commit 930a1fb0a8fa9432025442c4f4732058bb7af592 Author: zhunaipan Date: Fri Mar 27 14:49:12 2020 +0800 initial version Signed-off-by: leonwanghui diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000..c931e8f0689 --- /dev/null +++ b/.clang-format @@ -0,0 +1,152 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: +# - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 2 +UseTab: Never +SortIncludes: false +... + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..b5d31931012 --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# MindSpore +build/ +mindspore/lib +output +*.ir + +# Cmake files +CMakeFiles/ +cmake_install.cmake +CMakeCache.txt +Makefile +cmake-build-debug + +# Dynamic libraries +*.so +*.so.* +*.dylib + +# Static libraries +*.la +*.lai +*.a +*.lib + +# Protocol buffers +*_pb2.py +*.pb.h +*.pb.cc + +# Object files +*.o + +# Editor +.vscode +.idea/ + +# Cquery +.cquery_cached_index/ +compile_commands.json + +# Ctags and cscope +tags +TAGS +CTAGS +GTAGS +GRTAGS +GSYMS +GPATH +cscope.* + +# Python files +*__pycache__* +.pytest_cache + +# Mac files +*.DS_Store + +# Test results +test_temp_summary_event_file/ +*.dot +*.dat +*.svg +*.perf +*.info +*.ckpt +*.shp +*.pkl +.clangd +mindspore/version.py +mindspore/default_config.py +mindspore/.commit_id +onnx.proto +mindspore/ccsrc/onnx.proto diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..a241b6d69b9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,15 @@ +[submodule "third_party/flatbuffers"] + path = third_party/flatbuffers + url = https://github.com/google/flatbuffers.git +[submodule "third_party/googletest"] + path = third_party/googletest + url = https://github.com/google/googletest.git +[submodule "third_party/incubator-tvm"] + path = third_party/incubator-tvm + url = https://github.com/apache/incubator-tvm.git +[submodule "third_party/protobuf"] + path = third_party/protobuf + url = https://github.com/protocolbuffers/protobuf.git +[submodule "graphengine"] + path = graphengine + url = https://gitee.com/mindspore/graphengine.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..9cb73935ee9 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,55 @@ +cmake_minimum_required(VERSION 3.14) +project (MindSpore) + +include(${CMAKE_SOURCE_DIR}/cmake/options.cmake) + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") + +set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") +set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC") +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(PYBIND11_CPP_STANDARD -std=c++17) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTION_CXX_FLAGS}") + +find_package(Threads) +include(${CMAKE_SOURCE_DIR}/cmake/mind_expression.cmake) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include/flatbuffers) + +include(${CMAKE_SOURCE_DIR}/cmake/dependency_utils.cmake) +find_package(Python3 COMPONENTS Interpreter Development) +if(Python3_FOUND) + set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}") + set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") +else() + find_python_package(py_inc py_lib) + set(PYTHON_INCLUDE_DIRS "${py_inc}") + set(PYTHON_LIBRARIES "${py_lib}") +endif() +message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}") +message("PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") +include_directories(${PYTHON_INCLUDE_DIRS}) + +set(MS_CCSRC_PATH ${CMAKE_SOURCE_DIR}/mindspore/ccsrc) +set(MS_CCSRC_BUILD_PATH ${BUILD_PATH}/mindspore/mindspore/ccsrc) + +if (ENABLE_GE) + link_directories(${CMAKE_SOURCE_DIR}/third_party/ge/lib) +else() + include(${CMAKE_SOURCE_DIR}/cmake/dependency_graphengine.cmake) +endif() + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/inc) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/inc/external) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/inc/framework) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/third_party/fwkacllib/inc) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/third_party/fwkacllib/inc/toolchain) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") +add_subdirectory(mindspore/ccsrc) +if (ENABLE_TESTCASES) + add_subdirectory(tests) +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..591beaef820 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,115 @@ +# MindSpore contributing guidelines + + + +- [MindSpore contributing guidelines](#mindspore-contributing-guidelines) + - [Contributor License Agreement](#contributor-license-agreement) + - [Getting Started](#getting-started) + - [Contribution workflow](#contribution-workflow) + - [Code style](#code-style) + - [Fork-Pull development model](#fork-pull-development-model) + - [Report issues](#report-issues) + - [Propose PRs](#propose-prs) + + + +## Contributor License Agreement + +It's required to sign CLA before your first code submission to MindSpore community. + +For individual contributor, please refer to [ICLA online document](https://www.mindspore.cn/icla) for the detailed information. + +## Getting Started + +- Fork the repository on [Github](https://github.com/mindspore-ai/mindspore) or [Gitee](https://gitee.com/mindspore/mindspore). +- Read the [README.md](README.md) and [install page](https://www.mindspore.cn/install/en) for project information and build instructions. + +## Contribution Workflow + +### Code style + +Please follow this style to make MindSpore easy to review, maintain and develop. + +* Coding guidelines + + The *Python* coding style suggested by [Python PEP 8 Coding Style](https://pep8.org/) and *C++* coding style suggested by [Google C++ Coding Guidelines](http://google.github.io/styleguide/cppguide.html) are used in MindSpore community. + +* Unittest guidelines + + The *Python* unittest style suggested by [pytest](http://www.pytest.org/en/latest/) and *C++* unittest style suggested by [Googletest Primer](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) are used in MindSpore community. + +### Fork-Pull development model + +* Fork MindSpore repository + + Before submitting code to MindSpore project, please make sure that this project have been forked to your own repository. It means that there will be parallel development between MindSpore repository and your own repository, so be careful to avoid the inconsistency between them. + +* Clone the remote repository + + If you want to download the code to the local machine, `git` is the best way: + ```shell + # For GitHub + git clone https://github.com/{insert_your_forked_repo}/mindspore.git + git remote add upstream https://github.com/mindspore-ai/mindspore.git + # For Gitee + git clone https://gitee.com/{insert_your_forked_repo}/mindspore.git + git remote add upstream https://gitee.com/mindspore/mindspore.git + ``` + +* Develop code locally + + To avoid inconsistency between multiple branches, checking out to a new branch is `SUGGESTED`: + ```shell + git checkout -b {new_branch_name} origin/master + ``` + + Then you can change the code arbitrarily. + +* Push the code to the remote repository + + After updating the code, you should push the update in the formal way: + ```shell + git add . + git status # Check the update status + git commit -m "Your commit title" + git commit -s --amend #Add the concrete description of your commit + git push origin {new_branch_name} + ``` + +* Pull a request to MindSpore repository + + In the last step, your need to pull a compare request between your new branch and MindSpore `master` branch. After finishing the pull request, the Jekins CI will be automatically set up for building test. + +### Report issues + +A great way to contribute to the project is to send a detailed report when you encounter an issue. We always appreciate a well-written, thorough bug report, and will thank you for it! + +When reporting issues, refer to this format: + +- What version of env (mindspore, os, python etc) are you using? +- Is this a BUG REPORT or FEATURE REQUEST? +- What happened? +- What you expected to happen? +- How to reproduce it?(as minimally and precisely as possible) +- Special notes for your reviewers? + +**Issues advisory:** + +- **If you find an unclosed issue, which is exactly what you are going to solve,** please put some comments on that issue to tell others you would be in charge of it. +- **If an issue is opened for a while,** it's recommended for contributors to precheck before working on solving that issue. +- **If you resolve an issue which is reported by yourself,** it's also required to let others know before closing that issue. + +### Propose PRs + +* Raise your idea as an *issue* on [GitHub](https://github.com/mindspore-ai/mindspore/issues) or [Gitee](https://gitee.com/mindspore/mindspore/issues) +* If it is a new feature that needs lots of design details, a design proposal should also be submitted. +* After reaching consensus in the issue discussions and design proposal reviews, complete the development on the forked repo and submit a PR. +* None of PRs is not permitted until it receives **2+ LGTM** from approvers. Please NOTICE that approver is NOT allowed to add *LGTM* on his own PR. +* After PR is sufficiently discussed, it will get merged, abondoned or rejected depending on the outcome of the discussion. + +**PRs advisory:** + +- Any irrelevant changes should be avoided. +- Make sure your commit history being ordered. +- Always keep your branch up with the master branch. +- For bug-fix PRs, make sure all related issues being linked. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000000..676462131c7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +MindSpore +Copyright 2019-2020 Huawei Technologies Co., Ltd diff --git a/README.md b/README.md new file mode 100644 index 00000000000..a367d48ded4 --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +![MindSpore Logo](docs/MindSpore-logo.png "MindSpore logo") +============================================================ + +- [What is MindSpore?](#what-is-MindSpore) + - [Automatic Differentiation](#automatic-differentiation) + - [Automatic Parallel](#automatic-parallel) +- [Installation](#installation) + - [Binaries](#binaries) + - [From Source](#from-source) +- [Quickstart](#quickstart) +- [Docs](#docs) +- [Community](#community) + - [Governance](#governance) + - [Communication](#communication) +- [Contributing](#contributing) +- [Release Notes](#release-notes) +- [License](#license) + +## What Is MindSpore + +MindSpore is a new open source deep learning training/inference framework that +could be used for mobile, edge and cloud scenarios. MindSpore is designed to +provide development experience with friendly design and efficient execution for +the data scientists and algorithmic engineers, native support for Ascend AI +processor, and software hardware co-optimization. At the meantime MindSpore as +a global AI open source community, aims to further advance the development and +enrichment of the AI software/hardware application ecosystem. + +MindSpore Architecture + +For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/0.1.0-alpha/architecture.html). + +### Automatic Differentiation + +There are currently three automatic differentiation techniques in mainstream deep learning frameworks: + +- **Conversion based on static compute graph**: Convert the network into a static data flow graph at compile time, then turn the chain rule into a data flow graph to implement automatic differentiation. +- **Conversion based on dynamic compute graph**: Record the operation trajectory of the network during forward execution in an operator overloaded manner, then apply the chain rule to the dynamically generated data flow graph to implement automatic differentiation. +- **Conversion based on source code**: This technology is evolving from the functional programming framework and performs automatic differential transformation on the intermediate expression (the expression form of the program during the compilation process) in the form of just-in-time compilation (JIT), supporting complex control flow scenarios, higher-order functions and closures. + +TensorFlow adopted static calculation diagrams in the early days, whereas PyTorch used dynamic calculation diagrams. Static maps can utilize static compilation technology to optimize network performance, however, building a network or debugging it is very complicated. The use of dynamic graphics is very convenient, but it is difficult to achieve extreme optimization in performance. + +But MindSpore finds another way, automatic differentiation based on source code conversion. On the one hand, it supports automatic differentiation of automatic control flow, so it is quite convenient to build models like PyTorch. On the other hand, MindSpore can perform static compilation optimization on neural networks to achieve great performance. + +Automatic Differentiation + +The implementation of MindSpore automatic differentiation can be understood as the symbolic differentiation of the program itself. Because MindSpore IR is a functional intermediate expression, it has an intuitive correspondence with the composite function in basic algebra. The derivation formula of the composite function composed of arbitrary basic functions can be derived. Each primitive operation in MindSpore IR can correspond to the basic functions in basic algebra, which can build more complex flow control. + +### Automatic Parallel + +The goal of MindSpore automatic parallel is to build a training method that combines data parallelism, model parallelism, and hybrid parallelism. It can automatically select a least cost model splitting strategy to achieve automatic distributed parallel training. + +Automatic Parallel + +At present, MindSpore uses a fine-grained parallel strategy of splitting operators, that is, each operator in the figure is splited into a cluster to complete parallel operations. The splitting strategy during this period may be very complicated, but as a developer advocating Pythonic, you don't need to care about the underlying implementation, as long as the top-level API compute is efficient. + +## Installation + +### Binaries + +MindSpore offers build options across multiple backends: + +| Hardware Platform | Operating System | Status | +| :---------------- | :--------------- | :----- | +| Ascend910 | Ubuntu-x86 | ✔️ | +| | EulerOS-x86 | ✔️ | +| | EulerOS-aarch64 | ✔️ | +| GPU CUDA 9.2 | Ubuntu-x86 | ✔️ | +| GPU CUDA 10.1 | Ubuntu-x86 | ✔️ | +| CPU | Ubuntu-x86 | ✔️ | + +For installation using pip, take `Ubuntu-x86` and `CPU` build version as an example: + +1. Download whl from [MindSpore website](https://www.mindspore.cn/), and install the package. + + ``` + pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.1.0-alpha/MindSpore/cpu/ubuntu-x86/mindspore-0.1.0-cp37-cp37m-linux_x86_64.whl + ``` + +2. Run the following command to verify the install. + + ``` + python -c 'import mindspore' + ``` + +### From Source + +[Install MindSpore](https://www.mindspore.cn/install/en). + +## Quickstart + +See the [Quick Start](https://www.mindspore.cn/tutorial/en/0.1.0-alpha/quick_start/quick_start.html) +to implement the image classification. + +## Docs + +More details about installation guide, tutorials and APIs, please see the +[User Documentation](https://gitee.com/mindspore/docs). + +## Community + +### Governance + +Check out how MindSpore Open Governance [works](https://gitee.com/mindspore/community/blob/master/governance.md). + +### Communication + +- [MindSpore Slack](https://join.slack.com/t/mindspore/shared_invite/enQtOTcwMTIxMDI3NjM0LTNkMWM2MzI5NjIyZWU5ZWQ5M2EwMTQ5MWNiYzMxOGM4OWFhZjI4M2E5OGI2YTg3ODU1ODE2Njg1MThiNWI3YmQ) - Communication platform for developers. +- IRC channel at `#mindspore` (only for meeting minutes logging purpose) +- Video Conferencing: meet.jit.si +- Mailing-list: https://mailweb.mindspore.cn/postorius/lists + +## Contributing + +Welcome contributions. See our [Contributor Wiki](CONTRIBUTING.md) for +more details. + +## Release Notes + +The release notes, see our [RELEASE](RELEASE.md). + +## License + +[Apache License 2.0](LICENSE) diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000000..76d86c8bffe --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,73 @@ +# Release 0.1.0-alpha + +## Main Features + +### Ascend 910 Training and Inference Framework +* Recommended OS: Ubuntu 16.04 (or later) or EulerOS 2.0 +* Python version: 3.7.5 +* Preset models + * ResNet-50: residual structure-based convolutional neural network (CNN) for image classification, which is widely used. + * AlexNet: classic CNN for image classification, achieving historical results in ImageNet LSVRC-2012. + * LeNet: classic CNN for image classification, which was proposed by Yann LeCun. + * VGG16: classic CNN for image classification, which was proposed by Oxford Visual Geometry Group. + * YoloV3: real-time object detection network. + * NEZHA: BERT-based Chinese pre-training network produced by Huawei Noah's Ark Laboratory. +* Execution modes + * Graph mode: provides graph optimization methods such as memory overcommitment, IR fusion, and buffer fusion to achieve optimal execution performance. + * PyNative mode: single-step execution mode, facilitating process debugging. +* Debugging capability and methods + * Save CheckPoints and Summary data during training. + * Support asynchronous printing. + * Dump the computing data. + * Support profiling analysis of the execution process performance. +* Distributed execution + * Support AllReduce, AllGather, and BroadCast collective communication. + * AllReduce data parallel: Each device obtains different training data, which accelerates the overall training process. + * Collective communication-based layerwise parallel: Models are divided and allocated to different devices to solve the problem of insufficient memory for large model processing and improve the training speed. + * Automatic parallel mode: The better data and model parallel mode can be predicted based on the cost model. It is recommended that this mode be used on ResNet series networks. +* Automatic differentiation + * Implement automatic differentiation based on Source to Source. + * Support distributed scenarios and automatic insertion of reverse communication operators. +* Data processing, augmentation, and save format + * Load common datasets such as ImageNet, MNIST, CIFAR-10, and CIFAR-100. + * Support common data loading pipeline operations, such as shuffle, repeat, batch, map, and sampler. + * Provide basic operator libraries to cover common CV scenarios. + * Support users to customize Python data augmentation operators through the Pyfunc mechanism. + * Support the access of user-defined datasets through the GeneratorDataset mechanism. + * Provide the MindSpore data format, data aggregation and storage, random access example, data partition, efficient parallel read, user-defined index, and dataset search. + * Convert user datasets to the MindSpore data format. + * After data processing and augmentation, provide training applications in feed and graph modes. +* FP32/16 mixed precision computation, supporting automatic and manual configuration +* Provide common operators such as nn, math, and array, which can be customized. + +### Inference Deployment +* Deploy models in MindSpore format on the Ascend 310 platform for inference. +* Save models in ONNX format. +* Support saving models in LITE format and running models based on the lightweight inference framework. + * Recommended OS: Android 4.3 or later + * Supported network type: LeNet + * Provide the generalization operators generated by TVM and operators generated after specific networks are tuned. + +### Other Hardware Support +* GPU platform training + * Recommended OS: Ubuntu 16.04 + * CUDA version: 9.2 or 10.1 + * CuDNN version: 7.6 or later + * Python version: 3.7.5 + * NCCL version: 2.4.8-1 + * OpenMPI version: 3.1.5 + * Supported models: AlexNet, LeNet, and LSTM + * Supported datasets: MNIST and CIFAR-10 + * Support data parallel. +* CPU platform training + * Recommended OS: Ubuntu 16.04 + * Python version: 3.7.5 + * Supported model: LeNet + * Supported dataset: MNIST + * Provide only the stand-alone operation version. + +## Peripherals and Tools +* [MindSpore Official Website] (https://www.mindspore.cn/) +* [MindInsight Visualization Debugging and Optimization] (https://gitee.com/mindspore/mindinsight) +* [MindArmour Model Security Hardening Package] (https://gitee.com/mindspore/mindarmour) +* [GraphEngine Computational Graph Engine] (https://gitee.com/mindspore/graphengine) \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..dc390c1bd87 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security Risk Description + +1. When MindSpore is used for AI model training, if the user-defined computational graph structure (for example, Python code for generating the MindSpore computational graph) is provided by an untrusted third party, malicious code may exist and will be loaded and executed to attack the system. +2. Model files are stored in binary mode. When MindSpore is used to optimize or infer AI models and the model files are loaded in deserialization mode, once malicious code is written into the model files, the code are loaded and executed, causing attacks on the system. +3. MindSpore performs only model training and inference based on the data provided by users. Users need to protect data security to avoid privacy leakage. +4. MindSpore is a distributed training platform. When MindSpore is used for distributed training, if an Ascend chip is used for training, a device provides a secure transmission protocol for gradient fusion. If GPUs or other clusters are used for training, identity authentication and secure transmission are not provided. + +# Security Usage Suggestions + +1. Run MindSpore in the sandbox. +2. Run MindSpore as a non-root user. +3. Ensure that the source of a computational graph structure is trustworthy. Do not write code irrelevant to model training in the network structure definition. +4. Ensure that the source of a network model is trustworthy or enter secure network model parameters to prevent model parameters from being tampered with. +5. Ensure that GPU distributed training is performed on an isolated cluster network. diff --git a/Third_Party_Open_Source_Software_Notice b/Third_Party_Open_Source_Software_Notice new file mode 100644 index 00000000000..498b5b8d1b9 --- /dev/null +++ b/Third_Party_Open_Source_Software_Notice @@ -0,0 +1,3502 @@ +OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice along with this product and/or this product firmware (in the following just “this product”). The open source software licenses are granted by the respective right holders. And the open source licenses prevail all other license information with regard to the respective open source software contained in the product, including but not limited to End User Software Licensing Agreement. This notice is provided on behalf of Huawei Technologies Co. Ltd. and any of its local subsidiaries which may have provided this product to you in your local country. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts + +Software: Eigen 3.3.7 +Copyright notice: +Copyright (C) 2014 Benoit Steiner +Copyright (C) 2013 Christian Seiler +Copyright (C) 2015 Eugene Brevdo +Copyright (C) 2014-2015 Benoit Steiner +Copyright (C) 2015 Navdeep Jaitly +Copyright (C) 2014 Eric Martin +Copyright (C) 2015 Benoit Steiner +Copyright (C) 2016 Rasmus Munk Larsen +Copyright (C) 2016 Benoit Steiner +Copyright (C) 2015 Jianwei Cui +Copyright (C) 2016 Eugene Brevdo +Copyright (C) 2015 Ke Yang +Copyright (C) 2016 Mehdi Goli, Codeplay Software Ltd +Copyright (C) 2014 Navdeep Jaitly +Copyright (C) 2016 Igor Babuschkin +Copyright (C) 2016 Dmitry Vyukov +Copyright (C) EDF R&D, lun sep 30 14:23:30 CEST 2002 +Copyright (C) 2008 Gael Guennebaud +Copyright (C) EDF R&D, lun sep 30 14:23:31 CEST 2002 +Copyright (C) 2008-2010 Gael Guennebaud +Copyright (C) 2008-2016 Gael Guennebaud +Copyright (C) 2009 Mark Borgerding mark a borgerding net +Copyright (C) 2008-2009 Gael Guennebaud +Copyright (C) 2013 Desire Nuentsa +Copyright (C) 2013 Gael Guennebaud +Copyright (C) 2011 Gael Guennebaud +Copyright (C) 2012 Desire NUENTSA WAKAM +Copyright (C) 2009 Benoit Jacob +Copyright (C) 2009 Gael Guennebaud +Copyright (C) 2006-2010 Benoit Jacob +Copyright (C) 2006-2008 Benoit Jacob +Copyright (C) EDF R&D, lun sep 30 14:23:28 CEST 2002 +Copyright (C) 2010 Manuel Yguel +Copyright (C) 2009 Claire Maurice +Copyright (C) 2010,2012 Jitse Niesen +Copyright (c) 2011, Intel Corporation. All rights reserved. +Copyright (C) 2012-2016 Gael Guennebaud +Copyright (C) 2016 Tobias Wood +Copyright (C) 2010 Jitse Niesen +Copyright (C) 2012 Alexey Korepanov +Copyright (C) 2010 Vincent Lejeune +Copyright (C) 2010 Gael Guennebaud +Copyright (C) 2010 Benoit Jacob +Copyright (C) 2017 Gael Guennebaud +Copyright (C) 2009-2010 Gael Guennebaud +Copyright (C) 2008 Benoit Jacob +Copyright (C) 2009 Mathieu Gautier +Copyright (C) 2010 Hauke Heibel +Copyright (C) 2009 Hauke Heibel +Copyright (C) 2008-2015 Gael Guennebaud +Copyright (C) EDF R&D, mar déc 3 18:59:36 CET 2002 +Copyright (C) EDF R&D, lun sep 30 14:23:17 CEST 2002 +Copyright (C) EDF R&D, mar déc 3 18:59:35 CET 2002 +Copyright (C) 2016 Konstantinos Margaritis +Copyright (C) 2007 Julien Pommier +Copyright (C) 2008-2011 Gael Guennebaud +Copyright (C) 2009 Keir Mierle +Copyright (C) 2011 Timothy E. Holy +Copyright (C) 2009 Hauke Heibel +Copyright (C) 2012 Desire Nuentsa +Copyright (C) 2014 Gael Guennebaud +Copyright (C) 2015 Tal Hadad +@copyright (c) 2009-2014 The University of Tennessee and The University of Tennessee Research Foundation. +@copyright (c) 2012-2016 Inria. All rights reserved. +@copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved. +Copyright 2007-2009 Kitware, Inc. +Copyright 2012-2013 Inria +Copyright 2012-2013 Emmanuel Agullo +Copyright 2012-2013 Mathieu Faverge +Copyright 2012 Cedric Castagnede +Copyright 2013-2016 Florent Pruvost +Copyright 2016 Codeplay Software Ltd. +Copyright (c) 2006, 2007 Montel Laurent, +Copyright (c) 2008, 2009 Gael Guennebaud, +Copyright (c) 2009 Boudewijn Rempt +@copyright (c) 2012-2014 Inria. All rights reserved. +Copyright 2013 Florent Pruvost +Copyright (c) 2010 Jitse Niesen, +Copyright (C) 2009 Benjamin Schindler +Copyright (C) 2016 Pedro Gonnet (pedro.gonnet@gmail.com) +Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com) +Copyright (C) 2009 Thomas Capricelli +Copyright (C) 2012-2013 Desire Nuentsa +Copyright (C) 2012-2014 Gael Guennebaud +Copyright Jorge More - Argonne National Laboratory +Copyright Burt Garbow - Argonne National Laboratory +Copyright Ken Hillstrom - Argonne National Laboratory +Copyright (C) 2009 Ilya Baran +Copyright (c) 2010, Intel Corp. +Copyright (C) 2009-2010 Benoit Jacob +Copyright (C) 2013-2016 Gael Guennebaud +Copyright (C) 2013 Gauthier Brun +Copyright (C) 2013 Nicolas Carre +Copyright (C) 2013 Jean Ceccato +Copyright (C) 2013 Pierre Zoppitelli +Copyright (C) 2013 Jitse Niesen +Copyright (C) 2014-2017 Gael Guennebaud +Copyright (C) 2013-2014 Gael Guennebaud +Copyright (C) 2011-2014 Gael Guennebaud +Copyright (C) 2012 Désiré Nuentsa-Wakam +Copyright (C) 2015 Gael Guennebaud +Copyright (C) 2012 Gael Guennebaud +Copyright (c) 1994 by Xerox Corporation. All rights reserved. +Copyright (C) 2001 Intel Corporation +Copyright (c) 2001 Intel Corporation. +Copyright (C) 2009 Gael Guennebaud +Copyright (C) 2013 Christoph Hertzberg +Copyright (C) 2015 Eugene Brevdo +Copyright (C) 2016 +Mehdi Goli Codeplay Software Ltd. +Ralph Potter Codeplay Software Ltd. +Luke Iwanski Codeplay Software Ltd. +Copyright (C) 2014 Jianwei Cui +Copyright (C) 2015 Vijay Vasudevan +Copyright (C) 2015 +Mehdi Goli Codeplay Software Ltd. +Ralph Potter Codeplay Software Ltd. +Luke Iwanski Codeplay Software Ltd. +Copyright (C) 2014 Navdeep Jaitly +Copyright (C) 2011 Gael Guennebaud +Copyright (C) 2012 desire Nuentsa +Copyright (C) 2012 Kolja Brix +Copyright (C) 2011 Kolja Brix +Copyright (C) 2011 Andreas Platen +Copyright (C) 2012 Chen-Pang He +Copyright (C) 2009 Jitse Niesen +Copyright (C) 2009-2011 Jitse Niesen +Copyright (C) 2012, 2013 Chen-Pang He +Copyright (C) 2011 Jitse Niesen +Copyright (C) 2012 Giacomo Po +Copyright (C) 2008-2010 Gael Guennebaud +Copyright (C) 2016 Gael Guennebaud +Copyright (C) 2010-2011 Hauke Heibel +Copyright (C) 2012 David Harmon +Copyright (C) 2007-2009 Benoit Jacob +Copyright (C) 2007-2010 Benoit Jacob +Copyright (C) 2008-2009 Benoit Jacob +Copyright (C) 2009 Kenneth Riddile +Copyright (C) 2010 Thomas Capricelli +Copyright (C) 2013 Pavel Holoborodko +Copyright (C) EDF R&D, lun sep 30 14:23:16 CEST 2002 +Copyright (C) EDF R&D, mar déc 3 18:59:37 CET 2002 +Copyright (C) 2006-2009 Benoit Jacob +Copyright (C) 2008-2010 Benoit Jacob +Copyright (c) 2008-2015 Pavel Holoborodko +Copyright (C) 20010-2011 Hauke Heibel +Copyright (c) 2006, Montel Laurent, +Copyright (c) 2007, Allen Winter, +Copyright (c) 2007, Alexander Neundorf, +Copyright (C) 2008 Guillaume Saupin +Copyright (C) 2008-2009 Guillaume Saupin +Copyright (C) 2009 Guillaume Saupin +Copyright (C) 2010-2016 Konstantinos Margaritis +Copyright (C) 2008-2016 Konstantinos Margaritis +Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com) +Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) +Copyright (c) Fabian Giesen, 2016 +Copyright (C) 2010 Konstantinos Margaritis +Copyright (C) 2007 Michael Olbrich +Copyright (C) 2011 Benoit Jacob +Copyright (C) 2011-2012 Jitse Niesen +Copyright (C) 2016 Rasmus Munk Larsen (rmlarsen@google.com) +Copyright (C) 2008-2014 Gael Guennebaud +Copyright (C) 2010-2013 Hauke Heibel +Copyright (C) 2006-2008, 2010 Benoit Jacob +Copyright (C) 2010-2016 Gael Guennebaud +Copyright (C) 2009-2015 Gael Guennebaud +Copyright (C) 2009 Ricard Marxer +Copyright (C) 2009-2014 Gael Guennebaud +Copyright (C) 2010-2011 Gael Guennebaud +Copyright (C) 2009 Rohit Garg +Copyright (c) 2006, Timothy A. Davis. +Copyright (c) 1998-2003 by the University of Florida. +Copyright (C) 2012 Désiré Nuentsa-Wakam +Copyright (C) 2008-2012 Gael Guennebaud +LDL Copyright (c) 2005 by Timothy A. Davis. All Rights Reserved. +Copyright (C) 2010 Daniel Lowengrub +Copyright (C) EDF R&D, lun sep 30 14:23:20 CEST 2002 +Copyright (C) EDF R&D, lun sep 30 14:23:19 CEST 2002 +Copyright (C) 2009, 2010, 2013 Jitse Niesen +Copyright (C) 2011, 2013 Chen-Pang He +Copyright (C) 2009-2011, 2013 Jitse Niesen +Copyright (C) 2011, 2013 Jitse Niesen +Copyright (C) 2011 Chen-Pang He +Copyright (C) 2010, 2013 Jitse Niesen +Copyright (C) 2010-2014 Gael Guennebaud +Copyright (C) 2012 The Android Open Source Project +(C) Desire NUENTSA WAKAM, INRIA +Copyright (C) EDF R&D, lun sep 30 14:23:18 CEST 2002 +Copyright (C) 2012 Keir Mierle +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +Copyright (C) EDF R&D, lun sep 30 14:23:23 CEST 2002 +Copyright (C) EDF R&D, lun sep 30 14:23:24 CEST 2002 +Copyright (C) EDF R&D, lun sep 30 14:23:27 CEST 2002 +Copyright (C) 2007 Free Software Foundation, Inc. +Copyright (C) 1991, 1999 Free Software Foundation, Inc. +Copyright (C) 2015 Benoit Jacob +Geometric Tools, LLC Copyright (c) 1998-2010 +Copyright (C) EDF R&D, lun sep 30 14:23:15 CEST 2002 +Copyright (C) 2002-2007 Yves Renard +Copyright (C) 2012, 2014 Kolja Brix +Copyright (C) 1997-2001 Andrew Lumsdaine Lie-Quan Lee +Copyright (C) 2012 Desire NUENTSA WAKAM +Copyright (C) 2013 Hauke Heibel +Copyright (C) 2010-2011 Jitse Niesen +Intel Copyright (C) .... +Copyright (C) 2010-2017 Gael Guennebaud +Copyright (C) 20013 Gael Guennebaud +Copyright (C) 2008 Daniel Gomez Ferro +Copyright (C) 2013 Désiré Nuentsa-Wakam +Copyright (C) 2011-2015 Gael Guennebaud +Copyright (C) 20015 Gael Guennebaud +Copyright (C) 2014-2015 Gael Guennebaud + + +License: Mozilla Public License (MPL) V2.0 + +Mozilla Public License +Version 2.0 +1. Definitions +1.1. “Contributor” +means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. +1.2. “Contributor Version” +means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. +1.3. “Contribution” +means Covered Software of a particular Contributor. +1.4. “Covered Software” +means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. +1.5. “Incompatible With Secondary Licenses” +means +that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or +that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. +1.6. “Executable Form” +means any form of the work other than Source Code Form. +1.7. “Larger Work” +means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. +1.8. “License” +means this document. +1.9. “Licensable” +means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. +1.10. “Modifications” +means any of the following: +any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or +any new file in Source Code Form that contains any Covered Software. +1.11. “Patent Claims” of a Contributor +means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. +1.12. “Secondary License” +means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. +1.13. “Source Code Form” +means the form of the work preferred for making modifications. +1.14. “You” (or “Your”) +means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. +2. License Grants and Conditions +2.1. Grants +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: +under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and +under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. +2.2. Effective Date +The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. +2.3. Limitations on Grant Scope +The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: +for any code that a Contributor has removed from Covered Software; or +for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or +under Patent Claims infringed by Covered Software in the absence of its Contributions. +This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). +2.4. Subsequent Licenses +No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). +2.5. Representation +Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. +2.6. Fair Use +This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. +2.7. Conditions +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. +3. Responsibilities +3.1. Distribution of Source Form +All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. +3.2. Distribution of Executable Form +If You distribute Covered Software in Executable Form then: +such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and +You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. +3.3. Distribution of a Larger Work +You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). +3.4. Notices +You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. +3.5. Application of Additional Terms +You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. +4. Inability to Comply Due to Statute or Regulation +If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. +5. Termination +5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. +5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. +6. Disclaimer of Warranty +Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. +7. Limitation of Liability +Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. +8. Litigation +Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. +9. Miscellaneous +This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. +10. Versions of the License +10.1. New Versions +Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. +10.2. Effect of New Versions +You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. +10.3. Modified Versions +If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses +If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. +Exhibit A - Source Code Form License Notice +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. +If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. +You may add additional accurate notices of copyright ownership. +Exhibit B - “Incompatible With Secondary Licenses” Notice +This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. + + +Software: JSON for Modern C++ 3.6.1 +Copyright notice: +Copyright 2015 Google Inc. All rights reserved. +Copyright 2018 Google Inc. All rights reserved. +Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +Copyright 2017 Roman Lebedev. All rights reserved. +Copyright (c) 2012 Two Blue Cubes Ltd. All rights reserved. +Copyright (c) 2015 Max Woolf +Copyright 2014 The Authors +Copyright (c) 2016 Nicolas Seriot +Copyright (c) 2015-2017 Niels Lohmann. +Copyright (c) 2015-2017 Niels Lohmann +Copyright (c) 2013-2019 Niels Lohmann . +Copyright (c) 2018 Vitaliy Manushkin . +Copyright (c) 2012, Erik Edlund +Copyright (c) 2013-2019 Niels Lohmann +Copyright 2013-2019 [Niels Lohmann](http:nlohmann.me) +Copyright (c) 2009 Google Inc. All rights reserved. +Copyright (C) 2009 Google Inc. + +License: MIT License +The MIT License +Copyright (c) +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +Software: MKL-DNN 1.1.2 +Copyright (c) 2009-2018 The MathJax Consortium +Copyright 2018 Intel Corporation +Copyright 2019 Intel Corporation +Copyright 2017-2018 Intel Corporation +Copyright (c) 2011-2015 The MathJax Consortium +Copyright (c) 2015-2017 Martin Hensel +Copyright 2018-2019 Intel Corporation +Copyright 2016-2018 Intel Corporation +Copyright 2016-2019 Intel Corporation +Copyright 2018 YANDEX LLC +Copyright 2017-2019 Intel Corporation +Copyright 2008, Google Inc. +Copyright 2005, Google Inc. +Copyright 2007, Google Inc. +Copyright 2008 Google Inc. +Copyright 2006, Google Inc. +Copyright 2015, Google Inc. +Copyright (c) 2010-2018 The MathJax Consortium +Copyright (c) 2007 MITSUNARI Shigeo +Copyright (c) 2017-2018 Intel Corporation +Copyright (c) 2011, Intel Corporation +Copyright 2000-2019 Kitware, Inc. and Contributors +Copyright (c) 2007, Apostolos Syropoulos ( +Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved. +Copyright (c) 2014-2017 Intel, Inc. All rights reserved. +Copyright © 2004-2005 The Trustees of Indiana University and Indiana +Copyright © 2004-2005 The University of Tennessee and The University +Copyright © 2004-2005 High Performance Computing Center Stuttgart, +Copyright © 2004-2005 The Regents of the University of California. +Copyright © 2010-2014 Inria. All rights reserved. +Copyright © 2009-2014 Cisco Systems, Inc. All rights reserved. +Copyright © 2004-2006 The Trustees of Indiana University and Indiana +Copyright © 2008-2014 Cisco Systems, Inc. All rights reserved. +Copyright © 2014 Inria. All rights reserved. +Copyright (C) 2009. QLogic Corporation. All rights reserved. +Copyright (c) 2004-2010 The University of Tennessee and The University +Copyright (c) 2006 QLogic Corporation. All rights reserved. +Copyright (c) 2013-2014 Intel, Inc. All rights reserved +Copyright (c) 2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. +Copyright (c) 2012-2015 Los Alamos National Security, LLC. +Copyright (c) 2014 Intel Corporation. All rights reserved. +Copyright (c) 2007-2011 University of Houston. All rights reserved. +Copyright (c) 2011-2013 Inria. All rights reserved. +Copyright (c) 2011-2013 Universite Bordeaux 1 +Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2012-2016 Los Alamos National Security, LLC. +Copyright (c) 2015 Mellanox Technologies. All rights reserved. +Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2006-2017 University of Houston. All rights reserved. +Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights +Copyright (c) 2007 Voltaire All rights reserved. +Copyright (c) 2006-2010 University of Houston. All rights reserved. +Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2013-2016 Intel, Inc. All rights reserved. +Copyright (c) 2006-2017 University of Houston. All rights reserved. +Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2004-2016 The University of Tennessee and The University +Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. +Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2011-2013 Los Alamos National Security, LLC. +Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +Copyright (c) 2012-2015 Los Alamos National Security, Inc. All rights +Copyright (c) 2004-2010 High Performance Computing Center Stuttgart, +Copyright (c) 2006-2007 University of Houston. All rights reserved. +Copyright (c) 2006-2010 University of Houston. All rights reserved. +Copyright (c) 2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2008 University of Houston. All rights reserved. +Copyright (c) 2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2004-2005 The Trustees of the University of Tennessee. +Copyright (c) 2004-2007 The Trustees of Indiana University. +Copyright (c) 2007 The Trustees of Indiana University. +Copyright (c) 2011-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2017 Los Alamos National Security, LLC. All +Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2013 Mellanox Technologies, Inc. +Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2017 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2010-2011 Alex Brick . +Copyright (c) 2014-2019 Intel, Inc. All rights reserved. +Copyright (c) 2015-2018 Cisco Systems, Inc. All rights reserved +Copyright (c) 2006-2013 Los Alamos National Security, LLC. +Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2014 NVIDIA Corporation. All rights reserved. +Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. +Copyright (c) 2017 Research Organization for Information Science +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. +Copyright (c) 2015-2017 Intel, Inc. All rights reserved. +Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2010-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2019 Mellanox Technologies, Inc. +Copyright (c) 2019 Intel, Inc. All rights reserved. +Copyright (c) 2011-2015 Los Alamos National Security, LLC. All +Copyright (c) 2016 Intel, Inc. All rights reserved. +Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2010-2017 IBM Corporation. All rights reserved. +Copyright (c) 2016 Intel, Inc. All rights reserved. +Copyright (c) 2013 NVIDIA Corporation. All rights reserved. +Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved. +Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2008-2015 University of Houston. All rights reserved. +Copyright (c) 2008-2016 University of Houston. All rights reserved. +Copyright (c) 2008-2011 University of Houston. All rights reserved. +Copyright (c) 2017 University of Houston. All rights reserved. +Copyright (c) 2008-2017 University of Houston. All rights reserved. +Copyright 2009 Cisco Systems, Inc. All rights reserved. +Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED. +Copyright (c) 2016 The University of Tennessee and The University +Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED. +Copyright (c) 2015 Mellanox Technologies, Inc. +Copyright (c) 2004-2005 The Trustees of Indiana University. +Copyright (c) 2004-2011 The Trustees of the University of Tennessee. +Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +Copyright (c) 2011-2015 Los Alamos National Security, LLC. +Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +Copyright (c) 2014-2015 Mellanox Technologies, Inc. +Copyright (c) 2016-2017 Research Organization for Information Science +Copyright (c) 2014-2018 Research Organization for Information Science +Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +Copyright (c) 2014-2017 Mellanox Technologies, Inc. +Copyright (c) 2010-2017 IBM Corporation. All rights reserved. +Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2018-2019 Intel, Inc. All rights reserved. +Copyright (c) 2004-2015 The University of Tennessee and The University +Copyright (c) 2007-2008 UT-Battelle, LLC +Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. +Copyright (c) 2016-2017 IBM Corporation. All rights reserved. +Copyright (c) 2004-2014 The University of Tennessee and The University +Copyright (c) 2006-2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2012 Los Alamos National Security, LLC. All rights +Copyright (c) 2006 University of Houston. All rights reserved. +Copyright (C) 2003 University of Chicago, Ohio Supercomputer Center. +(C) 2008 by Argonne National Laboratory. +See COPYRIGHT in top-level directory. +Copyright (c) 2004-2018 The University of Tennessee and The University +Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2006-2008 University of Houston. All rights reserved. +Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved +Copyright (c) 2011 Sandia National Laboratories. All rights reserved. +Copyright (c) 2015 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved +Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2012-2015 NVIDIA Corporation. All rights reserved. +Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2011-2012 Sandia National Laboratories. All rights reserved. +Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2012 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2012 NVIDIA Corporation. All rights reserved. +Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2011-2012 NVIDIA Corporation. All rights reserved. +Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2007 IBM Corp., All rights reserved. +Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights +Copyright (c) 2012 Los Alamos National Security, LLC +Copyright (c) 2013-2015 Mellanox Technologies, Inc. +Copyright (c) 2014-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2012 Sandia National Laboratories. All rights reserved. +Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2014-2017 The University of Tennessee and The University +Copyright (c) 2011 Sandia National Laboratories. All rights reserved. +Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2015-2018 Research Organization for Information Science +Copyright (c) 2017 The University of Tennessee and The University +Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +Copyright (C) 2012 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006 Voltaire. All rights reserved. +Copyright (c) 2007 Mellanox Technologies. All rights reserved. +Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011 The University of Tennessee and The University +Copyright (c) 2017-2019 Intel, Inc. All rights reserved. +Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2010-2015 Los Alamos National Security, LLC. +Copyright (c) 2016-2019 Research Organization for Information Science +Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2006 Los Alamos National Security, LLC. All rights +Copyright (c) 2010-2014 Los Alamos National Security, LLC. +Copyright (c) 2014 Hochschule Esslingen. All rights reserved. +Copyright (c) 2015-2018 Mellanox Technologies, Inc. +Copyright (c) 2016-2019 Intel, Inc. All rights reserved. +Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2004-2009 The University of Tennessee and The University +Copyright (c) 2011-2012 Los Alamos National Security, LLC. +Copyright (c) 2013 Intel, Inc. All rights reserved +Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2009-2011 The Trustees of Indiana University. +Copyright (c) 2010-2017 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2011 Oracle and/or all its affiliates. All rights reserved. +Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +Copyright (c) 2016 Mellanox Technologies, Inc. +Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. +Copyright (c) 2017 Amazon.com, Inc. or its affiliates. +Copyright (c) 2014-2019 Research Organization for Information Science +Copyright (c) 2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2018 The University of Tennessee and The University +Copyright (c) 2006-2019 Cisco Systems, Inc. All rights reserved +Copyright (c) 2008 Los Alamos National Security, LLC. All rights reserved. +Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. +Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2011 NVIDIA Corporation. All rights reserved. +Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2004-2012 The University of Tennessee and The University +Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP. +Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved +Copyright (c) 2015 Intel, Inc. All rights reserved +Copyright (c) 2015 Intel, Inc. All rights reserved. +(c) Class instantiation: static +Copyright (c) 2011 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2012 The University of Tennessee and The University +Copyright (c) 2011-2018 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2007-2012 Niels Provos and Nick Mathewson +Copyright (c) 2000-2007 Niels Provos +Copyright (c) 2007-2012 Niels Provos, Nick Mathewson +Copyright (c) 2009-2012 Niels Provos and Nick Mathewson +Copyright (c) 2006-2007 Niels Provos +Copyright (c) 2008-2012 Niels Provos and Nick Mathewson +Copyright (c) 2011-2012 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2010-2018 Cisco Systems, Inc. All rights reserved +Copyright (c) 2012 Sandia National Laboratories. All rights reserved. +Copyright (c) 2011-2013 The University of Tennessee and The University +Copyright (c) 2011-2013 Université Bordeaux 1 +Copyright (c) 2013-2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2012 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2012 University of Oregon. All rights reserved. +Copyright (c) 2012 Inria. All rights reserved. +Copyright (c) 2004-2014 High Performance Computing Center Stuttgart, +Copyright (c) 2019 Research Organization for Information Science +Copyright (c) 2007-2012 Los Alamos National Security, LLC. +Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana +Copyright (c) 2013 Intel, Inc. All rights reserved. +Copyright (c) 2011-2013 INRIA. All rights reserved. +Copyright (c) 2015-2019 Intel, Inc. All rights reserved. +Copyright (c) 2018 Cisco Systems, Inc. All rights reserved +Copyright 2002 Niels Provos +Copyright (c) 2010 ARM ltd. All rights reserved. +Copyright (c) 2008 The University of Tennessee and The University +Copyright (c) 2009 Cisco Systems, Inc. All Rights Reserved. +Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006 The Trustees of Indiana University and Indiana +Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights +Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2012-213 Los Alamos National Security, LLC. All rights +Copyright (c) 2014-2018 Cisco Systems, Inc. All rights reserved +Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights +Copyright (c) 2011 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2007-2011 Los Alamos National Security, LLC. +Copyright (c) 2007-2015 Los Alamos National Security, LLC. +Copyright (c) 2015-2016 Intel, Inc. All rights reserved. +Copyright (c) 2007-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2008-2013 University of Houston. All rights reserved. +Copyright (c) 2008-2014 University of Houston. All rights reserved. +Copyright (c) 2017-2018 Intel, Inc. All rights reserved. +Copyright (c) 2014-2016 Mellanox Technologies, Inc. +Copyright (c) 2014 Intel, Inc. All rights reserved +Copyright (c) 2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved. +Copyright (c) 2009-2012 Oak Ridge National Laboratory. All rights reserved. +Copyright (c) 2012-2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2014 The University of Tennessee and The University +Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2010-2011 Los Alamos National Security, LLC. +Copyright (c) 2010-2012 Los Alamos National Security, LLC. +Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2016 Broadcom Limited. All rights reserved. +Copyright (c) 2018 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2009-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006 Voltaire All rights reserved. +"Copyright (c) 2000-2003 The Regents of the University of California. +Copyright (c) 2004-2006 The Trustees of the University of Tennessee. +Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. +Copyright (c) 2016-2018 Intel, Inc. All rights reserved. +Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2017-2018 Intel, Inc. All rights reserved. +Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2012-2015 Sandia National Laboratories. All rights reserved. +Copyright (c) 2015 NVIDIA Corporation. All rights reserved. +Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2018 Intel, Inc. All rights reserved. +Copyright (c) 2011-2012 Sandia National Laboratories. All rights reserved. +Copyright (c) 2011 Mellanox Technologies. All rights reserved. +Copyright (c) 2011 Mellanox Technologies. All rights reserved. +Copyright (c) 2016-2017 Intel, Inc. All rights reserved. +Copyright (c) 2012-2013 Inria. All rights reserved. +Copyright (c) 2009-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2012-2017 Cisco Systems, Inc. All rights reserved +Copyright (C) 2003 University of Chicago. +(C) 2001 by Argonne National Laboratory. +(C) 2009 UChicago/Argonne LLC +(C) 2012 by Argonne National Laboratory. +(C) 2003 by Argonne National Laboratory. +Copyright (c) 2015-2018 Intel, Inc. All rights reserved. +Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved. +Copyright (c) 2015 Sandia National Laboratories. All rights reserved. +Copyright (c) 2013 Sandia National Laboratories. All rights reserved. +Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved. +Copyright (c) 2011-2018 Cisco Systems, Inc. All rights reserved +Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights +Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +Copyright (c) 2014 Artem Y. Polyakov . +Copyright (c) 2015-2018 Intel, Inc. All rights reserved. +Copyright (c) 2018 Intel, Inc. All rights reserved. +Copyright (c) 2015 NVIDIA, Inc. All rights reserved. +Copyright (c) 2007 Mellanox Technologies, Inc. All rights reserved. +Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2007-2008 Chelsio, Inc. All rights reserved. +Copyright (c) 2008 Mellanox Technologies. All rights reserved. +Copyright (c) 2009 Sandia National Laboratories. All rights reserved. +Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2011 Mellanox Technologies. All rights reserved. +Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved. +Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2014 Bull SAS. All rights reserved. +Copyright (c) 2016 Mellanox Technologies. All rights reserved. +Copyright (c) 2011 Los Alamos National Security, LLC. All +Copyright (c) 2007-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2004-2009 The Trustees of Indiana University. +Copyright (c) 2016-2019 IBM Corporation. All rights reserved. +Copyright (c) 2011 Los Alamos National Security, LLC. All rights +Copyright (c) 2011 IBM Corporation. All rights reserved. +Copyright (c) 2010 Oracle and/or its affiliates. All rights +Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2007-2012 Los Alamos National Security, LLC. All rights +Copyright (c) 2011-2017 IBM Corporation. All rights reserved. +Copyright (c) 2015-2016 Mellanox Technologies, Inc. +Copyright (c) 2004-2011 The Trustees of Indiana University. +Copyright (c) 2010-2013 The University of Tennessee and The University +Copyright (c) 2007 Evergrid, Inc. All rights reserved. +Copyright (c) 2008 Institut National de Recherche en Informatique +Copyright (c) 2016 Intel, Inc. All rights reserved +Copyright (C) 2004 University of Chicago. +Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2011-2012 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2008 Voltaire. All rights reserved +Copyright (c) 2015 Los Alamos National Security, LLC. +Copyright (c) 2014, Cisco Systems, Inc. All rights reserved. +Copyright (c) 2014-2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2015 Cisco Systems. All rights reserved. +Copyright (c) 2017 Amazon.com, Inc. or its affiliates. +Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights +Copyright (c) 2017-XXXX Amazon.com, Inc. or its affiliates. +Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +Copyright (c) 2008 UT-Battelle, LLC +Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2008-2010 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2004-2011 The Trustees of Indiana University and Indiana +Copyright (c) 2011-2014 NVIDIA Corporation. All rights reserved. +Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2007 High Performance Computing Center Stuttgart, +Copyright (c) 2007-2013 The University of Tennessee and The University of +Copyright (C) 2000-2004 by Etnus, LLC +Copyright (C) 1999 by Etnus, Inc. +Copyright (C) 1997-1998 Dolphin Interconnect Solutions Inc. +Copyright (C) 2000-2004 by Etnus, LLC. +Copyright (c) 2008 Cisco Systems, Inc. All rights reserved +Copyright (c) 2008-2009 Sun Microystems, Inc. All rights reserved +Copyright (c) 2017 IBM Corp. All rights reserved. +Copyright (c) 2009 Sun Microsystems, Inc All rights reserved. +Copyright (c) 2012-2013 The University of Tennessee and The University +Copyright (c) 2006 The Trustees of Indiana University and Indiana +Copyright (c) 2006 The Technical University of Chemnitz. All +Copyright (c) 2006 The Technical University of Chemnitz. All +Copyright (c) 2015 The University of Tennessee and The University +Copyright (c) 2008-2019 Cisco Systems, Inc. All rights reserved +Copyright (c) 2014-2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2008-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2008-2013 University of Houston. All rights reserved. +Copyright (c) 2013-2017 University of Houston. All rights reserved. +Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2012 IBM Corporation. All rights reserved. +Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2013-2014 Intel, Inc. All rights reserved. +Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2004-2007 The Trustees of the University of Tennessee. +(C)Copyright IBM Corp. 2007, 2008 +Copyright (C) 1997-2001 University of Chicago. +Copyright (c) 2006-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2009 Oak Ridge National Laboratory +Copyright (c) 2013-2015 NVIDIA Corporation. All rights reserved. +Copyright (c) 2008-2010 Oracle and/or its affiliates. All rights reserved +Copyright (c) 2015-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2015-2016 The University of Tennessee and The University +Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2013-2015 Intel, Inc. All rights reserved. +Copyright (c) 2006 High Performance Computing Center Stuttgart, +Copyright (c) 2010-2015 Los Alamos National Security, LLC. All rights +Copyright (c) 2018 Mellanox Technologies, Inc. +Copyright (c) 2011-2012 University of Houston. All rights reserved. +Copyright (c) 2010-2013 Los Alamos National Security, LLC. +Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2014-2016 Intel Corporation. All rights reserved. +Copyright (c) 2013-2016 Mellanox Technologies, Inc. +Copyright © 2012 Inria. All rights reserved. +Copyright © 2010-2012, 2014 Université Bordeaux +Copyright © 2010 Cisco Systems, Inc. All rights reserved. +Copyright © 2009 CNRS +Copyright © 2009-2015 Inria. All rights reserved. +Copyright © 2009, 2011 Université Bordeaux +Copyright © 2011 Cisco Systems, Inc. All rights reserved. +Copyright © 2009-2016 Inria. All rights reserved. +Copyright © 2009-2012 Université Bordeaux +Copyright © 2009 CNRS +Copyright © 2009-2017 Inria. All rights reserved. +Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. +Copyright © 2009-2010 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2006-2018 Cisco Systems, Inc. All rights reserved +Copyright (c) 2009 University of Houston. All rights reserved. +Copyright (c) 2006-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2009 University of Houston. All rights reserved. +Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. +Copyright (c) 2006-2017 Los Alamos National Security, LLC. +Copyright (c) 2011-2017 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2018 IBM Corporation. All rights reserved. +Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights +Copyright (c) 2015 Intel, Inc. All rights reserved. +Copyright (c) 2015 Intel, Inc. All rights reserved. +Copyright (c) 2014 Mellanox Technologies, Inc. +Copyright (c) 2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2004-2015 The Trustees of the University of Tennessee. +Copyright (c) 2011-2015 The University of Tennessee and The University +Copyright (c) 2011-2015 INRIA. All rights reserved. +Copyright (c) 2011-2015 Bordeaux Polytechnic Institute +Copyright (c) 2011-2015 Université Bordeaux 1 +Copyright (c) 2011-2017 The University of Tennessee and The University +Copyright (c) 2011-2016 INRIA. All rights reserved. +Copyright (c) 2012-2017 Bordeaux Polytechnic Institute +Copyright (c) 2015-2019 Mellanox Technologies, Inc. +Copyright (c) 2015-2017 Mellanox Technologies, Inc. All rights reserved. +Copyright (c) 2015 Artem Y. Polyakov . +Copyright (c) 2015-2017 Mellanox Technologies, Inc. +Copyright (c) 2017 Mellanox Technologies, Inc. +Copyright (c) 2007-2011 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2009-2011 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2012 Universite Bordeaux 1 +Copyright (c) 2012-2015 Mellanox Technologies, Inc. +Copyright (c) 2004-2005 The University of Tennbfropsee and The University +Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved. +Copyright (c) 1991, 1993 +Copyright (c) 2016-2018 Mellanox Technologies, Inc. +Copyright (c) 2010 The University of Tennessee and The University +Copyright (c) 2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2016-2018 IBM Corporation. All rights reserved. +Copyright (c) 2017-2018 Mellanox Technologies, Inc. +Copyright (c) 2009-2010 The Trustees of Indiana University and Indiana +Copyright (c) 2004-2008 The Regents of the University of California. +Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved. +Copyright (c) 2006-2017 Sandia National Laboratories. All rights reserved. +Copyright (c) 2006-2010 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2006-2017 The University of Houston. All rights reserved. +Copyright (c) 2006-2009 Myricom, Inc. All rights reserved. +Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2007-2017 IBM Corporation. All rights reserved. +Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing +Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany +Copyright (c) 2008 Chelsio, Inc. All rights reserved. +Copyright (c) 2008-2009 Institut National de Recherche en +Copyright (c) 2007 Lawrence Livermore National Security, LLC. +Copyright (c) 2007-2017 Mellanox Technologies. All rights reserved. +Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. +Copyright (c) 2008-2017 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2006-2012 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2009-2015 Bull SAS. All rights reserved. +Copyright (c) 2016 ARM, Inc. All rights reserved. +Copyright (c) 2010-2011 Alex Brick . All rights reserved. +Copyright (c) 2013-2016 Intel, Inc. All rights reserved. +Copyright (c) 2011-2017 NVIDIA Corporation. All rights reserved. +Copyright (c) 2016 Broadcom Limited. All rights reserved. +Copyright (c) 2011-2017 Fujitsu Limited. All rights reserved. +Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP. All +Copyright (c) 2013-2017 Research Organization for Information Science (RIST). +Copyright (c) 2004-2007 The Regents of the University of California. +Copyright (c) 2006-2018 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2011 Mellanox Technologies. All rights reserved. +Copyright (c) 2007 Myricom, Inc. All rights reserved. +Copyright (c) 2008-2017 IBM Corporation. All rights reserved. +Copyright (c) 2010 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2011 University of Houston. All rights reserved. +Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2013-2017 Intel, Inc. All rights reserved +Copyright (c) 2013-2015 Intel, Inc. All rights reserved +Copyright (c) 2013-2016 Intel, Inc. All rights reserved +Copyright (c) 2012-2017 Los Alamos National Security, LLC. +Copyright (C) 2018 Mellanox Technologies, Ltd. +Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. +Copyright (c) 2019 IBM Corporation. All rights reserved. +Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights reserved. +Copyright (c) 2014-2015 Artem Y. Polyakov . +Copyright (c) 2010 Oak Ridge National Laboratory. +Copyright (c) 2010-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2019 IBM Corporation. All rights reserved. +Copyright (c) 2011-2013 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2008-2018 University of Houston. All rights reserved. +Copyright (c) 2007 Lawrence Livermore National Security, LLC. All +Copyright (c) 2016 ARM, Inc. All rights reserved. +Copyright (C) 2008 University of Chicago. +Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2015 Los Alamos National Security, Inc. All rights +Copyright (c) 2008-2009 University of Houston. All rights reserved. +Copyright 2003 +Copyright (c) 2012-2014 The University of Tennessee and The University +Copyright (c) 2009 Sun Microsystmes, Inc. All rights reserved. +Copyright (c) 2013 Los Alamos National Security, LLC. All rights +Copyright (C) Mellanox Technologies Ltd. 2001-2017. ALL RIGHTS RESERVED. +Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. +Copyright (c) 2017 IBM Corporation. All rights reserved. +Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved +Copyright 2003-2007 Niels Provos +Copyright 2007-2012 Niels Provos and Nick Mathewson +Copyright 2008-2012 Niels Provos and Nick Mathewson +Copyright 2009-2012 Niels Provos and Nick Mathewson +Copyright (c) 2003-2007 Niels Provos +Copyright (c) 2010-2012 Niels Provos and Nick Mathewson +Copyright (c) 2009-2012 Nick Mathewson and Niels Provos +Copyright (c) 2002-2007 Niels Provos +tinytest.c -- Copyright 2009-2012 Nick Mathewson +tinytest.h -- Copyright 2009-2012 Nick Mathewson +tinytestmacros.h -- Copyright 2009-2012 Nick Mathewson +Copyright (c) 2008-2019 University of Houston. All rights reserved. +Copyright (c) 2010-2016 Los Alamos National Security, LLC. +Copyright (c) 2006-2011 Cisco Systems, Inc. All rights reserved. +Copyright (C) 2013 University of Chicago. +Copyright (c) 2013-2014 Mellanox Technologies, Inc. +Copyright (c) 2004-2005 The University of Tennptlee and The University +Copyright (c) 2017-2018 Research Organization for Information Science +Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. +Copyright (c) 2015-2017 Mellanox Technologies. All rights reserved. +Copyright (c) 2008-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. +Copyright (c) 2015 Cisco Systems, Inc. +(C) 2004 by Argonne National Laboratory. +Copyright (c) 2011-2013 Los Alamos National Security, LLC. All +Copyright (C) 2001-2004 Farooq Mela. +Copyright (C) 2001 Farooq Mela. +Copyright (c) 2011 The Trustees of Indiana University. +Copyright (c) 2016 Inria. All rights reserved. +Copyright (c) 2016-2018 Inria. All rights reserved. +Copyright (c) 2016-2017 Inria. All rights reserved. +Copyright (c) 2004-201 The University of Tennessee and The University +Copyright (c) 2010-201 Oak Ridge National Labs. All rights reserved. +Copyright (c) 201-2014 Cisco Systems, Inc. All rights reserved. +Copyright (c) 201-2013 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2008 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2013 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2007 The Trustees of Indiana University and Indiana +Copyright (c) 2017 Intel, Inc. All rights reserved +Copyright (c) 2009-2013 The University of Tennessee and The University +Copyright (c) 2015-2019 Research Organization for Information Science +Copyright (c) 2019 IBM Corporation. All rights reserved. +Copyright (c) 2009-2017 The University of Tennessee and The University +Copyright (c) 2016 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. ALl rights +Copyright (c) 2014-2015 Intel, Inc. All rights reserved +Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2012-2015 Los Alamos National Security, Inc. All rights reserved. +Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2017 Inria. All rights reserved. +Copyright (c) 2013-2017 The University of Tennessee and The University +Copyright (c) 2013-2016 Inria. All rights reserved. +Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2016 Broadcom Limited. All rights reserved. +Copyright (c) 2006-2015 Mellanox Technologies. All rights reserved. +Copyright (c) 2006-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2007 Voltaire All rights reserved. +Copyright (c) 2008-2012 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2014 Bull SAS. All rights reserved +Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved. +Copyright (c) 2013-2014 NVIDIA Corporation. All rights reserved. +Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved. +Copyright (c) 2011-2015 NVIDIA Corporation. All rights reserved. +Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved +Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved. +Copyright (c) 2010-2011 IBM Corporation. All rights reserved. +Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved +Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. +Copyright (c) 2008 Chelsio, Inc. All rights reserved. +Copyright (c) 2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. +Copyright (c) 2010 University of Houston. All rights reserved. +Copyright (c) 2012 Los Alamos Nat Security, LLC. All rights reserved. +Copyright (c) 2008 University of Houston, Inc. All rights reserved. +Copyright (c) 2008 University of Houston. All rights reserved. +Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2008 The Trustees of Indiana University and Indiana +Copyright (c) 2013-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2015 University of Houston. All rights reserved. +Copyright (c) 2009 University of Houston. All rights reserved. +Copyright (c) 2012-2013 Los Alamos Nat Security, LLC. All rights reserved. +Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. +Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. +Copyright (c) 2007-2018 Cisco Systems, Inc. All rights reserved +Copyright (c) 2006-2009 University of Houston. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2012-2013 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2014-2015 Los Alamos National Security, LLC. ALl rights +Copyright (c) 2014-2015 Los Alamos National Security, LLC. All right +Copyright (c) 2006-2009 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. +Copyright (C) 2007 University of Chicago. +Copyright (c) 2006 Sun Microsystems Inc. All rights reserved. +Copyright (c) 2019 The University of Tennessee and The University +Copyright (c) 2009-2014 The University of Tennessee and The University +Copyright (c) 2009-2010 The Trustees of Indiana University. +Copyright (c) 2014 Intel Corporation. All rights reserved. +Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2010-2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2013 Mellanox Technologies, Inc. All rights reserved. +Copyright (c) 2007-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2009 Institut National de Recherche en Informatique +Copyright (c) 2018-2019 Research Organization for Information Science +Copyright (c) 2016-2017 Mellanox Technologies, Inc. +Copyright (c) 2008-2013 Los Alamos National Security, LLC. +Copyright (c) 2010 Los Alamos National Security, LLC. +Copyright (c) 2006 Sandia National Laboratories. All rights +Copyright (c) 2013-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2014-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2013-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2008-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2007 The Regents of the University of California. +Copyright (c) 2013-2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2008-2016 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2009-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2013-2017 Cisco Systems, Inc. All rights reserved +Copyright (c) 2014 Los Alamos National Security, LLC. All right +Copyright (c) 2017 IBM Corp. All rights reserved. +Copyright (c) 2008 The Trustees of Indiana University and Indiana +Copyright (c) 2012-2015 Inria. All rights reserved. +Copyright (c) 2015 Cisco Systems, Inc. +Copyright (c) 2017 ARM, Inc. All rights reserved. +Copyright (C) 2007 Oak Ridge National Laboratory +Copyright (C) 2008 Sun Microsystems, Lustre group +Copyright © 2004-2006 The Trustees of Indiana University and Indiana University Research and Technology Corporation. All rights reserved. +Copyright © 2004-2005 The University of Tennessee and The University of Tennessee Research Foundation. All rights reserved. +Copyright © 2004-2005 High Performance Computing Center Stuttgart, University of Stuttgart. All rights reserved. +Copyright © 2004-2005 The Regents of the University of California. All rights reserved. +Copyright © 2009-2015 Université Bordeaux +Copyright © 2009-2015 Cisco Systems, Inc. All rights reserved. +Copyright © 2009-2012 Oracle and/or its affiliates. All rights reserved. +Copyright © 2010 IBM +Copyright © 2010 Jirka Hladky +Copyright © 2012 Aleksej Saushev, The NetBSD Foundation +Copyright © 2012 Blue Brain Project, EPFL. All rights reserved. +Copyright © 2015 Research Organization for Information Science and Technology (RIST). All rights reserved. +Copyright © 2015-2016 Intel, Inc. All rights reserved. +Copyright (c) 2010-2012 Sandia National Laboratories. All rights reserved. +Copyright (c) 2010 Sandia National Laboratories. All rights reserved. +Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 1996 by Internet Software Consortium. +Copyright (c) 1995 by International Business Machines, Inc. +Copyright © 2009-2010, 2012 Université Bordeaux +Copyright © 2011-2015 Cisco Systems, Inc. All rights reserved. +Copyright © 2009-2011 Université Bordeaux +Copyright © 2012 Université Bordeaux +Copyright © 2013-2014 Inria. All rights reserved. +Copyright © 2010-2016 Inria. All rights reserved. +Copyright © 2011-2012 Université Bordeaux +Copyright © 2009 inria. All rights reserved. +Copyright © 2009, 2012 Université Bordeaux +Copyright © 2009-2014 Inria. All rights reserved. +Copyright © 2009-2010 Université Bordeaux +Copyright © 2009-2011, 2013 Université Bordeaux +Copyright © 2013-2017 Inria. All rights reserved. +Copyright © 2011 Université Bordeaux +Copyright © 2012-2014 Inria. All rights reserved. +Copyright © 2011-2014 Inria. All rights reserved. +Copyright © 2009-2013 Université Bordeaux +Copyright © 2012-2013 Blue Brain Project, BBP/EPFL. All rights reserved. +Copyright © 2015-2016 Inria. All rights reserved. +Copyright © 2009-2010, 2013 Université Bordeaux +Copyright © 2009-2013, 2015 Université Bordeaux +Copyright © 2015 Intel, Inc. All rights reserved. +Copyright © 2010 IBM +Copyright © 2012 Aleksej Saushev, The NetBSD Foundation +Copyright © 2013 Université Bordeaux. All right reserved. +Copyright © 2014 Cisco Systems, Inc. All rights reserved. +Copyright © 2015 Research Organization for Information Science +Copyright © 2013 Université Bordeaux. All rights reserved. +Copyright © 2016 Inria. All rights reserved. +Copyright © 2011 Oracle and/or its affiliates. All rights reserved. +Copyright © 2010-2017 Inria. All rights reserved. +Copyright © 2010-2013 Université Bordeaux +Copyright © 2010-2011 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2013 Los Alamos National Security, LLC. All Rights +Copyright (c) 2013-2017 Los Alamos National Security, LLC. All Rights +Copyright (c) 2013-2016 Los Alamos National Security, LLC. All Rights +Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. +Copyright (c) 2012 Oak Rigde National Laboratory. +Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2016-2018 Research Organization for Information Science +Copyright (c) 2012-2013 NVIDIA Corporation. All rights reserved. +Copyright (c) 2018 Triad National Security, LLC. All rights +Copyright (c) 2009-2018 Cisco Systems, Inc. All rights reserved. +Copyright (C) 2005 University of Chicago. +Copyright (C) 2002 University of Chicago. +Copyright (C) 2007 UChicago/Argonne LLC +Copyright (C) 2013 UChicago/Argonne, LLC +Copyright (C) 2007 UChicago/Argonne LLC. +Copyright (C) 2014 UChicgo/Argonne, LLC. +COPYRIGHT +(C) 2007 by Argonne National Laboratory. +Copyright (c) 2011 FUJITSU LIMITED. All rights reserved. +Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2007 Sun Microsystems, Inc. All rights reserved. +Copyright (c) 2012-2014 Los Alamos National Security, LLC. +Copyright 2010 IPB, INRIA & CNRS +Copyright (C) 1997--2004, Makoto Matsumoto, Takuji Nishimura, and +Eric Landry; All rights reserved. +Copyright (c) 2003-2011, Troy D. Hanson http:uthash.sourceforge.net +Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2017 Rutgers, The State University of New Jersey. +Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved. +Copyright (c) 2006-2007 Voltaire. All rights reserved. +Copyright (c) 2013-2017 Research Organization for Information Science +Copyright (c) 2017-2019 Research Organization for Information Science +Copyright (c) 2004-2019 The University of Tennessee and The University +Copyright (c) 2004-2017 High Performance Computing Center Stuttgart, +Copyright (c) 2008-2009 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2015 NVIDIA, Inc. All rights reserved. +Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights +Copyright (c) 2010-2012 IBM Corporation. All rights reserved. +Copyright (c) 2012-2014 NVIDIA Corporation. All rights reserved. +Copyright (c) 2014-2016 The University of Tennessee and The University +Copyright (C) 2001-2011 Mellanox Technologies Ltd. ALL RIGHTS RESERVED. +Copyright (c) 2011-2015 NVIDIA. All rights reserved. +Copyright (c) 2016-2018 Inria. All rights reserved. +Copyright (c) 2016 Inria. All rights reserved. +Copyright (c) 2017 Inria. All rights reserved. +Copyright (c) 2006-2010 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights +Copyright © 2010-2011 Université Bordeaux +Copyright © 2009-2010 Cisco Systems, Inc. All rights reserved. +Copyright © 2013-2016 Inria. All rights reserved. +Copyright © 2012 Blue Brain Project, EPFL. All rights reserved. +Copyright © 2012-2013 Inria. All rights reserved. +Copyright © 2009-2013 inria. All rights reserved. +Copyright © 2009-2013 Inria. All rights reserved. +Copyright © 2010-2014 Inria. All rights reserved. +Copyright © 2012-2016 Inria. All rights reserved. +Copyright © 2013-2015 Inria. All rights reserved. +Copyright (c) 2006-2015 Los Alamos National Security, LLC. +Copyright (c) 2010-2015 Sandia National Laboratories. All rights reserved. +Copyright (c) 2011-2017 Los Alamos National Security, LLC. +Copyright (C) 1997 University of Chicago +Copyright (c) 2006-2010 Oracle and/or its affiliates. All rights reserved +Copyright (c) 2011-2016 Los Alamos National Security, LLC. All +Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights reserved. +Copyright (c) 2013-2015 Bull SAS. All rights reserved. +Copyright (c) 2013-2015 Inria. All rights reserved. +Copyright (c) 2015 Los Alamos National Security, Inc. All rights +(c) 2008-2013 Nathan Hjelm +(C) 2013 UChicago/Argonne LLC +(C) 2011 by Argonne National Laboratory. +University of Stuttgart. All rights reserved. +Copyright (c) 2010-2017 The University of Tennessee and The University +Copyright (c) 2014-2015 NVIDIA Corporation. All rights reserved. +Copyright (c) 2010 High Performance Computing Center Stuttgart, +Copyright (c) 2011-2017 Sandia National Laboratories. All rights reserved. +Copyright (C) 2006 University of Chicago. +Copyright (C) 2006 Unknown (TODO: fix this) +Copyright (c) 2007-2013 Los Alamos National Security, LLC. +Copyright (c) 2004-2008 The Trustees of the University of Tennessee. +Copyright (c) 2010-2011, Siberian State University of Telecommunications +Copyright (c) 2010-2011, A.V. Rzhanov Institute of Semiconductor Physics SB RAS. +Copyright (C) 2011 Mikhail Kurnosov +Copyright (c) 2004-2013 The Trustees of the University of Tennessee. +Copyright (c) 2004-2014 The Trustees of the University of Tennessee. +Copyright (c) 2006-2010 Los Alamos National Security, LLC. All rights +Copyright (c) 2006-2011 Sandia National Laboratories. All rights reserved. +Copyright (c) 2006-2010 The University of Houston. All rights reserved. +Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2007-2019 IBM Corporation. All rights reserved. +Copyright (c) 2007-2019 Mellanox Technologies. All rights reserved. +Copyright (c) 2008-2010 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2009 Bull SAS. All rights reserved. +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. +Copyright (c) 2019 Amazon.com, Inc. or its affiliates. All Rights +Copyright (c) 2008 IBM Corporation. All rights reserved. +Copyright (c) 2015 University of Houston. All rights reserved. +Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2016 Intel, Inc. All rights reserved. +Copyright (c) 2006-2012 Mellanox Technologies. All rights reserved. +Copyright (c) 2010-2017 Los Alamos National Security, LLC. +Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights +Copyright (c) 2010-2018 Los Alamos National Security, LLC. +Copyright (c) 2010-2014 Los Alamos National Security, LLC. All rights +Copyright (c) 2010 Chris Davis, Niels Provos, and Nick Mathewson +Copyright (c) 1996, David Mazieres +Copyright (c) 2008, Damien Miller +Copyright (c) 2002-2006 Niels Provos +Copyright (c) 2009-2012 Niels Provos, Nick Mathewson +Copyright 2000-2009 Niels Provos +Copyright 2000-2007 Niels Provos +Copyright 2007-2012 Niels Provos, Nick Mathewson +Copyright 2003-2009 Niels Provos +Copyright 2006-2007 Niels Provos +Copyright 2007-2012 Nick Mathewson and Niels Provos +Copyright (c) 2005-2007 Niels Provos +Copyright (c) 2003-2009 Niels Provos +Copyright (c) 2007 Sun Microsystems. All rights reserved. +Copyright (c) 2008-2012 Niels Provos, Nick Mathewson +Based on work Copyright 2002 Christopher Clark +Copyright 2005-2012 Nick Mathewson +Copyright 2001-2007 Niels Provos +Copyright (c) 2000 Dug Song +Copyright (c) 1993 The Regents of the University of California. +Copyright (c) 1998 Todd C. Miller +Copyright (c) 2003 Michael A. Davis +Copyright (c) 2007 Sun Microsystems +Copyright (c) 2002 Christopher Clark +Copyright (c) 2006 Maxim Yegorushkin +Copyright (c) 2005-2012 Niels Provos and Nick Mathewson +Copyright (c) 1993 +Copyright (c) 2012 UT-Battelle, LLC. All rights reserved. +Copyright 2003 Michael A. Davis +Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2004-2012 The Trustees of Indiana University. +Copyright (c) 2017 Los Alamos National Security, LLC. All +Copyright (c) 2016 Los Alamos National Security, LLC. All rights +Copyright (c) 2010-2016 IBM Corporation. All rights reserved. +Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved. + +License: BSD-3 with additional clause +Most files in this release are marked with the copyrights of the +organizations who have edited them. The copyrights below are in no +particular order and generally reflect members of the Open MPI core +team who have contributed code to this release. The copyrights for +code used under license from other parties are included in the +corresponding files. + +Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + University Research and Technology + Corporation. All rights reserved. +Copyright (c) 2004-2017 The University of Tennessee and The University + of Tennessee Research Foundation. All rights + reserved. +Copyright (c) 2004-2010 High Performance Computing Center Stuttgart, + University of Stuttgart. All rights reserved. +Copyright (c) 2004-2008 The Regents of the University of California. + All rights reserved. +Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights + reserved. +Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2006-2010 Voltaire, Inc. All rights reserved. +Copyright (c) 2006-2017 Sandia National Laboratories. All rights reserved. +Copyright (c) 2006-2010 Sun Microsystems, Inc. All rights reserved. + Use is subject to license terms. +Copyright (c) 2006-2017 The University of Houston. All rights reserved. +Copyright (c) 2006-2009 Myricom, Inc. All rights reserved. +Copyright (c) 2007-2017 UT-Battelle, LLC. All rights reserved. +Copyright (c) 2007-2017 IBM Corporation. All rights reserved. +Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing + Centre, Federal Republic of Germany +Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany +Copyright (c) 2007 Evergrid, Inc. All rights reserved. +Copyright (c) 2008 Chelsio, Inc. All rights reserved. +Copyright (c) 2008-2009 Institut National de Recherche en + Informatique. All rights reserved. +Copyright (c) 2007 Lawrence Livermore National Security, LLC. + All rights reserved. +Copyright (c) 2007-2017 Mellanox Technologies. All rights reserved. +Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. +Copyright (c) 2008-2017 Oak Ridge National Labs. All rights reserved. +Copyright (c) 2006-2012 Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2009-2015 Bull SAS. All rights reserved. +Copyright (c) 2010 ARM ltd. All rights reserved. +Copyright (c) 2016 ARM, Inc. All rights reserved. +Copyright (c) 2010-2011 Alex Brick . All rights reserved. +Copyright (c) 2012 The University of Wisconsin-La Crosse. All rights + reserved. +Copyright (c) 2013-2016 Intel, Inc. All rights reserved. +Copyright (c) 2011-2017 NVIDIA Corporation. All rights reserved. +Copyright (c) 2016 Broadcom Limited. All rights reserved. +Copyright (c) 2011-2017 Fujitsu Limited. All rights reserved. +Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP. All + rights reserved. +Copyright (c) 2013-2017 Research Organization for Information Science (RIST). + All rights reserved. +Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights + reserved. + +$COPYRIGHT$ + +Additional copyrights may follow + +$HEADER$ + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer listed + in this license in the documentation and/or other materials + provided with the distribution. + +- Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +The copyright holders provide no reassurances that the source code +provided does not infringe any patent, copyright, or any other +intellectual property rights of third parties. The copyright holders +disclaim any liability to any recipient for claims brought against +recipient by any third party for infringement of that parties +intellectual property rights. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Software: onnx 1.6.0 +Copyright notice: +Copyright (c) ONNX Project Contributors. +Copyright (c) Facebook Inc. and Microsoft Corporation. + +License: MIT License +Please see above. + +Software: flatbuffers 1.11.0 +Copyright notice: +Copyright (c) 2015 Google, Inc. +Copyright 2014 Google Inc. All rights reserved. +Copyright 2015 Google Inc. All rights reserved. +Copyright 2017 Google Inc. All rights reserved. +Copyright 2008 Google Inc. All rights reserved. +Copyright 2014 Google Inc. +Copyright (c) 2013 Google, Inc. +Copyright (c) 2014 Google, Inc. +Copyright 2012, the Dart project authors. All rights reserved. +Copyright 2018 Google Inc. All rights reserved. +Copyright 2014 Stefan.Eilemann@epfl.ch +Copyright 2016 Google Inc. All rights reserved. +Copyright 2015 The Chromium Authors. All rights reserved. +Copyright 2015, Google Inc. +Copyright 2015 Google Inc. +Copyright 2018 Dan Field. All rights reserved. +Copyright 2018 Dan Field + + +License: Apache License V2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +1. Definitions. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. +END OF TERMS AND CONDITIONS +APPENDIX: How to apply the Apache License to your work. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. +Copyright [yyyy] [name of copyright owner] +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Software: googletest 1.8.1 +Copyright notice: +Copyright 2009, Google Inc. +Copyright 2008, Google Inc. +Copyright 2007 Google Inc. +Copyright 2007, Google Inc. +Copyright 2013, Google Inc. +Copyright 2015, Google Inc. +Copyright 2005, Google Inc. +Copyright 2008 Google Inc. +Copyright 2006, Google Inc. +Copyright 2009 Google Inc. All Rights Reserved. +Copyright 2013 Google Inc. All Rights Reserved. +Copyright 2017 Google Inc. +Copyright 2007 Neal Norwitz +Copyright 2008 Google Inc. All Rights Reserved. +Copyright 2009 Neal Norwitz All Rights Reserved. +Copyright 2003 Google Inc. +Copyright 2009 Google Inc. +Copyright 2008 Google Inc. All Rights Reserved. +Copyright [2007] Neal Norwitz +Portions Copyright [2007] Google Inc. +Copyright 2010 Google Inc. All Rights Reserved. +Copyright 2010, Google Inc. +Copyright 2005 Google Inc. All Rights Reserved. +Copyright 2018, Google Inc. +Copyright 2003, Google Inc. +Copyright 2009 Google Inc. All rights reserved. +Copyright 2015 Google Inc. All rights reserved. +Copyright 2009 Google Inc. All rights reserved. +Copyright 2018 Google LLC. All rights reserved. +Copyright 2018, Google LLC. + + +License: BSD 3-Clause License +Please see above. + +Software: glog 0.4.0 +Copyright notice: +Copyright (c) 1999, Google Inc. +Copyright (c) 2007, Google Inc. +Copyright (c) 2006, Google Inc. +Copyright (c) 2003, Google Inc. +Copyright (c) 1999, 2007, Google Inc. +Copyright (c) 2008, Google Inc. +Copyright (c) 2009, Google Inc. +Copyright (c) 2002, Google Inc. +Copyright (c) 2000 - 2007, Google Inc. +Copyright (c) 2005 - 2007, Google Inc. +Copyright (c) 2004, Google Inc. +Copyright (c) 2003-2008, Jouni Malinen and contributors + + +License: BSD 3-Clause License +Please see above. + +Software: pybind11 2.4.3 +Copyright notice: +Copyright (c) 2015 Wenzel Jakob +Copyright (c) 2016 Wenzel Jakob +Copyright (c) 2016 Trent Houliston and Wenzel Jakob +Copyright (c) 2017 Wenzel Jakob +Copyright (c) 2017 Jason Rhinelander +Copyright (c) 2016 Klemens Morgenstern and +Copyright (c) 2017 Henry F. Schreiner +Copyright (c) 2016 Sergey Lyskov and Wenzel Jakob +Copyright (c) 2016 Wenzel Jakob , All rights reserved. +Copyright (c) 2016 Jason Rhinelander +Copyright (c) 2019 Google LLC +Copyright (c) 2019 Google Inc. +Copyright (c) 2016 Ben North +Copyright (c) 2016 Klemens D. Morgenstern +Copyright (c) 2016 Pim Schellart +Copyright (c) 2017 Borja Zarco (Google LLC) +Copyright (c) 2016 Ivan Smirnov +Copyright (c) 2016 Ivan Smirnov +Copyright (c) 2016 Sergey Lyskov +Copyright (c) 2018 Hudson River Trading LLC +Copyright (c) 2019 Roland Dreier +Copyright (c) 2006, 2007 Montel Laurent, +Copyright (c) 2008, 2009 Gael Guennebaud, +Copyright (c) 2009 Benoit Jacob +Copyright 2001-2009 Kitware, Inc. +Copyright 2012 Continuum Analytics, Inc. +Copyright (c) 2007-2012 University of Illinois at Urbana-Champaign. + +License:BSD 3-Clause License +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Please also refer to the file CONTRIBUTING.md, which clarifies licensing of +external contributions to this project including patches, pull requests, etc. + +Software: SQLite 3.31.1 +Copyright notice: +Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc. +Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, 2006, 2007 2008 Free Software Foundation, Inc. +(c) The page number is greater than the largest page that existed in +Copyright (c) 1991-2011 Unicode, Inc. +Copyright (c) 2002 by David Gravereaux. +Copyright (c) 2006 by Pat Thoyts + +License: Public Domain +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. + + +Software: incubator-tvm 0.6 +Copyright notice: +Copyright (c) 2019 by Contributors +COPYRIGHT (C) 2017 Institute of Electronics and Computer Science (EDI), Latvia. +Copyright (c) 2018 by Contributors +Copyright (c) 2009-2015 by llvm/compiler-rt contributors +Copyright 2009-2010 Cybozu Labs, Inc. +Copyright 2011-2014 Kazuho Oku +Copyright 2019 The Apache Software Foundation +© Contributors Licensed under an [Apache-2.0](LICENSE) license. +©2015-2016 Cameron Desrochers. +Copyright (c) 2015 Jeff Preshing +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT +Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT +Copyright (c) 2013-2016, Cameron Desrochers. +Copyright 2017 by Contributors + +License: Apache License V2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright {yyyy} {name of copyright owner} + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +------------------------------------------------------------------------------------ +This product bundles various third-party components under other open source licenses. +This section summarizes those components and their licenses. See licenses/ +for text of these licenses. + +Apache Software Foundation License 2.0 +-------------------------------------- + +3rdparty/bfloat16/bfloat16.cc +3rdparty/dlpack +3rdparty/dmlc-core + + +BSD 2-clause License +-------------------- + +3rdparty/picojson +3rdparty/dmlc-core/include/dmlc/concurrentqueue.h + + +BSD 2-clause License + zlib License +----------------------------------- + +3rdparty/dmlc-core/include/dmlc/blockingconcurrentqueue.h + + +MIT License +----------- + +3rdparty/cma +3rdparty/compiler-rt/builtin_fp16.h + + +The Unlicense +------------- + +3rdparty/rang + +Software: dlpack 0acb731e0e43d15deee27b66f10e4c5b4e667913 +Copyright notice: +Copyright by contributors +Copyright (c) 2017 by Contributors + + +License: Apache License V2.0 + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2017 by Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Software: dmlc-core 808f485387f9a03f78fa9f1159f387d0d91b7a28 +Copyright notice: +Copyright by contributors +Copyright (c) 2015-2018 by Contributors +Copyright (c) 2017 by Contributors +Copyright (c) 2013-2016, Cameron Desrochers. +Copyright (c) 2018 by Contributors +copyright = u'2015, dmlc-core developers' +Copyright (c) 2015 by Contributors +Copyright (c) 2016 by Contributors +©2015-2016 Cameron Desrochers. + + +License: Apache License V2.0 +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Software: rang cabe04d6d6b05356fa8f9741704924788f0dd762 +Copyright notice: + + +License: The Unlicense +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +Software: google/protobuf 3.8.0 +Copyright notice: +Copyright 2008, Google Inc. +Copyright 2008 Google Inc. All Rights Reserved. +Copyright [2007] Neal Norwitz +Portions Copyright [2007] Google Inc. +Copyright 2008 Google Inc. All rights reserved. +Copyright 2007 Google Inc. All Rights Reserved. +Copyright 2007, Google Inc. +Copyright 2013, Google Inc. +Copyright 2009, Google Inc. +Copyright 2006, Google Inc. +Copyright 2009 Google Inc. All rights reserved. +Copyright 2005, Google Inc. +Copyright 2008 Google Inc. +Copyright 2015, Google Inc. +Copyright (C) 1996-2015 Free Software Foundation, Inc. +Copyright (c) 2007-2010 Baptiste Lepilleur +Copyright 2007 Neal Norwitz +Copyright 2007 Google Inc. +Copyright 2008 Google Inc. All Rights Reserved. +Copyright 2014 Google Inc. All rights reserved. +Copyright 2015 Google Inc. All rights reserved. +Copyright (c) 2006, Google Inc. +Copyright 2012 Google Inc. All rights reserved. +Copyright 2005 Google Inc. +Copyright 2010 Google Inc. All Rights Reserved. +Copyright 2010, Google Inc. +Copyright 2005 Google Inc. All Rights Reserved. +Copyright 2009 Google Inc. All rights reserved. +Copyright 2009 Google Inc. All Rights Reserved. + + +License: BSD 3-Clause License +Please see above. + + +Software: libjpeg-turbo 2.0.4 +Copyright notice: +Copyright 2009 Pierre Ossman for Cendio AB +Copyright (C) 2011, Nokia Corporation and/or its subsidiary(-ies). +Copyright (C) 2009-2011, 2013-2014, 2016, 2018, D. R. Commander. +Copyright (C) 2015-2016, 2018, Matthieu Darbois. +Copyright (C) 1999-2006, MIYASAKA Masaru. +Copyright (C) 2014-2015, D. R. Commander. All Rights Reserved. +Copyright (C) 2014, Jay Foad. All Rights Reserved. +Copyright (C) 2014, D. R. Commander. All Rights Reserved. +Copyright (C) 2015, D. R. Commander. All Rights Reserved. +Copyright (C) 1991-1996, Thomas G. Lane. +Copyright (C) 2009-2011, 2014-2016, 2018, D. R. Commander. +Copyright (C) 2009 by Dimitri van Heesch. +Copyright (C) 1994-1997, Thomas G. Lane. +Copyright (C) 1991-1997, Thomas G. Lane. +Copyright (C) 2017, D. R. Commander. +Copyright (C) 1991-1998, Thomas G. Lane. +Copyright (C) 2010, 2013-2014, 2017, D. R. Commander. +Copyright (C) 2017-2018, D. R. Commander. +Copyright (C) 2010-2011, 2013-2017, D. R. Commander. +Copyright (C) 2015, Google, Inc. +Copyright (C) 2015, 2018, D. R. Commander. +Copyright (C) 1994-1998, Thomas G. Lane. +Copyright (C) 1994-1996, Thomas G. Lane. +Copyright (C) 2009-2012, 2015, D. R. Commander. +Copyright (C) 2014, MIPS Technologies, Inc., California. +Copyright (C) 2011, 2014-2015, D. R. Commander. +Copyright (C) 2009-2011, 2014-2016, 2018-2019, D. R. Commander. +Copyright (C) 2015, Matthieu Darbois. +Copyright (C) 1997-1998, Thomas G. Lane, Todd Newman. +Copyright (C) 2010, D. R. Commander. +Copyright (C) 2010, 2016, 2018, D. R. Commander. +Copyright (C) 1991-1994, Thomas G. Lane. +Copyright (C) 2009-2011, 2018, D. R. Commander. +Copyright (C) 1995-1997, Thomas G. Lane. +Copyright (C) 2011, 2015, 2018, D. R. Commander. +Copyright (C) 2016, 2018, Matthieu Darbois. +Copyright (C) 2015, D. R. Commander. +Copyright (C)2011 D. R. Commander. All Rights Reserved. +Copyright (C) 1995-1998, Thomas G. Lane. +Copyright (C) 2016, D. R. Commander. +Copyright (C) 2010, 2015-2018, D. R. Commander. +Copyright (C) 2015-2018, D. R. Commander. +Copyright (C) 2011, 2014, 2016, 2019, D. R. Commander. +Copyright (C) 2013, 2016, D. R. Commander. +Copyright (C) 2011, 2016, 2019, D. R. Commander. +Copyright (C) 2010, 2015-2016, D. R. Commander. +Copyright (C) 2013, Linaro Limited. +Copyright (C) 2014-2015, D. R. Commander. +Copyright (C) 2009, 2011, 2015, D. R. Commander. +Copyright (C) 2009, 2011-2012, 2014-2015, D. R. Commander. +Copyright (C) 2010, 2015, D. R. Commander. +Copyright (C) 2013, MIPS Technologies, Inc., California. +Copyright (C) 2009-2011, 2016, 2018-2019, D. R. Commander. +Copyright (C) 2010-2011, 2015-2016, D. R. Commander. +Copyright (C) 2010, 2016, D. R. Commander. +Copyright (C) 2012, 2015, D. R. Commander. +Copyright (C) 2009-2011, 2016, D. R. Commander. +Copyright (C) 1991-1995, Thomas G. Lane. +Copyright (C) 2009, 2011, 2014-2015, D. R. Commander. +Copyright (C) 2014-2015, 2018, D. R. Commander. +Copyright (C) 2011, 2015, D. R. Commander. +Copyright (C) 2015-2016, 2018, D. R. Commander. +Copyright (C) 2019, Arm Limited. +Copyright (C) 2014, 2017, D. R. Commander. +Copyright (C) 2014, D. R. Commander. +Copyright (C) 1992-1996, Thomas G. Lane. +Copyright (C) 1992-1997, Thomas G. Lane. +Copyright (C) 2009, 2011, 2014-2015, 2018, D. R. Commander. +Copyright (C) 2015-2016, D. R. Commander. +Copyright (C) 2009-2011, 2013-2014, 2016-2017, D. R. Commander. +Copyright (C) 1995-2010, Thomas G. Lane, Guido Vollbeding. +Copyright (C) 2010, 2014, 2017, D. R. Commander. +Copyright (C) 2009, 2015, D. R. Commander. +Copyright (C) 2009, 2014-2015, D. R. Commander. +Copyright (C) 2011, 2014, D. R. Commander. +Copyright (C) 2009-2011, 2014, D. R. Commander. +Copyright (C) 2013, D. R. Commander. +Copyright (C) 1991-2012, Thomas G. Lane, Guido Vollbeding. +Copyright (C) 2010, 2012-2019, D. R. Commander. +copyright or license text in that file. +Copyright (C)2009-2019 D. R. Commander. All Rights Reserved. +Copyright (C)2015 Viktor Szathmáry. All Rights Reserved. +Copyright (C) 2015, 2017-2018, D. R. Commander. +Copyright (C) 1988 by Jef Poskanzer. +Copyright (C) 2015-2017, D. R. Commander. +Copyright (C) 2010, 2018, D. R. Commander. +Copyright (C) 2018, D. R. Commander. +Copyright (C)2011-2012, 2014-2015, 2017, 2019 D. R. Commander. +Copyright (C)2009-2014, 2017-2019 D. R. Commander. All Rights Reserved. +Copyright (C)2011, 2019 D. R. Commander. All Rights Reserved. +Copyright (C) 1997-2011, Thomas G. Lane, Guido Vollbeding. +Copyright (C) 2010, 2017, D. R. Commander. +Copyright (C)2011-2019 D. R. Commander. All Rights Reserved. +Copyright (C)2009-2015, 2017 D. R. Commander. All Rights Reserved. +Copyright (C) 2014-2015, 2017, 2019, D. R. Commander. +Copyright (C) 2015, 2017, D. R. Commander. +Copyright (C) 1989 by Jef Poskanzer. +Copyright (C) 2017, 2019, D. R. Commander. +Copyright (C) 2014-2015, 2019, D. R. Commander. All Rights Reserved. +Copyright (C) 2016-2018, Loongson Technology Corporation Limited, BeiJing. +Copyright (C) 2011, 2014, D. R. Commander. All Rights Reserved. +Copyright (C) 2016-2017, Loongson Technology Corporation Limited, BeiJing. +Copyright (C) 2015, 2018, D. R. Commander. All Rights Reserved. +Copyright (C) 2011, 2015, D. R. Commander. All Rights Reserved. +Copyright (C) 2014, 2018, D. R. Commander. All Rights Reserved. +Copyright (C) 2014-2015, 2018, D. R. Commander. All Rights Reserved. +Copyright (C) 2018, D. R. Commander. All Rights Reserved. +Copyright (C) 2009-2011, 2014, 2016, 2018, D. R. Commander. +Copyright (C) 2013-2014, MIPS Technologies, Inc., California. +Copyright (C) 2015, 2018, Matthieu Darbois. +Copyright (C) 2019, D. R. Commander. All Rights Reserved. +Copyright (C)2018, D. R. Commander. All Rights Reserved. +Copyright (C)2018 D. R. Commander. All Rights Reserved. +Copyright (C)2013, 2016 D. R. Commander. All Rights Reserved. +Copyright (C)2016, 2018-2019 D. R. Commander. All Rights Reserved. +Copyright (C) 2019, Google LLC. +Copyright (C)2011-2013, 2017-2018 D. R. Commander. All Rights Reserved. +Copyright (C)2011-2015, 2018 D. R. Commander. All Rights Reserved. +Copyright (C)2011, 2013 D. R. Commander. All Rights Reserved. +Copyright (C)2017-2018 D. R. Commander. All Rights Reserved. +Copyright (C)2011, 2018 D. R. Commander. All Rights Reserved. +Copyright (C)2011, 2013, 2018 D. R. Commander. All Rights Reserved. +Copyright (C)2011, 2013-2015 D. R. Commander. All Rights Reserved. +Copyright (C)2014, 2017 D. R. Commander. All Rights Reserved. +Copyright (C)2009-2014, 2016-2019 D. R. Commander. All Rights Reserved. +Copyright (C)2011-2012, 2014-2015, 2017-2018 D. R. Commander. +Copyright (C)2011-2018 D. R. Commander. All Rights Reserved. +Copyright (C) 2018, Matthieu Darbois. +Copyright 2016, 2019 D. R. Commander +Copyright 2016 Dmitry Marakasov +Copyright 2016 Roger Leigh +Copyright 2015 Alex Turbov +Copyright 2014 Rolf Eike Beer +Copyright 2014 Daniele E. Domenichelli +Copyright 2013 Dimitri John Ledkov +Copyright 2011 Alex Neundorf +Copyright 2011 Eric NOULARD +Copyright 2011, 2013-2015 Kitware, Inc. +Copyright 2011 Nikita Krupen'ko +Copyright (C) 2011, 2014-2016, 2018, D. R. Commander. +Copyright (C) 2014, Linaro Limited. + +License: libjpeg-turbo Licenses +libjpeg-turbo is covered by three compatible BSD-style open source licenses: + +The IJG (Independent JPEG Group) License, which is listed in README.ijg + +This license applies to the libjpeg API library and associated programs (any code inherited from libjpeg, and any modifications to that code.) + +The Modified (3-clause) BSD License, which is listed below + +This license covers the TurboJPEG API library and associated programs, as well as the build system. + +The zlib License + +This license is a subset of the other two, and it covers the libjpeg-turbo SIMD extensions. + +Complying with the libjpeg-turbo Licenses +This section provides a roll-up of the libjpeg-turbo licensing terms, to the best of our understanding. + +If you are distributing a modified version of the libjpeg-turbo source, then: + +You cannot alter or remove any existing copyright or license notices from the source. + +Origin + +Clause 1 of the IJG License +Clause 1 of the Modified BSD License +Clauses 1 and 3 of the zlib License +You must add your own copyright notice to the header of each source file you modified, so others can tell that you modified that file (if there is not an existing copyright header in that file, then you can simply add a notice stating that you modified the file.) + +Origin + +Clause 1 of the IJG License +Clause 2 of the zlib License +You must include the IJG README file, and you must not alter any of the copyright or license text in that file. + +Origin + +Clause 1 of the IJG License +If you are distributing only libjpeg-turbo binaries without the source, or if you are distributing an application that statically links with libjpeg-turbo, then: + +Your product documentation must include a message stating: + +This software is based in part on the work of the Independent JPEG Group. + +Origin + +Clause 2 of the IJG license +If your binary distribution includes or uses the TurboJPEG API, then your product documentation must include the text of the Modified BSD License (see below.) + +Origin + +Clause 2 of the Modified BSD License +You cannot use the name of the IJG or The libjpeg-turbo Project or the contributors thereof in advertising, publicity, etc. + +Origin + +IJG License +Clause 3 of the Modified BSD License +The IJG and The libjpeg-turbo Project do not warrant libjpeg-turbo to be free of defects, nor do we accept any liability for undesirable consequences resulting from your use of the software. + +Origin + +IJG License +Modified BSD License +zlib License +The Modified (3-clause) BSD License +Copyright (C)2009-2019 D. R. Commander. All Rights Reserved. Copyright (C)2015 Viktor Szathmáry. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +Neither the name of the libjpeg-turbo Project nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS", AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Why Three Licenses? +The zlib License could have been used instead of the Modified (3-clause) BSD License, and since the IJG License effectively subsumes the distribution conditions of the zlib License, this would have effectively placed libjpeg-turbo binary distributions under the IJG License. However, the IJG License specifically refers to the Independent JPEG Group and does not extend attribution and endorsement protections to other entities. Thus, it was desirable to choose a license that granted us the same protections for new code that were granted to the IJG for code derived from their software. + + +Software: opencv 4.2.0 +Copyright notice: +Copyright (C) 2016, NVIDIA Corporation, all rights reserved. +Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved. +Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved. +Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +Copyright (C) 2009, Willow Garage Inc., all rights reserved. +Copyright (C) 2013, OpenCV Foundation, all rights reserved. +Copyright (c) 2010, Google Inc. All rights reserved. +Copyright 2013 Google Inc. All Rights Reserved. +Copyright 2011 Google Inc. All Rights Reserved. +Copyright 2015 Google Inc. All Rights Reserved. +Copyright 2010 Google Inc. All Rights Reserved. +Copyright 2012 Google Inc. All Rights Reserved. +Copyright 2014 Google Inc. All Rights Reserved. +Copyright 2017 Google Inc. All Rights Reserved. +Copyright 2016 Google Inc. All Rights Reserved. +Copyright 2018 Google Inc. All Rights Reserved. +Copyright (c) 2015 The Regents of the University of California (Regents) +Copyright (c) 2015, 2015 The Regents of the University of California (Regents) +Copyright (C) 2000, Intel Corporation, all rights reserved. +Copyright (C) 2016, Itseez, Inc, all rights reserved. +Copyright (C) 2014, Itseez, Inc, all rights reserved. +Copyright (C) 2008-2011, Willow Garage Inc., all rights reserved. +Copyright (C) 2018 Intel Corporation +Copyright (C) 2014, Itseez Inc, all rights reserved. +Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +Copyright (c) 2013, Bo Li (prclibo@gmail.com), ETH Zurich +Copyright (C) 2014, Samson Yilma (samsonyilma@yahoo.com), all rights reserved. +Copyright (C) 2018, Intel Corporation, all rights reserved. +Copyright 2017 Toby Collins +Copyright (c) 2007, Miroslav Balda +Copyright (C) 2015, Itseez Inc., all rights reserved. +Copyright (C) 2009, Intel Corporation and others, all rights reserved. +Copyright (C) 2019 Intel Corporation +Copyright (C) 2018-2019 Intel Corporation +Copyright (C) 2014, Advanced Micro Devices, Inc., all rights reserved. +Copyright (C) 2010-2012, MulticoreWare Inc., all rights reserved. +Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved. +Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved. +Copyright (C) 2014, Intel Corporation, all rights reserved. +Copyright (C) 2014, Itseez, Inc., all rights reserved. +Copyright (C) 2001 Fabrice Bellard +Copyright (C) 1991, 1999 Free Software Foundation, Inc. +Copyright (C) 2015, NVIDIA Corporation, all rights reserved. +Copyright (C) 2014-2015, NVIDIA Corporation, all rights reserved. +Copyright 2012. All rights reserved. +Copyright (c) Microsoft Open Technologies, Inc. +Copyright (C) 2014-2015, Itseez Inc., all rights reserved. +Copyright (C) 2014, Itseez Inc., all rights reserved. +Copyright (C) 2008-2013, Willow Garage Inc., all rights reserved. +Copyright 2015-2017 Philippe Tillet +Copyright (c) 2017, Intel Corporation +Copyright (C) 2000-2018, Intel Corporation, all rights reserved. +Copyright (c) 2011. Philipp Wagner . +Copyright (c) 2008 - 2012 The Khronos Group Inc. +Copyright (C) 2016, Intel Corporation, all rights reserved. +Copyright (C) 2015, Itseez, Inc., all rights reserved. +Copyright (C) 2000-2015, Intel Corporation, all rights reserved. +Copyright (C) 2015, OpenCV Foundation, all rights reserved. +Copyright (c) 2005-2014 Intel Corporation. All rights reserved. +Copyright (c) 2011, Intel Corporation +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +Copyright (c) 2008 - 2009 NVIDIA Corporation. All rights reserved. +Copyright (c) 2007-2009 +Copyright (c) 2014, 2015, The Regents of the University of California (Regents) +Copyright (c) 2014, 2015, the respective contributors +Copyright 2010 Argus Corp. All rights reserved. +Copyright (c) 2015, Piotr Dobrowolski dobrypd[at]gmail[dot]com +Copyright 2008, Google Inc. +Copyright 2005, Google Inc. +Copyright 2007, Google Inc. +Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved. +Copyright (C) 2009-2010, Willow Garage Inc., all rights reserved. +Copyright 2008 Google Inc. All rights reserved. +Copyright (C) 2017, Intel Corporation, all rights reserved. +Copyright (C) 2018-2019, Intel Corporation, all rights reserved. +Copyright (C) 2017-2019, Intel Corporation, all rights reserved. +Copyright (C) 2008-2012, Willow Garage Inc., all rights reserved. +Copyright( C) 2000, Intel Corporation, all rights reserved. +Copyright (c) 2000-2003 Chih-Chung Chang and Chih-Jen Lin +Copyright (C) 2016, Itseez Inc, all rights reserved. +Copyright (C) 2018 Ya-Chiu Wu, all rights reserved. +Copyright (C) 2016, OpenCV Foundation, all rights reserved. +Copyright (C) 2010 The Android Open Source Project +Copyright (C) 2016 The Android Open Source Project +Copyright (c) 2010-2011, Ethan Rublee +Copyright (c) 2011-2014, Andrey Kamaev +Copyright (c) 2013 NVIDIA Corporation. All rights reserved. +Copyright (c) 2014, The Regents of the University of California (Regents) +Copyright (c) 2014, the respective contributors +Copyright (c) 1988-1997 Sam Leffler +Copyright (c) 1991-1997 Silicon Graphics, Inc. +Copyright (c) 2009 Frank Warmerdam +Copyright (c) 1991-1997 Sam Leffler +Copyright (c) 1990-1997 Sam Leffler +Copyright (C) 1990, 1995 Frank D. Cringle. +Copyright (c) 1994-1997 Sam Leffler +Copyright (c) 1994-1997 Silicon Graphics, Inc. +Copyright (c) 1997 Greg Ward Larson +Copyright (c) 1997 Silicon Graphics, Inc. +Copyright (c) 2010, Andrey Kiselev +Copyright (c) Joris Van Damme +Copyright (c) AWare Systems +Copyright (c) 1996-1997 Sam Leffler +Copyright (c) 1996 Pixar +Copyright (c) 1995-1997 Sam Leffler +Copyright (c) 1995-1997 Silicon Graphics, Inc. +Copyright (c) 1988-1996 Sam Leffler +Copyright (c) 1991-1996 Silicon Graphics, Inc. +Copyright (c) 1992-1997 Sam Leffler +Copyright (c) 1992-1997 Silicon Graphics, Inc. +Copyright (c) 2018, Mapbox +Copyright (c) 2017, Planet Labs +Copyright (C) 2014, Itseez Inc. See the license at http:opencv.org +Copyright (c) 1995-2019 The PNG Reference Library Authors. +Copyright (c) 2018-2019 Cosmin Truta. +Copyright (c) 2000-2002, 2004, 2006-2018 Glenn Randers-Pehrson. +Copyright (c) 1996-1997 Andreas Dilger. +Copyright (c) 1995-1996 Guy Eric Schalnat, Group 42, Inc. +Copyright (c) 2000-2002, 2004, 2006-2018 Glenn Randers-Pehrson, are +Copyright (c) 1998-2000 Glenn Randers-Pehrson, are derived from +Copyright (c) 1996-1997 Andreas Dilger, are derived from libpng-0.88, +Copyright (c) 2018-2019 Cosmin Truta +Copyright (c) 1998-2002,2004,2006-2018 Glenn Randers-Pehrson +Copyright (c) 1996-1997 Andreas Dilger +Copyright (c) 1998-2002,2004,2006-2016,2018 Glenn Randers-Pehrson +Copyright (c) 2018 Cosmin Truta +Copyright (c) 1998-2002,2004,2006-2013 Glenn Randers-Pehrson +Copyright (c) 1998-2002,2004,2006-2017 Glenn Randers-Pehrson +Copyright (c) 1998-2002,2004,2006-2013,2018 Glenn Randers-Pehrson +Copyright (c) 1998-2002,2004,2006-2014,2016 Glenn Randers-Pehrson +Copyright (c) 1998-2018 Glenn Randers-Pehrson +Copyright (c) 1998-2002,2004,2006-2014,2016,2018 Glenn Randers-Pehrson +Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. +Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. +(C) Copyright Christopher Diggins 2005-2011 +(C) Copyright Pablo Aguilar 2005 +(C) Copyright Kevlin Henney 2001 +Copyright 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. +Copyright 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. +Copyright (c) 2004, Industrial Light & Magic, a division of Lucas +Copyright (c) 2004-2012, Industrial Light & Magic, a division of Lucas +Copyright (c) 2002-2012, Industrial Light & Magic, a division of Lucas +Copyright (c) 2012, Industrial Light & Magic, a division of Lucas +Copyright (c) 2011-2012, Industrial Light & Magic, a division of Lucas +Copyright (c) 2006-2012, Industrial Light & Magic, a division of Lucas +Copyright (c) 2002, Industrial Light & Magic, a division of Lucas +Copyright (c) 2016-2017 Fabian David Tschopp, all rights reserved. +Copyright (C) 2014, Intel, Inc., all rights reserved. +Copyright 2015, Google Inc. +Copyright 2014, Google Inc. All rights reserved. +Copyright 2012 Google Inc. All rights reserved. +Copyright 2013 Red Hat Inc. All rights reserved. +Copyright 2014 Bloomberg Finance LP. All rights reserved. +Copyright 2015 Google Inc. All rights reserved. +Copyright 2014 Google Inc. All rights reserved. +Copyright 2013 Google Inc. All rights reserved. +Copyright 2014 Google Inc. All rights reserved. +Copyright (c) 2006, Google Inc. +Copyright 2005-2008 Google Inc. All Rights Reserved. +Copyright 2005 Google Inc. +Copyright (C) 2010-2012 Daniel Beer +Copyright (c) 2001-2003 Michael David Adams. +Copyright (c) 2001-2006 Michael David Adams +Copyright (c) 1999-2000 Image Power, Inc. +Copyright (c) 1999-2000 The University of British Columbia +Copyright (c) 2002-2003 Michael David Adams. +Copyright (c) 2001-2002 Michael David Adams. +Copyright (c) 1999-2000 Image Power, Inc. and the University of +Copyright (c) 2004 Michael David Adams. +Copyright (c) Microsoft Corporation. All rights reserved. +Copyright (c) 2008-2013 The Khronos Group Inc. +Copyright (c) 2016 Glenn Randers-Pehrson +copyright or license text in that file. +Copyright (C)2009-2019 D. R. Commander. All Rights Reserved. +Copyright (C)2015 Viktor Szathmáry. All Rights Reserved. +Copyright (C) 2008-2010, Willow Garage Inc., all rights reserved. +Copyright (c) 2014-2016, Joseph Howse (Nummist Media Corporation Limited, Halifax, Nova Scotia, Canada). All rights reserved. +Copyright (c) 2017, Puttemans Steven, Can Ergun and Toon Goedeme +Copyright (c) 2015, Advanced Micro Devices, Inc. +Copyright (c) 2008-2012 The Khronos Group Inc. +Copyright (c) 2008-2009 The Khronos Group Inc. +Copyright (c) 2008-2010 The Khronos Group Inc. +Copyright (c) 2008-2015 The Khronos Group Inc. +Copyright (c) 2006, Industrial Light & Magic, a division of Lucasfilm +Copyright 2011, 2012, 2013, 2014, 2015, 2016, 2017 The Regents of the University of California. All rights reserved. +Copyright (c) 2014-2017 The Khronos Group Inc. +Copyright (c) 2015-2018 The Khronos Group Inc. +Copyright (c) 2002-2018, Industrial Light & Magic, a division of Lucas +Copyright (c) Microsoft. All rights reserved. +Copyright (c) Microsoft Corporation. All rights reserved +Copyright (c) 2014,2016 Glenn Randers-Pehrson +Copyright (c) 2017-2018 Arm Holdings. All rights reserved. +Copyright (C) Microsoft Corporation. All rights reserved. +Copyright (C) 2000-2016, Intel Corporation, all rights reserved. +Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved. +Copyright (C) 2015-2016, Itseez Inc., all rights reserved. +Copyright (c) 2017 Glenn Randers-Pehrson +Copyright (c) 2016-2017 Glenn Randers-Pehrson +Copyright (c) 2004, Hannes Kruppa and Bernt Schiele (ETH Zurich, Switzerland). +Copyright (c) 2011, Modesto Castrillon-Santana (IUSIANI, Universidad de +Copyright (C) 2014, NVIDIA Corporation, all rights reserved. +Copyright (C) 2012-2015, NVIDIA Corporation, all rights reserved. +Copyright (c) 2006, 2008 Edward Rosten +Copyright (C) 2012-2014, NVIDIA Corporation, all rights reserved. +Copyright (C) 2014-2016, NVIDIA Corporation, all rights reserved. +Copyright (C) 2013-2015, NVIDIA Corporation, all rights reserved. +Copyright (C) 2010 Elmar Mair +Copyright (C) 2008, Willow Garage Inc., all rights reserved. +Copyright (C) 2011 The Autonomous Systems Lab (ASL), ETH Zurich, Stefan Leutenegger, Simon Lynen and Margarita Chli. +Copyright (c) 2009, Willow Garage, Inc. +Copyright (C) 2009, Liu Liu All rights reserved. +Copyright (c) 2017 Joseph Redmon +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) +Copyright (C) 1991-2012, Thomas G. Lane, Guido Vollbeding. +Copyright (C) 1995-2012 Jean-loup Gailly and Mark Adler. +Copyright (c) 2001-2003 Michael David Adams Entertainment Company Ltd. Portions contributed and copyright held by others as indicated. All rights reserved. +Copyright (c) 2001 Fabrice Bellard +Copyright (C) 2008-2013, Itseez Inc., all rights reserved. +Copyright (C) 2013, Itseez Inc, all rights reserved. +Copyright (c) 1999-2000, Image Power, Inc. and the University of +Copyright (c) 2001-2004 Michael David Adams. +Copyright (C) 2000-2019, Intel Corporation, all rights reserved. +Copyright (C) 2019, Xperience AI, all rights reserved. +Copyright (c) 2005-2012, Industrial Light & Magic, a division of Lucas +Copyright (C) 2008, 2011, Nils Hasler, all rights reserved. +Copyright (C) 2008, Nils Hasler, all rights reserved. +Copyright (C) 2015, 2018, D. R. Commander. +Copyright (C) 1994-1998, Thomas G. Lane. +Copyright (C) 1994-1996, Thomas G. Lane. +Copyright (C) 1994-1997, Thomas G. Lane. +Copyright (C) 1991-1996, Thomas G. Lane. +Copyright (C) 2009-2012, 2015, D. R. Commander. +Copyright 2009 Pierre Ossman for Cendio AB +Copyright (C) 2014, MIPS Technologies, Inc., California. +Copyright (C) 1999-2006, MIYASAKA Masaru. +Copyright (C) 2011, 2014-2015, D. R. Commander. +Copyright (C) 1991-1997, Thomas G. Lane. +Copyright (C) 2009-2011, 2014-2016, 2018, D. R. Commander. +Copyright (C) 2015, Matthieu Darbois. +Copyright (C) 1997-1998, Thomas G. Lane, Todd Newman. +Copyright (C) 2017, D. R. Commander. +Copyright (C) 1991-1998, Thomas G. Lane. +Copyright (C) 2010, D. R. Commander. +Copyright (C) 2010, 2016, 2018, D. R. Commander. +Copyright (C) 2009-2011, 2018, D. R. Commander. +Copyright (C) 1995-1997, Thomas G. Lane. +Copyright (C) 2011, 2015, 2018, D. R. Commander. +Copyright (C) 2016, 2018, Matthieu Darbois. +Copyright (C) 2015, D. R. Commander. +Copyright (C) 1995-1998, Thomas G. Lane. +Copyright (C) 2016, D. R. Commander. +Copyright (C) 2010, 2015-2018, D. R. Commander. +Copyright (C) 2015, Google, Inc. +Copyright (C) 2015-2018, D. R. Commander. +Copyright (C) 2013, 2016, D. R. Commander. +Copyright (C) 2010, 2015-2016, D. R. Commander. +Copyright (C) 2013, Linaro Limited. +Copyright (C) 2014-2015, D. R. Commander. +Copyright (C) 2009, 2011, 2015, D. R. Commander. +Copyright (C) 2009, 2011-2012, 2014-2015, D. R. Commander. +Copyright (C) 2010, 2015, D. R. Commander. +Copyright (C) 2013, MIPS Technologies, Inc., California. +Copyright (C) 2009-2011, 2016, 2018, D. R. Commander. +Copyright (C) 2010-2011, 2015-2016, D. R. Commander. +Copyright (C) 2010, 2016, D. R. Commander. +Copyright (C) 2012, 2015, D. R. Commander. +Copyright (C) 2009-2011, 2016, D. R. Commander. +Copyright (C) 1991-1995, Thomas G. Lane. +Copyright (C) 2009, 2011, 2014-2015, D. R. Commander. +Copyright (C) 2014-2015, 2018, D. R. Commander. +Copyright (C) 2011, 2015, D. R. Commander. +Copyright (C) 2015-2016, 2018, D. R. Commander. +Copyright (C) 2014, 2017, D. R. Commander. +Copyright (C) 2014, D. R. Commander. +Copyright (C) 1991-1994, Thomas G. Lane. +Copyright (C) 1992-1996, Thomas G. Lane. +Copyright (C) 2017-2018, D. R. Commander. +Copyright (C) 1992-1997, Thomas G. Lane. +Copyright (C) 2009, 2011, 2014-2015, 2018, D. R. Commander. +Copyright (C) 2015-2016, D. R. Commander. +Copyright (C) 2009-2011, 2013-2014, 2016-2017, D. R. Commander. +Copyright (C) 2009, 2015, D. R. Commander. +Copyright (C) 2009, 2014-2015, D. R. Commander. +Copyright (C) 2011, 2014, D. R. Commander. +Copyright (C) 2015-2016, 2018, Matthieu Darbois. +Copyright (C) 2009-2011, 2014, D. R. Commander. +Copyright (C) 2013, D. R. Commander. +Copyright (C) 2010, 2012-2019, D. R. Commander. +Copyright (C) 1995-2011, 2016 Mark Adler +Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler +Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler +Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler +Copyright (C) 1995-2016 Jean-loup Gailly +Copyright (C) 2004, 2010 Mark Adler +Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler +Copyright (C) 2004-2017 Mark Adler +Copyright (C) 1995-2016 Mark Adler +Copyright (C) 1995-2017 Mark Adler +Copyright (C) 1995-2003, 2010 Mark Adler +Copyright (C) 1995-2005, 2010 Mark Adler(C) 1995-2017 Jean-loup Gailly and Mark Adler +Copyright (C) 1995-2017 Jean-loup Gailly +Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler +Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler +Copyright (C) 2013, NVIDIA Corporation, all rights reserved. +Copyright (c) 2009-2014 DreamWorks Animation LLC. +Copyright (c) 2007, Industrial Light & Magic, a division of Lucas +Copyright (c) 2006, Industrial Light & Magic, a division of Lucas +Copyright (c) 2009, Industrial Light & Magic, a division of Lucas +Copyright (c) 2003, Industrial Light & Magic, a division of Lucas +Copyright (c) 2012, Weta Digital Ltd +Copyright (c) 2011, Industrial Light & Magic, a division of Lucas +Copyright (c) 2013, Industrial Light & Magic, a division of Lucas +Copyright (c) 2007, Weta Digital Ltd(c) 2012 Weta Digital Ltd +Copyright (c) 2012, Autodesk, Inc. +Copyright (c) 2004, Pixar Animation Studios +Copyright (c) 2004, Industrial Light & Magic, a division of Lucasfilm +Copyright (c) 2005, Industrial Light & Magic, a division of Lucas +Copyright (C) 2000-2008, 2018, Intel Corporation, all rights reserved. +Copyright (C) 1998 Yossi Rubner +Copyright (C) 2009-2012, Willow Garage Inc., all rights reserved. +Copyright (C) 2006 Simon Perreault +Copyright (C) 2013, Ovidiu Parvu, all rights reserved. +Copyright (c) 2008-2011, William Lucas +Copyright (C) 2000-2008, 2017, Intel Corporation, all rights reserved. +Copyright (C) 2019, Intel Corporation, all rights reserved. +Copyright (C) 2000-2008,2019 Intel Corporation, all rights reserved. +Copyright (c) 2007-2008 Intel Corporation. All Rights Reserved. +Copyright (c) 2008, Industrial Light & Magic, a division of Lucas +Copyright (C) 1991-2018, Thomas G. Lane, Guido Vollbeding. + +License: 3-clause BSD License +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2000-2019, Intel Corporation, all rights reserved. +Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved. +Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved. +Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved. +Copyright (C) 2015-2016, Itseez Inc., all rights reserved. +Copyright (C) 2019, Xperience AI, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + + +Software: cmake-modules cf2e087039f81d13e687cf6c2b1b382b9c1e756f +Copyright notice: +Copyright 2009 Kitware, Inc. +Copyright 2009 Will Dicharry +Copyright 2005-2009 Kitware, Inc. +Copyright Iowa State University 2009-2010. +Copyright 2006-2009 Kitware, Inc. +Copyright 2006-2008 Andreas Schneider +Copyright 2007 Wengo +Copyright 2007 Mike Jackson +Copyright 2008 Andreas Pakulat +Copyright 2008-2010 Philip Lowman +Copyright 2009 Alexander Neundorf +Copyright (c) 2012 - 2017, Lars Bilke +Copyright (c) 2012-2016 Sascha Kratky +Copyright 2012-2018 Sascha Kratky +Copyright (c) 2012-2018, OpenGeoSys Community (http://www.opengeosys.org) +Copyright (c) 2012 - 2015, Lars Bilke +Copyright 2008-2009 Philip Lowman +Copyright 2010 Iowa State University (Ryan Pavlik ) +Copyright 2000-2009 Kitware, Inc., Insight Software Consortium +Copyright 2010-2011 Kitware, Inc. +Copyright Iowa State University 2009-2011 + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +Software: Myia 395fa4c7beb9479a1d6e323e4cc0ecf25733dfba +Copyright notice: +Copyright (c) 2017 MILA +Please visit the Myia web site for more information: + * https://github.com/mila-iqia/myia/ + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000000..65ee17a72a5 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +git submodule update --init --recursive + + diff --git a/build.sh b/build.sh new file mode 100755 index 00000000000..9d812d6dcc8 --- /dev/null +++ b/build.sh @@ -0,0 +1,468 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +BASEPATH=$(cd "$(dirname $0)"; pwd) +PROJECT_PATH="${BASEPATH}" +CUDA_PATH="" +CUDNN_PATH="" +export BUILD_PATH="${BASEPATH}/build/" +# print usage message +usage() +{ + echo "Usage:" + echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t on|off] [-g on|off] [-h] [-s] [-b ge|cpu] [-m infer|train] \\" + echo " [-a on|off] [-g on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|d|cpu] \\" + echo " [-P on|off] [-z] [-M on|off] [-V 9.2|10.1] [-I] [-K]" + echo "" + echo "Options:" + echo " -d Debug mode" + echo " -r Release mode, default mode" + echo " -v Display build command" + echo " -c Enable code coverage switch, default off" + echo " -t Run testcases switch, default on" + echo " -g Use glog to output log, default on" + echo " -h Print usage" + echo " -s Install or setup" + echo " -b Select other backend, available: \\" + echo " ge:graph engine, cpu" + echo " -m Select mode, available: infer, train, default is infer " + echo " -a Enable ASAN, default off" + echo " -p Enable pipeline profile, default off" + echo " -i Enable increment building, default off" + echo " -L Enable load ANF-IR as input of 'infer', default off" + echo " -R Enable the time_line record, default off" + echo " -j[n] Set the threads when building (Default: -j8)" + echo " -e Use gpu, d or cpu" + echo " -P Enable dump anf graph to file in ProtoBuffer format, default on" + echo " -Q Enable dump end to end, default off" + echo " -D Enable dumping of function graph ir, default on" + echo " -z Compile dataset & mindrecord, default off" + echo " -M Enable MPI and NCCL for GPU training, default off" + echo " -V Specify the minimum required cuda version, default CUDA 9.2" + echo " -I Compile predict, default off" + echo " -K Compile with AKG, default off" +} + +# check value of input is 'on' or 'off' +# usage: check_on_off arg_value arg_name +check_on_off() +{ + if [[ "X$1" != "Xon" && "X$1" != "Xoff" ]]; then + echo "Invalid value $1 for option -$2" + usage + exit 1 + fi +} + +# check and set options +checkopts() +{ + # Init default values of build options + THREAD_NUM=8 + DEBUG_MODE="off" + VERBOSE="" + ENABLE_COVERAGE="off" + RUN_TESTCASES="off" + EXECUTE_SETUP="off" + ENABLE_BACKEND="" + TRAIN_MODE="INFER" + ENABLE_ASAN="off" + ENABLE_PROFILE="off" + INC_BUILD="off" + ENABLE_LOAD_IR="off" + ENABLE_TIMELINE="off" + ENABLE_DUMP2PROTO="on" + ENABLE_DUMPE2E="off" + ENABLE_DUMP_IR="on" + COMPILE_MINDDATA="off" + ENABLE_MPI="off" + CUDA_VERSION="9.2" + COMPILE_PREDICT="off" + USE_GLOG="on" + PREDICT_PLATFORM="" + ENABLE_AKG="off" + + # Process the options + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K' opt + do + OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') + case "${opt}" in + d) + DEBUG_MODE="on" + ;; + r) + DEBUG_MODE="off" + ;; + v) + VERBOSE="VERBOSE=1" + ;; + j) + THREAD_NUM=$OPTARG + ;; + c) + check_on_off $OPTARG c + ENABLE_COVERAGE="$OPTARG" + ;; + t) + check_on_off $OPTARG t + RUN_TESTCASES="$OPTARG" + ;; + g) + check_on_off $OPTARG g + USE_GLOG="$OPTARG" + ;; + h) + usage + exit 0 + ;; + s) + EXECUTE_SETUP="on" + ;; + b) + if [[ "X$OPTARG" != "Xge" && "X$OPTARG" != "Xcpu" ]]; then + echo "Invalid value ${OPTARG} for option -b" + usage + exit 1 + fi + ENABLE_BACKEND=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]') + if [[ "X$ENABLE_BACKEND" == "XGE" ]]; then + ENABLE_GE="on" + fi + if [[ "X$ENABLE_BACKEND" != "XCPU" ]]; then + ENABLE_CPU="on" + fi + ;; + a) + check_on_off $OPTARG a + ENABLE_ASAN="$OPTARG" + ;; + p) + check_on_off $OPTARG p + ENABLE_PROFILE="$OPTARG" + ;; + i) + INC_BUILD="on" + ;; + m) + if [[ "X$OPTARG" != "Xinfer" && "X$OPTARG" != "Xtrain" ]]; then + echo "Invalid value ${OPTARG} for option -m" + usage + exit 1 + fi + TRAIN_MODE=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]') + ;; + L) + ENABLE_LOAD_IR="on" + echo "build with enable load anf ir" + ;; + R) + ENABLE_TIMELINE="on" + echo "enable time_line record" + ;; + e) + if [[ "X$OPTARG" == "Xgpu" ]]; then + ENABLE_GPU="on" + ENABLE_CPU="on" + elif [[ "X$OPTARG" == "Xd" ]]; then + ENABLE_D="on" + ENABLE_CPU="on" + elif [[ "X$OPTARG" == "Xcpu" ]]; then + ENABLE_CPU="on" + else + echo "Invalid value ${OPTARG} for option -e" + usage + exit 1 + fi + ;; + M) + check_on_off $OPTARG M + ENABLE_MPI="$OPTARG" + ;; + V) + if [[ "X$OPTARG" != "X9.2" && "X$OPTARG" != "X10.1" ]]; then + echo "Invalid value ${OPTARG} for option -V" + usage + exit 1 + fi + CUDA_VERSION="$OPTARG" + ;; + P) + check_on_off $OPTARG p + ENABLE_DUMP2PROTO="$OPTARG" + echo "enable dump anf graph to proto file" + ;; + Q) + check_on_off $OPTARG Q + ENABLE_DUMPE2E="$OPTARG" + echo "enable dump end to end" + ;; + D) + check_on_off $OPTARG D + ENABLE_DUMP_IR="$OPTARG" + echo "enable dump function graph ir" + ;; + z) + COMPILE_MINDDATA="on" + ;; + I) + COMPILE_PREDICT="on" + if [[ "$OPTARG" == "arm64" ]]; then + PREDICT_PLATFORM="arm64" + elif [[ "$OPTARG" == "x86_64" ]]; then + PREDICT_PLATFORM="x86_64" + else + echo "-I parameter must be arm64 or x86_64" + exit 1 + fi + ;; + K) + ENABLE_AKG="on" + echo "enable compile with akg" + ;; + *) + echo "Unknown option ${opt}!" + usage + exit 1 + esac + done +} +checkopts "$@" +echo "---------------- mindspore: build start ----------------" +mkdir -pv "${BUILD_PATH}/package/mindspore/lib" +git submodule update --init graphengine + +build_exit() +{ + echo "$@" >&2 + stty echo + exit 1 +} + +# Create building path +build_mindspore() +{ + echo "start build mindspore project." + mkdir -pv "${BUILD_PATH}/mindspore" + cd "${BUILD_PATH}/mindspore" + CMAKE_ARGS="-DDEBUG_MODE=$DEBUG_MODE -DBUILD_PATH=$BUILD_PATH" + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_LOAD_ANF_IR=$ENABLE_LOAD_IR" + if [[ "X$ENABLE_COVERAGE" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_COVERAGE=ON" + fi + if [[ "X$RUN_TESTCASES" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_TESTCASES=ON" + fi + if [[ -n "$ENABLE_BACKEND" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_${ENABLE_BACKEND}=ON" + fi + if [[ -n "$TRAIN_MODE" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_${TRAIN_MODE}=ON" + fi + if [[ "X$ENABLE_ASAN" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_ASAN=ON" + fi + if [[ "X$ENABLE_PROFILE" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PROFILE=ON" + fi + if [[ "X$ENABLE_TIMELINE" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_TIMELINE=ON" + fi + if [[ "X$ENABLE_DUMP2PROTO" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DUMP_PROTO=ON" + fi + if [[ "X$ENABLE_DUMPE2E" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DUMP_E2E=ON" + fi + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DUMP_IR=${ENABLE_DUMP_IR^^}" + if [[ "X$ENABLE_MPI" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_MPI=ON" + fi + if [[ "X$ENABLE_D" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_D=ON" + fi + if [[ "X$ENABLE_GPU" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GPU=ON -DCUDA_PATH=$CUDA_PATH -DCUDNN_PATH=$CUDNN_PATH -DMS_REQUIRE_CUDA_VERSION=${CUDA_VERSION}" + fi + if [[ "X$ENABLE_CPU" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_CPU=ON" + fi + if [[ "X$COMPILE_MINDDATA" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_MINDDATA=ON" + fi + if [[ "X$USE_GLOG" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DUSE_GLOG=ON" + fi + if [[ "X$ENABLE_AKG" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_AKG=ON" + fi + echo "${CMAKE_ARGS}" + if [[ "X$INC_BUILD" = "Xoff" ]]; then + cmake ${CMAKE_ARGS} ../.. + fi + make ${VERBOSE} -j$THREAD_NUM + if [[ "X$EXECUTE_SETUP" = "Xon" ]]; then + make install + fi + echo "success to build mindspore project!" +} + +build_predict() +{ + git submodule update --init --recursive third_party/incubator-tvm + echo "start build predict project" + + git submodule update --init --recursive third_party/flatbuffers + git submodule update --init --recursive third_party/googletest + git submodule update --init --recursive third_party/protobuf + + rm -rf "${BASEPATH}/predict/build" + mkdir -pv "${BASEPATH}/predict/build" + rm -rf "${BASEPATH}/predict/output" + mkdir -pv "${BASEPATH}/predict/output" + + if [[ "$PREDICT_PLATFORM" == "arm64" ]]; then + if [ "${ANDROID_NDK}" ]; then + echo -e "\e[31mANDROID_NDK_PATH=$ANDROID_NDK \e[0m" + else + echo -e "\e[31mplease set ANDROID_NDK_PATH in environment variable for example: export ANDROID_NDK=/root/usr/android-ndk-r16b/ \e[0m" + exit 1 + fi + fi + + #build flatbuf + cd "${BASEPATH}/third_party/flatbuffers" + rm -rf build && mkdir -p build && cd build && cmake .. && make -j$THREAD_NUM + FLATC="${BASEPATH}"/third_party/flatbuffers/build/flatc + cd "${BASEPATH}"/predict/schema && mkdir -p "${BASEPATH}"/predict/schema/inner + find . -name "*.fbs" -print0 | xargs -0 "${FLATC}" -c -b + find . -name "*.fbs" -print0 | xargs -0 "${FLATC}" -c -b --reflect-types --gen-mutable --reflect-names --gen-object-api -o ${BASEPATH}/predict/schema/inner + + # check LLVM_PATH + if [ "${LLVM_PATH}" == "" ]; then + echo "Please set LLVM_PATH in env for example export LLVM_PATH=/xxxx/bin/llvm-config" + exit + fi + + #build tvm + tvm_open_source="${BASEPATH}/third_party/incubator-tvm" + tvm_kernel_build="${BASEPATH}/predict/module/tvm_kernel" + if [ ! -f "${tvm_kernel_build}"/incubator-tvm/build/libtvm.so ]; then + rm -fr "${tvm_kernel_build}"/incubator-tvm + cp -fr "${tvm_open_source}" "${tvm_kernel_build}" + mkdir -p "${tvm_kernel_build}"/incubator-tvm/build + patch -d "${tvm_kernel_build}"/incubator-tvm -p1 < "${BASEPATH}"/third_party/patch/predict/0001-RetBugFix-CustomRuntime_v06.patch + cp "${tvm_kernel_build}"/lite/src/codegen/llvm/lite_rtfunc_reset.cc "${tvm_kernel_build}"/incubator-tvm/src/codegen/llvm/ + cp "${tvm_open_source}"/cmake/config.cmake "${tvm_kernel_build}"/incubator-tvm + if [ "${LLVM_PATH}" ]; then + sed -i "s#set(USE_LLVM .*)#set(USE_LLVM \"${LLVM_PATH}\")#g" "${tvm_kernel_build}"/incubator-tvm/config.cmake + else + echo "need set LLVM_PATH in env for example export LLVM_PATH=/xxxx/bin/llvm-config" + fi + cd "${tvm_kernel_build}"/incubator-tvm/build + cmake .. + make -j$THREAD_NUM + else + cd "${tvm_kernel_build}"/incubator-tvm/build + make -j$THREAD_NUM + fi + + #gen op + predict_tvm_op_lib_path="${BASEPATH}/predict/module/tvm_kernel/build/lib_x86" + predict_platform="x86" + if [[ "$PREDICT_PLATFORM" == "arm64" ]]; then + predict_tvm_op_lib_path="${BASEPATH}/predict/module/tvm_kernel/build/lib_arm64" + predict_platform="arm64" + fi + + need_get_libs=true + if [ -d "${predict_tvm_op_lib_path}" ]; then + file_list=$(ls "${predict_tvm_op_lib_path}") + if [ -n "${file_list}" ]; then + libstime=$(stat -c %Y "${predict_tvm_op_lib_path}"/* | sort -u | tail -n1) + pythontime=$(find "${BASEPATH}"/predict/module/tvm_kernel/lite/python/ -name "*.py" -exec stat -c %Y {} \; | + sort -u | tail -n1) + if [ "${libstime}" -ge "${pythontime}" ]; then + need_get_libs=false + else + rm -fr "${predict_tvm_op_lib_path}" + fi + fi + fi + + if $need_get_libs; then + PYTHONPATH_OLD=${PYTHONPATH} + export PYTHONPATH="${tvm_kernel_build}/incubator-tvm/python:${tvm_kernel_build}/incubator-tvm/topi/python:${tvm_kernel_build}/incubator-tvm/nnvm/python:${tvm_kernel_build}/lite/python:" + cd "${BASEPATH}"/predict/module/tvm_kernel/lite/python/at_ops + python3 at_gen_strip.py ${predict_platform} + export PYTHONPATH=${PYTHONPATH_OLD} + fi + + cd "${BASEPATH}/predict/build" + if [[ "$PREDICT_PLATFORM" == "arm64" ]]; then + cmake -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \ + -DANDROID_NATIVE_API_LEVEL=android-19 -DANDROID_NDK="${ANDROID_NDK}" \ + -DANDROID_TOOLCHAIN_NAME="aarch64-linux-android-clang" -DANDROID_STL="c++_shared" \ + -DANDROID_ABI="arm64-v8a" -DENABLE_PREDICT_ARM64=ON -DANDROID_ALLOW_UNDEFINED_SYMBOLS=TRUE .. + elif [[ "$PREDICT_PLATFORM" == "x86_64" ]]; then + cmake .. + fi + + make ${VERBOSE} -j$THREAD_NUM + if [[ "$PREDICT_PLATFORM" == "x86_64" ]]; then + cd "${BASEPATH}/predict/build/test" && ./run_tests.sh + fi + + # copy securec include files + mkdir -p "${BASEPATH}/predict/output/include/securec/include" + cp "${BASEPATH}"/third_party/securec/include/* "${BASEPATH}"/predict/output/include/securec/include + + cd "${BASEPATH}/predict/output/" + if [[ "$PREDICT_PLATFORM" == "x86_64" ]]; then + tar -cf MSPredict-0.1.0-linux_x86_64.tar.gz include/ lib/ --warning=no-file-changed + elif [[ "$PREDICT_PLATFORM" == "arm64" ]]; then + tar -cf MSPredict-0.1.0-linux_aarch64.tar.gz include/ lib/ --warning=no-file-changed + fi + echo "success to build predict project!" +} + +if [[ "X$COMPILE_PREDICT" = "Xon" ]]; then + build_predict + echo "---------------- mindspore: build end ----------------" + exit +else + build_mindspore +fi + +if [[ "X$INC_BUILD" = "Xoff" ]]; then + if [[ "X$ENABLE_GE" = "Xon" ]]; then + bash "${PROJECT_PATH}/package.sh" ge + elif [[ "X$ENABLE_GPU" = "Xon" ]]; then + bash "${PROJECT_PATH}/package.sh" ms gpu + elif [[ "X$ENABLE_D" = "Xon" ]] || [[ "X$ENABLE_CPU" = "Xon" ]]; then + bash "${PROJECT_PATH}/package.sh" ms + else + bash "${PROJECT_PATH}/package.sh" debug + fi +fi + +cp -rf ${BUILD_PATH}/package/mindspore/lib ${BUILD_PATH}/../mindspore +cp -rf ${BUILD_PATH}/package/mindspore/*.so ${BUILD_PATH}/../mindspore + +if [[ -d "${BUILD_PATH}/package/build" ]]; then + rm -rf "${BUILD_PATH}/package/build" +fi +echo "---------------- mindspore: build end ----------------" diff --git a/cmake/dependency_graphengine.cmake b/cmake/dependency_graphengine.cmake new file mode 100644 index 00000000000..dfe90de8361 --- /dev/null +++ b/cmake/dependency_graphengine.cmake @@ -0,0 +1,70 @@ +message(STATUS "ENABLE_GE set to FALSE, compiling GraphEngine") +set(GE_SOURCE_DIR ${CMAKE_SOURCE_DIR}/graphengine) + +message(STATUS "ge dir: ${GE_SOURCE_DIR}") +# download json headers, rather than whole repository +include(${GE_SOURCE_DIR}/cmake/ge_utils.cmake) +include(${GE_SOURCE_DIR}/cmake/external_libs/json.cmake) +include(${GE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) +include(${GE_SOURCE_DIR}/cmake/external_libs/gtest.cmake) +include(${GE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) +include(${GE_SOURCE_DIR}/cmake/external_libs/onnx.cmake) + +# for CPU/GPU mode, find c_sec and slog from local prebuild +if (NOT ENABLE_D) + set(GE_PREBUILD_PATH ${GE_SOURCE_DIR}/third_party/prebuild/${CMAKE_HOST_SYSTEM_PROCESSOR}) + find_library(c_sec libc_sec.so ${GE_PREBUILD_PATH}) + find_library(slog libslog.so ${GE_PREBUILD_PATH}) +elseif (DEFINED ENV{D_LINK_PATH}) + set(GE_LIB_PATH $ENV{D_LINK_PATH}) + set(GE_SYS_ARCH "") + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64") + # x86 ubuntu + set(GE_SYS_ARCH "x86_64") + elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + # arm euleros + set(GE_SYS_ARCH "aarch64") + else() + message(FATAL_ERROR "Running on a unsupported architecture: ${SYSTEM_TYPE}, build terminated") + endif() + set(GE_LIB_PATH ${GE_LIB_PATH}/${GE_SYS_ARCH}) + find_library(c_sec libc_sec.so ${GE_LIB_PATH}) + find_library(slog libslog.so ${GE_LIB_PATH}) + find_library(mmpa libmmpa.so ${GE_LIB_PATH}) + find_library(runtime libruntime.so ${GE_LIB_PATH}) + find_library(msprof libmsprof.so ${GE_LIB_PATH}) + find_library(register libregister.so ${GE_LIB_PATH}) + find_library(hccl libhccl.so ${GE_LIB_PATH}) + find_library(cce libcce.so ${GE_LIB_PATH}) + find_library(resource libresource.so ${GE_LIB_PATH}) +else() + set(HIAI_INSTALLED_DIR /usr/local/HiAI) + set(HIAI_DRIVER_DIR ${HIAI_INSTALLED_DIR}/driver/lib64) + set(HIAI_RUNTIME_DIR ${HIAI_INSTALLED_DIR}/runtime/lib64) + find_library(c_sec libc_sec.so ${HIAI_DRIVER_DIR}) + find_library(slog libslog.so ${HIAI_DRIVER_DIR}) + find_library(mmpa libmmpa.so ${HIAI_DRIVER_DIR}) + + find_library(cce libcce.so ${HIAI_RUNTIME_DIR}) + find_library(hccl libhccl.so ${HIAI_RUNTIME_DIR}) + find_library(runtime libruntime.so ${HIAI_RUNTIME_DIR}) + find_library(msprof libmsprof.so ${HIAI_RUNTIME_DIR}) + find_library(register libregister.so ${HIAI_RUNTIME_DIR}) + find_library(resource libresource.so ${HIAI_RUNTIME_DIR}) +endif() + +# compile libraries from following directories +# this cmake file is called only when NOT ENABLE_GE is set +set(_ge_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + +string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +# force __FILE__ to show relative path of file, from source directory +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILE__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"' -Wno-builtin-macro-redefined") +add_subdirectory(${GE_SOURCE_DIR}/src/common/graph) +if(ENABLE_D) + add_subdirectory(${GE_SOURCE_DIR}/src/ge/common) + add_subdirectory(${GE_SOURCE_DIR}/src/ge/ge_runtime) +endif() + +set(CMAKE_CXX_FLAGS ${_ge_tmp_CMAKE_CXX_FLAGS}) diff --git a/cmake/dependency_gtest.cmake b/cmake/dependency_gtest.cmake new file mode 100644 index 00000000000..af4d288c6a6 --- /dev/null +++ b/cmake/dependency_gtest.cmake @@ -0,0 +1,36 @@ +# googletest library +# +# +# GTest_LIBRARY +# + +if (NOT TARGET gtest) + set(BUILD_TESTING OFF CACHE BOOL "Disable glog test") + + set(_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE ${CMAKE_POSITION_INDEPENDENT_CODE}) + set(_ms_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + set(_ms_tmp_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) + set(_ms_tmp_CMAKE_MACOSX_RPATH ${CMAKE_MACOSX_RPATH}) + + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + set(BUILD_SHARED_LIBS ON) + set(CMAKE_MACOSX_RPATH TRUE) + set(CMAKE_CXX_FLAGS "${SECURE_CXX_FLAGS}") + + if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.0" AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64" AND SYSTEM_TYPE MATCHES "euleros") + # -D_GLIBCXX_USE_CXX11_ABI=0 added for the ABI incompatible for libtsdclient.so + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + endif() + + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest ${CMAKE_BINARY_DIR}/googletest) + + set(CMAKE_POSITION_INDEPENDENT_CODE ${_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE}) + set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS}) + set(BUILD_SHARED_LIBS ${_ms_tmp_BUILD_SHARED_LIBS}) + set(CMAKE_MACOSX_RPATH ${_ms_tmp_CMAKE_MACOSX_RPATH}) +endif() + +include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googletest/include) + +set(GTEST_LIBRARY gtest) + diff --git a/cmake/dependency_protobuf.cmake b/cmake/dependency_protobuf.cmake new file mode 100644 index 00000000000..21760b55728 --- /dev/null +++ b/cmake/dependency_protobuf.cmake @@ -0,0 +1,100 @@ +# Protobuf +# +# +# PROTOBUF_LIBRARY - Link this to use protobuf +# +if (NOT TARGET protobuf::libprotobuf) + set(protobuf_BUILD_TESTS OFF CACHE BOOL "Disable protobuf test") + set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "Gen shared library") + set(_ms_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + + string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/protobuf/cmake ${CMAKE_BINARY_DIR}/protobuf) + + set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS}) +endif () + +include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/protobuf/src) + +set(PROTOBUF_LIBRARY protobuf::libprotobuf) +set(PROTOC_EXECUTABLE $) + +function(ms_protobuf_generate c_var h_var) + if(NOT ARGN) + message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files") + return() + endif() + + set(${c_var}) + set(${h_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) + + + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + DEPENDS protobuf::protoc ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + +endfunction() + +function(ms_protobuf_generate_py c_var h_var py_var) + if(NOT ARGN) + message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files") + return() + endif() + + set(${c_var}) + set(${h_var}) + set(${py_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) + + + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") + list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" + DEPENDS protobuf::protoc ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + set(${py_var} ${${py_var}} PARENT_SCOPE) + +endfunction() diff --git a/cmake/dependency_securec.cmake b/cmake/dependency_securec.cmake new file mode 100644 index 00000000000..81714c21d4c --- /dev/null +++ b/cmake/dependency_securec.cmake @@ -0,0 +1,19 @@ +# securec library +# +# +# SECUREC_LIBRARY +# + +if (NOT TARGET securec) + set(_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE ${CMAKE_POSITION_INDEPENDENT_CODE}) + set(_ms_tmp_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + + set(CMAKE_C_FLAGS "${SECURE_CXX_FLAGS}") + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/securec ${CMAKE_BINARY_DIR}/securec) + set(CMAKE_POSITION_INDEPENDENT_CODE ${_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE}) + set(CMAKE_C_FLAGS ${_ms_tmp_CMAKE_C_FLAGS}) +endif() + +include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/securec/include) + +set(SECUREC_LIBRARY securec) diff --git a/cmake/dependency_utils.cmake b/cmake/dependency_utils.cmake new file mode 100644 index 00000000000..129365e8337 --- /dev/null +++ b/cmake/dependency_utils.cmake @@ -0,0 +1,25 @@ +# MS Utils +# + +function(find_python_package out_inc out_lib) + # Use PYTHON_EXECUTABLE if it is defined, otherwise default to python + if ("${PYTHON_EXECUTABLE}" STREQUAL "") + set(PYTHON_EXECUTABLE "python3") + else() + set(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") + endif() + + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())" + RESULT_VARIABLE result + OUTPUT_VARIABLE inc) + string(STRIP "${inc}" inc) + set(${out_inc} ${inc} PARENT_SCOPE) + + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c "import distutils.sysconfig as sysconfig; import os; print(os.path.join(sysconfig.get_config_var('LIBDIR'), sysconfig.get_config_var('LDLIBRARY')))" + RESULT_VARIABLE result + OUTPUT_VARIABLE lib) + string(STRIP "${lib}" lib) + set(${out_lib} ${lib} PARENT_SCOPE) +endfunction() diff --git a/cmake/external_libs/dlpack.cmake b/cmake/external_libs/dlpack.cmake new file mode 100644 index 00000000000..a2375c7d350 --- /dev/null +++ b/cmake/external_libs/dlpack.cmake @@ -0,0 +1,7 @@ +mindspore_add_pkg(dlpack + VER 0.2 + HEAD_ONLY ./ + URL https://github.com/dmlc/dlpack/archive/0acb731e0e43d15deee27b66f10e4c5b4e667913.zip + MD5 6b8093f17ad4e830d3c63eb3171c4b45) + + diff --git a/cmake/external_libs/dmlc_core.cmake b/cmake/external_libs/dmlc_core.cmake new file mode 100644 index 00000000000..386a52429d4 --- /dev/null +++ b/cmake/external_libs/dmlc_core.cmake @@ -0,0 +1,7 @@ +mindspore_add_pkg(dmlc_core + VER 0.3 + HEAD_ONLY ./ + URL https://github.com/dmlc/dmlc-core/archive/808f485387f9a03f78fa9f1159f387d0d91b7a28.zip + MD5 ea36f94c57752bf40fb02dfc362f1ed9) + + diff --git a/cmake/external_libs/eigen.cmake b/cmake/external_libs/eigen.cmake new file mode 100644 index 00000000000..8efee439073 --- /dev/null +++ b/cmake/external_libs/eigen.cmake @@ -0,0 +1,12 @@ +set(Eigen3_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(Eigen3_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(Eigen3 + VER 3.3.7 + URL https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz + MD5 9e30f67e8531477de4117506fe44669b + CMAKE_OPTION -DBUILD_TESTING=OFF) +find_package(Eigen3 3.3.7 REQUIRED ${MS_FIND_NO_DEFAULT_PATH}) +include_directories(${Eigen3_INC}) +include_directories(${EIGEN3_INCLUDE_DIR}) +set_property(TARGET Eigen3::Eigen PROPERTY IMPORTED_GLOBAL TRUE) +add_library(mindspore::eigen ALIAS Eigen3::Eigen) diff --git a/cmake/external_libs/flatbuffers.cmake b/cmake/external_libs/flatbuffers.cmake new file mode 100644 index 00000000000..7d7c74b9e16 --- /dev/null +++ b/cmake/external_libs/flatbuffers.cmake @@ -0,0 +1,53 @@ +set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(flatbuffers + VER 1.11.0 + LIBS flatbuffers + EXE flatc + URL https://github.com/google/flatbuffers/archive/v1.11.0.tar.gz + MD5 02c64880acb89dbd57eebacfd67200d8 + CMAKE_OPTION -DFLATBUFFERS_BUILD_TESTS=OFF ) + +include_directories(${flatbuffers_INC}) +add_library(mindspore::flatbuffers ALIAS flatbuffers::flatbuffers) +add_executable(mindspore::flatc ALIAS flatbuffers::flatc) +include_directories(${flatbuffers_INC}) +function(ms_build_flatbuffers source_schema_files + source_schema_dirs + custom_target_name + generated_output_dir) + + set(total_schema_dirs "") + set(total_generated_files "") + set(FLATC mindspore::flatc) + foreach (schema_dir ${source_schema_dirs}) + set(total_schema_dirs -I ${schema_dir} ${total_schema_dirs}) + endforeach() + + foreach(schema ${source_schema_files}) + get_filename_component(filename ${schema} NAME_WE) + if (NOT ${generated_output_dir} STREQUAL "") + set(generated_file ${generated_output_dir}/${filename}_generated.h) + add_custom_command( + OUTPUT ${generated_file} + COMMAND ${FLATC} --gen-mutable + --reflect-names --gen-object-api -o ${generated_output_dir} + ${total_schema_dirs} + -c -b --reflect-types ${schema} + DEPENDS ${FLATC} ${schema} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Running C++ flatbuffers compiler on ${schema}" VERBATIM) + list(APPEND total_generated_files ${generated_file}) + endif() + endforeach() + + add_custom_target(${custom_target_name} ALL + DEPENDS ${total_generated_files}) + + if (NOT ${generated_output_dir} STREQUAL "") + include_directories(${generated_output_dir}) + set_property(TARGET ${custom_target_name} + PROPERTY GENERATED_OUTPUT_DIR + ${generated_output_dir}) + endif() +endfunction() diff --git a/cmake/external_libs/glog.cmake b/cmake/external_libs/glog.cmake new file mode 100644 index 00000000000..d7942a4efd7 --- /dev/null +++ b/cmake/external_libs/glog.cmake @@ -0,0 +1,10 @@ +set(glog_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 ${SECURE_CXX_FLAGS}") +set(glog_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(glog + VER 0.4.0 + LIBS glog + URL https://github.com/google/glog/archive/v0.4.0.tar.gz + MD5 0daea8785e6df922d7887755c3d100d0 + CMAKE_OPTION -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON) +include_directories(${glog_INC}) +add_library(mindspore::glog ALIAS glog::glog) diff --git a/cmake/external_libs/gtest.cmake b/cmake/external_libs/gtest.cmake new file mode 100644 index 00000000000..5384b48825b --- /dev/null +++ b/cmake/external_libs/gtest.cmake @@ -0,0 +1,13 @@ +set(gtest_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(gtest_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(gtest + VER 1.8.0 + LIBS gtest + URL https://github.com/google/googletest/archive/release-1.8.0.tar.gz + MD5 16877098823401d1bf2ed7891d7dce36 + CMAKE_OPTION -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON + -DCMAKE_MACOSX_RPATH=TRUE -Dgtest_disable_pthreads=ON) +include_directories(${gtest_INC}) +add_library(mindspore::gtest ALIAS gtest::gtest) +file(COPY ${gtest_LIBPATH}/libgtest.so DESTINATION ${CMAKE_BINARY_DIR}/googletest/googlemock/gtest) +file(COPY ${gtest_LIBPATH}/libgtest_main.so DESTINATION ${CMAKE_BINARY_DIR}/googletest/googlemock/gtest) diff --git a/cmake/external_libs/jpeg_turbo.cmake b/cmake/external_libs/jpeg_turbo.cmake new file mode 100644 index 00000000000..84d6e3006c0 --- /dev/null +++ b/cmake/external_libs/jpeg_turbo.cmake @@ -0,0 +1,13 @@ + +set(jpeg_turbo_USE_STATIC_LIBS ON) +set(jpeg_turbo_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") +set(jpeg_turbo_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") +mindspore_add_pkg(jpeg_turbo + VER 2.0.4 + LIBS jpeg + URL https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.4.tar.gz + MD5 44c43e4a9fb352f47090804529317c88 + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=TRUE + ) +include_directories(${jpeg_turbo_INC}) +add_library(mindspore::jpeg_turbo ALIAS jpeg_turbo::jpeg) diff --git a/cmake/external_libs/json.cmake b/cmake/external_libs/json.cmake new file mode 100644 index 00000000000..1912e80d358 --- /dev/null +++ b/cmake/external_libs/json.cmake @@ -0,0 +1,9 @@ +set(nlohmann_json_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(nlohmann_json_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(nlohmann_json + VER 3.6.1 + HEAD_ONLY ./ + URL https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip + MD5 0dc903888211db3a0f170304cd9f3a89) +include_directories(${nlohmann_json_INC}) +add_library(mindspore::json ALIAS nlohmann_json) \ No newline at end of file diff --git a/cmake/external_libs/libtiff.cmake b/cmake/external_libs/libtiff.cmake new file mode 100644 index 00000000000..461b9c44818 --- /dev/null +++ b/cmake/external_libs/libtiff.cmake @@ -0,0 +1,16 @@ + +set(tiff_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -Wno-unused-result \ + -Wno-unused-but-set-variable -fPIC -D_FORTIFY_SOURCE=2 -O2") +set(tiff_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -Wno-unused-result \ + -Wno-unused-but-set-variable -fPIC -D_FORTIFY_SOURCE=2 -O2") +set(tiff_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + +mindspore_add_pkg(tiff + VER 4.1.0 + LIBS tiff + URL https://gitlab.com/libtiff/libtiff/-/archive/v4.1.0/libtiff-v4.1.0.tar.gz + MD5 21de8d35c1b21ac82663fa9f56d3350d + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -Djbig=OFF -Dlzma=OFF -Djpeg12=OFF -Dzstd=OFF -Dpixarlog=OFF + -Dold-jpeg=OFF -Dwebp=OFF -DBUILD_SHARED_LIBS=OFF) +message("tiff include = ${tiff_INC}") +message("tiff lib = ${tiff_LIB}") diff --git a/cmake/external_libs/mkl_dnn.cmake b/cmake/external_libs/mkl_dnn.cmake new file mode 100644 index 00000000000..17d8020d3a2 --- /dev/null +++ b/cmake/external_libs/mkl_dnn.cmake @@ -0,0 +1,11 @@ +set(mkl_dnn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(mkl_dnn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(mkl_dnn + VER 1.1.1 + LIBS dnnl mkldnn + URL https://github.com/intel/mkl-dnn/archive/v1.1.1.tar.gz + MD5 d6a422b00459600bdc22242590953f38 + CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF) +include_directories(${mkl_dnn_INC}) +add_library(mindspore::dnnl ALIAS mkl_dnn::dnnl) +add_library(mindspore::mkldnn ALIAS mkl_dnn::mkldnn) diff --git a/cmake/external_libs/nccl.cmake b/cmake/external_libs/nccl.cmake new file mode 100644 index 00000000000..b74946f7fb9 --- /dev/null +++ b/cmake/external_libs/nccl.cmake @@ -0,0 +1,12 @@ + +set(nccl_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(nccl + VER 2.4.8-1 + LIBS nccl + URL https://github.com/NVIDIA/nccl/archive/v2.4.8-1.tar.gz + MD5 f14b37d6af1c79db5f57cb029a753727 + BUILD_OPTION src.build NVCC_GENCODE="-gencode=arch=compute_70,code=sm_70" + INSTALL_INCS build/include/* + INSTALL_LIBS build/lib/*) +include_directories(${nccl_INC}) +add_library(mindspore::nccl ALIAS nccl::nccl) \ No newline at end of file diff --git a/cmake/external_libs/ompi.cmake b/cmake/external_libs/ompi.cmake new file mode 100644 index 00000000000..b220ca54e93 --- /dev/null +++ b/cmake/external_libs/ompi.cmake @@ -0,0 +1,11 @@ + +set(ompi_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(ompi + VER 3.1.5 + LIBS mpi + URL https://github.com/open-mpi/ompi/archive/v3.1.5.tar.gz + MD5 f7f220b26532c11a2efbc0bb73af3282 + PRE_CONFIGURE_COMMAND ./autogen.pl + CONFIGURE_COMMAND ./configure) +include_directories(${ompi_INC}) +add_library(mindspore::ompi ALIAS ompi::mpi) \ No newline at end of file diff --git a/cmake/external_libs/onnx.cmake b/cmake/external_libs/onnx.cmake new file mode 100644 index 00000000000..fc77100866e --- /dev/null +++ b/cmake/external_libs/onnx.cmake @@ -0,0 +1,5 @@ +mindspore_add_pkg(ms_onnx + VER 1.6.0 + HEAD_ONLY ./ + URL https://github.com/onnx/onnx/releases/download/v1.6.0/onnx-1.6.0.tar.gz + MD5 512f2779d6215d4a36f366b6b9acdf1e) diff --git a/cmake/external_libs/opencv.cmake b/cmake/external_libs/opencv.cmake new file mode 100644 index 00000000000..e67c3f232f8 --- /dev/null +++ b/cmake/external_libs/opencv.cmake @@ -0,0 +1,31 @@ + +set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") +set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") +set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + +mindspore_add_pkg(opencv + VER 4.2.0 + LIBS opencv_core opencv_imgcodecs opencv_imgproc + URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz + MD5 e8cb208ce2723481408b604b480183b6 + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF + -DBUILD_ZLIB=ON + -DBUILD_JPEG=ON + -DBUILD_PNG=ON + -DBUILD_OPENEXR=ON + -DBUILD_TESTS=OFF + -DBUILD_PERF_TESTS=OFF + -DBUILD_opencv_apps=OFF + -DCMAKE_SKIP_RPATH=TRUE + -DBUILD_opencv_python3=OFF + -DWITH_FFMPEG=OFF + -DWITH_TIFF=ON + -DBUILD_TIFF=OFF + -DWITH_JASPER=OFF + -DBUILD_JASPER=OFF + -DTIFF_INCLUDE_DIR=${tiff_INC} + -DTIFF_LIBRARY=${tiff_LIB}) +include_directories(${opencv_INC}/opencv4) +add_library(mindspore::opencv_core ALIAS opencv::opencv_core) +add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs) +add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc) diff --git a/cmake/external_libs/protobuf.cmake b/cmake/external_libs/protobuf.cmake new file mode 100644 index 00000000000..c354bcb65d4 --- /dev/null +++ b/cmake/external_libs/protobuf.cmake @@ -0,0 +1,96 @@ +mindspore_add_pkg(protobuf + VER 3.8.0 + HEAD_ONLY ./ + URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz + MD5 3d9e32700639618a4d2d342c99d4507a) + +set(protobuf_BUILD_TESTS OFF CACHE BOOL "Disable protobuf test") +set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "Gen shared library") +set(_ms_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + +string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +add_subdirectory(${protobuf_DIRPATH}/cmake ${protobuf_DIRPATH}/build) + +set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS}) + +set(PROTOBUF_LIBRARY protobuf::libprotobuf) +include_directories(${protobuf_DIRPATH}/src) +add_library(mindspore::protobuf ALIAS libprotobuf) + +function(ms_protobuf_generate c_var h_var) + if(NOT ARGN) + message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files") + return() + endif() + + set(${c_var}) + set(${h_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) + + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + DEPENDS protobuf::protoc ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + +endfunction() + +function(ms_protobuf_generate_py c_var h_var py_var) + if(NOT ARGN) + message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files") + return() + endif() + + set(${c_var}) + set(${h_var}) + set(${py_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) + + + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") + list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" + "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} + COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" + COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" + DEPENDS protobuf::protoc ${abs_file} + COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + set(${py_var} ${${py_var}} PARENT_SCOPE) + +endfunction() diff --git a/cmake/external_libs/pybind11.cmake b/cmake/external_libs/pybind11.cmake new file mode 100644 index 00000000000..e818c192e6e --- /dev/null +++ b/cmake/external_libs/pybind11.cmake @@ -0,0 +1,12 @@ +set(pybind11_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(pybind11_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(pybind11 + VER 2.4.3 + URL https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz + MD5 62254c40f89925bb894be421fe4cdef2 + CMAKE_OPTION -DPYBIND11_TEST=OFF -DPYBIND11_LTO_CXX_FLAGS=FALSE + ) +include_directories(${pybind11_INC}) +find_package(pybind11 REQUIRED) +set_property(TARGET pybind11::module PROPERTY IMPORTED_GLOBAL TRUE) +add_library(mindspore::pybind11_module ALIAS pybind11::module) diff --git a/cmake/external_libs/rang.cmake b/cmake/external_libs/rang.cmake new file mode 100644 index 00000000000..45ea375cb56 --- /dev/null +++ b/cmake/external_libs/rang.cmake @@ -0,0 +1,7 @@ +mindspore_add_pkg(rang + VER 3.1.0 + HEAD_ONLY ./ + URL https://github.com/agauniyal/rang/archive/cabe04d6d6b05356fa8f9741704924788f0dd762.zip + MD5 0c5c9b251fea9ee7ce32f188655be0ea) + + diff --git a/cmake/external_libs/sqlite.cmake b/cmake/external_libs/sqlite.cmake new file mode 100644 index 00000000000..35e48b2d0ec --- /dev/null +++ b/cmake/external_libs/sqlite.cmake @@ -0,0 +1,15 @@ + +set(sqlite_USE_STATIC_LIBS ON) +set(sqlite_CXXFLAGS) +set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") +set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + +mindspore_add_pkg(sqlite + VER 3.31.1 + LIBS sqlite3 + URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz + MD5 5f4e7b4016c15f4fb5855615279819da + PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001 + CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1) +include_directories(${sqlite_INC}) +add_library(mindspore::sqlite ALIAS sqlite::sqlite3) \ No newline at end of file diff --git a/cmake/external_libs/tvm_gpu.cmake b/cmake/external_libs/tvm_gpu.cmake new file mode 100644 index 00000000000..57a045cb035 --- /dev/null +++ b/cmake/external_libs/tvm_gpu.cmake @@ -0,0 +1,8 @@ +set(incubator_tvm_gpu_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(incubator_tvm_gpu_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(incubator_tvm_gpu + VER 0.6.0 + HEAD_ONLY ./ + URL https://github.com/apache/incubator-tvm/archive/v0.6.0.tar.gz + MD5 9cbbd32545a776023acabbba270449fe) + diff --git a/cmake/external_libs/tvm_predict.cmake b/cmake/external_libs/tvm_predict.cmake new file mode 100644 index 00000000000..9f07b09e2b4 --- /dev/null +++ b/cmake/external_libs/tvm_predict.cmake @@ -0,0 +1,10 @@ +set(incubator_tvm_predict_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") +set(incubator_tvm_predict_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") +mindspore_add_pkg(incubator_tvm_predict + VER 0.6.0 + HEAD_ONLY ./ + URL https://github.com/apache/incubator-tvm/release/download/v0.6.0/apache-tvm-src-v0.6.0-incubating.tar.gz + MD5 2d77a005f0046d937b99c67de82f6438 + PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/predict/0001-RetBugFix-CustomRuntime_v06.patch) +include_directories(${incubator_tvm_predict_INC}) +add_library(mindspore::incubator_tvm_predict ALIAS incubator_tvm_predict) diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake new file mode 100644 index 00000000000..103775a4c24 --- /dev/null +++ b/cmake/mind_expression.cmake @@ -0,0 +1,59 @@ +set(SECURE_CXX_FLAGS "") +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") +endif() +set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS}) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") + +include(cmake/utils.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/json.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/dependency_securec.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pybind11.cmake) +MESSAGE("go to link flatbuffers") +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/flatbuffers.cmake) +if(USE_GLOG) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake) +endif() + +find_package(Python3) +include_directories(${Python3_INCLUDE_DIRS}) +include_directories(${CMAKE_SOURCE_DIR}/third_party) +if (ENABLE_CPU) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/mkl_dnn.cmake) +endif() + +if (ENABLE_GPU) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/dlpack.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/dmlc_core.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/rang.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/tvm_gpu.cmake) +endif() + +if (ENABLE_MPI) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/nccl.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/ompi.cmake) +endif() + +if (ENABLE_GE) + include_directories(${CMAKE_SOURCE_DIR}/third_party/ge/include) + include_directories(${CMAKE_SOURCE_DIR}/third_party/ge/include/external) + include_directories(${CMAKE_SOURCE_DIR}/third_party/ge/include/external/graph) +else() + include_directories(${CMAKE_SOURCE_DIR}/graphengine/inc) + include_directories(${CMAKE_SOURCE_DIR}/graphengine/inc/ops) + include_directories(${CMAKE_SOURCE_DIR}/graphengine/inc/external) + include_directories(${CMAKE_SOURCE_DIR}/graphengine/inc/external/graph) +endif() + +if (ENABLE_MINDDATA) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/jpeg_turbo.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/libtiff.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/opencv.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/sqlite.cmake) +endif() + +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/gtest.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/external_libs/onnx.cmake) +set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS_F}) diff --git a/cmake/options.cmake b/cmake/options.cmake new file mode 100644 index 00000000000..6ec577f3120 --- /dev/null +++ b/cmake/options.cmake @@ -0,0 +1,109 @@ +option(ENABLE_D "Enable d" OFF) +option(ENABLE_GPU "Enable gpu" OFF) +option(ENABLE_CPU "Enable cpu" OFF) +option(ENABLE_GE "Enable graph engine as backend to execute" OFF) +option(ENABLE_MINDDATA "Enable minddata compile" OFF) +option(ENABLE_TRAIN "Enable ge train, default off(only infer)" OFF) +option(ENABLE_TESTCASES "Run testcases switch, default off" OFF) +option(DEBUG_MODE "Debug mode, default off" OFF) +option(ENABLE_ASAN "Enable Google Sanitizer to find memory bugs") +option(ENABLE_LOAD_ANF_IR "Enable load ANF-IR as input of 'infer' stage of pipeline" OFF) +option(ENABLE_COVERAGE "Enable code coverage report" OFF) +option(USE_GLOG "Use glog to output log" OFF) +option(ENABLE_PROFILE "Enable pipeline profile, default off" OFF) +option(ENABLE_TIMELINE "Enable time line record" OFF) +option(ENABLE_DUMP_PROTO "Enable dump anf graph to file in ProtoBuffer format, default on" ON) +option(ENABLE_DUMP_E2E "Enable dump e2e file, default on" OFF) +option(ENABLE_DUMP_IR "Enable dump funciton graph ir, default on" ON) +option(ENABLE_MPI "enable mpi" OFF) +option(ENABLE_AKG "enable akg" OFF) + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") +endif() + +if (CMAKE_SYSTEM_NAME MATCHES "Darwin") + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -Wsign-compare") +endif() + +if (ENABLE_COVERAGE) + set(COVERAGE_COMPILER_FLAGS "-g --coverage -fprofile-arcs -ftest-coverage") + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}") +endif() + +if (ENABLE_ASAN) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -static-libasan -fsanitize=undefined") + else() + set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -static-libsan -fsanitize=undefined") + endif() +endif() + +if (DEBUG_MODE) + set(CMAKE_BUILD_TYPE "Debug") +else() + add_compile_definitions(MEM_REUSE_DEBUG) + set(CMAKE_BUILD_TYPE "Release") +endif() + +if ((CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") OR (CMAKE_BUILD_TYPE STREQUAL Release)) + set(PYBIND11_LTO_CXX_FLAGS FALSE) +endif() + +if (NOT BUILD_PATH) + set(BUILD_PATH "${CMAKE_SOURCE_DIR}/build") +endif() + +if (ENABLE_GE OR ENABLE_D) + set(ENABLE_TDTQUE ON) +endif() + +if (ENABLE_GPU) + set(ENABLE_GPUQUE ON) +endif() + +if (ENABLE_GE) + add_compile_definitions(ENABLE_GE) + add_compile_definitions(CUSTOM_OP) +endif() + +if(ENABLE_TRAIN) + add_compile_definitions(ENABLE_TRAIN=1) +else() + add_compile_definitions(ENABLE_TRAIN=0) +endif() + +if(USE_GLOG) + add_compile_definitions(USE_GLOG) +endif() + +if (ENABLE_PROFILE) + add_compile_definitions(ENABLE_PROFILE) +endif() + +if (ENABLE_TIMELINE) + add_compile_definitions(ENABLE_TIMELINE) +endif() + +if (ENABLE_LOAD_ANF_IR) + add_compile_definitions(ENABLE_LOAD_ANF_IR) +endif() + +if (ENABLE_TESTCASES OR (NOT ENABLE_D AND NOT ENABLE_GE)) + add_compile_definitions(NO_DLIB=1) +endif() + +if(ENABLE_DUMP_IR) + add_compile_definitions(ENABLE_DUMP_IR) +endif(ENABLE_DUMP_IR) + +if(ENABLE_MINDDATA) + add_compile_definitions(ENABLE_MINDDATA) + if (ENABLE_TDTQUE) + add_compile_definitions(ENABLE_TDTQUE) + endif() +endif() + +if(ENABLE_DUMP_E2E) + add_compile_definitions(ENABLE_DUMP_E2E) +endif() \ No newline at end of file diff --git a/cmake/utils.cmake b/cmake/utils.cmake new file mode 100644 index 00000000000..4efaad61b94 --- /dev/null +++ b/cmake/utils.cmake @@ -0,0 +1,349 @@ +include(FetchContent) +set(FETCHCONTENT_QUIET OFF) + +function(mindspore_add_submodule_obj des_submodule_objs sub_dir submodule_name_obj) + + add_subdirectory(${sub_dir}) + + if(NOT TARGET ${submodule_name_obj}) + message(FATAL_ERROR "Can not find submodule '${submodule_name_obj}'. in ${CMAKE_CURRENT_LIST_FILE}") + endif() + if("$" IN_LIST ${des_submodule_objs}) + message(FATAL_ERROR "submodule '${submodule_name_obj}' added more than once. in ${CMAKE_CURRENT_LIST_FILE}") + endif() + + set(${des_submodule_objs} ${${des_submodule_objs}} $ PARENT_SCOPE) + +endfunction() + +get_filename_component(_MS_LIB_CACHE ~/.mslib REALPATH) +if (NOT EXISTS ${_MS_LIB_CACHE}) + file(MAKE_DIRECTORY ${_MS_LIB_CACHE}) +endif () +# set(FETCHCONTENT_BASE_DIR ${_MS_LIB_CACHE}) +# set(CMAKE_PREFIX_PATH ${_MS_LIB_CACHE}) +if (DEFINED ENV{MSLIBS_SERVER}) + set(LOCAL_LIBS_SERVER $ENV{MSLIBS_SERVER}) + message("LOCAL_LIBS_SERVER: ${LOCAL_LIBS_SERVER}") +endif () +if(LOCAL_LIBS_SERVER) + if (NOT ENV{no_proxy}) + set(ENV{no_proxy} "${LOCAL_LIBS_SERVER}") + else() + string(FIND $ENV{no_proxy} ${LOCAL_LIBS_SERVER} IP_POS) + if (${IP_POS} EQUAL -1) + set(ENV{no_proxy} "$ENV{no_proxy},${LOCAL_LIBS_SERVER}") + endif () + endif () +endif() + +function(__download_pkg pkg_name pkg_url pkg_md5) + + if(LOCAL_LIBS_SERVER) + get_filename_component(_URL_FILE_NAME ${pkg_url} NAME) + set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${_URL_FILE_NAME}" ${pkg_url}) + endif() + + FetchContent_Declare( + ${pkg_name} + URL ${pkg_url} + URL_HASH MD5=${pkg_md5} + ) + FetchContent_GetProperties(${pkg_name}) + message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}") + if(NOT ${pkg_name}_POPULATED) + FetchContent_Populate(${pkg_name}) + set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE) + endif() + +endfunction() + +function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5) + + if(LOCAL_LIBS_SERVER) + set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${pkg_git_commit}") + FetchContent_Declare( + ${pkg_name} + URL ${pkg_url} + URL_HASH MD5=${pkg_md5} + ) + else() + FetchContent_Declare( + ${pkg_name} + GIT_REPOSITORY ${pkg_url} + GIT_TAG ${pkg_git_commit}) + endif() + FetchContent_GetProperties(${pkg_name}) + message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}") + if(NOT ${pkg_name}_POPULATED) + FetchContent_Populate(${pkg_name}) + set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE) + endif() + +endfunction() + + +function(__find_pkg_then_add_target pkg_name pkg_exe) + + unset(${pkg_name}_LIBS) + + message("_FIND:${${pkg_name}_BASE_DIR}") + + if(pkg_exe) + find_program(${pkg_exe}_EXE ${pkg_exe} PATHS ${${pkg_name}_BASE_DIR}/bin NO_DEFAULT_PATH) + if(NOT ${pkg_exe}_EXE) + return() + endif() + add_executable(${pkg_name}::${pkg_exe} IMPORTED GLOBAL) + set_target_properties(${pkg_name}::${pkg_exe} PROPERTIES + IMPORTED_LOCATION ${${pkg_exe}_EXE} + ) + message("found ${${pkg_exe}_EXE}") + endif() + + foreach(_LIB_NAME ${ARGN}) + set(_LIB_SEARCH_NAME ${_LIB_NAME}) + set(_LIB_TYPE SHARED) + if (${pkg_name}_USE_STATIC_LIBS) + set(_LIB_SEARCH_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(_LIB_TYPE STATIC) + endif () + set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND) + find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH) + if(NOT ${_LIB_NAME}_LIB) + return() + endif() + add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL) + set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include" + IMPORTED_LOCATION ${${_LIB_NAME}_LIB} + ) + list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME}) + message("found ${${_LIB_NAME}_LIB}") + STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB}) + set(${pkg_name}_LIBPATH ${LIBPATH} CACHE STRING INTERNAL) + endforeach(_LIB_NAME) + + set(${pkg_name}_LIBS ${${pkg_name}_LIBS} PARENT_SCOPE) +endfunction() + +function(__exec_cmd) + set(options ) + set(oneValueArgs WORKING_DIRECTORY) + set(multiValueArgs COMMAND) + + cmake_parse_arguments(EXEC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) + + execute_process(COMMAND ${EXEC_COMMAND} + WORKING_DIRECTORY ${EXEC_WORKING_DIRECTORY} + RESULT_VARIABLE RESULT) + if(NOT RESULT EQUAL "0") + message(FATAL_ERROR "error! when ${EXEC_COMMAND} in ${EXEC_WORKING_DIRECTORY}") + endif() +endfunction() + +function(__check_patches pkg_patches) + # check patches + if (PKG_PATCHES) + file(TOUCH ${_MS_LIB_CACHE}/${pkg_name}_patch.md5) + file(READ ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${pkg_name}_PATCHES_MD5) + + message("patches md5:${${pkg_name}_PATCHES_MD5}") + + set(${pkg_name}_PATCHES_NEW_MD5 ) + foreach(_PATCH ${PKG_PATCHES}) + file(MD5 ${_PATCH} _PF_MD5) + set(${pkg_name}_PATCHES_NEW_MD5 "${${pkg_name}_PATCHES_NEW_MD5},${_PF_MD5}") + endforeach(_PATCH) + + if (NOT ${pkg_name}_PATCHES_MD5 STREQUAL ${pkg_name}_PATCHES_NEW_MD5) + set(${pkg_name}_PATCHES ${PKG_PATCHES}) + file(REMOVE_RECURSE "${_MS_LIB_CACHE}/${pkg_name}-subbuild") + file(WRITE ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${${pkg_name}_PATCHES_NEW_MD5}) + message("patches changed : ${${pkg_name}_PATCHES_NEW_MD5}") + endif () + endif () +endfunction() + +set(MS_FIND_NO_DEFAULT_PATH NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_BUILDS_PATH NO_CMAKE_PACKAGE_REGISTRY NO_CMAKE_SYSTEM_PATH + NO_CMAKE_SYSTEM_PACKAGE_REGISTRY) +set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE) +function(mindspore_add_pkg pkg_name ) + + set(options ) + set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY) + set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) + + set(__FIND_PKG_NAME ${pkg_name}) + string(TOLOWER ${pkg_name} pkg_name) + message("pkg name:${__FIND_PKG_NAME},${pkg_name}") + + set(${pkg_name}_PATCHES_HASH ) + foreach(_PATCH ${PKG_PATCHES}) + file(MD5 ${_PATCH} _PF_MD5) + set(${pkg_name}_PATCHES_HASH "${${pkg_name}_PATCHES_HASH},${_PF_MD5}") + endforeach(_PATCH) + + # check options + set(${pkg_name}_CONFIG_TXT + "${CMAKE_CXX_COMPILER_VERSION}-${CMAKE_C_COMPILER_VERSION} + ${ARGN} - ${${pkg_name}_USE_STATIC_LIBS}- ${${pkg_name}_PATCHES_HASH} + ${${pkg_name}_CXXFLAGS}--${${pkg_name}_CFLAGS}--${${pkg_name}_LDFLAGS}") + string(REPLACE ";" "-" ${pkg_name}_CONFIG_TXT ${${pkg_name}_CONFIG_TXT}) + string(MD5 ${pkg_name}_CONFIG_HASH ${${pkg_name}_CONFIG_TXT}) + + message("${pkg_name} config hash: ${${pkg_name}_CONFIG_HASH}") + + set(${pkg_name}_BASE_DIR ${_MS_LIB_CACHE}/${pkg_name}_${${pkg_name}_CONFIG_HASH}) + set(${pkg_name}_DIRPATH ${${pkg_name}_BASE_DIR} CACHE STRING INTERNAL) + + if(EXISTS ${${pkg_name}_BASE_DIR}/options.txt AND PKG_HEAD_ONLY) + set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) + add_library(${pkg_name} INTERFACE) + target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) + return() + endif () + + if(NOT PKG_EXE) + set(PKG_EXE 0) + endif() + + set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR}) + set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE) + + if (PKG_LIBS) + __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) + if(${pkg_name}_LIBS) + set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) + message("Found libs: ${${pkg_name}_LIBS}") + return() + endif() + elseif(NOT PKG_HEAD_ONLY) + find_package(${__FIND_PKG_NAME} ${PKG_VER} ${MS_FIND_NO_DEFAULT_PATH}) + if (${__FIND_PKG_NAME}_FOUND) + set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) + message("Found pkg: ${__FIND_PKG_NAME}") + return() + endif () + endif () + + if (NOT PKG_DIR) + if (PKG_GIT_REPOSITORY) + __download_pkg_with_git(${pkg_name} ${PKG_GIT_REPOSITORY} ${PKG_GIT_TAG} ${PKG_MD5}) + else() + __download_pkg(${pkg_name} ${PKG_URL} ${PKG_MD5}) + endif() + else() + set(${pkg_name}_SOURCE_DIR ${PKG_DIR}) + endif () + file(WRITE ${${pkg_name}_BASE_DIR}/options.txt ${${pkg_name}_CONFIG_TXT}) + message("${pkg_name}_SOURCE_DIR : ${${pkg_name}_SOURCE_DIR}") + + foreach(_PATCH_FILE ${PKG_PATCHES}) + message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_PATCH_FILE}") + execute_process(COMMAND patch -p1 INPUT_FILE ${_PATCH_FILE} + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR} + RESULT_VARIABLE Result) + if(NOT Result EQUAL "0") + message(FATAL_ERROR "Failed patch: ${_PATCH_FILE}") + endif() + endforeach(_PATCH_FILE) + + file(LOCK ${${pkg_name}_BASE_DIR} DIRECTORY GUARD FUNCTION RESULT_VARIABLE ${pkg_name}_LOCK_RET TIMEOUT 600) + if(NOT ${pkg_name}_LOCK_RET EQUAL "0") + message(FATAL_ERROR "error! when try lock ${${pkg_name}_BASE_DIR} : ${${pkg_name}_LOCK_RET}") + endif() + + if(${pkg_name}_SOURCE_DIR) + if (PKG_HEAD_ONLY) + file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*) + file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR}) + set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) + add_library(${pkg_name} INTERFACE) + target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) + + elseif (PKG_CMAKE_OPTION) + # in cmake + file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build) + if (${pkg_name}_CFLAGS) + set(${pkg_name}_CMAKE_CFLAGS "-DCMAKE_C_FLAGS=${${pkg_name}_CFLAGS}") + endif () + if (${pkg_name}_CXXFLAGS) + set(${pkg_name}_CMAKE_CXXFLAGS "-DCMAKE_CXX_FLAGS=${${pkg_name}_CXXFLAGS}") + endif () + + if (${pkg_name}_LDFLAGS) + if (${pkg_name}_USE_STATIC_LIBS) + #set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_STATIC_LINKER_FLAGS=${${pkg_name}_LDFLAGS}") + else() + set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_SHARED_LINKER_FLAGS=${${pkg_name}_LDFLAGS}") + endif () + endif () + + __exec_cmd(COMMAND ${CMAKE_COMMAND} ${PKG_CMAKE_OPTION} -G ${CMAKE_GENERATOR} + ${${pkg_name}_CMAKE_CFLAGS} ${${pkg_name}_CMAKE_CXXFLAGS} ${${pkg_name}_CMAKE_LDFLAGS} + -DCMAKE_INSTALL_PREFIX=${${pkg_name}_BASE_DIR} .. + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build) + + __exec_cmd(COMMAND ${CMAKE_COMMAND} --build . --target install -- -j8 + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build) + + else() + if (${pkg_name}_CFLAGS) + set(${pkg_name}_MAKE_CFLAGS "CFLAGS=${${pkg_name}_CFLAGS}") + endif () + if (${pkg_name}_CXXFLAGS) + set(${pkg_name}_MAKE_CXXFLAGS "CXXFLAGS=${${pkg_name}_CXXFLAGS}") + endif () + if (${pkg_name}_LDFLAGS) + set(${pkg_name}_MAKE_LDFLAGS "LDFLAGS=${${pkg_name}_LDFLAGS}") + endif () + # in configure && make + if (PKG_PRE_CONFIGURE_COMMAND) + __exec_cmd(COMMAND ${PKG_PRE_CONFIGURE_COMMAND} + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}) + endif () + + if (PKG_CONFIGURE_COMMAND) + __exec_cmd(COMMAND ${PKG_CONFIGURE_COMMAND} + ${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS} + --prefix=${${pkg_name}_BASE_DIR} + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}) + endif () + set(${pkg_name}_BUILD_OPTION ${PKG_BUILD_OPTION}) + if (NOT PKG_CONFIGURE_COMMAND) + set(${pkg_name}_BUILD_OPTION ${${pkg_name}_BUILD_OPTION} + ${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS}) + endif () + # build + __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_BUILD_OPTION} -j8 + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}) + + if (PKG_INSTALL_INCS OR PKG_INSTALL_LIBS) + file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS}) + file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS}) + file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include) + file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib) + else() + __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} install WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}) + endif () + endif () + endif() + + if (PKG_LIBS) + __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) + set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) + if(NOT ${pkg_name}_LIBS) + message(FATAL_ERROR "Can not find pkg: ${pkg_name}") + endif() + else() + find_package(${__FIND_PKG_NAME} ${PKG_VER} QUIET ${MS_FIND_NO_DEFAULT_PATH}) + if (${__FIND_PKG_NAME}_FOUND) + set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) + message("Found pkg: ${${__FIND_PKG_NAME}_LIBRARIES}") + return() + endif () + endif () +endfunction() diff --git a/config/e2e_dump_config.json b/config/e2e_dump_config.json new file mode 100644 index 00000000000..ad75c2f27f8 --- /dev/null +++ b/config/e2e_dump_config.json @@ -0,0 +1,22 @@ +{ + "DumpSettings": { + "enable": false, + "trans_flag": false, + "path": "/tmp/net/", + "net_name": "ResNet50", + "mode": 0, + "iteration": 0, + "kernels": ["TensorAdd"] + }, + + "DumpSettingsSpec": { + "enable": "true: dump enable false: dump disable", + "trans_flag": "true: trans to host format,false: not trans format", + "path": "the dump file folder", + "net_name": "net name eg:ResNet50", + "mode": "0: dump all kernels 1: dump kernels in kernels list", + "iteration": "0: all iteration others: specified iteration ", + "kernels": "kernel name list need to be dump" + }, + "other": {} +} \ No newline at end of file diff --git a/config/e2e_dump_config_0.json b/config/e2e_dump_config_0.json new file mode 100644 index 00000000000..a67a4daba07 --- /dev/null +++ b/config/e2e_dump_config_0.json @@ -0,0 +1,22 @@ +{ + "DumpSettings": { + "enable": false, + "trans_flag": false, + "path": "/tmp/hccllog/0", + "net_name": "ResNet50", + "mode": 0, + "iteration": 0, + "kernels": ["AllReduce","BiasAddGrad","Conv2DBackpropFilter","SparseSoftmaxCrossEntropyWithLogits"] + }, + + "DumpSettingsSpec": { + "enable": "true: dump enable false: dump disable", + "trans_flag": "true: trans to host format,false: not trans format", + "path": "the dump file folder", + "net_name": "net name eg:ResNet50", + "mode": "0: dump all kernels 1: dump kernels in kernels list", + "iteration": "0: all iteration others: specified iteration ", + "kernels": "kernel name list need to be dump" + }, + "other": {} +} diff --git a/config/e2e_dump_config_1.json b/config/e2e_dump_config_1.json new file mode 100644 index 00000000000..226b91ae097 --- /dev/null +++ b/config/e2e_dump_config_1.json @@ -0,0 +1,22 @@ +{ + "DumpSettings": { + "enable": false, + "trans_flag": false, + "path": "/tmp/hccllog/1", + "net_name": "ResNet50", + "mode": 0, + "iteration": 0, + "kernels": ["AllReduce","BiasAddGrad","Conv2DBackpropFilter","SparseSoftmaxCrossEntropyWithLogits"] + }, + + "DumpSettingsSpec": { + "enable": "true: dump enable false: dump disable", + "trans_flag": "true: trans to host format,false: not trans format", + "path": "the dump file folder", + "net_name": "net name eg:ResNet50", + "mode": "0: dump all kernels 1: dump kernels in kernels list", + "iteration": "0: all iteration others: specified iteration ", + "kernels": "kernel name list need to be dump" + }, + "other": {} +} diff --git a/config/hccl_evb_multi_rank.json b/config/hccl_evb_multi_rank.json new file mode 100644 index 00000000000..5bf3950a1ee --- /dev/null +++ b/config/hccl_evb_multi_rank.json @@ -0,0 +1,44 @@ +{ + "board_id": "0x3000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "2", + "server_num": "1", + "group_name": "", + "instance_count": "2", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "[device_ip]" + } + ], + "rank_id": "0", + "server_id": "[sever_id]" + }, + { + "devices": [ + { + "device_id": "1", + "device_ip": "[device_ip]" + } + ], + "rank_id": "1", + "server_id": "[sever_id]" + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1" + ], + "para_plane_nic_num": "2", + "status": "completed" +} + diff --git a/config/hccl_multi_machine_multi_rank.json b/config/hccl_multi_machine_multi_rank.json new file mode 100644 index 00000000000..4b494f7d71b --- /dev/null +++ b/config/hccl_multi_machine_multi_rank.json @@ -0,0 +1,175 @@ +{ + "board_id": "0x0000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [{ + "device_num": "16", + "server_num": "2", + "group_name": "", + "instance_count": "16", + "instance_list": [{ + "devices": [{ + "device_id": "0", + "device_ip": "[A_device_ip_0]" + }], + "rank_id": "0", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "1", + "device_ip": "[A_device_ip_1]" + }], + "rank_id": "1", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "2", + "device_ip": "[A_device_ip_2]" + }], + "rank_id": "2", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "3", + "device_ip": "[A_device_ip_3]" + }], + "rank_id": "3", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "4", + "device_ip": "[A_device_ip_4]" + }], + "rank_id": "4", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "5", + "device_ip": "[A_device_ip_5]" + }], + "rank_id": "5", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "6", + "device_ip": "[A_device_ip_6]" + }], + "rank_id": "6", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "7", + "device_ip": "[A_device_ip_7]" + }], + "rank_id": "7", + "server_id": "[server_id_A]" + }, + { + "devices": [{ + "device_id": "0", + "device_ip": "[B_device_ip_0]" + }], + "rank_id": "8", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "1", + "device_ip": "[B_device_ip_1]" + }], + "rank_id": "9", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "2", + "device_ip": "[B_device_ip_2]" + }], + "rank_id": "10", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "3", + "device_ip": "[B_device_ip_3]" + }], + "rank_id": "11", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "4", + "device_ip": "[B_device_ip_4]" + }], + "rank_id": "12", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "5", + "device_ip": "[B_device_ip_5]" + }], + "rank_id": "13", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "6", + "device_ip": "[B_device_ip_6]" + }], + "rank_id": "14", + "server_id": "[server_id_B]" + }, + { + "devices": [{ + "device_id": "7", + "device_ip": "[B_device_ip_7]" + }], + "rank_id": "15", + "server_id": "[server_id_B]" + } + ] + }], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1", + "eth2", + "eth3", + "eth4", + "eth5", + "eth6", + "eth7" + ], + "para_plane_nic_num": "8", + "status": "completed", + + "hccl_config_json_spec": { + "board_id": "board id, current support x0000 or 0x3000", + "chip_info": "chip info, current is 910", + "deploy_mode": "current use lab", + "group_count": "number of groups used", + "group_list": "detailed group information", + "device_num": "number of devices used, the value is the nth power of 2", + "server_num": "number of multiple machines, single machine is 1", + "group_name": "default is hccl_world_group or specified", + "instance_count": "number of instance used, generally equal to device_num", + "instance_list": "detailed instance information", + "device_id": "designated davinic device id to use, values start from 0, but no more than single machine total device num.if server_num greater than 1, the id can be restart from 0", + "device_ip": "ip corresponding to device_id", + "rank_id": "the first device must be 0 and then increase in order", + "server_id": "can be specified as the machine's ip address", + "para_plane_nic_location": "current use device", + "para_plane_nic_name": "network card corresponding to device ip", + "para_plane_nic_num": "number of network cards used", + "status": "current use completed" + } +} \ No newline at end of file diff --git a/config/hccl_single_machine_multi_rank.json b/config/hccl_single_machine_multi_rank.json new file mode 100644 index 00000000000..3916dad5d2c --- /dev/null +++ b/config/hccl_single_machine_multi_rank.json @@ -0,0 +1,111 @@ +{ + "board_id": "0x0000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [{ + "device_num": "8", + "server_num": "1", + "group_name": "", + "instance_count": "8", + "instance_list": [{ + "devices": [{ + "device_id": "0", + "device_ip": "[device_ip_0]" + }], + "rank_id": "0", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "1", + "device_ip": "[device_ip_1]" + }], + "rank_id": "1", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "2", + "device_ip": "[device_ip_2]" + }], + "rank_id": "2", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "3", + "device_ip": "[device_ip_3]" + }], + "rank_id": "3", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "4", + "device_ip": "[device_ip_4]" + }], + "rank_id": "4", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "5", + "device_ip": "[device_ip_5]" + }], + "rank_id": "5", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "6", + "device_ip": "[device_ip_6]" + }], + "rank_id": "6", + "server_id": "[server_id]" + }, + { + "devices": [{ + "device_id": "7", + "device_ip": "[device_ip_7]" + }], + "rank_id": "7", + "server_id": "[server_id]" + } + ] + }], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1", + "eth2", + "eth3", + "eth4", + "eth5", + "eth6", + "eth7" + ], + "para_plane_nic_num": "8", + "status": "completed", + + "hccl_config_json_spec": { + "board_id": "board id, current support x0000 or 0x3000", + "chip_info": "chip info, current is 910", + "deploy_mode": "current use lab", + "group_count": "number of groups used", + "group_list": "detailed group information", + "device_num": "number of devices used, the value is the nth power of 2", + "server_num": "single machine is 1", + "group_name": "default is hccl_world_group or specified", + "instance_count": "number of instance used, generally equal to device_num", + "instance_list": "detailed instance information", + "device_id": "designated davinic device id to use, values start from 0, but no more than single machine total device num", + "device_ip": "ip corresponding to device_id", + "rank_id": "the first device must be 0 and then increase in order", + "server_id": "can be specified as the machine's ip address", + "para_plane_nic_location": "current use device", + "para_plane_nic_name": "network card corresponding to device ip", + "para_plane_nic_num": "number of network cards used", + "status": "current use completed" + } +} \ No newline at end of file diff --git a/dbg_dump_parser.sh b/dbg_dump_parser.sh new file mode 100755 index 00000000000..1d1ec28248c --- /dev/null +++ b/dbg_dump_parser.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -x +set -e + +export SAVE_GRAPHS=YES + +# print usage message +function usage() +{ + echo "Usage:" + echo "bash $0 [-g] [-d] [-a] [-h] [-f file]" + echo "e.g. $0 -f 3_specialize.dat" + echo "" + echo "Options:" + echo " -g Generate ir file for debug" + echo " -d Debug dumped ir" + echo " -a Execute all steps, default" + echo " -f File to be parse" + echo " -h Print usage" +} + +# check and set options +function checkopts() +{ + # init variable + MODE_GEN=0 + MODE_DBG=1 + MODE_ALL=2 + FILE_NAME="3_optimize.dat" + mode="${MODE_ALL}" # default execute all steps + + # Process the options + while getopts 'gdaf:h' opt + do + case "${opt}" in + g) + mode="${MODE_GEN}" + ;; + d) + mode="${MODE_DBG}" + ;; + a) + mode="${MODE_ALL}" + ;; + f) + FILE_NAME="$OPTARG" + if ! [ -f "${FILE_NAME}" ]; then + echo "File $FILE_NAME does not exist" + usage + exit 1 + fi + ;; + h) + usage + exit 0 + ;; + *) + echo "Unknown option ${opt}!" + usage + exit 1 + esac + done +} + +# init variable +# check options +checkopts "$@" + + +cd build/mindspore/ +make -j8 +cp -v mindspore/ccsrc/_c_expression.cpython-*.so ../../mindspore/ +cd - + +UT_NAME="./tests/ut/python/model/test_lenet.py::test_lenet5_train_sens" +#UT_NAME="./tests/python/ops/test_math_ops.py::test_matmul_grad" +#UT_NAME="./tests/python/exec/resnet_example.py::test_compile" +#UT_NAME="./tests/perf_test/test_bert_train.py::test_bert_train" + +if [[ "${mode}" == "${MODE_GEN}" || "${mode}" == "${MODE_ALL}" ]]; then + rm -rf pkl_objs + mkdir -p pkl_objs + + echo "MS_IR_PATH=$(pwd)/pkl_objs pytest -s ${UT_NAME}" + MS_IR_PATH=$(pwd)/pkl_objs/ pytest -s "${UT_NAME}" + #pytest -s $UT_NAME + + # 1_resolve.dat + # 3_specialize.dat + # 4_simplify_data_structures.dat + # 5_opt.dat + # 6_opt2.dat + # 7_opt_ge_adaptor_special.dat + # 8_cconv.dat + # 9_validate.dat + cp "${FILE_NAME}" anf_ir_file.dbg + + rm -rf pkl_objs.dbg + cp -rf pkl_objs pkl_objs.dbg +fi + +if [[ "${mode}" == "${MODE_DBG}" || "${mode}" == "${MODE_ALL}" ]]; then + echo "MS_IR_FILE=$(pwd)/anf_ir_file.dbg MS_IR_PATH=$(pwd)/pkl_objs.dbg/ pytest -s ${UT_NAME}" + MS_IR_FILE=$(pwd)/anf_ir_file.dbg MS_IR_PATH=$(pwd)/pkl_objs.dbg/ pytest -s "${UT_NAME}" +fi diff --git a/docs/Automatic-differentiation.png b/docs/Automatic-differentiation.png new file mode 100644 index 00000000000..c8a9f0a194f Binary files /dev/null and b/docs/Automatic-differentiation.png differ diff --git a/docs/Automatic-parallel.png b/docs/Automatic-parallel.png new file mode 100644 index 00000000000..abf572d568d Binary files /dev/null and b/docs/Automatic-parallel.png differ diff --git a/docs/MindSpore-architecture.png b/docs/MindSpore-architecture.png new file mode 100644 index 00000000000..5211e1d6805 Binary files /dev/null and b/docs/MindSpore-architecture.png differ diff --git a/docs/MindSpore-logo.png b/docs/MindSpore-logo.png new file mode 100644 index 00000000000..9cdefce2d3b Binary files /dev/null and b/docs/MindSpore-logo.png differ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000000..eef57183c40 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +# MindSpore Documentation + +The MindSpore documentation is in the [MindSpore Docs](https://gitee.com/mindspore/docs) repository. diff --git a/example/Bert_NEZHA/config.py b/example/Bert_NEZHA/config.py new file mode 100644 index 00000000000..2f3b22fe500 --- /dev/null +++ b/example/Bert_NEZHA/config.py @@ -0,0 +1,55 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +network config setting, will be used in main.py +""" + +from easydict import EasyDict as edict +import mindspore.common.dtype as mstype +from mindspore.model_zoo.Bert_NEZHA import BertConfig +bert_cfg = edict({ + 'epoch_size': 10, + 'num_warmup_steps': 0, + 'start_learning_rate': 1e-4, + 'end_learning_rate': 1, + 'decay_steps': 1000, + 'power': 10.0, + 'save_checkpoint_steps': 2000, + 'keep_checkpoint_max': 10, + 'checkpoint_prefix': "checkpoint_bert", + 'DATA_DIR' = "/your/path/examples.tfrecord" + 'SCHEMA_DIR' = "/your/path/datasetSchema.json" + 'bert_config': BertConfig( + batch_size=16, + seq_length=128, + vocab_size=21136, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float16, + ) +}) diff --git a/example/Bert_NEZHA/main.py b/example/Bert_NEZHA/main.py new file mode 100644 index 00000000000..a5500f25a97 --- /dev/null +++ b/example/Bert_NEZHA/main.py @@ -0,0 +1,111 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +NEZHA (NEural contextualiZed representation for CHinese lAnguage understanding) is the Chinese pretrained language model currently based on BERT developed by Huawei. +1. Prepare data +Following the data preparation as in BERT, run command as below to get dataset for training: + python ./create_pretraining_data.py \ + --input_file=./sample_text.txt \ + --output_file=./examples.tfrecord \ + --vocab_file=./your/path/vocab.txt \ + --do_lower_case=True \ + --max_seq_length=128 \ + --max_predictions_per_seq=20 \ + --masked_lm_prob=0.15 \ + --random_seed=12345 \ + --dupe_factor=5 +2. Pretrain +First, prepare the distributed training environment, then adjust configurations in config.py, finally run main.py. +""" + +import os +import pytest +import numpy as np +from numpy import allclose +from config import bert_cfg as cfg +import mindspore.common.dtype as mstype +import mindspore.dataset.engine.datasets as de +import mindspore._c_dataengine as deMap +from mindspore import context +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +from mindspore.train.callback import Callback +from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertNetworkWithLoss, BertTrainOneStepCell +from mindspore.nn.optim import Lamb +from mindspore import log as logger +_current_dir = os.path.dirname(os.path.realpath(__file__)) +DATA_DIR = [cfg.DATA_DIR] +SCHEMA_DIR = cfg.SCHEMA_DIR + +def me_de_train_dataset(batch_size): + """test me de train dataset""" + # apply repeat operations + repeat_count = cfg.epoch_size + ds = de.StorageDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids", + "next_sentence_labels", "masked_lm_positions", + "masked_lm_ids", "masked_lm_weights"]) + type_cast_op = deMap.TypeCastOp("int32") + ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op) + ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op) + ds = ds.map(input_columns="next_sentence_labels", operations=type_cast_op) + ds = ds.map(input_columns="segment_ids", operations=type_cast_op) + ds = ds.map(input_columns="input_mask", operations=type_cast_op) + ds = ds.map(input_columns="input_ids", operations=type_cast_op) + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(repeat_count) + return ds + + +def weight_variable(shape): + """weight variable""" + np.random.seed(1) + ones = np.random.uniform(-0.1, 0.1, size=shape).astype(np.float32) + return Tensor(ones) + + +class ModelCallback(Callback): + def __init__(self): + super(ModelCallback, self).__init__() + self.loss_list = [] + + def step_end(self, run_context): + cb_params = run_context.original_args() + self.loss_list.append(cb_params.net_outputs.asnumpy()[0]) + logger.info("epoch: {}, outputs are {}".format(cb_params.cur_epoch_num, str(cb_params.net_outputs))) + +def test_bert_tdt(): + """test bert tdt""" + context.set_context(mode=context.GRAPH_MODE) + context.set_context(device_target="Ascend") + context.set_context(enable_task_sink=True) + context.set_context(enable_loop_sink=True) + context.set_context(enable_mem_reuse=True) + parallel_callback = ModelCallback() + ds = me_de_train_dataset(cfg.bert_config.batch_size) + config = cfg.bert_config + netwithloss = BertNetworkWithLoss(config, True) + optimizer = Lamb(netwithloss.trainable_params(), decay_steps=cfg.decay_steps, start_learning_rate=cfg.start_learning_rate, + end_learning_rate=cfg.end_learning_rate, power=cfg.power, warmup_steps=cfg.num_warmup_steps, decay_filter=lambda x: False) + netwithgrads = BertTrainOneStepCell(netwithloss, optimizer=optimizer) + netwithgrads.set_train(True) + model = Model(netwithgrads) + config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, keep_checkpoint_max=cfg.keep_checkpoint_max) + ckpoint_cb = ModelCheckpoint(prefix=cfg.checkpoint_prefix, config=config_ck) + model.train(ds.get_repeat_count(), ds, callbacks=[parallel_callback, ckpoint_cb], dataset_sink_mode=False) + +if __name__ == '__main__': + test_bert_tdt() diff --git a/example/lenet/config.py b/example/lenet/config.py new file mode 100644 index 00000000000..3ad78f3bde9 --- /dev/null +++ b/example/lenet/config.py @@ -0,0 +1,32 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in main.py +""" +from easydict import EasyDict as edict + +mnist_cfg = edict({ + 'num_classes': 10, + 'lr': 0.01, + 'momentum': 0.9, + 'epoch_size': 1, + 'batch_size': 32, + 'repeat_size': 1, + 'buffer_size': 1000, + 'image_height': 32, + 'image_width': 32, + 'save_checkpoint_steps': 1875, + 'keep_checkpoint_max': 10, +}) diff --git a/example/lenet/main.py b/example/lenet/main.py new file mode 100644 index 00000000000..fe20264d955 --- /dev/null +++ b/example/lenet/main.py @@ -0,0 +1,125 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +######################## train and test lenet example ######################## +1. train lenet and get network model files(.ckpt) : +python main.py --data_path /home/workspace/mindspore_dataset/Tutorial_Network/Lenet/MNIST_Data + +2. test lenet according to model file: +python main.py --data_path /home/workspace/mindspore_dataset/Tutorial_Network/Lenet/MNIST_Data + --mode test --ckpt_path checkpoint_lenet_1-1_1875.ckpt +""" +import os +import argparse +from config import mnist_cfg as cfg + +import mindspore.dataengine as de +import mindspore.nn as nn +from mindspore.model_zoo.lenet import LeNet5 +from mindspore import context, Tensor +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor +from mindspore.train import Model +import mindspore.ops.operations as P +import mindspore.transforms.c_transforms as C +from mindspore.transforms import Inter +from mindspore.nn.metrics import Accuracy +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype + + +class CrossEntropyLoss(nn.Cell): + """ + Define loss for network + """ + def __init__(self): + super(CrossEntropyLoss, self).__init__() + self.cross_entropy = P.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean() + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + + def construct(self, logits, label): + label = self.one_hot(label, F.shape(logits)[1], self.on_value, self.off_value) + loss = self.cross_entropy(logits, label)[0] + loss = self.mean(loss, (-1,)) + return loss + +def create_dataset(data_path, batch_size=32, repeat_size=1, + num_parallel_workers=1): + """ + create dataset for train or test + """ + # define dataset + ds1 = de.MnistDataset(data_path) + + # apply map operations on images + ds1 = ds1.map(input_columns="label", operations=C.TypeCast(mstype.int32)) + ds1 = ds1.map(input_columns="image", operations=C.Resize((cfg.image_height, cfg.image_width), + interpolation=Inter.LINEAR), + num_parallel_workers=num_parallel_workers) + ds1 = ds1.map(input_columns="image", operations=C.Rescale(1 / 0.3081, -1 * 0.1307 / 0.3081), + num_parallel_workers=num_parallel_workers) + ds1 = ds1.map(input_columns="image", operations=C.Rescale(1.0 / 255.0, 0.0), + num_parallel_workers=num_parallel_workers) + ds1 = ds1.map(input_columns="image", operations=C.HWC2CHW(), num_parallel_workers=num_parallel_workers) + + # apply DatasetOps + ds1 = ds1.shuffle(buffer_size=cfg.buffer_size) # 10000 as in LeNet train script + ds1 = ds1.batch(batch_size, drop_remainder=True) + ds1 = ds1.repeat(repeat_size) + + return ds1 + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='MindSpore MNIST Example') + parser.add_argument('--device_target', type=str, default="Ascend", choices=['Ascend', 'GPU', 'CPU'], + help='device where the code will be implemented (default: Ascend)') + parser.add_argument('--mode', type=str, default="train", choices=['train', 'test'], + help='implement phase, set to train or test') + parser.add_argument('--data_path', type=str, default="./MNIST_Data", + help='path where the dataset is saved') + parser.add_argument('--ckpt_path', type=str, default="", help='if mode is test, must provide\ + path where the trained ckpt file') + + args = parser.parse_args() + + context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) + + network = LeNet5(cfg.num_classes) + network.set_train() + # net_loss = nn.SoftmaxCrossEntropyWithLogits() # support this loss soon + net_loss = CrossEntropyLoss() + net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) + config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, + keep_checkpoint_max=cfg.keep_checkpoint_max) + ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) + + if args.mode == 'train': # train + ds = create_dataset(os.path.join(args.data_path, args.mode), batch_size=cfg.batch_size, + repeat_size=cfg.epoch_size) + print("============== Starting Training ==============") + model.train(cfg['epoch_size'], ds, callbacks=[ckpoint_cb, LossMonitor()], dataset_sink_mode=False) + elif args.mode == 'test': # test + print("============== Starting Testing ==============") + param_dict = load_checkpoint(args.ckpt_path) + load_param_into_net(network, param_dict) + ds_eval = create_dataset(os.path.join(args.data_path, "test"), 32, 1) + acc = model.eval(ds_eval, dataset_sink_mode=False) + print("============== Accuracy:{} ==============".format(acc)) + else: + raise RuntimeError('mode should be train or test, rather than {}'.format(args.mode)) diff --git a/example/resnet50_cifar10/config.py b/example/resnet50_cifar10/config.py new file mode 100755 index 00000000000..8e295537044 --- /dev/null +++ b/example/resnet50_cifar10/config.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed + +config = ed({ + "class_num": 10, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 90, + "buffer_size": 100, + "image_height": 224, + "image_width": 224, + "save_checkpoint": True, + "save_checkpoint_steps": 195, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "lr_init": 0.01, + "lr_end": 0.00001, + "lr_max": 0.1, + "warmup_epochs": 5, + "lr_decay_mode": "poly" +}) diff --git a/example/resnet50_cifar10/dataset.py b/example/resnet50_cifar10/dataset.py new file mode 100755 index 00000000000..9ed16f08b55 --- /dev/null +++ b/example/resnet50_cifar10/dataset.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +from config import config + + +def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + + Returns: + dataset + """ + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + + if device_num == 1: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=4, shuffle=True) + else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=4, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + resize_height = config.image_height + resize_width = config.image_width + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) + random_horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) + + resize_op = C.Resize((resize_height, resize_width)) + rescale_op = C.Rescale(rescale, shift) + normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) + + change_swap_op = C.HWC2CHW() + + trans = [] + if do_train: + trans += [random_crop_op, random_horizontal_flip_op] + + trans += [resize_op, rescale_op, normalize_op, change_swap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="label", operations=type_cast_op) + ds = ds.map(input_columns="image", operations=trans) + + # apply shuffle operations + ds = ds.shuffle(buffer_size=config.buffer_size) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/example/resnet50_cifar10/eval.py b/example/resnet50_cifar10/eval.py new file mode 100755 index 00000000000..243dc2a332e --- /dev/null +++ b/example/resnet50_cifar10/eval.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +eval. +""" +import os +import argparse +import random +import numpy as np +from dataset import create_dataset +from config import config +from mindspore import context +from mindspore.model_zoo.resnet import resnet50 +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.model import Model, ParallelMode +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore.dataset.engine as de +from mindspore.communication.management import init + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) +context.set_context(enable_task_sink=True, device_id=device_id) +context.set_context(enable_loop_sink=True) +context.set_context(enable_mem_reuse=True) + +if __name__ == '__main__': + if args_opt.do_eval: + context.set_context(enable_hccl=False) + else: + if args_opt.run_distribute: + context.set_context(enable_hccl=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + auto_parallel_context().set_all_reduce_fusion_split_indices([140]) + init() + else: + context.set_context(enable_hccl=False) + + epoch_size = config.epoch_size + net = resnet50(class_num=config.class_num) + loss = SoftmaxCrossEntropyWithLogits(sparse=True) + + if args_opt.do_eval: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + + model = Model(net, loss_fn=loss, metrics={'acc'}) + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_cifar10/lr_generator.py b/example/resnet50_cifar10/lr_generator.py new file mode 100755 index 00000000000..37c8e907d75 --- /dev/null +++ b/example/resnet50_cifar10/lr_generator.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""learning rate generator""" +import numpy as np + + +def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): + """ + generate learning rate array + + Args: + global_step(int): total steps of the training + lr_init(float): init learning rate + lr_end(float): end learning rate + lr_max(float): max learning rate + warmup_epochs(int): number of warmup epochs + total_epochs(int): total epoch of training + steps_per_epoch(int): steps of one epoch + lr_decay_mode(string): learning rate decay mode, including steps, poly or default + + Returns: + np.array, learning rate array + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + if lr_decay_mode == 'steps': + decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] + for i in range(total_steps): + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + elif lr_decay_mode == 'poly': + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 + for i in range(total_steps): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + else: + base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) + lr = float(lr_max) * base * base + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + else: + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append(lr) + + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + + return learning_rate diff --git a/example/resnet50_cifar10/run_distribute_train.sh b/example/resnet50_cifar10/run_distribute_train.sh new file mode 100755 index 00000000000..e78e2bf1045 --- /dev/null +++ b/example/resnet50_cifar10/run_distribute_train.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ] +then + echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH]" +exit 1 +fi + +if [ ! -f $1 ] +then + echo "error: DMINDSPORE_HCCL_CONFIG_PATH=$1 is not a file" +exit 1 +fi + +if [ ! -d $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory" +exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=8 +export RANK_SIZE=8 +export MINDSPORE_HCCL_CONFIG_PATH=$1 + +for((i=0; i<${DEVICE_NUM}; i++)) +do + export DEVICE_ID=$i + export RANK_ID=$i + rm -rf ./train_parallel$i + mkdir ./train_parallel$i + cp *.py ./train_parallel$i + cp *.sh ./train_parallel$i + cd ./train_parallel$i || exit + echo "start training for rank $RANK_ID, device $DEVICE_ID" + env > env.log + python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$2 &> log & + cd .. +done diff --git a/example/resnet50_cifar10/run_infer.sh b/example/resnet50_cifar10/run_infer.sh new file mode 100755 index 00000000000..5df659275e0 --- /dev/null +++ b/example/resnet50_cifar10/run_infer.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ] +then + echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +if [ ! -d $1 ] +then + echo "error: DATASET_PATH=$1 is not a directory" +exit 1 +fi + +if [ ! -f $2 ] +then + echo "error: CHECKPOINT_PATH=$2 is not a file" +exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_SIZE=$DEVICE_NUM +export RANK_ID=0 + +if [ -d "infer" ]; +then + rm -rf ./infer +fi +mkdir ./infer +cp *.py ./infer +cp *.sh ./infer +cd ./infer || exit +env > env.log +echo "start infering for device $DEVICE_ID" +python eval.py --do_eval=True --dataset_path=$1 --checkpoint_path=$2 &> log & +cd .. diff --git a/example/resnet50_cifar10/run_standalone_train.sh b/example/resnet50_cifar10/run_standalone_train.sh new file mode 100755 index 00000000000..90423630aa5 --- /dev/null +++ b/example/resnet50_cifar10/run_standalone_train.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 1 ] +then + echo "Usage: sh run_standalone_train.sh [DATASET_PATH]" +exit 1 +fi + +if [ ! -d $1 ] +then + echo "error: DATASET_PATH=$1 is not a directory" +exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_ID=0 + +if [ -d "train" ]; +then + rm -rf ./train +fi +mkdir ./train +cp *.py ./train +cp *.sh ./train +cd ./train || exit +echo "start training for device $DEVICE_ID" +env > env.log +python train.py --do_train=True --dataset_path=$1 &> log & +cd .. diff --git a/example/resnet50_cifar10/train.py b/example/resnet50_cifar10/train.py new file mode 100755 index 00000000000..b18c3778de0 --- /dev/null +++ b/example/resnet50_cifar10/train.py @@ -0,0 +1,96 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train_imagenet.""" +import os +import argparse +import random +import numpy as np +from dataset import create_dataset +from lr_generator import get_lr +from config import config +from mindspore import context +from mindspore import Tensor +from mindspore.model_zoo.resnet import resnet50 +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits + +from mindspore.train.model import Model, ParallelMode + +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.loss_scale_manager import FixedLossScaleManager +import mindspore.dataset.engine as de +from mindspore.communication.management import init + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) +context.set_context(enable_task_sink=True, device_id=device_id) +context.set_context(enable_loop_sink=True) +context.set_context(enable_mem_reuse=True) + +if __name__ == '__main__': + if args_opt.do_eval: + context.set_context(enable_hccl=False) + else: + if args_opt.run_distribute: + context.set_context(enable_hccl=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + auto_parallel_context().set_all_reduce_fusion_split_indices([140]) + init() + else: + context.set_context(enable_hccl=False) + + epoch_size = config.epoch_size + net = resnet50(class_num=config.class_num) + loss = SoftmaxCrossEntropyWithLogits(sparse=True) + + + if args_opt.do_train: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, + repeat_num=epoch_size, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, + lr_decay_mode='poly')) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + config.weight_decay, config.loss_scale) + + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) + + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_steps, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) + cb += [ckpt_cb] + model.train(epoch_size, dataset, callbacks=cb) diff --git a/example/yolov3_coco2017/config.py b/example/yolov3_coco2017/config.py new file mode 100644 index 00000000000..f00eb89dc72 --- /dev/null +++ b/example/yolov3_coco2017/config.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Config parameters for YOLOv3 models.""" + + +class ConfigYOLOV3ResNet18: + """ + Config parameters for YOLOv3. + + Examples: + ConfigYoloV3ResNet18. + """ + img_shape = [352, 640] + feature_shape = [32, 3, 352, 640] + num_classes = 80 + + backbone_input_shape = [64, 64, 128, 256] + backbone_shape = [64, 128, 256, 512] + backbone_layers = [2, 2, 2, 2] + backbone_stride = [1, 2, 2, 2] + + ignore_threshold = 0.5 + + anchor_scales = [(10, 13), + (16, 30), + (33, 23), + (30, 61), + (62, 45), + (59, 119), + (116, 90), + (156, 198), + (163, 326)] + out_channel = int(len(anchor_scales) / 3 * (num_classes + 5)) diff --git a/example/yolov3_coco2017/dataset.py b/example/yolov3_coco2017/dataset.py new file mode 100644 index 00000000000..56520704b41 --- /dev/null +++ b/example/yolov3_coco2017/dataset.py @@ -0,0 +1,389 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""YOLOv3 dataset""" +from __future__ import division + +import abc +import io +import os +import math +import json +import numpy as np +from PIL import Image +from matplotlib.colors import rgb_to_hsv, hsv_to_rgb +import mindspore.dataset as de +import mindspore.dataset.transforms.vision.py_transforms as P +from config import ConfigYOLOV3ResNet18 + +iter_cnt = 0 +_NUM_BOXES = 50 + +def preprocess_fn(image, box, is_training): + """Preprocess function for dataset.""" + config_anchors = [10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 163, 326] + anchors = np.array([float(x) for x in config_anchors]).reshape(-1, 2) + do_hsv = False + max_boxes = 20 + num_classes = ConfigYOLOV3ResNet18.num_classes + + def _rand(a=0., b=1.): + return np.random.rand() * (b - a) + a + + def _preprocess_true_boxes(true_boxes, anchors, in_shape=None): + """Get true boxes.""" + num_layers = anchors.shape[0] // 3 + anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] + true_boxes = np.array(true_boxes, dtype='float32') + # input_shape = np.array([in_shape, in_shape], dtype='int32') + input_shape = np.array(in_shape, dtype='int32') + boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2. + boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] + true_boxes[..., 0:2] = boxes_xy / input_shape[::-1] + true_boxes[..., 2:4] = boxes_wh / input_shape[::-1] + + grid_shapes = [input_shape // 32, input_shape // 16, input_shape // 8] + y_true = [np.zeros((grid_shapes[l][0], grid_shapes[l][1], len(anchor_mask[l]), + 5 + num_classes), dtype='float32') for l in range(num_layers)] + + anchors = np.expand_dims(anchors, 0) + anchors_max = anchors / 2. + anchors_min = -anchors_max + + valid_mask = boxes_wh[..., 0] >= 1 + + wh = boxes_wh[valid_mask] + + + if len(wh) >= 1: + wh = np.expand_dims(wh, -2) + boxes_max = wh / 2. + boxes_min = -boxes_max + + intersect_min = np.maximum(boxes_min, anchors_min) + intersect_max = np.minimum(boxes_max, anchors_max) + intersect_wh = np.maximum(intersect_max - intersect_min, 0.) + intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] + box_area = wh[..., 0] * wh[..., 1] + anchor_area = anchors[..., 0] * anchors[..., 1] + iou = intersect_area / (box_area + anchor_area - intersect_area) + + best_anchor = np.argmax(iou, axis=-1) + for t, n in enumerate(best_anchor): + for l in range(num_layers): + if n in anchor_mask[l]: + i = np.floor(true_boxes[t, 0] * grid_shapes[l][1]).astype('int32') + j = np.floor(true_boxes[t, 1] * grid_shapes[l][0]).astype('int32') + k = anchor_mask[l].index(n) + + c = true_boxes[t, 4].astype('int32') + y_true[l][j, i, k, 0:4] = true_boxes[t, 0:4] + y_true[l][j, i, k, 4] = 1. + y_true[l][j, i, k, 5 + c] = 1. + + pad_gt_box0 = np.zeros(shape=[50, 4], dtype=np.float32) + pad_gt_box1 = np.zeros(shape=[50, 4], dtype=np.float32) + pad_gt_box2 = np.zeros(shape=[50, 4], dtype=np.float32) + + mask0 = np.reshape(y_true[0][..., 4:5], [-1]) + gt_box0 = np.reshape(y_true[0][..., 0:4], [-1, 4]) + gt_box0 = gt_box0[mask0 == 1] + pad_gt_box0[:gt_box0.shape[0]] = gt_box0 + + mask1 = np.reshape(y_true[1][..., 4:5], [-1]) + gt_box1 = np.reshape(y_true[1][..., 0:4], [-1, 4]) + gt_box1 = gt_box1[mask1 == 1] + pad_gt_box1[:gt_box1.shape[0]] = gt_box1 + + mask2 = np.reshape(y_true[2][..., 4:5], [-1]) + gt_box2 = np.reshape(y_true[2][..., 0:4], [-1, 4]) + gt_box2 = gt_box2[mask2 == 1] + pad_gt_box2[:gt_box2.shape[0]] = gt_box2 + + return y_true[0], y_true[1], y_true[2], pad_gt_box0, pad_gt_box1, pad_gt_box2 + + def _data_aug(image, box, is_training, jitter=0.3, hue=0.1, sat=1.5, val=1.5, image_size=(352, 640)): + """Data augmentation function.""" + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + + iw, ih = image.size + ori_image_shape = np.array([ih, iw], np.int32) + h, w = image_size + + if not is_training: + image = image.resize((w, h), Image.BICUBIC) + image_data = np.array(image) / 255. + if len(image_data.shape) == 2: + image_data = np.expand_dims(image_data, axis=-1) + image_data = np.concatenate([image_data, image_data, image_data], axis=-1) + image_data = image_data.astype(np.float32) + + # correct boxes + box_data = np.zeros((max_boxes, 5)) + if len(box) >= 1: + np.random.shuffle(box) + if len(box) > max_boxes: + box = box[:max_boxes] + # xmin ymin xmax ymax + box[:, [0, 2]] = box[:, [0, 2]] * float(w) / float(iw) + box[:, [1, 3]] = box[:, [1, 3]] * float(h) / float(ih) + box_data[:len(box)] = box + else: + image_data, box_data = None, None + + # preprocess bounding boxes + bbox_true_1, bbox_true_2, bbox_true_3, gt_box1, gt_box2, gt_box3 = \ + _preprocess_true_boxes(box_data, anchors, image_size) + + return image_data, bbox_true_1, bbox_true_2, bbox_true_3, \ + ori_image_shape, gt_box1, gt_box2, gt_box3 + + flip = _rand() < .5 + # correct boxes + box_data = np.zeros((max_boxes, 5)) + while True: + # Prevent the situation that all boxes are eliminated + new_ar = float(w) / float(h) * _rand(1 - jitter, 1 + jitter) / \ + _rand(1 - jitter, 1 + jitter) + scale = _rand(0.25, 2) + + if new_ar < 1: + nh = int(scale * h) + nw = int(nh * new_ar) + else: + nw = int(scale * w) + nh = int(nw / new_ar) + + dx = int(_rand(0, w - nw)) + dy = int(_rand(0, h - nh)) + + if len(box) >= 1: + t_box = box.copy() + np.random.shuffle(t_box) + t_box[:, [0, 2]] = t_box[:, [0, 2]] * float(nw) / float(iw) + dx + t_box[:, [1, 3]] = t_box[:, [1, 3]] * float(nh) / float(ih) + dy + if flip: + t_box[:, [0, 2]] = w - t_box[:, [2, 0]] + t_box[:, 0:2][t_box[:, 0:2] < 0] = 0 + t_box[:, 2][t_box[:, 2] > w] = w + t_box[:, 3][t_box[:, 3] > h] = h + box_w = t_box[:, 2] - t_box[:, 0] + box_h = t_box[:, 3] - t_box[:, 1] + t_box = t_box[np.logical_and(box_w > 1, box_h > 1)] # discard invalid box + + if len(t_box) >= 1: + box = t_box + break + + box_data[:len(box)] = box + # resize image + image = image.resize((nw, nh), Image.BICUBIC) + # place image + new_image = Image.new('RGB', (w, h), (128, 128, 128)) + new_image.paste(image, (dx, dy)) + image = new_image + + # flip image or not + if flip: + image = image.transpose(Image.FLIP_LEFT_RIGHT) + + # convert image to gray or not + gray = _rand() < .25 + if gray: + image = image.convert('L').convert('RGB') + + # when the channels of image is 1 + image = np.array(image) + if len(image.shape) == 2: + image = np.expand_dims(image, axis=-1) + image = np.concatenate([image, image, image], axis=-1) + + # distort image + hue = _rand(-hue, hue) + sat = _rand(1, sat) if _rand() < .5 else 1 / _rand(1, sat) + val = _rand(1, val) if _rand() < .5 else 1 / _rand(1, val) + image_data = image / 255. + if do_hsv: + x = rgb_to_hsv(image_data) + x[..., 0] += hue + x[..., 0][x[..., 0] > 1] -= 1 + x[..., 0][x[..., 0] < 0] += 1 + x[..., 1] *= sat + x[..., 2] *= val + x[x > 1] = 1 + x[x < 0] = 0 + image_data = hsv_to_rgb(x) # numpy array, 0 to 1 + image_data = image_data.astype(np.float32) + + # preprocess bounding boxes + bbox_true_1, bbox_true_2, bbox_true_3, gt_box1, gt_box2, gt_box3 = \ + _preprocess_true_boxes(box_data, anchors, image_size) + + return image_data, bbox_true_1, bbox_true_2, bbox_true_3, \ + ori_image_shape, gt_box1, gt_box2, gt_box3 + + images, bbox_1, bbox_2, bbox_3, _, gt_box1, gt_box2, gt_box3 = _data_aug(image, box, is_training) + return images, bbox_1, bbox_2, bbox_3, gt_box1, gt_box2, gt_box3 + + +def anno_parser(annos_str): + """Annotation parser.""" + annos = [] + for anno_str in annos_str: + anno = list(map(int, anno_str.strip().split(','))) + annos.append(anno) + return annos + + +def expand_path(path): + """Get file list from path.""" + files = [] + if os.path.isdir(path): + for file in os.listdir(path): + if os.path.isfile(os.path.join(path, file)): + files.append(file) + else: + raise RuntimeError("Path given is not valid.") + return files + + +def read_image(img_path): + """Read image with PIL.""" + with open(img_path, "rb") as f: + img = f.read() + data = io.BytesIO(img) + img = Image.open(data) + return np.array(img) + + +class BaseDataset(): + """BaseDataset for GeneratorDataset iterator.""" + def __init__(self, image_dir, anno_path): + self.image_dir = image_dir + self.anno_path = anno_path + self.cur_index = 0 + self.samples = [] + self.image_anno_dict = {} + self._load_samples() + + def __getitem__(self, item): + sample = self.samples[item] + return self._next_data(sample, self.image_dir, self.image_anno_dict) + + def __len__(self): + return len(self.samples) + + @staticmethod + def _next_data(sample, image_dir, image_anno_dict): + """Get next data.""" + image = read_image(os.path.join(image_dir, sample)) + annos = image_anno_dict[sample] + return [np.array(image), np.array(annos)] + + @abc.abstractmethod + def _load_samples(self): + """Base load samples.""" + + +class YoloDataset(BaseDataset): + """YoloDataset for GeneratorDataset iterator.""" + def _load_samples(self): + """Load samples.""" + image_files_raw = expand_path(self.image_dir) + self.samples = self._filter_valid_data(self.anno_path, image_files_raw) + self.dataset_size = len(self.samples) + if self.dataset_size == 0: + raise RuntimeError("Valid dataset is none!") + + def _filter_valid_data(self, anno_path, image_files_raw): + """Filter valid data.""" + image_files = [] + anno_dict = {} + print("Start filter valid data.") + with open(anno_path, "rb") as f: + lines = f.readlines() + for line in lines: + line_str = line.decode("utf-8") + line_split = str(line_str).split(' ') + anno_dict[line_split[0].split("/")[-1]] = line_split[1:] + anno_set = set(anno_dict.keys()) + image_set = set(image_files_raw) + for image_file in (anno_set & image_set): + image_files.append(image_file) + self.image_anno_dict[image_file] = anno_parser(anno_dict[image_file]) + image_files.sort() + print("Filter valid data done!") + return image_files + + +class DistributedSampler(): + """DistributedSampler for YOLOv3""" + def __init__(self, dataset_size, batch_size, num_replicas=None, rank=None, shuffle=True): + if num_replicas is None: + num_replicas = 1 + if rank is None: + rank = 0 + self.dataset_size = dataset_size + self.num_replicas = num_replicas + self.rank = rank % num_replicas + self.epoch = 0 + self.num_samples = max(batch_size, int(math.ceil(dataset_size * 1.0 / self.num_replicas))) + self.total_size = self.num_samples * self.num_replicas + self.shuffle = shuffle + + def __iter__(self): + # deterministically shuffle based on epoch + if self.shuffle: + indices = np.random.RandomState(seed=self.epoch).permutation(self.dataset_size) + indices = indices.tolist() + else: + indices = list(range(self.dataset_size)) + + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size - len(indices))] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self): + return self.num_samples + + def set_epoch(self, epoch): + self.epoch = epoch + + +def create_yolo_dataset(image_dir, anno_path, batch_size=32, repeat_num=10, device_num=1, rank=0, + is_training=True, num_parallel_workers=8): + """Creatr YOLOv3 dataset with GeneratorDataset.""" + yolo_dataset = YoloDataset(image_dir=image_dir, anno_path=anno_path) + distributed_sampler = DistributedSampler(yolo_dataset.dataset_size, batch_size, device_num, rank) + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "annotation"], sampler=distributed_sampler) + ds.set_dataset_size(len(distributed_sampler)) + compose_map_func = (lambda image, annotation: preprocess_fn(image, annotation, is_training)) + hwc_to_chw = P.HWC2CHW() + ds = ds.map(input_columns=["image", "annotation"], + output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], + columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], + operations=compose_map_func, num_parallel_workers=num_parallel_workers) + ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers) + ds = ds.shuffle(buffer_size=256) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(repeat_num) + return ds diff --git a/example/yolov3_coco2017/run_distribute_train.sh b/example/yolov3_coco2017/run_distribute_train.sh new file mode 100644 index 00000000000..98e6690b8c3 --- /dev/null +++ b/example/yolov3_coco2017/run_distribute_train.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +echo "Please run the scipt as: " +echo "sh run_distribute_train.sh DEVICE_NUM EPOCH_SIZE IMAGE_DIR ANNO_PATH MINDSPORE_HCCL_CONFIG_PATH" +echo "for example: sh run_distribute_train.sh 8 100 ./dataset/coco/train2017 ./dataset/train.txt　./hccl.json" +echo "After running the scipt, the network runs in the background. The log will be generated in LOGx/log.txt" + +export RANK_SIZE=$1 +EPOCH_SIZE=$2 +IMAGE_DIR=$3 +ANNO_PATH=$4 +export MINDSPORE_HCCL_CONFIG_PATH=$5 + + +for((i=0;i env.log + python ../train.py \ + --distribute=1 \ + --device_num=$RANK_SIZE \ + --device_id=$DEVICE_ID \ + --image_dir=$IMAGE_DIR \ + --epoch_size=$EPOCH_SIZE \ + --anno_path=$ANNO_PATH > log.txt 2>&1 & + cd ../ +done diff --git a/example/yolov3_coco2017/run_standalone_train.sh b/example/yolov3_coco2017/run_standalone_train.sh new file mode 100644 index 00000000000..b3b8b0cd9a3 --- /dev/null +++ b/example/yolov3_coco2017/run_standalone_train.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +echo "Please run the scipt as: " +echo "sh run_standalone_train.sh DEVICE_ID EPOCH_SIZE IMAGE_DIR ANNO_PATH" +echo "for example: sh run_standalone_train.sh 0 50 ./dataset/coco/train2017 ./dataset/train.txt" + +python train.py --device_id=$1 --epoch_size=$2 --image_dir=$3 --anno_path=$4 diff --git a/example/yolov3_coco2017/train.py b/example/yolov3_coco2017/train.py new file mode 100644 index 00000000000..a3dd8f34fed --- /dev/null +++ b/example/yolov3_coco2017/train.py @@ -0,0 +1,115 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# less required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +######################## train YOLOv3 example ######################## +train YOLOv3 and get network model files(.ckpt) : +python train.py --image_dir dataset/coco/coco/train2017 --anno_path dataset/coco/train_coco.txt +""" + +import argparse +import numpy as np +import mindspore.nn as nn +from mindspore import context, Tensor +from mindspore.common.initializer import initializer +from mindspore.communication.management import init +from mindspore.train.callback import CheckpointConfig, ModelCheckpoint, LossMonitor, TimeMonitor +from mindspore.train import Model, ParallelMode +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from mindspore.model_zoo.yolov3 import yolov3_resnet18, YoloWithLossCell, TrainingWrapper +from dataset import create_yolo_dataset +from config import ConfigYOLOV3ResNet18 + + +def get_lr(learning_rate, start_step, global_step, decay_step, decay_rate, steps=False): + """Set learning rate""" + lr_each_step = [] + lr = learning_rate + for i in range(global_step): + if steps: + lr_each_step.append(lr * (decay_rate ** (i // decay_step))) + else: + lr_each_step.append(lr * (decay_rate ** (i / decay_step))) + lr_each_step = np.array(lr_each_step).astype(np.float32) + lr_each_step = lr_each_step[start_step:] + return lr_each_step + + +def init_net_param(net, init='ones'): + """Init the parameters in net.""" + params = net.trainable_params() + for p in params: + if isinstance(p.data, Tensor) and 'beta' not in p.name and 'gamma' not in p.name and 'bias' not in p.name: + p.set_parameter_data(initializer(init, p.data.shape(), p.data.dtype())) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="YOLOv3") + parser.add_argument("--distribute", type=bool, default=False, help="Run distribute, default is false.") + parser.add_argument("--device_id", type=int, default=0, help="Device id, default is 0.") + parser.add_argument("--device_num", type=int, default=1, help="Use device nums, default is 1.") + parser.add_argument("--mode", type=str, default="graph", help="Run graph mode or feed mode, default is graph") + parser.add_argument("--epoch_size", type=int, default=10, help="Epoch size, default is 10") + parser.add_argument("--batch_size", type=int, default=32, help="Batch size, default is 32.") + parser.add_argument("--checkpoint_path", type=str, default="", help="Checkpoint file path") + parser.add_argument("--save_checkpoint_epochs", type=int, default=5, help="Save checkpoint epochs, default is 5.") + parser.add_argument("--loss_scale", type=int, default=1024, help="Loss scale, default is 1024.") + parser.add_argument("--image_dir", type=str, required=True, help="Dataset image dir.") + parser.add_argument("--anno_path", type=str, required=True, help="Dataset anno path.") + args_opt = parser.parse_args() + + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=args_opt.device_id) + context.set_context(enable_task_sink=True, enable_loop_sink=True, enable_mem_reuse=True) + if args_opt.distribute: + device_num = args_opt.device_num + context.reset_auto_parallel_context() + context.set_context(enable_hccl=True) + context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, mirror_mean=True, + device_num=device_num) + init() + rank = args_opt.device_id + else: + context.set_context(enable_hccl=False) + rank = 0 + device_num = 1 + + loss_scale = float(args_opt.loss_scale) + dataset = create_yolo_dataset(args_opt.image_dir, args_opt.anno_path, repeat_num=args_opt.epoch_size, + batch_size=args_opt.batch_size, device_num=device_num, rank=rank) + dataset_size = dataset.get_dataset_size() + net = yolov3_resnet18(ConfigYOLOV3ResNet18()) + net = YoloWithLossCell(net, ConfigYOLOV3ResNet18()) + init_net_param(net, "XavierUniform") + + # checkpoint + ckpt_config = CheckpointConfig(save_checkpoint_steps=dataset_size * args_opt.save_checkpoint_epochs) + ckpoint_cb = ModelCheckpoint(prefix="yolov3", directory=None, config=ckpt_config) + if args_opt.checkpoint_path != "": + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + + lr = Tensor(get_lr(learning_rate=0.001, start_step=0, global_step=args_opt.epoch_size * dataset_size, + decay_step=1000, decay_rate=0.95)) + opt = nn.Adam(filter(lambda x: x.requires_grad, net.get_parameters()), lr, loss_scale=loss_scale) + net = TrainingWrapper(net, opt, loss_scale) + callback = [TimeMonitor(data_size=dataset_size), LossMonitor(), ckpoint_cb] + + model = Model(net) + dataset_sink_mode = False + if args_opt.mode == "graph": + dataset_sink_mode = True + print("Start train YOLOv3.") + model.train(args_opt.epoch_size, dataset, callbacks=callback, dataset_sink_mode=dataset_sink_mode) diff --git a/graphengine b/graphengine new file mode 160000 index 00000000000..5f763679fa3 --- /dev/null +++ b/graphengine @@ -0,0 +1 @@ +Subproject commit 5f763679fa33de1608d07f7651c6f16012b953ea diff --git a/mindspore/__init__.py b/mindspore/__init__.py new file mode 100755 index 00000000000..593597cc3f0 --- /dev/null +++ b/mindspore/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore package.""" + +from . import common, train +from .common import * +from .ops import _op_impl +from .train import * +from .log import * +from .version import __version__ + +__all__ = [] +__all__.extend(__version__) +__all__.extend(common.__all__) +__all__.extend(train.__all__) +__all__.extend(log.__all__) diff --git a/mindspore/_checkparam.py b/mindspore/_checkparam.py new file mode 100644 index 00000000000..61b7cc9818b --- /dev/null +++ b/mindspore/_checkparam.py @@ -0,0 +1,542 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Check parameters.""" +import re +from enum import Enum +from itertools import repeat +from collections import Iterable + +import numpy as np +from mindspore import log as logger +from .common import dtype as mstype + + +# Named string regular expression +_name_re = r"^\w+[0-9a-zA-Z\_\.]*$" + + +class Rel(Enum): + """Numerical relationship between variables, logical relationship enumeration definition of range.""" + # scalar compare + EQ = 1 # == + NE = 2 # != + LT = 3 # < + LE = 4 # <= + GT = 5 # > + GE = 6 # >= + # scalar range check + INC_NEITHER = 7 # (), include neither + INC_LEFT = 8 # [), include left + INC_RIGHT = 9 # (], include right + INC_BOTH = 10 # [], include both + # collection in, not in + IN = 11 + NOT_IN = 12 + + @staticmethod + def get_strs(rel): + """Get value from rel_strs.""" + return rel_strs.get(rel, "") + + @staticmethod + def get_fns(rel): + """Get value from rel_fns.""" + return rel_fns.get(rel, lambda *args: False) + + +rel_fns = { + # scalar compare + Rel.EQ: lambda x, y: x == y, + Rel.NE: lambda x, y: x != y, + Rel.LT: lambda x, y: x < y, + Rel.LE: lambda x, y: x <= y, + Rel.GT: lambda x, y: x > y, + Rel.GE: lambda x, y: x >= y, + # scalar range check + Rel.INC_NEITHER: lambda x, lower, upper: (lower < x < upper), + Rel.INC_LEFT: lambda x, lower, upper: (lower <= x < upper), + Rel.INC_RIGHT: lambda x, lower, upper: (lower < x <= upper), + Rel.INC_BOTH: lambda x, lower, upper: (lower <= x <= upper), + # collection in, not in + Rel.IN: lambda x, y: x in y, + Rel.NOT_IN: lambda x, y: x not in y, +} + +rel_strs = { + # scalar compare + Rel.EQ: "equal to {}", + Rel.NE: "not equal to {}", + Rel.LT: "less than {}", + Rel.LE: "less or equal to {}", + Rel.GT: "greater than {}", + Rel.GE: "greater or equal to {}", + # scalar range check + Rel.INC_NEITHER: "({}, {})", + Rel.INC_LEFT: "[{}, {})", + Rel.INC_RIGHT: "({}, {}]", + Rel.INC_BOTH: "[{}, {}]", + # collection in, not in + Rel.IN: "in {}", + Rel.NOT_IN: "not in {}", +} + + +class ParamValidator: + """Parameter validator.""" + + @staticmethod + def equal(arg_name, arg_value, cond_str, cond): + """Judging valid value.""" + if not cond: + raise ValueError(f'The `{arg_name}` must be {cond_str}, but got {arg_value}.') + + @staticmethod + def check(arg_name, arg_value, value_name, value, rel=Rel.EQ): + """This method is only used for check int values, since when compare float values, + we need consider float error.""" + rel_fn = Rel.get_fns(rel) + if not rel_fn(arg_value, value): + rel_str = Rel.get_strs(rel).format(f'{value_name}: {value}') + raise ValueError(f'The `{arg_name}` should be {rel_str}, but got {arg_value}.') + + @staticmethod + def check_integer(arg_name, arg_value, value, rel): + """Integer value judgment.""" + rel_fn = Rel.get_fns(rel) + type_mismatch = not isinstance(arg_value, int) or isinstance(arg_value, bool) + if type_mismatch or not rel_fn(arg_value, value): + rel_str = Rel.get_strs(rel).format(value) + raise ValueError(f'The `{arg_name}` should be an int and must {rel_str}, but got {arg_value}.') + return arg_value + + @staticmethod + def check_shape_length(arg_name, arg_value, value, rel): + """Shape length judgment.""" + rel_fn = Rel.get_fns(rel) + type_mismatch = not isinstance(arg_value, int) + if type_mismatch or not rel_fn(arg_value, value): + rel_str = Rel.get_strs(rel).format(value) + raise ValueError(f'The length of `{arg_name}` should be an int and must {rel_str}, but got {arg_value}') + return arg_value + + @staticmethod + def check_int_range(arg_name, arg_value, lower_limit, upper_limit, rel): + """This method is only used for check int values, + since when compare float values, we need consider float error.""" + rel_fn = Rel.get_fns(rel) + type_mismatch = not isinstance(arg_value, int) + if type_mismatch or not rel_fn(arg_value, lower_limit, upper_limit): + rel_str = Rel.get_strs(rel).format(lower_limit, upper_limit) + raise ValueError(f'The `{arg_name}` should be an int in range {rel_str}, but got {arg_value}.') + return arg_value + + @staticmethod + def check_isinstance(arg_name, arg_value, classes): + """Check arg isintance of classes""" + if not isinstance(arg_value, classes): + raise ValueError(f'The `{arg_name}` should be isintance of {classes}, but got {arg_value}.') + return arg_value + + @staticmethod + def check_number_range(arg_name, arg_value, lower_limit, upper_limit, rel): + """Is it necessary to consider error when comparing float values.""" + rel_fn = Rel.get_fns(rel) + if not rel_fn(arg_value, lower_limit, upper_limit): + rel_str = Rel.get_strs(rel).format(lower_limit, upper_limit) + raise ValueError(f'The `{arg_name}` should be in range {rel_str}, but got {arg_value}.') + return arg_value + + @staticmethod + def check_subclass(arg_name, type_, template_type, with_type_of=True): + """Check whether some type is sublcass of another type""" + if not isinstance(template_type, Iterable): + template_type = (template_type,) + if not any([mstype.issubclass_(type_, x) for x in template_type]): + type_str = (type(type_).__name__ if isinstance(type_, (tuple, list)) else "") + str(type_) + raise TypeError(f'The {"type of" if with_type_of else ""} `{arg_name}` should be subclass' + f' of {",".join((str(x) for x in template_type))}, but got {type_str}.') + + @staticmethod + def check_args_tensor(args): + """Check whether args are all tensor.""" + if not isinstance(args, dict): + raise TypeError("The args should be a dict.") + for arg, value in args.items(): + ParamValidator.check_subclass(arg, value, mstype.tensor) + + @staticmethod + def check_type(arg_name, arg_value, valid_types): + """Type checking.""" + def raise_error_msg(): + """func for raising error message when check failed""" + type_names = [t.__name__ for t in valid_types] + num_types = len(valid_types) + raise ValueError(f'The type of `{arg_name}` should be {"one of " if num_types > 1 else ""}' + f'{type_names if num_types > 1 else type_names[0]}, but got {type(arg_value).__name__}.') + + if isinstance(arg_value, type(mstype.tensor)): + arg_value = arg_value.element_type() + # Notice: bool is subclass of int, so `check_type('x', True, [int])` will check fail, and + # `check_type('x', True, [bool, int])` will check pass + if isinstance(arg_value, bool) and bool not in tuple(valid_types): + raise_error_msg() + if isinstance(arg_value, tuple(valid_types)): + return arg_value + raise_error_msg() + + @staticmethod + def check_typename(arg_name, arg_type, valid_types): + """Does it contain the _name_ attribute.""" + + def get_typename(t): + return t.__name__ if hasattr(t, '__name__') else str(t) + + if isinstance(arg_type, type(mstype.tensor)): + arg_type = arg_type.element_type() + + if arg_type in valid_types: + return arg_type + type_names = [get_typename(t) for t in valid_types] + if len(valid_types) == 1: + raise ValueError(f'The type of `{arg_name}` should be {type_names[0]},' + f' but got {get_typename(arg_type)}.') + raise ValueError(f'The type of `{arg_name}` should be one of {type_names},' + f' but got {get_typename(arg_type)}.') + + @staticmethod + def check_string(arg_name, arg_value, valid_values): + """String type judgment.""" + if isinstance(arg_value, str) and arg_value in valid_values: + return arg_value + if len(valid_values) == 1: + raise ValueError(f'The `{arg_name}` should be str and must be {valid_values[0]},' + f' but got {arg_value}.') + raise ValueError(f'The `{arg_name}` should be str and must be one of {valid_values},' + f' but got {arg_value}.') + + @staticmethod + def check_type_same(args, valid_values): + """Determine whether the types are the same.""" + name = list(args.keys())[0] + value = list(args.values())[0] + if isinstance(value, type(mstype.tensor)): + value = value.element_type() + for arg_name, arg_value in args.items(): + if isinstance(arg_value, type(mstype.tensor)): + arg_value = arg_value.element_type() + + if arg_value not in valid_values: + raise TypeError(f'The `{arg_name}` should be in {valid_values},' + f' but `{arg_name}` is {arg_value}.') + if arg_value != value: + raise TypeError(f'`{arg_name}` should be same as `{name}`,' + f' but `{arg_name}` is {arg_value}, `{name}` is {value}.') + + @staticmethod + def check_two_types_same(arg1_name, arg1_type, arg2_name, arg2_type): + """Determine whether the types of two variables are the same.""" + if arg1_type != arg2_type: + raise TypeError(f'The type of `{arg1_name}` and `{arg2_name}` should be same.') + + @staticmethod + def check_value_on_integer(arg_name, arg_value, value, rel): + """Judging integer type.""" + rel_fn = Rel.get_fns(rel) + type_match = isinstance(arg_value, int) + if type_match and (not rel_fn(arg_value, value)): + rel_str = Rel.get_strs(rel).format(value) + raise ValueError(f'The `{arg_name}` should be an int and must {rel_str}, but got {arg_value}.') + return arg_value + + @staticmethod + def check_param_equal(param1_name, param1_value, param2_name, param2_value): + """Judging the equality of parameters.""" + if param1_value != param2_value: + raise ValueError(f"`{param1_name}` must equal `{param2_name}`," + f" but got `{param1_name}` = {param1_value}," + f" `{param2_name}` = {param2_value}.") + + @staticmethod + def check_const_input(arg_name, arg_value): + """Check valid value.""" + if arg_value is None: + raise ValueError(f'The `{arg_name}` must be a const input, but got {arg_value}.') + + @staticmethod + def check_float_positive(arg_name, arg_value): + """Float type judgment.""" + if isinstance(arg_value, float): + if arg_value > 0: + return arg_value + raise ValueError(f"The `{arg_name}` must be positive, but got {arg_value}.") + + raise TypeError(f"`{arg_name}` must be float!") + + @staticmethod + def check_pad_value_by_mode(op_name, pad_mode, padding): + """Validate value of padding according to pad_mode""" + if pad_mode != 'pad' and padding != 0: + raise ValueError(f"For op '{op_name}', padding must be zero when pad_mode is '{pad_mode}'.") + return padding + + @staticmethod + def check_empty_shape_input(arg_name, arg_value): + """Check zeros value.""" + if 0 in arg_value: + raise ValueError(f"Input `{arg_name}` cannot be empty.") + + @staticmethod + def check_scalar_shape_input(arg_name, arg_value): + """Check scalar shape input.""" + if arg_value != []: + raise ValueError(f"Input `{arg_name}` shape should be (). got {arg_value}") + + +def check_int(input_param): + """Int type judgment.""" + if isinstance(input_param, int) and not isinstance(input_param, bool): + return input_param + raise TypeError("Input type must be int!") + + +def check_int_positive(input_param): + """Int type judgment.""" + if isinstance(input_param, bool): + raise TypeError("Input type must be int cannot be bool!") + if isinstance(input_param, int): + if input_param > 0: + return input_param + raise ValueError("The input_param must be positive, but got input_param {}.".format(input_param)) + raise TypeError("Input type must be int cannot be {}!".format(type(input_param))) + + +def check_int_non_negative(input_param): + """Non_negative type judgment.""" + if isinstance(input_param, bool): + raise TypeError("Input type must be int cannot be bool!") + if isinstance(input_param, int): + if input_param >= 0: + return input_param + raise ValueError("The input_param must be non_negative, but got input_param {}.".format(input_param)) + raise TypeError("Input type must be int cannot be {}!".format(type(input_param))) + + +def check_int_zero_one(input_param): + """Judge whether it is 0 or 1.""" + if input_param in (0, 1): + return input_param + raise ValueError("The data must be 0 or 1.") + + +def check_bool(input_param): + """Bool type judgment.""" + if isinstance(input_param, bool): + return input_param + raise TypeError("Input type must be bool!") + + +def check_input_format(input_param): + """Judge input format.""" + if input_param == "NCHW": + return input_param + raise ValueError("The data format must be NCHW.") + + +def check_padding(padding): + """Check padding.""" + if padding >= 0: + return padding + raise ValueError("The padding must be at least 0,"" but got padding {}.".format(padding)) + + +def check_padmode(mode): + """Check padmode.""" + if mode in ("same", "valid", "pad"): + return mode + raise ValueError("The pad mode must be same or valid or pad,"" but got mode {}.".format(mode)) + + +def check_tensor_supported_type(dtype): + """Check tensor dtype.""" + if dtype in (mstype.int32, mstype.float32): + return dtype + raise ValueError("The dtype must be mstype.int32 or mstype.float32, but got mstype {}.".format(dtype)) + + +def _expand_tuple(n_dimensions): + """To expand a number to tuple.""" + + def convert(m): + if not isinstance(m, tuple): + if isinstance(m, int): + return tuple(repeat(m, n_dimensions)) + raise TypeError("Input type must be int or tuple.") + + if not len(m) is n_dimensions: + raise TypeError("Input dimension is incorrect.") + + for i in m: + if not isinstance(i, int): + raise TypeError("Incorrect type inside of a tuple!") + return m + + return convert + + +def check_input_data(*data, data_class): + """Input data check.""" + for item in data: + if isinstance(item, (list, tuple)): + for v in item: + check_input_data(v, data_class=data_class) + else: + if not isinstance(item, data_class): + raise ValueError(f'Please provide as model inputs' + f' either a single' + f' or a list of {data_class.__name__},' + f' but got part data type is {str(type(item))}.') + if item.size() == 0: + msg = "Please provide non-empty data." + logger.error(msg) + raise ValueError(msg) + + +def check_output_data(data): + """Output data check.""" + if not data: + raise RuntimeError('Executor return data ' + str(data) + ', please check your net or input data.') + + +def check_axis_type_int(axis): + """Check axis type.""" + if not isinstance(axis, int): + raise TypeError('Wrong type for axis, should be int.') + + +def check_axis_range(axis, rank): + """Check axis range.""" + if not -rank <= axis < rank: + raise ValueError('The axis should be in range [{}, {}),'' but got {}.'.format(-rank, rank, axis)) + + +def check_attr_int(attr_name, attr): + """Check int type.""" + if not isinstance(attr, int): + raise TypeError("The attr {} should be int, but got {}.".format(attr_name, type(attr))) + + +def check_t_in_range(t): + """Check input range.""" + if t not in (mstype.float16, mstype.float32, mstype.float64, mstype.int32, mstype.int64): + raise ValueError("The param T should be (float16, float32, float64, int32, int64).") + + +once = _expand_tuple(1) +twice = _expand_tuple(2) +triple = _expand_tuple(3) +valid_data_types = (int, float, np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64, np.float16, + np.float32, np.float64, bool, np.bool_) + + +def check_type(arg_name, arg_value, valid_types): + """Check value type.""" + # if input type is Tensor ,get element type + if isinstance(arg_value, type(mstype.tensor)): + arg_value = arg_value.element_type() + + # First, check if arg_value has argvalid_types + if isinstance(arg_value, tuple(valid_types)): + return type(arg_value).__name__ + + # Second, wrap arg_value with numpy array so that it can be checked through numpy api + if isinstance(arg_value, (list, tuple)): + arg_value = np.array(arg_value) + + # Thirdly, check the data type by numpy's dtype api + valid = False + if isinstance(arg_value, np.ndarray): + valid = arg_value.dtype in valid_data_types + + # Notice: bool is subclass of int, so `check_type('x', True, [int])` will check fail, and + # `check_type('x', True, [bool, int])` will check pass + if isinstance(arg_value, bool) and bool not in tuple(valid_types): + valid = False + + if not valid: + type_names = [t.__name__ for t in valid_types] + if len(valid_types) == 1: + raise TypeError(f'The type of `{arg_name}` should be {type_names[0]},' + f' but got {type(arg_value).__name__}.') + raise TypeError(f'The type of `{arg_name}` should be one of {type_names},' + f' but got {type(arg_value).__name__}.') + + return type(arg_value).__name__ + + +def check_typename(arg_name, arg_type, valid_types): + """Check type name.""" + + def get_typename(t): + return t.__name__ if hasattr(t, '__name__') else str(t) + + if isinstance(arg_type, type(mstype.tensor)): + arg_type = arg_type.element_type() + + if arg_type in valid_types: + return arg_type + if isinstance(arg_type, tuple(valid_types)): + return arg_type + type_names = [get_typename(t) for t in valid_types] + if len(valid_types) == 1: + raise TypeError(f'The type of `{arg_name}` should be {type_names[0]},' + f' but got {get_typename(arg_type)}.') + raise TypeError(f'The type of `{arg_name}` should be one of {type_names},' + f' but got {get_typename(arg_type)}.') + + +def check_shape(arg_name, arg_value): + """Check shape.""" + # First, check if shape is a tuple + if not isinstance(arg_value, tuple): + raise TypeError(f'The type of `{arg_name}` should be one of {tuple.__name__},' + f' but got {type(arg_value).__name__}.') + + # Second, wrap arg_value with numpy array so that it can be checked through numpy api + arg_value = np.array(arg_value) + + # shape can not be () + if arg_value.size == 0: + raise ValueError('Shape can not be empty.') + + # shape's dimension should be 1 + if arg_value.ndim != 1: + raise ValueError('Shape of tensor should be 1-dim vector, but got {}-dim.'.format(arg_value.ndim)) + + # Thirdly, check each element's type of the shape + valid_types = (int, np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64) + for dim_size in arg_value: + if not isinstance(dim_size, valid_types) or dim_size <= 0: + raise ValueError('Every dimension size of the tensor shape should be a positive integer,' + ' but got {}.'.format(dim_size)) + + +def _check_str_by_regular(target, reg=None, flag=re.ASCII): + if reg is None: + reg = _name_re + if re.match(reg, target, flag) is None: + raise ValueError("'{}' is illegal, it should be match regular'{}' by flags'{}'".format(target, reg, flag)) + return True diff --git a/mindspore/_extends/__init__.py b/mindspore/_extends/__init__.py new file mode 100644 index 00000000000..5eabfcd97c8 --- /dev/null +++ b/mindspore/_extends/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Extension functions. + +Python functions that will be called in the c++ parts of MindSpore. +""" +from .utils import cell_attr_register + +__all__ = ["cell_attr_register"] diff --git a/mindspore/_extends/builtin_operations.py b/mindspore/_extends/builtin_operations.py new file mode 100644 index 00000000000..087b7047196 --- /dev/null +++ b/mindspore/_extends/builtin_operations.py @@ -0,0 +1,161 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""builtin_operations""" +import functools +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.common.dtype import dtype_to_nptype, get_py_obj_dtype + + +def scalar_add(x, y): + """Implement `scalar_add`.""" + return x + y + + +def scalar_mul(x, y): + """Implement `scalar_mul`.""" + return x * y + + +def scalar_mod(x, y): + """Implement `scalar_mul`.""" + return x % y + + +def scalar_sub(x, y): + """Implement `scalar_sub`.""" + return x - y + + +def scalar_usub(x): + """Implement `scalar_usub`.""" + return -x + + +def tuple_getitem(x, index): + """Implement `tuple_getitem`.""" + if isinstance(x, Tensor): + x = x.asnumpy() + y = x[index] + return Tensor(y) + return x[index] + + +def scalar_gt(x, y): + """Implement `scalar_gt`.""" + return x > y + + +def scalar_ne(x, y): + """Implement `scalar_ne`.""" + return x != y + + +def scalar_eq(x, y): + """Implement `scalar_eq`.""" + return x == y + + +def scalar_le(x, y): + """Implement `scalar_le`.""" + return x <= y + + +def scalar_lt(x, y): + """Implement `scalar_lt`.""" + return x < y + + +def identity(x): + """Implement `identity`.""" + return x + + +def zeros_like_tensor(x): + """Implement `zeros_like_tensor`.""" + x = x.asnumpy() + value = Tensor(np.zeros(x.shape)) + return value + + +def switch(c, x, y): + """Implement `switch`.""" + return x if c else y + + +def list_getitem(data, item): + """Implement `list_getitem`.""" + return data[item] + + +def bool_not(x): + """Implement `bool_not`.""" + return not x + + +def bool_and(x, y): + """Implement `bool_and`.""" + return x and y + + +def bool_or(x, y): + """Implement `bool_or`.""" + return x or y + + +def make_list(*xs): + """Implement `make_list`.""" + return list(xs) + + +def list_len(x): + """Implement `list_len`.""" + return len(x) + + +# only used in PyNative modes +def partial(*args): + """Implement `partial`.""" + func = args[0].__call__ + partial_func = functools.partial(func, *args[1:]) + return partial_func + + +# only used in PyNative modes +def depend(value, expr): + return value + + +def scalar_cast(x, t): + """Implement scalar_cast.""" + np_type = dtype_to_nptype(t) + value = np_type(x) + cast_value = np.ndarray.item(value) + return cast_value + + +def typeof(x): + """Implement typeof.""" + return get_py_obj_dtype(x) + + +def tuple_to_array(x): + """Implement `tuple_to_array`.""" + return Tensor(np.array(x)) + + +def stop_gradient(x): + """Implement `stop_gradient`.""" + return x diff --git a/mindspore/_extends/parallel_compile/__init__.py b/mindspore/_extends/parallel_compile/__init__.py new file mode 100644 index 00000000000..c336f0dafc7 --- /dev/null +++ b/mindspore/_extends/parallel_compile/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Extension functions. + +Python functions that will be called in the c++ parts of MindSpore. +""" diff --git a/mindspore/_extends/parallel_compile/multi_compiler.py b/mindspore/_extends/parallel_compile/multi_compiler.py new file mode 100644 index 00000000000..542167888b1 --- /dev/null +++ b/mindspore/_extends/parallel_compile/multi_compiler.py @@ -0,0 +1,107 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Providing multi process compile with json""" +import json +import math +import os +import subprocess +import sys +from multiprocessing import Pool + + +def _compiletask(platform, *jsons): + """ + compile func called in single process + + Parameters: + platform: str. AKG platform or TBE platform + *jsons: str. json str contain kernel info, suitable for json compile + api + + """ + if platform == "AKG": + p = __import__("akg", globals(), locals(), ['ms'], 0) + func = getattr(p.ms, "compilewithjson") + for json_item in jsons: + res = func(json_item) + if not res: + raise ValueError("Compile error") + if platform == "TBE": + tbe_compiler = os.path.join(os.path.split(os.path.realpath(__file__))[0], "tbe_compiler", "compiler.py") + for json_item in jsons: + res = subprocess.run([sys.executable, tbe_compiler], input=json_item, text=True) + if res.returncode != 0: + raise ValueError("Tbe compile error") + + +def compilekernelparallel(jsons, process, waitime): + """ + compile kernel use multi processes + + Parameters: + jsons: list. json str list contain kernel info + process: int. processes num + waittime: int. max time the function blocked + """ + if not isinstance(jsons, list): + raise ValueError("jsons must be a list") + if not isinstance(process, int): + raise ValueError("process must be a num") + if not isinstance(waitime, int): + raise ValueError("waittime must be a num") + + jsons_akg = [] + jsons_tbe = [] + for json_ in jsons: + j = json.loads(json_) + if j["platform"] == "TBE": + jsons_tbe.append(json_) + continue + if j["platform"] == "AKG": + jsons_akg.append(json_) + continue + raise RuntimeError( + "not support this platform {0}".format(j["platform"])) + if jsons_akg: + process_akg = math.floor(len(jsons)/len(jsons_akg)*process) + else: + process_akg = 0 + + if process_akg == 0 and jsons_akg: + process_akg = 1 + process_tbe = process-process_akg + if process_tbe == 0 and jsons_tbe: + process_tbe = 1 + raise RuntimeWarning("we add a process for compile more operator") + + args = [[] for _ in range(process_akg+process_tbe)] + args_lens = len(args) + for p in range(args_lens): + if p < process_tbe: + args[p].append("TBE") + else: + args[p].append("AKG") + jsons_tbe_lens = len(jsons_tbe) + for p in range(jsons_tbe_lens): + args[p % process_tbe].append(jsons_tbe[p]) + jsons_akg_lens = len(jsons_akg) + for p in range(jsons_akg_lens): + args[process-p % process_akg-1].append(jsons_akg[p]) + for p in range(args_lens): + args[p] = tuple(args[p]) + with Pool(processes=process) as pool: + res = pool.starmap_async(_compiletask, args) + res.get(timeout=waitime) + return True diff --git a/mindspore/_extends/parallel_compile/tbe_compiler/__init__.py b/mindspore/_extends/parallel_compile/tbe_compiler/__init__.py new file mode 100644 index 00000000000..e30774307ca --- /dev/null +++ b/mindspore/_extends/parallel_compile/tbe_compiler/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindspore/_extends/parallel_compile/tbe_compiler/common.py b/mindspore/_extends/parallel_compile/tbe_compiler/common.py new file mode 100644 index 00000000000..6cfa2afd0d7 --- /dev/null +++ b/mindspore/_extends/parallel_compile/tbe_compiler/common.py @@ -0,0 +1,138 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""tbe common""" +import json +import os + + +class TBEException(Exception): + """tbe exception class""" + + def __init__(self, err_msg): + super().__init__(self) + self.__error_msg = err_msg + + def __str__(self): + return self.__error_msg + + +def get_ddk_version(): + """get ddk version""" + ddk_version = os.environ.get("DDK_VERSION") + if ddk_version is None: + default_ddk_info_file = '/usr/local/HiAI/runtime/ddk_info' + backup_ddk_info_file = '/usr/local/Ascend/fwkacllib/ddk_info' + if os.path.exists(default_ddk_info_file): + with open(default_ddk_info_file, "r") as fp: + ddk_version = json.load(fp)["VERSION"] + elif os.path.exists(backup_ddk_info_file): + with open(backup_ddk_info_file, "r") as fp: + ddk_version = json.load(fp)["VERSION"] + else: + ddk_version = "1.60.T17.B830" + return ddk_version + + +def get_build_in_impl_path(): + """get build-in tbe implement path""" + tbe_impl_path = os.environ.get("TBE_IMPL_PATH") + if tbe_impl_path is None: + default_install_path = '/usr/local/HiAI/runtime/ops/op_impl/built-in/ai_core/tbe/' + backup_install_path = '/usr/local/Ascend/Ascend/opp/op_impl/built-in/ai_core/tbe/' + if os.path.exists(default_install_path): + tbe_impl_path = default_install_path + elif os.path.exists(backup_install_path): + tbe_impl_path = backup_install_path + if not tbe_impl_path: + raise ValueError("Can not find the env TBE_IMPL_PATH") + return tbe_impl_path + + +def _check_arg_info(item): + """ + Check parameter Validity. + + Args: + item (dict): A dict, to be checked. + + Raises: + Exception: If specific keyword is not found. + """ + if 'shape' not in item: + raise ValueError("Json string Errors, key:shape not found.") + if 'ori_shape' not in item: + raise ValueError("Json string Errors, key:ori_shape not found.") + if 'format' not in item or not item['format']: + raise ValueError("Json string Errors, key:format not found.") + if 'ori_format' not in item or not item['ori_format']: + raise ValueError("Json string Errors, key:ori_format not found.") + if 'dtype' not in item or not item['dtype']: + raise ValueError("Json string Errors, key:dtype not found.") + + +def get_args(op_info, arg_type): + """ + Parse args. + + Args: + op_info (dict): Op info dict. + arg_type (str): arg, to be parsed. + + Raises: + Exception: If specific keyword is not found. + """ + if arg_type not in op_info: + raise ValueError("Json string Errors, key:{} not found.".format(arg_type)) + args = [] + if not op_info[arg_type]: + return args + if arg_type in ['inputs', 'outputs']: + for item in op_info[arg_type]: + arg = [] + for info in item: + if 'valid' not in info: + raise ValueError("Json string Errors, key:valid not found.") + if info['valid']: + _check_arg_info(info) + del info['valid'] + del info['name'] + if len(item) > 1: + arg.append(info) + else: + args.append(info) + else: + if len(item) > 1: + arg.append(None) + else: + args.append(None) + if len(item) > 1: + args.append(arg) + + elif arg_type == 'attrs': + for item in op_info[arg_type]: + if 'value' not in item: + raise ValueError("Json string Errors, attr key:value not found.") + if item["name"] != "isRef": + args.append(item['value']) + return args + + +def check_kernel_info(kernel_info): + if 'op_info' not in kernel_info or not kernel_info['op_info']: + raise ValueError("Json string Errors, key:op_info not found.") + if 'name' not in kernel_info['op_info'] or not kernel_info['op_info']['name']: + raise ValueError("Json string Errors, key:name not found.") + if 'kernel_name' not in kernel_info['op_info'] or not kernel_info['op_info']['kernel_name']: + raise ValueError("Json string Errors, key:kernel_name not found.") diff --git a/mindspore/_extends/parallel_compile/tbe_compiler/compiler.py b/mindspore/_extends/parallel_compile/tbe_compiler/compiler.py new file mode 100755 index 00000000000..8af0f8d3b40 --- /dev/null +++ b/mindspore/_extends/parallel_compile/tbe_compiler/compiler.py @@ -0,0 +1,166 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""tbe compiler""" +import json +import os +import sys +from te.platform.cce_conf import te_set_version +from te.platform.fusion_manager import op_build_cfg_dis, op_build_cfg_en, set_current_op_name, \ + init_op_pattern, set_op_params, set_op_build_type, get_op_pattern, set_current_op_func_name +from te.platform.fusion_util import fusion_op +from common import check_kernel_info, get_args, get_build_in_impl_path, get_ddk_version + +ddk_version = get_ddk_version() +build_in_impl_path = get_build_in_impl_path() + +# op function list +op_build = "compile" +op_pre_build = "pre_build" + + +def _initialize(impl_path): + """Initialize""" + te_set_version(ddk_version) + if impl_path == "": + op_module_name = build_in_impl_path + else: + op_module_name = impl_path + if not op_module_name: + raise ValueError("Can not find the env TBE_IMPL_PATH") + + sys.path.insert(0, op_module_name) + + +def build_op(build_type, json_str): + """ + call op functions with function name and input args json_str + + Args: + build_type : op function name + json_str (str): op function input args + + Raises: + Exception: If specific keyword is not found. + """ + kernel_info = json.loads(json_str) + check_kernel_info(kernel_info) + + # import module + op_name = kernel_info['op_info']['name'] + + try: + custom_flag = False + if 'impl_path' in kernel_info and kernel_info['impl_path'] is not None: + impl_path = os.path.realpath(kernel_info['impl_path']) + if os.path.isfile(impl_path): + path, file_name = os.path.split(impl_path) + op_name, _ = os.path.splitext(file_name) + impl_path = path + custom_flag = True + else: + impl_path = "" + _initialize(impl_path) + + inputs_args = get_args(kernel_info['op_info'], 'inputs') + outputs_args = get_args(kernel_info['op_info'], 'outputs') + attrs_args = get_args(kernel_info['op_info'], 'attrs') + kernel_name = kernel_info['op_info']['kernel_name'] + + if custom_flag: + op_module = __import__(op_name) + else: + op_module = __import__("impl."+op_name, globals(), locals(), [op_name], 0) + # get function + if build_type == op_pre_build: + # set op parameter + op_build_cfg_dis() + set_current_op_func_name(op_name) + set_current_op_name(kernel_name) + init_op_pattern() + set_op_params(*outputs_args, *attrs_args, kernel_name=kernel_name) + set_op_build_type('prebuild') + if custom_flag: + py_fn_name = kernel_info['op_info']['name'] + else: + py_fn_name = op_name + elif build_type == op_build: + if custom_flag: + py_fn_name = kernel_info['op_info']['name'] + else: + py_fn_name = op_name + else: + raise ValueError("function {} is not supported by Tbe op {}.".format(build_type, op_name)) + op_func = getattr(op_module, py_fn_name, None) + if op_func is None: + raise ValueError("Op:{} function {} is not supported by Tbe.".format(op_name, build_type)) + + # pre build + if build_type == op_pre_build: + op_func(*inputs_args, *outputs_args, *attrs_args, kernel_name) + # disable only pattern configuration + op_build_cfg_en() + return get_op_pattern() + + # call function + return op_func(*inputs_args, *outputs_args, *attrs_args, kernel_name=kernel_name) + + except Exception as e: + if build_type == op_pre_build: + op_build_cfg_en() + raise RuntimeError(e) + + +def compile_fusion_op(json_str): + """ + compile fusion op with input args json_str + + Args: + json_str (str): op function input args + + Raises: + Exception: If specific keyword is not found. + """ + args = json.loads(json_str) + if 'fusion_op' not in args or not args['fusion_op']: + raise ValueError("Json string Errors, key:fusion_op not found.") + if 'prebuild_ops' not in args or not args['prebuild_ops']: + raise ValueError("Json string Errors, key:prebuild_ops not found.") + + pre_build_op_list = args['prebuild_ops'] + for op in pre_build_op_list: + build_op(op_pre_build, json.dumps(op)) + fusion_op_arg = args['fusion_op'] + return fusion_op(json.dumps(fusion_op_arg)) + + +def compile_with_json(json_str): + """ + Compile tbe with json. + + Args: + json_str (str): jason file path. + + """ + json_info = json.loads(json_str) + if "fusion_op" in json_info: + ret = compile_fusion_op(json_str) + else: + ret = build_op(op_build, json_str) + return ret + + +if __name__ == "__main__": + in_args = sys.stdin.readline() + compile_with_json(in_args) diff --git a/mindspore/_extends/parallel_compile/tbe_compiler/tbe_process.py b/mindspore/_extends/parallel_compile/tbe_compiler/tbe_process.py new file mode 100644 index 00000000000..91207b4f81a --- /dev/null +++ b/mindspore/_extends/parallel_compile/tbe_compiler/tbe_process.py @@ -0,0 +1,209 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""tbe process""" +import traceback +import multiprocessing +import subprocess +import sys +import os +import json +from .common import check_kernel_info, get_args, get_build_in_impl_path + +build_in_impl_path = get_build_in_impl_path() + + +def create_tbe_parallel_compiler(): + """ + create TBEParallelCompiler object + + Returns: + TBEParallelCompiler + """ + return compile_pool + +def op_select_format(op_json: str): + """ + call op's op_select_format to get op supported format + + Args: + op_json (str): json string of the op + + Returns: + op supported format + """ + ret = "" + kernel_info = json.loads(op_json) + check_kernel_info(kernel_info) + + # import module + op_name = kernel_info['op_info']['name'] + impl_path = build_in_impl_path + custom_flag = False + if 'impl_path' in kernel_info and kernel_info['impl_path'] is not None: + op_impl_path = os.path.realpath(kernel_info['impl_path']) + if os.path.isfile(op_impl_path): + path, file_name = os.path.split(op_impl_path) + op_name, _ = os.path.splitext(file_name) + impl_path = path + custom_flag = True + sys.path.insert(0, impl_path) + + if custom_flag: + op_module = __import__(op_name) + else: + op_module = __import__("impl." + op_name, globals(), locals(), [op_name], 0) + # get function + if not hasattr(op_module, "op_select_format"): + return "" + op_func = getattr(op_module, "op_select_format", None) + + # call function + inputs_args = get_args(kernel_info['op_info'], 'inputs') + outputs_args = get_args(kernel_info['op_info'], 'outputs') + attrs_args = get_args(kernel_info['op_info'], 'attrs') + kernel_name = kernel_info['op_info']['kernel_name'] + ret = op_func(*inputs_args, *outputs_args, *attrs_args, kernel_name=kernel_name) + return ret + + +def check_supported(op_json: str): + """ + call op's check_supported to check supported or not + + Args: + op_json (str): json string of the op + + Returns: + true or false + """ + ret = "" + kernel_info = json.loads(op_json) + check_kernel_info(kernel_info) + + # import module + op_name = kernel_info['op_info']['name'] + impl_path = build_in_impl_path + custom_flag = False + if 'impl_path' in kernel_info and kernel_info['impl_path'] is not None: + op_impl_path = os.path.realpath(kernel_info['impl_path']) + if os.path.isfile(op_impl_path): + path, file_name = os.path.split(op_impl_path) + op_name, _ = os.path.splitext(file_name) + impl_path = path + custom_flag = True + sys.path.insert(0, impl_path) + + if custom_flag: + op_module = __import__(op_name) + else: + op_module = __import__("impl." + op_name, globals(), locals(), [op_name], 0) + # get function + if not hasattr(op_module, "check_supported"): + return "" + op_func = getattr(op_module, "check_supported", None) + + # call function + inputs_args = get_args(kernel_info['op_info'], 'inputs') + outputs_args = get_args(kernel_info['op_info'], 'outputs') + attrs_args = get_args(kernel_info['op_info'], 'attrs') + kernel_name = kernel_info['op_info']['kernel_name'] + ret = op_func(*inputs_args, *outputs_args, *attrs_args, kernel_name=kernel_name) + return ret + + +def run_compiler(op_json): + """ + run compiler to compile op with subprocess + + Args: + op_json (str): json string of the op + + Returns: + result type, result. + """ + try: + tbe_compiler = os.path.join(os.path.split(os.path.realpath(__file__))[0], "compiler.py") + subprocess.run([sys.executable, tbe_compiler], input=op_json, timeout=300, + text=True, capture_output=True, check=True) + return "Success", "Success" + except subprocess.TimeoutExpired: + tb = traceback.format_exc() + return "TBEException", "CompileTimeOut: " + tb + "\ninput_args: " + op_json + except subprocess.CalledProcessError as e: + return "TBEException", "CompileProcessFailed:\n" + e.stdout + "\n" + e.stderr + "\ninput_args: " + op_json + + +class CompilerPool: + """compiler pool""" + + def __init__(self): + processes = multiprocessing.cpu_count() + if processes > 32: + processes = 32 + self.__pool = multiprocessing.Pool(processes=processes) + self.__next_task_id = 1 + self.__running_tasks = [] + + def __del__(self): + if self.__pool is not None: + self.__pool.terminate() + self.__pool.join() + del self.__pool + + def exit(self): + return + # self.__pool.terminate() + # self.__pool.join() + # if self.__pool is not None: + # del self.__pool + + def start_compile_op(self, op_json): + """ + start compile op async. + + Args: + op_json (str): json string of the op + + Returns: + int, task id(>0). -1 if error + """ + task_id = self.__next_task_id + self.__next_task_id = self.__next_task_id + 1 + task_future = self.__pool.apply_async(func=run_compiler, args=(op_json,)) + self.__running_tasks.append((task_id, task_future)) + return task_id + + def wait_one(self): + """ + wait until a compile task finish + + Returns: + int, id of the finished task. -1 if error,0 if no unfinished task + str, result of compile task + """ + ret = 0, "Success" + if self.__running_tasks: + task_id, task_future = self.__running_tasks.pop(0) + ret_type, result = task_future.get(330) + if ret_type == "Success": + ret = task_id, "Success" + elif ret_type in ("Exception", "TBEException"): + ret = task_id, ret_type + ":" + result + else: + ret = task_id, "Exception: Not support return type:" + str(ret_type) + return ret + + +compile_pool = CompilerPool() diff --git a/mindspore/_extends/parse/__init__.py b/mindspore/_extends/parse/__init__.py new file mode 100644 index 00000000000..f8a34057c52 --- /dev/null +++ b/mindspore/_extends/parse/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Interfaces for parser module in c++. +""" + +from .parser import (Parser, create_obj_instance, generate_scope, + get_bprop_method_of_class, get_class_instance_type, + get_class_member_namespace_symbol, + get_dataclass_attributes, get_dataclass_methods, + get_module_namespace, get_obj_type, get_object_key, + get_parse_method_of_class, get_scope_name, + is_class_member, parse_cb, resolve_symbol) +from .serialize import * + +__all__ = ['parse_cb', 'get_parse_method_of_class', 'get_bprop_method_of_class', 'resolve_symbol', + 'get_object_key', 'get_class_instance_type', 'is_class_member', 'get_obj_type', + 'create_obj_instance', 'get_module_namespace', 'get_class_member_namespace_symbol', + 'Parser', 'get_dataclass_attributes', 'get_dataclass_methods', 'dump_obj', 'load_obj', + 'get_dataclass_methods', 'get_scope_name'] diff --git a/mindspore/_extends/parse/namespace.py b/mindspore/_extends/parse/namespace.py new file mode 100644 index 00000000000..8d8b6fd30e5 --- /dev/null +++ b/mindspore/_extends/parse/namespace.py @@ -0,0 +1,110 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Define the namespace of parse.""" + +import builtins + + +class Namespace: + """ + Base class of namespace for resolve variables. + + Args: + name (str): The namespace's name. + dicts (dict): A list of dict containing the namespace's variable. + """ + def __init__(self, name, *dicts): + self.name = name + self.dicts = dicts + + def __contains__(self, name): + for d in self.dicts: + if name in d: + return True + return False + + def __getitem__(self, name): + for d in self.dicts: + if name in d: + return d[name] + raise NameError(name) + + def __repr__(self): + return f'Namespace:{self.name}' + + +class CellNamespace(Namespace): + """ + Namespace for Cell object. + + Args: + name (str): Valid module name, it can be imported. + """ + def __init__(self, name): + mod_dict = vars(__import__(name, fromlist=['_'])) + builtins_dict = vars(builtins) + super().__init__(name, mod_dict, builtins_dict) + + def __getstate__(self): + return (self.name,) + + def __setstate__(self, state): + name, = state + mod_dict = vars(__import__(name, fromlist=['_'])) + builtins_dict = vars(builtins) + super().__init__(name, mod_dict, builtins_dict) + + +class ClosureNamespace(Namespace): + """ + Namespace for function closure. + + Args: + fn (Function): A python function. + """ + def __init__(self, fn): + name = f'{fn.__module__}..<{fn.__name__}>' + names = fn.__code__.co_freevars + cells = fn.__closure__ + ns = dict(zip(names, cells or ())) + super().__init__(name, ns) + + def __getitem__(self, name): + d, = self.dicts + try: + return d[name].cell_contents + except ValueError: + raise UnboundLocalError(name) + + +class ClassMemberNamespace(Namespace): + """ + Namespace of a class's closure. + + Args: + obj (Object): A python class object. + """ + def __init__(self, obj): + label = f'{obj.__module__}..<{obj.__class__.__name__}::{id(obj)}>' + super().__init__(label, obj) + + def __getitem__(self, name): + d, = self.dicts + try: + return getattr(d, name) + except ValueError: + raise UnboundLocalError(name) diff --git a/mindspore/_extends/parse/parser.py b/mindspore/_extends/parse/parser.py new file mode 100644 index 00000000000..e88c9c15e92 --- /dev/null +++ b/mindspore/_extends/parse/parser.py @@ -0,0 +1,488 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The module of parser python object, called by c++.""" + +import ast +import types +import inspect +from textwrap import dedent +from dataclasses import is_dataclass +import asttokens +import mindspore.nn as nn +from mindspore import log as logger +from mindspore import ops +from mindspore.common.dtype import pytype_to_dtype +from mindspore.common.api import _MindSporeFunction +from .namespace import CellNamespace, ClosureNamespace, ClassMemberNamespace +from .resources import parse_object_map, convert_object_map, trope_ns, SYMBOL_UNDEFINE, NO_IMPLEMENT + +# define return value +RET_SUCCESS = 0 +RET_FAILURE = 0xFF + +# define resolve type +RESOLVE_TYPE_NONE = 0 # resolve None +RESOLVE_TYPE_FUNCTION = 1 # resolve function +RESOLVE_TYPE_METHOD = 2 # resolve class method +RESOLVE_TYPE_CLASS_TYPE = 3 # resolve class type +RESOLVE_TYPE_CLASS_INSTANCE = 4 # resolve the class instance of common class +RESOLVE_TYPE_INVALID = 0xFF + +# define the class instance detail type +# When the type is RESOLVE_TYPE_CLASS_INSTANCE +CLASS_INSTANCE_TYPE_CELL = 0 # class instance type is Cell +CLASS_INSTANCE_TYPE_PRIMITIVE = 1 # class instance type is Primitive +CLASS_INSTANCE_TYPE_INVALID = 0xFF + +# Ast main type +AST_MAIN_TYPE_STMT = 0 # ast.Stmt +AST_MAIN_TYPE_EXPR = 1 # ast.Expr +AST_MAIN_TYPE_SLICE = 2 # ast.Slice +AST_MAIN_TYPE_UNKNOWN = 0xFF # unknown + +# Ast sub type +AST_SUB_TYPE_AND = 3 # ast.And +AST_SUB_TYPE_OR = 4 # ast.Or +AST_SUB_TYPE_NAME = 5 # ast.Name +AST_SUB_TYPE_TUPLE = 6 # ast.Tuple +AST_SUB_TYPE_SUBSCRIPT = 7 # ast.Subscript +AST_SUB_TYPE_STARRED = 8 # ast.Starred +AST_SUB_TYPE_UNKNOWN = 0xFF # unknown + +# Process expr statement white list +# add as needed, eg: "clear", "extend", "insert", "remove", "reverse" +parse_expr_statement_white_list = ( + "append", +) + + +def parse_cb(func, parse_method=None): + """Implements the function of parse.""" + return Parser(func, parse_method) + + +def get_parse_method_of_class(obj, parse_method=None): + """ + Het parse method of class. + + Args: + obj(Object): Instance of class. + parse_method(str): Save the method name. Cell object has default method named 'construct'. + + Returns: + Function, obj's method. + """ + method = None + method_name = None + if parse_method is not None: + method_name = parse_method + else: + if isinstance(obj, nn.Cell): + method_name = "construct" + if method_name is not None: + if hasattr(obj, method_name): + method = getattr(obj, method_name) + return method + + +def get_bprop_method_of_class(obj, parse_method=None): + """ + Get bprop method of class. + + Args: + obj (Object): Instance of class. + parse_method(str): Save the method name. Cell object has default method named 'bprop'. + + Returns: + Function, obj's method. + """ + method = None + if isinstance(obj, nn.Cell): + method_name = "bprop" + if hasattr(obj, method_name): + method = getattr(obj, method_name) + return method + + +def resolve_symbol(namespace, symbol): + """ + Resolve a symbol. + + Note: + Can't get function when use closure function. So save the fn on namespace. + + Args: + namespace (Object): Symbol's namespace. + symbol (str): Need resolve symbol. + + Returns: + Object, resolve result of symbol. + """ + # All exceptions need to be caught in this function + try: + resolve_ = namespace[symbol] + + # list and dict is not hashable ,it can not be key for the map, just return the result + if isinstance(resolve_, (list, dict)): + return resolve_ + + # dataclass may not be hashable + if getattr(resolve_, "__hash__") is None: + return resolve_ + + # If need trope the obj + if resolve_ in convert_object_map: + resolve_ = convert_object_map.get(resolve_) + logger.debug("convert resolve = %r", resolve_) + if resolve_ == NO_IMPLEMENT: + raise NotImplementedError("not implemented for ", str(symbol)) + except Exception as e: + if isinstance(e, NotImplementedError): + raise e + resolve_ = None + logger.debug("resolve exception occurred, value = %r", e) + logger.debug("resolve type is invalid, namespace = %s, symbol = %s", + namespace.__str__(), symbol) + if isinstance(resolve_, _MindSporeFunction): + logger.debug("resolve class _MindSporeFunction, resolve fn instead.") + resolve_ = resolve_.fn + return resolve_ + + +def generate_scope(obj): + """Generate the scope for every cell object in the network.""" + if isinstance(obj, nn.Cell): + obj.generate_scope() + + +def get_scope_name(obj): + """Returns the scope of a cell object in one network.""" + if isinstance(obj, nn.Cell): + return obj.get_scope() + return None + + +def get_object_key(obj): + """Return the function key: module + name.""" + obj_key = "" + if hasattr(obj, "__name__"): + if hasattr(obj, "cell_init_args"): + obj_key = "%s_ID" % (str(obj.__class__.__name__) + str(obj.__name__) + obj.cell_init_args) + obj_id = "%s_ID%d" % (str(obj.__class__.__name__) + str(obj.__name__), id(obj)) + else: + if hasattr(obj, "cell_init_args"): + obj_key = "%s_ID" % (str(obj.__class__.__name__) + obj.cell_init_args) + obj_id = "%s_ID%d" % (str(obj.__class__.__name__), id(obj)) + logger.debug("obj_key %s obj_id = %s", obj_key, obj_id) + + # method has same id of different instance + if isinstance(obj, types.MethodType): + method_instance = obj.__self__ + instance_id = "%s_ID%d" % (str(method_instance.__class__.__name__), id(method_instance)) + obj_id = instance_id + obj_id + return obj_id, obj_key + + +def is_class_member(node): + """Check the attr is class member variable.""" + type_ = node.__class__.__name__ + if type_ == "Attribute": + if not hasattr(node.value, "id"): + return False + id_ = node.value.id + if id_ == "self": + return True + return False + + +def get_obj_type(obj): + """Get the obj type.""" + obj_type = RESOLVE_TYPE_INVALID + if obj is None: + obj_type = RESOLVE_TYPE_NONE + elif isinstance(obj, types.FunctionType): + obj_type = RESOLVE_TYPE_FUNCTION + elif isinstance(obj, types.MethodType): + obj_type = RESOLVE_TYPE_METHOD + elif isinstance(obj, type): + obj_type = RESOLVE_TYPE_CLASS_TYPE + elif _is_class_instance(obj): + obj_type = RESOLVE_TYPE_CLASS_INSTANCE + else: + # here for ndarray, just print its shape (in case of the array to large and print many data in screen) + is_ndarray = type(obj).__name__ == 'ndarray' and hasattr(obj, 'shape') + raise TypeError(f'Invalid object with type `{type(obj)}` and {"shape" if is_ndarray else "value"} ' + f'`{obj.shape if is_ndarray else obj}`.') + return obj_type + + +def get_class_instance_type(obj): + """Get the class instance detail type.""" + # check the obj type + logger.debug("Get the class type(%r)", obj) + class_type = CLASS_INSTANCE_TYPE_INVALID + if _is_class_instance(obj): + if isinstance(obj, nn.Cell): + class_type = CLASS_INSTANCE_TYPE_CELL + elif isinstance(obj, ops.Primitive): + class_type = CLASS_INSTANCE_TYPE_PRIMITIVE + # Add the other type base requirement + return class_type + + +def _is_class_instance(obj): + """Confirm the obj is class instance.""" + return isinstance(obj, (nn.Cell, ops.Primitive)) or _is_dataclass_instance(obj) + + +def _is_dataclass_instance(obj): + """check whether a class is an instance of a dataclass (and not a dataclass itself)""" + return is_dataclass(obj) and not isinstance(obj, type) + + +def create_obj_instance(cls_type, args_tuple=None): + """Create python instance.""" + obj = None + if isinstance(cls_type, type): + # check the type, now only support nn.Cell and Primitive + if issubclass(cls_type, (nn.Cell, ops.Primitive)): + if args_tuple is not None: + obj = cls_type(*args_tuple) + else: + obj = cls_type() + return obj + + +def get_module_namespace(obj): + """Get the module's namespace.""" + logger.debug("get module namespace, module = %r", obj) + mod_namespace = None + if isinstance(obj, types.ModuleType): + mod_namespace = CellNamespace(obj.__name__) + else: + logger.warning("Module(%r) is invalid, get namespace failure!", obj) + return mod_namespace + + +def get_class_member_namespace_symbol(obj): + """Get obj class member type.""" + logger.debug("get class instance namespace, object = %r", obj) + class_namespace = ClassMemberNamespace(obj) + logger.debug("class namesapce = %r", class_namespace) + return class_namespace + + +def get_dataclass_attributes(cls): + """Get attributes of dataclass.""" + fields = cls.__dataclass_fields__ + attributes = {name: pytype_to_dtype(field.type) + for name, field in fields.items()} + return attributes + + +def get_dataclass_methods(cls): + """Get functions of dataclass.""" + methods = {name: getattr(cls, name) + for name in dir(cls) + if isinstance(getattr(cls, name), (types.FunctionType,))} + return methods + + +class Parser: + """ + Parser python code to ast tree. + + Args: + fn(FunctionType/MethodType): Need parse object instance. + parse_method(ExtendInfoOfParseObj): Extend information for parse the function. + """ + def __init__(self, fn: (types.FunctionType, types.MethodType), parse_method=None) -> None: + self.fn = fn + self.parse_method = parse_method + _, self.line_offset = inspect.getsourcelines(self.fn) + self.filename: str = inspect.getfile(self.fn) + + # Used to resolve the function's globals Namespace. + self.global_namespace = CellNamespace(fn.__module__) + self.function_module = fn.__module__ + # Used to resolve the function's nonlocals. + self.closure_namespace = ClosureNamespace(fn) + self.function_name = fn.__name__ + self.col_offset = 0 + + def parse(self): + """Parse the function or method.""" + logger.debug("fn = %r", self.fn) + tree = None + if isinstance(self.fn, (types.FunctionType, types.MethodType)): + original_src = inspect.getsource(self.fn) + src = dedent(original_src) + self.col_offset = \ + len(original_src.split('\n')[0]) - len(src.split('\n')[0]) + logger.debug("get source = %s", src) + tree = asttokens.ASTTokens(src, parse=True).tree + else: + logger.error("Fn type is invalid") + return tree + + def get_args(self, node): + """Get the arg of parse object.""" + args = [] + # process position args + for arg in node.args.args: + args.append(arg) + + # process kwonlyargs: kwonlyargs is append after position args + if node.args.kwonlyargs: + for kwarg in node.args.kwonlyargs: + args.append(kwarg) + # process vararg: vararg is append after kwonlyargs + if node.args.vararg: + args.append(node.args.vararg) + # process kwarg: kwarg is append after vararg + if node.args.kwarg: + args.append(node.args.kwarg) + return args + + def get_args_default_values(self, node): + """get the args'default values of parse object.""" + nondefaults = [None] * (len(node.args.args) - len(node.args.defaults)) + defaults = nondefaults + node.args.defaults + node.args.kw_defaults + if node.args.vararg: + defaults.append(None) + if node.args.kwarg: + defaults.append(None) + return defaults + + def get_node_type(self, node): + """Process an ast node.""" + method_name = f'{node.__class__.__name__}' + node_type = [method_name] + # judge the ast main type + if isinstance(node, ast.stmt): + node_type.append(AST_MAIN_TYPE_STMT) + elif isinstance(node, (ast.expr, ast.slice)) or node is None: + # ast.slice and ast.expr should be expr + node_type.append(AST_MAIN_TYPE_EXPR) + else: + node_type.append(AST_MAIN_TYPE_UNKNOWN) + return node_type + + def get_ast_type(self, node): + """Get the ast type.""" + ast_type = AST_SUB_TYPE_UNKNOWN + if isinstance(node, ast.And): + ast_type = AST_SUB_TYPE_AND + elif isinstance(node, ast.Or): + ast_type = AST_SUB_TYPE_OR + elif isinstance(node, ast.Name): + ast_type = AST_SUB_TYPE_NAME + elif isinstance(node, ast.Tuple): + ast_type = AST_SUB_TYPE_TUPLE + elif isinstance(node, ast.Subscript): + ast_type = AST_SUB_TYPE_SUBSCRIPT + elif isinstance(node, ast.Starred): + ast_type = AST_SUB_TYPE_STARRED + else: + ast_type = AST_SUB_TYPE_UNKNOWN + return ast_type + + def get_namespace_symbol(self, var: str): + """Get symbol type and namespace and symbol.""" + if var in self.closure_namespace: + ops_info = (self.closure_namespace, var) + logger.debug("in closure_namespace") + elif var in self.global_namespace: + ops_info = (self.global_namespace, var) + logger.debug("in global_namespace") + else: + ops_info = parse_object_map.get(SYMBOL_UNDEFINE) + ops_info = [ops_info[0], var] + return ops_info + + def get_operation_namespace_symbol(self, var: str): + """Get operation namespace and symbol.""" + ops_info = (trope_ns, var) + logger.debug("get operation ops info = %r", ops_info) + return ops_info + + def get_ast_namespace_symbol(self, obj): + """Get obj type and namespace and symbol.""" + # step 1:get symbol from object map + ops_info = parse_object_map.get(type(obj), SYMBOL_UNDEFINE) + logger.debug("ops info = %r", ops_info) + return ops_info + + def get_location(self, node): + """ + Get location of node start and end line no. + + Args: + node: AST op node or tuple or List. This is a node in the ANF diagram, + here is the code location to get this node. + + Returns: + List, [fileName, linestart, colstart, lineend, colend]. + """ + ret = [self.filename] + err_exit = 0 + if isinstance(node, (list, tuple)): + node_size = len(node) + if node_size == 0: + err_exit = 1 + else: + start_node = node[0] + end_node = node[-1] + else: + start_node = node + end_node = node + + if err_exit == 0: + if hasattr(start_node, "lineno") and \ + hasattr(end_node, "col_offset"): + start_lineno, start_colno = start_node.first_token.start + end_lineno, end_colno = end_node.last_token.end + start_lineno += self.line_offset - 1 + start_colno += self.col_offset + end_lineno += self.line_offset - 1 + end_colno += self.col_offset + ret = ret + [start_lineno, start_colno, end_lineno, end_colno] + else: + ret = ret + [0, 0, 0, 0] + return ret + + def expand_expr_statement(self, node): + """ + Process the expr statement and expand it. + + Returns: + tuple, (True, expr.value, x)/(False, None, None). + """ + if isinstance(node, ast.Expr) and hasattr(node, "value"): + expr_value = node.value + if isinstance(expr_value, ast.Call): + func = expr_value.func + if isinstance(func, ast.Attribute) and \ + hasattr(func, "attr") and \ + hasattr(func, "value"): + method = func.attr + target = func.value + if method in parse_expr_statement_white_list: + logger.debug("Expand expr, target:%s, method:%s", target, method) + return True, expr_value, target + return True, expr_value + return False, None, None diff --git a/mindspore/_extends/parse/resources.py b/mindspore/_extends/parse/resources.py new file mode 100644 index 00000000000..1f6f4b91b55 --- /dev/null +++ b/mindspore/_extends/parse/resources.py @@ -0,0 +1,137 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Resources for ast tree parse.""" +import ast +import math +from mindspore.ops.composite import multitype_ops +from mindspore.ops import functional as F, composite as C +from . import standard_method as M +from . import trope as T +from .namespace import CellNamespace + + +# namespace define +functional_ns = CellNamespace('mindspore.ops.functional') +composite_ns = CellNamespace('mindspore.ops.composite') +trope_ns = CellNamespace('mindspore._extends.parse.trope') + +NO_IMPLEMENT = None # not implemented +SYMBOL_UNDEFINE = 0xFF # Undefined var and function + +# ops map: {op.type:(Namespace, symbol)} +# Some space set aside for readability of code +parse_object_map = { + # ast grammar + ast.Add: (trope_ns, 'add'), + ast.Sub: (trope_ns, 'sub'), + ast.Mult: (trope_ns, 'mul'), + ast.Div: (trope_ns, 'truediv'), + ast.FloorDiv: (trope_ns, 'floordiv'), + ast.Mod: (trope_ns, 'mod'), + ast.Pow: (trope_ns, 'pow'), + ast.MatMult: (trope_ns, 'matmul'), + ast.LShift: (trope_ns, 'lshift'), + ast.RShift: (trope_ns, 'rshift'), + ast.And: (trope_ns, 'and_'), + ast.Or: (trope_ns, 'or_'), + ast.BitXor: (trope_ns, 'xor'), + ast.UAdd: (trope_ns, 'pos'), + ast.USub: (trope_ns, 'neg'), + ast.Invert: (trope_ns, 'invert'), + ast.Not: (trope_ns, 'not_'), + ast.Eq: (trope_ns, 'eq'), + ast.NotEq: (trope_ns, 'ne'), + ast.Lt: (trope_ns, 'lt'), + ast.Gt: (trope_ns, 'gt'), + ast.LtE: (trope_ns, 'le'), + ast.GtE: (trope_ns, 'ge'), + ast.Is: (trope_ns, 'is_'), + ast.IsNot: (trope_ns, 'is_not'), + ast.In: (trope_ns, 'contains'), + ast.NotIn: (trope_ns, 'not_contains'), + + # operation symbol type + 'getitem': (composite_ns, 'getitem'), + 'ms_iter': (composite_ns, 'ms_iter'), + 'ms_next': (composite_ns, 'ms_next'), + 'hasnext': (composite_ns, 'hasnext'), + + # undefined type + SYMBOL_UNDEFINE: (None, 'undefine'), +} + +# convert map: {obj:(Namespace, symbol)} +# Escape an object to another object, eg: system function(len,xxx) +# Some space set aside for readability of code +convert_object_map = { + T.add: multitype_ops.add, + T.sub: multitype_ops.sub, + T.mul: multitype_ops.mul, + T.truediv: multitype_ops.div, + T.getitem: multitype_ops.getitem, + T.floordiv: NO_IMPLEMENT, + T.mod: F.scalar_mod, + T.pow: F.scalar_pow, + T.matmul: F.dot, + T.lshift: NO_IMPLEMENT, + T.rshift: NO_IMPLEMENT, + T.and_: multitype_ops.logical_and, + T.or_: multitype_ops.logical_or, + T.xor: NO_IMPLEMENT, + T.pos: F.scalar_uadd, + T.neg: multitype_ops.negative, + T.invert: NO_IMPLEMENT, + T.not_: F.bool_not, + T.eq: multitype_ops.equal, + T.ne: F.scalar_ne, + T.lt: multitype_ops.less, + T.gt: F.scalar_gt, + T.le: multitype_ops.less_equal, + T.ge: F.scalar_ge, + T.is_: F.is_, + T.is_not: F.is_not, + T.contains: NO_IMPLEMENT, + T.not_contains: NO_IMPLEMENT, + + # system function + T.len: M.ms_len, + T.bool: M.bool_, + T.map: C.HyperMap(), + T.partial: F.partial, + T.zip: C.zip_operation, + + # custom define operation + T.iter: M.ms_iter, + T.next: M.ms_next, + T.hasnext: M.hasnext, + T.setitem: M.setitem, + + T.make_tuple: F.make_tuple, + T.make_dict: F.make_dict, + T.make_list: F.make_list, + T.make_slice: F.make_slice, + T.range: F.make_range, + + # lib function + math.floor: NO_IMPLEMENT, + math.trunc: NO_IMPLEMENT, + math.exp: NO_IMPLEMENT, + math.log: F.scalar_log, + math.sin: NO_IMPLEMENT, + math.cos: NO_IMPLEMENT, + math.tan: NO_IMPLEMENT, +} diff --git a/mindspore/_extends/parse/serialize.py b/mindspore/_extends/parse/serialize.py new file mode 100644 index 00000000000..6fecab45cbd --- /dev/null +++ b/mindspore/_extends/parse/serialize.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The functions in this file is used to dump and load python object in anf graphs.""" + +import pickle +import os +import stat + + +def dump_obj(obj, path): + """Dump object to file.""" + + file_name = hex(id(obj)) + file_path = path + file_name + with open(file_path, 'wb') as f: + os.chmod(file_path, stat.S_IWUSR | stat.S_IRUSR) + pickle.dump(obj, f) + return file_name + + +def load_obj(file_path): + """Load object from file.""" + obj = None + try: + real_file_path = os.path.realpath(file_path) + except Exception as ex: + raise RuntimeError(ex) + with open(real_file_path, 'rb') as f: + obj = pickle.load(f) + return obj + + +__all__ = ['dump_obj', 'load_obj'] diff --git a/mindspore/_extends/parse/standard_method.py b/mindspore/_extends/parse/standard_method.py new file mode 100644 index 00000000000..2c94240ba2c --- /dev/null +++ b/mindspore/_extends/parse/standard_method.py @@ -0,0 +1,235 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""standard_method""" +from dataclasses import dataclass +from ...ops import functional as F +from ...ops import operations as P +from ...ops.composite import tail, core, MultitypeFuncGraph, env_get, hyper_add, \ + zeros_like, ones_like +from ...ops.composite.base import _append + +__all__ = ['MultitypeFuncGraph', 'env_get', 'hyper_add', 'zeros_like', 'ones_like'] + +trans = P.Transpose() + + +def transpose(x): + """Implementation of `transpose`.""" + shape = F.shape(x) + length = F.tuple_len(shape) + perm = F.make_range(0, length) + revert_perm = F.tuple_reversed(perm) + out = trans(x, revert_perm) + return out + + +def getitem(data, item): + """Implementation of `getitem`.""" + return data.__getitem__(item) + + +def setitem(data, item, value): + """Implementation of `setitem`.""" + return data.__setitem__(item, value) + + +def ms_iter(xs): + """Implementation of `iter`.""" + return xs.__ms_iter__() + + +def ms_next(it): + """Implementation of `next`.""" + return it.__ms_next__() + + +def hasnext(it): + """Implementation of `hasnext`.""" + return it.__ms_hasnext__() + + +def ms_len(data): + """Implementation of `len`.""" + return data.__len__() + + +def floor(x): + """Implementation of `floor`.""" + return x.__floor__() + + +def trunc(x): + """Implementation of `trunc`.""" + return x.__trunc__() + + +def uadd(x): + """Implementation of `uadd`.""" + return x.__pos__() + + +def usub(x): + """Implementation of `usub`.""" + return x.__neg__() + + +def scalar_truediv(x, y): + """Implementation of `scalar_truediv`.""" + return x.__truediv__(y) + + +def scalar_floordiv(x, y): + """Implementation of `scalar_floordiv`.""" + return x.__floordiv__(y) + + +def bool_(x): + """Implementation of `bool`.""" + return x.__bool__() + + +def tensor_bool(x): + """return immedate x, x is a tensor of bool value""" + return x + + +def and_(x, y): + """Implementation of `and` (`&`).""" + return x.__and__(y) + + +def or_(x, y): + """Implementation of `or` (`|`).""" + return x.__or__(y) + + +def matmul(x, y): + """Implementation of `matmul` (`@`).""" + return x.__matmul__(y) + + +def float_bool(x): + """Implementation of `float_bool`.""" + return x != 0.0 + + +def int_bool(x): + """Implementation of `int_bool`.""" + return x != 0 + + +def str_bool(x): + """Implementation of `str_bool`.""" + if x == "": + return False + return True + + +def list_bool(x): + """Implementation of `tuple_bool`.""" + return len(x) != 0 + + +def tuple_bool(x): + """Implementation of `tuple_bool`.""" + return len(x) != 0 + + +def dict_bool(x): + """Implementation of `dict_bool`.""" + return len(x) != 0 + + +def none_bool(x): + """Implementation of `none_bool`.""" + return False + + +def float_floordiv(x, y): + """Implementation of `float_floordiv`.""" + return floor(x / y) + + +############# +# Iteration # +############# + + +@dataclass(frozen=True) +class SequenceIterator: + """ + SequenceIterator is a util dataclass for iterating sequence object. + + Iterator to use for sequences like List, Array. + """ + + idx: int + seq: list + + @core(ignore_values=True) + def __ms_hasnext__(self): + """Whether the index is past the length of the sequence.""" + return self.idx < ms_len(self.seq) + + @core(ignore_values=True) + def __ms_next__(self): + """Return the next element and a new iterator.""" + return self.seq[self.idx], SequenceIterator(self.idx + 1, self.seq) + + +def list_iter(xs): + """Iterator for List.""" + return SequenceIterator(0, xs) + + +def array_iter(xs): + """Iterator for Array.""" + return SequenceIterator(0, xs) + + +def tuple_next(xs): + """Next tuple.""" + return xs[0], tail(xs) + + +def tuple_hasnext(xs): + """Whether the tuple is empty or not.""" + return len(xs) > 0 + + +def list_next(xs): + """Next list.""" + return xs[0], tail(xs) + + +def list_hasnext(xs): + """Whether the list is empty or not.""" + return len(xs) > 0 + + +def list_append(self_, item): + return _append(self_, item) + + +################# +# Array methods # +################# + + +def to_array(x): + """Implementation of `to_array`.""" + return x.__ms_to_array__() diff --git a/mindspore/_extends/parse/trope.py b/mindspore/_extends/parse/trope.py new file mode 100644 index 00000000000..9f8f67fba59 --- /dev/null +++ b/mindspore/_extends/parse/trope.py @@ -0,0 +1,93 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Trope some system function symbol to ops.""" + +# This operation function is not meant to be called directly + +# support operator symbol, ast +from operator import ( # noqa + add, sub, mul, truediv, floordiv, mod, eq, ne, lt, gt, le, ge, pos, neg, + not_, and_, or_, xor, lshift, rshift, invert, is_, is_not, contains, + matmul, getitem, setitem +) + +# support system function call +from builtins import ( # noqa + bool, getattr, setattr, len, iter, next, pow, range, map, zip +) + +# support functools +from functools import ( # noqa + partial +) + +# support numpy symbol +from numpy import ( # noqa + exp, log, sin, cos, tan +) + +__all__ = ['add', 'sub', 'mul', 'truediv', 'floordiv', 'mod', 'eq', 'ne', 'lt', 'gt', 'le', 'ge', 'pos', 'neg', + 'not_', 'and_', 'or_', 'xor', 'lshift', 'rshift', 'invert', 'is_', 'is_not', 'contains', + 'matmul', 'getitem', 'setitem', + 'bool', 'getattr', 'setattr', 'len', 'iter', 'next', 'pow', 'range', 'map', 'zip', + 'partial', + 'exp', 'log', 'sin', 'cos', 'tan'] + + +def make_tuple(*elts): # pragma: no cover + """Tuple builder.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def make_dict(key, value): # pragma: no cover + """Dict builder.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def make_list(*elts): # pragma: no cover + """List builder.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def make_slice(*elts): # pragma: no cover + """Slice builder.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def make_range(*elts): # pragma: no cover + """Range tuple builder.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def switch(cond, tb, fb): # pragma: no cover + """Switch statement, returns one of the two values.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def hasnext(it): # pragma: no cover + """Hasnext function.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def to_array(x): # pragma: no cover + """The to_array function.""" + raise RuntimeError('This operation is not meant to be called directly.') + + +def not_contains(x): # pragma: no cover + """Not in function.""" + raise RuntimeError('This operation is not meant to be called directly.') diff --git a/mindspore/_extends/pynative_helper.py b/mindspore/_extends/pynative_helper.py new file mode 100644 index 00000000000..0b93ab926b5 --- /dev/null +++ b/mindspore/_extends/pynative_helper.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Pynative mode help module.""" +from inspect import signature +from functools import wraps + + +def args_type_check(*type_args, **type_kwargs): + """Check whether input data type is correct.""" + + def type_check(func): + sig = signature(func) + bound_types = sig.bind_partial(*type_args, **type_kwargs).arguments + + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal bound_types + bound_values = sig.bind(*args, **kwargs) + argument_dict = bound_values.arguments + if "kwargs" in bound_types: + bound_types = bound_types["kwargs"] + if "kwargs" in argument_dict: + argument_dict = argument_dict["kwargs"] + for name, value in argument_dict.items(): + if name in bound_types: + if value is not None and not isinstance(value, bound_types[name]): + raise TypeError('Argument {} must be {}'.format(name, bound_types[name])) + return func(*args, **kwargs) + + return wrapper + + return type_check diff --git a/mindspore/_extends/utils.py b/mindspore/_extends/utils.py new file mode 100644 index 00000000000..8469ddda8b8 --- /dev/null +++ b/mindspore/_extends/utils.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Some utils.""" +import hashlib +import logging +import os +import inspect +from functools import wraps + + +def cal_sha256(file_path): + """ + Calculate sha256 value of input file. + + Args: + file_path (str): file path to calculate. + + Returns: + str, returns sha256 value of input file or empty str if cal failed. + """ + buf_size = 64 * 1024 # once read 64kb + sha256 = hashlib.sha256() + file_real_path = os.path.realpath(file_path) + if not os.path.isfile(file_real_path) or not os.access(file_real_path, os.R_OK): + return "" + _, extension = os.path.splitext(file_path) + try: + if extension == '.ptx': + with open(file_real_path, 'r') as kf: + while True: + data = kf.read(buf_size) + if not data: + break + sha256.update(data.encode("utf-8")) + else: + with open(file_real_path, 'rb') as kf: + while True: + data = kf.read(buf_size) + if not data: + break + sha256.update(data) + except IOError: + logging.error("Open file %s failed.", file_path) + return "" + return sha256.hexdigest() + + +def cell_attr_register(fn=None, attrs=None): + """ + Cell init attributes register. + + Registering the decorator of the built-in operator cell __init__ + function will add save all the parameters of __init__ as operator attributes. + + Args: + fn (function): __init__ function of cell. + attrs (list(string) | string): attr list. + + Returns: + function, original function. + """ + def wrap_cell(fn): + @wraps(fn) + def deco(self, *args, **kwargs): + arguments = [] + if attrs is None: + bound_args = inspect.signature(fn).bind(self, *args, **kwargs) + arguments = bound_args.arguments + del arguments['self'] + arguments = arguments.values() + fn(self, *args, **kwargs) + if attrs is not None: + if isinstance(attrs, list): + for item in attrs: + if not isinstance(item, str): + raise ValueError(f"attr must be a string") + if hasattr(self, item): + arguments.append(getattr(self, item)) + elif isinstance(attrs, str): + if hasattr(self, attrs): + arguments = getattr(self, attrs) + else: + raise ValueError(f"attrs must be list or string") + self.cell_init_args = type(self).__name__ + str(arguments) + return deco + if fn is not None: + return wrap_cell(fn) + return wrap_cell diff --git a/mindspore/akg/__init__.py b/mindspore/akg/__init__.py new file mode 100644 index 00000000000..a0c0364bd67 --- /dev/null +++ b/mindspore/akg/__init__.py @@ -0,0 +1,63 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""__init__""" +from __future__ import absolute_import as _abs +import sys +import os + +def AKGAddPath(): + """akg add path.""" + pwd = os.path.dirname(os.path.realpath(__file__)) + tvm_path = os.path.realpath(pwd) + if tvm_path not in sys.path: + sys.path.insert(0, tvm_path) + else: + sys.path.remove(tvm_path) + sys.path.insert(0, tvm_path) + + +class AKGMetaPathFinder: + """class AKGMetaPath finder.""" + + def find_module(self, fullname, path=None): + """method akg find module.""" + if fullname.startswith("akg.tvm"): + rname = fullname[4:] + return AKGMetaPathLoader(rname) + if fullname.startswith("akg.topi"): + rname = fullname[4:] + return AKGMetaPathLoader(rname) + return None + + +class AKGMetaPathLoader: + """class AKGMetaPathLoader loader.""" + def __init__(self, rname): + self.__rname = rname + + def load_module(self, fullname): + if self.__rname in sys.modules: + sys.modules.pop(self.__rname) + AKGAddPath() + __import__(self.__rname, globals(), locals()) + self.__target_module = sys.modules[self.__rname] + sys.modules[fullname] = self.__target_module + return self.__target_module + + +sys.meta_path.insert(0, AKGMetaPathFinder()) + +from .op_build import op_build +from .message import compilewithjson diff --git a/mindspore/akg/gpu/__init__.py b/mindspore/akg/gpu/__init__.py new file mode 100644 index 00000000000..86334cfcd3f --- /dev/null +++ b/mindspore/akg/gpu/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""__init__""" +from .equal import Equal +from .equal import gpu_schedule_Equal +from .tile import Tile +from .tile import gpu_schedule_Tile +from .cast import Cast +from .cast import gpu_schedule_Cast +from .relu6 import ReLU6, gpu_schedule_ReLU6 +from .relu6_grad import ReLU6Grad, gpu_schedule_ReLU6Grad +from .squeeze import Squeeze, gpu_schedule_Squeeze +from .squeeze_grad import SqueezeGrad, gpu_schedule_SqueezeGrad +from .mean import SimpleMean, gpu_schedule_SimpleMean +from .mean_grad import SimpleMeanGrad, gpu_schedule_SimpleMeanGrad +from .mul import Mul, gpu_schedule_Mul diff --git a/mindspore/akg/gpu/cast.py b/mindspore/akg/gpu/cast.py new file mode 100644 index 00000000000..458772a8032 --- /dev/null +++ b/mindspore/akg/gpu/cast.py @@ -0,0 +1,43 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""cast""" +import logging +import akg.tvm +from akg.ops.math import cast +from akg.topi.generic import schedule_elemwise + +def Cast(x, dst_type): + """cast.""" + return cast.cast(x, dst_type) + + +def gpu_schedule_Cast(outs): + """ + gpu schedule for cast. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = akg.tvm.context(device, 0) + if not ctx.exist: + logging.info("Skip because %s is not enabled", device) + return None + with akg.tvm.target.create(device): + sch = schedule_elemwise(outs) + return sch diff --git a/mindspore/akg/gpu/default_schedule.py b/mindspore/akg/gpu/default_schedule.py new file mode 100644 index 00000000000..2e2892c0556 --- /dev/null +++ b/mindspore/akg/gpu/default_schedule.py @@ -0,0 +1,56 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""default schedule function for GPU""" +from queue import Queue + +import akg.tvm as tvm + +DEFAULT_GPU_THREAD = 1024 + + +def default_schedule(outs): + """ + default schedule function. + + Args: + outs (Union[tvm.tensor.Tensor, list[tvm.tensor.Tensor]]): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + if not isinstance(outs, tvm.tensor.Tensor) and not isinstance(outs, list): + raise ValueError("outs should be list of akg.tvm.tensor.Tensor or akg.tvm.tensor.Tensor") + device = 'cuda' + ctx = tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + outs_list = [outs] if isinstance(outs, tvm.tensor.Tensor) else outs + with tvm.target.create(device): + sch = tvm.create_schedule(outs_list[0].op) + outputs_tensor = Queue() + outputs_tensor.put(outs_list[0]) + op_list = [] + while not outputs_tensor.empty(): + out = outputs_tensor.get() + if out.op not in op_list and isinstance(out.op, tvm.tensor.ComputeOp): + op_list.append(out.op) + for input_tensor in out.op.input_tensors: + outputs_tensor.put(input_tensor) + for op in op_list: + stage = sch[op.output(0)] + bx, tx = stage.split(op.axis[0], factor=DEFAULT_GPU_THREAD) + stage.bind(bx, tvm.thread_axis("blockIdx.x")) + stage.bind(tx, tvm.thread_axis("threadIdx.x")) + return sch diff --git a/mindspore/akg/gpu/equal.py b/mindspore/akg/gpu/equal.py new file mode 100644 index 00000000000..05dce89622d --- /dev/null +++ b/mindspore/akg/gpu/equal.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""equal""" +import akg.tvm +from akg.ops.math import equal +from akg.topi.generic import schedule_elemwise + +def Equal(x, y): + """equal.""" + return equal.equal(x, y) + + +def gpu_schedule_Equal(outs): + """ + gpu schedule for Equal. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = akg.tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + with akg.tvm.target.create(device): + sch = schedule_elemwise(outs) + return sch diff --git a/mindspore/akg/gpu/mean.py b/mindspore/akg/gpu/mean.py new file mode 100644 index 00000000000..a68e9294091 --- /dev/null +++ b/mindspore/akg/gpu/mean.py @@ -0,0 +1,80 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""mean op compute and schedule""" +import akg.tvm as tvm +from akg.ops.math.mean import mean +from .default_schedule import DEFAULT_GPU_THREAD + +def Mean(x, axis=None, keepdims=True): + """mean.""" + outs = mean(x, axis, keepdims) + + # remove useless mean_output + if isinstance(outs, tuple): + outs = outs[0] + if outs.op.name == "mean_output": + outs = outs.op.input_tensors[0] + return outs + + +def gpu_schedule_Mean(outs): + """ + gpu schedule function for mean. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + out = outs[0] if isinstance(outs, list) else outs + + device = "cuda" + with tvm.target.create(device): + sch = tvm.create_schedule(out.op) + if out.op.name == "T_divide": + tensor_c = out + else: # squeeze + tensor_c = out.op.input_tensors[0] + + tensor_b = tensor_c.op.input_tensors[0] + if len(tensor_c.op.axis) >= 2: + sch[tensor_b].compute_at(sch[tensor_c], tensor_c.op.axis[1]) + else: + sch[tensor_b].compute_at(sch[tensor_c], tensor_c.op.axis[0]) + + bx, tx = sch[tensor_c].split(tensor_c.op.axis[0], factor=DEFAULT_GPU_THREAD) + sch[tensor_c].bind(bx, tvm.thread_axis("blockIdx.x")) + sch[tensor_c].bind(tx, tvm.thread_axis("threadIdx.x")) + return sch + +def SimpleMean(x): + """ + SimpleMean compute the mean of the input 4D Tensor over last two axises and keep reduced dimensions. + + Args: + x (tvm.tensor.Tensor): Tensor of type float16, float32. + + Returns: + tvm.tensor.Tensor, has the same type as x, output shape will be (a, b, 1, 1) if input Tensor x is (a, b, c, d). + """ + axis = (2, 3) + keepdims = True + return Mean(x, axis, keepdims) + + +def gpu_schedule_SimpleMean(outs): + """gpu schedule function for SimpleMean.""" + return gpu_schedule_Mean(outs) diff --git a/mindspore/akg/gpu/mean_grad.py b/mindspore/akg/gpu/mean_grad.py new file mode 100644 index 00000000000..ef77690a5dd --- /dev/null +++ b/mindspore/akg/gpu/mean_grad.py @@ -0,0 +1,90 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""mean_grad""" +import akg.tvm as tvm +import akg +from akg.ops.math import mean +from .default_schedule import DEFAULT_GPU_THREAD + + +def mean_ad(head, input_shape, axis, keepdims): + """mean autodiff.""" + tensor_a = tvm.placeholder(input_shape, head.dtype, "A") + tensor_b = mean.mean(tensor_a, axis, keepdims) + + # remove useless mean_output + if isinstance(tensor_b, tuple): + tensor_b = tensor_b[0] + if tensor_b.op.name == "mean_output": + tensor_b = tensor_b.op.input_tensors[0] + + jacs = list(akg.differentiate(tensor_b, [tensor_a], head)) + return jacs[0] + + +def MeanGrad(y_grad, input_shape, axis=None, keepdims=True): + """Mean Grad.""" + if axis is None and not keepdims: + raise ValueError("Mean not support (axis=None && keepdims=False) now") + return mean_ad(y_grad, input_shape, axis, keepdims) + + +def gpu_schedule_MeanGrad(outs): + """gpu schedule MeanGrad.""" + out = outs[0] if isinstance(outs, list) else outs + + device = "cuda" + with tvm.target.create(device): + sch = tvm.create_schedule(out.op) + tensor_c = out + tensor_b = tensor_c.op.input_tensors[0] + if len(tensor_c.op.axis) >= 2: + sch[tensor_b].compute_at(sch[tensor_c], tensor_c.op.axis[1]) + else: + sch[tensor_b].compute_at(sch[tensor_c], tensor_c.op.axis[0]) + + bx, tx = sch[tensor_c].split(tensor_c.op.axis[0], factor=DEFAULT_GPU_THREAD) + sch[tensor_c].bind(bx, tvm.thread_axis("blockIdx.x")) + sch[tensor_c].bind(tx, tvm.thread_axis("threadIdx.x")) + + return sch + +def SimpleMeanGrad(HEAD, input_shape): + """ + Compute Simple Mean Grad. + + Args: + HEAD (tvm.tensor.Tensor): output gradient, dy, defined in Primitive. + input_shape (Union[list[int], tuple[int]]): shape of mean input, x.shape. + + Returns: + tvm.tensor.Tensor, gradient of mean input. + """ + axis = (2, 3) + keepdims = True + return MeanGrad(HEAD, input_shape, axis, keepdims) + + +def gpu_schedule_SimpleMeanGrad(outs): + """ + gpu schedule SimpleMeanGrad. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + return gpu_schedule_MeanGrad(outs) diff --git a/mindspore/akg/gpu/mul.py b/mindspore/akg/gpu/mul.py new file mode 100644 index 00000000000..975a237837a --- /dev/null +++ b/mindspore/akg/gpu/mul.py @@ -0,0 +1,41 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""mul""" +import akg.topi as topi +import akg.tvm as tvm +from akg.ops.math import mul + +def Mul(x, y): + """mul.""" + return mul.mul(x, y) + + +def gpu_schedule_Mul(outs): + """ + gpu schedule for mul. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + with tvm.target.create(device): + sch = topi.cuda.schedule_broadcast(outs) + return sch diff --git a/mindspore/akg/gpu/relu6.py b/mindspore/akg/gpu/relu6.py new file mode 100644 index 00000000000..bdcf23f05a0 --- /dev/null +++ b/mindspore/akg/gpu/relu6.py @@ -0,0 +1,54 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""relu6""" +import akg.topi as topi +import akg.tvm as tvm +from akg.topi import tag + +@tvm.tag_scope(tag=tag.ELEMWISE) +def topi_nn_relu6(x): + """topi nn relu6.""" + return tvm.compute(x.shape, lambda *i: tvm.min(tvm.max(x(*i), tvm.const(0, x.dtype)), tvm.const(6, x.dtype))) + +def ReLU6(x): + """ + Compute elementwise with function: min(max(x, 0), 6). + + Args: + x (tvm.tensor.Tensor): Tensor of type float16, float32. + + Returns: + tvm.tensor.Tensor, has same type and shape as input. + """ + return topi_nn_relu6(x) + + +def gpu_schedule_ReLU6(outs): + """ + gpu schedule ReLU6. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + with tvm.target.create(device): + sch = topi.cuda.schedule_elemwise(outs) + return sch diff --git a/mindspore/akg/gpu/relu6_grad.py b/mindspore/akg/gpu/relu6_grad.py new file mode 100644 index 00000000000..e0590cf6efd --- /dev/null +++ b/mindspore/akg/gpu/relu6_grad.py @@ -0,0 +1,59 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""relu6 grad""" +import akg.topi as topi +import akg.tvm as tvm + +def ReLU6Grad(y_grad, x): + """ + Computes Gradients of Rectified Linear 6. + + Args: + y_grad (tvm.tensor.Tensor): Tensor of type float16, float32, gradients backpropagated to the ReLU6 op. + x (tvm.tensor.Tensor): Tensor of type float16/float32, inputs that where passed to the ReLU6 op, or its outputs. + + Returns: + tvm.tensor.Tensor, has same type and shape as x. + """ + shape = x.shape + dtype = x.dtype + + zero = tvm.const(0, dtype) + six = tvm.const(6, dtype) + + res0 = tvm.compute(shape, lambda *i: tvm.if_then_else(x(*i) >= zero, x(*i), zero)) + res6 = tvm.compute(shape, lambda *i: tvm.if_then_else(x(*i) >= six, zero, res0(*i))) + res = tvm.compute(shape, lambda *i: tvm.if_then_else(res6(*i) == zero, zero, y_grad(*i))) + return res + + +def gpu_schedule_ReLU6Grad(outs): + """ + gpu schedule ReLU6Grad. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + + with tvm.target.create(device): + sch = topi.cuda.schedule_elemwise(outs) + return sch diff --git a/mindspore/akg/gpu/squeeze.py b/mindspore/akg/gpu/squeeze.py new file mode 100644 index 00000000000..34fa423b8c7 --- /dev/null +++ b/mindspore/akg/gpu/squeeze.py @@ -0,0 +1,50 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""squeeze""" +import akg.topi as topi +import akg.tvm as tvm + +def Squeeze(x, axis=None): + """ + Remove the dimensions which have shape size 1. + + Args: + x (tvm.tensor.Tensor): Tensor, input whose shape is to be squeeze. + axis (Union[list, tuple, int, None]): specify which size 1 dimension to be removed. + + Returns: + tvm.tensor.Tensor, has the same type and element as x, but some size 1 dimensions are removed. + """ + return topi.squeeze(x, axis) + + +def gpu_schedule_Squeeze(outs): + """ + gpu schedule Squeeze. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + + with tvm.target.create(device): + sch = topi.cuda.schedule_injective(outs) + return sch diff --git a/mindspore/akg/gpu/squeeze_grad.py b/mindspore/akg/gpu/squeeze_grad.py new file mode 100644 index 00000000000..ef6a4242bab --- /dev/null +++ b/mindspore/akg/gpu/squeeze_grad.py @@ -0,0 +1,44 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""squeeze grad""" +import akg.topi as topi + +def SqueezeGrad(y_grad, x_shape, axis=None): + """ + Computes gradients for squeeze op. + + Args: + y_grad (tvm.tensor.Tensor): the gradient needed to be propagation. + x_shape (Union[list, tuple]): output Tensor shape. + axis (Union[list, tuple, int, None], optional): eliminated axis by squeeze. + + Returns: + tvm.tensor.Tensor: output gradient. + """ + return topi.reshape(y_grad, x_shape) + + +def gpu_schedule_SqueezeGrad(outs): + """ + gpu schedule SqueezeGrad. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + from .default_schedule import default_schedule + return default_schedule(outs) diff --git a/mindspore/akg/gpu/tile.py b/mindspore/akg/gpu/tile.py new file mode 100644 index 00000000000..cd3c663f978 --- /dev/null +++ b/mindspore/akg/gpu/tile.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tile""" +import akg.tvm +from akg.ops.array import tile +from akg.topi.generic import schedule_elemwise + +def Tile(x, multiples): + """tile.""" + return tile.tile(x, multiples) + +def gpu_schedule_Tile(outs): + """ + gpu schedule for tile. + + Args: + outs (tvm.tensor.Tensor): outputs of compute. + + Returns: + sch (schedule.Schedule): The created schedule. + """ + device = 'cuda' + ctx = akg.tvm.context(device, 0) + if not ctx.exist: + raise SystemError("Skip because %s is not enabled" % device) + with akg.tvm.target.create(device): + s = schedule_elemwise(outs) + return s diff --git a/mindspore/akg/message.py b/mindspore/akg/message.py new file mode 100644 index 00000000000..86bdf2899cf --- /dev/null +++ b/mindspore/akg/message.py @@ -0,0 +1,111 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""message""" +import importlib.util +import json +import json.decoder as jd +import logging +import traceback +import os.path +from pathlib import Path +import akg.tvm +from akg.utils import validation_check as vc_util +from akg.utils.dsl_create import TensorUtils +from . import gpu +from . import op_build + + +@vc_util.check_input_type(str) +def compilewithjson(json_str): + """compile with json.""" + try: + kernel_info = json.loads(json_str) + except jd.JSONDecodeError: + logging.error(traceback.format_exc()) + return False + + op_name = kernel_info['name'] + op_func = None + processor = 'aicore' + if 'process' in kernel_info: + processor = kernel_info['process'] + # get custom ops implementation first. + if 'impl_path' in kernel_info and kernel_info['impl_path'] is not None: + impl_path = os.path.realpath(kernel_info['impl_path']) + if os.path.isfile(impl_path): + custom_mod_name = Path(impl_path).resolve().stem + mod_spec = importlib.util.spec_from_file_location(custom_mod_name, impl_path) + custom_mod = importlib.util.module_from_spec(mod_spec) + mod_spec.loader.exec_module(custom_mod) + op_func = getattr(custom_mod, op_name, None) + + # get built-in ops. + if op_func is None: + if processor == 'cuda': + op_func = getattr(gpu, op_name, None) + + if op_func is None: + logging.error("this op not supported, please check op name %s", str(op_name)) + return False + + args = {} + tsr = [] + for input_desc in kernel_info['input_desc']: + if len(input_desc) == 1: + tensor_shape = input_desc[0]['shape'] + tensor_shape = (1,) if not tensor_shape else tensor_shape + vc_util.shape_dtype_max_size_check(tensor_shape) + args[input_desc[0]['name']] = akg.tvm.placeholder( + shape=tensor_shape, name=input_desc[0]['tensor_name'], dtype=input_desc[0]['data_type']) + tsr.append(args[input_desc[0]['name']]) + else: + tmp_input = [] + for tmp_desc in input_desc: + tensor_shape = tmp_desc['shape'] + tensor_shape = (1,) if not tensor_shape else tensor_shape + vc_util.shape_dtype_max_size_check(tensor_shape) + tmp_input.append(akg.tvm.placeholder( + shape=tensor_shape, name=tmp_desc['tensor_name'], dtype=tmp_desc['data_type'])) + args[input_desc[0]['name']] = tmp_input + tsr = tsr + tmp_input + + if kernel_info['attr']: + for ext_arg in kernel_info['attr']: + args[ext_arg['name']] = ext_arg['value'] + + output = op_func(**args) + + schedule_func = None + attrs = {} + if isinstance(output, (list, tuple)): + from inspect import isfunction + tmp_outputs = [] + for elem in output: + if isfunction(elem): + schedule_func = elem + elif isinstance(elem, dict): + for key, value in elem.items(): + if key not in attrs or not attrs[key]: + attrs[key] = value + else: + tmp_outputs.append(elem) + + output = tmp_outputs + else: + output = [output] + + + tsr = tsr + [i for i in output if TensorUtils.is_output_value(i)] + return op_build([op_name], output, tsr, schedule_func, processor, kernel_info['op'], attrs) diff --git a/mindspore/akg/op_build.py b/mindspore/akg/op_build.py new file mode 100644 index 00000000000..e3d3ec2b78b --- /dev/null +++ b/mindspore/akg/op_build.py @@ -0,0 +1,68 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""op_build""" +import os +import fcntl +import types +import typing +import logging +import traceback +import akg.tvm +import akg +from akg import save_gpu_param as gpu_utils +from akg.utils import validation_check as vc_util + +MS_CUDA_KERNEL_PATH = "/tmp/cuda_meta/" + +@vc_util.check_input_type(list, (list, tuple), (list, tuple), (types.FunctionType, type(None)), str, str, dict) +def op_build(opnames, computes, args, custom_schedule, device, kernel_name, attrs): + """op_build""" + if device == "cuda": + cuda_path = os.path.realpath(MS_CUDA_KERNEL_PATH) + if not os.path.isdir(cuda_path): + os.makedirs(cuda_path) + if not opnames: + logging.error("no opname given.") + return None + + schedule_name = 'gpu_schedule_' + opnames[0] + schedule_func = getattr(akg.gpu, schedule_name) + if not isinstance(schedule_func, (types.FunctionType, typing.Callable)): + logging.error("no schedule func found %s", str(schedule_name)) + return None + + ptx_file = os.path.realpath(MS_CUDA_KERNEL_PATH + kernel_name + ".ptx") + if os.path.exists(ptx_file): + os.remove(ptx_file) + try: + with open(ptx_file, 'at') as file: + fcntl.flock(file.fileno(), fcntl.LOCK_EX) + file.seek(0, 2) + if file.tell() == 0: + s = schedule_func(computes) + foo = akg.tvm.build(s, args, device, name=kernel_name) + ptx_code = foo.imported_modules[0].get_source("ptx") + file.write(ptx_code) + json_file = os.path.realpath(MS_CUDA_KERNEL_PATH + kernel_name + ".json") + kernel_info = (ptx_code, json_file, kernel_name) + gpu_utils.save_gpu_params(s, args, kernel_info) + os.chmod(ptx_file, 0o400) + except Exception: + logging.error(traceback.format_exc()) + return None + return True + + logging.error("Not support device %s.", device) + return None diff --git a/mindspore/akg/ops/__init__.py b/mindspore/akg/ops/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mindspore/akg/ops/array/__init__.py b/mindspore/akg/ops/array/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mindspore/akg/ops/array/tile.py b/mindspore/akg/ops/array/tile.py new file mode 100644 index 00000000000..e60fcc4ffb0 --- /dev/null +++ b/mindspore/akg/ops/array/tile.py @@ -0,0 +1,36 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: tile""" +import akg.tvm +import akg.topi +from akg.utils import validation_check as vc_util + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, (list, tuple)) +def tile(data, multiples): + """ + Repeats the data in the specified dimensions according to the multiples. + + Args: + data (tvm.tensor.Tensor): Tensor. + multiples (Union[list, tuple]): Elements must be int. The number of repetitions. + + Returns: + tvm.tensor.Tensor, has the same dtype as data. + """ + vc_util.check_shape(data.shape) + vc_util.check_int_list(multiples, "multiples") + output = akg.topi.tile(data, multiples) + return output diff --git a/mindspore/akg/ops/math/__init__.py b/mindspore/akg/ops/math/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mindspore/akg/ops/math/cast.py b/mindspore/akg/ops/math/cast.py new file mode 100644 index 00000000000..7266fd60c1d --- /dev/null +++ b/mindspore/akg/ops/math/cast.py @@ -0,0 +1,36 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: cast""" +import akg.tvm +import akg.topi +from akg.utils import validation_check as vc_util + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, str) +def cast(data, dst_type): + """ + cast data to target type. + + Args: + data (tvm.tensor.Tensor): Tensor to be casted. + dst_type (str): target cast type. + + Returns: + tvm.tensor.Tensor, type is dst_type. + """ + vc_util.check_shape(data.shape) + out = akg.topi.cast(data, dst_type) + + return out diff --git a/mindspore/akg/ops/math/equal.py b/mindspore/akg/ops/math/equal.py new file mode 100644 index 00000000000..eb446ac52bb --- /dev/null +++ b/mindspore/akg/ops/math/equal.py @@ -0,0 +1,54 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: equal""" +import akg.tvm +import akg.topi +from akg.utils.dsl_create import produce_shapes +from akg.utils import validation_check as vc_util + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, akg.tvm.tensor.Tensor) +def equal(input1, input2): + """ + check whether input1 equals to input2. + + Args: + input1 (tvm.tensor.Tensor): Tensor. + input2 (tvm.tensor.Tensor): Tensor. + + Returns: + tvm.tensor.Tensor. If input1 equal to input2 return True, else return False. + """ + shape1 = [x.value for x in input1.shape] + shape2 = [x.value for x in input2.shape] + vc_util.check_shape(shape1) + vc_util.check_shape(shape2) + + shape1, shape2, shape = produce_shapes(shape1, shape2) + + vc_util.elemwise_dtype_check(input1.dtype, input2.dtype) + dtype = input1.dtype + + # get equal compute + t_value = akg.tvm.compute(shape, lambda *indice: akg.tvm.const(1, dtype), "T") + f_value = akg.tvm.compute(shape, lambda *indice: akg.tvm.const(0, dtype), "F") + + input1_bro = akg.topi.broadcast_to(input1, shape) + input2_bro = akg.topi.broadcast_to(input2, shape) + c_out = akg.tvm.compute(shape, lambda *indice: akg.tvm.expr.Select(input1_bro[indice] == input2_bro[indice], + t_value[indice], f_value[indice]), name="C") + res = akg.tvm.compute(shape, lambda *indice: c_out(*indice).astype("bool"), name="res") + + return res diff --git a/mindspore/akg/ops/math/mean.py b/mindspore/akg/ops/math/mean.py new file mode 100644 index 00000000000..a26bc29087e --- /dev/null +++ b/mindspore/akg/ops/math/mean.py @@ -0,0 +1,47 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: mean""" +import akg.topi +import akg.tvm +from akg.utils import format_transform as ft_util +from akg.utils import validation_check as vc_util +from akg.ops.math import sum + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, (list, tuple, int, type(None)), (bool, type(None))) +def mean(data, axis=None, keepdims=False): + """ + Computes the mean of the values of a Tensor over the whole dataset. + + Args: + data (tvm.tensor.Tensor): Tensor. + axis (Union[list, tuple, int, None]): If the tuple is empty, the axis equal to None. + keepdims (bool): If keepdims equal to True, the result shape length is same to input shape length. + + Returns: + tvm.tensor.Tensor, has the same type as data. If keepdims equal to True, all reduced dimensions are + retained with length 1. else these reduced axis will be eliminate. + """ + shape = [x.value for x in data.shape] + vc_util.reduce_axis_check(shape, axis) + axis = ft_util.refine_reduce_axis(data, axis) + + count = 1 + for i in axis: + count *= shape[i] + output, _ = sum.sum_value(data, axis, keepdims) + res = akg.topi.divide(output, count) + + return res diff --git a/mindspore/akg/ops/math/mul.py b/mindspore/akg/ops/math/mul.py new file mode 100644 index 00000000000..8377a63d697 --- /dev/null +++ b/mindspore/akg/ops/math/mul.py @@ -0,0 +1,43 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: mul""" +import akg.topi +from akg.utils import validation_check as vc_util + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, akg.tvm.tensor.Tensor) +def mul(l_input, r_input): + """ + Calculate x * y element-wise. + + Note: + mul supports broadcasting. + + Args: + l_input (tvm.tensor.Tensor): Tensor. + r_input (tvm.tensor.Tensor): Tensor. + + Returns: + tvm.tensor.Tensor, has the same type as l_input and r_input. + """ + shape1 = [x.value for x in l_input.shape] + shape2 = [x.value for x in r_input.shape] + vc_util.check_shape(shape1) + vc_util.check_shape(shape2) + vc_util.auto_broadcast_check(shape1, shape2) + vc_util.elemwise_dtype_check(l_input.dtype, r_input.dtype) + output = akg.topi.multiply(l_input, r_input) + + return output diff --git a/mindspore/akg/ops/math/sub.py b/mindspore/akg/ops/math/sub.py new file mode 100644 index 00000000000..a4a85b0a09f --- /dev/null +++ b/mindspore/akg/ops/math/sub.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: sub""" +import akg.topi +import akg.tvm +from akg.utils import validation_check as vc_util + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, akg.tvm.tensor.Tensor) +def sub(data1, data2): + """ + Computes data1 - data2 elementwise, broadcast is supported. + + Args: + data1 (tvm.tensor.Tensor): Tensor. + data2 (tvm.tensor.Tensor): Tensor of same type as data1, if shape(data2) != shape(data1), broadcast will happen. + + Returns: + tvm.tensor.Tensor, subtracted result, with same type as input tensors and broadcasted shape of data1 and data2. + """ + vc_util.elemwise_dtype_check(data1.dtype, data2.dtype) + vc_util.check_shape(data1.shape) + vc_util.check_shape(data2.shape) + vc_util.auto_broadcast_check(data1.shape, data2.shape) + + res = akg.topi.subtract(data1, data2) + + return res diff --git a/mindspore/akg/ops/math/sum.py b/mindspore/akg/ops/math/sum.py new file mode 100644 index 00000000000..ea71bab9c48 --- /dev/null +++ b/mindspore/akg/ops/math/sum.py @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""operator dsl function: sum""" + +import akg.topi +import akg.tvm +from akg.utils import format_transform as ft_util +from akg.utils import validation_check as vc_util + + +@vc_util.check_input_type(akg.tvm.tensor.Tensor, (list, tuple, int, type(None)), (bool, type(None))) +def sum_value(inputs, axis=None, keepdims=False): + """ + Compute the sum of elements across dimensions of a tensor. + + Args: + inputs (tvm.tensor.Tensor): Tensor. + axis (Union[list, tuple, int, None]): If the list or tuple is empty, the axis equal to None. + keepdims (bool): If keepdims equal to True, the result shape length is same to input shape length. + + Returns: + tvm.tensor.Tensor, has same type as input. If keepdims is True, all reduced dimensions are retained + with length 1, else these reduced axis will be eliminate. + """ + axis = ft_util.refine_reduce_axis(inputs, axis) + vc_util.check_shape(inputs.shape) + + if not axis: + output = akg.topi.identity(inputs) + else: + output = akg.topi.sum(inputs, axis=axis, keepdims=keepdims) + + return output diff --git a/mindspore/akg/save_gpu_param.py b/mindspore/akg/save_gpu_param.py new file mode 100644 index 00000000000..228bdf32ca5 --- /dev/null +++ b/mindspore/akg/save_gpu_param.py @@ -0,0 +1,87 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""save gpu param""" +import os +import hashlib +import akg.tvm +from akg.tvm import schedule +from akg.utils import validation_check as vc_util + + +def get_dim(dim, axis=True): + """get dim info""" + dims_str = { + "grid_dim0": "// attr [iter_var(blockIdx.x, , blockIdx.x)] thread_extent = ", + "grid_dim1": "// attr [iter_var(blockIdx.y, , blockIdx.y)] thread_extent = ", + "grid_dim2": "// attr [iter_var(blockIdx.z, , blockIdx.z)] thread_extent = ", + "block_dim0": "// attr [iter_var(threadIdx.x, , threadIdx.x)] thread_extent = ", + "block_dim1": "// attr [iter_var(threadIdx.y, , threadIdx.y)] thread_extent = ", + "block_dim2": "// attr [iter_var(threadIdx.z, , threadIdx.z)] thread_extent = " + } + dim_to_axis = { + "grid_dim0": '"blockIdx.x" : ', + "grid_dim1": '"blockIdx.y" : ', + "grid_dim2": '"blockIdx.z" : ', + "block_dim0": '"threadIdx.x" : ', + "block_dim1": '"threadIdx.y" : ', + "block_dim2": '"threadIdx.z" : ' + } + if axis: + return dim_to_axis.get(dim) + return dims_str.get(dim) + + +def parse_params(file, dim, ir): + """parse parameters""" + dim_str = get_dim(dim, axis=False) + pos = ir.find(dim_str) + if pos != -1: + index = pos + len(dim_str) + param_temp = get_dim(dim) + + while ir[index].isdigit(): + param_temp += ir[index] + index += 1 + file.write(param_temp + ",\n") + else: + param_temp = get_dim(dim) + '1' + file.write(param_temp + ",\n") + + +@vc_util.check_input_type(schedule.Schedule, (list, tuple), tuple) +def save_gpu_params(s, args, kernel_info): + """save gpu parameters""" + ptx_code = kernel_info[0] + file_name = kernel_info[1] + kernel_name = kernel_info[2] + ir = str(akg.tvm.lower(s, args, simple_mode=True)) + file_path = os.path.realpath(file_name) + if os.path.exists(file_path): + os.remove(file_path) + + sha256 = hashlib.sha256() + sha256.update(ptx_code.encode("utf-8")) + hash_str = sha256.hexdigest() + with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT, 0o400), 'w') as fo: + fo.write("{\n") + fo.write('"kernelName" : ' + '"' + kernel_name + "_kernel0" + '",\n') + parse_params(fo, "grid_dim0", ir) + parse_params(fo, "grid_dim1", ir) + parse_params(fo, "grid_dim2", ir) + parse_params(fo, "block_dim0", ir) + parse_params(fo, "block_dim1", ir) + parse_params(fo, "block_dim2", ir) + fo.write('"sha256" : ' + '"' + hash_str + '"\n') + fo.write("}\n") diff --git a/mindspore/akg/utils/__init__.py b/mindspore/akg/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mindspore/akg/utils/dsl_create.py b/mindspore/akg/utils/dsl_create.py new file mode 100644 index 00000000000..aaea913143e --- /dev/null +++ b/mindspore/akg/utils/dsl_create.py @@ -0,0 +1,122 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""dsl create helping function""" +import akg +from akg.utils import format_transform as ft_util + +class TensorUtils: + """Class for creating tensor.""" + CREATE_SCH_ONLY = 'create_sch_only' + + @classmethod + def get_tensor_attrs(cls, tensor): + """get tensor attrs.""" + tensor_attrs = dict() + if "attrs" in dir(tensor.op): + tensor_attrs = dict(tensor.op.attrs.items()) + return tensor_attrs + + @classmethod + def update_tensor_attrs(cls, tensor, attrs): + """update tensor attrs.""" + tensor_attrs = cls.get_tensor_attrs(tensor) + tensor_attrs.update(attrs) + tensor = akg.tvm.compute(tensor.shape, + lambda *indice: tensor[indice], + name=tensor.op.name, + tag=tensor.op.tag, + attrs=tensor_attrs) + return tensor + + @classmethod + def is_create_sch_only(cls, tensor): + tensor_attrs = cls.get_tensor_attrs(tensor) + if cls.CREATE_SCH_ONLY in tensor_attrs.keys(): + return True + return False + + @classmethod + def is_output_value(cls, tensor): + """check output value.""" + return not cls.is_create_sch_only(tensor) + + @classmethod + def inplace_set(cls, input_tensor, output_tensor, buffer_name="data_buf"): + """inplace set.""" + input_tensor_shape = ft_util.get_shape(input_tensor) + output_tensor_shape = ft_util.get_shape(output_tensor) + if not input_tensor_shape == output_tensor_shape: + raise RuntimeError("Shape of the input_tensor and the output_tensor should be equal, " + "but got %s and %s"%(input_tensor_shape, output_tensor_shape)) + output_tensor = cls.update_tensor_attrs(output_tensor, {cls.CREATE_SCH_ONLY: 1}) + data_buf = akg.tvm.decl_buffer(input_tensor.shape, input_tensor.dtype, name=buffer_name) + binds_info = {input_tensor: data_buf, output_tensor: data_buf} + return output_tensor, binds_info + + @classmethod + def inplace_set_tensors(cls, input_tensors, output_tensors, buffer_names=None): + """ + inplace set for tensors + + Args: + in_tensors (Union[list, tuple]): Origin input tensors. + out_tensors (Union[list, tuple]): Origin output tensors. + buffer_names (Union[list, tuple] or None): Buffer names used to bind. + + Return: + inplace_tensors (list): Output tensors with the inplace info. + binds_infos (dict): Dictionary that maps the input tensor and the output + tensor to buffer. + """ + if not buffer_names: + buffer_names = ["data_buf_%s" % i for i in range(len(input_tensors))] + for arg in (input_tensors, output_tensors, buffer_names): + if not isinstance(arg, (tuple, list)): + raise RuntimeError("arg must be tuple or list!") + if len(input_tensors) != len(output_tensors) or len(input_tensors) != len(buffer_names): + raise RuntimeError("length of the input_tensors, output_tensors and buffer_names must be equal!") + + inplace_tensors = [] + binds_infos = dict() + for input_tensor, output_tensor, buffer_name in zip(input_tensors, output_tensors, buffer_names): + inplace_tensor, binds_info = cls.inplace_set(input_tensor, output_tensor, buffer_name) + inplace_tensors.append(inplace_tensor) + binds_infos.update(binds_info) + return inplace_tensors, binds_infos + +def produce_shapes(shape1, shape2): + """two input shapes produce three output shape.""" + shape1 = list(shape1) + shape2 = list(shape2) + flag = 0 + if len(shape1) < len(shape2): + shape1, shape2 = shape2, shape1 + flag = 1 + + output_shape_len = len(shape1) + dec = output_shape_len - len(shape2) + for i in range(dec): + shape2 = [1] + shape2 + + out_shape = [] + for i in range(output_shape_len): + if (shape1[i] != shape2[i]) and (shape1[i] != 1) and (shape2[i] != 1): + raise RuntimeError("input shapes not match!") + out_shape.append(shape1[i] if shape1[i] > shape2[i] else shape2[i]) + + if flag == 1: + shape1, shape2 = shape2, shape1 + + return shape1, shape2, out_shape diff --git a/mindspore/akg/utils/format_transform.py b/mindspore/akg/utils/format_transform.py new file mode 100644 index 00000000000..816cbcaadb8 --- /dev/null +++ b/mindspore/akg/utils/format_transform.py @@ -0,0 +1,80 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""format transform function""" +import akg + +def refine_reduce_axis(input, axis): + """make reduce axis legal.""" + shape = get_shape(input) + if axis is None: + axis = [i for i in range(len(shape))] + elif isinstance(axis, int): + axis = [axis] + elif not isinstance(axis, (tuple, list)): + raise TypeError("axis must be one of the type int,tuple,list or None") + + if len(axis) > len(shape): + raise ValueError("axis size must not larger than shape size") + + axis = list(axis) + + for i, _ in enumerate(axis): + if axis[i] < 0: + axis[i] += len(shape) + + if axis[i] >= len(shape): + raise ValueError(("axis value-{} exceeds len(axis) which is invalid".format(axis[i]))) + + axis.sort(reverse=True) + + return axis + + +def get_shape_from_tensor(data): + """translate akg.tvm.shape to list type in python.""" + tvm_shape = data.shape + py_shape = [] + for i in tvm_shape: + if isinstance(i, akg.tvm.expr.Var): + py_shape.append(i) + else: + py_shape.append(i.value) + return py_shape + + +def tvm_shape_to_list(tvm_shape): + """translate akg.tvm.shape to list type in python.""" + py_shape = [] + for i in tvm_shape: + if isinstance(i, akg.tvm.expr.Var): + py_shape.append(i) + else: + py_shape.append(i.value) + return py_shape + + +def get_shape(data): + """get shape and save it as list.""" + if isinstance(data, akg.tvm.tensor.Tensor): + shape = get_shape_from_tensor(data) + elif isinstance(data, akg.tvm.container.Array): + shape = tvm_shape_to_list(data) + elif isinstance(data, int): + shape = [data] + elif isinstance(data, (tuple, list)): + shape = list(data) + else: + raise TypeError("Refine axis does not support type {} for now.".format(type(data))) + return shape diff --git a/mindspore/akg/utils/validation_check.py b/mindspore/akg/utils/validation_check.py new file mode 100644 index 00000000000..72494c52816 --- /dev/null +++ b/mindspore/akg/utils/validation_check.py @@ -0,0 +1,233 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""validation check functions""" +from functools import wraps, reduce +from akg.utils.format_transform import get_shape + +MAX_DATA_SIZE = 2 ** 31 + +def check_input_type_dict(input_dict, input_key, input_name): + """ + check input parameter type for new type: dict. + + Note: + rule1: key of input_dict should be in the input_key + rule2: type of input_dict[shape] should be in (list, tuple), if have shape + rule3: type of input_dict[dtype] should be in (str), if have dtype + + Args: + input_dict (dict): input_dict + input_key (list or tuple): all input key list, the key of input must in input_key + input_name (str): input param name, only used for error print + + Returns: + None + """ + def _check_input_type(input_key, input_type): + if not isinstance(input_dict[input_key], input_type): + raise RuntimeError( + "the input parameter %s[%s] must be %s, while type of input is %s" % + (input_name, input_key, input_type, type(input_dict[input_key]))) + + for key in input_dict.keys(): + if key not in input_key: + raise RuntimeError( + "the input parameter %s must have arrt <%s>" % + (input_name, key)) + + # check shape's type of input_dict, if have shape + if key == "shape": + _check_input_type(key, (list, tuple)) + + # check dtype's type of input_dict, if have dtype + if key == "dtype": + _check_input_type(key, (str,)) + + +def check_input_type_list_tuple(inputs, expect): + """check inputs by a list or tuple of expected types.""" + if not isinstance(inputs, expect[1][0]): + raise RuntimeError("the input parameter %s must be (list, tuple), while" + " type of input is %s" % (expect[0], type(inputs))) + for inp in inputs: + if not isinstance(inp, expect[1][1]): + raise RuntimeError("The element in parameter %s must be %s, while " + "type of input is %s" % ( + expect[0], expect[1][1], type(inp))) + + +def check_input_type(*type_args, **_type_kwargs): + """check input parameter type.""" + def out_wrapper(func): + """outer wrapper function.""" + formal_parameter = func.__code__.co_varnames + formal_parameter_list = list(zip(formal_parameter, type_args)) + + @wraps(func) + def in_wrapper(*args, **kwargs): + """inner wrapper function.""" + for i, arg_v in enumerate(args): + # add for new input dict, if dict, will check shape and dtype + if isinstance(arg_v, dict): + check_input_type_dict(arg_v, arg_v.keys(), + formal_parameter_list[i][0]) + + if isinstance(formal_parameter_list[i][1], tuple): + if isinstance(formal_parameter_list[i][1][0], tuple) \ + and len(formal_parameter_list[i][1]) == 2: + check_input_type_list_tuple(arg_v, formal_parameter_list[i]) + continue + + if not isinstance(arg_v, formal_parameter_list[i][1]): + raise RuntimeError("the %sth input parameter %s must be %s, " + "while type of input is %s" % (str(i), formal_parameter_list[i][0], + formal_parameter_list[i][1], + type(arg_v))) + for i in kwargs: + for j in formal_parameter_list: + if i in j: + if not isinstance(kwargs[i], j[1]): + raise RuntimeError("the input parameter %s must be " + "%s, while type of input is %s" + "" % (i, j[1], type(kwargs[i]))) + break + return func(*args, **kwargs) + + return in_wrapper + + return out_wrapper + + +def shape_dtype_max_size_check(shape): + """check validation of tensor's shape.""" + if shape: + mul = int(reduce(lambda x, y: int(x) * int(y), shape)) + if mul > MAX_DATA_SIZE: + error_msg = "*".join([str(sh) for sh in shape]) + raise RuntimeError("Invalid shape, data is {} bytes ({}), which " + "exceed max data size {} bytes" + .format(mul, error_msg, MAX_DATA_SIZE)) + + +def check_shape(tensor, length=None, tensor_name=""): + """The common check rule for placeholder data.""" + shape = get_shape(tensor) + if not shape: + raise RuntimeError("The ndim of input tensor {} must more than 0, " + "actual input is {}".format(tensor_name, len(shape))) + + for shape_v in shape: + if not isinstance(shape_v, int) or shape_v <= 0: + raise RuntimeError("The type of tensor {} axis value must be " + "positive int and value more than 0," + "actual input is ({}) {}". + format(tensor_name, type(shape_v), shape_v)) + + if length and len(shape) != length: + raise ValueError('The length of {} should be {}, while actual length is {}'. + format(tensor_name, length, len(shape))) + + +def ops_dtype_check(dtype, args): + """check validation of op's dtype.""" + expected_dtype = list() + + def _get_expect_dtype(expected_dtype, arg): + if isinstance(arg, str): + expected_dtype.append(arg) + elif isinstance(arg, (list, tuple)): + for t in arg: + _get_expect_dtype(expected_dtype, t) + else: + raise TypeError("arg should be either a string, " + "or a list/tuple of string, " + "while current is {}".format(type(arg))) + + _get_expect_dtype(expected_dtype, args) + + if isinstance(dtype, (list, tuple)): + checking_dtype = [d.lower() for d in dtype] + elif isinstance(dtype, str): + checking_dtype = [dtype.lower()] + else: + raise TypeError("dtype should be either a string or a tuple/list of string") + error_msg = "Supported dtype: {}, while received dtype: {}" + if not set(checking_dtype).issubset(set(expected_dtype)): + raise RuntimeError(error_msg.format(expected_dtype, checking_dtype)) + + +def reduce_axis_check(reduce_shape, reduce_axis): + """check validation of reduce axis for certain reduce shape.""" + dim = len(reduce_shape) + if dim == 1 and int(reduce_shape[0]) == 1: + raise RuntimeError("Error, reduce shape is 1. Scalar is not supported " + "for reduction, please input a vector.") + if isinstance(reduce_axis, int): + if reduce_axis not in range(-dim, dim): + raise RuntimeError("Reduce axis should be in range [%d. %d)" + "" % (-dim, dim)) + elif isinstance(reduce_axis, (tuple, list)): + if len(reduce_axis) > len(reduce_shape): + raise RuntimeError("Reduce axis list exceed reduce shape length: " + "%d vs %d, error" % (len(reduce_axis), len(reduce_shape))) + processed_axis = [] + for axis in reduce_axis: + processed_axis.append(int(axis + dim) if axis < 0 else int(axis)) + if len(set(processed_axis)) < len(processed_axis): + raise RuntimeError("Reduce axis list contains %d duplicated element, please check" + % (len(processed_axis) - len(set(processed_axis)))) + for axis in processed_axis: + if axis >= dim: + raise RuntimeError("Invalid reduce axis, axis should less than %d" % dim) + elif reduce_axis is not None: + raise RuntimeError("axis should be a list, tuple or int.") + + +def elemwise_dtype_check(dtype_a, dtype_b, supported_type=None): + """check validation of tensor's dtype for element-wise op.""" + if supported_type: + ops_dtype_check(dtype_a, supported_type) + ops_dtype_check(dtype_b, supported_type) + if dtype_a.lower() != dtype_b.lower(): + raise RuntimeError("Element-wise operation needs same data type, while " + "current is %s vs %s" % (dtype_a.lower(), dtype_b.lower())) + + +def auto_broadcast_check(shape_a, shape_b): + """automatic broadcast check.""" + shape_l = get_shape(shape_a) + shape_r = get_shape(shape_b) + + if len(shape_l) <= len(shape_r): + shape_short = shape_l + shape_long = shape_r + else: + shape_short = shape_r + shape_long = shape_l + + dim_diff = len(shape_long) - len(shape_short) + for i in range(dim_diff): + shape_short.insert(0, 1) + for i, shp in enumerate(shape_short): + if int(shp) != int(shape_long[i]) and 1 not in [int(shp), int(shape_long[i])]: + raise RuntimeError("Invalid auto broadcast, dim %d should be 1 or equal, " + "while now is %d vs %d" % (i, shp, shape_long[i])) + + +def check_int_list(array, array_name): + """check whether all the elements are integers.""" + for num in array: + if not isinstance(num, int): + raise RuntimeError("Type of value in %s should be int, but got type %s" % (array_name, type(num))) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt new file mode 100644 index 00000000000..72ad4d7b09b --- /dev/null +++ b/mindspore/ccsrc/CMakeLists.txt @@ -0,0 +1,541 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +if(ENABLE_CPU) + include(ExternalProject) + add_compile_definitions(CPUSESSION) + file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/cpu/*.cc" + ) +endif() + +if(ENABLE_GPU) + find_package(CUDA REQUIRED) + find_package(Threads) + if(${CUDA_VERSION} VERSION_LESS ${MS_REQUIRE_CUDA_VERSION}) + message(FATAL_ERROR "The minimum CUDA version ${MS_REQUIRE_CUDA_VERSION} is required, but only CUDA ${CUDA_VERSION} found.") + endif() + enable_language(CUDA) + if(NOT CUDA_PATH OR CUDA_PATH STREQUAL "") + if(DEFINED ENV{CUDA_HOME}) + set(CUDA_PATH $ENV{CUDA_HOME}) + else() + set(CUDA_PATH ${CUDA_TOOLKIT_ROOT_DIR}) + endif() + endif() + if(NOT CUDNN_PATH OR CUDNN_PATH STREQUAL "") + set(CUDNN_PATH ${CUDA_PATH}) + endif() + message("CUDA_PATH: ${CUDA_PATH}") + message("CUDA_INCLUDE_DIRS: ${CUDA_INCLUDE_DIRS}") + message("CUDNN_PATH: ${CUDNN_PATH}") + include_directories(${CUDNN_PATH} ${CUDA_PATH} ${CUDA_INCLUDE_DIRS}) + + file(GLOB_RECURSE GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/gpu/*.cc" + "device/gpu/*.cu" + "kernel/gpu/*.cu" + "kernel/akg/gpu/*.cc" + ) + file(GLOB_RECURSE GPU_KERNEL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel/gpu/*.cc" + ) + list(APPEND CUDA_NVCC_FLAGS -arch=sm_53) + list(REMOVE_ITEM GPU_SRC_LIST "device/gpu/blocking_queue.cc" "device/gpu/gpu_buffer_mgr.cc") + add_library(gpu_queue SHARED "device/gpu/blocking_queue.cc" "device/gpu/gpu_buffer_mgr.cc") + target_link_libraries(gpu_queue ${CMAKE_THREAD_LIBS_INIT} ${CUDA_PATH}/lib64/libcudart.so) + + + file(GLOB_RECURSE MS_STEPS_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "session/gpu_session.cc" + ) + list(REMOVE_ITEM GPU_SRC_LIST "device/gpu/mpi/mpi_initializer.cc" + "device/gpu/distribution/collective_wrapper.cc" + "device/gpu/distribution/mpi_wrapper.cc" + "device/gpu/distribution/nccl_wrapper.cc" + ) + list(REMOVE_ITEM GPU_KERNEL_SRC_LIST "device/gpu/mpi/mpi_initializer.cc" + "kernel/gpu/nccl/nccl_gpu_kernel.cc" + ) + + set(NVCC_TMP_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + string(REPLACE "-std=c++17" "-std=c++11" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + cuda_add_library(gpu_cuda_lib STATIC ${GPU_SRC_LIST}) + set(CMAKE_CXX_FLAGS ${NVCC_TMP_CMAKE_CXX_FLAGS}) + + if(ENABLE_MPI) + include(ExternalProject) + + file(GLOB_RECURSE GPU_NCCL_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel/gpu/nccl/*.cc" + ) + file(GLOB_RECURSE GPU_MPI_PYTHON_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/gpu/mpi/mpi_initializer.cc" + ) + add_library(gpu_collective SHARED "device/gpu/distribution/collective_wrapper.cc" + "device/gpu/distribution/mpi_wrapper.cc" + "device/gpu/distribution/nccl_wrapper.cc" + ) + endif() +endif() + +include_directories("${CMAKE_BINARY_DIR}/predict/schema/inner") +file(GLOB_RECURSE FLATBUFFER_IN RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "predict/schema/*.fbs") +set(FLATBUFFER_OU "${CMAKE_BINARY_DIR}/predict/schema/inner") +ms_build_flatbuffers("${FLATBUFFER_IN}" "${FLATBUFFER_IN}" GENERATED_OUTPUT_DIR "${FLATBUFFER_OU}") + +file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "ir/*.cc" + "ir/dtype/*.cc" + "utils/*.cc" + "common/*.cc" + "parallel/*.cc" + "pipeline/pipeline.cc" + "pipeline/resource.cc" + "pipeline/pass.cc" + "pipeline/action.cc" + "pipeline/validator.cc" + "pipeline/remove_value_node_dup.cc" + "pipeline/parse/*.cc" + "pipeline/static_analysis/*.cc" + "optimizer/*.cc" + "debug/*.cc" + "onnx/onnx_exporter.cc" + "operator/*.cc" + "transform/*.cc" + "session/kernel_graph.cc" + "utils/node_utils.cc" + "session/session_basic.cc" + "session/session_factory.cc" + "session/anf_runtime_algorithm.cc" + "vm/*.cc" + "pynative/*.cc" + "pybind_api/*.cc" + "device/common/*.cc" + "kernel/kernel_query.cc" + "kernel/kernel_build_info.cc" + "kernel/kash/*.cc" + "device/kernel_info.cc" + "device/kernel_runtime.cc" + "device/kernel_runtime_manager.cc" + "device/convert_tensor_utils.cc" + "pre_activate/ascend/*.cc" + "pre_activate/common/*.cc" + "pre_activate/pass/*.cc" + "pre_activate/gpu/*.cc" + "pre_activate/mem_reuse/*.cc" + "predict/predict.cc" + "predict/generator/utils/ir_model_util.cc" + "predict/converter/*.cc" + "predict/converter/attr_utils/*.cc" + "predict/converter/lite_model/*.cc" + "predict/converter/lite_model/operations/*.cc" + "kernel/common_utils.cc" + "kernel/oplib/*.cc" + "kernel/kash/*.cc" + "device/gpu/distribution/collective_init.cc" + ) + +file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "pre_activate/mem_reuse/*.cc" + ) +if(NOT ENABLE_DUMP_E2E) + list(REMOVE_ITEM MINDSPORE_SRC_LIST "debug/e2e_dump.cc") +endif() + +file(COPY "${ms_onnx_INC}/onnx/onnx.proto" DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}) +file(GLOB_RECURSE ONNX_PROTO RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/onnx.proto") +message(“onnx proto path is : ${ONNX_PROTO}”) +ms_protobuf_generate(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${ONNX_PROTO}) +list(APPEND MINDSPORE_PROTO_LIST ${ONNX_PROTO_SRCS}) + +if(ENABLE_DUMP_PROTO) + include_directories(${CMAKE_BINARY_DIR}) + file(GLOB_RECURSE PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "utils/node_strategy.proto" + ) + ms_protobuf_generate(PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) + + file(GLOB_RECURSE PROTO_PY RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "utils/anf_ir.proto" + "utils/summary.proto" + "utils/checkpoint.proto" + ) + ms_protobuf_generate_py(PY_SRCS PY_HDRS PY_PYS ${PROTO_PY}) + + list(APPEND MINDSPORE_PROTO_DUMP_LIST ${PROTO_SRCS}) + list(APPEND MINDSPORE_PROTO_DUMP_LIST ${PY_SRCS}) + list(APPEND MINDSPORE_SRC_LIST "debug/dump_proto.cc") + list(APPEND MINDSPORE_SRC_LIST "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc") + add_compile_definitions(ENABLE_DUMP_PROTO) +endif() + +if(ENABLE_D) + include_directories("${CMAKE_BINARY_DIR}/kernel/aicpu") + file(GLOB_RECURSE PROTO_IN RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel/aicpu/proto/*.proto" + ) + ms_protobuf_generate(PROTOSRCS PROTOHDRS ${PROTO_IN}) + + include_directories("${CMAKE_BINARY_DIR}/predict/generator/ir") + file(GLOB_RECURSE PROTO_INNER RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "predict/proto/*.proto" + ) + ms_protobuf_generate(PREDICT_PROTOSRCS PREDICT_PROTOHDRS ${PROTO_INNER}) + + file(GLOB_RECURSE D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/ascend/*.cc" + "device/ascend/profiling/*.cc" + "device/ascend/tasksink/*.cc" + "kernel/akg/cce/*.cc" + "device/kernel_adjust.cc" + "kernel/kernel_fusion.cc" + "kernel/tbe/*.cc" + ) + list(APPEND MINDSPORE_SRC_LIST ${D_SRC_LIST}) + list(APPEND MINDSPORE_PROTO_AICPU_LIST ${PROTOSRCS}) + list(APPEND MINDSPORE_PROTO_PREDICT_LIST ${PREDICT_PROTOSRCS}) + + file(GLOB_RECURSE MS_STEPS_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "session/ascend_session.cc" + ) + file(GLOB_RECURSE MS_TASKINFO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/ascend/tasksink/taskinfo/*.cc") + file(GLOB_RECURSE MS_AICPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel/aicpu/*.cc" + ) + file(GLOB_RECURSE MS_RT_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel/mng/*.cc" + ) + file(GLOB_RECURSE MS_HCCL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel/hccl/*.cc" + ) + file(GLOB_RECURSE MS_PREDICT_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "predict/generator/ir/*.cc" + ) + add_compile_definitions(ENABLE_D) +endif() + +file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "gvar/*.cc" + ) + +add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST}) +add_library(mindspore STATIC ${MINDSPORE_SRC_LIST}) + +if(ENABLE_D) + list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST}) +endif() +if(ENABLE_DUMP_PROTO) + list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_DUMP_LIST}) +endif() +list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_PREDICT_LIST}) +if(MINDSPORE_PROTO_LIST) + add_library(proto_input STATIC ${MINDSPORE_PROTO_LIST}) + set_target_properties(proto_input PROPERTIES COMPILE_FLAGS "-Wno-unused-variable") + target_link_libraries(mindspore proto_input) +endif() + +if(APPLE) + set_target_properties(mindspore_gvar PROPERTIES MACOSX_RPATH ON) +endif() + +link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine) + +if (ENABLE_GE) + if(ENABLE_TRAIN) + target_link_libraries(mindspore graph ge_client_train) + else() + target_link_libraries(mindspore graph ge_client) + endif() + target_link_libraries(mindspore tsdclient) +else() + add_compile_definitions(NO_GE_CLIENT) + target_link_libraries(mindspore graph) +endif() + +if(ENABLE_D) + if (DEFINED ENV{D_LINK_PATH}) + if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + MESSAGE("system processor matches aarch64") + set(D_LIB_PATH $ENV{D_LINK_PATH}/aarch64) + elseif (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64") + MESSAGE("system processor matches x86_64") + set(D_LIB_PATH $ENV{D_LINK_PATH}/x86_64) + else () + MESSAGE("system ${CMAKE_HOST_SYSTEM_PROCESSOR} not support") + endif() + else() + MESSAGE("use system default lib") + set(D_LIB_PATH "/usr/local/HiAI/runtime/lib64/") + endif() + + MESSAGE("USE DAV LIB PATH: ${D_LIB_PATH}") + find_library(HCCL hccl ${D_LIB_PATH}) + find_library(CCE_LIB cce ${D_LIB_PATH}) + find_library(RUNTIME_LIB runtime ${D_LIB_PATH}) + find_library(TSDCLIENT tsdclient ${D_LIB_PATH}) + find_library(PROFILING msprof ${D_LIB_PATH}) + target_link_libraries(mindspore ge_runtime ${CCE_LIB} ${RUNTIME_LIB} ${TSDCLIENT} ${PROFILING} ${HCCL} ${TSDCLIENT}) +endif() + +target_link_libraries(mindspore securec) +target_link_libraries(mindspore dl) +target_link_libraries(mindspore mindspore::flatbuffers) +# link protobuf +if (ENABLE_D) + target_link_libraries(mindspore protobuf::libprotobuf) +endif() + +# set c_expression building +set(PYTHON_MODULE_SOURCE + pipeline/init.cc + kernel/oplib/oplib.cc + kernel/akg/akgkernelbuild.cc + kernel/akg/akg_kernel_attrs_process.cc + ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} + ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) + +set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) +pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE}) +target_link_options(_c_expression PRIVATE -Wl,-init,mindspore_log_init) + +MESSAGE(STATUS "operation system is ${CMAKE_SYSTEM}") +if (CMAKE_SYSTEM_NAME MATCHES "Linux") + set(ORIGIN_PATH $ORIGIN) +elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") + set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON) + set(ORIGIN_PATH @loader_path) +else () + MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}") +endif () + + +set(ORIGIN_PATH ${ORIGIN_PATH}/lib) +set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}) +target_link_libraries(_c_expression PRIVATE + mindspore::pybind11_module + mindspore + mindspore_gvar + ) + +if(ENABLE_GPU) + execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh + ${CMAKE_BINARY_DIR} + ${dlpack_DIRPATH} + ${dmlc_core_DIRPATH} + ${rang_DIRPATH} + ${incubator_tvm_gpu_DIRPATH}) + set(TVM_DIR "${CMAKE_BINARY_DIR}/incubator-tvm") + # Utility functions + include(${TVM_DIR}/cmake/util/Util.cmake) + include(${TVM_DIR}/cmake/util/FindCUDA.cmake) + + # include directories + include_directories(AFTER "${TVM_DIR}/include") + include_directories(AFTER "${TVM_DIR}/src") + include_directories(AFTER "${TVM_DIR}") + include_directories(AFTER "${TVM_DIR}/src/schedule") + + include_directories(AFTER "${TVM_DIR}/3rdparty/dmlc-core/include") + include_directories(AFTER "${TVM_DIR}/3rdparty/dlpack/include") + include_directories(AFTER "${TVM_DIR}/3rdparty/compiler-rt") + include_directories(AFTER "${TVM_DIR}/3rdparty/rang/include") + + # lib contain dlopen and dlclose + set(TVM_RUNTIME_LINKER_LIBS ${CMAKE_DL_LIBS}) + + # add source group + file(GLOB_RECURSE GROUP_SOURCE "${TVM_DIR}/src/*.cc" "src/*.cc") + file(GLOB_RECURSE GROUP_INCLUDE "${TVM_DIR}/src/*.h" + "${TVM_DIR}/include/*.h" "src/*.h" "include/*.h") + assign_source_group("Source" ${GROUP_SOURCE}) + assign_source_group("Include" ${GROUP_INCLUDE}) + + file(GLOB COMPILER_SRCS + ${TVM_DIR}/src/api/*.cc + ${TVM_DIR}/src/arithmetic/*.cc + ${TVM_DIR}/src/autotvm/*.cc + ${TVM_DIR}/src/codegen/*.cc + ${TVM_DIR}/src/lang/*.cc + ${TVM_DIR}/src/pass/*.cc + ${TVM_DIR}/src/op/*.cc + ${TVM_DIR}/src/node/*.cc + ${TVM_DIR}/src/schedule/*.cc + ${TVM_DIR}/src/runtime/*.cc + ${TVM_DIR}/src/runtime/cce/*.cc + ${TVM_DIR}/src/runtime/vm/*.cc + ${TVM_DIR}/src/runtime/vm/profiler/*.cc + ${TVM_DIR}/src/codegen/stackvm/*.cc) + + file(GLOB_RECURSE RELAY_SRCS ${TVM_DIR}/src/relay/*.cc) + list(APPEND COMPILER_SRCS ${RELAY_SRCS}) + + file(GLOB DATATYPE_SRCS ${TVM_DIR}/src/codegen/datatype/*.cc) + list(APPEND COMPILER_SRCS ${DATATYPE_SRCS}) + + file(GLOB COMPILER_VERILOG_SRCS ${TVM_DIR}/src/codegen/verilog/*.cc) + list(APPEND COMPILER_SRCS ${COMPILER_VERILOG_SRCS}) + + file(GLOB TOPI_SRCS ${TVM_DIR}/topi/src/*.cc) + + file(GLOB RUNTIME_SRCS + ${TVM_DIR}/src/runtime/*.cc + ${TVM_DIR}/src/runtime/cce/*.cc + ${TVM_DIR}/src/runtime/vm/*.cc + ${TVM_DIR}/src/runtime/stub/*.cc + ${TVM_DIR}/src/runtime/stackvm/*.cc) + + + file(GLOB COMPILER_OFF_SRCS + ${TVM_DIR}/src/codegen/opt/build_*_off.cc) + set(USE_CUDA "OFF") + if(ENABLE_GPU) + list(REMOVE_ITEM COMPILER_OFF_SRCS + ${TVM_DIR}/src/codegen/opt/build_cuda_off.cc) + set(USE_CUDA "ON") + endif() + list(APPEND COMPILER_SRCS ${COMPILER_OFF_SRCS}) + # Module rules + include(${TVM_DIR}/cmake/modules/CUDA.cmake) + + set(CMAKE_C_FLAGS_AKG -pipe -Wall -fPIC -fstack-protector-all) + set(CMAKE_C_FLAGS_AKG ${CMAKE_C_FLAGS_AKG} -Wl,-z,relro,-z,now,-z,noexecstack) + + set(CMAKE_CXX_FLAGS_AKG -std=c++11 -pipe -Wall -fPIC -fstack-protector-all) + set(CMAKE_CXX_FLAGS_AKG ${CMAKE_CXX_FLAGS_AKG} -Wl,-z,relro,-z,now,-z,noexecstack) + + if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + message("-- Build in Debug mode") + set(CMAKE_C_FLAGS_AKG ${CMAKE_C_FLAGS_AKG} -O0 -g -rdynamic) + set(CMAKE_CXX_FLAGS_AKG ${CMAKE_CXX_FLAGS_AKG} -O0 -g -rdynamic) + else() + message("-- Build in Release mode") + set(CMAKE_C_FLAGS_AKG ${CMAKE_C_FLAGS_AKG} -O2 -Werror) + set(CMAKE_CXX_FLAGS_AKG ${CMAKE_CXX_FLAGS_AKG} -O2 -Werror) + endif() + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION + VERSION_GREATER 7.0) + set(CMAKE_CXX_FLAGS_AKG ${CMAKE_CXX_FLAGS_AKG} -faligned-new) + endif() + + add_library(akg OBJECT ${COMPILER_SRCS} ${RUNTIME_SRCS} ${TOPI_SRCS}) + + target_link_libraries(akg ${TVM_LINKER_LIBS} ${TVM_RUNTIME_LINKER_LIBS}) + target_compile_options(akg PRIVATE + $<$:${CMAKE_C_FLAGS_AKG}> + $<$:${CMAKE_CXX_FLAGS_AKG}>) + target_include_directories(akg PRIVATE "${TVM_DIR}/topi/include") + + add_dependencies(_c_expression akg) + target_link_libraries(_c_expression PRIVATE akg) +endif() + +if(ENABLE_DUMP_PROTO) + target_link_libraries(_c_expression PRIVATE protobuf::libprotobuf) +endif() + +if(ENABLE_GPU) + message("add gpu lib to c_expression") + target_link_libraries(_c_expression PRIVATE + gpu_cuda_lib + gpu_queue + cublas + ${CUDNN_PATH}/lib64/libcudnn.so + ${CUDA_PATH}/lib64/libcudart.so + ${CUDA_PATH}/lib64/stubs/libcuda.so) + if(ENABLE_MPI) + pybind11_add_module(_ms_mpi ${GPU_MPI_PYTHON_LIST}) + target_link_libraries(_ms_mpi PRIVATE mindspore::pybind11_module mindspore::ompi) + target_link_libraries(gpu_collective PRIVATE mindspore::ompi mindspore::nccl) + endif() +endif() + + +if(ENABLE_CPU) + target_link_libraries(_c_expression PRIVATE mindspore::dnnl mindspore::mkldnn) +endif() + +if(ENABLE_MINDDATA) + add_subdirectory(mindrecord) + add_subdirectory(dataset) +endif() +set(MS_PACK_PATH ${CMAKE_SOURCE_DIR}/build/package/mindspore/) +set(MS_LIB_PATH ${CMAKE_SOURCE_DIR}/build/package/mindspore/lib/) + +add_custom_target(add_ms_lib ALL + COMMAND mkdir -pv ${MS_LIB_PATH} + COMMAND cp ${MS_CCSRC_BUILD_PATH}/_c_expression* ${MS_PACK_PATH} + COMMAND cp ${MS_CCSRC_BUILD_PATH}/libmindspore_gvar.so ${MS_LIB_PATH} +) +add_dependencies(add_ms_lib _c_expression) + +if (NOT ENABLE_GE) + if (ENABLE_D) + add_custom_target(add_ge_lib ALL + COMMAND cp ${MS_CCSRC_BUILD_PATH}/../../graphengine/src/common/graph/libgraph.so ${MS_LIB_PATH} + COMMAND cp ${MS_CCSRC_BUILD_PATH}/../../graphengine/src/ge/common/libge_common.so ${MS_LIB_PATH} + COMMAND cp ${MS_CCSRC_BUILD_PATH}/../../graphengine/src/ge/ge_runtime/libge_runtime.so ${MS_LIB_PATH} + COMMAND cp /usr/local/HiAI/driver/lib64/libslog.so ${MS_LIB_PATH} + COMMAND cp /usr/local/HiAI/driver/lib64/libc_sec.so ${MS_LIB_PATH} + ) + add_dependencies(add_ge_lib add_ms_lib) + add_dependencies(add_ge_lib graph) + add_dependencies(add_ge_lib ge_runtime) + else() + add_custom_target(add_ge_lib ALL + COMMAND cp ${MS_CCSRC_BUILD_PATH}/../../graphengine/src/common/graph/libgraph.so ${MS_LIB_PATH} + COMMAND cp ${CMAKE_SOURCE_DIR}/graphengine/third_party/prebuild/${CMAKE_HOST_SYSTEM_PROCESSOR}/libslog.so ${MS_LIB_PATH} + COMMAND cp ${CMAKE_SOURCE_DIR}/graphengine/third_party/prebuild/${CMAKE_HOST_SYSTEM_PROCESSOR}/libc_sec.so ${MS_LIB_PATH} + ) + add_dependencies(add_ge_lib add_ms_lib) + add_dependencies(add_ge_lib graph) + endif() +endif() + +if (ENABLE_GPU) + if (ENABLE_MPI) + add_custom_target(add_mpi_lib ALL + COMMAND cp ${MS_CCSRC_BUILD_PATH}/_ms_mpi* ${MS_PACK_PATH} + ) + add_dependencies(add_mpi_lib _ms_mpi) + add_custom_target(add_gpu_collective_lib ALL + COMMAND mkdir -pv ${MS_LIB_PATH} + COMMAND cp ${MS_CCSRC_BUILD_PATH}/libgpu_collective* ${MS_LIB_PATH} + ) + add_dependencies(add_gpu_collective_lib gpu_collective) + endif() + add_custom_target(add_gpu_queue_lib ALL + COMMAND cp ${MS_CCSRC_BUILD_PATH}/libgpu_queue* ${MS_LIB_PATH} + ) + add_dependencies(add_gpu_queue_lib add_ms_lib) +endif() + +if (ENABLE_CPU) + add_custom_target(add_cpu_lib ALL + COMMAND cp ${mkl_dnn_LIBPATH}/libdnnl.so.1.1 ${MS_LIB_PATH}/libdnnl.so.1 + ) + add_dependencies(add_cpu_lib add_ms_lib) +endif() + +if (ENABLE_MINDDATA) + add_custom_target(add_minddata_lib ALL + COMMAND cp ${MS_CCSRC_BUILD_PATH}/dataset/*.so ${MS_PACK_PATH} + COMMAND cp ${MS_CCSRC_BUILD_PATH}/mindrecord/*.so ${MS_PACK_PATH} + COMMAND cp ${opencv_LIBPATH}/libopencv_core.so.4.2.0 ${MS_LIB_PATH}/libopencv_core.so.4.2 + COMMAND cp ${opencv_LIBPATH}/libopencv_imgcodecs.so.4.2.0 ${MS_LIB_PATH}/libopencv_imgcodecs.so.4.2 + COMMAND cp ${opencv_LIBPATH}/libopencv_imgproc.so.4.2.0 ${MS_LIB_PATH}/libopencv_imgproc.so.4.2 + ) + add_dependencies(add_minddata_lib add_ms_lib) + add_dependencies(add_minddata_lib _c_mindrecord) + add_dependencies(add_minddata_lib _c_dataengine) + + add_dependencies(_c_mindrecord mindspore) + add_dependencies(_c_dataengine mindspore) +endif() + +if (USE_GLOG) + target_link_libraries(_c_expression PRIVATE mindspore::glog) + add_custom_target(add_glog_lib ALL + COMMAND cp ${glog_LIBPATH}/libglog*.so.0 ${MS_LIB_PATH} + ) + add_dependencies(add_glog_lib add_ms_lib) +endif() diff --git a/mindspore/ccsrc/common.h b/mindspore/ccsrc/common.h new file mode 100644 index 00000000000..0928dcfcf68 --- /dev/null +++ b/mindspore/ccsrc/common.h @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_COMMON_H_ +#define MINDSPORE_CCSRC_COMMON_H_ + +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include "pipeline/static_analysis/dshape.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pipeline/static_analysis/abstract_function.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/parse.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/resolve.h" + +namespace py = pybind11; +#endif // MINDSPORE_CCSRC_COMMON_H_ diff --git a/mindspore/ccsrc/common/CMakeLists.txt b/mindspore/ccsrc/common/CMakeLists.txt new file mode 100644 index 00000000000..1a1a5ae9e68 --- /dev/null +++ b/mindspore/ccsrc/common/CMakeLists.txt @@ -0,0 +1,2 @@ + +add_library(_mindspore_common_obj OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/*.cc) \ No newline at end of file diff --git a/mindspore/ccsrc/common/trans.cc b/mindspore/ccsrc/common/trans.cc new file mode 100644 index 00000000000..ea84537c1a3 --- /dev/null +++ b/mindspore/ccsrc/common/trans.cc @@ -0,0 +1,744 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/trans.h" +#include +#include +#include +#include +#include "./securec.h" +#include "common/utils.h" +#include "device/convert_tensor_utils.h" +#include "utils/convert_utils.h" +#include "utils/log_adapter.h" +#include "utils/utils.h" + +namespace mindspore { +namespace trans { +const size_t kNchwDims = 4; +const std::map type_map = {{kNumberTypeBool, 1}, {kNumberTypeInt, 4}, {kNumberTypeInt8, 1}, + {kNumberTypeInt16, 2}, {kNumberTypeInt32, 4}, {kNumberTypeInt64, 8}, + {kNumberTypeUInt, 4}, {kNumberTypeUInt8, 1}, {kNumberTypeUInt16, 2}, + {kNumberTypeUInt32, 4}, {kNumberTypeUInt64, 8}, {kNumberTypeFloat, 4}, + {kNumberTypeFloat16, 2}, {kNumberTypeFloat32, 4}, {kNumberTypeFloat64, 8}}; + +template +T Ceil(T n1, T n2) { + return (n2 != 0) ? (n1 - 1) / n2 + 1 : 0; +} + +enum DataTypeTransMode { + FROM_FLOAT_TO_FLOAT16, + FROM_FLOAT_TO_INT32, + FROM_FLOAT16_TO_FLOAT, + FROM_FLOAT16_TO_INT32, + FROM_INT32_TO_FLOAT, + FROM_INT32_TO_FLOAT16, + FROM_INT32_TO_UINT8, + FROM_INT32_TO_INT8, + FROM_UINT8_TO_FLOAT, + FROM_UINT8_TO_INT32, + FROM_INT8_TO_FLOAT, + FROM_INT8_TO_INT32, + FROM_INT64_TO_INT32, +}; + +const std::map, DataTypeTransMode> mode_map{ + {std::pair(kNumberTypeFloat32, kNumberTypeFloat16), FROM_FLOAT_TO_FLOAT16}, + {std::pair(kNumberTypeFloat32, kNumberTypeInt32), FROM_FLOAT_TO_INT32}, + {std::pair(kNumberTypeFloat16, kNumberTypeFloat32), FROM_FLOAT16_TO_FLOAT}, + {std::pair(kNumberTypeFloat16, kNumberTypeInt32), FROM_FLOAT16_TO_INT32}, + {std::pair(kNumberTypeInt32, kNumberTypeFloat32), FROM_INT32_TO_FLOAT}, + {std::pair(kNumberTypeInt32, kNumberTypeFloat16), FROM_INT32_TO_FLOAT16}, + {std::pair(kNumberTypeInt32, kNumberTypeUInt8), FROM_INT32_TO_UINT8}, + {std::pair(kNumberTypeInt32, kNumberTypeInt8), FROM_INT32_TO_INT8}, + {std::pair(kNumberTypeUInt8, kNumberTypeFloat32), FROM_UINT8_TO_FLOAT}, + {std::pair(kNumberTypeUInt8, kNumberTypeInt32), FROM_UINT8_TO_INT32}, + {std::pair(kNumberTypeInt8, kNumberTypeFloat32), FROM_INT8_TO_FLOAT}, + {std::pair(kNumberTypeInt8, kNumberTypeInt32), FROM_INT8_TO_INT32}, + {std::pair(kNumberTypeInt64, kNumberTypeInt32), FROM_INT64_TO_INT32}}; + +template +void TransDataSrc2Dst(const TypeIdArgs &args, void *dst, const size_t data_size) { + for (size_t idx = 0; idx != data_size; idx++) { + SrcT src_data = static_cast(args.data)[idx]; + static_cast(dst)[idx] = static_cast(src_data); + } +} + +bool CastKernel(const TypeIdArgs &args, void *dst, const size_t data_size, const DataTypeTransMode mode) { + switch (mode) { + case FROM_FLOAT_TO_FLOAT16: + device::FloatToHalf(dst, args.data, data_size); + break; + case FROM_FLOAT16_TO_FLOAT: + device::HalfToFloat(dst, args.data, data_size); + break; + case FROM_FLOAT_TO_INT32: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_FLOAT16_TO_INT32: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_INT32_TO_FLOAT: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_INT32_TO_INT8: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_INT32_TO_UINT8: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_UINT8_TO_INT32: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_UINT8_TO_FLOAT: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_INT8_TO_FLOAT: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_INT8_TO_INT32: + TransDataSrc2Dst(args, dst, data_size); + break; + case FROM_INT64_TO_INT32: + TransDataSrc2Dst(args, dst, data_size); + break; + default: + MS_LOG(ERROR) << "unsupported datatype trans"; + return false; + } + return true; +} + +size_t CubeSizeByType(const TypeId data_type) { + const size_t default_error = 0; + auto dt_size = TypeIdSize(data_type); + if (dt_size < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return default_error; + } else if (dt_size == 1) { + return kCubeSize * 2; + } + return kCubeSize; +} + +size_t ShapeSize(const std::vector &shape) { + size_t product = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + return product; +} + +size_t TypeIdSize(const TypeId data_type) { + const size_t unsupport_type_error = 0; + auto iter = type_map.find(data_type); + if (iter != type_map.end()) { + return iter->second; + } + return unsupport_type_error; +} + +std::vector TransShapeTo4d(const std::vector &shape) { + std::vector shape_4d(4, 1); + switch (shape.size()) { + case 0: + break; + case 1: + shape_4d[1] = shape[0]; + break; + case 2: + shape_4d[0] = shape[0]; + shape_4d[1] = shape[1]; + break; + case 3: + MS_LOG(EXCEPTION) << "Unexpected shape size = 3,it should has a default format"; + case 4: + for (size_t i = 0; i < 4; ++i) { + shape_4d[i] = shape[i]; + } + break; + default: + MS_LOG(EXCEPTION) << "Unexpeted shape size = " << shape.size(); + } + return shape_4d; +} + +std::vector TransShapeToDevice(const std::vector &shape, const std::string &format) { + std::vector device_shape; + if (format == kOpFormat_FRAC_NZ) { + if (shape.size() < 2) { + MS_EXCEPTION(NotSupportError) << "format " << format << " is not support shape " << shape.size(); + } + if (shape.size() > 2) { + (void)std::copy(shape.begin(), shape.end() - 2, std::back_inserter(device_shape)); + } + auto h1 = (shape[shape.size() - 2] - 1) / kCubeSize + 1; + auto w1 = (shape[shape.size() - 1] - 1) / kCubeSize + 1; + device_shape.push_back(w1); + device_shape.push_back(h1); + device_shape.push_back(kCubeSize); + device_shape.push_back(kCubeSize); + return device_shape; + } + if (shape.size() != 4) { + MS_LOG(EXCEPTION) << "shape_4d size should be 4"; + } + if (format == kOpFormat_NC1HWC0) { + size_t C1 = (shape[1] + kCubeSize - 1) / kCubeSize; + size_t C0 = kCubeSize; + device_shape.push_back(shape[0]); + device_shape.push_back(C1); + device_shape.push_back(shape[2]); + device_shape.push_back(shape[3]); + device_shape.push_back(C0); + return device_shape; + } else if (format == kOpFormat_FRAC_Z) { + size_t cout16 = ((shape[0] + kCubeSize - 1) / kCubeSize) * kCubeSize; + size_t cin16 = ((shape[1] + kCubeSize - 1) / kCubeSize) * kCubeSize; + device_shape.push_back(shape[2] * shape[3] * cin16 / kCubeSize); + device_shape.push_back(cout16 / kCubeSize); + device_shape.push_back(kCubeSize); + device_shape.push_back(kCubeSize); + return device_shape; + } else if (format == kOpFormat_NHWC) { + device_shape.push_back(shape[0]); + device_shape.push_back(shape[2]); + device_shape.push_back(shape[3]); + device_shape.push_back(shape[1]); + return device_shape; + } else if (format == kOpFormat_NCHW) { + return shape; + } else if (format == kOpFormat_HWCN) { + return {shape[2], shape[3], shape[1], shape[0]}; + } + MS_LOG(EXCEPTION) << "Unexpected format[" << format << "]"; +} + +bool TransDataType(const TypeIdArgs &args, void *result) { + MS_LOG(DEBUG) << "begin trans datatype from " << TypeIdLabel(args.host_data_type) << " to " + << TypeIdLabel(args.device_data_type); + MS_EXCEPTION_IF_NULL(result); + std::pair type_info(args.host_data_type, args.device_data_type); + auto iter = mode_map.find(type_info); + if (iter == mode_map.end()) { + MS_LOG(ERROR) << "unsupported datatype trans. src_type :" << TypeIdLabel(args.host_data_type) + << ", dst_type:" << TypeIdLabel(args.device_data_type); + return false; + } + auto trans_mode = iter->second; + auto type_size = TypeIdSize(args.device_data_type); + if (type_size < 1) { + MS_LOG(ERROR) << "invalid host data type."; + return false; + } + if (args.host_shape_size < 1) { + MS_LOG(ERROR) << "invalid host data size."; + return false; + } + if (!CastKernel(args, result, args.host_shape_size, trans_mode)) { + MS_LOG(ERROR) << "failed to trans datatype.."; + return false; + } + return true; +} + +bool TransFormat(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "start trans format."; + if (TypeIdSize(args.src_data_type) < 1) { + MS_LOG(ERROR) << "invalid datatype.."; + return false; + } + if ((args.host_format == kOpFormat_NCHW || args.host_format == kOpFormat_ND) && + args.device_format == kOpFormat_FRAC_Z) { + return NchwToFracZ(args, result); + } else if (args.device_format == kOpFormat_FRAC_NZ) { + return NchwToFracNz(args, result); + } else if (args.device_format == kOpFormat_NC1HWC0) { + return NchwToNc1hwc0(args, result); + } + return true; +} + +bool TransFormatFromDeviceToHost(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "start trans format."; + if (TypeIdSize(args.src_data_type) < 1) { + MS_LOG(ERROR) << "invalid datatype.."; + return false; + } + if ((args.host_format == kOpFormat_NCHW || args.host_format == kOpFormat_ND) && + args.device_format == kOpFormat_FRAC_Z) { + return FracZToNchw(args, result); + } else if (args.device_format == kOpFormat_FRAC_NZ) { + return FracNzToNchw(args, result); + } else if (args.device_format == kOpFormat_NC1HWC0) { + return Nc1hwc0ToNchw(args, result); + } + return true; +} + +bool NchwToFracZ(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "trans format from nchw to frac_z"; + MS_EXCEPTION_IF_NULL(result); + if (args.host_shape.size() != kNchwDims) { + MS_LOG(ERROR) << "invalid host shape, host shape dims:" << args.host_shape.size() << ", expect dims:" << kNchwDims; + return false; + } + size_t size = TypeIdSize(args.src_data_type); + if (size < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return false; + } + auto n = args.host_shape[0]; + auto c = args.host_shape[1]; + auto h = args.host_shape[2]; + auto w = args.host_shape[3]; + + size_t c0 = CubeSizeByType(args.src_data_type); + if (c0 < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return false; + } + size_t c1 = Ceil(c, c0); + size_t hw = h * w; + size_t chw = c * hw; + size_t hwc0 = hw * c0; + size_t nchw = n * chw; + + size_t hf_cnt = Ceil(n, kCubeSize); + size_t vf_cnt = c1 * hw; + size_t fractal_ele_cnt = c0 * kCubeSize; + size_t total_ele_cnt = hf_cnt * vf_cnt * fractal_ele_cnt; + + size_t dst_size = total_ele_cnt * size; + if (dst_size != args.device_size) { + MS_LOG(ERROR) << "illegal total data size." + << "dst size is :" << dst_size << "device size is :" << args.device_size; + return false; + } + for (size_t vfi = 0; vfi < vf_cnt; vfi++) { + auto vf_base_i = vfi * hf_cnt; // vertical fractal matrix base index + for (size_t hfi = 0; hfi < hf_cnt; hfi++) { + auto gfi = vf_base_i + hfi; // global fractal matrix index + auto src_n_offset = hfi * chw * kCubeSize; + auto src_f_offset = src_n_offset + vfi % hw + vfi / hw * hwc0; + for (size_t row = 0; row < c0; row++) { + auto src_ci = vfi / hw * c0 + row; + auto src_row_offset = src_f_offset + row * hw; + for (size_t col = 0; col < kCubeSize; col++) { + auto src_ni = hfi * kCubeSize + col; + auto src_offset = src_row_offset + chw * col; + + auto need_pad_zero = src_ni >= n || src_offset >= nchw || src_ci >= c; + auto idx = gfi * fractal_ele_cnt + col * c0 + row; + auto offset = idx * size; + auto protected_size = dst_size - offset < static_cast(SECUREC_MEM_MAX_LEN) + ? dst_size - offset + : static_cast(SECUREC_MEM_MAX_LEN); + errno_t ret; + if (need_pad_zero) { + ret = memset_s(static_cast(result) + offset, protected_size, 0, size); + } else { + ret = memcpy_s(static_cast(result) + offset, protected_size, + static_cast(args.data) + src_offset * size, size); + } + if (ret != 0) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << ret; + return false; + } + } + } + } + } + return true; +} + +bool FracZToNchw(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "trans format from frac_z to nchw"; + MS_EXCEPTION_IF_NULL(result); + if (args.host_shape.size() != kNchwDims) { + MS_LOG(ERROR) << "invalid host shape, host shape dims:" << args.host_shape.size() << ", expect dims:" << kNchwDims; + return false; + } + size_t size = TypeIdSize(args.src_data_type); + if (size < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return false; + } + size_t total_size = ShapeSize(args.device_shape) * size; + if (total_size != args.device_size) { + MS_LOG(ERROR) << "illegal total data size, total_size:" << total_size << ", device_size:" << args.device_size; + return false; + } + + auto n0 = args.device_shape.at(1); + auto ni = args.device_shape.at(2); + auto c0 = args.device_shape.at(3); + + auto n = args.host_shape[0]; + auto c = args.host_shape[1]; + auto h = args.host_shape[2]; + auto w = args.host_shape[3]; + + size_t nc = ni * n0; + size_t ncc0 = nc * c0; + size_t wncc0 = w * ncc0; + size_t hwncc0 = h * wncc0; + size_t hw = h * w; + size_t chw = c * hw; + + for (size_t n_idx = 0; n_idx < n; n_idx++) { + size_t n_head_addr = n_idx * chw; + for (size_t c_idx = 0; c_idx < c; c_idx++) { + size_t c_head_addr = n_head_addr + c_idx * hw; + for (size_t h_idx = 0; h_idx < h; h_idx++) { + size_t h_head_addr = c_head_addr + h_idx * w; + for (size_t w_idx = 0; w_idx < w; w_idx++) { + size_t dst_idx = h_head_addr + w_idx; + size_t c1_idx = c_idx / c0; + size_t c0_idx = c_idx % c0; + size_t nc_idx = n_idx; + size_t src_idx = c1_idx * hwncc0 + h_idx * wncc0 + w_idx * ncc0 + nc_idx * c0 + c0_idx; + auto src_offset = src_idx * size; + auto dst_offset = dst_idx * size; + auto protected_size = total_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? total_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + auto ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size); + if (ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << ret; + return false; + } + } + } + } + } + return true; +} + +bool TransShapeToNz(const std::vector &host_shape, std::vector *hw_shape) { + MS_EXCEPTION_IF_NULL(hw_shape); + if (host_shape.empty()) { + MS_LOG(ERROR) << "size of vector is 0."; + return false; + } + switch (host_shape.size()) { + case 1: + hw_shape->push_back(1); + hw_shape->push_back(1); + hw_shape->push_back(host_shape[0]); + return true; + default: + auto size = host_shape.size(); + if (size < 2) { + MS_LOG(ERROR) << "illegal size."; + return false; + } + size_t times = 1; + for (size_t i = 0; i != size - 2; i++) { + times *= host_shape[i]; + } + hw_shape->push_back(times); + hw_shape->push_back(host_shape[size - 2]); + hw_shape->push_back(host_shape[size - 1]); + return true; + } +} + +bool NchwToFracNz(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "trans format from nchw to frac_nz."; + MS_EXCEPTION_IF_NULL(result); + std::vector hw_shape; + if (!TransShapeToNz(args.host_shape, &hw_shape)) { + MS_LOG(ERROR) << "trans shape failed.."; + return false; + } + if (hw_shape.size() < 3 || args.device_shape.size() < 4) { + MS_LOG(ERROR) << "invalid shape size."; + return false; + } + auto size = TypeIdSize(args.src_data_type); + if (size < 1) { + MS_LOG(ERROR) << "illegal dtype"; + return false; + } + + auto dst_size = ShapeSize(args.device_shape) * size; + if (dst_size != args.device_size) { + MS_LOG(ERROR) << "illegal total data size, total_size:" << dst_size << ", device_size:" << args.device_size; + return false; + } + auto times = hw_shape.at(0); + auto h = hw_shape.at(1); + auto w = hw_shape.at(2); + auto hw = h * w; + + auto shape_size = args.device_shape.size(); + auto w1 = args.device_shape[shape_size - 4]; + auto h1 = args.device_shape[shape_size - 3]; + auto h0 = args.device_shape[shape_size - 2]; + auto w0 = args.device_shape[shape_size - 1]; + auto h1h0w0 = h1 * h0 * w0; + auto w1h1h0w0 = w1 * h1h0w0; + auto num_w1 = w / w0; + + for (size_t times_idx = 0; times_idx < times; times_idx++) { + auto times_head = times_idx * w1h1h0w0; + auto src_times_head = times_idx * hw; + for (size_t h1h0_idx = 0; h1h0_idx < h; h1h0_idx++) { + auto h1h0_head = times_head + h1h0_idx * w0; + auto src_h_head = src_times_head + h1h0_idx * w; + for (size_t w1_idx = 0; w1_idx < num_w1; w1_idx++) { + size_t dst_offset = (h1h0_head + w1_idx * h1h0w0) * size; + size_t src_offset = (src_h_head + w1_idx * w0) * size; + auto protected_size = dst_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? dst_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + auto cp_ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size * w0); + if (cp_ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory, error-code " << cp_ret; + return false; + } + } + auto w1_head = num_w1 * w0; + for (size_t w0_idx = 0; w1_head + w0_idx < w; w0_idx++) { + auto src_w_idx = w1_head + w0_idx; + size_t dst_offset = (h1h0_head + num_w1 * h1h0w0 + w0_idx) * size; + size_t src_offset = (src_h_head + src_w_idx) * size; + auto protected_size = dst_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? dst_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + auto cp_ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size); + if (cp_ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << cp_ret; + return false; + } + } + } + } + return true; +} + +bool FracNzToNchw(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "trans format from frac_nz to nchw"; + MS_EXCEPTION_IF_NULL(result); + std::vector hw_shape; + if (!TransShapeToNz(args.host_shape, &hw_shape)) { + MS_LOG(ERROR) << "trans shape failed.."; + return false; + } + if (hw_shape.size() < 3 || args.device_shape.size() < 4) { + MS_LOG(ERROR) << "invalid shape size."; + return false; + } + auto size = TypeIdSize(args.src_data_type); + if (size < 1) { + MS_LOG(ERROR) << "illegal dtype"; + return false; + } + + auto dst_size = ShapeSize(args.device_shape) * size; + if (dst_size != args.device_size) { + MS_LOG(ERROR) << "illegal total data size, total_size:" << dst_size << ", device_size:" << args.device_size; + return false; + } + auto times = hw_shape.at(0); + auto h = hw_shape.at(1); + auto w = hw_shape.at(2); + auto hw = h * w; + + auto shape_size = args.device_shape.size(); + auto w1 = args.device_shape[shape_size - 4]; + auto h1 = args.device_shape[shape_size - 3]; + auto h0 = args.device_shape[shape_size - 2]; + auto w0 = args.device_shape[shape_size - 1]; + auto h1h0w0 = h1 * h0 * w0; + auto w1h1h0w0 = w1 * h1h0w0; + auto num_w1 = w / w0; + + for (size_t times_idx = 0; times_idx < times; times_idx++) { + auto times_head = times_idx * w1h1h0w0; + auto src_times_head = times_idx * hw; + for (size_t h1h0_idx = 0; h1h0_idx < h; h1h0_idx++) { + auto h1h0_head = times_head + h1h0_idx * w0; + auto src_h_head = src_times_head + h1h0_idx * w; + for (size_t w1_idx = 0; w1_idx < num_w1; w1_idx++) { + size_t src_offset = (h1h0_head + w1_idx * h1h0w0) * size; + size_t dst_offset = (src_h_head + w1_idx * w0) * size; + auto protected_size = dst_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? dst_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + auto cp_ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size * w0); + if (cp_ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory, error-code " << cp_ret; + return false; + } + } + auto w1_head = num_w1 * w0; + for (size_t w0_idx = 0; w1_head + w0_idx < w; w0_idx++) { + auto src_w_idx = w1_head + w0_idx; + size_t src_offset = (h1h0_head + num_w1 * h1h0w0 + w0_idx) * size; + size_t dst_offset = (src_h_head + src_w_idx) * size; + auto protected_size = dst_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? dst_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + auto cp_ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size); + if (cp_ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << cp_ret; + return false; + } + } + } + } + return true; +} + +bool NchwToNc1hwc0(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "trans format from nchw to Nc1h1wc0"; + MS_EXCEPTION_IF_NULL(result); + if (args.host_shape.size() != kNchwDims) { + MS_LOG(ERROR) << "invalid host shape, host shape dims:" << args.host_shape.size() << ", expect dims:" << kNchwDims; + return false; + } + size_t size = TypeIdSize(args.src_data_type); + if (size < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return false; + } + auto total_size = ShapeSize(args.device_shape) * size; + if (total_size != args.device_size) { + MS_LOG(ERROR) << "illegal total data size, total_size:" << total_size << ", device_size:" << args.device_size; + return false; + } + + auto n = args.host_shape[0]; + auto c = args.host_shape[1]; + auto h = args.host_shape[2]; + auto w = args.host_shape[3]; + size_t c0 = CubeSizeByType(args.src_data_type); + if (c0 < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return false; + } + size_t c1 = Ceil(c, c0); + size_t hw = h * w; + size_t chw = c * hw; + size_t c1hwc0 = c1 * hw * c0; + size_t wc0 = w * c0; + + for (size_t n_idx = 0; n_idx < n; n_idx++) { + size_t n_head_addr = n_idx * c1hwc0; + for (size_t c1_idx = 0; c1_idx < c1; c1_idx++) { + size_t c1_head_addr = n_head_addr + c1_idx * hw * c0; + for (size_t h_idx = 0; h_idx < h; h_idx++) { + size_t h_head_addr = c1_head_addr + h_idx * wc0; + for (size_t w_idx = 0; w_idx < w; w_idx++) { + size_t w_head_addr = h_head_addr + w_idx * c0; + for (size_t c0_idx = 0; c0_idx < c0; c0_idx++) { + size_t dst_index = c0_idx + w_head_addr; + size_t dst_offset = dst_index * size; + auto protected_size = total_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? total_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + size_t c_idx = c0_idx + c1_idx * c0; + size_t src_idx = n_idx * chw + c_idx * hw + h_idx * w + w_idx; + auto src_offset = src_idx * size; + + if (c_idx < c) { + auto ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size); + if (ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << ret; + return false; + } + } else { + auto ret = memset_s(static_cast(result) + dst_offset, protected_size, 0, size); + if (ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << ret; + return false; + } + } + } + } + } + } + } + return true; +} + +bool Nc1hwc0ToNchw(const FormatArgs &args, void *result) { + MS_LOG(DEBUG) << "trans format from nc1h1wc0 to nchw"; + MS_EXCEPTION_IF_NULL(result); + if (args.host_shape.size() != kNchwDims) { + MS_LOG(ERROR) << "invalid host shape, host shape dims:" << args.host_shape.size() << ", expect dims:" << kNchwDims; + return false; + } + size_t size = TypeIdSize(args.src_data_type); + if (size < 1) { + MS_LOG(ERROR) << "illegal dtype."; + return false; + } + size_t total_size = ShapeSize(args.device_shape) * size; + if (total_size != args.device_size) { + MS_LOG(ERROR) << "illegal total data size, total_size:" << total_size << ", device_size:" << args.device_size; + return false; + } + + auto n = args.host_shape[0]; + auto c = args.host_shape[1]; + auto h = args.host_shape[2]; + auto w = args.host_shape[3]; + auto c1 = args.device_shape[1]; + auto c0 = args.device_shape[4]; + + size_t hw = h * w; + size_t chw = c * hw; + size_t wc0 = w * c0; + size_t hwc0 = h * wc0; + size_t c1hwc0 = c1 * hwc0; + + for (size_t n_idx = 0; n_idx < n; n_idx++) { + size_t n_head_addr = n_idx * chw; + for (size_t c_idx = 0; c_idx < c; c_idx++) { + size_t c_head_addr = n_head_addr + c_idx * hw; + for (size_t h_idx = 0; h_idx < h; h_idx++) { + size_t h_head_addr = c_head_addr + h_idx * w; + for (size_t w_idx = 0; w_idx < w; w_idx++) { + size_t dst_idx = h_head_addr + w_idx; + size_t c1_idx = c_idx / c0; + size_t c0_idx = c_idx % c0; + size_t src_idx = n_idx * c1hwc0 + c1_idx * hwc0 + h_idx * wc0 + w_idx * c0 + c0_idx; + auto src_offset = src_idx * size; + auto dst_offset = dst_idx * size; + auto protected_size = total_size - dst_offset < static_cast(SECUREC_MEM_MAX_LEN) + ? total_size - dst_offset + : static_cast(SECUREC_MEM_MAX_LEN); + auto ret = memcpy_s(static_cast(result) + dst_offset, protected_size, + static_cast(args.data) + src_offset, size); + if (ret != EOK) { + MS_LOG(ERROR) << "Failed to operate the dst memory error-code " << ret; + return false; + } + } + } + } + } + return true; +} +} // namespace trans +} // namespace mindspore diff --git a/mindspore/ccsrc/common/trans.h b/mindspore/ccsrc/common/trans.h new file mode 100644 index 00000000000..cf815985ff1 --- /dev/null +++ b/mindspore/ccsrc/common/trans.h @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_COMMON_TRANS_H +#define MINDSPORE_CCSRC_COMMON_TRANS_H + +#include +#include +#include +#include +#include +#include +#include +#include "ir/dtype.h" +#include "ir/dtype/type.h" + +namespace mindspore { +namespace trans { +struct TypeIdArgs { + const void *data; + size_t host_shape_size; // Multiply each dimension elements. [a, b, c, d] => a*b*c*d + TypeId host_data_type; + TypeId device_data_type; +}; + +struct FormatArgs { + const void *data; + const size_t device_size; + std::string host_format; + std::string device_format; + std::vector host_shape; + std::vector device_shape; + TypeId src_data_type; +}; + +size_t TypeIdSize(const TypeId data_type); +size_t ShapeSize(const std::vector &shape); +size_t CubeSizeByType(const TypeId data_type); + +std::vector TransShapeTo4d(const std::vector &shape); +std::vector TransShapeToDevice(const std::vector &shape, const std::string &format); +bool TransDataType(const TypeIdArgs &args, void *result); +bool TransFormat(const FormatArgs &args, void *result); +bool TransFormatFromDeviceToHost(const FormatArgs &args, void *result); + +// host to device +bool NchwToFracZ(const FormatArgs &args, void *result); +bool NchwToFracNz(const FormatArgs &args, void *result); +bool NchwToNc1hwc0(const FormatArgs &args, void *result); +// device to host +bool FracZToNchw(const FormatArgs &args, void *result); +bool FracNzToNchw(const FormatArgs &args, void *result); +bool Nc1hwc0ToNchw(const FormatArgs &args, void *result); +} // namespace trans +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_COMMON_TRANS_H diff --git a/mindspore/ccsrc/common/utils.cc b/mindspore/ccsrc/common/utils.cc new file mode 100644 index 00000000000..328a0591131 --- /dev/null +++ b/mindspore/ccsrc/common/utils.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/utils.h" +#include +#include +#include + +namespace mindspore { +namespace common { +const int CACHED_STR_NUM = 1 << 8; +const int CACHED_STR_MASK = CACHED_STR_NUM - 1; +std::vector STR_HOLDER(CACHED_STR_NUM); +const char* SafeCStr(const std::string&& str) { + static std::atomic index{0}; + uint32_t cur_index = index++; + cur_index = cur_index & CACHED_STR_MASK; + STR_HOLDER[cur_index] = str; + return STR_HOLDER[cur_index].c_str(); +} +} // namespace common +} // namespace mindspore diff --git a/mindspore/ccsrc/common/utils.h b/mindspore/ccsrc/common/utils.h new file mode 100644 index 00000000000..7cee933ac8c --- /dev/null +++ b/mindspore/ccsrc/common/utils.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_COMMON_UTILS_H_ +#define MINDSPORE_CCSRC_COMMON_UTILS_H_ + +#include +#include +#include + +#define DISABLE_COPY_AND_ASSIGN(ClassType) \ + ClassType(const ClassType&) = delete; \ + ClassType& operator=(const ClassType&) = delete; + +namespace mindspore { +namespace common { +inline const char* SafeCStr(const std::string& str) { return str.c_str(); } +const char* SafeCStr(const std::string&& str); + +static inline std::string GetEnv(const std::string& envvar) { + const char* value = ::getenv(envvar.c_str()); + + if (value == nullptr) { + return std::string(); + } + + return std::string(value); +} +} // namespace common +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_COMMON_UTILS_H_ diff --git a/mindspore/ccsrc/dataset/CMakeLists.txt b/mindspore/ccsrc/dataset/CMakeLists.txt new file mode 100644 index 00000000000..d6791f2b9b2 --- /dev/null +++ b/mindspore/ccsrc/dataset/CMakeLists.txt @@ -0,0 +1,101 @@ +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sequence-point") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes") + +############################# Options ################################ +if (ENABLE_GPUQUE) + add_definitions(-D ENABLE_GPUQUE) + message(STATUS "GPU queue is enabled") +endif () +if (ENABLE_TDTQUE) + add_definitions(-D ENABLE_TDTQUE) + message(STATUS "TDT queue is enabled") +endif () + +add_definitions(-D ENABLE_MINDRECORD) + +# conde coverage +# option(ENABLE_COVERAGE "Enable code coverage report" OFF) +# if (ENABLE_COVERAGE) +# include(${CMAKE_SOURCE_DIR}/cmake/CodeCoverage.cmake) +# append_coverage_compiler_flags() +# endif () + +########### Set up the include directories ########################### +include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc) +include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/device/ascend/platform) + +include_directories(${CMAKE_BINARY_DIR}) # for protobuf generated .h + +include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/mindrecord/include) +###################################################################### + +####################### Flags ######################################## +# compile flags +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") + +################## Include sub-modules ############################### +add_subdirectory(util) +add_subdirectory(core) +add_subdirectory(kernels) +add_subdirectory(engine) +add_subdirectory(api) +###################################################################### + +################### Create _c_dataengine Library ###################### +set(submodules + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + ) + +if (ENABLE_TDTQUE) + add_library(_c_dataengine SHARED ${submodules} $) +else () + add_library(_c_dataengine SHARED ${submodules}) +endif () + + +set_target_properties(_c_dataengine PROPERTIES + PREFIX "${PYTHON_MODULE_PREFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}" + ) + +###################################################################### + +################# Link with external libraries ######################## +target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) +target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl protobuf::libprotobuf ${SECUREC_LIBRARY}) +target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs + mindspore::opencv_imgproc) +if (ENABLE_GPUQUE) + target_link_libraries(_c_dataengine PRIVATE gpu_queue + ${CUDNN_PATH}/lib64/libcudnn.so + ${CUDA_PATH}/lib64/libcudart.so + ${CUDA_PATH}/lib64/stubs/libcuda.so) +endif () + +if (ENABLE_TDTQUE) + target_link_libraries(_c_dataengine PRIVATE ${TSDCLIENT}) +endif () + +add_dependencies(_c_dataengine _c_mindrecord) +target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) + +if (USE_GLOG) + target_link_libraries(_c_dataengine PRIVATE mindspore::glog) +endif() + diff --git a/mindspore/ccsrc/dataset/api/CMakeLists.txt b/mindspore/ccsrc/dataset/api/CMakeLists.txt new file mode 100644 index 00000000000..bf575e12c92 --- /dev/null +++ b/mindspore/ccsrc/dataset/api/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library(APItoPython OBJECT + de_pipeline.cc + python_bindings.cc + ) +target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS}) diff --git a/mindspore/ccsrc/dataset/api/de_pipeline.cc b/mindspore/ccsrc/dataset/api/de_pipeline.cc new file mode 100644 index 00000000000..d51204f6590 --- /dev/null +++ b/mindspore/ccsrc/dataset/api/de_pipeline.cc @@ -0,0 +1,969 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/api/de_pipeline.h" + +#include +#include + +#include "common/utils.h" +#include "dataset/kernels/py_func_op.h" +#include "dataset/engine/datasetops/source/image_folder_op.h" +#include "dataset/engine/datasetops/source/mnist_op.h" +#include "dataset/engine/datasetops/source/voc_op.h" +#include "dataset/util/make_unique.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/engine/datasetops/source/manifest_op.h" +#include "dataset/engine/datasetops/source/cifar_op.h" +#include "dataset/engine/datasetops/source/celeba_op.h" +#ifdef ENABLE_MINDRECORD +#include "./shard_category.h" +#include "./shard_sample.h" +#include "./shard_shuffle.h" +#endif + +#include "dataset/util/random.h" +#include "dataset/util/status.h" +#include "utils/log_adapter.h" +#include "pybind11/stl.h" + +namespace mindspore { +namespace dataset { +using pFunction = Status (DEPipeline::*)(const py::dict &, std::shared_ptr *); + +static std::unordered_map g_parse_op_func_ = {{kStorage, &DEPipeline::ParseStorageOp}, + {kShuffle, &DEPipeline::ParseShuffleOp}, +#ifdef ENABLE_MINDRECORD + {kMindrecord, &DEPipeline::ParseMindRecordOp}, +#endif + {kMap, &DEPipeline::ParseMapOp}, + {kBatch, &DEPipeline::ParseBatchOp}, + {kRepeat, &DEPipeline::ParseRepeatOp}, + {kZip, &DEPipeline::ParseZipOp}, + {kRename, &DEPipeline::ParseRenameOp}, + {kDeviceQueue, &DEPipeline::ParseDeviceQueueOp}, + {kGenerator, &DEPipeline::ParseGeneratorOp}, + {kTfReader, &DEPipeline::ParseTFReaderOp}, + {kProject, &DEPipeline::ParseProjectOp}, + {kImageFolder, &DEPipeline::ParseImageFolderOp}, + {kMnist, &DEPipeline::ParseMnistOp}, + {kManifest, &DEPipeline::ParseManifestOp}, + {kVoc, &DEPipeline::ParseVOCOp}, + {kCifar10, &DEPipeline::ParseCifar10Op}, + {kCifar100, &DEPipeline::ParseCifar100Op}, + {kCelebA, &DEPipeline::ParseCelebAOp}}; + +DEPipeline::DEPipeline() : iterator_(nullptr) { + try { + // One time init + (void)GlobalInit(); + + // Instantiate the execution tree + tree_ = std::make_shared(); + repeat_num_ = 1; + batch_size_ = 1; + num_rows_ = 0; + num_classes_ = 0; + temp_batch_size_ = 1; + temp_drop_remainder_ = false; + } catch (const std::exception &err) { + MS_LOG(ERROR) << "Dataset pipeline exception caught on init: " << err.what() << "."; + return; + } +} + +DEPipeline::~DEPipeline() { + { + // Release GIL before joining all threads + py::gil_scoped_release gil_release; + // Release tree + tree_.reset(); + } +} + +// Function to add a Node to the Execution Tree. +Status DEPipeline::AddNodeToTree(const OpName &op_name, const py::dict &args, DsOpPtr *out) { + // For each operator, Parse through the list of arguments, + // then call the respective builder/constructor. + auto iter = g_parse_op_func_.find(op_name); + if (iter != g_parse_op_func_.end()) { + pFunction func = iter->second; + RETURN_IF_NOT_OK((this->*func)(args, out)); + } else { + RETURN_STATUS_UNEXPECTED("No such Op"); + } + // Associate current dataset op node with the tree. + RETURN_IF_NOT_OK(tree_->AssociateNode(*out)); + return Status::OK(); +} +// Function to add a child and parent relationship. +Status DEPipeline::AddChildToParentNode(const DsOpPtr &child_op, const DsOpPtr &parent_op) { + // Link this relationship. + // Note parent node takes ownership of the child + return (parent_op->AddChild(child_op)); +} + +// Function to assign the node as root. +Status DEPipeline::AssignRootNode(const DsOpPtr &dataset_op) { return (tree_->AssignRoot(dataset_op)); } + +// Function to launch the tree execution. +Status DEPipeline::LaunchTreeExec() { + RETURN_IF_NOT_OK(tree_->Prepare()); + RETURN_IF_NOT_OK(tree_->Launch()); + iterator_ = make_unique(tree_); + if (iterator_ == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create an Iterator."); + return Status::OK(); +} + +void DEPipeline::PrintTree() { + for (auto itr = tree_->begin(); itr != tree_->end(); ++itr) { + std::stringstream ss; + ss << *itr; + MS_LOG(INFO) << "Operator ID is " << itr->id() << ". Details: " << ss.str().c_str() << "."; + } +} + +Status DEPipeline::GetNextAsMap(py::dict *output) { + TensorMap row; + Status s; + { + py::gil_scoped_release gil_release; + s = iterator_->GetNextAsMap(&row); + } + RETURN_IF_NOT_OK(s); + // Generate Python dict as return + for (auto el : row) { + (*output)[common::SafeCStr(el.first)] = el.second; + } + return Status::OK(); +} + +Status DEPipeline::GetNextAsList(py::list *output) { + TensorRow row; + Status s; + { + py::gil_scoped_release gil_release; + s = iterator_->FetchNextTensorRow(&row); + } + RETURN_IF_NOT_OK(s); + // Generate Python list as return + for (auto el : row) { + output->append(el); + } + return Status::OK(); +} + +Status DEPipeline::GetOutputShapes(py::list *output) { + std::vector shapes; + Status s; + { + py::gil_scoped_release gil_release; + s = iterator_->GetOutputShapes(&shapes); + } + RETURN_IF_NOT_OK(s); + for (auto el : shapes) { + py::list shape; + for (auto dim : el.AsVector()) { + shape.append(dim); + } + output->append(shape); + } + return Status::OK(); +} + +Status DEPipeline::GetOutputTypes(py::list *output) { + std::vector types; + Status s; + { + py::gil_scoped_release gil_release; + s = iterator_->GetOutputTypes(&types); + } + RETURN_IF_NOT_OK(s); + for (auto el : types) { + output->append(el.AsNumpyType()); + } + return Status::OK(); +} + +int DEPipeline::GetDatasetSize() const { return num_rows_ / batch_size_; } + +int DEPipeline::GetBatchSize() const { return batch_size_; } + +int DEPipeline::GetRepeatCount() const { return repeat_num_; } + +int ToInt(const py::handle &handle) { return py::reinterpret_borrow(handle); } + +bool ToBool(const py::handle &handle) { return py::reinterpret_borrow(handle); } + +std::string ToString(const py::handle &handle) { return py::reinterpret_borrow(handle); } + +std::vector ToStringVector(const py::handle handle) { + py::list list = py::reinterpret_borrow(handle); + std::vector vector; + for (auto l : list) { + if (!l.is_none()) + vector.push_back(py::str(l)); + else + vector.emplace_back(""); + } + return vector; +} + +std::set ToStringSet(const py::handle handle) { + py::list list = py::reinterpret_borrow(handle); + std::set set; + for (auto l : list) { + if (!l.is_none()) { + (void)set.insert(py::str(l)); + } + } + return set; +} + +std::map ToStringMap(const py::handle handle) { + py::dict dict = py::reinterpret_borrow(handle); + std::map map; + for (auto p : dict) { + (void)map.insert(std::make_pair(ToString(p.first), ToInt(p.second))); + } + return map; +} + +std::vector ToIntVector(const py::handle handle) { + py::list list = py::reinterpret_borrow(handle); + std::vector vector; + for (auto l : list) { + if (!l.is_none()) { + vector.push_back(ToInt(l)); + } + } + return vector; +} + +std::vector ToTypeVector(const py::handle handle) { + py::list list = py::reinterpret_borrow(handle); + std::vector vector; + for (auto l : list) { + if (l.is_none()) { + vector.emplace_back(DataType()); + } else { + vector.push_back(l.cast()); + } + } + return vector; +} + +Status DEPipeline::SetBatchParameters(const py::dict &args) { + if (args["batch_size"].is_none()) { + std::string err_msg = "Error: batchSize is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + temp_batch_size_ = ToInt(args["batch_size"]); + CHECK_FAIL_RETURN_UNEXPECTED(temp_batch_size_ > 0, "Error: batchSize is invalid."); + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "drop_remainder") { + temp_drop_remainder_ = ToBool(value); + } + } + } + + return Status::OK(); +} + +Status DEPipeline::ValidateArgStorageOp(const py::dict &args) { + // Required arguments + if (((args.contains("dataset_files") && args["dataset_files"].is_none()) || args["schema"].is_none()) && + ((args.contains("dataset_dir") && args["dataset_dir"].is_none()) || + (args["schema"].is_none() && args["schema_json_string"].is_none()))) { + std::string err_msg = "Error: at least one of dataset_files or schema_file is missing"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + return Status::OK(); +} + +Status DEPipeline::ParseStorageOp(const py::dict &args, std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(ValidateArgStorageOp(args)); + std::shared_ptr builder; + if (args.contains("dataset_files") && !args["dataset_files"].is_none()) { + builder = std::make_shared(); + (void)builder->SetDatasetFileList(ToStringVector(args["dataset_files"])); + (void)builder->SetSchemaFile(ToString(args["schema"])); + } else if (args.contains("dataset_dir") && !args["dataset_dir"].is_none()) { + builder = std::make_shared(); + (void)builder->SetDatasetFilesDir(ToString(args["dataset_dir"])); + if (!args["schema"].is_none()) { + (void)builder->SetSchemaFile(ToString(args["schema"])); + } else if (!args["schema_json_string"].is_none()) { + std::unique_ptr schema = make_unique(); + std::string s = ToString(args["schema_json_string"]); + RETURN_IF_NOT_OK(schema->LoadSchemaString(s, std::vector())); + (void)builder->SetNumRows(schema->num_rows()); + (void)builder->SetSchema(std::move(schema)); + } + } + + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "prefetch_size") { + (void)builder->SetOpConnectorSize(ToInt(value)); + } else if (key == "columns_list") { + (void)builder->SetColumnsToLoad(ToStringVector(value)); + } else if (key == "distribution") { + (void)builder->SetDataDistributionFile(ToString(value)); + } else if (key == "labels_filename") { + (void)builder->setLabelsFileName(ToString(value)); + } else if (key == "dataset_usage") { + (void)builder->SetDatasetUsage(ToString(value)); + } + } + } + (void)builder->SetBatchSize(temp_batch_size_); + (void)builder->SetDropRemainder(temp_drop_remainder_); + + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + num_rows_ = op->num_rows(); + num_classes_ = op->num_classes(); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr *ptr) { + std::shared_ptr builder = std::make_shared(); + if (!args["buffer_size"].is_none()) { + (void)builder->SetShuffleSize(ToInt(args["buffer_size"])); + } else { + std::string err_msg = "Error: Shuffle buffer size is missing"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +#ifdef ENABLE_MINDRECORD +Status DEPipeline::CheckMindRecordPartitionInfo(const py::dict &args, std::vector *in_partitions) { + if (args["partitions"].is_none()) { + std::string err_msg = "Error: partitions is not set (None)"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + py::list list = py::reinterpret_borrow(args["partitions"]); + for (auto l : list) { + if (!l.is_none()) { + in_partitions->push_back(ToInt(l)); + } + } + + if (in_partitions->size() != 2) { + std::string err_msg = "Error: partitions is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + constexpr int kMaxPartitions = 64; + if (in_partitions->at(0) <= 0 || in_partitions->at(0) > kMaxPartitions) { + std::string err_msg = "Error: partitions is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + if (in_partitions->at(1) < 0 || in_partitions->at(1) >= in_partitions->at(0)) { + std::string err_msg = "Error: partitions is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + return Status::OK(); +} + +Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr *ptr) { + if (args["dataset_file"].is_none()) { + std::string err_msg = "Error: at least one of dataset_files is missing"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + std::shared_ptr builder = std::make_shared(); + (void)builder->SetDatasetFile(ToString(args["dataset_file"])); + + std::vector in_col_names; + if (!args["columns_list"].is_none()) { + in_col_names = ToStringVector(args["columns_list"]); + if (in_col_names.empty() || in_col_names[0].empty()) { + std::string err_msg = "Error: columns_list is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + (void)builder->SetColumnsToLoad(in_col_names); + } + + std::vector> operators; + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_parallel_workers") { + (void)builder->SetNumMindRecordWorkers(ToInt(value)); + } else if (key == "block_reader" && ToBool(value) == true) { + (void)builder->SetBlockReader(); + } else if (key == "global_shuffle" && ToBool(value) == true) { + uint32_t seed = args["partitions"].is_none() ? GetSeed() : 0; + operators.push_back(std::make_shared(seed)); + } + } + } + + std::vector in_partitions; + if (!args["partitions"].is_none()) { + auto ret = CheckMindRecordPartitionInfo(args, &in_partitions); + if (Status::OK() != ret) { + return ret; + } + operators.push_back(std::make_shared(1, in_partitions[0], in_partitions[1])); + } + + if (!operators.empty()) { + (void)builder->SetOperators(operators); + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + num_rows_ = op->num_rows(); + *ptr = op; + return Status::OK(); +} +#endif + +Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr *ptr) { + std::shared_ptr builder = std::make_shared(); + std::vector> tensor_op_list; + + if (args["operations"].is_none()) RETURN_STATUS_UNEXPECTED("Error: 'operations' is not set. \n"); + + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "input_columns") { + std::vector in_col_names = ToStringVector(args["input_columns"]); + (void)builder->SetInColNames(in_col_names); + } else if (key == "output_columns") { + (void)builder->SetOutColNames(ToStringVector(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "prefetch_size") { + (void)builder->SetOpConnectorSize(ToInt(value)); + } else if (key == "operations") { + py::handle tensor_ops = args["operations"]; + // operation can be a list of TensorOps or a single TensorOp. + if (py::isinstance(tensor_ops)) { + for (auto op : tensor_ops) { + std::shared_ptr tensor_op; + if (py::isinstance(op)) { + tensor_op = op.cast>(); + } else if (py::isinstance(op)) { + tensor_op = std::make_shared(op.cast()); + } else { + RETURN_STATUS_UNEXPECTED("Error: tensor_op is not recognised (not TensorOp and not pyfunc)."); + } + tensor_op_list.push_back(tensor_op); + } + } + if (tensor_op_list.empty()) RETURN_STATUS_UNEXPECTED("Error: tensor_op is invalid or not set."); + (void)builder->SetTensorFuncs(std::move(tensor_op_list)); + } else { + RETURN_STATUS_UNEXPECTED("Error: Unhandled key: " + key); + } + } + } + + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr *ptr) { + if (args["count"].is_none()) { + std::string err_msg = "Error: count is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + repeat_num_ = ToInt(args["count"]); + std::shared_ptr op; + RETURN_IF_NOT_OK(RepeatOp::Builder(ToInt(args["count"])).Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr *ptr) { + std::shared_ptr builder = std::make_shared(); + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "generator_function") { + py::object obj = py::cast(&value); + if (!py::isinstance(obj)) { + std::string err_msg = "Error: generator is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + (void)builder->SetGeneratorFunction(obj.cast()); + } else if (key == "column_names") { + (void)builder->SetColumnNames(ToStringVector(value)); + } else if (key == "column_types") { + (void)builder->SetColumnTypes(ToTypeVector(value)); + } + } + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr *ptr) { + std::shared_ptr builder; + if (py::isinstance(args["batch_size"])) { + batch_size_ = ToInt(args["batch_size"]); + CHECK_FAIL_RETURN_UNEXPECTED(batch_size_ > 0, "Error: batch_size is invalid."); + builder = std::make_shared(ToInt(args["batch_size"])); + } else if (py::isinstance(args["batch_size"])) { + builder = std::make_shared(1); + (void)builder->SetBatchSizeFunc(args["batch_size"].cast()); + } else { + std::string err_msg = "Error: batch_size is neither an Integer nor a python function"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "drop_remainder") { + (void)builder->SetDrop(ToBool(value)); + } + if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } + if (key == "per_batch_map") { + (void)builder->SetBatchMapFunc(value.cast()); + } + if (key == "input_columns") { + (void)builder->SetColumnsToMap(ToStringVector(value)); + } + } + } + + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *ptr) { + int32_t prefetch_size = 0; + if (args.contains("prefetch_size")) { + if (args["prefetch_size"].is_none()) { + prefetch_size = 16; + } else { + prefetch_size = ToInt(args["prefetch_size"]); + } + } + std::shared_ptr builder = std::make_shared(prefetch_size); + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "queue_name") { + (void)builder->SetChannelName(ToString(value)); + } else if (key == "device_type") { + (void)builder->SetDeviceType(ToString(value)); + } else if (key == "device_id") { + (void)builder->SetDeviceId(ToInt(value)); + } else if (key == "num_batch") { + (void)builder->SetNumBatch(ToInt(value)); + } + } + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptr *ptr) { + std::vector in_col_names; + std::vector out_col_names; + std::shared_ptr builder = std::make_shared(); + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "input_columns") { + in_col_names = ToStringVector(value); + } else if (key == "output_columns") { + out_col_names = ToStringVector(value); + } + } + } + if (in_col_names.empty() || in_col_names[0].empty()) { + std::string err_msg = "Error: input_column_names is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + if (out_col_names.empty() || out_col_names[0].empty()) { + std::string err_msg = "Error: output_column_names is invalid or not set."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + (void)builder->SetInColNames(in_col_names); + (void)builder->SetOutColNames(out_col_names); + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +DsOpPtr DEPipeline::ParseTakeOp(const py::dict &args) const { return DsOpPtr(); } + +Status DEPipeline::ParseZipOp(const py::dict &args, std::shared_ptr *ptr) { + std::shared_ptr builder = std::make_shared(); + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +DsOpPtr DEPipeline::ParseFilterOp(const py::dict &args) const { return DsOpPtr(); } + +Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + std::shared_ptr builder = std::make_shared(); + if (!args["dataset_files"].is_none()) { + (void)builder->SetDatasetFilesList(ToStringVector(args["dataset_files"])); + } else { + std::string err_msg = "Error: at least one of dataset_files or schema_file is missing"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::vector columns_to_load; + bool schema_exists = false; + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "columns_list") { + columns_to_load = ToStringVector(value); + (void)builder->SetColumnsToLoad(columns_to_load); + } else if (key == "shuffle_files") { + (void)builder->SetShuffleFiles(ToBool(value)); + } else if (key == "schema_file_path" || key == "schema_json_string") { + schema_exists = true; + } else if (key == "num_samples") { + (void)builder->setTotalRows(ToInt(value)); + } else if (key == "num_shards") { + (void)builder->SetNumDevices(ToInt(value)); + } else if (key == "shard_id") { + (void)builder->SetDeviceId(ToInt(value)); + } else if (key == "shard_equal_rows") { + (void)builder->SetShardEqualRows(ToBool(value)); + } + } + } + if (schema_exists) { + std::unique_ptr schema = make_unique(); + if (args.contains("schema_file_path")) { + RETURN_IF_NOT_OK(schema->LoadSchemaFile(ToString(args["schema_file_path"]), columns_to_load)); + } else { + RETURN_IF_NOT_OK(schema->LoadSchemaString(ToString(args["schema_json_string"]), columns_to_load)); + } + (void)builder->SetDataSchema(std::move(schema)); + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr *ptr) { + if (args["columns"].is_none()) { + std::string err_msg = "Error: columns is missing"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::vector columns_to_project = ToStringVector(args["columns"]); + std::shared_ptr builder = std::make_shared(columns_to_project); + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + if (args["dataset_dir"].is_none()) { + std::string err_msg = "Error: No dataset path specified"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::shared_ptr builder = std::make_shared(); + (void)builder->SetImageFolderDir(ToString(args["dataset_dir"])); + + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } else if (key == "extensions") { + (void)builder->SetExtensions(ToStringSet(value)); + } else if (key == "class_indexing") { + (void)builder->SetClassIndex(ToStringMap(value)); + } else if (key == "decode") { + (void)builder->SetDecode(ToBool(value)); + } + } + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + if (args["dataset_file"].is_none()) { + std::string err_msg = "Error: No dataset files specified for manifest"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::shared_ptr builder = std::make_shared(); + (void)builder->SetManifestFile(ToString(args["dataset_file"])); + + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } else if (key == "class_indexing") { + (void)builder->SetClassIndex(ToStringMap(value)); + } else if (key == "decode") { + (void)builder->SetDecode(ToBool(value)); + } else if (key == "usage") { + (void)builder->SetUsage(ToString(value)); + } + } + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr *ptr) { + if (args["dataset_dir"].is_none()) { + std::string err_msg = "Error: No dataset path specified"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + std::shared_ptr builder = std::make_shared(); + (void)builder->SetDir(ToString(args["dataset_dir"])); + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } else if (key == "decode") { + (void)builder->SetDecode(ToBool(value)); + } + } + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + if (args["dataset_dir"].is_none()) { + std::string err_msg = "Error: No dataset path specified"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + std::shared_ptr builder = std::make_shared(); + (void)builder->SetCifarDir(ToString(args["dataset_dir"])); + + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } + } + } + + (void)builder->SetCifarType(true); + + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + if (args["dataset_dir"].is_none()) { + std::string err_msg = "Error: No dataset path specified"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + std::shared_ptr builder = std::make_shared(); + (void)builder->SetCifarDir(ToString(args["dataset_dir"])); + + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } + } + } + + (void)builder->SetCifarType(false); + + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +int32_t DEPipeline::GetNumClasses() const { return num_classes_; } + +Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + if (args["dataset_dir"].is_none()) { + std::string err_msg = "Error: No dataset path specified"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + std::shared_ptr builder = std::make_shared(); + (void)builder->SetDir(ToString(args["dataset_dir"])); + + // Optional arguments + for (auto arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } + } + } + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} + +Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr *ptr) { + // Required arguments + if (args["dataset_dir"].is_none()) { + std::string err_msg = "Error: No dataset path specified"; + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); + } + + std::shared_ptr builder = std::make_shared(); + if (builder == nullptr) { + std::string err_msg = "Create celebaop builder failed"; + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); + } + (void)builder->SetCelebADir(ToString(args["dataset_dir"])); + for (const auto &arg : args) { + std::string key = py::str(arg.first); + py::handle value = arg.second; + if (!value.is_none()) { + if (key == "num_parallel_workers") { + (void)builder->SetNumWorkers(ToInt(value)); + } else if (key == "sampler") { + auto create = py::reinterpret_borrow(value).attr("create"); + std::shared_ptr sampler = create().cast>(); + (void)builder->SetSampler(std::move(sampler)); + } else if (key == "decode") { + (void)builder->SetDecode(ToBool(value)); + } else if (key == "extensions") { + (void)builder->SetExtensions(ToStringSet(value)); + } else if (key == "num_samples") { + (void)builder->SetNumSamples(ToInt(value)); + } else if (key == "dataset_type") { + (void)builder->SetDatasetType(ToString(value)); + } + } + } + + std::shared_ptr op; + RETURN_IF_NOT_OK(builder->Build(&op)); + *ptr = op; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/api/de_pipeline.h b/mindspore/ccsrc/dataset/api/de_pipeline.h new file mode 100644 index 00000000000..e8dde85a772 --- /dev/null +++ b/mindspore/ccsrc/dataset/api/de_pipeline.h @@ -0,0 +1,171 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_API_DE_PIPELINE_H_ +#define DATASET_API_DE_PIPELINE_H_ + +#include +#include +#include +#include +#include +#include +#include "dataset/core/client.h" // DE client +#include "dataset/engine/dataset_iterator.h" +#include "dataset/util/status.h" +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +namespace py = pybind11; +namespace mindspore { +namespace dataset { +using DsOpPtr = std::shared_ptr; + +// enum for the dataset operator names +enum OpName { + kStorage = 0, + kShuffle, +#ifdef ENABLE_MINDRECORD + kMindrecord, +#endif + kBatch, + kCache, + kRepeat, + kTake, + kZip, + kMap, + kFilter, + kDeviceQueue, + kGenerator, + kRename, + kTfReader, + kProject, + kImageFolder, + kMnist, + kManifest, + kVoc, + kCifar10, + kCifar100, + kCelebA +}; + +// The C++ binder class that we expose to the python script. +class DEPipeline { + public: + DEPipeline(); + + ~DEPipeline(); + + // Function to add a Node to the Execution Tree. + Status AddNodeToTree(const OpName &op_name, const py::dict &args, DsOpPtr *out); + + // Function to add a child and parent relationship. + static Status AddChildToParentNode(const DsOpPtr &child_op, const DsOpPtr &parent_op); + + // Function to assign the node as root. + Status AssignRootNode(const DsOpPtr &dataset_op); + + // Function to launch the tree execution. + Status LaunchTreeExec(); + + // Get a row of data as dictionary of column name to the value. + Status GetNextAsMap(py::dict *output); + + // Get a row of data as list. + Status GetNextAsList(py::list *output); + + Status GetOutputShapes(py::list *output); + + Status GetOutputTypes(py::list *output); + + int GetDatasetSize() const; + + int GetBatchSize() const; + + int GetRepeatCount() const; + + Status ParseStorageOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseShuffleOp(const py::dict &args, std::shared_ptr *ptr); + +#ifdef ENABLE_MINDRECORD + Status CheckMindRecordPartitionInfo(const py::dict &args, std::vector *ptr); + + Status ParseMindRecordOp(const py::dict &args, std::shared_ptr *ptr); +#endif + + Status ParseMapOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseRepeatOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseBatchOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseGeneratorOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseRenameOp(const py::dict &args, std::shared_ptr *ptr); + + DsOpPtr ParseTakeOp(const py::dict &args) const; + + Status ParseZipOp(const py::dict &args, std::shared_ptr *ptr); + + DsOpPtr ParseFilterOp(const py::dict &args) const; + + Status ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseTFReaderOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseProjectOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseImageFolderOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseManifestOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseVOCOp(const py::dict &args, std::shared_ptr *ptr); + + Status ParseCifar10Op(const py::dict &args, std::shared_ptr *ptr); + + Status ParseCifar100Op(const py::dict &args, std::shared_ptr *ptr); + + void PrintTree(); + + int32_t GetNumClasses() const; + + Status ParseMnistOp(const py::dict &args, std::shared_ptr *ptr); + + Status SetBatchParameters(const py::dict &args); + + Status ParseCelebAOp(const py::dict &args, std::shared_ptr *ptr); + + private: + // Execution tree that links the dataset operators. + std::shared_ptr tree_; + + std::unique_ptr iterator_; + + // Validate required args passed to storage op. + Status ValidateArgStorageOp(const py::dict &args); + + int batch_size_; + int repeat_num_; + int num_rows_; + int num_classes_; + + int temp_batch_size_; + bool temp_drop_remainder_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_API_DE_PIPELINE_H_ diff --git a/mindspore/ccsrc/dataset/api/python_bindings.cc b/mindspore/ccsrc/dataset/api/python_bindings.cc new file mode 100644 index 00000000000..86b0a5d66ab --- /dev/null +++ b/mindspore/ccsrc/dataset/api/python_bindings.cc @@ -0,0 +1,474 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "dataset/api/de_pipeline.h" +#include "dataset/kernels/no_op.h" +#include "dataset/kernels/data/one_hot_op.h" +#include "dataset/kernels/image/center_crop_op.h" +#include "dataset/kernels/image/change_mode_op.h" +#include "dataset/kernels/image/cut_out_op.h" +#include "dataset/kernels/image/decode_op.h" +#include "dataset/kernels/image/distort_bounding_box_crop_op.h" +#include "dataset/kernels/image/hwc_to_chw_op.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/image/normalize_op.h" +#include "dataset/kernels/image/pad_op.h" +#include "dataset/kernels/image/random_color_adjust_op.h" +#include "dataset/kernels/image/random_crop_decode_resize_op.h" +#include "dataset/kernels/image/random_crop_and_resize_op.h" +#include "dataset/kernels/image/random_crop_op.h" +#include "dataset/kernels/image/random_horizontal_flip_op.h" +#include "dataset/kernels/image/random_resize_op.h" +#include "dataset/kernels/image/random_rotation_op.h" +#include "dataset/kernels/image/random_vertical_flip_op.h" +#include "dataset/kernels/image/rescale_op.h" +#include "dataset/kernels/image/resize_bilinear_op.h" +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/data/type_cast_op.h" +#include "dataset/engine/datasetops/source/cifar_op.h" +#include "dataset/engine/datasetops/source/image_folder_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/mnist_op.h" +#include "dataset/engine/datasetops/source/manifest_op.h" +#ifdef ENABLE_MINDRECORD +#include "dataset/engine/datasetops/source/mindrecord_op.h" +#endif +#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h" +#include "dataset/engine/datasetops/source/sampler/pk_sampler.h" +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" +#include "dataset/engine/datasetops/source/tf_reader_op.h" +#include "dataset/engine/jagged_connector.h" +#include "dataset/kernels/data/to_float16_op.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +namespace py = pybind11; + +namespace mindspore { +namespace dataset { +#define THROW_IF_ERROR(s) \ + do { \ + Status rc = std::move(s); \ + if (rc.IsError()) throw std::runtime_error(rc.ToString()); \ + } while (false) + +void bindDEPipeline(py::module *m) { + (void)py::class_(*m, "DEPipeline") + .def(py::init<>()) + .def( + "AddNodeToTree", + [](DEPipeline &de, const OpName &op_name, const py::dict &args) { + DsOpPtr op; + THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &op)); + return op; + }, + py::return_value_policy::reference) + .def_static("AddChildToParentNode", + [](const DsOpPtr &child_op, const DsOpPtr &parent_op) { + THROW_IF_ERROR(DEPipeline::AddChildToParentNode(child_op, parent_op)); + }) + .def("AssignRootNode", + [](DEPipeline &de, const DsOpPtr &dataset_op) { THROW_IF_ERROR(de.AssignRootNode(dataset_op)); }) + .def("SetBatchParameters", + [](DEPipeline &de, const py::dict &args) { THROW_IF_ERROR(de.SetBatchParameters(args)); }) + .def("LaunchTreeExec", [](DEPipeline &de) { THROW_IF_ERROR(de.LaunchTreeExec()); }) + .def("GetNextAsMap", + [](DEPipeline &de) { + py::dict out; + THROW_IF_ERROR(de.GetNextAsMap(&out)); + return out; + }) + .def("GetNextAsList", + [](DEPipeline &de) { + py::list out; + THROW_IF_ERROR(de.GetNextAsList(&out)); + return out; + }) + .def("GetOutputShapes", + [](DEPipeline &de) { + py::list out; + THROW_IF_ERROR(de.GetOutputShapes(&out)); + return out; + }) + .def("GetOutputTypes", + [](DEPipeline &de) { + py::list out; + THROW_IF_ERROR(de.GetOutputTypes(&out)); + return out; + }) + .def("GetDatasetSize", &DEPipeline::GetDatasetSize) + .def("GetBatchSize", &DEPipeline::GetBatchSize) + .def("GetNumClasses", &DEPipeline::GetNumClasses) + .def("GetRepeatCount", &DEPipeline::GetRepeatCount); +} +void bindDatasetOps(py::module *m) { + (void)py::class_>(*m, "TFReaderOp") + .def_static("get_num_rows", [](const py::list &files, int64_t numParallelWorkers, bool estimate = false) { + int64_t count = 0; + std::vector filenames; + for (auto l : files) { + !l.is_none() ? filenames.push_back(py::str(l)) : (void)filenames.emplace_back(""); + } + THROW_IF_ERROR(TFReaderOp::CountTotalRows(&count, filenames, numParallelWorkers, estimate)); + return count; + }); + + (void)py::class_>(*m, "CifarOp") + .def_static("get_num_rows", [](const std::string &dir, int64_t numSamples, bool isCifar10) { + int64_t count = 0; + THROW_IF_ERROR(CifarOp::CountTotalRows(dir, numSamples, isCifar10, &count)); + return count; + }); + + (void)py::class_>(*m, "ImageFolderOp") + .def_static("get_num_rows_and_classes", [](const std::string &path, int64_t numSamples) { + int64_t count = 0, num_classes = 0; + THROW_IF_ERROR( + ImageFolderOp::CountRowsAndClasses(path, numSamples, std::set{}, &count, &num_classes)); + return py::make_tuple(count, num_classes); + }); + +#ifdef ENABLE_MINDRECORD + (void)py::class_>(*m, "MindRecordOp") + .def_static("get_num_rows", [](const std::string &path) { + int64_t count = 0; + THROW_IF_ERROR(MindRecordOp::CountTotalRows(path, &count)); + return count; + }); +#endif + + (void)py::class_>(*m, "ManifestOp") + .def_static("get_num_rows_and_classes", + [](const std::string &file, int64_t numSamples, const py::dict &dict, const std::string &usage) { + int64_t count = 0, num_classes = 0; + THROW_IF_ERROR(ManifestOp::CountTotalRows(file, numSamples, dict, usage, &count, &num_classes)); + return py::make_tuple(count, num_classes); + }) + .def_static("get_class_indexing", + [](const std::string &file, int64_t numSamples, const py::dict &dict, const std::string &usage) { + std::map output_class_indexing; + THROW_IF_ERROR(ManifestOp::GetClassIndexing(file, numSamples, dict, usage, &output_class_indexing)); + return output_class_indexing; + }); + + (void)py::class_>(*m, "MnistOp") + .def_static("get_num_rows", [](const std::string &dir, int64_t numSamples) { + int64_t count = 0; + THROW_IF_ERROR(MnistOp::CountTotalRows(dir, numSamples, &count)); + return count; + }); +} +void bindTensor(py::module *m) { + (void)py::class_(*m, "GlobalContext") + .def_static("config_manager", &GlobalContext::config_manager, py::return_value_policy::reference); + + (void)py::class_>(*m, "ConfigManager") + .def("__str__", &ConfigManager::ToString) + .def("set_rows_per_buffer", &ConfigManager::set_rows_per_buffer) + .def("set_num_parallel_workers", &ConfigManager::set_num_parallel_workers) + .def("set_worker_connector_size", &ConfigManager::set_worker_connector_size) + .def("set_op_connector_size", &ConfigManager::set_op_connector_size) + .def("set_seed", &ConfigManager::set_seed) + .def("get_rows_per_buffer", &ConfigManager::rows_per_buffer) + .def("get_num_parallel_workers", &ConfigManager::num_parallel_workers) + .def("get_worker_connector_size", &ConfigManager::worker_connector_size) + .def("get_op_connector_size", &ConfigManager::op_connector_size) + .def("get_seed", &ConfigManager::seed) + .def("load", [](ConfigManager &c, std::string s) { (void)c.LoadFile(s); }); + + (void)py::class_>(*m, "Tensor", py::buffer_protocol()) + .def(py::init([](py::array arr) { + std::shared_ptr out; + THROW_IF_ERROR(Tensor::CreateTensor(&out, arr)); + return out; + })) + .def_buffer([](Tensor &tensor) { + py::buffer_info info; + THROW_IF_ERROR(Tensor::GetBufferInfo(tensor, &info)); + return info; + }) + .def("__str__", &Tensor::ToString) + .def("shape", &Tensor::shape) + .def("type", &Tensor::type) + .def("as_array", [](py::object &t) { + auto &tensor = py::cast(t); + py::buffer_info info; + THROW_IF_ERROR(Tensor::GetBufferInfo(tensor, &info)); + return py::array(pybind11::dtype(info), info.shape, info.strides, info.ptr, t); + }); + + (void)py::class_(*m, "TensorShape") + .def(py::init()) + .def("__str__", &TensorShape::ToString) + .def("as_list", &TensorShape::AsPyList) + .def("is_known", &TensorShape::known); + + (void)py::class_(*m, "DataType") + .def(py::init()) + .def(py::self == py::self) + .def("__str__", &DataType::ToString); +} + +void bindTensorOps1(py::module *m) { + (void)py::class_>(*m, "TensorOp"); + + (void)py::class_>( + *m, "NormalizeOp", "Tensor operation to normalize an image. Takes mean and std.") + .def(py::init(), py::arg("meanR"), py::arg("meanG"), py::arg("meanB"), + py::arg("stdR"), py::arg("stdG"), py::arg("stdB")); + + (void)py::class_>( + *m, "RescaleOp", "Tensor operation to rescale an image. Takes scale and shift.") + .def(py::init(), py::arg("rescale"), py::arg("shift")); + + (void)py::class_>( + *m, "CenterCropOp", "Tensor operation to crop and image in the middle. Takes height and width (optional)") + .def(py::init(), py::arg("height"), py::arg("width") = CenterCropOp::kDefWidth); + + (void)py::class_>( + *m, "ResizeOp", "Tensor operation to resize an image. Takes height, width and mode") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = ResizeOp::kDefWidth, py::arg("interpolation") = ResizeOp::kDefInterpolation); + + (void)py::class_>( + *m, "ResizeBilinearOp", + "Tensor operation to resize an image using " + "Bilinear mode. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), py::arg("targetWidth") = ResizeBilinearOp::kDefWidth); + + (void)py::class_>(*m, "DecodeOp", + "Tensor operation to decode a jpg image") + .def(py::init<>()) + .def(py::init(), py::arg("rgb_format") = DecodeOp::kDefRgbFormat); + + (void)py::class_>( + *m, "RandomHorizontalFlipOp", "Tensor operation to randomly flip an image horizontally.") + .def(py::init(), py::arg("probability") = RandomHorizontalFlipOp::kDefProbability); +} + +void bindTensorOps2(py::module *m) { + (void)py::class_>( + *m, "RandomVerticalFlipOp", "Tensor operation to randomly flip an image vertically.") + .def(py::init(), py::arg("probability") = RandomVerticalFlipOp::kDefProbability); + + (void)py::class_>(*m, "RandomCropOp", + "Gives random crop of specified size " + "Takes crop size") + .def(py::init(), + py::arg("cropHeight"), py::arg("cropWidth"), py::arg("padTop") = RandomCropOp::kDefPadTop, + py::arg("padBottom") = RandomCropOp::kDefPadBottom, py::arg("padLeft") = RandomCropOp::kDefPadLeft, + py::arg("padRight") = RandomCropOp::kDefPadRight, py::arg("borderType") = RandomCropOp::kDefBorderType, + py::arg("padIfNeeded") = RandomCropOp::kDefPadIfNeeded, py::arg("fillR") = RandomCropOp::kDefFillR, + py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); + (void)py::class_>(*m, "ChannelSwapOp").def(py::init<>()); + + (void)py::class_>( + *m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB") + .def(py::init<>()); + + (void)py::class_>( + *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") + .def(py::init()); + + (void)py::class_>( + *m, "RandomRotationOp", + "Tensor operation to apply RandomRotation." + "Takes a range for degrees and " + "optional parameters for rotation center and image expand") + .def(py::init(), + py::arg("startDegree"), py::arg("endDegree"), py::arg("centerX") = RandomRotationOp::kDefCenterX, + py::arg("centerY") = RandomRotationOp::kDefCenterY, + py::arg("interpolation") = RandomRotationOp::kDefInterpolation, + py::arg("expand") = RandomRotationOp::kDefExpand, py::arg("fillR") = RandomRotationOp::kDefFillR, + py::arg("fillG") = RandomRotationOp::kDefFillG, py::arg("fillB") = RandomRotationOp::kDefFillB); +} + +void bindTensorOps3(py::module *m) { + (void)py::class_>( + *m, "RandomCropAndResizeOp", + "Tensor operation to randomly crop an image and resize to a given size." + "Takes output height and width and" + "optional parameters for lower and upper bound for aspect ratio (h/w) and scale," + "interpolation mode, and max attempts to crop") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth"), py::arg("scaleLb") = RandomCropAndResizeOp::kDefScaleLb, + py::arg("scaleUb") = RandomCropAndResizeOp::kDefScaleUb, + py::arg("aspectLb") = RandomCropAndResizeOp::kDefAspectLb, + py::arg("aspectUb") = RandomCropAndResizeOp::kDefAspectUb, + py::arg("interpolation") = RandomCropAndResizeOp::kDefInterpolation, + py::arg("maxIter") = RandomCropAndResizeOp::kDefMaxIter); + + (void)py::class_>( + *m, "RandomColorAdjustOp", + "Tensor operation to adjust an image's color randomly." + "Takes range for brightness, contrast, saturation, hue and") + .def(py::init(), py::arg("bright_factor_start"), + py::arg("bright_factor_end"), py::arg("contrast_factor_start"), py::arg("contrast_factor_end"), + py::arg("saturation_factor_start"), py::arg("saturation_factor_end"), py::arg("hue_factor_start"), + py::arg("hue_factor_end")); + + (void)py::class_>( + *m, "RandomResizeOp", + "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = RandomResizeOp::kDefTargetWidth); + + (void)py::class_>( + *m, "CutOutOp", "Tensor operation to randomly erase a portion of the image. Takes height and width.") + .def(py::init(), py::arg("boxHeight"), + py::arg("boxWidth"), py::arg("numPatches"), py::arg("randomColor") = CutOutOp::kDefRandomColor, + py::arg("fillR") = CutOutOp::kDefFillR, py::arg("fillG") = CutOutOp::kDefFillG, + py::arg("fillB") = CutOutOp::kDefFillB); +} + +void bindTensorOps4(py::module *m) { + (void)py::class_>( + *m, "DistortBoundingBoxCropOp", + "Tensor operator to crop an image randomly as long as the cropped image has sufficient " + "overlap with any one bounding box associated with original image" + "Takes aspect ratio of the generated crop box, the intersection ratio of crop box and bounding box," + "crop ratio lower and upper bounds" + "Optional parameters: number of attempts for crop, number of attempts of crop box generation") + .def(py::init(), py::arg("aspect_ratio"), py::arg("intersect_ratio"), + py::arg("crop_ratio_lower_bound"), py::arg("crop_ratio_upper_bound"), + py::arg("max_attempts") = DistortBoundingBoxCropOp::kDefMaxAttempts, + py::arg("box_gen_attempts") = DistortBoundingBoxCropOp::kDefBoxGenAttempts); + + (void)py::class_>( + *m, "TypeCastOp", "Tensor operator to type cast data to a specified type.") + .def(py::init(), py::arg("data_type")) + .def(py::init(), py::arg("data_type")); + + (void)py::class_>(*m, "NoOp", + "TensorOp that does nothing, for testing purposes only.") + .def(py::init<>()); + + (void)py::class_>( + *m, "ToFloat16Op", py::dynamic_attr(), "Tensor operator to type cast float32 data to a float16 type.") + .def(py::init<>()); + + (void)py::class_>( + *m, "RandomCropDecodeResizeOp", "equivalent to RandomCropAndResize but crops before decoding") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth"), py::arg("scaleLb") = RandomCropDecodeResizeOp::kDefScaleLb, + py::arg("scaleUb") = RandomCropDecodeResizeOp::kDefScaleUb, + py::arg("aspectLb") = RandomCropDecodeResizeOp::kDefAspectLb, + py::arg("aspectUb") = RandomCropDecodeResizeOp::kDefAspectUb, + py::arg("interpolation") = RandomCropDecodeResizeOp::kDefInterpolation, + py::arg("maxIter") = RandomCropDecodeResizeOp::kDefMaxIter); + + (void)py::class_>( + *m, "PadOp", + "Pads image with specified color, default black, " + "Takes amount to pad for top, bottom, left, right of image, boarder type and color") + .def(py::init(), py::arg("padTop"), + py::arg("padBottom"), py::arg("padLeft"), py::arg("padRight"), py::arg("borderTypes") = PadOp::kDefBorderType, + py::arg("fillR") = PadOp::kDefFillR, py::arg("fillG") = PadOp::kDefFillG, py::arg("fillB") = PadOp::kDefFillB); +} + +void bindSamplerOps(py::module *m) { + (void)py::class_>(*m, "Sampler"); + + (void)py::class_>(*m, "DistributedSampler") + .def(py::init(), py::arg("numDev"), py::arg("devId"), py::arg("shuffle"), + py::arg("seed")); + + (void)py::class_>(*m, "PKSampler") + .def(py::init(), py::arg("kVal"), py::arg("shuffle")); + + (void)py::class_>(*m, "RandomSampler") + .def(py::init(), py::arg("replacement"), py::arg("numSamples")) + .def(py::init(), py::arg("replacement")); + + (void)py::class_>(*m, "SequentialSampler") + .def(py::init<>()); + (void)py::class_>(*m, "SubsetRandomSampler") + .def(py::init>(), py::arg("indices")); + + (void)py::class_>(*m, "WeightedRandomSampler") + .def(py::init, int64_t, bool>(), py::arg("weights"), py::arg("numSamples"), + py::arg("replacement")); +} + +void bindInfoObjects(py::module *m) { + (void)py::class_(*m, "CBatchInfo") + .def(py::init()) + .def("get_epoch_num", &BatchOp::CBatchInfo::get_epoch_num) + .def("get_batch_num", &BatchOp::CBatchInfo::get_batch_num); +} + +// This is where we externalize the C logic as python modules +PYBIND11_MODULE(_c_dataengine, m) { + m.doc() = "pybind11 for _c_dataengine"; + (void)py::class_>(m, "DatasetOp"); + + (void)py::enum_(m, "OpName", py::arithmetic()) + .value("STORAGE", OpName::kStorage) + .value("SHUFFLE", OpName::kShuffle) + .value("BATCH", OpName::kBatch) +#ifdef ENABLE_MINDRECORD + .value("MINDRECORD", OpName::kMindrecord) +#endif + .value("CACHE", OpName::kCache) + .value("REPEAT", OpName::kRepeat) + .value("TAKE", OpName::kTake) + .value("ZIP", OpName::kZip) + .value("MAP", OpName::kMap) + .value("FILTER", OpName::kFilter) + .value("DEVICEQUEUE", OpName::kDeviceQueue) + .value("GENERATOR", OpName::kGenerator) + .export_values() + .value("RENAME", OpName::kRename) + .value("TFREADER", OpName::kTfReader) + .value("PROJECT", OpName::kProject) + .value("IMAGEFOLDER", OpName::kImageFolder) + .value("MNIST", OpName::kMnist) + .value("MANIFEST", OpName::kManifest) + .value("VOC", OpName::kVoc) + .value("CIFAR10", OpName::kCifar10) + .value("CIFAR100", OpName::kCifar100) + .value("CELEBA", OpName::kCelebA); + + (void)py::enum_(m, "InterpolationMode", py::arithmetic()) + .value("DE_INTER_LINEAR", InterpolationMode::kLinear) + .value("DE_INTER_CUBIC", InterpolationMode::kCubic) + .value("DE_INTER_AREA", InterpolationMode::kArea) + .value("DE_INTER_NEAREST_NEIGHBOUR", InterpolationMode::kNearestNeighbour) + .export_values(); + + (void)py::enum_(m, "BorderType", py::arithmetic()) + .value("DE_BORDER_CONSTANT", BorderType::kConstant) + .value("DE_BORDER_EDGE", BorderType::kEdge) + .value("DE_BORDER_REFLECT", BorderType::kReflect) + .value("DE_BORDER_SYMMETRIC", BorderType::kSymmetric) + .export_values(); + bindDEPipeline(&m); + bindTensor(&m); + bindTensorOps1(&m); + bindTensorOps2(&m); + bindTensorOps3(&m); + bindTensorOps4(&m); + bindSamplerOps(&m); + bindDatasetOps(&m); + bindInfoObjects(&m); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/CMakeLists.txt b/mindspore/ccsrc/dataset/core/CMakeLists.txt new file mode 100644 index 00000000000..e49ce7ff263 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(core OBJECT + client.cc + config_manager.cc + cv_tensor.cc + data_type.cc + global_context.cc + tensor.cc + tensor_shape.cc + ) +target_include_directories(core PRIVATE ${pybind11_INCLUDE_DIRS}) diff --git a/mindspore/ccsrc/dataset/core/client.cc b/mindspore/ccsrc/dataset/core/client.cc new file mode 100644 index 00000000000..6247ddae7d5 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/client.cc @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/client.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/global_context.h" +#include "dataset/util/services.h" +#include "dataset/util/sig_handler.h" + +namespace mindspore { +namespace dataset { +// This is a one-time global initializer which includes the call to instantiate singletons. +// It is external api call and not a member of the GlobalContext directly. +Status GlobalInit() { + // Bring up all the services (logger, task, bufferpool) + return (Services::CreateInstance()); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/client.h b/mindspore/ccsrc/dataset/core/client.h new file mode 100644 index 00000000000..ac289a0e078 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/client.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_CLIENT_H_ +#define DATASET_CORE_CLIENT_H_ + +// client.h +// Include file for DE client functions + +#include "dataset/core/constants.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/engine/datasetops/batch_op.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/engine/datasetops/device_queue_op.h" +#include "dataset/engine/datasetops/map_op.h" +#include "dataset/engine/datasetops/project_op.h" +#include "dataset/engine/datasetops/rename_op.h" +#include "dataset/engine/datasetops/repeat_op.h" +#include "dataset/engine/datasetops/shuffle_op.h" +#include "dataset/engine/datasetops/source/generator_op.h" +#include "dataset/engine/datasetops/source/mindrecord_op.h" +#include "dataset/engine/datasetops/source/storage_op.h" +#include "dataset/engine/datasetops/source/tf_reader_op.h" +#include "dataset/engine/datasetops/zip_op.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// This is a one-time global initializer that needs to be called at the +// start of any minddata applications. +extern Status GlobalInit(); +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_CORE_CLIENT_H_ diff --git a/mindspore/ccsrc/dataset/core/config_manager.cc b/mindspore/ccsrc/dataset/core/config_manager.cc new file mode 100644 index 00000000000..3f659555f4b --- /dev/null +++ b/mindspore/ccsrc/dataset/core/config_manager.cc @@ -0,0 +1,92 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/config_manager.h" + +#include +#include +#include + +#include "dataset/util/system_pool.h" + +namespace mindspore { +namespace dataset { +// A print method typically used for debugging +void ConfigManager::Print(std::ostream &out) const { + // Don't show the test/internal ones. Only display the main ones here. + // fyi, boolalpha tells the output stream to write "true" and "false" for bools + out << "\nClient config settings :" + << "\nDataCache Rows per buffer : " << rows_per_buffer_ + << "\nParallelOp workers : " << num_parallel_workers_ + << "\nParallelOp worker connector size : " << worker_connector_size_ + << "\nSize of each Connector : " << op_connector_size_ << std::endl; +} + +// Private helper function that taks a nlohmann json format and populates the settings +Status ConfigManager::FromJson(const nlohmann::json &j) { + set_rows_per_buffer(j.value("rowsPerBuffer", rows_per_buffer_)); + set_num_parallel_workers(j.value("numParallelWorkers", num_parallel_workers_)); + set_worker_connector_size(j.value("workerConnectorSize", worker_connector_size_)); + set_op_connector_size(j.value("opConnectorSize", op_connector_size_)); + set_seed(j.value("seed", seed_)); + return Status::OK(); +} + +// Loads a json file with the default settings and populates all the settings +Status ConfigManager::LoadFile(const std::string &settingsFile) { + Status rc; + if (!Path(settingsFile).Exists()) { + RETURN_STATUS_UNEXPECTED("File is not found"); + } + // Some settings are mandatory, others are not (with default). If a setting + // is optional it will set a default value if the config is missing from the file. + try { + std::ifstream in(settingsFile); + nlohmann::json js; + in >> js; + rc = FromJson(js); + } catch (const nlohmann::json::type_error &e) { + std::ostringstream ss; + ss << "Client settings failed to load:\n" << e.what(); + std::string err_msg = ss.str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } catch (const std::exception &err) { + std::ostringstream ss; + ss << "Client settings failed to load:\n" << err.what(); + std::string err_msg = ss.str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + return rc; +} + +// Setter function +void ConfigManager::set_rows_per_buffer(int32_t rows_per_buffer) { rows_per_buffer_ = rows_per_buffer; } + +// Setter function +void ConfigManager::set_num_parallel_workers(int32_t num_parallel_workers) { + num_parallel_workers_ = num_parallel_workers; +} + +// Setter function +void ConfigManager::set_worker_connector_size(int32_t connector_size) { worker_connector_size_ = connector_size; } + +// Setter function +void ConfigManager::set_op_connector_size(int32_t connector_size) { op_connector_size_ = connector_size; } + +uint32_t ConfigManager::seed() const { return seed_; } + +void ConfigManager::set_seed(uint32_t seed) { seed_ = seed; } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/config_manager.h b/mindspore/ccsrc/dataset/core/config_manager.h new file mode 100644 index 00000000000..654d5f930c4 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/config_manager.h @@ -0,0 +1,128 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_CONFIG_MANAGER_H_ +#define DATASET_CORE_CONFIG_MANAGER_H_ + +#include +#include +#include + +#include + +#include "dataset/core/constants.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" + +// Config settings for the client-side +// example config file: +// { +// "rowsPerBuffer": 3 +// } +// + +namespace mindspore { +namespace dataset { +// The ConfigManager is a class for managing default values. When a user is constructing any objects +// in the framework, often they may choose to omit some settings instead of overriding them. +// This class manages some of the default values, for cases when the user does not manually specify +// those values. +class ConfigManager { + public: + ConfigManager() = default; + + // destructor + ~ConfigManager() = default; + + // A print method typically used for debugging + // @param out - The output stream to write output to + void Print(std::ostream &out) const; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param cS - reference to the ConfigManager to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const ConfigManager &cS) { + cS.Print(out); + return out; + } + + // Another debug print helper. Converts the print info to a string for you. + // @return The string version of the debug print + std::string ToString() { + std::stringstream ss; + ss << *this; + return ss.str(); + } + + // Loads a json file with the default settings and populates all the settings + // @param settingsFile - A json file with a set of default settings + // @return Status error code + Status LoadFile(const std::string &settingsFile); + + // getter function + // @return The rows per buffer setting + int32_t rows_per_buffer() const { return rows_per_buffer_; } + + // getter function + // @return The number of workers setting + int32_t num_parallel_workers() const { return num_parallel_workers_; } + + // getter function + // @return The queue size of the operator's output connector + int32_t op_connector_size() const { return op_connector_size_; } + + // getter function + // @return The internal worker-to-master connector queue size + int32_t worker_connector_size() const { return worker_connector_size_; } + + // setter function + // @param rows_per_buffer - The setting to apply to the config + void set_rows_per_buffer(int32_t rows_per_buffer); + + // setter function + // @param num_parallel_workers - The setting to apply to the config + void set_num_parallel_workers(int32_t num_parallel_workers); + + // setter function + // @param connector_size - The setting to apply to the config + void set_worker_connector_size(int32_t connector_size); + + // setter function + // @param connector_size - The setting to apply to the config + void set_op_connector_size(int32_t connector_size); + + uint32_t seed() const; + + // setter function + // @param seed - The default seed to use + void set_seed(uint32_t seed); + + private: + int32_t rows_per_buffer_{kCfgRowsPerBuffer}; + int32_t num_parallel_workers_{kCfgParallelWorkers}; + int32_t worker_connector_size_{kCfgWorkerConnectorSize}; + int32_t op_connector_size_{kCfgOpConnectorSize}; + uint32_t seed_{kCfgDefaultSeed}; + + // Private helper function that taks a nlohmann json format and populates the settings + // @param j - The json nlohmann json info + Status FromJson(const nlohmann::json &j); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_CORE_CONFIG_MANAGER_H_ diff --git a/mindspore/ccsrc/dataset/core/constants.h b/mindspore/ccsrc/dataset/core/constants.h new file mode 100644 index 00000000000..9c0e24acc63 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/constants.h @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_CONSTANTS_H_ +#define DATASET_CORE_CONSTANTS_H_ + +#include +#include +#include + +namespace mindspore { +namespace dataset { +// Various type defines for convenience +using uchar = unsigned char; +using dsize_t = int64_t; + +// Possible dataset types for holding the data and client type +enum class DatasetType { kUnknown, kArrow, kTf }; + +// Possible flavours of Tensor implementations +enum class TensorImpl { kNone, kFlexible, kCv, kNP }; + +// convenience functions for 32bit int bitmask +inline bool BitTest(uint32_t bits, uint32_t bitMask) { return (bits & bitMask) == bitMask; } + +inline void BitSet(uint32_t *bits, uint32_t bitMask) { *bits |= bitMask; } + +inline void BitClear(uint32_t *bits, uint32_t bitMask) { *bits &= (~bitMask); } + +constexpr int32_t kDeMaxDim = std::numeric_limits::max(); // 2147483647 or 2^32 -1 +constexpr int32_t kDeMaxRank = std::numeric_limits::max(); + +constexpr uint32_t kCfgRowsPerBuffer = 1; +constexpr uint32_t kCfgParallelWorkers = 4; +constexpr uint32_t kCfgWorkerConnectorSize = 16; +constexpr uint32_t kCfgOpConnectorSize = 16; +constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed; + +// Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h) +constexpr uint8_t kCVInvalidType = 255; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_CORE_CONSTANTS_H_ diff --git a/mindspore/ccsrc/dataset/core/cv_tensor.cc b/mindspore/ccsrc/dataset/core/cv_tensor.cc new file mode 100644 index 00000000000..b09751128d2 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/cv_tensor.cc @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/cv_tensor.h" + +#include +#include + +#include "dataset/core/constants.h" +#include "dataset/core/tensor.h" + +namespace mindspore { +namespace dataset { +CVTensor::CVTensor(const TensorShape &shape, const DataType &type) : Tensor(shape, type) { + (void)this->MatInit(StartAddr(), shape_, type_, &mat_); +} + +CVTensor::CVTensor(const TensorShape &shape, const DataType &type, const uchar *data) : Tensor(shape, type, data) { + (void)this->MatInit(StartAddr(), shape_, type_, &mat_); +} + +CVTensor::CVTensor(std::shared_ptr tensor) : Tensor(std::move(*tensor)) { + (void)this->MatInit(StartAddr(), shape_, type_, &mat_); +} + +std::pair, int> CVTensor::IsValidImage(const TensorShape &shape, const DataType &type) { + std::array size = {1, 1}; + if (shape.Rank() <= 2 || (shape.Rank() == 3 && shape[2] <= CV_CN_MAX)) { + uint8_t ch = 1; + if (shape.Rank() == 3) { + ch = static_cast(shape[2]); + } + if (shape.Rank() > 0) size[0] = static_cast(shape[0]); + if (shape.Rank() > 1) size[1] = static_cast(shape[1]); + if (type.AsCVType() == kCVInvalidType) return std::make_pair(size, -1); + + int cv_type = CV_MAKETYPE(type.AsCVType(), ch); + return std::make_pair(size, cv_type); + } + return std::make_pair(size, -1); +} + +std::shared_ptr CVTensor::AsCVTensor(std::shared_ptr t) { + std::shared_ptr cv_t = std::dynamic_pointer_cast(t); + if (cv_t != nullptr) { + return cv_t; + } else { + return std::make_shared(t); + } +} + +Status CVTensor::MatInit(uchar *data, const TensorShape &shape, const DataType &type, cv::Mat *mat) { + std::pair, int> cv_shape_type = IsValidImage(shape, type); + if (cv_shape_type.second == -1) { + std::vector sizes = shape.AsVector(); + std::vector sizes32(sizes.begin(), sizes.end()); // convert long to int for usage with OpenCV + if (static_cast(shape.Rank()) != shape.Rank()) { + RETURN_STATUS_UNEXPECTED("Error in creating CV mat. Wrong shape."); + } + + uint8_t cv_type = type.AsCVType(); + if (cv_type == kCVInvalidType) { + RETURN_STATUS_UNEXPECTED("Error in creating CV mat. Invalid type."); + } + *mat = cv::Mat(static_cast(shape.Rank()), &sizes32[0], cv_type, data); + } else { + *mat = cv::Mat(2, &(cv_shape_type.first[0]), cv_shape_type.second, data); + } + return Status::OK(); +} + +Status CVTensor::Reshape(const TensorShape &shape) { + RETURN_IF_NOT_OK(Tensor::Reshape(shape)); + RETURN_IF_NOT_OK(this->MatInit(StartAddr(), shape_, type_, &mat_)); + return Status::OK(); +} + +Status CVTensor::ExpandDim(const dsize_t &axis) { + RETURN_IF_NOT_OK(Tensor::ExpandDim(axis)); + RETURN_IF_NOT_OK(this->MatInit(StartAddr(), shape_, type_, &mat_)); + return Status::OK(); +} + +void CVTensor::Squeeze() { + Tensor::Squeeze(); + (void)this->MatInit(StartAddr(), shape_, type_, &mat_); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/cv_tensor.h b/mindspore/ccsrc/dataset/core/cv_tensor.h new file mode 100644 index 00000000000..8c136f5f3cf --- /dev/null +++ b/mindspore/ccsrc/dataset/core/cv_tensor.h @@ -0,0 +1,106 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_CV_TENSOR_H_ +#define DATASET_CORE_CV_TENSOR_H_ + +#include +#include +#include + +#include + +#include "./securec.h" + +#include "dataset/core/constants.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" + +namespace mindspore { +namespace dataset { +class CVTensor : public Tensor { + public: + // Create an empty CVTensor of shape `shape` and type `type`. + // @note The shape and type information should be known and valid. + // @param shape TensorShape + // @param type DataType + CVTensor(const TensorShape &shape, const DataType &type); + + // Create a CVTensor from a given buffer, shape and type. + // @note This constructor allocates a new space in the memory and copies the buffer into it. + // @note The buffer should be valid and the shape and type information should be known and valid. + // @param shape TensorShape + // @param type DataType + // @param data unsigned char*, pointer to the data. + CVTensor(const TensorShape &shape, const DataType &type, const uchar *data); + + // Create a CVTensor from a given CV::Mat. + // @note This constructor allocates a new space in the memory and copies the CV::Mat buffer into it. + // @param mat CV::Mat + explicit CVTensor(const cv::Mat &mat) + : CVTensor(TensorShape(mat.size, mat.type()), DataType::FromCVType(mat.type()), mat.data) {} + + ~CVTensor() = default; + + // Static function to cast a given Tensor as CVTensor. If the input tensor is already of type CVTensor, + // this function would be treated as a no-op. Fot other tensor types, a new CVTensor is created based on the data + // provided. The Passed Tensor will be invalidated. + // @note there is no memory copying here, the buffer will be assigned to the constructed tensor. + // @param tensor + // @return CVTensor + static std::shared_ptr AsCVTensor(std::shared_ptr tensor); + + // Create a CVTensor from a given tensor. The input tensor will be invalidated (i.e., the shape and type will be + // set to unknown and the data buffer will point to null. + // @note there is no memory copying here, the buffer will be assigned to the constructed tensor. + // @param tensor + explicit CVTensor(std::shared_ptr tensor); + + // Getter function for the CV::Mat + // @return + cv::Mat mat() const { return mat_; } + + // Static function to check if the passed information (shape and type) can be treated as a valid description + // of an image in OpenCV. Moreover, it returns OpenCV shape and type + // For example, if the shape is <512,512,3> and type is DE_UINT8, the output would be [512,512] and CV_8UC3. + // In case of invalid shape or type, the function will return pair + // @param shape TensorShape + // @param type DataType + // @return std::pair of OpenCV shape and type + std::pair, int> IsValidImage(const TensorShape &shape, const DataType &type); + + Status Reshape(const TensorShape &shape) override; + + Status ExpandDim(const dsize_t &axis) override; + + void Squeeze() override; + + Status Mat(const std::vector &index, cv::Mat *mat) { + uchar *start = nullptr; + TensorShape remaining({-1}); + RETURN_IF_NOT_OK(this->StartAddrOfIndex(index, &start, &remaining)); + RETURN_IF_NOT_OK(this->MatInit(start, remaining, type_, mat)); + return Status::OK(); + } + + private: + cv::Mat mat_; + + // Initialize CV::Mat with the data_, shape_ and type_ + Status MatInit(uchar *data, const TensorShape &shape, const DataType &type, cv::Mat *mat); +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_CORE_CV_TENSOR_H_ diff --git a/mindspore/ccsrc/dataset/core/data_type.cc b/mindspore/ccsrc/dataset/core/data_type.cc new file mode 100644 index 00000000000..5451e2bbda9 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/data_type.cc @@ -0,0 +1,311 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/data_type.h" + +#include + +#include "utils/log_adapter.h" + +#include "dataset/core/constants.h" +#include "dataset/core/pybind_support.h" +#include "dataset/util/de_error.h" + +namespace mindspore { +namespace dataset { +uint8_t DataType::SizeInBytes() const { + switch (type_) { + case DataType::DE_BOOL: + case DataType::DE_INT8: + case DataType::DE_UINT8: + return 1; + case DataType::DE_INT16: + case DataType::DE_UINT16: + case DataType::DE_FLOAT16: + return 2; + case DataType::DE_INT32: + case DataType::DE_UINT32: + case DataType::DE_FLOAT32: + return 4; + case DataType::DE_INT64: + case DataType::DE_UINT64: + case DataType::DE_FLOAT64: + return 8; + default: + return 0; + } +} + +py::dtype DataType::AsNumpyType() const { + std::string s; + switch (type_) { + case DataType::DE_BOOL: + s = "bool"; + break; + case DataType::DE_INT8: + s = "int8"; + break; + case DataType::DE_UINT8: + s = "uint8"; + break; + case DataType::DE_INT16: + s = "int16"; + break; + case DataType::DE_UINT16: + s = "uint16"; + break; + case DataType::DE_INT32: + s = "int32"; + break; + case DataType::DE_UINT32: + s = "uint32"; + break; + case DataType::DE_INT64: + s = "int64"; + break; + case DataType::DE_UINT64: + s = "uint64"; + break; + case DataType::DE_FLOAT16: + s = "float16"; + break; + case DataType::DE_FLOAT32: + s = "float32"; + break; + case DataType::DE_FLOAT64: + s = "double"; + break; + case DataType::DE_UNKNOWN: + s = "unknown"; + break; + default: + s = "unknown"; + break; + } + return py::dtype(s); +} + +uint8_t DataType::AsCVType() const { + switch (type_) { + case DataType::DE_BOOL: + return CV_8U; + case DataType::DE_INT8: + return CV_8S; + case DataType::DE_UINT8: + return CV_8U; + case DataType::DE_INT16: + return CV_16S; + case DataType::DE_UINT16: + return CV_16U; + case DataType::DE_INT32: + return CV_32S; + case DataType::DE_FLOAT16: + return CV_16F; + case DataType::DE_FLOAT32: + return CV_32F; + case DataType::DE_FLOAT64: + return CV_64F; + case DataType::DE_UINT32: + case DataType::DE_INT64: + case DataType::DE_UINT64: + default: + MS_LOG(ERROR) << "Cannot convert to OpenCV type. Return invalid type!"; + return kCVInvalidType; + } +} + +DataType DataType::FromCVType(int cv_type) { + auto depth = static_cast(cv_type) & static_cast(CV_MAT_DEPTH_MASK); + switch (depth) { + case CV_8S: + return DataType(DataType::DE_INT8); + case CV_8U: + return DataType(DataType::DE_UINT8); + case CV_16S: + return DataType(DataType::DE_INT16); + case CV_16U: + return DataType(DataType::DE_UINT16); + case CV_32S: + return DataType(DataType::DE_INT32); + case CV_16F: + return DataType(DataType::DE_FLOAT16); + case CV_32F: + return DataType(DataType::DE_FLOAT32); + case CV_64F: + return DataType(DataType::DE_FLOAT64); + default: + MS_LOG(ERROR) << "Cannot convert from OpenCV type, unknown CV type. Unknown data type is returned!"; + return DataType(DataType::DE_UNKNOWN); + } +} + +DataType::DataType(const std::string &type_str) { + if (type_str == "bool") + type_ = DE_BOOL; + else if (type_str == "int8") + type_ = DE_INT8; + else if (type_str == "uint8") + type_ = DE_UINT8; + else if (type_str == "int16") + type_ = DE_INT16; + else if (type_str == "uint16") + type_ = DE_UINT16; + else if (type_str == "int32") + type_ = DE_INT32; + else if (type_str == "uint32") + type_ = DE_UINT32; + else if (type_str == "int64") + type_ = DE_INT64; + else if (type_str == "uint64") + type_ = DE_UINT64; + else if (type_str == "float16") + type_ = DE_FLOAT16; + else if (type_str == "float32") + type_ = DE_FLOAT32; + else if (type_str == "float64") + type_ = DE_FLOAT64; + else + type_ = DE_UNKNOWN; +} + +std::string DataType::ToString() const { + switch (type_) { + case DataType::DE_BOOL: + return "bool"; + case DataType::DE_INT8: + return "int8"; + case DataType::DE_UINT8: + return "uint8"; + case DataType::DE_INT16: + return "int16"; + case DataType::DE_UINT16: + return "uint16"; + case DataType::DE_INT32: + return "int32"; + case DataType::DE_UINT32: + return "uint32"; + case DataType::DE_INT64: + return "int64"; + case DataType::DE_UINT64: + return "uint64"; + case DataType::DE_FLOAT16: + return "float16"; + case DataType::DE_FLOAT32: + return "float32"; + case DataType::DE_FLOAT64: + return "float64"; + case DataType::DE_UNKNOWN: + return "unknown"; + default: + return "unknown"; + } +} + +DataType DataType::FromNpType(const py::dtype &type) { + if (type.is(py::dtype("bool"))) { + return DataType(DataType::DE_BOOL); + } else if (type.is(py::dtype("int8"))) { + return DataType(DataType::DE_INT8); + } else if (type.is(py::dtype("uint8"))) { + return DataType(DataType::DE_UINT8); + } else if (type.is(py::dtype("int16"))) { + return DataType(DataType::DE_INT16); + } else if (type.is(py::dtype("uint16"))) { + return DataType(DataType::DE_UINT16); + } else if (type.is(py::dtype("int32"))) { + return DataType(DataType::DE_INT32); + } else if (type.is(py::dtype("uint32"))) { + return DataType(DataType::DE_UINT32); + } else if (type.is(py::dtype("int64"))) { + return DataType(DataType::DE_INT64); + } else if (type.is(py::dtype("uint64"))) { + return DataType(DataType::DE_UINT64); + } else if (type.is(py::dtype("float16"))) { + return DataType(DataType::DE_FLOAT16); + } else if (type.is(py::dtype("float32"))) { + return DataType(DataType::DE_FLOAT32); + } else if (type.is(py::dtype("double"))) { + return DataType(DataType::DE_FLOAT64); + } else { + MS_LOG(ERROR) << "Cannot convert from numpy type. Unknown data type is returned!"; + return DataType(DataType::DE_UNKNOWN); + } +} + +DataType DataType::FromNpArray(const py::array &arr) { + if (py::isinstance>(arr)) { + return DataType(DataType::DE_BOOL); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_INT8); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_UINT8); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_INT16); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_UINT16); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_INT32); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_UINT32); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_INT64); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_UINT64); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_FLOAT16); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_FLOAT32); + } else if (py::isinstance>(arr)) { + return DataType(DataType::DE_FLOAT64); + } else { + MS_LOG(ERROR) << "Cannot convert from numpy type. Unknown data type is returned!"; + return DataType(DataType::DE_UNKNOWN); + } +} + +std::string DataType::GetPybindFormat() const { + switch (type_) { + case DataType::DE_BOOL: + return py::format_descriptor::format(); + case DataType::DE_INT8: + return py::format_descriptor::format(); + case DataType::DE_UINT8: + return py::format_descriptor::format(); + case DataType::DE_INT16: + return py::format_descriptor::format(); + case DataType::DE_UINT16: + return py::format_descriptor::format(); + case DataType::DE_INT32: + return py::format_descriptor::format(); + case DataType::DE_UINT32: + return py::format_descriptor::format(); + case DataType::DE_INT64: + return py::format_descriptor::format(); + case DataType::DE_UINT64: + return py::format_descriptor::format(); + case DataType::DE_FLOAT16: + // Eigen 3.3.7 doesn't support py::format_descriptor::format() + return "e"; + case DataType::DE_FLOAT32: + return py::format_descriptor::format(); + case DataType::DE_FLOAT64: + return py::format_descriptor::format(); + default: + MS_LOG(ERROR) << "Cannot convert from data type to pybind format descriptor!"; + return ""; + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/data_type.h b/mindspore/ccsrc/dataset/core/data_type.h new file mode 100644 index 00000000000..70a9ffdedf7 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/data_type.h @@ -0,0 +1,292 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_DATA_TYPE_H_ +#define DATASET_CORE_DATA_TYPE_H_ + +#include +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" +#include "dataset/core/pybind_support.h" + +namespace py = pybind11; +namespace mindspore { +namespace dataset { +// Class that represents basic data types in DataEngine. +class DataType { + public: + enum Type : uint8_t { + DE_BOOL, + DE_INT8, + DE_UINT8, + DE_INT16, + DE_UINT16, + DE_INT32, + DE_UINT32, + DE_INT64, + DE_UINT64, + DE_FLOAT16, + DE_FLOAT32, + DE_FLOAT64, + DE_UNKNOWN + }; + + static constexpr uint8_t DE_BOOL_SIZE = 1; + static constexpr uint8_t DE_UINT8_SIZE = 1; + static constexpr uint8_t DE_INT8_SIZE = 1; + static constexpr uint8_t DE_UINT16_SIZE = 2; + static constexpr uint8_t DE_INT16_SIZE = 2; + static constexpr uint8_t DE_UINT32_SIZE = 4; + static constexpr uint8_t DE_INT32_SIZE = 4; + static constexpr uint8_t DE_INT64_SIZE = 8; + static constexpr uint8_t DE_UINT64_SIZE = 8; + static constexpr uint8_t DE_FLOAT32_SIZE = 4; + static constexpr uint8_t DE_FLOAT64_SIZE = 8; + + // No arg constructor to create an unknown shape + DataType() : type_(DE_UNKNOWN) {} + + // Create a type from a given string + // @param type_str + explicit DataType(const std::string &type_str); + + // Default destructor + ~DataType() = default; + + // Create a type from a given enum + // @param d + constexpr explicit DataType(Type d) : type_(d) {} + + constexpr bool operator==(const DataType a) const { return type_ == a.type_; } + + constexpr bool operator==(const Type a) const { return type_ == a; } + + constexpr bool operator!=(const DataType a) const { return type_ != a.type_; } + + constexpr bool operator!=(const Type a) const { return type_ != a; } + + // Disable this usage `if(d)` where d is of type DataType + // @return + operator bool() = delete; + + // To be used in Switch/case + // @return + operator Type() const { return type_; } + + // The number of bytes needed to store one value of this type + // @return + uint8_t SizeInBytes() const; + + // Convert from DataType to OpenCV type + // @return + uint8_t AsCVType() const; + + // Convert from OpenCV type to DataType + // @param cv_type + // @return + static DataType FromCVType(int cv_type); + + // Returns a string representation of the type + // @return + std::string ToString() const; + + // returns true if the template type is the same as the Tensor type_ + // @tparam T + // @return true or false + template + bool IsCompatible() const; + + // returns true if the template type is the same as the Tensor type_ + // @tparam T + // @return true or false + template + bool IsLooselyCompatible() const; + + // << Stream output operator overload + // @notes This allows you to print the info using stream operators + // @param out - reference to the output stream being overloaded + // @param rO - reference to the DataType to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const DataType &so) { + out << so.ToString(); + return out; + } + + // Convert from DataType to Pybind type + // @return + py::dtype AsNumpyType() const; + + // Convert from NP type to DataType + // @param type + // @return + static DataType FromNpType(const py::dtype &type); + + // Convert from NP array to DataType + // @param py array + // @return + static DataType FromNpArray(const py::array &arr); + + // Get the buffer string format of the current type. Used in pybind buffer protocol. + // @return + std::string GetPybindFormat() const; + + bool IsSignedInt() const { + return type_ == DataType::DE_INT8 || type_ == DataType::DE_INT16 || type_ == DataType::DE_INT32 || + type_ == DataType::DE_INT64; + } + + bool IsUnsignedInt() const { + return type_ == DataType::DE_UINT8 || type_ == DataType::DE_UINT16 || type_ == DataType::DE_UINT32 || + type_ == DataType::DE_UINT64; + } + + bool IsInt() const { return IsSignedInt() || IsUnsignedInt(); } + + bool IsFloat() const { + return type_ == DataType::DE_FLOAT16 || type_ == DataType::DE_FLOAT32 || type_ == DataType::DE_FLOAT64; + } + + bool IsBool() const { return type_ == DataType::DE_BOOL; } + + Type value() const { return type_; } + + private: + Type type_; +}; + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_BOOL; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_FLOAT64; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_FLOAT32; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_FLOAT16; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_INT64; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_UINT64; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_INT32; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_UINT32; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_INT16; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_UINT16; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_INT8; +} + +template <> +inline bool DataType::IsCompatible() const { + return type_ == DataType::DE_UINT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_BOOL; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_FLOAT64 || type_ == DataType::DE_FLOAT32; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_FLOAT32; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_FLOAT16; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_INT64 || type_ == DataType::DE_INT32 || type_ == DataType::DE_INT16 || + type_ == DataType::DE_INT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_UINT64 || type_ == DataType::DE_UINT32 || type_ == DataType::DE_UINT16 || + type_ == DataType::DE_UINT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_INT32 || type_ == DataType::DE_INT16 || type_ == DataType::DE_INT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_UINT32 || type_ == DataType::DE_UINT16 || type_ == DataType::DE_UINT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_INT16 || type_ == DataType::DE_INT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_UINT16 || type_ == DataType::DE_UINT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_INT8; +} + +template <> +inline bool DataType::IsLooselyCompatible() const { + return type_ == DataType::DE_UINT8; +} +} // namespace dataset +} // namespace mindspore +#endif // DATASET_CORE_DATA_TYPE_H_ diff --git a/mindspore/ccsrc/dataset/core/global_context.cc b/mindspore/ccsrc/dataset/core/global_context.cc new file mode 100644 index 00000000000..7e361a1f2ca --- /dev/null +++ b/mindspore/ccsrc/dataset/core/global_context.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/global_context.h" + +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/tensor.h" +#include "dataset/util/allocator.h" +#include "dataset/util/circular_pool.h" +#include "dataset/util/system_pool.h" + +namespace mindspore { +namespace dataset { +// Global static pointer for the singleton GlobalContext +std::unique_ptr GlobalContext::global_context_ = nullptr; +std::once_flag GlobalContext::init_instance_flag_; + +constexpr int GlobalContext::kArenaSize; +constexpr int GlobalContext::kMaxSize; +constexpr bool GlobalContext::kInitArena; + +// Singleton initializer +GlobalContext *GlobalContext::Instance() { + // If the single global context is not created yet, then create it. Otherwise the + // existing one is returned. + std::call_once(init_instance_flag_, []() { + global_context_.reset(new GlobalContext()); + Status rc = global_context_->Init(); + if (rc.IsError()) { + std::terminate(); + } + }); + return global_context_.get(); +} + +Status GlobalContext::Init() { + config_manager_ = std::make_shared(); + mem_pool_ = std::make_shared(); + // For testing we can use Dummy pool instead + + // Create some tensor allocators for the different types and hook them into the pool. + tensor_allocator_ = mindspore::make_unique>(mem_pool_); + cv_tensor_allocator_ = mindspore::make_unique>(mem_pool_); + int_allocator_ = mindspore::make_unique(mem_pool_); + return Status::OK(); +} + +// A print method typically used for debugging +void GlobalContext::Print(std::ostream &out) const { + out << "GlobalContext contains the following default config: " << *config_manager_ << "\n"; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/global_context.h b/mindspore/ccsrc/dataset/core/global_context.h new file mode 100644 index 00000000000..ee0cbfbbe0e --- /dev/null +++ b/mindspore/ccsrc/dataset/core/global_context.h @@ -0,0 +1,108 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_GLOBAL_CONTEXT_H_ +#define DATASET_CORE_GLOBAL_CONTEXT_H_ + +#include +#include + +#include "dataset/core/constants.h" +#include "dataset/util/allocator.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// forward declare +class MemoryPool; +class ConfigManager; +class Tensor; +class CVTensor; + +using TensorAlloc = Allocator; // An allocator for Tensors +using CVTensorAlloc = Allocator; // An allocator CVTensors +using IntAlloc = Allocator; + +class GlobalContext { + // some consts for pool config + static constexpr int kArenaSize = 128; + static constexpr int kMaxSize = -1; + static constexpr bool kInitArena = true; + + public: + // Singleton pattern. This method either: + // - creates the single version of the GlobalContext for the first time and returns it + // OR + // - returns the already existing single instance of the GlobalContext + // @return the single global context + static GlobalContext *Instance(); + + // Destructor + ~GlobalContext() = default; + + // A print method typically used for debugging + // @param out - The output stream to write output to + void Print(std::ostream &out) const; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param g_c - reference to the GlobalContext to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const GlobalContext &g_c) { + g_c.Print(out); + return out; + } + + // Getter method + // @return the client config as raw const pointer + static std::shared_ptr config_manager() { return Instance()->config_manager_; } + + // Getter method + // @return the mem pool + std::shared_ptr mem_pool() const { return mem_pool_; } + + // Getter method + // @return the tensor allocator as raw pointer + const TensorAlloc *tensor_allocator() const { return tensor_allocator_.get(); } + + // Getter method + // @return the CVTensor allocator as raw pointer + const CVTensorAlloc *cv_tensor_allocator() const { return cv_tensor_allocator_.get(); } + + // Getter method + // @return the integer allocator as raw pointer + const IntAlloc *int_allocator() const { return int_allocator_.get(); } + + private: + // Constructor. + // @note Singleton. Instantiation flows through instance() + // @return This is a constructor. + GlobalContext() = default; + + Status Init(); + + static std::once_flag init_instance_flag_; + static std::unique_ptr global_context_; // The instance of the singleton (global) + std::shared_ptr mem_pool_; // A global memory pool + std::shared_ptr config_manager_; // The configs + std::unique_ptr tensor_allocator_; // An allocator for Tensors + std::unique_ptr cv_tensor_allocator_; // An allocator for CV Tensors + std::unique_ptr int_allocator_; // An allocator for ints +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_CORE_GLOBAL_CONTEXT_H_ diff --git a/mindspore/ccsrc/dataset/core/pybind_support.h b/mindspore/ccsrc/dataset/core/pybind_support.h new file mode 100644 index 00000000000..01c39987c15 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/pybind_support.h @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDDATA_PYBINDSUPPORT_H +#define MINDDATA_PYBINDSUPPORT_H + +#include + +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" +#include "Eigen/Core" + +namespace py = pybind11; +using float16 = Eigen::half; + +namespace pybind11 { +namespace detail { +// Similar to enums in `pybind11/numpy.h`. Determined by doing: +// python3 -c 'import numpy as np; print(np.dtype(np.float16).num)' +constexpr int kNpyFloat16 = 23; + +template +struct npy_scalar_caster { + PYBIND11_TYPE_CASTER(T, _("PleaseOverride")); + using Array = array_t; + + bool load(handle src, bool convert) { + // Taken from Eigen casters. Permits either scalar dtype or scalar array. + handle type = dtype::of().attr("type"); // Could make more efficient. + if (!convert && !isinstance(src) && !isinstance(src, type)) return false; + + Array tmp = Array::ensure(src); + if (tmp && tmp.size() == 1 && tmp.ndim() == 0) { + this->value = *tmp.data(); + return true; + } + + return false; + } + + static handle cast(T src, return_value_policy, handle) { + Array tmp({1}); + tmp.mutable_at(0) = src; + tmp.resize({}); + + // You could also just return the array if you want a scalar array. + object scalar = tmp[tuple()]; + return scalar.release(); + } +}; + +template <> +struct npy_format_descriptor { + static constexpr auto name = "float16"; + static pybind11::dtype dtype() { + handle ptr = npy_api::get().PyArray_DescrFromType_(kNpyFloat16); + return reinterpret_borrow(ptr); + } + virtual ~npy_format_descriptor() {} + + static std::string format() { + // following: https://docs.python.org/3/library/struct.html#format-characters + return "e"; + } +}; + +template <> +struct type_caster : public npy_scalar_caster { + static constexpr auto name = "float16"; +}; +} // namespace detail +} // namespace pybind11 + +#endif // MINDDATA_PYBINDSUPPORT_H diff --git a/mindspore/ccsrc/dataset/core/tensor.cc b/mindspore/ccsrc/dataset/core/tensor.cc new file mode 100644 index 00000000000..6aa34fa3422 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/tensor.cc @@ -0,0 +1,644 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/tensor.h" + +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "dataset/core/constants.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/global_context.h" +#include "dataset/core/pybind_support.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/util/make_unique.h" + +namespace py = pybind11; +namespace mindspore { +namespace dataset { +// Helper macros for printing tensor elements +#define CASE_PRINT(de_type, native_type) \ + case de_type: { \ + native_type o; \ + rc = GetItemAt(&o, index); \ + out << o; \ + break; \ + } + +#define CASE_PRINT_HEX(de_type, native_type) \ + case de_type: { \ + native_type o; \ + rc = GetItemAt(&o, index); \ + out << std::hex << std::setw(2) << std::setfill('0') << o << std::dec << std::setfill(' '); \ + break; \ + } + +Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) { + // grab the mem pool from global context and create the allocator for char data area + std::shared_ptr global_pool = GlobalContext::Instance()->mem_pool(); + data_allocator_ = mindspore::make_unique>(global_pool); +} + +Tensor::Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data) : Tensor(shape, type) { + // If the data pointer was given, then we can also populate the tensor with data + if (data != nullptr) { + // Given the shape/type of this tensor, compute the data size and copy in the input bytes. + int64_t byte_size = this->SizeInBytes(); + static_cast(this->StartAddr()); // Allocates data_ inside itself + if (data_ != nullptr) { + int ret_code = memcpy_s(data_, byte_size, data, byte_size); + if (ret_code != 0) { + MS_LOG(ERROR) << "Failed to copy data into Tensor!"; + } + } else { + MS_LOG(ERROR) << "Failed to create memory for Tensor!"; + } + } +} + +Tensor::Tensor(Tensor &&other) noexcept + : shape_(other.shape()), + type_(other.type()), + data_(other.StartAddr()), + data_allocator_(std::move(other.data_allocator_)) { + other.Invalidate(); +} + +Tensor &Tensor::operator=(Tensor &&other) noexcept { + if (&other != this) { + shape_ = other.shape(); + type_ = other.type(); + data_ = other.StartAddr(); + data_allocator_ = std::move(other.data_allocator_); + other.Invalidate(); + } + return *this; +} + +Status Tensor::CreateTensor(std::shared_ptr *ptr, TensorImpl tensor_impl, const TensorShape &shape, + DataType type, const unsigned char *data) { + if (!shape.known()) { + RETURN_STATUS_UNEXPECTED("Invalid shape."); + } + if (type == DataType::DE_UNKNOWN) { + RETURN_STATUS_UNEXPECTED("Invalid data type."); + } + + switch (tensor_impl) { + case TensorImpl::kFlexible: { + // The flex tensor is really just the base class tensor implementation + const TensorAlloc *alloc = GlobalContext::Instance()->tensor_allocator(); + *ptr = std::allocate_shared(*alloc, shape, type, data); + break; + } + case TensorImpl::kCv: { + const CVTensorAlloc *alloc = GlobalContext::Instance()->cv_tensor_allocator(); + *ptr = std::allocate_shared(*alloc, shape, type, data); + break; + } + default: { + std::string err_msg("Invalid tensor implementation type."); + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + return Status::OK(); // returns base-class shared_ptr +} + +Status Tensor::CreateTensor(std::shared_ptr *ptr, py::array arr) { + const TensorAlloc *alloc = GlobalContext::Instance()->tensor_allocator(); + *ptr = std::allocate_shared(*alloc, TensorShape({}), DataType(DataType::DE_UNKNOWN)); + + std::vector shape; + for (dsize_t i = 0; i < arr.ndim(); i++) { + shape.push_back(static_cast(arr.shape()[i])); + } + + (*ptr)->shape_ = TensorShape(shape); + (*ptr)->type_ = DataType::FromNpArray(arr); + if (!(*ptr)->shape_.known()) RETURN_STATUS_UNEXPECTED("Invalid shape."); + + if ((*ptr)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type."); + + std::shared_ptr global_pool = GlobalContext::Instance()->mem_pool(); + (*ptr)->data_allocator_ = mindspore::make_unique>(global_pool); + static_cast((*ptr)->StartAddr()); + int64_t byte_size = (*ptr)->SizeInBytes(); + unsigned char *data = static_cast(arr.request().ptr); + if ((*ptr)->data_ == nullptr) { + RETURN_STATUS_UNEXPECTED("Failed to create memory for Tensor."); + } + + std::vector strides; + for (dsize_t i = 0; i < arr.ndim(); i++) { + strides.push_back(static_cast(arr.strides()[i])); + } + + // check if strides are contiguous + bool is_strided = false; + dsize_t count = (*ptr)->shape_.NumOfElements(); + for (size_t i = 0; i < shape.size(); i++) { + count /= shape[i]; + if (strides[i] != (*ptr)->type_.SizeInBytes() * count) { + is_strided = true; + break; + } + } + + if (is_strided) { + RETURN_IF_NOT_OK(CopyStridedArray((*ptr)->data_, data, shape, strides, (*ptr)->type_.SizeInBytes())); + } else { + int ret_code = memcpy_s((*ptr)->data_, byte_size, data, byte_size); + if (ret_code != 0) { + RETURN_STATUS_UNEXPECTED("Failed to copy data into Tensor."); + } + } + + return Status::OK(); // returns base-class shared_ptr +} + +// Memcpy the given strided array's used part to consecutive memory +// Consider a 3-d array +// A[(i * shape[1] + j) * shape[2] + k] = B[i][j][k] = C[i * strides[0] + j * strides[1] + k * strides[2]] +// Here we convert array C to array A, by memcpy index by index (Note that not all elements in C is copied) +Status Tensor::CopyStridedArray(unsigned char *dst, unsigned char *src, std::vector shape, + std::vector strides, uint8_t type_size) { + dsize_t size = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + for (dsize_t i = 0; i < size; ++i) { + dsize_t offset = 0; + dsize_t count = i; + for (size_t j = 0; j < shape.size(); ++j) { + // convert 1d array's index to 3d array's index (A -> B) + dsize_t idx = count % shape[shape.size() - 1 - j]; + count /= shape[shape.size() - 1 - j]; + // calculate the raw data offset based on strides (B -> C) + offset += idx * strides[shape.size() - 1 - j]; + // once count = 0, the following idxes are all zero, skip them + if (count == 0) break; + } + // strides already consider byte size of the data type, but dst doesn't. + // dst[i] = dst + i * type_size = src + offset + int ret_code = memcpy_s(dst + i * type_size, type_size, src + offset, type_size); + if (ret_code != 0) { + RETURN_STATUS_UNEXPECTED("Failed to copy data into Tensor."); + } + } + return Status::OK(); +} + +// Name: Destructor +// Description: Destructor +Tensor::~Tensor() { + if (data_ != nullptr) { + if (data_allocator_ != nullptr) { + data_allocator_->deallocate(data_); + data_ = nullptr; + } else { + // If we didn't have an allocator, but data_ is not null then it must + // be a stand-alone tensor that used malloc directly. + free(data_); + data_ = nullptr; + } + } +} + +bool Tensor::operator==(const Tensor &rhs) const { + // 1. different shape 2. different type 3. one data_ is nullptr and the other is not + if (shape_ != rhs.shape() || type_ != rhs.type_ || (data_ == nullptr && rhs.data_ != nullptr) || + (data_ != nullptr && rhs.data_ == nullptr)) { + return false; + } + if (data_ == nullptr && rhs.data_ == nullptr) { + return true; + } + // use mem compare to compare the two data, size are already verified + return memcmp(data_, rhs.data_, SizeInBytes()) == 0; +} + +// Name: PrintItemAt() +// Description: A function that print the value as specified by its index +void Tensor::PrintItemAt(const std::vector &index, std::ostream &out) const { + Status rc; + DS_ASSERT(data_); + + switch (type_.value()) { + CASE_PRINT_HEX(DataType::DE_BOOL, uint8_t); + + CASE_PRINT_HEX(DataType::DE_INT8, int8_t); + + CASE_PRINT_HEX(DataType::DE_UINT8, uint8_t); + + CASE_PRINT(DataType::DE_INT16, int16_t); + + CASE_PRINT(DataType::DE_UINT16, uint16_t); + + CASE_PRINT(DataType::DE_INT32, int32_t); + + CASE_PRINT(DataType::DE_UINT32, uint32_t); + + CASE_PRINT(DataType::DE_INT64, int64_t); + + CASE_PRINT(DataType::DE_UINT64, uint64_t); + + CASE_PRINT(DataType::DE_FLOAT16, float16); + + CASE_PRINT(DataType::DE_FLOAT32, float); + + CASE_PRINT(DataType::DE_FLOAT64, double); + + default: { + out << "?"; + break; + } + } + if (rc.IsError()) { + out << rc.ToString(); + } +} + +// Name: PrintRecursive() +// Description: A function that prints Tensor recursively, first called by print +void Tensor::PrintRecursive(std::ostream &out, int32_t cur_dim, const std::vector &cur_index) const { + if (cur_index.size() == shape_.Rank()) { + PrintItemAt(cur_index, out); + } else { + out << "["; + for (dsize_t i = 0; i < shape_[cur_dim]; i++) { + std::vector new_index = cur_index; + new_index.push_back(i); + PrintRecursive(out, cur_dim + 1, new_index); + if (i < shape_[cur_dim] - 1) { + out << ","; + } + } + out << "]"; + } +} + +// Name: Print() +// Description: A function that prints info about the tensor +void Tensor::Print(std::ostream &out) const { + out << "Tensor (shape: "; + out << shape_; + out << ", Type: " << type_ << ")\n"; + if (data_) { + PrintRecursive(out, 0, std::vector{}); + } else { + out << "[Data area is null]"; + } +} + +// Name: ToFlatIndex() +// Description: convert a vector style index to number, used to access memory internal use only +Status Tensor::ToFlatIndex(const std::vector &index, dsize_t *flat_index) const { + if (!shape_.IsValidIndex(index)) { + std::string err = "Not a valid index"; + RETURN_STATUS_UNEXPECTED(err); + } + *flat_index = 0; + for (size_t k = 0; k < index.size(); k++) { + dsize_t product = 1; + for (size_t l = k + 1; l < index.size(); l++) { + product *= shape_[l]; + } + *flat_index += index[k] * product; + } + return Status::OK(); +} + +const unsigned char *Tensor::StartAddr() const { + // This version cannot modify anything. data_ could possibly be null. + return data_; +} + +unsigned char *Tensor::StartAddr() { + if (!shape_.known() || type_ == DataType::DE_UNKNOWN) { + return nullptr; + } + // If the data area is already created, return the pointer to it + if (data_ != nullptr) { + return data_; + } else { + // If the data area is not created, then identify the memory size based + // on the shape and type and allocate it. + if (data_allocator_ != nullptr) { + data_ = data_allocator_->allocate(this->SizeInBytes()); + } else { + data_ = static_cast(malloc(this->SizeInBytes())); + if (data_ == nullptr) { + return nullptr; + } + } + return data_; + } +} + +Status Tensor::Reshape(const TensorShape &shape) { + if (shape.NumOfElements() == shape_.NumOfElements()) { + shape_ = shape; + return Status::OK(); + } else { + std::string err = "Cannot reshape, Number of elements do not match"; + RETURN_STATUS_UNEXPECTED(err); + } +} + +void Tensor::Invalidate() { + shape_ = TensorShape::CreateUnknownRankShape(); + type_ = DataType(DataType::DE_UNKNOWN); + data_ = nullptr; + data_allocator_ = nullptr; +} + +template +Status Tensor::GetItemPtr(T **ptr, const std::vector &index) const { + if (type_.IsCompatible()) { + if (data_ == nullptr) { + std::string err = "Data is not allocated yet"; + RETURN_STATUS_UNEXPECTED(err); + } + dsize_t flat_idx; + RETURN_IF_NOT_OK(ToFlatIndex(index, &flat_idx)); + *ptr = reinterpret_cast(data_ + flat_idx * type_.SizeInBytes()); + return Status::OK(); + } else { + std::string err = "data type not compatible"; + RETURN_STATUS_UNEXPECTED(err); + } +} + +Status Tensor::StartAddrOfIndex(std::vector ind, uchar **start_addr_of_index, TensorShape *remaining) { + dsize_t flat_ind; + std::vector t_shape = shape().AsVector(); + std::vector r(t_shape.begin() + ind.size(), t_shape.end()); + *remaining = TensorShape(r); + ind.resize(this->Rank(), 0); // same as -> while (ind.size() < this->Rank()) ind.push_back(0); + RETURN_IF_NOT_OK(ToFlatIndex(ind, &flat_ind)); + // check if StartAddr() returns null, we should flag this as an error, this sanity check will only + // be true is the tensor failed to allocate memory. + if (StartAddr() == nullptr) { + RETURN_STATUS_UNEXPECTED("Invalid StartAddr in Tensor, got nullptr"); + } + *start_addr_of_index = StartAddr() + flat_ind * this->type().SizeInBytes(); + return Status::OK(); +} + +Status Tensor::InsertTensor(const std::vector &ind, const std::shared_ptr &tensor) { + std::string err_msg; + err_msg += (!this->shape().known() || !tensor->shape().known()) ? "[Tensor] unknown shape\n" : ""; + err_msg += (ind.size() + tensor->Rank() != this->Rank()) ? "[Tensor] incorrect index\n" : ""; + err_msg += tensor->type().SizeInBytes() != this->type().SizeInBytes() ? "[Tensor] incorrect datatype\n" : ""; + uchar *start_addr_of_ind = nullptr; + TensorShape remaining_shape({-1}); + err_msg += (!StartAddrOfIndex(ind, &start_addr_of_ind, &remaining_shape).IsOk()) ? "[Tensor] incorrect index\n" : ""; + err_msg += !(remaining_shape == tensor->shape()) ? "[Tensor] memory error\n" : ""; + if (!err_msg.empty()) { + MS_LOG(INFO) << "Insert tensor message: " << err_msg; + RETURN_STATUS_UNEXPECTED(err_msg); + } else { + if (start_addr_of_ind != nullptr) { + int ret_code = memcpy_s(start_addr_of_ind, tensor->SizeInBytes(), tensor->StartAddr(), tensor->SizeInBytes()); + if (ret_code == 0) { + return Status::OK(); + } else { + err_msg += "[Tensor] error in memcpy_s when inserting tensor\n"; + MS_LOG(INFO) << "Tensor message: " << err_msg; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } else { + RETURN_STATUS_UNEXPECTED("Failed to create memory for Tensor."); + } + } +} + +Status Tensor::ExpandDim(const dsize_t &axis) { + if (axis > Rank()) { + std::string err = "Axis is out of bound"; + RETURN_STATUS_UNEXPECTED(err); + } + if (axis == Rank()) { + shape_ = shape_.AppendDim(1); + } else { + shape_ = shape_.InsertDim(axis, 1); + } + return Status::OK(); +} + +std::vector Tensor::Strides() { + std::vector strides(Rank()); + dsize_t count = shape_.NumOfElements(); + for (dsize_t i = 0; i < Rank(); i++) { + count /= shape_[i]; + strides[i] = type_.SizeInBytes() * count; + } + return strides; +} + +Status Tensor::GetBufferInfo(Tensor &t, py::buffer_info *out) { + std::string format_desc = t.type().GetPybindFormat(); + if (format_desc.empty()) { + RETURN_STATUS_UNEXPECTED("Cannot convert DE type tp pybind format"); + } + *out = py::buffer_info(t.StartAddr(), /* Pointer to buffer */ + t.type().SizeInBytes(), /* Size of one scalar */ + format_desc, /* Python struct-style format descriptor */ + t.Rank(), /* Number of dimensions */ + t.shape().AsVector(), /* Buffer dimensions */ + t.Strides()); + return Status::OK(); +} + +template +Status Tensor::GetItemAt(T *o, const std::vector &index) const { + if (data_ == nullptr) { + RETURN_STATUS_UNEXPECTED("Data is not allocated yet"); + } + if (!type_.IsLooselyCompatible()) { + std::string err = "Template type and Tensor type are not compatible"; + RETURN_STATUS_UNEXPECTED(err); + } + if (type_.IsUnsignedInt()) { + RETURN_IF_NOT_OK(GetUnsignedIntAt(o, index)); + } else if (type_.IsSignedInt()) { + RETURN_IF_NOT_OK(GetSignedIntAt(o, index)); + } else if (type_.IsFloat()) { + RETURN_IF_NOT_OK(GetFloatAt(o, index)); + } else if (type_.IsBool()) { + bool *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + } else { + std::string err = "Tensor Type is unknown"; + RETURN_STATUS_UNEXPECTED(err); + } + return Status::OK(); +} + +// return data as numpy, should return status +Status Tensor::GetDataAsNumpy(py::array *data) { + if (type_ == DataType::DE_BOOL) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_INT8) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_INT16) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_INT32) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_INT64) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_UINT8) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_UINT16) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_UINT32) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_UINT64) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_FLOAT16) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_FLOAT32) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else if (type_ == DataType::DE_FLOAT64) { + *data = py::array_t(shape_.AsVector(), reinterpret_cast(data_)); + } else { + RETURN_STATUS_UNEXPECTED("Got unexpected type when returning numpy"); + } + return Status::OK(); +} + +void Tensor::Squeeze() { shape_ = shape_.Squeeze(); } + +template +Status Tensor::GetUnsignedIntAt(T *o, const std::vector &index) const { + if (data_ == nullptr) { + RETURN_STATUS_UNEXPECTED("Data is not allocated yet"); + } + if (!type_.IsLooselyCompatible()) { + std::string err = "Template type and Tensor type are not compatible"; + RETURN_STATUS_UNEXPECTED(err); + } + switch (type_.value()) { + case DataType::DE_UINT8: { + uint8_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_UINT16: { + uint16_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_UINT32: { + uint32_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_UINT64: { + uint64_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + default: + std::string err = "Tensor Type is not an unsigned Integer"; + RETURN_STATUS_UNEXPECTED(err); + } + return Status::OK(); +} + +template +Status Tensor::GetSignedIntAt(T *o, const std::vector &index) const { + if (data_ == nullptr) { + RETURN_STATUS_UNEXPECTED("Data is not allocated yet"); + } + if (!type_.IsLooselyCompatible()) { + std::string err = "Template type and Tensor type are not compatible"; + RETURN_STATUS_UNEXPECTED(err); + } + switch (type_.value()) { + case DataType::DE_INT8: { + int8_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_INT16: { + int16_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_INT32: { + int32_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_INT64: { + int64_t *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + default: + std::string err = "Tensor Type is not a signed Integer"; + RETURN_STATUS_UNEXPECTED(err); + } + return Status::OK(); +} + +template +Status Tensor::GetFloatAt(T *o, const std::vector &index) const { + if (data_ == nullptr) { + RETURN_STATUS_UNEXPECTED("Data is not allocated yet"); + } + if (!type_.IsLooselyCompatible()) { + std::string err = "Template type and Tensor type are not compatible"; + RETURN_STATUS_UNEXPECTED(err); + } + switch (type_.value()) { + case DataType::DE_FLOAT16: { + float16 *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_FLOAT32: { + float *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + case DataType::DE_FLOAT64: { + double *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *o = static_cast(*ptr); + break; + } + default: + std::string err = "Tensor Type is not a float/double"; + RETURN_STATUS_UNEXPECTED(err); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/tensor.h b/mindspore/ccsrc/dataset/core/tensor.h new file mode 100644 index 00000000000..2017c2dfab6 --- /dev/null +++ b/mindspore/ccsrc/dataset/core/tensor.h @@ -0,0 +1,404 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_TENSOR_H_ +#define DATASET_CORE_TENSOR_H_ + +#include +#include +#include +#include +#include "./securec.h" +#include "utils/log_adapter.h" +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "dataset/core/constants.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/util/allocator.h" +#include "dataset/util/de_error.h" +#include "dataset/util/status.h" + +namespace py = pybind11; +namespace mindspore { +namespace dataset { +class Tensor; + +using CharAllocPtr = std::unique_ptr>; +using TensorAllocPtr = std::shared_ptr>; // An allocator shared_ptr for Tensors +using TensorRow = std::vector>; // A row is a set of Tensor pointers +using TensorTable = std::vector; // The table of tensors is a vector of rows +using TensorQTable = std::deque; // A different flavour of tensor table, this one has queue functionality + +// Tensor base class which holds the data in an unsigned char* buffer. + +class Tensor { + public: + Tensor() = delete; + + // Create a new tensor, does not internally allocate storage. This constructor is protected, use CreateTensor. + // @note The shape and type information should be known and valid. + // @param shape TensorShape + // @param type DataType + Tensor(const TensorShape &shape, const DataType &type); + + // Create a new tensor, allocates storage and copies in data. This constructor is protected, use CreateTensor. + // @note The buffer should be valid and the shape and type information should be known and valid. + // @param shape TensorShape + // @param type DataType + // @param data unsigned char*, pointer to the data. + Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data); + + Tensor(const Tensor &other) = delete; + + Tensor &operator=(const Tensor &other) = delete; + + Tensor(Tensor &&other) noexcept; + + Tensor &operator=(Tensor &&other) noexcept; + + // A static factory method to create the given flavour of derived Tensor + // Returns the base class reference for the Tensor. + // @param ptr output argument to hold the created Tensor of given tensor_impl + // @param tensor_impl - which implementation of Tensor + // @param shape - shape of the tensor + // @param type - datatype of the tensor + // @param data - data to be copied to Tensor new allocation + // @return Status Code + static Status CreateTensor(std::shared_ptr *, TensorImpl tensor_impl, const TensorShape &shape, DataType type, + const unsigned char *data = nullptr); + + // A static factory method to create a Tensor from a given py::array. + // @param ptr output argument to hold the created Tensor + // @param arr py::array + // @return Status Code + static Status CreateTensor(std::shared_ptr *ptr, py::array arr); + + // Copy raw data of a array based on shape and strides to the destination pointer + // @param dst Pointer to the destination array where the content is to be copied + // @param src Pointer to the source of stided array to be copied + // @param shape - shape of the source array + // @param strides - strides of the source array + // @param type_size - number of bytes needed to store one array elment's type + // @return Status Code + static Status CopyStridedArray(unsigned char *dst, unsigned char *src, std::vector shape, + std::vector strides, uint8_t type_size); + + // Release the memory using the allocator + virtual ~Tensor(); + + // compare the tensor shape and data + bool operator==(const Tensor &rhs) const; + + bool operator!=(const Tensor &rhs) const { return !((*this) == rhs); } + + // Get item located at `index`, caller needs to provide the type. + // @tparam T + // @param index vector + // @return return the item specified at index + template + Status GetItemAt(T *o, const std::vector &index) const; + + template + Status GetUnsignedIntAt(T *o, const std::vector &index) const; + + template + Status GetSignedIntAt(T *o, const std::vector &index) const; + + template + Status GetFloatAt(T *o, const std::vector &index) const; + + // set item at location specified by index + // @tparam `T` + // @param index + // @param value of type `T` + template + Status SetItemAt(const std::vector &index, const T &value) { + static_cast(StartAddr()); + T *ptr = nullptr; + RETURN_IF_NOT_OK(GetItemPtr(&ptr, index)); + *ptr = value; + return Status::OK(); + } + + Status Zero() { + dsize_t size = SizeInBytes(); + int retCode = memset_sp(StartAddr(), size, 0, size); + if (retCode != 0) return Status(StatusCode::kUnexpectedError, "Failed to fill tensor with zeroes."); + return Status::OK(); + } + + // Fill all elements in the Tensor with the given value of type `T` + // @tparam T + // @param value + template + Status Fill(const T &value) { + static_cast(StartAddr()); + int64_t cellSize = type_.SizeInBytes(); + if ((data_ != nullptr) && type_.IsCompatible()) { + for (dsize_t i = 0; i < Size(); i++) { + int retCode = memcpy_s((data_ + i * cellSize), cellSize, &value, cellSize); + if (retCode != 0) { + return Status(StatusCode::kUnexpectedError, "Failed to fill tensor."); + } + } + return Status::OK(); + } else { + std::string err; + err += (data_ == nullptr) ? "data_ is nullptr \t" : ""; + err += type_.IsCompatible() ? "data type not compatible\t" : ""; + return Status(StatusCode::kUnexpectedError, err); + } + } + + // Getter function for shape + // @return + const TensorShape &shape() const { return shape_; } + + // Reshape the tensor. The given shape should have the same number of elements in the Tensor + // @param shape + virtual Status Reshape(const TensorShape &shape); + + // @return number of elements in this tensor + dsize_t Size() const { return shape().NumOfElements(); } + + // @return the number of bytes this tensor is needs + dsize_t SizeInBytes() const { return Size() * type_.SizeInBytes(); } + + // @return the rank of the tensor + dsize_t Rank() const { return shape().Rank(); } + + // Get the starting memory address as a constant for the data of the tensor. This potentially + // drives an allocation if the data area. + // @return const unsigned char* + const unsigned char *StartAddr() const; + + // Get the starting memory address for the data of the tensor. This potentially + // drives an allocation if the data area. + // @return unsigned char* + unsigned char *StartAddr(); + + // Getter of the type + // @return + DataType type() const { return type_; } + + // Provide stream operator for displaying it + // @param output stream + // @param so the Tensor object to be printed + // @return output stream + friend std::ostream &operator<<(std::ostream &out, const Tensor &so) { + so.Print(out); + return out; + } + + // Invalidate this Tensor by setting the type and shape to unknown and MData to null. + // Calling this method will make the Tensor and its data inaccessible, use it with caution. + void Invalidate(); + + // Copy input tensor into self at the location index. + // Index is a vector of axises which can be incomplete: + // Ex: shape <2,3>, inserting into index {0} will replace the first row. index {1,2} will replace the last cell. + // @param index + // @param input + // @return Status code + Status InsertTensor(const std::vector &index, const std::shared_ptr &input); + + // Find the address of the given index. Used in InsertTensor. + // Example: + // Tensor t= [[1,2],[3,4]] , StartAddrOfIndex({0}) -> &1 + // @param index incomplete index + // @param output: startAddrofIndex + // @param output: remaining + // @return Status code + Status StartAddrOfIndex(std::vector ind, uchar **start_addr_of_index, TensorShape *remaining); + + // Expand the shape of the Tensor with one extra dimension. + // For example, if the shape is <512,512,3>: + // *- ExpandDim(0) gives: <1,512,512,3> + // *- ExpandDim(1) gives: <512,1,512,3> + // *- ExpandDim(3) gives: <512,512,3,1> + // @param axis location of the dim + virtual Status ExpandDim(const dsize_t &axis); + + virtual void Squeeze(); + + // Calculates the strides of the Tensor + // Ex: Tensor of shape <4,2,2> and type DE_UINT8 (1 byte) + // The strides will be {6,2,1}. + // Ex: Tensor of shape <4,2,2> and type DE_UINT32 (4 byte) + // The strides will be {24,8,4}. + // @return vector of integers + std::vector Strides(); + + std::string ToString() { + std::stringstream ss; + this->Print(ss); + return ss.str(); + } + + // Constructs numpy array from input tensor + // @param data this data is the location of python data + // @return Status code + Status GetDataAsNumpy(py::array *data); + + static Status GetBufferInfo(Tensor &t, py::buffer_info *out); + + // TensorIterator is a linear iterator that can be used to iterate over the elements of the Tensor + // The order elements is as the memory layout (i.e., row-major) [[1,2,3],[4,5,6] --> 1,2,3,4,5,6 + // @tparam T type of values in the Tensor Iterator + template + class TensorIterator { + public: + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T *; + using reference = T &; + + explicit TensorIterator(uchar *ptr = nullptr) { ptr_ = reinterpret_cast(ptr); } + + TensorIterator(const TensorIterator &raw_iterator) = default; + + ~TensorIterator() = default; + + TensorIterator &operator=(const TensorIterator &rhs) = default; + + TensorIterator &operator=(T *rhs) { + ptr_ = rhs; + return *this; + } + + bool operator==(const TensorIterator &rhs) { return ptr_ == rhs.ptr_; } + + bool operator!=(const TensorIterator &rhs) { return !(*this == rhs); } + + operator bool() const { return ptr_ != nullptr; } + + T &operator*() { return *ptr_; } + + const T &operator*() const { return *ptr_; } + + T *operator->() { return ptr_; } + + TensorIterator &operator+=(const ptrdiff_t &inc) { + ptr_ += inc; + return *this; + } + + TensorIterator &operator-=(const ptrdiff_t &inc) { + ptr_ -= inc; + return *this; + } + + TensorIterator &operator++() { + ++ptr_; + return *this; + } + + TensorIterator &operator--() { + --ptr_; + return *this; + } + + TensorIterator operator++(int) { + auto temp(*this); + ++ptr_; + return temp; + } + + TensorIterator operator--(int) { + auto temp(*this); + --ptr_; + return temp; + } + + TensorIterator operator+(const ptrdiff_t &inc) { + auto oldPtr = ptr_; + ptr_ += inc; + auto temp(*this); + ptr_ = oldPtr; + return temp; + } + + TensorIterator operator-(const ptrdiff_t &inc) { + auto oldPtr = ptr_; + ptr_ -= inc; + auto temp(*this); + ptr_ = oldPtr; + return temp; + } + + protected: + T *ptr_; + }; + + // Return a TensorIterator that points to the start of the Tensor. + // It's the user responsibility to use the correct type that matches the Tensor type + // @tparam T The type of values in the Tensor + // @return TensorIterator + template + TensorIterator begin() { + return TensorIterator(data_); + } + + // Return a linear iterator that points to the place after the last element of the Tensor. + // @tparam T The type of values in the Tensor + // @return TensorIterator + template + TensorIterator end() { + return TensorIterator(data_ + SizeInBytes()); + } + + protected: + // Returns the location of the item assuming row major memory layout. + // @param index + // @return + Status ToFlatIndex(const std::vector &index, dsize_t *flat_index) const; + + // A function that prints Tensor recursively, first called by print + // @param out + // @param cur_dim + // @param cur_index + void PrintRecursive(std::ostream &out, int32_t cur_dim, const std::vector &cur_index) const; + + // A function that prints info about the tensor + // @param out output stream + void Print(std::ostream &out) const; + + // A function that print the value as specified by its index + // @param index vector representing the index + // @param out + void PrintItemAt(const std::vector &index, std::ostream &out) const; + + // Get pointer to item located at `index`, caller needs to provide the type. + // @tparam T + // @param index vector + // @return return a pointer to the item specified at index of type `T` + template + Status GetItemPtr(T **, const std::vector &index) const; + + // all access to shape_ should be via shape + TensorShape shape_; + // data type of tensor + DataType type_; + // pointer to the start of the physical data + unsigned char *data_; + // An allocator for data_ + CharAllocPtr data_allocator_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_CORE_TENSOR_H_ diff --git a/mindspore/ccsrc/dataset/core/tensor_shape.cc b/mindspore/ccsrc/dataset/core/tensor_shape.cc new file mode 100644 index 00000000000..24520dc381b --- /dev/null +++ b/mindspore/ccsrc/dataset/core/tensor_shape.cc @@ -0,0 +1,215 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define MAX_INTEGER_DTYPE 9223372036854775807 + +#include "dataset/core/tensor_shape.h" + +#include + +#include "common/utils.h" +#include "utils/log_adapter.h" +#include "dataset/core/constants.h" +#include "dataset/util/de_error.h" + +namespace mindspore { +namespace dataset { +constexpr dsize_t TensorShape::kDimUnknown; + +bool multi_ok(dsize_t x, dsize_t y) { + dsize_t p = x * y; + if (x == 0) { + return true; + } + return p / x == y; +} + +dsize_t TensorShape::NumOfElements() const { + if (!known()) { + return 0; + } + dsize_t num = 1; + for (auto i : raw_shape_) { + if (multi_ok(num, i)) { + num *= i; + } else { + // dsize_t can wrap since it is signed int, we double check here + MS_LOG(ERROR) << "Tensor shape larger than maximum allowed value!"; + } + } + return num; +} + +void TensorShape::Print(std::ostream &out) const { + if (!known() && raw_shape_.empty()) { + out << ""; + } else { + out << "<"; + for (auto i = 0; i < this->Rank(); i++) { + if (raw_shape_[i] == kDimUnknown) { + out << "*"; + } else { + out << raw_shape_[i]; + } + if (i != this->Rank() - 1) { + out << ","; + } + } + out << ">"; + } +} + +TensorShape::TensorShape(const std::initializer_list &list) + : raw_shape_(*GlobalContext::Instance()->int_allocator()) { + AddListToShape(list); +} + +TensorShape::TensorShape(const std::vector &list) : raw_shape_(*GlobalContext::Instance()->int_allocator()) { + AddListToShape(list); +} + +TensorShape::TensorShape(const TensorShape &shape) : raw_shape_(*GlobalContext::Instance()->int_allocator()) { + AddListToShape(shape.AsVector()); + known_ = shape.known_; // override with the input shape in case of unknown-rank tensor shape. +} + +TensorShape::TensorShape(py::list l) : raw_shape_(*GlobalContext::Instance()->int_allocator()) { + std::vector list_c; + for (auto i : l) { + list_c.push_back(i.cast()); + } + AddListToShape(list_c); +} + +TensorShape TensorShape::CreateUnknownRankShape() { + TensorShape s({}); + s.known_ = false; + return s; +} + +TensorShape TensorShape::InsertDim(dsize_t axis, dsize_t dim) const { + std::vector tmp = AsVector(); + (void)tmp.insert(tmp.begin() + axis, dim); + return TensorShape(tmp); +} + +TensorShape::TensorShape(cv::MatSize cv_size, uint32_t type) : raw_shape_(*GlobalContext::Instance()->int_allocator()) { + for (int i = 0; i < cv_size.dims(); i++) { + raw_shape_.push_back(cv_size[i]); + } + auto channels = static_cast(1 + (type >> static_cast(CV_CN_SHIFT))); + if (channels != 1) { + raw_shape_.push_back(channels); + } + known_ = true; +} + +std::vector TensorShape::AsVector() const { + return std::vector(raw_shape_.begin(), raw_shape_.end()); +} + +bool TensorShape::IsValidIndex(const std::vector &index) const { + dsize_t s_rank = Rank(); + if (index.size() != s_rank) { + return false; + } + for (dsize_t i = 0; i < s_rank; i++) { + if (index[i] < 0 || raw_shape_[i] <= index[i]) { + return false; + } + } + return true; +} + +template +void TensorShape::AddListToShape(const T &list) { + known_ = true; + dsize_t num = 1; + dsize_t size = 0; + for (const auto &itr : list) { + if (itr > 0) { + if (num > std::numeric_limits::max() / itr) { + MS_LOG(ERROR) << "Invalid shape data, overflow occurred!"; + known_ = false; + raw_shape_.clear(); + return; + } + num *= itr; + } + if (itr < 0) { + known_ = false; + } + if (itr > kDeMaxDim) { + std::stringstream ss; + ss << "Invalid shape data, dim (" << size << ") is larger than the maximum dim size(" << kDeMaxDim << ")!"; + MS_LOG(ERROR) << ss.str().c_str(); + known_ = false; + raw_shape_.clear(); + return; + } + raw_shape_.push_back(itr); + size++; + } + if (size > kDeMaxRank) { + std::stringstream ss; + ss << "Invalid shape data, rank (" << size << ") is larger than the maximum rank size(" << kDeMaxRank << ")."; + MS_LOG(ERROR) << ss.str().c_str(); + known_ = false; + raw_shape_.clear(); + return; + } +} + +TensorShape TensorShape::CreateUnknownShapeWithRank(dsize_t rank) { + TensorShape s({}); + for (dsize_t i = 0; i < rank; i++) { + s.raw_shape_.push_back(kDimUnknown); + } + s.known_ = false; + return s; +} + +TensorShape TensorShape::PrependDim(dsize_t dim) const { + if (Size() == 0) { + return TensorShape({dim}); + } + return InsertDim(0, dim); +} + +TensorShape TensorShape::AppendDim(dsize_t dim) const { + auto vec = AsVector(); + vec.push_back(dim); + return TensorShape(vec); +} + +py::list TensorShape::AsPyList() { + py::list list; + for (auto i : raw_shape_) { + list.append(i); + } + return list; +} + +TensorShape TensorShape::Squeeze() const { + std::vector new_shape; + for (auto s : AsVector()) { + if (s != 1) { + new_shape.push_back(s); + } + } + return TensorShape(new_shape); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/core/tensor_shape.h b/mindspore/ccsrc/dataset/core/tensor_shape.h new file mode 100644 index 00000000000..f908a00eccc --- /dev/null +++ b/mindspore/ccsrc/dataset/core/tensor_shape.h @@ -0,0 +1,173 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_CORE_TENSOR_SHAPE_H_ +#define DATASET_CORE_TENSOR_SHAPE_H_ + +#include +#include +#include +#include +#include + +#include + +#include "pybind11/pybind11.h" + +#include "dataset/core/constants.h" +#include "dataset/core/global_context.h" +#include "dataset/util/allocator.h" + +namespace py = pybind11; +namespace mindspore { +namespace dataset { +// Class that represents a shape of a Tensor. A shape can be: +// -# Known shape (mKnown = true) +// -# Scalar --> empty vector --> <> +// -# n-Dim --> not empty vector --> where di is >= 0\n +// Example: <1,2>, <1>, <1,13,10,11,1> +// -# Unknown shape (mKnown = false) +// -# Rank is unknown --> empty vector --> <> +// -# one or more dim is unknown --> not empty vector --> where di is unknown\n +// Example: <3,?> (the 1st dim is unknown)\n +// <2,?,?,?> (all dims but the 0th dim are unknown) +// TensorShape supports any dim > 0 and < 2^31-1 +class TensorShape { + public: + static constexpr dsize_t kDimUnknown = -1; // constant for an unknown dimension + + // Force the compiler to not create a no-arg constructor + TensorShape() = delete; + + // Create a Shape from an initialization list (e.g., TensorShape s = {2,2}). + // If one of the dims is set to DIM_UNKNOWN, the shape will flagged as unKnown + // @param list + explicit TensorShape(const std::initializer_list &list); + + // Create a Shape from a vector (e.g., TensorShape s = std::vector({2,2}) ). + // If one of the dims is set to DIM_UNKNOWN, the shape will flagged as unKnown + // @param list + explicit TensorShape(const std::vector &list); + + // Copy constructor + // @param shape + TensorShape(const TensorShape &shape); + + ~TensorShape() = default; + + // Create a scalar Shape (i.e., empty shape with mKnown = true) + // @return TensorShape + static TensorShape CreateScalar() { return TensorShape({}); } + + // Create a shape with an unknown rank. + // @return TensorShape + static TensorShape CreateUnknownRankShape(); + + // Create a shape with a known rank . + // @return TensorShape + static TensorShape CreateUnknownShapeWithRank(dsize_t rank); + + // Insert a new dim into a copy of the current shape. + // @param dim to be added + // @param axis the index where dim should be added + // @return New modified shape + TensorShape InsertDim(dsize_t axis, dsize_t dim) const; + + // Insert new dim at index 0. For example, <2,4> --> PrependDim(4) --> <4,2,4> + // @param dim + // @return + TensorShape PrependDim(dsize_t dim) const; + + // Insert a new dim at the end of the shape. For example, <2,4> --> PrependDim(4) --> <2,4,4> + // @param dim + // @return + TensorShape AppendDim(dsize_t dim) const; + + // Create a shape based on OpenCV shape and type + // @param cv_size + // @param type int that represent the type in OpenCV, example CV_8U, CV_64S + TensorShape(cv::MatSize cv_size, uint32_t type); + + dsize_t Size() const { return raw_shape_.size(); } + + dsize_t Rank() const { return raw_shape_.size(); } + + bool known() const { return known_; } + + bool empty() const { return raw_shape_.empty(); } + + dsize_t NumOfElements() const; + + bool operator==(const TensorShape &rhs) const { return known_ == rhs.known_ && raw_shape_ == rhs.raw_shape_; } + + bool operator!=(const TensorShape &rhs) const { return !(rhs == *this); } + + dsize_t operator[](const dsize_t index) const { return raw_shape_[index]; } + + // Return the Shape as a vector + // @return + std::vector AsVector() const; + + // Returns the class info as a string + // @return + std::string ToString() const { + std::stringstream ss; + ss << *this; + return ss.str(); + } + + // Actual print function used by operator<< + // @param out output string stream + void Print(std::ostream &out) const; + + // << Stream output operator overload + // @notes This allows you to print the info using stream operators + // @param out - reference to the output stream being overloaded + // @param rO - reference to the TensorShape to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const TensorShape &so) { + so.Print(out); + return out; + } + + explicit TensorShape(py::list l); + + py::list AsPyList(); + + // Checks if the given index is a valid index for this tensor. + // For example: Tensor<3,4> Index<1,1> is valid. But Index<4,1> or <1> are not. + // @param index + // @return bool + bool IsValidIndex(const std::vector &index) const; + + TensorShape Squeeze() const; + + private: + // True if known and valid shape, false otherwise + bool known_; + // Vector to keep the dims of the shape. + std::vector raw_shape_; + + // Internal utility function to iterate over a list, check if the dim is valid and then insert it into the shape. + // @tparam T list + // @param list Iterable list + // @return true if the shape is valid and no overflow would be generated when counting the number of elements. + // False otherwise. + template + void AddListToShape(const T &list); +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_CORE_TENSOR_SHAPE_H_ diff --git a/mindspore/ccsrc/dataset/engine/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/CMakeLists.txt new file mode 100644 index 00000000000..c7277de8def --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/CMakeLists.txt @@ -0,0 +1,18 @@ +add_subdirectory(datasetops) +if (ENABLE_TDTQUE) + add_subdirectory(tdt) +endif () + +add_library(engine OBJECT + execution_tree.cc + data_buffer.cc + data_schema.cc + dataset_iterator.cc + ) +target_include_directories(engine PRIVATE ${pybind11_INCLUDE_DIRS}) + +if (ENABLE_TDTQUE) + add_dependencies(engine engine-datasetops engine-datasetops-source engine-tdt) +else() + add_dependencies(engine engine-datasetops engine-datasetops-source) +endif () diff --git a/mindspore/ccsrc/dataset/engine/connector.h b/mindspore/ccsrc/dataset/engine/connector.h new file mode 100644 index 00000000000..de8ebb8ba3e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/connector.h @@ -0,0 +1,188 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_CONNECTOR_H_ +#define DATASET_ENGINE_CONNECTOR_H_ + +#include +#include +#include +#include +#include "dataset/util/task_manager.h" +#include "dataset/util/queue.h" +#include "dataset/util/services.h" +#include "dataset/util/cond_var.h" + +namespace mindspore { +namespace dataset { +// Connector is a communication data structure between two group of threads that +// preserve the order. +// +// Example use case: +// An initial tasks-list of [1,2,3,4,5,6,7,8,9] with 5 threads getting/processing elements from that list, +// and pushing the processed elements to a Connector in any order whoever finishes processing first. +// If the consumer of the Connector is single threaded, when the consumer pop() the +// element from the Connector one by one, it will get [1,2,3,4,5,6,7,8,9]. +// +// Requirements: +// 1. Each thread in the group of consumer or producer threads must be assigned ids starting from 0. +// 2. If your multi-threads program is not reading from a Connector class but +// want to push to a Connector class, you must follow roundrobin element distribution, +// i.e., the thread-id0 must have the first element, thread-id1 has the second element, +// and so on; then each of this worker can push to the Connector class async in parallel. +// +// Blocking conditions: +// 1. Connector.push(int, T) can block when the internal queue it's trying to push is full. +// 2. Connector.pop(int) can block when +// - The internal queue it's trying to pop is empty. +// - The caller thread of pop() is not equal to the _expectConsumer. This is to enforce +// the ordering. +// +// Future improvement: +// 1. Fault tolerant: Right now, if one of the worker dies, the Connector will not work +// properly. +template +class Connector { + public: + // Name: Constructor + // Description: Initializing private members with the given input arguments. + // expect_consumer_ and pop_from_ is initialized to 0 as part of + // our requirements. We instantiate nProducers number of internal + // queues so that each producer thread can push to its queue without + // any sync overhead. + // Constructor of Connector + // Initializing private members with the given input arguments. + // _expectConsumer and _popFrom is initialized to 0 as part of + // our requirements. We instantiate nProducers number of internal + // queues so that each producer thread can push to its queue without + // any sync overhead. + // @param n_producers The number of threads producing data into this DbConnector. + // @param n_consumers The number of thread consuming data from this DbConnector. + // @param queue_capacity The number of element (DataBuffer) for each queue. + Connector(int32_t n_producers, int32_t n_consumers, int32_t queue_capacity) + : num_producers_(n_producers), num_consumers_(n_consumers) { + MS_LOG(INFO) << "A connector is created with " << n_producers << " producers and " << n_consumers << " consumers."; + my_name_ = Services::GetUniqueID(); + // We require the consumers to have ids sequentially from 0 to the num_consumers_-1, + // Otherwise a ordered list of consumer ids have to be passed here. (not implemented yet) + expect_consumer_ = 0; + + // Roundrobin pop starts from index 0 of the queues_. + pop_from_ = 0; + + // Initialize the queues_ to have num_producers_ number of queues. + // Each queue is a blocking queue and has the same queue_capacity. + queues_.Init(num_producers_, queue_capacity); + } + + // Destructor of Connector + virtual ~Connector() = default; + + // Get an element from the Connector. + // @not Call to pop() can block the caller thread, see the blocking condition at the top of this file. + // @param worker_id The id of a worker thread calling this method. + // @param result The address of an object where the popped element will be placed. + virtual Status Pop(int32_t worker_id, // The worker-id of the caller. See the requirement at the top of this file. + T *result) noexcept { + { + DS_ASSERT(worker_id < num_consumers_); + std::unique_lock lk(m_); + RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); + RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); + pop_from_ = (pop_from_ + 1) % num_producers_; + expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; + } + cv_.NotifyAll(); + return Status::OK(); + } + + // Add an element into the DbConnector without the overhead of synchronization. + // It may block when the internal queue is full. + // The element passed to this function will be copied into the internal queue. + // @param worker_id The id of a worker thread calling this method. + // @param el A const lvalue element to be passed/added/pushed. + Status Push(int32_t worker_id, const T &el) noexcept { + DS_ASSERT(worker_id < static_cast(queues_.size())); + DS_ASSERT(queues_[worker_id] != nullptr); + return (queues_[worker_id]->Add(el)); + } + + // Add an element into the DbConnector without the overhead of synchronization. + // It may block when the internal queue is full. + // The element passed to this function will be forwarded into the internal queue. + // @param worker_id The id of a worker thread calling this method. + // @param el An element to be passed/added/pushed. + virtual Status Push(int32_t worker_id, T &&el) noexcept { + DS_ASSERT(worker_id < static_cast(queues_.size())); + DS_ASSERT(queues_[worker_id] != nullptr); + return (queues_[worker_id]->Add(std::forward(el))); + } + + // Resets the internal index tracking of the queue so that it can be used again with new inputs, + // starting from the beginning. + void Reset() { + for (int i = 0; i < queues_.size(); ++i) { + queues_[i]->ResetQue(); + } + expect_consumer_ = 0; + pop_from_ = 0; + MS_LOG(INFO) << "Connector counters reset."; + } + + void Print(std::ostream &out, bool showAll) const { + out << "\n--------- Connector ------------" + << "\nConnector Name : " << my_name_ << "\nNumber of consumers : " << num_consumers_ + << "\nNumber of producers : " << num_producers_ << "\n"; + } + + friend std::ostream &operator<<(std::ostream &out, const Connector &con) { + con.print(out, false); + return out; + } + + // Register the internal resources with Task group for interruption service. + // @param vg + // @return + Status Register(TaskGroup *vg) { + Status rc = queues_.Register(vg); + if (rc.IsOk()) { + rc = cv_.Register(vg->GetIntrpService()); + } + return rc; + } + + protected: + std::string my_name_; + + // A list of Queues that are thread safe. + QueueList queues_; + + // The consumer that we allow to get the next data from pop() + int32_t expect_consumer_; + + // The index to the queues_ where the next data should be popped. + int32_t pop_from_; + + int32_t num_producers_; + int32_t num_consumers_; + + // Used in the Pop(), when a thread call pop() but it is not the expect_consumer_. + std::mutex m_; + CondVar cv_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_CONNECTOR_H_ diff --git a/mindspore/ccsrc/dataset/engine/data_buffer.cc b/mindspore/ccsrc/dataset/engine/data_buffer.cc new file mode 100644 index 00000000000..a0f47512ec6 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/data_buffer.cc @@ -0,0 +1,132 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/data_buffer.h" +#include "dataset/util/allocator.h" +#include "dataset/core/global_context.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/engine/datasetops/source/tf_buffer.h" + +namespace mindspore { +namespace dataset { +// Name: Constructor #1 +// Description: This is the main constructor that is used for making a buffer +DataBuffer::DataBuffer(int32_t id, BufferFlags flags) : buffer_id_(id), tensor_table_(nullptr), buffer_flags_(flags) {} + +// Name: CreateDataBuffer() +// Description: A static factory method to create the appropriate type of derived class +// buffer. Returns the base class reference for DataBuffer. +Status DataBuffer::CreateDataBuffer( + int32_t id, // In: The id for the new buffer + std::shared_ptr storage_client, // In: The storage client that is related to this buffer type + std::unique_ptr *ptr) { + std::unique_ptr new_data_buffer; + try { + DatasetType ds_type = storage_client->schema()->dataset_type(); + switch (ds_type) { + case DatasetType::kTf: { + // This type of buffer is for TF record data. + // Allocate derived class version for a TF buffers + new_data_buffer = mindspore::make_unique(id, kDeBFlagNone, storage_client); + break; + } + default: { + std::string errMsg("Invalid buffer type"); + RETURN_STATUS_UNEXPECTED(errMsg); + } + } + } catch (std::bad_alloc &e) { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__, e.what()); + } catch (std::exception &e) { + RETURN_STATUS_UNEXPECTED(e.what()); + } + *ptr = std::move(new_data_buffer); + return Status::OK(); +} + +// Name: print() +// Description: A function that prints info about the DataBuffer (base class version) +void DataBuffer::Print(std::ostream &out, // In: The output stream to print to + bool show_all) const { // In: T/F if it should show everything + out << "bufferId: " << buffer_id_ << "\nflags: " << std::hex << buffer_flags_ << std::dec << "\n"; + + // If the column counts are set then it means that data has been set into + // the tensor table. Display the tensor table here. + if (this->NumCols() > 0) { + out << "Tensor table:\n"; + for (int32_t row = 0; row < DataBuffer::NumRows(); ++row) { + out << "Row # : " << row << "\n"; + TensorRow currRow = (*tensor_table_)[row]; + for (int32_t col = 0; col < this->NumCols(); ++col) { + out << "Column #: " << col << "\n"; // Should add the column name here as well? + // Call the tensor display + out << *(currRow[col]) << "\n"; + } + } + } +} + +Status DataBuffer::Load() { + std::string err_msg = "Base class load called, but it does not have an implementation!"; + RETURN_STATUS_UNEXPECTED(err_msg); +} + +// Remove me!! Callers should fetch rows via pop +Status DataBuffer::GetTensor(std::shared_ptr *ptr, int32_t row_id, int32_t col_id) const { + if (row_id < tensor_table_->size() && col_id < tensor_table_->at(row_id).size()) { + *ptr = (tensor_table_->at(row_id)).at(col_id); + } else { + std::string err_msg = + "indices for mTensorTable out of range: (" + std::to_string(row_id) + "," + std::to_string(col_id) + ")."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + return Status::OK(); +} + +// Remove me!! Callers should fetch rows via pop +Status DataBuffer::GetRow(int32_t row_id, TensorRow *ptr) const { + if (row_id < tensor_table_->size()) { + *ptr = tensor_table_->at(row_id); + } else { + std::string err_msg = "rowId for mTensorTable out of range: " + std::to_string(row_id); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + return Status::OK(); +} + +Status DataBuffer::PopRow(TensorRow *ptr) { + if (tensor_table_ && !tensor_table_->empty()) { + *ptr = std::move(tensor_table_->front()); + tensor_table_->pop_front(); + } + + return Status::OK(); +} + +Status DataBuffer::SliceOff(int64_t number_of_rows) { + while (number_of_rows > 0) { + tensor_table_->pop_back(); + number_of_rows--; + } + + return Status::OK(); +} + +// Destructor +DataBuffer::~DataBuffer() {} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/data_buffer.h b/mindspore/ccsrc/dataset/engine/data_buffer.h new file mode 100644 index 00000000000..c624ce4339d --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/data_buffer.h @@ -0,0 +1,142 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATA_BUFFER_H_ +#define DATASET_ENGINE_DATA_BUFFER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/status.h" +#include "dataset/core/constants.h" +#include "dataset/core/tensor.h" + +namespace mindspore { +namespace dataset { +// Forward declares +class StorageClient; + +// The DataBuffer class is a base class that will represent the data for n values based +// on a unique row id for each row of data. +// There can be different types of DataBuffers to abstract over how the data is stored +// in memory and acquired from storage. +// Each buffer holds a range of consecutive row id's. +class DataBuffer { + public: + // Buffer flags + enum BufferFlags : uint32_t { + kDeBFlagNone = 0, + kDeBFlagEOF = 1, // The buffer is an eof end-of-data msg + kDeBFlagEOE = 1u << 1 // The buffer is an eoe end-of-epoch msg + }; + + // Name: Constructor #1 + // Description: This is the main constructor that is used for making a buffer + DataBuffer(int32_t id, BufferFlags flags); + + // Destructor + virtual ~DataBuffer(); + + // Name: CreateDataBuffer() + // Description: A factory method to create the appropriate type of derived class + // buffer. Returns the base class reference for DataBuffer. + static Status CreateDataBuffer( + int32_t id, // In: The id for the new buffer + std::shared_ptr, // In: The StorageClient is used to choose the buffer type to create + std::unique_ptr *); + + // Name: print() + // Description: A function that prints info about the DataBuffer (base class version) + virtual void Print(std::ostream &out, // In: The output stream to print to + bool show_all) const; // In: T/F if it should show everything + + // Provide stream operator for displaying it + friend std::ostream &operator<<(std::ostream &out, const DataBuffer &cb) { + cb.Print(out, false); + return out; + } + + // Name: load() + // Description: populates the DataBuffer with data based on it's id + virtual Status Load(); + + // Convenience getter functions for flag checking + bool eof() const { return (static_cast(buffer_flags_) & static_cast(kDeBFlagEOF)); } + + bool eoe() const { return (static_cast(buffer_flags_) & static_cast(kDeBFlagEOE)); } + + // Simple getter funcs + int32_t id() const { return buffer_id_; } + + void set_id(int32_t id) { buffer_id_ = id; } + + int32_t NumRows() const { return ((tensor_table_) ? tensor_table_->size() : 0); } + + int32_t NumCols() const { + return (tensor_table_ == nullptr || tensor_table_->empty()) ? 0 : tensor_table_->at(0).size(); + } + + BufferFlags buffer_flags() const { return buffer_flags_; } + + // Remove me!! Callers should fetch rows via pop + Status GetTensor(std::shared_ptr *, int32_t row_id, int32_t col_id) const; + + // Remove me!! Callers should drain rows via pop. + Status GetRow(int32_t row_id, TensorRow *) const; + + // Get a row from the TensorTable + Status PopRow(TensorRow *); + + Status SliceOff(int64_t number_of_rows); + + // Return a mapping from col names to col id. + std::unordered_map column_name_map() const { return column_name_map_; } + + // Update the column name to index mapping. + void set_column_name_map(const std::unordered_map &new_col_name_map) { + column_name_map_ = new_col_name_map; + } + + // Replacing mTensorTable, the unique_ptr assignment will release the old TensorTable. + void set_tensor_table(std::unique_ptr new_table) { tensor_table_ = std::move(new_table); } + + void set_flag(BufferFlags in_flag) { + buffer_flags_ = static_cast(static_cast(buffer_flags_) | static_cast(in_flag)); + } + + void Shuffle() {} // does nothing right now. possibly remove later + + // ***** column_name_map_ manipulation methods ***** + + // Append Column to mColumnNameMap + Status AppendColumn(const std::string &name, const int32_t &old_id) const { // does nothing right now + return Status::OK(); + } + + protected: + int32_t buffer_id_; // An id for the buffer. + std::unique_ptr tensor_table_; // A table (row major) of Tensors + BufferFlags buffer_flags_; // bit mask for various buffer properties + std::unordered_map column_name_map_; // A mapping between column index to column name. +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATA_BUFFER_H_ diff --git a/mindspore/ccsrc/dataset/engine/data_schema.cc b/mindspore/ccsrc/dataset/engine/data_schema.cc new file mode 100644 index 00000000000..68666796be8 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/data_schema.cc @@ -0,0 +1,470 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/data_schema.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "dataset/util/status.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/util/make_unique.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +// A macro for converting an input string representing the column type to it's actual +// numeric column type. +#define STR_TO_TENSORIMPL(in_col_str, out_type) \ + do { \ + if (in_col_str == "cvmat") { \ + out_type = TensorImpl::kCv; \ + } else if (in_col_str == "flex") { \ + out_type = TensorImpl::kFlexible; \ + } else if (in_col_str == "np") { \ + out_type = TensorImpl::kNP; \ + } else { \ + out_type = TensorImpl::kNone; \ + } \ + } while (false) + +// Constructor 1: Simple constructor that leaves things uninitialized. +ColDescriptor::ColDescriptor() + : type_(DataType::DE_UNKNOWN), rank_(0), tensor_impl_(TensorImpl::kNone), tensor_shape_(nullptr) {} + +// Constructor 2: Main constructor +ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, TensorImpl tensor_impl, int32_t rank, + const TensorShape *in_shape) + : type_(col_type), rank_(rank), tensor_impl_(tensor_impl), col_name_(col_name) { + // If a shape was provided, create unique pointer for it and copy construct it into + // our shape. Otherwise, set our shape to be empty. + if (in_shape != nullptr) { + // Create a shape and copy construct it into our column's shape. + tensor_shape_ = mindspore::make_unique(*in_shape); + } else { + tensor_shape_ = nullptr; + } + // If the user input a shape, then the rank of the input shape needs to match + // the input rank + if (in_shape != nullptr && in_shape->known() && in_shape->Size() != rank_) { + rank_ = in_shape->Size(); + MS_LOG(INFO) << "Rank does not match the number of dimensions in the provided shape." + << " Overriding rank with the number of dimensions in the provided shape."; + } +} + +// Explicit copy constructor is required +ColDescriptor::ColDescriptor(const ColDescriptor &in_cd) + : type_(in_cd.type_), rank_(in_cd.rank_), tensor_impl_(in_cd.tensor_impl_), col_name_(in_cd.col_name_) { + // If it has a tensor shape, make a copy of it with our own unique_ptr. + tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique(in_cd.shape()) : nullptr; +} + +// Assignment overload +ColDescriptor &ColDescriptor::operator=(const ColDescriptor &in_cd) { + if (&in_cd != this) { + type_ = in_cd.type_; + rank_ = in_cd.rank_; + tensor_impl_ = in_cd.tensor_impl_; + col_name_ = in_cd.col_name_; + // If it has a tensor shape, make a copy of it with our own unique_ptr. + tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique(in_cd.shape()) : nullptr; + } + return *this; +} + +// Destructor +ColDescriptor::~ColDescriptor() = default; + +// A print method typically used for debugging +void ColDescriptor::Print(std::ostream &out) const { + out << " Name : " << col_name_ << "\n Type : " << type_ << "\n Rank : " << rank_ + << "\n Shape : ("; + if (tensor_shape_) { + out << *tensor_shape_ << ")\n"; + } else { + out << "no shape provided)\n"; + } +} + +// Given a number of elements, this function will compute what the actual Tensor shape would be. +// If there is no starting TensorShape in this column, or if there is a shape but it contains +// an unknown dimension, then the output shape returned shall resolve dimensions as needed. +Status ColDescriptor::MaterializeTensorShape(int32_t num_elements, TensorShape *out_shape) const { + if (out_shape == nullptr) { + RETURN_STATUS_UNEXPECTED("Unexpected null output shape argument."); + } + + // If the shape is not given in this column, then we assume the shape will be: {numElements} + if (tensor_shape_ == nullptr) { + if (this->rank() == 0 && num_elements == 1) { + *out_shape = TensorShape::CreateScalar(); + return Status::OK(); + } + *out_shape = TensorShape({num_elements}); + return Status::OK(); + } + + // Build the real TensorShape based on the requested shape and the number of elements in the data. + // If there are unknown dimensions, then the unknown dimension needs to be filled in. + // Example: requestedShape: {?,4,3}. + // If numElements is 24, then the output shape can be computed to: {2,4,3} + std::vector requested_shape = tensor_shape_->AsVector(); + int64_t num_elements_of_shape = 1; // init to 1 as a starting multiplier. + + // unknownDimPosition variable is overloaded to provide 2 meanings: + // 1) If it's set to DIM_UNKNOWN, then it provides a boolean knowledge to tell us if there are + // any unknown dimensions. i.e. if it's set to unknown, then there are no unknown dimensions. + // 2) If it's set to a numeric value, then this is the vector index position within the shape + // where the single unknown dimension can be found. + int64_t unknown_dim_position = TensorShape::kDimUnknown; // Assume there are no unknown dims to start + + for (int i = 0; i < requested_shape.size(); ++i) { + // If we already had an unknown dimension, then we cannot have a second unknown dimension. + // We only support the compute of a single unknown dim. + if (requested_shape[i] == TensorShape::kDimUnknown && unknown_dim_position != TensorShape::kDimUnknown) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Requested shape has more than one unknown dimension!"); + } + + // If the current dimension in the requested shape is a known value, then compute the number of + // elements so far. + if (requested_shape[i] != TensorShape::kDimUnknown) { + num_elements_of_shape *= requested_shape[i]; + } else { + // This dimension is unknown so track which dimension position has it. + unknown_dim_position = i; + } + } + + // Sanity check the the computed element counts divide evenly into the input element count + if (num_elements < num_elements_of_shape || num_elements_of_shape == 0 || num_elements % num_elements_of_shape != 0) { + RETURN_STATUS_UNEXPECTED("Requested shape has an invalid element count!"); + } + + // If there was any unknown dimensions, then update the requested shape to fill in the unknown + // dimension with the correct value. If there were no unknown dim's then the output shape will + // remain to be the same as the requested shape. + if (unknown_dim_position != TensorShape::kDimUnknown) { + requested_shape[unknown_dim_position] = (num_elements / num_elements_of_shape); + } + + // Any unknown dimension is filled in now. Set the output shape + *out_shape = TensorShape(requested_shape); + return Status::OK(); +} + +// getter function for the shape +TensorShape ColDescriptor::shape() const { + if (tensor_shape_ != nullptr) { + return *tensor_shape_; // copy construct a shape to return + } else { + return TensorShape::CreateUnknownRankShape(); // empty shape to return + } +} + +const char DataSchema::DEFAULT_DATA_SCHEMA_FILENAME[] = "datasetSchema.json"; + +// Constructor 1: Simple constructor that leaves things uninitialized. +DataSchema::DataSchema() : dataset_type_(DatasetType::kUnknown), num_rows_(0) {} + +DatasetType DataSchema::GetDatasetTYpeFromString(const std::string &type) const { + // Convert the string to a more easy to manage enum flavour of the buffer type. + if (type == "ARROW") { + return DatasetType::kArrow; + } else if (type == "TF") { + return DatasetType::kTf; + } else { + return DatasetType::kUnknown; + } +} + +Status DataSchema::LoadDatasetType(const std::string &schema_file_path) { + try { + std::ifstream in(schema_file_path); + nlohmann::json js; + in >> js; + // First, get the column for the type of dataset. + dataset_type_str_ = js.value("datasetType", ""); + dataset_type_ = GetDatasetTYpeFromString(dataset_type_str_); + dir_structure_ = js.value("directoryStructure", ""); + } + // Catch any exception and convert to Status return code + catch (const std::exception &err) { + RETURN_STATUS_UNEXPECTED("Schema file failed to load"); + } + return Status::OK(); +} + +// Internal helper function. Parses the json schema file in any order and produces a schema that +// does not follow any particular order (json standard does not enforce any ordering protocol). +// This one produces a schema that contains all of the columns from the schema file. +Status DataSchema::AnyOrderLoad(nlohmann::json column_tree) { + // Iterate over the json file. Each parent json node is the column name, + // followed by the column properties in the child tree under the column. + // Outer loop here iterates over the parents (i.e. the column name) + if (!column_tree.is_array()) { + for (nlohmann::json::iterator it = column_tree.begin(); it != column_tree.end(); ++it) { + std::string col_name = it.key(); + nlohmann::json column_child_tree = it.value(); + RETURN_IF_NOT_OK(ColumnLoad(column_child_tree, col_name)); + } + } else { + // Case where the schema is a list of columns not a dict + for (nlohmann::json::iterator it = column_tree.begin(); it != column_tree.end(); ++it) { + nlohmann::json column_child_tree = it.value(); + RETURN_IF_NOT_OK(ColumnLoad(column_child_tree, "")); + } + } + return Status::OK(); +} + +// Internal helper function. For each input column name, perform a lookup to the json document to +// find the matching column. When the match is found, process that column to build the column +// descriptor and add to the schema in the order in which the input column names are given.id +Status DataSchema::ColumnOrderLoad(nlohmann::json column_tree, const std::vector &columns_to_load) { + if (!column_tree.is_array()) { + // the json file is dict (e.g., {image: ...}) + // Loop over the column name list + for (const auto &curr_col_name : columns_to_load) { + // Find the column in the json document + auto column_info = column_tree.find(common::SafeCStr(curr_col_name)); + if (column_info == column_tree.end()) { + RETURN_STATUS_UNEXPECTED("Failed to find column " + curr_col_name); + } + // At this point, columnInfo.value() is the subtree in the json document that contains + // all of the data for a given column. This data will formulate our schema column. + const std::string &col_name = column_info.key(); + nlohmann::json column_child_tree = column_info.value(); + RETURN_IF_NOT_OK(ColumnLoad(column_child_tree, col_name)); + } + } else { + // the json file is array (e.g., [name: image...]) + // Loop over the column name list + for (const auto &curr_col_name : columns_to_load) { + // Find the column in the json document + int32_t index = -1; + int32_t i = 0; + for (const auto &it_child : column_tree.items()) { + auto name = it_child.value().find("name"); + if (name == it_child.value().end()) { + RETURN_STATUS_UNEXPECTED("Name field is missing for this column."); + } + if (name.value() == curr_col_name) { + index = i; + break; + } + i++; + } + if (index == -1) { + RETURN_STATUS_UNEXPECTED("Failed to find column " + curr_col_name); + } + nlohmann::json column_child_tree = column_tree[index]; + RETURN_IF_NOT_OK(ColumnLoad(column_child_tree, curr_col_name)); + } + } + return Status::OK(); +} + +// Internal helper function for parsing shape info and building a vector for the shape construction. +static Status buildShape(const nlohmann::json &shapeVal, std::vector *outShape) { + if (outShape == nullptr) { + RETURN_STATUS_UNEXPECTED("null output shape"); + } + if (shapeVal.empty()) return Status::OK(); + + // Iterate over the integer list and add those values to the output shape tensor + auto items = shapeVal.items(); + using it_type = decltype(items.begin()); + (void)std::transform(items.begin(), items.end(), std::back_inserter(*outShape), [](it_type j) { return j.value(); }); + return Status::OK(); +} + +// Internal helper function. Given the json tree for a given column, load it into our schema. +Status DataSchema::ColumnLoad(nlohmann::json column_child_tree, const std::string &col_name) { + int32_t rank_value = -1; + TensorImpl t_impl_value = TensorImpl::kFlexible; + std::string name, type_str; + std::vector tmp_shape = {}; + bool shape_field_exists = false; + // Iterate over this column's attributes. + // Manually iterating each of the child nodes/trees here so that we can provide our own error handling. + for (const auto &it_child : column_child_tree.items()) { + // Save the data for each of the attributes into variables. We'll use these to construct later. + if (it_child.key() == "name") { + name = it_child.value(); + } else if (it_child.key() == "type") { + type_str = it_child.value(); + } else if (it_child.key() == "rank") { + rank_value = it_child.value(); + } else if (it_child.key() == "t_impl") { + STR_TO_TENSORIMPL(it_child.value(), t_impl_value); + } else if (it_child.key() == "shape") { + shape_field_exists = true; + RETURN_IF_NOT_OK(buildShape(it_child.value(), &tmp_shape)); + } else { + std::string err_msg = "Unexpected column attribute " + it_child.key() + " for column " + col_name; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + if (!name.empty()) { + if (!col_name.empty() && col_name != name) { + std::string err_msg = + "json schema file for column " + col_name + " has column name that does not match columnsToLoad"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } else { + if (col_name.empty()) { + std::string err_msg = "json schema file for column " + col_name + " has invalid or missing column name."; + RETURN_STATUS_UNEXPECTED(err_msg); + } else { + name = col_name; + } + } + // data type is mandatory field + if (type_str.empty()) + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "json schema file for column " + col_name + " has invalid or missing column type."); + + // rank number is mandatory field + if (rank_value <= -1) + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "json schema file for column " + col_name + " must define a positive rank value."); + + // Create the column descriptor for this column from the data we pulled from the json file + TensorShape col_shape = TensorShape(tmp_shape); + if (shape_field_exists) + (void)this->AddColumn(ColDescriptor(name, DataType(type_str), t_impl_value, rank_value, &col_shape)); + else + // Create a column descriptor that doesn't have a shape + (void)this->AddColumn(ColDescriptor(name, DataType(type_str), t_impl_value, rank_value)); + return Status::OK(); +} + +// Parses a schema json file and populates the columns and meta info. +Status DataSchema::LoadSchemaFile(const std::string &schema_file_path, + const std::vector &columns_to_load) { + try { + std::ifstream in(schema_file_path); + + nlohmann::json js; + in >> js; + RETURN_IF_NOT_OK(PreLoadExceptionCheck(js)); + try { + num_rows_ = js.at("numRows").get(); + } catch (nlohmann::json::out_of_range &e) { + num_rows_ = 0; + } catch (nlohmann::json::exception &e) { + RETURN_STATUS_UNEXPECTED("Unable to parse \"numRows\" from schema"); + } + nlohmann::json column_tree = js.at("columns"); + if (column_tree.empty()) { + RETURN_STATUS_UNEXPECTED("columns is null"); + } + if (columns_to_load.empty()) { + // Parse the json tree and load the schema's columns in whatever order that the json + // layout decides + RETURN_IF_NOT_OK(this->AnyOrderLoad(column_tree)); + } else { + RETURN_IF_NOT_OK(this->ColumnOrderLoad(column_tree, columns_to_load)); + } + } catch (const std::exception &err) { + // Catch any exception and convert to Status return code + RETURN_STATUS_UNEXPECTED("Schema file failed to load"); + } + return Status::OK(); +} + +// Parses a schema json string and populates the columns and meta info. +Status DataSchema::LoadSchemaString(const std::string &schema_json_string, + const std::vector &columns_to_load) { + try { + nlohmann::json js = nlohmann::json::parse(schema_json_string); + RETURN_IF_NOT_OK(PreLoadExceptionCheck(js)); + num_rows_ = js.value("numRows", 0); + dataset_type_str_ = js.value("datasetType", ""); + dataset_type_ = GetDatasetTYpeFromString(dataset_type_str_); + nlohmann::json column_tree = js.at("columns"); + if (column_tree.empty()) { + RETURN_STATUS_UNEXPECTED("columns is null"); + } + if (columns_to_load.empty()) { + // Parse the json tree and load the schema's columns in whatever order that the json + // layout decides + RETURN_IF_NOT_OK(this->AnyOrderLoad(column_tree)); + } else { + RETURN_IF_NOT_OK(this->ColumnOrderLoad(column_tree, columns_to_load)); + } + } catch (const std::exception &err) { + // Catch any exception and convert to Status return code + RETURN_STATUS_UNEXPECTED("Schema file failed to load"); + } + return Status::OK(); +} + +// Destructor +DataSchema::~DataSchema() = default; + +// Getter for the ColDescriptor by index +const ColDescriptor &DataSchema::column(int32_t idx) const { + DS_ASSERT(idx < static_cast(col_descs_.size())); + return col_descs_[idx]; +} + +// A print method typically used for debugging +void DataSchema::Print(std::ostream &out) const { + out << "Dataset type string : ("; + if (dataset_type_str_.empty()) { + out << "none specified)\n"; + } else { + out << dataset_type_str_ << ")\n"; + } + for (const auto &col_desc : col_descs_) { + out << col_desc << "\n"; + } + out << "Dataset type: " << static_cast(dataset_type_) << "\n"; +} + +// Adds a column descriptor to the schema +Status DataSchema::AddColumn(const ColDescriptor &cd) { + // Sanity check there's not a duplicate name before adding the column + for (int32_t i = 0; i < col_descs_.size(); ++i) { + if (col_descs_[i].name() == cd.name()) { + std::ostringstream ss; + ss << "column name '" << cd.name() << "' already exists in schema."; + std::string err_msg = ss.str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + col_descs_.push_back(cd); + return Status::OK(); +} + +// Internal helper function. Performs sanity checks on the json file setup. +Status DataSchema::PreLoadExceptionCheck(const nlohmann::json &js) { + // Check if columns node exists. It is required for building schema from file. + if (js.find("columns") == js.end()) + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "\"columns\" node is required in the schema json file."); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/data_schema.h b/mindspore/ccsrc/dataset/engine/data_schema.h new file mode 100644 index 00000000000..4b2be76f073 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/data_schema.h @@ -0,0 +1,221 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATA_SCHEMA_H_ +#define DATASET_ENGINE_DATA_SCHEMA_H_ + +#include +#include +#include +#include +#include +#include +#include "dataset/core/constants.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// A simple class to provide meta info about a column. +class ColDescriptor { + public: + // Constructor 1: Simple constructor that leaves things uninitialized. + ColDescriptor(); + + // Constructor 2: Main constructor + // @param col_name - The name of the column + // @param col_type - The DE Datatype of the column + // @param tensor_impl - The (initial) type of tensor implementation for the column + // @param rank - The number of dimension of the data + // @param in_shape - option argument for input shape + ColDescriptor(const std::string &col_name, DataType col_type, TensorImpl tensor_impl, int32_t rank, + const TensorShape *in_shape = nullptr); + + // Explicit copy constructor is required + // @param in_cd - the source ColDescriptor + ColDescriptor(const ColDescriptor &in_cd); + + // Assignment overload + // @param in_cd - the source ColDescriptor + ColDescriptor &operator=(const ColDescriptor &in_cd); + + // Destructor + ~ColDescriptor(); + + // A print method typically used for debugging + // @param out - The output stream to write output to + void Print(std::ostream &out) const; + + // Given a number of elements, this function will compute what the actual Tensor shape would be. + // If there is no starting TensorShape in this column, or if there is a shape but it contains + // an unknown dimension, then the output shape returned shall resolve dimensions as needed. + // @param num_elements - The number of elements in the data for a Tensor + // @param out_shape - The materialized output Tensor shape + // @return Status - The error code return + Status MaterializeTensorShape(int32_t num_elements, TensorShape *out_shape) const; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param cd - reference to the ColDescriptor to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const ColDescriptor &cd) { + cd.Print(out); + return out; + } + + // getter function + // @return The column's DataType + DataType type() const { return type_; } + + // getter function + // @return The column's rank + int32_t rank() const { return rank_; } + + // getter function + // @return The column's name + std::string name() const { return col_name_; } + + // getter function + // @return The column's shape + TensorShape shape() const; + + // getter function + // @return TF if the column has an assigned fixed shape. + bool hasShape() const { return tensor_shape_ != nullptr; } + + // getter function + // @return The column's tensor implementation type + TensorImpl tensorImpl() const { return tensor_impl_; } + + private: + DataType type_; // The columns type + int32_t rank_; // The rank for this column (number of dimensions) + TensorImpl tensor_impl_; // The initial flavour of the tensor for this column. + std::unique_ptr tensor_shape_; // The fixed shape (if given by user) + std::string col_name_; // The name of the column +}; + +// A list of the columns. +class DataSchema { + public: + // Constructor + DataSchema(); + + // Destructor + ~DataSchema(); + + // Populates the schema with a dataset type from a json file. It does not populate any of the + // column info. To populate everything, use loadSchema() afterwards. + // @param schema_file_path - Absolute path to the schema file to use for getting dataset type info. + Status LoadDatasetType(const std::string &schema_file_path); + + // Parses a schema json file and populates the columns and meta info. + // @param schema_file_path - the schema file that has the column's info to load + // @param columns_to_load - list of strings for columns to load. if empty, assumes all columns. + // @return Status - The error code return + Status LoadSchemaFile(const std::string &schema_file_path, const std::vector &columns_to_load); + + // Parses a schema JSON string and populates the columns and meta info. + // @param schema_json_string - the schema file that has the column's info to load + // @param columns_to_load - list of strings for columns to load. if empty, assumes all columns. + // @return Status - The error code return + Status LoadSchemaString(const std::string &schema_json_string, const std::vector &columns_to_load); + + // A print method typically used for debugging + // @param out - The output stream to write output to + void Print(std::ostream &out) const; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param ds - reference to the DataSchema to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const DataSchema &ds) { + ds.Print(out); + return out; + } + + // Adds a column descriptor to the schema + // @param cd - The ColDescriptor to add + // @return Status - The error code return + Status AddColumn(const ColDescriptor &cd); + + // Setter + // @param in_type - The Dataset type to set into the schema + void set_dataset_type(DatasetType in_type) { dataset_type_ = in_type; } + + // getter + // @return The dataset type of the schema + DatasetType dataset_type() const { return dataset_type_; } + + // getter + // @return The reference to a ColDescriptor to get (const version) + const ColDescriptor &column(int32_t idx) const; + + // getter + // @return The number of columns in the schema + int32_t NumColumns() const { return col_descs_.size(); } + + bool Empty() const { return NumColumns() == 0; } + + std::string dir_structure() const { return dir_structure_; } + + std::string dataset_type_str() const { return dataset_type_str_; } + + int64_t num_rows() const { return num_rows_; } + + static const char DEFAULT_DATA_SCHEMA_FILENAME[]; + + private: + // Internal helper function. Parses the json schema file in any order and produces a schema that + // does not follow any particular order (json standard does not enforce any ordering protocol). + // This one produces a schema that contains all of the columns from the schema file. + // @param column_tree - The nlohmann tree from the json file to parse + // @return Status - The error code return + Status AnyOrderLoad(nlohmann::json column_tree); + + // Internal helper function. For each input column name, perform a lookup to the json document to + // find the matching column. When the match is found, process that column to build the column + // descriptor and add to the schema in the order in which the input column names are given. + // @param column_tree - The nlohmann tree from the json file to parse + // @param columns_to_load - list of strings for the columns to add to the schema + // @return Status - The error code return + Status ColumnOrderLoad(nlohmann::json column_tree, const std::vector &columns_to_load); + + // Internal helper function. Given the json tree for a given column, load it into our schema. + // @param columnTree - The nlohmann child tree for a given column to load. + // @param col_name - The string name of the column for that subtree. + // @return Status - The error code return + Status ColumnLoad(nlohmann::json column_child_tree, const std::string &col_name); + + // Internal helper function. Performs sanity checks on the json file setup. + // @param js - The nlohmann tree for the schema file + // @return Status - The error code return + Status PreLoadExceptionCheck(const nlohmann::json &js); + + DatasetType GetDatasetTYpeFromString(const std::string &type) const; + + std::vector col_descs_; // Vector of column descriptors + std::string dataset_type_str_; // A string that represents the type of dataset + DatasetType dataset_type_; // The numeric form of the dataset type from enum + std::string dir_structure_; // Implicit or flatten + int64_t num_rows_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATA_SCHEMA_H_ diff --git a/mindspore/ccsrc/dataset/engine/dataset_iterator.cc b/mindspore/ccsrc/dataset/engine/dataset_iterator.cc new file mode 100644 index 00000000000..a9cbf06978e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/dataset_iterator.cc @@ -0,0 +1,230 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/dataset_iterator.h" +#include +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/status.h" +#include "dataset/engine/datasetops/dataset_op.h" + +namespace mindspore { +namespace dataset { +// Constructor of the IteratorBase +IteratorBase::IteratorBase() : curr_buffer_(nullptr), eof_handled_(false) {} + +IteratorBase::~IteratorBase() = default; + +// Fetches one row of data from the iterator as a column map. +Status IteratorBase::GetNextAsMap(TensorMap *out_map) { + if (out_map == nullptr) { + RETURN_STATUS_UNEXPECTED("Null output map in iterator!"); + } + + out_map->clear(); + + TensorRow curr_row; + RETURN_IF_NOT_OK(FetchNextTensorRow(&curr_row)); + + // Return empty map if there's no data + if (curr_row.empty()) { + return Status::OK(); + } + + // Populate the out map from the row and return it + for (auto colMap : col_name_id_map_) { + (*out_map)[colMap.first] = std::move(curr_row[colMap.second]); + } + + return Status::OK(); +} + +// Fetches one row of data from the iterator. +// The base class version simply performs error handling and returns empty row. Actual +// functionality exists in the derived versions of this function. +Status IteratorBase::FetchNextTensorRow(TensorRow *out_row) { + if (out_row == nullptr) { + RETURN_STATUS_UNEXPECTED("Null output row in iterator!"); + } + + // clear the old tensor row + out_row->clear(); + + return Status::OK(); +} + +// Constructor of the DatasetIterator +DatasetIterator::DatasetIterator(std::shared_ptr exe_tree) : IteratorBase(), root_(exe_tree->root()) {} + +DatasetIterator::~DatasetIterator() = default; + +// Fetches one row of data from the iterator. Overrides the base class. This one fetches +// from the tree root node directly. +Status DatasetIterator::FetchNextTensorRow(TensorRow *out_row) { + // Common code init and error checking in the base class. + RETURN_IF_NOT_OK(IteratorBase::FetchNextTensorRow(out_row)); + + // Once eof is handled, always return empty row. Class must be destroyed and recreated if you + // want to iterate again. + if (eof_handled_) { + return Status::OK(); + } + + // Check if we need to get a new DataBuffer to iterate. + if (curr_buffer_ == nullptr || curr_buffer_->NumRows() == 0) { + col_name_id_map_.clear(); + RETURN_IF_NOT_OK(root_->GetNextBuffer(&curr_buffer_)); + + // Since GetNextBuffer was used rather than GetNextInput(), it means we need to manually + // handle eoe and eof messages here. + // + // An eoe buffer means we have iterated fully to the end of the tree. + // An eoe buffer will be immediately followed by an eof buffer, which signals the shutdown of + // all operators. + if (curr_buffer_->eoe()) { + MS_LOG(INFO) << "End of data iteration. Fetch eof and then return empty row."; + + // Before returning the last empty vector, fetch the eof buffer which should be the last + // buffer, and then free it. + RETURN_IF_NOT_OK(root_->GetNextBuffer(&curr_buffer_)); + + if (!curr_buffer_->eof()) { + RETURN_STATUS_UNEXPECTED("Non-eof after getting eoe in iterator!"); + } + eof_handled_ = true; + curr_buffer_.reset(); // explicitly free the eof buffer + + return Status::OK(); + } + + if (curr_buffer_->eof()) { + // An eof by itself, without being preceded by an eoe, is possible if a repeat operator + // exists below us in the stack. Repeat operator eats eoe's but eventually allows the + // flow of an eof up the pipeline by itself. + eof_handled_ = true; + curr_buffer_.reset(); // explicitly free the eof buffer + return Status::OK(); + } + + col_name_id_map_ = curr_buffer_->column_name_map(); + } + + // If we got this far, now it's time to pop that next row for return to caller + RETURN_IF_NOT_OK(curr_buffer_->PopRow(out_row)); + + return Status::OK(); +} + +Status DatasetIterator::GetOutputShapes(std::vector *out_shapes) { + if (out_shapes == nullptr) { + RETURN_STATUS_UNEXPECTED("Null output shape argument"); + } + if (device_queue_row_.empty()) { + RETURN_IF_NOT_OK(FetchNextTensorRow(&device_queue_row_)); + } + for (auto ts : device_queue_row_) { + out_shapes->push_back(ts->shape()); + } + + return Status::OK(); +} + +Status DatasetIterator::GetOutputTypes(std::vector *out_types) { + if (out_types == nullptr) { + RETURN_STATUS_UNEXPECTED("Null output type argument"); + } + if (device_queue_row_.empty()) { + RETURN_IF_NOT_OK(FetchNextTensorRow(&device_queue_row_)); + } + for (auto ts : device_queue_row_) { + out_types->push_back(ts->type()); + } + return Status::OK(); +} + +// Constructor of the ChildIterator +ChildIterator::ChildIterator(DatasetOp *current_op, int32_t worker_id, int32_t child_idx) + : IteratorBase(), current_op_(current_op), child_idx_(child_idx), worker_id_(worker_id), end_epoch_(false) {} + +ChildIterator::~ChildIterator() { current_op_ = nullptr; } + +// Fetches one row of data from the iterator. Overrides the base class. This one fetches +// only from the child/worker id as given from the constructor. +Status ChildIterator::FetchNextTensorRow(TensorRow *out_row) { + // Common code init and error checking in the base class. + RETURN_IF_NOT_OK(IteratorBase::FetchNextTensorRow(out_row)); + + // Once eof is handled, always return empty row. Class must be destroyed and recreated if you + // want to iterate again. + if (eof_handled_) { + return Status::OK(); + } + + // Check if we need to get a new DataBuffer to iterate. + if (curr_buffer_ == nullptr || curr_buffer_->NumRows() == 0) { + col_name_id_map_.clear(); + RETURN_IF_NOT_OK(current_op_->GetNextInput(&curr_buffer_, worker_id_, child_idx_)); + + // Unlike the DatasetIterator, this child iterator does not quit after eoe. + // Instead, if an eoe is picked up here, we simply return an empty vector and it's up to the + // caller to decide what it wants to do next. + if (curr_buffer_->eoe()) { + MS_LOG(INFO) << "Child iterator picked up EOE."; + end_epoch_ = true; + return Status::OK(); + } + + if (curr_buffer_->eof()) { + MS_LOG(INFO) << "Child iterator picked up EOF."; + eof_handled_ = true; + return Status::OK(); + } + + col_name_id_map_ = curr_buffer_->column_name_map(); + } + + // If we got this far, now it's time to pop that next row for return to caller + RETURN_IF_NOT_OK(curr_buffer_->PopRow(out_row)); + + return Status::OK(); +} + +// drain till the next eoe +Status ChildIterator::Drain() { + if (end_epoch_ == true) { + // Calling drain against a child that is already at it's eoe state will not result in any action. + // This allows you to do: + // - fetch until empty row + // - drain (will not actually drain because you are already at the end of the iteration) + // However, the next time after that, it will perform it's normal draining activities. + end_epoch_ = false; + MS_LOG(INFO) << "No operation drain, already at end of epoch."; + return Status::OK(); + } + MS_LOG(INFO) << "Child draining buffers until eoe."; + // else we drain until eoe or eof, eof here is for sanity check + while (!curr_buffer_->eoe() && !curr_buffer_->eof()) { + RETURN_IF_NOT_OK(current_op_->GetNextInput(&curr_buffer_, worker_id_, child_idx_)); + } + if (curr_buffer_->eof()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Child iterator picked up EOF in drain."); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/dataset_iterator.h b/mindspore/ccsrc/dataset/engine/dataset_iterator.h new file mode 100644 index 00000000000..032fc8e9a8e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/dataset_iterator.h @@ -0,0 +1,146 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASET_ITERATOR_H_ +#define DATASET_ENGINE_DATASET_ITERATOR_H_ + +#include +#include +#include +#include +#include "dataset/util/status.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { +using TensorMap = std::unordered_map>; + +// forward declare +class ExecutionTree; + +class DataBuffer; + +// IteratorBase class is used to iterate data from an executionTree one row at a time. +// The base class provides the general interface, whereas derived classes provide slightly +// different implementations. +class IteratorBase { + public: + // Constructor of IteratorBase + IteratorBase(); + + // Destructor + virtual ~IteratorBase(); + + // Fetches one row of data from the iterator. + // the base class version simply performs error handling and returns empty row. Actual + // functionality exists in the derived versions of this function. + // @param out_row - A TensorRow (vector of shared pointers to Tensors). If any of the of data + // messages are encountered (such as eoe or eof), then an empty TensorRow is returned back. + // @return Status - The error code return + // @note The position of a Tensor/column might be different from the initial column order + // in the storageOp. User must be aware that MapOp, ZipOps, and others might change + // the column ordering. + virtual Status FetchNextTensorRow(TensorRow *out_row); + + // Fetches one row of data from the iterator as a column map. + // @return A unordered map from column name to shared pointer to Tensor. + Status GetNextAsMap(TensorMap *out_map); + + // Getter + // @return T/F if this iterator is completely done after getting an eof + bool eof_handled() const { return eof_handled_; } + + // Getter + // @return The string to column id mapping. + std::unordered_map col_name_id_map() const { return col_name_id_map_; } + + protected: + std::unique_ptr curr_buffer_; // holds the current buffer + + // The column name-id mapping for the current data buffer. + std::unordered_map col_name_id_map_; + + bool eof_handled_; // T/F if this op got an eof +}; + +// The DatasetIterator derived class is for fetching rows off the end/root of the execution tree. +class DatasetIterator : public IteratorBase { + public: + // Constructor of the DatasetIterator + // @param exe_tree The execution tree we want to pull/iterate the data from using it's root node. + explicit DatasetIterator(std::shared_ptr exe_tree); + + // Destructor + ~DatasetIterator(); + + // Fetches one row of data from the iterator. Overrides the base class. This one fetches + // from the tree root node directly. + // @param out_row - A TensorRow (vector of shared pointers to Tensors). If any of the of data + // messages are encountered (such as eoe or eof), then an empty TensorRow is returned back. + // @return Status - The error code return + Status FetchNextTensorRow(TensorRow *out_row) override; + + // Fetches the next tensor row into device row, and returns it's shape. + // @param out_shapes - A vector of tensor shapes (one shape per column) + // @return Status - The error code return + Status GetOutputShapes(std::vector *out_shapes); + + // Fetches the next tensor row into device row, and returns it's shape. + // @param outShapes - A vector of tensor shapes (one shape per column) + // @return Status - The error code return + Status GetOutputTypes(std::vector *out_types); + + private: + std::shared_ptr root_; // saves the root of the executionTree + TensorRow device_queue_row_; +}; + +// The ChildIterator derived class is for fetching rows from intermediate nodes of execution tree. +// This one should only be used by internal Dataset operators, rather than an end-user. +class ChildIterator : public IteratorBase { + public: + // Constructor of the DatasetIterator + // @param current_op - The parent op from which we'll fetch from it's children. + // @param worker_id - The worker id to use when fetching from the children. + // @param child_idx - The index to the child to fetch from. + ChildIterator(DatasetOp *current_op, int32_t worker_id, int32_t child_idx); + + // Destructor + ~ChildIterator(); + + // Fetches one row of data from the iterator. Overrides the base class. This one fetches + // only from the child/worker id as given from the constructor. + // @param out_row - A TensorRow (vector of shared pointers to Tensors). If any of the of data + // messages are encountered (such as eoe or eof), then an empty TensorRow is returned back. + // @return Status - The error code return + Status FetchNextTensorRow(TensorRow *out_row) override; + + // This function drains buffer until next eoe has been received. + // It will be a no-op if the previous row returned is empty. + // @return Status - The error code return + Status Drain(); + + private: + DatasetOp *current_op_; // The parent operator. We consume from it's children. + int32_t child_idx_; // The specific child this iterator will fetch from. + int32_t worker_id_; // The worker id uses for fetching the child data. + bool end_epoch_; // the flag used when an empty row has been returned. +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASET_ITERATOR_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/datasetops/CMakeLists.txt new file mode 100644 index 00000000000..d23d6bccb8e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/CMakeLists.txt @@ -0,0 +1,17 @@ +add_subdirectory(source) + +add_library(engine-datasetops OBJECT + dataset_op.cc + parallel_op.cc + pipeline_op.cc + batch_op.cc + batch_op.cc + device_queue_op.cc + map_op.cc + project_op.cc + rename_op.cc + repeat_op.cc + shuffle_op.cc + zip_op.cc + ) + diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc new file mode 100644 index 00000000000..7c5d4bd4c89 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc @@ -0,0 +1,303 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/batch_op.h" +#include +#include "common/utils.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" + +namespace mindspore { +namespace dataset { +BatchOp::Builder::Builder(int32_t batch_size) : builder_drop_(false) { + builder_batch_size_ = batch_size; + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_num_workers_ = cfg->num_parallel_workers(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status BatchOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(builder_batch_size_, builder_drop_, builder_op_connector_size_, builder_num_workers_, + builder_cols_to_map_, builder_batch_size_func_, builder_batch_map_func_); + return Status::OK(); +} + +Status BatchOp::Builder::SanityCheck() { + std::string err; + err += builder_op_connector_size_ <= 0 ? "connector size <= 0\n" : ""; + err += builder_batch_size_ <= 0 ? "batch size <= 0\n" : ""; + err += builder_num_workers_ <= 0 ? "batch num_parallel_workers <= 0\n" : ""; + return err.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, common::SafeCStr(err)); +} + +BatchOp::BatchOp(int32_t batch_size, bool drop, int32_t op_queue_size, int32_t num_workers, + const std::vector &cols_to_map, py::function batch_size_func, py::function batch_map_func) + : ParallelOp(num_workers, op_queue_size), + start_batch_size_(batch_size), + drop_(drop), + input_column_names_(cols_to_map), + batch_size_func_(batch_size_func), + batch_map_func_(batch_map_func) { + worker_queues_.Init(num_workers, op_queue_size); +} + +Status BatchOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + TaskManager::FindMe()->Post(); + int32_t epoch_num = 0, batch_num = 0, cnt = 0; + TensorRow new_row; + std::unique_ptr table = make_unique(); + child_iterator_ = mindspore::make_unique(this, 0, 0); + RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); + column_name_map_ = child_iterator_->col_name_id_map(); + int32_t cur_batch_size = 0; + RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(0, 0, 0))); + while (child_iterator_->eof_handled() == false) { + while (new_row.empty() == false) { + table->emplace_back(new_row); + // if # of rows is enough to make 1 batch (1 batch is buffer), send it to worker_queue + if (table->size() == static_cast(cur_batch_size)) { + RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( + std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); + table = make_unique(); + RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num))); + } + RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); + } + // Reminder logic, execute only when there is a remainder (table is non empty) and don't drop + if (drop_ == false && table->empty() == false) { + RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack( + std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num)))); + } + table = make_unique(); // this drops when drop == true + // end of the current epoch, batch_num should start from 0 again + batch_num = 0; + epoch_num++; + RETURN_IF_NOT_OK( + worker_queues_[cnt++ % num_workers_]->EmplaceBack(std::make_pair(nullptr, CBatchInfo(batchCtrl::kEOE)))); + RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num))); + RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); + } // end of eof_handled() == false + RETURN_IF_NOT_OK( + worker_queues_[cnt++ % num_workers_]->EmplaceBack(std::make_pair(nullptr, CBatchInfo(batchCtrl::kEOF)))); + // EOF received, send quit signal (an empty buffer) to all workers + for (int32_t ind = 0; ind < num_workers_; ind++) { + RETURN_IF_NOT_OK( + worker_queues_[cnt++ % num_workers_]->EmplaceBack(std::make_pair(nullptr, CBatchInfo(batchCtrl::kQuit)))); + } + return Status::OK(); +} + +void BatchOp::Print(std::ostream &out, bool show_all) const { + ParallelOp::Print(out, show_all); + out << "\nBatchOp:\n" + << "number of parallel workers: " << num_workers_ << "\nBatch size: " << start_batch_size_ + << "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n"; +} + +Status BatchOp::BatchRows(const std::unique_ptr *source_table, + const std::unique_ptr *dest_table, size_t batch_size) { + if ((*source_table)->size() < batch_size || (*source_table)->size() == 0) { + RETURN_STATUS_UNEXPECTED("[Internal Batch ERROR] Insufficient rows in source_table\n"); + } + TensorRow row = std::move((*source_table)->front()); + (*source_table)->pop_front(); + if (batch_size == 1) { + for (std::shared_ptr tensor : row) { + RETURN_IF_NOT_OK(tensor->ExpandDim(0)); + } + (*dest_table)->push_back(row); + } else { // batch_size > 1 + std::vector row_shapes; + TensorRow batched_row; + for (size_t i = 0; i < row.size(); i++) { // Handle the first row popped + row_shapes.push_back(row[i]->shape()); + std::shared_ptr ts; + RETURN_IF_NOT_OK(Tensor::CreateTensor( + &ts, TensorImpl::kFlexible, row[i]->shape().PrependDim(static_cast(batch_size)), row[i]->type())); + batched_row.emplace_back(ts); + RETURN_IF_NOT_OK(batched_row[i]->InsertTensor(std::vector(1, 0), row[i])); // {j} = 0 + } + for (size_t j = 1; j < batch_size; j++) { // Handle the rest of the rows + row = std::move((*source_table)->front()); + (*source_table)->pop_front(); + for (size_t i = 0; i < row.size(); i++) { + if (row[i]->shape() == row_shapes[i]) { // check the newly popped rows have the same dim as the first + RETURN_IF_NOT_OK(batched_row[i]->InsertTensor(std::vector(1, j), row[i])); + } else { + RETURN_STATUS_UNEXPECTED("[Batch ERROR] Inconsistent TensorShapes\n"); + } + } + } + (*dest_table)->emplace_back(batched_row); + } + return Status::OK(); +} + +Status BatchOp::WorkerEntry(int32_t workerId) { + TaskManager::FindMe()->Post(); + std::pair, CBatchInfo> table_pair; + RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair)); + while (table_pair.second.ctrl_ != batchCtrl::kQuit) { + if (table_pair.second.ctrl_ == batchCtrl::kEOE) { + RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique(0, DataBuffer::kDeBFlagEOE))); + } else if (table_pair.second.ctrl_ == batchCtrl::kEOF) { + RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique(0, DataBuffer::kDeBFlagEOF))); + } else if (table_pair.second.ctrl_ == batchCtrl::kNoCtrl) { + std::unique_ptr db = nullptr; + RETURN_IF_NOT_OK(MakeBatchedBuffer(std::move(table_pair), &db)); + RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::move(db))); + } + RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair)); + } + return Status::OK(); +} + +Status BatchOp::MakeBatchedBuffer(std::pair, CBatchInfo> table_pair, + std::unique_ptr *db) { + RETURN_UNEXPECTED_IF_NULL(table_pair.first); + if (!input_column_names_.empty()) RETURN_IF_NOT_OK(MapColumns(&table_pair)); // pass it through pyfunc + (*db) = make_unique(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone); + std::unique_ptr dest_table = make_unique(); + RETURN_IF_NOT_OK(BatchRows(&table_pair.first, &dest_table, table_pair.first->size())); + (*db)->set_tensor_table(std::move(dest_table)); + (*db)->set_column_name_map(column_name_map_); + return Status::OK(); +} + +Status BatchOp::LaunchThreadsAndInitOp() { + RETURN_UNEXPECTED_IF_NULL(tree_); + RETURN_IF_NOT_OK(worker_queues_.Register(tree_->AllTasks())); + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&BatchOp::WorkerEntry, this, std::placeholders::_1))); + return Status::OK(); +} + +Status BatchOp::EofReceived(int32_t) { return Status::OK(); } + +Status BatchOp::EoeReceived(int32_t) { + state_ = OpState::kDeOpIdle; + return Status::OK(); +} + +Status BatchOp::MapColumns(std::pair, CBatchInfo> *table_pair) { + TensorBatchTable input_table; + input_table.reserve(input_column_names_.size()); + for (std::string col_name : input_column_names_) { + if (column_name_map_.find(col_name) == column_name_map_.end()) { + RETURN_STATUS_UNEXPECTED("column : '" + col_name + "' does not exist\n"); + } + TensorBatch tensor_batch; + tensor_batch.reserve(table_pair->first->size()); + size_t col_idx = static_cast(column_name_map_[col_name]); + for (size_t row_idx = 0; row_idx < table_pair->first->size(); row_idx++) { + tensor_batch.push_back(std::move(table_pair->first->at(row_idx)[col_idx])); + } + input_table.push_back(std::move(tensor_batch)); + } + + // Perform batch map + TensorBatchTable output_table; + RETURN_IF_NOT_OK(InvokeBatchMapFunc(&input_table, &output_table, table_pair->second)); + + // Write back to TensorQTable + for (size_t input_idx = 0; input_idx < input_column_names_.size(); input_idx++) { + size_t col_idx = static_cast(column_name_map_[input_column_names_[input_idx]]); + size_t row_id = 0; + for (TensorRow &row : *(table_pair->first)) { + row[col_idx] = std::move(output_table[input_idx][row_id++]); + } + } + return Status::OK(); +} + +Status BatchOp::GetBatchSize(int32_t *batch_size, CBatchInfo info) { + if (batch_size_func_ != nullptr) { + RETURN_IF_NOT_OK(InvokeBatchSizeFunc(batch_size, info)); + } else { + (*batch_size) = start_batch_size_; + } + return Status::OK(); +} + +Status BatchOp::InvokeBatchSizeFunc(int32_t *batch_size, CBatchInfo info) { + { + // Acquire Python GIL + py::gil_scoped_acquire gil_acquire; + if (Py_IsInitialized() == 0) { + return Status(StatusCode::kPythonInterpreterFailure, "Python Interpreter is finalized"); + } + try { + py::object size = batch_size_func_(info); + *batch_size = size.cast(); + if (*batch_size <= 0) { + return Status(StatusCode::kPyFuncException, "Batch size function should return an integer > 0"); + } + } catch (const py::error_already_set &e) { + return Status(StatusCode::kPyFuncException, e.what()); + } catch (const py::cast_error &e) { + return Status(StatusCode::kPyFuncException, "Batch size function should return an integer > 0"); + } + } + return Status(StatusCode::kOK, "Batch size func call succeed"); +} + +Status BatchOp::InvokeBatchMapFunc(TensorBatchTable *input, TensorBatchTable *output, CBatchInfo info) { + { + // Acquire Python GIL + py::gil_scoped_acquire gil_acquire; + if (Py_IsInitialized() == 0) { + return Status(StatusCode::kPythonInterpreterFailure, "Python Interpreter is finalized"); + } + try { + // Prepare batch map call back parameters + py::tuple input_args(input->size() + 1); + for (size_t i = 0; i < input->size(); i++) { + std::vector np_batch; + for (std::shared_ptr t : input->at(i)) { + py::array np_array; + RETURN_IF_NOT_OK(t->GetDataAsNumpy(&np_array)); + np_batch.push_back(std::move(np_array)); + } + input_args[i] = np_batch; + } + input_args[input->size()] = info; + // Invoke batch map func + py::object ret_py_obj = batch_map_func_(*input_args); + // Parse batch map return value + py::tuple ret_tuple = py::cast(ret_py_obj); + if (ret_tuple.size() != input_column_names_.size() || !py::isinstance(ret_tuple)) { + return Status(StatusCode::kPyFuncException, "Batch map function should return an tuple if size(input_columns)"); + } + for (size_t i = 0; i < ret_tuple.size(); i++) { + TensorBatch output_batch; + py::list output_list = py::cast(ret_tuple[i]); + for (size_t j = 0; j < output_list.size(); j++) { + std::shared_ptr out; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&out, py::cast(output_list[j]))); + output_batch.push_back(std::move(out)); + } + output->push_back(std::move(output_batch)); + } + } catch (const py::error_already_set &e) { + return Status(StatusCode::kPyFuncException, e.what()); + } catch (const py::cast_error &e) { + return Status(StatusCode::kPyFuncException, "Batch map function should return an tuple of list of numpy array"); + } + } + return Status(StatusCode::kOK); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.h b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.h new file mode 100644 index 00000000000..9037b8e94e6 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.h @@ -0,0 +1,240 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_BATCH_OP_H_ +#define DATASET_ENGINE_DATASETOPS_BATCH_OP_H_ + +#include +#include +#include +#include +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class DataBuffer; + +using TensorBatch = std::vector>; +using TensorBatchTable = std::vector; + +class BatchOp : public ParallelOp { + public: + class Builder { + public: + // Builder constructor for Batch, batch size needs to be specified + // @param int32_t batch_size + explicit Builder(int32_t batch_size); + + // Builder constructor for Batch, batch size function needs to be specified + // @param py::function batch_size_func + explicit Builder(py::function batch_size_func); + + // Default destructor + ~Builder() = default; + + // set number of parallel Workers on batch + // @param int32_t num_workers + // @return Builder & reference to builder class object + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // set drop for batch op,default false + // @param bool drop + // @return Builder & reference to builder class object + Builder &SetDrop(bool drop) { + builder_drop_ = drop; + return *this; + } + + // set connector size for batch + // @param int32_t op_conn_size + // @return Builder & reference to builder class object + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = (op_connector_size == 0 ? builder_op_connector_size_ : op_connector_size); + return *this; + } + + // set columns to perform map on + // @param const std::vector & cols_to_map - name of columns to perform map on + // @return Builder & reference to builder class object + Builder &SetColumnsToMap(const std::vector &cols_to_map) { + builder_cols_to_map_ = cols_to_map; + return *this; + } + + // set columns to perform map on + // @param const std::vector & cols_to_map - name of columns to perform map on + // @return Builder & reference to builder class object + Builder &SetBatchMapFunc(py::function batch_map_func) { + builder_batch_map_func_ = batch_map_func; + return *this; + } + + // SetBatchSizeFunc, a function that calls to python after every batch is made + // @param py::function batch_size_func - python function to call, GIL required before calling + // @return Builder & reference to builder class object + Builder &SetBatchSizeFunc(py::function batch_size_func) { + builder_batch_size_func_ = batch_size_func; + return *this; + } + + // @param std::shared_ptr *ptr pointer to shared_ptr, actual return arg + // @return Status - The error code return + Status Build(std::shared_ptr *); + + private: + // Sanity check for builder class args + // @return Status - The error code return + Status SanityCheck(); + + bool builder_drop_; + int32_t builder_batch_size_; + int32_t builder_num_workers_; + int32_t builder_op_connector_size_; + std::vector builder_cols_to_map_; + + py::function builder_batch_size_func_; + py::function builder_batch_map_func_; + }; + + enum batchCtrl : int8_t { kNoCtrl = 0, kEOE = 1, kEOF = 2, kQuit = 3 }; + + // Parameters associate with one batch. + // This struct is used for both internal control and python callback. + // This struct is bound to python with read-only access. + struct CBatchInfo { + CBatchInfo(int32_t ep, int32_t bat, int32_t cur, batchCtrl ctrl) + : epoch_num_(ep), batch_num_(bat), total_batch_num_(cur), ctrl_(ctrl) {} + CBatchInfo(int32_t ep, int32_t bat, int32_t cur) : CBatchInfo(ep, bat, cur, batchCtrl::kNoCtrl) {} + CBatchInfo() : CBatchInfo(0, 0, 0, batchCtrl::kNoCtrl) {} + explicit CBatchInfo(batchCtrl ctrl) : CBatchInfo(0, 0, 0, ctrl) {} + int32_t epoch_num_; // i-th epoch. i starts from 0 + int32_t batch_num_; // i-th batch since the start of current epoch. i starts from 0 + int32_t total_batch_num_; // i-th batch since the start of first epoch. i starts from 0 + batchCtrl ctrl_; // No control=0, EOE=1, EOF=2, Quit=3 + const int32_t get_batch_num() const { return batch_num_; } + const int32_t get_epoch_num() const { return epoch_num_; } + }; + + // BatchOp constructor + // @param int32_t batch_size + // @param bool drop + // @param int32_t op_queue_size + // @param int32_t rows_per_buf + // @param int32_t num_workers + BatchOp(int32_t batch_size, bool drop, int32_t op_queue_size, int32_t num_workers, const std::vector &, + py::function batch_size_func, py::function batch_map_func); + + // BatchOp destructor + ~BatchOp() {} + + // @param int32_t workerId + // @return Status - The error code return + Status EofReceived(int32_t) override; + + // @param int32_t workerId + // @return Status - The error code return + Status EoeReceived(int32_t) override; + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param sO - reference to the BatchOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const BatchOp &bo) { + bo.Print(out, false); + return out; + } + + // Main loop of batch + // @return Status - The error code return + Status operator()() override; + + private: + // Worker thread for doing the memcpy of batch + // @param int32_t param workerId + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Generate buffer with batched tensors + // @return Status - The error code return + Status MakeBatchedBuffer(std::pair, CBatchInfo> table_pair, + std::unique_ptr *db); + + // batch the rows in src table then put it to dest table + // @param const std::unique_ptr *src - table that has the rows for batching + // @param const std::unique_ptr *dest - dest_table to hold batched rows + // @param int32_t size - batch_size + // @return Status - The error code return + Status BatchRows(const std::unique_ptr *src, const std::unique_ptr *dest, size_t size); + + // Function that calls pyfunc to perform map on batch + // @param (std::pair, batch_stats> *table_pair - contains un-batched tensor + // @return Status - The error code return + Status MapColumns(std::pair, CBatchInfo> *table_pair); + + // the number of thread pulling from the mOutConnector of the Op below + // @return int32_t, 1 + int32_t num_consumers() const override { return 1; } + + // get the batch size for next batch + // @return Status - The error code return + Status GetBatchSize(int32_t *batch_size, CBatchInfo info); + + // Do the initialization of all queues then start all worker threads + // @return Status - The error code return + Status LaunchThreadsAndInitOp(); + + // Invoke batch size function with current BatchInfo to generate batch size. + // @return Status - The error code return + Status InvokeBatchSizeFunc(int32_t *batch_size, CBatchInfo info); + + // Invoke batch map function with current BatchInfo to generate tensors to batch. + // @return Status - The error code return + Status InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBatchInfo info); + + int32_t start_batch_size_; + bool drop_; + // Name of the columns to perform map op on + std::vector input_column_names_; + // Iterator for fetching + std::unique_ptr child_iterator_; + // Map of column_name: column_index + std::unordered_map column_name_map_; + // Internal queue for task distribution + QueueList, CBatchInfo>> worker_queues_; + // Function pointer of batch size function + py::function batch_size_func_; + // Function pointer of per batch map function + py::function batch_map_func_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_BATCH_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc new file mode 100644 index 00000000000..d3b85b84fb5 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -0,0 +1,202 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/dataset_op.h" + +#include +#include +#include +#include +#include + +#include "dataset/engine/execution_tree.h" +#include "dataset/engine/datasetops/device_queue_op.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +// Constructor +DatasetOp::DatasetOp(int32_t op_connector_size) + : oc_queue_size_(op_connector_size), + operator_id_(kInvalidOperatorId), + tree_(nullptr), + state_(OpState::kDeOpIdle), + op_ctrl_flags_(kDeOpNone) { + // The operator starts out with an invalid operator id. The only way to + // get it out of invalid state is to assign the operator to an execution tree. +} + +// Adds a operator to become our child. +Status DatasetOp::AddChild(std::shared_ptr child) { + if (std::dynamic_pointer_cast(child) != nullptr) { + std::string err_msg("DeviceQueueOp cannot be added as a child, DeviceQueueOp must be a root node"); + RETURN_STATUS_UNEXPECTED(err_msg); + } + if (operator_id_ == kInvalidOperatorId) { + std::string err_msg( + "Cannot add child node. Tree node connections can only" + "be made if the node belongs to a tree."); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // disallow relationships with other trees + if (tree_ != child->tree_) { + std::string err_msg( + "Cannot add child node. Tree node connections can only be made if both nodes belong to the same tree."); + RETURN_STATUS_UNEXPECTED(err_msg); + } + child_.push_back(child); + child->AddParent(this); + return Status::OK(); +} + +// Adds a parent operator to this operator +void DatasetOp::AddParent(const DatasetOp *parent) { parent_.push_back(parent); } + +// Getter function to get a shared pointer to our childAdds a operator to become our child. +std::shared_ptr DatasetOp::child(int32_t child_index) const { + DS_ASSERT(child_index < static_cast(child_.size())); + // Return a shared pointer + return child_[child_index]; +} + +// Creates the connector within this operator +void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) { + MS_LOG(INFO) << "Creating connector in tree operator: " << operator_id_ << ". Producer: " << num_producers + << ". Consumer: " << num_consumers << "."; + if (oc_queue_size_ > 0) { + out_connector_ = mindspore::make_unique(num_producers, // The number of producers + num_consumers, // Only one consumer (the training App) + oc_queue_size_); + } else { + // Some op's may choose not to have an output connector + MS_LOG(INFO) << "Bypassed connector creation for tree operator: " << operator_id_ << "."; + out_connector_ = nullptr; + } +} + +// A print method typically used for debugging. showAll of true will recursively descend to child prints +void DatasetOp::Print(std::ostream &out, bool show_all) const { + if (show_all) { + for (size_t i = 0; i < child_.size(); i++) { + child_[i]->Print(out, show_all); + } + } + out << "\n-------------------------" + << "\nOperator # : " << operator_id_ << "\nNumber of children : " << child_.size() + << "\nNumber of parents : " << parent_.size() << "\nConnector queue size : " << oc_queue_size_ + << "\nOperator control flags : 0x" << std::hex << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec + << std::setfill(' ') << "\nHas parents:\n"; + for (size_t i = 0; i < parent_.size(); i++) { + out << "Parent[" << i << "] id: " << parent_[i]->id() << "\n"; + } +} + +// Gets the next buffer from the given child +Status DatasetOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) { + std::unique_ptr next_buff; + // pop is a blocked call and will throw an interruption if the whole group shuts down. + RETURN_IF_NOT_OK(out_connector_->PopWithRetry(static_cast(worker_id), &next_buff, retry_if_eoe)); + + *p_buffer = std::move(next_buff); + return Status::OK(); +} + +// Gets the next buffer from the given child . This function also has built-in eoe and eof +// message handling so that child classes don't have to manually code pass-through logic when +// those messages are received. +Status DatasetOp::GetNextInput(std::unique_ptr *p_buffer, int32_t worker_id, int32_t child_index) { + if (child_.size() == 0) { + return this->GetNextBuffer(p_buffer, worker_id); + } + CHECK_FAIL_RETURN_UNEXPECTED(child_index < child_.size(), "Child index too big : " + std::to_string(child_index)); + std::shared_ptr child = child_[child_index]; + std::unique_ptr buf; + RETURN_IF_NOT_OK(child->GetNextBuffer(&buf, worker_id)); + // Loop until non EOE is received + while (buf->eoe()) { + RETURN_IF_NOT_OK(EoeReceived(worker_id)); + if (state_ == OpState::kDeOpIdle) { + *p_buffer = std::move(buf); + return Status::OK(); + } + RETURN_IF_NOT_OK(child->GetNextBuffer(&buf, worker_id)); + } + // Check if the last buf is next eof + if (buf->eof()) { + RETURN_IF_NOT_OK(EofReceived(worker_id)); + } + *p_buffer = std::move(buf); + return Status::OK(); +} + +// Performs handling for when an eoe message is received. +// The base class implementation simply flows the eoe message to output. Derived classes +// may override if they need to perform special eoe handling. +Status DatasetOp::EoeReceived(int32_t worker_id) { + std::unique_ptr eoe_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + return (out_connector_->Add(static_cast(worker_id), std::move(eoe_buffer))); +} + +// Performs handling for when an eof message is received. +// The base class implementation simply flows the eof message to output. Derived classes +// may override if they need to perform special eof handling. +Status DatasetOp::EofReceived(int32_t worker_id) { + std::unique_ptr eof_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOF); + return (out_connector_->Add(static_cast(worker_id), std::move(eof_buffer))); +} + +// During tree prepare phase, operators may have specific operations to perform depending on +// their role. +Status DatasetOp::PrepareNodeAction() { + // If this op does not have any children and it is in a repeat path of the tree... + if (child_.size() == 0 && BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepRepeat)) { + // Then, flag this operator as a leaf node in a repeat path of tree execution. + BitSet(&op_ctrl_flags_, kDeOpRepeated); + + // Secondly, push ourselves onto the tree repeat stack. Later, the repeat operator + // above us will consume them. + tree_->AddToRepeatStack(shared_from_this()); + } + + // Creating Connector object for each op. + // The consumer of the root node is assumed to be one thread. + // If multiple threads are consuming from the root node, they will get the ordered data in round robin fashion. + if (parent_.empty()) { + this->CreateConnector(num_producers(), 1); + } else { + this->CreateConnector(num_producers(), parent_[0]->num_consumers()); + } + if (out_connector_) { + RETURN_IF_NOT_OK(out_connector_->Register(tree_->AllTasks())); + } + RETURN_IF_NOT_OK(this->RegisterWorkerConnectors()); + return Status::OK(); +} + +// Getter function. Base class does not have any special flags setting. +uint32_t DatasetOp::PrepareFlags() const { return ExecutionTree::kDePrepNone; } + +// Derived classes may implement the reset function if the operator is stateful and needs +// specific reset handling that is not contained in this common code version of the reset. +Status DatasetOp::Reset() { + state_ = OpState::kDeOpRunning; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h new file mode 100644 index 00000000000..a7d87c30922 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -0,0 +1,220 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_DATASET_OP_H_ +#define DATASET_ENGINE_DATASETOPS_DATASET_OP_H_ + +#include +#include +#include "dataset/core/constants.h" +#include "dataset/engine/db_connector.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Forward declare +class ExecutionTree; + +class DataBuffer; + +// The base class DatasetOp is the main tree node. It is an abstract class, so +// the actual implementation of the operators will be derived from here. +class DatasetOp : public std::enable_shared_from_this { + // Allow execution tree to access internal members + friend class ExecutionTree; + + public: + static constexpr int32_t kInvalidOperatorId = -1; + + // Flags that control operator runtime behaviours + enum OpControlFlags { + kDeOpNone = 0, + kDeOpRepeated = 1, // Operator is a leaf node in a repeat path + kDeOpLastRepeat = 1 << 1 // We are in the last repeat loop + }; + + // Flags that control operator runtime behaviours + enum OpState { kDeOpRunning = 0, kDeOpIdle = 1 }; + + // Constructor + // @param op_connector_size - The size for the output connector of this operator. + explicit DatasetOp(int32_t op_connector_size); + + // Destructor + virtual ~DatasetOp() { tree_ = nullptr; } + + // Adds a operator to become our child. + // @param child - shared pointer to the child to add. + Status AddChild(std::shared_ptr child); + + // Getter function to get a shared pointer to our childAdds a operator to become our child. + // @param child_index - An operator can have n children. Indicates choose which child to return. + std::shared_ptr child(int32_t child_index) const; + + // Creates the connector within this operator + // @param num_producers - number of threads that write into this connector + // @param num_consumers - number of threads that read from this connector + void CreateConnector(int32_t num_producers, int32_t num_consumers); + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + virtual void Print(std::ostream &out, bool show_all) const; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param dO - reference to the DatasetOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const DatasetOp &dO) { + dO.Print(out, false); + return out; + } + + // Class functor operator (). + // DatasetOps operate by launching a thread (see ExecutionTree). + // This pure virtual version makes the requirement that derived classes must provide a functor + // that will execute their main runtime loop code. + // @return Status - The error code return + virtual Status operator()() = 0; + + // Gets the next buffer from the given child + // @notes See GetNextInput for similar function that has built-in message handling + // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) + // @param worker_id - The worker id + // @return Status - The error code return + virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id) { + return GetNextBuffer(p_buffer, worker_id, false); + } + + // Gets the next buffer from the given child + // @notes See GetNextInput for similar function that has built-in message handling + // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) + // @return Status - The error code return + virtual Status GetNextBuffer(std::unique_ptr *p_buffer) { return GetNextBuffer(p_buffer, 0, false); } + + // Gets the next buffer from the given child + // @notes See GetNextInput for similar function that has built-in message handling + // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) + // @param worker_id - The worker id + // @param retry_if_eoe Set this flag to true to allow calling pop() again after the first pop() returns EOE. + // @return Status - The error code return + virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe); + + // Gets the next buffer from the given child . This function also has built-in eoe and eof + // message handling so that child classes don't have to manually code pass-through logic when + // those messages are received. + // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) + // @param worker_id - The worker id + // @return Status - The error code return + Status GetNextInput(std::unique_ptr *p_buffer, int32_t worker_id = 0, int32_t child_index = 0); + + // Performs handling for when an eoe message is received. + // The base class implementation simply flows the eoe message to output. Derived classes + // may override if they need to perform special eoe handling. + // @param worker_id - The worker id + // @return Status - The error code return + virtual Status EoeReceived(int32_t worker_id); + + // Performs handling for when an eof message is received. + // The base class implementation simply flows the eof message to output. Derived classes + // may override if they need to perform special eof handling. + // @param worker_id - The worker id + // @return Status - The error code return + virtual Status EofReceived(int32_t worker_id); + + // Derived classes may implement the reset function if the operator is stateful and needs + // specific reset handling that is not contained in this common code version of the reset + // @return Status - The error code return + virtual Status Reset(); + + // This calls the reset function on this subtree in pre-order + // @return Status - The error code return + virtual Status ResetSubtree() { + RETURN_IF_NOT_OK(Reset()); + for (const auto &c : child_) { + RETURN_IF_NOT_OK(c->ResetSubtree()); + } + return Status::OK(); + } + + // During tree prepare phase, operators may have specific operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + virtual Status PrepareNodeAction(); + + // Getter function + // @return The operator id + int32_t id() const { return operator_id_; } + + // Getter function + // @return The prepare flags + virtual uint32_t PrepareFlags() const; + + // Getter function + // @return The number of workers in this op + virtual int32_t num_workers() const = 0; + + // Getter function + // @return The number of threads consuming from previous op. + virtual int32_t num_consumers() const = 0; + + // Getter function + // @return The number of threads producing to the output connector. + virtual int32_t num_producers() const = 0; + + // Getter function + // @return T/F if this is an inlined operator + bool inlined() const { return (oc_queue_size_ == 0); } + + // Setter function + // @return Sets the control flags + void set_control_flag(uint64_t flag) { BitSet(&op_ctrl_flags_, flag); } + + // Register the internal worker connectors. No op unless it is a parallel op + // @return Status + virtual Status RegisterWorkerConnectors() { return Status::OK(); } + + protected: + // Adds a parent operator to this operator + // @notes External callers do not have access to this function. + // @param parent - The parent node to add + void AddParent(const DatasetOp *parent); + + std::vector> child_; // Child nodes + std::vector parent_; // Parent nodes. No ownership and read-only + int32_t oc_queue_size_; // Capacity for each out_connector_ + int32_t operator_id_; // Generated id for the node + ExecutionTree *tree_; // Back pointer to our tree. + OpState state_; // The state of the operator, Running, Idle, Terminated + uint32_t op_ctrl_flags_; // Flags for the operator + std::unique_ptr out_connector_; // Output Connector + + private: + // Sets the operator id. + // @notes No public interface. Only the class itself, or it's friend the execution tree can set + // this + // @param op_id - the Id value to set into the operator + void set_id(int32_t op_id) { operator_id_ = op_id; } + + // Sets the tree into the op so that the operator has a back pointer to the tree. + // @param tree - the tree to assign to the op. + void set_tree(ExecutionTree *tree) { tree_ = tree; } +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_DATASET_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc new file mode 100644 index 00000000000..3c2eea16ee2 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc @@ -0,0 +1,254 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/device_queue_op.h" + +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/util/status.h" +#include "dataset/util/task_manager.h" + +#ifdef ENABLE_TDTQUE +#include "tdt/tsd_client.h" +#endif + +namespace mindspore { +namespace dataset { +DeviceQueueOp::DeviceQueueOp(std::string channel_name, DeviceType device_type, int32_t device_id, int32_t prefetch_size, + int32_t op_connector_size, int64_t num_batch) + : PipelineOp(op_connector_size), + channel_name_(channel_name), + device_type_(device_type), + device_id_(device_id), + prefetch_size_(prefetch_size), + num_batch_(num_batch) {} + +DeviceQueueOp::~DeviceQueueOp() {} + +#ifdef ENABLE_GPUQUE +void ReleaseData(void *addr) { + if (addr != nullptr) { + free(addr); + } +} +#endif + +DeviceQueueOp::Builder::Builder(int32_t prefetch_size) + : builder_prefetch_size_(prefetch_size), + builder_device_id_(0), + builder_device_type_(DeviceType::CPU), + builder_channel_name_(""), + builder_num_batch_(0) { + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status DeviceQueueOp::EoeReceived(int32_t worker_id) { + state_ = OpState::kDeOpIdle; + return Status::OK(); +} + +Status DeviceQueueOp::operator()() { + TaskManager::FindMe()->Post(); + + if (device_type_ == DeviceType::Ascend) { +#ifdef ENABLE_TDTQUE + RETURN_IF_NOT_OK(SendDataToAscend()); +#endif + } else if (device_type_ == DeviceType::GPU) { +#ifdef ENABLE_GPUQUE + RETURN_IF_NOT_OK(SendDataToGPU()); +#endif + } else if (device_type_ == DeviceType::CPU) { + RETURN_IF_NOT_OK(SendDataToCPU()); + } + + return Status::OK(); +} + +Status DeviceQueueOp::CheckExceptions(const std::unique_ptr &buffer) const { + // this method checks if the buffer meets the conditions to be sent to TDT + return Status::OK(); +} + +#ifdef ENABLE_TDTQUE +Status DeviceQueueOp::SendDataToAscend() { + MS_LOG(INFO) << "Device queue, sending data to Ascend."; + int64_t total_batch = 0; + bool is_break_loop = false; + + std::unique_ptr current_buffer; + RETURN_IF_NOT_OK(GetNextInput(¤t_buffer)); + + while (!current_buffer->eof() && !is_break_loop) { + while (!current_buffer->eoe() && !is_break_loop) { + RETURN_IF_NOT_OK(CheckExceptions(current_buffer)); + TensorRow currRow; + for (int row_id = 0; row_id < current_buffer->NumRows() && !is_break_loop; row_id++) { + RETURN_IF_NOT_OK(current_buffer->GetRow(row_id, &currRow)); + auto status = tdtInstancePtr->hostPush(currRow, true, channel_name_); + if (status == TdtStatus::FAILED) { + return Status(StatusCode::kTDTPushFailure, "TDT Push Failed"); + } + total_batch++; + if (num_batch_ > 0 && total_batch == num_batch_) { + is_break_loop = true; + } + } + RETURN_IF_NOT_OK(GetNextInput(¤t_buffer)); + } + RETURN_IF_NOT_OK(GetNextInput(¤t_buffer)); + } + + MS_LOG(INFO) << "Device queue total batch is " << total_batch << ", number of batches is " << num_batch_ << "."; + + return Status::OK(); +} +#endif + +#ifdef ENABLE_GPUQUE +Status DeviceQueueOp::SendDataToGPU() { + MS_LOG(INFO) << "Device queue, sending data to GPU."; + int64_t total_batch = 0; + bool is_break_loop = false; + bool is_open = false; + uint32_t handle = INVALID_HANDLE; + + std::unique_ptr current_buffer; + RETURN_IF_NOT_OK(GetNextInput(¤t_buffer)); + + while (!current_buffer->eof() && !is_break_loop && !GpuBufferMgr::GetInstance().IsClosed()) { + while (!current_buffer->eoe() && !is_break_loop && !GpuBufferMgr::GetInstance().IsClosed()) { + RETURN_IF_NOT_OK(CheckExceptions(current_buffer)); + TensorRow curr_row; // batch data + for (int row_id = 0; + row_id < current_buffer->NumRows() && !is_break_loop && !GpuBufferMgr::GetInstance().IsClosed(); row_id++) { + RETURN_IF_NOT_OK(current_buffer->GetRow(row_id, &curr_row)); + if (curr_row.size() < 2) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Invalid tensor size"); + } + uint32_t feature_size = static_cast(curr_row[0]->SizeInBytes()); + uint32_t label_size = static_cast(curr_row[1]->SizeInBytes()); + if (!is_open) { + handle = GpuBufferMgr::GetInstance().Open(0, channel_name_, feature_size, label_size, ReleaseData); + if (handle == INVALID_HANDLE) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "open failed"); + } + is_open = true; + } + RETURN_IF_NOT_OK(RetryPushGPUData(feature_size, label_size, curr_row, handle)); + total_batch++; + if (num_batch_ > 0 && total_batch == num_batch_) { + is_break_loop = true; + } + } + RETURN_IF_NOT_OK(GetNextInput(¤t_buffer)); + } + RETURN_IF_NOT_OK(GetNextInput(¤t_buffer)); + } + + MS_LOG(INFO) << "Device queue total batch is " << total_batch << ", number of batches is " << num_batch_ << "."; + + GpuBufferMgr::GetInstance().Close(handle); + + GpuBufferMgr::GetInstance().CloseConfirm(); + + return Status::OK(); +} + +Status DeviceQueueOp::RetryPushGPUData(uint32_t feature_size, uint32_t label_size, const TensorRow &curr_row, + uint32_t handle) { + unsigned char *feature_addr = nullptr; + unsigned char *label_addr = nullptr; + while (true && !GpuBufferMgr::GetInstance().IsClosed()) { + RETURN_IF_NOT_OK(MallocForGPUData(&feature_addr, feature_size, &label_addr, label_size, curr_row)); + auto ret = GpuBufferMgr::GetInstance().Push(handle, feature_addr, feature_size, label_addr, label_size, WAIT_TIME); + if (ret) { + free(feature_addr); + free(label_addr); + MS_LOG(WARNING) << "Retry pushing data..."; + continue; + } else { + break; + } + } + return Status::OK(); +} + +Status DeviceQueueOp::MallocForGPUData(unsigned char **feature_addr, uint32_t feature_size, unsigned char **label_addr, + uint32_t label_size, const TensorRow &curr_row) { + *feature_addr = (unsigned char *)malloc(feature_size); + if (*feature_addr == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "feature memory malloc failed."); + } + (void)memset_s(*feature_addr, feature_size, 0, feature_size); + unsigned char *feature = curr_row[0]->StartAddr(); + if (memcpy_s(*feature_addr, feature_size, feature, static_cast(curr_row[0]->SizeInBytes())) != 0) { + MS_LOG(ERROR) << "Feature memcpy_s failed!"; + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "feature memcpy_s failed."); + } + + *label_addr = (unsigned char *)malloc(label_size); + if (*label_addr == nullptr) { + free(*feature_addr); + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "label memory malloc failed."); + } + (void)memset_s(*label_addr, label_size, 0, label_size); + unsigned char *label = curr_row[1]->StartAddr(); + if (memcpy_s(*label_addr, label_size, label, static_cast(curr_row[1]->SizeInBytes())) != 0) { + MS_LOG(ERROR) << "Label memcpy_s failed!"; + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "label memcpy_s failed."); + } + + return Status::OK(); +} +#endif + +Status DeviceQueueOp::SendDataToCPU() { + MS_LOG(INFO) << "Device queue, sending data to CPU."; + int64_t total_batch = 0; + + std::unique_ptr child_iterator = mindspore::make_unique(this, 0, 0); + while (!(child_iterator->eof_handled())) { + TensorRow curr_row; + RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&curr_row)); + + if (!curr_row.empty()) { + MS_LOG(DEBUG) << "Feature size is " << curr_row[0]->SizeInBytes() << "."; + MS_LOG(DEBUG) << "Label size is " << curr_row[1]->SizeInBytes() << "."; + total_batch++; + if (num_batch_ > 0 && total_batch == num_batch_) { + break; + } + } + } + + MS_LOG(INFO) << "Device queue total batch is " << total_batch << ", number of batches is " << num_batch_ << "."; + + return Status::OK(); +} + +void DeviceQueueOp::Print(std::ostream &out, bool show_all) const { + PipelineOp::Print(out, show_all); + + out << "DeviceQueueOp: channelName: " << channel_name_ << ", prefetchSize: " << prefetch_size_ << '\n'; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.h b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.h new file mode 100644 index 00000000000..11ff1bde95a --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.h @@ -0,0 +1,166 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_DEVICE_QUEUE_OP_H_ +#define DATASET_ENGINE_DATASETOPS_DEVICE_QUEUE_OP_H_ + +#include +#include +#include + +#include "dataset/engine/datasetops/pipeline_op.h" +#include "dataset/util/status.h" + +#ifdef ENABLE_TDTQUE +#include "dataset/engine/tdt/tdt_plugin.h" + +#endif + +#ifdef ENABLE_GPUQUE +#include "device/gpu/gpu_buffer_mgr.h" +using mindspore::device::GpuBufferMgr; +#endif + +namespace mindspore { +namespace dataset { +class DeviceQueueOp : public PipelineOp { + public: + static const uint32_t INVALID_HANDLE = 0xffffffffUL; + static const uint32_t WAIT_TIME = 5; + + enum class DeviceType { Ascend = 0, GPU = 1, CPU = 2 }; + + // The nested builder class inside of the DeviceQueueOp is used to help manage all of + // the arguments for constructing it. Use the builder by setting each argument + // with the provided set methods, and then finally call the build method to execute + // the actual construction. + class Builder { + public: + explicit Builder(int32_t prefetch_size); + + // Default destructor + ~Builder() = default; + + Builder &SetPrefetchSize(int32_t prefetch_size) { + builder_prefetch_size_ = prefetch_size; + return *this; + } + + Builder &SetChannelName(const std::string &channel_name) { + builder_channel_name_ = channel_name; + return *this; + } + + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = op_connector_size; + return *this; + } + + Builder &SetDeviceType(const std::string &device_type) { + if (device_type == "Ascend") { + builder_device_type_ = DeviceType::Ascend; + } else if (device_type == "GPU") { + builder_device_type_ = DeviceType::GPU; + } else if (device_type == "CPU") { + builder_device_type_ = DeviceType::CPU; + } + return *this; + } + + Builder &SetDeviceId(int32_t device_id) { + builder_device_id_ = device_id; + return *this; + } + + Builder &SetNumBatch(int64_t num_batch) { + builder_num_batch_ = num_batch; + return *this; + } + + // Name: Build() + // Description: The final step for building a DeviceQueueOp via the Builder is + // to call this Build() method. It will instantiate the DeviceQueueOp + // and return it to caller as a shared pointer. + Status Build(std::shared_ptr *ptr) { + *ptr = std::make_shared(builder_channel_name_, builder_device_type_, builder_device_id_, + builder_prefetch_size_, builder_op_connector_size_, builder_num_batch_); + return Status::OK(); + } + + private: + int32_t builder_prefetch_size_; + int32_t builder_device_id_; + DeviceType builder_device_type_; + std::string builder_channel_name_; + int64_t builder_num_batch_; + int32_t builder_op_connector_size_; + }; + + // Name: constructor + // Description + DeviceQueueOp(std::string channel_name, DeviceType device_type, int32_t device_id, int32_t prefetch_size, + int32_t op_connector_size, int64_t num_batch); + + // Name: destructor + // Description + ~DeviceQueueOp(); + + Status EoeReceived(int32_t worker_id) override; + + const int32_t get_prefetch_size() { return prefetch_size_; } + + // Name: Print() + // Description: A function that prints info about the node + void Print(std::ostream &out, // In: The output stream to print to + bool show_all) const override; // In: T/F if it should print everything + + // Provide stream operator for displaying it + friend std::ostream &operator<<(std::ostream &out, const DeviceQueueOp &to) { + to.Print(out, false); + return out; + } + + Status operator()() override; + + private: + // Name: checkExceptions(DataBuffer); + // Description: Check whether the dataBuffer meets the condition for performing DeviceQueueOp + Status CheckExceptions(const std::unique_ptr &buffer) const; + +#ifdef ENABLE_TDTQUE + Status SendDataToAscend(); +#endif + +#ifdef ENABLE_GPUQUE + Status SendDataToGPU(); + Status RetryPushGPUData(uint32_t feature_size, uint32_t label_size, const TensorRow &curr_row, uint32_t handle); + Status MallocForGPUData(unsigned char **feature_addr, uint32_t feature_size, unsigned char **label_addr, + uint32_t label_size, const TensorRow &curr_row); +#endif + + Status SendDataToCPU(); + std::string channel_name_; + DeviceType device_type_; + const int32_t device_id_; + const int32_t prefetch_size_; + const int64_t num_batch_; + +#ifdef ENABLE_TDTQUE + std::shared_ptr tdtInstancePtr; +#endif +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_DEVICE_QUEUE_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc new file mode 100644 index 00000000000..c9f1f98ae06 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -0,0 +1,363 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/map_op.h" +#include +#include +#include +#include +#include "dataset/core/config_manager.h" + +#include "dataset/core/constants.h" +#include "dataset/core/global_context.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/kernels/tensor_op.h" +#include "utils/log_adapter.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +// Builder constructor. Creates the builder object. +MapOp::Builder::Builder() : build_perf_mode_(true) { + std::shared_ptr cfg = GlobalContext::config_manager(); + build_num_workers_ = cfg->num_parallel_workers(); + build_op_connector_size_ = cfg->op_connector_size(); +} + +// Check if the required parameters are set by the builder. +Status MapOp::Builder::sanityCheck() const { + if (build_tensor_funcs_.empty()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Building a MapOp that has not provided any function/operation to apply"); + } + return Status::OK(); +} + +// The builder "build" method creates the final object. +Status MapOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(sanityCheck()); + *ptr = std::make_shared(std::move(build_in_col_names_), std::move(build_out_col_names_), + std::move(build_tensor_funcs_), build_num_workers_, build_op_connector_size_, + build_perf_mode_); + return Status::OK(); +} + +// Constructor of MapOp +MapOp::MapOp(const std::vector &in_col_names, const std::vector &out_col_names, + std::vector> tensor_funcs, int32_t num_workers, int32_t op_connector_size, + bool perf_mode) + : ParallelOp(num_workers, op_connector_size), + tfuncs_(std::move(tensor_funcs)), + in_columns_(in_col_names), + out_columns_(out_col_names), + perf_mode_(perf_mode) { + // If caller didn't specify the out_col_names, assume they are same as the in_columns. + if (out_columns_.empty() || out_columns_[0].empty()) { + out_columns_ = in_columns_; + } + MS_LOG(DEBUG) << "Performance Mode in map operator is " << perf_mode_ << "."; +} + +// The number of threads consuming data from previous op's output Connector. +int32_t MapOp::num_consumers() const { + // When Performance Mode is on, there is only one thread consuming from the previous Connector. + return perf_mode_ == true ? 1 : num_workers_; +} + +// A print method typically used for debugging +void MapOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + ParallelOp::Print(out, show_all); + + // Then display our own stuff + out << "\nMapOp:"; + out << "\n Input column names:"; + for (size_t i = 0; i < in_columns_.size(); i++) { + out << " " << in_columns_[i]; + } + out << "\n TensorOps:"; + for (size_t i = 0; i < tfuncs_.size(); i++) { + out << " " << tfuncs_[i]; + } + out << "\n"; +} + +// This class functor will provide the master loop that drives the logic for performing the work +Status MapOp::operator()() { + if (perf_mode_) { + // Create and register the local queues. + local_queues_.Init(num_workers_, oc_queue_size_); + RETURN_IF_NOT_OK(local_queues_.Register(tree_->AllTasks())); + } + + // The operator class just starts off threads by calling the tree_ function + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&MapOp::WorkerEntry, this, std::placeholders::_1))); + // Synchronize with TaskManager + TaskManager::FindMe()->Post(); + + if (perf_mode_) { + int64_t que_id = 0; + std::unique_ptr buff; + bool is_eof = false; + // Draining output connector of the previous op and distribute it to local queues. + // Stop when all worker threads are finished (received EOF). + while (!is_eof) { + RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buff, 0)); + is_eof = buff->eof(); + RETURN_IF_NOT_OK(local_queues_[que_id]->Add(std::move(buff))); + que_id = (que_id + 1) % num_workers_; + } + } + + return Status::OK(); +} + +// Private function for worker/thread to loop continuously. It comprises the main +// logic of MapOp: getting the data from previous Op, validating user specified column names, +// applying a list of TensorOps to each of the data, process the results and then +// pushing them back to MapOp's output Connector to be fetched by the next Op. +Status MapOp::WorkerEntry(int32_t worker_id) { + // Handshake with TaskManager that thread creation is successful. + TaskManager::FindMe()->Post(); + + // MapOp is not using the ChildIterator class to fetch rows because it needs to track + // rows at a per-buffer level. ChildIterator abstracts the concept of buffers making it + // less convenient to use that interface for fetching. + std::unique_ptr in_buffer; + + // Loop until eof buffer is encountered + while (true) { + // Getting a databuffer to work on. + // When PerformanceMode is enabled, workers pop from the local queue. + // Otherwise, workers pop from the first child output Connector. + if (perf_mode_) { + RETURN_IF_NOT_OK(local_queues_[worker_id]->PopFront(&in_buffer)); + } else { + RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&in_buffer, worker_id)); + } + + // Handle EOE and EOF ourselves. Implicit eoe/eof handling in GetNextInput does not work + // with Performance Mode design. + if (in_buffer->eoe()) { + // Calling base class EoeReceived to forward eoe buffer. + RETURN_IF_NOT_OK(EoeReceived(worker_id)); + continue; + } else if (in_buffer->eof()) { + // Calling base class EofReceived to forward eof buffer. + RETURN_IF_NOT_OK(EofReceived(worker_id)); + break; + } + + // Boolean mapping, true means to keep the column. + std::vector keep_input_columns; + // Indices of the columns to process. + std::vector to_process_indices; + // The final column mapping after performing this map + std::unordered_map final_col_name_id_map; + + // Thread local variables to avoid lock. When in_columns_ is empty and workers will write + // the name of the first column into input_columns (thread local) instead of in_columns_ (thread global). + std::vector input_columns = in_columns_; + std::vector output_columns = out_columns_; + + // Initialize the above data structures + RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get(), &keep_input_columns, &to_process_indices, &final_col_name_id_map, + &input_columns, &output_columns)); + + std::unique_ptr new_tensor_table(mindspore::make_unique()); + // Perform the compute function of TensorOp(s) and store the result in new_tensor_table. + RETURN_IF_NOT_OK(WorkerCompute(in_buffer.get(), to_process_indices, new_tensor_table.get(), keep_input_columns, + &input_columns, &output_columns)); + + // Update column name to index mapping because tensorOp might add/remove column. + in_buffer->set_column_name_map(final_col_name_id_map); + // Replace the TensorTable in DataBuffer with the new one. + in_buffer->set_tensor_table(std::move(new_tensor_table)); + + // Push the buffer onto the connector for next operator to consume. + RETURN_IF_NOT_OK(out_connector_->Add(static_cast(worker_id), std::move(in_buffer))); + } + + return Status::OK(); +} + +Status MapOp::WorkerCompute(DataBuffer *in_buffer, const std::vector &to_process_indices, + TensorQTable *new_tensor_table, const std::vector &keep_input_columns, + std::vector *input_columns, std::vector *output_columns) { + // Getting number of rows and cols in this buffer. + int32_t num_rows = in_buffer->NumRows(); + int32_t num_cols = in_buffer->NumCols(); + + for (int32_t r = 0; r < num_rows; r++) { + // to_process : A vector of Tensors only holding cols in input_columns. + // result_row; : A vector of Tensors to hold the result after Compute(). + // cur_row : A vector of Tensors holding all the columns from DataBuffer. + TensorRow to_process, result_row, cur_row; + RETURN_IF_NOT_OK(in_buffer->PopRow(&cur_row)); + + // Populate the Tensor from the current row to be processed by TensorOp + for (const auto &idx : to_process_indices) { + to_process.push_back(std::move(cur_row[idx])); + } + + // Looping over multiple TensorOps supplied in to MapOp. + // The assumption is that the result of one TensorOp matches the required input to the next TensorOp. + for (size_t i = 0; i < tfuncs_.size(); i++) { + // TensorOp can operate on single col or multiple cols. MapOp always call compute for multiple cols. + // TensorOp base class will call the single column Compute() depending on the ops. + // Note: The columns of the result_row is not preallocated, the compute function of each tensor op are + // required to resize/push back the result_row + RETURN_IF_NOT_OK(tfuncs_[i]->Compute(to_process, &result_row)); + + // Assign result_row to to_process for the next TensorOp processing, except for the last TensorOp in the list. + if (i + 1 < tfuncs_.size()) { + to_process = std::move(result_row); + } + } + + if (output_columns->size() != result_row.size()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Result of a tensorOp doesn't match output column names"); + } + + if (input_columns->size() == output_columns->size()) { + for (size_t i = 0; i < result_row.size(); i++) { + cur_row[to_process_indices[i]] = std::move(result_row[i]); + } + new_tensor_table->push_back(std::move(cur_row)); + } else { + // Add the columns we did not touch to the result_row. + for (int32_t i = 0; i < num_cols; i++) { + if (keep_input_columns[i]) { + result_row.push_back(std::move(cur_row[i])); + } + } + + // Add this final result_row to our new TensorTable. + new_tensor_table->push_back(std::move(result_row)); + } + } + + return Status::OK(); +} + +// Validating if each of the input_columns exists in the DataBuffer. +Status MapOp::ValidateInColumns(const std::unordered_map &col_name_id_map, + std::vector *input_columns) { + for (const auto &inCol : *input_columns) { + bool found = col_name_id_map.find(inCol) != col_name_id_map.end() ? true : false; + if (!found) { + std::string err_msg = "input column name: " + inCol + " doesn't exist in the dataset columns."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + return Status::OK(); +} + +// initialize some internal data structure used by WorkerEntry() +Status MapOp::WorkerEntryInit(const DataBuffer *in_buf, std::vector *keep_input_columns, + std::vector *to_process_indices, + std::unordered_map *final_col_name_id_map, + std::vector *input_columns, std::vector *output_columns) { + int32_t num_rows = in_buf->NumRows(); + int32_t num_cols = in_buf->NumCols(); + if (num_rows == 0 || num_cols == 0) { + RETURN_STATUS_UNEXPECTED("MapOp is getting an empty DataBuffer."); + } + std::unordered_map col_name_id_map = in_buf->column_name_map(); + // Check if there is invalid column name in the inColumns. + RETURN_IF_NOT_OK(ValidateInColumns(col_name_id_map, input_columns)); + + // If input_columns is empty(), The col at index-0 will be picked. + if (input_columns->empty()) { + for (const auto &pair : col_name_id_map) { + if (pair.second == 0) { + MS_LOG(INFO) << "Input columns in map operator is empty, will apply to the first column in the current table."; + input_columns->push_back(pair.first); + break; + } + } + + // If caller didn't specify the out_col_names, assume they are same as the input_columns. + if (output_columns->empty() || (*output_columns)[0].empty()) { + *output_columns = *input_columns; + } + } + + // initialize keep_input_columns, true means to keep the column. + keep_input_columns->resize(num_cols, true); + for (const auto &col_name : *input_columns) { + int32_t missed = col_name_id_map[col_name]; + (*keep_input_columns)[missed] = false; + } + + // initialize to_process_indices. + for (const auto &col_name : *input_columns) { + to_process_indices->push_back(col_name_id_map[col_name]); + } + + // Create the final column name to index mapping. + *final_col_name_id_map = CreateFinalColMap(&col_name_id_map, *keep_input_columns, input_columns, output_columns); + + return Status::OK(); +} + +// Create the final column name to index mapping and get indices of the columns this mapop does not use. +std::unordered_map MapOp::CreateFinalColMap( + std::unordered_map *col_name_id_map, const std::vector &keep_input_columns, + std::vector *input_columns, std::vector *output_columns) { + std::unordered_map final_col_name_id_map; + size_t num_cols = col_name_id_map->size(); + std::vector new_ids(num_cols); + if (input_columns->size() == output_columns->size()) { + for (size_t i = 0; i < input_columns->size(); i++) { + int32_t loc = (*col_name_id_map)[(*input_columns)[i]]; + (void)col_name_id_map->erase((*input_columns)[i]); + (*col_name_id_map)[(*output_columns)[i]] = loc; + } + + return *col_name_id_map; + } else { + int32_t fill_idx = 0; + // First columns of the tables are occupied by the output columns from tensorOp. + for (const auto &col_name : *output_columns) { + final_col_name_id_map[col_name] = fill_idx++; + } + + // Creating new_ids mapping for the columns we keep. + for (size_t i = 0; i < num_cols; i++) { + if (keep_input_columns[i]) { + new_ids[i] = fill_idx++; + } + } + + // Iterating through the old mapping to update the final mapping for the columns we kept. + std::string name; + for (const auto &pair : *col_name_id_map) { + name = pair.first; + int32_t old_id = pair.second; + if (keep_input_columns[old_id]) { + final_col_name_id_map[name] = new_ids[old_id]; + } + } + + return final_col_name_id_map; + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h new file mode 100644 index 00000000000..4c9d27f9c75 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h @@ -0,0 +1,253 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_MAP_OP_H_ +#define DATASET_ENGINE_DATASETOPS_MAP_OP_H_ + +#include +#include +#include +#include +#include +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/queue.h" + +namespace mindspore { +namespace dataset { +// Forward declare +class DataBuffer; +class ExecutionTree; + +// MapOp class implements the Map operator. It will apply a list of operations to each record specified by column names. +// The column order behavior after MapOp is as follows. +// [Case 1] If the number of Input Columns == the number of Output Column, column ordering after MapOp +// is the same as the original column order where the Remainder Columns stay in the same position, +// and the Output Columns are placed the same position of the Input Columns. +// For example, initially if the dataset has column order |A, B, C, D, E|, +// and we apply MapOp() with Input Columns {B, C} and Output Columns {X, Y}. +// The column order after applying MapOp will be |A, X, Y, D, E|. +// Note that in this case, |X, Y| is the Output Columns and |A, D, E| which is the Remainder Columns stay in +// their original position, and column B is replaced by column X and column C is replace by column Y. +// [Case 2] If the number of Input Columns != the number of Output Column, column ordering after MapOp +// is Output Columns followed by Remainder Columns. +// For example, initially if the dataset has column order |A, B, C, D, E|, +// and we apply MapOp() with Input Columns {B, C, A} and Output Columns {X, Y}. +// The column order after applying MapOp will be |X, Y, D, E|. +// Note that in this case, |X, Y| is the Output Columns and |D, E| is the Remainder Columns, +// and the Input Columns are gone and replaced by the Output Columns. + +// Keywords: +// Input Columns : a vector of column names (string) passed to MapOp specifying the column names from which +// Tensors are taken and passed to the TensorOp Compute(). +// Output Columns : a vector of column names (string) passed to MapOp specifying what are the column names +// for the Tensors produced by TensorOp Compute(). +// Remainder Columns : columns that exist in the dataset but are not mentioned in Input Columns. +// These columns will not be passed to TensorOp Compute(), but will be appended to the end of the Output Columns. +class MapOp : public ParallelOp { + public: + // The nested builder class inside of the MapOp is used to help manage all of + // the arguments for constructing it. Use the builder by setting each argument + // with the provided set methods, and then finally call the build method to execute + // the actual construction. + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + // Default destructor + ~Builder() = default; + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetInColNames(const std::vector &in_col_names) { + build_in_col_names_ = in_col_names; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOutColNames(const std::vector &out_col_names) { + build_out_col_names_ = out_col_names; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetTensorFuncs(std::vector> funcs) { + build_tensor_funcs_ = std::move(funcs); + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + build_num_workers_ = num_workers; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t connector_size) { + build_op_connector_size_ = connector_size; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetPerformanceMode(bool perf_mode) { + build_perf_mode_ = perf_mode; + return *this; + } + + // The builder "build" method creates the final object. + // @param ptr The shared_ptr to the new MapOp object + // @return Status + Status Build(std::shared_ptr *ptr); + + private: + std::vector build_in_col_names_; + std::vector build_out_col_names_; + std::vector> build_tensor_funcs_; + int32_t build_num_workers_; + int32_t build_op_connector_size_; + bool build_perf_mode_; // Default true. + + // Check if the required parameters are set by the builder. + // @return Status The error code return + Status sanityCheck() const; + }; + + // Constructor of MapOp + // @note The builder class should be used to call it. + // @param in_col_names A list of input column names (should match the input/output \p tensorFuncs). + // @param out_col_names A list of output column names (should match the input/output \p tensorFuncs). + // @param tensor_funcs A list of TensorOp pointers for MapOp to apply to each data. + // @param num_workers The number of worker threads. + // @param op_connector_size The size of each queue in the connector. + MapOp(const std::vector &in_col_names, const std::vector &out_col_names, + std::vector> tensor_funcs, int32_t num_workers, int32_t op_connector_size, + bool perf_mode); + + // Destructor + ~MapOp() = default; + + // A print method typically used for debugging + // @param out The output stream to write output to + // @param show_all A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out reference to the output stream being overloaded + // @param mo reference to the MapOp to display + // @return the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const MapOp &mo) { + mo.Print(out, false); + return out; + } + + // Class functor operator () override. + // All dataset ops operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work + // @return Status The error code return + Status operator()() override; + + // Getter + // @return the number of threads consuming data from previous op's output Connector. + int32_t num_consumers() const override; + + private: + // Local queues where worker threads can pop from. + // Popping directly from the Connector can block if the previous designated threads haven't pop. + // Setting the size of these queues to 0 is essentially the same as pulling directly from Connector. + QueueList> local_queues_; + + // Static variables to be ready by worker threads, no modification and readonly + const std::vector> tfuncs_; + + // Variable to store the column name that the tensorOps are consuming + std::vector in_columns_; + + // Variable to store the column name that the tensorOps are producing + std::vector out_columns_; + + // Performance mode is when the main thread creates local queues, pulls databuffers from the previous + // op's Connector and distributes them to the local queues. Workers pull from the local queues. + // If this flag is false, each worker pulls directly from the Connector. This use less resources + // (thread and memory), but when the computation cost is heavy (e.g. DecodeOp) and fluctuating, it can + // cause additional blocking because pop calls to Connector from the threads are synchronized to enforce the order. + bool perf_mode_; + + // Private function for worker/thread to loop continuously. It comprises the main + // logic of MapOp: getting the data from previous Op, validating user specified column names, + // applying a list of TensorOps to each of the data, process the results and then + // pushing them back to MapOp's output Connector to be fetched by the next Op. + // @param worker_id The id assigned to this thread/worker upon creation. + // @return Status The error code return + Status WorkerEntry(int32_t worker_id) override; // In: workerId assigned by tree_ + + // Private function for worker thread to perform TensorOp's compute function and get the result. + // @param in_buffer A raw pointer to the DataBuffer. A raw pointer is fine because this function doesn't manage memory + // and is not shared with other threads. + // @param to_process_indices Indices of columns to be processed by the TensorOp. + // @param[out] new_tensor_table A new Tensor Table to be populated in this function. + // @param keep_input_columns Keeping track of which columns to keep (not used by TensorOp). + // @param input_columns The vector of input column names used in the current thread. + // @param output_columns The vector of output column names used in the current thread. + Status WorkerCompute(DataBuffer *in_buffer, const std::vector &to_process_indices, + TensorQTable *new_tensor_table, const std::vector &keep_input_columns, + std::vector *input_columns, std::vector *output_columns); + + // Private function for validating if each of the user specified input column names + // exist in the DataBuffer. + // @param col_name_id_map The column name to index mapping obtained from DataBuffer. + // @param input_columns The vector of input column names used in the current thread. + // @return Status The error code return + Status ValidateInColumns(const std::unordered_map &col_name_id_map, + std::vector *input_columns); + + // Private function that create the final column name to index mapping and + // get indices of the columns this mapop does not use. + // @param col_name_id_map The column name to index mapping obtained from DataBuffer. + // @param keep_input_columns To mark which columns are to be kept (not used in mapOp). + // @param input_columns The vector of input column names used in the current thread. + // @param output_columns The vector of output column names used in the current thread. + // @return finalColNameIdMap The final column name to index mapping. + std::unordered_map CreateFinalColMap(std::unordered_map *col_name_id_map, + const std::vector &keep_input_columns, + std::vector *input_columns, + std::vector *output_columns); + + // Private function that initialize some internal data structure used by WorkerEntry() + // @param in_buf A raw pointer to the DataBuffer. A raw pointer is fine because this function does not manage memory + // and is not shared with other threads. + // @param[out] keep_input_columns Keeping track of which columns to keep (not used by TensorOp) + // @param[out] to_process_indices Indices of columns to be processed by the TensorOp + // @param[out] final_col_name_id_map Create the final column name id map. This final mapping will replace the old one + // if the TensorOp Compute() is successful. + // @param input_columns The vector of input column names used in the current thread. + // @param output_columns The vector of output column names used in the current thread. + Status WorkerEntryInit(const DataBuffer *in_buf, std::vector *keep_input_columns, + std::vector *to_process_indices, + std::unordered_map *final_col_name_id_map, + std::vector *input_columns, std::vector *output_columns); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_MAP_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc new file mode 100644 index 00000000000..d9792312a30 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/parallel_op.h" + +#include +#include +#include +#include +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/core/config_manager.h" +#include "dataset/engine/db_connector.h" + +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +// Constructor +ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size) + : DatasetOp(op_connector_size), + num_workers_(num_workers), + num_producers_(num_workers), + worker_connector_size_(1), + worker_connector_(nullptr) {} + +// Creates the internal worker connector for the parallel op if the derived class wants to use it +Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) { + if (worker_connector_size == 0) { + RETURN_STATUS_UNEXPECTED("Worker connector size 0 is invalid."); + } + num_producers_ = 1; + worker_connector_size_ = worker_connector_size; + // Instantiate the worker connector. This is the internal connector, not the operators + // output connector. It has single master consuming from it (num producers is 1), and the number + // of workers is the defined count from the op. + worker_connector_ = mindspore::make_unique(num_workers_, num_producers_, worker_connector_size); + + return Status::OK(); +} + +// A print method typically used for debugging +void ParallelOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + DatasetOp::Print(out, show_all); + + // Then show our own stuff + out << "ParallelOp:"; + out << "\n Num workers : " << num_workers_ << "\n"; +} + +// Override base class reset to provide reset actions specific to the ParallelOp class. +Status ParallelOp::Reset() { + RETURN_IF_NOT_OK(DatasetOp::Reset()); // Perform any super class reset work + + // ParallelOp is abstract, but we do own the connector between workers and master + // (if the parallel op is configured for this). Reset that connector here. + if (worker_connector_) { + worker_connector_->Reset(); + } + + return Status::OK(); +} + +// Register the internal worker connectors +Status ParallelOp::RegisterWorkerConnectors() { + if (worker_connector_) { + return (worker_connector_->Register(tree_->AllTasks())); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h new file mode 100644 index 00000000000..ceb7f2c4acf --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h @@ -0,0 +1,115 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_PARALLEL_OP_H_ +#define DATASET_ENGINE_DATASETOPS_PARALLEL_OP_H_ + +#include +#include +#include "dataset/core/constants.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// global const in our namespace +constexpr int32_t kEndOfActions = -1; + +// Forward declares +class DataBuffer; + +class DbConnector; + +// A ParallelOp provides a multi-threaded DatasetOp +class ParallelOp : public DatasetOp { + public: + // Constructor + // @param num_workers + // @param op_connector_size - size of the output connector for this operator + ParallelOp(int32_t num_workers, int32_t op_connector_size); + + // Destructor + ~ParallelOp() = default; + + // Creates the internal worker connector for the parallel op if the derived class wants to use it. + // @notes This changes the number of producers of this op to 1, since it establishes a master/worker + // relationship within the op, making all production flow through a single master. + // @return Status - The error return code + Status CreateWorkerConnector(int32_t worker_connector_size); + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param pO - reference to the ParallelOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const ParallelOp &po) { + po.Print(out, false); + return out; + } + + // During tree prepare phase, operators may have specific operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + // @return Status - The error return code + Status PrepareNodeAction() override { + // Run common code from super class before adding ParallelOp specific logic + return (DatasetOp::PrepareNodeAction()); + } + + // Override base class reset to provide reset actions specific to the ParallelOp class. + // @return Status - The error code return + Status Reset() override; + + // Getter + // @return the number of workers + int32_t num_workers() const override { return num_workers_; } + + // Getter + // @return the number of threads consuming from the previous Connector + int32_t num_consumers() const override { return num_workers_; } + + // Getter + // @return the number of producers pushing to the output Connector + // @notes The number of producers is commonly the same as number of workers, except in the case + // when a worker connector is set up. In that case, there are n workers, and a single master + // such that only 1 thread is a producer rather than the n workers. + // @return the number of producers + int32_t num_producers() const override { return num_producers_; } + + // Register the internal worker connectors. + // @return Status + Status RegisterWorkerConnectors() override; + + protected: + // Interface for derived classes to implement. All derived classes must provide the entry + // function with the main execution loop for worker threads. + // @return Status - The error code return + virtual Status WorkerEntry(int32_t workerId) = 0; + + int32_t num_workers_; // The number of worker threads + int32_t num_producers_; // The number of threads pushing to the out_connector_ + int32_t worker_connector_size_; + std::unique_ptr worker_connector_; // The internal connector for worker threads +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_PARALLEL_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc new file mode 100644 index 00000000000..56fc24883a3 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "dataset/engine/datasetops/pipeline_op.h" + +namespace mindspore { +namespace dataset { +// Constructor +PipelineOp::PipelineOp(int32_t op_connector_size) : DatasetOp(op_connector_size) {} + +// A print method typically used for debugging +void PipelineOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + DatasetOp::Print(out, show_all); + + // Then display our own stuff for the pipeline op + // out << "This is a pipeline op print. nothing to display here at the moment.\n"; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h new file mode 100644 index 00000000000..ee20f1d3735 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h @@ -0,0 +1,88 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_PIPELINE_OP_H_ +#define DATASET_ENGINE_DATASETOPS_PIPELINE_OP_H_ + +#include +#include +#include "dataset/engine/datasetops/dataset_op.h" + +namespace mindspore { +namespace dataset { +// forward declare +class ExecutionTree; + +class DataBuffer; + +class PipelineOp : public DatasetOp { + public: + // Constructor + // @param op_connector_size - size of the output connector + // @return Builder setter method returns reference to the builder. + explicit PipelineOp(int32_t op_connector_size); + + // Destructor + ~PipelineOp() = default; + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param po - reference to the PipelineOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const PipelineOp &po) { + po.Print(out, false); + return out; + } + + // Getter + // @return The number of workers inside this op. Pipeline ops only have a single worker. + int32_t num_workers() const override { return 1; } + + // Getter + // @return the number of threads consuming from the previous Connector + int32_t num_consumers() const override { return 1; } + + // Getter + // @return The number of threads that push data to the output connector + int32_t num_producers() const override { return 1; } + + // During tree prepare phase, operators may have specific operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + Status PrepareNodeAction() override { + // Run common code from super class before adding PipelineOp specific logic + return (DatasetOp::PrepareNodeAction()); + } + + protected: + // ******************************************************************************* + // I'm predicting there will be common arguments or functionality for pipeline ops, + // just not sure yet what those are. perhaps this intermediate class between + // DatasetOp and the actual ops is not needed at all? + // For example, if there's no common code for all of the non-parallel ops, then + // they can just inherit from DatasetOp directly and we can put this class into the + // trash. +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_PIPELINE_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc new file mode 100644 index 00000000000..11296f84f42 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc @@ -0,0 +1,131 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/engine/datasetops/project_op.h" +#include +#include +#include +#include +#include +#include +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +ProjectOp::Builder::Builder(const std::vector &columns_to_project) + : builder_columns_to_project_(columns_to_project) {} + +Status ProjectOp::Builder::SanityCheck() const { + if (builder_columns_to_project_.empty()) { + std::string err_msg("Columns to project is empty."); + RETURN_STATUS_UNEXPECTED(err_msg); + } + return Status::OK(); +} + +Status ProjectOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(builder_columns_to_project_); + return Status::OK(); +} + +ProjectOp::ProjectOp(const std::vector &columns_to_project) + : PipelineOp(0), columns_to_project_(columns_to_project) {} + +void ProjectOp::Print(std::ostream &out, bool show_all) const { + PipelineOp::Print(out, show_all); + out << "ProjectOp: columns that are projected: "; + for (size_t i = 0; i < columns_to_project_.size(); i++) { + out << columns_to_project_[i] << " "; + } + out << '\n'; +} + +// Gets a buffer from the child operator and projects the buffer. +Status ProjectOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) { + RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(p_buffer, worker_id, retry_if_eoe)); + if (!((*p_buffer)->eoe()) && !((*p_buffer)->eof())) { + RETURN_IF_NOT_OK(Project(p_buffer)); + } + return Status::OK(); +} + +Status ProjectOp::Project(std::unique_ptr *data_buffer) { + std::unordered_map column_name_mapping = (*data_buffer)->column_name_map(); + std::unordered_map new_column_name_mapping; + std::vector projected_column_indices; + for (size_t i = 0; i < columns_to_project_.size(); i++) { + std::string ¤t_column = columns_to_project_[i]; + if (column_name_mapping.find(current_column) == column_name_mapping.end()) { + std::string err_msg = "ProjectOp: column " + current_column + " does not exist in this buffer."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + new_column_name_mapping[current_column] = i; + projected_column_indices.push_back(column_name_mapping[current_column]); + } + std::unique_ptr new_tensor_table = mindspore::make_unique(); + while ((*data_buffer)->NumRows() > 0) { + TensorRow current_row; + RETURN_IF_NOT_OK((*data_buffer)->PopRow(¤t_row)); + TensorRow new_row; + (void)std::transform(projected_column_indices.begin(), projected_column_indices.end(), std::back_inserter(new_row), + [¤t_row](uint32_t x) { return current_row[x]; }); + new_tensor_table->push_back(new_row); + } + (*data_buffer)->set_tensor_table(std::move(new_tensor_table)); + (*data_buffer)->set_column_name_map(new_column_name_mapping); + return Status::OK(); +} + +// Class functor operator () override. +// Most dataset ops operate by launching a thread (see ExecutionTree). +// However, the ProjectOp is defined as a inlined operator, so it is invalid to launch the +// functor since this op runs inlined inside another operator. The function is overloaded to +// ensure that it is not called by mistake (it will generate an error). +Status ProjectOp::operator()() { RETURN_STATUS_UNEXPECTED("Logic error. ProjectOp is an inlined operator."); } + +int32_t ProjectOp::num_consumers() const { + if (parent_.empty()) { + MS_LOG(INFO) << "Project operator, no parent node, assuming it's the root and returning 1."; + return 1; + } else if (parent_[0] == nullptr) { + MS_LOG(INFO) << "Project operator, pointer to the first parent is null. Returning 0."; + return 0; + } else { + return parent_[0]->num_consumers(); + } +} + +int32_t ProjectOp::num_producers() const { + if (child_.empty() || child_[0] == nullptr) { + MS_LOG(INFO) << "Project operator, pointer to child node is null. Returning 0."; + return 0; + } else { + return child_[0]->num_producers(); + } +} + +Status ProjectOp::EoeReceived(int32_t worker_id) { + state_ = OpState::kDeOpIdle; + return Status::OK(); +} + +Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.h b/mindspore/ccsrc/dataset/engine/datasetops/project_op.h new file mode 100644 index 00000000000..09d022da86d --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.h @@ -0,0 +1,112 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_PROJECT_OP_H_ +#define DATASET_ENGINE_DATASETOPS_PROJECT_OP_H_ + +#include +#include +#include + +#include "dataset/engine/datasetops/pipeline_op.h" + +namespace mindspore { +namespace dataset { +class ProjectOp : public PipelineOp { + public: + // The nested builder class inside of the ProjectOp is used to help manage all of the arguments + // for constructing it. This repeat op is very simple though, so this builder is really just + // provided for a consistent look and feel for creators of Dataset operators overall. + class Builder { + public: + // Builder constructor. Creates the builder object. + // @param columns_to_project - + // @return This is a constructor. + explicit Builder(const std::vector &columns_to_project); + + // Builder destructor. + ~Builder() = default; + + // The builder "build" method creates the final object. + // @return shared_ptr to the new StorageOp object. + Status Build(std::shared_ptr *); + + private: + std::vector builder_columns_to_project_; + Status SanityCheck() const; + }; + + // Constructor of the ProjectOp. + // @param columnsToProject - + explicit ProjectOp(const std::vector &columns_to_project); + + // Destructor. + ~ProjectOp() = default; + + // A print method typically used for debugging. + // @param out - The output stream to write output to. + // @param show_all - A bool to control if you want to show all info or just a summary. + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload. + // @notes This allows you to write the debug print info using stream operators. + // @param out - reference to the output stream being overloaded. + // @param project_op - reference to the ProjectOp to display. + // @return - the output stream must be returned. + friend std::ostream &operator<<(std::ostream &out, const ProjectOp &project_op) { + project_op.Print(out, false); + return out; + } + + // Class functor operator () override. + // Most dataset ops operate by launching a thread (see ExecutionTree). + // However, the ProjectOp is defined as a inlined operator, so it is invalid to launch the + // functor since this op runs inlined inside another operator. The function is overloaded to + // ensure that it is not called by mistake (it will generate an error). + // @return Status - The error code returned. + Status operator()() override; + + // Gets a buffer from the child node and projects that buffer. The caller is typically our parent node. + // @param p_buffer - output pointer to the projected buffer. + // @param worker_id - The worker id + Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) override; + + // Base-class override. Return the number of workers in the first parent. + // @param workerId - The worker id + int32_t num_consumers() const override; + + // Base-class override. Return the number of producers in the first child. + // @param workerId - The worker id + int32_t num_producers() const override; + + // Base-class override for special eoe handler. + // Inline operators must override this because there is no connector to push eoe onto. + // @return Status - The error code returned. + Status EoeReceived(int32_t worker_id) override; + + // Base-class override for special eof handler. + // Inline operators must override this because there is no connector to push eof onto. + // @return Status - The error code returned. + Status EofReceived(int32_t worker_id) override; + + private: + std::vector columns_to_project_; + + Status Project(std::unique_ptr *data_buffer); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_PROJECT_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc new file mode 100644 index 00000000000..c09f56141e6 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc @@ -0,0 +1,159 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/rename_op.h" + +#include +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/constants.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +// builds +RenameOp::Builder::Builder() { + // Some arguments to the RenameOp constructor have a default argument that is taken + // from the client config. + // The user may choose to change these values for the construction of the RenameOp by + // using the various builder set methods. + + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status RenameOp::Builder::SanityCheck() const { return Status::OK(); } + +// build method for RenameOp +Status RenameOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(builder_in_columns_, builder_out_columns_, builder_op_connector_size_); + return Status::OK(); +} + +// constructor +RenameOp::RenameOp(const std::vector &in_col_names, const std::vector &out_col_names, + int32_t op_connector_size) + : PipelineOp(op_connector_size), in_columns_(in_col_names), out_columns_(out_col_names) { + // check input & output sizes + if (in_columns_.size() != out_columns_.size()) { + MS_LOG(ERROR) << "Rename operator number of in columns != number of out columns."; + } +} + +// destructor +RenameOp::~RenameOp() {} + +// main entry point for rename +Status RenameOp::operator()() { + TaskManager::FindMe()->Post(); + std::unique_ptr curr_buffer; + RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); + if (curr_buffer->buffer_flags() != DataBuffer::kDeBFlagNone) { + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(curr_buffer))); + std::string err_msg = "Rename first buffer got was control signal"; + // if 1st eoe or eof, pass it on then return + RETURN_STATUS_UNEXPECTED(err_msg); + } + while (curr_buffer->eof() == false) { + while (curr_buffer->eoe() == false) { + // core rename functionality + RETURN_IF_NOT_OK(RenameBuffer(&curr_buffer)); + // push the renamed input buffer + MS_LOG(DEBUG) << "Rename operator pushing next buffer."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(curr_buffer))); + RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); + } // end of while eoe loop + + // we got eoe, now try again until we get eof + MS_LOG(INFO) << "Rename operator EOE Received."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique(0, DataBuffer::kDeBFlagEOE)))); + MS_LOG(DEBUG) << "Rename operator fetching buffer after EOE."; + RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); + } // end of while eof loop + + MS_LOG(INFO) << "Rename opeerator EOF Received."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique(0, DataBuffer::kDeBFlagEOF)))); + return Status::OK(); +} + +// renames buffer +Status RenameOp::RenameBuffer(std::unique_ptr *input_buffer) { + // iterate over my index in input vector, find the corresponding position + const std::unordered_map col_name_id_map = (*input_buffer)->column_name_map(); + std::unordered_map new_col_name_id_map = {}; + // parameter for input check + size_t found = 0; + + // iterate over all the pairs and if there is a name match with rename, rename the column and add it to new map + // by doing it this way we recreate a new ColNameIdMap and allow for switching + for (const auto &pair : col_name_id_map) { + std::string name = pair.first; + int32_t id = pair.second; + // find name + std::vector::iterator it; + it = std::find(in_columns_.begin(), in_columns_.end(), name); + // for c input checks here we have to count the number of times we find the stuff in in_columns_ + // because we iterate over the mInputList n times + if (it != in_columns_.end()) { + // found + found += 1; + int index = std::distance(in_columns_.begin(), it); + MS_LOG(INFO) << "Rename operator index found " << index << " value " << id << "."; + + new_col_name_id_map[out_columns_[index]] = id; + } else { + // not found + MS_LOG(INFO) << "Rename operator index not found: " << id << " is the column id."; + new_col_name_id_map[name] = id; + } + } + // only checks number of renamed columns have been found, this input check doesn't check everything + if (found != in_columns_.size()) { + MS_LOG(INFO) << "Rename operator column names found: " << found << " out of " << in_columns_.size() << "."; + std::string err_msg = "Renamed column doesn't exist in dataset"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + (*input_buffer)->set_column_name_map(new_col_name_id_map); + return Status::OK(); +} + +// prints rename +void RenameOp::Print(std::ostream &out, // In: The output stream to print to + bool show_all) const { // In: T/F if it should print everything + // Call base class printer first + PipelineOp::Print(out, show_all); + out << "\nRenameOp:\n"; + for (size_t i = 0; i < in_columns_.size(); ++i) { + out << "\nin Columns: " << in_columns_[i] << "\nOut Columns: " << out_columns_[i] << "\n\n"; + } +} + +Status RenameOp::EofReceived(int32_t) { + MS_LOG(INFO) << "Rename operator EOF received, do nothing now."; + return Status::OK(); +} + +Status RenameOp::EoeReceived(int32_t) { + state_ = OpState::kDeOpIdle; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h new file mode 100644 index 00000000000..5d820a3d0ba --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h @@ -0,0 +1,127 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_RENAME_OP_H_ +#define DATASET_ENGINE_DATASETOPS_RENAME_OP_H_ + +#include +#include +#include +#include +#include "dataset/core/tensor.h" +#include "dataset/engine/datasetops/pipeline_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// forward declare +class DataBuffer; + +class RenameOp : public PipelineOp { + public: + // The nested builder class inside of the RenameOp is used to help manage all of + // the arguments for constructing it. Use the builder by setting each argument + // with the provided set methods, and then finally call the build method to execute + // the actual construction. + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + // Default destructor + ~Builder() = default; + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetInColNames(const std::vector &in_col_names) { + builder_in_columns_ = in_col_names; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOutColNames(const std::vector &out_col_names) { + builder_out_columns_ = out_col_names; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = op_connector_size; + return *this; + } + + // The builder "build" method creates the ZipOp dataset Operator. + // @return shared_ptr to the new StorageOp object + Status Build(std::shared_ptr *); + + private: + std::vector builder_in_columns_; + std::vector builder_out_columns_; + int32_t builder_op_connector_size_; + + Status SanityCheck() const; + }; + + // Constructor for RenameOp + // @param in_col_names names of columns to rename + // @param out_col_names names of columns after rename + // @param op_connector_size connector size + RenameOp(const std::vector &in_col_names, // In: Col names to consume + const std::vector &out_col_names, // In: Col names to produce + int32_t op_connector_size); + + // Destructor + ~RenameOp(); + + Status EofReceived(int32_t) override; + + Status EoeReceived(int32_t) override; + + // Print function for Rename + // @param out output stream to print to + // @param show_all if it should print everything + void Print(std::ostream &out, bool show_all) const override; + + // Provide stream operator for displaying it + friend std::ostream &operator<<(std::ostream &out, const RenameOp &ro) { + ro.Print(out, false); + return out; + } + + // Class functor operator () override. + // All dataset ops operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work + // @return Status - The error code return + Status operator()() override; + + protected: + // Rename core functionality + // @param input_buffer buffer to run rename on + Status RenameBuffer(std::unique_ptr *input_buffer); + + // Variable to store the input column names + std::vector in_columns_; + + // Variable to store the output column names + std::vector out_columns_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_RENAME_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc new file mode 100644 index 00000000000..32723a9bd4b --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -0,0 +1,182 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "dataset/engine/execution_tree.h" +#include "dataset/engine/datasetops/repeat_op.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +// Builder constructor. Creates the builder object. +RepeatOp::Builder::Builder(int32_t count) : build_max_repeats_(count) {} + +Status RepeatOp::Builder::SanityCheck() const { + if (build_max_repeats_ < kInfiniteRepeat || build_max_repeats_ == 0) { + std::string err_msg("Repeat count must be > 0 or -1."); + RETURN_STATUS_UNEXPECTED(err_msg); + } + return Status::OK(); +} + +// The builder "build" method creates the final object. +Status RepeatOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(build_max_repeats_); + return Status::OK(); +} + +// Constructor of the RepeatOp. +RepeatOp::RepeatOp(int32_t count) : PipelineOp(0), max_repeats_(count), repeat_count_(0) {} + +// Destructor +RepeatOp::~RepeatOp() {} + +// A print method typically used for debugging +void RepeatOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + PipelineOp::Print(out, show_all); + + // Then display our own stuff + out << "RepeatOp:" + << "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_ + << "\nLeaf Nodes in my execution path:"; + if (!leaf_ops_.empty()) { + out << "\n"; + for (size_t i = 0; i < leaf_ops_.size(); i++) { + out << " Operator: " << leaf_ops_[i]->id() << "\n"; + } + } else { + out << " kNone."; + } + out << "\n-------------------------\n\n"; // End the display with this line +} + +// Base-class override for executing specific RepeatOp configurations. This code will be called +// during the execution tree prepare phase when it is visiting this operator. +Status RepeatOp::PrepareNodeAction() { + // Run any common code from super class first before adding our own specific logic + RETURN_IF_NOT_OK(PipelineOp::PrepareNodeAction()); + std::shared_ptr leaf_op = tree_->PopFromRepeatStack(); + while (leaf_op != nullptr) { + // Track the leaf operators that are under this repeat op. + leaf_ops_.push_back(leaf_op); + + // Special case. If the repeat count is 1, then pre-flag the leaf nodes + // to tell them they are already at their last op: + if (max_repeats_ == 1) { + leaf_op->set_control_flag(kDeOpLastRepeat); + } + leaf_op = tree_->PopFromRepeatStack(); + } + return Status::OK(); +} + +// Base-class override for setting specific RepeatOp configurations. This code will be called +// during the execution tree prepare phase BEFORE traversing down to child operators. +uint32_t RepeatOp::PrepareFlags() const { return ExecutionTree::kDePrepRepeat; } + +// This function returns the buffer that is at the top of our output connector. The caller is +// typically our parent node, when the parent is asking us to provide the next buffer of data. +// Since RepeatOp is an inlined op, getting a buffer from us will simply bounce you to get +// a buffer from our child. +// This function sets the `retryIfEoe` flag when popping from the child connector. This way, +// this function will retry to pop the connector again and will get the non-EOE buffer if any. +Status RepeatOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) { + if (child_.empty()) { + RETURN_STATUS_UNEXPECTED("RepeatOp can't be the leaf node."); + } + + std::unique_ptr buf; + RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buf, worker_id, true)); + // Loop until non EOE is received + while (buf->eoe()) { + RETURN_IF_NOT_OK(EoeReceived(worker_id)); + if (state_ == OpState::kDeOpIdle) { + *p_buffer = std::move(buf); + return Status::OK(); + } + RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buf, worker_id, true)); + } + // Check if the last buf is next eof + if (buf->eof()) { + RETURN_IF_NOT_OK(EofReceived(worker_id)); + } + *p_buffer = std::move(buf); + return Status::OK(); +} + +// Base-class override for handling cases when an eoe is received. +Status RepeatOp::EoeReceived(int32_t worker_id) { + repeat_count_++; + MS_LOG(INFO) << "Repeat operator end of epoch message received. Repeat count is now: " << repeat_count_ << "."; + + // If we've reached the requested repeat count, then flag the leaf nodes + // to tell them they've got one more epoch to perform. When they reach the end + // of the last epoch, they quit rather than loop again. + if (max_repeats_ != kInfiniteRepeat && repeat_count_ == (max_repeats_ - 1)) { + for (size_t i = 0; i < leaf_ops_.size(); i++) { + leaf_ops_[i]->set_control_flag(kDeOpLastRepeat); + } + } + if (repeat_count_ == max_repeats_) { + state_ = OpState::kDeOpIdle; + return Status::OK(); + } + + // base-class ResetSubtree + return (DatasetOp::ResetSubtree()); +} + +// Class functor operator () override. +// Most dataset ops operate by launching a thread (see ExecutionTree). +// However, the RepeatOp is defined as a inlined operator, so it is invalid to launch the +// functor since this op runs inlined inside another operator. The function is overloaded to +// ensure that it is not called by mistake (it will generate an error). +Status RepeatOp::operator()() { RETURN_STATUS_UNEXPECTED("Logic error. RepeatOp is an inlined operator."); } + +// Base-class override for handling cases when an eof is received. +Status RepeatOp::EofReceived(int32_t worker_id) { + MS_LOG(INFO) << "Repeat operator EOF received, do nothing now."; + return Status::OK(); +} + +int32_t RepeatOp::num_consumers() const { + if (parent_.empty()) { + MS_LOG(INFO) << "Repeat operator, no parent node, assuming it's root and returning 1."; + return 1; + } else if (parent_[0] == nullptr) { + MS_LOG(INFO) << "Repeat operator, pointer to the first parent is null. Returning 0."; + return 0; + } else { + return parent_[0]->num_consumers(); + } +} + +int32_t RepeatOp::num_producers() const { + if (child_.empty() || child_[0] == nullptr) { + MS_LOG(INFO) << "Repeat operator, pointer to child node is null. Returning 0."; + return 0; + } else { + return child_[0]->num_producers(); + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.h b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.h new file mode 100644 index 00000000000..5cc7ec2efa7 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.h @@ -0,0 +1,129 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_REPEAT_OP_H_ +#define DATASET_ENGINE_DATASETOPS_REPEAT_OP_H_ + +#include +#include +#include +#include "dataset/engine/datasetops/pipeline_op.h" + +namespace mindspore { +namespace dataset { +class RepeatOp : public PipelineOp { + public: + static constexpr int32_t kInfiniteRepeat = -1; + + // The nested builder class inside of the RepeatOp is used to help manage all of the arguments + // for constructing it. This repeat op is very simple though, so this builder is really just + // provided for a consistent look and feel for creators of Dataset operators overall. + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @param count - The number of repeats to do + // @return This is a constructor. + explicit Builder(int32_t count); + + // Default destructor + ~Builder() = default; + + // The builder "build" method creates the final object. + // @return shared_ptr to the new StorageOp object + Status Build(std::shared_ptr *); + + private: + int32_t build_max_repeats_; + + Status SanityCheck() const; + }; + + // Constructor of the RepeatOp. + // @note The builder class should be used to call it + // @param count - The number of repeats to do + explicit RepeatOp(int32_t count); + + // Destructor + ~RepeatOp(); + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param ro - reference to the RepeatOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const RepeatOp &ro) { + ro.Print(out, false); + return out; + } + + // Class functor operator () override. + // Most dataset ops operate by launching a thread (see ExecutionTree). + // However, the RepeatOp is defined as a inlined operator, so it is invalid to launch the + // functor since this op runs inlined inside another operator. The function is overloaded to + // ensure that it is not called by mistake (it will generate an error). + // @return Status - The error code return + Status operator()() override; + + // Base-class override for setting specific RepeatOp configurations. This code will be called + // during the execution tree prepare phase BEFORE traversing down to child operators. + uint32_t PrepareFlags() const override; + + // Base-class override for executing specific RepeatOp configurations. This code will be called + // during the execution tree prepare phase when it is visiting this operator. + Status PrepareNodeAction() override; + + // This function returns the buffer that is at the top of our output connector. The caller is + // typically our parent node, when the parent is asking us to provide the next buffer of data. + // Since RepeatOp is an inlined op, getting a buffer from us will simply bounce you to get + // a buffer from our child. + // @note This function sets the `retryIfEoe` flag when popping from the child connector. This way, + // this function will retry to pop the connector again and will get the non-EOE buffer if any. + // @param p_buffer - output pointer to the buffer that it will fetch. + // @param worker_id - The worker id + // @param retry_if_eoe Set this flag to true to allow calling pop() again after the first pop() returns EOE. + // @return Status - The error code return + Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) override; + + // Base-class override for handling cases when an eoe is received. + // @param worker_id - The worker id + Status EoeReceived(int32_t worker_id) override; + + // Base-class override for handling cases when an eof is received. + // @param worker_id - The worker id + Status EofReceived(int32_t worker_id) override; + + // Base-class override. Return the number of workers in the first parent. + // @param workerId - The worker id + int32_t num_consumers() const override; + + // Base-class override. Return the number of producers in the first child. + // @param workerId - The worker id + int32_t num_producers() const override; + + private: + int32_t max_repeats_; // The number of repeats that the user requested + int32_t repeat_count_; // A counter for the current number of executed repeats + std::vector> leaf_ops_; // List of leaf operators underneath this repeat. +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_REPEAT_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc new file mode 100644 index 00000000000..5dae48ad739 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -0,0 +1,294 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/engine/datasetops/shuffle_op.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +constexpr int32_t ShuffleOp::kShuffleStateInit; +constexpr int32_t ShuffleOp::kShuffleStateActive; +constexpr int32_t ShuffleOp::kShuffleStateDrain; + +// Builder constructor. Creates the builder object. +ShuffleOp::Builder::Builder() : build_shuffle_size_(0), build_reshuffle_each_epoch_(true) { + std::shared_ptr cfg = GlobalContext::config_manager(); + build_op_connector_size_ = cfg->op_connector_size(); + build_rows_per_buffer_ = cfg->rows_per_buffer(); + build_shuffle_seed_ = GetSeed(); +} + +Status ShuffleOp::Builder::SanityCheck() const { + if (build_shuffle_size_ < 2) { + RETURN_STATUS_UNEXPECTED("Shuffle buffer size must be greater than 1."); + } + return Status::OK(); +} + +// The builder "build" method creates the final object. +Status ShuffleOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(build_shuffle_size_, build_shuffle_seed_, build_op_connector_size_, + build_reshuffle_each_epoch_, build_rows_per_buffer_); + return Status::OK(); +} + +// Constructor of the ShuffleOp +ShuffleOp::ShuffleOp(int32_t shuffle_size, uint32_t shuffle_seed, int32_t op_connector_size, bool reset_every_epoch, + int32_t rows_per_buffer) + : PipelineOp(op_connector_size), + shuffle_size_(shuffle_size), + shuffle_seed_(shuffle_seed), + reshuffle_each_epoch_(reset_every_epoch), + rng_(shuffle_seed), + buffer_counter_(0), + rows_per_buffer_(rows_per_buffer), + shuffle_buffer_(mindspore::make_unique()), + shuffle_last_row_idx_(0), + shuffle_buffer_state_(kShuffleStateInit) {} + +// Private function to re-init the shuffle op for another epoch. Shuffle op calls this by +// itself rather than waiting for the reset driven from operators above it in the pipeline. +Status ShuffleOp::SelfReset() { + MS_LOG(INFO) << "Shuffle operator performing a self-reset."; + // If ReshuffleEachEpoch is false, then we always use the same seed for every + // epoch. + // If ReshuffleEachEpoch is true, then the first epoch uses the given seed, + // and all subsequent epochs will then reset the seed based on random device. + if (!reshuffle_each_epoch_) { + rng_ = std::mt19937_64(shuffle_seed_); + } else { + std::random_device random_device("/dev/urandom"); + std::uniform_int_distribution distribution(0, std::numeric_limits::max()); + shuffle_seed_ = distribution(random_device); + rng_ = std::mt19937_64(shuffle_seed_); + } + shuffle_buffer_ = mindspore::make_unique(); + buffer_counter_ = 0; + shuffle_last_row_idx_ = 0; + shuffle_buffer_state_ = kShuffleStateInit; + return Status::OK(); +} + +// A print method typically used for debugging +void ShuffleOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + PipelineOp::Print(out, show_all); + + // Then display our own stuff + out << "ShuffleOp:\n Shuffle size: " << shuffle_size_ << "\n rows_per_buffer_: " << rows_per_buffer_ + << "\n shuffle_buffer_state_: " << shuffle_buffer_state_ << "\n shuffle_seed_: " << shuffle_seed_; + out << "\n-------------------------\n\n"; // End the display with this line +} + +// Private function to add a new row to the shuffle buffer. +Status ShuffleOp::AddRowToShuffleBuffer(TensorRow new_shuffle_row) { + // If the last slot of our shuffle buffer was not the full size of the shuffle buffer then we are + // filling it during the initial fill codepath and thus growing it's size. In that case, we push + // back the new row to grow our shuffle buffer size by 1. + // If we are already at the full size, then we overwrite the last slot with our row (and the last + // slot better be empty because it should already have been swapped out during the random row + // selection that was done previously!) + if (shuffle_last_row_idx_ < (shuffle_size_ - 1)) { + shuffle_buffer_->push_back(std::move(new_shuffle_row)); + shuffle_last_row_idx_ = (shuffle_buffer_->size()) - 1; + } else { + if (!(*shuffle_buffer_)[shuffle_last_row_idx_].empty()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Last row of shuffle buffer should not be occupied!"); + } + (*shuffle_buffer_)[shuffle_last_row_idx_] = std::move(new_shuffle_row); + } + return Status::OK(); +} + +// Class functor operator () override. +// All dataset ops operate by launching a thread (see ExecutionTree). This class functor will +// provide the master loop that drives the logic for performing the work +Status ShuffleOp::operator()() { + std::unique_ptr new_buffer_table; // A tensor table to be used for output. + + // Synchronize with TaskManager once the thread is launched. + TaskManager::FindMe()->Post(); + + // Shuffle op does not have workers, and only consumes from child 0. + // Create the child iterator to fetch our data from. + int32_t worker_id = 0; + int32_t child_idx = 0; + child_iterator_ = mindspore::make_unique(this, worker_id, child_idx); + + // Main operator loop + while (true) { + // Do an initial populate of the shuffle buffer + RETURN_IF_NOT_OK(InitShuffleBuffer()); + + // This is our main loop exit condition, when the iterator has no more data completely. + if (child_iterator_->eof_handled()) { + break; + } + + // Next, enter into the main execution loop of the shuffle op. + // When the tail index position of our shuffle buffer goes negative it means that we've + // fully drained the data from the shuffle buffer and we're done. + while (shuffle_last_row_idx_ >= 0) { + // Step 1) + // Create an output tensor table if one is not created yet. + if (!new_buffer_table) { + new_buffer_table = mindspore::make_unique(); + } + + // Step 2) + // Randomly select a slot from our shuffle buffer and copy that row into the output + // tensor table. We remove the data from the shuffle buffer, leaving that slot + // in the table as an empty vector + int64_t random_slot = rng_() % (shuffle_last_row_idx_ + 1); + new_buffer_table->push_back(std::move((*shuffle_buffer_)[random_slot])); + + // Step 3) + // If the output tensor table is at the requested size, then create a buffer for it + // and send this buffer on it's way up the pipeline. Special case is if this is the + // last row then we also send it. + if (new_buffer_table->size() == rows_per_buffer_ || shuffle_last_row_idx_ == 0) { + auto new_buffer = mindspore::make_unique(buffer_counter_, DataBuffer::kDeBFlagNone); + new_buffer->set_tensor_table(std::move(new_buffer_table)); + new_buffer->set_column_name_map(column_name_map_); + buffer_counter_++; + MS_LOG(DEBUG) << "Shuffle operator sending a buffer to output."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(new_buffer))); + } + + // Step 4) + // Take the last row from shuffle buffer, and swap it into the row position that was + // just vacated. This makes the shuffle buffer contiguous, with an empty slot at the + // tail of the shuffle buffer. + if (random_slot != shuffle_last_row_idx_) { + (*shuffle_buffer_)[random_slot] = std::move((*shuffle_buffer_)[shuffle_last_row_idx_]); + } + + // Step 5) + // Refill the last slot of the shuffle buffer with the next row from input if we are in the + // active state. + // If we are in the draining state, we do not need to fetch another row to replace the one we + // just drained. + if (shuffle_buffer_state_ == kShuffleStateActive) { + TensorRow new_row; + RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); + + if (!new_row.empty()) { + RETURN_IF_NOT_OK(AddRowToShuffleBuffer(std::move(new_row))); + } else { + shuffle_buffer_state_ = kShuffleStateDrain; + } + } + + // If we are draining, reposition (decrement) our tail index in the shuffle buffer since we + // just drained a row from it. + if (shuffle_buffer_state_ == kShuffleStateDrain) { + shuffle_last_row_idx_--; + } + } + + // Since we overloaded eoeReceived function, we are responsible to flow the EOE up the + // pipepline manually now that we are done draining the shuffle buffer + MS_LOG(INFO) << "Shuffle operator sending EOE."; + auto eoe_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); + + // Do not wait for any reset to be flown down from operators above us. + // Instead, manually update ourselves and then go reloop to start fetching from child operator + // right away. Any Reset() from the parent will still perform common reset actions. + RETURN_IF_NOT_OK(this->SelfReset()); + } + return Status::OK(); +} + +// Private function populate the shuffle buffer initially by fetching from the child output +// connector until the shuffle buffer is full (or there is no more data coming). +Status ShuffleOp::InitShuffleBuffer() { + MS_LOG(INFO) << "Shuffle operator initializing the shuffle buffer."; + + // The first phase of this operator is to read incoming buffers and then drain those + // rows from the buffers, putting them into our own local table of tensors (the shuffle + // buffer). + // This shuffle buffer initialization phase stops when we've either filled up the + // shuffle buffer to it's max size, or the dataset below us is not providing any more + // rows. + if (shuffle_buffer_state_ != kShuffleStateInit) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Invalid shuffle buffer state (SHUFFLE_STATE_INIT expected)"); + } + + // Before we drop into the fetching loop, call the fetch once for the first time + // to fill the first row and grab the first buffer. + TensorRow new_row; + RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); + + if (child_iterator_->eof_handled()) { + MS_LOG(INFO) << "Shuffle operator init picked up EOF. No more epochs."; + return Status::OK(); + } + + if (new_row.empty()) { + RETURN_STATUS_UNEXPECTED("Unable to fetch a single row for shuffle buffer."); + } + + // Take a copy of the column name mapping. We'll use this when constructing output buffers later. + column_name_map_ = child_iterator_->col_name_id_map(); + + // Now fill the rest of the shuffle buffer until we are unable to get the next row or we reached + // the desired shuffle buffer size. + while (!new_row.empty() && shuffle_buffer_->size() < static_cast(shuffle_size_ - 1)) { + // Add the previously fetched row + RETURN_IF_NOT_OK(AddRowToShuffleBuffer(std::move(new_row))); + + // Fetch the next row + RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); + } + + // If we quit the loop due to being at the shuffle size, still need to add the last row here. + if (!new_row.empty()) { + RETURN_IF_NOT_OK(AddRowToShuffleBuffer(std::move(new_row))); + shuffle_buffer_state_ = kShuffleStateActive; // Transition to the active state + } else { + // If init phase doesn't have more rows, then skip the active state and jump straight to the + // shuffle buffer draining state + shuffle_buffer_state_ = kShuffleStateDrain; + } + + MS_LOG(INFO) << "Shuffle operator finished intializing the shuffle buffer."; + return Status::OK(); +} + +Status ShuffleOp::EoeReceived(int32_t worker_id) { + state_ = OpState::kDeOpIdle; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.h b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.h new file mode 100644 index 00000000000..0294744ebd2 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.h @@ -0,0 +1,195 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SHUFFLE_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SHUFFLE_OP_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/engine/datasetops/pipeline_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Forward declare +class ExecutionTree; + +class DbConnector; + +class DataBuffer; + +class ShuffleOp : public PipelineOp { + // Shuffle buffer state flags + // + // Shuffle buffer is in a state of being initialized + static constexpr int32_t kShuffleStateInit = 0; + + // Shuffle buffer is in a state of being actively drained from, but refilling as well + static constexpr int32_t kShuffleStateActive = 1; + + // Shuffle buffer is in a state of being drained + static constexpr int32_t kShuffleStateDrain = 2; + + public: + // The nested builder class inside of the ShuffleOp is used to help manage all of the arguments + // for constructing it. The shuffle op is fairly simple though, but the builder provides a + // consistent look and feel for creators of Dataset operators overall. + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + // Default destructor + ~Builder() = default; + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetShuffleSize(int32_t shuffle_size) { + build_shuffle_size_ = shuffle_size; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetShuffleSeed(uint32_t shuffle_seed) { + build_shuffle_seed_ = shuffle_seed; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + build_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetReshuffleEachEpoch(bool reshuffle_each_epoch) { + build_reshuffle_each_epoch_ = reshuffle_each_epoch; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t op_connector_size) { + build_op_connector_size_ = op_connector_size; + return *this; + } + + // The builder "build" method creates the final object. + // @return shared_ptr to the new StorageOp object + Status Build(std::shared_ptr *); + + private: + // The builder saves all ShuffleOp construction arguments internally. + // The following are the arguments. + int32_t build_shuffle_size_; + uint32_t build_shuffle_seed_; + int32_t build_rows_per_buffer_; + bool build_reshuffle_each_epoch_; + int32_t build_op_connector_size_; + + Status SanityCheck() const; + }; + + // Constructor of the ShuffleOp + // @note The builder class should be used to call it + // @param shuffle_size - The size for the shuffle buffer + // @param shuffle_seed - The seed to use for random number generation + // @param op_connector_size - The output connector queue size + // @param rows_per_buffer - The requested number of rows per buffer + ShuffleOp(int32_t shuffle_size, uint32_t shuffle_seed, int32_t op_connector_size, bool reset_every_epoch, + int32_t rows_per_buffer); + + // Destructor + ~ShuffleOp() = default; + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param so - reference to the ShuffleOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const ShuffleOp &so) { + so.Print(out, false); + return out; + } + + // Class functor operator () override. + // All dataset ops operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work + // @return Status - The error code return + Status operator()() override; + + // Base-class override for special eoe handler. + // ShuffleOp must override this because it shall not perform default handling of eoe. Instead + // the ShuffleOp needs to manage actions related to the end of the epoch itself. + // @return Status - The error code return + Status EoeReceived(int32_t worker_id) override; + + private: + // Private function to add a new row to the shuffle buffer. + // @return Status - The error code return + Status AddRowToShuffleBuffer(TensorRow new_shuffle_row); + + // Private function to populate the shuffle buffer initially by fetching from the child output + // connector until the shuffle buffer is full (or there is no more data coming). + // @return Status - The error code return + Status InitShuffleBuffer(); + + // Private function to re-init the shuffle op for another epoch. Shuffle op calls this by + // itself rather than waiting for the reset driven from operators above it in the pipeline. + // @return Status - The error code return + Status SelfReset(); + + int32_t shuffle_size_; // User config for the size of the shuffle buffer (number of rows) + uint32_t shuffle_seed_; + bool reshuffle_each_epoch_; + // rng_ is seeded initially with shuffle_seed_. mt19937 is used for its large period. + // specifically mt19937_64 is used to generate larger random numbers to reduce bias when + // modding to fit within our desired range. we dont use a distribution + // (ie uniform_int_distribution) because we will need to create up to |dataset| instances + // of the distribution object in the common case of a perfect shuffle + std::mt19937_64 rng_; + int32_t buffer_counter_; // For creating new buffer id's + int32_t rows_per_buffer_; // Number of rows to pack into output buffer + // A single (potentially large) buffer of tensor rows for performing shuffling. + std::unique_ptr shuffle_buffer_; + int32_t shuffle_last_row_idx_; // Internal tracking of the last slot of our shuffle buffer + int32_t shuffle_buffer_state_; // State tracking for the shuffle buffer phases of work + std::unordered_map column_name_map_; // A mapping between column index to column name. + + std::unique_ptr child_iterator_; // An iterator for fetching. +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SHUFFLE_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/datasetops/source/CMakeLists.txt new file mode 100644 index 00000000000..5a02a2ec31d --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/CMakeLists.txt @@ -0,0 +1,23 @@ +ms_protobuf_generate(EXAMPLE_SRCS EXAMPLE_HDRS example.proto) +ms_protobuf_generate(FEATURE_SRCS FEATURE_HDRS feature.proto) +add_subdirectory(sampler) +add_library(engine-datasetops-source OBJECT + generator_op.cc + io_block.cc + mindrecord_op.cc + storage_client.cc + storage_op.cc + tf_buffer.cc + tf_client.cc + tf_reader_op.cc + image_folder_op.cc + mnist_op.cc + voc_op.cc + ${EXAMPLE_SRCS} + ${FEATURE_SRCS} + manifest_op.cc + cifar_op.cc + celeba_op.cc + ) + +add_dependencies(engine-datasetops-source protobuf::libprotobuf) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc new file mode 100644 index 00000000000..570fc9f4549 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -0,0 +1,443 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +#include "dataset/engine/datasetops/source/celeba_op.h" + +#include +#include "dataset/core/config_manager.h" +#include "dataset/util/path.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/kernels/image/image_utils.h" + +namespace mindspore { +namespace dataset { +CelebAOp::Builder::Builder() : builder_decode_(false), builder_sampler_(nullptr), builder_num_samples_(0) { + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_num_workers_ = cfg->num_parallel_workers(); + builder_rows_per_buffer_ = cfg->rows_per_buffer(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status CelebAOp::Builder::Build(std::shared_ptr *op) { + MS_LOG(INFO) << "Celeba dataset directory is " << builder_dir_.c_str() << "."; + MS_LOG(INFO) << "Celeba dataset type is " << builder_dataset_type_.c_str() << "."; + RETURN_IF_NOT_OK(SanityCheck()); + if (builder_sampler_ == nullptr) { + builder_sampler_ = std::make_shared(); + } + + builder_schema_ = make_unique(); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); + // label is like this:0 1 0 0 1...... + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("attr", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 1))); + *op = + std::make_shared(builder_num_workers_, builder_rows_per_buffer_, builder_dir_, builder_op_connector_size_, + builder_decode_, builder_dataset_type_, builder_extensions_, std::move(builder_schema_), + std::move(builder_sampler_), builder_num_samples_); + if (*op == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "CelebAOp is null"); + } + + return Status::OK(); +} + +Status CelebAOp::Builder::SanityCheck() { + Path dir(builder_dir_); + std::string err_msg; + err_msg += dir.IsDirectory() ? "" : "CelebA path is invalid or not set\n"; + err_msg += builder_num_workers_ <= 0 ? "Num of parallel workers is smaller than 1\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::string &dir, int32_t queue_size, + bool decode, const std::string &dataset_type, const std::set &exts, + std::unique_ptr schema, std::shared_ptr sampler, int64_t num_samples) + : ParallelOp(num_workers, queue_size), + rows_per_buffer_(rows_per_buffer), + folder_path_(dir), + decode_(decode), + extensions_(exts), + data_schema_(std::move(schema)), + sampler_(std::move(sampler)), + num_rows_in_attr_file_(0), + num_rows_exact_(0), + num_samples_(num_samples), + dataset_type_(dataset_type) { + for (int32_t index = 0; index < data_schema_->NumColumns(); index++) { + col_name_map_[data_schema_->column(index).name()] = index; + } + + attr_info_queue_ = make_unique>>(queue_size); + io_block_queues_.Init(num_workers_, queue_size); +} + +Status CelebAOp::LaunchThreadsAndInitOp() { + if (tree_ == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "tree_ not set"); + } + + RETURN_IF_NOT_OK(io_block_queues_.Register(tree_->AllTasks())); + RETURN_IF_NOT_OK(attr_info_queue_->Register(tree_->AllTasks())); + wp_.Register(tree_->AllTasks()); + + RETURN_IF_NOT_OK(tree_->AllTasks()->CreateAsyncTask("Walking attr file", std::bind(&CelebAOp::ParseAttrFile, this))); + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&CelebAOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + RETURN_IF_NOT_OK(ParseImageAttrInfo()); + RETURN_IF_NOT_OK(sampler_->Init(this)); + + return Status::OK(); +} + +Status CelebAOp::ParseAttrFile() { + TaskManager::FindMe()->Post(); + Path folder_path(folder_path_); + std::ifstream attr_file((folder_path / "list_attr_celeba.txt").toString()); + if (!attr_file.is_open()) { + return Status(StatusCode::kFileNotExist, __LINE__, __FILE__, "Celeba attr file does not exist"); + } + + const auto PushBackToQueue = [this](std::vector &vec, std::ifstream &attr_file, + std::ifstream &partition_file) { + Status s = attr_info_queue_->EmplaceBack(vec); + if (s.IsError()) { + CLOSE_FILE(attr_file, partition_file); + return s; + } + return Status::OK(); + }; + + std::string rows_num; + std::string attr_name; + (void)getline(attr_file, rows_num); + try { + num_rows_in_attr_file_ = static_cast(std::stoul(rows_num)); // First line is rows number in attr file + } catch (std::invalid_argument &e) { + RETURN_STATUS_UNEXPECTED("Conversion to ulong failed, invalid argument."); + } catch (std::out_of_range &e) { + RETURN_STATUS_UNEXPECTED("Conversion to ulong failed, out of range."); + } + + (void)getline(attr_file, attr_name); // Second line is attribute name,ignore it + std::string image_info; + std::vector image_infos; + image_infos.reserve(oc_queue_size_); + while (getline(attr_file, image_info)) { + if ((image_info.empty()) || (dataset_type_ != "all" && !CheckDatasetTypeValid())) { + continue; + } + image_infos.push_back(image_info); + if (image_info.size() % oc_queue_size_ == 0) { + RETURN_IF_NOT_OK(PushBackToQueue(image_infos, attr_file, partition_file_)); + image_infos.clear(); + } + } + if (!image_infos.empty()) { + RETURN_IF_NOT_OK(PushBackToQueue(image_infos, attr_file, partition_file_)); + } + std::vector end_indicator = std::vector(0); + RETURN_IF_NOT_OK(PushBackToQueue(end_indicator, attr_file, partition_file_)); // end indicator + CLOSE_FILE(attr_file, partition_file_); + return Status::OK(); +} + +bool CelebAOp::CheckDatasetTypeValid() { + if (!partition_file_.is_open()) { + Path folder_path(folder_path_); + partition_file_.open((folder_path / "list_eval_partition.txt").toString()); + if (!partition_file_.is_open()) { + MS_LOG(ERROR) << "Celeba partition file does not exist!"; + return false; + } + } + std::string line; + (void)getline(partition_file_, line); + std::vector vec = Split(line); + if (vec.size() != 2) { + return false; + } + int32_t type; + try { + type = std::stoi(vec[1]); + } catch (std::invalid_argument &e) { + MS_LOG(WARNING) << "Conversion to unsigned long failed, invalid argument, " << vec[0] << "."; + return false; + } catch (std::out_of_range &e) { + MS_LOG(WARNING) << "Conversion to unsigned long failed, out of range, " << vec[0] << "."; + return false; + } + // train:0, valid=1, test=2 + if (dataset_type_ == "train" && (type == 0)) { + return true; + } else if (dataset_type_ == "valid" && (type == 1)) { + return true; + } else if (dataset_type_ == "test" && (type == 2)) { + return true; + } + + return false; +} + +Status CelebAOp::ParseImageAttrInfo() { + std::vector image_infos; + bool needMoreData = true; + RETURN_IF_NOT_OK(attr_info_queue_->PopFront(&image_infos)); + while (!image_infos.empty() && needMoreData) { + for (uint32_t index = 0; index < image_infos.size(); index++) { + if (num_samples_ != 0 && image_labels_vec_.size() >= num_samples_) { + MS_LOG(WARNING) << "Image number(" << image_labels_vec_.size() << " is more than" + << " rows num eval attr file(" << num_rows_in_attr_file_ << ") or num samples(" << num_samples_ + << ")."; + needMoreData = false; + break; + } + std::string image_info = image_infos[index]; + std::vector split = Split(image_info); + std::pair> image_labels; + + Path path(folder_path_); + Path file_path = path / split[0]; + if (!extensions_.empty() && extensions_.find(file_path.Extension()) == extensions_.end()) { + MS_LOG(INFO) << "Unsupported file found at " << file_path.toString().c_str() << ", its extension is " + << file_path.Extension().c_str() << "."; + continue; + } + image_labels.first = split[0]; + for (uint32_t label_index = 1; label_index < split.size(); label_index++) { + int32_t value; + try { + value = std::stoi(split[label_index]); + } catch (std::invalid_argument &e) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed, invalid argument."); + } catch (std::out_of_range &e) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed, out of range."); + } + image_labels.second.push_back(value); + } + + image_labels_vec_.push_back(image_labels); + } + + RETURN_IF_NOT_OK(attr_info_queue_->PopFront(&image_infos)); + } + + num_rows_exact_ = image_labels_vec_.size(); + num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_exact_) ? num_rows_exact_ : num_samples_; + if (num_rows_exact_ == 0) { + RETURN_STATUS_UNEXPECTED("Number of rows in celeba dataset is zero"); + } + MS_LOG(INFO) << "Celeba dataset rows number is " << num_rows_exact_ << "."; + return Status::OK(); +} + +std::vector CelebAOp::Split(const std::string &line) { + std::string str = line; + std::string::size_type pos; + std::vector split; + str += " "; + int size = str.size(); + for (uint32_t index = 0; index < size;) { + pos = str.find(" ", index); + if (pos != index) { // skip space + std::string s = str.substr(index, pos - index); + split.push_back(s); + } + index = pos + 1; + } + + return split; +} + +// Derived from RandomAccessOp +Status CelebAOp::GetNumSamples(int64_t *num) const { + if (num == nullptr || num_samples_ == 0) { + RETURN_STATUS_UNEXPECTED("NumSample not set"); + } + (*num) = num_samples_; + return Status::OK(); +} + +Status CelebAOp::GetNumRowsInDataset(int64_t *num) const { + if (num == nullptr || num_rows_exact_ == 0) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "NumRow not set"); + } + + *num = num_rows_exact_; + return Status::OK(); +} + +// Main logic, Register Queue with TaskGroup, launch all threads and do the functor's work +Status CelebAOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + std::unique_ptr data_buffer; + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&data_buffer)); + RETURN_IF_NOT_OK(AddIOBlock(&data_buffer)); + return Status::OK(); +} + +Status CelebAOp::AddIOBlock(std::unique_ptr *data_buffer) { + int64_t buff_count = 0; + while (true) { + std::vector keys; + keys.reserve(rows_per_buffer_); + int64_t row_count = 0; + while (!(*data_buffer)->eoe()) { + TensorRow sample_row; + RETURN_IF_NOT_OK((*data_buffer)->PopRow(&sample_row)); + std::shared_ptr sample_ids = sample_row[0]; + for (auto itr = sample_ids->begin(); itr != sample_ids->end(); ++itr) { + if ((*itr) >= num_rows_exact_) { + MS_LOG(WARNING) << "Sample Id (" << *itr << ") is out of bounds, skipping. Max id is " << num_rows_exact_ + << "."; + continue; + } + keys.push_back(*itr); + row_count++; + if (row_count % rows_per_buffer_ == 0) { + RETURN_IF_NOT_OK(io_block_queues_[buff_count++ % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + keys.clear(); + } + } + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer)); + } + + if (!keys.empty()) { + RETURN_IF_NOT_OK(io_block_queues_[(buff_count++) % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + RETURN_IF_NOT_OK( + io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK( + io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEof))); + for (int32_t i = 0; i < num_workers_; i++) { + RETURN_IF_NOT_OK( + io_block_queues_[i]->Add(std::move(make_unique(std::vector(), IOBlock::kDeIoBlockNone)))); + } + return Status::OK(); + } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset + RETURN_IF_NOT_OK( + io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks + wp_.Clear(); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer)); + } + } +} + +Status CelebAOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + int64_t buffer_id = worker_id; + std::unique_ptr io_block; + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + while (io_block != nullptr) { + if (io_block->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique(0, DataBuffer::kDeBFlagEOE)))); + buffer_id = worker_id; + } else if (io_block->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique(0, DataBuffer::kDeBFlagEOF)))); + } else { + std::vector keys; + RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); + if (keys.empty()) { + return Status::OK(); // empty key is a quit signal for workers + } + std::unique_ptr db = make_unique(buffer_id, DataBuffer::kDeBFlagNone); + RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); + buffer_id += num_workers_; + } + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + } + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Unexpected nullptr received in worker"); +} + +Status CelebAOp::LoadBuffer(const std::vector &keys, std::unique_ptr *db) { + std::unique_ptr deq = make_unique(); + for (const auto &key : keys) { + TensorRow row; + RETURN_IF_NOT_OK(LoadTensorRow(image_labels_vec_[key], &row)); + deq->push_back(std::move(row)); + } + + (*db)->set_tensor_table(std::move(deq)); + (*db)->set_column_name_map(col_name_map_); + return Status::OK(); +} + +Status CelebAOp::LoadTensorRow(const std::pair> &image_label, TensorRow *row) { + std::shared_ptr image; + std::shared_ptr label; + + Path path(folder_path_); + Path image_path = path / image_label.first; + std::ifstream handle(image_path.toString(), std::ios::binary | std::ios::in); + if (handle.fail()) { + std::string err_msg = "Fail to open file: " + image_path.toString(); + return Status(StatusCode::kFileNotExist, __LINE__, __FILE__, err_msg); + } + + (void)handle.seekg(0, std::ios::end); + int64_t num_elements = handle.tellg(); + (void)handle.seekg(0, std::ios::beg); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), + TensorShape(std::vector(1, num_elements)), + data_schema_->column(0).type())); + (void)handle.read(reinterpret_cast(image->StartAddr()), num_elements); + if (decode_ == true) { + Status rc = Decode(image, &image); + if (rc.IsError()) { + image = nullptr; + std::string err_msg = "Fail to decode image: " + image_path.toString(); + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); + } + } + + RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), + TensorShape({1, (uint32_t)image_label.second.size()}), + data_schema_->column(1).type())); + RETURN_IF_NOT_OK(label->Zero()); + for (uint32_t index = 0; index < image_label.second.size(); index++) { + if (image_label.second[index] == 1) { + label->SetItemAt({0, static_cast(index)}, 1); + } else { + label->SetItemAt({0, static_cast(index)}, 0); + } + } + label->Squeeze(); + + (*row) = {std::move(image), std::move(label)}; + return Status::OK(); +} + +void CelebAOp::Print(std::ostream &out, bool show_all) const { + DatasetOp::Print(out, show_all); + out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_exact_ + << "\nceleba dir: " << folder_path_ << "\n-------------------------\n"; +} + +// Reset Sampler and wakeup Master thread (functor) +Status CelebAOp::Reset() { + RETURN_IF_NOT_OK(sampler_->Reset()); + wp_.Set(); // wake up master thread after reset is done + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h new file mode 100644 index 00000000000..55056c2a491 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h @@ -0,0 +1,248 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_CELEBA_OP_H +#define DATASET_ENGINE_DATASETOPS_SOURCE_CELEBA_OP_H + +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/util/status.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/util/queue.h" +#include "dataset/engine/datasetops/source/io_block.h" + +#define CLOSE_FILE(attr_file, pairition_file) \ + do { \ + attr_file.close(); \ + if (pairition_file.is_open()) { \ + pairition_file.close(); \ + } \ + } while (false) + +namespace mindspore { +namespace dataset { +class CelebAOp : public ParallelOp, RandomAccessOp { + public: + class Builder { + public: + // Constructor for Builder class of CelebAOp + // @return Builder setter method returns reference to the builder. + Builder(); + + // Destructor. + ~Builder() = default; + + // Setter method + // @param int32_t rows_per_buffer + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method + // @param int32_t size + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t size) { + builder_op_connector_size_ = size; + return *this; + } + + // Setter method + // @param std::set & exts, file extensions to be read + // @return Builder setter method returns reference to the builder. + Builder &SetExtensions(const std::set &exts) { + builder_extensions_ = exts; + return *this; + } + + // Setter method + // @param bool decode + // @return Builder setter method returns reference to the builder. + Builder &SetDecode(bool decode) { + builder_decode_ = decode; + return *this; + } + + // Setter method + // @param int32_t num_workers + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + + // Setter method + // @param const std::string &dir + // @return Builder setter method returns reference to the builder. + Builder &SetCelebADir(const std::string &dir) { + builder_dir_ = dir; + return *this; + } + + // Setter method + // @param int64_t num_samples + // @return Builder setter method returns reference to the builder. + Builder &SetNumSamples(int64_t num_samples) { + builder_num_samples_ = num_samples; + return *this; + } + + // Setter method + // @param const std::string dataset_type: type to be read + // @return Builder setter method returns reference to the builder. + Builder &SetDatasetType(const std::string &dataset_type) { + builder_dataset_type_ = dataset_type; + return *this; + } + // Check validity of input args + // @return - The error code return + Status SanityCheck(); + + // The builder "build" method creates the final object. + // @param std::shared_ptr *op - DatasetOp + // @return - The error code return + Status Build(std::shared_ptr *op); + + private: + bool builder_decode_; + std::string builder_dir_; + int32_t builder_num_workers_; + int32_t builder_rows_per_buffer_; + int32_t builder_op_connector_size_; + std::set builder_extensions_; + std::shared_ptr builder_sampler_; + std::unique_ptr builder_schema_; + int64_t builder_num_samples_; + std::string builder_dataset_type_; + }; + + // Constructor + // @param int32_t - num_workers - Num of workers reading images in parallel + // @param int32_t - rows_per_buffer Number of images (rows) in each buffer + // @param std::string - dir directory of celeba dataset + // @param int32_t queueSize - connector queue size + // @param std::unique_ptr sampler - sampler tells CelebAOp what to read + CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::string &dir, int32_t queue_size, bool decode, + const std::string &dataset_type, const std::set &exts, std::unique_ptr schema, + std::shared_ptr sampler, int64_t num_samples); + + ~CelebAOp() override = default; + + // Main Loop of CelebaOp + // Master thread: Fill IOBlockQueue, then goes to sleep + // Worker thread: pulls IOBlock from IOBlockQueue, work on it then put buffer to mOutConnector + // @return Status - The error code return + Status operator()() override; + + // Method derived from RandomAccess Op, enable Sampler to get numRows + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumSamples(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get numRows + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumRowsInDataset(int64_t *num) const override; + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t worker_id - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // A print method typically used for debugging + // @param out + // @param show_all + void Print(std::ostream &out, bool show_all) const override; + + // Method in operator(), to fill IOBlockQueue + // @param std::unique_ptr sampler_buffer - to fill IOBlockQueue + // @return Status - The error code return + Status AddIOBlock(std::unique_ptr *data_buffer); + + private: + // Called first when function is called + // @return + Status LaunchThreadsAndInitOp(); + + // Parse attribute file + // @return + Status ParseAttrFile(); + + // Parse each image line in attribute file + // @return + Status ParseImageAttrInfo(); + + // Split attribute info with space + // @param std::string - line - Line from att or partition file + // @return std::vector - string after split + std::vector Split(const std::string &line); + + // @param const std::vector &keys - keys in ioblock + // @param std::unique_ptr db + // @return Status - The error code return + Status LoadBuffer(const std::vector &keys, std::unique_ptr *db); + + // Load a tensor row according to a pair + // @param std::pair - > + // @param TensorRow row - image & label read into this tensor row + // @return Status - The error code return + Status LoadTensorRow(const std::pair> &image_label, TensorRow *row); + + // Check if need read according to dataset type + // @return bool - if need read + bool CheckDatasetTypeValid(); + + // reset Op + // @return Status - The error code return + Status Reset() override; + + int32_t rows_per_buffer_; + std::string folder_path_; // directory of celeba folder + bool decode_; + std::set extensions_; // extensions allowed + std::unique_ptr data_schema_; + std::shared_ptr sampler_; + std::unordered_map col_name_map_; + std::unique_ptr>> attr_info_queue_; + int64_t num_rows_in_attr_file_; // rows number specified in attr file + int64_t num_rows_exact_; // exact rows number,maybe is less than rows_num_in_attr_file_ + QueueList> io_block_queues_; + WaitPost wp_; + std::vector>> image_labels_vec_; + int64_t num_samples_; + std::string dataset_type_; + std::ifstream partition_file_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_CELEBA_OP_H diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc new file mode 100644 index 00000000000..260a4a4dc5d --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -0,0 +1,458 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/cifar_op.h" + +#include +#include +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { +constexpr uint32_t kCifarImageHeight = 32; +constexpr uint32_t kCifarImageWidth = 32; +constexpr uint32_t kCifarImageChannel = 3; +constexpr uint32_t kCifarBlockImageNum = 5; +constexpr uint32_t kCifarImageSize = kCifarImageHeight * kCifarImageWidth * kCifarImageChannel; + +CifarOp::Builder::Builder() : num_samples_(0), sampler_(nullptr) { + std::shared_ptr cfg = GlobalContext::config_manager(); + num_workers_ = cfg->num_parallel_workers(); + rows_per_buffer_ = cfg->rows_per_buffer(); + op_connect_size_ = cfg->op_connector_size(); + cifar_type_ = kCifar10; +} + +Status CifarOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + if (sampler_ == nullptr) { + sampler_ = std::make_shared(); + } + schema_ = make_unique(); + TensorShape scalar = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK(schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); + if (cifar_type_ == kCifar10) { + RETURN_IF_NOT_OK( + schema_->AddColumn(ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar))); + } else { + RETURN_IF_NOT_OK(schema_->AddColumn( + ColDescriptor("coarse_label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar))); + TensorShape another_scalar = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK(schema_->AddColumn( + ColDescriptor("fine_label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &another_scalar))); + } + + *ptr = std::make_shared(cifar_type_, num_workers_, rows_per_buffer_, dir_, op_connect_size_, num_samples_, + std::move(schema_), std::move(sampler_)); + return Status::OK(); +} + +Status CifarOp::Builder::SanityCheck() { + Path dir(dir_); + std::string err_msg; + err_msg += dir.IsDirectory() == false ? "Cifar path is invalid or not set\n" : ""; + err_msg += num_workers_ <= 0 ? "Num of parallel workers is negative or 0\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const std::string &file_dir, + int32_t queue_size, int64_t num_samples, std::unique_ptr data_schema, + std::shared_ptr sampler) + : ParallelOp(num_works, queue_size), + cifar_type_(type), + rows_per_buffer_(rows_per_buf), + folder_path_(file_dir), + num_samples_(num_samples), + data_schema_(std::move(data_schema)), + sampler_(std::move(sampler)), + num_rows_(0), + row_cnt_(0), + buf_cnt_(0) { + for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { + col_name_map_[data_schema_->column(i).name()] = i; + } + constexpr uint64_t kUtilQueueSize = 512; + cifar_raw_data_block_ = make_unique>>(kUtilQueueSize); + io_block_queues_.Init(num_workers_, queue_size); +} + +// Main logic, Register Queue with TaskGroup, launch all threads and do the functor's work +Status CifarOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + std::unique_ptr sampler_buffer; + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + while (true) { // each iterator is 1 epoch + std::vector keys; + keys.reserve(rows_per_buffer_); + while (sampler_buffer->eoe() == false) { + TensorRow sample_row; + RETURN_IF_NOT_OK(sampler_buffer->PopRow(&sample_row)); + std::shared_ptr sample_ids = sample_row[0]; + for (auto itr = sample_ids->begin(); itr != sample_ids->end(); itr++) { + keys.push_back(*itr); + row_cnt_++; + if ((*itr) >= num_rows_) continue; // index out of bound, skipping + if (row_cnt_ >= num_samples_) break; // enough row read, break for loop + if (row_cnt_ % rows_per_buffer_ == 0) { + RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + keys.clear(); + } + } + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + if (keys.empty() == false) { + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEof))); + for (int32_t i = 0; i < num_workers_; i++) { + RETURN_IF_NOT_OK( + io_block_queues_[i]->Add(make_unique(std::vector(), IOBlock::kDeIoBlockNone))); + } + return Status::OK(); + } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks + wp_.Clear(); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + } +} + +Status CifarOp::LaunchThreadsAndInitOp() { + if (tree_ == nullptr) { + RETURN_STATUS_UNEXPECTED("tree_ not set"); + } + RETURN_IF_NOT_OK(io_block_queues_.Register(tree_->AllTasks())); + wp_.Register(tree_->AllTasks()); + RETURN_IF_NOT_OK( + tree_->AllTasks()->CreateAsyncTask("Get cifar data block", std::bind(&CifarOp::ReadCifarBlockDataAsync, this))); + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&CifarOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + // The order of the following 2 functions must not be changed! + RETURN_IF_NOT_OK(ParseCifarData()); // Parse cifar data and get num rows, blocking + RETURN_IF_NOT_OK(InitSampler()); // Pass numRows to Sampler + return Status::OK(); +} + +// contains the main logic of pulling a IOBlock from IOBlockQueue, load a buffer and push the buffer to out_connector_ +// IMPORTANT: 1 IOBlock produces 1 DataBuffer +Status CifarOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + int64_t buffer_id = worker_id; + std::unique_ptr io_block; + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + while (io_block != nullptr) { + if (io_block->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOE))); + buffer_id = worker_id; + } else if (io_block->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOF))); + } else { + std::vector keys; + RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); + if (keys.empty() == true) { + return Status::OK(); // empty key is a quit signal for workers + } + std::unique_ptr db = make_unique(buffer_id, DataBuffer::kDeBFlagNone); + RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); + buffer_id += num_workers_; + } + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + } + RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); +} + +// Load 1 TensorRow (image,label). 1 function call produces 1 TensorTow in a DataBuffer +Status CifarOp::LoadTensorRow(uint64_t index, TensorRow *trow) { + std::shared_ptr label; + std::shared_ptr fine_label; + std::shared_ptr ori_image = cifar_image_label_pairs_[index].first; + std::shared_ptr copy_image = + std::make_shared(ori_image->shape(), ori_image->type(), ori_image->StartAddr()); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(), + data_schema_->column(1).type(), + reinterpret_cast(&cifar_image_label_pairs_[index].second[0]))); + if (cifar_image_label_pairs_[index].second.size() > 1) { + RETURN_IF_NOT_OK(Tensor::CreateTensor( + &fine_label, data_schema_->column(2).tensorImpl(), data_schema_->column(2).shape(), + data_schema_->column(2).type(), reinterpret_cast(&cifar_image_label_pairs_[index].second[1]))); + (*trow) = {copy_image, std::move(label), std::move(fine_label)}; + } else { + (*trow) = {copy_image, std::move(label)}; + } + + return Status::OK(); +} + +// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer +Status CifarOp::LoadBuffer(const std::vector &keys, std::unique_ptr *db) { + std::unique_ptr deq = make_unique(); + for (const int64_t &key : keys) { + TensorRow trow; + RETURN_IF_NOT_OK(LoadTensorRow(key, &trow)); + deq->push_back(std::move(trow)); + } + (*db)->set_tensor_table(std::move(deq)); + (*db)->set_column_name_map(col_name_map_); + return Status::OK(); +} + +void CifarOp::Print(std::ostream &out, bool show_all) const { + DatasetOp::Print(out, show_all); + out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ + << "\nCifar Directory: " << folder_path_ << "\n-------------------------\n"; +} + +// Reset Sampler and wakeup Master thread (functor) +Status CifarOp::Reset() { + RETURN_IF_NOT_OK(sampler_->Reset()); + row_cnt_ = 0; + wp_.Set(); // wake up master thread after reset is done + return Status::OK(); +} + +// hand shake with Sampler, allow Sampler to call RandomAccessOp's functions to get NumRows +Status CifarOp::InitSampler() { + RETURN_IF_NOT_OK(sampler_->Init(this)); + return Status::OK(); +} + +// Derived from RandomAccessOp +Status CifarOp::GetNumSamples(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_samples_; + return Status::OK(); +} + +// Derived from RandomAccessOp +Status CifarOp::GetNumRowsInDataset(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_rows_; + return Status::OK(); +} + +Status CifarOp::ReadCifarBlockDataAsync() { + TaskManager::FindMe()->Post(); + RETURN_IF_NOT_OK(GetCifarFiles()); + if (cifar_type_ == kCifar10) { + RETURN_IF_NOT_OK(ReadCifar10BlockData()); + } else { + RETURN_IF_NOT_OK(ReadCifar100BlockData()); + } + + return Status::OK(); +} + +Status CifarOp::ReadCifar10BlockData() { + constexpr uint32_t num_cifar10_records = 10000; + uint32_t block_size = (kCifarImageSize + 1) * kCifarBlockImageNum; // about 2M + std::vector image_data(block_size * sizeof(unsigned char), 0); + for (auto &file : cifar_files_) { + std::ifstream in(file, std::ios::binary); + if (!in.is_open()) { + std::string err_msg = file + " can not be opened."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + for (uint32_t index = 0; index < num_cifar10_records / kCifarBlockImageNum; ++index) { + (void)in.read(reinterpret_cast(&(image_data[0])), block_size * sizeof(unsigned char)); + if (in.fail()) { + RETURN_STATUS_UNEXPECTED("Fail to read cifar file" + file); + } + (void)cifar_raw_data_block_->EmplaceBack(image_data); + } + in.close(); + } + (void)cifar_raw_data_block_->EmplaceBack(std::vector()); // end block + + return Status::OK(); +} + +Status CifarOp::ReadCifar100BlockData() { + uint32_t num_cifar100_records = 0; // test:10000, train:50000 + uint32_t block_size = (kCifarImageSize + 2) * kCifarBlockImageNum; // about 2M + std::vector image_data(block_size * sizeof(unsigned char), 0); + for (auto &file : cifar_files_) { + int pos = file.find_last_of('/'); + if (pos == std::string::npos) { + RETURN_STATUS_UNEXPECTED("Invalid cifar100 file path"); + } + std::string file_name(file.substr(pos + 1)); + if (file_name.find("test") != std::string::npos) { + num_cifar100_records = 10000; + } else if (file_name.find("train") != std::string::npos) { + num_cifar100_records = 50000; + } else { + RETURN_STATUS_UNEXPECTED("Cifar 100 file not found!"); + } + + std::ifstream in(file, std::ios::binary); + if (!in.is_open()) { + RETURN_STATUS_UNEXPECTED(file + " can not be opened."); + } + + for (uint32_t index = 0; index < num_cifar100_records / kCifarBlockImageNum; index++) { + (void)in.read(reinterpret_cast(&(image_data[0])), block_size * sizeof(unsigned char)); + if (in.fail()) { + RETURN_STATUS_UNEXPECTED("Fail to read cifar file" + file); + } + (void)cifar_raw_data_block_->EmplaceBack(image_data); + } + in.close(); + } + (void)cifar_raw_data_block_->EmplaceBack(std::vector()); // block end + return Status::OK(); +} + +Status CifarOp::GetCifarFiles() { + // Initialize queue to hold the file names + const std::string kExtension = ".bin"; + Path dataset_directory(folder_path_); + auto dirIt = Path::DirIterator::OpenDirectory(&dataset_directory); + if (dirIt) { + while (dirIt->hasNext()) { + Path file = dirIt->next(); + std::string filename = file.toString(); + if (filename.find(kExtension) != std::string::npos) { + cifar_files_.push_back(filename); + MS_LOG(INFO) << "Cifar operator found file at " << filename << "."; + } + } + } else { + std::string err_msg = "Unable to open directory " + dataset_directory.toString(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::sort(cifar_files_.begin(), cifar_files_.end()); + return Status::OK(); +} + +Status CifarOp::ParseCifarData() { + std::vector block; + RETURN_IF_NOT_OK(cifar_raw_data_block_->PopFront(&block)); + uint32_t cur_block_index = 0; + while (!block.empty()) { + for (uint32_t index = 0; index < kCifarBlockImageNum; ++index) { + std::vector labels; + uint32_t label = block[cur_block_index++]; + labels.push_back(label); + if (cifar_type_ == kCifar100) { + uint32_t fine_label = block[cur_block_index++]; + labels.push_back(fine_label); + } + + std::shared_ptr image_tensor; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image_tensor, data_schema_->column(0).tensorImpl(), + TensorShape({kCifarImageHeight, kCifarImageWidth, kCifarImageChannel}), + data_schema_->column(0).type())); + for (int ch = 0; ch < kCifarImageChannel; ++ch) { + for (int pix = 0; pix < kCifarImageHeight * kCifarImageWidth; ++pix) { + (image_tensor->StartAddr())[pix * kCifarImageChannel + ch] = block[cur_block_index++]; + } + } + cifar_image_label_pairs_.emplace_back(std::make_pair(image_tensor, labels)); + } + RETURN_IF_NOT_OK(cifar_raw_data_block_->PopFront(&block)); + cur_block_index = 0; + } + cifar_image_label_pairs_.shrink_to_fit(); + num_rows_ = cifar_image_label_pairs_.size(); + num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; + if (num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("Init Cifar failed, not a single row read from dataset!"); + } + cifar_raw_data_block_->Reset(); + return Status::OK(); +} + +// Derived from RandomAccessOp +Status CifarOp::GetClassIds(std::map> *cls_ids) const { + if (cls_ids == nullptr || !cls_ids->empty()) { + RETURN_STATUS_UNEXPECTED("ImageLabelPair not set"); + } + + for (uint64_t index = 0; index < cifar_image_label_pairs_.size(); ++index) { + uint32_t label = (cifar_image_label_pairs_[index].second)[0]; + (*cls_ids)[label].push_back(index); + } + + for (auto &pair : (*cls_ids)) { + pair.second.shrink_to_fit(); + } + return Status::OK(); +} + +Status CifarOp::CountTotalRows(const std::string &dir, int64_t numSamples, bool isCIFAR10, int64_t *count) { + // the logic of counting the number of samples is copied from ReadCifar100Block() and ReadCifar10Block() + std::shared_ptr op; + *count = 0; + RETURN_IF_NOT_OK(Builder().SetCifarDir(dir).SetNumSamples(numSamples).SetCifarType(isCIFAR10).Build(&op)); + RETURN_IF_NOT_OK(op->GetCifarFiles()); + if (op->cifar_type_ == kCifar10) { + constexpr int64_t num_cifar10_records = 10000; + for (auto &file : op->cifar_files_) { + std::ifstream in(file, std::ios::binary); + if (!in.is_open()) { + std::string err_msg = file + " can not be opened."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + *count = *count + num_cifar10_records; + } + *count = *count < numSamples || numSamples == 0 ? *count : numSamples; + return Status::OK(); + } else { + int64_t num_cifar100_records = 0; + for (auto &file : op->cifar_files_) { + size_t pos = file.find_last_of('/'); + if (pos == std::string::npos) { + std::string err_msg = "Invalid cifar100 file path"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::string file_name(file.substr(pos + 1)); + if (file_name.find("test") != std::string::npos) { + num_cifar100_records = 10000; + } else if (file_name.find("train") != std::string::npos) { + num_cifar100_records = 50000; + } + std::ifstream in(file, std::ios::binary); + if (!in.is_open()) { + std::string err_msg = file + " can not be opened."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + *count = num_cifar100_records < numSamples || numSamples == 0 ? num_cifar100_records : numSamples; + return Status::OK(); + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h new file mode 100644 index 00000000000..3cb26e56798 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h @@ -0,0 +1,247 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_CIFAR_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_CIFAR_OP_H_ + +#include +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/util/path.h" +#include "dataset/util/queue.h" +#include "dataset/util/services.h" +#include "dataset/util/status.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +class CifarOp : public ParallelOp, public RandomAccessOp { + public: + enum CifarType { kCifar10, kCifar100 }; + + class Builder { + public: + // Constructor for Builder class of CifarOp + // @return Builder setter method returns reference to the builder. + Builder(); + + // Destructor. + ~Builder() = default; + + // Setter method + // @param uint32_t rows_per_buffer + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method + // @param uint32_t size + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t size) { + op_connect_size_ = size; + return *this; + } + + // Setter method + // @param uint32_t num_workers + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + num_workers_ = num_workers; + return *this; + } + + // Setter method + // @param uint64_t num_samples + // @return Builder setter method returns reference to the builder. + Builder &SetNumSamples(uint64_t num_samples) { + num_samples_ = num_samples; + return *this; + } + + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + sampler_ = std::move(sampler); + return *this; + } + + // Setter method + // @param const std::string & dir + // @return + Builder &SetCifarDir(const std::string &dir) { + dir_ = dir; + return *this; + } + + // Setter method + // @param const std::string & dir + // @return + Builder &SetCifarType(const bool cifar10) { + if (cifar10) { + cifar_type_ = kCifar10; + } else { + cifar_type_ = kCifar100; + } + return *this; + } + + // Check validity of input args + // @return - The error code return + Status SanityCheck(); + + // The builder "build" method creates the final object. + // @param std::shared_ptr *op - DatasetOp + // @return - The error code return + Status Build(std::shared_ptr *op); + + private: + std::string dir_; + int32_t num_workers_; + uint64_t num_samples_; + int32_t rows_per_buffer_; + int32_t op_connect_size_; + std::shared_ptr sampler_; + std::unique_ptr schema_; + CifarType cifar_type_; + }; + + // Constructor + // @param CifarType type - Cifar10 or Cifar100 + // @param uint32_t numWorks - Num of workers reading images in parallel + // @param uint32_t - rowsPerBuffer Number of images (rows) in each buffer + // @param std::string - dir directory of cifar dataset + // @param uint32_t - queueSize - connector queue size + // @param std::unique_ptr sampler - sampler tells ImageFolderOp what to read + CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const std::string &file_dir, int32_t queue_size, + int64_t num_samples, std::unique_ptr data_schema, std::shared_ptr sampler); + // Destructor. + ~CifarOp() = default; + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param uint32_t workerId - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Main Loop of CifarOp + // Master thread: Fill IOBlockQueue, then goes to sleep + // Worker thread: pulls IOBlock from IOBlockQueue, work on it then put buffer to mOutConnector + // @return Status - The error code return + Status operator()() override; + + // Method derived from RandomAccess Op, enable Sampler to get numRows + // @param uint64_t num - to return numRows + // @return Status - The error code return + Status GetNumSamples(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get total numRows in dataset + // @param uint64_t num - to return numRows + // @return Status - The error code return + Status GetNumRowsInDataset(int64_t *num) const override; + + // A print method typically used for debugging + // @param out + // @param show_all + void Print(std::ostream &out, bool show_all) const override; + + // Function to count the number of samples in the CIFAR dataset + // @param dir path to the CIFAR directory + // @param numSamples maximum number of samples requested + // @param isCIFAR10 true if CIFAR10 and false if CIFAR100 + // @param count output arg that will hold the minimum of the actual dataset size and numSamples + // @return + static Status CountTotalRows(const std::string &dir, int64_t numSamples, bool isCIFAR10, int64_t *count); + + private: + // Initialize Sampler, calls sampler->Init() within + // @return Status - The error code return + Status InitSampler(); + + // Load a tensor row according to a pair + // @param uint64_t index - index need to load + // @param TensorRow row - image & label read into this tensor row + // @return Status - The error code return + Status LoadTensorRow(uint64_t index, TensorRow *row); + + // @param const std::vector &keys - keys in ioblock + // @param std::unique_ptr db + // @return Status - The error code return + Status LoadBuffer(const std::vector &keys, std::unique_ptr *db); + + // Read block data from cifar file + // @return + Status ReadCifarBlockDataAsync(); + + // Called first when function is called + // @return + Status LaunchThreadsAndInitOp(); + + // reset Op + // @return Status - The error code return + Status Reset() override; + + // Get cifar files in dir + // @return + Status GetCifarFiles(); + + // Read cifar10 data as block + // @return + Status ReadCifar10BlockData(); + + // Read cifar100 data as block + // @return + Status ReadCifar100BlockData(); + + // Parse cifar data + // @return + Status ParseCifarData(); + + // Method derived from RandomAccess Op, enable Sampler to get all ids for each calss + // @param (std::unordered_map> * map - key label, val all ids for this class + // @return Status - The error code return + Status GetClassIds(std::map> *cls_ids) const override; + + CifarType cifar_type_; + int32_t rows_per_buffer_; + std::string folder_path_; + int64_t num_samples_; + std::unique_ptr data_schema_; + std::shared_ptr sampler_; + int64_t num_rows_; + int64_t row_cnt_; + int64_t buf_cnt_; + + WaitPost wp_; + std::unordered_map col_name_map_; + QueueList> io_block_queues_; + std::unique_ptr>> cifar_raw_data_block_; + std::vector cifar_files_; + std::vector, std::vector>> cifar_image_label_pairs_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_CIFAR_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/example.proto b/mindspore/ccsrc/dataset/engine/datasetops/source/example.proto new file mode 100644 index 00000000000..eb893f4be5f --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/example.proto @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Protocol messages for describing input data Examples for machine learning +// model training or inference. +syntax = "proto3"; + +import "feature.proto"; +option cc_enable_arenas = true; +option java_outer_classname = "ExampleProtos"; +option java_multiple_files = true; +option java_package = "org.dataengine.example"; + +package dataengine; + +message Example { + Features features = 1; +}; + +message SequenceExample { + Features context = 1; + FeatureLists feature_lists = 2; +}; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/feature.proto b/mindspore/ccsrc/dataset/engine/datasetops/source/feature.proto new file mode 100644 index 00000000000..03c0f0938bd --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/feature.proto @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +option cc_enable_arenas = true; +option java_multiple_files = true; +option java_package = "org.dataengine.example"; +option java_outer_classname = "FeatureProtos"; + +package dataengine; + +// define Int64 +message Int64List { + repeated int64 value = 1 [packed = true]; +} + +// define Float +message FloatList { + repeated float value = 1 [packed = true]; +} + +//define Bytes +message BytesList { + repeated bytes value = 1; +} + +// define Feature +message Feature { + oneof kind { + BytesList bytes_list = 1; + FloatList float_list = 2; + Int64List int64_list = 3; + } +}; + +//define Features as Feature map +message Features { + map feature = 1; +}; + +// define FeatureList +message FeatureList { + repeated Feature feature = 1; +}; + +// define FeatureLists as FeatureList map +message FeatureLists { + map feature_list = 1; +}; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc new file mode 100644 index 00000000000..ceb88ceb0e9 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc @@ -0,0 +1,234 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/generator_op.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +GeneratorOp::Builder::Builder() { + // Some arguments to the StorageOp constructor have a default argument that is taken + // from the client config. + build_buffer_size_ = kCfgRowsPerBuffer; + build_op_connector_size_ = kCfgOpConnectorSize; +} + +Status GeneratorOp::Builder::SanityCheck() { + // Update queue size to fit the prefetch requirement + MS_LOG(INFO) << "Generator operator sanity check, prefetch size is " << build_prefetch_size_ << "."; + if (build_prefetch_size_ > 0) { + build_op_connector_size_ = (build_prefetch_size_ + build_buffer_size_ - 1) / build_buffer_size_; + } + return Status::OK(); +} + +Status GeneratorOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(build_generator_function_, build_column_names_, build_column_types_, + build_prefetch_size_, build_buffer_size_, build_op_connector_size_); + return (*ptr)->Init(); +} + +GeneratorOp::GeneratorOp(py::function generator_function, std::vector column_names, + std::vector column_types, int32_t prefetch_size, int32_t buffer_size, + int32_t connector_size) + : PipelineOp(connector_size), + generator_function_(generator_function), + column_names_(column_names), + column_types_(column_types), + prefetch_size_(prefetch_size), + buffer_size_(buffer_size), + buffer_id_(0) {} + +GeneratorOp::~GeneratorOp() { this->Dealloc(); } + +void GeneratorOp::Dealloc() noexcept { + // Setup GIL state + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + // GC the generator object within GIL + (void)generator_.dec_ref(); + // Release GIL + PyGILState_Release(gstate); +} + +// Reentrant init method. +Status GeneratorOp::Init() { + // Reset BufferID + buffer_id_ = 0; + // Setup column names map. + if (column_names_map_.empty()) { + for (int i = 0; i < column_names_.size(); ++i) { + (void)column_names_map_.insert(std::make_pair(column_names_[i], i)); + } + } + Status ret; + { + // Acquire Python GIL + py::gil_scoped_acquire gil_acquire; + if (Py_IsInitialized() == 0) { + return Status(StatusCode::kPythonInterpreterFailure, "Python Interpreter is finalized"); + } + // Invoke the generatorFunction to get generator object + try { + generator_ = generator_function_(); + } catch (const py::error_already_set &e) { + ret = Status(StatusCode::kPyFuncException, e.what()); + } + } + return ret; +} + +Status GeneratorOp::PyRowToTensorRow(py::object py_data, TensorRow *tensor_row) { + if (!py::isinstance(py_data)) { + return Status(StatusCode::kPyFuncException, __LINE__, __FILE__, "Generator should return a tuple of numpy arrays."); + } + py::tuple py_row = py_data.cast(); + // Check if returned number of columns matches with column names + if (py_row.size() != column_names_.size()) { + return Status(StatusCode::kPyFuncException, __LINE__, __FILE__, + "Generator should return same number of numpy arrays as specified in column names."); + } + // Iterate over two containers simultaneously for memory copy + for (int i = 0; i < py_row.size(); ++i) { + py::object ret_py_ele = py_row[i]; + if (!py::isinstance(ret_py_ele)) { + return Status(StatusCode::kPyFuncException, __LINE__, __FILE__, + "Generator should return a tuple of numpy arrays."); + } + std::shared_ptr tensor; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&tensor, ret_py_ele.cast())); + if ((!column_types_.empty()) && (column_types_[i] != DataType::DE_UNKNOWN) && + (column_types_[i] != tensor->type())) { + return Status(StatusCode::kPyFuncException, __LINE__, __FILE__, "Generator type check failed."); + } + tensor_row->push_back(tensor); + } + return Status(StatusCode::kOK, ""); +} + +Status GeneratorOp::FillBuffer(TensorQTable *tt) { + for (int i = 0; i < buffer_size_; i++) { + TensorRow row; + RETURN_IF_NOT_OK(PyRowToTensorRow(generator_.attr("__next__")(), &row)); + tt->push_back(std::move(row)); + } + return Status::OK(); +} + +// Entry point for Generator, called by launch() +// Note that this function is very easy to break because of the Python GIL mechanism +// The master thread has the following workflow +// +// while !eof: +// Try: +// Prepare one data buffer GIL, Can throw +// Catch: +// Fetch Python Exception GIL +// Check if Exception is StopIteration (EOE) GIL +// Restore Python Exception GIL +// If not StopIteration: +// Return Status PyFuncException +// +// Push data buffer to connector Block +// +// if EOE +// Push EOE Block +// if more epoch: +// Block until next epoch Block +// else: +// Push EOF Block +// eof = true +// Return Status OK +// +// Note that any modification of this function need to guarantee: +// 1. All "Require GIL" operations are protected by GIL +// SegFault / Deadlock will occur if this condition is not fulfilled. +// 2. All "Block" operations are free from GIL, all block target are registered with tree. +// Deadlock will occur if this condition is not fulfilled +// 3. No Python GC should be triggered outside of GIL. +// SegFault will occur is this condition is not fulfilled +// +Status GeneratorOp::operator()() { + // Handshake with TaskManager to synchronize thread creation + TaskManager::FindMe()->Post(); + wp_.Register(tree_->AllTasks()); + std::unique_ptr fetched_buffer; + bool eof = false; + while (!eof) { + // Create new buffer each iteration + fetched_buffer = mindspore::make_unique(buffer_id_++, DataBuffer::kDeBFlagNone); + fetched_buffer->set_column_name_map(column_names_map_); + std::unique_ptr fetched_table = mindspore::make_unique(); + bool eoe = false; + { + py::gil_scoped_acquire gil_acquire; + if (Py_IsInitialized() == 0) { + return Status(StatusCode::kPythonInterpreterFailure, "Python Interpreter is finalized"); + } + try { + RETURN_IF_NOT_OK(FillBuffer(fetched_table.get())); + } catch (py::error_already_set &e) { + eoe = e.matches(PyExc_StopIteration); + // Restore exception to python + e.restore(); + // Pop up non StopIteration Python Exception + if (!eoe) { + return Status(StatusCode::kPyFuncException, __LINE__, __FILE__, e.what()); + } + } + } + if (fetched_table->size() > 0) { + fetched_buffer->set_tensor_table(std::move(fetched_table)); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(fetched_buffer))); + } + if (eoe) { + // Push out EOE upon StopIteration exception from generator + MS_LOG(INFO) << "Generator operator sends out EOE."; + std::unique_ptr eoe_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + // If last repeat or not repeated, push out EOF and exit master loop + MS_LOG(INFO) << "Generator operator sends out EOF."; + std::unique_ptr eof_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOF); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); + MS_LOG(INFO) << "Generator operator main execution loop complete."; + eof = true; + } else { + // Waiting for repeatOp to start new epoch + // If Reset() is called first by repeat op, this wait() will return right away. + // If Reset() is not called yet, this wait() will block until reset. + RETURN_IF_NOT_OK(wp_.Wait()); + // Clear the status of the wait post + wp_.Clear(); + } + } + } + return Status::OK(); +} + +Status GeneratorOp::Reset() { + // Reset Op state + RETURN_IF_NOT_OK(this->Init()); + // Wake up master thread + wp_.Set(); + return Status(StatusCode::kOK, "GeneratorOp Reset Succeed"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h new file mode 100644 index 00000000000..a5407a9b096 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h @@ -0,0 +1,146 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_GENERATOR_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_GENERATOR_OP_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/pipeline_op.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +#pragma GCC visibility push(hidden) + +class GeneratorOp : public PipelineOp { + public: + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + ~Builder() = default; + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetGeneratorFunction(py::function generator_function) { + build_generator_function_ = generator_function; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetColumnNames(const std::vector &column_names) { + build_column_names_ = column_names; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetColumnTypes(const std::vector &column_types) { + build_column_types_ = column_types; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetPrefetchSize(int32_t prefetch_size) { + build_prefetch_size_ = prefetch_size; + return *this; + } + + // The builder "build" method creates the final object. + // @return shared_ptr to the new StorageOp object + Status Build(std::shared_ptr *); + + private: + // The builder saves all GeneratorOp construction arguments internally. + // The following are the arguments. + py::function build_generator_function_; + std::vector build_column_names_; + std::vector build_column_types_; + + int32_t build_prefetch_size_ = 0; + int32_t build_buffer_size_; + int32_t build_op_connector_size_; + + Status SanityCheck(); + }; + + GeneratorOp(py::function generator_function, std::vector column_names, + std::vector column_types, int32_t prefetch_size, int32_t buffer_size, int32_t connector_size); + + ~GeneratorOp(); + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param generator_op - reference to the GeneratorOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const GeneratorOp &generator_op) { + generator_op.Print(out, false); + return out; + } + + // Class functor operator () override. + // All DatasetOps operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work. + // @return Status - The error code return + Status operator()() override; + + // Overrides base class reset method. When an operator does a reset, it cleans up any state + // info from it's previous execution and then initializes itself so that it can be executed + // again. + // @return Status - The error code return + Status Reset() override; + + private: + py::function generator_function_; + std::vector column_names_; + std::vector column_types_; + int32_t prefetch_size_; + int32_t buffer_size_; + + py::object generator_; + std::unordered_map column_names_map_; + int32_t buffer_id_; + + WaitPost wp_; + + Status Init(); + + void Dealloc() noexcept; + + Status PyRowToTensorRow(py::object py_data, TensorRow *tensor_row); + + Status FillBuffer(TensorQTable *tt); +}; + +#pragma GCC visibility pop +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_GENERATOR_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc new file mode 100644 index 00000000000..b8044fb38ab --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -0,0 +1,442 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/image_folder_op.h" + +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { +ImageFolderOp::Builder::Builder() + : builder_decode_(false), builder_recursive_(false), builder_num_samples_(0), builder_sampler_(nullptr) { + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_num_workers_ = cfg->num_parallel_workers(); + builder_rows_per_buffer_ = cfg->rows_per_buffer(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status ImageFolderOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + if (builder_sampler_ == nullptr) { + builder_sampler_ = std::make_shared(); + } + builder_schema_ = make_unique(); + TensorShape scalar = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); + RETURN_IF_NOT_OK(builder_schema_->AddColumn( + ColDescriptor("label", DataType(DataType::DE_INT32), TensorImpl::kFlexible, 0, &scalar))); + *ptr = std::make_shared(builder_num_workers_, builder_rows_per_buffer_, builder_dir_, + builder_op_connector_size_, builder_num_samples_, builder_recursive_, + builder_decode_, builder_extensions_, builder_labels_to_read_, + std::move(builder_schema_), std::move(builder_sampler_)); + return Status::OK(); +} + +Status ImageFolderOp::Builder::SanityCheck() { + Path dir(builder_dir_); + std::string err_msg; + err_msg += dir.IsDirectory() == false ? "ImageFolder path is invalid or not set\n" : ""; + err_msg += builder_num_workers_ <= 0 ? "Num of parallel workers is set to 0\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::string file_dir, int32_t queue_size, + int64_t num_samples, bool recursive, bool do_decode, const std::set &exts, + const std::map &map, std::unique_ptr data_schema, + std::shared_ptr sampler) + : ParallelOp(num_wkrs, queue_size), + rows_per_buffer_(rows_per_buffer), + folder_path_(file_dir), + num_samples_(num_samples), + recursive_(recursive), + decode_(do_decode), + extensions_(exts), + class_index_(map), + data_schema_(std::move(data_schema)), + sampler_(std::move(sampler)), + num_rows_(0), + row_cnt_(0), + buf_cnt_(0), + sampler_ind_(0), + dirname_offset_(0) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + col_name_map_[data_schema_->column(i).name()] = i; + } + folder_name_queue_ = make_unique>(num_wkrs * queue_size); + image_name_queue_ = make_unique>(num_wkrs * queue_size); + io_block_queues_.Init(num_workers_, queue_size); +} + +// Master thread that pulls the prescan worker's results. +// Keep collecting results until all prescan workers quit +// Then consolidate 2 level shuffles together into 1 giant vector +// calculate numRows then return +Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) { + std::vector v; + int64_t cnt = 0; + while (cnt != num_workers_) { // count number of end signals + FolderImagesPair p; + RETURN_IF_NOT_OK(image_name_queue_->PopFront(&p)); + if (p == nullptr) { + cnt++; + } else { + v.push_back(p); + } + } + std::sort(v.begin(), v.end(), + [](const FolderImagesPair &lhs, const FolderImagesPair &rhs) { return lhs->first < rhs->first; }); + // following loop puts the 2 level of shuffles together into 1 vector + for (size_t ind = 0; ind < v.size(); ++ind) { + while (v[ind]->second.empty() == false) { + DS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound + v[ind]->second.front()->second = class_index_.empty() ? ind : class_index_[v[ind]->first.substr(1)]; + image_label_pairs_.push_back(v[ind]->second.front()); + v[ind]->second.pop(); + } + } + image_label_pairs_.shrink_to_fit(); + num_rows_ = image_label_pairs_.size(); + num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; + // free memory of two queues used for pre-scan + folder_name_queue_->Reset(); + image_name_queue_->Reset(); + return Status::OK(); +} + +// Main logic, Register Queue with TaskGroup, launch all threads and do the functor's work +Status ImageFolderOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + std::unique_ptr sampler_buffer; + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + while (true) { // each iterator is 1 epoch + std::vector keys; + keys.reserve(rows_per_buffer_); + while (sampler_buffer->eoe() == false) { + TensorRow sample_row; + RETURN_IF_NOT_OK(sampler_buffer->PopRow(&sample_row)); + std::shared_ptr sample_ids = sample_row[0]; + if (sample_ids->type() != DataType(DataType::DE_INT64)) RETURN_STATUS_UNEXPECTED("Sampler Tensor isn't int64"); + for (auto itr = sample_ids->begin(); itr != sample_ids->end(); ++itr) { + if ((*itr) >= num_rows_) continue; // index out of bound, skipping + if (row_cnt_ >= num_samples_) break; // enough row read, break for loop + keys.push_back(*itr); + row_cnt_++; + if (row_cnt_ % rows_per_buffer_ == 0) { + RETURN_IF_NOT_OK( + io_block_queues_[buf_cnt_++ % num_workers_]->Add(make_unique(keys, IOBlock::kDeIoBlockNone))); + keys.clear(); + } + } + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + if (keys.empty() == false) { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(keys, IOBlock::kDeIoBlockNone))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + std::unique_ptr eoe_block = make_unique(IOBlock::kDeIoBlockFlagEoe); + std::unique_ptr eof_block = make_unique(IOBlock::kDeIoBlockFlagEof); + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); + for (int32_t i = 0; i < num_workers_; ++i) { + RETURN_IF_NOT_OK( + io_block_queues_[i]->Add(make_unique(std::vector(), IOBlock::kDeIoBlockNone))); + } + return Status::OK(); + } else { // not the last repeat. Sleep master thread, wait for the wake-up from reset + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks + wp_.Clear(); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + } +} + +// contains the main logic of pulling a IOBlock from IOBlockQueue, load a buffer and push the buffer to out_connector_ +// IMPORTANT: 1 IOBlock produces 1 DataBuffer +Status ImageFolderOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + int64_t buffer_id = worker_id; + std::unique_ptr io_block; + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + while (io_block != nullptr) { + if (io_block->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOE))); + buffer_id = worker_id; + } else if (io_block->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOF))); + } else { + std::vector keys; + RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); + if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers + std::unique_ptr db = make_unique(buffer_id, DataBuffer::kDeBFlagNone); + RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); + buffer_id += num_workers_; + } + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + } + RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); +} + +// Load 1 TensorRow (image,label) using 1 ImageLabelPair. 1 function call produces 1 TensorTow in a DataBuffer +Status ImageFolderOp::LoadTensorRow(ImageLabelPair pairPtr, TensorRow *trow) { + std::shared_ptr image, label; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(), + data_schema_->column(1).type(), + reinterpret_cast(&pairPtr->second))); + std::ifstream fs; + fs.open(folder_path_ + (pairPtr->first), std::ios::binary | std::ios::in); + if (fs.fail()) { + RETURN_STATUS_UNEXPECTED("Fail to open file: " + pairPtr->first); + } + int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); + (void)fs.seekg(0, std::ios::beg); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), + TensorShape(std::vector(1, num_elements)), + data_schema_->column(0).type(), nullptr)); + (void)fs.read(reinterpret_cast(image->StartAddr()), num_elements); + fs.close(); + if (decode_ == true) { + Status rc = Decode(image, &image); + if (rc.IsError()) { + std::string err = "Fail to decode image:" + folder_path_ + (pairPtr->first); + RETURN_STATUS_UNEXPECTED(err); + } + } + (*trow) = {std::move(image), std::move(label)}; + return Status::OK(); +} + +// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer +Status ImageFolderOp::LoadBuffer(const std::vector &keys, std::unique_ptr *db) { + std::unique_ptr deq = make_unique(); + TensorRow trow; + for (const int64_t &key : keys) { + RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); + deq->push_back(std::move(trow)); + } + (*db)->set_tensor_table(std::move(deq)); + (*db)->set_column_name_map(col_name_map_); + return Status::OK(); +} + +void ImageFolderOp::Print(std::ostream &out, bool show_all) const { + DatasetOp::Print(out, show_all); + out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ + << "\nImageFolder Directory: " << folder_path_ << "\n-------------------------\n"; +} + +// Reset Sampler and wakeup Master thread (functor) +Status ImageFolderOp::Reset() { + RETURN_IF_NOT_OK(sampler_->Reset()); + row_cnt_ = 0; + wp_.Set(); // wake up master thread after reset is done + return Status::OK(); +} + +// hand shake with Sampler, allow Sampler to call RandomAccessOp's functions to get NumRows +Status ImageFolderOp::InitSampler() { + RETURN_IF_NOT_OK(sampler_->Init(this)); + return Status::OK(); +} + +// Derived from RandomAccessOp +Status ImageFolderOp::GetNumSamples(int64_t *num) const { + if (num == nullptr || num_samples_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_samples_; + return Status::OK(); +} + +// Derived from RandomAccessOp +Status ImageFolderOp::GetNumRowsInDataset(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_rows_; + return Status::OK(); +} + +// Derived from RandomAccessOp +Status ImageFolderOp::GetClassIds(std::map> *cls_ids) const { + if (cls_ids == nullptr || !cls_ids->empty() || image_label_pairs_.empty()) { + RETURN_STATUS_UNEXPECTED("ImageLabelPair not set"); + } + for (size_t i = 0; i < image_label_pairs_.size(); ++i) { + (*cls_ids)[image_label_pairs_[i]->second].push_back(i); + } + for (auto &pair : (*cls_ids)) { + pair.second.shrink_to_fit(); + } + return Status::OK(); +} + +// Worker Entry for pre-scanning all the folders and do the 1st level shuffle +// Worker pull a file name from mFoldernameQueue (which is a Queue), walks all the images under that foldername +// After walking is complete, sort all the file names (relative path to all jpeg files under the same directory ) +// (Sort is automatically conducted using a set which is implemented using a Red-Black Tree) +// Add the sorted filenames in to a queue. The make a pair (foldername, queue*), +// foldername is used for 2nd level sorting. +// FYI: 1st level sorting: sort all images under the same directory. +// FYI: 2nd level sorting: sort all folder names +// push this pair to mImagenameQueue (which is again a Queue) +Status ImageFolderOp::PrescanWorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + std::string folder_name; + RETURN_IF_NOT_OK(folder_name_queue_->PopFront(&folder_name)); + while (folder_name.empty() == false) { + Path folder(folder_path_ + folder_name); + std::shared_ptr dirItr = Path::DirIterator::OpenDirectory(&folder); + if (folder.Exists() == false || dirItr == nullptr) { + RETURN_STATUS_UNEXPECTED("Error unable to open: " + folder_name); + } + std::set imgs; // use this for ordering + while (dirItr->hasNext()) { + Path file = dirItr->next(); + if (extensions_.empty() || extensions_.find(file.Extension()) != extensions_.end()) { + (void)imgs.insert(file.toString().substr(dirname_offset_)); + } else { + MS_LOG(INFO) << "Image folder operator unsupported file found: " << file.toString() + << ", extension: " << file.Extension() << "."; + } + } + FolderImagesPair p = std::make_shared>>(); + p->first = folder_name; + for (const std::string &img : imgs) { + p->second.push(std::make_shared>(img, 0)); + } + RETURN_IF_NOT_OK(image_name_queue_->EmplaceBack(p)); + RETURN_IF_NOT_OK(folder_name_queue_->PopFront(&folder_name)); + } + RETURN_IF_NOT_OK(image_name_queue_->EmplaceBack(nullptr)); // end signal + return Status::OK(); +} + +// This helper function recursively walks all foldernames, and send each foldername to mFoldernameQueue +// if mRecursive == false, don't go into folder of folders +Status ImageFolderOp::RecursiveWalkFolder(Path *dir) { + std::shared_ptr dir_itr = Path::DirIterator::OpenDirectory(dir); + if (dir_itr == nullptr) { + RETURN_STATUS_UNEXPECTED("Error encountered when indexing files"); + } + while (dir_itr->hasNext()) { + Path subdir = dir_itr->next(); + if (subdir.IsDirectory()) { + if (class_index_.empty() || + class_index_.find(subdir.toString().substr(dirname_offset_ + 1)) != class_index_.end()) { + RETURN_IF_NOT_OK(folder_name_queue_->EmplaceBack(subdir.toString().substr(dirname_offset_))); + } + if (recursive_ == true) { + RETURN_IF_NOT_OK(RecursiveWalkFolder(&subdir)); + } + } + } + return Status::OK(); +} + +// A thread that calls RecursiveWalkFolder +Status ImageFolderOp::startAsyncWalk() { + TaskManager::FindMe()->Post(); + Path dir(folder_path_); + if (dir.Exists() == false || dir.IsDirectory() == false) { + RETURN_STATUS_UNEXPECTED("Error unable to open: " + folder_path_); + } + dirname_offset_ = folder_path_.length(); + RETURN_IF_NOT_OK(RecursiveWalkFolder(&dir)); + // send out num_workers_ end signal to mFoldernameQueue, 1 for each worker. + // Upon receiving end Signal, worker quits and set another end Signal to mImagenameQueue. + for (int32_t ind = 0; ind < num_workers_; ++ind) { + RETURN_IF_NOT_OK(folder_name_queue_->EmplaceBack("")); // end signal + } + return Status::OK(); +} + +Status ImageFolderOp::LaunchThreadsAndInitOp() { + if (tree_ == nullptr) { + RETURN_STATUS_UNEXPECTED("tree_ not set"); + } + // Registers QueueList and individual Queues for interrupt services + RETURN_IF_NOT_OK(io_block_queues_.Register(tree_->AllTasks())); + RETURN_IF_NOT_OK(folder_name_queue_->Register(tree_->AllTasks())); + RETURN_IF_NOT_OK(image_name_queue_->Register(tree_->AllTasks())); + wp_.Register(tree_->AllTasks()); + // The following code launch 3 threads group + // 1) A thread that walks all folders and push the folder names to a util:Queue mFoldernameQueue. + // 2) Workers that pull foldername from mFoldernameQueue, walk it and return the sorted images to mImagenameQueue + // 3) Launch main workers that load DataBuffers by reading all images + RETURN_IF_NOT_OK(tree_->AllTasks()->CreateAsyncTask("walk dir", std::bind(&ImageFolderOp::startAsyncWalk, this))); + RETURN_IF_NOT_OK( + tree_->LaunchWorkers(num_workers_, std::bind(&ImageFolderOp::PrescanWorkerEntry, this, std::placeholders::_1))); + RETURN_IF_NOT_OK( + tree_->LaunchWorkers(num_workers_, std::bind(&ImageFolderOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + // The order of the following 2 functions must not be changed! + RETURN_IF_NOT_OK(this->PrescanMasterEntry(folder_path_)); // Master thread of pre-scan workers, blocking + RETURN_IF_NOT_OK(this->InitSampler()); // pass numRows to Sampler + return Status::OK(); +} + +Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const int64_t &num_samples, + const std::set &exts, int64_t *num_rows, int64_t *num_classes, + int64_t dev_id, int64_t num_dev) { + Path dir(path); + std::string err_msg = ""; + int64_t row_cnt = 0; + err_msg += (dir.Exists() == false || dir.IsDirectory() == false) ? "unable to open dir " + path : ""; + err_msg += (num_classes == nullptr || num_rows == nullptr) ? "num_class/num_rows is null\n" : ""; + err_msg += (dev_id >= num_dev || num_dev <= 0) ? "invalid sharding config\n" : ""; + err_msg += num_samples < 0 ? "num_samples can't be negative! set it to 0 to use all samples\n" : ""; + if (err_msg.empty() == false) { + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::queue foldernames; + std::shared_ptr dir_itr = Path::DirIterator::OpenDirectory(&dir); + while (dir_itr->hasNext()) { + Path subdir = dir_itr->next(); + if (subdir.IsDirectory()) { + foldernames.push(subdir.toString()); + } + } + (*num_classes) = foldernames.size(); + while (foldernames.empty() == false) { + Path subdir(foldernames.front()); + dir_itr = Path::DirIterator::OpenDirectory(&subdir); + while (dir_itr->hasNext()) { + if (exts.empty() || exts.find(subdir.Extension()) != exts.end()) { + ++row_cnt; + if (row_cnt == num_samples * num_dev) { + (*num_rows) = (row_cnt / num_dev) + (row_cnt % num_dev == 0 ? 0 : 1); + return Status::OK(); + } + } + } + foldernames.pop(); + } + (*num_rows) = (row_cnt / num_dev) + (row_cnt % num_dev == 0 ? 0 : 1); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h new file mode 100644 index 00000000000..63fcfa483c2 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h @@ -0,0 +1,285 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_IMAGE_FOLDER_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_IMAGE_FOLDER_OP_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/path.h" +#include "dataset/util/queue.h" +#include "dataset/util/services.h" +#include "dataset/util/status.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +// Forward declares +template +class Queue; + +using ImageLabelPair = std::shared_ptr>; +using FolderImagesPair = std::shared_ptr>>; + +class ImageFolderOp : public ParallelOp, public RandomAccessOp { + public: + class Builder { + public: + // Constructor for Builder class of ImageFolderOp + // @param int32_t numWrks - number of parallel workers + // @param dir - directory folder got ImageNetFolder + Builder(); + + // Destructor. + ~Builder() = default; + + // Setter method + // @param int32_t rows_per_buffer + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method + // @param int32_t size + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t size) { + builder_op_connector_size_ = size; + return *this; + } + + // Setter method + // @param std::set & exts, file extensions to be read + // @return Builder setter method returns reference to the builder. + Builder &SetExtensions(const std::set &exts) { + builder_extensions_ = exts; + return *this; + } + + // Setter method + // @paramconst std::map& map - a class name to label map + // @return + Builder &SetClassIndex(const std::map &map) { + builder_labels_to_read_ = map; + return *this; + } + + // Setter method + // @param bool do_decode + // @return Builder setter method returns reference to the builder. + Builder &SetDecode(bool do_decode) { + builder_decode_ = do_decode; + return *this; + } + + // Setter method + // @param int32_t num_workers + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // Setter method + // @param int64_t num_samples + // @return Builder setter method returns reference to the builder. + Builder &SetNumSamples(int64_t num_samples) { + builder_num_samples_ = num_samples; + return *this; + } + + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + + // Setter method + // @param const std::string & dir + // @return + Builder &SetImageFolderDir(const std::string &dir) { + builder_dir_ = dir; + return *this; + } + + // Whether dir are walked recursively + // @param bool recursive - if set to false, only get dirs in top level dir + // @return + Builder &SetRecursive(bool recursive) { + builder_recursive_ = recursive; + return *this; + } + + // Check validity of input args + // @return - The error code return + Status SanityCheck(); + + // The builder "build" method creates the final object. + // @param std::shared_ptr *op - DatasetOp + // @return - The error code return + Status Build(std::shared_ptr *op); + + private: + bool builder_decode_; + bool builder_recursive_; + std::string builder_dir_; + int32_t builder_num_workers_; + int64_t builder_num_samples_; + int32_t builder_rows_per_buffer_; + int32_t builder_op_connector_size_; + std::set builder_extensions_; + std::shared_ptr builder_sampler_; + std::unique_ptr builder_schema_; + std::map builder_labels_to_read_; + }; + + // Constructor + // @param int32_t num_wkrs - Num of workers reading images in parallel + // @param int32_t - rows_per_buffer Number of images (rows) in each buffer + // @param std::string - dir directory of ImageNetFolder + // @param int32_t queue_size - connector queue size + // @param std::set exts - set of file extensions to read, if empty, read everything under the dir + // @param td::unique_ptr sampler - sampler tells ImageFolderOp what to read + ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::string file_dir, int32_t queue_size, + int64_t num_samples, bool recursive, bool do_decode, const std::set &exts, + const std::map &map, std::unique_ptr, + std::shared_ptr sampler); + + // Destructor. + ~ImageFolderOp() = default; + + // Initialize ImageFOlderOp related var, calls the function to walk all files + // @param - std::string dir file directory to ImageNetFolder + // @return - The error code return + Status PrescanMasterEntry(const std::string &dir); + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t workerId - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t workerId - id of each worker + // @return Status - The error code return + Status PrescanWorkerEntry(int32_t worker_id); + + // Main Loop of ImageFolderOp + // Master thread: Fill IOBlockQueue, then goes to sleep + // Worker thread: pulls IOBlock from IOBlockQueue, work on it then put buffer to mOutConnector + // @return Status - The error code return + Status operator()() override; + + // Method derived from RandomAccess Op, enable Sampler to get numRows + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumSamples(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get total numRows in dataset + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumRowsInDataset(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get all ids for each class + // @param (std::unordered_map> * map - key label, val all ids for this class + // @return Status - The error code return + Status GetClassIds(std::map> *cls_ids) const override; + + // A print method typically used for debugging + // @param out + // @param show_all + void Print(std::ostream &out, bool show_all) const override; + + // This function is a hack! It is to return the num_class and num_rows the old storageOp does. The result + // returned by this function may not be consistent with what image_folder_op is going to return + // user this at your own risk! + static Status CountRowsAndClasses(const std::string &path, const int64_t &num_samples, + const std::set &exts, int64_t *num_rows, int64_t *num_classes, + int64_t dev_id = 0, int64_t num_dev = 1); + + private: + // Initialize Sampler, calls sampler->Init() within + // @return Status - The error code return + Status InitSampler(); + + // Load a tensor row according to a pair + // @param ImageLabelPair pair - + // @param TensorRow row - image & label read into this tensor row + // @return Status - The error code return + Status LoadTensorRow(ImageLabelPair pair, TensorRow *row); + + // @param const std::vector &keys - keys in ioblock + // @param std::unique_ptr db + // @return Status - The error code return + Status LoadBuffer(const std::vector &keys, std::unique_ptr *db); + + // @param std::string & dir - dir to walk all images + // @param int64_t * cnt - number of non folder files under the current dir + // @return + Status RecursiveWalkFolder(Path *dir); + + // start walking of all dirs + // @return + Status startAsyncWalk(); + + // Called first when function is called + // @return + Status LaunchThreadsAndInitOp(); + + // reset Op + // @return Status - The error code return + Status Reset() override; + + int32_t rows_per_buffer_; + std::string folder_path_; // directory of image folder + int64_t num_samples_; + bool recursive_; + bool decode_; + std::set extensions_; // extensions allowed + std::map class_index_; + std::unique_ptr data_schema_; + std::shared_ptr sampler_; + int64_t num_rows_; // total number of images in ImageFolder + int64_t row_cnt_; + int64_t buf_cnt_; + int64_t sampler_ind_; + int64_t dirname_offset_; + WaitPost wp_; + std::vector image_label_pairs_; + std::unordered_map col_name_map_; + QueueList> io_block_queues_; // queues of IOBlocks + std::unique_ptr> folder_name_queue_; + std::unique_ptr> image_name_queue_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_IMAGE_FOLDER_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/io_block.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/io_block.cc new file mode 100644 index 00000000000..9f45e2179fb --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/io_block.cc @@ -0,0 +1,85 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/io_block.h" + +#include +#include + +namespace mindspore { +namespace dataset { +// IOBlock Class // + +// Constructor of the IOBlock (1). A simpler one for the case when the block only has 1 key. +IOBlock::IOBlock(int64_t inKey, IOBlockFlags io_block_flags) : index_keys_(1, inKey), io_block_flags_(io_block_flags) {} + +// Constructor of the IOBlock (2) +IOBlock::IOBlock(const std::vector &in_keys, IOBlockFlags io_block_flags) : io_block_flags_(io_block_flags) { + index_keys_.insert(index_keys_.end(), in_keys.begin(), in_keys.end()); +} + +// Constructor of the IOBlock (3). A special IOBlock that is used for control messaging. +IOBlock::IOBlock(IOBlockFlags io_block_flags) : io_block_flags_(io_block_flags) {} + +// Fetches the first key from this block +Status IOBlock::GetKey(int64_t *out_key) const { + if (out_key == nullptr || index_keys_.empty()) { + RETURN_STATUS_UNEXPECTED("Failed to get the key from IOBlock"); + } + *out_key = index_keys_[0]; + return Status::OK(); +} + +// Fetches the list of keys from this block. +Status IOBlock::GetKeys(std::vector *out_keys) const { + if (out_keys == nullptr) { + RETURN_STATUS_UNEXPECTED("Output arg for GetKeys is null"); + } + *out_keys = index_keys_; // vector copy assign + return Status::OK(); +} + +// FilenameBlock derived class // + +// Constructor of the FilenameBlock (1) +FilenameBlock::FilenameBlock(int64_t key, int64_t start_offset, int64_t end_offset, IOBlockFlags io_block_flags) + : IOBlock(key, io_block_flags), start_offset_(start_offset), end_offset_(end_offset) {} + +// Constructor of the FilenameBlock (2). A special IOBlock that is used for control messaging. +FilenameBlock::FilenameBlock(IOBlockFlags io_block_flags) + : IOBlock(io_block_flags), start_offset_(kInvalidOffset), end_offset_(kInvalidOffset) {} + +// Gets the filename from the block using the provided index container +Status FilenameBlock::GetFilename(std::string *out_filename, const AutoIndexObj &index) const { + if (out_filename == nullptr) { + RETURN_STATUS_UNEXPECTED("Failed to get filename from FilenameBlock"); + } + + // a FilenameBlock only has one key. Call base class method to fetch that key + int64_t fetched_key; + RETURN_IF_NOT_OK(IOBlock::GetKey(&fetched_key)); + + // Do an index lookup using that key to get the filename. + auto it = index.Search(fetched_key); + if (it != index.end()) { + *out_filename = it.value(); + } else { + RETURN_STATUS_UNEXPECTED("Could not find filename from index"); + } + + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/io_block.h b/mindspore/ccsrc/dataset/engine/datasetops/source/io_block.h new file mode 100644 index 00000000000..87b417f027e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/io_block.h @@ -0,0 +1,125 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ + +#include +#include + +#include "dataset/util/auto_index.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// The IOBlock class is used to describe a "unit of work" that a storage leaf operator worker thread +// is responsible for acting on. +// The IOBlocks and it's derived classes abstracts a key-store and key-lookup interface where each +// block contains 1 to n keys, and the keys are used in conjunction with an index to provide the meta +// information for satisfying an IO request. +class IOBlock { + public: + enum IOBlockFlags : uint32_t { + kDeIoBlockNone = 0, + kDeIoBlockFlagEoe = 1u, // end of IOBlocks for one epoch + kDeIoBlockFlagEof = 1u << 1 // end of IOBlocks for entire program + }; + + // Constructor of the IOBlock (1). A simpler one for the case when the block only has 1 key. + // @param inKey - A single key to add into the block + // @param io_block_flags - The flag setting for the block + IOBlock(int64_t inKey, IOBlockFlags io_block_flags); + + // Constructor of the IOBlock (2). + // @param in_keys - A vector of keys to add into the block + // @param io_block_flags - The flag setting for the block + IOBlock(const std::vector &in_keys, IOBlockFlags io_block_flags); + + // Constructor of the IOBlock (3). A special IOBlock that is used for control messaging. + // @param io_block_flags - The flag setting for the block + explicit IOBlock(IOBlockFlags io_block_flags); + + // Destructor + virtual ~IOBlock() = default; + + // Fetches the first key from the block. + // @note Only useful if you know the block only has 1 key. + // @return A copy of the first key from the block + // @return Status - The error code return + Status GetKey(int64_t *out_key) const; + + // Fetches the list of keys from this block. + // @param out_keys - A copy of the vector of keys from the block. + // @return Status - The error code return + Status GetKeys(std::vector *out_keys) const; + + // Does this block have the eoe flag turned on? + // @return T/F if the IOBlock is eoe + bool eoe() const { return static_cast(io_block_flags_) & static_cast(kDeIoBlockFlagEoe); } + + // Does this block have the eof flag turned on? + // @return T/F if the IOBlock is eof + bool eof() const { return static_cast(io_block_flags_) & static_cast(kDeIoBlockFlagEof); } + + // Adds a key to this block + // @param key - The key to add to this block + void AddKey(int64_t key) { index_keys_.push_back(key); } + + protected: + std::vector index_keys_; // keys used for lookups to the meta info for the data + IOBlockFlags io_block_flags_; +}; // class IOBlock + +const int64_t kInvalidOffset = -1; + +// The Filename block derived class implements a style of IO block where each block contains only a +// single key that maps to a filename. +class FilenameBlock : public IOBlock { + public: + // Constructor of the FilenameBlock (1) + // @param key - The key identifier that can be used to find the data for this block + // @param start_offset - Start offset + // @param end_offset - End offset + // @param io_block_flags - The flag setting for the block + FilenameBlock(int64_t key, int64_t start_offset, int64_t end_offset, IOBlockFlags io_block_flags); + + // Constructor of the FilenameBlock (2). A special IOBlock that is used for control messaging. + // @param io_block_flags - The flag setting for the block + explicit FilenameBlock(IOBlockFlags io_block_flags); + + // Destructor + ~FilenameBlock() = default; + + // Gets the filename from the block using the provided index container + // @param out_filename - The filename to add to the block + // @param index - The index to perform lookup against + // @return Status - The error code return + Status GetFilename(std::string *out_filename, const AutoIndexObj &index) const; + + // Get the start offset of file + // @return int64_t - Start offset + int64_t GetStartOffset() const { return start_offset_; } + + // Get the end offset of the file + // @return int64_t - Start offset + int64_t GetEndOffset() const { return end_offset_; } + + private: + int64_t start_offset_; + int64_t end_offset_; +}; // class TFBlock +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc new file mode 100644 index 00000000000..52db199e5be --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -0,0 +1,453 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/manifest_op.h" + +#include +#include +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { +ManifestOp::Builder::Builder() : builder_sampler_(nullptr), builder_num_samples_(0), builder_decode_(false) { + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_num_workers_ = cfg->num_parallel_workers(); + builder_rows_per_buffer_ = cfg->rows_per_buffer(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status ManifestOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + if (builder_sampler_ == nullptr) { + builder_sampler_ = std::make_shared(); + } + builder_schema_ = make_unique(); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 1))); + *ptr = std::make_shared( + builder_num_workers_, builder_rows_per_buffer_, builder_file_, builder_op_connector_size_, builder_num_samples_, + builder_decode_, builder_labels_to_read_, std::move(builder_schema_), std::move(builder_sampler_), builder_usage_); + return Status::OK(); +} + +Status ManifestOp::Builder::SanityCheck() { + std::string err_msg; + err_msg += builder_file_.empty() ? "Manifest file is not set\n" : ""; + err_msg += builder_num_workers_ <= 0 ? "Num of parallel workers smaller than 1\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +ManifestOp::ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string file, int32_t queue_size, + int64_t num_samples, bool decode, const std::map &class_index, + std::unique_ptr data_schema, std::shared_ptr sampler, std::string usage) + : ParallelOp(num_works, queue_size), + rows_per_buffer_(rows_per_buffer), + io_block_pushed_(0), + row_cnt_(0), + sampler_ind_(0), + data_schema_(std::move(data_schema)), + file_(file), + class_index_(class_index), + sampler_(std::move(sampler)), + num_samples_(num_samples), + num_rows_(0), + decode_(decode), + usage_(usage), + buf_cnt_(0) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + col_name_map_[data_schema_->column(i).name()] = i; + } + io_block_queues_.Init(num_workers_, queue_size); + (void)std::transform(usage_.begin(), usage_.end(), usage_.begin(), ::tolower); +} + +// Main logic, Register Queue with TaskGroup, launch all threads and do the functor's work +Status ManifestOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + std::unique_ptr sampler_buffer; + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + return AddIoBlock(&sampler_buffer); +} + +Status ManifestOp::AddIoBlock(std::unique_ptr *sampler_buffer) { + while (true) { // each iterator is 1 epoch + std::vector keys; + keys.reserve(rows_per_buffer_); + while (!(*sampler_buffer)->eoe()) { + TensorRow sample_row; + RETURN_IF_NOT_OK((*sampler_buffer)->PopRow(&sample_row)); + std::shared_ptr sample_ids = sample_row[0]; + for (auto itr = sample_ids->begin(); itr != sample_ids->end(); ++itr) { + if ((*itr) >= num_rows_) continue; // index out of bound, skipping + if (row_cnt_ >= num_samples_) break; // enough row read, break for loop + keys.push_back(*itr); + row_cnt_++; + if (row_cnt_ % rows_per_buffer_ == 0) { + RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + keys.clear(); + } + } + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer)); + } + if (keys.empty() == false) { + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEof))); + for (int32_t i = 0; i < num_workers_; i++) { + RETURN_IF_NOT_OK( + io_block_queues_[i]->Add(make_unique(std::vector(), IOBlock::kDeIoBlockNone))); + } + return Status::OK(); + } else { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks + wp_.Clear(); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer)); + } + } +} + +Status ManifestOp::LaunchThreadsAndInitOp() { + if (tree_ == nullptr) { + RETURN_STATUS_UNEXPECTED("tree_ not set"); + } + RETURN_IF_NOT_OK(io_block_queues_.Register(tree_->AllTasks())); + wp_.Register(tree_->AllTasks()); + + RETURN_IF_NOT_OK( + tree_->LaunchWorkers(num_workers_, std::bind(&ManifestOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + RETURN_IF_NOT_OK(ParseManifestFile()); + RETURN_IF_NOT_OK(CountDatasetInfo()); + RETURN_IF_NOT_OK(InitSampler()); + return Status::OK(); +} + +// contains the main logic of pulling a IOBlock from IOBlockQueue, load a buffer and push the buffer to out_connector_ +// IMPORTANT: 1 IOBlock produces 1 DataBuffer +Status ManifestOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + int64_t buffer_id = worker_id; + std::unique_ptr io_block; + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + while (io_block != nullptr) { + if (io_block->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOE))); + buffer_id = worker_id; + } else if (io_block->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOF))); + } else { + std::vector keys; + RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); + if (keys.empty()) { + return Status::OK(); // empty key is a quit signal for workers + } + std::unique_ptr db = make_unique(buffer_id, DataBuffer::kDeBFlagNone); + RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); + buffer_id += num_workers_; + } + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + } + RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); +} + +// Load 1 TensorRow (image,label) using 1 ImageLabelPair. 1 function call produces 1 TensorTow in a DataBuffer +Status ManifestOp::LoadTensorRow(const std::pair> &data, TensorRow *trow) { + std::shared_ptr image; + std::shared_ptr label; + std::vector label_index(data.second.size()); + (void)std::transform(data.second.begin(), data.second.end(), label_index.begin(), + [this](const std::string &label_name) { return label_index_[label_name]; }); + if (label_index.size() == 1) { + RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), TensorShape({}), + data_schema_->column(1).type(), + reinterpret_cast(&label_index[0]))); + } else { + RETURN_IF_NOT_OK(Tensor::CreateTensor( + &label, data_schema_->column(1).tensorImpl(), TensorShape(std::vector(1, label_index.size())), + data_schema_->column(1).type(), reinterpret_cast(&label_index[0]))); + } + + std::ifstream fs; + fs.open(data.first, std::ios::binary | std::ios::in); + if (!fs.is_open()) { + RETURN_STATUS_UNEXPECTED("Fail to open file: " + data.first); + } + + int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); + (void)fs.seekg(0, std::ios::beg); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), + TensorShape(std::vector(1, num_elements)), + data_schema_->column(0).type(), nullptr)); + (void)fs.read(reinterpret_cast(image->StartAddr()), num_elements); + if (fs.fail()) { + fs.close(); + RETURN_STATUS_UNEXPECTED("Fail to read file: " + data.first); + } + fs.close(); + if (decode_ == true) { + Status rc = Decode(image, &image); + if (rc.IsError()) { + std::string err = "Fail to decode image:" + data.first; + RETURN_STATUS_UNEXPECTED(err); + } + } + (*trow) = {std::move(image), std::move(label)}; + return Status::OK(); +} + +// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer +Status ManifestOp::LoadBuffer(const std::vector &keys, std::unique_ptr *db) { + std::unique_ptr deq = make_unique(); + for (const auto &key : keys) { + TensorRow trow; + RETURN_IF_NOT_OK(LoadTensorRow(image_labelname_[static_cast(key)], &trow)); + deq->push_back(std::move(trow)); + } + (*db)->set_tensor_table(std::move(deq)); + (*db)->set_column_name_map(col_name_map_); + return Status::OK(); +} + +void ManifestOp::Print(std::ostream &out, bool show_all) const { + DatasetOp::Print(out, show_all); + out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ + << "\nManifest file: " << file_ << "\n-------------------------\n"; +} + +// Reset Sampler and wakeup Master thread (functor) +Status ManifestOp::Reset() { + RETURN_IF_NOT_OK(sampler_->Reset()); + row_cnt_ = 0; + wp_.Set(); // wake up master thread after reset is done + return Status::OK(); +} + +// hand shake with Sampler, allow Sampler to call RandomAccessOp's functions to get NumRows +Status ManifestOp::InitSampler() { + RETURN_IF_NOT_OK(sampler_->Init(this)); + return Status::OK(); +} + +// Derived from RandomAccessOp +Status ManifestOp::GetNumSamples(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_samples_; + return Status::OK(); +} + +// Derived from RandomAccessOp +Status ManifestOp::GetNumRowsInDataset(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_rows_; + return Status::OK(); +} + +// Derived from RandomAccessOp +Status ManifestOp::GetClassIds(std::map> *cls_ids) const { + if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) { + RETURN_STATUS_UNEXPECTED("Number rows is 0"); + } + + for (size_t i = 0; i < image_labelname_.size(); i++) { + size_t image_index = i; + for (size_t j = 0; j < image_labelname_[image_index].second.size(); j++) { + std::string label_name = (image_labelname_[image_index].second)[j]; + int32_t label_index = label_index_.at(label_name); + (*cls_ids)[label_index].emplace_back(image_index); + } + } + + for (auto &pair : (*cls_ids)) { + pair.second.shrink_to_fit(); + } + return Status::OK(); +} + +// Manifest file content +// {"source": "/path/to/image1.jpg", "usage":"train", annotation": ...} +// {"source": "/path/to/image2.jpg", "usage":"eval", "annotation": ...} +Status ManifestOp::ParseManifestFile() { + std::ifstream file_handle(file_); + if (!file_handle.is_open()) { + RETURN_STATUS_UNEXPECTED("Manifest file " + file_ + " can not open."); + } + std::string line; + while (getline(file_handle, line)) { + try { + nlohmann::json js = nlohmann::json::parse(line); + std::string image_file_path = js.value("source", ""); + // If image is not JPEG/PNG/GIF/BMP, drop it + bool valid = false; + RETURN_IF_NOT_OK(CheckImageType(image_file_path, &valid)); + if (!valid) { + continue; + } + std::string usage = js.value("usage", ""); + (void)std::transform(usage.begin(), usage.end(), usage.begin(), ::tolower); + if (usage != usage_) { + continue; + } + std::vector labels; + nlohmann::json annotations = js.at("annotation"); + for (nlohmann::json::iterator it = annotations.begin(); it != annotations.end(); ++it) { + nlohmann::json annotation = it.value(); + std::string label_name = annotation.value("name", ""); + if (label_name == "") { + file_handle.close(); + RETURN_STATUS_UNEXPECTED("Label name is not found in manifest file for " + image_file_path); + } + if (class_index_.empty() || class_index_.find(label_name) != class_index_.end()) { + if (label_index_.find(label_name) == label_index_.end()) { + label_index_[label_name] = 0; + } + labels.emplace_back(label_name); + } + } + if (!labels.empty()) { + image_labelname_.emplace_back(std::make_pair(image_file_path, labels)); + } + } catch (const std::exception &err) { + file_handle.close(); + RETURN_STATUS_UNEXPECTED("Parse manifest file failed"); + } + } + file_handle.close(); + + return Status::OK(); +} + +// Only support JPEG/PNG/GIF/BMP +Status ManifestOp::CheckImageType(const std::string &file_name, bool *valid) { + std::ifstream file_handle; + constexpr int read_num = 3; + *valid = false; + file_handle.open(file_name, std::ios::binary | std::ios::in); + if (!file_handle.is_open()) { + RETURN_STATUS_UNEXPECTED("Can not open image file " + file_name); + } + unsigned char file_type[read_num]; + (void)file_handle.read(reinterpret_cast(file_type), read_num); + + if (file_handle.fail()) { + file_handle.close(); + RETURN_STATUS_UNEXPECTED("Read image file failed " + file_name); + } + file_handle.close(); + if (file_type[0] == 0xff && file_type[1] == 0xd8 && file_type[2] == 0xff) { + // Normal JPEGs start with \xff\xd8\xff\xe0 + // JPEG with EXIF stats with \xff\xd8\xff\xe1 + // Use \xff\xd8\xff to cover both. + *valid = true; + } else if (file_type[0] == 0x89 && file_type[1] == 0x50 && file_type[2] == 0x4e) { + // It's a PNG + *valid = true; + } else if (file_type[0] == 0x47 && file_type[1] == 0x49 && file_type[2] == 0x46) { + // It's a GIF + *valid = true; + } else if (file_type[0] == 0x42 && file_type[1] == 0x4d) { + // It's a BMP + *valid = true; + } + return Status::OK(); +} + +Status ManifestOp::CountDatasetInfo() { + int32_t index = 0; + for (auto &label : label_index_) { + label.second = class_index_.empty() ? index : class_index_[label.first]; + index++; + } + + num_rows_ = static_cast(image_labelname_.size()); + num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; + if (num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("Number of rows is 0"); + } + return Status::OK(); +} + +Status ManifestOp::CountTotalRows(const std::string &file, int64_t numSamples, const py::dict &dict, + const std::string &usage, int64_t *count, int64_t *numClasses) { + // the logic of counting the number of samples is copied from ParseManifestFile() + std::map map; + for (auto p : dict) { + (void)map.insert(std::pair(py::reinterpret_borrow(p.first), + py::reinterpret_borrow(p.second))); + } + + std::shared_ptr op; + *count = 0; + RETURN_IF_NOT_OK( + Builder().SetManifestFile(file).SetNumSamples(numSamples).SetClassIndex(map).SetUsage(usage).Build(&op)); + RETURN_IF_NOT_OK(op->ParseManifestFile()); + *numClasses = static_cast(op->label_index_.size()); + *count = static_cast(op->image_labelname_.size()); + *count = (*count < numSamples || numSamples == 0) ? *count : numSamples; + return Status::OK(); +} + +Status ManifestOp::GetClassIndexing(const std::string &file, int64_t numSamples, const py::dict &dict, + const std::string &usage, std::map *output_class_indexing) { + std::map input_class_indexing; + for (auto p : dict) { + (void)input_class_indexing.insert(std::pair(py::reinterpret_borrow(p.first), + py::reinterpret_borrow(p.second))); + } + + if (!input_class_indexing.empty()) { + *output_class_indexing = input_class_indexing; + } else { + std::shared_ptr op; + RETURN_IF_NOT_OK(Builder() + .SetManifestFile(file) + .SetNumSamples(numSamples) + .SetClassIndex(input_class_indexing) + .SetUsage(usage) + .Build(&op)); + RETURN_IF_NOT_OK(op->ParseManifestFile()); + RETURN_IF_NOT_OK(op->CountDatasetInfo()); + uint32_t count = 0; + for (const auto label : op->label_index_) { + (*output_class_indexing).insert(std::make_pair(label.first, count)); + count++; + } + } + + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h new file mode 100644 index 00000000000..0f589930005 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h @@ -0,0 +1,258 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_MANIFEST_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_MANIFEST_OP_H_ + +#include +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/queue.h" +#include "dataset/util/services.h" +#include "dataset/util/status.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +class ManifestOp : public ParallelOp, public RandomAccessOp { + public: + class Builder { + public: + // Constructor for Builder class of ManifestOp + Builder(); + + // Destructor + ~Builder() = default; + + // Setter method + // @param int32_t rows_per_buffer + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method + // @param int32_t size + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t size) { + builder_op_connector_size_ = size; + return *this; + } + + // Setter method + // @param const std::map& map - a class name to label map + // @return + Builder &SetClassIndex(const std::map &map) { + builder_labels_to_read_ = map; + return *this; + } + + // Setter method + // @param bool do_decode + // @return Builder setter method returns reference to the builder. + Builder &SetDecode(bool do_decode) { + builder_decode_ = do_decode; + return *this; + } + + // Setter method + // @param int32_t num_workers + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // Setter method + // @param int64_t num_samples + // @return Builder setter method returns reference to the builder. + Builder &SetNumSamples(int64_t num_samples) { + builder_num_samples_ = num_samples; + return *this; + } + + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + + // Setter method + // @param const std::string & dir + // @return Builder setter method returns reference to the builder. + Builder &SetManifestFile(const std::string &file) { + builder_file_ = file; + return *this; + } + + // Setter method + // @param const std::string & dir + // @return Builder setter method returns reference to the builder. + Builder &SetUsage(const std::string &usage) { + builder_usage_ = usage; + return *this; + } + + // Check validity of input args + // @return Status - The error code return + Status SanityCheck(); + + // The builder "build" method creates the final object. + // @param std::shared_ptr *op - DatasetOp + // @return - The error code return + Status Build(std::shared_ptr *op); + + private: + std::shared_ptr builder_sampler_; + int64_t builder_num_samples_; + bool builder_decode_; + + std::string builder_file_; + int32_t builder_num_workers_; + int32_t builder_rows_per_buffer_; + int32_t builder_op_connector_size_; + std::unique_ptr builder_schema_; + std::string builder_usage_; + std::map builder_labels_to_read_; + }; + + // Constructor + // @param int32_t num_works - Num of workers reading images in parallel + // @param int32_t - rows_per_buffer Number of images (rows) in each buffer + // @param std::string - file list of Manifest + // @param int32_t queue_size - connector queue size + // @param td::unique_ptr sampler - sampler tells ImageFolderOp what to read + ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string file, int32_t queue_size, int64_t num_samples, + bool decode, const std::map &class_index, std::unique_ptr data_schema, + std::shared_ptr sampler, std::string usage); + // Destructor. + ~ManifestOp() = default; + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t worker_id - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Main Loop of ManifestOp + // Master thread: Fill IOBlockQueue, then goes to sleep + // Worker thread: pulls IOBlock from IOBlockQueue, work on it then put buffer to mOutConnector + // @return Status - The error code return + Status operator()() override; + + // Method derived from RandomAccess Op, enable Sampler to get numRows + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumSamples(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get total number of Rows in dataset + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumRowsInDataset(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get all ids for each class + // @param (std::unordered_map> * map - key label, val all ids for this class + // @return Status - The error code return + Status GetClassIds(std::map> *cls_ids) const override; + + // A print method typically used for debugging + // @param out + // @param show_all + void Print(std::ostream &out, bool show_all) const override; + + static Status CountTotalRows(const std::string &file, int64_t numSamples, const py::dict &dict, + const std::string &usage, int64_t *count, int64_t *numClasses); + + // Get str-to-int mapping from label name to index + static Status GetClassIndexing(const std::string &file, int64_t numSamples, const py::dict &dict, + const std::string &usage, std::map *output_class_indexing); + + private: + // Initialize Sampler, calls sampler->Init() within + // @return Status - The error code return + Status InitSampler(); + + // Method in operator(), to fill IOBlockQueue + // @param std::unique_ptr sampler_buffer - to fill IOBlockQueue + // @return Status - The error code return + Status AddIoBlock(std::unique_ptr *sampler_buffer); + + // Load a tensor row according to a pair + // @param std::pair> - > + // @param TensorRow row - image & label read into this tensor row + // @return Status - The error code return + Status LoadTensorRow(const std::pair> &data, TensorRow *row); + + // @param const std::vector &keys - keys in ioblock + // @param std::unique_ptr db + // @return Status - The error code return + Status LoadBuffer(const std::vector &keys, std::unique_ptr *db); + + // Parse manifest file to get image path and label and so on. + // @return Status - The error code return + Status ParseManifestFile(); + + // Called first when function is called + // @return Status - The error code return + Status LaunchThreadsAndInitOp(); + + // reset Op + // @return Status - The error code return + Status Reset() override; + + // Check if image ia valid.Only support JPEG/PNG/GIF/BMP + // @return + Status CheckImageType(const std::string &file_name, bool *valid); + + // Count label index,num rows and num samples + // @return Status - The error code return + Status CountDatasetInfo(); + + int32_t rows_per_buffer_; + int64_t io_block_pushed_; + int64_t row_cnt_; + int64_t sampler_ind_; + std::unique_ptr data_schema_; + std::string file_; // file that store the information of images + std::map class_index_; + std::shared_ptr sampler_; + int64_t num_samples_; + int64_t num_rows_; + bool decode_; + std::string usage_; + int64_t buf_cnt_; + + WaitPost wp_; + std::unordered_map col_name_map_; + QueueList> io_block_queues_; + std::map label_index_; + std::vector>> image_labelname_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_MANIFEST_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc new file mode 100644 index 00000000000..b062371d7fb --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -0,0 +1,668 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef ENABLE_MINDRECORD + +#include "dataset/engine/datasetops/source/mindrecord_op.h" + +#include +#include +#include +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/constants.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/make_unique.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +using mindrecord::kInt64Len; +using mindrecord::MSRStatus; +using mindrecord::Schema; +using mindrecord::ShardOperator; +using mindrecord::ShardReader; + +// Builder constructor. Creates the builder object. +MindRecordOp::Builder::Builder() : build_dataset_file_("") { + // Some arguments to the MindRecordOp constructor have a default argument that is taken + // from the client config. + // The user may choose to change these values for the construction of the StorageOp by + // using the various builder set methods. + + std::shared_ptr cfg = GlobalContext::config_manager(); + build_num_mind_record_workers_ = kDefaultMindRecordWorkers; + build_rows_per_buffer_ = cfg->rows_per_buffer(); + build_op_connector_queue_size_ = cfg->op_connector_size(); + build_block_reader_ = false; + builder_num_workers_ = 0; +} + +// The builder "build" method creates the final object. +Status MindRecordOp::Builder::Build(std::shared_ptr *ptr) { + std::shared_ptr new_mind_record_op; + + if (build_dataset_file_.empty()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Building a MindRecordOp that has not provided a file."); + } + + new_mind_record_op = std::make_shared(build_num_mind_record_workers_, build_rows_per_buffer_, + build_dataset_file_, build_op_connector_queue_size_, + build_columns_to_load_, build_operators_, build_block_reader_); + + RETURN_IF_NOT_OK(new_mind_record_op->Init()); + + *ptr = std::move(new_mind_record_op); + return Status::OK(); +} + +Status MindRecordOp::Builder::SanityCheck() const { return Status::OK(); } + +// Constructor of the MindRecordOp. +MindRecordOp::MindRecordOp(int32_t num_mind_record_workers, int32_t rows_per_buffer, std::string dataset_file, + int32_t op_connector_queue_size, const std::vector &columns_to_load, + const std::vector> &operators, const bool &block_reader) + : ParallelOp(num_mind_record_workers, op_connector_queue_size), + rows_per_buffer_(rows_per_buffer), + dataset_file_(dataset_file), + columns_to_load_(columns_to_load), + operators_(operators), + num_mind_record_workers_(num_mind_record_workers), + block_reader_(block_reader), + buffers_needed_(0), + buf_cnt_(0), + num_rows_(0), + ended_worker_(0), + buffer_water_mark_(0) { + io_blk_queues_.Init(num_workers_, op_connector_queue_size); + if (!block_reader_) return; + for (int32_t i = 0; i < num_workers_; ++i) { + block_buffer_.emplace_back(make_unique>(std::vector{})); + } +} + +// Private helper method to encapsulate some common construction/reset tasks +Status MindRecordOp::Init() { + shard_reader_ = mindspore::make_unique(); + auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_); + + CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed."); + + data_schema_ = mindspore::make_unique(); + + std::vector> schema_vec = shard_reader_->get_shard_header()->get_schemas(); + // check whether schema exists, if so use the first one + CHECK_FAIL_RETURN_UNEXPECTED(!schema_vec.empty(), "No schema found"); + mindrecord::json mr_schema = schema_vec[0]->GetSchema()["schema"]; + + bool load_all_cols = columns_to_load_.empty(); // if columns_to_load_ is empty it means load everything + std::map colname_to_ind; + for (mindrecord::json::iterator it = mr_schema.begin(); it != mr_schema.end(); ++it) { + std::string colname = it.key(); // key of the json, column name + mindrecord::json it_value = it.value(); // value, which contains type info and may contain shape + ColDescriptor col_desc; + TensorShape t_shape = TensorShape::CreateUnknownRankShape(); // shape of tensor, default unknown + std::string type_str = (it_value["type"] == "bytes" || it_value["type"] == "string") ? "uint8" : it_value["type"]; + DataType t_dtype = DataType(type_str); // valid types: {"bytes", "string", "int32", "int64", "float32", "float64"} + if (it_value["type"] == "bytes") { // rank = 1 + col_desc = ColDescriptor(colname, t_dtype, TensorImpl::kFlexible, 1); + } else if (it_value.find("shape") != it_value.end()) { + std::vector vec(it_value["shape"].size()); // temporary vector to hold shape + (void)std::copy(it_value["shape"].begin(), it_value["shape"].end(), vec.begin()); + t_shape = TensorShape(vec); + col_desc = ColDescriptor(colname, t_dtype, TensorImpl::kFlexible, t_shape.Rank(), &t_shape); + } else { // unknown shape + // create colDesc and add it to schema + col_desc = ColDescriptor(colname, t_dtype, TensorImpl::kFlexible, t_shape.Rank(), &t_shape); + } + + colname_to_ind[colname] = data_schema_->NumColumns(); + RETURN_IF_NOT_OK(data_schema_->AddColumn(col_desc)); + + if (load_all_cols) { + columns_to_load_.emplace_back(colname); + } + } + + if (!load_all_cols) { + std::unique_ptr tmp_schema = make_unique(); + for (std::string colname : columns_to_load_) { + CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), colname + ": doesn't exist"); + RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname]))); + } + data_schema_ = std::move(tmp_schema); + } + + for (int i = 0; i < static_cast(columns_to_load_.size()); i++) { + column_name_mapping_[columns_to_load_[i]] = i; + } + + num_rows_ = shard_reader_->get_num_rows(); + // Compute how many buffers we would need to accomplish rowsPerBuffer + buffers_needed_ = (num_rows_ + rows_per_buffer_ - 1) / rows_per_buffer_; + RETURN_IF_NOT_OK(SetColumnsBlob()); + + return Status::OK(); +} + +Status MindRecordOp::SetColumnsBlob() { + columns_blob_ = shard_reader_->get_blob_fields().second; + columns_blob_index_ = std::vector(columns_to_load_.size(), -1); + int32_t iBlob = 0; + for (uint32_t i = 0; i < columns_blob_.size(); ++i) { + if (column_name_mapping_.count(columns_blob_[i])) { + columns_blob_index_[column_name_mapping_[columns_blob_[i]]] = iBlob++; + } + } + return Status::OK(); +} + +// Destructor +MindRecordOp::~MindRecordOp() {} + +// A print method typically used for debugging +void MindRecordOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + ParallelOp::Print(out, show_all); + + // Then display our own stuff + out << "\nMindRecordOp:"; + out << "\n 1 Dataset file : " << dataset_file_; + out << "\n Number of rows : " << num_rows_; + out << "\n Rows per buffer : " << rows_per_buffer_; + out << "\n Number of buffers : " << buffers_needed_; + out << "\n Number of ShardReader workers : " << num_mind_record_workers_; + + out << "\n\n"; +} + +template +Status MindRecordOp::LoadFeature(std::shared_ptr *tensor, int32_t i_col, + const std::vector &columns_blob, const mindrecord::json &columns_json) const { + TensorShape new_shape = TensorShape::CreateUnknownRankShape(); + const unsigned char *data = nullptr; + + std::unique_ptr array_data; + std::string string_data; + + const ColDescriptor &cur_column = data_schema_->column(i_col); + std::string column_name = columns_to_load_[i_col]; + DataType type = cur_column.type(); + + // load blob column + if (columns_blob_index_[i_col] >= 0 && columns_blob.size() > 0) { + int32_t pos = columns_blob_.size() == 1 ? -1 : columns_blob_index_[i_col]; + RETURN_IF_NOT_OK(LoadBlob(&new_shape, &data, columns_blob, pos, cur_column)); + } else { + switch (type.value()) { + case DataType::DE_UINT8: { + // For strings (Assume DE_UINT8 is reserved for strings) + RETURN_IF_NOT_OK(LoadByte(&new_shape, &string_data, column_name, columns_json)); + data = reinterpret_cast(common::SafeCStr(string_data)); + break; + } + case DataType::DE_FLOAT32: { + // For both float scalars and arrays + RETURN_IF_NOT_OK(LoadFloat(&new_shape, &array_data, column_name, columns_json, cur_column, false)); + data = reinterpret_cast(array_data.get()); + break; + } + case DataType::DE_FLOAT64: { + // For both double scalars and arrays + RETURN_IF_NOT_OK(LoadFloat(&new_shape, &array_data, column_name, columns_json, cur_column, true)); + data = reinterpret_cast(array_data.get()); + break; + } + default: { + // For both integers scalars and arrays + RETURN_IF_NOT_OK(LoadInt(&new_shape, &array_data, column_name, columns_json, cur_column)); + data = reinterpret_cast(array_data.get()); + break; + } + } + } + // Create Tensor with given details + RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, cur_column.tensorImpl(), new_shape, type, data)); + + return Status::OK(); +} + +Status MindRecordOp::LoadBlob(TensorShape *new_shape, const unsigned char **data, + const std::vector &columns_blob, const int32_t pos, + const ColDescriptor &column) { + const auto kColumnSize = column.type().SizeInBytes(); + if (kColumnSize == 0) { + RETURN_STATUS_UNEXPECTED("column size is null"); + } + if (pos == -1) { + if (column.hasShape()) { + *new_shape = TensorShape::CreateUnknownRankShape(); + RETURN_IF_NOT_OK( + column.MaterializeTensorShape(static_cast(columns_blob.size() / kColumnSize), new_shape)); + } else { + std::vector shapeDetails = {static_cast(columns_blob.size() / kColumnSize)}; + *new_shape = TensorShape(shapeDetails); + } + *data = reinterpret_cast(&(columns_blob[0])); + return Status::OK(); + } + auto uint64_from_bytes = [&](int64_t pos) { + uint64_t result = 0; + for (uint64_t n = 0; n < kInt64Len; n++) { + result = (result << 8) + columns_blob[pos + n]; + } + return result; + }; + uint64_t iStart = 0; + for (int32_t i = 0; i < pos; i++) { + uint64_t num_bytes = uint64_from_bytes(iStart); + iStart += kInt64Len + num_bytes; + } + uint64_t num_bytes = uint64_from_bytes(iStart); + iStart += kInt64Len; + if (column.hasShape()) { + *new_shape = TensorShape::CreateUnknownRankShape(); + RETURN_IF_NOT_OK(column.MaterializeTensorShape(static_cast(num_bytes / kColumnSize), new_shape)); + } else { + std::vector shapeDetails = {static_cast(num_bytes / kColumnSize)}; + *new_shape = TensorShape(shapeDetails); + } + *data = reinterpret_cast(&(columns_blob[iStart])); + return Status::OK(); +} + +template +Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr *array_data, const std::string &column_name, + const mindrecord::json &columns_json, const ColDescriptor &column, bool use_double) { + if (!columns_json[column_name].is_array()) { + T value = 0; + RETURN_IF_NOT_OK(GetFloat(&value, columns_json[column_name], use_double)); + + *new_shape = TensorShape::CreateScalar(); + *array_data = mindspore::make_unique(1); + (*array_data)[0] = value; + } else { + if (column.hasShape()) { + *new_shape = TensorShape(column.shape()); + } else { + std::vector shapeDetails = {static_cast(columns_json[column_name].size())}; + *new_shape = TensorShape(shapeDetails); + } + + int idx = 0; + *array_data = mindspore::make_unique(new_shape->NumOfElements()); + for (auto &element : columns_json[column_name]) { + T value = 0; + RETURN_IF_NOT_OK(GetFloat(&value, element, use_double)); + + (*array_data)[idx++] = value; + } + } + + return Status::OK(); +} + +template +Status MindRecordOp::GetFloat(T *value, const mindrecord::json &data, bool use_double) { + if (data.is_number()) { + *value = data; + } else if (data.is_string()) { + try { + if (use_double) { + *value = data.get(); + } else { + *value = data.get(); + } + } catch (mindrecord::json::exception &e) { + RETURN_STATUS_UNEXPECTED("Conversion to float failed."); + } + } else { + RETURN_STATUS_UNEXPECTED("Conversion to float failed."); + } + + return Status::OK(); +} + +template +Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr *array_data, const std::string &column_name, + const mindrecord::json &columns_json, const ColDescriptor &column) { + if (!columns_json[column_name].is_array()) { + T value = 0; + RETURN_IF_NOT_OK(GetInt(&value, columns_json[column_name])); + + *new_shape = TensorShape::CreateScalar(); + *array_data = mindspore::make_unique(1); + (*array_data)[0] = value; + } else { + if (column.hasShape()) { + *new_shape = TensorShape(column.shape()); + } else { + std::vector shapeDetails = {static_cast(columns_json[column_name].size())}; + *new_shape = TensorShape(shapeDetails); + } + + int idx = 0; + *array_data = mindspore::make_unique(new_shape->NumOfElements()); + for (auto &element : columns_json[column_name]) { + T value = 0; + RETURN_IF_NOT_OK(GetInt(&value, element)); + + (*array_data)[idx++] = value; + } + } + + return Status::OK(); +} + +template +Status MindRecordOp::GetInt(T *value, const mindrecord::json &data) { + int64_t temp_value = 0; + bool less_than_zero = false; + + if (data.is_number_integer()) { + const mindrecord::json json_zero = 0; + if (data < json_zero) less_than_zero = true; + temp_value = data; + } else if (data.is_string()) { + std::string string_value = data; + + if (!string_value.empty() && string_value[0] == '-') { + try { + temp_value = std::stoll(string_value); + less_than_zero = true; + } catch (std::invalid_argument &e) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed, invalid argument."); + } catch (std::out_of_range &e) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed, out of range."); + } + } else { + try { + temp_value = static_cast(std::stoull(string_value)); + } catch (std::invalid_argument &e) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed, invalid argument."); + } catch (std::out_of_range &e) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed, out of range."); + } + } + } else { + RETURN_STATUS_UNEXPECTED("Conversion to int failed."); + } + + if ((less_than_zero && temp_value < static_cast(std::numeric_limits::min())) || + (!less_than_zero && static_cast(temp_value) > static_cast(std::numeric_limits::max()))) { + RETURN_STATUS_UNEXPECTED("Conversion to int failed. Out of range"); + } + *value = static_cast(temp_value); + + return Status::OK(); +} + +Status MindRecordOp::LoadByte(TensorShape *new_shape, std::string *string_data, const std::string &column_name, + const mindrecord::json &columns_json) { + *string_data = columns_json[column_name]; + std::vector shape_details = {static_cast(string_data->size())}; + *new_shape = TensorShape(shape_details); + + return Status::OK(); +} + +Status MindRecordOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + std::unique_ptr io_block; + RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); + while (io_block != nullptr) { + if (io_block->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique(0, DataBuffer::kDeBFlagEOE)))); + RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); + continue; + } + if (io_block->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique(0, DataBuffer::kDeBFlagEOF)))); + RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); + continue; + } + + // load data buffer + std::vector keys; + RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); + if (keys.empty() == true) { + { + std::unique_lock lock(ended_worker_mutex_); + ended_worker_++; + if (ended_worker_ == num_workers_) shard_reader_->Close(); + } + return Status::OK(); // empty key is a quit signal for workers + } + + const uint64_t buffer_id = keys[0]; + std::unique_ptr fetched_buffer; + + // Get the next buffer. Push it up to the output connector. + if (buffer_id % LOG_INTERVAL == 0) { + MS_LOG(DEBUG) << "MindRecord operator consumed buffer " << buffer_id << " by worker " << worker_id << "."; + } + RETURN_IF_NOT_OK(GetBufferFromReader(&fetched_buffer, buffer_id, worker_id)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(fetched_buffer))); + if (!block_reader_) { + RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); + continue; + } + + // update block-reader buffer + block_buffer_[buffer_id % num_workers_]->clear(); + { + std::unique_lock lck(mtx_block_reader_); + if (buffer_id == buffer_water_mark_) { + buffer_water_mark_++; + while (block_set_.count(buffer_water_mark_) > 0) (void)block_set_.erase(buffer_water_mark_++); + } else { + (void)block_set_.insert(buffer_id); + } + } + cv_reader_.notify_one(); + RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block)); + } + RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); +} + +Status MindRecordOp::GetBufferFromReader(std::unique_ptr *fetched_buffer, int64_t buffer_id, + int32_t worker_id) { + *fetched_buffer = mindspore::make_unique(buffer_id, DataBuffer::kDeBFlagNone); + (*fetched_buffer)->set_column_name_map(column_name_mapping_); + std::unique_ptr tensor_table = mindspore::make_unique(); + for (int32_t i = 0; i < rows_per_buffer_; ++i) { + ShardTuple tupled_buffer; + if (block_reader_) { + if (i >= block_buffer_[buffer_id % num_workers_]->size()) break; + tupled_buffer = block_buffer_[buffer_id % num_workers_]->at(i); + } else { + int32_t row_id = buffer_id * rows_per_buffer_ + i; + tupled_buffer = shard_reader_->GetNextById(row_id, worker_id); + if (tupled_buffer.empty()) break; + } + for (const auto &tupled_row : tupled_buffer) { + std::vector columnsBlob = std::get<0>(tupled_row); + mindrecord::json columns_json = std::get<1>(tupled_row); + TensorRow tensor_row; + for (uint32_t j = 0; j < columns_to_load_.size(); ++j) { + std::shared_ptr tensor; + + const ColDescriptor &cur_column = data_schema_->column(j); + DataType type = cur_column.type(); + RETURN_IF_NOT_OK(SwitchLoadFeature(type, &tensor, j, columnsBlob, columns_json)); + + tensor_row.push_back(std::move(tensor)); + } + + tensor_table->push_back(std::move(tensor_row)); + } + } + + // Replace the TensorTable in DataBuffer with the new one. + (*fetched_buffer)->set_tensor_table(std::move(tensor_table)); + return Status::OK(); +} + +Status MindRecordOp::SwitchLoadFeature(const DataType &type, std::shared_ptr *tensor, int32_t i_col, + const std::vector &columns_blob, + const mindrecord::json &columns_json) const { + switch (type.value()) { + case DataType::DE_BOOL: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_INT8: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_UINT8: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_INT16: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_UINT16: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_INT32: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_UINT32: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_INT64: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_UINT64: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_FLOAT32: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + case DataType::DE_FLOAT64: { + return LoadFeature(tensor, i_col, columns_blob, columns_json); + } + default: { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "mindrecord column list type does not match any known types"); + } + } +} + +Status MindRecordOp::FetchBlockBuffer(const int32_t &buffer_id) { + { + std::unique_lock lck(mtx_block_reader_); + cv_reader_.wait(lck, [buffer_id, this] { return buffer_id < buffer_water_mark_ + num_workers_; }); + } + for (int32_t i = 0; i < rows_per_buffer_; i++) { + // Block reader does NOT care about argument + ShardTuple tuple_buffer = shard_reader_->GetNextById(i, i); + if (tuple_buffer.empty()) break; + block_buffer_[buffer_id % num_workers_]->push_back(std::move(tuple_buffer)); + } + return Status::OK(); +} + +// Class functor operator () override. +// All dataset ops operate by launching a thread (see ExecutionTree). This class functor will +// provide the master loop that drives the logic for performing the work +// Main logic, Register Queue with TaskGroup, launch all threads and do the functor's work +Status MindRecordOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadAndInitOp()); + num_rows_ = shard_reader_->get_num_rows(); + + buffers_needed_ = num_rows_ / rows_per_buffer_; + if (num_rows_ % rows_per_buffer_ != 0) { + buffers_needed_++; + } + + while (true) { // each iterator is 1 epoch + for (int32_t i = 0; i < buffers_needed_; ++i) { + if (block_reader_) RETURN_IF_NOT_OK(FetchBlockBuffer(i)); + std::vector keys(1, i); + RETURN_IF_NOT_OK( + io_blk_queues_[buf_cnt_++ % num_workers_]->Add(make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + RETURN_IF_NOT_OK( + io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK( + io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEof))); + for (int32_t i = 0; i < num_workers_; i++) { + RETURN_IF_NOT_OK( + io_blk_queues_[i]->Add(std::move(make_unique(std::vector(), IOBlock::kDeIoBlockNone)))); + } + return Status::OK(); + } else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset + RETURN_IF_NOT_OK( + io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + + // reset our buffer count and go to loop again. + RETURN_IF_NOT_OK(shard_reader_wait_post_.Wait()); + shard_reader_wait_post_.Clear(); + } + } +} + +// Overrides base class reset method. When an operator does a reset, it cleans up any state +// info from it's previous execution and then initializes itself so that it can be executed +// again. +Status MindRecordOp::Reset() { + RETURN_IF_NOT_OK(ParallelOp::Reset()); // Call our super class reset first. + + if (block_reader_) { + shard_reader_->Reset(); + buffer_water_mark_ = 0; + } else { + shard_reader_->ShuffleTask(); + } + shard_reader_wait_post_.Set(); + + return Status::OK(); +} + +Status MindRecordOp::LaunchThreadAndInitOp() { + if (tree_ == nullptr) { + RETURN_STATUS_UNEXPECTED("tree_ not set"); + } + + RETURN_IF_NOT_OK(io_blk_queues_.Register(tree_->AllTasks())); + shard_reader_wait_post_.Register(tree_->AllTasks()); + if (shard_reader_->Launch(!block_reader_) == MSRStatus::FAILED) { + RETURN_STATUS_UNEXPECTED("MindRecordOp launch failed."); + } + // Launch main workers that load DataBuffers by reading all images + RETURN_IF_NOT_OK( + tree_->LaunchWorkers(num_workers_, std::bind(&MindRecordOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + return Status::OK(); +} + +Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *count) { + std::unique_ptr shard_reader = mindspore::make_unique(); + MSRStatus rc = shard_reader->CountTotalRows(dataset_path, count); + if (rc == MSRStatus::FAILED) { + RETURN_STATUS_UNEXPECTED("MindRecordOp count total rows failed."); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h new file mode 100644 index 00000000000..2535acbc503 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h @@ -0,0 +1,280 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_MINDRECORD_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_MINDRECORD_OP_H_ +#ifdef ENABLE_MINDRECORD +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/util/queue.h" +#include "dataset/util/status.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/common/shard_utils.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +// Forward declares +template +class Queue; +class DataBuffer; + +using mindrecord::ShardOperator; +using mindrecord::ShardReader; +using ShardTuple = std::vector, mindrecord::json>>; // Row of data from ShardReader + +const int32_t LOG_INTERVAL = 19; + +class MindRecordOp : public ParallelOp { + public: + // The nested builder class inside of the MindRecordOp is used to help manage all of the arguments + // for constructing it. Use the builder by setting each argument with the provided set methods, + // and then finally call the build method to execute the actual construction. + class Builder { + public: + Builder(); + + ~Builder() = default; + + Status Build(std::shared_ptr *); + + Builder &SetRowsPerBuffer(int rows_per_buffer) { + build_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + Builder &SetNumMindRecordWorkers(int32_t num_mind_record_workers) { + build_num_mind_record_workers_ = num_mind_record_workers; + return *this; + } + + Builder &SetOpConnectorQueueSize(int32_t queue_size) { + build_op_connector_queue_size_ = queue_size; + return *this; + } + + Builder &SetDatasetFile(const std::string &file) { + build_dataset_file_ = file; + return *this; + } + + Builder &SetColumnsToLoad(const std::vector &columns) { + build_columns_to_load_ = columns; + return *this; + } + + Builder &SetOperators(const std::vector> &operators) { + build_operators_ = operators; + return *this; + } + + Builder &SetBlockReader() { + build_block_reader_ = true; + return *this; + } + + Status SanityCheck() const; + + static int32_t num_mind_record_workers() { return kDefaultMindRecordWorkers; } + + private: + static constexpr int32_t kDefaultMindRecordWorkers = 4; + // The builder saves all MindRecordOp construction arguments internally. + // The following are the arguments. + int32_t build_num_mind_record_workers_; + int32_t builder_num_workers_; + int32_t build_rows_per_buffer_; + int32_t build_op_connector_queue_size_; + std::string build_dataset_file_; + std::vector build_columns_to_load_; + std::vector> build_operators_; + bool build_block_reader_; + }; + + // Constructor of the MindRecordOp. + // @note The builder class should be used to call it + // @param num_mind_record_workers - The number of workers for the op (run by ShardReader) + // @param rows_per_buffer - The requested number of rows per buffer + // @param dataset_file - A shard file + // @param op_connector_queue_size - The output connector queue size + // @param columns_to_load - The list of columns to use (column name) + // @param operators - ShardOperators for Shuffle, Category, Sample + MindRecordOp(int32_t num_mind_record_workers, int32_t rows_per_buffer, std::string dataset_file, + int32_t op_connector_queue_size, const std::vector &columns_to_load, + const std::vector> &operators, const bool &block_reader); + + // Destructor + ~MindRecordOp() override; + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param op - reference to the MindRecordOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const MindRecordOp &op) { + op.Print(out, false); + return out; + } + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t workerId - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Class functor operator () override. + // All DatasetOps operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work. + // @return Status - The error code return + Status operator()() override; + + // Called first when function is called + // @return + Status LaunchThreadAndInitOp(); + + // Overrides base class reset method. When an operator does a reset, it cleans up any state + // info from it's previous execution and then initializes itself so that it can be executed + // again. + // @return Status - The error code return + Status Reset() override; + + // Getter method + int32_t num_rows() const { return num_rows_; } + + // Getter method + static Status CountTotalRows(const std::string dataset_path, int64_t *count); + + // Getter method + int32_t rows_per_buffer() const { return rows_per_buffer_; } + + // Getter method + std::string dataset_file() const { return dataset_file_; } + + // Getter method + std::vector columns_to_load() const { return columns_to_load_; } + + bool block_reader() const { return block_reader_; } + + Status Init(); + + Status SetColumnsBlob(); + + private: + Status GetBufferFromReader(std::unique_ptr *fetched_buffer, int64_t buffer_id, int32_t worker_id); + + // Parses a single cell and puts the data into a tensor + // @param tensor - the tensor to put the parsed data in + // @param i_col - the id of column to parse + // @param columns_blob - the blob data received from the reader + // @param columns_json - the data for fields received from the reader + template + Status LoadFeature(std::shared_ptr *tensor, int32_t i_col, const std::vector &columns_blob, + const mindrecord::json &columns_json) const; + + Status SwitchLoadFeature(const DataType &type, std::shared_ptr *tensor, int32_t i_col, + const std::vector &columns_blob, const mindrecord::json &columns_json) const; + + static Status LoadBlob(TensorShape *new_shape, const unsigned char **data, const std::vector &columns_blob, + const int32_t pos, const ColDescriptor &column); + + // Get shape and data (scalar or array) for tensor to be created (for floats and doubles) + // @param new_shape - the shape of tensor to be created. + // @param array_data - the array where data should be put in + // @param column_name - name of current column to be processed + // @param columns_json - the data for fields received from the reader + // @param column - description of current column from schema + // @param use_double - boolean to choose between float32 and float64 + template + static Status LoadFloat(TensorShape *new_shape, std::unique_ptr *array_data, const std::string &column_name, + const mindrecord::json &columns_json, const ColDescriptor &column, bool use_double); + + // Get shape and data (scalar or array) for tensor to be created (for integers) + // @param new_shape - the shape of tensor to be created. + // @param array_data - the array where data should be put in + // @param column_name - name of current column to be processed + // @param columns_json - the data for fields received from the reader + // @param column - description of current column from schema + template + static Status LoadInt(TensorShape *new_shape, std::unique_ptr *array_data, const std::string &column_name, + const mindrecord::json &columns_json, const ColDescriptor &column); + + static Status LoadByte(TensorShape *new_shape, std::string *string_data, const std::string &column_name, + const mindrecord::json &columns_json); + + // Get a single float value from the given json + // @param value - the float to put the value in + // @param arrayData - the given json containing the float + // @param use_double - boolean to choose between float32 and float64 + template + static Status GetFloat(T *value, const mindrecord::json &data, bool use_double); + + // Get a single integer value from the given json + // @param value - the integer to put the value in + // @param arrayData - the given json containing the integer + template + static Status GetInt(T *value, const mindrecord::json &data); + + Status FetchBlockBuffer(const int32_t &buffer_id); + + int32_t rows_per_buffer_; // The number of requested rows per buffer. + std::string dataset_file_; // A dataset file + std::vector columns_to_load_; // Columns to load from dataset + std::vector> operators_; // ShardOperators to use + int32_t num_mind_record_workers_; // number of workers to be spawned by ShardReader + bool block_reader_; // block reader switch + int32_t buffers_needed_; // Counter for the buffers that were fetched + int64_t buf_cnt_; // Buffer counter + int32_t num_rows_; // One more than the last row id in the range for this cache + std::atomic ended_worker_; + std::atomic buffer_water_mark_; + + std::unique_ptr data_schema_; // Data schema for column typing + std::vector columns_blob_; // Blob Columns to load from dataset + std::vector columns_blob_index_; // Blob Columns to load from dataset + + std::unordered_map column_name_mapping_; + std::unique_ptr shard_reader_; + WaitPost shard_reader_wait_post_; + QueueList> io_blk_queues_; + + // For block reader + std::mutex mtx_block_reader_; + std::condition_variable cv_reader_; + std::vector>> block_buffer_; + std::unordered_set block_set_; + + std::mutex ended_worker_mutex_; +}; +} // namespace dataset +} // namespace mindspore +#endif +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_MINDRECORD_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc new file mode 100644 index 00000000000..f76fb9314de --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -0,0 +1,445 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/mnist_op.h" + +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { +const int32_t kMnistImageFileMagicNumber = 2051; +const int32_t kMnistLabelFileMagicNumber = 2049; +const int32_t kMnistImageRows = 28; +const int32_t kMnistImageCols = 28; + +MnistOp::Builder::Builder() : builder_num_samples_(0), builder_sampler_(nullptr) { + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_num_workers_ = cfg->num_parallel_workers(); + builder_rows_per_buffer_ = cfg->rows_per_buffer(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status MnistOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + if (builder_sampler_ == nullptr) { + builder_sampler_ = std::make_shared(); + } + builder_schema_ = make_unique(); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); + TensorShape scalar = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK(builder_schema_->AddColumn( + ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar))); + *ptr = + std::make_shared(builder_num_workers_, builder_rows_per_buffer_, builder_dir_, builder_op_connector_size_, + builder_num_samples_, std::move(builder_schema_), std::move(builder_sampler_)); + return Status::OK(); +} + +Status MnistOp::Builder::SanityCheck() { + Path dir(builder_dir_); + std::string err_msg; + err_msg += dir.IsDirectory() == false ? "MNIST path is invalid or not set\n" : ""; + err_msg += builder_num_workers_ <= 0 ? "Number of parallel workers is set to 0 or negative\n" : ""; + err_msg += builder_num_samples_ < 0 ? "Number of samples is set to negative\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +MnistOp::MnistOp(int32_t num_workers, int32_t rows_per_buffer, std::string folder_path, int32_t queue_size, + int64_t num_samples, std::unique_ptr data_schema, std::shared_ptr sampler) + : ParallelOp(num_workers, queue_size), + buf_cnt_(0), + row_cnt_(0), + num_rows_(0), + num_samples_(num_samples), + folder_path_(folder_path), + rows_per_buffer_(rows_per_buffer), + sampler_(std::move(sampler)), + data_schema_(std::move(data_schema)) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + col_name_map_[data_schema_->column(i).name()] = i; + } + io_block_queues_.Init(num_workers, queue_size); +} + +Status MnistOp::TraversalSampleIds(const std::shared_ptr &sample_ids, std::vector *keys) { + for (auto itr = sample_ids->begin(); itr != sample_ids->end(); ++itr) { + if ((*itr) >= num_rows_) continue; // index out of bound, skipping + if (row_cnt_ >= num_samples_) break; // enough row read, break for loop + keys->push_back(*itr); + row_cnt_++; + if (row_cnt_ % rows_per_buffer_ == 0) { + RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( + make_unique(IOBlock(*keys, IOBlock::kDeIoBlockNone)))); + keys->clear(); + } + } + return Status::OK(); +} + +// functor that contains the main logic of MNIST op +Status MnistOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + std::unique_ptr sampler_buffer; + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + while (true) { // each iterator is 1 epoch + std::vector keys; + keys.reserve(rows_per_buffer_); + while (sampler_buffer->eoe() == false) { + std::shared_ptr sample_ids; + RETURN_IF_NOT_OK(sampler_buffer->GetTensor(&sample_ids, 0, 0)); + if (sample_ids->type() != DataType(DataType::DE_INT64)) { + RETURN_STATUS_UNEXPECTED("Sampler Tensor isn't UINT64"); + } + RETURN_IF_NOT_OK(TraversalSampleIds(sample_ids, &keys)); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + if (keys.empty() == false) { + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEof))); + for (int32_t i = 0; i < num_workers_; ++i) { + RETURN_IF_NOT_OK( + io_block_queues_[i]->Add(make_unique(std::vector(), IOBlock::kDeIoBlockNone))); + } + return Status::OK(); + } else { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks + wp_.Clear(); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + } +} + +// contains the logic of pulling a IOBlock from IOBlockQueue, load a buffer and push the buffer to out_connector_ +Status MnistOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + int64_t buffer_id = worker_id; + std::unique_ptr iOBlock; + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock)); + while (iOBlock != nullptr) { + if (iOBlock->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOE))); + buffer_id = worker_id; + } else if (iOBlock->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOF))); + } else { + std::vector keys; + RETURN_IF_NOT_OK(iOBlock->GetKeys(&keys)); + if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers + std::unique_ptr db = make_unique(buffer_id, DataBuffer::kDeBFlagNone); + RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); + buffer_id += num_workers_; + } + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock)); + } + RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); +} + +// Load 1 TensorRow (image,label) using 1 MnistLabelPair. +Status MnistOp::LoadTensorRow(const MnistLabelPair &mnist_pair, TensorRow *trow) { + std::shared_ptr image, label; + int32_t l = mnist_pair.second; + // make a copy of cached tensor + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), mnist_pair.first->shape(), + mnist_pair.first->type(), mnist_pair.first->StartAddr())); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(), + data_schema_->column(1).type(), reinterpret_cast(&l))); + (*trow) = {std::move(image), std::move(label)}; + return Status::OK(); +} + +// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer +Status MnistOp::LoadBuffer(const std::vector &keys, std::unique_ptr *db) { + std::unique_ptr deq = make_unique(); + TensorRow trow; + for (const int64_t &key : keys) { + RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow)); + deq->push_back(std::move(trow)); + } + (*db)->set_tensor_table(std::move(deq)); + (*db)->set_column_name_map(col_name_map_); + return Status::OK(); +} + +void MnistOp::Print(std::ostream &out, bool show_all) const { + DatasetOp::Print(out, show_all); + out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ + << "\nMNIST Directory: " << folder_path_ << "\n-------------------------\n"; +} + +// Reset Sampler and wakeup Master thread (functor) +Status MnistOp::Reset() { + RETURN_IF_NOT_OK(sampler_->Reset()); + row_cnt_ = 0; + wp_.Set(); // wake up master thread after reset is done + return Status::OK(); +} + +// hand shake with Sampler, allow Sampler to call RandomAccessOp's functions to get NumRows +Status MnistOp::InitSampler() { + RETURN_IF_NOT_OK(sampler_->Init(this)); + return Status::OK(); +} + +// Derived from RandomAccessOp +Status MnistOp::GetNumSamples(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_samples_; + return Status::OK(); +} + +// Derived from RandomAccessOp +Status MnistOp::GetClassIds(std::map> *cls_ids) const { + if (cls_ids == nullptr || !cls_ids->empty() || image_label_pairs_.empty()) { + RETURN_STATUS_UNEXPECTED("ImageLabelPair not set"); + } + for (size_t i = 0; i < image_label_pairs_.size(); ++i) { + (*cls_ids)[image_label_pairs_[i].second].push_back(i); + } + for (auto &pair : (*cls_ids)) { + pair.second.shrink_to_fit(); + } + return Status::OK(); +} + +Status MnistOp::ReadFromReader(std::ifstream *reader, uint32_t *result) { + uint32_t res = 0; + reader->read(reinterpret_cast(&res), 4); + if (reader->fail()) { + RETURN_STATUS_UNEXPECTED("Failed to read 4 bytes from file"); + } + *result = SwapEndian(res); + return Status::OK(); +} + +uint32_t MnistOp::SwapEndian(uint32_t val) const { + val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF); + return (val << 16) | (val >> 16); +} + +Status MnistOp::CheckImage(const std::string &file_name, std::ifstream *image_reader, uint32_t *num_images) { + if (image_reader->is_open() == false) { + RETURN_STATUS_UNEXPECTED("Cannot open mnist image file: " + file_name); + } + int64_t image_len = image_reader->seekg(0, std::ios::end).tellg(); + (void)image_reader->seekg(0, std::ios::beg); + // The first 16 bytes of the image file are type, number, row and column + if (image_len < 16) { + RETURN_STATUS_UNEXPECTED("Mnist file is corrupted."); + } + uint32_t magic_number; + RETURN_IF_NOT_OK(ReadFromReader(image_reader, &magic_number)); + CHECK_FAIL_RETURN_UNEXPECTED(magic_number == kMnistImageFileMagicNumber, + "This is not the mnist image file: " + file_name); + + uint32_t num_items; + RETURN_IF_NOT_OK(ReadFromReader(image_reader, &num_items)); + uint32_t rows; + RETURN_IF_NOT_OK(ReadFromReader(image_reader, &rows)); + uint32_t cols; + RETURN_IF_NOT_OK(ReadFromReader(image_reader, &cols)); + // The image size of the Mnist dataset is fixed at [28,28] + if ((rows != kMnistImageRows) || (cols != kMnistImageCols)) { + RETURN_STATUS_UNEXPECTED("Wrong shape of image."); + } + if ((image_len - 16) != num_items * rows * cols) { + RETURN_STATUS_UNEXPECTED("Wrong number of image."); + } + *num_images = num_items; + return Status::OK(); +} + +Status MnistOp::CheckLabel(const std::string &file_name, std::ifstream *label_reader, uint32_t *num_labels) { + if (label_reader->is_open() == false) { + RETURN_STATUS_UNEXPECTED("Cannot open mnist label file: " + file_name); + } + int64_t label_len = label_reader->seekg(0, std::ios::end).tellg(); + (void)label_reader->seekg(0, std::ios::beg); + // The first 8 bytes of the image file are type and number + if (label_len < 8) { + RETURN_STATUS_UNEXPECTED("Mnist file is corrupted."); + } + uint32_t magic_number; + RETURN_IF_NOT_OK(ReadFromReader(label_reader, &magic_number)); + CHECK_FAIL_RETURN_UNEXPECTED(magic_number == kMnistLabelFileMagicNumber, + "This is not the mnist label file: " + file_name); + uint32_t num_items; + RETURN_IF_NOT_OK(ReadFromReader(label_reader, &num_items)); + if ((label_len - 8) != num_items) { + RETURN_STATUS_UNEXPECTED("Wrong number of labels!"); + } + *num_labels = num_items; + return Status::OK(); +} + +Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *label_reader, size_t index) { + uint32_t num_images, num_labels; + RETURN_IF_NOT_OK(CheckImage(image_names_[index], image_reader, &num_images)); + RETURN_IF_NOT_OK(CheckLabel(label_names_[index], label_reader, &num_labels)); + CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num_images != num_labels"); + // The image size of the Mnist dataset is fixed at [28,28] + int64_t size = kMnistImageRows * kMnistImageCols; + auto images_buf = mindspore::make_unique(size * num_images); + auto labels_buf = mindspore::make_unique(num_images); + if (images_buf == nullptr || labels_buf == nullptr) { + std::string err_msg = "Fail to allocate memory for MNIST Buffer."; + MS_LOG(ERROR) << err_msg.c_str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + (void)image_reader->read(images_buf.get(), size * num_images); + if (image_reader->fail()) { + RETURN_STATUS_UNEXPECTED("Fail to read:" + image_names_[index] + " size:" + std::to_string(size * num_images)); + } + (void)label_reader->read(labels_buf.get(), num_images); + if (label_reader->fail()) { + RETURN_STATUS_UNEXPECTED("Fail to read:" + label_names_[index] + " size: " + std::to_string(num_images)); + } + TensorShape img_tensor_shape = TensorShape({kMnistImageRows, kMnistImageCols, 1}); + for (int64_t j = 0; j != num_images; ++j) { + auto pixels = &images_buf[j * size]; + for (int64_t m = 0; m < size; ++m) { + pixels[m] = (pixels[m] == 0) ? 0 : 255; + } + std::shared_ptr image; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), img_tensor_shape, + data_schema_->column(0).type(), reinterpret_cast(pixels))); + image_label_pairs_.emplace_back(std::make_pair(image, labels_buf[j])); + } + return Status::OK(); +} + +Status MnistOp::ParseMnistData() { + for (size_t i = 0; i < image_names_.size(); ++i) { + std::ifstream image_reader, label_reader; + image_reader.open(image_names_[i], std::ios::binary); + label_reader.open(label_names_[i], std::ios::binary); + + Status s = ReadImageAndLabel(&image_reader, &label_reader, i); + // Close the readers + image_reader.close(); + label_reader.close(); + RETURN_IF_NOT_OK(s); + } + image_label_pairs_.shrink_to_fit(); + num_rows_ = image_label_pairs_.size(); + num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; + return Status::OK(); +} + +Status MnistOp::WalkAllFiles() { + const std::string kImageExtension = "idx3-ubyte"; + const std::string kLabelExtension = "idx1-ubyte"; + + Path dir(folder_path_); + auto dir_it = Path::DirIterator::OpenDirectory(&dir); + if (dir_it != nullptr) { + while (dir_it->hasNext()) { + Path file = dir_it->next(); + std::string filename = file.toString(); + if (filename.find(kImageExtension) != std::string::npos) { + image_names_.push_back(filename); + MS_LOG(INFO) << "Mnist operator found image file at " << filename << "."; + } else if (filename.find(kLabelExtension) != std::string::npos) { + label_names_.push_back(filename); + MS_LOG(INFO) << "Mnist Operator found label file at " << filename << "."; + } + } + } else { + MS_LOG(INFO) << "Mnist operator unable to open directory " << dir.toString() << "."; + } + + std::sort(image_names_.begin(), image_names_.end()); + std::sort(label_names_.begin(), label_names_.end()); + + if (image_names_.size() != label_names_.size()) { + RETURN_STATUS_UNEXPECTED("num of images does not equal to num of labels"); + } + + return Status::OK(); +} + +Status MnistOp::LaunchThreadsAndInitOp() { + if (tree_ == nullptr) { + RETURN_STATUS_UNEXPECTED("tree_ not set"); + } + RETURN_IF_NOT_OK(io_block_queues_.Register(tree_->AllTasks())); + wp_.Register(tree_->AllTasks()); + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&MnistOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + RETURN_IF_NOT_OK(this->WalkAllFiles()); + RETURN_IF_NOT_OK(this->ParseMnistData()); + RETURN_IF_NOT_OK(this->InitSampler()); // handle shake with sampler + return Status::OK(); +} + +Status MnistOp::CountTotalRows(const std::string &dir, int64_t numSamples, int64_t *count) { + // the logic of counting the number of samples is copied from ParseMnistData() and uses CheckReader() + std::shared_ptr op; + *count = 0; + RETURN_IF_NOT_OK(Builder().SetDir(dir).SetNumSamples(numSamples).Build(&op)); + + RETURN_IF_NOT_OK(op->WalkAllFiles()); + + for (size_t i = 0; i < op->image_names_.size(); ++i) { + std::ifstream image_reader; + image_reader.open(op->image_names_[i], std::ios::binary); + std::ifstream label_reader; + label_reader.open(op->label_names_[i], std::ios::binary); + + uint32_t num_images; + RETURN_IF_NOT_OK(op->CheckImage(op->image_names_[i], &image_reader, &num_images)); + uint32_t num_labels; + RETURN_IF_NOT_OK(op->CheckLabel(op->label_names_[i], &label_reader, &num_labels)); + CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num of images does not equal to num of labels"); + *count = *count + num_images; + + // Close the readers + image_reader.close(); + label_reader.close(); + } + + *count = (numSamples == 0 || *count < numSamples) ? *count : numSamples; + + return Status::OK(); +} + +// Derived from RandomAccessOp +Status MnistOp::GetNumRowsInDataset(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_rows_; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h new file mode 100644 index 00000000000..e558dca473b --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h @@ -0,0 +1,263 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_MNIST_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_MNIST_OP_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/util/path.h" +#include "dataset/util/queue.h" +#include "dataset/util/status.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +// Forward declares +template +class Queue; + +using MnistLabelPair = std::pair, int32_t>; + +class MnistOp : public ParallelOp, public RandomAccessOp { + public: + class Builder { + public: + // Constructor for Builder class of MnistOp + // @param uint32_t numWrks - number of parallel workers + // @param dir - directory folder got ImageNetFolder + Builder(); + + // Destructor. + ~Builder() = default; + + // Setter method + // @param int32_t rows_per_buffer + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method + // @param int32_t op_connector_size + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = op_connector_size; + return *this; + } + + // Setter method + // @param int32_t num_workers + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // Setter method + // @param int64_t num_samples + // @return Builder setter method returns reference to the builder. + Builder &SetNumSamples(int64_t num_samples) { + builder_num_samples_ = num_samples; + return *this; + } + + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + + // Setter method + // @param const std::string & dir + // @return + Builder &SetDir(const std::string &dir) { + builder_dir_ = dir; + return *this; + } + + // Check validity of input args + // @return - The error code return + Status SanityCheck(); + + // The builder "Build" method creates the final object. + // @param std::shared_ptr *op - DatasetOp + // @return - The error code return + Status Build(std::shared_ptr *op); + + private: + std::string builder_dir_; + int32_t builder_num_workers_; + int64_t builder_num_samples_; + int32_t builder_rows_per_buffer_; + int32_t builder_op_connector_size_; + std::shared_ptr builder_sampler_; + std::unique_ptr builder_schema_; + }; + + // Constructor + // @param int32_t num_workers - number of workers reading images in parallel + // @param int32_t rows_per_buffer - number of images (rows) in each buffer + // @param std::string folder_path - dir directory of mnist + // @param int32_t queue_size - connector queue size + // @param int64_t num_samples - number of samples to read + // @param std::unique_ptr data_schema - the schema of the mnist dataset + // @param td::unique_ptr sampler - sampler tells MnistOp what to read + MnistOp(int32_t num_workers, int32_t rows_per_buffer, std::string folder_path, int32_t queue_size, + int64_t num_samples, std::unique_ptr data_schema, std::shared_ptr sampler); + + // Destructor. + ~MnistOp() = default; + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t worker_id - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Main Loop of MnistOp + // Master thread: Fill IOBlockQueue, then goes to sleep + // Worker thread: pulls IOBlock from IOBlockQueue, work on it then put buffer to mOutConnector + // @return Status - The error code return + Status operator()() override; + + // Method derived from RandomAccess Op, enable Sampler to get numRows + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumSamples(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get total numRows in dataset + // @param int64_t num - to return numRows + // @return Status - The error code return + Status GetNumRowsInDataset(int64_t *num) const override; + + // Method derived from RandomAccess Op, enable Sampler to get all ids for each class + // @param (std::unordered_map> * map - key label, val all ids for this class + // @return Status - The error code return + Status GetClassIds(std::map> *cls_ids) const override; + + // A print method typically used for debugging + // @param out + // @param show_all + void Print(std::ostream &out, bool show_all) const override; + + // Function to count the number of samples in the MNIST dataset + // @param dir path to the MNSIT directory + // @param numSamples maximum number of samples requested + // @param count output arg that will hold the minimum of the actual dataset size and numSamples + // @return + static Status CountTotalRows(const std::string &dir, int64_t numSamples, int64_t *count); + + private: + // Initialize Sampler, calls sampler->Init() within + // @return Status - The error code return + Status InitSampler(); + + // Load a tensor row according to a pair + // @param ImageLabelPair pair - + // @param TensorRow row - image & label read into this tensor row + // @return Status - The error code return + Status LoadTensorRow(const MnistLabelPair &mnist_pair, TensorRow *row); + + // @param const std::vector &keys - keys in ioblock + // @param std::unique_ptr db + // @return Status - The error code return + Status LoadBuffer(const std::vector &keys, std::unique_ptr *db); + + // Iterate through all members in sampleIds and fill them into IOBlock. + // @param std::shared_ptr sample_ids - + // @param std::vector *keys - keys in ioblock + // @return Status - The error code return + Status TraversalSampleIds(const std::shared_ptr &sample_ids, std::vector *keys); + + // Check image file stream. + // @param const std::string *file_name - image file name + // @param std::ifstream *image_reader - image file stream + // @param uint32_t num_images - returns the number of images + // @return Status - The error code return + Status CheckImage(const std::string &file_name, std::ifstream *image_reader, uint32_t *num_images); + + // Check label stream. + // @param const std::string &file_name - label file name + // @param std::ifstream *label_reader - label file stream + // @param uint32_t num_labels - returns the number of labels + // @return Status - The error code return + Status CheckLabel(const std::string &file_name, std::ifstream *label_reader, uint32_t *num_labels); + + // Read 4 bytes of data from a file stream. + // @param std::ifstream *reader - file stream to read + // @return uint32_t - read out data + Status ReadFromReader(std::ifstream *reader, uint32_t *result); + + // Swap endian + // @param uint32_t val - + // @return uint32_t - swap endian data + uint32_t SwapEndian(uint32_t val) const; + + // Read the specified number of images and labels from the file stream + // @param std::ifstream *image_reader - image file stream + // @param std::ifstream *label_reader - label file stream + // @param int64_t read_num - number of image to read + // @return Status - The error code return + Status ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *label_reader, size_t index); + + // Parse all mnist dataset files + // @return Status - The error code return + Status ParseMnistData(); + + // Read all files in the directory + // @return Status - The error code return + Status WalkAllFiles(); + + // Called first when function is called + // @return Status - The error code return + Status LaunchThreadsAndInitOp(); + + // reset Op + // @return Status - The error code return + Status Reset() override; + + int64_t buf_cnt_; + int64_t row_cnt_; + int64_t num_rows_; // total number of images in Mnist + WaitPost wp_; + int64_t num_samples_; + std::string folder_path_; // directory of image folder + int32_t rows_per_buffer_; + std::shared_ptr sampler_; + std::unique_ptr data_schema_; + std::vector image_label_pairs_; + std::unordered_map col_name_map_; + std::vector image_names_; + std::vector label_names_; + QueueList> io_block_queues_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_MNIST_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/CMakeLists.txt new file mode 100644 index 00000000000..5d55c8276a5 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(engine-datasetops-source-sampler OBJECT + distributed_sampler.cc + pk_sampler.cc + random_sampler.cc + sampler.cc + sequential_sampler.cc + subset_random_sampler.cc + weighted_random_sampler.cc + ) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc new file mode 100644 index 00000000000..51ad71e8cf6 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h" + +#include +#include + +#include "dataset/engine/data_buffer.h" +#include "dataset/util/random.h" + +namespace mindspore { +namespace dataset { +DistributedSampler::DistributedSampler(int64_t num_dev, int64_t dev_id, bool shuffle, uint32_t seed) + : Sampler(), + cnt_(0), + seed_(seed == std::numeric_limits::max() ? GetSeed() : seed), + device_id_(dev_id), + num_devices_(num_dev), + shuffle_(shuffle) {} + +Status DistributedSampler::Init(const RandomAccessOp *op) { + RETURN_IF_NOT_OK(Sampler::Init(op)); + CHECK_FAIL_RETURN_UNEXPECTED(device_id_ < num_devices_ && device_id_ >= 0 && num_rows_ > 0 && num_samples_ > 0, + "fail to init DistributedSampler"); + rnd_.seed(seed_++); + samples_per_buffer_ = (num_rows_ + num_devices_ - 1) / num_devices_; // equals to ceil(num_rows/num_devices) + samples_per_buffer_ = num_samples_ < samples_per_buffer_ ? num_samples_ : samples_per_buffer_; + if (shuffle_ == true) { + shuffle_vec_.reserve(num_rows_); + for (int64_t i = 0; i < num_rows_; i++) { + shuffle_vec_.push_back(i); + } + std::shuffle(shuffle_vec_.begin(), shuffle_vec_.end(), rnd_); + } + return Status::OK(); +} + +Status DistributedSampler::GetNextBuffer(std::unique_ptr *out_buffer) { + if (cnt_ > samples_per_buffer_) { + RETURN_STATUS_UNEXPECTED("Distributed Sampler Error"); + } else if (cnt_ == samples_per_buffer_) { + (*out_buffer) = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + } else { + (*out_buffer) = mindspore::make_unique(cnt_, DataBuffer::kDeBFlagNone); + std::shared_ptr sample_ids; + RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_)); + int64_t *id_ptr = reinterpret_cast(sample_ids->StartAddr()); + while (cnt_ < samples_per_buffer_) { + int64_t next_id = (num_devices_ * (cnt_++) + device_id_) % num_rows_; + *(id_ptr++) = shuffle_ ? shuffle_vec_[static_cast(next_id)] : next_id; + } + TensorRow row(1, sample_ids); + (*out_buffer)->set_tensor_table(make_unique(1, row)); + } + return Status::OK(); +} + +Status DistributedSampler::Reset() { + CHECK_FAIL_RETURN_UNEXPECTED(cnt_ == samples_per_buffer_, "ERROR Reset() called early/late"); + cnt_ = 0; + rnd_.seed(seed_++); + if (shuffle_ == true) { + std::shuffle(shuffle_vec_.begin(), shuffle_vec_.end(), rnd_); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.h new file mode 100644 index 00000000000..ef25b6bccf7 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_DISTRIBUTED_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_DISTRIBUTED_SAMPLER_H_ + +#include +#include +#include +#include + +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +class DistributedSampler : public Sampler { + public: + // @param int64_t numDev + // @param int64_t devId + // @param bool shuffle + DistributedSampler(int64_t num_dev, int64_t dev_id, bool shuffle = true, + uint32_t seed = std::numeric_limits::max()); + + // default destructor + ~DistributedSampler() = default; + + // @param std::unique_ptr * pBuffer + // @param int32_t workerId + // @return - The error code return + Status GetNextBuffer(std::unique_ptr *out_buffer) override; + + // first handshake between StorageOp and Sampler + // @param op - StorageOp pointer, pass in so Sampler can call GetNumSamples() and get ClassIds() + // @return + Status Init(const RandomAccessOp *) override; + + // for next epoch of sampleIds + // @return - The error code return + Status Reset() override; + + private: + int64_t cnt_; // number of samples that have already been filled in to buffer + uint32_t seed_; + int64_t device_id_; + int64_t num_devices_; + bool shuffle_; + std::mt19937 rnd_; + std::vector shuffle_vec_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_DISTRIBUTED_SAMPLER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc new file mode 100644 index 00000000000..04a6ad17a24 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/pk_sampler.h" +#include +#include +#include "dataset/util/random.h" + +namespace mindspore { +namespace dataset { +PKSampler::PKSampler(int64_t val, bool shuffle, int64_t samples_per_buffer) + : Sampler(samples_per_buffer), + shuffle_(shuffle), + seed_(GetSeed()), + next_id_(0), + num_pk_samples_(0), + samples_per_class_(val) {} + +Status PKSampler::Init(const RandomAccessOp *op) { + RETURN_UNEXPECTED_IF_NULL(op); + RETURN_IF_NOT_OK(op->GetClassIds(&label_to_ids_)); + labels_.reserve(label_to_ids_.size()); + for (const auto &pair : label_to_ids_) { + if (pair.second.empty() == false) { + labels_.push_back(pair.first); + } + } + rnd_.seed(seed_++); + num_pk_samples_ = samples_per_class_ * static_cast(labels_.size()); + samples_per_buffer_ = (samples_per_buffer_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_; + if (shuffle_ == true) { + std::shuffle(labels_.begin(), labels_.end(), rnd_); + } else { + std::sort(labels_.begin(), labels_.end()); + } + CHECK_FAIL_RETURN_UNEXPECTED(num_pk_samples_ > 0, "num_class or K (num samples per class) is not positive"); + return Status::OK(); +} + +Status PKSampler::GetNextBuffer(std::unique_ptr *out_buffer) { + if (next_id_ > num_pk_samples_ || num_pk_samples_ == 0) { + RETURN_STATUS_UNEXPECTED("Index out of bound in PKSampler"); + } else if (next_id_ == num_pk_samples_) { + (*out_buffer) = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + } else { + (*out_buffer) = mindspore::make_unique(next_id_, DataBuffer::kDeBFlagNone); + std::shared_ptr sample_ids; + int64_t last_id = + (samples_per_buffer_ + next_id_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_ + next_id_; + RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, last_id - next_id_)); + int64_t *id_ptr = reinterpret_cast(sample_ids->StartAddr()); + while (next_id_ < last_id) { + int64_t cls_id = next_id_++ / samples_per_class_; + const std::vector &samples = label_to_ids_[labels_[cls_id]]; + int64_t rnd_ind = std::uniform_int_distribution(0, samples.size() - 1)(rnd_); + *(id_ptr++) = samples[rnd_ind]; + } + TensorRow row(1, sample_ids); + (*out_buffer)->set_tensor_table(make_unique(1, row)); + } + return Status::OK(); +} + +Status PKSampler::Reset() { + CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_pk_samples_, "ERROR Reset() called early/late"); + next_id_ = 0; + rnd_.seed(seed_++); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h new file mode 100644 index 00000000000..fa2b4ed0c79 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_PK_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_PK_SAMPLER_H_ + +#include +#include +#include +#include +#include + +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +class PKSampler : public Sampler { // NOT YET FINISHED + public: + // @param int64_t kVal + // @param bool shuffle - shuffle all classIds or not, if true, classes may be 5,1,4,3,2 + // @param int64_t samplesPerBuffer - Num of Sampler Ids to fetch via 1 GetNextBuffer call + explicit PKSampler(int64_t val, bool shuffle = false, + int64_t samples_per_buffer = std::numeric_limits::max()); + + // default destructor + ~PKSampler() = default; + + // @param std::unique_ptr *out_buffer) override; + + // first handshake between StorageOp and Sampler + // @param op - StorageOp pointer, pass in so Sampler can call GetNumSamples() and get ClassIds() + // @return + Status Init(const RandomAccessOp *op) override; + + // for next epoch of sampleIds + // @return - The error code return + Status Reset() override; + + private: + bool shuffle_; + uint32_t seed_; + int64_t next_id_; + int64_t num_pk_samples_; + int64_t samples_per_class_; + std::mt19937 rnd_; + std::vector labels_; + std::map> label_to_ids_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_PK_SAMPLER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc new file mode 100644 index 00000000000..de4d89d950e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc @@ -0,0 +1,80 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include +#include +#include "dataset/util/random.h" + +namespace mindspore { +namespace dataset { +RandomSampler::RandomSampler(bool replacement, int64_t num_samples, int64_t samples_per_buffer) + : Sampler(samples_per_buffer), + seed_(GetSeed()), + replacement_(replacement), + user_num_samples_(num_samples), + next_id_(0), + dist(nullptr) {} + +Status RandomSampler::GetNextBuffer(std::unique_ptr *out_buffer) { + if (next_id_ > num_samples_) { + RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error"); + } else if (next_id_ == num_samples_) { + (*out_buffer) = make_unique(0, DataBuffer::kDeBFlagEOE); + } else { + (*out_buffer) = make_unique(next_id_, DataBuffer::kDeBFlagNone); + std::shared_ptr sampleIds; + int64_t last_id = samples_per_buffer_ + next_id_ > num_samples_ ? num_samples_ : samples_per_buffer_ + next_id_; + RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_)); + int64_t *id_ptr = reinterpret_cast(sampleIds->StartAddr()); + for (int64_t i = 0; i < (last_id - next_id_); i++) { + *(id_ptr + i) = replacement_ ? (*dist)(rnd_) : shuffled_ids_[static_cast(i + next_id_)]; + } + next_id_ = last_id; + TensorRow row(1, sampleIds); + (*out_buffer)->set_tensor_table(make_unique(1, row)); + } + return Status::OK(); +} + +Status RandomSampler::Init(const RandomAccessOp *op) { + RETURN_IF_NOT_OK(Sampler::Init(op)); + num_samples_ = (user_num_samples_ < num_samples_) ? user_num_samples_ : num_samples_; + CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && num_rows_ > 0, "Fail to init RandomSampler"); + samples_per_buffer_ = samples_per_buffer_ > num_samples_ ? num_samples_ : samples_per_buffer_; + if (replacement_ == false) { + shuffled_ids_.reserve(num_rows_); + for (int64_t i = 0; i < num_rows_; i++) { + shuffled_ids_.push_back(i); + } + std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_); + } else { + dist = make_unique>(0, num_rows_ - 1); + } + rnd_.seed(seed_++); + return Status::OK(); +} + +Status RandomSampler::Reset() { + CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "ERROR Reset() called early/late"); + next_id_ = 0; + rnd_.seed(seed_++); + if (replacement_ == false) { + std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.h new file mode 100644 index 00000000000..54f26f352b7 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_RANDOM_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_RANDOM_SAMPLER_H_ + +#include +#include +#include + +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +class RandomSampler : public Sampler { + public: + // Constructor + // @param bool replacement - put he id back / or not after a sample + // @param int64_t numSamples - number samples to draw + // @param int64_t samplesPerBuffer - Num of Sampler Ids to fetch via 1 GetNextBuffer call + explicit RandomSampler(bool replacement = false, int64_t num_samples = std::numeric_limits::max(), + int64_t samples_per_buffer = std::numeric_limits::max()); + + // Destructor. + ~RandomSampler() = default; + + // Op calls this to get next Buffer that contains all the sampleIds + // @param std::unique_ptr pBuffer - Buffer to be returned to StorageOp + // @param int32_t workerId - not meant to be used + // @return - The error code return + Status GetNextBuffer(std::unique_ptr *out_buffer) override; + + // first handshake between StorageOp and Sampler + // @param op - StorageOp pointer, pass in so Sampler can call GetNumSamples() and get ClassIds() + // @return + Status Init(const RandomAccessOp *op) override; + + // for next epoch of sampleIds + // @return - The error code return + Status Reset() override; + + private: + uint32_t seed_; + bool replacement_; + int64_t user_num_samples_; + std::vector shuffled_ids_; // only used for NO REPLACEMENT + int64_t next_id_; + std::mt19937 rnd_; + std::unique_ptr> dist; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_RANDOM_SAMPLER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc new file mode 100644 index 00000000000..9818cd8a171 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +Sampler::Sampler(int64_t samples_per_buffer) + : DatasetOp(0), num_rows_(0), num_samples_(0), samples_per_buffer_(samples_per_buffer), col_desc_(nullptr) {} + +Status Sampler::Init(const RandomAccessOp *op) { + CHECK_FAIL_RETURN_UNEXPECTED(op != nullptr && samples_per_buffer_ > 0, "Fail to init Sampler()\n"); + RETURN_IF_NOT_OK(op->GetNumSamples(&num_samples_)); + RETURN_IF_NOT_OK(op->GetNumRowsInDataset(&num_rows_)); + // It's up to the derived class to check the validity of the two args + // Because some sampler only needs one of the arg (weighted_random_sampler) + return Status::OK(); +} + +Status Sampler::CreateSamplerTensor(std::shared_ptr *sample_ids, int64_t num_elements) { + if (num_elements == 0) { + RETURN_STATUS_UNEXPECTED("num of Elements is 0"); + } + if (col_desc_ == nullptr) { + // a ColDescriptor for Tensor that holds SampleIds + col_desc_ = make_unique("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1); + } + TensorShape shape(std::vector(1, num_elements)); + RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type())); + (void)(*sample_ids)->StartAddr(); // allocate memory in case user forgets! + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.h new file mode 100644 index 00000000000..d9a20f91700 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.h @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SAMPLER_H_ + +#include +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/util/make_unique.h" + +namespace mindspore { +namespace dataset { +// RandomAccessOp is a base class that all data-producing leaf operators +// must inherit from if those leaf operator wish to support sampling. +class RandomAccessOp { + public: + // Sampler get numRows from StorageOp + // @param int64_t num - return number of rows, normally num of samples + // @return - The error code return + virtual Status GetNumSamples(int64_t *num_samples) const { + // CI complains num_samples not used if the following line is not added + CHECK_FAIL_RETURN_UNEXPECTED(num_samples != nullptr, "num_samples == nullptr"); + RETURN_STATUS_UNEXPECTED("function GetNumSamples needs to overridden to support this sampler"); + } + + // Sampler get number of rows in the dataset! + // @param int64_t num - return number of rows for this dataset + // @return - The error code return + virtual Status GetNumRowsInDataset(int64_t *num_rows) const { + // CI complains num_rows not used if the following line is not added + CHECK_FAIL_RETURN_UNEXPECTED(num_rows != nullptr, "num_rows == nullptr"); + RETURN_STATUS_UNEXPECTED("function GetNumRowsInDataset needs to overridden to support this sampler"); + } + + // sampler gets label , imageIds from storageOp, this function is unique to PK + // @param std::unordered_map> * map + // @return - The error code return + virtual Status GetClassIds(std::map> *map) const { + RETURN_STATUS_UNEXPECTED("GetClassIds needs to be override to support PK"); + } + + // default destructor + virtual ~RandomAccessOp() = default; +}; + +class Sampler : public DatasetOp { + public: + // @param int64_t samplesPerBuffer: Num of Sampler Ids to fetch via 1 GetNextBuffer call + explicit Sampler(int64_t samples_per_buffer = std::numeric_limits::max()); + + // default destructor + ~Sampler() = default; + + // Get a list of sample ids. + // @note It is Sampler responsibility to make sure that the id is not out of bound. + // @param std::unique_ptr pBuffer - Buffer to be returned to StorageOp + // @param int32_t workerId - not meant to be used + // @return - The error code return + Status GetNextBuffer(std::unique_ptr *out_buffer) override = 0; + + // for next epoch of sampleIds + // @return - The error code return + Status Reset() override = 0; + + // first handshake between StorageOp and Sampler. Base class init will call both GetNumRows and GetNumSamples + // @param op - StorageOp pointer, pass in so Sampler can call GetNumSamples() and get ClassIds() + // @return + virtual Status Init(const RandomAccessOp *op); + + // Not meant to be called + // @return + int32_t num_workers() const final { return 0; } + + // Not meant to be called + // @return + int32_t num_consumers() const final { return 0; } + + // Not meant to be called + // @return + int32_t num_producers() const final { return 0; } + + // Not meant to be called! + // @return - The error code return + Status operator()() final { RETURN_STATUS_UNEXPECTED("Functor not supported in Sampler"); } + + // A helper function to create a int64_t 1-D Tensor specifically used to hold sampleIds for Sampler + // @param std::shared_ptr* sampleIds + // @param int64_t numElements - must be a non 0 number + // @return + Status CreateSamplerTensor(std::shared_ptr *sample_ids, int64_t num_elements); + + protected: + int64_t num_rows_; + int64_t num_samples_; + int64_t samples_per_buffer_; + std::unique_ptr col_desc_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SAMPLER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc new file mode 100644 index 00000000000..71c3dd07c49 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" + +#include + +namespace mindspore { +namespace dataset { +SequentialSampler::SequentialSampler(int64_t samples_per_buffer) : Sampler(samples_per_buffer), next_id_(0) {} + +Status SequentialSampler::GetNextBuffer(std::unique_ptr *out_buffer) { + if (next_id_ > num_samples_) { + RETURN_STATUS_UNEXPECTED("Sequential Sampler Internal Error"); + } else if (next_id_ == num_samples_) { + (*out_buffer) = make_unique(0, DataBuffer::kDeBFlagEOE); + } else { + (*out_buffer) = make_unique(next_id_, DataBuffer::kDeBFlagNone); + std::shared_ptr sampleIds; + int64_t lastId = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_; + RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, lastId - next_id_)); + int64_t *idPtr = reinterpret_cast(sampleIds->StartAddr()); + while (next_id_ < lastId) { + *(idPtr++) = next_id_++; + } + TensorRow row(1, sampleIds); + (*out_buffer)->set_tensor_table(make_unique(1, row)); + } + return Status::OK(); +} + +Status SequentialSampler::Init(const RandomAccessOp *op) { + RETURN_UNEXPECTED_IF_NULL(op); + RETURN_IF_NOT_OK(op->GetNumSamples(&num_samples_)); + CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init Sequential Sampler"); + samples_per_buffer_ = samples_per_buffer_ > num_samples_ ? num_samples_ : samples_per_buffer_; + return Status::OK(); +} + +Status SequentialSampler::Reset() { + CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "ERROR Reset() called early/late"); + next_id_ = 0; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h new file mode 100644 index 00000000000..d119fd8d088 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SEQUENTIAL_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SEQUENTIAL_SAMPLER_H_ + +#include +#include + +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +class SequentialSampler : public Sampler { + public: + // Constructor + // @param int64_t samplesPerBuffer - Num of Sampler Ids to fetch via 1 GetNextBuffer call + explicit SequentialSampler(int64_t samples_per_buffer = std::numeric_limits::max()); + + // Destructor. + ~SequentialSampler() = default; + + // Initialize the sampler. + // @param op + // @return Status + Status Init(const RandomAccessOp *op) override; + + // for next epoch of sampleIds + // @return - The error code return + Status Reset() override; + + // Op calls this to get next Buffer that contains all the sampleIds + // @param std::unique_ptr pBuffer - Buffer to be returned to StorageOp + // @param int32_t workerId - not meant to be used + // @return - The error code return + Status GetNextBuffer(std::unique_ptr *out_buffer) override; + + private: + int64_t next_id_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SEQUENTIAL_SAMPLER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc new file mode 100644 index 00000000000..4f727fcd04d --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" + +#include +#include +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/global_context.h" +#include "dataset/util/random.h" + +namespace mindspore { +namespace dataset { +// Constructor. +SubsetRandomSampler::SubsetRandomSampler(const std::vector &indices, int64_t samples_per_buffer) + : Sampler(samples_per_buffer), indices_(indices), sample_id_(0), buffer_id_(0) {} + +// Initialized this Sampler. +Status SubsetRandomSampler::Init(const RandomAccessOp *op) { + // Calling base class init. + RETURN_IF_NOT_OK(Sampler::Init(op)); + + // Initialize random generator with seed from config manager + rand_gen_.seed(GetSeed()); + + if (static_cast(samples_per_buffer_) > indices_.size()) { + samples_per_buffer_ = static_cast(indices_.size()); + } + + std::shuffle(indices_.begin(), indices_.end(), rand_gen_); + + return Status::OK(); +} + +// Reset the internal variable to the initial state. +Status SubsetRandomSampler::Reset() { + // Reset the internal counters. + sample_id_ = 0; + buffer_id_ = 0; + + // Randomized the indices again. + rand_gen_.seed(GetSeed()); + std::shuffle(indices_.begin(), indices_.end(), rand_gen_); + + return Status::OK(); +} + +// Get the sample ids. +Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr *out_buffer) { + // All samples have been drawn + if (sample_id_ == indices_.size()) { + (*out_buffer) = make_unique(buffer_id_++, DataBuffer::kDeBFlagEOE); + } else { + (*out_buffer) = make_unique(buffer_id_++, DataBuffer::kDeBFlagNone); + std::shared_ptr outputIds; + + int64_t last_id = sample_id_ + samples_per_buffer_; + // Handling the return all samples at once, and when last draw is not a full batch. + if (static_cast(last_id) > indices_.size()) { + last_id = indices_.size(); + } + + // Allocate tensor + RETURN_IF_NOT_OK(CreateSamplerTensor(&outputIds, last_id - sample_id_)); + + // Initialize tensor + int64_t *id_ptr = reinterpret_cast(outputIds->StartAddr()); + while (sample_id_ < last_id) { + if (indices_[sample_id_] >= num_rows_) { + std::string err_msg = + "Generated id is bigger than numRows (out of bound). indices_: " + std::to_string(indices_[sample_id_]) + + " num_rows_: " + std::to_string(num_rows_); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + *(id_ptr++) = indices_[sample_id_++]; + } + + // Create a TensorTable from that single tensor and push into DataBuffer + (*out_buffer)->set_tensor_table(make_unique(1, TensorRow(1, outputIds))); + } + + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h new file mode 100644 index 00000000000..38fae6b20b7 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SUBSET_RANDOM_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SUBSET_RANDOM_SAMPLER_H_ + +#include +#include +#include + +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +// Randomly samples elements from a given list of indices, without replacement. +class SubsetRandomSampler : public Sampler { + public: + // Constructor. + // @param indices List of indices from where we will randomly draw samples. + // @param samples_per_buffer The number of ids we draw on each call to GetNextBuffer(). + // When samplesPerBuffer=0, GetNextBuffer() will draw all the sample ids and return them at once. + explicit SubsetRandomSampler(const std::vector &indices, + std::int64_t samples_per_buffer = std::numeric_limits::max()); + + // Destructor. + ~SubsetRandomSampler() = default; + + // Initialize the sampler. + // @param op (Not used in this sampler) + // @return Status + Status Init(const RandomAccessOp *op) override; + + // Reset the internal variable to the initial state and reshuffle the indices. + // @return Status + Status Reset() override; + + // Get the sample ids. + // @param[out] out_buffer The address of a unique_ptr to DataBuffer where the sample ids will be placed. + // @note the sample ids (int64_t) will be placed in one Tensor and be placed into pBuffer. + Status GetNextBuffer(std::unique_ptr *out_buffer) override; + + private: + // A list of indices (already randomized in constructor). + std::vector indices_; + + // Current sample id. + int64_t sample_id_; + + // Current buffer id. + int64_t buffer_id_; + + // A random number generator. + std::mt19937 rand_gen_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_SUBSET_RANDOM_SAMPLER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc new file mode 100644 index 00000000000..f3054741828 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc @@ -0,0 +1,141 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" + +#include +#include +#include +#include +#include + +#include "dataset/core/global_context.h" +#include "dataset/util/random.h" + +namespace mindspore { +namespace dataset { +// Constructor. +WeightedRandomSampler::WeightedRandomSampler(const std::vector &weights, int64_t num_samples, bool replacement, + int64_t samples_per_buffer) + : Sampler(samples_per_buffer), weights_(weights), replacement_(replacement), sample_id_(0), buffer_id_(0) { + num_samples_ = num_samples; // this variable is defined in base class sampler +} + +// Initialized this Sampler. +Status WeightedRandomSampler::Init(const RandomAccessOp *op) { + RETURN_UNEXPECTED_IF_NULL(op); + RETURN_IF_NOT_OK(op->GetNumRowsInDataset(&num_rows_)); + + // Initialize random generator with seed from config manager + rand_gen_.seed(GetSeed()); + + samples_per_buffer_ = (samples_per_buffer_ > num_samples_) ? num_samples_ : samples_per_buffer_; + + CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init WeightedRandomSampler"); + + if (!replacement_) { + exp_dist_ = mindspore::make_unique>(1); + InitOnePassSampling(); + } else { + discrete_dist_ = mindspore::make_unique>(weights_.begin(), weights_.end()); + } + + return Status::OK(); +} + +// Initialized the computation for generating weighted random numbers without replacement using onepass method. +void WeightedRandomSampler::InitOnePassSampling() { + exp_dist_->reset(); + onepass_ids_.clear(); + std::vector> val_idx; + for (size_t i = 0; i < weights_.size(); i++) { + val_idx.emplace_back(std::make_pair((*exp_dist_)(rand_gen_) / weights_[i], i)); + } + + // Partial sort the first `numSamples` elements. + std::partial_sort(val_idx.begin(), val_idx.begin() + num_samples_, val_idx.end()); + for (int64_t i = 0; i < num_samples_; i++) { + onepass_ids_.push_back(val_idx[i].second); + } +} + +// Reset the internal variable to the initial state and reshuffle the indices. +Status WeightedRandomSampler::Reset() { + sample_id_ = 0; + buffer_id_ = 0; + rand_gen_.seed(GetSeed()); + if (!replacement_) { + InitOnePassSampling(); + } else { + discrete_dist_->reset(); + } + return Status::OK(); +} + +// Get the sample ids. +Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr *out_buffer) { + if (weights_.size() > static_cast(num_rows_)) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "number of samples weights is more than num of rows. Might generate id out of bound OR other errors"); + } + + if (!replacement_ && (weights_.size() < static_cast(num_samples_))) { + RETURN_STATUS_UNEXPECTED("Without replacement, sample weights less than numSamples"); + } + + if (sample_id_ == num_samples_) { + (*out_buffer) = make_unique(buffer_id_++, DataBuffer::kDeBFlagEOE); + } else { + (*out_buffer) = make_unique(buffer_id_++, DataBuffer::kDeBFlagNone); + std::shared_ptr outputIds; + + int64_t last_id = sample_id_ + samples_per_buffer_; + // Handling the return all samples at once, and when last draw is not a full batch. + if (last_id > num_samples_) { + last_id = num_samples_; + } + + // Allocate tensor. + RETURN_IF_NOT_OK(CreateSamplerTensor(&outputIds, last_id - sample_id_)); + + // Initialize tensor. + int64_t *id_ptr = reinterpret_cast(outputIds->StartAddr()); + // Assign the data to tensor element. + while (sample_id_ < last_id) { + int64_t genId; + if (replacement_) { + genId = (*discrete_dist_)(rand_gen_); + } else { + // Draw sample without replacement. + genId = onepass_ids_.front(); + onepass_ids_.pop_front(); + } + + if (genId >= num_rows_) { + RETURN_STATUS_UNEXPECTED("generated id is bigger than numRows (out of bound)."); + } + + *(id_ptr++) = genId; + sample_id_++; + } + + // Create a TensorTable from that single tensor and push into DataBuffer + (*out_buffer)->set_tensor_table(make_unique(1, TensorRow(1, outputIds))); + } + + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h new file mode 100644 index 00000000000..bccc9e599d1 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h @@ -0,0 +1,89 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_WEIGHTED_RANDOM_SAMPLER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_SAMPLER_WEIGHTED_RANDOM_SAMPLER_H_ + +#include +#include +#include +#include + +#include "dataset/engine/datasetops/source/sampler/sampler.h" + +namespace mindspore { +namespace dataset { +// Samples elements from id `0, 1, ..., weights.size()-1` with given probabilities (weights). +class WeightedRandomSampler : public Sampler { + public: + // Constructor. + // @param weights A lift of sample weights. + // @param num_samples Number of samples to be drawn. + // @param replacement Determine if samples are drawn with/without replacement. + // @param samples_per_buffer The number of ids we draw on each call to GetNextBuffer(). + // When samplesPerBuffer=0, GetNextBuffer() will draw all the sample ids and return them at once. + WeightedRandomSampler(const std::vector &weights, int64_t num_samples, bool replacement = true, + int64_t samples_per_buffer = std::numeric_limits::max()); + + // Destructor. + ~WeightedRandomSampler() = default; + + // Initialize the sampler. + // @param op (Not used in this sampler) + // @return Status + Status Init(const RandomAccessOp *op) override; + + // Reset the internal variable to the initial state and reshuffle the indices. + Status Reset() override; + + // Get the sample ids. + // @param[out] out_buffer The address of a unique_ptr to DataBuffer where the sample ids will be placed. + // @note the sample ids (int64_t) will be placed in one Tensor and be placed into pBuffer. + Status GetNextBuffer(std::unique_ptr *out_buffer) override; + + private: + // A list of weights for each sample. + std::vector weights_; + + // A flag indicating if samples are drawn with/without replacement. + bool replacement_; + + // Current sample id. + int64_t sample_id_; + + // Current buffer id. + int64_t buffer_id_; + + // Random engine and device + std::mt19937 rand_gen_; + + // Discrete distribution for generating weighted random numbers with replacement. + std::unique_ptr> discrete_dist_; + + // Exponential distribution for generating weighted random numbers without replacement. + // based on "Accelerating weighted random sampling without replacement" by Kirill Muller. + std::unique_ptr> exp_dist_; + + // Initialized the computation for generating weighted random numbers without replacement + // using onepass method. + void InitOnePassSampling(); + + // Store the random weighted ids generated by onepass method in `InitOnePassSampling` + std::deque onepass_ids_; +}; +} // namespace dataset +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.cc new file mode 100644 index 00000000000..95720a97be1 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.cc @@ -0,0 +1,187 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define MAX_INTEGER_INT32 2147483647 + +#include +#include +#include +#include +#include "dataset/core/constants.h" +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/engine/datasetops/source/storage_op.h" +#include "dataset/engine/datasetops/source/tf_client.h" +#include "dataset/util/make_unique.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Name: Constructor +// Description: +StorageClient::StorageClient(std::unique_ptr schema, // In: The schema for this storage client. + StorageOp *store_op) // In: The StorageOp that's using this client + : data_schema_(std::move(schema)), num_rows_in_dataset_(0), storage_op_(store_op), num_classes_(0) {} + +// Name: Print() +// Description: A function that prints info about the StorageClient +// In: The output stream to print to +void StorageClient::Print(std::ostream &out) const { + // not much to show here folks! + // out << "Storage client:\n"; +} + +// This is a local-only static function to drive the switch statement for creating +// the storage client (not a static member function) +static Status CreateStorageClientSwitch( + std::unique_ptr schema, // In: The schema to set into the client + StorageOp *store_op, // In: The StorageOp we are operating on + std::shared_ptr *out_client) { // Out: the created storage client + switch (schema->dataset_type()) { + case DatasetType::kArrow: { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Storage client not implemented yet for arrow dataset type."); + } + case DatasetType::kTf: { + // Construct the derived class TFClient, stored as base class StorageClient + store_op->set_rows_per_buffer(32); + *out_client = mindspore::make_unique(std::move(schema), store_op); + break; + } + case DatasetType::kUnknown: + default: { + RETURN_STATUS_UNEXPECTED("Invalid dataset type."); + } + } + if (*out_client) { + RETURN_IF_NOT_OK((*out_client)->Init()); + } + return Status::OK(); +} + +// Name: CreateStorageClient() +// Description: A factory method to create the derived storage client. +// Every dataset has a required field for the dataset type in a config +// file. This type will determine the child class to return for the +// type of storage client. It also creates the schema and sticks it +// into the cache. +Status StorageClient::CreateStorageClient( + StorageOp *store_op, // In: A backpointer to the owning cache for this client. + std::string dataset_schema_path, // In: The path to the schema + std::shared_ptr *out_client) { // Out: the created storage client + // Make a new schema first. This only assigns the dataset type. It does not + // create the columns yet. + auto new_schema = mindspore::make_unique(); + RETURN_IF_NOT_OK(new_schema->LoadDatasetType(dataset_schema_path)); + RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); + return Status::OK(); +} + +// Name: CreateStorageClient() +// Description: A factory method to create the derived storage client. +// This creator is a user-override for the schema properties where +// the user has input the layout of the data (typically used in testcases) +Status StorageClient::CreateStorageClient( + StorageOp *store_op, // In: A backpointer to the owning cache for this client. + DatasetType in_type, // In: The type of dataset + std::shared_ptr *out_client) { // Out: the created storage client + // The dataset type is passed in by the user. Create an empty schema with only + // only the dataset type filled in and then create the client with it. + auto new_schema = mindspore::make_unique(); + new_schema->set_dataset_type(in_type); + RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client)); + return Status::OK(); +} + +// Name: LoadDatasetLayout() +// Description: There are 2 ways to define the properties of the data in the storage +// layer: LoadDatasetLayout() and AssignDatasetLayout(). +// LoadDatasetLayout() will parse the json config file that comes with +// the dataset. +Status StorageClient::LoadDatasetLayout() { + // Access the json file to populate our schema, assume the json file is accessible + // locally. + RETURN_IF_NOT_OK(data_schema_->LoadSchemaFile(storage_op_->schema_file(), storage_op_->columns_to_load())); + + // The number of rows in the schema file is an optional config. For example, + // maybe the derived storage client will know how to determine the total number + // of rows a different way rather than having it in the schema config json file. + // Thus, mNumRowsInDataset can still be zero and force the derived class override + // to determine it another way. + uint32_t num_rows = 0; + RETURN_IF_NOT_OK(this->numRowsFromFile(num_rows)); + CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647"); + if (num_rows_in_dataset_ == 0 || num_rows < num_rows_in_dataset_) { + num_rows_in_dataset_ = num_rows; + } + + return Status::OK(); +} + +// Name: AssignDatasetLayout() +// Description: There are 2 ways to define the properties of the data in the storage +// layer: LoadDatasetLayout() and AssignDatasetLayout(). +// AssignDatasetLayout() will take input from the caller and assign that +// info into the storage client. +Status StorageClient::AssignDatasetLayout(uint32_t num_rows, // In: The number of rows in the dataset + const DataSchema &schema) { // In: The schema for the dataset + // Since this is just an assignment into the storage client, you probably won't need + // to override this one in a derived class. First some sanity checks + CHECK_FAIL_RETURN_UNEXPECTED(data_schema_->dataset_type() == schema.dataset_type(), + "Assigning a schema into StorageClient with mismatched dataset types!"); + CHECK_FAIL_RETURN_UNEXPECTED(data_schema_->NumColumns() == 0, + "Assigning a schema into StorageClient that already has non-empty schema!"); + + // The current schema was just an empty one with only the dataset field populated. + // Let's copy construct a new one that will be a copy of the input schema (releasing the old + // one) and then set the number of rows that the user requested. + data_schema_ = mindspore::make_unique(schema); + CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647"); + num_rows_in_dataset_ = num_rows; + + return Status::OK(); +} + +// Name: numRowsFromFile() +// Description: Reads the schema json file to see if the optional numRows field has +// been set and returns it. +Status StorageClient::numRowsFromFile(uint32_t &num_rows) const { + std::string schemaFile = storage_op_->schema_file(); + try { + std::ifstream in(schemaFile); + nlohmann::json js; + in >> js; + num_rows = js.value("numRows", 0); + if (num_rows == 0) { + std::string err_msg = + "Storage client has not properly done dataset " + "handshake to initialize schema and number of rows."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + // Catch any exception and rethrow it as our own + catch (const std::exception &err) { + std::ostringstream ss; + ss << "Schema file failed to load:\n" << err.what(); + std::string err_msg = ss.str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + return Status::OK(); +} + +// Get'r function +DataSchema *StorageClient::schema() const { return data_schema_.get(); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.h b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.h new file mode 100644 index 00000000000..6198f4233f0 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_client.h @@ -0,0 +1,128 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_STORAGE_CLIENT_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_STORAGE_CLIENT_H_ + +#include +#include +#include +#include +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/source/storage_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// The Storage Client is the interface and base class that the StorageOp +// will use to perform any interactions with the storage layer. +// The different types of datasets will have different derived classes +// under that storage client super class. +class StorageClient { + public: + // Name: Constructor + // Description: + StorageClient(std::unique_ptr schema, // In: The schema for this storage client. + StorageOp *store_op); // In: The StorageOp that's using this client + + // Destructor + virtual ~StorageClient() { storage_op_ = nullptr; } + + virtual Status Init() { return Status::OK(); } + + // Name: CreateStorageClient() + // Description: A factory method to create the derived storage client. + // Every dataset has a required field for the dataset type in a config + // file. This type will determine the child class to return for the + // type of storage client. + static Status CreateStorageClient(StorageOp *store_op, // In: A backpointer to the owning storage op for this client. + std::string dataset_schema_path, // In: The path to the dataset + std::shared_ptr *out_client); // Out: the created storage client + + // Name: CreateStorageClient() + // Description: A factory method to create the derived storage client. + // This creator is a user-override for the schema properties where + // the user has input the layout of the data (typically used in testcases) + static Status CreateStorageClient(StorageOp *store_op, // In: A backpointer to the owning cache for this client. + DatasetType in_type, // In: The type of dataset + std::shared_ptr *out_client); // Out: the created storage client + + // Name: Print() + // Description: A function that prints info about the StorageClient + virtual void Print(std::ostream &out) const; // In: The output stream to print to + + // Provide stream operator for displaying + friend std::ostream &operator<<(std::ostream &out, const StorageClient &storage_client) { + storage_client.Print(out); + return out; + } + + // Name: LoadDatasetLayout() + // Description: There are 2 ways to define the properties of the data in the storage + // layer: LoadDatasetLayout() and AssignDatasetLayout(). + // LoadDatasetLayout() will parse the json config file that comes with + // the dataset and internally populate row counts and schema. + virtual Status LoadDatasetLayout(); + + // Name: AssignDatasetLayout() + // Description: There are 2 ways to define the properties of the data in the storage + // layer: LoadDatasetLayout() and AssignDatasetLayout(). + // AssignDatasetLayout() will take input from the caller and assign that + virtual Status AssignDatasetLayout(uint32_t num_rows, // In: The number of rows in the dataset + const DataSchema &schema); // In: The schema for the dataset + + // Name: Reset() + // Description: Resets any state info inside the client back to it's initialized + // state. + virtual Status Reset() = 0; + + // Name: IsMoreData + // Description: General routine to ask if more data exists in the storage side for + // a given buffer id. + virtual bool IsMoreData(uint32_t id) { return true; } + + // Name: numRowsFromFile() + // Description: Reads the schema json file to see if the optional numRows field has + // been set and returns it. + Status numRowsFromFile(uint32_t &num_rows) const; + + // Get'r functions + DataSchema *schema() const; + + uint32_t num_rows() const { return num_rows_in_dataset_; } + + // Name: rows_per_buffer() + // Description: This default version simply gives you the count of the requested + // rows per buffer that the user defined in the storage op. + // However, if some condition down in the storage client layers + // could result in a buffer that has a different number of rows, + // then the derived class can override this method to provide their + // own implementation. + virtual uint32_t rows_per_buffer() { return storage_op_->rows_per_buffer(); } + + // Description: Get the label classes num. Only manifest and Imagenet dataset support this parameter + virtual uint32_t num_classes() const { return 0; } + + protected: + std::unique_ptr data_schema_; // The schema for the data + uint32_t num_rows_in_dataset_; // The number of rows in the dataset + StorageOp *storage_op_; // Back pointer to the owning storage operator. + std::vector col_names_; + uint32_t num_classes_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_STORAGE_CLIENT_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc new file mode 100644 index 00000000000..9da27eac04a --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc @@ -0,0 +1,613 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define MAX_INTEGER_UINT32 4294967295 +#define MAX_INTEGER_INT32 2147483647 + +#include "dataset/engine/datasetops/source/storage_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/constants.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/queue.h" +#include "dataset/engine/datasetops/source/storage_op.h" +#include "dataset/util/task_manager.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +// Builder constructor. Creates the builder object. +StorageOp::Builder::Builder() + : build_dataset_files_dir_(""), + build_schema_file_(""), + build_num_rows_(0), + build_data_distribution_file_(""), + build_batch_size_(1), + build_drop_remainder_(false) { + // Some arguments to the StorageOp constructor have a default argument that is taken + // from the client config. + // The user may choose to change these values for the construction of the StorageOp by + // using the various builder set methods. + + std::shared_ptr cfg = GlobalContext::config_manager(); + build_rows_per_buffer_ = cfg->rows_per_buffer(); + build_worker_connector_size_ = cfg->worker_connector_size(); + build_num_workers_ = cfg->num_parallel_workers(); + build_op_connector_size_ = cfg->op_connector_size(); +} + +// The builder "build" method creates the final object. +Status StorageOp::Builder::Build(std::shared_ptr *ptr) { + // There are 2 "flavours" of construction for a StorageOp: + // + // 1) Does a handshake with the dataset to identify row ranges and to identify + // the schema (internally the handshake does lookup against a json file in the dataset) + // + // 2) The user manually creates a schema and defines the row ranges, so there is no real + // dataset handshake. + // + // The decision about which style is called will depend on if the user supplied the + // schema and row range fields. + + const std::string dataset_schema_file("datasetSchema.json"); + if (build_schema_ != nullptr && build_num_rows_ == 0) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Building a StorageOp with a given schema, but the number of rows not specified!"); + } + if (build_schema_ == nullptr && build_num_rows_ != 0) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Building a StorageOp with a given number of rows but schema not specified!"); + } + if (build_dataset_files_dir_.empty() && build_dataset_file_list_.empty()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Building a StorageOp that has not provided the location of the data files."); + } + if (!build_dataset_files_dir_.empty() && !build_dataset_file_list_.empty()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Building a StorageOp that has provided conflicting location of the data files."); + } + + std::shared_ptr new_storage_op = std::make_shared( + build_num_workers_, build_worker_connector_size_, build_rows_per_buffer_, build_op_connector_size_, + build_columns_to_load_, build_data_distribution_file_, build_batch_size_, build_drop_remainder_); + + // If there is no schema or number of rows given, then we go with construction method 1 + // where we need to handshake with storage client to find out what the schema (and + // number of rows) are based on schema file. + if (build_schema_ == nullptr && build_num_rows_ == 0) { + if (!build_dataset_files_dir_.empty()) { + // We have a dataset files dir, but do not have a schema file. + // Set the default schema file to be inside the same path as the dataset files dir. + if (build_schema_file_.empty()) { + build_schema_file_ = build_dataset_files_dir_ + "/" + dataset_schema_file; + } + RETURN_IF_NOT_OK(new_storage_op->InitOp(build_dataset_files_dir_, build_schema_file_, build_labels_file_name_, + build_dataset_usage_)); + } else { + // dataset is provided by list of files not dir_path + RETURN_IF_NOT_OK(new_storage_op->InitOp(build_dataset_file_list_, build_schema_file_)); + } + } else { + // else, the user gave us a schema and a row range, go with construction method 2, where we use + // the user-provided schema, but we still need to identify our data files. + RETURN_IF_NOT_OK(new_storage_op->InitOp(build_num_rows_, build_dataset_files_dir_, std::move(build_schema_), + build_labels_file_name_, build_dataset_usage_)); + } + + // Call the actual workhorse of the constructor + RETURN_IF_NOT_OK(new_storage_op->init()); + *ptr = std::move(new_storage_op); + return Status::OK(); +} + +StorageOp::StorageOp(int32_t num_workers, int32_t worker_connector_size, int32_t rows_per_buffer, + int32_t op_connector_size, std::vector columns_to_load, + std::string data_distribution_file, int32_t batch_size, bool drop_remainder) + : ParallelOp(num_workers, op_connector_size), + worker_conn_size_(worker_connector_size), + rows_per_buffer_(rows_per_buffer), + num_rows_(0), + buffers_fetched_(0), + columns_to_load_(columns_to_load), + data_distribution_file_(data_distribution_file), + device_num_(1), + device_id_(0), + shard_config_("ALL"), + seed_(0), + shuffle_config_(false), + num_classes_(0), + batch_size_(batch_size), + drop_remainder_(drop_remainder) {} + +// Init of the StorageOp. This is 1 of 3 init. +// This version of the init does not take the schema in it's arguments. It must perform an +// internal handshake with the dataset to produce the schema. +Status StorageOp::InitOp(const std::string &dataset_files_dir, const std::string &schema_file, + const std::string &labels_file_name, const std::string &dataset_usage) { + dataset_files_dir_ = dataset_files_dir; + schema_file_ = schema_file; + labels_file_name_ = labels_file_name; + dataset_usage_ = dataset_usage; + + // Storage ops require the internal master/worker connector. create it here + RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_conn_size_)); + + // Get parameter for distribution. + RETURN_IF_NOT_OK(LoadParallelConfig()); + + // Create the storage client. This will read the json file to determine what + // type of client we're creating. + RETURN_IF_NOT_OK(StorageClient::CreateStorageClient(this, schema_file_, &store_client_)); + + // Perform the initial handshake with the storage client to further read the + // dataset info to populate schema info and the number of rows in the client. + RETURN_IF_NOT_OK(store_client_->LoadDatasetLayout()); + + // Pull out the number of rows from the client and save into the op. + num_rows_ = store_client_->num_rows(); + num_classes_ = store_client_->num_classes(); + + return Status::OK(); +} + +// Init of the StorageOp. This is 2 of 3 init. +// This version of the init allows the user to input the schema and other dataset properties rather +// than get it from the dataset itself. +Status StorageOp::InitOp(int32_t num_rows, const std::string &dataset_files_dir, + std::unique_ptr data_schema, const std::string &labels_file_name, + const std::string &dataset_usage) { + num_rows_ = num_rows; + dataset_files_dir_ = dataset_files_dir; + labels_file_name_ = labels_file_name; + dataset_usage_ = dataset_usage; + + // Storage ops require the internal master/worker connector. create it here + RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_conn_size_)); + + // Get parameter for distribution. + RETURN_IF_NOT_OK(LoadParallelConfig()); + + // Create the storage client based on the dataset type given from the input schema. + RETURN_IF_NOT_OK(StorageClient::CreateStorageClient(this, data_schema->dataset_type(), &store_client_)); + + // Perform the initial handshake with the storage client to initialize the schema + // and the number of rows in the set. In this case, since the schema and the number + // of rows is input by the user directly, it's not much of a "handshake", it's more + // like an assign. + RETURN_IF_NOT_OK(store_client_->AssignDatasetLayout(num_rows_, *data_schema)); + num_classes_ = store_client_->num_classes(); + + return Status::OK(); +} + +// Init of the StorageOp. This is 3 of 3 init. +// This version of the init does not take the schema in it's arguments. It must perform an +// internal handshake with the dataset to produce the schema. Unlike constructor 1, it takes a +// list of files rather than a directory. +Status StorageOp::InitOp(const std::vector &files_list, const std::string &schema_file) { + dataset_file_list_ = files_list; + schema_file_ = schema_file; + + // Storage ops require the internal master/worker connector. create it here + RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_conn_size_)); + + // Get parameter for distribution. + RETURN_IF_NOT_OK(LoadParallelConfig()); + + // Create the storage client. This will read the json file to determine what + // type of client we're creating. + RETURN_IF_NOT_OK(StorageClient::CreateStorageClient(this, schema_file_, &store_client_)); + + // Perform the initial handshake with the storage client to further read the + // dataset info to populate schema info and the number of rows in the client. + RETURN_IF_NOT_OK(store_client_->LoadDatasetLayout()); + + // Pull out the number of rows from the client and save into the op. + num_rows_ = store_client_->num_rows(); + + return Status::OK(); +} + +// Private helper method. This one encapsulates some common construction/reset tasks and is +// designed to be re-entrant so that you can re-init a previously used StorageOp without needing +// to redo the storage client handshake. +Status StorageOp::init() { + // First a sanity check to make sure the StorageClient initialization has done the proper + // handshake and initialized both the schema and the number of rows for the dataset. + if (store_client_->schema()->NumColumns() == 0 || num_rows_ == 0) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Storage client did not run handshake to init schema and number of rows."); + } + + // If the data buffer vector is not empty, then we may be redoing a scan again after a repeat. + // In such a case, we have vector of nullptrs that used to hold the buffers. get rid of this + // so we can reuse the vector. + if (!data_buffers_.empty()) { + data_buffers_.clear(); + } + int32_t buffers_needed; + + // We have our range of row id's, but we must carve this up into buffers now so that + // each buffer holds a subset of the overall range. + // Instantiate the buffers now, but this does not actually drive a load of actual + // data at this point. + + // First, compute how many buffers we would need to accomplish rowsPerBuffer + buffers_needed = this->num_rows() / rows_per_buffer_; + + // If an extra partial buffer is needed, adjust for that. + if (this->num_rows() % rows_per_buffer_ != 0) { + buffers_needed++; + } + MS_LOG(INFO) << "Master: Initializing StorageOp. Dataset files dir: " << dataset_files_dir_ << " Dataset type: " + << static_cast::type>(store_client_->schema()->dataset_type()) + << " Dataset schema file: " << schema_file_ << " Number of rows: " << num_rows_ + << " Rows per buffer: " << rows_per_buffer_ << " Num buffers (computed): " << buffers_needed + << " Number of workers: " << num_workers_ << "."; + + // Next, create each buffer in a loop. + int32_t buff_id = 0; + for (buff_id = 0; buff_id < buffers_needed; buff_id++) { + // Create a new data buffer as a base class pointer, using the factory method from + // DataBuffer class + std::unique_ptr new_data_buffer; + RETURN_IF_NOT_OK(DataBuffer::CreateDataBuffer(buff_id, store_client_, &new_data_buffer)); + + // Insert the buffer into our vector + data_buffers_.push_back(std::move(new_data_buffer)); + } + + // Instantiate the action queues. If this was a re-entrant call then these already exist. + // We cannot drop and recreate them because there are threads waiting on them currently. + // They should be empty anyway in a reset codepath + if (action_queue_.empty()) { + // The max size of these queues should ensure they will never get full and they support + // precisely the amount of data that we know they will hold (the total number of buffers). + // There needs to be one queue for each worker, to support the Connector design for how + // data will be fetched and pushed into a Connector in parallel. + // + // Say the total buffers is 5, and we have 2 workers. + // To support this, we'd need 1 queue of size 2 and the other of size 3. + // For simplicity, we'll make both of them 3 so they are the same size. + int32_t action_queue_size = (buffers_needed / num_workers_) + 1; + for (int32_t i = 0; i < num_workers_; ++i) { + auto new_queue = mindspore::make_unique>(action_queue_size); + action_queue_.push_back(std::move(new_queue)); + } + } + + // Extract the list of buffer id's from the vector and use this as our starting action + // queue of buffers. + RETURN_IF_NOT_OK(this->FillActionQueue(false)); + return Status::OK(); +} + +// Destructor +StorageOp::~StorageOp() {} + +// A print method typically used for debugging +void StorageOp::Print(std::ostream &out, bool show_all) const { + // Call base class printer first + ParallelOp::Print(out, show_all); + + // Then display our own stuff + out << "\nStorageOp:"; + out << "\n Dataset files dir : " << dataset_files_dir_ << "\n Dataset schema file : " << schema_file_; + if (!dataset_file_list_.empty()) { + out << "\n Dataset Files List:\n"; + for (auto filename : dataset_file_list_) { + out << " " << filename << "\n"; + } + } + out << "\n\n"; + if (!data_buffers_.empty()) { + out << std::boolalpha << " Number of DataBuffers inside StorageOp: " << data_buffers_.size() + << "\n Number of rows: " << num_rows_ << "\n Rows per buffer: " << rows_per_buffer_ << "\n\n DataBuffers:\n"; + + // Iterate over each DataBuffer and display the buffer id and the buffer + int32_t i = 0; + for (i = 0; i < data_buffers_.size(); i++) { + out << " " << i << ")\n"; + data_buffers_[i]->Print(out, show_all); + } + } else { + out << "DataCache is empty!\n"; + } +} + +// Private helper method. This one posts a control indicator for each worker thread to consume +// from the action queue. When the worker pops this msg, it will shut itself down gracefully. +Status StorageOp::PostEndOfData() { + MS_LOG(INFO) << "Master: Processed all of the buffers. Send end-of-data message to workers."; + + // For each worker we add the message so that they can all get the memo + for (int32_t i = 0; i < num_workers_; ++i) { + RETURN_IF_NOT_OK(action_queue_[i]->Add(kEndOfActions)); + } + return Status::OK(); +} + +// Private helper method. This one populates the action queue with the list of buffer ids. +Status StorageOp::FillActionQueue(bool randomize) { + // We only support adding the new list of id's to the queue if we are sure the old list + // of actions is already done. This might change in the future though + for (int32_t i = 0; i < num_workers_; ++i) { + if (!(action_queue_[i]->empty())) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Attempt to get buffer id's into a queue, but the queue not empty!"); + } + } + if (!data_buffers_.empty()) { + // Add buffer id's to the queue. Buffer id's in our vector are just numbers from 0 up, so + // basically just a list of consecutive numbers starting from 0 (incremented by 1). + // If randomize is requested, the list of id's will be jumbled up (so not consecutive + // order) + if (!randomize) { + // Round robin of filling each worker with the buffer id's + int32_t curr_worker = 0; + for (int32_t i = 0; i < data_buffers_.size(); ++i) { + RETURN_IF_NOT_OK(action_queue_[curr_worker]->Add(i)); + curr_worker++; + if (curr_worker == num_workers_) { + curr_worker = 0; + } + } + } else { + std::vector random_ids; + int32_t i; + for (i = 0; i < data_buffers_.size(); ++i) { + random_ids.push_back(i); + } + uint32_t seed = std::chrono::system_clock::now().time_since_epoch().count(); + std::shuffle(random_ids.begin(), random_ids.end(), std::default_random_engine(seed)); + + // Round robin of filling each worker with the buffer id's from randomized list + int32_t curr_worker = 0; + for (i = 0; i < random_ids.size(); ++i) { + RETURN_IF_NOT_OK(action_queue_[curr_worker]->Add(random_ids[i])); + curr_worker++; + if (curr_worker == num_workers_) { + curr_worker = 0; + } + } + } + } + return Status::OK(); +} + +// The entry point code for when workers are launched. +// Given the input bufferId, it returns a shared_ptr to that buffer back to you by driving a +// load operation. This function is intended to be run by worker threads, when they are +// populating the memory with the actual data of the buffer. +Status StorageOp::GetBuffer(int32_t buffer_id, std::unique_ptr *ptr) { + if (!data_buffers_.empty()) { + if (static_cast(buffer_id) >= data_buffers_.size()) { + std::ostringstream ss; + ss << "Error. Buffer id " << buffer_id << " is out of range."; + std::string err_msg = ss.str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // execute a load operation to fill this buffer (may result in call to storage layers) + RETURN_IF_NOT_OK(data_buffers_[buffer_id]->Load()); + + // Return the buffer + // Important: The share pointer remains counted for the caller as well as locally in the + // mDataBuffers array. Later when the buffer is sent on it's way up the pipeline, the + // shared_ptr in the array will be reset so that the StorageOp will not hang on to old + // buffers that it has already passed up the pipeline. + *ptr = std::move(data_buffers_[buffer_id]); + } else { + RETURN_STATUS_UNEXPECTED("Requested to get a buffer from an empty cache."); + } + return Status::OK(); +} + +// Class functor operator () override. +// All dataset ops operate by launching a thread (see ExecutionTree). This class functor will +// provide the master loop that drives the logic for performing the work +Status StorageOp::operator()() { + // Before we enter our master loop, kick off our workers and assign them to + // use the StorageOp worker entry code. + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&StorageOp::WorkerEntry, this, std::placeholders::_1))); + // Handshake with TaskManager to synchronize thread creation + TaskManager::FindMe()->Post(); + int32_t num_buffers_to_fetch = data_buffers_.size(); + + // The storage op is the bottom node in the tree, so it does not listen to an input + // queue from an operator below us. Instead, we'll will read from the internal queue + // that our workers produce into, and then push that into output queue. + bool done = false; + std::unique_ptr fetched_buffer; + while (!done) { + // Get the next buffer. We are single thread master so thread id hard coded to 0 + // on the connector pop. Count this buffer towards our count, and then push + // it up to the output connector. + RETURN_IF_NOT_OK(worker_connector_->PopWithRetry(0, &fetched_buffer)); + buffers_fetched_++; + int32_t buffer_id = fetched_buffer->id(); + + if (buffers_fetched_ == 1) { + num_buffers_to_fetch = static_cast(data_buffers_.size()); + } + + // There should be 2 holders of this buffer currently. We have one in the mDataBuffers + // table, and then ourselves right now with fetchedBuffer. + // Reduce the shared_ptr ref count of this buffer by removing it from the mDataBuffers + // table first before we push the buffer to output connector. + data_buffers_[buffer_id].reset(); + MS_LOG(INFO) << "StorageOp master: Consumed buffer " << buffer_id << " from internal worker connector."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(fetched_buffer))); + MS_LOG(INFO) << "StorageOp master: pushed buffer " << buffer_id << " to output connector."; + + // Now, check our loop exit conditions and perform appropriate end of data handling if + // we've reached the end of our scan. + if (buffers_fetched_ == num_buffers_to_fetch) { + MS_LOG(INFO) << "StorageOp master: Reached end of data."; + + // If we are not inside of a Repeat path in the tree, or we are in a repeat path but + // this was our last repeat, then we do a full quit here with eof control message. + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + // Post the control message to tell the workers to stop waiting on action queue + // because we are done! + RETURN_IF_NOT_OK(this->PostEndOfData()); + std::unique_ptr eoeBuffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoeBuffer))); + MS_LOG(INFO) << "StorageOp master: Flow end-of-data eof message."; + std::unique_ptr eofBuffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOF); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eofBuffer))); + MS_LOG(INFO) << "StorageOp master: Main execution loop complete."; + done = true; // while loop exit + } else { + // We are in a repeat path and it's not the last repeat. + // Flow an end-of-epoch control message up the pipeline. + // RepeatOp above us somewhere in the tree will re-init us with the data to fetch again + // once it gets the end-of-epoch message. + MS_LOG(INFO) << "StorageOp master: Flow end-of-epoch eoe message."; + std::unique_ptr eoe_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); + + // reset our buffer count and go to loop again. + buffers_fetched_ = 0; + + // This is a bit of a cheat. Only the repeat op should perform resetting actions + // against us (currently). However, if we go to block/wait on the worker_connector_ + // right now before the reset is done (driven from the repeat op), then we end + // up using stale connector index info and blocking on the wrong thing, causing + // invalid order during the next epoch. + // For now then, do a quick reset of just the connector queue so that we block + // at a safe starting point in the connector. + worker_connector_->Reset(); + } + } + } + return Status::OK(); +} + +// The entry point code for when workers are launched. +Status StorageOp::WorkerEntry(int32_t worker_id) { + int32_t next_action_id = 0; + MS_LOG(INFO) << "Worker: StorageOp worker entry point."; + + // Handshake with TaskManager to synchronize the creation + TaskManager::FindMe()->Post(); + + // While there is still some actions to perform + RETURN_IF_NOT_OK(action_queue_[worker_id]->PopFront(&next_action_id)); + while (next_action_id != kEndOfActions) { + // Drive a load of this buffer and get a pointer to the buffer after it's loaded in + std::unique_ptr dB; + RETURN_IF_NOT_OK(this->GetBuffer(next_action_id, &dB)); + MS_LOG(INFO) << "Worker: Loaded buffer " << next_action_id << "."; + + // Add the buffer to the internal queue for master to consume from later. + // This could end up blocking if the queue is full in which case it waits here + // until the master can drain a buffer off the queue. + RETURN_IF_NOT_OK(worker_connector_->Add(worker_id, std::move(dB))); + MS_LOG(INFO) << "Worker: Pushed buffer " << next_action_id << " to internal worker connector."; + + // Get the next action id and loop + RETURN_IF_NOT_OK(action_queue_[worker_id]->PopFront(&next_action_id)); + } + MS_LOG(INFO) << "Worker: Received end-of-data message. Worker complete."; + return Status::OK(); +} + +const DataSchema *StorageOp::schema() const { return store_client_->schema(); } + +// Overrides base class reset method. When an operator does a reset, it cleans up any state +// info from it's previous execution and then initializes itself so that it can be executed +// again. +Status StorageOp::Reset() { + RETURN_IF_NOT_OK(ParallelOp::Reset()); // Call our super class reset first. + + // We do not need to redo the handshake with the storage client, since that + // info should be the same as the last time. However there may be stale + // state info in the client from the last execution. The client provides + // a reset method as well to re-initialize. + RETURN_IF_NOT_OK(store_client_->Reset()); + + // init method is re-entrant and will refresh everything. + RETURN_IF_NOT_OK(this->init()); + return Status::OK(); +} + +// Name: LoadParallelConfig +// Description: Load parallel config info from a specific config file. In multi-P cases (or single-P cases), we +// need to know deviceID, rank, device number, shard mode +// , shuffle (or not) and seed to prepare to scatter files. +Status StorageOp::LoadParallelConfig() { + if (data_distribution_file_ == "") { + return Status::OK(); + } + try { + std::ifstream in(data_distribution_file_); + nlohmann::json js; + in >> js; + device_num_ = js.value("deviceNum", 0); + device_id_ = js.value("deviceId", 0); + if (device_num_ == 0 || device_num_ > MAX_INTEGER_INT32) { + RETURN_STATUS_UNEXPECTED("Invalid deviceNum"); + } + if (device_id_ > MAX_INTEGER_INT32 || device_id_ >= device_num_) { + MS_LOG(INFO) << "In parallel config file " << data_distribution_file_ << ", wrong deviceID provided."; + RETURN_STATUS_UNEXPECTED("Invalid deviceId"); + } + shard_config_ = js.value("shardConfig", ""); + if (shard_config_ != "ALL" && shard_config_ != "UNIQUE" && shard_config_ != "RANDOM") { + MS_LOG(INFO) << "In parallel config file " << data_distribution_file_ << " wrong mShardConfig provided."; + RETURN_STATUS_UNEXPECTED("Invalid shardConfig"); + } + std::string shuffle_str = js.value("shuffle", ""); + if (shuffle_str == "ON") { + shuffle_config_ = true; + } else if (shuffle_str == "OFF") { + shuffle_config_ = false; + } else { + MS_LOG(INFO) << "In parallel config file " << data_distribution_file_ + << ", shuffle config is wrong: it's not ON or OFF"; + RETURN_STATUS_UNEXPECTED("Invalid shuffle option"); + } + seed_ = js.value("seed", 0); + if (seed_ > MAX_INTEGER_UINT32) { + RETURN_STATUS_UNEXPECTED("Invalid seed"); + } + } catch (const std::exception &e) { + RETURN_STATUS_UNEXPECTED("Load parallel config failed"); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.h new file mode 100644 index 00000000000..9334addc342 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.h @@ -0,0 +1,389 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_STORAGE_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_STORAGE_OP_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Forward declares +template +class Queue; + +// A type for a container of DataBuffer shared_ptr's +using DataBuffers = std::vector>; + +// A type for the queue of buffer id's for workers to fetch. +using ActionQueue = std::vector>>; + +// Forward declare +class DataBuffer; + +class StorageClient; + +class StorageOp : public ParallelOp { + public: + // The nested builder class inside of the StorageOp is used to help manage all of the arguments + // for constructing it. Use the builder by setting each argument with the provided set methods, + // and then finally call the build method to execute the actual construction. + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + // Default destructor + ~Builder() = default; + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetNumRows(int num_rows) { + build_num_rows_ = num_rows; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int rows_per_buffer) { + build_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetSchema(std::unique_ptr schema) { + build_schema_ = std::move(schema); + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + build_num_workers_ = num_workers; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetWorkerConnectorSize(int32_t connector_size) { + build_worker_connector_size_ = connector_size; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t connector_size) { + build_op_connector_size_ = connector_size; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetSchemaDir(const std::string &schema_dir) { + build_schema_file_ = schema_dir + "/datasetSchema.json"; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetSchemaFile(const std::string &schema_file) { + build_schema_file_ = schema_file; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetDatasetFilesDir(const std::string &files_dir) { + build_dataset_files_dir_ = files_dir; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetDatasetFileList(const std::vector &file_list) { + build_dataset_file_list_ = file_list; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetColumnsToLoad(const std::vector &columns) { + build_columns_to_load_ = columns; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetDataDistributionFile(const std::string &data_distribution_file) { + build_data_distribution_file_ = data_distribution_file; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &setLabelsFileName(const std::string &labels_file_name) { + build_labels_file_name_ = labels_file_name; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetDatasetUsage(const std::string &dataset_usage) { + build_dataset_usage_ = dataset_usage; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetBatchSize(int32_t batch_size) { + build_batch_size_ = batch_size; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetDropRemainder(bool drop_remainder) { + build_drop_remainder_ = drop_remainder; + return *this; + } + + // The builder "build" method creates the final object. + // @param shared_ptr to the new StorageOp object + // @return Status - The error code return + Status Build(std::shared_ptr *); + + private: + // The builder saves all StorageOp construction arguments internally. + // The following are the arguments. + std::string build_dataset_files_dir_; + std::string build_schema_file_; + int32_t build_num_rows_; + std::string build_data_distribution_file_; + int32_t build_rows_per_buffer_; + int32_t build_worker_connector_size_; + int32_t build_num_workers_; + int32_t build_op_connector_size_; + std::unique_ptr build_schema_; + std::vector build_dataset_file_list_; + std::vector build_columns_to_load_; + std::string build_labels_file_name_; + std::string build_dataset_usage_; + int32_t build_batch_size_; + bool build_drop_remainder_; + }; + + // Constructor of the StorageOp. + // @note The builder class should be used to call it + // @param num_workers - The number of workers for the op + // @param worker_connector_size - The internal connector size between workers and master + // @param rows_per_buffer - The requested number of rows per buffer + // @param op_connector_size - The output connector queue size + // @param columns_to_load - The list of columns to use (column name) + StorageOp(int32_t num_workers, int32_t worker_connector_size, int32_t rows_per_buffer, int32_t op_connector_size, + std::vector columns_to_load, std::string data_distribution_file, int32_t batch_size, + bool drop_remainder); + + // Init the StorageOp. This is 1 of 3 init. + // This version of the init does not take the schema in it's arguments. It must perform an + // internal handshake with the dataset to produce the schema. + // @note The builder class should be used to call it + // @param dataset_files_dir - The directory that has the dataset files + // @param schema_file - The schema file for providing column info + Status InitOp(const std::string &dataset_files_dir, const std::string &schema_file, + const std::string &labels_file_name, const std::string &dataset_usage); + + // Init the StorageOp. This is 2 of 3 init. + // This version of the init allows the user to input the schema and other dataset properties rather + // than get it from the dataset itself. + // @note The builder class should be used to call it + // @param num_rows - The number of rows in the dataset + // @param dataset_files_dir - The directory that has the dataset files + // @param data_schema - The schema to use + Status InitOp(int32_t num_rows, const std::string &dataset_files_dir, std::unique_ptr data_schema, + const std::string &labels_file_name, const std::string &dataset_usage); + + // Init the StorageOp. This is 3 of 3 init. + // This version of the init does not take the schema in it's arguments. It must perform an + // internal handshake with the dataset to produce the schema. Unlike constructor 1, it takes a + // list of files rather than a directory. + // @note The builder class should be used to call it + // @param files_list - The list of files to use for the dataset + // @param schema_file - The schema file for providing column info + Status InitOp(const std::vector &files_list, const std::string &schema_file); + + // Destructor + ~StorageOp(); + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param storage_op - reference to the StorageOp to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, const StorageOp &storage_op) { + storage_op.Print(out, false); + return out; + } + + // Class functor operator () override. + // All DatasetOps operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work. + // @return Status - The error code return + Status operator()() override; + + // The entry point code for when workers are launched. + // @param worker_id - The worker id + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // The entry point code for when workers are launched. + // Given the input bufferId, it returns a shared_ptr to that buffer back to you by driving a + // load operation. This function is intended to be run by worker threads, when they are + // populating the memory with the actual data of the buffer. + // @param buffer_id - The buffer id to get. + // @param ptr - Pointer to shared_ptr to the buffer that was loaded in. + // @return Status - The error code return + Status GetBuffer(int32_t buffer_id, std::unique_ptr *ptr); + + // Overrides base class reset method. When an operator does a reset, it cleans up any state + // info from it's previous execution and then initializes itself so that it can be executed + // again. + // @return Status - The error code return + Status Reset() override; + + // Getter method + int32_t num_rows() const { return num_rows_; } + + // Setter method + void set_num_rows(int32_t num_rows) { num_rows_ = num_rows; } + + // Getter method + int32_t rows_per_buffer() const { return rows_per_buffer_; } + + // Setter method + void set_rows_per_buffer(int32_t rows_per_buffer) { rows_per_buffer_ = rows_per_buffer; } + + // Getter method + std::string dataset_files_dir() const { return dataset_files_dir_; } + + // Getter method + std::vector dataset_file_list() const { return dataset_file_list_; } + + // Getter method + std::string schema_file() const { return schema_file_; } + + // Getter method + const DataSchema *schema() const; + + // Getter method + const std::vector columns_to_load() const { return columns_to_load_; } + + // Getter method + std::string data_distribution_file() const { return data_distribution_file_; } + + // Getter method + int32_t device_num() const { return device_num_; } + + // Getter method + int32_t device_id() const { return device_id_; } + + // Getter method + std::string shard_config() const { return shard_config_; } + + // Getter method + uint32_t seed() const { return seed_; } + + // Getter method + bool shuffle_config() const { return shuffle_config_; } + + // Getter method + int32_t num_classes() const { return num_classes_; } + + // Getter method + std::string labels_file_name() const { return labels_file_name_; } + + // Getter method + std::string dataset_usage() const { return dataset_usage_; } + + // Getter method + int32_t batch_size() const { return batch_size_; } + + // Getter method + bool drop_remainder() const { return drop_remainder_; } + + private: + // Private helper method. This one populates the action queue with the list of buffer ids. + // @param randomize - T/F if the id's in the action queue should be randomized or sequential. + Status FillActionQueue(bool randomize); + + // Private helper method. This one encapsulates some common construction/reset tasks and is + // designed to be re-entrant so that you can re-init a previously used StorageOp without needing + // to redo the storage client handshake. + // @return Status - The error code return + Status init(); + + // Private helper method. This one posts a control indicator for each worker thread to consume + // from the action queue. When the worker pops this msg, it will shut itself down gracefully. + // @return Status - The error code return + Status PostEndOfData(); + + Status LoadParallelConfig(); + + DataBuffers data_buffers_; // A vector of pointers to buffers + std::shared_ptr store_client_; // The client for interacting with storage + ActionQueue action_queue_; // The queues of buffer id's for workers to fetch. + int32_t worker_conn_size_; // connector size for internal worker queue + int32_t rows_per_buffer_; // The number of requested rows per buffer. + int32_t num_rows_; // One more than the last row id in the range for this cache + std::string dataset_files_dir_; // The path for the dataset files + std::vector dataset_file_list_; // List of paths to files for the dataset + int32_t buffers_fetched_; // Counter for the buffers that were fetched + std::string schema_file_; // Path to the schema json file + std::vector columns_to_load_; // Columns to load from dataset + std::string data_distribution_file_; // Distribution configuration file + int32_t device_num_; // All device number + int32_t device_id_; // Device id + std::string shard_config_; // ALL UNIQUE RANDOM + uint32_t seed_; // Used for shuffle + bool shuffle_config_; // True or false + std::string labels_file_name_; // File name of labels + int32_t num_classes_; // Label class number + std::string dataset_usage_; // train/eval/inference + int32_t batch_size_; + bool drop_remainder_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_STORAGE_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.cc new file mode 100644 index 00000000000..766c2149c42 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.cc @@ -0,0 +1,333 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/tf_buffer.h" +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "utils/log_adapter.h" + +#include "dataset/engine/datasetops/source/tf_client.h" +#include "dataset/core/data_type.h" +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/engine/data_schema.h" +#include "dataset/util/make_unique.h" + +namespace mindspore { +namespace dataset { +// constructor +TFBuffer::TFBuffer( + uint32_t id, // In: The id for this buffer + BufferFlags flags, // In: The flags for this buffer + const std::shared_ptr &storage_client) // In: Storage client that is related to this buffer type + : DataBuffer(id, flags), storage_client_(storage_client) { + // Initializing mColumnNameMap from the schema file + const DataSchema *the_schema = storage_client_->schema(); + for (int32_t i = 0; i < the_schema->NumColumns(); ++i) { + column_name_map_[the_schema->column(i).name()] = i; + } +} + +// destructor +TFBuffer::~TFBuffer() {} + +// Name: print() +// Description: A function that prints info +void TFBuffer::Print(std::ostream &out, // In: The output stream to print to + bool show_all) const { // In: T/F if it should print everything + out << "TFBuffer print\n"; + + // Call base class printer + DataBuffer::Print(out, show_all); +} + +// Name: load() +// Description: populates the DataBuffer with data +// Overrides base-class method. +Status TFBuffer::Load() { + const DataSchema *the_schema = storage_client_->schema(); + uint32_t num_columns = the_schema->NumColumns(); + uint32_t num_rows_requested = storage_client_->rows_per_buffer(); + uint32_t remaining_rows = storage_client_->num_rows() > buffer_id_ * storage_client_->rows_per_buffer() + ? storage_client_->num_rows() - buffer_id_ * storage_client_->rows_per_buffer() + : 0; + if (remaining_rows < num_rows_requested) { + num_rows_requested = remaining_rows; + } + + // Construct the Tensor table for this buffer. + tensor_table_ = mindspore::make_unique(); + + // At each position in the tensor table, instantiate the shared pointer to it's Tensor. + uint32_t row = 0; + while (row < num_rows_requested && (cur_reader_.peek() != EOF || storage_client_->IsMoreData(buffer_id_))) { + TensorRow new_row; + + // Read the data from storage into a tf_file format + dataengine::Example tf_file; + RETURN_IF_NOT_OK(ParseSingleExample(&tf_file)); + for (uint32_t col = 0; col < num_columns; ++col) { + std::shared_ptr new_t; + const ColDescriptor current_col = the_schema->column(col); + const dataengine::Features &example_features = tf_file.features(); + const google::protobuf::Map &feature_map = example_features.feature(); + const dataengine::Feature &column_values_list = feature_map.at(current_col.name()); + const dataengine::Feature::KindCase column_list_type = column_values_list.kind_case(); + RETURN_IF_NOT_OK(LoadFeature(column_list_type, column_values_list, current_col, &new_t)); + + // Add the column to the current tensor row + new_row.push_back(std::move(new_t)); + } + + // Add the new row of tensors to the end of our tensor table + tensor_table_->push_back(new_row); + row++; + } + cur_reader_.close(); + return Status::OK(); +} + +// Name: ParseSingleExample() +// Description: Drives the calls to TFClient for fetching the tf_file info from +// the tf_file files. Returns a single row of data from the tf_file +// files. +Status TFBuffer::ParseSingleExample(dataengine::Example *ptr) { + if (cur_reader_.peek() == EOF) { + auto client = std::dynamic_pointer_cast(storage_client_); + if (client == nullptr) { + std::string errMsg = "Unexpected storage client type for TFBuffer"; + RETURN_STATUS_UNEXPECTED(errMsg); + } + RETURN_IF_NOT_OK(client->NextFileInfo(buffer_id_, &cur_f_info_)); + cur_reader_.close(); + cur_reader_.open(cur_f_info_.fileName); + // Seek to the offset + (void)cur_reader_.seekg(static_cast(cur_f_info_.startOffset)); + MS_LOG(INFO) << "got new file " << cur_f_info_.fileName << "."; + } + + // one record in tf_file looks like: + // Format of a single record: + // uint64 length + // uint32 masked crc of length + // byte data[length] + // uint32 masked crc of data + // read length + if (cur_reader_.peek() == EOF) { + MS_LOG(ERROR) << "ParseSingleExample failed"; + } + + dataengine::Example tf_file; + try { + uint64_t record_length = 0; + (void)cur_reader_.read(reinterpret_cast(&record_length), static_cast(sizeof(uint64_t))); + + // ignore crc header + (void)cur_reader_.ignore(static_cast(sizeof(uint32_t))); + + // read serialized Example + std::string serialized_example; + serialized_example.resize(record_length); + (void)cur_reader_.read(&serialized_example[0], static_cast(record_length)); + + // ignore crc footer + (void)cur_reader_.ignore(static_cast(sizeof(uint32_t))); + + if (!tf_file.ParseFromString(serialized_example)) { + std::string err_msg = "parse tf_file failed"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } catch (const std::exception &err) { + std::string err_msg = "Please check if the data file is complete!"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + *ptr = tf_file; + return Status::OK(); +} + +// Name: LoadFeature() +// Description: Given the column type of the tf record and the values list, +// constructs the tensor and returns it. +Status TFBuffer::LoadFeature(const dataengine::Feature::KindCase &column_list_type, + const dataengine::Feature &column_values_list, const ColDescriptor ¤t_col, + std::shared_ptr *out_tensor) { + std::string element_str; // For staging data from protobuf deserialization + std::unique_ptr int_array; // For staging data from protobuf deserialization + std::unique_ptr float_array; // For staging data from protobuf deserialization + const unsigned char *data_ptr = nullptr; // Generic pointer used for populating the Tensor + // This variable will point into the above staging + // variables. + uint32_t num_elements = 0; // Generic counter used for setting shape attributes + + // Depending on the type of data from the tf_file, we want to extract 2 things: + // 1) A pointer to the data as a const unsigned char * + // 2) The number of elements of the data + // After those are determined, we can then build the tensor to represent this data. + + switch (column_list_type) { + // CASE : TF record type: kBytesList + case dataengine::Feature::KindCase::kBytesList: { + RETURN_IF_NOT_OK(LoadBytesList(current_col, column_values_list, &element_str)); + + // Get the const pointer representation of this data, and the number of elements + // (number of bytes) for this tensor. + data_ptr = reinterpret_cast(common::SafeCStr(element_str)); + num_elements = element_str.length(); + break; + } + + // CASE : TF record type: kFloatList + case dataengine::Feature::KindCase::kFloatList: { + RETURN_IF_NOT_OK(LoadFloatList(current_col, column_values_list, &num_elements, &float_array)); + + data_ptr = reinterpret_cast(float_array.get()); + break; + } + + // CASE : TF record type: kInt64List + case dataengine::Feature::KindCase::kInt64List: { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, &num_elements, &int_array)); + + data_ptr = reinterpret_cast(int_array.get()); + break; + } + case dataengine::Feature::KindCase::KIND_NOT_SET: { + std::string errMsg = "tf_file column list type enum is KIND_NOT_SET"; + RETURN_STATUS_UNEXPECTED(errMsg); + } + default: { + std::string errMsg = "tf_file column list type enum does not match any known DE type"; + RETURN_STATUS_UNEXPECTED(errMsg); + } + } + + // At this point we have a raw pointer to the data, and we have the number of elements. + // Along with the tensor implementation type and the data type from the schema, we + // enough info to construct the Tensor for it. + TensorShape current_shape = TensorShape::CreateUnknownRankShape(); + RETURN_IF_NOT_OK(CreateTensorShapeForColumn(current_col, num_elements, ¤t_shape)); + + // Now, create this tensor directly into the appropriate slot in our tensor + // table. + RETURN_IF_NOT_OK( + Tensor::CreateTensor(out_tensor, current_col.tensorImpl(), current_shape, current_col.type(), data_ptr)); + + return Status::OK(); +} + +Status TFBuffer::LoadBytesList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + std::string *element_str) { + // kBytesList can map to the following DE types ONLY! + // DE_UINT8, DE_INT8 + // Must be single byte type for each element! + if (current_col.type() != DataType::DE_UINT8 && current_col.type() != DataType::DE_INT8) { + std::string err_msg = "Invalid datatype for Tensor at column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + const dataengine::BytesList &bytes_list = column_values_list.bytes_list(); + + // A bytesList is a special case where the entire list of data can be + // deserialized into a single string. For example, it is not a list + // of bytes, it is a list of strings, where each string represents + // a list of bytes (this is different from the other cases like IntList etc) + // As such, if there is more than one string in this list, that is invalid. + if (bytes_list.value_size() > 1) { + std::string err_msg = "Bytes list contains more than one element for column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // Extract the string that contains the bytes we need. Position 0 is the only + // valid string here. + *element_str = bytes_list.value(0); + + return Status::OK(); +} + +Status TFBuffer::LoadFloatList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + uint32_t *num_elements, std::unique_ptr *float_array) { + // KFloatList can only map to DE types: + // DE_FLOAT32 + if (current_col.type() != DataType::DE_FLOAT32) { + std::string err_msg = "Invalid datatype for Tensor at column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + const dataengine::FloatList &float_list = column_values_list.float_list(); + + // Identify how many values we have and then create a local array of these + // to deserialize into + *num_elements = float_list.value_size(); + *float_array = mindspore::make_unique(*num_elements); + for (int i = 0; i < float_list.value_size(); i++) { + (*float_array)[i] = float_list.value(i); + } + + return Status::OK(); +} + +Status TFBuffer::LoadIntList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + uint32_t *num_elements, std::unique_ptr *int_array) { + // KInt64List can only map to DE types: + // DE_UINT64, DE_INT64, DE_UINT32, DE_INT32, DE_UINT16, DE_INT16, DE_UINT8, DE_INT8 + if (!(current_col.type().IsInt())) { + std::string err_msg = "Invalid datatype/rank for column label in TFBuffer."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + const dataengine::Int64List &int64_list = column_values_list.int64_list(); + + // Identify how many values we have and then create a local array of these + // to deserialize into + *num_elements = int64_list.value_size(); + *int_array = mindspore::make_unique(*num_elements); + for (int i = 0; i < int64_list.value_size(); i++) { + (*int_array)[i] = int64_list.value(i); + } + + return Status::OK(); +} + +Status TFBuffer::CreateTensorShapeForColumn(const ColDescriptor ¤t_col, uint32_t num_elements, + TensorShape *current_shape) { + // If the shape is assigned by user, we have an assumption that the data is + // already in the appropriate format that we can copy into the Tensor as-is. + if (current_col.hasShape()) { + *current_shape = current_col.shape(); + } else if (current_col.rank() == 1) { + // If shape was not given, then we support 2 possible shapes. + // 1) It's a scalar (rank 0), in which case the shape is empty but we need to flag + // it as a scalar value (empty shape but has a single value) + // 2) It's a rank 1 shape, and the dimension value for that single dimension will + // be comprised of the entire bytes-size of the input data. + *current_shape = TensorShape({num_elements}); + } else if (current_col.rank() == 0) { + // Make this shape into a single value scalar. + *current_shape = TensorShape::CreateScalar(); + } else if (current_col.rank() > 1) { + // All other ranks, except for 0, are invalid because we cannot guess + // what the shape will be. For example, if we have rank 3 and 12 bytes + // of data, is it shape {2,2,3} or is it {2,6,1}. We can't guess at + // the shape dimensions. + const std::string kErrMsg = "Invalid rank (rank>1) for dynamic shape construction. Specify shape in schema."; + RETURN_STATUS_UNEXPECTED(kErrMsg); + } + + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.h new file mode 100644 index 00000000000..5745ff80718 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_buffer.h @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_TF_BUFFER_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_TF_BUFFER_H_ + +#include +#include +#include +#include +#include "dataset/engine/data_buffer.h" +#include "./example.pb.h" +#include "dataset/engine/datasetops/source/tf_client.h" + +namespace mindspore { +namespace dataset { +// This TFBuffer is the buffer type for dealing with tf record data. +class TFBuffer : public DataBuffer { + public: + // constructor + TFBuffer(uint32_t id, // In: The id for this buffer + DataBuffer::BufferFlags flags, // In: The flags for this buffer + const std::shared_ptr + &storage_client); // In: The storage client that is related to this buffer type + + // destructor + ~TFBuffer() override; + + // Name: print() + // Description: A function that prints info + void Print(std::ostream &out, // In: The output stream to print to + bool show_all) const override; // In: T/F if it should print everything + + // Provide stream operator for displaying it + friend std::ostream &operator<<(std::ostream &out, const TFBuffer &tf_buffer) { + tf_buffer.Print(out, false); // Show meta info only + return out; + } + + // Name: load() + // Description: populates the DataBuffer with data. + // Overrides base-class method. + Status Load() override; + + private: + std::ifstream cur_reader_; + FileInfo cur_f_info_; + + std::shared_ptr storage_client_; // The storage client for populating the buffer initially. + + // Name: ParseSingleExample() + // Description: Drives the calls to TFClient for fetching the tf_file info from + // the tf_file files. Returns a single row of data from the tf_file + // files. + Status ParseSingleExample(dataengine::Example *ptr); + + // Name: LoadFeature() + // Description: Given the column type of the tf record and the values list, + // constructs the tensor and returns it. + Status LoadFeature(const dataengine::Feature::KindCase &column_list_type, + const dataengine::Feature &column_values_list, const ColDescriptor ¤t_col, + std::shared_ptr *out_tensor); + + Status LoadBytesList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + std::string *element_str); + + Status LoadFloatList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + uint32_t *num_elements, std::unique_ptr *float_array); + + Status LoadIntList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + uint32_t *num_elements, std::unique_ptr *int_array); + + Status CreateTensorShapeForColumn(const ColDescriptor ¤t_col, uint32_t num_elements, + TensorShape *current_shape); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_TF_BUFFER_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_client.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_client.cc new file mode 100644 index 00000000000..d41ff121af4 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_client.cc @@ -0,0 +1,376 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/engine/datasetops/source/tf_client.h" + +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "./example.pb.h" +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "dataset/engine/datasetops/source/storage_op.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +// Name: Constructor +// Description: Creates the TFClient. +TFClient::TFClient(std::unique_ptr schema, // In: The schema for this storage client. + StorageOp *so) // In: The StorageOp that's using this client + : StorageClient(std::move(schema), so), + rows_per_buffer_(so->rows_per_buffer()), + random_seed_generator_(so->seed()), + random_seed_distribution_(0, std::numeric_limits::max()), + rows_per_shard_(0) {} + +Status TFClient::Init() { + // Initialize queue to hold the tf file names + const std::string kExtensionData = ".data"; + const std::string kExtensionTF = ".tfrecord"; + bool schema_init = false; + if (!storage_op_->dataset_files_dir().empty()) { + MS_LOG(INFO) << "Reading dataset using datasetPath."; + Path data_set_directory(storage_op_->dataset_files_dir()); + auto dirIt = Path::DirIterator::OpenDirectory(&data_set_directory); + if (dirIt) { + while (dirIt->hasNext()) { + Path file = dirIt->next(); + std::string filename = file.toString(); + if ((file.Extension() == kExtensionData) || (file.Extension() == kExtensionTF)) { + const std::vector recs_lengths = ParseTfFileLines(filename); + v_total_file_rows_.emplace_back( + std::pair>(filename, std::move(recs_lengths))); + + // schema + if (!schema_init) { + RETURN_IF_NOT_OK(ParseTfFileSchema(filename)); + schema_init = true; + } + MS_LOG(INFO) << "found tf file: " << filename << ", num rows " << recs_lengths.size() << "."; + } + } + } else { + RETURN_STATUS_UNEXPECTED("Unable to open directory " + data_set_directory.toString()); + } + } else { + MS_LOG(INFO) << "Reading dataset using dataset files list."; + for (auto filename : storage_op_->dataset_file_list()) { + const std::vector recs_lengths = ParseTfFileLines(filename); + v_total_file_rows_.emplace_back(std::pair>(filename, std::move(recs_lengths))); + + // schema + if (!schema_init) { + RETURN_IF_NOT_OK(ParseTfFileSchema(filename)); + schema_init = true; + } + MS_LOG(INFO) << "Processed tf file: " << filename << ", num rows " << recs_lengths.size() << "."; + } + } + + RETURN_IF_NOT_OK(CalculateRowsPerDevice()); + std::sort(v_total_file_rows_.begin(), v_total_file_rows_.end()); + RETURN_IF_NOT_OK(ScatterFileRows(static_cast(storage_op_->device_id()), storage_op_->shard_config(), + storage_op_->seed(), storage_op_->shuffle_config())); + + CalculateNumRows(); + InitStateInfo(); + return Status::OK(); +} + +// Sharding will reduce the number of rows. Doing this in constructor as we only want to do this once. +void TFClient::CalculateNumRows() { + num_rows_in_dataset_ = 0; + for (auto rows : file_start_end_offset_) { + num_rows_in_dataset_ += (rows.second - rows.first); + } +} + +Status TFClient::CalculateRowsPerDevice() { + uint64_t num = std::accumulate( + v_total_file_rows_.begin(), v_total_file_rows_.end(), 0, + [](uint64_t value, const std::pair> &a) { return value + a.second.size(); }); + if (static_cast(std::floor(num * 1.0 / storage_op_->device_num())) == 0) { + RETURN_STATUS_UNEXPECTED("Num rows of dataset is less than device number"); + } + rows_per_shard_ = static_cast(std::ceil(num * 1.0 / storage_op_->device_num())); + return Status::OK(); +} + +bool TFClient::ValidFileForShard(const uint64_t file_rows, uint64_t *start_offset, uint64_t *end_offset, + const uint64_t &pre_count, uint32_t device_id) const { + *start_offset = 0; + *end_offset = 0; + bool valid = false; + uint64_t start_index = device_id * rows_per_shard_; + uint64_t end_index = (device_id + 1) * rows_per_shard_; + + // First valid file + if (pre_count <= start_index && pre_count + file_rows > start_index) { + *start_offset = start_index - pre_count; + valid = true; + if (pre_count < end_index && pre_count + file_rows >= end_index) { + *end_offset = end_index - pre_count; + } else { + *end_offset = file_rows; + } + } + + // Second and subsequent files + if (pre_count > start_index && pre_count < end_index) { + *start_offset = 0; + valid = true; + if (pre_count + file_rows >= end_index) { + *end_offset = end_index - pre_count; + } else { + *end_offset = file_rows; + } + } + + return valid; +} + +void TFClient::GetValidFileForShard(const std::vector>> &v_files, + uint32_t device_id) { + uint64_t start_offset = 0; + uint64_t end_offset = 0; + uint64_t pre_count = 0; + bool finish = false; + while (!finish) { + for (const auto &file : v_files) { + if (ValidFileForShard(file.second.size(), &start_offset, &end_offset, pre_count, device_id)) { + std::pair offset(start_offset, end_offset); + file_start_end_offset_.emplace_back(offset); + v_file_rows_.emplace_back(file); + } + pre_count += file.second.size(); + } + if (pre_count < (device_id + 1) * rows_per_shard_) { + finish = false; + } else { + finish = true; + } + } +} + +// Description: Scatter file rows to local single-P according to config info. +// There are 3 modes: ALL, UNIQUE, RANDOM. For UNIQUE and RANDOM mode, shuffleConfig controls +// whether file row vector would be shuffled or not before a new mEopch. +// For ALL mode, temporarily, we deal with epoch in python part. +Status TFClient::ScatterFileRows(uint32_t device_id, const std::string &shard_config, uint32_t seed, + bool shuffle_config) { + if (shard_config == "UNIQUE" || shard_config == "RANDOM") { + std::vector>> v_shuffled_total_file_rows = + ShuffleVector(v_total_file_rows_, seed); + GetValidFileForShard(v_shuffled_total_file_rows, device_id); + if (shuffle_config) { + v_total_file_rows_ = v_shuffled_total_file_rows; + } + } else if (shard_config == "ALL") { + v_file_rows_.insert(v_file_rows_.end(), v_total_file_rows_.begin(), v_total_file_rows_.end()); + if (shuffle_config) { + v_total_file_rows_ = ShuffleVector(v_total_file_rows_, seed); + } + + for (const auto &file : v_file_rows_) { + std::pair offset(0, file.second.size()); + file_start_end_offset_.emplace_back(offset); + } + } else { + RETURN_STATUS_UNEXPECTED("In parallel config file, wrong shuffleConfig or shardConfig provided."); + } + + return Status::OK(); +} + +std::vector>> TFClient::ShuffleVector( + std::vector>> v, uint32_t seed = 1) { + std::default_random_engine randomEngine(seed); + std::shuffle(std::begin(v), std::end(v), randomEngine); + return v; +} + +void TFClient::CalculateStartOffset(const uint64_t start_index, const uint64_t end_index, + const std::vector &vec_length, uint64_t *start_offset) const { + for (size_t i = start_index; i < end_index; i++) { + // Format of a single record: + // uint64 length + // uint32 masked crc of length + // byte data[length] + // uint32 masked crc of data + *start_offset += sizeof(uint64_t) + 2 * sizeof(uint32_t) + vec_length[i]; + } +} + +void TFClient::InitStateInfo() { + uint32_t start_idx = 0, record_num = 0, buffer_id = 0; + uint64_t start_offset = 0; + bool first_buffer = true; + f_info_queue_.emplace_back(QFile()); + std::vector>>::iterator itr = v_file_rows_.begin(); + uint32_t index = 0; + while (itr != v_file_rows_.end()) { + uint32_t file_start_index = file_start_end_offset_[index].first; + uint32_t file_end_index = file_start_end_offset_[index].second; + FileInfo f_info; + f_info.fileName = itr->first; + f_info.startRecordIdx = start_idx > file_start_index ? start_idx : file_start_index; + if (first_buffer && f_info.startRecordIdx != 0) { + CalculateStartOffset(0, f_info.startRecordIdx, itr->second, &start_offset); + start_idx = static_cast(f_info.startRecordIdx); + } + first_buffer = false; + f_info.startOffset = start_offset; + if (start_idx + rows_per_buffer_ - record_num < itr->second.size()) { + uint64_t end_idx = start_idx + rows_per_buffer_ - record_num - 1; + f_info.endRecordIdx = end_idx > (file_end_index - 1) ? (file_end_index - 1) : end_idx; + f_info_queue_[buffer_id].push(f_info); + CalculateStartOffset(start_idx, f_info.endRecordIdx + 1, itr->second, &start_offset); + start_idx = start_idx + rows_per_buffer_ - record_num; + record_num = 0; + buffer_id++; + f_info_queue_.emplace_back(QFile()); + if (end_idx >= file_end_index - 1) { + start_idx = start_offset = 0; + ++itr; + ++index; + } + } else { + f_info.endRecordIdx = itr->second.size() - 1 > file_end_index - 1 ? file_end_index - 1 : itr->second.size() - 1; + f_info_queue_[buffer_id].push(f_info); + if (start_idx + rows_per_buffer_ - record_num == itr->second.size()) { + record_num = start_idx = start_offset = 0; + buffer_id++; + if (itr + 1 != v_file_rows_.end()) { + f_info_queue_.emplace_back(QFile()); + } + } else { + record_num += static_cast(itr->second.size()) - start_idx; + start_idx = start_offset = 0; + } + ++itr; + ++index; + } + } +} + +// Name: Print() +// Description: A function that prints info about the TFClient +void TFClient::Print(std::ostream &out) const { // In: The output stream to print to + out << "TF client."; +} + +std::vector TFClient::ParseTfFileLines(const std::string &filename) { + std::vector recs_lengths; + std::ifstream reader; + reader.open(filename); + while (true) { + if (reader.peek() == EOF) { + reader.close(); + break; + } + + // read length + uint64_t record_length = 0; + (void)reader.read(reinterpret_cast(&record_length), static_cast(sizeof(uint64_t))); + recs_lengths.push_back(record_length); + + // ignore crc header + (void)reader.ignore(static_cast(sizeof(uint32_t))); + + // ignore data length + (void)reader.ignore(static_cast(record_length)); + + // ignore crc footer + (void)reader.ignore(static_cast(sizeof(uint32_t))); + } + return recs_lengths; +} + +Status TFClient::ParseTfFileSchema(const std::string &filename) { + std::ifstream reader; + reader.open(filename); + std::string serialized_example; + // read length + uint64_t record_length = 0; + (void)reader.read(reinterpret_cast(&record_length), static_cast(sizeof(uint64_t))); + + // ignore crc header + (void)reader.ignore(static_cast(sizeof(uint32_t))); + + // read serialized Example + serialized_example.resize(record_length); + (void)reader.read(&serialized_example[0], static_cast(record_length)); + + // ignore crc footer + (void)reader.ignore(static_cast(sizeof(uint32_t))); + + reader.close(); + dataengine::Example tf_file; + if (!tf_file.ParseFromString(serialized_example)) { + std::string err_msg = "parse tf_file failed"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + const dataengine::Features &example_features = tf_file.features(); + const google::protobuf::Map &feature_map = example_features.feature(); + for (auto it = feature_map.begin(); it != feature_map.end(); ++it) { + col_names_.push_back(it->first); + } + return Status::OK(); +} + +// Name: Reset() +// Description: Resets any state info inside the client back to it's initialized +// state. +Status TFClient::Reset() { + v_file_rows_.clear(); + file_start_end_offset_.clear(); + + uint32_t next_seed = random_seed_distribution_(random_seed_generator_); + RETURN_IF_NOT_OK(ScatterFileRows(static_cast(storage_op_->device_id()), storage_op_->shard_config(), + next_seed, storage_op_->shuffle_config())); + + CalculateNumRows(); + uint32_t num_rows_in_file = 0; + RETURN_IF_NOT_OK(this->numRowsFromFile(num_rows_in_file)); + if (num_rows_in_file < num_rows_in_dataset_) { + num_rows_in_dataset_ = num_rows_in_file; + } + + storage_op_->set_num_rows(static_cast(num_rows_in_dataset_)); + InitStateInfo(); + + return Status::OK(); +} + +Status TFClient::NextFileInfo(uint32_t id, FileInfo *ptr) { + if (f_info_queue_.empty() || id >= f_info_queue_.size() || f_info_queue_[id].empty()) { + RETURN_STATUS_UNEXPECTED("cannot find next FileInfo in mFInfoQueue"); + } + *ptr = f_info_queue_[id].front(); + f_info_queue_[id].pop(); + return Status::OK(); +} + +bool TFClient::IsMoreData(uint32_t id) { return (!f_info_queue_[id].empty()); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_client.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_client.h new file mode 100644 index 00000000000..6ff76e202a3 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_client.h @@ -0,0 +1,111 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_TF_CLIENT_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_TF_CLIENT_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "./example.pb.h" +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/util/status.h" + +struct FileInfo { + std::string fileName; + uint64_t startRecordIdx; + uint64_t endRecordIdx; + uint64_t startOffset; +}; + +using QFile = std::queue; + +namespace mindspore { +namespace dataset { +// forward declares +class DataSchema; +class ParallelOp; + +class TFClient : public StorageClient { + public: + // Name: Constructor + // Description: Creates the TFClient. + TFClient(std::unique_ptr schema, // In: The schema for this storage client. + StorageOp *so); // In: The ParallelOp that's using this client + + ~TFClient() {} + + Status Init() override; + + // Name: Print() + // Description: A function that prints info about the TFClient + void Print(std::ostream &out) const override; // In: The output stream to print to + + std::vector ParseTfFileLines(const std::string &filename); + + Status ParseTfFileSchema(const std::string &filename); + + Status NextFileInfo(uint32_t id, FileInfo *); + + bool IsMoreData(uint32_t id) override; + + // Name: Reset() + // Description: Resets any state info inside the client back to it's initialized + // state. + Status Reset() override; + + Status ScatterFileRows(uint32_t device_id, const std::string &shard_config, uint32_t seed, bool shuffle_config); + + private: + // hardcoded, put this in json schema + // const static int32_t BERT_DATASET_TOTAL_ROWS = 43900; + uint32_t rows_per_buffer_; + std::default_random_engine random_seed_generator_; + std::uniform_int_distribution random_seed_distribution_; + + std::vector>> v_file_rows_; + std::vector>> v_total_file_rows_; + std::vector f_info_queue_; + uint64_t rows_per_shard_; + std::vector> file_start_end_offset_; + + void InitStateInfo(); + + std::vector>> ShuffleVector( + std::vector>> v, uint32_t seed); + + Status CalculateRowsPerDevice(); + + bool ValidFileForShard(const uint64_t file_rows, uint64_t *start_offset, uint64_t *end_offset, + const uint64_t &pre_count, uint32_t device_id) const; + + void CalculateNumRows(); + + void GetValidFileForShard(const std::vector>> &v_files, + uint32_t device_id); + + void CalculateStartOffset(const uint64_t start_index, const uint64_t end_index, + const std::vector &vec_length, uint64_t *start_offset) const; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_TF_CLIENT_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc new file mode 100644 index 00000000000..16a77000124 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -0,0 +1,930 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/tf_reader_op.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "./example.pb.h" +#include "./securec.h" +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/connector.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/storage_client.h" +#include "dataset/engine/datasetops/source/tf_client.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/engine/jagged_connector.h" +#include "dataset/util/make_unique.h" +#include "dataset/util/path.h" +#include "dataset/util/queue.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" +#include "dataset/util/task_manager.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +TFReaderOp::Builder::Builder() + : builder_device_id_(0), builder_num_devices_(1), builder_total_rows_(0), builder_equal_rows_per_shard_(false) { + std::shared_ptr config_manager = GlobalContext::config_manager(); + builder_num_workers_ = config_manager->num_parallel_workers(); + builder_worker_connector_size_ = config_manager->worker_connector_size(); + builder_op_connector_size_ = config_manager->op_connector_size(); + builder_rows_per_buffer_ = config_manager->rows_per_buffer(); + builder_shuffle_files_ = false; + builder_data_schema_ = make_unique(); +} + +Status TFReaderOp::Builder::ValidateInputs() const { + std::string err_msg; + err_msg += builder_num_workers_ <= 0 ? "Number of parallel workers is smaller or equal to 0\n" : ""; + if (!builder_equal_rows_per_shard_) { + err_msg += builder_dataset_files_list_.size() < static_cast(builder_num_devices_) + ? "No enough tf_file files provided\n" + : ""; + } + err_msg += builder_device_id_ >= builder_num_devices_ || builder_num_devices_ < 1 ? "Wrong sharding configs\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +Status TFReaderOp::Builder::Build(std::shared_ptr *out_tf_reader_op) { + RETURN_IF_NOT_OK(ValidateInputs()); + + // Throttle the number of workers if we have more workers than files! + if (static_cast(builder_num_workers_) > builder_dataset_files_list_.size()) { + builder_num_workers_ = builder_dataset_files_list_.size(); + MS_LOG(WARNING) << "TFReader operator parallelism reduced to " << builder_num_workers_ << " workers."; + } + + std::shared_ptr new_tf_reader_op = std::make_shared( + builder_num_workers_, builder_worker_connector_size_, builder_rows_per_buffer_, builder_total_rows_, + builder_dataset_files_list_, std::move(builder_data_schema_), builder_op_connector_size_, builder_columns_to_load_, + builder_shuffle_files_, builder_num_devices_, builder_device_id_, builder_equal_rows_per_shard_); + + RETURN_IF_NOT_OK(new_tf_reader_op->Init()); + *out_tf_reader_op = std::move(new_tf_reader_op); + return Status::OK(); +} + +TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, + int64_t total_num_rows, std::vector dataset_files_list, + std::unique_ptr data_schema, int32_t op_connector_size, + std::vector columns_to_load, bool shuffle_files, int32_t num_device, + int32_t device_id, bool equal_rows_per_shard) + : ParallelOp(num_workers, op_connector_size), + device_id_(device_id), + num_devices_(num_device), + rows_per_buffer_(rows_per_buffer), + total_rows_(total_num_rows), + dataset_files_list_(std::move(dataset_files_list)), + columns_to_load_(std::move(columns_to_load)), + finished_reading_dataset_(false), + shuffle_files_(shuffle_files), + data_schema_(std::move(data_schema)), + filename_index_(make_unique()), + load_io_block_queue_(true), + num_rows_(0), + num_rows_per_shard_(0), + equal_rows_per_shard_(equal_rows_per_shard) { + worker_connector_size_ = worker_connector_size; +} + +Status TFReaderOp::Init() { + if (data_schema_->Empty()) { + RETURN_IF_NOT_OK(CreateSchema(dataset_files_list_[0], columns_to_load_)); + } + + if (total_rows_ == 0) { + total_rows_ = data_schema_->num_rows(); + } + + // Build the index with our files such that each file corresponds to a key id. + RETURN_IF_NOT_OK(filename_index_->insert(dataset_files_list_)); + + // The creation of the internal connector has been delayed until now, since we may have adjusted the + // number of workers. Now that the worker count is established, create the connector now in the + // parallel op base. + RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); + + jagged_buffer_connector_ = mindspore::make_unique(num_workers_, 1, worker_connector_size_); + + // temporary: make size large enough to hold all files + EOE to avoid hangs + int32_t safe_queue_size = static_cast(std::ceil(dataset_files_list_.size() / num_workers_)) + 1; + io_block_queues_.Init(num_workers_, safe_queue_size); + dataset_files_list_.clear(); // no longer need the original list of files + + return Status::OK(); +} + +Status TFReaderOp::CalculateNumRowsPerShard() { + if (!equal_rows_per_shard_) { + return Status::OK(); + } + + for (auto it = filename_index_->begin(); it != filename_index_->end(); ++it) { + std::vector file(1, it.value()); + int64_t num = CountTotalRowsSectioned(file, 0, 1); + filename_numrows_[it.value()] = num; + num_rows_ += num; + } + num_rows_per_shard_ = static_cast(std::ceil(num_rows_ * 1.0 / num_devices_)); + if (num_rows_per_shard_ == 0) { + RETURN_STATUS_UNEXPECTED("Number of rows can not be zero"); + } + return Status::OK(); +} +// Class functor operator () override. +// All dataset operators operate by launching a thread (see ExecutionTree). This class functor will +// provide the master loop that drives the logic for performing the work +Status TFReaderOp::operator()() { + RETURN_IF_NOT_OK(CalculateNumRowsPerShard()); + + // launch one thread, responsible for filling mIOBlockQueue + RETURN_IF_NOT_OK(tree_->LaunchWorkers(1, std::bind(&TFReaderOp::WaitToFillIOBlockQueue, this))); + + // launch num_workers_ worker threads, responsible for pulling from the IOBlockQueue and reading + // data from disk into buffers + RETURN_IF_NOT_OK( + tree_->LaunchWorkers(num_workers_, std::bind(&TFReaderOp::WorkerEntry, this, std::placeholders::_1))); + + // must be called after launching workers. workers can't be spawned after this post, + // so workers have to be kept alive until the end of the program + TaskManager::FindMe()->Post(); + + io_block_queue_wait_post_.Register(tree_->AllTasks()); + + NotifyToFillIOBlockQueue(); + while (!finished_reading_dataset_) { + int64_t buffer_id = 0; + int32_t workers_done = 0; + int64_t rows_read = 0; + { + std::unique_lock lock(load_io_block_queue_mutex_); + load_io_block_queue_ = true; + } + + while (workers_done < num_workers_) { + std::unique_ptr fetched_buffer; + RETURN_IF_NOT_OK(jagged_buffer_connector_->Pop(0, &fetched_buffer)); + if (fetched_buffer->eoe()) { + workers_done++; + } else if (total_rows_ == 0 || rows_read < total_rows_) { + // we need to push a buffer + if (total_rows_ > 0 && rows_read + fetched_buffer->NumRows() > total_rows_) { + // this is last buffer we need, and we only need a part of it + int64_t rowsToRemove = fetched_buffer->NumRows() - (total_rows_ - rows_read); + RETURN_IF_NOT_OK(fetched_buffer->SliceOff(rowsToRemove)); + } + + rows_read += fetched_buffer->NumRows(); + fetched_buffer->set_id(buffer_id); + buffer_id++; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(fetched_buffer))); + } else { + std::unique_lock lock(load_io_block_queue_mutex_); + load_io_block_queue_ = false; + } + } + + // all workers finished reading for this epoch, and we have read all the data from all workers + std::unique_ptr eoe_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOE); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer))); + + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + finished_reading_dataset_ = true; + NotifyToFillIOBlockQueue(); + } else { + jagged_buffer_connector_->DoReset(); + buffer_id = 0; + } + } + + std::unique_ptr eof_buffer = mindspore::make_unique(0, DataBuffer::kDeBFlagEOF); + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); + + RETURN_IF_NOT_OK(PostEndOfData()); + + return Status::OK(); +} + +// static local-only helper function +static void shuffleKeys(std::vector *i_keys, uint32_t seed) { + std::mt19937 rng(seed); + std::shuffle(i_keys->begin(), i_keys->end(), rng); +} + +// The entry point for when workers are launched. +Status TFReaderOp::WorkerEntry(int32_t worker_id) { + // must be called first if called by worker spawned by taskgroup + TaskManager::FindMe()->Post(); + + std::unique_ptr io_block; + RETURN_IF_NOT_OK(PopIoBlockQueue(worker_id, &io_block)); + + while (!io_block->eof()) { + if (!io_block->eoe()) { + std::string filename; + RETURN_IF_NOT_OK(io_block->GetFilename(&filename, *filename_index_)); + int64_t start_offset = io_block->GetStartOffset(); + int64_t end_offset = io_block->GetEndOffset(); + RETURN_IF_NOT_OK(LoadFile(filename, start_offset, end_offset, worker_id)); + MS_LOG(INFO) << "TFReader operator worker " << worker_id << " loaded file " << common::SafeCStr(filename) << "."; + } else { + std::unique_ptr eoe_buffer = mindspore::make_unique(1, DataBuffer::kDeBFlagEOE); + RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(eoe_buffer))); + } + + RETURN_IF_NOT_OK(PopIoBlockQueue(worker_id, &io_block)); + } + + return Status::OK(); +} + +// Pushes a control indicator onto the IOBlockQueue for each worker to consume. +// When the worker pops this control indicator, it will shut itself down gracefully. +Status TFReaderOp::PostEndOfData() { + for (int i = 0; i < num_workers_; ++i) { + std::unique_ptr eof = mindspore::make_unique(IOBlock::kDeIoBlockFlagEof); + RETURN_IF_NOT_OK(PushIoBlockQueue(i, std::move(eof))); + } + + return Status::OK(); +} + +// Pushes a control indicator onto the IOBlockQueue for each worker to consume. When the worker +// pops this control indicator, it will wait until the next epoch starts and then resume execution. +Status TFReaderOp::PostEndOfEpoch(int32_t queue_index) { + for (int i = 0; i < num_workers_; ++i) { + std::unique_ptr eoe = mindspore::make_unique(IOBlock::kDeIoBlockFlagEoe); + RETURN_IF_NOT_OK(PushIoBlockQueue((queue_index + i) % num_workers_, std::move(eoe))); + } + + return Status::OK(); +} + +bool TFReaderOp::NeedPushFileToblockQueue(const std::string &file_name, int64_t *start_offset, int64_t *end_offset, + const int64_t &pre_count) { + *start_offset = 0; + *end_offset = 0; + bool push = false; + int64_t start_index = device_id_ * num_rows_per_shard_; + if (device_id_ + 1 < 0) { + MS_LOG(ERROR) << "Device id is invalid"; + return false; + } + int64_t end_index = (static_cast(device_id_) + 1) * num_rows_per_shard_; + + if (pre_count <= start_index && pre_count + filename_numrows_[file_name] > start_index) { + *start_offset = start_index - pre_count; + push = true; + if (pre_count < end_index && pre_count + filename_numrows_[file_name] >= end_index) { + *end_offset = end_index - pre_count; + } else { + *end_offset = filename_numrows_[file_name]; + } + } + + if (pre_count >= start_index && pre_count < end_index) { + *start_offset = 0; + push = true; + if (pre_count + filename_numrows_[file_name] >= end_index) { + *end_offset = end_index - pre_count; + } else { + *end_offset = filename_numrows_[file_name]; + } + } + + return push; +} + +Status TFReaderOp::FillIOBlockShuffle(const std::vector &i_keys) { + int32_t queue_index = 0; + int32_t key_index = 0; + int64_t pre_count = 0; + int64_t start_offset = 0; + int64_t end_offset = 0; + bool finish = false; + while (!finish) { + for (auto it = i_keys.begin(); it != i_keys.end(); ++it) { + { + std::unique_lock lock(load_io_block_queue_mutex_); + if (load_io_block_queue_ == false) { + break; + } + } + if (!equal_rows_per_shard_) { + if (key_index++ % num_devices_ == device_id_) { + auto ioBlock = make_unique(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone); + RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); + queue_index = (queue_index + 1) % num_workers_; + } + } else { + // Do an index lookup using that key to get the filename. + auto file_it = filename_index_->Search(*it); + std::string file_name = file_it.value(); + if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { + auto ioBlock = make_unique(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone); + RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); + MS_LOG(DEBUG) << "File name " << *it << " start offset " << start_offset << " end_offset " << end_offset; + queue_index = (queue_index + 1) % num_workers_; + } + + pre_count += filename_numrows_[file_name]; + } + } + if (equal_rows_per_shard_ && pre_count < (static_cast(device_id_) + 1) * num_rows_per_shard_) { + finish = false; + } else { + finish = true; + } + } + RETURN_IF_NOT_OK(PostEndOfEpoch(queue_index)); + return Status::OK(); +} + +Status TFReaderOp::FillIOBlockNoShuffle() { + int32_t queue_index = 0; + int32_t key_index = 0; + int64_t pre_count = 0; + int64_t start_offset = 0; + int64_t end_offset = 0; + bool finish = false; + while (!finish) { + // Iterate over all the keys and add one key to each block. + for (auto it = filename_index_->begin(); it != filename_index_->end(); ++it) { + { + std::unique_lock lock(load_io_block_queue_mutex_); + if (load_io_block_queue_ == false) { + break; + } + } + if (!equal_rows_per_shard_) { + if (key_index++ % num_devices_ == device_id_) { + auto ioBlock = make_unique(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone); + RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); + queue_index = (queue_index + 1) % num_workers_; + } + } else { + std::string file_name = it.value(); + if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) { + auto ioBlock = make_unique(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone); + RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock))); + queue_index = (queue_index + 1) % num_workers_; + } + + pre_count += filename_numrows_[file_name]; + } + } + if (equal_rows_per_shard_ && pre_count < (static_cast(device_id_) + 1) * num_rows_per_shard_) { + finish = false; + } else { + finish = true; + } + } + + RETURN_IF_NOT_OK(PostEndOfEpoch(queue_index)); + return Status::OK(); +} + +// Called asynchronously by another thread. Will wait until notified to fill the IOBlockQueue. +Status TFReaderOp::WaitToFillIOBlockQueue() { + // must be called first if called by worker spawned by taskgroup + TaskManager::FindMe()->Post(); + + std::vector i_keys; + // Generate a vector of keys that we can shuffle + if (shuffle_files_) { + for (auto it = filename_index_->begin(); it != filename_index_->end(); ++it) { + i_keys.push_back(it.key()); + } + } + uint32_t seed = 0; + while (true) { + RETURN_IF_NOT_OK(io_block_queue_wait_post_.Wait()); + io_block_queue_wait_post_.Clear(); + + if (finished_reading_dataset_) { + break; + } + + if (shuffle_files_) { + shuffleKeys(&i_keys, num_devices_ == 1 ? GetSeed() : ++seed); + RETURN_IF_NOT_OK(FillIOBlockShuffle(i_keys)); + } else { // shuffle_files_ == false + RETURN_IF_NOT_OK(FillIOBlockNoShuffle()); + } + } + + return Status::OK(); +} + +// Notifies the thread which called WaitToFillIOBlockQueue to resume execution. +void TFReaderOp::NotifyToFillIOBlockQueue() { io_block_queue_wait_post_.Set(); } + +// Pops an element from a queue in io_block_queues +Status TFReaderOp::PopIoBlockQueue(int32_t index, std::unique_ptr *out_block) { + RETURN_IF_NOT_OK(io_block_queues_[index]->PopFront(out_block)); + + return Status::OK(); +} + +// Pushes an element to a queue in io_block_queues +Status TFReaderOp::PushIoBlockQueue(int32_t index, std::unique_ptr &&io_block) { + RETURN_IF_NOT_OK(io_block_queues_[index]->Add(std::move(io_block))); + + return Status::OK(); +} + +// Reads a tf_file file and loads the data into multiple buffers. +Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_offset, const int64_t end_offset, + const int32_t &worker_id) { + std::ifstream reader; + reader.open(filename); + if (!reader) { + RETURN_STATUS_UNEXPECTED("failed to open file: " + filename); + } + + int64_t rows_read = 0; + int64_t rows_total = 0; + std::unique_ptr current_buffer = + mindspore::make_unique(0, DataBuffer::BufferFlags::kDeBFlagNone); + std::unordered_map column_name_map; + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_map[data_schema_->column(i).name()] = i; + } + current_buffer->set_column_name_map(column_name_map); + std::unique_ptr new_tensor_table = make_unique(); + + while (reader.peek() != EOF) { + // read length + int64_t record_length = 0; + (void)reader.read(reinterpret_cast(&record_length), static_cast(sizeof(int64_t))); + + // ignore crc header + (void)reader.ignore(static_cast(sizeof(int32_t))); + + // read serialized Example + std::string serialized_example; + serialized_example.resize(record_length); + (void)reader.read(&serialized_example[0], static_cast(record_length)); + if (start_offset == kInvalidOffset || (rows_total >= start_offset && rows_total < end_offset)) { + dataengine::Example tf_file; + if (!tf_file.ParseFromString(serialized_example)) { + std::string errMsg = "parse tfrecord failed"; + RETURN_STATUS_UNEXPECTED(errMsg); + } + RETURN_IF_NOT_OK(LoadExample(&tf_file, &new_tensor_table, rows_read)); + rows_read++; + } + // ignore crc footer + (void)reader.ignore(static_cast(sizeof(int32_t))); + rows_total++; + + if (rows_read == rows_per_buffer_) { + current_buffer->set_tensor_table(std::move(new_tensor_table)); + RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer))); + + current_buffer = make_unique(0, DataBuffer::BufferFlags::kDeBFlagNone); + current_buffer->set_column_name_map(column_name_map); + new_tensor_table = make_unique(); + rows_read = 0; + } + } + + if (rows_read > 0) { + current_buffer->set_tensor_table(std::move(new_tensor_table)); + RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer))); + } + + return Status::OK(); +} + +// Parses a single row and puts the data into a tensor table. +Status TFReaderOp::LoadExample(const dataengine::Example *tf_file, std::unique_ptr *tensor_table, + int64_t row) { + int32_t num_columns = data_schema_->NumColumns(); + TensorRow newRow(num_columns, nullptr); + (*tensor_table)->push_back(std::move(newRow)); + + for (int32_t col = 0; col < num_columns; ++col) { + const ColDescriptor current_col = data_schema_->column(col); + const dataengine::Features &example_features = tf_file->features(); + const google::protobuf::Map &feature_map = example_features.feature(); + const dataengine::Feature &column_values_list = feature_map.at(current_col.name()); + RETURN_IF_NOT_OK(LoadFeature(tensor_table, column_values_list, current_col, row, col)); + } + + return Status::OK(); +} + +// Parses a single cell and puts the data into a tensor table. +Status TFReaderOp::LoadFeature(const std::unique_ptr *tensor_table, + const dataengine::Feature &column_values_list, const ColDescriptor ¤t_col, + int64_t row, int32_t col) { + const dataengine::Feature::KindCase column_list_type = column_values_list.kind_case(); + std::unique_ptr float_array; // For staging data from protobuf deserialization + const unsigned char *data_ptr = nullptr; // Generic pointer used for populating the Tensor + + // This variable will point into the above staging variables. + // Also used for creating shape attributes. + int32_t num_elements = 0; + + // we build a tensor first a read directly into it if we need to cast + std::shared_ptr ts; + + // Depending on the type of data from the tf_file, we want to extract 2 things: + // 1) A pointer to the data as a const unsigned char * + // 2) The number of elements of the data + // After those are determined, we can then build the tensor to represent this data. + switch (column_list_type) { + case dataengine::Feature::KindCase::kBytesList: { + RETURN_IF_NOT_OK(LoadBytesList(current_col, column_values_list, &num_elements, &ts)); + + break; + } + case dataengine::Feature::KindCase::kFloatList: { + RETURN_IF_NOT_OK(LoadFloatList(current_col, column_values_list, &num_elements, &float_array)); + + data_ptr = reinterpret_cast(float_array.get()); + + // only floatList needs to create the tensor here, other two lists read directly + // into the tensor + TensorShape current_shape = TensorShape::CreateUnknownRankShape(); + RETURN_IF_NOT_OK(current_col.MaterializeTensorShape(num_elements, ¤t_shape)); + RETURN_IF_NOT_OK( + Tensor::CreateTensor(&ts, current_col.tensorImpl(), current_shape, current_col.type(), data_ptr)); + break; + } + case dataengine::Feature::KindCase::kInt64List: { + RETURN_IF_NOT_OK(LoadIntListSwitch(current_col, column_values_list, &num_elements, &ts)); + break; + } + case dataengine::Feature::KindCase::KIND_NOT_SET: { + std::string err_msg = "tf_file column list type enum is KIND_NOT_SET"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + default: { + std::string err_msg = "tf_file column list type enum does not match any known DE type"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + } + + (**tensor_table)[row][col] = std::move(ts); + + return Status::OK(); +} + +// Overrides base class reset method. Cleans up any state info from it's previous execution and +// reinitializes itself so that it can be executed again, as if it was just created. +Status TFReaderOp::Reset() { + { + std::unique_lock lock(load_io_block_queue_mutex_); + load_io_block_queue_ = true; + } + + RETURN_IF_NOT_OK(ParallelOp::Reset()); + NotifyToFillIOBlockQueue(); + + return Status::OK(); +} + +Status TFReaderOp::LoadBytesList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor) { + // kBytesList can map to the following DE types ONLY! + // DE_UINT8, DE_INT8 + // Must be single byte type for each element! + if (current_col.type() != DataType::DE_UINT8 && current_col.type() != DataType::DE_INT8) { + std::string err_msg = "Invalid datatype for Tensor at column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + const dataengine::BytesList &bytes_list = column_values_list.bytes_list(); + + uint64_t max_size = 0; + for (uint32_t i = 0; i < bytes_list.value_size(); ++i) max_size = std::max(max_size, bytes_list.value(i).size()); + + *num_elements = bytes_list.value_size(); + + int64_t pad_size = max_size; + + // if user provides a shape in the form of [-1, d1, 2d, ... , dn], we need to pad to d1 * d2 * ... * dn + if (current_col.hasShape()) { + TensorShape cur_shape = current_col.shape(); + if (cur_shape.Size() >= 2 && cur_shape[0] == TensorShape::kDimUnknown) { + int64_t new_pad_size = 1; + for (int i = 1; i < cur_shape.Size(); ++i) { + if (cur_shape[i] == TensorShape::kDimUnknown) { + std::string err_msg = "More than one unknown dimension in the shape of column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + new_pad_size *= cur_shape[i]; + } + pad_size = new_pad_size; + } + } + + // know how many elements there are and the total bytes, create tensor here: + TensorShape current_shape = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK(current_col.MaterializeTensorShape((*num_elements) * pad_size, ¤t_shape)); + RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, current_col.tensorImpl(), current_shape, current_col.type())); + + // Tensors are lazily allocated, this eagerly allocates memory for the tensor. + unsigned char *current_tensor_addr = (*tensor)->StartAddr(); + int64_t tensor_bytes_remaining = (*num_elements) * pad_size; + + if (current_tensor_addr == nullptr) { + std::string err_msg = "tensor memory allocation failed"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + RETURN_IF_NOT_OK(LoadAndPadBytes(current_tensor_addr, bytes_list, tensor_bytes_remaining, pad_size)); + + return Status::OK(); +} + +Status TFReaderOp::LoadAndPadBytes(unsigned char *current_tensor_addr, const dataengine::BytesList &bytes_list, + int64_t tensor_bytes_remaining, int64_t pad_size) { + if (current_tensor_addr == nullptr) { + std::string err_msg = "current_tensor_addr is null"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + for (int i = 0; i < bytes_list.value_size(); i++) { + // read string data into tensor + const std::string ¤t_element = bytes_list.value(i); + int return_code = + memcpy_s(current_tensor_addr, tensor_bytes_remaining, common::SafeCStr(current_element), current_element.size()); + if (return_code != 0) { + std::string err_msg = "memcpy_s failed when reading bytesList element into Tensor"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + current_tensor_addr += current_element.size(); + tensor_bytes_remaining -= current_element.size(); + + // pad + int64_t chars_to_pad = pad_size - current_element.size(); + return_code = memset_s(current_tensor_addr, tensor_bytes_remaining, static_cast(' '), chars_to_pad); + if (return_code != 0) { + std::string err_msg = "memset_s failed when padding bytesList in Tensor"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + + current_tensor_addr += chars_to_pad; + tensor_bytes_remaining -= chars_to_pad; + } + + return Status::OK(); +} + +Status TFReaderOp::LoadFloatList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::unique_ptr *float_array) { + // KFloatList can only map to DE types: + // DE_FLOAT32 + if (current_col.type() != DataType::DE_FLOAT32) { + std::string err_msg = "Invalid datatype for Tensor at column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + const dataengine::FloatList &float_list = column_values_list.float_list(); + + // Identify how many values we have and then create a local array of these + // to deserialize into + *num_elements = float_list.value_size(); + *float_array = mindspore::make_unique(*num_elements); + for (int i = 0; i < float_list.value_size(); ++i) { + (*float_array)[i] = float_list.value(i); + } + + return Status::OK(); +} + +// Determines which template type to use and calls LoadIntList +Status TFReaderOp::LoadIntListSwitch(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor) { + if (current_col.type() == DataType::DE_UINT64) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_INT64) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_UINT32) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_INT32) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_UINT16) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_INT16) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_UINT8) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else if (current_col.type() == DataType::DE_INT8) { + RETURN_IF_NOT_OK(LoadIntList(current_col, column_values_list, num_elements, tensor)); + } else { + std::string err_msg = "Invalid datatype for Tensor at column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + return Status::OK(); +} + +// Reads values from a bytes list and casts the value to type T, must be an integral type +// compatible with int64_t +template +Status TFReaderOp::LoadIntList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor) { + if (!(current_col.type().IsInt())) { + std::string err_msg = "Invalid datatype for Tensor at column: " + current_col.name(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + const dataengine::Int64List &int64_list = column_values_list.int64_list(); + + // Identify how many values we have and then create a local array of these + // to deserialize into + *num_elements = int64_list.value_size(); + + // know how many elements there are, create tensor here: + TensorShape current_shape = TensorShape::CreateUnknownRankShape(); + RETURN_IF_NOT_OK(current_col.MaterializeTensorShape(*num_elements, ¤t_shape)); + RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, current_col.tensorImpl(), current_shape, current_col.type())); + + // Tensors are lazily allocated, this eagerly allocates memory for the tensor. + (void)(*tensor)->StartAddr(); + + int64_t i = 0; + auto it = (*tensor)->begin(); + for (; it != (*tensor)->end(); i++, ++it) { + T element = static_cast(int64_list.value(i)); + *it = element; + } + + return Status::OK(); +} + +Status TFReaderOp::CreateSchema(const std::string tf_file, const std::vector &columns_to_load) { + std::ifstream reader; + reader.open(tf_file); + + // read length + int64_t record_length = 0; + (void)reader.read(reinterpret_cast(&record_length), static_cast(sizeof(int64_t))); + + // ignore crc header + (void)reader.ignore(static_cast(sizeof(int32_t))); + + // read serialized Example + std::string serialized_example; + serialized_example.resize(record_length); + (void)reader.read(&serialized_example[0], static_cast(record_length)); + + dataengine::Example example; + if (!example.ParseFromString(serialized_example)) RETURN_STATUS_UNEXPECTED("parse tf_file failed"); + + const dataengine::Features &example_features = example.features(); + const google::protobuf::Map &feature_map = example_features.feature(); + std::vector columns = columns_to_load; + + if (columns_to_load.empty()) + (void)std::transform(feature_map.begin(), feature_map.end(), std::back_inserter(columns), + [](const auto &it) -> std::string { return it.first; }); + for (const auto &curr_col_name : columns) { + auto it = feature_map.find(curr_col_name); + std::string column_name = it->first; + + std::string column_type; + + const dataengine::Feature &feature = it->second; + const dataengine::Feature::KindCase kind_case = feature.kind_case(); + switch (kind_case) { + case dataengine::Feature::KindCase::kBytesList: + column_type = "uint8"; + break; + + case dataengine::Feature::KindCase::kFloatList: + column_type = "float32"; + break; + + case dataengine::Feature::KindCase::kInt64List: + column_type = "int64"; + break; + + case dataengine::Feature::KindCase::KIND_NOT_SET: + RETURN_STATUS_UNEXPECTED("trying to make schema, tf_file column list type enum is KIND_NOT_SET"); + + default: + RETURN_STATUS_UNEXPECTED( + "trying to make schema, tf_file column list type enum does not match any known DE type"); + } + + RETURN_IF_NOT_OK( + data_schema_->AddColumn(ColDescriptor(column_name, DataType(column_type), TensorImpl::kFlexible, 1))); + } + + return Status::OK(); +} + +Status TFReaderOp::CountTotalRows(int64_t *out_total_rows, const std::vector &filenames, int64_t threads, + bool estimate) { + try { + if (threads > filenames.size()) { + threads = filenames.size(); + } + + std::vector> async_results; + + int64_t chunk_size = filenames.size() / threads; + int64_t remainder = filenames.size() % threads; + + int64_t begin = 0; + int64_t end = begin; + for (int i = 0; i < threads; i++) { + end += chunk_size; + if (remainder > 0) { + end++; + remainder--; + } + + if (estimate) { + // Parse a single file for each chunk with estimate mode on + async_results.push_back(std::async(std::launch::async, &CountTotalRowsSectioned, filenames, begin, begin + 1)); + } else { + // Parse the whole chunk with estimate mode off + async_results.push_back(std::async(std::launch::async, &CountTotalRowsSectioned, filenames, begin, end)); + } + + begin = end; + } + + int64_t total_rows = 0; + for (int i = 0; i < async_results.size(); i++) { + total_rows += async_results[i].get(); + } + + if (estimate) { + // Each thread only scans 1 file + // Estimated total rows = Average rows * total number of files + total_rows = total_rows / threads * filenames.size(); + } + + *out_total_rows = total_rows; + } catch (const std::exception &e) { + std::string err_msg = "Unexpected error occurred: "; + err_msg += e.what(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + return Status::OK(); +} + +int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector &filenames, int64_t begin, int64_t end) { + int64_t rows_read = 0; + for (int i = begin; i < end; i++) { + std::ifstream reader; + reader.open(filenames[i]); + if (!reader) { + MS_LOG(DEBUG) << "TFReader operator failed to open file " << filenames[i] << "."; + } + + while (reader.peek() != EOF) { + // read length + int64_t record_length = 0; + (void)reader.read(reinterpret_cast(&record_length), static_cast(sizeof(int64_t))); + + // ignore crc header + (void)reader.ignore(static_cast(sizeof(int32_t))); + + // ignore tf_file contents + (void)reader.ignore(static_cast(record_length)); + + // ignore crc footer + (void)reader.ignore(static_cast(sizeof(int32_t))); + + rows_read++; + } + } + + return rows_read; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h new file mode 100644 index 00000000000..69de068f9b0 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -0,0 +1,384 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_TF_READER_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_TF_READER_OP_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dataset/util/wait_post.h" +#include "dataset/util/auto_index.h" +#include "dataset/util/status.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" + +namespace dataengine { +class Example; +class Feature; +class BytesList; +} // namespace dataengine + +namespace mindspore { +namespace dataset { +template +class Queue; + +template +class Connector; + +class JaggedConnector; +class FilenameBlock; + +using StringIndex = AutoIndexObj; + +class TFReaderOp : public ParallelOp { + public: + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + // Default destructor + ~Builder() = default; + + // Checks if the inputs of the builder is valid. + // @return Status - the error code returned. + Status ValidateInputs() const; + + Status Build(std::shared_ptr *out_tf_reader_op); + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetDataSchema(std::unique_ptr data_schema) { + builder_data_schema_ = std::move(data_schema); + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetWorkerConnectorSize(int32_t size) { + builder_worker_connector_size_ = size; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = op_connector_size; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int64_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetNumDevices(int64_t num_dev) { + builder_num_devices_ = num_dev; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetDeviceId(int64_t dev_id) { + builder_device_id_ = dev_id; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &setTotalRows(int64_t total_rows) { + builder_total_rows_ = total_rows; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetDatasetFilesList(const std::vector &dataset_files_list) { + builder_dataset_files_list_ = dataset_files_list; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetColumnsToLoad(const std::vector &columns_to_load) { + builder_columns_to_load_ = columns_to_load; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetShuffleFiles(bool shuffle_files) { + builder_shuffle_files_ = shuffle_files; + return *this; + } + + // Setter method. + // @return Builder - setter method returns reference to the builder. + Builder &SetShardEqualRows(bool shard_equal_rows) { + builder_equal_rows_per_shard_ = shard_equal_rows; + return *this; + } + + private: + std::unique_ptr builder_data_schema_; + int32_t builder_device_id_; + int32_t builder_num_devices_; + int32_t builder_num_workers_; + int32_t builder_worker_connector_size_; + int32_t builder_op_connector_size_; + int64_t builder_rows_per_buffer_; + int64_t builder_total_rows_; + std::vector builder_dataset_files_list_; + std::vector builder_columns_to_load_; + bool builder_shuffle_files_; + bool builder_equal_rows_per_shard_; + }; + + // Constructor of TFReaderOp (2) + // @note The builder class should be used to call this constructor. + // @param num_workers - number of worker threads reading data from tf_file files. + // @param worker_connector_size - size of each internal queue. + // @param rows_per_buffer - number of rows that a full buffer will contain. + // @param total_num_rows - Number of rows to read + // @param dataset_files_list - list of filepaths for the dataset files. + // @param data_schema - the data schema object. + // @param op_connector_size - size of each queue in the connector that the child operator pulls from. + // @param columns_to_load - the names of the columns to load data from. + // @param shuffle_files - whether or not to shuffle the files before reading data. + // @param equal_rows_per_shard - whether or not to get equal rows for each process. + TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, int64_t total_num_rows, + std::vector dataset_files_list, std::unique_ptr data_schema, + int32_t op_connector_size, std::vector columns_to_load, bool shuffle_files, + int32_t num_devices, int32_t device_id, bool equal_rows_per_shard); + + // Default destructor + ~TFReaderOp() = default; + + // Instantiates the internal queues and connectors. + // @return Status - the error code returned. + Status Init(); + + // Class functor operator () override. + // All dataset operators operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work + // @return Status - the error code returned. + Status operator()() override; + + // Overrides base class reset method. Cleans up any state info from it's previous execution and + // reinitializes itself so that it can be executed again, as if it was just created. + // @return Status - the error code returned. + Status Reset() override; + + // Getter method + int64_t rows_per_buffer() const { return rows_per_buffer_; } + + // Reads all the provided tf_file files and counts the total number of rows. filenames will + // first be sectioned into equal parts, then sections are read in parallel. If threads is + // greater than the number of files, threads will be clamped to the number of files. + // @param out_total_tows - output parameter which contains the total number of rows + // @param filenames - a list of tf_file filenames. + // @param threads - number of threads to use to read the tf_file files. + // @param estimate - estimate mode, under this mode each threads will sample a single file from each chunk + // @return Status - the error code returned. + static Status CountTotalRows(int64_t *out_total_rows, const std::vector &filenames, int64_t threads = 1, + bool estimate = false); + + private: + // The entry point for when workers are launched. + // @param worker_id - the id of the worker that is executing this function. + // @return Status - the error code returned. + Status WorkerEntry(int32_t worker_id) override; + + // Pushes a control indicator onto the IOBlockQueue for each worker to consume. + // When the worker pops this control indicator, it will shut itself down gracefully. + // @return Status - the error code returned. + Status PostEndOfData(); + + // Pushes a control indicator onto the IOBlockQueue for each worker to consume. When the worker + // pops this control indicator, it will wait until the next epoch starts and then resume execution. + // @return Status - the error code returned. + Status PostEndOfEpoch(int32_t queue_index); + + // Called asynchronously by another thread. Will wait until notified to fill the IOBlockQueue. + // @return Status - the error code returned. + Status WaitToFillIOBlockQueue(); + + // Notifies the thread which called WaitToFillIOBlockQueue to resume execution. + void NotifyToFillIOBlockQueue(); + + // Pops an element from a queue in IOBlockQueue. + // @param index - the index of the queue to pop from. + // @param out_block - the popped element. + // @return Status - the error code returned. + Status PopIoBlockQueue(int32_t index, std::unique_ptr *out_block); + + // Pushes an element to a queue in IOBlockQueue. + // @param index - the index of the queue to push to. + // @param io_block - the element to push onto the queue. + // @return Status - the error code returned. + Status PushIoBlockQueue(int32_t index, std::unique_ptr &&io_block); + + // Reads a tf_file file and loads the data into multiple buffers. + // @param filename - the tf_file file to read. + // @param start_offset - the start offset of file. + // @param end_offset - the end offset of file. + // @param worker_id - the id of the worker that is executing this function. + // @return Status - the error code returned. + Status LoadFile(const std::string &filename, const int64_t start_offset, const int64_t end_offset, + const int32_t &worker_id); + + // Parses a single row and puts the data into a tensor table. + // @param tf_file - the row to be parsed. + // @param tensor_table - the tensor table to put the parsed data in. + // @param row - the id of the row filled in the tensor table. + // @return Status - the error code returned. + Status LoadExample(const dataengine::Example *tf_file, std::unique_ptr *tensor_table, int64_t row); + + // Parses a single cell and puts the data into a tensor table. + // @param tensor_table - the tensor table to put the parsed data in. + // @param column_values_list - the cell to parse. + // @param current_col - the column descriptor containing the expected shape and type of the data. + // @return Status - the error code returned. + Status LoadFeature(const std::unique_ptr *tensor_table, const dataengine::Feature &column_values_list, + const ColDescriptor ¤t_col, int64_t row, int32_t col); + + // Reads values from a bytes list + // @param current_col - the column descriptor containing the expected shape and type of the data. + // @param column_values_list - the cell that contains the bytes list to read from. + // @param elementStr - the string we read the value into. + // @return Status - the error code returned. + Status LoadBytesList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor); + + // Loads all the strings in bytes_list into the memory at current_tensor_addr. + // @param current_tensor_addr - the memory address to load the strings to. + // @param bytes_list - the list of strings to load. + // @param tensor_bytes_remaining - the number of bytes available for this function to use. + // @param pad_size - number of bytes to pad to. + // @return Status - the error code returned. + Status LoadAndPadBytes(unsigned char *current_tensor_addr, const dataengine::BytesList &bytes_list, + int64_t tensor_bytes_remaining, int64_t pad_size); + + // Reads values from a float list + // @param current_col - the column descriptor containing the expected shape and type of the data. + // @param column_values_list - the cell that contains the float list to read from. + // @Param numElements - number of values in the float list. + // @param float_array - the array we read the values into. + // @return Status - the error code returned. + Status LoadFloatList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::unique_ptr *float_array); + + // Reads values from a bytes list and casts the value to type T, must be an integral + // type compatible with int64_t + // @param current_col - the column descriptor containing the expected shape and type of the data. + // @param column_values_list - the cell that contains the int list to read from. + // @Param num_elements - number of values in the int list. + // @param tensor - the tensor we read the values into. + // @return Status - the error code returned. + template + Status LoadIntList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor); + + // Determines which template type to use and calls LoadIntList + // @param current_col - the column descriptor containing the expected shape and type of the data. + // @param column_values_list - the cell that contains the int list to read from. + // @Param numElements - number of values in the int list. + // @param tensor - the tensor we read the values into. + // @return Status - the error code returned. + Status LoadIntListSwitch(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor); + + // Reads one row of data from a tf file and creates a schema based on that row + // @return Status - the error code returned. + Status CreateSchema(const std::string tf_file, const std::vector &columns_to_load); + + // Meant to be called async. Will read files in the range [begin, end) and return the total rows + // @param filenames - a list of tf data filenames. + // @param begin - index of first file to read. + // @param end - one greater than the index of the last file to read. + // @return int63_t - the total number of rows of files read. + static int64_t CountTotalRowsSectioned(const std::vector &filenames, const int64_t begin, + const int64_t end); + // Fill IO block queue if shuffle is true + // @param i_keys - shuffle keys. + // @return Status - the error code returned. + Status FillIOBlockShuffle(const std::vector &i_keys); + + /** + * Fill IO block queue if shuffle is false + * @param i_keys - shuffle keys. + * @return Status - the error code returned. + */ + Status FillIOBlockNoShuffle(); + + // Select file and push it to the block queue. + // @param file_name - File name. + // @param start_file - If file contains the first sample of data. + // @param end_file - If file contains the end sample of data. + // @param pre_count - Total rows of previous files. + // @return Status - the error code returned. + bool NeedPushFileToblockQueue(const std::string &file_name, int64_t *start_offset, int64_t *end_offset, + const int64_t &pre_count); + + // Caculate number of rows in each shard. + // @return Status - the error code returned. + Status CalculateNumRowsPerShard(); + + int32_t device_id_; + int32_t num_devices_; + int64_t rows_per_buffer_; + int64_t total_rows_; + std::vector dataset_files_list_; + std::vector columns_to_load_; + bool finished_reading_dataset_; + bool shuffle_files_; + std::unique_ptr data_schema_; + std::unique_ptr filename_index_; + bool load_io_block_queue_; + + std::unique_ptr jagged_buffer_connector_; + QueueList> io_block_queues_; + WaitPost io_block_queue_wait_post_; + std::mutex load_io_block_queue_mutex_; + std::map filename_numrows_; + int64_t num_rows_; + int64_t num_rows_per_shard_; + bool equal_rows_per_shard_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_TF_READER_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc new file mode 100644 index 00000000000..d3785f4660d --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -0,0 +1,270 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/source/voc_op.h" + +#include + +#include "common/utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/db_connector.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { +VOCOp::Builder::Builder() : builder_decode_(false), builder_num_samples_(0), builder_sampler_(nullptr) { + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_num_workers_ = cfg->num_parallel_workers(); + builder_rows_per_buffer_ = cfg->rows_per_buffer(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status VOCOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + if (builder_sampler_ == nullptr) { + builder_sampler_ = std::make_shared(); + } + builder_schema_ = make_unique(); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); + RETURN_IF_NOT_OK( + builder_schema_->AddColumn(ColDescriptor("target", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1))); + *ptr = std::make_shared(builder_num_workers_, builder_rows_per_buffer_, builder_dir_, + builder_op_connector_size_, builder_num_samples_, builder_decode_, + std::move(builder_schema_), std::move(builder_sampler_)); + return Status::OK(); +} + +Status VOCOp::Builder::SanityCheck() { + Path dir(builder_dir_); + std::string err_msg; + err_msg += dir.IsDirectory() == false ? "VOC path is invalid or not set\n" : ""; + err_msg += builder_num_workers_ <= 0 ? "Num of parallel workers is set to 0 or negative\n" : ""; + err_msg += builder_num_samples_ < 0 ? "num_samples is negative\n" : ""; + return err_msg.empty() ? Status::OK() : Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, err_msg); +} + +VOCOp::VOCOp(int32_t num_workers, int32_t rows_per_buffer, const std::string &folder_path, int32_t queue_size, + int64_t num_samples, bool decode, std::unique_ptr data_schema, + std::shared_ptr sampler) + : ParallelOp(num_workers, queue_size), + decode_(decode), + row_cnt_(0), + buf_cnt_(0), + num_rows_(0), + num_samples_(num_samples), + folder_path_(folder_path), + rows_per_buffer_(rows_per_buffer), + sampler_(std::move(sampler)), + data_schema_(std::move(data_schema)) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + col_name_map_[data_schema_->column(i).name()] = i; + } + io_block_queues_.Init(num_workers_, queue_size); +} + +Status VOCOp::TraverseSampleIds(const std::shared_ptr &sample_ids, std::vector *keys) { + for (auto itr = sample_ids->begin(); itr != sample_ids->end(); ++itr) { + if ((*itr) > num_rows_) continue; + if (row_cnt_ == num_samples_) break; + keys->push_back(*itr); + row_cnt_++; + if (row_cnt_ % rows_per_buffer_ == 0) { + RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add( + make_unique(IOBlock(*keys, IOBlock::kDeIoBlockNone)))); + keys->clear(); + } + } + return Status::OK(); +} + +Status VOCOp::operator()() { + RETURN_IF_NOT_OK(LaunchThreadsAndInitOp()); + std::unique_ptr sampler_buffer; + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + while (true) { + std::vector keys; + keys.reserve(rows_per_buffer_); + while (sampler_buffer->eoe() == false) { + std::shared_ptr sample_ids; + RETURN_IF_NOT_OK(sampler_buffer->GetTensor(&sample_ids, 0, 0)); + if (sample_ids->type() != DataType(DataType::DE_INT64)) { + RETURN_STATUS_UNEXPECTED("Sampler Tensor isn't int64"); + } + RETURN_IF_NOT_OK(TraverseSampleIds(sample_ids, &keys)); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + if (keys.empty() == false) { + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add( + make_unique(IOBlock(keys, IOBlock::kDeIoBlockNone)))); + } + if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) { + std::unique_ptr eoe_block = make_unique(IOBlock::kDeIoBlockFlagEoe); + std::unique_ptr eof_block = make_unique(IOBlock::kDeIoBlockFlagEof); + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block))); + RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block))); + for (int32_t i = 0; i < num_workers_; i++) { + RETURN_IF_NOT_OK( + io_block_queues_[i]->Add(make_unique(std::vector(), IOBlock::kDeIoBlockNone))); + } + return Status::OK(); + } else { + RETURN_IF_NOT_OK( + io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique(IOBlock::kDeIoBlockFlagEoe))); + RETURN_IF_NOT_OK(wp_.Wait()); + wp_.Clear(); + RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer)); + } + } +} + +void VOCOp::Print(std::ostream &out, bool show_all) const { + DatasetOp::Print(out, show_all); + out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ + << "\nVOC Directory: " << folder_path_ << "\n-------------------\n"; +} + +Status VOCOp::Reset() { + RETURN_IF_NOT_OK(sampler_->Reset()); + row_cnt_ = 0; + wp_.Set(); + return Status::OK(); +} + +Status VOCOp::GetNumSamples(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_samples_; + return Status::OK(); +} + +Status VOCOp::LoadTensorRow(const std::string &image_id, TensorRow *trow) { + std::shared_ptr image, target; + const std::string kImageDir = folder_path_ + "/JPEGImages/" + image_id + ".jpg"; + const std::string kTargetDir = folder_path_ + "/SegmentationClass/" + image_id + ".png"; + RETURN_IF_NOT_OK(ReadImageToTensor(kImageDir, data_schema_->column(0), &image)); + RETURN_IF_NOT_OK(ReadImageToTensor(kTargetDir, data_schema_->column(1), &target)); + (*trow) = {std::move(image), std::move(target)}; + return Status::OK(); +} + +Status VOCOp::LoadBuffer(const std::vector &keys, std::unique_ptr *db) { + std::unique_ptr deq = make_unique(); + TensorRow trow; + for (const uint64_t &key : keys) { + RETURN_IF_NOT_OK(this->LoadTensorRow(image_ids_[key], &trow)); + deq->push_back(std::move(trow)); + } + (*db)->set_tensor_table(std::move(deq)); + (*db)->set_column_name_map(col_name_map_); + return Status::OK(); +} + +Status VOCOp::WorkerEntry(int32_t worker_id) { + TaskManager::FindMe()->Post(); + int64_t buffer_id = worker_id; + std::unique_ptr io_block; + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + while (io_block != nullptr) { + if (io_block->eoe() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique(0, DataBuffer::kDeBFlagEOE))); + buffer_id = worker_id; + } else if (io_block->eof() == true) { + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (make_unique(0, DataBuffer::kDeBFlagEOF)))); + } else { + std::vector keys; + RETURN_IF_NOT_OK(io_block->GetKeys(&keys)); + if (keys.empty() == true) return Status::OK(); + std::unique_ptr db = make_unique(buffer_id, DataBuffer::kDeBFlagNone); + RETURN_IF_NOT_OK(LoadBuffer(keys, &db)); + RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db))); + buffer_id += num_workers_; + } + RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); + } + RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); +} + +Status VOCOp::ParseImageIds() { + const std::string kImageSets = "/ImageSets/Segmentation/train.txt"; + std::string image_sets_file = folder_path_ + kImageSets; + std::ifstream in_file; + in_file.open(image_sets_file); + if (in_file.fail()) { + RETURN_STATUS_UNEXPECTED("Fail to open file: " + image_sets_file); + } + std::string id; + while (getline(in_file, id)) { + image_ids_.push_back(id); + } + in_file.close(); + image_ids_.shrink_to_fit(); + num_rows_ = image_ids_.size(); + num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; + return Status::OK(); +} + +Status VOCOp::InitSampler() { + RETURN_IF_NOT_OK(sampler_->Init(this)); + return Status::OK(); +} + +Status VOCOp::LaunchThreadsAndInitOp() { + if (tree_ == nullptr) { + RETURN_STATUS_UNEXPECTED("tree_ not set"); + } + RETURN_IF_NOT_OK(io_block_queues_.Register(tree_->AllTasks())); + wp_.Register(tree_->AllTasks()); + RETURN_IF_NOT_OK(tree_->LaunchWorkers(num_workers_, std::bind(&VOCOp::WorkerEntry, this, std::placeholders::_1))); + TaskManager::FindMe()->Post(); + RETURN_IF_NOT_OK(this->ParseImageIds()); + RETURN_IF_NOT_OK(this->InitSampler()); + return Status::OK(); +} + +Status VOCOp::ReadImageToTensor(const std::string &path, const ColDescriptor &col, std::shared_ptr *tensor) { + std::ifstream fs; + fs.open(path, std::ios::binary | std::ios::in); + if (fs.fail()) { + RETURN_STATUS_UNEXPECTED("Fail to open file: " + path); + } + int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); + (void)fs.seekg(0, std::ios::beg); + RETURN_IF_NOT_OK( + Tensor::CreateTensor(tensor, col.tensorImpl(), TensorShape(std::vector(1, num_elements)), col.type())); + (void)fs.read(reinterpret_cast((*tensor)->StartAddr()), num_elements); + fs.close(); + if (decode_ == true) { + Status rc = Decode(*tensor, tensor); + if (rc.IsError()) { + RETURN_STATUS_UNEXPECTED("fail to decode file: " + path); + } + } + return Status::OK(); +} + +// Derived from RandomAccessOp +Status VOCOp::GetNumRowsInDataset(int64_t *num) const { + if (num == nullptr || num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED("NumRow not set"); + } + (*num) = num_rows_; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h new file mode 100644 index 00000000000..a267ef866a4 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h @@ -0,0 +1,220 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_VOC_OP_H_ +#define DATASET_ENGINE_DATASETOPS_SOURCE_VOC_OP_H_ + +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/data_schema.h" +#include "dataset/engine/datasetops/parallel_op.h" +#include "dataset/engine/datasetops/source/io_block.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/path.h" +#include "dataset/util/queue.h" +#include "dataset/util/status.h" +#include "dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +// Forward declares +template +class Queue; + +class VOCOp : public ParallelOp, public RandomAccessOp { + public: + class Builder { + public: + // Constructor for Builder class of ImageFolderOp + // @param uint32_t numWrks - number of parallel workers + // @param dir - directory folder got ImageNetFolder + Builder(); + + // Destructor. + ~Builder() = default; + + // Setter method. + // @param const std::string & build_dir + // @return Builder setter method returns reference to the builder. + Builder &SetDir(const std::string &build_dir) { + builder_dir_ = build_dir; + return *this; + } + + // Setter method. + // @param int32_t num_workers + // @return Builder setter method returns reference to the builder. + Builder &SetNumWorkers(int32_t num_workers) { + builder_num_workers_ = num_workers; + return *this; + } + + // Setter method. + // @param int32_t op_connector_size + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = op_connector_size; + return *this; + } + + // Setter method. + // @param int32_t rows_per_buffer + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method. + // @param int64_t num_samples + // @return Builder setter method returns reference to the builder. + Builder &SetNumSamples(int64_t num_samples) { + builder_num_samples_ = num_samples; + return *this; + } + + // Setter method. + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + + // Setter method. + // @param bool do_decode + // @return Builder setter method returns reference to the builder. + Builder &SetDecode(bool do_decode) { + builder_decode_ = do_decode; + return *this; + } + + // Check validity of input args + // @return = The error code return + Status SanityCheck(); + + // The builder "Build" method creates the final object. + // @param std::shared_ptr *op - DatasetOp + // @return - The error code return + Status Build(std::shared_ptr *op); + + private: + bool builder_decode_; + std::string builder_dir_; + int32_t builder_num_workers_; + int32_t builder_op_connector_size_; + int32_t builder_rows_per_buffer_; + int64_t builder_num_samples_; + std::shared_ptr builder_sampler_; + std::unique_ptr builder_schema_; + }; + + // Constructor + // @param int32_t num_workers - number of workers reading images in parallel + // @param int32_t rows_per_buffer - number of images (rows) in each buffer + // @param std::string folder_path - dir directory of VOC + // @param int32_t queue_size - connector queue size + // @param int64_t num_samples - number of samples to read + // @param bool decode - whether to decode images + // @param std::unique_ptr data_schema - the schema of the VOC dataset + // @param std::shared_ptr sampler - sampler tells VOCOp what to read + VOCOp(int32_t num_workers, int32_t rows_per_buffer, const std::string &folder_path, int32_t queue_size, + int64_t num_samples, bool decode, std::unique_ptr data_schema, std::shared_ptr sampler); + + // Destructor + ~VOCOp() = default; + + // Worker thread pulls a number of IOBlock from IOBlock Queue, make a buffer and push it to Connector + // @param int32_t workerId - id of each worker + // @return Status - The error code return + Status WorkerEntry(int32_t worker_id) override; + + // Main Loop of VOCOp + // Master thread: Fill IOBlockQueue, then goes to sleep + // Worker thread: pulls IOBlock from IOBlockQueue, work on it the put buffer to mOutConnector + // @return Status - The error code return + Status operator()() override; + + // Method derived from RandomAccessOp, enable Sampler to get numRows + // @param uint64_t num - to return numRows + // return Status - The error code return + Status GetNumSamples(int64_t *num) const override; + + // Method derived from RandomAccessOp, enable Sampler to get total number of rows in dataset + // @param uint64_t num - to return numRows + Status GetNumRowsInDataset(int64_t *num) const override; + + // A print method typically used for debugging + // @param out + // @param show_all + void Print(std::ostream &out, bool show_all) const override; + + private: + // Initialize Sampler, calls sampler->Init() within + // @return Status - The error code return + Status InitSampler(); + + // Load a tensor row according to image id + // @param std::string image_id - image id + // @param TensorRow row - image & target read into this tensor row + // @return Status - The error code return + Status LoadTensorRow(const std::string &image_id, TensorRow *row); + + // @param const std::string &path - path to the image file + // @param const ColDescriptor &col - contains tensor implementation and datatype + // @param std::shared_ptr tensor - return + // @return Status - The error code return + Status ReadImageToTensor(const std::string &path, const ColDescriptor &col, std::shared_ptr *tensor); + + // @param const std::vector &keys - keys in ioblock + // @param std::unique_ptr db + // @return Status - The error code return + Status LoadBuffer(const std::vector &keys, std::unique_ptr *db); + + Status ParseImageIds(); + + Status TraverseSampleIds(const std::shared_ptr &sample_ids, std::vector *keys); + + // Called first when function is called + // @return Status - The error code return + Status LaunchThreadsAndInitOp(); + + Status Reset() override; + + bool decode_; + uint64_t row_cnt_; + uint64_t buf_cnt_; + int64_t num_rows_; + int64_t num_samples_; + std::string folder_path_; + int32_t rows_per_buffer_; + std::shared_ptr sampler_; + std::unique_ptr data_schema_; + + WaitPost wp_; + std::vector image_ids_; + std::unordered_map col_name_map_; + QueueList> io_block_queues_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_DATASETOPS_SOURCE_VOC_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc new file mode 100644 index 00000000000..716c8534886 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc @@ -0,0 +1,247 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/datasetops/zip_op.h" +#include +#include "dataset/core/constants.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/db_connector.h" +#include "dataset/core/config_manager.h" +#include "dataset/core/global_context.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +ZipOp::Builder::Builder() { + // Some arguments to the ZipOp constructor have a default argument that is taken + // from the client config. + // The user may choose to change these values for the construction of the ZipOp by + // using the various builder set methods. + + std::shared_ptr cfg = GlobalContext::config_manager(); + builder_rows_per_buffer_ = cfg->rows_per_buffer(); + builder_op_connector_size_ = cfg->op_connector_size(); +} + +Status ZipOp::Builder::SanityCheck() const { return Status::OK(); } + +Status ZipOp::Builder::Build(std::shared_ptr *ptr) { + RETURN_IF_NOT_OK(SanityCheck()); + *ptr = std::make_shared(builder_rows_per_buffer_, builder_op_connector_size_); + return Status::OK(); +} + +// Construct ZipOp here, local variables initialized in operator due to tree construction restrictions +ZipOp::ZipOp(int32_t rows_per_buffer, int32_t op_connector_size) + : PipelineOp(op_connector_size), + children_num_(0), + rows_per_buffer_(rows_per_buffer), + buffer_id_(0), + draining_(false), + eof_(false) {} + +// destructor +ZipOp::~ZipOp() {} + +// Entry point for Zip, called by launch() +Status ZipOp::operator()() { + // The children_num_ parameter needs to be put here + children_num_ = child_.size(); + // Synchronize with TaskManager once the thread is created. + TaskManager::FindMe()->Post(); + + // initialize the iterators + for (int32_t i = 0; i < children_num_; ++i) { + // magic number 0 since Zip is not a parallel Op + child_iterators_.push_back(mindspore::make_unique(this, 0, i)); + } + + // Loop until eof is true + while (!eof_) { + // Create tensor table and prepare it by fetching and packing the first zipped row into it. + std::unique_ptr curr_table = mindspore::make_unique(); + RETURN_IF_NOT_OK(prepare(curr_table.get())); + + // If an eof got picked up during the above prepare, then we're done + if (eof_) { + break; + } + while (!draining_) { + // 1. If a previous loop iteration sent the current table out, then create a new one. + if (curr_table == nullptr) { + curr_table = mindspore::make_unique(); + } + + // 2 fill the table. Note: draining mode might get turned on if any of the child inputs were done + RETURN_IF_NOT_OK(fillBuffer(curr_table.get())); + + // 3 create and update buffer and send it to the out connector + if (!curr_table->empty()) { + std::unique_ptr curr_buffer = + mindspore::make_unique(buffer_id_, DataBuffer::kDeBFlagNone); + curr_buffer->set_tensor_table(std::move(curr_table)); + curr_buffer->set_column_name_map(col_name_id_map_); + MS_LOG(DEBUG) << "Zip operator finished one buffer, pushing, rows " << curr_buffer->NumRows() << ", cols " + << curr_buffer->NumCols() << ", map " << col_name_id_map_.size() << "."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(curr_buffer))); + buffer_id_++; + } + } + + // 4 handle drain state. + if (draining_) { + MS_LOG(DEBUG) << "Zip operator is now draining child inputs."; + RETURN_IF_NOT_OK(drainPipeline()); + // Now that we have drained child inputs, send the eoe up. + RETURN_IF_NOT_OK( + out_connector_->Add(0, std::move(mindspore::make_unique(0, DataBuffer::kDeBFlagEOE)))); + } + } + + // 5 handle eof + // propagate eof here. + MS_LOG(INFO) << "Zip operator got EOF, propagating."; + RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique(0, DataBuffer::kDeBFlagEOF)))); + return Status::OK(); +} + +// Handles preprocessing of the main loop, used when starting new epoch +Status ZipOp::prepare(TensorQTable *const table) { + MS_LOG(DEBUG) << "Zip operator prepares for new epoch."; + draining_ = false; + buffer_id_ = 0; + if (table == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "ZipOp prepare phase requires a tensor table."); + } + // fill initial row + TensorRow new_row; + RETURN_IF_NOT_OK(getNextTensorRow(&new_row)); + + // If the first row fetching resulted in eof, then we are done. + if (eof_) { + return Status::OK(); + } + if (new_row.empty()) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "ZipOp prepare phase got empty row!"); + } + + // Pack this first row into our tensor table + table->push_back(std::move(new_row)); + + // At this point we have at least 1 row produced, so all child iterators have their column names such that we + // can produce our column name map now. + col_name_id_map_ = {}; + for (int32_t i = 0; i < children_num_; ++i) { + // Initializing col_name_id_map_ from the first data buffer. + const std::unordered_map col_name_id_map = child_iterators_[i]->col_name_id_map(); + int32_t colsCurrent = col_name_id_map_.size(); + // the update code below shouldn't do anything bad if the column name already exists. + for (const auto &pair : col_name_id_map) { + std::string name = pair.first; + int32_t old_id = pair.second; + // check if name already exists in column name descriptor + if (col_name_id_map_.count(name) == 1) { + RETURN_STATUS_UNEXPECTED("key already exists when zipping datasets"); + } + col_name_id_map_[name] = old_id + colsCurrent; + } + } + return Status::OK(); +} + +// fillBuffer always expects a new table to fill +Status ZipOp::fillBuffer(TensorQTable *const table) { + if (table == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "ZipOp fillBuffer null table pointer."); + } + TensorRow new_row; + while (table->size() < static_cast(rows_per_buffer_)) { + RETURN_IF_NOT_OK(getNextTensorRow(&new_row)); + // Early exit the loop if we got empty row from any of our child iterations + if (new_row.empty()) { + return Status::OK(); + } + // else we got a row so pack it into the tensor table. + table->push_back(std::move(new_row)); + } + return Status::OK(); +} + +// fetches next zip buffer row (merged row) +Status ZipOp::getNextTensorRow(TensorRow *const new_zip_row) { + // iterate over all iterators and generate a row + for (int32_t i = 0; i < children_num_; ++i) { + TensorRow new_row = {}; + RETURN_IF_NOT_OK((child_iterators_[i])->FetchNextTensorRow(&new_row)); + // add each new row to iterator, check if row is empty, if row from iterator is empty return empty row + if (new_row.empty()) { + // If we did not get a row from any of the children, then it's the end of an epoch and we can move + // to drain state. + MS_LOG(INFO) << "Zip operator child iterator produced empty row."; + draining_ = true; + new_zip_row->clear(); + // If we picked up an eof here, then we are completely done. + if ((child_iterators_[i])->eof_handled()) { + MS_LOG(INFO) << "Zip operator iterator got EOF."; + eof_ = true; + } + return Status::OK(); + } else { + MS_LOG(DEBUG) << "Zip operator got row from child " << i << ". Num cols: " << new_row.size() << "."; + // if row isn't empty then we can append the fetched row with new_zip_row + new_zip_row->insert(new_zip_row->end(), new_row.begin(), new_row.end()); + } + } + MS_LOG(DEBUG) << "Zip operator builds a zipped row. Number of columns in row: " << new_zip_row->size() << "."; + return Status::OK(); +} + +// drain end of epoch messages from iterator for this epoch +Status ZipOp::drainPipeline() { + // we don't need to drain if we reached eof + if (eof_) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "ZipOp draining should not be done if already at eof!"); + } + for (int32_t con = 0; con < children_num_; ++con) { + MS_LOG(DEBUG) << "Zip operator draining child at " << con << "."; + RETURN_IF_NOT_OK(child_iterators_[con]->Drain()); + } + // at this point all connectors don't contain end of epoch messages. next iteration should be clean + return Status::OK(); +} + +// A function that prints info about the Operator +void ZipOp::Print(std::ostream &out, // In: The output stream to print to + bool show_all) const { // In: T/F if it should print everything + // Call base class printer first + PipelineOp::Print(out, show_all); + out << "\nZipOp:\n" + << "\nDatasets: " << children_num_ << "\n\n"; +} + +// overwrite function and handle eof +Status ZipOp::EofReceived(int32_t) { + MS_LOG(DEBUG) << "Zip operator EOF received, do nothing now."; + return Status::OK(); +} + +// overwrite function and handle eoe +Status ZipOp::EoeReceived(int32_t) { + state_ = OpState::kDeOpIdle; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h new file mode 100644 index 00000000000..f14ecba733e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h @@ -0,0 +1,145 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DATASETOPS_ZIP_OP_H_ +#define DATASET_ENGINE_DATASETOPS_ZIP_OP_H_ + +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/engine/dataset_iterator.h" +#include "dataset/engine/datasetops/pipeline_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// forward declare +class DataBuffer; + +class ZipOp : public PipelineOp { + public: + // The nested builder class inside of the BatchOp is used to help manage all of + // the arguments for constructing it. Use the builder by setting each argument + // with the provided set methods, and then finally call the build method to execute + // the actual construction. + // NOTE: the rows per buffer with initial value 0 means to default to the number of rows from the first child + + class Builder { + public: + // Builder constructor. Creates the builder object. + // @note No default args + // @return This is a constructor. + Builder(); + + // Default destructor + ~Builder() = default; + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetRowsPerBuffer(int32_t rows_per_buffer) { + builder_rows_per_buffer_ = rows_per_buffer; + return *this; + } + + // Setter method. + // @return Builder setter method returns reference to the builder. + Builder &SetOpConnectorSize(int32_t op_connector_size) { + builder_op_connector_size_ = op_connector_size; + return *this; + } + + // The builder "build" method creates the ZipOp dataset Operator. + // @return shared_ptr to the new StorageOp object + Status Build(std::shared_ptr *); + + private: + int32_t builder_rows_per_buffer_; + int32_t builder_op_connector_size_; + + Status SanityCheck() const; + }; + + // Constructor for ZipOp + // @param rows_per_buffer number of rows in output buffer + // @param op_connector_size connector + ZipOp(int32_t rows_per_buffer, int32_t op_connector_size); + + // Destructor + ~ZipOp(); + + Status EofReceived(int32_t) override; + + Status EoeReceived(int32_t) override; + + // Print function for Zip + // @param out output stream to print to + // @param show_all if it should print everything + void Print(std::ostream &out, bool show_all) const override; + + // Provide stream operator for displaying it + friend std::ostream &operator<<(std::ostream &out, const ZipOp &zo) { + zo.Print(out, false); + return out; + } + + // Class functor operator () override. + // All dataset ops operate by launching a thread (see ExecutionTree). This class functor will + // provide the master loop that drives the logic for performing the work + // @return Status - The error code return + Status operator()() override; + + private: + // Handles preprocessing of the main loop, used when starting new epoch + Status prepare(TensorQTable *const table); + + // This function calls takes a table repeatedly adds rows to it. + // @param table a table of tensors to be moved into a buffer + Status fillBuffer(TensorQTable *const table); + + // Special handle case where an empty row has been received from child iterator + // @note we need to drain eoe signals from all children connectors. + // @details when this function is called, then we encountered eoe at child iterator + // we have to drain rows from other child iterators until we hit eoe from all other child iterators + Status drainPipeline(); + + // Merges 1 row from each childIterator together + // @param new_zip_row input and output, will return a non-empty row if all rows from childConnectors are non-empty + // @param updateColumnMapping generates a new column name to index mapping (mColNameIdMap) if set to true + // @details merge rows from iterator together. This is the main functionality for ZipOp + // this function takes one row and fills it with tensors from rows fetched + // from childIterators. + // @example: + // Zips multiple rows at a time, the output is store in newZipRow + // 1 a T + // \ | / + // 1, a, T + Status getNextTensorRow(TensorRow *const new_zip_row); + + int32_t children_num_; + int32_t rows_per_buffer_; + int32_t buffer_id_; + bool draining_; + bool eof_; + std::unordered_map col_name_id_map_; + std::vector> child_iterators_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DATASETOPS_ZIP_OP_H_ diff --git a/mindspore/ccsrc/dataset/engine/db_connector.h b/mindspore/ccsrc/dataset/engine/db_connector.h new file mode 100644 index 00000000000..243e77e6ac8 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/db_connector.h @@ -0,0 +1,97 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_DB_CONNECTOR_H_ +#define DATASET_ENGINE_DB_CONNECTOR_H_ + +#include +#include +#include "dataset/engine/connector.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/core/constants.h" + +namespace mindspore { +namespace dataset { +// DbConnector is a derived class from Connector with added logic to handle EOE and EOF. +// The Connector class itself is responsible to ensure deterministic order on every run. +class DbConnector : public Connector> { + public: + // Constructor of DbConnector + // @note DbConnector will create internal N number of blocking queues, where N = nProducers. + // See Connector.h for more details. + // @param n_producers The number of threads producing data into this DbConnector. + // @param n_consumers The number of thread consuming data from this DbConnector. + // @param queue_capacity The number of element (DataBuffer) for each internal queue. + DbConnector(int32_t n_producers, int32_t n_consumers, int32_t queue_capacity) + : Connector>(n_producers, n_consumers, queue_capacity), end_of_file_(false) {} + + // Destructor of DbConnector + ~DbConnector() = default; + + // Add a unique_ptr into the DbConnector. + // @note The caller of this add method should use std::move to pass the ownership to DbConnector. + // @param worker_id The id of a worker thread calling this method. + // @param el A rvalue reference to an element to be passed/added/pushed. + Status Add(int32_t worker_id, std::unique_ptr &&el) noexcept { + return (Connector>::Push(worker_id, std::move(el))); + } + + // Get a unique_ptr from the DbConnector. + // @note After the first EOF Buffer is encountered, subsequent pop()s will return EOF Buffer. + // This will provide/propagate the EOF to all consumer threads of this Connector. + // Thus, When the num_consumers < num_producers, there will be extra EOF messages in some of the internal queues + // and reset() must be called before reusing DbConnector. + // @param worker_id The id of a worker thread calling this method. + // @param result The address of a unique_ptr where the popped element will be placed. + // @param retry_if_eoe A flag to allow the same thread invoke pop() again if the current pop returns eoe buffer. + Status PopWithRetry(int32_t worker_id, std::unique_ptr *result, bool retry_if_eoe = false) noexcept { + if (result == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "[ERROR] nullptr detected when getting data from db connector"); + } else { + std::unique_lock lk(m_); + RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); + // Once an EOF message is encountered this flag will be set and we can return early. + if (end_of_file_) { + *result = mindspore::make_unique(0, DataBuffer::kDeBFlagEOF); + } else { + RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); + if (*result == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "[ERROR] nullptr detected when getting data from db connector"); + } + // Setting the internal flag once the first EOF is encountered. + if ((*result)->eof()) { + end_of_file_ = true; + } + pop_from_ = (pop_from_ + 1) % num_producers_; + } + // Do not increment expect_consumer_ when result is eoe and retry_if_eoe is set. + if (!((*result)->eoe() && retry_if_eoe)) { + expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; + } + } + cv_.NotifyAll(); + return Status::OK(); + } + + private: + // A flag to indicate the end of stream has been encountered. + bool end_of_file_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_DB_CONNECTOR_H_ diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc new file mode 100644 index 00000000000..3dbeaa5ed1e --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -0,0 +1,205 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/execution_tree.h" +#include +#include +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/engine/datasetops/shuffle_op.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +// Constructor +ExecutionTree::ExecutionTree() : id_count_(0) { + tg_ = mindspore::make_unique(); + tree_state_ = kDeTStateInit; + prepare_flags_ = kDePrepNone; +} + +// Destructor +ExecutionTree::~ExecutionTree() { (void)tg_->ServiceStop(); } + +// Associates a DatasetOp with this tree. This assigns a valid node id to the operator and +// provides it with a link to the tree. A node cannot form any relationships (parent/child) with +// other nodes unless they are associated with the same tree. +Status ExecutionTree::AssociateNode(const std::shared_ptr &op) { + if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding) { + std::string err_msg = + "Invalid tree state for adding a node. Current state: " + std::to_string(static_cast(tree_state_)) + + " Expected states: " + std::to_string(static_cast(kDeTStateInit)) + " or " + + std::to_string(static_cast(kDeTStateBuilding)); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // Enter the building state if we were not already there + tree_state_ = kDeTStateBuilding; + + // Assign an id to the operator + op->set_id(id_count_); + id_count_++; + + // Assign our tree into the op so that each op has a link back to the tree + op->set_tree(this); + return Status::OK(); +} + +// Sets the root node of the tree +Status ExecutionTree::AssignRoot(const std::shared_ptr &op) { + // Tree must be in building state before we can assign root to it + if (tree_state_ != kDeTStateBuilding) { + std::string err_msg = + "Invalid tree state for assigning a root node. Current state: " + std::to_string(static_cast(tree_state_)) + + " Expected state: " + std::to_string(static_cast(kDeTStateBuilding)); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // If they didn't already call AssociateNode for this node before calling AssignRoot, + // then do so now. + if (op->operator_id_ == DatasetOp::kInvalidOperatorId) { + RETURN_IF_NOT_OK(this->AssociateNode(op)); + } + + // Then add it as the root. + root_ = op; + + // The tree has an assigned root now and it's ready to be prepared. + tree_state_ = kDeTStatePrepare; + return Status::OK(); +} + +// A print method typically used for debugging +void ExecutionTree::Print(std::ostream &out, bool show_all) const { + out << "Total number of nodes in the ExecutionTree (may or may not be connected nodes): " << id_count_ + << "\nTree state: " << static_cast(tree_state_) << "\n"; + if (root_ != nullptr) { + // Just call the printer on the root node. Each node descends to it's children to print them if + // showAll is true. + root_->Print(out, show_all); + } +} + +// Start the execution of the tree +Status ExecutionTree::Launch() { + // Tree must be built and prepared before it can be launched! + if (tree_state_ != kDeTStateReady) { + std::string err_msg = + "Invalid tree state for launching tree. Current state: " + std::to_string(static_cast(tree_state_)) + + " Expected state: " + std::to_string(static_cast(kDeTStateReady)); + RETURN_STATUS_UNEXPECTED(err_msg); + } + for (auto itr = this->begin(); itr != this->end(); ++itr) { + // An inlined operator is one that has an output connector size of 0, and it does not + // require a thread to execute. Instead, the work of this operator is executed inlined + // from the tree node directly above it (or in the case of a root node, it runs from within + // the launching tree/user thread. Do not exec any thread for an inlined op. + itr->state_ = DatasetOp::OpState::kDeOpRunning; + if (!itr->inlined()) { + RETURN_IF_NOT_OK(tg_->CreateAsyncTask("Op launched, OperatorId:" + std::to_string(itr->id()), std::ref(*itr))); + // Set the state of the Operator as running. This only matters in Leaf ops, CacheOp and TakeOp + } + } + tree_state_ = kDeTStateExecuting; + return Status::OK(); +} + +// A function that traverse the tree in postorder then save the results in nodes +void ExecutionTree::Iterator::PostOrderTraverse(const std::shared_ptr &node) { + if (node == nullptr) { + return; + } + for (int32_t i = 0; i < node->child_.size(); ++i) { + PostOrderTraverse(node->child_[i]); + } + nodes_.push_back(node); +} + +ExecutionTree::Iterator::Iterator(const std::shared_ptr &root) : ind_(0) { + // post-order traverse the tree, if root is null, it return + PostOrderTraverse(root); + nodes_.emplace_back(nullptr); +} + +// Given the number of workers, launches the worker entry function for each. Essentially a +// wrapper for the TaskGroup handling that is stored inside the execution tree. +Status ExecutionTree::LaunchWorkers(int32_t num_workers, std::function func) { + // Launch the workers + for (int32_t i = 0; i < num_workers; ++i) { + RETURN_IF_NOT_OK(tg_->CreateAsyncTask("Parallel Op Worker", std::bind(func, i))); + } + return Status::OK(); +} + +// The driver of the prepare phase of the execution tree. The prepare phase will recursively +// walk the tree to perform modifications to the tree or specific nodes within the tree to get +// it ready for execution. +Status ExecutionTree::Prepare() { + // Tree must be in pending prepare state before we can assign root to it + if (tree_state_ != kDeTStatePrepare) { + std::string err_msg = + "Invalid tree state for preparing the tree. Current state: " + std::to_string(static_cast(tree_state_)) + + " Expected state: " + std::to_string(static_cast(kDeTStatePrepare)); + RETURN_STATUS_UNEXPECTED(err_msg); + } + // Start the recursive prepare + RETURN_IF_NOT_OK(this->PrepareNode(root_)); + tree_state_ = kDeTStateReady; + return Status::OK(); +} + +// Recursive function used during prepare phase to visit a node and drive any pre- and post- +// node actions during a tree walk. +Status ExecutionTree::PrepareNode(const std::shared_ptr &dataset_op) { + int32_t num_children = dataset_op->child_.size(); + + // Before going down into children, make any prepare flags updates based on this + // operator. + uint32_t op_prep_flags = dataset_op->PrepareFlags(); + // Sanity check. In future we can support nested repeats. for now it's not allowed. + // If somebody above us already set the repeat flag, and now we are another repeat... + if (BitTest(op_prep_flags, kDePrepRepeat) && BitTest(prepare_flags_, kDePrepRepeat)) { + std::string err_msg("Nested RepeatOp detected! This is not supported yet."); + RETURN_STATUS_UNEXPECTED(err_msg); + } + BitSet(&prepare_flags_, op_prep_flags); + + // Now, descend to children + for (int32_t i = 0; i < num_children; ++i) { + RETURN_IF_NOT_OK(this->PrepareNode(dataset_op->child_[i])); + } + + // No more children, now we execute any prepare actions before going back up the + // the tree on recursive function exit + RETURN_IF_NOT_OK(dataset_op->PrepareNodeAction()); + + // Then clear the flags from this op now that we have prepared it. + BitClear(&prepare_flags_, op_prep_flags); + return Status::OK(); +} + +// Adds an operator to the repeat stack during prepare phase. +void ExecutionTree::AddToRepeatStack(std::shared_ptr dataset_op) { repeat_stack_.push(dataset_op); } + +// Pops an operator from the repeat stack during prepare phase. +std::shared_ptr ExecutionTree::PopFromRepeatStack() { + std::shared_ptr top_op = nullptr; + if (!repeat_stack_.empty()) { + top_op = repeat_stack_.top(); + repeat_stack_.pop(); + } + return top_op; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.h b/mindspore/ccsrc/dataset/engine/execution_tree.h new file mode 100644 index 00000000000..0f6cdfc1652 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/execution_tree.h @@ -0,0 +1,191 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_EXECUTION_TREE_H_ +#define DATASET_ENGINE_EXECUTION_TREE_H_ + +#include +#include +#include +#include +#include "dataset/engine/datasetops/dataset_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Forward declares +class TaskGroup; +class DatasetOp; + +class ExecutionTree { + public: + // Prepare flags used during tree prepare phase + enum PrepareFlags { + kDePrepNone = 0, + kDePrepRepeat = 1 // Processing a repeat operation + }; + + // State flags for the lifecycle of the tree + enum TreeState { + kDeTStateInit = 0, // The freshly initialized state after construction + kDeTStateBuilding, // The tree is being built, nodes are being added + kDeTStatePrepare, // The tree has been assigned a root node and is pending prepare + kDeTStateReady, // The tree has been prepared and is ready to be launched + kDeTStateExecuting // The tree has been launched and is executing + }; + + class Iterator { + public: + // Constructor + // @param root The root node to start iterating from + explicit Iterator(const std::shared_ptr &root = nullptr); + + // Destructor + ~Iterator() {} + + Iterator &operator++() { + ++ind_; + return *this; + } // prefix ++ overload + Iterator operator++(int) { + Iterator it = *this; + it.ind_ = ind_; + ind_++; + return it; + } // post-fix ++ overload + Iterator &operator--() { + --ind_; + return *this; + } // prefix -- overload + Iterator operator--(int) { + Iterator it = *this; + it.ind_ = ind_; + ind_--; + return it; + } // post-fix -- overload + DatasetOp &operator*() { return *nodes_[ind_]; } // dereference operator + std::shared_ptr operator->() { return nodes_[ind_]; } + + // getter function + // @return Shared pointer to the current operator + std::shared_ptr get() { return nodes_[ind_]; } + + bool operator!=(const Iterator &rhs) { return nodes_[ind_] != rhs.nodes_[rhs.ind_]; } + + private: + int ind_; // the cur node our Iterator points to + std::vector> nodes_; // store the nodes in post order + void PostOrderTraverse(const std::shared_ptr &); + }; + + // Constructor + ExecutionTree(); + + // Destructor + ~ExecutionTree(); + + // Associates a DatasetOp with this tree. This assigns a valid node id to the operator and + // provides it with a link to the tree. A node cannot form any relationships (parent/child) with + // other nodes unless they are associated with the same tree. + // @param op - The operator to associate + // @return Status - The error code return + Status AssociateNode(const std::shared_ptr &op); + + // Sets the root node of the tree + // @param op - The operator to assign as root + // @return Status - The error code return + Status AssignRoot(const std::shared_ptr &op); + + // Start the execution of the tree + // @return Status - The error code return + Status Launch(); + + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const; + + // Returns an iterator positioned at the start + // @return Iterator - The iterator + ExecutionTree::Iterator begin(const std::shared_ptr &root = nullptr) const { + return Iterator((root == nullptr) ? root_ : root); + } + + // Returns an iterator positioned at the end + // @return Iterator - The iterator + ExecutionTree::Iterator end() const { return Iterator(nullptr); } + + // << Stream output operator overload + // @notes This allows you to write the debug print info using stream operators + // @param out - reference to the output stream being overloaded + // @param exe_tree - reference to the execution tree to display + // @return - the output stream must be returned + friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) { + exe_tree.Print(out, false); + return out; + } + + // Given the number of workers, launches the worker entry function for each. Essentially a + // wrapper for the TaskGroup handling that is stored inside the execution tree. + // @param num_workers - The number of workers to launch + // @param func - The function entry point that workers will execute + // @return Status - The error code return + Status LaunchWorkers(int32_t num_workers, std::function func); + + // Getter method + // @return shared_ptr to the root operator + std::shared_ptr root() const { return root_; } + + // Getter method + // @return the prepare flags + uint32_t PrepareFlags() const { return prepare_flags_; } + + // The driver of the prepare phase of the execution tree. The prepare phase will recursively + // walk the tree to perform modifications to the tree or specific nodes within the tree to get + // it ready for execution. + // @return Status - The error code return + Status Prepare(); + + // Recursive function used during prepare phase to visit a node and drive any pre- and post- + // node actions during a tree walk. + // @param op - The dataset op to work on + // @return Status - The error code return + Status PrepareNode(const std::shared_ptr &dataset_op); + + // Adds an operator to the repeat stack during prepare phase. + // @param op - The dataset op to work add to repeat stack + // @return Status - The error code return + void AddToRepeatStack(std::shared_ptr dataset_op); + + // Pops an operator from the repeat stack during prepare phase. + // @return shared_ptr to the popped operator + std::shared_ptr PopFromRepeatStack(); + + // Return the pointer to the TaskGroup + // @return raw pointer to the TaskGroup + TaskGroup *AllTasks() const { return tg_.get(); } + + private: + std::unique_ptr tg_; // Class for worker management + std::shared_ptr root_; // The root node of the tree + int32_t id_count_; // Counter for generating operator id's + uint32_t prepare_flags_; // Flags used during tree prepare + TreeState tree_state_; // Tracking the current tree state + std::stack> repeat_stack_; // A stack used during prepare phase +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_EXECUTION_TREE_H_ diff --git a/mindspore/ccsrc/dataset/engine/jagged_connector.h b/mindspore/ccsrc/dataset/engine/jagged_connector.h new file mode 100644 index 00000000000..5f9f255f2b5 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/jagged_connector.h @@ -0,0 +1,88 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_JAGGED_CONNECTOR_H_ +#define DATASET_ENGINE_JAGGED_CONNECTOR_H_ + +#include +#include +#include +#include +#include "dataset/engine/connector.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/util/status.h" +#include "dataset/core/constants.h" + +namespace mindspore { +namespace dataset { +class JaggedConnector : public Connector> { + public: + JaggedConnector(int32_t num_producers, int32_t num_consumers, int32_t queue_capacity) + : Connector>(num_producers, num_consumers, queue_capacity) { + for (int i = 0; i < num_producers; i++) { + is_queue_finished_.push_back(false); + } + } + + ~JaggedConnector() = default; + + Status Add(int32_t worker_d, std::unique_ptr &&element) noexcept { + return Connector>::Push(worker_d, std::move(element)); + } + + Status Pop(int32_t worker_id, std::unique_ptr *result) noexcept override { + { + DS_ASSERT(worker_id < num_consumers_); + std::unique_lock lock(m_); + RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; })); + if (is_queue_finished_[pop_from_]) { + std::string errMsg = "ERROR: popping from a finished queue in JaggedConnector"; + RETURN_STATUS_UNEXPECTED(errMsg); + } + + RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); + if ((*result)->eoe()) { + is_queue_finished_[pop_from_] = true; + } + + for (int offset = 1; offset <= num_producers_; offset++) { + int32_t nextQueueIndex = (pop_from_ + offset) % num_producers_; + if (is_queue_finished_[nextQueueIndex] == false) { + pop_from_ = nextQueueIndex; + break; + } + } + + expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; + } + + cv_.NotifyAll(); + return Status::OK(); + } + + void DoReset() { + for (int i = 0; i < is_queue_finished_.size(); i++) { + is_queue_finished_[i] = false; + } + + Connector>::Reset(); + } + + private: + std::vector is_queue_finished_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_JAGGED_CONNECTOR_H_ diff --git a/mindspore/ccsrc/dataset/engine/tdt/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/tdt/CMakeLists.txt new file mode 100644 index 00000000000..4a2adff3102 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/tdt/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(engine-tdt OBJECT + tdt_plugin.cc + ${EXAMPLE_SRCS} + ${FEATURE_SRCS} + ) + +add_dependencies(engine-tdt protobuf::libprotobuf) diff --git a/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc b/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc new file mode 100644 index 00000000000..27c408d681c --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc @@ -0,0 +1,121 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/engine/tdt/tdt_plugin.h" +#include "common/utils.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +static std::shared_ptr instance_ptr_ = nullptr; + +std::shared_ptr TdtPlugin::GetInstance() { + if (instance_ptr_ == nullptr) { + instance_ptr_ = std::shared_ptr(new TdtPlugin); + } + return instance_ptr_; +} + +TdtStatus TdtPlugin::hostPush(TensorRow ts_row, bool is_wait, std::string channel_name) { + MS_LOG(INFO) << "TDT channel name is " << channel_name << "."; + std::vector items; + auto ret = translate(ts_row, items); + if (ret != SUCCESS) { + MS_LOG(ERROR) << "TDT converting tensor failed!"; + return FAILED; + } + if (tdt::TdtHostPushData(channel_name, items) != 0) { + MS_LOG(ERROR) << "TDT pushing data failed!"; + return FAILED; + } + return SUCCESS; +} + +TdtStatus TdtPlugin::getTdtType(DataType d_type, std::string &datatype) { + switch (d_type.value()) { + case DataType::DE_BOOL: + datatype = "bool"; + break; + case DataType::DE_INT8: + datatype = "int8"; + break; + case DataType::DE_UINT8: + datatype = "uint8"; + break; + case DataType::DE_INT16: + datatype = "int16"; + break; + case DataType::DE_UINT16: + datatype = "uint16"; + break; + case DataType::DE_INT32: + datatype = "int32"; + break; + case DataType::DE_UINT32: + datatype = "uint32"; + break; + case DataType::DE_FLOAT16: + datatype = "float16"; + break; + case DataType::DE_FLOAT32: + datatype = "float32"; + break; + case DataType::DE_FLOAT64: + datatype = "float64"; + break; + case DataType::DE_INT64: + datatype = "int64"; + break; + case DataType::DE_UINT64: + datatype = "uint64"; + break; + default: + return FAILED; + } + return SUCCESS; +} + +TdtStatus TdtPlugin::translate(const TensorRow &ts_row, std::vector &items) { + if (ts_row.size() == 0) { + MS_LOG(ERROR) << "TDT the size of row is zero."; + return SUCCESS; + } + for (auto ts : ts_row) { + std::string datatype; + TdtStatus status = getTdtType(ts->type(), datatype); + if (status != SUCCESS) { + return status; + } + TensorShape tsShape = ts->shape(); + std::string dataShapes = "["; + for (auto dim : tsShape.AsVector()) { + (void)dataShapes.append(std::to_string(dim)).append(","); + } + dataShapes.pop_back(); + (void)dataShapes.append("]"); + DataItem data_item; + data_item.dataType_ = tdt::TDT_TENSOR; + data_item.tensorShape_ = dataShapes; + data_item.tensorType_ = datatype; + data_item.dataLen_ = ts->SizeInBytes(); + data_item.dataPtr_ = std::shared_ptr(reinterpret_cast(ts->StartAddr()), [](void *elem) {}); + items.emplace_back(data_item); + MS_LOG(INFO) << "TDT data type is " << datatype << ", data shape is " << dataShapes << ", data length is " + << ts->Size() << "."; + } + return SUCCESS; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.h b/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.h new file mode 100644 index 00000000000..a25deb4aab3 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.h @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_ENGINE_TDT_TDT_PLUGIN_H_ +#define DATASET_ENGINE_TDT_TDT_PLUGIN_H_ + +#include +#include +#include +#include +#include +#include +#include "tdt/tdt_host_interface.h" + +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" + +namespace mindspore { +namespace dataset { +enum TdtStatus { SUCCESS, FAILED }; + +using tdt::DataItem; + +class TdtPlugin { + public: + static std::shared_ptr GetInstance(); + + TdtStatus hostPush(TensorRow ts_row, bool is_wait, std::string channel_name); + + private: + TdtPlugin() {} + + TdtStatus getTdtType(DataType d_type, std::string &datatype); + + TdtStatus translate(const TensorRow &ts_row, std::vector &items); + + void *tdt_handle_ = nullptr; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_ENGINE_TDT_TDT_PLUGIN_H_ diff --git a/mindspore/ccsrc/dataset/kernels/CMakeLists.txt b/mindspore/ccsrc/dataset/kernels/CMakeLists.txt new file mode 100644 index 00000000000..4ffb279df5c --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/CMakeLists.txt @@ -0,0 +1,6 @@ +add_subdirectory(image) +add_subdirectory(data) +add_library(kernels OBJECT + py_func_op.cc + tensor_op.cc) +target_include_directories(kernels PRIVATE ${pybind11_INCLUDE_DIRS}) diff --git a/mindspore/ccsrc/dataset/kernels/data/CMakeLists.txt b/mindspore/ccsrc/dataset/kernels/data/CMakeLists.txt new file mode 100644 index 00000000000..a654de381ae --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/CMakeLists.txt @@ -0,0 +1,6 @@ +add_library(kernels-data OBJECT + data_utils.cc + one_hot_op.cc + type_cast_op.cc + to_float16_op.cc + ) diff --git a/mindspore/ccsrc/dataset/kernels/data/data_utils.cc b/mindspore/ccsrc/dataset/kernels/data/data_utils.cc new file mode 100644 index 00000000000..f1b190578a5 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/data_utils.cc @@ -0,0 +1,224 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/kernels/data/data_utils.h" +#include +#include "dataset/core/constants.h" +#include "dataset/core/tensor.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/core/data_type.h" +#include "dataset/core/pybind_support.h" + +namespace mindspore { +namespace dataset { +Status OneHotEncodingUnsigned(const std::shared_ptr &input, std::shared_ptr *output, + dsize_t num_classes, int64_t index) { + uint64_t class_idx; + if (input->Rank() == 0) { + RETURN_IF_NOT_OK(input->GetItemAt(&class_idx, {})); + } else { + RETURN_IF_NOT_OK(input->GetItemAt(&class_idx, {index})); + } + if (class_idx >= static_cast(num_classes)) { + RETURN_STATUS_UNEXPECTED("One_hot index values are not in range"); + } + if (input->type() == DataType::DE_UINT64) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else if (input->type() == DataType::DE_UINT32) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else if (input->type() == DataType::DE_UINT16) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else if (input->type() == DataType::DE_UINT8) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else { + RETURN_STATUS_UNEXPECTED("One hot unsigned only supports unsigned int as input."); + } + return Status::OK(); +} + +Status OneHotEncodingSigned(const std::shared_ptr &input, std::shared_ptr *output, dsize_t num_classes, + int64_t index) { + int64_t class_idx; + if (input->Rank() == 0) { + RETURN_IF_NOT_OK(input->GetItemAt(&class_idx, {})); + } else { + RETURN_IF_NOT_OK(input->GetItemAt(&class_idx, {index})); + } + if (class_idx >= static_cast(num_classes)) { + RETURN_STATUS_UNEXPECTED("One_hot index values are not in range"); + } + if (input->type() == DataType::DE_INT64) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else if (input->type() == DataType::DE_INT32) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else if (input->type() == DataType::DE_INT16) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else if (input->type() == DataType::DE_INT8) { + RETURN_IF_NOT_OK((*output)->SetItemAt({index, static_cast(class_idx)}, 1)); + } else { + RETURN_STATUS_UNEXPECTED("One hot signed only supports signed int as input."); + } + return Status::OK(); +} + +Status OneHotEncoding(std::shared_ptr input, std::shared_ptr *output, dsize_t num_classes) { + input->Squeeze(); + if (input->Rank() > 1) { // We expect the input to be int he first dimension + RETURN_STATUS_UNEXPECTED("One hot only supports scalars or 1D shape Tensors."); + } + if (!input->type().IsInt()) { + RETURN_STATUS_UNEXPECTED("One hot does not support input of this type."); + } + try { + dsize_t num_elements = 1; + if (input->Rank() == 1) num_elements = input->shape()[0]; + TensorShape out_shape({num_elements, num_classes}); + std::shared_ptr out; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&out, TensorImpl::kFlexible, out_shape, input->type())); + RETURN_IF_NOT_OK(out->Zero()); + for (dsize_t i = 0; i < num_elements; ++i) { + if (input->type().IsUnsignedInt()) { + RETURN_IF_NOT_OK(OneHotEncodingUnsigned(input, &out, num_classes, i)); + } else { + RETURN_IF_NOT_OK(OneHotEncodingSigned(input, &out, num_classes, i)); + } + } + out->Squeeze(); + *output = out; + return Status::OK(); + } catch (const std::exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in OneHotOp"); + } +} + +template +void Cast(const std::shared_ptr &input, std::shared_ptr *output) { + auto in_itr = input->begin(); + auto out_itr = (*output)->begin(); + auto out_end = (*output)->end(); + for (; out_itr != out_end; static_cast(in_itr++), static_cast(out_itr++)) + *out_itr = static_cast(*in_itr); +} + +template +void CastFrom(const std::shared_ptr &input, std::shared_ptr *output) { + switch ((*output)->type().value()) { + case DataType::DE_BOOL: + Cast(input, output); + break; + case DataType::DE_INT8: + Cast(input, output); + break; + case DataType::DE_UINT8: + Cast(input, output); + break; + case DataType::DE_INT16: + Cast(input, output); + break; + case DataType::DE_UINT16: + Cast(input, output); + break; + case DataType::DE_INT32: + Cast(input, output); + break; + case DataType::DE_UINT32: + Cast(input, output); + break; + case DataType::DE_INT64: + Cast(input, output); + break; + case DataType::DE_UINT64: + Cast(input, output); + break; + case DataType::DE_FLOAT16: + Cast(input, output); + break; + case DataType::DE_FLOAT32: + Cast(input, output); + break; + case DataType::DE_FLOAT64: + Cast(input, output); + break; + case DataType::DE_UNKNOWN: + MS_LOG(ERROR) << "Unknown data type."; + break; + } +} + +// Type cast operator +Status TypeCast(const std::shared_ptr &input, std::shared_ptr *output, const DataType &data_type) { + RETURN_IF_NOT_OK(Tensor::CreateTensor(output, TensorImpl::kFlexible, input->shape(), data_type)); + + static_cast((*output)->StartAddr()); + switch (input->type().value()) { + case DataType::DE_BOOL: + CastFrom(input, output); + break; + case DataType::DE_INT8: + CastFrom(input, output); + break; + case DataType::DE_UINT8: + CastFrom(input, output); + break; + case DataType::DE_INT16: + CastFrom(input, output); + break; + case DataType::DE_UINT16: + CastFrom(input, output); + break; + case DataType::DE_INT32: + CastFrom(input, output); + break; + case DataType::DE_UINT32: + CastFrom(input, output); + break; + case DataType::DE_INT64: + CastFrom(input, output); + break; + case DataType::DE_UINT64: + CastFrom(input, output); + break; + case DataType::DE_FLOAT16: + CastFrom(input, output); + break; + case DataType::DE_FLOAT32: + CastFrom(input, output); + break; + case DataType::DE_FLOAT64: + CastFrom(input, output); + break; + case DataType::DE_UNKNOWN: + // sanity check, unreachable code. + RETURN_STATUS_UNEXPECTED("TypeCast does not support input of this type."); + } + return Status::OK(); +} + +Status ToFloat16(const std::shared_ptr &input, std::shared_ptr *output) { + // initiate new tensor for type cast + DataType new_type = DataType("float16"); + RETURN_IF_NOT_OK(Tensor::CreateTensor(output, TensorImpl::kFlexible, input->shape(), new_type)); + static_cast((*output)->StartAddr()); + + auto in_itr = input->begin(); + auto out_itr = (*output)->begin(); + auto out_end = (*output)->end(); + for (; out_itr != out_end; in_itr++, out_itr++) *out_itr = Eigen::half(*in_itr); + + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/data/data_utils.h b/mindspore/ccsrc/dataset/kernels/data/data_utils.h new file mode 100644 index 00000000000..2bd168a1fec --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/data_utils.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_DATA_DATA_UTILS_H_ +#define DATASET_KERNELS_DATA_DATA_UTILS_H_ + +#include +#include +#include "dataset/core/constants.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" + +namespace mindspore { +namespace dataset { +// Returns Onehot encoding of the input tensor. +// Example: if input=2 and numClasses=3, the output is [0 0 1]. +// @param input: Tensor has type DE_UINT64, the non-one hot values are stored +// along the first dimensions or rows.. +// If the rank of input is not 1 or the type is not DE_UINT64, +// then it will fail. +// @param output: Tensor. The shape of the output tensor is +// and the type is same as input. +// @param num_classes: Number of classes to. +Status OneHotEncoding(std::shared_ptr input, std::shared_ptr *output, dsize_t num_classes); + +Status OneHotEncodingUnsigned(const std::shared_ptr &input, std::shared_ptr *output, + dsize_t num_classes, int64_t index); + +Status OneHotEncodingSigned(const std::shared_ptr &input, std::shared_ptr *output, dsize_t num_classes, + int64_t index); + +// Returns a type changed input tensor. +// Example: if input tensor is float64, the output will the specified dataType. See DataTypes.cpp +// @param input Tensor +// @param output Tensor. The shape of the output tensor is same as input with the type changed. +// @param data_type: type of data to cast data to +// @note: this operation will do a memcpy and if the value is truncated then precision will be lost + +template +void CastFrom(const std::shared_ptr &input, std::shared_ptr *output); + +template +void Cast(const std::shared_ptr &input, std::shared_ptr *output); + +Status ToFloat16(const std::shared_ptr &input, std::shared_ptr *output); + +Status TypeCast(const std::shared_ptr &input, std::shared_ptr *output, const DataType &data_type); +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_DATA_DATA_UTILS_H_ diff --git a/mindspore/ccsrc/dataset/kernels/data/one_hot_op.cc b/mindspore/ccsrc/dataset/kernels/data/one_hot_op.cc new file mode 100644 index 00000000000..65d1a183b33 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/one_hot_op.cc @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/data/one_hot_op.h" + +#include "dataset/core/tensor.h" +#include "dataset/kernels/data/data_utils.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +Status OneHotOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + Status s = OneHotEncoding(input, output, num_classes_); + return s; +} + +Status OneHotOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + std::vector inputs_copy; + inputs_copy.push_back(inputs[0].Squeeze()); + if (inputs_copy[0].Rank() == 0) outputs.emplace_back(std::vector{num_classes_}); + if (inputs_copy[0].Rank() == 1) outputs.emplace_back(std::vector{inputs_copy[0][0], num_classes_}); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/data/one_hot_op.h b/mindspore/ccsrc/dataset/kernels/data/one_hot_op.h new file mode 100644 index 00000000000..80494dc5c01 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/one_hot_op.h @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_DATA_ONE_HOT_OP_H_ +#define DATASET_KERNELS_DATA_ONE_HOT_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +class OneHotOp : public TensorOp { + public: + explicit OneHotOp(int num_classes) : num_classes_(num_classes) {} + + ~OneHotOp() override = default; + + void Print(std::ostream &out) const override { out << "OneHotOp"; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + private: + int num_classes_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_KERNELS_DATA_ONE_HOT_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/data/to_float16_op.cc b/mindspore/ccsrc/dataset/kernels/data/to_float16_op.cc new file mode 100644 index 00000000000..1cd79456e05 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/to_float16_op.cc @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/data/to_float16_op.h" +#include "dataset/core/tensor.h" +#include "dataset/kernels/data/data_utils.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +Status ToFloat16Op::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + return ToFloat16(input, output); +} +Status ToFloat16Op::OutputType(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs)); + outputs[0] = DataType(DataType::DE_FLOAT16); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/data/to_float16_op.h b/mindspore/ccsrc/dataset/kernels/data/to_float16_op.h new file mode 100644 index 00000000000..3fca50bf07f --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/to_float16_op.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDDATA_TOFLOAT16OP_H +#define MINDDATA_TOFLOAT16OP_H + +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +class ToFloat16Op : public TensorOp { + public: + ToFloat16Op() = default; + + ~ToFloat16Op() override = default; + + // Overrides the base class compute function + // Calls the ToFloat16 function in ImageUtils, this function takes an input tensor + // and transforms its data to float16, the output memory is manipulated to contain the result + // @return Status - The error code return + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + void Print(std::ostream &out) const override { out << "ToFloat16Op"; } + + Status OutputType(const std::vector &inputs, std::vector &outputs) override; +}; +} // namespace dataset +} // namespace mindspore + +#endif // MINDDATA_TOFLOAT16OP_H diff --git a/mindspore/ccsrc/dataset/kernels/data/type_cast_op.cc b/mindspore/ccsrc/dataset/kernels/data/type_cast_op.cc new file mode 100644 index 00000000000..74c84a668ab --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/type_cast_op.cc @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/data/type_cast_op.h" + +#include "dataset/core/tensor.h" +#include "dataset/kernels/data/data_utils.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +TypeCastOp::TypeCastOp(const DataType &new_type) : type_(new_type) {} + +TypeCastOp::TypeCastOp(const std::string &data_type) { type_ = DataType(data_type); } + +Status TypeCastOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + return TypeCast(input, output, type_); +} +Status TypeCastOp::OutputType(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs)); + outputs[0] = type_; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/data/type_cast_op.h b/mindspore/ccsrc/dataset/kernels/data/type_cast_op.h new file mode 100644 index 00000000000..1b3f2c3290c --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/data/type_cast_op.h @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_DATA_TYPE_CAST_OP_H_ +#define DATASET_KERNELS_DATA_TYPE_CAST_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +class TypeCastOp : public TensorOp { + public: + // Constructor for TypecastOp + // @param data_type datatype to cast to + explicit TypeCastOp(const DataType &data_type); + + // Constructor for TypecastOp + // @param data_type datatype to cast to + explicit TypeCastOp(const std::string &data_type); + + ~TypeCastOp() override = default; + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + void Print(std::ostream &out) const override { out << "TypeCastOp"; } + Status OutputType(const std::vector &inputs, std::vector &outputs) override; + + private: + DataType type_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_DATA_TYPE_CAST_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt new file mode 100644 index 00000000000..7a6240d1732 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt @@ -0,0 +1,22 @@ +add_library(kernels-image OBJECT + center_crop_op.cc + change_mode_op.cc + cut_out_op.cc + decode_op.cc + distort_bounding_box_crop_op.cc + hwc_to_chw_op.cc + image_utils.cc + normalize_op.cc + pad_op.cc + random_color_adjust_op.cc + random_crop_decode_resize_op.cc + random_crop_and_resize_op.cc + random_crop_op.cc + random_horizontal_flip_op.cc + random_resize_op.cc + random_rotation_op.cc + random_vertical_flip_op.cc + rescale_op.cc + resize_bilinear_op.cc + resize_op.cc + ) diff --git a/mindspore/ccsrc/dataset/kernels/image/center_crop_op.cc b/mindspore/ccsrc/dataset/kernels/image/center_crop_op.cc new file mode 100644 index 00000000000..a5129e9c713 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/center_crop_op.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/center_crop_op.h" +#include +#include "common/utils.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t CenterCropOp::kDefWidth = 0; + +Status CenterCropOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + std::string err_msg; + dsize_t rank = input->shape().Rank(); + err_msg += (rank < 2 || rank > 3) ? "Rank received::" + std::to_string(rank) + " Expected: 2 or 3 \t" : ""; + err_msg += (crop_het_ <= 0 || crop_wid_ <= 0) ? "crop size needs to be positive integers\t" : ""; + + if (err_msg.length() != 0) RETURN_STATUS_UNEXPECTED(common::SafeCStr(err_msg)); + + int32_t top = crop_het_ - input->shape()[0]; // number of pixels to pad (top and bottom) + int32_t left = crop_wid_ - input->shape()[1]; + std::shared_ptr pad_image; + if (top > 0 && left > 0) { // padding only + return Pad(input, output, top / 2 + top % 2, top / 2, left / 2 + left % 2, left / 2, BorderType::kConstant); + } else if (top > 0) { + RETURN_IF_NOT_OK(Pad(input, &pad_image, top / 2 + top % 2, top / 2, 0, 0, BorderType::kConstant)); + return Crop(pad_image, output, (static_cast(pad_image->shape()[1]) - crop_wid_) / 2, + (static_cast(pad_image->shape()[0]) - crop_het_) / 2, crop_wid_, crop_het_); + } else if (left > 0) { + RETURN_IF_NOT_OK(Pad(input, &pad_image, 0, 0, left / 2 + left % 2, left / 2, BorderType::kConstant)); + return Crop(pad_image, output, (static_cast(pad_image->shape()[1]) - crop_wid_) / 2, + (static_cast(pad_image->shape()[0]) - crop_het_) / 2, crop_wid_, crop_het_); + } + return Crop(input, output, (input->shape()[1] - crop_wid_) / 2, (input->shape()[0] - crop_het_) / 2, crop_wid_, + crop_het_); +} + +void CenterCropOp::Print(std::ostream &out) const { + out << "CenterCropOp: " + << "cropWidth: " << crop_wid_ << "cropHeight: " << crop_het_ << "\n"; +} +Status CenterCropOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape out = TensorShape{crop_het_, crop_wid_}; + if (inputs[0].Rank() == 2) outputs.emplace_back(out); + if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/center_crop_op.h b/mindspore/ccsrc/dataset/kernels/image/center_crop_op.h new file mode 100644 index 00000000000..eb8e71ba7c7 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/center_crop_op.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_CENTER_CROP_OP_H_ +#define DATASET_KERNELS_IMAGE_CENTER_CROP_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class CenterCropOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefWidth; + + explicit CenterCropOp(int32_t het, int32_t wid = kDefWidth) : crop_het_(het), crop_wid_(wid == 0 ? het : wid) {} + + ~CenterCropOp() override = default; + + void Print(std::ostream &out) const override; + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + private: + int32_t crop_het_; + int32_t crop_wid_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_CENTER_CROP_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/change_mode_op.cc b/mindspore/ccsrc/dataset/kernels/image/change_mode_op.cc new file mode 100644 index 00000000000..9007044a734 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/change_mode_op.cc @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/change_mode_op.h" + +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +Status ChangeModeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + MS_LOG(INFO) << "WARN_DEPRECATED: ChangeModeOp is disabled, colour mode has NOT been changed."; + *output = input; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/change_mode_op.h b/mindspore/ccsrc/dataset/kernels/image/change_mode_op.h new file mode 100644 index 00000000000..e43350146f9 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/change_mode_op.h @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_CHANGE_MODE_OP_H_ +#define DATASET_KERNELS_IMAGE_CHANGE_MODE_OP_H_ + +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class ChangeModeOp : public TensorOp { + public: + // Name: Print() + // Description: A function that prints info about the node + void Print(std::ostream &out) const override { out << "ChangeModeOp"; } + + // Name: Compute() + // Description: The input second and last dimensions are swapped + // i.e. NHWC becomes NCHW + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_KERNELS_IMAGE_CHANGE_MODE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/cut_out_op.cc b/mindspore/ccsrc/dataset/kernels/image/cut_out_op.cc new file mode 100644 index 00000000000..9327d785fc4 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/cut_out_op.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +#include "dataset/kernels/image/cut_out_op.h" + +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const bool CutOutOp::kDefRandomColor = false; +const uint8_t CutOutOp::kDefFillR = 0; +const uint8_t CutOutOp::kDefFillG = 0; +const uint8_t CutOutOp::kDefFillB = 0; + +// constructor +CutOutOp::CutOutOp(int32_t box_height, int32_t box_width, int32_t num_patches, bool random_color, uint8_t fill_r, + uint8_t fill_g, uint8_t fill_b) + : box_height_(box_height), + box_width_(box_width), + num_patches_(num_patches), + random_color_(random_color), + fill_r_(fill_r), + fill_g_(fill_g), + fill_b_(fill_b) {} + +// main function call for cut out +Status CutOutOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + std::shared_ptr inputCV = CVTensor::AsCVTensor(input); + // cut out will clip the erasing area if the box is near the edge of the image and the boxes are black + RETURN_IF_NOT_OK( + Erase(inputCV, output, box_height_, box_width_, num_patches_, false, random_color_, fill_r_, fill_g_, fill_b_)); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/cut_out_op.h b/mindspore/ccsrc/dataset/kernels/image/cut_out_op.h new file mode 100644 index 00000000000..9a76572a548 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/cut_out_op.h @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +#ifndef DATASET_KERNELS_IMAGE_CUT_OUT_OP_H_ +#define DATASET_KERNELS_IMAGE_CUT_OUT_OP_H_ + +#include +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class CutOutOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const bool kDefRandomColor; + static const uint8_t kDefFillR; + static const uint8_t kDefFillG; + static const uint8_t kDefFillB; + + // Constructor for CutOutOp + // @param box_height box height + // @param box_width box_width + // @param num_patches how many patches to erase from image + // @param random_color boolean value to indicate fill patch with random color + // @param fill_r R value for the color to fill patch with + // @param fill_g G value for the color to fill patch with + // @param fill_b B value for the color to fill patch with + // @note maybe using unsigned long int isn't the best here according to our coding rules + CutOutOp(int32_t box_height, int32_t box_width, int32_t num_patches, bool random_color = kDefRandomColor, + uint8_t fill_r = kDefFillR, uint8_t fill_g = kDefFillG, uint8_t fill_b = kDefFillB); + + ~CutOutOp() override = default; + + void Print(std::ostream &out) const override { + out << "CutOut:: box_height: " << box_height_ << " box_width: " << box_width_ << " num_patches: " << num_patches_; + } + + // Overrides the base class compute function + // Calls the erase function in ImageUtils, this function takes an input tensor + // and overwrites some of its data using openCV, the output memory is manipulated to contain the result + // @return Status - The error code return + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + private: + int32_t box_height_; + int32_t box_width_; + int32_t num_patches_; + bool random_color_; + uint8_t fill_r_; + uint8_t fill_g_; + uint8_t fill_b_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_CUT_OUT_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/decode_op.cc b/mindspore/ccsrc/dataset/kernels/image/decode_op.cc new file mode 100644 index 00000000000..e3cfed7aa25 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/decode_op.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/decode_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const bool DecodeOp::kDefRgbFormat = true; + +DecodeOp::DecodeOp(bool is_rgb_format) : is_rgb_format_(is_rgb_format) { + if (is_rgb_format_) { // RGB colour mode + MS_LOG(INFO) << "Decode colour mode is RGB."; + } else { + MS_LOG(INFO) << "Decode colour mode is BGR."; + } +} + +Status DecodeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + if (is_rgb_format_) { // RGB colour mode + return Decode(input, output); + } else { // BGR colour mode + RETURN_STATUS_UNEXPECTED("Decode BGR is deprecated"); + } +} +Status DecodeOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape out({-1, -1, 3}); // we don't know what is output image size, but we know it should be 3 channels + if (inputs[0].Rank() == 1) outputs.emplace_back(out); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} + +Status DecodeOp::OutputType(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs)); + outputs[0] = DataType(DataType::DE_UINT8); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/decode_op.h b/mindspore/ccsrc/dataset/kernels/image/decode_op.h new file mode 100644 index 00000000000..50d2d3cb680 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/decode_op.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_DECODE_OP_H_ +#define DATASET_KERNELS_IMAGE_DECODE_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class DecodeOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const bool kDefRgbFormat; + + explicit DecodeOp(bool is_rgb_format = true); + + ~DecodeOp() = default; + + Status Compute(const std::shared_ptr& input, std::shared_ptr* output) override; + + void Print(std::ostream& out) const override { out << "DecodeOp"; } + Status OutputShape(const std::vector& inputs, std::vector& outputs) override; + Status OutputType(const std::vector& inputs, std::vector& outputs) override; + + private: + bool is_rgb_format_ = true; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_DECODE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/distort_bounding_box_crop_op.cc b/mindspore/ccsrc/dataset/kernels/image/distort_bounding_box_crop_op.cc new file mode 100644 index 00000000000..e7a8cc34962 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/distort_bounding_box_crop_op.cc @@ -0,0 +1,117 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/distort_bounding_box_crop_op.h" +#include +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t DistortBoundingBoxCropOp::kDefMaxAttempts = 100; +const int32_t DistortBoundingBoxCropOp::kDefBoxGenAttempts = 10; + +DistortBoundingBoxCropOp::DistortBoundingBoxCropOp(float aspect_ratio, float intersect_ratio, float crop_ratio_lb, + float crop_ratio_ub, int32_t max_attempts, int32_t box_gen_attempts) + : max_attempts_(max_attempts), + box_gen_attempts_(box_gen_attempts), + aspect_ratio_(aspect_ratio), + intersect_ratio_(intersect_ratio), + crop_ratio_lb_(crop_ratio_lb), + crop_ratio_ub_(crop_ratio_ub) { + seed_ = GetSeed(); + rnd_.seed(seed_); +} + +Status DistortBoundingBoxCropOp::Compute(const std::vector>& input, + std::vector>* output) { + IO_CHECK_VECTOR(input, output); + if (input.size() != NumInput()) + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Number of inputs is not 5"); + + CHECK_FAIL_RETURN_UNEXPECTED(input[1]->shape().Size() >= 1, "The shape of the second tensor is abnormal"); + int64_t num_boxes = 0; + for (uint64_t i = 1; i < input.size(); i++) { + if (i == 1) num_boxes = input[i]->shape()[0]; + if (num_boxes != input[i]->shape()[0]) + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Numbers of boxes do not match"); + + if (input[i]->type() != DataType::DE_FLOAT32) + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "boxes' type is not DE_FLOAT21"); + } + + // assume input Tensor vector in the order of [img, bbox_y_min, bbox_y_max, bbox_x_min, bbox_x_max] + CHECK_FAIL_RETURN_UNEXPECTED(input[0]->shape().Size() >= 2, "The shape of the first tensor is abnormal"); + int h_in = input[0]->shape()[0]; + int w_in = input[0]->shape()[1]; + + std::vector bounding_boxes; + for (int64_t i = 0; i < num_boxes; ++i) { + // bbox coordinates are floats relative to the image width and height + float y_min, y_max, x_min, x_max; + RETURN_IF_NOT_OK(input[1]->GetItemAt(&y_min, {i})); + RETURN_IF_NOT_OK(input[2]->GetItemAt(&y_max, {i})); + RETURN_IF_NOT_OK(input[3]->GetItemAt(&x_min, {i})); + RETURN_IF_NOT_OK(input[4]->GetItemAt(&x_max, {i})); + bounding_boxes.emplace_back(static_cast(x_min * w_in), static_cast(y_min * h_in), + static_cast((x_max - x_min) * w_in), static_cast((y_max - y_min) * h_in)); + } + cv::Rect output_box; + bool should_crop = false; + + // go over iterations, if no satisfying box found we return the original image + for (int32_t t = 0; t < max_attempts_; ++t) { + // try to generate random box + RETURN_IF_NOT_OK(GenerateRandomCropBox(h_in, w_in, aspect_ratio_, crop_ratio_lb_, crop_ratio_ub_, + box_gen_attempts_, // int maxIter, should not be needed here + &output_box, seed_)); + RETURN_IF_NOT_OK(CheckOverlapConstraint(output_box, + bounding_boxes, // have to change, should take tensor or add bbox logic + intersect_ratio_, &should_crop)); + if (should_crop) { + // found a box to crop + break; + } + } + // essentially we have to check this again at the end to return original tensor + if (should_crop) { + std::shared_ptr out; + RETURN_IF_NOT_OK(Crop(input[0], &out, output_box.x, output_box.y, output_box.width, output_box.height)); + output->push_back(out); + } else { + output->push_back(input[0]); + } + return Status::OK(); +} + +Status DistortBoundingBoxCropOp::OutputShape(const std::vector& inputs, + std::vector& outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape out = TensorShape{-1, -1}; + if (inputs[0].Rank() == 2) outputs.emplace_back(out); + if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +Status DistortBoundingBoxCropOp::OutputType(const std::vector& inputs, std::vector& outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs)); + outputs[0] = inputs[0]; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/distort_bounding_box_crop_op.h b/mindspore/ccsrc/dataset/kernels/image/distort_bounding_box_crop_op.h new file mode 100644 index 00000000000..6d5dca99fb9 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/distort_bounding_box_crop_op.h @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_DISTORT_BOUNDING_BOX_CROP_OP_H_ +#define DATASET_KERNELS_IMAGE_DISTORT_BOUNDING_BOX_CROP_OP_H_ + +#include +#include +#include +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class DistortBoundingBoxCropOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefMaxAttempts; + static const int32_t kDefBoxGenAttempts; + + // Constructor for DistortBoundingBoxCropOp + // @param max_attempts tries before the crop happens + // @param box_gen_attempts crop box generation attempts + // @param aspect_ratio aspect ratio of the generated crop box + // @param intersect_ratio area overlap ratio, condition for crop only if area over lap between the generated + // crop box has sufficient overlap with any 1 bounding box + // @param crop_ratio_lb the crop ratio lower bound + // @param crop_ratio_ub the crop ratio upper bound + // @param seed + DistortBoundingBoxCropOp(float aspect_ratio, float intersect_ratio, float crop_ratio_lb, float crop_ratio_ub, + int32_t max_attempts = kDefMaxAttempts, int32_t box_gen_attempts = kDefBoxGenAttempts); + + ~DistortBoundingBoxCropOp() override = default; + + void Print(std::ostream& out) const override { + out << "DistortBoundingBoxCropOp: " << max_attempts_ << " " << intersect_ratio_; + } + + Status Compute(const std::vector>& input, + std::vector>* output) override; + + uint32_t NumInput() override { return 5; } + Status OutputShape(const std::vector& inputs, std::vector& outputs) override; + Status OutputType(const std::vector& inputs, std::vector& outputs) override; + + private: + int32_t max_attempts_; + int32_t box_gen_attempts_; + float aspect_ratio_; + float intersect_ratio_; + float crop_ratio_lb_; + float crop_ratio_ub_; + std::mt19937 rnd_; + uint32_t seed_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_DISTORT_BOUNDING_BOX_CROP_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/hwc_to_chw_op.cc b/mindspore/ccsrc/dataset/kernels/image/hwc_to_chw_op.cc new file mode 100644 index 00000000000..8ed2229cd1d --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/hwc_to_chw_op.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/hwc_to_chw_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +Status HwcToChwOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + // input.shape == HWC + // output.shape == CHW + return HwcToChw(input, output); +} +Status HwcToChwOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape in = inputs[0]; + TensorShape out = TensorShape{in[2], in[0], in[1]}; + if (inputs[0].Rank() == 3) outputs.emplace_back(out); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/hwc_to_chw_op.h b/mindspore/ccsrc/dataset/kernels/image/hwc_to_chw_op.h new file mode 100644 index 00000000000..825ffa4443c --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/hwc_to_chw_op.h @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_CHANNEL_SWAP_OP_H_ +#define DATASET_KERNELS_IMAGE_CHANNEL_SWAP_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class HwcToChwOp : public TensorOp { + public: + void Print(std::ostream &out) const override { out << "HwcToChw"; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_CHANNEL_SWAP_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc new file mode 100644 index 00000000000..0412f07636a --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc @@ -0,0 +1,787 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/image_utils.h" +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "dataset/core/constants.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/tensor.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/util/make_unique.h" +#include "dataset/util/random.h" + +#define MAX_INT_PRECISION 16777216 // float int precision is 16777216 +namespace mindspore { +namespace dataset { +int GetCVInterpolationMode(InterpolationMode mode) { + switch (mode) { + case InterpolationMode::kLinear: + return static_cast(cv::InterpolationFlags::INTER_LINEAR); + case InterpolationMode::kCubic: + return static_cast(cv::InterpolationFlags::INTER_CUBIC); + case InterpolationMode::kArea: + return static_cast(cv::InterpolationFlags::INTER_AREA); + case InterpolationMode::kNearestNeighbour: + return static_cast(cv::InterpolationFlags::INTER_NEAREST); + default: + return static_cast(cv::InterpolationFlags::INTER_LINEAR); + } +} + +int GetCVBorderType(BorderType type) { + switch (type) { + case BorderType::kConstant: + return static_cast(cv::BorderTypes::BORDER_CONSTANT); + case BorderType::kEdge: + return static_cast(cv::BorderTypes::BORDER_REPLICATE); + case BorderType::kReflect: + return static_cast(cv::BorderTypes::BORDER_REFLECT101); + case BorderType::kSymmetric: + return static_cast(cv::BorderTypes::BORDER_REFLECT); + default: + return static_cast(cv::BorderTypes::BORDER_CONSTANT); + } +} + +Status Flip(std::shared_ptr input, std::shared_ptr *output, int flip_code) { + std::shared_ptr input_cv = CVTensor::AsCVTensor(std::move(input)); + + std::shared_ptr output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + (void)output_cv->StartAddr(); + if (input_cv->mat().data) { + try { + cv::flip(input_cv->mat(), output_cv->mat(), flip_code); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in flip op."); + } + } else { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor, the input data is null"); + } +} + +Status HorizontalFlip(std::shared_ptr input, std::shared_ptr *output) { + return Flip(std::move(input), output, 1); +} + +Status VerticalFlip(std::shared_ptr input, std::shared_ptr *output) { + return Flip(std::move(input), output, 0); +} + +Status Resize(const std::shared_ptr &input, std::shared_ptr *output, int32_t output_height, + int32_t output_width, double fx, double fy, InterpolationMode mode) { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->Rank() != 2) { + RETURN_STATUS_UNEXPECTED("Input Tensor is not in shape of or "); + } + cv::Mat in_image = input_cv->mat(); + // resize image too large or too small + if (output_height == 0 || output_height > in_image.rows * 1000 || output_width == 0 || + output_width > in_image.cols * 1000) { + std::string err_msg = + "The resizing width or height 1) is too big, it's up to " + "1000 times the original image; 2) can not be 0."; + return Status(StatusCode::kShapeMisMatch, err_msg); + } + try { + TensorShape shape{output_height, output_width}; + if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + std::shared_ptr output_cv = std::make_shared(shape, input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + auto cv_mode = GetCVInterpolationMode(mode); + cv::resize(in_image, output_cv->mat(), cv::Size(output_width, output_height), fx, fy, cv_mode); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in image resize."); + } +} + +bool HasJpegMagic(const unsigned char *data, size_t data_size) { + const unsigned char *kJpegMagic = (unsigned char *)"\xFF\xD8\xFF"; + constexpr size_t kJpegMagicLen = 3; + return data_size >= kJpegMagicLen && memcmp(data, kJpegMagic, kJpegMagicLen) == 0; +} + +Status Decode(const std::shared_ptr &input, std::shared_ptr *output) { + if (input->StartAddr() == nullptr) { + RETURN_STATUS_UNEXPECTED("Tensor is nullptr"); + } + if (HasJpegMagic(input->StartAddr(), input->SizeInBytes())) { + return JpegCropAndDecode(input, output); + } else { + return DecodeCv(input, output); + } +} + +Status DecodeCv(const std::shared_ptr &input, std::shared_ptr *output) { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + try { + cv::Mat img_mat = cv::imdecode(input_cv->mat(), cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION); + if (img_mat.data == nullptr) { + std::string err = "Error in decoding\t"; + RETURN_STATUS_UNEXPECTED(err); + } + cv::cvtColor(img_mat, img_mat, static_cast(cv::COLOR_BGR2RGB)); + std::shared_ptr output_cv = std::make_shared(img_mat); + RETURN_UNEXPECTED_IF_NULL(output_cv); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in image Decode"); + } +} + +static void JpegInitSource(j_decompress_ptr cinfo) {} + +static boolean JpegFillInputBuffer(j_decompress_ptr cinfo) { + if (cinfo->src->bytes_in_buffer == 0) { + ERREXIT(cinfo, JERR_INPUT_EMPTY); + return FALSE; + } + return TRUE; +} + +static void JpegTermSource(j_decompress_ptr cinfo) {} + +static void JpegSkipInputData(j_decompress_ptr cinfo, int64_t jump) { + if (jump < 0) { + return; + } + if (static_cast(jump) > cinfo->src->bytes_in_buffer) { + cinfo->src->bytes_in_buffer = 0; + return; + } else { + cinfo->src->bytes_in_buffer -= jump; + cinfo->src->next_input_byte += jump; + } +} + +void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) { + cinfo->src = static_cast( + (*cinfo->mem->alloc_small)(reinterpret_cast(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); + cinfo->src->init_source = JpegInitSource; + cinfo->src->fill_input_buffer = JpegFillInputBuffer; + cinfo->src->skip_input_data = JpegSkipInputData; + cinfo->src->resync_to_restart = jpeg_resync_to_restart; + cinfo->src->term_source = JpegTermSource; + cinfo->src->bytes_in_buffer = datasize; + cinfo->src->next_input_byte = static_cast(data); +} + +static Status JpegReadScanlines(jpeg_decompress_struct *const cinfo, int max_scanlines_to_read, JSAMPLE *buffer, + int buffer_size, int crop_w, int crop_w_aligned, int offset, int stride) { + // scanlines will be read to this buffer first, must have the number + // of components equal to the number of components in the image + int64_t scanline_size = crop_w_aligned * cinfo->output_components; + std::vector scanline(scanline_size); + JSAMPLE *scanline_ptr = &scanline[0]; + while (cinfo->output_scanline < static_cast(max_scanlines_to_read)) { + int num_lines_read = jpeg_read_scanlines(cinfo, &scanline_ptr, 1); + if (cinfo->out_color_space == JCS_CMYK && num_lines_read > 0) { + for (int i = 0; i < crop_w; ++i) { + int cmyk_pixel = 4 * i + offset; + const int c = scanline_ptr[cmyk_pixel]; + const int m = scanline_ptr[cmyk_pixel + 1]; + const int y = scanline_ptr[cmyk_pixel + 2]; + const int k = scanline_ptr[cmyk_pixel + 3]; + int r, g, b; + if (cinfo->saw_Adobe_marker) { + r = (k * c) / 255; + g = (k * m) / 255; + b = (k * y) / 255; + } else { + r = (255 - c) * (255 - k) / 255; + g = (255 - m) * (255 - k) / 255; + b = (255 - y) * (255 - k) / 255; + } + buffer[3 * i + 0] = r; + buffer[3 * i + 1] = g; + buffer[3 * i + 2] = b; + } + } else if (num_lines_read > 0) { + int copy_status = memcpy_s(buffer, buffer_size, scanline_ptr + offset, stride); + if (copy_status != 0) { + jpeg_destroy_decompress(cinfo); + RETURN_STATUS_UNEXPECTED("memcpy failed"); + } + } else { + jpeg_destroy_decompress(cinfo); + std::string err_msg = "failed to read scanline"; + RETURN_STATUS_UNEXPECTED(err_msg); + } + buffer += stride; + buffer_size = buffer_size - stride; + } + return Status::OK(); +} + +static Status JpegSetColorSpace(jpeg_decompress_struct *cinfo) { + switch (cinfo->num_components) { + case 1: + // we want to output 3 components if it's grayscale + cinfo->out_color_space = JCS_RGB; + return Status::OK(); + case 3: + cinfo->out_color_space = JCS_RGB; + return Status::OK(); + case 4: + // Need to manually convert to RGB + cinfo->out_color_space = JCS_CMYK; + return Status::OK(); + default: + jpeg_destroy_decompress(cinfo); + std::string err_msg = "wrong number of components"; + RETURN_STATUS_UNEXPECTED(err_msg); + } +} + +void JpegErrorExitCustom(j_common_ptr cinfo) { + char jpeg_last_error_msg[JMSG_LENGTH_MAX]; + (*(cinfo->err->format_message))(cinfo, jpeg_last_error_msg); + throw std::runtime_error(jpeg_last_error_msg); +} + +Status JpegCropAndDecode(const std::shared_ptr &input, std::shared_ptr *output, int crop_x, int crop_y, + int crop_w, int crop_h) { + struct jpeg_decompress_struct cinfo; + auto DestroyDecompressAndReturnError = [&cinfo](const std::string &err) { + jpeg_destroy_decompress(&cinfo); + RETURN_STATUS_UNEXPECTED(err); + }; + struct JpegErrorManagerCustom jerr; + cinfo.err = jpeg_std_error(&jerr.pub); + jerr.pub.error_exit = JpegErrorExitCustom; + try { + jpeg_create_decompress(&cinfo); + JpegSetSource(&cinfo, input->StartAddr(), input->SizeInBytes()); + (void)jpeg_read_header(&cinfo, TRUE); + RETURN_IF_NOT_OK(JpegSetColorSpace(&cinfo)); + jpeg_calc_output_dimensions(&cinfo); + } catch (std::runtime_error &e) { + return DestroyDecompressAndReturnError(e.what()); + } + if (crop_x == 0 && crop_y == 0 && crop_w == 0 && crop_h == 0) { + crop_w = cinfo.output_width; + crop_h = cinfo.output_height; + } else if (crop_w == 0 || static_cast(crop_w + crop_x) > cinfo.output_width || crop_h == 0 || + static_cast(crop_h + crop_y) > cinfo.output_height) { + return DestroyDecompressAndReturnError("Crop window is not valid"); + } + const int mcu_size = cinfo.min_DCT_scaled_size; + unsigned int crop_x_aligned = (crop_x / mcu_size) * mcu_size; + unsigned int crop_w_aligned = crop_w + crop_x - crop_x_aligned; + try { + (void)jpeg_start_decompress(&cinfo); + jpeg_crop_scanline(&cinfo, &crop_x_aligned, &crop_w_aligned); + } catch (std::runtime_error &e) { + return DestroyDecompressAndReturnError(e.what()); + } + JDIMENSION skipped_scanlines = jpeg_skip_scanlines(&cinfo, crop_y); + // three number of output components, always convert to RGB and output + constexpr int kOutNumComponents = 3; + TensorShape ts = TensorShape({crop_h, crop_w, kOutNumComponents}); + auto output_tensor = std::make_shared(ts, DataType(DataType::DE_UINT8)); + const int buffer_size = output_tensor->SizeInBytes(); + JSAMPLE *buffer = static_cast(output_tensor->StartAddr()); + const int max_scanlines_to_read = skipped_scanlines + crop_h; + // stride refers to output tensor, which has 3 components at most + const int stride = crop_w * kOutNumComponents; + // offset is calculated for scanlines read from the image, therefore + // has the same number of components as the image + const int offset = (crop_x - crop_x_aligned) * cinfo.output_components; + RETURN_IF_NOT_OK( + JpegReadScanlines(&cinfo, max_scanlines_to_read, buffer, buffer_size, crop_w, crop_w_aligned, offset, stride)); + *output = output_tensor; + jpeg_destroy_decompress(&cinfo); + return Status::OK(); +} + +Status Rescale(const std::shared_ptr &input, std::shared_ptr *output, float rescale, float shift) { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + cv::Mat input_image = input_cv->mat(); + std::shared_ptr output_cv = std::make_shared(input_cv->shape(), DataType(DataType::DE_FLOAT32)); + RETURN_UNEXPECTED_IF_NULL(output_cv); + try { + input_image.convertTo(output_cv->mat(), CV_32F, rescale, shift); + *output = std::static_pointer_cast(output_cv); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in image rescale"); + } + return Status::OK(); +} + +Status Crop(const std::shared_ptr &input, std::shared_ptr *output, int x, int y, int w, int h) { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->Rank() != 2) { + RETURN_STATUS_UNEXPECTED("Shape not or "); + } + try { + TensorShape shape{h, w}; + if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + std::shared_ptr output_cv = std::make_shared(shape, input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + cv::Rect roi(x, y, w, h); + (input_cv->mat())(roi).copyTo(output_cv->mat()); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in crop."); + } +} + +Status HwcToChw(std::shared_ptr input, std::shared_ptr *output) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->shape().Size() != 3 && input_cv->shape()[2] != 3) { + RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels is not equal 3"); + } + cv::Mat output_img; + + int height = input_cv->shape()[0]; + int width = input_cv->shape()[1]; + int num_channels = input_cv->shape()[2]; + + auto output_cv = mindspore::make_unique(TensorShape{num_channels, height, width}, input_cv->type()); + for (int i = 0; i < num_channels; ++i) { + cv::Mat mat; + RETURN_IF_NOT_OK(output_cv->Mat({i}, &mat)); + cv::extractChannel(input_cv->mat(), mat, i); + } + *output = std::move(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in ChannelSwap."); + } +} + +Status SwapRedAndBlue(std::shared_ptr input, std::shared_ptr *output) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(std::move(input)); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->shape().Size() != 3 && input_cv->shape()[2] != 3) { + RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels is not equal 3"); + } + auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + cv::cvtColor(input_cv->mat(), output_cv->mat(), static_cast(cv::COLOR_BGR2RGB)); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in ChangeMode."); + } +} + +Status CropAndResize(const std::shared_ptr &input, std::shared_ptr *output, int x, int y, + int crop_height, int crop_width, int target_height, int target_width, InterpolationMode mode) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->Rank() != 2) { + RETURN_STATUS_UNEXPECTED("Ishape not or "); + } + // image too large or too small + if (crop_height == 0 || crop_width == 0 || target_height == 0 || target_height > crop_height * 1000 || + target_width == 0 || target_height > crop_width * 1000) { + std::string err_msg = + "The resizing width or height 1) is too big, it's up to " + "1000 times the original image; 2) can not be 0."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + cv::Rect roi(x, y, crop_width, crop_height); + auto cv_mode = GetCVInterpolationMode(mode); + cv::Mat cv_in = input_cv->mat(); + TensorShape shape{target_height, target_width}; + if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + std::shared_ptr cvt_out = std::make_shared(shape, input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(cvt_out); + cv::resize(cv_in(roi), cvt_out->mat(), cv::Size(target_width, target_height), 0, 0, cv_mode); + *output = std::static_pointer_cast(cvt_out); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in CropAndResize."); + } +} + +Status Rotate(const std::shared_ptr &input, std::shared_ptr *output, float fx, float fy, float degree, + InterpolationMode interpolation, bool expand, uint8_t fill_r, uint8_t fill_g, uint8_t fill_b) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + cv::Mat input_img = input_cv->mat(); + if (input_img.cols > (MAX_INT_PRECISION * 2) || input_img.rows > (MAX_INT_PRECISION * 2)) { + RETURN_STATUS_UNEXPECTED("Image too large center not precise"); + } + // default to center of image + if (fx == -1 && fy == -1) { + fx = (input_img.cols - 1) / 2.0; + fy = (input_img.rows - 1) / 2.0; + } + cv::Mat output_img; + cv::Scalar fill_color = cv::Scalar(fill_b, fill_g, fill_r); + // maybe don't use uint32 for image dimension here + cv::Point2f pc(fx, fy); + cv::Mat rot = cv::getRotationMatrix2D(pc, degree, 1.0); + std::shared_ptr output_cv; + if (!expand) { + // this case means that the shape doesn't change, size stays the same + // We may not need this memcpy if it is in place. + output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + // using inter_nearest to comply with python default + cv::warpAffine(input_img, output_cv->mat(), rot, input_img.size(), GetCVInterpolationMode(interpolation), + cv::BORDER_CONSTANT, fill_color); + } else { + // we resize here since the shape changes + // create a new bounding box with the rotate + cv::Rect2f bbox = cv::RotatedRect(cv::Point2f(), input_img.size(), degree).boundingRect2f(); + rot.at(0, 2) += bbox.width / 2.0 - input_img.cols / 2.0; + rot.at(1, 2) += bbox.height / 2.0 - input_img.rows / 2.0; + // use memcpy and don't compute the new shape since openCV has a rounding problem + cv::warpAffine(input_img, output_img, rot, bbox.size(), GetCVInterpolationMode(interpolation), + cv::BORDER_CONSTANT, fill_color); + output_cv = std::make_shared(output_img); + RETURN_UNEXPECTED_IF_NULL(output_cv); + } + *output = std::static_pointer_cast(output_cv); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in image rotation"); + } + return Status::OK(); +} + +Status Normalize(const std::shared_ptr &input, std::shared_ptr *output, + const std::shared_ptr &mean, const std::shared_ptr &std) { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (!(input_cv->mat().data && input_cv->Rank() == 3)) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + cv::Mat in_image = input_cv->mat(); + std::shared_ptr output_cv = std::make_shared(input_cv->shape(), DataType(DataType::DE_FLOAT32)); + RETURN_UNEXPECTED_IF_NULL(output_cv); + mean->Squeeze(); + if (mean->type() != DataType::DE_FLOAT32 || mean->Rank() != 1 || mean->shape()[0] != 3) { + std::string err_msg = "Mean tensor should be of size 3 and type float."; + return Status(StatusCode::kShapeMisMatch, err_msg); + } + std->Squeeze(); + if (std->type() != DataType::DE_FLOAT32 || std->Rank() != 1 || std->shape()[0] != 3) { + std::string err_msg = "Std tensor should be of size 3 and type float."; + return Status(StatusCode::kShapeMisMatch, err_msg); + } + try { + // NOTE: We are assuming the input image is in RGB and the mean + // and std are in RGB + cv::Mat rgb[3]; + cv::split(in_image, rgb); + for (uint8_t i = 0; i < 3; i++) { + float mean_c, std_c; + RETURN_IF_NOT_OK(mean->GetItemAt(&mean_c, {i})); + RETURN_IF_NOT_OK(std->GetItemAt(&std_c, {i})); + rgb[i].convertTo(rgb[i], CV_32F, 1.0 / std_c, (-mean_c / std_c)); + } + cv::merge(rgb, 3, output_cv->mat()); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in Normalize"); + } +} + +Status AdjustBrightness(const std::shared_ptr &input, std::shared_ptr *output, const float &alpha) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + cv::Mat input_img = input_cv->mat(); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->shape()[2] != 3) { + RETURN_STATUS_UNEXPECTED("Shape not or "); + } + auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + output_cv->mat() = input_img * alpha; + *output = std::static_pointer_cast(output_cv); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in adjust brightness"); + } + return Status::OK(); +} + +Status AdjustContrast(const std::shared_ptr &input, std::shared_ptr *output, const float &alpha) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + cv::Mat input_img = input_cv->mat(); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->shape()[2] != 3) { + RETURN_STATUS_UNEXPECTED("Shape not or "); + } + cv::Mat gray, output_img; + cv::cvtColor(input_img, gray, CV_RGB2GRAY); + int mean_img = static_cast(cv::mean(gray).val[0] + 0.5); + std::shared_ptr output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + output_img = cv::Mat::zeros(input_img.rows, input_img.cols, CV_8UC1); + output_img = output_img + mean_img; + cv::cvtColor(output_img, output_img, CV_GRAY2RGB); + output_cv->mat() = output_img * (1.0 - alpha) + input_img * alpha; + *output = std::static_pointer_cast(output_cv); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in adjust contrast"); + } + return Status::OK(); +} + +Status AdjustSaturation(const std::shared_ptr &input, std::shared_ptr *output, const float &alpha) { + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + cv::Mat input_img = input_cv->mat(); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->shape()[2] != 3) { + RETURN_STATUS_UNEXPECTED("Shape not or "); + } + auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + cv::Mat output_img = output_cv->mat(); + cv::Mat gray; + cv::cvtColor(input_img, gray, CV_RGB2GRAY); + cv::cvtColor(gray, output_img, CV_GRAY2RGB); + output_cv->mat() = output_img * (1.0 - alpha) + input_img * alpha; + *output = std::static_pointer_cast(output_cv); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in adjust saturation"); + } + return Status::OK(); +} + +Status AdjustHue(const std::shared_ptr &input, std::shared_ptr *output, const float &hue) { + if (hue > 0.5 || hue < -0.5) { + MS_LOG(ERROR) << "Hue factor is not in [-0.5, 0.5]."; + RETURN_STATUS_UNEXPECTED("hue_factor is not in [-0.5, 0.5]."); + } + try { + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + cv::Mat input_img = input_cv->mat(); + if (!input_cv->mat().data) { + RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); + } + if (input_cv->Rank() != 3 && input_cv->shape()[2] != 3) { + RETURN_STATUS_UNEXPECTED("Shape not or "); + } + auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); + RETURN_UNEXPECTED_IF_NULL(output_cv); + cv::Mat output_img; + cv::cvtColor(input_img, output_img, CV_RGB2HSV_FULL); + for (int y = 0; y < output_img.cols; y++) { + for (int x = 0; x < output_img.rows; x++) { + uint8_t cur1 = output_img.at(cv::Point(y, x))[0]; + uint8_t h_hue = 0; + h_hue = static_cast(hue * 255); + cur1 += h_hue; + output_img.at(cv::Point(y, x))[0] = cur1; + } + } + cv::cvtColor(output_img, output_cv->mat(), CV_HSV2RGB_FULL); + *output = std::static_pointer_cast(output_cv); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in adjust hue"); + } + return Status::OK(); +} + +Status GenerateRandomCropBox(int input_height, int input_width, float ratio, float lb, float ub, int max_itr, + cv::Rect *crop_box, uint32_t seed) { + try { + std::mt19937 rnd; + rnd.seed(GetSeed()); + if (input_height <= 0 || input_width <= 0 || ratio <= 0.0 || lb <= 0.0 || lb > ub) { + RETURN_STATUS_UNEXPECTED("Invalid inputs GenerateRandomCropBox"); + } + std::uniform_real_distribution rd_crop_ratio(lb, ub); + float crop_ratio; + int crop_width, crop_height; + bool crop_success = false; + int64_t input_area = input_height * input_width; + for (auto i = 0; i < max_itr; i++) { + crop_ratio = rd_crop_ratio(rnd); + crop_width = static_cast(std::round(std::sqrt(input_area * static_cast(crop_ratio) / ratio))); + crop_height = static_cast(std::round(crop_width * ratio)); + if (crop_width <= input_width && crop_height <= input_height) { + crop_success = true; + break; + } + } + if (crop_success == false) { + ratio = static_cast(input_height) / input_width; + crop_ratio = rd_crop_ratio(rnd); + crop_width = static_cast(std::lround(std::sqrt(input_area * static_cast(crop_ratio) / ratio))); + crop_height = static_cast(std::lround(crop_width * ratio)); + crop_height = (crop_height > input_height) ? input_height : crop_height; + crop_width = (crop_width > input_width) ? input_width : crop_width; + } + std::uniform_int_distribution<> rd_x(0, input_width - crop_width); + std::uniform_int_distribution<> rd_y(0, input_height - crop_height); + *crop_box = cv::Rect(rd_x(rnd), rd_y(rnd), crop_width, crop_height); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("error in GenerateRandomCropBox."); + } +} + +Status CheckOverlapConstraint(const cv::Rect &crop_box, const std::vector &bounding_boxes, + float min_intersect_ratio, bool *is_satisfied) { + try { + // not satisfied if the crop box contains no pixel + if (crop_box.area() < 1.0) { + *is_satisfied = false; + } + for (const auto &b_box : bounding_boxes) { + const float b_box_area = b_box.area(); + // not satisfied if the bounding box contains no pixel + if (b_box_area < 1.0) { + continue; + } + const float intersect_ratio = (crop_box & b_box).area() / b_box_area; + if (intersect_ratio >= min_intersect_ratio) { + *is_satisfied = true; + break; + } + } + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("error in CheckOverlapConstraint."); + } +} + +Status Erase(const std::shared_ptr &input, std::shared_ptr *output, int32_t box_height, + int32_t box_width, int32_t num_patches, bool bounded, bool random_color, uint8_t fill_r, uint8_t fill_g, + uint8_t fill_b) { + try { + std::mt19937 rnd; + rnd.seed(GetSeed()); + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + if (input_cv->mat().data == nullptr || (input_cv->Rank() != 3 && input_cv->shape()[2] != 3)) { + RETURN_STATUS_UNEXPECTED("bad CV Tensor input for erase"); + } + cv::Mat input_img = input_cv->mat(); + int32_t image_h = input_cv->shape()[0]; + int32_t image_w = input_cv->shape()[1]; + // check if erase size is bigger than image itself + if (box_height > image_h || box_width > image_w) { + RETURN_STATUS_UNEXPECTED("input box size too large for image erase"); + } + + // for random color + std::normal_distribution normal_distribution(0, 1); + std::uniform_int_distribution height_distribution_bound(0, image_h - box_height); + std::uniform_int_distribution width_distribution_bound(0, image_w - box_width); + std::uniform_int_distribution height_distribution_unbound(0, image_h + box_height); + std::uniform_int_distribution width_distribution_unbound(0, image_w + box_width); + // core logic + // update values based on random erasing or cutout + + for (int32_t i = 0; i < num_patches; i++) { + // rows in cv mat refers to the height of the cropped box + // we determine h_start and w_start using two different distributions as erasing is used by two different + // image augmentations. The bounds are also different in each case. + int32_t h_start = (bounded) ? height_distribution_bound(rnd) : (height_distribution_unbound(rnd) - box_height); + int32_t w_start = (bounded) ? width_distribution_bound(rnd) : (width_distribution_unbound(rnd) - box_width); + + int32_t max_width = (w_start + box_width > image_w) ? image_w : w_start + box_width; + int32_t max_height = (h_start + box_height > image_h) ? image_h : h_start + box_height; + // check for starting range >= 0, here the start range is checked after for cut out, for random erasing + // w_start and h_start will never be less than 0. + h_start = (h_start < 0) ? 0 : h_start; + w_start = (w_start < 0) ? 0 : w_start; + for (int y = w_start; y < max_width; y++) { + for (int x = h_start; x < max_height; x++) { + if (random_color) { + // fill each box with a random value + input_img.at(cv::Point(y, x))[0] = static_cast(normal_distribution(rnd)); + input_img.at(cv::Point(y, x))[1] = static_cast(normal_distribution(rnd)); + input_img.at(cv::Point(y, x))[2] = static_cast(normal_distribution(rnd)); + } else { + input_img.at(cv::Point(y, x))[0] = fill_r; + input_img.at(cv::Point(y, x))[1] = fill_g; + input_img.at(cv::Point(y, x))[2] = fill_b; + } + } + } + } + *output = std::static_pointer_cast(input); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Error in erasing"); + } +} + +Status Pad(const std::shared_ptr &input, std::shared_ptr *output, const int32_t &pad_top, + const int32_t &pad_bottom, const int32_t &pad_left, const int32_t &pad_right, const BorderType &border_types, + uint8_t fill_r, uint8_t fill_g, uint8_t fill_b) { + try { + // input image + std::shared_ptr input_cv = CVTensor::AsCVTensor(input); + // get the border type in openCV + auto b_type = GetCVBorderType(border_types); + // output image + cv::Mat out_image; + if (b_type == cv::BORDER_CONSTANT) { + cv::Scalar fill_color = cv::Scalar(fill_b, fill_g, fill_r); + cv::copyMakeBorder(input_cv->mat(), out_image, pad_top, pad_bottom, pad_left, pad_right, b_type, fill_color); + } else { + cv::copyMakeBorder(input_cv->mat(), out_image, pad_top, pad_bottom, pad_left, pad_right, b_type); + } + std::shared_ptr output_cv = std::make_shared(out_image); + RETURN_UNEXPECTED_IF_NULL(output_cv); + *output = std::static_pointer_cast(output_cv); + return Status::OK(); + } catch (const cv::Exception &e) { + RETURN_STATUS_UNEXPECTED("Unexpected error in pad"); + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.h b/mindspore/ccsrc/dataset/kernels/image/image_utils.h new file mode 100644 index 00000000000..d289f6f56e1 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.h @@ -0,0 +1,243 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_ +#define DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_ + +#include + +#include +#include +#include +#include +#include "./jpeglib.h" +#include "./jerror.h" +#include +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +enum class InterpolationMode { kLinear = 0, kNearestNeighbour = 1, kCubic = 2, kArea = 3 }; + +enum class BorderType { kConstant = 0, kEdge = 1, kReflect = 2, kSymmetric = 3 }; + +void JpegErrorExitCustom(j_common_ptr cinfo); + +struct JpegErrorManagerCustom { + // "public" fields + struct jpeg_error_mgr pub; + // for return to caller + jmp_buf setjmp_buffer; +}; + +// Returns the interpolation mode in openCV format +// @param mode: interpolation mode in DE format +int GetCVInterpolationMode(InterpolationMode mode); + +// Returns the openCV equivalent of the border type used for padding. +// @param type +// @return +int GetCVBorderType(BorderType type); + +// Returns flipped image +// @param input/output: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param flip_code: 1 for Horizontal (around y-axis), 0 for Vertical (around x-axis), -1 for both +// The flipping happens in place. +Status Flip(std::shared_ptr input, std::shared_ptr *output, int flip_code); + +// Returns Horizontally flipped image +// @param input/output: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// The flipping happens in place. +Status HorizontalFlip(std::shared_ptr input, std::shared_ptr *output); + +// Returns Vertically flipped image +// @param input/output: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// The flipping happens in place. +Status VerticalFlip(std::shared_ptr input, std::shared_ptr *output); + +// Returns Resized image. +// @param input/output: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param output_height: height of output +// @param output_width: width of output +// @param fx: horizontal scale +// @param fy: vertical scale +// @param InterpolationMode: the interpolation mode +// @param output: Resized image of shape or +// and same type as input +Status Resize(const std::shared_ptr &input, std::shared_ptr *output, int32_t output_height, + int32_t output_width, double fx = 0.0, double fy = 0.0, + InterpolationMode mode = InterpolationMode::kLinear); + +// Returns Decoded image +// Supported images: +// - Windows bitmaps - \*.bmp, \*.dib (always supported) +// - JPEG files - \*.jpeg, \*.jpg, \*.jpe (see the *Note* section) +// - JPEG 2000 files - \*.jp2 (see the *Note* section) +// - Portable Network Graphics - \*.png (see the *Note* section) +// - WebP - \*.webp (see the *Note* section) +// - Portable image format - \*.pbm, \*.pgm, \*.ppm \*.pxm, \*.pnm (always supported) +// - PFM files - \*.pfm (see the *Note* section) +// - Sun rasters - \*.sr, \*.ras (always supported) +// - TIFF files - \*.tiff, \*.tif (see the *Note* section) +// - OpenEXR Image files - \*.exr (see the *Note* section) +// - Radiance HDR - \*.hdr, \*.pic (always supported) +// - Raster and Vector geospatial data supported by GDAL (see the *Note* section) +// @param input: CVTensor containing the not decoded image 1D bytes +// @param output: Decoded image Tensor of shape and type DE_UINT8. Pixel order is RGB +Status Decode(const std::shared_ptr &input, std::shared_ptr *output); + +Status DecodeCv(const std::shared_ptr &input, std::shared_ptr *output); + +bool HasJpegMagic(const unsigned char *data, size_t data_size); + +void JpegSetSource(j_decompress_ptr c_info, const void *data, int64_t data_size); + +Status JpegCropAndDecode(const std::shared_ptr &input, std::shared_ptr *output, int x = 0, int y = 0, + int w = 0, int h = 0); +// Returns Rescaled image +// @param input: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param rescale: rescale parameter +// @param shift: shift parameter +// @param output: Rescaled image Tensor of same input shape and type DE_FLOAT32 +Status Rescale(const std::shared_ptr &input, std::shared_ptr *output, float rescale, float shift); + +// Returns cropped ROI of an image +// @param input: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param x: starting horizontal position of ROI +// @param y: starting vertical position of ROI +// @param w: width of the ROI +// @param h: height of the ROI +// @param output: Cropped image Tensor of shape or and same input type. +Status Crop(const std::shared_ptr &input, std::shared_ptr *output, int x, int y, int w, int h); + +// Swaps the channels in the image, i.e. converts HWC to CHW +// @param input: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param output: Tensor of shape or and same input type. +Status HwcToChw(std::shared_ptr input, std::shared_ptr *output); + +// Swap the red and blue pixels (RGB <-> BGR) +// @param input: Tensor of shape and any OpenCv compatible type, see CVTensor. +// @param output: Swapped image of same shape and type +Status SwapRedAndBlue(std::shared_ptr input, std::shared_ptr *output); + +// Crops and resizes the image +// @param input: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param x: horizontal start point +// @param y: vertical start point +// @param crop_height: height of the cropped ROI +// @param crop_width: width of the cropped ROI +// @param target_width: width of the final resized image +// @param target_height: height of the final resized image +// @param InterpolationMode: the interpolation used in resize operation +// @param output: Tensor of shape or +// and same type as input +Status CropAndResize(const std::shared_ptr &input, std::shared_ptr *output, int x, int y, + int crop_height, int crop_width, int target_height, int target_width, InterpolationMode mode); + +// Returns rotated image +// @param input: Tensor of shape or and any OpenCv compatible type, see CVTensor. +// @param fx: rotation center x coordinate +// @param fy: rotation center y coordinate +// @param degree: degree to rotate +// @param expand: if reshape is necessary +// @param output: rotated image of same input type. +Status Rotate(const std::shared_ptr &input, std::shared_ptr *output, float fx, float fy, float degree, + InterpolationMode interpolation = InterpolationMode::kNearestNeighbour, bool expand = false, + uint8_t fill_r = 0, uint8_t fill_g = 0, uint8_t fill_b = 0); + +// Returns Normalized image +// @param input: Tensor of shape in RGB order and any OpenCv compatible type, see CVTensor. +// @param mean: Tensor of shape <3> and type DE_FLOAT32 which are mean of each channel in RGB order +// @param std: Tensor of shape <3> and type DE_FLOAT32 which are std of each channel in RGB order +// @param output: Normalized image Tensor of same input shape and type DE_FLOAT32 +Status Normalize(const std::shared_ptr &input, std::shared_ptr *output, + const std::shared_ptr &mean, const std::shared_ptr &std); + +// Returns image with adjusted brightness. +// @param input: Tensor of shape in RGB order and any OpenCv compatible type, see CVTensor. +// @param alpha: Alpha value to adjust brightness by. Should be a positive number. +// If user input one value in python, the range is [1 - value, 1 + value]. +// This will output original image multiplied by alpha. 0 gives a black image, 1 gives the +// original image while 2 increases the brightness by a factor of 2. +// @param output: Adjusted image of same shape and type. +Status AdjustBrightness(const std::shared_ptr &input, std::shared_ptr *output, const float &alpha); + +// Returns image with adjusted contrast. +// @param input: Tensor of shape in RGB order and any OpenCv compatible type, see CVTensor. +// @param alpha: Alpha value to adjust contrast by. Should be a positive number. +// If user input one value in python, the range is [1 - value, 1 + value]. +// 0 gives a solid gray image, 1 gives the original image while 2 increases +// the contrast by a factor of 2. +// @param output: Adjusted image of same shape and type. +Status AdjustContrast(const std::shared_ptr &input, std::shared_ptr *output, const float &alpha); + +// Returns image with adjusted saturation. +// @param input: Tensor of shape in RGB order and any OpenCv compatible type, see CVTensor. +// @param alpha: Alpha value to adjust saturation by. Should be a positive number. +// If user input one value in python, the range is [1 - value, 1 + value]. +// 0 will give a black and white image, 1 will give the original image while +// 2 will enhance the saturation by a factor of 2. +// @param output: Adjusted image of same shape and type. +Status AdjustSaturation(const std::shared_ptr &input, std::shared_ptr *output, const float &alpha); + +// Returns image with adjusted hue. +// @param input: Tensor of shape in RGB order and any OpenCv compatible type, see CVTensor. +// @param hue: Hue value to adjust by, should be within range [-0.5, 0.5]. 0.5 and - 0.5 will reverse the hue channel +// completely. +// If user input one value in python, the range is [-value, value]. +// @param output: Adjusted image of same shape and type. +Status AdjustHue(const std::shared_ptr &input, std::shared_ptr *output, const float &hue); + +Status GenerateRandomCropBox(int input_height, int input_width, float ratio, float lb, float ub, int max_itr, + cv::Rect *crop_box, uint32_t seed = std::mt19937::default_seed); + +Status CheckOverlapConstraint(const cv::Rect &crop_box, const std::vector &bounding_boxes, + float min_intersect_ratio, bool *is_satisfied); + +// Masks out a random section from the image with set dimension +// @param input: input Tensor +// @param output: cutOut Tensor +// @param box_height: height of the cropped box +// @param box_width: width of the cropped box +// @param num_patches: number of boxes to cut out from the image +// @param bounded: boolean flag to toggle between random erasing and cutout +// @param random_color: whether or not random fill value should be used +// @param fill_r: red fill value for erase +// @param fill_g: green fill value for erase +// @param fill_b: blue fill value for erase. +Status Erase(const std::shared_ptr &input, std::shared_ptr *output, int32_t box_height, + int32_t box_width, int32_t num_patches, bool bounded, bool random_color, uint8_t fill_r = 0, + uint8_t fill_g = 0, uint8_t fill_b = 0); + +// Pads the input image and puts the padded image in the output +// @param input: input Tensor +// @param output: padded Tensor +// @param pad_top: amount of padding done in top +// @param pad_bottom: amount of padding done in bottom +// @param pad_left: amount of padding done in left +// @param pad_right: amount of padding done in right +// @param border_types: the interpolation to be done in the border +// @param fill_r: red fill value for pad +// @param fill_g: green fill value for pad +// @param fill_b: blue fill value for pad. +Status Pad(const std::shared_ptr &input, std::shared_ptr *output, const int32_t &pad_top, + const int32_t &pad_bottom, const int32_t &pad_left, const int32_t &pad_right, const BorderType &border_types, + uint8_t fill_r = 0, uint8_t fill_g = 0, uint8_t fill_b = 0); +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/normalize_op.cc b/mindspore/ccsrc/dataset/kernels/image/normalize_op.cc new file mode 100644 index 00000000000..638eaad264a --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/normalize_op.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/normalize_op.h" + +#include + +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +NormalizeOp::NormalizeOp(float mean_r, float mean_g, float mean_b, float std_r, float std_g, float std_b) { + int size[] = {3}; + cv::Mat mean_cv(1, size, CV_32F); + mean_cv.at(0) = mean_r; + mean_cv.at(1) = mean_g; + mean_cv.at(2) = mean_b; + mean_ = std::make_shared(mean_cv); + mean_->Squeeze(); + + cv::Mat std_cv(1, size, CV_32F); + std_cv.at(0) = std_r; + std_cv.at(1) = std_g; + std_cv.at(2) = std_b; + std_ = std::make_shared(std_cv); + std_->Squeeze(); +} + +Status NormalizeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + // Doing the normalization + return Normalize(input, output, mean_, std_); +} + +void NormalizeOp::Print(std::ostream &out) const { + out << "NormalizeOp, mean: " << mean_->mat().at(0) << ", " << mean_->mat().at(1) << ", " + << mean_->mat().at(2) << "std: " << std_->mat().at(0) << ", " << std_->mat().at(1) << ", " + << std_->mat().at(2) << std::endl; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/normalize_op.h b/mindspore/ccsrc/dataset/kernels/image/normalize_op.h new file mode 100644 index 00000000000..7aa6fa69bd7 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/normalize_op.h @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_NORMALIZE_OP_H_ +#define DATASET_KERNELS_IMAGE_NORMALIZE_OP_H_ + +#include + +#include "dataset/core/cv_tensor.h" +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class NormalizeOp : public TensorOp { + public: + NormalizeOp(float mean_r, float mean_g, float mean_b, float std_r, float std_g, float std_b); + + ~NormalizeOp() override = default; + + void Print(std::ostream &out) const override; + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + private: + std::shared_ptr mean_; + std::shared_ptr std_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_NORMALIZE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/pad_op.cc b/mindspore/ccsrc/dataset/kernels/image/pad_op.cc new file mode 100644 index 00000000000..b4d9c2bbf0b --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/pad_op.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/pad_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const BorderType PadOp::kDefBorderType = BorderType::kConstant; +const uint8_t PadOp::kDefFillR = 0; +const uint8_t PadOp::kDefFillG = 0; +const uint8_t PadOp::kDefFillB = 0; + +PadOp::PadOp(int32_t pad_top, int32_t pad_bottom, int32_t pad_left, int32_t pad_right, BorderType border_types, + uint8_t fill_r, uint8_t fill_g, uint8_t fill_b) + : pad_top_(pad_top), + pad_bottom_(pad_bottom), + pad_left_(pad_left), + pad_right_(pad_right), + boarder_type_(border_types), + fill_r_(fill_r), + fill_g_(fill_g), + fill_b_(fill_b) {} + +Status PadOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + return Pad(input, output, pad_top_, pad_bottom_, pad_left_, pad_right_, boarder_type_, fill_r_, fill_g_, fill_b_); +} + +Status PadOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape out({-1, -1, 3}); // we don't know what is output image size, but we know it should be 3 channels + if (inputs[0].Rank() == 1) outputs.emplace_back(out); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/pad_op.h b/mindspore/ccsrc/dataset/kernels/image/pad_op.h new file mode 100644 index 00000000000..76d99d01626 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/pad_op.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_PAD_OP_H_ +#define DATASET_KERNELS_IMAGE_PAD_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class PadOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const BorderType kDefBorderType; + static const uint8_t kDefFillR; + static const uint8_t kDefFillG; + static const uint8_t kDefFillB; + + // Constructor for PadOp. + // @param pad_top number of pixels to pad the top of image with. + // @param pad_bottom number of pixels to pad the bottom of the image with. + // @param pad_left number of pixels to pad the left of the image with. + // @param pad_right number of pixels to pad the right of the image with. + // @param border_types BorderType enum, the type of boarders that we are using. + // @param fill_r R value for the color to pad with. + // @param fill_g G value for the color to pad with. + // @param fill_b B value for the color to pad with. + PadOp(int32_t pad_top, int32_t pad_bottom, int32_t pad_left, int32_t pad_right, BorderType border_types, + uint8_t fill_r = kDefFillR, uint8_t fill_g = kDefFillG, uint8_t fill_b = kDefFillB); + + ~PadOp() override = default; + + void Print(std::ostream &out) const override { out << "PadOp: "; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + private: + int32_t pad_top_; + int32_t pad_bottom_; + int32_t pad_left_; + int32_t pad_right_; + BorderType boarder_type_; + uint8_t fill_r_; + uint8_t fill_g_; + uint8_t fill_b_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_PAD_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_color_adjust_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_color_adjust_op.cc new file mode 100644 index 00000000000..e420f86e9a2 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_color_adjust_op.cc @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_color_adjust_op.h" + +#include + +#include "dataset/core/config_manager.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +RandomColorAdjustOp::RandomColorAdjustOp(float s_bright_factor, float e_bright_factor, float s_contrast_factor, + float e_contrast_factor, float s_saturation_factor, float e_saturation_factor, + float s_hue_factor, float e_hue_factor) + : bright_factor_start_(s_bright_factor), + bright_factor_end_(e_bright_factor), + contrast_factor_start_(s_contrast_factor), + contrast_factor_end_(e_contrast_factor), + saturation_factor_start_(s_saturation_factor), + saturation_factor_end_(e_saturation_factor), + hue_factor_start_(s_hue_factor), + hue_factor_end_(e_hue_factor) { + rnd_.seed(GetSeed()); +} + +Status RandomColorAdjustOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + + // randomly select an augmentation to apply to the input image until all the transformations run + std::vector params_vector = {"brightness", "contrast", "saturation", "hue"}; + + std::shuffle(params_vector.begin(), params_vector.end(), rnd_); + + *output = std::static_pointer_cast(input); + // determine if certain augmentation needs to be executed: + for (const auto ¶m : params_vector) { + // case switch + if (param == "brightness") { + if (CmpFloat(bright_factor_start_, bright_factor_end_) && CmpFloat(bright_factor_start_, 1.0f)) { + MS_LOG(DEBUG) << "Not running brightness."; + } else { + // adjust the brightness of an image + float random_factor = std::uniform_real_distribution(bright_factor_start_, bright_factor_end_)(rnd_); + RETURN_IF_NOT_OK(AdjustBrightness(*output, output, random_factor)); + } + } else if (param == "contrast") { + if (CmpFloat(contrast_factor_start_, contrast_factor_end_) && CmpFloat(contrast_factor_start_, 1.0f)) { + MS_LOG(DEBUG) << "Not running contrast."; + } else { + float random_factor = std::uniform_real_distribution(contrast_factor_start_, contrast_factor_end_)(rnd_); + RETURN_IF_NOT_OK(AdjustContrast(*output, output, random_factor)); + } + } else if (param == "saturation") { + // adjust the Saturation of an image + if (CmpFloat(saturation_factor_start_, saturation_factor_end_) && CmpFloat(saturation_factor_start_, 1.0f)) { + MS_LOG(DEBUG) << "Not running saturation."; + } else { + float random_factor = + std::uniform_real_distribution(saturation_factor_start_, saturation_factor_end_)(rnd_); + RETURN_IF_NOT_OK(AdjustSaturation(*output, output, random_factor)); + } + } else if (param == "hue") { + if (CmpFloat(hue_factor_start_, hue_factor_end_) && CmpFloat(hue_factor_start_, 0.0f)) { + MS_LOG(DEBUG) << "Not running hue."; + } else { + // adjust the Hue of an image + float random_factor = std::uniform_real_distribution(hue_factor_start_, hue_factor_end_)(rnd_); + RETURN_IF_NOT_OK(AdjustHue(*output, output, random_factor)); + } + } + } + // now after we do all the transformations, the last one is fine + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_color_adjust_op.h b/mindspore/ccsrc/dataset/kernels/image/random_color_adjust_op.h new file mode 100644 index 00000000000..74d1ec450b1 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_color_adjust_op.h @@ -0,0 +1,78 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_COLOR_ADJUST_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_COLOR_ADJUST_OP_H_ + +#include +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomColorAdjustOp : public TensorOp { + public: + static const uint32_t kDefSeed; + + // Constructor for RandomColorAdjustOp. + // @param s_bright_factor brightness change range start value. + // @param e_bright_factor brightness change range end value. + // @param s_contrast_factor contrast change range start value. + // @param e_contrast_factor contrast change range start value. + // @param s_saturation_factor saturation change range end value. + // @param e_saturation_factor saturation change range end value. + // @param s_hue_factor hue change factor start value, this should be greater than -0.5. + // @param e_hue_factor hue change factor start value, this should be less than 0.5. + // @param seed optional seed to pass in to the constructor. + // @details the randomly chosen degree is uniformly distributed. + RandomColorAdjustOp(float s_bright_factor, float e_bright_factor, float s_contrast_factor, float e_contrast_factor, + float s_saturation_factor, float e_saturation_factor, float s_hue_factor, float e_hue_factor); + + ~RandomColorAdjustOp() override = default; + + // Print function for RandomJitter. + // @param out output stream to print to. + void Print(std::ostream &out) const override { out << "RandomColorAdjustOp: "; } + + // Overrides the base class compute function. + // Calls multiple transform functions in ImageUtils, this function takes an input tensor. + // and transforms its data using openCV, the output memory is manipulated to contain the result. + // @return Status - The error code return. + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + private: + std::mt19937 rnd_; + float bright_factor_start_; + float bright_factor_end_; + float contrast_factor_start_; + float contrast_factor_end_; + float saturation_factor_start_; + float saturation_factor_end_; + float hue_factor_start_; + float hue_factor_end_; + // Compare two floating point variables. Return true if they are same / very close. + inline bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f) const { + return (std::fabs(a - b) < epsilon); + } +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_COLOR_ADJUST_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_op.cc new file mode 100644 index 00000000000..3cf60656595 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_op.cc @@ -0,0 +1,97 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_crop_and_resize_op.h" +#include + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const float RandomCropAndResizeOp::kDefScaleLb = 0.08; +const float RandomCropAndResizeOp::kDefScaleUb = 1.0; +const float RandomCropAndResizeOp::kDefAspectLb = 0.75; +const float RandomCropAndResizeOp::kDefAspectUb = 1.333333; +const InterpolationMode RandomCropAndResizeOp::kDefInterpolation = InterpolationMode::kLinear; +const int32_t RandomCropAndResizeOp::kDefMaxIter = 10; + +RandomCropAndResizeOp::RandomCropAndResizeOp(int32_t target_height, int32_t target_width, float scale_lb, + float scale_ub, float aspect_lb, float aspect_ub, + InterpolationMode interpolation, int32_t max_iter) + : target_height_(target_height), + target_width_(target_width), + rnd_scale_(scale_lb, scale_ub), + rnd_aspect_(aspect_lb, aspect_ub), + interpolation_(interpolation), + max_iter_(max_iter) { + rnd_.seed(GetSeed()); +} + +Status RandomCropAndResizeOp::Compute(const std::shared_ptr& input, std::shared_ptr* output) { + IO_CHECK(input, output); + CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "The shape of input is abnormal"); + + int h_in = input->shape()[0]; + int w_in = input->shape()[1]; + int x = 0; + int y = 0; + int crop_height = 0; + int crop_width = 0; + (void)GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width); + return CropAndResize(input, output, x, y, crop_height, crop_width, target_height_, target_width_, interpolation_); +} +Status RandomCropAndResizeOp::OutputShape(const std::vector& inputs, std::vector& outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape out = TensorShape{target_height_, target_width_}; + if (inputs[0].Rank() == 2) outputs.emplace_back(out); + if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +Status RandomCropAndResizeOp::GetCropBox(int h_in, int w_in, int* x, int* y, int* crop_height, int* crop_width) { + double scale, aspect; + *crop_width = w_in; + *crop_height = h_in; + bool crop_success = false; + for (int32_t i = 0; i < max_iter_; i++) { + scale = rnd_scale_(rnd_); + aspect = rnd_aspect_(rnd_); + *crop_width = static_cast(std::round(std::sqrt(h_in * w_in * scale / aspect))); + *crop_height = static_cast(std::round(*crop_width * aspect)); + if (*crop_width <= w_in && *crop_height <= h_in) { + crop_success = true; + break; + } + } + if (!crop_success) { + CHECK_FAIL_RETURN_UNEXPECTED(w_in != 0, "Width is 0"); + aspect = static_cast(h_in) / w_in; + scale = rnd_scale_(rnd_); + *crop_width = static_cast(std::round(std::sqrt(h_in * w_in * scale / aspect))); + *crop_height = static_cast(std::round(*crop_width * aspect)); + *crop_height = (*crop_height > h_in) ? h_in : *crop_height; + *crop_width = (*crop_width > w_in) ? w_in : *crop_width; + } + std::uniform_int_distribution<> rd_x(0, w_in - *crop_width); + std::uniform_int_distribution<> rd_y(0, h_in - *crop_height); + *x = rd_x(rnd_); + *y = rd_y(rnd_); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_op.h b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_op.h new file mode 100644 index 00000000000..97ee9f60926 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_op.h @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_CROP_AND_RESIZE_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_CROP_AND_RESIZE_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomCropAndResizeOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const float kDefScaleLb; + static const float kDefScaleUb; + static const float kDefAspectLb; + static const float kDefAspectUb; + static const InterpolationMode kDefInterpolation; + static const int32_t kDefMaxIter; + + RandomCropAndResizeOp(int32_t target_height, int32_t target_width, float scale_lb = kDefScaleLb, + float scale_ub = kDefScaleUb, float aspect_lb = kDefAspectLb, float aspect_ub = kDefAspectUb, + InterpolationMode interpolation = kDefInterpolation, int32_t max_iter = kDefMaxIter); + + ~RandomCropAndResizeOp() override = default; + + void Print(std::ostream &out) const override { + out << "RandomCropAndResize: " << target_height_ << " " << target_width_; + } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + Status GetCropBox(int h_in, int w_in, int *x, int *y, int *crop_height, int *crop_width); + + protected: + int32_t target_height_; + int32_t target_width_; + std::uniform_real_distribution rnd_scale_; + std::uniform_real_distribution rnd_aspect_; + std::mt19937 rnd_; + InterpolationMode interpolation_; + int32_t max_iter_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_CROP_AND_RESIZE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc new file mode 100644 index 00000000000..732aaa90313 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_crop_decode_resize_op.h" +#include +#include "dataset/kernels/image/image_utils.h" +#include "dataset/core/config_manager.h" +#include "dataset/kernels/image/decode_op.h" + +namespace mindspore { +namespace dataset { +RandomCropDecodeResizeOp::RandomCropDecodeResizeOp(int32_t target_height, int32_t target_width, float scale_lb, + float scale_ub, float aspect_lb, float aspect_ub, + InterpolationMode interpolation, int32_t max_iter) + : RandomCropAndResizeOp(target_height, target_width, scale_lb, scale_ub, aspect_lb, aspect_ub, interpolation, + max_iter) {} + +Status RandomCropDecodeResizeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + if (input == nullptr) { + RETURN_STATUS_UNEXPECTED("input tensor is null"); + } + if (!HasJpegMagic(input->StartAddr(), input->SizeInBytes())) { + DecodeOp op(true); + std::shared_ptr decoded; + RETURN_IF_NOT_OK(op.Compute(input, &decoded)); + return RandomCropAndResizeOp::Compute(decoded, output); + } else { + struct jpeg_decompress_struct cinfo {}; + struct JpegErrorManagerCustom jerr {}; + cinfo.err = jpeg_std_error(&jerr.pub); + jerr.pub.error_exit = JpegErrorExitCustom; + try { + jpeg_create_decompress(&cinfo); + JpegSetSource(&cinfo, input->StartAddr(), input->SizeInBytes()); + (void)jpeg_read_header(&cinfo, TRUE); + jpeg_calc_output_dimensions(&cinfo); + } catch (std::runtime_error &e) { + jpeg_destroy_decompress(&cinfo); + RETURN_STATUS_UNEXPECTED(e.what()); + } + int h_in = cinfo.output_height; + int w_in = cinfo.output_width; + jpeg_destroy_decompress(&cinfo); + + int x = 0; + int y = 0; + int crop_height = 0; + int crop_width = 0; + (void)GetCropBox(h_in, w_in, &x, &y, &crop_height, &crop_width); + + std::shared_ptr decoded; + RETURN_IF_NOT_OK(JpegCropAndDecode(input, &decoded, x, y, crop_width, crop_height)); + return Resize(decoded, output, target_height_, target_width_, 0.0, 0.0, interpolation_); + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.h b/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.h new file mode 100644 index 00000000000..95661699464 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.h @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_CROP_DECODE_RESIZE_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_CROP_DECODE_RESIZE_OP_H_ + +#include +#include +#include +#include +#include "dataset/core/tensor.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/image/random_crop_and_resize_op.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomCropDecodeResizeOp : public RandomCropAndResizeOp { + public: + RandomCropDecodeResizeOp(int32_t target_height, int32_t target_width, float scale_lb = kDefScaleLb, + float scale_ub = kDefScaleUb, float aspect_lb = kDefAspectLb, float aspect_ub = kDefAspectUb, + InterpolationMode interpolation = kDefInterpolation, int32_t max_iter = kDefMaxIter); + + ~RandomCropDecodeResizeOp() override = default; + + void Print(std::ostream &out) const override { + out << "RandomCropDecodeResize: " << RandomCropAndResizeOp::target_height_ << " " + << RandomCropAndResizeOp::target_width_; + } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_CROP_DECODE_RESIZE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_op.cc new file mode 100644 index 00000000000..7662c64cc40 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_op.cc @@ -0,0 +1,99 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_crop_op.h" +#include +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t RandomCropOp::kDefPadTop = 0; +const int32_t RandomCropOp::kDefPadBottom = 0; +const int32_t RandomCropOp::kDefPadLeft = 0; +const int32_t RandomCropOp::kDefPadRight = 0; +const BorderType RandomCropOp::kDefBorderType = BorderType::kConstant; +const bool RandomCropOp::kDefPadIfNeeded = false; +const uint8_t RandomCropOp::kDefFillR = 0; +const uint8_t RandomCropOp::kDefFillG = 0; +const uint8_t RandomCropOp::kDefFillB = 0; + +RandomCropOp::RandomCropOp(int32_t crop_height, int32_t crop_width, int32_t pad_top, int32_t pad_bottom, + int32_t pad_left, int32_t pad_right, BorderType border_types, bool pad_if_needed, + uint8_t fill_r, uint8_t fill_g, uint8_t fill_b) + : crop_height_(crop_height), + crop_width_(crop_width), + pad_top_(pad_top), + pad_bottom_(pad_bottom), + pad_left_(pad_left), + pad_right_(pad_right), + pad_if_needed_(pad_if_needed), + border_type_(border_types), + fill_r_(fill_r), + fill_g_(fill_g), + fill_b_(fill_b) { + rnd_.seed(GetSeed()); +} + +Status RandomCropOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + + // Apply padding first then crop + std::shared_ptr pad_image; + + RETURN_IF_NOT_OK( + Pad(input, &pad_image, pad_top_, pad_bottom_, pad_left_, pad_right_, border_type_, fill_r_, fill_g_, fill_b_)); + CHECK_FAIL_RETURN_UNEXPECTED(pad_image->shape().Size() >= 2, "Abnormal shape"); + int32_t padded_image_h = pad_image->shape()[0]; + int32_t padded_image_w = pad_image->shape()[1]; + // no need to crop if same size + if (padded_image_h == crop_height_ && padded_image_w == crop_width_) { + *output = pad_image; + return Status::OK(); + } + if (pad_if_needed_) { + // check the dimensions of the image for padding, if we do need padding, then we change the pad values + if (padded_image_h < crop_height_) { + RETURN_IF_NOT_OK(Pad(pad_image, &pad_image, crop_height_ - padded_image_h, crop_height_ - padded_image_h, 0, 0, + border_type_, fill_r_, fill_g_, fill_b_)); + } + if (padded_image_w < crop_width_) { + RETURN_IF_NOT_OK(Pad(pad_image, &pad_image, 0, 0, crop_width_ - padded_image_w, crop_width_ - padded_image_w, + border_type_, fill_r_, fill_g_, fill_b_)); + } + padded_image_h = pad_image->shape()[0]; + padded_image_w = pad_image->shape()[1]; + } + if (padded_image_h < crop_height_ || padded_image_w < crop_width_ || crop_height_ == 0 || crop_width_ == 0) { + return Status(StatusCode::kShapeMisMatch, __LINE__, __FILE__, + "Crop size is greater than the image dimensions or is zero."); + } + // random top corner + int x = std::uniform_int_distribution(0, padded_image_w - crop_width_)(rnd_); + int y = std::uniform_int_distribution(0, padded_image_h - crop_height_)(rnd_); + return Crop(pad_image, output, x, y, crop_width_, crop_height_); +} +Status RandomCropOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + TensorShape out = TensorShape{crop_height_, crop_width_}; + if (inputs[0].Rank() == 2) outputs.emplace_back(out); + if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_op.h b/mindspore/ccsrc/dataset/kernels/image/random_crop_op.h new file mode 100644 index 00000000000..d4ec49cd7b4 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_op.h @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_CROP_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_CROP_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomCropOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefPadTop; + static const int32_t kDefPadBottom; + static const int32_t kDefPadLeft; + static const int32_t kDefPadRight; + static const BorderType kDefBorderType; + static const bool kDefPadIfNeeded; + static const uint8_t kDefFillR; + static const uint8_t kDefFillG; + static const uint8_t kDefFillB; + + RandomCropOp(int32_t crop_height, int32_t crop_width, int32_t pad_top = kDefPadTop, + int32_t pad_bottom = kDefPadBottom, int32_t pad_left = kDefPadLeft, int32_t pad_right = kDefPadRight, + BorderType border_types = kDefBorderType, bool pad_if_needed = kDefPadIfNeeded, + uint8_t fill_r = kDefFillR, uint8_t fill_g = kDefFillG, uint8_t fill_b = kDefFillB); + + ~RandomCropOp() override = default; + + void Print(std::ostream &out) const override { out << "RandomCropOp: " << crop_height_ << " " << crop_width_; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + private: + int32_t crop_height_ = 0; + int32_t crop_width_ = 0; + int32_t pad_top_ = 0; + int32_t pad_bottom_ = 0; + int32_t pad_left_ = 0; + int32_t pad_right_ = 0; + bool pad_if_needed_ = false; + BorderType border_type_; + uint8_t fill_r_ = 0; + uint8_t fill_g_ = 0; + uint8_t fill_b_ = 0; + std::mt19937 rnd_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_CROP_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_horizontal_flip_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_horizontal_flip_op.cc new file mode 100644 index 00000000000..ae76e1bf591 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_horizontal_flip_op.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_horizontal_flip_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const float RandomHorizontalFlipOp::kDefProbability = 0.5; + +Status RandomHorizontalFlipOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + if (distribution_(rnd_)) { + return HorizontalFlip(input, output); + } + *output = input; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_horizontal_flip_op.h b/mindspore/ccsrc/dataset/kernels/image/random_horizontal_flip_op.h new file mode 100644 index 00000000000..efea1245334 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_horizontal_flip_op.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_HORIZONTAL_FLIP_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_HORIZONTAL_FLIP_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomHorizontalFlipOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const float kDefProbability; + + explicit RandomHorizontalFlipOp(float probability = kDefProbability) : distribution_(probability) { + rnd_.seed(GetSeed()); + } + + ~RandomHorizontalFlipOp() override = default; + + // Provide stream operator for displaying it + friend std::ostream &operator<<(std::ostream &out, const RandomHorizontalFlipOp &so) { + so.Print(out); + return out; + } + + void Print(std::ostream &out) const override { out << "RandomHorizontalFlipOp"; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + private: + std::mt19937 rnd_; + std::bernoulli_distribution distribution_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_HORIZONTAL_FLIP_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_resize_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_resize_op.cc new file mode 100644 index 00000000000..c14224a9308 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_resize_op.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_resize_op.h" + +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t RandomResizeOp::kDefTargetWidth = 0; + +Status RandomResizeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + // Randomly selects from the following four interpolation methods + // 0-bilinear, 1-nearest_neighbor, 2-bicubic, 3-area + interpolation_ = static_cast(distribution_(random_generator_)); + return ResizeOp::Compute(input, output); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_resize_op.h b/mindspore/ccsrc/dataset/kernels/image/random_resize_op.h new file mode 100644 index 00000000000..af23803d4cb --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_resize_op.h @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_RESIZE_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_RESIZE_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomResizeOp : public ResizeOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefTargetWidth; + + explicit RandomResizeOp(int32_t size_1, int32_t size_2 = kDefTargetWidth) : ResizeOp(size_1, size_2) { + random_generator_.seed(GetSeed()); + } + + ~RandomResizeOp() = default; + + // Description: A function that prints info about the node + void Print(std::ostream &out) const override { + out << "RandomResizeOp: " << ResizeOp::size1_ << " " << ResizeOp::size2_; + } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + private: + std::mt19937 random_generator_; + std::uniform_int_distribution distribution_{0, 3}; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_RESIZE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_rotation_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_rotation_op.cc new file mode 100644 index 00000000000..65e024865b2 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_rotation_op.cc @@ -0,0 +1,82 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_rotation_op.h" + +#include + +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const float RandomRotationOp::kDefCenterX = -1; +const float RandomRotationOp::kDefCenterY = -1; +const InterpolationMode RandomRotationOp::kDefInterpolation = InterpolationMode::kNearestNeighbour; +const bool RandomRotationOp::kDefExpand = false; +const uint8_t RandomRotationOp::kDefFillR = 0; +const uint8_t RandomRotationOp::kDefFillG = 0; +const uint8_t RandomRotationOp::kDefFillB = 0; + +// constructor +RandomRotationOp::RandomRotationOp(float start_degree, float end_degree, float center_x, float center_y, + InterpolationMode interpolation, bool expand, uint8_t fill_r, uint8_t fill_g, + uint8_t fill_b) + : degree_start_(start_degree), + degree_end_(end_degree), + center_x_(center_x), + center_y_(center_y), + interpolation_(interpolation), + expand_(expand), + fill_r_(fill_r), + fill_g_(fill_g), + fill_b_(fill_b) { + rnd_.seed(GetSeed()); +} + +// main function call for random rotation : Generate the random degrees +Status RandomRotationOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + float random_double = distribution_(rnd_); + // get the degree rotation range, mod by 360 because full rotation doesn't affect + // the way this op works (uniform distribution) + // assumption here is that mDegreesEnd > mDegreeStart so we always get positive number + // Note: the range technically is greater than 360 degrees, but will be halved + float degree_range = (degree_end_ - degree_start_) / 2; + float mid = (degree_end_ + degree_start_) / 2; + float degree = mid + random_double * degree_range; + + return Rotate(input, output, center_x_, center_y_, degree, interpolation_, expand_, fill_r_, fill_g_, fill_b_); +} +Status RandomRotationOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + int32_t outputH = -1, outputW = -1; + // if expand_, then we cannot know the shape. We need the input image to find the output shape --> set it to + // <-1,-1[,3]> + if (!expand_) { + outputH = inputs[0][0]; + outputW = inputs[0][1]; + } + TensorShape out = TensorShape{outputH, outputW}; + if (inputs[0].Rank() == 2) outputs.emplace_back(out); + if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_rotation_op.h b/mindspore/ccsrc/dataset/kernels/image/random_rotation_op.h new file mode 100644 index 00000000000..d30cd242882 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_rotation_op.h @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_ROTATION_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_ROTATION_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" +#include "dataset/kernels/image/image_utils.h" + +namespace mindspore { +namespace dataset { +class RandomRotationOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const float kDefCenterX; + static const float kDefCenterY; + static const InterpolationMode kDefInterpolation; + static const bool kDefExpand; + static const uint8_t kDefFillR; + static const uint8_t kDefFillG; + static const uint8_t kDefFillB; + + // Constructor for RandomRotationOp + // @param startDegree starting range for random degree + // @param endDegree ending range for random degree + // @param centerX x coordinate for center of image rotation + // @param centerY y coordinate for center of image rotation + // @param interpolation DE interpolation mode for rotation + // @param expand option for the output image shape to change + // @param fill_r R value for the color to pad with + // @param fill_g G value for the color to pad with + // @param fill_b B value for the color to pad with + // @details the randomly chosen degree is uniformly distributed + // @details the output shape, if changed, will contain the entire rotated image + // @note maybe using unsigned long int isn't the best here according to our coding rules + RandomRotationOp(float start_degree, float end_degree, float center_x = kDefCenterX, float center_y = kDefCenterY, + InterpolationMode interpolation = kDefInterpolation, bool expand = kDefExpand, + uint8_t fill_r = kDefFillR, uint8_t fill_g = kDefFillG, uint8_t fill_b = kDefFillB); + + ~RandomRotationOp() override = default; + + // Print function for RandomRotation + // @param out output stream to print to + void Print(std::ostream &out) const override { out << "RandomRotationOp: "; } + + // Overrides the base class compute function + // Calls the rotate function in ImageUtils, this function takes an input tensor + // and transforms its data using openCV, the output memory is manipulated to contain the result + // @return Status - The error code return + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + private: + float degree_start_; + float degree_end_; + float center_x_; + float center_y_; + InterpolationMode interpolation_; + bool expand_; + uint8_t fill_r_; + uint8_t fill_g_; + uint8_t fill_b_; + std::uniform_real_distribution distribution_{-1.0, 1.0}; + std::mt19937 rnd_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_ROTATION_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_op.cc new file mode 100644 index 00000000000..096923a9ecd --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_op.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/kernels/image/random_vertical_flip_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const float RandomVerticalFlipOp::kDefProbability = 0.5; + +Status RandomVerticalFlipOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + if (distribution_(rnd_)) { + return VerticalFlip(input, output); + } + *output = input; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_op.h b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_op.h new file mode 100644 index 00000000000..18693bc0eb2 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_op.h @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RANDOM_VERTICAL_FLIP_OP_H_ +#define DATASET_KERNELS_IMAGE_RANDOM_VERTICAL_FLIP_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" +#include "dataset/util/random.h" + +namespace mindspore { +namespace dataset { +class RandomVerticalFlipOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const float kDefProbability; + + explicit RandomVerticalFlipOp(float probability = kDefProbability) : distribution_(probability) { + rnd_.seed(GetSeed()); + } + + ~RandomVerticalFlipOp() override = default; + + void Print(std::ostream &out) const override { out << "RandomVerticalFlipOp"; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + private: + std::mt19937 rnd_; + std::bernoulli_distribution distribution_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_VERTICAL_FLIP_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/rescale_op.cc b/mindspore/ccsrc/dataset/kernels/image/rescale_op.cc new file mode 100644 index 00000000000..fd1807991c0 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/rescale_op.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/rescale_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +Status RescaleOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + return Rescale(input, output, rescale_, shift_); +} +Status RescaleOp::OutputType(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs)); + outputs[0] = DataType(DataType::DE_FLOAT32); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/rescale_op.h b/mindspore/ccsrc/dataset/kernels/image/rescale_op.h new file mode 100644 index 00000000000..8aee75b0c1b --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/rescale_op.h @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RESCALE_OP_H_ +#define DATASET_KERNELS_IMAGE_RESCALE_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RescaleOp : public TensorOp { + public: + RescaleOp(float rescale_ratio, float shift_ratio) : rescale_(rescale_ratio), shift_(shift_ratio) {} + + ~RescaleOp() override = default; + + void Print(std::ostream &out) const override { + out << "RescaleOp: shift: " << shift_ << ", Rescale: " << rescale_ << std::endl; + } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputType(const std::vector &inputs, std::vector &outputs) override; + + private: + float rescale_; + float shift_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_KERNELS_IMAGE_RESCALE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_bilinear_op.cc b/mindspore/ccsrc/dataset/kernels/image/resize_bilinear_op.cc new file mode 100644 index 00000000000..658caac6a50 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_bilinear_op.cc @@ -0,0 +1,27 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/resize_bilinear_op.h" +#include + +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t ResizeBilinearOp::kDefWidth = 0; + +void ResizeBilinearOp::Print(std::ostream &out) const { out << "ResizeBilinearOp: "; } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_bilinear_op.h b/mindspore/ccsrc/dataset/kernels/image/resize_bilinear_op.h new file mode 100644 index 00000000000..c8c2a5185b6 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_bilinear_op.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RESIZE_BILINEAR_OP_H_ +#define DATASET_KERNELS_IMAGE_RESIZE_BILINEAR_OP_H_ + +#include +#include +#include +#include +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class ResizeBilinearOp : public ResizeOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefWidth; + + // Name: constructor + // Resizes the image to the output specified size using Bilinear interpolation. + // If only one value is provided, the it will resize the smaller size and maintains + // the aspect ratio. + // @param size1: the first size of output. If only this parameter is provided + // the smaller dimension will be resized to this and then the other dimension changes + // such that the aspect ratio is maintained. + // @param size2: the second size of output. If this is also provided, the output size + // will be (size1, size2) + explicit ResizeBilinearOp(int32_t size1, int32_t size2 = kDefWidth) + : ResizeOp(size1, size2, ResizeOp::kDefInterpolation) {} + + // Name: Destructor + // Description: Destructor + ~ResizeBilinearOp() = default; + + // Name: Print() + // Description: A function that prints info about the node + void Print(std::ostream &out) const override; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RESIZE_BILINEAR_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_op.cc b/mindspore/ccsrc/dataset/kernels/image/resize_op.cc new file mode 100644 index 00000000000..7c0252188e2 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_op.cc @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/resize_op.h" + +#include "dataset/kernels/image/image_utils.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t ResizeOp::kDefWidth = 0; +const InterpolationMode ResizeOp::kDefInterpolation = InterpolationMode::kLinear; + +Status ResizeOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "The shape size " + std::to_string(input->shape().Size()) + + " of input tensor is invalid"); + int32_t output_h, output_w = 0; + int32_t input_h = static_cast(input->shape()[0]); + int32_t input_w = static_cast(input->shape()[1]); + if (size2_ == 0) { + if (input_h < input_w) { + CHECK_FAIL_RETURN_UNEXPECTED(input_h != 0, "The input height is 0"); + output_h = size1_; + output_w = static_cast(std::lround(static_cast(input_w) / input_h * output_h)); + } else { + CHECK_FAIL_RETURN_UNEXPECTED(input_w != 0, "The input width is 0"); + output_w = size1_; + output_h = static_cast(std::lround(static_cast(input_h) / input_w * output_w)); + } + } else { + output_h = size1_; + output_w = size2_; + } + return Resize(input, output, output_h, output_w, 0, 0, interpolation_); +} + +Status ResizeOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + int32_t outputH = -1, outputW = -1; + // if size2_ == 0, then we cannot know the shape. We need the input image to find the output shape --> set it to + // <-1,-1[,3]> + if (size2_ != 0) { + outputH = size1_; + outputW = size2_; + } + TensorShape out = TensorShape{outputH, outputW}; + if (inputs[0].Rank() == 2) outputs.emplace_back(out); + if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kUnexpectedError, "Input has a wrong shape"); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_op.h b/mindspore/ccsrc/dataset/kernels/image/resize_op.h new file mode 100644 index 00000000000..5a35a6076c4 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_op.h @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RESIZE_OP_H_ +#define DATASET_KERNELS_IMAGE_RESIZE_OP_H_ + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class ResizeOp : public TensorOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefWidth; + static const InterpolationMode kDefInterpolation; + + // Resizes the image to the output specified size. If only one value is provided, + // the it will resize the smaller size and maintains the aspect ratio. + // @param size1: the first size of output. If only this parameter is provided + // the smaller dimension will be resized to this and then the other dimension changes + // such that the aspect ratio is maintained. + // @param size2: the second size of output. If this is also provided, the output size + // will be (size1, size2) + // @param InterpolationMode: the interpolation mode being used. + explicit ResizeOp(int32_t size1, int32_t size2 = kDefWidth, InterpolationMode mInterpolation = kDefInterpolation) + : size1_(size1), size2_(size2), interpolation_(mInterpolation) {} + + ~ResizeOp() override = default; + + void Print(std::ostream &out) const override { out << "ResizeOp: " << size1_ << " " << size2_; } + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + protected: + int32_t size1_; + int32_t size2_; + InterpolationMode interpolation_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RESIZE_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/no_op.h b/mindspore/ccsrc/dataset/kernels/no_op.h new file mode 100644 index 00000000000..bfbdf43b366 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/no_op.h @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_NO_OP_H_ +#define DATASET_KERNELS_NO_OP_H_ + +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +class NoOp : public TensorOp { + public: + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override { + *output = input; + return Status::OK(); + } + + void Print(std::ostream &out) const override { out << "NoOp"; }; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_KERNELS_NO_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/py_func_op.cc b/mindspore/ccsrc/dataset/kernels/py_func_op.cc new file mode 100644 index 00000000000..69bd3443c49 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/py_func_op.cc @@ -0,0 +1,85 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/py_func_op.h" + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/make_unique.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +Status PyFuncOp::Compute(const std::vector> &input, + std::vector> *output) { + IO_CHECK_VECTOR(input, output); + Status ret = Status(StatusCode::kOK, "PyFunc Call Succeed"); + { + // Acquire Python GIL + py::gil_scoped_acquire gil_acquire; + if (Py_IsInitialized() == 0) { + ret = Status(StatusCode::kPythonInterpreterFailure, "Python Interpreter is finalized"); + goto ComputeReturn; + } + try { + // Transform input tensor vector into numpy array vector + py::tuple input_args(input.size()); + for (size_t i = 0; i < input.size(); i++) { + py::array new_data; + RETURN_IF_NOT_OK(input.at(i)->GetDataAsNumpy(&new_data)); + // possible memcpy here + input_args[i] = new_data; + } + // Invoke python function + py::object ret_py_obj = this->py_func_ptr_(*input_args); + // Process the return value + if (py::isinstance(ret_py_obj)) { + // In case of a n-1 mapping, the return value will be a numpy array + std::shared_ptr out; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&out, ret_py_obj.cast())); + output->push_back(out); + } else if (py::isinstance(ret_py_obj)) { + // In case of a n-m mapping, the return value will be a tuple of numpy arrays + py::tuple ret_py_tuple = ret_py_obj.cast(); + // Iterate over two containers simultaneously for memory copy + for (size_t i = 0; i < ret_py_tuple.size(); i++) { + py::object ret_py_ele = ret_py_tuple[i]; + if (!py::isinstance(ret_py_ele)) { + goto ShapeMisMatch; + } + std::shared_ptr out; + RETURN_IF_NOT_OK(Tensor::CreateTensor(&out, ret_py_ele.cast())); + output->push_back(out); + } + } else { + goto ShapeMisMatch; + } + } catch (const py::error_already_set &e) { + ret = Status(StatusCode::kPyFuncException, e.what()); + } + } + +ComputeReturn: + return ret; + +ShapeMisMatch: + ret = Status(StatusCode::kShapeMisMatch, "PyFunc should return a numpy array or a numpy array tuple"); + goto ComputeReturn; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/py_func_op.h b/mindspore/ccsrc/dataset/kernels/py_func_op.h new file mode 100644 index 00000000000..af61f6ac55c --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/py_func_op.h @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_KERNELS_PY_FUNC_OP_H_ +#define DATASET_KERNELS_PY_FUNC_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +class __attribute__((visibility("hidden"))) PyFuncOp : public TensorOp { + public: + explicit PyFuncOp(py::function func) : py_func_ptr_(std::move(func)) {} + + ~PyFuncOp() override = default; + + uint32_t NumInput() override { return 0; } + uint32_t NumOutput() override { return 0; } + + // Compute function for n-n mapping. + Status Compute(const std::vector> &input, + std::vector> *output) override; + + private: + py::function py_func_ptr_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_PY_FUNC_OP_H_ diff --git a/mindspore/ccsrc/dataset/kernels/tensor_op.cc b/mindspore/ccsrc/dataset/kernels/tensor_op.cc new file mode 100644 index 00000000000..390dd42a71c --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/tensor_op.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/tensor_op.h" +#include +#include +#include +#include + +namespace mindspore { +namespace dataset { +// Name: Compute() +// Description: This Compute() take 1 Tensor and produce 1 Tensor. +// The derived class should override this function otherwise error. +Status TensorOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + if (!OneToOne()) { + return Status(StatusCode::kUnexpectedError, "Wrong Compute() function is called. This is not 1-1 TensorOp."); + } else { + return Status(StatusCode::kUnexpectedError, + "Is this TensorOp 1-1? If yes, please implement this Compute() in the derived class."); + } +} + +// Name: Compute() +// Description: This Compute() take multiple Tensors from different columns and produce multiple Tensors too. +// The derived class should override this function otherwise error. +Status TensorOp::Compute(const std::vector> &input, + std::vector> *output) { + IO_CHECK_VECTOR(input, output); + if (OneToOne()) { + output->resize(1); + return Compute(input[0], &(*output)[0]); + } + + return Status(StatusCode::kUnexpectedError, + "Is this TensorOp oneToOne? If no, please implement this Compute() in the derived class."); +} + +void TensorOp::Print(std::ostream &out) const { out << "TensorOp" << std::endl; } + +Status TensorOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + if (inputs.size() != NumInput()) + return Status(StatusCode::kUnexpectedError, + "The size of the input argument vector does not match the number of inputs"); + outputs = inputs; + return Status::OK(); +} + +Status TensorOp::OutputType(const std::vector &inputs, std::vector &outputs) { + if (inputs.size() != NumInput()) + return Status(StatusCode::kUnexpectedError, + "The size of the input argument vector does not match the number of inputs"); + outputs = inputs; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/tensor_op.h b/mindspore/ccsrc/dataset/kernels/tensor_op.h new file mode 100644 index 00000000000..73fba4e28d0 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/tensor_op.h @@ -0,0 +1,110 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_TENSOR_OP_H_ +#define DATASET_KERNELS_TENSOR_OP_H_ + +#include +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/util/status.h" + +#define IO_CHECK(input, output) \ + do { \ + if (input == nullptr || output == nullptr) { \ + RETURN_STATUS_UNEXPECTED("input or output is null."); \ + } \ + } while (false) + +#define IO_CHECK_VECTOR(input, output) \ + do { \ + if (output == nullptr) { \ + RETURN_STATUS_UNEXPECTED("output is null."); \ + } \ + for (auto &_i : input) { \ + if (_i == nullptr) { \ + RETURN_STATUS_UNEXPECTED("input is null."); \ + } \ + } \ + } while (false) + +namespace mindspore { +namespace dataset { +// A class that does a computation on a Tensor +class TensorOp { + public: + TensorOp() = default; + + virtual ~TensorOp() = default; + + // A function that prints info about the tensor operation + // @param out + virtual void Print(std::ostream &out) const; + + // Provide stream operator for displaying it + // @param output stream + // @param so the TensorOp object to be printed + // @return output stream + friend std::ostream &operator<<(std::ostream &out, const TensorOp &so) { + so.Print(out); + return out; + } + + // Perform an operation on one Tensor and produce one Tensor. This is for 1-to-1 column MapOp + // @param input shares the ownership of the Tensor (increase the ref count). + // @param output the address to a shared_ptr where the result will be placed. + // @return Status + virtual Status Compute(const std::shared_ptr &input, std::shared_ptr *output); + + // Perform an operation on Tensors from multiple columns, and produce multiple Tensors. + // This is for m-to-n column MapOp. + // @param input is a vector of shared_ptr to Tensor (pass by const reference). + // @param output is the address to an empty vector of shared_ptr to Tensor. + // @return Status + virtual Status Compute(const std::vector> &input, + std::vector> *output); + + // Returns true oif the TensorOp takes one input and returns one output. + // @return true/false + bool OneToOne() { return NumInput() == 1 && NumOutput() == 1; } + + // Function to determine the number of inputs the TensorOp can take. 0: means undefined. + // @return uint32_t + virtual uint32_t NumInput() { return 1; } + + // Function to determine the number of output the TensorOp generates. 0: means undefined. + // @return uint32_t + virtual uint32_t NumOutput() { return 1; } + + // Function to determine the shapes of the output tensor given the input tensors' shapes. + // If a subclass did not override this function, it means that the shape does not change. + // @param inputs in: vector of the shapes of the input tensors. + // @param outputs out: vector of the shapes of the output tensors to be filled. + // @return Status + virtual Status OutputShape(const std::vector &inputs, std::vector &outputs); + + // Function to determine the types of the output tensor given the input tensor's types. + // If a subclass did not override this function, it means that the type does not change. + // @param inputs in: vector of the types of the input tensors. + // @param outputs out: vector of the types of the output tensors to be filled. + // @return Status + virtual Status OutputType(const std::vector &inputs, std::vector &outputs); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_TENSOR_OP_H_ diff --git a/mindspore/ccsrc/dataset/util/.gitignore b/mindspore/ccsrc/dataset/util/.gitignore new file mode 100644 index 00000000000..b86007a5ffa --- /dev/null +++ b/mindspore/ccsrc/dataset/util/.gitignore @@ -0,0 +1,3 @@ +*.o +*.a +*~ \ No newline at end of file diff --git a/mindspore/ccsrc/dataset/util/CMakeLists.txt b/mindspore/ccsrc/dataset/util/CMakeLists.txt new file mode 100644 index 00000000000..ff14d772cad --- /dev/null +++ b/mindspore/ccsrc/dataset/util/CMakeLists.txt @@ -0,0 +1,17 @@ +add_library(utils OBJECT + arena.cc + circular_pool.cc + memory_pool.cc + cond_var.cc + semaphore.cc + intrp_service.cc + task.cc + task_manager.cc + service.cc + services.cc + lock.cc + status.cc + path.cc + wait_post.cc + sig_handler.cc + random.cc) diff --git a/mindspore/ccsrc/dataset/util/README.md b/mindspore/ccsrc/dataset/util/README.md new file mode 100644 index 00000000000..f62d77d1dfd --- /dev/null +++ b/mindspore/ccsrc/dataset/util/README.md @@ -0,0 +1,72 @@ +# Event +The header file WaitPost.h contains the implementation of an event which is a type of synchronization mechanism that is used to indicate to waiting processes when a particular condition has become true. + +An event is created with initial state set to false. It provides the following operations: +* `wait` - causes the suspension of the executing process until the state of the event is set to true. If the state is already set to true has no effect. +* `set` - sets the event's state to true, releasing all waiting processes. +* `clear` - sets the event's state to false. + +# Counting Semaphore +The header file Semaphore.h contains the implementation of counting semaphore. Conceptually, a semaphore is a nonnegative integer count. Semaphores are typically used to coordinate access to resources, with the semaphore count initialized to the number of free resources. Threads then atomically increment the count when resources are added and atomically decrement the count when resources are removed. + +When the semaphore count becomes zero, indicating that no more resources are present, threads trying to decrement the semaphore block wait until the count becomes greater than zero. + +Two operations are provided +* `P`(). Decrement the semaphore count. If the count is 0, the current thread is blocked. +* `V`(). Increment the semaphore count. Wake up one of the threads that are currently blocked. Note that the current implementation wakes up one of the blocked threads instead of waking up all of them. + +# List +It is a doubly linked structure used solely by Buffer Manager. List can used for general purpose. The reason we use a home grown linked list because Buffer Manager manages several linked lists and an element can simultaneously in more than one list. Using STL C++ container is not as efficient as the home grown linked list. + +# Consumer/Producer Queue +The header file Queue.h contains a generic implementation of producer/consumer queue. The problem describes two processes, the producer and the consumer, who share a common, fixed-size buffer used as a queue. The producer's job is to generate data, put it into the buffer, and start again. At the same time, the consumer is consuming the data (i.e., removing it from the buffer), one piece at a time. + +It has the following template signature +``` + template + class Queue { + +``` +_SIZE_ is the capacity of the queue. +_T_ is the object class that represents the data that are produced and consumed by the producer and consumer respectively. + +Initially the Queue is empty and all consumers are blocked. + +The implementation of Queue is based on counting semaphore above. + +The following operations are provided +* void `push_back`(const T&) used by producer to add data to the queue. +* T `pop_front`() used by consumer to retrieve the data from the queue. + + +# Memory Pool +Two different kinds of memory pools are provided. While they behave differently, they have identical interfaces +* void * `allocate`(size_t reqSize). It allocates memory from the pool where reqSize is the size of memory requested +* void `deallocate`(void *p) returns the memory previously acquired by allocate pointed to by p back to the memory pool +* void `Reallocate`(void **pp, size_t oldSize, size_t newSize). Enlarge or shrink the memory acquired previously by allocate to the new size. The old pointer is passed in and a new pointer (or maybe the same ond) is returned. + +C++ operator **new** and **delete** are also overloaded to make use of the customized memory pools. + +Both functions allocate and deallocate can throw `std::bad_alloc` if running out of memory from the arena. It is user's responsibility to catch the out of memory exception. + +An allocator header file Allocator.h is created to provided additional support to hook into the C++ STL container such as vector or map to allocate memory from the customized memory pools. + +## BuddyArena +The first kind of memory pool is BuddyArena. The corresponding header file is BuddyArena.h. + +BuddyArena is a general purpose arena and the constructor takes K (in unit of MB) as input. The default value is 4096 which is 4G if no value is given to the constructor. + +BuddyArena is implemented based on Buddy System. + +## CircularPool +The second kind of memory pool is CircularPool. The corresponding header file is CircularPool.h. + +CircularPool is built upon multiple BuddyArena. Initially there is one BuddyArena. More BuddyArena are gradually added to the memory pool as needed until it reaches the specified maximum capacity. There is no guarantee the newly added BuddyArena is contiguous. Maximum size of allocated block in CircularPool is determined by the maximum block allowed by a BuddyArena. By default the maximum capacity is 32G and each BuddyArena is 4G.. The constructor takes unit of GB as input. + +There are one important assumption of this kind of memory pool +* Allocated memory is not kept for the whole duration of the memory pool and will be released soon. + +User allocates memory from the _logical_ end of the pool while allocated memory will be returned to the _logical_ head of the pool. When a new BuddyArena is added to the pool, it will become the new logical end. When a BuddyArena becomes full, the next BuddyArena (in a round robin fashion) will become the new tail. + + + diff --git a/mindspore/ccsrc/dataset/util/allocator.h b/mindspore/ccsrc/dataset/util/allocator.h new file mode 100644 index 00000000000..ba6c7786df5 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/allocator.h @@ -0,0 +1,90 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_ALLOCATOR_H_ +#define DATASET_UTIL_ALLOCATOR_H_ + +#include +#include +#include +#include "dataset/util/memory_pool.h" + +namespace mindspore { +namespace dataset { +// The following conforms to the requirements of +// std::allocator. Do not rename/change any needed +// requirements, e.g. function names, typedef etc. +template +class Allocator { + public: + template + friend class Allocator; + + using value_type = T; + using pointer = T *; + using const_pointer = const T *; + using reference = T &; + using const_reference = const T &; + using size_type = uint64_t; + + template + struct rebind { + using other = Allocator; + }; + + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + + explicit Allocator(const std::shared_ptr &b) : pool_(b) {} + + ~Allocator() = default; + + template + explicit Allocator(Allocator const &rhs) : pool_(rhs.pool_) {} + + template + bool operator==(Allocator const &rhs) const { + return pool_ == rhs.pool_; + } + + template + bool operator!=(Allocator const &rhs) const { + return pool_ != rhs.pool_; + } + + pointer allocate(std::size_t n) { + void *p; + Status rc = pool_->Allocate(n * sizeof(T), &p); + if (rc.IsOk()) { + return reinterpret_cast(p); + } else if (rc.IsOutofMemory()) { + throw std::bad_alloc(); + } else { + throw std::exception(); + } + } + + void deallocate(pointer p, std::size_t n = 0) noexcept { pool_->Deallocate(p); } + + size_type max_size() { return pool_->get_max_size(); } + + private: + std::shared_ptr pool_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_ALLOCATOR_H_ diff --git a/mindspore/ccsrc/dataset/util/arena.cc b/mindspore/ccsrc/dataset/util/arena.cc new file mode 100644 index 00000000000..856f7fef24d --- /dev/null +++ b/mindspore/ccsrc/dataset/util/arena.cc @@ -0,0 +1,258 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/arena.h" +#include +#include +#include "dataset/util/make_unique.h" +#include "dataset/util/system_pool.h" +#include "dataset/util/de_error.h" +#include "./securec.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +struct MemHdr { + uint32_t sig; + uint64_t addr; + uint64_t blk_size; + MemHdr(uint64_t a, uint64_t sz) : sig(0xDEADBEEF), addr(a), blk_size(sz) {} + static void setHdr(void *p, uint64_t addr, uint64_t sz) { new (p) MemHdr(addr, sz); } + static void getHdr(void *p, MemHdr *hdr) { + auto *tmp = reinterpret_cast(p); + *hdr = *tmp; + } +}; +Status Arena::Init() { + RETURN_IF_NOT_OK(DeMalloc(size_in_MB_ * 1048576L, &ptr_, false)); + // Divide the memory into blocks. Ignore the last partial block. + uint64_t num_blks = size_in_bytes_ / ARENA_BLK_SZ; + MS_LOG(INFO) << "Size of memory pool is " << num_blks << ", number of blocks of size is " << ARENA_BLK_SZ << "."; + tr_.Insert(0, num_blks); + return Status::OK(); +} + +Status Arena::Allocate(size_t n, void **p) { + if (n == 0) { + *p = nullptr; + return Status::OK(); + } + std::unique_lock lck(mux_); + // Round up n to 1K block + uint64_t req_size = static_cast(n) + ARENA_WALL_OVERHEAD_SZ; + if (req_size > this->get_max_size()) { + RETURN_STATUS_UNEXPECTED("Request size too big : " + std::to_string(n)); + } + uint64_t reqBlk = SizeToBlk(req_size); + // Do a first fit search + auto blk = tr_.Top(); + if (blk.second && reqBlk <= blk.first.priority) { + uint64_t addr = blk.first.key; + uint64_t size = blk.first.priority; + // Trim to the required size and return the rest to the tree. + tr_.Pop(); + if (size > reqBlk) { + tr_.Insert(addr + reqBlk, size - reqBlk); + } + lck.unlock(); + char *q = static_cast(ptr_) + addr * ARENA_BLK_SZ; + MemHdr::setHdr(q, addr, reqBlk); + *p = get_user_addr(q); + } else { + return Status(StatusCode::kOutOfMemory); + } + return Status::OK(); +} + +void Arena::Deallocate(void *p) { + auto *q = get_base_addr(p); + MemHdr hdr(0, 0); + MemHdr::getHdr(q, &hdr); + DS_ASSERT(hdr.sig == 0xDEADBEEF); + // We are going to insert a free block back to the treap. But first, check if we can combine + // with the free blocks before and after to form a bigger block. + std::unique_lock lck(mux_); + // Query if we have a free block after us. + auto nextBlk = tr_.Search(hdr.addr + hdr.blk_size); + if (nextBlk.second) { + // Form a bigger block + hdr.blk_size += nextBlk.first.priority; + tr_.DeleteKey(nextBlk.first.key); + } + // Next find a block in front of us. + auto result = FindPrevBlk(hdr.addr); + if (result.second) { + // We can combine with this block + hdr.addr = result.first.first; + hdr.blk_size += result.first.second; + tr_.DeleteKey(result.first.first); + } + // Now we can insert the free node + tr_.Insert(hdr.addr, hdr.blk_size); +} + +Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) { + DS_ASSERT(pp); + DS_ASSERT(*pp); + uint64_t actual_size = static_cast(new_sz) + ARENA_WALL_OVERHEAD_SZ; + if (actual_size > this->get_max_size()) { + RETURN_STATUS_UNEXPECTED("Request size too big : " + std::to_string(new_sz)); + } + uint64_t req_blk = SizeToBlk(actual_size); + char *oldAddr = reinterpret_cast(*pp); + auto *oldHdr = get_base_addr(oldAddr); + MemHdr hdr(0, 0); + MemHdr::getHdr(oldHdr, &hdr); + DS_ASSERT(hdr.sig == 0xDEADBEEF); + std::unique_lock lck(mux_); + if (hdr.blk_size > req_blk) { + // Refresh the header with the new smaller size. + MemHdr::setHdr(oldHdr, hdr.addr, req_blk); + // Return the unused memory back to the tree. Unlike allocate, we we need to merge with the block after us. + auto next_blk = tr_.Search(hdr.addr + hdr.blk_size); + if (next_blk.second) { + hdr.blk_size += next_blk.first.priority; + tr_.DeleteKey(next_blk.first.key); + } + tr_.Insert(hdr.addr + req_blk, hdr.blk_size - req_blk); + } else if (hdr.blk_size < req_blk) { + uint64_t addr = hdr.addr; + // Attempt a block enlarge. No guarantee it is always successful. + bool success = BlockEnlarge(&addr, hdr.blk_size, req_blk); + if (success) { + auto *newHdr = static_cast(ptr_) + addr * ARENA_BLK_SZ; + MemHdr::setHdr(newHdr, addr, req_blk); + if (addr != hdr.addr) { + errno_t err = + memmove_s(get_user_addr(newHdr), (req_blk * ARENA_BLK_SZ) - ARENA_WALL_OVERHEAD_SZ, oldAddr, old_sz); + if (err) { + RETURN_STATUS_UNEXPECTED("Error from memmove: " + std::to_string(err)); + } + } + *pp = get_user_addr(newHdr); + return Status::OK(); + } + // If we reach here, allocate a new block and simply move the content from the old to the new place. + // Unlock since allocate will grab the lock again. + lck.unlock(); + return FreeAndAlloc(pp, old_sz, new_sz); + } + return Status::OK(); +} + +std::ostream &operator<<(std::ostream &os, const Arena &s) { + for (auto &it : s.tr_) { + os << "Address : " << it.key << ". Size : " << it.priority << "\n"; + } + return os; +} + +Arena::Arena(size_t val_in_MB) : ptr_(nullptr), size_in_MB_(val_in_MB), size_in_bytes_(val_in_MB * 1048576L) {} + +Status Arena::CreateArena(std::shared_ptr *p_ba, size_t val_in_MB) { + if (p_ba == nullptr) { + RETURN_STATUS_UNEXPECTED("p_ba is null"); + } + Status rc; + auto ba = new (std::nothrow) Arena(val_in_MB); + if (ba == nullptr) { + return Status(StatusCode::kOutOfMemory); + } + rc = ba->Init(); + if (rc.IsOk()) { + (*p_ba).reset(ba); + } else { + delete ba; + } + return rc; +} + +int Arena::PercentFree() const { + uint64_t sz = 0; + for (auto &it : tr_) { + sz += it.priority; + } + double ratio = static_cast(sz * ARENA_BLK_SZ) / static_cast(size_in_bytes_); + return static_cast(ratio * 100.0); +} + +uint64_t Arena::get_max_size() const { return (size_in_bytes_ - ARENA_WALL_OVERHEAD_SZ); } + +std::pair, bool> Arena::FindPrevBlk(uint64_t addr) { + for (auto &it : tr_) { + if (it.key + it.priority == addr) { + return std::make_pair(std::make_pair(it.key, it.priority), true); + } else if (it.key > addr) { + break; + } + } + return std::make_pair(std::make_pair(0, 0), false); +} + +bool Arena::BlockEnlarge(uint64_t *addr, uint64_t old_sz, uint64_t new_sz) { + uint64_t size = old_sz; + // The logic is very much identical to Deallocate. We will see if we can combine with the blocks before and after. + auto next_blk = tr_.Search(*addr + old_sz); + if (next_blk.second) { + size += next_blk.first.priority; + if (size >= new_sz) { + // In this case, we can just enlarge the block without doing any moving. + tr_.DeleteKey(next_blk.first.key); + // Return unused back to the tree. + if (size > new_sz) { + tr_.Insert(*addr + new_sz, size - new_sz); + } + } + return true; + } + // If we still get here, we have to look at the block before us. + auto result = FindPrevBlk(*addr); + if (result.second) { + // We can combine with this block together with the next block (if any) + size += result.first.second; + *addr = result.first.first; + if (size >= new_sz) { + // We can combine with this block together with the next block (if any) + tr_.DeleteKey(*addr); + if (next_blk.second) { + tr_.DeleteKey(next_blk.first.key); + } + // Return unused back to the tree. + if (size > new_sz) { + tr_.Insert(*addr + new_sz, size - new_sz); + } + return true; + } + } + return false; +} + +Status Arena::FreeAndAlloc(void **pp, size_t old_sz, size_t new_sz) { + DS_ASSERT(pp); + DS_ASSERT(*pp); + void *p = nullptr; + void *q = *pp; + RETURN_IF_NOT_OK(Allocate(new_sz, &p)); + errno_t err = memmove_s(p, new_sz, q, old_sz); + if (err) { + RETURN_STATUS_UNEXPECTED("Error from memmove: " + std::to_string(err)); + } + *pp = p; + // Free the old one. + Deallocate(q); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/arena.h b/mindspore/ccsrc/dataset/util/arena.h new file mode 100644 index 00000000000..8c5d1e1093f --- /dev/null +++ b/mindspore/ccsrc/dataset/util/arena.h @@ -0,0 +1,105 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_ARENA_H_ +#define DATASET_UTIL_ARENA_H_ + +#include +#include +#include +#include "dataset/util/memory_pool.h" +#include "dataset/util/treap.h" + +#define ARENA_LOG_BLK_SZ (6u) +#define ARENA_BLK_SZ (static_cast(1u << ARENA_LOG_BLK_SZ)) +#define ARENA_WALL_OVERHEAD_SZ 32 +namespace mindspore { +namespace dataset { +// This is a memory arena based on a treap data structure. +// The constructor of the Arena takes the size of the initial memory size (in MB). +// Internally we divide the memory into multiple blocks. Each block is 64 bytes. +// The treap contains all the free blocks with the relative memory address as key +// and the size of the block as priority. +// +// Initially the treap has only one root which is the whole memory piece. +// +// For memory suballocation, we pop the root node of the treap which contains the largest free block. +// We allocate what we need and return the rest back to the treap. We search for the first fit instead +// of the best fit so to give us a constant time in memory allocation. +// +// When a block of memory is freed. It is joined with the blocks before and after (if they are available) to +// form a bigger block. +class Arena : public MemoryPool { + public: + Arena(const Arena &) = delete; + + Arena &operator=(const Arena &) = delete; + + ~Arena() override { + if (ptr_ != nullptr) { + free(ptr_); + ptr_ = nullptr; + } + } + + Status Allocate(size_t n, void **p) override; + + Status Reallocate(void **, size_t old_sz, size_t new_sz) override; + + void Deallocate(void *) override; + + uint64_t get_max_size() const override; + + static uint64_t SizeToBlk(uint64_t sz) { + uint64_t req_blk = sz / ARENA_BLK_SZ; + if (sz % ARENA_BLK_SZ) { + ++req_blk; + } + return req_blk; + } + + int PercentFree() const override; + + const void *get_base_addr() const { return ptr_; } + + friend std::ostream &operator<<(std::ostream &os, const Arena &s); + + static Status CreateArena(std::shared_ptr *p_ba, size_t val_in_MB = 4096); + + private: + std::mutex mux_; + Treap tr_; + void *ptr_; + size_t size_in_MB_; + size_t size_in_bytes_; + + explicit Arena(size_t val_in_MB = 4096); + + std::pair, bool> FindPrevBlk(uint64_t addr); + + Status Init(); + + bool BlockEnlarge(uint64_t *addr, uint64_t old_sz, uint64_t new_sz); + + Status FreeAndAlloc(void **pp, size_t old_sz, size_t new_sz); + + void *get_user_addr(void *base_addr) const { return reinterpret_cast(base_addr) + ARENA_WALL_OVERHEAD_SZ; } + + void *get_base_addr(void *user_addr) const { return reinterpret_cast(user_addr) - ARENA_WALL_OVERHEAD_SZ; } +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_ARENA_H_ diff --git a/mindspore/ccsrc/dataset/util/auto_index.h b/mindspore/ccsrc/dataset/util/auto_index.h new file mode 100644 index 00000000000..e34d9e1288f --- /dev/null +++ b/mindspore/ccsrc/dataset/util/auto_index.h @@ -0,0 +1,89 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_AUTO_INDEX_H_ +#define DATASET_UTIL_AUTO_INDEX_H_ + +#include +#include +#include + +#include "dataset/util/btree.h" +#include "dataset/util/system_pool.h" + +namespace mindspore { +namespace dataset { +// This is a B+ tree with generated uint64_t value as key. +// Use minKey() function to query the min key. +// Use maxKey() function to query the max key. +// @tparam T +template +class AutoIndexObj : public BPlusTree { + public: + using my_tree = BPlusTree; + using key_type = typename my_tree::key_type; + using value_type = typename my_tree::value_type; + + explicit AutoIndexObj(const typename my_tree::value_allocator &alloc = Allocator{std::make_shared()}) + : my_tree::BPlusTree(alloc), inx_(kMinKey) {} + + ~AutoIndexObj() = default; + + // Insert an object into the tree. + // @param val + // @return + Status insert(const value_type &val, key_type *key = nullptr) { + key_type my_inx = inx_.fetch_add(1); + if (key) { + *key = my_inx; + } + return my_tree::DoInsert(my_inx, val); + } + + // Insert a vector of objects into the tree. + // @param v + // @return + Status insert(std::vector v) { + uint64_t num_ele = v.size(); + if (num_ele > 0) { + // reserve a range of keys rather than getting it one by one. + key_type my_inx = inx_.fetch_add(num_ele); + for (uint64_t i = 0; i < num_ele; i++) { + RETURN_IF_NOT_OK(my_tree::DoInsert(my_inx + i, v.at(i))); + } + } + return Status::OK(); + } + + // @return the minimum key + key_type min_key() const { + auto it = this->cbegin(); + return it.key(); + } + + // @return the maximum key + key_type max_key() const { + auto it = this->cend(); + --it; + return it.key(); + } + + private: + static constexpr key_type kMinKey = 1; + std::atomic inx_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_AUTO_INDEX_H_ diff --git a/mindspore/ccsrc/dataset/util/bit.h b/mindspore/ccsrc/dataset/util/bit.h new file mode 100644 index 00000000000..f02f39585cb --- /dev/null +++ b/mindspore/ccsrc/dataset/util/bit.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_BIT_H_ +#define DATASET_UTIL_BIT_H_ + +namespace mindspore { +namespace dataset { +template +Enum operator|(Enum lhs, Enum rhs) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + return static_cast(static_cast(lhs) | static_cast(rhs)); +} + +template +Enum operator&(Enum lhs, Enum rhs) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + return static_cast(static_cast(lhs) & static_cast(rhs)); +} + +template +Enum operator^(Enum lhs, Enum rhs) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + return static_cast(static_cast(lhs) ^ static_cast(rhs)); +} + +template +Enum &operator|=(Enum &lhs, Enum rhs) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + lhs = static_cast(static_cast(lhs) | static_cast(rhs)); + return lhs; +} + +template +Enum &operator&=(Enum &lhs, Enum rhs) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + lhs = static_cast(static_cast(lhs) & static_cast(rhs)); + return lhs; +} + +template +Enum &operator^=(Enum &lhs, Enum rhs) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + lhs = static_cast(static_cast(lhs) ^ static_cast(rhs)); + return lhs; +} + +template +Enum operator~(Enum v) { + static_assert(std::is_enum::value, "template parameter is not an enum type"); + using underlying = typename std::underlying_type::type; + return static_cast(~static_cast(v)); +} +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_BIT_H_ diff --git a/mindspore/ccsrc/dataset/util/btree.h b/mindspore/ccsrc/dataset/util/btree.h new file mode 100644 index 00000000000..42c0499e5fd --- /dev/null +++ b/mindspore/ccsrc/dataset/util/btree.h @@ -0,0 +1,446 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_INDEX_H_ +#define DATASET_UTIL_INDEX_H_ + +#include +#include +#include +#include +#include +#include +#include "./securec.h" +#include "dataset/util/allocator.h" +#include "dataset/util/list.h" +#include "dataset/util/lock.h" +#include "dataset/util/memory_pool.h" +#include "dataset/util/services.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Default traits for a B+ tree +struct BPlusTreeTraits { + // This determines the limit of number of keys in a node. + using slot_type = uint16_t; + // Number of slots in each leaf of the tree. + static constexpr slot_type kLeafSlots = 256; + // Number of slots in each inner node of the tree + static constexpr slot_type kInnerSlots = 128; + // If kAppendMode is true, we will split high instead of 50/50 split + static constexpr bool kAppendMode = false; +}; + +// Implementation of B+ tree +// @tparam K +// @tparam V +// @tparam C +// @tparam T +template , typename T = BPlusTreeTraits> +class BPlusTree { + public: + enum class IndexRc : char { + kOk = 0, + kDuplicateKey = 1, + kSlotFull = 2, + kKeyNotFound = 3, + kNullPointer = 4, + kOutOfMemory = 5, + kRetry = 6, + kUnexpectedError = 127 + }; +#define RETURN_IF_BAD_RC(_s) \ + do { \ + IndexRc __rc = (_s); \ + if (__rc != IndexRc::kOk) { \ + return __rc; \ + } \ + } while (false) + + Status IndexRc2Status(IndexRc rc) { + if (rc == IndexRc::kOk) { + return Status(StatusCode::kOK); + } else if (rc == IndexRc::kOutOfMemory) { + return Status(StatusCode::kOutOfMemory); + } else if (rc == IndexRc::kDuplicateKey) { + return Status(StatusCode::kDuplicateKey); + } else { + RETURN_STATUS_UNEXPECTED(std::to_string(static_cast(rc))); + } + } + + using key_type = K; + using value_type = V; + using key_compare = C; + using slot_type = typename T::slot_type; + using traits = T; + using key_allocator = Allocator; + using value_allocator = Allocator; + using slot_allocator = Allocator; + + explicit BPlusTree(const value_allocator &alloc); + + ~BPlusTree() noexcept; + + BPlusTree(const BPlusTree &) = delete; + + BPlusTree(BPlusTree &&) = delete; + + BPlusTree &operator=(const BPlusTree &) = delete; + + BPlusTree &operator=(BPlusTree &&) = delete; + + key_compare key_comp() const { return key_less_; } + + size_t size() const { return stats_.size_; } + + bool empty() const { return (size() == 0); } + + // @param key + // @param value + // @return + Status DoInsert(const key_type &key, const value_type &value); + + void PopulateNumKeys(); + + key_type KeyAtPos(uint64_t inx); + + // Statistics + struct tree_stats { + std::atomic size_; + uint32_t leaves_; + uint32_t inner_nodes_; + uint32_t level_; + bool num_keys_array_valid_; + + tree_stats() : size_(0), leaves_(0), inner_nodes_(0), level_(0), num_keys_array_valid_(false) {} + }; + + private: + // Abstract class of a node (leaf or inner) + class BaseNode { + public: + friend class BPlusTree; + + virtual bool is_leafnode() const = 0; + + virtual bool is_full() const = 0; + + explicit BaseNode(const value_allocator &alloc) : alloc_(alloc) {} + + virtual ~BaseNode() = default; + + protected: + RWLock rw_lock_; + value_allocator alloc_; + + private: + Node lru_; + }; + + uint64_t PopulateNumKeys(BaseNode *n); + + key_type KeyAtPos(BaseNode *n, uint64_t inx); + + // This control block keeps track of all the nodes we traverse on insert. + // To maximize concurrency, internal nodes are latched S. If a node split + // is required, we must releases all the latches and redo it again and change + // the latch mode from S to X. + struct LockPathCB { + enum class LockMode : char { kShared = 0, kExclusive = 1, kNone = 2 }; + + struct path { + BaseNode *node_; + bool locked_; + + path() : node_(nullptr), locked_(false) {} + + path(BaseNode *p, LockMode lockmode) : node_(p), locked_(false) { + if (lockmode == LockMode::kExclusive) { + p->rw_lock_.LockExclusive(); + locked_ = true; + } else if (lockmode == LockMode::kShared) { + p->rw_lock_.LockShared(); + locked_ = true; + } + } + }; + + LockPathCB(BPlusTree *tree, bool retryWithXlock) : self_(tree), latch_shared_(true) { + if (retryWithXlock) { + latch_shared_ = false; + } + if (latch_shared_) { + tree->rw_lock_.LockShared(); + } else { + tree->rw_lock_.LockExclusive(); + } + } + + ~LockPathCB() noexcept { + // Make sure all locks are released. + while (!paths_.empty()) { + path p = paths_.back(); + paths_.pop_back(); + if (p.locked_) { + p.node_->rw_lock_.Unlock(); + } + } + self_->rw_lock_.Unlock(); + self_ = nullptr; + } + + void LockNode(BaseNode *p, LockMode locktype) { paths_.emplace_back(p, locktype); } + + void UnlockMyParents(BaseNode *me) { + path p = paths_.front(); + while (p.node_ != me) { + if (p.locked_) { + p.node_->rw_lock_.Unlock(); + } + paths_.pop_front(); + p = paths_.front(); + } + } + + BPlusTree *self_; + std::deque paths_; + bool latch_shared_; + }; + + // Definition of inner node which fans to either inner node or leaf node. + class InnerNode : public BaseNode { + public: + friend class BPlusTree; + + using alloc_type = typename value_allocator::template rebind::other; + + bool is_leafnode() const override { return false; } + + bool is_full() const override { return (slotuse_ == traits::kInnerSlots); } + + IndexRc Sort(); + + // 50/50 split + IndexRc Split(InnerNode *to, key_type *split_key); + + IndexRc InsertIntoSlot(slot_type slot, const key_type &key, BaseNode *ptr); + + explicit InnerNode(const value_allocator &alloc) : BaseNode::BaseNode(alloc), slotuse_(0) {} + + ~InnerNode() = default; + + slot_type slot_dir_[traits::kInnerSlots]; + key_type keys_[traits::kInnerSlots]; + BaseNode *data_[traits::kInnerSlots + 1] = {nullptr}; + uint64_t num_keys_[traits::kInnerSlots + 1] = {0}; + slot_type slotuse_; + }; + + // Definition of a leaf node which contains the key/value pair + class LeafNode : public BaseNode { + public: + friend class BPlusTree; + + using alloc_type = typename value_allocator::template rebind::other; + Node link_; + + bool is_leafnode() const override { return true; } + + bool is_full() const override { return (slotuse_ == traits::kLeafSlots); } + + IndexRc Sort(); + + // 50/50 split + IndexRc Split(LeafNode *to); + + IndexRc InsertIntoSlot(LockPathCB *insCB, slot_type slot, const key_type &key, std::shared_ptr value); + + explicit LeafNode(const value_allocator &alloc) : BaseNode::BaseNode(alloc), slotuse_(0) {} + + ~LeafNode() = default; + + slot_type slot_dir_[traits::kLeafSlots]; + key_type keys_[traits::kLeafSlots]; + std::shared_ptr data_[traits::kLeafSlots]; + slot_type slotuse_; + }; + + RWLock rw_lock_; + value_allocator alloc_; + // All the leaf nodes. Used by the iterator to traverse all the key/values. + List leaf_nodes_; + // All the nodes (inner + leaf). Used by the destructor to free the memory of all the nodes. + List all_; + // Pointer to the root of the tree. + BaseNode *root_; + // Key comparison object + key_compare key_less_; + // Stat + tree_stats stats_; + + bool LessThan(const key_type &a, const key_type &b) const { return key_less_(a, b); } + + bool EqualOrLessThan(const key_type &a, const key_type &b) const { return !key_less_(b, a); } + + bool Equal(const key_type &a, const key_type &b) const { return !key_less_(a, b) && !key_less_(b, a); } + + IndexRc AllocateInner(InnerNode **p); + + IndexRc AllocateLeaf(LeafNode **p); + + template + slot_type FindSlot(const node_type *node, const key_type &key, bool *duplicate = nullptr) const { + slot_type lo = 0; + while (lo < node->slotuse_ && key_comp()(node->keys_[node->slot_dir_[lo]], key)) { + ++lo; + } + bool keymatch = (lo < node->slotuse_ && Equal(key, node->keys_[node->slot_dir_[lo]])); + if (keymatch && !node->is_leafnode()) { + // For an inner node and we match a key during search, we should look into the next slot. + ++lo; + } + if (duplicate != nullptr) { + *duplicate = keymatch; + } + return lo; + } + + IndexRc LeafInsertKeyValue(LockPathCB *ins_cb, LeafNode *node, const key_type &key, std::shared_ptr value, + key_type *split_key, LeafNode **split_node); + + IndexRc InnerInsertKeyChild(InnerNode *node, const key_type &key, BaseNode *ptr, key_type *split_key, + InnerNode **split_node); + + inline BaseNode *FindBranch(InnerNode *inner, slot_type slot) const { + BaseNode *child = nullptr; + if (slot == 0) { + child = inner->data_[0]; + } else { + child = inner->data_[inner->slot_dir_[slot - 1] + 1]; + } + return child; + } + + IndexRc InsertKeyValue(LockPathCB *ins_cb, BaseNode *n, const key_type &key, std::shared_ptr value, + key_type *split_key, BaseNode **split_node); + + IndexRc Locate(BaseNode *top, const key_type &key, LeafNode **ln, slot_type *s) const; + + public: + class Iterator : public std::iterator { + public: + using reference = BPlusTree::value_type &; + using pointer = BPlusTree::value_type *; + + explicit Iterator(BPlusTree *btree) : cur_(btree->leaf_nodes_.head), slot_(0) {} + + Iterator(LeafNode *leaf, slot_type slot) : cur_(leaf), slot_(slot) {} + + ~Iterator() = default; + + pointer operator->() const { return cur_->data_[cur_->slot_dir_[slot_]].get(); } + + reference operator*() const { return *(cur_->data_[cur_->slot_dir_[slot_]].get()); } + + const key_type &key() { return cur_->keys_[cur_->slot_dir_[slot_]]; } + + const value_type &value() { return *(cur_->data_[cur_->slot_dir_[slot_]].get()); } + + // Prefix++ + Iterator &operator++(); + + // Postfix++ + Iterator operator++(int); + + // Prefix-- + Iterator &operator--(); + + // Postfix-- + Iterator operator--(int); + + bool operator==(const Iterator &x) const { return (x.cur_ == cur_) && (x.slot_ == slot_); } + + bool operator!=(const Iterator &x) const { return (x.cur_ != cur_) || (x.slot_ != slot_); } + + private: + typename BPlusTree::LeafNode *cur_; + slot_type slot_; + }; + + class ConstIterator : public std::iterator { + public: + using reference = BPlusTree::value_type &; + using pointer = BPlusTree::value_type *; + + explicit ConstIterator(const BPlusTree *btree) : cur_(btree->leaf_nodes_.head), slot_(0) {} + + ~ConstIterator() = default; + + ConstIterator(const LeafNode *leaf, slot_type slot) : cur_(leaf), slot_(slot) {} + + pointer operator->() const { return cur_->data_[cur_->slot_dir_[slot_]].get(); } + + reference operator*() const { return *(cur_->data_[cur_->slot_dir_[slot_]].get()); } + + const key_type &key() const { return cur_->keys_[cur_->slot_dir_[slot_]]; } + + const value_type &value() const { return *(cur_->data_[cur_->slot_dir_[slot_]].get()); } + + // Prefix++ + ConstIterator &operator++(); + + // Postfix++ + ConstIterator operator++(int); + + // Prefix-- + ConstIterator &operator--(); + + // Postfix-- + ConstIterator operator--(int); + + bool operator==(const ConstIterator &x) const { return (x.cur_ == cur_) && (x.slot_ == slot_); } + + bool operator!=(const ConstIterator &x) const { return (x.cur_ != cur_) || (x.slot_ != slot_); } + + private: + const typename BPlusTree::LeafNode *cur_; + slot_type slot_; + }; + + Iterator begin(); + + Iterator end(); + + ConstIterator begin() const; + + ConstIterator end() const; + + ConstIterator cbegin() const; + + ConstIterator cend() const; + + // Locate the entry with key + ConstIterator Search(const key_type &key) const; + + value_type operator[](key_type key); +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_INDEX_H_ + +#include "btree_impl.tpp" +#include "btree_iterator.tpp" diff --git a/mindspore/ccsrc/dataset/util/btree_impl.tpp b/mindspore/ccsrc/dataset/util/btree_impl.tpp new file mode 100644 index 00000000000..dab94002d5e --- /dev/null +++ b/mindspore/ccsrc/dataset/util/btree_impl.tpp @@ -0,0 +1,561 @@ +/* Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_BTREE_H_ +#define DATASET_UTIL_BTREE_H_ + +#include "btree.h" + +namespace mindspore { +namespace dataset { +template +typename BPlusTree::IndexRc BPlusTree::InnerNode::Sort() { + // Build an inverse map. Basically it means keys[i] should be relocated to keys[inverse[i]]; + Allocator alloc(this->alloc_); + slot_type *inverse = nullptr; + try { + inverse = alloc.allocate(traits::kInnerSlots); + } catch (std::bad_alloc &e) { + return IndexRc::kOutOfMemory; + } catch (std::exception &e) { + return IndexRc::kUnexpectedError; + } + + for (slot_type i = 0; i < slotuse_; i++) { + inverse[slot_dir_[i]] = i; + } + for (slot_type i = 0; i < slotuse_; i++) { + while (inverse[i] != i) { + slot_type j = inverse[i]; + slot_type k = inverse[j]; + // Swap the key + std::swap(keys_[j], keys_[i]); + // Swap the pointers. + std::swap(data_[j + 1], data_[i + 1]); + // one key in order. + inverse[j] = j; + // continue to move + inverse[i] = k; + } + slot_dir_[i] = i; + } + if (inverse != nullptr) { + alloc.deallocate(inverse); + inverse = nullptr; + } + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc BPlusTree::InnerNode::Split(BPlusTree::InnerNode *to, + key_type *split_key) { + DS_ASSERT(to); + DS_ASSERT(to->slotuse_ == 0); + // It is simpler to sort first, then split. Other alternative is to move key by key to the + // new node. Also we need to deal with the 'holes' after a key is moved. + RETURN_IF_BAD_RC(this->Sort()); + slot_type mid = slotuse_ >> 1; + slot_type num_keys_to_move = slotuse_ - (mid + 1); + *split_key = keys_[mid]; + errno_t err = memmove_s(to->keys_, sizeof(to->keys_), keys_ + mid + 1, num_keys_to_move * sizeof(key_type)); + if (err != EOK) { + return IndexRc::kUnexpectedError; + } + err = memcpy_s(to->data_, sizeof(to->data_), data_ + mid + 1, (num_keys_to_move + 1) * sizeof(BaseNode * )); + if (err != EOK) { + return IndexRc::kUnexpectedError; + } + for (slot_type i = 0; i < num_keys_to_move; i++) { + to->slot_dir_[i] = i; + } + slotuse_ -= (num_keys_to_move + 1); // the split key is moved up. So one less + to->slotuse_ += num_keys_to_move; + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc +BPlusTree::InnerNode::InsertIntoSlot(slot_type slot, const key_type &key, + BPlusTree::BaseNode *ptr) { + if (is_full()) { + return IndexRc::kSlotFull; + } + // Shift the slot entries to the right and make room for the new comer. + // We don't sort the key and/or the data array until node split + auto num_keys_to_move = slotuse_ - slot; + if (num_keys_to_move > 0) { + auto *src = &slot_dir_[slot]; + auto *dest = &slot_dir_[slot + 1]; + auto destMax = sizeof(slot_dir_) - sizeof(slot_type) * (slot + 1); + auto amt = sizeof(slot_type) * num_keys_to_move; + errno_t err = memmove_s(dest, destMax, src, amt); + if (err) { + return IndexRc::kUnexpectedError; + } + } + slot_dir_[slot] = slotuse_; + keys_[slotuse_] = key; + data_[slotuse_ + 1] = ptr; + ++slotuse_; + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc BPlusTree::LeafNode::Sort() { + // Build an inverse map. Basically it means keys[i] should be relocated to keys[inverse[i]]; + Allocator alloc(this->alloc_); + slot_type *inverse = nullptr; + try { + inverse = alloc.allocate(traits::kLeafSlots); + } catch (std::bad_alloc &e) { + return IndexRc::kOutOfMemory; + } catch (std::exception &e) { + return IndexRc::kUnexpectedError; + } + + for (slot_type i = 0; i < slotuse_; i++) { + inverse[slot_dir_[i]] = i; + } + for (slot_type i = 0; i < slotuse_; i++) { + while (inverse[i] != i) { + slot_type j = inverse[i]; + slot_type k = inverse[j]; + // Swap the key + std::swap(keys_[j], keys_[i]); + // Swap the shared pointers + std::swap(data_[j], data_[i]); + // one key in order. + inverse[j] = j; + // continue to move + inverse[i] = k; + } + slot_dir_[i] = i; + } + if (inverse != nullptr) { + alloc.deallocate(inverse); + inverse = nullptr; + } + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc BPlusTree::LeafNode::Split(BPlusTree::LeafNode *to) { + DS_ASSERT(to); + DS_ASSERT(to->slotuse_ == 0); + // It is simpler to sort first, then split. Other alternative is to move key by key to the + // new node. Also we need to deal with the 'holes' after a key is moved. + RETURN_IF_BAD_RC(this->Sort()); + slot_type mid = slotuse_ >> 1; + slot_type num_keys_to_move = slotuse_ - mid; + errno_t err = memmove_s(to->keys_, sizeof(to->keys_), keys_ + mid, num_keys_to_move * sizeof(key_type)); + if (err) { + return IndexRc::kUnexpectedError; + } + for (slot_type i = 0; i < num_keys_to_move; i++) { + to->data_[i] = std::move(data_[i + mid]); + to->slot_dir_[i] = i; + } + slotuse_ -= num_keys_to_move; + to->slotuse_ += num_keys_to_move; + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc +BPlusTree::LeafNode::InsertIntoSlot(BPlusTree::LockPathCB *insCB, slot_type slot, + const key_type &key, + std::shared_ptr value) { + if (is_full()) { + // If we need to do node split, we need to ensure all the intermediate nodes are locked exclusive. + // Otherwise we need to do a retry. + if (insCB == nullptr || !insCB->latch_shared_) { + return IndexRc::kSlotFull; + } else { + return IndexRc::kRetry; + } + } + // We can now let go all the locks of the parent. Nothing we do from now on will change the + // structure of the tree. + if (insCB) { + insCB->UnlockMyParents(this); + } + // Shift the slot entries to the right and make room for the new comer. + // We don't sort the key and/or the data array until node split + auto num_keys_to_move = slotuse_ - slot; + if (num_keys_to_move > 0) { + auto *src = &slot_dir_[slot]; + auto *dest = &slot_dir_[slot + 1]; + auto destMax = sizeof(slot_dir_) - sizeof(slot_type) * (slot + 1); + auto amt = sizeof(slot_type) * num_keys_to_move; + errno_t err = memmove_s(dest, destMax, src, amt); + if (err) { + return IndexRc::kUnexpectedError; + } + } + slot_dir_[slot] = slotuse_; + keys_[slotuse_] = key; + data_[slotuse_] = std::move(value); + ++slotuse_; + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc BPlusTree::AllocateInner(BPlusTree::InnerNode **p) { + if (p == nullptr) { + return IndexRc::kNullPointer; + } + typename InnerNode::alloc_type alloc(alloc_); + InnerNode *ptr = nullptr; + try { + ptr = alloc.allocate(1); + } catch (std::bad_alloc &e) { + return IndexRc::kOutOfMemory; + } catch (std::exception &e) { + return IndexRc::kUnexpectedError; + } + *p = new(ptr) InnerNode(alloc_); + all_.Prepend(ptr); + stats_.inner_nodes_++; + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc BPlusTree::AllocateLeaf(BPlusTree::LeafNode **p) { + if (p == nullptr) { + return IndexRc::kNullPointer; + } + typename LeafNode::alloc_type alloc(this->alloc_); + LeafNode *ptr = nullptr; + try { + ptr = alloc.allocate(1); + } catch (std::bad_alloc &e) { + return IndexRc::kOutOfMemory; + } catch (std::exception &e) { + return IndexRc::kUnexpectedError; + } + *p = new(ptr) LeafNode(alloc_); + all_.Prepend(ptr); + stats_.leaves_++; + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc +BPlusTree::LeafInsertKeyValue(BPlusTree::LockPathCB *ins_cb, + BPlusTree::LeafNode *node, const key_type &key, + std::shared_ptr value, key_type *split_key, + BPlusTree::LeafNode **split_node) { + bool duplicate; + slot_type slot = FindSlot(node, key, &duplicate); + if (duplicate) { + return IndexRc::kDuplicateKey; + } + IndexRc rc = node->InsertIntoSlot(ins_cb, slot, key, value); + if (rc == IndexRc::kSlotFull) { + LeafNode *new_leaf = nullptr; + rc = AllocateLeaf(&new_leaf); + RETURN_IF_BAD_RC(rc); + leaf_nodes_.InsertAfter(node, new_leaf); + *split_node = new_leaf; + if (slot == node->slotuse_ && traits::kAppendMode) { + // Split high. Good for bulk load and keys are in asending order on insert + *split_key = key; + // Just insert the new key to the new leaf. No further need to move the keys + // from one leaf to the other. + rc = new_leaf->InsertIntoSlot(nullptr, 0, key, value); + RETURN_IF_BAD_RC(rc); + } else { + // 50/50 split + rc = node->Split(new_leaf); + RETURN_IF_BAD_RC(rc); + *split_key = new_leaf->keys_[0]; + if (LessThan(key, *split_key)) { + rc = node->InsertIntoSlot(nullptr, slot, key, value); + RETURN_IF_BAD_RC(rc); + } else { + slot -= node->slotuse_; + rc = new_leaf->InsertIntoSlot(nullptr, slot, key, value); + RETURN_IF_BAD_RC(rc); + } + } + } + return rc; +} + +template +typename BPlusTree::IndexRc +BPlusTree::InnerInsertKeyChild(BPlusTree::InnerNode *node, const key_type &key, + BPlusTree::BaseNode *ptr, + key_type *split_key, BPlusTree::InnerNode **split_node) { + bool duplicate; + slot_type slot = FindSlot(node, key, &duplicate); + if (duplicate) { + return IndexRc::kDuplicateKey; + } + IndexRc rc = node->InsertIntoSlot(slot, key, ptr); + if (rc == IndexRc::kSlotFull) { + InnerNode *new_inner = nullptr; + rc = AllocateInner(&new_inner); + RETURN_IF_BAD_RC(rc); + *split_node = new_inner; + if (slot == node->slotuse_ && traits::kAppendMode) { + *split_key = key; + new_inner->data_[0] = node->data_[node->slotuse_]; + rc = new_inner->InsertIntoSlot(0, key, ptr); + RETURN_IF_BAD_RC(rc); + } else { + rc = node->Split(new_inner, split_key); + RETURN_IF_BAD_RC(rc); + if (LessThan(key, *split_key)) { + // Need to readjust the slot position since the split key is no longer in the two children. + slot = FindSlot(node, key); + rc = node->InsertIntoSlot(slot, key, ptr); + RETURN_IF_BAD_RC(rc); + } else { + // Same reasoning as above + slot = FindSlot(new_inner, key); + rc = new_inner->InsertIntoSlot(slot, key, ptr); + RETURN_IF_BAD_RC(rc); + } + } + } + return rc; +} + +template +typename BPlusTree::IndexRc +BPlusTree::InsertKeyValue(BPlusTree::LockPathCB *ins_cb, BPlusTree::BaseNode *n, + const key_type &key, + std::shared_ptr value, key_type *split_key, + BPlusTree::BaseNode **split_node) { + if (split_key == nullptr || split_node == nullptr) { + return IndexRc::kUnexpectedError; + } + if (n->is_leafnode()) { + if (ins_cb) { + // Always lock the leaf in X. + ins_cb->LockNode(n, LockPathCB::LockMode::kExclusive); + } + auto *leaf = static_cast(n); + LeafNode *new_leaf = nullptr; + RETURN_IF_BAD_RC(LeafInsertKeyValue(ins_cb, leaf, key, std::move(value), split_key, &new_leaf)); + if (new_leaf) { + *split_node = new_leaf; + } + } else { + if (ins_cb) { + // For internal node, lock in S unless we are doing retry. + if (ins_cb->latch_shared_) { + ins_cb->LockNode(n, LockPathCB::LockMode::kShared); + } else { + ins_cb->LockNode(n, LockPathCB::LockMode::kExclusive); + } + } + auto *inner = static_cast(n); + slot_type slot = FindSlot(inner, key); + BaseNode *new_child = nullptr; + key_type new_key = key_type(); + RETURN_IF_BAD_RC(InsertKeyValue(ins_cb, FindBranch(inner, slot), key, std::move(value), &new_key, &new_child)); + if (new_child) { + InnerNode *new_inner = nullptr; + RETURN_IF_BAD_RC(InnerInsertKeyChild(inner, new_key, new_child, split_key, &new_inner)); + if (new_inner) { + *split_node = new_inner; + } + } + } + return IndexRc::kOk; +} + +template +typename BPlusTree::IndexRc +BPlusTree::Locate(BPlusTree::BaseNode *top, const key_type &key, + BPlusTree::LeafNode **ln, + slot_type *s) const { + if (ln == nullptr || s == nullptr) { + return IndexRc::kNullPointer; + } + if (top == nullptr) { + return IndexRc::kKeyNotFound; + } + if (top->is_leafnode()) { + bool duplicate; + auto *leaf = static_cast(top); + slot_type slot = FindSlot(leaf, key, &duplicate); + // Need exact match. + if (duplicate) { + *ln = leaf; + *s = slot; + } else { + return IndexRc::kKeyNotFound; + } + } else { + auto *inner = static_cast(top); + slot_type slot = FindSlot(inner, key); + return Locate(FindBranch(inner, slot), key, ln, s); + } + return IndexRc::kOk; +} + +template +BPlusTree::BPlusTree(const value_allocator &alloc) + : alloc_(alloc), leaf_nodes_(&LeafNode::link_), all_(&BaseNode::lru_), root_(nullptr) {} + +template +BPlusTree::~BPlusTree() noexcept { + // We have a list of all the nodes allocated. Traverse them and free all the memory + BaseNode *n = all_.head; + BaseNode *t = nullptr; + while (n) { + t = n->lru_.next; + all_.Remove(n); + if (n->is_leafnode()) { + auto *leaf = static_cast(n); + typename LeafNode::alloc_type alloc(alloc_); + leaf->~LeafNode(); + alloc.deallocate(leaf, 1); + } else { + auto *in = static_cast(n); + typename InnerNode::alloc_type alloc(alloc_); + in->~InnerNode(); + alloc.deallocate(in, 1); + } + n = t; + } + root_ = nullptr; +} + +template +Status BPlusTree::DoInsert(const key_type &key, const value_type &value) { + IndexRc rc; + if (root_ == nullptr) { + UniqueLock lck(&rw_lock_); + // Check again after we get the lock. Other thread may have created the root node already. + if (root_ == nullptr) { + LeafNode *leaf = nullptr; + rc = AllocateLeaf(&leaf); + if (rc != IndexRc::kOk) { + return IndexRc2Status(rc); + } + leaf_nodes_.Append(leaf); + root_ = leaf; + } + // lock will be unlocked when it goes out of scope. + } + bool retry = false; + do { + // Track all the paths to the target and lock each internal node in S. + LockPathCB InsCB(this, retry); + // Mark the numKeysArray invalid. We may latch the tree in S and multiple guys are doing insert. + // But it is okay as we all set the same value. + stats_.num_keys_array_valid_ = false; + // Initially we lock path in S unless we need to do node split. + retry = false; + BaseNode *new_child = nullptr; + key_type new_key = key_type(); + // We don't store the value directly into the leaf node as it is expensive to move it during node split. + // Rather we store a pointer instead. The value_type must support the copy constructor. + std::shared_ptr ptr_value = std::make_shared(value); + rc = InsertKeyValue(&InsCB, root_, key, std::move(ptr_value), &new_key, &new_child); + if (rc == IndexRc::kRetry) { + retry = true; + } else if (rc != IndexRc::kOk) { + return IndexRc2Status(rc); + } else if (new_child != nullptr) { + // root is full + InnerNode *new_root = nullptr; + rc = AllocateInner(&new_root); + if (rc == IndexRc::kOk) { + rc = new_root->InsertIntoSlot(0, new_key, new_child); + if (rc != IndexRc::kOk) { + return IndexRc2Status(rc); + } + new_root->data_[0] = root_; + root_ = new_root; + stats_.level_++; + } else { + return IndexRc2Status(rc); + } + } + } while (retry); + (void) stats_.size_++; + return Status::OK(); +} + +template +void BPlusTree::PopulateNumKeys() { + // Start from the root and we calculate how many leaf nodes as pointed to by each inner node. + // The results are stored in the numKeys array in each inner node. + (void)PopulateNumKeys(root_); + // Indicate the result is accurate since we have the tree locked exclusive. + stats_.num_keys_array_valid_ = true; +} + +template +uint64_t BPlusTree::PopulateNumKeys(BPlusTree::BaseNode *n) { + if (n->is_leafnode()) { + auto *leaf = static_cast(n); + return leaf->slotuse_; + } else { + auto *inner = static_cast(n); + uint64_t num_keys = 0; + for (auto i = 0; i < inner->slotuse_ + 1; i++) { + inner->num_keys_[i] = PopulateNumKeys(inner->data_[i]); + num_keys += inner->num_keys_[i]; + } + return num_keys; + } +} + +template +typename BPlusTree::key_type BPlusTree::KeyAtPos(uint64_t inx) { + if (stats_.num_keys_array_valid_ == false) { + // We need exclusive access to the tree. If concurrent insert is going on, it is hard to get accurate numbers + UniqueLock lck(&rw_lock_); + // Check again. + if (stats_.num_keys_array_valid_ == false) { + PopulateNumKeys(); + } + } + // Now we know how many keys each inner branch contains, we can now traverse the correct node in log n time. + return KeyAtPos(root_, inx); +} + +template +typename BPlusTree::key_type BPlusTree::KeyAtPos(BPlusTree::BaseNode *n, uint64_t inx) { + if (n->is_leafnode()) { + auto *leaf = static_cast(n); + return leaf->keys_[leaf->slot_dir_[inx]]; + } else { + auto *inner = static_cast(n); + if ((inx + 1) > inner->num_keys_[0]) { + inx -= inner->num_keys_[0]; + } else { + return KeyAtPos(inner->data_[0], inx); + } + for (auto i = 0; i < inner->slotuse_; i++) { + if ((inx + 1) > inner->num_keys_[inner->slot_dir_[i] + 1]) { + inx -= inner->num_keys_[inner->slot_dir_[i]+1]; + } else { + return KeyAtPos(inner->data_[inner->slot_dir_[i] + 1], inx); + } + } + } + // If we get here, inx is way too big. Instead of throwing exception, we will just return the default value + // of key_type whatever it is. + return key_type(); +} +} // namespace dataset +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/dataset/util/btree_iterator.tpp b/mindspore/ccsrc/dataset/util/btree_iterator.tpp new file mode 100644 index 00000000000..5677b581bb9 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/btree_iterator.tpp @@ -0,0 +1,186 @@ +/* COPYRIGHT 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_BTREE_ITERATOR_H_ +#define DATASET_UTIL_BTREE_ITERATOR_H_ + +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" +#include "btree.h" + +namespace mindspore { +namespace dataset { +template +typename BPlusTree::Iterator &BPlusTree::Iterator::operator++() { + if (slot_ + 1u < cur_->slotuse_) { + ++slot_; + } else if (cur_->link_.next) { + cur_ = cur_->link_.next; + slot_ = 0; + } else { + slot_ = cur_->slotuse_; + } + return *this; +} + +template +typename BPlusTree::Iterator BPlusTree::Iterator::operator++(int) { + Iterator tmp = *this; + if (slot_ + 1u < cur_->slotuse_) { + ++slot_; + } else if (cur_->link_.next) { + cur_ = cur_->link_.next; + slot_ = 0; + } else { + slot_ = cur_->slotuse_; + } + return tmp; +} + +template +typename BPlusTree::Iterator &BPlusTree::Iterator::operator--() { + if (slot_ > 0) { + --slot_; + } else if (cur_->link_.prev) { + cur_ = cur_->link_.prev; + slot_ = cur_->slotuse_ - 1; + } else { + slot_ = 0; + } + return *this; +} + +template +typename BPlusTree::Iterator BPlusTree::Iterator::operator--(int) { + Iterator tmp = *this; + if (slot_ > 0) { + --slot_; + } else if (cur_->link_.prev) { + cur_ = cur_->link_.prev; + slot_ = cur_->slotuse_ - 1; + } else { + slot_ = 0; + } + return tmp; +} + +template +typename BPlusTree::ConstIterator &BPlusTree::ConstIterator::operator++() { + if (slot_ + 1u < cur_->slotuse_) { + ++slot_; + } else if (cur_->link_.next) { + cur_ = cur_->link_.next; + slot_ = 0; + } else { + slot_ = cur_->slotuse_; + } + return *this; +} + +template +typename BPlusTree::ConstIterator BPlusTree::ConstIterator::operator++(int) { + Iterator tmp = *this; + if (slot_ + 1u < cur_->slotuse_) { + ++slot_; + } else if (cur_->link_.next) { + cur_ = cur_->link_.next; + slot_ = 0; + } else { + slot_ = cur_->slotuse_; + } + return tmp; +} + +template +typename BPlusTree::ConstIterator &BPlusTree::ConstIterator::operator--() { + if (slot_ > 0) { + --slot_; + } else if (cur_->link_.prev) { + cur_ = cur_->link_.prev; + slot_ = cur_->slotuse_ - 1; + } else { + slot_ = 0; + } + return *this; +} + +template +typename BPlusTree::ConstIterator BPlusTree::ConstIterator::operator--(int) { + Iterator tmp = *this; + if (slot_ > 0) { + --slot_; + } else if (cur_->link_.prev) { + cur_ = cur_->link_.prev; + slot_ = cur_->slotuse_ - 1; + } else { + slot_ = 0; + } + return tmp; +} + +template +typename BPlusTree::ConstIterator BPlusTree::Search(const key_type &key) const { + if (root_ != nullptr) { + LeafNode *leaf = nullptr; + slot_type slot; + IndexRc rc = Locate(root_, key, &leaf, &slot); + if (rc == IndexRc::kOk) { + return ConstIterator(leaf, slot); + } else { + MS_LOG(INFO) << "Key not found. rc = " << static_cast(rc) << "."; + return end(); + } + } else { + return end(); + } +} + +template +typename BPlusTree::value_type BPlusTree::operator[](key_type key) { + ConstIterator it = Search(key); + return it.value(); +} + +template +typename BPlusTree::Iterator BPlusTree::begin() { + return Iterator(this); +} + +template +typename BPlusTree::Iterator BPlusTree::end() { + return Iterator(this->leaf_nodes_.tail, this->leaf_nodes_.tail ? this->leaf_nodes_.tail->slotuse_ : 0); +} + +template +typename BPlusTree::ConstIterator BPlusTree::begin() const { + return ConstIterator(this); +} + +template +typename BPlusTree::ConstIterator BPlusTree::end() const { + return ConstIterator(this->leaf_nodes_.tail, this->leaf_nodes_.tail ? this->leaf_nodes_.tail->slotuse_ : 0); +} + +template +typename BPlusTree::ConstIterator BPlusTree::cbegin() const { + return ConstIterator(this); +} + +template +typename BPlusTree::ConstIterator BPlusTree::cend() const { + return ConstIterator(this->leaf_nodes_.tail, this->leaf_nodes_.tail ? this->leaf_nodes_.tail->slotuse_ : 0); +} +} // namespace dataset +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/dataset/util/circular_pool.cc b/mindspore/ccsrc/dataset/util/circular_pool.cc new file mode 100644 index 00000000000..f6e43f35bf1 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/circular_pool.cc @@ -0,0 +1,224 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/circular_pool.h" + +#include +#include +#include + +#include "./securec.h" + +#include "dataset/util/make_unique.h" +#include "dataset/util/system_pool.h" + +namespace mindspore { +namespace dataset { +Status CircularPool::AddOneArena() { + Status rc; + std::shared_ptr b; + RETURN_IF_NOT_OK(Arena::CreateArena(&b, arena_size_)); + tail_ = b.get(); + cur_size_in_mb_ += arena_size_; + mem_segments_.push_back(std::move(b)); + return Status::OK(); +} + +ListOfArenas::iterator CircularPool::CircularIterator::Next() { + ListOfArenas::iterator it = dp_->mem_segments_.begin(); + uint32_t size = dp_->mem_segments_.size(); + // This is what we return + it += cur_; + // Prepare for the next round + cur_++; + if (cur_ == size) { + if (start_ == 0) { + has_next_ = false; + } else { + wrap_ = true; + cur_ = 0; + } + } else if (cur_ == start_) { + has_next_ = false; + } + return it; +} + +bool CircularPool::CircularIterator::has_next() const { return has_next_; } + +void CircularPool::CircularIterator::Reset() { + wrap_ = false; + has_next_ = false; + if (!dp_->mem_segments_.empty()) { + // Find the buddy arena that corresponds to the tail. + cur_tail_ = dp_->tail_; + auto list_end = dp_->mem_segments_.end(); + auto it = std::find_if(dp_->mem_segments_.begin(), list_end, + [this](const std::shared_ptr &b) { return b.get() == cur_tail_; }); + DS_ASSERT(it != list_end); + start_ = std::distance(dp_->mem_segments_.begin(), it); + cur_ = start_; + has_next_ = true; + } +} + +CircularPool::CircularIterator::CircularIterator(CircularPool *dp) : dp_(dp) { Reset(); } + +Status CircularPool::Allocate(size_t n, void **p) { + if (p == nullptr) { + RETURN_STATUS_UNEXPECTED("p is null"); + } + Status rc; + void *ptr = nullptr; + do { + SharedLock lock_s(&rw_lock_); + int prevSzInMB = cur_size_in_mb_; + bool move_tail = false; + CircularIterator cirIt(this); + while (cirIt.has_next()) { + auto it = cirIt.Next(); + Arena *ba = it->get(); + // If we are asked to move forward the tail + if (move_tail) { + Arena *expected = cirIt.cur_tail_; + (void)atomic_compare_exchange_weak(&tail_, &expected, ba); + move_tail = false; + } + rc = ba->Allocate(n, &ptr); + if (rc.IsOk()) { + *p = ptr; + break; + } else if (rc.IsOutofMemory()) { + // Make the next arena a new tail and continue. + move_tail = true; + } else { + return rc; + } + } + + // Handle the case we have done one round robin search. + if (ptr == nullptr) { + // If we have room to expand. + if (unlimited_ || cur_size_in_mb_ < max_size_in_mb_) { + // lock in exclusively mode. + lock_s.Upgrade(); + // Check again if someone has already expanded. + if (cur_size_in_mb_ == prevSzInMB) { + RETURN_IF_NOT_OK(AddOneArena()); + } + // Re-acquire the shared lock and try again + lock_s.Downgrade(); + } else { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__); + } + } + } while (ptr == nullptr); + return rc; +} + +void CircularPool::Deallocate(void *p) { + // Lock in the chain in shared mode and find out which + // segment it comes from + SharedLock lock(&rw_lock_); + auto it = std::find_if(mem_segments_.begin(), mem_segments_.end(), [p](std::shared_ptr &b) -> bool { + char *q = reinterpret_cast(p); + char *base = const_cast(reinterpret_cast(b->get_base_addr())); + return (q > base && q < base + b->get_max_size()); + }); + lock.Unlock(); + it->get()->Deallocate(p); +} + +Status CircularPool::Reallocate(void **pp, size_t old_sz, size_t new_sz) { + // Lock in the chain in shared mode and find out which + // segment it comes from + if (pp == nullptr) { + RETURN_STATUS_UNEXPECTED("pp is null"); + } + void *p = *pp; + SharedLock lock(&rw_lock_); + auto it = std::find_if(mem_segments_.begin(), mem_segments_.end(), [p](std::shared_ptr &b) -> bool { + char *q = reinterpret_cast(p); + char *base = const_cast(reinterpret_cast(b->get_base_addr())); + return (q > base && q < base + b->get_max_size()); + }); + lock.Unlock(); + DS_ASSERT(it != mem_segments_.end()); + Arena *ba = it->get(); + Status rc = ba->Reallocate(pp, old_sz, new_sz); + if (rc.IsOutofMemory()) { + // The current arena has no room for the bigger size. + // Allocate free space from another arena and copy + // the content over. + void *q = nullptr; + rc = this->Allocate(new_sz, &q); + RETURN_IF_NOT_OK(rc); + errno_t err = memcpy_s(q, new_sz, p, old_sz); + if (err) { + this->Deallocate(q); + RETURN_STATUS_UNEXPECTED(std::to_string(err)); + } + *pp = q; + ba->Deallocate(p); + } + return Status::OK(); +} + +uint64_t CircularPool::get_max_size() const { return mem_segments_.front()->get_max_size(); } + +int CircularPool::PercentFree() const { + int percent_free = 0; + int num_arena = 0; + for (auto const &p : mem_segments_) { + percent_free += p->PercentFree(); + num_arena++; + } + if (num_arena) { + return percent_free / num_arena; + } else { + return 100; + } +} + +CircularPool::CircularPool(int max_size_in_gb, int arena_size) + : unlimited_(max_size_in_gb <= 0), + max_size_in_mb_(unlimited_ ? std::numeric_limits::max() : max_size_in_gb * 1024), + arena_size_(arena_size), + cur_size_in_mb_(0) {} + +Status CircularPool::CreateCircularPool(std::shared_ptr *out_pool, int max_size_in_gb, int arena_size, + bool createOneArena) { + Status rc; + if (out_pool == nullptr) { + RETURN_STATUS_UNEXPECTED("pPool is null"); + } + auto pool = new (std::nothrow) CircularPool(max_size_in_gb, arena_size); + if (pool == nullptr) { + return Status(StatusCode::kOutOfMemory); + } + if (createOneArena) { + rc = pool->AddOneArena(); + } + if (rc.IsOk()) { + (*out_pool).reset(pool); + } else { + delete pool; + } + return rc; +} + +CircularPool::~CircularPool() = default; +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/circular_pool.h b/mindspore/ccsrc/dataset/util/circular_pool.h new file mode 100644 index 00000000000..3c526597991 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/circular_pool.h @@ -0,0 +1,108 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_CIRCULAR_POOL_H_ +#define DATASET_UTIL_CIRCULAR_POOL_H_ + +#include +#include +#include +#include "dataset/util/memory_pool.h" +#include "dataset/util/arena.h" +#include "dataset/util/lock.h" + +namespace mindspore { +namespace dataset { +using ListOfArenas = std::vector>; + +// This is a dynamic memory pool built on top of memory +// segment each of which is 4G in size. Initially we start +// with one segment, and gradually add segments (not +// guaranteed contiguous) until we reach 32G in size. There +// is an assumption about this kind of memory pool. Allocated +// memory is not held for the whole duration of the pool and +// will be released soon. Based on this assumption, memory is +// obtained from the tail while allocated memory is returned +// to the head of the pool. +class CircularPool : public MemoryPool { + public: + class CircularIterator { + friend class CircularPool; + + public: + explicit CircularIterator(CircularPool *dp); + + ~CircularIterator() = default; + + bool has_next() const; + + ListOfArenas::iterator Next(); + + void Reset(); + + private: + CircularPool *dp_; + Arena *cur_tail_{}; + uint32_t start_{}; + uint32_t cur_{}; + bool wrap_{}; + bool has_next_{}; + }; + + CircularPool(const CircularPool &) = delete; + + CircularPool &operator=(const CircularPool &) = delete; + + ~CircularPool() override; + + Status Allocate(size_t n, void **) override; + + Status Reallocate(void **, size_t old_size, size_t new_size) override; + + void Deallocate(void *) override; + + uint64_t get_max_size() const override; + + int PercentFree() const override; + + friend std::ostream &operator<<(std::ostream &os, const CircularPool &s) { + int i = 0; + for (auto it = s.mem_segments_.begin(); it != s.mem_segments_.end(); ++it, ++i) { + os << "Dumping segment " << i << "\n" << *(it->get()); + } + return os; + } + + static Status CreateCircularPool(std::shared_ptr *out_pool, int max_size_in_gb = -1, + int arena_size = 4096, bool create_one_arena = false); + + private: + ListOfArenas mem_segments_; + std::atomic tail_{}; + bool unlimited_; + int max_size_in_mb_; + int arena_size_; + int cur_size_in_mb_; + RWLock rw_lock_; + + // We can take negative or 0 as input which means unlimited. + CircularPool(int max_size_in_gb, int arena_size); + + Status AddOneArena(); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_CIRCULAR_POOL_H_ diff --git a/mindspore/ccsrc/dataset/util/cond_var.cc b/mindspore/ccsrc/dataset/util/cond_var.cc new file mode 100644 index 00000000000..ada401f2161 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/cond_var.cc @@ -0,0 +1,86 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/cond_var.h" +#include +#include "dataset/util/services.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +CondVar::CondVar() : svc_(nullptr), my_name_(std::move(Services::GetUniqueID())) {} + +Status CondVar::Wait(std::unique_lock *lck, const std::function &pred) { + // Append an additional condition on top of the given predicate. + // We will also bail out if this cv got interrupted. + auto f = [this, &pred]() -> bool { return (pred() || (CurState() == State::kInterrupted)); }; + // If we have interrupt service, just wait on the cv unconditionally. + // Otherwise fall back to the old way of checking interrupt. + if (svc_) { + cv_.wait(*lck, f); + if (CurState() == State::kInterrupted) { + Task *my_task = TaskManager::FindMe(); + if (my_task->IsMasterThread() && my_task->CaughtSevereException()) { + return TaskManager::GetMasterThreadRc(); + } else { + return Status(StatusCode::kInterrupted); + } + } + } else { + RETURN_IF_NOT_OK(interruptible_wait(&cv_, lck, pred)); + if (CurState() == State::kInterrupted) { + return Status(StatusCode::kInterrupted); + } + } + return Status::OK(); +} + +CondVar::~CondVar() noexcept { + if (svc_ != nullptr) { + (void)svc_->Deregister(my_name_); + svc_ = nullptr; + } +} + +void CondVar::NotifyOne() noexcept { cv_.notify_one(); } + +void CondVar::NotifyAll() noexcept { cv_.notify_all(); } + +Status CondVar::Register(std::shared_ptr svc) { + Status rc = svc->Register(my_name_, this); + if (rc.IsOk()) { + svc_ = svc; + } + return rc; +} + +Status CondVar::Interrupt() { + RETURN_IF_NOT_OK(IntrpResource::Interrupt()); + cv_.notify_all(); + return Status::OK(); +} + +std::string CondVar::my_name() const { return my_name_; } + +Status CondVar::Deregister() { + if (svc_) { + Status rc = svc_->Deregister(my_name_); + svc_ = nullptr; + return rc; + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/cond_var.h b/mindspore/ccsrc/dataset/util/cond_var.h new file mode 100644 index 00000000000..fc29b3315da --- /dev/null +++ b/mindspore/ccsrc/dataset/util/cond_var.h @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_COND_VAR_H_ +#define DATASET_UTIL_COND_VAR_H_ + +#include +#include +#include +#include +#include +#include "dataset/util/intrp_resource.h" +#include "dataset/util/intrp_service.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class CondVar : public IntrpResource { + public: + CondVar(); + + ~CondVar() noexcept; + + Status Wait(std::unique_lock *lck, const std::function &pred); + + Status Interrupt() override; + + void NotifyOne() noexcept; + + void NotifyAll() noexcept; + + Status Register(std::shared_ptr svc); + + std::string my_name() const; + + Status Deregister(); + + protected: + std::condition_variable cv_; + std::shared_ptr svc_; + + private: + std::string my_name_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_COND_VAR_H_ diff --git a/mindspore/ccsrc/dataset/util/de_error.h b/mindspore/ccsrc/dataset/util/de_error.h new file mode 100644 index 00000000000..07d7c125f75 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/de_error.h @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_DE_ERROR_H_ +#define DATASET_UTIL_DE_ERROR_H_ + +#include +#include "utils/error_code.h" + +namespace mindspore { +namespace dataset { +DE_ERRORNO_DATASET(CATCH_EXCEPTION, 0, "try catch exception error"); +DE_ERRORNO_DATASET(FILE_NOT_FOUND, 1, "file is not found"); +DE_ERRORNO_DATASET(PARSE_FAILED, 2, "parse failed"); +DE_ERRORNO_DATASET(COPY_ERROR, 3, "copy data error"); +DE_ERRORNO_DATASET(BOUND_ERROR, 4, "variable overflow or lost of precision"); +DE_ERRORNO_DATASET(ALLOC_FAILED, 5, "dynamic memory allocation failed"); +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_DE_ERROR_H_ diff --git a/mindspore/ccsrc/dataset/util/intrp_resource.h b/mindspore/ccsrc/dataset/util/intrp_resource.h new file mode 100644 index 00000000000..462a0bd7ef6 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/intrp_resource.h @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_INTRP_RESOURCE_H_ +#define DATASET_UTIL_INTRP_RESOURCE_H_ + +#include +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class IntrpResource { + public: + enum class State : int { kRunning, kInterrupted }; + + IntrpResource() : st_(State::kRunning) {} + + virtual ~IntrpResource() = default; + + virtual Status Interrupt() { + st_ = State::kInterrupted; + return Status::OK(); + } + + virtual void ResetIntrpState() { st_ = State::kRunning; } + + State CurState() const { return st_; } + + bool Interrupted() const { return CurState() == State::kInterrupted; } + + protected: + std::atomic st_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_INTRP_RESOURCE_H_ diff --git a/mindspore/ccsrc/dataset/util/intrp_service.cc b/mindspore/ccsrc/dataset/util/intrp_service.cc new file mode 100644 index 00000000000..85b56ec00ec --- /dev/null +++ b/mindspore/ccsrc/dataset/util/intrp_service.cc @@ -0,0 +1,92 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/intrp_service.h" +#include +#include "common/utils.h" +#include "dataset/util/services.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +IntrpService::IntrpService() : high_water_mark_(0) { (void)ServiceStart(); } + +IntrpService::~IntrpService() noexcept { + MS_LOG(INFO) << "Number of registered resources is " << high_water_mark_ << "."; + if (!all_intrp_resources_.empty()) { + try { + (void)InterruptAll(); + } catch (const std::exception &e) { + // Ignore all error as we can't throw in the destructor. + } + } + (void)ServiceStop(); +} + +Status IntrpService::Register(const std::string &name, IntrpResource *res) { + SharedLock stateLck(&state_lock_); + // Now double check the state + if (ServiceState() != STATE::kRunning) { + return Status(StatusCode::kInterrupted, __LINE__, __FILE__, "Interrupt service is shutting down"); + } else { + std::lock_guard lck(mutex_); + try { + std::ostringstream ss; + ss << this_thread::get_id(); + MS_LOG(DEBUG) << "Register resource with name " << name << ". Thread ID " << ss.str() << "."; + (void)all_intrp_resources_.emplace(name, res); + high_water_mark_++; + } catch (std::exception &e) { + RETURN_STATUS_UNEXPECTED(e.what()); + } + } + return Status::OK(); +} + +Status IntrpService::Deregister(const std::string &name) noexcept { + std::lock_guard lck(mutex_); + try { + std::ostringstream ss; + ss << this_thread::get_id(); + MS_LOG(DEBUG) << "De-register resource with name " << name << ". Thread ID is " << ss.str() << "."; + auto it = all_intrp_resources_.find(name); + if (it != all_intrp_resources_.end()) { + (void)all_intrp_resources_.erase(it); + } else { + MS_LOG(INFO) << "Key " << name << " not found."; + } + } catch (std::exception &e) { + RETURN_STATUS_UNEXPECTED(e.what()); + } + return Status::OK(); +} + +Status IntrpService::InterruptAll() noexcept { + Status rc; + for (auto const &it : all_intrp_resources_) { + std::string kName = it.first; + try { + Status rc2 = it.second->Interrupt(); + if (rc2.IsError()) { + rc = rc2; + } + } catch (const std::exception &e) { + // continue the clean up. + } + } + return rc; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/intrp_service.h b/mindspore/ccsrc/dataset/util/intrp_service.h new file mode 100644 index 00000000000..47a7d8d9192 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/intrp_service.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_INTRP_SERVICE_H_ +#define DATASET_UTIL_INTRP_SERVICE_H_ + +#include +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/intrp_resource.h" +#include "dataset/util/service.h" +#include "dataset/util/services.h" +#include "dataset/util/status.h" + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +using SvcAllocator = Allocator>; + +class IntrpService : public Service { + public: + IntrpService(); + + ~IntrpService() noexcept override; + + IntrpService(const IntrpService &) = delete; + + IntrpService &operator=(const IntrpService &) = delete; + + Status Register(const std::string &name, IntrpResource *res); + + Status Deregister(const std::string &name) noexcept; + + Status InterruptAll() noexcept; + + Status DoServiceStart() override { return Status::OK(); } + + Status DoServiceStop() override { return Status::OK(); } + + private: + int high_water_mark_; + std::mutex mutex_; + std::map all_intrp_resources_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_INTRP_SERVICE_H_ diff --git a/mindspore/ccsrc/dataset/util/list.h b/mindspore/ccsrc/dataset/util/list.h new file mode 100644 index 00000000000..f01201e34d7 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/list.h @@ -0,0 +1,198 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_LIST_H_ +#define DATASET_UTIL_LIST_H_ + +#include +#include + +#include "dataset/util/make_unique.h" + +namespace mindspore { +namespace dataset { +template +struct Node { + using value_type = T; + using pointer = T *; + pointer prev; + pointer next; + + Node() { + prev = nullptr; + next = nullptr; + } +}; + +template +struct List { + using value_type = T; + using pointer = T *; + using const_pointer = const T *; + using reference = T &; + using const_reference = const T &; + int count; + pointer head; + pointer tail; + Node T::*node; + + // Constructor + explicit List(Node T::*m) : count(0), head(nullptr), tail(nullptr), node(m) {} + + // Destructor + virtual ~List() { + head = nullptr; + tail = nullptr; + } + + // Prepend + virtual void Prepend(pointer elem) { + Node &elem_node = elem->*node; + elem_node.prev = nullptr; + elem_node.next = head; + if (head != nullptr) { + Node &base_node = head->*node; + base_node.prev = elem; + } + head = elem; + if (tail == nullptr) { + tail = elem; + } + ++count; + } + + // Append + virtual void Append(pointer elem) { + Node &elem_node = elem->*node; + elem_node.next = nullptr; + elem_node.prev = tail; + if (tail != nullptr) { + Node &base_node = tail->*node; + base_node.next = elem; + } + tail = elem; + if (head == nullptr) { + head = elem; + } + ++count; + } + + // Insert elem2 after elem1 in the list. + virtual void InsertAfter(pointer elem1, pointer elem2) { + DS_ASSERT(elem1 != elem2); + Node &elem1_node = elem1->*node; + Node &elem2_node = elem2->*node; + elem2_node.prev = elem1; + elem2_node.next = elem1_node.next; + if (elem1_node.next != nullptr) { + Node &next_node = elem1_node.next->*node; + next_node.prev = elem2; + } + elem1_node.next = elem2; + if (tail == elem1) { + tail = elem2; + } + ++count; + } + + // Remove an element in the list + virtual void Remove(pointer elem) noexcept { + Node &elem_node = elem->*node; + if (elem_node.next != nullptr) { + Node &next_node = elem_node.next->*node; + next_node.prev = elem_node.prev; + } else { + tail = elem_node.prev; + } + if (elem_node.prev != nullptr) { + Node &prev_node = elem_node.prev->*node; + prev_node.next = elem_node.next; + } else { + head = elem_node.next; + } + elem_node.prev = nullptr; + elem_node.next = nullptr; + --count; + } + + // Iterator + class Iterator : public std::iterator { + public: + pointer elem_; + + explicit Iterator(const List &v, pointer p = nullptr) : elem_(p), li_(v) {} + + ~Iterator() = default; + + reference operator*() { return *elem_; } + + pointer operator->() { return elem_; } + + const_reference operator*() const { return *elem_; } + + const_pointer operator->() const { return elem_; } + + bool operator==(const Iterator &rhs) const { return elem_ == rhs.elem_; } + + bool operator!=(const Iterator &rhs) const { return elem_ != rhs.elem_; } + + // Prefix increment + Iterator &operator++() { + Node &elem_node = elem_->*(li_.node); + elem_ = elem_node.next; + return *this; + } + + // Postfix increment + Iterator operator++(int junk) { + Iterator tmp(*this); + Node &elem_node = elem_->*(li_.node); + elem_ = elem_node.next; + return tmp; + } + + // Prefix decrement + Iterator &operator--() { + Node &elem_node = elem_->*(li_.node); + elem_ = elem_node.prev; + return *this; + } + + // Postfix decrement + Iterator operator--(int junk) { + Iterator tmp(*this); + Node &elem_node = elem_->*(li_.node); + elem_ = elem_node.prev; + return tmp; + } + + private: + const List &li_; + }; + + Iterator begin() { + Iterator it(*this, head); + return it; + } + + Iterator end() { + Iterator it(*this); + return it; + } +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_LIST_H_ diff --git a/mindspore/ccsrc/dataset/util/lock.cc b/mindspore/ccsrc/dataset/util/lock.cc new file mode 100644 index 00000000000..7e92a1e53fe --- /dev/null +++ b/mindspore/ccsrc/dataset/util/lock.cc @@ -0,0 +1,184 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/lock.h" + +namespace mindspore { +namespace dataset { +void SpinLock::Lock() { + while (true) { + int expected = kUnlocked; + if (val_.compare_exchange_weak(expected, kLocked)) { + break; + } + } +} + +bool SpinLock::TryLock() { + int expected = kUnlocked; + return val_.compare_exchange_strong(expected, kLocked); +} + +void SpinLock::Unlock() noexcept { val_.store(kUnlocked); } + +void RWLock::LockShared() { + std::unique_lock lck(mtx_); + waiting_readers_ += 1; + read_cv_.wait(lck, [this]() { return (waiting_writers_ == 0 && status_ >= 0); }); + waiting_readers_ -= 1; + status_ += 1; +} + +void RWLock::Unlock() noexcept { + std::unique_lock lck(mtx_); + if (status_ == -1) { + // I am the writer. By definition, no other writer nor reader. + status_ = 0; + } else if (status_ > 0) { + // One less reader + status_ -= 1; + } + // Wake up writer only if there is no reader. + if (waiting_writers_ > 0) { + if (status_ == 0) { + write_cv_.notify_one(); + } + } else { + read_cv_.notify_all(); + } +} + +void RWLock::Upgrade() { + std::unique_lock lck(mtx_); + DS_ASSERT(status_); + if (status_ == -1) { + // I am a writer already. + return; + } else if (status_ == 1) { + // If I am the only reader. Just change the status. + status_ = -1; + return; + } else { + // In all other cases, let of the shared lock and relock in exclusive. + lck.unlock(); + this->Unlock(); + this->LockExclusive(); + } +} + +void RWLock::Downgrade() { + std::unique_lock lck(mtx_); + DS_ASSERT(status_); + if (status_ == -1) { + // If there are no other writers waiting, just change the status + if (waiting_writers_ == 0) { + status_ = 1; + } else { + // Otherwise just unlock and relock in shared + lck.unlock(); + this->Unlock(); + this->LockShared(); + } + } else if (status_ > 0) { + return; + } +} + +SharedLock::SharedLock(RWLock *rw) : rw_(rw), ownlock_(false) { + rw_->LockShared(); + ownlock_ = true; +} + +SharedLock::~SharedLock() { + if (ownlock_) { + rw_->Unlock(); + ownlock_ = false; + } + rw_ = nullptr; +} + +void SharedLock::Unlock() { + DS_ASSERT(ownlock_ == true); + rw_->Unlock(); + ownlock_ = false; +} + +void SharedLock::Lock() { + DS_ASSERT(ownlock_ == false); + rw_->LockShared(); + ownlock_ = true; +} + +void SharedLock::Upgrade() { + DS_ASSERT(ownlock_ == true); + rw_->Upgrade(); +} + +void SharedLock::Downgrade() { + DS_ASSERT(ownlock_ == true); + rw_->Downgrade(); +} + +UniqueLock::UniqueLock(RWLock *rw) : rw_(rw), ownlock_(false) { + rw_->LockExclusive(); + ownlock_ = true; +} + +UniqueLock::~UniqueLock() { + if (ownlock_) { + rw_->Unlock(); + ownlock_ = false; + } + rw_ = nullptr; +} + +void UniqueLock::Unlock() { + DS_ASSERT(ownlock_ == true); + rw_->Unlock(); + ownlock_ = false; +} + +void UniqueLock::Lock() { + DS_ASSERT(ownlock_ == false); + rw_->LockExclusive(); + ownlock_ = true; +} + +LockGuard::LockGuard(SpinLock *lock) : lck_(lock), own_lock_(false) { + lck_->Lock(); + own_lock_ = true; +} + +LockGuard::~LockGuard() { + if (own_lock_) { + lck_->Unlock(); + own_lock_ = false; + } + lck_ = nullptr; +} + +void LockGuard::Unlock() { + DS_ASSERT(own_lock_); + lck_->Unlock(); + own_lock_ = false; +} + +void LockGuard::Lock() { + DS_ASSERT(own_lock_ == false); + lck_->Lock(); + own_lock_ = true; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/lock.h b/mindspore/ccsrc/dataset/util/lock.h new file mode 100644 index 00000000000..8fef6a143a6 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/lock.h @@ -0,0 +1,174 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_LOCK_H_ +#define DATASET_UTIL_LOCK_H_ + +#include +#include +#include +#include "dataset/util/make_unique.h" + +namespace mindspore { +namespace dataset { +class SpinLock { + public: + void Lock(); + + bool TryLock(); + + void Unlock() noexcept; + + SpinLock() : val_(kUnlocked) {} + + SpinLock(const SpinLock &) = delete; + + SpinLock(SpinLock &&) = delete; + + ~SpinLock() = default; + + SpinLock &operator=(const SpinLock &) = delete; + + SpinLock &operator=(SpinLock &&) = delete; + + private: + static constexpr int kUnlocked = 0; + static constexpr int kLocked = 1; + std::atomic val_; +}; + +// C++11 has no shared mutex. The following class is an alternative. It favors writer and is suitable for the case +// where writer is rare. +class RWLock { + public: + RWLock() : status_(0), waiting_readers_(0), waiting_writers_(0) {} + + RWLock(const RWLock &) = delete; + + RWLock(RWLock &&) = delete; + + ~RWLock() = default; + + RWLock &operator=(const RWLock &) = delete; + + RWLock &operator=(RWLock &&) = delete; + + void LockShared(); + + void LockExclusive() { + std::unique_lock lck(mtx_); + waiting_writers_ += 1; + write_cv_.wait(lck, [this]() { return status_ == 0; }); + waiting_writers_ -= 1; + status_ = -1; + } + + void Unlock() noexcept; + + // Upgrade a shared lock to exclusive lock + void Upgrade(); + + // Downgrade an exclusive lock to shared lock + void Downgrade(); + + private: + // -1 : one writer + // 0 : no reader and no writer + // n > 0 : n reader + int32_t status_; + int32_t waiting_readers_; + int32_t waiting_writers_; + std::mutex mtx_; + std::condition_variable read_cv_; + std::condition_variable write_cv_; +}; + +// A Wrapper for RWLock. The destructor will release the lock if we own it. +class SharedLock { + public: + explicit SharedLock(RWLock *rw); + + ~SharedLock(); + + SharedLock(const SharedLock &) = delete; + + SharedLock(SharedLock &&) = delete; + + SharedLock &operator=(const SharedLock &) = delete; + + SharedLock &operator=(SharedLock &&) = delete; + + void Unlock(); + + void Lock(); + + void Upgrade(); + + void Downgrade(); + + private: + RWLock *rw_; + bool ownlock_; +}; + +class UniqueLock { + public: + explicit UniqueLock(RWLock *rw); + + ~UniqueLock(); + + UniqueLock(const UniqueLock &) = delete; + + UniqueLock(UniqueLock &&) = delete; + + UniqueLock &operator=(const UniqueLock &) = delete; + + UniqueLock &operator=(UniqueLock &&) = delete; + + void Unlock(); + + void Lock(); + + private: + RWLock *rw_; + bool ownlock_; +}; + +class LockGuard { + public: + explicit LockGuard(SpinLock *lock); + + ~LockGuard(); + + LockGuard(const LockGuard &) = delete; + + LockGuard(LockGuard &&) = delete; + + LockGuard &operator=(const LockGuard &) = delete; + + LockGuard &operator=(LockGuard &&) = delete; + + void Unlock(); + + void Lock(); + + private: + SpinLock *lck_; + bool own_lock_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_LOCK_H_ diff --git a/mindspore/ccsrc/dataset/util/make_unique.h b/mindspore/ccsrc/dataset/util/make_unique.h new file mode 100644 index 00000000000..2fe0bf45506 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/make_unique.h @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_UTIL_MAKE_UNIQUE_H_ +#define DATASET_UTIL_MAKE_UNIQUE_H_ + +#ifdef DEBUG +#include +#define DS_ASSERT(f) assert(f) +#else +#define DS_ASSERT(f) ((void)0) +#endif + +#include +#include +#include +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +namespace mindspore { +using std::make_unique; +} // namespace mindspore + +#endif // DATASET_UTIL_MAKE_UNIQUE_H_ diff --git a/mindspore/ccsrc/dataset/util/memory_pool.cc b/mindspore/ccsrc/dataset/util/memory_pool.cc new file mode 100644 index 00000000000..5d66b4bd6de --- /dev/null +++ b/mindspore/ccsrc/dataset/util/memory_pool.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/memory_pool.h" +#include "./securec.h" + +namespace mindspore { +namespace dataset { +Status DeMalloc(std::size_t s, void **p, bool init_to_zero = false) { + if (p == nullptr) { + RETURN_STATUS_UNEXPECTED("p is null"); + } + void *q = ::malloc(s); + if (q == nullptr) { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__); + } else { + *p = q; + if (init_to_zero) { + (void)memset_s(q, s, 0, s); + } + return Status::OK(); + } +} +} // namespace dataset +} // namespace mindspore + +void *operator new(std::size_t s, mindspore::dataset::Status *rc, std::shared_ptr b) { + void *ptr = nullptr; + *rc = b->Allocate(s, &ptr); + return ptr; +} + +void *operator new[](std::size_t s, mindspore::dataset::Status *rc, std::shared_ptr b) { + void *ptr = nullptr; + *rc = b->Allocate(s, &ptr); + return ptr; +} + +void operator delete(void *p, std::shared_ptr b) { + if (p != nullptr) b->Deallocate(p); +} + +void operator delete[](void *p, std::shared_ptr b) { + if (p != nullptr) b->Deallocate(p); +} diff --git a/mindspore/ccsrc/dataset/util/memory_pool.h b/mindspore/ccsrc/dataset/util/memory_pool.h new file mode 100644 index 00000000000..70876a81417 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/memory_pool.h @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_MEMORY_POOL_H_ +#define DATASET_UTIL_MEMORY_POOL_H_ + +#include +#include +#include +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +// Abstract class of a memory pool +class MemoryPool { + public: + // Allocate a block of size n + virtual Status Allocate(size_t, void **) = 0; + + // Enlarge or shrink a block from oldSz to newSz + virtual Status Reallocate(void **, size_t old_sz, size_t new_sz) = 0; + + // Free a pointer + virtual void Deallocate(void *) = 0; + + // What is the maximum size I can allocate ? + virtual uint64_t get_max_size() const = 0; + + virtual int PercentFree() const = 0; + + // Destructor + virtual ~MemoryPool() {} +}; + +// Used by unique_ptr +template +class Deleter { + public: + explicit Deleter(std::shared_ptr &mp) : mp_(mp) {} + + ~Deleter() = default; + + void operator()(T *ptr) const { mp_->Deallocate(ptr); } + + private: + std::shared_ptr mp_; +}; + +Status DeMalloc(std::size_t s, void **p, bool); +} // namespace dataset +} // namespace mindspore + +void *operator new(std::size_t, mindspore::dataset::Status *, std::shared_ptr); + +void *operator new[](std::size_t, mindspore::dataset::Status *, std::shared_ptr); + +void operator delete(void *, std::shared_ptr); + +void operator delete[](void *, std::shared_ptr); + +#endif // DATASET_UTIL_MEMORY_POOL_H_ diff --git a/mindspore/ccsrc/dataset/util/path.cc b/mindspore/ccsrc/dataset/util/path.cc new file mode 100644 index 00000000000..dd72f807667 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/path.cc @@ -0,0 +1,225 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/path.h" + +#include +#include +#include +#include + +#include "common/utils.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +#ifdef _WIN32 +char Path::_separator = '\\'; +#else +char Path::separator_ = '/'; +#endif + +Path::Path(const std::string &s) : path_(s) {} + +Path::Path(const char *p) : path_(p) {} + +Path::Path(const Path &p) : path_(p.path_) {} + +Path &Path::operator=(const Path &p) { + if (&p != this) { + this->path_ = p.path_; + } + return *this; +} + +Path &Path::operator=(Path &&p) noexcept { + if (&p != this) { + this->path_ = std::move(p.path_); + } + return *this; +} + +Path::Path(Path &&p) noexcept { this->path_ = std::move(p.path_); } + +Path Path::operator+(const Path &p) { + std::string q = path_ + p.toString(); + return Path(q); +} + +Path Path::operator+(const std::string &p) { + std::string q = path_ + p; + return Path(q); +} + +Path Path::operator+(const char *p) { + std::string q = path_ + p; + return Path(q); +} + +Path &Path::operator+=(const Path &rhs) { + path_ += rhs.toString(); + return *this; +} + +Path &Path::operator+=(const std::string &p) { + path_ += p; + return *this; +} + +Path &Path::operator+=(const char *p) { + path_ += p; + return *this; +} + +Path Path::operator/(const Path &p) { + std::string q = path_ + separator_ + p.toString(); + return Path(q); +} + +Path Path::operator/(const std::string &p) { + std::string q = path_ + separator_ + p; + return Path(q); +} + +Path Path::operator/(const char *p) { + std::string q = path_ + separator_ + p; + return Path(q); +} + +std::string Path::Extension() const { + std::size_t found = path_.find_last_of('.'); + if (found != std::string::npos) { + return path_.substr(found); + } else { + return std::string(""); + } +} + +bool Path::Exists() { + struct stat sb; + int rc = stat(common::SafeCStr(path_), &sb); + if (rc == -1 && errno != ENOENT) { + MS_LOG(INFO) << "Unable to query the status of " << path_ << ". Errno = " << errno << "."; + } + return (rc == 0); +} + +bool Path::IsDirectory() { + struct stat sb; + int rc = stat(common::SafeCStr(path_), &sb); + if (rc == 0) { + return S_ISDIR(sb.st_mode); + } else { + return false; + } +} + +Status Path::CreateDirectory() { + if (!Exists()) { + int rc = mkdir(common::SafeCStr(path_), 0700); + if (rc) { + std::ostringstream oss; + oss << "Unable to create directory " << path_ << ". Errno = " << errno; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + return Status::OK(); + } else { + if (IsDirectory()) { + return Status::OK(); + } else { + std::ostringstream oss; + oss << "Unable to create directory " << path_ << ". It exists but is not a directory"; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + } +} + +std::string Path::ParentPath() { + std::string r(""); + std::size_t found = path_.find_last_of(separator_); + if (found != std::string::npos) { + if (found == 0) { + r += separator_; + } else { + r = std::string(path_.substr(0, found)); + } + } + return r; +} + +Status Path::CreateDirectories() { + if (IsDirectory()) { + MS_LOG(INFO) << "Directory " << toString() << " already exists."; + return Status::OK(); + } else { + MS_LOG(INFO) << "Creating directory " << toString() << "."; + std::string parent = ParentPath(); + if (!parent.empty()) { + if (Path(parent).CreateDirectories()) { + return CreateDirectory(); + } + } else { + return CreateDirectory(); + } + } + return Status::OK(); +} + +std::shared_ptr Path::DirIterator::OpenDirectory(Path *f) { + auto it = new (std::nothrow) DirIterator(f); + + if (it == nullptr) { + return nullptr; + } + + if (it->dp_) { + return std::shared_ptr(it); + } else { + delete it; + return nullptr; + } +} + +Path::DirIterator::~DirIterator() { + if (dp_) { + (void)closedir(dp_); + } + dp_ = nullptr; + dir_ = nullptr; + entry_ = nullptr; +} + +Path::DirIterator::DirIterator(Path *f) : dir_(f), dp_(nullptr), entry_(nullptr) { + MS_LOG(INFO) << "Open directory " << f->toString() << "."; + dp_ = opendir(common::SafeCStr(f->toString())); +} + +bool Path::DirIterator::hasNext() { + do { + entry_ = readdir(dp_); + if (entry_) { + if (strcmp(entry_->d_name, ".") == 0 || strcmp(entry_->d_name, "..") == 0) { + continue; + } + } + break; + } while (true); + return (entry_ != nullptr); +} + +Path Path::DirIterator::next() { return (*(this->dir_) / Path(entry_->d_name)); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/path.h b/mindspore/ccsrc/dataset/util/path.h new file mode 100644 index 00000000000..efe01a7d16c --- /dev/null +++ b/mindspore/ccsrc/dataset/util/path.h @@ -0,0 +1,100 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_PATH_H_ +#define DATASET_UTIL_PATH_H_ + +#include +#include +#include + +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class Path { + public: + class DirIterator { + public: + static std::shared_ptr OpenDirectory(Path *f); + + ~DirIterator(); + + bool hasNext(); + + Path next(); + + private: + explicit DirIterator(Path *f); + + Path *dir_; + DIR *dp_; + struct dirent *entry_; + }; + + explicit Path(const std::string &); + + explicit Path(const char *); + + ~Path() = default; + + Path(const Path &); + + Path &operator=(const Path &); + + Path(Path &&) noexcept; + + Path &operator=(Path &&) noexcept; + + std::string toString() const { return path_; } + + Path operator+(const Path &); + + Path operator+(const std::string &); + + Path operator+(const char *); + + Path &operator+=(const Path &rhs); + + Path &operator+=(const std::string &); + + Path &operator+=(const char *); + + Path operator/(const Path &); + + Path operator/(const std::string &); + + Path operator/(const char *); + + bool Exists(); + + bool IsDirectory(); + + Status CreateDirectory(); + + Status CreateDirectories(); + + std::string Extension() const; + + std::string ParentPath(); + + private: + static char separator_; + std::string path_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_PATH_H_ diff --git a/mindspore/ccsrc/dataset/util/queue.h b/mindspore/ccsrc/dataset/util/queue.h new file mode 100644 index 00000000000..4048deb86b4 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/queue.h @@ -0,0 +1,243 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_QUEUE_H_ +#define DATASET_UTIL_QUEUE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" +#include "dataset/util/allocator.h" +#include "dataset/util/services.h" +#include "dataset/util/cond_var.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +template +struct is_shared_ptr : public std::false_type {}; + +template +struct is_shared_ptr> : public std::true_type {}; + +template +struct is_unique_ptr : public std::false_type {}; + +template +struct is_unique_ptr> : public std::true_type {}; + +// A simple thread safe queue using a fixed size array +template +class Queue { + public: + using value_type = T; + using pointer = T *; + using const_pointer = const T *; + using reference = T &; + using const_reference = const T &; + + void Init() { + if (sz_ > 0) { + // We allocate a block of memory and then call the default constructor for each slot. Maybe simpler to call + // new[] but we want to control where the memory is allocated from. + arr_ = alloc_.allocate(sz_); + for (uint64_t i = 0; i < sz_; i++) { + std::allocator_traits>::construct(alloc_, &(arr_[i])); + } + } + } + + explicit Queue(int sz) + : sz_(sz), + arr_(nullptr), + head_(0), + tail_(0), + my_name_(std::move(Services::GetUniqueID())), + alloc_(Services::GetInstance().GetServiceMemPool()) { + Init(); + MS_LOG(DEBUG) << "Create Q with uuid " << my_name_ << " of size " << sz_ << "."; + } + + virtual ~Queue() { + ResetQue(); + if (arr_) { + // Simply free the pointer. Since there is nothing in the queue. We don't want to invoke the destructor + // of T in each slot. + alloc_.deallocate(arr_); + arr_ = nullptr; + } + } + + int size() const { + int v = tail_ - head_; + return (v >= 0) ? v : 0; + } + + int capacity() const { return sz_; } + + bool empty() const { return head_ == tail_; } + + void Reset() { ResetQue(); } + + // Producer + Status Add(const_reference ele) noexcept { + std::unique_lock _lock(mux_); + // Block when full + Status rc = full_cv_.Wait(&_lock, [this]() -> bool { return (size() != capacity()); }); + if (rc.IsOk()) { + uint32_t k = tail_++ % sz_; + arr_[k] = ele; + empty_cv_.NotifyAll(); + _lock.unlock(); + } else { + (void)empty_cv_.Interrupt(); + } + return rc; + } + + Status Add(T &&ele) noexcept { + std::unique_lock _lock(mux_); + // Block when full + Status rc = full_cv_.Wait(&_lock, [this]() -> bool { return (size() != capacity()); }); + if (rc.IsOk()) { + uint32_t k = tail_++ % sz_; + arr_[k] = std::forward(ele); + empty_cv_.NotifyAll(); + _lock.unlock(); + } else { + (void)empty_cv_.Interrupt(); + } + return rc; + } + + template + Status EmplaceBack(Ts &&... args) noexcept { + std::unique_lock _lock(mux_); + // Block when full + Status rc = full_cv_.Wait(&_lock, [this]() -> bool { return (size() != capacity()); }); + if (rc.IsOk()) { + uint32_t k = tail_++ % sz_; + new (&(arr_[k])) T(std::forward(args)...); + empty_cv_.NotifyAll(); + _lock.unlock(); + } else { + (void)empty_cv_.Interrupt(); + } + return rc; + } + + // Consumer + Status PopFront(pointer p) { + std::unique_lock _lock(mux_); + // Block when empty + Status rc = empty_cv_.Wait(&_lock, [this]() -> bool { return !empty(); }); + if (rc.IsOk()) { + uint32_t k = head_++ % sz_; + *p = std::move(arr_[k]); + if (std::is_destructible::value) { + arr_[k].~T(); + } + full_cv_.NotifyAll(); + _lock.unlock(); + } else { + (void)full_cv_.Interrupt(); + } + return rc; + } + + void ResetQue() noexcept { + std::unique_lock _lock(mux_); + // If there are elements in the queue, invoke its destructor one by one. + if (!empty() && std::is_destructible::value) { + for (uint64_t i = head_; i < tail_; i++) { + uint32_t k = i % sz_; + arr_[k].~T(); + } + } + empty_cv_.ResetIntrpState(); + full_cv_.ResetIntrpState(); + head_ = 0; + tail_ = 0; + } + + Status Register(TaskGroup *vg) { + Status rc1 = empty_cv_.Register(vg->GetIntrpService()); + Status rc2 = full_cv_.Register(vg->GetIntrpService()); + if (rc1.IsOk()) { + return rc2; + } else { + return rc1; + } + } + + private: + uint64_t sz_; + pointer arr_; + uint64_t head_; + uint64_t tail_; + std::string my_name_; + std::mutex mux_; + CondVar empty_cv_; + CondVar full_cv_; + Allocator alloc_; +}; + +// A container of queues with [] operator accessors. Basically this is a wrapper over of a vector of queues +// to help abstract/simplify code that is maintaining multiple queues. +template +class QueueList { + public: + QueueList() {} + + void Init(int num_queues, int capacity) { + queue_list_.reserve(num_queues); + for (int i = 0; i < num_queues; i++) { + queue_list_.emplace_back(mindspore::make_unique>(capacity)); + } + } + + Status Register(TaskGroup *vg) { + if (vg == nullptr) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Null task group during QueueList registration."); + } + for (int i = 0; i < queue_list_.size(); ++i) { + RETURN_IF_NOT_OK(queue_list_[i]->Register(vg)); + } + return Status::OK(); + } + + int size() const { return queue_list_.size(); } + + std::unique_ptr> &operator[](const int index) { return queue_list_[index]; } + + ~QueueList() = default; + + private: + // Queue contains non-copyable objects, so it cannot be added to a vector due to the vector + // requirement that objects must have copy semantics. To resolve this, we use a vector of unique + // pointers. This allows us to provide dynamic creation of queues in a container. + std::vector>> queue_list_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_QUEUE_H_ diff --git a/mindspore/ccsrc/dataset/util/random.cc b/mindspore/ccsrc/dataset/util/random.cc new file mode 100644 index 00000000000..e4bab6094c9 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/random.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_RANDOM_H_ +#define DATASET_UTIL_RANDOM_H_ + +#include "dataset/util/random.h" + +#include +#include +#include +#include + +#include "dataset/core/config_manager.h" +#include "dataset/core/global_context.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +uint32_t GetSeed() { + uint32_t seed = GlobalContext::config_manager()->seed(); + if (seed == std::mt19937::default_seed) { + std::random_device random_device("/dev/urandom"); + std::uniform_int_distribution distribution(0, std::numeric_limits::max()); + seed = distribution(random_device); + } + + return seed; +} +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_RANDOM_H_ diff --git a/mindspore/ccsrc/dataset/util/random.h b/mindspore/ccsrc/dataset/util/random.h new file mode 100644 index 00000000000..fa9e18f7079 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/random.h @@ -0,0 +1,24 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_RANDOM_H_ +#define DATASET_UTIL_RANDOM_H_ +namespace mindspore { +namespace dataset { +uint32_t GetSeed(); +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_RANDOM_H_ diff --git a/mindspore/ccsrc/dataset/util/semaphore.cc b/mindspore/ccsrc/dataset/util/semaphore.cc new file mode 100644 index 00000000000..983c387df5c --- /dev/null +++ b/mindspore/ccsrc/dataset/util/semaphore.cc @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/semaphore.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +Status Semaphore::P() { + std::unique_lock lck(mutex_); + return (wait_cond_.Wait(&lck, [this]() { return value_ != 0; })); +} + +void Semaphore::V() { + std::unique_lock lck(mutex_); + ++value_; + wait_cond_.NotifyOne(); +} + +void Semaphore::Register(TaskGroup *vg) { (void)wait_cond_.Register(vg->GetIntrpService()); } + +Status Semaphore::Deregister() { return (wait_cond_.Deregister()); } + +void Semaphore::ResetIntrpState() { wait_cond_.ResetIntrpState(); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/semaphore.h b/mindspore/ccsrc/dataset/util/semaphore.h new file mode 100644 index 00000000000..74c344f7d38 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/semaphore.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SEMAPHORE_H_ +#define DATASET_UTIL_SEMAPHORE_H_ + +#include "dataset/util/cond_var.h" + +namespace mindspore { +namespace dataset { +class TaskGroup; + +class Semaphore { + public: + explicit Semaphore(int init) : value_(init) {} + + virtual ~Semaphore() {} + + Status P(); + + void V(); + + void Register(TaskGroup *vg); + + Status Deregister(); + + void ResetIntrpState(); + + private: + int value_; + + std::mutex mutex_; + CondVar wait_cond_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_SEMAPHORE_H_ diff --git a/mindspore/ccsrc/dataset/util/service.cc b/mindspore/ccsrc/dataset/util/service.cc new file mode 100644 index 00000000000..c89f7287f61 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/service.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/service.h" +#include + +namespace mindspore { +namespace dataset { +Status Service::ServiceStart() { + do { + UniqueLock lck(&state_lock_); + // No-op if it is already up or some other thread is + // in the process of bring it up. + if (state_ == STATE::kRunning || state_ == STATE::kStartInProg) { + return Status::OK(); + } + // If a stop is in progress, we line up after it + // is done. + if (state_ == STATE::kStopInProg) { + std::this_thread::yield(); + } else { + state_ = STATE::kStartInProg; + // At this point, we will let go of the lock. This allow others to proceed. + lck.Unlock(); + RETURN_IF_NOT_OK(DoServiceStart()); + // Lock again to change state. + lck.Lock(); + state_ = STATE::kRunning; + return Status::OK(); + } + } while (true); +} + +Status Service::ServiceStop() noexcept { + do { + UniqueLock lck(&state_lock_); + // No-op if it is already stopped or some other thread is + // in the process of shutting it down + if (state_ == STATE::kStopped || state_ == STATE::kStopInProg) { + return Status::OK(); + } + // If a start is in progress, we line up after it + // is done. + if (state_ == STATE::kStartInProg) { + std::this_thread::yield(); + } else { + state_ = STATE::kStopInProg; + // At this point, we will let go of the lock. This allows others to proceed. + lck.Unlock(); + RETURN_IF_NOT_OK(DoServiceStop()); + // Lock again to change state. + lck.Lock(); + state_ = STATE::kStopped; + return Status::OK(); + } + } while (true); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/service.h b/mindspore/ccsrc/dataset/util/service.h new file mode 100644 index 00000000000..1113fc1d14c --- /dev/null +++ b/mindspore/ccsrc/dataset/util/service.h @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SERVICE_H_ +#define DATASET_UTIL_SERVICE_H_ + +#include +#include "dataset/util/lock.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class Service { + public: + enum class STATE : int { kStartInProg = 1, kRunning, kStopInProg, kStopped }; + + Service() : state_(STATE::kStopped) {} + + Service(const Service &) = delete; + + Service &operator=(const Service &) = delete; + + virtual ~Service() {} + + STATE ServiceState() const { return state_; } + + virtual Status DoServiceStart() = 0; + + virtual Status DoServiceStop() = 0; + + Status ServiceStart(); + + Status ServiceStop() noexcept; + + protected: + STATE state_; + RWLock state_lock_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_SERVICE_H_ diff --git a/mindspore/ccsrc/dataset/util/services.cc b/mindspore/ccsrc/dataset/util/services.cc new file mode 100644 index 00000000000..7dcb5b14c90 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/services.cc @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/services.h" + +#include +#include +#include +#include +#include "dataset/util/circular_pool.h" +#include "dataset/util/task_manager.h" + +#define SLOT_TASK_MGR 0 +namespace mindspore { +namespace dataset { +std::unique_ptr Services::instance_ = nullptr; +std::once_flag Services::init_instance_flag_; + +std::string Services::GetUserName() { + char user[LOGIN_NAME_MAX]; + (void)getlogin_r(user, sizeof(user)); + return std::string(user); +} + +std::string Services::GetHostName() { + char host[LOGIN_NAME_MAX]; + (void)gethostname(host, sizeof(host)); + return std::string(host); +} + +int Services::GetLWP() { return syscall(SYS_gettid); } + +std::string Services::GetUniqueID() { + const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789"; + std::mt19937 gen{std::random_device{"/dev/urandom"}()}; + std::uniform_int_distribution<> dist(0, kStr.size() - 1); + char buffer[UNIQUEID_LEN]; + for (int i = 0; i < UNIQUEID_LEN; i++) { + buffer[i] = kStr[dist(gen)]; + } + return std::string(buffer, UNIQUEID_LEN); +} + +TaskManager &Services::getTaskMgrInstance() { + Services &sm = GetInstance(); + return *(static_cast(sm.sa_[SLOT_TASK_MGR])); +} + +Status Services::CreateAllInstances() { + // In order, TaskMgr, BufferMgr + Status rc; + sa_[SLOT_TASK_MGR] = new (&rc, pool_) TaskManager(); + RETURN_IF_NOT_OK(rc); + rc = sa_[SLOT_TASK_MGR]->ServiceStart(); + return rc; +} + +Services::Services() : pool_(nullptr), sa_{nullptr} { + Status rc = CircularPool::CreateCircularPool(&pool_, -1, 16, true); // each arena 16M + if (rc.IsError()) { + std::terminate(); + } +} + +Services::~Services() noexcept { + try { + // In reverse order + TaskManager *tm = static_cast(sa_[SLOT_TASK_MGR]); + if (tm) { + (void)tm->ServiceStop(); + tm->~TaskManager(); + pool_->Deallocate(tm); + } + } catch (const std::exception &e) { + // Do nothing. + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/services.h b/mindspore/ccsrc/dataset/util/services.h new file mode 100644 index 00000000000..65a302af918 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/services.h @@ -0,0 +1,92 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SERVICES_H_ +#define DATASET_UTIL_SERVICES_H_ + +#include +#include +#include +#include "dataset/util/memory_pool.h" +#include "dataset/util/service.h" + +#define UNIQUEID_LEN 36 +namespace mindspore { +namespace dataset { +class TaskManager; + +class Services { + public: + static Status CreateInstance() { + std::call_once(init_instance_flag_, [&]() -> Status { + instance_.reset(new Services()); + return (instance_->CreateAllInstances()); + }); + + if (instance_ == nullptr) { + instance_.reset(new Services()); + return (instance_->CreateAllInstances()); + } + + return Status::OK(); + } + + static Services &GetInstance() { + if (instance_ == nullptr) { + if (!CreateInstance()) { + std::terminate(); + } + } + return *instance_; + } + + Services(const Services &) = delete; + + Services &operator=(const Services &) = delete; + + ~Services() noexcept; + + static TaskManager &getTaskMgrInstance(); + + std::shared_ptr GetServiceMemPool() { return pool_; } + + static std::string GetUserName(); + + static std::string GetHostName(); + + static int GetLWP(); + + static std::string GetUniqueID(); + + private: + static std::once_flag init_instance_flag_; + static std::unique_ptr instance_; + // A small pool used for small objects that last until the + // Services Manager shuts down. Used by all sub-services. + std::shared_ptr pool_; + // We use pointers here instead of unique_ptr because we + // want to have ultimate control on the order of + // construction and destruction. + static constexpr int kNumServices_ = 1; + Service *sa_[kNumServices_]; + + Services(); + + Status CreateAllInstances(); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_SERVICES_H_ diff --git a/mindspore/ccsrc/dataset/util/sig_handler.cc b/mindspore/ccsrc/dataset/util/sig_handler.cc new file mode 100644 index 00000000000..1b6a3701c3c --- /dev/null +++ b/mindspore/ccsrc/dataset/util/sig_handler.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/sig_handler.h" +#include +#include +#include +#include +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +// Register the custom signal handlers +void RegisterHandlers() { + struct sigaction new_int_action; + + // For the interrupt handler, we do not use SA_RESETHAND so this handler remains in play + // permanently, do not use the OS default handler for it. + new_int_action.sa_sigaction = &IntHandler; + (void)sigemptyset(&new_int_action.sa_mask); + new_int_action.sa_flags = SA_RESTART | SA_SIGINFO; + (void)sigaction(SIGINT, &new_int_action, nullptr); +} + +extern void IntHandler(int sig_num, // The signal that was raised + siginfo_t *sig_info, // The siginfo structure. + void *context) { // context info + // Wake up the watchdog which is designed as async-signal-safe. + TaskManager::WakeUpWatchDog(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/sig_handler.h b/mindspore/ccsrc/dataset/util/sig_handler.h new file mode 100644 index 00000000000..6c5e1f015c7 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/sig_handler.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SIG_HANDLER_H_ +#define DATASET_UTIL_SIG_HANDLER_H_ + +#include +#include + +namespace mindspore { +namespace dataset { +// Register the custom signal handlers +extern void RegisterHandlers(); + +// A signal handler for SIGINT. Drives interrupt to watchdog +extern void IntHandler(int sig_num, // The signal that was raised + siginfo_t *sig_info, // The siginfo structure. + void *context); // context info +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_SIG_HANDLER_H_ diff --git a/mindspore/ccsrc/dataset/util/status.cc b/mindspore/ccsrc/dataset/util/status.cc new file mode 100644 index 00000000000..84d8ee582c9 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/status.cc @@ -0,0 +1,117 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/status.h" +#include +#include "common/utils.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +std::string CodeAsString(const StatusCode c) { + const char *s = nullptr; + if (c == StatusCode::kOK) { + // Optimize the most frequent case + return std::string("OK"); + } else { + switch (c) { + case StatusCode::kOutOfMemory: + s = "Out of memory"; + break; + case StatusCode::kInterrupted: + s = "Interrupted system call"; + break; + case StatusCode::kShapeMisMatch: + s = "Shape is incorrect."; + break; + case StatusCode::kNoSpace: + s = "No space left on device"; + break; + case StatusCode::kPyFuncException: + s = "Exception thrown from PyFunc"; + break; + case StatusCode::kDuplicateKey: + s = "Duplicate key"; + break; + case StatusCode::kUnexpectedError: + default: + s = "Unexpected error"; + break; + } + } + return std::string(s); +} + +Status::Status(StatusCode c) noexcept : code_(c), err_msg_(std::move(CodeAsString(c))) {} + +Status::Status() noexcept : code_(StatusCode::kOK), err_msg_("") {} + +Status::~Status() noexcept {} + +Status::Status(const Status &s) : code_(s.code_), err_msg_(s.err_msg_) {} + +Status &Status::operator=(const Status &s) { + if (this == &s) { + return *this; + } + code_ = s.code_; + err_msg_ = s.err_msg_; + return *this; +} + +Status::Status(Status &&s) noexcept { + code_ = s.code_; + s.code_ = StatusCode::kOK; + err_msg_ = std::move(s.err_msg_); +} + +Status &Status::operator=(Status &&s) noexcept { + if (this == &s) { + return *this; + } + code_ = s.code_; + s.code_ = StatusCode::kOK; + err_msg_ = std::move(s.err_msg_); + return *this; +} + +Status::Status(const StatusCode code, const std::string &msg) : code_(code), err_msg_(msg) {} + +Status::Status(const StatusCode code, int line_of_code, const char *file_name, const std::string &extra) { + code_ = code; + std::ostringstream ss; + ss << "Thread ID " << this_thread::get_id() << " " << CodeAsString(code) << ". "; + if (!extra.empty()) { + ss << extra; + } + ss << "\n"; + ss << "Line of code : " << line_of_code << "\n"; + if (file_name != nullptr) { + ss << "File : " << file_name << "\n"; + } + err_msg_ = ss.str(); + MS_LOG(INFO) << err_msg_; +} + +std::ostream &operator<<(std::ostream &os, const Status &s) { + os << s.ToString(); + return os; +} + +std::string Status::ToString() const { return err_msg_; } + +StatusCode Status::get_code() const { return code_; } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/status.h b/mindspore/ccsrc/dataset/util/status.h new file mode 100644 index 00000000000..38ed1fef899 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/status.h @@ -0,0 +1,134 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_STATUS_H_ +#define DATASET_UTIL_STATUS_H_ + +#if defined(__GNUC__) || defined(__clang__) +#define DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define DEPRECATED __declspec(deprecated) +#else +#pragma message("WARNING: You need to implement DEPRECATED for this compiler") +#define DEPRECATED +#endif + +#include +#include +#include + +namespace mindspore { +namespace dataset { +#define RETURN_IF_NOT_OK(_s) \ + do { \ + Status __rc = (_s); \ + if (__rc.IsError()) { \ + return __rc; \ + } \ + } while (false) + +#define RETURN_STATUS_UNEXPECTED(_e) \ + do { \ + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, _e); \ + } while (false) + +#define CHECK_FAIL_RETURN_UNEXPECTED(_condition, _e) \ + do { \ + if (!(_condition)) { \ + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, _e); \ + } \ + } while (false) + +#define RETURN_UNEXPECTED_IF_NULL(_ptr) \ + do { \ + if ((_ptr) == nullptr) { \ + std::string err_msg = "The pointer[" + std::string(#_ptr) + "] is null."; \ + RETURN_STATUS_UNEXPECTED(err_msg); \ + } \ + } while (false) + +enum class StatusCode : char { + kOK = 0, + kOutOfMemory = 1, + kShapeMisMatch = 2, + kInterrupted = 3, + kNoSpace = 4, + kPyFuncException = 5, + kDuplicateKey = 6, + kPythonInterpreterFailure = 7, + kTDTPushFailure = 8, + kFileNotExist = 9, + // Make this error code the last one. Add new error code above it. + kUnexpectedError = 127 +}; + +std::string CodeAsString(const StatusCode c); + +class Status { + public: + Status() noexcept; + + explicit Status(StatusCode c) noexcept; + + ~Status() noexcept; + + // Copy constructor + Status(const Status &s); + + Status &operator=(const Status &s); + + // Move constructor + Status(Status &&) noexcept; + + Status &operator=(Status &&) noexcept; + + Status(const StatusCode code, const std::string &msg); + + Status(const StatusCode code, int line_of_code, const char *file_name, const std::string &extra = ""); + + // Return a success status + static Status OK() { return Status(StatusCode::kOK); } + + std::string ToString() const; + + StatusCode get_code() const; + + friend std::ostream &operator<<(std::ostream &os, const Status &s); + + explicit operator bool() const { return (get_code() == StatusCode::kOK); } + + bool operator==(const Status &other) const { return (this->get_code() == other.get_code()); } + + bool operator!=(const Status &other) const { return !(*this == other); } + + bool IsOk() const { return (get_code() == StatusCode::kOK); } + + bool IsError() const { return !IsOk(); } + + bool IsOutofMemory() const { return (get_code() == StatusCode::kOutOfMemory); } + + bool IsInterrupted() const { return (get_code() == StatusCode::kInterrupted); } + + bool IsShapeIncorrect() const { return (get_code() == StatusCode::kShapeMisMatch); } + + bool IsNoSpace() const { return (get_code() == StatusCode::kNoSpace); } + + private: + StatusCode code_; + std::string err_msg_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_STATUS_H_ diff --git a/mindspore/ccsrc/dataset/util/system_pool.h b/mindspore/ccsrc/dataset/util/system_pool.h new file mode 100644 index 00000000000..bd15ad11ddf --- /dev/null +++ b/mindspore/ccsrc/dataset/util/system_pool.h @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SYSTEM_POOL_H_ +#define DATASET_UTIL_SYSTEM_POOL_H_ + +#include +#include +#include +#include +#include "./securec.h" +#include "dataset/util/memory_pool.h" + +namespace mindspore { +namespace dataset { +// This class demonstrate how to implement a simple MemoryPool +// for minddata/dataset using malloc/free/realloc. We need to +// implement 4 virtual functions. Other MemoryPool +// implementation, e.g., are BuddyArena and CircularPool. All +// these MemoryPool can be used together with Allocator.h for +// C++ STL containers. +class SystemPool : public MemoryPool { + public: + ~SystemPool() override {} + + Status Allocate(size_t n, void **pp) override { return DeMalloc(n, pp, false); } + + void Deallocate(void *p) override { free(p); } + + Status Reallocate(void **p, size_t old_sz, size_t new_sz) override { + if (old_sz >= new_sz) { + // Do nothing if we shrink. + return Status::OK(); + } else { + void *ptr = *p; + void *q = nullptr; + RETURN_IF_NOT_OK(DeMalloc(new_sz, &q, false)); + errno_t err = memcpy_s(q, new_sz, ptr, old_sz); + if (err) { + free(q); + RETURN_STATUS_UNEXPECTED(std::to_string(err)); + } + free(ptr); + *p = q; + return Status::OK(); + } + } + + uint64_t get_max_size() const override { return std::numeric_limits::max(); } + + int PercentFree() const override { return 100; } +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_SYSTEM_POOL_H_ diff --git a/mindspore/ccsrc/dataset/util/task.cc b/mindspore/ccsrc/dataset/util/task.cc new file mode 100644 index 00000000000..7fabf6cac5a --- /dev/null +++ b/mindspore/ccsrc/dataset/util/task.cc @@ -0,0 +1,136 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/task.h" +#include "common/utils.h" +#include "dataset/util/task_manager.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +thread_local Task *gMyTask = nullptr; + +void Task::operator()() { + gMyTask = this; + std::stringstream ss; + ss << this_thread::get_id(); + MS_LOG(INFO) << my_name_ << " Thread ID " << ss.str() << " Started."; + try { + // Previously there is a timing hole where the thread is spawn but hit error immediately before we can set + // the TaskGroup pointer and register. We move the registration logic to here (after we spawn) so we can + // get the thread id. + TaskGroup *vg = MyTaskGroup(); + rc_ = vg->GetIntrpService()->Register(ss.str(), this); + if (rc_.IsOk()) { + // Now we can run the given task. + rc_ = fnc_obj_(); + } + // Some error codes are ignored, e.g. interrupt. Others we just shutdown the group. + if (rc_.IsError() && !rc_.IsInterrupted()) { + ShutdownGroup(); + } + } catch (const std::bad_alloc &e) { + rc_ = Status(StatusCode::kOutOfMemory, __LINE__, __FILE__, e.what()); + ShutdownGroup(); + } catch (const std::exception &e) { + rc_ = Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, e.what()); + ShutdownGroup(); + } +} + +void Task::ShutdownGroup() { // Wake up watch dog and shutdown the engine. + { + std::lock_guard lk(mux_); + caught_severe_exception_ = true; + } + TaskGroup *vg = MyTaskGroup(); + // If multiple threads hit severe errors in the same group. Keep the first one and + // discard the rest. + if (vg->rc_.IsOk()) { + std::unique_lock rcLock(vg->rc_mux_); + // Check again after we get the lock + if (vg->rc_.IsOk()) { + vg->rc_ = rc_; + rcLock.unlock(); + TaskManager::InterruptMaster(rc_); + TaskManager::InterruptGroup(*gMyTask); + } + } +} + +Status Task::GetTaskErrorIfAny() { + std::lock_guard lk(mux_); + if (caught_severe_exception_) { + return rc_; + } else { + return Status::OK(); + } +} + +Task::Task(const std::string &myName, const std::function &f) + : my_name_(myName), + rc_(), + fnc_obj_(f), + task_group_(nullptr), + is_master_(false), + running_(false), + caught_severe_exception_(false) { + IntrpResource::ResetIntrpState(); + wp_.ResetIntrpState(); + wp_.Clear(); +} + +Status Task::Run() { + Status rc; + if (running_ == false) { + try { + thrd_ = std::thread(std::ref(*this)); + id_ = thrd_.get_id(); + running_ = true; + caught_severe_exception_ = false; + } catch (const std::exception &e) { + rc = Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, e.what()); + } + } + return rc; +} + +Status Task::Join() { + if (running_) { + try { + thrd_.join(); + std::stringstream ss; + ss << get_id(); + MS_LOG(INFO) << MyName() << " Thread ID " << ss.str() << " Stopped."; + running_ = false; + RETURN_IF_NOT_OK(wp_.Deregister()); + if (MyTaskGroup()) { + RETURN_IF_NOT_OK(MyTaskGroup()->GetIntrpService()->Deregister(ss.str())); + } + } catch (const std::exception &e) { + RETURN_STATUS_UNEXPECTED(e.what()); + } + } + return Status::OK(); +} + +TaskGroup *Task::MyTaskGroup() { return task_group_; } + +void Task::set_task_group(TaskGroup *vg) { task_group_ = vg; } + +Task::~Task() { task_group_ = nullptr; } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/task.h b/mindspore/ccsrc/dataset/util/task.h new file mode 100644 index 00000000000..aaf2f80a3d5 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/task.h @@ -0,0 +1,124 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_TASK_H_ +#define DATASET_UTIL_TASK_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dataset/util/de_error.h" +#include "dataset/util/make_unique.h" +#include "dataset/util/intrp_resource.h" +#include "dataset/util/list.h" +#include "dataset/util/memory_pool.h" +#include "dataset/util/services.h" +#include "dataset/util/wait_post.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +class TaskManager; + +class Task : public IntrpResource { + public: + friend class TaskManager; + + friend class TaskGroup; + + Task(const std::string &myName, const std::function &f); + + // Future objects are not copyable. + Task(const Task &) = delete; + + ~Task() override; + + Task &operator=(const Task &) = delete; + + // Move constructor and Assignment are not supported. + // Too many things in this class. + Task(Task &&) = delete; + + Task &operator=(Task &&) = delete; + + Status GetTaskErrorIfAny(); + + void ChangeName(const std::string &newName) { my_name_ = newName; } + + // To execute the _fncObj + void operator()(); + + Node node; + Node group; + Node free; + + // Run the task + Status Run(); + + Status Join(); + + bool Running() const { return running_; } + + bool CaughtSevereException() const { return caught_severe_exception_; } + + bool IsMasterThread() const { return is_master_; } + + std::thread::id get_id() { return id_; } + + std::string MyName() { return my_name_; } + + // An operator used by std::find + bool operator==(const Task &other) const { return (this == &other); } + + bool operator!=(const Task &other) const { return !(*this == other); } + + void Post() { wp_.Set(); } + + Status Wait() { return (wp_.Wait()); } + + void set_task_group(TaskGroup *vg); + + private: + std::mutex mux_; + std::string my_name_; + Status rc_; + WaitPost wp_; + // Task need to provide definition for this function. It + // will be called by thread function. + std::function fnc_obj_; + // Misc fields used by TaskManager. + TaskGroup *task_group_; + std::thread thrd_; + std::thread::id id_; + bool is_master_; + volatile bool running_; + volatile bool caught_severe_exception_; + + void ShutdownGroup(); + TaskGroup *MyTaskGroup(); +}; + +extern thread_local Task *gMyTask; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_TASK_H_ diff --git a/mindspore/ccsrc/dataset/util/task_manager.cc b/mindspore/ccsrc/dataset/util/task_manager.cc new file mode 100644 index 00000000000..a9f509385e5 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/task_manager.cc @@ -0,0 +1,327 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "./securec.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +// This takes the same parameter as Task constructor. +Status TaskManager::CreateAsyncTask(const std::string &my_name, const std::function &f, TaskGroup *vg, + Task **task) { + // We need to block destructor coming otherwise we will deadlock. We will grab the + // stateLock in shared allowing CreateAsyncTask to run concurrently. + SharedLock stateLck(&state_lock_); + // Now double check the state + if (ServiceState() == STATE::kStopInProg || ServiceState() == STATE::kStopped) { + return Status(StatusCode::kInterrupted, __LINE__, __FILE__, "TaskManager is shutting down"); + } + RETURN_IF_NOT_OK(GetFreeTask(my_name, f, task)); + if (vg == nullptr) { + RETURN_STATUS_UNEXPECTED("TaskGroup is null"); + } + // Previously there is a timing hole where the thread is spawn but hit error immediately before we can set + // the TaskGroup pointer. We will do the set here before we call run(). The run() will do the registration. + (*task)->set_task_group(vg); + // Link to the master lru list. + { + UniqueLock lck(&lru_lock_); + lru_.Append(*task); + } + // Link to the group list as well before we spawn. + { + UniqueLock lck(&vg->rw_lock_); + vg->grp_list_.Append(*task); + } + // Track all the TaskGroup. Used for control-c + { + LockGuard lck(&tg_lock_); + this->grp_list_.insert(vg); + } + (*task)->wp_.Register(vg); + RETURN_IF_NOT_OK((*task)->Run()); + // Wait for the thread to initialize successfully. + RETURN_IF_NOT_OK((*task)->Wait()); + return Status::OK(); +} + +Status TaskManager::join_all() { + Status rc; + Status rc2; + SharedLock lck(&lru_lock_); + for (Task &tk : lru_) { + rc = tk.Join(); + if (rc.IsError()) { + rc2 = rc; + } + } + return rc2; +} + +void TaskManager::interrupt_all() noexcept { + global_interrupt_ = 1; + LockGuard lck(&tg_lock_); + for (TaskGroup *vg : grp_list_) { + auto svc = vg->GetIntrpService(); + if (svc) { + // Stop the interrupt service. No new request is accepted. + svc->ServiceStop(); + svc->InterruptAll(); + } + } + (void)master_->Interrupt(); +} + +Task *TaskManager::FindMe() { return gMyTask; } + +TaskManager::TaskManager() try : global_interrupt_(0), + lru_(&Task::node), + free_lst_(&Task::free), + watchdog_grp_(nullptr), + watchdog_(nullptr) { + std::shared_ptr mp = Services::GetInstance().GetServiceMemPool(); + Allocator alloc(mp); + // Create a dummy Task for the master thread (this thread) + master_ = std::allocate_shared(alloc, "master", []() -> Status { return Status::OK(); }); + master_->id_ = this_thread::get_id(); + master_->running_ = true; + master_->is_master_ = true; + gMyTask = master_.get(); + // Initialize the semaphore for the watchdog + errno_t rc = sem_init(&sem_, 0, 0); + if (rc == -1) { + MS_LOG(INFO) << "Unable to initialize a semaphore. Errno = " << rc << "."; + std::terminate(); + } +} catch (const std::exception &e) { + MS_LOG(ERROR) << "MindData initialization failed: " << e.what() << "."; + std::terminate(); +} + +TaskManager::~TaskManager() { + if (watchdog_) { + WakeUpWatchDog(); + watchdog_->thrd_.join(); + // watchdog_grp_ and watchdog_ pointers come from Services::GetInstance().GetServiceMemPool() which we will free it + // on shutdown. So no need to free these pointers one by one. + watchdog_grp_ = nullptr; + watchdog_ = nullptr; + } + (void)sem_destroy(&sem_); +} + +Status TaskManager::DoServiceStart() { + MS_LOG(INFO) << "Starting Task Manager."; + // Create a watchdog for control-c + std::shared_ptr mp = Services::GetInstance().GetServiceMemPool(); + // A dummy group just for the watchdog. We aren't really using it. But most code assumes a thread must + // belong to a group. + auto f = std::bind(&TaskManager::WatchDog, this); + Status rc; + watchdog_grp_ = new (&rc, mp) TaskGroup(); + RETURN_IF_NOT_OK(rc); + rc = watchdog_grp_->CreateAsyncTask("Watchdog", f, &watchdog_); + if (rc.IsError()) { + ::operator delete(watchdog_grp_, mp); + watchdog_grp_ = nullptr; + return rc; + } + grp_list_.erase(watchdog_grp_); + lru_.Remove(watchdog_); + return Status::OK(); +} + +Status TaskManager::DoServiceStop() { + WakeUpWatchDog(); + interrupt_all(); + return Status::OK(); +} + +Status TaskManager::WatchDog() { + TaskManager::FindMe()->Post(); + errno_t err = sem_wait(&sem_); + if (err == -1) { + RETURN_STATUS_UNEXPECTED("Errno = " + std::to_string(errno)); + } + // We are woken up by control-c and we are going to stop all threads that are running. + // In addition, we also want to prevent new thread from creating. This can be done + // easily by calling the parent function. + RETURN_IF_NOT_OK(ServiceStop()); + return Status::OK(); +} + +// Follow the group link and interrupt other +// Task in the same group. It is used by +// Watchdog only. +void TaskManager::InterruptGroup(Task &curTk) { + TaskGroup *vg = curTk.MyTaskGroup(); + vg->interrupt_all(); +} + +void TaskManager::InterruptMaster(const Status &rc) { + TaskManager &tm = TaskManager::GetInstance(); + std::shared_ptr master = tm.master_; + std::lock_guard lck(master->mux_); + (void)master->Interrupt(); + if (rc.IsError() && master->rc_.IsOk()) { + master->rc_ = rc; + master->caught_severe_exception_ = true; + } +} + +Status TaskManager::GetMasterThreadRc() { + TaskManager &tm = TaskManager::GetInstance(); + std::shared_ptr master = tm.master_; + Status rc = tm.master_->GetTaskErrorIfAny(); + if (rc.IsError()) { + // Reset the state once we retrieve the value. + std::lock_guard lck(master->mux_); + master->rc_ = Status::OK(); + master->caught_severe_exception_ = false; + master->ResetIntrpState(); + } + return rc; +} + +void TaskManager::ReturnFreeTask(Task *p) noexcept { + // Take it out from lru_ if any + { + UniqueLock lck(&lru_lock_); + auto it = std::find(lru_.begin(), lru_.end(), *p); + if (it != lru_.end()) { + lru_.Remove(p); + } + } + // We need to deallocate the string resources associated with the Task class + // before we cache its memory for future use. + p->~Task(); + // Put it back into free list + { + LockGuard lck(&free_lock_); + free_lst_.Append(p); + } +} + +Status TaskManager::GetFreeTask(const std::string &my_name, const std::function &f, Task **p) { + if (p == nullptr) { + RETURN_STATUS_UNEXPECTED("p is null"); + } + Task *q = nullptr; + // First try the free list + { + LockGuard lck(&free_lock_); + if (free_lst_.count > 0) { + q = free_lst_.head; + free_lst_.Remove(q); + } + } + if (q) { + new (q) Task(my_name, f); + } else { + std::shared_ptr mp = Services::GetInstance().GetServiceMemPool(); + Status rc; + q = new (&rc, mp) Task(my_name, f); + RETURN_IF_NOT_OK(rc); + } + *p = q; + return Status::OK(); +} + +Status TaskGroup::CreateAsyncTask(const std::string &my_name, const std::function &f, Task **ppTask) { + auto pMytask = TaskManager::FindMe(); + // We need to block ~TaskGroup coming otherwise we will deadlock. We will grab the + // stateLock in shared allowing CreateAsyncTask to run concurrently. + SharedLock state_lck(&state_lock_); + // Now double check the state + if (ServiceState() != STATE::kRunning) { + return Status(StatusCode::kInterrupted, __LINE__, __FILE__, "Taskgroup is shutting down"); + } + TaskManager &dm = TaskManager::GetInstance(); + Task *pTask = nullptr; + // If the group is already in error, early exit too. + // We can't hold the rc_mux_ throughout because the thread spawned by CreateAsyncTask may hit error which + // will try to shutdown the group and grab the rc_mux_ and we will deadlock. + { + std::unique_lock rcLock(rc_mux_); + if (rc_.IsError()) { + return pMytask->IsMasterThread() ? rc_ : Status(StatusCode::kInterrupted); + } + } + RETURN_IF_NOT_OK(dm.CreateAsyncTask(my_name, f, this, &pTask)); + if (ppTask) { + *ppTask = pTask; + } + return Status::OK(); +} + +void TaskGroup::interrupt_all() noexcept { (void)intrp_svc_->InterruptAll(); } + +Status TaskGroup::join_all() { + Status rc; + Status rc2; + SharedLock lck(&rw_lock_); + for (Task &tk : grp_list_) { + rc = tk.Join(); + if (rc.IsError()) { + rc2 = rc; + } + } + return rc2; +} + +Status TaskGroup::DoServiceStop() { + intrp_svc_->ServiceStop(); + interrupt_all(); + return (join_all()); +} + +TaskGroup::TaskGroup() : grp_list_(&Task::group), intrp_svc_(nullptr) { + std::shared_ptr mp = Services::GetInstance().GetServiceMemPool(); + Allocator alloc(mp); + intrp_svc_ = std::allocate_shared(alloc); + (void)Service::ServiceStart(); +} + +TaskGroup::~TaskGroup() { + (void)Service::ServiceStop(); + // The TaskGroup is going out of scope, and we can return the Task list to the free list. + Task *cur = grp_list_.head; + TaskManager &tm = TaskManager::GetInstance(); + while (cur) { + Task *next = cur->group.next; + grp_list_.Remove(cur); + tm.ReturnFreeTask(cur); + cur = next; + } + { + LockGuard lck(&tm.tg_lock_); + (void)tm.grp_list_.erase(this); + } +} + +Status TaskGroup::GetTaskErrorIfAny() { + SharedLock lck(&rw_lock_); + for (Task &tk : grp_list_) { + RETURN_IF_NOT_OK(tk.GetTaskErrorIfAny()); + } + return Status::OK(); +} + +std::shared_ptr TaskGroup::GetIntrpService() { return intrp_svc_; } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/task_manager.h b/mindspore/ccsrc/dataset/util/task_manager.h new file mode 100644 index 00000000000..cd670524498 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/task_manager.h @@ -0,0 +1,185 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_TASK_MANAGER_H_ +#define DATASET_UTIL_TASK_MANAGER_H_ + +#include +#include // for sig_atomic_t +#include +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/intrp_service.h" +#include "dataset/util/lock.h" +#include "dataset/util/services.h" +#include "dataset/util/status.h" +#include "dataset/util/task.h" + +namespace mindspore { +namespace dataset { +namespace thread { +using id = std::thread::id; +} // namespace thread + +namespace this_thread { +inline thread::id get_id() { return std::this_thread::get_id(); } +} // namespace this_thread + +class TaskManager : public Service { + public: + friend class Services; + + friend class TaskGroup; + + ~TaskManager() override; + + TaskManager(const TaskManager &) = delete; + + TaskManager &operator=(const TaskManager &) = delete; + + static TaskManager &GetInstance() noexcept { return Services::getTaskMgrInstance(); } + + Status DoServiceStart() override; + + Status DoServiceStop() override; + + // A public global interrupt flag for signal handlers + volatile sig_atomic_t global_interrupt_; + + // API + // This takes the same parameter as Task constructor. Take a look + // of the test-thread.cc for usage. + Status CreateAsyncTask(const std::string &my_name, const std::function &f, TaskGroup *vg, Task **); + + // Same usage as boot thread group + Status join_all(); + + void interrupt_all() noexcept; + + // Locate a particular Task. + static Task *FindMe(); + + static void InterruptGroup(Task &); + + static Status GetMasterThreadRc(); + + static void InterruptMaster(const Status &rc = Status::OK()); + + static void WakeUpWatchDog() { + TaskManager &tm = TaskManager::GetInstance(); + (void)sem_post(&tm.sem_); + } + + void ReturnFreeTask(Task *p) noexcept; + + Status GetFreeTask(const std::string &my_name, const std::function &f, Task **p); + + Status WatchDog(); + + private: + RWLock lru_lock_; + SpinLock free_lock_; + SpinLock tg_lock_; + std::shared_ptr master_; + List lru_; + List free_lst_; + sem_t sem_; + TaskGroup *watchdog_grp_; + std::set grp_list_; + Task *watchdog_; + + TaskManager(); +}; + +// A group of related tasks. +class TaskGroup : public Service { + public: + friend class Task; + friend class TaskManager; + + Status CreateAsyncTask(const std::string &my_name, const std::function &f, Task **pTask = nullptr); + + void interrupt_all() noexcept; + + Status join_all(); + + int size() const noexcept { return grp_list_.count; } + + Status DoServiceStart() override { return Status::OK(); } + + Status DoServiceStop() override; + + TaskGroup(); + + ~TaskGroup() override; + + Status GetTaskErrorIfAny(); + + std::shared_ptr GetIntrpService(); + + private: + Status rc_; + // Can't use rw_lock_ as we will lead to deadlatch. Create another mutex to serialize access to rc_. + std::mutex rc_mux_; + RWLock rw_lock_; + List grp_list_; + std::shared_ptr intrp_svc_; +}; + +namespace this_thread { +inline bool is_interrupted() { + TaskManager &tm = TaskManager::GetInstance(); + if (tm.global_interrupt_ == 1) { + return true; + } + Task *my_task = TaskManager::FindMe(); + return (my_task != nullptr) ? my_task->Interrupted() : false; +} +} // namespace this_thread + +#define RETURN_IF_INTERRUPTED() \ + do { \ + if (mindspore::dataset::this_thread::is_interrupted()) { \ + Task *myTask = TaskManager::FindMe(); \ + if (myTask->IsMasterThread() && myTask->CaughtSevereException()) { \ + return TaskManager::GetMasterThreadRc(); \ + } else { \ + return Status(StatusCode::kInterrupted); \ + } \ + } \ + } while (false) + +inline Status interruptible_wait(std::condition_variable *cv, std::unique_lock *lk, + const std::function &pred) noexcept { + if (!pred()) { + do { + RETURN_IF_INTERRUPTED(); + try { + (void)cv->wait_for(*lk, std::chrono::milliseconds(1)); + } catch (std::exception &e) { + // Anything thrown by wait_for is considered system error. + RETURN_STATUS_UNEXPECTED(e.what()); + } + } while (!pred()); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_TASK_MANAGER_H_ diff --git a/mindspore/ccsrc/dataset/util/treap.h b/mindspore/ccsrc/dataset/util/treap.h new file mode 100644 index 00000000000..777b7c5701d --- /dev/null +++ b/mindspore/ccsrc/dataset/util/treap.h @@ -0,0 +1,407 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_TREAP_H_ +#define DATASET_UTIL_TREAP_H_ + +#include +#include +#include +#include +#include + +namespace mindspore { +namespace dataset { +// A treap is a combination of binary search tree and heap. Each key is given a priority. The priority +// for any non-leaf node is greater than or equal to the priority of its children. +// @tparam K +// Data type of key +// @tparam P +// Data type of priority +// @tparam KC +// Class to compare key. Default to std::less +// @tparam KP +// Class to compare priority. Default to std:less +template , typename KP = std::less

> +class Treap { + public: + using key_type = K; + using priority_type = P; + using key_compare = KC; + using priority_compare = KP; + + struct NodeValue { + key_type key; + priority_type priority; + }; + + class TreapNode { + public: + TreapNode() : left(nullptr), right(nullptr) {} + ~TreapNode() { + left = nullptr; + right = nullptr; + } + NodeValue nv; + TreapNode *left; + TreapNode *right; + }; + + // search API + // @param k + // key to search for + // @return + // a pair is returned. The 2nd value of type bool indicate if the search is successful. + // If true, the first value of the pair contains the key and the priority. + std::pair Search(key_type k) const { + auto *n = Search(root_, k); + if (n != nullptr) { + return std::make_pair(n->nv, true); + } else { + return std::make_pair(NodeValue{key_type(), priority_type()}, false); + } + } + + // @return + // Return the root of the heap. It has the highest priority. But not necessarily the first key. + std::pair Top() const { + if (root_) { + return std::make_pair(root_->nv, true); + } else { + return std::make_pair(NodeValue{key_type(), priority_type()}, false); + } + } + + // Remove the root of the heap. + void Pop() { + if (root_) { + DeleteKey(root_->nv.key); + } + } + + // Insert API. + // @param k + // The key to insert. + // @param p + // The priority of the key. + void Insert(key_type k, priority_type p) { root_ = Insert(root_, k, p); } + + // Delete a key. + // @param k + void DeleteKey(key_type k) { root_ = DeleteNode(root_, k); } + + Treap() : root_(nullptr), count_(0) { free_list_.reserve(kResvSz); } + + ~Treap() noexcept { + DeleteTreap(root_); + while (!free_list_.empty()) { + TreapNode *n = free_list_.back(); + delete (n); + free_list_.pop_back(); + } + } + + class iterator : public std::iterator { + public: + explicit iterator(Treap *tr) : tr_(tr), cur_(nullptr) { + if (tr_) { + cur_ = tr_->root_; + while (cur_) { + stack_.push(cur_); + cur_ = cur_->left; + } + } + if (!stack_.empty()) { + cur_ = stack_.top(); + } else { + cur_ = nullptr; + } + } + ~iterator() { + tr_ = nullptr; + cur_ = nullptr; + } + + NodeValue &operator*() { return cur_->nv; } + + NodeValue *operator->() { return &(cur_->nv); } + + const TreapNode &operator*() const { return *cur_; } + + const TreapNode *operator->() const { return cur_; } + + bool operator==(const iterator &rhs) const { return cur_ == rhs.cur_; } + + bool operator!=(const iterator &rhs) const { return cur_ != rhs.cur_; } + + // Prefix increment + iterator &operator++() { + if (cur_) { + stack_.pop(); + if (cur_->right) { + TreapNode *n = cur_->right; + while (n) { + stack_.push(n); + n = n->left; + } + } + } + if (!stack_.empty()) { + cur_ = stack_.top(); + } else { + cur_ = nullptr; + } + return *this; + } + + // Postfix increment + iterator operator++(int junk) { + iterator tmp(*this); + if (cur_) { + stack_.pop(); + if (cur_->right) { + TreapNode *n = cur_->right; + while (n) { + stack_.push(n); + n = n->left; + } + } + } + if (!stack_.empty()) { + cur_ = stack_.top(); + } else { + cur_ = nullptr; + } + return tmp; + } + + private: + Treap *tr_; + TreapNode *cur_; + std::stack stack_; + }; + + class const_iterator : public std::iterator { + public: + explicit const_iterator(const Treap *tr) : tr_(tr), cur_(nullptr) { + if (tr_) { + cur_ = tr_->root_; + while (cur_) { + stack_.push(cur_); + cur_ = cur_->left; + } + } + if (!stack_.empty()) { + cur_ = stack_.top(); + } else { + cur_ = nullptr; + } + } + ~const_iterator() { + tr_ = nullptr; + cur_ = nullptr; + } + + const NodeValue &operator*() const { return cur_->nv; } + + const NodeValue *operator->() const { return &(cur_->nv); } + + bool operator==(const const_iterator &rhs) const { return cur_ == rhs.cur_; } + + bool operator!=(const const_iterator &rhs) const { return cur_ != rhs.cur_; } + + // Prefix increment + const_iterator &operator++() { + if (cur_) { + stack_.pop(); + if (cur_->right) { + TreapNode *n = cur_->right; + while (n) { + stack_.push(n); + n = n->left; + } + } + } + if (!stack_.empty()) { + cur_ = stack_.top(); + } else { + cur_ = nullptr; + } + return *this; + } + + // Postfix increment + const_iterator operator++(int junk) { + iterator tmp(*this); + if (cur_) { + stack_.pop(); + if (cur_->right) { + TreapNode *n = cur_->right; + while (n) { + stack_.push(n); + n = n->left; + } + } + } + if (!stack_.empty()) { + cur_ = stack_.top(); + } else { + cur_ = nullptr; + } + return tmp; + } + + private: + const Treap *tr_; + TreapNode *cur_; + std::stack stack_; + }; + + iterator begin() { return iterator(this); } + + iterator end() { return iterator(nullptr); } + + const_iterator begin() const { return const_iterator(this); } + + const_iterator end() const { return const_iterator(nullptr); } + + const_iterator cbegin() { return const_iterator(this); } + + const_iterator cend() { return const_iterator(nullptr); } + + bool empty() { return root_ == nullptr; } + + size_t size() { return count_; } + + private: + TreapNode *NewNode() { + TreapNode *n = nullptr; + if (!free_list_.empty()) { + n = free_list_.back(); + free_list_.pop_back(); + new (n) TreapNode(); + } else { + n = new TreapNode(); + } + return n; + } + + void FreeNode(TreapNode *n) { free_list_.push_back(n); } + + void DeleteTreap(TreapNode *n) noexcept { + if (n == nullptr) { + return; + } + TreapNode *x = n->left; + TreapNode *y = n->right; + delete (n); + DeleteTreap(x); + DeleteTreap(y); + } + + TreapNode *RightRotate(TreapNode *y) { + TreapNode *x = y->left; + TreapNode *T2 = x->right; + x->right = y; + y->left = T2; + return x; + } + + TreapNode *LeftRotate(TreapNode *x) { + TreapNode *y = x->right; + TreapNode *T2 = y->left; + y->left = x; + x->right = T2; + return y; + } + + TreapNode *Search(TreapNode *n, key_type k) const { + key_compare keyCompare; + if (n == nullptr) { + return n; + } else if (keyCompare(k, n->nv.key)) { + return Search(n->left, k); + } else if (keyCompare(n->nv.key, k)) { + return Search(n->right, k); + } else { + return n; + } + } + + TreapNode *Insert(TreapNode *n, key_type k, priority_type p) { + key_compare keyCompare; + priority_compare priorityCompare; + if (n == nullptr) { + n = NewNode(); + n->nv.key = k; + n->nv.priority = p; + count_++; + return n; + } + if (keyCompare(k, n->nv.key)) { + n->left = Insert(n->left, k, p); + if (priorityCompare(n->nv.priority, n->left->nv.priority)) { + n = RightRotate(n); + } + } else if (keyCompare(n->nv.key, k)) { + n->right = Insert(n->right, k, p); + if (priorityCompare(n->nv.priority, n->right->nv.priority)) { + n = LeftRotate(n); + } + } else { + // If we insert the same key again, do nothing. + return n; + } + return n; + } + + TreapNode *DeleteNode(TreapNode *n, key_type k) { + key_compare keyCompare; + priority_compare priorityCompare; + if (n == nullptr) { + return n; + } + if (keyCompare(k, n->nv.key)) { + n->left = DeleteNode(n->left, k); + } else if (keyCompare(n->nv.key, k)) { + n->right = DeleteNode(n->right, k); + } else if (n->left == nullptr) { + TreapNode *t = n; + n = n->right; + FreeNode(t); + count_--; + } else if (n->right == nullptr) { + TreapNode *t = n; + n = n->left; + FreeNode(t); + count_--; + } else if (priorityCompare(n->left->nv.priority, n->right->nv.priority)) { + n = LeftRotate(n); + n->left = DeleteNode(n->left, k); + } else { + n = RightRotate(n); + n->right = DeleteNode(n->right, k); + } + return n; + } + + static constexpr int kResvSz = 512; + TreapNode *root_; + size_t count_; + std::vector free_list_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_TREAP_H_ diff --git a/mindspore/ccsrc/dataset/util/wait_post.cc b/mindspore/ccsrc/dataset/util/wait_post.cc new file mode 100644 index 00000000000..99ee0cb77f1 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/wait_post.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/wait_post.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +WaitPost::WaitPost() : value_(0) {} + +Status WaitPost::Wait() { + std::unique_lock lck(mutex_); + return (wait_cond_.Wait(&lck, [this]() { return value_ != 0; })); +} + +void WaitPost::Set() { + std::unique_lock lck(mutex_); + value_ = 1; + wait_cond_.NotifyAll(); +} + +void WaitPost::Clear() { + std::unique_lock lck(mutex_); + value_ = 0; +} + +void WaitPost::Register(TaskGroup *vg) { (void)wait_cond_.Register(vg->GetIntrpService()); } + +void WaitPost::ResetIntrpState() { wait_cond_.ResetIntrpState(); } + +Status WaitPost::Deregister() { return wait_cond_.Deregister(); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/wait_post.h b/mindspore/ccsrc/dataset/util/wait_post.h new file mode 100644 index 00000000000..bac43f7a4ec --- /dev/null +++ b/mindspore/ccsrc/dataset/util/wait_post.h @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_WAIT_POST_H_ +#define DATASET_UTIL_WAIT_POST_H_ + +#include +#include "dataset/util/cond_var.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class TaskGroup; + +class WaitPost { + public: + WaitPost(); + + ~WaitPost() = default; + + Status Wait(); + + void Set(); + + void Clear(); + + void Register(TaskGroup *vg); + + Status Deregister(); + + void ResetIntrpState(); + + private: + std::mutex mutex_; + CondVar wait_cond_; + int value_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_WAIT_POST_H_ diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt new file mode 100644 index 00000000000..a88745b864f --- /dev/null +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -0,0 +1,12 @@ + +set(_DEBUG_SRC_LIST + "${CMAKE_CURRENT_SOURCE_DIR}/anf_ir_dump.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/anf_ir_utils.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/draw.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/dump_proto.cc") + +if(ENABLE_DUMP_E2E) + list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/e2e_dump.cc") +endif(ENABLE_DUMP_E2E) + +add_library(_mindspore_debug_obj OBJECT ${_DEBUG_SRC_LIST}) \ No newline at end of file diff --git a/mindspore/ccsrc/debug/anf_ir_dump.cc b/mindspore/ccsrc/debug/anf_ir_dump.cc new file mode 100644 index 00000000000..9e24a17edee --- /dev/null +++ b/mindspore/ccsrc/debug/anf_ir_dump.cc @@ -0,0 +1,483 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/anf_ir_dump.h" +#include +#include +#include +#include + +#include "ir/primitive.h" +#include "ir/func_graph.h" +#include "device/kernel_info.h" +#include "utils/graph_utils.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +const std::string ToShortString(const TypeId &typeId) { + std::string label = TypeIdLabel(typeId); + std::string prefix = "kNumberType"; + if (prefix.length() > label.length()) { + return label; + } + auto position = label.find(prefix); + // position is 0 when label begins with prefix + if (position != 0) { + return label; + } + auto sub_position = position + prefix.length(); + if (sub_position >= label.length()) { + return label; + } + return label.substr(sub_position); +} + +void PrintKernelFormatAndType(std::ostringstream &buffer, const std::string &fmt, const TypeId &type, + const std::vector &shape) { + buffer << "<" << ToShortString(type); + if (!fmt.empty()) { + buffer << "x" << fmt << shape; + } + buffer << ">"; +} + +void PrintNodeOutputType(std::ostringstream &buffer, const AnfNodePtr &nd) { + if (nd == nullptr) { + return; + } + + abstract::ShapePtr shape = dyn_cast(nd->Shape()); + TypePtr type = dyn_cast(nd->Type()); + if ((nullptr != shape) && (nullptr != type)) { + buffer << "<" << type << "x" << shape->shape() << ">"; + } else if (nullptr != type) { + buffer << "<" << type << ">"; + } else { + buffer << ""; + } +} + +void PrintNodeInputType(std::ostringstream &buffer, const AnfNodePtr &nd) { + if (nd == nullptr) { + return; + } + + std::vector inputs = SuccIncoming(nd); + size_t len = inputs.size(); + if (len > 1) { + // skip inputs[0] which is Primitive value node + for (size_t i = 1; i < len; ++i) { + AnfNodePtr in = inputs[i]; + if (i != 1) { + buffer << ", "; + } + PrintNodeOutputType(buffer, in); + } + } +} + +struct SubGraphIRInfo { + int32_t local_var; + std::ostringstream buffer; + OrderedMap local_var_map; +}; + +void DumpGrobalInfoEntry(const FuncGraphPtr &graph, std::ostringstream &buffer) { + if (graph == nullptr) { + return; + } + + buffer << "#IR entry : @" << graph->ToString() << "." << graph->debug_info()->get_id() << std::endl; + buffer << "#flags :" << std::endl; + for (const auto &flag : graph->flags()) { + buffer << flag.first << " : " << flag.second << std::endl; + } +} + +void DumpKernelInfo(const CNodePtr &node, const std::shared_ptr &gsub) { + if (node == nullptr || gsub == nullptr) { + return; + } + auto kernel_info = node->kernel_info(); + if (kernel_info == nullptr || kernel_info->select_kernel_build_info() == nullptr) { + return; + } + + gsub->buffer << " : ("; + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(node); ++i) { + if (i != 0) { + gsub->buffer << ", "; + } + auto format = AnfAlgo::GetInputFormat(node, i); + auto type = AnfAlgo::GetInputDeviceDataType(node, i); + auto shape = AnfAlgo::GetInputDeviceShape(node, i); + PrintKernelFormatAndType(gsub->buffer, format, type, shape); + } + gsub->buffer << ") -> ("; + for (size_t i = 0; i < AnfAlgo::GetOutputTensorNum(node); ++i) { + if (i != 0) { + gsub->buffer << ", "; + } + auto format = AnfAlgo::GetOutputFormat(node, i); + auto type = AnfAlgo::GetOutputDeviceDataType(node, i); + auto shape = AnfAlgo::GetOutputDeviceShape(node, i); + PrintKernelFormatAndType(gsub->buffer, format, type, shape); + } + gsub->buffer << ")"; + gsub->buffer << std::endl; +} + +void DumpParams(const FuncGraphPtr &graph, std::ostringstream &buffer, OrderedMap *para_map) { + if (graph == nullptr) { + MS_LOG(INFO) << "param graph is nullptr."; + return; + } + std::vector parameters = graph->parameters(); + buffer << "#Total params : " << parameters.size() << std::endl; + buffer << std::endl; + + // dump parameters + int32_t para = 1; + for (const auto &p : parameters) { + if (p == nullptr) { + continue; + } + auto parameter_ptr = p->cast(); + if (parameter_ptr == nullptr) { + MS_LOG(EXCEPTION) << "p cannot cast to ParameterPtr"; + } + buffer << "%para" << para << " = " << parameter_ptr->name() << " : "; + // print parameters' type and shape + PrintNodeOutputType(buffer, p); + auto kernel_info = p->kernel_info(); + if (kernel_info != nullptr && kernel_info->select_kernel_build_info() != nullptr) { + buffer << " : "; + auto type = AnfAlgo::GetOutputDeviceDataType(p, 0); + auto format = AnfAlgo::GetOutputFormat(p, 0); + auto shape = AnfAlgo::GetOutputDeviceShape(p, 0); + PrintKernelFormatAndType(buffer, format, type, shape); + buffer << " : IsWeight:" << std::boolalpha << AnfAlgo::IsParameterWeight(parameter_ptr); + } + buffer << std::endl; + + if (para_map != nullptr) { + (*para_map)[p] = para++; + } + MS_LOG(DEBUG) << "record param: " << p->ToString() << " graph belong : " << p->func_graph()->ToString(); + } +} + +void DumpOperator(const AnfNodePtr &op, const std::shared_ptr &gsub) { + if (op == nullptr) { + MS_LOG(INFO) << "param op is nullptr"; + return; + } + if (gsub == nullptr) { + MS_LOG(INFO) << "param gsub is nullptr"; + return; + } + + if (IsValueNode(op)) { + FuncGraphPtr fg = GetValueNode(op); + if (fg != nullptr) { + gsub->buffer << "call @" << fg->ToString() << "." << fg->debug_info()->get_id(); + } + } else if (op->isa()) { + if (gsub->local_var_map.find(op) != gsub->local_var_map.end()) { + gsub->buffer << "%" << gsub->local_var_map[op]; + } + } else if (op->isa()) { + gsub->buffer << GetValueNode(op)->ToString(); + } else { + gsub->buffer << op->ToString(); + } +} + +void DumpOperands(const AnfNodePtr &nd, OrderedMap *para_map, + const std::shared_ptr &gsub) { + if (nd == nullptr || para_map == nullptr || gsub == nullptr) { + return; + } + + gsub->buffer << "("; + std::vector inputs = SuccIncoming(nd); + size_t len = inputs.size(); + if (len > 1) { + // skip inputs[0] which is Primitive valuenode + for (size_t i = 1; i < len; ++i) { + AnfNodePtr in = inputs[i]; + MS_EXCEPTION_IF_NULL(in); + if (i != 1) { + gsub->buffer << ", "; + } + if (in->isa()) { + if (!(*para_map)[in]) { + gsub->buffer << "%arg"; + } else { + gsub->buffer << "%para" << (*para_map)[in]; + } + } else if (in->isa()) { + gsub->buffer << "%" << gsub->local_var_map[in]; + } else if (in->isa() && !IsValueNode(in)) { + // non Primitive valuenode + gsub->buffer << GetValueNode(in)->ToString(); + } else if (IsValueNode(in)) { + FuncGraphPtr fg = GetValueNode(in); + gsub->buffer << "@" << fg->ToString() << "." << fg->debug_info()->get_id(); + } else { + gsub->buffer << in->ToString(); + } + } + } + gsub->buffer << ")"; +} + +void DumpParallelInfo(const CNodePtr &node, const std::shared_ptr &gsub) { + if ((node == nullptr) || (gsub == nullptr)) { + return; + } + + auto operator_info = node->operator_info(); + if (operator_info == nullptr) { + return; + } + + auto strategy = operator_info->strategy(); + if (strategy == nullptr) { + return; + } + + ValuePtr temp = MakeValue(strategy->GetInputDim()); + gsub->buffer << " { strategy: "; + gsub->buffer << temp->ToString(); + gsub->buffer << " }"; +} + +void DumpOperateAttrs(const AnfNodePtr &op, const std::shared_ptr &gsub) { + if (op == nullptr || gsub == nullptr) { + return; + } + + if (IsValueNode(op)) { + PrimitivePtr primitive = GetValueNode(op); + if (!primitive->instance_name().empty()) { + gsub->buffer << " {"; + gsub->buffer << "instance name" + << ": "; + gsub->buffer << primitive->instance_name(); + gsub->buffer << "}"; + } + auto attrs = primitive->attrs(); + if (!attrs.empty()) { + gsub->buffer << " {"; + int i = 0; + for (const auto &attr : attrs) { + if (attr.first == PARALLEL_STRATEGY) { + continue; // skip the strategy + } + if (i++ != 0) { + gsub->buffer << ", "; + } + gsub->buffer << attr.first << ": "; + if (attr.second == nullptr) { + gsub->buffer << "null"; + } else { + gsub->buffer << attr.second->ToString(); + } + } + gsub->buffer << "}"; + } + } + gsub->buffer << std::endl; +} + +void DumpShape(const AnfNodePtr &nd, const FuncGraphPtr &sub_graph, const std::shared_ptr &gsub) { + if (nd == nullptr || sub_graph == nullptr || gsub == nullptr) { + return; + } + + if (nd != sub_graph->get_return()) { + gsub->buffer << " : ("; + PrintNodeInputType(gsub->buffer, nd); + gsub->buffer << ") -> ("; + PrintNodeOutputType(gsub->buffer, nd); + gsub->buffer << ")"; + } else { + gsub->buffer << " : ("; + PrintNodeInputType(gsub->buffer, nd); + gsub->buffer << ")"; + } + + gsub->buffer << std::endl; +} + +void DumpCNode(const CNodePtr &nd, const FuncGraphPtr &sub_graph, OrderedMap *const para_map, + const std::shared_ptr &gsub, bool dump_full_name = false) { + if (nd == nullptr || sub_graph == nullptr || para_map == nullptr || gsub == nullptr) { + return; + } + + if (nd != sub_graph->get_return()) { + gsub->buffer << " %" << gsub->local_var << "(" << nd->ToString() << ")" + << " = "; + gsub->local_var_map[nd] = gsub->local_var++; + } else { + gsub->buffer << " "; + } + + if (nd->inputs().empty()) { + MS_LOG(EXCEPTION) << "input of apply node is empty"; + } + + // print operator + AnfNodePtr op = nd->input(0); + DumpOperator(op, gsub); + + // print operands + DumpOperands(nd, para_map, gsub); + + // print operator attrs + DumpOperateAttrs(op, gsub); + + // print parallel info + DumpParallelInfo(nd, gsub); + + // print shape info + DumpShape(nd, sub_graph, gsub); + + // print kernel info + DumpKernelInfo(nd, gsub); + + if (dump_full_name) { + gsub->buffer << " : (" << nd->fullname_with_scope() << ")" << std::endl; + } +} + +void DumpIRInSubgraph(const std::vector &nodes, OrderedMap *para_map, + OrderedMap> *const sub_graphs, + bool dump_full_name = false) { + if (para_map == nullptr || sub_graphs == nullptr) { + return; + } + + for (const auto &nd : nodes) { + MS_EXCEPTION_IF_NULL(nd); + FuncGraphPtr sub_graph = nd->func_graph(); + if (sub_graph == nullptr) { + MS_LOG(DEBUG) << "node[" << nd->ToString() << "] belongs to no graph!"; + continue; + } + std::shared_ptr gsub = (*sub_graphs)[sub_graph]; + if (gsub == nullptr) { + gsub = std::make_shared(); + gsub->local_var = 0; + (*sub_graphs)[sub_graph] = gsub; + } + if (!nd->isa()) { + if (nd->isa()) { + // print and record output of operator if it is not 'Return' + DumpCNode(nd->cast(), sub_graph, para_map, gsub, dump_full_name); + } else { + gsub->buffer << " " << nd->ToString() << std::endl; + } + } + } +} + +void DumpSubgraph(const OrderedMap> *sub_graphs, + const FuncGraphPtr &graph, std::ofstream &fout) { + if (sub_graphs == nullptr || graph == nullptr) { + return; + } + + fout << "#Total subgraph : " << sub_graphs->size() << std::endl; + fout << std::endl; + + for (const auto &sg : *sub_graphs) { + fout << "subgraph flag:" << std::endl; + MS_EXCEPTION_IF_NULL(sg.first); + for (const auto &flag : sg.first->flags()) { + fout << flag.first << " : " << flag.second << std::endl; + } + fout << "subgraph @" << sg.first->ToString() << "."; + fout << sg.first->debug_info()->get_id() << "("; + if (sg.first != graph) { + fout << "%arg"; + } + fout << ") {" << std::endl; + MS_EXCEPTION_IF_NULL(sg.second); + fout << sg.second->buffer.str(); + fout << "}" << std::endl; + fout << std::endl; + } +} + +#ifdef ENABLE_DUMP_IR +void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_full_name) { + if (graph == nullptr) { + return; + } + if (filename.size() > PATH_MAX) { + MS_LOG(ERROR) << "file path " << filename << " is too long."; + return; + } + char real_path[PATH_MAX] = {0}; + if (nullptr == realpath(filename.c_str(), real_path)) { + MS_LOG(DEBUG) << "dir " << filename << " does not exit."; + } + + OrderedMap para_map; + std::string path_string = real_path; + ChangeFileMode(path_string, S_IRWXU); + std::ofstream fout(real_path); + std::ostringstream buffer; + + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open dump file '" << real_path << "' failed!"; + return; + } + + auto nodes = TopoSort(graph->get_return(), SuccDeeperSimple, AlwaysInclude); + + // dump global info + DumpGrobalInfoEntry(graph, buffer); + DumpParams(graph, buffer, ¶_map); + + OrderedMap> sub_graphs; + // dump ir in each sub graph + DumpIRInSubgraph(nodes, ¶_map, &sub_graphs, dump_full_name); + + // output global info + fout << buffer.str() << std::endl; + + // output each sub graph + DumpSubgraph(&sub_graphs, graph, fout); + + fout.close(); + // set file mode to read only by user + ChangeFileMode(path_string, S_IRUSR); +} +#else +void DumpIR(const std::string &, const FuncGraphPtr &, bool) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " + << "please recompile source to enable it. See help of building script."; +} +#endif +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/anf_ir_dump.h b/mindspore/ccsrc/debug/anf_ir_dump.h new file mode 100644 index 00000000000..5c4bc5eacd6 --- /dev/null +++ b/mindspore/ccsrc/debug/anf_ir_dump.h @@ -0,0 +1,29 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_ANF_IR_DUMP_H_ +#define MINDSPORE_CCSRC_DEBUG_ANF_IR_DUMP_H_ + +#include +#include +#include "ir/anf.h" + +namespace mindspore { +constexpr char PARALLEL_STRATEGY[] = "strategy"; +void DumpIR(const std::string& filename, const FuncGraphPtr& func_graph, bool dump_full_name = false); + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_ANF_IR_DUMP_H_ diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc new file mode 100644 index 00000000000..0891e211a07 --- /dev/null +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -0,0 +1,2281 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/anf_ir_utils.h" + +#include +#include +#include +#include +#include +#include + +#include "utils/graph_utils.h" +#include "utils/symbolic.h" +#include "ir/meta_func_graph.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/resolve.h" +#include "operator/composite/composite.h" +#include "utils/ordered_map.h" +#include "utils/ordered_set.h" +#include "utils/utils.h" +#include "debug/trace.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +// max number of elements in sequence +const int NUM_MAX_SEQUENCE_ELEMS = 0x00FFFFFF; + +// ============================================== MindSpore IR Common ============================================== +// get MindSpore Intermediate Representation Path +std::string GetMsIrPath(void) { + std::string path; + const char* path_ptr = getenv("MS_IR_PATH"); + if (path_ptr != nullptr) { + path = path_ptr; + char real_path[PATH_MAX] = {0}; + if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) { + MS_LOG(EXCEPTION) << "MS IR Path error, " << path_ptr; + } + path = real_path; + } + return path; +} + +std::string dump_obj(const py::object& obj, const std::string& path) { + py::module mod = parse::python_adapter::GetPyModule(parse::PYTHON_MOD_PARSE_MODULE); + py::object name = parse::python_adapter::CallPyModFn(mod, "dump_obj", obj, py::str(path)); + return py::str(name); +} + +py::object load_obj(const std::string& path) { + py::module mod = parse::python_adapter::GetPyModule(parse::PYTHON_MOD_PARSE_MODULE); + py::object obj = parse::python_adapter::CallPyModFn(mod, "load_obj", py::str(path)); + return obj; +} + +// ============================================= MindSpore IR Exporter ============================================= + +std::string GetNodeType(const AnfNodePtr& nd) { + abstract::ShapePtr shape = nd->Shape() == nullptr ? nullptr : dyn_cast(nd->Shape()); + TypePtr type = dyn_cast(nd->Type()); + std::ostringstream oss; + if ((nullptr != shape) && (nullptr != type)) { + oss << type->DumpText() << shape->DumpText(); + } else if (nullptr != type) { + oss << type->DumpText(); + } else { + oss << "Undefined"; + } + return oss.str(); +} + +std::string AnfExporter::DumpObject(const py::object& obj, const std::string& category) const { + std::string pkl_path = GetMsIrPath(); + // if not specified env 'MS_IR_PATH', do not create any files + if (pkl_path.empty() || (getenv("MS_IR_FILE") != nullptr)) { + return "null"; + } + std::string file_prefix = id_ + "." + category; + std::string file_name = dump_obj(obj, pkl_path + "/" + file_prefix); + return file_prefix + file_name; +} + +int AnfExporter::GetParamIndex(const FuncGraphPtr& func_graph, const AnfNodePtr& param, bool throw_excp) { + if (func_graph == nullptr || param == nullptr) { + return -1; + } + + FuncGraphPtr fg = func_graph; + while (fg != nullptr) { + if (exported.find(fg) == exported.end()) { + if (!export_used_) { + break; + } + MS_LOG(EXCEPTION) << "Can not find func graph '" << fg->DumpText() << "." << fg->debug_info()->get_id() << "'"; + } + auto param_map = exported[fg]; + if (param_map.find(param) != param_map.end()) { + return param_map[param]; + } + fg = fg->parent(); + } + if (throw_excp) { + MS_LOG(EXCEPTION) << "Can not find index for param '" << param->DumpText() << "' for func graph '" + << func_graph->DumpText() << "." << func_graph->debug_info()->get_id() << "'"; + } + return -1; +} + +// try to find index of parameter for SymbolicKeyInstance from all exported graphs +// NOTICE: Suppose name of all parameters in SymbolicKeyInstance are different +int AnfExporter::GetParamIndexFromExported(const AnfNodePtr& param) { + if (param == nullptr) { + return -1; + } + + int ret = -1; + for (const auto& item : exported) { + auto pram_iter = item.second.find(param); + if (pram_iter != item.second.end()) { + return pram_iter->second; + } + } + return ret; +} + +std::string AnfExporter::GetValueNodeText(const FuncGraphPtr& fg, const ValueNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + return GetValueText(fg, node->value()); +} + +std::string AnfExporter::GetMultitypeFuncGraphText(const prim::MultitypeFuncGraphPtr& mt_func_graph) { + auto py_funs = mt_func_graph->GetPyFunctions(); + if (py_funs.empty()) { + return ""; + } + + std::ostringstream oss; + + oss << "{"; + bool is_first = true; + for (const auto& py_func : py_funs) { + if (is_first) { + is_first = false; + } else { + oss << ", "; + } + oss << "("; + for (size_t i = 0; i < py_func.first.size(); ++i) { + if (i > 0) { + oss << ", "; + } + oss << py_func.first[i]->DumpText(); + } + oss << ")"; + + // dump Python Function object + oss << "@" << DumpObject(py_func.second, "F"); + } + oss << "}"; + + return oss.str(); +} + +/* inherit relation of MetaFuncGraph + * + * MetaGraph + * ├── MultitypeGraph + * ├── HyperMap + * │ └── HyperMapPy + * ├── Tail + * ├── MakeTupleGradient + * ├── GradOperation + * └── TupleAdd + */ +std::string AnfExporter::GetMetaFuncGraphText(const MetaFuncGraphPtr& meta_func_graph) { + if (meta_func_graph == nullptr) { + return ""; + } + + std::ostringstream oss; + oss << meta_func_graph->type_name() << "::" << meta_func_graph->name(); + + if (meta_func_graph->isa()) { + prim::MultitypeFuncGraphPtr mt_func_graph = meta_func_graph->cast(); + oss << GetMultitypeFuncGraphText(mt_func_graph); + } else if (meta_func_graph + ->isa()) { // this statement must before 'meta_graph->isa()' + prim::HyperMapPyPtr hyper_map = meta_func_graph->cast(); + MS_EXCEPTION_IF_NULL(hyper_map); + if (hyper_map->GetFnLeaf() != nullptr) { + oss << "{fn_leaf=" << GetMetaFuncGraphText(hyper_map->GetFnLeaf()) << "}"; + } + } else if (meta_func_graph->isa()) { + prim::HyperMapPtr hyper_map = meta_func_graph->cast(); + MS_EXCEPTION_IF_NULL(hyper_map); + if (hyper_map->GetFnLeaf() != nullptr) { + oss << "{fn_leaf=" << GetMetaFuncGraphText(hyper_map->GetFnLeaf()) << "}"; + } + } else if (meta_func_graph->isa()) { + prim::GradOperationPtr grad_op = meta_func_graph->cast(); + oss << "{get_all=" << grad_op->get_all_ << ", get_by_list=" << grad_op->get_by_list_ + << ", sens_param=" << grad_op->sens_param_ << "}"; + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else if (meta_func_graph->isa()) { + // do nothing + } else { + MS_LOG(EXCEPTION) << "Unknown MetaFuncGraph type " << meta_func_graph->type_name(); + } + + return oss.str(); +} + +std::string AnfExporter::GetPrimitiveText(const PrimitivePtr& prim) { + std::ostringstream oss; + if (prim == nullptr) { + return oss.str(); + } + oss << prim->type_name() << "::" << prim->name(); + // need to serialize internal python function of PrimitivePy and record its prim_type + if (prim->isa()) { + PrimitivePyPtr primpy = prim->cast(); + + // dump related function in PrimitivePy + oss << "@" << DumpObject(primpy->GetPyObj(), "P"); + + // output primitive type + oss << "{prim_type=" << static_cast(prim->prim_type()) << "}"; + } + + // output primitive attributes + auto attrs = prim->attrs(); + if (attrs.size() > 0) { + oss << "["; + int i = 0; + for (auto& attr : attrs) { + oss << (i > 0 ? ", " : "") << attr.first << "=" << attr.second->DumpText(); + i++; + } + oss << "]"; + } + + return oss.str(); +} + +std::string AnfExporter::GetNameSpaceText(const parse::NameSpacePtr& ns) { + std::ostringstream oss; + if (ns == nullptr) { + return oss.str(); + } + + // dump related module information in Namespace + oss << ns->type_name() << "::" << ns->module() << "@" << DumpObject(ns->obj(), "N"); + + return oss.str(); +} + +std::string AnfExporter::GetSymbolicKeyInstanceText(const FuncGraphPtr& func_graph, + const SymbolicKeyInstancePtr& sym_inst) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(sym_inst); + AnfNodePtr sym_node = sym_inst->node(); + MS_EXCEPTION_IF_NULL(sym_node); + std::ostringstream oss; + if (sym_node->isa()) { + int idx = GetParamIndex(func_graph, sym_node, false); + // if can not find SymbolicKeyInstance related parameter from ancestors, + // try to find from all exported graphs + if (idx < 0) { + idx = GetParamIndexFromExported(sym_node); + } + if (idx < 0) { + ParameterPtr p = dyn_cast(sym_node); + if (p == nullptr) { + MS_LOG(EXCEPTION) << "Sym_inst's node could not cast to parameter"; + } + MS_LOG(WARNING) << "Can not find SymbolicKeyInstance: " << p->name(); + } + oss << "SymInst(%para" << idx << ")"; + } else { + MS_LOG(EXCEPTION) << "SymbolicKeyInstance does not embed a parameter: " << sym_node->ToString(); + } + + return oss.str(); +} + +std::string AnfExporter::GetSequenceText(const FuncGraphPtr& func_graph, const ValuePtr& value) { + std::ostringstream oss; + // output ValueList, ValueTuple + ValueSequeuePtr seq = dyn_cast(value); + MS_EXCEPTION_IF_NULL(seq); + MS_EXCEPTION_IF_NULL(value); + bool is_tuple = value->isa(); + oss << (is_tuple ? "(" : "["); + bool first_flag = true; + for (auto elem : seq->value()) { + if (first_flag) { + first_flag = false; + } else { + oss << ", "; + } + oss << GetValueText(func_graph, elem); + } + oss << (is_tuple ? ")" : "]"); + return oss.str(); +} + +std::string AnfExporter::GetDictText(const FuncGraphPtr& func_graph, const ValuePtr& value) { + std::ostringstream oss; + ValueDictionaryPtr dict = value->cast(); + oss << "{"; + bool first_flag = true; + for (const auto& elem : dict->value()) { + if (first_flag) { + first_flag = false; + } else { + oss << ", "; + } + oss << "\"" << elem.first << "\": " << GetValueText(func_graph, elem.second); + } + oss << "}"; + return oss.str(); +} + +std::string AnfExporter::GetOtherValueText(const FuncGraphPtr&, const ValuePtr& value) { + std::ostringstream oss; + + if (export_used_) { + MS_LOG(EXCEPTION) << "Need to process type: " << value->type_name() << ", dump text: " << value->DumpText(); + } + oss << value->type_name() << "[" << value->DumpText() << "]"; + + return oss.str(); +} + +std::string AnfExporter::GetValueText(const FuncGraphPtr& func_graph, const ValuePtr& value) { + std::ostringstream oss; + bool is_null_ptr = (func_graph == nullptr || value == nullptr); + if (is_null_ptr) { + return oss.str(); + } + + if (value->isa()) { + oss << GetPrimitiveText(value->cast()); + } else if (value->isa()) { + MetaFuncGraphPtr meta_func_graph = value->cast(); + oss << GetMetaFuncGraphText(meta_func_graph); + } else if (value->isa()) { + oss << GetSymbolicKeyInstanceText(func_graph, value->cast()); + } else if (value->isa()) { + oss << value->DumpText(); + } else if (value->isa() || value->isa()) { + oss << value->DumpText(); + } else if (value->isa()) { + auto tensor_ptr = dyn_cast(value); + oss << value->DumpText() << "@" << DumpObject(tensor_ptr->data(), "T"); + } else if (value->isa() || value->isa() || value->isa()) { + oss << value->DumpText(); + } else if (value->isa()) { + oss << GetSequenceText(func_graph, value); + } else if (value->isa()) { + oss << GetDictText(func_graph, value); + } else if (value->isa()) { + ValueSlicePtr slice = value->cast(); + oss << slice->DumpText(); + } else if (value->isa()) { + oss << value->DumpText(); + } else if (value->isa()) { + oss << GetNameSpaceText(value->cast()); + } else if (value->isa()) { + oss << value->type_name(); + } else if (value->isa()) { + KeywordArgPtr keyword_arg = value->cast(); + oss << keyword_arg->DumpText(); + } else { + return GetOtherValueText(func_graph, value); + } + + return oss.str(); +} + +// this function is used to output node in CNode's inputs +std::string AnfExporter::GetAnfNodeText(const FuncGraphPtr& func_graph, const AnfNodePtr& node, + const std::map& apply_map) { + std::ostringstream oss; + if (func_graph == nullptr || node == nullptr) { + return oss.str(); + } + + if (node->isa()) { + auto iter = apply_map.find(node); + if (iter == apply_map.end()) { + MS_LOG(EXCEPTION) << "Can not find node '" << node->DumpText() << "' in apply_map"; + } + oss << "%" << iter->second; + } else if (node->isa()) { + oss << "%para" << GetParamIndex(func_graph, node, export_used_); + } else if (IsValueNode(node)) { + FuncGraphPtr fg = GetValueNode(node); + oss << fg->type_name() << "::fg_" << fg->debug_info()->get_id(); + + if (!func_graph_set.contains(fg) && exported.find(fg) == exported.end() && export_used_) { + func_graph_set.add(fg); + } + } else if (node->isa()) { + oss << GetValueNodeText(func_graph, node->cast()); + } else { + MS_LOG(EXCEPTION) << "Unknown node '" << node->DumpText() << "'"; + } + + return oss.str(); +} + +void AnfExporter::OutputParameters(std::ofstream& ofs, const std::vector& parameters, + OrderedMap* param_map) { + bool first_flag = true; + for (const AnfNodePtr& param : parameters) { + if (first_flag) { + first_flag = false; + ofs << " "; + } else { + ofs << " , "; + } + (*param_map)[param] = param_index; + std::string type_info = GetNodeType(param); + // output parameter and type + if (type_info == "Undefined") { + ofs << "%para" << param_index; + } else { + ofs << "%para" << param_index << " : " << type_info; + } + + // dump Default value of parameter if exists + const ParameterPtr param_ptr = dyn_cast(param); + if (param_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Param could not cast to parameter"; + } + if (param_ptr->has_default()) { + ofs << " = @" << DumpObject(param_ptr->default_param(), "D"); + } + + // output comment + ofs << " # " << param->DumpText() << "\n"; + + param_index += 1; + } +} + +void AnfExporter::OutputStatementComment(std::ofstream& ofs, const CNodePtr& node) { + if (node == nullptr) { + return; + } + + // output type of each input argument + auto& inputs = node->inputs(); + if (inputs.size() > 1) { + ofs << " #("; + for (size_t i = 1; i < inputs.size(); ++i) { + if (i != 1) { + ofs << ", "; + } + AnfNodePtr arg = inputs[i]; + ofs << GetNodeType(arg); + } + ofs << ")"; + } + // output other comment, map the graph name to original representation(containing unicode character) + std::ostringstream comment; + comment << " #"; + bool has_comment = false; + for (size_t i = 0; i < inputs.size(); ++i) { + AnfNodePtr arg = inputs[i]; + if (!IsValueNode(arg)) { + continue; + } + if (!has_comment) { + has_comment = true; + } else { + comment << ","; + } + FuncGraphPtr fg = GetValueNode(arg); + std::string func_graph_id = fg->debug_info()->get_id(); + comment << " fg_" << func_graph_id << "=" << fg->ToString() << "." << func_graph_id; + } + if (has_comment) { + ofs << comment.str(); + } + ofs << " #scope: " << node->scope()->name(); +} + +void AnfExporter::OutputCNodes(std::ofstream& ofs, const std::vector& nodes, + const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return; + } + + int idx = 1; + std::map apply_map; + for (const AnfNodePtr& node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + + auto iter = tagged_cnodes_.find(node); + if (iter != tagged_cnodes_.end()) { + ofs << "\n#------------------------> " << iter->second << "\n"; + } + + auto cnode = node->cast(); + auto& inputs = cnode->inputs(); + std::string op_text = GetAnfNodeText(func_graph, inputs[0], apply_map); + // non-return node + if (node != func_graph->get_return()) { + int apply_idx = idx++; + apply_map[node] = apply_idx; + std::string type_info = GetNodeType(node); + if (type_info == "Undefined") { + ofs << " %" << apply_idx << " = " << op_text << "("; + } else { + ofs << " %" << apply_idx << " : " << type_info << " = " << op_text << "("; + } + } else { + ofs << " " << op_text << "("; + } + + for (size_t i = 1; i < inputs.size(); ++i) { + if (i != 1) { + ofs << ", "; + } + AnfNodePtr arg = inputs[i]; + ofs << GetAnfNodeText(func_graph, arg, apply_map); + } + ofs << ")"; + + // output comment + OutputStatementComment(ofs, cnode); + ofs << "\n"; + if (label_manage::GetGlobalTraceLabelType() == label_manage::TraceLabelType::kWithUniqueId) { + ofs << trace::GetDebugInfo(cnode->debug_info(), " # ", kSourceLineTipDiscard) << "#" + << label_manage::Label(cnode->debug_info()) << "\n"; + } else { + ofs << trace::GetDebugInfo(cnode->debug_info(), " # ", kSourceLineTipDiscard) << "\n"; + } + } +} + +void AnfExporter::ExportOneFuncGraph(std::ofstream& ofs, const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return; + } + + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + std::vector parameters = func_graph->parameters(); + OrderedMap param_map; + + ofs << "# [No." << (exported.size() + 1) << "] " << func_graph->DumpText() << "." + << func_graph->debug_info()->get_id() << "\n"; + if (label_manage::GetGlobalTraceLabelType() == label_manage::TraceLabelType::kWithUniqueId) { + ofs << trace::GetDebugInfo(func_graph->debug_info(), "# ", kSourceLineTipDiscard) << "#" + << label_manage::Label(func_graph->debug_info()) << "\n"; + } else { + ofs << trace::GetDebugInfo(func_graph->debug_info(), "# ", kSourceLineTipDiscard) << "\n"; + } + ofs << "funcgraph fg_" << func_graph->debug_info()->get_id(); + // output name of parent of graph if exists + if (func_graph->parent() != nullptr) { + ofs << "[fg_" << func_graph->parent()->debug_info()->get_id() << "]"; + } + ofs << "(\n"; + + OutputParameters(ofs, parameters, ¶m_map); + + exported[func_graph] = param_map; + ofs << (!parameters.empty() ? " " : "") << ") {\n"; + + OutputCNodes(ofs, nodes, func_graph); + + ofs << "}\n"; +} + +void AnfExporter::ExportFuncGraph(const std::string& filename, const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return; + } + + std::ofstream ofs(filename); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file '" << filename << "' failed!"; + return; + } + + param_index = 1; + + func_graph_set.add(func_graph); + while (!func_graph_set.empty()) { + FuncGraphPtr fg = *func_graph_set.begin(); + ExportOneFuncGraph(ofs, fg); + ofs << "\n\n"; + (void)func_graph_set.erase(fg); + } + ofs << "# num of total funcgraphs: " << exported.size(); + + ofs.close(); +} + +void AnfExporter::ExportFuncGraph(const std::string& filename, const std::vector& graphs) { + if (graphs.empty()) { + return; + } + + std::ofstream ofs(filename); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file '" << filename << "' failed!"; + return; + } + + param_index = 1; + + for (const auto& tagged_graph : graphs) { + tagged_cnodes_ = tagged_graph.second; + ExportOneFuncGraph(ofs, tagged_graph.first); + tagged_cnodes_.clear(); + ofs << "\n\n"; + } + + ofs << "# num of total funcgraphs: " << graphs.size(); + + ofs.close(); +} + +#ifdef ENABLE_DUMP_IR +void ExportIR(const std::string& filename, const std::string& id, const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return; + } + + AnfExporter exporter(id); + ChangeFileMode(filename, S_IRWXU); + exporter.ExportFuncGraph(filename, func_graph); + // set file mode to read only by user + ChangeFileMode(filename, S_IRUSR); +} + +void ExportIR(const std::string& filename, const std::vector& graphs) { + AnfExporter exporter("", false); + ChangeFileMode(filename, S_IRWXU); + exporter.ExportFuncGraph(filename, graphs); + // set file mode to read only by user + ChangeFileMode(filename, S_IRUSR); +} +#else +void ExportIR(const std::string&, const std::string&, const FuncGraphPtr&) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " + << "please recompile source to enable it. See help of building script."; +} + +void ExportIR(const std::string& filename, const std::vector& graphs) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR is disabled, " + << "please recompile source to enable it. See help of building script."; +} +#endif + +// ============================================= MindSpore IR Importer ============================================= + +enum Token : int { + TOK_INVALID = 0, // invalid token + TOK_LPARENTHESIS, // ( left parenthesis + TOK_RPARENTHESIS, // ) right parenthesis + TOK_LBRACKET, // [ left bracket + TOK_RBRACKET, // ] right bracket + TOK_LBRACE, // { left brace + TOK_RBRACE, // } right brace + TOK_COMMA, // , comma + TOK_EQUALITY, // = equality + TOK_COLON, // : colon + TOK_STAR, // * star + TOK_VARIABLE, // variable + TOK_AT_FILE, // @filename + TOK_PARAMETER, // parameter + TOK_IDENTIFIER, // identifier + TOK_FUNCGRAPH, // keyword 'funcgraph' + TOK_RETURN, // id prim::return + TOK_STRING, // string + TOK_NUMBER, // number + TOK_COMMENT, // comment + TOK_EOL, // end of line + TOK_EOF, // end of file + TOK_ERROR // file read error +}; + +std::map token_text = { + {TOK_INVALID, "invalid"}, // invalid token + {TOK_LPARENTHESIS, "("}, // ( left parenthesis + {TOK_RPARENTHESIS, ")"}, // ) right parenthesis + {TOK_LBRACKET, "["}, // [ left bracket + {TOK_RBRACKET, "]"}, // ] right bracket + {TOK_LBRACE, "{"}, // { left brace + {TOK_RBRACE, "}"}, // } right brace + {TOK_COMMA, ","}, // , comma + {TOK_EQUALITY, "="}, // = equality + {TOK_COLON, ":"}, // : colon + {TOK_STAR, "*"}, // * start + {TOK_VARIABLE, nullptr}, // variable + {TOK_AT_FILE, nullptr}, // @file + {TOK_PARAMETER, nullptr}, // parameter + {TOK_IDENTIFIER, nullptr}, // identifier + {TOK_FUNCGRAPH, "funcgraph"}, // keyword 'funcgraph' + {TOK_RETURN, nullptr}, // id prim::return + {TOK_STRING, nullptr}, // string + {TOK_NUMBER, nullptr}, // number + {TOK_COMMENT, nullptr}, // comment + {TOK_EOL, "\n"}, // end of line + {TOK_EOF, ""}, // end of file + {TOK_ERROR, "error"} // file read error +}; + +class Lexer { + public: + // filename is checked in ImportIR; + explicit Lexer(const char* filename) : fin(filename) {} + + ~Lexer() { + try { + if (fin.is_open()) { + fin.close(); + } + } catch (const std::exception& e) { + MS_LOG(ERROR) << "exception when closing file"; + } catch (...) { + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(ERROR) << "Error occurred when closing file. Exception name: " << exName; + } + } + + bool IsSingleCharToken(char ch, Token* token_ptr) { + // clang-format off + std::unordered_map char_to_token = { + {'(', TOK_LPARENTHESIS}, + {')', TOK_RPARENTHESIS}, + {'[', TOK_LBRACKET}, + {']', TOK_RBRACKET}, + {'{', TOK_LBRACE}, + {'}', TOK_RBRACE}, + {',', TOK_COMMA}, + {'=', TOK_EQUALITY}, + {':', TOK_COLON}, + {'*', TOK_STAR}}; + // clang-format on + + auto iter = char_to_token.find(ch); + if (iter == char_to_token.end()) { + return false; + } + + if (token_ptr != nullptr) { + *token_ptr = iter->second; + } + + return true; + } + + Token GetNextToken() { +#ifdef DEBUG + Token token = GetNextTokenInner(); + const char* str = token_text[token]; + std::string text = (str == nullptr ? GetTokenText() : str); + MS_LOG(DEBUG) << "------parse token] " << text; + return token; + } + + Token GetNextTokenInner() { +#endif + tok_idx = 0; + Token tok = TOK_ERROR; + char ch = SkipTabAndSpace(); + if (ch == CODE_EOF) { + return TOK_EOF; + } else if (ch == CODE_ERROR) { + return TOK_ERROR; + } else if (IsSingleCharToken(ch, &tok)) { + return tok; + } else if (ch == '\r') { + char c = GetChar(); + if (c == '\n') { + line_++; + return TOK_EOL; + } + UnGetChar(c); + line_++; + return TOK_EOL; + } else if (ch == '\n') { + line_++; + return TOK_EOL; + } else if (ch == '#') { + return ParseComment(ch); + } else if (ch == '"') { + return ParseString(); + } else if (ch == '%') { + return ParseVariableOrParameter(ch); + } else if (ch == '@') { + return ParseAtFile(); + } else if (IsDigit(ch) || ch == '-') { + return ParseNumber(ch); + } else if (IsAlpha(ch) || ch == '_') { + return ParseIdentifier(ch); + } else { + return TOK_ERROR; + } + } + + Token SkipWhiteToken() { + Token tok = GetNextToken(); + while (tok == TOK_EOL || tok == TOK_COMMENT) { + tok = GetNextToken(); + } + return tok; + } + + std::string GetTokenText() const { return std::string(tok_buf); } + + int GetLineNo() const { return line_; } + + private: + Token ParseComment(char ch) { + char c = GetChar(); + while (c != '\r' && c != '\n' && c != CODE_EOF) { + c = GetChar(); + } + if (ch != CODE_EOF) { + UnGetChar(c); + } + tok_buf[0] = '#'; + tok_buf[1] = '\0'; + return TOK_COMMENT; + } + + Token ParseString() { + tok_idx = 0; + char c = GetChar(); + while (c != '"') { + if (tok_idx >= BUF_SIZE) { + MS_LOG(EXCEPTION) << "Length of token which is " << tok_idx << " exceeds " << BUF_SIZE; + } + if (c == '\r' || c == '\n') { + MS_LOG(EXCEPTION) << "Literal newline characters are not allowed within the quote at line " << line_; + } + if (c == CODE_EOF) { + MS_LOG(EXCEPTION) << "Encounter EOF within the quote at line " << line_; + } + tok_buf[tok_idx++] = c; + c = GetChar(); + } + tok_buf[tok_idx] = '\0'; + return TOK_STRING; + } + + Token ParseVariableOrParameter(char ch) { + tok_idx = 0; + tok_buf[tok_idx++] = ch; + char c = GetChar(); + while (IsAlphaNumeric(c)) { + if (tok_idx >= BUF_SIZE) { + MS_LOG(EXCEPTION) << "Length of token which is " << tok_idx << " exceeds " << BUF_SIZE; + } + tok_buf[tok_idx++] = c; + c = GetChar(); + } + tok_buf[tok_idx] = '\0'; + UnGetChar(c); + + // judge parameter: %para[0-9]+ + tok_buf[tok_idx] = '\0'; + std::string param_key = "%para"; + if (strncmp(tok_buf, param_key.c_str(), param_key.size()) == 0) { + if (tok_idx <= param_key.size()) { + return TOK_ERROR; + } + for (auto i = static_cast(param_key.size()); i < tok_idx; ++i) { + if (!IsDigit(tok_buf[i])) { + return TOK_ERROR; + } + } + return TOK_PARAMETER; + } + + // judge local variable: %[0-9]+ + if (tok_idx == 1) { + return TOK_ERROR; + } + for (unsigned i = 1; i < tok_idx; ++i) { + if (!IsDigit(tok_buf[i])) { + return TOK_ERROR; + } + } + return TOK_VARIABLE; + } + + Token ParseAtFile() { + tok_idx = 0; + char c = GetChar(); + while (IsAlphaNumeric(c) || c == '_' || c == '.') { + if (tok_idx >= BUF_SIZE) { + MS_LOG(EXCEPTION) << "Length of token which is " << tok_idx << " exceeds " << BUF_SIZE; + } + tok_buf[tok_idx++] = c; + c = GetChar(); + } + tok_buf[tok_idx] = '\0'; + UnGetChar(c); + + if (tok_idx == 0) { + return TOK_ERROR; + } + + return TOK_AT_FILE; + } + + Token ParseNumber(char ch) { + tok_buf[tok_idx++] = ch; + char c = GetChar(); + // parse number, e.g. 10, 15.6, 1e-5 + while (IsDigit(c) || c == '.' || c == 'e' || c == '-') { + if (tok_idx >= BUF_SIZE) { + MS_LOG(EXCEPTION) << "Length of token which is " << tok_idx << " exceeds " << BUF_SIZE; + } + tok_buf[tok_idx++] = c; + c = GetChar(); + } + UnGetChar(c); + tok_buf[tok_idx] = '\0'; + return TOK_NUMBER; + } + + Token ParseIdentifier(char ch) { + tok_idx = 0; + tok_buf[tok_idx++] = ch; + char c = GetChar(); + while (IsAlphaNumeric(c) || c == '.' || c == ':' || c == '_') { + if (tok_idx >= BUF_SIZE) { + MS_LOG(EXCEPTION) << "Length of token which is " << tok_idx << " exceeds " << BUF_SIZE; + } + tok_buf[tok_idx++] = c; + c = GetChar(); + } + UnGetChar(c); + tok_buf[tok_idx] = '\0'; + + if (strcmp(tok_buf, "funcgraph") == 0) { + return TOK_FUNCGRAPH; + } + if (strcmp(tok_buf, "Primitive::return") == 0) { + return TOK_RETURN; + } + return TOK_IDENTIFIER; + } + + // Suppose the file only contain ASCII character + char GetChar() { + if (ungot_char != UNGOT_CHAR) { + char ch = ungot_char; + ungot_char = UNGOT_CHAR; + return ch; + } + if (idx >= cnt) { + if (fin.eof()) { + return CODE_EOF; + } + cnt = fin.read(buffer, BUF_SIZE).gcount(); + if ((fin.bad() || fin.fail()) && !fin.eof()) { + MS_LOG(EXCEPTION) << "Read file error!"; + } + idx = 0; + } + return buffer[idx++]; + } + + void UnGetChar(char ch) { + if (ungot_char == UNGOT_CHAR) { + ungot_char = ch; + } + } + + static bool IsTabOrSpace(char ch) { return ch == ' ' || ch == '\t'; } + + static bool IsDigit(char ch) { return ch >= '0' && ch <= '9'; } + + static bool IsAlpha(char ch) { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); } + + static bool IsAlphaNumeric(char ch) { return IsDigit(ch) || IsAlpha(ch); } + + // skip whitespace(including comment) to read a valid character + char SkipTabAndSpace() { + char ch = GetChar(); + while (IsTabOrSpace(ch)) { + ch = GetChar(); + } + return ch; + } + + std::ifstream fin; + + static const unsigned BUF_SIZE = 4096; // lexer buffer size + char buffer[BUF_SIZE + 1] = {0}; // buffer for holding text read from text + std::streamsize cnt = 0; // number of valid characters in the buffer + unsigned idx = 0; // index of next char the lexer to read from + + char tok_buf[BUF_SIZE + 1] = {0}; // token buffer + unsigned tok_idx = 0; // token buffer index + + char ungot_char = UNGOT_CHAR; // store ungot char + + static const int CODE_EOF = -1; // return code of GetChar + static const int CODE_ERROR = -2; // read file error + static const char UNGOT_CHAR = -3; // value of ungot char + + int line_ = 1; // current line number +}; + +const unsigned Lexer::BUF_SIZE; + +class IrParser { + public: + explicit IrParser(const char* filename) : lexer_(filename) {} + + ~IrParser() {} + + py::object LoadObject(const std::string& file_name) const { + std::string pkl_path = GetMsIrPath(); + py::object default_obj = load_obj(pkl_path + "/" + file_name); + return default_obj; + } + + void ParseFile() { + FuncGraphPtr func_graph = ParseFuncGraph(); + while (func_graph != nullptr) { + func_graphs_.push_back(func_graph); + func_graph = ParseFuncGraph(); + } + if (error_flag_) { + MS_LOG(EXCEPTION) << "Parse Error at line: " << lexer_.GetLineNo(); + } + + MS_LOG(INFO) << "Total graphs: " << func_graphs_.size(); + } + + Token ParseParent(FuncGraphPtr* const parent_ptr) { + if (lexer_.GetNextToken() != TOK_IDENTIFIER) { + return TOK_ERROR; + } + + std::string parent_name = lexer_.GetTokenText(); + // NOTICE: require definition of parent graph must before child graph + auto iter = func_graphs_map_.find(parent_name); + if (iter == func_graphs_map_.end()) { + MS_LOG(EXCEPTION) << "Can not find definition of parent func graph '" << parent_name << "' at line " + << lexer_.GetLineNo(); + } + if (parent_ptr != nullptr) { + *parent_ptr = iter->second; + } + + if (lexer_.GetNextToken() != TOK_RBRACKET) { + return TOK_ERROR; + } + + return lexer_.GetNextToken(); + } + + FuncGraphPtr ParseFuncGraph() { + cnodes_.clear(); + + Token tok = lexer_.SkipWhiteToken(); + if (tok != TOK_FUNCGRAPH) { + error_flag_ = tok != TOK_EOF; + return nullptr; + } + + if (lexer_.GetNextToken() != TOK_IDENTIFIER) { + error_flag_ = true; + return nullptr; + } + + std::string func_graph_name = lexer_.GetTokenText(); + if (func_graphs_map_.find(func_graph_name) == func_graphs_map_.end()) { + func_graphs_map_[func_graph_name] = std::make_shared(); + } + FuncGraphPtr func_graph = func_graphs_map_[func_graph_name]; + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(func_graph->debug_info()); + func_graph->debug_info()->set_name(func_graph_name); // for debugging + + FuncGraphPtr parent = nullptr; + tok = lexer_.GetNextToken(); + if (tok == TOK_LBRACKET) { + tok = ParseParent(&parent); + if (parent != nullptr) { + parents_map_[func_graph] = parent; + } + } + + if (tok != TOK_LPARENTHESIS) { + error_flag_ = true; + return nullptr; + } + + if (ParseParameters(func_graph) == nullptr) { + error_flag_ = true; + return nullptr; + } + + if (lexer_.SkipWhiteToken() != TOK_LBRACE) { + error_flag_ = true; + return nullptr; + } + + // parse statements + if (ParseStatements(func_graph) == nullptr) { + error_flag_ = true; + return nullptr; + } + + func_graphs_map_[func_graph_name] = func_graph; + + return func_graph; + } + + FuncGraphPtr ParseStatements(const FuncGraphPtr& func_graph) { + Token tok = lexer_.SkipWhiteToken(); + while (tok == TOK_VARIABLE) { + if (ParseStatement(func_graph) == nullptr) { + return nullptr; + } + tok = lexer_.SkipWhiteToken(); + } + if (tok == TOK_RETURN) { + return ParseReturn(func_graph); + } + return nullptr; + } + + FuncGraphPtr ParseStatement(FuncGraphPtr func_graph) { + std::string var_name = lexer_.GetTokenText(); + Token tok = lexer_.GetNextToken(); + AbstractBasePtr type = nullptr; + if (tok == TOK_COLON) { + tok = ParseType(func_graph, &type); + } + if (tok != TOK_EQUALITY) { + return nullptr; + } + + std::vector inputs; + AnfNodePtr node = nullptr; + ValuePtr val = nullptr; + tok = ParseItem(func_graph, &node, &val); + if (tok != TOK_LPARENTHESIS) { + return nullptr; + } + inputs.push_back(node); + + int lineno = lexer_.GetLineNo(); + + if (ParseArguments(func_graph, &inputs) == nullptr) { + return nullptr; + } + + tok = lexer_.GetNextToken(); + if (tok == TOK_COMMENT) { + tok = lexer_.GetNextToken(); + } + if (tok != TOK_EOL) { + return nullptr; + } + + MS_EXCEPTION_IF_NULL(func_graph); + cnodes_[var_name] = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(cnodes_[var_name]); + cnodes_[var_name]->set_debug_info(std::make_shared(var_name + "@" + std::to_string(lineno))); + return func_graph; + } + + FuncGraphPtr ParseReturn(FuncGraphPtr func_graph) { + if (lexer_.GetNextToken() != TOK_LPARENTHESIS) { + return nullptr; + } + + AnfNodePtr input1 = nullptr; + ValuePtr value = nullptr; + Token tok = ParseItem(func_graph, &input1, &value, lexer_.GetNextToken()); + int lineno = lexer_.GetLineNo(); + + if (tok != TOK_RPARENTHESIS) { + return nullptr; + } + + tok = lexer_.GetNextToken(); + if (tok == TOK_COMMENT) { + tok = lexer_.GetNextToken(); + } + if (tok != TOK_EOL) { + return nullptr; + } + + if (lexer_.SkipWhiteToken() != TOK_RBRACE) { + return nullptr; + } + + PrimitivePtr prim = std::make_shared("return"); + ValueNodePtr input0 = std::make_shared(prim); + std::vector inputs; + inputs.push_back(input0); + inputs.push_back(input1); + MS_EXCEPTION_IF_NULL(func_graph); + CNodePtr ret = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(ret); + ret->set_debug_info(std::make_shared(std::string("ret@") + std::to_string(lineno))); + + func_graph->set_return(ret); + + return func_graph; + } + + void SetBasicType(TypePtr* ptr, const TypePtr& dtype) const { + if (ptr == nullptr) { + return; + } + *ptr = dtype; + } + + void SetTupleType(TypePtr* ptr) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(); + } + + void SetTupleType(TypePtr* ptr, const TypePtrList& elems) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(elems); + } + + void SetArrayType(TypePtr* const ptr, const TypePtr& elem_type, const std::vector&) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(elem_type); + } + + void SetListType(TypePtr* ptr) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(); + } + + void SetListType(TypePtr* ptr, const TypePtrList& elems) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(elems); + } + + void SetJTaggedType(TypePtr* ptr, const TypePtr& elem) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(elem); + } + + void SetBasicType(AbstractBasePtr* ptr, const TypePtr& dtype) const { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(dtype); + } + + // void SetBasicType(AbstractBasePtr *ptr, const SymbolicKeyTypePtr& dtype) {} + void SetBasicType(AbstractBasePtr* const ptr, const TypeNonePtr&) const { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(); + } + + void SetBasicType(AbstractBasePtr*, const FunctionPtr&) const {} + void SetBasicType(AbstractBasePtr*, const TensorTypePtr&) const {} + + void SetTupleType(AbstractBasePtr* const ptr, const AbstractBasePtrList& elems) { + if (ptr == nullptr) { + return; + } + // if one of elems is nullptr, just return + if (std::any_of(std::begin(elems), std::end(elems), [](const AbstractBasePtr& elem) { return elem == nullptr; })) { + return; + } + *ptr = std::make_shared(elems); + } + + void SetArrayType(AbstractBasePtr* const ptr, const TypePtr& elem_type, const std::vector& shape) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(elem_type, shape); + } + + void SetListType(AbstractBasePtr* const ptr, const AbstractBasePtrList& elems) { + if (ptr == nullptr) { + return; + } + if (std::any_of(std::begin(elems), std::end(elems), [](const AbstractBasePtr& elem) { return elem == nullptr; })) { + return; + } + *ptr = std::make_shared(elems); + } + + void SetJTaggedType(AbstractBasePtr* const ptr, const AbstractBasePtr& elem) { + if (ptr == nullptr) { + return; + } + *ptr = std::make_shared(elem); + } + + template + Token ParseTypeVector(const FuncGraphPtr& func_graph, Token tok, const std::string& type, T* const ptr = nullptr) { + if (tok != TOK_LBRACKET) { + MS_LOG(EXCEPTION) << "Illegal case, , wrong token start symbol."; + return tok; + } + + bool first_flag = true; + std::vector elems; + do { + tok = lexer_.GetNextToken(); + if (first_flag) { + if (tok == TOK_RBRACKET) { + return lexer_.GetNextToken(); + } + first_flag = false; + } + T elem = nullptr; + tok = ParseOneType(func_graph, tok, &elem); + elems.push_back(elem); + if (tok == TOK_STAR) { + if (lexer_.GetNextToken() != TOK_NUMBER) { + return TOK_ERROR; + } + int num_elems = StringToScalar(lexer_.GetTokenText()); + if (num_elems < 1 || num_elems > NUM_MAX_SEQUENCE_ELEMS) { + MS_LOG(EXCEPTION) << "Number of elements " << num_elems << " is out of range [1, " << NUM_MAX_SEQUENCE_ELEMS + << "]"; + } + for (int i = 0; i < num_elems - 1; ++i) { + elems.push_back(elem); + } + tok = lexer_.GetNextToken(); + } + } while (tok == TOK_COMMA); + if (tok != TOK_RBRACKET) { + return TOK_ERROR; + } + if (type == "Tuple") { + SetTupleType(ptr, elems); + } else if (type == "List") { + SetListType(ptr, elems); + } else { + MS_LOG(EXCEPTION) << "This method does not support " << type << " parse."; + } + return lexer_.GetNextToken(); + } + + template + Token ParseTypeArray(const FuncGraphPtr& func_graph, Token tok, T* const ptr = nullptr) { + if (tok != TOK_LPARENTHESIS) { + if (ptr != nullptr) { + SetBasicType(ptr, std::make_shared()); + } + return tok; + } + // process Array element type + TypePtr elem_type = nullptr; + std::vector shape; + tok = ParseOneType(func_graph, lexer_.GetNextToken(), &elem_type); + if (tok != TOK_RPARENTHESIS) { + return TOK_ERROR; + } + tok = lexer_.GetNextToken(); + if (tok != TOK_LBRACKET) { + // NOTICE: if shape.size == 0, is this ok? + SetArrayType(ptr, elem_type, shape); + return tok; + } + // process Array shape + do { + tok = lexer_.GetNextToken(); + // case: Array(I32)[] + if (tok != TOK_NUMBER) { + break; + } + shape.push_back(StringToScalar(lexer_.GetTokenText())); + tok = lexer_.GetNextToken(); + } while (tok == TOK_COMMA); + if (tok != TOK_RBRACKET) { + return TOK_ERROR; + } + + SetArrayType(ptr, elem_type, shape); + + return lexer_.GetNextToken(); + } + + bool IsNumberType(const std::string& type, TypeId* typeid_ptr) { + // clang-format off + static std::unordered_map basic_types = { + {"Bool", kNumberTypeBool}, + {"I8", kNumberTypeInt8}, + {"I16", kNumberTypeInt16}, + {"I32", kNumberTypeInt32}, + {"I64", kNumberTypeInt64}, + {"U8", kNumberTypeUInt8}, + {"U16", kNumberTypeUInt16}, + {"U32", kNumberTypeUInt32}, + {"U64", kNumberTypeUInt64}, + {"F16", kNumberTypeFloat16}, + {"F32", kNumberTypeFloat32}, + {"F64", kNumberTypeFloat64}, + {"Int", kNumberTypeInt}, + {"UInt", kNumberTypeUInt}, + {"Float", kNumberTypeFloat}, + {"Number", kObjectTypeNumber}}; + // clang-format on + + auto iter = basic_types.find(type); + if (iter == basic_types.end()) { + return false; + } + if (typeid_ptr != nullptr) { + *typeid_ptr = iter->second; + } + return true; + } + + template + void ParseNumberType(const std::string& type, TypeId typeId, T* const ptr = nullptr) { + TypePtr dtype = nullptr; + + std::unordered_map type_map = { + {static_cast(kNumberTypeBool), std::make_shared()}, // Bool + {static_cast(kNumberTypeInt8), std::make_shared(8)}, // Int8 + {static_cast(kNumberTypeInt16), std::make_shared(16)}, // Int16 + {static_cast(kNumberTypeInt32), std::make_shared(32)}, // Int32 + {static_cast(kNumberTypeInt64), std::make_shared(64)}, // Int64 + {static_cast(kNumberTypeUInt8), std::make_shared(8)}, // UInt8 + {static_cast(kNumberTypeUInt16), std::make_shared(16)}, // UInt16 + {static_cast(kNumberTypeUInt32), std::make_shared(32)}, // UInt32 + {static_cast(kNumberTypeUInt64), std::make_shared(64)}, // UInt64 + {static_cast(kNumberTypeFloat16), std::make_shared(16)}, // Float16 + {static_cast(kNumberTypeFloat32), std::make_shared(32)}, // Float32 + {static_cast(kNumberTypeFloat64), std::make_shared(64)}, // Float64 + {static_cast(kNumberTypeInt), std::make_shared()}, // Int + {static_cast(kNumberTypeUInt), std::make_shared()}, // UInt + {static_cast(kNumberTypeFloat), std::make_shared()}, // Float + {static_cast(kObjectTypeNumber), std::make_shared()}, // Number + }; + + auto iter = type_map.find(static_cast(typeId)); + if (iter != type_map.end()) { + dtype = iter->second; + } else { + MS_LOG(EXCEPTION) << "Unknown number type " << type; + } + + SetBasicType(ptr, dtype); + } + + template + Token ParseTrivalType(const std::string& type, T* const ptr = nullptr) { + if (type == "NoneType") { + SetBasicType(ptr, std::make_shared()); + return lexer_.GetNextToken(); + } else if (type == "ProblemType") { + SetBasicType(ptr, std::make_shared()); + return lexer_.GetNextToken(); + } else if (type == "ExternalType") { + SetBasicType(ptr, std::make_shared()); + return lexer_.GetNextToken(); + } else if (type == "AnythingType") { + SetBasicType(ptr, kAnyType); + return lexer_.GetNextToken(); + } else if (type == "TypeType") { + SetBasicType(ptr, std::make_shared()); + return lexer_.GetNextToken(); + } else { + MS_LOG(EXCEPTION) << "Unknown type error at line " << lexer_.GetLineNo(); + } + } + + template + Token ParseOneType(const FuncGraphPtr& func_graph, Token tok, T* const ptr = nullptr) { + if (tok != TOK_IDENTIFIER) { + return TOK_ERROR; + } + std::string type = lexer_.GetTokenText(); + TypeId typeId = kTypeUnknown; + if (IsNumberType(type, &typeId)) { + ParseNumberType(type, typeId, ptr); + return lexer_.GetNextToken(); + } else if (type == "Tuple") { + return ParseTypeVector(func_graph, lexer_.GetNextToken(), type, ptr); + } else if (type == "Array") { + return ParseTypeArray(func_graph, lexer_.GetNextToken(), ptr); + } else if (type == "List") { + return ParseTypeVector(func_graph, lexer_.GetNextToken(), type, ptr); + } else if (type == "Func") { + tok = lexer_.GetNextToken(); + if (tok != TOK_LBRACKET) { + SetBasicType(ptr, std::make_shared()); + return tok; + } + MS_LOG(EXCEPTION) << "Need to process function parameter types at line " << lexer_.GetLineNo(); + } else if (type == "JT") { + tok = lexer_.GetNextToken(); + if (tok != TOK_LBRACKET) { + return tok; + } + T elem = nullptr; + tok = ParseOneType(func_graph, lexer_.GetNextToken(), &elem); + SetJTaggedType(ptr, elem); + if (tok != TOK_RBRACKET) { + return TOK_ERROR; + } + return lexer_.GetNextToken(); + } else if (type == "SymType") { + SetBasicType(ptr, std::make_shared()); + return lexer_.GetNextToken(); + } else if (type == "EnvType") { + SetBasicType(ptr, std::make_shared()); + return lexer_.GetNextToken(); + } else if (Match(type, "Cls.")) { + MS_LOG(EXCEPTION) << "Need to do class type at line " << lexer_.GetLineNo(); + } else { + return ParseTrivalType(type, ptr); + } + } + + Token ParseType(const FuncGraphPtr& func_graph, AbstractBasePtr* const abstract = nullptr) { + return ParseOneType(func_graph, lexer_.GetNextToken(), abstract); + } + + Token ParseAttributes(const FuncGraphPtr& func_graph, const PrimitivePtr& prim) { + Token tok = ParseAttribute(func_graph, prim); + while (tok == TOK_COMMA) { + tok = ParseAttribute(func_graph, prim); + } + if (tok != TOK_RBRACKET) { + return TOK_ERROR; + } + return lexer_.GetNextToken(); + } + + Token ParseAttribute(const FuncGraphPtr& func_graph, const PrimitivePtr& prim) { + Token tok = lexer_.GetNextToken(); + if (tok != TOK_IDENTIFIER) { + return TOK_ERROR; + } + std::string attr_name = lexer_.GetTokenText(); + + if (lexer_.GetNextToken() != TOK_EQUALITY) { + return TOK_ERROR; + } + + ValuePtr value = nullptr; + tok = ParseValue(func_graph, lexer_.GetNextToken(), &value); + + if (prim != nullptr) { + prim->set_attr(attr_name, value); + } else { + MS_LOG(EXCEPTION) << "Non primitive obj has attributes"; + } + + return tok; + } + + FuncGraphPtr ParseParameters(FuncGraphPtr func_graph) { + Token tok = lexer_.SkipWhiteToken(); + while (tok == TOK_PARAMETER) { + ParameterPtr param = std::make_shared(func_graph); + param->set_name(lexer_.GetTokenText()); + param_nodes_[lexer_.GetTokenText()] = param; + int lineno = lexer_.GetLineNo(); + param->set_debug_info(std::make_shared(lexer_.GetTokenText() + "@" + std::to_string(lineno))); + func_graph->add_parameter(param); + + tok = lexer_.GetNextToken(); + // parse type + if (tok == TOK_COLON) { + AbstractBasePtr type = nullptr; + tok = ParseType(func_graph, &type); + } + // parse default value + if (tok == TOK_EQUALITY) { + if (lexer_.GetNextToken() != TOK_AT_FILE) { + MS_LOG(EXCEPTION) << "Expect @file at line " << lexer_.GetLineNo(); + } + + // load prameter default value from serialized file + py::object default_obj = LoadObject(lexer_.GetTokenText()); + param->set_default_param(default_obj); + + tok = lexer_.GetNextToken(); + } + if (tok == TOK_COMMENT || tok == TOK_EOL) { + tok = lexer_.SkipWhiteToken(); + } + + Token next = tok; + if (next == TOK_RPARENTHESIS) { + return func_graph; + } else if (next == TOK_COMMA) { + tok = lexer_.SkipWhiteToken(); + } else { + return nullptr; + } + } + return tok == TOK_RPARENTHESIS ? func_graph : nullptr; + } + + FuncGraphPtr ParseArguments(FuncGraphPtr func_graph, std::vector* const inputs_ptr) { + Token tok = ParseArgument(func_graph, inputs_ptr); + while (tok == TOK_COMMA) { + tok = ParseArgument(func_graph, inputs_ptr); + } + if (tok != TOK_RPARENTHESIS) { + return nullptr; + } + return func_graph; + } + + AnfNodePtr FindParameter(FuncGraphPtr func_graph, const std::string& param_name) { + while (func_graph != nullptr) { + for (auto& ptr : func_graph->parameters()) { + MS_EXCEPTION_IF_NULL(ptr); + ParameterPtr param = ptr->cast(); + MS_EXCEPTION_IF_NULL(param); + if (param->name() == param_name) { + return ptr; + } + } + auto iter = parents_map_.find(func_graph); + if (iter == parents_map_.end()) { + break; + } + func_graph = iter->second; + } + + return nullptr; + } + + bool Match(const std::string& str, const std::string& pattern) const { + return strncmp(str.c_str(), pattern.c_str(), pattern.length()) == 0; + } + + template + Token ParseScalar(ValuePtr* const val_ptr) { + if (lexer_.GetNextToken() != TOK_NUMBER) { + return TOK_ERROR; + } + std::stringstream ss; + ss << lexer_.GetTokenText(); + + if (lexer_.GetNextToken() != TOK_RPARENTHESIS) { + return TOK_ERROR; + } + + V val; + ss >> val; + *val_ptr = std::make_shared(val); + + return lexer_.GetNextToken(); + } + + template + Token ParseScalar(ValuePtr* const val_ptr, Token tok) { + if (tok != TOK_LPARENTHESIS) { + *val_ptr = std::make_shared(); + return tok; + } + + return ParseScalar(val_ptr); + } + + template + Token ParseScalar(ValuePtr* const val_ptr, Token tok) { + if (tok != TOK_LPARENTHESIS) { + *val_ptr = std::make_shared(nbits); + return tok; + } + + return ParseScalar(val_ptr); + } + + template + T StringToScalar(const std::string& text) { + std::stringstream ss; + T value; + ss << text; + ss >> value; + return value; + } + + Token ParseTensor(ValuePtr* const val_ptr) { + // parse type + TypeId type; + if (lexer_.GetNextToken() != TOK_LPARENTHESIS) { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_NUMBER) { + return TOK_ERROR; + } + type = static_cast(StringToScalar(lexer_.GetTokenText())); + if (lexer_.GetNextToken() != TOK_RPARENTHESIS) { + return TOK_ERROR; + } + + // parse shape + std::vector shape; + Token tok = lexer_.GetNextToken(); + if (tok != TOK_LBRACKET) { + return TOK_ERROR; + } + + do { + tok = lexer_.GetNextToken(); + // consider case: Tensor(23)[] + if (tok != TOK_NUMBER) { + break; + } + shape.push_back(StringToScalar(lexer_.GetTokenText())); + + tok = lexer_.GetNextToken(); + } while (tok == TOK_COMMA); + + if (tok != TOK_RBRACKET) { + return TOK_ERROR; + } + + if (lexer_.GetNextToken() != TOK_AT_FILE) { + return TOK_ERROR; + } + + py::object tensor_obj = LoadObject(lexer_.GetTokenText()); + py::array tensor_data = py::cast(tensor_obj); + if (tensor_data == nullptr) { + return TOK_ERROR; + } + *val_ptr = std::make_shared(tensor_data, TypeIdToType(type)); + + return lexer_.GetNextToken(); + } + + Token ParsePrimType(Token tok, PrimType* prim_type_ptr) { + if (tok != TOK_LBRACE) { + return tok; + } + if (lexer_.GetNextToken() != TOK_IDENTIFIER) { + return TOK_ERROR; + } + if (lexer_.GetTokenText() != "prim_type") { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_EQUALITY) { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_NUMBER) { + return TOK_ERROR; + } + int val = 0; + std::stringstream ss; + ss << lexer_.GetTokenText(); + ss >> val; + *prim_type_ptr = PrimType(val); + if (lexer_.GetNextToken() != TOK_RBRACE) { + return TOK_ERROR; + } + return lexer_.GetNextToken(); + } + + Token ParseMultitypeFuncGraphItem(const prim::MultitypeFuncGraphPtr& mt_func_graph, Token tok) { + if (tok != TOK_LPARENTHESIS) { + return TOK_ERROR; + } + TypePtrList type_list; + do { + TypePtr type = nullptr; + tok = ParseOneType(nullptr, lexer_.GetNextToken(), &type); + type_list.push_back(type); + } while (tok == TOK_COMMA); + if (tok != TOK_RPARENTHESIS) { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_AT_FILE) { + return TOK_ERROR; + } + + // load Python function from serialized file + py::object py_func = LoadObject(lexer_.GetTokenText()); + MS_EXCEPTION_IF_NULL(mt_func_graph); + mt_func_graph->Register(type_list, py::function(py_func)); + + return lexer_.GetNextToken(); + } + + Token ParseMultitypeFuncGraph(const prim::MultitypeFuncGraphPtr& mt_func_graph, Token tok) { + if (tok != TOK_LBRACE) { + return tok; + } + do { + tok = ParseMultitypeFuncGraphItem(mt_func_graph, lexer_.GetNextToken()); + } while (tok == TOK_COMMA); + if (tok != TOK_RBRACE) { + return TOK_ERROR; + } + return lexer_.GetNextToken(); + } + + Token ParseBoolValue(const std::string& key, bool* val_ptr) { + if (lexer_.GetNextToken() != TOK_IDENTIFIER || lexer_.GetTokenText() != key) { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_EQUALITY) { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_NUMBER) { + return TOK_ERROR; + } + bool value = false; + { + std::stringstream ss; + ss << lexer_.GetTokenText(); + ss >> value; + } + + if (val_ptr != nullptr) { + *val_ptr = value; + } + + return lexer_.GetNextToken(); + } + + Token ParseValueGradOperation(const std::string& name, ValuePtr* const val_ptr) { + if (lexer_.GetNextToken() != TOK_LBRACE) { + return TOK_ERROR; + } + // get_all=0, get_by_list=1, sens_param=1 + bool get_all = false; + Token tok = ParseBoolValue("get_all", &get_all); + if (tok != TOK_COMMA) { + return TOK_ERROR; + } + + bool get_by_list = false; + tok = ParseBoolValue("get_by_list", &get_by_list); + if (tok != TOK_COMMA) { + return TOK_ERROR; + } + + bool sens_param = false; + tok = ParseBoolValue("sens_param", &sens_param); + if (tok != TOK_RBRACE) { + return TOK_ERROR; + } + + *val_ptr = std::make_shared(name, get_all, get_by_list, sens_param); + + return lexer_.GetNextToken(); + } + + Token ParseSymbolicKeyInstance(const FuncGraphPtr& func_graph, AnfNodePtr* const node_ptr = nullptr) { + if (lexer_.GetNextToken() != TOK_LPARENTHESIS) { + return TOK_ERROR; + } + if (lexer_.GetNextToken() != TOK_PARAMETER) { + return TOK_ERROR; + } + std::string param_name = lexer_.GetTokenText(); + if (lexer_.GetNextToken() != TOK_RPARENTHESIS) { + return TOK_ERROR; + } + auto iter = param_nodes_.find(param_name); + if (iter == param_nodes_.end()) { + MS_LOG(EXCEPTION) << "Can not find param '" << param_name << "' for SymbolicKeyInstance at line " + << lexer_.GetLineNo(); + } + + PrimitivePtr embed = std::make_shared("embed"); + std::vector inputs; + inputs.push_back(std::make_shared(embed)); + inputs.push_back(iter->second); + if (node_ptr != nullptr) { + MS_EXCEPTION_IF_NULL(func_graph); + *node_ptr = func_graph->NewCNode(inputs); + } else { + MS_LOG(EXCEPTION) << "Not processed SymbolicKeyInstance '" << param_name << "' at line " << lexer_.GetLineNo() + << "."; + } + return lexer_.GetNextToken(); + } + + Token ParsePrimitivePy(const FuncGraphPtr& func_graph, const std::string& id, ValuePtr* const val_ptr) { + if (lexer_.GetNextToken() != TOK_AT_FILE) { + return TOK_ERROR; + } + + // restore python funciton of PrimitivePy from serialized file + py::object py_obj = LoadObject(lexer_.GetTokenText()); + PrimitivePyPtr ptr = nullptr; + if (py::hasattr(py_obj, "__setattr_flag__") && py::hasattr(py_obj, "_clone")) { + auto clone_fn = py_obj.attr("_clone"); + py::object new_obj = clone_fn(); + ptr = new_obj.cast(); + if (ptr == nullptr) { + MS_LOG(EXCEPTION) << "cast to type 'PrimitivePyPtr' error"; + } + } else { + ptr = std::make_shared(id.substr(strlen("PrimitivePy::")), py_obj); + } + *val_ptr = ptr; + + PrimType prim_type = kPrimTypeUnknown; + Token next = ParsePrimType(lexer_.GetNextToken(), &prim_type); + if (prim_type != kPrimTypeUnknown) { + ptr->set_prim_type(prim_type); + } + if (next != TOK_LBRACKET) { + return next; + } + // parse attributes + next = ParseAttributes(func_graph, ptr); + return next; + } + + Token ParseValueGraphAndNamespace(const std::string& id, ValuePtr* val_ptr) { + if (Match(id, "MultitypeFuncGraph::")) { + std::string name = id.substr(strlen("MultitypeFuncGraph::")); + auto mt_func_graph = std::make_shared(name); + *val_ptr = mt_func_graph; + Token next = ParseMultitypeFuncGraph(mt_func_graph, lexer_.GetNextToken()); + return next; + } else if (Match(id, "HyperMapPy::")) { + *val_ptr = std::make_shared(); + Token next = lexer_.GetNextToken(); + // process case: fn_leaf is not null + if (next == TOK_LBRACE) { + MS_LOG(EXCEPTION) << "Need to process fn_leaf at line " << lexer_.GetLineNo(); + } + return next; + } else if (Match(id, "FuncGraph::")) { + std::string func_graph_name = id.substr(strlen("FuncGraph::")); + // if the graph does not exist, create a null graph, then fill the graph when encounter the definition + // of the graph + if (func_graphs_map_.find(func_graph_name) == func_graphs_map_.end()) { + func_graphs_map_[func_graph_name] = std::make_shared(); + } + *val_ptr = func_graphs_map_[func_graph_name]; + return lexer_.GetNextToken(); + } else if (Match(id, "NameSpace::")) { + std::string module_name = id.substr(strlen("NameSpace::")); + if (lexer_.GetNextToken() != TOK_AT_FILE) { + MS_LOG(ERROR) << "Expect TOK_AT_FILE at line " << lexer_.GetLineNo(); + return TOK_ERROR; + } + // load Python module information from serialized file + py::object py_obj = LoadObject(lexer_.GetTokenText()); + *val_ptr = std::make_shared(module_name, py_obj); + + return lexer_.GetNextToken(); + } else { + MS_LOG(EXCEPTION) << "Unknown id " << id << " at line " << lexer_.GetLineNo(); + } + } + + Token ParseValueBasic(const FuncGraphPtr& func_graph, const std::string& id, ValuePtr* val_ptr, + AnfNodePtr* const node_ptr = nullptr) { + if (id == "None") { + *val_ptr = std::make_shared(); + return lexer_.GetNextToken(); + } else if (id == "Bool") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "I8") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "I16") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "I32") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "I64") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "U8") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "U16") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "U32") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "U64") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "F16") { + // Notice: Since there is no basic data type for storing fp16, just use float instead + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "F32") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "F64") { + return ParseScalar(val_ptr, lexer_.GetNextToken()); + } else if (id == "Tensor") { + return ParseTensor(val_ptr); + } else if (id == "SymInst") { + return ParseSymbolicKeyInstance(func_graph, node_ptr); + } else if (id == "Array") { + TypePtr type = nullptr; + Token ret = ParseTypeArray(func_graph, lexer_.GetNextToken(), &type); + *val_ptr = type; + return ret; + } else if (Match(id, "PrimitivePy::")) { + return ParsePrimitivePy(func_graph, id, val_ptr); + } else if (Match(id, "Primitive::")) { + *val_ptr = std::make_shared(id.substr(strlen("Primitive::"))); + return lexer_.GetNextToken(); + } else if (Match(id, "GradOperation::")) { + return ParseValueGradOperation(id.substr(strlen("GradOperation::")), val_ptr); + } else { + return ParseValueGraphAndNamespace(id, val_ptr); + } + } + + Token SetListOrTupleValue(const FuncGraphPtr& func_graph, Token left_tok, Token next, bool node_is_valid, + const std::vector& elems, const std::vector& nodes, + ValuePtr* const val_ptr, AnfNodePtr* node_ptr) { + if (left_tok == TOK_LPARENTHESIS && next == TOK_RPARENTHESIS) { + if (node_is_valid && node_ptr != nullptr) { + MS_EXCEPTION_IF_NULL(func_graph); + *node_ptr = func_graph->NewCNode(nodes); + } else { + *val_ptr = std::make_shared(elems); + } + return lexer_.GetNextToken(); + } else if (left_tok == TOK_LBRACKET && next == TOK_RBRACKET) { + if (node_is_valid && node_ptr != nullptr) { + MS_LOG(EXCEPTION) << "Encounter valid node in value list"; + } + *val_ptr = std::make_shared(elems); + return lexer_.GetNextToken(); + } else { + return TOK_ERROR; + } + } + + Token ParseListOrTupleValue(const FuncGraphPtr& func_graph, Token tok, ValuePtr* const val_ptr, + AnfNodePtr* node_ptr = nullptr) { + Token left_tok = tok; + + std::vector elems; + std::vector nodes; + nodes.push_back(std::make_shared(std::make_shared("make_tuple"))); + ValuePtr elem = nullptr; + AnfNodePtr node = nullptr; + bool node_is_valid = false; + bool first_flag = true; + Token next = TOK_ERROR; + do { + next = lexer_.GetNextToken(); + if (first_flag) { + first_flag = false; + // case (), zero elements + if ((left_tok == TOK_LPARENTHESIS && next == TOK_RPARENTHESIS) || + (left_tok == TOK_LBRACKET && next == TOK_RBRACKET)) { + if (left_tok == TOK_LPARENTHESIS) { + *val_ptr = std::make_shared(elems); + } else { + *val_ptr = std::make_shared(elems); + } + return lexer_.GetNextToken(); + } + } + node = nullptr; + next = ParseValue(func_graph, next, &elem, &node); + elems.push_back(elem); + if (node != nullptr) { + nodes.push_back(node); + node_is_valid = true; + } else { + nodes.push_back(std::make_shared(elem)); + } + } while (next == TOK_COMMA); + + return SetListOrTupleValue(func_graph, left_tok, next, node_is_valid, elems, nodes, val_ptr, node_ptr); + } + + Token ParseValue(const FuncGraphPtr& func_graph, Token tok, ValuePtr* const val_ptr, AnfNodePtr* node_ptr = nullptr) { + // tuple or list + if (tok == TOK_LPARENTHESIS || tok == TOK_LBRACKET) { + return ParseListOrTupleValue(func_graph, tok, val_ptr, node_ptr); + } else if (tok == TOK_IDENTIFIER) { + return ParseValueBasic(func_graph, lexer_.GetTokenText(), val_ptr, node_ptr); + } else if (tok == TOK_STRING) { + *val_ptr = std::make_shared(lexer_.GetTokenText()); + return lexer_.GetNextToken(); + } + MS_LOG(ERROR) << "Parse error!"; + return TOK_ERROR; + } + + Token ParseItem(const FuncGraphPtr& func_graph, AnfNodePtr* node_ptr, ValuePtr* const val_ptr, + Token tok = TOK_INVALID) { + if (tok == TOK_INVALID) { + tok = lexer_.GetNextToken(); + } + if (tok == TOK_VARIABLE) { + auto iter = cnodes_.find(lexer_.GetTokenText()); + if (iter == cnodes_.end()) { + MS_LOG(EXCEPTION) << "Can not find definition of '" << lexer_.GetTokenText() << "'"; + } + *node_ptr = iter->second; + } else if (tok == TOK_PARAMETER) { + AnfNodePtr param = FindParameter(func_graph, lexer_.GetTokenText()); + if (param == nullptr) { + MS_LOG(EXCEPTION) << "Can not find definition of '" << lexer_.GetTokenText() << "' at line " + << lexer_.GetLineNo(); + } + *node_ptr = param; + } else if (tok == TOK_IDENTIFIER || tok == TOK_LPARENTHESIS || tok == TOK_STRING) { + ValuePtr value; + AnfNodePtr node; + tok = ParseValue(func_graph, tok, &value, &node); + if (tok == TOK_ERROR) { + MS_LOG(ERROR) << "Parse value error!"; + return tok; + } + if (node == nullptr) { + *val_ptr = value; + *node_ptr = std::make_shared(value); + } else { + *node_ptr = node; + } + + return tok; + } else { + MS_LOG(EXCEPTION) << "tok_type = " << tok; + } + + return lexer_.GetNextToken(); + } + + Token ParseArgument(const FuncGraphPtr& func_graph, std::vector* const inputs_ptr) { + Token tok = lexer_.GetNextToken(); + if (tok == TOK_RPARENTHESIS) { + return tok; + } + AnfNodePtr node = nullptr; + ValuePtr value = nullptr; + tok = ParseItem(func_graph, &node, &value, tok); + if (tok != TOK_ERROR) { + MS_EXCEPTION_IF_NULL(inputs_ptr); + inputs_ptr->push_back(node); + } + return tok; + } + + const std::vector& GetFuncGraphs() const { return func_graphs_; } + + private: + Lexer lexer_; + std::vector func_graphs_; + bool error_flag_ = false; + + // store all parsed graphs + std::map func_graphs_map_; + // map from child to parent, consider adding a 'parent' field in class Graph + std::map parents_map_; + + // map for buffering cnodes when parsing a graph + std::map cnodes_; + + std::map param_nodes_; // map parameter name to parameter +}; + +std::vector ImportIR(const std::string& filename) { + IrParser paser(filename.c_str()); + paser.ParseFile(); + return paser.GetFuncGraphs(); +} + +#ifdef ENABLE_DUMP_IR +void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) { + if (func_graph == nullptr) { + MS_LOG(ERROR) << "func graph is nullptr"; + return; + } + auto ms_context = MsContext::GetInstance(); + if (ms_context == nullptr) { + MS_LOG(ERROR) << "ms_context is nullptr"; + return; + } + auto save_graphs_path = ms_context->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + std::string file_path = save_graphs_path + "/" + "ms_output_" + suffix + ".pb"; + if (file_path.size() > PATH_MAX) { + MS_LOG(ERROR) << "file path " << file_path << " is too long."; + return; + } + char real_path[PATH_MAX] = {0}; + if (nullptr == realpath(file_path.c_str(), real_path)) { + MS_LOG(DEBUG) << "dir " << file_path << " does not exit."; + } else { + std::string path_string = real_path; + if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) { + MS_LOG(ERROR) << "modify file:" << real_path << " to rw fail."; + return; + } + } + + // write to pb file + std::ofstream ofs(real_path); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file '" << real_path << "' failed!"; + return; + } + ofs << GetFuncGraphProtoString(func_graph); + ofs.close(); + // set file mode to read only by user + ChangeFileMode(file_path, S_IRUSR); +} +#else +void DumpIRProto(const FuncGraphPtr&, const std::string&) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR in protobuf format is disabled, " + << "please recompile source to enable it. See help of building script."; +} +#endif +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/anf_ir_utils.h b/mindspore/ccsrc/debug/anf_ir_utils.h new file mode 100644 index 00000000000..5b9ac9d2f02 --- /dev/null +++ b/mindspore/ccsrc/debug/anf_ir_utils.h @@ -0,0 +1,121 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_ANF_IR_UTILS_H_ +#define MINDSPORE_CCSRC_DEBUG_ANF_IR_UTILS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/meta_func_graph.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/resolve.h" +#include "operator/composite/composite.h" +#include "utils/symbolic.h" +#include "utils/ordered_map.h" +#include "utils/ordered_set.h" +#include "utils/utils.h" + +namespace mindspore { + +struct ParamPtrEqual { + bool operator()(AnfNodePtr const& t1, AnfNodePtr const& t2) const { + const ParameterPtr param1 = dyn_cast(t1); + const ParameterPtr param2 = dyn_cast(t2); + + if (param1 == nullptr || param2 == nullptr) { + return false; + } + + return *param1 == *param2; + } +}; + +struct ParamPtrHasher { + std::size_t operator()(AnfNodePtr const& param) const { + const ParameterPtr parameter = dyn_cast(param); + if (parameter == nullptr) { + return 0; + } + std::size_t hash = std::hash()(parameter->name()); + return hash; + } +}; + +class AnfExporter { + public: + explicit AnfExporter(const std::string& id, bool export_used = true) + : param_index(-1), id_(id), export_used_(export_used) { + func_graph_set.clear(); + exported.clear(); + } + ~AnfExporter() {} + + void ExportFuncGraph(const std::string& filename, const FuncGraphPtr& func_graph); + void ExportFuncGraph(const std::string& filename, const std::vector& graphs); + + private: + int GetParamIndex(const FuncGraphPtr& func_graph, const AnfNodePtr& param, bool throw_excp = true); + int GetParamIndexFromExported(const AnfNodePtr& param); + std::string DumpObject(const py::object& obj, const std::string& category) const; + std::string GetValueNodeText(const FuncGraphPtr& func_graph, const ValueNodePtr& node); + std::string GetMultitypeFuncGraphText(const prim::MultitypeFuncGraphPtr& mt_func_graph); + std::string GetSymbolicKeyInstanceText(const FuncGraphPtr& func_graph, const SymbolicKeyInstancePtr& sym_inst); + std::string GetSequenceText(const FuncGraphPtr& func_graph, const ValuePtr& value); + std::string GetValueText(const FuncGraphPtr& func_graph, const ValuePtr& value); + std::string GetOtherValueText(const FuncGraphPtr& func_graph, const ValuePtr& value); + std::string GetPrimitiveText(const PrimitivePtr& prim); + std::string GetDictText(const FuncGraphPtr& func_graph, const ValuePtr& value); + std::string GetNameSpaceText(const parse::NameSpacePtr& ns); + std::string GetMetaFuncGraphText(const MetaFuncGraphPtr& meta_func_graph); + std::string GetAnfNodeText(const FuncGraphPtr& func_graph, const AnfNodePtr& node, + const std::map& apply_map); + void ExportOneFuncGraph(std::ofstream& ofs, const FuncGraphPtr& func_graph); + void OutputParameters(std::ofstream& ofs, const std::vector& parameters, + OrderedMap* param_map); + + void OutputStatementComment(std::ofstream& ofs, const CNodePtr& node); + void OutputCNodes(std::ofstream& ofs, const std::vector& nodes, const FuncGraphPtr& func_graph); + + int param_index; + OrderedSet func_graph_set{}; + OrderedMap> exported; + std::string id_; + bool export_used_ = true; // whether export function graphs used in current exporting function graph + TaggedNodeMap tagged_cnodes_; +}; + +void ExportIR(const std::string& filename, const std::string& id, const FuncGraphPtr& func_graph); +void ExportIR(const std::string& filename, const std::vector& graphs); + +std::vector ImportIR(const std::string& filename); + +std::string GetFuncGraphProtoString(const FuncGraphPtr& func_graph); + +void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix); + +std::string GetOnnxProtoString(const FuncGraphPtr& func_graph); +std::string GetNodeType(const AnfNodePtr& nd); +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_ANF_IR_UTILS_H_ diff --git a/mindspore/ccsrc/debug/draw.cc b/mindspore/ccsrc/debug/draw.cc new file mode 100644 index 00000000000..e0949c04194 --- /dev/null +++ b/mindspore/ccsrc/debug/draw.cc @@ -0,0 +1,641 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/draw.h" + +#include +#include +#include +#include + +#include "ir/meta_func_graph.h" +#include "utils/graph_utils.h" +#include "utils/utils.h" +#include "operator/composite/composite.h" +#include "ir/meta_tensor.h" + +namespace mindspore { + +// namespace to support debug utils +namespace draw { + +namespace { +// Only for ValueNode +std::string ValueType(const ValueNodePtr& node) { + if (node == nullptr) { + return ""; + } + auto v = node->value(); + + return v->type_name(); +} + +std::string ReplaceSpecialChar(const std::string& str) { + std::ostringstream oss; + for (size_t i = 0; i < str.size(); i++) { + if (str[i] == '<') { + oss << "「"; + } else if (str[i] == '>') { + oss << "」"; + } else { + oss << str[i]; + } + } + return oss.str(); +} +} // namespace + +// API of debug utils +void DrawNodes(const std::vector& nodes, OrderedMap>* sub_graphs, + bool is_user) { + if (sub_graphs == nullptr) { + return; + } + for (auto& nd : nodes) { + MS_EXCEPTION_IF_NULL(nd); + auto sub_graph = nd->func_graph(); + if (sub_graph != nullptr) { + auto gsub = (*sub_graphs)[sub_graph]; + if (gsub == nullptr) { + if (is_user) { + gsub = std::make_shared(sub_graph->ToString()); + } else { + gsub = std::make_shared(sub_graph->ToString()); + } + (*sub_graphs)[sub_graph] = gsub; + } + if (!nd->isa()) { + gsub->Node(nd); + } + } + } +} + +void DrawValueNodes(const std::vector& nodes, + OrderedMap>* sub_graphs) { + if (sub_graphs == nullptr) { + return; + } + + int dup_idx = 0; + + for (auto& nd : nodes) { + for (auto& t : SuccIncoming(nd)) { + MS_EXCEPTION_IF_NULL(t); + MS_EXCEPTION_IF_NULL(nd); + if (t->isa() && (*sub_graphs).find(nd->func_graph()) != (*sub_graphs).end()) { + (*sub_graphs)[nd->func_graph()]->Node(t, dup_idx); + dup_idx++; + } else if (t->isa() && (*sub_graphs).find(t->func_graph()) != (*sub_graphs).end()) { + (*sub_graphs)[t->func_graph()]->Node(t, dup_idx); + dup_idx++; + } + } + } +} + +void DrawEdges(const std::vector& nodes, const std::shared_ptr& digraph, bool is_user) { + if (digraph == nullptr) { + return; + } + + int dup_idx = 0; + + int offset = 0; + if (is_user) { + offset = 1; + } + + // Draw edge + for (auto& nd : nodes) { + auto succs = SuccIncoming(nd); + auto num = succs.size(); + for (size_t i = 0; i < num; i++) { + auto& t = succs.at(i); + MS_EXCEPTION_IF_NULL(t); + if (t->isa() || t->isa()) { + if ((!is_user) || (i != 0)) { + // `SizeToInt(i) - offset` is just for printing as text + digraph->Edge(t, nd, SizeToInt(i) - offset, dup_idx); + } + if (IsValueNode(t)) { + auto const_graph = GetValueNode(t); + digraph->Edge(t, const_graph, dup_idx); + } + dup_idx++; + } else { + digraph->Edge(t, nd, SizeToInt(i) - offset); + } + } + } +} + +void DrawByOpt(std::string filename, const FuncGraphPtr& func_graph, bool is_user) { + if (func_graph == nullptr) { + return; + } + auto ret = func_graph->get_return(); + auto nodes = DeepScopedGraphSearch(ret); + + std::shared_ptr digraph; + OrderedMap> sub_graphs; + ChangeFileMode(filename, S_IRWXU); + if (is_user) { + digraph = std::make_shared("mindspore", filename); + } else { + digraph = std::make_shared("mindspore", filename); + } + + MS_EXCEPTION_IF_NULL(digraph); + digraph->Start(); + + // Draw nodes + DrawNodes(nodes, &sub_graphs, is_user); + + // Draw ValueNodes on CNodes + DrawValueNodes(nodes, &sub_graphs); + + // Draw subgraph + for (const auto& gsub : sub_graphs) { + digraph->SubGraph(gsub.first, gsub.second); + } + + // Draw edge + DrawEdges(nodes, digraph, is_user); + + digraph->End(); + // set file mode to read only by user + ChangeFileMode(filename, S_IRUSR); +} + +#ifdef ENABLE_DUMP_IR +void Draw(const std::string& filename, const FuncGraphPtr& func_graph) { + const std::string dot_suffix = ".dot"; + std::string filename_with_suffix = + (filename.rfind(dot_suffix) != (filename.size() - dot_suffix.size())) ? (filename + dot_suffix) : filename; + DrawByOpt(filename_with_suffix, func_graph, false); +} + +void DrawUserFuncGraph(const std::string& filename, const FuncGraphPtr& func_graph) { + DrawByOpt(filename, func_graph, true); +} +#else +void Draw(const std::string&, const FuncGraphPtr&) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR in graphviz dot format is disabled, " + << "please recompile source to enable it. See help of building script."; +} + +void DrawUserFuncGraph(const std::string&, const FuncGraphPtr&) { + static bool already_printed = false; + if (already_printed) { + return; + } + already_printed = true; + MS_LOG(WARNING) << "The functionality of dumping function graph IR in graphviz dot format is disabled, " + << "please recompile source to enable it. See help of building script."; +} +#endif + +std::string Graphviz::Shape(AnfNodePtr node) { + if (node == nullptr) { + return ""; + } + + if (node->isa()) { + return "plaintext"; + } + + if (node->isa()) { + return "octagon"; + } + + if (IsValueNode(node)) { + return "oval"; + } + + return "plaintext"; +} + +std::string Graphviz::Color(const AnfNodePtr& node) { + if (node == nullptr) { + return ""; + } + + if (node->isa()) { + return "cornsilk"; + } + + if (node->isa()) { + return "paleturquoise"; + } + + if (IsValueNode(node)) { + return "palegreen"; + } + + return "lavender"; +} + +void BaseDigraph::Start() { + buffer_ << "digraph " << name_ << " {" << std::endl; + buffer_ << "compound=true" << std::endl; +} + +void BaseDigraph::Head(const AnfNodePtr& node, int id) { + if (node == nullptr) { + return; + } + + buffer_ << "node" << node << "_" << id; + if (node->isa() || (node->isa() && !IsValueNode(node))) { + buffer_ << ":core"; + } +} + +void BaseDigraph::Tail(const AnfNodePtr& node, int idx, int id) { + if (node == nullptr) { + return; + } + + buffer_ << "node" << node << "_" << id; + buffer_ << ":" << idx; +} + +void BaseDigraph::Tail(const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return; + } + buffer_ << "node" << func_graph->get_return() << "_" << 0; +} + +void BaseDigraph::Edge(AnfNodePtr start, FuncGraphPtr end, int id_start) { + Head(start, id_start); + buffer_ << "->"; + Tail(end); + + buffer_ << "[lhead=cluster_" << end; + buffer_ << ",dir=both,arrowhead=dot,style=filled,color=blue]"; + buffer_ << std::endl; +} + +void BaseDigraph::End() { + buffer_ << "}" << std::endl; + + if (fout_.is_open()) { + fout_ << buffer_.str(); + } +} + +void BaseDigraph::FuncGraphParameters(const FuncGraphPtr& key) { + buffer_ << "parameters_" << key << "[shape=plaintext "; + buffer_ << "label=<"; + buffer_ << ""; + int count = 0; + for (auto& parameter : key->parameters()) { + buffer_ << ""; + count++; + // wrap the text. + if (count % 10 == 0) { + buffer_ << "\n"; + } + } + buffer_ << "
parameters
"; + buffer_ << parameter->ToString(); + auto py_p = dyn_cast(parameter)->default_param(); + if (py::hasattr(py_p, "default_input")) { + py_p = py_p.attr("default_input"); + if (py::hasattr(py_p, PYTHON_TENSOR_FLAG)) { + std::shared_ptr m_tensor = py_p.cast>(); + py::tuple shape = m_tensor->GetPyTupleShape(); + buffer_ << "[" << std::string(py::str(shape)) << "]"; + } + } + buffer_ << "
>,];"; +} + +void BaseDigraph::SubGraph(const FuncGraphPtr& key, const std::shared_ptr& gsub) { + if (key == nullptr || gsub == nullptr) { + return; + } + + std::string label = key->debug_info()->get_full_name(); + if (label.empty()) { + label = gsub->name(); + } + + std::string label_managed = "[managed]"; + if (key->manager() == nullptr) { + label_managed = "[not managed]"; + } + label += label_managed; + + gsub->FuncGraphParameters(key); + buffer_ << "subgraph cluster_" << key << "{" << std::endl; + buffer_ << "id=cluster_" << key << std::endl; + buffer_ << "label=\"" << label << "\"" << std::endl; + buffer_ << "fontname=\"Courier New\"" << std::endl; + buffer_ << gsub->buffer().str(); + buffer_ << "}" << std::endl; +} + +Digraph::~Digraph() { + try { + if (fout_.is_open()) { + fout_.close(); + } + } catch (const std::exception& e) { + MS_LOG(ERROR) << "exception when closing file " << filename_; + } +} + +static std::string ReplaceAll(std::string str, const std::string& from, const std::string& to) { + size_t start_pos = 0; + while ((start_pos = str.find(from, start_pos)) != std::string::npos) { + (void)str.replace(start_pos, from.length(), to); + start_pos += to.length(); // Handles case where 'to' is a substring of 'from' + } + return str; +} + +static void DrawValueNode(Graphviz* const graph_obj, const ValueNodePtr& node) { + MS_EXCEPTION_IF_NULL(graph_obj); + graph_obj->buffer() << "label=<"; + graph_obj->buffer() << ""; + graph_obj->buffer() << ""; + graph_obj->buffer() << ""; + graph_obj->buffer() << ""; + graph_obj->buffer() << "
"; + graph_obj->buffer() << ValueType(node); + graph_obj->buffer() << "
"; + if (IsValueNode(node)) { + graph_obj->buffer() << node->value()->cast()->name(); + } else if (IsValueNode(node)) { + graph_obj->buffer() << node->value()->cast()->name(); + } else if (IsValueNode(node)) { + graph_obj->buffer() << ReplaceSpecialChar(node->value()->cast()->name()); + } else { + std::ostringstream ss; + ss << node->value()->ToString(); + std::string s = ReplaceAll(ss.str(), ", ", "
"); + graph_obj->buffer() << s; + ValuePtr value = node->value(); + if (value->isa()) { + PrimitivePtr primitive = value->cast(); + graph_obj->buffer() << "
"; + if (!primitive->instance_name().empty()) { + graph_obj->buffer() << "instance name:" + << " "; + graph_obj->buffer() << primitive->instance_name(); + graph_obj->buffer() << "
"; + } + auto attrs = primitive->attrs(); + if (attrs.size() > 0) { + graph_obj->buffer() << "
"; + int i = 0; + for (const auto& attr : attrs) { + if (i != 0) { + graph_obj->buffer() << "
"; + } + graph_obj->buffer() << attr.first << " "; + graph_obj->buffer() << attr.second->ToString(); + i++; + } + } + } + } + graph_obj->buffer() << "
>,"; +} + +static void DrawParallelInfo(Graphviz* const graph_obj, const CNodePtr& node) { + if (graph_obj == nullptr || node == nullptr) { + return; + } + auto distributed_operation_info = node->operator_info(); + if (distributed_operation_info != nullptr) { + auto strategyPtr = distributed_operation_info->strategy(); + if (strategyPtr != nullptr) { + auto num = node->inputs().size(); + graph_obj->buffer() << "buffer() << "bgcolor='" << graph_obj->Color(node) << "'>"; + std::vector temp = {MakeValue(strategyPtr->GetInputStage()), MakeValue(strategyPtr->GetInputDim())}; + ValueTuplePtr strategy_tuple = std::make_shared(temp); + graph_obj->buffer() << "Strategy " << strategy_tuple->ToString(); + graph_obj->buffer() << "" << std::endl; + } + } +} + +static void DrawCNode(Graphviz* const graph_obj, const CNodePtr& node) { + if (graph_obj == nullptr || node == nullptr || node->size() == 0) { + return; + } + auto num = node->size(); + bool is_modelgraph = false; + if (typeid(*graph_obj) == typeid(ModelDigraph)) { + is_modelgraph = true; + num -= 1; + } + + graph_obj->buffer() << "label=<" << std::endl; + // Draw ports for CNode + if (num > 0) { + graph_obj->buffer() << ""; + for (size_t i = 0; i < num; i++) { + graph_obj->buffer() << ""; + } + graph_obj->buffer() << "" << std::endl; + } + + // Draw op name + graph_obj->buffer() << " 0) { + graph_obj->buffer() << " colspan='" << num << "'"; + } + graph_obj->buffer() << " bgcolor='" << graph_obj->Color(node) << "'>"; + + if (IsValueNode(node->input(0)) && is_modelgraph) { + auto primitive = GetValueNode(node->input(0)); + graph_obj->buffer() << ReplaceAll(primitive->ToString(), ", ", "
"); + auto attrs = primitive->attrs(); + if (attrs.size() > 0) { + graph_obj->buffer() << "" << std::endl << " 0) { + graph_obj->buffer() << " colspan='" << num << "'"; + } + graph_obj->buffer() << ">"; + int i = 0; + for (auto& attr : attrs) { + if (i != 0) { + graph_obj->buffer() << "
"; + } + graph_obj->buffer() << attr.first << " " << attr.second->ToString(); + i++; + } + } + graph_obj->buffer() << "CNode"; + } else { + graph_obj->buffer() << "CNode(" << node->ToString() << ")"; + } + + graph_obj->buffer() << "" << std::endl; + DrawParallelInfo(graph_obj, node); + graph_obj->buffer() << "
" << i << "
>,"; +} + +void Digraph::Node(AnfNodePtr node, int id) { + if (node == nullptr) { + return; + } + + buffer_ << "node" << node << "_" << id; + buffer_ << "["; + + // set fontname + buffer_ << "fontname=\"Courier New\","; + // set label and shape + buffer_ << "shape=" << Shape(node) << ","; + if (node->isa()) { + DrawCNode(this, node->cast()); + } else if (node->isa() && !IsValueNode(node)) { + DrawValueNode(this, node->cast()); + } else { + buffer_ << "label=\"" << node->ToString(); + if (IsValueNode(node)) { + FuncGraphPtr nextNet = GetValueNode(node); + std::string nextNetName = nextNet->debug_info()->get_full_name(); + if (!nextNetName.empty()) { + buffer_ << "[" << nextNet->debug_info()->get_full_name().c_str() << "]"; + } + } + buffer_ << "\"," + << "style=filled,fillcolor=" << Color(node) << ","; + } + + // Set URL for func graph + if (IsValueNode(node)) { + buffer_ << "URL=\"#cluster_" << GetValueNode(node) << "\","; + } + + buffer_ << "]" << std::endl; +} + +void Digraph::Edge(AnfNodePtr start, AnfNodePtr end, int idx, int id_start) { + if (start == nullptr || end == nullptr) { + return; + } + + Head(start, id_start); + buffer_ << "->"; + Tail(end, idx); + + buffer_ << "[arrowhead=vee,"; + + // check how many inputs for end + if (end->isa()) { + auto cnode = end->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto num = cnode->inputs().size(); + if (idx == 0 && num > 1) { + buffer_ << "style=dashed"; + } + } + buffer_ << "]" << std::endl; +} + +ModelDigraph::~ModelDigraph() { + try { + if (fout_.is_open()) { + fout_.close(); + } + } catch (const std::exception& e) { + MS_LOG(ERROR) << "exception when closing file " << filename_; + } +} + +std::string ModelDigraph::Shape(AnfNodePtr node) { + if (node == nullptr) { + return ""; + } + + if (node->isa()) { + return "plaintext"; + } + + if (node->isa()) { + return "ellipse"; + } + + if (IsValueNode(node)) { + return "oval"; + } + + return "plaintext"; +} + +void ModelDigraph::Node(AnfNodePtr node, int id) { + if (node == nullptr) { + return; + } + + if (IsValueNode(node)) { + return; + } + + buffer_ << "node" << node << "_" << id; + buffer_ << "["; + + // set fontname + buffer_ << "fontname=\"Courier New\","; + // set label and shape + buffer_ << "shape=" << Shape(node) << ","; + if (node->isa()) { + DrawCNode(this, node->cast()); + } else if (node->isa() && !IsValueNode(node)) { + DrawValueNode(this, node->cast()); + } else { + buffer_ << "label=\"" << node->ToString() << "\","; + buffer_ << "style=filled,fillcolor=" << Color(node) << ","; + } + + // Set URL for func graph + if (IsValueNode(node)) { + buffer_ << "URL=\"#cluster_" << GetValueNode(node) << "\","; + } + + buffer_ << "]" << std::endl; +} + +void ModelDigraph::Edge(AnfNodePtr start, AnfNodePtr end, int idx, int id_start) { + if (start == nullptr || end == nullptr) { + return; + } + + Head(start, id_start); + buffer_ << "->"; + Tail(end, idx); + + buffer_ << "[arrowhead=vee,"; + buffer_ << "]" << std::endl; +} +} // namespace draw +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/draw.h b/mindspore/ccsrc/debug/draw.h new file mode 100644 index 00000000000..4781a6c231f --- /dev/null +++ b/mindspore/ccsrc/debug/draw.h @@ -0,0 +1,105 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DRAW_H_ +#define MINDSPORE_CCSRC_DEBUG_DRAW_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "utils/any.h" +#include "pipeline/parse/resolve.h" + +namespace mindspore { +namespace draw { + +namespace parse = mindspore::parse; + +class Graphviz { + public: + Graphviz(const std::string& name, const std::string& filename) : name_(name), filename_(filename), fout_(filename_) {} + + explicit Graphviz(const std::string& name) : name_(name) {} + + virtual ~Graphviz() {} + + virtual void Start() {} + virtual void End() {} + + virtual std::string Shape(AnfNodePtr node); + std::string Color(const AnfNodePtr& node); + std::ostringstream& buffer() { return buffer_; } + std::ostringstream buffer_; + + protected: + std::string name_; + std::string filename_; + std::ofstream fout_; +}; + +class BaseDigraph : public Graphviz { + public: + BaseDigraph(const std::string& name, const std::string& filename) : Graphviz(name, filename) {} + explicit BaseDigraph(const std::string& name) : Graphviz(name) {} + ~BaseDigraph() override = default; + + virtual void Node(AnfNodePtr node, int id = 0) = 0; + virtual void Edge(AnfNodePtr start, AnfNodePtr end, int idx, int idx_start = 0) = 0; + + void Start() override; + void End() override; + virtual void Edge(AnfNodePtr start, FuncGraphPtr end, int id_start); + void FuncGraphParameters(const FuncGraphPtr& key); + void SubGraph(const FuncGraphPtr& key, const std::shared_ptr& gsub); + + const std::string& name() const { return name_; } + + protected: + void Head(const AnfNodePtr& node, int id = 0); + void Tail(const AnfNodePtr& node, int idx, int id = 0); + void Tail(const FuncGraphPtr& func_graph); +}; + +class Digraph : public BaseDigraph { + public: + Digraph(const std::string& name, const std::string& filename) : BaseDigraph(name, filename) {} + explicit Digraph(const std::string& name) : BaseDigraph(name) {} + ~Digraph() override; + + void Node(AnfNodePtr node, int id = 0) override; + void Edge(AnfNodePtr start, AnfNodePtr end, int idx, int idx_start = 0) override; +}; + +class ModelDigraph : public BaseDigraph { + public: + ModelDigraph(const std::string& name, const std::string& filename) : BaseDigraph(name, filename) {} + explicit ModelDigraph(const std::string& name) : BaseDigraph(name) {} + ~ModelDigraph() override; + + std::string Shape(AnfNodePtr node) override; + void Node(AnfNodePtr node, int id = 0) override; + void Edge(AnfNodePtr start, AnfNodePtr end, int idx, int idx_start = 0) override; +}; + +// API to draw +void Draw(const std::string& filename, const FuncGraphPtr& func_graph); +void DrawUserFuncGraph(const std::string& filename, const FuncGraphPtr& func_graph); + +} // namespace draw +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_DRAW_H_ diff --git a/mindspore/ccsrc/debug/dump_proto.cc b/mindspore/ccsrc/debug/dump_proto.cc new file mode 100644 index 00000000000..a12cf8b47f1 --- /dev/null +++ b/mindspore/ccsrc/debug/dump_proto.cc @@ -0,0 +1,513 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "debug/anf_ir_utils.h" +#include "utils/anf_ir.pb.h" +#include "utils/graph_utils.h" +#include "utils/symbolic.h" + +namespace mindspore { +class ProtoExporter { + public: + ProtoExporter() {} + ~ProtoExporter() {} + + std::string GetFuncGraphProtoString(const FuncGraphPtr& func_graph); + + private: + void InitModelInfo(); + void GetOpNodeTypeAndAttrs(const FuncGraphPtr& func_graph, const AnfNodePtr& node, irpb::NodeProto* node_proto); + std::string GetOpNodeInputId(const FuncGraphPtr& func_graph, const AnfNodePtr& node, + const std::map& apply_map, + std::map* const_map_ptr); + void SetValueToProto(const ValuePtr& attr_value, irpb::ValueProto* value_proto); + void SetScalarToProto(const ScalarPtr& val, irpb::ValueProto* value_proto); + void SetSequenceToProto(const ValueSequeuePtr& val, irpb::ValueProto* value_proto); + void SetDictionaryToProto(const ValueDictionaryPtr& val, irpb::ValueProto* value_proto); + void SetNodeOutputType(const AnfNodePtr& node, irpb::TypeProto* type_proto); + void SetNodeOutputType(const TypePtr& node, const BaseShapePtr& shape, irpb::TypeProto* type_proto); + + void ExportFuncGraph(const FuncGraphPtr& func_graph, irpb::GraphProto* graph_proto); + void ExportParameters(const FuncGraphPtr& func_graph, irpb::GraphProto* graph_proto); + void ExportCNodes(const FuncGraphPtr& func_graph, irpb::GraphProto* graph_proto, + std::map* const_map_ptr); + void ExportCNode(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* apply_map_ptr, + std::map* const_map_ptr, irpb::GraphProto* graph_proto); + void ExportFuncGraphOutput(const FuncGraphPtr& func_graph, const CNodePtr& ret_node, + const std::map& apply_map, std::map* const_map_ptr, + irpb::GraphProto* graph_proto); + void ExportValueNodes(const std::map& const_map, irpb::GraphProto* graph_proto); + + static std::string GetConstNodeId(size_t idx) { return std::string("cst") + std::to_string(idx); } + + irpb::ModelProto model_; +}; + +static irpb::DataType GetNumberDataType(const TypePtr& type) { + switch (type->type_id()) { + case kNumberTypeBool: + return irpb::DT_BOOL; + case kNumberTypeInt8: + return irpb::DT_INT8; + case kNumberTypeInt16: + return irpb::DT_INT16; + case kNumberTypeInt32: + return irpb::DT_INT32; + case kNumberTypeInt64: + return irpb::DT_INT64; + case kNumberTypeUInt8: + return irpb::DT_UINT8; + case kNumberTypeUInt16: + return irpb::DT_UINT16; + case kNumberTypeUInt32: + return irpb::DT_UINT32; + case kNumberTypeUInt64: + return irpb::DT_UINT64; + case kNumberTypeFloat16: + return irpb::DT_FLOAT16; + case kNumberTypeFloat32: + return irpb::DT_FLOAT32; + case kNumberTypeFloat64: + return irpb::DT_FLOAT64; + case kNumberTypeInt: + return irpb::DT_BASE_INT; + case kNumberTypeUInt: + return irpb::DT_BASE_UINT; + case kNumberTypeFloat: + return irpb::DT_BASE_FLOAT; + default: + MS_LOG(EXCEPTION) << "Unexpected type " << type->type_name(); + } +} + +void ProtoExporter::SetNodeOutputType(const TypePtr& type, const BaseShapePtr& shape, irpb::TypeProto* type_proto) { + if (type_proto == nullptr) { + return; + } + + if (type == nullptr) { + type_proto->set_data_type(irpb::DT_UNDEFINED); + } else if (type->isa()) { + type_proto->set_data_type(GetNumberDataType(type)); + } else if (type->isa()) { + TypePtr elem_type = dyn_cast(type)->element(); + type_proto->mutable_tensor_type()->set_elem_type(GetNumberDataType(elem_type)); + type_proto->set_data_type(irpb::DT_TENSOR); + if (shape != nullptr && shape->isa()) { + abstract::ShapePtr shape_info = dyn_cast(shape); + for (const auto& elem : shape_info->shape()) { + type_proto->mutable_tensor_type()->mutable_shape()->add_dim()->set_size(elem); + } + } + } else if (type->isa()) { + TuplePtr tuple_type = dyn_cast(type); + type_proto->set_data_type(irpb::DT_TUPLE); + for (const auto& elem_type : tuple_type->elements()) { + SetNodeOutputType(elem_type, nullptr, type_proto->mutable_sequence_type()->add_elem_types()); + } + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_TYPE); + } else if (type->isa()) { + ListPtr list_type = dyn_cast(type); + type_proto->set_data_type(irpb::DT_LIST); + for (const auto& elem_type : list_type->elements()) { + SetNodeOutputType(elem_type, nullptr, type_proto->mutable_sequence_type()->add_elem_types()); + } + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_ANYTHING); + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_REFKEY); + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_REF); + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_GRAPH); + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_NONE); + } else if (type->isa()) { + type_proto->set_data_type(irpb::DT_STRING); + } else if (type->isa()) { + // Do Nothing. + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << type->type_name(); + } +} + +void ProtoExporter::SetNodeOutputType(const AnfNodePtr& node, irpb::TypeProto* type_proto) { + if (node == nullptr || type_proto == nullptr) { + return; + } + SetNodeOutputType(node->Type(), node->Shape(), type_proto); +} + +void ProtoExporter::SetValueToProto(const ValuePtr& val, irpb::ValueProto* value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const StringImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_STRING); + value_proto->set_str_val(value->value()); + } else if (val->isa()) { + SetScalarToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + value_proto->set_dtype(irpb::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(irpb::DT_BOOL); + } else if (val->isa()) { + value_proto->set_dtype(irpb::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(irpb::DT_BASE_INT); + } else if (val->isa()) { + value_proto->set_dtype(irpb::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(irpb::DT_BASE_FLOAT); + } else if (val->isa()) { + SetSequenceToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + value_proto->set_dtype(irpb::DT_NONE); + value_proto->set_str_val("None"); + } else if (val->isa()) { + SymbolicKeyInstancePtr sym_inst = dyn_cast(val); + ParameterPtr sym_node = dyn_cast(sym_inst->node()); + value_proto->set_dtype(irpb::DT_SYM_INST); + value_proto->set_str_val(sym_node == nullptr ? std::string("nullptr") : sym_node->ToString()); + } else if (val->isa()) { + SetDictionaryToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + tensor::TensorPtr tensor_ptr = dyn_cast(val); + value_proto->set_dtype(irpb::DT_TENSOR); + irpb::TensorProto* tensor_proto = value_proto->mutable_tensor_val(); + tensor_proto->set_data_type(GetNumberDataType(tensor_ptr->Dtype())); + for (auto& elem : tensor_ptr->shape()) { + tensor_proto->add_dims(elem); + } + } else if (val->isa()) { + value_proto->set_dtype(irpb::DT_TYPE); + + irpb::TypeProto* type_proto = value_proto->mutable_type_val(); + type_proto->set_data_type(irpb::DT_TENSOR); + TypePtr elem_type = dyn_cast(val)->element(); + type_proto->mutable_tensor_type()->set_elem_type(GetNumberDataType(elem_type)); + } else { + MS_LOG(WARNING) << "Not supported type " << val->type_name(); + } +} + +void ProtoExporter::SetScalarToProto(const ScalarPtr& val, irpb::ValueProto* value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const BoolImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_BOOL); + value_proto->set_bool_val(value->value()); + } else if (val->isa()) { + const Int8ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_INT8); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int16ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_INT16); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int32ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_INT32); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int64ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_INT64); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const UInt8ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_UINT8); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt16ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_UINT16); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt32ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_UINT32); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt64ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_UINT64); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const FP32ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_FLOAT32); + value_proto->set_float_val(value->value()); + } else if (val->isa()) { + const FP64ImmPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_FLOAT64); + value_proto->set_double_val(value->value()); + } else { + MS_LOG(EXCEPTION) << "Unknown scalar type " << val->ToString(); + } +} + +void ProtoExporter::SetSequenceToProto(const ValueSequeuePtr& val, irpb::ValueProto* value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const ValueTuplePtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_TUPLE); + for (const auto& item : value->value()) { + SetValueToProto(item, value_proto->add_values()); + } + } else if (val->isa()) { + const ValueListPtr& value = dyn_cast(val); + value_proto->set_dtype(irpb::DT_LIST); + for (const auto& item : value->value()) { + SetValueToProto(item, value_proto->add_values()); + } + } +} + +void ProtoExporter::SetDictionaryToProto(const ValueDictionaryPtr& val, irpb::ValueProto* value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + value_proto->set_dtype(irpb::DT_DICT); + for (const auto& item : val->value()) { + irpb::NamedValueProto* named_val = value_proto->add_dict_val(); + named_val->set_key(item.first); + SetValueToProto(item.second, named_val->mutable_value()); + } +} + +void ProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr&, const AnfNodePtr& node, irpb::NodeProto* node_proto) { + if (node == nullptr || node_proto == nullptr) { + return; + } + + if (node->isa() || node->isa() || IsValueNode(node)) { + MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << node->ToString(); + } + + if (!IsValueNode(node)) { + MS_LOG(EXCEPTION) << "Op node is not primitive: " << node->ToString(); + } + + const PrimitivePtr& prim = GetValueNode(node); + node_proto->set_op_type(prim->name()); + for (const auto& attr : prim->attrs()) { + irpb::AttributeProto* attr_proto = node_proto->add_attribute(); + attr_proto->set_name(attr.first); + SetValueToProto(attr.second, attr_proto->mutable_value()); + } + node_proto->set_scope(node->scope()->name()); +} + +std::string ProtoExporter::GetOpNodeInputId(const FuncGraphPtr&, const AnfNodePtr& node, + const std::map& apply_map, + std::map* const_map_ptr) { + if (node == nullptr || const_map_ptr == nullptr) { + return ""; + } + + if (node->isa()) { + auto iter = apply_map.find(node); + if (iter == apply_map.end()) { + MS_LOG(EXCEPTION) << "Can not find node '" << node->ToString() << "' in apply_map"; + } + return std::to_string(iter->second); + } + + if (node->isa()) { + return node->ToString(); + } + + if (node->isa()) { + auto iter = const_map_ptr->find(node); + if (iter == const_map_ptr->end()) { + // Start index number from 1 + auto const_idx = const_map_ptr->size() + 1; + (*const_map_ptr)[node] = const_idx; + } + return GetConstNodeId((*const_map_ptr)[node]); + } + + MS_LOG(EXCEPTION) << "Unknown node type. node is '" << node->ToString() << "'"; +} + +std::string ProtoExporter::GetFuncGraphProtoString(const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return ""; + } + + InitModelInfo(); + irpb::GraphProto* graph_proto = model_.mutable_graph(); + ExportFuncGraph(func_graph, graph_proto); + return model_.SerializeAsString(); +} + +void ProtoExporter::ExportFuncGraph(const FuncGraphPtr& func_graph, irpb::GraphProto* graph_proto) { + if (func_graph == nullptr || graph_proto == nullptr) { + return; + } + + // map for store ValueNodes of this graph + std::map const_map; + + // set graph name + graph_proto->set_name(func_graph->ToString()); + + ExportParameters(func_graph, graph_proto); + + ExportCNodes(func_graph, graph_proto, &const_map); + + ExportValueNodes(const_map, graph_proto); +} + +void ProtoExporter::ExportParameters(const FuncGraphPtr& func_graph, irpb::GraphProto* graph_proto) { + if (func_graph == nullptr || graph_proto == nullptr) { + return; + } + + std::vector parameters = func_graph->parameters(); + for (auto& param : parameters) { + irpb::ParameterProto* param_proto = graph_proto->add_parameters(); + param_proto->set_name(param->ToString()); + + SetNodeOutputType(param, param_proto->mutable_type()); + + const ParameterPtr param_ptr = dyn_cast(param); + if (param_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Parameter '" << param->ToString() << "' could not cast to parameter."; + } + } +} + +void ProtoExporter::ExportCNodes(const FuncGraphPtr& func_graph, irpb::GraphProto* graph_proto, + std::map* const_map_ptr) { + if (func_graph == nullptr || graph_proto == nullptr || const_map_ptr == nullptr) { + return; + } + // topo sort nodes + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + std::map apply_map; + for (const AnfNodePtr& node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + if (cnode != func_graph->get_return()) { + ExportCNode(func_graph, cnode, &apply_map, const_map_ptr, graph_proto); + } else { + ExportFuncGraphOutput(func_graph, cnode, apply_map, const_map_ptr, graph_proto); + } + } +} + +void ProtoExporter::ExportCNode(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* apply_map_ptr, + std::map* const_map_ptr, irpb::GraphProto* graph_proto) { + if (func_graph == nullptr || node == nullptr || apply_map_ptr == nullptr || const_map_ptr == nullptr || + graph_proto == nullptr) { + return; + } + + auto apply_idx = apply_map_ptr->size() + 1; + (*apply_map_ptr)[node] = apply_idx; + + auto& inputs = node->inputs(); + if (inputs.size() < 1) { + MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; + } + AnfNodePtr op = inputs[0]; + irpb::NodeProto* node_proto = graph_proto->add_node(); + + // CNode/ConstGraph/Const/Parameter + if (op->isa() || IsValueNode(op) || op->isa()) { + MS_LOG(WARNING) << "Operator must be a primitive"; + } else { + GetOpNodeTypeAndAttrs(func_graph, op, node_proto); + node_proto->set_name(std::to_string(apply_idx)); + node_proto->set_scope(node->scope()->name()); + + // process OP inputs + for (size_t i = 1; i < inputs.size(); ++i) { + irpb::InputProto* input_proto = node_proto->add_input(); + input_proto->set_type(irpb::InputProto_EdgeType_DATA_EDGE); + std::string id = GetOpNodeInputId(func_graph, inputs[i], *apply_map_ptr, const_map_ptr); + input_proto->set_name(id); + } + + // set node output type + SetNodeOutputType(node, node_proto->mutable_output_type()); + } +} + +void ProtoExporter::ExportFuncGraphOutput(const FuncGraphPtr& func_graph, const CNodePtr& ret_node, + const std::map& apply_map, + std::map* const_map_ptr, irpb::GraphProto* graph_proto) { + if (ret_node == nullptr || !ret_node->isa()) { + MS_LOG(EXCEPTION) << "Graph return node is illegal"; + } + AnfNodePtr arg = ret_node->input(1); + if (graph_proto == nullptr) { + MS_LOG(EXCEPTION) << "graph_proto is nullptr"; + } + irpb::OutputProto* output_proto = graph_proto->add_outputs(); + if (output_proto == nullptr) { + MS_LOG(EXCEPTION) << "output_proto is nullptr"; + } + std::string id = GetOpNodeInputId(func_graph, arg, apply_map, const_map_ptr); + output_proto->set_name(id); + SetNodeOutputType(arg, output_proto->mutable_type()); +} + +static bool CompareValue(const std::pair& x, const std::pair& y) { + return x.second < y.second; +} + +void ProtoExporter::ExportValueNodes(const std::map& const_map, irpb::GraphProto* graph_proto) { + std::vector> nodes; + (void)std::transform(const_map.cbegin(), const_map.cend(), std::back_inserter(nodes), + [](const std::pair& item) { return item; }); + + sort(nodes.begin(), nodes.end(), CompareValue); + + for (auto& item : nodes) { + if (graph_proto == nullptr) { + MS_LOG(EXCEPTION) << "graph_proto is nullptr"; + } + irpb::NamedValueProto* named_value = graph_proto->add_const_vals(); + MS_EXCEPTION_IF_NULL(named_value); + named_value->set_key(GetConstNodeId(item.second)); + SetValueToProto(GetValueNode(item.first), named_value->mutable_value()); + } +} + +void ProtoExporter::InitModelInfo() { model_.set_ir_version(irpb::IR_VERSION); } + +std::string GetFuncGraphProtoString(const FuncGraphPtr& func_graph) { + ProtoExporter exporter; + return exporter.GetFuncGraphProtoString(func_graph); +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/e2e_dump.cc b/mindspore/ccsrc/debug/e2e_dump.cc new file mode 100644 index 00000000000..3006bb66da8 --- /dev/null +++ b/mindspore/ccsrc/debug/e2e_dump.cc @@ -0,0 +1,253 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/e2e_dump.h" +#include +#include +#include +#include +#include "utils/log_adapter.h" +#include "utils/system/file_system.h" +#include "utils/system/env.h" +#include "utils/convert_utils.h" +#include "utils/context/ms_context.h" + +using json = nlohmann::json; + +namespace mindspore { +Dump::Dump() + : dump_enable_(false), + trans_flag_(false), + dump_path_("/tmp/"), + dump_net_name_("net_name"), + dump_mode_(0), + dump_iter_(0), + cur_iter_(0) {} + +bool Dump::IsKernelNeedDump(const std::string& kernel_name) { + if (dump_mode_ == 0) { + // Dump All Kernels mode + return true; + } else { + auto iter = std::find(dump_kernels_.begin(), dump_kernels_.end(), kernel_name); + if (iter != dump_kernels_.end()) { + return true; + } + } + return false; +} + +bool Dump::ParseDumpConfig(const string& dump_config_file) { + std::ifstream jsonFile(dump_config_file); + if (!jsonFile.is_open()) { + MS_LOG(ERROR) << dump_config_file << " open failed."; + dump_enable_ = false; + return false; + } + json j; + jsonFile >> j; + if (j.find("DumpSettings") == j.end()) { + MS_LOG(ERROR) << "DumpSettings is not exist."; + dump_enable_ = false; + return false; + } else { + json dumpSettings = j.at("DumpSettings"); + // convert json to string + std::stringstream ss; + ss << dumpSettings; + std::string cfg = ss.str(); + MS_LOG(INFO) << "E2E Dump Settings Json: " << cfg; + if (!IsConfigExist(dumpSettings)) { + return false; + } + if (!IsConfigValid(dumpSettings)) { + return false; + } + } + return true; +} + +bool Dump::IsConfigExist(const nlohmann::json& dumpSettings) { + if (dumpSettings.find("trans_flag") == dumpSettings.end() || dumpSettings.find("enable") == dumpSettings.end() || + dumpSettings.find("mode") == dumpSettings.end() || dumpSettings.find("path") == dumpSettings.end() || + dumpSettings.find("net_name") == dumpSettings.end() || dumpSettings.find("iteration") == dumpSettings.end() || + dumpSettings.find("kernels") == dumpSettings.end()) { + MS_LOG(ERROR) << "DumpSettings keys is not exist."; + dump_enable_ = false; + return false; + } + return true; +} + +bool Dump::IsConfigValid(const nlohmann::json& dumpSettings) { + auto trans_flag = dumpSettings.at("trans_flag"); + auto enable = dumpSettings.at("enable"); + auto mode = dumpSettings.at("mode"); + auto path = dumpSettings.at("path"); + auto net_name = dumpSettings.at("net_name"); + auto iteration = dumpSettings.at("iteration"); + auto kernels = dumpSettings.at("kernels"); + if (!(enable.is_boolean() && trans_flag.is_boolean() && mode.is_number() && path.is_string() && + net_name.is_string() && iteration.is_number() && kernels.is_array())) { + MS_LOG(ERROR) << "element's type in Dump config json is invalid."; + dump_enable_ = false; + return false; + } + + dump_enable_ = enable; + trans_flag_ = trans_flag; + dump_mode_ = mode; + dump_path_ = path; + dump_net_name_ = net_name; + dump_iter_ = iteration; + for (const auto& kernel : kernels) { + dump_kernels_.push_back(kernel); + } + return true; +} + +bool Dump::SetDumpConfFromJsonFile() { + const char* config_path_str = std::getenv("MINDSPORE_CONFIG_PATH"); + if (config_path_str != nullptr) { + MS_LOG(INFO) << "getenv MINDSPORE_CONFIG_PATH :" << config_path_str; + } else { + MS_LOG(INFO) << "No need E2E Dump. please export MINDSPORE_CONFIG_PATH eg: MINDSPORE_CONFIG_PATH=/etc"; + dump_enable_ = false; + return false; + } + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + auto id = context_ptr->device_id(); + char real_path[PATH_MAX] = {0}; + if (nullptr == realpath(config_path_str, real_path)) { + MS_LOG(ERROR) << "env e2e dump path error, " << config_path_str; + dump_enable_ = false; + return false; + } + std::string dump_config_file = std::string(real_path) + "/e2e_dump_config_" + std::to_string(id) + ".json"; + std::shared_ptr fs = system::Env::GetFileSystem(); + MS_EXCEPTION_IF_NULL(fs); + if (!fs->FileExist(dump_config_file)) { + MS_LOG(ERROR) << dump_config_file << " not exist."; + dump_enable_ = false; + return false; + } + + return ParseDumpConfig(dump_config_file); +} + +bool Dump::DumpToFile(const std::string& filename, const void* data, size_t len) { + if (filename.empty() || data == nullptr || len == 0) { + MS_LOG(ERROR) << "incorrect parameter."; + return false; + } + + std::string realpath; + bool ret = GetRealPath(filename, &realpath); + if (!ret) { + MS_LOG(ERROR) << "get real path failed."; + return false; + } + std::ofstream fd; + fd.open(realpath, std::ios::binary | std::ios::out); + if (!fd.is_open()) { + MS_LOG(ERROR) << "open file " << realpath << " fail."; + return false; + } + (void)fd.write(reinterpret_cast(data), SizeToLong(len)); + fd.close(); + return true; +} + +bool Dump::GetRealPath(const std::string& inpath, std::string* outpath) { + MS_EXCEPTION_IF_NULL(outpath); + auto path_split_pos = inpath.find_last_of('/'); + if (path_split_pos == std::string::npos) { + path_split_pos = inpath.find_last_of('\\'); + } + // get real path + char real_path[PATH_MAX] = {0}; + if (path_split_pos != std::string::npos) { + std::string prefix_path = inpath.substr(0, path_split_pos); + if (prefix_path.length() >= PATH_MAX) { + MS_LOG(ERROR) << "prefix path is too longer!"; + return false; + } + std::string last_path = inpath.substr(path_split_pos, inpath.length() - path_split_pos); + auto ret = CreateNotExistDirs(prefix_path); + if (ret == false) { + MS_LOG(ERROR) << "CreateNotExistDirs Failed!"; + return false; + } + + if (nullptr == realpath(prefix_path.c_str(), real_path)) { + MS_LOG(ERROR) << "dir " << prefix_path << " does not exit."; + return false; + } + *outpath = std::string(real_path) + last_path; + } + + if (path_split_pos == std::string::npos) { + if (inpath.length() >= PATH_MAX) { + MS_LOG(ERROR) << "prefix path is too longer!"; + return false; + } + if (nullptr == realpath(inpath.c_str(), real_path)) { + MS_LOG(ERROR) << "file " << inpath << " does not exit, it will be created."; + } + *outpath = std::string(real_path); + } + + return true; +} + +bool Dump::CreateNotExistDirs(const std::string& path) { + std::shared_ptr fs = system::Env::GetFileSystem(); + MS_EXCEPTION_IF_NULL(fs); + char temp_path[PATH_MAX] = {0}; + if (path.length() > PATH_MAX) { + MS_LOG(ERROR) << "path lens is max than " << PATH_MAX; + return false; + } + for (uint32_t i = 0; i < path.length(); i++) { + temp_path[i] = path[i]; + if (temp_path[i] == '\\' || temp_path[i] == '/') { + if (i != 0) { + char tmp_char = temp_path[i]; + temp_path[i] = '\0'; + std::string path_handle(temp_path); + if (!fs->FileExist(temp_path)) { + MS_LOG(INFO) << "dir " << path_handle << " does not exit, creating..."; + if (!fs->CreateDir(temp_path)) { + MS_LOG(ERROR) << "Create " << path_handle << " dir error"; + return false; + } + } + temp_path[i] = tmp_char; + } + } + } + + if (!fs->FileExist(path)) { + MS_LOG(INFO) << "dir " << path << " does not exit, creating..."; + if (!fs->CreateDir(path)) { + MS_LOG(ERROR) << "Create " << path << " dir error"; + return false; + } + } + + return true; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/e2e_dump.h b/mindspore/ccsrc/debug/e2e_dump.h new file mode 100644 index 00000000000..2410dfb09af --- /dev/null +++ b/mindspore/ccsrc/debug/e2e_dump.h @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_E2E_DUMP_H +#define MINDSPORE_E2E_DUMP_H +#include +#include +#include +#include +#include +#include + +namespace mindspore { +class Dump { + public: + Dump(); + + ~Dump() = default; + + bool dump_enable() const { return dump_enable_; } + + bool trans_flag() const { return trans_flag_; } + + std::string dump_path() const { return dump_path_; } + + std::string dump_net_name() const { return dump_net_name_; } + + uint32_t dump_iter() const { return dump_iter_; } + + void UpdataCurIter() { cur_iter_++; } + + uint32_t cur_iter() const { return cur_iter_; } + + bool IsKernelNeedDump(const std::string& kernel_name); + + bool SetDumpConfFromJsonFile(); + + static bool DumpToFile(const std::string& filename, const void* data, size_t len); + + protected: + bool dump_enable_; + bool trans_flag_; + std::string dump_path_; + std::string dump_net_name_; + uint32_t dump_mode_; + uint32_t dump_iter_; + uint32_t cur_iter_; + std::vector dump_kernels_; + + static bool GetRealPath(const std::string& inpath, std::string* outpath); + + static bool CreateNotExistDirs(const std::string& path); + + private: + bool ParseDumpConfig(const std::string& dump_config_file); + bool IsConfigExist(const nlohmann::json& dumpSettings); + bool IsConfigValid(const nlohmann::json& dumpSettings); +}; + +using DumpConfPtr = std::shared_ptr; +} // namespace mindspore +#endif // MINDSPORE_E2E_DUMP_H diff --git a/mindspore/ccsrc/debug/info.cc b/mindspore/ccsrc/debug/info.cc new file mode 100644 index 00000000000..6f966e335ca --- /dev/null +++ b/mindspore/ccsrc/debug/info.cc @@ -0,0 +1,217 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/info.h" +#include +#include +#include +#include "ir/anf.h" +#include "pipeline/parse/parse.h" +#include "pipeline/parse/python_adapter.h" + +namespace mindspore { +std::string HighLightLine(const std::string& line, int col_begin, int col_end, SourceLineTip tip) { + std::string temp_line = line; + if (col_begin < col_end && col_begin != -1 && col_end <= SizeToInt(temp_line.length()) && + tip != kSourceLineTipDiscard) { + std::string start = temp_line.substr(0, IntToSize(col_begin)); + std::string trimmed = temp_line.substr(IntToSize(col_begin), IntToSize(col_end - col_begin)); + std::string end = temp_line.substr(IntToSize(col_end), IntToSize(SizeToInt(temp_line.length()) - col_end)); + std::stringstream oss; + std::stringstream tip_ss; + std::string start_spaces(start.length(), ' '); + if (tip == kSourceLineTipInLine) { + temp_line = start + "<" + trimmed + ">" + end; + } else if (tip == kSourceLineTipNextLine) { + tip_ss << start_spaces << "^"; + } + oss << temp_line << "\n" << tip_ss.str(); + return oss.str(); + } + return temp_line; +} +// Generate debug information for the location node . +// print the file name, line no and column no, and part of the content +std::string Location::ToString(SourceLineTip tip) { + std::stringstream debug_info_ss; + debug_info_ss << " In file " << file_name_ << "(" << line_ << ")" << std::endl; + if (line_ <= 0) { + return debug_info_ss.str(); + } + + char path[PATH_MAX + 1] = {0x00}; + if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) { + return debug_info_ss.str(); + } + + auto src_path = std::string(path); + std::ifstream file(src_path); + if (!file.is_open()) { + return debug_info_ss.str(); + } + + int line_num = 0; + std::string line; + (void)getline(file, line); + while (line_num != line_ - 1) { + (void)getline(file, line); + line_num++; + } + file.close(); + + debug_info_ss << HighLightLine(line, column_, column_end_, tip) << std::endl; + return debug_info_ss.str(); +} + +void TraceContext::ProcessAttributeFromContext() { + trace_info_ = nullptr; + location_ = nullptr; + func_name_ = ""; + // if there is trace context, get info from previous context + if (!TraceManager::trace_context_stack_.empty()) { + TraceContextPtr top = TraceManager::trace_context_stack_.top(); + trace_info_ = top->trace_info_; + location_ = top->location_; + func_name_ = top->func_name_; + } +} + +DebugInfo::DebugInfo() { + InitValueFromContext(); + unique_id_ = gen_unique_id(); + debug_id_ = -1; + name_ = ""; +} + +DebugInfo::DebugInfo(const std::string& name) { + InitValueFromContext(); + unique_id_ = gen_unique_id(); + debug_id_ = -1; + name_ = name; +} + +DebugInfo::DebugInfo(const LocationPtr& loc) { + InitValueFromContext(); + unique_id_ = gen_unique_id(); + debug_id_ = -1; + location_ = loc; +} + +int64_t DebugInfo::debug_id() { + // cppcheck-suppress variableScope + static int64_t cur_debug_id = 0; + if (debug_id_ == -1) { + debug_id_ = cur_debug_id; + cur_debug_id++; + } + return debug_id_; +} + +int64_t DebugInfo::unique_id_through_copy() const { + TraceInfoPtr trace_info = const_cast(this)->trace_info(); + if (trace_info != nullptr) { + if (trace_info->isa() && trace_info->debug_info() != nullptr) { + return trace_info->debug_info()->unique_id_through_copy(); + } + } + return unique_id(); +} + +std::string DebugInfo::debug_name() { + if (!name_.empty()) { + return name_; + } + std::string debug_name = std::to_string(debug_id()); + name_ = debug_name; + return debug_name; +} + +std::string NodeDebugInfo::debug_name() { + if (!name_.empty()) { + return name_; + } + std::string prefix = ""; + if (node_.lock() != nullptr) { + std::ostringstream oss; + oss << "[" << node_.lock()->type_name() << "]"; + prefix = oss.str(); + } + name_ = prefix + DebugInfo::debug_name(); + return name_; +} + +std::string GraphDebugInfo::debug_name() { + std::string prefix = ""; + return prefix + DebugInfo::debug_name(); +} + +LocationPtr GraphDebugInfo::location() { + // function may have decorator which is included in its location + if (deco_loc_ != nullptr) { + LocationPtr loc = std::make_shared(*DebugInfo::location()); + loc->set_line(loc->line() + (deco_loc_->line_end() - deco_loc_->line() + 1)); + return loc; + } + return DebugInfo::location(); +} +void GraphDebugInfo::set_deco_location(const LocationPtr& deco_list_loc) { deco_loc_ = deco_list_loc; } + +TraceContextPtr TraceManager::CurrentContextInfo() { + if (!TraceManager::trace_context_stack_.empty()) { + return TraceManager::trace_context_stack_.top(); + } + return nullptr; +} + +void TraceManager::DebugTrace(const std::string& func_name, const LocationPtr& location) { + TraceContextPtr context = std::make_shared(location); + context->set_func_name(func_name); + TraceManager::trace_context_stack_.push(context); +} + +void TraceManager::DebugTrace(const LocationPtr& location) { + TraceContextPtr context = std::make_shared(location); + TraceManager::trace_context_stack_.push(context); +} + +void TraceManager::DebugTrace(const TraceInfoPtr& trace_info) { + if (trace_info == nullptr) { + MS_LOG(EXCEPTION) << "DebugTrace wrong traced info is null"; + } + TraceContextPtr context = std::make_shared(trace_info); + if (trace_info->debug_info() == nullptr) { + MS_LOG(EXCEPTION) << "trace debug info is null"; + } + TraceManager::trace_context_stack_.push(context); +} + +void TraceManager::DebugTrace(const DebugInfoPtr& debug_info, const TraceInfoPtr& trace_info) { + if (trace_info == nullptr) { + MS_LOG(EXCEPTION) << "DebugTrace wrong traced info is null"; + } + auto cloned_info = trace_info->clone(); + cloned_info->set_debug_info(debug_info); + if (cloned_info->debug_info() == nullptr) { + MS_LOG(EXCEPTION) << "trace debug info is null with cloned trace"; + } + TraceContextPtr context = std::make_shared(cloned_info); + TraceManager::trace_context_stack_.push(context); +} + +void TraceManager::EndTrace() { TraceManager::trace_context_stack_.pop(); } + +std::stack TraceManager::trace_context_stack_; +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/info.h b/mindspore/ccsrc/debug/info.h new file mode 100644 index 00000000000..da641ab74bb --- /dev/null +++ b/mindspore/ccsrc/debug/info.h @@ -0,0 +1,243 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_INFO_H_ +#define MINDSPORE_CCSRC_IR_INFO_H_ + +#include +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "ir/base.h" +#include "debug/trace_info.h" + +namespace mindspore { +namespace py = pybind11; + +// namespace to support intermediate representation definition +enum SourceLineTip { kSourceLineTipDiscard = 0, kSourceLineTipNextLine = 1, kSourceLineTipInLine = 2 }; + +// Location class record the location in source code. +class Location { + public: + Location(const std::string& file_name, int line, int column, int line_end, int column_end) + : file_name_(file_name), line_(line), column_(column), line_end_(line_end), column_end_(column_end) {} + Location(const Location& loc) + : file_name_(loc.file_name_), + line_(loc.line_), + column_(loc.column_), + line_end_(loc.line_end_), + column_end_(loc.column_end_) {} + std::string ToString(SourceLineTip tip = kSourceLineTipNextLine); + std::string file_name() { return file_name_; } + int line() const { return line_; } + void set_line(int line) { line_ = line; } + int line_end() const { return line_end_; } + void set_line_end(int line) { line_end_ = line; } + int column() const { return column_; } + void set_column(int column) { column_ = column; } + int column_end() const { return column_end_; } + void set_column_end(int column) { column_end_ = column; } + ~Location() = default; + + private: + std::string file_name_; + int line_; + int column_; + int line_end_; + int column_end_; +}; +class TraceContext; +using TraceContextPtr = std::shared_ptr; +class FuncGraph; +using FuncGraphPtr = std::shared_ptr; +using FuncGraphWeakPtr = std::weak_ptr; +class AnfNode; +using AnfNodeWeakPtr = std::weak_ptr; + +class TraceManager { + public: + TraceManager() = default; + ~TraceManager() = default; + static TraceContextPtr CurrentContextInfo(); + static void DebugTrace(const std::string& func_name, const LocationPtr& location); + static void DebugTrace(const LocationPtr& location); + static void DebugTrace(const TraceInfoPtr& trace_info); + // debug trace with a cloned trace info with debug_info + static void DebugTrace(const DebugInfoPtr& debug_info, const TraceInfoPtr& trace_info); + static void EndTrace(); + static std::stack trace_context_stack_; +}; + +class TraceGuard { + public: + explicit TraceGuard(const std::string func_name, const LocationPtr& location) { + TraceManager::DebugTrace(func_name, location); + } + explicit TraceGuard(const LocationPtr& location) { TraceManager::DebugTrace(location); } + ~TraceGuard() { TraceManager::EndTrace(); } +}; + +class TraceContext { + public: + LocationPtr location_; + TraceInfoPtr trace_info_; + std::string func_name_; + + protected: + void ProcessAttributeFromContext(); + + public: + ~TraceContext() = default; + explicit TraceContext(const LocationPtr& loc) { + ProcessAttributeFromContext(); + location_ = loc; + } + explicit TraceContext(const std::string& func_name) { + ProcessAttributeFromContext(); + func_name_ = func_name; + } + explicit TraceContext(const TraceInfoPtr& trace_info) { + ProcessAttributeFromContext(); + trace_info_ = trace_info; + } + void set_location(const LocationPtr& loc) { location_ = loc; } + LocationPtr location() { return location_; } + void set_trace_info(const TraceInfoPtr& trace_info) { trace_info_ = trace_info; } + TraceInfoPtr trace_info() { return trace_info_; } + void set_func_name(const std::string& func_name) { func_name_ = func_name; } + std::string func_name() { return func_name_; } +}; + +class DebugInfo : public Base { + public: + DebugInfo(); + + explicit DebugInfo(const std::string& name); + + explicit DebugInfo(const LocationPtr& loc); + + virtual ~DebugInfo() = default; + MS_DECLARE_PARENT(DebugInfo, Base); + int64_t debug_id(); + int64_t unique_id() const { return unique_id_; } + int64_t unique_id_through_copy() const; + std::string get_id() { return std::to_string(debug_id()); } + + void set_trace_info(const TraceInfoPtr& trace_info) { trace_info_ = trace_info; } + TraceInfoPtr trace_info() { return trace_info_; } + void set_location(const LocationPtr& loc) { location_ = loc; } + virtual LocationPtr location() { return location_; } + std::string name() { return name_; } + void set_name(const std::string& name) { name_ = name; } + virtual std::string debug_name(); + + virtual std::string get_python_func_belonged() { return ""; } + + protected: + template + std::shared_ptr shared_from_base() { + return std::static_pointer_cast(shared_from_this()); + } + + private: + void InitValueFromContext() { + if (TraceManager::CurrentContextInfo() != nullptr) { + auto context_info = TraceManager::CurrentContextInfo(); + trace_info_ = context_info->trace_info(); + location_ = context_info->location(); + } + } + static int64_t gen_unique_id() { + static int64_t cur_unique_id = 0; + return cur_unique_id++; + } + + protected: + int64_t unique_id_; + int64_t debug_id_; + TraceInfoPtr trace_info_; + LocationPtr location_; + std::string name_; +}; + +class NodeDebugInfo : public DebugInfo { + public: + NodeDebugInfo() { + if (TraceManager::CurrentContextInfo() != nullptr) { + auto context_info = TraceManager::CurrentContextInfo(); + py_func_belonged_ = context_info->func_name(); + } + } + explicit NodeDebugInfo(const std::string& name) : DebugInfo(name) { + if (TraceManager::CurrentContextInfo() != nullptr) { + auto context_info = TraceManager::CurrentContextInfo(); + py_func_belonged_ = context_info->func_name(); + } + } + ~NodeDebugInfo() override = default; + + std::string debug_name() override; + void set_node(const std::shared_ptr& node) { node_ = AnfNodeWeakPtr(node); } + std::shared_ptr get_node() const { return node_.lock(); } + void set_py_func_belonged(const std::string& name) { py_func_belonged_ = name; } + std::string get_python_func_belonged() override { return py_func_belonged_; } + AnfNodeWeakPtr node_; + std::string py_func_belonged_; +}; +using NodeDebugInfoPtr = std::shared_ptr; + +class GraphDebugInfo : public DebugInfo { + public: + GraphDebugInfo() { + if (TraceManager::CurrentContextInfo() != nullptr) { + auto context_info = TraceManager::CurrentContextInfo(); + py_func_name_ = context_info->func_name(); + deco_loc_ = nullptr; + } + } + + explicit GraphDebugInfo(const std::string& name) : DebugInfo(name) { + if (TraceManager::CurrentContextInfo() != nullptr) { + auto context_info = TraceManager::CurrentContextInfo(); + py_func_name_ = context_info->func_name(); + deco_loc_ = nullptr; + } + } + ~GraphDebugInfo() override = default; + std::string debug_name() override; + LocationPtr location() override; + LocationPtr deco_location() { return deco_loc_; } + void set_graph(const FuncGraphPtr& func_graph) { func_graph_ = FuncGraphWeakPtr(func_graph); } + FuncGraphPtr get_graph() const { return func_graph_.lock(); } + void set_full_name(const std::string& name) { full_name_ = name; } + std::string get_full_name() { return full_name_; } + void set_deco_location(const LocationPtr& deco_list_loc); + std::string get_python_func_belonged() override { return py_func_name_; } + FuncGraphWeakPtr func_graph_; + LocationPtr deco_loc_; + std::string py_func_name_; + std::string full_name_; +}; + +using GraphDebugInfoPtr = std::shared_ptr; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_INFO_H_ diff --git a/mindspore/ccsrc/debug/label.cc b/mindspore/ccsrc/debug/label.cc new file mode 100644 index 00000000000..794151b952f --- /dev/null +++ b/mindspore/ccsrc/debug/label.cc @@ -0,0 +1,115 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/label.h" +#include +#include +#include + +#include "debug/info.h" +#include "ir/func_graph.h" + +namespace mindspore { +namespace label_manage { +static TraceLabelType trace_type = TraceLabelType::kShortSymbol; +TraceLabelType GetGlobalTraceLabelType() { return trace_type; } +void SetGlobalTraceLabelType(TraceLabelType label_type) { trace_type = label_type; } +struct NameWithTrace { + std::string name; + std::vector trace_labels; +}; +static std::string GetTraceName(const TraceInfoPtr& trace_info, TraceLabelType trace_label) { + switch (trace_label) { + case TraceLabelType::kShortSymbol: + return trace_info->symbol(); + case TraceLabelType::kFullName: + return "_" + trace_info->full_name() + "_"; + default: + return ""; + } +} + +NameWithTrace RootName(const DebugInfoPtr& debug_info, TraceLabelType trace_label) { + NameWithTrace trace_name; + // find debug info after Resolve/ExpandJ/GenMetaFuncGraph, it is a new node + auto temp_info = debug_info; + while (temp_info != nullptr) { + if (temp_info->trace_info() != nullptr) { + if (temp_info->trace_info()->isa() || temp_info->trace_info()->isa() || + temp_info->trace_info()->isa()) { + break; + } + trace_name.trace_labels.push_back(GetTraceName(temp_info->trace_info(), trace_label)); + temp_info = temp_info->trace_info()->debug_info(); + } else { + break; + } + } + if (!temp_info->name().empty()) { + trace_name.name = temp_info->name(); + } else { + trace_name.name = temp_info->debug_name(); + } + return trace_name; +} + +std::string CombineTraceTypes(const std::string& root_name, std::vector trace_labels) { + std::string tags = ""; + for (auto& itr : trace_labels) { + std::string symbol = itr; + tags = tags + symbol; + } + return tags + root_name; +} + +// get the label name of the node debug info +std::string LabelString(const DebugInfoPtr& debug_info, TraceLabelType trace_label) { + NameWithTrace trace_name = RootName(debug_info, trace_label); + return CombineTraceTypes(trace_name.name, trace_name.trace_labels); +} + +std::string CombineUniqueID(const DebugInfoPtr& debug_info) { + auto temp_info = debug_info; + std::string label = ""; + while (temp_info != nullptr) { + if (!temp_info->name().empty()) { + label = label + temp_info->name(); + } else { + // the symbol 'U' is for identification of number + label = label + "U" + std::to_string(temp_info->unique_id()); + } + + if (temp_info->trace_info() != nullptr) { + label = label + "_" + temp_info->trace_info()->full_name() + "_"; + temp_info = temp_info->trace_info()->debug_info(); + } else { + temp_info = nullptr; + } + } + return label; +} + +// get trace with unique id chain +std::string LabelStringUnique(const DebugInfoPtr& debug_info) { return CombineUniqueID(debug_info); } + +std::string Label(const DebugInfoPtr& debug_info, TraceLabelType trace_label) { + if (GetGlobalTraceLabelType() == TraceLabelType::kWithUniqueId) { + return LabelStringUnique(debug_info); + } + return LabelString(debug_info, trace_label); +} +} // namespace label_manage +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/label.h b/mindspore/ccsrc/debug/label.h new file mode 100644 index 00000000000..275f7dea8ac --- /dev/null +++ b/mindspore/ccsrc/debug/label.h @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_LABEL_H_ +#define MINDSPORE_CCSRC_UTILS_LABEL_H_ +#include +#include +#include +#include +#include +#include "utils/any.h" +#include "ir/anf.h" + +namespace mindspore { +namespace label_manage { +enum class TraceLabelType { kShortSymbol, kFullName, kWithUniqueId }; +TraceLabelType GetGlobalTraceLabelType(); +void SetGlobalTraceLabelType(TraceLabelType label_type); +std::string Label(const DebugInfoPtr& debug_info, TraceLabelType trace_type = TraceLabelType::kShortSymbol); +} // namespace label_manage +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_LABEL_H_ diff --git a/mindspore/ccsrc/debug/trace.cc b/mindspore/ccsrc/debug/trace.cc new file mode 100644 index 00000000000..4b0f4c4fb30 --- /dev/null +++ b/mindspore/ccsrc/debug/trace.cc @@ -0,0 +1,293 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/trace.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/meta_func_graph.h" +#include "utils/graph_utils.h" +#include "operator/composite/composite.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_utils.h" +#include "pipeline/static_analysis/evaluator.h" + +namespace mindspore { +// namespace to support debug trace infomation +namespace trace { +std::string GetAbstractStr(const abstract::AbstractBasePtr& abs) { + if (abs == nullptr) { + return "Null Abstract"; + } + auto shape = abs->BuildShape()->cast(); + TypePtr type = abs->BuildType(); + std::ostringstream oss; + if ((shape != nullptr) && (type != nullptr)) { + oss << type->DumpText() << shape->DumpText(); + } else if (type != nullptr) { + oss << type->DumpText(); + } else { + oss << "Undefined"; + } + return oss.str(); +} + +std::vector GetSourceCodeDebugInfoVec(DebugInfoPtr debug_info) { + std::vector debug_with_loc_vec; + while (debug_info != nullptr) { + if (debug_info->location() != nullptr) { + debug_with_loc_vec.push_back(debug_info); + } + if (debug_info->trace_info() != nullptr) { + debug_info = debug_info->trace_info()->debug_info(); + } else { + break; + } + } + return debug_with_loc_vec; +} + +DebugInfoPtr GetSourceCodeDebugInfo(const DebugInfoPtr& info) { + auto debug_with_loc_vec = GetSourceCodeDebugInfoVec(info); + if (debug_with_loc_vec.size() > 0) { + return debug_with_loc_vec[0]; + } else { + return info; + } +} + +std::string GetDebugInfo(const DebugInfoPtr& info, SourceLineTip tip) { + if (info == nullptr) { + return ""; + } + auto src_info = GetSourceCodeDebugInfo(info); + if (src_info->location() != nullptr) { + return src_info->location()->ToString(tip); + } + return ""; +} + +// a trace info identifys a node transform, so we can trace the node transform through +// a link of trace info and debug info +std::string GetInfoWithAction(const std::vector& info_vec, SourceLineTip tip) { + if (info_vec.size() < 1) { + return ""; + } + if (info_vec.size() == 1) { + return info_vec[0]->location()->ToString(tip); + } + std::string traced_info = info_vec[0]->location()->ToString(tip); + for (size_t i = 1; i < info_vec.size(); i++) { + auto action_name = info_vec[i - 1]->trace_info()->GetActionBetweenNode(info_vec[i]); + if (action_name == "") { + break; + } + traced_info = traced_info + action_name + info_vec[i]->location()->ToString(tip); + } + return traced_info; +} + +std::string GetTracedDebugInfo(const DebugInfoPtr& info, SourceLineTip tip) { + if (info == nullptr) { + return ""; + } + auto info_vec = GetSourceCodeDebugInfoVec(info); + if (info_vec.size() == 0) { + return ""; + } else if (info_vec.size() == 1) { + return info_vec[0]->location()->ToString(tip); + } else if (info_vec.size() > 1) { + return GetInfoWithAction(info_vec, tip); + } + return ""; +} + +std::string GetDebugInfo(const DebugInfoPtr& info, const std::string& prefix, SourceLineTip tip) { + std::ostringstream oss; + if (info == nullptr) { + return ""; + } + + auto debug_info = GetTracedDebugInfo(info, tip); + if (tip == kSourceLineTipDiscard) { + std::replace(debug_info.begin(), debug_info.end(), '\r', '/'); + std::replace(debug_info.begin(), debug_info.end(), '\n', '/'); + } + oss << prefix << debug_info; + return oss.str(); +} + +std::string GetGraphParamString(const FuncGraphPtr& graph, abstract::AbstractBasePtrList args_spec_list) { + std::ostringstream oss; + oss << "graph:" << graph->ToString() << " with args["; + auto params = graph->parameters(); + for (size_t i = 0; i < args_spec_list.size(); i++) { + oss << params[i]->ToString() << ":<" << GetAbstractStr(args_spec_list[i]) << ">,"; + } + oss << "]"; + oss << GetDebugInfo(graph->debug_info(), kSourceLineTipDiscard); + return oss.str(); +} + +void DumpInferStack(std::ostringstream& oss) { + auto& infer_stack = GetCurrenGraphInferStack(); + if (infer_stack.empty()) { + return; + } + std::vector> infer_vec; + while (!infer_stack.empty()) { + auto top = infer_stack.top(); + infer_vec.push_back(top); + infer_stack.pop(); + } + std::reverse(infer_vec.begin(), infer_vec.end()); + int index = 0; + for (auto& item : infer_vec) { + auto graph_infer = std::dynamic_pointer_cast(item.first); + if (graph_infer == nullptr) { + MS_LOG(WARNING) << "DumpInferStack failed, got null graph evaluator"; + infer_vec.clear(); + break; + } + auto graph_context = graph_infer->graph_context(); + if (graph_context == nullptr) { + MS_LOG(INFO) << "null context continue"; + continue; + } + auto graph = graph_context->func_graph(); + auto args_spec_list = graph_context->args_spec_list(); + oss << " #" << index++ << " " << GetGraphParamString(graph, args_spec_list); + } +} + +void TraceGraphInfer() { + auto& infer_stack = GetCurrenGraphInferStack(); + std::ostringstream oss; + if (infer_stack.empty()) { + return; + } + MS_LOG(INFO) << "\n*******************************graph evaluate stack**********************************"; + oss << std::endl; + DumpInferStack(oss); + MS_LOG(INFO) << oss.str(); + MS_LOG(INFO) << "\n*************************************************************************************"; +} + +void OutputAnalysisGraphInfo() { + MS_LOG(INFO) << "Output analysis graph begin"; + std::unordered_map index_map; + std::vector tagged_graphs; + + auto& list = GetCNodeDebugStack(); + for (size_t i = 0; i < list.size(); ++i) { + auto& node_cfg = list[i]; + auto fg = node_cfg->context()->func_graph(); + auto node = node_cfg->node(); + auto idx = tagged_graphs.size(); + std::pair item(fg, idx); + if (index_map.insert(item).second) { + tagged_graphs.emplace_back(TaggedGraph(fg, TaggedNodeMap())); + } + tagged_graphs[index_map[fg]].second[node] = i; + } + + ExportIR("analyze_fail.dat", tagged_graphs); + MS_LOG(INFO) << "Output analysis graph *end*"; +} + +void GetInferStackInfo(std::ostringstream& oss) { + MS_LOG(INFO) << "Get graph analysis information begin"; + auto& stack = GetCNodeDebugStack(); + if (stack.empty()) { + MS_LOG(INFO) << "Length of analysis information stack is empty."; + return; + } + + OutputAnalysisGraphInfo(); + oss << "\nThe function call stack:\n"; + + int index = 0; + std::string last_py_func = ""; + for (size_t i = 0; i < stack.size(); ++i) { + auto node_cfg = stack[i]; + + auto cnode = dyn_cast(node_cfg->node()); + if (cnode == nullptr) { + MS_LOG(DEBUG) << "CNode of elements[" << i << "] is nullptr."; + continue; + } + + auto debug_info = cnode->debug_info(); + auto this_py_func = debug_info->get_python_func_belonged(); + if (i > 0 && (this_py_func == last_py_func)) { + MS_LOG(DEBUG) << "Python function of elements[" << i << "] is same as previous."; + continue; + } + last_py_func = this_py_func; + oss << "# " << index++ << " " << trace::GetDebugInfo(debug_info, std::string("")); + } + + stack.clear(); + MS_LOG(INFO) << "Get graph analysis information *end*"; +} + +// trace the graph evaluator statck +static std::stack> graph_infer_stack; +// trace the cnode infer debug info +static std::vector cnode_debug_stack{}; +void TraceGraphInferEnter(const abstract::EvaluatorPtr& eval, const abstract::AnfNodeConfigPtr& node) { + if (eval == nullptr) { + MS_LOG(EXCEPTION) << "GraphInferEnter got null eval"; + } + if (eval->isa() || eval->isa()) { + graph_infer_stack.emplace(std::pair(eval, node)); + } +} + +void TraceGraphInferLeave(const abstract::EvaluatorPtr& eval) { + if (eval == nullptr) { + MS_LOG(EXCEPTION) << "GraphInferEnter got null eval"; + } + if (eval->isa() || eval->isa()) { + graph_infer_stack.pop(); + } +} + +void TraceInferCNodeEnter(const abstract::AnfNodeConfigPtr& node_cfg) { cnode_debug_stack.push_back(node_cfg); } + +void TraceInferCNodeLeave() { cnode_debug_stack.pop_back(); } + +std::vector& GetCNodeDebugStack() { return cnode_debug_stack; } + +std::stack>& GetCurrenGraphInferStack() { + return graph_infer_stack; +} +void ClearTraceStack() { + while (!graph_infer_stack.empty()) { + graph_infer_stack.pop(); + } + cnode_debug_stack.clear(); +} +} // namespace trace +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/trace.h b/mindspore/ccsrc/debug/trace.h new file mode 100644 index 00000000000..5fba86fddd1 --- /dev/null +++ b/mindspore/ccsrc/debug/trace.h @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_TRACE_H_ +#define MINDSPORE_CCSRC_DEBUG_TRACE_H_ + +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "utils/any.h" +#include "ir/func_graph.h" +#include "debug/info.h" +#include "pipeline/static_analysis/static_analysis.h" + +namespace mindspore { +namespace trace { +std::string GetDebugInfo(const DebugInfoPtr& info, SourceLineTip tip = kSourceLineTipNextLine); +std::string GetDebugInfo(const DebugInfoPtr& info, const std::string& prefix, + SourceLineTip tip = kSourceLineTipNextLine); +DebugInfoPtr GetSourceCodeDebugInfo(const DebugInfoPtr& info); +void TraceGraphInfer(); +void GetInferStackInfo(std::ostringstream& oss); +void TraceGraphInferEnter(const abstract::EvaluatorPtr& eval, const abstract::AnfNodeConfigPtr& node); +void TraceGraphInferLeave(const abstract::EvaluatorPtr& eval); +void TraceInferCNodeEnter(const abstract::AnfNodeConfigPtr& node_cfg); +void TraceInferCNodeLeave(); +std::vector& GetCNodeDebugStack(); +std::stack>& GetCurrenGraphInferStack(); +std::string GetAbstractStr(const abstract::AbstractBasePtr& abs); +void ClearTraceStack(); +} // namespace trace +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_TRACE_H_ diff --git a/mindspore/ccsrc/debug/trace_info.cc b/mindspore/ccsrc/debug/trace_info.cc new file mode 100644 index 00000000000..e2da4ffcce9 --- /dev/null +++ b/mindspore/ccsrc/debug/trace_info.cc @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "debug/trace_info.h" +#include +#include +#include +#include "ir/anf.h" +#include "pipeline/parse/parse.h" +#include "pipeline/parse/python_adapter.h" + +namespace mindspore { +std::string TraceInfo::GetActionBetweenNode(const DebugInfoPtr& info) { + if (info == nullptr) { + return ""; + } + std::string act_name = action_name(); + if (debug_info() == nullptr) { + MS_LOG(EXCEPTION) << "Traced debug info is null"; + } + if (debug_info() == info) { + return act_name; + } else if (debug_info()->trace_info() != nullptr) { + return act_name + debug_info()->trace_info()->GetActionBetweenNode(info); + } + return "not in the traced info"; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/trace_info.h b/mindspore/ccsrc/debug/trace_info.h new file mode 100644 index 00000000000..16be9031e2c --- /dev/null +++ b/mindspore/ccsrc/debug/trace_info.h @@ -0,0 +1,402 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEBUG_TRACE_INFO_H_ +#define MINDSPORE_CCSRC_DEBUG_TRACE_INFO_H_ + +#include +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "ir/base.h" + +namespace mindspore { +namespace py = pybind11; + +class TraceInfo; +using TraceInfoPtr = std::shared_ptr; +class Location; +using LocationPtr = std::shared_ptr; +class DebugInfo; +using DebugInfoPtr = std::shared_ptr; + +// namespace to support intermediate representation definition +class TraceInfo : public Base { + public: + TraceInfo(const DebugInfoPtr& info, const std::string& full_name, const std::string& symbol) { + symbol_ = symbol; + full_name_ = full_name; + name_ = full_name_; + debug_info_ = info; + } + TraceInfo(const TraceInfo& info) + : Base(), debug_info_(info.debug_info_), symbol_(info.symbol_), full_name_(info.full_name_), name_(info.name_) {} + virtual ~TraceInfo() = default; + MS_DECLARE_PARENT(TraceInfo, Base); + virtual std::string name() { return name_; } + virtual std::string symbol() { return symbol_; } + virtual std::string full_name() { return full_name_; } + virtual TraceInfoPtr clone() { return shared_from_base(); } + virtual std::string action_name() { return ""; } + virtual std::string GetActionBetweenNode(const DebugInfoPtr& info); + void set_debug_info(const DebugInfoPtr& info) { debug_info_ = info; } + DebugInfoPtr debug_info() { return debug_info_; } + DebugInfoPtr DebugInfoHasLoc(); + std::vector> GetSourceCodeDebugInfo(); + + protected: + DebugInfoPtr debug_info_; + std::string symbol_; + std::string full_name_; + std::string name_; +}; + +class TracePhi : public TraceInfo { + public: + explicit TracePhi(const DebugInfoPtr& info) : TraceInfo(info, "phi", "Φ") {} + MS_DECLARE_PARENT(TracePhi, TraceInfo); + ~TracePhi() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceIfStmtTrueBranch : public TraceInfo { + public: + TraceIfStmtTrueBranch(const TraceIfStmtTrueBranch&) = default; + explicit TraceIfStmtTrueBranch(const DebugInfoPtr& info) : TraceInfo(info, "if_true", "✓") {} + MS_DECLARE_PARENT(TraceIfStmtTrueBranch, TraceInfo); + ~TraceIfStmtTrueBranch() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceIfStmtFalseBranch : public TraceInfo { + public: + TraceIfStmtFalseBranch(const TraceIfStmtFalseBranch&) = default; + explicit TraceIfStmtFalseBranch(const DebugInfoPtr& info) : TraceInfo(info, "if_false", "✗") {} + MS_DECLARE_PARENT(TraceIfStmtFalseBranch, TraceInfo); + ~TraceIfStmtFalseBranch() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceIfStmtAfterBranch : public TraceInfo { + public: + explicit TraceIfStmtAfterBranch(const DebugInfoPtr& info) : TraceInfo(info, "if_after", "↓") {} + MS_DECLARE_PARENT(TraceIfStmtAfterBranch, TraceInfo); + ~TraceIfStmtAfterBranch() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceIfExpTrueBranch : public TraceInfo { + public: + explicit TraceIfExpTrueBranch(const DebugInfoPtr& info) : TraceInfo(info, "ifexp_true", "↰") {} + MS_DECLARE_PARENT(TraceIfExpTrueBranch, TraceInfo); + ~TraceIfExpTrueBranch() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceIfExpFalseBranch : public TraceInfo { + public: + explicit TraceIfExpFalseBranch(const DebugInfoPtr& info) : TraceInfo(info, "ifexp_false", "↱") {} + MS_DECLARE_PARENT(TraceIfExpFalseBranch, TraceInfo); + ~TraceIfExpFalseBranch() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceCopy : public TraceInfo { + public: + TraceCopy() : TraceInfo(nullptr, "copy", "") {} + explicit TraceCopy(const DebugInfoPtr& info) : TraceInfo(info, "copy", "") {} + MS_DECLARE_PARENT(TraceCopy, TraceInfo); + ~TraceCopy() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceIterator : public TraceInfo { + public: + explicit TraceIterator(const DebugInfoPtr& info) : TraceInfo(info, "iterator", "@") {} + MS_DECLARE_PARENT(TraceIterator, TraceInfo); + ~TraceIterator() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceWhileHeader : public TraceInfo { + public: + explicit TraceWhileHeader(const DebugInfoPtr& info) : TraceInfo(info, "while_header", "⤾") {} + MS_DECLARE_PARENT(TraceWhileHeader, TraceInfo); + ~TraceWhileHeader() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceWhileBody : public TraceInfo { + public: + explicit TraceWhileBody(const DebugInfoPtr& info) : TraceInfo(info, "while_body", "⥁") {} + MS_DECLARE_PARENT(TraceWhileBody, TraceInfo); + ~TraceWhileBody() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceWhileAfter : public TraceInfo { + public: + explicit TraceWhileAfter(const DebugInfoPtr& info) : TraceInfo(info, "while_after", "↓") {} + MS_DECLARE_PARENT(TraceWhileAfter, TraceInfo); + ~TraceWhileAfter() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceForHeader : public TraceInfo { + public: + explicit TraceForHeader(const DebugInfoPtr& info) : TraceInfo(info, "for_header", "⤾") {} + MS_DECLARE_PARENT(TraceForHeader, TraceInfo); + ~TraceForHeader() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceForBody : public TraceInfo { + public: + explicit TraceForBody(const DebugInfoPtr& info) : TraceInfo(info, "for_body", "⥁") {} + MS_DECLARE_PARENT(TraceForBody, TraceInfo); + ~TraceForBody() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceForAfter : public TraceInfo { + public: + explicit TraceForAfter(const DebugInfoPtr& info) : TraceInfo(info, "for_after", "↓") {} + MS_DECLARE_PARENT(TraceForAfter, TraceInfo); + ~TraceForAfter() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceEquiv : public TraceInfo { + public: + explicit TraceEquiv(const DebugInfoPtr& info) : TraceInfo(info, "equiv", "equiv") {} + MS_DECLARE_PARENT(TraceEquiv, TraceInfo); + ~TraceEquiv() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceGradFpropApp : public TraceInfo { + public: + TraceGradFpropApp() : TraceInfo(nullptr, "grad_fprop_app", "▲") {} + explicit TraceGradFpropApp(const DebugInfoPtr& info) : TraceInfo(info, "grad_fprop_app", "▲") {} + MS_DECLARE_PARENT(TraceGradFpropApp, TraceInfo); + ~TraceGradFpropApp() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceGradBpropApp : public TraceInfo { + public: + TraceGradBpropApp() : TraceInfo(nullptr, "grad_bprop_app", "▼") {} + explicit TraceGradBpropApp(const DebugInfoPtr& info) : TraceInfo(info, "grad_bprop_app", "▼") {} + MS_DECLARE_PARENT(TraceGradBpropApp, TraceInfo); + ~TraceGradBpropApp() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceGradFprop : public TraceInfo { + public: + TraceGradFprop() : TraceInfo(nullptr, "grad_fprop", "▶") {} + explicit TraceGradFprop(const DebugInfoPtr& info) : TraceInfo(info, "grad_fprop", "▶") {} + MS_DECLARE_PARENT(TraceGradFprop, TraceInfo); + ~TraceGradFprop() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceGradBprop : public TraceInfo { + public: + TraceGradBprop() : TraceInfo(nullptr, "grad_bprop", "◀") {} + explicit TraceGradBprop(const DebugInfoPtr& info) : TraceInfo(info, "grad_bprop", "◀") {} + MS_DECLARE_PARENT(TraceGradBprop, TraceInfo); + ~TraceGradBprop() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceGradSens : public TraceInfo { + public: + TraceGradSens() : TraceInfo(nullptr, "grad_sens", "∇") {} + explicit TraceGradSens(const DebugInfoPtr& info) : TraceInfo(info, "grad_sens", "∇") {} + MS_DECLARE_PARENT(TraceGradSens, TraceInfo); + ~TraceGradSens() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceSpecialize : public TraceInfo { + public: + explicit TraceSpecialize(const std::string& counter) : TraceInfo(nullptr, "specialize", "") { counter_ = counter; } + MS_DECLARE_PARENT(TraceSpecialize, TraceInfo); + std::string name() override { return full_name_ + counter_; } + std::string symbol() override { return counter_ + "_"; } + std::string full_name() override { return full_name_ + counter_ + "_"; } + ~TraceSpecialize() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } + std::string counter_; +}; + +class TraceGradOperation : public TraceInfo { + public: + explicit TraceGradOperation(const DebugInfoPtr& info) : TraceInfo(info, "grad_ops", "") {} + MS_DECLARE_PARENT(TraceGradOperation, TraceInfo); + ~TraceGradOperation() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceForceBool : public TraceInfo { + public: + explicit TraceForceBool(const DebugInfoPtr& info) : TraceInfo(info, "force_bool", "") {} + MS_DECLARE_PARENT(TraceForceBool, TraceInfo); + ~TraceForceBool() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceExpandJ : public TraceInfo { + public: + explicit TraceExpandJ(const DebugInfoPtr& info) : TraceInfo(info, "expand_j", "") {} + MS_DECLARE_PARENT(TraceExpandJ, TraceInfo); + ~TraceExpandJ() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceGenMetaFuncGraph : public TraceInfo { + public: + explicit TraceGenMetaFuncGraph(const DebugInfoPtr& info) : TraceInfo(info, "GenMetaFuncGraph", "") {} + MS_DECLARE_PARENT(TraceGenMetaFuncGraph, TraceInfo); + ~TraceGenMetaFuncGraph() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceEvaluatorGenGraph : public TraceInfo { + public: + explicit TraceEvaluatorGenGraph(const DebugInfoPtr& info) : TraceInfo(info, "GenEvaluatorGraph", "") {} + MS_DECLARE_PARENT(TraceEvaluatorGenGraph, TraceInfo); + ~TraceEvaluatorGenGraph() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceResolve : public TraceInfo { + public: + explicit TraceResolve(const DebugInfoPtr& info) : TraceInfo(info, "resolve", "") {} + MS_DECLARE_PARENT(TraceResolve, TraceInfo); + ~TraceResolve() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceTransform : public TraceInfo { + public: + TraceTransform() : TraceInfo(nullptr, "transform", "") { transform_name_ = ""; } + explicit TraceTransform(const std::string& transform_name) : TraceInfo(nullptr, "transform", "") { + transform_name_ = transform_name; + } + + std::string full_name() override { return full_name_ + transform_name_; } + MS_DECLARE_PARENT(TraceTransform, TraceInfo); + virtual std::string symbol() { + if (transform_name_.empty()) { + return ""; + } + return transform_name_ + "_"; + } + + ~TraceTransform() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } + std::string transform_name_; +}; + +class TraceGenerateVarArg : public TraceInfo { + public: + explicit TraceGenerateVarArg(const DebugInfoPtr& info) : TraceInfo(info, "GenerateVarArg", "") {} + MS_DECLARE_PARENT(TraceGenerateVarArg, TraceInfo); + ~TraceGenerateVarArg() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceGenerateKwArg : public TraceInfo { + public: + explicit TraceGenerateKwArg(const DebugInfoPtr& info) : TraceInfo(info, "GenerateKwArg", "") {} + MS_DECLARE_PARENT(TraceGenerateKwArg, TraceInfo); + ~TraceGenerateKwArg() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceTrasformK : public TraceInfo { + public: + explicit TraceTrasformK(const DebugInfoPtr& info) : TraceInfo(info, "TraceTrasformK", "") {} + MS_DECLARE_PARENT(TraceTrasformK, TraceInfo); + ~TraceTrasformK() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TracePartialTransform : public TraceInfo { + public: + explicit TracePartialTransform(const DebugInfoPtr& info) : TraceInfo(info, "PartialTransform", "") {} + MS_DECLARE_PARENT(TracePartialTransform, TraceInfo); + ~TracePartialTransform() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; + +class TraceGetEnv : public TraceInfo { + public: + explicit TraceGetEnv(const DebugInfoPtr& info) : TraceInfo(info, "get_env", "") {} + MS_DECLARE_PARENT(TraceGetEnv, TraceInfo); + ~TraceGetEnv() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceDoSignature : public TraceInfo { + public: + explicit TraceDoSignature(const DebugInfoPtr& info) : TraceInfo(info, "DoSignature", "") {} + MS_DECLARE_PARENT(TraceDoSignature, TraceInfo); + ~TraceDoSignature() override = default; + TraceInfoPtr clone() override { return std::make_shared(*shared_from_base()); } +}; + +class TraceCombileLikeGraphs : public TraceInfo { + public: + TraceCombileLikeGraphs() : TraceInfo(nullptr, "CombileLike", "L-") {} + explicit TraceCombileLikeGraphs(const DebugInfoPtr& info) : TraceInfo(info, "CombileLike", "L-") {} + MS_DECLARE_PARENT(TraceCombileLikeGraphs, TraceInfo); + ~TraceCombileLikeGraphs() override = default; + TraceInfoPtr clone() override { + return std::make_shared(*shared_from_base()); + } +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_TRACE_INFO_H_ diff --git a/mindspore/ccsrc/device/CMakeLists.txt b/mindspore/ccsrc/device/CMakeLists.txt new file mode 100644 index 00000000000..93ef7adc840 --- /dev/null +++ b/mindspore/ccsrc/device/CMakeLists.txt @@ -0,0 +1,34 @@ +file(GLOB_RECURSE _DEVICE_ALL_SRC_FILES *.cc) +add_library(_mindspore_device_obj OBJECT ${_DEVICE_ALL_SRC_FILES}) + +if(ENABLE_CPU) + target_compile_definitions(_mindspore_device_obj PRIVATE CPUSESSION) + file(GLOB_RECURSE _CPU_SRC_LIST cpu/*.cc) + add_library(_c_expression_cpu_device_obj OBJECT ${_CPU_SRC_LIST}) +endif() + +if(ENABLE_GPU) + file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "gpu/*.cc" + "gpu/*.cu" + ) + list(REMOVE_ITEM _GPU_SRC_LIST "gpu/blocking_queue.cc" + "gpu/gpu_buffer_mgr.cc" + "gpu/mpi/mpi_initializer.cc" + "gpu/distribution/collective_wrapper.cc" + "gpu/distribution/mpi_wrapper.cc" + "gpu/distribution/nccl_wrapper.cc") + add_library(_cuda_gpu_device_obj OBJECT ${_GPU_SRC_LIST}) +endif() + +if(ENABLE_D) + file(GLOB_RECURSE _D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "ascend/*.cc" + "ascend/profiling/*.cc" + "ascend/tasksink/*.cc" + "kernel_adjust.cc" + "ascend/tasksink/taskinfo/*.cc" + ) + target_sources(_mindspore_device_obj PRIVATE ${_D_SRC_LIST}) +endif() + diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.cc b/mindspore/ccsrc/device/ascend/ascend_device_address.cc new file mode 100644 index 00000000000..a521f1516f9 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.cc @@ -0,0 +1,316 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/ascend/ascend_device_address.h" +#include +#include +#include +#include "runtime/mem.h" +#include "device/kernel_runtime_manager.h" +#include "device/convert_tensor_utils.h" +#include "ir/dtype/type.h" +#include "ir/meta_tensor.h" +#include "kernel/common_utils.h" +#include "utils/utils.h" +#include "common/utils.h" +#include "common/trans.h" +#ifdef ENABLE_DUMP_E2E +#include "debug/e2e_dump.h" +#endif +namespace mindspore { +namespace device { +namespace ascend { +const int FLOAT_LEN = sizeof(float); +const int FLOAT16_LEN = 2; // sizeof(float16); +void SyncMemory(void *dst, const void *src, uint64_t size, rtMemcpyKind_t kind) { + auto ret_rt_memcpy = rtMemcpy(dst, size, src, size, kind); + if (ret_rt_memcpy != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtMemcpy failed"; + } +} + +bool FloatToHalfAndSyncHostToDevice(void *dst, size_t dst_size, const void *src, size_t src_size) { + auto elem_num = src_size / FLOAT_LEN; + if (elem_num != (dst_size / FLOAT16_LEN)) { + MS_EXCEPTION(ArgumentError) << "FloatToHalf failed. size not match src_size[" << src_size << "], dst_size[" + << dst_size << "]"; + } + std::vector half_data(elem_num); + FloatToHalf(half_data.data(), src, elem_num); + SyncMemory(dst, half_data.data(), dst_size, RT_MEMCPY_HOST_TO_DEVICE); + return true; +} + +bool Float64ToFloatAndSyncHostToDevice(void *dst, size_t dst_size, const void *src, size_t src_size) { + if (src_size / 2 != dst_size) { + MS_EXCEPTION(ArgumentError) << "src_size[" << src_size << "], dst_size[" << dst_size << "]"; + } + size_t elem_num = dst_size / sizeof(float); + auto host_tmp = std::vector(elem_num); + DoubleToFloat(host_tmp.data(), src, elem_num); + SyncMemory(dst, host_tmp.data(), dst_size, RT_MEMCPY_HOST_TO_DEVICE); + return true; +} + +bool SyncDeviceToHostAndHalfToFloat(void *dst, size_t dst_size, const void *src, size_t src_size) { + auto elem_num = src_size / FLOAT16_LEN; + if (elem_num != (dst_size / FLOAT_LEN)) { + MS_EXCEPTION(ArgumentError) << "HalfToFloat failed. size not match src_size[" << src_size << "], dst_size[" + << dst_size << "]"; + } + std::vector half_data(elem_num); + SyncMemory(half_data.data(), src, src_size, RT_MEMCPY_DEVICE_TO_HOST); + HalfToFloat(dst, half_data.data(), elem_num); + return true; +} + +bool SyncDeviceToHostAndFloatToFloat64(void *dst, size_t dst_size, const void *src, size_t src_size) { + if (src_size != dst_size / 2) { + MS_EXCEPTION(ArgumentError) << "src_size[" << src_size << "], dst_size[" << dst_size << "]"; + } + size_t elem_num = src_size / sizeof(float); + auto host_tmp = std::vector(elem_num); + SyncMemory(host_tmp.data(), src, src_size, RT_MEMCPY_DEVICE_TO_HOST); + FloatToDouble(dst, host_tmp.data(), elem_num); + return true; +} + +bool AscendDeviceAddress::SyncDeviceToHost(const std::vector &shape, size_t size, mindspore::TypeId type, + void *host_ptr) const { + MS_LOG(INFO) << "SyncDeviceToHost, Device(format:" << format_ << ", type_id:" << TypeIdLabel(type_id_) + << ", size:" << size_ << "), Host(type_id:" << TypeIdLabel(type) << ", size:" << size << ")"; + bool sync_ok = false; + std::vector host_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(host_shape), IntToSize); + if (host_shape.empty()) { + host_shape.emplace_back(1); + } + if (format_ == kOpFormat_NCHW || format_ == kOpFormat_DEFAULT) { + if (type_id_ == type) { + SyncMemory(host_ptr, ptr_, size, RT_MEMCPY_DEVICE_TO_HOST); + sync_ok = true; + } else if (type_id_ == kNumberTypeFloat32 && type == kNumberTypeFloat64) { + sync_ok = SyncDeviceToHostAndFloatToFloat64(host_ptr, size, ptr_, size_); + } else { + auto shape_size = trans::ShapeSize(host_shape); + auto host = std::vector(size_); + const trans::TypeIdArgs type_args{ptr_, shape_size, type_id_, type}; + sync_ok = trans::TransDataType(type_args, host.data()); + if (!sync_ok) { + MS_LOG(ERROR) << "trans data type failed."; + return false; + } + SyncMemory(host_ptr, host.data(), size, RT_MEMCPY_DEVICE_TO_HOST); + } + } else if (format_ == kOpFormat_NC1HWC0 || format_ == kOpFormat_FRAC_Z || format_ == kOpFormat_FRAC_NZ) { + sync_ok = SyncDeviceToHostAndConvertFormat(shape, size, type, host_ptr); + } + if (!sync_ok) { + MS_LOG(ERROR) << "Not support to trans, dev_format:" << format_ << ", dev_type:" << TypeIdLabel(type_id_) + << ", host_type:" << TypeIdLabel(type); + return false; + } + return sync_ok; +} + +bool AscendDeviceAddress::SyncDeviceToHostAndConvertFormat(const std::vector &shape, size_t size, + mindspore::TypeId type, void *host_ptr) const { + MS_LOG(INFO) << "SyncDeviceToHostAndConvertFormat, Device(format:" << format_ << ", type_id:" << TypeIdLabel(type_id_) + << ", size:" << size_ << "), Host(type_id:" << TypeIdLabel(type) << ", size:" << size << ")"; + bool sync_ok = false; + auto host_tmp = std::vector(size_); + SyncMemory(host_tmp.data(), ptr_, size_, RT_MEMCPY_DEVICE_TO_HOST); + std::vector host_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(host_shape), IntToSize); + std::vector device_shape; + if (host_shape.empty()) { + host_shape.emplace_back(1); + } + if (format_ == kOpFormat_FRAC_NZ) { + device_shape = trans::TransShapeToDevice(host_shape, format_); + } else { + host_shape = trans::TransShapeTo4d(host_shape); + device_shape = trans::TransShapeToDevice(host_shape, format_); + } + if (type_id_ != type) { + const trans::FormatArgs format_args{host_tmp.data(), size_, kOpFormat_NCHW, format_, + host_shape, device_shape, type_id_}; + auto host = std::vector(size_); + sync_ok = trans::TransFormatFromDeviceToHost(format_args, host.data()); + if (!sync_ok) { + MS_LOG(ERROR) << "trans format failed."; + return false; + } + auto shape_size = trans::ShapeSize(host_shape); + const trans::TypeIdArgs type_args{host.data(), shape_size, type_id_, type}; + sync_ok = trans::TransDataType(type_args, host_ptr); + if (!sync_ok) { + MS_LOG(ERROR) << "trans format failed."; + return false; + } + } else { + const trans::FormatArgs format_args{host_tmp.data(), size_, kOpFormat_NCHW, format_, + host_shape, device_shape, type_id_}; + sync_ok = trans::TransFormatFromDeviceToHost(format_args, host_ptr); + if (!sync_ok) { + MS_LOG(ERROR) << "trans format failed."; + return false; + } + } + return sync_ok; +} + +bool AscendDeviceAddress::SyncHostToDevice(const std::vector &shape, size_t size, mindspore::TypeId type, + const void *host_ptr) const { + MS_LOG(INFO) << "SyncHostToDevice, Device(format:" << format_ << ", type_id:" << TypeIdLabel(type_id_) + << ", size:" << size_ << "), Host(type_id:" << TypeIdLabel(type) << ", size:" << size << ")"; + bool sync_ok = false; + std::vector host_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(host_shape), IntToSize); + if (host_shape.empty()) { + host_shape.emplace_back(1); + } + if (format_ == kOpFormat_NCHW || format_ == kOpFormat_DEFAULT) { + if (type_id_ == type) { + SyncMemory(ptr_, host_ptr, size_, RT_MEMCPY_HOST_TO_DEVICE); + sync_ok = true; + } else if (type_id_ == kNumberTypeFloat32 && type == kNumberTypeFloat64) { + sync_ok = Float64ToFloatAndSyncHostToDevice(ptr_, size_, host_ptr, size); + } else { + auto shape_size = trans::ShapeSize(host_shape); + const trans::TypeIdArgs type_args{host_ptr, shape_size, type, type_id_}; + auto host_tmp = std::vector(size_); + sync_ok = trans::TransDataType(type_args, host_tmp.data()); + if (!sync_ok) { + MS_LOG(ERROR) << "trans data type failed."; + return false; + } + SyncMemory(ptr_, host_tmp.data(), size_, RT_MEMCPY_HOST_TO_DEVICE); + } + } else if (format_ == kOpFormat_NC1HWC0 || format_ == kOpFormat_FRAC_Z || format_ == kOpFormat_FRAC_NZ) { + sync_ok = ConvertFormatAndSyncHostToDevice(shape, size, type, host_ptr); + } + if (!sync_ok) { + MS_LOG(ERROR) << "Not support to trans, dev_format:" << format_ << ", dev_type:" << TypeIdLabel(type_id_) + << ", host_type:" << TypeIdLabel(type); + return false; + } + return sync_ok; +} + +bool AscendDeviceAddress::ConvertFormatAndSyncHostToDevice(const std::vector &shape, size_t size, + mindspore::TypeId type, const void *host_ptr) const { + bool sync_ok = false; + MS_LOG(INFO) << "ConvertFormatAndSyncHostToDevice, Device(format:" << format_ << ", type_id:" << TypeIdLabel(type_id_) + << ", size:" << size_ << "), Host(type_id:" << TypeIdLabel(type) << ", size:" << size << ")"; + std::vector host_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(host_shape), IntToSize); + if (host_shape.empty()) { + host_shape.emplace_back(1); + } + std::vector device_shape; + if (format_ == kOpFormat_FRAC_NZ) { + device_shape = trans::TransShapeToDevice(host_shape, format_); + } else { + host_shape = trans::TransShapeTo4d(host_shape); + device_shape = trans::TransShapeToDevice(host_shape, format_); + } + if (type_id_ != type) { + auto shape_size = trans::ShapeSize(host_shape); + const trans::TypeIdArgs type_args{host_ptr, shape_size, type, type_id_}; + auto host_tmp = std::vector(size_); + sync_ok = trans::TransDataType(type_args, host_tmp.data()); + if (!sync_ok) { + MS_LOG(ERROR) << "trans datatype failed."; + return false; + } + const trans::FormatArgs format_args{host_tmp.data(), size_, kOpFormat_NCHW, format_, + host_shape, device_shape, type_id_}; + auto dst_tmp = std::vector(size_); + sync_ok = trans::TransFormat(format_args, dst_tmp.data()); + if (!sync_ok) { + MS_LOG(ERROR) << "trans format failed."; + return false; + } + SyncMemory(ptr_, dst_tmp.data(), size_, RT_MEMCPY_HOST_TO_DEVICE); + } else { + const trans::FormatArgs format_args{host_ptr, size_, kOpFormat_NCHW, format_, host_shape, device_shape, type_id_}; + auto host_tmp = std::vector(size_); + sync_ok = trans::TransFormat(format_args, host_tmp.data()); + if (!sync_ok) { + MS_LOG(ERROR) << "trans format failed."; + return false; + } + SyncMemory(ptr_, host_tmp.data(), size_, RT_MEMCPY_HOST_TO_DEVICE); + } + return sync_ok; +} + +AscendDeviceAddress::~AscendDeviceAddress() { + if (ptr_ == nullptr) { + return; + } + if (mem_dynamic_alloc_) { + AscendMemoryAllocator::GetInstance().FreeTensorMem(ptr_); + ptr_ = nullptr; + } +} + +#ifdef ENABLE_DUMP_E2E +bool AscendDeviceAddress::DumpMemToFile(bool trans_flag, const std::string &filepath, const std::string &host_fmt, + const std::vector &host_shape, TypeId host_type) const { + bool ret = false; + if (filepath.empty()) { + MS_LOG(ERROR) << "Dump file path is null!"; + return ret; + } + std::string shape = "shape"; + if (host_shape.size()) { + for (auto &value : host_shape) { + shape = shape + '_' + std::to_string(value); + } + } else { + shape = shape + "_0"; + } + std::string file_extension = ".bin"; + if (trans_flag) { + std::string path = filepath + '_' + shape + '_' + TypeIdLabel(host_type) + '_' + host_fmt + file_extension; + MS_LOG(INFO) << "E2E Dump path is " << path; + mindspore::tensor::TensorPtr out_tensor = std::make_shared(host_type, host_shape); + size_t host_size = out_tensor->data().nbytes(); + ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c(true)); + if (!ret) { + MS_LOG(ERROR) << "Copy device mem to host failed"; + return ret; + } + ret = mindspore::Dump::DumpToFile(path, out_tensor->data_c(false), host_size); + } else { + auto host_tmp = std::vector(size_); + auto ret_rt_memcpy = rtMemcpy(host_tmp.data(), size_, ptr_, size_, RT_MEMCPY_DEVICE_TO_HOST); + if (ret_rt_memcpy != RT_ERROR_NONE) { + MS_LOG(ERROR) << "SyncDeviceToHost: rtMemcpy mem size[" << size_ << "] fail, ret[" << ret_rt_memcpy << "]"; + } + std::string path = + filepath + '_' + shape + '_' + TypeIdToType(type_id_)->ToString() + '_' + format_ + file_extension; + MS_LOG(INFO) << "E2E Dump path is " << path; + ret = mindspore::Dump::DumpToFile(path, host_tmp.data(), size_); + } + + return ret; +} +#endif +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.h b/mindspore/ccsrc/device/ascend/ascend_device_address.h new file mode 100644 index 00000000000..60cc64cca7c --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.h @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_DEVICE_ADDRESS_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_DEVICE_ADDRESS_H_ + +#include +#include +#include +#include "device/device_address.h" +#include "device/ascend/ascend_memory_allocator.h" +#include "ir/dtype.h" + +namespace mindspore { +namespace device { +namespace ascend { +class AscendDeviceAddress : public DeviceAddress { + public: + explicit AscendDeviceAddress(void *ptr, size_t size) : DeviceAddress(ptr, size) {} + explicit AscendDeviceAddress(void *ptr, size_t size, const std::string &format, TypeId type_id) + : DeviceAddress(ptr, size, format, type_id) {} + ~AscendDeviceAddress() override; + bool SyncDeviceToHost(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const override; + bool SyncHostToDevice(const std::vector &shape, size_t size, TypeId type, const void *host_ptr) const override; +#ifdef ENABLE_DUMP_E2E + bool DumpMemToFile(bool dump_mode, const std::string &filepath, const std::string &host_fmt, + const std::vector &host_shape, TypeId host_type) const; +#endif + private: + bool SyncDeviceToHostAndConvertFormat(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const; + bool ConvertFormatAndSyncHostToDevice(const std::vector &shape, size_t size, TypeId type, + const void *host_ptr) const; +}; +using AscendDeviceAddressPtr = std::shared_ptr; +} // namespace ascend +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_DEVICE_ADDRESS_H_ diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc new file mode 100644 index 00000000000..7be480e5e37 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -0,0 +1,531 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/ascend_kernel_runtime.h" + +#include +#include +#include +#include +#include +#include + +#include "device/ascend/ascend_device_address.h" +#include "utils/context/ms_context.h" +#include "device/ascend/profiling/profiling_manager.h" +#include "hccl/hcom.h" +#include "runtime/context.h" +#include "device/ascend/ascend_stream_assign.h" +#include "device/ascend/ascend_memory_allocator.h" +#include "framework/ge_runtime/model_runner.h" +#include "device/ascend/tasksink/task_generator.h" +#include "session/anf_runtime_algorithm.h" +#include "device/ascend/profiling/profiling_utils.h" +#include "kernel/tbe/tbe_utils.h" +#include "kernel/tbe/tbe_python_funcs.h" +#include "pre_activate/mem_reuse/mem_reuse_checker.h" + +using mindspore::device::ascend::ProfilingManager; +using mindspore::device::ascend::ProfilingUtils; +using mindspore::device::ascend::tasksink::TaskGenerator; +using mindspore::kernel::tbe::TbeUtils; +using std::vector; + +namespace mindspore { +namespace device { +namespace ascend { +static const uint64_t ASCEND_MEM_SIZE = 20; +static const uint64_t ASCEND_MEM_SIZE_BYTE = (ASCEND_MEM_SIZE << 30); +static const size_t PRAMATER_OUTPUT_INDEX = 0; + +AscendKernelRuntime::~AscendKernelRuntime() { graph_model_map_.clear(); } + +void AscendKernelRuntime::ClearGraphModelMap() { + for (auto &iter : graph_model_id_map_) { + MS_LOG(INFO) << "Ge UnloadModel " << iter.second; + auto ret = ge::model_runner::ModelRunner::Instance().UnloadModel(iter.second); + if (!ret) { + MS_LOG(ERROR) << "UnloadModel failed"; + } + } +} + +bool AscendKernelRuntime::NeedDestroyHccl() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (!context_ptr->enable_hccl()) { + MS_LOG(INFO) << "hccl is not enabled"; + return false; + } + // Note: make sure hcom_connectivity_detection api never be used. + return true; +} + +void AscendKernelRuntime::ReleaseDeviceRes() { + MS_LOG(INFO) << "ascend finalize start"; + // release ge runtime + ClearGraphModelMap(); + + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + auto ret = rtSetDevice(context_ptr->device_id()); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtSetDevice, ret[" << static_cast(ret) << "]"; + } + + FreeDeviceMemory(); + (void)DestroyHccl(); + (void)ResetDevice(); + (void)ProfilingManager::GetInstance().StopProfiling(); + MS_LOG(INFO) << "ascend finalize end"; +} + +bool AscendKernelRuntime::Init() { + if (initialized_) { + return true; + } + bool ret = false; +#ifdef ENABLE_DUMP_E2E + ret = SetDumpConf(); + if (!ret) { + MS_LOG(INFO) << "no dump conf to set!"; + } +#endif + + ret = InitDevice(); + if (!ret) { + return ret; + } + + ret = MallocDeviceMemory(); + if (!ret) { + return ret; + } + + ret = ProfilingManager::GetInstance().StartupProfiling(device_id_); + if (!ret) { + MS_EXCEPTION(DeviceProcessError) << "StartupProfiling failed."; + } + + initialized_ = true; + return ret; +} + +#ifdef ENABLE_DUMP_E2E +namespace { +void DumpOutput(mindspore::session::KernelGraph *graph, const string &dump_path, DumpConfPtr dump_conf) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(dump_conf); + bool trans_flag = dump_conf->trans_flag(); + const auto &apply_kernels = graph->execution_order(); + for (const auto &node : apply_kernels) { + MS_EXCEPTION_IF_NULL(node); + auto node_name = AnfAlgo::GetCNodeName(node); + std::string kernel_name = node->fullname_with_scope(); + if (!dump_conf->IsKernelNeedDump(kernel_name)) { + continue; + } + const std::string strsrc = "/"; + const std::string strdst = "--"; + std::string::size_type pos = 0; + std::string::size_type srclen = strsrc.size(); + std::string::size_type dstlen = strdst.size(); + while ((pos = kernel_name.find(strsrc, pos)) != std::string::npos) { + kernel_name.replace(pos, srclen, strdst); + pos += dstlen; + } + auto output_size = AnfAlgo::GetOutputTensorNum(node); + for (size_t j = 0; j < output_size; ++j) { + auto addr = AnfAlgo::GetOutputAddr(node, j); + auto shape = AnfAlgo::GetOutputDeviceShape(node, j); + auto type = AnfAlgo::GetOutputDeviceDataType(node, j); + auto format = AnfAlgo::GetOutputFormat(node, j); + string filepath = dump_path + '/' + kernel_name + '_' + "output_" + std::to_string(j); + auto ascend_addr = dynamic_cast(addr); + std::vector int_shapes; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes), + [](size_t inner_item) { return SizeToInt(inner_item); }); + auto ret = ascend_addr->DumpMemToFile(trans_flag, filepath, format, int_shapes, type); + if (!ret) { + MS_LOG(ERROR) << "DumpMemToFile Failed: flag:" << trans_flag << ", path:" << filepath + << ", host_format:" << format << ".!"; + } + } + } +} + +void DumpParameters(mindspore::session::KernelGraph *graph, const string &dump_path, DumpConfPtr dump_conf) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(dump_conf); + bool trans_flag = dump_conf->trans_flag(); + const auto ¶meters = graph->inputs(); + for (auto &item : parameters) { + if (!item->isa()) { + continue; + } + std::string parameter_name = item->fullname_with_scope(); + if (!dump_conf->IsKernelNeedDump(parameter_name)) { + continue; + } + auto addr = AnfAlgo::GetOutputAddr(item, PRAMATER_OUTPUT_INDEX); + auto shape = AnfAlgo::GetOutputDeviceShape(item, PRAMATER_OUTPUT_INDEX); + auto type = AnfAlgo::GetOutputDeviceDataType(item, PRAMATER_OUTPUT_INDEX); + auto format = AnfAlgo::GetOutputFormat(item, PRAMATER_OUTPUT_INDEX); + string filepath = dump_path + '/' + parameter_name + '_' + "output_0"; + auto ascend_addr = dynamic_cast(addr); + std::vector int_shapes; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes), + [](size_t inner_item) { return SizeToInt(inner_item); }); + auto ret = ascend_addr->DumpMemToFile(trans_flag, filepath, format, int_shapes, type); + if (!ret) { + MS_LOG(ERROR) << "DumpMemToFile Failed: flag:" << trans_flag << ", path:" << filepath + << ", host_format:" << format << ".!"; + } + } +} +} // namespace +#endif + +bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); +#ifdef ENABLE_DUMP_E2E + MS_LOG(INFO) << "start dump step"; + DumpConfPtr dump_conf = GetDumpConf(); + MS_EXCEPTION_IF_NULL(dump_conf); + dump_conf->UpdataCurIter(); + bool dump_flag = dump_conf->dump_enable(); + if (!dump_flag) { + MS_LOG(INFO) << "dump flag is disable, pass dump step"; + return true; + } + uint32_t cur_iter = dump_conf->cur_iter(); + if (dump_conf->dump_iter() != 0) { + if (cur_iter != dump_conf->dump_iter()) { + return true; + } + } + MS_LOG(INFO) << "cur iter is " << cur_iter; + std::string net_name = dump_conf->dump_net_name(); + std::string iterator = to_string(cur_iter); + std::string dump_path = dump_conf->dump_path(); + if (dump_path.back() == '/') { + dump_path = dump_path + net_name + '/' + iterator; + } else { + dump_path = dump_path + '/' + net_name + '/' + iterator; + } + // dump output + DumpOutput(graph, dump_path, dump_conf); + // dump parameters + DumpParameters(graph, dump_path, dump_conf); +#endif + return true; +} + +DeviceAddressPtr AscendKernelRuntime::CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) { + return std::make_shared(device_ptr, device_size, format, type_id); +} + +void AscendKernelRuntime::MallocOpMemory(const DeviceAddressPtr address, size_t size, int flag) { + MS_EXCEPTION_IF_NULL(MsContext::GetInstance()); + if (MsContext::GetInstance()->enable_dynamic_mem_pool()) { + auto device_ptr = AscendMemoryAllocator::GetInstance().AllocTensorMem(size); + MS_EXCEPTION_IF_NULL(device_ptr); + address->ptr_ = device_ptr; + address->mem_dynamic_alloc_ = true; + return; + } + if (flag == kStaticMem) { + address->ptr_ = MallocStaticMem(size, false); + } else if (flag == kDynamicMem) { + address->ptr_ = MallocDynamicMem(size, false); + } else { + MS_LOG(EXCEPTION) << "Unknown memory type!"; + } +} + +bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool is_task_sink = context_ptr->enable_task_sink(); + if (!is_task_sink) { + return true; + } +#ifdef MEM_REUSE_DEBUG + if (!context_ptr->enable_mem_reuse()) { + // Get normal graph ir for memreuse + mindspore::memreuse::MemReuseChecker::GetInstance().CheckNormalIR(graph); + } +#endif + if (graph == nullptr) { + MS_EXCEPTION(NotExistsError) << "session::KernelGraph is NULL!"; + } + vector> task_info_list; + auto anf_node_list = graph->execution_order(); + TaskGenerator::GenTasks(anf_node_list, &task_info_list, graph->graph_id()); + + AscendStreamAssign &assign_instance = AscendStreamAssign::GetInstance(); + // the streams' flag not HEAD_STREAM + std::vector wait_active_stream_list = assign_instance.GetWaitStreams(); + std::vector force_copy_stream_list = assign_instance.GetHcomStreams(); + + MS_LOG(INFO) << "call DavinciModel total stream num:" << assign_instance.GetTotalStreamNum() + << ", total event num:" << assign_instance.GetTotalEventNum() + << ", wait_active_stream_list size:" << wait_active_stream_list.size() + << ", force_copy_stream_list size:" << force_copy_stream_list.size(); + + std::vector> empty_list; + std::shared_ptr model = std::make_shared( + task_info_list, empty_list, empty_list, empty_list, empty_list, wait_active_stream_list, force_copy_stream_list, 0, + 0, 0, 0, 0, 0, assign_instance.GetTotalStreamNum(), 1, assign_instance.GetTotalEventNum(), 0); + + graph_model_map_[graph] = model; + graph_model_id_map_[graph] = graph->graph_id(); + MS_LOG(INFO) << "TaskGenerator GetTaskInfo end..."; + + // Store the task_info_list + task_map_.insert(std::make_pair(graph, task_info_list)); + + return true; +} + +uint32_t AscendKernelRuntime::GetGraphModelId(const session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto iter = graph_model_id_map_.find(kernel_graph); + if (iter == graph_model_id_map_.end()) { + MS_LOG(EXCEPTION) << "graph not in the map"; + } + return iter->second; +} + +bool AscendKernelRuntime::LoadTask(const session::KernelGraph *graph) { + if (graph == nullptr) { + MS_EXCEPTION(NotExistsError) << "Null pointer graph, LoadTask failed. "; + } + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool is_task_sink = context_ptr->enable_task_sink(); + if (!is_task_sink) { + return true; + } + + auto task_iter = graph_model_map_.find(graph); + if (task_iter == graph_model_map_.end()) { + MS_LOG(ERROR) << "task not exist"; + return false; + } + + auto model_id = GetGraphModelId(graph); + std::shared_ptr listener; + MS_LOG(INFO) << "LoadDavinciModel mode_id:" << model_id; + bool status = + ge::model_runner::ModelRunner::Instance().LoadDavinciModel(device_id_, 0, model_id, task_iter->second, listener); + if (!status) { + MS_LOG(INFO) << "load task failed"; + return false; + } + if (ProfilingManager::GetInstance().IsProfiling()) { + std::vector task_ids = ge::model_runner::ModelRunner::Instance().GetTaskIdList(model_id); + ProfilingUtils::ReportProfilingData(graph->graph_id(), task_ids); + } + return true; +} + +bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + ge::InputData input_tensors = ge::InputData(); + ge::OutputData *output_tensors = nullptr; + auto model_id = GetGraphModelId(graph); + bool status = ge::model_runner::ModelRunner::Instance().RunModel(model_id, input_tensors, output_tensors); + if (!status) { + MS_LOG(INFO) << "run task failed"; + return false; + } + return true; +} + +bool AscendKernelRuntime::SyncStream() { + if (RT_ERROR_NONE != rtStreamSynchronize(stream_)) { // o for switch stream + MS_LOG(ERROR) << "Call runtime rtStreamSynchronize error."; + return false; + } + return true; +} + +bool AscendKernelRuntime::InitDevice() { + int device_count = 0; + auto ret = rtGetDeviceCount(&device_count); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtGetDeviceCount, ret[" << static_cast(ret) << "]"; + } + + ret = rtSetDevice(device_id_); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtSetDevice, ret[" << static_cast(ret) << "]"; + } + + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr == nullptr) { + MS_LOG(ERROR) << "get MsContext instance failed"; + return false; + } + if (context_ptr->enable_hccl()) { + if (!HcclInit()) { + MS_LOG(ERROR) << "HcclInit init failed"; + return false; + } + } + + ret = rtCtxCreate(&rt_context_, 0, device_id_); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtCtxCreate, ret[" << static_cast(ret) << "]"; + } + + ret = rtCtxSetCurrent(rt_context_); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtCtxSetCurrent, ret[" << ret << "]"; + } + + ret = rtStreamCreate(&stream_, 0); + if (ret != RT_ERROR_NONE) { + MS_LOG(EXCEPTION) << "rtStreamCreate, ret[" << ret << "]"; + } + + return true; +} + +bool AscendKernelRuntime::ResetDevice() { + auto ret = rtCtxSetCurrent(rt_context_); + if (ret != RT_ERROR_NONE) { + MS_LOG(ERROR) << "call rtCtxSetCurrent failed"; + return false; + } + + if (stream_ != nullptr) { + ret = rtStreamDestroy(stream_); + if (ret != RT_ERROR_NONE) { + MS_LOG(EXCEPTION) << "rtStreamDestroy, ret[" << ret << "]"; + } + stream_ = nullptr; + } + + if (rt_context_ != nullptr) { + ret = rtCtxDestroy(rt_context_); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtCtxDestroy, ret[" << ret << "]"; + } + rt_context_ = nullptr; + } + return true; +} + +bool AscendKernelRuntime::HcclInit() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (!context_ptr->IsTsdOpened()) { + MS_LOG(EXCEPTION) << "Hccl dependent tsd is not open"; + } + + MS_LOG(INFO) << "do hcom init"; + std::string path; + const char *config_path_str = std::getenv("MINDSPORE_HCCL_CONFIG_PATH"); + if (config_path_str == nullptr) { + MS_LOG(ERROR) << "get hccl json config failed, please set env MINDSPORE_HCCL_CONFIG_PATH"; + return false; + } + path = config_path_str; + char fullPath[PATH_MAX] = {0}; + if (path.size() > PATH_MAX || realpath(path.c_str(), fullPath) == nullptr) { + MS_LOG(ERROR) << "file " << path << " is not exist"; + return false; + } + const char *identify = std::getenv("RANK_ID"); + if (identify == nullptr) { + MS_LOG(ERROR) << "get hccl rankid failed, please set env RANK_ID"; + return false; + } + MS_LOG(INFO) << "MINDSPORE_HCCL_CONFIG_PATH : " << fullPath << ", RANK_ID: " << identify; + hcclResult_t res = hcom_init(fullPath, identify); + if (res != HCCL_SUCCESS) { + MS_LOG(ERROR) << "hcom init failed, res is " << static_cast(res); + return false; + } + return true; +} + +bool AscendKernelRuntime::DestroyHccl() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (!NeedDestroyHccl()) { + MS_LOG(INFO) << "hccl is not enable, no need to close."; + return true; + } + hcclResult_t res = hcom_destroy(); + if (res != HCCL_SUCCESS) { + MS_LOG(ERROR) << "hccl destroy failed"; + return false; + } + MS_LOG(INFO) << "hccl destroy successful, status = " << res << "."; + context_ptr->set_enable_hccl(false); + return true; +} + +bool AscendKernelRuntime::MallocDeviceMemory() { + device_mem_size_ = ASCEND_MEM_SIZE_BYTE; + MS_EXCEPTION_IF_NULL(MsContext::GetInstance()); + if (MsContext::GetInstance()->enable_dynamic_mem_pool()) { + static_mem_offset_ = FloatToSize(device_mem_size_ * GRAPH_INIT_DAVINCI_MEM_RATIO); + device_mem_pool_size_ = FloatToSize(device_mem_size_ * (1 - GRAPH_INIT_DAVINCI_MEM_RATIO)); + auto ret = rtMalloc(reinterpret_cast(&device_mem_pool_base_), device_mem_pool_size_, RT_MEMORY_HBM); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtMalloc mem size[" << device_mem_pool_size_ << "] fail, ret[" << ret << "]"; + } + AscendMemoryAllocator::GetInstance().set_device_mem_pool_base(device_mem_pool_base_); + AscendMemoryAllocator::GetInstance().set_device_mem_pool_size(device_mem_pool_size_); + } else { + static_mem_offset_ = device_mem_size_; + } + auto ret = rtMalloc(reinterpret_cast(&device_mem_base_), device_mem_size_, RT_MEMORY_HBM); + if (ret != RT_ERROR_NONE) { + MS_EXCEPTION(DeviceProcessError) << "rtMalloc mem size[" << device_mem_size_ << "] fail, ret[" << ret << "]"; + } + return true; +} + +void AscendKernelRuntime::FreeDeviceMemory() { + if (device_mem_base_ != nullptr) { + auto ret = rtFree(device_mem_base_); + if (ret != RT_ERROR_NONE) { + MS_LOG(ERROR) << "rtFree mem size[" << device_mem_size_ << "] fail, ret[" << ret << "]"; + } + device_mem_base_ = nullptr; + } + if (device_mem_pool_base_ != nullptr) { + auto ret = rtFree(device_mem_pool_base_); + if (ret != RT_ERROR_NONE) { + MS_LOG(ERROR) << "rtFree mem size[" << device_mem_pool_size_ << "] fail, ret[" << ret << "]"; + } + device_mem_pool_base_ = nullptr; + } +} + +void AscendKernelRuntime::FreeHostMemory() { dynamic_mem_offset_ = 0; } +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h new file mode 100644 index 00000000000..dbd1460d24f --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_KERNEL_RUNTIME_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_KERNEL_RUNTIME_H_ +#include +#include +#include +#include +#include "device/kernel_runtime.h" +#include "runtime/context.h" +#include "framework/ge_runtime/davinci_model.h" +#include "device/kernel_runtime_manager.h" + +using ge::model_runner::TaskInfo; +using std::unordered_map; +using std::vector; +namespace mindspore { +namespace device { +namespace ascend { +class AscendKernelRuntime : public KernelRuntime { + public: + AscendKernelRuntime() = default; + ~AscendKernelRuntime() override; + bool Init() override; + bool DumpData(session::KernelGraph *graph) override; + bool GenTask(const session::KernelGraph *graph) override; + bool RunTask(const session::KernelGraph *graph) override; + bool LoadTask(const session::KernelGraph *graph) override; + void FreeHostMemory() override; + + protected: + DeviceAddressPtr CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) override; + bool SyncStream() override; + void MallocOpMemory(const DeviceAddressPtr address, size_t size, int flag) override; + + private: + bool InitDevice(); + bool ResetDevice(); + bool HcclInit(); + bool NeedDestroyHccl(); + bool DestroyHccl(); + bool MallocDeviceMemory(); + void FreeDeviceMemory(); + void ClearGraphModelMap(); + void ReleaseDeviceRes() override; + uint32_t GetGraphModelId(const session::KernelGraph *kernel_graph); + rtContext_t rt_context_{nullptr}; + bool initialized_{false}; + unordered_map>> task_map_; + unordered_map> graph_model_map_; + unordered_map graph_model_id_map_; +}; + +MS_REG_KERNEL_RUNTIME(kAscendDevice, AscendKernelRuntime); +} // namespace ascend +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_KERNEL_RUNTIME_H_ diff --git a/mindspore/ccsrc/device/ascend/ascend_memory_allocator.cc b/mindspore/ccsrc/device/ascend/ascend_memory_allocator.cc new file mode 100644 index 00000000000..06b921f5090 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_memory_allocator.cc @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/ascend_memory_allocator.h" +#include "device/ascend/ascend_kernel_runtime.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +namespace ascend { +const uint64_t MEM_SIZE = 20; +const uint64_t MEM_SIZE_BYTE = (MEM_SIZE << 30); + +AscendMemoryAllocator::AscendMemoryAllocator() { + hasMalloc_ = false; + free_mem_size_ = FloatToSize(MEM_SIZE_BYTE * (1 - GRAPH_INIT_DAVINCI_MEM_RATIO)); + total_mem_size_ = free_mem_size_; +} + +size_t AscendMemoryAllocator::AllocDeviceMem(size_t size, DeviceMemPtr* addr) { + if (hasMalloc_) { + MS_LOG(EXCEPTION) << "Has alloc memory pool memory !"; + } + if (size == 0 || size > free_mem_size_) { + MS_LOG(EXCEPTION) << "Failed to alloc memory pool memory !"; + } + *addr = device_mem_pool_base_; + if (*addr == nullptr) { + MS_LOG(EXCEPTION) << "Device memory pool base is nullptr, failed to alloc memory pool memory!"; + } + hasMalloc_ = true; + free_mem_size_ -= size; + return size; +} + +bool AscendMemoryAllocator::FreeDeviceMem(const DeviceMemPtr& addr) { + MS_EXCEPTION_IF_NULL(addr); + hasMalloc_ = false; + free_mem_size_ = total_mem_size_; + return true; +} + +size_t AscendMemoryAllocator::AlignMemorySize(size_t size) const { + if (size == 0) { + return DYNAMIC_MEM_ALIGN_SIZE; + } + return ((size + DYNAMIC_MEM_ALIGN_SIZE + 31) / DYNAMIC_MEM_ALIGN_SIZE) * DYNAMIC_MEM_ALIGN_SIZE; +} + +size_t AscendMemoryAllocator::mem_alloc_unit_size() const { return free_mem_size_ - 512; } + +void AscendMemoryAllocator::set_device_mem_pool_base(uint8_t* device_mem_pool_base) { + MS_EXCEPTION_IF_NULL(device_mem_pool_base); + device_mem_pool_base_ = device_mem_pool_base; +} + +size_t AscendMemoryAllocator::free_mem_size() { return free_mem_size_; } + +size_t AscendMemoryAllocator::total_mem_size() { return total_mem_size_; } +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/ascend_memory_allocator.h b/mindspore/ccsrc/device/ascend/ascend_memory_allocator.h new file mode 100644 index 00000000000..74cc50834fd --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_memory_allocator.h @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_MEMORY_ALLOCATOR_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_MEMORY_ALLOCATOR_H_ + +#include +#include "pre_activate/mem_reuse/mem_dynamic_allocator.h" + +namespace mindspore { +namespace device { +namespace ascend { +// The fraction of total ascend memory used to compute the graph. +static const float GRAPH_INIT_DAVINCI_MEM_RATIO = 0.8; + +class AscendMemoryAllocator : public DynamicMemPoolBestFit { + public: + ~AscendMemoryAllocator() override = default; + + size_t AllocDeviceMem(size_t size, DeviceMemPtr* addr) override; + bool FreeDeviceMem(const DeviceMemPtr& addr) override; + void set_device_mem_pool_base(uint8_t* device_mem_pool_base); + void set_device_mem_pool_size(uint64_t device_mem_pool_size) { device_mem_pool_size_ = device_mem_pool_size; } + size_t free_mem_size() override; + size_t total_mem_size() override; + + static AscendMemoryAllocator& GetInstance() { + static AscendMemoryAllocator instance; + return instance; + } + + protected: + // The real size by memory alloc aligned. + size_t AlignMemorySize(size_t size) const override; + // Get the minimum memory unit size using for dynamic extend. + size_t mem_alloc_unit_size() const override; + + private: + AscendMemoryAllocator(); + AscendMemoryAllocator(const AscendMemoryAllocator&) = delete; + AscendMemoryAllocator& operator=(const AscendMemoryAllocator&) = delete; + bool hasMalloc_; + uint8_t* device_mem_pool_base_{nullptr}; + uint64_t device_mem_pool_size_{0}; + size_t free_mem_size_; + size_t total_mem_size_; +}; +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_MEMORY_ALLOCATOR_H_ diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc new file mode 100644 index 00000000000..4f16c596c7a --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc @@ -0,0 +1,713 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/ascend_stream_assign.h" + +#include +#include + +#include "ir/manager.h" +#include "utils/context/ms_context.h" +#include "common/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_adjust.h" +#include "predict/generator/utils/ir_model_util.h" +#include "device/kernel_info.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace device { +namespace ascend { +const uint32_t kHcomMaxTask = 5; +const uint32_t kCommonMaxTask = 350; +const uint32_t kIndependFirstStreamId = 1024; + +bool AscendStreamAssign::IsHcom(const CNodePtr &apply_kernel) { + MS_EXCEPTION_IF_NULL(apply_kernel); + return AnfAlgo::GetKernelType(apply_kernel) == HCCL_KERNEL; +} + +void AscendStreamAssign::ResetNew() { + total_common_stream_num_ = 0; + total_independ_stream_num_ = 0; + total_event_num_ = 0; + first_physic_id_ = UINT32_MAX; + first_logic_id_ = UINT32_MAX; + independent_id_ = kIndependFirstStreamId; + logic_to_independent_map_.clear(); + processed_logic_id_.clear(); + logic_to_physic_map_.clear(); + independent_before_physic_id_.clear(); + inner_parallel_streams_.clear(); + processed_parallel_streams_.clear(); + hcom_stream_list_.clear(); +} + +void AscendStreamAssign::AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr, uint32_t processing_logic_id) { + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + auto it = logic_to_independent_map_.find(processing_logic_id); + if (it == logic_to_independent_map_.end()) { + (void)logic_to_independent_map_.insert(std::make_pair(processing_logic_id, independent_id_)); + AnfAlgo::SetStreamId(independent_id_, cur_cnode_ptr.get()); + independent_id_++; + } else { + AnfAlgo::SetStreamId(it->second, cur_cnode_ptr.get()); + } + + if (first_physic_id_ == UINT32_MAX) { + auto res = std::find(independent_before_physic_id_.begin(), independent_before_physic_id_.end(), + AnfAlgo::GetStreamId(cur_cnode_ptr)); + if (res == independent_before_physic_id_.end()) { + independent_before_physic_id_.push_back(AnfAlgo::GetStreamId(cur_cnode_ptr)); + } + } +} + +void AscendStreamAssign::AssignCommonStreamId(const CNodePtr &cur_cnode_ptr, CNodePtr *pre_cnode_ptr, + uint32_t *cur_index, uint32_t *cur_stream_id) { + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + MS_EXCEPTION_IF_NULL(*pre_cnode_ptr); + bool over_max_hcom_task = (IsHcom(cur_cnode_ptr) && (*cur_index) % kHcomMaxTask == 0); + bool over_max_common_task = (!IsHcom(cur_cnode_ptr) && (*cur_index) % kCommonMaxTask == 0); + bool pre_common_cur_hcom = (IsHcom(cur_cnode_ptr) && !IsHcom(*pre_cnode_ptr)); + bool pre_hcom_cur_common = (!IsHcom(cur_cnode_ptr) && IsHcom(*pre_cnode_ptr)); + if (over_max_hcom_task || over_max_common_task || pre_common_cur_hcom || pre_hcom_cur_common) { + *cur_index = 0; + ++(*cur_stream_id); + } + + if (over_max_hcom_task || pre_common_cur_hcom) { + hcom_stream_list_.emplace_back(*cur_stream_id); + } + ++(*cur_index); + AnfAlgo::SetStreamId(*cur_stream_id, cur_cnode_ptr.get()); + *pre_cnode_ptr = cur_cnode_ptr; +} + +bool AscendStreamAssign::IsProcessed(uint32_t logic_id) { + auto it = std::find(processed_logic_id_.begin(), processed_logic_id_.end(), logic_id); + if (it == processed_logic_id_.end()) { + return false; + } + + return true; +} + +void AscendStreamAssign::RecordIdMap(uint32_t logic_id, uint32_t physic_id) { + auto it = logic_to_physic_map_.find(logic_id); + if (it == logic_to_physic_map_.end()) { + MS_LOG(INFO) << "New logic_id[" << logic_id << "] to physic_id[" << physic_id << "]"; + (void)logic_to_physic_map_.insert(std::make_pair(logic_id, physic_id)); + } +} + +void AscendStreamAssign::RecordFirstCommonOp(const CNodePtr &cur_cnode_ptr, uint32_t cur_node_logic_id, + uint32_t cur_stream_id) { + AnfAlgo::SetStreamId(cur_stream_id, cur_cnode_ptr.get()); + RecordIdMap(cur_node_logic_id, cur_stream_id); + first_physic_id_ = cur_stream_id; + first_logic_id_ = cur_node_logic_id; +} + +uint32_t AscendStreamAssign::GetLogicId(const CNodePtr &cur_cnode_ptr) { + uint32_t logic_id = AnfAlgo::GetStreamDistinctionLabel(cur_cnode_ptr.get()); + if (logic_id == kInvalidDistincLabel) { + MS_LOG(EXCEPTION) << "node[" << cur_cnode_ptr->DebugString() << "] logic id is invalid"; + } + return logic_id; +} + +void AscendStreamAssign::SetCommonStreamNum(uint32_t cur_stream_id) { + if (first_physic_id_ == UINT32_MAX) { + MS_LOG(INFO) << "cur common node size is zero"; + total_common_stream_num_ = 0; + } else { + total_common_stream_num_ = cur_stream_id + 1; + } +} + +void AscendStreamAssign::AssignAllNodesStream(const shared_ptr &graph_ptr) { + MS_EXCEPTION_IF_NULL(graph_ptr); + auto cnode_ptr_list = graph_ptr->execution_order(); + CNodePtr pre_cnode_ptr = nullptr; + uint32_t cur_index = 0; + uint32_t cur_stream_id = 0; + uint32_t processing_logic_id = UINT32_MAX; + + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + // get logic id + uint32_t cur_node_logic_id = GetLogicId(cur_cnode_ptr); + if (IsIndependentNode(cur_cnode_ptr)) { + AssignIndependentStreamId(cur_cnode_ptr, cur_node_logic_id); + continue; + } + if (pre_cnode_ptr == nullptr) { + RecordFirstCommonOp(cur_cnode_ptr, cur_node_logic_id, cur_stream_id); + processing_logic_id = cur_node_logic_id; + ++cur_index; + pre_cnode_ptr = cur_cnode_ptr; + continue; + } + + // 1.has been processed + if (IsProcessed(cur_node_logic_id)) { + continue; + } + + if (cur_node_logic_id == processing_logic_id) { + AssignCommonStreamId(cur_cnode_ptr, &pre_cnode_ptr, &cur_index, &cur_stream_id); + } else { + // 1.find other same logic id + for (size_t j = i; j < cnode_ptr_list.size(); ++j) { + CNodePtr cnode_ptr = cnode_ptr_list[j]; + MS_EXCEPTION_IF_NULL(cnode_ptr); + uint32_t logic_id = AnfAlgo::GetStreamDistinctionLabel(cnode_ptr.get()); + if (logic_id == processing_logic_id) { + AssignCommonStreamId(cnode_ptr, &pre_cnode_ptr, &cur_index, &cur_stream_id); + } + } + // 2.after deal: + processed_logic_id_.push_back(processing_logic_id); + cur_cnode_ptr = cnode_ptr_list[i]; + // 3. new stream + ++cur_stream_id; + AnfAlgo::SetStreamId(cur_stream_id, cur_cnode_ptr.get()); + cur_index = 1; + + pre_cnode_ptr = cur_cnode_ptr; + processing_logic_id = cur_node_logic_id; + RecordIdMap(processing_logic_id, cur_stream_id); + } + } + + SetCommonStreamNum(cur_stream_id); + total_independ_stream_num_ = independent_id_ - kIndependFirstStreamId; + MS_LOG(INFO) << "stream nums:common:" << total_common_stream_num_ << ",independ:" << total_independ_stream_num_; +} + +vector AscendStreamAssign::TransLogicToPhysic(const vector &logic_ids) { + vector physic_ids; + for (auto &id : logic_ids) { + auto it = logic_to_physic_map_.find(id); + if (it != logic_to_physic_map_.end()) { + MS_LOG(INFO) << "logic id[" << id << "] to physic id[" << it->second << "]"; + physic_ids.push_back(it->second); + } else { + MS_LOG(EXCEPTION) << "logic id[" << id << "] has no correspond physic id"; + } + + auto it_independ = logic_to_independent_map_.find(id); + if (it_independ != logic_to_independent_map_.end()) { + MS_LOG(INFO) << "logic id[" << id << "] to independent id[" << it_independ->second << "]"; + physic_ids.push_back(it_independ->second); + } + } + return physic_ids; +} + +void AscendStreamAssign::UpdateStreamActive(const CNodePtr &active_ptr) { + MS_LOG(INFO) << "start update outter active op[" << active_ptr->DebugString() << "] "; + MS_EXCEPTION_IF_NULL(active_ptr); + auto primitive = AnfAlgo::GetCNodePrimitive(active_ptr); + MS_EXCEPTION_IF_NULL(primitive); + vector active_logic_ids = GetValue>(primitive->GetAttr(kAttrActiveStreamList)); + // out StreamAcitve active physic stream is not parallel now, if parallel, should deal here. + vector active_physic_ids = TransLogicToPhysic(active_logic_ids); + ValuePtr active_physic_value = MakeValue>(active_physic_ids); + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, active_physic_value, active_ptr); +} + +void AscendStreamAssign::UpdateStreamSwitch(const CNodePtr &switch_ptr, const CNodePtr &active_ptr) { + MS_LOG(INFO) << "start update switch op[" << switch_ptr->DebugString() << "]"; + MS_EXCEPTION_IF_NULL(switch_ptr); + MS_EXCEPTION_IF_NULL(active_ptr); + auto primitive = AnfAlgo::GetCNodePrimitive(switch_ptr); + MS_EXCEPTION_IF_NULL(primitive); + auto true_logic_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); + MS_LOG(INFO) << "streamswtich stream id[" << AnfAlgo::GetStreamId(switch_ptr) << "], true_logic_id[" << true_logic_id + << "]"; + vector logic_ids{true_logic_id}; + vector physic_ids = TransLogicToPhysic(logic_ids); + if (physic_ids.empty()) { + MS_LOG(EXCEPTION) << "stream switch true logic id[" << true_logic_id << "] has no physical id"; + } + ValuePtr true_index = MakeValue(physic_ids[0]); + AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, true_index, switch_ptr); + + MS_LOG(INFO) << "start update StreamActive op[" << active_ptr->DebugString() << "]"; + AnfAlgo::SetStreamId(physic_ids[0], active_ptr.get()); + vector active_ids; + for (size_t i = 0; i < physic_ids.size(); i++) { + if (i == 0) { + MS_LOG(INFO) << "StreamActive op self stream id[" << physic_ids[i] << "]"; + } else { + MS_LOG(INFO) << "StreamActive op active stream id[" << physic_ids[i] << "]"; + active_ids.emplace_back(physic_ids[i]); + } + } + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_ids), active_ptr); +} + +void AscendStreamAssign::FindAllReduceParallel(const shared_ptr &graph_ptr) { + MS_EXCEPTION_IF_NULL(graph_ptr); + CNodePtr cur_cnode_ptr = nullptr; + CNodePtr pre_cnode_ptr = nullptr; + uint32_t pre_stream_id = UINT32_MAX; + auto cnode_ptr_list = graph_ptr->execution_order(); + for (uint32_t i = 0; i < cnode_ptr_list.size(); ++i) { + cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + if (i == 0) { + pre_cnode_ptr = cur_cnode_ptr; + pre_stream_id = cur_stream_id; + continue; + } + + bool diff_stream = (pre_stream_id != cur_stream_id) && (pre_stream_id < cur_stream_id); + bool pre_hcom = IsHcom(pre_cnode_ptr); + if (diff_stream && pre_hcom) { + inner_parallel_streams_.emplace_back(std::vector{pre_stream_id, cur_stream_id}); + } + + pre_cnode_ptr = cur_cnode_ptr; + pre_stream_id = cur_stream_id; + } +} + +void AscendStreamAssign::InsertSendRecvForHcomParallel(const shared_ptr &graph_ptr) { + MS_LOG(INFO) << "start"; + MS_EXCEPTION_IF_NULL(graph_ptr); + auto cnode_ptr_list = graph_ptr->execution_order(); + vector cnodes = cnode_ptr_list; + uint32_t cur_event_id = 0; + auto it = cnodes.begin(); + while (it != cnodes.end() && (it + 1) != cnodes.end()) { + MS_EXCEPTION_IF_NULL(*it); + MS_EXCEPTION_IF_NULL(*(it + 1)); + if (IsHcom(*it) && !IsHcom(*(it + 1))) { + CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); + it = cnodes.insert(it + 1, send_cnode_ptr); + + auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); + if (target == cnodes.end()) { + MS_LOG(WARNING) << "hcom node[" << (*(it - 1))->fullname_with_scope() + << "] can't find target for insert recv op, no insert send/recv"; + it = cnodes.erase(it); + continue; + } + + // deal recv op + uint32_t stream_id = AnfAlgo::GetStreamId(*target); + CNodePtr recv_cnode_ptr = CreateRecvApplyKernel(graph_ptr, cur_event_id, stream_id); + (void)cnodes.insert(target, recv_cnode_ptr); + ++cur_event_id; + } + ++it; + } + graph_ptr->set_execution_order(cnodes); + total_event_num_ = cur_event_id; + MS_LOG(INFO) << "after insert send/recv for hcom parallel, total event nums[" << total_event_num_ << "]"; + MS_LOG(INFO) << "end"; +} + +bool AscendStreamAssign::IsProcessedParallelStream(uint32_t stream_id) { + auto it = std::find(processed_parallel_streams_.begin(), processed_parallel_streams_.end(), stream_id); + if (it != processed_parallel_streams_.end()) { + return true; + } + return false; +} + +vector AscendStreamAssign::GetParallelStream(uint32_t cur_stream_id, uint32_t stream_acitve_id) { + vector parallel_streams; + for (size_t i = 0; i < inner_parallel_streams_.size(); i++) { + auto cur_parallel_streams = inner_parallel_streams_[i]; + auto it = std::find(cur_parallel_streams.begin(), cur_parallel_streams.end(), cur_stream_id); + if (it != cur_parallel_streams.end()) { + MS_LOG(INFO) << "stream id:" << cur_stream_id << " is parallel stream"; + for (size_t j = 0; j < cur_parallel_streams.size(); j++) { + if (cur_parallel_streams[j] == stream_acitve_id) { + MS_LOG(INFO) << "one of parallel stream id" << cur_parallel_streams[j] + << "is same with streamacvite stream id" << stream_acitve_id; + continue; + } + parallel_streams.emplace_back(cur_parallel_streams[j]); + } + + // record processed parallel streams + (void)std::copy(parallel_streams.begin(), parallel_streams.end(), + std::back_inserter(processed_parallel_streams_)); + return parallel_streams; + } + } + + return vector{cur_stream_id}; +} + +void AscendStreamAssign::InsertActiveNew(const std::shared_ptr &graph_ptr) { + MS_LOG(INFO) << "start"; + MS_EXCEPTION_IF_NULL(graph_ptr); + std::vector update_cnode_list; + CNodePtr cur_cnode_ptr = nullptr; + CNodePtr pre_cnode_ptr = nullptr; + uint32_t pre_stream_id = UINT32_MAX; + + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + if (cur_stream_id >= kIndependFirstStreamId) { + update_cnode_list.emplace_back(cur_cnode_ptr); + continue; + } + + bool inner_active = pre_stream_id != cur_stream_id && pre_stream_id < cur_stream_id && + AnfAlgo::GetCNodeName(pre_cnode_ptr) != "StreamSwitch" && + AnfAlgo::GetCNodeName(pre_cnode_ptr) != "StreamActive"; + bool processed = IsProcessedParallelStream(cur_stream_id); + // 1)inner stream assign, need insert active op + if (inner_active && !processed) { + MS_LOG(INFO) << "Inner insert active op, self stream id[" << pre_stream_id << "]"; + CNodePtr active_ptr = KernelAdjust::GetInstance().CreateSteamActiveOp(graph_ptr); + update_cnode_list.emplace_back(active_ptr); + update_cnode_list.emplace_back(cur_cnode_ptr); + + // 1.set stream id + AnfAlgo::SetStreamId(pre_stream_id, active_ptr.get()); + // 2.set active stream ids + vector active_index_list = GetParallelStream(cur_stream_id, pre_stream_id); + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_index_list), active_ptr); + } else if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == "StreamActive" && + AnfAlgo::GetStreamDistinctionLabel(cur_cnode_ptr.get()) != UINT32_MAX) { + // 2)outter stream assign, update active op + update_cnode_list.emplace_back(cur_cnode_ptr); + UpdateStreamActive(cur_cnode_ptr); + } else if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == "StreamSwitch") { + // 3)update switch op + MS_LOG(INFO) << "Insert active op after switch"; + CNodePtr active_ptr = KernelAdjust::GetInstance().CreateSteamActiveOp(graph_ptr); + update_cnode_list.emplace_back(cur_cnode_ptr); + update_cnode_list.emplace_back(active_ptr); + UpdateStreamSwitch(cur_cnode_ptr, active_ptr); + } else { + update_cnode_list.emplace_back(cur_cnode_ptr); + } + + pre_stream_id = cur_stream_id; + pre_cnode_ptr = cur_cnode_ptr; + } + graph_ptr->set_execution_order(update_cnode_list); + MS_LOG(INFO) << "end"; +} + +void AscendStreamAssign::UpdateStreamId(const shared_ptr &graph_ptr) { + MS_LOG(INFO) << "start"; + MS_EXCEPTION_IF_NULL(graph_ptr); + CNodePtr cur_cnode_ptr = nullptr; + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + if (cur_stream_id < kIndependFirstStreamId) { + if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == "StreamActive") { + auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); + MS_EXCEPTION_IF_NULL(primitive); + vector active_ids = GetValue>(primitive->GetAttr(kAttrActiveStreamList)); + for (size_t j = 0; j < active_ids.size(); j++) { + if (active_ids[j] >= kIndependFirstStreamId) { + active_ids[j] = active_ids[j] - kIndependFirstStreamId + total_common_stream_num_; + } + } + ValuePtr active_value = MakeValue>(active_ids); + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, active_value, cur_cnode_ptr); + } + } else { + uint32_t update_id = cur_stream_id - kIndependFirstStreamId + total_common_stream_num_; + AnfAlgo::SetStreamId(update_id, cur_cnode_ptr.get()); + } + + // update AtomicAddrClean stream same witch the next node + if (i > 0 && AnfAlgo::GetCNodeName(cnode_ptr_list[i - 1]) == "AtomicAddrClean") { + MS_LOG(INFO) << "update AtomicAddrClean stream id from[" << AnfAlgo::GetStreamId(cnode_ptr_list[i - 1]) + << "] to [" << AnfAlgo::GetStreamId(cur_cnode_ptr) << "]"; + AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(cur_cnode_ptr), cnode_ptr_list[i - 1].get()); + } + } + + // update logic_to_independent_map_ + for (auto &indep : logic_to_independent_map_) { + if (indep.second >= kIndependFirstStreamId) { + indep.second = indep.second - kIndependFirstStreamId + total_common_stream_num_; + } + } + + // update independent_before_physic_id_ + for (auto &id : independent_before_physic_id_) { + if (id >= kIndependFirstStreamId) { + id = id - kIndependFirstStreamId + total_common_stream_num_; + } + } + + // update independent_id_ + independent_id_ = independent_id_ - kIndependFirstStreamId + total_common_stream_num_; + MS_LOG(INFO) << "end"; +} + +void AscendStreamAssign::AssignStreamNew(const shared_ptr &graph_ptr) { + if (IsTaskSink()) { + ResetNew(); + AssignAllNodesStream(graph_ptr); + FindAllReduceParallel(graph_ptr); + InsertActiveNew(graph_ptr); + InsertSendRecvForHcomParallel(graph_ptr); + InsertSendRecvForIndependent(graph_ptr); + UpdateStreamId(graph_ptr); + + MS_LOG(INFO) << "after finish stream assign"; + PrintGraphExeOrders(graph_ptr); + + // Get info for D Model + generator::IRModelUtil::GetInstance().set_event_num(GetTotalEventNum()); + generator::IRModelUtil::GetInstance().set_stream_num(GetTotalCommonStreamNum() + GetTotalIndependStreamNum()); + // Init to 1,temporarily + generator::IRModelUtil::GetInstance().set_batch_num(1); + } +} + +CNodePtr AscendStreamAssign::CreateSendApplyKernel(const std::shared_ptr &graph_ptr, + uint32_t event_id, uint32_t stream_id) { + MS_EXCEPTION_IF_NULL(graph_ptr); + auto send_op = std::make_shared("Send"); + MS_EXCEPTION_IF_NULL(send_op); + auto send_apply = std::make_shared(send_op); + MS_EXCEPTION_IF_NULL(send_apply); + std::vector send_input_list = {send_apply}; + CNodePtr send_node_ptr = graph_ptr->NewCNode(send_input_list); + MS_EXCEPTION_IF_NULL(send_node_ptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder; + selected_kernel_builder.SetKernelType(KernelType::RT_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), send_node_ptr.get()); + AnfAlgo::SetNodeAttr("event_id", MakeValue(event_id), send_node_ptr); + auto abstract_none = std::make_shared(); + MS_EXCEPTION_IF_NULL(abstract_none); + send_node_ptr->set_abstract(abstract_none); + AnfAlgo::SetStreamId(stream_id, send_node_ptr.get()); + return send_node_ptr; +} + +CNodePtr AscendStreamAssign::CreateRecvApplyKernel(const std::shared_ptr &graph_ptr, + uint32_t event_id, uint32_t stream_id) { + MS_EXCEPTION_IF_NULL(graph_ptr); + auto recv_op = std::make_shared("Recv"); + MS_EXCEPTION_IF_NULL(recv_op); + auto recv_apply = std::make_shared(recv_op); + MS_EXCEPTION_IF_NULL(recv_apply); + std::vector recv_input_list = {recv_apply}; + CNodePtr recv_node_ptr = graph_ptr->NewCNode(recv_input_list); + MS_EXCEPTION_IF_NULL(recv_node_ptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder; + selected_kernel_builder.SetKernelType(KernelType::RT_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), recv_node_ptr.get()); + AnfAlgo::SetNodeAttr("event_id", MakeValue(event_id), recv_node_ptr); + AnfAlgo::SetStreamId(stream_id, recv_node_ptr.get()); + auto abstract_none = std::make_shared(); + MS_EXCEPTION_IF_NULL(abstract_none); + recv_node_ptr->set_abstract(abstract_none); + return recv_node_ptr; +} + +vector::iterator AscendStreamAssign::FindTargetOp(vector::iterator begin, + vector::iterator end, const CNodePtr &node) { + while (begin != end) { + auto inputs = (*begin)->inputs(); + for (size_t i = 1; i < inputs.size(); i++) { + auto input = inputs[i]; + if (opt::IsNopNode(input)) { + CNodePtr cnode = input->cast(); + auto new_inputs = cnode->inputs(); + for (size_t j = 1; j < new_inputs.size(); j++) { + auto new_real_input = AnfAlgo::VisitKernel(new_inputs[j], 0); + if (node == new_real_input.first) { + MS_LOG(INFO) << "Nop node find target op[" << (*begin)->DebugString() << "]"; + return begin; + } + } + } else { + auto real_input = AnfAlgo::VisitKernel(input, 0); + if (node == real_input.first) { + MS_LOG(INFO) << "find target op[" << (*begin)->DebugString() << "]"; + return begin; + } + } + } + ++begin; + } + return end; +} // namespace ascend + +void AscendStreamAssign::InsertSendRecvForIndependent(const shared_ptr &graph_ptr) { + MS_LOG(INFO) << "start"; + MS_EXCEPTION_IF_NULL(graph_ptr); + auto cnode_ptr_list = graph_ptr->execution_order(); + vector cnodes = cnode_ptr_list; + uint32_t cur_event_id = total_event_num_; + auto it = cnodes.begin(); + while (it != cnodes.end()) { + MS_EXCEPTION_IF_NULL(*it); + if (IsIndependentNode(*it)) { + MS_LOG(INFO) << "deal independent op[" << (*it)->DebugString() << "]"; + CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); + it = cnodes.insert(it + 1, send_cnode_ptr); + + auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); + if (target == cnodes.end()) { + MS_LOG(DEBUG) << "independ node[" << (*(it - 1))->fullname_with_scope() + << "] can't find target for insert recv op, no insert send/recv"; + it = cnodes.erase(it); + continue; + } + + // deal recv op + uint32_t stream_id = AnfAlgo::GetStreamId(*target); + CNodePtr recv_cnode_ptr = CreateRecvApplyKernel(graph_ptr, cur_event_id, stream_id); + (void)cnodes.insert(target, recv_cnode_ptr); + ++cur_event_id; + } + ++it; + } + graph_ptr->set_execution_order(cnodes); + total_event_num_ = cur_event_id; + MS_LOG(INFO) << "total event nums[" << total_event_num_ << "]"; + MS_LOG(INFO) << "end"; +} + +bool AscendStreamAssign::IsIndependentNode(const CNodePtr &node_ptr) { + MS_EXCEPTION_IF_NULL(node_ptr); + if (AnfAlgo::GetKernelType(node_ptr) != AICPU_KERNEL) { + return false; + } + + if (AnfAlgo::GetCNodeName(node_ptr) == "GetNext") { + MS_LOG(INFO) << "GetNext should not be independent node"; + return false; + } + + uint32_t input_nums = AnfAlgo::GetInputTensorNum(node_ptr); + if (input_nums == 0) { + MS_LOG(INFO) << "node " << node_ptr->fullname_with_scope() << " is independent, as inputs nums is zero"; + return true; + } + + auto inputs = node_ptr->inputs(); + for (size_t i = 1; i < inputs.size(); i++) { + if (!inputs[i]->isa()) { + return false; + } + } + MS_LOG(INFO) << "node " << node_ptr->fullname_with_scope() << " is independent, as inputs is all value node"; + return true; +} + +bool AscendStreamAssign::IsTaskSink() { + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (!ms_context->enable_task_sink()) { + MS_LOG(INFO) << "task sink mode is not enable"; + return false; + } else { + MS_LOG(INFO) << "task sink mode is enable"; + return true; + } +} + +std::vector AscendStreamAssign::GetWaitStreams() { + vector wait_active_stream_list; + if (total_common_stream_num_ == 0) { + MS_LOG(INFO) << "total_common_stream_num is zero"; + return wait_active_stream_list; + } + + // common stream:active first common stream + MS_LOG(INFO) << "active physic id[" << first_physic_id_ << "]"; + for (uint32_t i = first_physic_id_ + 1; i < total_common_stream_num_; i++) { + MS_LOG(INFO) << "wait common stream id = " << i; + wait_active_stream_list.push_back(i); + } + + auto it = logic_to_independent_map_.find(first_logic_id_); + if (it != logic_to_independent_map_.end()) { + uint32_t independent_id = it->second; + auto res = std::find(independent_before_physic_id_.begin(), independent_before_physic_id_.end(), independent_id); + if (res == independent_before_physic_id_.end()) { + // first physical to independ id may be not in independent_before_physic_id_ + independent_before_physic_id_.push_back(independent_id); + } + MS_LOG(INFO) << "active independent id[" << independent_id << "]"; + } + + uint32_t max_before_physic = 0; + for (size_t i = 0; i < independent_before_physic_id_.size(); i++) { + if (independent_before_physic_id_[i] > max_before_physic) { + max_before_physic = independent_before_physic_id_[i]; + } + MS_LOG(INFO) << "independent id[" << independent_before_physic_id_[i] << "] before first physic is active"; + } + + for (uint32_t i = 0; i < total_independ_stream_num_; i++) { + if (i + total_common_stream_num_ <= max_before_physic) { + continue; + } + MS_LOG(INFO) << "wait independent stream id:" << i + total_common_stream_num_; + wait_active_stream_list.push_back(i + total_common_stream_num_); + } + + return wait_active_stream_list; +} + +std::vector AscendStreamAssign::GetHcomStreams() { + MS_LOG(INFO) << "hcom total stream nums:" << hcom_stream_list_.size(); + return hcom_stream_list_; +} + +uint32_t AscendStreamAssign::GetTotalStreamNum() const { return total_common_stream_num_ + total_independ_stream_num_; } + +void AscendStreamAssign::PrintGraphExeOrders(const shared_ptr &graph_ptr) { + MS_EXCEPTION_IF_NULL(graph_ptr); + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == "Send" || AnfAlgo::GetCNodeName(cur_cnode_ptr) == "Recv") { + auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); + MS_LOG(INFO) << "node name[" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "], logic id[" + << AnfAlgo::GetStreamDistinctionLabel(cur_cnode_ptr.get()) << "], stream id[" + << AnfAlgo::GetStreamId(cur_cnode_ptr) << "], event_id[" + << GetValue(primitive->GetAttr(kAttrEventId)) << "]"; + } else { + MS_LOG(INFO) << "node name[" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "], logic id[" + << AnfAlgo::GetStreamDistinctionLabel(cur_cnode_ptr.get()) << "], stream id[" + << AnfAlgo::GetStreamId(cur_cnode_ptr) << "]"; + } + } +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.h b/mindspore/ccsrc/device/ascend/ascend_stream_assign.h new file mode 100755 index 00000000000..6c4ef01da87 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.h @@ -0,0 +1,121 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_STREAM_ASSIGN_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_STREAM_ASSIGN_H_ + +#include +#include +#include +#include +#include +#include +#include "runtime/base.h" +#include "runtime/rt_model.h" +#include "runtime/stream.h" +#include "session/kernel_graph.h" + +namespace mindspore { +namespace device { +namespace ascend { +using std::map; +using std::shared_ptr; +using std::unordered_map; +using std::unordered_set; +using std::vector; + +class AscendStreamAssign { + public: + static AscendStreamAssign& GetInstance() { + static AscendStreamAssign instance; // Guaranteed to be destroyed. + return instance; + } + + AscendStreamAssign(const AscendStreamAssign&) = delete; + AscendStreamAssign& operator=(const AscendStreamAssign&) = delete; + + uint32_t GetTotalStreamNum() const; + // new stream policy + uint32_t GetTotalCommonStreamNum() const { return total_common_stream_num_; } + uint32_t GetTotalIndependStreamNum() const { return total_independ_stream_num_; } + uint32_t GetTotalEventNum() const { return total_event_num_; } + const uint32_t GetFisrtPhysicId() const { return first_physic_id_; } + const uint32_t GetFirstLogicId() const { return first_logic_id_; } + + void InsertActiveNew(const std::shared_ptr& graph_ptr); + void AssignAllNodesStream(const std::shared_ptr& graph_ptr); + void ResetNew(); + void AssignStreamNew(const std::shared_ptr& graph_ptr); + bool IsIndependentNode(const CNodePtr& node_ptr); + const std::unordered_map GetIndependentMap() { return logic_to_independent_map_; } + const std::unordered_map GetPhysicMap() { return logic_to_physic_map_; } + std::vector GetWaitStreams(); + std::vector GetHcomStreams(); + + private: + AscendStreamAssign() = default; + ~AscendStreamAssign() = default; + + CNodePtr CreateSendApplyKernel(const std::shared_ptr& graph_ptr, uint32_t event_id, + uint32_t stream_id); + CNodePtr CreateRecvApplyKernel(const std::shared_ptr& graph_ptr, uint32_t event_id, + uint32_t stream_id); + + vector::iterator FindTargetOp(vector::iterator begin, vector::iterator end, + const CNodePtr& node); + + bool IsHcom(const CNodePtr& apply_kernel); + bool IsProcessed(uint32_t logic_id); + vector TransLogicToPhysic(const vector& logic_ids); + void AssignCommonStreamId(const CNodePtr& cur_cnode_ptr, CNodePtr* pre_cnode_ptr, uint32_t* cur_index, + uint32_t* cur_stream_id); + void RecordIdMap(uint32_t logic_id, uint32_t physic_id); + void UpdateStreamActive(const CNodePtr& active_ptr); + void UpdateStreamSwitch(const CNodePtr& switch_ptr, const CNodePtr& active_ptr); + bool IsTaskSink(); + void AssignIndependentStreamId(const CNodePtr& cur_cnode_ptr, uint32_t deal_logic_id); + void UpdateStreamId(const std::shared_ptr& graph_ptr); + void PrintGraphExeOrders(const std::shared_ptr& graph_ptr); + void RecordFirstCommonOp(const CNodePtr& cur_cnode_ptr, uint32_t cur_node_logic_id, uint32_t cur_stream_id); + uint32_t GetLogicId(const CNodePtr& cur_cnode_ptr); + void SetCommonStreamNum(uint32_t cur_stream_id); + void FindAllReduceParallel(const std::shared_ptr& graph_ptr); + bool IsProcessedParallelStream(uint32_t stream_id); + vector GetParallelStream(uint32_t cur_stream_id, uint32_t stream_acitve_id); + void InsertSendRecvForIndependent(const std::shared_ptr& graph_ptr); + void InsertSendRecvForHcomParallel(const std::shared_ptr& graph_ptr); + + uint32_t total_common_stream_num_{0}; + uint32_t total_independ_stream_num_{0}; + uint32_t total_event_num_{0}; + + uint32_t first_physic_id_{UINT32_MAX}; + uint32_t first_logic_id_{UINT32_MAX}; + uint32_t independent_id_{UINT32_MAX}; + vector processed_logic_id_{}; + std::unordered_map logic_to_physic_map_{}; // key:logic id, value: first physic id + std::unordered_map logic_to_independent_map_{}; // key:logic id, value: dependent id + std::vector independent_before_physic_id_{}; // record independent id before first physic id + std::vector> inner_parallel_streams_{}; + std::vector processed_parallel_streams_{}; + std::vector hcom_stream_list_{}; + // new policy end +}; +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_ASCEND_STREAM_ASSIGN_H_ diff --git a/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc b/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc new file mode 100644 index 00000000000..66ce697ffca --- /dev/null +++ b/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc @@ -0,0 +1,235 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/kernel_build_ascend.h" + +#include +#include +#include +#include + +#include "device/ascend/kernel_select_ascend.h" +#include "device/kernel_info.h" +#include "kernel/kernel.h" +#include "kernel/tbe/tbe_kernel_build.h" +#include "kernel/tbe/tbe_kernel_parallel_build.h" +#include "kernel/aicpu/aicpu_kernel_build.h" +#include "kernel/hccl/hccl_kernel_build.h" +#include "kernel/mng/rt_kernel_build.h" +#include "kernel/tbe/tbe_utils.h" +#include "operator/ops.h" +#include "session/anf_runtime_algorithm.h" +#include "./common.h" + +namespace mindspore { +namespace device { +namespace ascend { +using mindspore::kernel::tbe::TbeUtils; +using std::make_shared; +static kernel::KernelModPtr SerialCompileImpl(const AnfNodePtr &anf_node) { + kernel::KernelModPtr kernel_mod_ptr = nullptr; + KernelType kernel_type = AnfAlgo::GetKernelType(anf_node); + switch (kernel_type) { + case KernelType::AICPU_KERNEL: { + kernel_mod_ptr = kernel::AicpuOpBuild(anf_node); + break; + } + case KernelType::RT_KERNEL: { + kernel_mod_ptr = kernel::RtOpBuild(anf_node); + break; + } + case KernelType::HCCL_KERNEL: { + kernel_mod_ptr = kernel::HcclOpBuild(anf_node); + break; + } + default: { + MS_LOG(EXCEPTION) << "node [" << anf_node->DebugString() << "] Unsupported kernel_type:" << kernel_type; + } + } + return kernel_mod_ptr; +} + +static bool KernelBuildParallelCompile(const mindspore::session::KernelGraph *kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + std::vector tbe_nodes; + std::vector other_nodes; + for (const auto &anf_node : kernel_graph_ptr->execution_order()) { + MS_EXCEPTION_IF_NULL(anf_node); + if (!AnfAlgo::IsRealKernel(anf_node)) { + continue; + } + KernelType kernel_type = AnfAlgo::GetKernelType(anf_node); + switch (kernel_type) { + case KernelType::TBE_KERNEL: { + if (AnfAlgo::GetKernelMod(anf_node) == nullptr) { + tbe_nodes.push_back(anf_node); + } + break; + } + default: { + other_nodes.push_back(anf_node); + break; + } + } + } + bool ret = kernel::TbeOpParallelBuild(tbe_nodes); + for (const auto &anf_node : other_nodes) { + kernel::KernelModPtr kernel_mod_ptr = SerialCompileImpl(anf_node); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + } + return ret; +} + +static vector CalCleanZerosSize(const CNodePtr &pre_node) { + MS_EXCEPTION_IF_NULL(pre_node); + std::vector clean_size_list; + // clean output + if (AnfAlgo::HasNodeAttr(kAttrAutomicOutputIndexs, pre_node)) { + auto clean_output_indexs = AnfAlgo::GetNodeAttr>(pre_node, kAttrAutomicOutputIndexs); + for (auto index : clean_output_indexs) { + TypeId output_type_id = AnfAlgo::GetOutputDeviceDataType(pre_node, index); + size_t type_size = GetTypeByte(TypeIdToType(output_type_id)); + std::vector shape = AnfAlgo::GetOutputDeviceShape(pre_node, index); + auto size = std::accumulate(shape.begin(), shape.end(), type_size, std::multiplies()); + clean_size_list.push_back((size + kMemAlignSize + 31) / kMemAlignSize * kMemAlignSize); + } + } + // clean workspace + auto workspaces_size = 0; + if (AnfAlgo::HasNodeAttr(kAttrAutomicWorkspaceSize, pre_node)) { + workspaces_size = AnfAlgo::GetNodeAttr(pre_node, kAttrAutomicWorkspaceSize); + clean_size_list.push_back(workspaces_size); + } + MS_LOG(INFO) << "clear output size:" << clean_size_list.size() << ", workspace size:" << workspaces_size + << ",pre_node:" << pre_node->fullname_with_scope(); + return clean_size_list; +} + +static void AddTbeClearZeroNode(mindspore::session::KernelGraph *const kernel_graph, + const mindspore::CNodePtr &pre_node, std::vector *new_nodes) { + MS_EXCEPTION_IF_NULL(kernel_graph); + MS_EXCEPTION_IF_NULL(pre_node); + MS_EXCEPTION_IF_NULL(new_nodes); + auto clear_zero_prim = std::make_shared(kAtomicAddrCleanOpName); + MS_EXCEPTION_IF_NULL(clear_zero_prim); + auto new_value_node = NewValueNode(clear_zero_prim); + MS_EXCEPTION_IF_NULL(new_value_node); + std::vector inputs = {new_value_node}; + inputs.push_back(pre_node); + CNodePtr clear_zero = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(clear_zero); + AbstractBasePtr abstract = std::make_shared(); + MS_EXCEPTION_IF_NULL(abstract); + clear_zero->set_abstract(abstract); + auto builder = std::make_shared(); + builder->SetKernelType(KernelType::TBE_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), clear_zero.get()); + auto clean_size = CalCleanZerosSize(pre_node); + AnfAlgo::SetNodeAttr(kAttrAutomicAddMemSize, MakeValue(clean_size), clear_zero); + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(pre_node.get()), clear_zero.get()); + new_nodes->push_back(clear_zero); +} + +bool IsAtomicNode(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto parameters_indexs = kernel_mod->GenParameters(); + if (parameters_indexs.empty()) { + return false; + } + auto atomic_flag = false; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + auto workspace_size_list = kernel_mod->GetWorkspaceSizeList(); + size_t workspace_num = kernel_mod->GetWorkspaceSizeList().size(); + if (input_num + workspace_num + output_num > parameters_indexs.size()) { + size_t lossNum = (input_num + workspace_num + output_num) - parameters_indexs.size(); + for (size_t i = 0; i < lossNum; i++) { + parameters_indexs.push_back(0); + } + } + std::vector clean_output_indexs; + // in parameters data sort as input->workspace->output + size_t index = 0; + while (index < output_num) { + if (parameters_indexs[input_num + workspace_num + index] == 1) { + atomic_flag = true; + clean_output_indexs.push_back(index); + } + index++; + } + if (atomic_flag) { + AnfAlgo::SetNodeAttr(kAttrAutomicOutputIndexs, MakeValue(clean_output_indexs), kernel_node); + } + for (size_t i = 0; i < workspace_num; ++i) { + if (parameters_indexs[input_num + i] == 1) { + atomic_flag = true; + AnfAlgo::SetNodeAttr(kAttrAutomicWorkspaceSize, + MakeValue(std::accumulate(workspace_size_list.begin(), workspace_size_list.end(), 0)), + kernel_node); + break; + } + } + return atomic_flag; +} + +bool KernelBuild(const mindspore::session::KernelGraph *kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + TbeUtils::LoadCache(); + bool ret; + ret = device::ascend::KernelBuildParallelCompile(kernel_graph_ptr); + return ret; +} + +void KernelBuildPreprocess(mindspore::session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + std::vector new_nodes; + for (const auto &anf_node : kernel_graph->execution_order()) { + std::string apply_function_name = AnfAlgo::GetCNodeName(anf_node); + if (apply_function_name == prim::kPrimMaxPoolGrad->name() && + AnfAlgo::GetKernelType(anf_node) == KernelType::AUTO_DIFF_KERNEL) { + auto clear_zero_prim = std::make_shared(kClearZeroOpName); + MS_EXCEPTION_IF_NULL(clear_zero_prim); + auto new_value_node = NewValueNode(clear_zero_prim); + MS_EXCEPTION_IF_NULL(new_value_node); + std::vector inputs = {new_value_node}; + inputs.push_back(anf_node); + CNodePtr clear_zero = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(clear_zero); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + clear_zero->set_kernel_info(kernel_info); + AbstractBasePtr abstract = std::make_shared(); + MS_EXCEPTION_IF_NULL(abstract); + AnfAlgo::SetNodeAttr("input_names", MakeValue(std::vector({"x"})), clear_zero); + SelectKernelInfo(clear_zero); + // set the distinction label of clear same with anf + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(anf_node.get()), clear_zero.get()); + new_nodes.push_back(clear_zero); + } else if (AnfAlgo::GetKernelType(anf_node) == KernelType::TBE_KERNEL) { + if (IsAtomicNode(anf_node)) { + AddTbeClearZeroNode(kernel_graph, anf_node, &new_nodes); + } + } + new_nodes.push_back(anf_node); + } + kernel_graph->set_execution_order(new_nodes); +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/kernel_build_ascend.h b/mindspore/ccsrc/device/ascend/kernel_build_ascend.h new file mode 100644 index 00000000000..5dea36a1835 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/kernel_build_ascend.h @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_KERNEL_BUILD_ASCEND_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_KERNEL_BUILD_ASCEND_H_ + +#include "session/kernel_graph.h" + +namespace mindspore { +namespace device { +namespace ascend { +/** + * @brief kernel build for ascend. + */ +bool KernelBuild(const mindspore::session::KernelGraph *kernel_graph_ptr); +/** + * @brief preporcess of kernel build for ascend, e.g. inserting clear_zero node for maxpool, bn. + * Must DO these changes just before kernel build, and after all of other optimizations on AnfGraph + */ +void KernelBuildPreprocess(mindspore::session::KernelGraph *kernel_graph); +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_KERNEL_BUILD_ASCEND_H_ diff --git a/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc b/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc new file mode 100644 index 00000000000..347d63be39f --- /dev/null +++ b/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc @@ -0,0 +1,573 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/kernel_select_ascend.h" +#include +#include +#include +#include +#include +#include "kernel/oplib/oplib.h" +#include "kernel/kernel_query.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/kernel_build_info.h" +#include "utils/context/ms_context.h" +#include "operator/ops.h" + +namespace mindspore { +namespace device { +namespace ascend { +namespace { +enum MatchCountPriority : int { + MATCH_COUNT_PRIORITY_BEGIN = 0, + MATCH_DTYPE_COUNT = MATCH_COUNT_PRIORITY_BEGIN, + MATCH_FORMAT_COUNT, + MATCH_5D_FORMAT_COUNT, + MATCH_OUTPUT_DTYPE_COUNT, + MATCH_COUNT_PRIORITY_END +}; + +const size_t kMaxCount = 0xffffffff; +const int kUnSupportMixedDataTypeIndex = -1; + +const std::set kOpFormatList = { + kOpFormat_DEFAULT, kOpFormat_NC1KHKWHWC0, kOpFormat_ND, kOpFormat_NCHW, kOpFormat_NHWC, + kOpFormat_HWCN, kOpFormat_NC1HWC0, kOpFormat_FRAC_Z, kOpFormat_C1HWNCoC0, kOpFormat_FRAC_NZ}; + +bool IsShapeMatchFormat(const std::vector &shape, const std::string &format) { + // if format is default, it remarkes support all format + if (kOpFormatList.find(format) == kOpFormatList.end()) { + MS_LOG(EXCEPTION) << "got the unknown format " << format; + } + if (format == kOpFormat_DEFAULT) { + return true; + } + // if shape size is 0, the shape will be a scalar + if (shape.empty()) { + return true; + } + if (shape.size() > kShapeSupportFormatMap.size()) { + return false; + } + if (format == kOpFormat_FRAC_NZ && shape.size() >= 2) { + return true; + } + return !(kShapeSupportFormatMap[shape.size() - 1].find(format) == kShapeSupportFormatMap[shape.size() - 1].end()); +} + +bool IsValidKernelInfo(const std::shared_ptr &kernel_node, const kernel::KernelBuildInfo &kernel_build_info) { + MS_EXCEPTION_IF_NULL(kernel_node); + auto check_function = [](const std::vector &shape, const std::string &format) -> bool { + if (!IsShapeMatchFormat(shape, format)) { + return false; + } + for (auto shape_value : shape) { + if (shape_value == 0) { + MS_LOG(EXCEPTION) << "dimension size of the tensor shape should be a positive integer, but got " << shape_value; + } + } + return true; + }; + if (AnfAlgo::GetCNodeName(kernel_node) == prim::kPrimCast->name()) { + return AnfAlgo::GetOutputInferDataType(kernel_node, 0) == kernel_build_info.GetOutputDeviceType(0) && + AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0) == kernel_build_info.GetInputDeviceType(0); + } + for (size_t index = 0; index < kernel_build_info.GetOutputNum(); ++index) { + auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, index); + if (!check_function(output_shape, kernel_build_info.GetOutputFormat(index))) { + return false; + } + } + for (size_t index = 0; index < kernel_build_info.GetInputNum(); ++index) { + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, index); + if (!check_function(input_shape, kernel_build_info.GetInputFormat(index))) { + return false; + } + } + return true; +} + +bool MatchInferOutputDataType(const CNodePtr &cnode, const kernel::KernelBuildInfo &kernel_build_info) { + MS_EXCEPTION_IF_NULL(cnode); + // Check input data type + for (size_t input_index = 0; input_index < kernel_build_info.GetInputNum(); ++input_index) { + AnfNodePtr cur_input = AnfAlgo::GetInputNode(cnode, input_index); + MS_EXCEPTION_IF_NULL(cur_input); + TypeId input_origin_type; + if (cur_input->isa() && AnfAlgo::IsParameterWeight(cur_input->cast())) { + // weight + input_origin_type = AnfAlgo::GetOutputDeviceDataType(cur_input, 0); + } else if (cur_input->isa()) { + input_origin_type = AnfAlgo::GetOutputDeviceDataType(cur_input, 0); + } else { + // feature map + input_origin_type = AnfAlgo::GetPrevNodeOutputInferDataType(cnode, input_index); + } + if (input_origin_type == kTypeUnknown) { + continue; + } + if (kernel_build_info.GetInputDeviceType(input_index) != input_origin_type) { + return false; + } + } + // Check output data type + for (size_t output_index = 0; output_index < kernel_build_info.GetOutputNum(); ++output_index) { + if (kernel_build_info.GetOutputDeviceType(output_index) != AnfAlgo::GetOutputInferDataType(cnode, output_index)) { + return false; + } + } + return true; +} + +/** + * compare two vector by priority, select a better vector, like compare two num, first compare highest num location, + * if equal then next num location + * example:[3,1,1,1] > [2,2,2,2] > [2,2,1,2] > [2,1,1,3] + */ +bool PriorityChooseItem(const std::vector &cur_item, std::vector *best_item) { + MS_EXCEPTION_IF_NULL(best_item); + if (cur_item.size() != best_item->size()) { + MS_LOG(ERROR) << "item size should be same!"; + return false; + } + // Update the best_item by comparing the cur_item and best_item + for (size_t i = 0; i < cur_item.size(); i++) { + if (cur_item[i] > best_item->at(i)) { + *best_item = cur_item; + return true; + } else if (cur_item[i] == best_item->at(i)) { + continue; + } else { + return false; + } + } + return false; +} + +void UpdateCurMatchCounts(const kernel::KernelBuildInfo &kernel_build_info, const std::shared_ptr &kernel_node, + std::vector *const cur_kernelinfo_match_counts) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(cur_kernelinfo_match_counts); + if (cur_kernelinfo_match_counts->size() < MATCH_COUNT_PRIORITY_END) { + MS_LOG(EXCEPTION) << "Out of range cur_kernelinfo_match_counts " << MATCH_COUNT_PRIORITY_END; + } + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(kernel_node); ++input_index) { + AnfNodePtr input_anf_node = AnfAlgo::GetInputNode(kernel_node, input_index); + MS_EXCEPTION_IF_NULL(input_anf_node); + // if a input parameter is a weight with default format, the input shouldn't participate the judge + if (input_anf_node->isa()) { + auto para = input_anf_node->cast(); + if (AnfAlgo::IsParameterWeight(para) && AnfAlgo::GetOutputDeviceDataType(para, 0) == kTypeUnknown) { + continue; + } + } + if (input_anf_node->isa()) { + if (AnfAlgo::GetOutputDeviceDataType(input_anf_node, 0) == kTypeUnknown) { + continue; + } + } + if (kernel_build_info.GetInputFormat(input_index) == AnfAlgo::GetPrevNodeOutputFormat(kernel_node, input_index)) { + (*cur_kernelinfo_match_counts)[MATCH_FORMAT_COUNT]++; + } + if (kernel_build_info.GetInputDeviceType(input_index) == + AnfAlgo::GetPrevNodeOutputDeviceDataType(kernel_node, input_index)) { + (*cur_kernelinfo_match_counts)[MATCH_DTYPE_COUNT]++; + } + if (kernel_build_info.GetInputFormat(input_index) == kOpFormat_NC1HWC0) { + // input is from a feature map & this input's shape is not 4d + if (AnfAlgo::IsFeatureMapInput(kernel_node, input_index) && + AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, input_index).size() != kShape4dDims) { + continue; + } + (*cur_kernelinfo_match_counts)[MATCH_5D_FORMAT_COUNT]++; + } + } + + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(kernel_node); ++output_index) { + // cal count of same output dtype between abstract and kernel info + if (kernel_build_info.GetOutputDeviceType(output_index) == + AnfAlgo::GetOutputInferDataType(kernel_node, output_index)) { + (*cur_kernelinfo_match_counts)[MATCH_OUTPUT_DTYPE_COUNT]++; + } + } +} + +void SetTensorDeviceInfo(const kernel::KernelBuildInfo &selected_kernel_info, const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(kernel_node); ++input_index) { + auto input_kernel_node = AnfAlgo::GetInputNode(kernel_node, input_index); + MS_EXCEPTION_IF_NULL(input_kernel_node); + if (AnfAlgo::IsFeatureMapInput(kernel_node, input_index)) { + continue; + } + auto input_with_index = AnfAlgo::VisitKernel(input_kernel_node, 0); + MS_EXCEPTION_IF_NULL(input_with_index.first); + auto real_input_node = input_with_index.first; + std::shared_ptr builder = + std::make_shared(); + // we set special device info of a input tensor. + bool is_ref = false; + auto op_info = mindspore::kernel::OpLib::FindOp(AnfAlgo::GetCNodeName(kernel_node), kernel::kTBE); + if (op_info != nullptr) { + is_ref = op_info->is_ref(); + } + MS_EXCEPTION_IF_NULL(MsContext::GetInstance()); + if (MsContext::GetInstance()->execution_mode() == kPynativeMode && + AnfAlgo::GetOutputDeviceDataType(real_input_node, 0) != kTypeUnknown) { + continue; + } + if (AnfAlgo::GetOutputDeviceDataType(real_input_node, 0) == kTypeUnknown || is_ref) { + std::vector output_format = {selected_kernel_info.GetInputFormat(input_index)}; + builder->SetOutputsFormat(output_format); + std::vector output_type = {selected_kernel_info.GetInputDeviceType(input_index)}; + builder->SetOutputsDeviceType(output_type); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), real_input_node.get()); + } + } +} + +void AddSupportMixedPrecisionDataTypeIndex(TypeId data_type, std::vector *support_index) { + int index = kUnSupportMixedDataTypeIndex; + switch (data_type) { + case kNumberTypeFloat16: + index = 0; + break; + case kNumberTypeFloat32: + case kNumberTypeFloat: + index = 1; + break; + default: + break; + } + support_index->push_back(index); +} + +void AddKernelInputSupportDataType(const kernel::KernelBuildInfo &kernel_build_info, size_t input_index, + std::vector *support_datatype_index, std::vector *support_datatype) { + auto data_type = kernel_build_info.GetInputDeviceType(input_index); + support_datatype->push_back(data_type); + AddSupportMixedPrecisionDataTypeIndex(data_type, support_datatype_index); +} + +void AddKernelOutputSupportDataType(const kernel::KernelBuildInfo &kernel_build_info, size_t output_index, + std::vector *support_datatype_index, std::vector *support_datatype) { + auto data_type = kernel_build_info.GetOutputDeviceType(output_index); + support_datatype->push_back(data_type); + AddSupportMixedPrecisionDataTypeIndex(data_type, support_datatype_index); +} + +void AddNodeInputDataType(const CNodePtr &kernel_node, size_t input_index, + std::vector *node_mix_precision_datatype_index, + std::vector *node_mix_precision_datatype) { + AnfNodePtr cur_input = AnfAlgo::GetInputNode(kernel_node, input_index); + MS_EXCEPTION_IF_NULL(cur_input); + TypeId input_origin_type; + if (cur_input->isa() && AnfAlgo::IsParameterWeight(cur_input->cast())) { + // weight + input_origin_type = AnfAlgo::GetOutputDeviceDataType(cur_input, 0); + } else if (cur_input->isa()) { + input_origin_type = AnfAlgo::GetOutputDeviceDataType(cur_input, 0); + } else { + // feature map + input_origin_type = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, input_index); + } + AddSupportMixedPrecisionDataTypeIndex(input_origin_type, node_mix_precision_datatype_index); + node_mix_precision_datatype->push_back(input_origin_type); +} + +void AddNodeOutputDataType(const CNodePtr &kernel_node, size_t output_index, + std::vector *node_mix_precision_datatype_index, + std::vector *node_mix_precision_datatype) { + auto output_origin_type = AnfAlgo::GetOutputInferDataType(kernel_node, output_index); + AddSupportMixedPrecisionDataTypeIndex(output_origin_type, node_mix_precision_datatype_index); + node_mix_precision_datatype->push_back(output_origin_type); +} + +void CheckDataTypeInputs(const std::vector &node_mix_precision_datatype_index, + const std::vector &node_mix_precision_datatype, + const std::unordered_map> &kernel_support_datatypes, + std::unordered_map> *kernel_match_datatype_idx) { + if (node_mix_precision_datatype_index.size() != node_mix_precision_datatype.size()) { + MS_LOG(EXCEPTION) << "node datatype index size " << node_mix_precision_datatype_index.size() << " != datatype size " + << node_mix_precision_datatype.size(); + } + MS_EXCEPTION_IF_NULL(kernel_match_datatype_idx); + if (kernel_support_datatypes.size() != kernel_match_datatype_idx->size()) { + MS_LOG(EXCEPTION) << "kernel datatype index size " << kernel_match_datatype_idx->size() << " != datatype size " + << kernel_support_datatypes.size(); + } +} + +int RaiseDataTypePrecisionSelect(const std::vector &node_mix_precision_datatype_index, + const std::vector &node_mix_precision_datatype, + const std::unordered_map> &kernel_support_datatypes, + std::unordered_map> *kernel_match_datatype_idx) { + CheckDataTypeInputs(node_mix_precision_datatype_index, node_mix_precision_datatype, kernel_support_datatypes, + kernel_match_datatype_idx); + for (size_t i = 0; i < node_mix_precision_datatype_index.size(); ++i) { + if (node_mix_precision_datatype[i] == kTypeUnknown) { + continue; + } + auto iter = kernel_match_datatype_idx->begin(); + while (iter != kernel_match_datatype_idx->end()) { + if (node_mix_precision_datatype_index[i] == kUnSupportMixedDataTypeIndex) { + auto find_iter = kernel_support_datatypes.find(iter->first); + if (find_iter == kernel_support_datatypes.end()) { + MS_LOG(EXCEPTION) << "kernel datatype index:%lu can not be found " << iter->first; + } + if (i >= find_iter->second.size()) { + MS_LOG(EXCEPTION) << "node index " << i << "kernel datatype size " << find_iter->second.size(); + } + if (node_mix_precision_datatype[i] != find_iter->second[i]) { + iter = kernel_match_datatype_idx->erase(iter); + } else { + ++iter; + } + continue; + } + auto datatype_indexes = iter->second; + if (i >= datatype_indexes.size()) { + MS_LOG(EXCEPTION) << "node datatype index: " << i << " kernel support size " << datatype_indexes.size(); + } + if (datatype_indexes[i] < node_mix_precision_datatype_index[i]) { + iter = kernel_match_datatype_idx->erase(iter); + } else { + ++iter; + } + } + } + + if (kernel_match_datatype_idx->size() >= 1) { + return SizeToInt(kernel_match_datatype_idx->begin()->first); + } + return -1; +} + +int GetMinReducePrecisionCountIndex(std::unordered_map> *kernel_match_datatype_idx, + const std::unordered_map &precision_reduce_count) { + int selected_index = -1; + size_t min_reduce_precision_count = kMaxCount; + auto iter = kernel_match_datatype_idx->begin(); + while (iter != kernel_match_datatype_idx->end()) { + auto find_iter = precision_reduce_count.find(iter->first); + if (find_iter == precision_reduce_count.end()) { + continue; + } + if (min_reduce_precision_count > find_iter->second) { + selected_index = SizeToInt(iter->first); + min_reduce_precision_count = find_iter->second; + } + ++iter; + } + return selected_index; +} + +int RaiseOrReduceDataTypePrecisionSelect( + const std::vector &node_mix_precision_datatype_index, const std::vector &node_mix_precision_datatype, + const std::unordered_map> &kernel_support_datatypes, + std::unordered_map> *kernel_match_datatype_idx) { + CheckDataTypeInputs(node_mix_precision_datatype_index, node_mix_precision_datatype, kernel_support_datatypes, + kernel_match_datatype_idx); + // reduce / raise + std::unordered_map precision_reduce_count; + for (size_t i = 0; i < node_mix_precision_datatype_index.size(); ++i) { + if (node_mix_precision_datatype[i] == kTypeUnknown) { + continue; + } + auto iter = kernel_match_datatype_idx->begin(); + while (iter != kernel_match_datatype_idx->end()) { + if (node_mix_precision_datatype_index[i] == kUnSupportMixedDataTypeIndex) { + auto find_iter = kernel_support_datatypes.find(iter->first); + if (find_iter == kernel_support_datatypes.end()) { + MS_LOG(EXCEPTION) << "kernel datatype index:%lu can not be found " << iter->first; + } + if (i >= find_iter->second.size()) { + MS_LOG(EXCEPTION) << "node index " << i << " >= kernel datatype size " << find_iter->second.size(); + } + if (node_mix_precision_datatype[i] != find_iter->second[i]) { + iter = kernel_match_datatype_idx->erase(iter); + } else { + ++iter; + } + continue; + } + auto datatype_indexes = iter->second; + if (i >= datatype_indexes.size()) { + MS_LOG(EXCEPTION) << "index " << i << "> kernel datatype indexes size " << datatype_indexes.size(); + } + if (datatype_indexes[i] == kUnSupportMixedDataTypeIndex) { + iter = kernel_match_datatype_idx->erase(iter); + } else { + if (datatype_indexes[i] < node_mix_precision_datatype_index[i]) { + auto count_iter = precision_reduce_count.find(iter->first); + if (count_iter != precision_reduce_count.end()) { + count_iter->second++; + } else { + precision_reduce_count[iter->first] = 1; + } + } + ++iter; + } + } + } + + return GetMinReducePrecisionCountIndex(kernel_match_datatype_idx, precision_reduce_count); +} + +void AddNodeAndKernelDataType(const CNodePtr &kernel_node, const kernel::KernelBuildInfo &kernel_build_info, + std::vector *support_indexes, std::vector *node_mix_precision_datatype, + std::vector *support_datatypes, + std::vector *node_mix_precision_datatype_index) { + bool add_node_datatype_flag = false; + if (node_mix_precision_datatype->size() == 0) { + add_node_datatype_flag = true; + } + for (size_t input_index = 0; input_index < kernel_build_info.GetInputNum(); ++input_index) { + AddKernelInputSupportDataType(kernel_build_info, input_index, support_indexes, support_datatypes); + if (add_node_datatype_flag) { + AddNodeInputDataType(kernel_node, input_index, node_mix_precision_datatype_index, node_mix_precision_datatype); + } + } + // Check output data type + for (size_t output_index = 0; output_index < kernel_build_info.GetOutputNum(); ++output_index) { + AddKernelOutputSupportDataType(kernel_build_info, output_index, support_indexes, support_datatypes); + if (add_node_datatype_flag) { + AddNodeOutputDataType(kernel_node, output_index, node_mix_precision_datatype_index, node_mix_precision_datatype); + } + } +} + +int PrecisionReduce(const std::vector &node_mix_precision_datatype_index, + const std::vector &node_mix_precision_datatype, + const std::unordered_map> &kernel_support_datatype, + std::unordered_map> *kernel_match_datatype_idx, bool *precision_reduce) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + MS_EXCEPTION_IF_NULL(precision_reduce); + std::unordered_map> kernel_match_datatype_idx_copy = *kernel_match_datatype_idx; + // raise precision + int selected_index = RaiseDataTypePrecisionSelect(node_mix_precision_datatype_index, node_mix_precision_datatype, + kernel_support_datatype, kernel_match_datatype_idx); + if (selected_index == -1 && context_ptr->enable_reduce_precision()) { + selected_index = + RaiseOrReduceDataTypePrecisionSelect(node_mix_precision_datatype_index, node_mix_precision_datatype, + kernel_support_datatype, &kernel_match_datatype_idx_copy); + if (selected_index != -1) { + *precision_reduce = true; + } + } + return selected_index; +} + +void SelectKernel(const CNodePtr &kernel_node, bool precision_reduce, const std::vector &node_datatype, + const std::shared_ptr &selected_kernel_info_ptr) { + MS_EXCEPTION_IF_NULL(selected_kernel_info_ptr); + if (precision_reduce) { + std::ostringstream datatype; + size_t input_num = selected_kernel_info_ptr->GetInputNum(); + size_t i = 0; + datatype << "("; + for (; i < input_num && i < node_datatype.size(); ++i) { + datatype << static_cast(node_datatype[i]); + if (i < input_num - 1) { + datatype << ", "; + } + } + datatype << ") -> ("; + for (; i < node_datatype.size(); ++i) { + datatype << static_cast(node_datatype[i]); + if (i < node_datatype.size() - 1) { + datatype << ", "; + } + } + datatype << ")"; + MS_LOG(WARNING) << kernel_node->DebugString() << " reduce precision, node datatype: " << datatype.str() + << ", select kernel: %s" << selected_kernel_info_ptr->ToString(); + } + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_info_ptr, kernel_node.get()); + // Set format and data type for input tensor. + SetTensorDeviceInfo(*selected_kernel_info_ptr, kernel_node); +} +} // namespace + +void SelectKernelInfo(const CNodePtr &kernel_node) { + std::vector> kernel_info_list; + MS_EXCEPTION_IF_NULL(kernel_node); + kernel::KernelQuery(kernel_node, &kernel_info_list); + std::vector most_match_counts = {-1, -1, -1, -1, -1}; + int selected_index = -1; + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool auto_mixed_precision = context_ptr->auto_mixed_precision_flag(); + std::unordered_map> kernel_match_datatype_idx; + std::unordered_map> kernel_support_datatype; + std::vector node_mix_precision_datatype_index; + std::vector node_mix_precision_datatype; + for (size_t info_index = 0; info_index < kernel_info_list.size(); ++info_index) { + std::vector cur_kernel_info_match_counts = {0, 0, 0, 0, 0}; + auto kernel_build_info = *(kernel_info_list[info_index]); + if (!IsValidKernelInfo(kernel_node, kernel_build_info)) { + continue; + } + std::vector support_indexes; + std::vector support_datatypes; + AddNodeAndKernelDataType(kernel_node, kernel_build_info, &support_indexes, &node_mix_precision_datatype, + &support_datatypes, &node_mix_precision_datatype_index); + kernel_match_datatype_idx[info_index] = support_indexes; + kernel_support_datatype[info_index] = support_datatypes; + if (!auto_mixed_precision && !MatchInferOutputDataType(kernel_node, kernel_build_info)) { + continue; + } + std::shared_ptr kernel_info_ptr = kernel_info_list[info_index]; + UpdateCurMatchCounts(*kernel_info_ptr, kernel_node, &cur_kernel_info_match_counts); + // Currently the selection policy is the match format count first, and then is datatype counts. + if (PriorityChooseItem(cur_kernel_info_match_counts, &most_match_counts)) { + selected_index = SizeToInt(info_index); + } + } + + bool precision_reduce = false; + if (selected_index == -1) { + selected_index = PrecisionReduce(node_mix_precision_datatype_index, node_mix_precision_datatype, + kernel_support_datatype, &kernel_match_datatype_idx, &precision_reduce); + } + if (selected_index == -1) { + MS_LOG(EXCEPTION) << kernel_node->DebugString() << "Cannot find valid kernel Info !"; + } + auto index = IntToSize(selected_index); + if (index >= kernel_info_list.size()) { + MS_LOG(EXCEPTION) << "index outof range"; + } + std::shared_ptr selected_kernel_info_ptr = kernel_info_list[index]; + MS_EXCEPTION_IF_NULL(selected_kernel_info_ptr); + SelectKernel(kernel_node, precision_reduce, node_mix_precision_datatype, selected_kernel_info_ptr); +} + +bool CheckKernelAccuracySupported(const CNodePtr &kernel_node, + const kernel::KernelBuildInfoPtr &new_kernel_build_info) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector> kernel_info_list; + kernel::KernelQuery(kernel_node, &kernel_info_list); + auto result = std::find_if(kernel_info_list.begin(), kernel_info_list.end(), + [&new_kernel_build_info](const kernel::KernelBuildInfoPtr item) { + MS_EXCEPTION_IF_NULL(item); + return *item == *new_kernel_build_info; + }); + return result != kernel_info_list.end(); +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/kernel_select_ascend.h b/mindspore/ccsrc/device/ascend/kernel_select_ascend.h new file mode 100644 index 00000000000..100cd8e1e1a --- /dev/null +++ b/mindspore/ccsrc/device/ascend/kernel_select_ascend.h @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_KERNEL_SELECT_ASCEND_ANFALGO_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_KERNEL_SELECT_ASCEND_ANFALGO_H_ +#include "ir/anf.h" +#include "kernel/kernel_build_info.h" +namespace mindspore { +namespace device { +namespace ascend { +void SelectKernelInfo(const CNodePtr &kernel_node); +bool CheckKernelAccuracySupported(const CNodePtr &kernel_node, const kernel::KernelBuildInfoPtr &new_kernel_build_info); +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_KERNEL_SELECT_ASCEND_ANFALGO_H_ diff --git a/mindspore/ccsrc/device/ascend/profiling/plugin_impl.cc b/mindspore/ccsrc/device/ascend/profiling/plugin_impl.cc new file mode 100644 index 00000000000..7790107aa10 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/plugin_impl.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/ascend/profiling/plugin_impl.h" +#include +#include "utils/log_adapter.h" +using std::string; + +namespace mindspore { +namespace device { +namespace ascend { +Reporter *PluginImpl::reporter_ = nullptr; + +PluginImpl::PluginImpl(const std::string &module) : module_(module) { MS_LOG(INFO) << "Create PluginImpl."; } + +int PluginImpl::Init(const Reporter *reporter) { + MS_LOG(INFO) << "PluginImpl init"; + MS_EXCEPTION_IF_NULL(reporter); + reporter_ = const_cast(reporter); + return 0; +} + +int PluginImpl::UnInit() { + MS_LOG(INFO) << " PluginImpl Uninit "; + reporter_ = nullptr; + return 0; +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/profiling/plugin_impl.h b/mindspore/ccsrc/device/ascend/profiling/plugin_impl.h new file mode 100644 index 00000000000..668b54b78cd --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/plugin_impl.h @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PLUGIN_IMPL_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PLUGIN_IMPL_H_ + +#include +#include "./prof_engine.h" +using Msprof::Engine::PluginIntf; +using Msprof::Engine::Reporter; +using std::string; + +namespace mindspore { +namespace device { +namespace ascend { +class PluginImpl : public PluginIntf { + public: + explicit PluginImpl(const std::string& module); + ~PluginImpl() override = default; + int Init(const Reporter* reporter) override; + int UnInit() override; + static Reporter* GetPluginReporter() { return reporter_; } + + private: + static Reporter* reporter_; + std::string module_; +}; +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PLUGIN_IMPL_H_ diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc new file mode 100644 index 00000000000..3a1dc4689bb --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/ascend/profiling/profiling_engine_impl.h" +#include "utils/log_adapter.h" +#include "device/ascend/profiling/plugin_impl.h" + +namespace mindspore { +namespace device { +namespace ascend { +PluginIntf* ProfilingEngineImpl::CreatePlugin() { + MS_LOG(INFO) << "Create Plugin."; + return new (std::nothrow) PluginImpl("Framework"); +} + +int ProfilingEngineImpl::ReleasePlugin(PluginIntf* plugin) { + if (plugin != nullptr) { + delete plugin; + } + return 0; +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.h b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.h new file mode 100644 index 00000000000..e8dbfc7087c --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_ENGINE_IMPL_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_ENGINE_IMPL_H_ + +#include "./prof_engine.h" + +using Msprof::Engine::EngineIntf; +using Msprof::Engine::PluginIntf; + +namespace mindspore { +namespace device { +namespace ascend { +class ProfilingEngineImpl : public EngineIntf { + public: + ProfilingEngineImpl() = default; + ~ProfilingEngineImpl() override = default; + + PluginIntf* CreatePlugin() override; + int ReleasePlugin(PluginIntf* plugin) override; +}; +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_ENGINE_IMPL_H_ diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc new file mode 100644 index 00000000000..4bc68e647ac --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc @@ -0,0 +1,195 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/profiling/profiling_manager.h" + +#include +#include + +#include +#include "securec/include/securec.h" +#include "./prof_mgr_core.h" +#include "device/ascend/profiling/plugin_impl.h" +#include "device/ascend/profiling/profiling_engine_impl.h" +#include "utils/log_adapter.h" +#include "utils/context/ms_context.h" +#include "common/utils.h" + +using std::vector; +using Json = nlohmann::json; + +namespace mindspore { +namespace device { +namespace ascend { +ProfilingManager& ProfilingManager::GetInstance() { + static ProfilingManager inst; + return inst; +} + +ProfilingManager::ProfilingManager() : device_id_(0), prof_handle_(nullptr) { + engine_0_ = std::make_shared(); +} + +uint64_t ProfilingManager::GetJobId() const { + const char* job_id = std::getenv("JOB_ID"); + return ((job_id != nullptr) ? std::strtoul(job_id, nullptr, 10) : 0); +} + +bool ProfilingManager::ReportProfilingData(const map& op_taskId_map) const { + if (!IsProfiling()) { + MS_LOG(INFO) << "No need profiling. please export PROFILING_MODE and in train mode."; + return false; + } + if (op_taskId_map.empty()) { + MS_LOG(WARNING) << "op_taskId_map is empty."; + return false; + } + auto reporter = PluginImpl::GetPluginReporter(); + if (reporter == nullptr) { + MS_LOG(ERROR) << "No profiling data report!"; + return false; + } + MS_LOG(INFO) << "DistributeTask: op tasId map size = " << op_taskId_map.size(); + + Msprof::Engine::ReporterData reporter_data = {}; + for (const auto& iter : op_taskId_map) { + auto data = iter.second + ' ' + std::to_string(iter.first) + ';'; + reporter_data.deviceId = UintToInt(device_id_); + reporter_data.data = (unsigned char*)(const_cast(data.c_str())); + reporter_data.dataLen = data.size(); + auto ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "framework", sizeof("framework")); + if (ret != 0) { + MS_LOG(ERROR) << "memcpy_s error, errorno(" << ret << ")"; + return false; + } + ret = reporter->Report(&reporter_data); + if (ret != 0) { + MS_LOG(ERROR) << "reporter data fail, errorno(" << ret << ")"; + return false; + } + } + return true; +} + +static std::vector Split(const std::string& str, const char delim) { + std::vector elems; + + if (str.empty()) { + elems.emplace_back(""); + return elems; + } + + std::stringstream ss(str); + std::string item; + + while (getline(ss, item, delim)) { + elems.push_back(item); + } + auto str_size = str.size(); + if (str_size > 0 && str[str_size - 1] == delim) { + elems.emplace_back(""); + } + + return elems; +} + +bool ProfilingManager::StartupProfiling(uint32_t device_id) { + auto is_profiling = IsProfiling(); + if (!is_profiling) { + MS_LOG(INFO) << "No need profiling. please export PROFILING_MODE and in train mode."; + return true; + } + device_id_ = device_id; + // exp: export PROFILING_MODE=true + // export PROFILING_OPTIONS=training_trace + const char* prof_options_str = std::getenv("PROFILING_OPTIONS"); + // register Framework to profiling + int result = Msprof::Engine::RegisterEngine("Framework", engine_0_.get()); + if (result != 0) { + MS_LOG(ERROR) << "Register profiling Engine failed."; + return false; + } + + if (prof_options_str != nullptr) { + const string prof_options_str_tmp = prof_options_str; + vector opts = Split(prof_options_str_tmp, ':'); + if (!opts.empty()) { + // current one docker only use one device` + Json p_device; + + // device_id + p_device["deviceID"] = std::to_string(device_id); + + // features:'training_trace', 'task_trace' etc + Json features; + for (vector::size_type i = 0; i < opts.size(); i++) { + Json f; + f["name"] = opts[i]; + features[i] = f; + } + p_device["features"] = features; + + // only one device, but sProfMgrStartUp API require for device list + Json devices; + devices[0] = p_device; + + Json startCfg; + startCfg["startCfg"] = devices; + + // convert json to string + std::stringstream ss; + ss << startCfg; + std::string cfg = ss.str(); + + MS_LOG(INFO) << "profiling config " << cfg; + + // call profiling startup API + ProfMgrCfg prof_cfg = {cfg}; + prof_handle_ = ProfMgrStartUp(&prof_cfg); + if (prof_handle_ == nullptr) { + MS_LOG(ERROR) << "Startup profiling failed."; + return false; + } + } + } + + return true; +} + +bool ProfilingManager::StopProfiling() const { + MS_LOG(INFO) << "StopProfiling"; + if (!IsProfiling()) { + MS_LOG(INFO) << "No need profiling. please export PROFILING_MODE and in train mode."; + return true; + } + Msprof::Engine::Reporter* reporter = PluginImpl::GetPluginReporter(); + if (reporter != nullptr) { + MS_LOG(INFO) << "report data end, ret = " << reporter->Flush(); + } + + if (prof_handle_ != nullptr) { + int result = ProfMgrStop(prof_handle_); + if (result != 0) { + MS_LOG(ERROR) << "ProfMgr stop return fail:" << result << "."; + return false; + } + } + + return true; +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h new file mode 100644 index 00000000000..de8f6a7d0af --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_MANAGER_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_MANAGER_H_ + +#include +#include +#include +#include +using std::map; +using std::string; + +namespace mindspore { +namespace device { +namespace ascend { +// PROFILING_CUSTOM_LOGID_START 3 +const uint64_t kProfilingFpStartLogId = 1; +const uint64_t kProfilingBpEndLogId = 2; +const uint64_t kProfilingAllReduce1Start = 3; +const uint64_t kProfilingAllReduce1End = 4; +const uint64_t kProfilingAllReduce2Start = 5; +const uint64_t kProfilingAllReduce2End = 6; +const uint64_t kProfilingIterEndLogId = 255; + +class ProfilingEngineImpl; +class ProfilingManager { + public: + static ProfilingManager &GetInstance(); + uint64_t GetJobId() const; + bool ReportProfilingData(const map &op_taskId_map) const; + bool StartupProfiling(uint32_t device_id); + bool StopProfiling() const; + + inline bool IsProfiling() const { + const char *is_profiling = std::getenv("PROFILING_MODE"); + return (is_profiling != nullptr && (strcmp("true", is_profiling) == 0)); + } + + protected: + ProfilingManager(); + ~ProfilingManager() { prof_handle_ = nullptr; } + + private: + std::shared_ptr engine_0_; + uint32_t device_id_; + void *prof_handle_; +}; +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_MANAGER_H_ diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc new file mode 100644 index 00000000000..0d7088300e6 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc @@ -0,0 +1,231 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/profiling/profiling_utils.h" + +#include + +#include "kernel/kernel.h" +#include "device/ascend/profiling/profiling_manager.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" +#include "utils/utils.h" + +namespace mindspore { +namespace device { +namespace ascend { +const char ProfilingUtils::kProfiling[] = "Profiling"; +const char ProfilingUtils::kNotify[] = "notify"; +const char ProfilingUtils::kProfilerTraceId[] = "profiler_trace_id"; +const char ProfilingUtils::kFlags[] = "flags"; +std::unordered_map> ProfilingUtils::graph_kernel_name_; +bool ProfilingUtils::GetProfilingTraceInfo(const std::shared_ptr &graph_ptr, + ProfilingTraceInfo *profiling_trace_info) { + MS_EXCEPTION_IF_NULL(profiling_trace_info); + MS_EXCEPTION_IF_NULL(graph_ptr); + bool find_begin = false; + bool first_allreduce = true; + for (const auto &anf_node : graph_ptr->execution_order()) { + if (anf_node->isa()) { + const std::string kernel_name = AnfAlgo::GetCNodeName(anf_node); + if ((kernel_name == "Cast" || kernel_name == "Four2Five") && !find_begin) { + profiling_trace_info->profiling_trace_begin = anf_node->fullname_with_scope(); + find_begin = true; + } + if (kernel_name == "Conv2DBackpropFilter") { + profiling_trace_info->profiling_trace_bp_end = anf_node->fullname_with_scope(); + } + if (kernel_name == kFusedMulApplyMomentumOpName || kernel_name == kApplyMomentumOpName) { + profiling_trace_info->profiling_trace_netoutput = anf_node->fullname_with_scope(); + } + if (kernel_name == kAllReduceOpName) { + if (first_allreduce) { + profiling_trace_info->profiling_allreduce1_start = anf_node->fullname_with_scope(); + profiling_trace_info->profiling_allreduce1_end = anf_node->fullname_with_scope(); + first_allreduce = false; + } else { + profiling_trace_info->profiling_allreduce2_start = anf_node->fullname_with_scope(); + profiling_trace_info->profiling_allreduce2_end = anf_node->fullname_with_scope(); + } + } + } + } + MS_LOG(INFO) << "[profiling]begin:" << profiling_trace_info->profiling_trace_begin + << ", net_output:" << profiling_trace_info->profiling_trace_netoutput + << ", end:" << profiling_trace_info->profiling_trace_bp_end + << ", allreduce1:" << profiling_trace_info->profiling_allreduce1_start + << ", allreduce2:" << profiling_trace_info->profiling_allreduce2_start; + return profiling_trace_info->IsValid(); +} + +bool ProfilingUtils::GetNetOutput(AnfNodePtr anf_node, std::string *profiling_trace_net_output) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(profiling_trace_net_output); + MS_LOG(INFO) << "[profiling]Anf node's full name with scope:" << anf_node->fullname_with_scope(); + if (!profiling_trace_net_output->empty()) { + MS_LOG(INFO) << "[profiling]Has got the net_output:" << profiling_trace_net_output->c_str(); + return true; + } + + if (AnfAlgo::IsRealKernel(anf_node)) { + *profiling_trace_net_output = anf_node->fullname_with_scope(); + return true; + } + + auto cnode = anf_node->cast(); + if (cnode == nullptr) { + MS_LOG(ERROR) << "[profiling]Anf node should be a CNode"; + return false; + } + + auto inputs = cnode->inputs(); + auto input_size = inputs.size(); + if (input_size < 2) { + MS_LOG(ERROR) << "[profiling]Anf node' input size(" << input_size << ") < 2, don't support get apply kernel node."; + return false; + } + return GetNetOutput(inputs[1], profiling_trace_net_output); +} + +CNodePtr ProfilingUtils::CreateProfilingCNode(const std::shared_ptr &graph_ptr, bool notify, + uint64_t profiler_trace_id, uint32_t flags) { + MS_EXCEPTION_IF_NULL(graph_ptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder; + selected_kernel_builder.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + selected_kernel_builder.SetInputsDeviceType({TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32}); + selected_kernel_builder.SetFusionType(kernel::FusionType::OPAQUE); + selected_kernel_builder.SetProcessor(kernel::Processor::AICORE); + selected_kernel_builder.SetKernelType(KernelType::RT_KERNEL); + abstract::AbstractBasePtr type_none_abstract = std::make_shared(); + auto primitive = std::make_shared(ProfilingUtils::kProfiling); + std::vector inputs; + inputs.emplace_back(NewValueNode(primitive)); + CNodePtr cnode_ptr = graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(cnode_ptr); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), cnode_ptr.get()); + cnode_ptr->set_abstract(type_none_abstract); + // set attr + ValuePtr notify_value = MakeValue(notify); + ValuePtr trace_id_value = MakeValue(profiler_trace_id); + ValuePtr flags_value = MakeValue(flags); + AnfAlgo::SetNodeAttr(ProfilingUtils::kNotify, notify_value, cnode_ptr); + AnfAlgo::SetNodeAttr(ProfilingUtils::kProfilerTraceId, trace_id_value, cnode_ptr); + AnfAlgo::SetNodeAttr(ProfilingUtils::kFlags, flags_value, cnode_ptr); + return cnode_ptr; +} + +void ProfilingUtils::ProfilingTraceFpStart(const std::shared_ptr &graph_ptr, + const mindspore::AnfNodePtr &anf_node, + const mindspore::device::ascend::ProfilingTraceInfo &profiling_trace_info, + std::vector *kernel_list) { + if (profiling_trace_info.IsValid() && profiling_trace_info.profiling_trace_begin == anf_node->fullname_with_scope()) { + if (graph_ptr == nullptr || kernel_list == nullptr || anf_node == nullptr) { + MS_LOG(ERROR) << "[profiling]input param invalid"; + return; + } + auto job_id = ProfilingManager::GetInstance().GetJobId(); + // job task info + CNodePtr job_kernel_ptr = CreateProfilingCNode(graph_ptr, false, job_id, 0); + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(anf_node.get()), job_kernel_ptr.get()); + AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(anf_node), job_kernel_ptr.get()); + // fp task info + CNodePtr start_kernel_ptr = CreateProfilingCNode(graph_ptr, false, kProfilingFpStartLogId, 0); + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(anf_node.get()), start_kernel_ptr.get()); + AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(anf_node), start_kernel_ptr.get()); + kernel_list->emplace_back(job_kernel_ptr); + kernel_list->emplace_back(start_kernel_ptr); + } +} + +void ProfilingUtils::ProfilingAllReduce(const std::shared_ptr &graph_ptr, + const AnfNodePtr &anf_node, int job_id, const std::string &profiling_node_name, + std::vector *kernel_list) { + MS_EXCEPTION_IF_NULL(graph_ptr); + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(kernel_list); + auto full_scope_name = anf_node->fullname_with_scope(); + if (profiling_node_name == full_scope_name) { + CNodePtr allreduce_kernel_ptr = CreateProfilingCNode(graph_ptr, false, job_id, 0); + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(anf_node.get()), allreduce_kernel_ptr.get()); + AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(anf_node), allreduce_kernel_ptr.get()); + kernel_list->emplace_back(allreduce_kernel_ptr); + } +} + +void ProfilingUtils::ProfilingTraceEnd(const std::shared_ptr &graph_ptr, + const mindspore::AnfNodePtr &anf_node, + const mindspore::device::ascend::ProfilingTraceInfo &profiling_trace_info, + std::vector *kernel_list) { + MS_EXCEPTION_IF_NULL(graph_ptr); + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(kernel_list); + if (profiling_trace_info.IsValid()) { + auto full_scope_name = anf_node->fullname_with_scope(); + if (profiling_trace_info.profiling_trace_netoutput == full_scope_name) { + CNodePtr bp_kernel_ptr = CreateProfilingCNode(graph_ptr, true, kProfilingIterEndLogId, 0); + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(anf_node.get()), bp_kernel_ptr.get()); + AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(anf_node), bp_kernel_ptr.get()); + kernel_list->emplace_back(bp_kernel_ptr); + } + + if (profiling_trace_info.profiling_trace_bp_end == full_scope_name) { + CNodePtr end_task_info = CreateProfilingCNode(graph_ptr, false, kProfilingBpEndLogId, 0); + AnfAlgo::SetStreamDistinctionLabel(AnfAlgo::GetStreamDistinctionLabel(anf_node.get()), end_task_info.get()); + AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(anf_node), end_task_info.get()); + kernel_list->emplace_back(end_task_info); + } + } +} + +void ProfilingUtils::SetGraphKernelName(uint32_t graph_id, const std::vector &kernel_names) { + auto iter = graph_kernel_name_.find(graph_id); + if (iter == graph_kernel_name_.end()) { + graph_kernel_name_[graph_id] = kernel_names; + } else { + MS_LOG(ERROR) << "[profiling]graph kernel names already exist"; + } +} + +void ProfilingUtils::ReportProfilingData(uint32_t graph_id, const std::vector &task_ids) { + auto iter = graph_kernel_name_.find(graph_id); + if (iter == graph_kernel_name_.end()) { + MS_LOG(ERROR) << "[profiling]graph id " << graph_id << " not in graph_kernel_name_"; + return; + } + auto &kernel_names = iter->second; + + MS_LOG(INFO) << "kernel_names size:" << kernel_names.size() << ", task_ids size:" << task_ids.size(); + if (kernel_names.size() != task_ids.size()) { + MS_LOG(ERROR) << "[profiling]kernel name and task id not match"; + return; + } + std::map op_task_id_map; + size_t size = kernel_names.size(); + for (size_t i = 0; i < size; ++i) { + auto it = op_task_id_map.find(task_ids[i]); + if (it != op_task_id_map.end()) { + MS_LOG(WARNING) << "task_id " << task_ids[i] << " exist, " << kernel_names[i]; + continue; + } + op_task_id_map[task_ids[i]] = kernel_names[i]; + } + if (!ProfilingManager::GetInstance().ReportProfilingData(op_task_id_map)) { + MS_LOG(ERROR) << "ReportProfilingData failed"; + } +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h new file mode 100644 index 00000000000..ca0ef6f1f0b --- /dev/null +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_UTILS_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_UTILS_H_ + +#include +#include +#include +#include +#include "session/kernel_graph.h" + +namespace mindspore { +namespace device { +namespace ascend { +struct ProfilingTraceInfo { + // execute order's first execute op(like: Cast or Four2Five ...), except tdt op(GetNext ...) + std::string profiling_trace_begin; + // get first net_output(apply kernel) from graph outputs: fp ->net_output<- bp + std::string profiling_trace_bp_end; + // execute order's end execute (like: Conv2DBackpropFilter) + std::string profiling_trace_netoutput; + + std::string profiling_allreduce1_start; + + std::string profiling_allreduce1_end; + + std::string profiling_allreduce2_start; + + std::string profiling_allreduce2_end; + + // 1. insert profiling_trace_begin if profiling_trace_bp_end is not empty. + // 2. op lanuch get task info with callback func. + // 3. insert profiling_trace_bp_end. + // 4. insert profiling_trace_net_output if profiling_trace_bp_end is not empty. + + bool IsValid() const { return !(profiling_trace_begin.empty() || profiling_trace_bp_end.empty()); } +}; + +class ProfilingUtils { + public: + ProfilingUtils() = default; + ~ProfilingUtils() = default; + static bool GetProfilingTraceInfo(const std::shared_ptr &graph_ptr, + ProfilingTraceInfo *profiling_trace_info); + static void ProfilingTraceFpStart(const std::shared_ptr &graph_ptr, const AnfNodePtr &anf_node, + const ProfilingTraceInfo &profiling_trace_info, std::vector *kernel_list); + static void ProfilingAllReduce(const std::shared_ptr &graph_ptr, const AnfNodePtr &anf_node, + int job_id, const std::string &profiling_node_name, + std::vector *kernel_list); + static void ProfilingTraceEnd(const std::shared_ptr &graph_ptr, const AnfNodePtr &anf_node, + const ProfilingTraceInfo &profiling_trace_info, std::vector *kernel_list); + static void SetGraphKernelName(uint32_t graph_id, const std::vector &kernel_names); + static void ReportProfilingData(uint32_t graph_id, const std::vector &task_ids); + + static const char kProfiling[]; + static const char kNotify[]; + static const char kProfilerTraceId[]; + static const char kFlags[]; + + private: + static bool GetNetOutput(AnfNodePtr anf_node, std::string *profiling_trace_net_output); + static CNodePtr CreateProfilingCNode(const std::shared_ptr &graph_ptr, bool notify, + uint64_t profiler_trace_id, uint32_t flags); + // graph id --> (kernel name list) + static std::unordered_map> graph_kernel_name_; +}; +} // namespace ascend +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_PROFILING_PROFILING_UTILS_H_ diff --git a/mindspore/ccsrc/device/ascend/readme.md b/mindspore/ccsrc/device/ascend/readme.md new file mode 100644 index 00000000000..037e3b658ad --- /dev/null +++ b/mindspore/ccsrc/device/ascend/readme.md @@ -0,0 +1 @@ +ascend diff --git a/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc new file mode 100644 index 00000000000..7a95ddc844c --- /dev/null +++ b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc @@ -0,0 +1,107 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/tasksink/runtime_utils.h" + +#include + +#include "hccl/hcom.h" +#include "utils/log_adapter.h" +#include "utils/utils.h" + +constexpr auto kHcomBroadcast = "hcom_broadcast_"; +constexpr auto kHcomAllGather = "hcom_all_gather_"; +constexpr auto kHcomAllReduce = "hcom_all_reduce_"; +constexpr auto kHcomReduceScatter = "hcom_reduce_scatter_"; +constexpr auto kUnderline = "_"; +namespace mindspore { +namespace device { +namespace ascend { +namespace tasksink { +bool RuntimeUtils::HcomBindModel(rtModel_t model, rtStream_t stream) { + hcclResult_t ret = hcom_bind_model(model, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "Call hcom_bind_model failed, ret: 0x" << static_cast(ret); + return false; + } + return true; +} + +bool RuntimeUtils::HcomUnbindModel(rtModel_t model) { + hcclResult_t ret = hcom_unbind_model(model); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "Call hcom_unbind_model failed, ret: 0x" << static_cast(ret); + return false; + } + return true; +} + +bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info, rtStream_t stream) { + MS_LOG(INFO) << "hccl distribute start"; + MS_EXCEPTION_IF_NULL(task_info); + hcclResult_t ret; + static uint32_t task_counter = 0; + + if (task_info->hccl_type() == kBroadcastOpName) { + // call hcom broadcast interface to run op + const string tag_broadcast = kHcomBroadcast + std::to_string(task_counter++) + kUnderline + std::to_string(0); + ret = hcom_broadcast(tag_broadcast.c_str(), reinterpret_cast(task_info->input_data_addr()), + static_cast(task_info->count()), static_cast(task_info->data_type()), + static_cast(task_info->root_id()), nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "hcom_broadcast fail, return ret: " << static_cast(ret); + return false; + } + } else if (task_info->hccl_type() == kAllGatherOpName) { + // call hcom allgather interface to run op + const string tag_all_gather = kHcomAllGather + std::to_string(task_counter++) + kUnderline + std::to_string(0); + ret = hcom_all_gather(tag_all_gather.c_str(), reinterpret_cast(task_info->input_data_addr()), + reinterpret_cast(task_info->output_data_addr()), static_cast(task_info->count()), + static_cast(task_info->data_type()), nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "hcom_all_gather fail, return ret: " << ret; + return false; + } + } else if (task_info->hccl_type() == kAllReduceOpName) { + // call hcom allreduce interface to run op + const string tag_all_reduce = kHcomAllReduce + std::to_string(task_counter++) + kUnderline + std::to_string(0); + ret = hcom_all_reduce(tag_all_reduce.c_str(), reinterpret_cast(task_info->input_data_addr()), + reinterpret_cast(task_info->output_data_addr()), static_cast(task_info->count()), + static_cast(task_info->data_type()), + static_cast(task_info->op_type()), nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "hcom_all_reduce fail, return ret: " << ret; + return false; + } + } else if (task_info->hccl_type() == kReduceScatterOpName) { + // call hcom reducescatter interface to run op + const string tag_reduce_scatter = + kHcomReduceScatter + std::to_string(task_counter++) + kUnderline + std::to_string(0); + ret = hcom_reduce_scatter(tag_reduce_scatter.c_str(), reinterpret_cast(task_info->input_data_addr()), + reinterpret_cast(task_info->output_data_addr()), + static_cast(task_info->count()), static_cast(task_info->data_type()), + static_cast(task_info->op_type()), nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "hcom_reduce_scatter fail, return ret: " << ret; + return false; + } + } + return true; +} +} // namespace tasksink +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.h b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.h new file mode 100644 index 00000000000..874de54e8a7 --- /dev/null +++ b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_TASKSINK_RUNTIME_UTILS_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_TASKSINK_RUNTIME_UTILS_H_ + +#include +#include "runtime/rt.h" +#include "framework/ge_runtime/task_info.h" + +using ge::model_runner::HcclTaskInfo; + +namespace mindspore { +namespace device { +namespace ascend { +namespace tasksink { +class RuntimeUtils { + public: + static bool HcomBindModel(rtModel_t model, rtStream_t stream); + static bool HcomUnbindModel(rtModel_t model); + static bool HcomDistribute(const std::shared_ptr &task_info, rtStream_t stream); +}; +} // namespace tasksink +} // namespace ascend +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_ASCEND_TASKSINK_RUNTIME_UTILS_H_ diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc new file mode 100644 index 00000000000..88d00bc6c2c --- /dev/null +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc @@ -0,0 +1,156 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/ascend/tasksink/task_generator.h" + +#include +#include "kernel/task_stream.h" +#include "utils/context/ms_context.h" +#include "common/utils.h" +#include "device/ascend/profiling/profiling_utils.h" +#include "device/ascend/profiling/profiling_manager.h" + +namespace mindspore { +namespace device { +namespace ascend { +namespace tasksink { +bool TaskGenerator::GenTasks(const std::vector &anf_node_list, std::vector *task_info_list, + uint32_t graph_id) { + MS_LOG(INFO) << "GenTasks start..."; + MS_EXCEPTION_IF_NULL(task_info_list); + // Traverse graph applykernel list and run + if (!LaunchAllKernel(anf_node_list, task_info_list, graph_id)) { + MS_LOG(ERROR) << "LaunchAllKernel failed"; + return false; + } + MS_LOG(INFO) << "GenTasks end..."; + return true; +} + +void TaskGenerator::LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs) { + MS_EXCEPTION_IF_NULL(anf_node_ptr); + if (anf_node_ptr->inputs().size() != 2) { + MS_LOG(EXCEPTION) << "atomic Addr clean Node Input nodes not equal 2."; + } + auto pre_node = anf_node_ptr->inputs()[1]; + // set clean output addr + if (AnfAlgo::HasNodeAttr(kAttrAutomicOutputIndexs, pre_node)) { + auto clean_output_indexs = AnfAlgo::GetNodeAttr>(pre_node, kAttrAutomicOutputIndexs); + for (auto index : clean_output_indexs) { + auto device_address = AnfAlgo::GetOutputAddr(pre_node, index); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(input->addr); + input->size = device_address->size_; + kernel_inputs->push_back(input); + } + MS_LOG(INFO) << "AtomicAddClean clean output size:" << clean_output_indexs.size(); + } + // set clean workspace address + if (AnfAlgo::HasNodeAttr(kAttrAutomicWorkspaceSize, pre_node)) { + auto clean_workspaces = AnfAlgo::GetNodeAttr(pre_node, kAttrAutomicWorkspaceSize); + if (clean_workspaces != 0) { + auto device_address = AnfAlgo::GetWorkspaceAddr(pre_node, 0); + kernel::AddressPtr workspace = std::make_shared(); + MS_EXCEPTION_IF_NULL(workspace); + workspace->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(workspace->addr); + workspace->size = device_address->size_; + kernel_inputs->push_back(workspace); + } + MS_LOG(INFO) << "AtomicAddClean clean workspace size" << clean_workspaces; + } + auto clear_mems = AnfAlgo::GetNodeAttr>(anf_node_ptr, kAttrAutomicAddMemSize); + if (kernel_inputs->size() != clear_mems.size()) { + MS_LOG(EXCEPTION) << "AtomicAddClean kernel inputs size not equal clear memory size,kerenl_inputs size:" + << kernel_inputs->size() << ",clean mem size" << clear_mems.size(); + } +} + +bool TaskGenerator::LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, + std::vector *task_info_list) { + MS_LOG(INFO) << "LaunchKernel start..."; + MS_EXCEPTION_IF_NULL(task_info_list); + MS_EXCEPTION_IF_NULL(anf_node_ptr); + AddressPtrList kernel_inputs; + AddressPtrList kernel_workspaces; + AddressPtrList kernel_outputs; + auto kernel_mod = AnfAlgo::GetKernelMod(anf_node_ptr); + MS_EXCEPTION_IF_NULL(kernel_mod); + if (AnfAlgo::GetCNodeName(anf_node_ptr) != kAtomicAddrCleanOpName) { + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(anf_node_ptr); ++i) { + auto real_input_index = AnfAlgo::GetRealInputIndex(anf_node_ptr, i); + auto device_address = AnfAlgo::GetPrevNodeOutputAddr(anf_node_ptr, real_input_index); + AddressPtr input = std::make_shared

(); + input->addr = device_address->ptr_; + input->size = device_address->size_; + kernel_inputs.push_back(input); + } + + for (size_t i = 0; i < AnfAlgo::GetOutputTensorNum(anf_node_ptr); ++i) { + auto it = AnfAlgo::GetOutputAddr(anf_node_ptr, i); + AddressPtr output = std::make_shared
(); + output->addr = it->ptr_; + output->size = it->size_; + kernel_outputs.push_back(output); + } + + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + auto device_address = AnfAlgo::GetWorkspaceAddr(anf_node_ptr, i); + kernel::AddressPtr workspace = std::make_shared(); + MS_EXCEPTION_IF_NULL(workspace); + workspace->addr = device_address->ptr_; + workspace->size = device_address->size_; + kernel_workspaces.push_back(workspace); + } + } else { + LaunchAddrCleanKernel(anf_node_ptr, &kernel_inputs); + } + + std::vector task_info_ptrs = + kernel_mod->GenTask(kernel_inputs, kernel_workspaces, kernel_outputs, stream_id); + task_info_list->insert(task_info_list->end(), task_info_ptrs.begin(), task_info_ptrs.end()); + return true; +} + +bool TaskGenerator::LaunchAllKernel(const std::vector &anf_node_list, + std::vector *task_info_list, uint32_t graph_id) { + uint32_t current_op_index = 0; + std::vector kernel_name_list; + for (const auto &anf_node_ptr : anf_node_list) { + size_t old_size = task_info_list->size(); + uint32_t stream_id = AnfAlgo::GetStreamId(anf_node_ptr); + MS_LOG(INFO) << "Task gen launch begin, current_op_idx:" << current_op_index + << " type:" << (AnfAlgo::GetCNodeName(anf_node_ptr)) << ", stream id:" << stream_id; + if (!LaunchKernel(anf_node_ptr, stream_id, task_info_list)) { + MS_LOG(ERROR) << "LaunchKernel failed."; + return false; + } + for (size_t i = old_size; i < task_info_list->size(); ++i) { + kernel_name_list.emplace_back(anf_node_ptr->fullname_with_scope()); + } + current_op_index++; + } + if (ProfilingManager::GetInstance().IsProfiling()) { + ProfilingUtils::SetGraphKernelName(graph_id, kernel_name_list); + } + return true; +} +} // namespace tasksink +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.h b/mindspore/ccsrc/device/ascend/tasksink/task_generator.h new file mode 100644 index 00000000000..4d1fba9db4b --- /dev/null +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_ASCEND_TASK_TASK_BUILD_H_ +#define MINDSPORE_CCSRC_DEVICE_ASCEND_TASK_TASK_BUILD_H_ + +#include +#include +#include +#include +#include +#include +#include "cce/aicpu_engine_struct.h" +#include "cce/taskdown_api.h" +#include "cce/fwk_adpt_struct.h" +#include "device/kernel_runtime.h" +#include "ir/anf.h" +#include "kernel/kernel.h" +#include "framework/ge_runtime/task_info.h" + +namespace mindspore { +namespace device { +namespace ascend { +namespace tasksink { +using mindspore::kernel::Address; +using mindspore::kernel::AddressPtr; +using AddressPtrList = std::vector; +using ge::model_runner::TaskInfo; +using TaskInfoPtr = std::shared_ptr; +class TaskGenerator { + public: + TaskGenerator() = default; + ~TaskGenerator() = default; + TaskGenerator(const TaskGenerator &in) = delete; + TaskGenerator &operator=(const TaskGenerator &in) = delete; + + static bool GenTasks(const std::vector &anf_node_list, std::vector *task_info_list, + uint32_t graph_id); + + private: + static void LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); + static bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector *task_info_list); + static bool LaunchAllKernel(const std::vector &anf_node_list, std::vector *task_info_list, + uint32_t graph_id); +}; +} // namespace tasksink +} // namespace ascend +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_ASCEND_TASK_TASK_BUILD_H_ diff --git a/mindspore/ccsrc/device/convert_tensor_utils.cc b/mindspore/ccsrc/device/convert_tensor_utils.cc new file mode 100644 index 00000000000..bac72727c2c --- /dev/null +++ b/mindspore/ccsrc/device/convert_tensor_utils.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/convert_tensor_utils.h" +#include +namespace mindspore { +namespace device { +void HalfToFloat(void *dst, const void *src, size_t elem_num) { + auto half_data = static_cast(src); + auto float_data = static_cast(dst); + for (size_t i = 0; i < elem_num; ++i) { + float tmp = Eigen::half_impl::half_to_float(half_data[i]); + float_data[i] = tmp; + } +} + +void FloatToHalf(void *dst, const void *src, size_t elem_num) { + auto float_data = static_cast(src); + auto half_data = static_cast(dst); + for (size_t i = 0; i < elem_num; ++i) { + half_data[i] = Eigen::half(float_data[i]); + } +} + +void DoubleToFloat(void *dst, const void *src, size_t elem_num) { + auto double_data = static_cast(src); + auto float_data = static_cast(dst); + for (size_t i = 0; i < elem_num; ++i) { + float_data[i] = static_cast(double_data[i]); + } +} + +void FloatToDouble(void *dst, const void *src, size_t elem_num) { + auto float_data = static_cast(src); + auto double_data = static_cast(dst); + for (size_t i = 0; i < elem_num; ++i) { + double_data[i] = static_cast(float_data[i]); + } +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/convert_tensor_utils.h b/mindspore/ccsrc/device/convert_tensor_utils.h new file mode 100644 index 00000000000..13ca6df0898 --- /dev/null +++ b/mindspore/ccsrc/device/convert_tensor_utils.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_CONVERT_TENSOR_UTILS_H_ +#define MINDSPORE_CCSRC_DEVICE_CONVERT_TENSOR_UTILS_H_ + +#include +#include +#include "ir/meta_tensor.h" + +namespace mindspore { +namespace device { +void HalfToFloat(void *dst, const void *src, size_t elem_num); +void FloatToHalf(void *dst, const void *src, size_t elem_num); +void DoubleToFloat(void *dst, const void *src, size_t elem_num); +void FloatToDouble(void *dst, const void *src, size_t elem_num); +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CONVERT_TENSOR_UTILS_H_ diff --git a/mindspore/ccsrc/device/cpu/cpu_device_address.cc b/mindspore/ccsrc/device/cpu/cpu_device_address.cc new file mode 100644 index 00000000000..56e9b6d36e8 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_device_address.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/cpu_device_address.h" +#include +#include "device/convert_tensor_utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +bool CPUDeviceAddress::SyncDeviceToHost(const std::vector & /*shape*/, size_t size, TypeId type, + void *host_ptr) const { + if (type == kNumberTypeFloat16) { + FloatToHalf(host_ptr, ptr_, size / 2); + } else if (type == kNumberTypeFloat64) { + FloatToDouble(host_ptr, ptr_, size / sizeof(double)); + } + return true; +} + +bool CPUDeviceAddress::SyncHostToDevice(const std::vector & /*shape*/, size_t size, TypeId type, + const void *host_ptr) const { + if (type == kNumberTypeFloat16) { + HalfToFloat(ptr_, host_ptr, size / 2); + } else if (type == kNumberTypeFloat64) { + DoubleToFloat(ptr_, host_ptr, size / sizeof(double)); + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_device_address.h b/mindspore/ccsrc/device/cpu/cpu_device_address.h new file mode 100644 index 00000000000..9d51abe6256 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_device_address.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CPU_DEVICE_ADDRESS_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CPU_DEVICE_ADDRESS_H_ + +#include +#include +#include "device/device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +class CPUDeviceAddress : public DeviceAddress { + public: + CPUDeviceAddress(void *ptr, size_t size) : DeviceAddress(ptr, size) {} + + CPUDeviceAddress(void *ptr, size_t size, const string &format, TypeId type_id) + : DeviceAddress(ptr, size, format, type_id) {} + + ~CPUDeviceAddress() override = default; + + bool SyncDeviceToHost(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const override; + bool SyncHostToDevice(const std::vector &shape, size_t size, TypeId type, const void *host_ptr) const override; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CPU_DEVICE_ADDRESS_H_ diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel.cc b/mindspore/ccsrc/device/cpu/cpu_kernel.cc new file mode 100644 index 00000000000..5f810ff522b --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_kernel.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +void CPUKernel::InitInputOutputSize(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + size_t type_size = sizeof(float); + for (size_t input_index = 0; input_index < input_num; ++input_index) { + std::vector shape = AnfAlgo::GetInputDeviceShape(kernel_node, input_index); + size_t tensor_size = + shape.empty() ? type_size : std::accumulate(shape.begin(), shape.end(), type_size, std::multiplies()); + input_size_list_.emplace_back(tensor_size); + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + for (size_t output_index = 0; output_index < output_num; ++output_index) { + std::vector shape = AnfAlgo::GetOutputDeviceShape(kernel_node, output_index); + size_t tensor_size = + shape.empty() ? type_size : std::accumulate(shape.begin(), shape.end(), type_size, std::multiplies()); + output_size_list_.emplace_back(tensor_size); + } +} + +void CPUKernel::Init(const CNodePtr &kernel_node) { + InitInputOutputSize(kernel_node); + InitKernel(kernel_node); +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel.h b/mindspore/ccsrc/device/cpu/cpu_kernel.h new file mode 100644 index 00000000000..ebd182ee492 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_kernel.h @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_H_ + +#include +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "ir/anf.h" +#include "session/anf_runtime_algorithm.h" + +using mindspore::kernel::Address; +using mindspore::kernel::AddressPtr; +namespace mindspore { +namespace device { +namespace cpu { +const char KSIZE[] = "ksize"; +const char STRIDE[] = "stride"; +const char STRIDES[] = "strides"; +const char DILATION[] = "dilation"; +const char PAD[] = "pad"; +const char PAD_MODE[] = "pad_mode"; +const char PADDING[] = "padding"; +const char PAD_MODE_LOWER_SAME[] = "same"; +const char PAD_MODE_LOWER_VALID[] = "valid"; +const char PAD_MODE_UPPER_SAME[] = "SAME"; +const char PAD_MODE_UPPER_VALID[] = "VALID"; +const char TRANSPOSE_A[] = "transpose_a"; +const char TRANSPOSE_B[] = "transpose_b"; +const char IS_GRAD[] = "is_grad"; +const char TRANSPOSE_NO = 'N'; +const char TRANSPOSE_YES = 'T'; +const char AXIS[] = "axis"; + +class CPUKernel : public kernel::KernelMod { + public: + CPUKernel() = default; + ~CPUKernel() override = default; + void Init(const CNodePtr &kernel_node); + virtual void InitKernel(const CNodePtr &kernel_node) = 0; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t /*stream_ptr*/) override { + return Launch(inputs, workspace, outputs); + }; + virtual bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) = 0; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + protected: + virtual void InitInputOutputSize(const CNodePtr &kernel_node); + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_factory.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_factory.cc new file mode 100644 index 00000000000..5aba329e12a --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_factory.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/cpu/cpu_kernel_factory.h" + +#include +#include +#include + +namespace mindspore { +namespace device { +namespace cpu { +CPUKernelFactory &CPUKernelFactory::Get() { + static CPUKernelFactory instance; + return instance; +} + +void CPUKernelFactory::Register(const std::string &kernel_name, CPUKernelCreator &&kernel_creator) { + if (kernel_creators_.find(kernel_name) == kernel_creators_.end()) { + (void)kernel_creators_.emplace(kernel_name, kernel_creator); + MS_LOG(DEBUG) << "CPUKernelFactory register operator: " << kernel_name; + } +} + +std::shared_ptr CPUKernelFactory::Create(const std::string &kernel_name) { + auto iter = kernel_creators_.find(kernel_name); + if (iter != kernel_creators_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + return (iter->second)(); + } + return nullptr; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_factory.h b/mindspore/ccsrc/device/cpu/cpu_kernel_factory.h new file mode 100644 index 00000000000..6a86f947099 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_factory.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_FACTORY_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_FACTORY_H_ + +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "device/cpu/cpu_kernel.h" +namespace mindspore { +namespace device { +namespace cpu { +using CPUKernelCreator = std::function()>; +class CPUKernelFactory { + public: + static CPUKernelFactory &Get(); + void Register(const std::string &kernel_name, CPUKernelCreator &&kernel_creator); + std::shared_ptr Create(const std::string &kernel_name); + + private: + CPUKernelFactory() = default; + ~CPUKernelFactory() = default; + DISABLE_COPY_AND_ASSIGN(CPUKernelFactory) + std::map kernel_creators_; +}; + +class CPUKernelRegistrar { + public: + CPUKernelRegistrar(const std::string &kernel_name, CPUKernelCreator &&kernel_creator) { + CPUKernelFactory::Get().Register(kernel_name, std::move(kernel_creator)); + } + ~CPUKernelRegistrar() = default; +}; + +#define MS_REG_CPU_KERNEL(KERNEL_NAME, KERNEL_CLASS) \ + static const CPUKernelRegistrar g_cpu_kernel_##KERNEL_NAME##_reg(#KERNEL_NAME, \ + []() { return std::make_shared(); }); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_FACTORY_H_ diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc new file mode 100644 index 00000000000..5bf9689f0bc --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc @@ -0,0 +1,275 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/cpu_kernel_runtime.h" +#include +#include +#include +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "device/cpu/cpu_device_address.h" +#include "utils/context/ms_context.h" +#include "utils/config_manager.h" +#include "common/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" + +namespace mindspore { +namespace device { +namespace cpu { +const size_t INIT_NODE_REF = 1; +void CPUKernelRuntime::AssignKernelAddress(session::KernelGraph *kernel_graph) { + AssignValueNodeAddress(kernel_graph); + AssignInputNodeAddress(kernel_graph); + AssignKernelOutputAddress(kernel_graph); + resource_manager_.MemPlan(kernel_graph); + resource_manager_.MemMalloc(kernel_graph); +} + +void CPUKernelRuntime::AssignValueNodeAddress(session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + size_t type_size = sizeof(float); + for (auto &item : kernel_graph->value_nodes()) { + auto item_node = item.first; + MS_EXCEPTION_IF_NULL(item_node); + if (item_node->isa()) { + auto value_node = item_node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto node_value = value_node->value(); + MS_EXCEPTION_IF_NULL(node_value); + if (!node_value->isa()) { + continue; + } + auto tensor = node_value->cast(); + MS_EXCEPTION_IF_NULL(tensor); + std::vector data_shape = tensor->shape(); + size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), type_size, std::multiplies()); + DeviceAddressPtr address = CreateDeviceAddress(nullptr, tensor_size, kOpFormat_DEFAULT, kNumberTypeFloat32); + if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { + address->ptr_ = tensor->data_c(false); + } else { + address->ptr_ = resource_manager_.MemMalloc(tensor_size); + if (!address->SyncHostToDevice(data_shape, LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c(false))) { + MS_LOG(EXCEPTION) << "value node sync host to device failed!"; + } + } + address->ref_count_ = INIT_NODE_REF; + AnfAlgo::SetOutputAddr(address, 0, item_node.get()); + } + } +} + +void CPUKernelRuntime::AssignInputNodeAddress(const session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + size_t type_size = sizeof(float); + for (auto &item : kernel_graph->inputs()) { + MS_EXCEPTION_IF_NULL(item); + if (item->isa()) { + auto output_num = AnfAlgo::GetOutputTensorNum(item); + for (size_t index = 0; index < output_num; index++) { + TypeId output_type_id = AnfAlgo::GetOutputDeviceDataType(item, index); + std::vector fmt_shape = AnfAlgo::GetOutputDeviceShape(item, index); + size_t tensor_size = + fmt_shape.empty() ? type_size + : std::accumulate(fmt_shape.begin(), fmt_shape.end(), type_size, std::multiplies()); + auto format = AnfAlgo::GetOutputFormat(item, index); + auto address = CreateDeviceAddress(nullptr, tensor_size, format, output_type_id); + AnfAlgo::SetOutputAddr(address, index, item.get()); + } + } + } +} + +void CPUKernelRuntime::AssignKernelOutputAddress(const session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto kernels = kernel_graph->execution_order(); + for (auto &kernel : kernels) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto output_sizes = kernel_mod->GetOutputSizeList(); + for (size_t i = 0; i < output_sizes.size(); ++i) { + auto output_format = AnfAlgo::GetOutputFormat(kernel, i); + auto output_type = AnfAlgo::GetOutputDeviceDataType(kernel, i); + AnfAlgo::SetOutputAddr(CreateDeviceAddress(nullptr, output_sizes[i], output_format, output_type), i, + kernel.get()); + } + auto workspace_sizes = kernel_mod->GetWorkspaceSizeList(); + for (size_t i = 0; i < workspace_sizes.size(); ++i) { + AnfAlgo::SetWorkspaceAddr(CreateDeviceAddress(nullptr, workspace_sizes[i], kOpFormat_DEFAULT, kNumberTypeFloat32), + i, kernel.get()); + } + } +} + +DeviceAddressPtr CPUKernelRuntime::CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) { + return std::make_shared(device_ptr, device_size, format, type_id); +} + +BaseRef CPUKernelRuntime::CreatTensorForOutput(const AnfNodePtr &input_node, size_t index, + const std::unordered_map &input_map) { + MS_EXCEPTION_IF_NULL(input_node); + if (input_node->isa() && AnfAlgo::GetCNodeName(input_node) == prim::kPrimMakeTuple->name()) { + auto cnode = input_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + VectorRef ret; + for (size_t i = 1; i < cnode->inputs().size(); i++) { + auto item_with_index = AnfAlgo::VisitKernelWithReturnType(cnode->input(i), 0); + auto out = CreatTensorForOutput(item_with_index.first, item_with_index.second, input_map); + ret.push_back(out); + } + return ret; + } + if (input_node->isa()) { + auto node = input_node->cast(); + MS_EXCEPTION_IF_NULL(node); + size_t output_size = AnfAlgo::GetOutputTensorNum(node); + if (index >= output_size) { + MS_LOG(EXCEPTION) << "invalid input index " << index; + } + auto address = AnfAlgo::GetMutableOutputAddr(node, index); + MS_EXCEPTION_IF_NULL(address); + auto shape = AnfAlgo::GetOutputInferShape(node, index); + std::vector temp_shape; + (void)temp_shape.insert(temp_shape.end(), shape.begin(), shape.end()); + TypeId type_id = AnfAlgo::GetOutputInferDataType(node, index); + if (type_id == kNumberTypeUInt32) { + type_id = kNumberTypeInt32; + } + if (type_id == kNumberTypeFloat || type_id == kNumberTypeFloat16 || type_id == kNumberTypeFloat32 || + type_id == kNumberTypeFloat64) { + type_id = kNumberTypeFloat32; + } + if (type_id != kNumberTypeInt32 && type_id != kNumberTypeFloat32) { + MS_LOG(EXCEPTION) << "check output type failed."; + } + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + MS_EXCEPTION_IF_NULL(tensor); + address->ptr_ = tensor->data_c(true); + address->ref_count_ = INIT_NODE_REF; + tensor->set_dirty(false); + return tensor; + } else if (input_node->isa() || input_node->isa()) { + auto iter = input_map.find(input_node.get()); + if (iter != input_map.end()) { + return iter->second; + } + } + return BaseRef(); +} + +void CPUKernelRuntime::BindInputOutput(const session::KernelGraph *kernel_graph, + const std::vector &inputs, VectorRef *outputs) { + MS_EXCEPTION_IF_NULL(kernel_graph); + MS_EXCEPTION_IF_NULL(outputs); + // bind input ptr + auto &input_nodes = kernel_graph->inputs(); + if (input_nodes.size() != inputs.size()) { + MS_LOG(EXCEPTION) << "input size not equal to input node size!"; + } + + std::unordered_map input_map; + size_t input_idx = 0; + size_t type_size = sizeof(float); + for (auto &item : input_nodes) { + MS_EXCEPTION_IF_NULL(item); + input_map[item.get()] = inputs[input_idx]; + if (item->isa()) { + auto address = AnfAlgo::GetMutableOutputAddr(item, 0); + auto tensor = inputs[input_idx]; + MS_EXCEPTION_IF_NULL(address); + MS_EXCEPTION_IF_NULL(tensor); + std::vector data_shape = tensor->shape(); + size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), type_size, std::multiplies()); + if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { + address->ptr_ = tensor->data_c(false); + } else { + address->ptr_ = resource_manager_.MemMalloc(tensor_size); + if (!address->SyncHostToDevice(data_shape, LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c(false))) { + MS_LOG(EXCEPTION) << "parameter node sync host to device failed!"; + } + tensor->set_dirty(true); + } + address->ref_count_ = INIT_NODE_REF; + tensor->set_device_address(address); + } + input_idx++; + } + + // new output and bind ptr + auto output_nodes = kernel_graph->outputs(); + for (const auto &item : output_nodes) { + auto item_with_index = AnfAlgo::VisitKernelWithReturnType(item, 0); + auto out = CreatTensorForOutput(item_with_index.first, item_with_index.second, input_map); + outputs->push_back(std::move(out)); + } +} + +void CPUKernelRuntime::AddRuntimeAddress(DeviceAddress *address, std::vector *input_list) { + MS_EXCEPTION_IF_NULL(address); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + if (address->ptr_ == nullptr) { + address->ptr_ = resource_manager_.MemMalloc(address->size_); + } + MS_EXCEPTION_IF_NULL(address->ptr_); + input->addr = address->ptr_; + input->size = address->size_; + input_list->push_back(input); +} + +bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + resource_manager_.ResetAddressRefCount(kernel_graph); + auto kernels = kernel_graph->execution_order(); + for (const auto &kernel : kernels) { + std::vector kernel_inputs; + std::vector kernel_workspaces; + std::vector kernel_outputs; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel); + for (size_t i = 0; i < input_num; ++i) { + auto device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i).get(); + MS_EXCEPTION_IF_NULL(device_address); + AddRuntimeAddress(device_address, &kernel_inputs); + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel); + for (size_t i = 0; i < output_num; ++i) { + auto device_address = AnfAlgo::GetMutableOutputAddr(kernel, i).get(); + MS_EXCEPTION_IF_NULL(device_address); + AddRuntimeAddress(device_address, &kernel_outputs); + } + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + auto device_address = AnfAlgo::GetWorkspaceAddr(kernel, i); + MS_EXCEPTION_IF_NULL(device_address); + AddRuntimeAddress(device_address, &kernel_workspaces); + } + auto ret = kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, 0); + resource_manager_.DecreaseAddressRefCount(kernel); + if (!ret) { + MS_LOG(EXCEPTION) << "Launch kernel failed."; + } + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h new file mode 100644 index 00000000000..28e61c14791 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_RUNTIME_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_RUNTIME_H_ + +#include +#include +#include +#include +#include "device/kernel_runtime.h" +#include "session/kernel_graph.h" +#include "device/cpu/cpu_resource_manager.h" +#include "utils/any.h" +namespace mindspore { +namespace device { +namespace cpu { +class CPUKernelRuntime : public KernelRuntime { + public: + CPUKernelRuntime() = default; + ~CPUKernelRuntime() override = default; + + bool Init() override { return true; } + bool Run(session::KernelGraph *graph) override; + void AssignKernelAddress(session::KernelGraph *kernel_graph); + void BindInputOutput(const session::KernelGraph *kernel_graph, const std::vector &inputs, + VectorRef *outputs); + + protected: + bool SyncStream() override { return true; }; + DeviceAddressPtr CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) override; + + private: + BaseRef CreatTensorForOutput(const AnfNodePtr &input_node, size_t index, + const std::unordered_map &input_map); + void AssignValueNodeAddress(session::KernelGraph *kernel_graph); + void AssignInputNodeAddress(const session::KernelGraph *kernel_graph); + void AssignKernelOutputAddress(const session::KernelGraph *kernel_graph); + void AddRuntimeAddress(DeviceAddress *address, std::vector *input_list); + CPUResourceManager resource_manager_; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CPU_KERNEL_RUNTIME_H_ diff --git a/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc b/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc new file mode 100644 index 00000000000..5b2b7a07503 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_resource_manager.cc @@ -0,0 +1,133 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/cpu_resource_manager.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace device { +namespace cpu { +CPUResourceManager::~CPUResourceManager() { MemFree(); } + +void CPUResourceManager::MemFree() { + if (mem_ptr_ != nullptr) { + free(mem_ptr_); + mem_ptr_ = nullptr; + mem_size_ = 0; + } + + for (auto &&iter : dynamic_mem_) { + free(iter.first); + } + dynamic_mem_.clear(); +} + +void CPUResourceManager::MemPlan(const session::KernelGraph *graph) { + mem_plan_.MemPlan(graph); + size_t graph_mem_size = mem_plan_.GetGraphMemSize(graph); + if (graph_mem_size > mem_size_) { + MemFree(); + mem_ptr_ = reinterpret_cast(malloc(graph_mem_size)); + if (mem_ptr_ != nullptr) { + mem_size_ = graph_mem_size; + dynamic_malloc_ = false; + } else { + MS_LOG(INFO) << "switch to dynamic malloc"; + dynamic_malloc_ = true; + } + } +} + +void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) { + if (dynamic_malloc_) { + return; + } + mem_plan_.MemAssign(graph, mem_ptr_); +} + +void *CPUResourceManager::MemMalloc(size_t mem_size) { + void *ptr = malloc(mem_size); + if (ptr != nullptr) { + dynamic_mem_[ptr] = mem_size; + return ptr; + } else { + MS_LOG(EXCEPTION) << "malloc memory failed: size " << mem_size; + } +} + +void CPUResourceManager::MemFree(void *ptr) { + auto iter = dynamic_mem_.find(ptr); + if (iter != dynamic_mem_.end()) { + (void)dynamic_mem_.erase(iter); + free(ptr); + } +} + +void CPUResourceManager::ResetAddressRefCount(const session::KernelGraph *graph) { + if (!dynamic_malloc_) { + return; + } + MS_EXCEPTION_IF_NULL(graph); + auto kernels = graph->execution_order(); + for (const auto &kernel : kernels) { + MS_EXCEPTION_IF_NULL(kernel); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel); + for (size_t i = 0; i < input_num; ++i) { + auto address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + address->ref_count_++; + } + + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + auto address = AnfAlgo::GetWorkspaceAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + address->ref_count_++; + } + } +} + +void CPUResourceManager::DecreaseAddressRefCount(const AnfNodePtr &kernel) { + if (!dynamic_malloc_) { + return; + } + MS_EXCEPTION_IF_NULL(kernel); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel); + for (size_t i = 0; i < input_num; ++i) { + auto address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + address->ref_count_--; + if (address->ref_count_ == 0 && address->ptr_ != nullptr) { + MemFree(address->ptr_); + address->ptr_ = nullptr; + } + } + + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + auto address = AnfAlgo::GetWorkspaceAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + address->ref_count_--; + if (address->ref_count_ == 0 && address->ptr_ != nullptr) { + MemFree(address->ptr_); + address->ptr_ = nullptr; + } + } +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_resource_manager.h b/mindspore/ccsrc/device/cpu/cpu_resource_manager.h new file mode 100644 index 00000000000..96cf00f3d85 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_resource_manager.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CPU_RESOURCE_MANAGER_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CPU_RESOURCE_MANAGER_H_ + +#include +#include +#include "session/kernel_graph.h" +#include "device/device_address.h" +#include "device/cpu/cpu_simple_mem_plan.h" +namespace mindspore { +namespace device { +namespace cpu { +class CPUResourceManager { + public: + CPUResourceManager() = default; + ~CPUResourceManager(); + + void MemPlan(const session::KernelGraph *graph); + void MemMalloc(const session::KernelGraph *graph); + void ResetAddressRefCount(const session::KernelGraph *graph); + void DecreaseAddressRefCount(const AnfNodePtr &kernel); + void *MemMalloc(size_t mem_size); + void MemFree(void *ptr); + + private: + void MemFree(); + CPUSimpleMemPlan mem_plan_; + + size_t mem_size_{0}; + uint8_t *mem_ptr_{nullptr}; + bool dynamic_malloc_{false}; + std::unordered_map dynamic_mem_; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CPU_RESOURCE_MANAGER_H_ diff --git a/mindspore/ccsrc/device/cpu/cpu_session.cc b/mindspore/ccsrc/device/cpu/cpu_session.cc new file mode 100644 index 00000000000..59d0318874f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_session.cc @@ -0,0 +1,123 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/cpu/cpu_session.h" +#include +#include "ir/meta_tensor.h" +#include "ir/anf.h" +#include "kernel/kernel.h" +#include "common/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_runtime.h" +#include "predict/predict.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace session { +GraphId CPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) { + auto graph_id = graph_sum_; + auto graph = ConstructKernelGraph(lst, outputs); + MS_EXCEPTION_IF_NULL(graph); + MS_LOG(INFO) << "set kernel info"; + SetKernelInfo(graph.get()); + predictmodel::StepConvertGraph(graph); + MS_LOG(INFO) << "build kernel"; + BuildKernel(graph.get()); + MS_LOG(INFO) << "assign kernel address"; + runtime_.AssignKernelAddress(graph.get()); + return graph_id; +} + +void CPUSession::RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *outputs) { + auto &kernel_graph = graphs_[graph_id]; + MS_EXCEPTION_IF_NULL(kernel_graph); + MS_LOG(INFO) << "bind input output address"; + runtime_.BindInputOutput(kernel_graph.get(), inputs, outputs); + MS_LOG(INFO) << "run graph start"; + predictmodel::StepConvertWeight(inputs); + auto execution_order = kernel_graph->execution_order(); + Reorder(&execution_order); + kernel_graph->set_execution_order(execution_order); + bool ret = runtime_.Run(kernel_graph.get()); + if (!ret) { + MS_LOG(EXCEPTION) << "run graph failed"; + } + MS_LOG(INFO) << "run graph end"; +} + +void CPUSession::SetKernelInfo(const KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto &kernel_nodes = kernel_graph->execution_order(); + for (const auto &kernel_node : kernel_nodes) { + MS_EXCEPTION_IF_NULL(kernel_node); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + for (size_t input_index = 0; input_index < input_num; ++input_index) { + auto input_kernel_node = kernel_node->input(input_index + 1); + MS_EXCEPTION_IF_NULL(input_kernel_node); + if (!input_kernel_node->isa()) { + continue; + } + auto builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(builder); + std::vector output_formats = {kOpFormat_DEFAULT}; + builder->SetOutputsFormat(output_formats); + std::vector output_types{kNumberTypeFloat32}; + builder->SetOutputsDeviceType(output_types); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), input_kernel_node.get()); + } + + auto builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(builder); + std::vector input_formats; + std::vector input_types; + std::vector output_formats; + std::vector output_types; + + for (size_t input_index = 0; input_index < input_num; ++input_index) { + input_formats.emplace_back(kOpFormat_DEFAULT); + input_types.emplace_back(kNumberTypeFloat32); + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + for (size_t output_index = 0; output_index < output_num; ++output_index) { + output_formats.emplace_back(kOpFormat_DEFAULT); + output_types.emplace_back(kNumberTypeFloat32); + } + builder->SetInputsFormat(input_formats); + builder->SetInputsDeviceType(input_types); + builder->SetOutputsFormat(output_formats); + builder->SetOutputsDeviceType(output_types); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), kernel_node.get()); + } +} + +void CPUSession::BuildKernel(const KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto &kernel_nodes = kernel_graph->execution_order(); + for (const auto &kernel_node : kernel_nodes) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::string kernel_name = AnfAlgo::GetCNodeName(kernel_node); + MS_LOG(INFO) << "Cpu building operator[" << kernel_name << "]."; + std::shared_ptr cpu_kernel = device::cpu::CPUKernelFactory::Get().Create(kernel_name); + if (cpu_kernel == nullptr) { + MS_LOG(EXCEPTION) << "Operator[" << kernel_name << "] is not support."; + } + cpu_kernel->Init(kernel_node); + AnfAlgo::SetKernelMod(cpu_kernel, kernel_node.get()); + MS_LOG(INFO) << "Cpu build success operator[" << kernel_name << "]."; + } +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_session.h b/mindspore/ccsrc/device/cpu/cpu_session.h new file mode 100644 index 00000000000..c53b0d2d8cb --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_session.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_CPU_SESSION_H +#define MINDSPORE_CCSRC_SESSION_CPU_SESSION_H +#include +#include +#include +#include "session/session_basic.h" +#include "session/kernel_graph.h" +#include "device/cpu/cpu_kernel_runtime.h" +#include "session/session_factory.h" +namespace mindspore { +namespace session { +class CPUSession : public SessionBasic { + public: + CPUSession() = default; + ~CPUSession() override = default; + void Init(uint32_t device_id) override { + SessionBasic::Init(device_id); + context_ = std::make_shared(kCPUDevice, device_id); + } + GraphId CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) override; + void RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *outputs) override; + + private: + void SetKernelInfo(const KernelGraph *kernel_graph); + void BuildKernel(const KernelGraph *kernel_graph); + device::cpu::CPUKernelRuntime runtime_; +}; +MS_REG_SESSION(kCPUDevice, CPUSession); +} // namespace session +} // namespace mindspore +#endif // MINDSPORE_CCSRC_SESSION_CPU_SESSION_H diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc new file mode 100644 index 00000000000..3fb76d38db2 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc @@ -0,0 +1,102 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/cpu_simple_mem_plan.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace device { +namespace cpu { +void CPUSimpleMemPlan::MemPlan(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + size_t total_mem_size = 0; + auto kernels = graph->execution_order(); + for (const auto &kernel : kernels) { + MS_EXCEPTION_IF_NULL(kernel); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel); + for (size_t i = 0; i < input_num; ++i) { + auto address = AnfAlgo::GetPrevNodeOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + if (address->ptr_ == nullptr) { + total_mem_size += address->size_; + } + } + + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel); + for (size_t i = 0; i < output_num; ++i) { + auto address = AnfAlgo::GetOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + if (address->ptr_ == nullptr) { + total_mem_size += address->size_; + } + } + + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + auto address = AnfAlgo::GetWorkspaceAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + if (address->ptr_ == nullptr) { + total_mem_size += address->size_; + } + } + } + graph_mem_size_[graph] = total_mem_size; +} + +size_t CPUSimpleMemPlan::GetGraphMemSize(const session::KernelGraph *graph) { return graph_mem_size_[graph]; } + +void CPUSimpleMemPlan::MemAssign(const session::KernelGraph *graph, uint8_t *base_ptr) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(base_ptr); + uint8_t *mem_ptr = base_ptr; + auto kernels = graph->execution_order(); + for (const auto &kernel : kernels) { + MS_EXCEPTION_IF_NULL(kernel); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel); + for (size_t i = 0; i < input_num; ++i) { + auto address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + if (address->ptr_ == nullptr) { + address->ptr_ = mem_ptr; + mem_ptr = mem_ptr + address->size_; + } + } + + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel); + for (size_t i = 0; i < output_num; ++i) { + auto address = AnfAlgo::GetMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + if (address->ptr_ == nullptr) { + address->ptr_ = mem_ptr; + mem_ptr = mem_ptr + address->size_; + } + } + + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + auto address = AnfAlgo::GetWorkspaceAddr(kernel, i); + MS_EXCEPTION_IF_NULL(address); + if (address->ptr_ == nullptr) { + address->ptr_ = mem_ptr; + mem_ptr = mem_ptr + address->size_; + } + } + } +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h new file mode 100644 index 00000000000..5f88966103b --- /dev/null +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CPU_SIMPLE_MEM_PLAN_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CPU_SIMPLE_MEM_PLAN_H_ + +#include +#include +#include "session/kernel_graph.h" +#include "device/device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +class CPUSimpleMemPlan { + public: + CPUSimpleMemPlan() = default; + ~CPUSimpleMemPlan() = default; + + void MemPlan(const session::KernelGraph *graph); + void MemAssign(const session::KernelGraph *graph, uint8_t *base_ptr); + size_t GetGraphMemSize(const session::KernelGraph *graph); + + private: + std::unordered_map graph_mem_size_; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CPU_SIMPLE_MEM_PLAN_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/apply_momentum_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/apply_momentum_cpu_kernel.cc new file mode 100644 index 00000000000..94b6c306ef7 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/apply_momentum_cpu_kernel.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/apply_momentum_cpu_kernel.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void ApplyMomentumCPUKernel::InitKernel(const CNodePtr & /*kernel_node*/) {} + +bool ApplyMomentumCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector & /*outputs*/) { + if (inputs.size() < 5) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + if (inputs[0]->size != inputs[1]->size || inputs[0]->size != inputs[3]->size) { + MS_LOG(EXCEPTION) << "error input data size!"; + } + auto weight = reinterpret_cast(inputs[0]->addr); + auto accumulate = reinterpret_cast(inputs[1]->addr); + float learning_rate = reinterpret_cast(inputs[2]->addr)[0]; + auto gradient = reinterpret_cast(inputs[3]->addr); + float moment = reinterpret_cast(inputs[4]->addr)[0]; + size_t elem_num = inputs[0]->size / sizeof(float); + for (size_t i = 0; i < elem_num; ++i) { + accumulate[i] = accumulate[i] * moment + gradient[i]; + weight[i] -= accumulate[i] * learning_rate; + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/apply_momentum_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/apply_momentum_cpu_kernel.h new file mode 100644 index 00000000000..84ed3404309 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/apply_momentum_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_APPLY_MOMENTUM_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_APPLY_MOMENTUM_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class ApplyMomentumCPUKernel : public MKLCPUKernel { + public: + ApplyMomentumCPUKernel() = default; + ~ApplyMomentumCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(ApplyMomentum, ApplyMomentumCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_APPLY_MOMENTUM_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/argmax_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/argmax_cpu_kernel.cc new file mode 100644 index 00000000000..a492bf79696 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/argmax_cpu_kernel.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/argmax_cpu_kernel.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void ArgmaxCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + if (shape.size() != 2) { + MS_LOG(EXCEPTION) << "argmax kernel dims invalid " << shape.size(); + } + batch_size_ = shape[0]; + class_num_ = shape[1]; + + int axis = AnfAlgo::GetNodeAttr(kernel_node, AXIS); + if (axis != -1 && axis != 1) { + MS_LOG(EXCEPTION) << "argmax kernel not support axis " << axis; + } +} + +bool ArgmaxCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspaces*/, + const std::vector &outputs) { + if (inputs.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "input or output empty!"; + } + + size_t batch_float_size = batch_size_ * sizeof(float); + size_t batch_class_float_size = class_num_ * batch_float_size; + if (inputs[0]->size != batch_class_float_size || outputs[0]->size != batch_float_size) { + MS_LOG(EXCEPTION) << "invalid input or output data size!"; + } + auto input = reinterpret_cast(inputs[0]->addr); + auto output = reinterpret_cast(outputs[0]->addr); + size_t row_start = 0; + for (size_t i = 0; i < batch_size_; ++i) { + size_t max_index = 0; + float max_value = input[row_start]; + for (size_t j = 1; j < class_num_; ++j) { + size_t index = row_start + j; + if (input[index] > max_value) { + max_value = input[index]; + max_index = j; + } + } + output[i] = SizeToInt(max_index); + row_start += class_num_; + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/argmax_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/argmax_cpu_kernel.h new file mode 100644 index 00000000000..993b012c4e0 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/argmax_cpu_kernel.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_ARGMAX_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_ARGMAX_CPU_KERNEL_H_ +#include +#include +#include "device/cpu/cpu_kernel.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace device { +namespace cpu { +class ArgmaxCPUKernel : public CPUKernel { + public: + ArgmaxCPUKernel() = default; + ~ArgmaxCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; + + private: + size_t class_num_{0}; + size_t batch_size_{0}; +}; + +MS_REG_CPU_KERNEL(Argmax, ArgmaxCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_ARGMAX_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/bias_add_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/bias_add_cpu_kernel.cc new file mode 100644 index 00000000000..4661ee73cd1 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/bias_add_cpu_kernel.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/cpu/kernel/bias_add_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +void BiasAddCpuKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + input_shape_ = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + bias_shape_ = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + if (input_shape_.size() == 4) { + data_shape_ = 4; + } else if (input_shape_.size() == 2) { + data_shape_ = 2; + } else { + MS_LOG(EXCEPTION) << "bias add input data format should be NCHW or NC"; + } + if (input_shape_.size() != 2 && input_shape_.size() != 4) { + MS_LOG(EXCEPTION) << "bias add input shape nchw or nc"; + } + if (bias_shape_.size() != 1) { + MS_LOG(EXCEPTION) << "bias shape invalid"; + } + if (input_shape_[1] != bias_shape_[0]) { + MS_LOG(EXCEPTION) << "bias shape not match"; + } +} + +bool BiasAddCpuKernel::Launch(const std::vector &inputs, const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() != 2 || outputs.size() != 1) { + MS_LOG(EXCEPTION) << "inputs outputs size not supoort"; + } + + auto src_addr = reinterpret_cast(inputs[0]->addr); + auto bias_addr = reinterpret_cast(inputs[1]->addr); + auto output_addr = reinterpret_cast(outputs[0]->addr); + + if (data_shape_ == 4) { + size_t h_size = input_shape_[3]; + size_t c_size = input_shape_[2] * h_size; + size_t n_size = input_shape_[1] * c_size; + size_t hw_size = input_shape_[2] * input_shape_[3]; + size_t n_offset = 0; + for (size_t n = 0; n < input_shape_[0]; ++n) { + size_t c_offset = 0; + for (size_t c = 0; c < input_shape_[1]; ++c) { + for (size_t hw = 0; hw < hw_size; ++hw) { + size_t offset = n_offset + c_offset + hw; + output_addr[offset] = src_addr[offset] + bias_addr[c]; + } + c_offset += c_size; + } + n_offset += n_size; + } + } else { + size_t n_offset = 0; + for (size_t n = 0; n < input_shape_[0]; ++n) { + for (size_t c = 0; c < input_shape_[1]; ++c) { + output_addr[n_offset + c] = src_addr[n_offset + c] + bias_addr[c]; + } + n_offset += input_shape_[1]; + } + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/bias_add_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/bias_add_cpu_kernel.h new file mode 100644 index 00000000000..45028523bd7 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/bias_add_cpu_kernel.h @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_CPU_KERNEL_BIAS_ADD_CPU_KERNEL_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_CPU_KERNEL_BIAS_ADD_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/cpu_kernel.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace device { +namespace cpu { +class BiasAddCpuKernel : public CPUKernel { + public: + BiasAddCpuKernel() = default; + ~BiasAddCpuKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; + + private: + uint8_t data_shape_{0}; + std::vector input_shape_; + std::vector bias_shape_; +}; +MS_REG_CPU_KERNEL(BiasAdd, BiasAddCpuKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_CPU_KERNEL_BIAS_ADD_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/bias_add_grad_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/bias_add_grad_cpu_kernel.cc new file mode 100644 index 00000000000..6846ca2555a --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/bias_add_grad_cpu_kernel.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/cpu/kernel/bias_add_grad_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +void BiasAddGradCpuKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + input_shape_ = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + if (input_shape_.size() != 4 && input_shape_.size() != 2) { + MS_LOG(EXCEPTION) << "input data format not support"; + } +} + +bool BiasAddGradCpuKernel::Launch(const std::vector &inputs, const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() != 1 || outputs.size() != 1) { + MS_LOG(EXCEPTION) << "input output size not support"; + } + auto output_addr = reinterpret_cast(outputs[0]->addr); + auto input_addr = reinterpret_cast(inputs[0]->addr); + + if (input_shape_.size() == 4) { + size_t h_size = input_shape_[3]; + size_t c_size = h_size * input_shape_[2]; + size_t n_size = c_size * input_shape_[1]; + size_t hw_size = input_shape_[2] * input_shape_[3]; + size_t c_offset = 0; + for (size_t c = 0; c < input_shape_[1]; ++c) { + output_addr[c] = 0; + size_t n_offset = 0; + for (size_t n = 0; n < input_shape_[0]; ++n) { + for (size_t hw = 0; hw < hw_size; ++hw) { + size_t offset = c_offset + n_offset + hw; + output_addr[c] += input_addr[offset]; + } + n_offset += n_size; + } + c_offset += c_size; + } + } else if (input_shape_.size() == 2) { + for (size_t c = 0; c < input_shape_[1]; ++c) { + output_addr[c] = 0; + size_t n_offset = 0; + for (size_t n = 0; n < input_shape_[0]; ++n) { + output_addr[c] += input_addr[c + n_offset]; + n_offset += input_shape_[1]; + } + } + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/bias_add_grad_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/bias_add_grad_cpu_kernel.h new file mode 100644 index 00000000000..736540b8a36 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/bias_add_grad_cpu_kernel.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_CPU_KERNEL_BIASADDGRADCPUKERNEL_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_CPU_KERNEL_BIASADDGRADCPUKERNEL_H_ + +#include +#include +#include "device/cpu/cpu_kernel.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace device { +namespace cpu { +class BiasAddGradCpuKernel : public CPUKernel { + public: + BiasAddGradCpuKernel() = default; + ~BiasAddGradCpuKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; + + private: + std::vector input_shape_; +}; +MS_REG_CPU_KERNEL(BiasAddGrad, BiasAddGradCpuKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_CPU_KERNEL_BIASADDGRADCPUKERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/equal_count_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/equal_count_cpu_kernel.cc new file mode 100644 index 00000000000..ee6bb9f144f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/equal_count_cpu_kernel.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/equal_count_cpu_kernel.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void EqualCountCPUKernel::InitKernel(const CNodePtr & /*kernel_node*/) {} + +bool EqualCountCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "input or output empty!"; + } + if (inputs[0]->size != inputs[1]->size) { + MS_LOG(EXCEPTION) << "input or output size!"; + } + int count = 0; + auto left = reinterpret_cast(inputs[0]->addr); + auto right = reinterpret_cast(inputs[1]->addr); + size_t elem_num = inputs[0]->size / sizeof(int); + for (size_t i = 0; i < elem_num; i++) { + if (left[i] == right[i]) { + count++; + } + } + auto output = reinterpret_cast(outputs[0]->addr); + output[0] = count; + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/equal_count_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/equal_count_cpu_kernel.h new file mode 100644 index 00000000000..26f343e855f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/equal_count_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_EQUAL_COUNT_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_EQUAL_COUNT_CPU_KERNEL_H_ +#include +#include +#include "device/cpu/cpu_kernel.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace device { +namespace cpu { +class EqualCountCPUKernel : public CPUKernel { + public: + EqualCountCPUKernel() = default; + ~EqualCountCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(EqualCount, EqualCountCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_EQUAL_COUNT_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_cpu_kernel.cc new file mode 100644 index 00000000000..f7527c47507 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_cpu_kernel.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/conv2d_cpu_kernel.h" +#include +#include "common/utils.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void Conv2dCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + std::vector weight_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + std::vector dst_shape = AnfAlgo::GetOutputDeviceShape(kernel_node, 0); + if (src_shape.size() != 4 || weight_shape.size() != 4) { + MS_LOG(EXCEPTION) << "conv2d only support nchw input!"; + } + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + dnnl::memory::desc weights_desc = GetDefaultMemDesc(weight_shape); + dnnl::memory::desc dst_desc = GetDefaultMemDesc(dst_shape); + + int kernel_size = SizeToInt(weight_shape[3]); + int stride = AnfAlgo::GetNodeAttr(kernel_node, STRIDE); + int dilation = AnfAlgo::GetNodeAttr(kernel_node, DILATION); + + dnnl::memory::dims strides{stride, stride}; + dnnl::memory::dims dilates{dilation - 1, dilation - 1}; + std::vector int_padding_l; + std::vector int_padding_r; + + const std::string pad_mode = AnfAlgo::GetNodeAttr(kernel_node, PAD_MODE); + GetPadding(kernel_node, pad_mode, src_shape, kernel_size, stride, &int_padding_l, &int_padding_r); + if (int_padding_l.size() != 2 || int_padding_r.size() != 2) { + MS_LOG(EXCEPTION) << "get padding failed"; + } + dnnl::memory::dims padding_l{int_padding_l[0], int_padding_l[1]}; + dnnl::memory::dims padding_r{int_padding_r[0], int_padding_r[1]}; + dnnl::convolution_forward::desc desc = + dnnl::convolution_forward::desc(dnnl::prop_kind::forward_training, dnnl::algorithm::convolution_auto, src_desc, + weights_desc, dst_desc, strides, dilates, padding_l, padding_r); + + auto prim_desc = dnnl::convolution_forward::primitive_desc(desc, MKLKernelEngine::Get().engine()); + primitive_ = std::make_shared(prim_desc); + + AddArgument(DNNL_ARG_SRC, src_desc); + AddArgument(DNNL_ARG_WEIGHTS, weights_desc); + AddArgument(DNNL_ARG_DST, dst_desc); +} + +bool Conv2dCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + SetArgumentHandle(DNNL_ARG_SRC, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_WEIGHTS, inputs[1]->addr); + SetArgumentHandle(DNNL_ARG_DST, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_cpu_kernel.h new file mode 100644 index 00000000000..d975b537ca6 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class Conv2dCPUKernel : public MKLCPUKernel { + public: + Conv2dCPUKernel() = default; + ~Conv2dCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Conv2D, Conv2dCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.cc new file mode 100644 index 00000000000..f4c0e58350e --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.h" +#include +#include "common/utils.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void Conv2dGradFilterCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + std::vector weight_shape = AnfAlgo::GetOutputDeviceShape(kernel_node, 0); + std::vector dst_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + if (src_shape.size() != 4 || weight_shape.size() != 4) { + MS_LOG(EXCEPTION) << ("conv2d grad filter only support nchw input!"); + } + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + dnnl::memory::desc weights_desc = GetDefaultMemDesc(weight_shape); + dnnl::memory::desc dst_desc = GetDefaultMemDesc(dst_shape); + + int kernel_size = SizeToInt(weight_shape[3]); + int stride = AnfAlgo::GetNodeAttr(kernel_node, STRIDE); + int dilation = AnfAlgo::GetNodeAttr(kernel_node, DILATION); + + dnnl::memory::dims strides{stride, stride}; + dnnl::memory::dims dilates{dilation - 1, dilation - 1}; + const std::string pad_mode = AnfAlgo::GetNodeAttr(kernel_node, PAD_MODE); + std::vector int_padding_l; + std::vector int_padding_r; + GetPadding(kernel_node, pad_mode, src_shape, kernel_size, stride, &int_padding_l, &int_padding_r); + if (int_padding_l.size() != 2 || int_padding_r.size() != 2) { + MS_LOG(EXCEPTION) << "get padding failed"; + } + dnnl::memory::dims padding_l{int_padding_l[0], int_padding_l[1]}; + dnnl::memory::dims padding_r{int_padding_r[0], int_padding_r[1]}; + dnnl::convolution_forward::desc forward_desc = + dnnl::convolution_forward::desc(dnnl::prop_kind::forward_training, dnnl::algorithm::convolution_auto, src_desc, + weights_desc, dst_desc, strides, dilates, padding_l, padding_r); + + auto forward_prim_desc = dnnl::convolution_forward::primitive_desc(forward_desc, MKLKernelEngine::Get().engine()); + + dnnl::convolution_backward_weights::desc backward_desc = dnnl::convolution_backward_weights::desc( + dnnl::algorithm::convolution_auto, src_desc, weights_desc, dst_desc, strides, dilates, padding_l, padding_r); + + auto backward_prim_desc = dnnl::convolution_backward_weights::primitive_desc( + backward_desc, MKLKernelEngine::Get().engine(), forward_prim_desc); + primitive_ = std::make_shared(backward_prim_desc); + + AddArgument(DNNL_ARG_SRC, src_desc); + AddArgument(DNNL_ARG_DIFF_DST, dst_desc); + AddArgument(DNNL_ARG_DIFF_WEIGHTS, weights_desc); +} + +bool Conv2dGradFilterCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + SetArgumentHandle(DNNL_ARG_SRC, inputs[1]->addr); + SetArgumentHandle(DNNL_ARG_DIFF_DST, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_DIFF_WEIGHTS, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.h new file mode 100644 index 00000000000..d42c1166f25 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_filter_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_GRAD_FILTER_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_GRAD_FILTER_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class Conv2dGradFilterCPUKernel : public MKLCPUKernel { + public: + Conv2dGradFilterCPUKernel() = default; + ~Conv2dGradFilterCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Conv2DBackpropFilter, Conv2dGradFilterCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_GRAD_FILTER_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.cc new file mode 100644 index 00000000000..492e2d6280f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.cc @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.h" +#include +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void Conv2dGradInputCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetOutputDeviceShape(kernel_node, 0); + std::vector weight_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + std::vector dst_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + if (src_shape.size() != 4 || weight_shape.size() != 4) { + MS_LOG(EXCEPTION) << "conv2d grad filter only support nchw input!"; + } + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + dnnl::memory::desc weights_desc = GetDefaultMemDesc(weight_shape); + dnnl::memory::desc dst_desc = GetDefaultMemDesc(dst_shape); + + int kernel_size = SizeToInt(weight_shape[3]); + int stride = AnfAlgo::GetNodeAttr(kernel_node, STRIDE); + int dilation = AnfAlgo::GetNodeAttr(kernel_node, DILATION); + dnnl::memory::dims strides{stride, stride}; + dnnl::memory::dims dilates{dilation - 1, dilation - 1}; + std::vector int_padding_l; + std::vector int_padding_r; + const std::string pad_mode = AnfAlgo::GetNodeAttr(kernel_node, PAD_MODE); + GetPadding(kernel_node, pad_mode, src_shape, kernel_size, stride, &int_padding_l, &int_padding_r); + if (int_padding_l.size() != 2 || int_padding_r.size() != 2) { + MS_LOG(EXCEPTION) << "conv2d grad get padding failed"; + } + dnnl::memory::dims padding_l{int_padding_l[0], int_padding_l[1]}; + dnnl::memory::dims padding_r{int_padding_r[0], int_padding_r[1]}; + dnnl::convolution_forward::desc forward_desc = + dnnl::convolution_forward::desc(dnnl::prop_kind::forward_training, dnnl::algorithm::convolution_auto, src_desc, + weights_desc, dst_desc, strides, dilates, padding_l, padding_r); + + auto forward_prim_desc = dnnl::convolution_forward::primitive_desc(forward_desc, MKLKernelEngine::Get().engine()); + + dnnl::convolution_backward_data::desc backward_desc = dnnl::convolution_backward_data::desc( + dnnl::algorithm::convolution_auto, src_desc, weights_desc, dst_desc, strides, dilates, padding_l, padding_r); + + auto backward_prim_desc = + dnnl::convolution_backward_data::primitive_desc(backward_desc, MKLKernelEngine::Get().engine(), forward_prim_desc); + primitive_ = std::make_shared(backward_prim_desc); + + AddArgument(DNNL_ARG_DIFF_SRC, src_desc); + AddArgument(DNNL_ARG_DIFF_DST, dst_desc); + AddArgument(DNNL_ARG_WEIGHTS, weights_desc); +} + +bool Conv2dGradInputCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + SetArgumentHandle(DNNL_ARG_DIFF_DST, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_WEIGHTS, inputs[1]->addr); + SetArgumentHandle(DNNL_ARG_DIFF_SRC, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.h new file mode 100644 index 00000000000..fb6e14688db --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/conv2d_grad_input_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_GRAD_INPUT_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_GRAD_INPUT_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class Conv2dGradInputCPUKernel : public MKLCPUKernel { + public: + Conv2dGradInputCPUKernel() = default; + ~Conv2dGradInputCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Conv2DBackpropInput, Conv2dGradInputCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_CONV2D_GRAD_INPUT_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/matmul_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/matmul_cpu_kernel.cc new file mode 100644 index 00000000000..84d9508c71a --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/matmul_cpu_kernel.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/matmul_cpu_kernel.h" +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "common/utils.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void MatMulCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + std::vector weight_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + std::vector dst_shape = AnfAlgo::GetOutputDeviceShape(kernel_node, 0); + + if (src_shape.size() != 2 || weight_shape.size() != 2 || dst_shape.size() != 2) { + MS_LOG(EXCEPTION) << "matmul invalid input size"; + } + bool trans_a = AnfAlgo::GetNodeAttr(kernel_node, TRANSPOSE_A); + bool trans_b = AnfAlgo::GetNodeAttr(kernel_node, TRANSPOSE_B); + if (trans_a) { + trans_a_ = TRANSPOSE_YES; + dim_m_ = static_cast(src_shape[1]); + dim_k_ = static_cast(src_shape[0]); + } else { + dim_m_ = static_cast(src_shape[0]); + dim_k_ = static_cast(src_shape[1]); + } + if (trans_b) { + trans_b_ = TRANSPOSE_YES; + } + dim_n_ = static_cast(dst_shape[1]); +} + +bool MatMulCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "matmul error input output size!"; + } + dnnl_dim_t lda = dim_m_; + if (trans_a_ == TRANSPOSE_NO) { + lda = dim_k_; + } + dnnl_dim_t ldb = dim_k_; + if (trans_b_ == TRANSPOSE_NO) { + ldb = dim_n_; + } + auto input_a = reinterpret_cast(inputs[0]->addr); + auto input_b = reinterpret_cast(inputs[1]->addr); + auto output = reinterpret_cast(outputs[0]->addr); + (void)dnnl_sgemm(trans_a_, trans_b_, dim_m_, dim_n_, dim_k_, 1.f, input_a, lda, input_b, ldb, 0.f, output, dim_n_); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/matmul_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/matmul_cpu_kernel.h new file mode 100644 index 00000000000..b096e767409 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/matmul_cpu_kernel.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_MATMUL_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_MATMUL_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class MatMulCPUKernel : public MKLCPUKernel { + public: + MatMulCPUKernel() = default; + ~MatMulCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; + + private: + char trans_a_{TRANSPOSE_NO}; + char trans_b_{TRANSPOSE_NO}; + dnnl_dim_t dim_m_{0}; + dnnl_dim_t dim_n_{0}; + dnnl_dim_t dim_k_{0}; +}; + +MS_REG_CPU_KERNEL(MatMul, MatMulCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_MATMUL_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_cpu_kernel.cc new file mode 100644 index 00000000000..677df1d0f80 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_cpu_kernel.cc @@ -0,0 +1,104 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" +#include +#include +#include +#include "common/utils.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" + +namespace mindspore { +namespace device { +namespace cpu { +void MKLCPUKernel::GetPadding(const CNodePtr &kernel_node, const std::string &pad_mode, + const std::vector &src_shape, int kernel_size, int stride, + std::vector *padding_l, std::vector *padding_r) { + MS_EXCEPTION_IF_NULL(kernel_node); + if (src_shape.size() < 2) { + MS_LOG(EXCEPTION) << "set pad only support src dim >= 2!"; + } + std::vector weight_height; + weight_height.emplace_back(src_shape[src_shape.size() - 2]); + weight_height.emplace_back(src_shape[src_shape.size() - 1]); + int rad = kernel_size / 2; + int need_pad = kernel_size - 1; + MS_LOG(INFO) << "pad mode " << pad_mode; + if (pad_mode == PAD_MODE_LOWER_SAME || pad_mode == PAD_MODE_UPPER_SAME) { + for (auto wh : weight_height) { + int re = (wh - 1) % stride; + int pad = std::max(rad - (re / 2), 0); + padding_r->emplace_back(pad); + pad = std::max(need_pad - pad - re, 0); + padding_l->emplace_back(pad); + } + } else if (pad_mode == PAD_MODE_LOWER_VALID || pad_mode == PAD_MODE_UPPER_VALID) { + MS_LOG(INFO) << "pad valid"; + padding_l->emplace_back(0); + padding_l->emplace_back(0); + padding_r->emplace_back(0); + padding_r->emplace_back(0); + } else { + std::vector pad = AnfAlgo::GetNodeAttr>(kernel_node, PAD); + if (pad.size() != 4) { + MS_LOG(EXCEPTION) << "wrong pad size in max pooling " << pad.size(); + } + padding_l->emplace_back(pad[0]); + padding_l->emplace_back(pad[1]); + padding_r->emplace_back(pad[2]); + padding_r->emplace_back(pad[3]); + } +} + +dnnl::memory::format_tag MKLCPUKernel::GetDefaultFormatTag(const dnnl::memory::dims &dims) const { + dnnl::memory::format_tag mem_tag; + auto dim_size = dims.size(); + if (dim_size == 4) { + mem_tag = dnnl::memory::format_tag::abcd; + } else if (dim_size == 3) { + mem_tag = dnnl::memory::format_tag::abc; + } else if (dim_size == 2) { + mem_tag = dnnl::memory::format_tag::ab; + } else if (dim_size == 1) { + mem_tag = dnnl::memory::format_tag::a; + } else { + MS_LOG(EXCEPTION) << "kernel dims invalid " << dim_size; + } + return mem_tag; +} + +dnnl::memory::desc MKLCPUKernel::GetDefaultMemDesc(const std::vector &shape) { + dnnl::memory::dims dims; + dims.insert(dims.end(), shape.begin(), shape.end()); + dnnl::memory::format_tag mem_tag = GetDefaultFormatTag(dims); + dnnl::memory::desc mem_desc(dims, dnnl::memory::data_type::f32, mem_tag); + return mem_desc; +} + +void MKLCPUKernel::AddArgument(int arg_key, const dnnl::memory::desc &mem_desc, bool alloc) { + arguments_[arg_key] = MKLKernelEngine::Get().CreateMemory(mem_desc, alloc); +} + +void MKLCPUKernel::SetArgumentHandle(int arg_key, void *ptr) { + auto arg_iter = arguments_.find(arg_key); + if (arg_iter != arguments_.end()) { + arg_iter->second.set_data_handle(ptr); + } +} + +void MKLCPUKernel::ExecutePrimitive() { MKLKernelEngine::Get().Execute(primitive_, arguments_); } +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_cpu_kernel.h new file mode 100644 index 00000000000..0a38de70601 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_cpu_kernel.h @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_MKL_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_MKL_CPU_KERNEL_H_ + +#include +#include +#include +#include +#include "dnnl.hpp" +#include "device/cpu/cpu_kernel.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace device { +namespace cpu { +class MKLCPUKernel : public CPUKernel { + public: + MKLCPUKernel() = default; + ~MKLCPUKernel() override = default; + + protected: + void GetPadding(const CNodePtr &kernel_node, const std::string &pad_mode, const std::vector &src_shape, + int kernel_size, int stride, std::vector *padding_l, std::vector *padding_r); + void AddArgument(int arg_key, const dnnl::memory::desc &mem_desc, bool alloc = false); + void SetArgumentHandle(int arg_key, void *ptr); + dnnl::memory::format_tag GetDefaultFormatTag(const dnnl::memory::dims &dims) const; + dnnl::memory::desc GetDefaultMemDesc(const std::vector &shape); + void ExecutePrimitive(); + std::unordered_map arguments_; + std::shared_ptr primitive_{nullptr}; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_MKL_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_kernel_engine.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_kernel_engine.cc new file mode 100644 index 00000000000..70251487327 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_kernel_engine.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "utils/log_adapter.h" +#include "dnnl.hpp" + +namespace mindspore { +namespace device { +namespace cpu { +void MKLKernelEngine::Execute(const std::shared_ptr &primitive, + const std::unordered_map &arguments) { + MS_EXCEPTION_IF_NULL(primitive); + primitive->execute(stream_, arguments); + (void)stream_.wait(); +} + +dnnl::memory MKLKernelEngine::CreateMemory(const dnnl::memory::desc &mem_desc, bool alloc) { + if (alloc) { + return dnnl::memory(mem_desc, engine_); + } else { + return dnnl::memory(mem_desc, engine_, nullptr); + } +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_kernel_engine.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_kernel_engine.h new file mode 100644 index 00000000000..ea764359b6f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mkl_kernel_engine.h @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MKL_KERNEL_ENGINE_H_ +#define MINDSPORE_MKL_KERNEL_ENGINE_H_ + +#include +#include +#include +#include "dnnl.hpp" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +class MKLKernelEngine { + public: + static MKLKernelEngine &Get() { + static MKLKernelEngine instance; + return instance; + } + DISABLE_COPY_AND_ASSIGN(MKLKernelEngine) + + const dnnl::engine &engine() const { return engine_; } + + dnnl::memory CreateMemory(const dnnl::memory::desc &mem_desc, bool alloc = false); + + void Execute(const std::shared_ptr &primitive, + const std::unordered_map &arguments); + + private: + MKLKernelEngine() : engine_(dnnl::engine::kind::cpu, 0), stream_(engine_) {} + ~MKLKernelEngine() = default; + dnnl::engine engine_; + dnnl::stream stream_; +}; +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_MKL_KERNEL_ENGINE_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/mul_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mul_cpu_kernel.cc new file mode 100644 index 00000000000..bdaa85559ee --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mul_cpu_kernel.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/mul_cpu_kernel.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void MulCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src0_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + std::vector src1_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + std::vector dst_shape = AnfAlgo::GetOutputDeviceShape(kernel_node, 0); + if (src0_shape.size() != src1_shape.size() && src1_shape.size() > 1) { + MS_LOG(EXCEPTION) << "mul only support same dim input or tensor * scalar " << src0_shape.size() << " vs " + << src1_shape.size(); + } + if (src1_shape.size() < src0_shape.size()) { + for (size_t i = src1_shape.size(); i < src0_shape.size(); ++i) { + src1_shape.emplace_back(1); + } + } + dnnl::memory::desc src0_mem_desc = GetDefaultMemDesc(src0_shape); + dnnl::memory::desc src1_mem_desc = GetDefaultMemDesc(src1_shape); + dnnl::memory::desc dst_mem_desc = GetDefaultMemDesc(dst_shape); + dnnl::binary::desc desc = dnnl::binary::desc(dnnl::algorithm::binary_mul, src0_mem_desc, src1_mem_desc, dst_mem_desc); + auto prim_desc = dnnl::binary::primitive_desc(desc, MKLKernelEngine::Get().engine()); + primitive_ = std::make_shared(prim_desc); + AddArgument(DNNL_ARG_SRC_0, src0_mem_desc); + AddArgument(DNNL_ARG_SRC_1, src1_mem_desc); + AddArgument(DNNL_ARG_DST, dst_mem_desc); +} + +bool MulCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "mul error input output size!"; + } + SetArgumentHandle(DNNL_ARG_SRC_0, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_SRC_1, inputs[1]->addr); + SetArgumentHandle(DNNL_ARG_DST, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/mul_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mul_cpu_kernel.h new file mode 100644 index 00000000000..e666197632c --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/mul_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_MUL_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_MUL_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class MulCPUKernel : public MKLCPUKernel { + public: + MulCPUKernel() = default; + ~MulCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Mul, MulCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_MUL_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_cpu_kernel.cc new file mode 100644 index 00000000000..9417105e2f5 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_cpu_kernel.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/pooling_cpu_kernel.h" +#include +#include +#include "common/utils.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void PoolingCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + std::vector dst_shape = AnfAlgo::GetOutputDeviceShape(kernel_node, 0); + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + dnnl::memory::desc dst_desc = GetDefaultMemDesc(dst_shape); + std::vector kernel_sizes = AnfAlgo::GetNodeAttr>(kernel_node, KSIZE); + std::vector strides = AnfAlgo::GetNodeAttr>(kernel_node, STRIDES); + if (kernel_sizes.size() != 4 || strides.size() != 4) { + MS_LOG(EXCEPTION) << "invalid kernel size " << kernel_sizes.size() << " or stride size " << strides.size(); + } + dnnl::memory::dims strides_dims{strides[2], strides[3]}; + dnnl::memory::dims kernels_dims{kernel_sizes[2], kernel_sizes[3]}; + const std::string pad_mode = AnfAlgo::GetNodeAttr(kernel_node, PADDING); + std::vector int_padding_l; + std::vector int_padding_r; + GetPadding(kernel_node, pad_mode, src_shape, kernel_sizes[3], strides[3], &int_padding_l, &int_padding_r); + if (int_padding_l.size() != 2 || int_padding_r.size() != 2) { + MS_LOG(EXCEPTION) << "pooling get padding failed"; + } + dnnl::memory::dims padding_l{int_padding_l[0], int_padding_l[1]}; + dnnl::memory::dims padding_r{int_padding_r[0], int_padding_r[1]}; + dnnl::pooling_forward::desc desc = + dnnl::pooling_forward::desc(dnnl::prop_kind::forward_training, dnnl::algorithm::pooling_max, src_desc, dst_desc, + strides_dims, kernels_dims, padding_l, padding_r); + auto prim_desc = dnnl::pooling_forward::primitive_desc(desc, MKLKernelEngine::Get().engine()); + primitive_ = std::make_shared(prim_desc); + AddArgument(DNNL_ARG_SRC, src_desc); + AddArgument(DNNL_ARG_DST, dst_desc); + AddArgument(DNNL_ARG_WORKSPACE, prim_desc.workspace_desc()); +} + +bool PoolingCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + SetArgumentHandle(DNNL_ARG_SRC, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_DST, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_cpu_kernel.h new file mode 100644 index 00000000000..50f4ff5da73 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_POOLING_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_POOLING_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class PoolingCPUKernel : public MKLCPUKernel { + public: + PoolingCPUKernel() = default; + ~PoolingCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(MaxPool, PoolingCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_POOLING_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.cc new file mode 100644 index 00000000000..724b78f19f3 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.cc @@ -0,0 +1,126 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.h" +#include +#include +#include +#include "common/utils.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void PoolingGradCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + src_shape_ = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + dst_shape_ = AnfAlgo::GetInputDeviceShape(kernel_node, 1); + std::vector kernel_sizes = AnfAlgo::GetNodeAttr>(kernel_node, KSIZE); + std::vector strides = AnfAlgo::GetNodeAttr>(kernel_node, STRIDES); + if (kernel_sizes.size() != 4 || strides.size() != 4 || src_shape_.size() != 4 || dst_shape_.size() != 4) { + MS_LOG(EXCEPTION) << "pooling grad invalid input size"; + } + std::vector padding_r; + const std::string pad_mode = AnfAlgo::GetNodeAttr(kernel_node, PADDING); + kernel_size_ = kernel_sizes[3]; + stride_ = strides[3]; + GetPadding(kernel_node, pad_mode, src_shape_, kernel_size_, stride_, &padding_l_, &padding_r); +} + +void PoolingGradCPUKernel::RowPoolingGrad(const float *input, float *output, float diff, + const std::vector> &box, + std::vector> *row_max_pair) { + float max_value = 0; + size_t max_index = box[1].second; + size_t src_width = src_shape_[3]; + size_t index_start; + size_t index; + for (size_t i = box[1].first; i < box[1].second; ++i) { + if ((*row_max_pair)[i].first == 0) { + index_start = box[0].first * src_width; + for (size_t j = box[0].first; j < box[0].second; ++j) { + index = index_start + i; + if (input[index] > (*row_max_pair)[i].second || j == box[0].first) { + (*row_max_pair)[i].second = input[index]; + (*row_max_pair)[i].first = index; + } + index_start += src_width; + } + } + if ((*row_max_pair)[i].second > max_value || max_index == box[1].second) { + max_value = (*row_max_pair)[i].second; + max_index = i; + } + } + + output[(*row_max_pair)[max_index].first] += diff; +} + +void PoolingGradCPUKernel::ChannelPoolingGrad(const float *input, const float *diff, float *output) { + int src_width = SizeToInt(src_shape_[3]); + int src_height = SizeToInt(src_shape_[2]); + std::vector> row_max_pair(src_shape_[3]); + std::vector> box(2); + int h_start = -padding_l_[0]; + size_t diff_index = 0; + for (size_t h = 0; h < dst_shape_[2]; ++h) { + box[0].first = IntToSize(std::max(h_start, 0)); + box[0].second = IntToSize(std::min(h_start + kernel_size_, src_height)); + for (size_t w = 0; w < src_shape_[3]; ++w) { + row_max_pair[w].first = 0; + row_max_pair[w].second = 0; + } + int w_start = -padding_l_[1]; + for (size_t w = 0; w < dst_shape_[3]; ++w) { + box[1].first = IntToSize(std::max(w_start, 0)); + box[1].second = IntToSize(std::min(w_start + kernel_size_, src_width)); + RowPoolingGrad(input, output, diff[diff_index], box, &row_max_pair); + diff_index += 1; + w_start += stride_; + } + h_start += stride_; + } +} + +bool PoolingGradCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 3 || outputs.empty()) { + MS_LOG(EXCEPTION) << "pooling grad error input output size!"; + } + + auto input = reinterpret_cast(inputs[0]->addr); + auto diff = reinterpret_cast(inputs[2]->addr); + auto output = reinterpret_cast(outputs[0]->addr); + auto ret = memset_s(output, outputs[0]->size, 0, outputs[0]->size); + if (ret != 0) { + MS_LOG(EXCEPTION) << "pooling grad memset error"; + } + size_t src_wh = src_shape_[2] * src_shape_[3]; + size_t dst_wh = dst_shape_[2] * dst_shape_[3]; + for (size_t n = 0; n < src_shape_[0]; ++n) { + for (size_t c = 0; c < src_shape_[1]; ++c) { + ChannelPoolingGrad(input, diff, output); + input = input + src_wh; + output = output + src_wh; + diff = diff + dst_wh; + } + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.h new file mode 100644 index 00000000000..e557f7f95f7 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/pooling_grad_cpu_kernel.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_POOLING_GRAD_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_POOLING_GRAD_CPU_KERNEL_H_ + +#include +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class PoolingGradCPUKernel : public MKLCPUKernel { + public: + PoolingGradCPUKernel() = default; + ~PoolingGradCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; + + private: + void RowPoolingGrad(const float *input, float *output, float diff, const std::vector> &box, + std::vector> *row_max_pair); + void ChannelPoolingGrad(const float *input, const float *diff, float *output); + int stride_{0}, kernel_size_{0}; + std::vector padding_l_; + std::vector src_shape_; + std::vector dst_shape_; +}; + +MS_REG_CPU_KERNEL(MaxPoolGrad, PoolingGradCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_POOLING_GRAD_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_cpu_kernel.cc new file mode 100644 index 00000000000..c8bf63eaf06 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_cpu_kernel.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/relu_cpu_kernel.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void ReluCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + if (src_shape.size() != 4 && src_shape.size() != 2) { + MS_LOG(EXCEPTION) << "relu kernel dims invalid " << src_shape.size(); + } + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + + dnnl::eltwise_forward::desc desc = + dnnl::eltwise_forward::desc(dnnl::prop_kind::forward_training, dnnl::algorithm::eltwise_relu, src_desc, 0.0); + auto prim_desc = dnnl::eltwise_forward::primitive_desc(desc, MKLKernelEngine::Get().engine()); + primitive_ = std::make_shared(prim_desc); + + AddArgument(DNNL_ARG_SRC, src_desc); + AddArgument(DNNL_ARG_DST, src_desc); +} + +bool ReluCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + SetArgumentHandle(DNNL_ARG_SRC, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_DST, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_cpu_kernel.h new file mode 100644 index 00000000000..8811539f407 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_RELU_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_RELU_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class ReluCPUKernel : public MKLCPUKernel { + public: + ReluCPUKernel() = default; + ~ReluCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(ReLU, ReluCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_RELU_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.cc new file mode 100644 index 00000000000..b831562d10f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void ReluGradCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + if (src_shape.size() != 4 && src_shape.size() != 2) { + MS_LOG(EXCEPTION) << "relu grad kernel dims invalid " << src_shape.size(); + } + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + + dnnl::eltwise_forward::desc forward_desc = + dnnl::eltwise_forward::desc(dnnl::prop_kind::forward_training, dnnl::algorithm::eltwise_relu, src_desc, 0.0); + auto forward_prim_desc = dnnl::eltwise_forward::primitive_desc(forward_desc, MKLKernelEngine::Get().engine()); + + dnnl::eltwise_backward::desc backward_desc = + dnnl::eltwise_backward::desc(dnnl::algorithm::eltwise_relu, src_desc, src_desc, 0.0, 0.0); + auto backward_prim_desc = + dnnl::eltwise_backward::primitive_desc(backward_desc, MKLKernelEngine::Get().engine(), forward_prim_desc); + primitive_ = std::make_shared(backward_prim_desc); + + AddArgument(DNNL_ARG_SRC, src_desc); + AddArgument(DNNL_ARG_DIFF_SRC, src_desc); + AddArgument(DNNL_ARG_DIFF_DST, src_desc); +} + +bool ReluGradCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 2 || outputs.empty()) { + MS_LOG(EXCEPTION) << "relu grad error input output size!"; + } + if (inputs[0]->size != outputs[0]->size) { + MS_LOG(EXCEPTION) << "relu grad error input output data size!"; + } + + SetArgumentHandle(DNNL_ARG_SRC, inputs[1]->addr); + SetArgumentHandle(DNNL_ARG_DIFF_SRC, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_DIFF_DST, inputs[0]->addr); + ExecutePrimitive(); + size_t mem_bits = outputs[0]->size; + auto ret = memcpy_s(outputs[0]->addr, mem_bits, inputs[0]->addr, mem_bits); + if (ret != 0) { + MS_LOG(EXCEPTION) << "memcpy_s error, errorno " << ret; + return false; + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.h new file mode 100644 index 00000000000..81b84916ba5 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/relu_grad_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_RELU_GRAD_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_RELU_GRAD_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class ReluGradCPUKernel : public MKLCPUKernel { + public: + ReluGradCPUKernel() = default; + ~ReluGradCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(ReluGrad, ReluGradCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_RELU_GRAD_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/softmax_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/softmax_cpu_kernel.cc new file mode 100644 index 00000000000..495f833c164 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/softmax_cpu_kernel.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/softmax_cpu_kernel.h" +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void SoftmaxCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector src_shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + std::vector axis_list = AnfAlgo::GetNodeAttr>(kernel_node, AXIS); + if (axis_list.size() != 1) { + MS_LOG(EXCEPTION) << "cpu softmax only support input axis size 1"; + } + int axis = axis_list[0]; + if (axis == -1 || axis >= SizeToInt(src_shape.size())) { + axis = SizeToInt(src_shape.size()) - 1; + } + dnnl::memory::desc src_desc = GetDefaultMemDesc(src_shape); + dnnl::softmax_forward::desc desc = dnnl::softmax_forward::desc(dnnl::prop_kind::forward_training, src_desc, axis); + auto prim_desc = dnnl::softmax_forward::primitive_desc(desc, MKLKernelEngine::Get().engine()); + primitive_ = std::make_shared(prim_desc); + AddArgument(DNNL_ARG_SRC, src_desc); + AddArgument(DNNL_ARG_DST, src_desc); +} + +bool SoftmaxCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "softmax error input output size!"; + } + SetArgumentHandle(DNNL_ARG_SRC, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_DST, outputs[0]->addr); + ExecutePrimitive(); + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/softmax_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/softmax_cpu_kernel.h new file mode 100644 index 00000000000..8f4ccae1b20 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/softmax_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_SOFTMAX_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_SOFTMAX_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class SoftmaxCPUKernel : public MKLCPUKernel { + public: + SoftmaxCPUKernel() = default; + ~SoftmaxCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Softmax, SoftmaxCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_SOFTMAX_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.cc new file mode 100644 index 00000000000..ca06b4a617e --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.cc @@ -0,0 +1,131 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.h" +#include +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_kernel_engine.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace cpu { +void SparseSoftmaxCrossEntropyWithLogitsCPUKernel::InitInputOutputSize(const CNodePtr &kernel_node) { + CPUKernel::InitInputOutputSize(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_node); + size_t type_size = sizeof(float); + std::vector shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + size_t tensor_size = std::accumulate(shape.begin(), shape.end(), type_size, std::multiplies()); + workspace_size_list_.emplace_back(tensor_size); +} + +void SparseSoftmaxCrossEntropyWithLogitsCPUKernel::InitKernel(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + std::vector shape = AnfAlgo::GetInputDeviceShape(kernel_node, 0); + dnnl::memory::dims mem_dims; + mem_dims.insert(mem_dims.end(), shape.begin(), shape.end()); + if (mem_dims.size() != 2) { + MS_LOG(EXCEPTION) << "SparseSoftmaxCrossEntropyWithLogits kernel dims invalid " << mem_dims.size(); + } + batch_size_ = shape[0]; + class_num_ = shape[1]; + if (batch_size_ == 0 || class_num_ == 0) { + MS_LOG(EXCEPTION) << "invalid batch size or class num input!"; + } + is_grad_ = AnfAlgo::GetNodeAttr(kernel_node, IS_GRAD); + dnnl::memory::desc mem_desc(mem_dims, dnnl::memory::data_type::f32, dnnl::memory::format_tag::nc); + + dnnl::softmax_forward::desc desc = dnnl::softmax_forward::desc(dnnl::prop_kind::forward_training, mem_desc, 1); + auto prim_desc = dnnl::softmax_forward::primitive_desc(desc, MKLKernelEngine::Get().engine()); + primitive_ = std::make_shared(prim_desc); + + AddArgument(DNNL_ARG_SRC, mem_desc); + AddArgument(DNNL_ARG_DST, mem_desc); +} + +void SparseSoftmaxCrossEntropyWithLogitsCPUKernel::ForwardPostExecute(const int *labels, const float *losses, + float *output) const { + float total_loss = 0; + for (size_t i = 0; i < batch_size_; ++i) { + if (labels[i] < 0) { + MS_LOG(EXCEPTION) << "label value must >= 0"; + } + size_t label = IntToSize(labels[i]); + if (label > class_num_) { + MS_LOG(EXCEPTION) << "error label input!"; + } + total_loss -= logf(losses[i * class_num_ + label]); + } + output[0] = total_loss / batch_size_; +} + +void SparseSoftmaxCrossEntropyWithLogitsCPUKernel::GradPostExecute(const int *labels, const float *losses, + float *output) const { + size_t row_start = 0; + for (size_t i = 0; i < batch_size_; ++i) { + if (labels[i] < 0) { + MS_LOG(EXCEPTION) << "label value must >= 0"; + } + size_t label = IntToSize(labels[i]); + if (label > class_num_) { + MS_LOG(EXCEPTION) << "error label input!"; + } + for (size_t j = 0; j < class_num_; ++j) { + size_t index = row_start + j; + if (j == label) { + output[index] = (losses[index] - 1) / batch_size_; + } else { + output[index] = losses[index] / batch_size_; + } + } + row_start += class_num_; + } +} + +bool SparseSoftmaxCrossEntropyWithLogitsCPUKernel::Launch(const std::vector &inputs, + const std::vector &workspace, + const std::vector &outputs) { + if (inputs.empty() || workspace.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "error input output size!"; + } + size_t batch_float_size = batch_size_ * sizeof(float); + size_t batch_class_float_size = class_num_ * batch_float_size; + if (inputs[0]->size != workspace[0]->size || inputs[0]->size != batch_class_float_size || + inputs[1]->size != batch_float_size) { + MS_LOG(EXCEPTION) << "error input data size!"; + } + if (is_grad_ && outputs[0]->size != batch_class_float_size) { + MS_LOG(EXCEPTION) << "error output data size!"; + } else if (!is_grad_ && outputs[0]->size != sizeof(float)) { + MS_LOG(EXCEPTION) << "error output data size!"; + } + SetArgumentHandle(DNNL_ARG_SRC, inputs[0]->addr); + SetArgumentHandle(DNNL_ARG_DST, workspace[0]->addr); + ExecutePrimitive(); + auto labels = reinterpret_cast(inputs[1]->addr); + auto losses = reinterpret_cast(workspace[0]->addr); + auto output = reinterpret_cast(outputs[0]->addr); + if (is_grad_) { + GradPostExecute(labels, losses, output); + } else { + ForwardPostExecute(labels, losses, output); + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.h new file mode 100644 index 00000000000..1d7169ea3d8 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/mkldnn/sparse_softmax_cross_entropy_with_logits_cpu_kernel.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_CPU_KERNEL_H_ + +#include +#include +#include "device/cpu/kernel/mkldnn/mkl_cpu_kernel.h" + +namespace mindspore { +namespace device { +namespace cpu { +class SparseSoftmaxCrossEntropyWithLogitsCPUKernel : public MKLCPUKernel { + public: + SparseSoftmaxCrossEntropyWithLogitsCPUKernel() = default; + ~SparseSoftmaxCrossEntropyWithLogitsCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; + + protected: + void InitInputOutputSize(const CNodePtr &kernel_node) override; + + private: + void ForwardPostExecute(const int *labels, const float *losses, float *output) const; + void GradPostExecute(const int *labels, const float *losses, float *output) const; + bool is_grad_{false}; + size_t class_num_{0}; + size_t batch_size_{0}; +}; + +MS_REG_CPU_KERNEL(SparseSoftmaxCrossEntropyWithLogits, SparseSoftmaxCrossEntropyWithLogitsCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/kernel/reshape_cpu_kernel.cc b/mindspore/ccsrc/device/cpu/kernel/reshape_cpu_kernel.cc new file mode 100644 index 00000000000..a742e3a5505 --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/reshape_cpu_kernel.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/cpu/kernel/reshape_cpu_kernel.h" +#include "device/cpu/cpu_device_address.h" + +namespace mindspore { +namespace device { +namespace cpu { +void ReshapeCPUKernel::InitKernel(const CNodePtr &kernel_node) { MS_EXCEPTION_IF_NULL(kernel_node); } + +bool ReshapeCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "input or output empty!"; + } + if (inputs[0]->size != outputs[0]->size) { + return false; + } + + if (inputs[0]->addr == outputs[0]->addr) { + return true; + } + + size_t mem_bits = outputs[0]->size; + auto ret = memcpy_s(outputs[0]->addr, mem_bits, inputs[0]->addr, mem_bits); + if (ret != 0) { + MS_LOG(EXCEPTION) << "memcpy_s error, errorno" << ret; + return false; + } + return true; +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/cpu/kernel/reshape_cpu_kernel.h b/mindspore/ccsrc/device/cpu/kernel/reshape_cpu_kernel.h new file mode 100644 index 00000000000..908c3df2d9f --- /dev/null +++ b/mindspore/ccsrc/device/cpu/kernel/reshape_cpu_kernel.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_CPU_RESHAPE_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_DEVICE_CPU_RESHAPE_CPU_KERNEL_H_ +#include +#include +#include "device/cpu/cpu_kernel.h" +#include "device/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace device { +namespace cpu { +class ReshapeCPUKernel : public CPUKernel { + public: + ReshapeCPUKernel() = default; + ~ReshapeCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Reshape, ReshapeCPUKernel); +} // namespace cpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_CPU_RESHAPE_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/device/cpu/readme.md b/mindspore/ccsrc/device/cpu/readme.md new file mode 100644 index 00000000000..40a991e2f3a --- /dev/null +++ b/mindspore/ccsrc/device/cpu/readme.md @@ -0,0 +1 @@ +cpu diff --git a/mindspore/ccsrc/device/device_address.h b/mindspore/ccsrc/device/device_address.h new file mode 100644 index 00000000000..1610d433724 --- /dev/null +++ b/mindspore/ccsrc/device/device_address.h @@ -0,0 +1,85 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_DEVICE_TENSOR_H +#define MINDSPORE_DEVICE_TENSOR_H + +#include +#include +#include +#include "ir/dtype.h" + +using std::string; + +namespace mindspore { +namespace device { +namespace cpu { +class CPUSimpleMemPlan; +class CPUResourceManager; +class CPUKernelRuntime; +} // namespace cpu +namespace ascend { +class AscendKernelRuntime; +namespace tasksink { +class TaskGenerator; +} // namespace tasksink +} // namespace ascend +namespace gpu { +class GPUKernelRuntime; +} // namespace gpu +} // namespace device +} // namespace mindspore + +namespace mindspore { +namespace device { +class DeviceAddress { + public: + explicit DeviceAddress(void *ptr, size_t size) : ptr_(ptr), size_(size) {} + explicit DeviceAddress(void *ptr, size_t size, const string &format, TypeId type_id) + : ptr_(ptr), size_(size), format_(format), type_id_(type_id) {} + virtual ~DeviceAddress() { ptr_ = nullptr; } + virtual bool SyncDeviceToHost(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const = 0; + virtual bool SyncHostToDevice(const std::vector &shape, size_t size, TypeId type, + const void *host_ptr) const = 0; + const void *GetPtr() const { return ptr_; } + size_t GetSize() const { return size_; } + std::string format() const { return format_; } + TypeId type_id() const { return type_id_; } + + protected: + const void *ptr() const { return ptr_; } + size_t size() const { return size_; } + void set_ptr(void *ptr) { ptr_ = ptr; } + void *ptr_{nullptr}; + size_t size_{0}; + size_t ref_count_{0}; + string format_{"DefaultFormat"}; + TypeId type_id_{kNumberTypeFloat16}; + bool mem_dynamic_alloc_{false}; + friend class KernelRuntime; + friend class mindspore::device::ascend::tasksink::TaskGenerator; + friend class mindspore::device::cpu::CPUSimpleMemPlan; + friend class mindspore::device::cpu::CPUResourceManager; + friend class mindspore::device::cpu::CPUKernelRuntime; + friend class mindspore::device::gpu::GPUKernelRuntime; + friend class mindspore::device::ascend::AscendKernelRuntime; +}; + +using DeviceAddressPtr = std::shared_ptr; +using DeviceAddressPtrList = std::vector; +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_DEVICE_TENSOR_H diff --git a/mindspore/ccsrc/device/gpu/blocking_queue.cc b/mindspore/ccsrc/device/gpu/blocking_queue.cc new file mode 100644 index 00000000000..7417115ae94 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/blocking_queue.cc @@ -0,0 +1,161 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/blocking_queue.h" +#include +#include "device/gpu/gpu_common.h" +#include "dataset/util/make_unique.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +GpuQueue::GpuQueue(void *addr, size_t feature_size, size_t label_size, size_t capacity) + : buffer_(addr), + head_(0), + tail_(0), + feature_size_(feature_size), + label_size_(label_size), + capacity_(capacity), + stream_(0), + node_info_(nullptr) { + CHECK_CUDA_RET_WITH_ERROR(cudaStreamCreate(&stream_), "Cuda Create Stream Failed"); + node_info_ = mindspore::make_unique(capacity); +} + +GpuQueue::~GpuQueue() { buffer_ = nullptr; } + +BlockQueueStatus_T GpuQueue::Push(void *feature_addr, size_t feature_size, void *label_addr, size_t label_size) { + if ((feature_addr == nullptr) || (label_addr == nullptr)) { + MS_LOG(ERROR) << "input nullptr"; + return ERROR_INPUT; + } + if ((feature_size != feature_size_) || (label_size != label_size_)) { + MS_LOG(ERROR) << "Data input error. Input data size: (" << feature_size << ", " << label_size << "), with (" + << feature_size_ << ", " << label_size_ << ") expect"; + return ERROR_INPUT; + } + void *feature_start_addr = reinterpret_cast(buffer_) + tail_ * (feature_size + label_size); + if (feature_start_addr == nullptr) { + MS_LOG(ERROR) << "feature start addr is nullptr"; + return INTERNAL_ERROR; + } + CHECK_CUDA_RET_WITH_ERROR( + cudaMemcpyAsync(feature_start_addr, feature_addr, feature_size, cudaMemcpyHostToDevice, stream_), + "Cuda Memcpy Error"); + void *label_start_addr = reinterpret_cast(feature_start_addr) + feature_size; + if (label_start_addr == nullptr) { + MS_LOG(ERROR) << "label start addr is nullptr"; + return INTERNAL_ERROR; + } + CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(label_start_addr, label_addr, label_size, cudaMemcpyHostToDevice, stream_), + "Cuda Memcpy Error"); + node_info_[tail_].event_.reset(new cudaEvent_t()); + CHECK_CUDA_RET_WITH_ERROR(cudaEventCreate(&(*(node_info_[tail_].event_))), "Cuda Create Event Failed"); + node_info_[tail_].host_feature_addr_ = feature_addr; + node_info_[tail_].host_label_addr_ = label_addr; + tail_ = (tail_ + 1) % (capacity_); + return SUCCESS; +} + +BlockQueueStatus_T GpuQueue::Front(void **feature_addr, size_t *feature_size, void **label_addr, + size_t *label_size) const { + CHECK_CUDA_RET_WITH_ERROR(cudaEventSynchronize(*(node_info_[head_].event_)), "Cuda Event Syn Failed"); + CHECK_CUDA_RET_WITH_ERROR(cudaEventDestroy(*(node_info_[head_].event_)), "Cuda Destroy Event Failed"); + *feature_addr = (unsigned char *)buffer_ + head_ * (feature_size_ + label_size_); + *feature_size = feature_size_; + *label_addr = (unsigned char *)buffer_ + head_ * (feature_size_ + label_size_) + feature_size_; + *label_size = label_size_; + host_release_(node_info_[head_].host_feature_addr_); + host_release_(node_info_[head_].host_label_addr_); + return SUCCESS; +} + +BlockQueueStatus_T GpuQueue::Pop() { + head_ = (head_ + 1) % (capacity_); + return SUCCESS; +} + +bool GpuQueue::Destroy() { + if (stream_ != nullptr) { + auto ret = cudaStreamDestroy(stream_); + if (ret == cudaSuccess) { + return true; + } else { + return false; + } + } else { + return true; + } +} + +BlockQueueStatus_T BlockingQueue::Create(void *addr, size_t feature_size, size_t label_size, size_t capacity) { + if (addr == nullptr) { + MS_LOG(ERROR) << "addr is nullptr"; + return INTERNAL_ERROR; + } + queue_ = std::make_shared(addr, feature_size, label_size, capacity); + return SUCCESS; +} + +void BlockingQueue::RegisterRelease(const std::function &func) { queue_->RegisterRelease(func); } + +BlockQueueStatus_T BlockingQueue::Push(void *feature_addr, size_t feature_size, void *label_addr, size_t label_size, + unsigned int timeout_in_sec) { + std::unique_lock locker(mutex_); + if (queue_->IsFull()) { + if (not_full_cond_.wait_for(locker, std::chrono::seconds(timeout_in_sec)) == std::cv_status::timeout) { + return TIMEOUT; + } + } + auto ret = queue_->Push(feature_addr, feature_size, label_addr, label_size); + if (ret) { + return ret; + } + not_empty_cond_.notify_one(); + return SUCCESS; +} + +BlockQueueStatus_T BlockingQueue::Front(void **feature_addr, size_t *feature_size, void **label_addr, + size_t *label_size) { + std::unique_lock locker(mutex_); + bool timeout = not_empty_cond_.wait_for(locker, std::chrono::seconds(30), [this] { return !queue_->IsEmpty(); }); + if (!timeout) { + return TIMEOUT; + } + + return queue_->Front(feature_addr, feature_size, label_addr, label_size); +} + +BlockQueueStatus_T BlockingQueue::Pop() { + std::unique_lock locker(mutex_); + not_empty_cond_.wait(locker, [this] { return !queue_->IsEmpty(); }); + auto ret = queue_->Pop(); + if (ret) { + return ret; + } + not_full_cond_.notify_one(); + return SUCCESS; +} + +bool BlockingQueue::Destroy() { + if (queue_ != nullptr) { + return queue_->Destroy(); + } else { + return true; + } +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/blocking_queue.h b/mindspore/ccsrc/device/gpu/blocking_queue.h new file mode 100644 index 00000000000..ccf481858f7 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/blocking_queue.h @@ -0,0 +1,92 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_BLOCKING_QUEUE_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_BLOCKING_QUEUE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mindspore { +namespace device { +enum BlockQueueStatus_T : int { SUCCESS = 0, QUEUE_NOT_EXIST, HANDLE_NOT_EXIST, ERROR_INPUT, INTERNAL_ERROR, TIMEOUT }; + +class GpuQueue { + public: + GpuQueue(void* addr, size_t feature_size, size_t label_size, size_t capacity); + virtual ~GpuQueue(); + + void RegisterRelease(const std::function& func) { host_release_ = func; } + + inline bool IsEmpty() const { return head_ == tail_; } + inline bool IsFull() const { return head_ == ((tail_ + 1) % (capacity_)); } + + BlockQueueStatus_T Push(void* feature_addr, size_t feature_size, void* label_addr, size_t label_size); + BlockQueueStatus_T Front(void** feature_addr, size_t* feature_size, void** label_addr, size_t* label_size) const; + BlockQueueStatus_T Pop(); + bool Destroy(); + + private: + struct NodeInfo { + std::unique_ptr event_; + void* host_feature_addr_; + void* host_label_addr_; + }; + + void* buffer_; + size_t head_; + size_t tail_; + size_t feature_size_; + size_t label_size_; + size_t capacity_; + cudaStream_t stream_; + std::unique_ptr node_info_; + std::function host_release_; + + GpuQueue(const GpuQueue&) = delete; + GpuQueue& operator=(const GpuQueue&) = delete; +}; + +class BlockingQueue { + public: + BlockingQueue() : queue_(nullptr) {} + ~BlockingQueue() = default; + + BlockQueueStatus_T Create(void* addr, size_t feature_size, size_t label_size, size_t capacity); + void RegisterRelease(const std::function& func); + BlockQueueStatus_T Push(void* feature_addr, size_t feature_size, void* label_addr, size_t label_size, + unsigned int timeout_in_sec); + BlockQueueStatus_T Front(void** feature_addr, size_t* feature_size, void** label_addr, size_t* label_size); + BlockQueueStatus_T Pop(); + bool Destroy(); + + private: + std::mutex mutex_; + std::condition_variable not_full_cond_; + std::condition_variable not_empty_cond_; + std::shared_ptr queue_; +}; +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_BLOCKING_QUEUE_H_ diff --git a/mindspore/ccsrc/device/gpu/cuda_common.h b/mindspore/ccsrc/device/gpu/cuda_common.h new file mode 100644 index 00000000000..5a5b6416cee --- /dev/null +++ b/mindspore/ccsrc/device/gpu/cuda_common.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_CUDA_COMMON_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_CUDA_COMMON_H_ + +#include +#include "device/gpu/gpu_device_manager.h" + +namespace mindspore { +namespace device { +namespace gpu { +class CudaCommon { + public: + inline int threads_num() const { return threads_per_block_; } + inline int major_sm() const { return major_sm_; } + inline int blocks_num(const int total_threads) const { + return std::min(((total_threads - 1) / threads_per_block_) + 1, max_blocks_); + } + + static CudaCommon &GetInstance() { + static CudaCommon instance; + return instance; + } + + private: + CudaCommon() { + uint32_t device_id = GPUDeviceManager::GetInstance().cur_device_id(); + cudaDeviceProp prop; + (void)cudaGetDeviceProperties(&prop, device_id); + threads_per_block_ = prop.maxThreadsPerBlock; + max_blocks_ = prop.multiProcessorCount; + major_sm_ = prop.major; + } + ~CudaCommon() = default; + CudaCommon(const CudaCommon &) = delete; + CudaCommon &operator=(const CudaCommon &) = delete; + + int max_blocks_; + int threads_per_block_; + int major_sm_; +}; +#define GET_BLOCKS(total_threads) mindspore::device::gpu::CudaCommon::GetInstance().blocks_num(total_threads) +#define GET_THREADS mindspore::device::gpu::CudaCommon::GetInstance().threads_num() +#define GET_MAJOR_SM mindspore::device::gpu::CudaCommon::GetInstance().major_sm() +#define MINIUM_SM 7 +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_CUDA_COMMON_H_ diff --git a/mindspore/ccsrc/device/gpu/cuda_driver.cc b/mindspore/ccsrc/device/gpu/cuda_driver.cc new file mode 100644 index 00000000000..3693157d2b7 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/cuda_driver.cc @@ -0,0 +1,144 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/cuda_driver.h" + +#include + +#include "utils/log_adapter.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace device { +namespace gpu { +size_t CudaDriver::AllocDeviceMem(size_t size, DeviceMemPtr *addr) { + size_t retreat_count = 0; + auto ret = cudaMalloc(reinterpret_cast(addr), size); + // If free memory is not enough, then retry with mem_malloc_retry_rate_. + while (ret == cudaErrorMemoryAllocation) { + size = FloatToSize(size * mem_malloc_retry_rate_); + size = (size / mem_malloc_align_size_) * mem_malloc_align_size_; + ret = cudaMalloc(reinterpret_cast(addr), size); + retreat_count++; + if (retreat_count > mem_malloc_retry_conut_max_) { + break; + } + } + + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaMalloc failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return 0; + } + return size; +} + +bool CudaDriver::FreeDeviceMem(const DeviceMemPtr &addr) { + auto ret = cudaFree(addr); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaFree failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} + +bool CudaDriver::CopyHostMemToDevice(const DeviceMemPtr &dst, const void *src, size_t size) { + auto ret = cudaMemcpy(dst, src, size, cudaMemcpyHostToDevice); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaMemcpy failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} + +bool CudaDriver::CopyDeviceMemToHost(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size) { + auto ret = cudaMemcpy(dst, src, size, cudaMemcpyDeviceToHost); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaMemcpy failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} + +size_t CudaDriver::total_mem_size() { + size_t free; + size_t total; + auto ret = cudaMemGetInfo(&free, &total); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaMemGetInfo failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return 0; + } + return total; +} + +size_t CudaDriver::free_mem_size() { + size_t free; + size_t total; + auto ret = cudaMemGetInfo(&free, &total); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaMemGetInfo failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return 0; + } + + return free; +} + +bool CudaDriver::CreateStream(DeviceStream *stream) { + auto ret = cudaStreamCreate(reinterpret_cast(stream)); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaStreamCreate failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} + +bool CudaDriver::DestroyStream(const DeviceStream &stream) { + auto ret = cudaStreamDestroy((cudaStream_t)stream); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaStreamDestroy failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} + +bool CudaDriver::SyncStream(const DeviceStream &stream) { + auto ret = cudaStreamSynchronize((cudaStream_t)stream); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaStreamSynchronize failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} + +int CudaDriver::device_count() { + int dev_count; + auto ret = cudaGetDeviceCount(&dev_count); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaGetDeviceCount failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + } + return dev_count; +} + +bool CudaDriver::set_current_device(int index) { + auto ret = cudaSetDevice(index); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaSetDevice failed, ret[" << static_cast(ret) << "], " << cudaGetErrorString(ret); + return false; + } + return true; +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/cuda_driver.h b/mindspore/ccsrc/device/gpu/cuda_driver.h new file mode 100644 index 00000000000..860e747f38d --- /dev/null +++ b/mindspore/ccsrc/device/gpu/cuda_driver.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_CUDA_DRIVER_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_CUDA_DRIVER_H_ + +#include + +namespace mindspore { +namespace device { +namespace gpu { +typedef void *DeviceStream; +typedef void *DeviceEvent; +typedef void *HostMemPtr; +typedef void *DeviceMemPtr; + +class CudaDriver { + public: + // Encapsulate the cuda APIs associated with memory operations + // such as malloc/free and memory copy from host to device and reverse. + static size_t AllocDeviceMem(size_t size, DeviceMemPtr *addr); + static bool FreeDeviceMem(const DeviceMemPtr &addr); + static bool CopyHostMemToDevice(const DeviceMemPtr &dst, const void *src, size_t size); + static bool CopyDeviceMemToHost(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size); + static size_t total_mem_size(); + static size_t free_mem_size(); + + // Encapsulate the cuda APIs associated with device resource + // such as Stream and Event. + static bool CreateStream(DeviceStream *stream); + static bool DestroyStream(const DeviceStream &stream); + static bool SyncStream(const DeviceStream &stream); + + // Encapsulate the cuda APIs associated with device management. + static int device_count(); + static bool set_current_device(int index); + + private: + CudaDriver() = delete; + ~CudaDriver() = delete; + CudaDriver(const CudaDriver &) = delete; + CudaDriver &operator=(const CudaDriver &) = delete; + + static constexpr float mem_malloc_retry_rate_{0.99}; + static constexpr size_t mem_malloc_retry_conut_max_{10}; + static constexpr size_t mem_malloc_align_size_{4}; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_CUDA_DRIVER_H_ diff --git a/mindspore/ccsrc/device/gpu/distribution/collective_common.h b/mindspore/ccsrc/device/gpu/distribution/collective_common.h new file mode 100644 index 00000000000..f9564a0c747 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/collective_common.h @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_COLLECTIVE_COMMON_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_COLLECTIVE_COMMON_H_ + +#include +#include "pybind11/pybind11.h" + +namespace mindspore { +namespace device { +namespace gpu { +#define MAX_HOSTNAME_LEN 1024 +#define CHECK_RET(expression, result, message) \ + { \ + auto ret = (expression); \ + if (ret != result) { \ + std::ostringstream oss; \ + oss << "Error in file " << __FILE__ << " | Error on line " << __LINE__ << " | GPU collective Error " << message \ + << " | Error Number " << ret; \ + pybind11::pybind11_fail(oss.str()); \ + } \ + } +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_COLLECTIVE_COMMON_H_ diff --git a/mindspore/ccsrc/device/gpu/distribution/collective_init.cc b/mindspore/ccsrc/device/gpu/distribution/collective_init.cc new file mode 100644 index 00000000000..d212c56ae73 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/collective_init.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/distribution/collective_init.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +namespace gpu { +CollectiveInitializer& CollectiveInitializer::instance() { + static CollectiveInitializer instance = {}; + return instance; +} + +bool CollectiveInitializer::collective_inited() const { return collective_inited_; } + +const void* CollectiveInitializer::collective_handle() const { return collective_handle_; } + +void CollectiveInitializer::InitCollective() { + void* handle = dlopen("libgpu_collective.so", RTLD_LAZY); + if (handle == nullptr) { + MS_LOG(EXCEPTION) + << "Loading libgpu_collective.so failed. Many reasons could cause this:\n1.libgpu_collective.so is not " + "installed.\n2.nccl is not " + "installed or found.\n3.mpi is not installed or found"; + } + auto mpi_init_funcptr = reinterpret_cast(dlsym(handle, "InitMPI")); + MS_EXCEPTION_IF_NULL(mpi_init_funcptr); + (*mpi_init_funcptr)(); + + CollectiveInitializer::instance().collective_inited_ = true; + CollectiveInitializer::instance().collective_handle_ = handle; +} + +void CollectiveInitializer::FinalizeCollective() { + if (CollectiveInitializer::instance().collective_handle_ != nullptr) { + if (dlclose(CollectiveInitializer::instance().collective_handle_) != 0) { + MS_LOG(EXCEPTION) << "Closing libgpu_collective.so handle failed."; + } + } +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/distribution/collective_init.h b/mindspore/ccsrc/device/gpu/distribution/collective_init.h new file mode 100644 index 00000000000..424abcf4700 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/collective_init.h @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_INIT_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_INIT_H_ + +#include + +namespace mindspore { +namespace device { +namespace gpu { +using InitMPI = void (*)(); +using InitNCCLComm = void (*)(); +using GetLocalRankId = int (*)(); + +class CollectiveInitializer { + public: + CollectiveInitializer(CollectiveInitializer const &) = delete; + CollectiveInitializer &operator=(const CollectiveInitializer &) = delete; + static CollectiveInitializer &instance(); + bool collective_inited() const; + const void *collective_handle() const; + static void InitCollective(); + static void FinalizeCollective(); + + private: + CollectiveInitializer() : collective_inited_(false) {} + ~CollectiveInitializer() = default; + + bool collective_inited_; + void *collective_handle_{nullptr}; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_INIT_H_ diff --git a/mindspore/ccsrc/device/gpu/distribution/collective_wrapper.cc b/mindspore/ccsrc/device/gpu/distribution/collective_wrapper.cc new file mode 100644 index 00000000000..5fb0f748492 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/collective_wrapper.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "device/gpu/distribution/mpi_wrapper.h" +#include "device/gpu/distribution/nccl_wrapper.h" + +#ifndef EXPORT_WRAPPER +#define EXPORT_WRAPPER __attribute__((visibility("default"))) +#endif + +using MPIWrapper = mindspore::device::gpu::MPIWrapper; +using NCCLWrapper = mindspore::device::gpu::NCCLWrapper; + +extern "C" EXPORT_WRAPPER void InitMPI() { MPIWrapper::instance(); } + +extern "C" EXPORT_WRAPPER int local_rank_id() { return MPIWrapper::instance().local_rank_id(); } + +extern "C" EXPORT_WRAPPER void InitNCCLComm() { NCCLWrapper::instance().InitNCCLComm(); } + +extern "C" EXPORT_WRAPPER ncclResult_t AllReduce(const void *input_addr, void *output_addr, size_t count, + ncclDataType_t data_type, ncclRedOp_t reduce_type, + cudaStream_t stream) { + return NCCLWrapper::instance().AllReduce(input_addr, output_addr, count, data_type, reduce_type, stream); +} + +extern "C" EXPORT_WRAPPER ncclResult_t AllGather(const void *input_addr, void *output_addr, size_t count, + ncclDataType_t data_type, cudaStream_t stream) { + return NCCLWrapper::instance().AllGather(input_addr, output_addr, count, data_type, stream); +} + +extern "C" EXPORT_WRAPPER ncclResult_t ReduceScatter(const void *input_addr, void *output_addr, size_t count, + ncclDataType_t data_type, ncclRedOp_t reduce_type, + cudaStream_t stream) { + return NCCLWrapper::instance().ReduceScatter(input_addr, output_addr, count, data_type, reduce_type, stream); +} diff --git a/mindspore/ccsrc/device/gpu/distribution/mpi_wrapper.cc b/mindspore/ccsrc/device/gpu/distribution/mpi_wrapper.cc new file mode 100644 index 00000000000..46b574c575d --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/mpi_wrapper.cc @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/distribution/mpi_wrapper.h" + +#include +#include +#include "device/gpu/distribution/nccl_wrapper.h" + +namespace mindspore { +namespace device { +namespace gpu { +MPIWrapper::MPIWrapper() : rank_id_(0), rank_size_(0), local_rank_id_(0) { Init(); } + +MPIWrapper::~MPIWrapper() { + int finalized; + MPI_Finalized(&finalized); + if (finalized == 0) { + MPI_Finalize(); + } +} + +MPIWrapper &MPIWrapper::instance() { + static MPIWrapper instance; + return instance; +} + +int MPIWrapper::local_rank_id() const { return local_rank_id_; } + +void MPIWrapper::Init() { + int initialized; + CHECK_RET(MPI_Initialized(&initialized), MPI_SUCCESS, "Failed to check mpi initialization status."); + + if (initialized == 0) { + MPI_Init(nullptr, nullptr); + } + CHECK_RET(MPI_Comm_rank(MPI_COMM_WORLD, &rank_id_), MPI_SUCCESS, "Failed to init mpi rank id."); + CHECK_RET(MPI_Comm_size(MPI_COMM_WORLD, &rank_size_), MPI_SUCCESS, "Failed to init mpi rank size."); + NCCLWrapper::instance().set_rank(rank_id_, rank_size_); + AssignLocalRankId(); + + ncclUniqueId unique_id; + if (rank_id_ == 0) { + unique_id = NCCLWrapper::instance().nccl_unique_id(); + } + CHECK_RET(MPI_Bcast(reinterpret_cast(&unique_id), sizeof(unique_id), MPI_BYTE, 0, MPI_COMM_WORLD), + MPI_SUCCESS, "Failed to broadcast nccl unique id."); + NCCLWrapper::instance().set_nccl_unique_id(unique_id); + return; +} + +void MPIWrapper::AssignLocalRankId() { + char host_name[MAX_HOSTNAME_LEN] = {0}; + CHECK_RET(gethostname(host_name, MAX_HOSTNAME_LEN), 0, "Getting host name failed."); + size_t host_hash = std::hash()(host_name); + + const int kRankSize = rank_size_; + size_t all_host_hashs[kRankSize]; + all_host_hashs[rank_id_] = host_hash; + CHECK_RET(MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, all_host_hashs, sizeof(size_t), MPI_BYTE, MPI_COMM_WORLD), + MPI_SUCCESS, "MPI_Allgather host hashs failed."); + for (int global_rank = 0; global_rank < kRankSize; global_rank++) { + if (global_rank == rank_id_) { + break; + } + if (all_host_hashs[global_rank] == all_host_hashs[rank_id_]) { + local_rank_id_++; + } + } + return; +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/distribution/mpi_wrapper.h b/mindspore/ccsrc/device/gpu/distribution/mpi_wrapper.h new file mode 100644 index 00000000000..6dfedea9220 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/mpi_wrapper.h @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_MPI_WRAPPER_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_MPI_WRAPPER_H_ + +#include +#include +#include +#include +#include +#include "device/gpu/distribution/collective_common.h" + +namespace mindspore { +namespace device { +namespace gpu { +class MPIWrapper { + public: + MPIWrapper(MPIWrapper const &) = delete; + MPIWrapper &operator=(const MPIWrapper &) = delete; + static MPIWrapper &instance(); + int local_rank_id() const; + + private: + MPIWrapper(); + ~MPIWrapper(); + void Init(); + void AssignLocalRankId(); + + int rank_id_; + int rank_size_; + int local_rank_id_; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_MPI_WRAPPER_H_ diff --git a/mindspore/ccsrc/device/gpu/distribution/nccl_wrapper.cc b/mindspore/ccsrc/device/gpu/distribution/nccl_wrapper.cc new file mode 100644 index 00000000000..aa4756a69f7 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/nccl_wrapper.cc @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/distribution/nccl_wrapper.h" + +namespace mindspore { +namespace device { +namespace gpu { +NCCLWrapper &NCCLWrapper::instance() { + static NCCLWrapper instance; + return instance; +} + +ncclUniqueId NCCLWrapper::nccl_unique_id() const { + ncclUniqueId unique_id; + CHECK_RET(ncclGetUniqueId(&unique_id), ncclSuccess, "Failed to create nccl unique id."); + return unique_id; +} + +void NCCLWrapper::set_nccl_unique_id(ncclUniqueId unique_id) { unique_id_ = unique_id; } + +void NCCLWrapper::set_rank(int rank_id, int rank_size) { + rank_id_ = rank_id; + rank_size_ = rank_size; +} + +void NCCLWrapper::InitNCCLComm() { + CHECK_RET(ncclCommInitRank(&comm_, rank_size_, unique_id_, rank_id_), ncclSuccess, + "Failed to init nccl communicator."); +} + +ncclResult_t NCCLWrapper::AllReduce(const void *input_addr, void *output_addr, size_t count, ncclDataType_t data_type, + ncclRedOp_t reduce_type, cudaStream_t stream) { + return ncclAllReduce(input_addr, output_addr, count, data_type, reduce_type, comm_, stream); +} + +ncclResult_t NCCLWrapper::AllGather(const void *input_addr, void *output_addr, size_t count, ncclDataType_t data_type, + cudaStream_t stream) { + return ncclAllGather(input_addr, output_addr, count, data_type, comm_, stream); +} + +ncclResult_t NCCLWrapper::ReduceScatter(const void *input_addr, void *output_addr, size_t count, + ncclDataType_t data_type, ncclRedOp_t reduce_type, cudaStream_t stream) { + return ncclReduceScatter(input_addr, output_addr, count, data_type, reduce_type, comm_, stream); +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/distribution/nccl_wrapper.h b/mindspore/ccsrc/device/gpu/distribution/nccl_wrapper.h new file mode 100644 index 00000000000..5df1e63bb8b --- /dev/null +++ b/mindspore/ccsrc/device/gpu/distribution/nccl_wrapper.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_NCCL_WRAPPER_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_NCCL_WRAPPER_H_ + +#include +#include +#include +#include "device/gpu/distribution/collective_common.h" + +namespace mindspore { +namespace device { +namespace gpu { +class NCCLWrapper { + public: + NCCLWrapper(NCCLWrapper const &) = delete; + NCCLWrapper &operator=(const NCCLWrapper &) = delete; + static NCCLWrapper &instance(); + ncclUniqueId nccl_unique_id() const; + void set_nccl_unique_id(ncclUniqueId unique_id); + void set_rank(int rank_id, int rank_size); + void InitNCCLComm(); + ncclResult_t AllReduce(const void *input_addr, void *output_addr, size_t count, ncclDataType_t datatype, + ncclRedOp_t op, cudaStream_t stream); + ncclResult_t AllGather(const void *input_addr, void *output_addr, size_t count, ncclDataType_t datatype, + cudaStream_t stream); + ncclResult_t ReduceScatter(const void *input_addr, void *output_addr, size_t count, ncclDataType_t datatype, + ncclRedOp_t op, cudaStream_t stream); + + private: + NCCLWrapper() : rank_id_(-1), rank_size_(0) {} + ~NCCLWrapper() = default; + + private: + int rank_id_; + int rank_size_; + ncclUniqueId unique_id_; + ncclComm_t comm_; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_NCCL_WRAPPER_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_buffer_mgr.cc b/mindspore/ccsrc/device/gpu/gpu_buffer_mgr.cc new file mode 100644 index 00000000000..bfffb9fc059 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_buffer_mgr.cc @@ -0,0 +1,204 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/gpu_buffer_mgr.h" +#include +#include +#include "utils/log_adapter.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +unsigned int HandleMgr::AllocHandle() { + for (size_t i = 0; i < MAX_HANDLE_NUM; ++i) { + if (!handle_list_[i]) { + handle_list_[i] = true; + return (unsigned int)i; + } + } + return INVALID_HANDLE; +} + +void HandleMgr::FreeHandle(unsigned int handle_id) { + if (handle_id >= MAX_HANDLE_NUM) { + return; + } + handle_list_[handle_id] = false; +} + +GpuBufferMgr &GpuBufferMgr::GetInstance() noexcept { + static GpuBufferMgr instance; + return instance; +} + +BlockQueueStatus_T GpuBufferMgr::Create(unsigned int device_id, const std::string &channel_name, void *addr, + const size_t &feature_len, const size_t &label_size, const size_t &capacity) { + std::string name = std::to_string(device_id) + std::string("_") + channel_name; + if (name_queue_map_.count(name)) { + MS_LOG(ERROR) << "Queue not exist " << name; + return QUEUE_NOT_EXIST; + } + std::shared_ptr queue = std::make_shared(); + BlockQueueStatus_T rt = queue->Create(addr, feature_len, label_size, capacity); + if (rt != SUCCESS) { + return rt; + } + (void)name_queue_map_.insert(std::make_pair(name, queue)); + init_ = true; + return SUCCESS; +} + +unsigned int GpuBufferMgr::Open(unsigned int device_id, const std::string &channel_name, const size_t &, const size_t &, + const std::function func) { + set_device(); + std::string name = std::to_string(device_id) + std::string("_") + channel_name; + if (!name_queue_map_.count(name)) { + MS_LOG(ERROR) << "Queue not exist " << name; + return HandleMgr::INVALID_HANDLE; + } + unsigned int handle = handle_mgr_.AllocHandle(); + if (handle == HandleMgr::INVALID_HANDLE) { + MS_LOG(ERROR) << "handle is invalid"; + return HandleMgr::INVALID_HANDLE; + } + (void)handle_queue_map_.insert(std::make_pair(handle, name_queue_map_[name])); + name_queue_map_[name]->RegisterRelease(func); + open_by_dataset_++; + return handle; +} + +unsigned int GpuBufferMgr::Open(unsigned int device_id, const std::string &channel_name, const size_t &, + const size_t &) { + set_device(); + std::string name = std::to_string(device_id) + std::string("_") + channel_name; + if (!name_queue_map_.count(name)) { + MS_LOG(ERROR) << "Queue not exist " << name; + return HandleMgr::INVALID_HANDLE; + } + unsigned int handle = handle_mgr_.AllocHandle(); + if (handle == HandleMgr::INVALID_HANDLE) { + MS_LOG(ERROR) << "handle is invalid"; + return HandleMgr::INVALID_HANDLE; + } + (void)handle_queue_map_.insert(std::make_pair(handle, name_queue_map_[name])); + return handle; +} + +void GpuBufferMgr::set_device_id(int device_id) { cur_dev_id_ = device_id; } + +void GpuBufferMgr::set_device() const { + auto ret = cudaSetDevice(cur_dev_id_); + if (ret != cudaSuccess) { + MS_LOG(ERROR) << "cudaSetDevice, ret[" << static_cast(ret) << "]"; + } +} + +BlockQueueStatus_T GpuBufferMgr::Push(unsigned int handle, void *feature_addr, size_t feature_size, void *label_addr, + size_t label_size, unsigned int timeout_in_sec) { + auto iter = handle_queue_map_.find(handle); + if (iter == handle_queue_map_.end()) { + return HANDLE_NOT_EXIST; + } + return iter->second->Push(feature_addr, feature_size, label_addr, label_size, timeout_in_sec); +} + +BlockQueueStatus_T GpuBufferMgr::Front(unsigned int handle, void **feature_addr, size_t *feature_size, + void **label_addr, size_t *label_size) { + auto iter = handle_queue_map_.find(handle); + if (iter == handle_queue_map_.end()) { + return HANDLE_NOT_EXIST; + } + return iter->second->Front(feature_addr, feature_size, label_addr, label_size); +} + +BlockQueueStatus_T GpuBufferMgr::Pop(unsigned int handle) { + auto iter = handle_queue_map_.find(handle); + if (iter == handle_queue_map_.end()) { + return HANDLE_NOT_EXIST; + } + return iter->second->Pop(); +} + +void GpuBufferMgr::Close(unsigned int handle) noexcept { + if (!handle_queue_map_.count(handle)) { + return; + } + (void)handle_queue_map_.erase(handle); + handle_mgr_.FreeHandle(handle); + return; +} + +bool GpuBufferMgr::IsInit() const { return init_; } + +bool GpuBufferMgr::IsClosed() const { return closed_; } + +bool GpuBufferMgr::Destroy() { + for (auto iter = name_queue_map_.begin(); iter != name_queue_map_.end(); ++iter) { + std::shared_ptr queue = iter->second; + if (queue != nullptr) { + if (!queue->Destroy()) { + return false; + } + queue.reset(); + } + } + name_queue_map_.clear(); + return true; +} + +inline bool GpuBufferMgr::isCreated(unsigned int device_id, const std::string &channel_name) { + std::string name = std::to_string(device_id) + std::string("_") + channel_name; + if (name_queue_map_.count(name) != 0) { + return true; + } + return false; +} + +bool GpuBufferMgr::CloseNotify() { + bool result = true; + // lock scope + { + std::lock_guard lk(close_mutex_); + // set closed_ to be true, all the dataset retry can be jumped out of the while + closed_ = true; // set closed_ to be true, all the dataset retry can be jumped out of the while + // notify all the waiting dataset threads + close_confirm_cond_.notify_all(); // notify all the waiting dataset threads + } + + // wati for the dataset threads' ack + for (int i = 0; i < open_by_dataset_; i++) { + if (sema.Wait() == false) { + MS_LOG(ERROR) << "time out of receiving signals"; + result = false; + } + MS_LOG(DEBUG) << "receive one signal (" << i + 1 << "/" << open_by_dataset_ << ")"; + } + return result; +} + +void GpuBufferMgr::CloseConfirm() { + // lock scope + { + std::unique_lock lk(close_mutex_); + // dataset threads wait for the closed_ flag from false to true + close_confirm_cond_.wait( + lk, [this] { return closed_; }); // dataset threads wait for the closed_ flag from false to true + } + + sema.Signal(); +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/gpu_buffer_mgr.h b/mindspore/ccsrc/device/gpu/gpu_buffer_mgr.h new file mode 100644 index 00000000000..447564914a1 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_buffer_mgr.h @@ -0,0 +1,141 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPU_BUFFER_MGR_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPU_BUFFER_MGR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "device/gpu/blocking_queue.h" + +#define EXPORT __attribute__((visibility("default"))) + +namespace mindspore { +namespace device { +static const unsigned int MAX_WAIT_TIME_IN_SEC = 60; + +class Semaphore { + public: + explicit Semaphore(int count = 0) : count_(count) {} + + inline void Signal() { + std::unique_lock lock(mutex_); + ++count_; + cv_.notify_one(); + } + + inline bool Wait() { + std::unique_lock lock(mutex_); + while (count_ == 0) { + if (cv_.wait_for(lock, std::chrono::seconds(MAX_WAIT_TIME_IN_SEC)) == std::cv_status::timeout) { + return false; + } + } + --count_; + return true; + } + + private: + std::mutex mutex_; + std::condition_variable cv_; + int count_; +}; + +class HandleMgr { + public: + static const unsigned int MAX_HANDLE_NUM = 32; + static const unsigned int INVALID_HANDLE = 0xffffffffUL; + + unsigned int AllocHandle(); + void FreeHandle(unsigned int); + + private: + bool handle_list_[MAX_HANDLE_NUM]; +}; + +class GpuBufferMgr { + public: + EXPORT GpuBufferMgr() : cur_dev_id_(0), init_(false), closed_(false), open_by_dataset_(0) {} + + EXPORT virtual ~GpuBufferMgr() = default; + + EXPORT static GpuBufferMgr &GetInstance() noexcept; + + EXPORT BlockQueueStatus_T Create(unsigned int device_id, const std::string &channel_name, void *addr, + const size_t &feature_len, const size_t &label_size, const size_t &capacity); + + // call for Push thread + EXPORT unsigned int Open(unsigned int device_id, const std::string &channel_name, const size_t &feature_len, + const size_t &label_size, std::function func); + + // call for Front/Pop thread + EXPORT unsigned int Open(unsigned int device_id, const std::string &channel_name, const size_t &feature_len, + const size_t &label_size); + + EXPORT BlockQueueStatus_T Push(unsigned int handle, void *feature_addr, size_t feature_size, void *label_addr, + size_t label_size, unsigned int timeout_in_sec); + EXPORT BlockQueueStatus_T Front(unsigned int handle, void **feature_addr, size_t *feature_size, void **label_addr, + size_t *label_size); + EXPORT BlockQueueStatus_T Pop(unsigned int handle); + + EXPORT void set_device_id(int device_id); + + EXPORT void Close(unsigned int handle) noexcept; + + EXPORT bool IsInit() const; + + EXPORT bool IsClosed() const; + + EXPORT bool Destroy(); + + // call for Release GPU Resources + EXPORT bool CloseNotify(); + + // call for dataset send thread + EXPORT void CloseConfirm(); + + private: + void set_device() const; + + int cur_dev_id_; + bool init_; + bool closed_; + std::mutex mutex_; + std::mutex close_mutex_; + std::condition_variable close_confirm_cond_; + // how many queues opened by dataset + int open_by_dataset_; + Semaphore sema; + + HandleMgr handle_mgr_; + + std::map> handle_queue_map_; + std::map> name_queue_map_; + + inline bool isCreated(unsigned int device_id, const std::string &channel_name); + + GpuBufferMgr(const GpuBufferMgr &) = delete; + GpuBufferMgr &operator=(const GpuBufferMgr &) = delete; +}; +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPU_BUFFER_MGR_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_common.h b/mindspore/ccsrc/device/gpu/gpu_common.h new file mode 100644 index 00000000000..c1b61464875 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_common.h @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPU_COMMON_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPU_COMMON_H_ + +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +namespace gpu { +#define CHECK_OP_RET_WITH_EXCEPT(expression, message) \ + { \ + bool success = (expression); \ + if (!success) { \ + MS_LOG(EXCEPTION) << "Op Error: " << message << " | Error Number: " << success; \ + } \ + } + +#define CHECK_OP_RET_WITH_ERROR(expression, message) \ + { \ + bool success = (expression); \ + if (!success) { \ + MS_LOG(ERROR) << "Op Error: " << message << " | Error Number: " << success; \ + } \ + } + +#define CHECK_CUDA_RET_WITH_ERROR(expression, message) \ + { \ + cudaError_t status = (expression); \ + if (status != cudaSuccess) { \ + MS_LOG(ERROR) << "CUDA Error: " << message << " | Error Number: " << status << " " \ + << cudaGetErrorString(status); \ + } \ + } + +#define CHECK_CUDA_RET_WITH_EXCEPT(expression, message) \ + { \ + cudaError_t status = (expression); \ + if (status != cudaSuccess) { \ + MS_LOG(EXCEPTION) << "CUDA Error: " << message << " | Error Number: " << status << " " \ + << cudaGetErrorString(status); \ + } \ + } + +#define CHECK_CUDNN_RET_WITH_EXCEPT(expression, message) \ + { \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + MS_LOG(EXCEPTION) << "cuDNN Error: " << message << " | Error Number: " << status << " " \ + << cudnnGetErrorString(status); \ + } \ + } + +#define CHECK_CUDNN_RET_WITH_ERROR(expression, message) \ + { \ + cudnnStatus_t status = (expression); \ + if (status != CUDNN_STATUS_SUCCESS) { \ + MS_LOG(ERROR) << "cuDNN Error: " << message << " | Error Number: " << status << " " \ + << cudnnGetErrorString(status); \ + } \ + } + +#define CHECK_CUBLAS_RET_WITH_EXCEPT(expression, message) \ + { \ + cublasStatus_t status = (expression); \ + if (status != CUBLAS_STATUS_SUCCESS) { \ + MS_LOG(EXCEPTION) << "cuBLAS Error: " << message << " | Error Number: " << status; \ + } \ + } + +#define CHECK_CUBLAS_RET_WITH_ERROR(expression, message) \ + { \ + cublasStatus_t status = (expression); \ + if (status != CUBLAS_STATUS_SUCCESS) { \ + MS_LOG(ERROR) << "cuBLAS Error: " << message << " | Error Number: " << status; \ + } \ + } + +#define CHECK_NCCL_RET_WITH_EXCEPT(expression, message) \ + { \ + int result = (expression); \ + if (result != ncclSuccess) { \ + MS_LOG(EXCEPTION) << "NCCL Error: " << message << " | Error Number: " << result; \ + } \ + } + +#define VARIABLE_NOT_USED(var) \ + { (void)(var); } + +inline bool CheckNullInput(std::vector input_shape) { + // If input_shape.size() == 0, it means a scalar input; If input_shape.size() != 0 and input_shape contains 0, + // it means a null input. Just return a null output. + if (input_shape.size() != 0) { + if (std::any_of(input_shape.begin(), input_shape.end(), [](size_t i) { return i == 0; })) { + return true; + } + } + return false; +} +#define CHECK_NULL_INPUT(input_shape) mindspore::device::gpu::CheckNullInput(input_shape) +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPU_COMMON_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_device_address.cc b/mindspore/ccsrc/device/gpu/gpu_device_address.cc new file mode 100644 index 00000000000..36391d27db6 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_device_address.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/gpu_device_address.h" + +#include + +#include "device/gpu/gpu_device_manager.h" +#include "utils/log_adapter.h" +#include "utils/context/ms_context.h" +#include "device/gpu/gpu_memory_allocator.h" + +namespace mindspore { +namespace device { +namespace gpu { +bool GPUDeviceAddress::SyncDeviceToHost(const std::vector &, size_t size, TypeId, void *host_ptr) const { + MS_EXCEPTION_IF_NULL(host_ptr); + if (size != size_) { + MS_LOG(WARNING) << "SyncDeviceToHost ignored, host size: " << size << ", device size " << size_; + return true; + } + return GPUDeviceManager::GetInstance().CopyDeviceMemToHost(host_ptr, ptr_, size_); +} + +bool GPUDeviceAddress::SyncHostToDevice(const std::vector &, size_t, TypeId, const void *host_ptr) const { + MS_EXCEPTION_IF_NULL(host_ptr); + return GPUDeviceManager::GetInstance().CopyHostMemToDevice(ptr_, host_ptr, size_); +} + +GPUDeviceAddress::~GPUDeviceAddress() { + if (ptr_ == nullptr) { + return; + } + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (mem_dynamic_alloc_) { + GPUMemoryAllocator::GetInstance().FreeTensorMem(ptr_); + ptr_ = nullptr; + } +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/gpu_device_address.h b/mindspore/ccsrc/device/gpu/gpu_device_address.h new file mode 100644 index 00000000000..ac54102a0a8 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_device_address.h @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPU_DEVICE_ADDRESS_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPU_DEVICE_ADDRESS_H_ + +#include +#include +#include "device/device_address.h" + +namespace mindspore { +namespace device { +namespace gpu { +class GPUDeviceAddress : public DeviceAddress { + public: + GPUDeviceAddress(void *ptr, size_t size) : DeviceAddress(ptr, size) {} + GPUDeviceAddress(void *ptr, size_t size, const string &format, TypeId type_id) + : DeviceAddress(ptr, size, format, type_id) {} + ~GPUDeviceAddress() override; + + bool SyncDeviceToHost(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const override; + bool SyncHostToDevice(const std::vector &shape, size_t size, TypeId type, const void *host_ptr) const override; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPU_DEVICE_ADDRESS_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_device_manager.cc b/mindspore/ccsrc/device/gpu/gpu_device_manager.cc new file mode 100644 index 00000000000..59c8fde5a2c --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_device_manager.cc @@ -0,0 +1,86 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/gpu_device_manager.h" +#include "device/gpu/gpu_common.h" +#include "utils/log_adapter.h" +#include "utils/convert_utils.h" +#include "device/gpu/gpu_buffer_mgr.h" + +namespace mindspore { +namespace device { +namespace gpu { +void GPUDeviceManager::InitDevice() { + CHECK_OP_RET_WITH_EXCEPT(CudaDriver::set_current_device(SizeToInt(cur_dev_id_)), "Failed to set current device id"); + CHECK_OP_RET_WITH_EXCEPT(CudaDriver::CreateStream(&stream_), "Failed to create CUDA stream."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreate(&cudnn_handle_), "Failed to create cuDNN handle"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetStream(cudnn_handle_, reinterpret_cast(default_stream())), + "Failed to set stream for cuDNN handle."); + CHECK_CUBLAS_RET_WITH_EXCEPT(cublasCreate(&cublas_handle_), "Failed to create cuBLAS handle."); + CHECK_CUBLAS_RET_WITH_EXCEPT(cublasSetStream(cublas_handle_, reinterpret_cast(default_stream())), + "Failed to set stream for cuBLAS handle."); + CHECK_OP_RET_WITH_EXCEPT(GPUMemoryAllocator::GetInstance().Init(), "Failed to Init gpu memory allocator") +} + +void GPUDeviceManager::ReleaseDevice() { + if (stream_ != nullptr) { + CHECK_OP_RET_WITH_ERROR(CudaDriver::DestroyStream(stream_), "Failed to destroy cuda stream."); + } + if (cudnn_handle_ != nullptr) { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroy(cudnn_handle_), "Failed to destroy cudnn handle"); + } + if (cublas_handle_ != nullptr) { + CHECK_CUBLAS_RET_WITH_ERROR(cublasDestroy(cublas_handle_), "Failed to destroy cublas handle."); + } + CHECK_OP_RET_WITH_ERROR(GPUMemoryAllocator::GetInstance().Finalize(), "Failed to destroy gpu memory allocator"); +} + +const DeviceStream& GPUDeviceManager::default_stream() const { return stream_; } + +int GPUDeviceManager::device_count() const { return CudaDriver::device_count(); } + +bool GPUDeviceManager::set_cur_device_id(uint32_t device_id) { + if (!dev_id_init_) { + dev_id_init_ = true; + cur_dev_id_ = device_id; + mindspore::device::GpuBufferMgr::GetInstance().set_device_id(UintToInt(device_id)); + return true; + } else { + MS_LOG(ERROR) << "Device already been set."; + return false; + } +} + +uint32_t GPUDeviceManager::cur_device_id() const { return cur_dev_id_; } + +bool GPUDeviceManager::is_device_id_init() const { return dev_id_init_; } + +const cudnnHandle_t& GPUDeviceManager::GetCudnnHandle() const { return cudnn_handle_; } + +const cublasHandle_t& GPUDeviceManager::GetCublasHandle() const { return cublas_handle_; } + +bool GPUDeviceManager::SyncStream(const DeviceStream& stream) const { return CudaDriver::SyncStream(stream); } + +bool GPUDeviceManager::CopyDeviceMemToHost(const HostMemPtr& dst, const DeviceMemPtr& src, size_t size) const { + return CudaDriver::CopyDeviceMemToHost(dst, src, size); +} + +bool GPUDeviceManager::CopyHostMemToDevice(const DeviceMemPtr& dst, const void* src, size_t size) const { + return CudaDriver::CopyHostMemToDevice(dst, src, size); +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/gpu_device_manager.h b/mindspore/ccsrc/device/gpu/gpu_device_manager.h new file mode 100644 index 00000000000..6bfaf856736 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_device_manager.h @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPU_DEVICE_MANAGER_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPU_DEVICE_MANAGER_H_ + +#include +#include +#include +#include "device/gpu/cuda_driver.h" +#include "device/gpu/gpu_memory_allocator.h" + +namespace mindspore { +namespace device { +namespace gpu { +class GPUDeviceManager { + public: + void InitDevice(); + void ReleaseDevice(); + + int device_count() const; + bool set_cur_device_id(uint32_t device_id); + uint32_t cur_device_id() const; + bool is_device_id_init() const; + + const DeviceStream& default_stream() const; + const cudnnHandle_t& GetCudnnHandle() const; + const cublasHandle_t& GetCublasHandle() const; + + bool CopyDeviceMemToHost(const HostMemPtr& dst, const DeviceMemPtr& src, size_t size) const; + bool CopyHostMemToDevice(const DeviceMemPtr& dst, const void* src, size_t size) const; + bool SyncStream(const DeviceStream& stream) const; + + static GPUDeviceManager& GetInstance() { + static GPUDeviceManager instance; + return instance; + } + + private: + GPUDeviceManager() : dev_id_init_(false), cur_dev_id_(0) {} + ~GPUDeviceManager() = default; + GPUDeviceManager(const GPUDeviceManager&) = delete; + GPUDeviceManager& operator=(const GPUDeviceManager&) = delete; + + // default cuda stream used for all the kernels. + DeviceStream stream_{nullptr}; + + // handle used for cudnn kernels. + cudnnHandle_t cudnn_handle_{nullptr}; + + // handle used for cublas kernels. + cublasHandle_t cublas_handle_{nullptr}; + + bool dev_id_init_; + uint32_t cur_dev_id_; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPU_DEVICE_MANAGER_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_kernel_build.cc b/mindspore/ccsrc/device/gpu/gpu_kernel_build.cc new file mode 100644 index 00000000000..0467b59e06c --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_kernel_build.cc @@ -0,0 +1,66 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/gpu/gpu_kernel_build.h" +#include +#include "kernel/kernel.h" +#include "kernel/akg/akgkernelbuild.h" +#include "kernel/akg/gpu/akg_gpu_kernel_build.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "operator/ops.h" +#include "pybind11/stl.h" +#include "transform/convert.h" +#include "session/anf_runtime_algorithm.h" +namespace mindspore { +namespace device { +namespace gpu { +namespace py = pybind11; +void GpuBuild(const KernelGraphPtr &kernel_graph) { + kernel::KernelMeta *bin_map = kernel::KernelMeta::GetInstance(); + if (!bin_map->ReadIndex(kernel::kGpuKernelMeta)) { + MS_LOG(INFO) << "kernel cache miss, cache directory will be created later."; + } else { + MS_LOG(INFO) << "cache initialize to[" << kernel::kGpuKernelMeta << "]."; + } + MS_EXCEPTION_IF_NULL(kernel_graph); + auto kernels = kernel_graph->execution_order(); + for (const auto &kernel : kernels) { + std::string kernel_name = session::AnfRuntimeAlgorithm::GetCNodeName(kernel); + if (kernel_name == prim::kPrimTupleGetItem->name() || kernel_name == prim::kPrimMakeTuple->name() || + kernel_name == prim::kPrimDepend->name() || kernel_name == prim::kPrimStateSetItem->name()) { + continue; + } + + if (session::AnfRuntimeAlgorithm::GetKernelType(kernel) == KernelType::AUTO_DIFF_KERNEL) { + auto gpu_kernel_ptr = kernel::AkgGpuKernelBuild(kernel); + if (!gpu_kernel_ptr) { + MS_LOG(EXCEPTION) << "Build akg kernel op[" << kernel_name << "] failed"; + } + session::AnfRuntimeAlgorithm::SetKernelMod(gpu_kernel_ptr, kernel.get()); + } else { + auto gpu_kernel_ptr = kernel::GpuKernelFactory::GetInstance().Create(kernel_name, kernel); + if (!gpu_kernel_ptr) { + MS_LOG(EXCEPTION) << "Build gpu kernel op[" << kernel_name << "] failed"; + } + if (!gpu_kernel_ptr->Init(kernel)) { + MS_LOG(EXCEPTION) << "Initialize gpu kernel op[" << kernel_name << "] failed."; + } + session::AnfRuntimeAlgorithm::SetKernelMod((kernel::KernelModPtr)gpu_kernel_ptr, kernel.get()); + } + } +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/gpu_kernel_build.h b/mindspore/ccsrc/device/gpu/gpu_kernel_build.h new file mode 100644 index 00000000000..5770e4d3b18 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_kernel_build.h @@ -0,0 +1,28 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPUKERNELBUILD_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPUKERNELBUILD_H_ + +#include +#include "session/kernel_graph.h" +namespace mindspore { +namespace device { +namespace gpu { +void GpuBuild(const std::shared_ptr &kernel_graph); +} // namespace gpu +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPUKERNELBUILD_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc new file mode 100644 index 00000000000..9eeb1062f74 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc @@ -0,0 +1,488 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/gpu_kernel_runtime.h" +#include "device/gpu/gpu_device_address.h" +#include "device/gpu/cuda_driver.h" +#include "device/gpu/gpu_buffer_mgr.h" +#include "device/gpu/gpu_device_manager.h" +#include "device/gpu/gpu_memory_allocator.h" +#include "device/gpu/distribution/collective_init.h" +#include "utils/convert_utils.h" +#include "utils/context/ms_context.h" +#include "device/kernel_runtime_manager.h" +#include "device/gpu/gpu_common.h" +#include "common/utils.h" + +namespace mindspore { +namespace device { +namespace gpu { +bool GPUKernelRuntime::SyncStream() { return GPUDeviceManager::GetInstance().SyncStream(stream_); } + +bool GPUKernelRuntime::Init() { + if (device_init_ == true) { + return true; + } + + auto ret = InitDevice(); + if (!ret) { + MS_LOG(ERROR) << "InitDevice error."; + return ret; + } + + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + // If use the dynamic memory pool, then alloc the first memory block to init. + if (context_ptr->enable_dynamic_mem_pool()) { + auto device_addr = AllocTensorMemDynamic(1); + if (!device_addr) { + MS_LOG(ERROR) << "Dynamic memory pool init error."; + return false; + } + } else { + MallocDeviceMemory(); + } + + const void *collective_handle_ = CollectiveInitializer::instance().collective_handle(); + bool collective_inited = CollectiveInitializer::instance().collective_inited(); + if (collective_inited && collective_handle_ != nullptr) { + auto init_nccl_comm_funcptr = + reinterpret_cast(dlsym(const_cast(collective_handle_), "InitNCCLComm")); + MS_EXCEPTION_IF_NULL(init_nccl_comm_funcptr); + (*init_nccl_comm_funcptr)(); + } + device_init_ = true; + return ret; +} + +DeviceAddressPtr GPUKernelRuntime::CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) { + return std::make_shared(device_ptr, device_size, format, type_id); +} + +bool GPUKernelRuntime::InitDevice() { + if (GPUDeviceManager::GetInstance().device_count() <= 0) { + MS_LOG(ERROR) << "No GPU device found."; + return false; + } + const void *collective_handle_ = CollectiveInitializer::instance().collective_handle(); + bool collective_inited = CollectiveInitializer::instance().collective_inited(); + if (collective_inited && collective_handle_ != nullptr) { + auto get_local_rank_funcptr = + reinterpret_cast(dlsym(const_cast(collective_handle_), "local_rank_id")); + MS_EXCEPTION_IF_NULL(get_local_rank_funcptr); + device_id_ = IntToUint((*get_local_rank_funcptr)()); + } + if (!GPUDeviceManager::GetInstance().is_device_id_init()) { + if (!GPUDeviceManager::GetInstance().set_cur_device_id(device_id_)) { + MS_LOG(ERROR) << "Failed to set current device to " << SizeToInt(device_id_); + return false; + } + } + GPUDeviceManager::GetInstance().InitDevice(); + stream_ = GPUDeviceManager::GetInstance().default_stream(); + if (stream_ == nullptr) { + MS_LOG(ERROR) << "No default CUDA stream found."; + return false; + } + return true; +} + +void GPUKernelRuntime::MallocDeviceMemory() { + // Need to reserve 20% space for dynamic memory + const float init_gpu_mem_ratio = 0.8; + size_t mem_size = FloatToSize(GPUMemoryAllocator::GetInstance().free_mem_size() * init_gpu_mem_ratio); + auto alloc_size = + GPUMemoryAllocator::GetInstance().AllocDeviceMem(mem_size, reinterpret_cast(&device_mem_base_)); + device_mem_size_ = alloc_size; + static_mem_offset_ = device_mem_size_; +} + +void GPUKernelRuntime::ReleaseDeviceRes() { + // For dataset mode. + if (GpuBufferMgr::GetInstance().IsInit()) { + if (!GpuBufferMgr::GetInstance().IsClosed()) { + if (!GpuBufferMgr::GetInstance().CloseNotify()) { + MS_LOG(EXCEPTION) << "Could not close gpu data queue."; + } + } + CHECK_OP_RET_WITH_EXCEPT(GpuBufferMgr::GetInstance().Destroy(), "Could not destroy gpu data queue."); + } + GPUDeviceManager::GetInstance().ReleaseDevice(); + if (device_mem_base_ != nullptr) { + if (!GPUMemoryAllocator::GetInstance().FreeDeviceMem(device_mem_base_)) { + MS_LOG(EXCEPTION) << "Could not free gpu device memory."; + } + } + GPUMemoryAllocator::GetInstance().ReleaseDeviceRes(); +} + +void GPUKernelRuntime::FreeHostMemory() { dynamic_mem_offset_ = 0; } + +void *GPUKernelRuntime::AllocTensorMemDynamic(size_t size) { + return GPUMemoryAllocator::GetInstance().AllocTensorMem(size); +} + +void GPUKernelRuntime::FreeTensorMemDynamic(void *device_ptr) { + GPUMemoryAllocator::GetInstance().FreeTensorMem(device_ptr); +} + +void GPUKernelRuntime::AssignMemory(session::KernelGraph *graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + AssignStaticMemory(graph); + bool is_enable_mem_reuse = context_ptr->enable_mem_reuse(); + bool is_enable_dynamic_mem = context_ptr->enable_dynamic_mem_pool(); + if (is_enable_dynamic_mem) { + // Use the dynamic memory pool. + InitKernelRefCount(graph); + InitKernelOutputAddress(graph); + } else if (is_enable_mem_reuse) { + // Use the memory reuse. + ReuseAssignDynamicMemory(graph); + } else { + // Normal way. + AssignDynamicMemory(graph); + } +} + +bool GPUKernelRuntime::Run(session::KernelGraph *graph) { + bool ret; + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool is_enable_dynamic_mem = context_ptr->enable_dynamic_mem_pool(); + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + if (is_enable_dynamic_mem) { + ret = LaunchKernelDynamic(graph); + } else { + ret = LaunchKernel(graph); + } + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(DEBUG) << "kernel runtime run graph in " << cost << " us"; + return ret; +} + +uint8_t *GPUKernelRuntime::MallocStaticMem(size_t size, bool) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_dynamic_mem_pool()) { + auto device_ptr = AllocTensorMemDynamic(size); + MS_EXCEPTION_IF_NULL(device_ptr); + return AddressOffset(device_ptr, 0); + } + + auto align_size = GetCommonAlignSize(size); + if (static_mem_offset_ < align_size) { + MS_LOG(EXCEPTION) << "Out of memory!!! total[" << device_mem_size_ << "](dynamic[" << total_dynamic_size_ + << "] static[" << total_static_size_ << "])" + << " malloc [" << align_size << "] failed!"; + } + auto offset = static_mem_offset_ - align_size; + if (dynamic_mem_offset_ > offset) { + MS_LOG(EXCEPTION) << "Out of memory!!! total[" << device_mem_size_ << "](dynamic[" << total_dynamic_size_ + << "] static[" << total_static_size_ << "])" + << " malloc [" << align_size << "] failed!"; + } + total_static_size_ += align_size; + static_mem_offset_ = offset; + return device_mem_base_ + offset; +} + +void GPUKernelRuntime::InitKernelRefCount(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + MemReuseUtilPtr mem_reuse_util_ptr = std::make_shared(); + MS_EXCEPTION_IF_NULL(mem_reuse_util_ptr); + // Init the kernel reference count. + if (!mem_reuse_util_ptr->InitDynamicKernelRef(graph)) { + MS_LOG(EXCEPTION) << "Init kernel reference count failed"; + } + mem_reuse_util_ptr->SetKernelDefMap(); + mem_reuse_util_ptr->SetReuseRefCount(); + // Can't free the device address of graph output, so set the reference count of graph output specially, + mem_reuse_util_ptr->SetGraphOutputRefCount(); + mem_reuse_util_ptr_ = mem_reuse_util_ptr; +} + +void GPUKernelRuntime::InitKernelOutputAddress(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + auto &kernels = graph->execution_order(); + for (const auto &kernel : kernels) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto output_sizes = kernel_mod->GetOutputSizeList(); + for (size_t i = 0; i < output_sizes.size(); ++i) { + if (AnfAlgo::OutputAddrExist(kernel, i)) { + continue; + } + std::string output_format = AnfAlgo::GetOutputFormat(kernel, i); + auto output_type = AnfAlgo::GetOutputDeviceDataType(kernel, i); + auto device_address = CreateDeviceAddress(nullptr, output_sizes[i], output_format, output_type); + AnfAlgo::SetOutputAddr(device_address, i, kernel.get()); + } + } +} + +bool GPUKernelRuntime::LaunchKernelDynamic(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + // The inputs and outputs memory of communication kernel are special, so separate processing. + AllocCommunicationOpDynamicRes(graph); + + auto &kernels = graph->execution_order(); + for (const auto &kernel : kernels) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + AddressPtrList kernel_inputs; + AddressPtrList kernel_workspaces; + AddressPtrList kernel_outputs; + AllocKernelDynamicRes(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); + if (!kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast(stream_))) { + MS_LOG(ERROR) << "Launch kernel failed."; + return false; + } + FreeKernelDynamicRes(kernel, kernel_workspaces); + } + + if (!SyncStream()) { + MS_LOG(ERROR) << "SyncStream failed."; + return false; + } + return true; +} + +void GPUKernelRuntime::AllocKernelDynamicRes(const mindspore::kernel::KernelMod &kernel_mod, + const mindspore::AnfNodePtr &kernel, AddressPtrList *kernel_inputs, + AddressPtrList *kernel_workspaces, AddressPtrList *kernel_outputs) { + MS_EXCEPTION_IF_NULL(kernel); + MS_EXCEPTION_IF_NULL(kernel_inputs); + MS_EXCEPTION_IF_NULL(kernel_workspaces); + MS_EXCEPTION_IF_NULL(kernel_outputs); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(kernel); ++i) { + auto device_address = AnfAlgo::GetPrevNodeOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(device_address); + MS_EXCEPTION_IF_NULL(device_address->ptr_); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->addr = device_address->ptr_; + input->size = device_address->size_; + kernel_inputs->push_back(input); + } + + auto output_sizes = kernel_mod.GetOutputSizeList(); + for (size_t i = 0; i < output_sizes.size(); ++i) { + auto device_address = AnfAlgo::GetMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(device_address); + auto device_ptr = device_address->ptr_; + if (device_ptr == nullptr) { + device_ptr = AllocTensorMemDynamic(output_sizes[i]); + MS_EXCEPTION_IF_NULL(device_ptr); + device_address->ptr_ = device_ptr; + } + kernel::AddressPtr output = std::make_shared(); + MS_EXCEPTION_IF_NULL(output); + output->addr = device_ptr; + output->size = output_sizes[i]; + kernel_outputs->push_back(output); + } + + auto workspace_sizes = kernel_mod.GetWorkspaceSizeList(); + for (size_t i = 0; i < workspace_sizes.size(); ++i) { + if (workspace_sizes[i] == 0) { + kernel_workspaces->emplace_back(nullptr); + continue; + } + auto device_ptr = AllocTensorMemDynamic(workspace_sizes[i]); + MS_EXCEPTION_IF_NULL(device_ptr); + kernel::AddressPtr workspace = std::make_shared(); + MS_EXCEPTION_IF_NULL(workspace); + workspace->addr = device_ptr; + workspace->size = workspace_sizes[i]; + kernel_workspaces->push_back(workspace); + } +} + +void GPUKernelRuntime::AllocCommunicationOpDynamicRes(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + auto &kernels = graph->execution_order(); + for (auto &kernel : kernels) { + MS_EXCEPTION_IF_NULL(kernel); + auto kernel_name = AnfAlgo::GetCNodeName(kernel); + if (kernel_name == kAllReduceOpName) { + AllocCommunicationOpInputDynamicRes(kernel); + AllocCommunicationOpOutputDynamicRes(kernel); + return; + } + } +} + +void GPUKernelRuntime::AllocCommunicationOpInputDynamicRes(const mindspore::AnfNodePtr &kernel) { + MS_EXCEPTION_IF_NULL(kernel); + // The reference count of communication kernel input is not 0. + if (communication_op_input_ref_count_ != 0) { + MS_LOG(ERROR) << "The reference count of communication kernel input is not 0."; + return; + } + + size_t total = 0; + std::vector> addr_size; + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(kernel); ++i) { + auto device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(device_address); + // The inputs of communication kernel are not released. + if ((i == 0) && (device_address->ptr_ != nullptr)) { + MS_LOG(ERROR) << "The inputs of communication kernel are not released."; + return; + } + auto output_size = device_address->size_; + total += output_size; + addr_size.emplace_back(device_address.get(), output_size); + } + + auto device_mem_ptr = AllocTensorMemDynamic(total); + MS_EXCEPTION_IF_NULL(device_mem_ptr); + for (const auto &iter : addr_size) { + MS_EXCEPTION_IF_NULL(iter.first); + iter.first->set_ptr(device_mem_ptr); + communication_op_input_ref_count_++; + device_mem_ptr = AddressOffset(device_mem_ptr, iter.second); + } +} + +void GPUKernelRuntime::AllocCommunicationOpOutputDynamicRes(const mindspore::AnfNodePtr &kernel) { + MS_EXCEPTION_IF_NULL(kernel); + // The reference count of communication kernel output is not 0. + if (communication_op_output_ref_count_ != 0) { + MS_LOG(ERROR) << "The reference count of communication kernel output is not 0."; + return; + } + + size_t total = 0; + std::vector> addr_size; + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto output_sizes = kernel_mod->GetOutputSizeList(); + for (size_t i = 0; i < output_sizes.size(); ++i) { + auto device_address = AnfAlgo::GetMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(device_address); + // The outputs of communication kernel are not released. + if ((i == 0) && (device_address->ptr_ != nullptr)) { + MS_LOG(ERROR) << "The outputs of communication kernel are not released."; + return; + } + total += output_sizes[i]; + addr_size.emplace_back(device_address.get(), output_sizes[i]); + } + + auto device_mem_ptr = AllocTensorMemDynamic(total); + MS_EXCEPTION_IF_NULL(device_mem_ptr); + for (const auto &iter : addr_size) { + MS_EXCEPTION_IF_NULL(iter.first); + iter.first->set_ptr(device_mem_ptr); + communication_op_output_ref_count_++; + device_mem_ptr = AddressOffset(device_mem_ptr, iter.second); + } +} + +void GPUKernelRuntime::FreeKernelDynamicRes(const mindspore::AnfNodePtr &kernel, + const AddressPtrList &kernel_workspaces) { + MS_EXCEPTION_IF_NULL(kernel); + auto cnode = kernel->cast(); + MS_EXCEPTION_IF_NULL(cnode); + // Free the input of kernel by reference count. + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(kernel); ++i) { + auto kernel_ref_count_ptr = mem_reuse_util_ptr_->GetKernelInputRef(cnode, i); + if (kernel_ref_count_ptr == nullptr) { + continue; + } + kernel_ref_count_ptr->ref_count_dynamic_use_--; + if (kernel_ref_count_ptr->ref_count_dynamic_use_ == 0) { + // Reset the reference count. + kernel_ref_count_ptr->ref_count_dynamic_use_ = kernel_ref_count_ptr->ref_count_; + bool is_communication_op = false; + // The inputs and outputs memory of communication kernel are special, so separate processing. + FreeCommunicationOpDynamicRes(kernel, i, &is_communication_op); + if (!is_communication_op) { + auto device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + MS_EXCEPTION_IF_NULL(device_address); + MS_EXCEPTION_IF_NULL(device_address->ptr_); + FreeTensorMemDynamic(device_address->ptr_); + device_address->ptr_ = nullptr; + } + } + } + + // Free the workspace of kernel. + for (size_t i = 0; i < kernel_workspaces.size(); ++i) { + auto workspace = kernel_workspaces[i]; + if (workspace != nullptr) { + MS_EXCEPTION_IF_NULL(workspace->addr); + FreeTensorMemDynamic(workspace->addr); + workspace->addr = nullptr; + } + } +} + +void GPUKernelRuntime::FreeCommunicationOpDynamicRes(const mindspore::AnfNodePtr &kernel, size_t input_idx, + bool *is_communication_op) { + MS_EXCEPTION_IF_NULL(kernel); + // The inputs memory of communication kernel is one piece memory, need release together. + if (AnfAlgo::GetCNodeName(kernel) == kAllReduceOpName) { + communication_op_input_ref_count_--; + if (communication_op_input_ref_count_ == 0) { + auto device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, 0); + MS_EXCEPTION_IF_NULL(device_address); + MS_EXCEPTION_IF_NULL(device_address->ptr_); + FreeTensorMemDynamic(device_address->ptr_); + device_address->ptr_ = nullptr; + } + *is_communication_op = true; + return; + } + + auto cnode = kernel->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << cnode->inputs().size() - 1 + << "."; + } + auto input_node = cnode->input(input_idx + 1); + auto kernel_input = AnfAlgo::VisitKernel(input_node, 0); + // The outputs memory of communication kernel is one piece memory, need release together. + if (AnfAlgo::GetCNodeName(kernel_input.first) == kAllReduceOpName) { + communication_op_output_ref_count_--; + if (communication_op_output_ref_count_ == 0) { + auto device_address = AnfAlgo::GetMutableOutputAddr(kernel_input.first, 0); + MS_EXCEPTION_IF_NULL(device_address); + MS_EXCEPTION_IF_NULL(device_address->ptr_); + FreeTensorMemDynamic(device_address->ptr_); + device_address->ptr_ = nullptr; + } + *is_communication_op = true; + } +} + +void GPUKernelRuntime::MallocOpMemory(const DeviceAddressPtr address, size_t size, int) { + auto device_ptr = AllocTensorMemDynamic(size); + MS_EXCEPTION_IF_NULL(device_ptr); + address->ptr_ = device_ptr; + address->mem_dynamic_alloc_ = true; +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.h b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.h new file mode 100644 index 00000000000..f3fdb5fa98f --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.h @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPU_KERNEL_RUNTIME_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPU_KERNEL_RUNTIME_H_ + +#include +#include +#include +#include +#include "device/kernel_runtime.h" +#include "device/kernel_runtime_manager.h" + +namespace mindspore { +namespace device { +namespace gpu { +class GPUKernelRuntime : public KernelRuntime { + public: + GPUKernelRuntime() = default; + ~GPUKernelRuntime() override = default; + bool Init() override; + void ReleaseDeviceRes() override; + void FreeHostMemory() override; + void AssignMemory(session::KernelGraph *graph) override; + bool Run(session::KernelGraph *graph) override; + + protected: + DeviceAddressPtr CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) override; + bool SyncStream() override; + // Alloc memory use the dynamic memory pool. + void *AllocTensorMemDynamic(size_t size) override; + // Free memory use the dynamic memory pool. + void FreeTensorMemDynamic(void *device_ptr) override; + void MallocOpMemory(const DeviceAddressPtr address, size_t size, int flag) override; + uint8_t *MallocStaticMem(size_t size, bool communication_mem) override; + + private: + GPUKernelRuntime(const GPUKernelRuntime &); + GPUKernelRuntime &operator=(const GPUKernelRuntime &); + bool InitDevice(); + void MallocDeviceMemory(); + bool device_init_{false}; + + // The related functions and members for using dynamic memory pool. + void InitKernelRefCount(const session::KernelGraph *graph); + void InitKernelOutputAddress(const session::KernelGraph *graph); + bool LaunchKernelDynamic(const session::KernelGraph *graph); + void AllocKernelDynamicRes(const mindspore::kernel::KernelMod &kernel_mod, const mindspore::AnfNodePtr &kernel, + AddressPtrList *kernel_inputs, AddressPtrList *kernel_workspaces, + AddressPtrList *kernel_outputs); + void AllocCommunicationOpDynamicRes(const session::KernelGraph *graph); + void AllocCommunicationOpInputDynamicRes(const mindspore::AnfNodePtr &kernel); + void AllocCommunicationOpOutputDynamicRes(const mindspore::AnfNodePtr &kernel); + void FreeKernelDynamicRes(const mindspore::AnfNodePtr &kernel, const AddressPtrList &kernel_workspaces); + void FreeCommunicationOpDynamicRes(const mindspore::AnfNodePtr &kernel, size_t input_idx, bool *is_communication_op); + size_t communication_op_input_ref_count_{0}; + size_t communication_op_output_ref_count_{0}; +}; +MS_REG_KERNEL_RUNTIME(kGPUDevice, GPUKernelRuntime); +} // namespace gpu +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPU_KERNEL_RUNTIME_H_ diff --git a/mindspore/ccsrc/device/gpu/gpu_memory_allocator.cc b/mindspore/ccsrc/device/gpu/gpu_memory_allocator.cc new file mode 100644 index 00000000000..cbd43645abf --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_memory_allocator.cc @@ -0,0 +1,78 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/gpu_memory_allocator.h" +#include "device/gpu/cuda_driver.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +namespace gpu { +bool GPUMemoryAllocator::Init() { + size_t total_size = total_mem_size(); + size_t free_size = free_mem_size(); + if (total_size > 0 && free_size > 0) { + MS_LOG(INFO) << "GPU device total memory size " << total_size << ", current free memory size " << free_size; + } else { + MS_LOG(EXCEPTION) << "GPU device memory error, total memory size " << total_size << ", current free memory size " + << free_size; + } + return true; +} + +bool GPUMemoryAllocator::Finalize() { + if (buffer_q_addr_ != nullptr) { + if (!CudaDriver::FreeDeviceMem(buffer_q_addr_)) { + MS_LOG(ERROR) << "Could not free buffer queue memory."; + return false; + } + } + return true; +} + +bool GPUMemoryAllocator::AllocBufferQueueMem(size_t size, DeviceMemPtr* addr) { + auto alloc_size = AllocDeviceMem(size, addr); + buffer_q_addr_ = *addr; + // Buffer queue needs to ensure that the alloc_size and size is equal. + return (alloc_size == size) ? true : false; +} + +size_t GPUMemoryAllocator::AllocDeviceMem(size_t size, DeviceMemPtr* addr) { + if (size == 0) { + MS_LOG(EXCEPTION) << "The memory alloc size is 0."; + } + auto free_size = free_mem_size(); + if (size > free_size) { + MS_LOG(EXCEPTION) << "Memory not enough: current free memory size[" << free_size + << "] is smaller than required size[" << size << "]."; + } + + auto alloc_size = CudaDriver::AllocDeviceMem(size, addr); + if (alloc_size == 0) { + MS_LOG(EXCEPTION) << "Alloc device memory[" << size << "] failed."; + } + MS_LOG(INFO) << "Current free memory size[" << free_size << "], current alloc size[" << alloc_size << "]."; + return alloc_size; +} + +bool GPUMemoryAllocator::FreeDeviceMem(const DeviceMemPtr& addr) { return CudaDriver::FreeDeviceMem(addr); } + +size_t GPUMemoryAllocator::free_mem_size() { return CudaDriver::free_mem_size(); } + +size_t GPUMemoryAllocator::total_mem_size() { return CudaDriver::total_mem_size(); } +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/gpu_memory_allocator.h b/mindspore/ccsrc/device/gpu/gpu_memory_allocator.h new file mode 100644 index 00000000000..0d2f0f8a39a --- /dev/null +++ b/mindspore/ccsrc/device/gpu/gpu_memory_allocator.h @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_GPU_MEMORY_ALLOCATOR_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_GPU_MEMORY_ALLOCATOR_H_ + +#include +#include "device/gpu/cuda_driver.h" +#include "pre_activate/mem_reuse/mem_dynamic_allocator.h" + +namespace mindspore { +namespace device { +namespace gpu { +class GPUMemoryAllocator : public DynamicMemPoolBestFit { + public: + ~GPUMemoryAllocator() override = default; + bool Init(); + bool Finalize(); + bool AllocBufferQueueMem(size_t size, DeviceMemPtr* addr); + + size_t AllocDeviceMem(size_t size, DeviceMemPtr* addr) override; + bool FreeDeviceMem(const DeviceMemPtr& addr) override; + size_t free_mem_size() override; + size_t total_mem_size() override; + + static GPUMemoryAllocator& GetInstance() { + static GPUMemoryAllocator instance; + return instance; + } + + private: + GPUMemoryAllocator() = default; + GPUMemoryAllocator(const GPUMemoryAllocator&) = delete; + GPUMemoryAllocator& operator=(const GPUMemoryAllocator&) = delete; + + // Used to track address of data buffer queue. + DeviceMemPtr buffer_q_addr_{nullptr}; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_GPU_MEMORY_ALLOCATOR_H_ diff --git a/mindspore/ccsrc/device/gpu/kernel_info_setter.cc b/mindspore/ccsrc/device/gpu/kernel_info_setter.cc new file mode 100644 index 00000000000..3faf7d01c80 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/kernel_info_setter.cc @@ -0,0 +1,199 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/kernel_info_setter.h" +#include +#include +#include "kernel/kernel.h" +#include "utils/utils.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/kernel_build_info.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/common_utils.h" +#include "common/utils.h" +#include "kernel/oplib/oplib.h" +#include "kernel/oplib/opinfo.h" + +namespace mindspore { +namespace device { +namespace gpu { +using AnfAlgo = mindspore::session::AnfRuntimeAlgorithm; +using mindspore::kernel::KernelBuildInfo; +namespace { +bool CheckKernelInfo(const std::shared_ptr& alternative_kernel_info, + const std::shared_ptr& selected_kernel_info) { + MS_EXCEPTION_IF_NULL(selected_kernel_info); + MS_EXCEPTION_IF_NULL(alternative_kernel_info); + size_t selected_input_num = selected_kernel_info->GetInputNum(); + size_t alternative_input_num = alternative_kernel_info->GetInputNum(); + if (selected_input_num != alternative_input_num) { + return false; + } + for (size_t i = 0; i < selected_input_num; i++) { + if (selected_kernel_info->GetInputFormat(i) != alternative_kernel_info->GetInputFormat(i)) { + return false; + } + if (selected_kernel_info->GetInputDeviceType(i) != alternative_kernel_info->GetInputDeviceType(i)) { + return false; + } + } + + size_t selected_output_num = selected_kernel_info->GetOutputNum(); + size_t alternative_output_num = alternative_kernel_info->GetOutputNum(); + if (selected_output_num != alternative_output_num) { + return false; + } + for (size_t i = 0; i < selected_output_num; i++) { + if (selected_kernel_info->GetOutputFormat(i) != alternative_kernel_info->GetOutputFormat(i)) { + return false; + } + if (selected_kernel_info->GetOutputDeviceType(i) != alternative_kernel_info->GetOutputDeviceType(i)) { + return false; + } + } + return true; +} + +std::string SupportedTypeList(const CNodePtr& kernel_node) { + std::string supported_type_lists = + kernel::GpuKernelFactory::GetInstance().SupportedTypeList(AnfAlgo::GetCNodeName(kernel_node)); + if (!supported_type_lists.empty()) { + return supported_type_lists; + } + std::vector> kernel_info_list; + std::string op_name = AnfAlgo::GetCNodeName(kernel_node); + auto op_info_ptr = mindspore::kernel::OpLib::FindOp(op_name, kernel::OpImplyType::kAKG); + if (op_info_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Unsupported op [" << op_name << "]"; + } + (void)ParseMetadata(kernel_node, op_info_ptr, kernel::Processor::CUDA, &kernel_info_list); + for (size_t i = 0; i < kernel_info_list.size(); i++) { + auto supported_akg_type = kernel_info_list[i]->GetAllInputDeviceTypes(); + std::string supported_akg_type_list = "["; + for (auto type : supported_akg_type) { + supported_akg_type_list = supported_akg_type_list + mindspore::kernel::TypeId2String(type); + } + supported_type_lists = supported_type_lists + supported_akg_type_list + "] "; + } + return supported_type_lists; +} + +bool SelectAkgKernel(const CNodePtr& kernel_node, const shared_ptr& selected_kernel_info) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(selected_kernel_info); + std::vector> kernel_info_list; + std::string op_name = AnfAlgo::GetCNodeName(kernel_node); + + auto op_info_ptr = mindspore::kernel::OpLib::FindOp(op_name, kernel::OpImplyType::kAKG); + if (op_info_ptr == nullptr) { + MS_LOG(ERROR) << "Not find op[" << op_name << "] in akg"; + return false; + } + if (!ParseMetadata(kernel_node, op_info_ptr, kernel::Processor::CUDA, &kernel_info_list)) { + MS_LOG(EXCEPTION) << "Parsed metadata of op[" << op_name << "] failed."; + } + if (kernel_info_list.empty()) { + MS_LOG(EXCEPTION) << "Akg dose not has metadata of op[" << op_name << "]."; + } + + bool match = std::any_of(kernel_info_list.begin(), kernel_info_list.end(), + [&](const std::shared_ptr& alternative_kernel_info) { + return CheckKernelInfo(alternative_kernel_info, selected_kernel_info); + }); + if (!match) { + MS_LOG(ERROR) << "Not find op[" << op_name << "] in akg"; + return false; + } + return true; +} + +void SetTensorDeviceInfo(const kernel::KernelBuildInfo& selected_kernel_info, const CNodePtr& kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(kernel_node); ++input_index) { + auto input_kernel_node = kernel_node->input(input_index + 1); + MS_EXCEPTION_IF_NULL(input_kernel_node); + if (!input_kernel_node->isa()) { + continue; + } + std::shared_ptr builder = + std::make_shared(); + + auto param = input_kernel_node->cast(); + MS_EXCEPTION_IF_NULL(param); + if (!AnfAlgo::IsParameterWeight(param)) { + std::vector output_format = {kOpFormat_DEFAULT}; + builder->SetOutputsFormat(output_format); + std::vector output_type = {AnfAlgo::GetOutputInferDataType(input_kernel_node, 0)}; + builder->SetOutputsDeviceType(output_type); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), input_kernel_node.get()); + continue; + } + if ((AnfAlgo::GetOutputDeviceDataType(input_kernel_node, 0) == kTypeUnknown) || + (AnfAlgo::GetCNodeName(kernel_node) == "ApplyMomentum")) { + std::vector output_format = {selected_kernel_info.GetInputFormat(input_index)}; + builder->SetOutputsFormat(output_format); + std::vector output_type = {selected_kernel_info.GetInputDeviceType(input_index)}; + builder->SetOutputsDeviceType(output_type); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), input_kernel_node.get()); + } + } +} +} // namespace + +void SetKernelInfo(const CNodePtr& kernel_node) { + std::vector inputs_format; + std::vector inputs_type; + std::shared_ptr builder = + std::make_shared(); + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(kernel_node); ++input_index) { + inputs_format.emplace_back(kOpFormat_DEFAULT); + inputs_type.push_back(AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, input_index)); + } + builder->SetInputsFormat(inputs_format); + builder->SetInputsDeviceType(inputs_type); + std::vector outputs_format; + std::vector outputs_type; + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(kernel_node); ++output_index) { + outputs_format.emplace_back(kOpFormat_DEFAULT); + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(kernel_node, output_index)); + } + builder->SetOutputsFormat(outputs_format); + builder->SetOutputsDeviceType(outputs_type); + + bool result = + kernel::GpuKernelFactory::GetInstance().SearchRegistered(AnfAlgo::GetCNodeName(kernel_node), builder->Build()); + KernelType kernel_type = UNKNOWN_KERNEL_TYPE; + + if (!result) { + result = SelectAkgKernel(kernel_node, builder->Build()); + kernel_type = AUTO_DIFF_KERNEL; + } + + if (!result) { + auto kernel_name = AnfAlgo::GetCNodeName(kernel_node); + + auto supported_type_lists = SupportedTypeList(kernel_node); + MS_LOG(EXCEPTION) << "Select GPU kernel op[" << kernel_name + << "] fail! Incompatible data type!\nThe supported data types are " << supported_type_lists; + } + builder->SetKernelType(kernel_type); + builder->SetProcessor(kernel::Processor::CUDA); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), kernel_node.get()); + SetTensorDeviceInfo(*(builder->Build()), kernel_node); +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/kernel_info_setter.h b/mindspore/ccsrc/device/gpu/kernel_info_setter.h new file mode 100644 index 00000000000..e3dc2241a9a --- /dev/null +++ b/mindspore/ccsrc/device/gpu/kernel_info_setter.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_KERNEL_INFO_SETTER_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_KERNEL_INFO_SETTER_H_ + +#include +#include +#include +#include "ir/anf.h" +#include "ir/dtype.h" +#include "utils/utils.h" + +namespace mindspore { +namespace device { +namespace gpu { +void SetKernelInfo(const CNodePtr& apply_kernel_ptr); + +class KernelAttr { + public: + using DataType = std::pair; + KernelAttr() : all_same_(false) {} + ~KernelAttr() = default; + + KernelAttr& AddInputAttr(const TypeId& ms_type, const std::string& format = kOpFormat_DEFAULT) { + input_type_.emplace_back(ms_type, format); + return *this; + } + + KernelAttr& AddOutputAttr(const TypeId& ms_type, const std::string& format = kOpFormat_DEFAULT) { + output_type_.emplace_back(ms_type, format); + return *this; + } + + KernelAttr& AddAllSameAttr(const bool& all_same) { + all_same_ = all_same; + return *this; + } + + const DataType& GetInputAttr(const size_t index) const { return input_type_[index]; } + const DataType& GetOutputAttr(const size_t index) const { return output_type_[index]; } + const bool& GetAllSame() const { return all_same_; } + + size_t GetInputSize() const { return input_type_.size(); } + size_t GetOutputSize() const { return output_type_.size(); } + + private: + std::vector input_type_; + std::vector output_type_; + bool all_same_; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_KERNEL_INFO_SETTER_H_ diff --git a/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc b/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc new file mode 100644 index 00000000000..f2dbd4491b1 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/gpu/mpi/mpi_initializer.h" + +#include +#include +#include + +namespace mindspore { +namespace device { +namespace gpu { +MPIInitializer::MPIInitializer() { + MPI_Comm_rank(MPI_COMM_WORLD, &rank_id_); + MPI_Comm_size(MPI_COMM_WORLD, &rank_size_); +} + +MPIInitializer &MPIInitializer::GetInstance() { + static MPIInitializer instance; + return instance; +} + +int MPIInitializer::get_rank_id() { return MPIInitializer::GetInstance().rank_id_; } + +int MPIInitializer::get_rank_size() { return MPIInitializer::GetInstance().rank_size_; } + +PYBIND11_MODULE(_ms_mpi, mpi_initializer) { + mpi_initializer.doc() = "mindspore mpi python wrapper"; + mpi_initializer.def("get_rank_id", &MPIInitializer::get_rank_id, "get rank id"); + mpi_initializer.def("get_rank_size", &MPIInitializer::get_rank_size, "get rank size"); +} +} // namespace gpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.h b/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.h new file mode 100644 index 00000000000..00f3b9d7136 --- /dev/null +++ b/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_GPU_MPI_MPI_INITIALIZER_H_ +#define MINDSPORE_CCSRC_DEVICE_GPU_MPI_MPI_INITIALIZER_H_ + +namespace mindspore { +namespace device { +namespace gpu { +class MPIInitializer { + public: + MPIInitializer(MPIInitializer const &) = delete; + MPIInitializer &operator=(const MPIInitializer &) = delete; + static MPIInitializer &GetInstance(); + static int get_rank_id(); + static int get_rank_size(); + + private: + MPIInitializer(); + ~MPIInitializer() = default; + + int rank_id_; + int rank_size_; +}; +} // namespace gpu +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_GPU_MPI_MPI_INITIALIZER_H_ diff --git a/mindspore/ccsrc/device/gpu/readme.md b/mindspore/ccsrc/device/gpu/readme.md new file mode 100644 index 00000000000..927fa94a8de --- /dev/null +++ b/mindspore/ccsrc/device/gpu/readme.md @@ -0,0 +1 @@ +gpu diff --git a/mindspore/ccsrc/device/kernel_adjust.cc b/mindspore/ccsrc/device/kernel_adjust.cc new file mode 100644 index 00000000000..a4d316d601c --- /dev/null +++ b/mindspore/ccsrc/device/kernel_adjust.cc @@ -0,0 +1,480 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/kernel_adjust.h" + +#include +#include +#include +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "utils/context/ms_context.h" +#include "utils/config_manager.h" +#include "common/utils.h" +#include "kernel/kernel_build_info.h" +#include "utils/utils.h" +#include "device/ascend/profiling/profiling_manager.h" +#include "device/ascend/kernel_select_ascend.h" +#include "device/kernel_info.h" + +constexpr auto kLoopCountParamName = "loop_count"; +constexpr auto kIterLoopParamName = "iter_loop"; +constexpr auto kZeroParamName = "zero"; +constexpr auto kOneParamName = "one"; +constexpr auto kStreamSwitch = "StreamSwitch"; +constexpr auto kStreamActive = "StreamActive"; +constexpr auto kAssignAdd = "AssignAdd"; +namespace mindspore { +namespace device { +using device::ascend::ProfilingUtils; +void KernelAdjust::Reorder(const std::shared_ptr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + const std::vector &origin_cnode_list = kernel_graph_ptr->execution_order(); + std::vector momentum_list; + std::vector other_list; + for (const auto &cnode : origin_cnode_list) { + if (kOptOpeatorSet.find(AnfAlgo::GetCNodeName(cnode)) != kOptOpeatorSet.end()) { + momentum_list.emplace_back(cnode); + } else { + other_list.emplace_back(cnode); + } + } + std::vector new_order_list; + new_order_list.insert(new_order_list.end(), other_list.begin(), other_list.end()); + new_order_list.insert(new_order_list.end(), momentum_list.begin(), momentum_list.end()); + kernel_graph_ptr->set_execution_order(new_order_list); +} + +bool KernelAdjust::NeedInsertSwitch() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + return (context_ptr->enable_task_sink() && context_ptr->loop_sink_flag() && + ConfigManager::GetInstance().iter_num() > 1); +} + +void KernelAdjust::InsertSwitchLoop(const std::shared_ptr &kernel_graph_ptr) { + if (!NeedInsertSwitch()) { + return; + } + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + std::map switch_loop_input; + CreateSwitchOpParameters(kernel_graph_ptr, &switch_loop_input); + + std::vector *mute_inputs = kernel_graph_ptr->MutableInputs(); + MS_EXCEPTION_IF_NULL(mute_inputs); + mute_inputs->push_back(switch_loop_input[kLoopCountParamName]); + mute_inputs->push_back(switch_loop_input[kIterLoopParamName]); + mute_inputs->push_back(switch_loop_input[kZeroParamName]); + mute_inputs->push_back(switch_loop_input[kOneParamName]); + for (const auto &input : kernel_graph_ptr->inputs()) { + MS_EXCEPTION_IF_NULL(input); + if (input->isa()) { + ParameterPtr param_ptr = input->cast(); + if (param_ptr == nullptr) { + MS_EXCEPTION(NotSupportError) << "Cast to parameter point failed !"; + } + } + } + std::vector exec_order; + CNodePtr stream_switch_app = CreateStreamSwitchOp(kernel_graph_ptr, switch_loop_input); + MS_EXCEPTION_IF_NULL(stream_switch_app); + exec_order.push_back(stream_switch_app); + + CNodePtr stream_active_switch_app = CreateStreamActiveSwitchOp(kernel_graph_ptr); + MS_EXCEPTION_IF_NULL(stream_active_switch_app); + + CNodePtr assign_add_one = CreateStreamAssignAddnOP(kernel_graph_ptr, switch_loop_input); + MS_EXCEPTION_IF_NULL(assign_add_one); + exec_order.push_back(assign_add_one); + + auto original_exec_order = kernel_graph_ptr->execution_order(); + (void)std::copy(original_exec_order.begin(), original_exec_order.end(), std::back_inserter(exec_order)); + exec_order.push_back(stream_active_switch_app); + kernel_graph_ptr->set_execution_order(exec_order); +} + +void KernelAdjust::CreateSwitchOpParameters(const std::shared_ptr &kernel_graph_ptr, + std::map *switch_loop_input) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + MS_EXCEPTION_IF_NULL(switch_loop_input); + std::vector shp = {1}; + tensor::TensorPtr tensor_ptr = std::make_shared(kInt32->type_id(), shp); + MS_EXCEPTION_IF_NULL(tensor_ptr); + mindspore::abstract::AbstractBasePtr paremeter_abstract_ptr = tensor_ptr->ToAbstract(); + if (paremeter_abstract_ptr == nullptr) { + MS_LOG(EXCEPTION) << "create abstract brfore insert switch op failed!"; + } + + ParameterPtr loop_count = std::make_shared(kernel_graph_ptr); + MS_EXCEPTION_IF_NULL(loop_count); + loop_count->set_name(kLoopCountParamName); + loop_count->set_abstract(paremeter_abstract_ptr); + ParameterPtr loop_count_new = kernel_graph_ptr->NewParameter(loop_count); + + (*switch_loop_input)[kLoopCountParamName] = loop_count_new; + + ParameterPtr iter_loop = std::make_shared(kernel_graph_ptr); + iter_loop->set_name(kIterLoopParamName); + iter_loop->set_abstract(paremeter_abstract_ptr); + ParameterPtr iter_loop_new = kernel_graph_ptr->NewParameter(iter_loop); + (*switch_loop_input)[kIterLoopParamName] = iter_loop_new; + + ParameterPtr zero = std::make_shared(kernel_graph_ptr); + zero->set_name(kZeroParamName); + zero->set_abstract(paremeter_abstract_ptr); + ParameterPtr zero_new = kernel_graph_ptr->NewParameter(zero); + (*switch_loop_input)[kZeroParamName] = zero_new; + + ParameterPtr one = std::make_shared(kernel_graph_ptr); + one->set_name(kOneParamName); + one->set_abstract(paremeter_abstract_ptr); + ParameterPtr one_new = kernel_graph_ptr->NewParameter(one); + (*switch_loop_input)[kOneParamName] = one_new; +} + +kernel::KernelBuildInfo::KernelBuildInfoBuilder KernelAdjust::CreateMngKernelBuilder( + const std::vector &formats, const std::vector &type_ids) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder; + selected_kernel_builder.SetInputsFormat(formats); + selected_kernel_builder.SetInputsDeviceType(type_ids); + + selected_kernel_builder.SetFusionType(kernel::FusionType::OPAQUE); + selected_kernel_builder.SetProcessor(kernel::Processor::AICORE); + selected_kernel_builder.SetKernelType(KernelType::RT_KERNEL); + return selected_kernel_builder; +} + +CNodePtr KernelAdjust::CreateStreamSwitchOp(const std::shared_ptr &kernel_graph_ptr, + const std::map &switch_loop_input) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder = CreateMngKernelBuilder( + {kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32}); + auto typeNone_abstract = std::make_shared(); + auto stream_switch = std::make_shared(kStreamSwitch); + std::vector inputs; + inputs.push_back(NewValueNode(stream_switch)); + inputs.push_back(switch_loop_input.at(kLoopCountParamName)); + inputs.push_back(switch_loop_input.at(kIterLoopParamName)); + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + CNodePtr stream_switch_app = kernel_graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(stream_switch_app); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), stream_switch_app.get()); + stream_switch_app->set_abstract(typeNone_abstract); + // set attr: cond_ RT_LESS + int condition = static_cast(RT_LESS); + ValuePtr cond = MakeValue(condition); + AnfAlgo::SetNodeAttr(kAttrSwitchCondition, cond, stream_switch_app); + // set attr:true branch graph id ,which is same to stream distinction label + if (kernel_graph_ptr->execution_order().empty()) { + MS_LOG(EXCEPTION) << "empty execution order"; + } + auto first_node = kernel_graph_ptr->execution_order()[0]; + auto first_stream = AnfAlgo::GetStreamDistinctionLabel(first_node.get()); + AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(first_stream), stream_switch_app); + // set attr:data_type + int data_type = static_cast(RT_SWITCH_INT64); + ValuePtr dt = MakeValue(data_type); + AnfAlgo::SetNodeAttr(kAttrDataType, dt, stream_switch_app); + // set distinction label and graph id + AnfAlgo::SetGraphId(kInvalidGraphId - 1, stream_switch_app.get()); + AnfAlgo::SetStreamDistinctionLabel(kInvalidDistincLabel - 1, stream_switch_app.get()); + return stream_switch_app; +} + +CNodePtr KernelAdjust::CreateSteamActiveOp(const std::shared_ptr &kernel_graph_ptr) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder = CreateMngKernelBuilder( + {kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32}); + abstract::AbstractBasePtr typeNone_abstract = std::make_shared(); + auto stream_active_others = std::make_shared(kStreamActive); + std::vector inputs; + inputs.push_back(NewValueNode(stream_active_others)); + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + CNodePtr stream_active_others_app = kernel_graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(stream_active_others_app); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), stream_active_others_app.get()); + stream_active_others_app->set_abstract(typeNone_abstract); + return stream_active_others_app; +} + +CNodePtr KernelAdjust::CreateStreamActiveSwitchOp(const std::shared_ptr &kernel_graph_ptr) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder = CreateMngKernelBuilder( + {kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32}); + abstract::AbstractBasePtr typeNone_abstract = std::make_shared(); + auto stream_active_switch = std::make_shared(kStreamActive); + std::vector inputs; + inputs.push_back(NewValueNode(stream_active_switch)); + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + CNodePtr stream_active_switch_app = kernel_graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(stream_active_switch_app); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), stream_active_switch_app.get()); + stream_active_switch_app->set_abstract(typeNone_abstract); + // set attr,which stream to active + std::vector active_index_value = {kInvalidDistincLabel - 1}; + auto value = MakeValue>(active_index_value); + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, value, stream_active_switch_app); + // set the distinction label of stream active + if (kernel_graph_ptr->execution_order().empty()) { + MS_LOG(EXCEPTION) << "empty execution order"; + } + auto first_node = kernel_graph_ptr->execution_order()[0]; + auto label = AnfAlgo::GetStreamDistinctionLabel(first_node.get()); + // find the first switch's distinction label + for (auto node : kernel_graph_ptr->execution_order()) { + if (AnfAlgo::GetCNodeName(node) == "StreamSwitch") { + label = AnfAlgo::GetStreamDistinctionLabel(node.get()); + break; + } + } + AnfAlgo::SetStreamDistinctionLabel(label, stream_active_switch_app.get()); + return stream_active_switch_app; +} + +CNodePtr KernelAdjust::CreateStreamActiveOtherOp(const std::shared_ptr &kernel_graph_ptr) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder = CreateMngKernelBuilder( + {kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32}); + abstract::AbstractBasePtr typeNone_abstract = std::make_shared(); + auto stream_active_others = std::make_shared(kStreamActive); + std::vector inputs; + inputs.push_back(NewValueNode(stream_active_others)); + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + CNodePtr stream_active_others_app = kernel_graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(stream_active_others_app); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), stream_active_others_app.get()); + stream_active_others_app->set_abstract(typeNone_abstract); + // set attr + ValuePtr active_target = MakeValue(kValueTargetOther); + AnfAlgo::SetNodeAttr(kAttrActiveTarget, active_target, stream_active_others_app); + return stream_active_others_app; +} + +CNodePtr KernelAdjust::CreateStreamAssignAddnOP( + const std::shared_ptr &kernel_graph_ptr, + const std::map &switch_loop_input) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder = CreateMngKernelBuilder( + {kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {TypeId::kNumberTypeInt32, TypeId::kNumberTypeInt32}); + selected_kernel_builder.SetOutputsFormat({kOpFormat_DEFAULT}); + selected_kernel_builder.SetOutputsDeviceType({kNumberTypeInt32}); + // AssignAdd + auto assign_add = std::make_shared(kAssignAdd); + std::vector inputs; + inputs.push_back(NewValueNode(assign_add)); + inputs.push_back(switch_loop_input.at(kLoopCountParamName)); + inputs.push_back(switch_loop_input.at(kOneParamName)); + CNodePtr assign_add_one = kernel_graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(assign_add_one); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), assign_add_one.get()); + std::vector input_names = {"ref", "value"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + AnfAlgo::SetNodeAttr("input_names", input_names_v, assign_add_one); + AnfAlgo::SetNodeAttr("output_names", output_names_v, assign_add_one); + selected_kernel_builder.SetKernelType(KernelType::TBE_KERNEL); + MS_EXCEPTION_IF_NULL(switch_loop_input.at(kLoopCountParamName)); + assign_add_one->set_abstract(switch_loop_input.at(kLoopCountParamName)->abstract()); + // set the distinction label of assign add + if (kernel_graph_ptr->execution_order().empty()) { + MS_LOG(EXCEPTION) << "empty execution order"; + } + auto first_node = kernel_graph_ptr->execution_order()[0]; + auto label = AnfAlgo::GetStreamDistinctionLabel(first_node.get()); + AnfAlgo::SetStreamDistinctionLabel(label, assign_add_one.get()); + return assign_add_one; +} + +void KernelAdjust::SetStreamActiveOPs(const std::shared_ptr &kernel_graph_ptr, + const std::unordered_set &ctrl_stream_list, + const std::unordered_set &comm_stream_list, + const std::unordered_set &momentum_stream_list) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + for (const auto &cnode_ptr : kernel_graph_ptr->execution_order()) { + MS_EXCEPTION_IF_NULL(cnode_ptr); + if (AnfAlgo::GetCNodeName(cnode_ptr) == kStreamActive) { + auto primitive = AnfAlgo::GetCNodePrimitive(cnode_ptr); + ValuePtr active_target = primitive->GetAttr(kAttrActiveTarget); + std::vector index_list; + index_list.clear(); + if (GetValue(active_target) == kValueTargetSwitch) { + index_list.insert(index_list.end(), ctrl_stream_list.begin(), ctrl_stream_list.end()); + } else if (GetValue(active_target) == kValueTargetOther) { + for (uint32_t index : comm_stream_list) { + if (AnfAlgo::GetStreamId(cnode_ptr) == index) { + continue; + } + index_list.emplace_back(index); + } + index_list.insert(index_list.end(), momentum_stream_list.begin(), momentum_stream_list.end()); + } + ValuePtr index_list_value = MakeValue(index_list); + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, index_list_value, cnode_ptr); + } + } +} + +void KernelAdjust::SetStreamSwitchOps(const std::shared_ptr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + CNodePtr switch_cnode_ptr = nullptr; + uint32_t target_stream_id = 0; + for (const auto &cnode_ptr : kernel_graph_ptr->execution_order()) { + MS_EXCEPTION_IF_NULL(cnode_ptr); + if (AnfAlgo::GetCNodeName(cnode_ptr) == kStreamSwitch) { + switch_cnode_ptr = cnode_ptr; + } + if (AnfAlgo::GetCNodeName(cnode_ptr) == kStreamActive) { + auto primitive = AnfAlgo::GetCNodePrimitive(cnode_ptr); + ValuePtr active_target = primitive->GetAttr(kAttrActiveTarget); + if (GetValue(active_target) == kValueTargetOther) { + target_stream_id = AnfAlgo::GetStreamId(cnode_ptr); + } + } + } + if (switch_cnode_ptr != nullptr) { + // set attr:true stream + ValuePtr true_index = MakeValue(target_stream_id); + AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, true_index, switch_cnode_ptr); + MS_LOG(INFO) << "switch to true_index:" << target_stream_id; + } +} + +bool KernelAdjust::StepLoadCtrlInputs(const std::shared_ptr &context, + const std::shared_ptr &kernel_graph_ptr) { + if (!NeedInsertSwitch()) { + return true; + } + MS_EXCEPTION_IF_NULL(context); + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + auto input_nodes = kernel_graph_ptr->inputs(); + std::vector inputs; + LoadSwitchInputs(&inputs); + std::shared_ptr> inputsPtr = std::make_shared>(inputs); + context->SetResult(session::kInputCtrlTensors, inputsPtr); + size_t input_ctrl_size = inputs.size(); + // inputs_node:include four ctrl nodes in the back. such as:conv,loop_cnt, ites_loop, zero, one. + // deal four ctrl nodes. + for (size_t i = 0; i < inputs.size(); ++i) { + auto tensor = inputs[i]; + size_t deal_index = input_nodes.size() - input_ctrl_size + i; + if (deal_index >= input_nodes.size()) { + MS_LOG(EXCEPTION) << "deak_index[" << deal_index << "] outof range"; + } + auto input_node = input_nodes[deal_index]; + bool need_sync = false; + MS_EXCEPTION_IF_NULL(input_node); + if (input_node->isa()) { + auto pk_node = input_node->cast(); + MS_EXCEPTION_IF_NULL(tensor); + MS_EXCEPTION_IF_NULL(pk_node); + if (tensor->is_dirty() || !pk_node->has_default()) { + need_sync = true; + } + } + if (need_sync) { + auto pk_node = input_node->cast(); + MS_EXCEPTION_IF_NULL(pk_node); + auto device_address = AnfAlgo::GetMutableOutputAddr(pk_node, 0); + MS_EXCEPTION_IF_NULL(device_address); + tensor->set_device_address(device_address); + if (!device_address->SyncHostToDevice(tensor->shape(), LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c(false))) { + MS_LOG(INFO) << "SyncHostToDevice failed."; + return false; + } + } + tensor->set_dirty(false); + } + return true; +} + +void KernelAdjust::LoadSwitchInputs(std::vector *inputs) { + MS_LOG(INFO) << "---------------- LoadSwitchInputs---"; + MS_EXCEPTION_IF_NULL(inputs); + std::vector shp = {1}; + tensor::TensorPtr loop_count_tensor = std::make_shared(kInt32->type_id(), shp); + MS_EXCEPTION_IF_NULL(loop_count_tensor); + int32_t *val = nullptr; + val = static_cast(loop_count_tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(val); + *val = 0; + inputs->push_back(loop_count_tensor); + + tensor::TensorPtr iter_loop_tensor = std::make_shared(kInt32->type_id(), shp); + MS_EXCEPTION_IF_NULL(iter_loop_tensor); + val = static_cast(iter_loop_tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(val); + *val = SizeToInt(LongToSize(ConfigManager::GetInstance().iter_num())); + MS_LOG(INFO) << "iter_loop_tensor = " << *val; + inputs->push_back(iter_loop_tensor); + + tensor::TensorPtr zero_tensor = std::make_shared(kInt32->type_id(), shp); + MS_EXCEPTION_IF_NULL(zero_tensor); + val = static_cast(zero_tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(val); + *val = 0; + inputs->push_back(zero_tensor); + + tensor::TensorPtr one_tensor = std::make_shared(kInt32->type_id(), shp); + MS_EXCEPTION_IF_NULL(one_tensor); + val = static_cast(one_tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(val); + *val = 1; + inputs->push_back(one_tensor); + MS_LOG(INFO) << "---------------- LoadSwitchInputs End--"; +} + +void KernelAdjust::Profiling(const std::shared_ptr &kernel_graph_ptr) { + if (!ascend::ProfilingManager::GetInstance().IsProfiling()) { + MS_LOG(INFO) << "no need to profiling"; + return; + } + ProfilingTraceInfo profiling_trace_info; + if (ProfilingUtils::GetProfilingTraceInfo(kernel_graph_ptr, &profiling_trace_info)) { + InsertProfilingKernel(kernel_graph_ptr, profiling_trace_info); + } else { + MS_LOG(WARNING) << "[profiling] GetProfilingTraceInfo failed"; + } +} + +void KernelAdjust::InsertProfilingKernel(const std::shared_ptr &kernel_graph_ptr, + const ProfilingTraceInfo &profiling_trace_info) { + MS_LOG(INFO) << "[profiling] insert profiling kernel start"; + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + if (!profiling_trace_info.IsValid()) { + MS_LOG(WARNING) << "profiling trace point not found"; + return; + } + std::vector new_cnode_list; + std::vector cnode_ptr_list = kernel_graph_ptr->execution_order(); + for (const auto &cnode_ptr : cnode_ptr_list) { + ProfilingUtils::ProfilingTraceFpStart(kernel_graph_ptr, cnode_ptr, profiling_trace_info, &new_cnode_list); + ProfilingUtils::ProfilingAllReduce(kernel_graph_ptr, cnode_ptr, ascend::kProfilingAllReduce1Start, + profiling_trace_info.profiling_allreduce1_start, &new_cnode_list); + ProfilingUtils::ProfilingAllReduce(kernel_graph_ptr, cnode_ptr, ascend::kProfilingAllReduce2Start, + profiling_trace_info.profiling_allreduce2_start, &new_cnode_list); + new_cnode_list.emplace_back(cnode_ptr); + + ProfilingUtils::ProfilingAllReduce(kernel_graph_ptr, cnode_ptr, ascend::kProfilingAllReduce1End, + profiling_trace_info.profiling_allreduce1_end, &new_cnode_list); + ProfilingUtils::ProfilingAllReduce(kernel_graph_ptr, cnode_ptr, ascend::kProfilingAllReduce2End, + profiling_trace_info.profiling_allreduce2_end, &new_cnode_list); + ProfilingUtils::ProfilingTraceEnd(kernel_graph_ptr, cnode_ptr, profiling_trace_info, &new_cnode_list); + } + kernel_graph_ptr->set_execution_order(new_cnode_list); +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/kernel_adjust.h b/mindspore/ccsrc/device/kernel_adjust.h new file mode 100644 index 00000000000..62c64d98b90 --- /dev/null +++ b/mindspore/ccsrc/device/kernel_adjust.h @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_KERNEL_ADJUST_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_KERNEL_ADJUST_H_ + +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "session/kernel_graph.h" +#include "kernel/kernel_build_info.h" +#include "session/session_context.h" +#include "ir/meta_tensor.h" +#include "device/ascend/profiling/profiling_utils.h" + +using mindspore::device::ascend::ProfilingTraceInfo; +using mindspore::device::ascend::ProfilingUtils; +namespace mindspore { +namespace device { +class KernelAdjust { + public: + static KernelAdjust &GetInstance() { + static KernelAdjust instance; + return instance; + } + void Reorder(const std::shared_ptr &kernel_graph_ptr); + void InsertSwitchLoop(const std::shared_ptr &kernel_graph_ptr); + void SetStreamActiveOPs(const std::shared_ptr &kernel_graph_ptr, + const std::unordered_set &ctrl_stream_list, + const std::unordered_set &comm_stream_list, + const std::unordered_set &momentum_stream_list); + void SetStreamSwitchOps(const std::shared_ptr &kernel_graph_ptr); + bool StepLoadCtrlInputs(const std::shared_ptr &context, + const std::shared_ptr &kernel_graph_ptr); + void Profiling(const std::shared_ptr &kernel_graph_ptr); + static bool NeedInsertSwitch(); + CNodePtr CreateSteamActiveOp(const std::shared_ptr &kernel_graph_ptr); + + private: + KernelAdjust() = default; + ~KernelAdjust() = default; + void CreateSwitchOpParameters(const std::shared_ptr &kernel_graph_ptr, + std::map *switch_loop_input); + CNodePtr CreateStreamSwitchOp(const std::shared_ptr &kernel_graph_ptr, + const std::map &switch_loop_input); + CNodePtr CreateStreamActiveSwitchOp(const std::shared_ptr &kernel_graph_ptr); + CNodePtr CreateStreamActiveOtherOp(const std::shared_ptr &kernel_graph_ptr); + CNodePtr CreateStreamAssignAddnOP(const std::shared_ptr &kernel_graph_ptr, + const std::map &switch_loop_input); + kernel::KernelBuildInfo::KernelBuildInfoBuilder CreateMngKernelBuilder(const std::vector &formats, + const std::vector &type_ids); + void LoadSwitchInputs(std::vector *inputs); + void InsertProfilingKernel(const std::shared_ptr &kernel_graph_ptr, + const ProfilingTraceInfo &profiling_trace_info); +}; +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_KERNEL_ADJUST_H_ diff --git a/mindspore/ccsrc/device/kernel_info.cc b/mindspore/ccsrc/device/kernel_info.cc new file mode 100644 index 00000000000..59c9b0f4111 --- /dev/null +++ b/mindspore/ccsrc/device/kernel_info.cc @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/kernel_info.h" + +namespace mindspore { +namespace device { +const kernel::KernelBuildInfo *KernelInfo::select_kernel_build_info() const { return select_kernel_build_info_.get(); } + +kernel::KernelBuildInfoPtr KernelInfo::GetMutableSelectKernelBuildInfo() const { return select_kernel_build_info_; } + +const DeviceAddress *KernelInfo::GetOutputAddr(size_t index) const { + if (index >= output_address_list_.size()) { + MS_LOG(ERROR) << "Index [" << index << "] out of range"; + return nullptr; + } + return output_address_list_[index].get(); +} + +DeviceAddressPtr KernelInfo::GetMutableOutputAddr(size_t index) const { + if (index >= output_address_list_.size()) { + MS_LOG(ERROR) << "Index [" << index << "] out of range"; + return nullptr; + } + return output_address_list_[index]; +} + +bool KernelInfo::OutputAddrExist(size_t index) const { + if (index >= output_address_list_.size()) { + return false; + } + return output_address_list_[index] != nullptr; +} + +bool KernelInfo::SetOutputAddr(const DeviceAddressPtr &output_address, size_t index) { + // parameter and valuenode + if (kernel_mod_ == nullptr && index >= output_address_list_.size()) { + for (size_t i = output_address_list_.size(); i <= index; i++) { + output_address_list_.emplace_back(nullptr); + } + } else if (output_address_list_.empty()) { + // set cnode + for (size_t i = 0; i < kernel_mod_->GetOutputSizeList().size(); i++) { + output_address_list_.emplace_back(nullptr); + } + } + if (index >= output_address_list_.size()) { + MS_LOG(ERROR) << "Index [" << index << "] out of range"; + return false; + } + output_address_list_[index] = output_address; + return true; +} + +DeviceAddress *KernelInfo::GetWorkspaceAddr(size_t index) const { + if (index >= workspace_address_list_.size()) { + MS_LOG(ERROR) << "Index [" << index << "] out of range"; + return nullptr; + } + return workspace_address_list_[index].get(); +} + +bool KernelInfo::SetWorkspaceAddr(const DeviceAddressPtr &output_address, size_t index) { + if (workspace_address_list_.empty()) { + // parameter and valuenode + if (kernel_mod_ == nullptr) { + workspace_address_list_.emplace_back(nullptr); + } else { + // set cnode + for (size_t i = 0; i < kernel_mod_->GetWorkspaceSizeList().size(); i++) { + workspace_address_list_.emplace_back(nullptr); + } + } + } + if (index >= workspace_address_list_.size()) { + MS_LOG(ERROR) << "Index" << index << " out of range"; + return false; + } + workspace_address_list_[index] = output_address; + return true; +} + +void KernelInfo::set_kernel_mod(const kernel::KernelModPtr &kernel_mod) { kernel_mod_ = kernel_mod; } + +kernel::KernelMod *KernelInfo::MutableKernelMod() const { return kernel_mod_.get(); } + +const kernel::KernelMod *KernelInfo::kernel_mod() const { return kernel_mod_.get(); } + +bool KernelInfo::operator==(const KernelInfo &other) const { + if (stream_id_ != other.stream_id_ || stream_distinction_label_ != other.stream_distinction_label_ || + graph_id_ != other.graph_id_) { + return false; + } + if ((select_kernel_build_info_ != nullptr && other.select_kernel_build_info_ == nullptr) || + (select_kernel_build_info_ == nullptr && other.select_kernel_build_info_ != nullptr)) { + return false; + } + if (select_kernel_build_info_ != nullptr && other.select_kernel_build_info_ != nullptr) { + if (!(*select_kernel_build_info_ == *(other.select_kernel_build_info_))) { + return false; + } + } + // Currently we only check whether both the kernel_mod_ are initialized or uninitialized. + if ((kernel_mod_ == nullptr && other.kernel_mod_ != nullptr) || + (kernel_mod_ != nullptr && other.kernel_mod_ == nullptr)) { + return false; + } + // Currently we only check whether both the sizes are equal of output_address_list_ and workspace_address_list_ or + // not. We can complete this check in the future. + if (output_address_list_.size() != other.output_address_list_.size() || + workspace_address_list_.size() != other.workspace_address_list_.size()) { + return false; + } + return true; +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/kernel_info.h b/mindspore/ccsrc/device/kernel_info.h new file mode 100644 index 00000000000..93521587743 --- /dev/null +++ b/mindspore/ccsrc/device/kernel_info.h @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_DEVICE_KERNEL_INFO_H_ +#define MINDSPORE_DEVICE_KERNEL_INFO_H_ + +#include +#include +#include "kernel/kernel_build_info.h" +#include "device/ascend/ascend_device_address.h" +#include "kernel/kernel.h" + +namespace mindspore { +const uint32_t kInvalidGraphId = UINT32_MAX; +const uint32_t kInvalidDistincLabel = UINT32_MAX; +namespace device { +class KernelInfo { + public: + KernelInfo() { + kernel_mod_ = nullptr; + select_kernel_build_info_ = nullptr; + output_address_list_ = {}; + workspace_address_list_ = {}; + stream_id_ = 0; + stream_distinction_label_ = kInvalidDistincLabel; + graph_id_ = kInvalidGraphId; + } + virtual ~KernelInfo() = default; + + const kernel::KernelBuildInfo *select_kernel_build_info() const; + kernel::KernelBuildInfoPtr GetMutableSelectKernelBuildInfo() const; + void set_select_kernel_build_info(const kernel::KernelBuildInfoPtr &select_kernel_build_info) { + select_kernel_build_info_ = select_kernel_build_info; + } + const DeviceAddress *GetOutputAddr(size_t index) const; + DeviceAddressPtr GetMutableOutputAddr(size_t index) const; + bool OutputAddrExist(size_t index) const; + bool SetOutputAddr(const DeviceAddressPtr &output_address, size_t index); + DeviceAddress *GetWorkspaceAddr(size_t index) const; + bool SetWorkspaceAddr(const DeviceAddressPtr &output_address, size_t index); + void set_kernel_mod(const kernel::KernelModPtr &kernel_mod); + kernel::KernelMod *MutableKernelMod() const; + const kernel::KernelMod *kernel_mod() const; + uint32_t stream_id() const { return stream_id_; } + void set_stream_id(uint32_t stream_id) { stream_id_ = stream_id; } + uint32_t stream_distinction_label() const { return stream_distinction_label_; } + void set_stream_distinction_label(uint32_t stream_distinction_label) { + stream_distinction_label_ = stream_distinction_label; + } + void set_graph_id(uint32_t graph_id) { graph_id_ = graph_id; } + uint32_t graph_id() const { return graph_id_; } + bool operator==(const KernelInfo &other) const; + + private: + kernel::KernelBuildInfoPtr select_kernel_build_info_; + std::vector> output_address_list_; + std::vector> workspace_address_list_; + kernel::KernelModPtr kernel_mod_; + // stream_id_ is the index of stream object vector + uint32_t stream_id_; + // stream_distinction_label_ is used mark different op in different stream + uint32_t stream_distinction_label_; + // record which graph the node belong to + uint32_t graph_id_; +}; +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_DEVICE_KERNEL_INFO_H_ diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc new file mode 100644 index 00000000000..f320aa9cf65 --- /dev/null +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -0,0 +1,782 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/kernel_runtime.h" +#include +#include +#include +#include "common/utils.h" +#include "common/trans.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "operator/ops.h" +#include "pipeline/parse/python_adapter.h" +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/common_utils.h" +#include "kernel/oplib/oplib.h" +#include "ir/value.h" +using mindspore::kernel::Address; +using mindspore::kernel::AddressPtr; +using mindspore::memreuse::BestFitMemReuse; +using mindspore::memreuse::MemReuseUtilPtr; + +namespace mindspore { +namespace device { +KernelRuntime::~KernelRuntime() { + device_mem_base_ = nullptr; + device_mem_pool_base_ = nullptr; +#ifdef ENABLE_DUMP_E2E + dump_conf_ptr_ = nullptr; +#endif + reuse_mem_base_ = nullptr; + mem_reuse_util_ptr_ = nullptr; +} + +bool KernelRuntime::Run(session::KernelGraph *graph) { + bool ret = false; + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool is_task_sink = context_ptr->enable_task_sink(); + if (is_task_sink) { + ret = RunTask(graph); + } else { + ret = LaunchKernel(graph); + } + return ret; +} + +// for D to impl +bool KernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { + if (graph != nullptr) { + return true; + } + return false; +} + +// for D to impl +bool KernelRuntime::GenTask(const session::KernelGraph *graph) { + if (graph != nullptr) { + return true; + } + return false; +} + +bool KernelRuntime::LoadTask(const session::KernelGraph *graph) { + if (graph != nullptr) { + return true; + } + return false; +} + +void KernelRuntime::FreeHostMemory() { + dynamic_mem_offset_ = 0; + static_mem_offset_ = 0; +} + +// for D to impl +bool KernelRuntime::RunTask(const session::KernelGraph *graph) { + if (graph != nullptr) { + return true; + } + return false; +} + +size_t KernelRuntime::CountNodeDeviceMemorySize(const mindspore::AnfNodePtr &node, size_t output_index) { + MS_EXCEPTION_IF_NULL(node); + if (output_index >= AnfAlgo::GetOutputTensorNum(node)) { + MS_EXCEPTION(ArgumentError) << "output index [" << output_index << "] large than the output size [" + << AnfAlgo::GetOutputTensorNum(node) << "] of node!"; + } + TypeId output_type_id = AnfAlgo::GetOutputDeviceDataType(node, output_index); + if (output_type_id == kTypeUnknown) { + output_type_id = AnfAlgo::GetOutputInferDataType(node, output_index); + } + size_t type_size = GetTypeByte(TypeIdToType(output_type_id)); + std::vector shape = AnfAlgo::GetOutputDeviceShape(node, output_index); + auto format = AnfAlgo::GetOutputFormat(node, output_index); + if (shape.empty() && format != kOpFormat_DEFAULT) { + shape = trans::TransShapeTo4d(shape); + shape = trans::TransShapeToDevice(shape, format); + } + // scalar's output shape is a empty vector + size_t tensor_size = std::accumulate(shape.begin(), shape.end(), type_size, std::multiplies()); + return tensor_size; +} + +void KernelRuntime::AssignMemory(session::KernelGraph *graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + AssignStaticMemory(graph); + bool is_enable_mem_reuse = context_ptr->enable_mem_reuse(); + if (is_enable_mem_reuse) { + ReuseAssignDynamicMemory(graph); + } else { + AssignDynamicMemory(graph); + } + UpdateRefNodeOutputMem(graph); +} + +void KernelRuntime::RunOpAssignMemory(const std::vector &input_tensors, + const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + // assign memory for input nodes + RunOpAssignInputMemory(input_tensors, graph); + for (const auto &cnode : graph->execution_order()) { + // assign memory for output nodes + RunOpAssignOutputMemory(cnode); + // assign memory for workspace + RunOpAssignWorkSpaceMemory(cnode); + } + UpdateRefNodeOutputMem(graph); +} + +void KernelRuntime::AssignStaticMemory(session::KernelGraph *graph) { + AssignStaticMemoryInput(graph); + AssignStaticMemoryValueNode(graph); + AssignStaticMemoryOutput(graph); +} + +void KernelRuntime::RunOpAssignInputMemory(const std::vector &input_tensors, + const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + for (size_t input_index = 0; input_index < graph->inputs().size(); ++input_index) { + auto item = graph->inputs()[input_index]; + MS_EXCEPTION_IF_NULL(item); + if (!item->isa()) { + continue; + } + auto output_size = AnfAlgo::GetOutputTensorNum(item); + for (size_t index = 0; index < output_size; index++) { + MS_EXCEPTION_IF_NULL(input_tensors[input_index]); + if (input_tensors[input_index]->device_address().get() != nullptr) { + AnfAlgo::SetOutputAddr(input_tensors[input_index]->device_address(), index, item.get()); + continue; + } + TypeId output_type_id = AnfAlgo::GetOutputDeviceDataType(item, index); + if (output_type_id == kTypeUnknown) { + output_type_id = AnfAlgo::GetOutputInferDataType(item, index); + } + auto tensor_size = CountNodeDeviceMemorySize(item, index); + auto device_address = + CreateDeviceAddress(nullptr, tensor_size, AnfAlgo::GetOutputFormat(item, index), output_type_id); + MS_EXCEPTION_IF_NULL(device_address); + MallocOpMemory(device_address, tensor_size, kStaticMem); + AnfAlgo::SetOutputAddr(device_address, index, item.get()); + } + } +} + +void KernelRuntime::RunOpAssignOutputMemory(const AnfNodePtr &kernel) { + MS_EXCEPTION_IF_NULL(kernel); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto output_sizes = kernel_mod->GetOutputSizeList(); + if (output_sizes.empty()) { + return; + } + if (AnfAlgo::GetCNodeName(kernel) == "ApplyMomentum") { + auto device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, 0); + AnfAlgo::SetOutputAddr(device_address, 0, kernel.get()); + return; + } + + for (size_t i = 0; i < output_sizes.size(); ++i) { + if (AnfAlgo::OutputAddrExist(kernel, i)) { + continue; + } + std::string output_format = AnfAlgo::GetOutputFormat(kernel, i); + auto output_type = AnfAlgo::GetOutputDeviceDataType(kernel, i); + auto device_address = CreateDeviceAddress(nullptr, output_sizes[i], output_format, output_type); + MS_EXCEPTION_IF_NULL(device_address); + MallocOpMemory(device_address, output_sizes[i], kDynamicMem); + AnfAlgo::SetOutputAddr(device_address, i, kernel.get()); + } +} + +void KernelRuntime::RunOpAssignWorkSpaceMemory(const AnfNodePtr &kernel) { + MS_EXCEPTION_IF_NULL(kernel); + if (kernel->isa()) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto workspace_lists = kernel_mod->GetWorkspaceSizeList(); + for (size_t i = 0; i < workspace_lists.size(); ++i) { + auto device_address = CreateDeviceAddress(nullptr, workspace_lists[i], "", kTypeUnknown); + MS_EXCEPTION_IF_NULL(device_address); + MallocOpMemory(device_address, workspace_lists[i], kDynamicMem); + AnfAlgo::SetWorkspaceAddr(device_address, i, kernel.get()); + } + } +} + +void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + for (auto &item : graph->inputs()) { + MS_EXCEPTION_IF_NULL(item); + if (!item->isa()) { + continue; + } + if (AnfAlgo::OutputAddrExist(item, 0)) { + continue; + } + auto output_size = AnfAlgo::GetOutputTensorNum(item); + for (size_t index = 0; index < output_size; index++) { + TypeId output_type_id = AnfAlgo::GetOutputDeviceDataType(item, index); + // if graph output is a weight and doesn't link to any cnode,it's data type will be unkonwn + if (output_type_id == kTypeUnknown) { + MS_LOG(WARNING) << "It is not suggested to use a lonely weight parameter as the output of graph"; + output_type_id = AnfAlgo::GetOutputInferDataType(item, index); + } + auto tensor_size = CountNodeDeviceMemorySize(item, index); + auto ptr = MallocStaticMem(tensor_size, false); + auto address = CreateDeviceAddress(ptr, tensor_size, AnfAlgo::GetOutputFormat(item, index), output_type_id); + AnfAlgo::SetOutputAddr(address, index, item.get()); + } + } +} + +void KernelRuntime::AssignStaticMemoryOutput(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + auto nodes = AnfAlgo::GetAllOutput(graph->output(), {prim::kPrimTupleGetItem}); + for (const auto &node : nodes) { + auto item_with_index = AnfAlgo::VisitKernelWithReturnType(node, 0); + MS_EXCEPTION_IF_NULL(item_with_index.first); + if (!item_with_index.first->isa() || !AnfAlgo::IsRealKernel(item_with_index.first)) { + continue; + } + AssignNodeOutputMem(kStaticMem, item_with_index.first, SizeToInt(item_with_index.second)); + } +} + +void KernelRuntime::UpdateRefNodeOutputMem(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + auto &kernels = graph->execution_order(); + for (auto &kernel : kernels) { + MS_EXCEPTION_IF_NULL(kernel); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + + auto output_sizes = kernel_mod->GetOutputSizeList(); + if (output_sizes.empty()) { + MS_LOG(INFO) << "This kernel has no output size."; + continue; + } + for (size_t i = 0; i < output_sizes.size(); ++i) { + session::AnfWithOutIndex out_pair(kernel, i); + if (graph->IsInRefOutputMap(out_pair)) { + auto origin_pair = graph->GetRefCorrespondOutput(out_pair); + MS_EXCEPTION_IF_NULL(origin_pair.first); + auto origin_node_output_addr = AnfAlgo::GetMutableOutputAddr(origin_pair.first, origin_pair.second); + MS_EXCEPTION_IF_NULL(origin_node_output_addr); + auto cur_node_output_addr = AnfAlgo::GetMutableOutputAddr(kernel, i); + if (origin_node_output_addr.get() != cur_node_output_addr.get()) { + MS_LOG(INFO) << "REF address is not same, ref node output need address update"; + MS_LOG(INFO) << "REF origin op is " << origin_pair.first->DebugString() << ", output index is " + << origin_pair.second << ", cur op is " << kernel->DebugString() << ", out index is " << i; + AnfAlgo::SetOutputAddr(origin_node_output_addr, i, kernel.get()); + } + } + } + } +} + +void KernelRuntime::AssignCommunicationNodeOutputMem(int flag, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_mod = AnfAlgo::GetKernelMod(node); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto output_sizes = kernel_mod->GetOutputSizeList(); + if (output_sizes.empty()) { + MS_LOG(INFO) << "This kernel[" << node->DebugString() << "] has no output size."; + return; + } + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + size_t total_size = 0; + std::vector align_size_list; + for (uint64_t mem_size : output_sizes) { + if (context_ptr->enable_hccl()) { + mem_size = GetCommonAlignSize(mem_size); + } + total_size += mem_size; + align_size_list.emplace_back(mem_size); + } + uint8_t *output_ptr = CalDeviceMem(node, total_size, flag, 0); + for (size_t j = 0; j < align_size_list.size(); ++j) { + std::string output_format = AnfAlgo::GetOutputFormat(node, j); + auto output_type = AnfAlgo::GetOutputDeviceDataType(node, j); + auto address = CreateDeviceAddress(output_ptr, output_sizes[j], output_format, output_type); + AnfAlgo::SetOutputAddr(address, j, node.get()); + output_ptr += align_size_list[j]; + } +} + +void KernelRuntime::UpdateCommunicationOpInputMem(const AnfNodePtr &node) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + MS_EXCEPTION_IF_NULL(node); + size_t total_size = 0; + std::vector> addr_size; + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(node); ++i) { + auto address = AnfAlgo::GetPrevNodeMutableOutputAddr(node, i); + MS_EXCEPTION_IF_NULL(address); + auto mem_size = address->size(); + if (context_ptr->enable_hccl()) { + mem_size = GetCommonAlignSize(mem_size); + } + total_size += mem_size; + addr_size.emplace_back(address.get(), mem_size); + } + uint8_t *input_ptr = CalDeviceMem(node, total_size, kDynamicMem, 0); + for (const auto &iter : addr_size) { + MS_EXCEPTION_IF_NULL(iter.first); + iter.first->set_ptr(input_ptr); + input_ptr += iter.second; + } +} + +void KernelRuntime::AssignNodeOutputMem(int flag, const AnfNodePtr &node, int index) { + MS_EXCEPTION_IF_NULL(node); + if (IsCommunicationOp(node)) { + UpdateCommunicationOpInputMem(node); + AssignCommunicationNodeOutputMem(flag, node); + return; + } + auto kernel_mod = AnfAlgo::GetKernelMod(node); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto output_sizes = kernel_mod->GetOutputSizeList(); + if (output_sizes.empty()) { + MS_LOG(INFO) << "This kernel[" << node->DebugString() << "] has no output size."; + return; + } + for (size_t i = 0; i < output_sizes.size(); ++i) { + if ((kGetAllOuts != index) && (SizeToInt(i) != index)) { + continue; + } + if (AnfAlgo::OutputAddrExist(node, i)) { + MS_LOG(INFO) << "already malloc index:" << i; + continue; + } + auto ptr = CalDeviceMem(node, output_sizes[i], flag, i); + if (ptr == nullptr) { + // reused ptr, no need alloc, continue; + continue; + } + std::string output_format = AnfAlgo::GetOutputFormat(node, i); + auto output_type = AnfAlgo::GetOutputDeviceDataType(node, i); + AnfAlgo::SetOutputAddr(CreateDeviceAddress(ptr, output_sizes[i], output_format, output_type), i, node.get()); + } +} + +void KernelRuntime::AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, + size_t output_idx) { + MS_EXCEPTION_IF_NULL(value_node); + MS_EXCEPTION_IF_NULL(node_value); + auto tensor = node_value->cast(); + if (tensor == nullptr) { + MS_LOG(WARNING) << "tensor is null"; + return; + } + size_t tensor_size = tensor->data().nbytes(); + auto node_size = CountNodeDeviceMemorySize(value_node, output_idx); + auto ptr = MallocStaticMem(node_size, false); + TypeId output_type_id = AnfAlgo::GetOutputDeviceDataType(value_node, output_idx); + if (output_type_id == kTypeUnknown) { + output_type_id = AnfAlgo::GetOutputInferDataType(value_node, output_idx); + } + auto address = CreateDeviceAddress(ptr, node_size, AnfAlgo::GetOutputFormat(value_node, output_idx), output_type_id); + MS_EXCEPTION_IF_NULL(address); + AnfAlgo::SetOutputAddr(address, output_idx, value_node.get()); + if (!address->SyncHostToDevice(tensor->shape(), tensor_size, tensor->data_type(), tensor->data_c(false))) { + MS_EXCEPTION(NotExistsError) << "kValueNode SyncHostToDevice fail!" << value_node->DebugString() << "node format is" + << AnfAlgo::GetOutputFormat(value_node, output_idx) << "node dtype is " + << AnfAlgo::GetOutputInferDataType(value_node, output_idx); + } +} + +void KernelRuntime::AssignStaticMemoryValueNode(session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + for (auto &value_node : graph->graph_value_nodes()) { + MS_EXCEPTION_IF_NULL(value_node); + if (AnfAlgo::OutputAddrExist(value_node, 0)) { + MS_LOG(INFO) << "value_node[" << value_node->DebugString() << "] address already exist"; + continue; + } + auto &node_value = value_node->value(); + MS_EXCEPTION_IF_NULL(node_value); + if (node_value->isa()) { + AssignValueNodeTensor(value_node, node_value, 0); + } else if (node_value->isa()) { + auto value_tuple = node_value->cast(); + if (value_tuple == nullptr) { + MS_LOG(WARNING) << "value_tuple is null"; + continue; + } + size_t i = 0; + auto value_list = value_tuple->value(); + for (auto value_ptr : value_list) { + if (value_ptr->isa()) { + AssignValueNodeTensor(value_node, value_ptr, i++); + } + } + } else if (node_value->isa()) { + auto value = GetValue(node_value); + size_t tensor_size = value.size(); + auto ptr = MallocStaticMem(tensor_size, false); + auto address = CreateDeviceAddress(ptr, tensor_size, kOpFormat_DEFAULT, kNumberTypeUInt8); + MS_EXCEPTION_IF_NULL(address); + AnfAlgo::SetOutputAddr(address, 0, value_node.get()); + std::vector shape = {1, SizeToInt(tensor_size)}; + if (!address->SyncHostToDevice(shape, tensor_size, kNumberTypeUInt8, value.data())) { + MS_LOG(EXCEPTION) << "kValueNode SyncHostToDevice fail!"; + } + } + } +} + +void KernelRuntime::AssignDynamicMemory(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + // reset dynamic mem offset + dynamic_mem_offset_ = 0; + auto &kernels = graph->execution_order(); + for (auto &kernel : kernels) { + AssignNodeOutputMem(kDynamicMem, kernel, kGetAllOuts); + AssignWorkSpaceMem(kernel); + } +} + +void KernelRuntime::ReuseAssignDynamicMemory(session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + dynamic_mem_offset_ = 0; + MemReuseUtilPtr mem_reuse_util_ptr = std::make_shared(); + MS_EXCEPTION_IF_NULL(mem_reuse_util_ptr); + // set all infos + mem_reuse_util_ptr->SetAllInfo(graph); + auto bestfit_mem_reuse = std::make_shared(); + MS_EXCEPTION_IF_NULL(bestfit_mem_reuse); + bestfit_mem_reuse->Reuse(mem_reuse_util_ptr.get()); + size_t total_allocated_size = bestfit_mem_reuse->GetAllocatedSize(); + MS_LOG(INFO) << "TotalReuseDynamicSize [" << total_allocated_size << "]"; + auto base_ptr = MallocDynamicMem(total_allocated_size, false); + reuse_mem_base_ = base_ptr; + mem_reuse_util_ptr_ = mem_reuse_util_ptr; + auto &kernels = graph->execution_order(); + for (auto &kernel : kernels) { + AssignNodeOutputMem(kReuseDynamicMem, kernel, kGetAllOuts); + AssignReuseWorkSpaceMem(kernel); + } +} + +void KernelRuntime::AssignReuseWorkSpaceMem(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto key = node.get(); + auto kernel_mod = AnfAlgo::GetKernelMod(node); + MS_EXCEPTION_IF_NULL(kernel_mod); + size_t index = 0; + auto iter = mem_reuse_util_ptr_->kernel_workspace_refs_.find(key); + for (auto &size : kernel_mod->GetWorkspaceSizeList()) { + if (iter != mem_reuse_util_ptr_->kernel_workspace_refs_.end()) { + if (index >= iter->second.size()) { + MS_LOG(EXCEPTION) << "index:[" << index << "] is larger than it's workspace size:[" << iter->second.size() + << "]"; + } + auto wk_ref = iter->second[index]; + auto wk_ptr = reuse_mem_base_ + wk_ref->offset_; + AnfAlgo::SetWorkspaceAddr(CreateDeviceAddress(wk_ptr, size, "", kTypeUnknown), index, node.get()); + index++; + } + } +} + +void KernelRuntime::AssignWorkSpaceMem(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + auto kernel_mod = AnfAlgo::GetKernelMod(node); + MS_EXCEPTION_IF_NULL(kernel_mod); + size_t index = 0; + for (auto &size : kernel_mod->GetWorkspaceSizeList()) { + auto ptr = MallocDynamicMem(size, false); + AnfAlgo::SetWorkspaceAddr(CreateDeviceAddress(ptr, size, "", kTypeUnknown), index, node.get()); + index++; + } + } +} + +bool KernelRuntime::IsCommunicationOp(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_name = AnfAlgo::GetCNodeName(node); + auto kernel_type = AnfAlgo::GetKernelType(node); + if (kernel_name == kAllReduceOpName || kernel_type == HCCL_KERNEL) { + return true; + } + return false; +} + +uint8_t *KernelRuntime::CalDeviceMem(const AnfNodePtr &node, size_t size, int flag, size_t index) { + MS_EXCEPTION_IF_NULL(node); + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + uint8_t *ptr = nullptr; + if (IsCommunicationOp(node)) { + bool communication_mem = false; + if (context_ptr->enable_hccl()) { + communication_mem = true; + } + if (flag == kStaticMem) { + ptr = MallocStaticMem(size, communication_mem); + } else { + ptr = MallocDynamicMem(size, communication_mem); + } + return ptr; + } + + if (flag == kStaticMem) { + ptr = MallocStaticMem(size, false); + } else if (flag == kDynamicMem) { + ptr = MallocDynamicMem(size, false); + } else if (flag == kReuseDynamicMem) { + auto key = node.get(); + auto iter = mem_reuse_util_ptr_->kernel_output_refs_.find(key); + if (iter != mem_reuse_util_ptr_->kernel_output_refs_.end()) { + // private member form KernelRuntime + memreuse::KernelRefCountPtr kernel_ref_count_ptr = mem_reuse_util_ptr_->kernel_output_refs_[key][index]; + if (kernel_ref_count_ptr == nullptr) { + return ptr; + } + ptr = reuse_mem_base_ + kernel_ref_count_ptr->offset_; + } else { + MS_LOG(EXCEPTION) << "node [" << AnfAlgo::GetCNodeName(node) << "] don't exist in kernel_output_refs"; + } + } + return ptr; +} + +void KernelRuntime::GenLaunchArgs(const mindspore::kernel::KernelMod &kernel_mod, const mindspore::AnfNodePtr &kernel, + AddressPtrList *kernel_inputs, AddressPtrList *const kernel_workspaces, + AddressPtrList *kernel_outputs) { + MS_EXCEPTION_IF_NULL(kernel); + MS_EXCEPTION_IF_NULL(kernel_inputs); + MS_EXCEPTION_IF_NULL(kernel_workspaces); + MS_EXCEPTION_IF_NULL(kernel_outputs); + auto cnode = kernel->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (AnfAlgo::GetCNodeName(cnode) == kAtomicAddrCleanOpName) { + return GenAddrCleanLaunchArgs(cnode, kernel_inputs); + } + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(kernel); ++i) { + auto real_input = AnfAlgo::GetRealInputIndex(kernel, i); + auto device_address = AnfAlgo::GetPrevNodeOutputAddr(kernel, real_input); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(input->addr); + input->size = device_address->size_; + kernel_inputs->emplace_back(input); + } + + for (size_t i = 0; i < kernel_mod.GetOutputSizeList().size(); ++i) { + auto device_address = AnfAlgo::GetOutputAddr(kernel, i); + kernel::AddressPtr output = std::make_shared(); + MS_EXCEPTION_IF_NULL(output); + output->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(output->addr); + output->size = device_address->size_; + kernel_outputs->emplace_back(output); + } + + for (size_t i = 0; i < kernel_mod.GetWorkspaceSizeList().size(); ++i) { + auto device_address = AnfAlgo::GetWorkspaceAddr(kernel, i); + kernel::AddressPtr workspace = std::make_shared(); + MS_EXCEPTION_IF_NULL(workspace); + workspace->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(workspace->addr); + workspace->size = device_address->size_; + kernel_workspaces->emplace_back(workspace); + } +} + +void KernelRuntime::GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList *kernel_inputs) { + if (cnode->inputs().size() != 2) { + MS_LOG(EXCEPTION) << "atomic Addr clean Node Input nodes not equal 2."; + } + auto pre_node = cnode->inputs()[1]; + // set clean output address + if (AnfAlgo::HasNodeAttr(kAttrAutomicOutputIndexs, pre_node)) { + auto clean_output_indexs = AnfAlgo::GetNodeAttr>(pre_node, kAttrAutomicOutputIndexs); + for (auto index : clean_output_indexs) { + auto device_address = AnfAlgo::GetOutputAddr(pre_node, index); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(input->addr); + input->size = device_address->size_; + kernel_inputs->emplace_back(input); + } + MS_LOG(INFO) << "AtomicAddClean clean output size:" << clean_output_indexs.size(); + } + // set clean workspace address + if (AnfAlgo::HasNodeAttr(kAttrAutomicWorkspaceSize, pre_node)) { + auto clean_workspaces = AnfAlgo::GetNodeAttr(pre_node, kAttrAutomicWorkspaceSize); + if (clean_workspaces != 0) { + auto device_address = AnfAlgo::GetWorkspaceAddr(pre_node, 0); + kernel::AddressPtr workspace = std::make_shared(); + MS_EXCEPTION_IF_NULL(workspace); + workspace->addr = device_address->ptr_; + MS_EXCEPTION_IF_NULL(workspace->addr); + workspace->size = device_address->size_; + kernel_inputs->emplace_back(workspace); + } + MS_LOG(INFO) << "AtomicAddClean clean workspace size" << clean_workspaces; + } +} + +bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) { + auto &kernels = graph.execution_order(); + for (const auto &kernel : kernels) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + + AddressPtrList kernel_inputs; + AddressPtrList kernel_workspaces; + AddressPtrList kernel_outputs; + GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + auto ret = + kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast(stream_)); + if (!ret) { + MS_LOG(ERROR) << "Launch kernel failed."; + return false; + } else { + if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) { + MS_LOG(EXCEPTION) << "SyncStream failed."; + } + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us"; + } + } + return true; +} + +size_t KernelRuntime::GetCommonAlignSize(size_t input_size) const { + return (input_size + mem_align_size_ + 31) / mem_align_size_ * mem_align_size_; +} + +size_t KernelRuntime::GetCommunicationAlignSize(size_t input_size) const { + return (input_size + mem_align_size_ - 1) / mem_align_size_ * mem_align_size_ + 2 * mem_align_size_; +} + +uint8_t *KernelRuntime::MallocStaticMem(size_t size, bool communication_mem) { + size_t align_size = 0; + if (communication_mem) { + align_size = GetCommunicationAlignSize(size); + } else { + align_size = GetCommonAlignSize(size); + } + if (static_mem_offset_ < align_size) { + MS_LOG(EXCEPTION) << "Out of memory!!! total[" << device_mem_size_ << "](dynamic[" << total_dynamic_size_ + << "] static[" << total_static_size_ << "])" + << " malloc [" << align_size << "] failed!"; + } + total_static_size_ += align_size; + auto offset = static_mem_offset_ - align_size; + if (dynamic_mem_offset_ > offset) { + MS_LOG(EXCEPTION) << "Out of memory!!! total[" << device_mem_size_ << "](dynamic[" << total_dynamic_size_ + << "] static[" << total_static_size_ << "])" + << " malloc [" << align_size << "] failed!"; + } + static_mem_offset_ = offset; + if (communication_mem) { + return device_mem_base_ + offset + mem_align_size_; + } else { + return device_mem_base_ + offset; + } +} + +uint8_t *KernelRuntime::MallocDynamicMem(size_t size, bool communication_mem) { + size_t align_size = 0; + if (communication_mem) { + align_size = GetCommunicationAlignSize(size); + } else { + align_size = GetCommonAlignSize(size); + } + uint64_t offset = dynamic_mem_offset_; + auto new_offset = dynamic_mem_offset_ + align_size; + if (new_offset > static_mem_offset_) { + MS_LOG(EXCEPTION) << "Out of memory!!! total[" << device_mem_size_ << "](dynamic[" << total_dynamic_size_ + << "] static[" << total_static_size_ << "])" + << " malloc [" << align_size << "] failed!"; + } + total_dynamic_size_ += align_size; + dynamic_mem_offset_ = new_offset; + + if (communication_mem) { + return device_mem_base_ + offset + mem_align_size_; + } else { + return device_mem_base_ + offset; + } +} + +bool KernelRuntime::LaunchKernel(const session::KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + if (!LaunchKernelMod(*graph)) { + MS_LOG(ERROR) << "LaunchKernelMod failed."; + return false; + } + if (!SyncStream()) { + MS_LOG(ERROR) << "SyncStream failed."; + return false; + } + return true; +} + +void KernelRuntime::MallocOpMemory(const DeviceAddressPtr address, size_t size, int flag) { + if (flag == kStaticMem) { + address->ptr_ = MallocStaticMem(size, false); + } else if (flag == kDynamicMem) { + address->ptr_ = MallocDynamicMem(size, false); + } else { + MS_LOG(EXCEPTION) << "Unknown memory type!"; + } +} + +void *KernelRuntime::AllocTensorMemDynamic(size_t size) { + if (size == 0) { + MS_LOG(ERROR) << "AllocTensorMemDynamic size is 0."; + } + return nullptr; +} + +void KernelRuntime::FreeTensorMemDynamic(void *device_ptr) { + if (device_ptr == nullptr) { + MS_LOG(ERROR) << "FreeTensorMemDynamic device_ptr is null."; + } +} + +#ifdef ENABLE_DUMP_E2E +bool KernelRuntime::SetDumpConf() { + dump_conf_ptr_ = std::make_shared(); + MS_EXCEPTION_IF_NULL(dump_conf_ptr_); + bool ret = dump_conf_ptr_->SetDumpConfFromJsonFile(); + return ret; +} + +DumpConfPtr KernelRuntime::GetDumpConf() { return dump_conf_ptr_; } +#endif +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/kernel_runtime.h b/mindspore/ccsrc/device/kernel_runtime.h new file mode 100644 index 00000000000..afdb45a6989 --- /dev/null +++ b/mindspore/ccsrc/device/kernel_runtime.h @@ -0,0 +1,139 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_KERNEL_RUNTIME_H_ +#define MINDSPORE_CCSRC_DEVICE_KERNEL_RUNTIME_H_ +#include +#include +#include +#include +#include "pre_activate/mem_reuse/mem_reuse.h" +#include "pre_activate/mem_reuse/mem_reuse_allocator.h" +#include "device/device_address.h" +#include "ir/meta_tensor.h" +#include "predict/generator/utils/ir_model_util.h" +#ifdef ENABLE_DUMP_E2E +#include "debug/e2e_dump.h" +#endif +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/kernel.h" +#include "utils/context/ms_context.h" + +// using mindspore::session::KernelGraph; +using mindspore::tensor::Tensor; +using TensorPtr = std::shared_ptr; +using MemReuseUtilPtr = mindspore::memreuse::MemReuseUtilPtr; +using mindspore::kernel::AddressPtr; +using AddressPtrList = std::vector; + +namespace mindspore { +namespace device { +const int kStaticMem = 0; +const int kDynamicMem = 1; +const int kReuseDynamicMem = 2; +const int kGetAllOuts = -1; + +class KernelRuntime { + public: + KernelRuntime() = default; + virtual ~KernelRuntime(); + virtual bool Init() = 0; + virtual void AssignMemory(session::KernelGraph *graph); + void RunOpAssignMemory(const std::vector &input_tensors, const session::KernelGraph *graph); + virtual bool Run(session::KernelGraph *graph); + virtual bool DumpData(session::KernelGraph *graph); + virtual bool RunTask(const session::KernelGraph *graph); + virtual bool GenTask(const session::KernelGraph *graph); + bool LaunchKernel(const session::KernelGraph *graph); + virtual void AssignStaticMemoryInput(const session::KernelGraph *graph); + +#ifdef ENABLE_DUMP_E2E + DumpConfPtr GetDumpConf(); +#endif + virtual bool LoadTask(const session::KernelGraph *graph); + virtual void FreeHostMemory(); + // for GPU and D to impl + virtual void ReleaseDeviceRes() {} + void set_device_id(uint32_t device_id) { device_id_ = device_id; } + + protected: + virtual DeviceAddressPtr CreateDeviceAddress(void *device_ptr, size_t device_size, const string &format, + TypeId type_id) = 0; + virtual bool SyncStream() = 0; + void AssignStaticMemory(session::KernelGraph *graph); + void AssignDynamicMemory(const session::KernelGraph *graph); + void ReuseAssignDynamicMemory(session::KernelGraph *graph); + void AssignNodeOutputMem(int flag, const AnfNodePtr &node, int index); + void AssignWorkSpaceMem(const AnfNodePtr &node); + void AssignReuseWorkSpaceMem(const AnfNodePtr &node); + void AssignCommunicationNodeOutputMem(int flag, const AnfNodePtr &node); + void UpdateRefNodeOutputMem(const session::KernelGraph *graph); + void UpdateCommunicationOpInputMem(const AnfNodePtr &node); + bool IsCommunicationOp(const AnfNodePtr &node); + size_t GetCommonAlignSize(size_t input_size) const; + size_t GetCommunicationAlignSize(size_t input_size) const; + + uint8_t *CalDeviceMem(const AnfNodePtr &node, size_t size, int flag, size_t index); + virtual uint8_t *MallocStaticMem(size_t size, bool communication_mem); + uint8_t *MallocDynamicMem(size_t size, bool communication_mem); +#ifdef ENABLE_DUMP_E2E + bool SetDumpConf(); +#endif + // Alloc memory use the dynamic memory pool. + virtual void *AllocTensorMemDynamic(size_t size); + // Free memory use the dynamic memory pool. + virtual void FreeTensorMemDynamic(void *device_ptr); + virtual void MallocOpMemory(const DeviceAddressPtr address, size_t size, int flag); + + private: + void AssignStaticMemoryOutput(const session::KernelGraph *graph); + void AssignStaticMemoryValueNode(session::KernelGraph *graph); + void GenLaunchArgs(const mindspore::kernel::KernelMod &kernel_mod, const AnfNodePtr &kernel, + AddressPtrList *kernel_inputs, AddressPtrList *kernel_workspaces, AddressPtrList *kernel_outputs); + bool LaunchKernelMod(const session::KernelGraph &graph); + void GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList *kernel_inputs); + size_t CountNodeDeviceMemorySize(const AnfNodePtr &node, size_t output_index); + void RunOpAssignInputMemory(const std::vector &input_tensors, const session::KernelGraph *graph); + void RunOpAssignOutputMemory(const AnfNodePtr &kernel); + void RunOpAssignWorkSpaceMemory(const AnfNodePtr &kernel); + void AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, size_t output_idx); + + protected: + uint32_t device_id_{0}; + uint8_t *device_mem_base_{nullptr}; + uint8_t *device_mem_pool_base_{nullptr}; + uint64_t device_mem_size_{0}; + uint64_t device_mem_pool_size_{0}; + uint64_t dynamic_mem_offset_{0}; + uint64_t static_mem_offset_{0}; + const uint64_t mem_align_size_ = 512; +#ifdef ENABLE_DUMP_E2E + DumpConfPtr dump_conf_ptr_; +#endif + void *stream_ = nullptr; + size_t total_static_size_ = 0; + size_t total_dynamic_size_ = 0; + MemReuseUtilPtr mem_reuse_util_ptr_{nullptr}; + + private: + uint8_t *reuse_mem_base_{nullptr}; +}; +using KernelRuntimePtr = std::shared_ptr; +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_KERNEL_RUNTIME_H_ diff --git a/mindspore/ccsrc/device/kernel_runtime_manager.cc b/mindspore/ccsrc/device/kernel_runtime_manager.cc new file mode 100644 index 00000000000..6303bc88a8e --- /dev/null +++ b/mindspore/ccsrc/device/kernel_runtime_manager.cc @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "device/kernel_runtime_manager.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +void KernelRuntimeManager::ClearRuntimeResource() { + std::lock_guard guard(lock_); + for (auto &iter : runtime_map_) { + MS_LOG(INFO) << "Release device " << iter.first; + MS_EXCEPTION_IF_NULL(iter.second); + iter.second->ReleaseDeviceRes(); + } + runtime_map_.clear(); +} + +void KernelRuntimeManager::Register(const std::string &device_name, KernelRuntimeCreator &&runtime_creator) { + if (runtime_creators_.find(device_name) == runtime_creators_.end()) { + (void)runtime_creators_.emplace(device_name, runtime_creator); + } +} + +KernelRuntime *KernelRuntimeManager::GetSingleKernelRuntime(const std::string &device_name, uint32_t device_id) { + std::string runtime_key = device_name + "_" + std::to_string(device_id); + auto runtime_iter = runtime_map_.find(runtime_key); + if (runtime_iter != runtime_map_.end()) { + return runtime_iter->second.get(); + } else if (runtime_map_.size() > 0) { + auto cur_runtime_key = runtime_map_.begin()->first; + if (!cur_runtime_key.empty()) { + auto cur_device_id = cur_runtime_key.substr(cur_runtime_key.rfind('_') + 1); + MS_LOG(EXCEPTION) << "Can't change device id in runtime, already set device id: " << cur_device_id + << ", set device id: " << device_id << " failed"; + } + } + return GetKernelRuntime(device_name, device_id); +} + +KernelRuntime *KernelRuntimeManager::GetKernelRuntime(const std::string &device_name, uint32_t device_id) { + std::lock_guard guard(lock_); + std::string runtime_key = device_name + "_" + std::to_string(device_id); + auto runtime_iter = runtime_map_.find(runtime_key); + if (runtime_iter != runtime_map_.end()) { + return runtime_iter->second.get(); + } + std::shared_ptr kernel_runtime; + auto creator_iter = runtime_creators_.find(device_name); + if (creator_iter != runtime_creators_.end()) { + MS_EXCEPTION_IF_NULL(creator_iter->second); + kernel_runtime = (creator_iter->second)(); + kernel_runtime->set_device_id(device_id); + MS_EXCEPTION_IF_NULL(kernel_runtime); + runtime_map_[runtime_key] = kernel_runtime; + } else { + MS_LOG(EXCEPTION) << "no kernel runtime creator for " << device_name << " with device id " << device_id; + } + + return kernel_runtime.get(); +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/kernel_runtime_manager.h b/mindspore/ccsrc/device/kernel_runtime_manager.h new file mode 100644 index 00000000000..f8b292bd60b --- /dev/null +++ b/mindspore/ccsrc/device/kernel_runtime_manager.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_DEVICE_KERNEL_RUNTIME_MANAGER_H_ +#define MINDSPORE_MINDSPORE_CCSRC_DEVICE_KERNEL_RUNTIME_MANAGER_H_ +#include +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "device/kernel_runtime.h" +namespace mindspore { +namespace device { +using KernelRuntimeCreator = std::function()>; + +class KernelRuntimeManager { + public: + static KernelRuntimeManager &Instance() { + static KernelRuntimeManager instance; + return instance; + } + void Register(const std::string &device_name, KernelRuntimeCreator &&runtime_creator); + KernelRuntime *GetKernelRuntime(const std::string &device_name, uint32_t device_id); + KernelRuntime *GetSingleKernelRuntime(const std::string &device_name, uint32_t device_id); + void ClearRuntimeResource(); + + private: + KernelRuntimeManager() = default; + ~KernelRuntimeManager() = default; + DISABLE_COPY_AND_ASSIGN(KernelRuntimeManager); + std::map > runtime_map_; + std::map runtime_creators_; + std::mutex lock_; +}; + +class KernelRuntimeRegistrar { + public: + KernelRuntimeRegistrar(const std::string &device_name, KernelRuntimeCreator &&runtime_creator) { + KernelRuntimeManager::Instance().Register(device_name, std::move(runtime_creator)); + } + ~KernelRuntimeRegistrar() = default; +}; + +#define MS_REG_KERNEL_RUNTIME(DEVICE_NAME, RUNTIME_CLASS) \ + static const KernelRuntimeRegistrar g_kernel_runtime_##DEVICE_NAME##_reg( \ + DEVICE_NAME, []() { return std::make_shared(); }); +} // namespace device +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_DEVICE_KERNEL_RUNTIME_MANAGER_H_ diff --git a/mindspore/ccsrc/gvar/typeid_manager.cc b/mindspore/ccsrc/gvar/typeid_manager.cc new file mode 100644 index 00000000000..97250a65710 --- /dev/null +++ b/mindspore/ccsrc/gvar/typeid_manager.cc @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "ir/base.h" + +namespace mindspore { + +struct TypeIdManager* TypeIdManager::Get() { + static TypeIdManager manager; + return &manager; +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/CMakeLists.txt b/mindspore/ccsrc/ir/CMakeLists.txt new file mode 100644 index 00000000000..278ad492e2c --- /dev/null +++ b/mindspore/ccsrc/ir/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _IR_ALL_SRC_FILES + ./*.cc + dtype/*.cc) + +add_library(_mindspore_ir_obj OBJECT ${_IR_ALL_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/ir/anf.cc b/mindspore/ccsrc/ir/anf.cc new file mode 100644 index 00000000000..c1348bf7d74 --- /dev/null +++ b/mindspore/ccsrc/ir/anf.cc @@ -0,0 +1,224 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/anf.h" + +#include +#include +#include +#include + +#include "ir/visitor.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "operator/ops.h" +#include "parallel/ops_info/ops_utils.h" + +namespace mindspore { +// namespace to support intermediate representation definition +// Methods of AnfNode +TypePtr AnfNode::Type() const { return (abstract_ == nullptr) ? nullptr : abstract_->BuildType(); } +BaseShapePtr AnfNode::Shape() const { return (abstract_ == nullptr) ? nullptr : abstract_->BuildShape(); } + +std::string AnfNode::ToString() const { + return mindspore::label_manage::Label(const_cast(this)->shared_from_base()->debug_info()); +} + +CNode::CNode(const std::vector& inputs, const FuncGraphPtr& func_graph) + : AnfNode(func_graph), inputs_(inputs), stop_gradient_(false) {} + +// Check if CNode is an apply with the specific Primitive. +bool CNode::IsApply(const PrimitivePtr& value) const { + if (value == nullptr) { + return false; + } + + if (inputs_.size() != 0 && IsValueNode(inputs_[0])) { + PrimitivePtr fn_value = GetValueNode(inputs_[0]); + if (fn_value->Hash() == value->Hash() && fn_value->name() == value->name()) { + return true; + } + } + + return false; +} + +void CNode::set_input(size_t i, const AnfNodePtr& new_input) { inputs_[i] = new_input; } + +std::string CNode::DebugString(int recursive_level) const { + std::ostringstream buffer; + if (recursive_level > 0) { + if (func_graph() != nullptr) { + buffer << func_graph()->ToString() << ":"; + } + buffer << ToString() << "{"; + bool is_first_node = true; + int idx = 0; + for (auto& node : inputs_) { + MS_EXCEPTION_IF_NULL(node); + if (is_first_node) { + is_first_node = false; + } else { + buffer << ", "; + } + buffer << "[" << idx << "]: " << node->DebugString(recursive_level - 1); + idx++; + } + buffer << "}"; + } else { + buffer << ToString(); + } + return buffer.str(); +} + +OperatorInfoPtr CNode::set_operator_info(const OperatorInfoPtr& operator_info) { + if (operator_info_ != nullptr) { + MS_LOG(WARNING) << "The CNode: " << ToString() << " has already been set OperatorInfo: " << operator_info_->name() + << ", using the new one: " << operator_info->name(); + auto old_ptr = operator_info_; + operator_info_ = operator_info; + return old_ptr; + } + operator_info_ = operator_info; + return nullptr; +} + +std::string CNode::fullname_with_scope() { + // if full name is set, return its name immediately + if (!fullname_with_scope_.empty()) { + return fullname_with_scope_; + } + + if (IsApply(prim::kPrimScalarSummary) || IsApply(prim::kPrimTensorSummary) || IsApply(prim::kPrimImageSummary)) { + std::string tag = GetValue(GetValueNode(input(1))); + if (tag == "") { + MS_LOG(EXCEPTION) << "The tag name is null, should be valid string"; + } + std::string name; + if (IsApply(prim::kPrimScalarSummary)) { + name = tag + "[:Scalar]"; + } else if (IsApply(prim::kPrimTensorSummary)) { + name = tag + "[:Tensor]"; + } else { + name = tag + "[:Image]"; + } + fullname_with_scope_ = name; + } else { + // cnode input 0 should be primitive ptr + auto value_ptr = input(0)->cast(); + if (value_ptr == nullptr) { + MS_LOG(WARNING) << "Input 0 of cnode is not a value node, its type is " << input(0)->type_name() << "."; + fullname_with_scope_ = id_generator::get_id(shared_from_base()); + return fullname_with_scope_; + } + auto input_value = value_ptr->value(); + if (input_value == nullptr) { + MS_LOG(WARNING) << "Value of input 0 of cnode is nullptr."; + fullname_with_scope_ = id_generator::get_id(shared_from_base()); + return fullname_with_scope_; + } + + PrimitivePtr prim = GetValue(input_value); + MS_EXCEPTION_IF_NULL(scope()); + MS_EXCEPTION_IF_NULL(prim); + fullname_with_scope_ = + scope()->name() + "/" + prim->name() + "-op" + id_generator::get_id(shared_from_base()); + } + + return fullname_with_scope_; +} + +std::string ValueNode::ToString() const { + MS_EXCEPTION_IF_NULL(value_); + if (value_->isa()) { + return value_->cast()->ToString(); + } + std::ostringstream buffer; + buffer << AnfNode::ToString(); + buffer << "(" << value_->ToString() << ")"; + return buffer.str(); +} + +std::string ValueNode::DebugString(int) const { + MS_EXCEPTION_IF_NULL(value_); + std::ostringstream buffer; + buffer << "ValueNode<" << value_->type_name() << "> " << value_->ToString(); + return buffer.str(); +} + +std::string ValueNode::fullname_with_scope() { + if (!fullname_with_scope_.empty()) { + return fullname_with_scope_; + } + + MS_EXCEPTION_IF_NULL(scope()); + fullname_with_scope_ = scope()->name() + "/" + "data-" + id_generator::get_id(shared_from_base()); + return fullname_with_scope_; +} + +void CNode::accept(AnfVisitor* v) { v->Visit(shared_from_base()); } +void ValueNode::accept(AnfVisitor* v) { v->Visit(shared_from_base()); } +void Parameter::accept(AnfVisitor* v) { v->Visit(shared_from_base()); } + +bool IsPrimitiveCNode(const AnfNodePtr& node, const PrimitivePtr& value) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + if (cnode != nullptr) { + return cnode->IsApply(value); + } + return false; +} + +PrimitivePtr GetCNodePrimitive(const AnfNodePtr& node) { + if (node == nullptr) { + return nullptr; + } + auto cnode = node->cast(); + if (cnode != nullptr) { + if (cnode->size() > 0) { + auto prim = GetValueNode(cnode->input(0)); + return prim; + } + } + return nullptr; +} + +bool IsPrimitive(const AnfNodePtr& node, const PrimitivePtr& value) { + if (IsValueNode(node)) { + PrimitivePtr fn_value = GetValueNode(node); + MS_EXCEPTION_IF_NULL(value); + if (fn_value->Hash() == value->Hash() && fn_value->name() == value->name()) { + return true; + } + } + return false; +} +namespace id_generator { +static std::unordered_map node_ids; +std::string get_id(const AnfNodePtr& node) { + auto type_name = node->type_name(); + if (node_ids.find(type_name) == node_ids.end()) { + node_ids[type_name] = 0; + } else { + node_ids[type_name]++; + } + return std::to_string(node_ids[type_name]); +} + +void reset_id() { node_ids.clear(); } +} // namespace id_generator +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/anf.h b/mindspore/ccsrc/ir/anf.h new file mode 100644 index 00000000000..9050a4ed16a --- /dev/null +++ b/mindspore/ccsrc/ir/anf.h @@ -0,0 +1,439 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_ANF_H_ +#define MINDSPORE_CCSRC_IR_ANF_H_ + +#include +#include +#include +#include +#include +#include + +#include "ir/base.h" +#include "debug/info.h" +#include "ir/scope.h" + +// A MindSpore ANF IR defined here. +// with BNF followed: +// ::= Scalar | Named | Tensor | Var | +// Prim | MetaFuncGraph | FuncGraph | Type| +// Shape | Param +// ::= ( ...) +// ::= | +// ANode: Atomic Node +// CNode: Complex Node +namespace mindspore { +namespace parallel { +class TensorLayout; +class OperatorInfo; +} // namespace parallel +using OperatorInfoPtr = std::shared_ptr; + +namespace abstract { +class BaseShape; +class AbstractBase; +} // namespace abstract +using BaseShapePtr = std::shared_ptr; +using AbstractBasePtr = std::shared_ptr; + +class ValueNode; +using ValueNodePtr = std::shared_ptr; +class CNode; +using CNodePtr = std::shared_ptr; + +class FuncGraph; +using FuncGraphSet = OrderedSet; +using FuncGraphPtrList = std::vector; + +class Primitive; +using PrimitivePtr = std::shared_ptr; + +class BaseRef; + +class Var; +using VarPtr = std::shared_ptr; + +namespace device { +class KernelInfo; +} // namespace device +using KernelInfoDevice = device::KernelInfo; +using KernelInfoDevicePtr = std::shared_ptr; + +class AnfVisitor; + +// AnfNode is the basic class of the IR definition derived from Base. +// Only two types of nodes are derived: CNode and ANode. +// Methods: +// func_graph: return FuncGraph that this AnfNode belongs to. +// scope: return the scope namespace of this AnfNode. Set it using set_scope. +// abstract: return the cached inferred abstract value. It cantains type, shape +// value. Set New cache using set_abstract. +// intermediate_abstract: return the cached inferring abstract value. +// Type/Shape: return the related info of this AnfNode. When this AnfNode is an +// input of other CNodes, you can get the related info by this method. +// debug_info: return the information retrived from parser. Set it using set_debug_info. +// fullname_with_scope: return the detailed debug info. +class AnfNode : public Base { + public: + explicit AnfNode(const FuncGraphPtr &func_graph) + : func_graph_(FuncGraphWeakPtr(func_graph)), + abstract_(nullptr), + intermediate_abstract_(nullptr), + debug_info_(std::make_shared()), + fullname_with_scope_(""), + hash_(std::hash()), + kernel_info_(nullptr) { + scope_ = ScopeManager::GetInstance().GetCurrentScope(); + } + + ~AnfNode() override = default; + MS_DECLARE_PARENT(AnfNode, Base); + + virtual void accept(AnfVisitor *) {} + FuncGraphPtr func_graph() const { return func_graph_.lock(); } + + void set_func_graph(const FuncGraphPtr &func_graph) { func_graph_ = FuncGraphWeakPtr(func_graph); } + + ScopePtr scope() { return scope_; } + void set_scope(const ScopePtr &scope) { scope_ = scope; } + + const KernelInfoDevice *kernel_info() const { return kernel_info_.get(); } + KernelInfoDevice *kernel_info() { return kernel_info_.get(); } + void set_kernel_info(const KernelInfoDevicePtr &kernel_info) { kernel_info_ = kernel_info; } + + AbstractBasePtr abstract() const { return abstract_; } + void set_abstract(const AbstractBasePtr &abs) { abstract_ = abs; } + + AbstractBasePtr intermediate_abstract() { return intermediate_abstract_; } + void set_intermediate_abstract(const AbstractBasePtr &abs) { intermediate_abstract_ = abs; } + + NodeDebugInfoPtr debug_info() { + MS_EXCEPTION_IF_NULL(debug_info_); + if (debug_info_->get_node() == nullptr) { + debug_info_->set_node(shared_from_base()); + } + return debug_info_; + } + void set_debug_info(const NodeDebugInfoPtr &debug_info) { + debug_info_ = debug_info; + if (debug_info_->get_node() == nullptr) { + debug_info_->set_node(shared_from_base()); + } + } + + TypePtr Type() const; + BaseShapePtr Shape() const; + + std::size_t hash() const override { return this->hash_(this); } + virtual std::string fullname_with_scope() { return ""; } + + virtual std::string DebugString(int recursive_level = 1) const { return ToString(); } + virtual std::string DebugString(bool recursive) const { return DebugString(recursive ? 1 : 0); } + std::string ToString() const override; + void dump() const override { std::cout << DebugString() << std::endl; } + std::string UniqueId() { return std::to_string(debug_info()->unique_id()); } + std::string UniqueIdThroughCopy() { return std::to_string(debug_info()->unique_id_through_copy()); } + virtual bool operator==(const AnfNode &other) const { return &other == this; } + friend std::ostream &operator<<(std::ostream &os, const AnfNode &node) { + os << node.ToString(); + return os; + } + + protected: + // Hold a weak ref to Graph as Graph also hold ref to AnfNode. + // Otherwise, func_graph_ and AnfNode will make a reference cycle. + FuncGraphWeakPtr func_graph_; + AbstractBasePtr abstract_; + AbstractBasePtr intermediate_abstract_; + NodeDebugInfoPtr debug_info_; + std::string fullname_with_scope_; + + private: + std::hash hash_; + ScopePtr scope_; + KernelInfoDevicePtr kernel_info_; +}; + +// CNode represents the complex node with a set of arguments. +// Fields: +// inputs_: represents all of the inputs for this CNode. +// Using input(i) to get the index i input. +// Using inputs() to get all the inputs as a vector. +// Using add_input(input) to append a new input for a CNode. +// Using set_input(i, input) to change some input of these inputs. +// Using set_inputs(inputs) to refresh all of the inputs of a CNode. +// func_graph_as_var_: used in opt pattern matching to match a real FuncGraph. +// stop_gradient_: a flag used to stop gradient. +// Using stop_gradient() to get this flag, mainly used in ad. +// Using set_stop_gradient() to set this flag. +class CNode : public AnfNode { + public: + CNode(const std::vector &inputs, const FuncGraphPtr &func_graph); + CNode(const std::vector &inputs, const VarPtr &func_graph_as_var) + : AnfNode(nullptr), inputs_(inputs), func_graph_as_var_(func_graph_as_var), stop_gradient_(false) {} + + ~CNode() override = default; + MS_DECLARE_PARENT(CNode, AnfNode); + + void accept(AnfVisitor *v) override; + // check whether this cnode has some primitive value as the first input. + bool IsApply(const PrimitivePtr &) const; + + const size_t size() const { return inputs_.size(); } + const AnfNodePtr input(size_t i) const { return inputs_[i]; } + const std::vector &inputs() const { return inputs_; } + void add_input(const AnfNodePtr &input) { inputs_.push_back(input); } + void set_input(size_t i, const AnfNodePtr &input); + void set_inputs(const std::vector &inputs) { inputs_ = inputs; } + + bool stop_gradient() const { return stop_gradient_; } + void set_stop_gradient(bool stop_gradient) { stop_gradient_ = stop_gradient; } + + std::string fullname_with_scope() override; + std::string DebugString(int recursive_level = 1) const override; + std::string DebugString(bool recursive) const override { return DebugString(recursive ? 1 : 0); } + + OperatorInfoPtr set_operator_info(const OperatorInfoPtr &operator_info); + OperatorInfoPtr operator_info() { return operator_info_; } + + void set_in_forward_flag(bool flag) { in_forward_flag_ = flag; } + bool in_forward_flag() const { return in_forward_flag_; } + + VarPtr func_graph_as_var() const { return func_graph_as_var_; } + + private: + std::vector inputs_; + VarPtr func_graph_as_var_; + bool stop_gradient_; + OperatorInfoPtr operator_info_ = nullptr; + bool in_forward_flag_ = false; +}; + +// ANode represents the atomic node. It's derived Parameter and ValueNode. +class ANode : public AnfNode { + public: + ANode() : AnfNode(nullptr) {} + explicit ANode(const FuncGraphPtr &func_graph) : AnfNode(func_graph) {} + virtual ~ANode() = default; + + MS_DECLARE_PARENT(ANode, AnfNode); +}; + +// Parameter represents the parameter inputs of a function. They have no value. +// Attributes: +// default_param_: used to hold the inputting tensor of the model. +class Parameter : public ANode { + public: + explicit Parameter(const FuncGraphPtr &func_graph) + : ANode(func_graph), name_(""), has_default_(false), default_param_(py::none()), tensor_layout_(nullptr) {} + ~Parameter() override = default; + MS_DECLARE_PARENT(Parameter, ANode); + + void accept(AnfVisitor *v) override; + + std::string name() const { return name_; } + void set_name(const std::string &name) { name_ = name; } + std::string fullname_with_scope() override { return name(); }; + + bool has_default() const { return has_default_; } + + py::object default_param() { return default_param_; } + void set_default_param(const py::object &obj) { + default_param_ = obj; + has_default_ = true; + } + + std::shared_ptr tensor_layout() const { return tensor_layout_; } + void set_tensor_layout(const std::shared_ptr &tensor_layout) { + tensor_layout_ = tensor_layout; + } + + bool operator==(const AnfNode &other) const override { + if (!other.isa()) { + return false; + } + auto p = static_cast(other); + if (name_.length() > 0 && p.name_.length() > 0) { + return p.name_ == name_; + } + return shared_from_this() == other.shared_from_this(); + } + + private: + std::string name_; + bool has_default_; + py::object default_param_; + std::shared_ptr tensor_layout_; +}; +using ParameterPtr = std::shared_ptr; + +// Value is used to represent the atomic expression metioned in BNF. +// It mainly be stored in ValueNode. Value and ValueNode is related definition. +class Value : public Base { + public: + Value() = default; + explicit Value(const TypePtr t) : type_(t) {} + Value(const Value &other) : Base(other) { this->type_ = other.type_; } + ~Value() override = default; + MS_DECLARE_PARENT(Value, Base) + + TypePtr type() const { return type_; } + virtual abstract::AbstractBasePtr ToAbstract() { MS_LOG(EXCEPTION) << "ToAbstract error"; } + + virtual bool operator==(const Value &rhs) const = 0; + virtual Value &operator=(const Value &other) { + if (&other == this) { + return *this; + } + this->type_ = other.type_; + return *this; + } + + protected: + TypePtr type_{nullptr}; +}; +using ValuePtr = std::shared_ptr; +using ValuePtrList = std::vector; + +// ValueNode is used to hold value. Unlike CNode and Parameter, ValueNode +// do not belong to any particular function graph. +class ValueNode : public ANode { + public: + explicit ValueNode(const ValuePtr &value) : value_(value) {} + ~ValueNode() override = default; + MS_DECLARE_PARENT(ValueNode, ANode); + + void accept(AnfVisitor *v) override; + const ValuePtr &value() const { return value_; } + std::string fullname_with_scope() override; + + std::string ToString() const override; + std::string DebugString(int recursive_level = 1) const override; + std::string DebugString(bool recursive) const override { return DebugString(recursive ? 1 : 0); } + + bool operator==(const AnfNode &other) const override { + if (!other.isa()) { + return false; + } + auto v = static_cast(other); + return *v.value() == *value(); + } + friend std::ostream &operator<<(std::ostream &os, const ValueNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + os << node->ToString(); + return os; + } + + private: + ValuePtr value_; +}; + +template +struct ImmTraits {}; + +#define IMM_TRAITS(typeimm, prototype) \ + template <> \ + struct ImmTraits { \ + using type = typeimm; \ + }; + +inline ValuePtr MakeValue(const ValuePtr &value) { return value; } + +template ::type::element_type> +inline ValuePtr MakeValue(S v) { + return std::make_shared(v); +} + +template ::type> +static S GetValue(const ValuePtr &value) { + MS_EXCEPTION_IF_NULL(value); + + U imm = value->cast(); + if (imm == nullptr) { + MS_LOG(EXCEPTION) << "Cast failed, original value: " << value->ToString() << ", type: " << value->type_name(); + } + return imm->value(); +} + +template ::value && std::is_base_of::value, + S>::type * = nullptr> +static S GetValue(const ValuePtr &value) { + MS_EXCEPTION_IF_NULL(value); + S v = value->cast(); + if (v == nullptr) { + MS_LOG(EXCEPTION) << "Cast failed, original value: " << value->ToString() << ", type: " << value->type_name(); + } + return v; +} + +// used to check whether an AnfNode is a cnode with a kind of Primitive as first input +bool IsPrimitiveCNode(const AnfNodePtr &node, const PrimitivePtr &value); + +// used to check whether an AnfNode is a cnode with a Primitive as first input +PrimitivePtr GetCNodePrimitive(const AnfNodePtr &node); + +// used to check whether an AnfNode is a valuenode having some Primitive value +bool IsPrimitive(const AnfNodePtr &node, const PrimitivePtr &value); + +// used to check whether a ValueNode has some kind of value +template +static bool IsValueNode(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto anode = node->cast(); + if (anode != nullptr) { + auto value = anode->value(); + if (value == nullptr) { + MS_LOG(EXCEPTION) << "Const value is nullptr."; + } + return value->isa(); + } + return false; +} + +inline ValuePtr GetValueNode(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return nullptr; + } + return node->cast()->value(); +} + +template ::value && std::is_base_of::value, + S>::type * = nullptr> +inline S GetValueNode(const AnfNodePtr &node) { + auto value = GetValueNode(node); + if (value == nullptr) { + return nullptr; + } + auto s = value->cast(); + return s; +} +namespace id_generator { +std::string get_id(const AnfNodePtr &node); +void reset_id(); +} // namespace id_generator +using TaggedNodeMap = std::unordered_map; +using TaggedGraph = std::pair; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_ANF_H_ diff --git a/mindspore/ccsrc/ir/base.cc b/mindspore/ccsrc/ir/base.cc new file mode 100644 index 00000000000..7a03269ad8f --- /dev/null +++ b/mindspore/ccsrc/ir/base.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/base.h" +#include +#include +#include + +namespace mindspore { +const bool Base::IsFromTypeId(uint32_t tid) const { + static const uint32_t node_id = GetTypeId(typeid(Base).name()); + return tid == node_id; +} + +uint32_t Base::GetTypeId(const char *const type_name) { + TypeIdManager *t = TypeIdManager::Get(); + std::lock_guard(t->mutex); + auto it = t->map.find(type_name); + if (it != t->map.end()) { + return it->second; + } + uint32_t tid = ++(t->type_counter); + t->map[type_name] = tid; + return tid; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/base.h b/mindspore/ccsrc/ir/base.h new file mode 100644 index 00000000000..6a3537306f6 --- /dev/null +++ b/mindspore/ccsrc/ir/base.h @@ -0,0 +1,151 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_BASE_H_ +#define MINDSPORE_CCSRC_IR_BASE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "utils/visible.h" +#include "utils/log_adapter.h" +#include "utils/ordered_set.h" + +namespace mindspore { +template +struct is_shared_ptr : public std::false_type {}; +template +struct is_shared_ptr> : public std::true_type {}; + +class Base : public std::enable_shared_from_this { + public: + constexpr Base() = default; + Base(const Base& other) : std::enable_shared_from_this(other) {} + virtual bool operator==(const Base& rhs) { + if (this == &rhs) { + return true; + } + return false; + } + + virtual Base& operator=(const Base&) { return *this; } + virtual ~Base() = default; + virtual std::size_t hash() const { return tid(); } + virtual std::string ToString() const { return type_name(); } + virtual void dump() const { std::cout << ToString() << std::endl; } + + virtual std::string DumpText() const { return ToString(); } + + virtual const bool IsFromTypeId(uint32_t tid) const; + virtual std::string type_name() const { return "Base"; } + static uint32_t GetTypeId(const char* const type_key); + virtual uint32_t tid() const { + static const uint32_t tid = GetTypeId(typeid(Base).name()); + return tid; + } + + template ::value && std::is_base_of::value, T>::type* = nullptr> + inline bool isa() const { + static const uint32_t tid = GetTypeId(typeid(T).name()); + return this->IsFromTypeId(tid); + } + + template ::value, typename T::element_type>::type> + inline T cast() { + if (isa()) { + return std::static_pointer_cast(shared_from_this()); + } else { + return nullptr; + } + } + + protected: + template + std::shared_ptr shared_from_base() { + return std::static_pointer_cast(shared_from_this()); + } +}; + +using BasePtr = std::shared_ptr; +using BaseWeakPtr = std::weak_ptr; + +template +inline T* cast(U* source) { + if (source != nullptr && source->template isa()) { + return static_cast(source); + } else { + return nullptr; + } +} + +template < + typename T, typename U, + typename std::enable_if::value && std::is_base_of::value, T>::type* = nullptr> +inline std::shared_ptr dyn_cast(const std::shared_ptr r) { + if (r != nullptr && r->template isa()) { + return std::static_pointer_cast(r); + } else { + return std::shared_ptr(); + } +} + +#define MS_DECLARE_PARENT(current_t, parent_t) \ + uint32_t tid() const override { \ + static const uint32_t tid = GetTypeId(typeid(current_t).name()); \ + return tid; \ + } \ + const bool IsFromTypeId(uint32_t from_tid) const override { \ + static const uint32_t tid = Base::GetTypeId(typeid(current_t).name()); \ + if (tid == from_tid) { \ + return true; \ + } \ + return parent_t::IsFromTypeId(from_tid); \ + } \ + std::string type_name() const override { return #current_t; } + +class Type; +using TypePtr = std::shared_ptr; + +class AnfNode; +using AnfNodePtr = std::shared_ptr; +using AnfNodePtrList = std::vector; +using AnfNodeSet = OrderedSet; + +namespace abstract { +class AbstractBase; +using AbstractBasePtr = std::shared_ptr; +using AbstractAttribute = std::pair; +class AnalysisContext; +using AnalysisContextPtr = std::shared_ptr; +} // namespace abstract + +struct MS_EXPORT TypeIdManager { + std::mutex mutex; + std::atomic type_counter{0}; + std::unordered_map map; + static TypeIdManager* Get(); + TypeIdManager() : mutex(), type_counter(0), map() {} +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_BASE_H_ diff --git a/mindspore/ccsrc/ir/dtype.cc b/mindspore/ccsrc/ir/dtype.cc new file mode 100644 index 00000000000..65a42bc3fa6 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype.cc @@ -0,0 +1,703 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/dtype.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +TypePtr Keyword::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + MS_EXCEPTION_IF_NULL(value_); + std::string key = key_; + return std::make_shared(key, value_->DeepCopy()); + } +} + +std::string Keyword::ToString() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Keyword"; + } else { + MS_EXCEPTION_IF_NULL(value_); + buffer << "Keyword["; + buffer << "key : " << key_; + buffer << "value : " << value_->ToString(); + buffer << "]"; + } + return buffer.str(); +} + +bool Keyword::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + const auto& other_keyword = static_cast(other); + return (other_keyword.key_ == key_ && *other_keyword.value_ == *value_); +} + +std::string Keyword::DumpText() const { return ToString(); } + +TypePtr Slice::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + auto copy = std::make_shared(start_->DeepCopy(), stop_->DeepCopy(), step_->DeepCopy()); + return copy; + } +} + +std::string Slice::ToString() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Slice"; + } else { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + buffer << "Slice["; + buffer << start_->ToString() << " : "; + buffer << stop_->ToString() << " : "; + buffer << step_->ToString(); + buffer << "]"; + } + return buffer.str(); +} + +bool Slice::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_slice = static_cast(other); + return (*start_ == *other_slice.start_ && *stop_ == *other_slice.stop_ && *step_ == *other_slice.step_); +} + +std::string Slice::DumpText() const { return ToString(); } + +TypePtr TensorType::DeepCopy() const { + MS_EXCEPTION_IF_NULL(element_type_); + if (IsGeneric()) { + return std::make_shared(); + } else { + return std::make_shared(element_type_->DeepCopy()); + } +} + +std::string TensorType::ToString() const { + if (element_type_ == nullptr) { + return "Tensor"; + } else { + return "Tensor[" + element_type_->ToString() + "]"; + } +} + +std::string TensorType::DumpText() const { + if (element_type_ == nullptr) { + return "Tensor"; + } else { + return "Tensor(" + element_type_->DumpText() + ")"; + } +} + +bool TensorType::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_elem_type = static_cast(other).element_type_; + // When element_type_ = nullptr, which means any type of Array. + if (element_type_ == nullptr && other_elem_type == nullptr) { + return true; + } else if (element_type_ == nullptr || other_elem_type == nullptr) { + return false; + } + return *element_type_ == *other_elem_type; +} + +Function::Function() : Object(kObjectTypeFunction) { + args_ = std::vector(); + retval_ = nullptr; +} + +Function::Function(const std::vector& args, const TypePtr retval) + : Object(kObjectTypeFunction, false), args_(args), retval_(retval) {} + +TypePtr Function::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + TypePtrList args; + TypePtr retval = nullptr; + (void)std::transform(args_.begin(), args_.end(), std::back_inserter(args), + [](const TypePtr& arg) { return arg->DeepCopy(); }); + if (retval_ != nullptr) { + retval = retval_->DeepCopy(); + } + return std::make_shared(args, retval); + } +} + +bool Function::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + + const auto& other_function = static_cast(other); + if ((retval_ != nullptr) && (other_function.retval_ != nullptr)) { + if (*retval_ != *other_function.retval_) { + return false; + } + } else if ((retval_ == nullptr) && (other_function.retval_ != nullptr)) { + return false; + } + if (args_.size() != other_function.args_.size()) { + return false; + } + for (size_t i = 0; i < args_.size(); ++i) { + if (*args_[i] != *other_function.args_[i]) return false; + } + return true; +} + +std::string Function::ToString() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Func"; + } else { + buffer << "Func[("; + bool begin = true; + for (auto& attr : args_) { + if (!begin) { + buffer << ", "; + } else { + begin = false; + } + buffer << attr->ToString(); + } + buffer << ")"; + if (retval_ != nullptr) { + buffer << ", " << retval_->ToString() << "]"; + } else { + buffer << "]"; + } + } + return buffer.str(); +} + +TypePtr TypeAnything::DeepCopy() const { return kAnyType; } + +TypePtr JTagged::DeepCopy() const { + MS_EXCEPTION_IF_NULL(subtype_); + if (IsGeneric()) { + return std::make_shared(); + } else { + auto subtype = subtype_->DeepCopy(); + return std::make_shared(subtype); + } +} + +std::string JTagged::ToString() const { + MS_EXCEPTION_IF_NULL(subtype_); + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "JT"; + } else { + buffer << "JT["; + buffer << subtype_->ToString() << "]"; + } + return buffer.str(); +} + +std::string JTagged::DumpText() const { + MS_EXCEPTION_IF_NULL(subtype_); + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "JT"; + } else { + buffer << "JT["; + buffer << subtype_->DumpText() << "]"; + } + return buffer.str(); +} + +std::ostream& operator<<(std::ostream& os, const std::shared_ptr problem) { + MS_EXCEPTION_IF_NULL(problem); + os << problem->ToString(); + return os; +} + +std::size_t TypeHasher::operator()(TypePtr const& type) const { + MS_EXCEPTION_IF_NULL(type); + std::size_t hash = std::hash()(type->type_id()); + return hash; +} + +std::size_t TypeListHasher::operator()(const TypePtrList& type_list) const { + std::size_t hash_sum = 0; + for (auto& type : type_list) { + auto type_id = static_cast(type->type_id()); + hash_sum = hash_combine(hash_sum, type_id); + } + return hash_sum; +} + +bool TypeEqual::operator()(TypePtr const& t1, TypePtr const& t2) const { + MS_EXCEPTION_IF_NULL(t1); + MS_EXCEPTION_IF_NULL(t2); + return t1->type_id() == t2->type_id(); +} + +bool TypeListEqual::operator()(TypePtrList const& lhs, TypePtrList const& rhs) const { + if (lhs.size() != rhs.size()) { + return false; + } + std::size_t size = lhs.size(); + for (std::size_t i = 0; i < size; ++i) { + MS_EXCEPTION_IF_NULL(lhs[i]); + MS_EXCEPTION_IF_NULL(rhs[i]); + if (*lhs[i] != *rhs[i]) { + return false; + } + } + return true; +} + +TypePtr TypeIdToType(TypeId id) { + switch (id) { + case kNumberTypeFloat16: + return kFloat16; + case kNumberTypeFloat: + case kNumberTypeFloat32: + return kFloat32; + case kNumberTypeFloat64: + return kFloat64; + case kNumberTypeInt8: + return kInt8; + case kNumberTypeInt16: + return kInt16; + case kNumberTypeInt32: + return kInt32; + case kNumberTypeInt64: + return kInt64; + case kNumberTypeUInt8: + return kUInt8; + case kNumberTypeUInt16: + return kUInt16; + case kNumberTypeUInt32: + return kUInt32; + case kNumberTypeUInt64: + return kUInt64; + case kNumberTypeBool: + return kBool; + case kMetaTypeExternal: + return kTypeExternal; + case kMetaTypeAnything: + return kAnyType; + case kMetaTypeNone: + return kTypeNone; + case kObjectTypeEnvType: + return kTypeEnv; + case kObjectTypeRefKey: + return kRefKeyType; + case kObjectTypeRef: + return kRefType; + case kTypeUnknown: + return kTypeNone; + default: + MS_LOG(EXCEPTION) << "Not support the type: " << id; + } +} + +namespace { +template +TypePtr StringToNumberType(const std::string& type_name, const std::string& num_type_name) { + TypePtr type = nullptr; + if (type_name == num_type_name) { + type = std::make_shared(); + } else { + try { + if (num_type_name.size() >= type_name.size()) { + MS_LOG(EXCEPTION) << "Convert type is error, type_name(" << type_name << "), num_type_name(" << num_type_name + << ")"; + } + auto bits = std::stoi(type_name.substr(num_type_name.size())); + type = std::make_shared(bits); + } catch (const std::exception& e) { + MS_LOG(EXCEPTION) << "" << num_type_name << " convert from string error " << e.what(); + } + } + return type; +} + +std::vector StringToVectorOfType(const std::string& type_names) { + std::vector types; + if (type_names.length() == 0) { + return types; + } + std::string::size_type start = 0; + std::string::size_type end = type_names.find_first_of(','); + while (end != std::string::npos) { + types.push_back(StringToType(type_names.substr(start, end))); + // Skip ',' to find the next element. + start = end + 1; + end = type_names.find_first_of(',', start); + } + if (start >= type_names.size()) { + MS_LOG(EXCEPTION) << "Type name is empty string."; + } + types.push_back(StringToType(type_names.substr(start))); + return types; +} + +TypePtr TensorStrToType(const std::string& type_name) { + TypePtr type = nullptr; + if (type_name == "Tensor") { + type = std::make_shared(); + } else { + try { + auto start = type_name.find_first_of('[') + 1; + auto end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + auto element_str = type_name.substr(start, end - start); + auto element_type = StringToType(element_str); + if (element_type == nullptr) { + return nullptr; + } + type = std::make_shared(element_type); + } catch (const std::exception& e) { + MS_LOG(EXCEPTION) << "" << type_name << " convert from string error " << e.what(); + } + } + + return type; +} + +TypePtr ListStrToType(const std::string& type_name) { + TypePtr type = nullptr; + if (type_name == "List") { + type = std::make_shared(); + } else { + try { + auto start = type_name.find_first_of('[') + 1; + auto end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + std::string element_strs = type_name.substr(start, end - start); + std::vector element_types = StringToVectorOfType(element_strs); + bool wrong = + std::any_of(element_types.begin(), element_types.end(), [](const TypePtr& x) { return x == nullptr; }); + if (wrong) { + return nullptr; + } + type = std::make_shared(element_types); + } catch (const std::exception& e) { + MS_LOG(EXCEPTION) << "" << type_name << " convert from string error " << e.what(); + } + } + + return type; +} + +TypePtr TupleStrToType(const std::string& type_name) { + TypePtr type = nullptr; + if (type_name == "Tuple") { + type = std::make_shared(); + } else { + try { + size_t start = type_name.find_first_of('[') + 1; + size_t end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + std::string element_strs = type_name.substr(start, end - start); + std::vector element_types = StringToVectorOfType(element_strs); + bool wrong = + std::any_of(element_types.begin(), element_types.end(), [](const TypePtr& x) { return x == nullptr; }); + if (wrong) { + return nullptr; + } + type = std::make_shared(element_types); + } catch (const std::exception& e) { + MS_LOG(EXCEPTION) << "" << type_name << " convert from string error " << e.what(); + } + } + return type; +} + +TypePtr FunctionStrToType(const std::string& type_name) { + TypePtr type = nullptr; + + if (type_name == "Function") { + type = std::make_shared(); + } else { + try { + // format: [(para1, para2, para3, ...) retval] + size_t start = type_name.find_first_of('[') + 1; + size_t end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + std::string str_all = type_name.substr(start, end - start); + size_t start_a = str_all.find_first_of('(') + 1; + size_t end_a = str_all.find_last_of(')'); + if (start_a >= str_all.size()) { + return nullptr; + } + std::string str_args = str_all.substr(start_a, end_a - start_a); + // bypass " " between ")" and retval + start = end_a + 2; + if (start >= str_all.size()) { + return nullptr; + } + std::string str_retval = str_all.substr(start); + + std::vector args_type = StringToVectorOfType(str_args); + TypePtr retval = StringToType(str_retval); + bool wrong = std::any_of(args_type.begin(), args_type.end(), [](const TypePtr& x) { return x == nullptr; }); + if (retval == nullptr || wrong) { + return nullptr; + } + type = std::make_shared(args_type, retval); + } catch (const std::exception& e) { + MS_LOG(EXCEPTION) << "" << type_name << " convert from string error " << e.what(); + } + } + return type; +} +} // namespace + +TypePtr StringToType(const std::string& type_name) { + TypePtr type = nullptr; + if (type_name.compare("None") == 0) { + type = std::make_shared(); + } else if (type_name.compare("TypeType") == 0) { + type = std::make_shared(); + } else if (type_name.compare("SymbolicKeyType") == 0) { + type = std::make_shared(); + } else if (type_name.compare("RefKeyType") == 0) { + type = std::make_shared(); + } else if (type_name.compare("EnvType") == 0) { + type = std::make_shared(); + } else if (type_name.compare("Number") == 0) { + type = std::make_shared(); + } else if (type_name.compare("Bool") == 0) { + type = std::make_shared(); + } else if (type_name.compare(0, strlen("Int"), "Int") == 0) { + type = StringToNumberType(type_name, "Int"); + } else if (type_name.compare(0, strlen("UInt"), "UInt") == 0) { + type = StringToNumberType(type_name, "UInt"); + } else if (type_name.compare(0, strlen("Float"), "Float") == 0) { + type = StringToNumberType(type_name, "Float"); + } else if (type_name.compare(0, strlen("Tensor"), "Tensor") == 0) { + type = TensorStrToType(type_name); + } else if (type_name.compare(0, strlen("List"), "List") == 0) { + type = ListStrToType(type_name); + } else if (type_name.compare(0, strlen("Tuple"), "Tuple") == 0) { + type = TupleStrToType(type_name); + } else if (type_name.compare("Slice") == 0) { + type = std::make_shared(); + } else if (type_name.compare("Dictionary") == 0) { + type = std::make_shared(); + } else if (type_name.compare("String") == 0) { + type = std::make_shared(); + } else if (type_name.compare("Problem") == 0) { + type = std::make_shared(); + } else if (type_name.compare(0, strlen("Function"), "Function") == 0) { + type = FunctionStrToType(type_name); + } else { + // - unsupported to convert + // Class + // SymbolicType + // JTagged + // Anything + // External + // Problem + MS_LOG(EXCEPTION) << "Unsupported type name: " << type_name << "!"; + } + return type; +} + +bool IsIdentidityOrSubclass(TypePtr const& x, TypePtr const& base_type) { + if (x == nullptr || base_type == nullptr) { + MS_LOG(ERROR) << "Type is nullptr."; + return false; + } + if (base_type->type_id() == kTypeUnknown || x->type_id() == kTypeUnknown) { + return false; + } else if (!(base_type->IsGeneric())) { + return *(base_type) == *(x); + } else if (base_type->type_id() == x->type_id()) { + return true; + } else if (base_type->type_id() == x->generic_type_id()) { + return true; + } else if (base_type->type_id() == x->object_type()) { + return true; + } else if (base_type->type_id() == x->meta_type()) { + return true; + } else { + return false; + } +} + +bool IsSubType(TypePtr const& t1, TypePtr const& t2) { + MS_EXCEPTION_IF_NULL(t1); + if (t1->type_id() == kTypeUnknown) { + return false; + } else if (t2 != nullptr) { + return IsIdentidityOrSubclass(t1, t2); + } else { + return true; + } +} + +REGISTER_PYBIND_DEFINE( + typing, ([](py::module* const m) { + auto m_sub = m->def_submodule("typing", "submodule for dtype"); + py::enum_(m_sub, "TypeId"); + (void)m_sub.def("is_subclass", &IsIdentidityOrSubclass, "is equal or subclass"); + (void)m_sub.def("load_type", &TypeIdToType, "load type"); + (void)m_sub.def( + "dump_type", [](const TypePtr& t) { return t->type_id(); }, "dump type"); + (void)py::class_>(m_sub, "Type") + .def_readonly(PYTHON_DTYPE_FLAG, &mindspore::Type::parse_info_) + .def("__eq__", + [](const TypePtr& t1, const TypePtr& t2) { + if (t1 != nullptr && t2 != nullptr) { + return *t1 == *t2; + } + return false; + }) + .def("__hash__", &Type::hash) + .def("__str__", &Type::ToString) + .def("__repr__", &Type::ReprString) + .def("__deepcopy__", [](const TypePtr& t, py::dict) { + if (t == nullptr) { + return static_cast(nullptr); + } + return t->DeepCopy(); + }); + (void)py::class_>(m_sub, "Number").def(py::init()); + (void)py::class_>(m_sub, "Bool") + .def(py::init()) + .def(py::pickle( + [](const Bool&) { // __getstate__ + return py::make_tuple(); + }, + [](const py::tuple&) { // __setstate__ + return std::make_shared(); + })); + (void)py::class_>(m_sub, "Int") + .def(py::init()) + .def(py::init(), py::arg("nbits")) + .def(py::pickle( + [](const Int& t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(py::int_(t.nbits())); + }, + [](const py::tuple& t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + Int data(t[0].cast()); + return data; + })); + (void)py::class_>(m_sub, "UInt") + .def(py::init()) + .def(py::init(), py::arg("nbits")) + .def(py::pickle( + [](const UInt& t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(py::int_(t.nbits())); + }, + [](const py::tuple& t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + UInt data(t[0].cast()); + return data; + })); + (void)py::class_>(m_sub, "Float") + .def(py::init()) + .def(py::init(), py::arg("nbits")) + .def(py::pickle( + [](const Float& t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(py::int_(t.nbits())); + }, + [](const py::tuple& t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + Float data(t[0].cast()); + return data; + })); + (void)py::class_>(m_sub, "List") + .def(py::init()) + .def(py::init>(), py::arg("elements")); + (void)py::class_>(m_sub, "Tuple") + .def(py::init()) + .def(py::init>(), py::arg("elements")); + (void)py::class_>(m_sub, "TensorType") + .def(py::init()) + .def(py::init(), py::arg("element")) + .def("element_type", &TensorType::element) + .def(py::pickle( + [](const TensorType& t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(py::int_(static_cast(t.element()->type_id()))); + }, + [](const py::tuple& t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + TensorType data(TypeIdToType(TypeId(static_cast(t[0].cast())))); + return data; + })); + (void)py::class_>(m_sub, "Function") + .def(py::init()) + .def(py::init, TypePtr>(), py::arg("args"), py::arg("retval")); + (void)py::class_>(m_sub, "Class").def(py::init()); + (void)py::class_>(m_sub, "SymbolicKeyType").def(py::init()); + (void)py::class_>(m_sub, "EnvType").def(py::init()); + (void)py::class_>(m_sub, "TypeNone").def(py::init()); + (void)py::class_>(m_sub, "TypeType").def(py::init()); + (void)py::class_>(m_sub, "String").def(py::init()); + (void)py::class_>(m_sub, "RefKeyType").def(py::init()); + (void)py::class_>(m_sub, "RefType").def(py::init()); + })); + +const TypePtr kTypeExternal = std::make_shared(); +const TypePtr kTypeEnv = std::make_shared(); +const TypePtr kTypeType = std::make_shared(); +const TypePtr kTensorType = std::make_shared(); +const TypePtr kString = std::make_shared(); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype.h b/mindspore/ccsrc/ir/dtype.h new file mode 100644 index 00000000000..11099e460e3 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype.h @@ -0,0 +1,281 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_DTYPE_H_ +#define MINDSPORE_CCSRC_IR_DTYPE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/named.h" + +#include "ir/dtype/type.h" +#include "ir/dtype/ref.h" +#include "ir/dtype/number.h" +#include "ir/dtype/container.h" +#include "ir/dtype/empty.h" + +/* namespace to support intermediate representation definition */ +namespace mindspore { +// Only few type supported now. +TypePtr TypeIdToType(TypeId id); + +class String : public Object { + public: + String() : Object(kObjectTypeString, false) {} + ~String() override = default; + MS_DECLARE_PARENT(String, Object) + + TypeId generic_type_id() const override { return kObjectTypeString; } + + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToString() const override { return std::string("String:"); } + std::string ToReprString() const override { return "string"; } + std::string DumpText() const override { return "String"; } +}; +using StringPtr = std::shared_ptr; + +class Keyword : public Object { + public: + Keyword() : Object(kObjectTypeKeyword, false), key_(""), value_(nullptr) {} + Keyword(const std::string& key, const TypePtr& value) : Object(kObjectTypeKeyword, false), key_(key), value_(value) {} + + ~Keyword() override = default; + MS_DECLARE_PARENT(Keyword, Object) + + TypeId generic_type_id() const override { return kObjectTypeKeyword; } + TypePtr DeepCopy() const override; + + std::string ToString() const override; + std::string DumpText() const override; + bool operator==(const Type& other) const override; + + std::string GetKey() const { return key_; } + TypePtr GetValue() const { return value_; } + + private: + std::string key_; + TypePtr value_; +}; +using KeywordPtr = std::shared_ptr; + +class Slice : public Object { + public: + Slice() : Object(kObjectTypeSlice), start_(nullptr), stop_(nullptr), step_(nullptr) {} + Slice(const TypePtr& start, const TypePtr& stop, const TypePtr& step) + : Object(kObjectTypeSlice, false), start_(start), stop_(stop), step_(step) {} + + ~Slice() override = default; + MS_DECLARE_PARENT(Slice, Object) + + TypeId generic_type_id() const override { return kObjectTypeSlice; } + TypePtr DeepCopy() const override; + + std::string ToString() const override; + std::string DumpText() const override; + bool operator==(const Type& other) const override; + + TypePtr get_start() const { return start_; } + TypePtr get_stop() const { return stop_; } + TypePtr get_step() const { return step_; } + + private: + TypePtr start_; + TypePtr stop_; + TypePtr step_; +}; +using SlicePtr = std::shared_ptr; + +class TensorType : public Object { + public: + TensorType() : Object(kObjectTypeTensorType) {} + explicit TensorType(const TypePtr& ele) : Object(kObjectTypeTensorType, false), element_type_(ele) {} + ~TensorType() override = default; + MS_DECLARE_PARENT(TensorType, Object) + + TypeId generic_type_id() const override { return kObjectTypeTensorType; } + const TypePtr element() const { return element_type_; } + void set_element(const TypePtr& element_type) { element_type_ = element_type; } + + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string ToReprString() const override { return "tensor"; } + std::string DumpText() const override; + bool operator==(const Type& other) const override; + + private: + TypePtr element_type_; +}; +using TensorTypePtr = std::shared_ptr; + +class Function : public Object { + public: + Function(); + Function(const std::vector& args, const TypePtr retval); + ~Function() override = default; + MS_DECLARE_PARENT(Function, Object) + + TypeId generic_type_id() const override { return kObjectTypeFunction; } + + // Add temporarily for return abstraction to avoid type checking. + bool IsTransparent() const { return (args_.empty()) && (retval_ == nullptr); } + const std::vector& args() const { return args_; } + const TypePtr& retval() const { return retval_; } + + TypePtr DeepCopy() const override; + bool operator==(const Type& other) const override; + std::string ToString() const override; + std::string ToReprString() const override { return "function"; } + + private: + std::vector args_; + TypePtr retval_; +}; +using FunctionPtr = std::shared_ptr; + +class JTagged : public Object { + public: + JTagged() : Object(kObjectTypeJTagged) {} + explicit JTagged(const TypePtr& subtype) : Object(kObjectTypeJTagged, false), subtype_(subtype) {} + ~JTagged() override = default; + MS_DECLARE_PARENT(JTagged, Object) + + TypeId generic_type_id() const override { return kObjectTypeJTagged; } + + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string DumpText() const override; + + private: + TypePtr subtype_; +}; +using JTaggedPtr = std::shared_ptr; + +class SymbolicKeyType : public Object { + public: + SymbolicKeyType() : Object(kObjectTypeSymbolicKeyType) {} + ~SymbolicKeyType() override = default; + MS_DECLARE_PARENT(SymbolicKeyType, Object) + + TypeId generic_type_id() const override { return kObjectTypeSymbolicKeyType; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToReprString() const override { return "symbolic_key"; } + std::string DumpText() const override { return "SymType"; } +}; + +class EnvType : public Object { + public: + EnvType() : Object(kObjectTypeEnvType) {} + ~EnvType() override = default; + MS_DECLARE_PARENT(EnvType, Object) + + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToReprString() const override { return "env_type"; } + std::string DumpText() const override { return "EnvType"; } +}; +using EnvTypePtr = std::shared_ptr; + +class TypeType : public Type { + public: + TypeType() : Type(kMetaTypeTypeType) {} + ~TypeType() override = default; + MS_DECLARE_PARENT(TypeType, Type) + + TypeId generic_type_id() const override { return kMetaTypeTypeType; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToReprString() const override { return "type_type"; } + std::string DumpText() const override { return "TypeType"; } +}; +using TypeTypePtr = std::shared_ptr; + +class Problem : public Type { + public: + Problem() : Type(kMetaTypeProblem), kind_(Named("unknown")) {} + explicit Problem(const Named& kind) : Type(kMetaTypeProblem), kind_(kind) {} + ~Problem() override = default; + MS_DECLARE_PARENT(Problem, Type) + + TypeId generic_type_id() const override { return kMetaTypeProblem; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToString() const override { return kind_.name(); } + std::string DumpText() const override { return "ProblemType"; } + + friend std::ostream& operator<<(std::ostream& os, const std::shared_ptr problem); + + private: + Named kind_; +}; +using ProblemPtr = std::shared_ptr; + +class External : public Type { + public: + External() : Type(kMetaTypeExternal) {} + ~External() override = default; + MS_DECLARE_PARENT(External, Type) + + TypeId generic_type_id() const override { return kMetaTypeExternal; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string DumpText() const override { return "ExternalType"; } + + private: + TypePtr kind; +}; +using ExternalPtr = std::shared_ptr; + +// helper template +template +TypePtr Clone(const T& t) { + return t.Clone(); +} + +TypePtr StringToType(const std::string& type_name); + +// Judge whether x is predicate or is a subclass of predicate. +bool IsIdentidityOrSubclass(TypePtr const& x, TypePtr const& base_type); + +// Whether t1 is identity or a subclass of t2. +bool IsSubType(TypePtr const& t1, TypePtr const& t2 = nullptr); + +struct TypeHasher { + std::size_t operator()(TypePtr const& type) const; +}; +struct TypeListHasher { + std::size_t operator()(const TypePtrList& type_list) const; +}; +struct TypeEqual { + bool operator()(TypePtr const& t1, TypePtr const& t2) const; +}; +struct TypeListEqual { + bool operator()(TypePtrList const& lhs, TypePtrList const& rhs) const; +}; + +extern const TypePtr kTypeExternal; +extern const TypePtr kTypeEnv; +extern const TypePtr kTypeType; +extern const TypePtr kString; +extern const TypePtr kTensorType; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_DTYPE_H_ diff --git a/mindspore/ccsrc/ir/dtype/container.cc b/mindspore/ccsrc/ir/dtype/container.cc new file mode 100644 index 00000000000..8bca29f7935 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/container.cc @@ -0,0 +1,298 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/dtype/container.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +static std::string DumpTypeVector(const std::vector& elements, bool is_dumptext) { + std::ostringstream oss; + bool begin = true; + int cnt = 0; + // write 'Tuple[Bool, Bool, Bool, Int, Float, Float]' as 'Tuple[Bool...3, Int, Float...2]' + for (size_t i = 0; i < elements.size(); ++i) { + TypePtr elem = elements[i]; + cnt += 1; + bool print = false; + if (i + 1 < elements.size()) { + TypePtr next = elements[i + 1]; + if (*elem != *next) { + print = true; + } + } else { + // encounter last element + print = true; + } + if (!print) { + continue; + } + if (!begin) { + oss << ","; + } else { + begin = false; + } + oss << (is_dumptext ? elem->DumpText() : elem->ToString()); + if (cnt > 1) { + oss << "*" << cnt; + } + cnt = 0; + } + return oss.str(); +} + +TypePtr List::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + TypePtrList elements; + (void)std::transform(elements_.begin(), elements_.end(), std::back_inserter(elements), + [](const TypePtr& ele) { return ele->DeepCopy(); }); + auto copy = std::make_shared(elements); + return copy; + } +} + +const TypePtr List::operator[](std::size_t dim) const { + if (dim >= size()) { + MS_LOG(EXCEPTION) << "Out of the size of the List."; + } + return elements_[dim]; +} + +bool List::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + const List& other_list = static_cast(other); + if (elements_.size() != other_list.elements_.size()) { + return false; + } + for (size_t i = 0; i < elements_.size(); ++i) { + if (*elements_[i] != *other_list.elements_[i]) { + return false; + } + } + return true; +} + +Class::Class(const Named& tag, const ClassAttrVector& attributes, + const std::unordered_map& methods) + : Object(kObjectTypeClass, false), attributes_(attributes), tag_(tag), methods_(methods) {} + +std::string List::ToString() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "List"; + } else { + buffer << "List["; + buffer << DumpTypeVector(elements_, false); + buffer << "]"; + } + return buffer.str(); +} + +std::string List::DumpText() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "List"; + } else { + buffer << "List["; + buffer << DumpTypeVector(elements_, true); + buffer << "]"; + } + return buffer.str(); +} + +bool Class::operator==(const Type& other) const { + // Class is cached for each pyobj in ParseDataClass, so ClassPtr is one by one map to pyobj. + return &other == this; +} + +TypePtr Class::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + auto copy = std::make_shared(tag_, attributes_, methods_); + return copy; + } +} + +std::string Class::ToString() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "cls"; + } else { + bool begin = true; + buffer << "cls." << tag_ << "["; + for (auto& attr : attributes_) { + if (!begin) { + buffer << ", "; + } else { + begin = false; + } + buffer << attr.first << ":" << attr.second->ToString(); + } + buffer << "]"; + } + return buffer.str(); +} + +std::string Class::DumpText() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Cls"; + } else { + bool begin = true; + buffer << "Cls." << tag_ << "["; + for (auto& attr : attributes_) { + if (!begin) { + buffer << ", "; + } else { + begin = false; + } + buffer << attr.first << ":" << attr.second->DumpText(); + } + buffer << "]"; + } + return buffer.str(); +} + +TypePtr Tuple::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + TypePtrList elements; + (void)std::transform(elements_.begin(), elements_.end(), std::back_inserter(elements), + [](const TypePtr& ele) { return ele->DeepCopy(); }); + auto copy = std::make_shared(elements); + return copy; + } +} + +bool Tuple::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_tuple = static_cast(other); + if (elements_.size() != other_tuple.elements_.size()) { + return false; + } + for (size_t i = 0; i < elements_.size(); ++i) { + if (*elements_[i] != *other_tuple.elements_[i]) { + return false; + } + } + return true; +} + +const TypePtr Tuple::operator[](std::size_t dim) const { + if (dim >= size()) { + MS_LOG(EXCEPTION) << "Out of the size of the tuple."; + } + return elements_[dim]; +} + +std::string Tuple::ToString() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Tuple"; + } else { + buffer << "Tuple["; + buffer << DumpTypeVector(elements_, false); + buffer << "]"; + } + return buffer.str(); +} + +std::string Tuple::DumpText() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Tuple"; + } else { + buffer << "Tuple["; + buffer << DumpTypeVector(elements_, true); + buffer << "]"; + } + return buffer.str(); +} + +TypePtr Dictionary::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + std::vector> kv; + (void)std::transform( + key_values_.begin(), key_values_.end(), std::back_inserter(kv), + [](const std::pair& item) { return std::make_pair(item.first, item.second->DeepCopy()); }); + return std::make_shared(kv); + } +} + +std::string DumpKeyVector(std::vector keys) { + std::ostringstream buffer; + for (auto key : keys) { + buffer << key << ","; + } + return buffer.str(); +} + +std::string Dictionary::ToString() const { + std::ostringstream buffer; + std::vector keys; + std::vector values; + for (const auto& kv : key_values_) { + keys.push_back(kv.first); + values.push_back(kv.second); + } + if (IsGeneric()) { + buffer << "Dictionary"; + } else { + buffer << "Dictionary["; + buffer << "[" << DumpKeyVector(keys) << "],"; + buffer << "[" << DumpTypeVector(values, false) << "]"; + buffer << "]"; + } + return buffer.str(); +} + +std::string Dictionary::DumpText() const { return ToString(); } + +bool Dictionary::operator==(const mindspore::Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + + const auto& other_dict = static_cast(other); + if (key_values_.size() != other_dict.key_values_.size()) { + return false; + } + for (size_t index = 0; index < key_values_.size(); index++) { + if (key_values_[index].first != other_dict.key_values_[index].first) { + return false; + } + if (*key_values_[index].second != *other_dict.key_values_[index].second) { + return false; + } + } + return true; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/container.h b/mindspore/ccsrc/ir/dtype/container.h new file mode 100644 index 00000000000..04ed484cf7e --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/container.h @@ -0,0 +1,150 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_DTYPE_CONTAINER_H_ +#define MINDSPORE_CCSRC_IR_DTYPE_CONTAINER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/named.h" +#include "ir/dtype/type.h" + +namespace mindspore { +// TypeRefKey type + +// List +class List : public Object { + public: + List() : Object(kObjectTypeList) {} + List(const std::initializer_list& objs) + : Object(kObjectTypeList, false), elements_(objs.begin(), objs.end()) {} + // Shadow copy; + explicit List(const TypePtrList& obj) : Object(kObjectTypeList, false), elements_(obj) {} + ~List() override {} + MS_DECLARE_PARENT(List, Object) + + const TypePtr operator[](size_t dim) const; + TypeId generic_type_id() const override { return kObjectTypeList; } + TypePtr DeepCopy() const override; + + bool operator==(const Type& other) const override; + std::size_t size() const { return elements_.size(); } + TypePtrList elements() const { return elements_; } + std::string ToString() const override; + std::string ToReprString() const override { return "list_"; } + std::string DumpText() const override; + + private: + TypePtrList elements_; +}; +using ListPtr = std::shared_ptr; + +using ClassAttrVector = std::vector>; + +class Class : public Object { + public: + Class() : Object(kObjectTypeClass), tag_(Named("Class")) {} + Class(const Named& tag, const ClassAttrVector& attributes, const std::unordered_map& methods); + ~Class() override {} + MS_DECLARE_PARENT(Class, Object) + + TypeId generic_type_id() const override { return kObjectTypeClass; } + + bool operator==(const Type& other) const override; + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string DumpText() const override; + void set_value(const std::unordered_map& v) { attributes_value_ = v; } + + Named tag() { return tag_; } + std::unordered_map GetValue() { return attributes_value_; } + std::unordered_map methods() { return methods_; } + ClassAttrVector& GetAttributes() { return attributes_; } + + ClassAttrVector attributes_; + + private: + Named tag_; + std::unordered_map methods_; + // For AbstractClass build value + std::unordered_map attributes_value_; +}; +using ClassPtr = std::shared_ptr; + +class Tuple : public Object { + public: + Tuple() : Object(kObjectTypeTuple) {} + // usage : Tuple t = {std::make_shared(), std::make_shared(32)}; + Tuple(const std::initializer_list& objs) + : Object(kObjectTypeTuple, false), elements_(objs.begin(), objs.end()) {} + + // Shadow copy + explicit Tuple(const TypePtrList& objs) : Object(kObjectTypeTuple, false), elements_(objs.begin(), objs.end()) {} + + ~Tuple() override {} + MS_DECLARE_PARENT(Tuple, Object) + + TypeId generic_type_id() const override { return kObjectTypeTuple; } + TypePtr DeepCopy() const override; + + std::string ToString() const override; + std::string ToReprString() const override { return "tuple_"; } + std::string DumpText() const override; + const TypePtr operator[](size_t dim) const; + bool operator==(const Type& other) const override; + + TypePtrList elements() const { return elements_; } + std::size_t size() const { return elements_.size(); } + + private: + TypePtrList elements_; +}; +using TuplePtr = std::shared_ptr; + +class Dictionary : public Object { + public: + Dictionary() : Object(kObjectTypeDictionary) {} + explicit Dictionary(const std::vector>& key_values) + : Object(kObjectTypeDictionary, false), key_values_(key_values) {} + + ~Dictionary() override {} + MS_DECLARE_PARENT(Dictionary, Object) + + TypeId generic_type_id() const override { return kObjectTypeDictionary; } + + bool operator==(const Type& other) const override; + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string DumpText() const override; + + private: + std::vector> key_values_; +}; +using DictionaryPtr = std::shared_ptr; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_DTYPE_CONTAINER_H_ diff --git a/mindspore/ccsrc/ir/dtype/empty.cc b/mindspore/ccsrc/ir/dtype/empty.cc new file mode 100644 index 00000000000..3d4f74bf312 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/empty.cc @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/dtype/empty.h" + +namespace mindspore { +const TypePtr kTypeNone = std::make_shared(); +const TypePtr kTypeAnything = std::make_shared(); +const TypePtr kAnyType = std::make_shared(); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/empty.h b/mindspore/ccsrc/ir/dtype/empty.h new file mode 100644 index 00000000000..a13dc084ca0 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/empty.h @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_DTYPE_EMPTY_H_ +#define MINDSPORE_CCSRC_IR_DTYPE_EMPTY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/named.h" +#include "ir/dtype/type.h" + +namespace mindspore { +class TypeAnything : public Type { + public: + TypeAnything() : Type(kMetaTypeAnything) {} + ~TypeAnything() override {} + MS_DECLARE_PARENT(TypeAnything, Type) + + TypeId generic_type_id() const override { return kMetaTypeAnything; } + TypePtr DeepCopy() const override; + std::string DumpText() const override { return "AnythingType"; } +}; +using TypeAnythingPtr = std::shared_ptr; + +class TypeNone : public Type { + public: + TypeNone() : Type(kMetaTypeNone) {} + ~TypeNone() override {} + MS_DECLARE_PARENT(TypeNone, Type) + + TypeId generic_type_id() const override { return kMetaTypeNone; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToReprString() const override { return "type_none"; } + std::string DumpText() const override { return "NoneType"; } +}; +using TypeNonePtr = std::shared_ptr; + +class TypeNull : public Type { + public: + TypeNull() : Type(kMetaTypeNull) {} + ~TypeNull() override {} + MS_DECLARE_PARENT(TypeNull, Type) + + TypeId generic_type_id() const override { return kMetaTypeNull; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string DumpText() const override { return "NullType"; } +}; +using TypeNullPtr = std::shared_ptr; + +extern const TypePtr kTypeNone; +extern const TypePtr kTypeAnything; +extern const TypePtr kAnyType; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_DTYPE_EMPTY_H_ diff --git a/mindspore/ccsrc/ir/dtype/number.cc b/mindspore/ccsrc/ir/dtype/number.cc new file mode 100644 index 00000000000..70d0aaeeaa3 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/number.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/dtype/number.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +bool Number::operator==(const Type& other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_number = static_cast(other); + return ((number_type_ == other_number.number_type_) && (nbits_ == other_number.nbits_)); +} + +Int::Int(const int nbits) : Number(IntBitsToTypeId(nbits), nbits, false) { + if (nbits != 8 && nbits != 16 && nbits != 32 && nbits != 64) { + MS_LOG(EXCEPTION) << "wrong number of bits."; + } +} + +UInt::UInt(const int nbits) : Number(UIntBitsToTypeId(nbits), nbits, false) { + if (nbits != 8 && nbits != 16 && nbits != 32 && nbits != 64) { + MS_LOG(EXCEPTION) << "wrong number of bits."; + } +} + +Float::Float(const int nbits) : Number(FloatBitsToTypeId(nbits), nbits, false) { + if (nbits != 16 && nbits != 32 && nbits != 64) { + MS_LOG(EXCEPTION) << "wrong number of bits."; + } +} + +const TypePtr kBool = std::make_shared(); +const TypePtr kInt8 = std::make_shared(8); +const TypePtr kInt16 = std::make_shared(16); +const TypePtr kInt32 = std::make_shared(32); +const TypePtr kInt64 = std::make_shared(64); +const TypePtr kUInt8 = std::make_shared(8); +const TypePtr kUInt16 = std::make_shared(16); +const TypePtr kUInt32 = std::make_shared(32); +const TypePtr kUInt64 = std::make_shared(64); +const TypePtr kFloat16 = std::make_shared(16); +const TypePtr kFloat32 = std::make_shared(32); +const TypePtr kFloat64 = std::make_shared(64); +const TypePtr kInt = std::make_shared(); +const TypePtr kUInt = std::make_shared(); +const TypePtr kFloat = std::make_shared(); +const TypePtr kNumber = std::make_shared(); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/number.h b/mindspore/ccsrc/ir/dtype/number.h new file mode 100644 index 00000000000..cb3b0a607c8 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/number.h @@ -0,0 +1,154 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_DTYPE_NUMBER_H_ +#define MINDSPORE_CCSRC_IR_DTYPE_NUMBER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/named.h" +#include "ir/dtype/type.h" + +namespace mindspore { +// Number, abstract class. +class Number : public Object { + public: + Number() : Object(kObjectTypeNumber), number_type_(kObjectTypeNumber), nbits_(0) {} + Number(const TypeId number_type, const int nbits, bool is_generic = true) + : Object(kObjectTypeNumber, is_generic), number_type_(number_type), nbits_(nbits) {} + ~Number() override = default; + MS_DECLARE_PARENT(Number, Object) + + int nbits() const { return nbits_; } + + TypeId number_type() const override { return number_type_; } + TypeId type_id() const override { return number_type_; } + TypeId generic_type_id() const override { return kObjectTypeNumber; } + + bool operator==(const Type& other) const override; + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToString() const override { return "Number"; } + std::string ToReprString() const override { return "number"; } + std::string DumpText() const override { return "Number"; } + std::string GetTypeName(const std::string& type_name) const { + std::ostringstream oss; + oss << type_name; + if (nbits() != 0) { + oss << nbits(); + } + return oss.str(); + } + + private: + const TypeId number_type_; + const int nbits_; +}; + +// Bool +class Bool : public Number { + public: + Bool() : Number(kNumberTypeBool, 8) {} + ~Bool() override = default; + MS_DECLARE_PARENT(Bool, Number) + + TypeId generic_type_id() const override { return kNumberTypeBool; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToString() const override { return "Bool_"; } + std::string ToReprString() const override { return "bool_"; } + std::string DumpText() const override { return "Bool_"; } +}; + +// Int +class Int : public Number { + public: + Int() : Number(kNumberTypeInt, 0) {} + explicit Int(const int nbits); + ~Int() override = default; + MS_DECLARE_PARENT(Int, Number) + TypeId generic_type_id() const override { return kNumberTypeInt; } + TypePtr DeepCopy() const override { return std::make_shared(nbits()); } + std::string ToString() const override { return GetTypeName("Int"); } + std::string ToReprString() const override { return nbits() == 0 ? "int_" : GetTypeName("int"); } + std::string DumpText() const override { + return nbits() == 0 ? std::string("Int") : std::string("I") + std::to_string(nbits()); + } +}; + +// UInt +class UInt : public Number { + public: + UInt() : Number(kNumberTypeUInt, 0) {} + explicit UInt(const int nbits); + TypeId generic_type_id() const override { return kNumberTypeUInt; } + + ~UInt() override {} + MS_DECLARE_PARENT(UInt, Number) + + TypePtr DeepCopy() const override { return std::make_shared(nbits()); } + std::string ToString() const override { return GetTypeName("UInt"); } + std::string ToReprString() const override { return GetTypeName("uint"); } + std::string DumpText() const override { + return nbits() == 0 ? std::string("UInt") : std::string("U") + std::to_string(nbits()); + } +}; + +// Float +class Float : public Number { + public: + Float() : Number(kNumberTypeFloat, 0) {} + explicit Float(const int nbits); + ~Float() override {} + MS_DECLARE_PARENT(Float, Number) + + TypeId generic_type_id() const override { return kNumberTypeFloat; } + TypePtr DeepCopy() const override { return std::make_shared(nbits()); } + std::string ToString() const override { return GetTypeName("Float"); } + std::string ToReprString() const override { return nbits() == 0 ? "float_" : GetTypeName("float"); } + std::string DumpText() const override { + return nbits() == 0 ? std::string("Float") : std::string("F") + std::to_string(nbits()); + } +}; + +extern const TypePtr kBool; +extern const TypePtr kInt8; +extern const TypePtr kInt16; +extern const TypePtr kInt32; +extern const TypePtr kInt64; +extern const TypePtr kUInt8; +extern const TypePtr kUInt16; +extern const TypePtr kUInt32; +extern const TypePtr kUInt64; +extern const TypePtr kFloat16; +extern const TypePtr kFloat32; +extern const TypePtr kFloat64; +extern const TypePtr kInt; +extern const TypePtr kUInt; +extern const TypePtr kFloat; +extern const TypePtr kNumber; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_DTYPE_NUMBER_H_ diff --git a/mindspore/ccsrc/ir/dtype/ref.cc b/mindspore/ccsrc/ir/dtype/ref.cc new file mode 100644 index 00000000000..9fa7f6750b9 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/ref.cc @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/dtype/ref.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +TypePtr RefType::DeepCopy() const { + if (IsGeneric()) { + return std::make_shared(); + } else { + auto subtype = subtype_->DeepCopy(); + auto subtype_origin = subtype_origin_->DeepCopy(); + return std::make_shared(subtype, subtype_origin); + } +} + +std::string RefType::ToString() const { return DumpText(); } + +std::string RefType::DumpText() const { + std::ostringstream buffer; + if (IsGeneric()) { + buffer << "Ref"; + } else { + buffer << "Ref["; + buffer << subtype_->DumpText() << "]"; + } + return buffer.str(); +} + +const TypePtr kRefKeyType = std::make_shared(); +const TypePtr kRefType = std::make_shared(); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/ref.h b/mindspore/ccsrc/ir/dtype/ref.h new file mode 100644 index 00000000000..7f1dc4a95fc --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/ref.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_DTYPE_REF_H_ +#define MINDSPORE_CCSRC_IR_DTYPE_REF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/named.h" +#include "ir/dtype/type.h" + +namespace mindspore { +// TypeRefKey type +class RefKeyType : public Object { + public: + RefKeyType() : Object(kObjectTypeRefKey) {} + ~RefKeyType() override {} + MS_DECLARE_PARENT(RefKeyType, Object) + + TypeId generic_type_id() const override { return kObjectTypeRefKey; } + TypePtr DeepCopy() const override { return std::make_shared(); } + std::string ToReprString() const override { return "type_refkey"; } + std::string DumpText() const override { return "RefKeyType"; } +}; + +// TypeRef type +class RefType : public Object { + public: + RefType() : Object(kObjectTypeRef) {} + RefType(const TypePtr& subtype, const TypePtr& subtype_origin) + : Object(kObjectTypeRef, false), subtype_(subtype), subtype_origin_(subtype_origin) {} + ~RefType() override {} + MS_DECLARE_PARENT(RefType, Object) + + TypePtr subtype() const { return subtype_; } + TypeId generic_type_id() const override { return kObjectTypeRef; } + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string DumpText() const override; + + private: + TypePtr subtype_; + TypePtr subtype_origin_; +}; +using RefTypePtr = std::shared_ptr; + +extern const TypePtr kRefKeyType; +extern const TypePtr kRefType; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_DTYPE_REF_H_ diff --git a/mindspore/ccsrc/ir/dtype/type.cc b/mindspore/ccsrc/ir/dtype/type.cc new file mode 100644 index 00000000000..6169acabbc6 --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/type.cc @@ -0,0 +1,268 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/dtype/type.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +TypeId IntBitsToTypeId(const int nbits) { + switch (nbits) { + case 8: + return kNumberTypeInt8; + case 16: + return kNumberTypeInt16; + case 32: + return kNumberTypeInt32; + case 64: + return kNumberTypeInt64; + default: + MS_LOG(EXCEPTION) << "wrong number of bits."; + } +} + +TypeId UIntBitsToTypeId(const int nbits) { + switch (nbits) { + case 8: + return kNumberTypeUInt8; + case 16: + return kNumberTypeUInt16; + case 32: + return kNumberTypeUInt32; + case 64: + return kNumberTypeUInt64; + default: + MS_LOG(EXCEPTION) << "wrong number of bits."; + } +} + +TypeId FloatBitsToTypeId(const int nbits) { + switch (nbits) { + case 16: + return kNumberTypeFloat16; + case 32: + return kNumberTypeFloat32; + case 64: + return kNumberTypeFloat64; + default: + MS_LOG(EXCEPTION) << "wrong number of bits."; + } +} + +const char* MetaIdLabel(const TypeId& v) { + switch (v) { + case kTypeUnknown: + return "kTypeUnknown"; + case kMetaTypeType: + return "kMetaTypeType"; + case kMetaTypeAnything: + return "kMetaTypeAnything"; + case kMetaTypeObject: + return "kMetaTypeObject"; + case kMetaTypeTypeType: + return "kMetaTypeTypeType"; + case kMetaTypeProblem: + return "kMetaTypeProblem"; + case kMetaTypeExternal: + return "kMetaTypeExternal"; + case kMetaTypeNone: + return "kMetaTypeNone"; + default: + return "[Unknown Type Id]"; + } +} + +const char* ObjectIdLabel(const TypeId& v) { + switch (v) { + case kObjectTypeNumber: + return "kObjectTypeNumber"; + case kObjectTypeString: + return "kObjectTypeString"; + case kObjectTypeList: + return "kObjectTypeList"; + case kObjectTypeTuple: + return "kObjectTypeTuple"; + case kObjectTypeSlice: + return "kObjectTypeSlice"; + case kObjectTypeKeyword: + return "kObjectTypeKeyword"; + case kObjectTypeTensorType: + return "kObjectTypeTensorType"; + case kObjectTypeDictionary: + return "kObjectTypeDictionary"; + case kObjectTypeClass: + return "kObjectTypeClass"; + case kObjectTypeFunction: + return "kObjectTypeFunction"; + case kObjectTypeJTagged: + return "kObjectTypeJTagged"; + case kObjectTypeSymbolicKeyType: + return "kObjectTypeSymbolicKeyType"; + case kObjectTypeEnvType: + return "kObjectTypeEnvType"; + case kObjectTypeRefKey: + return "kObjectTypeRefKey"; + case kObjectTypeRef: + return "kObjectTypeRef"; + default: + return "[Unknown Type Id]"; + } +} + +const char* NumberIdLabel(const TypeId& v) { + switch (v) { + case kNumberTypeBool: + return "kNumberTypeBool"; + case kNumberTypeInt: + return "kNumberTypeInt"; + case kNumberTypeInt8: + return "kNumberTypeInt8"; + case kNumberTypeInt16: + return "kNumberTypeInt16"; + case kNumberTypeInt32: + return "kNumberTypeInt32"; + case kNumberTypeInt64: + return "kNumberTypeInt64"; + case kNumberTypeUInt: + return "kNumberTypeUInt"; + case kNumberTypeUInt8: + return "kNumberTypeUInt8"; + case kNumberTypeUInt16: + return "kNumberTypeUInt16"; + case kNumberTypeUInt32: + return "kNumberTypeUInt32"; + case kNumberTypeUInt64: + return "kNumberTypeUInt64"; + case kNumberTypeFloat: + return "kNumberTypeFloat"; + case kNumberTypeFloat16: + return "kNumberTypeFloat16"; + case kNumberTypeFloat32: + return "kNumberTypeFloat32"; + case kNumberTypeFloat64: + return "kNumberTypeFloat64"; + default: + return "[Unknown Type Id]"; + } +} + +const char* TypeIdLabel(const TypeId& v) { + if (v < kMetaTypeEnd) { + return MetaIdLabel(v); + } else { + if (v < kObjectTypeEnd) { + return ObjectIdLabel(v); + } else { + return NumberIdLabel(v); + } + } +} + +TypeId NormalizeTypeId(const TypeId type_id) { + if ((type_id == kNumberTypeInt) || (type_id == kNumberTypeInt8) || (type_id == kNumberTypeInt16) || + (type_id == kNumberTypeInt32) || (type_id == kNumberTypeInt64)) { + return kNumberTypeInt; + } else if ((type_id == kNumberTypeFloat) || (type_id == kNumberTypeFloat16) || (type_id == kNumberTypeFloat32) || + (type_id == kNumberTypeFloat64)) { + return kNumberTypeFloat; + } else { + return type_id; + } +} + +bool IsSameObjectType(const Type& lhs, const Type& rhs) { + if ((lhs.meta_type() != kMetaTypeObject) || (rhs.meta_type() != kMetaTypeObject)) { + return false; + } + return lhs.object_type() == rhs.object_type(); +} + +size_t GetTypeByte(const TypePtr& type_ptr) { + if (type_ptr && type_ptr->isa()) { + auto number = dyn_cast(type_ptr); + if (!number) { + MS_LOG(DEBUG) << "Invalid TypePtr got from ApplyKernel."; + return 0; + } else { + return IntToSize(number->nbits() / 8); + } + } else { + MS_LOG(DEBUG) << "Invalid TypePtr got from ApplyKernel."; + return 0; + } +} + +bool Type::operator==(const Value& other) const { + if (other.isa()) { + auto other_type = static_cast(&other); + return *this == *other_type; + } else { + return false; + } +} + +abstract::AbstractBasePtr Type::ToAbstract() { + abstract::AbstractBasePtr ptr = std::make_shared(shared_from_base()); + return ptr; +} + +std::ostream& operator<<(std::ostream& os, const Type& type) { + os << type.ToString(); + return os; +} + +std::ostream& operator<<(std::ostream& os, const TypePtr type) { + os << type->ToString(); + return os; +} + +bool Object::equal(const TypePtr other) const { + auto same_other = dyn_cast(other); + if (same_other != nullptr) { + return *this == *same_other; + } + return false; +} + +std::ostream& operator<<(std::ostream& os, const Object& obj) { + os << obj.ToString(); + return os; +} + +std::ostream& operator<<(std::ostream& os, const std::shared_ptr obj) { + os << obj->ToString(); + return os; +} + +std::ostream& operator<<(std::ostream& os, const TypePtrList& types) { + os << "["; + for (size_t i = 0; i < types.size(); ++i) { + if (i > 0) { + os << ", "; + } + os << (types[i] == nullptr ? "nullptr" : types[i]->ToString()); + } + os << "]"; + return os; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/type.h b/mindspore/ccsrc/ir/dtype/type.h new file mode 100644 index 00000000000..9454596538b --- /dev/null +++ b/mindspore/ccsrc/ir/dtype/type.h @@ -0,0 +1,176 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_DTYPE_TYPE_H_ +#define MINDSPORE_CCSRC_IR_DTYPE_TYPE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/named.h" + +namespace mindspore { +// +// Supported meta type +// +enum TypeId : int { + kTypeUnknown = 0, + kMetaTypeBegin = kTypeUnknown, + kMetaTypeType, // Type + kMetaTypeAnything, + kMetaTypeObject, + kMetaTypeTypeType, // TypeType + kMetaTypeProblem, + kMetaTypeExternal, + kMetaTypeNone, + kMetaTypeNull, + kMetaTypeEnd, + // + // Object types + // + kObjectTypeBegin = kMetaTypeEnd, + kObjectTypeNumber, + kObjectTypeString, + kObjectTypeList, + kObjectTypeTuple, + kObjectTypeSlice, + kObjectTypeKeyword, + kObjectTypeTensorType, + kObjectTypeClass, + kObjectTypeDictionary, + kObjectTypeFunction, + kObjectTypeJTagged, + kObjectTypeSymbolicKeyType, + kObjectTypeEnvType, + kObjectTypeRefKey, + kObjectTypeRef, + kObjectTypeEnd, + // + // Number Types + // + kNumberTypeBegin = kObjectTypeEnd, + kNumberTypeBool, + kNumberTypeInt, + kNumberTypeInt8, + kNumberTypeInt16, + kNumberTypeInt32, + kNumberTypeInt64, + kNumberTypeUInt, + kNumberTypeUInt8, + kNumberTypeUInt16, + kNumberTypeUInt32, + kNumberTypeUInt64, + kNumberTypeFloat, + kNumberTypeFloat16, + kNumberTypeFloat32, + kNumberTypeFloat64, + kNumberTypeEnd +}; + +TypeId IntBitsToTypeId(const int nbits); +TypeId UIntBitsToTypeId(const int nbits); +TypeId FloatBitsToTypeId(const int nbits); +const char* TypeIdLabel(const TypeId& v); +TypeId NormalizeTypeId(const TypeId type_id); +bool IsSameObjectType(const Type& lhs, const Type& rhs); +size_t GetTypeByte(const TypePtr& type_ptr); + +// Base class for all types +// forward declaration. + +class Type : public Value { + public: + Type() : meta_type_(kMetaTypeType), is_generic_(true) {} + explicit Type(TypeId t, bool is_generic = true) : meta_type_(t), is_generic_(is_generic) {} + ~Type() override = default; + MS_DECLARE_PARENT(Type, Value) + + bool operator==(const Value& other) const override; + TypeId meta_type() const { return meta_type_; } + + virtual TypeId type_id() const { return meta_type_; } + virtual TypeId generic_type_id() const { return kMetaTypeType; } + + virtual bool operator!=(const Type& other) const { return !(*this == other); } + virtual bool operator==(const Type& other) const { return this->type_id() == other.type_id(); } + virtual bool equal(const TypePtr other) const { return *this == *other; } + + virtual TypeId object_type() const { return kTypeUnknown; } + virtual TypeId number_type() const { return kTypeUnknown; } + virtual TypePtr DeepCopy() const = 0; + virtual TypePtr Clone() const { return DeepCopy(); } + + std::size_t hash() const override { return std::hash{}(static_cast(type_id())); } + + std::string ToString() const override { return TypeIdLabel(meta_type_); } + virtual std::string ToReprString() const { return ToString(); } + std::string ReprString() const { return "mindspore." + ToReprString(); } + void dump() const override { std::cout << ToString() << std::endl; } + bool IsUnknown() const { return (meta_type_ == kMetaTypeType); } + bool IsGeneric() const { return is_generic_; } + abstract::AbstractBasePtr ToAbstract() override; + friend std::ostream& operator<<(std::ostream& os, const Type& type); + friend std::ostream& operator<<(std::ostream& os, const TypePtr type); + + const bool parse_info_ = true; + + private: + TypeId meta_type_; + bool is_generic_; +}; + +using TypePtrList = std::vector; + +// +// Base class for normal objects +// +class Object : public Type { + public: + Object() : Type(kMetaTypeObject), object_type_(kMetaTypeObject) {} + explicit Object(const TypeId object_type, bool is_generic = true) + : Type(kMetaTypeObject, is_generic), object_type_(object_type) {} + ~Object() override = default; + MS_DECLARE_PARENT(Object, Type) + + TypeId object_type() const override { return object_type_; } + TypeId type_id() const override { return object_type_; } + TypeId generic_type_id() const override { return kMetaTypeObject; } + bool equal(const TypePtr other) const override; + std::string ToString() const override { return std::string("Object:") + TypeIdLabel(object_type_); } + + friend std::ostream& operator<<(std::ostream& os, const Object& obj); + friend std::ostream& operator<<(std::ostream& os, const std::shared_ptr obj); + + private: + const TypeId object_type_; +}; + +std::ostream& operator<<(std::ostream& os, const TypePtrList& types); +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_DTYPE_TYPE_H_ diff --git a/mindspore/ccsrc/ir/func_graph.cc b/mindspore/ccsrc/ir/func_graph.cc new file mode 100644 index 00000000000..f5153676354 --- /dev/null +++ b/mindspore/ccsrc/ir/func_graph.cc @@ -0,0 +1,846 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/func_graph.h" + +#include +#include +#include + +#include "ir/manager.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" +#include "utils/ordered_set.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "pipeline/static_analysis/abstract_function.h" + +#include "debug/anf_ir_dump.h" +#include "debug/trace.h" +#include "debug/draw.h" +#include "debug/label.h" + +namespace mindspore { +using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractFunctionPtr; +using mindspore::abstract::AnalysisContextPtr; +using mindspore::abstract::PrimitiveAbstractClosure; +using mindspore::abstract::VirtualAbstractClosure; +/* + * Methods of Graph + */ +FuncGraph::FuncGraph() + : flags_(), + transforms_(), + parameter_default_value_(), + parameters_(), + has_vararg_(false), + has_kwarg_(false), + kwonlyargs_count_(0), + hyper_param_count_(0), + is_generated_(false), + return_(nullptr), + manager_(std::weak_ptr()) { + debug_info_ = std::make_shared(); +} + +AbstractFunctionPtr FuncGraph::abstract() { + AbstractBasePtrList args_spec_list; + + for (auto& p : parameters_) { + MS_EXCEPTION_IF_NULL(p); + if (p->abstract() == nullptr) { + MS_LOG(ERROR) << "error!!"; + return nullptr; + } + args_spec_list.push_back(p->abstract()); + } + + if (nullptr == output()) { + MS_LOG(ERROR) << "error func graph no output"; + return nullptr; + } + + return std::make_shared(args_spec_list, output()->abstract()); +} + +abstract::AbstractBasePtr FuncGraph::MakeAbstractClosure(const abstract::AnalysisContextPtr& context) { + AnalysisContextPtr temp_context = context; + if (temp_context == nullptr) { + temp_context = abstract::AnalysisContext::DummyContext(); + } + return std::make_shared(shared_from_base(), temp_context); +} + +AnfNodePtr FuncGraph::output() const { + // If return value is set, return should have two inputs. + if (return_ != nullptr && return_->inputs().size() == 2) { + return return_->input(1); + } else { + // If not set yet, return nullptr. + return nullptr; + } +} + +void FuncGraph::set_output(const AnfNodePtr& value, bool force_new_ret) { + if (force_new_ret || return_ == nullptr) { + std::vector params({NewValueNode(prim::kPrimReturn), value}); + FuncGraphPtr this_graph = shared_from_base(); + return_ = this_graph->NewCNode(params); + } else { + if (manager_.lock()) { + manager_.lock()->SetEdge(return_, 1, value); + } else { + return_->set_input(1, value); + } + } + + return_->set_abstract(value->abstract()); + + AnfNodePtr input0 = return_->input(0); + + PrimitivePtr return_prim = prim::kPrimReturn; + auto f = std::make_shared(return_prim, input0); + input0->set_abstract(f); +} + +ParameterPtr FuncGraph::add_parameter() { + FuncGraphPtr this_func_graph = shared_from_base(); + ParameterPtr p = std::make_shared(this_func_graph); + add_parameter(p); + return p; +} + +void FuncGraph::add_parameter(const ParameterPtr& p) { + if (manager_.lock()) { + std::vector new_params = parameters_; + new_params.push_back(p); + manager_.lock()->SetParameters(shared_from_base(), new_params); + } else { + parameters_.push_back(p); + } +} + +ParameterPtr FuncGraph::AddWeightParameter(const std::string& name) { + FuncGraphPtr this_graph = shared_from_base(); + ParameterPtr p = std::make_shared(this_graph); + p->set_name(name); + p->debug_info()->set_name(name); + + std::vector new_params = parameters_; + // append parameter + new_params.push_back(p); + + if (manager_.lock()) { + manager_.lock()->SetParameters(shared_from_base(), new_params); + } else { + parameters_.push_back(p); + } + hyper_param_count_++; + return p; +} + +bool FuncGraph::has_flag(const std::string& flag) { + if (flags_.count(flag)) { + return flags_[flag]; + } + return false; +} + +CNodePtr FuncGraph::NewCNode(const std::vector& inputs) { + CNodePtr cnode = std::make_shared(inputs, shared_from_base()); + if (has_flag(GRAPH_FLAG_HAS_EFFECT)) { + order_.push_back(cnode); + MS_LOG(INFO) << "Graph: " << ToString() << ", push back " << cnode->DebugString() << " in order."; + } + return cnode; +} + +CNodePtr FuncGraph::NewCNodeWithScope(const std::vector& inputs, const ScopePtr& scope) { + CNodePtr app = NewCNode(inputs); + app->set_scope(scope); + return app; +} + +void FuncGraph::DumpCNodeList() { + MS_LOG(INFO) << "FuncGraph " << ToString() << " has following CNode in code order:"; + for (const auto& cnode : order_) { + MS_LOG(INFO) << cnode->DebugString(); + } +} + +std::string FuncGraph::ToString() const { + return mindspore::label_manage::Label(const_cast(this)->shared_from_base()->debug_info()); +} + +GraphDebugInfoPtr FuncGraph::debug_info() { + MS_EXCEPTION_IF_NULL(this->debug_info_); + if (this->debug_info_->get_graph() == nullptr) { + this->debug_info_->set_graph(shared_from_base()); + } + return this->debug_info_; +} + +const AnfNodeSet& FuncGraph::nodes() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& nodes = mng->nodes(); + return nodes[shared_from_base()]; +} + +const AnfNodeCounterMap& FuncGraph::value_nodes() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& cts = mng->valuenodes(); + return cts[shared_from_base()]; +} + +const AnfNodeCounterMap& FuncGraph::free_variables_direct() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& fv_direct = mng->free_variables_direct(); + return fv_direct[shared_from_base()]; +} + +const BaseRefCounterMap& FuncGraph::free_variables_total() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& fv_total = mng->free_variables_total(); + return fv_total[shared_from_base()]; +} + +std::vector FuncGraph::free_variables_nodes() { + std::vector nodes; + const auto& fv_total = this->free_variables_total(); + for (auto& p : fv_total) { + auto key = p.first; + if (utils::isa(key)) { + nodes.push_back(utils::cast(key)); + } + } + + return nodes; +} + +std::vector FuncGraph::free_variables_func_graphs() { + std::vector func_graphs; + const auto& fv_total = this->free_variables_total(); + for (auto& p : fv_total) { + auto key = p.first; + if (utils::isa(key)) { + func_graphs.push_back(utils::cast(key)); + } + } + + return func_graphs; +} + +const FuncGraphCounterMap& FuncGraph::func_graphs_used() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& used = mng->func_graphs_used(); + return used[shared_from_base()]; +} + +const FuncGraphSet& FuncGraph::func_graphs_used_total() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& used = mng->func_graphs_used_total(shared_from_base()); + return used; +} + +const FuncGraphCounterMap& FuncGraph::func_graph_users() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& users = mng->func_graph_users(); + return users[shared_from_base()]; +} + +const AnfNodeCounterMap& FuncGraph::func_graph_user_cnodes() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + auto& users = mng->func_graph_user_cnodes(); + return users[shared_from_base()]; +} + +FuncGraphPtr FuncGraph::parent() { + // report the bug early. + if (manager_.lock() == nullptr) { + MS_LOG(EXCEPTION) << "BUG: no manager for this func graph: " << ToString() + << " NodeInfo: " << trace::GetDebugInfo(debug_info()); + } + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + return mng->parent(shared_from_base()); +} + +const FuncGraphSet& FuncGraph::children() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + return mng->children(shared_from_base()); +} + +const FuncGraphSet& FuncGraph::scope() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + return mng->scopes(shared_from_base()); +} + +bool FuncGraph::recursive() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + return mng->recursive(shared_from_base()); +} + +std::shared_ptr> FuncGraph::recursive_graphs() { + auto mng = manager_.lock(); + MS_EXCEPTION_IF_NULL(mng); + return mng->recursive_graphs(shared_from_base()); +} + +void FuncGraph::DumpFuncGraph(const std::string& path) { draw::Draw(path + ".dot", shared_from_base()); } + +AnfNodePtr FuncGraph::GetDefaultValueByName(const std::string& name) { + auto itr = this->parameter_default_value_.find(name); + if (itr == parameter_default_value_.end()) { + return nullptr; + } + auto default_value = itr->second; + if (default_value == nullptr) { + MS_LOG(EXCEPTION) << "Graph parameter " << name << " not exist"; + } + if (IsValueNode(default_value)) { + return nullptr; + } + return default_value; +} + +// set the default values +void FuncGraph::SetDefaultValues(const std::vector& name_list, const std::vector& value_list) { + auto all_is_null = std::all_of(value_list.begin(), value_list.end(), + [](const AnfNodePtr& node) { return IsValueNode(node); }); + if (value_list.empty()) { + all_is_null = true; + } + for (size_t i = 0; i < name_list.size(); ++i) { + if (!all_is_null) { + this->parameter_default_value_[name_list[i]] = value_list[i]; + } + } +} + +void FuncGraph::ClearDefaultValues() { parameter_default_value_.clear(); } + +size_t FuncGraph::GetDefaultValueCount() { + int null_count = + std::count_if(parameter_default_value_.begin(), parameter_default_value_.end(), + [](const std::pair& pair) { return IsValueNode(pair.second); }); + return parameter_default_value_.size() - IntToSize(null_count); +} + +AnfNodePtr FuncGraph::GetVariableArgParameter() { + if (!has_vararg_) { + return nullptr; + } + + if (has_kwarg_) { + if (parameters_.size() < hyper_param_count_ + 2) { + MS_LOG(EXCEPTION) << "Length of parameters is " << parameters_.size() << ", hyper_param_count is " + << hyper_param_count_ << ", parameters is less than 2 + hyper_param_count"; + } + return parameters_[parameters_.size() - hyper_param_count_ - 2]; + } + + if (parameters_.size() < hyper_param_count_ + 1) { + MS_LOG(EXCEPTION) << "Length of parameters is " << parameters_.size() << ", hyper_param_count is " + << hyper_param_count_ << ", parameters is less than 1 + hyper_param_count"; + } + return parameters_[parameters_.size() - hyper_param_count_ - 1]; +} + +std::string FuncGraph::GetVariableArgName() { + if (!has_vararg_) { + return ""; + } + + if (has_kwarg_) { + if (parameters_.size() < hyper_param_count_ + 2) { + MS_LOG(EXCEPTION) << "Length of parameters is " << parameters_.size() << ", hyper_param_count is " + << hyper_param_count_ << ", parameters is less than 2 + hyper_param_count"; + } + return parameters_[parameters_.size() - hyper_param_count_ - 2]->cast()->name(); + } + + if (parameters_.size() < hyper_param_count_ + 1) { + MS_LOG(EXCEPTION) << "Length of parameters is " << parameters_.size() << ", hyper_param_count is " + << hyper_param_count_ << ", parameters is less than 1 + hyper_param_count"; + } + return parameters_[parameters_.size() - hyper_param_count_ - 1]->cast()->name(); +} + +AnfNodePtr FuncGraph::GetVariableKwargParameter() { + if (has_kwarg_) { + if (parameters_.size() < hyper_param_count_ + 1) { + MS_LOG(EXCEPTION) << "Length of parameters is " << parameters_.size() << ", hyper_param_count is " + << hyper_param_count_ << ", parameters is less than 1 + hyper_param_count"; + } + return parameters_[parameters_.size() - hyper_param_count_ - 1]; + } + return nullptr; +} + +std::string FuncGraph::GetVariableKwargName() { + if (has_kwarg_) { + if (parameters_.size() < hyper_param_count_ + 1) { + MS_LOG(EXCEPTION) << "Length of parameters is " << parameters_.size() << ", hyper_param_count is " + << hyper_param_count_ << ", parameters is less than 1 + hyper_param_count"; + } + return parameters_[parameters_.size() - hyper_param_count_ - 1]->cast()->name(); + } + return ""; +} + +int FuncGraph::GetPositionalArgsCount() const { + int count = SizeToInt(parameters_.size()); + if (has_kwarg_) { + count--; + } + if (has_vararg_) { + count--; + } + return count - kwonlyargs_count_ - SizeToInt(hyper_param_count_); +} + +AnfNodePtr FuncGraph::GetParameterByName(const std::string& name) { + for (size_t i = 0; i < parameters_.size(); ++i) { + MS_EXCEPTION_IF_NULL(parameters_[i]); + auto param_cast = parameters_[i]->cast(); + MS_EXCEPTION_IF_NULL(param_cast); + if (param_cast->name() == name) { + return parameters_[i]; + } + } + return nullptr; +} + +void FuncGraph::GenerateVarParams(const FuncGraphPtr& specialized_graph, + std::vector* specialized_parameter_list, + std::unordered_map* repl_nodes, int variable_args_count, + int pos_args_input_count) { + // if there is variable argument, pass the input arguments that does not match positional args to it as a tuple + if (specialized_graph->has_vararg()) { + TraceManager::DebugTrace( + std::make_shared(specialized_graph->GetVariableArgParameter()->debug_info())); + std::vector var_param_tuple_nodes; + var_param_tuple_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + + if (variable_args_count < 0) { + MS_LOG(EXCEPTION) << "Function:" << this->ToString() << ", variable_args_count " << variable_args_count + << " were given."; + } + // for python variable argument input , there is no upper limit + for (int i = 0; i < variable_args_count; ++i) { + ParameterPtr p = std::make_shared(specialized_graph); + std::string param_name = specialized_graph->GetVariableArgName() + std::to_string(i); + p->set_name(param_name); + MS_EXCEPTION_IF_NULL(p->debug_info()); + p->debug_info()->set_name(param_name); + var_param_tuple_nodes.push_back(p); + MS_EXCEPTION_IF_NULL(specialized_parameter_list); + specialized_parameter_list->push_back(p); + } + auto var_tuple_param = specialized_graph->NewCNode(var_param_tuple_nodes); + (void)repl_nodes->emplace(specialized_graph->GetVariableArgParameter(), var_tuple_param); + TraceManager::EndTrace(); + } else if (variable_args_count > 0) { + MS_LOG(EXCEPTION) << "Function:" << this->ToString() << " takes " << this->GetPositionalArgsCount() + << " positional arguments, but " << pos_args_input_count << " were given."; + } +} + +void FuncGraph::GenerateKwParams(const FuncGraphPtr& specialized_graph, + std::vector* specialized_parameter_list, + const std::vector& kwarg_list, + std::unordered_map* repl_nodes) { + std::vector kwarg_keys_tuple_nodes = {NewValueNode(prim::kPrimMakeTuple)}; + std::vector kwarg_values_tuple_nodes = {NewValueNode(prim::kPrimMakeTuple)}; + + for (const auto& kwarg : kwarg_list) { + MS_EXCEPTION_IF_NULL(kwarg); + std::string kw_param_name = kwarg->get_key(); + MS_EXCEPTION_IF_NULL(specialized_graph); + AnfNodePtr param_node = specialized_graph->GetParameterByName(kw_param_name); + // if not find correspoding parameter node + if (param_node == nullptr) { + if (!has_kwarg()) { + MS_LOG(EXCEPTION) << "Got unexpected keyword argument: " << kw_param_name; + } else { + ParameterPtr p = std::make_shared(specialized_graph); + std::string param_name = specialized_graph->GetVariableKwargName() + "[" + kw_param_name + "]"; + MS_EXCEPTION_IF_NULL(specialized_parameter_list); + auto find_kw_arg_in_list = std::any_of(specialized_parameter_list->begin(), specialized_parameter_list->end(), + [param_name](const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + auto param = node->cast(); + return param != nullptr && param->name() == param_name; + }); + if (find_kw_arg_in_list) { + MS_LOG(EXCEPTION) << "Multiply values for keyword argument:" << kw_param_name; + } + p->set_name(param_name); + p->debug_info()->set_name(param_name); + kwarg_keys_tuple_nodes.push_back(NewValueNode(kw_param_name)); + auto extract_node = + specialized_graph->NewCNode({NewValueNode(prim::kPrimExtractKeywordArg), NewValueNode(kw_param_name), p}); + kwarg_values_tuple_nodes.push_back(extract_node); + specialized_parameter_list->push_back(p); + } + } else { + auto node_itr = std::find(specialized_parameter_list->begin(), specialized_parameter_list->end(), param_node); + // multiply values found given for parameter + if (node_itr != specialized_parameter_list->end()) { + MS_LOG(EXCEPTION) << "Multiply values for specific argument:" << kw_param_name; + } else { + specialized_parameter_list->push_back(param_node); + auto extract_node = specialized_graph->NewCNode( + {NewValueNode(prim::kPrimExtractKeywordArg), NewValueNode(kw_param_name), param_node}); + (void)repl_nodes->emplace(param_node, extract_node); + } + } + } + + GenerateKwargReplNode(specialized_graph, repl_nodes, kwarg_keys_tuple_nodes, kwarg_values_tuple_nodes); +} + +void FuncGraph::GenerateKwargReplNode(const FuncGraphPtr& specialized_graph, + std::unordered_map* repl_nodes, + const std::vector& kwarg_keys_tuple_nodes, + const std::vector& kwarg_values_tuple_nodes) { + if (has_kwarg()) { + MS_EXCEPTION_IF_NULL(specialized_graph); + TraceManager::DebugTrace( + std::make_shared(specialized_graph->GetVariableKwargParameter()->debug_info())); + auto make_tuple_keys = specialized_graph->NewCNode(kwarg_keys_tuple_nodes); + auto make_tuple_values = specialized_graph->NewCNode(kwarg_values_tuple_nodes); + auto make_dict_node = + specialized_graph->NewCNode({NewValueNode(prim::kPrimMakeDict), make_tuple_keys, make_tuple_values}); + MS_EXCEPTION_IF_NULL(repl_nodes); + (void)repl_nodes->emplace(specialized_graph->GetVariableKwargParameter(), make_dict_node); + TraceManager::EndTrace(); + } +} +bool FuncGraph::NeedGenerate(const std::vector& kwarg_list) { + // if the function does not have any vararg/kwarg/kwonly/default value/kw args input + // return the original graph + if (!has_vararg() && kwonlyargs_count() == 0 && !has_kwarg() && GetDefaultValueCount() == 0 && kwarg_list.empty()) { + return false; + } + + // if the graph is generated for specific input, do not need to generate again + if (is_generated()) { + return false; + } + return true; +} +void FuncGraph::GenerateDefaultValue(const FuncGraphPtr& specialized_graph, + const std::vector& specialized_parameter_list, + std::unordered_map* repl_nodes) { + MS_EXCEPTION_IF_NULL(specialized_graph); + for (size_t i = 0; i < specialized_graph->parameters().size() - hyper_param_count(); ++i) { + auto param_node = specialized_graph->parameters()[i]; + MS_EXCEPTION_IF_NULL(param_node); + auto param_name = param_node->cast()->name(); + auto node_itr = std::find(specialized_parameter_list.begin(), specialized_parameter_list.end(), param_node); + if (node_itr != specialized_parameter_list.end()) { + continue; + } + if (param_name == specialized_graph->GetVariableArgName() || + param_name == specialized_graph->GetVariableKwargName()) { + continue; + } + auto default_value = specialized_graph->GetDefaultValueByName(param_name); + if (default_value == nullptr) { + MS_LOG(EXCEPTION) << "Miss argument input for parameter:" << param_name; + } + MS_EXCEPTION_IF_NULL(repl_nodes); + (void)repl_nodes->emplace(param_node, default_value); + } +} + +FuncGraphPtr FuncGraph::GenerateGraph(const AbstractBasePtrList& args_spec_list) { + std::vector kwarg_list; + size_t arguments_count = args_spec_list.size(); + for (const auto& arg : args_spec_list) { + // if it is a keyword argument + MS_EXCEPTION_IF_NULL(arg); + if (arg->isa()) { + kwarg_list.push_back(dyn_cast(arg)); + } + } + if (!NeedGenerate(kwarg_list)) { + return shared_from_base(); + } + FuncGraphPtr specialized_graph = BasicClone(shared_from_base()); + size_t kwarg_count = kwarg_list.size(); + int pos_args_input_count = SizeToInt(arguments_count - kwarg_count - hyper_param_count()); + int pos_args_count = std::min(pos_args_input_count, this->GetPositionalArgsCount()); + int variable_args_count = pos_args_input_count - pos_args_count; + std::vector specialized_parameter_list; + std::unordered_map repl_nodes; + // the parameters that has arg input, copy from original parameters + for (size_t i = 0; i < IntToSize(pos_args_count); ++i) { + specialized_parameter_list.push_back(specialized_graph->parameters()[i]); + } + + GenerateVarParams(specialized_graph, &specialized_parameter_list, &repl_nodes, variable_args_count, + pos_args_input_count); + + GenerateKwParams(specialized_graph, &specialized_parameter_list, kwarg_list, &repl_nodes); + + GenerateDefaultValue(specialized_graph, specialized_parameter_list, &repl_nodes); + + // append hyper parameter to specialized_parameter_list + MS_EXCEPTION_IF_NULL(specialized_graph); + auto params = specialized_graph->parameters(); + (void)std::transform(params.end() - SizeToInt(hyper_param_count()), params.end(), + std::back_inserter(specialized_parameter_list), [](const AnfNodePtr& node) { return node; }); + + std::shared_ptr manager = mindspore::Manage(specialized_graph, false); + auto tr = manager->Transact(); + for (auto& node_pair : repl_nodes) { + MS_LOG(DEBUG) << "GenerateGraph replace:" << node_pair.first->DebugString() << "-" + << node_pair.second->DebugString(); + (void)tr.Replace(node_pair.first, node_pair.second); + } + tr.SetParameters(specialized_graph, specialized_parameter_list); + tr.Commit(); + specialized_graph->set_has_kwarg(false); + specialized_graph->set_has_vararg(false); + specialized_graph->set_kwonlyargs_count(0); + specialized_graph->ClearDefaultValues(); + specialized_graph->set_is_generate(true); + return specialized_graph; +} + +void FuncGraph::add_parameter_obj_node(const AnfNodePtr& p) { paramter_obj_nodes_.push_back(p); } + +std::list FuncGraph::GetOrderedCnodes(bool force_use_topo_sort) { + if (has_flag(GRAPH_FLAG_HAS_EFFECT) && !force_use_topo_sort) { + MS_LOG(DEBUG) << "Return ordered cnodes."; + return order_; + } else { + auto this_ptr = shared_from_base(); + auto BelongSameGraph = std::bind(IncludeBelongGraph, this_ptr, std::placeholders::_1); + auto SuccDepends = std::bind(SuccIncludeFV, this_ptr, std::placeholders::_1); + + std::list cnodes; + auto nodes = TopoSort(get_return(), SuccDepends, BelongSameGraph); + for (const auto& node : nodes) { + auto cnode = dyn_cast(node); + if (cnode) { + cnodes.push_back(cnode); + } + } + return cnodes; + } +} + +void FuncGraph::EraseUnusedNodeInOrder() { + if (has_flag(GRAPH_FLAG_HAS_EFFECT)) { + auto mng = manager_.lock(); + if (mng) { + auto nodes = mng->nodes()[shared_from_base()]; + // Erase unusued cnode. + for (auto it = order_.begin(); it != order_.end();) { + if (nodes.count(*it)) { + (void)it++; + } else { + MS_LOG(DEBUG) << "Remove node " << (*it)->ToString() << " in graph " << ToString() << " order."; + it = order_.erase(it); + } + } + } + } +} + +void FuncGraph::EraseUnusedNodeInOrder(const AnfNodePtr& n) { + if (has_flag(GRAPH_FLAG_HAS_EFFECT) && n && n->isa()) { + order_.remove(n->cast()); + MS_LOG(DEBUG) << "Remove the node" << n->DebugString() << " from order list."; + } +} + +void FuncGraph::CheckOrder() { + if (has_flag(GRAPH_FLAG_HAS_EFFECT)) { + MS_LOG(DEBUG) << "Check graph " << ToString(); + for (auto it = order_.begin(); it != order_.end(); (void)it++) { + for (const auto& input_node : (*it)->inputs()) { + if (input_node && input_node->isa() && input_node->func_graph() == shared_from_base()) { + // Need to reorder the wrong order node. + auto found = std::find(order_.begin(), it, input_node); + if (found == it) { + DumpCNodeList(); + MS_LOG(EXCEPTION) << "The cnode " << (*it)->DebugString() << " order in " << ToString() + << " doesn't obey the input denpency, " + << "as input " << input_node->DebugString() << " is not ahead of itself."; + } + } + } + } + auto topo_sort = GetOrderedCnodes(true); + if (topo_sort.size() != order_.size()) { + DumpCNodeList(); + DumpIR(ToString(), shared_from_base()); + MS_LOG(INFO) << "Dump graph: " << ToString() << "."; + DumpFuncGraph(ToString()); + MS_LOG(EXCEPTION) << "CNode order size " << order_.size() << " is not equal to topo sort list size " + << topo_sort.size() << "."; + } + MS_LOG(DEBUG) << "Check order okay."; + } +} + +const char kPrimHasEffect[] = "_side_effect_flag"; + +bool FuncGraph::HasEffect(const CNodePtr& cnode) { + auto prim = GetCNodePrimitive(cnode); + if (prim != nullptr && prim->isa()) { + auto do_sig = prim->cast(); + auto prim_val = do_sig->function(); + if (prim_val != nullptr && prim_val->isa()) { + prim = prim_val->cast(); + } else { + prim = nullptr; + } + } + if (prim != nullptr) { + auto effect_val = prim->GetAttr(kPrimHasEffect); + if (effect_val && effect_val->isa()) { + auto effect_bool = GetValue(effect_val); + return effect_bool; + } + } + return false; +} + +std::shared_ptr> FindRoots(const std::vector& segment) { + std::shared_ptr> roots = std::make_shared>(segment); + for (const auto& node : segment) { + if (roots->size() == 1) { + return roots; + } + auto input_size = node->size(); + for (size_t i = 0; i < input_size; i++) { + auto in_node = node->input(i); + auto in_cnode = in_node->cast(); + if (in_cnode != nullptr) { + (void)roots->erase(in_cnode); + } + } + } + return roots; +} + +std::shared_ptr> FindLeaves(const std::vector& segment) { + std::shared_ptr> nodes = std::make_shared>(segment); + for (const auto& node : segment) { + if (nodes->size() == 1) { + return nodes; + } + if (IsPrimitiveCNode(node, prim::kPrimSwitch)) { + (void)nodes->erase(node); + continue; + } + auto input_size = node->size(); + for (size_t i = 0; i < input_size; i++) { + auto in_node = node->input(i); + if (!in_node->isa()) { + continue; + } + auto in_cnode = in_node->cast(); + if (in_cnode != nullptr) { + if (std::find(segment.begin(), segment.end(), in_cnode) != segment.end()) { + (void)nodes->erase(node); + break; + } + } + } + } + return nodes; +} + +void FuncGraph::ReleaseFullOrderToEffectOrder() { + MS_LOG(DEBUG) << "Flag has_effect " << has_flag(GRAPH_FLAG_HAS_EFFECT) << "."; + if (has_flag(GRAPH_FLAG_HAS_EFFECT)) { + std::list depends_order; + std::vector segment; + for (const auto& cnode : order_) { + if (IsPrimitiveCNode(cnode, prim::kPrimReturn)) { + continue; + } + if (HasEffect(cnode)) { + MS_LOG(DEBUG) << "Meet a effect node " << cnode->DebugString() << "."; + if (segment.size() > 0) { + auto roots = FindRoots(segment); + for (auto iter = roots->begin(); iter != roots->end(); (void)iter++) { + depends_order.push_back(*iter); + } + } + segment.clear(); + depends_order.push_back(cnode); + } else { + MS_LOG(DEBUG) << "Meet a general node " << cnode->DebugString() << "."; + segment.push_back(cnode); + } + } + if (segment.size() > 1) { + auto roots = FindRoots(segment); + for (auto iter = roots->begin(); iter != roots->end(); (void)iter++) { + depends_order.push_back(*iter); + } + } + std::vector depend_inputs; + auto old_ret = output(); + for (auto iter = depends_order.rbegin(); iter != depends_order.rend(); (void)iter++) { + if (*iter != old_ret) { + depend_inputs.push_back(*iter); + } + } + set_flags(GRAPH_FLAG_HAS_EFFECT, false); + set_flags(GRAPH_FLAG_EFFECT_PATIAL_ORDER, true); + if (!depend_inputs.empty()) { + SetEffectDepends(depend_inputs); + } + } +} + +void FuncGraph::SetEffectDepends(const std::vector& depend_inputs) { + auto old_ret = output(); + std::vector inputs{NewValueNode(prim::kPrimDepend), old_ret}; + (void)inputs.insert(inputs.end(), depend_inputs.begin(), depend_inputs.end()); + auto new_ret = NewCNode(inputs); + auto mng = manager(); + if (mng) { + (void)mng->Replace(old_ret, new_ret); + } else { + return_->set_input(1, new_ret); + } +} + +const PrimitivePtr FuncGraphTransform::func_graph_prim_ = std::make_shared("FuncGraph"); +const char kFuncGraphFlagUndetermin[] = "Undeterminate"; +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/func_graph.h b/mindspore/ccsrc/ir/func_graph.h new file mode 100644 index 00000000000..95b26de4735 --- /dev/null +++ b/mindspore/ccsrc/ir/func_graph.h @@ -0,0 +1,323 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_FUNC_GRAPH_H_ +#define MINDSPORE_CCSRC_IR_FUNC_GRAPH_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/manager.h" +#include "utils/any.h" +#include "utils/ordered_set.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { +using BaseRefCounterMap = OrderedMap; +using FuncGraphCounterMap = OrderedMap; +using AnfNodeCounterMap = OrderedMap; + +const char FUNC_GRAPH_FLAG_IGNORE_VALUES[] = "ignore_values"; +const char FUNC_GRAPH_FLAG_DEFER_INLINE[] = "defer_inline"; +const char FUNC_GRAPH_FLAG_CORE[] = "core"; + +// ANF transform class +// either a primitive or a func_graph +class FuncGraphTransform { + public: + enum Type { kGtPrimitive, kGtFuncGraph }; + + explicit FuncGraphTransform(const PrimitivePtr prim, const FuncGraphPtr func_graph = nullptr) + : prim_(prim), func_graph_(FuncGraphWeakPtr(func_graph)) {} + + explicit FuncGraphTransform(const FuncGraphPtr &func_graph, const PrimitivePtr &prim = func_graph_prim_) + : prim_(prim), func_graph_(FuncGraphWeakPtr(func_graph)) {} + + FuncGraphTransform(const FuncGraphTransform &t) : prim_(t.prim_), func_graph_(t.func_graph_) {} + + ~FuncGraphTransform() = default; + + Type type() const { + if (IsFuncGraph()) { + return kGtFuncGraph; + } else { + return kGtPrimitive; + } + } + + bool IsPrimitive() const { return (func_graph_.lock() == nullptr); } + bool IsFuncGraph() const { return (func_graph_.lock() != nullptr); } + FuncGraphPtr func_graph() const { return func_graph_.lock(); } + PrimitivePtr primitive() const { return prim_; } + + FuncGraphTransform &operator=(const FuncGraphTransform &t) { + if (this != &t) { + prim_ = t.prim_; + func_graph_ = t.func_graph_; + } + return *this; + } + + private: + PrimitivePtr prim_; + // FuncGraph will be hold by FuncGraphManager, so weak_ptr is enough here. + // And use weak_ptr can break the reference cycle between "primal" and "grad" graph in + // FPropRemapper::FinalizeGraph(). + FuncGraphWeakPtr func_graph_; + static const PrimitivePtr func_graph_prim_; +}; + +class FuncGraphBase : public Value { + public: + FuncGraphBase() = default; + + ~FuncGraphBase() override = default; + MS_DECLARE_PARENT(FuncGraphBase, Value); +}; + +extern const char kFuncGraphFlagUndetermin[]; + +class FuncGraph : public FuncGraphBase { + public: + FuncGraph(); + + ~FuncGraph() override = default; + MS_DECLARE_PARENT(FuncGraph, FuncGraphBase); + + // get the graph's abstract + abstract::AbstractFunctionPtr abstract(); + abstract::AbstractBasePtr MakeAbstractClosure(const abstract::AnalysisContextPtr &context); + + // return the graph's output, or nullptr if not yet deduced + AnfNodePtr output() const; + void set_output(const AnfNodePtr &value, bool force_new_ret = false); + + const std::vector ¶meters() const { return parameters_; } + virtual ParameterPtr add_parameter(); + void add_parameter(const ParameterPtr &p); + void set_parameters(const std::vector ¶ms) { parameters_ = params; } + // add a weight parameter with specific name + ParameterPtr AddWeightParameter(const std::string &name); + + // create a cnode with given inputs, bound to this graph + virtual CNodePtr NewCNode(const std::vector &inputs = std::vector()); + + // create a cnode with given inputs, bound to this graph, and set to specific scope + CNodePtr NewCNodeWithScope(const std::vector &inputs, const ScopePtr &scope); + + // Functions for handling variable argument, keyword-only arguments and variable keyword argument + AnfNodePtr GetDefaultValueByName(const std::string &name); + void set_param_default_value(const std::string &name, const AnfNodePtr &node) { + parameter_default_value_[name] = node; + } + void SetDefaultValues(const std::vector &name_list, const std::vector &value_list); + void ClearDefaultValues(); + size_t GetDefaultValueCount(); + std::map ¶meter_default_value() { return parameter_default_value_; } + void set_has_vararg(bool has_) { has_vararg_ = has_; } + bool has_vararg() const { return has_vararg_; } + AnfNodePtr GetVariableArgParameter(); + std::string GetVariableArgName(); + void set_has_kwarg(bool has_) { has_kwarg_ = has_; } + bool has_kwarg() const { return has_kwarg_; } + void set_kwonlyargs_count(int count) { kwonlyargs_count_ = count; } + int kwonlyargs_count() const { return kwonlyargs_count_; } + AnfNodePtr GetVariableKwargParameter(); + std::string GetVariableKwargName(); + void set_hyper_param_count(size_t count) { hyper_param_count_ = count; } + size_t hyper_param_count() const { return hyper_param_count_; } + int GetPositionalArgsCount() const; + AnfNodePtr GetParameterByName(const std::string &name); + bool NeedGenerate(const std::vector &kwarg_list); + FuncGraphPtr GenerateGraph(const AbstractBasePtrList &args_spec_list); + void set_is_generate(bool generated) { is_generated_ = generated; } + bool is_generated() const { return is_generated_; } + + bool has_flag(const std::string &flag); + std::unordered_map &flags() { return flags_; } + void set_flags(const std::unordered_map &flags) { flags_ = flags; } + void set_flags(const std::string &key, const bool value) { flags_[key] = value; } + + std::unordered_map &transforms() { return transforms_; } + void set_transforms(const std::unordered_map &transforms) { + transforms_ = transforms; + } + + CNodePtr get_return() const { return return_; } + void set_return(const CNodePtr &cnode) { return_ = cnode; } + + FuncGraphManagerPtr manager() const { return manager_.lock(); } + void set_manager(const FuncGraphManagerPtr &m) { manager_ = std::weak_ptr(m); } + + std::string ToString() const override; + GraphDebugInfoPtr debug_info(); + void set_debug_info(const GraphDebugInfoPtr &info) { + if (info == nullptr) { + MS_LOG(EXCEPTION) << "graph set null debug info"; + } + this->debug_info_ = info; + } + + // get all nodes belonging to this func graph + const AnfNodeSet &nodes(); + + // get all value_nodes belonging to this func graph + const AnfNodeCounterMap &value_nodes(); + + // get all vars directly pointed to in this func graph + const AnfNodeCounterMap &free_variables_direct(); + + // get all vars required by this func graph + const BaseRefCounterMap &free_variables_total(); + + // Return the set of graphs free_variables_total belong to. + std::vector free_variables_nodes(); + + // get all vars that are func graphs + std::vector free_variables_func_graphs(); + + // get all func graphs directly used by this func graph + const FuncGraphCounterMap &func_graphs_used(); + + // get all func graphs nestedly used by this func graph + const FuncGraphSet &func_graphs_used_total(); + + // get all users of this func graph + const FuncGraphCounterMap &func_graph_users(); + + // get all user cnodes of this func graph + const AnfNodeCounterMap &func_graph_user_cnodes(); + + // Return the parent of this graph. + FuncGraphPtr parent(); + + // Return the children of this graph. + const FuncGraphSet &children(); + + // Return the scope of this graph, scope have graph self but children not have. + const FuncGraphSet &scope(); + + // Return whether this graph is recursive + bool recursive(); + + // Return graphs which forms a recursive loop + std::shared_ptr> recursive_graphs(); + + std::size_t hash() const override { return std::hash{}(this); } + + void DumpFuncGraph(const std::string &path = "./func_graph.dot"); + + bool operator==(const Value &other) const override { + if (other.isa()) { + return &other == this; + } else { + return false; + } + } + void GenerateVarParams(const FuncGraphPtr &specialized_graph, std::vector *specialized_parameter_list, + std::unordered_map *repl_nodes, int variable_args_count, + int pos_args_input_count); + + void GenerateKwParams(const FuncGraphPtr &specialized_graph, std::vector *specialized_parameter_list, + const std::vector &kwarg_list, + std::unordered_map *repl_nodes); + + void GenerateDefaultValue(const FuncGraphPtr &specialized_graph, + const std::vector &specialized_parameter_list, + std::unordered_map *repl_nodes); + + const std::vector ¶mter_obj_nodes() const { return paramter_obj_nodes_; } + void add_parameter_obj_node(const AnfNodePtr &p); + + std::unordered_map &make_ref_params() { return make_ref_params_; } + + std::unordered_map flags_; + std::unordered_map transforms_; + // parameter default value + std::map parameter_default_value_; + std::unordered_map make_ref_params_; + + std::list GetOrderedCnodes(bool force_use_topo_sort = false); + void EraseUnusedNodeInOrder(const AnfNodePtr &n); + void EraseUnusedNodeInOrder(); + void CheckOrder(); + void DumpCNodeList(); + void ReleaseFullOrderToEffectOrder(); + void SetEffectDepends(const std::vector &depend_inputs); + bool HasEffect(const CNodePtr &cnode); + + private: + // graph is manipulated by manager and others + friend FuncGraphManager; + + // parameters of this function + std::vector parameters_; + std::vector paramter_obj_nodes_; + + // whether there is a *args and **kwargs, and count kwonlyargs'number + bool has_vararg_; + bool has_kwarg_; + int kwonlyargs_count_; + // the hyper param is placed on the top graph, + // and positioned in the end of the param list, so we record the number to trace the position + size_t hyper_param_count_; + // the argument input list for the graph used to generate this graph + bool is_generated_; + + // the cnode that calls 'return' primitive + // we use shared pointer to manage it. + CNodePtr return_; + + // back-ref to its manager + // hold a weak ref to FuncGraphManager as FuncGraphManager also hold many ref to FuncGraph. + // Otherwise, FuncGraph and FuncGraphManager will make a reference cycles. + // Notes: Normally, there will be a global FuncGraphManager, it will hold all FuncGraphs. + // In some ut test cases, they may use local FuncGraphManager in function which + // generating the func graph, when go outside of that function, func graph will have no + // FuncGraphManager. In that special case, Manage() should be called to make the func graph + // managed. + std::weak_ptr manager_; + + GraphDebugInfoPtr debug_info_; + void GenerateKwargReplNode(const FuncGraphPtr &specialized_graph, + std::unordered_map *repl_nodes, + const std::vector &kwarg_keys_tuple_nodes, + const std::vector &kwarg_values_tuple_nodes); + + // CNode order which relates to origin code order + std::list order_; +}; + +inline CNodePtr NewCNode(const std::vector &inputs, const FuncGraphPtr &fg) { + MS_EXCEPTION_IF_NULL(fg); + return fg->NewCNode(inputs); +} + +// Find the root cnodes of a segment of cnodes. +std::shared_ptr> FindRoots(const std::vector &segment); +// Find the leaf cnodes of a segment of cnodes. +std::shared_ptr> FindLeaves(const std::vector &segment); +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_FUNC_GRAPH_H_ diff --git a/mindspore/ccsrc/ir/func_graph_cloner.cc b/mindspore/ccsrc/ir/func_graph_cloner.cc new file mode 100644 index 00000000000..d90cdbacf27 --- /dev/null +++ b/mindspore/ccsrc/ir/func_graph_cloner.cc @@ -0,0 +1,636 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/func_graph_cloner.h" + +#include + +#include "ir/manager.h" +#include "operator/ops.h" +#include "utils/log_adapter.h" +#include "utils/profile.h" +#include "utils/context/ms_context.h" + +// namespace to support intermediate representation definition +namespace mindspore { +Cloner::Cloner(const FuncGraphPtrList& func_graphs, bool clone_all_valuenodes, bool clone_all_child_graphs, + bool clone_all_used_graphs, const TraceInfoPtr& relation, const TraceInfoPtr& target_relation) + : clone_all_valuenodes_(clone_all_valuenodes), + clone_all_child_graphs_(clone_all_child_graphs), + clone_all_used_graphs_(clone_all_used_graphs), + relation_(relation), + target_relation_(target_relation == nullptr ? relation : target_relation) { + for (auto& func_graph : func_graphs) { + AddClone(func_graph); + } + scope_ = kDefaultScope; + type_ = kBasic; +} + +void Cloner::AddClone(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph, + const AnfNodePtrList& params, CloneType type) { + if (func_graph != nullptr) { + todo_.push_back({.origin = func_graph, .target = target_func_graph, .params = params}); + type_ = type; + } +} + +void Cloner::CloneNode(const AnfNodePtr& node, const FuncGraphPtr& target) { + MS_EXCEPTION_IF_NULL(node); + if (repl_node_.find(node) != repl_node_.end() || node->isa()) { + return; + } + if (node->isa()) { + CloneParameter(node, target); + } else if (node->isa()) { + CloneCNode(node, target); + } +} + +void Cloner::CloneParameter(const AnfNodePtr& node, const FuncGraphPtr& target, bool is_add) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(target); + TraceManager::DebugTrace(node->debug_info(), relation_); + auto new_param = (is_add) ? target->add_parameter() : std::make_shared(target); + auto old_param = node->cast(); + new_param->set_abstract(old_param->abstract()); + new_param->set_name(old_param->name()); + if (old_param->has_default()) { + new_param->set_default_param(old_param->default_param()); + } + ScopePtr scope = (node->scope() != kDefaultScope) ? node->scope() : this->scope(); + new_param->set_scope(scope); + repl_node_[node] = new_param; + TraceManager::EndTrace(); +} + +void Cloner::CloneCNode(const AnfNodePtr& node, const FuncGraphPtr& target) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(target); + TraceManager::DebugTrace(node->debug_info(), relation_); + CNodePtr new_node = std::make_shared(AnfNodePtrList{}, target); + auto old_node = node->cast(); + new_node->set_abstract(old_node->abstract()); + ScopePtr scope = (node->scope() != kDefaultScope) ? node->scope() : this->scope(); + new_node->set_scope(scope); + repl_node_[old_node] = new_node; + nodes_.emplace_back(old_node, new_node); + TraceManager::EndTrace(); +} + +void Cloner::CloneValueNode(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + TraceManager::DebugTrace(node->debug_info(), relation_); + ValueNodePtr new_const = NewValueNode(GetValueNode(node)); + ScopePtr scope = (node->scope() != kDefaultScope) ? node->scope() : this->scope(); + new_const->set_scope(scope); + new_const->set_abstract(node->abstract()); + repl_node_[node] = new_const; + TraceManager::EndTrace(); +} + +void Cloner::CloneValueNode(const AnfNodePtr& node, const FuncGraphPtr& target) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(target); + TraceManager::DebugTrace(node->debug_info(), relation_); + ValueNodePtr new_const = NewValueNode(target); + ScopePtr scope = (node->scope() != kDefaultScope) ? node->scope() : this->scope(); + new_const->set_scope(scope); + new_const->set_abstract(node->abstract()); + repl_node_[node] = new_const; + TraceManager::EndTrace(); +} + +void Cloner::CloneValueNodes(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(manager_); + if (!clone_all_valuenodes_) { + return; + } + auto& value_nodes = manager_->valuenodes()[func_graph]; + for (auto& value_node : value_nodes) { + auto old_node = value_node.first; + MS_EXCEPTION_IF_NULL(old_node); + if (repl_node_.count(old_node) == 0) { + CloneValueNode(old_node); + } + } +} + +void Cloner::AddChildGraphs(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(manager_); + if (!clone_all_child_graphs_) { + return; + } + auto& scopes = manager_->scopes(func_graph); + for (auto& graph : scopes) { + if (graph != func_graph) { + todo_.push_back({graph, nullptr, {}}); + } + } +} + +void Cloner::AddTotalGraphs(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(manager_); + if (!clone_all_used_graphs_) { + return; + } + auto& used_graphs = manager_->func_graphs_used()[func_graph]; + for (auto& used_graph : used_graphs) { + todo_.push_back({used_graph.first, nullptr, {}}); + } +} + +void Cloner::CloneFuncGraphDefaultValues(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(target_func_graph); + for (auto& item : func_graph->parameter_default_value()) { + auto nodes = DeepLinkedGraphSearch(item.second); + for (auto& node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + CloneNode(node, target_func_graph); + } else if (node->isa()) { + CloneValueNode(node); + } + } + } +} + +void Cloner::CloneFuncGraphValueNodes(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(target_func_graph); + MS_EXCEPTION_IF_NULL(manager_); + auto return_node = repl_node_[func_graph->get_return()]->cast(); + if (return_node == nullptr) { + MS_LOG(EXCEPTION) << "Can't find replicate node for return."; + } + target_func_graph->set_return(return_node); + + auto& value_nodes = manager_->func_graph_valuenodes()[func_graph]; + for (auto& value_node : value_nodes) { + CloneValueNode(value_node.first, target_func_graph); + } +} + +void Cloner::InlineCloneParameters(const FuncGraphPtr& func_graph, const AnfNodePtrList& params) { + MS_EXCEPTION_IF_NULL(func_graph); + auto& old_params = func_graph->parameters(); + if (old_params.size() != params.size()) { + MS_LOG(EXCEPTION) << "Origin params size[" << old_params.size() << "], inline params size[" << params.size() << "]"; + return; + } + for (size_t i = 0; i < old_params.size(); ++i) { + repl_node_[old_params[i]] = params[i]; + } +} + +void Cloner::SetFuncGraphInfo(const FuncGraphPtr& func_graph, FuncGraphPtr* const target_func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(target_func_graph); + TraceManager::DebugTrace(func_graph->debug_info(), target_relation_); + *target_func_graph = std::make_shared(); + (*target_func_graph)->set_flags(func_graph->flags()); + (*target_func_graph)->set_transforms(func_graph->transforms()); + (*target_func_graph)->set_has_vararg(func_graph->has_vararg()); + (*target_func_graph)->set_has_kwarg(func_graph->has_kwarg()); + (*target_func_graph)->set_kwonlyargs_count(func_graph->kwonlyargs_count()); + (*target_func_graph)->set_hyper_param_count(func_graph->hyper_param_count()); + (*target_func_graph)->set_is_generate(func_graph->is_generated()); + TraceManager::EndTrace(); +} + +void Cloner::CloneParameters(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(target_func_graph); + auto& params = func_graph->parameters(); + for (auto& param : params) { + CloneParameter(param, target_func_graph, true); + } + repl_func_graph_[func_graph] = target_func_graph; +} + +void Cloner::GenParameters(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + auto& free_vars = manager_->free_variables_total(); + auto iter = free_vars.find(func_graph); + if (iter == free_vars.end()) { + return; + } + + for (auto& fv_map : iter->second) { + auto& free_var = fv_map.first; + if (utils::isa(free_var)) { + repl_func_graph_params_[func_graph].push_back(AddParameter(func_graph, utils::cast(free_var))); + } + } +} + +void Cloner::CloneParameter(const ParameterPtr& param, const AnfNodePtr& node) { + param->set_abstract(node->abstract()); + if (node->isa()) { + ParameterPtr old_param = dyn_cast(node); + if (old_param->has_default()) { + param->set_default_param(old_param->default_param()); + } + param->set_name(old_param->name()); + } +} + +ParameterPtr Cloner::AddParameter(const FuncGraphPtr& func_graph, const AnfNodePtr& node, bool is_add) { + TraceManager::DebugTrace(std::make_shared(node->debug_info())); + ParameterPtr param = std::make_shared(func_graph); + TraceManager::EndTrace(); + CloneParameter(param, node); + if (is_add) { + func_graph->add_parameter(param); + } + repl_node_[param] = node; + repl_map_node_[func_graph][node] = param; + return param; +} + +void Cloner::AddParameters(const FuncGraphPtr& func_graph, const AnfNodePtrList& params, + AnfNodePtrList* const lift_params, AnfNodePtrList* const input_params) { + AnfNodePtrList parameters; + std::unordered_set old_params; + for (auto& param : func_graph->parameters()) { + auto iter = repl_node_.find(param); + if (iter != repl_node_.end()) { + (void)old_params.insert(iter->second); + parameters.push_back(param); + } else { + parameters.push_back(AddParameter(func_graph, param, false)); + (void)old_params.insert(param); + } + } + AnfNodePtr new_param = nullptr; + for (auto& param : params) { + auto old_param = repl_node_[param]; + if (old_param->isa() && old_param->func_graph() == func_graph) { + repl_node_[old_param] = old_param; + repl_map_node_[func_graph][old_param] = old_param; + input_params->push_back(old_param); + continue; + } + if (old_params.find(old_param) != old_params.end()) { + new_param = repl_map_node_[func_graph][old_param]; + input_params->push_back(new_param); + continue; + } + new_param = AddParameter(func_graph, old_param, false); + parameters.push_back(new_param); + lift_params->push_back(new_param); + input_params->push_back(new_param); + } + func_graph->set_parameters(parameters); +} + +void Cloner::AddInputs(const FuncGraphPtr& func_graph_user, const FuncGraphPtr& func_graph, + const AnfNodePtrList& params) { + AnfNodePtr node = nullptr; + auto& repl_func_graph = repl_map_func_graph_[func_graph_user]; + auto iter = repl_func_graph.find(func_graph); + if (iter == repl_func_graph.end()) { + node = func_graph_user->NewCNode({NewValueNode(prim::kPrimPartial), NewValueNode(func_graph)}); + repl_func_graph[func_graph] = node; + } else { + node = iter->second; + } + if (node == nullptr || !node->isa()) { + return; + } + auto cnode = node->cast(); + auto inputs = cnode->inputs(); + (void)std::copy(params.begin(), params.end(), std::back_inserter(inputs)); + cnode->set_inputs(inputs); + OrderParameters(func_graph, inputs); +} + +void Cloner::OrderParameters(const FuncGraphPtr& func_graph, const AnfNodePtrList& inputs) { + std::unordered_set old_params; + for (auto& param : func_graph->parameters()) { + (void)old_params.insert(repl_node_[param]); + } + std::unordered_set new_params; + AnfNodePtrList parameters; + // Ignore the 1st and 2nd param of inputs(such as. partial graph) + for (size_t i = 2; i < inputs.size(); ++i) { + auto input = inputs[i]; + auto param = repl_node_[input]; + if (old_params.find(param) != old_params.end()) { + auto new_param = repl_map_node_[func_graph][param]; + parameters.push_back(new_param); + (void)new_params.insert(new_param); + } + } + for (auto& param : func_graph->parameters()) { + if (new_params.find(param) == new_params.end()) { + parameters.push_back(param); + } + } + func_graph->set_parameters(parameters); +} + +void Cloner::SetEdges(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + for (auto& node : func_graph->nodes()) { + if (node == nullptr) { + continue; + } + // Only cnode needed to be handled + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + auto& inputs = cnode->inputs(); + for (size_t i = 0; i < inputs.size(); i++) { + auto& input = inputs[i]; + if (IsValueNode(input)) { + auto graph = GetValueNode(input); + auto& repl_func_graph = repl_map_func_graph_[func_graph]; + if (repl_func_graph.find(graph) != repl_func_graph.end()) { + transaction_.SetEdge(cnode, SizeToInt(i), repl_func_graph[graph]); + } + } else { + auto& repl_node = repl_map_node_[func_graph]; + if (repl_node.find(input) != repl_node.end()) { + transaction_.SetEdge(cnode, SizeToInt(i), repl_node[input]); + } + } + } + } +} + +void Cloner::LiftParameters(const FuncGraphPtr& func_graph_user, const FuncGraphPtr& func_graph, + const AnfNodePtrList& params) { + AnfNodePtrList lift_params; + AnfNodePtrList input_params; + AddParameters(func_graph_user, params, &lift_params, &input_params); + AddInputs(func_graph_user, func_graph, input_params); + if (lift_params.empty()) { + return; + } + for (auto& user : func_graph_user->func_graph_users()) { + LiftParameters(user.first, func_graph_user, lift_params); + } +} + +void Cloner::Lift() { + for (auto& func_graph_params : repl_func_graph_params_) { + auto& func_graph = func_graph_params.first; + auto& params = func_graph_params.second; + for (auto& user : func_graph->func_graph_users()) { + LiftParameters(user.first, func_graph, params); + } + } +} + +void Cloner::LiftParameters() { + MS_EXCEPTION_IF_NULL(manager_); + transaction_ = manager_->Transact(); + const FuncGraphSet& func_graphs = manager_->func_graphs(); + for (auto& func_graph : func_graphs) { + GenParameters(func_graph); + } + Lift(); + for (auto& func_graph : func_graphs) { + SetEdges(func_graph); + } + transaction_.Commit(); +} + +bool Cloner::CheckStatus(const FuncGraphPtr& func_graph, bool is_inline) { + MS_EXCEPTION_IF_NULL(func_graph); + // Make sure only inline once + if (status_.count(func_graph) != 0) { + if (is_inline == status_[func_graph]) { + return false; + } + if (clone_all_used_graphs_) { + MS_LOG(ERROR) << "Try setting the `clone_all_used_graphs` option to False."; + return false; + } + } + return true; +} + +void Cloner::CloneAllNodes(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(target_func_graph); + MS_EXCEPTION_IF_NULL(manager_); + const AnfNodeSet& nodes = manager_->nodes()[func_graph]; + for (auto& node : nodes) { + CloneNode(node, target_func_graph); + } +} + +void Cloner::Run() { + if (todo_.empty()) { + return; + } + + if (type_ < kLifting) { + // Basic and Inline Clone + FuncGraphPtrList func_graphs; + (void)std::transform(todo_.begin(), todo_.end(), std::back_inserter(func_graphs), + [](const CloneInfo& item) -> FuncGraphPtr { return item.origin; }); + manager_ = Manage(func_graphs, false); + CloneNodes(); + LinkEdges(); + SetDefaults(); + } else { + // Lifting Clone + CloneInfo item = todo_.back(); + manager_ = Manage(item.origin); + LiftParameters(); + } +} + +void Cloner::CloneNodes() { + while (!todo_.empty()) { + CloneInfo item = todo_.back(); + todo_.pop_back(); + + bool is_inline = (item.target != nullptr); + FuncGraphPtr func_graph = item.origin; + FuncGraphPtr target_func_graph = item.target; + (void)graph_set_.insert(func_graph); + + if (!CheckStatus(func_graph, is_inline)) { + continue; + } + + if (is_inline) { + InlineCloneParameters(func_graph, item.params); + CloneAllNodes(func_graph, target_func_graph); + } else { + SetFuncGraphInfo(func_graph, &target_func_graph); + CloneParameters(func_graph, target_func_graph); + CloneAllNodes(func_graph, target_func_graph); + CloneFuncGraphValueNodes(func_graph, target_func_graph); + CloneFuncGraphDefaultValues(func_graph, target_func_graph); + } + + CloneValueNodes(func_graph); + AddChildGraphs(func_graph); + AddTotalGraphs(func_graph); + status_[func_graph] = is_inline; + } +} + +void Cloner::LinkEdges() { + for (auto& node_pair : nodes_) { + CNodePtr old_node = node_pair.first; + CNodePtr new_node = node_pair.second; + MS_EXCEPTION_IF_NULL(old_node); + MS_EXCEPTION_IF_NULL(new_node); + for (auto& input : old_node->inputs()) { + auto& new_input = (repl_node_.count(input) == 0) ? input : repl_node_[input]; + new_node->add_input(new_input); + } + } +} + +// For the graphs cloned, update its default value map to the cloned nodes +void Cloner::SetDefaults() { + for (auto& item : graph_set_) { + MS_EXCEPTION_IF_NULL(item); + if (repl_func_graph_.count(item) != 0) { + for (auto& param_def : item->parameter_default_value()) { + MS_EXCEPTION_IF_NULL(repl_func_graph_[item]); + if (repl_node_.count(param_def.second) != 0) { + repl_func_graph_[item]->set_param_default_value(param_def.first, repl_node_[param_def.second]); + } else { + repl_func_graph_[item]->set_param_default_value(param_def.first, param_def.second); + } + } + } + } +} + +AnfNodePtr Cloner::CloneDisconnected(const AnfNodePtr& root) { + MS_EXCEPTION_IF_NULL(root); + if (repl_func_graph_.find(root->func_graph()) == repl_func_graph_.end()) { + MS_LOG(EXCEPTION) << "Cannot find func graph " << root->func_graph()->ToString() << " in cloner."; + } + CloneNode(root, repl_func_graph_[root->func_graph()]); + auto iter = repl_node_.find(root); + if (iter != repl_node_.end()) { + return iter->second; + } + MS_LOG(EXCEPTION) << "Failed in clone for node " << root->DebugString() << "."; +} + +AnfNodePtr Cloner::operator[](const AnfNodePtr& node) { +#ifdef ENABLE_PROFILE + double time = GetTime(); +#endif + Run(); +#ifdef ENABLE_PROFILE + MsProfile::StatTime("func_graph_cloner_run.FuncGraphClonerNode", GetTime() - time); +#endif + return ((repl_node_.count(node) == 0) ? node : repl_node_[node]); +} + +FuncGraphPtr Cloner::operator[](const FuncGraphPtr& func_graph) { +#ifdef ENABLE_PROFILE + double time = GetTime(); +#endif + Run(); +#ifdef ENABLE_PROFILE + MsProfile::StatTime("func_graph_cloner_run.FuncGraphClonerGraph", GetTime() - time); +#endif + return ((repl_func_graph_.count(func_graph) == 0) ? func_graph : repl_func_graph_[func_graph]); +} + +FuncGraphPtr BasicClone(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + Cloner cloner({func_graph}, false, true, true, std::make_shared(), nullptr); + return cloner[func_graph]; +} + +AnfNodePtr InlineClone(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph, + const AnfNodePtrList& func_graph_args, const ScopePtr& scope) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(target_func_graph); + Cloner cloner({}, false); + if (scope != nullptr) { + cloner.set_scope(scope); + } + cloner.AddClone(func_graph, target_func_graph, func_graph_args, kInline); + return cloner[func_graph->output()]; +} + +FuncGraphPtr LiftingClone(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + Cloner cloner({}, false); + cloner.AddClone(func_graph, nullptr, {}, kLifting); + return cloner[func_graph]; +} + +ClonerPtr SpecializerClone(const FuncGraphPtr& func_graph, const TraceInfoPtr& relation) { + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphPtrList func_graphs = {func_graph}; + ClonerPtr cloner = + std::make_shared(func_graphs, false, false, false, std::make_shared(), relation); +#ifdef ENABLE_PROFILE + double time = GetTime(); +#endif + cloner->Run(); +#ifdef ENABLE_PROFILE + MsProfile::StatTime("func_graph_cloner_run.FuncGraphSpecializer", GetTime() - time); +#endif + return cloner; +} + +FuncGraphPtr TransformableClone(const FuncGraphPtr& func_graph, const TraceInfoPtr& relation) { + MS_EXCEPTION_IF_NULL(func_graph); + TraceManager::DebugTrace(func_graph->debug_info(), relation); + auto new_func_graph = std::make_shared(); + TraceManager::EndTrace(); + + auto& parameters = func_graph->parameters(); + (void)std::for_each(parameters.begin(), parameters.end(), [&new_func_graph](const AnfNodePtr& param) -> void { + MS_EXCEPTION_IF_NULL(param); + TraceManager::DebugTrace(std::make_shared(param->debug_info())); + (void)new_func_graph->add_parameter(); + TraceManager::EndTrace(); + }); + + Cloner cloner = Cloner(); + cloner.AddClone(func_graph, new_func_graph, new_func_graph->parameters()); + AnfNodePtr output = cloner[func_graph->output()]; + new_func_graph->set_output(output); + new_func_graph->set_has_vararg(func_graph->has_vararg()); + new_func_graph->set_has_kwarg(func_graph->has_kwarg()); + new_func_graph->set_kwonlyargs_count(func_graph->kwonlyargs_count()); + new_func_graph->set_hyper_param_count(func_graph->hyper_param_count()); + new_func_graph->set_is_generate(func_graph->is_generated()); + for (auto& item : func_graph->parameter_default_value()) { + new_func_graph->set_param_default_value(item.first, cloner[item.second]); + } + + if (MsContext::GetInstance()->is_multi_graph_sink()) { + if (func_graph->has_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES)) { + new_func_graph->set_flags(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + } + } + return new_func_graph; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/func_graph_cloner.h b/mindspore/ccsrc/ir/func_graph_cloner.h new file mode 100644 index 00000000000..dd228cf79f2 --- /dev/null +++ b/mindspore/ccsrc/ir/func_graph_cloner.h @@ -0,0 +1,134 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_FUNC_GRAPH_CLONER_H_ +#define MINDSPORE_CCSRC_IR_FUNC_GRAPH_CLONER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" + +namespace mindspore { +class Cloner; +using ClonerPtr = std::shared_ptr; + +enum CloneType { kBasic = 0, kInline = 1, kLifting = 2, kDropping = 3 }; + +struct CloneInfo { + FuncGraphPtr origin; + FuncGraphPtr target; + AnfNodePtrList params; +}; + +class Cloner { + public: + explicit Cloner(const FuncGraphPtrList& func_graphs = {}, bool clone_all_valuenodes = false, + bool clone_all_child_graphs = true, bool clone_all_used_graphs = false, + const TraceInfoPtr& relation = std::make_shared(), + const TraceInfoPtr& target_relation = nullptr); + ~Cloner() = default; + void AddClone(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph = nullptr, + const AnfNodePtrList& params = {}, CloneType type = kBasic); + void Run(); + + // Interfaces for specializer + AnfNodePtr CloneDisconnected(const AnfNodePtr& root); + AnfNodePtr operator[](const AnfNodePtr& node); + FuncGraphPtr operator[](const FuncGraphPtr& func_graph); + + // Map of replicate nodes and graphs + std::unordered_map* cloned_node() { return &repl_node_; } + std::unordered_map cloned_func_graph() { return repl_func_graph_; } + + // Scope of cloned graphs + void set_scope(const ScopePtr& scope) { scope_ = scope; } + const ScopePtr scope() const { return scope_; } + + std::unordered_map repl_node_; + + private: + void CloneNodes(); + void LinkEdges(); + void SetDefaults(); + void CloneNode(const AnfNodePtr& node, const FuncGraphPtr& target); + void CloneValueNode(const AnfNodePtr& node); + void CloneValueNode(const AnfNodePtr& node, const FuncGraphPtr& target); + void CloneCNode(const AnfNodePtr& node, const FuncGraphPtr& target); + void CloneParameter(const AnfNodePtr& node, const FuncGraphPtr& target, bool is_add = false); + void CloneValueNodes(const FuncGraphPtr& func_graph); + void AddChildGraphs(const FuncGraphPtr& func_graph); + void AddTotalGraphs(const FuncGraphPtr& func_graph); + bool CheckStatus(const FuncGraphPtr& func_graph, bool is_inline); + void CloneAllNodes(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph); + void CloneFuncGraphValueNodes(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph); + void CloneFuncGraphDefaultValues(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph); + void InlineCloneParameters(const FuncGraphPtr& func_graph, const AnfNodePtrList& params); + void SetFuncGraphInfo(const FuncGraphPtr& func_graph, FuncGraphPtr* const target_func_graph); + void CloneParameters(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph); + void GenParameters(const FuncGraphPtr& func_graph); + void CloneParameter(const ParameterPtr& param, const AnfNodePtr& node); + ParameterPtr AddParameter(const FuncGraphPtr& func_graph, const AnfNodePtr& node, bool is_add = true); + void AddParameters(const FuncGraphPtr& func_graph, const AnfNodePtrList& params, AnfNodePtrList* const lift_params, + AnfNodePtrList* const input_params); + void AddInputs(const FuncGraphPtr& func_graph_user, const FuncGraphPtr& func_graph, const AnfNodePtrList& params); + void OrderParameters(const FuncGraphPtr& func_graph, const AnfNodePtrList& inputs); + void SetEdges(const FuncGraphPtr& func_graph); + void LiftParameters(const FuncGraphPtr& func_graph_user, const FuncGraphPtr& func_graph, + const AnfNodePtrList& params); + void Lift(); + void LiftParameters(); + + bool clone_all_valuenodes_; + bool clone_all_child_graphs_; + bool clone_all_used_graphs_; + TraceInfoPtr relation_; + TraceInfoPtr target_relation_; + FuncGraphManagerPtr manager_; + FuncGraphTransaction transaction_; + FuncGraphSet graph_set_; + ScopePtr scope_; + CloneType type_; + std::list todo_; + std::list> nodes_; + std::unordered_map status_; + std::unordered_map repl_func_graph_; + std::unordered_map> repl_map_node_; + std::unordered_map> repl_map_func_graph_; + std::unordered_map repl_func_graph_params_; +}; + +FuncGraphPtr BasicClone(const FuncGraphPtr& func_graph); + +AnfNodePtr InlineClone(const FuncGraphPtr& func_graph, const FuncGraphPtr& target_func_graph, + const AnfNodePtrList& func_graph_args, const ScopePtr& scope = nullptr); + +FuncGraphPtr LiftingClone(const FuncGraphPtr& func_graph); + +ClonerPtr SpecializerClone(const FuncGraphPtr& func_graph, const TraceInfoPtr& relation); + +FuncGraphPtr TransformableClone(const FuncGraphPtr& func_graph, + const TraceInfoPtr& relation = std::make_shared()); +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_FUNC_GRAPH_CLONER_H_ diff --git a/mindspore/ccsrc/ir/manager.cc b/mindspore/ccsrc/ir/manager.cc new file mode 100644 index 00000000000..09bc7127c1e --- /dev/null +++ b/mindspore/ccsrc/ir/manager.cc @@ -0,0 +1,1202 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/manager.h" +#include +#include +#include +#include "./common.h" +#include "utils/profile.h" +#include "operator/ops.h" +#include "debug/trace.h" + +namespace mindspore { + +FuncGraphManagerPtr MakeManager(const std::vector& func_graphs, bool manage) { + auto m = std::make_shared(func_graphs, manage); + m->Init(); + return m; +} + +FuncGraphManagerPtr Manage(const std::vector& func_graphs, bool manage) { + FuncGraphManagerPtr m = nullptr; + bool root = false; + + for (auto& fg : func_graphs) { + if (fg == nullptr) { + continue; + } + if (fg->manager() != nullptr) { + m = fg->manager(); + break; + } + } + + if (m == nullptr) { + std::vector tmp; + m = MakeManager(tmp, manage); + root = true; + } + + for (auto& fg : func_graphs) { + if (fg == nullptr) { + continue; + } + m->AddFuncGraph(fg, root); + } + return m; +} + +FuncGraphManagerPtr Manage(FuncGraphPtr func_graph, bool manage) { + std::vector func_graphs = {func_graph}; + return Manage(func_graphs, manage); +} + +FuncGraphManager::FuncGraphManager(const std::vector& roots, bool manage) + : roots_(roots), is_manage_(manage) { + Reset(); +} + +void FuncGraphManager::Reset() { + func_graphs_ = FuncGraphSet(); + all_nodes_ = AnfNodeSet(); + node_users_ = NodeUsersMap(); + + signals_ = std::make_shared(); + nodes_ = std::make_shared(this); + valuenodes_ = std::make_shared(this); + free_variables_direct_ = std::make_shared(this); + func_graph_valuenodes_ = std::make_shared(this); + func_graphs_used_ = std::make_shared(this); + func_graph_users_ = std::make_shared(this); + func_graph_user_cnodes_ = std::make_shared(this); + func_graph_child_direct_ = std::make_shared(this); + func_graph_parents_direct_ = std::make_shared(this); + func_graph_j_direct_ = std::make_shared(this); + + func_graph_parents_total_ = std::make_shared(this); + func_graph_parent_ = std::make_shared(this); + children_ = std::make_shared(this); + scopes_ = std::make_shared(this); + free_variables_total_ = std::make_shared(this); + func_graphs_used_total_ = std::make_shared(this); + recursive_ = std::make_shared(this); + j_total_ = std::make_shared(this); +} + +void FuncGraphManager::Init() { + auto roots = roots_; + roots_ = FuncGraphSet(); + + for (auto& fg : roots) { + AddFuncGraph(fg, true); + } +} + +FuncGraphSet& FuncGraphManager::func_graph_parents_total(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(fg); + MS_LOG(DEBUG) << "Start func_graph_parents_total func graph " << fg->ToString(); + func_graph_parents_total_->Recompute(fg); + MS_LOG(DEBUG) << "End func_graph_parents func graph " << fg->ToString(); + return func_graph_parents_total_->func_graph_parents_total_analysis()[fg]; +} + +FuncGraphPtr FuncGraphManager::parent(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(fg); + MS_EXCEPTION_IF_NULL(func_graph_parent_); + MS_LOG(DEBUG) << "Start parents func graph " << fg->ToString(); + func_graph_parent_->Recompute(fg); + if (func_graph_parent_->parent_analysis().count(fg) == 0) { + MS_LOG(WARNING) << "This func graph is not in manager:" << fg->ToString(); + return nullptr; + } + MS_LOG(DEBUG) << "End parents func graph " << fg->ToString(); + return func_graph_parent_->parent_analysis()[fg]; +} + +FuncGraphSet& FuncGraphManager::children(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(fg); + MS_EXCEPTION_IF_NULL(children_); + MS_LOG(DEBUG) << "Start child func graph " << fg->ToString(); + children_->Recompute(fg); + return children_->children_analysis()[fg]; +} + +FuncGraphSet& FuncGraphManager::scopes(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(fg); + MS_EXCEPTION_IF_NULL(scopes_); + MS_LOG(DEBUG) << "Start scopes func graph:" << fg->ToString(); + scopes_->Recompute(fg); + MS_LOG(DEBUG) << "End scopes func graph:" << fg->ToString(); + return scopes_->scope_analysis()[fg]; +} + +FVTotalMap& FuncGraphManager::free_variables_total() const { + MS_EXCEPTION_IF_NULL(free_variables_total_); + free_variables_total_->Recompute(); + return free_variables_total_->fv_total_analysis(); +} + +FuncGraphSet& FuncGraphManager::func_graphs_used_total(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(func_graphs_used_total_); + func_graphs_used_total_->Recompute(fg); + return func_graphs_used_total_->func_graph_used_total_analysis()[fg]; +} + +bool FuncGraphManager::recursive(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(fg); + recursive_->Recompute(fg); + if (recursive_->recursive_analysis().count(fg) == 0) { + MS_LOG(WARNING) << "This func graph is not in manager: " << fg->ToString(); + return false; + } + return recursive_->recursive_analysis()[fg]; +} + +std::shared_ptr> FuncGraphManager::recursive_graphs(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(fg); + if (recursive(fg)) { + if (!recursive_->recursive_map().count(fg)) { + auto trace = std::list(); + recursive_->CheckRecursiveGraphs(fg, &trace); + } + if (recursive_->recursive_map().count(fg) == 0) { + MS_LOG(WARNING) << "This func graph is not in manager: " << fg->ToString(); + return nullptr; + } + return recursive_->recursive_map()[fg]; + } else { + return nullptr; + } +} + +bool FuncGraphManager::func_graph_j_total(const FuncGraphPtr& fg) const { + MS_EXCEPTION_IF_NULL(j_total_); + MS_EXCEPTION_IF_NULL(fg); + j_total_->Recompute(fg); + if (j_total_->j_total_analysis().count(fg) == 0) { + MS_LOG(WARNING) << "This func graph is not in manager: " << fg->ToString(); + return false; + } + return j_total_->j_total_analysis()[fg]; +} + +// add a func graph to this manager, optionally as a root func graph. +void FuncGraphManager::AddFuncGraph(FuncGraphPtr func_graph, bool is_root) { + MS_EXCEPTION_IF_NULL(func_graph); + if (is_root) { + roots_.add(func_graph); + } + if (func_graphs_.contains(func_graph)) { + return; + } + AddIntoManaged(func_graph); + MS_EXCEPTION_IF_NULL(signals_); + signals_->AddFuncGraph(func_graph); + std::vector para = func_graph->parameters(); + AcquireNodes(para); + std::vector return_vec({func_graph->get_return()}); + AcquireNodes(return_vec); +} + +// clear the all information in manager +void FuncGraphManager::Clear() { + func_graphs_.clear(); + all_nodes_.clear(); + node_users_.clear(); + roots_.clear(); + + signals_->InvalidateCollector(); + signals_->InvalidateComputer(); +} + +void FuncGraphManager::KeepRoots(const std::vector& func_graphs) { + MS_LOG(DEBUG) << "Start keep roots"; + bool root_exist = false; + for (auto& item : func_graphs) { + if (roots_.contains(item)) { + root_exist = true; + break; + } + } + + // if the new_root in roots_, we add new_root first, then calculate the func_graphs + // relation to new_root, remove the func_graphs not relation to new_root + // if the new_root not in roots_, we clear the all func_graphs in manager + // then add the new_root + if (root_exist || func_graphs.empty()) { + FuncGraphSet roots(func_graphs); + if (roots.empty()) { + roots = roots_; + } else { + roots_.clear(); + for (auto& item : roots) { + AddFuncGraph(item, true); + } + } + + FuncGraphSet keep; + for (auto& item : roots) { + MS_LOG(DEBUG) << "roots: " << item->ToString(); + keep.update(func_graphs_used_total(item)); +#ifdef DEBUG + for (auto& k : keep) { + MS_LOG(DEBUG) << "keep: " << k->ToString(); + } +#endif + } + MaybeDropFuncGraphs(func_graphs_ - keep, true); + } else { + Clear(); + FuncGraphSet roots(func_graphs); + for (auto& item : roots) { + AddFuncGraph(item, true); + } + } +} + +void FuncGraphManager::RemoveRoots() { + MS_LOG(DEBUG) << "Start remove roots"; + roots_.clear(); + MaybeDropFuncGraphs(func_graphs_, true); +} + +void FuncGraphManager::AddIntoManaged(const FuncGraphPtr& fg) { + MS_EXCEPTION_IF_NULL(fg); + if (is_manage_) { + if (fg->manager() != nullptr && (&(*fg->manager()) != this)) { + MS_LOG(WARNING) << "A func graph can only have one manager."; + } + FuncGraphManagerPtr this_manager = shared_from_this(); + fg->set_manager(this_manager); + } + func_graphs_.add(fg); +} + +void FuncGraphManager::MaybeDropFuncGraphs(const FuncGraphSet& func_graphs, bool ignore_users) { + FuncGraphSet todo(func_graphs); + std::set dropped; + // int count = 0; + while (!todo.empty()) { + FuncGraphPtr func_graph = todo.pop(); + MS_EXCEPTION_IF_NULL(func_graph); + MS_LOG(DEBUG) << "Maybe drop func graph " << func_graph->ToString(); + if (roots_.contains(func_graph)) { + MS_LOG(DEBUG) << "Cannot drop as roots contains func graph: " << func_graph->ToString(); + continue; + } + MS_EXCEPTION_IF_NULL(func_graph_users_); + auto& users = func_graph_users_->count_func_graphs_map()[func_graph]; + if (!users.empty() && !ignore_users) { + MS_LOG(DEBUG) << "Cannot drop as users not empty: " << func_graph->ToString(); + continue; + } + if (dropped.find(func_graph) != dropped.end()) { + MS_LOG(DEBUG) << "Func graph had been dropped " << func_graph->ToString(); + continue; + } + (void)dropped.insert(func_graph); + std::vector return_vec = {func_graph->get_return()}; + todo.update(MaybeDropNodes(return_vec)); + } + MS_EXCEPTION_IF_NULL(signals_); + for (auto& fg : dropped) { + MS_EXCEPTION_IF_NULL(fg); + signals_->DropFuncGraph(fg); + all_nodes_.difference_update(fg->parameters()); + (void)func_graphs_.erase(fg); + if (fg->manager().get() == this) { + fg->set_manager(nullptr); + } + MS_LOG(DEBUG) << "Func graph dropped " << fg->ToString(); + } +} + +void FuncGraphManager::ProcessEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(inp); + if (direction == kDecEdge) { + MS_LOG(DEBUG) << "Remove node " << node->ToString() << " input[" << index << "] " << inp->ToString(); + auto& users_node = node_users_[inp]; + if (!users_node.contains(make_pair(node, index))) { + return; + } + (void)users_node.erase(make_pair(node, index)); + signals_->DropEdge(node, index, inp); + } else { + MS_LOG(DEBUG) << "Add node " << node->ToString() << " input[" << index << "] " << inp->ToString(); + if (inp->func_graph() != nullptr) { + AddFuncGraph(inp->func_graph()); + } + if (IsValueNode(inp)) { + MS_LOG(DEBUG) << "Input[" << index << "] is const graph " << inp->ToString(); + AddFuncGraph(GetValueNode(inp)); + } + auto& users_node = node_users_[inp]; + users_node.add(make_pair(node, index)); + MS_EXCEPTION_IF_NULL(signals_); + signals_->AddEdge(node, index, inp); + } +} + +void FuncGraphManager::ProcessInputs(const AnfNodePtr& node, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + auto cnode = node->cast(); + int index = 0; + for (auto& inp : cnode->inputs()) { + ProcessEdge(cnode, index, inp, direction); + ++index; + } + } +} + +IncludeType FuncGraphManager::Limit(const AnfNodePtr& node) { + if (all_nodes_.contains(node)) { + return EXCLUDE; + } else { + return FOLLOW; + } +} + +void FuncGraphManager::AcquireNodes(const std::vector& nodes) { + AnfNodeSet acq; + for (auto& node : nodes) { + std::function limit = std::bind(&FuncGraphManager::Limit, this, std::placeholders::_1); + + AnfNodeSet new_nodes = AnfNodeSet(DeepScopedGraphSearch(node, limit)); + + all_nodes_.update(new_nodes); + acq.update(new_nodes); + } + + for (auto& node : acq) { + MS_EXCEPTION_IF_NULL(node); + FuncGraphPtr fg = node->func_graph(); + if (fg != nullptr) { + AddFuncGraph(fg); + } + signals_->AddNode(node); + ProcessInputs(node, kIncEdge); + } +} + +FuncGraphSetPtr FuncGraphManager::MaybeDropNodes(const std::vector& nodes) { + AnfNodeSet nodes_ordered(nodes); + FuncGraphSetPtr func_graphs_to_check = std::make_shared(); + MS_EXCEPTION_IF_NULL(signals_); + + while (!nodes_ordered.empty()) { + AnfNodePtr node = nodes_ordered.pop(); + MS_EXCEPTION_IF_NULL(node); + if (!all_nodes_.contains(node)) { + continue; + } + AnfNodeIndexSet& users = node_users_[node]; + + std::vector parameters; + if (!users.empty() || + (node->isa() && parameters.end() != std::find(parameters.begin(), parameters.end(), node))) { + continue; + } + if (IsValueNode(node)) { + auto fg = GetValueNode(node); + func_graphs_to_check->add(fg); + MS_LOG(DEBUG) << "Set value of node " << node->DebugString() << " from func graph " << fg->ToString() + << " to null"; + } + ProcessInputs(node, kDecEdge); + (void)all_nodes_.erase(node); + signals_->DropNode(node); + if (node->isa()) { + auto cnode = node->cast(); + nodes_ordered.update(cnode->inputs()); + } + (void)node_users_.erase(node); + } + return func_graphs_to_check; +} + +void FuncGraphManager::SetParameters(const FuncGraphPtr& fg, const std::vector& parameters) { + auto tr = Transact(); + tr.SetParameters(fg, parameters); + tr.Commit(); +} + +bool FuncGraphManager::Replace(const AnfNodePtr& old_node, const AnfNodePtr& new_node) { + auto tr = Transact(); + bool success = tr.Replace(old_node, new_node); + if (success) { + tr.Commit(); + } + return success; +} + +void FuncGraphManager::SetEdge(const AnfNodePtr& node, int index, const AnfNodePtr& value) { + auto tr = Transact(); + tr.SetEdge(node, index, value); + tr.Commit(); +} + +void FuncGraphManager::MoveAllCNodeDropGraph(FuncGraphPtr source, FuncGraphPtr target, const ScopePtr& scope) { + AnfNodePtr source_return = source->get_return(); + AnfNodePtr source_output = source->output(); + AnfNodePtr source_prim = source_return->cast()->input(0); + + int index = 0; + (void)node_users_[source_prim].erase(make_pair(source_return, index)); + signals_->DropEdge(source_return, index, source_prim); + index = 1; + (void)node_users_[source_output].erase(make_pair(source_return, index)); + signals_->DropEdge(source_return, index, source_output); + (void)all_nodes_.erase(source_return); + (void)node_users_.erase(source_return); + signals_->DropNode(source_return); + for (auto& node : source->nodes()) { + node->set_func_graph(target); + if (node->scope() == kDefaultScope) { + node->set_scope(scope); + } + } + for (auto& used : source->func_graphs_used()) { + (void)func_graph_users_->Inc(used.first, target, used.second); + (void)this->func_graph_users()[used.first].erase(source); + } + for (auto& child : this->func_graph_child_direct()[source]) { + (void)func_graph_parents_direct_->Inc(child.first, target, child.second); + (void)this->func_graph_parents_direct()[child.first].erase(source); + } + for (auto& fv_count : this->free_variables_direct()[source]) { + auto fv_g = fv_count.first->func_graph(); + auto& count_on_g = this->func_graph_child_direct()[fv_g]; + auto pair = count_on_g.find(source); + if (fv_g != target && pair != count_on_g.end()) { + (void)func_graph_child_direct_->Inc(fv_g, target, pair->second); + } + (void)count_on_g.erase(source); + } + signals_->MoveAllCNode(source, target); + signals_->InvalidateComputer(); + signals_->DropFuncGraph(source); + all_nodes_.difference_update(source->parameters()); + (void)func_graphs_.erase(source); + if (source->manager().get() == this) { + source->set_manager(nullptr); + } +} + +FuncGraphTransaction FuncGraphManager::Transact() { + auto tr = FuncGraphTransaction(this); + return tr; +} + +void FuncGraphManager::ParseChanges(const std::vector& changes, EdgeTupleCounter* add_edges, + EdgeTupleCounter* rm_edges, Counter* adds, Counter* rms) { + for (auto& iter : changes) { + auto operation = iter.op; + auto args = iter.args; + if (operation == Change::kTxSetEdge) { + auto edge = args.cast(); + auto old_node = edge.root_node->input(edge.index); + (*rm_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, old_node))] += 1; + (*add_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, edge.new_node))] += 1; + (*rms)[old_node] += 1; + (*adds)[edge.new_node] += 1; + edge.root_node->set_input(edge.index, edge.new_node); + } else if (operation == Change::kTxSetParams) { + auto param = args.cast(); + MS_EXCEPTION_IF_NULL(param.func_graph); + auto old_parameters = param.func_graph->parameters(); + for (auto& p : param.params) { + (*adds)[p] += 1; + } + for (auto& p : old_parameters) { + (*rms)[p] += 1; + } + param.func_graph->set_parameters(param.params); + } + } +} + +void FuncGraphManager::CommitChanges(const std::vector& changes) { + EdgeTupleCounter add_edges; + EdgeTupleCounter rm_edges; + Counter adds; + Counter rms; + ParseChanges(changes, &add_edges, &rm_edges, &adds, &rms); + + auto sub_edges = add_edges - rm_edges; + for (auto& iter : sub_edges) { + auto root_node = iter.first.first; + int index = iter.first.second.first; + auto new_node = iter.first.second.second; + ProcessEdge(root_node, index, new_node, kIncEdge); + } + + auto sub_nodes = adds - rms; + std::vector nodes; + (void)std::transform(sub_nodes.begin(), sub_nodes.end(), std::back_inserter(nodes), + [](const std::pair& iter) -> AnfNodePtr { return iter.first; }); + + AcquireNodes(nodes); + + auto sub_edges_reverse = rm_edges - add_edges; + for (auto& iter : sub_edges_reverse) { + auto root_node = iter.first.first; + int index = iter.first.second.first; + auto old_node = iter.first.second.second; + ProcessEdge(root_node, index, old_node, kDecEdge); + } + + auto sub_nodes_reverse = rms - adds; + std::vector nodes_reverse; + + (void)std::transform(sub_nodes_reverse.begin(), sub_nodes_reverse.end(), std::back_inserter(nodes_reverse), + [](const std::pair& iter) -> AnfNodePtr { return iter.first; }); + + auto drop_func_graphs = MaybeDropNodes(nodes_reverse); + MaybeDropFuncGraphs(*drop_func_graphs); +} + +void FuncGraphTransaction::SetParameters(FuncGraphPtr fg, const std::vector& params) { + changes_.emplace_back(Change::kTxSetParams, ArgsOfSetParams{fg, params}); +} + +bool FuncGraphTransaction::Replace(const AnfNodePtr& old_node, const AnfNodePtr& new_node) { + MS_EXCEPTION_IF_NULL(old_node); + MS_EXCEPTION_IF_NULL(new_node); + FuncGraphPtr old_func_graph = old_node->func_graph(); + if (old_func_graph != nullptr && old_func_graph->get_return() == old_node) { + MS_LOG(WARNING) << "Cannot replace the return node of a func graph " << old_func_graph->ToString(); + return false; + } + auto users = manager_->node_users()[old_node]; + for (auto& node : users) { + SetEdge(node.first, node.second, new_node); + } + + return true; +} + +void FuncGraphTransaction::SetEdge(const AnfNodePtr& src_node, int k, const AnfNodePtr& v) { + if (k < 0) { + MS_LOG(EXCEPTION) << "Invalid value k = " << k; + } + MS_EXCEPTION_IF_NULL(src_node); + auto cnode = src_node->cast(); + if (cnode == nullptr) { + MS_LOG(EXCEPTION) << "src_node should be a cnode, but cast failed."; + } + changes_.emplace_back(Change::kTxSetEdge, ArgsOfSetEdge{cnode, v, IntToSize(k)}); +} + +void FuncGraphTransaction::Commit() { + std::vector changes; + changes_.swap(changes); + manager_->CommitChanges(changes); +} + +FuncGraphAnalysis::FuncGraphAnalysis(const FuncGraphManager* const manager) + : manager_(manager), include_func_graph_none_(false) { + manager_->signals()->AddFuncGraph.connect(this, &FuncGraphAnalysis::OnAddFuncGraph); + manager_->signals()->DropFuncGraph.connect(this, &FuncGraphAnalysis::OnDropFuncGraph); + manager_->signals()->AddEdge.connect(this, &FuncGraphAnalysis::OnAddEdge); + manager_->signals()->DropEdge.connect(this, &FuncGraphAnalysis::OnDropEdge); + manager_->signals()->MoveAllCNode.connect(this, &FuncGraphAnalysis::OnMoveAllCNode); +} + +NodesCollector::NodesCollector(const FuncGraphManager* const m) : DepCollector(m), nodes_analysis_() { + include_func_graph_none_ = true; + nodes_analysis_[nullptr] = AnfNodeSet(); + + manager_->signals()->AddNode.connect(this, &NodesCollector::OnAddNode); + manager_->signals()->DropNode.connect(this, &NodesCollector::OnDropNode); +} + +void NodesCollector::OnAddNode(AnfNodePtr n) { + if (nodes_analysis_.find(n->func_graph()) == nodes_analysis_.end()) { + nodes_analysis_[n->func_graph()] = AnfNodeSet(); + } + + nodes_analysis_[n->func_graph()].add(n); +} + +void NodesCollector::OnDropNode(AnfNodePtr n) { + (void)nodes_analysis_[n->func_graph()].erase(n); + auto graph = n->func_graph(); + // Remove the node from order list. + if (graph) { + graph->EraseUnusedNodeInOrder(n); + } +} + +void NodesCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + // change the owner of node except for the src's return node + for (auto& it : nodes_analysis_[src]) { + nodes_analysis_[dst].add(it); + } + (void)nodes_analysis_.erase(src); +} + +void DepCollector::OnAddEdge(AnfNodePtr node, int index, AnfNodePtr inp) { OnModEdge(node, index, inp, kIncEdge); } + +DepCollector::DepCollector(const FuncGraphManager* const manager) : FuncGraphAnalysis(manager) { + MS_EXCEPTION_IF_NULL(manager_); + manager_->signals()->InvalidateCollector.connect(this, &DepCollector::OnInvalidateCollector); +} + +void DepCollector::OnDropEdge(AnfNodePtr node, int index, AnfNodePtr inp) { OnModEdge(node, index, inp, kDecEdge); } + +bool CounterAnfNodeCollector::Inc(const FuncGraphPtr& func_graph, const AnfNodePtr& key, int count = 1) { + auto& d = count_nodes_map_[func_graph]; + if (d.count(key) == 0) { + d[key] = count; + return true; + } else { + d[key] += count; + } + return false; +} + +bool CounterAnfNodeCollector::Dec(const FuncGraphPtr& func_graph, const AnfNodePtr& key, int count = 1) { + MS_EXCEPTION_IF_NULL(func_graph); + auto& d = count_nodes_map_[func_graph]; + if (d.count(key) != 0) { + if (d[key] == count) { + (void)d.erase(key); + return true; + } else { + d[key] -= count; + if (d[key] < 0) { + MS_LOG(EXCEPTION) << "Count of key '" << key->ToString() + << "' dec from 0. NodeInfo: " << trace::GetDebugInfo(func_graph->debug_info()); + } + } + } + return false; +} + +bool CounterAnfNodeCollector::Mod(const FuncGraphPtr& func_graph, const AnfNodePtr& key, int count) { + if (count > 0) { + return Inc(func_graph, key, count); + } else if (count < 0) { + return Dec(func_graph, key, -count); + } else { + MS_LOG(EXCEPTION) << "Count of key '" << key->ToString() + << "' cannot be 0. NodeInfo: " << trace::GetDebugInfo(func_graph->debug_info()); + } +} + +bool CounterFuncGraphCollector::Inc(const FuncGraphPtr& func_graph, const FuncGraphPtr& key, int count = 1) { + auto& d = count_func_graphs_map_[func_graph]; + if (d.count(key) == 0) { + d[key] = count; + return true; + } else { + d[key] += count; + } + return false; +} + +bool CounterFuncGraphCollector::Dec(const FuncGraphPtr& func_graph, const FuncGraphPtr& key, int count = 1) { + auto& d = count_func_graphs_map_[func_graph]; + if (d.count(key) != 0) { + if (d[key] == count) { + (void)d.erase(key); + return true; + } else { + d[key] -= count; + if (d[key] < 0) { + MS_LOG(EXCEPTION) << "Count of key '" << key->ToString() + << "' dec from 0. NodeInfo: " << trace::GetDebugInfo(func_graph->debug_info()); + } + } + } + return false; +} + +bool CounterFuncGraphCollector::Mod(const FuncGraphPtr& func_graph, const FuncGraphPtr& key, int count) { + if (count > 0) { + return Inc(func_graph, key, count); + } else if (count < 0) { + return Dec(func_graph, key, -count); + } else { + MS_LOG(EXCEPTION) << "Count of key '" << key->ToString() + << "' cannot be 0. NodeInfo: " << trace::GetDebugInfo(func_graph->debug_info()); + } +} + +void ValueNodesCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + if (inp->isa()) { + (void)Mod(node->func_graph(), inp, direction); + } +} + +void ValueNodesCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + for (auto& it : count_nodes_map_[src]) { + (void)Inc(dst, it.first, it.second); + } + (void)count_nodes_map_.erase(src); +} + +// if inp is a graph ValueNode, this graph's FuncGraphValueNodesCollector's value is inp self +void FuncGraphValueNodesCollector::OnModEdge(AnfNodePtr, int, AnfNodePtr inp, EdgeProcessDirection direction) { + if (IsValueNode(inp)) { + (void)Mod(GetValueNode(inp), inp, direction); + } +} + +void FuncGraphValueNodesCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + for (auto& it : count_nodes_map_[src]) { + (void)Inc(dst, it.first, it.second); + } + (void)count_nodes_map_.erase(src); +} + +void FVDirectCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(inp); + FuncGraphPtr fg1 = node->func_graph(); + FuncGraphPtr fg2 = inp->func_graph(); + if (nullptr != fg1 && nullptr != fg2 && fg1 != fg2) { + (void)Mod(fg1, inp, direction); + } +} + +void FVDirectCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + for (auto& it : count_nodes_map_[src]) { + FuncGraphPtr fg2 = it.first->func_graph(); + if (fg2 != dst) { + (void)Inc(dst, it.first, it.second); + } + } + (void)count_nodes_map_.erase(src); +} + +static FuncGraphPtr ParentProxy(const FuncGraphPtr& fg) { + FuncGraphPtr gn = std::make_shared(); + (void)gn->transforms().insert(std::make_pair("proxy", FuncGraphTransform(fg))); + return gn; +} + +void FuncGraphChildDirect::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(inp); + FuncGraphPtr fg1 = node->func_graph(); + FuncGraphPtr fg2 = inp->func_graph(); + if (nullptr != fg1 && nullptr != fg2 && fg1 != fg2) { + (void)Mod(fg2, fg1, direction); + } +} + +void FuncGraphChildDirect::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + for (auto& it : count_func_graphs_map_[src]) { + FuncGraphPtr fg = it.first; + if (fg != dst) { + (void)Inc(dst, fg, it.second); + } + } + (void)count_func_graphs_map_.erase(src); +} + +void FuncGraphParentsDirectCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + FuncGraphPtr fg1 = node->func_graph(); + // possible chirld parent + if (IsValueNode(inp)) { + FuncGraphPtr fg2 = GetValueNode(inp); + if (Mod(fg1, ParentProxy(fg2), direction)) { + manager_->signals()->InvalidateComputer(); + } + } + // from fv + FuncGraphPtr fg2 = inp->func_graph(); + if (nullptr != fg1 && nullptr != fg2 && fg1 != fg2) { + // node use fv will in here, fg1's node use fg2's node, so fg1 is child and fg2 is parent + if (Mod(fg1, fg2, direction)) { + manager_->signals()->InvalidateComputer(); + } + } +} + +void FuncGraphParentsDirectCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + for (auto& it : count_func_graphs_map_[src]) { + if (it.first != dst) { + (void)Inc(dst, it.first, it.second); + } + } + (void)count_func_graphs_map_.erase(src); +} + +void FuncGraphsUsedCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + if (IsValueNode(inp)) { + (void)Mod(node->func_graph(), GetValueNode(inp), direction); + } +} + +void FuncGraphsUsedCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + // all graph use in src need to change to dst, so meger the to dst use + for (auto& it : count_func_graphs_map_[src]) { + (void)Inc(dst, it.first, it.second); + } + (void)count_func_graphs_map_[dst].erase(src); + (void)count_func_graphs_map_.erase(src); +} + +void FuncGraphUsersCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + if (IsValueNode(inp)) { + (void)Mod(GetValueNode(inp), node->func_graph(), direction); + } +} + +void FuncGraphUsersCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr) { + // all graph use in src need to change to dst, so add dst user + (void)count_func_graphs_map_.erase(src); +} + +void FuncGraphUserNodesCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + MS_EXCEPTION_IF_NULL(node); + if (IsValueNode(inp)) { + (void)Mod(GetValueNode(inp), node, direction); + } +} + +void FuncGraphUserNodesCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + for (auto& it : count_nodes_map_[src]) { + (void)Inc(dst, it.first, it.second); + } + (void)count_nodes_map_.erase(src); +} + +void FuncGraphJDirectCollector::OnModEdge(AnfNodePtr node, int, AnfNodePtr inp, EdgeProcessDirection direction) { + if (IsValueNode(inp) && IsPrimitiveCNode(node, prim::kPrimJ)) { + (void)Mod(node->func_graph(), GetValueNode(inp), direction); + MS_LOG(DEBUG) << "" << node->func_graph()->ToString() << " users func graph " + << GetValueNode(inp)->ToString() << " which contains J(func_graph), dir: " << direction; + } +} + +void FuncGraphJDirectCollector::OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) { + // all graph use in src need to change to dst, so meger the to dst use + for (auto& it : count_func_graphs_map_[src]) { + (void)Inc(dst, it.first, it.second); + } + (void)count_func_graphs_map_.erase(src); +} + +DepComputer::DepComputer(const FuncGraphManager* const manager) : FuncGraphAnalysis(manager) { + MS_EXCEPTION_IF_NULL(manager_); + manager_->signals()->InvalidateComputer.connect(this, &DepComputer::OnInvalidateComputer); + validate_ = false; +} + +void DepComputer::Recompute() { + if (!validate_) { + RealRecompute(); + validate_ = true; + } +} + +void DepComputer::Recompute(const FuncGraphPtr& fg) { + if (func_graphs_validate_.count(fg) == 0 || !func_graphs_validate_[fg]) { + RealRecompute(fg); + func_graphs_validate_[fg] = true; + } +} + +FuncGraphSetPtr FuncGraphParentsTotalComputer::SeekParents(const FuncGraphPtr& fg, const FuncGraphSetPtr& path) { + if (path == nullptr || path->contains(fg)) { + return std::make_shared(); + } + FuncGraphSetPtr parents = std::make_shared(); + FuncGraphToFuncGraphCounterMap& deps = *all_parents_direct_; + for (auto& dep : deps[fg]) { + MS_EXCEPTION_IF_NULL(dep.first); + auto proxy = dep.first->transforms().find("proxy"); + if (proxy != dep.first->transforms().end()) { + path->add(fg); + auto gt = proxy->second.func_graph(); + parents->update(SeekParents(gt, path)); + } else { + parents->add(dep.first); + } + } + (void)parents->erase(fg); + return parents; +} + +void FuncGraphParentsTotalComputer::RealRecompute(FuncGraphPtr fg) { + MS_EXCEPTION_IF_NULL(fg); + all_parents_direct_ = &(manager_->func_graph_parents_direct()); + MS_LOG(DEBUG) << "" << fg->ToString() << " total func graph dep size:" << (*all_parents_direct_)[fg].size(); + func_graph_parents_total_analysis_[fg].update(SeekParents(fg)); + MS_LOG(DEBUG) << "FuncGraphParentsTotalComputer end: " << func_graph_parents_total_analysis_[fg].size(); +} + +bool set_len_compare(const FuncGraphSetPair& lhs, const FuncGraphSetPair& rhs) { + auto l1 = lhs.second.size(); + auto l2 = rhs.second.size(); + return l1 < l2; +} + +void ParentComputer::RealRecompute(FuncGraphPtr fg) { + this->parent_analysis_[fg] = nullptr; + // Note: must be a copy other than reference as it is modified thereafter. + auto deps = this->manager_->func_graph_parents_total(fg); + + if (deps.empty()) { + this->parent_analysis_[fg] = nullptr; + return; + } else if (deps.size() == 1) { + this->parent_analysis_[fg] = deps.pop(); + return; + } else { + // return nearest parent as parent + FuncGraphSet deps_copy(deps); + for (auto& dep : deps) { + auto parent_deps = this->manager_->func_graph_parents_total(dep); + for (auto& p_d : parent_deps) { + if (deps_copy.count(p_d)) { + (void)deps_copy.erase(p_d); + } + } + if (deps_copy.size() == 1) { + this->parent_analysis_[fg] = deps_copy.pop(); + return; + } + } + } +} + +// children include: +// A. func graphs which use variables in fg as free variables; (child_direct_) +// B. func graphs which call func func graph in A. (all_users_) +FuncGraphSetPtr ChildrenComputer::SeekChildren(const FuncGraphPtr& fg, const FuncGraphSetPtr& path) { + if (path == nullptr || path->contains(fg)) { + return std::make_shared(); + } + std::shared_ptr children = std::make_shared(); + auto& deps = *child_direct_; + auto& users = *all_users_; + MS_LOG(DEBUG) << "" << fg->ToString() << " start func graph dep size:" << deps[fg].size(); + for (auto& dep : deps[fg]) { + FuncGraphPtr child = dep.first; + children->add(child); + path->add(child); + MS_LOG(DEBUG) << "Child func graph:" << fg->ToString() << " child " << child->ToString(); + for (auto& user : users[child]) { + auto user_func_graph = user.first; + MS_LOG(DEBUG) << "Func graph:" << fg->ToString() << " user " << user_func_graph->ToString(); + children->add(user_func_graph); + path->add(user_func_graph); + } + children->update(SeekChildren(child, path)); + } + (void)children->erase(fg); + MS_LOG(DEBUG) << "End in children: " << children->size(); + return children; +} + +void ChildrenComputer::RealRecompute(FuncGraphPtr fg) { + MS_EXCEPTION_IF_NULL(manager_); + child_direct_ = &manager_->func_graph_child_direct(); + all_users_ = &manager_->func_graph_users(); + children_analysis_[fg].update(SeekChildren(fg)); +} + +void ScopeComputer::RealRecompute(FuncGraphPtr fg) { + MS_EXCEPTION_IF_NULL(manager_); + auto& children = manager_->children(fg); + + scope_analysis_[fg] = FuncGraphSet(); + scope_analysis_[fg].add(fg); + for (auto& child : children) { + scope_analysis_[fg].add(child); + } +} + +void FVTotalComputer::RealRecompute() { + auto manager = DepComputer::manager_; + MS_EXCEPTION_IF_NULL(manager); + + for (auto& fg : manager->func_graphs()) { + fv_total_analysis_[fg] = OrderedMap(); + count_nodes_map_[fg] = OrderedMap(); + count_func_graphs_map_[fg] = OrderedMap(); + } + + for (auto& fg : manager->func_graphs()) { + AnfNodeCounterMap items = manager->free_variables_direct()[fg]; + for (auto& iter : items) { + auto curr = fg; + while (curr) { + (void)CounterAnfNodeCollector::Mod(curr, iter.first, iter.second); + curr = manager->parent(curr); + const AnfNodeSet& nodes = manager->nodes()[curr]; + if (nodes.contains(iter.first)) { + break; + } + } + } + + auto items_fg = manager->func_graphs_used()[fg]; + for (auto& iter : items_fg) { + auto p = manager->parent(iter.first); + if (p == nullptr) { + continue; + } + auto curr = fg; + while (curr != p) { + (void)CounterFuncGraphCollector::Mod(curr, iter.first, iter.second); + curr = manager->parent(curr); + } + } + } + for (auto& fg : manager->func_graphs()) { + auto& fvp = count_nodes_map_[fg]; + auto& fvg = count_func_graphs_map_[fg]; + for (auto& item : fvp) { + fv_total_analysis_[fg][item.first] = item.second; + } + for (auto& item : fvg) { + fv_total_analysis_[fg][item.first] = item.second; + } + } +} + +void FuncGraphsUsedTotalComputer::RealRecompute(FuncGraphPtr fg) { + MS_EXCEPTION_IF_NULL(manager_); + auto& used = this->manager_->func_graphs_used(); + std::vector todo; + std::vector todo_new; + + todo.push_back(fg); + while (!todo.empty()) { + todo_new.clear(); + for (auto& gt : todo) { + for (auto& item : used[gt]) { + auto used_fg = item.first; + if (used_fg == fg) { + func_graph_used_total_analysis_[fg].add(used_fg); + continue; + } + if (func_graph_used_total_analysis_[fg].count(used_fg) == 0) { + todo_new.push_back(used_fg); + } + MS_LOG(DEBUG) << "" << fg->ToString() << " add func graph " << used_fg->ToString(); + func_graph_used_total_analysis_[fg].add(used_fg); + } + } + todo = todo_new; + } +} + +bool CheckRecursive(const FuncGraphManager* const manager, const FuncGraphPtr& fg) { + MS_EXCEPTION_IF_NULL(manager); + auto& used = manager->func_graphs_used(); + std::vector todo; + std::vector todo_new; + todo.push_back(fg); + FuncGraphSet used_total; + while (!todo.empty()) { + todo_new.clear(); + for (auto& gt : todo) { + for (auto& item : used[gt]) { + auto used_g = item.first; + if (used_g == fg) { + return true; + } + if (used_total.count(used_g) == 0) { + todo_new.push_back(used_g); + } + used_total.add(used_g); + } + } + todo = todo_new; + } + return false; +} + +void RecursiveComputer::RealRecompute(FuncGraphPtr fg) { + this->recursive_analysis_[fg] = CheckRecursive(this->manager_, fg); +} + +void RecursiveComputer::CheckRecursiveGraphs(const FuncGraphPtr& fg, std::list* trace) { + MS_EXCEPTION_IF_NULL(trace); + auto res = std::find(trace->begin(), trace->end(), fg); + // find recursive + if (res != trace->end()) { + auto recur_ptr = std::make_shared>(res, trace->end()); + for (auto iter = res; iter != trace->end(); (void)iter++) { + MS_LOG(DEBUG) << "Recursive graph " << (*iter)->ToString(); + recursive_map_[*iter] = recur_ptr; + } + } else { + trace->push_back(fg); + auto& used_fgs = manager_->func_graphs_used()[fg]; + for (auto iter = used_fgs.begin(); iter != used_fgs.end(); (void)iter++) { + CheckRecursiveGraphs(iter->first, trace); + } + trace->pop_back(); + if (!recursive_map_.count(fg)) { + recursive_map_[fg] = nullptr; + } + } +} + +bool FuncGraphJTotalComputer::SeekJ(const FuncGraphPtr& fg, const FuncGraphSetPtr& path) { + MS_EXCEPTION_IF_NULL(path); + if (path->contains(fg)) { + MS_LOG(DEBUG) << "" << fg->ToString() << " had been checked"; + return false; + } + MS_EXCEPTION_IF_NULL(manager_); + auto& func_graph_counter_map = manager_->func_graph_j_direct(); + if (!func_graph_counter_map[fg].empty()) { + // check g1->J(fg)->g2->g cycle; + auto contains_j = + std::find_if(func_graph_counter_map[fg].begin(), func_graph_counter_map[fg].end(), + [path](const std::pair iter) { return !path->contains(iter.first); }); + if (contains_j != func_graph_counter_map[fg].end()) { + MS_LOG(DEBUG) << "" << fg->ToString() << " contains J(" << contains_j->first->ToString() << ")"; + return true; + } + } + path->add(fg); + + // checkg if func graphs used contains J(func_graph); + auto& used = this->manager_->func_graphs_used(); + for (auto& item : used[fg]) { + auto used_g = item.first; + if (SeekJ(used_g, path)) { + MS_LOG(DEBUG) << "" << fg->ToString() << " users func graph " << used_g->ToString() + << " which contains J(func_graph)"; + return true; + } + } + MS_LOG(DEBUG) << "" << fg->ToString() << " doesn't contain J(func_graph)"; + return false; +} + +void FuncGraphJTotalComputer::RealRecompute(FuncGraphPtr fg) { + std::shared_ptr path = std::make_shared(); + this->j_total_analysis_[fg] = SeekJ(fg, path); +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/manager.h b/mindspore/ccsrc/ir/manager.h new file mode 100644 index 00000000000..8036bd68c0f --- /dev/null +++ b/mindspore/ccsrc/ir/manager.h @@ -0,0 +1,711 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_MANAGER_H_ +#define MINDSPORE_CCSRC_IR_MANAGER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/any.h" +#include "utils/misc.h" +#include "utils/signal.h" +#include "utils/ordered_set.h" +#include "utils/ordered_map.h" +#include "utils/graph_utils.h" +#include "utils/counter.h" +#include "utils/hashing.h" +#include "ir/anf.h" + +namespace mindspore { +struct Change; +class FuncGraphTransaction; +class FuncGraphManager; +using FuncGraphManagerPtr = std::shared_ptr; + +struct AnfNodeIndexPairHasher { + std::size_t operator()(const std::pair& p1) const { + return std::hash{}(p1.first.get()); + } +}; + +struct AnfNodeIndexPairEqual { + bool operator()(const std::pair& lhs, const std::pair& rhs) const { + return lhs == rhs; + } +}; +using AnfNodeIndexSet = OrderedSet, AnfNodeIndexPairHasher, AnfNodeIndexPairEqual>; +// NodeUsersMap, for node B input i use node A, it will be one item in map with key: A, and value: (B, i) +using NodeUsersMap = OrderedMap; +using FuncGraphSetPair = std::pair; +using FuncGraphSetPtr = std::shared_ptr; +using EdgeTuple = std::pair>; +struct EdgeTupleHasher { + std::size_t operator()(const EdgeTuple& p1) const { + return hash_combine({std::hash{}(p1.first.get()), std::hash{}(p1.second.first), + std::hash{}(p1.second.second.get())}); + } +}; + +struct EdgeTupleEqual { + bool operator()(const EdgeTuple& lhs, const EdgeTuple& rhs) const { + return lhs.first == rhs.first && lhs.second.first == rhs.second.first && lhs.second.second == rhs.second.second; + } +}; +using EdgeTupleCounter = Counter; +// manage the func graphs. +// if no manager exist, just create one and associate it to all func graphs; else reuse simply. +// func_graph, be managed graph +// manage: if true, created manager will be set in func_graph +// FuncGraphManagerPtr: return created manager +FuncGraphManagerPtr Manage(FuncGraphPtr func_graph, bool manage = true); + +FuncGraphManagerPtr Manage(const std::vector& func_graphs, bool manage = true); + +FuncGraphManagerPtr MakeManager(const std::vector& func_graphs = {}, bool manage = true); + +struct Signals { + Signal AddFuncGraph; + Signal DropFuncGraph; + Signal AddNode; + Signal DropNode; + Signal AddEdge; + Signal DropEdge; + Signal MoveAllCNode; + Signal InvalidateCollector; + Signal InvalidateComputer; +}; + +enum EdgeProcessDirection { kDecEdge = -1, kIncEdge = 1 }; + +using FuncGraphToFuncGraphCounterMap = OrderedMap>; +using FuncGraphToAnfNodeCounterMap = OrderedMap>; + +// analysis base class +class FuncGraphAnalysis { + public: + explicit FuncGraphAnalysis(const FuncGraphManager* const manager); + + virtual ~FuncGraphAnalysis() { manager_ = nullptr; } + + virtual size_t size() const { return 0; } + + virtual void OnAddFuncGraph(FuncGraphPtr) {} + + virtual void OnDropFuncGraph(FuncGraphPtr) {} + + virtual void OnMoveAllCNode(FuncGraphPtr, FuncGraphPtr) {} + + protected: + // subclass can reset their own member; + virtual void ExtraReset() {} + + virtual void OnAddNode(AnfNodePtr n) {} + + virtual void OnDropNode(AnfNodePtr n) {} + + virtual void OnAddEdge(AnfNodePtr, int, AnfNodePtr) {} + + virtual void OnDropEdge(AnfNodePtr, int, AnfNodePtr) {} + + const FuncGraphManager* manager_; + bool include_func_graph_none_; +}; + +using FuncGraphToAnfNodeMap = OrderedMap; + +// graphs analysis which compute in write, read needn't recompute +class DepCollector : public FuncGraphAnalysis { + public: + explicit DepCollector(const FuncGraphManager* manager); + ~DepCollector() override = default; + + void Reset() { ExtraReset(); } + void OnInvalidateCollector() { Reset(); } + + protected: + // inherit from FuncGraphAnalysis + void OnAddEdge(AnfNodePtr node, int index, AnfNodePtr inp) override; + void OnDropEdge(AnfNodePtr node, int index, AnfNodePtr inp) override; + // subclass can override; + virtual void OnModEdge(AnfNodePtr, int, AnfNodePtr, EdgeProcessDirection) {} +}; + +class NodesCollector final : public DepCollector { + public: + explicit NodesCollector(const FuncGraphManager* m); + ~NodesCollector() override = default; + + const FuncGraphToAnfNodeMap& nodes_analysis() const { return nodes_analysis_; } + size_t size() const override { return nodes_analysis_.size(); } + void OnAddFuncGraph(FuncGraphPtr fg) override { nodes_analysis_[fg] = AnfNodeSet(); } + + void OnDropFuncGraph(FuncGraphPtr fg) override { (void)nodes_analysis_.erase(fg); } + + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + + FuncGraphToAnfNodeMap nodes_analysis_; + + protected: + void ExtraReset() override { nodes_analysis_.clear(); } + void OnAddNode(AnfNodePtr n) override; + void OnDropNode(AnfNodePtr n) override; +}; + +class CounterFuncGraphCollector : public DepCollector { + public: + explicit CounterFuncGraphCollector(const FuncGraphManager* m) : DepCollector(m) {} + ~CounterFuncGraphCollector() override = default; + FuncGraphToFuncGraphCounterMap& count_func_graphs_map() { return count_func_graphs_map_; } + // inherit from FuncGraphAnalysis + size_t size() const override { return count_func_graphs_map_.size(); } + void OnAddFuncGraph(FuncGraphPtr fg) final { count_func_graphs_map_[fg] = OrderedMap(); } + void OnDropFuncGraph(FuncGraphPtr fg) final { (void)count_func_graphs_map_.erase(fg); } + bool Inc(const FuncGraphPtr& func_graph, const FuncGraphPtr& key, int count); + bool Dec(const FuncGraphPtr& func_graph, const FuncGraphPtr& key, int count); + bool Mod(const FuncGraphPtr& func_graph, const FuncGraphPtr& key, int count); + + FuncGraphToFuncGraphCounterMap count_func_graphs_map_; + + protected: + void ExtraReset() override { count_func_graphs_map_.clear(); } +}; + +class CounterAnfNodeCollector : public DepCollector { + public: + explicit CounterAnfNodeCollector(const FuncGraphManager* m) : DepCollector(m) {} + ~CounterAnfNodeCollector() override = default; + FuncGraphToAnfNodeCounterMap& count_nodes_map() { return count_nodes_map_; } + + size_t size() const override { return count_nodes_map_.size(); } + void OnAddFuncGraph(FuncGraphPtr fg) final { count_nodes_map_[fg] = OrderedMap(); } + void OnDropFuncGraph(FuncGraphPtr fg) final { (void)count_nodes_map_.erase(fg); } + + bool Inc(const FuncGraphPtr& func_graph, const AnfNodePtr& key, int count); + bool Dec(const FuncGraphPtr& func_graph, const AnfNodePtr& key, int count); + bool Mod(const FuncGraphPtr& func_graph, const AnfNodePtr& key, int count); + + FuncGraphToAnfNodeCounterMap count_nodes_map_; + + protected: + void ExtraReset() override { count_nodes_map_.clear(); } +}; + +class ValueNodesCollector final : public CounterAnfNodeCollector { + public: + explicit ValueNodesCollector(const FuncGraphManager* m) : CounterAnfNodeCollector(m) {} + ~ValueNodesCollector() override = default; + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +class FuncGraphValueNodesCollector final : public CounterAnfNodeCollector { + public: + explicit FuncGraphValueNodesCollector(const FuncGraphManager* m) : CounterAnfNodeCollector(m) {} + ~FuncGraphValueNodesCollector() override = default; + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +class FVDirectCollector final : public CounterAnfNodeCollector { + public: + explicit FVDirectCollector(const FuncGraphManager* m) : CounterAnfNodeCollector(m) {} + ~FVDirectCollector() override = default; + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +class FuncGraphChildDirect final : public CounterFuncGraphCollector { + public: + explicit FuncGraphChildDirect(const FuncGraphManager* m) : CounterFuncGraphCollector(m) {} + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + + ~FuncGraphChildDirect() override = default; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +// graph's all parents, parentsdirect have a map, which key is graph, value is this graph's all direct and proxy +// parents: +// 1.proxy parent: graph g use graph f, key is g, value is ParentProxy(f) because f's parent will be g's parent +// 2.direct parent: if graph g's node a used free_variable node in graph f, g's direct parent is f key is g, value is f +class FuncGraphParentsDirectCollector final : public CounterFuncGraphCollector { + public: + explicit FuncGraphParentsDirectCollector(const FuncGraphManager* m) : CounterFuncGraphCollector(m) {} + ~FuncGraphParentsDirectCollector() override = default; + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +// graph's all used graphs: key is g, value is g used graph +class FuncGraphsUsedCollector final : public CounterFuncGraphCollector { + public: + explicit FuncGraphsUsedCollector(const FuncGraphManager* m) : CounterFuncGraphCollector(m) {} + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + ~FuncGraphsUsedCollector() override = default; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +// graph's all user graphs: key is g, value is graphs who used g +class FuncGraphUsersCollector final : public CounterFuncGraphCollector { + public: + explicit FuncGraphUsersCollector(const FuncGraphManager* m) : CounterFuncGraphCollector(m) {} + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + ~FuncGraphUsersCollector() override = default; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +// graph's all user cnodes: key is g, value is cnodes who used g +class FuncGraphUserNodesCollector final : public CounterAnfNodeCollector { + public: + explicit FuncGraphUserNodesCollector(const FuncGraphManager* m) : CounterAnfNodeCollector(m) {} + void OnMoveAllCNode(FuncGraphPtr src, FuncGraphPtr dst) override; + ~FuncGraphUserNodesCollector() override = default; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +class FuncGraphJDirectCollector final : public CounterFuncGraphCollector { + public: + explicit FuncGraphJDirectCollector(const FuncGraphManager* m) : CounterFuncGraphCollector(m) {} + void OnMoveAllCNode(FuncGraphPtr src, const FuncGraphPtr dst) override; + ~FuncGraphJDirectCollector() override = default; + + protected: + void OnModEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction) override; +}; + +using FuncGraphToFuncGraphSetMap = OrderedMap; + +// graphs analysis which need dynamic compute by DepCollector in each read +class DepComputer : public FuncGraphAnalysis { + public: + explicit DepComputer(const FuncGraphManager* manager); + ~DepComputer() override = default; + + void Reset() { + ExtraReset(); + validate_ = false; + func_graphs_validate_.clear(); + } + + void OnInvalidateComputer() { Reset(); } + + void Recompute(); + + void Recompute(const FuncGraphPtr& fg); + + bool IsValidate() const { return validate_; } + + bool IsValidate(const FuncGraphPtr& fg) { return func_graphs_validate_[fg]; } + + void OnAddFuncGraph(FuncGraphPtr) final { Reset(); } + + void OnDropFuncGraph(FuncGraphPtr) final { Reset(); } + + protected: + // subclass do the real compute + virtual void RealRecompute() {} + virtual void RealRecompute(FuncGraphPtr) {} + + bool validate_; + OrderedMap func_graphs_validate_; + + private: + friend FuncGraphManager; +}; + +// graph g's all direct or proxy parents +class FuncGraphParentsTotalComputer final : public DepComputer { + public: + explicit FuncGraphParentsTotalComputer(const FuncGraphManager* m) : DepComputer(m), all_parents_direct_(nullptr) {} + ~FuncGraphParentsTotalComputer() override { all_parents_direct_ = nullptr; } + + FuncGraphToFuncGraphSetMap& func_graph_parents_total_analysis() { return func_graph_parents_total_analysis_; } + + size_t size() const override { return func_graph_parents_total_analysis_.size(); } + + FuncGraphToFuncGraphSetMap func_graph_parents_total_analysis_; + + protected: + void ExtraReset() override { func_graph_parents_total_analysis_.clear(); } + + void RealRecompute(FuncGraphPtr fg) override; + + private: + FuncGraphSetPtr SeekParents(const FuncGraphPtr& fg, const FuncGraphSetPtr& path = std::make_shared()); + // when SeekParents calls itself recursively, it can access these variables by class member + // other than pass by formal parameters, it can save 1 parameter for SeekParents(). + FuncGraphToFuncGraphCounterMap* all_parents_direct_; +}; + +using FuncGraphToFuncGraphMap = OrderedMap; + +// graph's nearest parent in parents total +class ParentComputer final : public DepComputer { + public: + explicit ParentComputer(const FuncGraphManager* m) : DepComputer(m) {} + ~ParentComputer() override = default; + + FuncGraphToFuncGraphMap& parent_analysis() { return parent_analysis_; } + + size_t size() const override { return parent_analysis_.size(); } + + FuncGraphToFuncGraphMap parent_analysis_; + + protected: + void ExtraReset() override { parent_analysis_.clear(); } + + void RealRecompute(FuncGraphPtr fg) override; +}; + +// graph's children graph except self +class ChildrenComputer final : public DepComputer { + public: + explicit ChildrenComputer(const FuncGraphManager* m) : DepComputer(m), all_users_(nullptr), child_direct_(nullptr) {} + ~ChildrenComputer() override { + all_users_ = nullptr; + child_direct_ = nullptr; + } + + FuncGraphToFuncGraphSetMap& children_analysis() { return children_analysis_; } + + size_t size() const override { return children_analysis_.size(); } + + FuncGraphToFuncGraphSetMap children_analysis_; + + protected: + void ExtraReset() override { children_analysis_.clear(); } + + void RealRecompute(FuncGraphPtr fg) override; + + private: + FuncGraphSetPtr SeekChildren(const FuncGraphPtr& fg, const FuncGraphSetPtr& path = std::make_shared()); + // when SeekChildren calls itself recursively, it can access these variables by class member + // other than pass by formal parameters, it can save 2 parameters for SeekChildren(). + FuncGraphToFuncGraphCounterMap* all_users_; + FuncGraphToFuncGraphCounterMap* child_direct_; +}; + +// graph's children graph include self +class ScopeComputer final : public DepComputer { + public: + explicit ScopeComputer(const FuncGraphManager* m) : DepComputer(m) {} + ~ScopeComputer() override = default; + + FuncGraphToFuncGraphSetMap& scope_analysis() { return scope_analysis_; } + + size_t size() const override { return scope_analysis_.size(); } + + FuncGraphToFuncGraphSetMap scope_analysis_; + + protected: + void ExtraReset() override { scope_analysis_.clear(); } + + void RealRecompute(FuncGraphPtr fg) override; +}; + +using FVTotalMap = OrderedMap>; + +class FVTotalComputer final : public DepComputer, public CounterAnfNodeCollector, public CounterFuncGraphCollector { + public: + explicit FVTotalComputer(const FuncGraphManager* m) + : DepComputer(m), CounterAnfNodeCollector(m), CounterFuncGraphCollector(m) {} + ~FVTotalComputer() override = default; + + FVTotalMap& fv_total_analysis() { return fv_total_analysis_; } + + size_t size() const override { return fv_total_analysis_.size(); } + + FVTotalMap fv_total_analysis_; + + protected: + void ExtraReset() override { fv_total_analysis_.clear(); } + + void RealRecompute() override; +}; + +class FuncGraphsUsedTotalComputer final : public DepComputer { + public: + explicit FuncGraphsUsedTotalComputer(const FuncGraphManager* m) : DepComputer(m) {} + ~FuncGraphsUsedTotalComputer() override = default; + + FuncGraphToFuncGraphSetMap& func_graph_used_total_analysis() { return func_graph_used_total_analysis_; } + + size_t size() const override { return func_graph_used_total_analysis_.size(); } + + FuncGraphToFuncGraphSetMap func_graph_used_total_analysis_; + + protected: + void ExtraReset() override { func_graph_used_total_analysis_.clear(); } + + void RealRecompute(FuncGraphPtr fg) override; +}; + +using FuncGraphToBoolMap = OrderedMap; +using RecursiveMap = OrderedMap>>; + +class RecursiveComputer final : public DepComputer { + public: + explicit RecursiveComputer(const FuncGraphManager* m) : DepComputer(m) {} + ~RecursiveComputer() override = default; + + RecursiveMap& recursive_map() { return recursive_map_; } + FuncGraphToBoolMap& recursive_analysis() { return recursive_analysis_; } + + void CheckRecursiveGraphs(const FuncGraphPtr& fg, std::list* trace); + + size_t size() const override { return recursive_analysis_.size(); } + + RecursiveMap recursive_map_; + FuncGraphToBoolMap recursive_analysis_; + + protected: + void ExtraReset() override { + recursive_analysis_.clear(); + recursive_map_.clear(); + } + + void RealRecompute(FuncGraphPtr fg) override; +}; + +class FuncGraphJTotalComputer final : public DepComputer { + public: + explicit FuncGraphJTotalComputer(const FuncGraphManager* m) : DepComputer(m) {} + ~FuncGraphJTotalComputer() override = default; + + FuncGraphToBoolMap& j_total_analysis() { return j_total_analysis_; } + + size_t size() const override { return j_total_analysis_.size(); } + + FuncGraphToBoolMap j_total_analysis_; + + protected: + void ExtraReset() override { j_total_analysis_.clear(); } + + void RealRecompute(FuncGraphPtr fg) override; + bool SeekJ(const FuncGraphPtr& fg, const FuncGraphSetPtr& path); +}; + +class FuncGraphManager : public std::enable_shared_from_this { + public: + explicit FuncGraphManager(const std::vector& roots, bool manage = true); + ~FuncGraphManager() { + if (is_manage_) { + RemoveRoots(); + } + } + + void Reset(); + void Init(); + void Clear(); + void AddFuncGraph(FuncGraphPtr func_graph, bool is_root = false); + void KeepRoots(const std::vector& roots = {}); + void RemoveRoots(); + void SetParameters(const FuncGraphPtr& fg, const std::vector& parameters); + void MaybeDropFuncGraphs(const FuncGraphSet& func_graphs, bool ignore_users = false); + bool Replace(const AnfNodePtr& old_node, const AnfNodePtr& new_node); + void SetEdge(const AnfNodePtr& node, int index, const AnfNodePtr& value); + void MoveAllCNodeDropGraph(FuncGraphPtr source, FuncGraphPtr target, const ScopePtr& scope); + + FuncGraphTransaction Transact(); + void CommitChanges(const std::vector& changes); + + bool IsManaged() const { return is_manage_; } + + const FuncGraphSet& roots() const { return roots_; } + + const FuncGraphSet& func_graphs() const { return func_graphs_; } + + AnfNodeSet& all_nodes() { return all_nodes_; } + + NodeUsersMap& node_users() { return node_users_; } + + FuncGraphToAnfNodeMap& nodes() const { return nodes_->nodes_analysis_; } + + FuncGraphToAnfNodeCounterMap& valuenodes() const { return valuenodes_->count_nodes_map_; } + + FuncGraphToAnfNodeCounterMap& free_variables_direct() const { return free_variables_direct_->count_nodes_map_; } + + FuncGraphToAnfNodeCounterMap& func_graph_valuenodes() const { return func_graph_valuenodes_->count_nodes_map_; } + + FuncGraphToFuncGraphCounterMap& func_graphs_used() const { return func_graphs_used_->count_func_graphs_map_; } + + FuncGraphToFuncGraphCounterMap& func_graph_users() const { return func_graph_users_->count_func_graphs_map_; } + + FuncGraphToAnfNodeCounterMap& func_graph_user_cnodes() const { return func_graph_user_cnodes_->count_nodes_map_; } + + FuncGraphToFuncGraphCounterMap& func_graph_child_direct() const { + return func_graph_child_direct_->count_func_graphs_map_; + } + + FuncGraphToFuncGraphCounterMap& func_graph_parents_direct() const { + return func_graph_parents_direct_->count_func_graphs_map_; + } + + FuncGraphToFuncGraphCounterMap& func_graph_j_direct() const { return func_graph_j_direct_->count_func_graphs_map_; } + + FVTotalMap& free_variables_total() const; + + FuncGraphSet& func_graph_parents_total(const FuncGraphPtr& fg) const; + + FuncGraphSet& scopes(const FuncGraphPtr& fg) const; + + FuncGraphPtr parent(const FuncGraphPtr& fg) const; + + FuncGraphSet& children(const FuncGraphPtr& fg) const; + + FuncGraphSet& func_graphs_used_total(const FuncGraphPtr& fg) const; + + bool recursive(const FuncGraphPtr& fg) const; + std::shared_ptr> recursive_graphs(const FuncGraphPtr& fg) const; + + bool func_graph_j_total(const FuncGraphPtr& fg) const; + + std::shared_ptr signals() const { return signals_; } + + IncludeType Limit(const AnfNodePtr& node); + + // Static Analysis + NodeUsersMap node_users_; + AnfNodeSet all_nodes_; // managed nodes + std::shared_ptr nodes_; + std::shared_ptr valuenodes_; + std::shared_ptr free_variables_direct_; + std::shared_ptr func_graph_valuenodes_; + std::shared_ptr func_graphs_used_; + std::shared_ptr func_graph_users_; + std::shared_ptr func_graph_user_cnodes_; + std::shared_ptr func_graph_child_direct_; + std::shared_ptr func_graph_parents_direct_; + std::shared_ptr func_graph_j_direct_; + + // Dynamic Analysis + std::shared_ptr func_graph_parent_; + + private: + void AddIntoManaged(const FuncGraphPtr& fg); + void ProcessEdge(AnfNodePtr node, int index, AnfNodePtr inp, EdgeProcessDirection direction); + void ProcessInputs(const AnfNodePtr& node, EdgeProcessDirection direction); + void AcquireNodes(const std::vector& nodes); + FuncGraphSetPtr MaybeDropNodes(const std::vector& nodes); + void ParseChanges(const std::vector& changes, EdgeTupleCounter* add_edges, EdgeTupleCounter* rm_edges, + Counter* adds, Counter* rms); + + FuncGraphSet roots_; // managed roots + FuncGraphSet func_graphs_; // managed func graphs + + std::shared_ptr signals_; + + // Dynamic Analysis + std::shared_ptr func_graph_parents_total_; + std::shared_ptr children_; + std::shared_ptr scopes_; + std::shared_ptr free_variables_total_; + std::shared_ptr func_graphs_used_total_; + std::shared_ptr recursive_; + std::shared_ptr j_total_; + + bool is_manage_; +}; + +class FuncGraphTransaction { + public: + explicit FuncGraphTransaction(FuncGraphManager* manager) : manager_(manager), changes_() { + MS_EXCEPTION_IF_NULL(manager_); + if (!manager_->IsManaged()) { + MS_LOG(DEBUG) << "the manager is not managed yet"; + } + } + + FuncGraphTransaction() : manager_(nullptr) {} + ~FuncGraphTransaction() { manager_ = nullptr; } + + // set parameters of a func graph + void SetParameters(FuncGraphPtr fg, const std::vector& params); + + // replace old_node with new_node + bool Replace(const AnfNodePtr& old_node, const AnfNodePtr& new_node); + + // set esge, i.e., declare setting node.inputs[key] to value. + void SetEdge(const AnfNodePtr& src_node, int k, const AnfNodePtr& v); + + // commit all changes + void Commit(); + + private: + FuncGraphManager* manager_; + std::vector changes_; +}; + +// args for set param +struct ArgsOfSetParams { + FuncGraphPtr func_graph; + std::vector params; + bool operator==(const ArgsOfSetParams& other) const { return &other == this; } + + friend std::ostream& operator<<(std::ostream& os, const ArgsOfSetParams&) { + os << "[ArgsOfSetParams]"; + return os; + } +}; + +// args for set edge +struct ArgsOfSetEdge { + CNodePtr root_node; + AnfNodePtr new_node; + size_t index; + bool operator==(const ArgsOfSetEdge& other) const { return &other == this; } + + friend std::ostream& operator<<(std::ostream& os, const ArgsOfSetEdge& other) { + os << "[ArgsOfSetEdge]"; + return os; + } +}; + +struct Change { + enum OpName { kTxSetParams, kTxSetEdge }; + OpName op; + Any args; + Change(OpName name, const Any& para) : op(name), args(para) {} +}; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_MANAGER_H_ diff --git a/mindspore/ccsrc/ir/meta_func_graph.cc b/mindspore/ccsrc/ir/meta_func_graph.cc new file mode 100644 index 00000000000..58a500a28cb --- /dev/null +++ b/mindspore/ccsrc/ir/meta_func_graph.cc @@ -0,0 +1,35 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/meta_func_graph.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "pipeline/static_analysis/abstract_function.h" + +// namespace to support intermediate representation definition +namespace mindspore { +abstract::AbstractBasePtr MetaFuncGraph::MakeAbstractClosure(const AnfNodePtr &anf_node) { + abstract::MetaFuncGraphAbstractClosurePtr meta_func_graph_fn; + if (anf_node == nullptr) { + meta_func_graph_fn = std::make_shared(shared_from_base()); + } else { + meta_func_graph_fn = + std::make_shared(shared_from_base(), anf_node->scope()); + } + return meta_func_graph_fn; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/meta_func_graph.h b/mindspore/ccsrc/ir/meta_func_graph.h new file mode 100644 index 00000000000..69da925e3df --- /dev/null +++ b/mindspore/ccsrc/ir/meta_func_graph.h @@ -0,0 +1,115 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_META_FUNC_GRAPH_H_ +#define MINDSPORE_CCSRC_IR_META_FUNC_GRAPH_H_ + +#include +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" + +#include "ir/dtype.h" +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace py = pybind11; + +namespace mindspore { +// namespace to support intermediate representation definition +// Graph generator. +// Can be called with a pipeline's resources and a list of argument types to +// generate a graph corresponding to these types. +class MetaFuncGraph : public FuncGraphBase { + public: + explicit MetaFuncGraph(const std::string& name) : name_(name) { cache_.clear(); } + + ~MetaFuncGraph() override = default; + + MS_DECLARE_PARENT(MetaFuncGraph, FuncGraphBase); + abstract::AbstractBasePtr MakeAbstractClosure(const AnfNodePtr& anf_node); + // Return normalized versions of the arguments. + // By default, this returns args unchanged. + virtual abstract::AbstractBasePtrList NormalizeArgs(const abstract::AbstractBasePtrList& args_spec_list) const { + return args_spec_list; + } + + const std::vector& signatures() const { return signatures_; } + void set_signatures(const std::vector& signatures) { signatures_ = signatures; } + // Generate a Graph for the given abstract arguments. + virtual FuncGraphPtr GenerateFuncGraph(const abstract::AbstractBasePtrList& args_spec_list) { + TypePtrList types; + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(types), + [](const AbstractBasePtr& arg) -> TypePtr { + MS_EXCEPTION_IF_NULL(arg); + return arg->BuildType(); + }); + // filter unsafe characters in log print since name_ is from outside + auto iter = cache_.find(types); + if (iter == cache_.end()) { + FuncGraphPtr fg = GenerateFromTypes(types); + MS_EXCEPTION_IF_NULL(fg); + MS_LOG(INFO) << "MetaFuncgraph: cache miss for types: " << mindspore::ToString(args_spec_list) + << ", g: " << fg->ToString(); + cache_[types] = fg; + return fg; + } else { + MS_LOG(DEBUG) << "MetaFuncgraph: cache hit for types: " << mindspore::ToString(args_spec_list) + << ", g: " << iter->second->ToString(); + return iter->second; + } + } + + // Generate a Graph for this type signature. + virtual FuncGraphPtr GenerateFromTypes(const TypePtrList&) { + MS_LOG(EXCEPTION) << "Undefine the method of generating graph from types."; + } + + std::string name() { return name_; } + std::string ToString() const override { return name_; } + std::size_t hash() const override { return tid(); } + + virtual bool operator==(const MetaFuncGraph& other) const { return &other == this; } + bool operator==(const Value& other) const override { + if (other.isa()) { + return &other == this; + } else { + return false; + } + } + const bool parse_info_ = true; + + protected: + template + std::shared_ptr shared_from_base() { + return std::static_pointer_cast(shared_from_this()); + } + std::string name_; + std::vector signatures_; + std::unordered_map cache_; +}; + +using MetaFuncGraphPtr = std::shared_ptr; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_META_FUNC_GRAPH_H_ diff --git a/mindspore/ccsrc/ir/meta_tensor.cc b/mindspore/ccsrc/ir/meta_tensor.cc new file mode 100644 index 00000000000..d33bc10c273 --- /dev/null +++ b/mindspore/ccsrc/ir/meta_tensor.cc @@ -0,0 +1,562 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/meta_tensor.h" + +#include +#include +#include +#include +#include + +#include "device/device_address.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" +#include "pynative/pynative_execute.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { + +namespace tensor { + +void DataBuf2Contiguous(const py::array& src, py::array* const dest) { + if (dest == nullptr) { + MS_LOG(EXCEPTION) << "Failed to copy data to a contiguous buffer as dest is nullptr!"; + } + + Py_buffer pybuf_src; + if (PyObject_GetBuffer(src.ptr(), &pybuf_src, PyBUF_ANY_CONTIGUOUS)) { + MS_LOG(EXCEPTION) << "Failed to get buffer info from the src!"; + } + + if (!PyBuffer_IsContiguous(&pybuf_src, 'C')) { + if (PyBuffer_ToContiguous(dest->request(true).ptr, &pybuf_src, pybuf_src.len, 'C')) { + MS_LOG(EXCEPTION) << "Can't copy numpy.ndarray to a contiguous buffer."; + } + } else { + *dest = src; + } + + PyBuffer_Release(&pybuf_src); +} + +// MetaTensor has default type_id_ which is TypeId::kTypeUnknown. +MetaTensor::MetaTensor() : data_type_(TypeId::kTypeUnknown) {} + +MetaTensor::MetaTensor(const TypeId data_type, const std::vector& shape) : data_type_(data_type), shape_(shape) {} + +MetaTensor::MetaTensor(const TypePtr& type_ptr, const py::tuple& shape) { + TypeId data_type = TypeId::kTypeUnknown; + if (type_ptr != nullptr) { + data_type = type_ptr->type_id(); + } + data_type_ = data_type; + shape_.resize(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + shape_[i] = py::int_(shape[i]); + } +} + +MetaTensor::MetaTensor(const MetaTensor& meta_tensor) + : Value(meta_tensor), data_type_(meta_tensor.data_type()), shape_(meta_tensor.shape()) {} + +MetaTensor& MetaTensor::operator=(const MetaTensor& meta_tensor) { + if (&meta_tensor == this) { + return *this; + } + + data_type_ = meta_tensor.data_type(); + shape_ = meta_tensor.shape(); + device_info_ = meta_tensor.device_info(); + + return *this; +} + +bool MetaTensor::operator==(const MetaTensor& meta_tensor) const { + return data_type_ == meta_tensor.data_type() && shape_ == meta_tensor.shape(); +} + +// Get the size of a given dimension by its index number. +// The given index number should be in [0, shape_.size()). +// param index Dimension index number. +// return The size of the dimension if succeed, or -1 if failed. +int MetaTensor::DimensionSize(const size_t index) const { + int dim_size = -1; + if (index < shape_.size()) { + dim_size = shape_[index]; + } else { + MS_LOG(ERROR) << "Dimension index is wrong: " << index; + } + return dim_size; +} + +int MetaTensor::ElementsNum() const { + return std::accumulate(shape_.begin(), shape_.end(), 1LL, std::multiplies()); +} + +TypePtr MetaTensor::Dtype() const { return TypeIdToType(data_type_); } + +TypePtr MetaTensor::SetDtype(const TypePtr type_ptr) { + if (type_ptr == nullptr) { + MS_LOG(ERROR) << "Dtype to be set is nullptr."; + return nullptr; + } + (void)set_data_type(type_ptr->type_id()); + return type_ptr; +} + +void MetaTensor::SetDeviceInfo(const std::string& format, const TypePtr& data_type) { + DeviceInfo info(format, data_type); + set_device_info(info); +} + +std::string MetaTensor::ToString() const { + std::ostringstream buf; + buf << "MetaTensor shape:[" << shape() << "]"; + return buf.str(); +} + +std::string MetaTensor::DumpText() const { + std::ostringstream oss; + oss << type_name() << "(" << SizeToInt(data_type_) << ")["; + for (size_t i = 0; i < shape_.size(); ++i) { + oss << (i > 0 ? ", " : "") << shape_[i]; + } + oss << "]"; + return oss.str(); +} + +Tensor::Tensor(const TypePtr& type_ptr, const py::tuple& shape) { + TypeId data_type = TypeId::kTypeUnknown; + if (type_ptr != nullptr) { + data_type = type_ptr->type_id(); + } + data_type_ = data_type; + shape_.resize(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + shape_[i] = py::int_(shape[i]); + } + init(data_type_, shape_, &data_); +} + +Tensor::Tensor(TypeId data_type, const std::vector& shape) { init(data_type, shape, &data_); } + +Tensor::Tensor(const py::array& input, const TypePtr& data_type) { init(input, data_type); } + +Tensor::Tensor(const py::list& input, const TypePtr& data_type) { init(py::array(input), data_type); } + +Tensor::Tensor(const py::tuple& input, const TypePtr& data_type) { init(py::array(input), data_type); } + +Tensor::Tensor(const py::float_& input, const TypePtr& data_type) { init(py::array(input), data_type); } + +Tensor::Tensor(const py::int_& input, const TypePtr& data_type) { init(py::array(input), data_type); } + +Tensor::Tensor(const Tensor& tensor, const TypePtr& data_type) + : MetaTensor(tensor), device_address_(tensor.device_address()) { + init(tensor.data_, data_type); +} + +Tensor& Tensor::operator=(const Tensor& tensor) { + if (this != &tensor) { + MetaTensor::operator=(tensor); + dirty_ = tensor.is_dirty(); + device_address_ = tensor.device_address(); + data_ = tensor.data_; + } + return *this; +} + +bool Tensor::operator==(const Tensor& tensor) const { + return (MetaTensor::operator==(tensor) && data_ == tensor.data_); +} + +bool Tensor::ValueEqualPy(const py::object& other) const { + if (!py::isinstance(other)) { + MS_LOG(WARNING) << "compare other not a tensor"; + return false; + } + return ValueEqual(py::cast(other)); +} + +bool Tensor::ValueEqual(const Tensor& other) const { + auto equal = [&other, this]() -> bool { + auto np = py::module::import("numpy"); + auto equal = np.attr("equal")(data_, other.data_); + auto all_equal = np.attr("all")(equal); + return all_equal.cast(); + }; + return (MetaTensor::operator==(other) && (data_.is(other.data_) || equal())); +} + +int Tensor::DataDim() const { return static_cast(data_.ndim()); } + +int Tensor::DataSize() const { return static_cast(data_.size()); } + +py::tuple Tensor::GetPyTupleShape() const { + py::tuple dims(shape_.size()); + for (size_t i = 0; i < dims.size(); ++i) { + dims[i] = py::int_(shape_[i]); + } + return dims; +} + +py::array Tensor::data() const { return data_; } + +int Tensor::data_type_c() const { return static_cast(data_type_); } + +std::vector Tensor::shape_c(void) const { return shape(); } + +void* Tensor::data_c(bool writable) { + // operand of bit operation should be unsigned int. + unsigned int flags = ((unsigned int)data_.flags()) & pybind11::detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_; + bool is_c_contiguous = (flags != 0) ? true : false; + if (!is_c_contiguous) { + py::array data_c; + init(data_type_, shape_, &data_c); + DataBuf2Contiguous(data_, &data_c); + data_ = data_c; + } + return data_.request(writable).ptr; +} + +TypeId Tensor::GetDataType(const py::buffer_info& buf) const { + TypeId data_type = TypeId::kTypeUnknown; + if (buf.format.compare("e") == 0) { + data_type = TypeId::kNumberTypeFloat16; + } else if (buf.format.compare("f") == 0) { + data_type = TypeId::kNumberTypeFloat32; + } else if (buf.format.compare("d") == 0) { + data_type = TypeId::kNumberTypeFloat64; + } else if (buf.format.compare("B") == 0) { + data_type = TypeId::kNumberTypeUInt8; + } else if (buf.format.compare("H") == 0) { + data_type = TypeId::kNumberTypeUInt16; + } else if (buf.format.compare("I") == 0) { + data_type = TypeId::kNumberTypeUInt32; + } else if (buf.format.compare("L") == 0 || buf.format.compare("Q") == 0) { + data_type = TypeId::kNumberTypeUInt64; + } else if (buf.format.compare("b") == 0) { + data_type = TypeId::kNumberTypeInt8; + } else if (buf.format.compare("h") == 0) { + data_type = TypeId::kNumberTypeInt16; + } else if (buf.format.compare("i") == 0) { + data_type = TypeId::kNumberTypeInt32; + } else if (buf.format.compare("l") == 0 || buf.format.compare("q") == 0) { + data_type = TypeId::kNumberTypeInt64; + } else if (buf.format.compare("?") == 0) { + data_type = TypeId::kNumberTypeBool; + } else { + MS_LOG(WARNING) << "Get unsupported DataType " << buf.format << "."; + } + return data_type; +} + +void Tensor::init(const py::array& input, const TypePtr& type_ptr) { + TypeId data_type = TypeId::kTypeUnknown; + if (type_ptr != nullptr) { + data_type = type_ptr->type_id(); + } + init(input, data_type); +} + +void Tensor::init(const py::array& input, const TypeId& data_type) { + py::buffer_info buf = input.request(); + + data_type_ = GetDataType(buf); + if (TypeId::kTypeUnknown == data_type && TypeId::kTypeUnknown == data_type_) { + MS_LOG(EXCEPTION) << "Unsupported tensor type!"; + } + + std::vector tm = buf.shape; + size_t len = tm.size(); + std::vector dims(len); + for (size_t i = 0; i < len; ++i) { + dims[i] = static_cast(tm[i]); + } + (void)set_shape(dims); + + if (TypeId::kTypeUnknown != data_type && TypeId::kTypeUnknown != data_type_ && data_type_ != data_type) { + // If user defined data type is not same as GetDataType from the data + bool success = convert_data(input, data_type_, &data_, data_type); + if (success) { + data_type_ = data_type; + } else { + data_type_ = TypeId::kTypeUnknown; + MS_LOG(EXCEPTION) << "Convert data from " << data_type_ << " to " << data_type << " failed!"; + } + } else { + data_ = input; + } +} + +void Tensor::init(TypeId data_type, const std::vector& shape, py::array* const data) { + data_type_ = data_type; + shape_ = shape; + switch (data_type) { + case kNumberTypeBool: + *data = py::array_t(shape); + break; + case kNumberTypeInt8: + *data = py::array_t(shape); + break; + case kNumberTypeInt16: + *data = py::array_t(shape); + break; + case kNumberTypeInt32: + *data = py::array_t(shape); + break; + case kNumberTypeInt64: + *data = py::array_t(shape); + break; + case kNumberTypeUInt8: + *data = py::array_t(shape); + break; + case kNumberTypeUInt16: + *data = py::array_t(shape); + break; + case kNumberTypeUInt32: + *data = py::array_t(shape); + break; + case kNumberTypeUInt64: + *data = py::array_t(shape); + break; + case kNumberTypeFloat16: + *data = py::array_t(shape); + break; + case kNumberTypeFloat32: + *data = py::array_t(shape); + break; + case kNumberTypeFloat64: + *data = py::array_t(shape); + break; + default: + MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; + break; + } +} + +TypePtr Tensor::SetDtype(const TypePtr type_ptr) { + MS_EXCEPTION_IF_NULL(type_ptr); + (void)set_data_type(type_ptr->type_id()); + return type_ptr; +} + +TypeId Tensor::set_data_type(const TypeId data_type) { + if (data_.size() > 0 && data_type_ != data_type) { + bool success = convert_data(data_, data_type_, &data_, data_type); + if (success) { + data_type_ = data_type; + } else { + MS_LOG(EXCEPTION) << "Convert data from " << data_type_ << " to " << data_type << " failed!"; + } + } else if (data_.size() == 0) { + data_type_ = data_type; + } + + return data_type_; +} + +bool Tensor::convert_data(const py::array& in, const TypeId in_data_type, py::array* const out, + const TypeId out_data_type) { + if (out == nullptr) { + return false; + } + + bool result = true; + if (TypeId::kTypeUnknown == in_data_type || TypeId::kTypeUnknown == out_data_type) { + result = false; + } else if (in_data_type == out_data_type) { + *out = in; + } else if (TypeId::kNumberTypeFloat64 == out_data_type) { + *out = in.attr("astype").cast()("float64").cast(); + } else if (TypeId::kNumberTypeFloat32 == out_data_type) { + *out = in.attr("astype").cast()("float32").cast(); + } else if (TypeId::kNumberTypeFloat16 == out_data_type) { + *out = in.attr("astype").cast()("float16").cast(); + } else if (TypeId::kNumberTypeInt64 == out_data_type) { + *out = in.attr("astype").cast()("int64").cast(); + } else if (TypeId::kNumberTypeInt32 == out_data_type) { + *out = in.attr("astype").cast()("int32").cast(); + } else if (TypeId::kNumberTypeInt16 == out_data_type) { + *out = in.attr("astype").cast()("int16").cast(); + } else if (TypeId::kNumberTypeInt8 == out_data_type) { + *out = in.attr("astype").cast()("int8").cast(); + } else if (TypeId::kNumberTypeUInt8 == out_data_type) { + *out = in.attr("astype").cast()("uint8").cast(); + } else if (TypeId::kNumberTypeUInt16 == out_data_type) { + *out = in.attr("astype").cast()("uint16").cast(); + } else if (TypeId::kNumberTypeUInt32 == out_data_type) { + *out = in.attr("astype").cast()("uint32").cast(); + } else if (TypeId::kNumberTypeUInt64 == out_data_type) { + *out = in.attr("astype").cast()("uint64").cast(); + } else { + data_type_ = TypeId::kTypeUnknown; + MS_LOG(EXCEPTION) << "Cannot convert from " << TypeIdLabel(in_data_type) << " to " << TypeIdLabel(out_data_type) + << "."; + } + + return result; +} + +abstract::AbstractBasePtr Tensor::ToAbstract() { + auto tens = shared_from_base(); + auto dtype = tens->Dtype(); + if (!IsSubType(dtype, kNumber)) { + MS_LOG(EXCEPTION) << "Expect tensor type kNumber but got: " << dtype->ToString() << "."; + } + auto tensor_shape = tens->shape(); + auto abs_tensor = std::make_shared(dtype, tensor_shape); + abs_tensor->set_value(shared_from_base()); + return abs_tensor; +} + +std::string Tensor::GetShapeAndDataTypeInfo() const { + std::ostringstream buf; + buf << "Tensor \nshape:[" << shape() << "]" << this->Dtype()->ToString(); + return buf.str(); +} + +std::string Tensor::ToString() const { + const int small_tensor_size = 30; + std::ostringstream buf; + buf << "Tensor \nshape:[" << shape() << "]" << this->Dtype()->ToString(); + // only print small tensor + if (DataSize() < small_tensor_size) { + buf << "val:" << std::string(py::str(data())); + } + return buf.str(); +} + +std::string Tensor::ToStringRepr() const { + std::ostringstream buf; + auto type_ptr = this->Dtype(); + MS_EXCEPTION_IF_NULL(type_ptr); + buf << "Tensor shape:[" << shape() << "]" << type_ptr->ToString(); + buf << "\nval:" << std::string(py::str(data())); + return buf.str(); +} + +py::array Tensor::data_sync() { + if (device_address_ != nullptr) { + if (!device_address_->SyncDeviceToHost(this->shape(), static_cast(this->data().nbytes()), this->data_type(), + this->data_c(true))) { + MS_LOG(EXCEPTION) << "SyncDeviceToHost when asnumpy."; + } + } + return data_; +} + +REGISTER_PYBIND_DEFINE(Tensor, ([](const py::module* m) { + // dtype should define before Tensor, because Tensor init depend dtype + (void)py::class_>(*m, "Tensor") + .def(py::init(), py::arg("dtype"), py::arg("shape")) + .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) + .def_readonly(PYTHON_TENSOR_FLAG, &Tensor::parse_info_) + .def("asnumpy", &Tensor::data_sync, R"mydelimiter( + Convert tensor to numpy.ndarray. + + Returns: + numpy.ndarray. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> array = data.asnumpy() + >>> array + array([[1., 1., 1.], + [1., 1., 1.]]) + )mydelimiter") + .def("size", &Tensor::DataSize, R"mydelimiter( + Get tensor's data size. + + Returns: + int, the size of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.size() + 6 + )mydelimiter") + .def("dim", &Tensor::DataDim, R"mydelimiter( + Get tensor's data dimension. + + Returns: + int, the dimension of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.dim() + 2 + )mydelimiter") + .def("dtype", &Tensor::Dtype, R"mydelimiter( + Get the tensor's data type. + + Returns: + type, the data type of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 1), np.int32)) + >>> data.dtype() + Int32 + )mydelimiter") + .def("set_dtype", &Tensor::SetDtype, R"mydelimiter( + Set the tensor's data type. + + Arg: + dtype (:class:`mindspore.dtype`): The type of output tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) + >>> data.set_dtype(mindspore.int32) + mindspore.int32 + )mydelimiter") + .def("shape", &Tensor::GetPyTupleShape, R"mydelimiter( + Get the tensor's shape. + + Returns: + tuple[int], the shape of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((3, 3))) + >>> data.shape() + (3, 3) + )mydelimiter") + .def("__str__", &Tensor::ToString) + .def("__repr__", &Tensor::ToStringRepr) + .def("__eq__", &Tensor::ValueEqualPy) + .def(py::pickle( + [](const Tensor& t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(t.data()); + }, + [](const py::tuple& t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + Tensor tensor(t[0].cast()); + return tensor; + })); + (void)py::class_>(*m, "MetaTensor") + .def(py::init(), py::arg("dtype"), py::arg("shape")); + })); + +} // namespace tensor +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/meta_tensor.h b/mindspore/ccsrc/ir/meta_tensor.h new file mode 100644 index 00000000000..57c37851b67 --- /dev/null +++ b/mindspore/ccsrc/ir/meta_tensor.h @@ -0,0 +1,434 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_META_TENSOR_H_ +#define MINDSPORE_CCSRC_IR_META_TENSOR_H_ + +#include +#include +#include +#include +#include "device/device_address.h" + +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" + +#include "Eigen/Core" +#include "ir/base.h" +#include "ir/dtype.h" +#include "utils/log_adapter.h" +#include "utils/convert_utils.h" +#include "utils/hashing.h" + +namespace py = pybind11; + +using float16 = Eigen::half; + +namespace pybind11 { + +namespace detail { + +// Similar to enums in `pybind11/numpy.h`. Determined by doing: +// python3 -c 'import numpy as np; print(np.dtype(np.float16).num)' +constexpr int NPY_FLOAT16 = 23; + +template +struct npy_scalar_caster { + PYBIND11_TYPE_CASTER(T, _("PleaseOverride")); + using Array = array_t; + + bool load(handle src, bool convert) { + // Taken from Eigen casters. Permits either scalar dtype or scalar array. + handle type = dtype::of().attr("type"); + if (!convert && !isinstance(src) && !isinstance(src, type)) return false; + + Array tmp = Array::ensure(src); + if (tmp && tmp.size() == 1 && tmp.ndim() == 0) { + this->value = *tmp.data(); + return true; + } + + return false; + } + + static handle cast(T src, return_value_policy, handle) { + Array tmp({1}); + tmp.mutable_at(0) = src; + tmp.resize({}); + + // You could also just return the array if you want a scalar array. + object scalar = tmp[tuple()]; + return scalar.release(); + } +}; + +template <> +struct npy_format_descriptor { + static constexpr auto name = "float16"; + static pybind11::dtype dtype() { + handle ptr = npy_api::get().PyArray_DescrFromType_(NPY_FLOAT16); + return reinterpret_borrow(ptr); + } + virtual ~npy_format_descriptor() {} +}; + +template <> +struct type_caster : public npy_scalar_caster { + static constexpr auto name = "float16"; +}; + +} // namespace detail +} // namespace pybind11 + +using mindspore::device::DeviceAddress; +using DeviceAddressPtr = std::shared_ptr; +// brief mindspore namespace. +// +// mindspore namespace is the top level namespace of Mindsporeession project. +// Other namespace should be a sub namespace of mindspore namespace in the ME project. +namespace mindspore { + +// brief mindspore::tensor namespace +// +// A sub namespace in ME to support tensor related definition. +namespace tensor { + +// brief Device info of Tensor +// +// Includes the format and data type of a tensor. +struct DeviceInfo { + explicit DeviceInfo(std::string format = "DefaultFormat", TypePtr data_type = nullptr) + : format_(std::move(format)), data_type_(std::move(data_type)) {} + std::string format_ = "DefaultFormat"; + TypePtr data_type_ = nullptr; +}; + +// brief Metadata of Tensor +// +// Includes the metadata information of a tensor, such as data type, shape +// and so on. But it does not contain values of a tensor. +class MetaTensor : public Value { + public: + // Construction + MetaTensor(); + + // brief Constructs a meta tensor of a tensor having data_type data and shape. + // + // The constructed MetaTensor is not a Tensor, but it has the data type and shape + // information of a Tensor. The following codes will create a 2x3 float + // param data_type The data type of the tensor. + // param shape The shape of the tensor. + MetaTensor(const TypeId data_type, const std::vector& shape); + + MetaTensor(const TypePtr& type_ptr, const py::tuple& shape); + // brief Constructs a MetaTensor object from an existing MetaTensor instance. + // + // The constructed MetaTensor object will have the same data type and shape as the + // meta_tensor. + // + // param meta_tensor An existing MetaTensor object. + MetaTensor(const MetaTensor& meta_tensor); + ~MetaTensor() override = default; + MS_DECLARE_PARENT(MetaTensor, Value) + + // brief Overloads operator = for MetaTensor. + // + // The constructed MetaTensor object has the same type and shape with meta_tensor. + // + // param meta_tensor An exisiting MetaTensor object. + virtual MetaTensor& operator=(const MetaTensor& meta_tensor); + + // brief Compares two MetaTensor objects. + // + // The constructed MetaTensor object has the same type and shape with meta_tensor. + // + // param meta_tensor The MetaTensor object to be compared. + // return true: If having same type and shape, return true, or return false. + virtual bool operator==(const MetaTensor& meta_tensor) const; + + // brief Returns the data type of the tensor in its MetaTensor. + // + // All the types are defined in "ir/dtype.h". + TypePtr Dtype() const; + TypeId data_type() const { return data_type_; } + std::string ToString() const override; + std::string DumpText() const override; + // bried Sets the data type of a tensor in its MetaTensor. + // + // param data_type The data type of the tensor to be set. + virtual TypeId set_data_type(const TypeId data_type) { + data_type_ = data_type; + return data_type_; + } + virtual TypePtr SetDtype(const TypePtr type_ptr); + // brief Get tensor's shape. + // + // The shape of a tensor is stored in a vector. Each + // element of the vector represents the size of a dimension of the tensor. + // The order of each element in the vector is as same as the the dimension's + // order it represents. + // + // return A const vector which represents the shape of the tensor. + std::vector shape() const { return shape_; } + + // brief Sets the shape of a tensor. + // + // The shape of a tensor is stored in a vector. Each + // element of the vector represents the size of a dimension of the tensor. + // The order of each element in the vector is as same as the the dimension's + // order it represents. + // + // param shape The shape of the tensor. + // return The shape's size. + size_t set_shape(const std::vector& shape) { + this->shape_ = shape; + return shape_.size(); + } + + // Get tensor's device info. + DeviceInfo device_info() const { return device_info_; } + + // Set tensor's device info. + void set_device_info(const DeviceInfo& device_info) { device_info_ = device_info; } + + void SetDeviceInfo(const std::string& format, const TypePtr& data_type); + + // Get the size of a given dimension by its index number. + int DimensionSize(size_t index) const; + + // Get total number of elements in a tensor. + int ElementsNum() const; + + std::size_t hash() const override { + std::size_t hash_value = std::hash{}(SizeToInt(data_type_)); + hash_value = hash_combine(hash_value, std::hash{}(shape_.size())); + // hash all elements may costly, so only take at most 4 elements into account based on + // some experiments. + for (size_t i = 0; (i < shape_.size()) && (i < 4); ++i) { + hash_value = hash_combine(hash_value, (std::hash{}(shape_[i]))); + } + return hash_value; + } + bool operator==(const Value& other) const override { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } + } + + protected: + // brief Data type of the tensor. + // + // All support data type is in Number Types of [TypeId], + // including [kNumberTypeBool], [kNumberTypeInt], + // [kNumberTypeUInt32], [kNumberTypeFloat32] and [kNumberTypeFloat64]. + TypeId data_type_; + + // brief Shape of the tensor. + // + // A std::vector container is used to store the shape of a tensor. + // Each element of the vector represents the size of a dimension of the tensor. + // The order of each element in the vector is as same as the the dimension's + // order it represents. If the dimension size is not set, its value will be -1. + std::vector shape_; + + // brief Device info of Tensor + // + // Includes the format and data type of a tensor on device. + DeviceInfo device_info_; +}; + +// Tensor entity class +class Tensor : public MetaTensor { + public: + Tensor() = default; + abstract::AbstractBasePtr ToAbstract() override; + // brief Constructor for Python. + // + // param type_ptr [TypePty] Data type of the tensor. + // param py_shape [py::tuple] The shape represented by py::tuple of the tensor. + Tensor(const TypePtr& type_ptr, const py::tuple& shape); + + // brief Constructor for C++. + // + // param data_type [TypeId] Data type of the tensor. + // param shape The shape represented by std::vector of the tensor. + Tensor(TypeId data_type, const std::vector& shape); + + // brief Constructor for Python. + // + // param input [py::array] Data value of the tensor. + // param data_type [TypeId] Data type of the tensor. + explicit Tensor(const py::array& input, const TypePtr& data_type = nullptr); + + // brief Constructor + // + // param input [py::list] the data for tensor + // param data_type [TypeId] data type + explicit Tensor(const py::list& input, const TypePtr& data_type = nullptr); + + // brief Constructor + // + // param input [py::tuple] the data for tensor + // param data_type [TypeId] data type + explicit Tensor(const py::tuple& input, const TypePtr& data_type = nullptr); + + // brief Constructor + // + // param input [py::float_] the data for tensor + // param data_type [TypeId] data type + explicit Tensor(const py::float_& input, const TypePtr& data_type = nullptr); + + // brief Constructor + // + // param input [py::int_] the data for tensor + // param data_type [TypeId] data type + explicit Tensor(const py::int_& input, const TypePtr& data_type = nullptr); + + // brief Constructor + // + // param input [Tensor] the data for tensor + // param data_type [TypeId] data type + Tensor(const Tensor& tensor, const TypePtr& data_type = nullptr); + + ~Tensor() override = default; + + MS_DECLARE_PARENT(Tensor, MetaTensor); + + // brief Overloads operator = for Tensor. + // + // The constructed Tensor object has the same type and shape with tensor. + // + // param tensor An exisiting Tensor object. + Tensor& operator=(const Tensor& tensor); + + // brief Compares two Tensor objects. + // + // Compare two tensor objects to see if they have same data type, shape and + // data value. + // + // param tensor The Tensor object to be compared. + // return true: If having same type, shape and data, return true, or return false. + bool operator==(const Tensor& tensor) const; + + // It is different from 'operator==' which just compare shape/type/address, it do real value comparison. + bool ValueEqual(const Tensor& other) const; + + // It is different from 'operator==' which just compare shape/type/address, it do real value comparison. + bool ValueEqualPy(const py::object& other) const; + + bool operator==(const Value& other) const override { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } + } + + // brief Gets tensor's dimension + // + // return The number of dimensions of the tensor data. + int DataDim() const; + + // brief Getting tensor data size + // + // return The total number of elements of the tensor data. + int DataSize() const; + + // brief Get tensor's shape + // + // return [py::tuple] The tensor's shape + py::tuple GetPyTupleShape() const; + + // brief Tensor's data value. + // + // return [py::array] The tensor's data in py::array. + py::array data() const; + + // brief Get the data type fo the tensor for C++ + // + // return [int] The tensor's data type will be cast to int to return. + int data_type_c() const; + + // brief Get the tensor's shape for C++ + // + // return [std::vector] + std::vector shape_c(void) const; + + // brief Get Tensor data pointer for c++ type + // + // param writable true if writable, false if read only + // return The pointer to the object + void* data_c(bool writable = false); + + // brief Get data type from tensor data. + // + // param buf The buffer info of the py::array data. + // return The [TypeId] of the tensor data. + TypeId GetDataType(const py::buffer_info& buf) const; + + // bried Sets the data type of a tensor. + // + // param data_type The data type of the tensor to be set. + // + TypeId set_data_type(const TypeId data_type) override; + TypePtr SetDtype(const TypePtr type_ptr) override; + std::string GetShapeAndDataTypeInfo() const; + std::string ToString() const override; + std::string ToStringRepr() const; + py::array data_; // < Tensor's data value + const bool parse_info_ = true; + + private: + // brief init tensor + // + // param input [py::array] the data for tensor + // param data_type [TypeId] data type + // return true if succeed, false if failed. + void init(const py::array& input, const TypeId& data_type); + void init(const py::array& input, const TypePtr& type_ptr); + + // brief init tensor attribute + // + // param data_type [TypeId] Data type of the tensor. + // param shape [py::array] The shape of the tensor. + // return true if succeed, false if failed. + void init(TypeId data_type, const std::vector& shape, py::array* data); + + bool convert_data(const py::array& in, const TypeId in_data_type, py::array* out, const TypeId out_data_type); + + public: + bool is_dirty() const { return dirty_; } + void set_dirty(const bool dirty) { dirty_ = dirty; } + DeviceAddressPtr device_address() const { return device_address_; } + void set_device_address(const DeviceAddressPtr& device_address) { device_address_ = device_address; } + py::array data_sync(); + + private: + bool dirty_{true}; + DeviceAddressPtr device_address_{nullptr}; +}; + +using TensorPtr = std::shared_ptr; +using TensorPtrList = std::vector>; + +} // namespace tensor +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_META_TENSOR_H_ diff --git a/mindspore/ccsrc/ir/named.cc b/mindspore/ccsrc/ir/named.cc new file mode 100644 index 00000000000..3d12e8a4536 --- /dev/null +++ b/mindspore/ccsrc/ir/named.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/named.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { +bool Named::operator==(const Value& other) const { + if (other.isa()) { + auto other_named = static_cast(other); + return *this == other_named; + } else { + return false; + } +} + +abstract::AbstractBasePtr None::ToAbstract() { return std::make_shared(); } +const NamedPtr kNone = std::make_shared(); + +abstract::AbstractBasePtr NullObj::ToAbstract() { return std::make_shared(); } +const NamedPtr kNullObj = std::make_shared(); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/named.h b/mindspore/ccsrc/ir/named.h new file mode 100644 index 00000000000..0651307a91b --- /dev/null +++ b/mindspore/ccsrc/ir/named.h @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_NAMED_H_ +#define MINDSPORE_CCSRC_IR_NAMED_H_ + +#include +#include +#include + +#include "ir/base.h" +#include "ir/anf.h" + +namespace mindspore { +class Named : public Value { + public: + explicit Named(const std::string& name) : name_(name) { hash_id_ = std::hash{}(name); } + Named(const Named& other) : Value(other) { + this->name_ = other.name_; + hash_id_ = std::hash{}(other.name_); + } + ~Named() override = default; + MS_DECLARE_PARENT(Named, Value); + + const std::string& name() const { return name_; } + virtual bool operator==(const Named& other) const { return name_ == other.name(); } + bool operator==(const Value& other) const override; + Named& operator=(const Named& other) { + if (&other != this) { + this->type_ = other.type_; + this->name_ = other.name_; + hash_id_ = std::hash{}(name_); + } + return *this; + } + + std::size_t Hash() const { return hash_id_; } + std::size_t hash() const override { return hash_id_; } + + friend std::ostream& operator<<(std::ostream& os, const Named& nmd) { + os << nmd.name(); + return os; + } + + std::string ToString() const override { return name(); } + + private: + std::string name_; + std::size_t hash_id_; +}; + +using NamedPtr = std::shared_ptr; + +class None : public Named { + public: + None() : Named("None") {} + ~None() override = default; + MS_DECLARE_PARENT(None, Named); + abstract::AbstractBasePtr ToAbstract() override; +}; + +extern const NamedPtr kNone; + +class NullObj : public Named { + public: + NullObj() : Named("Null") {} + ~NullObj() override = default; + MS_DECLARE_PARENT(NullObj, Named); + abstract::AbstractBasePtr ToAbstract() override; +}; + +extern const NamedPtr kNullObj; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_IR_NAMED_H_ diff --git a/mindspore/ccsrc/ir/primitive.cc b/mindspore/ccsrc/ir/primitive.cc new file mode 100644 index 00000000000..d62553ef603 --- /dev/null +++ b/mindspore/ccsrc/ir/primitive.cc @@ -0,0 +1,181 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/primitive.h" +#include +#include +#include "ir/signature.h" +#include "operator/ops.h" +#include "./common.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/data_converter.h" +#include "pybind11/pytypes.h" +#include "utils/convert_utils.h" + +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +using mindspore::abstract::AbstractFunction; + +abstract::AbstractBasePtr Primitive::ToPrimAbstract(const AnfNodePtr& anf_node) { + auto prim_func = std::make_shared(shared_from_base(), anf_node); + return prim_func; +} + +static py::function GetBpropFunctionByObj(py::object obj) { + static const std::string get_bprop_fn = "get_bprop_fn"; + static const std::string ad_module = "mindspore.ops._grad"; + py::function fn = parse::python_adapter::GetPyFn(ad_module, get_bprop_fn)(obj); + return fn; +} + +py::function Primitive::GetBpropFunction() { + auto fn = GetBpropFunctionByObj(py::str(name())); + if (fn.is_none()) { + MS_LOG(WARNING) << "Can't find bprop function for " << name(); + } + return fn; +} + +py::function Primitive::GetComputeFunction() { + static const std::string module = "mindspore._extends.builtin_operations"; + py::module mod = py::module::import(common::SafeCStr(module)); + if (!py::hasattr(mod, common::SafeCStr(name()))) { + PyErr_SetString(PyExc_NotImplementedError, common::SafeCStr(name())); + // If raise AttributeError, user can't understand. This case need raise NotImplementedError. + throw py::error_already_set(); + } + py::object fn = mod.attr(common::SafeCStr(name())); + return fn; +} + +bool Primitive::operator==(const Value& other) const { + if (other.isa()) { + auto other_prim = static_cast(other); + return *this == other_prim; + } else { + return false; + } +} + +bool Primitive::operator==(const Primitive& other) const { + if (name() != other.name()) { + return false; + } + if (attrs_.size() != other.attrs_.size()) { + return false; + } + auto all = std::all_of(attrs_.begin(), attrs_.end(), [&other](const std::pair& item) -> bool { + if (item.second == nullptr) { + return false; + } + auto iter = other.attrs_.find(item.first); + if (iter == other.attrs_.end()) { + return false; + } + return *item.second == *iter->second; + }); + return all; +} + +void Primitive::set_signatures( + std::vector> signatures) { + signatures_.clear(); + for (auto& signature : signatures) { + std::string name; + SignatureEnumRW rw; + SignatureEnumKind kind; + py::object default_value; + SignatureEnumDType dtype; + std::tie(name, rw, kind, default_value, dtype) = signature; + signatures_.emplace_back(Signature(name, rw, kind, default_value, dtype)); + } +} + +py::function PrimitivePy::GetBpropFunction() { + static const char* const get_bprop_func_name = "get_bprop"; + if (py::hasattr(python_obj_, get_bprop_func_name)) { + py::function fn = python_obj_.attr(get_bprop_func_name)().cast(); + return fn; + } else { + auto fn = GetBpropFunctionByObj(python_obj_); + if (fn.is_none()) { + MS_LOG(WARNING) << "Can't find bprop function for " << name(); + } + return fn; + } +} + +py::function PrimitivePy::GetComputeFunction() { + static const char* const compute_func_name = "vm_impl"; + + if (py::hasattr(python_obj_, compute_func_name)) { + MS_LOG(INFO) << "" << name() << " compute_func_name"; + py::function fn = python_obj_.attr(compute_func_name).cast(); + return fn; + } + + static const std::string vm_module = "mindspore.ops.vm_impl_registry"; + static const std::string get_vm_impl_fn = "get_vm_impl_fn"; + MS_LOG(INFO) << "" << name() << ": get_vm_impl_fn"; + py::function get_fn = parse::python_adapter::GetPyFn(vm_module, get_vm_impl_fn); + py::function vm_fn = get_fn(python_obj_); + + if (py::isinstance(vm_fn)) { + MS_LOG(DEBUG) << "Cannot find " << python_obj_.attr("__class__").attr("__name__").cast(); + vm_fn = Primitive::GetComputeFunction(); + } + return vm_fn; +} + +void PrimitivePy::AddPyAttr(const py::str& name, const py::object& obj) { + std::string attr_name = name; + ValuePtr converted_ret = nullptr; + if (py::isinstance(obj)) { + MS_LOG(EXCEPTION) << "AddPyAttr failed, obj should not be py::module"; + } + bool converted = parse::ConvertData(obj, &converted_ret); + if (!converted) { + MS_LOG(EXCEPTION) << "Attribute convert error with type:" << std::string(py::str(obj)); + } + (void)this->AddAttr(attr_name, converted_ret); +} + +py::dict PrimitivePy::GetAttrDict() { + py::dict attr_dict; + for (auto& attr : attrs_) { + attr_dict[py::str(attr.first)] = ValuePtrToPyData(attr.second); + } + return attr_dict; +} + +REGISTER_PYBIND_DEFINE(Primitive_, ([](const py::module* m) { + (void)py::enum_(*m, "prim_type", py::arithmetic()) + .value("unknown", PrimType::kPrimTypeUnknown) + .value("builtin", PrimType::kPrimTypeBuiltIn) + .value("py_infer_shape", PrimType::kPrimTypePyInferShape) + .value("user_custom", PrimType::kPrimTypeUserCustom); + (void)py::class_>(*m, "Primitive_") + .def_readonly(PYTHON_PRIMITIVE_FLAG, &PrimitivePy::parse_info_) + .def(py::init()) + .def("add_attr", &PrimitivePy::AddPyAttr, "add primitive attr") + .def("get_attr_dict", &PrimitivePy::GetAttrDict, "get primitive attr") + .def("set_prim_type", &PrimitivePy::set_prim_type, "Set primitive type.") + .def("set_signatures", &PrimitivePy::set_signatures, "Set primitive inputs signature.") + .def("set_instance_name", &PrimitivePy::set_instance_name, "Set primitive instance name."); + })); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/primitive.h b/mindspore/ccsrc/ir/primitive.h new file mode 100644 index 00000000000..8a60412e441 --- /dev/null +++ b/mindspore/ccsrc/ir/primitive.h @@ -0,0 +1,160 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_PRIMITIVE_H_ +#define MINDSPORE_CCSRC_IR_PRIMITIVE_H_ + +#include +#include +#include +#include +#include +#include "pybind11/pybind11.h" + +#include "pipeline/static_analysis/abstract_value.h" +#include "utils/misc.h" +#include "utils/log_adapter.h" +#include "ir/signature.h" + +#include "parallel/ops_info/operator_info.h" + +namespace py = pybind11; + +namespace mindspore { +using abstract::AbstractBasePtr; +using abstract::AbstractBasePtrList; +// Supported meta type +enum PrimType { + kPrimTypeUnknown = 0, + kPrimTypeBegin = kTypeUnknown, + kPrimTypeBuiltIn, // Built-in primitive operator + kPrimTypePyInferShape, // Primitive operator defined by custom + kPrimTypePyInferTensor, // Primitive operator defined by custom + kPrimTypeUserCustom +}; + +class Primitive : public Named { + public: + explicit Primitive(const std::string& name, const PrimType prim_type = kPrimTypeBuiltIn) + : Named(name), signatures_(), prim_type_(prim_type) {} + + Primitive(const Primitive& prim) + : Named(prim), attrs_(prim.attrs_), signatures_(prim.signatures_), prim_type_(prim.prim_type_) {} + + MS_DECLARE_PARENT(Primitive, Named); + + abstract::AbstractBasePtr ToPrimAbstract(const AnfNodePtr& anf_node); + std::string ToString() const override { return name(); } + virtual py::function GetBpropFunction(); + virtual py::function GetComputeFunction(); + Primitive& AddAttr(const std::string& name, const ValuePtr& attr) { + attrs_[name] = attr; + return *this; + } + + Primitive& SetAttrs(const std::unordered_map& attrs) { + for (auto& attr : attrs) { + attrs_[attr.first] = attr.second; + } + return *this; + } + + void set_signatures( + std::vector> + signatures); + + const std::vector& signatures() const { return signatures_; } + + void set_attr(const std::string& attrName, const ValuePtr& attr) { attrs_[attrName] = attr; } + void EraseAttr(const std::string& attrName) { (void)attrs_.erase(attrName); } + + ValuePtr GetAttr(const std::string& attrName) const { + auto iter = attrs_.find(attrName); + return iter == attrs_.cend() ? nullptr : iter->second; + } + + const std::unordered_map& attrs() const { return attrs_; } + + // if Primitive has any attribute, for Primitives like scalar_add, return, etc, don't have any attribute. + bool HasAttr() const { return !attrs_.empty(); } + bool HasAttr(const std::string& attrName) const { + auto iter = attrs_.find(attrName); + return !(iter == attrs_.cend()); + } + void set_prim_type(const PrimType t) { prim_type_ = t; } + void set_instance_name(const std::string s) { instance_name_ = s; } + bool HasPyEvaluator() const { return prim_type_ == kPrimTypePyInferShape || prim_type_ == kPrimTypeUserCustom; } + bool HasPyInferTensor() const { return prim_type_ == kPrimTypePyInferTensor; } + bool IsCustomPrim() const { return prim_type_ == kPrimTypeUserCustom; } + + PrimType prim_type() const { return prim_type_; } + std::string instance_name() const { return instance_name_; } + bool operator==(const Value& other) const override; + bool operator==(const Primitive& other) const; + ~Primitive() override = default; + + protected: + std::unordered_map attrs_; + + private: + std::vector signatures_; + std::string instance_name_; + PrimType prim_type_; +}; + +class PrimitivePy : public Primitive { + public: + PrimitivePy(const py::str& name, const py::object& python_obj) : Primitive(name), python_obj_(python_obj) {} + ~PrimitivePy() override = default; + MS_DECLARE_PARENT(PrimitivePy, Primitive); + py::function GetBpropFunction() override; + py::function GetComputeFunction() override; + + void AddPyAttr(const py::str& name, const py::object& obj); + + py::dict GetAttrDict(); + + const bool parse_info_ = true; + const py::object& GetPyObj() const { return python_obj_; } + bool is_tuple_input_ = false; + + private: + py::object python_obj_; +}; + +using PrimitivePyPtr = std::shared_ptr; + +inline std::ostream& operator<<(std::ostream& os, const PrimitivePtr& p) { + os << *p; + return os; +} + +struct PrimitiveEqual { + bool operator()(PrimitivePtr const& t1, PrimitivePtr const& t2) const { + MS_EXCEPTION_IF_NULL(t1); + MS_EXCEPTION_IF_NULL(t2); + return t1->name() == t2->name(); + } +}; + +struct PrimitiveHasher { + std::size_t operator()(PrimitivePtr const& prim) const { + std::size_t hash = std::hash()(prim->name()); + return hash; + } +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_IR_PRIMITIVE_H_ diff --git a/mindspore/ccsrc/ir/scalar.h b/mindspore/ccsrc/ir/scalar.h new file mode 100644 index 00000000000..3e0a827b072 --- /dev/null +++ b/mindspore/ccsrc/ir/scalar.h @@ -0,0 +1,359 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_SCALAR_H_ +#define MINDSPORE_CCSRC_IR_SCALAR_H_ + +namespace mindspore { +/* namespace to support inference engine */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/base.h" +#include "ir/dtype.h" + +class Scalar : public Value { + public: + Scalar() = default; + explicit Scalar(const TypePtr t) : Value(t) {} + ~Scalar() override = default; + MS_DECLARE_PARENT(Scalar, Value) + virtual bool IsZero() = 0; + virtual bool IsOne() = 0; + abstract::AbstractBasePtr ToAbstract() override; + + protected: + std::size_t hash_ = 0; +}; +using ScalarPtr = std::shared_ptr; + +class BoolImm : public Scalar { + public: + explicit BoolImm(bool b) : Scalar(kBool), v_(b) { hash_ = std::hash{}(v_); } + ~BoolImm() override = default; + MS_DECLARE_PARENT(BoolImm, Scalar) + std::size_t hash() const override { return hash_; } + bool value() const { return v_; } + bool IsZero() override { return v_ == false; } + bool IsOne() override { return v_ == true; } + bool operator==(const Value& other) const override; + bool operator==(const BoolImm& other) const; + std::string ToString() const override { + if (v_) { + return "true"; + } else { + return "false"; + } + } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "Bool(" << v_ << ")"; + return oss.str(); + } + + private: + bool v_; +}; +using BoolImmPtr = std::shared_ptr; +IMM_TRAITS(BoolImmPtr, bool) + +class IntergerImm : public Scalar { + public: + IntergerImm() = default; + explicit IntergerImm(const TypePtr& t) : Scalar(t) {} + ~IntergerImm() override = default; + MS_DECLARE_PARENT(IntergerImm, Scalar) +}; + +class Int8Imm : public IntergerImm { + public: + Int8Imm() : IntergerImm(kInt8), v_(0) {} + explicit Int8Imm(int8_t v) : IntergerImm(kInt8), v_(v) { hash_ = std::hash{}(v_); } + ~Int8Imm() override = default; + MS_DECLARE_PARENT(Int8Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + int8_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const Int8Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "I8(" << v_ << ")"; + return oss.str(); + } + + private: + int8_t v_; +}; +using Int8ImmPtr = std::shared_ptr; +IMM_TRAITS(Int8ImmPtr, int8_t) + +class Int16Imm : public IntergerImm { + public: + Int16Imm() : IntergerImm(kInt16), v_(0) {} + explicit Int16Imm(int16_t v) : IntergerImm(kInt16), v_(v) { hash_ = std::hash{}(v_); } + ~Int16Imm() override = default; + MS_DECLARE_PARENT(Int16Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + int16_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const Int16Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "I16(" << v_ << ")"; + return oss.str(); + } + + private: + int16_t v_; +}; +using Int16ImmPtr = std::shared_ptr; +IMM_TRAITS(Int16ImmPtr, int16_t) + +class Int32Imm : public IntergerImm { + public: + Int32Imm() : IntergerImm(kInt32), v_(0) {} + explicit Int32Imm(int v) : IntergerImm(kInt32), v_(v) { hash_ = std::hash{}(v_); } + ~Int32Imm() override = default; + MS_DECLARE_PARENT(Int32Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + int32_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const Int32Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "I32(" << v_ << ")"; + return oss.str(); + } + + private: + int32_t v_; +}; +using Int32ImmPtr = std::shared_ptr; +IMM_TRAITS(Int32ImmPtr, int32_t) + +class Int64Imm : public IntergerImm { + public: + Int64Imm() : IntergerImm(kInt64), v_(0) {} + explicit Int64Imm(int64_t v) : IntergerImm(kInt64), v_(v) { hash_ = std::hash{}(v_); } + ~Int64Imm() override = default; + MS_DECLARE_PARENT(Int64Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + int64_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const Int64Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "I64(" << v_ << ")"; + return oss.str(); + } + + private: + int64_t v_; +}; +using Int64ImmPtr = std::shared_ptr; +IMM_TRAITS(Int64ImmPtr, int64_t) + +class UInt8Imm : public IntergerImm { + public: + UInt8Imm() : IntergerImm(kUInt8), v_(0) {} + explicit UInt8Imm(uint8_t v) : IntergerImm(kUInt8), v_(v) { hash_ = std::hash{}(v_); } + ~UInt8Imm() override = default; + MS_DECLARE_PARENT(UInt8Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + uint8_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const UInt8Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "U8(" << v_ << ")"; + return oss.str(); + } + + private: + uint8_t v_; +}; +using UInt8ImmPtr = std::shared_ptr; +IMM_TRAITS(UInt8ImmPtr, uint8_t); + +class UInt16Imm : public IntergerImm { + public: + UInt16Imm() : IntergerImm(kUInt16), v_(0) {} + explicit UInt16Imm(uint16_t v) : IntergerImm(kUInt16), v_(v) { hash_ = std::hash{}(v_); } + ~UInt16Imm() override = default; + MS_DECLARE_PARENT(UInt16Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + uint16_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const UInt16Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "U16(" << v_ << ")"; + return oss.str(); + } + + private: + uint16_t v_; +}; +using UInt16ImmPtr = std::shared_ptr; +IMM_TRAITS(UInt16ImmPtr, uint16_t); + +class UInt32Imm : public IntergerImm { + public: + UInt32Imm() : IntergerImm(kUInt32), v_(0) {} + explicit UInt32Imm(uint32_t v) : IntergerImm(kUInt32), v_(v) { hash_ = std::hash{}(v_); } + ~UInt32Imm() override = default; + MS_DECLARE_PARENT(UInt32Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + uint32_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const UInt32Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "U32(" << v_ << ")"; + return oss.str(); + } + + private: + uint32_t v_; +}; +using UInt32ImmPtr = std::shared_ptr; +IMM_TRAITS(UInt32ImmPtr, uint32_t); + +class UInt64Imm : public IntergerImm { + public: + UInt64Imm() : IntergerImm(kUInt64), v_(0) {} + explicit UInt64Imm(uint64_t v) : IntergerImm(kUInt64), v_(v) { hash_ = std::hash{}(v); } + ~UInt64Imm() override = default; + MS_DECLARE_PARENT(UInt64Imm, IntergerImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return v_ == 0; } + bool IsOne() override { return v_ == 1; } + uint64_t value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const UInt64Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "U64(" << v_ << ")"; + return oss.str(); + } + + private: + uint64_t v_; +}; +using UInt64ImmPtr = std::shared_ptr; +IMM_TRAITS(UInt64ImmPtr, uint64_t); + +class FloatImm : public Scalar { + public: + FloatImm() = default; + explicit FloatImm(const TypePtr& t) : Scalar(t) {} + ~FloatImm() override = default; + MS_DECLARE_PARENT(FloatImm, Scalar) +}; +using FloatImmPtr = std::shared_ptr; + +class FP32Imm : public FloatImm { + public: + FP32Imm() : FloatImm(kFloat32), v_(0.0) {} + explicit FP32Imm(float v) : FloatImm(kFloat32), v_(v) { hash_ = std::hash{}(v_); } + ~FP32Imm() override = default; + MS_DECLARE_PARENT(FP32Imm, FloatImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return fabs(v_) <= FLT_EPSILON; } + bool IsOne() override { return fabs(v_ - 1.0) <= FLT_EPSILON; } + float value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const FP32Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "F32(" << v_ << ")"; + return oss.str(); + } + + private: + float v_; +}; +using FP32ImmPtr = std::shared_ptr; +IMM_TRAITS(FP32ImmPtr, float) + +class FP64Imm : public FloatImm { + public: + FP64Imm() : FloatImm(kFloat64), v_(0.0) {} + explicit FP64Imm(double v) : FloatImm(kFloat64), v_(v) { hash_ = std::hash{}(v_); } + ~FP64Imm() override = default; + MS_DECLARE_PARENT(FP64Imm, FloatImm) + std::size_t hash() const override { return hash_; } + bool IsZero() override { return fabs(v_) <= DBL_EPSILON; } + bool IsOne() override { return fabs(v_ - 1.0) <= DBL_EPSILON; } + double value() const { return v_; } + bool operator==(const Value& other) const override; + bool operator==(const FP64Imm& other) const; + std::string ToString() const override { return std::to_string(v_); } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "F64(" << v_ << ")"; + return oss.str(); + } + + private: + double v_; +}; +using FP64ImmPtr = std::shared_ptr; +IMM_TRAITS(FP64ImmPtr, double) + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_SCALAR_H_ diff --git a/mindspore/ccsrc/ir/scope.cc b/mindspore/ccsrc/ir/scope.cc new file mode 100644 index 00000000000..a5e7837179c --- /dev/null +++ b/mindspore/ccsrc/ir/scope.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/scope.h" +namespace mindspore { +const ScopePtr kDefaultScope = std::make_shared("Default"); + +void ScopeManager::EnterScope(const ScopePtr &scope) { + if (scope != kDefaultScope) { + scope_stack_.push(scope); + } +} + +void ScopeManager::LeaveScope(const ScopePtr &scope) noexcept { + if (scope != kDefaultScope) { + scope_stack_.pop(); + } +} +ScopePtr ScopeManager::GetCurrentScope() { + // if the scope stack is empty, return the default scope + if (scope_stack_.empty()) { + return kDefaultScope; + } + return scope_stack_.top(); +} +void ScopeManager::ClearScope() { + while (!scope_stack_.empty()) { + scope_stack_.pop(); + } +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/scope.h b/mindspore/ccsrc/ir/scope.h new file mode 100644 index 00000000000..07318a817e4 --- /dev/null +++ b/mindspore/ccsrc/ir/scope.h @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IR_SCOPE_H_ +#define IR_SCOPE_H_ +#include +#include +#include +namespace mindspore { +class Scope; +using ScopePtr = std::shared_ptr; +extern const ScopePtr kDefaultScope; + +class Scope { + public: + // using the default scope + explicit Scope(const std::string &name) : name_(name) {} + ~Scope() = default; + std::string name() { return name_; } + std::string name_; +}; + +class ScopeManager { + public: + static ScopeManager &GetInstance() noexcept { + static ScopeManager instance; + return instance; + } + ScopeManager(const ScopeManager &) = delete; + ScopeManager &operator=(const ScopeManager &) = delete; + ~ScopeManager() = default; + void EnterScope(const ScopePtr &scope); + void LeaveScope(const ScopePtr &scope) noexcept; + ScopePtr GetCurrentScope(); + void ClearScope(); + + private: + ScopeManager() = default; + std::stack scope_stack_; +}; +// ScopeGuard is a class that help generate the anf node of specified scope +// in the current c++ action scope. +class ScopeGuard { + public: + explicit ScopeGuard(const ScopePtr &scope) { + scope_ = scope; + ScopeManager::GetInstance().EnterScope(scope); + } + ~ScopeGuard() { ScopeManager::GetInstance().LeaveScope(scope_); } + + private: + ScopePtr scope_; +}; +} // namespace mindspore +#endif // IR_SCOPE_H_ diff --git a/mindspore/ccsrc/ir/signature.cc b/mindspore/ccsrc/ir/signature.cc new file mode 100644 index 00000000000..b7eec921d45 --- /dev/null +++ b/mindspore/ccsrc/ir/signature.cc @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/signature.h" + +#include "pybind11/operators.h" +#include "pybind_api/api_register.h" +#include "pipeline/parse/data_converter.h" + +namespace mindspore { +Signature::Signature(const std::string& arg_name, const SignatureEnumRW& rw_tag, const SignatureEnumKind& arg_kind, + const py::object& arg_default, const SignatureEnumDType& arg_dtype) + : name(arg_name), rw(rw_tag), kind(arg_kind), dtype(arg_dtype) { + if (py::isinstance(arg_default) && + py::cast(arg_default) == SignatureEnumKind::kKindEmptyDefaultValue) { + default_value = nullptr; + } else { + default_value = parse::data_converter::PyDataToValue(arg_default); + } +} + +Signature::Signature(const std::string& arg_name, const SignatureEnumRW& rw_tag, const SignatureEnumKind& arg_kind) + : name(arg_name), + rw(rw_tag), + kind(arg_kind), + default_value(nullptr), + dtype(SignatureEnumDType::kDTypeEmptyDefaultValue) {} + +REGISTER_PYBIND_DEFINE(SignatureEnumRW, ([](const py::module* m) { + (void)py::enum_(*m, "signature_rw", py::arithmetic()) + .value("RW_READ", SignatureEnumRW::kRWRead) + .value("RW_WRITE", SignatureEnumRW::kRWWrite) + .value("RW_REF", SignatureEnumRW::kRWRef) + .value("RW_EMPTY_DEFAULT_VALUE", SignatureEnumRW::kRWEmptyDefaultValue); + (void)py::enum_(*m, "signature_kind", py::arithmetic()) + .value("KIND_POSITIONAL_KEYWORD", SignatureEnumKind::kKindPositionalKeyword) + .value("KIND_VAR_POSITIONAL", SignatureEnumKind::kKindVarPositional) + .value("KIND_KEYWORD_ONLY", SignatureEnumKind::kKindKeywordOnly) + .value("KIND_VAR_KEYWARD", SignatureEnumKind::kKindVarKeyword) + .value("KIND_EMPTY_DEFAULT_VALUE", SignatureEnumKind::kKindEmptyDefaultValue); + (void)py::enum_(*m, "signature_dtype", py::arithmetic()) + .value("T", SignatureEnumDType::kDType) + .value("T1", SignatureEnumDType::kDType1) + .value("T2", SignatureEnumDType::kDType2) + .value("T3", SignatureEnumDType::kDType3) + .value("T4", SignatureEnumDType::kDType4) + .value("T5", SignatureEnumDType::kDType5) + .value("T6", SignatureEnumDType::kDType6) + .value("T7", SignatureEnumDType::kDType7) + .value("T8", SignatureEnumDType::kDType8) + .value("T9", SignatureEnumDType::kDType9) + .value("T_EMPTY_DEFAULT_VALUE", SignatureEnumDType::kDTypeEmptyDefaultValue); + })); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/signature.h b/mindspore/ccsrc/ir/signature.h new file mode 100644 index 00000000000..8e7409ab264 --- /dev/null +++ b/mindspore/ccsrc/ir/signature.h @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_SIGNATURE_H_ +#define MINDSPORE_CCSRC_IR_SIGNATURE_H_ +#include +#include + +#include "pybind11/operators.h" +#include "ir/value.h" + +namespace py = pybind11; + +namespace mindspore { +// Input signature, support type +enum SignatureEnumRW { + // describe the arguments action on read and write + kRWRead = 0, // use the value of the input + kRWWrite, // use the key of the input + kRWRef, // use the ref of the input + kRWEmptyDefaultValue, + kRWDefault = kRWRead +}; +enum SignatureEnumKind { + kKindPositionalKeyword = 0, // use value of the input start from this arg + kKindVarPositional, // use key of the input start from this arg + kKindKeywordOnly, + kKindVarKeyword, // use ref of the input start from this arg + kKindEmptyDefaultValue, + kKindDefault = kKindPositionalKeyword +}; +enum SignatureEnumDType { + kDType = 0, + kDType1, + kDType2, + kDType3, + kDType4, + kDType5, + kDType6, + kDType7, + kDType8, + kDType9, + kDTypeEmptyDefaultValue +}; +struct Signature { + std::string name; + SignatureEnumRW rw; + SignatureEnumKind kind; + ValuePtr default_value; // nullptr for no default value + SignatureEnumDType dtype; + Signature(const std::string& arg_name, const SignatureEnumRW& rw_tag, const SignatureEnumKind& arg_kind, + const py::object& arg_default, const SignatureEnumDType& arg_dtype); + Signature(const std::string& arg_name, const SignatureEnumRW& rw_tag, const SignatureEnumKind& arg_kind); +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_SIGNATURE_H_ diff --git a/mindspore/ccsrc/ir/value.cc b/mindspore/ccsrc/ir/value.cc new file mode 100644 index 00000000000..f9e8abaee9d --- /dev/null +++ b/mindspore/ccsrc/ir/value.cc @@ -0,0 +1,370 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/value.h" +#include +#include +#include +#include + +#include "pybind_api/api_register.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { +const ValuePtr ValueSequeue::operator[](const std::size_t& dim) const { + if (dim >= size()) { + MS_LOG(EXCEPTION) << "List index [" << dim << "] is out of range [" << size() << "]."; + } + return elements_[dim]; +} + +bool ValueSequeue::erase(size_t idx) { + if (idx < size()) { + (void)elements_.erase(elements_.begin() + SizeToInt(idx)); + return true; + } else { + return false; + } +} + +bool BoolImm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool BoolImm::operator==(const BoolImm& other) const { return v_ == other.v_; } + +bool Int8Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool Int8Imm::operator==(const Int8Imm& other) const { return v_ == other.v_; } +bool Int16Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool Int16Imm::operator==(const Int16Imm& other) const { return v_ == other.v_; } +bool Int32Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool Int32Imm::operator==(const Int32Imm& other) const { return v_ == other.v_; } +bool Int64Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool Int64Imm::operator==(const Int64Imm& other) const { return v_ == other.v_; } +bool UInt8Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool UInt8Imm::operator==(const UInt8Imm& other) const { return v_ == other.v_; } +bool UInt16Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool UInt16Imm::operator==(const UInt16Imm& other) const { return v_ == other.v_; } +bool UInt32Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool UInt32Imm::operator==(const UInt32Imm& other) const { return v_ == other.v_; } +bool UInt64Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool UInt64Imm::operator==(const UInt64Imm& other) const { return v_ == other.v_; } +bool FP32Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool FP32Imm::operator==(const FP32Imm& other) const { return fabs(v_ - other.v_) < FLT_EPSILON; } +bool FP64Imm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool ValueSequeue::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool ValueSequeue::operator==(const ValueSequeue& other) const { + if (other.elements_.size() != elements_.size()) { + return false; + } + return std::equal(elements_.begin(), elements_.end(), other.elements_.begin(), + [](const ValuePtr& lhs, const ValuePtr& rhs) { return *lhs == *rhs; }); +} + +std::string ValueSequeue::ToString() const { + std::ostringstream buffer; + bool begin = true; + for (auto& attr : elements_) { + if (!begin) { + buffer << ", "; + } else { + begin = false; + } + MS_EXCEPTION_IF_NULL(attr); + buffer << attr->ToString(); + } + return buffer.str(); +} + +std::string ValueSequeue::DumpText() const { + std::ostringstream oss; + for (size_t i = 0; i < elements_.size(); ++i) { + MS_EXCEPTION_IF_NULL(elements_[i]); + oss << (i > 0 ? ", " : "") << elements_[i]->DumpText(); + } + return oss.str(); +} + +bool FP64Imm::operator==(const FP64Imm& other) const { return fabs(v_ - other.v_) < DBL_EPSILON; } +bool StringImm::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool StringImm::operator==(const StringImm& other) const { return str_ == other.str_; } + +bool RefKey::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} +bool RefKey::operator==(const RefKey& other) const { return tag_ == other.tag_; } + +bool AnyValue::operator==(const Value& other) const { + if (other.isa()) { + return true; + } else { + return false; + } +} +const ValuePtr kAnyValue = std::make_shared(); +using ContextPtr = abstract::AnalysisContextPtr; + +abstract::AbstractBasePtr Scalar::ToAbstract() { + return std::make_shared(shared_from_base()); +} + +abstract::AbstractBasePtr StringImm::ToAbstract() { + return std::make_shared(shared_from_base(), std::make_shared()); +} + +abstract::AbstractBasePtr RefKey::ToAbstract() { + auto refkey = std::make_shared(); + refkey->set_value(shared_from_base()); + return refkey; +} + +abstract::AbstractBasePtr AnyValue::ToAbstract() { return std::make_shared(); } + +abstract::AbstractBasePtr ValueTuple::ToAbstract() { + abstract::AbstractBasePtrList a_list; + (void)std::transform(elements_.begin(), elements_.end(), std::back_inserter(a_list), [](const ValuePtr& ele) { + MS_EXCEPTION_IF_NULL(ele); + return ele->ToAbstract(); + }); + return std::make_shared(a_list); +} + +abstract::AbstractBasePtr ValueList::ToAbstract() { + abstract::AbstractBasePtrList a_list; + (void)std::transform(elements_.begin(), elements_.end(), std::back_inserter(a_list), [](const ValuePtr& ele) { + MS_EXCEPTION_IF_NULL(ele); + return ele->ToAbstract(); + }); + return std::make_shared(a_list); +} + +std::size_t ValueSlice::hash() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + return hash_combine({tid(), start_->hash(), stop_->hash(), step_->hash()}); +} + +bool ValueSlice::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} + +bool ValueSlice::operator==(const ValueSlice& other) const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + return (*start_ == *other.start_ && *stop_ == *other.stop_ && *step_ == *other.step_); +} + +std::string ValueSlice::ToString() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + std::ostringstream buffer; + buffer << "Slice["; + buffer << start_->ToString() << " : "; + buffer << stop_->ToString() << " : "; + buffer << step_->ToString(); + buffer << "]"; + return buffer.str(); +} + +abstract::AbstractBasePtr ValueSlice::ToAbstract() { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + abstract::AbstractBasePtr start = start_->ToAbstract(); + abstract::AbstractBasePtr end = stop_->ToAbstract(); + abstract::AbstractBasePtr step = step_->ToAbstract(); + return std::make_shared(start, end, step); +} + +std::size_t KeywordArg::hash() const { + MS_EXCEPTION_IF_NULL(value_); + return hash_combine({tid(), std::hash{}(key_), value_->hash()}); +} + +bool KeywordArg::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} + +bool KeywordArg::operator==(const KeywordArg& other) const { return (other.key_ == key_ && *other.value_ == *value_); } + +std::string KeywordArg::ToString() const { + std::ostringstream buffer; + buffer << "KeywordArg["; + buffer << "key : " << key_; + MS_EXCEPTION_IF_NULL(value_); + buffer << "value : " << value_->ToString(); + buffer << "]"; + return buffer.str(); +} + +abstract::AbstractBasePtr KeywordArg::ToAbstract() { + MS_EXCEPTION_IF_NULL(value_); + abstract::AbstractBasePtr argument = value_->ToAbstract(); + return std::make_shared(key_, argument); +} + +const ValuePtr ValueDictionary::operator[](const std::string& key) const { + auto it = std::find_if(key_values_.begin(), key_values_.end(), + [key](const std::pair& item) { return item.first == key; }); + if (it == key_values_.end()) { + MS_LOG(EXCEPTION) << "The key " << key << " is not in the map"; + } + return it->second; +} + +bool ValueDictionary::operator==(const Value& other) const { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } +} + +bool ValueDictionary::operator==(const ValueDictionary& other) const { + if (key_values_.size() != other.key_values_.size()) { + return false; + } + for (size_t index = 0; index < key_values_.size(); index++) { + if (key_values_[index].first != other.key_values_[index].first) { + return false; + } + if (!(*key_values_[index].second == *other.key_values_[index].second)) { + return false; + } + } + return true; +} + +abstract::AbstractBasePtr ValueDictionary::ToAbstract() { + std::vector> kv; + (void)std::transform( + key_values_.begin(), key_values_.end(), std::back_inserter(kv), + [](const std::pair& item) { return std::make_pair(item.first, item.second->ToAbstract()); }); + return std::make_shared(kv); +} + +REGISTER_PYBIND_DEFINE( + RefKey, ([](const py::module* m) { + (void)py::class_>(*m, "RefKey").def(py::init(), py::arg("tag")); + })); +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/value.h b/mindspore/ccsrc/ir/value.h new file mode 100644 index 00000000000..85f514b57b3 --- /dev/null +++ b/mindspore/ccsrc/ir/value.h @@ -0,0 +1,302 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_VALUE_H_ +#define MINDSPORE_CCSRC_IR_VALUE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/base.h" +#include "ir/anf.h" +#include "ir/dtype.h" +#include "ir/scalar.h" +#include "utils/hashing.h" +#include "common/utils.h" + +namespace mindspore { +class ValueSequeue : public Value { + public: + explicit ValueSequeue(const ValuePtrList& elements) : elements_(elements) { + TypePtrList t_list; + (void)std::transform(elements.begin(), elements.end(), std::back_inserter(t_list), [](const ValuePtr& ele) { + MS_EXCEPTION_IF_NULL(ele); + return ele->type(); + }); + TypePtr t = std::make_shared(t_list); + type_ = t; + } + ValueSequeue(const std::initializer_list& elements) : elements_(elements.begin(), elements.end()) { + TypePtrList t_list; + (void)std::transform(elements_.begin(), elements_.end(), std::back_inserter(t_list), + [](const ValuePtr& ele) { return ele->type(); }); + TypePtr t = std::make_shared(t_list); + type_ = t; + } + ~ValueSequeue() override = default; + MS_DECLARE_PARENT(ValueSequeue, Value) + std::size_t hash() const override { return hash_combine(tid(), std::hash{}(elements_.size())); } + std::size_t size() const { return elements_.size(); } + bool erase(size_t idx); + const ValuePtr operator[](const std::size_t& dim) const; + const ValuePtrList& value() const { return elements_; } + bool operator==(const Value& other) const override; + bool operator==(const ValueSequeue& other) const; + std::string ToString() const override; + std::string DumpText() const override; + + protected: + ValuePtrList elements_; +}; +using ValueSequeuePtr = std::shared_ptr; + +class ValueTuple : public ValueSequeue { + public: + explicit ValueTuple(const std::vector& elements) : ValueSequeue(elements) {} + ValueTuple(const std::initializer_list& elements) : ValueSequeue(elements) {} + ~ValueTuple() override = default; + MS_DECLARE_PARENT(ValueTuple, ValueSequeue) + abstract::AbstractBasePtr ToAbstract() override; + + std::string DumpText() const override { return "(" + ValueSequeue::DumpText() + ")"; } + std::string ToString() const override { return "(" + ValueSequeue::ToString() + ")"; } +}; +using ValueTuplePtr = std::shared_ptr; + +class ValueList : public ValueSequeue { + public: + explicit ValueList(const std::vector& elements) : ValueSequeue(elements) {} + ValueList(const std::initializer_list& elements) : ValueSequeue(elements) {} + ~ValueList() override = default; + MS_DECLARE_PARENT(ValueList, ValueSequeue) + abstract::AbstractBasePtr ToAbstract() override; + + std::string DumpText() const override { return "[" + ValueSequeue::DumpText() + "]"; } + std::string ToString() const override { return "[" + ValueSequeue::ToString() + "]"; } +}; +using ValueListPtr = std::shared_ptr; + +inline ValuePtr MakeValue(const std::vector& v) { return std::make_shared(v); } +inline ValuePtr MakeValue(std::initializer_list v) { return std::make_shared(v); } + +template +struct is_vector : public std::false_type {}; +template +struct is_vector> : public std::true_type {}; + +template ::value, typename T::value_type>::type> +ValuePtr MakeValue(const T& vec) { + std::vector list; + (void)std::transform(vec.begin(), vec.end(), std::back_inserter(list), [](U ele) { return MakeValue(ele); }); + return std::make_shared(list); +} + +class ValueSlice : public Value { + public: + ValueSlice(const ValuePtr& start, const ValuePtr& stop, const ValuePtr& step) + : start_(start), stop_(stop), step_(step) {} + ~ValueSlice() override = default; + MS_DECLARE_PARENT(ValueSlice, Value) + std::size_t hash() const override; + bool operator==(const Value& other) const override; + bool operator==(const ValueSlice& other) const; + + std::string ToString() const override; + + abstract::AbstractBasePtr ToAbstract() override; + std::string DumpText() const override { return ToString(); } + + private: + ValuePtr start_; + ValuePtr stop_; + ValuePtr step_; +}; +using ValueSlicePtr = std::shared_ptr; + +class KeywordArg : public Value { + public: + KeywordArg(const std::string& key, const ValuePtr& value) : key_(key), value_(value) {} + ~KeywordArg() override = default; + MS_DECLARE_PARENT(KeywordArg, Value) + std::size_t hash() const override; + ValuePtr get_value() const { return value_; } + bool operator==(const Value& other) const override; + bool operator==(const KeywordArg& other) const; + + std::string ToString() const override; + + abstract::AbstractBasePtr ToAbstract() override; + std::string DumpText() const override { return ToString(); } + + private: + std::string key_; + ValuePtr value_; +}; +using KeywordArgPtr = std::shared_ptr; + +class ValueDictionary : public Value { + public: + explicit ValueDictionary(const std::vector>& key_values) : key_values_(key_values) {} + ~ValueDictionary() override = default; + MS_DECLARE_PARENT(ValueDictionary, Value) + std::size_t hash() const override { return hash_combine(tid(), std::hash{}(key_values_.size())); } + std::size_t size() const { return key_values_.size(); } + const ValuePtr operator[](const std::string& key) const; + const std::vector>& value() const { return key_values_; } + bool operator==(const Value& other) const override; + bool operator==(const ValueDictionary& other) const; + + std::string ToString() const override { + std::ostringstream buffer; + std::vector keys; + std::vector values; + for (const auto& kv : key_values_) { + keys.push_back(kv.first); + values.push_back(kv.second); + } + buffer << "(Dict: " + << " keys:("; + for (const auto& key : keys) { + buffer << key << ", "; + } + buffer << ") values:("; + for (const auto& value : values) { + MS_EXCEPTION_IF_NULL(value); + buffer << value->DumpText() << ", "; + } + buffer << ")"; + return buffer.str(); + } + abstract::AbstractBasePtr ToAbstract() override; + std::string DumpText() const override { return ToString(); } + + private: + std::vector> key_values_; +}; +using ValueDictionaryPtr = std::shared_ptr; + +class StringImm : public Value { + public: + explicit StringImm(const std::string& str) : Value(kString), str_(str), hash_(std::hash{}(str_)) {} + + ~StringImm() override = default; + MS_DECLARE_PARENT(StringImm, Value) + std::size_t hash() const override { return hash_; } + const std::string& value() const { return str_; } + bool operator==(const Value& other) const override; + bool operator==(const StringImm& other) const; + abstract::AbstractBasePtr ToAbstract() override; + std::string ToString() const override { return str_; } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "\"" << str_ << "\""; + return oss.str(); + } + + private: + std::string str_; + std::size_t hash_ = 0; +}; +using StringImmPtr = std::shared_ptr; +IMM_TRAITS(StringImmPtr, std::string) +IMM_TRAITS(StringImmPtr, const char*) + +class RefKey : public Value { + public: + explicit RefKey(const std::string& tag) : Value(kRefKeyType), tag_(tag), hash_(std::hash{}(tag)) {} + + ~RefKey() override = default; + MS_DECLARE_PARENT(RefKey, Value) + std::size_t hash() const override { return hash_; } + const std::string& tag() const { return tag_; } + bool operator==(const Value& other) const override; + bool operator==(const RefKey& other) const; + abstract::AbstractBasePtr ToAbstract() override; + std::string ToString() const override { return "RefKey[" + tag_ + "]"; } + + std::string DumpText() const override { + std::ostringstream oss; + oss << "RefKey[\"" << tag_ << "\"]"; + return oss.str(); + } + + private: + std::string tag_; + std::size_t hash_ = 0; +}; +using RefKeyPtr = std::shared_ptr; + +class AnyValue : public Value { + public: + AnyValue() = default; + ~AnyValue() override = default; + MS_DECLARE_PARENT(AnyValue, Value) + std::size_t hash() const override { return tid(); } + bool operator==(const Value& other) const override; + abstract::AbstractBasePtr ToAbstract() override; +}; +extern const ValuePtr kAnyValue; + +template <> +inline const char* GetValue(const ValuePtr& value) { + if (value == nullptr) { + MS_LOG(EXCEPTION) << "Value is nullptr"; + } + auto imm = value->cast(); + if (imm == nullptr) { + MS_LOG(EXCEPTION) << "GetValue:" << value->ToString() << ", Type:" << value->type_name(); + } + return common::SafeCStr(imm->value()); +} + +template ::type, + typename U = typename std::enable_if::value, typename S::value_type>::type> +std::vector GetValue(const ValuePtr& value) { + if (value == nullptr) { + MS_LOG(EXCEPTION) << "Value is nullptr"; + } + + if (!value->isa()) { + MS_LOG(EXCEPTION) << "Error GetValue for value: " << value->ToString() << ", type: vector<" << typeid(U).name() + << ">"; + } + std::vector rets; + const std::vector& vals = value->cast()->value(); + (void)std::transform(vals.begin(), vals.end(), std::back_inserter(rets), + [](const ValuePtr& v) { return GetValue(v); }); + return rets; +} + +inline ValueNodePtr NewValueNode(const ValuePtr& t) { return std::make_shared(t); } + +template ::value>::type> +inline ValueNodePtr NewValueNode(const std::shared_ptr& x) { + return NewValueNode(MakeValue(x)); +} + +template ::value>::type> +inline ValueNodePtr NewValueNode(const T& x) { + return NewValueNode(MakeValue(x)); +} +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_VALUE_H_ diff --git a/mindspore/ccsrc/ir/visitor.cc b/mindspore/ccsrc/ir/visitor.cc new file mode 100644 index 00000000000..70e22b28085 --- /dev/null +++ b/mindspore/ccsrc/ir/visitor.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/visitor.h" +#include "ir/func_graph.h" + +namespace mindspore { +AnfNodePtr AnfVisitor::operator()(const opt::OptimizerPtr &, const AnfNodePtr &) { return nullptr; } +void AnfVisitor::Visit(const AnfNodePtr &node) { node->accept(this); } + +void AnfVisitor::Visit(const CNodePtr &cnode) { + for (auto &input : cnode->inputs()) { + Visit(input); + } +} + +void AnfVisitor::Visit(const ValueNodePtr &vnode) { + if (IsValueNode(vnode)) { + auto func_graph = GetValueNode(vnode); + Visit(func_graph->output()); + } +} + +void AnfVisitor::Visit(const ParameterPtr &) {} + +VisitFuncType AnfVisitor::Match(const PrimitivePtr &prim, const std::vector &funcs) { + auto fn = [prim, funcs, this](const AnfNodePtr &node) { + if (!IsPrimitiveCNode(node, prim)) { + return; + } + + auto &inputs = node->cast()->inputs(); + // infact, funcs_size == inps_size - 1 + auto funcs_size = funcs.size(); + auto inps_size = inputs.size(); + + // check the inputs are matched with the predicate functions + if (funcs_size > 0) { + // use the predicate function list to check the number of inputs + if (funcs_size != (inps_size - 1)) { + return; + } + + // check per input + for (size_t i = 0; i < funcs_size; i++) { + if (!funcs[i](inputs[i + 1])) { + return; + } + } + } + + // visit the inputs + for (size_t i = 1; i < inps_size; i++) { + this->Visit(inputs[i]); + } + }; + + return fn; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/visitor.h b/mindspore/ccsrc/ir/visitor.h new file mode 100644 index 00000000000..5305d1fe858 --- /dev/null +++ b/mindspore/ccsrc/ir/visitor.h @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_VISITOR_H_ +#define MINDSPORE_CCSRC_IR_VISITOR_H_ + +#include +#include "ir/anf.h" +#include "optimizer/opt.h" + +namespace mindspore { +using VisitFuncType = std::function; +class AnfVisitor { + public: + virtual AnfNodePtr operator()(const opt::OptimizerPtr&, const AnfNodePtr&); + virtual void Visit(const AnfNodePtr&); + virtual void Visit(const CNodePtr&); + virtual void Visit(const ValueNodePtr&); + virtual void Visit(const ParameterPtr&); + VisitFuncType Match(const PrimitivePtr&, const std::vector& = {}); + virtual ~AnfVisitor() = default; +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_IR_VISITOR_H_ diff --git a/mindspore/ccsrc/kernel/CMakeLists.txt b/mindspore/ccsrc/kernel/CMakeLists.txt new file mode 100644 index 00000000000..4cd60b591c4 --- /dev/null +++ b/mindspore/ccsrc/kernel/CMakeLists.txt @@ -0,0 +1,35 @@ +file(GLOB_RECURSE _SESSION_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel_query.cc" + "kernel_fusion.cc" + "kernel_build_info.cc" + "kash/*.cc" + "common_utils.cc" + "oplib/*.cc" + ) + +add_library(_mindspore_kernel_obj OBJECT ${_SESSION_ALL_SRC_FILES}) + +if(ENABLE_GPU) + file(GLOB_RECURSE _CUDA_GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "gpu/*.cu" + "akg/gpu/*.cc" + ) + add_library(_cuda_gpu_kernel_obj OBJECT ${_CUDA_GPU_SRC_LIST}) + + file(GLOB_RECURSE _C_EXPRESSION_GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "gpu/*.cc" + ) + list(REMOVE_ITEM _C_EXPRESSION_GPU_SRC_LIST "gpu/nccl/nccl_gpu_kernel.cc") + add_library(_c_expression_gpu_device_obj OBJECT ${_C_EXPRESSION_GPU_SRC_LIST}) +endif() + +if(ENABLE_D) + file(GLOB_RECURSE _D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "akg/cce/*.cc" + "tbe/*.cc" + "aicpu/*.cc" + "mng/*.cc" + "hccl/*.cc" + ) + target_sources(_mindspore_kernel_obj PRIVATE ${_D_SRC_LIST}) +endif() \ No newline at end of file diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_build.cc b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_build.cc new file mode 100644 index 00000000000..c89e27c8ced --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_build.cc @@ -0,0 +1,305 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/aicpu/aicpu_kernel_build.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "device/kernel_runtime.h" +#include "kernel/aicpu/aicpu_kernel_mod.h" +#include "kernel/akg/akgkernelbuild.h" +#include "proto/tensor.pb.h" +#include "proto/tensor_shape.pb.h" +#include "proto/attr.pb.h" +#include "proto/node_def.pb.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" +#include "kernel/aicpu/aicpu_util.h" +#include "session/kernel_graph.h" +#include "kernel/common_utils.h" + +namespace mindspore { +namespace kernel { +using FNodeAttrHandle = std::function &anf_node, mindspore::NodeDef *proto)>; + +const std::vector local_framework_op_vec = {kInitDataSetQueue, kGetNext, kDropoutGenMask, kPrint}; + +void InitDataSetQueueAttr(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(proto); + + ::google::protobuf::Map<::std::string, ::mindspore::AttrValue> *node_attr = proto->mutable_attrs(); + MS_EXCEPTION_IF_NULL(node_attr); + std::string channel_name = AnfAlgo::GetNodeAttr(anf_node, kQueueName); + (*node_attr)[kChannelName].set_s(channel_name); +} + +void GetNextAttr(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(proto); + + ::google::protobuf::Map<::std::string, ::mindspore::AttrValue> *node_attr = proto->mutable_attrs(); + MS_EXCEPTION_IF_NULL(node_attr); + std::string shared_name = AnfAlgo::GetNodeAttr(anf_node, kSharedName); + (*node_attr)[kChannelName].set_s(shared_name); +} + +void DropoutGenMaskAttr(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(proto); + + ::google::protobuf::Map<::std::string, ::mindspore::AttrValue> *node_attr = proto->mutable_attrs(); + MS_EXCEPTION_IF_NULL(node_attr); + int seed = AnfAlgo::GetNodeAttr(anf_node, kSeed); + int seed2 = AnfAlgo::GetNodeAttr(anf_node, kSeed2); + (*node_attr)["seed"].set_i(seed); + (*node_attr)["seed2"].set_i(seed2); +} + +void CreateAttrFuncMap(std::map *mOpAttrFuncMap) { + (void)mOpAttrFuncMap->emplace(std::pair(kInitDataSetQueue, InitDataSetQueueAttr)); + (void)mOpAttrFuncMap->emplace(std::pair(kGetNext, GetNextAttr)); + (void)mOpAttrFuncMap->emplace(std::pair(kDropoutGenMask, DropoutGenMaskAttr)); +} + +bool SetIOIputSize(const std::shared_ptr &anf_node, const size_t &input_num, + std::vector *input_size_list) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(input_size_list); + for (size_t i = 0; i < input_num; i++) { + std::vector shape_i = AnfAlgo::GetInputDeviceShape(anf_node, i); + if (AnfAlgo::GetInputDeviceDataType(anf_node, i) == kObjectTypeString) { + if (!anf_node->isa()) { + MS_LOG(EXCEPTION) << "anf_node is not CNode."; + } + auto cnode = anf_node->cast(); + auto input_node = cnode->inputs()[i + 1]; + if (input_node->isa()) { + auto value_ptr = GetValueNode(input_node); + auto value = GetValue(value_ptr); + input_size_list->push_back(value.size()); + } + } else { + auto type_ptr = TypeIdToType(AnfAlgo::GetInputDeviceDataType(anf_node, i)); + MS_EXCEPTION_IF_NULL(type_ptr); + int size_i = 1; + for (size_t j = 0; j < shape_i.size(); j++) { + IntMulWithOverflowCheck(size_i, static_cast(shape_i[j]), &size_i); + } + size_t type_byte = GetTypeByte(type_ptr); + if (type_byte == 0) { + return false; + } + IntMulWithOverflowCheck(size_i, SizeToInt(type_byte), &size_i); + input_size_list->push_back(IntToSize(size_i)); + } + } + return true; +} + +bool SetIOSize(const std::shared_ptr &anf_node, const std::shared_ptr &kernel_mod_ptr) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + std::vector input_size_list; + std::vector output_size_list; + size_t input_num = AnfAlgo::GetInputTensorNum(anf_node); + size_t output_num = AnfAlgo::GetOutputTensorNum(anf_node); + + if (!SetIOIputSize(anf_node, input_num, &input_size_list)) { + return false; + } + kernel_mod_ptr->SetInputSizeList(input_size_list); + + for (size_t i = 0; i < output_num; i++) { + std::vector shape_i = AnfAlgo::GetOutputDeviceShape(anf_node, i); + TypePtr type_ptr = TypeIdToType(AnfAlgo::GetOutputDeviceDataType(anf_node, i)); + MS_EXCEPTION_IF_NULL(type_ptr); + int size_i = 1; + for (size_t j = 0; j < shape_i.size(); j++) { + IntMulWithOverflowCheck(size_i, static_cast(shape_i[j]), &size_i); + } + size_t type_byte = GetTypeByte(type_ptr); + if (type_byte == 0) { + return false; + } + IntMulWithOverflowCheck(size_i, SizeToInt(type_byte), &size_i); + output_size_list.push_back(IntToSize(size_i)); + } + kernel_mod_ptr->SetOutputSizeList(output_size_list); + + return true; +} + +void SetNodeAttr(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (op_name == "InitDataSetQueue") { + op_name = "InitData"; + } + if (op_name == "Print") { + return; + } + std::map mOpAttrFuncMap; + CreateAttrFuncMap(&mOpAttrFuncMap); + FNodeAttrHandle func_ptr = nullptr; + auto iter = mOpAttrFuncMap.find(op_name); + if (iter != mOpAttrFuncMap.end()) { + func_ptr = iter->second; + MS_EXCEPTION_IF_NULL(func_ptr); + func_ptr(anf_node, proto); + } else { + MS_LOG(ERROR) << "Don't support node [" << op_name << "] to set nodedef of attr"; + } + MS_LOG(INFO) << "Set node attr end!"; +} + +void SetNodeInputs(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + size_t input_num = AnfAlgo::GetInputTensorNum(anf_node); + if (input_num == 0) { + MS_LOG(INFO) << "Node [" << AnfAlgo::GetCNodeName(anf_node) << "] does not have input. "; + return; + } + + for (size_t input_index = 0; input_index < input_num; input_index++) { + ::mindspore::Tensor *node_inputs = proto->add_inputs(); + MS_EXCEPTION_IF_NULL(node_inputs); + TypeId input_type = AnfAlgo::GetInputDeviceDataType(anf_node, input_index); + std::vector input_shape; + int32_t input_data_type; + if (input_type == kObjectTypeString) { + auto cnode = anf_node->cast(); + auto input_node = cnode->inputs()[input_index + 1]; + auto value_ptr = GetValueNode(input_node); + auto value = GetValue(value_ptr); + input_shape.push_back(1); + input_shape.push_back(value.size()); + input_data_type = AicpuOpUtil::MsTypeToProtoType(kTypeUnknown); + } else { + input_shape = AnfAlgo::GetInputDeviceShape(anf_node, input_index); + input_data_type = AicpuOpUtil::MsTypeToProtoType(input_type); + } + mindspore::TensorShape *tensorShape = node_inputs->mutable_tensor_shape(); + for (auto item : input_shape) { + mindspore::TensorShape_Dim *dim = tensorShape->add_dim(); + dim->set_size((::google::protobuf::int64)item); + } + + node_inputs->set_tensor_type((mindspore::DataType)input_data_type); + + node_inputs->set_mem_device("HBM"); + } +} + +void SetNodeOutputs(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + size_t output_num = AnfAlgo::GetOutputTensorNum(anf_node); + if (output_num == 0) { + MS_LOG(INFO) << "Node [" << AnfAlgo::GetCNodeName(anf_node) << "] does not have output. "; + return; + } + + for (size_t output_index = 0; output_index < output_num; output_index++) { + ::mindspore::Tensor *node_outputs = proto->add_outputs(); + std::vector output_shape = AnfAlgo::GetOutputDeviceShape(anf_node, output_index); + mindspore::TensorShape *tensorShape = node_outputs->mutable_tensor_shape(); + for (auto item : output_shape) { + mindspore::TensorShape_Dim *dim = tensorShape->add_dim(); + dim->set_size((::google::protobuf::int64)item); + } + + TypeId output_type = AnfAlgo::GetOutputDeviceDataType(anf_node, output_index); + + int32_t output_data_type = AicpuOpUtil::MsTypeToProtoType(output_type); + node_outputs->set_tensor_type((mindspore::DataType)output_data_type); + + node_outputs->set_mem_device("HBM"); + } +} + +void SetNodedefProto(const std::shared_ptr &anf_node, mindspore::NodeDef *proto) { + MS_LOG(INFO) << "SetNodedefProto entry"; + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(proto); + + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (op_name == "InitDataSetQueue") { + op_name = "InitData"; + } + // set op name + proto->set_op(op_name); + + // set inputs tensor + SetNodeInputs(anf_node, proto); + + // set outputs tensor + SetNodeOutputs(anf_node, proto); + + // set node attr + SetNodeAttr(anf_node, proto); + + MS_LOG(INFO) << "SetNodedefProto end!"; +} + +bool CreateNodeDefBytes(const std::shared_ptr &anf_node, + const std::shared_ptr &kernel_mod_ptr) { + MS_LOG(INFO) << "CreateNodeDefBytes entry"; + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + mindspore::NodeDef proto; + + SetNodedefProto(anf_node, &proto); + + std::string nodeDefStr; + if (!proto.SerializeToString(&nodeDefStr)) { + MS_LOG(ERROR) << "Serialize nodeDef to string failed."; + return false; + } + + kernel_mod_ptr->SetNodeDef(nodeDefStr); + + MS_LOG(INFO) << "CreateNodeDefBytes end!"; + return true; +} + +KernelModPtr AicpuOpBuild(const std::shared_ptr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (op_name == "InitDataSetQueue") { + op_name = "InitData"; + } + auto kernel_mod_ptr = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + kernel_mod_ptr->SetAnfNode(anf_node); + kernel_mod_ptr->SetNodeName(op_name); + auto iter = std::find(local_framework_op_vec.begin(), local_framework_op_vec.end(), op_name); + if (iter != local_framework_op_vec.end()) { + if (!CreateNodeDefBytes(anf_node, kernel_mod_ptr)) { + MS_LOG(EXCEPTION) << "Create nodeDefBytes faild!"; + } + } else { + MS_LOG(EXCEPTION) << "Aicpu don't support node [" << op_name << "]"; + } + + if (!SetIOSize(anf_node, kernel_mod_ptr)) { + MS_LOG(EXCEPTION) << "Set input output size list failed."; + } + + return kernel_mod_ptr; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_build.h b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_build.h new file mode 100644 index 00000000000..a3c24ae49e9 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_build.h @@ -0,0 +1,27 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_BUILD_H_ +#define MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_BUILD_H_ +#include +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { +KernelModPtr AicpuOpBuild(const std::shared_ptr &anf_node); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_BUILD_H_ diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.cc b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.cc new file mode 100644 index 00000000000..ac0b0d9f7af --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.cc @@ -0,0 +1,103 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/aicpu/aicpu_kernel_metadata.h" +#include +#include +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +constexpr auto kInitDataSetQueueOpName = "InitDataSetQueue"; +constexpr auto kGetNext = "GetNext"; +constexpr auto kDropoutGenMask = "DropoutGenMask"; +constexpr auto kPrint = "Print"; +const std::vector AICPU_OPS = {kInitDataSetQueueOpName, kGetNext, kDropoutGenMask, kPrint}; + +std::shared_ptr CreateKernelInfo(const std::vector &inputs_format, + const std::vector &inputs_device_type, + const std::vector &outputs_format, + const std::vector &outputs_device_type) { + auto builder = KernelBuildInfo::KernelBuildInfoBuilder(); + builder.SetInputsFormat(inputs_format); + builder.SetInputsDeviceType(inputs_device_type); + builder.SetOutputsFormat(outputs_format); + builder.SetOutputsDeviceType(outputs_device_type); + builder.SetProcessor(AICPU); + builder.SetKernelType(AICPU_KERNEL); + builder.SetFusionType(OPAQUE); + return builder.Build(); +} + +bool CheckIfExistAicpuMeta(const std::string &op_name) { + if (std::find(AICPU_OPS.begin(), AICPU_OPS.end(), op_name) != AICPU_OPS.end()) { + return false; + } + return true; +} + +void AicpuMetadataInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list) { + MS_LOG(INFO) << "AicpuMetadataInfo."; + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_info_list); + std::string op_name = AnfAlgo::GetCNodeName(kernel_node); + if (CheckIfExistAicpuMeta(op_name)) { + MS_LOG(DEBUG) << "Aicpu doesn't have metadata of op [" << op_name << "]."; + return; + } + + if (op_name == kInitDataSetQueueOpName) { + kernel_info_list->push_back(CreateKernelInfo({}, {}, {}, {})); + } + + if (op_name == kGetNext) { + std::vector outputs_format; + std::vector outputs_type; + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(kernel_node); ++output_index) { + outputs_format.emplace_back(kOpFormat_DEFAULT); + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(kernel_node, output_index)); + } + kernel_info_list->push_back(CreateKernelInfo({}, {}, outputs_format, outputs_type)); + } + + if (op_name == kDropoutGenMask) { + kernel_info_list->push_back(CreateKernelInfo({kOpFormat_NCHW, kOpFormat_NCHW}, + {kInt32->type_id(), kFloat16->type_id()}, {kOpFormat_NCHW}, + {kUInt8->type_id()})); + } + + if (op_name == kPrint) { + std::vector inputs_format; + std::vector inputs_type; + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(kernel_node); ++input_index) { + inputs_format.emplace_back(kOpFormat_DEFAULT); + inputs_type.push_back(AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, input_index)); + } + std::vector outputs_format; + std::vector outputs_type; + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(kernel_node); ++output_index) { + outputs_format.emplace_back(kOpFormat_DEFAULT); + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(kernel_node, output_index)); + } + kernel_info_list->push_back(CreateKernelInfo(inputs_format, inputs_type, outputs_format, outputs_type)); + } + + if (kernel_info_list->empty()) { + MS_LOG(INFO) << "Aicpu dose not has metadata of op[ " << op_name << "]."; + } +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.h b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.h new file mode 100644 index 00000000000..74e667856e5 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.h @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_META_DATA_H_ +#define MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_META_DATA_H_ + +#include +#include +#include +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace kernel { +void AicpuMetadataInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list); +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_META_DATA_H_ diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_mod.cc b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_mod.cc new file mode 100644 index 00000000000..5028c757f2d --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_mod.cc @@ -0,0 +1,147 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/aicpu/aicpu_kernel_mod.h" + +#include +#include +#include +#include + +#include "runtime/mem.h" +#include "runtime/rt.h" +#include "kernel/aicpu/aicpu_kernel_build.h" +#include "utils/convert_utils.h" +#include "kernel/aicpu/aicpu_util.h" + +using AicpuTaskInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace kernel { +constexpr auto AICPU_OPS_SO_NAME = "libaicpu_kernels.so"; + +AicpuOpKernelMod::AicpuOpKernelMod() : anf_node_(nullptr) {} + +AicpuOpKernelMod::~AicpuOpKernelMod() { + args_.clear(); + inputList_.clear(); + outputList_.clear(); + anf_node_ = nullptr; + input_size_list_.clear(); + output_size_list_.clear(); + workspace_size_list_.clear(); +} + +void AicpuOpKernelMod::SetInputSizeList(const std::vector &size_list) { input_size_list_ = size_list; } +const std::vector &AicpuOpKernelMod::GetInputSizeList() const { return input_size_list_; } +void AicpuOpKernelMod::SetOutputSizeList(const std::vector &size_list) { output_size_list_ = size_list; } +const std::vector &AicpuOpKernelMod::GetOutputSizeList() const { return output_size_list_; } +void AicpuOpKernelMod::SetWorkspaceSizeList(const std::vector &size_list) { workspace_size_list_ = size_list; } +const std::vector &AicpuOpKernelMod::GetWorkspaceSizeList() const { return workspace_size_list_; } +void AicpuOpKernelMod::SetInputList(const std::vector &inputList) { inputList_ = inputList; } +void AicpuOpKernelMod::SetOutputList(const std::vector &outputList) { outputList_ = outputList; } +void AicpuOpKernelMod::SetNodeDef(const std::string &nodeDef) { (void)node_def_str_.assign(nodeDef); } +void AicpuOpKernelMod::SetNodeName(const std::string &node_name) { node_name_ = node_name; } +void AicpuOpKernelMod::SetAnfNode(const mindspore::AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + anf_node_ = anf_node; +} + +void AicpuOpKernelMod::CreateCpuKernelInfo(const std::vector &inputs, + const std::vector &outputs) { + MS_LOG(INFO) << "CreateCpuKernelInfoOffline start"; + + node_so_ = AICPU_OPS_SO_NAME; + + // InputOutputAddr + vector io_addrs; + (void)std::transform(std::begin(inputs), std::end(inputs), std::back_inserter(io_addrs), + [](const AddressPtr &input) -> void * { return input->addr; }); + (void)std::transform(std::begin(outputs), std::end(outputs), std::back_inserter(io_addrs), + [](const AddressPtr &output) -> void * { return output->addr; }); + + auto io_addrs_num = io_addrs.size(); + // calculate paramLen: AicpuParamHead.len + ioAddrsSize + notifyId.len + customizedAttr.len + auto param_len = sizeof(AicpuParamHead); + + // get input and output addrs size, no need to check overflow + auto io_addrs_size = io_addrs_num * sizeof(uint64_t); + // refresh paramLen, no need to check overflow + param_len += io_addrs_size; + + auto node_def_len = node_def_str_.length(); + param_len += node_def_len; + + // Create taskArgs: AicpuParamHead + ioAddrs + notifyId + customizedAttr + AicpuParamHead paramHead = {static_cast(param_len), static_cast(io_addrs_num)}; + args_.clear(); + (void)args_.append(reinterpret_cast(¶mHead), sizeof(AicpuParamHead)); + // TaskArgs append ioAddrs + if (io_addrs_size != 0) { + (void)args_.append(reinterpret_cast(io_addrs.data()), io_addrs_size); + } + + // When it's aicpu customized ops, taskArgs should append customized attr + if (node_def_len != 0) { + (void)args_.append(reinterpret_cast(node_def_str_.data()), node_def_len); + } + + MS_LOG(INFO) << "CreateCpuKernelInfoOffline end"; +} + +bool AicpuOpKernelMod::Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) { + if (stream_ptr == 0) { + MS_LOG(ERROR) << "stream_ptr should not be nullptr."; + return false; + } + + CreateCpuKernelInfo(inputs, outputs); + auto *stream = reinterpret_cast(stream_ptr); + MS_LOG(INFO) << "Aicpu launch, node_so_:" << node_so_ << ", node name:" << node_name_ + << ", args_size:" << args_.length(); + if (rtCpuKernelLaunch(reinterpret_cast(node_so_.c_str()), + reinterpret_cast(node_name_.c_str()), 1, + reinterpret_cast(args_.data()), static_cast(args_.length()), nullptr, + stream) != RT_ERROR_NONE) { + MS_LOG(ERROR) << "Aicpu op launch failed!"; + + return false; + } + return true; +} + +vector AicpuOpKernelMod::GenTask(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uint32_t stream_id) { + MS_LOG(INFO) << "AicpuOpKernelMod GenTask start"; + + node_so_ = AICPU_OPS_SO_NAME; + std::vector input_data_addrs; + (void)std::transform(std::begin(inputs), std::end(inputs), std::back_inserter(input_data_addrs), + [](const AddressPtr &input) -> void * { return input->addr; }); + + std::vector output_data_addrs; + (void)std::transform(std::begin(outputs), std::end(outputs), std::back_inserter(output_data_addrs), + [](const AddressPtr &output) -> void * { return output->addr; }); + + AicpuTaskInfoPtr task_info_ptr = make_shared( + stream_id, node_so_, node_name_, node_def_str_, input_data_addrs, output_data_addrs); + + MS_LOG(INFO) << "AicpuOpKernelMod GenTask end"; + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_mod.h b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_mod.h new file mode 100644 index 00000000000..d1e6f69b230 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_kernel_mod.h @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_MOD_H_ +#define MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_MOD_H_ +#include +#include +#include +#include "kernel/kernel.h" +#include "kernel/aicpu/aicpu_util.h" +namespace mindspore { +namespace kernel { +class AicpuOpKernelMod : public KernelMod { + public: + AicpuOpKernelMod(); + ~AicpuOpKernelMod() override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + + vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + void SetInputList(const std::vector &inputList); + void SetOutputList(const std::vector &outputList); + void SetAnfNode(const AnfNodePtr &anf_node); + void SetNodeDef(const std::string &nodeDef); + void SetNodeName(const std::string &node_name); + + /** + * @brief Build AICPU Engine kernel structure, and allocate device memory for offline task generate + * @return SUCCESS + * @return FAIL + * + */ + void CreateCpuKernelInfo(const std::vector &inputs, const std::vector &outputs); + + void SetInputSizeList(const std::vector &size_list); + void SetOutputSizeList(const std::vector &size_list); + void SetWorkspaceSizeList(const std::vector &size_list); + const std::vector &GetInputSizeList() const override; + const std::vector &GetOutputSizeList() const override; + const std::vector &GetWorkspaceSizeList() const override; + + private: + std::string args_; + std::string node_def_str_; + std::string node_name_; + std::string node_so_; + std::vector inputList_; + std::vector outputList_; + AnfNodePtr anf_node_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; + +using AicpuOpKernelModPtr = std::shared_ptr; +using AicputOpKernelModPtrList = std::vector; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_KERNEL_MOD_H_ diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_util.cc b/mindspore/ccsrc/kernel/aicpu/aicpu_util.cc new file mode 100644 index 00000000000..316df639226 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_util.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/aicpu/aicpu_util.h" +#include +#include +#include "proto/types.pb.h" +#include "runtime/mem.h" +#include "runtime/rt.h" +#include "utils/convert_utils.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +static std::map MS_PROTO_DATA_TYPE_MAP = { + {mindspore::TypeId::kTypeUnknown, mindspore::DataType::MS_UNKNOWN}, + {mindspore::TypeId::kNumberTypeBool, mindspore::DataType::MS_BOOL}, + {mindspore::TypeId::kNumberTypeInt8, mindspore::DataType::MS_INT8}, + {mindspore::TypeId::kNumberTypeInt16, mindspore::DataType::MS_INT16}, + {mindspore::TypeId::kNumberTypeInt32, mindspore::DataType::MS_INT32}, + {mindspore::TypeId::kNumberTypeInt64, mindspore::DataType::MS_INT64}, + {mindspore::TypeId::kNumberTypeUInt, mindspore::DataType::MS_UINT32}, + {mindspore::TypeId::kNumberTypeUInt8, mindspore::DataType::MS_UINT8}, + {mindspore::TypeId::kNumberTypeUInt16, mindspore::DataType::MS_UINT16}, + {mindspore::TypeId::kNumberTypeUInt64, mindspore::DataType::MS_UINT64}, + {mindspore::TypeId::kNumberTypeFloat16, mindspore::DataType::MS_FLOAT16}, + {mindspore::TypeId::kNumberTypeFloat32, mindspore::DataType::MS_FLOAT32}, + {mindspore::TypeId::kNumberTypeFloat64, mindspore::DataType::MS_FLOAT64}, +}; + +int AicpuOpUtil::MsTypeToProtoType(TypeId ms_type) { + auto iter = MS_PROTO_DATA_TYPE_MAP.find(ms_type); + if (iter != MS_PROTO_DATA_TYPE_MAP.end()) { + return MS_PROTO_DATA_TYPE_MAP[ms_type]; + } else { + MS_LOG(ERROR) << "UnSupported ms_type value" << static_cast(ms_type); + return -1; + } +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/aicpu/aicpu_util.h b/mindspore/ccsrc/kernel/aicpu/aicpu_util.h new file mode 100644 index 00000000000..f521418f6b3 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/aicpu_util.h @@ -0,0 +1,59 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_UTIL_H_ +#define MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_UTIL_H_ + +#include +#include +#include +#include +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { +constexpr auto kInitDataSetQueue = "InitData"; +constexpr auto kGetNext = "GetNext"; +constexpr auto kDropoutGenMask = "DropoutGenMask"; +constexpr auto kPrint = "Print"; + +constexpr auto kOutputTypes = "output_types"; +constexpr auto kOutputShapes = "output_shapes"; +constexpr auto kChannelName = "channel_name"; +constexpr auto kSharedName = "shared_name"; +constexpr auto kShapes = "shapes"; +constexpr auto kTypes = "types"; +constexpr auto kQueueName = "queue_name"; + +constexpr auto kSeed = "Seed0"; +constexpr auto kSeed2 = "Seed1"; + +struct AicpuParamHead { + uint32_t length; // Total length: include cunstom message + uint32_t ioAddrNum; // Input and output address number +} __attribute__((packed)); + +class AicpuOpUtil { + public: + static int MsTypeToProtoType(TypeId ms_type); + + private: + // kernel id + static uint64_t KernelId_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_KERNEL_AICPU_AICPU_UTIL_H_ diff --git a/mindspore/ccsrc/kernel/aicpu/proto/attr.proto b/mindspore/ccsrc/kernel/aicpu/proto/attr.proto new file mode 100644 index 00000000000..a0ab4bd1e76 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/proto/attr.proto @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package mindspore; +import "tensor.proto"; +import "tensor_shape.proto"; +import "types.proto"; + +message AttrValue { + + message ArrayValue { + repeated bytes s = 2; //"array(string)" + repeated int64 i = 3 [ packed = true ]; //"array(int)" + repeated float f = 4 [ packed = true ]; //"array(float)" + repeated bool b = 5 [ packed = true ]; //"array(bool)" + repeated DataType type = 6 [ packed = true ]; //"array(type)" + repeated TensorShape shape = 7; //"array(shape)" + repeated Tensor tensor = 8; //"array(tensor)" + } + + oneof value { + ArrayValue array = 1; + bytes s = 2; //"string" + int64 i = 3; //"int" + float f = 4; //"float" + bool b = 5; //"bool" + DataType type = 6; //"type" + TensorShape shape = 7; //"shape" + Tensor tensor = 8; //"tensor" + } +} diff --git a/mindspore/ccsrc/kernel/aicpu/proto/node_def.proto b/mindspore/ccsrc/kernel/aicpu/proto/node_def.proto new file mode 100644 index 00000000000..b0c0e0f349b --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/proto/node_def.proto @@ -0,0 +1,26 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package mindspore; +import "attr.proto"; +import "tensor.proto"; + +message NodeDef { + string op = 2; + map attrs = 3; + repeated Tensor inputs = 4; + repeated Tensor outputs = 5; +} diff --git a/mindspore/ccsrc/kernel/aicpu/proto/tensor.proto b/mindspore/ccsrc/kernel/aicpu/proto/tensor.proto new file mode 100644 index 00000000000..b4fd66595a1 --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/proto/tensor.proto @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; + +option cc_enable_arenas = true; +import "types.proto"; +import "tensor_shape.proto"; +package mindspore; + +message Tensor { + + // tensor shape info + TensorShape tensor_shape = 1; + + // tensor content data type + DataType tensor_type = 2; + + // tensor memory device + // data located memory device , "DDR" "HBM" OR "NONE" + string mem_device = 3; +} diff --git a/mindspore/ccsrc/kernel/aicpu/proto/tensor_shape.proto b/mindspore/ccsrc/kernel/aicpu/proto/tensor_shape.proto new file mode 100644 index 00000000000..70534e8ebab --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/proto/tensor_shape.proto @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package mindspore; + +message TensorShape { + // One dimension of the tensor. + message Dim { + // size must >=0 + int64 size = 1; + }; + + // group dim info + repeated Dim dim = 2; + + // If true, the number of dimensions in the shape is unknown. + // If true, "dim.size()" must be 0. + bool unknown_rank = 3; + + // data format "NHWC" "NCHW" "NC1HWC0" OR "NONE" + string data_format = 4; +}; diff --git a/mindspore/ccsrc/kernel/aicpu/proto/types.proto b/mindspore/ccsrc/kernel/aicpu/proto/types.proto new file mode 100644 index 00000000000..574259d97df --- /dev/null +++ b/mindspore/ccsrc/kernel/aicpu/proto/types.proto @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; + +option cc_enable_arenas = true; +package mindspore; + +enum DataType { + MS_UNKNOWN = 0; + MS_BOOL = 1; + MS_INT8 = 2; + MS_UINT8 = 3; + MS_INT16 = 4; + MS_UINT16 = 5; + MS_INT32 = 6; + MS_UINT32 = 7; + MS_INT64 = 8; + MS_UINT64 = 9; + MS_FLOAT16 = 10; + MS_FLOAT32 = 11; + MS_FLOAT64 = 12; +} diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc b/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc new file mode 100644 index 00000000000..c9ff41dc552 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc @@ -0,0 +1,176 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/akg/akg_kernel_attrs_process.h" + +#include +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace kernel { +void SetAkgAttrsForFour2Five(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + // The x and output are akg op input and output param. + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + AnfAlgo::SetNodeAttr("input_names", MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr("output_names", MakeValue(output_names), anf_node); + + TypeId dst_type_id = AnfAlgo::GetOutputDeviceDataType(anf_node, 0); + std::string dst_type; + if (dst_type_id == kFloat32->type_id()) { + dst_type = "float32"; + } else if (dst_type_id == kFloat16->type_id()) { + dst_type = "float16"; + } + AnfAlgo::SetNodeAttr("dst_type", MakeValue(dst_type), anf_node); +} + +void SetAkgAttrsForFive2Four(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + AnfAlgo::SetNodeAttr("input_names", MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr("output_names", MakeValue(output_names), anf_node); + std::vector origin_shape = AnfAlgo::GetOutputInferShape(anf_node, 0); + if (origin_shape.size() != kShape4dDims) { + MS_LOG(EXCEPTION) << "The dim of origin_shape is not equal to 4, but it's dim is " << origin_shape.size() << "."; + } + std::vector shape_transform; + (void)std::transform(origin_shape.begin(), origin_shape.end(), std::back_inserter(shape_transform), + [](const int &origin_shape) { return static_cast(origin_shape); }); + AnfAlgo::SetNodeAttr("shape4d", MakeValue(shape_transform), anf_node); + AnfAlgo::SetNodeAttr("output_format", MakeValue(kOpFormat_NCHW), anf_node); + + TypeId dst_type_id = AnfAlgo::GetOutputDeviceDataType(anf_node, 0); + std::string dst_type; + if (dst_type_id == kFloat32->type_id()) { + dst_type = "float32"; + } else if (dst_type_id == kFloat16->type_id()) { + dst_type = "float16"; + } + AnfAlgo::SetNodeAttr("dstType", MakeValue(dst_type), anf_node); +} + +void SetAkgAttrsForCast(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + // The x and output are akg op input and output param. + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + AnfAlgo::SetNodeAttr("input_names", MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr("output_names", MakeValue(output_names), anf_node); + + std::string dst_type; + TypeId output_type = AnfAlgo::GetOutputDeviceDataType(anf_node, 0); + if (output_type == kFloat32->type_id()) { + dst_type = "float32"; + } else if (output_type == kFloat16->type_id()) { + dst_type = "float16"; + } + AnfAlgo::SetNodeAttr("dst_type", MakeValue(dst_type), anf_node); +} + +void SetAkgAttrsForBNGrad1(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::vector input_names{"dy", "data", "mean"}; + std::vector output_names{"dgamma_red_hw", "dbeta_red_hw", "data_minus_mean"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(output_names), anf_node); +} + +void SetAkgAttrsForBNGrad2(const AnfNodePtr &anf_node) { + const size_t kBNGrad2InputSize = 5; + MS_EXCEPTION_IF_NULL(anf_node); + std::vector input_names{"dgamma_red_hw", "dbeta_red_hw", "variance", "gamma"}; + std::vector output_names{"bn_scale", "bn_bias", "rs", "dgamma_dx", "dbeta_dx"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(output_names), anf_node); + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() < kBNGrad2InputSize) { + MS_LOG(EXCEPTION) << "The inputs size of BNGrad2 is less then " << kBNGrad2InputSize; + } + auto input1 = cnode->input(1); + MS_EXCEPTION_IF_NULL(input1); + auto tuple_getitem = input1->cast(); + MS_EXCEPTION_IF_NULL(tuple_getitem); + if (tuple_getitem->inputs().size() < kTupleGetItemInputSize) { + MS_LOG(EXCEPTION) << "The inputs size of tuple_getitem is less then " << kTupleGetItemInputSize; + } + auto bn_grad1 = tuple_getitem->input(kRealInputNodeIndexInTupleGetItem); + std::vector data_shape = AnfAlgo::GetInputDeviceShape(bn_grad1, 0); + AnfAlgo::SetNodeAttr(kAttrDataShape, MakeValue(opt::Convert2Int(data_shape)), anf_node); +} + +void SetAkgAttrsForBNGrad3(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::vector input_names{"dy", "rs", "dgamma_dx", "dbeta_dx", "data_minus_mean"}; + std::vector output_names{"dx"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(output_names), anf_node); +} + +void SetAkgAttrsForFusedBN1(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + // Set attr for fused_bn1 + std::vector fused_bn1_input_names{"data"}; + std::vector fused_bn1_output_names{"mean", "var_part"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(fused_bn1_input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(fused_bn1_output_names), anf_node); +} + +void SetAkgAttrsForFusedBN2(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + // Set attr for fused_bn2 + std::vector fused_bn2_input_names{"mean", "var_part", "running_mean", "running_var"}; + std::vector fused_bn2_output_names{"variance", "running_mean", "running_variance"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(fused_bn2_input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(fused_bn2_output_names), anf_node); +} + +void SetAkgAttrsForFusedBN3(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + // Set attr for fused_bn3 + std::vector fused_bn3_input_names{"data", "mean", "variance", "gamma", "beta"}; + std::vector fused_bn3_output_names{"y"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(fused_bn3_input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(fused_bn3_output_names), anf_node); +} + +void SetAkgAttrsForConvBN1(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::vector conv_bn1_output_names{"data", "var_part", "mean"}; + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(conv_bn1_output_names), anf_node); +} + +void SetAkgAttrsForBN2AddRelu(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::vector bn2_add_relu_input_names{"data", "var_part", "mean", "other_branch_data", + "gamma", "beta", "running_mean", "running_var"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(bn2_add_relu_input_names), anf_node); + std::vector bn2_add_relu_output_names{"output", "running_mean", "running_variance", "save_inv_variance"}; + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(bn2_add_relu_output_names), anf_node); +} + +void SetAkgAttrsForBN2Relu(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::vector bn2_input_names{"data", "var_part", "mean", "gamma", "beta", "running_mean", "running_var"}; + std::vector bn2_output_names{"y", "running_mean", "running_variance", "save_inv_variance"}; + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(bn2_input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(bn2_output_names), anf_node); +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.h b/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.h new file mode 100644 index 00000000000..9d15d4f9e9b --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.h @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_KERNEL_AKG_AKG_KERNEL_ATTRS_PROCESS_H +#define MINDSPORE_CCSRC_KERNEL_AKG_AKG_KERNEL_ATTRS_PROCESS_H + +#include +#include +#include +#include +#include "ir/anf.h" +#include "utils/utils.h" +#include "operator/ops.h" + +namespace mindspore { +namespace kernel { +void SetAkgAttrsForFour2Five(const AnfNodePtr &anf_node); +void SetAkgAttrsForFive2Four(const AnfNodePtr &anf_node); +void SetAkgAttrsForCast(const AnfNodePtr &anf_node); +void SetAkgAttrsForBNGrad1(const AnfNodePtr &anf_node); +void SetAkgAttrsForBNGrad2(const AnfNodePtr &anf_node); +void SetAkgAttrsForBNGrad3(const AnfNodePtr &anf_node); +void SetAkgAttrsForFusedBN1(const AnfNodePtr &anf_node); +void SetAkgAttrsForFusedBN2(const AnfNodePtr &anf_node); +void SetAkgAttrsForFusedBN3(const AnfNodePtr &anf_node); +void SetAkgAttrsForConvBN1(const AnfNodePtr &anf_node); +void SetAkgAttrsForBN2AddRelu(const AnfNodePtr &anf_node); +void SetAkgAttrsForBN2Relu(const AnfNodePtr &anf_node); + +const std::unordered_map> kAkgKernelAttrsProcessMap = { + {kFour2FiveOpName, SetAkgAttrsForFour2Five}, + {kFive2FourOpName, SetAkgAttrsForFive2Four}, + {"Cast", SetAkgAttrsForCast}, + {kBNGrad1OpName, SetAkgAttrsForBNGrad1}, + {kBNGrad2OpName, SetAkgAttrsForBNGrad2}, + {kBNGrad3OpName, SetAkgAttrsForBNGrad3}, + {kFusedBN1OpName, SetAkgAttrsForFusedBN1}, + {kFusedBN2OpName, SetAkgAttrsForFusedBN2}, + {kFusedBN3OpName, SetAkgAttrsForFusedBN3}, + {kConvBN1OpName, SetAkgAttrsForConvBN1}, + {kBN2AddReluOpName, SetAkgAttrsForBN2AddRelu}, + {kBN2ReLUOpName, SetAkgAttrsForBN2Relu}, +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_AKG_AKG_KERNEL_ATTRS_PROCESS_H diff --git a/mindspore/ccsrc/kernel/akg/akgkernelbuild.cc b/mindspore/ccsrc/kernel/akg/akgkernelbuild.cc new file mode 100644 index 00000000000..8413208c4d4 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/akgkernelbuild.cc @@ -0,0 +1,518 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/akg/akgkernelbuild.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "utils/convert_utils.h" +#include "utils/any.h" +#include "utils/utils.h" +#include "transform/convert.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/akg/akg_kernel_attrs_process.h" + +namespace mindspore { +namespace kernel { +constexpr int ME_MAX_KERNEL_NAME_LENGTH = 200; +constexpr int32_t ARGS_SIZE = 1; +constexpr auto kCompileWithJsonFunc = "compilewithjson"; +// json key +constexpr auto kInputDesc = "input_desc"; +constexpr auto kShape = "shape"; +constexpr auto kDataType = "data_type"; +constexpr auto kOutputDesc = "output_desc"; +constexpr auto kName = "name"; +constexpr auto kTensorName = "tensor_name"; +constexpr auto kValue = "value"; +constexpr auto KInpputNames = "input_names"; +constexpr auto KInput = "input"; +constexpr auto KDtype = "dtype"; +int AkgKernelBuild::op_cnt_ = 0; +std::mutex AkgKernelBuild::op_cnt_mtx_; + +std::string PyObjectToStr(PyObject *const PyObj) { + char *pChar = nullptr; + std::string str_res; + if (PyObj == nullptr) { + MS_LOG(ERROR) << "Input parameter is nullptr."; + return str_res; + } + PyObject *strArgs = PyObject_Str(PyObj); + if (strArgs != nullptr) { + (void)PyArg_Parse(strArgs, "s", &pChar); + } + if (pChar == nullptr) { + MS_LOG(ERROR) << "pChar is nullptr."; + return str_res; + } + str_res = pChar; + return str_res; +} + +std::string AkgKernelBuild::GetProcessor(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string device; + switch (AnfAlgo::GetProcessor(anf_node)) { + case Processor::AICORE: + device = kProcessorAiCore; + break; + + case Processor::AICPU: + device = kProcessorAiCpu; + break; + + case Processor::CUDA: + device = kProcessorCuda; + break; + + default: + MS_LOG(ERROR) << "Unknown processor type."; + break; + } + + return device; +} + +bool GetIOSize(const nlohmann::json &node_json, std::vector *const input_size, + std::vector *const output_size) { + if (input_size == nullptr || output_size == nullptr) { + MS_LOG(ERROR) << "input size or output size is nullptr"; + return false; + } + input_size->clear(); + output_size->clear(); + + for (size_t i = 0; i < node_json[kInputDesc].size(); i++) { + for (size_t m = 0; m < node_json[kInputDesc][i].size(); m++) { + std::string dtype = node_json[kInputDesc][i][m][kDataType]; + size_t nbyte = GetDtypeNbyte(dtype); + size_t size_i = std::accumulate(node_json[kInputDesc][i][m][kShape].begin(), + node_json[kInputDesc][i][m][kShape].end(), nbyte, std::multiplies()); + input_size->push_back(size_i); + } + } + + for (size_t i = 0; i < node_json[kOutputDesc].size(); i++) { + std::string dtype = node_json[kOutputDesc][i][kDataType]; + size_t nbyte = GetDtypeNbyte(dtype); + size_t size_i = std::accumulate(node_json[kOutputDesc][i][kShape].begin(), node_json[kOutputDesc][i][kShape].end(), + nbyte, std::multiplies()); + output_size->push_back(size_i); + } + + return true; +} + +int AkgKernelBuild::GetOpCntInc() { + op_cnt_mtx_.lock(); + int cnt = op_cnt_++; + op_cnt_mtx_.unlock(); + return cnt; +} + +bool AkgKernelBuild::CreateInputDescJson(const AnfNodePtr &anf_node, nlohmann::json *const inputs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(inputs_json); + + // for dynamic input number, dyn_input_sizes has the info of dynamic input num for each input. + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + auto op_info = mindspore::kernel::OpLib::FindOp(op_name, OpImplyType::kAKG); + if (op_info == nullptr) { + MS_LOG(ERROR) << "Apply kernel [" << op_name << "] op_info is nullptr"; + return false; + } + + std::vector> inputs_ptr = op_info->inputs_ptr(); + if (inputs_ptr.empty()) { + MS_LOG(INFO) << "Apply kernel [" << op_name << "] regist info has no input info"; + return true; + } + auto op_info_input_num = inputs_ptr.size(); + + // for dynamic input number, dyn_input_sizes has the info of dynamic input num for each input. + std::vector dyn_input_sizes; + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + + if (primitive->GetAttr(kAttrDynInputSizes) != nullptr) { + dyn_input_sizes = GetValue>(primitive->GetAttr(kAttrDynInputSizes)); + } + + size_t real_input_index = 0; + std::vector input_list; + for (size_t i = 0; i < op_info_input_num; i++) { + size_t input_tensor_num; + std::shared_ptr input_ptr = inputs_ptr[i]; + std::string op_input_name; + if (input_ptr == nullptr) { + MS_LOG(ERROR) << "Apply kernel [" << op_name << "] regist input[" << i << "] is nullptr"; + return false; + } + + op_input_name = input_ptr->name(); + if (dyn_input_sizes.empty()) { + input_tensor_num = 1; + } else { + input_tensor_num = IntToSize(dyn_input_sizes[i]); + } + + input_list.clear(); + for (size_t input_i = 0; input_i < input_tensor_num; input_i++) { + // dtype : float16 + auto type_id = AnfAlgo::GetInputDeviceDataType(anf_node, real_input_index); + TypePtr type_ptr = TypeIdToType(type_id); + MS_EXCEPTION_IF_NULL(type_ptr); + std::string dtype = type_ptr->ToString(); + dtype = Dtype2String(dtype); + if (dtype.empty()) { + MS_LOG(ERROR) << "Op [" << op_name << "] input [" << input_i << "] data type is null. "; + return false; + } + nlohmann::json input_desc_json; + input_desc_json[kDataType] = dtype; + input_desc_json[kName] = op_input_name; + input_desc_json[kTensorName] = + op_input_name + "_" + std::to_string(real_input_index) + "_" + std::to_string(input_i); + input_desc_json[kShape] = AnfAlgo::GetInputDeviceShape(anf_node, real_input_index); + input_list.emplace_back(input_desc_json); + } + inputs_json->emplace_back(input_list); + real_input_index++; + } + return true; +} + +bool AkgKernelBuild::CreateOutputDescJson(const AnfNodePtr &anf_node, nlohmann::json *const outputs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(outputs_json); + size_t output_tensor_num = AnfAlgo::GetOutputTensorNum(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + + auto op_info_ptr = mindspore::kernel::OpLib::FindOp(op_name, OpImplyType::kAKG); + auto outputs = op_info_ptr->outputs_ptr(); + for (size_t i = 0; i < output_tensor_num; i++) { + nlohmann::json output_json; + auto type_id = AnfAlgo::GetOutputDeviceDataType(anf_node, i); + TypePtr type_ptr = TypeIdToType(type_id); + MS_EXCEPTION_IF_NULL(type_ptr); + std::string dtype = type_ptr->ToString(); + dtype = Dtype2String(dtype); + if (dtype.empty()) { + MS_LOG(ERROR) << "Op [" << op_name << "] output [" << i << "] data type is null. "; + return false; + } + + std::string output_name = outputs[i]->name(); + output_json[kDataType] = dtype; + output_json[kName] = output_name; + output_json[kTensorName] = output_name + "_" + std::to_string(i); + output_json[kShape] = AnfAlgo::GetOutputDeviceShape(anf_node, i); + outputs_json->push_back(output_json); + } + return true; +} + +void GetJson(const AnfNodePtr &anf_node, const vector &dyn_input_sizes, const shared_ptr &op_attr, + nlohmann::json *const attr_json, const ValuePtr &attr_value) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(op_attr); + MS_EXCEPTION_IF_NULL(attr_json); + std::string type = op_attr->type(); + if (type == "int") { + (*attr_json)[kValue] = GetValue(attr_value); + } else if (type == "str") { + (*attr_json)[kValue] = GetValue(attr_value); + } else if (type == "bool") { + (*attr_json)[kValue] = GetValue(attr_value); + } else if (type == "float") { + (*attr_json)[kValue] = GetValue(attr_value); + } else if (type == "listInt") { + (*attr_json)[kValue] = GetValue>(attr_value); + } else if (type == "listStr") { + std::vector data_format; + if (op_attr->name() == kArgDataformat) { + size_t tensor_args_num = !dyn_input_sizes.empty() ? dyn_input_sizes.size() : AnfAlgo::GetInputTensorNum(anf_node); + for (size_t format_i = 0; format_i < tensor_args_num; format_i++) { + auto input_format = AnfAlgo::GetInputFormat(anf_node, format_i); + data_format.push_back(input_format); + } + } else { + data_format = GetValue>(attr_value); + } + (*attr_json)[kValue] = data_format; + } else { + MS_LOG(WARNING) << "attr type:" << type; + } +} + +bool AkgKernelBuild::CreateAttrDescJson(const AnfNodePtr &anf_node, const std::string &op_name, + const std::shared_ptr &op_info, nlohmann::json *const attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + MS_EXCEPTION_IF_NULL(op_info); + std::vector> attrs = op_info->attrs_ptr(); + if (attrs.empty()) { + MS_LOG(INFO) << "Apply kernel [" << op_name << "] op info attrs is empty"; + return true; + } + std::vector> inputs = op_info->inputs_ptr(); + + std::vector dyn_input_sizes; + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr(kAttrDynInputSizes) != nullptr) { + dyn_input_sizes = GetValue>(primitive->GetAttr(kAttrDynInputSizes)); + } + + if (inputs.empty()) { + MS_LOG(ERROR) << "Apply kernel [" << op_name << "] op info inputs is empty"; + return false; + } + + // create input name list for atch "x_shape" in att with "x" in primitive. + std::map op_info_shape_name; + for (size_t op_info_input_i = 0; op_info_input_i < inputs.size(); op_info_input_i++) { + std::string input_name = inputs[op_info_input_i]->name(); + std::string x_shape_name = input_name + "_shape"; + (void)op_info_shape_name.insert(make_pair(op_info_input_i, x_shape_name)); + } + + for (const auto &op_attr : attrs) { + nlohmann::json attr_json; + ValuePtr attr_value = primitive->GetAttr(op_attr->name()); + if (attr_value == nullptr && op_attr->name() != kArgDataformat) { + if (op_attr->param_type() == "required") { + // match "x_shape" in att with "x" in primitive. + std::string attr_name = op_attr->name(); + auto find_item = std::find_if( + op_info_shape_name.begin(), op_info_shape_name.end(), + [attr_name](const std::map::value_type item) { return item.second == attr_name; }); + if (find_item != op_info_shape_name.end()) { + if (!dyn_input_sizes.empty()) { + if (find_item->first >= dyn_input_sizes.size() - 1) { + MS_LOG(EXCEPTION) << "dyn_input_sizes list index:" << find_item->first + << " is out of range:" << dyn_input_sizes.size() - 1 << "."; + return false; + } + size_t tensor_idx = IntToSize(std::accumulate(&dyn_input_sizes[0], &dyn_input_sizes[find_item->first], 0)); + for (int input_i = 0; input_i < dyn_input_sizes[find_item->first]; input_i++) { + attr_json[kValue] = AnfAlgo::GetPrevNodeOutputInferShape(anf_node, tensor_idx); + attr_json[kName] = op_attr->name(); + attrs_json->push_back(attr_json); + tensor_idx++; + } + } else { + attr_json[kValue] = AnfAlgo::GetPrevNodeOutputInferShape(anf_node, find_item->first); + attr_json[kName] = op_attr->name(); + attrs_json->push_back(attr_json); + } + } else { + MS_LOG(ERROR) << "op [" << op_name << "] should have attr :" << op_attr->name(); + return false; + } + } + continue; + } + + GetJson(anf_node, dyn_input_sizes, op_attr, &attr_json, attr_value); + + attr_json[kName] = op_attr->name(); + attrs_json->push_back(attr_json); + } + return true; +} + +bool AkgKernelBuild::GenerateSingleKernelJson(const AnfNodePtr &anf_node, const std::string &op_name, + nlohmann::json *const node_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(node_json); + int op_cnt = GetOpCntInc(); + auto op_info_ptr = mindspore::kernel::OpLib::FindOp(op_name, OpImplyType::kAKG); + MS_EXCEPTION_IF_NULL(op_info_ptr); + + // get basic params from currentNodeOpDesc + (*node_json)["platform"] = "AKG"; + (*node_json)[kName] = op_name; + (*node_json)["fusion_type"] = AnfAlgo::GetFusionType(anf_node); + (*node_json)["impl_path"] = op_info_ptr->impl_path(); + (*node_json)["process"] = AkgKernelBuild::GetProcessor(anf_node); + + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + ValuePtr input_names_v = primitive->GetAttr(KInpputNames); + if (input_names_v == nullptr) { + MS_LOG(ERROR) << "ApplyKernel has no input_names, op[" << op_name << "]."; + return false; + } + std::vector prim_input_names = GetValue>(input_names_v); + std::string inputs_name; + for (const auto &prim_input_name : prim_input_names) { + (void)inputs_name.append("_input_").append(prim_input_name).append("_"); + } + + // input desc + nlohmann::json inputs_json; + if (!CreateInputDescJson(anf_node, &inputs_json)) { + MS_LOG(ERROR) << "Create input desc json failed, op[" << op_name << "]."; + return false; + } + (*node_json)[kInputDesc] = inputs_json; + MS_LOG(INFO) << "Akg create input desc json success."; + std::string inputs_shape = "inputs_shape_"; + for (auto &i : inputs_json) { + for (auto &m : i) { + std::string data_type = m[kDataType]; + (void)inputs_shape.append("_").append(data_type).append("_"); + for (auto &j : m[kShape]) { + size_t n = j; + (void)inputs_shape.append(std::to_string(n)).append("_"); + } + } + } + + // output desc + nlohmann::json outputs_json; + if (!CreateOutputDescJson(anf_node, &outputs_json)) { + MS_LOG(ERROR) << "Create output desc json failed, op[" << op_name << "]."; + return false; + } + + (*node_json)[kOutputDesc] = outputs_json; + MS_LOG(INFO) << "Akg create output desc json success."; + std::string outputs_shape = "outputs_shape_"; + for (auto &i : outputs_json) { + std::string data_type = i[kDataType]; + (void)outputs_shape.append("_").append(data_type).append("_"); + for (auto &j : i[kShape]) { + size_t m = j; + (void)outputs_shape.append(std::to_string(m)).append("_"); + } + } + + // attribute desc + nlohmann::json attrs_json; + if (!CreateAttrDescJson(anf_node, op_name, op_info_ptr, &attrs_json)) { + MS_LOG(ERROR) << "Create attr desc json failed, op[" << op_name << "]."; + return false; + } + (*node_json)["attr"] = attrs_json; + std::string json_str = node_json->dump(); + size_t hash_id = std::hash()(json_str); + json_name_ = op_name + "_"; + (void)json_name_.append(std::to_string(hash_id)); + MS_LOG(INFO) << "full scope name is : " << anf_node->fullname_with_scope() << ", json info name is : " << json_name_; + json_info_ = json_str; + (*node_json)["id"] = op_cnt; + (*node_json)["op"] = json_name_; + MS_LOG(INFO) << "Akg create node desc json success."; + return true; +} + +KernelPackPtr AkgKernelBuild::OpBuild(const std::string &node_json, const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + auto processor = AkgKernelBuild::GetProcessor(anf_node); + auto cached_kernel_pack = SearchCache(json_name_, processor); + if (cached_kernel_pack != nullptr) { + MS_LOG(INFO) << "Use cached kernel, json_name_[" << json_name_ << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + return cached_kernel_pack; + } + + PyObject *pModule = nullptr; + PyObject *pFunc = nullptr; + PyObject *pArg = nullptr; + PyObject *pRes = nullptr; + + pModule = PyImport_ImportModule(kAkgModule); + if (pModule == nullptr) { + MS_LOG(ERROR) << "Failed to import [" << kAkgModule << "]."; + return nullptr; + } + + pFunc = PyObject_GetAttrString(pModule, kCompileWithJsonFunc); + pArg = PyTuple_New(ARGS_SIZE); + (void)PyTuple_SetItem(pArg, 0, Py_BuildValue("s", node_json.c_str())); + + (void)alarm(AUTODIFF_COMPILE_OVERTIME); + pRes = PyEval_CallObject(pFunc, pArg); + (void)alarm(0); + if (pRes == nullptr) { + MS_LOG(ERROR) << "No ret got, failed to call function [" << kCompileWithJsonFunc << "], args:\n(" + << PyObjectToStr(pArg) << ")."; + return nullptr; + } + if (PyObject_IsTrue(pRes) != 1) { + MS_LOG(ERROR) << "Illegal ret, failed to call function [" << kCompileWithJsonFunc << "], args:\n(" + << PyObjectToStr(pArg) << ")."; + return nullptr; + } + + auto new_kernel_pack = InsertCache(json_name_, processor); + kernel::SaveJsonInfo(json_name_, json_info_); + if (new_kernel_pack == nullptr) { + MS_LOG(ERROR) << "Insert to cache failed, json_name_[" << json_name_ << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + return nullptr; + } + return new_kernel_pack; +} + +KernelPackPtr AkgKernelBuild::BuildByJson(const AnfNodePtr &anf_node, std::vector *const input_size, + std::vector *const output_size) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + auto it = kAkgKernelAttrsProcessMap.find(op_name); + if (it != kAkgKernelAttrsProcessMap.end()) { + it->second(anf_node); + } + MS_LOG(INFO) << "Akg start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; + nlohmann::json node_json; + if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { + MS_LOG(ERROR) << "Op[" << op_name << "] create single kernel json failed."; + } + + std::string json_str = node_json.dump(); + auto kernel_pack = OpBuild(json_str, anf_node); + if (kernel_pack == nullptr) { + MS_LOG(ERROR) << "Akg build failed op[" << op_name << "], json:" << json_str; + return nullptr; + } + + if (!GetIOSize(node_json, input_size, output_size)) { + MS_LOG(ERROR) << "Cal mem size failed."; + return nullptr; + } + MS_LOG(INFO) << "Akg compile success, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) + << "]"; + return kernel_pack; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/akgkernelbuild.h b/mindspore/ccsrc/kernel/akg/akgkernelbuild.h new file mode 100644 index 00000000000..f8127843bd6 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/akgkernelbuild.h @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_AKG_AKGKERNELBUILD_H_ +#define MINDSPORE_CCSRC_KERNEL_AKG_AKGKERNELBUILD_H_ +#include +#include +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "ir/dtype.h" +#include +#include "kernel/common_utils.h" +#include "kernel/oplib/oplib.h" + +namespace mindspore { +namespace kernel { +class AkgKernelBuild { + public: + AkgKernelBuild() = default; + ~AkgKernelBuild() = default; + + KernelPackPtr BuildByJson(const AnfNodePtr &anf_node, std::vector *const input_size, + std::vector *const output_size); + + private: + bool CreateInputDescJson(const AnfNodePtr &anf_node, nlohmann::json *const inputs_json); + bool CreateOutputDescJson(const AnfNodePtr &anf_node, nlohmann::json *const outputs_json); + bool CreateAttrDescJson(const AnfNodePtr &anf_node, const std::string &op_name, + const std::shared_ptr &op_info, nlohmann::json *const attrs_json); + bool GenerateSingleKernelJson(const AnfNodePtr &anf_node, const std::string &op_name, + nlohmann::json *const node_json); + KernelPackPtr OpBuild(const std::string &node_json, const AnfNodePtr &anf_node); + + int GetOpCntInc(); + std::string GetProcessor(const AnfNodePtr &anf_node); + static int op_cnt_; + // lock for variable fusionOpCnt in singleton mode + static std::mutex op_cnt_mtx_; + std::string json_name_; + std::string json_info_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_AKG_AKGKERNELBUILD_H_ diff --git a/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_build.cc b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_build.cc new file mode 100644 index 00000000000..2bb2cfd2678 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_build.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/akg/gpu/akg_gpu_kernel_build.h" +#include +#include +#include "kernel/kernel.h" +#include "kernel/akg/akgkernelbuild.h" +#include "kernel/akg/gpu/akg_gpu_kernel_mod.h" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +KernelModPtr AkgGpuKernelBuild(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + AkgKernelBuild akg_kernel_build; + + std::vector input_size_list; + std::vector output_size_list; + KernelPackPtr kernel_pack = akg_kernel_build.BuildByJson(anf_node, &input_size_list, &output_size_list); + MS_EXCEPTION_IF_NULL(kernel_pack); + + auto kernel_mod_ptr = std::make_shared(kernel_pack); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + kernel_mod_ptr->SetInputSizeList(input_size_list); + kernel_mod_ptr->SetOutputSizeList(output_size_list); + return kernel_mod_ptr; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_build.h b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_build.h new file mode 100644 index 00000000000..3a1145140f8 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_build.h @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_AKG_GPU_AKG_GPU_KERNEL_BUILD_H_ +#define MINDSPORE_CCSRC_KERNEL_AKG_GPU_AKG_GPU_KERNEL_BUILD_H_ +#include "kernel/kernel.h" +#include "ir/base.h" + +namespace mindspore { +namespace kernel { +KernelModPtr AkgGpuKernelBuild(const AnfNodePtr &anf_node); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_AKG_GPU_AKG_GPU_KERNEL_BUILD_H_ diff --git a/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_mod.cc b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_mod.cc new file mode 100644 index 00000000000..7ce434b2d35 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_mod.cc @@ -0,0 +1,116 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/akg/gpu/akg_gpu_kernel_mod.h" +#include +#include +#include "nlohmann/json.hpp" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +using std::fstream; +using std::string; +using std::vector; + +GpuKernelManagerPtr GpuKernelMod::kernelmanager_ = std::make_shared(); +GpuKernelManager::GpuKernelManager() {} + +CUresult GpuKernelManager::GetFunction(const KernelPackPtr &kernel_pack, bool force_reload, + vector *thread_info, CUfunction *func) { + if (kernel_pack->GetJson() == nullptr || kernel_pack->GetJson()->contents == nullptr || + kernel_pack->GetKernel() == nullptr || kernel_pack->GetKernel()->contents == nullptr) { + MS_LOG(ERROR) << "GPU:Invalid kernel pack, json or kernel is nullptr."; + return CUDA_ERROR_INVALID_IMAGE; + } + auto js = nlohmann::json::parse(kernel_pack->GetJson()->contents, + kernel_pack->GetJson()->contents + kernel_pack->GetJson()->len); + string fn = js["kernelName"]; + if (!force_reload) { + auto iter = infotable_.find(fn); + if (iter != infotable_.end()) { + auto kernelmeta = iter->second; + *thread_info = kernelmeta->thread_info_; + *func = kernelmeta->func_addr_; + return CUDA_SUCCESS; + } + } + thread_info->emplace_back(js["blockIdx.x"]); + thread_info->emplace_back(js["blockIdx.y"]); + thread_info->emplace_back(js["blockIdx.z"]); + thread_info->emplace_back(js["threadIdx.x"]); + thread_info->emplace_back(js["threadIdx.y"]); + thread_info->emplace_back(js["threadIdx.z"]); + CUmodule module; + CUresult result = cuModuleLoadData(&module, kernel_pack->GetKernel()->contents); + if (result != CUDA_SUCCESS) { + MS_LOG(ERROR) << "cuModuleLoadData failed."; + return result; + } + result = cuModuleGetFunction(func, module, fn.c_str()); + if (result != CUDA_SUCCESS) { + MS_LOG(ERROR) << "cuModuleGetFunction failed."; + return result; + } + infotable_[fn] = std::make_shared(*func, module, *thread_info); + return result; +} + +GpuKernelMod::GpuKernelMod(const KernelPackPtr &kernel_pack) : kernel_pack_(kernel_pack) {} + +void GpuKernelMod::SetInputSizeList(const std::vector &size_list) { input_size_list_ = size_list; } + +void GpuKernelMod::SetOutputSizeList(const std::vector &size_list) { output_size_list_ = size_list; } + +const std::vector &GpuKernelMod::GetInputSizeList() const { return input_size_list_; } + +const std::vector &GpuKernelMod::GetOutputSizeList() const { return output_size_list_; } + +const std::vector &GpuKernelMod::GetWorkspaceSizeList() const { return workspace_size_list_; } + +bool GpuKernelMod::Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) { + if (stream_ptr == 0) { + MS_LOG(ERROR) << "stream_ptr should not be nullptr."; + return false; + } + if (kernel_pack_ == nullptr) { + MS_LOG(ERROR) << "kernel pack should not be nullptr."; + return false; + } + vector thread_info; + CUfunction kernel_addr; + CUresult result = kernelmanager_->GetFunction(kernel_pack_, false, &thread_info, &kernel_addr); + if (result != CUDA_SUCCESS) { + MS_LOG(ERROR) << "GetFunction failed."; + return false; + } + std::vector runtimeargs; + (void)std::transform(std::begin(inputs), std::end(inputs), std::back_inserter(runtimeargs), + [](const AddressPtr &input) -> void * { return reinterpret_cast(&(input->addr)); }); + (void)std::transform(std::begin(outputs), std::end(outputs), std::back_inserter(runtimeargs), + [](const AddressPtr &output) -> void * { return reinterpret_cast(&(output->addr)); }); + result = cuLaunchKernel(kernel_addr, thread_info[0], thread_info[1], thread_info[2], thread_info[3], thread_info[4], + thread_info[5], 0, reinterpret_cast(stream_ptr), + reinterpret_cast(&runtimeargs[0]), 0); + if (result != CUDA_SUCCESS) { + MS_LOG(ERROR) << "Launch Kernel failed."; + return false; + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_mod.h b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_mod.h new file mode 100644 index 00000000000..4ef903f1f32 --- /dev/null +++ b/mindspore/ccsrc/kernel/akg/gpu/akg_gpu_kernel_mod.h @@ -0,0 +1,82 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_AKG_GPU_AKG_GPU_KERNEL_MOD_H_ +#define MINDSPORE_CCSRC_KERNEL_AKG_GPU_AKG_GPU_KERNEL_MOD_H_ +#include +#include +#include +#include +#include +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { +struct GpuKernelMeta { + CUfunction func_addr_; + CUmodule module_; + std::vector thread_info_; + GpuKernelMeta(CUfunction funcAddr, CUmodule module, const std::vector &thread_info) + : func_addr_(funcAddr), module_(module), thread_info_(thread_info) {} +}; +using GpuKernelMetaPtr = std::shared_ptr; + +class GpuKernelManager { + public: + GpuKernelManager(); + virtual ~GpuKernelManager() { + for (auto iter = infotable_.begin(); iter != infotable_.end(); ++iter) { + CUresult ret = cuModuleUnload(iter->second->module_); + if (ret != CUDA_SUCCESS && ret != CUDA_ERROR_DEINITIALIZED) { + MS_LOG(ERROR) << "Unload GPU Module failed."; + } + } + } + CUresult GetFunction(const KernelPackPtr &kernel_pack, bool force_reload, std::vector *thread_info, + CUfunction *func); + + private: + std::unordered_map infotable_; +}; +using GpuKernelManagerPtr = std::shared_ptr; + +class GpuKernelMod : public KernelMod { + public: + explicit GpuKernelMod(const KernelPackPtr &kernel_pack); + virtual ~GpuKernelMod() {} + + void SetInputSizeList(const std::vector &size_list); + void SetOutputSizeList(const std::vector &size_list); + const std::vector &GetInputSizeList() const override; + const std::vector &GetOutputSizeList() const override; + const std::vector &GetWorkspaceSizeList() const override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + + static GpuKernelManagerPtr kernelmanager_; + + private: + KernelPackPtr kernel_pack_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; + +using GpuKernelModPtr = std::shared_ptr; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_AKG_GPU_AKG_GPU_KERNEL_MOD_H_ diff --git a/mindspore/ccsrc/kernel/common_utils.cc b/mindspore/ccsrc/kernel/common_utils.cc new file mode 100644 index 00000000000..d610ea736d6 --- /dev/null +++ b/mindspore/ccsrc/kernel/common_utils.cc @@ -0,0 +1,528 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/common_utils.h" +#include +#include +#include +#include +#include "runtime/rt.h" +#include "nlohmann/json.hpp" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +const std::unordered_map type_id_maps = { + {"float", TypeId::kNumberTypeFloat32}, {"float16", TypeId::kNumberTypeFloat16}, + {"float32", TypeId::kNumberTypeFloat32}, {"float64", TypeId::kNumberTypeFloat64}, + {"int", TypeId::kNumberTypeInt}, {"int8", TypeId::kNumberTypeInt8}, + {"int16", TypeId::kNumberTypeInt16}, {"int32", TypeId::kNumberTypeInt32}, + {"int64", TypeId::kNumberTypeInt64}, {"uint", TypeId::kNumberTypeUInt}, + {"uint8", TypeId::kNumberTypeUInt8}, {"uint16", TypeId::kNumberTypeUInt16}, + {"uint32", TypeId::kNumberTypeUInt32}, {"uint64", TypeId::kNumberTypeUInt64}, + {"bool", TypeId::kNumberTypeBool}, +}; + +const std::map type_id_str_map = { + {TypeId::kNumberTypeFloat32, "float32"}, {TypeId::kNumberTypeFloat16, "float16"}, + {TypeId::kNumberTypeFloat, "float"}, {TypeId::kNumberTypeFloat64, "float64"}, + {TypeId::kNumberTypeInt, "int"}, {TypeId::kNumberTypeInt8, "int8"}, + {TypeId::kNumberTypeInt16, "int16"}, {TypeId::kNumberTypeInt32, "int32"}, + {TypeId::kNumberTypeInt64, "int64"}, {TypeId::kNumberTypeUInt, "uint"}, + {TypeId::kNumberTypeUInt8, "uint8"}, {TypeId::kNumberTypeUInt16, "uint16"}, + {TypeId::kNumberTypeUInt32, "uint32"}, {TypeId::kNumberTypeUInt64, "uint64"}, + {TypeId::kNumberTypeBool, "bool"}, +}; + +const std::map DATATYPE_STRING_MAP{ + {"Float32", "float32"}, {"Float16", "float16"}, {"Int8", "int8"}, {"Int16", "int16"}, + {"UInt16", "uint16"}, {"UInt8", "uint8"}, {"Int32", "int32"}, {"UInt32", "uint32"}, + {"Int64", "int64"}, {"UInt64", "uint64"}, {"Bool_", "bool"}, {"Float64", "double"}, +}; + +const std::unordered_map dtype_shortdtype_map_ = { + {"float16", "f16"}, {"float32", "f32"}, {"float64", "f64"}, {"int8", "i8"}, {"int16", "i16"}, {"int32", "i32"}, + {"int64", "i64"}, {"uint8", "u8"}, {"uint16", "u16"}, {"uint32", "u32"}, {"uint64", "u64"}, {"bool", "bool"}, +}; + +const std::unordered_map dtype_nbyte_map = { + {"float16", sizeof(float) / 2}, {"float32", sizeof(float)}, {"float64", sizeof(float) * 2}, + {"int8", sizeof(int) / 4}, {"int16", sizeof(int) / 2}, {"int32", sizeof(int)}, + {"int64", sizeof(int) * 2}, {"uint8", sizeof(int) / 4}, {"uint16", sizeof(int) / 2}, + {"uint32", sizeof(int)}, {"uint64", sizeof(int) * 2}, {"bool", sizeof(char)}, +}; + +const std::unordered_map fusion_type_maps = { + {"CONVLUTION", FusionType::CONVLUTION}, {"ELEMWISE", FusionType::ELEMWISE}, {"COMMREDUCE", FusionType::COMMREDUCE}, + {"SEGMENT", FusionType::SEGMENT}, {"OPAQUE", FusionType::OPAQUE}, +}; + +bool IsAtomicNode(const CNodePtr &kernel_node) { + MS_EXCEPTION_IF_NULL(kernel_node); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto parameters_indexs = kernel_mod->GenParameters(); + if (parameters_indexs.empty()) { + return false; + } + auto atomic_flag = false; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + auto workspace_size_list = kernel_mod->GetWorkspaceSizeList(); + size_t workspace_num = kernel_mod->GetWorkspaceSizeList().size(); + if (input_num + workspace_num + output_num > parameters_indexs.size()) { + size_t lossNum = (input_num + workspace_num + output_num) - parameters_indexs.size(); + for (size_t i = 0; i < lossNum; i++) { + parameters_indexs.push_back(0); + } + } + std::vector clean_output_indexs; + // in parameters data sort as input->workspace->output + size_t index = 0; + while (index < output_num) { + if (parameters_indexs[input_num + workspace_num + index] == 1) { + atomic_flag = true; + clean_output_indexs.push_back(index); + } + index++; + } + if (atomic_flag) { + AnfAlgo::SetNodeAttr(kAttrAutomicOutputIndexs, MakeValue(clean_output_indexs), kernel_node); + } + for (size_t i = 0; i < workspace_num; ++i) { + if (parameters_indexs[input_num + i] == 1) { + atomic_flag = true; + AnfAlgo::SetNodeAttr(kAttrAutomicWorkspaceSize, + MakeValue(std::accumulate(workspace_size_list.begin(), workspace_size_list.end(), 0)), + kernel_node); + break; + } + } + return atomic_flag; +} + +bool KernelMeta::ReadIndex(const std::string &bin_dir) { + DIR *dir = opendir(bin_dir.c_str()); + if (dir == nullptr) { + auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU); + if (ret != 0) { + MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "]."; + return false; + } + dir = opendir(bin_dir.c_str()); + } + + struct dirent *entry; + while ((entry = readdir(dir)) != nullptr) { + string bin_dir_tmp = bin_dir; + std::string cce_json = entry->d_name; + if (cce_json.length() <= 5) { + continue; + } + + std::string suffix = cce_json.substr(cce_json.length() - 5); + if (suffix != kJsonSuffix) { + continue; + } + + auto sp = cce_json.rfind('/'); + if (sp != std::string::npos) { + continue; + } + + sp = cce_json.rfind('.'); + if (sp == std::string::npos) { + continue; + } + auto kernel_name = cce_json.substr(0, sp); + (void)bin_dir_tmp.append("/"); + (void)bin_dir_tmp.append(cce_json); + kernel_meta_map_[kernel_name] = bin_dir_tmp; + } + (void)closedir(dir); + + MS_LOG(INFO) << "Cache kernel initialized, kernel size[" << kernel_meta_map_.size() << "]."; + initialized_ = true; + return true; +} + +std::string KernelMeta::Search(const std::string &kernel_name) const { + if (!initialized_) { + return ""; + } + + auto iter = kernel_meta_map_.find(kernel_name); + if (iter == kernel_meta_map_.end()) { + return ""; + } else { + return iter->second; + } +} + +bool KernelMeta::Insert(const std::string &kernel_name, const std::string &cce_json) { + if (!initialized_) { + return false; + } + kernel_meta_map_[kernel_name] = cce_json; + return true; +} + +bool CheckCache(const std::string &kernel_name) { + // check cache. + KernelMeta *bin_map = KernelMeta::GetInstance(); + if (bin_map == nullptr) { + MS_LOG(DEBUG) << "kernel cache is invalid."; + return false; + } + std::string cce_json = bin_map->Search(kernel_name); + bool ret = (!cce_json.empty()); + if (ret) { + MS_LOG(INFO) << "Kernel name:" << kernel_name << " has registed."; + } else { + MS_LOG(INFO) << "Kernel name:" << kernel_name << " will been registed."; + } + return ret; +} + +KernelPackPtr SearchCache(const std::string &kernel_name, const std::string &processor) { + // search cache. + KernelMeta *bin_map = KernelMeta::GetInstance(); + if (bin_map == nullptr) { + MS_LOG(DEBUG) << "kernel cache is invalid."; + return nullptr; + } + + std::string cce_json = bin_map->Search(kernel_name); + if (!cce_json.empty()) { + KernelPackPtr kernel_pack = std::make_shared(); + // just a tmp solution. + if (!kernel_pack->ReadFromJsonFile(cce_json, processor)) { + MS_LOG(DEBUG) << "Read cache json and bin file failed[" << cce_json << "]."; + return nullptr; + } else { + return kernel_pack; + } + } else { + MS_LOG(INFO) << "cache kernel not found[" << kernel_name << "]."; + return nullptr; + } +} + +KernelPackPtr InsertCache(const std::string &kernel_name, const std::string &processor) { + MS_LOG(INFO) << "kernel name:" << kernel_name << ", processr:" << processor; + std::string cce_json; + if (processor == kProcessorAiCore || processor == kProcessorAiCpu) { + cce_json = kCceKernelMeta; + } else { + cce_json = kGpuKernelMeta; + } + (void)cce_json.append(kernel_name).append(kJsonSuffix); + KernelPackPtr kernel_pack = std::make_shared(); + if (!kernel_pack->ReadFromJsonFile(cce_json, processor)) { + MS_LOG(DEBUG) << "Read json and bin file failed[" << cce_json << "]."; + return nullptr; + } + + KernelMeta *bin_map = KernelMeta::GetInstance(); + if (bin_map == nullptr) { + MS_LOG(DEBUG) << "kernel cache is invalid."; + return nullptr; + } + if (bin_map->Insert(kernel_name, cce_json)) { + MS_LOG(INFO) << "Insert to cache success[" << cce_json << "], kernelname[" << kernel_name << "]."; + } + return kernel_pack; +} + +TypeId DtypeToTypeId(const std::string &dtypes) { + auto iter = type_id_maps.find(dtypes); + if (iter != type_id_maps.end()) { + return iter->second; + } else { + MS_EXCEPTION(ArgumentError) << "Illegal input device dtype:" << dtypes; + } +} + +std::string Dtype2String(const std::string &dtypes) { + auto iter = DATATYPE_STRING_MAP.find(dtypes); + if (iter == DATATYPE_STRING_MAP.end()) { + MS_EXCEPTION(ArgumentError) << "Illegal input dtype:" << dtypes; + } + return iter->second; +} + +std::string TypeId2String(TypeId type_id) { + auto iter = type_id_str_map.find(type_id); + if (iter == type_id_str_map.end()) { + MS_EXCEPTION(ArgumentError) << "Illegal input dtype." << TypeIdLabel(type_id); + } + return iter->second; +} + +std::string Dtype2ShortType(const std::string &dtypes) { + auto iter = dtype_shortdtype_map_.find(dtypes); + if (iter != dtype_shortdtype_map_.end()) { + return iter->second; + } else { + MS_EXCEPTION(ArgumentError) << "Illegal input dtype:" << dtypes; + } +} + +size_t GetDtypeNbyte(const std::string &dtypes) { + auto iter = dtype_nbyte_map.find(dtypes); + if (iter != dtype_nbyte_map.end()) { + return iter->second; + } else { + MS_EXCEPTION(ArgumentError) << "Illegal input dtype:" << dtypes; + } +} + +bool SetInputKernelBuilderInfo(const std::vector> &inputs, size_t real_input_num, + size_t builder_idex, const std::vector &dyn_input_sizes, + const std::shared_ptr &builder) { + MS_EXCEPTION_IF_NULL(builder); + + std::vector inputs_device_type; + std::vector inputs_format; + size_t dyn_input_idx = 0; + size_t kernel_info_index = 0; + MS_EXCEPTION_IF_NULL(inputs[0]); + size_t kernel_info_cnt = inputs[0]->dtypes().size(); + + for (const auto &input : inputs) { + MS_EXCEPTION_IF_NULL(input); + std::string param_type = input->param_type(); + std::vector dtypes = input->dtypes(); + std::vector formats = input->formats(); + if (dtypes.size() != kernel_info_cnt || formats.size() != kernel_info_cnt) { + MS_LOG(DEBUG) << "Set input kernel builder info, dtyps size != formats size."; + return false; + } + + if (param_type == "dynamic") { + if (dyn_input_sizes.empty()) { + MS_LOG(DEBUG) << "Set input kernel builder info, dyn_input_sizes's size is 0 when param_type is dynamic"; + return false; + } + + for (int t = 0; t < dyn_input_sizes[dyn_input_idx]; t++) { + kernel_info_index++; + auto type_id = DtypeToTypeId(dtypes[builder_idex]); + inputs_device_type.push_back(type_id); + inputs_format.push_back(formats[builder_idex]); + } + dyn_input_idx++; + } else if (param_type == "required") { + kernel_info_index++; + auto type_id = DtypeToTypeId(dtypes[builder_idex]); + inputs_device_type.push_back(type_id); + inputs_format.push_back(formats[builder_idex]); + } else { + if (kernel_info_index < real_input_num) { + MS_LOG(INFO) << "Set input kernel builder info, input type is optional, input index is :" << kernel_info_index; + kernel_info_index++; + auto type_id = DtypeToTypeId(dtypes[builder_idex]); + inputs_device_type.push_back(type_id); + inputs_format.push_back(formats[builder_idex]); + } + } + } + + builder->SetInputsDeviceType(inputs_device_type); + builder->SetInputsFormat(inputs_format); + return true; +} + +bool SetOutputKernelBuilderInfo(const std::vector> &outputs, size_t builder_idex, + const size_t &real_output_num, + const std::shared_ptr &builder) { + // not now but in the next we need to support dynamic output case + MS_EXCEPTION_IF_NULL(builder); + + size_t output_idx = 0; + std::vector outputs_device_type; + std::vector outputs_format; + MS_EXCEPTION_IF_NULL(outputs[0]); + size_t kernel_info_cnt = outputs[0]->dtypes().size(); + + for (const auto &output : outputs) { + MS_EXCEPTION_IF_NULL(output); + if (output_idx >= real_output_num) { + MS_LOG(DEBUG) << "real_output_num:" << real_output_num << ", output_idx:" << output_idx << " is out of limit!"; + continue; + } + size_t output_num = 0; + if (output->param_type() == "dynamic") { + if (outputs.size() > 1) { + MS_EXCEPTION(ArgumentError) << "Dynamic output is unsupported multi output!"; + } + output_num = real_output_num; + } else if (output->param_type() == "required") { + output_num = 1; + } else { + if (output_idx < real_output_num) { + MS_LOG(INFO) << "Set output kernel builder info, output type is optional, output index is :" << output_idx; + output_num = 1; + } + } + + for (size_t i = 0; i < output_num; i++) { + std::vector dtypes = output->dtypes(); + std::vector formats = output->formats(); + if (dtypes.size() != kernel_info_cnt || formats.size() != kernel_info_cnt) { + MS_LOG(DEBUG) << "Set output kernel builder info, dtyps size != formats size."; + return false; + } + auto type_id = DtypeToTypeId(dtypes[builder_idex]); + outputs_device_type.push_back(type_id); + outputs_format.push_back(formats[builder_idex]); + output_idx++; + } + } + + builder->SetOutputsFormat(outputs_format); + builder->SetOutputsDeviceType(outputs_device_type); + return true; +} + +void SetKernelBuildInfo(const std::shared_ptr &builder, Processor processor, + const std::shared_ptr &op_info_ptr) { + MS_EXCEPTION_IF_NULL(builder); + MS_EXCEPTION_IF_NULL(op_info_ptr); + + auto imply_type = op_info_ptr->imply_type(); + builder->SetProcessor(processor); + std::string fusion_type = op_info_ptr->fusion_type(); + auto iter = fusion_type_maps.find(fusion_type); + if (iter != fusion_type_maps.end()) { + builder->SetFusionType(iter->second); + } else { + if (imply_type == kAKG) { + MS_EXCEPTION(NotExistsError) << "Illegal fusion type from dsl register:" << fusion_type; + } + } + + if (imply_type == kAKG) { + builder->SetKernelType(AUTO_DIFF_KERNEL); + } else { + builder->SetKernelType(TBE_KERNEL); + } +} + +bool ParseMetadata(const CNodePtr &kernel_node, const std::shared_ptr &op_info_ptr, Processor processor, + std::vector> *const kernel_info_list) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_info_list); + size_t real_input_num = AnfAlgo::GetInputTensorNum(kernel_node); + size_t real_output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + std::vector> inputs = op_info_ptr->inputs_ptr(); + std::vector> outputs = op_info_ptr->outputs_ptr(); + std::vector dyn_input_sizes; + auto primitive = AnfAlgo::GetCNodePrimitive(kernel_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr("dyn_input_sizes") != nullptr) { + dyn_input_sizes = GetValue>(primitive->GetAttr("dyn_input_sizes")); + } + if (inputs.size() > 0) { + MS_EXCEPTION_IF_NULL(inputs[0]); + size_t kernel_info_cnt = inputs[0]->dtypes().size(); + for (size_t j = 0; j < kernel_info_cnt; j++) { + auto builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(builder); + SetKernelBuildInfo(builder, processor, op_info_ptr); + + if (!SetInputKernelBuilderInfo(inputs, real_input_num, j, dyn_input_sizes, builder)) { + MS_LOG(DEBUG) << "Parse kernel metadata, set inputs kernel builder info failed."; + return false; + } + + if (outputs.size() > 0) { + if (!SetOutputKernelBuilderInfo(outputs, j, real_output_num, builder)) { + MS_LOG(DEBUG) << "Parse kernel metadata, set outputs kernel builder info failed."; + return false; + } + } + + kernel_info_list->push_back(builder->Build()); + } + } else if (outputs.size() > 0) { + MS_EXCEPTION_IF_NULL(outputs[0]); + size_t kernel_info_cnt = outputs[0]->dtypes().size(); + for (size_t j = 0; j < kernel_info_cnt; j++) { + auto builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(builder); + SetKernelBuildInfo(builder, processor, op_info_ptr); + + if (!SetOutputKernelBuilderInfo(outputs, j, real_output_num, builder)) { + MS_LOG(DEBUG) << "Parse kernel metadata, set outputs kernel builder info failed."; + return false; + } + + kernel_info_list->push_back(builder->Build()); + } + } + return true; +} + +void SaveJsonInfo(const std::string &json_name, const std::string &info) { + char real_path[PATH_MAX] = {0}; + std::string path = kCceKernelMeta + json_name + kInfoSuffix; + if (path.size() > PATH_MAX) { + MS_LOG(DEBUG) << "file path " << path << " is too long."; + return; + } + std::ofstream filewrite; + filewrite.open(path); + if (!filewrite.is_open()) { + return; + } + filewrite << info << endl; + filewrite.close(); + if (nullptr == realpath(path.c_str(), real_path)) { + MS_LOG(DEBUG) << "dir " << path << " does not exit."; + return; + } + MS_LOG(INFO) << "real path is :" << real_path; + if (chmod(real_path, S_IRUSR) == -1) { + MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail."; + } +} + +std::string GetProcessor(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string device; + switch (AnfAlgo::GetProcessor(anf_node)) { + case Processor::AICORE: + device = kProcessorAiCore; + break; + + case Processor::AICPU: + device = kProcessorAiCpu; + break; + + case Processor::CUDA: + device = kProcessorCuda; + break; + + default: + MS_LOG(DEBUG) << "Unknown processor type."; + break; + } + return device; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/common_utils.h b/mindspore/ccsrc/kernel/common_utils.h new file mode 100644 index 00000000000..6e3635d9045 --- /dev/null +++ b/mindspore/ccsrc/kernel/common_utils.h @@ -0,0 +1,85 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_COMMON_UTILS_H_ +#define MINDSPORE_CCSRC_KERNEL_COMMON_UTILS_H_ + +#include +#include +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "kernel/oplib/opinfo.h" +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace kernel { +constexpr auto kCceKernelMeta = "./kernel_meta/"; +constexpr auto kGpuKernelMeta = "/tmp/cuda_meta/"; +constexpr auto kProcessorAiCore = "aicore"; +constexpr auto kProcessorAiCpu = "aicpu"; +constexpr auto kProcessorCuda = "cuda"; +constexpr auto kJsonSuffix = ".json"; +constexpr auto kInfoSuffix = ".info"; +constexpr unsigned int AUTODIFF_COMPILE_OVERTIME = 600; +constexpr auto kAkgModule = "akg"; +constexpr auto kArgDataformat = "data_format"; + +const std::vector support_devices = {"aicore", "aicpu", "cuda"}; + +struct KernelMetaInfo { + uintptr_t func_stub_; + uint32_t block_dim_; +}; +using KernelMetaPtr = std::shared_ptr; + +class KernelMeta { + public: + KernelMeta() = default; + bool ReadIndex(const std::string &bin_dir); + std::string Search(const std::string &kernel_name) const; + bool Insert(const std::string &kernel_name, const std::string &cce_json); + + static KernelMeta *GetInstance() { + static KernelMeta kernel_meta; + return &kernel_meta; + } + ~KernelMeta() = default; + + private: + bool initialized_ = false; + std::unordered_map kernel_meta_map_; +}; + +bool CheckCache(const std::string &kernel_name); +KernelPackPtr SearchCache(const std::string &kernel_name, const std::string &processor); +KernelPackPtr InsertCache(const std::string &kernel_name, const std::string &processor); +TypeId DtypeToTypeId(const std::string &dtypes); +std::string Dtype2String(const std::string &dtypes); +std::string Dtype2ShortType(const std::string &dtypes); +std::string TypeId2String(TypeId type_id); +size_t GetDtypeNbyte(const std::string &dtypes); +bool ParseMetadata(const CNodePtr &kernel_node, const std::shared_ptr &op_info_ptr, Processor processor, + std::vector> *const kernel_info_list); +bool IsAtomicNode(const CNodePtr &kernel_node); +void SaveJsonInfo(const std::string &json_name, const std::string &info); +std::string GetProcessor(const AnfNodePtr &anf_node); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_COMMON_UTILS_H_ diff --git a/mindspore/ccsrc/kernel/gpu/arrays/argmax_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/argmax_gpu_kernel.cc new file mode 100644 index 00000000000..71f612d07c3 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/argmax_gpu_kernel.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/argmax_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(Argmax, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeInt32), + ArgmaxGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(Argmax, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeInt32), + ArgmaxGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/argmax_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/argmax_gpu_kernel.h new file mode 100644 index 00000000000..ccd691ae91e --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/argmax_gpu_kernel.h @@ -0,0 +1,106 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_ARGMAXGPUKERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_ARGMAXGPUKERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/argmax_impl.cuh" +namespace mindspore { +namespace kernel { +#define ARGMAX_MAX_DIMENSION 2 +template +class ArgmaxGpuKernel : public GpuKernel { + public: + ArgmaxGpuKernel() : input_size_(0), output_size_(0), workspace_size_(0), batch_size_(0), channel_size_(0), axis_(0) {} + ~ArgmaxGpuKernel() override = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *input = GetDeviceAddress(inputs, 0); + int *output = GetDeviceAddress(outputs, 0); + CalArgmax(input, SizeToInt(batch_size_), SizeToInt(channel_size_), axis_, output, + reinterpret_cast(stream_ptr)); + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but argmax needs 1 input."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but argmax needs 1 output."; + return false; + } + auto output_type = GetValue(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("output_type")); + if (output_type->type_id() != TypeId::kNumberTypeInt32) { + MS_LOG(EXCEPTION) << "Argmax only supports int32 output type."; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + if (input_shape.size() > ARGMAX_MAX_DIMENSION) { + MS_LOG(EXCEPTION) << "Input is " << input_shape.size() << "-D, but argmax supports max " << ARGMAX_MAX_DIMENSION + << "-D inputs."; + } + + axis_ = GetAttr(kernel_node, "axis"); + if (axis_ < 0) { + axis_ += SizeToInt(input_shape.size()); + } + if (input_shape.size() == 1) { + batch_size_ = 0; + channel_size_ = input_shape[0]; + input_size_ = sizeof(T) * channel_size_; + output_size_ = sizeof(int); + } else { + batch_size_ = input_shape[0]; + channel_size_ = input_shape[1]; + input_size_ = sizeof(T) * batch_size_ * channel_size_; + output_size_ = (axis_ == 1) ? sizeof(int) * batch_size_ : sizeof(int) * channel_size_; + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + } + + private: + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + size_t batch_size_; + size_t channel_size_; + int axis_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_ARGMAXGPUKERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.cc new file mode 100644 index 00000000000..f3786046243 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/array_reduce_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(ReduceMax, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ArrayReduceGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(ReduceMax, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ArrayReduceGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(ReduceMean, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ArrayReduceGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(ReduceMean, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ArrayReduceGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(ReduceSum, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ArrayReduceGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(ReduceSum, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ArrayReduceGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h new file mode 100644 index 00000000000..3ab8beab0b4 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h @@ -0,0 +1,249 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_ARRAYREDUCE_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_ARRAYREDUCE_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +namespace mindspore { +namespace kernel { +const std::map kReduceTypeMap = { + {"ReduceMax", CUDNN_REDUCE_TENSOR_MAX}, + {"ReduceMean", CUDNN_REDUCE_TENSOR_AVG}, + {"ReduceSum", CUDNN_REDUCE_TENSOR_ADD}, +}; +template +class ArrayReduceGpuKernel : public GpuKernel { + public: + ArrayReduceGpuKernel() + : cudnn_handle_(nullptr), + reduce_tensor_op_(CUDNN_REDUCE_TENSOR_ADD), + data_type_(CUDNN_DATA_FLOAT), + nan_prop_(CUDNN_NOT_PROPAGATE_NAN), + reduce_indices_(CUDNN_REDUCE_TENSOR_NO_INDICES), + reduce_tensor_descriptor_(nullptr), + inputA_descriptor_(nullptr), + outputC_descriptor_(nullptr), + keep_dims_(false), + is_reduce_dim_one_(true), + is_null_input_(false), + input_size_(0), + output_size_(0), + workspace_size_(0) {} + ~ArrayReduceGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *input_addr = GetDeviceAddress(inputs, 0); + T *output_addr = GetDeviceAddress(outputs, 0); + T *workspace_addr = GetDeviceAddress(workspace, 0); + + const float alpha = 1; + const float beta = 0; + if (is_reduce_dim_one_) { + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(output_addr, input_addr, inputs[0]->size, cudaMemcpyDeviceToDevice, + reinterpret_cast(stream_ptr)), + "cudaMemcpyAsync failed in ArrayReduceGpuKernel::Launch."); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnReduceTensor(cudnn_handle_, reduce_tensor_descriptor_, nullptr, 0, workspace_addr, workspace_size_, &alpha, + inputA_descriptor_, input_addr, &beta, outputC_descriptor_, output_addr), + "cudnnReduceTensor failed."); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but reduce op needs 1 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but reduce op needs 1 output."; + return false; + } + int input_dim_length = SizeToInt(AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0).size()); + + if (AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa()) { + auto attr_axis = GetAttr>(kernel_node, "axis"); + if (attr_axis.empty()) { + axis_.push_back(-1); + } else { + for (auto axis : attr_axis) { + axis < 0 ? axis_.push_back(axis + input_dim_length) : axis_.push_back(axis); + } + } + } else if (AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa()) { + int axis = GetAttr(kernel_node, "axis"); + axis < 0 ? axis_.push_back(axis + input_dim_length) : axis_.push_back(axis); + } else { + MS_LOG(EXCEPTION) << "Attribute axis type is invalid."; + } + keep_dims_ = GetAttr(kernel_node, "keep_dims"); + + auto inputA_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto outputC_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(inputA_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "ArrayReduceGpuKernel input is null"; + InitSizeLists(); + return true; + } + InferInAndOutDesc(inputA_shape, outputC_shape); + InferArrayReduceType(kernel_node); + + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateReduceTensorDescriptor(&reduce_tensor_descriptor_), + "cudnnCreateReduceTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&inputA_descriptor_), + "cudnnCreateTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&outputC_descriptor_), + "cudnnCreateTensorDescriptor failed."); + } + void InitSizeLists() override { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(inputA_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed."); + input_size_list_.push_back(input_size_); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(outputC_descriptor_, &output_size_), + "cudnnGetTensorSizeInBytes failed."); + output_size_list_.push_back(output_size_); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetReductionWorkspaceSize(cudnn_handle_, reduce_tensor_descriptor_, inputA_descriptor_, outputC_descriptor_, + &workspace_size_), + "cudnnGetReductionWorkspaceSize failed."); + workspace_size_list_.push_back(workspace_size_); + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyReduceTensorDescriptor(reduce_tensor_descriptor_), + "cudnnDestroyReduceTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(inputA_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(outputC_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + } + void InferArrayReduceType(const CNodePtr &kernel_node) { + std::string kernel_name = AnfAlgo::GetCNodeName(kernel_node); + auto iter = kReduceTypeMap.find(kernel_name); + if (iter == kReduceTypeMap.end()) { + MS_LOG(EXCEPTION) << "Array reduce kernel type " << kernel_name << " is not supported."; + } else { + reduce_tensor_op_ = iter->second; + } + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetReduceTensorDescriptor(reduce_tensor_descriptor_, reduce_tensor_op_, CUDNN_DATA_FLOAT, nan_prop_, + reduce_indices_, CUDNN_32BIT_INDICES), + "cudnnSetReduceTensorDescriptor failed"); + return; + } + void InferInAndOutDesc(const std::vector &input_shape, const std::vector &output_shape) { + std::vector inputA_shape = input_shape; + std::vector outputC_shape = output_shape; + int shapeA_n, shapeA_c, shapeA_h, shapeA_w; + shapeA_n = inputA_shape.size() < 4 ? 1 : SizeToInt(inputA_shape[inputA_shape.size() - 4]); + shapeA_c = inputA_shape.size() < 3 ? 1 : SizeToInt(inputA_shape[inputA_shape.size() - 3]); + shapeA_h = inputA_shape.size() < 2 ? 1 : SizeToInt(inputA_shape[inputA_shape.size() - 2]); + shapeA_w = inputA_shape.size() == 0 ? 1 : SizeToInt(inputA_shape[inputA_shape.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(inputA_descriptor_, CUDNN_TENSOR_NCHW, data_type_, shapeA_n, + shapeA_c, shapeA_h, shapeA_w), + "cudnnSetTensor4dDescriptor failed"); + + int shapeC_n, shapeC_c, shapeC_h, shapeC_w; + if (axis_[0] == -1) { + shapeC_n = 1; + shapeC_c = 1; + shapeC_h = 1; + shapeC_w = 1; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(outputC_descriptor_, CUDNN_TENSOR_NCHW, data_type_, + shapeC_n, shapeC_c, shapeC_h, shapeC_w), + "cudnnSetTensor4dDescriptor failed"); + is_reduce_dim_one_ = false; + return; + } + + if (!keep_dims_) { + for (auto i : axis_) { + (void)(outputC_shape.insert(outputC_shape.begin() + i, 1)); + } + } + for (auto i : axis_) { + if (inputA_shape[IntToSize(i)] != 1) { + // To avoid cudnnReduceTensor bug when the dimension which needs to be + // reduced is already 1. + is_reduce_dim_one_ = false; + } + } + + shapeC_n = outputC_shape.size() < 4 ? 1 : SizeToInt(outputC_shape[outputC_shape.size() - 4]); + shapeC_c = outputC_shape.size() < 3 ? 1 : SizeToInt(outputC_shape[outputC_shape.size() - 3]); + shapeC_h = outputC_shape.size() < 2 ? 1 : SizeToInt(outputC_shape[outputC_shape.size() - 2]); + shapeC_w = SizeToInt(outputC_shape[outputC_shape.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(outputC_descriptor_, CUDNN_TENSOR_NCHW, data_type_, shapeC_n, + shapeC_c, shapeC_h, shapeC_w), + "cudnnSetTensor4dDescriptor failed"); + return; + } + + cudnnHandle_t cudnn_handle_; + cudnnReduceTensorOp_t reduce_tensor_op_; + cudnnDataType_t data_type_; + cudnnNanPropagation_t nan_prop_; + cudnnReduceTensorIndices_t reduce_indices_; + cudnnReduceTensorDescriptor_t reduce_tensor_descriptor_; + cudnnTensorDescriptor_t inputA_descriptor_; + cudnnTensorDescriptor_t outputC_descriptor_; + + std::vector axis_; + bool keep_dims_; + bool is_reduce_dim_one_; + bool is_null_input_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_ARRAYREDUCE_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/arrays/concatv2_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/concatv2_gpu_kernel.cc new file mode 100644 index 00000000000..af86ff8e9bd --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/concatv2_gpu_kernel.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/concatv2_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + Concat, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ConcatV2GpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + Concat, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + ConcatV2GpuFwdKernel, int) +MS_REG_GPU_KERNEL_ONE( + Concat, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ConcatV2GpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/concatv2_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/concatv2_gpu_kernel.h new file mode 100644 index 00000000000..eba6bb87f0e --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/concatv2_gpu_kernel.h @@ -0,0 +1,112 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CONCATV2_GPU_KERNEL_H +#define MINDSPORE_CCSRC_KERNEL_GPU_CONCATV2_GPU_KERNEL_H + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/concatv2_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class ConcatV2GpuFwdKernel : public GpuKernel { + public: + ConcatV2GpuFwdKernel() : axis_(0), input0_size_(0), input1_size_(0), output_size_(0), workspace_size_(0) {} + ~ConcatV2GpuFwdKernel() override = default; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *input_0 = GetDeviceAddress(inputs, 0); + T *input_1 = GetDeviceAddress(inputs, 1); + T *output = GetDeviceAddress(outputs, 0); + + CalConcatV2(output_size_ / sizeof(T), w_[0], w_[1], input_0, input_1, output, + reinterpret_cast(stream_ptr)); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + if (!CheckParam(kernel_node)) { + return false; + } + + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + input0_size_ = sizeof(T); + for (size_t i = 0; i < input_shape.size(); i++) { + input0_size_ *= input_shape[i]; + } + auto input_shape1 = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + input1_size_ = sizeof(T); + for (size_t i = 0; i < input_shape1.size(); i++) { + input1_size_ *= input_shape1[i]; + } + output_size_ = input0_size_ + input1_size_; + axis_ = GetAttr(kernel_node, "axis"); + if (axis_ < 0) { + axis_ += SizeToInt(input_shape.size()); + } + w_[0] = 1; + w_[1] = 1; + for (size_t i = IntToSize(axis_); i < input_shape.size(); i++) { + w_[0] *= SizeToInt(input_shape[i]); + w_[1] *= SizeToInt(input_shape1[i]); + } + + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input0_size_); + input_size_list_.push_back(input1_size_); + output_size_list_.push_back(output_size_); + } + + private: + bool CheckParam(const CNodePtr &kernel_node) { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but ConcatV2GpuFwdKernel needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but ConcatV2GpuFwdKernel needs 1 output."; + return false; + } + return true; + } + int w_[2] = {1}; + int axis_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input0_size_; + size_t input1_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CONCATV2_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/arrays/gather_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/gather_gpu_kernel.cc new file mode 100644 index 00000000000..dc595e47936 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/gather_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/gather_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_TWO( + GatherV2, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeFloat32), + GatherGpuFwdKernel, float, int) +MS_REG_GPU_KERNEL_TWO( + GatherV2, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeFloat16), + GatherGpuFwdKernel, half, int) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/gather_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/gather_gpu_kernel.h new file mode 100644 index 00000000000..c4424df59c6 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/gather_gpu_kernel.h @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_GATHER_GPU_KERNEL_H +#define MINDSPORE_GATHER_GPU_KERNEL_H + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/gather.cuh" + +namespace mindspore { +namespace kernel { +template +class GatherGpuFwdKernel : public GpuKernel { + public: + GatherGpuFwdKernel() : axis_(0), handle_(nullptr) {} + ~GatherGpuFwdKernel() = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + T *input_addr = GetDeviceAddress(inputs, 0); + S *indices_addr = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + + auto input_dim1 = input_shapes_[IntToSize(axis_)]; + Gather(input_addr, indices_addr, output_addr, dims_[0], dims_[1], dims_[2], input_dim1, + reinterpret_cast(stream_ptr)); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but GatherGpuFwdKernel needs 2."; + } + input_shapes_ = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + indices_shapes_ = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + output_shapes_ = AnfAlgo::GetOutputInferShape(kernel_node, 0); + + axis_ = GetAttr(kernel_node, "axis"); + if (axis_ < 0) { + axis_ = axis_ + SizeToInt(input_shapes_.size()); + } + + Reshape(); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); } + void InitSizeLists() override { + size_t size = GetSize(input_shapes_); + input_size_list_.push_back(size); + + size = GetSize(indices_shapes_); + input_size_list_.push_back(size); + + size = GetSize(output_shapes_); + output_size_list_.push_back(size); + } + + private: + void Reshape() { + size_t dim_before_axis = 1; + for (size_t i = 0; i < IntToSize(axis_); i++) { + dim_before_axis *= output_shapes_[i]; + } + + size_t dim_of_indices = 1; + for (size_t i = 0; i < indices_shapes_.size(); i++) { + dim_of_indices *= indices_shapes_[i]; + } + + size_t dim_after_indices = 1; + for (size_t i = IntToSize(axis_) + indices_shapes_.size(); i < output_shapes_.size(); i++) { + dim_after_indices *= output_shapes_[i]; + } + + dims_[0] = dim_before_axis; + dims_[1] = dim_of_indices; + dims_[2] = dim_after_indices; + return; + } + size_t GetSize(const std::vector &shape) const { + if (shape.size() == 0) { + return 0; + } + size_t result = sizeof(T); + for (size_t i = 0; i < shape.size(); i++) { + result *= shape[i]; + } + return result; + } + + std::vector input_shapes_; + std::vector indices_shapes_; + std::vector output_shapes_; + + size_t dims_[3] = {}; + int axis_; + cudnnHandle_t handle_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_GATHER_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/arrays/one_hot_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/one_hot_gpu_kernel.cc new file mode 100644 index 00000000000..7c160f8f584 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/one_hot_gpu_kernel.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/one_hot_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_TWO(OneHot, + KernelAttr() + .AddInputAttr(kNumberTypeInt32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + OneHotGpuFwdKernel, float, int) +MS_REG_GPU_KERNEL_TWO(OneHot, + KernelAttr() + .AddInputAttr(kNumberTypeInt32) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + OneHotGpuFwdKernel, half, int) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/one_hot_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/one_hot_gpu_kernel.h new file mode 100644 index 00000000000..d8059869f29 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/one_hot_gpu_kernel.h @@ -0,0 +1,105 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_ONEHOT_GPU_KERNEL_H +#define MINDSPORE_CCSRC_KERNEL_GPU_ONEHOT_GPU_KERNEL_H + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/one_hot_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class OneHotGpuFwdKernel : public GpuKernel { + public: + OneHotGpuFwdKernel() : input_size_(1), output_size_(1), depth_(0), left_dim_size_(1), right_dim_size_(1) {} + ~OneHotGpuFwdKernel() = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + const S *indices = GetDeviceAddress(inputs, 0); + const T *on_value = GetDeviceAddress(inputs, 1); + const T *off_value = GetDeviceAddress(inputs, 2); + T *output = GetDeviceAddress(outputs, 0); + OneHot(indices, depth_, on_value, off_value, left_dim_size_, right_dim_size_, output, + reinterpret_cast(stream_ptr)); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + int axis = GetAttr(kernel_node, "axis"); + auto input = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto output = AnfAlgo::GetOutputInferShape(kernel_node, 0); + int input_size = SizeToInt(input.size()); + const int default_axis = -1; + + // Compress arbitrary tensor dimensions into three dimensions (left_dims, depth, right_dims). + for (int i = 0; i < input_size; i++) { + auto dim_size = input[IntToSize(i)]; + if (axis == default_axis || i < axis) { + left_dim_size_ *= dim_size; + } + if (axis != default_axis && i >= axis) { + right_dim_size_ *= dim_size; + } + } + for (auto size : input) { + input_size_ *= size; + } + for (auto size : output) { + output_size_ *= size; + } + if (axis >= input_size) { + MS_LOG(ERROR) << "invalid one hot axis value: " << axis << " for input dims size: " << input.size(); + return false; + } + if (axis == default_axis) { + depth_ = output[output.size() - 1]; + } else { + depth_ = output[IntToSize(axis)]; + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + // inputs: indices, depth + input_size_list_.push_back((input_size_ + 1) * sizeof(S)); + output_size_list_.push_back(output_size_ * sizeof(T)); + } + + private: + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input_size_; + size_t output_size_; + + size_t depth_; + size_t left_dim_size_; + size_t right_dim_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_ONEHOT_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/arrays/slice_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/slice_gpu_kernel.cc new file mode 100644 index 00000000000..53161c29c2a --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/slice_gpu_kernel.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/slice_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(Slice, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + SliceGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(Slice, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + SliceGpuFwdKernel, int) +MS_REG_GPU_KERNEL_ONE(Slice, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + SliceGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(StridedSlice, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + SliceGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(StridedSlice, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + SliceGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(StridedSlice, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + SliceGpuFwdKernel, int) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/slice_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/slice_gpu_kernel.h new file mode 100644 index 00000000000..96e899da60b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/slice_gpu_kernel.h @@ -0,0 +1,157 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_SLICE_GPU_KERNEL_H +#define MINDSPORE_CCSRC_KERNEL_GPU_SLICE_GPU_KERNEL_H + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/slice_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class SliceGpuFwdKernel : public GpuKernel { + public: + SliceGpuFwdKernel() : is_strided_slice_(false), input_size_(0), output_size_(0), workspace_size_(0) {} + ~SliceGpuFwdKernel() override = default; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + if (is_strided_slice_) { + CalStridedSlice(output_size_ / sizeof(T), input, input_shape_, begin_, size_, strides_, output, + reinterpret_cast(stream_ptr)); + } else { + CalSlice(output_size_ / sizeof(T), input, input_shape_, begin_, size_, output, + reinterpret_cast(stream_ptr)); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + if (!CheckParam(kernel_node)) { + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + int shape_n = input_shape.size() < 4 ? 1 : SizeToInt(input_shape[input_shape.size() - 4]); + int shape_c = input_shape.size() < 3 ? 1 : SizeToInt(input_shape[input_shape.size() - 3]); + int shape_h = input_shape.size() < 2 ? 1 : SizeToInt(input_shape[input_shape.size() - 2]); + int shape_w = SizeToInt(input_shape[input_shape.size() - 1]); + input_shape_.push_back(shape_n); + input_shape_.push_back(shape_c); + input_shape_.push_back(shape_h); + input_shape_.push_back(shape_w); + + auto strides = AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("strides"); + if (strides) { + strides_ = GetAttr>(kernel_node, "strides"); + for (auto i = strides_.size(); i < 4; i++) { + (void)strides_.insert(strides_.begin(), 1); + } + size_ = GetAttr>(kernel_node, "end"); + is_strided_slice_ = true; + } else { + size_ = GetAttr>(kernel_node, "size"); + } + for (auto i = begin_.size(); i < 4; i++) { + (void)begin_.insert(begin_.begin(), 0); + } + for (size_t i = size_.size(); i < 4; i++) { + (void)size_.insert(size_.begin(), 1); + } + for (size_t i = 0; i < begin_.size(); i++) { + if (begin_[i] < 0) { + begin_[i] = begin_[i] + input_shape_[i]; + } + } + for (size_t i = 0; i < size_.size(); i++) { + if (size_[i] < 0) { + size_[i] = (size_[i] + input_shape_[i]) > 0 ? (size_[i] + input_shape_[i]) : 0; + } + } + + input_size_ = IntToSize(shape_n * shape_c * shape_h * shape_w) * sizeof(T); + auto out_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + + output_size_ = sizeof(T); + for (size_t x : out_shape) { + output_size_ = output_size_ * x; + } + + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + } + + private: + bool CheckParam(const CNodePtr &kernel_node) { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but SliceGpuFwdKernel needs 1 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but SliceGpuFwdKernel needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + if (input_shape.size() > 4) { + MS_LOG(ERROR) << "Input dims is " << input_shape.size() << ", but SliceGpuFwdKernel olny support 4d or lower."; + return false; + } + if (input_shape.size() == 0) { + MS_LOG(ERROR) << "Input dims is " << input_shape.size() << ", scalar is not supported."; + return false; + } + begin_ = GetAttr>(kernel_node, "begin"); + for (size_t i = 0; i < input_shape.size(); i++) { + if ((begin_[i] > 0 && (begin_[i] >= SizeToInt(input_shape[i]))) || + (begin_[i] < 0 && (std::abs(begin_[i]) > SizeToInt(input_shape[i])))) { + MS_LOG(ERROR) << "Error input, out of bounds " << input_shape[i] << " in axis " << i << "."; + return false; + } + } + return true; + } + std::vector begin_; + std::vector size_; + std::vector strides_; + std::vector input_shape_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + bool is_strided_slice_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_SLICE_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/arrays/slice_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/slice_grad_gpu_kernel.cc new file mode 100644 index 00000000000..b91aafb7341 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/slice_grad_gpu_kernel.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/slice_grad_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + SliceGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + SliceGradGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + SliceGrad, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + SliceGradGpuKernel, int) +MS_REG_GPU_KERNEL_ONE( + SliceGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + SliceGradGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(StridedSliceGrad, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + SliceGradGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(StridedSliceGrad, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + SliceGradGpuKernel, int) +MS_REG_GPU_KERNEL_ONE(StridedSliceGrad, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + SliceGradGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/slice_grad_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/slice_grad_gpu_kernel.h new file mode 100644 index 00000000000..9660e1dd9b3 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/slice_grad_gpu_kernel.h @@ -0,0 +1,154 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_SLICE_GRAD_GPU_KERNEL_H +#define MINDSPORE_CCSRC_KERNEL_GPU_SLICE_GRAD_GPU_KERNEL_H + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/slice_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class SliceGradGpuKernel : public GpuKernel { + public: + SliceGradGpuKernel() : is_strided_slice_(false), input_size_(0), output_size_(0), workspace_size_(0) {} + ~SliceGradGpuKernel() override = default; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *dy = GetDeviceAddress(inputs, 0); + T *dx = GetDeviceAddress(outputs, 0); + FillDeviceArray(outputs[0]->size / sizeof(T), dx, 0.f, reinterpret_cast(stream_ptr)); + if (is_strided_slice_) { + CalStridedSliceGrad(output_size_ / sizeof(T), dy, input_shape_, begin_, size_, strides_, dx, + reinterpret_cast(stream_ptr)); + } else { + CalSliceGrad(output_size_ / sizeof(T), dy, input_shape_, begin_, size_, dx, + reinterpret_cast(stream_ptr)); + } + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + if (!CheckParam(kernel_node)) { + return false; + } + auto kernel_name = AnfAlgo::GetCNodeName(kernel_node); + if (kernel_name == "StridedSliceGrad") { + is_strided_slice_ = true; + input_shape_ = GetAttr>(kernel_node, "shapex"); + for (auto i = input_shape_.size(); i < 4; i++) { + (void)input_shape_.insert(input_shape_.begin(), 1); + } + strides_ = GetAttr>(kernel_node, "strides"); + for (auto i = strides_.size(); i < 4; i++) { + (void)strides_.insert(strides_.begin(), 1); + } + size_ = GetAttr>(kernel_node, "end"); + } else { + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + input_shape_.push_back(input_shape.size() < 4 ? 1 : SizeToInt(input_shape[input_shape.size() - 4])); + input_shape_.push_back(input_shape.size() < 3 ? 1 : SizeToInt(input_shape[input_shape.size() - 3])); + input_shape_.push_back(input_shape.size() < 2 ? 1 : SizeToInt(input_shape[input_shape.size() - 2])); + input_shape_.push_back(SizeToInt(input_shape[input_shape.size() - 1])); + size_ = GetAttr>(kernel_node, "size"); + } + + auto dy_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + dy_shape_.push_back(dy_shape.size() < 4 ? 1 : SizeToInt(dy_shape[dy_shape.size() - 4])); + dy_shape_.push_back(dy_shape.size() < 3 ? 1 : SizeToInt(dy_shape[dy_shape.size() - 3])); + dy_shape_.push_back(dy_shape.size() < 2 ? 1 : SizeToInt(dy_shape[dy_shape.size() - 2])); + dy_shape_.push_back(SizeToInt(dy_shape[dy_shape.size() - 1])); + + begin_ = GetAttr>(kernel_node, "begin"); + DealParam(); + input_size_ = IntToSize(input_shape_[0] * input_shape_[1] * input_shape_[2] * input_shape_[3]) * sizeof(T); + + output_size_ = sizeof(T); + for (auto x : dy_shape_) { + output_size_ = output_size_ * IntToSize(x); + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(output_size_); + input_size_list_.push_back(input_size_); + output_size_list_.push_back(input_size_); + } + + private: + bool CheckParam(const CNodePtr &kernel_node) { + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but SliceGradGpuKernel needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + if (input_shape.size() > 4) { + MS_LOG(ERROR) << "Input dims is " << input_shape.size() << ", but SliceGradGpuKernel only support 4d or lower."; + return false; + } + if (input_shape.size() == 0) { + MS_LOG(ERROR) << "Input dims is " << input_shape.size() << ", scalar is not supported."; + return false; + } + return true; + } + void DealParam() { + for (auto i = begin_.size(); i < 4; i++) { + (void)begin_.insert(begin_.begin(), 0); + } + for (auto i = size_.size(); i < 4; i++) { + (void)size_.insert(size_.begin(), 1); + } + for (size_t i = 0; i < begin_.size(); i++) { + if (begin_[i] < 0) { + begin_[i] = begin_[i] + input_shape_[i]; + } + } + for (size_t i = 0; i < size_.size(); i++) { + if (size_[i] < 0) { + size_[i] = (size_[i] + input_shape_[i]) > 0 ? (size_[i] + input_shape_[i]) : 0; + } + } + } + std::vector begin_; + std::vector size_; + std::vector strides_; + std::vector input_shape_; + std::vector dy_shape_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + bool is_strided_slice_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_SLICE_GRAD_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/arrays/transpose_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/arrays/transpose_gpu_kernel.cc new file mode 100644 index 00000000000..338e7a4093b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/transpose_gpu_kernel.cc @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/arrays/transpose_gpu_kernel.h" +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(Transpose, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + TransposeGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(Transpose, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + TransposeGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/arrays/transpose_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/transpose_gpu_kernel.h new file mode 100644 index 00000000000..198e8687fcb --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/arrays/transpose_gpu_kernel.h @@ -0,0 +1,111 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_TRANSPOSE_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_TRANSPOSE_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/transpose_impl.cuh" +namespace mindspore { +namespace kernel { +template +class TransposeGpuFwdKernel : public GpuKernel { + public: + TransposeGpuFwdKernel() : shape_size_(0), input_size_(0), output_size_(0), workspace_size_(0) {} + ~TransposeGpuFwdKernel() = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + int *input_shape = GetDeviceAddress(workspace, 0); + int *input_axis = GetDeviceAddress(workspace, 1); + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(input_shape, &input_shape_[0], workspace_size_, cudaMemcpyHostToDevice, + reinterpret_cast(stream_ptr)), + "cudaMemcpyAsync input_shape failed"); + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(input_axis, &input_axis_[0], workspace_size_, cudaMemcpyHostToDevice, + reinterpret_cast(stream_ptr)), + "cudaMemcphalfyAsync input_axis failed"); + int size = SizeToInt(input_size_ / sizeof(T)); + CalTranspose(size, input, input_shape, input_axis, SizeToInt(shape_size_), output, + reinterpret_cast(stream_ptr)); + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but transpose needs 1 input."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but transpose needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + shape_size_ = input_shape.size(); + if (shape_size_ > TRANSPOSE_MAX_DIMENSION) { + MS_LOG(EXCEPTION) << "Input is " << shape_size_ << "-D, but transpose supports max " << TRANSPOSE_MAX_DIMENSION + << "-D inputs."; + } + + input_size_ = 1; + for (size_t i = 0; i < shape_size_; i++) { + input_size_ *= input_shape[i]; + input_shape_.push_back(input_shape[i]); + } + input_size_ *= sizeof(T); + output_size_ = input_size_; + auto perm = GetAttr>(kernel_node, "perm"); + for (size_t j = 0; j < perm.size(); j++) { + input_axis_.push_back(perm[j]); + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + workspace_size_ = shape_size_ * sizeof(int); + workspace_size_list_.push_back(workspace_size_); + workspace_size_list_.push_back(workspace_size_); + return; + } + + private: + std::vector input_shape_; + std::vector input_axis_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + size_t shape_size_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_TRANSPOSE_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmax_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmax_impl.cu new file mode 100755 index 00000000000..e8fab27dda3 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmax_impl.cu @@ -0,0 +1,88 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "argmax_impl.cuh" +#include "device/gpu/cuda_common.h" +#include "include/cuda_fp16.h" +template +__global__ void Argmax1D(const T* input, const int channel_size, int* output) { + int max_index = 0; + T max = input[0]; + for (int pos = 1; pos < channel_size; pos++) { + if (max < input[pos]) { + max = input[pos]; + max_index = pos; + } + } + output[0] = max_index; + return; +} +template +__global__ void ArgmaxDefault2D(const T* input, const int batch_size, const int channel_size, int* output) { + int pos; + int max_index; + T max; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < batch_size; i += blockDim.x * gridDim.x) { + max = input[i * channel_size]; + max_index = 0; + for (int j = 1; j < channel_size; j++) { + pos = i * channel_size + j; + if (max < input[pos]) { + max = input[pos]; + max_index = j; + } + } + + output[i] = max_index; + } + return; +} +template +__global__ void ArgmaxAxis2D(const T* input, const int batch_size, const int channel_size, int* output) { + int pos; + int max_index; + T max; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < channel_size; i += blockDim.x * gridDim.x) { + max = input[i]; + max_index = 0; + for (int j = 1; j < batch_size; j++) { + pos = j * channel_size + i; + if (max < input[pos]) { + max = input[pos]; + max_index = j; + } + } + output[i] = max_index; + } + return; +} +template +void CalArgmax(const T* input, const int batch_size, const int channel_size, const int axis, int* output, + cudaStream_t cuda_stream) { + if (batch_size == 0) { + Argmax1D<<<1, 1, 0, cuda_stream>>>(input, channel_size, output); + } else if (axis == 1) { + ArgmaxDefault2D<<>>(input, batch_size, channel_size, output); + } else { + ArgmaxAxis2D<<>>(input, batch_size, channel_size, output); + } + return; +} + +template void CalArgmax(const float* input, const int batch_size, const int channel_size, const int axis, + int* output, cudaStream_t cuda_stream); +template void CalArgmax(const half* input, const int batch_size, const int channel_size, const int axis, + int* output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmax_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmax_impl.cuh new file mode 100755 index 00000000000..1e90f133ff8 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmax_impl.cuh @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAX_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAX_H_ +template +void CalArgmax(const T* input, const int batch_size, const int channel_size, const int axis, int* output, + cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAX_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/assign_add_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/assign_add_impl.cu new file mode 100644 index 00000000000..d44ad99202e --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/assign_add_impl.cu @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "assign_add_impl.cuh" +#include "device/gpu/cuda_common.h" +#include "include/cuda_fp16.h" +template +__global__ void AssignAdd(const size_t size, T* ref, const T* value, T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (size); pos += blockDim.x * gridDim.x) { + output[pos] = ref[pos] + value[pos]; + ref[pos] = output[pos]; + } + return; +} + +template +void CalAssignAdd(const size_t size, T* ref, const T* value, T* output, cudaStream_t cuda_stream) { + AssignAdd<<>>(size, ref, value, output); + + return; +} + +template void CalAssignAdd(const size_t size, float* ref, const float* value, float* output, + cudaStream_t cuda_stream); +template void CalAssignAdd(const size_t size, half* ref, const half* value, half* output, + cudaStream_t cuda_stream); +template void CalAssignAdd(const size_t size, int* ref, const int* value, int* output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/assign_add_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/assign_add_impl.cuh new file mode 100644 index 00000000000..b095384aced --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/assign_add_impl.cuh @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ASSIGNADD_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ASSIGNADD_H_ +template +void CalAssignAdd(const size_t size, T* ref, const T* value, T* output, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ASSIGNADD_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/concatv2_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/concatv2_impl.cu new file mode 100755 index 00000000000..ed330f6e0ac --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/concatv2_impl.cu @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "kernel/gpu/cuda_impl/concatv2_impl.cuh" +template +__global__ void ConcatV2(const size_t size, const int w1, const int w2, const T* input_1, const T* input_2, T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (size); pos += blockDim.x * gridDim.x) { + int n = pos / (w1 + w2); + int m = pos % (w1 + w2); + output[pos] = m >= w1 ? input_2[n * w2 + m - w1] : input_1[n * w1 + m]; + } + return; +} + +template +void CalConcatV2(const size_t size, const int w1, const int w2, const T* input_1, const T* input_2, T* output, + cudaStream_t cuda_stream) { + ConcatV2<<>>(size, w1, w2, input_1, input_2, output); + return; +} + +template void CalConcatV2(const size_t size, const int w1, const int w2, const float* input_1, const float* input_2, + float* output, cudaStream_t cuda_stream); +template void CalConcatV2(const size_t size, const int w1, const int w2, const int* input_1, const int* input_2, + int* output, cudaStream_t cuda_stream); +template void CalConcatV2(const size_t size, const int w1, const int w2, const half* input_1, const half* input_2, + half* output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/concatv2_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/concatv2_impl.cuh new file mode 100755 index 00000000000..5cbf61205b0 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/concatv2_impl.cuh @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_CONCATV2IMPL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_CONCATV2IMPL_H_ + +#include "device/gpu/cuda_common.h" +template +void CalConcatV2(const size_t size, const int w1, const int w2, const T* input_1, const T* input_2, T* output, + cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_CONCATV2IMPL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/cross_entropy_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/cross_entropy_impl.cu new file mode 100644 index 00000000000..4d0503ba97d --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/cross_entropy_impl.cu @@ -0,0 +1,133 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "cross_entropy_impl.cuh" + +template +__global__ void CrossEntropyWithSparseKernel(const T *logits, const S *labels, const size_t batch_size, + const size_t class_num, T *loss) { + double total_loss = 0.0; + T epsilon = 1e-6; + for (size_t i = 0; i < batch_size; ++i) { + T logit = logits[i * class_num + labels[i]]; + if (logit <= 0) { + logit += epsilon; + } + total_loss += -logf(logit); + } + total_loss /= batch_size; + loss[0] = static_cast(total_loss); + return; +} + +template +__global__ void CrossEntropyGradWithSparseKernel(const T *logits, const S *labels, const size_t batch_size, + const size_t class_num, T *grad) { + for (size_t i = 0; i < batch_size; i++) { + for (size_t j = blockIdx.x * blockDim.x + threadIdx.x; j < class_num; j += blockDim.x * gridDim.x) { + if (labels[i] == j) { + grad[i * class_num + j] = (logits[i * class_num + j] - 1) / batch_size; + } else { + grad[i * class_num + j] = logits[i * class_num + j] / batch_size; + } + } + } + return; +} + +template +__global__ void CrossEntropyWithoutSparseKernel(const T *logits, const S *labels, const size_t batch_size, + const size_t class_num, T *losses) { + T epsilon = 1e-6; + for (size_t i = 0; i < batch_size; ++i) { + T logit = 0.0; + for (size_t j = 0; j < class_num; j++) { + if (fabs(labels[i * class_num + j] - 1.0) <= 1e-8) { + logit = logits[i * class_num + j]; + break; + } + } + if (logit <= 0) { + logit += epsilon; + } + losses[i] = -logf(logit); + } + return; +} + +template +__global__ void CrossEntropyGradWithoutSparseKernel(const T *logits, const S *labels, const size_t batch_size, + const size_t class_num, T *grad) { + for (size_t i = 0; i < batch_size; i++) { + for (size_t j = blockIdx.x * blockDim.x + threadIdx.x; j < class_num; j += blockDim.x * gridDim.x) { + if (fabs(labels[i * class_num + j] - 1.0) <= 1e-8) { + grad[i * class_num + j] = (logits[i * class_num + j] - 1) / batch_size; + } else { + grad[i * class_num + j] = logits[i * class_num + j] / batch_size; + } + } + } + return; +} + +template +void CrossEntropyWithSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, T *loss, + cudaStream_t cuda_stream) { + CrossEntropyWithSparseKernel<<<1, 1, 0, cuda_stream>>>(logits, labels, batch_size, class_num, loss); + return; +} + +template +void CrossEntropyGradWithSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, + T *grad, cudaStream_t cuda_stream) { + CrossEntropyGradWithSparseKernel<<>>(logits, labels, batch_size, + class_num, grad); + return; +} + +template +void CrossEntropyWithoutSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, + T *losses, cudaStream_t cuda_stream) { + CrossEntropyWithoutSparseKernel<<<1, 1, 0, cuda_stream>>>(logits, labels, batch_size, class_num, losses); + return; +} + +template +void CrossEntropyGradWithoutSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, + T *grad, cudaStream_t cuda_stream) { + CrossEntropyGradWithoutSparseKernel<<>>( + logits, labels, batch_size, class_num, grad); + return; +} + +template void CrossEntropyWithSparse(const float *logits, const int *labels, const size_t batch_size, + const size_t class_num, float *loss, cudaStream_t cuda_stream); +template void CrossEntropyWithSparse(const float *logits, const int64_t *labels, + const size_t batch_size, const size_t class_num, float *loss, + cudaStream_t cuda_stream); +template void CrossEntropyGradWithSparse(const float *logits, const int *labels, const size_t batch_size, + const size_t class_num, float *grad, cudaStream_t cuda_stream); +template void CrossEntropyGradWithSparse(const float *logits, const int64_t *labels, + const size_t batch_size, const size_t class_num, float *grad, + cudaStream_t cuda_stream); +template void CrossEntropyWithoutSparse(const float *logits, const float *labels, const size_t batch_size, + const size_t class_num, float *losses, cudaStream_t cuda_stream); +template void CrossEntropyGradWithoutSparse(const float *logits, const float *labels, + const size_t batch_size, const size_t class_num, float *grad, + cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/cross_entropy_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/cross_entropy_impl.cuh new file mode 100644 index 00000000000..00ec13553d3 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/cross_entropy_impl.cuh @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_CROSSENTROPY_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_CROSSENTROPY_H_ + +#include "device/gpu/cuda_common.h" + +template +void CrossEntropyWithSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, T *loss, + cudaStream_t cuda_stream); + +template +void CrossEntropyGradWithSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, + T *grad, cudaStream_t cuda_stream); + +template +void CrossEntropyWithoutSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, + T *losses, cudaStream_t cuda_stream); + +template +void CrossEntropyGradWithoutSparse(const T *logits, const S *labels, const size_t batch_size, const size_t class_num, + T *grad, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_CROSSENTROPY_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/equalcount_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/equalcount_impl.cu new file mode 100755 index 00000000000..38dd79c441d --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/equalcount_impl.cu @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "equalcount_impl.cuh" +#include "device/gpu/cuda_common.h" +template +__global__ void EqualCount(const int size, const T* input1, const T* input2, T* output) { + T equal_count = 0; + + for (int i = 0; i < size; i++) { + if (input1[i] == input2[i]) { + equal_count++; + } + } + + output[0] = equal_count; + return; +} +template +void CalEqualCount(const int size, const T* input1, const T* input2, T* output, cudaStream_t cuda_stream) { + EqualCount<<<1, 1, 0, cuda_stream>>>(size, input1, input2, output); + return; +} + +template void CalEqualCount(const int size, const int* input1, const int* input2, int* output, + cudaStream_t cuda_stream); +template void CalEqualCount(const int size, const float* input1, const float* input2, float* output, + cudaStream_t cuda_stream); +template void CalEqualCount(const int size, const half* input1, const half* input2, half* output, + cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/equalcount_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/equalcount_impl.cuh new file mode 100755 index 00000000000..ba6004da3b8 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/equalcount_impl.cuh @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_EQUALCOUNT_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_EQUALCOUNT_H_ +template +void CalEqualCount(const int size, const T* input1, const T* input2, T* output, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_EQUALCOUNT_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/gather.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/gather.cu new file mode 100755 index 00000000000..6bde359d9b0 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/gather.cu @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "kernel/gpu/cuda_impl/gather.cuh" +#include "device/gpu/cuda_common.h" +template +__global__ void GatherKernel(T *input, S *indices, T *output, size_t output_dim0, size_t output_dim1, + size_t output_dim2, size_t input_dim1) { + int num = output_dim0 * output_dim1 * output_dim2; + int i, j, k; + for (int write_index = blockIdx.x * blockDim.x + threadIdx.x; write_index < num; + write_index += blockDim.x * gridDim.x) { + i = write_index / (output_dim1 * output_dim2) % output_dim0; + j = write_index / output_dim2 % output_dim1; + k = write_index % output_dim2; + + if ((indices[j] >= 0) && (indices[j] < input_dim1)) { + int read_index = i * input_dim1 * output_dim2 + indices[j] * output_dim2 + k; + output[write_index] = input[read_index]; + } else { + output[write_index] = 0; + } + } + + return; +} +template +void Gather(T *input, S *indices, T *output, size_t output_dim0, size_t output_dim1, size_t output_dim2, + size_t input_dim1, cudaStream_t stream) { + int size = output_dim0 * output_dim1 * output_dim2; + GatherKernel<<>>(input, indices, output, output_dim0, output_dim1, + output_dim2, input_dim1); + return; +} + +template void Gather(float *input, int *indices, float *output, size_t output_dim0, size_t output_dim1, + size_t output_dim2, size_t input_dim1, cudaStream_t stream); + +template void Gather(half *input, int *indices, half *output, size_t output_dim0, size_t output_dim1, + size_t output_dim2, size_t input_dim1, cudaStream_t stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/gather.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/gather.cuh new file mode 100755 index 00000000000..dae2115a91b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/gather.cuh @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_GATHER_GPU_CU_H +#define MINDSPORE_GATHER_GPU_CU_H +template +void Gather(T *input, S *indices, T *output, size_t output_dim0, size_t output_dim1, size_t output_dim2, + size_t input_dim1, cudaStream_t stream); + +#endif diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/momentum_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/momentum_impl.cu new file mode 100755 index 00000000000..ae24a8dec9f --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/momentum_impl.cu @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "momentum_impl.cuh" +template +__global__ void MomentumUpdateVariableKernel(const size_t size, T *variable, T *accumulation, const T *learning_rate, + const T *gradient, const T *momentum) { + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (size); i += blockDim.x * gridDim.x) { + accumulation[i] = momentum[0] * accumulation[i] + gradient[i]; + variable[i] -= learning_rate[0] * accumulation[i]; + } + return; +} +template +void MomentumUpdateVariable(const size_t size, T *variable, T *accumulation, const T *learning_rate, const T *gradient, + const T *momentum, cudaStream_t cuda_stream) { + MomentumUpdateVariableKernel<<>>(size, variable, accumulation, + learning_rate, gradient, momentum); + return; +} +template void MomentumUpdateVariable(const size_t size, float *variable, float *accumulation, + const float *learning_rate, const float *gradient, const float *momentum, + cudaStream_t cuda_stream); +template void MomentumUpdateVariable(const size_t size, half *variable, half *accumulation, + const half *learning_rate, const half *gradient, const half *momentum, + cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/momentum_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/momentum_impl.cuh new file mode 100755 index 00000000000..2993e04ff38 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/momentum_impl.cuh @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_MOMENTUMIMPL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_MOMENTUMIMPL_H_ + +#include "device/gpu/cuda_common.h" +template +void MomentumUpdateVariable(const size_t size, T *variable, T *accumulation, const T *learning_rate, const T *gradient, + const T *momentum, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_MOMENTUMIMPL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/one_hot_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/one_hot_impl.cu new file mode 100644 index 00000000000..cf5dc7ecd02 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/one_hot_impl.cu @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "one_hot_impl.cuh" +#include "device/gpu/cuda_common.h" +template +__global__ void OneHotKernel(size_t size, const S *indices, size_t depth, const T *on_value, const T *off_value, + size_t left_dim_size, size_t right_dim_size, T *output) { + T on_v = *on_value; + T off_v = *off_value; + for (int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; thread_idx < size; + thread_idx += blockDim.x * gridDim.x) { + if (thread_idx < size) { + int left_idx = (thread_idx / (depth * right_dim_size)) % left_dim_size; + int d_idx = thread_idx / right_dim_size % depth; + int right_idx = thread_idx % right_dim_size; + int input_idx = left_idx * right_dim_size + right_idx; + int output_idx = left_idx * depth * right_dim_size + d_idx * right_dim_size + right_idx; + if (indices[input_idx] == d_idx) { + output[output_idx] = on_v; + } else { + output[output_idx] = off_v; + } + } + } +} +template +void OneHot(const S *indices, size_t depth, const T *on_value, const T *off_value, size_t left_dim_size, + size_t right_dim_size, T *output, cudaStream_t cuda_stream) { + size_t size = left_dim_size * depth * right_dim_size; + OneHotKernel<<>>(size, indices, depth, on_value, off_value, + left_dim_size, right_dim_size, output); + return; +} +template void OneHot(const int *indices, size_t depth, const float *on_value, const float *off_value, + size_t left_dim_size, size_t right_dim_size, float *output, cudaStream_t cuda_stream); +template void OneHot(const int *indices, size_t depth, const half *on_value, const half *off_value, + size_t left_dim_size, size_t right_dim_size, half *output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/one_hot_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/one_hot_impl.cuh new file mode 100644 index 00000000000..5b5991256ec --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/one_hot_impl.cuh @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ONE_HOT_IMPL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ONE_HOT_IMPL_H_ +template +void OneHot(const S *indices, size_t depth_, const T *on_value, const T *off_value, size_t left_dim_size, + size_t right_dim_size, T *output, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ONE_HOT_IMPL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/pad_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/pad_impl.cu new file mode 100755 index 00000000000..ddc615d94ba --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/pad_impl.cu @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "kernel/gpu/cuda_impl/pad_impl.cuh" + +template +__global__ void Pad(const size_t size, const T* input, const int num, const int channels, const int old_height, + const int old_width, const int padded_height, const int padded_width, const int pad_top, + const int pad_left, float pad_value, T* output) { + T pad_value_ = static_cast(pad_value); + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (size); pos += blockDim.x * gridDim.x) { + int block_num = pos / padded_width / padded_height; + const int padded_w = pos % padded_width; + const int padded_h = pos / padded_width % padded_height; + if (padded_h - pad_top < 0 || padded_w - pad_left < 0 || padded_h - pad_top >= old_height || + padded_w - pad_left >= old_width) { + output[pos] = pad_value_; + } else { + output[pos] = input[(block_num * old_height + padded_h - pad_top) * old_width + padded_w - pad_left]; + } + } + return; +} + +template +__global__ void PadGrad(const size_t size, const T* dy, const int num, const int channels, const int old_height, + const int old_width, const int padded_height, const int padded_width, const int pad_top, + const int pad_left, T* dx) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (size); pos += blockDim.x * gridDim.x) { + int block_num = pos / old_width / old_height; + const int padded_w = pos % old_width + pad_left; + const int padded_h = pos / old_width % old_height + pad_top; + dx[pos] = dy[(block_num * padded_height + padded_h) * padded_width + padded_w]; + } + return; +} + +template +void CalPad(const size_t size, const T* input, const int num, const int channels, const int old_height, + const int old_width, const int padded_height, const int padded_width, const int pad_top, const int pad_left, + const float pad_value, T* output, cudaStream_t cuda_stream) { + Pad<<>>(size, input, num, channels, old_height, old_width, + padded_height, padded_width, pad_top, pad_left, pad_value, + output); + return; +} + +template +void CalPadGrad(const size_t size, const T* dy, const int num, const int channels, const int old_height, + const int old_width, const int padded_height, const int padded_width, const int pad_top, + const int pad_left, T* dx, cudaStream_t cuda_stream) { + PadGrad<<>>(size, dy, num, channels, old_height, old_width, + padded_height, padded_width, pad_top, pad_left, dx); + return; +} + +template void CalPad(const size_t size, const float* input, const int num, const int channels, + const int old_height, const int old_width, const int padded_height, const int padded_width, + const int pad_top, const int pad_left, float pad_value, float* output, + cudaStream_t cuda_stream); +template void CalPadGrad(const size_t size, const float* dy, const int num, const int channels, + const int old_height, const int old_width, const int padded_height, + const int padded_width, const int pad_top, const int pad_left, float* dx, + cudaStream_t cuda_stream); +template void CalPad(const size_t size, const half* input, const int num, const int channels, + const int old_height, const int old_width, const int padded_height, const int padded_width, + const int pad_top, const int pad_left, float pad_value, half* output, + cudaStream_t cuda_stream); +template void CalPadGrad(const size_t size, const half* dy, const int num, const int channels, + const int old_height, const int old_width, const int padded_height, + const int padded_width, const int pad_top, const int pad_left, half* dx, + cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/pad_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/pad_impl.cuh new file mode 100755 index 00000000000..dc3036b8b65 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/pad_impl.cuh @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_PADIMPL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_PADIMPL_H_ +#include +#include "device/gpu/cuda_common.h" + +template +void CalPad(const size_t size, const T* input, const int num, const int channels, const int old_height, + const int old_width, const int padded_height, const int padded_width, const int pad_top, const int pad_left, + float pad_value, T* output, cudaStream_t cuda_stream); +template +void CalPadGrad(const size_t size, const T* dy, const int num, const int channels, const int old_height, + const int old_width, const int padded_height, const int padded_width, const int pad_top, + const int pad_left, T* dx, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_PADIMPL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/slice_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/slice_impl.cu new file mode 100755 index 00000000000..7f5bcdf81f6 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/slice_impl.cu @@ -0,0 +1,187 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "kernel/gpu/cuda_impl/slice_impl.cuh" + +template +__global__ void Slice(const T* input, int p, int start, int length, T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (length); pos += blockDim.x * gridDim.x) { + output[p + pos] = input[start + pos]; + } + return; +} +template +__global__ void SliceGrad(const T* dy, int p, int start, int length, T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (length); pos += blockDim.x * gridDim.x) { + output[start + pos] = dy[p + pos]; + } + return; +} +template +__global__ void StridedSlice(const T* input, int p, int start, int begin, int stride, int ended, T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < ((ended - 1 - begin) / stride) + 1; + pos += blockDim.x * gridDim.x) { + output[p + pos] = input[start + pos * stride]; + } + return; +} +template +__global__ void StridedSliceGrad(const T* dy, int p, int start, int begin, int stride, int ended, T* dx) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < ((ended - 1 - begin) / stride) + 1; + pos += blockDim.x * gridDim.x) { + dx[start + pos * stride] = dy[p + pos]; + } + return; +} +template +__global__ void FillArray(T* addr, const size_t len, const float value) { + T value_ = static_cast(value); + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < len; pos += blockDim.x * gridDim.x) { + addr[pos] = value_; + } + return; +} +template +void FillDeviceArray(const size_t input_size, T* addr, const float value, cudaStream_t cuda_stream) { + FillArray<<>>(addr, input_size, value); + return; +} +template +void CalSlice(const size_t input_size, const T* input, const std::vector in_shape, const std::vector begin, + const std::vector size, T* output, cudaStream_t cuda_stream) { + int block = in_shape[1] * in_shape[2] * in_shape[3]; + int map = in_shape[2] * in_shape[3]; + int w = in_shape[3]; + int length = size[3]; + int p = 0; + for (int i = begin[0]; i < size[0] + begin[0]; i++) { + for (int j = begin[1]; j < size[1] + begin[1]; j++) { + for (int k = begin[2]; k < size[2] + begin[2]; k++) { + Slice<<>>(input, p, i * block + j * map + k * w + begin[3], + length, output); + p = p + size[3]; + } + } + } +} +template +void CalSliceGrad(const size_t input_size, const T* dy, const std::vector in_shape, const std::vector begin, + const std::vector size, T* output, cudaStream_t cuda_stream) { + int block = in_shape[1] * in_shape[2] * in_shape[3]; + int map = in_shape[2] * in_shape[3]; + int w = in_shape[3]; + int length = size[3]; + int p = 0; + for (int i = begin[0]; i < size[0] + begin[0]; i++) { + for (int j = begin[1]; j < size[1] + begin[1]; j++) { + for (int k = begin[2]; k < size[2] + begin[2]; k++) { + SliceGrad<<>>( + dy, p, i * block + j * map + k * w + begin[3], length, output); + p = p + size[3]; + } + } + } +} +template +void CalStridedSlice(const size_t input_size, const T* input, const std::vector in_shape, + const std::vector begin, const std::vector end, const std::vector strides, + T* output, cudaStream_t cuda_stream) { + int block = in_shape[1] * in_shape[2] * in_shape[3]; + int map = in_shape[2] * in_shape[3]; + int w = in_shape[3]; + int ended = end[3]; + int p = 0; + int start = 0; + for (int i = begin[0]; i < ((end[0] > begin[0]) ? end[0] : (2 * begin[0] - end[0])); i += std::abs(strides[0])) { + for (int j = begin[1]; j < ((end[1] > begin[1]) ? end[1] : (2 * begin[1] - end[1])); j += std::abs(strides[1])) { + for (int k = begin[2]; k < ((end[2] > begin[2]) ? end[2] : (2 * begin[2] - end[2])); k += std::abs(strides[2])) { + start = (strides[0] > 0 ? i : 2 * begin[0] - i) * block + (strides[1] > 0 ? j : 2 * begin[1] - j) * map + + (strides[2] > 0 ? k : 2 * begin[2] - k) * w + begin[3]; + StridedSlice<<>>(input, p, start, begin[3], strides[3], + ended, output); + p = p + (end[3] - 1 - begin[3]) / strides[3] + 1; + } + } + } +} +template +void CalStridedSliceGrad(const size_t input_size, const T* dy, const std::vector in_shape, + const std::vector begin, const std::vector end, const std::vector strides, + T* dx, cudaStream_t cuda_stream) { + int block = in_shape[1] * in_shape[2] * in_shape[3]; + int map = in_shape[2] * in_shape[3]; + int w = in_shape[3]; + int ended = end[3]; + int p = 0; + int start = 0; + for (int i = begin[0]; i < ((end[0] > begin[0]) ? end[0] : (2 * begin[0] - end[0] + 1)); i += std::abs(strides[0])) { + for (int j = begin[1]; j < ((end[1] > begin[1]) ? end[1] : (2 * begin[1] - end[1] + 1)); + j += std::abs(strides[1])) { + for (int k = begin[2]; k < ((end[2] > begin[2]) ? end[2] : (2 * begin[2] - end[2] + 1)); + k += std::abs(strides[2])) { + start = (strides[0] > 0 ? i : 2 * begin[0] - i) * block + (strides[1] > 0 ? j : 2 * begin[1] - j) * map + + (strides[2] > 0 ? k : 2 * begin[2] - k) * w + begin[3]; + StridedSliceGrad<<>>(dy, p, start, begin[3], strides[3], + ended, dx); + p = p + (end[3] - 1 - begin[3]) / strides[3] + 1; + } + } + } +} + +template void FillDeviceArray(const size_t input_size, float* addr, const float value, cudaStream_t cuda_stream); +template void CalSlice(const size_t input_size, const float* input, const std::vector in_shape, + const std::vector begin, const std::vector size, float* output, + cudaStream_t cuda_stream); +template void CalSliceGrad(const size_t input_size, const float* dy, const std::vector in_shape, + const std::vector begin, const std::vector size, float* output, + cudaStream_t cuda_stream); +template void CalStridedSlice(const size_t input_size, const float* input, const std::vector in_shape, + const std::vector begin, const std::vector end, + const std::vector strides, float* output, cudaStream_t cuda_stream); +template void CalStridedSliceGrad(const size_t input_size, const float* dy, const std::vector in_shape, + const std::vector begin, const std::vector end, + const std::vector strides, float* dx, cudaStream_t cuda_stream); +template void FillDeviceArray(const size_t input_size, half* addr, const float value, cudaStream_t cuda_stream); +template void CalSlice(const size_t input_size, const half* input, const std::vector in_shape, + const std::vector begin, const std::vector size, half* output, + cudaStream_t cuda_stream); +template void CalSliceGrad(const size_t input_size, const half* dy, const std::vector in_shape, + const std::vector begin, const std::vector size, half* output, + cudaStream_t cuda_stream); +template void CalStridedSlice(const size_t input_size, const half* input, const std::vector in_shape, + const std::vector begin, const std::vector end, + const std::vector strides, half* output, cudaStream_t cuda_stream); +template void CalStridedSliceGrad(const size_t input_size, const half* dy, const std::vector in_shape, + const std::vector begin, const std::vector end, + const std::vector strides, half* dx, cudaStream_t cuda_stream); +template void FillDeviceArray(const size_t input_size, int* addr, const float value, cudaStream_t cuda_stream); +template void CalSlice(const size_t input_size, const int* input, const std::vector in_shape, + const std::vector begin, const std::vector size, int* output, + cudaStream_t cuda_stream); +template void CalSliceGrad(const size_t input_size, const int* dy, const std::vector in_shape, + const std::vector begin, const std::vector size, int* output, + cudaStream_t cuda_stream); +template void CalStridedSlice(const size_t input_size, const int* input, const std::vector in_shape, + const std::vector begin, const std::vector end, + const std::vector strides, int* output, cudaStream_t cuda_stream); +template void CalStridedSliceGrad(const size_t input_size, const int* dy, const std::vector in_shape, + const std::vector begin, const std::vector end, + const std::vector strides, int* dx, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/slice_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/slice_impl.cuh new file mode 100755 index 00000000000..d88ce29c513 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/slice_impl.cuh @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_SLICEIMPL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_SLICEIMPL_H_ + +#include +#include +#include "device/gpu/cuda_common.h" + +template +void CalSlice(const size_t input_size, const T* input, const std::vector in_shape, const std::vector begin, + const std::vector size, T* output, cudaStream_t cuda_stream); +template +void CalSliceGrad(const size_t input_size, const T* input, const std::vector in_shape, + const std::vector begin, const std::vector size, T* output, cudaStream_t cuda_stream); +template +void CalStridedSlice(const size_t input_size, const T* input, const std::vector in_shape, + const std::vector begin, const std::vector end, const std::vector strides, + T* output, cudaStream_t cuda_stream); +template +void CalStridedSliceGrad(const size_t input_size, const T* dy, const std::vector in_shape, + const std::vector begin, const std::vector end, const std::vector strides, + T* dx, cudaStream_t cuda_stream); +template +void FillDeviceArray(const size_t input_size, T* addr, const float value, cudaStream_t cuda_stream); +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_SLICEIMPL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/transpose_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/transpose_impl.cu new file mode 100755 index 00000000000..a0fea901360 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/transpose_impl.cu @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "transpose_impl.cuh" +#include "device/gpu/cuda_common.h" +template +__global__ void Transpose(const int size, const T* input, const int* input_shape, const int* input_axis, + const int shape_size, T* output) { + int pos_size; + int temp_pos; + int newpos; + int newpos_size; + int pos_array[TRANSPOSE_MAX_DIMENSION]; + + // for example 4-D: pos = posArray[0] * input_shape[1] * input_shape[2] * input_shape[3] + + // posArray[1] * input_shape[2] * input_shape[3] + + // posArray[2] * input_shape[3] + + // posArray[3] + for (int pos = blockIdx.x * blockDim.x + threadIdx.x; pos < size; pos += blockDim.x * gridDim.x) { + temp_pos = pos; + pos_size = size / input_shape[0]; + pos_array[0] = temp_pos / pos_size; + for (int i = 1; i < shape_size; i++) { + temp_pos -= pos_array[i - 1] * pos_size; + pos_size = pos_size / input_shape[i]; + pos_array[i] = temp_pos / pos_size; + } + + newpos = pos_array[input_axis[shape_size - 1]]; + newpos_size = 1; + for (int j = shape_size - 2; j >= 0; j--) { + newpos_size *= input_shape[input_axis[j + 1]]; + newpos += pos_array[input_axis[j]] * newpos_size; + } + + output[newpos] = input[pos]; + } + return; +} +template +void CalTranspose(const int size, const T* input, const int* input_shape, const int* input_axis, const int shape_size, + T* output, cudaStream_t cuda_stream) { + Transpose<<>>(size, input, input_shape, input_axis, shape_size, + output); + return; +} + +template void CalTranspose(const int size, const float* input, const int* input_shape, const int* input_axis, + const int shape_size, float* output, cudaStream_t cuda_stream); +template void CalTranspose(const int size, const half* input, const int* input_shape, const int* input_axis, + const int shape_size, half* output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/transpose_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/transpose_impl.cuh new file mode 100755 index 00000000000..dbf7d140eb7 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/transpose_impl.cuh @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_TRANSPOSE_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_TRANSPOSE_H_ + +#define TRANSPOSE_MAX_DIMENSION 100 +template +void CalTranspose(const int size, const T* input, const int* input_shape, const int* input_axis, const int shape_size, + T* output, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_TRANSPOSE_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu new file mode 100755 index 00000000000..3cebefec179 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "unary_op_impl.cuh" +template +__global__ void ExponentialKernel(T *input, T *output, size_t count) { + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = exp(input[i]); + } + return; +} +template <> +__global__ void ExponentialKernel(half *input, half *output, size_t count) { + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = hexp(input[i]); + } + return; +} +template +__global__ void LogarithmKernel(T *input, T *output, size_t count) { + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = logf(input[i]); + } + return; +} +template +__global__ void NegativeKernel(T *input, T *output, size_t count) { + T neg_one = -1; + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = neg_one * input[i]; + } + return; +} +template +__global__ void ReciprocalKernel(T *input, T *output, size_t count) { + T one = 1.0; + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = one / input[i]; + } + return; +} +template +void Exponential(T *input, T *output, size_t count, cudaStream_t cuda_stream) { + ExponentialKernel<<>>(input, output, count); + return; +} +template +void Logarithm(T *input, T *output, size_t count, cudaStream_t cuda_stream) { + LogarithmKernel<<>>(input, output, count); + return; +} +template +void Negative(T *input, T *output, size_t count, cudaStream_t cuda_stream) { + NegativeKernel<<>>(input, output, count); + return; +} +template +void Reciprocal(T *input, T *output, size_t count, cudaStream_t cuda_stream) { + ReciprocalKernel<<>>(input, output, count); + return; +} + +template void Exponential(float *input, float *output, size_t count, cudaStream_t cuda_stream); +template void Logarithm(float *input, float *output, size_t count, cudaStream_t cuda_stream); +template void Negative(float *input, float *output, size_t count, cudaStream_t cuda_stream); +template void Reciprocal(float *input, float *output, size_t count, cudaStream_t cuda_stream); +template void Exponential(half *input, half *output, size_t count, cudaStream_t cuda_stream); +template void Logarithm(half *input, half *output, size_t count, cudaStream_t cuda_stream); +template void Negative(half *input, half *output, size_t count, cudaStream_t cuda_stream); +template void Reciprocal(half *input, half *output, size_t count, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cuh new file mode 100755 index 00000000000..2e7227eb323 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cuh @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_UNARYOPIMPL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_UNARYOPIMPL_H_ + +#include "device/gpu/cuda_common.h" +template +void Exponential(T *input, T *output, size_t count, cudaStream_t cuda_stream); +template +void Logarithm(T *input, T *output, size_t count, cudaStream_t cuda_stream); +template +void Negative(T *input, T *output, size_t count, cudaStream_t cuda_stream); +template +void Reciprocal(T *input, T *output, size_t count, cudaStream_t cuda_stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_UNARYOPIMPL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/data/dataset_init_kernel.cc b/mindspore/ccsrc/kernel/gpu/data/dataset_init_kernel.cc new file mode 100644 index 00000000000..bdf56d7827d --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/data/dataset_init_kernel.cc @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/data/dataset_init_kernel.h" +#include "device/gpu/gpu_buffer_mgr.h" +#include "device/gpu/gpu_memory_allocator.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace kernel { +using mindspore::device::GpuBufferMgr; + +DatasetInitKernel::DatasetInitKernel() : feature_size_(0), label_size_(0) {} + +const std::vector &DatasetInitKernel::GetInputSizeList() const { return input_size_list_; } + +const std::vector &DatasetInitKernel::GetOutputSizeList() const { return output_size_list_; } + +const std::vector &DatasetInitKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } + +size_t DatasetInitKernel::TensorSize(std::vector &shape) const { + if (shape.size() == 0) { + return 0; + } + + int size = 1; + for (size_t i = 0; i < shape.size(); i++) { + size *= shape[i]; + } + + return IntToSize(size); +} + +bool DatasetInitKernel::Init(const CNodePtr &kernel_node) { + queue_name_ = GetAttr(kernel_node, "queue_name"); + auto shapes = GetAttr>>(kernel_node, "shapes"); + auto data_num = shapes.size(); + if (data_num != 2) { + MS_LOG(EXCEPTION) << "Invalid Shapes " << data_num; + } + + auto &feature_Shapes = shapes[0]; + auto size = TensorSize(feature_Shapes); + feature_size_ = size * sizeof(float); + + auto types = GetAttr>(kernel_node, "types"); + if ((types[1]->type_id() != kNumberTypeInt32) && (types[1]->type_id() != kNumberTypeInt64)) { + MS_LOG(EXCEPTION) << "Invalid types " << types[1]->type_id(); + } + + size_t label_unit = (types[1]->type_id() == kNumberTypeInt32) ? sizeof(int32_t) : sizeof(int64_t); + size = TensorSize(shapes[1]); + label_size_ = size * label_unit; + return true; +} + +void DatasetInitKernel::InitSizeLists() { return; } + +bool DatasetInitKernel::Launch(const std::vector &, const std::vector &, + const std::vector &, uintptr_t) { + void *addr = nullptr; + size_t len = (feature_size_ + label_size_) * buffer_q_capacity_; + + if (!device::gpu::GPUMemoryAllocator::GetInstance().AllocBufferQueueMem(len, &addr)) { + MS_LOG(EXCEPTION) << "Memory not enough: failed to allocate GPU buffer queue memory[" << len << "]."; + } + + auto status = + GpuBufferMgr::GetInstance().Create(0, queue_name_, addr, feature_size_, label_size_, buffer_q_capacity_); + if (status) { + MS_LOG(EXCEPTION) << "Init Dataset Failed: " << queue_name_ << ", " << feature_size_ << ", " << label_size_ << ", " + << status; + } + + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/data/dataset_init_kernel.h b/mindspore/ccsrc/kernel/gpu/data/dataset_init_kernel.h new file mode 100644 index 00000000000..7750fb8da6d --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/data/dataset_init_kernel.h @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_DATASET_INIT_KERNEL_H +#define MINDSPORE_DATASET_INIT_KERNEL_H + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" + +namespace mindspore { +namespace kernel { +class DatasetInitKernel : public GpuKernel { + public: + DatasetInitKernel(); + ~DatasetInitKernel() = default; + + const std::vector &GetInputSizeList() const override; + const std::vector &GetOutputSizeList() const override; + const std::vector &GetWorkspaceSizeList() const override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + bool Init(const CNodePtr &kernel_node) override; + + protected: + void InitSizeLists() override; + + private: + size_t TensorSize(std::vector &) const; + + std::string queue_name_; + size_t feature_size_; + size_t label_size_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + // The capacity of buffer Q. + size_t buffer_q_capacity_{2}; +}; + +MS_REG_GPU_KERNEL(InitDataSetQueue, DatasetInitKernel) +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_QUEUE_CPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/data/dataset_iterator_kernel.cc b/mindspore/ccsrc/kernel/gpu/data/dataset_iterator_kernel.cc new file mode 100644 index 00000000000..342bfb98e3f --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/data/dataset_iterator_kernel.cc @@ -0,0 +1,136 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/data/dataset_iterator_kernel.h" + +#include +#include +#include + +#include "device/gpu/gpu_buffer_mgr.h" +#include "device/gpu/gpu_common.h" + +namespace mindspore { +namespace kernel { +using mindspore::device::GpuBufferMgr; +using mindspore::device::HandleMgr; + +DatasetIteratorKernel::DatasetIteratorKernel() + : output_num_(0), handle_(HandleMgr::INVALID_HANDLE), feature_size_(0), label_size_(0) {} + +DatasetIteratorKernel::~DatasetIteratorKernel() { GpuBufferMgr::GetInstance().Close(handle_); } + +const std::vector &DatasetIteratorKernel::GetInputSizeList() const { return input_size_list_; } + +const std::vector &DatasetIteratorKernel::GetOutputSizeList() const { return output_size_list_; } + +const std::vector &DatasetIteratorKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } + +size_t DatasetIteratorKernel::TensorSize(std::vector &shape) const { + if (shape.size() == 0) { + return 0; + } + + int size = 1; + for (size_t i = 0; i < shape.size(); i++) { + size *= shape[i]; + } + + return IntToSize(size); +} + +bool DatasetIteratorKernel::Init(const CNodePtr &kernel_node) { + output_num_ = GetAttr(kernel_node, "output_num"); + queue_name_ = GetAttr(kernel_node, "shared_name"); + auto shapes = GetAttr>>(kernel_node, "shapes"); + auto data_num = shapes.size(); + if (data_num != 2) { + MS_LOG(EXCEPTION) << "Invalid Shapes " << data_num; + } + + auto &feature_Shapes = shapes[0]; + auto size = TensorSize(feature_Shapes); + feature_size_ = size * sizeof(float); + + auto types = GetAttr>(kernel_node, "types"); + if ((types[1]->type_id() != kNumberTypeInt32) && (types[1]->type_id() != kNumberTypeInt64)) { + MS_LOG(EXCEPTION) << "Invalid types " << types[1]->type_id(); + } + + size_t label_unit = (types[1]->type_id() == kNumberTypeInt32) ? sizeof(int32_t) : sizeof(int64_t); + size = TensorSize(shapes[1]); + label_size_ = size * label_unit; + + InitSizeLists(); + + handle_ = GpuBufferMgr::GetInstance().Open(0, queue_name_, feature_size_, label_size_); + if (handle_ == HandleMgr::INVALID_HANDLE) { + MS_LOG(EXCEPTION) << "Gpu Queue(" << queue_name_ << ") Open Failed: feature_size(" << feature_size_ + << "), label_size(" << label_size_ << ")"; + } + + return true; +} + +void DatasetIteratorKernel::InitSizeLists() { + output_size_list_.push_back(feature_size_); + output_size_list_.push_back(label_size_); +} + +bool DatasetIteratorKernel::Launch(const std::vector &, const std::vector &, + const std::vector &outputs, uintptr_t) { + void *feature_addr{nullptr}, *label_addr{nullptr}; + size_t feature_size{0}, label_size{0}; + + int repeat = 0; + while (true) { + auto ret = GpuBufferMgr::GetInstance().Front(handle_, &feature_addr, &feature_size, &label_addr, &label_size); + if (ret == device::SUCCESS) { + break; + } + + if (ret == device::TIMEOUT) { + repeat++; + if (repeat < 10) { + MS_LOG(INFO) << "Waiting for data...(" << repeat << " / 10)"; + continue; + } else { + MS_LOG(ERROR) << "Get data timeout"; + return false; + } + } + + MS_LOG(ERROR) << "Get data failed, errcode " << ret; + return false; + } + + if (feature_size != feature_size_ || label_size != label_size_) { + MS_LOG(ERROR) << "DatasetIteratorKernel: Front Error: " << feature_addr << ", " << feature_size << ", " + << label_addr << ", " << label_size; + return false; + } + + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpy(outputs[0]->addr, feature_addr, feature_size, cudaMemcpyDeviceToDevice), + "Cuda Memcpy Failed"); + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpy(outputs[1]->addr, label_addr, label_size, cudaMemcpyDeviceToDevice), + "Cuda Memcpy Failed"); + + (void)GpuBufferMgr::GetInstance().Pop(handle_); + + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/data/dataset_iterator_kernel.h b/mindspore/ccsrc/kernel/gpu/data/dataset_iterator_kernel.h new file mode 100644 index 00000000000..5819a87b936 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/data/dataset_iterator_kernel.h @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_GET_NEXT_KERNEL_H +#define MINDSPORE_GET_NEXT_KERNEL_H + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" + +namespace mindspore { +namespace kernel { +class DatasetIteratorKernel : public GpuKernel { + public: + DatasetIteratorKernel(); + ~DatasetIteratorKernel(); + + const std::vector &GetInputSizeList() const override; + const std::vector &GetOutputSizeList() const override; + const std::vector &GetWorkspaceSizeList() const override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + bool Init(const CNodePtr &kernel_node) override; + + protected: + void InitSizeLists() override; + + private: + size_t TensorSize(std::vector &) const; + + std::string queue_name_; + int output_num_; + unsigned int handle_; + + size_t feature_size_; + size_t label_size_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; + +MS_REG_GPU_KERNEL(GetNext, DatasetIteratorKernel) +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_QUEUE_CPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/gpu_kernel.h new file mode 100644 index 00000000000..de7176eff73 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/gpu_kernel.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_GPUKERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_GPUKERNEL_H_ + +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "device/gpu/gpu_device_manager.h" +#include "device/gpu/gpu_common.h" +#include "session/anf_runtime_algorithm.h" +using AnfAlgo = mindspore::session::AnfRuntimeAlgorithm; + +namespace mindspore { +namespace kernel { +class GpuKernel : public KernelMod { + public: + virtual ~GpuKernel() = default; + virtual bool Init(const CNodePtr &kernel_node) = 0; + + protected: + virtual void InitResource() {} + virtual void InitSizeLists() = 0; + + template + + inline T *GetDeviceAddress(const std::vector &addr_list, size_t index) { + if (index >= addr_list.size()) { + MS_LOG(EXCEPTION) << "Address index(" << index << ") out of range(" << addr_list.size() << ")"; + } + // Kernels may run normally without workspace, the addr_list[index] maybe nullptr. + if ((addr_list[index] == nullptr) || (addr_list[index]->size == 0)) { + return nullptr; + } + MS_EXCEPTION_IF_NULL(addr_list[index]->addr); + return reinterpret_cast(addr_list[index]->addr); + } + + template + inline T GetAttr(const CNodePtr &kernel_node, const std::string &key) const { + const PrimitivePtr &prim = AnfAlgo::GetCNodePrimitive(kernel_node); + const ValuePtr &attr = prim->GetAttr(key); + if (attr == nullptr) { + const std::string &prim_name = AnfAlgo::GetCNodeName(kernel_node); + MS_LOG(EXCEPTION) << "The attr(" << key << ") of kernel(" << prim_name << ") not exist"; + } + return GetValue(attr); + } +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_GPUKERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/gpu_kernel_factory.cc b/mindspore/ccsrc/kernel/gpu/gpu_kernel_factory.cc new file mode 100644 index 00000000000..21f5d084a92 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/gpu_kernel_factory.cc @@ -0,0 +1,145 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/gpu_kernel_factory.h" + +#include +#include + +#include "common/utils.h" +#include "device/kernel_info.h" +#include "device/gpu/cuda_common.h" +#include "kernel/common_utils.h" + +namespace mindspore { +namespace kernel { +GpuKernelFactory &GpuKernelFactory::GetInstance() { + static GpuKernelFactory instance; + return instance; +} + +void GpuKernelFactory::Register(const std::string &kernel_name, const KernelAttr &kernel_attr, + GpuKernelCreater &&creater) { + map_kernel_name_to_creater_[kernel_name].emplace_back(kernel_attr, creater); +} + +void GpuKernelFactory::CheckIOParam(const std::string &kernel_name, const KernelBuildInfo *kernel_info, + std::vector> *iter_second, + size_t attr_index) { + if (kernel_info->GetInputNum() != iter_second->at(attr_index).first.GetInputSize()) { + if (iter_second->at(attr_index).first.GetAllSame()) { + for (size_t attr = 1; attr < kernel_info->GetInputNum(); ++attr) { + (void)iter_second->at(attr_index).first.AddInputAttr(kernel_info->GetInputDeviceType(0)); + } + } else { + MS_LOG(EXCEPTION) << "op[" << kernel_name << "] Input size is mismatching!"; + } + } + if (kernel_info->GetOutputNum() != iter_second->at(attr_index).first.GetOutputSize()) { + if (iter_second->at(attr_index).first.GetAllSame()) { + for (size_t attr = 1; attr < kernel_info->GetOutputNum(); ++attr) { + (void)iter_second->at(attr_index).first.AddOutputAttr(kernel_info->GetOutputDeviceType(0)); + } + } else { + MS_LOG(EXCEPTION) << "op[" << kernel_name << "] Output size is mismatching!"; + } + } +} + +std::string GpuKernelFactory::SupportedTypeList(const std::string &kernel_name) { + std::string type_lists = ""; + auto iter = map_kernel_name_to_creater_.find(kernel_name); + if (map_kernel_name_to_creater_.end() == iter) { + return type_lists; + } + for (size_t attr_index = 0; attr_index < (iter->second).size(); ++attr_index) { + std::string type_list = "["; + auto attr = (iter->second)[attr_index].first; + for (size_t input_index = 0; input_index < attr.GetInputSize(); ++input_index) { + type_list = type_list + TypeId2String(attr.GetInputAttr(input_index).first) + + ((input_index == (attr.GetInputSize() - 1)) ? "" : " "); + } + type_lists = type_lists + type_list + "] "; + } + return type_lists; +} + +std::pair GpuKernelFactory::GpuKernelAttrCheck(const std::string &kernel_name, + const KernelBuildInfo *kernel_info) { + auto iter = map_kernel_name_to_creater_.find(kernel_name); + const int marjor_sm = GET_MAJOR_SM; + if (map_kernel_name_to_creater_.end() == iter) { + MS_LOG(INFO) << "Not registered GPU kernel: op[" << kernel_name << "]!"; + return std::make_pair(false, 0); + } + if ((iter->second).size() == 1 && (iter->second)[0].first.GetInputSize() == 0) { + return std::make_pair(true, 0); + } + + for (size_t attr_index = 0; attr_index < (iter->second).size(); ++attr_index) { + CheckIOParam(kernel_name, kernel_info, &(iter->second), attr_index); + bool flag = true; + // data type matching check of all input parameters of kernel + for (size_t input_index = 0; input_index < kernel_info->GetInputNum(); input_index++) { + if (marjor_sm < MINIUM_SM && kernel_info->GetInputDeviceType(input_index) == kNumberTypeFloat16) { + MS_LOG(EXCEPTION) << "Half precision op can be used on Devices which compute capacity is above " << MINIUM_SM + << ", but your device's compute capacity is " << marjor_sm; + } + if (kernel_info->GetInputDeviceType(input_index) != + (iter->second)[attr_index].first.GetInputAttr(input_index).first) { + flag = false; + break; + } + } + if (!flag) { + continue; + } + // data type matching check of all output parameters of kernel + for (size_t output_index = 0; output_index < kernel_info->GetOutputNum(); output_index++) { + if (kernel_info->GetOutputDeviceType(output_index) != + (iter->second)[attr_index].first.GetOutputAttr(output_index).first) { + flag = false; + break; + } + } + // finish data type matching check and return a pair maintain the whether matching is success, + // if first is true, second is index of matching KernelAttr and creater pair in vector; + if (flag) { + size_t match_index = attr_index; + return std::make_pair(true, match_index); + } + } + return std::make_pair(false, 0); +} + +GpuKernel *GpuKernelFactory::Create(const std::string &kernel_name, const CNodePtr &apply_kernel) { + auto kernel_info = apply_kernel->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + const KernelBuildInfo *kernel_build_Info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(kernel_build_Info); + std::pair ret_pair = GpuKernelAttrCheck(kernel_name, kernel_build_Info); + if (ret_pair.first) { + return (map_kernel_name_to_creater_.find(kernel_name)->second)[ret_pair.second].second(); + } + return nullptr; +} + +bool GpuKernelFactory::SearchRegistered(const std::string &kernel_name, const KernelBuildInfoPtr &kernel_build_info) { + std::pair ret_pair = GpuKernelAttrCheck(kernel_name, kernel_build_info.get()); + return ret_pair.first; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/gpu_kernel_factory.h b/mindspore/ccsrc/kernel/gpu/gpu_kernel_factory.h new file mode 100644 index 00000000000..dc5f61a315a --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/gpu_kernel_factory.h @@ -0,0 +1,93 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_GPUKERNELFACTORY_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_GPUKERNELFACTORY_H_ + +#include +#include +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "device/gpu/kernel_info_setter.h" +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace kernel { +using mindspore::device::gpu::KernelAttr; +using GpuKernelCreater = std::function; +class GpuKernelFactory { + public: + ~GpuKernelFactory() = default; + + static GpuKernelFactory &GetInstance(); + + void Register(const std::string &kernel_name, const KernelAttr &kernel_attr, GpuKernelCreater &&creater); + + GpuKernel *Create(const std::string &kernel_name, const CNodePtr &apply_kernel); + + bool SearchRegistered(const std::string &kernel_name, const KernelBuildInfoPtr &kernel_info); + + std::string SupportedTypeList(const std::string &kernel_name); + + private: + GpuKernelFactory() = default; + + GpuKernelFactory(GpuKernelFactory const &); + + GpuKernelFactory &operator=(const GpuKernelFactory &); + + std::pair GpuKernelAttrCheck(const std::string &kernel_name, const KernelBuildInfo *kernel_info); + void CheckIOParam(const std::string &kernel_name, const KernelBuildInfo *kernel_info, + std::vector> *iter_second, size_t attr_index); + // map to maintain kernel and creater, KernelAttr object and creater must be registered as a pair. + std::map>> map_kernel_name_to_creater_; +}; + +class GpuKernelRegister { + public: + GpuKernelRegister(const std::string &kernel_name, const KernelAttr &kernel_attr, GpuKernelCreater &&creater) { + GpuKernelFactory::GetInstance().Register(kernel_name, kernel_attr, std::move(creater)); + } +}; + +#define MS_REG_GPU_KERNEL(OPNAME, OPCLASS) \ + static_assert(std::is_base_of::value, " must be base of GpuKernel"); \ + static const GpuKernelRegister g_##OPNAME##_gpu_kernel_reg(#OPNAME, KernelAttr(), []() { return new OPCLASS(); }); + +// regular register of fixed accuracy kernels +#define MS_REG_GPU_KERNEL_REGULAR(OPNAME, ATTR, OPCLASS) \ + static_assert(std::is_base_of::value, " must be base of GpuKernel"); \ + static const GpuKernelRegister g_##OPNAME##_gpu_kernel_reg(#OPNAME, ATTR, []() { return new OPCLASS(); }); + +// register of mixed accuracy kernels which use template and maintain one typename, ignore input num +#define MS_REG_GPU_KERNEL_SAME(OPNAME, ATTR, OPCLASS, T) \ + static_assert(std::is_base_of>::value, " must be base of GpuKernel"); \ + static const GpuKernelRegister g_##OPNAME##_##T##_gpu_kernel_reg(#OPNAME, ATTR, []() { return new OPCLASS(); }); + +// register of mixed accuracy kernels which use template and maintain one typename +#define MS_REG_GPU_KERNEL_ONE(OPNAME, ATTR, OPCLASS, T) \ + static_assert(std::is_base_of>::value, " must be base of GpuKernel"); \ + static const GpuKernelRegister g_##OPNAME##_##T##_gpu_kernel_reg(#OPNAME, ATTR, []() { return new OPCLASS(); }); + +// register of mixed accuracy kernels which use template and maintain two typename +#define MS_REG_GPU_KERNEL_TWO(OPNAME, ATTR, OPCLASS, T, S) \ + static_assert(std::is_base_of>::value, " must be base of GpuKernel"); \ + static const GpuKernelRegister g_##OPNAME##_##T##_##S##_gpu_kernel_reg(#OPNAME, ATTR, \ + []() { return new OPCLASS(); }); +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_GPU_GPUKERNELFACTORY_H_ diff --git a/mindspore/ccsrc/kernel/gpu/kernel_constants.h b/mindspore/ccsrc/kernel/gpu/kernel_constants.h new file mode 100644 index 00000000000..7c9f209a9ef --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/kernel_constants.h @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_KERNEL_CONSTANTS_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_KERNEL_CONSTANTS_H_ +#include +#include + +namespace mindspore { +namespace kernel { +// Used by Pooling and Conv2d +static constexpr char kSamePadModeUpperCase[] = "SAME"; + +// Used by Pooling and Conv2d +static constexpr char kSamePadModeLowerCase[] = "same"; + +// Used by Pooling and Conv2d +static constexpr char kValidPadModeUpperCase[] = "VALID"; + +// Used by Pooling and Conv2d +static constexpr char kValidPadModeLowerCase[] = "valid"; + +// Used by Pooling +static constexpr char kAvgPoolingModeUpperCase[] = "AVG"; + +// Used by Pooling +static constexpr char kAvgPoolingModeLowerCase[] = "avg"; + +// Used by MaxPool pad: The minimum value of float32 +static constexpr float kSignedMinFloat = -3.402823466e+38F; + +// Used by mixprecision, cudnn dtype select +static std::map kCudnnDtypeMap = {{"kNumberTypeFloat32", CUDNN_DATA_FLOAT}, + {"kNumberTypeFloat16", CUDNN_DATA_HALF}, + {"kNumberTypeInt32", CUDNN_DATA_INT32}}; +// Used by mixprecision, cuda dtype select +static std::map kCudaDtypeMap = {{"kNumberTypeFloat32", CUDA_R_32F}, + {"kNumberTypeFloat16", CUDA_R_16F}}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_KERNEL_CONSTANTS_H_ diff --git a/mindspore/ccsrc/kernel/gpu/math/addn_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/addn_gpu_kernel.cc new file mode 100644 index 00000000000..4683f015ae7 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/addn_gpu_kernel.cc @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/addn_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + AddN, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + AddNGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + AddN, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + AddNGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(AddN, + KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + AddNGpuFwdKernel, int) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/addn_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/addn_gpu_kernel.h new file mode 100644 index 00000000000..485d3b4f721 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/addn_gpu_kernel.h @@ -0,0 +1,134 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_ADDN_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_ADDN_GPU_KERNEL_H_ + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class AddNGpuFwdKernel : public GpuKernel { + public: + AddNGpuFwdKernel() + : cudnn_handle_(nullptr), + input_descriptor_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + input_size_(0), + output_size_(0), + workspace_size_(0), + is_null_input_(false), + num_input_(0) {} + ~AddNGpuFwdKernel() override { DestroyResource(); } + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t) override { + if (is_null_input_) { + return true; + } + T *output_addr = GetDeviceAddress(outputs, 0); + const float alpha = 1; + const float beta = 0; + for (size_t i = 0; i < IntToSize(num_input_); i++) { + T *input_addr = GetDeviceAddress(inputs, i); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnAddTensor(cudnn_handle_, &alpha, input_descriptor_, input_addr, + &(i > 0 ? alpha : beta), input_descriptor_, output_addr), + "cudnnAddTensor failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + num_input_ = GetAttr(kernel_node, "n"); + if (IntToSize(num_input_) != input_num) { + MS_LOG(ERROR) << "Input number is " << num_input_ << " in attr, but got " << input_num << "input."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but cudnnAddTensor needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "AddNGpuFwdKernel input is null"; + InitSizeLists(); + return true; + } + for (size_t i = input_shape.size(); i < 4; i++) { + (void)input_shape.insert(input_shape.begin(), 1); + } + int dimA[4]; + for (size_t i = 0; i < input_shape.size(); i++) { + dimA[i] = SizeToInt(input_shape[i]); + } + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(input_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + SizeToInt(input_shape.size()), dimA), + "cudnnSetTensorNdDescriptor failed"); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&input_descriptor_), "cudnnCreateTensorDescriptor failed"); + } + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetTensorSizeInBytes(input_descriptor_, reinterpret_cast(&input_size_)), + "cudnnGetTensorSizeInBytes failed"); + } + for (int i = 0; i < num_input_; i++) { + input_size_list_.push_back(input_size_); + } + output_size_list_.push_back(input_size_); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(input_descriptor_), "cudnnDestroyTensorDescriptor failed"); + } + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t input_descriptor_; + cudnnDataType_t cudnn_data_type_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + bool is_null_input_; + int num_input_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_ADDN_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/math/assign_add_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/assign_add_gpu_kernel.cc new file mode 100644 index 00000000000..2ae1728ca34 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/assign_add_gpu_kernel.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/assign_add_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + AssignAdd, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + AssignAddGpuFwdKernel, int) +MS_REG_GPU_KERNEL_ONE( + AssignAdd, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + AssignAddGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + AssignAdd, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + AssignAddGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/assign_add_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/assign_add_gpu_kernel.h new file mode 100644 index 00000000000..191bd0b1c8b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/assign_add_gpu_kernel.h @@ -0,0 +1,95 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_ASSIGNADD_GPU_KERNEL_H +#define MINDSPORE_CCSRC_KERNEL_GPU_ASSIGNADD_GPU_KERNEL_H + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/assign_add_impl.cuh" +namespace mindspore { +namespace kernel { +template +class AssignAddGpuFwdKernel : public GpuKernel { + public: + AssignAddGpuFwdKernel() : is_null_input_(false), input_size_(0) {} + ~AssignAddGpuFwdKernel() override = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *input_addr = GetDeviceAddress(inputs, 0); + T *input_addr2 = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + + CalAssignAdd(input_size_ / sizeof(T), input_addr, input_addr2, output_addr, + reinterpret_cast(stream_ptr)); + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but cudnnAddTensor needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but cudnnAddTensor needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "AssignAddGpuFwdKernel input is null"; + InitSizeLists(); + return true; + } + input_size_ = sizeof(T); + for (size_t i : input_shape) { + input_size_ = i * input_size_; + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + input_size_list_.push_back(input_size_); + output_size_list_.push_back(input_size_); + } + + private: + bool is_null_input_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_ASSIGNADD_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.cc new file mode 100644 index 00000000000..5684f0c424a --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/bias_add_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + BiasAdd, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + BiasAddGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + BiasAdd, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + BiasAddGpuKernel, float16) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.h new file mode 100644 index 00000000000..265180afe64 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/bias_add_gpu_kernel.h @@ -0,0 +1,133 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_BIAS_ADD_GPU_KERNEL_H +#define MINDSPORE_BIAS_ADD_GPU_KERNEL_H +#include +#include +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "dataset/util/make_unique.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class BiasAddGpuKernel : public GpuKernel { + public: + BiasAddGpuKernel() + : cudnn_handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + x_desc_(nullptr), + b_desc_(nullptr), + op_desc_(nullptr) {} + ~BiasAddGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + VARIABLE_NOT_USED(stream_ptr); + T *x_addr = GetDeviceAddress(inputs, 0); + T *b_addr = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + + const float alpha = 1; + const float beta = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnOpTensor(cudnn_handle_, op_desc_, &alpha, x_desc_, x_addr, &alpha, b_desc_, b_addr, + &beta, x_desc_, output_addr), + "cudnnOpTensor Add failed"); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto num_dims = x_shape.size(); + if (num_dims < 2) { + MS_LOG(EXCEPTION) << "input dims must be at least 2, but got " << num_dims; + } + + std::string format = GetAttr(kernel_node, "data_format"); + string::size_type pos = format.find("C"); + if (pos == std::string::npos || pos >= num_dims) { + MS_LOG(EXCEPTION) << "format '" << format << "' invalid"; + } + + // Expand to 4 dims for cudnnSetTensorNdDescriptorEx. + auto cudnn_dims = std::max(num_dims, 4UL); + std::unique_ptr x_dims = mindspore::make_unique(cudnn_dims); + std::unique_ptr b_dims = mindspore::make_unique(cudnn_dims); + for (size_t i = 0; i < cudnn_dims; i++) { + x_dims[i] = (i < num_dims) ? SizeToInt(x_shape[i]) : 1; + b_dims[i] = (i == pos) ? SizeToInt(x_shape[i]) : 1; + } + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(x_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(cudnn_dims), x_dims.get()), + "cudnnSetTensorNdDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(b_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(cudnn_dims), b_dims.get()), + "cudnnSetTensorNdDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetOpTensorDescriptor(op_desc_, CUDNN_OP_TENSOR_ADD, CUDNN_DATA_FLOAT, CUDNN_NOT_PROPAGATE_NAN), + "cudnnSetOpTensorDescriptor failed"); + + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&b_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateOpTensorDescriptor(&op_desc_), "cudnnCreateOpTensorDescriptor failed"); + } + void InitSizeLists() override { + size_t x_size, b_size; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(x_desc_, &x_size), "cudnnGetTensorSizeInBytes failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(b_desc_, &b_size), "cudnnGetTensorSizeInBytes failed."); + input_size_list_.push_back(x_size); + input_size_list_.push_back(b_size); + output_size_list_.push_back(x_size); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyOpTensorDescriptor(op_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(b_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_desc_), "cudnnDestroyOpTensorDescriptor failed"); + } + + cudnnHandle_t cudnn_handle_; + cudnnDataType_t cudnn_data_type_; + cudnnTensorDescriptor_t x_desc_; + cudnnTensorDescriptor_t b_desc_; + cudnnOpTensorDescriptor_t op_desc_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_BIAS_ADD_GPU_KERNEL_H diff --git a/mindspore/ccsrc/kernel/gpu/math/binary_op_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/binary_op_gpu_kernel.cc new file mode 100644 index 00000000000..56a0905e4e7 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/binary_op_gpu_kernel.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/binary_op_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + RealDiv, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + BinaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + RealDiv, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + BinaryOpGpuKernel, half) +MS_REG_GPU_KERNEL_ONE( + Mul, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + BinaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + Mul, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + BinaryOpGpuKernel, half) +MS_REG_GPU_KERNEL_ONE( + Sub, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + BinaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + Sub, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + BinaryOpGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/binary_op_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/binary_op_gpu_kernel.h new file mode 100644 index 00000000000..522ec2b37e0 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/binary_op_gpu_kernel.h @@ -0,0 +1,232 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_BINARYOP_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_BINARYOP_GPU_KERNEL_H_ + +#include +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/unary_op_impl.cuh" +#include "kernel/gpu/kernel_constants.h" +namespace mindspore { +namespace kernel { +enum BinaryOpType { BINARY_OP_ADD = 0, BINARY_OP_SUB, BINARY_OP_MUL, BINARY_OP_DIV, BINARY_OP_INVALID_TYPE = 255 }; +const std::map kBinaryOpTypeMap = { + {"Sub", BINARY_OP_SUB}, + {"Mul", BINARY_OP_MUL}, + {"RealDiv", BINARY_OP_DIV}, +}; +template +class BinaryOpGpuKernel : public GpuKernel { + public: + BinaryOpGpuKernel() + : cudnn_handle_(nullptr), + binary_op_type_(BINARY_OP_INVALID_TYPE), + tensor_op_(CUDNN_OP_TENSOR_MUL), + inputA_descriptor_(nullptr), + inputB_descriptor_(nullptr), + opTensor_descriptor_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + is_null_input_(false), + input_size_(0), + output_size_(0), + workspace_size_(0) {} + ~BinaryOpGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *input_addr = GetDeviceAddress(inputs, 0); + T *input_addr2 = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + const float alpha = 1; + const float beta = 0; + + T *inputB_addr = nullptr; + switch (binary_op_type_) { + case BINARY_OP_SUB: { + T *workspace_addr = GetDeviceAddress(workspace, 0); + Negative(input_addr2, workspace_addr, inputs[1]->size / sizeof(T), reinterpret_cast(stream_ptr)); + inputB_addr = workspace_addr; + break; + } + case BINARY_OP_MUL: { + inputB_addr = input_addr2; + break; + } + case BINARY_OP_DIV: { + T *workspace_addr = GetDeviceAddress(workspace, 0); + Reciprocal(input_addr2, workspace_addr, inputs[1]->size / sizeof(T), + reinterpret_cast(stream_ptr)); + inputB_addr = workspace_addr; + break; + } + default: { + MS_LOG(EXCEPTION) << "Binary operation " << binary_op_type_ << " is not supported."; + } + } + if (inputs[0]->size > inputs[1]->size) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnOpTensor(cudnn_handle_, opTensor_descriptor_, &alpha, inputA_descriptor_, input_addr, &alpha, + inputB_descriptor_, inputB_addr, &beta, inputA_descriptor_, output_addr), + "cudnnOpTensor failed"); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnOpTensor(cudnn_handle_, opTensor_descriptor_, &alpha, inputB_descriptor_, inputB_addr, &alpha, + inputA_descriptor_, input_addr, &beta, inputB_descriptor_, output_addr), + "cudnnOpTensor failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but binary operation needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but binary operation needs 1 output."; + return false; + } + InferBinaryType(kernel_node); + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto input_shapeB = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(input_shape) || CHECK_NULL_INPUT(input_shapeB); + if (is_null_input_) { + MS_LOG(WARNING) << "BinaryOpGpuKernel input is null"; + InitSizeLists(); + return true; + } + int shape_n = input_shape.size() < 4 ? 1 : SizeToInt(input_shape[input_shape.size() - 4]); + int shape_c = input_shape.size() < 3 ? 1 : SizeToInt(input_shape[input_shape.size() - 3]); + int shape_h = input_shape.size() < 2 ? 1 : SizeToInt(input_shape[input_shape.size() - 2]); + int shape_w = input_shape.size() == 0 ? 1 : SizeToInt(input_shape[input_shape.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(inputA_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + shape_n, shape_c, shape_h, shape_w), + "cudnnSetTensor4dDescriptor failed"); + int shapeB_n = input_shapeB.size() < 4 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 4]); + int shapeB_c = input_shapeB.size() < 3 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 3]); + int shapeB_h = input_shapeB.size() < 2 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 2]); + int shapeB_w = input_shapeB.size() == 0 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(inputB_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + shapeB_n, shapeB_c, shapeB_h, shapeB_w), + "cudnnSetTensor4dDescriptor failed"); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&inputA_descriptor_), + "cudnnCreateTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&inputB_descriptor_), + "cudnnCreateTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateOpTensorDescriptor(&opTensor_descriptor_), + "cudnnCreateOpTensorDescriptor failed."); + } + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(inputA_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed."); + input_size_list_.push_back(input_size_); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(inputB_descriptor_, &output_size_), + "cudnnGetTensorSizeInBytes failed."); + } + input_size_list_.push_back(output_size_); + if (binary_op_type_ == BINARY_OP_DIV || binary_op_type_ == BINARY_OP_SUB) { + workspace_size_ = output_size_; + } + workspace_size_list_.push_back(workspace_size_); + + if (output_size_ > input_size_) { + output_size_list_.push_back(output_size_); + } else { + output_size_list_.push_back(input_size_); + } + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(inputA_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(inputB_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyOpTensorDescriptor(opTensor_descriptor_), + "cudnnDestroyOpTensorDescriptor failed."); + } + void InferBinaryType(const CNodePtr &kernel_node) { + std::string kernel_name = AnfAlgo::GetCNodeName(kernel_node); + auto iter = kBinaryOpTypeMap.find(kernel_name); + if (iter == kBinaryOpTypeMap.end()) { + MS_LOG(EXCEPTION) << "Binary operation " << kernel_name << " is not supported."; + } else { + binary_op_type_ = iter->second; + } + + switch (binary_op_type_) { + case BINARY_OP_DIV: + case BINARY_OP_MUL: { + tensor_op_ = CUDNN_OP_TENSOR_MUL; + break; + } + case BINARY_OP_SUB: { + tensor_op_ = CUDNN_OP_TENSOR_ADD; + break; + } + default: { + MS_LOG(EXCEPTION) << "Binary operation " << binary_op_type_ << " is not supported."; + } + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetOpTensorDescriptor(opTensor_descriptor_, tensor_op_, cudnn_data_type_, CUDNN_NOT_PROPAGATE_NAN), + "cudnnSetOpTensorDescriptor failed"); + return; + } + + cudnnHandle_t cudnn_handle_; + BinaryOpType binary_op_type_; + cudnnOpTensorOp_t tensor_op_; + cudnnTensorDescriptor_t inputA_descriptor_; + cudnnTensorDescriptor_t inputB_descriptor_; + cudnnOpTensorDescriptor_t opTensor_descriptor_; + cudnnDataType_t cudnn_data_type_; + bool is_null_input_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_BINARYOP_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/math/equalcount_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/equalcount_gpu_kernel.cc new file mode 100644 index 00000000000..f3c3b6164d1 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/equalcount_gpu_kernel.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/equalcount_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + EqualCount, + KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + EqualCountGpuKernel, int) +MS_REG_GPU_KERNEL_ONE( + EqualCount, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + EqualCountGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + EqualCount, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + EqualCountGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/equalcount_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/equalcount_gpu_kernel.h new file mode 100644 index 00000000000..9e98d6be0f4 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/equalcount_gpu_kernel.h @@ -0,0 +1,89 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_EQUALCOUNT_GPU_KERNEL_H +#define MINDSPORE_EQUALCOUNT_GPU_KERNEL_H + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/equalcount_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class EqualCountGpuKernel : public GpuKernel { + public: + EqualCountGpuKernel() : input_size_(0), output_size_(0), workspace_size_(0) {} + ~EqualCountGpuKernel() = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + T *input1 = GetDeviceAddress(inputs, 0); + T *input2 = GetDeviceAddress(inputs, 1); + T *output = GetDeviceAddress(outputs, 0); + int size = SizeToInt(input_size_ / sizeof(T)); + CalEqualCount(size, input1, input2, output, reinterpret_cast(stream_ptr)); + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but equalcount needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but equalcount needs 1 output."; + return false; + } + + output_size_ = sizeof(T); + input_size_ = sizeof(T); + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + for (size_t i = 0; i < input_shape.size(); i++) { + input_size_ *= input_shape[i]; + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + return; + } + + private: + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/gpu/math/matmul_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/matmul_gpu_kernel.cc new file mode 100644 index 00000000000..a96cede94e7 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/matmul_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/matmul_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + MatMul, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + MatMulGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + MatMul, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + MatMulGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/matmul_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/matmul_gpu_kernel.h new file mode 100644 index 00000000000..36f4272c683 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/matmul_gpu_kernel.h @@ -0,0 +1,132 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MATMUL_GPU_KERNEL_H +#define MINDSPORE_MATMUL_GPU_KERNEL_H + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace kernel { +template +class MatMulGpuKernel : public GpuKernel { + public: + MatMulGpuKernel() + : batch_(0), + m_(0), + n_(0), + k_(0), + transpose_x1_(CUBLAS_OP_N), + transpose_x2_(CUBLAS_OP_N), + handle_(nullptr), + cudaDataType_(CUDA_R_32F) {} + ~MatMulGpuKernel() = default; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + VARIABLE_NOT_USED(stream_ptr); + auto input1_addr = GetDeviceAddress(inputs, 0); + auto input2_addr = GetDeviceAddress(inputs, 1); + auto output_addr = GetDeviceAddress(outputs, 0); + + const float alpha = 1; + const float beta = 0; + const int lda = (transpose_x2_ == CUBLAS_OP_T) ? SizeToInt(k_) : SizeToInt(n_); + const int ldb = (transpose_x1_ == CUBLAS_OP_T) ? SizeToInt(m_) : SizeToInt(k_); + + for (size_t i = 0; i < batch_; i++) { + auto input1_slice = input1_addr + i * m_ * k_; + auto input2_slice = input2_addr + i * k_ * n_; + auto output_slice = output_addr + i * m_ * n_; + + CHECK_CUBLAS_RET_WITH_EXCEPT(cublasSgemmEx(handle_, transpose_x2_, transpose_x1_, SizeToInt(n_), SizeToInt(m_), + SizeToInt(k_), &alpha, input2_slice, cudaDataType_, lda, input1_slice, + cudaDataType_, ldb, &beta, output_slice, cudaDataType_, SizeToInt(n_)), + "cublasSgemm Call Fail"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCublasHandle(); + cudaDataType_ = kCudaDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + auto dims = output_shape.size(); + if (dims < 2) { + MS_LOG(EXCEPTION) << "Output dims " << dims << " not support."; + } + + m_ = output_shape[dims - 2]; + n_ = output_shape[dims - 1]; + batch_ = 1; + for (size_t i = 0; i < dims - 2; i++) { + batch_ *= output_shape[i]; + } + + bool transpose = GetAttr(kernel_node, "transpose_x1"); + transpose_x1_ = transpose ? CUBLAS_OP_T : CUBLAS_OP_N; + auto input1_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + k_ = transpose ? input1_shape[dims - 2] : input1_shape[dims - 1]; + + transpose = GetAttr(kernel_node, "transpose_x2"); + transpose_x2_ = transpose ? CUBLAS_OP_T : CUBLAS_OP_N; + + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + size_t unit_size = sizeof(T); + + size_t input_size = batch_ * m_ * k_ * unit_size; + input_size_list_.push_back(input_size); + + input_size = batch_ * n_ * k_ * unit_size; + input_size_list_.push_back(input_size); + + size_t output_size = batch_ * m_ * n_ * unit_size; + output_size_list_.push_back(output_size); + } + + private: + size_t batch_; + size_t m_; + size_t n_; + size_t k_; + + cublasOperation_t transpose_x1_; + cublasOperation_t transpose_x2_; + + cublasHandle_t handle_; + cudaDataType_t cudaDataType_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/gpu/math/tensoradd_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/tensoradd_gpu_kernel.cc new file mode 100644 index 00000000000..1b7318c5114 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/tensoradd_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/tensoradd_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + TensorAdd, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + TensorAddGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + TensorAdd, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + TensorAddGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/tensoradd_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/tensoradd_gpu_kernel.h new file mode 100644 index 00000000000..a203567aa80 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/tensoradd_gpu_kernel.h @@ -0,0 +1,168 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_TENSORADD_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_TENSORADD_GPU_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +namespace mindspore { +namespace kernel { +template +class TensorAddGpuFwdKernel : public GpuKernel { + public: + TensorAddGpuFwdKernel() + : cudnn_handle_(nullptr), + inputA_descriptor_(nullptr), + inputB_descriptor_(nullptr), + opTensor_descriptor_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + input_size_(0), + output_size_(0), + workspace_size_(0), + is_null_input_(false) {} + ~TensorAddGpuFwdKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t) { + if (is_null_input_) { + return true; + } + T *input_addr = GetDeviceAddress(inputs, 0); + T *input_addr2 = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + const float alpha = 1; + const float beta = 0; + // A + B = C. [ C = op(alpha1[0] * A, alpha2[0] * B) + beta[0] * C ] + // InputA must match the corresponding dimension of the destination tensor outC, and each dimension of the inputB + // must match the corresponding dimension of outC or must be equal to 1. + if (inputs[0]->size > inputs[1]->size) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnOpTensor(cudnn_handle_, opTensor_descriptor_, &alpha, inputA_descriptor_, input_addr, &alpha, + inputB_descriptor_, input_addr2, &beta, inputA_descriptor_, output_addr), + "cudnnOpTensor Add failed"); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnOpTensor(cudnn_handle_, opTensor_descriptor_, &alpha, inputB_descriptor_, input_addr2, &alpha, + inputA_descriptor_, input_addr, &beta, inputB_descriptor_, output_addr), + "cudnnOpTensor Add failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but cudnnAddTensor needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but cudnnAddTensor needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto input_shapeB = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(input_shape) || CHECK_NULL_INPUT(input_shapeB); + if (is_null_input_) { + MS_LOG(WARNING) << "TensorAddGpuFwdKernel input is null"; + InitSizeLists(); + return true; + } + int shape_n = input_shape.size() < 4 ? 1 : SizeToInt(input_shape[input_shape.size() - 4]); + int shape_c = input_shape.size() < 3 ? 1 : SizeToInt(input_shape[input_shape.size() - 3]); + int shape_h = input_shape.size() < 2 ? 1 : SizeToInt(input_shape[input_shape.size() - 2]); + int shape_w = input_shape.size() == 0 ? 1 : SizeToInt(input_shape[input_shape.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(inputA_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + shape_n, shape_c, shape_h, shape_w), + "cudnnSetTensor4dDescriptor failed"); + int shapeB_n = input_shapeB.size() < 4 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 4]); + int shapeB_c = input_shapeB.size() < 3 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 3]); + int shapeB_h = input_shapeB.size() < 2 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 2]); + int shapeB_w = input_shapeB.size() == 0 ? 1 : SizeToInt(input_shapeB[input_shapeB.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(inputB_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + shapeB_n, shapeB_c, shapeB_h, shapeB_w), + "cudnnSetTensor4dDescriptor failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetOpTensorDescriptor(opTensor_descriptor_, CUDNN_OP_TENSOR_ADD, CUDNN_DATA_FLOAT, CUDNN_NOT_PROPAGATE_NAN), + "cudnnSetOpTensorDescriptor failed"); + + InitSizeLists(); + return true; + } + + protected: + void InitResource() { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&inputA_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&inputB_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateOpTensorDescriptor(&opTensor_descriptor_), + "cudnnCreateOpTensorDescriptor failed"); + } + void InitSizeLists() { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(inputA_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + input_size_list_.push_back(input_size_); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(inputB_descriptor_, &output_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(output_size_); + + if (output_size_ > input_size_) { + output_size_list_.push_back(output_size_); + } else { + output_size_list_.push_back(input_size_); + } + workspace_size_list_.push_back(workspace_size_); + + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(inputA_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(inputB_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyOpTensorDescriptor(opTensor_descriptor_), + "cudnnDestroyOpTensorDescriptor failed"); + } + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t inputA_descriptor_; + cudnnTensorDescriptor_t inputB_descriptor_; + cudnnOpTensorDescriptor_t opTensor_descriptor_; + cudnnDataType_t cudnn_data_type_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + bool is_null_input_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_TENSORADD_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/math/unary_op_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/unary_op_gpu_kernel.cc new file mode 100644 index 00000000000..d69706663e1 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/unary_op_gpu_kernel.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/math/unary_op_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(Exp, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + UnaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(Exp, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + UnaryOpGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(Log, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + UnaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(Log, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + UnaryOpGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(Neg, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + UnaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(Neg, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + UnaryOpGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(Reciprocal, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + UnaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(Reciprocal, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + UnaryOpGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(ZerosLike, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + UnaryOpGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(ZerosLike, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + UnaryOpGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/math/unary_op_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/math/unary_op_gpu_kernel.h new file mode 100644 index 00000000000..af78ea4e738 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/math/unary_op_gpu_kernel.h @@ -0,0 +1,131 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_UNARYOP_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_UNARYOP_GPU_KERNEL_H_ + +#include +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/unary_op_impl.cuh" + +namespace mindspore { +namespace kernel { +enum UnaryOptype { + UNARY_OP_EXP = 0, + UNARY_OP_LOG, + UNARY_OP_NEG, + UNARY_OP_RECIPROCAL, + UNARY_OP_ZEROSLIKE, + UNARY_OP_INVALID_TYPE = 255 +}; +const std::map kUnaryOpTypeMap = {{"Exp", UNARY_OP_EXP}, + {"Log", UNARY_OP_LOG}, + {"Neg", UNARY_OP_NEG}, + {"Reciprocal", UNARY_OP_RECIPROCAL}, + {"ZerosLike", UNARY_OP_ZEROSLIKE}}; +template +class UnaryOpGpuKernel : public GpuKernel { + public: + UnaryOpGpuKernel() + : unary_op_type_(UNARY_OP_INVALID_TYPE), input_size_(sizeof(T)), output_size_(sizeof(T)), workspace_size_(0) {} + ~UnaryOpGpuKernel() override = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + T *input_addr = GetDeviceAddress(inputs, 0); + T *output_addr = GetDeviceAddress(outputs, 0); + + switch (unary_op_type_) { + case UNARY_OP_EXP: { + Exponential(input_addr, output_addr, inputs[0]->size / sizeof(T), reinterpret_cast(stream_ptr)); + break; + } + case UNARY_OP_LOG: { + Logarithm(input_addr, output_addr, inputs[0]->size / sizeof(T), reinterpret_cast(stream_ptr)); + break; + } + case UNARY_OP_NEG: { + Negative(input_addr, output_addr, inputs[0]->size / sizeof(T), reinterpret_cast(stream_ptr)); + break; + } + case UNARY_OP_RECIPROCAL: { + Reciprocal(input_addr, output_addr, inputs[0]->size / sizeof(T), reinterpret_cast(stream_ptr)); + break; + } + case UNARY_OP_ZEROSLIKE: { + return true; + } + default: { + MS_LOG(EXCEPTION) << "Unary operation " << unary_op_type_ << " is not supported."; + } + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + std::string kernel_name = AnfAlgo::GetCNodeName(kernel_node); + auto iter = kUnaryOpTypeMap.find(kernel_name); + if (iter == kUnaryOpTypeMap.end()) { + MS_LOG(EXCEPTION) << "Unary operation " << kernel_name << " is not supported."; + } else { + unary_op_type_ = iter->second; + } + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but negative op needs 1 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but negative op needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + for (size_t i = 0; i < input_shape.size(); i++) { + input_size_ *= input_shape[i]; + } + output_size_ = input_size_; + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + } + + private: + UnaryOptype unary_op_type_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_UNARYOP_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nccl/nccl_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nccl/nccl_gpu_kernel.cc new file mode 100644 index 00000000000..6993085a75b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nccl/nccl_gpu_kernel.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nccl/nccl_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + AllReduce, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + NcclGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + AllReduce, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + NcclGpuKernel, half) +MS_REG_GPU_KERNEL_ONE( + AllGather, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + NcclGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + AllGather, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + NcclGpuKernel, half) +MS_REG_GPU_KERNEL_ONE( + ReduceScatter, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + NcclGpuKernel, float) +MS_REG_GPU_KERNEL_ONE( + ReduceScatter, KernelAttr().AddAllSameAttr(true).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + NcclGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nccl/nccl_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nccl/nccl_gpu_kernel.h new file mode 100644 index 00000000000..54e4eb92134 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nccl/nccl_gpu_kernel.h @@ -0,0 +1,174 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NCCL_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NCCL_GPU_KERNEL_H_ + +#include +#include +#include +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "device/gpu/distribution/collective_init.h" + +namespace mindspore { +namespace kernel { +enum NcclKernelType { NCCL_ALL_REDUCE = 0, NCCL_ALL_GATHER, NCCL_REDUCE_SCATTER, NCCL_INVALID_TYPE = 255 }; +const std::map kNcclTypeMap = { + {"AllReduce", NCCL_ALL_REDUCE}, + {"AllGather", NCCL_ALL_GATHER}, + {"ReduceScatter", NCCL_REDUCE_SCATTER}, +}; + +static std::map kNcclDtypeMap = { + {"kNumberTypeFloat32", ncclFloat}, {"kNumberTypeFloat16", ncclHalf}, {"kNumberTypeInt32", ncclInt}}; + +typedef ncclResult_t (*AllReduce)(const void *, void *, size_t, ncclDataType_t, ncclRedOp_t, cudaStream_t); +typedef ncclResult_t (*AllGather)(const void *, void *, size_t, ncclDataType_t, cudaStream_t); +typedef ncclResult_t (*ReduceScatter)(const void *, void *, size_t, ncclDataType_t, ncclRedOp_t, cudaStream_t); + +template +class NcclGpuKernel : public GpuKernel { + public: + NcclGpuKernel() + : nccl_kernel_type_(NCCL_INVALID_TYPE), + nccl_reduce_type_(ncclSum), + input_size_(0), + output_size_(0), + collective_handle_(nullptr) {} + ~NcclGpuKernel() override = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *input_addr = GetDeviceAddress(inputs, 0); + T *output_addr = GetDeviceAddress(outputs, 0); + + switch (nccl_kernel_type_) { + case NCCL_ALL_REDUCE: { + auto all_reduce_funcptr = + reinterpret_cast(dlsym(const_cast(collective_handle_), "AllReduce")); + MS_EXCEPTION_IF_NULL(all_reduce_funcptr); + CHECK_NCCL_RET_WITH_EXCEPT( + (*all_reduce_funcptr)(input_addr, output_addr, output_size_ / sizeof(T), nccl_data_type_, nccl_reduce_type_, + reinterpret_cast(stream_ptr)), + "ncclAllReduce failed"); + break; + } + case NCCL_ALL_GATHER: { + auto all_gather_funcptr = + reinterpret_cast(dlsym(const_cast(collective_handle_), "AllGather")); + MS_EXCEPTION_IF_NULL(all_gather_funcptr); + CHECK_NCCL_RET_WITH_EXCEPT((*all_gather_funcptr)(input_addr, output_addr, input_size_ / sizeof(T), + nccl_data_type_, reinterpret_cast(stream_ptr)), + "ncclAllGather failed"); + break; + } + case NCCL_REDUCE_SCATTER: { + auto reduce_scatter_funcptr = + reinterpret_cast(dlsym(const_cast(collective_handle_), "ReduceScatter")); + MS_EXCEPTION_IF_NULL(reduce_scatter_funcptr); + CHECK_NCCL_RET_WITH_EXCEPT( + (*reduce_scatter_funcptr)(input_addr, output_addr, output_size_ / sizeof(T), nccl_data_type_, + nccl_reduce_type_, reinterpret_cast(stream_ptr)), + "ncclReduceScatter failed"); + break; + } + default: { + MS_LOG(EXCEPTION) << "Kernel type " << nccl_kernel_type_ << " is not supported."; + } + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + nccl_data_type_ = kNcclDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + for (size_t i = 0; i < input_num; ++i) { + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, i); + size_t size = sizeof(T); + for (size_t j = 0; j < shape.size(); j++) { + size *= IntToSize(shape[j]); + } + input_size_list_.push_back(size); + input_size_ += size; + } + for (size_t i = 0; i < output_num; ++i) { + auto shape = AnfAlgo::GetOutputInferShape(kernel_node, i); + size_t size = sizeof(T); + for (size_t j = 0; j < shape.size(); j++) { + size *= IntToSize(shape[j]); + } + output_size_list_.push_back(size); + output_size_ += size; + } + InferCommType(kernel_node); + collective_handle_ = device::gpu::CollectiveInitializer::instance().collective_handle(); + MS_EXCEPTION_IF_NULL(collective_handle_); + return true; + } + + protected: + void InitSizeLists() override { return; } + + private: + void InferCommType(const CNodePtr &kernel_node) { + std::string kernel_name = AnfAlgo::GetCNodeName(kernel_node); + auto iter = kNcclTypeMap.find(kernel_name); + if (iter == kNcclTypeMap.end()) { + MS_LOG(EXCEPTION) << "Kernel " << kernel_name << " is not supported."; + } else { + nccl_kernel_type_ = iter->second; + } + + auto reduce_op = AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("op"); + if (reduce_op) { + std::string type = GetValue(reduce_op); + if (type == "sum") { + nccl_reduce_type_ = ncclSum; + } else if (type == "max") { + nccl_reduce_type_ = ncclMax; + } else if (type == "min") { + nccl_reduce_type_ = ncclMin; + } else if (type == "prod") { + nccl_reduce_type_ = ncclProd; + } else { + MS_LOG(EXCEPTION) << "Nccl reduce type " << type << " is not supported."; + } + } + return; + } + + NcclKernelType nccl_kernel_type_; + ncclRedOp_t nccl_reduce_type_; + ncclDataType_t nccl_data_type_; + size_t input_size_; + size_t output_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + const void *collective_handle_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NCCL_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.cc b/mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.cc new file mode 100644 index 00000000000..ce6c9beeb7a --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/bias_add_grad_gpu_kenel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(BiasAddGrad, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + BiasAddGradGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(BiasAddGrad, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + BiasAddGradGpuKernel, float16) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.h b/mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.h new file mode 100644 index 00000000000..b0e8102ee3a --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/bias_add_grad_gpu_kenel.h @@ -0,0 +1,159 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_BIAS_ADD_GRAD_GPU_KENEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_BIAS_ADD_GRAD_GPU_KENEL_H_ + +#include +#include +#include +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "dataset/util/make_unique.h" + +namespace mindspore { +namespace kernel { +template +class BiasAddGradGpuKernel : public GpuKernel { + public: + BiasAddGradGpuKernel() + : same_dims_(true), + cudnn_handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + dy_desc_(nullptr), + db_desc_(nullptr), + op_desc_(nullptr) {} + ~BiasAddGradGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *dy_addr = GetDeviceAddress(inputs, 0); + T *db_addr = GetDeviceAddress(outputs, 0); + T *indices_addr = GetDeviceAddress(workspace, 0); + T *workspace_addr = GetDeviceAddress(workspace, 1); + + const float alpha = 1; + const float beta = 0; + if (same_dims_) { + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(db_addr, dy_addr, output_size_list_[0], cudaMemcpyDeviceToDevice, + reinterpret_cast(stream_ptr)), + "cudaMemcpyAsync failed."); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnReduceTensor(cudnn_handle_, op_desc_, indices_addr, workspace_size_list_[0], workspace_addr, + workspace_size_list_[1], &alpha, dy_desc_, dy_addr, &beta, db_desc_, db_addr), + "cudnnReduceTensor failed"); + } + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto dy_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto num_dims = dy_shape.size(); + if (num_dims < 2) { + MS_LOG(EXCEPTION) << "input dims must be at least 2, but got " << num_dims; + } + + std::string format = GetAttr(kernel_node, "data_format"); + string::size_type pos = format.find("C"); + if (pos == std::string::npos || pos >= num_dims) { + MS_LOG(EXCEPTION) << "format '" << format << "' invalid"; + } + + // Expand to 4 dims for cudnnSetTensorNdDescriptorEx. + auto cudnn_dims = std::max(num_dims, 4UL); + std::unique_ptr dy_dims = mindspore::make_unique(cudnn_dims); + std::unique_ptr db_dims = mindspore::make_unique(cudnn_dims); + for (size_t i = 0; i < cudnn_dims; i++) { + dy_dims[i] = (i < num_dims) ? SizeToInt(dy_shape[i]) : 1; + db_dims[i] = (i == pos) ? SizeToInt(dy_shape[i]) : 1; + + if (dy_dims[i] != db_dims[i]) { + same_dims_ = false; + } + } + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(cudnn_dims), dy_dims.get()), + "cudnnSetTensorNdDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(db_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(cudnn_dims), db_dims.get()), + "cudnnSetTensorNdDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetReduceTensorDescriptor(op_desc_, CUDNN_REDUCE_TENSOR_ADD, cudnn_data_type_, CUDNN_NOT_PROPAGATE_NAN, + CUDNN_REDUCE_TENSOR_NO_INDICES, CUDNN_32BIT_INDICES), + "cudnnSetReduceTensorDescriptor failed"); + + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dy_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&db_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateReduceTensorDescriptor(&op_desc_), "cudnnCreateOpTensorDescriptor failed"); + } + void InitSizeLists() override { + size_t dy_size, db_size; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dy_desc_, &dy_size), "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(db_desc_, &db_size), "cudnnGetTensorSizeInBytes failed"); + input_size_list_.push_back(dy_size); + output_size_list_.push_back(db_size); + + size_t indices_size, workspace_size; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetReductionIndicesSize(cudnn_handle_, op_desc_, dy_desc_, db_desc_, &indices_size), + "cudnnGetReductionIndicesSize failed") + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetReductionWorkspaceSize(cudnn_handle_, op_desc_, dy_desc_, db_desc_, &workspace_size), + "cudnnGetReductionWorkspaceSize failed") + workspace_size_list_.push_back(indices_size); + workspace_size_list_.push_back(workspace_size); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnDestroyReduceTensorDescriptor(op_desc_), + "cudnnDestroyReduceTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(db_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dy_desc_), "cudnnDestroyOpTensorDescriptor failed"); + } + + bool same_dims_; + cudnnHandle_t cudnn_handle_; + cudnnDataType_t cudnn_data_type_; + cudnnTensorDescriptor_t dy_desc_; + cudnnTensorDescriptor_t db_desc_; + cudnnReduceTensorDescriptor_t op_desc_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_BIAS_ADD_GRAD_GPU_KENEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/conv2d_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/conv2d_gpu_kernel.cc new file mode 100644 index 00000000000..df6825e0793 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/conv2d_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/conv2d_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + Conv2D, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + Conv2dGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + Conv2D, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + Conv2dGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/conv2d_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/conv2d_gpu_kernel.h new file mode 100644 index 00000000000..7a4adff970c --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/conv2d_gpu_kernel.h @@ -0,0 +1,300 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2DGPUKERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2DGPUKERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/pad_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class Conv2dGpuFwdKernel : public GpuKernel { + public: + Conv2dGpuFwdKernel() + : cudnn_handle_(nullptr), + input_desc_(nullptr), + output_desc_(nullptr), + filter_desc_(nullptr), + conv_desc_(nullptr), + padded_desc_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + old_height_(0), + old_width_(0), + pad_height_(0), + pad_width_(0), + pad_top_(0), + pad_left_(0), + n_(0), + c_(0), + stride_(1), + dilation_(0), + group_(1), + is_null_input_(false), + input_size_(0), + filter_size_(0), + output_size_(0), + padded_size_(0), + workspace_size_(0), + use_pad_(true) {} + ~Conv2dGpuFwdKernel() override { DestroyResource(); } + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *input_addr = GetDeviceAddress(inputs, 0); + T *filter_addr = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + T *workspace_addr = nullptr; + if (workspace_size_ != 0) { + workspace_addr = GetDeviceAddress(workspace, 0); + } + + const float alpha = 1; + const float beta = 0; + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_) { + T *padded_addr = GetDeviceAddress(workspace, 1); + CalPad(padded_size_ / sizeof(T), input_addr, n_, c_, old_height_, old_width_, old_height_ + pad_height_, + old_width_ + pad_width_, pad_top_, pad_left_, pad_value_, padded_addr, + reinterpret_cast(stream_ptr)); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnConvolutionForward(cudnn_handle_, &alpha, padded_desc_, padded_addr, filter_desc_, filter_addr, conv_desc_, + conv_algorithm_, workspace_addr, workspace_size_, &beta, output_desc_, output_addr), + "cudnnConvolutionForward failed"); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnConvolutionForward(cudnn_handle_, &alpha, input_desc_, input_addr, filter_desc_, filter_addr, conv_desc_, + conv_algorithm_, workspace_addr, workspace_size_, &beta, output_desc_, output_addr), + "cudnnConvolutionForward failed"); + } + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + if (!CheckParam(kernel_node)) { + return false; + } + auto in_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto filter_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(in_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "Conv2dGpuFwdKernel input is null."; + InitSizeLists(); + return true; + } + Set4DDesc(in_shape, filter_shape, output_shape); + group_ = GetAttr(kernel_node, "group"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetConvolutionGroupCount(conv_desc_, group_), "cudnnSetConvGroupCount failed"); + pad_height_ = GetAttr(kernel_node, "pad"); + pad_width_ = pad_height_; + stride_ = GetAttr(kernel_node, "stride"); + dilation_ = GetAttr(kernel_node, "dilation"); + pad_mode_ = GetAttr(kernel_node, "pad_mode"); + cudnnTensorDescriptor_t input_descriptor_real = nullptr; + if (pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) { + SetPad(in_shape, kernel_node); + input_descriptor_real = use_pad_ ? padded_desc_ : input_desc_; + } else { + if (pad_mode_ == kValidPadModeUpperCase || pad_mode_ == kValidPadModeLowerCase) { + pad_height_ = 0; + pad_width_ = 0; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetConvolution2dDescriptor(conv_desc_, pad_height_, pad_width_, stride_, stride_, dilation_, dilation_, + CUDNN_CROSS_CORRELATION, cudnn_data_type_), + "cudnnSetConvolution2dDescriptor failed"); + input_descriptor_real = input_desc_; + } + SelectAlgorithm(input_descriptor_real); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&input_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&output_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&padded_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateFilterDescriptor(&filter_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateConvolutionDescriptor(&conv_desc_), + "cudnnCreateConvolutionDescriptor failed"); + } + + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(input_desc_, reinterpret_cast(&input_size_)), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetFilterSizeInBytes(filter_desc_, reinterpret_cast(&filter_size_)), + "cudnnGetFilterSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(output_desc_, reinterpret_cast(&output_size_)), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(padded_desc_, reinterpret_cast(&padded_size_)), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + input_size_list_.push_back(filter_size_); + output_size_list_.push_back(output_size_); + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_ && !is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionForwardWorkspaceSize(cudnn_handle_, padded_desc_, filter_desc_, conv_desc_, output_desc_, + conv_algorithm_, &workspace_size_), + "cudnnGetConvolutionForwardWorkspaceSize failed"); + workspace_size_list_.push_back(padded_size_); + } else { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionForwardWorkspaceSize(cudnn_handle_, input_desc_, filter_desc_, conv_desc_, output_desc_, + conv_algorithm_, &workspace_size_), + "cudnnGetConvolutionForwardWorkspaceSize failed"); + } + } + (void)workspace_size_list_.insert(workspace_size_list_.begin(), workspace_size_); + + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyConvolutionDescriptor(conv_desc_), + "cudnnDestroyConvolutionDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyFilterDescriptor(filter_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(padded_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(output_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(input_desc_), "cudnnDestroyTensorDescriptor failed"); + } + bool CheckParam(const CNodePtr &kernel_node) { + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but conv2d needs 2 inputs."; + return false; + } + + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but conv2d needs 1 output."; + return false; + } + return true; + } + void SetPad(const std::vector &in_shape, const CNodePtr &kernel_node) { + auto pad_list = GetAttr>(kernel_node, "pad_list"); + + n_ = SizeToInt(in_shape[0]); + c_ = SizeToInt(in_shape[1]); + old_height_ = SizeToInt(in_shape[2]); + old_width_ = SizeToInt(in_shape[3]); + pad_height_ = pad_list[0] + pad_list[1]; + pad_width_ = pad_list[2] + pad_list[3]; + pad_top_ = pad_list[0]; + pad_left_ = pad_list[2]; + + // if use_pad_ == true, using zero padding in advance, else using the default cudnn pad. + if (pad_height_ % 2 == 0 && pad_width_ % 2 == 0) { + use_pad_ = false; + } + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(padded_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, n_, c_, + old_height_ + pad_height_, old_width_ + pad_width_), + "cudnnSetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetConvolution2dDescriptor(conv_desc_, use_pad_ ? 0 : pad_top_, use_pad_ ? 0 : pad_left_, stride_, stride_, + dilation_, dilation_, CUDNN_CROSS_CORRELATION, cudnn_data_type_), + "cudnnSetConvolution2dDescriptor failed"); + } + + void Set4DDesc(const std::vector &in_shape, const std::vector &filter_shape, + const std::vector &output_shape) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(input_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(in_shape[0]), + SizeToInt(in_shape[1]), SizeToInt(in_shape[2]), SizeToInt(in_shape[3])), + "cudnnSetTensor4dDescriptor failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetFilter4dDescriptor(filter_desc_, cudnn_data_type_, CUDNN_TENSOR_NCHW, SizeToInt(filter_shape[0]), + SizeToInt(filter_shape[1]), SizeToInt(filter_shape[2]), SizeToInt(filter_shape[3])), + "cudnnSetFilter4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(output_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(output_shape[0]), + SizeToInt(output_shape[1]), SizeToInt(output_shape[2]), SizeToInt(output_shape[3])), + "cudnnSetTensor4dDescriptor failed"); + } + void SelectAlgorithm(cudnnTensorDescriptor_t input_descriptor_real) { + if (group_ > 1 || CUDNN_MAJOR < 7) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetConvolutionForwardAlgorithm( + cudnn_handle_, input_descriptor_real, filter_desc_, conv_desc_, output_desc_, + CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, 0, &conv_algorithm_), + "cudnnGetConvolutionForwardAlgorithm failed"); + } else { + constexpr int requested_algo_count = 1; + int returned_algo_count; + cudnnConvolutionFwdAlgoPerf_t perf_results; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionForwardAlgorithm_v7(cudnn_handle_, input_descriptor_real, filter_desc_, conv_desc_, + output_desc_, requested_algo_count, &returned_algo_count, &perf_results), + "cudnnGetConvolutionForwardAlgorithm_v7 failed"); + conv_algorithm_ = perf_results.algo; + } + } + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t input_desc_; + cudnnTensorDescriptor_t output_desc_; + cudnnFilterDescriptor_t filter_desc_; + cudnnConvolutionFwdAlgo_t conv_algorithm_; + cudnnConvolutionDescriptor_t conv_desc_; + cudnnTensorDescriptor_t padded_desc_; + std::string pad_mode_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + const float pad_value_ = 0.0; + cudnnDataType_t cudnn_data_type_; + int old_height_; + int old_width_; + int pad_height_; + int pad_width_; + int pad_top_; + int pad_left_; + int n_; + int c_; + int stride_; + int dilation_; + int group_; + bool is_null_input_; + size_t input_size_; + size_t filter_size_; + size_t output_size_; + size_t padded_size_; + size_t workspace_size_; + bool use_pad_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2DGPUKERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.cc new file mode 100644 index 00000000000..28e9a10ccc2 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + Conv2DBackpropFilter, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ConvGradFilterGpuBkwKernel, float) +MS_REG_GPU_KERNEL_ONE( + Conv2DBackpropFilter, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ConvGradFilterGpuBkwKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.h new file mode 100644 index 00000000000..f8afad4f847 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_filter_gpu_kernel.h @@ -0,0 +1,308 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2D_GRAD_FILTER_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2D_GRAD_FILTER_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/pad_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class ConvGradFilterGpuBkwKernel : public GpuKernel { + public: + ConvGradFilterGpuBkwKernel() + : cudnn_handle_(nullptr), + dw_desc_(nullptr), + conv_desc_(nullptr), + dy_desc_(nullptr), + x_desc_(nullptr), + padded_descriptor_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + old_height_(0), + old_width_(0), + pad_height_(0), + pad_width_(0), + pad_top_(0), + pad_left_(0), + n_(0), + c_(0), + stride_(1), + dilation_(0), + group_(1), + is_null_input_(false), + input_size_(0), + output_size_(0), + padded_size_(0), + workspace_size_(0), + use_pad_(true) {} + ~ConvGradFilterGpuBkwKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *dy = GetDeviceAddress(inputs, 0); + T *x = GetDeviceAddress(inputs, 1); + T *dw = GetDeviceAddress(outputs, 0); + T *work_space = nullptr; + if (workspace_size_ != 0) { + work_space = GetDeviceAddress(workspace, 0); + } + + const float alpha = 1; + const float beta = 0; + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_) { + T *padded = GetDeviceAddress(workspace, 1); + CalPad(padded_size_ / sizeof(T), x, n_, c_, old_height_, old_width_, old_height_ + pad_height_, + old_width_ + pad_width_, pad_top_, pad_left_, pad_value_, padded, + reinterpret_cast(stream_ptr)); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnConvolutionBackwardFilter(cudnn_handle_, &alpha, padded_descriptor_, padded, dy_desc_, dy, conv_desc_, + algo_, work_space, workspace_size_, &beta, dw_desc_, dw), + "ConvolutionBackwardFilter failed"); + return true; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnConvolutionBackwardFilter(cudnn_handle_, &alpha, x_desc_, x, dy_desc_, dy, conv_desc_, algo_, work_space, + workspace_size_, &beta, dw_desc_, dw), + "ConvolutionBackwardFilter failed"); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + if (!CheckParam(kernel_node)) { + return false; + } + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto dy_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto in_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(dy_shape) || CHECK_NULL_INPUT(in_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "ConvGradFilterGpuBkwKernel input is null."; + InitSizeLists(); + return true; + } + std::vector filter_shape; + GetFilterShape(kernel_node, &filter_shape); + Set4DDesc(dy_shape, filter_shape, in_shape); + group_ = GetAttr(kernel_node, "group"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetConvolutionGroupCount(conv_desc_, group_), "cudnnSetConvGroupCount failed"); + + pad_height_ = GetAttr(kernel_node, "pad"); + pad_width_ = pad_height_; + stride_ = GetAttr(kernel_node, "stride"); + dilation_ = GetAttr(kernel_node, "dilation"); + pad_mode_ = GetAttr(kernel_node, "pad_mode"); + cudnnTensorDescriptor_t x_desc_real = nullptr; + if (pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) { + SetPad(in_shape, kernel_node); + x_desc_real = use_pad_ ? padded_descriptor_ : x_desc_; + } else { + if (pad_mode_ == kValidPadModeUpperCase || pad_mode_ == kValidPadModeLowerCase) { + pad_height_ = 0; + pad_width_ = 0; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetConvolution2dDescriptor(conv_desc_, pad_height_, pad_width_, stride_, stride_, dilation_, dilation_, + CUDNN_CROSS_CORRELATION, cudnn_data_type_), + "GetConvolution2dDescriptor failed"); + x_desc_real = x_desc_; + } + SelectAlgorithm(x_desc_real); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dy_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&padded_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateFilterDescriptor(&dw_desc_), "cudnnCreateFilterDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateConvolutionDescriptor(&conv_desc_), + "cudnnCreateConvolutionDescriptor failed"); + } + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dy_desc_, reinterpret_cast(&input_size_)), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetFilterSizeInBytes(dw_desc_, reinterpret_cast(&output_size_)), + "cudnnGetFilterSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(x_desc_, reinterpret_cast(&input_size_)), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetFilterSizeInBytes(dw_desc_, reinterpret_cast(&output_size_)), + "cudnnGetFilterSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + input_size_list_.push_back(output_size_); + + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_ && !is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetTensorSizeInBytes(padded_descriptor_, reinterpret_cast(&padded_size_)), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardFilterWorkspaceSize(cudnn_handle_, padded_descriptor_, dy_desc_, conv_desc_, + dw_desc_, algo_, reinterpret_cast(&workspace_size_)), + "cudnnGetConvolutionBackwardFilterWorkspaceSize failed"); + workspace_size_list_.push_back(padded_size_); + } else { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardFilterWorkspaceSize(cudnn_handle_, x_desc_, dy_desc_, conv_desc_, dw_desc_, algo_, + reinterpret_cast(&workspace_size_)), + "cudnnGetConvolutionBackwardFilterWorkspaceSize failed"); + } + } + (void)workspace_size_list_.insert(workspace_size_list_.begin(), workspace_size_); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyConvolutionDescriptor(conv_desc_), + "cudnnDestroyConvolutionDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyFilterDescriptor(dw_desc_), "cudnnDestroyFilterDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(padded_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dy_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_desc_), "cudnnDestroyTensorDescriptor failed"); + } + bool CheckParam(const CNodePtr &kernel_node) { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but ConvGradFilter needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but ConvGradFilter needs 1 output."; + return false; + } + return true; + } + void SetPad(const std::vector &in_shape, const CNodePtr &kernel_node) { + auto pad_list = GetAttr>(kernel_node, "pad_list"); + n_ = SizeToInt(in_shape[0]); + c_ = SizeToInt(in_shape[1]); + old_height_ = SizeToInt(in_shape[2]); + old_width_ = SizeToInt(in_shape[3]); + pad_height_ = pad_list[0] + pad_list[1]; + pad_width_ = pad_list[2] + pad_list[3]; + pad_top_ = pad_list[0]; + pad_left_ = pad_list[2]; + if (pad_height_ % 2 == 0 && pad_width_ % 2 == 0) { + use_pad_ = false; + } + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(padded_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, n_, + c_, old_height_ + pad_height_, old_width_ + pad_width_), + "cudnnSetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetConvolution2dDescriptor(conv_desc_, use_pad_ ? 0 : pad_top_, use_pad_ ? 0 : pad_left_, stride_, stride_, + dilation_, dilation_, CUDNN_CROSS_CORRELATION, cudnn_data_type_), + "cudnnSetConvolution2dDescriptor failed"); + } + void SelectAlgorithm(cudnnTensorDescriptor_t x_desc_real) { + if (group_ > 1 || CUDNN_MAJOR < 7) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardFilterAlgorithm(cudnn_handle_, x_desc_real, dy_desc_, conv_desc_, dw_desc_, + CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, 0, &algo_), + "GetConvolutionBackwardFilterAlgorithm failed"); + } else { + constexpr int requested_algo_count = 1; + int returned_algo_count; + cudnnConvolutionBwdFilterAlgoPerf_t perf_results; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardFilterAlgorithm_v7(cudnn_handle_, x_desc_real, dy_desc_, conv_desc_, dw_desc_, + requested_algo_count, &returned_algo_count, &perf_results), + "GetConvolutionBackwardFilterAlgorithm failed"); + algo_ = perf_results.algo; + } + } + void GetFilterShape(const CNodePtr &kernel_node, std::vector *filter_shape) { + auto shp_tuple_x = AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("filter_sizes")->cast()->value(); + (void)std::transform(std::begin(shp_tuple_x), std::end(shp_tuple_x), std::back_inserter(*filter_shape), + [](const ValuePtr &e) -> int { return e->cast()->value(); }); + } + void Set4DDesc(const std::vector &dy_shape, const std::vector &filter_shape, + const std::vector &in_shape) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(dy_shape[0]), + SizeToInt(dy_shape[1]), SizeToInt(dy_shape[2]), SizeToInt(dy_shape[3])), + "SetTensor4dDescriptor failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetFilter4dDescriptor(dw_desc_, cudnn_data_type_, CUDNN_TENSOR_NCHW, SizeToInt(dy_shape[1]), filter_shape[1], + filter_shape[2], filter_shape[3]), + "SetFilter4dDescriptor failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(x_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(in_shape[0]), + SizeToInt(in_shape[1]), SizeToInt(in_shape[2]), SizeToInt(in_shape[3])), + "SetTensor4dDescriptor failed"); + } + + cudnnHandle_t cudnn_handle_; + cudnnFilterDescriptor_t dw_desc_; + cudnnConvolutionDescriptor_t conv_desc_; + cudnnTensorDescriptor_t dy_desc_; + cudnnTensorDescriptor_t x_desc_; + cudnnTensorDescriptor_t padded_descriptor_; + cudnnConvolutionBwdFilterAlgo_t algo_; + std::string pad_mode_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + const float pad_value_ = 0.0; + cudnnDataType_t cudnn_data_type_; + int old_height_; + int old_width_; + int pad_height_; + int pad_width_; + int pad_top_; + int pad_left_; + int n_; + int c_; + int stride_; + int dilation_; + int group_; + bool is_null_input_; + size_t input_size_; + size_t output_size_; + size_t padded_size_; + size_t workspace_size_; + bool use_pad_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2D_GRAD_FILTER_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_input_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_input_gpu_kernel.cc new file mode 100644 index 00000000000..12b6f91537e --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_input_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/conv2d_grad_input_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + Conv2DBackpropInput, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ConvGradInputGpuBkwKernel, float) +MS_REG_GPU_KERNEL_ONE( + Conv2DBackpropInput, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ConvGradInputGpuBkwKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_input_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_input_gpu_kernel.h new file mode 100644 index 00000000000..be7739981da --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/conv2d_grad_input_gpu_kernel.h @@ -0,0 +1,304 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2D_GRAD_INPUT_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2D_GRAD_INPUT_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/pad_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class ConvGradInputGpuBkwKernel : public GpuKernel { + public: + ConvGradInputGpuBkwKernel() + : cudnn_handle_(nullptr), + w_desc_(nullptr), + conv_desc_(nullptr), + dy_desc_(nullptr), + dx_desc_(nullptr), + padded_descriptor_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT), + old_height_(0), + old_width_(0), + pad_height_(0), + pad_width_(0), + pad_top_(0), + pad_left_(0), + n_(0), + c_(0), + stride_(1), + dilation_(0), + group_(1), + is_null_input_(false), + input_size_(0), + output_size_(0), + padded_size_(0), + workspace_size_(0), + use_pad_(true) {} + ~ConvGradInputGpuBkwKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *dy = GetDeviceAddress(inputs, 0); + T *w = GetDeviceAddress(inputs, 1); + T *dx = GetDeviceAddress(outputs, 0); + T *work_space = nullptr; + if (workspace_size_ != 0) { + work_space = GetDeviceAddress(workspace, 0); + } + + const float alpha = 1; + const float beta = 0; + + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_) { + T *padded = GetDeviceAddress(workspace, 1); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnConvolutionBackwardData(cudnn_handle_, &alpha, w_desc_, w, dy_desc_, dy, conv_desc_, algo_, work_space, + workspace_size_, &beta, padded_descriptor_, padded), + "ConvolutionBackwardData failed"); + CalPadGrad(padded_size_ / sizeof(T), padded, n_, c_, old_height_, old_width_, old_height_ + pad_height_, + old_width_ + pad_width_, pad_top_, pad_left_, dx, reinterpret_cast(stream_ptr)); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnConvolutionBackwardData(cudnn_handle_, &alpha, w_desc_, w, dy_desc_, dy, conv_desc_, algo_, work_space, + workspace_size_, &beta, dx_desc_, dx), + "ConvolutionBackwardData failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + if (!CheckParam(kernel_node)) { + return false; + } + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto dy_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto filter_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(dy_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "ConvGradInputGpuBkwKernel input is null."; + InitSizeLists(); + return true; + } + std::vector input_shape; + GetInputShape(kernel_node, &input_shape); + Set4DDesc(dy_shape, input_shape, filter_shape); + + group_ = GetAttr(kernel_node, "group"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetConvolutionGroupCount(conv_desc_, group_), "cudnnSetConvGroupCount failed"); + + pad_height_ = GetAttr(kernel_node, "pad"); + pad_width_ = pad_height_; + stride_ = GetAttr(kernel_node, "stride"); + dilation_ = GetAttr(kernel_node, "dilation"); + pad_mode_ = GetAttr(kernel_node, "pad_mode"); + cudnnTensorDescriptor_t dx_desc_real = nullptr; + if (pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) { + SetPad(input_shape, kernel_node); + dx_desc_real = use_pad_ ? padded_descriptor_ : dx_desc_; + } else { + if (pad_mode_ == kValidPadModeUpperCase || pad_mode_ == kValidPadModeLowerCase) { + pad_height_ = 0; + pad_width_ = 0; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetConvolution2dDescriptor(conv_desc_, pad_height_, pad_width_, stride_, stride_, dilation_, dilation_, + CUDNN_CROSS_CORRELATION, cudnn_data_type_), + "cudnnSetConvolution2dDescriptor failed"); + dx_desc_real = dx_desc_; + } + SelectAlgorithm(dx_desc_real); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dy_desc_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&padded_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateFilterDescriptor(&w_desc_), "cudnnCreateFilterDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateConvolutionDescriptor(&conv_desc_), + "cudnnCreateConvolutionDescriptor failed"); + } + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dy_desc_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dx_desc_, &output_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dx_desc_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetFilterSizeInBytes(w_desc_, &output_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + input_size_list_.push_back(output_size_); + + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_ && !is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(padded_descriptor_, &padded_size_), + "cudnnGetTensorSizeInBytes failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardDataWorkspaceSize(cudnn_handle_, w_desc_, dy_desc_, conv_desc_, padded_descriptor_, + algo_, &workspace_size_), + "cudnnGetConvolutionBackwardDataWorkspaceSize failed"); + workspace_size_list_.push_back(padded_size_); + } else { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetConvolutionBackwardDataWorkspaceSize( + cudnn_handle_, w_desc_, dy_desc_, conv_desc_, dx_desc_, algo_, &workspace_size_), + "cudnnGetConvolutionBackwardDataWorkspaceSize failed"); + } + } + (void)workspace_size_list_.insert(workspace_size_list_.begin(), workspace_size_); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyConvolutionDescriptor(conv_desc_), + "cudnnDestroyConvolutionDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyFilterDescriptor(w_desc_), "cudnnDestroyFilterDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(padded_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dy_desc_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dx_desc_), "cudnnDestroyTensorDescriptor failed"); + } + bool CheckParam(const CNodePtr &kernel_node) { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but ConvGradInput needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but ConvGradInput needs 1 output."; + return false; + } + return true; + } + void SetPad(const std::vector &input_shape, const CNodePtr &kernel_node) { + auto pad_list = GetAttr>(kernel_node, "pad_list"); + n_ = input_shape[0]; + c_ = input_shape[1]; + old_height_ = input_shape[2]; + old_width_ = input_shape[3]; + pad_height_ = pad_list[0] + pad_list[1]; + pad_width_ = pad_list[2] + pad_list[3]; + pad_top_ = pad_list[0]; + pad_left_ = pad_list[2]; + if (pad_height_ % 2 == 0 && pad_width_ % 2 == 0) { + use_pad_ = false; + } + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(padded_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, n_, + c_, old_height_ + pad_height_, old_width_ + pad_width_), + "cudnnSetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetConvolution2dDescriptor(conv_desc_, use_pad_ ? 0 : pad_top_, use_pad_ ? 0 : pad_left_, stride_, stride_, + dilation_, dilation_, CUDNN_CROSS_CORRELATION, cudnn_data_type_), + "cudnnSetConvolution2dDescriptor failed"); + } + void SelectAlgorithm(cudnnTensorDescriptor_t dx_desc_real) { + if (group_ > 1 || CUDNN_MAJOR < 7) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardDataAlgorithm(cudnn_handle_, w_desc_, dy_desc_, conv_desc_, dx_desc_real, + CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, 0, &algo_), + "cudnnGetConvolutionBackwardDataAlgorithm failed"); + } else { + constexpr int requested_algo_count = 1; + int returned_algo_count; + cudnnConvolutionBwdDataAlgoPerf_t perf_results; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetConvolutionBackwardDataAlgorithm_v7(cudnn_handle_, w_desc_, dy_desc_, conv_desc_, dx_desc_real, + requested_algo_count, &returned_algo_count, &perf_results), + "cudnnGetConvolutionBackwardDataAlgorithm_v7 failed"); + algo_ = perf_results.algo; + } + } + void GetInputShape(const CNodePtr &kernel_node, std::vector *input_shape) { + auto shp_tuple_x = AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("input_sizes")->cast()->value(); + (void)std::transform(std::begin(shp_tuple_x), std::end(shp_tuple_x), std::back_inserter(*input_shape), + [](const ValuePtr &e) -> int { return e->cast()->value(); }); + } + void Set4DDesc(const std::vector &dy_shape, const std::vector &input_shape, + const std::vector &filter_shape) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetFilter4dDescriptor(w_desc_, cudnn_data_type_, CUDNN_TENSOR_NCHW, SizeToInt(dy_shape[1]), + SizeToInt(filter_shape[1]), SizeToInt(filter_shape[2]), SizeToInt(filter_shape[3])), + "SetFilter4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(dy_shape[0]), + SizeToInt(dy_shape[1]), SizeToInt(dy_shape[2]), SizeToInt(dy_shape[3])), + "SetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, input_shape[0], input_shape[1], + input_shape[2], input_shape[3]), + "SetTensor4dDescriptor failed"); + } + cudnnHandle_t cudnn_handle_; + cudnnFilterDescriptor_t w_desc_; + cudnnConvolutionDescriptor_t conv_desc_; + cudnnTensorDescriptor_t dy_desc_; + cudnnTensorDescriptor_t dx_desc_; + cudnnTensorDescriptor_t padded_descriptor_; + cudnnConvolutionBwdDataAlgo_t algo_; + std::string pad_mode_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + cudnnDataType_t cudnn_data_type_; + int old_height_; + int old_width_; + int pad_height_; + int pad_width_; + int pad_top_; + int pad_left_; + int n_; + int c_; + int stride_; + int dilation_; + int group_; + bool is_null_input_; + size_t input_size_; + size_t output_size_; + size_t padded_size_; + size_t workspace_size_; + bool use_pad_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_CONV2D_GRAD_INPUT_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/flatten_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/flatten_gpu_kernel.cc new file mode 100644 index 00000000000..f9c993d31d1 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/flatten_gpu_kernel.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/flatten_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(Flatten, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + FlattenGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(Flatten, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + FlattenGpuFwdKernel, int) +MS_REG_GPU_KERNEL_ONE(Flatten, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + FlattenGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(Reshape, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + FlattenGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(Reshape, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + FlattenGpuFwdKernel, int) +MS_REG_GPU_KERNEL_ONE(Reshape, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + FlattenGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(ExpandDims, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + FlattenGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(ExpandDims, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + FlattenGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(ExpandDims, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + FlattenGpuFwdKernel, int) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/flatten_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/flatten_gpu_kernel.h new file mode 100644 index 00000000000..37d0aadfbca --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/flatten_gpu_kernel.h @@ -0,0 +1,82 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_FLATTEN_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_FLATTEN_GPU_KERNEL_H_ + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" + +namespace mindspore { +namespace kernel { +template +class FlattenGpuFwdKernel : public GpuKernel { + public: + FlattenGpuFwdKernel() : input_size_(0), output_size_(0), workspace_size_(0) {} + ~FlattenGpuFwdKernel() override = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t stream_ptr) override { + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + cudaError_t ret = + cudaMemcpyAsync(output, input, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)); + if (ret) { + MS_LOG(ERROR) << "cudaMemcpyAsync error in FlattenGpuFwdKernel::Launch, error code is " << ret; + return false; + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + for (size_t i = 0; i < shape.size(); ++i) { + if (input_size_ == 0) { + input_size_ = 1; + } + input_size_ *= shape[i]; + } + input_size_ = input_size_ * sizeof(T); + + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_ = input_size_; + output_size_list_.push_back(output_size_); + } + + private: + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_FLATTEN_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/flatten_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/flatten_grad_gpu_kernel.cc new file mode 100644 index 00000000000..0e079d137bc --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/flatten_grad_gpu_kernel.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/flatten_grad_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(FlattenGrad, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + FlattenGardGpuBkwKernel, float) +MS_REG_GPU_KERNEL_ONE(FlattenGrad, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + FlattenGardGpuBkwKernel, half) +MS_REG_GPU_KERNEL_ONE(FlattenGrad, KernelAttr().AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + FlattenGardGpuBkwKernel, int) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/flatten_grad_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/flatten_grad_gpu_kernel.h new file mode 100644 index 00000000000..bdae6404dcd --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/flatten_grad_gpu_kernel.h @@ -0,0 +1,89 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_FLATTEN_GRAD_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_FLATTEN_GRAD_GPU_KERNEL_H_ + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" + +namespace mindspore { +namespace kernel { +template +class FlattenGardGpuBkwKernel : public GpuKernel { + public: + FlattenGardGpuBkwKernel() : input_size_(0), output_size_(0), workspace_size_(0) {} + ~FlattenGardGpuBkwKernel() override = default; + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + cudaError_t ret = + cudaMemcpyAsync(output, input, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)); + if (ret) { + MS_LOG(ERROR) << "cudaMemcpyAsync error in FlattenGardGpuFwdKernel::Launch, error code is " << ret; + return false; + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Argument number is " << input_num << ", but FlattenGardGpuFwdKernel needs 1."; + return false; + } + + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + for (size_t i = 0; i < shape.size(); ++i) { + if (input_size_ == 0) { + input_size_ = 1; + } + input_size_ *= shape[i]; + } + input_size_ = input_size_ * sizeof(T); + + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_ = input_size_; + output_size_list_.push_back(output_size_); + } + + private: + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_FLATTEN_GRAD_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_batch_norm_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/fused_batch_norm_gpu_kernel.cc new file mode 100644 index 00000000000..4ddc710a4cb --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_batch_norm_gpu_kernel.cc @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/fused_batch_norm_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(FusedBatchNorm, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedBatchNormGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(FusedBatchNorm, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + FusedBatchNormGpuKernel, half) +MS_REG_GPU_KERNEL_ONE(BatchNorm, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedBatchNormGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(BatchNorm, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + FusedBatchNormGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_batch_norm_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/fused_batch_norm_gpu_kernel.h new file mode 100644 index 00000000000..6f0c59e29ac --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_batch_norm_gpu_kernel.h @@ -0,0 +1,192 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_BATCH_NORM_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_BATCH_NORM_GPU_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class FusedBatchNormGpuKernel : public GpuKernel { + public: + FusedBatchNormGpuKernel() + : batch_(0), + channel_(0), + height_(0), + width_(0), + mode_(CUDNN_BATCHNORM_SPATIAL), + epsilon_(10e-5), + exp_avg_factor_(0.1), + is_train_(false), + is_null_input_(false), + x_desc_(nullptr), + y_desc_(nullptr), + scale_bias_mean_var_desc_(nullptr), + handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT) {} + ~FusedBatchNormGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + VARIABLE_NOT_USED(stream_ptr); + if (is_null_input_) { + return true; + } + auto x = GetDeviceAddress(inputs, 0); + auto scale = GetDeviceAddress(inputs, 1); + auto bias = GetDeviceAddress(inputs, 2); + auto runing_mean = GetDeviceAddress(inputs, 3); + auto runnig_variance = GetDeviceAddress(inputs, 4); + auto y = GetDeviceAddress(outputs, 0); + + const float alpha = 1; + const float beta = 0; + if (is_train_) { + auto save_mean = GetDeviceAddress(outputs, 3); + auto save_variance = GetDeviceAddress(outputs, 4); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnBatchNormalizationForwardTraining(handle_, mode_, &alpha, &beta, x_desc_, x, y_desc_, y, + scale_bias_mean_var_desc_, scale, bias, exp_avg_factor_, runing_mean, + runnig_variance, epsilon_, save_mean, save_variance), + "Kernel launch failed"); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnBatchNormalizationForwardInference(handle_, mode_, &alpha, &beta, x_desc_, x, + y_desc_, y, scale_bias_mean_var_desc_, scale, + bias, runing_mean, runnig_variance, epsilon_), + "Kernel launch failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 5) { + MS_LOG(EXCEPTION) << "input tensor size is " << input_num << ", FusedBatchNormGpuKernel should be 5"; + } + + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + if (shape.size() != 4) { + MS_LOG(EXCEPTION) << "tensor shape is " << shape.size() << ", FusedBatchNormGpuKernel should be >= 4"; + } + is_null_input_ = CHECK_NULL_INPUT(shape); + if (is_null_input_) { + MS_LOG(WARNING) << "FusedBatchNormGpuKernel input is null"; + InitSizeLists(); + return true; + } + batch_ = SizeToInt(shape[0]); + channel_ = SizeToInt(shape[1]); + height_ = SizeToInt(shape[2]); + width_ = SizeToInt(shape[3]); + + mode_ = CUDNN_BATCHNORM_SPATIAL; + epsilon_ = GetAttr(kernel_node, "epsilon"); + // P.FusedBatchNorm is used for training; P.BatchNorm is used for inference + auto node_name = AnfAlgo::GetCNodeName(kernel_node); + if (node_name == "FusedBatchNorm") { + is_train_ = true; + exp_avg_factor_ = GetAttr(kernel_node, "momentum"); + } + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(x_desc_, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch_, channel_, height_, width_), + "Set x desc failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(y_desc_, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch_, channel_, height_, width_), + "Set y desc failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(scale_bias_mean_var_desc_, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 1, channel_, 1, 1), + "Set para desc failed"); + + InitSizeLists(); + + return true; + } + + protected: + void InitResource() override { + handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_), "Create x desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&y_desc_), "Create y desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&scale_bias_mean_var_desc_), "Create para desc failed"); + } + void InitSizeLists() override { + size_t input_size = 0; + size_t para_size = 0; + size_t output_size = 0; + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(x_desc_, &input_size), "Get input size failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(scale_bias_mean_var_desc_, ¶_size), + "Get para size failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(y_desc_, &output_size), "Get para size failed"); + } + input_size_list_.push_back(input_size); + input_size_list_.push_back(para_size); // scale + input_size_list_.push_back(para_size); // bias + input_size_list_.push_back(para_size); // mean + input_size_list_.push_back(para_size); // variance + + output_size_list_.push_back(output_size); + output_size_list_.push_back(para_size); // running mean + output_size_list_.push_back(para_size); // running variance + output_size_list_.push_back(para_size); // save mean + output_size_list_.push_back(para_size); // save variance + if (!is_train_) { + output_size_list_.push_back(para_size); // reserve + } + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_desc_), "Destroy x desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(y_desc_), "Destroy y desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(scale_bias_mean_var_desc_), "Destroy para desc failed"); + } + + int batch_; + int channel_; + int height_; + int width_; + cudnnBatchNormMode_t mode_; + double epsilon_; + double exp_avg_factor_; + bool is_train_; + bool is_null_input_; + cudnnTensorDescriptor_t x_desc_; + cudnnTensorDescriptor_t y_desc_; + cudnnTensorDescriptor_t scale_bias_mean_var_desc_; + cudnnHandle_t handle_; + cudnnDataType_t cudnn_data_type_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_BATCH_NORM_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.cc new file mode 100644 index 00000000000..3947aaea9ad --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(FusedBatchNormGrad, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedBatchNormGradGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(FusedBatchNormGrad, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + FusedBatchNormGradGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.h new file mode 100644 index 00000000000..08eac28af77 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_batchnorm_grad_gpu_kernel.h @@ -0,0 +1,178 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_BATCHNORM_GRAD_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_BATCHNORM_GRAD_GPU_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class FusedBatchNormGradGpuKernel : public GpuKernel { + public: + FusedBatchNormGradGpuKernel() + : batch_(0), + channel_(0), + height_(0), + width_(0), + mode_(CUDNN_BATCHNORM_SPATIAL), + epsilon_(10e-5), + is_null_input_(false), + x_desc_(nullptr), + dy_desc_(nullptr), + dx_desc_(nullptr), + scale_bias_desc_(nullptr), + handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT) {} + ~FusedBatchNormGradGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(workspace); + VARIABLE_NOT_USED(stream_ptr); + if (is_null_input_) { + return true; + } + auto dy = GetDeviceAddress(inputs, 0); + auto x = GetDeviceAddress(inputs, 1); + auto scale = GetDeviceAddress(inputs, 2); + auto save_mean = GetDeviceAddress(inputs, 3); + auto save_variance = GetDeviceAddress(inputs, 4); + auto dx = GetDeviceAddress(outputs, 0); + auto bn_scale = GetDeviceAddress(outputs, 1); + auto bn_bias = GetDeviceAddress(outputs, 2); + + const float alpha_data_diff = 1; + const float beta_data_diff = 0; + const float alpha_param_diff = 1; + const float beta_param_diff = 0; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnBatchNormalizationBackward(handle_, mode_, &alpha_data_diff, &beta_data_diff, &alpha_param_diff, + &beta_param_diff, x_desc_, x, dy_desc_, dy, dx_desc_, dx, scale_bias_desc_, scale, + bn_scale, bn_bias, epsilon_, save_mean, save_variance), + "Kernel Launch Failed."); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 5) { + MS_LOG(EXCEPTION) << "input tensor size is " << input_num << ", FusedBatchNormGradGpuKernel should be 5"; + } + + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + if (shape.size() != 4) { + MS_LOG(EXCEPTION) << "tensor shape is " << shape.size() << ", FusedBatchNormGradGpuKernel should be 4"; + return false; + } + is_null_input_ = CHECK_NULL_INPUT(shape); + if (is_null_input_) { + MS_LOG(WARNING) << "FusedBatchNormGradGpuKernel input is null"; + InitSizeLists(); + return true; + } + batch_ = SizeToInt(shape[0]); + channel_ = SizeToInt(shape[1]); + height_ = SizeToInt(shape[2]); + width_ = SizeToInt(shape[3]); + + mode_ = CUDNN_BATCHNORM_SPATIAL; + epsilon_ = GetAttr(kernel_node, "epsilon"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(x_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, batch_, channel_, height_, width_), + "Set x desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, batch_, channel_, height_, width_), + "Set dy desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, batch_, channel_, height_, width_), + "Set dx desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(scale_bias_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 1, channel_, 1, 1), + "Set para desc failed"); + + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_), "Create x desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dy_desc_), "Create dy desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_), "Create dx desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&scale_bias_desc_), "Create para desc failed"); + } + + void InitSizeLists() override { + size_t input_size = 0; + size_t para_size = 0; + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(x_desc_, &input_size), "Get input size failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(scale_bias_desc_, ¶_size), "Get input size failed"); + } + + input_size_list_.push_back(input_size); + input_size_list_.push_back(input_size); + input_size_list_.push_back(para_size); + input_size_list_.push_back(para_size); + input_size_list_.push_back(para_size); + + output_size_list_.push_back(input_size); + output_size_list_.push_back(para_size); + output_size_list_.push_back(para_size); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(scale_bias_desc_), "Destroy para desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dx_desc_), "Destroy dx desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dy_desc_), "Destroy dy desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_desc_), "Destroy x desc failed"); + } + + int batch_; + int channel_; + int height_; + int width_; + + cudnnBatchNormMode_t mode_; + double epsilon_; + bool is_null_input_; + cudnnTensorDescriptor_t x_desc_; + cudnnTensorDescriptor_t dy_desc_; + cudnnTensorDescriptor_t dx_desc_; + cudnnTensorDescriptor_t scale_bias_desc_; + + cudnnHandle_t handle_; + cudnnDataType_t cudnn_data_type_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_BATCHNORM_GRAD_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.cc new file mode 100644 index 00000000000..c745c216f7b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/lstm_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(LSTM, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + LstmGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(LSTM, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + LstmGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.h new file mode 100644 index 00000000000..2e284f72e86 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/lstm_gpu_kernel.h @@ -0,0 +1,248 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "dataset/util/make_unique.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class LstmGpuKernel : public GpuKernel { + public: + LstmGpuKernel() + : batch_size_(0), + seq_len_(0), + input_size_(0), + hidden_size_(0), + num_layers_(0), + has_bias_(false), + bidirectional_(false), + states_init_(false), + dropout_(0), + weight_size_(0), + reserved_size_(0), + x_desc_(nullptr), + hx_desc_(nullptr), + cx_desc_(nullptr), + w_desc_(nullptr), + dropout_desc_(nullptr), + y_desc_(nullptr), + hy_desc_(nullptr), + cy_desc_(nullptr), + rnn_desc_(nullptr), + handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT) {} + ~LstmGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(stream_ptr); + auto x_addr = GetDeviceAddress(inputs, 0); + auto hx_addr = GetDeviceAddress(inputs, 1); + auto cx_addr = GetDeviceAddress(inputs, 2); + auto w_addr = GetDeviceAddress(inputs, 3); + auto y_addr = GetDeviceAddress(outputs, 0); + auto hy_addr = GetDeviceAddress(outputs, 1); + auto cy_addr = GetDeviceAddress(outputs, 2); + auto reserved_addr = GetDeviceAddress(outputs, 3); + auto states_addr = GetDeviceAddress(outputs, 4); + void *workspace_addr = GetDeviceAddress(workspace, 0); + + if (!states_init_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetDropoutDescriptor(dropout_desc_, handle_, dropout_, states_addr, output_size_list_[4], 0), + "set dropout_desc failed"); + states_init_ = true; + } + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnRNNForwardTraining(handle_, rnn_desc_, seq_len_, x_desc_.get(), x_addr, hx_desc_, hx_addr, cx_desc_, cx_addr, + w_desc_, w_addr, y_desc_.get(), y_addr, hy_desc_, hy_addr, cy_desc_, cy_addr, + workspace_addr, workspace_size_list_[0], reserved_addr, reserved_size_), + "launch lstm kernel failed"); + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + seq_len_ = SizeToInt(input_shape[0]); + batch_size_ = SizeToInt(input_shape[1]); + input_size_ = SizeToInt(input_shape[2]); + + input_size_ = GetAttr(kernel_node, "input_size"); + hidden_size_ = GetAttr(kernel_node, "hidden_size"); + num_layers_ = GetAttr(kernel_node, "num_layers"); + has_bias_ = GetAttr(kernel_node, "has_bias"); + bidirectional_ = GetAttr(kernel_node, "bidirectional"); + dropout_ = GetAttr(kernel_node, "dropout"); + + cudnnRNNInputMode_t input_mode = CUDNN_LINEAR_INPUT; + cudnnDirectionMode_t direction = bidirectional_ ? CUDNN_BIDIRECTIONAL : CUDNN_UNIDIRECTIONAL; + cudnnRNNMode_t rnn_mode = CUDNN_LSTM; + cudnnRNNAlgo_t algo = CUDNN_RNN_ALGO_STANDARD; + CreateTensorDescGrp(); + int hx_dims[3]{num_layers_ * (bidirectional_ ? 2 : 1), batch_size_, hidden_size_}; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(hx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set hx_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(cx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set cx_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(hy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set hy_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(cy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set cy_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetDropoutDescriptor(dropout_desc_, handle_, dropout_, nullptr, 0, 0), + "set dropout_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetRNNDescriptor(handle_, rnn_desc_, hidden_size_, num_layers_, dropout_desc_, + input_mode, direction, rnn_mode, algo, cudnn_data_type_), + "set rnn_desc failed"); + cudnnRNNBiasMode_t bias_mode = has_bias_ ? CUDNN_RNN_DOUBLE_BIAS : CUDNN_RNN_NO_BIAS; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetRNNBiasMode(rnn_desc_, bias_mode), "set bias_mode failed"); + auto weight_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 3); + size_t weight_size = weight_shape[0] * weight_shape[1] * weight_shape[2] * sizeof(T); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetRNNParamsSize(handle_, rnn_desc_, x_desc_[0], &weight_size_, cudnn_data_type_), + "get weight_size_ failed"); + if (weight_size != weight_size_) { + MS_LOG(EXCEPTION) << "weight size: " << weight_size << " error, expect: " << weight_size_ << " ."; + } + int w_dims[3] = {SizeToInt(weight_size_ / 4), 1, 1}; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetFilterNdDescriptor(w_desc_, cudnn_data_type_, CUDNN_TENSOR_NCHW, 3, w_dims), + "set w_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetRNNTrainingReserveSize(handle_, rnn_desc_, seq_len_, x_desc_.get(), &reserved_size_), + "get reserve size failed"); + InitSizeLists(); + return true; + } + void CreateTensorDescGrp() { + int x_dims[3]{batch_size_, input_size_, 1}; + int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; + + x_desc_ = mindspore::make_unique(seq_len_); + y_desc_ = mindspore::make_unique(seq_len_); + + for (size_t i = 0; i < IntToSize(seq_len_); ++i) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(x_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, x_dims), "set x_desc failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&y_desc_[i]), "create y_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(y_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, y_dims), "set y_desc failed"); + } + } + + protected: + void InitResource() override { + handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&hx_desc_), "create hx_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&cx_desc_), "create cx_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateFilterDescriptor(&w_desc_), "create w_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&hy_desc_), "create hy_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&cy_desc_), "create cy_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateDropoutDescriptor(&dropout_desc_), "create dropout_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateRNNDescriptor(&rnn_desc_), "create rnn_desc failed"); + } + void InitSizeLists() override { + size_t x_size = IntToSize(seq_len_ * batch_size_ * input_size_) * sizeof(T); + + size_t h_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(hx_desc_, &h_size), "get h size failed"); + + input_size_list_.push_back(x_size); + input_size_list_.push_back(h_size); + input_size_list_.push_back(h_size); + input_size_list_.push_back(weight_size_); + + size_t y_size = IntToSize(seq_len_ * batch_size_ * hidden_size_ * (bidirectional_ ? 2 : 1)) * sizeof(T); + output_size_list_.push_back(y_size); + output_size_list_.push_back(h_size); + output_size_list_.push_back(h_size); + output_size_list_.push_back(reserved_size_); + size_t state_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnDropoutGetStatesSize(handle_, &state_size), "get dropout states size failed"); + output_size_list_.push_back(state_size); + + size_t workspace_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetRNNWorkspaceSize(handle_, rnn_desc_, seq_len_, x_desc_.get(), &workspace_size), + "get workspace size failed"); + workspace_size_list_.push_back(workspace_size); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyRNNDescriptor(rnn_desc_), "destroy rnn_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyDropoutDescriptor(dropout_desc_), "destroy dropout_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(cy_desc_), "destroy cy_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(hy_desc_), "destroy hy_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyFilterDescriptor(w_desc_), "destroy w_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(hx_desc_), "destroy hx_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(cx_desc_), "destroy cx_desc failed"); + + for (size_t i = 0; i < IntToSize(seq_len_); ++i) { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(y_desc_[i]), "destroy y_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_desc_[i]), "destroy x_desc failed"); + } + } + + int batch_size_; + int seq_len_; + int input_size_; + int hidden_size_; + int num_layers_; + + bool has_bias_; + bool bidirectional_; + bool states_init_; + float dropout_; + + size_t weight_size_; + size_t reserved_size_; + + // input desc + unique_ptr x_desc_; + cudnnTensorDescriptor_t hx_desc_; + cudnnTensorDescriptor_t cx_desc_; + cudnnFilterDescriptor_t w_desc_; + cudnnDropoutDescriptor_t dropout_desc_; + unique_ptr y_desc_; + cudnnTensorDescriptor_t hy_desc_; + cudnnTensorDescriptor_t cy_desc_; + cudnnRNNDescriptor_t rnn_desc_; + + cudnnHandle_t handle_; + cudnnDataType_t cudnn_data_type_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.cc new file mode 100644 index 00000000000..ab88308d4e9 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.cc @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/lstm_grad_data_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(LSTMGradData, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + LstmGradDataGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(LSTMGradData, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + LstmGradDataGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.h new file mode 100644 index 00000000000..2fadccb8ea4 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_data_gpu_kernel.h @@ -0,0 +1,285 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GRAD_DATA_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GRAD_DATA_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "dataset/util/make_unique.h" + +namespace mindspore { +namespace kernel { +template +class LstmGradDataGpuKernel : public GpuKernel { + public: + LstmGradDataGpuKernel() + : batch_size_(0), + seq_len_(0), + input_size_(0), + hidden_size_(0), + num_layers_(0), + has_bias_(false), + bidirectional_(false), + states_init_(false), + dropout_(0), + weight_size_(0), + reserved_size_(0), + rnn_desc_(nullptr), + y_desc_(nullptr), + dy_desc_(nullptr), + dhy_desc_(nullptr), + dcy_desc_(nullptr), + w_desc_(nullptr), + hx_desc_(nullptr), + cx_desc_(nullptr), + dropout_desc_(nullptr), + dx_desc_(nullptr), + dhx_desc_(nullptr), + dcx_desc_(nullptr), + handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT) {} + ~LstmGradDataGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(stream_ptr); + auto y_addr = GetDeviceAddress(inputs, 0); + auto dy_addr = GetDeviceAddress(inputs, 1); + auto dhy_addr = GetDeviceAddress(inputs, 2); + auto dcy_addr = GetDeviceAddress(inputs, 3); + auto w_addr = GetDeviceAddress(inputs, 4); + auto hx_addr = GetDeviceAddress(inputs, 5); + auto cx_addr = GetDeviceAddress(inputs, 6); + auto reserved_addr = GetDeviceAddress(inputs, 7); + auto states_addr = GetDeviceAddress(inputs, 8); + auto dx_addr = GetDeviceAddress(outputs, 0); + auto dhx_addr = GetDeviceAddress(outputs, 1); + auto dcx_addr = GetDeviceAddress(outputs, 2); + void *workspace_addr = GetDeviceAddress(workspace, 0); + + if (!states_init_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnRestoreDropoutDescriptor(dropout_desc_, handle_, dropout_, states_addr, input_size_list_[8], 0), + "restore dropout state failed"); + states_init_ = true; + } + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnRNNBackwardData(handle_, rnn_desc_, seq_len_, y_desc_.get(), y_addr, dy_desc_.get(), dy_addr, dhy_desc_, + dhy_addr, dcy_desc_, dcy_addr, w_desc_, w_addr, hx_desc_, hx_addr, cx_desc_, cx_addr, + dx_desc_.get(), dx_addr, dhx_desc_, dhx_addr, dcx_desc_, dcx_addr, workspace_addr, + workspace_size_list_[0], reserved_addr, reserved_size_), + "launch lstm back data kernel failed"); + + CHECK_CUDA_RET_WITH_EXCEPT(cudaStreamSynchronize(reinterpret_cast(stream_ptr)), + "stream synchronize failed."); + return true; + } + void GetAttrs(const CNodePtr &kernel_node) { + input_size_ = GetAttr(kernel_node, "input_size"); + hidden_size_ = GetAttr(kernel_node, "hidden_size"); + num_layers_ = GetAttr(kernel_node, "num_layers"); + has_bias_ = GetAttr(kernel_node, "has_bias"); + bidirectional_ = GetAttr(kernel_node, "bidirectional"); + dropout_ = GetAttr(kernel_node, "dropout"); + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto input_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + seq_len_ = SizeToInt(input_shape[0]); + batch_size_ = SizeToInt(input_shape[1]); + GetAttrs(kernel_node); + cudnnRNNInputMode_t input_mode = CUDNN_LINEAR_INPUT; + cudnnDirectionMode_t direction = bidirectional_ ? CUDNN_BIDIRECTIONAL : CUDNN_UNIDIRECTIONAL; + cudnnRNNMode_t rnn_mode = CUDNN_LSTM; + cudnnRNNAlgo_t algo = CUDNN_RNN_ALGO_STANDARD; + CreateTensorDescGrp(); + int hx_dims[3]{num_layers_ * (bidirectional_ ? 2 : 1), batch_size_, hidden_size_}; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dhy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), "set dhy_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dcy_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), "set dcy_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(hx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set hx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(cx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set cx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dhx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), "set dhx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dcx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), "set dcx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetDropoutDescriptor(dropout_desc_, handle_, dropout_, nullptr, 0, 0), + "set dropout_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetRNNDescriptor(handle_, rnn_desc_, hidden_size_, num_layers_, dropout_desc_, + input_mode, direction, rnn_mode, algo, cudnn_data_type_), + "set rnn_desc failed"); + cudnnRNNBiasMode_t bias_mode = has_bias_ ? CUDNN_RNN_DOUBLE_BIAS : CUDNN_RNN_NO_BIAS; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetRNNBiasMode(rnn_desc_, bias_mode), "set bias_mode failed"); + auto weight_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 4); + size_t weight_size = weight_shape[0] * weight_shape[1] * weight_shape[2] * sizeof(T); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetRNNParamsSize(handle_, rnn_desc_, dx_desc_[0], &weight_size_, cudnn_data_type_), + "get weight_size_ failed"); + if (weight_size != weight_size_) { + MS_LOG(EXCEPTION) << "weight size: " << weight_size << " error, expect: " << weight_size_ << " ."; + } + int w_dims[3] = {SizeToInt(weight_size_ / 4), 1, 1}; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetFilterNdDescriptor(w_desc_, cudnn_data_type_, CUDNN_TENSOR_NCHW, 3, w_dims), + "set w_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetRNNTrainingReserveSize(handle_, rnn_desc_, seq_len_, dx_desc_.get(), &reserved_size_), "get size failed"); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dhy_desc_), "create dhy_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dcy_desc_), "create dcy_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&hx_desc_), "create hx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&cx_desc_), "create cx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateFilterDescriptor(&w_desc_), "create w_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dhx_desc_), "create dhx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dcx_desc_), "create dcx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateDropoutDescriptor(&dropout_desc_), "create dropout_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateRNNDescriptor(&rnn_desc_), "create rnn_desc failed"); + } + + void InitSizeLists() override { + size_t y_size = IntToSize(seq_len_ * batch_size_ * hidden_size_ * (bidirectional_ ? 2 : 1)) * sizeof(T); + + size_t h_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(hx_desc_, &h_size), "get h size failed"); + + input_size_list_.push_back(y_size); + input_size_list_.push_back(y_size); + input_size_list_.push_back(h_size); + input_size_list_.push_back(h_size); + input_size_list_.push_back(weight_size_); + input_size_list_.push_back(h_size); + input_size_list_.push_back(h_size); + input_size_list_.push_back(reserved_size_); + size_t state_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnDropoutGetStatesSize(handle_, &state_size), "get dropout states size failed"); + input_size_list_.push_back(state_size); + + size_t x_size = IntToSize(seq_len_ * batch_size_ * input_size_) * sizeof(T); + output_size_list_.push_back(x_size); + output_size_list_.push_back(h_size); + output_size_list_.push_back(h_size); + + size_t workspace_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetRNNWorkspaceSize(handle_, rnn_desc_, seq_len_, dx_desc_.get(), &workspace_size), + "get workspace size failed"); + workspace_size_list_.push_back(workspace_size); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyRNNDescriptor(rnn_desc_), "destroy rnn_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyDropoutDescriptor(dropout_desc_), "destroy dropout_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dcx_desc_), "destroy dcx_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dhx_desc_), "destroy dhx_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyFilterDescriptor(w_desc_), "destroy w_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(cx_desc_), "destroy cx_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(hx_desc_), "destroy hx_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dcy_desc_), "destroy dcy_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dhy_desc_), "destroy dhy_desc_ failed"); + DestroyTensorDescGrp(); + } + void CreateTensorDescGrp() { + int x_dims[3]{batch_size_, input_size_, 1}; + int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; + + dx_desc_ = mindspore::make_unique(seq_len_); + y_desc_ = mindspore::make_unique(seq_len_); + dy_desc_ = mindspore::make_unique(seq_len_); + + for (size_t i = 0; i < IntToSize(seq_len_); ++i) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_[i]), "create x_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dx_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, x_dims), + "set dx_desc failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&y_desc_[i]), "create y_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(y_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, y_dims), "set y_desc failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dy_desc_[i]), "create dy_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(dy_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, y_dims), + "set dy_desc_ failed"); + } + } + + void DestroyTensorDescGrp() { + for (size_t i = 0; i < IntToSize(seq_len_); ++i) { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dy_desc_[i]), "destroy dy_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(y_desc_[i]), "destroy y_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dx_desc_[i]), "destroy x_desc failed"); + } + } + + int batch_size_; + int seq_len_; + int input_size_; + int hidden_size_; + int num_layers_; + + bool has_bias_; + bool bidirectional_; + bool states_init_; + float dropout_; + + size_t weight_size_; + size_t reserved_size_; + + cudnnRNNDescriptor_t rnn_desc_; + + // input desc + unique_ptr y_desc_; + unique_ptr dy_desc_; + cudnnTensorDescriptor_t dhy_desc_; + cudnnTensorDescriptor_t dcy_desc_; + cudnnFilterDescriptor_t w_desc_; + cudnnTensorDescriptor_t hx_desc_; + cudnnTensorDescriptor_t cx_desc_; + + cudnnDropoutDescriptor_t dropout_desc_; + + // output desc + unique_ptr dx_desc_; + cudnnTensorDescriptor_t dhx_desc_; + cudnnTensorDescriptor_t dcx_desc_; + + cudnnHandle_t handle_; + cudnnDataType_t cudnn_data_type_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GRAD_DATA_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.cc new file mode 100644 index 00000000000..856a986e073 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/lstm_grad_weight_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(LSTMGradWeight, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + LstmGradWeightGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(LSTMGradWeight, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + LstmGradWeightGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.h new file mode 100644 index 00000000000..6cf512f14a6 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/lstm_grad_weight_gpu_kernel.h @@ -0,0 +1,232 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GRAD_WEIGHT_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GRAD_WEIGHT_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "dataset/util/make_unique.h" +#include "kernel/gpu/kernel_constants.h" +namespace mindspore { +namespace kernel { +template +class LstmGradWeightGpuKernel : public GpuKernel { + public: + LstmGradWeightGpuKernel() + : batch_size_(0), + seq_len_(0), + input_size_(0), + hidden_size_(0), + num_layers_(0), + has_bias_(false), + bidirectional_(false), + states_init_(false), + dropout_(0), + weight_size_(0), + reserved_size_(0), + rnn_desc_(nullptr), + dropout_desc_(nullptr), + x_desc_(nullptr), + hx_desc_(nullptr), + y_desc_(nullptr), + dw_desc_(nullptr), + handle_(nullptr), + cudnn_data_type_(CUDNN_DATA_FLOAT) {} + ~LstmGradWeightGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + VARIABLE_NOT_USED(stream_ptr); + auto x_addr = GetDeviceAddress(inputs, 0); + auto hx_addr = GetDeviceAddress(inputs, 1); + auto y_addr = GetDeviceAddress(inputs, 2); + auto reserved_addr = GetDeviceAddress(inputs, 3); + auto states_addr = GetDeviceAddress(inputs, 4); + auto dw_addr = GetDeviceAddress(outputs, 0); + void *workspace_addr = GetDeviceAddress(workspace, 0); + + if (!states_init_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnRestoreDropoutDescriptor(dropout_desc_, handle_, dropout_, states_addr, input_size_list_[4], 0), + "restore dropout state failed"); + states_init_ = true; + } + + CHECK_CUDA_RET_WITH_EXCEPT( + cudaMemsetAsync(dw_addr, 0, outputs[0]->size, reinterpret_cast(stream_ptr)), "cudaMemSet Failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnRNNBackwardWeights(handle_, rnn_desc_, seq_len_, x_desc_.get(), x_addr, hx_desc_, hx_addr, y_desc_.get(), + y_addr, workspace_addr, workspace_size_list_[0], dw_desc_, dw_addr, reserved_addr, + reserved_size_), + "launch lstm back weight kernel failed"); + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + seq_len_ = SizeToInt(input_shape[0]); + batch_size_ = SizeToInt(input_shape[1]); + + input_size_ = GetAttr(kernel_node, "input_size"); + hidden_size_ = GetAttr(kernel_node, "hidden_size"); + num_layers_ = GetAttr(kernel_node, "num_layers"); + has_bias_ = GetAttr(kernel_node, "has_bias"); + bidirectional_ = GetAttr(kernel_node, "bidirectional"); + dropout_ = GetAttr(kernel_node, "dropout"); + + cudnnRNNInputMode_t input_mode = CUDNN_LINEAR_INPUT; + cudnnDirectionMode_t direction = bidirectional_ ? CUDNN_BIDIRECTIONAL : CUDNN_UNIDIRECTIONAL; + cudnnRNNMode_t rnn_mode = CUDNN_LSTM; + cudnnRNNAlgo_t algo = CUDNN_RNN_ALGO_STANDARD; + + CreateTensorDescGrp(); + int hx_dims[3]{num_layers_ * (bidirectional_ ? 2 : 1), batch_size_, hidden_size_}; + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensorNdDescriptorEx(hx_desc_, CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, hx_dims), + "set hx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetDropoutDescriptor(dropout_desc_, handle_, dropout_, nullptr, 0, 0), + "set dropout_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetRNNDescriptor(handle_, rnn_desc_, hidden_size_, num_layers_, dropout_desc_, + input_mode, direction, rnn_mode, algo, cudnn_data_type_), + "set rnn_desc failed"); + cudnnRNNBiasMode_t bias_mode = has_bias_ ? CUDNN_RNN_DOUBLE_BIAS : CUDNN_RNN_NO_BIAS; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetRNNBiasMode(rnn_desc_, bias_mode), "set bias_mode failed"); + + auto weight_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + size_t weight_size = weight_shape[0] * weight_shape[1] * weight_shape[2] * sizeof(T); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetRNNParamsSize(handle_, rnn_desc_, x_desc_[0], &weight_size_, cudnn_data_type_), + "get weight_size_ failed"); + if (weight_size != weight_size_) { + MS_LOG(EXCEPTION) << "weight size: " << weight_size << " error, expect: " << weight_size_ << " ."; + } + int w_dims[3] = {SizeToInt(weight_size_ / 4), 1, 1}; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetFilterNdDescriptor(dw_desc_, cudnn_data_type_, CUDNN_TENSOR_NCHW, 3, w_dims), + "set dw_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetRNNTrainingReserveSize(handle_, rnn_desc_, seq_len_, x_desc_.get(), &reserved_size_), + "get reserve size failed"); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&hx_desc_), "create hx_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateFilterDescriptor(&dw_desc_), "create dw_desc_ failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateDropoutDescriptor(&dropout_desc_), "create dropout_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateRNNDescriptor(&rnn_desc_), "create rnn_desc failed"); + } + void InitSizeLists() override { + size_t x_size = IntToSize(seq_len_ * batch_size_ * input_size_) * sizeof(T); + + size_t h_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(hx_desc_, &h_size), "get h size failed"); + + size_t y_size = IntToSize(seq_len_ * batch_size_ * hidden_size_ * (bidirectional_ ? 2 : 1)) * sizeof(T); + input_size_list_.push_back(x_size); + input_size_list_.push_back(h_size); + input_size_list_.push_back(y_size); + input_size_list_.push_back(reserved_size_); + size_t state_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnDropoutGetStatesSize(handle_, &state_size), "get dropout states size failed"); + input_size_list_.push_back(state_size); + + output_size_list_.push_back(weight_size_); + + size_t workspace_size = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetRNNWorkspaceSize(handle_, rnn_desc_, seq_len_, x_desc_.get(), &workspace_size), + "get workspace size failed"); + workspace_size_list_.push_back(workspace_size); + } + + private: + void CreateTensorDescGrp() { + int x_dims[3]{batch_size_, input_size_, 1}; + int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1}; + + x_desc_ = mindspore::make_unique(seq_len_); + y_desc_ = mindspore::make_unique(seq_len_); + + for (size_t i = 0; i < IntToSize(seq_len_); ++i) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(x_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, x_dims), "set x_desc failed"); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&y_desc_[i]), "create y_desc failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensorNdDescriptorEx(y_desc_[i], CUDNN_TENSOR_NCHW, cudnn_data_type_, 3, y_dims), "set y_desc failed"); + } + } + void DestroyTensorDescGrp() { + for (size_t i = 0; i < IntToSize(seq_len_); ++i) { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(y_desc_[i]), "destroy y_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_desc_[i]), "destroy x_desc failed"); + } + } + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyRNNDescriptor(rnn_desc_), "destroy rnn_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyDropoutDescriptor(dropout_desc_), "destroy dropout_desc failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyFilterDescriptor(dw_desc_), "destroy dw_desc_ failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(hx_desc_), "destroy hx_desc_ failed"); + DestroyTensorDescGrp(); + } + + int batch_size_; + int seq_len_; + int input_size_; + int hidden_size_; + int num_layers_; + + bool has_bias_; + bool bidirectional_; + bool states_init_; + float dropout_; + + size_t weight_size_; + size_t reserved_size_; + + cudnnRNNDescriptor_t rnn_desc_; + cudnnDropoutDescriptor_t dropout_desc_; + + // input desc + unique_ptr x_desc_; + cudnnTensorDescriptor_t hx_desc_; + unique_ptr y_desc_; + + // output desc + cudnnFilterDescriptor_t dw_desc_; + + cudnnHandle_t handle_; + cudnnDataType_t cudnn_data_type_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_LSTM_GRAD_WEIGHT_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/momentum_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/momentum_gpu_kernel.cc new file mode 100644 index 00000000000..4a77f7342bd --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/momentum_gpu_kernel.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/momentum_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(ApplyMomentum, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + MomentumGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(ApplyMomentum, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + MomentumGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/momentum_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/momentum_gpu_kernel.h new file mode 100644 index 00000000000..2d8afb5780f --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/momentum_gpu_kernel.h @@ -0,0 +1,100 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_MOMENTUM_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_MOMENTUM_GPU_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/momentum_impl.cuh" +namespace mindspore { +namespace kernel { +template +class MomentumGpuKernel : public GpuKernel { + public: + MomentumGpuKernel() + : variable_size_(0), accumulation_size_(0), learning_rate_size_(0), gradient_size_(0), momentum_size_(0) {} + ~MomentumGpuKernel() override = default; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, const std::vector &, + uintptr_t stream_ptr) override { + T *variable = GetDeviceAddress(inputs, 0); + T *accumulation = GetDeviceAddress(inputs, 1); + T *learning_rate = GetDeviceAddress(inputs, 2); + T *gradient = GetDeviceAddress(inputs, 3); + T *momentum = GetDeviceAddress(inputs, 4); + MomentumUpdateVariable(inputs[0]->size / sizeof(T), variable, accumulation, learning_rate, gradient, momentum, + reinterpret_cast(stream_ptr)); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 5) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but momentum needs 5 inputs."; + return false; + } + + variable_size_ = sizeof(T); + accumulation_size_ = sizeof(T); + learning_rate_size_ = sizeof(T); + gradient_size_ = sizeof(T); + momentum_size_ = sizeof(T); + + auto variable_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + for (size_t i = 0; i < variable_shape.size(); i++) { + variable_size_ *= variable_shape[i]; + } + auto accumulation_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + for (size_t i = 0; i < accumulation_shape.size(); i++) { + accumulation_size_ *= accumulation_shape[i]; + } + auto gradient_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 3); + for (size_t i = 0; i < gradient_shape.size(); i++) { + gradient_size_ *= gradient_shape[i]; + } + InitSizeLists(); + return true; + } + + protected: + void InitSizeLists() override { + input_size_list_.push_back(variable_size_); + input_size_list_.push_back(accumulation_size_); + input_size_list_.push_back(learning_rate_size_); + input_size_list_.push_back(gradient_size_); + input_size_list_.push_back(momentum_size_); + output_size_list_.push_back(0); + } + + private: + size_t variable_size_; + size_t accumulation_size_; + size_t learning_rate_size_; + size_t gradient_size_; + size_t momentum_size_; + + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_MOMENTUM_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/pooling_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/pooling_gpu_kernel.cc new file mode 100644 index 00000000000..e871af360a1 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/pooling_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/pooling_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(MaxPool, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + PoolingGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(MaxPool, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + PoolingGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(AvgPool, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + PoolingGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(AvgPool, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + PoolingGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/pooling_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/pooling_gpu_kernel.h new file mode 100644 index 00000000000..67a705cdc16 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/pooling_gpu_kernel.h @@ -0,0 +1,252 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_POOLING_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_POOLING_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/pad_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class PoolingGpuFwdKernel : public GpuKernel { + public: + PoolingGpuFwdKernel() + : cudnn_handle_(nullptr), + input_descriptor_(nullptr), + output_descriptor_(nullptr), + pooling_descriptor_(nullptr), + padded_descriptor_(nullptr), + pooling_mode_(CUDNN_POOLING_MAX), + cudnn_data_type_(CUDNN_DATA_FLOAT), + old_height_(0), + old_width_(0), + pad_height_(0), + pad_width_(0), + pad_top_(0), + pad_left_(0), + n_(0), + c_(0), + pad_value_(0), + is_null_input_(false), + input_size_(0), + output_size_(0), + padded_size_(0), + workspace_size_(0), + use_pad_(true) {} + ~PoolingGpuFwdKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + if (is_null_input_) { + return true; + } + T *input_addr = reinterpret_cast(inputs[0]->addr); + T *output_addr = reinterpret_cast(outputs[0]->addr); + const float alpha = 1; + const float beta = 0; + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_) { + T *padded_addr = reinterpret_cast(workspace[0]->addr); + CalPad(padded_size_ / sizeof(T), input_addr, n_, c_, old_height_, old_width_, old_height_ + pad_height_, + old_width_ + pad_width_, pad_top_, pad_left_, pad_value_, padded_addr, + reinterpret_cast(stream_ptr)); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnPoolingForward(cudnn_handle_, pooling_descriptor_, &alpha, padded_descriptor_, + padded_addr, &beta, output_descriptor_, output_addr), + "cudnnPoolingForward failed"); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnPoolingForward(cudnn_handle_, pooling_descriptor_, &alpha, input_descriptor_, + input_addr, &beta, output_descriptor_, output_addr), + "cudnnPoolingForward failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) { + InitResource(); + if (!CheckParam(kernel_node)) { + return false; + } + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "PoolingGpuFwdKernel input is null."; + InitSizeLists(); + return true; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(input_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(input_shape[0]), + SizeToInt(input_shape[1]), SizeToInt(input_shape[2]), SizeToInt(input_shape[3])), + "cudnnSetTensor4dDescriptor failed"); + + auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(output_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(output_shape[0]), + SizeToInt(output_shape[1]), SizeToInt(output_shape[2]), SizeToInt(output_shape[3])), + "cudnnSetTensor4dDescriptor failed"); + auto window = GetValue>(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("ksize")); + int window_height = window[3]; + int window_width = window[2]; + stride_ = GetValue>(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("strides")); + SetPoolingMode(kernel_node); + if (pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) { + SetPad(input_shape, window_height, window_width); + } else { + pad_height_ = 0; + pad_width_ = 0; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetPooling2dDescriptor(pooling_descriptor_, pooling_mode_, CUDNN_NOT_PROPAGATE_NAN, window_height, + window_width, pad_height_, pad_width_, stride_[2], stride_[3]), + "cudnnSetPooling2dDescriptor failed"); + } + + InitSizeLists(); + return true; + } + + protected: + void InitResource() { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&input_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&output_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&padded_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreatePoolingDescriptor(&pooling_descriptor_), + "cudnnCreatePoolingDescriptor failed"); + } + void InitSizeLists() { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetTensorSizeInBytes(input_descriptor_, reinterpret_cast(&input_size_)), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetTensorSizeInBytes(output_descriptor_, reinterpret_cast(&output_size_)), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_ && !is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnGetTensorSizeInBytes(padded_descriptor_, reinterpret_cast(&padded_size_)), + "cudnnGetTensorSizeInBytes failed"); + workspace_size_list_.push_back(padded_size_); + if (padded_size_ == 0) { + MS_LOG(EXCEPTION) << "Padded size is 0."; + } + } + return; + } + + private: + bool CheckParam(const CNodePtr &kernel_node) { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but pooling needs 1 inputs."; + return false; + } + return true; + } + void SetPad(const std::vector &input_shape, const int &window_height, const int &window_width) { + n_ = SizeToInt(input_shape[0]); + c_ = SizeToInt(input_shape[1]); + old_height_ = SizeToInt(input_shape[2]); + old_width_ = SizeToInt(input_shape[3]); + pad_height_ = + std::max(0, (((old_height_ / stride_[2]) * stride_[2] == old_height_ ? (old_height_ / stride_[2]) + : (old_height_ / stride_[2]) + 1) - + 1) * + stride_[2] + + window_height - old_height_); + pad_width_ = + std::max(0, (((old_width_ / stride_[3]) * stride_[3] == old_width_ ? (old_width_ / stride_[3]) + : (old_width_ / stride_[3]) + 1) - + 1) * + stride_[3] + + window_width - old_width_); + pad_top_ = pad_height_ / 2; + pad_left_ = pad_width_ / 2; + if (pad_height_ % 2 == 0 && pad_width_ % 2 == 0) { + use_pad_ = false; + } + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(padded_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, n_, + c_, old_height_ + pad_height_, old_width_ + pad_width_), + "cudnnSetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetPooling2dDescriptor(pooling_descriptor_, pooling_mode_, CUDNN_NOT_PROPAGATE_NAN, + window_height, window_width, use_pad_ ? 0 : pad_top_, + use_pad_ ? 0 : pad_left_, stride_[2], stride_[3]), + "cudnnSetPooling2dDescriptor failed"); + } + void SetPoolingMode(const CNodePtr &kernel_node) { + pad_mode_ = GetValue(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("padding")); + mode_ = AnfAlgo::GetCNodeName(kernel_node); + if (mode_ == "AvgPool") { + pooling_mode_ = CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING; + pad_value_ = 0.0; + } else { + pooling_mode_ = CUDNN_POOLING_MAX; + pad_value_ = kSignedMinFloat; + } + } + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyPoolingDescriptor(pooling_descriptor_), + "cudnnDestroyPoolingDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(padded_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(output_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(input_descriptor_), "cudnnDestroyTensorDescriptor failed"); + } + + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t input_descriptor_; + cudnnTensorDescriptor_t output_descriptor_; + cudnnPoolingDescriptor_t pooling_descriptor_; + cudnnTensorDescriptor_t padded_descriptor_; + cudnnPoolingMode_t pooling_mode_ = CUDNN_POOLING_MAX; + std::vector stride_; + std::string mode_; + std::string pad_mode_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + cudnnDataType_t cudnn_data_type_; + + int old_height_; + int old_width_; + int pad_height_; + int pad_width_; + int pad_top_; + int pad_left_; + int n_; + int c_; + float pad_value_; + bool is_null_input_; + size_t input_size_; + size_t output_size_; + size_t padded_size_; + size_t workspace_size_; + bool use_pad_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_POOLING_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/pooling_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/pooling_grad_gpu_kernel.cc new file mode 100644 index 00000000000..57bd2311291 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/pooling_grad_gpu_kernel.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/pooling_grad_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(MaxPoolGrad, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + PoolingGradGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(MaxPoolGrad, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + PoolingGradGpuFwdKernel, half) +MS_REG_GPU_KERNEL_ONE(AvgPoolGradGpu, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + PoolingGradGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(AvgPoolGradGpu, + KernelAttr() + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddInputAttr(kNumberTypeFloat16) + .AddOutputAttr(kNumberTypeFloat16), + PoolingGradGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/pooling_grad_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/pooling_grad_gpu_kernel.h new file mode 100644 index 00000000000..6c10635c0ce --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/pooling_grad_gpu_kernel.h @@ -0,0 +1,296 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_POOLING_GRAD_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_POOLING_GRAD_GPU_KERNEL_H_ + +#include +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/pad_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class PoolingGradGpuFwdKernel : public GpuKernel { + public: + PoolingGradGpuFwdKernel() + : cudnn_handle_(nullptr), + pooling_descriptor_(nullptr), + y_descriptor_(nullptr), + dy_descriptor_(nullptr), + x_descriptor_(nullptr), + dx_descriptor_(nullptr), + padded_descriptor_(nullptr), + pooling_mode_(CUDNN_POOLING_MAX), + cudnn_data_type_(CUDNN_DATA_FLOAT), + old_height_(0), + old_width_(0), + pad_height_(0), + pad_width_(0), + pad_top_(0), + pad_left_(0), + n_(0), + c_(0), + pad_value_(0), + is_null_input_(false), + input_size_(0), + output_size_(0), + padded_size_(0), + workspace_size_(0), + use_pad_(true) {} + ~PoolingGradGpuFwdKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *x_data = GetDeviceAddress(inputs, 0); + T *y = GetDeviceAddress(inputs, 1); + T *dy = GetDeviceAddress(inputs, 2); + T *dx = GetDeviceAddress(outputs, 0); + + const float alpha = 1; + const float beta = 0; + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_) { + T *padded = GetDeviceAddress(workspace, 0); + T *padded_dx = GetDeviceAddress(workspace, 1); + + CalPad(padded_size_ / sizeof(T), x_data, n_, c_, old_height_, old_width_, old_height_ + pad_height_, + old_width_ + pad_width_, pad_top_, pad_left_, pad_value_, padded, + reinterpret_cast(stream_ptr)); + + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnPoolingBackward(cudnn_handle_, pooling_descriptor_, &alpha, y_descriptor_, y, dy_descriptor_, dy, + padded_descriptor_, padded, &beta, padded_descriptor_, padded_dx), + "cudnnPoolingBackward failed"); + + CalPadGrad(padded_size_ / sizeof(T), padded_dx, n_, c_, old_height_, old_width_, old_height_ + pad_height_, + old_width_ + pad_width_, pad_top_, pad_left_, dx, reinterpret_cast(stream_ptr)); + } else { + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnPoolingBackward(cudnn_handle_, pooling_descriptor_, &alpha, y_descriptor_, y, dy_descriptor_, dy, + x_descriptor_, x_data, &beta, dx_descriptor_, dx), + "cudnnPoolingBackward failed"); + } + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + if (!CheckParam(kernel_node)) { + return false; + } + auto window = GetAttr>(kernel_node, "ksize"); + int window_height = window[3]; + int window_width = window[2]; + SetPoolingMode(kernel_node); + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + auto input_mask = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(input_shape) || CHECK_NULL_INPUT(input_mask); + if (is_null_input_) { + MS_LOG(WARNING) << "PoolingGradGpuFwdKernel input is null."; + InitSizeLists(); + return true; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(y_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(input_mask[0]), + SizeToInt(input_mask[1]), SizeToInt(input_mask[2]), SizeToInt(input_mask[3])), + "cudnnSetTensor4dDescriptor"); + + auto dout_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 2); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dy_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(dout_shape[0]), + SizeToInt(dout_shape[1]), SizeToInt(dout_shape[2]), SizeToInt(dout_shape[3])), + "cudnnSetTensor4dDescriptor"); + + auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(dx_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(output_shape[0]), + SizeToInt(output_shape[1]), SizeToInt(output_shape[2]), SizeToInt(output_shape[3])), + "cudnnSetTensor4dDescriptor failed"); + if (kSamePadModeUpperCase == pad_mode_ || kSamePadModeLowerCase == pad_mode_) { + SetPad(input_shape, window_height, window_width); + } else { + if (pad_mode_ == kValidPadModeUpperCase || pad_mode_ == kValidPadModeLowerCase) { + pad_height_ = 0; + pad_width_ = 0; + } + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetPooling2dDescriptor(pooling_descriptor_, pooling_mode_, CUDNN_NOT_PROPAGATE_NAN, window_height, + window_width, pad_height_, pad_width_, stride_[2], stride_[3]), + "cudnnSetPooling2dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(x_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(input_shape[0]), + SizeToInt(input_shape[1]), SizeToInt(input_shape[2]), SizeToInt(input_shape[3])), + "cudnnSetTensor4dDescriptor"); + } + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&y_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dy_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&padded_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreatePoolingDescriptor(&pooling_descriptor_), + "cudnnCreatePoolingDescriptor failed"); + } + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(y_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dx_descriptor_, &output_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(dy_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(x_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + + if ((pad_mode_ == kSamePadModeUpperCase || pad_mode_ == kSamePadModeLowerCase) && use_pad_ && !is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(padded_descriptor_, &padded_size_), + "cudnnGetTensorSizeInBytes failed"); + if (padded_size_ == 0) { + MS_LOG(EXCEPTION) << "Padded size is 0."; + } + workspace_size_list_.push_back(padded_size_); + workspace_size_list_.push_back(padded_size_); + } + return; + } + + private: + bool CheckParam(const CNodePtr &kernel_node) { + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 3) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but PoolingGradGpuFwdKernel needs 3 inputs."; + return false; + } + return true; + } + void SetPad(const std::vector &input_shape, const int &window_height, const int &window_width) { + n_ = SizeToInt(input_shape[0]); + c_ = SizeToInt(input_shape[1]); + old_height_ = SizeToInt(input_shape[2]); + old_width_ = SizeToInt(input_shape[3]); + pad_height_ = + std::max(0, (((old_height_ / stride_[2]) * stride_[2] == old_height_ ? (old_height_ / stride_[2]) + : (old_height_ / stride_[2]) + 1) - + 1) * + stride_[2] + + window_height - old_height_); + pad_width_ = + std::max(0, (((old_width_ / stride_[3]) * stride_[3] == old_width_ ? (old_width_ / stride_[3]) + : (old_width_ / stride_[3]) + 1) - + 1) * + stride_[3] + + window_width - old_width_); + pad_top_ = pad_height_ / 2; + pad_left_ = pad_width_ / 2; + if (pad_height_ % 2 == 0 && pad_width_ % 2 == 0) { + use_pad_ = false; + } + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(padded_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, n_, + c_, old_height_ + pad_height_, old_width_ + pad_width_), + "cudnnSetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(x_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(input_shape[0]), + SizeToInt(input_shape[1]), SizeToInt(input_shape[2]) + (use_pad_ ? pad_height_ : 0), + SizeToInt(input_shape[3]) + (use_pad_ ? pad_width_ : 0)), + "cudnnSetTensor4dDescriptor"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetPooling2dDescriptor(pooling_descriptor_, pooling_mode_, CUDNN_NOT_PROPAGATE_NAN, + window_height, window_width, use_pad_ ? 0 : pad_top_, + use_pad_ ? 0 : pad_left_, stride_[2], stride_[3]), + "cudnnSetPooling2dDescriptor failed"); + } + void SetPoolingMode(const CNodePtr &kernel_node) { + pad_mode_ = GetAttr(kernel_node, "padding"); + stride_ = GetAttr>(kernel_node, "strides"); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + mode_ = AnfAlgo::GetCNodeName(kernel_node); + if (mode_ == "AvgPoolGradGpu") { + pooling_mode_ = CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING; + pad_value_ = 0.0; + } else { + pooling_mode_ = CUDNN_POOLING_MAX; + pad_value_ = kSignedMinFloat; + } + } + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyPoolingDescriptor(pooling_descriptor_), + "cudnnDestroyPoolingDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(padded_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dx_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(x_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(dy_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(y_descriptor_), "cudnnDestroyTensorDescriptor failed"); + } + + cudnnHandle_t cudnn_handle_; + cudnnPoolingDescriptor_t pooling_descriptor_; + cudnnTensorDescriptor_t y_descriptor_; + cudnnTensorDescriptor_t dy_descriptor_; + cudnnTensorDescriptor_t x_descriptor_; + cudnnTensorDescriptor_t dx_descriptor_; + cudnnTensorDescriptor_t padded_descriptor_; + cudnnPoolingMode_t pooling_mode_ = CUDNN_POOLING_MAX; + std::vector stride_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + std::string mode_; + std::string pad_mode_; + cudnnDataType_t cudnn_data_type_; + int old_height_; + int old_width_; + int pad_height_; + int pad_width_; + int pad_top_; + int pad_left_; + int n_; + int c_; + float pad_value_; + bool is_null_input_; + size_t input_size_; + size_t output_size_; + size_t padded_size_; + size_t workspace_size_; + bool use_pad_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_POOLING_GRAD_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/relu_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/relu_gpu_kernel.cc new file mode 100644 index 00000000000..d4cefc73ca1 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/relu_gpu_kernel.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/relu_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(ReLU, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ReLUGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE(ReLU, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ReLUGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/relu_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/relu_gpu_kernel.h new file mode 100644 index 00000000000..7931dbaf1be --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/relu_gpu_kernel.h @@ -0,0 +1,140 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_RELU_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_RELU_GPU_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class ReLUGpuFwdKernel : public GpuKernel { + public: + ReLUGpuFwdKernel() + : cudnn_handle_(nullptr), + activation_desc_(nullptr), + mode_(CUDNN_ACTIVATION_RELU), + input_descriptor_(nullptr), + output_descriptor_(nullptr), + is_null_input_(false), + cudnn_data_type_(CUDNN_DATA_FLOAT), + input_size_(0), + output_size_(0), + workspace_size_(0) {} + ~ReLUGpuFwdKernel() override { DestroyResource(); } + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t) override { + if (is_null_input_) { + return true; + } + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + + const float alpha = 1; + const float beta = 0; + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnActivationForward(cudnn_handle_, activation_desc_, &alpha, input_descriptor_, + input, &beta, output_descriptor_, output), + "ReLUGpuFwdKernel failed"); + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Argument number is " << input_num << ", but ReLUGpuFwdKernel needs 1."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "ReLUGpuFwdKernel input is null."; + InitSizeLists(); + return true; + } + mode_ = CUDNN_ACTIVATION_RELU; + int batch_size = input_shape.size() < 4 ? 1 : SizeToInt(input_shape[input_shape.size() - 4]); + int channel_size = input_shape.size() < 3 ? 1 : SizeToInt(input_shape[input_shape.size() - 3]); + int height = input_shape.size() < 2 ? 1 : SizeToInt(input_shape[input_shape.size() - 2]); + int width = input_shape.size() == 0 ? 1 : SizeToInt(input_shape[input_shape.size() - 1]); + + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetActivationDescriptor(activation_desc_, mode_, CUDNN_NOT_PROPAGATE_NAN, 0.0), + "SetActivationDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(input_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + batch_size, channel_size, height, width), + "SetTensor4dDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(output_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + batch_size, channel_size, height, width), + "SetTensor4dDescriptor failed"); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&input_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&output_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateActivationDescriptor(&activation_desc_), + "cudnnCreateActivationDescriptor failed"); + } + + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(input_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(output_descriptor_, &output_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyActivationDescriptor(activation_desc_), + "cudnnDestroyActivationDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(output_descriptor_), "cudnnDestroyTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(input_descriptor_), "cudnnDestroyTensorDescriptor failed"); + } + + cudnnHandle_t cudnn_handle_; + cudnnActivationDescriptor_t activation_desc_; + cudnnActivationMode_t mode_; + cudnnTensorDescriptor_t input_descriptor_; + cudnnTensorDescriptor_t output_descriptor_; + bool is_null_input_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + cudnnDataType_t cudnn_data_type_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_RELU_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/relu_grad_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/relu_grad_kernel.cc new file mode 100644 index 00000000000..848007320f0 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/relu_grad_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/relu_grad_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE( + ReluGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + ReluGradGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + ReluGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + ReluGradGpuFwdKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/relu_grad_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/relu_grad_kernel.h new file mode 100644 index 00000000000..713c08c654b --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/relu_grad_kernel.h @@ -0,0 +1,129 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_RELU_GRAD_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_RELU_GRAD_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class ReluGradGpuFwdKernel : public GpuKernel { + public: + ReluGradGpuFwdKernel() + : cudnn_handle_(nullptr), + activation_desc_(nullptr), + mode_(CUDNN_ACTIVATION_RELU), + input_descriptor_(nullptr), + is_null_input_(false), + cudnn_data_type_(CUDNN_DATA_FLOAT), + input_size_(0) {} + ~ReluGradGpuFwdKernel() override { DestroyResource(); } + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, uintptr_t) override { + if (is_null_input_) { + return true; + } + T *y = GetDeviceAddress(inputs, 1); + T *dy = GetDeviceAddress(inputs, 0); + T *dx = GetDeviceAddress(outputs, 0); + + const float alpha = 1; + const float beta = 0; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnActivationBackward(cudnn_handle_, activation_desc_, &alpha, input_descriptor_, y, input_descriptor_, dy, + input_descriptor_, y, &beta, input_descriptor_, dx), + "cudnnActivationBackward failed"); + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Argument number is " << input_num << ", but ReluGradGpuFwdKernel needs 2."; + return false; + } + auto input_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + mode_ = CUDNN_ACTIVATION_RELU; + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "ReluGradGpuFwdKernel input is null."; + InitSizeLists(); + return true; + } + int batch_size = input_shape.size() < 4 ? 1 : SizeToInt(input_shape[input_shape.size() - 4]); + int channel_size = input_shape.size() < 3 ? 1 : SizeToInt(input_shape[input_shape.size() - 3]); + int height = input_shape.size() < 2 ? 1 : SizeToInt(input_shape[input_shape.size() - 2]); + int width = input_shape.size() == 0 ? 1 : SizeToInt(input_shape[input_shape.size() - 1]); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetActivationDescriptor(activation_desc_, mode_, CUDNN_PROPAGATE_NAN, 0.0), + "SetActivationDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(input_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + batch_size, channel_size, height, width), + "SetTensor4dDescriptor failed"); + + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&input_descriptor_), "cudnnCreateTensorDescriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateActivationDescriptor(&activation_desc_), + "cudnnCreateActivationDescriptor failed"); + } + void InitSizeLists() override { + if (!is_null_input_) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnGetTensorSizeInBytes(input_descriptor_, &input_size_), + "cudnnGetTensorSizeInBytes failed"); + } + input_size_list_.push_back(input_size_); + output_size_list_.push_back(input_size_); + input_size_list_.push_back(input_size_); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyActivationDescriptor(activation_desc_), + "cudnnDestroyActivationDescriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(input_descriptor_), "cudnnDestroyTensorDescriptor failed"); + } + + cudnnHandle_t cudnn_handle_; + cudnnActivationDescriptor_t activation_desc_; + cudnnActivationMode_t mode_; + cudnnTensorDescriptor_t input_descriptor_; + bool is_null_input_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + cudnnDataType_t cudnn_data_type_; + size_t input_size_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_RELU_GRAD_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.cc new file mode 100644 index 00000000000..160a26d2007 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.cc @@ -0,0 +1,29 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_TWO(SoftmaxCrossEntropyWithLogits, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + SoftmaxCrossEntropyWithLogitsGpuKernel, float, float) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.h new file mode 100644 index 00000000000..3822a326fbd --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/softmax_cross_entropy_with_logits_gpu_kernel.h @@ -0,0 +1,207 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_GPU_KERNEL_H_ + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/cross_entropy_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class SoftmaxCrossEntropyWithLogitsGpuKernel : public GpuKernel { + public: + SoftmaxCrossEntropyWithLogitsGpuKernel() + : cudnn_handle_(nullptr), + logits_descriptor_(nullptr), + softmax_output_descriptor_(nullptr), + algo_(CUDNN_SOFTMAX_ACCURATE), + mode_(CUDNN_SOFTMAX_MODE_INSTANCE), + cudnn_data_type_(CUDNN_DATA_FLOAT), + is_null_input_(false), + logits_size_(0), + labels_size_(0), + output1_size_(0), + output2_size_(0), + softmax_output_logits_size_(0), + batch_size_(0), + channel_size_(0), + height_(0), + width_(0) {} + ~SoftmaxCrossEntropyWithLogitsGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *logits_addr = GetDeviceAddress(inputs, 0); + S *labels_addr = GetDeviceAddress(inputs, 1); + T *output1_addr = GetDeviceAddress(outputs, 0); + T *output2_addr = GetDeviceAddress(outputs, 1); + T *softmax_output_logits = GetDeviceAddress(workspace, 0); + + const float alpha = 1; + const float beta = 0; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSoftmaxForward(cudnn_handle_, algo_, mode_, &alpha, logits_descriptor_, logits_addr, &beta, + softmax_output_descriptor_, softmax_output_logits), + "cudnnSoftmaxForward failed."); + + CrossEntropyWithoutSparse(softmax_output_logits, labels_addr, batch_size_, channel_size_, output1_addr, + reinterpret_cast(stream_ptr)); + CrossEntropyGradWithoutSparse(softmax_output_logits, labels_addr, batch_size_, channel_size_, output2_addr, + reinterpret_cast(stream_ptr)); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num + << ", but SoftmaxCrossEntropyWithLogitsGpuKernel needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 2) { + MS_LOG(ERROR) << "Output number is " << output_num + << ", but SoftmaxCrossEntropyWithLogitsGpuKernel needs 2 output."; + return false; + } + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + + InferInputOutputSize(kernel_node); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(logits_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + batch_size_, channel_size_, height_, width_), + "cudnnSetTensor4dDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(softmax_output_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, batch_size_, + channel_size_, height_, width_), + "cudnnSetTensor4dDescriptor failed."); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&logits_descriptor_), + "cudnnCreateTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&softmax_output_descriptor_), + "cudnnCreateTensorDescriptor failed."); + } + void InitSizeLists() override { + input_size_list_.push_back(logits_size_); + input_size_list_.push_back(labels_size_); + output_size_list_.push_back(output1_size_); + output_size_list_.push_back(output2_size_); + workspace_size_list_.push_back(softmax_output_logits_size_); + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(softmax_output_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(logits_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + } + void InferInputOutputSize(const CNodePtr &kernel_node) { + auto logits_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(logits_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "SoftmaxCrossEntropyWithLogitsGpuKernel input1 is null"; + InitSizeLists(); + return; + } + auto labels_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(logits_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "SoftmaxCrossEntropyWithLogitsGpuKernel input2 is null"; + InitSizeLists(); + return; + } + CheckShapeValidation(logits_shape, labels_shape); + + size_t logits_dims = logits_shape.size(); + batch_size_ = 1; + for (size_t i = 0; i < logits_dims - 1; i++) { + batch_size_ *= logits_shape[i]; + } + channel_size_ = logits_shape[logits_dims - 1]; + height_ = 1; + width_ = 1; + logits_size_ = sizeof(T) * batch_size_ * channel_size_ * height_ * width_; + + labels_size_ = 1; + size_t labels_dims = labels_shape.size(); + for (size_t i = 0; i < labels_dims; i++) { + labels_size_ *= labels_shape[i]; + } + labels_size_ *= sizeof(S); + + output1_size_ = logits_size_ / logits_shape[logits_dims - 1]; + output2_size_ = logits_size_; + softmax_output_logits_size_ = logits_size_; + return; + } + void CheckShapeValidation(const std::vector &logits_shape, const std::vector &labels_shape) { + size_t logits_dim_length = logits_shape.size(); + size_t labels_dim_length = labels_shape.size(); + if (labels_dim_length != logits_dim_length) { + MS_LOG(EXCEPTION) << "Labels shape length should be equal to Logits shape length for " + "SoftmaxCrossEntropyWithLogits, but got Labels " + "shape length:" + << labels_dim_length << ", Logits shape length:" << logits_dim_length; + } + if (!std::equal(labels_shape.begin(), labels_shape.end(), logits_shape.begin())) { + MS_LOG(EXCEPTION) << "The shape of labels should be the same as the shape of logits except its last demension."; + } + return; + } + + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t logits_descriptor_; + cudnnTensorDescriptor_t softmax_output_descriptor_; + cudnnSoftmaxAlgorithm_t algo_; + cudnnSoftmaxMode_t mode_; + cudnnDataType_t cudnn_data_type_; + bool is_null_input_; + + size_t logits_size_; + size_t labels_size_; + size_t output1_size_; + size_t output2_size_; + size_t softmax_output_logits_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + size_t batch_size_; + size_t channel_size_; + size_t height_; + size_t width_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/softmax_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/softmax_gpu_kernel.cc new file mode 100644 index 00000000000..9e33630c3bc --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/softmax_gpu_kernel.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/softmax_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(Softmax, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + SoftmaxGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(Softmax, KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + SoftmaxGpuKernel, half) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/softmax_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/softmax_gpu_kernel.h new file mode 100644 index 00000000000..89120d93c07 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/softmax_gpu_kernel.h @@ -0,0 +1,215 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_SOFTMAX_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_SOFTMAX_GPU_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "kernel/gpu/cuda_impl/transpose_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class SoftmaxGpuKernel : public GpuKernel { + public: + SoftmaxGpuKernel() + : cudnn_handle_(nullptr), + input_descriptor_(nullptr), + output_descriptor_(nullptr), + algo_(CUDNN_SOFTMAX_ACCURATE), + mode_(CUDNN_SOFTMAX_MODE_INSTANCE), + cudnn_data_type_(CUDNN_DATA_FLOAT), + is_null_input_(false), + input_size_(0), + output_size_(0), + workspace_size_(0), + axis_(0), + shape_size_(0), + batch_size_(0), + channel_size_(0), + height_(0), + width_(0) {} + ~SoftmaxGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *input_addr = GetDeviceAddress(inputs, 0); + T *output_addr = GetDeviceAddress(outputs, 0); + T *transpose_input_addr = GetDeviceAddress(workspace, 0); + T *transpose_output_addr = GetDeviceAddress(workspace, 1); + int *input_shape = GetDeviceAddress(workspace, 2); + int *transpose_shape = GetDeviceAddress(workspace, 3); + int *transpose_axis = GetDeviceAddress(workspace, 4); + const float alpha = 1; + const float beta = 0; + + if (axis_ == 1) { + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSoftmaxForward(cudnn_handle_, algo_, mode_, &alpha, input_descriptor_, + input_addr, &beta, output_descriptor_, output_addr), + "cudnnSoftmaxForward failed"); + } else { + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(input_shape, &input_shape_[0], workspace_size_, cudaMemcpyHostToDevice, + reinterpret_cast(stream_ptr)), + "cudaMemcpyAsync input_shape failed"); + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(transpose_shape, &transpose_shape_[0], workspace_size_, + cudaMemcpyHostToDevice, reinterpret_cast(stream_ptr)), + "cudaMemcpyAsync input_shape failed"); + CHECK_CUDA_RET_WITH_EXCEPT(cudaMemcpyAsync(transpose_axis, &transpose_axis_[0], workspace_size_, + cudaMemcpyHostToDevice, reinterpret_cast(stream_ptr)), + "cudaMemcpyAsync input_axis failed"); + int size = SizeToInt(input_size_ / sizeof(T)); + CalTranspose(size, input_addr, input_shape, transpose_axis, shape_size_, transpose_input_addr, + reinterpret_cast(stream_ptr)); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSoftmaxForward(cudnn_handle_, algo_, mode_, &alpha, input_descriptor_, transpose_input_addr, &beta, + output_descriptor_, transpose_output_addr), + "cudnnSoftmaxForward failed"); + CalTranspose(size, transpose_output_addr, transpose_shape, transpose_axis, shape_size_, output_addr, + reinterpret_cast(stream_ptr)); + } + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(ERROR) << "Input number is " << input_num << ", but softmax needs 1 input."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num << ", but softmax needs 1 output."; + return false; + } + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "SoftmaxGpuKernel input is null"; + InitSizeLists(); + return true; + } + shape_size_ = SizeToInt(input_shape.size()); + if (shape_size_ != 2) { + MS_LOG(EXCEPTION) << "Input is " << shape_size_ << "-D, but softmax only supports 2-D inputs."; + } + + auto axis = GetAttr>(kernel_node, "axis"); + InitSizeByAxis(input_shape, axis[0]); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(input_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(batch_size_), + SizeToInt(channel_size_), SizeToInt(height_), SizeToInt(width_)), + "set input_descriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(output_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, SizeToInt(batch_size_), + SizeToInt(channel_size_), SizeToInt(height_), SizeToInt(width_)), + "set output_descriptor failed"); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&input_descriptor_), "create input_descriptor failed"); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&output_descriptor_), "create output_descriptor failed"); + } + + void InitSizeLists() override { + input_size_list_.push_back(input_size_); + output_size_list_.push_back(output_size_); + workspace_size_list_.push_back(input_size_); + workspace_size_list_.push_back(output_size_); + workspace_size_list_.push_back(workspace_size_); + workspace_size_list_.push_back(workspace_size_); + workspace_size_list_.push_back(workspace_size_); + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(output_descriptor_), "destroy output_descriptor failed"); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(input_descriptor_), "destroy input_descriptor failed"); + } + + void InitSizeByAxis(const std::vector input_shape, const int axis) { + axis_ = axis; + if (axis_ < 0) { + axis_ += shape_size_; + } + if (axis_ == 1) { + batch_size_ = input_shape[0]; + channel_size_ = input_shape[1]; + } else if (axis_ == 0) { + batch_size_ = input_shape[1]; + channel_size_ = input_shape[0]; + input_shape_.push_back(input_shape[0]); + input_shape_.push_back(input_shape[1]); + transpose_shape_.push_back(input_shape[1]); + transpose_shape_.push_back(input_shape[0]); + transpose_axis_.push_back(1); + transpose_axis_.push_back(0); + } else { + MS_LOG(EXCEPTION) << "Input is " << shape_size_ << "-D, but axis(" << axis << ") is invalid."; + } + + height_ = 1; + width_ = 1; + input_size_ = sizeof(T) * batch_size_ * channel_size_ * height_ * width_; + output_size_ = input_size_; + workspace_size_ = IntToSize(shape_size_) * sizeof(int); + } + + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t input_descriptor_; + cudnnTensorDescriptor_t output_descriptor_; + cudnnSoftmaxAlgorithm_t algo_; + cudnnSoftmaxMode_t mode_; + cudnnDataType_t cudnn_data_type_; + bool is_null_input_; + size_t input_size_; + size_t output_size_; + size_t workspace_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + std::vector input_shape_; + std::vector transpose_shape_; + std::vector transpose_axis_; + int axis_; + int shape_size_; + + size_t batch_size_; + size_t channel_size_; + size_t height_; + size_t width_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_SOFTMAX_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.cc new file mode 100644 index 00000000000..537eeb5726a --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_TWO( + SparseSoftmaxCrossEntropyWithLogits, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeFloat32), + SparseSoftmaxCrossEntropyWithLogitsGpuKernel, float, int) +MS_REG_GPU_KERNEL_TWO( + SparseSoftmaxCrossEntropyWithLogits, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeInt64).AddOutputAttr(kNumberTypeFloat32), + SparseSoftmaxCrossEntropyWithLogitsGpuKernel, float, int64_t) +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.h new file mode 100644 index 00000000000..d232f7a131d --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/sparse_softmax_cross_entropy_with_logits_gpu_kernel.h @@ -0,0 +1,206 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_GPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_GPU_KERNEL_H_ + +#include +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/cross_entropy_impl.cuh" +#include "kernel/gpu/kernel_constants.h" + +namespace mindspore { +namespace kernel { +template +class SparseSoftmaxCrossEntropyWithLogitsGpuKernel : public GpuKernel { + public: + SparseSoftmaxCrossEntropyWithLogitsGpuKernel() + : cudnn_handle_(nullptr), + logits_descriptor_(nullptr), + softmax_output_descriptor_(nullptr), + algo_(CUDNN_SOFTMAX_ACCURATE), + mode_(CUDNN_SOFTMAX_MODE_INSTANCE), + cudnn_data_type_(CUDNN_DATA_FLOAT), + is_grad_(false), + is_null_input_(false), + logits_size_(0), + labels_size_(0), + output_size_(0), + softmax_output_logits_size_(0), + batch_size_(0), + channel_size_(0), + height_(0), + width_(0) {} + ~SparseSoftmaxCrossEntropyWithLogitsGpuKernel() override { DestroyResource(); } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override { + if (is_null_input_) { + return true; + } + T *logits_addr = GetDeviceAddress(inputs, 0); + S *labels_addr = GetDeviceAddress(inputs, 1); + T *output_addr = GetDeviceAddress(outputs, 0); + T *softmax_output_logits = GetDeviceAddress(workspace, 0); + + const float alpha = 1; + const float beta = 0; + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSoftmaxForward(cudnn_handle_, algo_, mode_, &alpha, logits_descriptor_, logits_addr, &beta, + softmax_output_descriptor_, softmax_output_logits), + "cudnnSoftmaxForward failed."); + + is_grad_ ? CrossEntropyGradWithSparse(softmax_output_logits, labels_addr, batch_size_, channel_size_, output_addr, + reinterpret_cast(stream_ptr)) + : CrossEntropyWithSparse(softmax_output_logits, labels_addr, batch_size_, channel_size_, output_addr, + reinterpret_cast(stream_ptr)); + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Input number is " << input_num + << ", but SparseSoftmaxCrossEntropyWithLogitsGpuKernel needs 2 inputs."; + return false; + } + size_t output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + if (output_num != 1) { + MS_LOG(ERROR) << "Output number is " << output_num + << ", but SparseSoftmaxCrossEntropyWithLogitsGpuKernel needs 1 output."; + return false; + } + is_grad_ = GetAttr(kernel_node, "is_grad"); + cudnn_data_type_ = kCudnnDtypeMap[TypeIdLabel(AnfAlgo::GetInputDeviceDataType(kernel_node, 0))]; + + InferInputOutputSize(kernel_node); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnSetTensor4dDescriptor(logits_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, + batch_size_, channel_size_, height_, width_), + "cudnnSetTensor4dDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT( + cudnnSetTensor4dDescriptor(softmax_output_descriptor_, CUDNN_TENSOR_NCHW, cudnn_data_type_, batch_size_, + channel_size_, height_, width_), + "cudnnSetTensor4dDescriptor failed."); + InitSizeLists(); + return true; + } + + protected: + void InitResource() override { + cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&logits_descriptor_), + "cudnnCreateTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&softmax_output_descriptor_), + "cudnnCreateTensorDescriptor failed."); + } + void InitSizeLists() override { + input_size_list_.push_back(logits_size_); + input_size_list_.push_back(labels_size_); + output_size_list_.push_back(output_size_); + workspace_size_list_.push_back(softmax_output_logits_size_); + return; + } + + private: + void DestroyResource() noexcept { + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(softmax_output_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + CHECK_CUDNN_RET_WITH_ERROR(cudnnDestroyTensorDescriptor(logits_descriptor_), + "cudnnDestroyTensorDescriptor failed."); + } + void InferInputOutputSize(const CNodePtr &kernel_node) { + auto logits_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(logits_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "SoftmaxCrossEntropyWithLogitsGpuKernel input1 is null"; + InitSizeLists(); + return; + } + auto labels_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); + is_null_input_ = CHECK_NULL_INPUT(logits_shape); + if (is_null_input_) { + MS_LOG(WARNING) << "SoftmaxCrossEntropyWithLogitsGpuKernel input2 is null"; + InitSizeLists(); + return; + } + CheckShapeValidation(logits_shape, labels_shape); + + size_t logits_dims = logits_shape.size(); + batch_size_ = 1; + for (size_t i = 0; i < logits_dims - 1; i++) { + batch_size_ *= logits_shape[i]; + } + channel_size_ = logits_shape[logits_dims - 1]; + height_ = 1; + width_ = 1; + logits_size_ = sizeof(T) * batch_size_ * channel_size_ * height_ * width_; + + labels_size_ = 1; + size_t labels_dims = labels_shape.size(); + for (size_t i = 0; i < labels_dims; i++) { + labels_size_ *= labels_shape[i]; + } + labels_size_ *= sizeof(S); + + output_size_ = is_grad_ ? logits_size_ : sizeof(T); + softmax_output_logits_size_ = logits_size_; + return; + } + void CheckShapeValidation(const std::vector &logits_shape, const std::vector &labels_shape) { + size_t logits_dim_length = logits_shape.size(); + size_t labels_dim_length = labels_shape.size(); + if (labels_dim_length != logits_dim_length - 1) { + MS_LOG(EXCEPTION) << "Labels shape length should be equal to Logits shape length minus 1 for " + "SparseSoftmaxCrossEntropyWithLogits, " + "but got Labels shape length:" + << labels_dim_length << ", Logits shape length:" << logits_dim_length; + } + if (!std::equal(labels_shape.begin(), labels_shape.end(), logits_shape.begin())) { + MS_LOG(EXCEPTION) << "The shape of labels should be the same as the shape of logits except its last demension."; + } + return; + } + + cudnnHandle_t cudnn_handle_; + cudnnTensorDescriptor_t logits_descriptor_; + cudnnTensorDescriptor_t softmax_output_descriptor_; + cudnnSoftmaxAlgorithm_t algo_; + cudnnSoftmaxMode_t mode_; + cudnnDataType_t cudnn_data_type_; + bool is_grad_; + bool is_null_input_; + + size_t logits_size_; + size_t labels_size_; + size_t output_size_; + size_t softmax_output_logits_size_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + size_t batch_size_; + size_t channel_size_; + size_t height_; + size_t width_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS_GPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/hccl/hccl_kernel.cc b/mindspore/ccsrc/kernel/hccl/hccl_kernel.cc new file mode 100755 index 00000000000..0c6d45f0087 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hccl_kernel.cc @@ -0,0 +1,154 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hccl_kernel.h" +#include "device/ascend/tasksink/runtime_utils.h" +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" + +using HcclTaskInfoPtr = std::shared_ptr; +using ge::model_runner::HcclTaskInfo; +using mindspore::device::ascend::tasksink::RuntimeUtils; + +namespace mindspore { +namespace kernel { +void HcclKernelFactory::Registe(const std::string &name, HcclKernelCreater &&fun) { + hcclKernelMap_.emplace(name, std::move(fun)); +} + +std::shared_ptr HcclKernelFactory::Get(const std::string &name) { + const auto &map = Get().hcclKernelMap_; + auto it = map.find(name); + if (it != map.end() && it->second) { + return (it->second)(); + } + return nullptr; +} + +HcclKernelFactory &HcclKernelFactory::Get() { + static HcclKernelFactory _this; + return _this; +} + +HcclKernel::HcclKernel() : hccl_count_(0), op_type_(HCCL_REP_OP_SUM), root_id_(0), anf_node_(nullptr) {} + +HcclKernel::~HcclKernel() { + hccl_kernel_input_shape_list_.clear(); + hccl_kernel_output_shape_list_.clear(); + hccl_data_type_list_.clear(); + hccl_count_ = 0; + op_type_ = HCCL_REP_OP_SUM; + root_id_ = 0; + input_size_list_.clear(); + output_size_list_.clear(); + workspace_size_list_.clear(); + anf_node_ = nullptr; +} + +bool HcclKernel::Init(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + op_name_ = AnfAlgo::GetCNodeName(anf_node); + + if (!HcomUtil::GetKernelInputShape(anf_node, &hccl_kernel_input_shape_list_)) { + MS_LOG(ERROR) << "GetKernelInputShape fail!"; + return false; + } + if (!HcomUtil::GetKernelOutputShape(anf_node, &hccl_kernel_output_shape_list_)) { + MS_LOG(ERROR) << "GetKernelOutputShape fail!"; + return false; + } + if (!HcomUtil::GetHcomDataType(anf_node, &hccl_data_type_list_)) { + MS_LOG(ERROR) << "GetHcomDataType fail!"; + return false; + } + if (!HcomUtil::GetHcomCount(anf_node, hccl_data_type_list_, hccl_kernel_input_shape_list_, &hccl_count_)) { + MS_LOG(ERROR) << "GetHcomCount fail!"; + return false; + } + if (op_name_ == kAllReduce || op_name_ == kReduceScatter) { + if (!HcomUtil::GetHcomOperationType(anf_node, &op_type_)) { + MS_LOG(ERROR) << "GetHcomOperationType fail!"; + return false; + } + } + if (op_name_ == kBroadcast) { + if (!HcomUtil::GetHcomRootId(anf_node, &root_id_)) { + MS_LOG(ERROR) << "GetHcomRootId fail!"; + return false; + } + } + anf_node_ = anf_node; + return true; +} + +const std::vector &HcclKernel::GetInputSizeList() const { + size_t size = 0; + if (!input_size_list_.empty()) { + return input_size_list_; + } + for (ulong i = 0; i < hccl_data_type_list_.size(); ++i) { + if (!HcomUtil::GetHcclOpSize(hccl_data_type_list_[i], hccl_kernel_input_shape_list_[i], &size)) { + MS_LOG(ERROR) << "GetHcclOpInputSize failed"; + } + input_size_list_.push_back(size); + } + return input_size_list_; +} + +const std::vector &HcclKernel::GetOutputSizeList() const { + size_t size = 0; + if (!output_size_list_.empty()) { + return output_size_list_; + } + for (ulong i = 0; i < hccl_data_type_list_.size(); ++i) { + if (!HcomUtil::GetHcclOpSize(hccl_data_type_list_[i], hccl_kernel_output_shape_list_[i], &size)) { + MS_LOG(ERROR) << "GetHcclOpOutputSize failed"; + } + output_size_list_.push_back(size); + } + return output_size_list_; +} + +const std::vector &HcclKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } + +vector HcclKernel::GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) { + if (inputs.empty() || outputs.empty()) { + MS_LOG(EXCEPTION) << "inputs or outputs is empty"; + } + std::string hccl_type = AnfAlgo::GetCNodeName(anf_node_); + MS_EXCEPTION_IF_NULL(inputs.at(0)); + auto input_data_addr = inputs.at(0)->addr; + MS_EXCEPTION_IF_NULL(outputs.at(0)); + auto output_data_addr = outputs.at(0)->addr; + void *workspace_address = nullptr; + const int64_t workspace_num = 0; + std::vector private_def; + hcclDataType_t data_type = hccl_data_type_list_[0]; + + MS_LOG(INFO) << "HCCL Task : stream_id=" << stream_id << ", ws_num=" << workspace_num << ", count=" << hccl_count_ + << ", root_id=" << root_id_ << ", op_type=" << static_cast(op_type_) + << ", data_type=" << static_cast(data_type); + + HcclTaskInfoPtr task_info_ptr = std::make_shared( + stream_id, hccl_type, input_data_addr, output_data_addr, workspace_address, workspace_num, 0, private_def, nullptr, + hccl_count_, root_id_, op_type_, data_type, RuntimeUtils::HcomBindModel, RuntimeUtils::HcomUnbindModel, + RuntimeUtils::HcomDistribute); + MS_EXCEPTION_IF_NULL(task_info_ptr); + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hccl_kernel.h b/mindspore/ccsrc/kernel/hccl/hccl_kernel.h new file mode 100644 index 00000000000..71d9e5ba6a0 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hccl_kernel.h @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_H_ + +#include +#include +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "kernel/hccl/hcom_util.h" +#include "hccl/hcom.h" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +class HcclKernel : public KernelMod { + public: + HcclKernel(); + ~HcclKernel() override; + virtual bool Init(const AnfNodePtr &anf_node); + const std::vector &GetInputSizeList() const override; + const std::vector &GetOutputSizeList() const override; + const std::vector &GetWorkspaceSizeList() const override; + vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + protected: + std::vector> hccl_kernel_input_shape_list_; + std::vector> hccl_kernel_output_shape_list_; + std::vector hccl_data_type_list_; + std::vector hccl_format_list_; + uint64_t hccl_count_; + hcclRedOp_t op_type_; + uint32_t root_id_; + mutable std::vector input_size_list_; + mutable std::vector output_size_list_; + mutable std::vector workspace_size_list_; + AnfNodePtr anf_node_; + std::string op_name_; +}; + +using HcclKernelCreater = std::function()>; + +class HcclKernelFactory { + HcclKernelFactory() = default; + ~HcclKernelFactory() = default; + + public: + static HcclKernelFactory &Get(); + void Registe(const string &name, HcclKernelCreater &&fun); + static std::shared_ptr Get(const string &name); + + private: + std::map hcclKernelMap_; +}; + +class _HcclKernelRegister { + public: + _HcclKernelRegister(const string &name, HcclKernelCreater &&fun) { + HcclKernelFactory::Get().Registe(name, std::move(fun)); + } + ~_HcclKernelRegister() = default; +}; + +#define _MS_HCCL_REG_KERNEL_REG(KNAME, clazz) \ + static_assert(std::is_base_of::value, " must be base of HcclKernel"); \ + static const _HcclKernelRegister g_##KNAME##_##_kernel_reg(#KNAME, []() { \ + std::shared_ptr ptr = nullptr; \ + ptr = std::make_shared(); \ + MS_EXCEPTION_IF_NULL(ptr); \ + return ptr; \ + }); + +#define MS_HCCL_REG_KERNEL(KNAME, clazz) _MS_HCCL_REG_KERNEL_REG(KNAME, clazz) +} // namespace kernel +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/kernel/hccl/hccl_kernel_build.cc b/mindspore/ccsrc/kernel/hccl/hccl_kernel_build.cc new file mode 100644 index 00000000000..d6e4aa09b9d --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hccl_kernel_build.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hccl_kernel_build.h" + +#include +#include +#include + +#include "kernel/hccl/hccl_kernel.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +KernelModPtr HcclOpBuild(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string opname = AnfAlgo::GetCNodeName(anf_node); + MS_LOG(INFO) << "Hccl op [" << opname << "]"; + auto kerPtr = HcclKernelFactory::Get(opname); + if (kerPtr == nullptr) { + MS_LOG(ERROR) << "Hccl can't find Kernel[" << opname << "]"; + return nullptr; + } + if (!kerPtr->Init(anf_node)) { + MS_LOG(ERROR) << "Kernel initialize failed!"; + return nullptr; + } + return kerPtr; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hccl_kernel_build.h b/mindspore/ccsrc/kernel/hccl/hccl_kernel_build.h new file mode 100644 index 00000000000..f20760a3ebb --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hccl_kernel_build.h @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_BUILD_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_BUILD_H_ + +#include +#include +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { +KernelModPtr HcclOpBuild(const AnfNodePtr &anf_node); +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/hccl/hccl_kernel_metadata.cc b/mindspore/ccsrc/kernel/hccl/hccl_kernel_metadata.cc new file mode 100755 index 00000000000..80e87b0c6d6 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hccl_kernel_metadata.cc @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hccl_kernel_metadata.h" +#include +#include "utils/utils.h" +#include "kernel/hccl/hcom_util.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +void HcclMetadataInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list) { + MS_EXCEPTION_IF_NULL(kernel_info_list); + MS_EXCEPTION_IF_NULL(kernel_node); + std::string op_name = AnfAlgo::GetCNodeName(kernel_node); + if (op_name != kAllGather && op_name != kAllReduce && op_name != kBroadcast && op_name != kReduceScatter) { + MS_LOG(INFO) << "Hccl can't find op [" << op_name << "]"; + return; + } + + std::vector data_type_list{kNumberTypeFloat32, kNumberTypeFloat16, kNumberTypeInt8, kNumberTypeInt32}; + std::vector input_format, output_format; + std::vector input_type, output_type; + for (const auto &data_type : data_type_list) { + for (const auto &format : k4DSupportFormat) { + auto builder = std::make_shared(); + input_format.clear(); + input_format.push_back(format); + input_type.clear(); + input_type.push_back(data_type); + output_format.clear(); + output_format.push_back(format); + output_type.clear(); + output_type.push_back(data_type); + + builder->SetInputsFormat(input_format); + builder->SetInputsDeviceType(input_type); + builder->SetOutputsFormat(output_format); + builder->SetOutputsDeviceType(output_type); + builder->SetKernelType(HCCL_KERNEL); + kernel_info_list->emplace_back(builder->Build()); + } + } +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hccl_kernel_metadata.h b/mindspore/ccsrc/kernel/hccl/hccl_kernel_metadata.h new file mode 100755 index 00000000000..b13393d3bd7 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hccl_kernel_metadata.h @@ -0,0 +1,29 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_METADATA_ANFALGO_H_ +#define MINDSPORE_MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_METADATA_ANFALGO_H_ +#include +#include +#include +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace kernel { +void HcclMetadataInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_KERNEL_HCCL_HCCL_KERNEL_METADATA_ANFALGO_H_ diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_broadcast.cc b/mindspore/ccsrc/kernel/hccl/hcom_all_broadcast.cc new file mode 100644 index 00000000000..3cc57fe6d88 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_broadcast.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hcom_all_broadcast.h" + +#include +#include +#include + +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace kernel { +bool HcomAllBroadCastKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_task_sink()) { + return true; + } + const char *tag = "Hccl-BroadCast"; + auto stream = reinterpret_cast(stream_ptr); + hcclResult_t ret = + hcom_broadcast(tag, inputs[0]->addr, hccl_count_, hccl_data_type_list_[0], root_id_, nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "HcomBroadcastOp : hcom_broadcast fail, return: " << static_cast(ret); + return false; + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_broadcast.h b/mindspore/ccsrc/kernel/hccl/hcom_all_broadcast.h new file mode 100644 index 00000000000..d7d02a94514 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_broadcast.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_BROADCAST_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_BROADCAST_H_ + +#include +#include +#include "hccl/hcom.h" +#include "kernel/hccl/hccl_kernel.h" + +namespace mindspore { +namespace kernel { +class HcomAllBroadCastKernel : public HcclKernel { + public: + HcomAllBroadCastKernel() = default; + ~HcomAllBroadCastKernel() override = default; + + /* Inherit from kernelmod */ + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + + private: +}; +MS_HCCL_REG_KERNEL(Broadcast, HcomAllBroadCastKernel); +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_gather.cc b/mindspore/ccsrc/kernel/hccl/hcom_all_gather.cc new file mode 100644 index 00000000000..fde1e3bb12d --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_gather.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hcom_all_gather.h" + +#include +#include +#include + +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace kernel { +bool HcomAllGatherKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_task_sink()) { + return true; + } + const char *tag = "Hccl-AllGather"; + auto stream = reinterpret_cast(stream_ptr); + hcclResult_t ret = + hcom_all_gather(tag, inputs[0]->addr, outputs[0]->addr, hccl_count_, hccl_data_type_list_[0], nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "HcomAllGatherKernelOp : hcom_all_gather fail, return: " << static_cast(ret); + return false; + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_gather.h b/mindspore/ccsrc/kernel/hccl/hcom_all_gather.h new file mode 100644 index 00000000000..f29b5cc0f69 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_gather.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_GATHER_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_GATHER_H_ + +#include +#include +#include "hccl/hcom.h" +#include "kernel/hccl/hccl_kernel.h" + +namespace mindspore { +namespace kernel { +class HcomAllGatherKernel : public HcclKernel { + public: + HcomAllGatherKernel() = default; + ~HcomAllGatherKernel() override = default; + + /* Inherit from kernelmod */ + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + + private: +}; +MS_HCCL_REG_KERNEL(AllGather, HcomAllGatherKernel); +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_reduce.cc b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce.cc new file mode 100644 index 00000000000..a0d96683c22 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hcom_all_reduce.h" + +#include +#include +#include + +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace kernel { +bool HcomAllReduceKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_task_sink()) { + return true; + } + const char *tag = "Hccl-AllReduce"; + auto stream = reinterpret_cast(stream_ptr); + hcclResult_t ret = hcom_all_reduce(tag, inputs[0]->addr, outputs[0]->addr, hccl_count_, hccl_data_type_list_[0], + op_type_, nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "HcomAllReduceKernelOp : hcom_all_reduce fail, return: " << static_cast(ret); + return false; + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_reduce.h b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce.h new file mode 100644 index 00000000000..0a3bdb32844 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_REDUCE_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_REDUCE_H_ + +#include +#include +#include "kernel/hccl/hccl_kernel.h" + +namespace mindspore { +namespace kernel { +class HcomAllReduceKernel : public HcclKernel { + public: + HcomAllReduceKernel() = default; + ~HcomAllReduceKernel() override = default; + + /* Inherit from kernelmod */ + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + + private: +}; + +MS_HCCL_REG_KERNEL(AllReduce, HcomAllReduceKernel); +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_reduce_scatter.cc b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce_scatter.cc new file mode 100644 index 00000000000..36341ed6a75 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce_scatter.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hcom_all_reduce_scatter.h" + +#include +#include +#include + +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace kernel { +bool HcomAllReduceScatterKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_task_sink()) { + return true; + } + const char *tag = "Hccl-ReduceScatter"; + auto stream = reinterpret_cast(stream_ptr); + hcclResult_t ret = hcom_reduce_scatter(tag, inputs[0]->addr, outputs[0]->addr, hccl_count_, hccl_data_type_list_[0], + op_type_, nullptr, stream); + if (ret != HCCL_SUCCESS) { + MS_LOG(ERROR) << "HcomReduceScatterOp : hcom_reduce_scatter fail, return: " << static_cast(ret); + return false; + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hcom_all_reduce_scatter.h b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce_scatter.h new file mode 100644 index 00000000000..4c4f821d36b --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_all_reduce_scatter.h @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_REDUCE_SCATTER_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_ALL_REDUCE_SCATTER_H_ + +#include +#include +#include "hccl/hcom.h" +#include "kernel/hccl/hccl_kernel.h" + +namespace mindspore { +namespace kernel { +class HcomAllReduceScatterKernel : public HcclKernel { + public: + HcomAllReduceScatterKernel() = default; + ~HcomAllReduceScatterKernel() override = default; + + /* Inherit from kernelmod */ + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + + private: +}; + +MS_HCCL_REG_KERNEL(ReduceScatter, HcomAllReduceScatterKernel); +} // namespace kernel +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/hccl/hcom_util.cc b/mindspore/ccsrc/kernel/hccl/hcom_util.cc new file mode 100644 index 00000000000..8e5f9cb7e66 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_util.cc @@ -0,0 +1,186 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/hccl/hcom_util.h" + +#include + +#include "kernel/common_utils.h" +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" + +namespace mindspore { +bool HcomUtil::GetKernelInputShape(const AnfNodePtr &anf_node, vector> *hccl_kernel_intput_shape_list) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(hccl_kernel_intput_shape_list); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(anf_node); ++i) { + std::vector shape_i = AnfAlgo::GetInputDeviceShape(anf_node, i); + hccl_kernel_intput_shape_list->emplace_back(shape_i); + } + + return true; +} + +bool HcomUtil::GetKernelOutputShape(const AnfNodePtr &anf_node, vector> *hccl_kernel_output_shape_list) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(hccl_kernel_output_shape_list); + for (size_t i = 0; i < AnfAlgo::GetOutputTensorNum(anf_node); ++i) { + std::vector shape_i = AnfAlgo::GetOutputDeviceShape(anf_node, i); + hccl_kernel_output_shape_list->emplace_back(shape_i); + } + + return true; +} + +bool HcomUtil::GetHcomDataType(const AnfNodePtr &anf_node, vector *data_type_list) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(data_type_list); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(anf_node); ++i) { + auto type_ptr = AnfAlgo::GetPrevNodeOutputInferDataType(anf_node, i); + auto iter = CONST_OP_HCOM_DATA_TYPE_MAP.find(type_ptr); + if (iter == CONST_OP_HCOM_DATA_TYPE_MAP.end()) { + MS_LOG(EXCEPTION) << "HcomDataType cann't support Current Ascend Data Type : " << type_ptr; + } + data_type_list->emplace_back(iter->second); + } + auto type_base = *(std::begin(*data_type_list)); + if (std::any_of(data_type_list->begin(), data_type_list->end(), + [&type_base](hcclDataType_t type) { return type != type_base; })) { + MS_LOG(ERROR) << "hccl have different data type"; + return false; + } + return true; +} + +bool HcomUtil::GetHcclOpSize(const hcclDataType_t &data_type, const vector &shape, size_t *size) { + int tmp_size = 1; + uint32_t type_size = 4; + for (size_t i = 0; i < shape.size(); i++) { + IntMulWithOverflowCheck(tmp_size, SizeToInt(shape[i]), &tmp_size); + } + + if (!GetHcomTypeSize(data_type, &type_size)) { + return false; + } + + IntMulWithOverflowCheck(tmp_size, UintToInt(type_size), &tmp_size); + *size = IntToSize(tmp_size); + + MS_LOG(INFO) << "size[" << *size << "]"; + return true; +} + +bool HcomUtil::GetHcomTypeSize(const hcclDataType_t &data_type, uint32_t *size) { + auto iter = CONST_OP_HCOM_DATA_TYPE_SIZE_MAP.find(data_type); + if (iter == CONST_OP_HCOM_DATA_TYPE_SIZE_MAP.end()) { + MS_LOG(ERROR) << "HcomUtil::HcomDataTypeSize, No DataTypeSize!"; + return false; + } + *size = iter->second; + return true; +} + +bool HcomUtil::GetHcomCount(const AnfNodePtr &anf_node, const vector &data_type_list, + const vector> &shape_list, uint64_t *total_count) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(total_count); + const uint32_t align_size = 512; + const uint32_t filled_size = 32; + uint64_t total_size = 0; + uint64_t block_size; + size_t input_size; + uint32_t type_size = 4; + + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(anf_node); ++i) { + if (!GetHcomTypeSize(data_type_list[i], &type_size)) { + return false; + } + + if (!GetHcclOpSize(data_type_list[i], shape_list[i], &input_size)) { + MS_LOG(ERROR) << "Get GetHcclOpSize failed"; + return false; + } + + if (AnfAlgo::GetCNodeName(anf_node) == kReduceScatterOpName) { + int32_t rank_size; + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr("rank_size") != nullptr) { + rank_size = GetValue(primitive->GetAttr("rank_size")); + } else { + MS_LOG(ERROR) << "Get rank size failed"; + return false; + } + block_size = input_size / IntToSize(rank_size); + total_size = total_size + block_size; + } else { + if (AnfAlgo::GetCNodeName(anf_node) == kAllGatherOpName) { + block_size = input_size; + } else { + block_size = (input_size + align_size - 1 + filled_size) / align_size * align_size; + } + total_size = total_size + block_size; + } + } + + if (total_size % type_size != 0) { + MS_LOG(ERROR) << "Total_size[" << total_size << "],Type_size[" << type_size << "] != 0, fail!"; + return false; + } + *total_count = total_size / type_size; + return true; +} + +bool HcomUtil::GetHcomOperationType(const AnfNodePtr &anf_node, hcclRedOp_t *op_type) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(op_type); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr("op") == nullptr) { + MS_LOG(ERROR) << "Get HCOM_ATTR_REDUCE_TYPE fail, not support!"; + return false; + } + auto hcom_op_type_get = GetValue(primitive->GetAttr("op")); + string hcom_op_type(hcom_op_type_get); + if (hcom_op_type == "min") { + *op_type = HCCL_REP_OP_MIN; + } else if (hcom_op_type == "max") { + *op_type = HCCL_REP_OP_MAX; + } else if (hcom_op_type == "prod") { + *op_type = HCCL_REP_OP_PROD; + } else if (hcom_op_type == "sum") { + *op_type = HCCL_REP_OP_SUM; + } else { + MS_LOG(ERROR) << "HcomUtil::Get HCOM_ATTR_REDUCE_TYPE fail, [" << hcom_op_type << "] not support!"; + return false; + } + return true; +} + +bool HcomUtil::GetHcomRootId(const AnfNodePtr &anf_node, uint32_t *root_id) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(root_id); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr("root_rank") != nullptr) { + *root_id = GetValue>(primitive->GetAttr("root_rank"))[0]; + } else { + MS_LOG(ERROR) << "HcomUtil::Get HCOM_ATTR_ROOT_INDEX fail, not support!"; + return false; + } + return true; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/hccl/hcom_util.h b/mindspore/ccsrc/kernel/hccl/hcom_util.h new file mode 100644 index 00000000000..e0524b80662 --- /dev/null +++ b/mindspore/ccsrc/kernel/hccl/hcom_util.h @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_UTILS_H_ +#define MINDSPORE_CCSRC_KERNEL_HCCL_HCOM_UTILS_H_ + +#include +#include +#include +#include +#include "ir/dtype.h" +#include "hccl/base.h" + +namespace mindspore { +using std::map; +using std::string; +using std::vector; + +constexpr auto kAllGather = "AllGather"; +constexpr auto kAllReduce = "AllReduce"; +constexpr auto kBroadcast = "Broadcast"; +constexpr auto kReduceScatter = "ReduceScatter"; + +/* Correspondence between data_type and hcom data type in Ascend */ +static map CONST_OP_HCOM_DATA_TYPE_MAP = { + {TypeId::kNumberTypeFloat32, HCCL_DATA_TYPE_FLOAT}, + {TypeId::kNumberTypeFloat16, HCCL_DATA_TYPE_HALF}, + {TypeId::kNumberTypeInt8, HCCL_DATA_TYPE_INT8}, + {TypeId::kNumberTypeInt32, HCCL_DATA_TYPE_INT}, +}; + +/* Correspondence between data_type and occupied byte size in hcom */ +static map CONST_OP_HCOM_DATA_TYPE_SIZE_MAP = { + {HCCL_DATA_TYPE_FLOAT, sizeof(float)}, + {HCCL_DATA_TYPE_HALF, sizeof(float) / 2}, + {HCCL_DATA_TYPE_INT8, sizeof(int8_t)}, + {HCCL_DATA_TYPE_INT, sizeof(int32_t)}, +}; + +class HcomUtil { + public: + static bool GetKernelInputShape(const AnfNodePtr &anf_node, vector> *hccl_kernel_shape_list); + static bool GetKernelOutputShape(const AnfNodePtr &anf_node, vector> *hccl_kernel_shape_list); + static bool GetHcomDataType(const AnfNodePtr &anf_node, vector *data_type_list); + static bool GetHcclOpSize(const hcclDataType_t &data_type, const vector &shape, size_t *size); + static bool GetHcomTypeSize(const hcclDataType_t &data_type, uint32_t *size); + static bool GetHcomCount(const AnfNodePtr &anf_node, const vector &data_type_list, + const vector> &shape_list, uint64_t *total_count); + static bool GetHcomOperationType(const AnfNodePtr &anf_node, hcclRedOp_t *op_type); + static bool GetHcomRootId(const AnfNodePtr &anf_node, uint32_t *root_id); +}; +} // namespace mindspore + +#endif diff --git a/mindspore/ccsrc/kernel/kash/kernel_pack.cc b/mindspore/ccsrc/kernel/kash/kernel_pack.cc new file mode 100644 index 00000000000..31f81d5d022 --- /dev/null +++ b/mindspore/ccsrc/kernel/kash/kernel_pack.cc @@ -0,0 +1,247 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "mindspore/ccsrc/kernel/kernel.h" +#include "kernel/kernel.h" +#include "kernel/akg/akgkernelbuild.h" +#include "nlohmann/json.hpp" +#include "securec/include/securec.h" +#include "pipeline/parse/python_adapter.h" +#include "utils/log_adapter.h" +#include "utils/convert_utils.h" +namespace mindspore { +namespace kernel { +constexpr auto kUtilsModule = "mindspore._extends.utils"; +constexpr auto kCalSha256Func = "cal_sha256"; + +namespace { +bool CheckHash(const std::string &json_file, const std::string &bin_file, const nlohmann::json &js) { + if (js.find("sha256") == js.end()) { + MS_LOG(ERROR) << "No sha256 found in " << json_file; + return false; + } + std::string sha256_str = js["sha256"]; + py::object ret = parse::python_adapter::CallPyFn(kUtilsModule, kCalSha256Func, bin_file); + std::string sha256_cal = py::cast(ret); + if (sha256_cal.empty()) { + MS_LOG(ERROR) << "Cal sha256 of " << bin_file << " failed."; + return false; + } + if (sha256_cal != sha256_str) { + MS_LOG(ERROR) << "Cal sha256 of " << bin_file << " failed."; + return false; + } + return true; +} +} // namespace + +const std::string KernelPack::Serialize() const { + std::string buffer; + (void)buffer.append((const char *)json_, json_->len + sizeof(json_->len)); + (void)buffer.append((const char *)kernel_, kernel_->len + sizeof(kernel_->len)); + return buffer; +} + +bool KernelPack::ReadFromJsonFileHelper(std::ifstream &kernelbin) { + size_t binsize = LongToSize(kernelbin.seekg(0, std::ios::end).tellg()); + // free old data + if (kernel_ != nullptr) { + delete[] kernel_; + kernel_ = nullptr; + } + + void *ptr = static_cast(new (std::nothrow) uint8_t[sizeof(KernelPack) + binsize]); + if (ptr != nullptr) { + kernel_ = static_cast(ptr); + } + if (kernel_ == nullptr) { + MS_LOG(ERROR) << "memory malloc failed."; + kernelbin.close(); + return false; + } + if (memset_s(kernel_, sizeof(KernelPack) + binsize, 0, sizeof(KernelPack) + binsize) != EOK) { + MS_LOG(ERROR) << "memset kernel_ failed."; + delete[] kernel_; + kernel_ = nullptr; + kernelbin.close(); + return false; + } + kernel_->len = binsize; + MS_LOG(INFO) << "kernel len:" << kernel_->len; + (void)kernelbin.seekg(0, std::ios::beg); + (void)kernelbin.read(kernel_->contents, SizeToLong(kernel_->len)); + return true; +} + +bool KernelPack::ReadFromJsonFile(const std::string &json_f, const std::string &processor) { + if (json_f.length() <= strlen(kJsonSuffix)) { + MS_LOG(ERROR) << "please check json path."; + return false; + } + + std::ifstream kerneljson(json_f); + if (!kerneljson.is_open()) { + MS_LOG(DEBUG) << "read json file error, please check kernelmeta."; + return false; + } + nlohmann::json js; + kerneljson >> js; + + size_t binsize = LongToSize(kerneljson.seekg(0, std::ios::end).tellg()); + void *ptr = static_cast(new (std::nothrow) uint8_t[sizeof(KernelPack) + binsize]); + if (ptr != nullptr) { + json_ = static_cast(ptr); + } + if (json_ == nullptr) { + MS_LOG(ERROR) << "memory malloc failed."; + kerneljson.close(); + return false; + } + json_->len = binsize; + (void)kerneljson.seekg(0, std::ios::beg); + (void)kerneljson.read(json_->contents, SizeToLong(json_->len)); + + if (processor == kProcessorCuda) { + std::string bin_f = json_f.substr(0, json_f.length() - 5) + ".ptx"; + std::ifstream kernelbin(bin_f); + if (!kernelbin.is_open()) { + MS_LOG(ERROR) << "read kernel ptx file error, please check kernelmeta."; + kerneljson.close(); + return false; + } + + if (ReadFromJsonFileHelper(kernelbin) == false) { + delete[] json_; + json_ = nullptr; + kerneljson.close(); + return false; + } + kerneljson.close(); + if (!CheckHash(json_f, bin_f, js)) { + return false; + } + return true; + } + + std::string binfilesuffix = js["binFileSuffix"]; + std::string bin_f = json_f.substr(0, json_f.length() - 5) + binfilesuffix; + if (binfilesuffix.compare(".so") == 0) { + // change "xx/xx.so" -> "xx/libxx.so" + auto sp = bin_f.rfind('/'); + if (sp == std::string::npos) { + MS_LOG(ERROR) << "illegal bin file path " << bin_f; + kerneljson.close(); + return false; + } + bin_f = bin_f.substr(0, sp + 1) + "lib" + bin_f.substr(sp + 1, bin_f.length() - sp - 1); + } + + std::ifstream kernelbin(bin_f, std::ios::binary); + if (!kernelbin.is_open()) { + MS_LOG(ERROR) << "read kernel binary file error, please check kernelmeta."; + kerneljson.close(); + delete[] json_; + json_ = nullptr; + return false; + } + + MS_LOG(INFO) << "kernelbin_name:" << bin_f; + if (ReadFromJsonFileHelper(kernelbin) == false) { + delete[] json_; + json_ = nullptr; + kerneljson.close(); + return false; + } + kerneljson.close(); + + if (!CheckHash(json_f, bin_f, js)) { + return false; + } + + return true; +} + +void KernelPack::ParseKernelJson(const nlohmann::json &js) { + kernel_json_info_.bin_file_name = js["binFileName"]; + kernel_json_info_.bin_file_suffix = js["binFileSuffix"]; + kernel_json_info_.block_dim = js["blockDim"]; + kernel_json_info_.kernel_name = js["kernelName"]; + kernel_json_info_.magic = js["magic"]; + if (js.find("parameters") != js.end()) { + if (!js.at("parameters").is_array()) { + MS_LOG(DEBUG) << "Format error!,parameters should be array."; + } + std::vector sizes = js.at("parameters"); + for (auto size : sizes) { + MS_LOG(INFO) << "parameter " << size; + kernel_json_info_.parameters.push_back(size); + } + } + if (js.find("workspace") != js.end()) { + auto workspace = js.at("workspace"); + std::vector sizes = workspace.at("size"); + for (auto size : sizes) { + MS_LOG(INFO) << "workspace_size_list " << size; + kernel_json_info_.workspaces.push_back(size); + } + } + kernel_json_info_.sha256 = js["sha256"]; +} + +bool KernelPack::LoadKernelMeta(const std::string &json_f, const std::string &processor) { + if (json_f.length() <= strlen(kJsonSuffix)) { + MS_LOG(ERROR) << "please check json path."; + return false; + } + std::ifstream kernel_json(json_f); + if (!kernel_json.is_open()) { + MS_LOG(DEBUG) << "read json file error, please check kernelmeta."; + return false; + } + nlohmann::json js; + kernel_json >> js; + ParseKernelJson(js); + kernel_json.close(); + + std::string bin_f = json_f.substr(0, json_f.length() - 5) + kernel_json_info_.bin_file_suffix; + if (kernel_json_info_.bin_file_suffix == ".so") { + // change "xx/xx.so" -> "xx/libxx.so" + auto sp = bin_f.rfind('/'); + if (sp == std::string::npos) { + MS_LOG(ERROR) << "illegal bin file path " << bin_f; + return false; + } + bin_f = bin_f.substr(0, sp + 1) + "lib" + bin_f.substr(sp + 1, bin_f.length() - sp - 1); + } + + std::ifstream kernelbin(bin_f, std::ios::binary); + if (!kernelbin.is_open()) { + MS_LOG(ERROR) << "read kernel binary file error, please check kernelmeta."; + return false; + } + + MS_LOG(INFO) << "kernelbin_name:" << bin_f; + if (!ReadFromJsonFileHelper(kernelbin)) { + return false; + } + + return CheckHash(json_f, bin_f, js); +} + +KernelJsonInfo KernelPack::kernel_json_info() const { return kernel_json_info_; } +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/kernel.h b/mindspore/ccsrc/kernel/kernel.h new file mode 100644 index 00000000000..aecc51794c1 --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel.h @@ -0,0 +1,135 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_KERNEL_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_KERNEL_H_ +#include +#include +#include +#include "nlohmann/json.hpp" +#include "ir/anf.h" +#include "ir/dtype.h" +#include "utils/utils.h" +#include "ir/meta_tensor.h" +#include "pipeline/static_analysis/dshape.h" +#include "utils/log_adapter.h" +#include "framework/ge_runtime/task_info.h" + +namespace mindspore { +enum KernelType : int { UNKNOWN_KERNEL_TYPE = 0, AUTO_DIFF_KERNEL, AICPU_KERNEL, RT_KERNEL, HCCL_KERNEL, TBE_KERNEL }; + +namespace kernel { + +enum Axis { + N = 0, + C, + H, + W, +}; + +// Supported fusion type +enum FusionType { + CONVLUTION = 0, + ELEMWISE, + COMMREDUCE, + SEGMENT, + OPAQUE, + UNKNOWN_FUSION_TYPE = -1, +}; + +// Backend processor +enum Processor { + AICORE = 0, + AICPU, + CUDA, +}; + +struct FlexArray { + size_t len; + char contents[]; +}; + +struct KernelJsonInfo { + std::string bin_file_name; + std::string bin_file_suffix; + uint32_t block_dim; + std::string kernel_name; + std::string magic; + std::vector parameters; + std::string sha256; + std::vector workspaces; + KernelJsonInfo() : block_dim(0) {} +}; + +class KernelPack { + public: + KernelPack() : json_(nullptr), kernel_(nullptr) {} + KernelPack(const KernelPack &) = default; + KernelJsonInfo kernel_json_info() const; + bool LoadKernelMeta(const std::string &json_f, const std::string &processor); + bool ReadFromJsonFile(const std::string &json_f, const std::string &processor); + const std::string Serialize() const; + const FlexArray *const GetJson() const { return json_; } + const FlexArray *const GetKernel() const { return kernel_; } + ~KernelPack() { + if (json_) { + delete[] json_; + json_ = nullptr; + } + if (kernel_) { + delete[] kernel_; + kernel_ = nullptr; + } + } + + private: + bool ReadFromJsonFileHelper(std::ifstream &kernelbin); + void ParseKernelJson(const nlohmann::json &js); + KernelJsonInfo kernel_json_info_; + FlexArray *json_; + FlexArray *kernel_; +}; +using KernelPackPtr = std::shared_ptr; + +/** + * @brief base class for autotensor kernel and cce kernel. + */ +struct Address { + void *addr; + size_t size; +}; +using AddressPtr = std::shared_ptr
; +using TaskInfoPtr = std::shared_ptr; + +class KernelMod { + public: + virtual const std::vector &GetInputSizeList() const = 0; + virtual const std::vector &GetOutputSizeList() const = 0; + virtual const std::vector &GetWorkspaceSizeList() const = 0; + virtual bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) = 0; + virtual std::vector GenTask(const std::vector &, const std::vector &, + const std::vector &, uint32_t) { + return {}; + } + virtual std::vector GenParameters() { return {}; } + + virtual ~KernelMod() = default; +}; +using KernelModPtr = std::shared_ptr; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/kernel_build_info.cc b/mindspore/ccsrc/kernel/kernel_build_info.cc new file mode 100644 index 00000000000..c52f71c1369 --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel_build_info.cc @@ -0,0 +1,167 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/kernel_build_info.h" +#include +#include "utils/log_adapter.h" +namespace mindspore { +namespace kernel { +std::string KernelBuildInfo::GetInputFormat(size_t input_index) const { + if (input_index >= inputs_format_.size()) { + MS_LOG(EXCEPTION) << "The index [" << input_index << "] is exceed the number of input node"; + } + return inputs_format_[input_index]; +} + +std::string KernelBuildInfo::GetOutputFormat(size_t output_index) const { + if (output_index >= outputs_format_.size()) { + MS_LOG(EXCEPTION) << "The index [" << output_index << "] is exceed the number of input node"; + } + return outputs_format_[output_index]; +} + +TypeId KernelBuildInfo::GetInputDeviceType(size_t input_index) const { + if (input_index >= inputs_device_type_.size()) { + MS_LOG(EXCEPTION) << "The index [" << input_index << "] is exceed the number of input node"; + } + return inputs_device_type_[input_index]; +} + +TypeId KernelBuildInfo::GetOutputDeviceType(size_t output_index) const { + if (output_index >= outputs_device_type_.size()) { + MS_LOG(EXCEPTION) << "The index [" << output_index << "] is exceed the number of input node"; + } + return outputs_device_type_[output_index]; +} + +std::vector KernelBuildInfo::GetAllInputFormats() const { return inputs_format_; } + +std::vector KernelBuildInfo::GetAllOutputFormats() const { return outputs_format_; } + +std::vector KernelBuildInfo::GetAllInputDeviceTypes() const { return inputs_device_type_; } + +std::vector KernelBuildInfo::GetAllOutputDeviceTypes() const { return outputs_device_type_; } + +size_t KernelBuildInfo::GetInputNum() const { return inputs_format_.size(); } + +size_t KernelBuildInfo::GetOutputNum() const { return outputs_format_.size(); } + +bool KernelBuildInfo::GetInputReshapeType(size_t input_index, std::vector *reshape_type) const { + MS_EXCEPTION_IF_NULL(reshape_type); + reshape_type->clear(); + if (input_index >= input_reshape_type_.size()) { + MS_LOG(WARNING) << "The index [" << input_index << "] is exceed the number of input node size " + << input_reshape_type_.size(); + return false; + } + (void)std::copy(input_reshape_type_[input_index].begin(), input_reshape_type_[input_index].end(), + std::inserter(*reshape_type, (*reshape_type).begin())); + return true; +} + +bool KernelBuildInfo::GetOutputReshapeType(size_t output_index, std::vector *reshape_type) const { + MS_EXCEPTION_IF_NULL(reshape_type); + reshape_type->clear(); + if (output_index >= output_reshape_type_.size()) { + MS_LOG(WARNING) << "The index [" << output_index << "] is exceed the number of output node dixr" + << output_reshape_type_.size(); + return false; + } + (void)std::copy(output_reshape_type_[output_index].begin(), output_reshape_type_[output_index].end(), + std::inserter(*reshape_type, (*reshape_type).begin())); + return true; +} + +std::string KernelBuildInfo::ToString() const { + std::ostringstream output_buffer; + output_buffer << "("; + for (size_t index = 0; index < GetInputNum(); ++index) { + if (index != 0) { + output_buffer << ", "; + } + output_buffer << "<" << static_cast(GetInputDeviceType(index)) << "x" << GetInputFormat(index) << ">"; + } + output_buffer << ") -> ("; + for (size_t index = 0; index < GetOutputNum(); ++index) { + if (index != 0) { + output_buffer << ", "; + } + output_buffer << "<" << static_cast(GetOutputDeviceType(index)) << "x" << GetOutputFormat(index) << ">"; + } + output_buffer << ")"; + return output_buffer.str(); +} + +bool KernelBuildInfo::operator==(const KernelBuildInfo &other) const { + if (kernel_type_ != other.kernel_type_ || fusion_type_ != other.fusion_type_ || processor_ != other.processor_) { + return false; + } + if (inputs_format_ != other.inputs_format_ || outputs_format_ != other.outputs_format_) { + return false; + } + return !(inputs_device_type_ != other.inputs_device_type_ || outputs_device_type_ != other.outputs_device_type_); +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetKernelType(const KernelType &kernel_type) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->kernel_type_ = kernel_type; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetInputsFormat(const std::vector &inputs_format) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->inputs_format_ = inputs_format; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetOutputsFormat(const std::vector &outputs_format) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->outputs_format_ = outputs_format; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetInputsDeviceType(const std::vector &inputs_device_type) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->inputs_device_type_ = inputs_device_type; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetOutputsDeviceType(const std::vector &outputs_device_type) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->outputs_device_type_ = outputs_device_type; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetFusionType(FusionType fusion_type) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->fusion_type_ = fusion_type; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetProcessor(Processor processor) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->processor_ = processor; +} + +std::shared_ptr KernelBuildInfo::KernelBuildInfoBuilder::Build() { return kernel_build_info_; } + +void KernelBuildInfo::KernelBuildInfoBuilder::SetInputReshapeType( + const std::vector> &input_reshape_type) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->input_reshape_type_ = input_reshape_type; +} + +void KernelBuildInfo::KernelBuildInfoBuilder::SetOutputReshapeType( + const std::vector> &output_reshape_type) { + MS_EXCEPTION_IF_NULL(kernel_build_info_); + kernel_build_info_->output_reshape_type_ = output_reshape_type; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/kernel_build_info.h b/mindspore/ccsrc/kernel/kernel_build_info.h new file mode 100644 index 00000000000..24552e0341a --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel_build_info.h @@ -0,0 +1,128 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_KERNEL_BUILD_INFO_H_ +#define MINDSPORE_CCSRC_KERNEL_KERNEL_BUILD_INFO_H_ +#include +#include +#include +#include +#include +#include "ir/dtype.h" +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { +class KernelBuildInfo { + public: + class KernelBuildInfoBuilder; + + KernelBuildInfo() { + kernel_type_ = AUTO_DIFF_KERNEL; + fusion_type_ = OPAQUE; + processor_ = AICORE; + input_reshape_type_ = {}; + output_reshape_type_ = {}; + inputs_format_ = {}; + outputs_format_ = {}; + inputs_device_type_ = {}; + outputs_device_type_ = {}; + } + + ~KernelBuildInfo() = default; + + KernelType kernel_type() const { return kernel_type_; } + + std::string GetInputFormat(size_t input_index) const; + + std::string GetOutputFormat(size_t output_index) const; + + TypeId GetInputDeviceType(size_t input_index) const; + + TypeId GetOutputDeviceType(size_t output_index) const; + + bool GetInputReshapeType(size_t input_index, std::vector *reshape_type) const; + + bool GetOutputReshapeType(size_t input_index, std::vector *reshape_type) const; + + std::vector GetAllInputFormats() const; + + std::vector GetAllOutputFormats() const; + + std::vector GetAllInputDeviceTypes() const; + + std::vector GetAllOutputDeviceTypes() const; + + FusionType fusion_type() const { return fusion_type_; } + + Processor processor() const { return processor_; } + + size_t GetInputNum() const; + + size_t GetOutputNum() const; + + std::string ToString() const; + + bool operator==(const KernelBuildInfo &other) const; + + private: + KernelType kernel_type_; + std::vector inputs_format_; + std::vector outputs_format_; + std::vector> input_reshape_type_; + std::vector> output_reshape_type_; + std::vector inputs_device_type_; + std::vector outputs_device_type_; + FusionType fusion_type_; + Processor processor_; +}; +using KernelBuildInfoPtr = std::shared_ptr; + +class KernelBuildInfo::KernelBuildInfoBuilder { + public: + KernelBuildInfoBuilder() { kernel_build_info_ = std::make_shared(); } + + explicit KernelBuildInfoBuilder(std::shared_ptr kernel_build_info) + : kernel_build_info_(std::move(kernel_build_info)) {} + + ~KernelBuildInfoBuilder() = default; + + void SetKernelType(const KernelType &kernel_type); + + void SetInputsFormat(const std::vector &inputs_format); + + void SetOutputsFormat(const std::vector &outputs_format); + + void SetInputsDeviceType(const std::vector &inputs_device_type); + + void SetOutputsDeviceType(const std::vector &outputs_device_type); + + void SetInputReshapeType(const std::vector> &input_reshape_type); + + void SetOutputReshapeType(const std::vector> &output_reshape_type); + + void SetFusionType(FusionType fusion_type); + + void SetProcessor(Processor processor); + + std::shared_ptr Build(); + + private: + std::shared_ptr kernel_build_info_; +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_KERNEL_BUILD_INFO_H_ diff --git a/mindspore/ccsrc/kernel/kernel_fusion.cc b/mindspore/ccsrc/kernel/kernel_fusion.cc new file mode 100644 index 00000000000..cd8936f2186 --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel_fusion.cc @@ -0,0 +1,123 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/kernel_fusion.h" + +#include +#include +#include +#include + +#include "common/utils.h" +#include "kernel/tbe/tbe_kernel_build.h" +#include "kernel/tbe/tbe_kernel_parallel_build.h" +#include "kernel/tbe/tbe_utils.h" +#include "kernel/tbe/tbe_convert_utils.h" + +namespace mindspore { +namespace kernel { +using mindspore::kernel::tbe::TbeUtils; +static bool GenPreBuildKernelJson(const std::vector &compute_nodes, + std::vector *prebuild_op_list) { + MS_EXCEPTION_IF_NULL(prebuild_op_list); + TbeKernelJsonCreator creator(PREBUILD); + for (const auto &anf_node : compute_nodes) { + nlohmann::json prebuild; + if (!creator.GenTbeSingleKernelJson(anf_node, &prebuild)) { + MS_LOG(ERROR) << "GenTbeSingleKernelJson failed"; + return false; + } + (*prebuild_op_list).push_back(prebuild); + } + return true; +} + +std::map KernelFusion(const std::vector &fusion_scopes) { + MS_LOG(INFO) << "kernel fusion build start, scope size:" << fusion_scopes.size(); + std::map kernel_mod_ret; + auto build_manger = std::make_shared(); + MS_EXCEPTION_IF_NULL(build_manger); + for (const auto &fusion_scope_iter : fusion_scopes) { + auto scope_id = fusion_scope_iter.scope_id; + nlohmann::json fusion_op; + string fusion_kernel = "te_fusion"; + if (!TbeKernelBuild::GenFusionScopeJson(fusion_scope_iter.input_nodes, fusion_scope_iter.compute_nodes, &fusion_op, + &fusion_kernel)) { + continue; + } + // gen kernel_name & check cache + std::string json_str = fusion_op.dump(); + size_t hash_id = std::hash()(json_str); + auto json_name = fusion_kernel.append("_").append(std::to_string(hash_id)); + fusion_op["fusion_op_name"] = json_name; + // gen json for prebuild + std::vector prebuild_op_list; + if (!GenPreBuildKernelJson(fusion_scope_iter.compute_nodes, &prebuild_op_list)) { + continue; + } + // get io size + std::vector input_size_list; + std::vector output_size_list; + if (!TbeKernelBuild::GetIOSize(fusion_op["op_list"], fusion_scope_iter.output_nodes, &input_size_list, + &output_size_list)) { + continue; + } + // search cache + auto kernel_pack = TbeUtils::SearchCache(json_name, tbe::kProcessorAiCore); + if (kernel_pack != nullptr) { + MS_LOG(INFO) << "Use cached kernel, kernel json name: " << json_name; + auto kernel_mod = + build_manger->GenKernelMod(json_name, tbe::kProcessorAiCore, input_size_list, output_size_list, kernel_pack); + if (kernel_mod != nullptr) { + kernel_mod_ret[scope_id] = kernel_mod; + continue; + } + } + // fusion build + nlohmann::json fusion_json; + fusion_json["fusion_op"] = fusion_op; + fusion_json["prebuild_ops"] = prebuild_op_list; + auto task_id = build_manger->StartCompileOp(fusion_json); + TbeUtils::SaveJsonInfo(json_name, fusion_json.dump()); + if (task_id < 0) { + MS_EXCEPTION(ArgumentError) << "start compile failed."; + } + build_manger->SaveTaskInfo(task_id, nullptr, json_name, input_size_list, output_size_list, scope_id); + } + + int build_failed_num = 0; + while (!build_manger->IsAllTaskFinish()) { + int task_id = -1; + char *task_result = nullptr; + auto ret = build_manger->WaitOne(&task_id, &task_result); + if (!ret) { + MS_EXCEPTION(ArgumentError) << "Build Failed. wait one ret:" << ret << ", task id:" << task_id; + } + + if ((task_result != nullptr) && (strcmp(task_result, "Success") != 0)) { + MS_LOG(DEBUG) << "fuison op build failed, err log: " << task_result << " change to single op build."; + build_failed_num++; + } + auto kernel_mod_item = build_manger->TaskFinishProcess(task_id, false); + if (kernel_mod_item.second != nullptr) { + (void)kernel_mod_ret.emplace(kernel_mod_item); + } + } + MS_LOG(INFO) << "Build Fusion Kernel Failed Num: " << build_failed_num; + return kernel_mod_ret; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/kernel_fusion.h b/mindspore/ccsrc/kernel/kernel_fusion.h new file mode 100644 index 00000000000..8ded21787c3 --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel_fusion.h @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_KERNELFUSION_H_ +#define MINDSPORE_CCSRC_KERNEL_KERNELFUSION_H_ +#include +#include +#include "kernel/kernel.h" +namespace mindspore { +namespace kernel { +/* + * @brief fuse op and return a callable mod + */ +struct FusionScopeInfo { + int32_t scope_id; + std::vector input_nodes; + std::vector compute_nodes; + std::vector output_nodes; +}; + +std::map KernelFusion(const std::vector &fusion_scopes); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_KERNELFUSION_H_ diff --git a/mindspore/ccsrc/kernel/kernel_query.cc b/mindspore/ccsrc/kernel/kernel_query.cc new file mode 100755 index 00000000000..7934bd0a5c4 --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel_query.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/kernel_query.h" +#include +#include +#include "kernel/aicpu/aicpu_kernel_metadata.h" +#include "kernel/mng/rt_kernel_info.h" +#include "kernel/hccl/hccl_kernel_metadata.h" +#include "kernel/tbe/tbe_kernel_select.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +namespace { +void FilterInvaildKernelInfo(const CNodePtr& kernel_node, + std::vector>* kernel_info_list) { + MS_EXCEPTION_IF_NULL(kernel_info_list); + std::vector> filtered_list; + (void)std::copy_if(kernel_info_list->begin(), kernel_info_list->end(), std::back_inserter(filtered_list), + [&](const std::shared_ptr& kernel_build_info) { + return AnfAlgo::GetOutputTensorNum(kernel_node) == kernel_build_info->GetOutputNum() && + AnfAlgo::GetInputTensorNum(kernel_node) == kernel_build_info->GetInputNum(); + }); + kernel_info_list->clear(); + if (!filtered_list.empty()) { + (void)std::copy(filtered_list.begin(), filtered_list.end(), std::back_inserter(*kernel_info_list)); + } else { + MS_LOG(EXCEPTION) << "node" << kernel_node->DebugString() << "'s output size : [" + << AnfAlgo::GetOutputTensorNum(kernel_node) << "]" + << "input size : [" << AnfAlgo::GetInputTensorNum(kernel_node) + << "] cannot match any kernelInfo !"; + } +} +} // namespace +void KernelQuery(const CNodePtr& kernel_node, std::vector>* kernel_info_list) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_info_list); + TbeMetadataInfo(kernel_node, kernel_info_list); + + if (kernel_info_list->empty()) { + AicpuMetadataInfo(kernel_node, kernel_info_list); + } + + if (kernel_info_list->empty()) { + GetRtKelInfo(kernel_node, kernel_info_list); + } + + if (kernel_info_list->empty()) { + HcclMetadataInfo(kernel_node, kernel_info_list); + } + if (kernel_info_list->empty()) { + MS_LOG(EXCEPTION) << "op" << kernel_node->DebugString() << "kernel query fail!"; + } + FilterInvaildKernelInfo(kernel_node, kernel_info_list); +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/kernel_query.h b/mindspore/ccsrc/kernel/kernel_query.h new file mode 100644 index 00000000000..72acbc09525 --- /dev/null +++ b/mindspore/ccsrc/kernel/kernel_query.h @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_KERNEL_QUERY_H_ +#define MINDSPORE_CCSRC_KERNEL_KERNEL_QUERY_H_ + +#include +#include +#include +#include "kernel/kernel.h" +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace kernel { +void KernelQuery(const CNodePtr &kernel_node, std::vector> *kernel_info_list); +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_KERNEL_QUERY_H_ diff --git a/mindspore/ccsrc/kernel/mng/assign.cc b/mindspore/ccsrc/kernel/mng/assign.cc new file mode 100644 index 00000000000..15ffd6ee948 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/assign.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/assign.h" + +#include + +#include "runtime/mem.h" +#include "common/utils.h" + +using ge::model_runner::MemcpyAsyncTaskInfo; +using MemcpyAsyncTaskInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace kernel { +AssignKernel::AssignKernel() {} + +AssignKernel::~AssignKernel() {} + +bool AssignKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + auto stream = reinterpret_cast(stream_ptr); + + if (inputs.size() != 2) { + MS_LOG(ERROR) << "inputs size is not two"; + return false; + } + + if (inputs[0]->addr == inputs[1]->addr) { + MS_LOG(INFO) << "first addr is same with second addr , no need assign"; + return true; + } + rtError_t status = rtMemcpyAsync(inputs[0]->addr, inputs[0]->size, inputs[1]->addr, inputs[1]->size, + RT_MEMCPY_DEVICE_TO_DEVICE, stream); + if (status != RT_ERROR_NONE) { + MS_LOG(ERROR) << "Assign op rtMemcpyAsync failed!"; + return false; + } + return true; +} + +std::vector AssignKernel::GenTask(const vector &inputs, + const vector &workspace, + const vector &outputs, + uint32_t stream_id) { + if (inputs.size() != 2) { + MS_LOG(EXCEPTION) << "inputs size is not two"; + } + + std::shared_ptr task_info_ptr = std::make_shared( + stream_id, inputs[0]->addr, inputs[0]->size, inputs[1]->addr, inputs[1]->size, RT_MEMCPY_DEVICE_TO_DEVICE); + MS_EXCEPTION_IF_NULL(task_info_ptr); + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/assign.h b/mindspore/ccsrc/kernel/mng/assign.h new file mode 100644 index 00000000000..a3446deecf6 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/assign.h @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_ASSIGN_H +#define MINDSPORE_CCSRC_KERNEL_MNG_ASSIGN_H + +#include +#include "kernel/mng/rt_kernel.h" +#include "kernel/mng/rt_kernel_info.h" + +namespace mindspore { +namespace kernel { +class AssignKernel : public RtKernel { + public: + AssignKernel(); + ~AssignKernel() override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + std::vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; +}; + +MS_REG_RTKERNEL(assign, AssignKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_ASSIGN_H diff --git a/mindspore/ccsrc/kernel/mng/memcpy_async.cc b/mindspore/ccsrc/kernel/mng/memcpy_async.cc new file mode 100644 index 00000000000..9ccbc424e73 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/memcpy_async.cc @@ -0,0 +1,148 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/memcpy_async.h" + +#include +#include + +#include "runtime/mem.h" +#include "common/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "common/trans.h" + +using ge::model_runner::MemcpyAsyncTaskInfo; +using MemcpyAsyncTaskInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace kernel { +MemCpyAsyncKernel::MemCpyAsyncKernel() {} + +MemCpyAsyncKernel::~MemCpyAsyncKernel() {} + +bool MemCpyAsyncKernel::Launch(const std::vector &inputs, const std::vector & /*workspace*/, + const std::vector &outputs, uintptr_t stream_ptr) { + auto stream = reinterpret_cast(stream_ptr); + + if (inputs.size() != 1) { + MS_LOG(ERROR) << "inputs size is not one"; + return false; + } + if (outputs.size() != 1) { + MS_LOG(ERROR) << "outputs size is not one"; + return false; + } + + if (inputs[0]->addr == outputs[0]->addr) { + MS_LOG(INFO) << "input addr is same with output addr , no need exe memcpy async"; + return true; + } + rtError_t status = rtMemcpyAsync(outputs[0]->addr, outputs[0]->size, inputs[0]->addr, inputs[0]->size, + RT_MEMCPY_DEVICE_TO_DEVICE, stream); + if (status != RT_ERROR_NONE) { + MS_LOG(ERROR) << "MemCpyAsync op rtMemcpyAsync failed!"; + return false; + } + return true; +} + +bool MemCpyAsyncKernel::Init(const mindspore::AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + GetInputOutputDataType(anf_node); + GetInputOutputTotalCount(anf_node); + return true; +} + +void MemCpyAsyncKernel::GetInputOutputDataType(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + size_t input_size = AnfAlgo::GetInputTensorNum(anf_node); + if (input_size != 1) { + MS_LOG(EXCEPTION) << "MemCpyAsync input size is not 1"; + } + input_type_id_ = AnfAlgo::GetPrevNodeOutputInferDataType(anf_node, 0); +} + +void MemCpyAsyncKernel::GetInputOutputTotalCount(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + size_t input_size = AnfAlgo::GetInputTensorNum(anf_node); + if (input_size != 1) { + MS_LOG(EXCEPTION) << "MemCpyAsync input size is not 1"; + } + size_t type_size = trans::TypeIdSize(input_type_id_); + std::vector shape_i = AnfAlgo::GetInputDeviceShape(anf_node, 0); + size_t total_size = 1; + for (size_t i = 0; i < shape_i.size(); i++) { + total_size = total_size * shape_i[i]; + } + total_size *= type_size; + MS_LOG(INFO) << "MemCpyAsync size[" << total_size << "]"; + input_size_list_.emplace_back(total_size); + output_size_list_.emplace_back(total_size); +} + +std::vector MemCpyAsyncKernel::GenTask(const vector &inputs, + const vector & /*workspace*/, + const vector &outputs, + uint32_t stream_id) { + if (inputs.size() != 1) { + MS_LOG(EXCEPTION) << "MemCpyAsync op inputs is not one"; + } + + if (outputs.size() != 1) { + MS_LOG(EXCEPTION) << "MemCpyAsync op output is not one"; + } + + std::shared_ptr task_info_ptr = std::make_shared( + stream_id, outputs[0]->addr, outputs[0]->size, inputs[0]->addr, inputs[0]->size, RT_MEMCPY_DEVICE_TO_DEVICE); + MS_EXCEPTION_IF_NULL(task_info_ptr); + return {task_info_ptr}; +} + +const std::vector data_type_list{kNumberTypeInt, kNumberTypeInt8, kNumberTypeInt16, kNumberTypeInt32, + kNumberTypeInt64, kNumberTypeUInt, kNumberTypeUInt8, kNumberTypeUInt16, + kNumberTypeUInt32, kNumberTypeUInt64, kNumberTypeFloat, kNumberTypeFloat16, + kNumberTypeFloat32, kNumberTypeFloat64}; +const std::vector format_list = {kOpFormat_DEFAULT, kOpFormat_NCHW, kOpFormat_NHWC, + kOpFormat_NC1HWC0, kOpFormat_FRAC_Z, kOpFormat_NC1KHKWHWC0, + kOpFormat_C1HWNCoC0}; + +MemCpyAsyncDesc::MemCpyAsyncDesc() {} + +MemCpyAsyncDesc::~MemCpyAsyncDesc() {} + +std::vector> MemCpyAsyncDesc::GetKernelInfo() { + std::vector> memcpy_build_info{}; + for (const auto &format : format_list) { + for (const auto &type : data_type_list) { + auto builder = KernelBuildInfo::KernelBuildInfoBuilder(); + vector input_format{format}; + vector input_type{type}; + vector output_format{format}; + vector output_type{type}; + builder.SetInputsFormat(input_format); + builder.SetInputsDeviceType(input_type); + builder.SetOutputsFormat(output_format); + builder.SetOutputsDeviceType(output_type); + builder.SetProcessor(AICORE); + builder.SetKernelType(RT_KERNEL); + builder.SetFusionType(OPAQUE); + memcpy_build_info.emplace_back(builder.Build()); + } + } + return memcpy_build_info; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/memcpy_async.h b/mindspore/ccsrc/kernel/mng/memcpy_async.h new file mode 100755 index 00000000000..b4e15aea61d --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/memcpy_async.h @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_MEMCPY_ASYNC_H +#define MINDSPORE_CCSRC_KERNEL_MNG_MEMCPY_ASYNC_H + +#include +#include +#include "kernel/mng/rt_kernel.h" +#include "kernel/mng/rt_kernel_info.h" + +namespace mindspore { +namespace kernel { +class MemCpyAsyncKernel : public RtKernel { + public: + MemCpyAsyncKernel(); + ~MemCpyAsyncKernel() override; + + bool Init(const AnfNodePtr &anf_node) override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + std::vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + private: + void GetInputOutputDataType(const AnfNodePtr &anf_node); + void GetInputOutputTotalCount(const AnfNodePtr &anf_node); + TypeId input_type_id_{}; +}; + +class MemCpyAsyncDesc : public RtKerDesc { + public: + MemCpyAsyncDesc(); + ~MemCpyAsyncDesc() override; + std::vector> GetKernelInfo() override; +}; + +MS_REG_RTKERNEL_DESC(memcpy_async, MemCpyAsyncDesc); +MS_REG_RTKERNEL(memcpy_async, MemCpyAsyncKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_MEMCPY_ASYNC_H diff --git a/mindspore/ccsrc/kernel/mng/profiling_kernel_mod.cc b/mindspore/ccsrc/kernel/mng/profiling_kernel_mod.cc new file mode 100644 index 00000000000..38b42989d2a --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/profiling_kernel_mod.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/profiling_kernel_mod.h" + +#include +#include +#include + +#include "framework/ge_runtime/task_info.h" +#include "device/ascend/profiling/profiling_utils.h" +#include "session/anf_runtime_algorithm.h" + +using ProfilerTraceTaskInfo = ge::model_runner::ProfilerTraceTaskInfo; +using mindspore::device::ascend::ProfilingUtils; + +namespace mindspore { +namespace kernel { +bool ProfilingKernelMod::Init(const AnfNodePtr &anf_node) { + MS_LOG(INFO) << "[profiling] init profiling kernel mod"; + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + + ValuePtr notify_ptr = primitive->GetAttr(ProfilingUtils::kNotify); + MS_EXCEPTION_IF_NULL(notify_ptr); + + ValuePtr log_id_ptr = primitive->GetAttr(ProfilingUtils::kProfilerTraceId); + MS_EXCEPTION_IF_NULL(log_id_ptr); + + ValuePtr flags_ptr = primitive->GetAttr(ProfilingUtils::kFlags); + MS_EXCEPTION_IF_NULL(flags_ptr); + + notify_ = GetValue(notify_ptr); + log_id_ = GetValue(log_id_ptr); + flags_ = GetValue(flags_ptr); + MS_LOG(INFO) << "[profiling] profiling kernel notify_:" << notify_ << ", log_id_:" << log_id_ + << ", flags_:" << flags_; + return true; +} + +bool ProfilingKernelMod::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + MS_LOG(INFO) << "gen task inputs size:" << inputs.size() << ", workspace size:" << workspace.size() + << ", outputs size:" << outputs.size() << ", stream_ptr:" << stream_ptr; + return true; +} + +std::vector ProfilingKernelMod::GenTask(const std::vector &inputs, + const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) { + MS_LOG(INFO) << "gen task inputs size:" << inputs.size() << ", workspace size:" << workspace.size() + << ", outputs size:" << outputs.size(); + std::shared_ptr task_info_ptr = + std::make_shared(stream_id, log_id_, notify_, flags_); + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/profiling_kernel_mod.h b/mindspore/ccsrc/kernel/mng/profiling_kernel_mod.h new file mode 100644 index 00000000000..7dc0ce05c62 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/profiling_kernel_mod.h @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_KERNEL_PROFILING_PROFILING_KERNEL_MOD_H_ +#define MINDSPORE_MINDSPORE_CCSRC_KERNEL_PROFILING_PROFILING_KERNEL_MOD_H_ +#include +#include "kernel/mng/rt_kernel.h" +namespace mindspore { +namespace kernel { +class ProfilingKernelMod : public RtKernel { + public: + ProfilingKernelMod() = default; + ~ProfilingKernelMod() override = default; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + std::vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + bool Init(const AnfNodePtr &anf_node) override; + + private: + uint64_t log_id_{0}; + bool notify_{true}; + uint32_t flags_{0}; +}; +MS_REG_RTKERNEL(profiling, ProfilingKernelMod); +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_KERNEL_PROFILING_PROFILING_KERNEL_MOD_H_ diff --git a/mindspore/ccsrc/kernel/mng/recv.cc b/mindspore/ccsrc/kernel/mng/recv.cc new file mode 100644 index 00000000000..bbf0308924a --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/recv.cc @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/recv.h" +#include +#include "runtime/stream.h" +#include "utils/context/ms_context.h" +#include "device/ascend/ascend_stream_assign.h" +#include "framework/ge_runtime/task_info.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +using ge::model_runner::EventWaitTaskInfo; +using mindspore::device::ascend::AscendStreamAssign; +using EventWaitTaskInfoPtr = std::shared_ptr; + +RecvKernel::RecvKernel() { event_id_ = 0; } + +RecvKernel::~RecvKernel() {} + +bool RecvKernel::Init(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + event_id_ = GetValue(primitive->GetAttr(kAttrEventId)); + MS_LOG(INFO) << "recv op event_id_:" << event_id_; + return true; +} + +bool RecvKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + rtEvent_t stream_event{}; + auto stream = reinterpret_cast(stream_ptr); + auto status = rtStreamWaitEvent(stream, stream_event); + if (status != RT_ERROR_NONE) { + MS_LOG(ERROR) << "Recv rtStreamWaitEvent failed!"; + return false; + } + return true; +} + +std::vector RecvKernel::GenTask(const std::vector &, const std::vector &, + const std::vector &, uint32_t stream_id) { + MS_LOG(INFO) << "RecvKernel GenTask event_id_:" << event_id_ << ", stream_id_:" << stream_id; + EventWaitTaskInfoPtr task_info_ptr = std::make_shared(stream_id, event_id_); + MS_EXCEPTION_IF_NULL(task_info_ptr); + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/recv.h b/mindspore/ccsrc/kernel/mng/recv.h new file mode 100644 index 00000000000..e1deaf49cb6 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/recv.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_RECV_H +#define MINDSPORE_CCSRC_KERNEL_MNG_RECV_H + +#include +#include +#include "kernel/mng/rt_kernel.h" +#include "kernel/mng/rt_kernel_info.h" + +namespace mindspore { +namespace kernel { +class RecvKernel : public RtKernel { + public: + RecvKernel(); + ~RecvKernel() override; + + bool Init(const AnfNodePtr &anf_node) override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + std::vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + private: + uint32_t event_id_; +}; + +MS_REG_RTKERNEL(recv, RecvKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_RECV_H diff --git a/mindspore/ccsrc/kernel/mng/rt_kernel.cc b/mindspore/ccsrc/kernel/mng/rt_kernel.cc new file mode 100644 index 00000000000..447f21dc079 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/rt_kernel.cc @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/rt_kernel.h" + +namespace mindspore { +namespace kernel { +void RtKernelFactory::Registe(const std::string &name, RtKernelCreater &&fun) { + (void)fmap_.emplace(name, std::move(fun)); +} + +std::shared_ptr RtKernelFactory::Create(const std::string &name) { + const auto &map = Get().fmap_; + auto it = map.find(name); + if (it != map.end() && it->second) { + return (it->second)(); + } + return nullptr; +} + +RtKernelFactory &RtKernelFactory::Get() { + static RtKernelFactory _this; + return _this; +} + +RtKernel::RtKernel() {} + +RtKernel::~RtKernel() {} + +bool RtKernel::Init(const mindspore::AnfNodePtr & /*anf_node*/) { return true; } + +const std::vector &RtKernel::GetInputSizeList() const { return input_size_list_; } + +const std::vector &RtKernel::GetOutputSizeList() const { return output_size_list_; } + +const std::vector &RtKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/rt_kernel.h b/mindspore/ccsrc/kernel/mng/rt_kernel.h new file mode 100644 index 00000000000..f86a86ef5d3 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/rt_kernel.h @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_H +#define MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_H + +#include +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "kernel/task_stream.h" + +namespace mindspore { +namespace kernel { +class RtKernel : public KernelMod { + public: + RtKernel(); + ~RtKernel() override; + virtual bool Init(const AnfNodePtr &anf_node); + const std::vector &GetInputSizeList() const override; + const std::vector &GetOutputSizeList() const override; + const std::vector &GetWorkspaceSizeList() const override; + + protected: + mutable std::vector input_size_list_; + mutable std::vector output_size_list_; + mutable std::vector workspace_size_list_; +}; + +using RTKernelPtr = std::shared_ptr; + +using RtKernelCreater = std::function()>; +class RtKernelFactory { + RtKernelFactory() = default; + ~RtKernelFactory() = default; + + public: + static RtKernelFactory &Get(); + void Registe(const std::string &name, RtKernelCreater &&fun); + static std::shared_ptr Create(const std::string &name); + + private: + std::map fmap_; +}; + +class _RtKernelRegister { + public: + _RtKernelRegister(const std::string &name, RtKernelCreater &&fun) { + RtKernelFactory::Get().Registe(name, std::move(fun)); + } + ~_RtKernelRegister() = default; +}; + +#define _MS_REG_RTKERNEL_REG(KNAME, clazz) \ + static_assert(std::is_base_of::value, " must be base of RtKernel"); \ + static const _RtKernelRegister g_##KNAME##_##_RtKernel_reg(#KNAME, []() { return std::make_shared(); }); + +#define MS_REG_RTKERNEL(KNAME, clazz) _MS_REG_RTKERNEL_REG(KNAME, clazz) +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_H diff --git a/mindspore/ccsrc/kernel/mng/rt_kernel_build.cc b/mindspore/ccsrc/kernel/mng/rt_kernel_build.cc new file mode 100644 index 00000000000..b88200dfdd7 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/rt_kernel_build.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/rt_kernel_build.h" + +#include +#include +#include +#include + +#include "kernel/mng/rt_kernel.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +KernelModPtr RtOpBuild(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + (void)std::transform(op_name.begin(), op_name.end(), op_name.begin(), ::tolower); + MS_LOG(INFO) << "Op Name(tolower)[" << op_name << "]"; + auto ker_ptr = RtKernelFactory::Create(op_name); + MS_EXCEPTION_IF_NULL(ker_ptr); + if (!ker_ptr->Init(anf_node)) { + MS_LOG(ERROR) << "Rt Op initialize failed!"; + return nullptr; + } + + return ker_ptr; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/rt_kernel_build.h b/mindspore/ccsrc/kernel/mng/rt_kernel_build.h new file mode 100644 index 00000000000..2e6ccdf94c0 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/rt_kernel_build.h @@ -0,0 +1,29 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_BUILD_H +#define MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_BUILD_H + +#include +#include +#include "kernel/kernel.h" +namespace mindspore { +namespace kernel { +KernelModPtr RtOpBuild(const AnfNodePtr &anf_node); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_BUILD_H diff --git a/mindspore/ccsrc/kernel/mng/rt_kernel_info.cc b/mindspore/ccsrc/kernel/mng/rt_kernel_info.cc new file mode 100755 index 00000000000..9bd22712d97 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/rt_kernel_info.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/rt_kernel_info.h" +#include +#include +#include "utils/convert_utils.h" +#include "utils/utils.h" +#include "common/utils.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace kernel { +void RtKerDescFactory::Register(const std::string &name, RtKerDescCreater &&fun) { + if (fmap_.find(name) == fmap_.end()) { + (void)fmap_.emplace(name, std::move(fun)); + } +} + +std::shared_ptr RtKerDescFactory::Create(const std::string &name) { + const auto &map = Get().fmap_; + auto it = map.find(name); + if (it != map.end() && it->second) { + return (it->second)(); + } + return nullptr; +} + +RtKerDescFactory &RtKerDescFactory::Get() { + static RtKerDescFactory _this; + return _this; +} + +void GetRtKelInfo(const CNodePtr &kernel_node, + std::vector> *kernel_info_list) { + MS_LOG(INFO) << "Mng kernel Info."; + MS_EXCEPTION_IF_NULL(kernel_info_list); + MS_EXCEPTION_IF_NULL(kernel_node); + std::string opNameLower = AnfAlgo::GetCNodeName(kernel_node); + (void)std::transform(opNameLower.begin(), opNameLower.end(), opNameLower.begin(), ::tolower); + + auto ker_desc_ptr = RtKerDescFactory::Create(opNameLower); + if (ker_desc_ptr == nullptr) { + MS_LOG(DEBUG) << "Mng can't find op [" << opNameLower << "]."; + return; + } + MS_EXCEPTION_IF_NULL(ker_desc_ptr); + auto kernel_info = ker_desc_ptr->GetKernelInfo(); + if (kernel_info.empty()) { + MS_LOG(WARNING) << "Rt dose not has op[" << opNameLower << "]."; + return; + } + *kernel_info_list = kernel_info; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/rt_kernel_info.h b/mindspore/ccsrc/kernel/mng/rt_kernel_info.h new file mode 100644 index 00000000000..de4f97f4c21 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/rt_kernel_info.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_INFO_H +#define MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_INFO_H + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/dtype.h" +#include "kernel/kernel_build_info.h" +#include "kernel/kernel.h" +#include "utils/utils.h" + +namespace mindspore { +namespace kernel { +class RtKerDesc { + public: + virtual ~RtKerDesc() {} + virtual std::vector> GetKernelInfo() { + return std::vector>{}; + } +}; + +using RtKerDescCreater = std::function()>; +class RtKerDescFactory { + RtKerDescFactory() = default; + ~RtKerDescFactory() = default; + + public: + static RtKerDescFactory &Get(); + void Register(const std::string &name, RtKerDescCreater &&fun); + static std::shared_ptr Create(const std::string &name); + + private: + std::map fmap_; +}; + +class _RtKerDescRegister { + public: + _RtKerDescRegister(const std::string &name, RtKerDescCreater &&fun) { + RtKerDescFactory::Get().Register(name, std::move(fun)); + } + ~_RtKerDescRegister() = default; +}; + +#define _MS_REG_RTKERNEL_DESC_REG(KNAME, clazz) \ + static_assert(std::is_base_of::value, " must be base of RtKerDesc"); \ + static const _RtKerDescRegister g_##KNAME##_##_rtkernel_desc_reg(#KNAME, []() { return std::make_shared(); }); + +#define MS_REG_RTKERNEL_DESC(KNAME, clazz) _MS_REG_RTKERNEL_DESC_REG(KNAME, clazz) + +void GetRtKelInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_RT_KERNEL_INFO_H diff --git a/mindspore/ccsrc/kernel/mng/send.cc b/mindspore/ccsrc/kernel/mng/send.cc new file mode 100644 index 00000000000..c63440ccc45 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/send.cc @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/send.h" +#include +#include "runtime/event.h" +#include "framework/ge_runtime/task_info.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" + +using ge::model_runner::EventRecordTaskInfo; +using EventRecordTaskInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace kernel { +SendKernel::SendKernel() { event_id_ = 0; } + +SendKernel::~SendKernel() {} + +bool SendKernel::Init(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + event_id_ = GetValue(primitive->GetAttr(kAttrEventId)); + MS_LOG(INFO) << "send op event id:" << event_id_; + return true; +} + +bool SendKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + rtEvent_t event{}; + auto stream = reinterpret_cast(stream_ptr); + rtError_t status = rtEventRecord(event, stream); + if (status != RT_ERROR_NONE) { + MS_LOG(ERROR) << "Send op rtEventRecord failed!"; + return false; + } + return true; +} + +std::vector SendKernel::GenTask(const std::vector &, const std::vector &, + const std::vector &, uint32_t stream_id) { + MS_LOG(INFO) << "SendKernel GenTask event id:" << event_id_ << ", stream id:" << stream_id; + EventRecordTaskInfoPtr task_info_ptr = std::make_shared(stream_id, event_id_); + MS_EXCEPTION_IF_NULL(task_info_ptr); + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/send.h b/mindspore/ccsrc/kernel/mng/send.h new file mode 100644 index 00000000000..43e27049aad --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/send.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_SEND_H +#define MINDSPORE_CCSRC_KERNEL_MNG_SEND_H +#include +#include +#include "kernel/mng/rt_kernel.h" +#include "kernel/mng/rt_kernel_info.h" + +namespace mindspore { +namespace kernel { +class SendKernel : public RtKernel { + public: + SendKernel(); + ~SendKernel() override; + bool Init(const AnfNodePtr &anf_node) override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + private: + uint32_t event_id_; +}; + +MS_REG_RTKERNEL(send, SendKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_SEND_H diff --git a/mindspore/ccsrc/kernel/mng/stream_active.cc b/mindspore/ccsrc/kernel/mng/stream_active.cc new file mode 100644 index 00000000000..b0caa689ac9 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/stream_active.cc @@ -0,0 +1,80 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/stream_active.h" +#include +#include +#include "runtime/stream.h" +#include "framework/ge_runtime/task_info.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" + +using ge::model_runner::StreamActiveTaskInfo; +using StreamActiveTaskInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace kernel { +StreamActiveKernel::StreamActiveKernel() { active_streams_index_ = {}; } + +StreamActiveKernel::~StreamActiveKernel() {} + +bool StreamActiveKernel::Init(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_LOG(INFO) << "stream active op init start"; + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + active_streams_index_ = GetValue>(primitive->GetAttr(kAttrActiveStreamList)); + return true; +} + +bool StreamActiveKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + MS_LOG(INFO) << "Stream active op launch start"; + auto stream = reinterpret_cast(stream_ptr); + + if (active_streams_index_.empty()) { + MS_LOG(ERROR) << "activeStreamList_ is empty!"; + return false; + } + + rtStream_t act_stream; + rtError_t status; + for (auto index : active_streams_index_) { + act_stream = kernel::TaskStream::GetInstance()->gen_stream_list()[index]; + status = rtStreamActive(act_stream, stream); + if (status != RT_ERROR_NONE) { + MS_LOG(ERROR) << "Stream active failed!"; + return false; + } + } + return true; +} + +std::vector StreamActiveKernel::GenTask(const std::vector &, const std::vector &, + const std::vector &, uint32_t stream_id) { + MS_LOG(INFO) << "StreamActiveKernel GenTask active stream size:" << active_streams_index_.size() + << ", stream id:" << stream_id; + std::vector task_info_list; + for (auto &index : active_streams_index_) { + std::shared_ptr task_info_ptr = std::make_shared(stream_id, index); + MS_EXCEPTION_IF_NULL(task_info_ptr); + task_info_list.emplace_back(task_info_ptr); + MS_LOG(INFO) << "StreamActiveKernel GenTask: streamId:" << stream_id << ", Active streamId:" << index; + } + return task_info_list; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/stream_active.h b/mindspore/ccsrc/kernel/mng/stream_active.h new file mode 100644 index 00000000000..51aeff9bbfa --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/stream_active.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_STREAM_ACTIVE_H +#define MINDSPORE_CCSRC_KERNEL_MNG_STREAM_ACTIVE_H +#include +#include +#include "kernel/mng/rt_kernel.h" +#include "kernel/mng/rt_kernel_info.h" + +namespace mindspore { +namespace kernel { +class StreamActiveKernel : public RtKernel { + public: + StreamActiveKernel(); + ~StreamActiveKernel() override; + + bool Init(const AnfNodePtr &anf_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + private: + std::vector active_streams_index_; +}; + +MS_REG_RTKERNEL(streamactive, StreamActiveKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_STREAM_ACTIVE_H diff --git a/mindspore/ccsrc/kernel/mng/stream_switch.cc b/mindspore/ccsrc/kernel/mng/stream_switch.cc new file mode 100644 index 00000000000..b4655020faa --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/stream_switch.cc @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/mng/stream_switch.h" + +#include +#include + +#include "runtime/stream.h" +#include "framework/ge_runtime/task_info.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" + +using ge::model_runner::StreamSwitchTaskInfo; +using StreamSwitchTaskInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace kernel { +StreamSwitchKernel::StreamSwitchKernel() { + cond_ = RT_EQUAL; + true_stream_index_ = 0; + data_type_ = RT_SWITCH_INT32; +} + +StreamSwitchKernel::~StreamSwitchKernel() {} + +bool StreamSwitchKernel::Init(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_LOG(INFO) << "stream switch op init start"; + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + cond_ = tagRtCondition(GetValue(primitive->GetAttr(kAttrSwitchCondition))); + true_stream_index_ = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); + data_type_ = tagRtSwitchDataType(GetValue(primitive->GetAttr(kAttrDataType))); + MS_LOG(INFO) << "cond_:" << static_cast(cond_) << ", true_stream_index_:" << true_stream_index_ + << ", data_type_:" << static_cast(data_type_); + return true; +} + +bool StreamSwitchKernel::Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + MS_LOG(INFO) << "stream switch op launch start"; + if (inputs.size() != 2) { + MS_LOG(ERROR) << "Stream switch inputs size is " << inputs.size() << ", only support 2"; + } + + void *loop_cnt = inputs[0]->addr; + void *ites_per_loop = inputs[1]->addr; + auto stream = reinterpret_cast(stream_ptr); + rtStream_t true_stream_ = kernel::TaskStream::GetInstance()->gen_stream_list()[true_stream_index_]; + rtError_t status = rtStreamSwitchEx(loop_cnt, cond_, ites_per_loop, true_stream_, stream, data_type_); + if (status != RT_ERROR_NONE) { + MS_LOG(ERROR) << "Stream switch failed!"; + return false; + } + return true; +} + +std::vector StreamSwitchKernel::GenTask(const std::vector &inputs, + const std::vector &, const std::vector &, + uint32_t stream_id) { + MS_LOG(INFO) << "StreamSwitchKernel GenTask start"; + if (inputs.size() != 2) { + MS_LOG(ERROR) << "stream switch inputs size is " << inputs.size() << ", is not two"; + } + MS_EXCEPTION_IF_NULL(inputs[0]); + MS_EXCEPTION_IF_NULL(inputs[1]); + auto loop_cnt = inputs[0]->addr; + auto ites_per_loop = inputs[1]->addr; + MS_LOG(INFO) << "cond_:" << static_cast(cond_) << ", true_stream_index_:" << true_stream_index_ + << ", stream_id:" << stream_id; + std::shared_ptr task_info_ptr = + std::make_shared(stream_id, true_stream_index_, loop_cnt, ites_per_loop, cond_, data_type_); + MS_EXCEPTION_IF_NULL(task_info_ptr); + return {task_info_ptr}; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/mng/stream_switch.h b/mindspore/ccsrc/kernel/mng/stream_switch.h new file mode 100644 index 00000000000..55cf323ec72 --- /dev/null +++ b/mindspore/ccsrc/kernel/mng/stream_switch.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_MNG_STREAM_SWITCH_H +#define MINDSPORE_CCSRC_KERNEL_MNG_STREAM_SWITCH_H + +#include +#include +#include "kernel/mng/rt_kernel.h" +#include "kernel/mng/rt_kernel_info.h" + +namespace mindspore { +namespace kernel { +class StreamSwitchKernel : public RtKernel { + public: + StreamSwitchKernel(); + ~StreamSwitchKernel() override; + + bool Init(const AnfNodePtr &anf_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + std::vector GenTask(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uint32_t stream_id) override; + + private: + rtCondition_t cond_; + uint32_t true_stream_index_; + rtSwitchDataType_t data_type_; +}; + +MS_REG_RTKERNEL(streamswitch, StreamSwitchKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_MNG_STREAM_SWITCH_H diff --git a/mindspore/ccsrc/kernel/oplib/opinfo.h b/mindspore/ccsrc/kernel/oplib/opinfo.h new file mode 100644 index 00000000000..7861da34d9d --- /dev/null +++ b/mindspore/ccsrc/kernel/oplib/opinfo.h @@ -0,0 +1,139 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_OPLIB_OPINFO_H_ +#define MINDSPORE_CCSRC_KERNEL_OPLIB_OPINFO_H_ +#include +#include +#include +#include +#include "ir/dtype.h" + +namespace mindspore { +namespace kernel { +enum OpImplyType { kAKG = 0, kTBE }; +enum OpIOType { kInput = 0, kOutput }; + +class OpAttr { + public: + OpAttr() = default; + ~OpAttr() = default; + + std::string name() const { return name_; } + std::string param_type() const { return param_type_; } + std::string type() const { return type_; } + std::string value() const { return value_; } + std::string default_value() const { return default_value_; } + + void set_name(const std::string& name) { name_ = name; } + void set_param_type(const std::string& param_type) { param_type_ = param_type; } + void set_type(const std::string& type) { type_ = type; } + void set_value(const std::string& value) { value_ = value; } + void set_default_value(const std::string& default_value) { default_value_ = default_value; } + + private: + std::string name_; + std::string param_type_; + std::string type_; + std::string value_; + std::string default_value_; +}; + +class OpIOInfo { + public: + OpIOInfo() = default; + ~OpIOInfo() = default; + + int index() const { return index_; } + std::string name() const { return name_; } + bool need_compile() const { return need_compile_; } + std::string param_type() const { return param_type_; } + std::string shape() const { return shape_; } + std::vector dtypes() const { return dtypes_; } + std::vector formats() const { return formats_; } + + void set_index(const int index) { index_ = index; } + void set_name(const std::string& name) { name_ = name; } + void set_need_compile(const bool need_compile) { need_compile_ = need_compile; } + void set_param_type(const std::string& param_type) { param_type_ = param_type; } + void set_shape(const std::string& shape) { shape_ = shape; } + void set_dtypes(const std::vector& dtype) { dtypes_ = dtype; } + void set_formats(const std::vector& formats) { formats_ = formats; } + + private: + int index_ = 0; + std::string name_; + bool need_compile_ = false; + std::string param_type_; + std::string shape_; + std::vector dtypes_; + std::vector formats_; +}; + +class OpInfo { + public: + OpInfo() = default; + ~OpInfo() = default; + std::string op_name() const { return op_name_; } + OpImplyType imply_type() const { return imply_type_; } + std::string impl_path() const { return impl_path_; } + std::string fusion_type() const { return fusion_type_; } + bool async_flag() const { return async_flag_; } + std::string binfile_name() const { return binfile_name_; } + int compute_cost() const { return compute_cost_; } + std::string kernel_name() const { return kernel_name_; } + bool partial_flag() const { return partial_flag_; } + std::vector> attrs_ptr() const { return attrs_ptr_; } + std::vector> inputs_ptr() const { return inputs_ptr_; } + std::vector> outputs_ptr() const { return outputs_ptr_; } + const std::unordered_map& ref_infos() const { return ref_infos_; } + + void set_op_name(const std::string& op_name) { op_name_ = op_name; } + void set_imply_type(const OpImplyType imply_type) { imply_type_ = imply_type; } + void set_impl_path(const std::string& impl_path) { impl_path_ = impl_path; } + void set_fusion_type(const std::string& fusion_type) { fusion_type_ = fusion_type; } + void set_async_flag(const bool async_flag) { async_flag_ = async_flag; } + void set_binfile_name(const std::string& binfile_name) { binfile_name_ = binfile_name; } + void set_compute_cost(const int compute_cost) { compute_cost_ = compute_cost; } + void set_kernel_name(const std::string& kernel_name) { kernel_name_ = kernel_name; } + void set_partial_flag(const bool partial_flag) { partial_flag_ = partial_flag; } + void add_attrs_ptr(const std::shared_ptr& attr) { attrs_ptr_.push_back(attr); } + void add_inputs_ptr(const std::shared_ptr& input) { inputs_ptr_.push_back(input); } + void add_outputs_ptr(const std::shared_ptr& output) { outputs_ptr_.push_back(output); } + void set_inputs_ptr(const std::vector>& inputs) { inputs_ptr_ = inputs; } + void set_outputs_ptr(const std::vector>& outputs) { outputs_ptr_ = outputs; } + bool is_ref() const { return !ref_infos_.empty(); } + bool has_ref_index(size_t out_index) const { return ref_infos_.find(out_index) != ref_infos_.end(); } + void add_ref_pair(size_t out_index, size_t in_index) { (void)ref_infos_.emplace(out_index, in_index); } + + private: + std::string op_name_; + OpImplyType imply_type_ = kTBE; + std::string impl_path_; + std::string fusion_type_; + bool async_flag_ = false; + std::string binfile_name_; + int compute_cost_ = 0; + std::string kernel_name_; + bool partial_flag_ = false; + std::vector> attrs_ptr_; + std::vector> inputs_ptr_; + std::vector> outputs_ptr_; + std::unordered_map ref_infos_; +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_OPLIB_OPINFO_H_ diff --git a/mindspore/ccsrc/kernel/oplib/oplib.cc b/mindspore/ccsrc/kernel/oplib/oplib.cc new file mode 100644 index 00000000000..23e70141040 --- /dev/null +++ b/mindspore/ccsrc/kernel/oplib/oplib.cc @@ -0,0 +1,254 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/oplib/oplib.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "kernel/oplib/opinfo.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace kernel { +constexpr auto kImplyType = "imply_type"; +constexpr auto kOpName = "op_name"; +constexpr auto kTbe = "TBE"; +constexpr auto kAkg = "akg"; +constexpr auto kAutodiff = "AutoDiff"; +constexpr auto kFusionType = "fusion_type"; +constexpr auto kAsyncFlag = "async_flag"; +constexpr auto kBinfileName = "binfile_name"; +constexpr auto kComputeCost = "compute_cost"; +constexpr auto kKernelName = "kernel_name"; +constexpr auto kPartialFlag = "partial_flag"; +constexpr auto kAttr = "attr"; +constexpr auto kIputs = "inputs"; +constexpr auto kOutputs = "outputs"; +constexpr auto kName = "name"; +constexpr auto kParamType = "param_type"; +constexpr auto kDtype = "dtype"; +constexpr auto kType = "type"; +constexpr auto kValue = "value"; +constexpr auto kDefaultValue = "default_value"; +constexpr auto kIndex = "index"; +constexpr auto kFormat = "format"; +constexpr auto kNeedCompile = "need_compile"; +constexpr auto kShape = "shape"; +std::vector> OpLib::op_info_; + +string ImplTypeToStr(OpImplyType impl_type) { + switch (impl_type) { + case kTBE: + return kTbe; + case kAKG: + return kAkg; + default: + return "unknow"; + } +} +bool OpLib::RegOp(const std::string& json_string, const std::string& impl_path) { + bool ret = false; + try { + auto op_json = nlohmann::json::parse(json_string); + std::string imply_type_string = op_json.at(kImplyType); + std::string op_name = op_json.at(kOpName); + if (imply_type_string == kTbe) { + OpImplyType imply_type = kTBE; + ret = DecodeOpInfo(op_json, imply_type, impl_path); + } else if (imply_type_string == kAutodiff) { + OpImplyType imply_type = kAKG; + ret = DecodeOpInfo(op_json, imply_type, impl_path); + } else { + MS_LOG(DEBUG) << "Not support imply_type"; + } + if (!ret) { + MS_LOG(DEBUG) << "RegOp failed: opname:" << op_name << "imply_type" << imply_type_string; + } + } catch (const std::exception& e) { + MS_LOG(DEBUG) << "get op_json elements failed:" << e.what(); + } + return ret; +} + +bool OpLib::DecodeOpInfo(const nlohmann::json& obj, const mindspore::kernel::OpImplyType imply_type, + const std::string& impl_path) { + std::shared_ptr op_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(op_info); + op_info->set_op_name(obj.at(kOpName)); + op_info->set_imply_type(imply_type); + op_info->set_impl_path(impl_path); + op_info->set_fusion_type(obj.at(kFusionType)); + if (imply_type == kTBE) { + op_info->set_async_flag(obj.at(kAsyncFlag)); + op_info->set_binfile_name(obj.at(kBinfileName)); + op_info->set_compute_cost(obj.at(kComputeCost)); + op_info->set_kernel_name(obj.at(kKernelName)); + op_info->set_partial_flag(obj.at(kPartialFlag)); + } + auto attrs = obj.at(kAttr); + for (const auto& attr : attrs) { + if (!DecodeAttr(attr, imply_type, op_info)) { + MS_LOG(DEBUG) << "DecodeAttr Failed"; + return false; + } + } + auto inputs = obj.at(kIputs); + for (const auto& input : inputs) { + if (!DecodeInputOutput(input, imply_type, kInput, op_info)) { + MS_LOG(DEBUG) << "DecodeInputOutput Failed"; + return false; + } + } + auto outputs = obj.at(kOutputs); + for (const auto& output : outputs) { + if (!DecodeInputOutput(output, imply_type, kOutput, op_info)) { + MS_LOG(DEBUG) << "DecodeInputOutput Failed"; + return false; + } + } + if (!GetRefInfo(op_info)) { + MS_LOG(DEBUG) << "GetRefInfo Failed"; + return false; + } + if (!CheckRepetition(op_info)) { + MS_LOG(DEBUG) << "CheckRepetition Failed"; + return false; + } + op_info_.push_back(op_info); + return true; +} + +bool OpLib::DecodeAttr(const nlohmann::json& obj, const OpImplyType imply_type, + const std::shared_ptr& op_info) { + MS_EXCEPTION_IF_NULL(op_info); + bool ret = true; + try { + std::shared_ptr op_attr = std::make_shared(); + MS_EXCEPTION_IF_NULL(op_attr); + op_attr->set_name(obj.at(kName)); + op_attr->set_param_type(obj.at(kParamType)); + op_attr->set_type(obj.at(kType)); + if (imply_type == kTBE) { + op_attr->set_value(obj.at(kValue)); + } + if (obj.find(kDefaultValue) != obj.end()) { + op_attr->set_default_value(obj.at(kDefaultValue)); + } + op_info->add_attrs_ptr(op_attr); + } catch (const std::exception& e) { + MS_LOG(DEBUG) << "DecodeAttr failed:" << e.what(); + ret = false; + } + return ret; +} + +bool OpLib::DecodeInputOutput(const nlohmann::json& obj, const OpImplyType imply_type, const OpIOType io_type, + const std::shared_ptr& op_info) { + bool ret = true; + try { + std::shared_ptr op_io = std::make_shared(); + MS_EXCEPTION_IF_NULL(op_io); + op_io->set_index(obj.at(kIndex)); + op_io->set_name(obj.at(kName)); + op_io->set_dtypes(obj.at(kDtype)); + op_io->set_formats(obj.at(kFormat)); + if (op_io->dtypes().size() != op_io->formats().size()) { + MS_LOG(DEBUG) << "op" << op_io->name() << "dtype size:" << op_io->dtypes() + << "is not equal to format size:" << op_io->formats(); + return false; + } + if (obj.find(kParamType) != obj.end()) { + op_io->set_param_type(obj.at(kParamType)); + } + if (imply_type == kTBE) { + if (obj.find(kNeedCompile) != obj.end()) { + op_io->set_need_compile(obj.at(kNeedCompile)); + } + if (obj.find(kShape) != obj.end()) { + op_io->set_shape(obj.at(kShape)); + } + } + + if (io_type == kInput) { + op_info->add_inputs_ptr(op_io); + } else if (io_type == kOutput) { + op_info->add_outputs_ptr(op_io); + } + } catch (const std::exception& e) { + MS_LOG(DEBUG) << "DecodeInputOutput failed" << e.what(); + ret = false; + } + return ret; +} + +std::shared_ptr OpLib::FindOp(const std::string& op_name, OpImplyType imply_type) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool is_gpu = (context->device_target() == kGPUDevice); + if ((is_gpu && imply_type == kTBE) || (!is_gpu && imply_type != kTBE)) { + MS_LOG(DEBUG) << "FindOp failed: opname:" << op_name << "imply_type:" << ImplTypeToStr(imply_type) + << "current op num:" << op_info_.size(); + return nullptr; + } + for (const auto& op_info : op_info_) { + MS_EXCEPTION_IF_NULL(op_info); + if (op_info->op_name() == op_name && op_info->imply_type() == imply_type) { + return op_info; + } + } + MS_LOG(DEBUG) << "FindOp failed: opname:" << op_name << "imply_type:" << ImplTypeToStr(imply_type) + << "current op num:" << op_info_.size(); + return nullptr; +} + +bool OpLib::GetRefInfo(const std::shared_ptr& op_info) { + MS_EXCEPTION_IF_NULL(op_info); + const auto& output_infos = op_info->outputs_ptr(); + const auto& input_infos = op_info->inputs_ptr(); + for (size_t out_index = 0; out_index < output_infos.size(); out_index++) { + const auto& out_name = output_infos[out_index]->name(); + for (size_t in_index = 0; in_index < input_infos.size(); in_index++) { + const auto& in_name = input_infos[in_index]->name(); + if (out_name == in_name) { + if (op_info->has_ref_index(out_index)) { + MS_LOG(DEBUG) << "The out_index" << out_index << "is already in ref_info"; + return false; + } + op_info->add_ref_pair(out_index, in_index); + MS_LOG(INFO) << "add ref info, op name is " << op_info->op_name() << ", outindex is " << out_index + << ", in_index is " << in_index; + } + } + } + return true; +} + +bool OpLib::CheckRepetition(const std::shared_ptr& op_info) { + MS_EXCEPTION_IF_NULL(op_info); + for (const auto& exist_op_info : op_info_) { + MS_EXCEPTION_IF_NULL(exist_op_info); + if (exist_op_info->op_name() == op_info->op_name() && exist_op_info->imply_type() == op_info->imply_type() && + exist_op_info->impl_path() != op_info->impl_path()) { + MS_LOG(DEBUG) << "Has already exist, drop the latter one, op name:" << op_info->op_name() + << "op type:" << ImplTypeToStr(op_info->imply_type()); + return false; + } + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/oplib/oplib.h b/mindspore/ccsrc/kernel/oplib/oplib.h new file mode 100644 index 00000000000..37c3fdcfc79 --- /dev/null +++ b/mindspore/ccsrc/kernel/oplib/oplib.h @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_OPLIB_OPLIB_H_ +#define MINDSPORE_CCSRC_KERNEL_OPLIB_OPLIB_H_ +#include +#include +#include +#include +#include "kernel/oplib/opinfo.h" + +namespace mindspore { +namespace kernel { +class OpLib { + public: + OpLib() = default; + virtual ~OpLib() = default; + bool RegOp(const std::string& json_string, const std::string& impl_path); + static std::shared_ptr FindOp(const std::string& op_name, OpImplyType imply_type); + + protected: + static std::vector> op_info_; + + private: + static bool DecodeOpInfo(const nlohmann::json& obj, const OpImplyType imply_type, const std::string& impl_path); + static bool DecodeAttr(const nlohmann::json& obj, const OpImplyType imply_type, + const std::shared_ptr& op_info); + static bool DecodeInputOutput(const nlohmann::json& obj, const OpImplyType imply_type, const OpIOType io_type, + const std::shared_ptr& op_info); + static bool GetRefInfo(const std::shared_ptr& op_info); + static bool CheckRepetition(const std::shared_ptr& op_info); +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_OPLIB_OPLIB_H_ diff --git a/mindspore/ccsrc/kernel/task_stream.h b/mindspore/ccsrc/kernel/task_stream.h new file mode 100644 index 00000000000..babfe7eecda --- /dev/null +++ b/mindspore/ccsrc/kernel/task_stream.h @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TASK_STREAM_H_ +#define MINDSPORE_CCSRC_KERNEL_TASK_STREAM_H_ + +#include +#include +#include +#include +#include "runtime/base.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace kernel { +class TaskStream { + public: + TaskStream() = default; + ~TaskStream() = default; + static std::shared_ptr GetInstance() { + static const std::shared_ptr instance = std::make_shared(); + return instance; + } + + void set_gen_stream_list(const std::vector &stream_list) { gen_stream_list_ = stream_list; } + void set_run_stream_list(const std::vector &stream_list) { run_stream_list_ = stream_list; } + void SetGenStreamIndex(uint32_t stream_id, uint32_t index) { gen_stream_index_map_[stream_id] = index; } + std::unordered_map GetGenStreamIndexMap() { return gen_stream_index_map_; } + uint32_t GetGenStreamIndex(uint32_t stream_id) { + auto iter = gen_stream_index_map_.find(stream_id); + if (iter == gen_stream_index_map_.end()) { + MS_LOG(EXCEPTION) << "stream_id not in gen_stream_index_map_"; + } + return iter->second; + } + const std::vector &gen_stream_list() const { return gen_stream_list_; } + const std::vector &run_stream_list() const { return run_stream_list_; } + + private: + std::vector gen_stream_list_; + std::vector run_stream_list_; + std::unordered_map gen_stream_index_map_; +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_TASK_STREAM_H_ diff --git a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc new file mode 100644 index 00000000000..ddb78a08df3 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc @@ -0,0 +1,472 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_adapter.h" + +#include +#include +#include +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "kernel/oplib/opinfo.h" + +namespace mindspore { +namespace kernel { +namespace tbe { +static std::map tbe_func_adapter_map = { + {"re_lu", "relu"}, + {"tensor_add", "add"}, + {"reduce_mean", "reduce_mean_d"}, + {"reduce_max", "reduce_max_d"}, + {"conv2d_backprop_filter", "conv2d_backprop_filter_d"}, + {"conv2d_backprop_input", "conv2d_backprop_input_d"}, + {"top_kv2", "top_k"}, + {"scatter_nd", "scatter_nd_d"}, + {"tile", "tile_d"}, + {"gather_v2", "gather_v2_d"}, + {"batch_mat_mul", "batch_matmul"}, + {"b_n_training_reduce", "bn_training_reduce"}, + {"b_n_training_update", "bn_training_update"}, + {"b_n_training_reduce_grad", "bn_training_reduce_grad"}, + {"b_n_training_update_grad", "bn_training_update_grad"}, + {"n_pu_clear_float_status", "n_p_u_clear_float_status"}, + {"n_pu_get_float_status", "n_p_u_get_float_status"}, + {"n_pu_alloc_float_status", "n_p_u_alloc_float_status"}, + {"dropout_do_mask", "drop_out_do_mask"}, + {"strided_slice", "strided_slice_d"}, + {"strided_slice_grad", "strided_slice_grad_d"}, + {"transpose", "transpose_d"}, + {"unsorted_segment_sum", "unsorted_segment_sum_d"}, + {"concat", "concat_d"}, + {"slice", "slice_d"}, + {"reduce_sum", "reduce_sum_d"}, + {"one_hot", "one_hot_d"}, + {"sum", "reduce_sum_d"}, + {"lamb_next_mv_with_decay_v1", "lamb_next_m_v_with_decay_v1"}, + {"lamb_next_mv", "lamb_next_m_v"}, + {"split", "split_d"}, + {"resize_nearest_neighbor", "resize_nearest_neighbor_d"}, + {"resize_nearest_neighbor_grad", "resize_nearest_neighbor_grad_d"}, + {"pad", "pad_d"}, + {"adam", "apply_adam"}}; + +void TbeAdapter::NormalizeFuncName(std::string *func_name) { + if (func_name == nullptr) { + MS_LOG(EXCEPTION) << "func_name is null"; + } + std::string name_tmp; + bool sub_head = false; + for (string::iterator iter = func_name->begin(); iter != func_name->end(); ++iter) { + if (islower(*iter)) { + sub_head = false; + } + if (isdigit(*iter)) { + sub_head = true; + } + if (isupper(*iter) && iter != func_name->begin()) { + if (!sub_head) { + (void)name_tmp.insert(name_tmp.end(), '_'); + sub_head = true; + } else { + string::iterator iter_next = iter + 1; + if (iter_next != func_name->end()) { + if (islower(*iter_next)) { + (void)name_tmp.insert(name_tmp.end(), '_'); + } + } + } + } + (void)name_tmp.insert(name_tmp.end(), *iter); + } + (void)transform(name_tmp.begin(), name_tmp.end(), name_tmp.begin(), ::tolower); + *func_name = name_tmp; + auto iter = tbe_func_adapter_map.find(*func_name); + if (iter != tbe_func_adapter_map.end()) { + MS_LOG(INFO) << "map actual op fron me " << func_name << "to tbe op" << iter->second; + *func_name = iter->second; + } +} + +void TbeAdapter::SetTbeAttrsForTransDataOp(const mindspore::AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + if (AnfAlgo::GetCNodeName(anf_node) == kTransDataOpName) { + std::string input_format = AnfAlgo::GetInputFormat(anf_node, 0); + std::string output_format = AnfAlgo::GetOutputFormat(anf_node, 0); + if (input_format == kOpFormat_DEFAULT) { + input_format = kOpFormat_NCHW; + } + if (output_format == kOpFormat_DEFAULT) { + output_format = kOpFormat_NCHW; + } + AnfAlgo::SetNodeAttr("src_format", MakeValue(input_format), anf_node); + AnfAlgo::SetNodeAttr("dst_format", MakeValue(output_format), anf_node); + } +} + +std::unordered_set input_order_adjusted_ops = { + "Conv2DBackpropInput", "Conv2DBackpropFilter", "LogSoftmaxGrad", "LayerNormGrad", + "LayerNormXBackprop", "LayerNormBetaGammaBackprop", "MinimumGrad", "MaximumGrad"}; + +void TbeAdapter::InputOrderPass(const std::string &op_name, std::vector> const &inputs_list, + nlohmann::json *inputs_json) { + MS_EXCEPTION_IF_NULL(inputs_json); + if (input_order_adjusted_ops.find(op_name) == input_order_adjusted_ops.end()) { + (void)std::copy(inputs_list.begin(), inputs_list.end(), std::back_inserter((*inputs_json))); + } else { + if (op_name == "MinimumGrad" || op_name == "MaximumGrad") { + inputs_json->push_back(inputs_list[2]); + inputs_json->push_back(inputs_list[0]); + inputs_json->push_back(inputs_list[1]); + for (size_t i = 3; i < inputs_list.size(); ++i) { + inputs_json->push_back(inputs_list[i]); + } + } else { + inputs_json->push_back(inputs_list[1]); + inputs_json->push_back(inputs_list[0]); + for (size_t i = 2; i < inputs_list.size(); ++i) { + inputs_json->push_back(inputs_list[i]); + } + } + } +} + +std::map TbeAdapter::build_json_attr_pass_map_ = { + {"MaxPoolWithArgmax", TbeAdapter::MaxPoolWithArgmaxAttrJsonPass}, + {"MaxPoolGradWithArgmax", TbeAdapter::MaxPoolGradWithArgmaxAttrJsonPass}, + {"Conv2D", TbeAdapter::Conv2DAttrJsonPass}, + {"Conv2DBackpropFilter", TbeAdapter::Conv2DBackpropFilterAttrJsonPass}, + {"Conv2DBackpropInput", TbeAdapter::Conv2DBackpropInputAttrJsonPass}, + {"MaximumGrad", TbeAdapter::MaximumGradAttrJsonPass}, + {"MinimumGrad", TbeAdapter::MinimumGradAttrJsonPass}, + {"Cast", TbeAdapter::CastAttrJsonPass}}; + +bool TbeAdapter::RunAttrPass(const mindspore::AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(attrs_json); + auto cnode_name = AnfAlgo::GetCNodeName(anf_node); + auto FPass = build_json_attr_pass_map_.find(cnode_name); + if (FPass != build_json_attr_pass_map_.end()) { + FPass->second(anf_node, op_info_attrs, attrs_json); + return true; + } + return false; +} + +void TbeAdapter::MaxPoolWithArgmaxAttrJsonPass( + const mindspore::AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attr_num = op_info_attrs.size(); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (size_t i = 0; i < attr_num; i++) { + nlohmann::json attr_obj; + MS_EXCEPTION_IF_NULL(op_info_attrs[i]); + std::string attr_name = op_info_attrs[i]->name(); + if (primitive->GetAttr(attr_name) != nullptr) { + auto value = primitive->GetAttr(attr_name); + if (attr_name == "pad_mode") { + std::string attr_value = GetValue(value); + (void)transform(attr_value.begin(), attr_value.end(), attr_value.begin(), ::toupper); + attr_obj["value"] = attr_value; + } else { + std::vector attr_value; + int data = GetValue(value); + attr_value.push_back(1); + attr_value.push_back(data); + attr_value.push_back(data); + attr_value.push_back(1); + attr_obj["value"] = attr_value; + } + attr_obj["valid"] = true; + } else { + attr_obj["valid"] = false; + } + attr_obj["name"] = attr_name; + attrs_json->push_back(attr_obj); + } +} + +void TbeAdapter::MaxPoolGradWithArgmaxAttrJsonPass( + const mindspore::AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MaxPoolWithArgmaxAttrJsonPass(anf_node, op_info_attrs, attrs_json); +} + +void TbeAdapter::Conv2DAttrJsonPass(const mindspore::AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attr_num = op_info_attrs.size(); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (size_t i = 0; i < attr_num; i++) { + nlohmann::json attr_obj; + MS_EXCEPTION_IF_NULL(op_info_attrs[i]); + std::string attr_name = op_info_attrs[i]->name(); + std::vector attr_value; + if (primitive->GetAttr(attr_name) != nullptr) { + auto value = primitive->GetAttr(attr_name); + int data = GetValue(value); + size_t list_int_size = 0; + if (attr_name == "stride") { + list_int_size = 4; + } else if (attr_name == "dilation") { + list_int_size = 4; + } else if (attr_name == "pad") { + value = primitive->GetAttr("pad_list"); + attr_value = GetValue>(value); + } + for (size_t j = 0; j < list_int_size; j++) { + attr_value.push_back(data); + } + attr_obj["value"] = attr_value; + } else { + attr_obj["value"] = 0; + } + attr_obj["name"] = attr_name; + attr_obj["valid"] = true; + (*attrs_json).push_back(attr_obj); + } + MS_LOG(INFO) << "Conv2DAttrPass done."; +} + +void TbeAdapter::Conv2DBackpropFilterAttrJsonPass( + const mindspore::AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attr_num = op_info_attrs.size(); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (size_t i = 0; i < attr_num; i++) { + nlohmann::json attr_obj; + MS_EXCEPTION_IF_NULL(op_info_attrs[i]); + std::string attr_name = op_info_attrs[i]->name(); + if (primitive->GetAttr(attr_name) != nullptr) { + auto value = primitive->GetAttr(attr_name); + if (attr_name == "pad_mode") { + std::string attr_value = GetValue(value); + (void)transform(attr_value.begin(), attr_value.end(), attr_value.begin(), ::toupper); + attr_obj["value"] = attr_value; + } else if (attr_name == "filter_sizes") { + std::vector attr_value = GetValue>(value); + attr_obj["value"] = attr_value; + } else { + std::vector attr_value; + int data = GetValue(value); + size_t list_int_size = 0; + if (attr_name == "stride") { + list_int_size = 2; + } else if (attr_name == "dilation") { + list_int_size = 4; + } + for (size_t j = 0; j < list_int_size; j++) { + attr_value.push_back(data); + } + attr_obj["value"] = attr_value; + } + attr_obj["valid"] = true; + } else { + attr_obj["valid"] = false; + } + attr_obj["name"] = attr_name; + attrs_json->push_back(attr_obj); + } + MS_LOG(INFO) << "Conv2DBackpropFilterAttrJsonPass done."; +} + +void TbeAdapter::Conv2DBackpropInputAttrJsonPass( + const mindspore::AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attr_num = op_info_attrs.size(); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (size_t i = 0; i < attr_num; i++) { + nlohmann::json attr_obj; + MS_EXCEPTION_IF_NULL(op_info_attrs[i]); + std::string attr_name = op_info_attrs[i]->name(); + if (primitive->GetAttr(attr_name) != nullptr) { + auto value = primitive->GetAttr(attr_name); + if (attr_name == "pad_mode") { + std::string attr_value = GetValue(value); + (void)transform(attr_value.begin(), attr_value.end(), attr_value.begin(), ::toupper); + attr_obj["value"] = attr_value; + } else if (attr_name == "input_sizes") { + std::vector attr_value = GetValue>(value); + attr_obj["value"] = attr_value; + } else { + std::vector attr_value; + int data = GetValue(value); + size_t list_int_size = 0; + if (attr_name == "stride") { + list_int_size = 2; + } else if (attr_name == "dilation") { + list_int_size = 4; + } + for (size_t j = 0; j < list_int_size; j++) { + attr_value.push_back(data); + } + attr_obj["value"] = attr_value; + } + attr_obj["valid"] = true; + } else { + attr_obj["valid"] = false; + } + attr_obj["name"] = attr_name; + attrs_json->push_back(attr_obj); + } + MS_LOG(INFO) << "Conv2DBackpropInputAttrJsonPass done."; +} + +void TbeAdapter::MaximumGradAttrJsonPass(const mindspore::AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attr_num = op_info_attrs.size(); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (size_t i = 0; i < attr_num; i++) { + nlohmann::json attr_obj; + MS_EXCEPTION_IF_NULL(op_info_attrs[i]); + std::string attr_name = op_info_attrs[i]->name(); + auto value = primitive->GetAttr(attr_name); + if (value != nullptr) { + bool attr_value = GetValue(value); + attr_obj["value"] = attr_value; + attr_obj["valid"] = true; + } else { + attr_obj["valid"] = false; + } + attr_obj["name"] = attr_name; + attrs_json->push_back(attr_obj); + } + MS_LOG(INFO) << "MaximumGradAttrJsonPass done."; +} + +void TbeAdapter::MinimumGradAttrJsonPass(const mindspore::AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attr_num = op_info_attrs.size(); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (size_t i = 0; i < attr_num; i++) { + nlohmann::json attr_obj; + MS_EXCEPTION_IF_NULL(op_info_attrs[i]); + std::string attr_name = op_info_attrs[i]->name(); + auto value = primitive->GetAttr(attr_name); + if (value != nullptr) { + bool attr_value = GetValue(value); + attr_obj["value"] = attr_value; + attr_obj["valid"] = true; + } else { + attr_obj["valid"] = false; + } + attr_obj["name"] = attr_name; + attrs_json->push_back(attr_obj); + } + MS_LOG(INFO) << "MinimumGradAttrJsonPass done."; +} + +static int TypeStrToDstType(const std::string &type_str) { + int ret = -1; + if (type_str == "Float" || type_str == "Float32") { + ret = 0; + } else if (type_str == "Float16") { + ret = 1; + } else if (type_str == "Int8") { + ret = 2; + } else if (type_str == "Int32") { + ret = 3; + } else if (type_str == "UInt8") { + ret = 4; + } else if (type_str == "UInt64") { + ret = 10; + } else if (type_str == "Bool_") { + ret = 12; + } else { + MS_EXCEPTION(ArgumentError) << "type str is invailed: " << type_str; + } + return ret; +} + +void TbeAdapter::CastAttrJsonPass(const mindspore::AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(attrs_json); + if (op_info_attrs.size() != 1) { + MS_LOG(INFO) << "cast node should has dst_type attr"; + return; + } + auto attr_name = op_info_attrs[0]->name(); + auto type_ptr = std::make_shared(TypeIdToType(AnfAlgo::GetOutputDeviceDataType(anf_node, 0))); + MS_EXCEPTION_IF_NULL(type_ptr); + auto type_element = type_ptr->element(); + MS_EXCEPTION_IF_NULL(type_element); + auto dtype = type_element->ToString(); + auto dst_type_value = TypeStrToDstType(dtype); + nlohmann::json attr_obj; + attr_obj["value"] = dst_type_value; + attr_obj["valid"] = true; + attr_obj["name"] = attr_name; + attrs_json->push_back(attr_obj); + MS_LOG(INFO) << "CastAttrJsonPass done."; +} + +void TbeAdapter::GenTopKV2IndicesTensorInfo(const std::shared_ptr &anf_node, + size_t real_input_index, std::vector *input_list, + mindspore::kernel::kCreaterType creater_type) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(input_list); + auto input_x_shape = AnfAlgo::GetOutputInferShape(anf_node, 0); + size_t last_dim = input_x_shape[input_x_shape.size() - 1]; + std::vector tensor_shape = {last_dim}; + std::vector tensor_origin_shape = {last_dim}; + std::string tensor_format = AnfAlgo::GetInputFormat(anf_node, static_cast(real_input_index)); + if (tensor_format == kOpFormat_DEFAULT) { + tensor_format = kOpFormat_NCHW; + } + std::string tensor_origin_format = kOpFormat_NCHW; + std::string tensor_dtype = "float16"; + nlohmann::json input_desc_json; + input_desc_json["dtype"] = tensor_dtype; + input_desc_json["name"] = AnfAlgo::GetCNodeName(anf_node); + input_desc_json["ori_shape"] = tensor_origin_shape; + input_desc_json["ori_format"] = tensor_origin_format; + input_desc_json["shape"] = tensor_shape; + if (creater_type == OP_SELECT_FORMAT) { + input_desc_json["format"] = tensor_origin_format; + } else { + input_desc_json["format"] = tensor_format; + } + input_desc_json["valid"] = true; + input_list->emplace_back(input_desc_json); +} +} // namespace tbe +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_adapter.h b/mindspore/ccsrc/kernel/tbe/tbe_adapter.h new file mode 100644 index 00000000000..3997318c869 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_adapter.h @@ -0,0 +1,77 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_TBE_ADAPTER_H +#define MINDSPORE_CCSRC_KERNEL_TBE_TBE_ADAPTER_H + +#include +#include +#include +#include +#include "nlohmann/json.hpp" +#include "ir/base.h" +#include "kernel/oplib/opinfo.h" +// Note: This file is mainly used to adapt the ME front-end operator description and +// the TBE back-end operator implementation difference +namespace mindspore { +namespace kernel { +enum kCreaterType : int { SINGLE_BUILD = 0, PREBUILD, OP_SELECT_FORMAT, CHECK_SUPPORTED }; +namespace tbe { +using FAttrsPass = void (*)(const AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); +class TbeAdapter { + public: + TbeAdapter() = default; + ~TbeAdapter() = default; + static void NormalizeFuncName(std::string *func_name); + static void SetTbeAttrsForTransDataOp(const AnfNodePtr &anf_node); + static void InputOrderPass(const std::string &op_name, std::vector> const &inputs_list, + nlohmann::json *inputs_json); + static bool RunAttrPass(const AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void GenTopKV2IndicesTensorInfo(const std::shared_ptr &anf_node, size_t real_input_index, + std::vector *input_list, kCreaterType creater_type); + + private: + static void MaxPoolWithArgmaxAttrJsonPass(const AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void MaxPoolGradWithArgmaxAttrJsonPass(const AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void Conv2DAttrJsonPass(const AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void Conv2DBackpropFilterAttrJsonPass(const AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void Conv2DBackpropInputAttrJsonPass(const AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void MaximumGradAttrJsonPass(const AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + static void MinimumGradAttrJsonPass(const AnfNodePtr &anf_node, + const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + + static void CastAttrJsonPass(const AnfNodePtr &anf_node, const std::vector> &op_info_attrs, + nlohmann::json *attrs_json); + + static std::map build_json_attr_pass_map_; +}; +} // namespace tbe +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_TBE_TBE_ADAPTER_H diff --git a/mindspore/ccsrc/kernel/tbe/tbe_convert_utils.cc b/mindspore/ccsrc/kernel/tbe/tbe_convert_utils.cc new file mode 100644 index 00000000000..025ff935e2c --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_convert_utils.cc @@ -0,0 +1,125 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_convert_utils.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +namespace tbe { +const std::unordered_map type_str_id_maps = { + {"float", TypeId::kNumberTypeFloat32}, {"float16", TypeId::kNumberTypeFloat16}, + {"float32", TypeId::kNumberTypeFloat32}, {"float64", TypeId::kNumberTypeFloat64}, + {"int", TypeId::kNumberTypeInt}, {"int8", TypeId::kNumberTypeInt8}, + {"int16", TypeId::kNumberTypeInt16}, {"int32", TypeId::kNumberTypeInt32}, + {"int64", TypeId::kNumberTypeInt64}, {"uint", TypeId::kNumberTypeUInt}, + {"uint8", TypeId::kNumberTypeUInt8}, {"uint16", TypeId::kNumberTypeUInt16}, + {"uint32", TypeId::kNumberTypeUInt32}, {"uint64", TypeId::kNumberTypeUInt64}, + {"bool", TypeId::kNumberTypeBool}, +}; + +const std::map type_id_str_maps = { + {TypeId::kNumberTypeFloat32, "float32"}, {TypeId::kNumberTypeFloat16, "float16"}, + {TypeId::kNumberTypeFloat, "float"}, {TypeId::kNumberTypeFloat64, "float64"}, + {TypeId::kNumberTypeInt, "int"}, {TypeId::kNumberTypeInt8, "int8"}, + {TypeId::kNumberTypeInt16, "int16"}, {TypeId::kNumberTypeInt32, "int32"}, + {TypeId::kNumberTypeInt64, "int64"}, {TypeId::kNumberTypeUInt, "uint"}, + {TypeId::kNumberTypeUInt8, "uint8"}, {TypeId::kNumberTypeUInt16, "uint16"}, + {TypeId::kNumberTypeUInt32, "uint32"}, {TypeId::kNumberTypeUInt64, "uint64"}, + {TypeId::kNumberTypeBool, "bool"}, +}; + +const std::map type_str_maps = { + {"Float32", "float32"}, {"Float16", "float16"}, {"Int8", "int8"}, {"Int16", "int16"}, + {"UInt16", "uint16"}, {"UInt8", "uint8"}, {"Int32", "int32"}, {"UInt32", "uint32"}, + {"Int64", "int64"}, {"UInt64", "uint64"}, {"Bool_", "int8"}, {"Float64", "double"}, +}; + +const std::unordered_map type_nbyte_maps = { + {"float16", sizeof(float) / 2}, {"float32", sizeof(float)}, {"float64", sizeof(float) * 2}, + {"int8", sizeof(int) / 4}, {"int16", sizeof(int) / 2}, {"int32", sizeof(int)}, + {"int64", sizeof(int) * 2}, {"uint8", sizeof(int) / 4}, {"uint16", sizeof(int) / 2}, + {"uint32", sizeof(int)}, {"uint64", sizeof(int) * 2}, {"bool", sizeof(char)}, +}; + +const std::unordered_map fusion_type_maps = { + {"CONVLUTION", FusionType::CONVLUTION}, {"ELEMWISE", FusionType::ELEMWISE}, {"COMMREDUCE", FusionType::COMMREDUCE}, + {"SEGMENT", FusionType::SEGMENT}, {"OPAQUE", FusionType::OPAQUE}, +}; + +TypeId DtypeToTypeId(const std::string &dtypes) { + auto iter = type_str_id_maps.find(dtypes); + if (iter == type_str_id_maps.end()) { + MS_LOG(EXCEPTION) << "Illegal input device dtype: " << dtypes; + } + return iter->second; +} + +std::string DtypeToString(const std::string &dtypes) { + auto iter = type_str_maps.find(dtypes); + if (iter == type_str_maps.end()) { + MS_LOG(EXCEPTION) << "Illegal input dtype: " << dtypes; + } + return iter->second; +} + +std::string TypeIdToString(TypeId type_id) { + auto iter = type_id_str_maps.find(type_id); + if (iter == type_id_str_maps.end()) { + MS_LOG(EXCEPTION) << "Illegal input dtype." << TypeIdLabel(type_id); + } + return iter->second; +} + +size_t GetDtypeNbyte(const std::string &dtypes) { + auto iter = type_nbyte_maps.find(dtypes); + if (iter == type_nbyte_maps.end()) { + MS_LOG(EXCEPTION) << "Illegal input dtype: " << dtypes; + } + return iter->second; +} + +FusionType GetFusionType(const std::string &pattern) { + auto iter = fusion_type_maps.find(pattern); + if (iter == fusion_type_maps.end()) { + MS_LOG(DEBUG) << "Illegal fusion pattern: " << pattern; + return UNKNOWN_FUSION_TYPE; + } + return iter->second; +} + +std::string GetProcessor(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string device; + switch (AnfAlgo::GetProcessor(anf_node)) { + case Processor::AICORE: + device = kProcessorAiCore; + break; + default: + MS_LOG(DEBUG) << "Unknown processor type." << anf_node->fullname_with_scope(); + break; + } + return device; +} +} // namespace tbe +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_convert_utils.h b/mindspore/ccsrc/kernel/tbe/tbe_convert_utils.h new file mode 100644 index 00000000000..9b9b3770dfd --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_convert_utils.h @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_COMMON_UTILS_H_ +#define MINDSPORE_CCSRC_KERNEL_TBE_COMMON_UTILS_H_ + +#include +#include "kernel/kernel.h" +#include "ir/base.h" +#include "ir/dtype/type.h" + +namespace mindspore { +namespace kernel { +namespace tbe { +constexpr auto kProcessorAiCore = "aicore"; +TypeId DtypeToTypeId(const std::string &dtypes); + +std::string DtypeToString(const std::string &dtypes); + +std::string TypeIdToString(TypeId type_id); + +size_t GetDtypeNbyte(const std::string &dtypes); + +FusionType GetFusionType(const std::string &pattern); + +std::string GetProcessor(const AnfNodePtr &anf_node); +} // namespace tbe +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_TBE_COMMON_UTILS_H_ diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc new file mode 100644 index 00000000000..496f99df1c7 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc @@ -0,0 +1,799 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_kernel_build.h" + +#include +#include +#include +#include + +#include "operator/ops.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/tbe/tbe_kernel_mod.h" +#include "kernel/tbe/tbe_adapter.h" +#include "kernel/tbe/tbe_python_funcs.h" +#include "kernel/tbe/tbe_convert_utils.h" +#include "kernel/tbe/tbe_utils.h" + +namespace mindspore { +namespace kernel { +using mindspore::kernel::tbe::TbeAdapter; +using mindspore::kernel::tbe::TbeUtils; +constexpr auto kFusionOpList = "op_list"; +constexpr auto kFusionKernelNamePrfix = "te_fusion"; +constexpr auto kOptional = "optional_"; +constexpr auto kOpFormat_FRACTAL_Z = "FRACTAL_Z"; + +std::string NormalizeFullScopeName(const string &full_scope_name) { + // exp:Default/ReLU-op0 -->Default_ReLU_op0 + string normal_ret = full_scope_name; + std::replace(normal_ret.begin(), normal_ret.end(), '/', '_'); + std::replace(normal_ret.begin(), normal_ret.end(), '-', '_'); + return normal_ret; +} + +bool TbeKernelJsonCreator::GenTbeSingleKernelJson(const shared_ptr &anf_node, + nlohmann::json *kernel_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(kernel_json); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + auto op_info_ptr = mindspore::kernel::OpLib::FindOp(op_name, OpImplyType::kTBE); + MS_EXCEPTION_IF_NULL(op_info_ptr); + (*kernel_json)["platform"] = "TBE"; + (*kernel_json)["gen_model"] = "single"; + (*kernel_json)["impl_path"] = op_info_ptr->impl_path(); + nlohmann::json op_info_json; + if (op_info_ptr->impl_path().empty()) { + tbe::TbeAdapter::NormalizeFuncName(&op_name); + } else { + op_name = op_info_ptr->kernel_name(); + } + op_info_json["name"] = op_name; + // generate inputs json + nlohmann::json inputs_json; + if (!GenTbeInputsJson(anf_node, op_info_ptr, &inputs_json)) { + MS_LOG(ERROR) << "Anf Node [" << op_name << "] generate inputs json failed"; + return false; + } + op_info_json["inputs"] = inputs_json; + // generate outputs json + nlohmann::json outputs_json; + if (!GenTbeOutputsJson(anf_node, op_info_ptr, &outputs_json)) { + MS_LOG(ERROR) << "Anf Node [" << op_name << "] generate outputs json failed"; + return false; + } + op_info_json["outputs"] = outputs_json; + // generate attrs json + nlohmann::json attrs_json; + (void)GenTbeAttrJson(anf_node, op_info_ptr, &attrs_json); + op_info_json["attrs"] = attrs_json; + std::string json_str = op_info_json.dump(); + size_t hash_id = std::hash()(json_str); + json_name_ = op_name + "_" + std::to_string(hash_id); + json_info_ = json_str; + if (creater_type_ == PREBUILD) { + op_info_json["kernel_name"] = NormalizeFullScopeName(anf_node->fullname_with_scope()); + } else { + op_info_json["kernel_name"] = json_name_; + } + (*kernel_json)["op_info"] = op_info_json; + if (creater_type_ == SINGLE_BUILD) { + TbeUtils::SaveJsonInfo(json_name_, json_info_); + } + + MS_LOG(INFO) << "Operate type:" << creater_type_ << ", full scope name is :" << anf_node->fullname_with_scope() + << ", json info name is : " << json_name_ << ", kernel json:" << kernel_json->dump(); + + return true; +} + +bool TbeKernelJsonCreator::GenInputDescJson(const shared_ptr &anf_node, size_t real_input_index, bool value, + const shared_ptr &input_ptr, const string &op_input_name, + size_t input_i, vector *input_list) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(input_ptr); + MS_EXCEPTION_IF_NULL(input_list); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (input_ptr->name() == "input_indices" && op_name == kTopKOpName) { + TbeAdapter::GenTopKV2IndicesTensorInfo(anf_node, real_input_index, input_list, creater_type_); + } else { + // dtype : float16 + auto tensor_dtype = + std::make_shared(TypeIdToType(AnfAlgo::GetInputDeviceDataType(anf_node, real_input_index))); + MS_EXCEPTION_IF_NULL(tensor_dtype); + std::string dtype = tensor_dtype->element()->ToString(); + dtype = tbe::DtypeToString(dtype); + + // format + std::string format = AnfAlgo::GetInputFormat(anf_node, real_input_index); + if (format == kOpFormat_DEFAULT) { + format = kOpFormat_NCHW; + } else if (format == kOpFormat_FRAC_Z) { + format = kOpFormat_FRACTAL_Z; + } + + nlohmann::json input_desc_json; + input_desc_json["dtype"] = dtype; + input_desc_json["name"] = op_input_name + std::to_string(input_i); + auto ori_shape = AnfAlgo::GetPrevNodeOutputInferShape(anf_node, real_input_index); + if (ori_shape.empty()) { + ori_shape.emplace_back(1); + } + input_desc_json["ori_shape"] = ori_shape; + input_desc_json["ori_format"] = kOpFormat_NCHW; + auto shape = AnfAlgo::GetInputDeviceShape(anf_node, real_input_index); + if (shape.empty()) { + shape.emplace_back(1); + } + if (creater_type_ == OP_SELECT_FORMAT || creater_type_ == CHECK_SUPPORTED) { + input_desc_json["shape"] = ori_shape; + input_desc_json["format"] = kOpFormat_NCHW; + } else { + input_desc_json["shape"] = shape; + input_desc_json["format"] = format; + } + input_desc_json["valid"] = value; + input_list->emplace_back(input_desc_json); + } + return true; +} + +bool TbeKernelJsonCreator::GenInputList(const shared_ptr &anf_node, size_t input_tensor_num, + const shared_ptr &input_ptr, size_t *real_input_index, + string *op_input_name, vector *input_list) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(input_ptr); + MS_EXCEPTION_IF_NULL(real_input_index); + MS_EXCEPTION_IF_NULL(op_input_name); + MS_EXCEPTION_IF_NULL(input_list); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + size_t real_input_num = AnfAlgo::GetInputTensorNum(anf_node); + bool value = true; + for (size_t input_i = 0; input_i < input_tensor_num; input_i++) { + if (*real_input_index >= real_input_num) { + if (input_ptr->param_type() == "optional") { + *op_input_name = input_ptr->name() + "_optional_"; + nlohmann::json input_desc_json; + input_desc_json["valid"] = false; + input_desc_json["name"] = *op_input_name + std::to_string(*real_input_index); + input_list->emplace_back(input_desc_json); + continue; + } + MS_LOG(ERROR) << "input num" << *real_input_index << "is not match op inputs"; + return false; + } + if (op_name == "BatchNorm") { + if (input_ptr->name() == "mean" || input_ptr->name() == "variance") { + auto attr = primitive->GetAttr("is_training"); + MS_EXCEPTION_IF_NULL(attr); + bool is_training = GetValue(attr); + MS_LOG(INFO) << "op_name" << op_name << ", tensor_name " << input_ptr->name() << ", is_training " + << is_training; + if (is_training) { + (*real_input_index)++; + break; + } + } + } + bool ret = GenInputDescJson(anf_node, *real_input_index, value, input_ptr, *op_input_name, input_i, input_list); + (*real_input_index)++; + if (!ret) { + return false; + } + } + return true; +} + +bool GetInputNameAndRealNum(const std::shared_ptr &anf_node, const shared_ptr &input_ptr, + size_t *dyn_input_index, size_t *input_num, std::string *op_input_name) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(input_ptr); + MS_EXCEPTION_IF_NULL(dyn_input_index); + MS_EXCEPTION_IF_NULL(input_num); + MS_EXCEPTION_IF_NULL(op_input_name); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + // for dynamic input number, dyn_input_sizes has the info of dynamic input num for each input. + std::vector dyn_input_sizes; + if (primitive->GetAttr(kAttrDynInputSizes) != nullptr) { + dyn_input_sizes = GetValue>(primitive->GetAttr(kAttrDynInputSizes)); + } + + if (input_ptr->param_type() == "dynamic") { + if (*dyn_input_index >= dyn_input_sizes.size()) { + MS_LOG(ERROR) << "dyn input index" << *dyn_input_index << "is over dyn input num" << dyn_input_sizes.size(); + return false; + } + *input_num = IntToSize(dyn_input_sizes[*dyn_input_index]); + *op_input_name = input_ptr->name() + "_dynamic_"; + (*dyn_input_index)++; + // if optional input is exist + } else { + *input_num = 1; + *op_input_name = input_ptr->name() + "_"; + } + return true; +} + +bool TbeKernelJsonCreator::GenTbeInputsJson(const std::shared_ptr &anf_node, + const std::shared_ptr &op_info, nlohmann::json *inputs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(op_info); + MS_EXCEPTION_IF_NULL(inputs_json); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (op_name == kAtomicAddrCleanOpName) { + return true; + } + std::vector> inputs_ptr = op_info->inputs_ptr(); + if (inputs_ptr.empty()) { + MS_LOG(INFO) << "Apply kernel " << op_name << "registration info has no input info"; + return true; + } + auto op_info_input_num = inputs_ptr.size(); + size_t dyn_input_index = 0; + size_t real_input_index = 0; + std::vector> inputs_list; + for (size_t i = 0; i < op_info_input_num; i++) { + size_t input_tensor_num; + std::shared_ptr input_ptr = inputs_ptr[i]; + std::string op_input_name; + MS_EXCEPTION_IF_NULL(input_ptr); + if (!GetInputNameAndRealNum(anf_node, input_ptr, &dyn_input_index, &input_tensor_num, &op_input_name)) { + return false; + } + std::vector input_list; + if (!GenInputList(anf_node, input_tensor_num, input_ptr, &real_input_index, &op_input_name, &input_list)) { + return false; + } + inputs_list.emplace_back(input_list); + } + + TbeAdapter::InputOrderPass(op_name, inputs_list, inputs_json); + return true; +} + +bool TbeKernelJsonCreator::GenTbeOutputsJson(const std::shared_ptr &anf_node, + const std::shared_ptr &op_info, nlohmann::json *outputs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(op_info); + MS_EXCEPTION_IF_NULL(outputs_json); + auto op_name = AnfAlgo::GetCNodeName(anf_node); + if (op_name == kAtomicAddrCleanOpName) { + return true; + } + auto outputs_ptr = op_info->outputs_ptr(); + return GenOutputDescJson(anf_node, outputs_ptr, outputs_json); +} + +bool TbeKernelJsonCreator::GenOutputDescJson(const shared_ptr &anf_node, + const vector> &outputs_ptr, + nlohmann::json *outputs_json) { + MS_EXCEPTION_IF_NULL(outputs_json); + size_t output_idx = 0; + auto op_name = AnfAlgo::GetCNodeName(anf_node); + size_t real_output_num = AnfAlgo::GetOutputTensorNum(anf_node); + + for (const auto &output_ptr : outputs_ptr) { + size_t output_obj_num = 0; + if (output_ptr->param_type() == "required") { + output_obj_num = 1; + } else if (output_ptr->param_type() == "dynamic") { + if (outputs_ptr.size() > 1) { + MS_LOG(ERROR) << "Dynamic output is unsupported multi output!"; + return false; + } + output_obj_num = real_output_num; + } else { + if (output_idx >= real_output_num) { + MS_LOG(INFO) << "op:" << op_name << ", output" << output_ptr->name() << " is optional, output is none."; + std::vector output_list; + nlohmann::json output_obj; + output_obj["name"] = output_ptr->name(); + output_obj["valid"] = false; + output_list.emplace_back(output_obj); + (*outputs_json).push_back(output_list); + continue; + } else { + output_obj_num = 1; + } + } + std::vector output_list; + GenOutputList(anf_node, output_obj_num, output_ptr, &output_idx, &output_list); + (*outputs_json).push_back(output_list); + } + return true; +} + +void TbeKernelJsonCreator::GenOutputList(const shared_ptr &anf_node, const size_t &output_obj_num, + const shared_ptr &output_ptr, size_t *output_idx, + vector *output_list) { + MS_EXCEPTION_IF_NULL(output_idx); + MS_EXCEPTION_IF_NULL(output_list); + for (size_t i = 0; i < output_obj_num; i++) { + nlohmann::json output_obj; + auto type_ptr = std::make_shared(TypeIdToType(AnfAlgo::GetOutputDeviceDataType(anf_node, *output_idx))); + std::string dtype = type_ptr->element()->ToString(); + dtype = tbe::DtypeToString(dtype); + std::string format = AnfAlgo::GetOutputFormat(anf_node, *output_idx); + if (format == kOpFormat_DEFAULT) { + format = kOpFormat_NCHW; + } else if (format == kOpFormat_FRAC_Z) { + format = kOpFormat_FRACTAL_Z; + } + std::vector ori_shape; + if (AnfAlgo::GetOutputInferShape(anf_node, *output_idx).empty()) { + ori_shape.emplace_back(1); + } else { + ori_shape = AnfAlgo::GetOutputInferShape(anf_node, *output_idx); + } + output_obj["dtype"] = dtype; + auto shape = AnfAlgo::GetOutputDeviceShape(anf_node, *output_idx); + if (shape.empty()) { + shape.emplace_back(1); + } + if (creater_type_ == OP_SELECT_FORMAT || creater_type_ == CHECK_SUPPORTED) { + output_obj["shape"] = ori_shape; + output_obj["format"] = kOpFormat_NCHW; + } else { + output_obj["shape"] = shape; + output_obj["format"] = format; + } + output_obj["ori_shape"] = ori_shape; + output_obj["ori_format"] = kOpFormat_NCHW; + output_obj["name"] = output_ptr->name(); + output_obj["valid"] = true; + + output_list->emplace_back(output_obj); + (*output_idx)++; + } +} + +bool TbeKernelJsonCreator::GenTbeAttrJson(const std::shared_ptr &anf_node, + const std::shared_ptr &op_info, nlohmann::json *attrs_json) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(op_info); + MS_EXCEPTION_IF_NULL(attrs_json); + auto attrs_ptr = op_info->attrs_ptr(); + if (TbeAdapter::RunAttrPass(anf_node, attrs_ptr, attrs_json)) { + return true; + } + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + for (const auto &attr_ptr : attrs_ptr) { + std::string attr_name = attr_ptr->name(); + if (primitive->GetAttr(attr_name) != nullptr) { + nlohmann::json attr_obj; + auto value = primitive->GetAttr(attr_name); + std::string type = attr_ptr->type(); + ParseAttrValue(type, value, &attr_obj); + attr_obj["name"] = attr_name; + attr_obj["valid"] = true; + (*attrs_json).push_back(attr_obj); + } + } + return true; +} + +void TbeKernelJsonCreator::ParseAttrValue(const std::string &type, const mindspore::ValuePtr &value, + nlohmann::json *attr_obj) { + MS_EXCEPTION_IF_NULL(value); + MS_EXCEPTION_IF_NULL(attr_obj); + if (type == "int") { + auto attr_value = GetValue(value); + (*attr_obj)["value"] = attr_value; + } else if (type == "str") { + auto attr_value = GetValue(value); + if (attr_value == kOpFormat_FRAC_Z) { + attr_value = kOpFormat_FRACTAL_Z; + } + (*attr_obj)["value"] = attr_value; + } else if (type == "bool") { + auto attr_value = GetValue(value); + (*attr_obj)["value"] = attr_value; + } else if (type == "float") { + auto attr_value = GetValue(value); + (*attr_obj)["value"] = attr_value; + } else if (type == "listInt") { + std::vector attr_value; + auto value_type = value->type(); + MS_EXCEPTION_IF_NULL(value_type); + auto value_type_str = value_type->ToString(); + if (value_type_str == "Int32") { + int data = GetValue(value); + attr_value.push_back(data); + } else { + attr_value = GetValue>(value); + } + (*attr_obj)["value"] = attr_value; + } else if (type == "listListInt") { + auto attr_value = GetValue>>(value); + (*attr_obj)["value"] = attr_value; + } else { + MS_LOG(EXCEPTION) << "type: " << type << "not support"; + } +} + +bool TbeKernelBuild::GetIOSize(const nlohmann::json &kernel_json, std::vector *input_size_list, + std::vector *output_size_list) { + if (input_size_list == nullptr || output_size_list == nullptr) { + MS_LOG(ERROR) << "input size or output size is nullptr"; + return false; + } + input_size_list->clear(); + output_size_list->clear(); + for (size_t i = 0; i < kernel_json["op_info"]["inputs"].size(); i++) { + for (size_t m = 0; m < kernel_json["op_info"]["inputs"][i].size(); m++) { + size_t size_i = 1; + if (kernel_json["op_info"]["inputs"][i][m]["valid"] == false) { + std::string input_name = kernel_json["op_info"]["inputs"][i][m]["name"]; + MS_LOG(INFO) << "Input name:" << input_name << "is optional, valid is false."; + continue; + } + for (const auto &j : kernel_json["op_info"]["inputs"][i][m]["shape"]) { + size_i *= static_cast(j); + } + std::string dtype = kernel_json["op_info"]["inputs"][i][m]["dtype"]; + size_t nbyte = tbe::GetDtypeNbyte(dtype); + size_i *= nbyte; + input_size_list->push_back(size_i); + } + } + for (size_t i = 0; i < kernel_json["op_info"]["outputs"].size(); i++) { + for (size_t m = 0; m < kernel_json["op_info"]["outputs"][i].size(); m++) { + size_t size_i = 1; + if (kernel_json["op_info"]["outputs"][i][m]["valid"] == false) { + std::string output_name = kernel_json["op_info"]["outputs"][i][m]["name"]; + MS_LOG(INFO) << "Output name:" << output_name << " is optional, valid is false."; + continue; + } + for (const auto &j : kernel_json["op_info"]["outputs"][i][m]["shape"]) { + size_i *= static_cast(j); + } + std::string dtype = kernel_json["op_info"]["outputs"][i][m]["dtype"]; + size_t nbyte = tbe::GetDtypeNbyte(dtype); + size_i *= nbyte; + output_size_list->push_back(size_i); + } + } + return true; +} + +bool TbeKernelBuild::GenFusionScopeJson(const vector &input_nodes, + const vector &compute_nodes, nlohmann::json *fusion_str, + std::string *fusion_kernel) { + MS_EXCEPTION_IF_NULL(fusion_str); + MS_EXCEPTION_IF_NULL(fusion_kernel); + // get input layer info + std::vector> input_layers; + if (!GetInputLayers(input_nodes, compute_nodes, &input_layers)) { + return false; + } + // gen fusion scopre_op jsom + vector compute_list; + (*fusion_kernel) = kFusionKernelNamePrfix; + // index: fusion build option input record, next one from 0 + static size_t index = 0; + auto layer_iter = input_layers.begin(); + auto compute_op_iter = compute_nodes.begin(); + for (; compute_op_iter != compute_nodes.end(); ++compute_op_iter, ++layer_iter) { + nlohmann::json compute_op_str; + (void)GenFusionComputeJson(*compute_op_iter, &layer_iter, &compute_op_str, fusion_kernel, &index); + compute_list.push_back(compute_op_str); + } + index = 0; + // gen data input json + vector data_list; + for (const auto &layer : input_layers) { + for (const auto &data_input : layer) { + nlohmann::json data_str; + if (!GenFusionDataInputJson(data_input, &data_str, &index)) { + MS_LOG(DEBUG) << "GenFusionDataInputJson faild."; + return false; + } + data_list.push_back(data_str); + } + } + index = 0; + data_list.insert(data_list.end(), compute_list.begin(), compute_list.end()); + (*fusion_str)[kFusionOpList] = data_list; + return true; +} + +void TbeKernelBuild::GenDescJson(const shared_ptr &anf_node, size_t out_idx, + nlohmann::json *output_desc) { + std::string output_desc_name = anf_node->fullname_with_scope(); + if (out_idx > 0) { + output_desc_name = output_desc_name + "_" + std::to_string(out_idx); + } + (*output_desc)["name"] = NormalizeFullScopeName(output_desc_name); + auto type_id = AnfAlgo::GetOutputDeviceDataType(anf_node, out_idx); + (*output_desc)["data_type"] = tbe::TypeIdToString(type_id); + auto ori_shape = AnfAlgo::GetOutputInferShape(anf_node, out_idx); + if (ori_shape.empty()) { + ori_shape.emplace_back(1); + } + (*output_desc)["ori_shape"] = ori_shape; + auto shape = AnfAlgo::GetOutputDeviceShape(anf_node, out_idx); + if (shape.empty()) { + shape.emplace_back(1); + } + (*output_desc)["shape"] = shape; + auto format = AnfAlgo::GetOutputFormat(anf_node, out_idx); + if (format == kOpFormat_DEFAULT) { + if (ori_shape.size() == 4) { + format = kOpFormat_NCHW; + } else { + format = "ND"; + } + } + (*output_desc)["format"] = format; + (*output_desc)["ori_format"] = kOpFormat_NCHW; + (*output_desc)["output_index"] = out_idx; +} + +void TbeKernelBuild::GenReusedOutputDesc(const shared_ptr &anf_node, size_t index, + size_t output_index, nlohmann::json *output_desc) { + std::string output_desc_name = anf_node->fullname_with_scope() + "_" + std::to_string(index); + (*output_desc)["name"] = NormalizeFullScopeName(output_desc_name); + (*output_desc)["data_type"] = tbe::TypeIdToString(kNumberTypeFloat32); + (*output_desc)["output_index"] = output_index; + std::vector shape; + (*output_desc)["shape"] = shape; +} + +bool TbeKernelBuild::GetInputLayers(const vector &input_nodes, + const vector &compute_nodes, + std::vector> *input_layers) { + size_t input_size = 0; + for (const auto &compute_node : compute_nodes) { + std::vector layer; + MS_EXCEPTION_IF_NULL(compute_node); + auto ccompute_node = compute_node->cast(); + if (ccompute_node == nullptr) { + MS_LOG(DEBUG) << "fusion compute node must be cnode"; + return false; + } + for (size_t i = 1; i < ccompute_node->inputs().size(); ++i) { + auto input = ccompute_node->input(i); + auto find_iter = std::find(input_nodes.begin(), input_nodes.end(), input); + if (find_iter != input_nodes.end()) { + layer.emplace_back((*find_iter)); + } + } + input_size += layer.size(); + input_layers->emplace_back(layer); + } + if (input_nodes.size() != input_size) { + MS_LOG(DEBUG) << "fusion scope error, layer input:" << input_size << ", input_node:" << input_nodes.size(); + return false; + } + return true; +} + +bool TbeKernelBuild::GenFusionDataInputJson(const shared_ptr &data_input, nlohmann::json *data_str, + size_t *index) { + MS_EXCEPTION_IF_NULL(data_str); + MS_EXCEPTION_IF_NULL(index); + std::vector output_desc_list; + if (!data_input) { + MS_LOG(INFO) << "data input is optional node"; + auto name = std::string(kOptional) + std::to_string(*index); + (*data_str)["name"] = name; + nlohmann::json output_desc; + output_desc["name"] = name; + output_desc["shape"] = "NULL"; + output_desc_list.push_back(output_desc); + (*index)++; + } else { + auto kernel_idx = AnfAlgo::VisitKernel(data_input, 0); + auto real_node = kernel_idx.first; + size_t real_idx = kernel_idx.second; + MS_LOG(INFO) << "real name " << real_node->fullname_with_scope() << " index:" << real_idx; + // "output_desc" + nlohmann::json output_desc; + GenDescJson(real_node, real_idx, &output_desc); + output_desc_list.push_back(output_desc); + (*data_str)["name"] = NormalizeFullScopeName(real_node->fullname_with_scope()); + } + (*data_str)["output_desc"] = output_desc_list; + (*data_str)["type"] = "Data"; + return true; +} + +bool TbeKernelBuild::IsDynamicInput(const mindspore::CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(cnode); + auto primitive = AnfAlgo::GetCNodePrimitive(cnode); + MS_EXCEPTION_IF_NULL(primitive); + // for dynamic input number, dyn_input_sizes has the info of dynamic input num for each input. + bool ret = false; + std::vector dyn_input_sizes; + auto dynamic_input_attr = primitive->GetAttr(kAttrDynInputSizes); + if (dynamic_input_attr != nullptr) { + dyn_input_sizes = GetValue>(dynamic_input_attr); + auto real_input_size = cnode->inputs().size() - 1; + auto dyn_input_size = dyn_input_sizes.size(); + if (dyn_input_size != 1) { + MS_LOG(DEBUG) << "fusion build not support dyn_input_sizes > 1"; + return ret; + } + if (IntToSize(dyn_input_sizes[0]) != real_input_size) { + MS_LOG(DEBUG) << " dyn_input_size" << dyn_input_sizes[0] << "not equal real_input_size" << real_input_size; + return ret; + } + ret = true; + } + return ret; +} + +size_t TbeKernelBuild::GetOptionalInput(const mindspore::CNodePtr &cnode, bool is_dynamic_input) { + if (is_dynamic_input) { + return 0; + } + MS_EXCEPTION_IF_NULL(cnode); + auto node_name = AnfAlgo::GetCNodeName(cnode); + auto op_info = OpLib::FindOp(node_name, kTBE); + MS_EXCEPTION_IF_NULL(cnode); + if (op_info->inputs_ptr().size() < (cnode->inputs().size() - 1)) { + MS_EXCEPTION(ArgumentError) << "op info error, node name:" << cnode->fullname_with_scope(); + } + return (op_info->inputs_ptr().size() + 1 - cnode->inputs().size()); +} + +bool TbeKernelBuild::GenFusionComputeInputeJson(const mindspore::CNodePtr &cnode, + std::vector>::iterator *layer_iter, + std::vector *input_desc_list, size_t *index) { + MS_EXCEPTION_IF_NULL(cnode); + MS_EXCEPTION_IF_NULL(input_desc_list); + bool is_dynamic_input = IsDynamicInput(cnode); + for (size_t i = 1; i < cnode->inputs().size(); ++i) { + auto input = cnode->input(i); + auto kernel_idx = AnfAlgo::VisitKernel(input, 0); + auto real_node = kernel_idx.first; + size_t real_idx = kernel_idx.second; + MS_LOG(INFO) << "real name" << real_node->fullname_with_scope() << "index:" << real_idx; + nlohmann::json input_desc; + GenDescJson(real_node, real_idx, &input_desc); + if (is_dynamic_input) { + MS_LOG(INFO) << "node has dynamic input."; + input_desc["dyn_index"] = (i - 1); + } + (*input_desc_list).emplace_back(input_desc); + } + size_t optional_num = GetOptionalInput(cnode, is_dynamic_input); + if (optional_num > 0) { + MS_LOG(INFO) << "node has optional input."; + for (size_t i = 0; i < optional_num; ++i) { + nlohmann::json optional_input_desc; + optional_input_desc["name"] = std::string(kOptional) + std::to_string(*index); + (*index)++; + (*layer_iter)->emplace_back(nullptr); + (*input_desc_list).emplace_back(optional_input_desc); + } + } + return true; +} + +bool TbeKernelBuild::GenFusionComputeJson(const mindspore::AnfNodePtr &compute_node, + std::vector>::iterator *layer_iter, + nlohmann::json *compute_op_str, std::string *fusion_kernel_name, + size_t *index) { + MS_EXCEPTION_IF_NULL(compute_node); + auto cnode = compute_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + // gen input desc + std::vector input_desc_list; + (void)GenFusionComputeInputeJson(cnode, layer_iter, &input_desc_list, index); + (*compute_op_str)["input_desc"] = input_desc_list; + // gen output desc + std::vector output_desc_list; + auto output_size = AnfAlgo::GetOutputTensorNum(cnode); + for (size_t i = 0; i < output_size; ++i) { + nlohmann::json output_desc; + GenDescJson(cnode, i, &output_desc); + output_desc_list.push_back(output_desc); + } + + if (AnfAlgo::GetCNodeName(cnode) == prim::kPrimConv2D->name()) { + if (AnfAlgo::HasNodeAttr(kAttrOutputUsedNum, compute_node)) { + auto output_used_num = AnfAlgo::GetNodeAttr(compute_node, kAttrOutputUsedNum); + for (size_t i = output_size; i < output_used_num; ++i) { + nlohmann::json output_desc; + GenReusedOutputDesc(cnode, i, 0, &output_desc); + output_desc_list.push_back(output_desc); + } + } + } + + (*compute_op_str)["output_desc"] = output_desc_list; + // gen others + auto type = AnfAlgo::GetCNodeName(cnode); + if (type == "TensorAdd") { + type = "Add"; + } + (*compute_op_str)["type"] = type; + tbe::TbeAdapter::NormalizeFuncName(&type); + (*compute_op_str)["func_name"] = type; + (*compute_op_str)["name"] = NormalizeFullScopeName(cnode->fullname_with_scope()); + (void)(*fusion_kernel_name).append("_"); + (void)(*fusion_kernel_name).append(type); + return true; +} + +size_t TbeKernelBuild::GetIOSizeImpl(const nlohmann::json &desc) { + size_t ret = 1; + for (const auto &shape_item : desc["shape"]) { + ret *= static_cast(shape_item); + } + std::string data_type = desc["data_type"]; + size_t nbyte = tbe::GetDtypeNbyte(data_type); + ret *= nbyte; + return ret; +} + +bool TbeKernelBuild::GetIOSize(const nlohmann::json &fusion_op_list, const vector &output_nodes, + std::vector *input_size_list, std::vector *output_size_list) { + MS_EXCEPTION_IF_NULL(input_size_list); + MS_EXCEPTION_IF_NULL(output_size_list); + input_size_list->clear(); + output_size_list->clear(); + + for (const auto &op : fusion_op_list) { + if (op["type"] == "Data") { + const auto &data_output_desc = op["output_desc"]; + for (const auto &data_output : data_output_desc) { + if (data_output["shape"] == "NULL") { + break; + } + auto ret = GetIOSizeImpl(data_output); + input_size_list->push_back(ret); + } + } + } + + for (const auto &output_node : output_nodes) { + auto kernel_idx = AnfAlgo::VisitKernel(output_node, 0); + auto real_node = kernel_idx.first; + size_t real_idx = kernel_idx.second; + for (const auto &op : fusion_op_list) { + auto normal_name = NormalizeFullScopeName(real_node->fullname_with_scope()); + if (op["name"] == normal_name) { + auto op_output_desces = op["output_desc"]; + if (output_node != real_node) { + // tuple_get item + MS_LOG(DEBUG) << "output is a tuple getitem node"; + auto output_desc = op_output_desces[real_idx]; + if (output_desc["shape"].empty()) { + continue; + } + auto ret = GetIOSizeImpl(output_desc); + output_size_list->push_back(ret); + } else { + for (const auto &output_desc : op_output_desces) { + if (output_desc["shape"].empty()) { + continue; + } + auto ret = GetIOSizeImpl(output_desc); + output_size_list->push_back(ret); + } + } + } + } + } + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.h b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.h new file mode 100644 index 00000000000..bc4895ac6f0 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.h @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_BUILD_H_ +#define MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_BUILD_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "ir/dtype.h" +#include "kernel/kernel.h" +#include "pybind11/stl.h" +#include "kernel/oplib/oplib.h" +#include "kernel/tbe/tbe_adapter.h" + +namespace mindspore { +namespace kernel { +// kernel operate type used for generate json + +class TbeKernelBuild { + public: + static bool GetIOSize(const nlohmann::json &kernel_json, std::vector *input_size_list, + std::vector *output_size_list); + // Ub Fuison + static bool GenFusionScopeJson(const std::vector &input_nodes, + const std::vector &compute_nodes, nlohmann::json *fusion_str, + std::string *fusion_kernel); + static bool GetIOSize(const nlohmann::json &fusion_op_list, const std::vector &output_nodes, + std::vector *input_size_list, std::vector *output_size_list); + + private: + TbeKernelBuild() = default; + ~TbeKernelBuild() = default; + static bool GenFusionDataInputJson(const shared_ptr &data_input, nlohmann::json *data_str, + size_t *index); + static bool GenFusionComputeJson(const mindspore::AnfNodePtr &compute_node, + std::vector>::iterator *layer_iter, + nlohmann::json *compute_op_str, std::string *fusion_kernel_name, size_t *index); + static bool GenFusionComputeInputeJson(const mindspore::CNodePtr &cnode, + std::vector>::iterator *layer_iter, + std::vector *input_desc_list, size_t *index); + static void GenDescJson(const shared_ptr &anf_node, size_t out_idx, nlohmann::json *output_desc); + static void GenReusedOutputDesc(const shared_ptr &anf_node, size_t index, size_t output_index, + nlohmann::json *output_desc); + static size_t GetIOSizeImpl(const nlohmann::json &desc); + static bool GetInputLayers(const vector &input_nodes, + const vector &compute_nodes, + std::vector> *input_layers); + static bool IsDynamicInput(const CNodePtr &cnode); + static size_t GetOptionalInput(const CNodePtr &cnode, bool is_dynamic_input); +}; + +class TbeKernelJsonCreator { + public: + explicit TbeKernelJsonCreator(kCreaterType creater_type = SINGLE_BUILD) : creater_type_(creater_type) {} + ~TbeKernelJsonCreator() = default; + bool GenTbeSingleKernelJson(const std::shared_ptr &anf_node, nlohmann::json *kernel_json); + std::string json_name() { return json_name_; } + + private: + bool GenTbeInputsJson(const std::shared_ptr &anf_node, const std::shared_ptr &op_info, + nlohmann::json *inputs_json); + bool GenTbeOutputsJson(const std::shared_ptr &anf_node, const std::shared_ptr &op_info, + nlohmann::json *outputs_json); + bool GenTbeAttrJson(const std::shared_ptr &anf_node, const std::shared_ptr &op_info, + nlohmann::json *attrs_json); + void ParseAttrValue(const std::string &type, const ValuePtr &value, nlohmann::json *attr_obj); + bool GenInputDescJson(const shared_ptr &anf_node, size_t real_input_index, bool value, + const shared_ptr &input_ptr, const string &op_input_name, size_t input_i, + vector *input_list); + bool GenOutputDescJson(const shared_ptr &anf_node, const vector> &outputs_ptr, + nlohmann::json *outputs_json); + bool GenInputList(const shared_ptr &anf_node, size_t input_tensor_num, const shared_ptr &input_ptr, + size_t *real_input_index, string *op_input_name, vector *input_list); + void GenOutputList(const shared_ptr &anf_node, const size_t &output_obj_num, + const shared_ptr &output_ptr, size_t *output_idx, vector *output_list); + kCreaterType creater_type_; + std::string json_name_; + std::string json_info_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_BUILD_H_ diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_mod.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_mod.cc new file mode 100644 index 00000000000..422a06a3365 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_mod.cc @@ -0,0 +1,114 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_kernel_mod.h" +#include +#include "runtime/rt.h" +#include "nlohmann/json.hpp" +#include "graphengine/inc/framework/ge_runtime/task_info.h" + +namespace mindspore { +namespace kernel { +using TbeTaskInfoPtr = std::shared_ptr; +using tbe::KernelManager; +bool TbeKernelMod::Launch(const std::vector &inputs, + const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) { + if (stream_ptr == 0) { + MS_LOG(ERROR) << "stream_ptr should not be nullptr."; + return false; + } + + if (kernel_pack_ == nullptr) { + MS_LOG(ERROR) << "kernel pack should not be nullptr."; + return false; + } + + uint32_t blockdim = 1; // default blockdim equal to 1. + auto func_stub = KernelManager::GenFuncStub(*kernel_pack_, false, &blockdim); + if (func_stub == 0) { + MS_LOG(ERROR) << "GenFuncStub failed."; + return false; + } + + // pack all addresses into a vector. + std::vector runtimeargs; + (void)std::transform(std::begin(inputs), std::end(inputs), std::back_inserter(runtimeargs), + [](const AddressPtr &input) -> void * { return input->addr; }); + (void)std::transform(std::begin(outputs), std::end(outputs), std::back_inserter(runtimeargs), + [](const AddressPtr &output) -> void * { return output->addr; }); + if (!workspace.empty()) { + (void)std::transform(std::begin(workspace), std::end(workspace), std::back_inserter(runtimeargs), + [](const AddressPtr &addr) -> void * { return addr->addr; }); + } + rtL2Ctrl_t *l2ctrl = nullptr; + auto *stream = reinterpret_cast(stream_ptr); + const void *stubFunc = reinterpret_cast(func_stub); + auto argsSize = static_cast(UlongToUint(sizeof(void *)) * runtimeargs.size()); + if (RT_ERROR_NONE != rtKernelLaunch(stubFunc, blockdim, runtimeargs.data(), argsSize, l2ctrl, stream)) { + MS_LOG(ERROR) << "Call runtime rtKernelLaunch error."; + return false; + } + + return true; +} + +vector TbeKernelMod::GenTask(const std::vector &inputs, + const std::vector &workspaces, + const std::vector &outputs, uint32_t stream_id) { + if (kernel_pack_ == nullptr) { + MS_EXCEPTION(ArgumentError) << "kernel pack should not be nullptr."; + } + + std::vector args; + std::vector sm_desc; + std::vector meta_data; + std::vector input_data_addrs; + std::vector output_data_addrs; + std::vector workspace_addrs; + + // pack all addresses into a vector. + (void)std::transform(std::begin(inputs), std::end(inputs), std::back_inserter(input_data_addrs), + [](const AddressPtr &input) -> void * { return input->addr; }); + (void)std::transform(std::begin(outputs), std::end(outputs), std::back_inserter(output_data_addrs), + [](const AddressPtr &output) -> void * { return output->addr; }); + if (!workspaces.empty()) { + (void)std::transform(std::begin(workspaces), std::end(workspaces), std::back_inserter(workspace_addrs), + [](const AddressPtr &workspace) -> void * { return workspace->addr; }); + } + + uint32_t block_dim = 1; // default blockdim equal to 1. + auto funcstub = KernelManager::GenFuncStub(*kernel_pack_, false, &block_dim); + if (funcstub == 0) { + MS_EXCEPTION(ArgumentError) << "GenFuncStub failed."; + } + + std::string stub_func = KernelManager::GetStubFuncName(kernel_pack_); + + MS_LOG(INFO) << "block_dim is:" << block_dim; + + TbeTaskInfoPtr task_info_ptr = + make_shared(stream_id, stub_func, block_dim, args, 0, sm_desc, nullptr, 0, meta_data, + input_data_addrs, output_data_addrs, workspace_addrs); + return {task_info_ptr}; +} + +vector TbeKernelMod::GenParameters() { + auto kernel_json_info = kernel_pack_->kernel_json_info(); + return kernel_json_info.parameters; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_mod.h b/mindspore/ccsrc/kernel/tbe/tbe_kernel_mod.h new file mode 100644 index 00000000000..35fc7f517de --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_mod.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_MOD_H_ +#define MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_MOD_H_ + +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "kernel/tbe/tbe_utils.h" + +namespace mindspore { +namespace kernel { +class TbeKernelMod : public KernelMod { + public: + explicit TbeKernelMod(KernelPackPtr kernel_pack) : kernel_pack_(std::move(kernel_pack)) {} + ~TbeKernelMod() override = default; + + void SetInputSizeList(const std::vector &size_list) { input_size_list_ = size_list; } + void SetOutputSizeList(const std::vector &size_list) { output_size_list_ = size_list; } + void SetWorkspaceSizeList(const std::vector &size_list) { workspace_size_list_ = size_list; } + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, uintptr_t stream_ptr) override; + vector GenTask(const std::vector &inputs, const std::vector &workspaces, + const std::vector &outputs, uint32_t stream_id) override; + std::vector GenParameters() override; + + private: + KernelPackPtr kernel_pack_; + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; +}; + +using TbeKernelModPtr = std::shared_ptr; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_MOD_H_ diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc new file mode 100644 index 00000000000..16ddec1b4a6 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc @@ -0,0 +1,238 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_kernel_parallel_build.h" + +#include +#include +#include +#include +#include +#include + +#include "utils/context/ms_context.h" +#include "kernel/tbe/tbe_adapter.h" +#include "kernel/tbe/tbe_kernel_build.h" +#include "kernel/tbe/tbe_kernel_mod.h" +#include "session/anf_runtime_algorithm.h" +#include "./common.h" +#include "kernel/tbe/tbe_python_funcs.h" +#include "kernel/tbe/tbe_convert_utils.h" +#include "kernel/tbe/tbe_utils.h" + +namespace mindspore { +namespace kernel { +using mindspore::kernel::tbe::TbeUtils; +constexpr auto kParallelCompileModule = "mindspore._extends.parallel_compile.tbe_compiler.tbe_process"; +constexpr auto kCreateParallelCompiler = "create_tbe_parallel_compiler"; +constexpr auto kStartCompileOp = "start_compile_op"; +constexpr auto kWaitOne = "wait_one"; + +bool TbeOpParallelBuild(std::vector anf_nodes) { + auto build_manger = std::make_shared(); + MS_EXCEPTION_IF_NULL(build_manger); + set processed_kernel; + for (const auto &anf_node : anf_nodes) { + // gen kernel json + tbe::TbeAdapter::SetTbeAttrsForTransDataOp(anf_node); + if (AnfAlgo::GetKernelMod(anf_node) != nullptr) { + continue; + } + const std::string &processor = tbe::GetProcessor(anf_node); + nlohmann::json kernel_json; + TbeKernelJsonCreator creator(SINGLE_BUILD); + if (!creator.GenTbeSingleKernelJson(anf_node, &kernel_json)) { + MS_LOG(ERROR) << "GenTbeSingleKernelJson failed"; + return false; + } + // get size + std::vector input_size_list; + std::vector output_size_list; + (void)TbeKernelBuild::GetIOSize(kernel_json, &input_size_list, &output_size_list); + // search cache + const std::string &json_name = creator.json_name(); + if (build_manger->SearchInCache(json_name, processor, input_size_list, output_size_list, anf_node.get())) { + MS_LOG(INFO) << "Use cached kernel, kernel json name:." << json_name; + continue; + } + // same op not need build, but need wait build finish to set kernel mode + if (processed_kernel.find(json_name) != processed_kernel.end()) { + build_manger->SaveSameOpInfo(anf_node, json_name, input_size_list, output_size_list); + continue; + } + (void)processed_kernel.insert(json_name); + // op build + auto task_id = build_manger->StartCompileOp(kernel_json); + build_manger->SaveTaskInfo(task_id, anf_node, json_name, input_size_list, output_size_list); + } + while (!build_manger->IsAllTaskFinish()) { + int task_id = -1; + char *task_result = nullptr; + auto ret = build_manger->WaitOne(&task_id, &task_result); + if (!ret) { + MS_EXCEPTION(ArgumentError) << "Build Failed. wait one ret:" << ret << ", task id:" << task_id; + } + + if ((task_result != nullptr) && (strcmp(task_result, "Success") != 0)) { + MS_EXCEPTION(ArgumentError) << "task compile Failed, task id:" << task_id << ", cause:" << task_result; + } + (void)build_manger->TaskFinishProcess(task_id); + } + return build_manger->GenSameOpKernelMod(); +} + +ParallelBuildManager::ParallelBuildManager() { tbe_parallel_compiler_ = TbePythonFuncs::TbeParallelCompiler(); } + +int32_t ParallelBuildManager::StartCompileOp(const nlohmann::json &kernel_json) const { + PyObject *pRes = nullptr; + PyObject *pArgs = PyTuple_New(1); + std::string json_str = kernel_json.dump(); + PyObject *arg1 = Py_BuildValue("s", json_str.c_str()); + (void)PyTuple_SetItem(pArgs, 0, arg1); + pRes = PyObject_CallMethod(tbe_parallel_compiler_, kStartCompileOp, "O", pArgs); + if (pRes == nullptr) { + PyErr_Print(); + MS_EXCEPTION(ArgumentError) << "Failed to call function start_compile_op"; + } + int task_id; + (void)PyArg_Parse(pRes, "i", &task_id); + MS_LOG(INFO) << "start compile , task id:" << task_id; + return task_id; +} + +bool ParallelBuildManager::WaitOne(int *task_id, char **task_result) const { + MS_LOG(INFO) << "wait task start."; + MS_EXCEPTION_IF_NULL(task_id); + MS_EXCEPTION_IF_NULL(task_result); + PyObject *pRes = nullptr; + PyObject *pArg = Py_BuildValue("()"); + pRes = PyObject_CallMethod(tbe_parallel_compiler_, kWaitOne, "O", pArg); + if (pRes == nullptr) { + PyErr_Print(); + MS_EXCEPTION(ArgumentError) << "Failed to call function wait_one"; + return false; + } + (void)PyArg_ParseTuple(pRes, "is", task_id, task_result); + return true; +} + +void ParallelBuildManager::SaveTaskInfo(int32_t task_id, const mindspore::AnfNodePtr &anf_node, + const std::string &json_name, const std::vector &input_size_list, + const std::vector &output_size_list, int32_t scope_id) { + MS_LOG(INFO) << "SaveTaskInfo, task id: " << task_id; + struct KernelBuildTaskInfo task_info; + task_info.node = anf_node.get(); + task_info.json_name = json_name; + if (anf_node == nullptr) { + task_info.processor = tbe::kProcessorAiCore; + } else { + task_info.processor = tbe::GetProcessor(anf_node); + } + task_info.input_size_list.assign(input_size_list.begin(), input_size_list.end()); + task_info.output_size_list.assign(output_size_list.begin(), output_size_list.end()); + task_info.scope_id = scope_id; + task_map_[task_id] = task_info; +} + +bool ParallelBuildManager::IsAllTaskFinish() const { + MS_LOG(INFO) << "wait process task_num: " << task_map_.size(); + return task_map_.empty(); +} + +std::pair ParallelBuildManager::TaskFinishProcess(int32_t task_id, bool set_kernel_mod) { + auto task_iter = task_map_.find(task_id); + if (task_iter == task_map_.end()) { + MS_EXCEPTION(ArgumentError) << "can find task_id:" << task_id; + } + auto json_name = task_iter->second.json_name; + auto processor = task_iter->second.processor; + auto kernel_pack = TbeUtils::InsertCache(json_name, processor); + if (kernel_pack == nullptr) { + if (set_kernel_mod) { + MS_EXCEPTION(ArgumentError) << "build kernel name:" << task_iter->second.json_name << " failed."; + } else { + MS_LOG(DEBUG) << "fusion build kernel name:" << task_iter->second.json_name << "failed."; + auto ret = std::make_pair(task_iter->second.scope_id, nullptr); + (void)task_map_.erase(task_iter); + return ret; + } + } + auto kernel_mod = GenKernelMod(json_name, processor, task_iter->second.input_size_list, + task_iter->second.output_size_list, kernel_pack); + MS_EXCEPTION_IF_NULL(kernel_mod); + if (set_kernel_mod) { + AnfAlgo ::SetKernelMod(kernel_mod, task_iter->second.node); + } + auto ret = std::make_pair(task_iter->second.scope_id, kernel_mod); + (void)task_map_.erase(task_iter); + MS_LOG(INFO) << "wait process remain task_num:" << task_map_.size(); + return ret; +} + +void ParallelBuildManager::SaveSameOpInfo(const mindspore::AnfNodePtr &anf_node, const std::string &json_name, + const std::vector &input_size_list, + const std::vector &output_size_list) { + struct KernelBuildTaskInfo task_info; + task_info.node = anf_node.get(); + task_info.json_name = json_name; + task_info.processor = tbe::GetProcessor(anf_node); + task_info.input_size_list.assign(input_size_list.begin(), input_size_list.end()); + task_info.output_size_list.assign(output_size_list.begin(), output_size_list.end()); + same_op_list_.push_back(task_info); +} + +bool ParallelBuildManager::GenSameOpKernelMod() const { + for (const auto &task_info : same_op_list_) { + bool ret = SearchInCache(task_info.json_name, task_info.processor, task_info.input_size_list, + task_info.output_size_list, task_info.node); + if (!ret) { + MS_LOG(DEBUG) << "can't find " << task_info.json_name << " in cache."; + return false; + } + } + return true; +} + +bool ParallelBuildManager::SearchInCache(const std::string &json_name, const std::string &processor, + const std::vector &input_size_list, + const std::vector &output_size_list, mindspore::AnfNode *node) const { + auto cached_kernel_pack = TbeUtils::SearchCache(json_name, processor); + if (cached_kernel_pack != nullptr) { + MS_LOG(INFO) << "Find cached kernel, kernel json name" << json_name; + auto kernel_mod_ptr = GenKernelMod(json_name, processor, input_size_list, output_size_list, cached_kernel_pack); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + AnfAlgo::SetKernelMod(kernel_mod_ptr, node); + return true; + } else { + return false; + } +} + +KernelModPtr ParallelBuildManager::GenKernelMod(const string &json_name, const string &processor, + const vector &input_size_list, + const vector &output_size_list, + const mindspore::kernel::KernelPackPtr &kernel_pack) const { + MS_EXCEPTION_IF_NULL(kernel_pack); + auto kernel_json_info = kernel_pack->kernel_json_info(); + auto kernel_mod_ptr = std::make_shared(kernel_pack); + MS_EXCEPTION_IF_NULL(kernel_mod_ptr); + kernel_mod_ptr->SetInputSizeList(input_size_list); + kernel_mod_ptr->SetOutputSizeList(output_size_list); + kernel_mod_ptr->SetWorkspaceSizeList(kernel_json_info.workspaces); + return kernel_mod_ptr; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h new file mode 100644 index 00000000000..45f56fdd0b1 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_PARALLEL_BUILD_H_ +#define MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_PARALLEL_BUILD_H_ + +#include +#include +#include +#include +#include "kernel/kernel.h" +#include "pybind11/stl.h" +#include +namespace mindspore { +namespace kernel { +bool TbeOpParallelBuild(std::vector anf_nodes); + +struct KernelBuildTaskInfo { + AnfNode *node; + std::string processor; + std::string json_name; + std::vector input_size_list; + std::vector output_size_list; + int32_t scope_id; +}; + +class ParallelBuildManager { + public: + ParallelBuildManager(); + ~ParallelBuildManager() = default; + int32_t StartCompileOp(const nlohmann::json &kernel_json) const; + void SaveTaskInfo(int32_t task_id, const AnfNodePtr &anf_node, const std::string &json_name, + const std::vector &input_size_list, const std::vector &output_size_list, + int32_t scope_id = 0); + void SaveSameOpInfo(const AnfNodePtr &anf_node, const std::string &json_name, + const std::vector &input_size_list, const std::vector &output_size_list); + bool GenSameOpKernelMod() const; + bool SearchInCache(const std::string &json_name, const std::string &processor, + const std::vector &input_size_list, const std::vector &output_size_list, + AnfNode *node) const; + + bool WaitOne(int *task_id, char **task_result) const; + bool IsAllTaskFinish() const; + std::pair TaskFinishProcess(int32_t task_id, bool set_kernel_mod = true); + KernelModPtr GenKernelMod(const string &json_name, const string &processor, const vector &input_size_list, + const vector &output_size_list, const KernelPackPtr &kernel_pack) const; + + private: + PyObject *tbe_parallel_compiler_; + std::map task_map_; + std::vector same_op_list_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_TBE_TBE_KERNEL_PARALLEL_BUILD_H_ diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select.cc new file mode 100644 index 00000000000..25495d1e687 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select.cc @@ -0,0 +1,546 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_kernel_select.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "kernel/oplib/oplib.h" +#include "kernel/tbe/tbe_kernel_build.h" +#include "nlohmann/json.hpp" +#include "common/utils.h" +#include "utils/context/ms_context.h" +#include "kernel/tbe/tbe_python_funcs.h" +#include "pre_activate/common/helper.h" +#include "kernel/tbe/tbe_convert_utils.h" + +namespace mindspore { +namespace kernel { +constexpr auto kName = "name"; +constexpr auto kDtype = "dtype"; +constexpr auto kFormat = "format"; +const std::map DYNAMIC_FORMAT_MAP = {{"NCHW", "DefaultFormat"}, + {"NHWC", "DefaultFormat"}, + {"ND", "DefaultFormat"}, + {"FRACTAL_Z", "FracZ"}, + {"NDHWC", "DefaultFormat"}}; +static const std::vector CHECK_SUPPORTED_OPTYPE{ + "MatMul", "BatchMatMul", "TopK", "InTopK", "Pack", "GatherNd", "UnsortedSegmentMinD", "UnsortedSegmentProdD", "Cast"}; + +bool CheckSupported(const AnfNodePtr &anf_node, const KernelBuildInfoPtr &select_kernel_build_info) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(select_kernel_build_info); + + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + auto iter = std::find(CHECK_SUPPORTED_OPTYPE.begin(), CHECK_SUPPORTED_OPTYPE.end(), op_name); + if (iter == CHECK_SUPPORTED_OPTYPE.end()) { + MS_LOG(DEBUG) << "Op " << op_name << "this op does not need to check op supported."; + return true; + } + + // replace kernel_info with current kernel info + auto ori_select_kernel_info = AnfAlgo::GetSelectKernelBuildInfo(anf_node); + AnfAlgo::SetSelectKernelBuildInfo(select_kernel_build_info, anf_node.get()); + + nlohmann::json kernel_json; + TbeKernelJsonCreator creator(CHECK_SUPPORTED); + bool ret = creator.GenTbeSingleKernelJson(anf_node, &kernel_json); + if (!ret) { + MS_LOG(DEBUG) << "GenTbeSingleKernelJson failed"; + AnfAlgo::SetSelectKernelBuildInfo(ori_select_kernel_info, anf_node.get()); + return false; + } + + ret = TbePythonFuncs::CheckSupported(kernel_json); + AnfAlgo::SetSelectKernelBuildInfo(ori_select_kernel_info, anf_node.get()); + return ret; +} + +bool CheckJsonItemValidity(const nlohmann::json &json_obj, const std::string &key_name, + const std::vector &keys) { + if (!json_obj[key_name].is_object()) { + MS_LOG(DEBUG) << key_name << "is not an object!"; + return false; + } + for (auto key : keys) { + if (json_obj[key_name].find(key) == json_obj[key_name].end()) { + MS_LOG(DEBUG) << "Key" << key << "of " << key_name << " is not found!"; + return false; + } + } + return true; +} + +std::vector SplitStr(const std::string &string, const std::string &sep) { + std::vector result; + size_t start = 0; + size_t index = string.find(sep, start); + std::string substr; + while (index != std::string::npos) { + if (string.size() > start) { + substr = string.substr(start, index - start); + } + (void)substr.erase(0, substr.find_first_not_of(' ')); + (void)substr.erase(substr.find_last_not_of(" ") + 1); + auto iter = DYNAMIC_FORMAT_MAP.find(substr); + if (iter != DYNAMIC_FORMAT_MAP.end()) { + substr = iter->second; + } + result.push_back(substr); + start = index + sep.size(); + index = string.find(sep, start); + } + + if (string.size() > start) { + substr = string.substr(start); + } + (void)substr.erase(0, substr.find_first_not_of(" ")); + (void)substr.erase(substr.find_last_not_of(" ") + 1); + auto iter = DYNAMIC_FORMAT_MAP.find(substr); + if (iter != DYNAMIC_FORMAT_MAP.end()) { + substr = iter->second; + } + result.push_back(substr); + return result; +} + +void ConvertFormatDtype(const std::string &format, const std::string &dtype, const std::shared_ptr io_info) { + MS_EXCEPTION_IF_NULL(io_info); + std::vector format_vec = SplitStr(format, ","); + std::vector dtype_vec = SplitStr(dtype, ","); + io_info->set_formats(format_vec); + io_info->set_dtypes(dtype_vec); +} + +bool ParseDynamicFormatJson(const std::string &jsonStr, std::vector> *const inputs, + std::vector> *const outputs) { + nlohmann::json json_obj = nlohmann::json::parse(jsonStr); + if (!json_obj.is_object()) { + MS_LOG(DEBUG) << "JsonStr is not an object, the jsonStr is:" << jsonStr; + return false; + } + std::vector keys = {kName, kDtype, kFormat}; + for (const auto &item : json_obj.items()) { + std::string key_name; + key_name = item.key(); + if (key_name.empty()) { + MS_LOG(DEBUG) << "Key name is empty!"; + return false; + } + if (!CheckJsonItemValidity(json_obj, key_name, keys)) { + return false; + } + if (key_name.find("input", 0) != std::string::npos) { + std::shared_ptr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->set_name(json_obj[key_name].at(kName)); + ConvertFormatDtype(json_obj[key_name].at(kFormat), json_obj[key_name].at(kDtype), input); + inputs->emplace_back(input); + } else if (key_name.find("output", 0) != std::string::npos) { + std::shared_ptr output = std::make_shared(); + MS_EXCEPTION_IF_NULL(output); + output->set_name(json_obj[key_name].at(kName)); + ConvertFormatDtype(json_obj[key_name].at(kFormat), json_obj[key_name].at(kDtype), output); + outputs->emplace_back(output); + } else { + MS_LOG(DEBUG) << "Key name:" << key_name << " is undefined!"; + return false; + } + } + return true; +} + +std::string OpSelectFormat(const shared_ptr &anf_node) { + nlohmann::json kernel_json; + std::string res_json_str; + TbeKernelJsonCreator creator(OP_SELECT_FORMAT); + bool ret = creator.GenTbeSingleKernelJson(anf_node, &kernel_json); + if (!ret) { + MS_LOG(DEBUG) << "GenTbeSingleKernelJson failed"; + return res_json_str; + } + res_json_str = TbePythonFuncs::OpSelectFormat(kernel_json); + MS_LOG(INFO) << "Dynamic select foramt response result:" << res_json_str; + return res_json_str; +} + +void SetTidyInputsInfo(const shared_ptr &anf_node, + const std::shared_ptr &builder, + const std::vector> &inputs) { + std::vector inputs_type; + std::vector inputs_format; + std::vector dyn_input_sizes; + size_t dyn_input_idx = 0; + size_t kernel_info_index = 0; + size_t real_input_num = AnfAlgo::GetInputTensorNum(anf_node); + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr("dyn_input_sizes") != nullptr) { + dyn_input_sizes = GetValue>(primitive->GetAttr("dyn_input_sizes")); + } + for (size_t i = 0; i < inputs.size(); i++) { + MS_EXCEPTION_IF_NULL(inputs[i]); + std::string param_type = inputs[i]->param_type(); + if (i >= real_input_num) { + MS_LOG(INFO) << "Input index:" << i << "is out of real_input_num:" << real_input_num; + continue; + } + auto type_id = AnfAlgo::GetPrevNodeOutputInferDataType(anf_node, i); + auto format = kOpFormat_DEFAULT; + if (param_type == "dynamic") { + if (!dyn_input_sizes.empty()) { + for (int t = 0; t < dyn_input_sizes[dyn_input_idx]; t++) { + kernel_info_index++; + inputs_type.emplace_back(type_id); + inputs_format.emplace_back(format); + } + dyn_input_idx++; + } + } else if (param_type == "required") { + kernel_info_index++; + inputs_type.emplace_back(type_id); + inputs_format.emplace_back(format); + } else { + if (kernel_info_index < real_input_num) { + MS_LOG(INFO) << "Input type is optional, input index is :" << kernel_info_index; + kernel_info_index++; + inputs_type.emplace_back(type_id); + inputs_format.emplace_back(format); + } + } + } + builder->SetInputsDeviceType(inputs_type); + builder->SetInputsFormat(inputs_format); +} + +void SetTidyOutputsInfo(const shared_ptr &anf_node, + const std::shared_ptr &builder, + const std::vector> &outputs) { + std::vector outputs_type; + std::vector outputs_format; + auto real_output_num = AnfAlgo::GetOutputTensorNum(anf_node); + size_t output_idx = 0; + for (const auto output : outputs) { + MS_EXCEPTION_IF_NULL(output); + if (output_idx >= real_output_num) { + continue; + } + size_t output_num = 0; + if (output->param_type() == "dynamic") { + if (outputs.size() > 1) { + MS_EXCEPTION(ArgumentError) << "Dynamic output is unsupported multi output!"; + } + output_num = real_output_num; + } else if (output->param_type() == "required") { + output_num = 1; + } else { + if (output_idx < real_output_num) { + MS_LOG(INFO) << "Set output kernel builder info, output type is optional, output index is :" << output_idx; + output_num = 1; + } + } + for (size_t i = 0; i < output_num; i++) { + auto type_id = AnfAlgo::GetOutputInferDataType(anf_node, output_idx); + outputs_type.emplace_back(type_id); + outputs_format.emplace_back(kOpFormat_DEFAULT); + output_idx++; + } + } + builder->SetOutputsDeviceType(outputs_type); + builder->SetOutputsFormat(outputs_format); +} + +void GenTidyKernelBuildInfo(const shared_ptr &anf_node, const std::vector> &inputs, + const std::vector> &outputs) { + auto builder_tmp = std::make_shared(); + builder_tmp->SetKernelType(TBE_KERNEL); + SetTidyInputsInfo(anf_node, builder_tmp, inputs); + SetTidyOutputsInfo(anf_node, builder_tmp, outputs); + AnfAlgo::SetSelectKernelBuildInfo(builder_tmp->Build(), anf_node.get()); +} + +void ReplaceByDynamicFormatDtype(const CNodePtr &kernel_node, const std::shared_ptr &op_info_ptr, + const std::shared_ptr op_info_new_ptr) { + std::vector> inputs_static = op_info_ptr->inputs_ptr(); + std::vector> outputs_static = op_info_ptr->outputs_ptr(); + std::vector> inputs_dyn; + std::vector> outputs_dyn; + if ((op_info_ptr->imply_type() == kTBE) && (!mindspore::opt::IsNopNode(kernel_node->cast()))) { + // 1. create tidy kernelBuildInfo in order to generate json for calling op_select_format + auto anf_node = kernel_node->cast>(); + auto kernel_build_info_ptr = AnfAlgo::GetSelectKernelBuildInfo(anf_node); + GenTidyKernelBuildInfo(kernel_node, inputs_static, outputs_static); + + // 2.get dynamic format from op_impl + std::string res_json_str; + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->execution_mode() != kPynativeMode) { + res_json_str = OpSelectFormat(kernel_node); + } + if (!res_json_str.empty()) { + (void)ParseDynamicFormatJson(res_json_str, &inputs_dyn, &outputs_dyn); + } + if (inputs_static.size() != inputs_dyn.size()) { + inputs_dyn.clear(); + } + if (outputs_static.size() != outputs_dyn.size()) { + outputs_dyn.clear(); + } + + // 3. resume kernel node's SelectKernelBuildInfo + // As it has been replaced by GenTidyKernelBuildInfo in order to call python func + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_ptr, anf_node.get()); + } + // 4.replace by dynamic format and dtype + if (inputs_dyn.empty() && outputs_dyn.empty()) { + MS_LOG(INFO) << "Dynamic select format response is empty, use static register info."; + op_info_new_ptr->set_inputs_ptr(inputs_static); + op_info_new_ptr->set_outputs_ptr(outputs_static); + } else { + MS_LOG(INFO) << "Dynamic select format response successful, use dynamic format."; + for (size_t i = 0; i < inputs_static.size(); i++) { + inputs_dyn[i]->set_param_type(inputs_static[i]->param_type()); + } + for (size_t j = 0; j < outputs_static.size(); j++) { + outputs_dyn[j]->set_param_type(outputs_static[j]->param_type()); + } + op_info_new_ptr->set_inputs_ptr(inputs_dyn); + op_info_new_ptr->set_outputs_ptr(outputs_dyn); + } + + // 5.copy other opinfo to new op_info_new + op_info_new_ptr->set_op_name(op_info_ptr->op_name()); + op_info_new_ptr->set_imply_type(op_info_ptr->imply_type()); + op_info_new_ptr->set_fusion_type(op_info_ptr->fusion_type()); +} + +bool SetKernelBuilderInputInfo(const std::vector> &inputs, size_t real_input_num, + size_t builder_idex, const std::vector &dyn_input_sizes, + const std::shared_ptr &builder) { + MS_EXCEPTION_IF_NULL(builder); + + std::vector inputs_device_type; + std::vector inputs_format; + size_t dyn_input_idx = 0; + size_t kernel_info_index = 0; + MS_EXCEPTION_IF_NULL(inputs[0]); + size_t kernel_info_cnt = inputs[0]->dtypes().size(); + + for (const auto &input : inputs) { + MS_EXCEPTION_IF_NULL(input); + std::string param_type = input->param_type(); + std::vector dtypes = input->dtypes(); + std::vector formats = input->formats(); + if (dtypes.size() != kernel_info_cnt || formats.size() != kernel_info_cnt) { + MS_LOG(ERROR) << "Set input kernel builder info, dtyps size != formats size."; + return false; + } + + if (param_type == "dynamic") { + if (dyn_input_sizes.empty()) { + MS_LOG(ERROR) << "Set input kernel builder info, dyn_input_sizes's size is 0 when param_type is dynamic"; + return false; + } + + for (int t = 0; t < dyn_input_sizes[dyn_input_idx]; t++) { + kernel_info_index++; + auto type_id = tbe::DtypeToTypeId(dtypes[builder_idex]); + inputs_device_type.push_back(type_id); + inputs_format.push_back(formats[builder_idex]); + } + dyn_input_idx++; + } else if (param_type == "required") { + kernel_info_index++; + auto type_id = tbe::DtypeToTypeId(dtypes[builder_idex]); + inputs_device_type.push_back(type_id); + inputs_format.push_back(formats[builder_idex]); + } else { + if (kernel_info_index < real_input_num) { + MS_LOG(INFO) << "Set input kernel builder info, input type is optional, input index is " << kernel_info_index; + kernel_info_index++; + auto type_id = tbe::DtypeToTypeId(dtypes[builder_idex]); + inputs_device_type.push_back(type_id); + inputs_format.push_back(formats[builder_idex]); + } + } + } + + builder->SetInputsDeviceType(inputs_device_type); + builder->SetInputsFormat(inputs_format); + return true; +} + +bool SetKernelBuilderOutputInfo(const std::vector> &outputs, size_t builder_idex, + const size_t &real_output_num, + const std::shared_ptr &builder) { + // not now but in the next we need to support dynamic output case + MS_EXCEPTION_IF_NULL(builder); + + size_t output_idx = 0; + std::vector outputs_device_type; + std::vector outputs_format; + MS_EXCEPTION_IF_NULL(outputs[0]); + size_t kernel_info_cnt = outputs[0]->dtypes().size(); + + for (const auto &output : outputs) { + MS_EXCEPTION_IF_NULL(output); + if (output_idx >= real_output_num) { + MS_LOG(WARNING) << "real_output_num: " << real_output_num << ", output_idx: " << output_idx << "is out of limit!"; + continue; + } + size_t output_num = 0; + if (output->param_type() == "dynamic") { + if (outputs.size() > 1) { + MS_LOG(EXCEPTION) << "Dynamic output is unsupported multi output!"; + } + output_num = real_output_num; + } else if (output->param_type() == "required") { + output_num = 1; + } else { + if (output_idx < real_output_num) { + MS_LOG(INFO) << "Set output kernel builder info, output type is optional, output index is " << output_idx; + output_num = 1; + } + } + + for (size_t i = 0; i < output_num; i++) { + std::vector dtypes = output->dtypes(); + std::vector formats = output->formats(); + if (dtypes.size() != kernel_info_cnt || formats.size() != kernel_info_cnt) { + MS_LOG(ERROR) << "Set output kernel builder info, dtyps size != formats size."; + return false; + } + auto type_id = tbe::DtypeToTypeId(dtypes[builder_idex]); + outputs_device_type.push_back(type_id); + outputs_format.push_back(formats[builder_idex]); + output_idx++; + } + } + + builder->SetOutputsFormat(outputs_format); + builder->SetOutputsDeviceType(outputs_device_type); + return true; +} + +void SetKernelBuildCommonInfo(const std::shared_ptr &builder, + Processor processor, const std::shared_ptr &op_info_ptr) { + MS_EXCEPTION_IF_NULL(builder); + MS_EXCEPTION_IF_NULL(op_info_ptr); + + builder->SetProcessor(processor); + std::string fusion_type = op_info_ptr->fusion_type(); + if (tbe::GetFusionType(fusion_type) != UNKNOWN_FUSION_TYPE) { + builder->SetFusionType(tbe::GetFusionType(fusion_type)); + } + builder->SetKernelType(TBE_KERNEL); +} + +bool ParseMetadata(const CNodePtr &kernel_node, const std::shared_ptr &op_info_ptr, + std::vector> *const kernel_info_list) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_info_list); + size_t real_input_num = AnfAlgo::GetInputTensorNum(kernel_node); + size_t real_output_num = AnfAlgo::GetOutputTensorNum(kernel_node); + std::vector> inputs = op_info_ptr->inputs_ptr(); + std::vector> outputs = op_info_ptr->outputs_ptr(); + std::vector dyn_input_sizes; + auto primitive = AnfAlgo::GetCNodePrimitive(kernel_node); + MS_EXCEPTION_IF_NULL(primitive); + if (primitive->GetAttr("dyn_input_sizes") != nullptr) { + dyn_input_sizes = GetValue>(primitive->GetAttr("dyn_input_sizes")); + } + if (inputs.size() > 0) { + MS_EXCEPTION_IF_NULL(inputs[0]); + size_t kernel_info_cnt = inputs[0]->dtypes().size(); + for (size_t j = 0; j < kernel_info_cnt; j++) { + auto builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(builder); + SetKernelBuildCommonInfo(builder, Processor::AICORE, op_info_ptr); + + if (!SetKernelBuilderInputInfo(inputs, real_input_num, j, dyn_input_sizes, builder)) { + MS_LOG(ERROR) << "Parse kernel metadata, set inputs kernel builder info failed."; + return false; + } + + if (outputs.size() > 0) { + if (!SetKernelBuilderOutputInfo(outputs, j, real_output_num, builder)) { + MS_LOG(ERROR) << "Parse kernel metadata, set outputs kernel builder info failed."; + return false; + } + } + + kernel_info_list->push_back(builder->Build()); + } + } else if (outputs.size() > 0) { + MS_EXCEPTION_IF_NULL(outputs[0]); + size_t kernel_info_cnt = outputs[0]->dtypes().size(); + for (size_t j = 0; j < kernel_info_cnt; j++) { + auto builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(builder); + SetKernelBuildCommonInfo(builder, Processor::AICORE, op_info_ptr); + + if (!SetKernelBuilderOutputInfo(outputs, j, real_output_num, builder)) { + MS_LOG(ERROR) << "Parse kernel metadata, set outputs kernel builder info failed."; + return false; + } + + kernel_info_list->push_back(builder->Build()); + } + } + return true; +} + +void TbeMetadataInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(kernel_info_list); + std::vector> parse_info_list; + std::string op_name = AnfAlgo::GetCNodeName(kernel_node); + auto op_info_ptr = mindspore::kernel::OpLib::FindOp(op_name, OpImplyType::kTBE); + if (op_info_ptr == nullptr) { + return; + } + // dynamic get op format and dtype and replace opinfo + auto op_info_new_ptr = std::make_shared(); + ReplaceByDynamicFormatDtype(kernel_node, op_info_ptr, op_info_new_ptr); + + if (!ParseMetadata(kernel_node, op_info_new_ptr, &parse_info_list)) { + MS_LOG(INFO) << "Tbe parsed metadata of op[" << op_name << "] failed."; + return; + } + + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + for (auto parse_info : parse_info_list) { + if (context_ptr->execution_mode() == kPynativeMode) { + kernel_info_list->push_back(parse_info); + } else { + if (CheckSupported(kernel_node, parse_info)) { + kernel_info_list->push_back(parse_info); + } else { + MS_LOG(INFO) << "CheckSupported Failed for TBE op" << op_name << " kernel info."; + } + } + } + if (kernel_info_list->empty()) { + MS_LOG(DEBUG) << "Tbe dose not has metadata of op[" << op_name << "]."; + } +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select.h b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select.h new file mode 100644 index 00000000000..4c85468f15a --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select.h @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_TBE_KERNEL_SELECT_H +#define MINDSPORE_TBE_KERNEL_SELECT_H + +#include +#include +#include +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace kernel { +void TbeMetadataInfo(const CNodePtr &kernel_node, std::vector> *kernel_info_list); +bool CheckSupported(const AnfNodePtr &anf_node, const KernelBuildInfoPtr &select_kernel_build_info); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_TBE_KERNEL_SELECT_H diff --git a/mindspore/ccsrc/kernel/tbe/tbe_python_funcs.cc b/mindspore/ccsrc/kernel/tbe/tbe_python_funcs.cc new file mode 100644 index 00000000000..1e60742fc40 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_python_funcs.cc @@ -0,0 +1,182 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_python_funcs.h" +#include "kernel/tbe/tbe_utils.h" +#include "common/utils.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace kernel { +using mindspore::kernel::tbe::TbeUtils; +constexpr auto kTbeProcessModule = "mindspore._extends.parallel_compile.tbe_compiler.tbe_process"; +constexpr auto kCreateTbeParallelCompilerFunc = "create_tbe_parallel_compiler"; +constexpr auto kOpSelectFormatFunc = "op_select_format"; +constexpr auto kCheckSupportedFunc = "check_supported"; + +PyObject *TbePythonFuncs::pCreateTbeParallelCompilerFunc_ = nullptr; +PyObject *TbePythonFuncs::pTbeCompiler_ = nullptr; +PyObject *TbePythonFuncs::pOpSelectFormatFunc_ = nullptr; +PyObject *TbePythonFuncs::pCheckSupportedFunc_ = nullptr; +bool TbePythonFuncs::Init() { + static bool initialized = false; + if (initialized) { + return true; + } + // Initialize cache + TbeUtils::LoadCache(); + + // tbe_process + PyObject *pTbeProcessModule = nullptr; + pTbeProcessModule = PyImport_ImportModule(kTbeProcessModule); + if (pTbeProcessModule == nullptr) { + MS_LOG(ERROR) << "Failed to import [" << kTbeProcessModule << "] module."; + return false; + } + + pCreateTbeParallelCompilerFunc_ = PyObject_GetAttrString(pTbeProcessModule, kCreateTbeParallelCompilerFunc); + if (pCreateTbeParallelCompilerFunc_ == nullptr) { + MS_LOG(ERROR) << "Failed to transform opModule and FuncName to PyObject, opModule:[" << kTbeProcessModule + << "], FuncName:[" << kCreateTbeParallelCompilerFunc << "]."; + return false; + } + + pTbeCompiler_ = PyEval_CallObject(pCreateTbeParallelCompilerFunc_, nullptr); + if (pTbeCompiler_ == nullptr) { + PyErr_Print(); + MS_EXCEPTION(ArgumentError) << "Failed to call function : create_parallel_compiler."; + return false; + } + + pOpSelectFormatFunc_ = PyObject_GetAttrString(pTbeProcessModule, kOpSelectFormatFunc); + if (pOpSelectFormatFunc_ == nullptr) { + MS_LOG(ERROR) << "Failed to transform opModule and FuncName to PyObject, opModule:[" << kTbeProcessModule + << "], FuncName:[" << kOpSelectFormatFunc << "]."; + return false; + } + + pCheckSupportedFunc_ = PyObject_GetAttrString(pTbeProcessModule, kCheckSupportedFunc); + if (pCheckSupportedFunc_ == nullptr) { + MS_LOG(ERROR) << "Failed to transform opModule and FuncName to PyObject, opModule:[" << kTbeProcessModule + << "], FuncName:[" << kCheckSupportedFunc << "]."; + return false; + } + initialized = true; + MS_LOG(INFO) << "TbePythonFuncs initialized Success."; + return true; +} + +std::string TbePythonFuncs::PyObjectToStr(PyObject *PyObj) { + char *pChar = nullptr; + std::string str_res; + if (PyObj == nullptr) { + MS_LOG(ERROR) << "Input parameter is nullptr."; + return str_res; + } + PyObject *strArgs = PyObject_Str(PyObj); + if (strArgs != nullptr) { + (void)PyArg_Parse(strArgs, "s", &pChar); + } + if (pChar == nullptr) { + MS_LOG(ERROR) << "pChar is nullptr."; + return str_res; + } + str_res = pChar; + return str_res; +} + +std::string TbePythonFuncs::OpSelectFormat(const nlohmann::json &kernel_json) { + PyObject *pArg = nullptr; + PyObject *pRet = nullptr; + std::string res_json_str; + + if (!Init()) { + MS_LOG(ERROR) << "TbePythonFuncs Initialize Failed !"; + return res_json_str; + } + + // assembly Args + pArg = PyTuple_New(1); + std::string json_str = kernel_json.dump(); + (void)PyTuple_SetItem(pArg, 0, Py_BuildValue("s", json_str.c_str())); + if (pArg == nullptr) { + MS_LOG(ERROR) << "Failed to generate parameter from kernel_json to PyObject."; + return res_json_str; + } + + // call functions + if (pOpSelectFormatFunc_ == nullptr) { + MS_LOG(ERROR) << "function is nullptr."; + return res_json_str; + } + + pRet = PyEval_CallObject(pOpSelectFormatFunc_, pArg); + if (pRet == nullptr) { + PyErr_Print(); + MS_EXCEPTION(ArgumentError) << "Failed to call function [" << kOpSelectFormatFunc + << "], function args:" << PyObjectToStr(pArg); + } + + char *pstr = nullptr; + (void)PyArg_Parse(pRet, "s", &pstr); + res_json_str = pstr; + return res_json_str; +} + +bool TbePythonFuncs::CheckSupported(const nlohmann::json &kernel_json) { + PyObject *pArg = nullptr; + PyObject *pRes = nullptr; + bool ret = false; + + if (!Init()) { + MS_LOG(ERROR) << "TbePythonFuncs Initialize Failed !"; + return ret; + } + // assembly Args + pArg = PyTuple_New(1); + std::string json_str = kernel_json.dump(); + PyObject *arg1 = Py_BuildValue("s", json_str.c_str()); + (void)PyTuple_SetItem(pArg, 0, arg1); + if (pArg == nullptr) { + MS_LOG(ERROR) << "Failed to generate parameter from kernel_json to PyObject."; + return ret; + } + + // call functions + if (pCheckSupportedFunc_ == nullptr) { + MS_LOG(ERROR) << "function is nullptr."; + return ret; + } + + pRes = PyEval_CallObject(pCheckSupportedFunc_, pArg); + if (pRes == nullptr) { + PyErr_Print(); + MS_EXCEPTION(ArgumentError) << "Failed to call function [" << kCheckSupportedFunc + << "], function args: " << PyObjectToStr(pArg); + } + ret = PyObject_IsTrue(pRes) != 0; + return ret; +} + +PyObject *TbePythonFuncs::TbeParallelCompiler() { + if (!Init()) { + MS_LOG(ERROR) << "TbePythonFuncs Initialize Failed !"; + return nullptr; + } + return pTbeCompiler_; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_python_funcs.h b/mindspore/ccsrc/kernel/tbe/tbe_python_funcs.h new file mode 100644 index 00000000000..4e1746475c5 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_python_funcs.h @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_TBE_PYTHON_FUNCS_H_ +#define MINDSPORE_CCSRC_KERNEL_TBE_TBE_PYTHON_FUNCS_H_ + +#include +#include +#include "pybind11/stl.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace kernel { +class TbePythonFuncs { + public: + TbePythonFuncs() = default; + ~TbePythonFuncs() = default; + static std::string OpSelectFormat(const nlohmann::json &kernel_json); + static bool CheckSupported(const nlohmann::json &kernel_json); + static PyObject *TbeParallelCompiler(); + + private: + static bool Init(); + static std::string PyObjectToStr(_object *PyObj); + static PyObject *pCreateTbeParallelCompilerFunc_; + static PyObject *pTbeCompiler_; + static PyObject *pOpSelectFormatFunc_; + static PyObject *pCheckSupportedFunc_; +}; +} // namespace kernel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_KERNEL_TBE_TBE_PYTHON_FUNCS_H_ diff --git a/mindspore/ccsrc/kernel/tbe/tbe_utils.cc b/mindspore/ccsrc/kernel/tbe/tbe_utils.cc new file mode 100644 index 00000000000..ab29ca69bbe --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_utils.cc @@ -0,0 +1,252 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/tbe/tbe_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel/oplib/oplib.h" +#include "utils/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" +#include "device/kernel_info.h" +#include "ir/dtype/type.h" +#include "kernel/tbe/tbe_convert_utils.h" +#include "securec/include/securec.h" +#include "operator/ops.h" + +namespace mindspore { +namespace kernel { +namespace tbe { +constexpr auto kCceKernelMeta = "./kernel_meta/"; +constexpr auto kJsonSuffix = ".json"; +constexpr auto kInfoSuffix = ".info"; + +uintptr_t KernelManager::kernel_stub_gen_ = 0; +std::unordered_map KernelManager::info_table_ = {}; + +void TbeUtils::SaveJsonInfo(const std::string &json_name, const std::string &info) { + char real_path[PATH_MAX] = {0}; + std::string path = kCceKernelMeta + json_name + kInfoSuffix; + if (path.size() > PATH_MAX) { + MS_LOG(ERROR) << "file path: " << path << "is too long."; + return; + } + std::ifstream fin(path); + if (fin) { + MS_LOG(INFO) << "json file exist, no need to create."; + return; + } + std::ofstream filewrite; + filewrite.open(path); + if (!filewrite.is_open()) { + return; + } + filewrite << info << std::endl; + filewrite.close(); + if (nullptr == realpath(path.c_str(), real_path)) { + MS_LOG(DEBUG) << "dir: " << path << "does not exit."; + return; + } + MS_LOG(INFO) << "real path is: " << real_path; + if (chmod(real_path, S_IRUSR) == -1) { + MS_LOG(DEBUG) << "modify file: " << real_path << "to read only fail."; + } +} + +void TbeUtils::LoadCache() { + static bool has_load = false; + if (!has_load) { + KernelMeta *bin_map = KernelMeta::GetInstance(); + if (bin_map != nullptr && !bin_map->ReadIndex(kCceKernelMeta)) { + MS_LOG(INFO) << "Cache initialize failed[" << kCceKernelMeta << "]"; + } else { + MS_LOG(INFO) << "Cache initialize to " << kCceKernelMeta; + } + has_load = true; + } +} + +KernelPackPtr TbeUtils::SearchCache(const std::string &kernel_name, const std::string &processor) { + // search cache. + KernelMeta *bin_map = KernelMeta::GetInstance(); + if (bin_map == nullptr) { + MS_LOG(DEBUG) << "kernel cache is invalid."; + return nullptr; + } + return bin_map->GetKernelPack(kernel_name, processor); +} + +KernelPackPtr TbeUtils::InsertCache(const std::string &kernel_name, const std::string &processor) { + MS_LOG(INFO) << "kernel name: " << kernel_name << ", processr:" << processor; + if (processor != kProcessorAiCore) { + MS_LOG(EXCEPTION) << "process type should be aicore, actually is: " << processor; + } + return SearchCache(kernel_name, processor); +} + +int KernelManager::BinaryRegister(const mindspore::kernel::FlexArray &kernel_buffer, void **module, + const string &magic) { + static std::map magic_maps = {{"RT_DEV_BINARY_MAGIC_ELF", RT_DEV_BINARY_MAGIC_ELF}, + {"RT_DEV_BINARY_MAGIC_PLAIN", RT_DEV_BINARY_MAGIC_PLAIN}, + {"RT_DEV_BINARY_MAGIC_PLAIN_AICPU", RT_DEV_BINARY_MAGIC_PLAIN_AICPU}, + {"RT_DEV_BINARY_MAGIC_ELF_AICPU", RT_DEV_BINARY_MAGIC_ELF_AICPU}}; + // object for device register. + rtDevBinary_t dev_bin; + dev_bin.data = kernel_buffer.contents; + auto iter = magic_maps.find(magic); + if (iter == magic_maps.end()) { + MS_LOG(DEBUG) << "Invalid magic number: " << magic; + return -1; + } + dev_bin.magic = iter->second; + dev_bin.length = kernel_buffer.len; + dev_bin.version = 2; + if (RT_ERROR_NONE != rtDevBinaryRegister(&dev_bin, module)) { + MS_LOG(DEBUG) << "Call runtime rtDevBinaryRegister error."; + return -1; + } + return 0; +} + +uintptr_t KernelManager::GenFuncStub(const mindspore::kernel::KernelPack &kernel_pack, bool force_reload, + uint32_t *block_dim) { + auto kernel = kernel_pack.GetKernel(); + if (kernel == nullptr) { + MS_LOG(EXCEPTION) << "Invalid kernel pack, json or kernel is nullptr."; + } + auto kernel_contents = kernel->contents; + if (kernel_contents == nullptr) { + MS_LOG(EXCEPTION) << "Invalid kernel context, json or kernel is nullptr."; + } + auto kernel_json_info = kernel_pack.kernel_json_info(); + + *block_dim = kernel_json_info.block_dim; + string funcname = kernel_json_info.kernel_name; + string magic = kernel_json_info.magic; + + if (!force_reload) { + // use the cached object. + auto iter = info_table_.find(funcname); + if (iter != info_table_.end()) { + auto kernelmeta = iter->second; + *block_dim = kernelmeta->block_dim_; + return kernelmeta->func_stub_; + } + } + void *module = nullptr; + if (0 != BinaryRegister((*kernel_pack.GetKernel()), &module, magic)) { + MS_LOG(DEBUG) << "Call runtime BinaryRegister error."; + return 0; + } + // to diff different funcs. + uintptr_t funcstub = ++kernel_stub_gen_; + if (RT_ERROR_NONE != + rtFunctionRegister(module, reinterpret_cast(funcstub), funcname.c_str(), funcname.c_str(), 0)) { + MS_LOG(DEBUG) << "Call runtime rtFunctionRegister error."; + return 0; + } + // cache the registered kernelmeta. + info_table_[funcname] = std::make_shared(KernelMetaInfo{funcstub, *block_dim}); + return funcstub; +} + +std::string KernelManager::GetStubFuncName(const KernelPackPtr &kernel_pack) { + auto kernel_json_info = kernel_pack->kernel_json_info(); + return kernel_json_info.kernel_name; +} + +KernelMeta *KernelMeta::GetInstance() { + static KernelMeta inst; + return &inst; +} + +bool KernelMeta::ReadIndex(const std::string &bin_dir) { + DIR *dir = opendir(bin_dir.c_str()); + if (dir == nullptr) { + auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU); + if (ret != 0) { + MS_LOG(INFO) << "kernel dir: " << bin_dir << "not exist"; + return false; + } + dir = opendir(bin_dir.c_str()); + } + struct dirent *entry; + while ((entry = readdir(dir)) != nullptr) { + string bin_dir_tmp = bin_dir; + std::string cce_json = entry->d_name; + if (cce_json.length() <= 5) { + continue; + } + std::string suffix = cce_json.substr(cce_json.length() - 5); + if (suffix != kJsonSuffix) { + continue; + } + auto sp = cce_json.rfind('/'); + if (sp != std::string::npos) { + continue; + } + sp = cce_json.rfind('.'); + if (sp == std::string::npos) { + continue; + } + auto kernel_name = cce_json.substr(0, sp); + (void)bin_dir_tmp.append("/"); + (void)bin_dir_tmp.append(cce_json); + kernel_index_map_[kernel_name] = bin_dir_tmp; + } + (void)closedir(dir); + + MS_LOG(INFO) << "Cache kernel initialized, kernel size: " << kernel_index_map_.size(); + return true; +} + +KernelPackPtr KernelMeta::GetKernelPack(const std::string &kernel_name, const std::string &processor) { + KernelPackPtr ret = nullptr; + // 1. pack has been created + auto kernel_pack_iter = kernel_pack_map_.find(kernel_name); + if (kernel_pack_iter != kernel_pack_map_.end()) { + MS_LOG(INFO) << "kernel pack [" << kernel_name << "]has been created."; + ret = kernel_pack_iter->second; + } else { + // 2. kernel file has been create, but pack does not been created. + std::string cce_json = kCceKernelMeta; + (void)cce_json.append(kernel_name).append(kJsonSuffix); + ret = std::make_shared(); + if (!ret->LoadKernelMeta(cce_json, processor)) { + MS_LOG(DEBUG) << "Read cache json and bin file failed[" << cce_json << "]"; + return nullptr; + } + kernel_pack_map_[kernel_name] = ret; + auto iter = kernel_index_map_.find(kernel_name); + if (iter == kernel_index_map_.end()) { + MS_LOG(INFO) << "kernel name [" << kernel_name << "] has been ceated first."; + kernel_index_map_[kernel_name] = cce_json; + } + } + return ret; +} +} // namespace tbe +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/tbe/tbe_utils.h b/mindspore/ccsrc/kernel/tbe/tbe_utils.h new file mode 100644 index 00000000000..56fbe7967a4 --- /dev/null +++ b/mindspore/ccsrc/kernel/tbe/tbe_utils.h @@ -0,0 +1,86 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_TBE_TBE_UTILS_H_ +#define MINDSPORE_CCSRC_KERNEL_TBE_TBE_UTILS_H_ +#include +#include +#include +#include +#include +#include + +#include "session/kernel_graph.h" +#include "ir/anf.h" +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { +namespace tbe { +using std::string; +using std::vector; + +class TbeUtils { + public: + TbeUtils() = default; + + ~TbeUtils() = default; + + static void SaveJsonInfo(const std::string &json_name, const std::string &info); + + static void LoadCache(); + + static KernelPackPtr SearchCache(const std::string &kernel_name, const std::string &processor); + + static KernelPackPtr InsertCache(const std::string &kernel_name, const std::string &processor); +}; + +struct KernelMetaInfo { + uintptr_t func_stub_; + uint32_t block_dim_; +}; +using KernelMetaPtr = std::shared_ptr; + +class KernelManager { + public: + static uintptr_t GenFuncStub(const KernelPack &kernel_pack, bool force_reload, uint32_t *block_dim); + static std::string GetStubFuncName(const KernelPackPtr &kernel_pack); + + private: + KernelManager() = default; + ~KernelManager() = default; + static int BinaryRegister(const FlexArray &kernel_buffer, void **module, const string &magic); + static std::unordered_map info_table_; + static uintptr_t kernel_stub_gen_; +}; + +class KernelMeta { + public: + static KernelMeta *GetInstance(); + bool ReadIndex(const std::string &bin_dir); + KernelPackPtr GetKernelPack(const std::string &kernel_name, const std::string &processor); + + private: + KernelMeta() = default; + ~KernelMeta() = default; + std::unordered_map kernel_index_map_{}; + std::unordered_map kernel_pack_map_{}; +}; +} // namespace tbe +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_TBE_TBE_UTILS_H_ diff --git a/mindspore/ccsrc/mindrecord/CMakeLists.txt b/mindspore/ccsrc/mindrecord/CMakeLists.txt new file mode 100644 index 00000000000..eb1c1fb591f --- /dev/null +++ b/mindspore/ccsrc/mindrecord/CMakeLists.txt @@ -0,0 +1,33 @@ +include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc) + +# This set up makes the source code more portable. +include_directories(${PYTHON_INCLUDE_DIRS}) + +# source directory +aux_source_directory(io DIR_LIB_SRCS) +aux_source_directory(meta DIR_LIB_SRCS) +aux_source_directory(common DIR_LIB_SRCS) + +# set(CMAKE_CXX_COMPILER "g++") +# set(CMAKE_CXX_FLAGS "-Wall -fvisibility=hidden") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") + +# add shared link library +add_library(_c_mindrecord SHARED ${DIR_LIB_SRCS}) + +set_target_properties(_c_mindrecord PROPERTIES + PREFIX "${PYTHON_MODULE_PREFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}" + ) + +# add link library +target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar protobuf::libprotobuf) + +if (USE_GLOG) + target_link_libraries(_c_mindrecord PRIVATE mindspore::glog) +endif() diff --git a/mindspore/ccsrc/mindrecord/common/shard_pybind.cc b/mindspore/ccsrc/mindrecord/common/shard_pybind.cc new file mode 100644 index 00000000000..338a17ac2de --- /dev/null +++ b/mindspore/ccsrc/mindrecord/common/shard_pybind.cc @@ -0,0 +1,231 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "common/utils.h" +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_index_generator.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/shard_segment.h" +#include "mindrecord/include/shard_writer.h" +#include "nlohmann/json.hpp" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "utils/log_adapter.h" + +namespace py = pybind11; + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +void BindSchema(py::module *m) { + (void)py::class_>(*m, "Schema", py::module_local()) + .def_static("build", (std::shared_ptr(*)(std::string, py::handle)) & Schema::Build) + .def("get_desc", &Schema::get_desc) + .def("get_schema_content", (py::object(Schema::*)()) & Schema::GetSchemaForPython) + .def("get_blob_fields", &Schema::get_blob_fields) + .def("get_schema_id", &Schema::get_schema_id); +} + +void BindStatistics(const py::module *m) { + (void)py::class_>(*m, "Statistics", py::module_local()) + .def_static("build", (std::shared_ptr(*)(std::string, py::handle)) & Statistics::Build) + .def("get_desc", &Statistics::get_desc) + .def("get_statistics", (py::object(Statistics::*)()) & Statistics::GetStatisticsForPython) + .def("get_statistics_id", &Statistics::get_statistics_id); +} + +void BindShardHeader(const py::module *m) { + (void)py::class_>(*m, "ShardHeader", py::module_local()) + .def(py::init<>()) + .def("add_schema", &ShardHeader::AddSchema) + .def("add_statistics", &ShardHeader::AddStatistic) + .def("add_index_fields", + (MSRStatus(ShardHeader::*)(const std::vector &)) & ShardHeader::AddIndexFields) + .def("get_meta", &ShardHeader::get_schemas) + .def("get_statistics", &ShardHeader::get_statistics) + .def("get_fields", &ShardHeader::get_fields) + .def("get_schema_by_id", &ShardHeader::GetSchemaByID) + .def("get_statistic_by_id", &ShardHeader::GetStatisticByID); +} + +void BindShardWriter(py::module *m) { + (void)py::class_(*m, "ShardWriter", py::module_local()) + .def(py::init<>()) + .def("open", &ShardWriter::Open) + .def("open_for_append", &ShardWriter::OpenForAppend) + .def("set_header_size", &ShardWriter::set_header_size) + .def("set_page_size", &ShardWriter::set_page_size) + .def("set_shard_header", &ShardWriter::SetShardHeader) + .def("write_raw_data", + (MSRStatus(ShardWriter::*)(std::map> &, vector> &, bool)) & + ShardWriter::WriteRawData) + .def("write_raw_nlp_data", (MSRStatus(ShardWriter::*)(std::map> &, + std::map> &, bool)) & + ShardWriter::WriteRawData) + .def("commit", &ShardWriter::Commit); +} + +void BindShardReader(const py::module *m) { + (void)py::class_>(*m, "ShardReader", py::module_local()) + .def(py::init<>()) + .def("open", (MSRStatus(ShardReader::*)(const std::string &, const int &, const std::vector &, + const std::vector> &)) & + ShardReader::OpenPy) + .def("launch", &ShardReader::Launch) + .def("get_header", &ShardReader::get_shard_header) + .def("get_blob_fields", &ShardReader::get_blob_fields) + .def("get_next", + (std::vector, pybind11::object>>(ShardReader::*)()) & ShardReader::GetNextPy) + .def("finish", &ShardReader::Finish) + .def("close", &ShardReader::Close); +} + +void BindShardIndexGenerator(const py::module *m) { + (void)py::class_(*m, "ShardIndexGenerator", py::module_local()) + .def(py::init()) + .def("build", &ShardIndexGenerator::Build) + .def("write_to_db", &ShardIndexGenerator::WriteToDatabase); +} + +void BindShardSegment(py::module *m) { + (void)py::class_(*m, "ShardSegment", py::module_local()) + .def(py::init<>()) + .def("open", (MSRStatus(ShardSegment::*)(const std::string &, const int &, const std::vector &, + const std::vector> &)) & + ShardSegment::OpenPy) + .def("get_category_fields", + (std::pair>(ShardSegment::*)()) & ShardSegment::GetCategoryFields) + .def("set_category_field", (MSRStatus(ShardSegment::*)(std::string)) & ShardSegment::SetCategoryField) + .def("read_category_info", (std::pair(ShardSegment::*)()) & ShardSegment::ReadCategoryInfo) + .def("read_at_page_by_id", (std::pair, pybind11::object>>>( + ShardSegment::*)(int64_t, int64_t, int64_t)) & + ShardSegment::ReadAtPageByIdPy) + .def("read_at_page_by_name", (std::pair, pybind11::object>>>( + ShardSegment::*)(std::string, int64_t, int64_t)) & + ShardSegment::ReadAtPageByNamePy) + .def("get_header", &ShardSegment::get_shard_header) + .def("get_blob_fields", + (std::pair>(ShardSegment::*)()) & ShardSegment::get_blob_fields); +} + +void BindGlobalParams(py::module *m) { + (*m).attr("MIN_HEADER_SIZE") = kMinHeaderSize; + (*m).attr("MAX_HEADER_SIZE") = kMaxHeaderSize; + (*m).attr("MIN_PAGE_SIZE") = kMinPageSize; + (*m).attr("MAX_PAGE_SIZE") = kMaxPageSize; + (*m).attr("MIN_SHARD_COUNT") = kMinShardCount; + (*m).attr("MAX_SHARD_COUNT") = kMaxShardCount; + (*m).attr("MIN_CONSUMER_COUNT") = kMinConsumerCount; + (void)(*m).def("get_max_thread_num", &GetMaxThreadNum); +} + +PYBIND11_MODULE(_c_mindrecord, m) { + m.doc() = "pybind11 mindrecord plugin"; // optional module docstring + (void)py::enum_(m, "MSRStatus", py::module_local()) + .value("SUCCESS", SUCCESS) + .value("FAILED", FAILED) + .export_values(); + (void)py::enum_(m, "ShardType", py::module_local()).value("NLP", kNLP).value("CV", kCV).export_values(); + BindGlobalParams(&m); + BindSchema(&m); + BindStatistics(&m); + BindShardHeader(&m); + BindShardWriter(&m); + BindShardReader(&m); + BindShardIndexGenerator(&m); + BindShardSegment(&m); +} +} // namespace mindrecord +} // namespace mindspore + +namespace nlohmann { +namespace detail { +py::object FromJsonImpl(const json &j) { + if (j.is_null()) { + return py::none(); + } else if (j.is_boolean()) { + return py::bool_(j.get()); + } else if (j.is_number()) { + double number = j.get(); + if (fabs(number - std::floor(number)) < mindspore::mindrecord::kEpsilon) { + return py::int_(j.get()); + } else { + return py::float_(number); + } + } else if (j.is_string()) { + return py::str(j.get()); + } else if (j.is_array()) { + py::list obj; + for (const auto &el : j) { + (void)obj.attr("append")(FromJsonImpl(el)); + } + return std::move(obj); + } else { + py::dict obj; + for (json::const_iterator it = j.cbegin(); it != j.cend(); ++it) { + obj[py::str(it.key())] = FromJsonImpl(it.value()); + } + return std::move(obj); + } +} + +json ToJsonImpl(const py::handle &obj) { + if (obj.is_none()) { + return nullptr; + } + if (py::isinstance(obj)) { + return obj.cast(); + } + if (py::isinstance(obj)) { + return obj.cast(); + } + if (py::isinstance(obj)) { + return obj.cast(); + } + if (py::isinstance(obj)) { + return obj.cast(); + } + if (py::isinstance(obj) || py::isinstance(obj)) { + auto out = json::array(); + for (const py::handle &value : obj) { + out.push_back(ToJsonImpl(value)); + } + return out; + } + if (py::isinstance(obj)) { + auto out = json::object(); + for (const py::handle &key : obj) { + out[py::str(key).cast()] = ToJsonImpl(obj[key]); + } + return out; + } + MS_LOG(ERROR) << "Python to json failed, obj is: " << py::cast(obj); + return json(); +} +} // namespace detail + +py::object adl_serializer::FromJson(const json &j) { return detail::FromJsonImpl(j); } + +void adl_serializer::ToJson(json *j, const py::object &obj) { + *j = detail::ToJsonImpl(obj); +} // namespace detail +} // namespace nlohmann diff --git a/mindspore/ccsrc/mindrecord/common/shard_utils.cc b/mindspore/ccsrc/mindrecord/common/shard_utils.cc new file mode 100644 index 00000000000..ca4bb8a261b --- /dev/null +++ b/mindspore/ccsrc/mindrecord/common/shard_utils.cc @@ -0,0 +1,180 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/common/shard_utils.h" +#include "common/utils.h" +#include "./securec.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::DEBUG; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +// split a string using a character +std::vector StringSplit(const std::string &field, char separator) { + std::vector res; + uint64_t s_pos = 0; + while (s_pos < field.length()) { + size_t e_pos = field.find_first_of(separator, s_pos); + if (e_pos != std::string::npos) { + res.push_back(field.substr(s_pos, e_pos - s_pos)); + } else { + res.push_back(field.substr(s_pos, field.length() - s_pos)); + break; + } + s_pos = e_pos + 1; + } + return std::move(res); +} + +bool ValidateFieldName(const std::string &str) { + std::string::const_iterator it = str.begin(); + if (it == str.end()) { + return false; + } + for (; it != str.end(); ++it) { + if (*it == '_' || ((*it >= '0') && (*it <= '9')) || ((*it >= 'A') && (*it <= 'Z')) || + ((*it >= 'a') && (*it <= 'z'))) { + continue; + } + return false; + } + return true; +} + +std::pair GetFileName(const std::string &path) { + char real_path[PATH_MAX] = {0}; + char buf[PATH_MAX] = {0}; + if (strncpy_s(buf, PATH_MAX, common::SafeCStr(path), path.length()) != EOK) { + MS_LOG(ERROR) << "Securec func [strncpy_s] failed, path: " << path; + return {FAILED, ""}; + } + char tmp[PATH_MAX] = {0}; + if (realpath(dirname(&(buf[0])), tmp) == nullptr) { + MS_LOG(ERROR) << "Invalid file path, path: " << buf; + return {FAILED, ""}; + } + if (realpath(common::SafeCStr(path), real_path) == nullptr) { + MS_LOG(DEBUG) << "Path: " << path << "check successfully"; + } + std::string s = real_path; + char sep = '/'; + size_t i = s.rfind(sep, s.length()); + if (i != std::string::npos) { + if (i + 1 < s.size()) { + return {SUCCESS, s.substr(i + 1)}; + } + } + return {SUCCESS, s}; +} + +std::pair GetParentDir(const std::string &path) { + char real_path[PATH_MAX] = {0}; + char buf[PATH_MAX] = {0}; + if (strncpy_s(buf, PATH_MAX, common::SafeCStr(path), path.length()) != EOK) { + MS_LOG(ERROR) << "Securec func [strncpy_s] failed, path: " << path; + return {FAILED, ""}; + } + char tmp[PATH_MAX] = {0}; + if (realpath(dirname(&(buf[0])), tmp) == nullptr) { + MS_LOG(ERROR) << "Invalid file path, path: " << buf; + return {FAILED, ""}; + } + if (realpath(common::SafeCStr(path), real_path) == nullptr) { + MS_LOG(DEBUG) << "Path: " << path << "check successfully"; + } + std::string s = real_path; + if (s.rfind('/') + 1 <= s.size()) { + return {SUCCESS, s.substr(0, s.rfind('/') + 1)}; + } + return {SUCCESS, "/"}; +} + +bool CheckIsValidUtf8(const std::string &str) { + int n = 0; + int ix = str.length(); + for (int i = 0; i < ix; ++i) { + uint8_t c = static_cast(str[i]); + if (c <= 0x7f) { + n = 0; + } else if ((c & 0xE0) == 0xC0) { + n = 1; + } else if (c == 0xed && i < (ix - 1) && (static_cast(str[i + 1]) & 0xa0) == 0xa0) { + return false; + } else if ((c & 0xF0) == 0xE0) { + n = 2; + } else if ((c & 0xF8) == 0xF0) { + n = 3; + } else { + return false; + } + for (int j = 0; j < n && i < ix; ++j) { + if ((++i == ix) || ((static_cast(str[i]) & 0xC0) != 0x80)) { + return false; + } + } + } + return true; +} + +bool IsLegalFile(const std::string &path) { + struct stat s; + if (stat(common::SafeCStr(path), &s) == 0) { + if (s.st_mode & S_IFDIR) { + return false; + } + return true; + } + return false; +} + +std::pair GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) { + uint64_t ll_count = 0; + struct statfs disk_info; + if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) { + MS_LOG(ERROR) << "Get disk size error"; + return {FAILED, 0}; + } + + switch (disk_type) { + case kTotalSize: + ll_count = disk_info.f_bsize * disk_info.f_blocks; + ll_count = ll_count >> 20; + break; + case kFreeSize: + ll_count = disk_info.f_bsize * disk_info.f_bavail; + ll_count = ll_count >> 20; + break; + default: + ll_count = 0; + break; + } + + return {SUCCESS, ll_count}; +} + +uint32_t GetMaxThreadNum() { + // define the number of thread + uint32_t thread_num = std::thread::hardware_concurrency(); + if (thread_num == 0) { + thread_num = kMaxConsumerCount; + } + return thread_num; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/include/common/shard_pybind.h b/mindspore/ccsrc/mindrecord/include/common/shard_pybind.h new file mode 100644 index 00000000000..86c71a0ea73 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/common/shard_pybind.h @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ +#define MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ + +#include +#include +#include "mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; +namespace nlohmann { +template <> +struct adl_serializer { + py::object FromJson(const json &j); + + void ToJson(json *j, const py::object &obj); +}; + +namespace detail { +py::object FromJsonImpl(const json &j); + +json ToJsonImpl(const py::handle &obj); +} // namespace detail +} // namespace nlohmann +#endif // MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ diff --git a/mindspore/ccsrc/mindrecord/include/common/shard_utils.h b/mindspore/ccsrc/mindrecord/include/common/shard_utils.h new file mode 100644 index 00000000000..c452b49fbca --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/common/shard_utils.h @@ -0,0 +1,162 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ +#define MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mindrecord/include/shard_error.h" +#include "nlohmann/json.hpp" +#include "./sqlite3.h" +#include "utils/log_adapter.h" + +/* To be used when dlog is ok #include "./slog.h" */ +#ifdef DEBUG +#define MS_ASSERT(f) assert(f) +#else +#define MS_ASSERT(f) ((void)0) +#endif + +namespace mindspore { +namespace mindrecord { +using json = nlohmann::json; + +const int kInt0 = 0; +const int kInt1 = 1; +const int kInt2 = 2; +const int kInt3 = 3; +const int kUnsignedInt4 = 4; + +enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel }; + +enum ShardType { + kNLP = 0, + kCV = 1, +}; + +const double kEpsilon = 1e-7; + +const int kThreadNumber = 14; + +// Shard default parameters +const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB +const uint64_t kDefaultPageSize = 1 << 25; // 32MB + +// HeaderSize [16KB, 128MB] +const int kMinHeaderSize = 1 << 14; // 16KB +const int kMaxHeaderSize = 1 << 27; // 128MB + +// PageSize [32KB, 256MB] +const int kMinPageSize = 1 << 15; // 32KB +const int kMaxPageSize = 1 << 28; // 256MB + +// used by value length / schema id length / statistic id length ... +const uint64_t kInt64Len = 8; + +// Minimum file size +const uint64_t kMinFileSize = kInt64Len; + +const int kMinShardCount = 1; +const int kMaxShardCount = 1000; + +const int kMinConsumerCount = 1; +const int kMaxConsumerCount = 128; + +const int kMaxSchemaCount = 1; +const int kMaxThreadCount = 32; +const int kMaxFieldCount = 100; + +// Minimum free disk size +const int kMinFreeDiskSize = 10; // 10M + +// dummy json +const json kDummyId = R"({"id": 0})"_json; + +// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) +const std::unordered_map kDbJsonMap = { + {"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"}, + {"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"}, + {"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"}, + {"bytes", "BLOB"}}; + +const char kPoint = '.'; + +// field type used by check schema validation +const std::set kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"}; + +/// \brief split a string using a character +/// \param[in] field target string +/// \param[in] separator a character for spliting +/// \return vector type result +std::vector StringSplit(const std::string &field, char separator); + +/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' +/// \param[in] str target string +/// \return +bool ValidateFieldName(const std::string &str); + +/// \brief get the filename by the path +/// \param s file path +/// \return +std::pair GetFileName(const std::string &s); + +/// \brief get parent dir +/// \param path file path +/// \return parent path +std::pair GetParentDir(const std::string &path); + +bool CheckIsValidUtf8(const std::string &str); + +/// \brief judge if a path is legal file +/// \param path file path +/// \return parent path +bool IsLegalFile(const std::string &path); + +enum DiskSizeType { kTotalSize = 0, kFreeSize }; + +/// \brief get the free space about the disk +/// \param str_dir file path +/// \param disk_type: kTotalSize / kFreeSize +/// \return size in Megabytes +std::pair GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type); + +/// \brief get the max hardware concurrency +/// \return max concurrency +uint32_t GetMaxThreadNum(); +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_category.h b/mindspore/ccsrc/mindrecord/include/shard_category.h new file mode 100644 index 00000000000..08e5ac9c2e9 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_category.h @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_CATEGORY_H_ +#define MINDRECORD_INCLUDE_SHARD_CATEGORY_H_ + +#include +#include +#include +#include "mindrecord/include/shard_operator.h" + +namespace mindspore { +namespace mindrecord { +class ShardCategory : public ShardOperator { + public: + explicit ShardCategory(const std::vector> &categories); + + ~ShardCategory() override{}; + + const std::vector> &get_categories() const; + + MSRStatus operator()(ShardTask &tasks) override; + + private: + std::vector> categories_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_CATEGORY_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_error.h b/mindspore/ccsrc/mindrecord/include/shard_error.h new file mode 100644 index 00000000000..026ee836e35 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_error.h @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_ERROR_H_ +#define MINDRECORD_INCLUDE_SHARD_ERROR_H_ + +#include +#include "utils/error_code.h" + +namespace mindspore { +namespace mindrecord { +DE_ERRORNO_MINDRECORD(OPEN_FILE_FAILED, 0, "open file failed"); +DE_ERRORNO_MINDRECORD(CLOSE_FILE_FAILED, 1, "close file failed"); +DE_ERRORNO_MINDRECORD(WRITE_METADATA_FAILED, 2, "write metadata failed"); +DE_ERRORNO_MINDRECORD(WRITE_RAWDATA_FAILED, 3, "write rawdata failed"); +DE_ERRORNO_MINDRECORD(GET_SCHEMA_FAILED, 4, "get schema failed"); +DE_ERRORNO_MINDRECORD(ILLEGAL_RAWDATA, 5, "illegal raw data"); +DE_ERRORNO_MINDRECORD(PYTHON_TO_JSON_FAILED, 6, "pybind: python object to json failed"); +DE_ERRORNO_MINDRECORD(DIR_CREATE_FAILED, 7, "directory create failed"); +DE_ERRORNO_MINDRECORD(OPEN_DIR_FAILED, 8, "open directory failed"); +DE_ERRORNO_MINDRECORD(INVALID_STATISTICS, 9, "invalid statistics object"); +DE_ERRORNO_MINDRECORD(OPEN_DATABASE_FAILED, 10, "open database failed"); +DE_ERRORNO_MINDRECORD(CLOSE_DATABASE_FAILED, 11, "close database failed"); +DE_ERRORNO_MINDRECORD(DATABASE_OPERATE_FAILED, 12, "database operate failed"); +DE_ERRORNO_MINDRECORD(BUILD_SCHEMA_FAILED, 13, "build schema failed"); +DE_ERRORNO_MINDRECORD(DIVISOR_IS_ILLEGAL, 14, "divisor is illegal"); +DE_ERRORNO_MINDRECORD(INVALID_FILE_PATH, 15, "file path is invalid"); +DE_ERRORNO_MINDRECORD(SECURE_FUNC_FAILED, 16, "secure function failed"); +DE_ERRORNO_MINDRECORD(ALLOCATE_MEM_FAILED, 17, "allocate memory failed"); +DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_NAME, 18, "illegal field name"); +DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_TYPE, 19, "illegal field type"); +DE_ERRORNO_MINDRECORD(SET_METADATA_FAILED, 20, "set metadata failed"); +DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_DEFINITION, 21, "illegal schema definition"); +DE_ERRORNO_MINDRECORD(ILLEGAL_COLUMN_LIST, 22, "illegal column list"); +DE_ERRORNO_MINDRECORD(SQL_ERROR, 23, "sql error"); +DE_ERRORNO_MINDRECORD(ILLEGAL_SHARD_COUNT, 24, "illegal shard count"); +DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_COUNT, 25, "illegal schema count"); +DE_ERRORNO_MINDRECORD(VERSION_ERROR, 26, "data version is not matched"); +DE_ERRORNO_MINDRECORD(ADD_SCHEMA_FAILED, 27, "add schema failed"); +DE_ERRORNO_MINDRECORD(ILLEGAL_Header_SIZE, 28, "illegal header size"); +DE_ERRORNO_MINDRECORD(ILLEGAL_Page_SIZE, 29, "illegal page size"); +DE_ERRORNO_MINDRECORD(ILLEGAL_SIZE_VALUE, 30, "illegal size value"); +DE_ERRORNO_MINDRECORD(INDEX_FIELD_FAILED, 31, "add index fields failed"); +DE_ERRORNO_MINDRECORD(GET_CANDIDATE_CATEGORYFIELDS_FAILED, 32, "get candidate categoryFields failed"); +DE_ERRORNO_MINDRECORD(GET_CATEGORY_INFO, 33, "get category information failed"); +DE_ERRORNO_MINDRECORD(ILLEGAL_CATEGORY_ID, 34, "illegal category id"); +DE_ERRORNO_MINDRECORD(ILLEGAL_ROWNUMBER_OF_PAGE, 35, "illegal row number of page"); +DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_ID, 36, "illegal schema id"); +DE_ERRORNO_MINDRECORD(DESERIALIZE_SCHEMA_FAILED, 37, "deserialize schema failed"); +DE_ERRORNO_MINDRECORD(DESERIALIZE_STATISTICS_FAILED, 38, "deserialize statistics failed"); +DE_ERRORNO_MINDRECORD(ILLEGAL_DB_FILE, 39, "illegal db file."); +DE_ERRORNO_MINDRECORD(OVERWRITE_DB_FILE, 40, "overwrite db file."); +DE_ERRORNO_MINDRECORD(OVERWRITE_MINDRECORD_FILE, 41, "overwrite mindrecord file."); +DE_ERRORNO_MINDRECORD(ILLEGAL_MINDRECORD_FILE, 42, "illegal mindrecord file."); +DE_ERRORNO_MINDRECORD(PARSE_JSON_FAILED, 43, "parse json failed."); +DE_ERRORNO_MINDRECORD(ILLEGAL_PARAMETERS, 44, "illegal parameters."); +DE_ERRORNO_MINDRECORD(GET_PAGE_BY_GROUP_ID_FAILED, 46, "get page by group id failed."); +DE_ERRORNO_MINDRECORD(GET_SYSTEM_STATE_FAILED, 47, "get system state failed."); +DE_ERRORNO_MINDRECORD(IO_FAILED, 48, "io operate failed."); + +enum MSRStatus { + SUCCESS = 0, + FAILED = 1, +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_ERROR_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_header.h b/mindspore/ccsrc/mindrecord/include/shard_header.h new file mode 100644 index 00000000000..ca4d3bd66fa --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_header.h @@ -0,0 +1,184 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_HEADER_H_ +#define MINDRECORD_INCLUDE_SHARD_HEADER_H_ + +#include +#include +#include +#include +#include +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_index.h" +#include "mindrecord/include/shard_page.h" +#include "mindrecord/include/shard_schema.h" +#include "mindrecord/include/shard_statistics.h" + +namespace mindspore { +namespace mindrecord { +class ShardHeader { + public: + ShardHeader(); + + MSRStatus Build(const std::string &file_path); + + ~ShardHeader() = default; + + /// \brief add the schema and save it + /// \param[in] schema the schema needs to be added + /// \return the last schema's id + int AddSchema(std::shared_ptr schema); + + /// \brief add the statistic and save it + /// \param[in] statistic the statistic needs to be added + /// \return the last statistic's id + void AddStatistic(std::shared_ptr statistic); + + /// \brief create index and add fields which from schema for each schema + /// \param[in] fields the index fields needs to be added + /// \return SUCCESS if add successfully, FAILED if not + MSRStatus AddIndexFields(std::vector> fields); + + MSRStatus AddIndexFields(const std::vector &fields); + + /// \brief get the schema + /// \return the schema + std::vector> get_schemas(); + + /// \brief get Statistics + /// \return the Statistic + std::vector> get_statistics(); + + /// \brief get the fields of the index + /// \return the fields of the index + std::vector> get_fields(); + + /// \brief get the index + /// \return the index + std::shared_ptr get_index(); + + /// \brief get the schema by schemaid + /// \param[in] schemaId the id of schema needs to be got + /// \return the schema obtained by schemaId + std::pair, MSRStatus> GetSchemaByID(int64_t schema_id); + + /// \brief get the filepath to shard by shardID + /// \param[in] shardID the id of shard which filepath needs to be obtained + /// \return the filepath obtained by shardID + std::string get_shard_address_by_id(int64_t shard_id); + + /// \brief get the statistic by statistic id + /// \param[in] statisticId the id of statistic needs to be get + /// \return the statistics obtained by statistic id + std::pair, MSRStatus> GetStatisticByID(int64_t statistic_id); + + MSRStatus InitByFiles(const std::vector &file_paths); + + void set_index(Index index) { index_ = std::make_shared(index); } + + std::pair, MSRStatus> GetPage(const int &shard_id, const int &page_id); + + MSRStatus SetPage(const std::shared_ptr &new_page); + + MSRStatus AddPage(const std::shared_ptr &new_page); + + int64_t GetLastPageId(const int &shard_id); + + int GetLastPageIdByType(const int &shard_id, const std::string &page_type); + + const std::pair> GetPageByGroupId(const int &group_id, const int &shard_id); + + std::vector get_shard_addresses() const { return shard_addresses_; } + + int get_shard_count() const { return shard_count_; } + + int get_schema_count() const { return schema_.size(); } + + uint64_t get_header_size() const { return header_size_; } + + uint64_t get_page_size() const { return page_size_; } + + void set_header_size(const uint64_t &header_size) { header_size_ = header_size; } + + void set_page_size(const uint64_t &page_size) { page_size_ = page_size; } + + const string get_version() { return version_; } + + std::vector SerializeHeader(); + + private: + MSRStatus InitializeHeader(const std::vector &headers); + + /// \brief get the headers from all the shard data + /// \param[in] the shard data real path + /// \param[in] the headers which readed from the shard data + /// \return SUCCESS/FAILED + MSRStatus get_headers(const vector &real_addresses, std::vector &headers); + + MSRStatus ValidateField(const std::vector &field_name, json schema, const uint64_t &schema_id); + + /// \brief check the binary file status + MSRStatus CheckFileStatus(const std::string &path); + + std::pair ValidateHeader(const std::string &path); + + void ParseHeader(const json &header); + + void GetHeadersOneTask(int start, int end, std::vector &headers, const vector &realAddresses); + + MSRStatus ParseIndexFields(const json &index_fields); + + MSRStatus CheckIndexField(const std::string &field, const json &schema); + + void ParsePage(const json &page); + + MSRStatus ParseStatistics(const json &statistics); + + MSRStatus ParseSchema(const json &schema); + + void ParseShardAddress(const json &address); + + std::string SerializeIndexFields(); + + std::vector SerializePage(); + + std::string SerializeStatistics(); + + std::string SerializeSchema(); + + std::string SerializeShardAddress(); + + std::shared_ptr InitIndexPtr(); + + MSRStatus GetAllSchemaID(std::set &bucket_count); + + uint32_t shard_count_; + uint64_t header_size_; + uint64_t page_size_; + string version_ = "2.0"; + + std::shared_ptr index_; + std::vector shard_addresses_; + std::vector> schema_; + std::vector> statistics_; + std::vector>> pages_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_HEADER_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_index.h b/mindspore/ccsrc/mindrecord/include/shard_index.h new file mode 100644 index 00000000000..6d4bc364578 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_index.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INDEX_H +#define MINDRECORD_INDEX_H +#pragma once + +#include +#include +#include +#include +#include +#include +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_schema.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +using std::cin; +using std::endl; +using std::pair; +using std::string; +using std::vector; + +class Index { + public: + Index(); + + ~Index() {} + + /// \brief Add field which from schema according to schemaId + /// \param[in] schemaId the id of schema to be added + /// \param[in] field the field need to be added + /// + /// add the field to the fields_ vector + void AddIndexField(const int64_t &schemaId, const std::string &field); + + /// \brief get stored fields + /// \return fields stored + std::vector > get_fields(); + + private: + std::vector > fields_; + string database_name_; + string table_name_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INDEX_H diff --git a/mindspore/ccsrc/mindrecord/include/shard_index_generator.h b/mindspore/ccsrc/mindrecord/include/shard_index_generator.h new file mode 100644 index 00000000000..f59dbe9bf00 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_index_generator.h @@ -0,0 +1,116 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_ +#define MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "mindrecord/include/shard_header.h" +#include "./sqlite3.h" + +namespace mindspore { +namespace mindrecord { +using INDEX_FIELDS = std::pair>>; +using ROW_DATA = std::pair>>>; +class ShardIndexGenerator { + public: + explicit ShardIndexGenerator(const std::string &file_path, bool append = false); + + MSRStatus Build(); + + static std::pair GenerateFieldName(const std::pair &field); + + ~ShardIndexGenerator() {} + + /// \brief fetch value in json by field path + /// \param[in] field_path + /// \param[in] schema + /// \return the vector of value + static std::vector GetField(const std::string &field_path, json schema); + + /// \brief fetch field type in schema n by field path + /// \param[in] field_path + /// \param[in] schema + /// \return the type of field + static std::string TakeFieldType(const std::string &field_path, json schema); + + /// \brief create databases for indexes + MSRStatus WriteToDatabase(); + + private: + static int Callback(void *not_used, int argc, char **argv, char **az_col_name); + + static MSRStatus ExecuteSQL(const std::string &statement, sqlite3 *db, const string &success_msg = ""); + + static std::string ConvertJsonToSQL(const std::string &json); + + std::pair CreateDatabase(int shard_no); + + std::pair> GetSchemaDetails(const std::vector &schema_lens, std::fstream &in); + + static std::pair GenerateRawSQL(const std::vector> &fields); + + std::pair CheckDatabase(const std::string &shard_address); + + /// + /// \param shard_no + /// \param blob_id_to_page_id + /// \param raw_page_id + /// \param in + /// \return field name, db type, field value + ROW_DATA GenerateRowData(int shard_no, const std::map &blob_id_to_page_id, int raw_page_id, + std::fstream &in); + /// + /// \param db + /// \param sql + /// \param data + /// \return + MSRStatus BindParamaterExecuteSQL( + sqlite3 *db, const std::string &sql, + const std::vector>> &data); + + INDEX_FIELDS GenerateIndexFields(const std::vector &schema_detail); + + MSRStatus ExcuteTransaction(const int &shard_no, const std::pair &db, + const std::vector &raw_page_ids, const std::map &blob_id_to_page_id); + + MSRStatus CreateShardNameTable(sqlite3 *db, const std::string &shard_name); + + MSRStatus AddBlobPageInfo(std::vector> &row_data, + const std::shared_ptr cur_blob_page, uint64_t &cur_blob_page_offset, + std::fstream &in); + + void AddIndexFieldByRawData(const std::vector &schema_detail, + std::vector> &row_data); + + std::string file_path_; + bool append_; + ShardHeader shard_header_; + uint64_t page_size_; + uint64_t header_size_; + int schema_count_; + std::vector> fields_; +}; +} // namespace mindrecord +} // namespace mindspore +#endif // MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_operator.h b/mindspore/ccsrc/mindrecord/include/shard_operator.h new file mode 100644 index 00000000000..9d00fb7628d --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_operator.h @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_OPERATOR_H_ +#define MINDRECORD_INCLUDE_SHARD_OPERATOR_H_ + +#include "mindrecord/include/shard_task.h" + +namespace mindspore { +namespace mindrecord { +class ShardOperator { + public: + virtual ~ShardOperator() = default; + virtual MSRStatus operator()(ShardTask &tasks) = 0; +}; +} // namespace mindrecord +} // namespace mindspore +#endif // MINDRECORD_INCLUDE_SHARD_OPERATOR_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_page.h b/mindspore/ccsrc/mindrecord/include/shard_page.h new file mode 100644 index 00000000000..8b7a5244bd0 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_page.h @@ -0,0 +1,106 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_PAGE_H_ +#define MINDRECORD_INCLUDE_SHARD_PAGE_H_ + +#include +#include +#include +#include +#include +#include +#include "mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +const std::string kPageTypeRaw = "RAW_DATA"; +const std::string kPageTypeBlob = "BLOB_DATA"; +const std::string kPageTypeNewColumn = "NEW_COLUMN_DATA"; + +class Page { + public: + Page(const int &page_id, const int &shard_id, const std::string &page_type, const int &page_type_id, + const uint64_t &start_row_id, const uint64_t end_row_id, + const std::vector> &row_group_ids, const uint64_t page_size) + : page_id_(page_id), + shard_id_(shard_id), + page_type_(page_type), + page_type_id_(page_type_id), + start_row_id_(start_row_id), + end_row_id_(end_row_id), + row_group_ids_(row_group_ids), + page_size_(page_size) {} + + ~Page() = default; + + /// \brief get the page and its description + /// \return the json format of the page and its description + json GetPage() const; + + int get_page_id() const { return page_id_; } + + int get_shard_id() const { return shard_id_; } + + int get_page_type_id() const { return page_type_id_; } + + std::string get_page_type() const { return page_type_; } + + uint64_t get_page_size() const { return page_size_; } + + uint64_t get_start_row_id() const { return start_row_id_; } + + uint64_t get_end_row_id() const { return end_row_id_; } + + void set_end_row_id(const uint64_t &end_row_id) { end_row_id_ = end_row_id; } + + void set_page_size(const uint64_t &page_size) { page_size_ = page_size; } + + std::pair get_last_row_group_id() const { return row_group_ids_.back(); } + + std::vector> get_row_group_ids() const { return row_group_ids_; } + + void set_row_group_ids(const std::vector> &last_row_group_ids) { + row_group_ids_ = last_row_group_ids; + } + + void DeleteLastGroupId(); + + private: + int page_id_; + int shard_id_; + std::string page_type_; + int page_type_id_; + uint64_t start_row_id_; + uint64_t end_row_id_; + std::vector> row_group_ids_; + uint64_t page_size_; + // JSON page: { + // "page_id":X, + // "shard_id":X, + // "page_type":"XXX", (enum "raw_data", "blob_data", "new_column") + // "page_type_id":X, + // "start_row_id":X, + // "end_row_id":X, + // "row_group_ids":[{"id":X, "offset":X}], + // "page_size":X, +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_PAGE_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_reader.h b/mindspore/ccsrc/mindrecord/include/shard_reader.h new file mode 100644 index 00000000000..c114b17951d --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_reader.h @@ -0,0 +1,335 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_READER_H_ +#define MINDRECORD_INCLUDE_SHARD_READER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_category.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_index_generator.h" +#include "mindrecord/include/shard_operator.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/shard_sample.h" +#include "mindrecord/include/shard_shuffle.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +using ROW_GROUPS = + std::tuple>>, std::vector>>; +using ROW_GROUP_BRIEF = + std::tuple>, std::vector>; +using TASK_RETURN_CONTENT = std::pair, json>>>; +const int kNumBatchInMap = 1000; // iterator buffer size in row-reader mode +const int kNumPageInBuffer = 16; // page buffer size in block-reader mode + +class ShardReader { + public: + ShardReader(); + + virtual ~ShardReader(); + + /// \brief open files and initialize reader, c++ API + /// \param[in] file_path the path of ONE file, any file in dataset is fine + /// \param[in] n_consumer number of threads when reading + /// \param[in] selected_columns column list to be populated + /// \param[in] operators operators applied to data, operator type is shuffle, sample or category + /// \param[in] block_reader block-reader mode if true, otherwise row-reader mode + /// \return MSRStatus the status of MSRStatus + MSRStatus Open(const std::string &file_path, int n_consumer = 4, + const std::vector &selected_columns = {}, + const std::vector> &operators = {}, const bool &block_reader = false); + + /// \brief open files and initialize reader, python API + /// \param[in] file_path the path of ONE file, any file in dataset is fine + /// \param[in] n_consumer number of threads when reading + /// \param[in] selected_columns column list to be populated + /// \param[in] operators operators applied to data, operator type is shuffle, sample or category + /// \return MSRStatus the status of MSRStatus + MSRStatus OpenPy(const std::string &file_path, const int &n_consumer = 4, + const std::vector &selected_columns = {}, + const std::vector> &operators = {}); + + /// \brief close reader + /// \return null + void Close(); + + /// \brief read the file, get schema meta,statistics and index, single-thread mode + /// \return MSRStatus the status of MSRStatus + MSRStatus Open(); + + /// \brief read the file, get schema meta,statistics and index, multiple-thread mode + /// \return MSRStatus the status of MSRStatus + MSRStatus Open(int n_consumer); + + /// \brief launch threads to get batches + /// \param[in] is_simple_reader trigger threads if false; do nothing if true + /// \return MSRStatus the status of MSRStatus + MSRStatus Launch(bool is_simple_reader = false); + + /// \brief aim to get the meta data + /// \return the metadata + std::shared_ptr get_shard_header() const; + + /// \brief get the number of shards + /// \return # of shards + int get_shard_count() const; + + /// \brief get the number of rows in database + /// \param[in] file_path the path of ONE file, any file in dataset is fine + /// \param[out] count # of rows + /// \return MSRStatus the status of MSRStatus + MSRStatus CountTotalRows(const std::string &file_path, int64_t *count); + + /// \brief shuffle task with incremental seed + /// \return void + void ShuffleTask(); + + /// \brief get the number of rows in database + /// \return # of rows + int get_num_rows() const; + + /// \brief Read the summary of row groups + /// \return the tuple of 4 elements + /// 1. Sharding ID + /// 2. Row group ID + /// 3. The row ID started in row group + /// 4. # of rows in row group + std::vector> ReadRowGroupSummary(); + + /// \brief Read 1 row group data, excluding images + /// \param[in] groupID row group ID + /// \param[in] shard_id sharding ID + /// \param[in] columns multi-columns retrieved + /// \return the tuple of 5 elements + /// 1. file name where row group is located + /// 2. Actual row group size + /// 3. Offset address of row group in file + /// 4. The list of image offset in page [startOffset, endOffset) + /// 5. The list of columns data + ROW_GROUP_BRIEF ReadRowGroupBrief(int group_id, int shard_id, + const std::vector &columns = std::vector()); + + /// \brief Read 1 row group data, excluding images, following an index field criteria + /// \param[in] groupID row group ID + /// \param[in] shard_id sharding ID + /// \param[in] column-value pair of criteria to fulfill + /// \param[in] columns multi-columns retrieved + /// \return the tuple of 5 elements + /// 1. file name where row group is located + /// 2. Actual row group size + /// 3. Offset address of row group in file + /// 4. The list of image offset in page [startOffset, endOffset) + /// 5. The list of columns data + ROW_GROUP_BRIEF ReadRowGroupCriteria(int group_id, int shard_id, const std::pair &criteria, + const std::vector &columns = std::vector()); + + /// \brief join all created threads + /// \return MSRStatus the status of MSRStatus + MSRStatus Finish(); + + /// \brief return a batch, given that one is ready + /// \return a batch of images and image data + std::vector, json>> GetNext(); + + /// \brief return a row by id + /// \return a batch of images and image data + std::vector, json>> GetNextById(const int64_t &task_id, const int32_t &consumer_id); + + /// \brief return a batch in block-reader mode, given that one is ready + /// \return a batch of images and image data + std::vector, json>> GetBlockNext(); + + /// \brief return a batch, given that one is ready, python API + /// \return a batch of images and image data + std::vector, pybind11::object>> GetNextPy(); + + /// \brief get blob filed list + /// \return blob field list + std::pair> get_blob_fields(); + + /// \brief reset reader + /// \return null + void Reset(); + + /// \brief set flag of all-in-index + /// \return null + void set_all_in_index(bool all_in_index) { all_in_index_ = all_in_index; } + + /// \brief get NLP flag + bool get_nlp_flag(); + + protected: + /// \brief sqlite call back function + static int SelectCallback(void *p_data, int num_fields, char **p_fields, char **p_col_names); + + private: + /// \brief wrap up labels to json format + MSRStatus ConvertLabelToJson(const std::vector> &labels, std::shared_ptr fs, + std::vector>> &offsets, int shard_id, + const std::vector &columns, std::vector> &column_values); + + /// \brief read all rows for specified columns + ROW_GROUPS ReadAllRowGroup(std::vector &columns); + + /// \brief read all rows in one shard + MSRStatus ReadAllRowsInShard(int shard_id, const std::string &sql, const std::vector &columns, + std::vector>> &offsets, + std::vector> &column_values); + + /// \brief initialize reader + MSRStatus Init(const std::string &file_path); + + /// \brief validate column list + MSRStatus CheckColumnList(const std::vector &selected_columns); + + /// \brief populate one row by task list in row-reader mode + MSRStatus ConsumerByRow(int consumer_id); + + /// \brief populate one row by task list in block-reader mode + MSRStatus ConsumerByBlock(int consumer_id); + + /// \brief get offset address of images within page + std::vector> GetImageOffset(int group_id, int shard_id, + const std::pair &criteria = {"", ""}); + + /// \brief execute sqlite query with prepare statement + MSRStatus QueryWithCriteria(sqlite3 *db, string &sql, string criteria, std::vector> &labels); + + /// \brief get column values + std::pair> GetLabels(int group_id, int shard_id, const std::vector &columns, + const std::pair &criteria = {"", ""}); + + /// \brief get column values from raw data page + std::pair> GetLabelsFromPage(int group_id, int shard_id, + const std::vector &columns, + const std::pair &criteria = {"", + ""}); + + /// \brief create task list in block-reader mode + MSRStatus CreateTasksByBlock(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief create category-applied task list + int CreateTasksByCategory(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief create task list in row-reader mode + MSRStatus CreateTasksByRow(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief crate task list + MSRStatus CreateTasks(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief set NLP flag + void CheckNlp(); + + /// \brief check if all specified columns are in index table + void CheckIfColumnInIndex(const std::vector &columns); + + /// \brief open multiple file handle + void FileStreamsOperator(); + + /// \brief read one row by one task + TASK_RETURN_CONTENT ConsumerOneTask(int task_id, uint32_t consumer_id); + + /// \brief get all the column names by schema + vector GetAllColumns(); + + /// \brief get one row from buffer in block-reader mode + std::shared_ptr, json>>> GetRowFromBuffer(int bufId, int rowId); + + /// \brief get labels from binary file + std::pair> GetLabelsFromBinaryFile( + int shard_id, const std::vector &columns, const std::vector> &label_offsets); + + MSRStatus ReadBlob(const int &shard_id, const uint64_t &page_offset, const int &page_length, const int &buf_id); + + protected: + uint64_t header_size_; // header size + uint64_t page_size_; // page size + int shard_count_; // number of shards + std::shared_ptr shard_header_; // shard header + bool nlp_ = false; // NLP data + + std::vector database_paths_; // sqlite handle list + std::vector file_paths_; // file paths + std::vector> file_streams_; // single-file handle list + std::vector>> file_streams_random_; // multiple-file handle list + + private: + int n_consumer_; // number of workers (threads) + std::vector selected_columns_; // columns which will be read + std::map column_schema_id_; // column-schema map + std::vector> operators_; // data operators, including shuffle, sample and category + ShardTask tasks_; // shard task + std::mutex shard_locker_; // locker of shard + + // flags + bool all_in_index_ = true; // if all columns are stored in index-table + bool interrupt_ = false; // reader interrupted + + // Delivery/Iterator mode begin + const std::string kThreadName = "THRD_ITER_"; // prefix of thread name + std::vector thread_set_; // thread list + int num_rows_; // number of rows + std::mutex mtx_delivery_; // locker for delivery + std::condition_variable cv_delivery_; // conditional variable for delivery + std::condition_variable cv_iterator_; // conditional variable for iterator + std::atomic task_id_; // task ID which is working + std::atomic deliver_id_; // delivery ID which is picked up by iterator + // map of delivery + std::unordered_map, json>>>> delivery_map_; + // Delivery/Iterator mode end + + // Block reader mode begin + bool block_reader_; // block-reader mode + int row_id_; // row id in one page + int num_blocks_; // number of pages + // raw data page + std::vector>, std::vector>>> delivery_block_; + std::unordered_set delivery_block_set_; // set of delivered pages + std::vector> buf_; // page buffer + // Block reader mode end +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_READER_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_sample.h b/mindspore/ccsrc/mindrecord/include/shard_sample.h new file mode 100644 index 00000000000..f6b074a65d2 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_sample.h @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_SAMPLE_H_ +#define MINDRECORD_INCLUDE_SHARD_SAMPLE_H_ + +#include +#include "mindrecord/include/shard_operator.h" + +namespace mindspore { +namespace mindrecord { +class ShardSample : public ShardOperator { + public: + explicit ShardSample(int n); + + ShardSample(int num, int den); + + ShardSample(int num, int den, int par); + + ~ShardSample() override{}; + + const std::pair get_partitions() const; + + MSRStatus operator()(ShardTask &tasks) override; + + private: + int numerator_; + int denominator_; + int no_of_samples_; + int partition_id_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_SAMPLE_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_schema.h b/mindspore/ccsrc/mindrecord/include/shard_schema.h new file mode 100644 index 00000000000..ee0222ec8ea --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_schema.h @@ -0,0 +1,90 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_SCHEMA_H_ +#define MINDRECORD_INCLUDE_SHARD_SCHEMA_H_ + +#include +#include +#include +#include +#include +#include "mindrecord/include/common/shard_pybind.h" +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_error.h" +#include "pybind11/pybind11.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +class Schema { + public: + ~Schema() = default; + + /// \brief obtain the json schema ,its description, its block fields + /// \param[in] desc the description of the schema + /// \param[in] schema the schema's json + static std::shared_ptr Build(std::string desc, const json &schema); + + /// \brief obtain the json schema and its description for python + /// \param[in] desc the description of the schema + /// \param[in] schema the schema's json + static std::shared_ptr Build(std::string desc, pybind11::handle schema); + + /// \brief compare two schema to judge if they are equal + /// \param b another schema to be judged + /// \return true if they are equal,false if not + bool operator==(const Schema &b) const; + + /// \brief get the schema and its description + /// \return the json format of the schema and its description + std::string get_desc() const; + + /// \brief get the schema and its description + /// \return the json format of the schema and its description + json GetSchema() const; + + /// \brief get the schema and its description for python method + /// \return the python object of the schema and its description + pybind11::object GetSchemaForPython() const; + + /// set the schema id + /// \param[in] id the id need to be set + void set_schema_id(int64_t id); + + /// get the schema id + /// \return the int64 schema id + int64_t get_schema_id() const; + + /// get the blob fields + /// \return the vector blob fields + std::vector get_blob_fields() const; + + private: + Schema() = default; + static bool ValidateNumberShape(const json &it_value); + static bool Validate(json schema); + static std::vector PopulateBlobFields(json schema); + + std::string desc_; + json schema_; + std::vector blob_fields_; + int64_t schema_id_ = -1; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_SCHEMA_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_segment.h b/mindspore/ccsrc/mindrecord/include/shard_segment.h new file mode 100644 index 00000000000..c13ea5dccf9 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_segment.h @@ -0,0 +1,104 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_SEGMENT_H_ +#define MINDRECORD_INCLUDE_SHARD_SEGMENT_H_ + +#include +#include +#include +#include +#include "mindrecord/include/shard_reader.h" + +namespace mindspore { +namespace mindrecord { +class ShardSegment : public ShardReader { + public: + ShardSegment(); + + ~ShardSegment() override = default; + + /// \brief Get candidate category fields + /// \return a list of fields names which are the candidates of category + std::pair> GetCategoryFields(); + + /// \brief Set category field + /// \param[in] category_field category name + /// \return true if category name is existed + MSRStatus SetCategoryField(std::string category_field); + + /// \brief Thread-safe implementation of ReadCategoryInfo + /// \return statistics data in json format with 2 field: "key" and "categories". + /// The value of "categories" is a list. Each Element in list is {count, id， name} + /// count: count of images in category + /// id: internal unique identification, persistent + /// name: category name + /// example: + /// { "key": "label", + /// "categories": [ { "count": 3, "id": 0, "name": "sport", }, + /// { "count": 3, "id": 1, "name": "finance", } ] } + std::pair ReadCategoryInfo(); + + /// \brief Thread-safe implementation of ReadAtPageById + /// \param[in] category_id category ID + /// \param[in] page_no page number + /// \param[in] n_rows_of_page rows number in one page + /// \return images array, image is a vector of uint8_t + std::pair>> ReadAtPageById(int64_t category_id, int64_t page_no, + int64_t n_rows_of_page); + + /// \brief Thread-safe implementation of ReadAtPageByName + /// \param[in] category_name category Name + /// \param[in] page_no page number + /// \param[in] n_rows_of_page rows number in one page + /// \return images array, image is a vector of uint8_t + std::pair>> ReadAtPageByName(std::string category_name, int64_t page_no, + int64_t n_rows_of_page); + + std::pair, json>>> ReadAllAtPageById(int64_t category_id, + int64_t page_no, + int64_t n_rows_of_page); + + std::pair, json>>> ReadAllAtPageByName( + std::string category_name, int64_t page_no, int64_t n_rows_of_page); + + std::pair, pybind11::object>>> ReadAtPageByIdPy( + int64_t category_id, int64_t page_no, int64_t n_rows_of_page); + + std::pair, pybind11::object>>> ReadAtPageByNamePy( + std::string category_name, int64_t page_no, int64_t n_rows_of_page); + + std::pair> get_blob_fields(); + + private: + std::pair>> WrapCategoryInfo(); + + std::string ToJsonForCategory(const std::vector> &tri_vec); + + std::string CleanUp(std::string fieldName); + + std::tuple, json> GetImageLabel(std::vector images, json label); + + std::pair> PackImages(int group_id, int shard_id, std::vector offset); + + std::vector candidate_category_fields_; + std::string current_category_field_; + const uint32_t kStartFieldId = 9; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_SEGMENT_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_shuffle.h b/mindspore/ccsrc/mindrecord/include/shard_shuffle.h new file mode 100644 index 00000000000..a9992ab4bcb --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_shuffle.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_ +#define MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_ + +#include +#include "mindrecord/include/shard_operator.h" + +namespace mindspore { +namespace mindrecord { +class ShardShuffle : public ShardOperator { + public: + explicit ShardShuffle(uint32_t seed = 0); + + ~ShardShuffle() override{}; + + MSRStatus operator()(ShardTask &tasks) override; + + private: + uint32_t shuffle_seed_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_statistics.h b/mindspore/ccsrc/mindrecord/include/shard_statistics.h new file mode 100644 index 00000000000..44956332e13 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_statistics.h @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifndef MINDRECORD_STATISTICS_H +#define MINDRECORD_STATISTICS_H + +#include +#include +#include +#include +#include + +#include "mindrecord/include/common/shard_pybind.h" +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_error.h" +#include "pybind11/pybind11.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +class Statistics { + public: + /// \brief save the statistic and its description + /// \param[in] desc the statistic's description + /// \param[in] statistics the statistic needs to be saved + static std::shared_ptr Build(std::string desc, const json &statistics); + + /// \brief save the statistic from python and its description + /// \param[in] desc the statistic's description + /// \param[in] statistics the statistic needs to be saved + static std::shared_ptr Build(std::string desc, pybind11::handle statistics); + + ~Statistics() = default; + + /// \brief compare two statistics to judge if they are equal + /// \param b another statistics to be judged + /// \return true if they are equal,false if not + bool operator==(const Statistics &b) const; + + /// \brief get the description + /// \return the description + std::string get_desc() const; + + /// \brief get the statistic + /// \return json format of the statistic + json get_statistics() const; + + /// \brief get the statistic for python + /// \return the python object of statistics + pybind11::object GetStatisticsForPython() const; + + /// \brief decode the bson statistics to json + /// \param[in] encodedStatistics the bson type of statistics + /// \return json type of statistic + void set_statistics_id(int64_t id); + + /// \brief get the statistics id + /// \return the int64 statistics id + int64_t get_statistics_id() const; + + private: + /// \brief validate the statistic + /// \return true / false + static bool Validate(const json &statistics); + + static bool LevelRecursive(json level); + + Statistics() = default; + + std::string desc_; + json statistics_; + int64_t statistics_id_ = -1; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_STATISTICS_H diff --git a/mindspore/ccsrc/mindrecord/include/shard_task.h b/mindspore/ccsrc/mindrecord/include/shard_task.h new file mode 100644 index 00000000000..30ea352ef39 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_task.h @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_TASK_H_ +#define MINDRECORD_INCLUDE_SHARD_TASK_H_ + +#include +#include +#include +#include +#include "mindrecord/include/common/shard_utils.h" + +namespace mindspore { +namespace mindrecord { +class ShardTask { + public: + void MakePerm(); + + void InsertTask(int shard_id, int group_id, const std::vector &offset, const json &label); + + void InsertTask(std::tuple, std::vector, json> task); + + void PopBack(); + + uint32_t Size() const; + + uint32_t SizeOfRows() const; + + std::tuple, std::vector, json> &get_task_by_id(size_t id); + + static ShardTask Combine(std::vector &category_tasks); + + uint32_t categories = 1; + + std::vector, std::vector, json>> task_list_; + std::vector permutation_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_TASK_H_ diff --git a/mindspore/ccsrc/mindrecord/include/shard_writer.h b/mindspore/ccsrc/mindrecord/include/shard_writer.h new file mode 100644 index 00000000000..6a22f07700b --- /dev/null +++ b/mindspore/ccsrc/mindrecord/include/shard_writer.h @@ -0,0 +1,226 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDRECORD_INCLUDE_SHARD_WRITER_H_ +#define MINDRECORD_INCLUDE_SHARD_WRITER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mindrecord/include/common/shard_utils.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_header.h" +#include "mindrecord/include/shard_index.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +class ShardWriter { + public: + ShardWriter(); + + ~ShardWriter(); + + /// \brief Open file at the beginning + /// \param[in] paths the file names list + /// \param[in] append new data at the end of file if true, otherwise overwrite file + /// \return MSRStatus the status of MSRStatus + MSRStatus Open(const std::vector &paths, bool append = false); + + /// \brief Open file at the ending + /// \param[in] paths the file names list + /// \return MSRStatus the status of MSRStatus + MSRStatus OpenForAppend(const std::string &path); + + /// \brief Write header to disk + /// \return MSRStatus the status of MSRStatus + MSRStatus Commit(); + + /// \brief Set file size + /// \param[in] header_size the size of header, only (1< header_data); + + /// \brief write raw data by group size + /// \param[in] raw_data the vector of raw json data, vector format + /// \param[in] blob_data the vector of image data + /// \param[in] sign validate data or not + /// \return MSRStatus the status of MSRStatus to judge if write successfully + MSRStatus WriteRawData(std::map> &raw_data, vector> &blob_data, + bool sign = true); + + /// \brief write raw data by group size for call from python + /// \param[in] raw_data the vector of raw json data, python-handle format + /// \param[in] blob_data the vector of image data + /// \param[in] sign validate data or not + /// \return MSRStatus the status of MSRStatus to judge if write successfully + MSRStatus WriteRawData(std::map> &raw_data, vector> &blob_data, + bool sign = true); + + /// \brief write raw data by group size for call from python + /// \param[in] raw_data the vector of raw json data, python-handle format + /// \param[in] blob_data the vector of blob json data, python-handle format + /// \param[in] sign validate data or not + /// \return MSRStatus the status of MSRStatus to judge if write successfully + MSRStatus WriteRawData(std::map> &raw_data, + std::map> &blob_data, bool sign = true); + + private: + /// \brief write shard header data to disk + MSRStatus WriteShardHeader(); + + /// \brief erase error data + void DeleteErrorData(std::map> &raw_data, std::vector> &blob_data); + + /// \brief populate error data + void PopulateMutexErrorData(const int &row, const std::string &message, std::map &err_raw_data); + + /// \brief check data + void CheckSliceData(int start_row, int end_row, json schema, const std::vector &sub_raw_data, + std::map &err_raw_data); + + /// \brief write shard header data to disk + std::tuple ValidateRawData(std::map> &raw_data, + std::vector> &blob_data, bool sign); + + /// \brief fill data array in multiple thread run + void FillArray(int start, int end, std::map> &raw_data, + std::vector> &bin_data); + + /// \brief serialized raw data + MSRStatus SerializeRawData(std::map> &raw_data, + std::vector> &bin_data, uint32_t row_count); + + /// \brief write all data parallel + MSRStatus ParallelWriteData(const std::vector> &blob_data, + const std::vector> &bin_raw_data); + + /// \brief write data shard by shard + MSRStatus WriteByShard(int shard_id, int start_row, int end_row, const std::vector> &blob_data, + const std::vector> &bin_raw_data); + + /// \brief break image data up into multiple row groups + MSRStatus CutRowGroup(int start_row, int end_row, const std::vector> &blob_data, + std::vector> &rows_in_group, const std::shared_ptr &last_raw_page, + const std::shared_ptr &last_blob_page); + + /// \brief append partial blob data to previous page + MSRStatus AppendBlobPage(const int &shard_id, const std::vector> &blob_data, + const std::vector> &rows_in_group, + const std::shared_ptr &last_blob_page); + + /// \brief write new blob data page to disk + MSRStatus NewBlobPage(const int &shard_id, const std::vector> &blob_data, + const std::vector> &rows_in_group, + const std::shared_ptr &last_blob_page); + + /// \brief shift last row group to next raw page for new appending + MSRStatus ShiftRawPage(const int &shard_id, const std::vector> &rows_in_group, + std::shared_ptr &last_raw_page); + + /// \brief write raw data page to disk + MSRStatus WriteRawPage(const int &shard_id, const std::vector> &rows_in_group, + std::shared_ptr &last_raw_page, const std::vector> &bin_raw_data); + + /// \brief generate empty raw data page + void EmptyRawPage(const int &shard_id, std::shared_ptr &last_raw_page); + + /// \brief append a row group at the end of raw page + MSRStatus AppendRawPage(const int &shard_id, const std::vector> &rows_in_group, + const int &chunk_id, int &last_row_groupId, std::shared_ptr last_raw_page, + const std::vector> &bin_raw_data); + + /// \brief write blob chunk to disk + MSRStatus FlushBlobChunk(const std::shared_ptr &out, const std::vector> &blob_data, + const std::pair &blob_row); + + /// \brief write raw chunk to disk + MSRStatus FlushRawChunk(const std::shared_ptr &out, + const std::vector> &rows_in_group, const int &chunk_id, + const std::vector> &bin_raw_data); + + /// \brief break up into tasks by shard + std::vector> BreakIntoShards(); + + /// \brief calculate raw data size row by row + MSRStatus SetRawDataSize(const std::vector> &bin_raw_data); + + /// \brief calculate blob data size row by row + MSRStatus SetBlobDataSize(const std::vector> &blob_data); + + /// \brief populate last raw page pointer + void SetLastRawPage(const int &shard_id, std::shared_ptr &last_raw_page); + + /// \brief populate last blob page pointer + void SetLastBlobPage(const int &shard_id, std::shared_ptr &last_blob_page); + + /// \brief check the data by schema + MSRStatus CheckData(const std::map> &raw_data); + + /// \brief check the data and type + MSRStatus CheckDataTypeAndValue(const std::string &key, const json &value, const json &data, const int &i, + std::map &err_raw_data); + + private: + int shard_count_; // number of files + uint64_t header_size_; // header size + uint64_t page_size_; // page size + uint32_t row_count_; // count of rows + uint32_t schema_count_; // count of schemas + + std::vector raw_data_size_; // Raw data size + std::vector blob_data_size_; // Blob data size + + std::vector file_paths_; // file paths + std::vector> file_streams_; // file handles + std::shared_ptr shard_header_; // shard headers + + std::map> err_mg_; // used for storing error raw_data info + + std::mutex check_mutex_; // mutex for data check + std::atomic flag_{false}; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDRECORD_INCLUDE_SHARD_WRITER_H_ diff --git a/mindspore/ccsrc/mindrecord/io/shard_index_generator.cc b/mindspore/ccsrc/mindrecord/io/shard_index_generator.cc new file mode 100644 index 00000000000..1c14d30f30c --- /dev/null +++ b/mindspore/ccsrc/mindrecord/io/shard_index_generator.cc @@ -0,0 +1,550 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_index_generator.h" +#include "common/utils.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::DEBUG; +using mindspore::MsLogLevel::ERROR; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +ShardIndexGenerator::ShardIndexGenerator(const std::string &file_path, bool append) + : file_path_(file_path), append_(append), page_size_(0), header_size_(0), schema_count_(0) {} + +MSRStatus ShardIndexGenerator::Build() { + ShardHeader header = ShardHeader(); + if (header.Build(file_path_) != SUCCESS) { + MS_LOG(ERROR) << "Build shard schema failed"; + return FAILED; + } + shard_header_ = header; + MS_LOG(INFO) << "Init header from mindrecord file for index successfully."; + return SUCCESS; +} + +std::vector ShardIndexGenerator::GetField(const string &field_path, json schema) { + std::vector field_name = StringSplit(field_path, kPoint); + std::vector res; + if (schema.empty()) { + res.emplace_back("null"); + return res; + } + for (uint64_t i = 0; i < field_name.size(); i++) { + // Check if field is part of an array of objects + auto &child = schema.at(field_name[i]); + if (child.is_array() && !child.empty() && child[0].is_object()) { + schema = schema[field_name[i]]; + std::string new_field_path; + for (uint64_t j = i + 1; j < field_name.size(); j++) { + if (j > i + 1) new_field_path += '.'; + new_field_path += field_name[j]; + } + // Return multiple field data since multiple objects in array + for (auto &single_schema : schema) { + auto child_res = GetField(new_field_path, single_schema); + res.insert(res.end(), child_res.begin(), child_res.end()); + } + return res; + } + schema = schema.at(field_name[i]); + } + + // Return vector of one field data (not array of objects) + return std::vector{schema.dump()}; +} + +std::string ShardIndexGenerator::TakeFieldType(const string &field_path, json schema) { + std::vector field_name = StringSplit(field_path, kPoint); + for (uint64_t i = 0; i < field_name.size(); i++) { + if (i != field_name.size() - 1) { + // Get type information from json schema + schema = schema.at(field_name[i]); + schema = schema.at("properties"); + } else { + // standard root layer exist "properties" if type is "object" + if (schema.find("properties") != schema.end()) { + schema = schema.at("properties"); + } + schema = schema.at(field_name[i]); + std::string field_type = schema.at("type").dump(); + if (field_type.length() <= 2) { + return ""; + } else { + return field_type.substr(1, field_type.length() - 2); + } + } + } + return ""; +} + +std::string ShardIndexGenerator::ConvertJsonToSQL(const std::string &json) { + if (kDbJsonMap.find(json) != kDbJsonMap.end()) { + return kDbJsonMap.at(json); + } else { + return "TEXT"; + } +} + +int ShardIndexGenerator::Callback(void *not_used, int argc, char **argv, char **az_col_name) { + for (auto i = 0; i < argc; i++) { + if (argv[i] != nullptr) { + MS_LOG(INFO) << az_col_name[i] << " = " << (argv[i] ? argv[i] : "nullptr"); + } + } + MS_LOG(INFO) << "\n"; + return 0; +} + +MSRStatus ShardIndexGenerator::ExecuteSQL(const std::string &sql, sqlite3 *db, const std::string &success_msg) { + char *z_err_msg = nullptr; + int rc = sqlite3_exec(db, common::SafeCStr(sql), Callback, nullptr, &z_err_msg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Sql error: " << z_err_msg; + sqlite3_free(z_err_msg); + return FAILED; + } else { + if (!success_msg.empty()) { + MS_LOG(DEBUG) << "Sqlite3_exec exec success, msg is: " << success_msg; + } + sqlite3_free(z_err_msg); + return SUCCESS; + } +} + +std::pair ShardIndexGenerator::GenerateFieldName( + const std::pair &field) { + // Replaces dots and dashes with underscores for SQL use + std::string field_name = field.second; + // white list to avoid sql injection + std::replace_if( + field_name.begin(), field_name.end(), [](char x) { return (x == '-' || x == '.'); }, '_'); + auto pos = std::find_if_not(field_name.begin(), field_name.end(), [](char x) { + return (x >= 'A' && x <= 'Z') || (x >= 'a' && x <= 'z') || x == '_' || (x >= '0' && x <= '9'); + }); + if (pos != field_name.end()) { + MS_LOG(ERROR) << "Field name must be composed of '0-9' or 'a-z' or 'A-Z' or '_', field_name: " << field_name; + return {FAILED, ""}; + } + return {SUCCESS, field_name + "_" + std::to_string(field.first)}; +} + +std::pair ShardIndexGenerator::CheckDatabase(const std::string &shard_address) { + sqlite3 *db = nullptr; + std::ifstream fin(common::SafeCStr(shard_address)); + if (!append_ && fin.good()) { + MS_LOG(ERROR) << "DB file already exist"; + fin.close(); + return {FAILED, nullptr}; + } + fin.close(); + int rc = sqlite3_open_v2(common::SafeCStr(shard_address), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); + if (rc) { + MS_LOG(ERROR) << "Can't open database, error: " << sqlite3_errmsg(db); + return {FAILED, nullptr}; + } else { + MS_LOG(DEBUG) << "Opened database successfully"; + return {SUCCESS, db}; + } +} + +MSRStatus ShardIndexGenerator::CreateShardNameTable(sqlite3 *db, const std::string &shard_name) { + // create shard_name table + std::string sql = "DROP TABLE IF EXISTS SHARD_NAME;"; + if (ExecuteSQL(sql, db, "drop table successfully.") != SUCCESS) { + return FAILED; + } + sql = "CREATE TABLE SHARD_NAME(NAME TEXT NOT NULL);"; + if (ExecuteSQL(sql, db, "create table successfully.") != SUCCESS) { + return FAILED; + } + sql = "INSERT INTO SHARD_NAME (NAME) VALUES ('" + shard_name + "');"; + if (ExecuteSQL(sql, db, "insert name successfully.") != SUCCESS) { + return FAILED; + } + return SUCCESS; +} + +std::pair ShardIndexGenerator::CreateDatabase(int shard_no) { + std::string shard_address = shard_header_.get_shard_address_by_id(shard_no); + if (shard_address.empty()) { + MS_LOG(ERROR) << "Shard address is null, shard no: " << shard_no; + return {FAILED, nullptr}; + } + + string shard_name = GetFileName(shard_address).second; + shard_address += ".db"; + auto ret1 = CheckDatabase(shard_address); + if (ret1.first != SUCCESS) { + return {FAILED, nullptr}; + } + sqlite3 *db = ret1.second; + std::string sql = "DROP TABLE IF EXISTS INDEXES;"; + if (ExecuteSQL(sql, db, "drop table successfully.") != SUCCESS) { + return {FAILED, nullptr}; + } + sql = + "CREATE TABLE INDEXES(" + " ROW_ID INT NOT NULL, PAGE_ID_RAW INT NOT NULL" + ", PAGE_OFFSET_RAW INT NOT NULL, PAGE_OFFSET_RAW_END INT NOT NULL" + ", ROW_GROUP_ID INT NOT NULL, PAGE_ID_BLOB INT NOT NULL" + ", PAGE_OFFSET_BLOB INT NOT NULL, PAGE_OFFSET_BLOB_END INT NOT NULL"; + + int field_no = 0; + for (const auto &field : fields_) { + uint64_t schema_id = field.first; + auto result = shard_header_.GetSchemaByID(schema_id); + if (result.second != SUCCESS) { + return {FAILED, nullptr}; + } + json json_schema = (result.first->GetSchema())["schema"]; + std::string type = ConvertJsonToSQL(TakeFieldType(field.second, json_schema)); + auto ret = GenerateFieldName(field); + if (ret.first != SUCCESS) { + return {FAILED, nullptr}; + } + sql += ",INC_" + std::to_string(field_no++) + " INT, " + ret.second + " " + type; + } + sql += ", PRIMARY KEY(ROW_ID"; + for (uint64_t i = 0; i < fields_.size(); ++i) sql += ",INC_" + std::to_string(i); + sql += "));"; + if (ExecuteSQL(sql, db, "create table successfully.") != SUCCESS) { + return {FAILED, nullptr}; + } + + if (CreateShardNameTable(db, shard_name) != SUCCESS) { + return {FAILED, nullptr}; + } + return {SUCCESS, db}; +} + +std::pair> ShardIndexGenerator::GetSchemaDetails(const std::vector &schema_lens, + std::fstream &in) { + std::vector schema_details; + if (schema_count_ <= kMaxSchemaCount) { + for (int sc = 0; sc < schema_count_; ++sc) { + std::vector schema_detail(schema_lens[sc]); + + auto &io_read = in.read(&schema_detail[0], schema_lens[sc]); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + in.close(); + return {FAILED, {}}; + } + + schema_details.emplace_back(json::from_msgpack(std::string(schema_detail.begin(), schema_detail.end()))); + } + } + + return {SUCCESS, schema_details}; +} + +std::pair ShardIndexGenerator::GenerateRawSQL( + const std::vector> &fields) { + std::string sql = + "INSERT INTO INDEXES (ROW_ID,ROW_GROUP_ID,PAGE_ID_RAW,PAGE_OFFSET_RAW,PAGE_OFFSET_RAW_END," + "PAGE_ID_BLOB,PAGE_OFFSET_BLOB,PAGE_OFFSET_BLOB_END"; + + int field_no = 0; + for (const auto &field : fields) { + auto ret = GenerateFieldName(field); + if (ret.first != SUCCESS) { + return {FAILED, ""}; + } + sql += ",INC_" + std::to_string(field_no++) + "," + ret.second; + } + sql += + ") VALUES( :ROW_ID,:ROW_GROUP_ID,:PAGE_ID_RAW,:PAGE_OFFSET_RAW,:PAGE_OFFSET_RAW_END,:PAGE_ID_BLOB," + ":PAGE_OFFSET_BLOB,:PAGE_OFFSET_BLOB_END"; + field_no = 0; + for (const auto &field : fields) { + auto ret = GenerateFieldName(field); + if (ret.first != SUCCESS) { + return {FAILED, ""}; + } + sql += ",:INC_" + std::to_string(field_no++) + ",:" + ret.second; + } + sql += " )"; + return {SUCCESS, sql}; +} + +MSRStatus ShardIndexGenerator::BindParamaterExecuteSQL( + sqlite3 *db, const std::string &sql, + const std::vector>> &data) { + sqlite3_stmt *stmt = nullptr; + if (sqlite3_prepare_v2(db, common::SafeCStr(sql), -1, &stmt, 0) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not prepare statement, sql: " << sql; + return FAILED; + } + for (auto &row : data) { + for (auto &field : row) { + const auto &place_holder = std::get<0>(field); + const auto &field_type = std::get<1>(field); + const auto &field_value = std::get<2>(field); + int index = sqlite3_bind_parameter_index(stmt, common::SafeCStr(place_holder)); + if (field_type == "INTEGER") { + if (sqlite3_bind_int(stmt, index, std::stoi(field_value)) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not bind parameter, index: " << index + << ", field value: " << std::stoi(field_value); + return FAILED; + } + } else if (field_type == "NUMERIC") { + if (sqlite3_bind_double(stmt, index, std::stod(field_value)) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not bind parameter, index: " << index + << ", field value: " << std::stoi(field_value); + return FAILED; + } + } else if (field_type == "NULL") { + if (sqlite3_bind_null(stmt, index) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not bind parameter, index: " << index << ", field value: NULL"; + return FAILED; + } + } else { + if (sqlite3_bind_text(stmt, index, common::SafeCStr(field_value), -1, SQLITE_STATIC) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not bind parameter, index: " << index << ", field value: " << field_value; + return FAILED; + } + } + } + if (sqlite3_step(stmt) != SQLITE_DONE) { + MS_LOG(ERROR) << "SQL error: Could not step (execute) stmt."; + return FAILED; + } + (void)sqlite3_reset(stmt); + } + (void)sqlite3_finalize(stmt); + return SUCCESS; +} + +MSRStatus ShardIndexGenerator::AddBlobPageInfo(std::vector> &row_data, + const std::shared_ptr cur_blob_page, + uint64_t &cur_blob_page_offset, std::fstream &in) { + row_data.emplace_back(":PAGE_ID_BLOB", "INTEGER", std::to_string(cur_blob_page->get_page_id())); + + // blob data start + row_data.emplace_back(":PAGE_OFFSET_BLOB", "INTEGER", std::to_string(cur_blob_page_offset)); + auto &io_seekg_blob = + in.seekg(page_size_ * cur_blob_page->get_page_id() + header_size_ + cur_blob_page_offset, std::ios::beg); + if (!io_seekg_blob.good() || io_seekg_blob.fail() || io_seekg_blob.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + in.close(); + return FAILED; + } + + uint64_t image_size = 0; + + auto &io_read = in.read(reinterpret_cast(&image_size), kInt64Len); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + in.close(); + return FAILED; + } + + cur_blob_page_offset += (kInt64Len + image_size); + row_data.emplace_back(":PAGE_OFFSET_BLOB_END", "INTEGER", std::to_string(cur_blob_page_offset)); + + return SUCCESS; +} + +void ShardIndexGenerator::AddIndexFieldByRawData( + const std::vector &schema_detail, std::vector> &row_data) { + auto result = GenerateIndexFields(schema_detail); + if (result.first == SUCCESS) { + int index = 0; + for (const auto &field : result.second) { + // assume simple field: string , number etc. + row_data.emplace_back(":INC_" + std::to_string(index++), "INTEGER", "0"); + row_data.emplace_back(":" + std::get<0>(field), std::get<1>(field), std::get<2>(field)); + } + } +} + +ROW_DATA ShardIndexGenerator::GenerateRowData(int shard_no, const std::map &blob_id_to_page_id, + int raw_page_id, std::fstream &in) { + std::vector>> full_data; + + // current raw data page + std::shared_ptr cur_raw_page = shard_header_.GetPage(shard_no, raw_page_id).first; + + // related blob page + vector> row_group_list = cur_raw_page->get_row_group_ids(); + + // pair: row_group id, offset in raw data page + for (pair blob_ids : row_group_list) { + // get blob data page according to row_group id + std::shared_ptr cur_blob_page = shard_header_.GetPage(shard_no, blob_id_to_page_id.at(blob_ids.first)).first; + + // offset in current raw data page + auto cur_raw_page_offset = static_cast(blob_ids.second); + uint64_t cur_blob_page_offset = 0; + for (unsigned int i = cur_blob_page->get_start_row_id(); i < cur_blob_page->get_end_row_id(); ++i) { + std::vector> row_data; + row_data.emplace_back(":ROW_ID", "INTEGER", std::to_string(i)); + row_data.emplace_back(":ROW_GROUP_ID", "INTEGER", std::to_string(cur_blob_page->get_page_type_id())); + row_data.emplace_back(":PAGE_ID_RAW", "INTEGER", std::to_string(cur_raw_page->get_page_id())); + + // raw data start + row_data.emplace_back(":PAGE_OFFSET_RAW", "INTEGER", std::to_string(cur_raw_page_offset)); + + // calculate raw data end + auto &io_seekg = + in.seekg(page_size_ * (cur_raw_page->get_page_id()) + header_size_ + cur_raw_page_offset, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + in.close(); + return {FAILED, {}}; + } + + std::vector schema_lens; + if (schema_count_ <= kMaxSchemaCount) { + for (int sc = 0; sc < schema_count_; sc++) { + uint64_t schema_size = 0; + + auto &io_read = in.read(reinterpret_cast(&schema_size), kInt64Len); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + in.close(); + return {FAILED, {}}; + } + + cur_raw_page_offset += (kInt64Len + schema_size); + schema_lens.push_back(schema_size); + } + } + row_data.emplace_back(":PAGE_OFFSET_RAW_END", "INTEGER", std::to_string(cur_raw_page_offset)); + + // Getting schema for getting data for fields + auto st_schema_detail = GetSchemaDetails(schema_lens, in); + if (st_schema_detail.first != SUCCESS) { + return {FAILED, {}}; + } + + // start blob page info + if (AddBlobPageInfo(row_data, cur_blob_page, cur_blob_page_offset, in) != SUCCESS) { + return {FAILED, {}}; + } + + // start index field + AddIndexFieldByRawData(st_schema_detail.second, row_data); + full_data.push_back(std::move(row_data)); + } + } + return {SUCCESS, full_data}; +} + +INDEX_FIELDS ShardIndexGenerator::GenerateIndexFields(const std::vector &schema_detail) { + std::vector> fields; + // index fields + std::vector> index_fields = shard_header_.get_fields(); + for (const auto &field : index_fields) { + if (field.first >= schema_detail.size()) { + return {FAILED, {}}; + } + auto field_value = GetField(field.second, schema_detail[field.first]); + auto result = shard_header_.GetSchemaByID(field.first); + if (result.second != SUCCESS) { + return {FAILED, {}}; + } + std::string field_type = ConvertJsonToSQL(TakeFieldType(field.second, result.first->GetSchema()["schema"])); + auto ret = GenerateFieldName(field); + if (ret.first != SUCCESS) { + return {FAILED, {}}; + } + fields.emplace_back(ret.second, field_type, field_value[0]); + } + return {SUCCESS, std::move(fields)}; +} + +MSRStatus ShardIndexGenerator::ExcuteTransaction(const int &shard_no, const std::pair &db, + const std::vector &raw_page_ids, + const std::map &blob_id_to_page_id) { + // Add index data to database + std::string shard_address = shard_header_.get_shard_address_by_id(shard_no); + if (shard_address.empty()) { + MS_LOG(ERROR) << "Shard address is null"; + return FAILED; + } + + std::fstream in; + in.open(common::SafeCStr(shard_address), std::ios::in | std::ios::binary); + (void)sqlite3_exec(db.second, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr); + for (int raw_page_id : raw_page_ids) { + auto sql = GenerateRawSQL(fields_); + if (sql.first != SUCCESS) { + return FAILED; + } + auto data = GenerateRowData(shard_no, blob_id_to_page_id, raw_page_id, in); + if (data.first != SUCCESS) { + return FAILED; + } + if (BindParamaterExecuteSQL(db.second, sql.second, data.second) == FAILED) { + return FAILED; + } + MS_LOG(INFO) << "Insert " << data.second.size() << " rows to index db."; + } + (void)sqlite3_exec(db.second, "END TRANSACTION;", nullptr, nullptr, nullptr); + in.close(); + + // Close database + if (sqlite3_close(db.second) != SQLITE_OK) { + MS_LOG(ERROR) << "Close database failed"; + return FAILED; + } + return SUCCESS; +} + +MSRStatus ShardIndexGenerator::WriteToDatabase() { + fields_ = shard_header_.get_fields(); + page_size_ = shard_header_.get_page_size(); + header_size_ = shard_header_.get_header_size(); + schema_count_ = shard_header_.get_schema_count(); + if (shard_header_.get_shard_count() <= kMaxShardCount) { + // Create one database per shard + for (int shard_no = 0; shard_no < shard_header_.get_shard_count(); ++shard_no) { + // Create database + auto db = CreateDatabase(shard_no); + if (db.first != SUCCESS || db.second == nullptr) { + return FAILED; + } + MS_LOG(INFO) << "Init index db for shard: " << shard_no << " successfully."; + + // Pre-processing page information + auto total_pages = shard_header_.GetLastPageId(shard_no) + 1; + + std::map blob_id_to_page_id; + std::vector raw_page_ids; + for (uint64_t i = 0; i < total_pages; ++i) { + std::shared_ptr cur_page = shard_header_.GetPage(shard_no, i).first; + if (cur_page->get_page_type() == "RAW_DATA") { + raw_page_ids.push_back(i); + } else if (cur_page->get_page_type() == "BLOB_DATA") { + blob_id_to_page_id[cur_page->get_page_type_id()] = i; + } + } + + if (ExcuteTransaction(shard_no, db, raw_page_ids, blob_id_to_page_id) != SUCCESS) { + return FAILED; + } + MS_LOG(INFO) << "Generate index db for shard: " << shard_no << " successfully."; + } + } + return SUCCESS; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/io/shard_reader.cc b/mindspore/ccsrc/mindrecord/io/shard_reader.cc new file mode 100644 index 00000000000..791de6c60ba --- /dev/null +++ b/mindspore/ccsrc/mindrecord/io/shard_reader.cc @@ -0,0 +1,1184 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_reader.h" +#include "common/utils.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::DEBUG; +using mindspore::MsLogLevel::ERROR; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +ShardReader::ShardReader() { + task_id_ = 0; + deliver_id_ = 0; + shard_count_ = 0; + n_consumer_ = 0; + page_size_ = 0; + header_size_ = 0; + num_rows_ = 0; + row_id_ = 0; + num_blocks_ = 0; + block_reader_ = false; +} + +MSRStatus ShardReader::Init(const std::string &file_path) { + if (!IsLegalFile(file_path)) { + return FAILED; + } + ShardHeader sh = ShardHeader(); + if (sh.Build(file_path) == FAILED) { + return FAILED; + } + shard_header_ = std::make_shared(sh); + header_size_ = shard_header_->get_header_size(); + page_size_ = shard_header_->get_page_size(); + file_paths_ = shard_header_->get_shard_addresses(); + + for (const auto &file : file_paths_) { + sqlite3 *db = nullptr; + // sqlite3_open create a database if not found, use sqlite3_open_v2 instead of it + int rc = sqlite3_open_v2(common::SafeCStr(file + ".db"), &db, SQLITE_OPEN_READONLY, nullptr); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Can't open database, error: " << sqlite3_errmsg(db); + return FAILED; + } + MS_LOG(DEBUG) << "Opened database successfully"; + + string sql = "select NAME from SHARD_NAME;"; + std::vector> name; + char *errmsg = nullptr; + rc = sqlite3_exec(db, common::SafeCStr(sql), SelectCallback, &name, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(db); + return FAILED; + } else { + MS_LOG(DEBUG) << "Get " << static_cast(name.size()) << " records from index."; + string shardName = GetFileName(file).second; + if (name.empty() || name[0][0] != shardName) { + MS_LOG(ERROR) << "DB file can not match file " << file; + sqlite3_free(errmsg); + sqlite3_close(db); + return FAILED; + } + } + database_paths_.push_back(db); + } + + num_rows_ = 0; + auto row_group_summary = ReadRowGroupSummary(); + for (const auto &rg : row_group_summary) { + num_rows_ += std::get<3>(rg); + } + + MS_LOG(INFO) << "Get meta from mindrecord file & index file successfully."; + + return SUCCESS; +} + +MSRStatus ShardReader::CheckColumnList(const std::vector &selected_columns) { + vector inSchema(selected_columns.size(), 0); + for (auto &p : get_shard_header()->get_schemas()) { + auto schema = p->GetSchema()["schema"]; + for (unsigned int i = 0; i < selected_columns.size(); ++i) { + if (schema.find(selected_columns[i]) != schema.end()) { + inSchema[i] = 1; + } + } + } + if (std::any_of(std::begin(inSchema), std::end(inSchema), [](int x) { return x == 0; })) { + return FAILED; + } + + return SUCCESS; +} + +MSRStatus ShardReader::Open() { + file_streams_.clear(); + + for (const auto &file : file_paths_) { + std::shared_ptr fs = std::make_shared(); + fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); + if (fs->fail()) { + fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); + if (fs->fail()) { + MS_LOG(ERROR) << "File could not opened"; + return FAILED; + } + } + MS_LOG(INFO) << "Open shard file successfully."; + file_streams_.push_back(fs); + } + + return SUCCESS; +} + +MSRStatus ShardReader::Open(int n_consumer) { + file_streams_random_ = + std::vector>>(n_consumer, std::vector>()); + for (const auto &file : file_paths_) { + for (int j = 0; j < n_consumer; ++j) { + std::shared_ptr fs = std::make_shared(); + fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); + if (fs->fail()) { + fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); + if (fs->fail()) { + MS_LOG(ERROR) << "File could not opened"; + return FAILED; + } + } + file_streams_random_[j].push_back(fs); + } + MS_LOG(INFO) << "Open shard file successfully."; + } + + return SUCCESS; +} + +void ShardReader::FileStreamsOperator() { + for (int i = static_cast(file_streams_.size()) - 1; i >= 0; --i) { + if (file_streams_[i] != nullptr) { + file_streams_[i]->close(); + } + } + for (int i = static_cast(file_streams_random_.size()) - 1; i >= 0; --i) { + for (int j = static_cast(file_streams_random_[i].size()) - 1; j >= 0; --j) { + if (file_streams_random_[i][j] != nullptr) { + file_streams_random_[i][j]->close(); + } + } + } + for (int i = static_cast(database_paths_.size()) - 1; i >= 0; --i) { + if (database_paths_[i] != nullptr) { + (void)sqlite3_close(database_paths_[i]); + } + } +} + +ShardReader::~ShardReader() { Close(); } + +void ShardReader::Close() { + (void)Finish(); // interrupt reading and stop threads + FileStreamsOperator(); +} + +std::shared_ptr ShardReader::get_shard_header() const { return shard_header_; } + +int ShardReader::get_shard_count() const { return shard_header_->get_shard_count(); } + +int ShardReader::get_num_rows() const { return num_rows_; } + +std::vector> ShardReader::ReadRowGroupSummary() { + std::vector> row_group_summary; + int shard_count = shard_header_->get_shard_count(); + if (shard_count <= 0) { + return row_group_summary; + } + if (shard_count <= kMaxShardCount) { + for (int shard_id = 0; shard_id < shard_count; ++shard_id) { + // return -1 when page's size equals to 0. + auto last_page_id = shard_header_->GetLastPageId(shard_id); + if (static_cast(last_page_id) == -1) { + continue; + } + for (uint64_t page_id = 0; page_id <= last_page_id; ++page_id) { + const auto &page_t = shard_header_->GetPage(shard_id, page_id); + const auto &page = page_t.first; + if (page->get_page_type() != kPageTypeBlob) continue; + uint64_t start_row_id = page->get_start_row_id(); + if (start_row_id > page->get_end_row_id()) { + return std::vector>(); + } + uint64_t number_of_rows = page->get_end_row_id() - start_row_id; + row_group_summary.emplace_back(shard_id, page->get_page_type_id(), start_row_id, number_of_rows); + } + } + } + return row_group_summary; +} + +MSRStatus ShardReader::ConvertLabelToJson(const std::vector> &labels, + std::shared_ptr fs, + std::vector>> &offsets, int shard_id, + const std::vector &columns, + std::vector> &column_values) { + for (int i = 0; i < static_cast(labels.size()); ++i) { + uint64_t group_id = std::stoull(labels[i][0]); + uint64_t offset_start = std::stoull(labels[i][1]) + kInt64Len; + uint64_t offset_end = std::stoull(labels[i][2]); + offsets[shard_id].emplace_back( + std::vector{static_cast(shard_id), group_id, offset_start, offset_end}); + if (!all_in_index_) { + int raw_page_id = std::stoi(labels[i][3]); + uint64_t label_start = std::stoull(labels[i][4]) + kInt64Len; + uint64_t label_end = std::stoull(labels[i][5]); + auto len = label_end - label_start; + auto label_raw = std::vector(len); + auto &io_seekg = fs->seekg(page_size_ * raw_page_id + header_size_ + label_start, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + fs->close(); + return FAILED; + } + + auto &io_read = fs->read(reinterpret_cast(&label_raw[0]), len); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + fs->close(); + return FAILED; + } + + json label_json = json::from_msgpack(label_raw); + json tmp; + if (!columns.empty()) { + for (auto &col : columns) { + if (label_json.find(col) != label_json.end()) { + tmp[col] = label_json[col]; + } + } + } else { + tmp = label_json; + } + column_values[shard_id].emplace_back(tmp); + } else { + string json_str = "{"; + for (unsigned int j = 0; j < columns.size(); ++j) { + // construct the string json "f1": value + json_str = json_str + "\"" + columns[j] + "\":" + labels[i][j + 3]; + if (j < columns.size() - 1) { + json_str += ","; + } + } + json_str += "}"; + column_values[shard_id].emplace_back(json::parse(json_str)); + } + } + + return SUCCESS; +} + +MSRStatus ShardReader::ReadAllRowsInShard(int shard_id, const std::string &sql, const std::vector &columns, + std::vector>> &offsets, + std::vector> &column_values) { + auto db = database_paths_[shard_id]; + std::vector> labels; + char *errmsg = nullptr; + int rc = sqlite3_exec(db, common::SafeCStr(sql), SelectCallback, &labels, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(db); + return FAILED; + } + MS_LOG(INFO) << "Get " << static_cast(labels.size()) << " records from shard " << shard_id << " index."; + + std::string file_name = file_paths_[shard_id]; + std::shared_ptr fs = std::make_shared(); + if (!all_in_index_) { + fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::out | std::ios::binary); + if (fs->fail()) { + fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); + if (fs->fail()) { + MS_LOG(ERROR) << "File could not opened"; + } + } + } + sqlite3_free(errmsg); + return ConvertLabelToJson(labels, fs, offsets, shard_id, columns, column_values); +} + +ROW_GROUPS ShardReader::ReadAllRowGroup(std::vector &columns) { + std::string fields = "ROW_GROUP_ID, PAGE_OFFSET_BLOB, PAGE_OFFSET_BLOB_END"; + std::vector>> offsets(shard_count_, std::vector>{}); + std::vector> column_values(shard_count_, std::vector{}); + if (all_in_index_) { + for (unsigned int i = 0; i < columns.size(); ++i) { + fields += ','; + auto ret = ShardIndexGenerator::GenerateFieldName(std::make_pair(column_schema_id_[columns[i]], columns[i])); + if (ret.first != SUCCESS) { + return std::make_tuple(FAILED, std::move(offsets), std::move(column_values)); + } + fields += ret.second; + } + } else { // fetch raw data from Raw page while some field is not index. + fields += ", PAGE_ID_RAW, PAGE_OFFSET_RAW, PAGE_OFFSET_RAW_END "; + } + + std::string sql = "SELECT " + fields + " FROM INDEXES ORDER BY ROW_ID ;"; + + std::vector thread_read_db = std::vector(shard_count_); + for (int x = 0; x < shard_count_; x++) { + thread_read_db[x] = + std::thread(&ShardReader::ReadAllRowsInShard, this, x, sql, columns, std::ref(offsets), std::ref(column_values)); + } + + for (int x = 0; x < shard_count_; x++) { + thread_read_db[x].join(); + } + return std::make_tuple(SUCCESS, std::move(offsets), std::move(column_values)); +} + +ROW_GROUP_BRIEF ShardReader::ReadRowGroupBrief(int group_id, int shard_id, const std::vector &columns) { + std::lock_guard lck(shard_locker_); + const auto &ret = shard_header_->GetPageByGroupId(group_id, shard_id); + if (SUCCESS != ret.first) { + return std::make_tuple(FAILED, "", 0, 0, std::vector>(), std::vector()); + } + const std::shared_ptr &page = ret.second; + std::string file_name = file_paths_[shard_id]; + uint64_t page_length = page->get_page_size(); + uint64_t page_offset = page_size_ * page->get_page_id() + header_size_; + std::vector> image_offset = GetImageOffset(page->get_page_id(), shard_id); + + auto status_labels = GetLabels(page->get_page_id(), shard_id, columns); + if (status_labels.first != SUCCESS) { + return std::make_tuple(FAILED, "", 0, 0, std::vector>(), std::vector()); + } + return std::make_tuple(SUCCESS, file_name, page_length, page_offset, std::move(image_offset), + std::move(status_labels.second)); +} + +ROW_GROUP_BRIEF ShardReader::ReadRowGroupCriteria(int group_id, int shard_id, + const std::pair &criteria, + const std::vector &columns) { + std::lock_guard lck(shard_locker_); + const auto &ret = shard_header_->GetPageByGroupId(group_id, shard_id); + if (SUCCESS != ret.first) { + return std::make_tuple(FAILED, "", 0, 0, std::vector>(), std::vector()); + } + vector criteria_list{criteria.first}; + if (CheckColumnList(criteria_list) == FAILED) { + return std::make_tuple(FAILED, "", 0, 0, std::vector>(), std::vector()); + } + const std::shared_ptr &page = ret.second; + std::string file_name = file_paths_[shard_id]; + uint64_t page_length = page->get_page_size(); + uint64_t page_offset = page_size_ * page->get_page_id() + header_size_; + std::vector> image_offset = GetImageOffset(page->get_page_id(), shard_id, criteria); + + auto status_labels = GetLabels(page->get_page_id(), shard_id, columns, criteria); + if (status_labels.first != SUCCESS) { + return std::make_tuple(FAILED, "", 0, 0, std::vector>(), std::vector()); + } + + return std::make_tuple(SUCCESS, file_name, page_length, page_offset, std::move(image_offset), + std::move(status_labels.second)); +} + +int ShardReader::SelectCallback(void *p_data, int num_fields, char **p_fields, char **p_col_names) { + auto *records = static_cast> *>(p_data); + if (num_fields > 0 && num_fields <= kMaxFieldCount) { + for (int i = 0; i < num_fields; ++i) + if (p_fields[i] == nullptr) p_fields[i] = const_cast(""); + } + records->emplace_back(p_fields, p_fields + num_fields); + return 0; +} + +std::vector> ShardReader::GetImageOffset(int page_id, int shard_id, + const std::pair &criteria) { + auto db = database_paths_[shard_id]; + + std::string sql = + "SELECT PAGE_OFFSET_BLOB, PAGE_OFFSET_BLOB_END FROM INDEXES WHERE PAGE_ID_BLOB = " + std::to_string(page_id); + + // whether use index search + if (!criteria.first.empty()) { + sql += " AND " + criteria.first + "_" + std::to_string(column_schema_id_[criteria.first]) + " = " + criteria.second; + } + sql += ";"; + std::vector> image_offsets; + char *errmsg = nullptr; + int rc = sqlite3_exec(db, common::SafeCStr(sql), SelectCallback, &image_offsets, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(db); + return std::vector>(); + } else { + MS_LOG(DEBUG) << "Get " << static_cast(image_offsets.size()) << "records from index."; + } + std::vector> res; + for (int i = static_cast(image_offsets.size()) - 1; i >= 0; i--) res.emplace_back(std::vector{0, 0}); + for (int i = 0; i < static_cast(image_offsets.size()); i++) { + const auto &image_offset = image_offsets[i]; + res[i][0] = std::stoull(image_offset[0]) + kInt64Len; + res[i][1] = std::stoull(image_offset[1]); + } + sqlite3_free(errmsg); + return res; +} + +void ShardReader::CheckNlp() { + nlp_ = false; + return; +} + +bool ShardReader::get_nlp_flag() { return nlp_; } + +std::pair> ShardReader::get_blob_fields() { + std::vector blob_fields; + for (auto &p : get_shard_header()->get_schemas()) { + // assume one schema + const auto &fields = p->get_blob_fields(); + blob_fields.assign(fields.begin(), fields.end()); + break; + } + return std::make_pair(nlp_ ? kNLP : kCV, blob_fields); +} + +void ShardReader::CheckIfColumnInIndex(const std::vector &columns) { + // assume different schemas do not contain same key. + if (columns.empty()) { + all_in_index_ = false; + return; + } + for (auto &field : get_shard_header()->get_fields()) { + column_schema_id_[field.second] = field.first; + } + for (auto &col : columns) { + if (column_schema_id_.find(col) == column_schema_id_.end()) { + all_in_index_ = false; + return; + } + } +} + +MSRStatus ShardReader::QueryWithCriteria(sqlite3 *db, string &sql, string criteria, + std::vector> &labels) { + sqlite3_stmt *stmt = nullptr; + if (sqlite3_prepare_v2(db, common::SafeCStr(sql), -1, &stmt, 0) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not prepare statement"; + return FAILED; + } + int index = sqlite3_bind_parameter_index(stmt, ":criteria"); + if (sqlite3_bind_text(stmt, index, common::SafeCStr(criteria), -1, SQLITE_STATIC) != SQLITE_OK) { + MS_LOG(ERROR) << "SQL error: could not bind parameter, index: " << index << ", field value: " << criteria; + return FAILED; + } + int rc = sqlite3_step(stmt); + while (rc != SQLITE_DONE) { + vector tmp; + int ncols = sqlite3_column_count(stmt); + for (int i = 0; i < ncols; i++) { + tmp.emplace_back(reinterpret_cast(sqlite3_column_text(stmt, i))); + } + labels.push_back(tmp); + rc = sqlite3_step(stmt); + } + (void)sqlite3_finalize(stmt); + return SUCCESS; +} + +std::pair> ShardReader::GetLabelsFromBinaryFile( + int shard_id, const std::vector &columns, const std::vector> &label_offsets) { + std::string file_name = file_paths_[shard_id]; + std::vector res; + std::shared_ptr fs = std::make_shared(); + fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::out | std::ios::binary); + if (fs->fail()) { + MS_LOG(ERROR) << "File could not opened"; + return {FAILED, {}}; + } + + // init the return + for (unsigned int i = 0; i < label_offsets.size(); ++i) { + res.emplace_back(json{}); + } + + for (unsigned int i = 0; i < label_offsets.size(); ++i) { + const auto &labelOffset = label_offsets[i]; + uint64_t label_start = std::stoull(labelOffset[1]) + kInt64Len; + uint64_t label_end = std::stoull(labelOffset[2]); + int raw_page_id = std::stoi(labelOffset[0]); + auto len = label_end - label_start; + auto label_raw = std::vector(len); + auto &io_seekg = fs->seekg(page_size_ * raw_page_id + header_size_ + label_start, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + fs->close(); + return {FAILED, {}}; + } + + auto &io_read = fs->read(reinterpret_cast(&label_raw[0]), len); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + fs->close(); + return {FAILED, {}}; + } + + json label_json = json::from_msgpack(label_raw); + json tmp = label_json; + for (auto &col : columns) { + if (label_json.find(col) != label_json.end()) { + tmp[col] = label_json[col]; + } + } + res[i] = tmp; + } + return {SUCCESS, res}; +} + +std::pair> ShardReader::GetLabelsFromPage( + int page_id, int shard_id, const std::vector &columns, + const std::pair &criteria) { + // get page info from sqlite + auto db = database_paths_[shard_id]; + std::string sql = "SELECT PAGE_ID_RAW, PAGE_OFFSET_RAW,PAGE_OFFSET_RAW_END FROM INDEXES WHERE PAGE_ID_BLOB = " + + std::to_string(page_id); + std::vector> label_offsets; + if (!criteria.first.empty()) { + sql += " AND " + criteria.first + "_" + std::to_string(column_schema_id_[criteria.first]) + " = :criteria"; + if (QueryWithCriteria(db, sql, criteria.second, label_offsets) == FAILED) { + return {FAILED, {}}; + } + } else { + sql += ";"; + char *errmsg = nullptr; + int rc = sqlite3_exec(db, common::SafeCStr(sql), SelectCallback, &label_offsets, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(db); + return {FAILED, {}}; + } + MS_LOG(DEBUG) << "Get " << label_offsets.size() << "records from index."; + sqlite3_free(errmsg); + } + // get labels from binary file + return GetLabelsFromBinaryFile(shard_id, columns, label_offsets); +} + +std::pair> ShardReader::GetLabels(int page_id, int shard_id, + const std::vector &columns, + const std::pair &criteria) { + if (all_in_index_) { + auto db = database_paths_[shard_id]; + std::string fields; + for (unsigned int i = 0; i < columns.size(); ++i) { + if (i > 0) fields += ','; + uint64_t schema_id = column_schema_id_[columns[i]]; + fields += columns[i] + "_" + std::to_string(schema_id); + } + if (fields.empty()) fields = "*"; + std::vector> labels; + std::string sql = "SELECT " + fields + " FROM INDEXES WHERE PAGE_ID_BLOB = " + std::to_string(page_id); + if (!criteria.first.empty()) { + sql += " AND " + criteria.first + "_" + std::to_string(column_schema_id_[criteria.first]) + " = " + ":criteria"; + if (QueryWithCriteria(db, sql, criteria.second, labels) == FAILED) { + return {FAILED, {}}; + } + } else { + sql += ";"; + char *errmsg = nullptr; + int rc = sqlite3_exec(db, common::SafeCStr(sql), SelectCallback, &labels, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(db); + return {FAILED, {}}; + } else { + MS_LOG(DEBUG) << "Get " << static_cast(labels.size()) << "records from index."; + } + sqlite3_free(errmsg); + } + std::vector ret; + for (unsigned int i = 0; i < labels.size(); ++i) ret.emplace_back(json{}); + for (unsigned int i = 0; i < labels.size(); ++i) { + string json_str = "{"; + for (unsigned int j = 0; j < columns.size(); ++j) { + // construct string json "f1": value + json_str = json_str + "\"" + columns[j] + "\":" + labels[i][j]; + if (j < columns.size() - 1) { + json_str += ","; + } + } + json_str += "}"; + ret[i] = json::parse(json_str); + } + return {SUCCESS, ret}; + } + return GetLabelsFromPage(page_id, shard_id, columns, criteria); +} + +bool ResortRowGroups(std::tuple a, std::tuple b) { + return std::get<1>(a) < std::get<1>(b) || (std::get<1>(a) == std::get<1>(b) && std::get<0>(a) < std::get<0>(b)); +} + +MSRStatus ShardReader::Finish() { + { + std::lock_guard lck(mtx_delivery_); + interrupt_ = true; + } + cv_delivery_.notify_all(); + + // Wait for all threads to finish + for (auto &i_thread : thread_set_) { + if (i_thread.joinable()) { + i_thread.join(); + } + } + return SUCCESS; +} + +MSRStatus ShardReader::CountTotalRows(const std::string &file_path, int64_t *count) { + if (Init(file_path) == FAILED) { + return FAILED; + } + *count = num_rows_; + return SUCCESS; +} + +MSRStatus ShardReader::Open(const std::string &file_path, int n_consumer, + const std::vector &selected_columns, + const std::vector> &operators, const bool &block_reader) { + // Open file and set header by ShardReader + if (Init(file_path) == FAILED) { + return FAILED; + } + auto thread_limit = GetMaxThreadNum(); + if (n_consumer > thread_limit) { + n_consumer = thread_limit; + } + if (n_consumer < kMinConsumerCount) { + n_consumer = kMinConsumerCount; + } + CheckNlp(); + if (nlp_) { + selected_columns_ = selected_columns; + } else { + vector blob_fields = get_blob_fields().second; + for (unsigned int i = 0; i < selected_columns.size(); ++i) { + if (!std::any_of(blob_fields.begin(), blob_fields.end(), + [&selected_columns, i](std::string item) { return selected_columns[i] == item; })) { + selected_columns_.push_back(selected_columns[i]); + } + } + } + + if (CheckColumnList(selected_columns_) == FAILED) { + MS_LOG(ERROR) << "Illegal column list"; + return FAILED; + } + + // Initialize argument + shard_count_ = static_cast(file_paths_.size()); + n_consumer_ = n_consumer; + + operators_ = operators; + + if (block_reader) { + block_reader_ = true; + if (Open() == FAILED) { + return FAILED; + } + delivery_block_ = std::vector>, std::vector>>>( + kNumPageInBuffer, std::shared_ptr>, std::vector>>{}); + buf_ = std::vector>(kNumPageInBuffer, std::vector(page_size_)); + } else { + block_reader_ = false; + if (Open(n_consumer) == FAILED) { + return FAILED; + } + } + return SUCCESS; +} + +MSRStatus ShardReader::OpenPy(const std::string &file_path, const int &n_consumer, + const std::vector &selected_columns, + const std::vector> &operators) { + // Open file and set header by ShardReader + if (Init(file_path) == FAILED) { + return FAILED; + } + // should remove blob field from selected_columns when call from python + std::vector columns(selected_columns); + auto blob_fields = get_blob_fields().second; + for (auto &blob_field : blob_fields) { + auto it = std::find(selected_columns.begin(), selected_columns.end(), blob_field); + if (it != selected_columns.end()) { + columns.erase(columns.begin() + std::distance(selected_columns.begin(), it)); + } + } + if (CheckColumnList(columns) == FAILED) { + MS_LOG(ERROR) << "Illegal column list"; + return FAILED; + } + if (Open(n_consumer) == FAILED) { + return FAILED; + } + CheckNlp(); + // Initialize argument + shard_count_ = static_cast(file_paths_.size()); + n_consumer_ = n_consumer; + + // Initialize columns which will be read + selected_columns_ = selected_columns; + operators_ = operators; + + return SUCCESS; +} + +MSRStatus ShardReader::Launch(bool isSimpleReader) { + // Get all row groups' info + auto row_group_summary = ReadRowGroupSummary(); + + // Sort row group by (group_id, shard_id), prepare for parallel reading + std::sort(row_group_summary.begin(), row_group_summary.end(), ResortRowGroups); + CreateTasks(row_group_summary, operators_); + MS_LOG(INFO) << "Launching read threads"; + + if (isSimpleReader) return SUCCESS; + + // Start provider consumer threads + thread_set_ = std::vector(n_consumer_); + if (n_consumer_ <= 0 || n_consumer_ > kMaxConsumerCount) { + return FAILED; + } + + for (int x = 0; x < n_consumer_; ++x) { + if (block_reader_) { + thread_set_[x] = std::thread(&ShardReader::ConsumerByBlock, this, x); + } else { + thread_set_[x] = std::thread(&ShardReader::ConsumerByRow, this, x); + } + } + return SUCCESS; +} + +vector ShardReader::GetAllColumns() { + vector columns; + if (nlp_) { + for (auto &c : selected_columns_) { + for (auto &p : get_shard_header()->get_schemas()) { + auto schema = p->GetSchema()["schema"]; // make sure schema is not reference since error occurred in arm. + for (auto it = schema.begin(); it != schema.end(); ++it) { + if (it.key() == c) { + columns.push_back(c); + } + } + } + } + } else { + columns = selected_columns_; + } + return std::move(columns); +} + +MSRStatus ShardReader::CreateTasksByBlock(const std::vector> &row_group_summary, + const std::vector> &operators) { + for (const auto &rg : row_group_summary) { + auto shard_id = std::get<0>(rg); + auto group_id = std::get<1>(rg); + auto n_Rows = std::get<3>(rg); + tasks_.InsertTask(shard_id, group_id, std::vector{n_Rows}, json{}); + } + return SUCCESS; +} + +int ShardReader::CreateTasksByCategory(const std::vector> &row_group_summary, + const std::vector> &operators) { + vector columns = GetAllColumns(); + CheckIfColumnInIndex(columns); + + int category_operator = -1; + for (uint32_t i = 0; i < operators.size(); ++i) { + const auto &op = operators[i]; + if (std::dynamic_pointer_cast(op)) category_operator = static_cast(i); + } + + if (category_operator == -1) return category_operator; + + auto categories = std::dynamic_pointer_cast(operators[category_operator])->get_categories(); + + // Generate task list, a task will create a batch + std::vector categoryTasks(categories.size()); + for (uint32_t categoryNo = 0; categoryNo < categories.size(); ++categoryNo) { + for (const auto &rg : row_group_summary) { + auto shard_id = std::get<0>(rg); + auto group_id = std::get<1>(rg); + + auto details = ReadRowGroupCriteria(group_id, shard_id, categories[categoryNo], columns); + if (SUCCESS != std::get<0>(details)) { + return -2; + } + auto offsets = std::get<4>(details); + + auto number_of_rows = offsets.size(); + for (uint32_t iStart = 0; iStart < number_of_rows; iStart += 1) { + categoryTasks[categoryNo].InsertTask(shard_id, group_id, std::get<4>(details)[iStart], + std::get<5>(details)[iStart]); + } + } + MS_LOG(INFO) << "Category #" << categoryNo << " has " << categoryTasks[categoryNo].Size() << " tasks"; + } + tasks_ = ShardTask::Combine(categoryTasks); + return category_operator; +} + +MSRStatus ShardReader::CreateTasksByRow(const std::vector> &row_group_summary, + const std::vector> &operators) { + vector columns = GetAllColumns(); + CheckIfColumnInIndex(columns); + + auto ret = ReadAllRowGroup(columns); + if (std::get<0>(ret) != SUCCESS) { + return FAILED; + } + auto offsets = std::get<1>(ret); + auto local_columns = std::get<2>(ret); + if (shard_count_ <= kMaxShardCount) { + for (int shard_id = 0; shard_id < shard_count_; shard_id++) { + for (uint32_t i = 0; i < offsets[shard_id].size(); i += 1) { + tasks_.InsertTask(offsets[shard_id][i][0], offsets[shard_id][i][1], + std::vector{offsets[shard_id][i][2], offsets[shard_id][i][3]}, + local_columns[shard_id][i]); + } + } + } else { + return FAILED; + } + return SUCCESS; +} + +MSRStatus ShardReader::CreateTasks(const std::vector> &row_group_summary, + const std::vector> &operators) { + if (block_reader_) { + CreateTasksByBlock(row_group_summary, operators); + } else { + int category_operator = CreateTasksByCategory(row_group_summary, operators); + if (category_operator == -1) { + CreateTasksByRow(row_group_summary, operators); + } + if (category_operator == -2) { + return FAILED; + } + } + + for (uint32_t operator_no = 0; operator_no < operators.size(); operator_no++) { + const auto &op = operators[operator_no]; + if (std::dynamic_pointer_cast(op)) continue; + if (block_reader_ && std::dynamic_pointer_cast(op)) continue; + if (SUCCESS != (*op)(tasks_)) { + return FAILED; + } + } + + if (tasks_.permutation_.empty()) tasks_.MakePerm(); + num_rows_ = block_reader_ ? tasks_.SizeOfRows() : tasks_.Size(); + num_blocks_ = block_reader_ ? tasks_.Size() : 0; + MS_LOG(INFO) << "Total rows is " << num_rows_; + return SUCCESS; +} + +TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_id) { + // All tasks are done + if (task_id >= static_cast(tasks_.Size())) { + return std::make_pair(FAILED, std::vector, json>>()); + } + + // Pick up task from task list + auto task = tasks_.get_task_by_id(tasks_.permutation_[task_id]); + + auto shard_id = std::get<0>(std::get<0>(task)); + auto group_id = std::get<1>(std::get<0>(task)); + auto addr = std::get<1>(task); + const auto &ret = shard_header_->GetPageByGroupId(group_id, shard_id); + if (SUCCESS != ret.first) { + return std::make_pair(FAILED, std::vector, json>>()); + } + const std::shared_ptr &page = ret.second; + // Pack image list + std::vector images(addr[1] - addr[0]); + auto file_offset = header_size_ + page_size_ * (page->get_page_id()) + addr[0]; + + auto &io_seekg = file_streams_random_[consumer_id][shard_id]->seekg(file_offset, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + file_streams_random_[consumer_id][shard_id]->close(); + return std::make_pair(FAILED, std::vector, json>>()); + } + + auto &io_read = + file_streams_random_[consumer_id][shard_id]->read(reinterpret_cast(&images[0]), addr[1] - addr[0]); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + file_streams_random_[consumer_id][shard_id]->close(); + return std::make_pair(FAILED, std::vector, json>>()); + } + + // Deliver batch data to output map + std::vector, json>> batch; + if (nlp_) { + json blob_fields = json::from_msgpack(images); + + json merge; + if (selected_columns_.size() > 0) { + for (auto &col : selected_columns_) { + if (blob_fields.find(col) != blob_fields.end()) { + merge[col] = blob_fields[col]; + } + } + } else { + merge = blob_fields; + } + auto label_json = std::get<2>(task); + if (label_json != nullptr) { + merge.update(label_json); + } + batch.emplace_back(std::vector{}, std::move(merge)); + } else { + batch.emplace_back(std::move(images), std::move(std::get<2>(task))); + } + return std::make_pair(SUCCESS, std::move(batch)); +} + +MSRStatus ShardReader::ConsumerByRow(int consumer_id) { + // Set thread name + auto thread_id = kThreadName + std::to_string(consumer_id); + prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); + + // Loop forever + for (;;) { + int task_id = 0; + + // Get next task ID + task_id = task_id_++; + + // All tasks are done + if (task_id >= static_cast(tasks_.Size())) { + return FAILED; + } + const auto &ret = ConsumerOneTask(task_id, consumer_id); + if (SUCCESS != ret.first) { + return FAILED; + } + const auto &batch = ret.second; + // Hanging if maximum map size exceeded + // otherwise, set batch data in map + { + std::unique_lock lck(mtx_delivery_); + cv_delivery_.wait(lck, [task_id, this] { return interrupt_ || task_id <= deliver_id_ + kNumBatchInMap; }); + if (interrupt_) { + return SUCCESS; + } + delivery_map_[task_id] = std::make_shared, json>>>(std::move(batch)); + } + cv_iterator_.notify_one(); + } +} + +MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset, const int &page_length, + const int &buf_id) { + auto &io_seekg = file_streams_[shard_id]->seekg(page_offset, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + auto &io_read = file_streams_[shard_id]->read(reinterpret_cast(&buf_[buf_id][0]), page_length); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + return SUCCESS; +} + +MSRStatus ShardReader::ConsumerByBlock(int consumer_id) { + // Set thread name + auto thread_id = kThreadName + std::to_string(consumer_id); + prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); + + // Loop forever + for (;;) { + int task_id = 0; + + // Get next task ID + task_id = task_id_++; + + // All tasks are done, either quit or repeat again + if (task_id >= num_blocks_) { + std::unique_lock lck(mtx_delivery_); + cv_delivery_.wait(lck, [this] { return interrupt_ || task_id_ < num_blocks_; }); + if (interrupt_) { + return SUCCESS; + } + continue; + } + + // Pick up task from task list + auto task = tasks_.get_task_by_id(tasks_.permutation_[task_id]); + + auto shard_id = std::get<0>(std::get<0>(task)); + auto group_id = std::get<1>(std::get<0>(task)); + auto row_group_brief = ReadRowGroupBrief(group_id, shard_id, selected_columns_); + if (SUCCESS != std::get<0>(row_group_brief)) { + return FAILED; + } + auto page_length = std::get<2>(row_group_brief); + auto page_offset = std::get<3>(row_group_brief); + + MS_LOG(DEBUG) << "Block task " << task_id << tasks_.permutation_[task_id] << ", shard " << shard_id << ", group " + << group_id << ", page length " << page_length << ", page offset " << page_offset; + + // Deliver block data to output map + auto offset_and_labels = std::make_pair(std::get<4>(row_group_brief), std::get<5>(row_group_brief)); + + int deliver_id = deliver_id_; + // Hanging if maximum map size exceeded otherwise, set batch data in buffer + { + std::unique_lock lck(mtx_delivery_); + cv_delivery_.wait(lck, [task_id, this] { return interrupt_ || task_id < deliver_id_ + kNumPageInBuffer; }); + if (interrupt_) { + return SUCCESS; + } + } + + auto buf_id = task_id % kNumPageInBuffer; + delivery_block_[buf_id] = + std::make_shared>, std::vector>>(offset_and_labels); + + // Read blob + if (ReadBlob(shard_id, page_offset, page_length, buf_id) != SUCCESS) { + return FAILED; + } + + { + std::unique_lock lck(mtx_delivery_); + delivery_block_set_.insert(task_id); + } + cv_iterator_.notify_one(); + } +} + +std::shared_ptr, json>>> ShardReader::GetRowFromBuffer(int buf_id, + int rowId) { + auto &blob_page = buf_[buf_id]; + auto &offsets = (*delivery_block_[buf_id]).first; + auto &labels = (*delivery_block_[buf_id]).second; + auto &addr_start = offsets[rowId][0]; + auto &addr_end = offsets[rowId][1]; + std::vector images(blob_page.begin() + addr_start, blob_page.begin() + addr_end); + std::vector, json>> batch; + batch.emplace_back(std::move(images), std::move(labels[rowId])); + return std::make_shared, json>>>(std::move(batch)); +} + +std::vector, json>> ShardReader::GetBlockNext() { + if (deliver_id_ >= num_blocks_) { + return std::vector, json>>(); + } + + if (row_id_ == 0) { + std::unique_lock lck(mtx_delivery_); + cv_iterator_.wait(lck, [this] { return interrupt_ || (delivery_block_set_.count(deliver_id_) > 0); }); + + if (interrupt_) { + return std::vector, json>>(); + } + } + auto buf_id = deliver_id_ % kNumPageInBuffer; + auto res = GetRowFromBuffer(buf_id, row_id_); + + row_id_++; + if (row_id_ == (*delivery_block_[buf_id]).first.size()) { + row_id_ = 0; + { + std::unique_lock lck(mtx_delivery_); + delivery_block_set_.erase(deliver_id_++); + } + cv_delivery_.notify_all(); + } + + return *res; +} + +std::vector, json>> ShardReader::GetNext() { + if (block_reader_) return GetBlockNext(); + if (deliver_id_ >= static_cast(tasks_.Size())) { + return std::vector, json>>(); + } + + std::shared_ptr, json>>> res; + { + std::unique_lock lck(mtx_delivery_); + cv_iterator_.wait(lck, [this] { return interrupt_ || (delivery_map_.count(deliver_id_) > 0); }); + if (interrupt_) { + return std::vector, json>>(); + } + res = delivery_map_[deliver_id_]; + delivery_map_.erase(deliver_id_++); + } + + cv_delivery_.notify_all(); + + return *res; +} + +std::vector, json>> ShardReader::GetNextById(const int64_t &task_id, + const int32_t &consumer_id) { + if (interrupt_) { + return std::vector, json>>(); + } + if (block_reader_) { + return GetBlockNext(); + } + const auto &ret = ConsumerOneTask(task_id, consumer_id); + if (SUCCESS != ret.first) { + return std::vector, json>>(); + } + return std::move(ret.second); +} + +std::vector, pybind11::object>> ShardReader::GetNextPy() { + auto res = GetNext(); + vector, pybind11::object>> jsonData; + std::transform(res.begin(), res.end(), std::back_inserter(jsonData), + [](const std::tuple, json> &item) { + auto &j = std::get<1>(item); + pybind11::object obj = nlohmann::detail::FromJsonImpl(j); + return std::make_tuple(std::get<0>(item), std::move(obj)); + }); + return jsonData; +} + +void ShardReader::Reset() { + { + std::lock_guard lck(mtx_delivery_); + task_id_ = 0; + deliver_id_ = 0; + } + cv_delivery_.notify_all(); +} + +void ShardReader::ShuffleTask() { + for (const auto &op : operators_) { + if (block_reader_ || !std::dynamic_pointer_cast(op)) continue; + if (SUCCESS != (*op)(tasks_)) { + MS_LOG(WARNING) << "Reshuffle reader tasks failed."; + } + } +} + +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/io/shard_segment.cc b/mindspore/ccsrc/mindrecord/io/shard_segment.cc new file mode 100644 index 00000000000..94ef0d81677 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/io/shard_segment.cc @@ -0,0 +1,402 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_segment.h" +#include "common/utils.h" + +#include "./securec.h" +#include "mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +ShardSegment::ShardSegment() { set_all_in_index(false); } + +std::pair> ShardSegment::GetCategoryFields() { + // Skip if already populated + if (!candidate_category_fields_.empty()) return {SUCCESS, candidate_category_fields_}; + + std::string sql = "PRAGMA table_info(INDEXES);"; + std::vector> field_names; + + char *errmsg = nullptr; + int rc = sqlite3_exec(database_paths_[0], common::SafeCStr(sql), SelectCallback, &field_names, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(database_paths_[0]); + return {FAILED, vector{}}; + } else { + MS_LOG(INFO) << "Get " << static_cast(field_names.size()) << " records from index."; + } + + uint32_t idx = kStartFieldId; + while (idx < field_names.size()) { + if (field_names[idx].size() < 2) { + sqlite3_free(errmsg); + sqlite3_close(database_paths_[0]); + return {FAILED, vector{}}; + } + candidate_category_fields_.push_back(field_names[idx][1]); + idx += 2; + } + sqlite3_free(errmsg); + return {SUCCESS, candidate_category_fields_}; +} + +MSRStatus ShardSegment::SetCategoryField(std::string category_field) { + if (GetCategoryFields().first != SUCCESS) { + MS_LOG(ERROR) << "Get candidate category field failed"; + return FAILED; + } + category_field = category_field + "_0"; + if (std::any_of(std::begin(candidate_category_fields_), std::end(candidate_category_fields_), + [category_field](std::string x) { return x == category_field; })) { + current_category_field_ = category_field; + return SUCCESS; + } + MS_LOG(ERROR) << "Field " << category_field << " is not a candidate category field."; + return FAILED; +} + +std::pair ShardSegment::ReadCategoryInfo() { + MS_LOG(INFO) << "Read category begin"; + auto ret = WrapCategoryInfo(); + if (ret.first != SUCCESS) { + MS_LOG(ERROR) << "Get category info failed"; + return {FAILED, ""}; + } + // Convert category info to json string + auto category_json_string = ToJsonForCategory(ret.second); + + MS_LOG(INFO) << "Read category end"; + + return {SUCCESS, category_json_string}; +} + +std::pair>> ShardSegment::WrapCategoryInfo() { + std::map counter; + + std::string sql = "SELECT " + current_category_field_ + ", COUNT(" + current_category_field_ + + ") AS `value_occurrence` FROM indexes GROUP BY " + current_category_field_ + ";"; + + for (auto &db : database_paths_) { + std::vector> field_count; + + char *errmsg = nullptr; + int rc = sqlite3_exec(db, common::SafeCStr(sql), SelectCallback, &field_count, &errmsg); + if (rc != SQLITE_OK) { + MS_LOG(ERROR) << "Error in select statement, sql: " << sql << ", error: " << errmsg; + sqlite3_free(errmsg); + sqlite3_close(db); + return {FAILED, std::vector>()}; + } else { + MS_LOG(INFO) << "Get " << static_cast(field_count.size()) << " records from index."; + } + + for (const auto &field : field_count) { + counter[field[0]] += std::stoi(field[1]); + } + sqlite3_free(errmsg); + } + + int idx = 0; + std::vector> category_vec(counter.size()); + (void)std::transform(counter.begin(), counter.end(), category_vec.begin(), [&idx](std::tuple item) { + return std::make_tuple(idx++, std::get<0>(item), std::get<1>(item)); + }); + return {SUCCESS, std::move(category_vec)}; +} + +std::string ShardSegment::ToJsonForCategory(const std::vector> &tri_vec) { + std::vector category_json_vec; + for (auto q : tri_vec) { + json j; + j["id"] = std::get<0>(q); + j["name"] = std::get<1>(q); + j["count"] = std::get<2>(q); + + category_json_vec.emplace_back(j); + } + + json j_vec(category_json_vec); + json category_info; + category_info["key"] = current_category_field_; + category_info["categories"] = j_vec; + return category_info.dump(); +} + +std::pair>> ShardSegment::ReadAtPageById(int64_t category_id, + int64_t page_no, + int64_t n_rows_of_page) { + auto ret = WrapCategoryInfo(); + if (ret.first != SUCCESS) { + MS_LOG(ERROR) << "Get category info"; + return {FAILED, std::vector>{}}; + } + if (category_id >= static_cast(ret.second.size()) || category_id < 0) { + MS_LOG(ERROR) << "Illegal category id, id: " << category_id; + return {FAILED, std::vector>{}}; + } + int total_rows_in_category = std::get<2>(ret.second[category_id]); + // Quit if category not found or page number is out of range + if (total_rows_in_category <= 0 || page_no < 0 || n_rows_of_page <= 0 || + page_no * n_rows_of_page >= total_rows_in_category) { + MS_LOG(ERROR) << "Illegal page no / page size, page no: " << page_no << ", page size: " << n_rows_of_page; + return {FAILED, std::vector>{}}; + } + + std::vector> page; + auto row_group_summary = ReadRowGroupSummary(); + + uint64_t i_start = page_no * n_rows_of_page; + uint64_t i_end = std::min(static_cast(total_rows_in_category), (page_no + 1) * n_rows_of_page); + uint64_t idx = 0; + for (const auto &rg : row_group_summary) { + if (idx >= i_end) break; + + auto shard_id = std::get<0>(rg); + auto group_id = std::get<1>(rg); + auto details = ReadRowGroupCriteria( + group_id, shard_id, std::make_pair(CleanUp(current_category_field_), std::get<1>(ret.second[category_id]))); + if (SUCCESS != std::get<0>(details)) { + return {FAILED, std::vector>{}}; + } + auto offsets = std::get<4>(details); + uint64_t number_of_rows = offsets.size(); + if (idx + number_of_rows < i_start) { + idx += number_of_rows; + continue; + } + + for (uint64_t i = 0; i < number_of_rows; ++i, ++idx) { + if (idx >= i_start && idx < i_end) { + auto ret1 = PackImages(group_id, shard_id, offsets[i]); + if (SUCCESS != ret1.first) { + return {FAILED, std::vector>{}}; + } + page.push_back(std::move(ret1.second)); + } + } + } + + return {SUCCESS, std::move(page)}; +} + +std::pair> ShardSegment::PackImages(int group_id, int shard_id, + std::vector offset) { + const auto &ret = shard_header_->GetPageByGroupId(group_id, shard_id); + if (SUCCESS != ret.first) { + return {FAILED, std::vector()}; + } + const std::shared_ptr &blob_page = ret.second; + + // Pack image list + std::vector images(offset[1] - offset[0]); + auto file_offset = header_size_ + page_size_ * (blob_page->get_page_id()) + offset[0]; + auto &io_seekg = file_streams_random_[0][shard_id]->seekg(file_offset, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + file_streams_random_[0][shard_id]->close(); + return {FAILED, {}}; + } + + auto &io_read = file_streams_random_[0][shard_id]->read(reinterpret_cast(&images[0]), offset[1] - offset[0]); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + file_streams_random_[0][shard_id]->close(); + return {FAILED, {}}; + } + + return {SUCCESS, std::move(images)}; +} + +std::pair>> ShardSegment::ReadAtPageByName(std::string category_name, + int64_t page_no, + int64_t n_rows_of_page) { + auto ret = WrapCategoryInfo(); + if (ret.first != SUCCESS) { + MS_LOG(ERROR) << "Get category info"; + return {FAILED, std::vector>{}}; + } + for (const auto &categories : ret.second) { + if (std::get<1>(categories) == category_name) { + auto result = ReadAtPageById(std::get<0>(categories), page_no, n_rows_of_page); + return result; + } + } + + return {FAILED, std::vector>()}; +} + +std::pair, json>>> ShardSegment::ReadAllAtPageById( + int64_t category_id, int64_t page_no, int64_t n_rows_of_page) { + auto ret = WrapCategoryInfo(); + if (ret.first != SUCCESS || category_id >= static_cast(ret.second.size())) { + MS_LOG(ERROR) << "Illegal category id, id: " << category_id; + return {FAILED, std::vector, json>>{}}; + } + int total_rows_in_category = std::get<2>(ret.second[category_id]); + // Quit if category not found or page number is out of range + if (total_rows_in_category <= 0 || page_no < 0 || page_no * n_rows_of_page >= total_rows_in_category) { + MS_LOG(ERROR) << "Illegal page no: " << page_no << ", page size: " << n_rows_of_page; + return {FAILED, std::vector, json>>{}}; + } + + std::vector, json>> page; + auto row_group_summary = ReadRowGroupSummary(); + + int i_start = page_no * n_rows_of_page; + int i_end = std::min(static_cast(total_rows_in_category), (page_no + 1) * n_rows_of_page); + int idx = 0; + for (const auto &rg : row_group_summary) { + if (idx >= i_end) break; + + auto shard_id = std::get<0>(rg); + auto group_id = std::get<1>(rg); + auto details = ReadRowGroupCriteria( + group_id, shard_id, std::make_pair(CleanUp(current_category_field_), std::get<1>(ret.second[category_id]))); + if (SUCCESS != std::get<0>(details)) { + return {FAILED, std::vector, json>>{}}; + } + auto offsets = std::get<4>(details); + auto labels = std::get<5>(details); + + int number_of_rows = offsets.size(); + if (idx + number_of_rows < i_start) { + idx += number_of_rows; + continue; + } + + if (number_of_rows > static_cast(labels.size())) { + MS_LOG(ERROR) << "Illegal row number of page: " << number_of_rows; + return {FAILED, std::vector, json>>{}}; + } + for (int i = 0; i < number_of_rows; ++i, ++idx) { + if (idx >= i_start && idx < i_end) { + auto ret1 = PackImages(group_id, shard_id, offsets[i]); + if (SUCCESS != ret1.first) { + return {FAILED, std::vector, json>>{}}; + } + auto imageLabel = GetImageLabel(ret1.second, labels[i]); + page.emplace_back(std::move(std::get<0>(imageLabel)), std::move(std::get<1>(imageLabel))); + } + } + } + return {SUCCESS, std::move(page)}; +} + +std::pair, json>>> ShardSegment::ReadAllAtPageByName( + std::string category_name, int64_t page_no, int64_t n_rows_of_page) { + auto ret = WrapCategoryInfo(); + if (ret.first != SUCCESS) { + MS_LOG(ERROR) << "Get category info"; + return {FAILED, std::vector, json>>{}}; + } + for (const auto &categories : ret.second) { + if (std::get<1>(categories) == category_name) { + auto result = ReadAllAtPageById(std::get<0>(categories), page_no, n_rows_of_page); + return {SUCCESS, result.second}; + } + } + + return {SUCCESS, std::vector, json>>{}}; +} + +std::pair, pybind11::object>>> ShardSegment::ReadAtPageByIdPy( + int64_t category_id, int64_t page_no, int64_t n_rows_of_page) { + auto res = ReadAllAtPageById(category_id, page_no, n_rows_of_page); + if (res.first != SUCCESS) { + return {FAILED, std::vector, pybind11::object>>{}}; + } + + vector, pybind11::object>> json_data; + std::transform(res.second.begin(), res.second.end(), std::back_inserter(json_data), + [](const std::tuple, json> &item) { + auto &j = std::get<1>(item); + pybind11::object obj = nlohmann::detail::FromJsonImpl(j); + return std::make_tuple(std::get<0>(item), std::move(obj)); + }); + return {SUCCESS, std::move(json_data)}; +} + +std::pair, pybind11::object>>> ShardSegment::ReadAtPageByNamePy( + std::string category_name, int64_t page_no, int64_t n_rows_of_page) { + auto res = ReadAllAtPageByName(category_name, page_no, n_rows_of_page); + if (res.first != SUCCESS) { + return {FAILED, std::vector, pybind11::object>>{}}; + } + vector, pybind11::object>> json_data; + std::transform(res.second.begin(), res.second.end(), std::back_inserter(json_data), + [](const std::tuple, json> &item) { + auto &j = std::get<1>(item); + pybind11::object obj = nlohmann::detail::FromJsonImpl(j); + return std::make_tuple(std::get<0>(item), std::move(obj)); + }); + return {SUCCESS, std::move(json_data)}; +} + +std::pair> ShardSegment::get_blob_fields() { + std::vector blob_fields; + for (auto &p : get_shard_header()->get_schemas()) { + // assume one schema + const auto &fields = p->get_blob_fields(); + blob_fields.assign(fields.begin(), fields.end()); + break; + } + return std::make_pair(get_nlp_flag() ? kNLP : kCV, blob_fields); +} + +std::tuple, json> ShardSegment::GetImageLabel(std::vector images, json label) { + if (get_nlp_flag()) { + vector columns; + for (auto &p : get_shard_header()->get_schemas()) { + auto schema = p->GetSchema()["schema"]; // make sure schema is not reference since error occurred in arm. + auto schema_items = schema.items(); + using it_type = decltype(schema_items.begin()); + std::transform(schema_items.begin(), schema_items.end(), std::back_inserter(columns), + [](it_type item) { return item.key(); }); + } + + json blob_fields = json::from_msgpack(images); + json merge; + if (columns.size() > 0) { + for (auto &col : columns) { + if (blob_fields.find(col) != blob_fields.end()) { + merge[col] = blob_fields[col]; + } + } + } else { + merge = blob_fields; + } + merge.update(label); + return std::make_tuple(std::vector{}, merge); + } + return std::make_tuple(images, label); +} + +std::string ShardSegment::CleanUp(std::string field_name) { + while (field_name.back() >= '0' && field_name.back() <= '9') field_name.pop_back(); + field_name.pop_back(); + return field_name; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/io/shard_writer.cc b/mindspore/ccsrc/mindrecord/io/shard_writer.cc new file mode 100644 index 00000000000..54cf0e156b6 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/io/shard_writer.cc @@ -0,0 +1,1054 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_writer.h" +#include "common/utils.h" +#include "mindrecord/include/common/shard_utils.h" +#include "./securec.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::DEBUG; +using mindspore::MsLogLevel::ERROR; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +ShardWriter::ShardWriter() + : shard_count_(1), + header_size_(kDefaultHeaderSize), + page_size_(kDefaultPageSize), + row_count_(0), + schema_count_(1) {} + +ShardWriter::~ShardWriter() { + for (int i = static_cast(file_streams_.size()) - 1; i >= 0; i--) { + file_streams_[i]->close(); + } +} + +MSRStatus ShardWriter::Open(const std::vector &paths, bool append) { + shard_count_ = paths.size(); + if (shard_count_ > kMaxShardCount || shard_count_ == 0) { + MS_LOG(ERROR) << "The Shard Count greater than max value or equal to 0."; + return FAILED; + } + if (schema_count_ > kMaxSchemaCount) { + MS_LOG(ERROR) << "The schema Count greater than max value."; + return FAILED; + } + + // Get full path from file name + for (const auto &path : paths) { + if (!CheckIsValidUtf8(path)) { + MS_LOG(ERROR) << "The filename contains invalid uft-8 data: " << path << "."; + return FAILED; + } + char resolved_path[PATH_MAX] = {0}; + char buf[PATH_MAX] = {0}; + if (strncpy_s(buf, PATH_MAX, common::SafeCStr(path), path.length()) != EOK) { + MS_LOG(ERROR) << "Securec func failed"; + return FAILED; + } + if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) { + MS_LOG(ERROR) << "Invalid file path"; + return FAILED; + } + if (realpath(common::SafeCStr(path), resolved_path) == nullptr) { + MS_LOG(DEBUG) << "Path " << resolved_path; + } + file_paths_.emplace_back(string(resolved_path)); + } + + // Open files + for (const auto &file : file_paths_) { + std::shared_ptr fs = std::make_shared(); + fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); + if (fs->fail()) { + fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); + if (fs->fail()) { + MS_LOG(ERROR) << "File could not opened"; + return FAILED; + } + } else { + if (!append) { + MS_LOG(ERROR) << "MindRecord file already existed"; + return FAILED; + } + } + MS_LOG(INFO) << "Open shard file successfully."; + file_streams_.push_back(fs); + } + return SUCCESS; +} + +MSRStatus ShardWriter::OpenForAppend(const std::string &path) { + if (!IsLegalFile(path)) { + return FAILED; + } + ShardHeader sh = ShardHeader(); + if (sh.Build(path) == FAILED) { + return FAILED; + } + shard_header_ = std::make_shared(sh); + auto paths = shard_header_->get_shard_addresses(); + MSRStatus ret = set_header_size(shard_header_->get_header_size()); + if (ret == FAILED) { + return FAILED; + } + ret = set_page_size(shard_header_->get_page_size()); + if (ret == FAILED) { + return FAILED; + } + ret = Open(paths, true); + if (ret == FAILED) { + MS_LOG(ERROR) << "Open file failed"; + return FAILED; + } + return SUCCESS; +} + +MSRStatus ShardWriter::Commit() { + if (WriteShardHeader() == FAILED) { + MS_LOG(ERROR) << "Write metadata failed"; + return FAILED; + } + MS_LOG(INFO) << "Write metadata successfully."; + return SUCCESS; +} + +MSRStatus ShardWriter::SetShardHeader(std::shared_ptr header_data) { + MSRStatus ret = header_data->InitByFiles(file_paths_); + if (ret == FAILED) { + return FAILED; + } + + // set fields in mindrecord when empty + std::vector> fields = header_data->get_fields(); + if (fields.empty()) { + MS_LOG(DEBUG) << "Missing index fields by user, auto generate index fields."; + std::vector> schemas = header_data->get_schemas(); + for (const auto &schema : schemas) { + json jsonSchema = schema->GetSchema()["schema"]; + for (const auto &el : jsonSchema.items()) { + if (el.value()["type"] == "string" || + (el.value()["type"] == "int32" && el.value().find("shape") == el.value().end()) || + (el.value()["type"] == "int64" && el.value().find("shape") == el.value().end()) || + (el.value()["type"] == "float32" && el.value().find("shape") == el.value().end()) || + (el.value()["type"] == "float64" && el.value().find("shape") == el.value().end())) { + fields.emplace_back(std::make_pair(schema->get_schema_id(), el.key())); + } + } + } + // only blob data + if (!fields.empty()) { + ret = header_data->AddIndexFields(fields); + if (ret == FAILED) { + MS_LOG(ERROR) << "Add index field failed"; + return FAILED; + } + } + } + + shard_header_ = header_data; + shard_header_->set_header_size(header_size_); + shard_header_->set_page_size(page_size_); + return SUCCESS; +} + +MSRStatus ShardWriter::set_header_size(const uint64_t &header_size) { + // header_size [16KB, 128MB] + if (header_size < kMinHeaderSize || header_size > kMaxHeaderSize) { + MS_LOG(ERROR) << "Header size should between 16KB and 128MB."; + return FAILED; + } + if (header_size % 4 != 0) { + MS_LOG(ERROR) << "Header size should be divided by four."; + return FAILED; + } + + header_size_ = header_size; + return SUCCESS; +} + +MSRStatus ShardWriter::set_page_size(const uint64_t &page_size) { + // PageSize [32KB, 256MB] + if (page_size < kMinPageSize || page_size > kMaxPageSize) { + MS_LOG(ERROR) << "Page size should between 16KB and 256MB."; + return FAILED; + } + if (page_size % 4 != 0) { + MS_LOG(ERROR) << "Page size should be divided by four."; + return FAILED; + } + page_size_ = page_size; + return SUCCESS; +} + +void ShardWriter::DeleteErrorData(std::map> &raw_data, + std::vector> &blob_data) { + // get wrong data location + std::set> delete_set; + for (auto &err_mg : err_mg_) { + uint64_t id = err_mg.first; + auto sub_err_mg = err_mg.second; + for (auto &subMg : sub_err_mg) { + int loc = subMg.first; + std::string message = subMg.second; + MS_LOG(ERROR) << "For schema " << id << ", " << loc + 1 << " th data is wrong: " << message; + (void)delete_set.insert(loc); + } + } + + auto it = raw_data.begin(); + if (delete_set.size() == it->second.size()) { + raw_data.clear(); + blob_data.clear(); + return; + } + + // delete wrong raw data + for (auto &loc : delete_set) { + // delete row data + for (auto &raw : raw_data) { + (void)raw.second.erase(raw.second.begin() + loc); + } + + // delete blob data + (void)blob_data.erase(blob_data.begin() + loc); + } +} + +void ShardWriter::PopulateMutexErrorData(const int &row, const std::string &message, + std::map &err_raw_data) { + std::lock_guard lock(check_mutex_); + (void)err_raw_data.insert(std::make_pair(row, message)); +} + +MSRStatus ShardWriter::CheckDataTypeAndValue(const std::string &key, const json &value, const json &data, const int &i, + std::map &err_raw_data) { + auto data_type = std::string(value["type"].get()); + + if ((data_type == "int32" && !data[key].is_number_integer()) || + (data_type == "int64" && !data[key].is_number_integer()) || + (data_type == "float32" && !data[key].is_number_float()) || + (data_type == "float64" && !data[key].is_number_float()) || (data_type == "string" && !data[key].is_string())) { + std::string message = "field: " + key + " type : " + data_type + " value: " + data[key].dump() + " is not matched"; + PopulateMutexErrorData(i, message, err_raw_data); + return FAILED; + } + + if (data_type == "int32" && data[key].is_number_integer()) { + int64_t temp_value = data[key]; + if (static_cast(temp_value) < static_cast(std::numeric_limits::min()) && + static_cast(temp_value) > static_cast(std::numeric_limits::max())) { + std::string message = + "field: " + key + " type : " + data_type + " value: " + data[key].dump() + " is out of range"; + PopulateMutexErrorData(i, message, err_raw_data); + return FAILED; + } + } + return SUCCESS; +} + +void ShardWriter::CheckSliceData(int start_row, int end_row, json schema, const std::vector &sub_raw_data, + std::map &err_raw_data) { + if (start_row < 0 || start_row > end_row || end_row > static_cast(sub_raw_data.size())) { + return; + } + for (int i = start_row; i < end_row; i++) { + json data = sub_raw_data[i]; + + for (auto iter = schema.begin(); iter != schema.end(); iter++) { + std::string key = iter.key(); + json value = iter.value(); + if (data.find(key) == data.end()) { + std::string message = "there is not '" + key + "' object in the raw data"; + PopulateMutexErrorData(i, message, err_raw_data); + break; + } + + if (value.size() == kInt2) { + // Skip check since all shaped data will store as blob + continue; + } + + if (CheckDataTypeAndValue(key, value, data, i, err_raw_data) != SUCCESS) { + break; + } + } + } +} + +MSRStatus ShardWriter::CheckData(const std::map> &raw_data) { + auto rawdata_iter = raw_data.begin(); + + // make sure rawdata match schema + for (; rawdata_iter != raw_data.end(); ++rawdata_iter) { + // used for storing error + std::map sub_err_mg; + int schema_id = rawdata_iter->first; + auto result = shard_header_->GetSchemaByID(schema_id); + if (result.second != SUCCESS) { + return FAILED; + } + json schema = result.first->GetSchema()["schema"]; + for (const auto &field : result.first->get_blob_fields()) { + (void)schema.erase(field); + } + std::vector sub_raw_data = rawdata_iter->second; + + // calculate start position and end position for each thread + int batch_size = rawdata_iter->second.size() / shard_count_; + int thread_num = shard_count_; + if (thread_num <= 0) { + return FAILED; + } + if (thread_num > kMaxThreadCount) { + thread_num = kMaxThreadCount; + } + std::vector thread_set(thread_num); + + // start multiple thread + int start_row = 0, end_row = 0; + for (int x = 0; x < thread_num; ++x) { + if (x != thread_num - 1) { + start_row = batch_size * x; + end_row = batch_size * (x + 1); + } else { + start_row = batch_size * x; + end_row = rawdata_iter->second.size(); + } + thread_set[x] = std::thread(&ShardWriter::CheckSliceData, this, start_row, end_row, schema, + std::ref(sub_raw_data), std::ref(sub_err_mg)); + } + if (thread_num > kMaxThreadCount) { + return FAILED; + } + // Wait for threads done + for (int x = 0; x < thread_num; ++x) { + thread_set[x].join(); + } + + (void)err_mg_.insert(std::make_pair(schema_id, sub_err_mg)); + } + return SUCCESS; +} + +std::tuple ShardWriter::ValidateRawData(std::map> &raw_data, + std::vector> &blob_data, bool sign) { + auto rawdata_iter = raw_data.begin(); + schema_count_ = raw_data.size(); + std::tuple failed(FAILED, 0, 0); + if (schema_count_ == 0) { + MS_LOG(ERROR) << "Data size is zero"; + return failed; + } + + // keep schema_id + std::set schema_ids; + row_count_ = (rawdata_iter->second).size(); + MS_LOG(DEBUG) << "Schema count is " << schema_count_; + + // Determine if the number of schemas is the same + if (shard_header_->get_schemas().size() != schema_count_) { + MS_LOG(ERROR) << "Data size is not equal with the schema size"; + return failed; + } + + // Determine raw_data size == blob_data size + if (raw_data[0].size() != blob_data.size()) { + MS_LOG(ERROR) << "Raw data size is not equal blob data size"; + return failed; + } + + // Determine whether the number of samples corresponding to each schema is the same + for (rawdata_iter = raw_data.begin(); rawdata_iter != raw_data.end(); ++rawdata_iter) { + if (row_count_ != rawdata_iter->second.size()) { + MS_LOG(ERROR) << "Data size is not equal"; + return failed; + } + (void)schema_ids.insert(rawdata_iter->first); + } + const std::vector> &schemas = shard_header_->get_schemas(); + if (std::any_of(schemas.begin(), schemas.end(), [schema_ids](const std::shared_ptr &schema) { + return schema_ids.find(schema->get_schema_id()) == schema_ids.end(); + })) { + // There is not enough data which is not matching the number of schema + MS_LOG(ERROR) << "Input rawdata schema id do not match real schema id."; + return failed; + } + + if (!sign) { + std::tuple success(SUCCESS, schema_count_, row_count_); + return success; + } + + // check the data according the schema + if (CheckData(raw_data) != SUCCESS) { + MS_LOG(ERROR) << "Data validate check failed"; + return std::tuple(FAILED, schema_count_, row_count_); + } + + // delete wrong data from raw data + DeleteErrorData(raw_data, blob_data); + + // update raw count + row_count_ = row_count_ - err_mg_.begin()->second.size(); + std::tuple success(SUCCESS, schema_count_, row_count_); + return success; +} + +void ShardWriter::FillArray(int start, int end, std::map> &raw_data, + std::vector> &bin_data) { + // Prevent excessive thread opening and cause cross-border + if (start >= end) { + flag_ = true; + return; + } + int schema_count = static_cast(raw_data.size()); + std::map>::const_iterator rawdata_iter; + for (int x = start; x < end; ++x) { + int cnt = 0; + for (rawdata_iter = raw_data.begin(); rawdata_iter != raw_data.end(); ++rawdata_iter) { + const json &line = raw_data.at(rawdata_iter->first)[x]; + std::vector bline = json::to_msgpack(line); + + // Storage form is [Sample1-Schema1, Sample1-Schema2, Sample2-Schema1, Sample2-Schema2] + bin_data[x * schema_count + cnt] = bline; + cnt++; + } + } +} + +MSRStatus ShardWriter::WriteRawData(std::map> &raw_data, + std::vector> &blob_data, bool sign) { + // check the free disk size + auto st_space = GetDiskSize(file_paths_[0], kFreeSize); + if (st_space.first != SUCCESS || st_space.second < kMinFreeDiskSize) { + MS_LOG(ERROR) << "IO error / there is no free disk to be used"; + return FAILED; + } + + // Add 4-bytes dummy blob data if no any blob fields + if (blob_data.size() == 0 && raw_data.size() > 0) { + blob_data = std::vector>(raw_data[0].size(), std::vector(kUnsignedInt4, 0)); + } + + // Add dummy id if all are blob fields + if (blob_data.size() > 0 && raw_data.size() == 0) { + raw_data.insert(std::pair>(0, std::vector(blob_data.size(), kDummyId))); + } + + auto v = ValidateRawData(raw_data, blob_data, sign); + if (std::get<0>(v) == FAILED) { + MS_LOG(ERROR) << "Validate raw data failed"; + return FAILED; + } + + // Get the count of schemas and rows + int schema_count = std::get<1>(v); + int row_count = std::get<2>(v); + + if (row_count == kInt0) { + MS_LOG(INFO) << "Raw data size is 0."; + return SUCCESS; + } + + std::vector> bin_raw_data(row_count * schema_count); + + // Serialize raw data + if (SerializeRawData(raw_data, bin_raw_data, row_count) == FAILED) { + MS_LOG(ERROR) << "Serialize raw data failed"; + return FAILED; + } + + // Set row size of raw data + if (SetRawDataSize(bin_raw_data) == FAILED) { + MS_LOG(ERROR) << "Set raw data size failed"; + return FAILED; + } + + // Set row size of blob data + if (SetBlobDataSize(blob_data) == FAILED) { + MS_LOG(ERROR) << "Set blob data size failed"; + return FAILED; + } + + // Write data to disk with multi threads + if (ParallelWriteData(blob_data, bin_raw_data) == FAILED) { + MS_LOG(ERROR) << "Parallel write data failed"; + return FAILED; + } + MS_LOG(INFO) << "Write " << bin_raw_data.size() << " records successfully."; + + return SUCCESS; +} + +MSRStatus ShardWriter::WriteRawData(std::map> &raw_data, + std::map> &blob_data, bool sign) { + std::map> raw_data_json; + std::map> blob_data_json; + + (void)std::transform(raw_data.begin(), raw_data.end(), std::inserter(raw_data_json, raw_data_json.end()), + [](const std::pair> &pair) { + auto &py_raw_data = pair.second; + std::vector json_raw_data; + (void)std::transform(py_raw_data.begin(), py_raw_data.end(), std::back_inserter(json_raw_data), + [](const py::handle &obj) { return nlohmann::detail::ToJsonImpl(obj); }); + return std::make_pair(pair.first, std::move(json_raw_data)); + }); + + (void)std::transform(blob_data.begin(), blob_data.end(), std::inserter(blob_data_json, blob_data_json.end()), + [](const std::pair> &pair) { + auto &py_blob_data = pair.second; + std::vector jsonBlobData; + (void)std::transform(py_blob_data.begin(), py_blob_data.end(), + std::back_inserter(jsonBlobData), + [](const py::handle &obj) { return nlohmann::detail::ToJsonImpl(obj); }); + return std::make_pair(pair.first, std::move(jsonBlobData)); + }); + + // Serialize blob page + auto blob_data_iter = blob_data.begin(); + auto schema_count = blob_data.size(); + auto row_count = blob_data_iter->second.size(); + + std::vector> bin_blob_data(row_count * schema_count); + // Serialize blob data + if (SerializeRawData(blob_data_json, bin_blob_data, row_count) == FAILED) { + MS_LOG(ERROR) << "Serialize raw data failed in write raw data"; + return FAILED; + } + return WriteRawData(raw_data_json, bin_blob_data, sign); +} + +MSRStatus ShardWriter::WriteRawData(std::map> &raw_data, + vector> &blob_data, bool sign) { + std::map> raw_data_json; + (void)std::transform(raw_data.begin(), raw_data.end(), std::inserter(raw_data_json, raw_data_json.end()), + [](const std::pair> &pair) { + auto &py_raw_data = pair.second; + std::vector json_raw_data; + (void)std::transform(py_raw_data.begin(), py_raw_data.end(), std::back_inserter(json_raw_data), + [](const py::handle &obj) { return nlohmann::detail::ToJsonImpl(obj); }); + return std::make_pair(pair.first, std::move(json_raw_data)); + }); + return WriteRawData(raw_data_json, blob_data, sign); +} + +MSRStatus ShardWriter::ParallelWriteData(const std::vector> &blob_data, + const std::vector> &bin_raw_data) { + auto shards = BreakIntoShards(); + // define the number of thread + int thread_num = static_cast(shard_count_); + if (thread_num < 0) { + return FAILED; + } + if (thread_num > kMaxThreadCount) { + thread_num = kMaxThreadCount; + } + int left_thread = shard_count_; + int current_thread = 0; + while (left_thread) { + if (left_thread < thread_num) { + thread_num = left_thread; + } + // Start one thread for one shard + std::vector thread_set(thread_num); + if (thread_num <= kMaxThreadCount) { + for (int x = 0; x < thread_num; ++x) { + int start_row = shards[current_thread + x].first; + int end_row = shards[current_thread + x].second; + thread_set[x] = std::thread(&ShardWriter::WriteByShard, this, current_thread + x, start_row, end_row, + std::ref(blob_data), std::ref(bin_raw_data)); + } + // Wait for threads done + for (int x = 0; x < thread_num; ++x) { + thread_set[x].join(); + } + left_thread -= thread_num; + current_thread += thread_num; + } + } + return SUCCESS; +} + +MSRStatus ShardWriter::WriteByShard(int shard_id, int start_row, int end_row, + const std::vector> &blob_data, + const std::vector> &bin_raw_data) { + MS_LOG(DEBUG) << "Shard: " << shard_id << ", start: " << start_row << ", end: " << end_row + << ", schema size: " << schema_count_; + if (start_row == end_row) { + return SUCCESS; + } + vector> rows_in_group; + std::shared_ptr last_raw_page = nullptr; + std::shared_ptr last_blob_page = nullptr; + SetLastRawPage(shard_id, last_raw_page); + SetLastBlobPage(shard_id, last_blob_page); + + if (CutRowGroup(start_row, end_row, blob_data, rows_in_group, last_raw_page, last_blob_page) == FAILED) { + MS_LOG(ERROR) << "Cut row group failed"; + return FAILED; + } + + if (AppendBlobPage(shard_id, blob_data, rows_in_group, last_blob_page) == FAILED) { + MS_LOG(ERROR) << "Append bolb page failed"; + return FAILED; + } + + if (NewBlobPage(shard_id, blob_data, rows_in_group, last_blob_page) == FAILED) { + MS_LOG(ERROR) << "New blob page failed"; + return FAILED; + } + + if (ShiftRawPage(shard_id, rows_in_group, last_raw_page) == FAILED) { + MS_LOG(ERROR) << "Shit raw page failed"; + return FAILED; + } + + if (WriteRawPage(shard_id, rows_in_group, last_raw_page, bin_raw_data) == FAILED) { + MS_LOG(ERROR) << "Write raw page failed"; + return FAILED; + } + + return SUCCESS; +} + +MSRStatus ShardWriter::CutRowGroup(int start_row, int end_row, const std::vector> &blob_data, + std::vector> &rows_in_group, + const std::shared_ptr &last_raw_page, + const std::shared_ptr &last_blob_page) { + auto n_byte_blob = last_blob_page ? last_blob_page->get_page_size() : 0; + + auto last_raw_page_size = last_raw_page ? last_raw_page->get_page_size() : 0; + auto last_raw_offset = last_raw_page ? last_raw_page->get_last_row_group_id().second : 0; + auto n_byte_raw = last_raw_page_size - last_raw_offset; + + int page_start_row = start_row; + if (start_row > end_row) { + return FAILED; + } + if (end_row > static_cast(blob_data_size_.size()) || end_row > static_cast(raw_data_size_.size())) { + return FAILED; + } + for (int i = start_row; i < end_row; ++i) { + // n_byte_blob(0) indicate appendBlobPage + if (n_byte_blob == 0 || n_byte_blob + blob_data_size_[i] > page_size_ || + n_byte_raw + raw_data_size_[i] > page_size_) { + rows_in_group.emplace_back(page_start_row, i); + page_start_row = i; + n_byte_blob = blob_data_size_[i]; + n_byte_raw = raw_data_size_[i]; + } else { + n_byte_blob += blob_data_size_[i]; + n_byte_raw += raw_data_size_[i]; + } + } + + // Not forget last one + rows_in_group.emplace_back(page_start_row, end_row); + return SUCCESS; +} + +MSRStatus ShardWriter::AppendBlobPage(const int &shard_id, const std::vector> &blob_data, + const std::vector> &rows_in_group, + const std::shared_ptr &last_blob_page) { + auto blob_row = rows_in_group[0]; + if (blob_row.first == blob_row.second) return SUCCESS; + + // Write disk + auto page_id = last_blob_page->get_page_id(); + auto bytes_page = last_blob_page->get_page_size(); + auto &io_seekp = file_streams_[shard_id]->seekp(page_size_ * page_id + header_size_ + bytes_page, std::ios::beg); + if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) { + MS_LOG(ERROR) << "File seekp failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + (void)FlushBlobChunk(file_streams_[shard_id], blob_data, blob_row); + + // Update last blob page + bytes_page += std::accumulate(blob_data_size_.begin() + blob_row.first, blob_data_size_.begin() + blob_row.second, 0); + last_blob_page->set_page_size(bytes_page); + uint64_t end_row = last_blob_page->get_end_row_id() + blob_row.second - blob_row.first; + last_blob_page->set_end_row_id(end_row); + (void)shard_header_->SetPage(last_blob_page); + return SUCCESS; +} + +MSRStatus ShardWriter::NewBlobPage(const int &shard_id, const std::vector> &blob_data, + const std::vector> &rows_in_group, + const std::shared_ptr &last_blob_page) { + auto page_id = shard_header_->GetLastPageId(shard_id); + auto page_type_id = last_blob_page ? last_blob_page->get_page_type_id() : -1; + auto current_row = last_blob_page ? last_blob_page->get_end_row_id() : 0; + // index(0) indicate appendBlobPage + for (uint32_t i = 1; i < rows_in_group.size(); ++i) { + auto blob_row = rows_in_group[i]; + + // Write 1 blob page to disk + auto &io_seekp = file_streams_[shard_id]->seekp(page_size_ * (page_id + 1) + header_size_, std::ios::beg); + if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) { + MS_LOG(ERROR) << "File seekp failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + (void)FlushBlobChunk(file_streams_[shard_id], blob_data, blob_row); + // Create new page info for header + auto page_size = + std::accumulate(blob_data_size_.begin() + blob_row.first, blob_data_size_.begin() + blob_row.second, 0); + std::vector> row_group_ids; + auto start_row = current_row; + auto end_row = start_row + blob_row.second - blob_row.first; + auto page = Page(++page_id, shard_id, kPageTypeBlob, ++page_type_id, start_row, end_row, row_group_ids, page_size); + (void)shard_header_->AddPage(std::make_shared(page)); + current_row = end_row; + } + return SUCCESS; +} + +MSRStatus ShardWriter::ShiftRawPage(const int &shard_id, const std::vector> &rows_in_group, + std::shared_ptr &last_raw_page) { + auto blob_row = rows_in_group[0]; + if (blob_row.first == blob_row.second) return SUCCESS; + auto last_raw_page_size = last_raw_page ? last_raw_page->get_page_size() : 0; + if (std::accumulate(raw_data_size_.begin() + blob_row.first, raw_data_size_.begin() + blob_row.second, 0) + + last_raw_page_size <= + page_size_) { + return SUCCESS; + } + auto page_id = shard_header_->GetLastPageId(shard_id); + auto last_row_group_id_offset = last_raw_page->get_last_row_group_id().second; + auto last_raw_page_id = last_raw_page->get_page_id(); + auto shift_size = last_raw_page_size - last_row_group_id_offset; + + std::vector buf(shift_size); + + // Read last row group from previous raw data page + if (shard_id < 0 || shard_id >= file_streams_.size()) { + return FAILED; + } + + auto &io_seekg = file_streams_[shard_id]->seekg( + page_size_ * last_raw_page_id + header_size_ + last_row_group_id_offset, std::ios::beg); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + MS_LOG(ERROR) << "File seekg failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + auto &io_read = file_streams_[shard_id]->read(reinterpret_cast(&buf[0]), buf.size()); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + // Merge into new row group at new raw data page + auto &io_seekp = file_streams_[shard_id]->seekp(page_size_ * (page_id + 1) + header_size_, std::ios::beg); + if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) { + MS_LOG(ERROR) << "File seekp failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + auto &io_handle = file_streams_[shard_id]->write(reinterpret_cast(&buf[0]), buf.size()); + if (!io_handle.good() || io_handle.fail() || io_handle.bad()) { + MS_LOG(ERROR) << "File write failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + last_raw_page->DeleteLastGroupId(); + (void)shard_header_->SetPage(last_raw_page); + + // Refresh page info in header + int row_group_id = last_raw_page->get_last_row_group_id().first + 1; + std::vector> row_group_ids; + row_group_ids.emplace_back(row_group_id, 0); + int page_type_id = last_raw_page->get_page_id(); + auto page = Page(++page_id, shard_id, kPageTypeRaw, ++page_type_id, 0, 0, row_group_ids, shift_size); + (void)shard_header_->AddPage(std::make_shared(page)); + + // Reset: last raw page + SetLastRawPage(shard_id, last_raw_page); + return SUCCESS; +} + +MSRStatus ShardWriter::WriteRawPage(const int &shard_id, const std::vector> &rows_in_group, + std::shared_ptr &last_raw_page, + const std::vector> &bin_raw_data) { + int last_row_group_id = last_raw_page ? last_raw_page->get_last_row_group_id().first : -1; + for (uint32_t i = 0; i < rows_in_group.size(); ++i) { + const auto &blob_row = rows_in_group[i]; + if (blob_row.first == blob_row.second) continue; + auto raw_size = + std::accumulate(raw_data_size_.begin() + blob_row.first, raw_data_size_.begin() + blob_row.second, 0); + if (!last_raw_page) { + EmptyRawPage(shard_id, last_raw_page); + } else if (last_raw_page->get_page_size() + raw_size > page_size_) { + (void)shard_header_->SetPage(last_raw_page); + EmptyRawPage(shard_id, last_raw_page); + } + if (AppendRawPage(shard_id, rows_in_group, i, last_row_group_id, last_raw_page, bin_raw_data) != SUCCESS) { + return FAILED; + } + } + (void)shard_header_->SetPage(last_raw_page); + return SUCCESS; +} + +void ShardWriter::EmptyRawPage(const int &shard_id, std::shared_ptr &last_raw_page) { + auto row_group_ids = std::vector>(); + auto page_id = shard_header_->GetLastPageId(shard_id); + auto page_type_id = last_raw_page ? last_raw_page->get_page_id() : -1; + auto page = Page(++page_id, shard_id, kPageTypeRaw, ++page_type_id, 0, 0, row_group_ids, 0); + (void)shard_header_->AddPage(std::make_shared(page)); + SetLastRawPage(shard_id, last_raw_page); +} + +MSRStatus ShardWriter::AppendRawPage(const int &shard_id, const std::vector> &rows_in_group, + const int &chunk_id, int &last_row_group_id, std::shared_ptr last_raw_page, + const std::vector> &bin_raw_data) { + std::vector> row_group_ids = last_raw_page->get_row_group_ids(); + auto last_raw_page_id = last_raw_page->get_page_id(); + auto n_bytes = last_raw_page->get_page_size(); + + // previous raw data page + auto &io_seekp = + file_streams_[shard_id]->seekp(page_size_ * last_raw_page_id + header_size_ + n_bytes, std::ios::beg); + if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) { + MS_LOG(ERROR) << "File seekp failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + if (chunk_id > 0) row_group_ids.emplace_back(++last_row_group_id, n_bytes); + n_bytes += std::accumulate(raw_data_size_.begin() + rows_in_group[chunk_id].first, + raw_data_size_.begin() + rows_in_group[chunk_id].second, 0); + (void)FlushRawChunk(file_streams_[shard_id], rows_in_group, chunk_id, bin_raw_data); + + // Update previous raw data page + last_raw_page->set_page_size(n_bytes); + last_raw_page->set_row_group_ids(row_group_ids); + (void)shard_header_->SetPage(last_raw_page); + + return SUCCESS; +} + +MSRStatus ShardWriter::FlushBlobChunk(const std::shared_ptr &out, + const std::vector> &blob_data, + const std::pair &blob_row) { + if (blob_row.first > blob_row.second) { + return FAILED; + } + if (blob_row.second > static_cast(blob_data.size()) || blob_row.first < 0) { + return FAILED; + } + for (int j = blob_row.first; j < blob_row.second; ++j) { + // Write the size of blob + uint64_t line_len = blob_data[j].size(); + auto &io_handle = out->write(reinterpret_cast(&line_len), kInt64Len); + if (!io_handle.good() || io_handle.fail() || io_handle.bad()) { + MS_LOG(ERROR) << "File write failed"; + out->close(); + return FAILED; + } + + // Write the data of blob + auto line = blob_data[j]; + auto &io_handle_data = out->write(reinterpret_cast(&line[0]), line_len); + if (!io_handle_data.good() || io_handle_data.fail() || io_handle_data.bad()) { + MS_LOG(ERROR) << "File write failed"; + out->close(); + return FAILED; + } + } + return SUCCESS; +} + +MSRStatus ShardWriter::FlushRawChunk(const std::shared_ptr &out, + const std::vector> &rows_in_group, const int &chunk_id, + const std::vector> &bin_raw_data) { + for (int i = rows_in_group[chunk_id].first; i < rows_in_group[chunk_id].second; i++) { + // Write the size of multi schemas + for (uint32_t j = 0; j < schema_count_; ++j) { + uint64_t line_len = bin_raw_data[i * schema_count_ + j].size(); + auto &io_handle = out->write(reinterpret_cast(&line_len), kInt64Len); + if (!io_handle.good() || io_handle.fail() || io_handle.bad()) { + MS_LOG(ERROR) << "File write failed"; + out->close(); + return FAILED; + } + } + // Write the data of multi schemas + for (uint32_t j = 0; j < schema_count_; ++j) { + auto line = bin_raw_data[i * schema_count_ + j]; + auto &io_handle = out->write(reinterpret_cast(&line[0]), line.size()); + if (!io_handle.good() || io_handle.fail() || io_handle.bad()) { + MS_LOG(ERROR) << "File write failed"; + out->close(); + return FAILED; + } + } + } + return SUCCESS; +} + +// Allocate data to shards evenly +std::vector> ShardWriter::BreakIntoShards() { + std::vector> shards; + int row_in_shard = row_count_ / shard_count_; + int remains = row_count_ % shard_count_; + + std::vector v_list(shard_count_); + std::iota(v_list.begin(), v_list.end(), 0); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(v_list.begin(), v_list.end(), g); + std::unordered_set set(v_list.begin(), v_list.begin() + remains); + + if (shard_count_ <= kMaxShardCount) { + int start_row = 0; + for (int i = 0; i < shard_count_; ++i) { + int end_row = start_row + row_in_shard; + if (set.count(i)) end_row++; + shards.emplace_back(start_row, end_row); + start_row = end_row; + } + } + return shards; +} + +MSRStatus ShardWriter::WriteShardHeader() { + if (shard_header_ == nullptr) { + MS_LOG(ERROR) << "Shard header is null"; + return FAILED; + } + auto shard_header = shard_header_->SerializeHeader(); + // Write header data to multi files + if (shard_count_ > static_cast(file_streams_.size()) || shard_count_ > static_cast(shard_header.size())) { + return FAILED; + } + if (shard_count_ <= kMaxShardCount) { + for (int shard_id = 0; shard_id < shard_count_; ++shard_id) { + auto &io_seekp = file_streams_[shard_id]->seekp(0, std::ios::beg); + if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) { + MS_LOG(ERROR) << "File seekp failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + std::vector bin_header(shard_header[shard_id].begin(), shard_header[shard_id].end()); + uint64_t line_len = bin_header.size(); + if (line_len + kInt64Len > header_size_) { + MS_LOG(ERROR) << "Shard header is too big"; + return FAILED; + } + + auto &io_handle = file_streams_[shard_id]->write(reinterpret_cast(&line_len), kInt64Len); + if (!io_handle.good() || io_handle.fail() || io_handle.bad()) { + MS_LOG(ERROR) << "File write failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + + auto &io_handle_header = file_streams_[shard_id]->write(reinterpret_cast(&bin_header[0]), line_len); + if (!io_handle_header.good() || io_handle_header.fail() || io_handle_header.bad()) { + MS_LOG(ERROR) << "File write failed"; + file_streams_[shard_id]->close(); + return FAILED; + } + file_streams_[shard_id]->close(); + } + } + return SUCCESS; +} + +MSRStatus ShardWriter::SerializeRawData(std::map> &raw_data, + std::vector> &bin_data, uint32_t row_count) { + // define the number of thread + uint32_t thread_num = std::thread::hardware_concurrency(); + if (thread_num == 0) thread_num = kThreadNumber; + // Set the number of samples processed by each thread + int group_num = ceil(row_count * 1.0 / thread_num); + std::vector thread_set(thread_num); + int work_thread_num = 0; + for (uint32_t x = 0; x < thread_num; ++x) { + int start_num = x * group_num; + int end_num = ((x + 1) * group_num > row_count) ? row_count : (x + 1) * group_num; + if (start_num >= end_num) { + continue; + } + // Define the run boundary and start the child thread + thread_set[x] = + std::thread(&ShardWriter::FillArray, this, start_num, end_num, std::ref(raw_data), std::ref(bin_data)); + work_thread_num++; + } + for (uint32_t x = 0; x < work_thread_num; ++x) { + // Set obstacles to prevent the main thread from running + thread_set[x].join(); + } + return flag_ == true ? FAILED : SUCCESS; +} + +MSRStatus ShardWriter::SetRawDataSize(const std::vector> &bin_raw_data) { + raw_data_size_ = std::vector(row_count_, 0); + for (uint32_t i = 0; i < row_count_; ++i) { + raw_data_size_[i] = std::accumulate( + bin_raw_data.begin() + (i * schema_count_), bin_raw_data.begin() + (i * schema_count_) + schema_count_, 0, + [](uint64_t accumulator, const std::vector &row) { return accumulator + kInt64Len + row.size(); }); + } + if (*std::max_element(raw_data_size_.begin(), raw_data_size_.end()) > page_size_) { + MS_LOG(ERROR) << "Page size is too small to save a row!"; + return FAILED; + } + return SUCCESS; +} + +MSRStatus ShardWriter::SetBlobDataSize(const std::vector> &blob_data) { + blob_data_size_ = std::vector(row_count_); + (void)std::transform(blob_data.begin(), blob_data.end(), blob_data_size_.begin(), + [](const std::vector &row) { return kInt64Len + row.size(); }); + if (*std::max_element(blob_data_size_.begin(), blob_data_size_.end()) > page_size_) { + MS_LOG(ERROR) << "Page size is too small to save a row!"; + return FAILED; + } + return SUCCESS; +} + +void ShardWriter::SetLastRawPage(const int &shard_id, std::shared_ptr &last_raw_page) { + // Get last raw page + auto last_raw_page_id = shard_header_->GetLastPageIdByType(shard_id, kPageTypeRaw); + if (last_raw_page_id >= 0) { + auto page = shard_header_->GetPage(shard_id, last_raw_page_id); + last_raw_page = page.first; + } +} + +void ShardWriter::SetLastBlobPage(const int &shard_id, std::shared_ptr &last_blob_page) { + // Get last blob page + auto last_blob_page_id = shard_header_->GetLastPageIdByType(shard_id, kPageTypeBlob); + if (last_blob_page_id >= 0) { + auto page = shard_header_->GetPage(shard_id, last_blob_page_id); + last_blob_page = page.first; + } +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_category.cc b/mindspore/ccsrc/mindrecord/meta/shard_category.cc new file mode 100644 index 00000000000..c64a7bfc708 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_category.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_category.h" + +namespace mindspore { +namespace mindrecord { +ShardCategory::ShardCategory(const std::vector> &categories) + : categories_(categories) {} + +const std::vector> &ShardCategory::get_categories() const { return categories_; } + +MSRStatus ShardCategory::operator()(ShardTask &tasks) { return SUCCESS; } +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_header.cc b/mindspore/ccsrc/mindrecord/meta/shard_header.cc new file mode 100644 index 00000000000..57b2e5fa9ea --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_header.cc @@ -0,0 +1,681 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_header.h" + +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_page.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +std::atomic thread_status(false); +ShardHeader::ShardHeader() : shard_count_(0), header_size_(0), page_size_(0) { index_ = std::make_shared(); } + +MSRStatus ShardHeader::InitializeHeader(const std::vector &headers) { + shard_count_ = headers.size(); + bool first = true; + for (const auto &header : headers) { + if (first) { + first = false; + if (ParseSchema(header["schema"]) != SUCCESS) { + return FAILED; + } + if (ParseIndexFields(header["index_fields"]) != SUCCESS) { + return FAILED; + } + if (ParseStatistics(header["statistics"]) != SUCCESS) { + return FAILED; + } + ParseShardAddress(header["shard_addresses"]); + header_size_ = header["header_size"].get(); + page_size_ = header["page_size"].get(); + } + ParsePage(header["page"]); + } + return SUCCESS; +} + +MSRStatus ShardHeader::CheckFileStatus(const std::string &path) { + std::ifstream fin(common::SafeCStr(path), std::ios::in | std::ios::binary); + if (!fin) { + MS_LOG(ERROR) << "File does not exist or permission denied. path: " << path; + return FAILED; + } + if (fin.fail()) { + MS_LOG(ERROR) << "Failed to open file. path: " << path; + return FAILED; + } + + // fetch file size + auto &io_seekg = fin.seekg(0, std::ios::end); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + fin.close(); + MS_LOG(ERROR) << "File seekg failed"; + return FAILED; + } + + size_t file_size = fin.tellg(); + if (file_size < kMinFileSize) { + fin.close(); + MS_LOG(ERROR) << "File size %d is smaller than the minimum value."; + return FAILED; + } + fin.close(); + return SUCCESS; +} + +std::pair ShardHeader::ValidateHeader(const std::string &path) { + if (CheckFileStatus(path) != SUCCESS) { + return {FAILED, {}}; + } + + // read header size + json json_header; + std::ifstream fin(common::SafeCStr(path), std::ios::in | std::ios::binary); + if (!fin.is_open()) { + MS_LOG(ERROR) << "File seekg failed"; + return {FAILED, json_header}; + } + + uint64_t header_size = 0; + auto &io_read = fin.read(reinterpret_cast(&header_size), kInt64Len); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + MS_LOG(ERROR) << "File read failed"; + fin.close(); + return {FAILED, json_header}; + } + + if (header_size > kMaxHeaderSize) { + fin.close(); + MS_LOG(ERROR) << "Header size is illegal."; + return {FAILED, json_header}; + } + + // read header content + std::vector header_content(header_size); + auto &io_read_content = fin.read(reinterpret_cast(&header_content[0]), header_size); + if (!io_read_content.good() || io_read_content.fail() || io_read_content.bad()) { + MS_LOG(ERROR) << "File read failed"; + fin.close(); + return {FAILED, json_header}; + } + + fin.close(); + std::string raw_header_content = std::string(header_content.begin(), header_content.end()); + // parse json content + try { + json_header = json::parse(raw_header_content); + } catch (json::parse_error &e) { + MS_LOG(ERROR) << "Json parse error: " << e.what(); + return {FAILED, json_header}; + } + return {SUCCESS, json_header}; +} + +MSRStatus ShardHeader::Build(const std::string &file_path) { + auto ret = ValidateHeader(file_path); + if (SUCCESS != ret.first) { + return FAILED; + } + json main_header = ret.second; + json addresses = main_header["shard_addresses"]; + vector real_addresses; + auto ret1 = GetParentDir(file_path); + if (SUCCESS != ret1.first) { + return FAILED; + } + std::string parent_dir = ret1.second; + + for (const auto &addr : addresses) { + std::string absolute_path = parent_dir + string(addr); + real_addresses.emplace_back(absolute_path); + } + uint32_t thread_num = std::thread::hardware_concurrency(); + if (thread_num == 0) thread_num = kThreadNumber; + uint32_t work_thread_num = 0; + uint32_t addr_count = real_addresses.size(); + int group_num = ceil(addr_count * 1.0 / thread_num); + std::vector thread_set(thread_num); + std::vector headers(addr_count); + for (uint32_t x = 0; x < thread_num; ++x) { + int start_num = x * group_num; + int end_num = ((x + 1) * group_num > addr_count) ? addr_count : (x + 1) * group_num; + if (start_num >= end_num) { + continue; + } + + thread_set[x] = + std::thread(&ShardHeader::GetHeadersOneTask, this, start_num, end_num, std::ref(headers), real_addresses); + work_thread_num++; + } + + for (uint32_t x = 0; x < work_thread_num; ++x) { + thread_set[x].join(); + } + if (thread_status) { + thread_status = false; + return FAILED; + } + if (SUCCESS != InitializeHeader(headers)) { + return FAILED; + } + return SUCCESS; +} + +void ShardHeader::GetHeadersOneTask(int start, int end, std::vector &headers, + const vector &realAddresses) { + if (thread_status || end > realAddresses.size()) { + return; + } + for (int x = start; x < end; ++x) { + auto ret = ValidateHeader(realAddresses[x]); + if (SUCCESS != ret.first) { + thread_status = true; + return; + } + json header; + header = ret.second; + header["shard_addresses"] = realAddresses; + if (header["version"] != version_) { + MS_LOG(ERROR) << "Version wrong, file version is: " << header["version"].dump() + << ", lib version is: " << version_; + thread_status = true; + return; + } + headers[x] = header; + } +} + +MSRStatus ShardHeader::InitByFiles(const std::vector &file_paths) { + std::vector file_names(file_paths.size()); + std::transform(file_paths.begin(), file_paths.end(), file_names.begin(), [](std::string fp) -> std::string { + if (GetFileName(fp).first == SUCCESS) { + return GetFileName(fp).second; + } + }); + + shard_addresses_ = std::move(file_names); + shard_count_ = file_paths.size(); + if (shard_count_ == 0) { + return FAILED; + } + if (shard_count_ <= kMaxShardCount) { + pages_.resize(shard_count_); + } else { + return FAILED; + } + return SUCCESS; +} + +void ShardHeader::ParseHeader(const json &header) {} + +MSRStatus ShardHeader::ParseIndexFields(const json &index_fields) { + std::vector> parsed_index_fields; + for (auto &index_field : index_fields) { + auto schema_id = index_field["schema_id"].get(); + std::string field_name = index_field["index_field"].get(); + std::pair parsed_index_field(schema_id, field_name); + parsed_index_fields.push_back(parsed_index_field); + } + if (!parsed_index_fields.empty() && AddIndexFields(parsed_index_fields) != SUCCESS) { + return FAILED; + } + return SUCCESS; +} + +void ShardHeader::ParsePage(const json &pages) { + if (pages_.empty() && shard_count_ <= kMaxShardCount) { + pages_.resize(shard_count_); + } + for (auto &page : pages) { + int page_id = page["page_id"]; + int shard_id = page["shard_id"]; + std::string page_type = page["page_type"]; + int page_type_id = page["page_type_id"]; + auto start_row_id = page["start_row_id"].get(); + auto end_row_id = page["end_row_id"].get(); + + std::vector> row_group_ids(page["row_group_ids"].size()); + std::transform(page["row_group_ids"].begin(), page["row_group_ids"].end(), row_group_ids.begin(), + [](json rg) { return std::make_pair(rg["id"], rg["offset"].get()); }); + + auto page_size = page["page_size"].get(); + + std::shared_ptr parsed_page = std::make_shared(page_id, shard_id, page_type, page_type_id, start_row_id, + end_row_id, row_group_ids, page_size); + pages_[shard_id].push_back(std::move(parsed_page)); + } +} + +MSRStatus ShardHeader::ParseStatistics(const json &statistics) { + for (auto &statistic : statistics) { + if (statistic.find("desc") == statistic.end() || statistic.find("statistics") == statistic.end()) { + MS_LOG(ERROR) << "Deserialize statistics failed, statistic: " << statistics.dump(); + return FAILED; + } + std::string statistic_description = statistic["desc"].get(); + json statistic_body = statistic["statistics"]; + std::shared_ptr parsed_statistic = Statistics::Build(statistic_description, statistic_body); + if (!parsed_statistic) { + return FAILED; + } + AddStatistic(parsed_statistic); + } + return SUCCESS; +} + +MSRStatus ShardHeader::ParseSchema(const json &schemas) { + for (auto &schema : schemas) { + // change how we get schemaBody once design is finalized + if (schema.find("desc") == schema.end() || schema.find("blob_fields") == schema.end() || + schema.find("schema") == schema.end()) { + MS_LOG(ERROR) << "Deserialize schema failed. schema: " << schema.dump(); + return FAILED; + } + std::string schema_description = schema["desc"].get(); + std::vector blob_fields = schema["blob_fields"].get>(); + json schema_body = schema["schema"]; + std::shared_ptr parsed_schema = Schema::Build(schema_description, schema_body); + if (!parsed_schema) { + return FAILED; + } + AddSchema(parsed_schema); + } + return SUCCESS; +} + +void ShardHeader::ParseShardAddress(const json &address) { + std::copy(address.begin(), address.end(), std::back_inserter(shard_addresses_)); +} + +std::vector ShardHeader::SerializeHeader() { + std::vector header; + auto index = SerializeIndexFields(); + auto stats = SerializeStatistics(); + auto schema = SerializeSchema(); + auto pages = SerializePage(); + auto address = SerializeShardAddress(); + if (shard_count_ > static_cast(pages.size())) { + return std::vector{}; + } + if (shard_count_ <= kMaxShardCount) { + for (int shardId = 0; shardId < shard_count_; shardId++) { + string s; + s += "{\"header_size\":" + std::to_string(header_size_) + ","; + s += "\"index_fields\":" + index + ","; + s += "\"page\":" + pages[shardId] + ","; + s += "\"page_size\":" + std::to_string(page_size_) + ","; + s += "\"schema\":" + schema + ","; + s += "\"shard_addresses\":" + address + ","; + s += "\"shard_id\":" + std::to_string(shardId) + ","; + s += "\"statistics\":" + stats + ","; + s += "\"version\":\"" + version_ + "\""; + s += "}"; + header.emplace_back(s); + } + } + return header; +} + +std::string ShardHeader::SerializeIndexFields() { + json j; + auto fields = index_->get_fields(); + for (const auto &field : fields) { + j.push_back({{"schema_id", field.first}, {"index_field", field.second}}); + } + return j.dump(); +} + +std::vector ShardHeader::SerializePage() { + std::vector pages; + for (auto &shard_pages : pages_) { + json j; + for (const auto &p : shard_pages) { + j.emplace_back(p->GetPage()); + } + pages.emplace_back(j.dump()); + } + return pages; +} + +std::string ShardHeader::SerializeStatistics() { + json j; + for (const auto &stats : statistics_) { + j.emplace_back(stats->get_statistics()); + } + return j.dump(); +} + +std::string ShardHeader::SerializeSchema() { + json j; + for (const auto &schema : schema_) { + j.emplace_back(schema->GetSchema()); + } + return j.dump(); +} + +std::string ShardHeader::SerializeShardAddress() { + json j; + for (const auto &addr : shard_addresses_) { + j.emplace_back(GetFileName(addr).second); + } + return j.dump(); +} + +std::pair, MSRStatus> ShardHeader::GetPage(const int &shard_id, const int &page_id) { + if (shard_id < static_cast(pages_.size()) && page_id < static_cast(pages_[shard_id].size())) { + return std::make_pair(pages_[shard_id][page_id], SUCCESS); + } else { + return std::make_pair(nullptr, FAILED); + } +} + +MSRStatus ShardHeader::SetPage(const std::shared_ptr &new_page) { + if (new_page == nullptr) { + return FAILED; + } + int shard_id = new_page->get_shard_id(); + int page_id = new_page->get_page_id(); + if (shard_id < static_cast(pages_.size()) && page_id < static_cast(pages_[shard_id].size())) { + pages_[shard_id][page_id] = new_page; + return SUCCESS; + } else { + return FAILED; + } +} + +MSRStatus ShardHeader::AddPage(const std::shared_ptr &new_page) { + if (new_page == nullptr) { + return FAILED; + } + int shard_id = new_page->get_shard_id(); + int page_id = new_page->get_page_id(); + if (shard_id < static_cast(pages_.size()) && page_id == static_cast(pages_[shard_id].size())) { + pages_[shard_id].push_back(new_page); + return SUCCESS; + } else { + return FAILED; + } +} + +int64_t ShardHeader::GetLastPageId(const int &shard_id) { + if (shard_id >= static_cast(pages_.size())) { + return 0; + } + return pages_[shard_id].size() - 1; +} + +int ShardHeader::GetLastPageIdByType(const int &shard_id, const std::string &page_type) { + if (shard_id >= static_cast(pages_.size())) { + return 0; + } + int last_page_id = -1; + for (uint64_t i = pages_[shard_id].size(); i >= 1; i--) { + if (pages_[shard_id][i - 1]->get_page_type() == page_type) { + last_page_id = pages_[shard_id][i - 1]->get_page_id(); + return last_page_id; + } + } + return last_page_id; +} + +const std::pair> ShardHeader::GetPageByGroupId(const int &group_id, + const int &shard_id) { + if (shard_id >= static_cast(pages_.size())) { + MS_LOG(ERROR) << "Shard id is more than sum of shards."; + return {FAILED, nullptr}; + } + for (uint64_t i = pages_[shard_id].size(); i >= 1; i--) { + auto page = pages_[shard_id][i - 1]; + if (page->get_page_type() == kPageTypeBlob && page->get_page_type_id() == group_id) { + return {SUCCESS, page}; + } + } + MS_LOG(ERROR) << "Could not get page by group id " << group_id; + return {FAILED, nullptr}; +} + +int ShardHeader::AddSchema(std::shared_ptr schema) { + if (schema == nullptr) { + MS_LOG(ERROR) << "Schema is illegal"; + return -1; + } + + if (!schema_.empty()) { + MS_LOG(ERROR) << "Only support one schema"; + return -1; + } + + int64_t schema_id = schema->get_schema_id(); + if (schema_id == -1) { + schema_id = schema_.size(); + schema->set_schema_id(schema_id); + } + schema_.push_back(schema); + return schema_id; +} + +void ShardHeader::AddStatistic(std::shared_ptr statistic) { + if (statistic) { + int64_t statistics_id = statistic->get_statistics_id(); + if (statistics_id == -1) { + statistics_id = statistics_.size(); + statistic->set_statistics_id(statistics_id); + } + statistics_.push_back(statistic); + } +} + +std::shared_ptr ShardHeader::InitIndexPtr() { + std::shared_ptr index = index_; + if (!index_) { + index = std::make_shared(); + index_ = index; + } + return index; +} + +MSRStatus ShardHeader::CheckIndexField(const std::string &field, const json &schema) { + // check field name is or is not valid + if (schema.find(field) == schema.end()) { + MS_LOG(ERROR) << "Schema do not contain the field: " << field << "."; + return FAILED; + } + + if (schema[field]["type"] == "bytes") { + MS_LOG(ERROR) << field << " is bytes type, can not be schema index field."; + return FAILED; + } + + if (schema.find(field) != schema.end() && schema[field].find("shape") != schema[field].end()) { + MS_LOG(ERROR) << field << " array can not be schema index field."; + return FAILED; + } + return SUCCESS; +} + +MSRStatus ShardHeader::AddIndexFields(const std::vector &fields) { + // create index Object + std::shared_ptr index = InitIndexPtr(); + + if (fields.size() == kInt0) { + MS_LOG(ERROR) << "There are no index fields"; + return FAILED; + } + + if (get_schemas().empty()) { + MS_LOG(ERROR) << "No schema is set"; + return FAILED; + } + + for (const auto &schemaPtr : schema_) { + auto result = GetSchemaByID(schemaPtr->get_schema_id()); + if (result.second != SUCCESS) { + MS_LOG(ERROR) << "Could not get schema by id."; + return FAILED; + } + + if (result.first == nullptr) { + MS_LOG(ERROR) << "Could not get schema by id."; + return FAILED; + } + + json schema = result.first->GetSchema().at("schema"); + + // checkout and add fields for each schema + std::set field_set; + for (const auto &item : index->get_fields()) { + field_set.insert(item.second); + } + for (const auto &field : fields) { + if (field_set.find(field) != field_set.end()) { + MS_LOG(ERROR) << "Add same index field twice"; + return FAILED; + } + + // check field name is or is not valid + if (CheckIndexField(field, schema) == FAILED) { + return FAILED; + } + field_set.insert(field); + + // add field into index + index.get()->AddIndexField(schemaPtr->get_schema_id(), field); + } + } + + index_ = index; + return SUCCESS; +} + +MSRStatus ShardHeader::GetAllSchemaID(std::set &bucket_count) { + // get all schema id + for (const auto &schema : schema_) { + auto bucket_it = bucket_count.find(schema->get_schema_id()); + if (bucket_it != bucket_count.end()) { + MS_LOG(ERROR) << "Schema duplication"; + return FAILED; + } else { + bucket_count.insert(schema->get_schema_id()); + } + } + return SUCCESS; +} + +MSRStatus ShardHeader::AddIndexFields(std::vector> fields) { + // create index Object + std::shared_ptr index = InitIndexPtr(); + + if (fields.size() == kInt0) { + MS_LOG(ERROR) << "There are no index fields"; + return FAILED; + } + + // get all schema id + std::set bucket_count; + if (GetAllSchemaID(bucket_count) != SUCCESS) { + return FAILED; + } + + // check and add fields for each schema + std::set> field_set; + for (const auto &item : index->get_fields()) { + field_set.insert(item); + } + for (const auto &field : fields) { + if (field_set.find(field) != field_set.end()) { + MS_LOG(ERROR) << "Add same index field twice"; + return FAILED; + } + + uint64_t schema_id = field.first; + std::string field_name = field.second; + + // check schemaId is or is not valid + if (bucket_count.find(schema_id) == bucket_count.end()) { + MS_LOG(ERROR) << "Illegal schema id: " << schema_id; + return FAILED; + } + + // check field name is or is not valid + auto result = GetSchemaByID(schema_id); + if (result.second != SUCCESS) { + MS_LOG(ERROR) << "Could not get schema by id."; + return FAILED; + } + json schema = result.first->GetSchema().at("schema"); + if (schema.find(field_name) == schema.end()) { + MS_LOG(ERROR) << "Schema " << schema_id << " do not contain the field: " << field_name; + return FAILED; + } + + if (CheckIndexField(field_name, schema) == FAILED) { + return FAILED; + } + + field_set.insert(field); + + // add field into index + index.get()->AddIndexField(schema_id, field_name); + } + index_ = index; + return SUCCESS; +} + +std::string ShardHeader::get_shard_address_by_id(int64_t shard_id) { + if (shard_id >= shard_addresses_.size()) { + return ""; + } + return shard_addresses_.at(shard_id); +} + +std::vector> ShardHeader::get_schemas() { return schema_; } + +std::vector> ShardHeader::get_statistics() { return statistics_; } + +std::vector> ShardHeader::get_fields() { return index_->get_fields(); } + +std::shared_ptr ShardHeader::get_index() { return index_; } + +std::pair, MSRStatus> ShardHeader::GetSchemaByID(int64_t schema_id) { + int64_t schemaSize = schema_.size(); + if (schema_id < 0 || schema_id >= schemaSize) { + MS_LOG(ERROR) << "Illegal schema id"; + return std::make_pair(nullptr, FAILED); + } + return std::make_pair(schema_.at(schema_id), SUCCESS); +} + +std::pair, MSRStatus> ShardHeader::GetStatisticByID(int64_t statistic_id) { + int64_t statistics_size = statistics_.size(); + if (statistic_id < 0 || statistic_id >= statistics_size) { + return std::make_pair(nullptr, FAILED); + } + return std::make_pair(statistics_.at(statistic_id), SUCCESS); +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_index.cc b/mindspore/ccsrc/mindrecord/meta/shard_index.cc new file mode 100644 index 00000000000..ddb85fc66e5 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_index.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_index.h" + +namespace mindspore { +namespace mindrecord { +// table name for index +const char TABLENAME[] = "index_table"; + +Index::Index() : database_name_(""), table_name_(TABLENAME) {} + +void Index::AddIndexField(const int64_t &schemaId, const std::string &field) { + fields_.emplace_back(pair(schemaId, field)); +} + +// Get attribute list +std::vector> Index::get_fields() { return fields_; } +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_page.cc b/mindspore/ccsrc/mindrecord/meta/shard_page.cc new file mode 100644 index 00000000000..6bb849ae1db --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_page.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_page.h" +#include "pybind11/pybind11.h" + +namespace mindspore { +namespace mindrecord { +json Page::GetPage() const { + json str_page; + str_page["page_id"] = page_id_; + str_page["shard_id"] = shard_id_; + str_page["page_type"] = page_type_; + str_page["page_type_id"] = page_type_id_; + str_page["start_row_id"] = start_row_id_; + str_page["end_row_id"] = end_row_id_; + if (row_group_ids_.size() == 0) { + json row_groups = json({}); + row_groups["id"] = 0; + row_groups["offset"] = 0; + str_page["row_group_ids"].push_back(row_groups); + } else { + for (const auto &rg : row_group_ids_) { + json row_groups = json({}); + row_groups["id"] = rg.first; + row_groups["offset"] = rg.second; + str_page["row_group_ids"].push_back(row_groups); + } + } + str_page["page_size"] = page_size_; + return str_page; +} + +void Page::DeleteLastGroupId() { + if (!row_group_ids_.empty()) { + page_size_ = row_group_ids_.back().second; + row_group_ids_.pop_back(); + } +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_sample.cc b/mindspore/ccsrc/mindrecord/meta/shard_sample.cc new file mode 100644 index 00000000000..ea365a0e2ac --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_sample.cc @@ -0,0 +1,103 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_sample.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +ShardSample::ShardSample(int n) { + numerator_ = 0; + denominator_ = 0; + no_of_samples_ = n; + partition_id_ = 0; +} + +ShardSample::ShardSample(int num, int den) { + if (num < 0 || den <= 0 || num > den) { + no_of_samples_ = 5; + numerator_ = 0; + denominator_ = 0; + partition_id_ = 0; + return; + } + numerator_ = num; + denominator_ = den; + no_of_samples_ = 0; + partition_id_ = 0; +} + +ShardSample::ShardSample(int num, int den, int par) { + numerator_ = num; + denominator_ = den; + no_of_samples_ = 0; + partition_id_ = par; +} + +const std::pair ShardSample::get_partitions() const { + if (numerator_ == 1 && denominator_ > 1) { + return std::pair(denominator_, partition_id_); + } + return std::pair(-1, -1); +} + +MSRStatus ShardSample::operator()(ShardTask &tasks) { + int no_of_categories = static_cast(tasks.categories); + int total_no = static_cast(tasks.Size()); + + int taking = 0; + if (no_of_samples_ > 0) { // non sharding case constructor #1 + no_of_samples_ = std::min(no_of_samples_, total_no); + taking = no_of_samples_ - no_of_samples_ % no_of_categories; + } else { // constructor #2 & #3 + if (numerator_ > 0 && denominator_ > 0 && numerator_ <= denominator_) { + if (numerator_ == 1 && denominator_ > 1) { // sharding + taking = (total_no / denominator_) + (total_no % denominator_ == 0 ? 0 : 1); + } else { // non sharding + taking = total_no * numerator_ / denominator_; + taking -= (taking % no_of_categories); + } + } else { + MS_LOG(ERROR) << "parameter numerator or denominator is illegal"; + return FAILED; + } + } + + if (tasks.permutation_.empty()) { + ShardTask new_tasks; + total_no = static_cast(tasks.Size()); + for (int i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { + new_tasks.InsertTask(tasks.get_task_by_id(i % total_no)); // rounding up. if overflow, go back to start + } + std::swap(tasks, new_tasks); + } else { + ShardTask new_tasks; + if (taking > static_cast(tasks.permutation_.size())) { + return FAILED; + } + total_no = static_cast(tasks.permutation_.size()); + for (size_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { + new_tasks.InsertTask(tasks.get_task_by_id(tasks.permutation_[i % total_no])); + } + std::swap(tasks, new_tasks); + } + return SUCCESS; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_schema.cc b/mindspore/ccsrc/mindrecord/meta/shard_schema.cc new file mode 100644 index 00000000000..0c2550e2dc5 --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_schema.cc @@ -0,0 +1,164 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_schema.h" +#include "common/utils.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +std::shared_ptr Schema::Build(std::string desc, const json &schema) { + // validate check + if (!Validate(schema)) { + return nullptr; + } + + std::vector blob_fields = PopulateBlobFields(schema); + Schema object_schema; + object_schema.desc_ = std::move(desc); + object_schema.blob_fields_ = std::move(blob_fields); + object_schema.schema_ = schema; + object_schema.schema_id_ = -1; + return std::make_shared(object_schema); +} + +std::shared_ptr Schema::Build(std::string desc, pybind11::handle schema) { + // validate check + json schema_json = nlohmann::detail::ToJsonImpl(schema); + return Build(std::move(desc), schema_json); +} + +std::string Schema::get_desc() const { return desc_; } + +json Schema::GetSchema() const { + json str_schema; + str_schema["desc"] = desc_; + str_schema["schema"] = schema_; + str_schema["blob_fields"] = blob_fields_; + return str_schema; +} + +pybind11::object Schema::GetSchemaForPython() const { + json schema_json = GetSchema(); + pybind11::object schema_py = nlohmann::detail::FromJsonImpl(schema_json); + return schema_py; +} + +void Schema::set_schema_id(int64_t id) { schema_id_ = id; } + +int64_t Schema::get_schema_id() const { return schema_id_; } + +std::vector Schema::get_blob_fields() const { return blob_fields_; } + +std::vector Schema::PopulateBlobFields(json schema) { + std::vector blob_fields; + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + json it_value = it.value(); + if ((it_value.size() == kInt2 && it_value.find("shape") != it_value.end()) || it_value["type"] == "bytes") { + blob_fields.emplace_back(it.key()); + } + } + return blob_fields; +} + +bool Schema::ValidateNumberShape(const json &it_value) { + if (it_value.find("shape") == it_value.end()) { + MS_LOG(ERROR) << "%s supports shape only." << it_value["type"].dump(); + return false; + } + + auto shape = it_value["shape"]; + if (!shape.is_array()) { + MS_LOG(ERROR) << "%s shape format is wrong." << it_value["type"].dump(); + return false; + } + + int num_negtive_one = 0; + for (const auto &i : shape) { + if (i == 0 || i < -1) { + MS_LOG(ERROR) << "Shape %s, number is wrong." << it_value["shape"].dump(); + return false; + } + if (i == -1) { + num_negtive_one++; + } + } + + if (num_negtive_one > 1) { + MS_LOG(ERROR) << "Shape %s, have at most 1 variable-length dimension." << it_value["shape"].dump(); + return false; + } + + return true; +} + +bool Schema::Validate(json schema) { + if (schema.size() == kInt0) { + MS_LOG(ERROR) << "Schema is null"; + return false; + } + + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + // make sure schema key name must be composed of '0-9' or 'a-z' or 'A-Z' or '_' + if (!ValidateFieldName(it.key())) { + MS_LOG(ERROR) << "Field name must be composed of '0-9' or 'a-z' or 'A-Z' or '_', fieldName: " << it.key(); + return false; + } + + json it_value = it.value(); + if (it_value.find("type") == it_value.end()) { + MS_LOG(ERROR) << "No 'type' field exist: " << it_value.dump(); + return false; + } + + if (kFieldTypeSet.find(it_value["type"]) == kFieldTypeSet.end()) { + MS_LOG(ERROR) << "Wrong type: " << it_value["type"].dump(); + return false; + } + + if (it_value.size() == kInt1) { + continue; + } + + if (it_value["type"] == "bytes" || it_value["type"] == "string") { + MS_LOG(ERROR) << it_value["type"].dump() << " can not 1 field only."; + return false; + } + + if (it_value.size() != kInt2) { + MS_LOG(ERROR) << it_value["type"].dump() << " can have at most 2 fields."; + return false; + } + + if (!ValidateNumberShape(it_value)) { + return false; + } + } + + return true; +} + +bool Schema::operator==(const mindrecord::Schema &b) const { + if (this->get_desc() != b.get_desc() || this->GetSchema() != b.GetSchema()) { + return false; + } + return true; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_shuffle.cc b/mindspore/ccsrc/mindrecord/meta/shard_shuffle.cc new file mode 100644 index 00000000000..14816e9e9fa --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_shuffle.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_shuffle.h" + +#include + +namespace mindspore { +namespace mindrecord { +ShardShuffle::ShardShuffle(uint32_t seed) : shuffle_seed_(seed) {} + +MSRStatus ShardShuffle::operator()(ShardTask &tasks) { + if (tasks.categories < 1) { + return FAILED; + } + uint32_t individual_size = tasks.Size() / tasks.categories; + std::vector> new_permutations(tasks.categories, std::vector(individual_size)); + for (uint32_t i = 0; i < tasks.categories; i++) { + for (uint32_t j = 0; j < individual_size; j++) new_permutations[i][j] = static_cast(j); + std::shuffle(new_permutations[i].begin(), new_permutations[i].end(), std::default_random_engine(shuffle_seed_)); + } + shuffle_seed_++; + tasks.permutation_.clear(); + for (uint32_t j = 0; j < individual_size; j++) { + for (uint32_t i = 0; i < tasks.categories; i++) { + tasks.permutation_.push_back(new_permutations[i][j] * static_cast(tasks.categories) + static_cast(i)); + } + } + return SUCCESS; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_statistics.cc b/mindspore/ccsrc/mindrecord/meta/shard_statistics.cc new file mode 100644 index 00000000000..deaf0b1874b --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_statistics.cc @@ -0,0 +1,112 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_statistics.h" +#include "pybind11/pybind11.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +std::shared_ptr Statistics::Build(std::string desc, const json &statistics) { + // validate check + if (!Validate(statistics)) { + return nullptr; + } + Statistics object_statistics; + object_statistics.desc_ = std::move(desc); + object_statistics.statistics_ = statistics; + object_statistics.statistics_id_ = -1; + return std::make_shared(object_statistics); +} + +std::shared_ptr Statistics::Build(std::string desc, pybind11::handle statistics) { + // validate check + json statistics_json = nlohmann::detail::ToJsonImpl(statistics); + if (!Validate(statistics_json)) { + return nullptr; + } + Statistics object_statistics; + object_statistics.desc_ = std::move(desc); + object_statistics.statistics_ = statistics_json; + object_statistics.statistics_id_ = -1; + return std::make_shared(object_statistics); +} + +std::string Statistics::get_desc() const { return desc_; } + +json Statistics::get_statistics() const { + json str_statistics; + str_statistics["desc"] = desc_; + str_statistics["statistics"] = statistics_; + return str_statistics; +} + +pybind11::object Statistics::GetStatisticsForPython() const { + json str_statistics = Statistics::get_statistics(); + return nlohmann::detail::FromJsonImpl(str_statistics); +} + +void Statistics::set_statistics_id(int64_t id) { statistics_id_ = id; } + +int64_t Statistics::get_statistics_id() const { return statistics_id_; } + +bool Statistics::Validate(const json &statistics) { + if (statistics.size() != kInt1) { + MS_LOG(ERROR) << "Statistics object is null"; + return false; + } + if (statistics.find("level") == statistics.end()) { + MS_LOG(ERROR) << "There is not 'level' object in statistic"; + return false; + } + return LevelRecursive(statistics["level"]); +} + +bool Statistics::LevelRecursive(json level) { + bool ini = true; + for (json::iterator it = level.begin(); it != level.end(); ++it) { + json a = it.value(); + if (a.size() == kInt2) { + if ((a.find("key") == a.end()) || (a.find("count") == a.end())) { + MS_LOG(ERROR) << "The node field is 2, but 'key'/'count' is not existed"; + return false; + } + } else if (a.size() == kInt3) { + if ((a.find("key") == a.end()) || (a.find("count") == a.end()) || a.find("level") == a.end()) { + MS_LOG(ERROR) << "The node field is 3, but 'key'/'count'/'level' is not existed"; + return false; + } else { + ini = LevelRecursive(a.at("level")); + } + } else { + MS_LOG(ERROR) << "The node field is not equal 2/3"; + return false; + } + } + return ini; +} + +bool Statistics::operator==(const Statistics &b) const { + if (this->get_statistics() != b.get_statistics()) { + return false; + } + return true; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindrecord/meta/shard_task.cc b/mindspore/ccsrc/mindrecord/meta/shard_task.cc new file mode 100644 index 00000000000..3744d881a4d --- /dev/null +++ b/mindspore/ccsrc/mindrecord/meta/shard_task.cc @@ -0,0 +1,89 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mindrecord/include/shard_task.h" +#include "common/utils.h" +#include "mindrecord/include/common/shard_utils.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::DEBUG; + +namespace mindspore { +namespace mindrecord { +void ShardTask::MakePerm() { + permutation_ = std::vector(task_list_.size()); + for (uint32_t i = 0; i < task_list_.size(); i++) { + permutation_[i] = static_cast(i); + } +} + +void ShardTask::InsertTask(int shard_id, int group_id, const std::vector &offset, const json &label) { + MS_LOG(DEBUG) << "Into insert task, shard_id: " << shard_id << ", group_id: " << group_id + << ", label: " << label.dump() << ", size of task_list_: " << task_list_.size() << "."; + task_list_.emplace_back(std::make_tuple(shard_id, group_id), offset, label); + MS_LOG(DEBUG) << "Out of insert task, shard_id: " << shard_id << ", group_id: " << group_id + << ", label: " << label.dump() << ", size of task_list_: " << task_list_.size() << "."; +} + +void ShardTask::InsertTask(std::tuple, std::vector, json> task) { + MS_LOG(DEBUG) << "Into insert task, shard_id: " << std::get<0>(std::get<0>(task)) + << ", group_id: " << std::get<1>(std::get<0>(task)) << ", label: " << std::get<2>(task).dump() + << ", size of task_list_: " << task_list_.size() << "."; + task_list_.push_back(std::move(task)); + MS_LOG(DEBUG) << "Out of insert task, shard_id: " << std::get<0>(std::get<0>(task)) + << ", group_id: " << std::get<1>(std::get<0>(task)) << ", label: " << std::get<2>(task).dump() + << ", size of task_list_: " << task_list_.size() << "."; +} + +void ShardTask::PopBack() { task_list_.pop_back(); } + +uint32_t ShardTask::Size() const { return static_cast(task_list_.size()); } + +uint32_t ShardTask::SizeOfRows() const { + if (task_list_.size() == 0) return static_cast(0); + + // 1 task is 1 page + auto sum_num_rows = [](int x, std::tuple, std::vector, json> y) { + return x + std::get<1>(y)[0]; + }; + uint32_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows); + return nRows; +} + +std::tuple, std::vector, json> &ShardTask::get_task_by_id(size_t id) { + MS_ASSERT(id < task_list_.size()); + return task_list_[id]; +} + +ShardTask ShardTask::Combine(std::vector &category_tasks) { + ShardTask res; + if (category_tasks.empty()) return res; + auto total_categories = category_tasks.size(); + res.categories = static_cast(total_categories); + auto minTasks = category_tasks[0].Size(); + for (uint32_t i = 1; i < total_categories; i++) { + minTasks = std::min(minTasks, category_tasks[i].Size()); + } + for (uint32_t task_no = 0; task_no < minTasks; task_no++) { + for (uint32_t i = 0; i < total_categories; i++) { + res.InsertTask(std::move(category_tasks[i].get_task_by_id(static_cast(task_no)))); + } + } + return res; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/mindspore/ccsrc/mindspore.cc b/mindspore/ccsrc/mindspore.cc new file mode 100644 index 00000000000..542814016f5 --- /dev/null +++ b/mindspore/ccsrc/mindspore.cc @@ -0,0 +1,24 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace mindspore { + +// cppcheck-suppress unusedFunction +std::string set_version(const std::string& version) { return version; } + +} // namespace mindspore diff --git a/mindspore/ccsrc/onnx/CMakeLists.txt b/mindspore/ccsrc/onnx/CMakeLists.txt new file mode 100644 index 00000000000..2a25e67634d --- /dev/null +++ b/mindspore/ccsrc/onnx/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _ONNX_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_onnx_obj OBJECT ${_ONNX_ALL_SRC_FILES}) diff --git a/mindspore/ccsrc/onnx/onnx_exporter.cc b/mindspore/ccsrc/onnx/onnx_exporter.cc new file mode 100644 index 00000000000..3bd4a38881d --- /dev/null +++ b/mindspore/ccsrc/onnx/onnx_exporter.cc @@ -0,0 +1,906 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug/anf_ir_utils.h" +#include "./onnx.pb.h" +#include "operator/ops.h" + +namespace mindspore { +enum OpMergeMode { + OP_MERGE_UNDEFINED = 0, // undefined behavior + OP_MERGE_IGNORE = 1, // indicate an input op merged into other op in compute node list + OP_MERGE_CONV = 2, // indicate `MindSpore Conv + BiasAdd` --> `ONNX Conv` + OP_MERGE_GEMM = 3, // indicate `MindSpore MatMul + BiasAdd` --> `ONNX Gemm` + OP_MERGE_BATCH_NORM = 4, // indicate `MindSpore BatchNorm(x)[0]` --> `ONNX BatchNormalization` +}; + +struct OpMergedInfo { + OpMergeMode mode = OP_MERGE_UNDEFINED; + int referred_count = 0; +}; + +using GenAttrFuncType = + std::function; + +template +void SetAttrValueToProto(const ValuePtr& value, onnx::AttributeProto_AttributeType attr_type, + onnx::AttributeProto* const attr_proto, const PrimitivePtr&) { + auto casted_value = dyn_cast(value); + if (casted_value == nullptr) { + MS_LOG(EXCEPTION) << "Cast value " << value->ToString() << " to type T failed."; + } + auto attr_value = casted_value->value(); + switch (attr_type) { + case onnx::AttributeProto_AttributeType_INT: + attr_proto->set_i(static_cast<::google::protobuf::int64>(attr_value)); + break; + case onnx::AttributeProto_AttributeType_FLOAT: + attr_proto->set_f(static_cast(attr_value)); + break; + case onnx::AttributeProto_AttributeType_INTS: + for (size_t i = 0; i < rep_cnt; ++i) { + attr_proto->add_ints(static_cast<::google::protobuf::int64>(attr_value)); + } + break; + case onnx::AttributeProto_AttributeType_FLOATS: + for (size_t i = 0; i < rep_cnt; ++i) { + attr_proto->add_floats(static_cast(attr_value)); + } + break; + default: + MS_LOG(EXCEPTION) << "Convert attribute fail, unexpected ONNX type " << attr_type; + } + attr_proto->set_type(attr_type); +} + +template +void SetAttrTupleValueToProto(const ValuePtr& value, onnx::AttributeProto_AttributeType attr_type, + onnx::AttributeProto* const attr_proto, const PrimitivePtr&) { + auto tuple_ptr = dyn_cast(value); + if (tuple_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Cast value from type " << value->type_name() << " to ValueTuple failed."; + } + switch (attr_type) { + case onnx::AttributeProto_AttributeType_INTS: + for (size_t i = beg_idx; i < tuple_ptr->size(); ++i) { + attr_proto->add_ints(GetValue((*tuple_ptr)[i])); + } + break; + case onnx::AttributeProto_AttributeType_FLOATS: + for (size_t i = beg_idx; i < tuple_ptr->size(); ++i) { + attr_proto->add_floats(GetValue((*tuple_ptr)[i])); + } + break; + default: + MS_LOG(EXCEPTION) << "Convert attribute fail, unexpected ONNX type " << attr_type; + } + attr_proto->set_type(attr_type); +} + +void SetPoolingPadMode(const ValuePtr& value, onnx::AttributeProto_AttributeType, + onnx::AttributeProto* const attr_proto, const PrimitivePtr&) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + auto attr_value = GetValue(value); + if (attr_value == "VALID") { + attr_proto->set_s("VALID"); + } else { + attr_proto->set_s("SAME_UPPER"); + } +} + +class OpAttrInfo { + public: + OpAttrInfo(const std::string& attr_name, const string& onnx_attr_name, + onnx::AttributeProto_AttributeType onnx_attr_type, const GenAttrFuncType& fn_gen_attr) + : attr_name_(attr_name), + onnx_attr_name_(onnx_attr_name), + onnx_attr_type_(onnx_attr_type), + fn_gen_attr_(fn_gen_attr) {} + ~OpAttrInfo() {} + + const std::string& attr_name() const { return attr_name_; } + const std::string& onnx_attr_name() const { return onnx_attr_name_; } + onnx::AttributeProto_AttributeType onnx_attr_type() const { return onnx_attr_type_; } + GenAttrFuncType fn_gen_attr() const { return fn_gen_attr_; } + + private: + std::string attr_name_; // attribute name of MindSpore + std::string onnx_attr_name_; // corresponding attribute name of ONNX + onnx::AttributeProto_AttributeType onnx_attr_type_; // corresponding attribute type of ONNX + GenAttrFuncType fn_gen_attr_; // function used convert +}; + +class OpNameInfo { + public: + OpNameInfo& set_op_type(const std::string& op_type) { + op_type_ = op_type; + return *this; + } + + const std::string& op_type() const { return op_type_; } + + OpNameInfo& set_onnx_type(const std::string& onnx_type) { + onnx_type_ = onnx_type; + return *this; + } + + const std::string& onnx_type() const { return onnx_type_; } + + OpNameInfo& Attr(const std::string& attr_name, const std::string& onnx_attr_name, + onnx::AttributeProto_AttributeType onnx_attr_type, const GenAttrFuncType& fn_gen_attr) { + op_attrs_.emplace_back(OpAttrInfo(attr_name, onnx_attr_name, onnx_attr_type, fn_gen_attr)); + return *this; + } + + const std::vector& op_attrs() const { return op_attrs_; } + + private: + std::string op_type_; // operator type of MindSpore + std::string onnx_type_; // corresponding ONNX operator type + std::vector op_attrs_; // operator attributes map info +}; + +#define OPERATOR_ONNX_CONVERT_DEFINE(name, onnx_name, impl) \ + OpNameInfo GetOpOnnxConvertInfo_##name() { return impl.set_op_type(#name).set_onnx_type(#onnx_name); } + +OPERATOR_ONNX_CONVERT_DEFINE(TensorAdd, Add, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Mul, Mul, OpNameInfo()) + +OPERATOR_ONNX_CONVERT_DEFINE(ReLU, Relu, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Sigmoid, Sigmoid, OpNameInfo()) + +OPERATOR_ONNX_CONVERT_DEFINE(Flatten, Flatten, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Squeeze, Squeeze, + OpNameInfo().Attr("axis", "axes", onnx::AttributeProto_AttributeType_INTS, + SetAttrTupleValueToProto)) + +OPERATOR_ONNX_CONVERT_DEFINE( + Conv2D, Conv, + OpNameInfo() + .Attr("dilation", "dilations", onnx::AttributeProto_AttributeType_INTS, SetAttrValueToProto) + .Attr("group", "group", onnx::AttributeProto_AttributeType_INT, SetAttrValueToProto) + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, + [](ValuePtr value, onnx::AttributeProto_AttributeType, onnx::AttributeProto* const attr_proto, + const PrimitivePtr& prim) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + auto attr_value = GetValue(value); + if (attr_value == "valid") { + attr_proto->set_s("VALID"); + } else if (attr_value == "same") { + attr_proto->set_s("SAME_UPPER"); + } else { // pad_mode is 'pad', use attribute 'pad_list' to fill ONNX attribute 'pads' + attr_proto->set_name("pads"); + SetAttrTupleValueToProto(prim->GetAttr("pad_list"), onnx::AttributeProto_AttributeType_INTS, attr_proto, + prim); + } + }) + .Attr("stride", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrValueToProto)) + +OPERATOR_ONNX_CONVERT_DEFINE(BiasAdd, Add, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(MatMul, Gemm, + OpNameInfo() + .Attr("transpose_a", "transA", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto) + .Attr("transpose_b", "transB", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto)) + +OPERATOR_ONNX_CONVERT_DEFINE(BatchNorm, BatchNormalization, + OpNameInfo().Attr("epsilon", "epsilon", onnx::AttributeProto_AttributeType_FLOAT, + SetAttrValueToProto)) + +OPERATOR_ONNX_CONVERT_DEFINE(Reshape, Reshape, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(ReduceMean, ReduceMean, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Cast, Cast, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(PReLU, PRelu, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Argmax, ArgMax, + OpNameInfo() + .Attr("axis", "axis", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto) + .Attr("", "keepdims", onnx::AttributeProto_AttributeType_INT, + [](ValuePtr, onnx::AttributeProto_AttributeType, + onnx::AttributeProto* const attr_proto, const PrimitivePtr&) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + attr_proto->set_i(0); + })) + +OPERATOR_ONNX_CONVERT_DEFINE(SimpleMean, AveragePool, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE( + MaxPool, MaxPool, + OpNameInfo() + .Attr("ksize", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>) + .Attr("padding", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetPoolingPadMode) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>)) + +OPERATOR_ONNX_CONVERT_DEFINE( + AvgPool, AveragePool, + OpNameInfo() + .Attr("ksize", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>) + .Attr("padding", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetPoolingPadMode) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>)) + +#define OP_CONVERT_FUNCTION_NAME(name) GetOpOnnxConvertInfo_##name + +void RegisterOpConverters(const std::function& fn) { + fn(OP_CONVERT_FUNCTION_NAME(TensorAdd)()); + fn(OP_CONVERT_FUNCTION_NAME(Mul)()); + + fn(OP_CONVERT_FUNCTION_NAME(ReLU)()); + fn(OP_CONVERT_FUNCTION_NAME(Sigmoid)()); + + fn(OP_CONVERT_FUNCTION_NAME(Conv2D)()); + fn(OP_CONVERT_FUNCTION_NAME(Argmax)()); + + fn(OP_CONVERT_FUNCTION_NAME(Flatten)()); + fn(OP_CONVERT_FUNCTION_NAME(MaxPool)()); + fn(OP_CONVERT_FUNCTION_NAME(AvgPool)()); + + fn(OP_CONVERT_FUNCTION_NAME(Squeeze)()); + fn(OP_CONVERT_FUNCTION_NAME(BatchNorm)()); + fn(OP_CONVERT_FUNCTION_NAME(MatMul)()); +} + +class OpConvertRegistry { + public: + ~OpConvertRegistry() { Clear(); } + + static void RegisterOneOpConverter(OpNameInfo&& op_info) { GetSingleton().op_map_[op_info.op_type()] = op_info; } + + static void RegisterAllOpConverters() { RegisterOpConverters(RegisterOneOpConverter); } + + static OpConvertRegistry& GetSingleton() { + static OpConvertRegistry registry = OpConvertRegistry(); + return registry; + } + + static const std::unordered_map& GetOpConvertMap() { return GetSingleton().op_map_; } + + void Clear() noexcept { op_map_.clear(); } + + private: + OpConvertRegistry() {} + + std::unordered_map op_map_; +}; + +class OnnxExporter { + public: + OnnxExporter() {} + ~OnnxExporter() {} + + std::string GetOnnxProtoString(const FuncGraphPtr& func_graph); + + private: + void InitModelInfo(); + + void ExportFuncGraph(const FuncGraphPtr& func_graph, onnx::GraphProto* graph_proto); + void ExportParameters(const FuncGraphPtr& func_graph, onnx::GraphProto* graph_proto); + + size_t ExportPrimitive(const FuncGraphPtr& func_graph, std::map* node_map_ptr, + const PrimitivePtr& prim, const std::vector& inputs, + onnx::GraphProto* graph_proto); + + static onnx::TensorProto_DataType GetOnnxDataType(TypeId type_id); + void SetValueInfoType(const AnfNodePtr& node, onnx::ValueInfoProto* value_proto, bool is_output = false); + void SetTensorProtoInfo(const ParameterPtr& param, onnx::TensorProto* tensor_proto); + + void MatchAndMark(const FuncGraphPtr& func_graph, const std::vector& nodes, + std::unordered_map* op_merged_infos_ptr); + void ExportNodes(const FuncGraphPtr& func_graph, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + + void ExportCNode(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + + void ExportPrimReshape(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* graph_proto); + void ExportPrimReduceMean(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* graph_proto); + void ExportPrimCast(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + void ExportPrimPReLU(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + + void ExportMergeConv(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + void ExportMergeGemm(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + void ExportMergeBatchNorm(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* graph_proto); + + void ExportOutput(const FuncGraphPtr& func_graph, const CNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* graph_proto); + std::string GetNodeInputName(const AnfNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* const graph_proto); + + void ConvertTupleToTensor(const ValuePtr& value, onnx::TensorProto* tensor_proto); + void SetNodeAttribute(const ValuePtr& value, onnx::NodeProto* node_proto); + + size_t AllocateNodeIndex() { return ++onnx_node_index_; } + + void ResetNodeIndex() { onnx_node_index_ = 0; } + + static int GetInt32Value(const AnfNodePtr& node) { + auto value_node_ptr = dyn_cast(node); + MS_EXCEPTION_IF_NULL(value_node_ptr); + return GetValue(value_node_ptr->value()); + } + + onnx::ModelProto model_; + + size_t onnx_node_index_ = 0; +}; + +std::string OnnxExporter::GetOnnxProtoString(const FuncGraphPtr& func_graph) { + if (func_graph == nullptr) { + return ""; + } + ResetNodeIndex(); + OpConvertRegistry::GetSingleton().Clear(); + OpConvertRegistry::RegisterAllOpConverters(); + InitModelInfo(); + onnx::GraphProto* graph_proto = model_.mutable_graph(); + ExportFuncGraph(func_graph, graph_proto); + return model_.SerializeAsString(); +} + +void OnnxExporter::InitModelInfo() { + model_.set_ir_version(onnx::IR_VERSION_2019_1_22); + model_.set_producer_name("MindSpore"); + model_.set_producer_version("1.0"); + onnx::OperatorSetIdProto* opset_proto = model_.add_opset_import(); + opset_proto->set_version(9); +} + +void OnnxExporter::ExportFuncGraph(const FuncGraphPtr& func_graph, onnx::GraphProto* const graph_proto) { + std::map node_map; + + onnx_node_index_ = func_graph->parameters().size(); + + // set graph name + graph_proto->set_name(func_graph->ToString()); + + // export parameters + // 1. all parameters (with or without default value) will be mapped to ONNX parameters + // 2. parameters with default value will mapped to ONNX initializers + ExportParameters(func_graph, graph_proto); + + // export computational nodes and output nodes + ExportNodes(func_graph, &node_map, graph_proto); +} + +void OnnxExporter::ExportParameters(const FuncGraphPtr& func_graph, onnx::GraphProto* const graph_proto) { + for (auto& param : func_graph->parameters()) { + const ParameterPtr param_ptr = dyn_cast(param); + if (param_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Parameter '" << param->ToString() << "' could not cast to parameter."; + } + + onnx::ValueInfoProto* input_proto = graph_proto->add_input(); + input_proto->set_name(param_ptr->ToString()); + SetValueInfoType(param_ptr, input_proto); + + if (!param_ptr->has_default()) { + continue; + } + // parameter with default value is an ONNX initializer + onnx::TensorProto* initializer_proto = graph_proto->add_initializer(); + initializer_proto->set_name(param_ptr->ToString()); + SetTensorProtoInfo(param_ptr, initializer_proto); + // set value for initializer + py::object obj = param_ptr->default_param(); + py::object data = obj.attr("data"); + if (py::isinstance(data)) { + auto method = data.attr("asnumpy"); + py::array npy_data = method(); + initializer_proto->set_raw_data(npy_data.request(true).ptr, static_cast(npy_data.nbytes())); + } + } +} + +onnx::TensorProto_DataType OnnxExporter::GetOnnxDataType(TypeId type_id) { + // clang-format off + static std::unordered_map type_map = { + {kNumberTypeBool, onnx::TensorProto_DataType_BOOL}, + {kNumberTypeInt8, onnx::TensorProto_DataType_INT8}, + {kNumberTypeInt16, onnx::TensorProto_DataType_INT16}, + {kNumberTypeInt32, onnx::TensorProto_DataType_INT32}, + {kNumberTypeInt64, onnx::TensorProto_DataType_INT64}, + {kNumberTypeUInt8, onnx::TensorProto_DataType_UINT8}, + {kNumberTypeUInt16, onnx::TensorProto_DataType_UINT16}, + {kNumberTypeUInt32, onnx::TensorProto_DataType_UINT32}, + {kNumberTypeUInt64, onnx::TensorProto_DataType_UINT64}, + {kNumberTypeFloat16, onnx::TensorProto_DataType_FLOAT16}, + {kNumberTypeFloat32, onnx::TensorProto_DataType_FLOAT}, + {kNumberTypeFloat64, onnx::TensorProto_DataType_DOUBLE}, + }; + // clang-format on + + auto iter = type_map.find(type_id); + if (iter == type_map.end()) { + MS_LOG(EXCEPTION) << "Convert type error, unsupported type " << type_id; + } + + return iter->second; +} + +void OnnxExporter::SetValueInfoType(const AnfNodePtr& node, onnx::ValueInfoProto* const value_proto, bool is_output) { + auto dtype = node->Type(); + auto shape = node->Shape(); + onnx::TypeProto* type_proto = value_proto->mutable_type(); + if (dtype->isa() && shape->isa()) { + auto tensor = dyn_cast(dtype); + auto elem_type = tensor->element(); + const auto& dims = dyn_cast(shape)->shape(); + // output type of 'Argmax' of MindSpore is int32, output type of 'ArgMax' of ONNX is int64 + auto type = is_output ? onnx::TensorProto_DataType_INT64 : GetOnnxDataType(elem_type->type_id()); + type_proto->mutable_tensor_type()->set_elem_type(type); + + for (const auto& dim : dims) { + type_proto->mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); + } + } +} + +void OnnxExporter::SetTensorProtoInfo(const ParameterPtr& param, onnx::TensorProto* const tensor_proto) { + auto dtype = param->Type(); + auto shape = param->Shape(); + if (!dtype->isa() || !shape->isa()) { + MS_LOG(EXCEPTION) << "Parameter " << param->name() << " is not a regular tensor, with value " << param->ToString(); + } + + auto tensor = dyn_cast(dtype); + auto elem_type = tensor->element(); + const auto& dims = dyn_cast(shape)->shape(); + tensor_proto->set_data_type(GetOnnxDataType(elem_type->type_id())); + for (const auto& dim : dims) { + tensor_proto->add_dims(dim); + } +} + +void OnnxExporter::MatchAndMark(const FuncGraphPtr& func_graph, const std::vector& nodes, + std::unordered_map* op_merged_infos_ptr) { + std::unordered_map& op_merged_infos = *op_merged_infos_ptr; + + for (auto& node : nodes) { + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + if (cnode == func_graph->get_return()) { + // if the key `input` does not exist, just create a new one + op_merged_infos[cnode].referred_count += 1; + } + for (auto& input : cnode->inputs()) { + if (!input->isa()) { + continue; + } + // if the key `input` does not exist, just create a new one + op_merged_infos[input].referred_count += 1; + } + // MindSpore Conv + BiasAdd --> ONNX Conv + if (cnode->IsApply(std::make_shared("BiasAdd")) && + IsPrimitiveCNode(cnode->input(1), prim::kPrimConv2D)) { + op_merged_infos[cnode].mode = OP_MERGE_CONV; + op_merged_infos[cnode->input(1)].mode = OP_MERGE_IGNORE; + op_merged_infos[cnode->input(1)].referred_count -= 1; + } else if (cnode->IsApply(std::make_shared("BiasAdd")) && + IsPrimitiveCNode(cnode->input(1), prim::kPrimMatMul)) { + op_merged_infos[cnode].mode = OP_MERGE_GEMM; + op_merged_infos[cnode->input(1)].mode = OP_MERGE_IGNORE; + op_merged_infos[cnode->input(1)].referred_count -= 1; + } else if (cnode->IsApply(prim::kPrimTupleGetItem) && + IsPrimitiveCNode(cnode->input(1), std::make_shared("BatchNorm")) && + GetInt32Value(cnode->input(2)) == 0) { + op_merged_infos[cnode].mode = OP_MERGE_BATCH_NORM; + op_merged_infos[cnode->input(1)].mode = OP_MERGE_IGNORE; + op_merged_infos[cnode->input(1)].referred_count -= 1; + } + } +} + +/** + * AnfNode + * +-- CNode + * +-- ANode + * | +-- Parameter + * | `-- ValueNode + */ +void OnnxExporter::ExportNodes(const FuncGraphPtr& func_graph, std::map* node_map_ptr, + onnx::GraphProto* const graph_proto) { + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + + std::unordered_map op_merged_infos; + MatchAndMark(func_graph, nodes, &op_merged_infos); + + for (const AnfNodePtr& node : nodes) { + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + auto iter = op_merged_infos.find(cnode); + // the node is not referenced by any other nodes, skip it + if (iter == op_merged_infos.end()) { + continue; + } + auto merged_info = iter->second; + // the op node is merged with other node and not used any more, skip it + if (merged_info.mode == OP_MERGE_IGNORE && merged_info.referred_count == 0) { + continue; + } + if (cnode == func_graph->get_return()) { + ExportOutput(func_graph, cnode, node_map_ptr, graph_proto); + continue; + } + switch (merged_info.mode) { + case OP_MERGE_CONV: + ExportMergeConv(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_GEMM: + ExportMergeGemm(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_BATCH_NORM: + ExportMergeBatchNorm(func_graph, cnode, node_map_ptr, graph_proto); + break; + default: + ExportCNode(func_graph, cnode, node_map_ptr, graph_proto); + break; + } + } +} + +void OnnxExporter::ExportPrimReshape(const FuncGraphPtr& /*func_graph*/, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + auto name_x = GetNodeInputName(node->input(1), node_map_ptr, graph_proto); + auto input_shape = node->input(2); + std::string name_shape; + if (input_shape->isa()) { + auto const_node_idx = AllocateNodeIndex(); + (*node_map_ptr)[input_shape] = const_node_idx; + onnx::NodeProto* node_proto = graph_proto->add_node(); + name_shape = std::to_string(const_node_idx); + node_proto->add_output(name_shape); + + node_proto->set_op_type("Constant"); + onnx::AttributeProto* attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + ConvertTupleToTensor(dyn_cast(input_shape)->value(), attr_proto->mutable_t()); + } else { + name_shape = GetNodeInputName(input_shape, node_map_ptr, graph_proto); + MS_LOG(EXCEPTION) << "Need to insert op convert variable from tuple to tensor for Reshape."; + } + + auto node_idx = AllocateNodeIndex(); + (*node_map_ptr)[node] = node_idx; + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->set_op_type(prim::kPrimReshape->name()); + node_proto->add_output(std::to_string(node_idx)); + node_proto->add_input(name_x); + node_proto->add_input(name_shape); +} + +void OnnxExporter::ExportPrimReduceMean(const FuncGraphPtr& /*func_graph*/, const CNodePtr& node, + std::map* node_map_ptr, + onnx::GraphProto* const graph_proto) { + auto input_data = GetNodeInputName(node->input(1), node_map_ptr, graph_proto); + auto input_axis = node->input(2); + + auto node_idx = AllocateNodeIndex(); + (*node_map_ptr)[node] = node_idx; + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->set_op_type(prim::kPrimReduceMean->name()); + node_proto->add_output(std::to_string(node_idx)); + node_proto->add_input(input_data); + + if (input_axis->isa()) { + onnx::AttributeProto* attr_proto = node_proto->add_attribute(); + attr_proto->set_name("axes"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + auto axis_value = dyn_cast(input_axis)->value(); + auto tuple_ptr = dyn_cast(axis_value); + MS_EXCEPTION_IF_NULL(tuple_ptr); + for (size_t i = 0; i < tuple_ptr->size(); ++i) { + attr_proto->add_ints(GetValue((*tuple_ptr)[i])); + } + } else { + MS_LOG(EXCEPTION) << "Need to insert op convert variable from tuple to attributes for ReduceMean."; + } +} + +void OnnxExporter::ExportPrimCast(const FuncGraphPtr& /*func_graph*/, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + auto input_data = GetNodeInputName(node->input(1), node_map_ptr, graph_proto); + auto input_type = node->input(2); + + auto node_idx = AllocateNodeIndex(); + (*node_map_ptr)[node] = node_idx; + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->set_op_type(prim::kPrimCast->name()); + node_proto->add_output(std::to_string(node_idx)); + node_proto->add_input(input_data); + + if (input_type->isa()) { + onnx::AttributeProto* attr_proto = node_proto->add_attribute(); + attr_proto->set_name("to"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + auto type_value = dyn_cast(input_type)->value(); + auto type_ptr = dyn_cast(type_value); + MS_EXCEPTION_IF_NULL(type_ptr); + attr_proto->set_i(GetOnnxDataType(type_ptr->type_id())); + } else { + MS_LOG(EXCEPTION) << "Need to convert MindSpore Cast input(1) to ONNX Cast to attribute."; + } +} + +void OnnxExporter::ExportPrimPReLU(const FuncGraphPtr& /*func_graph*/, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + auto input_x = GetNodeInputName(node->input(1), node_map_ptr, graph_proto); + auto input_slope = GetNodeInputName(node->input(2), node_map_ptr, graph_proto); + + auto x_shape = dyn_cast(node->input(1)->Shape()); + auto slope_shape = dyn_cast(node->input(2)->Shape()); + MS_EXCEPTION_IF_NULL(x_shape); + MS_EXCEPTION_IF_NULL(slope_shape); + + // format of x is NCHW, input format is NCHW, if length of input_slope is 1, insert Unsqueeze [1,2] + if (x_shape->shape().size() == 4 && slope_shape->shape().size() == 1) { + auto node_idx = AllocateNodeIndex(); + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->set_op_type("Unsqueeze"); + node_proto->add_output(std::to_string(node_idx)); + + onnx::AttributeProto* attr_proto = node_proto->add_attribute(); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + attr_proto->set_name("axes"); + attr_proto->add_ints(1); + attr_proto->add_ints(2); + + node_proto->add_input(input_slope); + input_slope = std::to_string(node_idx); + } + + auto node_idx = AllocateNodeIndex(); + (*node_map_ptr)[node] = node_idx; + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->set_op_type("PRelu"); + node_proto->add_output(std::to_string(node_idx)); + node_proto->add_input(input_x); + node_proto->add_input(input_slope); +} + +void OnnxExporter::ExportCNode(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + // Type of the 2nd input of 'Reshape' of MindSpore is tuple, but ONNX's is tensor, need to do some convert + if (node->IsApply(prim::kPrimReshape)) { + return ExportPrimReshape(func_graph, node, node_map_ptr, graph_proto); + } + + if (node->IsApply(prim::kPrimReduceMean)) { + return ExportPrimReduceMean(func_graph, node, node_map_ptr, graph_proto); + } + + // MindSpore Cast(x, T) --> ONNX Cast[to=T](x) + if (node->IsApply(prim::kPrimCast)) { + return ExportPrimCast(func_graph, node, node_map_ptr, graph_proto); + } + + // ONNX PRelu requires unidirectional broadcasting, here need some process + if (node->IsApply(std::make_shared("PReLU"))) { + return ExportPrimPReLU(func_graph, node, node_map_ptr, graph_proto); + } + + auto inputs = node->inputs(); + if (inputs.size() < 1) { + MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; + } + + AnfNodePtr op = inputs[0]; + std::vector op_inputs; + // first process node input 1,2,..., since when node input is a ValueNode, here need to create a Constant Operator + for (size_t i = 1; i < inputs.size(); i++) { + op_inputs.push_back(inputs[i]); + } + auto op_value = dyn_cast(op); + if (op_value == nullptr) { + MS_LOG(EXCEPTION) << "Need to support node op type " << op->type_name(); + } + auto prim = dyn_cast(op_value->value()); + if (prim == nullptr) { + MS_LOG(EXCEPTION) << "Need to support node op type " << op_value->value()->type_name(); + } + + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim, op_inputs, graph_proto); +} + +size_t OnnxExporter::ExportPrimitive(const FuncGraphPtr& /*func_graph*/, std::map* node_map_ptr, + const PrimitivePtr& prim, const std::vector& inputs, + onnx::GraphProto* const graph_proto) { + auto op_map = OpConvertRegistry::GetOpConvertMap(); + auto op_iter = op_map.find(prim->name()); + if (op_iter == op_map.end()) { + MS_LOG(EXCEPTION) << "Can not find key " << prim->name() << " in convert map"; + } + const OpNameInfo& op_convert_info = op_iter->second; + + auto node_idx = AllocateNodeIndex(); + + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->add_output(std::to_string(node_idx)); + node_proto->set_op_type(op_convert_info.onnx_type()); + + // Set inputs + for (const auto& input : inputs) { + auto input_name = GetNodeInputName(input, node_map_ptr, graph_proto); + node_proto->add_input(input_name); + } + + // Set node attribute + for (const OpAttrInfo& attr : op_convert_info.op_attrs()) { + const std::string& attr_name = attr.attr_name(); + ValuePtr attr_value = nullptr; + if (!attr_name.empty()) { + attr_value = prim->GetAttr(attr_name); + if (attr_value == nullptr) { + MS_LOG(EXCEPTION) << "Primitive " << prim->name() << " does not have attribute " << attr_name; + } + } + onnx::AttributeProto* onnx_attr_proto = node_proto->add_attribute(); + onnx_attr_proto->set_name(attr.onnx_attr_name()); + attr.fn_gen_attr()(attr_value, attr.onnx_attr_type(), onnx_attr_proto, prim); + } + return node_idx; +} + +void OnnxExporter::ExportMergeConv(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + auto conv_node = dyn_cast(node->input(1)); + auto input_x = conv_node->input(1); // conv input x + auto input_w = conv_node->input(2); // conv weight(filter) + auto input_b = node->input(2); // conv bias + + PrimitivePtr prim_conv = dyn_cast((dyn_cast(conv_node->input(0)))->value()); + std::vector inputs{input_x, input_w, input_b}; + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_conv, inputs, graph_proto); +} + +void OnnxExporter::ExportMergeGemm(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + auto matmul_node = dyn_cast(node->input(1)); + auto input_x = matmul_node->input(1); // matmul input x + auto input_y = matmul_node->input(2); // matmul input y + auto input_b = node->input(2); // matmul bias + + PrimitivePtr prim_matmul = dyn_cast((dyn_cast(matmul_node->input(0)))->value()); + std::vector inputs{input_x, input_y, input_b}; + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_matmul, inputs, graph_proto); +} + +void OnnxExporter::ExportMergeBatchNorm(const FuncGraphPtr& func_graph, const CNodePtr& node, + std::map* node_map_ptr, + onnx::GraphProto* const graph_proto) { + auto batch_norm_node = dyn_cast(node->input(1)); + + PrimitivePtr prim_batch_norm = dyn_cast((dyn_cast(batch_norm_node->input(0)))->value()); + std::vector inputs; + for (size_t i = 1; i < batch_norm_node->inputs().size(); i++) { + inputs.push_back(batch_norm_node->input(i)); + } + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_batch_norm, inputs, graph_proto); +} + +void OnnxExporter::ExportOutput(const FuncGraphPtr& /*func_graph*/, const CNodePtr& node, + std::map* node_map_ptr, onnx::GraphProto* const graph_proto) { + if (node->inputs().size() != 2) { + MS_LOG(EXCEPTION) << "Number of inputs of return node is not equal to 2."; + } + AnfNodePtr arg = node->input(1); + std::string name = GetNodeInputName(arg, node_map_ptr, graph_proto); + onnx::ValueInfoProto* output_proto = graph_proto->add_output(); + output_proto->set_name(name); + SetValueInfoType(arg, output_proto, false); +} + +std::string OnnxExporter::GetNodeInputName(const AnfNodePtr& node, std::map* node_map_ptr, + onnx::GraphProto* const graph_proto) { + if (node->isa()) { + auto iter = node_map_ptr->find(node); + if (iter == node_map_ptr->end()) { + MS_LOG(EXCEPTION) << "Can not find node '" << node->ToString() << "' in node_map"; + } + return std::to_string(iter->second); + } + + if (node->isa()) { + return node->ToString(); + } + + // for ValueNode input, create a Constant Operator + if (node->isa()) { + auto iter = node_map_ptr->find(node); + if (iter != node_map_ptr->end()) { + return std::to_string(iter->second); + } + // the id number starts at 1, so the id of created node should be size of map plus one + auto node_idx = AllocateNodeIndex(); + (*node_map_ptr)[node] = node_idx; + std::string node_name = std::to_string(node_idx); + + onnx::NodeProto* node_proto = graph_proto->add_node(); + node_proto->add_output(node_name); + + SetNodeAttribute(node->cast()->value(), node_proto); + + return node_name; + } + + MS_LOG(EXCEPTION) << "Unexpected node type " << node->type_name(); +} + +void OnnxExporter::ConvertTupleToTensor(const ValuePtr& value, onnx::TensorProto* const tensor_proto) { + auto tuple_ptr = dyn_cast(value); + MS_EXCEPTION_IF_NULL(tuple_ptr); + if (tuple_ptr->size() == 0) { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, the size of converted tuple is 0."; + } + auto type_id = (*tuple_ptr)[0]->type()->type_id(); + for (size_t i = 1; i < tuple_ptr->size(); ++i) { + if ((*tuple_ptr)[i]->type()->type_id() != type_id) { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, type of tuple elements is not same."; + } + } + + tensor_proto->add_dims(static_cast<::google::protobuf::int64>(tuple_ptr->size())); + tensor_proto->set_data_type(onnx::TensorProto_DataType_INT64); + for (size_t i = 0; i < tuple_ptr->size(); ++i) { + ValuePtr elem = (*tuple_ptr)[i]; + if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, unexpected tuple element type " << elem->type()->type_name() + << "."; + } + } +} + +void OnnxExporter::SetNodeAttribute(const ValuePtr& value, onnx::NodeProto* const node_proto) { + node_proto->set_op_type("Constant"); + onnx::AttributeProto* attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + MS_LOG(EXCEPTION) << "Need to set value " << value->ToString() << " attribute for Constant node"; +} + +std::string GetOnnxProtoString(const FuncGraphPtr& func_graph) { + OnnxExporter exporter; + return exporter.GetOnnxProtoString(func_graph); +} +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/CMakeLists.txt b/mindspore/ccsrc/operator/CMakeLists.txt new file mode 100644 index 00000000000..328b4cf787d --- /dev/null +++ b/mindspore/ccsrc/operator/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _OPERATOR_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_operator_obj OBJECT ${_OPERATOR_ALL_SRC_FILES}) diff --git a/mindspore/ccsrc/operator/cc_implementations.cc b/mindspore/ccsrc/operator/cc_implementations.cc new file mode 100644 index 00000000000..5ff49758b4d --- /dev/null +++ b/mindspore/ccsrc/operator/cc_implementations.cc @@ -0,0 +1,400 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/cc_implementations.h" +#include +#include +#include +#include +#include +#include "utils/misc.h" +#include "utils/log_adapter.h" +#include "utils/convert_utils.h" +#include "common/utils.h" + +namespace mindspore { +// namespace to support primitive operators definition +namespace prim { +enum class DataType { kInt, kFloat, kDouble, kUnknown }; + +// Whether has a T type data in AnyPtrList. +template +bool HasType(const AnyPtrList& list) { + bool ret = std::any_of(list.begin(), list.end(), [](const AnyPtr& ptr) { return ptr->is(); }); + return ret; +} + +DataType InferType(const AnyPtrList& list) { + if (HasType(list)) { + return DataType::kDouble; + } else if (HasType(list)) { + return DataType::kFloat; + } else if (HasType(list)) { + return DataType::kInt; + } + return DataType::kUnknown; +} + +enum OpType { ADD, SUB, MUL, DIV, MOD }; + +template +bool IsSignedIntOverflow(T x, T y, OpType opType) { + auto max = std::numeric_limits::max(); + auto min = std::numeric_limits::min(); + + if (opType == OpType::ADD) { + return (y > 0 && (max - y) < x) || (y < 0 && (min - y) > x); + } + + if (opType == OpType::SUB) { + return (y < 0 && (max + y) < x) || (y > 0 && (min + y) > x); + } + + if (opType == OpType::MUL) { + return (x > 0 && y > 0 && (max / y) < x) || (x < 0 && y < 0 && (max / y) > x) || + (x > 0 && y < 0 && (min / y) < x) || (x < 0 && y > 0 && (min / y) > x); + } + + if (opType == OpType::DIV || opType == OpType::MOD) { + return x == min && static_cast(y) == -1; + } + + MS_LOG(EXCEPTION) << "Unsupported operation type."; +} + +template +T InnerScalarAdd(T x, T y) { + if (std::is_integral::value && std::is_signed::value && IsSignedIntOverflow(x, y, OpType::ADD)) { + MS_LOG(EXCEPTION) << "Overflow of the sum of two signed number x: " << std::to_string(x) + << ", y: " << std::to_string(y) << "."; + } + return x + y; +} + +template +T InnerScalarSub(T x, T y) { + if (std::is_integral::value && std::is_signed::value && IsSignedIntOverflow(x, y, OpType::SUB)) { + MS_LOG(EXCEPTION) << "Overflow of the sub of two signed number x: " << std::to_string(x) + << ", y: " << std::to_string(y) << "."; + } + return x - y; +} + +template +T InnerScalarMul(T x, T y) { + if (std::is_integral::value && std::is_signed::value && IsSignedIntOverflow(x, y, OpType::MUL)) { + MS_LOG(EXCEPTION) << "Overflow of the mul of two signed number x: " << std::to_string(x) + << ", y: " << std::to_string(y) << "."; + } + return x * y; +} + +template +T InnerScalarDiv(T x, T y) { + if (y == 0) { + MS_LOG(EXCEPTION) << "Divisor could not be zero"; + } + if (std::is_integral::value && std::is_signed::value && IsSignedIntOverflow(x, y, OpType::DIV)) { + MS_LOG(EXCEPTION) << "Overflow of the div of two signed number x: " << std::to_string(x) + << ", y: " << std::to_string(y) << "."; + } + return x / y; +} + +int32_t InnerScalarMod(int32_t x, int32_t y) { + if (y == 0) { + MS_LOG(EXCEPTION) << "Could not mod to zero."; + } + if (IsSignedIntOverflow(x, y, OpType::MOD)) { + MS_LOG(EXCEPTION) << "Overflow of the mod of two signed number x: " << std::to_string(x) + << ", y: " << std::to_string(y) << "."; + } + return x % y; +} + +float InnerScalarMod(float, float) { MS_LOG(EXCEPTION) << "Float does not support mod operator."; } + +double InnerScalarMod(double, double) { MS_LOG(EXCEPTION) << "Double does not support mod operator."; } + +template +bool InnerScalarEq(T x, U y) { + double error = static_cast(x) - static_cast(y); + error = fabs(error); + return error < DBL_EPSILON; +} + +template +bool InnerScalarLt(T x, U y) { + return x < y; +} + +template +bool InnerScalarGt(T x, U y) { + return x > y; +} + +template +bool InnerScalarNe(T x, U y) { + return !InnerScalarEq(x, y); +} + +template +bool InnerScalarLe(T x, U y) { + return x <= y; +} + +template +bool InnerScalarGe(T x, U y) { + return x >= y; +} + +#define SCALAR_OP(op_t) \ + ValuePtr Scalar##op_t(const ValuePtrList& list) { \ + do { \ + if (list.size() < 2) { \ + MS_LOG(EXCEPTION) << "length of input list for Scalar" << #op_t << " is less than 2."; \ + } \ + ValuePtr x = list[0]; \ + ValuePtr y = list[1]; \ + MS_EXCEPTION_IF_NULL(x); \ + MS_EXCEPTION_IF_NULL(y); \ + if (x->isa() && y->isa()) { \ + double sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + float sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + int sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + MS_LOG(EXCEPTION) << "Unsupported Value for Scalar" << #op_t << ", x: " << x->ToString() \ + << ", y: " << y->ToString(); \ + } while (0); \ + } + +SCALAR_OP(Add) +SCALAR_OP(Sub) +SCALAR_OP(Mul) +SCALAR_OP(Div) +SCALAR_OP(Mod) + +#define LOGIC_OP(op_t) \ + ValuePtr Scalar##op_t(const ValuePtrList& list) { \ + if (list.size() < 2) { \ + MS_LOG(EXCEPTION) << "length of input list for Scalar" << #op_t << " is less than 2."; \ + } \ + ValuePtr x = list[0]; \ + ValuePtr y = list[1]; \ + MS_EXCEPTION_IF_NULL(x); \ + MS_EXCEPTION_IF_NULL(y); \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + if (x->isa() && y->isa()) { \ + bool sum = InnerScalar##op_t(GetValue(x), GetValue(y)); \ + return MakeValue(sum); \ + } \ + MS_LOG(EXCEPTION) << "Unsupported Value for Scalar" << #op_t << ", x: " << x->ToString() \ + << ", y: " << y->ToString() << "."; \ + } + +LOGIC_OP(Eq) +LOGIC_OP(Lt) +LOGIC_OP(Gt) +LOGIC_OP(Ne) +LOGIC_OP(Le) +LOGIC_OP(Ge) + +ValuePtr ScalarUAdd(const ValuePtrList& list) { + if (list.size() != 1) { + MS_LOG(EXCEPTION) << "Input number of ScalarUAdd should be 1, but got " << list.size(); + } + ValuePtr x = list[0]; + MS_EXCEPTION_IF_NULL(x); + return x; +} + +ValuePtr ScalarUSub(const ValuePtrList& list) { + if (list.size() != 1) { + MS_LOG(EXCEPTION) << "Input number of ScalarUSub should be 1, but got " << list.size(); + } + ValuePtr x = list[0]; + MS_EXCEPTION_IF_NULL(x); + + if (x->isa()) { + int32_t sum = -1 * GetValue(x); + return MakeValue(sum); + } + if (x->isa()) { + float sum = -1.0f * GetValue(x); + return MakeValue(sum); + } + + MS_LOG(EXCEPTION) << "Unsported Value for ScalarUSub, x: " << x->ToString() << "."; +} + +ValuePtr ScalarLog(const ValuePtrList& list) { + if (list.empty()) { + MS_LOG(EXCEPTION) << "Input list of ScalarLog is empty."; + } + ValuePtr x = list[0]; + MS_EXCEPTION_IF_NULL(x); + + if (x->isa()) { + double v = log(GetValue(x)); + return MakeValue(v); + } + if (x->isa()) { + auto v = static_cast(log(GetValue(x))); + return MakeValue(v); + } + + MS_LOG(EXCEPTION) << "Unsported Value for ScalarLog, x: " << x->ToString(); +} + +ValuePtr BoolNot(const ValuePtrList& list) { + if (list.empty()) { + MS_LOG(EXCEPTION) << "value list of BoolNot is empty"; + } + ValuePtr x = list[0]; + MS_EXCEPTION_IF_NULL(x); + bool convert = false; + + if (ValueToBool(x, &convert)) { + auto res = !convert; + return MakeValue(res); + } + + MS_LOG(EXCEPTION) << "Unsported Value for BoolNot, x: " << x->ToString(); +} + +ValuePtr BoolAnd(const ValuePtrList& list) { + if (list.size() < 2) { + MS_LOG(EXCEPTION) << "Input number " << list.size() << " of BoolAnd is less then 2."; + } + ValuePtr x = list[0]; + ValuePtr y = list[1]; + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(y); + bool x_b = false; + bool y_b = false; + + if (ValueToBool(x, &x_b) && ValueToBool(y, &y_b)) { + auto res = x_b && y_b; + return MakeValue(res); + } + + MS_LOG(EXCEPTION) << "Unsported Value for BoolAnd, x: " << x->ToString() << "."; +} + +ValuePtr BoolOr(const ValuePtrList& list) { + if (list.size() < 2) { + MS_LOG(EXCEPTION) << "Input number " << list.size() << " of BoolOr is less then 2."; + } + ValuePtr x = list[0]; + ValuePtr y = list[1]; + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(y); + bool x_b = false; + bool y_b = false; + + if (ValueToBool(x, &x_b) && ValueToBool(y, &y_b)) { + auto res = x_b || y_b; + return MakeValue(res); + } + + MS_LOG(EXCEPTION) << "Unsported Value for BoolOr, x: " << x->ToString() << "."; +} + +ValuePtr BoolEq(const ValuePtrList& list) { + if (list.size() < 2) { + MS_LOG(EXCEPTION) << "Input number " << list.size() << " of BoolEq is less than 2."; + } + ValuePtr x = list[0]; + ValuePtr y = list[1]; + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(y); + bool x_b = false; + bool y_b = false; + + if (ValueToBool(x, &x_b) && ValueToBool(y, &y_b)) { + auto res = x_b == y_b; + return MakeValue(res); + } + + MS_LOG(EXCEPTION) << "Unsported Value for BoolEq, x: " << x->ToString() << "."; +} + +std::vector BroadcastShape_(std::vector shpx, std::vector shpy) { + int dlen = SizeToInt(shpx.size()) - SizeToInt(shpy.size()); + if (dlen < 0) { + for (int i = 0; i < -dlen; ++i) { + (void)shpx.insert(shpx.begin(), 1); + } + } else if (dlen > 0) { + for (int i = 0; i < dlen; i++) { + (void)shpy.insert(shpy.begin(), 1); + } + } + if (shpx.size() != shpy.size()) { + MS_LOG(EXCEPTION) << "Failure: shpx.size() != shpy.size()."; + } + std::vector shp; + for (size_t i = 0; i < shpx.size(); i++) { + auto a = shpx[i]; + auto b = shpy[i]; + if (a == 1) { + shp.push_back(b); + } else if (b == 1) { + shp.push_back(a); + } else if (a == -1) { + shp.push_back(b); + } else if (b == -1) { + shp.push_back(a); + } else if (a == b) { + shp.push_back(a); + } else { + return std::vector(); + } + } + return shp; +} +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/cc_implementations.h b/mindspore/ccsrc/operator/cc_implementations.h new file mode 100644 index 00000000000..2c2936fc92e --- /dev/null +++ b/mindspore/ccsrc/operator/cc_implementations.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_CC_IMPLEMENTATIONS_H_ +#define MINDSPORE_CCSRC_OPERATOR_CC_IMPLEMENTATIONS_H_ + +#include +#include +#include "ir/anf.h" +#include "ir/value.h" +#include "utils/any.h" + +namespace mindspore { +// namespace to support primitive operators definition +namespace prim { +using Any = mindspore::Any; +using AnyPtrList = std::vector>; +using ValuePtrList = std::vector; +using OpsFunction = std::function; +using AnfNodeOpsFunction = std::function&)>; + +ValuePtr ScalarAdd(const ValuePtrList& list); +ValuePtr ScalarSub(const ValuePtrList& list); +ValuePtr ScalarMul(const ValuePtrList& list); +ValuePtr ScalarDiv(const ValuePtrList& list); +ValuePtr ScalarMod(const ValuePtrList& list); +ValuePtr ScalarUAdd(const ValuePtrList& list); +ValuePtr ScalarUSub(const ValuePtrList& list); +ValuePtr ScalarUSub(const ValuePtrList& list); +ValuePtr ScalarLog(const ValuePtrList& list); +ValuePtr ScalarEq(const ValuePtrList& list); +ValuePtr ScalarLt(const ValuePtrList& list); +ValuePtr ScalarGt(const ValuePtrList& list); +ValuePtr ScalarNe(const ValuePtrList& list); +ValuePtr ScalarLe(const ValuePtrList& list); +ValuePtr ScalarGe(const ValuePtrList& list); +ValuePtr BoolNot(const ValuePtrList& list); +ValuePtr BoolAnd(const ValuePtrList& list); +ValuePtr BoolOr(const ValuePtrList& list); +ValuePtr BoolEq(const ValuePtrList& list); +std::vector BroadcastShape_(std::vector s1, std::vector s2); +} // namespace prim +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPERATOR_CC_IMPLEMENTATIONS_H_ diff --git a/mindspore/ccsrc/operator/composite/composite.cc b/mindspore/ccsrc/operator/composite/composite.cc new file mode 100644 index 00000000000..0c55b9480c8 --- /dev/null +++ b/mindspore/ccsrc/operator/composite/composite.cc @@ -0,0 +1,1267 @@ + +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/composite/composite.h" +#include +#include +#include + +#include "ir/anf.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pipeline/static_analysis/abstract_function.h" +#include "pipeline/static_analysis/dshape.h" +#include "pipeline/static_analysis/param_validator.h" +#include "operator/cc_implementations.h" +#include "optimizer/opt.h" +#include "utils/symbolic.h" +#include "pybind_api/api_register.h" +#include "./common.h" +#include "ir/signature.h" +#include "debug/trace.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +using AbstractTensor = mindspore::abstract::AbstractTensor; +using FuncGraphAbstractClosure = mindspore::abstract::FuncGraphAbstractClosure; + +using mindspore::abstract::AbstractAttribute; +using mindspore::abstract::AbstractBase; +using mindspore::abstract::AbstractClass; +using mindspore::abstract::AbstractDictionary; +using mindspore::abstract::AbstractDictionaryPtr; +using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractFunctionPtr; +using mindspore::abstract::AbstractList; +using mindspore::abstract::AbstractNone; +using mindspore::abstract::AbstractScalar; +using mindspore::abstract::AbstractSlice; +using mindspore::abstract::AbstractTuple; + +ElemwiseMap kElemwiseMap = {{"__add__", kPrimScalarAdd}, {"__sub__", kPrimScalarSub}, {"__mul__", kPrimScalarMul}, + {"__truediv__", nullptr}, {"__floordiv__", nullptr}, {"__mod__", kPrimScalarMod}, + {"__pow__", kPrimScalarPow}, {"__eq__", kPrimScalarEq}, {"__lt__", kPrimScalarLt}, + {"__gt__", kPrimScalarGt}, {"__ne__", kPrimScalarNe}, {"__le__", kPrimScalarLe}, + {"__ge__", kPrimScalarGe}}; + +const MetaFuncGraphPtr kTail = std::make_shared("tail"); + +// copy from python API: reduce. +// Apply a function of two arguments cumulatively to the items of a sequence, +// from left to right, so as to reduce the sequence to a single value.For example, +// reduce(lambda x, y: x + y, [ 1, 2, 3, 4, 5 ]) calculates ((((1 + 2) + 3) + 4) + 5). +AnyPtr Reduce(const OpsFunction& func, const AnyPtrList& list) { + std::shared_ptr ret; + size_t size = list.size(); + if (size < 2) { + MS_LOG(EXCEPTION) << "length of inputs of Reduce is less than 2"; + } + + AnyPtrList input; + input.push_back(list[0]); + input.push_back(list[1]); + ret = std::make_shared(func(input)); + + for (size_t i = 2; i < size; ++i) { + input.clear(); + input.push_back(ret); + input.push_back(list[i]); + ret = std::make_shared(func(input)); + } + + return ret; +} + +AnfNodePtr Reduce(const AnfNodeOpsFunction& func, const std::vector& list) { + size_t size = list.size(); + if (size < 2) { + MS_LOG(EXCEPTION) << "length of inputs of Reduce is less than 2"; + } + + std::vector input; + input.push_back(list[0]); + input.push_back(list[1]); + AnfNodePtr ret = func(input); + + for (size_t i = 2; i < size; ++i) { + input.clear(); + input.push_back(ret); + input.push_back(list[i]); + ret = func(input); + } + + return ret; +} + +ValuePtr kCompositeHyperMap = std::make_shared(); + +void HyperMap::Init() { + if (fn_leaf_) { + name_ = "hyper_map[" + fn_leaf_->name() + "]"; + } + signatures_ = + // def hypermap(func:read, *args:ref): + std::vector({{"func", SignatureEnumRW::kRWRead, SignatureEnumKind::kKindDefault}, + {"args", SignatureEnumRW::kRWRef, SignatureEnumKind::kKindVarPositional}}); +} + +HyperMap::HyperMap(const std::shared_ptr& fn_leaf) + : MetaFuncGraph("hyper_map"), + fn_leaf_(fn_leaf), + broadcast_(false), + nonleaf_({kObjectTypeList, kObjectTypeTuple, kObjectTypeClass}) { + Init(); +} + +HyperMap::HyperMap(const HyperMap& h) + : MetaFuncGraph("hyper_map"), fn_leaf_(h.fn_leaf_), broadcast_(h.broadcast_), nonleaf_(h.nonleaf_) { + Init(); +} + +AnfNodePtr HyperMap::FullMake(TypePtr, const FuncGraphPtr& func_graph, const AnfNodePtr& fn_arg, + const ArgsPairList& arg_map) { + MS_EXCEPTION_IF_NULL(func_graph); + std::vector inputs; + if (fn_arg != nullptr) { + inputs.push_back(fn_arg); + } else { + inputs.push_back(NewValueNode(fn_leaf_)); + } + + (void)std::transform(arg_map.begin(), arg_map.end(), std::back_inserter(inputs), + [](const std::pair& item) { return item.first; }); + return func_graph->NewCNode(inputs); +} + +AnfNodePtr HyperMap::FullMake(const std::shared_ptr& type, const FuncGraphPtr& func_graph, + const AnfNodePtr& fn_arg, const ArgsPairList& arg_map) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(type); + + std::size_t size = type->elements().size(); + bool is_not_same = std::any_of(arg_map.begin(), arg_map.end(), [size](const std::pair& item) { + auto lhs = std::static_pointer_cast(item.second); + MS_EXCEPTION_IF_NULL(lhs); + return lhs->elements().size() != size; + }); + if (is_not_same) { + MS_LOG(EXCEPTION) << "List in HyperMap should have same length"; + } + + // cannot use shared_from_base() also known as this, as it will make a reference cycle on + // hypermap and graph generated, it will cause memory leak. + auto fn_rec = std::make_shared(*this); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimMakeList)); + + for (int i = 0; i < SizeToInt(size); ++i) { + std::vector inputs2; + inputs2.push_back(NewValueNode(fn_rec)); + if (fn_arg != nullptr) { + inputs2.push_back(fn_arg); + } + + (void)std::transform( + arg_map.begin(), arg_map.end(), std::back_inserter(inputs2), + [&func_graph, i](const std::pair& item) { + return func_graph->NewCNode({NewValueNode(prim::kPrimListGetItem), item.first, NewValueNode(i)}); + }); + + inputs.push_back(func_graph->NewCNode(inputs2)); + } + return func_graph->NewCNode(inputs); +} + +AnfNodePtr HyperMap::FullMake(const std::shared_ptr& type, const FuncGraphPtr& func_graph, + const AnfNodePtr& fn_arg, const ArgsPairList& arg_map) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(type); + + std::size_t size = type->elements().size(); + bool is_not_same = std::any_of(arg_map.begin(), arg_map.end(), [size](const std::pair& item) { + auto lhs = std::static_pointer_cast(item.second); + MS_EXCEPTION_IF_NULL(lhs); + return lhs->elements().size() != size; + }); + if (is_not_same) { + MS_LOG(EXCEPTION) << "tuple in HyperMap should have same length"; + } + + // cannot use shared_from_base() also known as this, as it will make a reference cycle on + // hypermap and graph generated, it will cause memory leak. + auto fn_rec = std::make_shared(*this); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + + for (int i = 0; i < SizeToInt(size); ++i) { + std::vector inputs2; + inputs2.push_back(NewValueNode(fn_rec)); + if (fn_arg != nullptr) { + inputs2.push_back(fn_arg); + } + + (void)std::transform( + arg_map.begin(), arg_map.end(), std::back_inserter(inputs2), [&func_graph, &i](std::pair item) { + return func_graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), item.first, NewValueNode(i)}); + }); + + inputs.push_back(func_graph->NewCNode(inputs2)); + } + return func_graph->NewCNode(inputs); +} + +AnfNodePtr HyperMap::FullMake(const std::shared_ptr& type, const FuncGraphPtr& func_graph, + const AnfNodePtr& fn_arg, const ArgsPairList& arg_map) { + MS_EXCEPTION_IF_NULL(type); + MS_EXCEPTION_IF_NULL(func_graph); + + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimMakeRecord)); + inputs.push_back(NewValueNode(type)); + + // cannot use shared_from_base() also known as this, as it will make a reference cycle on + // hypermap and graph generated, it will cause memory leak. + std::shared_ptr fn_rec = std::make_shared(*this); + std::size_t attrSize = type->GetAttributes().size(); + for (std::size_t i = 0; i < attrSize; ++i) { + std::vector inputs2; + inputs2.push_back(NewValueNode(fn_rec)); + if (fn_arg) { + inputs2.push_back(fn_arg); + } + + int j = 0; + for (auto item : arg_map) { + inputs2.push_back(func_graph->NewCNode({NewValueNode(prim::kPrimGetAttr), item.first, NewValueNode(j)})); + j++; + } + + inputs.push_back(func_graph->NewCNode(inputs2)); + } + return func_graph->NewCNode(inputs); +} + +AnfNodePtr HyperMap::Make(const FuncGraphPtr& func_graph, const AnfNodePtr& fn_arg, const ArgsPairList& arg_map) { + bool found = false; + TypeId id = kObjectTypeEnd; + std::pair pair; + for (auto& item : arg_map) { + pair = item; + id = item.second->type_id(); + if (nonleaf_.count(id)) { + found = true; + break; + } + } + + if (found) { + // In a nonleaf situation, all arguments must have the same generic. + bool is_not_same = std::any_of(arg_map.begin(), arg_map.end(), [pair](const std::pair& item) { + if (item.first != pair.first) { + return item.second->type_id() != pair.second->type_id(); + } + return false; + }); + if (is_not_same) { + std::ostringstream oss; + oss << "There are " << arg_map.size() << " inputs of `" << name_ << "`, corresponding type info:\n" + << trace::GetDebugInfo(func_graph->debug_info()) << "\n"; + int idx = 0; + for (auto& item : arg_map) { + oss << ++idx << ": " << item.second->ToString() << "\n"; + } + MS_LOG(EXCEPTION) << "HyperMap cannot match up all input types of arguments.\n" << oss.str(); + } + } + + switch (id) { + case kObjectTypeList: { + auto type = std::static_pointer_cast(pair.second); + return FullMake(type, func_graph, fn_arg, arg_map); + } + case kObjectTypeTuple: { + auto type = std::static_pointer_cast(pair.second); + return FullMake(type, func_graph, fn_arg, arg_map); + } + case kObjectTypeClass: { + auto type = std::static_pointer_cast(pair.second); + return FullMake(type, func_graph, fn_arg, arg_map); + } + default: + return FullMake(pair.second, func_graph, fn_arg, arg_map); + } +} + +ArgsPairList HyperMap::Harmonize(const FuncGraphPtr& func_graph, const ArgsPairList& args_spec_list) { + TypePtr type_tensor = std::make_shared(); + bool flag = std::any_of( + args_spec_list.begin(), args_spec_list.end(), + [type_tensor](const std::pair& item) { return IsSubType(item.second, type_tensor); }); + if (flag && broadcast_) { + ArgsPairList ret; + for (auto& item : args_spec_list) { + if (!IsSubType(item.second, type_tensor)) { + TypePtr type_tensor_ele = std::make_shared(item.second); + ret.push_back( + std::make_pair(func_graph->NewCNode({NewValueNode(prim::kPrimScalarToArray), item.first}), type_tensor_ele)); + } else { + ret.push_back(std::make_pair(item.first, item.second)); + } + } + return ret; + } + return args_spec_list; +} + +FuncGraphPtr HyperMap::GenerateFromTypes(const TypePtrList& args_spec_list) { + FuncGraphPtr ptrGraph = std::make_shared(); + ptrGraph->set_flags(FUNC_GRAPH_FLAG_CORE, true); + ptrGraph->debug_info()->set_name("hyper_map"); + + AnfNodePtr ptrFnArg = nullptr; + std::size_t i = 0; + ArgsPairList argmap; + ArgsPairList argmap2; + if (fn_leaf_ == nullptr) { + ptrFnArg = ptrGraph->add_parameter(); + i = 1; + } + + std::size_t size = args_spec_list.size(); + for (; i < size; ++i) { + argmap.push_back(std::make_pair(ptrGraph->add_parameter(), args_spec_list[i])); + } + + argmap2 = Harmonize(ptrGraph, argmap); + ptrGraph->set_output(Make(ptrGraph, ptrFnArg, argmap2)); + return ptrGraph; +} + +abstract::AbstractBasePtrList HyperMap::NormalizeArgs(const AbstractBasePtrList& args_spec_list) const { + if (fn_leaf_ == nullptr) { + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + // Assert that hypermap's function param does not contain free variables + if (args_spec_list[0]->isa()) { + auto graph_func = dyn_cast(args_spec_list[0]); + auto func_graph = graph_func->func_graph(); + if (func_graph->parent() != nullptr) { + MS_LOG(EXCEPTION) << "HyperMap don't support Closure with free variable yet."; + } + } + } + + AbstractBasePtrList broadened; + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(broadened), + [](const AbstractBasePtr& arg) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(arg); + return arg->Broaden(); + }); + return broadened; +} + +REGISTER_PYBIND_DEFINE(HyperMap_, ([](const py::module* m) { + (void)py::class_>(*m, "HyperMap_") + .def(py::init>(), py::arg("leaf")) + .def(py::init<>()); + })); + +FuncGraphPtr Tail::GenerateTupleFuncGraph(const abstract::AbstractTuplePtr& a_tuple) { + MS_EXCEPTION_IF_NULL(a_tuple); + + FuncGraphPtr ret = std::make_shared(); + ret->set_flags(FUNC_GRAPH_FLAG_CORE, true); + ret->debug_info()->set_name("tail"); + AnfNodePtr ptrTup = ret->add_parameter(); + + std::vector elems; + elems.push_back(NewValueNode(prim::kPrimMakeTuple)); + + int tuple_size = SizeToInt(a_tuple->size()); + for (int i = 1; i < tuple_size; ++i) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimTupleGetItem), ptrTup, NewValueNode(i)})); + } + + ret->set_output(ret->NewCNode(elems)); + return ret; +} + +FuncGraphPtr Tail::GenerateListFuncGraph(const abstract::AbstractListPtr& a_list) { + MS_EXCEPTION_IF_NULL(a_list); + + FuncGraphPtr ret = std::make_shared(); + ret->set_flags(FUNC_GRAPH_FLAG_CORE, true); + ret->debug_info()->set_name("tail"); + AnfNodePtr ptrList = ret->add_parameter(); + + std::vector elems; + elems.push_back(NewValueNode(prim::kPrimMakeList)); + + int list_size = SizeToInt(a_list->size()); + for (int i = 1; i < list_size; ++i) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimListGetItem), ptrList, NewValueNode(i)})); + } + + ret->set_output(ret->NewCNode(elems)); + return ret; +} + +FuncGraphPtr Tail::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + if (args_spec_list.size() != 1) { + MS_LOG(EXCEPTION) << "tail requires a non-empty tuple."; + } + + AbstractBasePtr a = args_spec_list[0]; + abstract::AbstractTuplePtr a_tuple = dyn_cast(a); + if (a_tuple != nullptr) { + return GenerateTupleFuncGraph(a_tuple); + } + + abstract::AbstractListPtr a_list = dyn_cast(a); + if (a_list != nullptr) { + return GenerateListFuncGraph(a_list); + } + + MS_LOG(EXCEPTION) << "arg0 must be AbstractTuple or AbstractList, but: " << a->ToString(); +} + +REGISTER_PYBIND_DEFINE( + Tail_, ([](const py::module* m) { + (void)py::class_>(*m, "Tail_").def(py::init()); + })); + +FuncGraphPtr MakeTupleGradient::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + int tuple_size = SizeToInt(args_spec_list.size()); + + std::ostringstream ss; + ss << "▶make_tuple_" << tuple_size; + FuncGraphPtr fg = std::make_shared(); + fg->debug_info()->set_name(ss.str()); + + std::vector params; + params.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (int i = 0; i < tuple_size; ++i) { + params.push_back(fg->add_parameter()); + } + + // make fprob first result, maketuple's forward result. + AnfNodePtr out = fg->NewCNode(params); + + // make fprob second result, maketuple's backward function. + FuncGraphPtr b = std::make_shared(); + + ss.clear(); + ss << "◀make_tuple_" << tuple_size; + b->debug_info()->set_name(ss.str()); + AnfNodePtr dout = b->add_parameter(); + + std::vector grads; + grads.push_back(NewValueNode(prim::kPrimMakeTuple)); + grads.push_back(NewValueNode(newenv)); + for (int i = 0; i < tuple_size; ++i) { + grads.push_back(b->NewCNode({NewValueNode(prim::kPrimTupleGetItem), dout, NewValueNode(i)})); + } + + b->set_flags(FUNC_GRAPH_FLAG_CORE, true); + b->set_output(b->NewCNode(grads)); + + fg->set_flags(FUNC_GRAPH_FLAG_CORE, true); + fg->set_output(fg->NewCNode({NewValueNode(prim::kPrimMakeTuple), out, NewValueNode(b)})); + (void)fg->transforms().emplace("primal", FuncGraphTransform(prim::kPrimMakeTuple)); + return fg; +} + +GradOperation::GradOperation(const std::string& name, bool get_all, bool get_by_list, bool sens_param) + : MetaFuncGraph(name), get_all_(get_all), get_by_list_(get_by_list), sens_param_(sens_param) { + if (get_by_list) { + signatures_ = + // def grad(func:read, weight_list:ref): + std::vector({{"func", SignatureEnumRW::kRWRead, SignatureEnumKind::kKindDefault}, + {"weight_list", SignatureEnumRW::kRWRef, SignatureEnumKind::kKindDefault}}); + } +} + +FuncGraphPtr GradOperation::GetGrad(AnfNodePtr node, const AnfNodePtr& weights, + const std::vector& params_list, bool applyJ) { + FuncGraphPtr ret = std::make_shared(); + ret->set_flags(FUNC_GRAPH_FLAG_CORE, true); + + ValueNodePtr opsJ = NewValueNode(prim::kPrimJ); + ValueNodePtr opsTupleItem = NewValueNode(prim::kPrimTupleGetItem); + + std::vector inputs; + if (applyJ) { + inputs.push_back(opsJ); + inputs.push_back(node); + node = ret->NewCNode(inputs); + } + + std::vector params; + for (size_t i = 0; i < params_list.size(); ++i) { + params.push_back(ret->add_parameter()); + } + + inputs.clear(); + inputs.push_back(node); + (void)std::copy(params.begin(), params.end(), std::back_inserter(inputs)); + AnfNodePtr cnode = ret->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(opsTupleItem); + inputs.push_back(cnode); + inputs.push_back(NewValueNode(0)); + auto out = ret->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(opsTupleItem); + inputs.push_back(cnode); + inputs.push_back(NewValueNode(1)); + AnfNodePtr ptrBprop = ret->NewCNode(inputs); + + doGetGrad(ret, out, ptrBprop, weights, opsTupleItem); + return ret; +} + +void GradOperation::doGetGrad(const FuncGraphPtr& func_graph, AnfNodePtr out, AnfNodePtr ptrBprop, AnfNodePtr weights, + ValueNodePtr opsTupleItem) { + MS_EXCEPTION_IF_NULL(func_graph); + + AnfNodePtr ptrBPropArg = nullptr; + if (sens_param_) { + ptrBPropArg = func_graph->add_parameter(); + } else { + auto ones_like = prim::GetPythonOps("ones_like"); + ptrBPropArg = func_graph->NewCNode({NewValueNode(ones_like), out}); + } + + AnfNodePtr ptrBApp = func_graph->NewCNode({ptrBprop, ptrBPropArg}); + + CNodePtr fv_bprop = nullptr; + if (get_by_list_) { + // python code: grads = hyper_map(F.partial(env_get, env), weights) + AnfNodePtr env = func_graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), ptrBApp, NewValueNode(0)}); + AnfNodePtr partial_env_get = + func_graph->NewCNode({NewValueNode(prim::kPrimPartial), NewValueNode(prim::GetPythonOps("env_get")), env}); + MetaFuncGraphPtr hyper_map = std::make_shared(); + fv_bprop = func_graph->NewCNode({NewValueNode(hyper_map), partial_env_get, weights}); + } + + CNodePtr inputs_bprop = nullptr; + if (get_all_) { + inputs_bprop = func_graph->NewCNode({NewValueNode(kTail), ptrBApp}); + } + + // Gradients wrt inputs and parameters + if (fv_bprop != nullptr && inputs_bprop != nullptr) { + func_graph->set_output(func_graph->NewCNode({NewValueNode(kPrimMakeTuple), inputs_bprop, fv_bprop})); + return; + } + + // Gradients wrt parameters + if (fv_bprop != nullptr) { + func_graph->set_output(fv_bprop); + return; + } + + // Gradients wrt inputs + if (inputs_bprop != nullptr) { + func_graph->set_output(inputs_bprop); + return; + } + + // Gradients wrt first input. + // ptrBApp returns (EnvInstance(grads wrt params), grads wrt input0, grads wrt input1, ...), so 1 is for first input + func_graph->set_output(func_graph->NewCNode({opsTupleItem, ptrBApp, NewValueNode(1)})); +} + +// Generate the graph. +FuncGraphPtr GradOperation::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + if (args_spec_list.size() < 1) { + MS_LOG(EXCEPTION) << "GenerateGraph requires at least 1 parameters, while the input size is " + << args_spec_list.size() << "."; + } + + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + AbstractFunctionPtr fn = dyn_cast(args_spec_list[0]); + if (fn == nullptr) { + MS_LOG(EXCEPTION) << "GradOperation arg0 must be AbstractFunction, but " << args_spec_list[0]->ToString(); + } + + // Waiting for implementation. + auto real_fn = dyn_cast(fn); + MS_EXCEPTION_IF_NULL(real_fn); + + FuncGraphPtr ptrGraph = real_fn->func_graph(); + MS_EXCEPTION_IF_NULL(ptrGraph); + TraceManager::DebugTrace(std::make_shared(ptrGraph->debug_info())); + FuncGraphPtr dfBuilder = std::make_shared(); + TraceManager::EndTrace(); + auto nparam = ptrGraph->parameters().size(); + + std::ostringstream ss; + ss << "grad{" << nparam << "}"; + dfBuilder->set_flags(FUNC_GRAPH_FLAG_CORE, true); + dfBuilder->debug_info()->set_name(ss.str()); + ParameterPtr param_graph = dfBuilder->add_parameter(); + + AnfNodePtr weights = nullptr; + if (get_by_list_) { + weights = dfBuilder->add_parameter(); + } + + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimJ)); + inputs.push_back(param_graph); + auto jf = dfBuilder->NewCNode(inputs); + // df is checked in GetGrad + TraceManager::DebugTrace(std::make_shared(ptrGraph->debug_info())); + auto df = GetGrad(jf, weights, ptrGraph->parameters()); + TraceManager::EndTrace(); + dfBuilder->set_output(NewValueNode(df)); + + return dfBuilder; +} + +REGISTER_PYBIND_DEFINE(GradOperation_, ([](const py::module* m) { + (void)py::class_>( + *m, "GradOperation_") + .def(py::init(), py::arg("fn")) + .def(py::init(), py::arg("fn"), py::arg("get_all"), + py::arg("get_by_list"), py::arg("sens_param")); + })); + +MultitypeFuncGraph::MultitypeFuncGraph(const std::string& name) : MetaFuncGraph(name) { + fn_cache_.clear(); + signatures_ = std::vector({// def multitype(*args:ref): + {"args", SignatureEnumRW::kRWRef, SignatureEnumKind::kKindVarPositional}}); +} + +void MultitypeFuncGraph::Register(const TypePtrList& types, specialize_fn s_fn) { + MS_LOG(DEBUG) << "Register type (" << ::mindspore::ToString(types) << "."; + auto fn = fn_cache_.find(types); + if (fn != fn_cache_.end()) { + MS_LOG(EXCEPTION) << "Cannot register as (" << ::mindspore::ToString(types) << ", already registered."; + } + fn_cache_[types] = s_fn; +} + +void MultitypeFuncGraph::Register(const TypePtrList& types, const py::function& py_fn) { + MS_LOG(DEBUG) << "Register type (" << ::mindspore::ToString(types) << ", " << std::string(py_fn.str()) << ")."; + auto fn = fn_cache_.find(types); + if (fn != fn_cache_.end()) { + MS_LOG(EXCEPTION) << "Cannot register as (" << ::mindspore::ToString(types) << ", already registered."; + } + fn_cache_py_[types] = py_fn; +} + +void MultitypeFuncGraph::Register(const std::vector& types_name, const py::function& py_fn) { + TypePtrList types; + for (auto& type_name : types_name) { + auto type_ptr = StringToType(type_name); + if (type_ptr == nullptr) { + MS_LOG(EXCEPTION) << "" << type_name << " convert from string error "; + } + types.push_back(type_ptr); + } + Register(types, py_fn); +} + +void MultitypeFuncGraph::PyRegister(const py::tuple& tuple, const py::function& py_fn) { + std::vector types_name; + for (size_t it = 0; it < tuple.size(); ++it) { + py::object name_py = tuple[it]; + if (py::isinstance(name_py)) { + types_name.push_back(name_py.cast()); + continue; + } + MS_LOG(EXCEPTION) << "Register must be string"; + } + Register(types_name, py_fn); +} +static TypePtr UnwrapRef(const TypePtr& type) { + if (type->isa()) { + return type->cast()->subtype(); + } + return type; +} +FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList& types) { + bool find_fn = false; + py::function py_fn; + for (auto& item : fn_cache_py_) { + TypePtrList sign = item.first; + if (sign.size() != types.size()) { + continue; + } + bool match = true; + for (size_t i = 0; i < sign.size(); ++i) { + if (!IsIdentidityOrSubclass(UnwrapRef(types[i]), sign[i])) { + match = false; + break; + } + } + if (!match) { + continue; + } + find_fn = true; + py_fn = item.second; + break; + } + std::ostringstream buffer; + buffer << types; + if (find_fn) { + FuncGraphPtr func_graph = parse::ParsePythonCode(py_fn); + if (func_graph == nullptr) { + MS_LOG(EXCEPTION) << "Fail to parse overload function " << buffer.str(); + } + MS_LOG(DEBUG) << "Find overload function " << buffer.str() << ", function: " << func_graph->ToString(); + return func_graph; + } + std::ostringstream oss; + oss << "There are " << fn_cache_py_.size() << " prototypes for overload function `" << name_ + << "`, corresponding location info:\n"; + int idx = 0; + for (auto& item : fn_cache_py_) { + FuncGraphPtr func_graph = parse::ParsePythonCode(item.second); + if (func_graph == nullptr) { + MS_LOG(WARNING) << "Fail to parse Python code for function `" << name_ << "`."; + continue; + } + oss << ++idx << ". " << item.first << "\n " << trace::GetDebugInfo(func_graph->debug_info()) << "\n"; + } + MS_LOG(EXCEPTION) << "Fail to find overload function for `" << name_ << "` with type " << buffer.str() << "\n" + << oss.str(); +} + +REGISTER_PYBIND_DEFINE(MultitypeFuncGraph_, ([](const py::module* m) { + (void)py::class_>( + *m, "MultitypeFuncGraph_") + .def(py::init()) + .def("register_fn", &MultitypeFuncGraph::PyRegister); + })); + +// Generate the ListMap func graph. +FuncGraphPtr ListMap::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + size_t args_num = args_spec_list.size(); + // args: fn, list1, list2, ... + if (args_num < 2) { + MS_LOG(EXCEPTION) << "list_map takes at least two arguments"; + } + + for (size_t i = 1; i < args_num; ++i) { + if (typeid(args_spec_list[i]) != typeid(AbstractBase)) { + // The function currently not be use + MS_LOG(EXCEPTION) << "list_map requires lists, not {t}'"; + } + } + + FuncGraphPtr fg_ptr = std::make_shared(); + fg_ptr->set_flags(FUNC_GRAPH_FLAG_CORE, true); + fg_ptr->debug_info()->set_name("list_map"); + AnfNodePtr fn = fg_ptr->add_parameter(); + + std::vector lists; + for (size_t i = 1; i < args_num; ++i) { + lists.push_back(fg_ptr->add_parameter()); + } + + std::vector iters; + (void)std::transform(lists.begin(), lists.end(), std::back_inserter(iters), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(std::string("list_iter")), item}); + }); + + std::vector nexts; + (void)std::transform(iters.begin(), iters.end(), std::back_inserter(nexts), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(std::string("next")), item}); + }); + + std::vector values; + (void)std::transform(nexts.begin(), nexts.end(), std::back_inserter(values), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(prim::kPrimTupleGetItem), item}); + }); + + (void)std::transform(nexts.begin(), nexts.end(), std::back_inserter(iters), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(prim::kPrimTupleGetItem), item, NewValueNode(1)}); + }); + + (void)values.insert(values.begin(), fn); + AnfNodePtr cnode_graph = fg_ptr->NewCNode(values); + AnfNodePtr resl = fg_ptr->NewCNode({NewValueNode(prim::kPrimMakeList), cnode_graph}); + + FuncGraphPtr fgnext_ptr = std::make_shared(); + fgnext_ptr->debug_info()->set_name("body"); + + FuncGraphPtr fgcond_ptr = std::make_shared(); + fgcond_ptr->debug_info()->set_name("cond"); + + MakeCond(lists, fgnext_ptr, fgcond_ptr); + MakeNext(lists, fgcond_ptr, fgnext_ptr); + + CNodePtr output_cnode = fg_ptr->NewCNode({NewValueNode(fgcond_ptr), fn, resl}); + + auto inputs = output_cnode->inputs(); + (void)inputs.insert(inputs.end(), iters.begin(), iters.end()); + output_cnode->set_inputs(inputs); + + fg_ptr->set_output(output_cnode); + return fg_ptr; +} + +void ListMap::MakeCond(const std::vector& lists, const FuncGraphPtr& fgnext_ptr, + const FuncGraphPtr& fg_ptr) { + MS_EXCEPTION_IF_NULL(fg_ptr); + + AnfNodePtr fn = fg_ptr->add_parameter(); + AnfNodePtr resl = fg_ptr->add_parameter(); + + std::vector iters; + (void)std::transform(lists.begin(), lists.end(), std::back_inserter(iters), + [fg_ptr](AnfNodePtr) { return fg_ptr->add_parameter(); }); + + std::vector hasnexts; + (void)std::transform(iters.begin(), iters.end(), std::back_inserter(hasnexts), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(std::string("hasnext")), item}); + }); + + // cond = reduce(lambda a, b: g.apply(P.bool_and, a, b), hasnexts) + FuncGraphPtr fgtrue_ptr = std::make_shared(); + fgtrue_ptr->debug_info()->set_name("ftrue"); + fgtrue_ptr->set_flags(FUNC_GRAPH_FLAG_CORE, true); + + CNodePtr fgtrue_output_cnode = fgtrue_ptr->NewCNode({NewValueNode(fgnext_ptr), fn, resl}); + auto inputs = fgtrue_output_cnode->inputs(); + (void)inputs.insert(inputs.end(), iters.begin(), iters.end()); + fgtrue_output_cnode->set_inputs(inputs); + fgtrue_ptr->set_output(fgtrue_output_cnode); + + FuncGraphPtr fgfalse_ptr = std::make_shared(); + fgfalse_ptr->debug_info()->set_name("ffalse"); + fgfalse_ptr->set_flags(FUNC_GRAPH_FLAG_CORE, true); + fgfalse_ptr->set_output(resl); + + AnfNodePtr output_cnode = fg_ptr->NewCNode({NewValueNode(prim::kPrimSwitch), NewValueNode(std::string("cond")), + NewValueNode(fgtrue_ptr), NewValueNode(fgfalse_ptr)}); + fgtrue_ptr->set_output(output_cnode); +} + +void ListMap::MakeNext(const std::vector& lists, const FuncGraphPtr& fgcond_ptr, + const FuncGraphPtr& fg_ptr) { + MS_EXCEPTION_IF_NULL(fg_ptr); + AnfNodePtr fn = fg_ptr->add_parameter(); + + std::vector iters; + (void)std::transform(lists.begin(), lists.end(), std::back_inserter(iters), + [fg_ptr](AnfNodePtr) { return fg_ptr->add_parameter(); }); + + std::vector nexts; + (void)std::transform(iters.begin(), iters.end(), std::back_inserter(nexts), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(std::string("next")), item}); + }); + + std::vector values; + (void)std::transform(nexts.begin(), nexts.end(), std::back_inserter(values), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(prim::kPrimTupleGetItem), item, nullptr}); + }); + + iters.clear(); + (void)std::transform(nexts.begin(), nexts.end(), std::back_inserter(iters), [fg_ptr](AnfNodePtr item) { + return fg_ptr->NewCNode({NewValueNode(prim::kPrimTupleGetItem), item, NewValueNode(1)}); + }); + + (void)values.insert(values.begin(), fn); + AnfNodePtr cnode_graph = fg_ptr->NewCNode(values); + AnfNodePtr resl = fg_ptr->NewCNode({NewValueNode(prim::kPrimListAppend), cnode_graph}); + CNodePtr output_cnode = fg_ptr->NewCNode({NewValueNode(fgcond_ptr), fn, resl}); + + auto inputs = output_cnode->inputs(); + (void)inputs.insert(inputs.end(), iters.begin(), iters.end()); + output_cnode->set_inputs(inputs); + fg_ptr->set_output(output_cnode); +} + +FuncGraphPtr TupleAdd::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + // args: tuple1, tuple2 + abstract::CheckArgsSize("TupleAdd", args_spec_list, 2); + AbstractBasePtr abs_a = args_spec_list[0]; + AbstractBasePtr abs_b = args_spec_list[1]; + + abstract::AbstractTuplePtr a_tuple = dyn_cast(abs_a); + abstract::AbstractTuplePtr b_tuple = dyn_cast(abs_b); + if (a_tuple == nullptr || b_tuple == nullptr) { + MS_LOG(EXCEPTION) << "TupleAdd argument should be tuple,but " << args_spec_list[0]->ToString() << ", " + << args_spec_list[1]->ToString(); + } + + FuncGraphPtr ret = std::make_shared(); + ret->set_flags(FUNC_GRAPH_FLAG_CORE, true); + AnfNodePtr p_tup_a = ret->add_parameter(); + AnfNodePtr p_tup_b = ret->add_parameter(); + + std::vector elems; + elems.push_back(NewValueNode(prim::kPrimMakeTuple)); + + int tuple_size = SizeToInt(a_tuple->size()); + for (int i = 0; i < tuple_size; ++i) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimTupleGetItem), p_tup_a, NewValueNode(i)})); + } + + tuple_size = SizeToInt(b_tuple->size()); + for (int i = 0; i < tuple_size; ++i) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimTupleGetItem), p_tup_b, NewValueNode(i)})); + } + + ret->set_output(ret->NewCNode(elems)); + return ret; +} + +int GetArgScalarValue(const abstract::AbstractScalarPtr& scalar, const std::string&) { + MS_EXCEPTION_IF_NULL(scalar); + return GetValue(scalar->BuildValue()); +} + +bool CheckIndexInRange(int index, int min, int max) { return (index >= min && index <= max); } + +int GetPositiveIndex(int index, int length) { + if (index < 0) { + index += length; + } + return index; +} + +int CheckSliceMember(const AbstractBasePtr& member, int default_value, const std::string& member_name) { + MS_EXCEPTION_IF_NULL(member); + + if (member->isa()) { + return GetArgScalarValue(dyn_cast(member), member_name); + } + + if (member->isa()) { + return default_value; + } + + MS_LOG(EXCEPTION) << "" << member_name << " should be a AbstractScalar or AbstractNone, but got " + << member->ToString(); +} + +void GenerateTupleSliceParameter(const AbstractTuplePtr& tuple, const AbstractSlicePtr& slice, int* start_index, + int* stop_index, int* step_value) { + MS_EXCEPTION_IF_NULL(tuple); + MS_EXCEPTION_IF_NULL(slice); + MS_EXCEPTION_IF_NULL(start_index); + MS_EXCEPTION_IF_NULL(stop_index); + MS_EXCEPTION_IF_NULL(step_value); + + const std::string start_name("Slice start index"); + const std::string stop_name("Slice stop index"); + const std::string step_name("Slice step value"); + + int tuple_size = SizeToInt(tuple->size()); + int start_default = 0; + int stop_default = tuple_size; + int step_default = 1; + + *step_value = CheckSliceMember(slice->step(), step_default, step_name); + if (*step_value == 0) { + MS_LOG(EXCEPTION) << "TupleSlice require the step value could not be 0, but got 0."; + } + + if (*step_value < 0) { + start_default = tuple_size - 1; + stop_default = -1; + } + + *start_index = CheckSliceMember(slice->start(), start_default, start_name); + *stop_index = CheckSliceMember(slice->stop(), stop_default, stop_name); + if (!CheckIndexInRange(*start_index, -tuple_size, tuple_size - 1) || + !CheckIndexInRange(*stop_index, -tuple_size - 1, tuple_size)) { + MS_LOG(EXCEPTION) << "TupleSlice the start index " << *start_index << " or end end index " << *stop_index + << " out of range, tuple size " << tuple_size << "."; + } + + *start_index = GetPositiveIndex(*start_index, tuple_size); + if (!slice->stop()->isa()) { + *stop_index = GetPositiveIndex(*stop_index, tuple_size); + } +} + +FuncGraphPtr TupleSlice::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + // slice a tuple + // args: tuple, start index, end index, step + const std::string op_name("TupleSlice"); + abstract::CheckArgsSize(op_name, args_spec_list, 2); + AbstractTuplePtr tuple = abstract::CheckArg(op_name, args_spec_list, 0); + AbstractSlicePtr slice = abstract::CheckArg(op_name, args_spec_list, 1); + + int start_index; + int stop_index; + int step_value; + GenerateTupleSliceParameter(tuple, slice, &start_index, &stop_index, &step_value); + + FuncGraphPtr ret = std::make_shared(); + ret->set_flags(FUNC_GRAPH_FLAG_CORE, true); + AnfNodePtr p_tuple = ret->add_parameter(); + (void)ret->add_parameter(); + + std::vector elems; + elems.push_back(NewValueNode(prim::kPrimMakeTuple)); + if (step_value > 0) { + for (int index = start_index; index < stop_index; index = index + step_value) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimTupleGetItem), p_tuple, NewValueNode(index)})); + } + } else { + for (int index = start_index; index > stop_index; index = index + step_value) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimTupleGetItem), p_tuple, NewValueNode(index)})); + } + } + + ret->set_output(ret->NewCNode(elems)); + return ret; +} + +int ConvertBinaryToDecimal(const std::vector& number_bin) { + unsigned int number_dec = 0; + for (size_t index = 0; index < number_bin.size(); index++) { + number_dec |= number_bin[index] << index; + } + return static_cast(number_dec); +} + +void ParseSlice(const AbstractSlicePtr& slice, std::vector* begin, std::vector* end, + std::vector* strides, int length) { + MS_EXCEPTION_IF_NULL(slice); + MS_EXCEPTION_IF_NULL(begin); + MS_EXCEPTION_IF_NULL(end); + MS_EXCEPTION_IF_NULL(strides); + if (length <= 0) { + MS_LOG(EXCEPTION) << "Could not slice a dim when it's length less than 1"; + } + + int start_default = 0; + int stop_default = length; + int step_default = 1; + int step_value = CheckSliceMember(slice->step(), step_default, "step"); + if (step_value < 0) { + start_default = -1; + stop_default = -(length + 1); + } + + begin->push_back(CheckSliceMember(slice->start(), start_default, "begin")); + end->push_back(CheckSliceMember(slice->stop(), stop_default, "stop")); + strides->push_back(step_value); +} + +int GenerateStridedSliceParametersFromTuple(const AbstractTuplePtr& slice_tuple, const std::vector& shape, + std::vector* begin, std::vector* end, std::vector* strides) { + MS_EXCEPTION_IF_NULL(slice_tuple); + MS_EXCEPTION_IF_NULL(begin); + MS_EXCEPTION_IF_NULL(end); + MS_EXCEPTION_IF_NULL(strides); + + size_t slice_tuple_size = slice_tuple->size(); + size_t shape_size = shape.size(); + if (slice_tuple_size > shape_size) { + MS_LOG(EXCEPTION) << "The number of slice data to slice tensor should be less than the rank of tensor," + "when the rank of tensor is " + << shape_size << ", the number of slice is " << slice_tuple_size; + } + + std::vector shrink; + auto slice_tuple_eles = slice_tuple->elements(); + for (size_t index = 0; index < slice_tuple_size; index++) { + if (slice_tuple_eles[index]->isa()) { + AbstractSlicePtr slice = dyn_cast(slice_tuple_eles[index]); + ParseSlice(slice, begin, end, strides, shape[index]); + shrink.push_back(0); + continue; + } + + if (slice_tuple_eles[index]->isa()) { + int ele_index = GetArgScalarValue(dyn_cast(slice_tuple_eles[index]), "slice_tuple"); + begin->push_back(ele_index); + end->push_back(ele_index + 1); + strides->push_back(1); + shrink.push_back(1); + continue; + } + + MS_LOG(EXCEPTION) << "Slice tuple only could contain slice or int number, but got " + << slice_tuple_eles[index]->ToString(); + } + + for (size_t index = slice_tuple_size; index < shape_size; index++) { + begin->push_back(0); + end->push_back(shape[index]); + strides->push_back(1); + } + + return ConvertBinaryToDecimal(shrink); +} + +int GenerateStridedSliceParametersFromSlice(const AbstractSlicePtr& slice, const std::vector& shape, + std::vector* begin, std::vector* end, std::vector* strides) { + MS_EXCEPTION_IF_NULL(begin); + MS_EXCEPTION_IF_NULL(end); + MS_EXCEPTION_IF_NULL(strides); + size_t shape_size = shape.size(); + if (shape_size == 0) { + MS_LOG(EXCEPTION) << "Could slice a scalar tensor"; + } + + ParseSlice(slice, begin, end, strides, shape[0]); + + for (size_t index = 1; index < shape_size; index++) { + begin->push_back(0); + end->push_back(shape[index]); + strides->push_back(1); + } + + return 0; +} + +int GenerateStridedSliceParametersFromNumber(const AbstractScalarPtr& scalar, const std::vector& shape, + std::vector* begin, std::vector* end, + std::vector* strides) { + MS_EXCEPTION_IF_NULL(begin); + MS_EXCEPTION_IF_NULL(end); + MS_EXCEPTION_IF_NULL(strides); + int ele_index = GetArgScalarValue(scalar, "slice_tuple"); + + begin->push_back(ele_index); + end->push_back(ele_index + 1); + strides->push_back(1); + + for (size_t index = 1; index < shape.size(); index++) { + begin->push_back(0); + end->push_back(shape[index]); + strides->push_back(1); + } + + return 1; +} + +FuncGraphPtr TensorSlice::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + // slice a tensor + // args: tensor, slice or slice tuple + const std::string op_name = std::string("TensorSlice"); + abstract::CheckArgsSize(op_name, args_spec_list, 2); + AbstractTensorPtr tensorPtr = abstract::CheckArg(op_name, args_spec_list, 0); + + auto shape = tensorPtr->shape()->shape(); + std::vector begin; + std::vector end; + std::vector strides; + int shrink_axis_mask; + + if (args_spec_list[1]->isa()) { + AbstractTuplePtr tuple_ptr = dyn_cast(args_spec_list[1]); + shrink_axis_mask = GenerateStridedSliceParametersFromTuple(tuple_ptr, shape, &begin, &end, &strides); + } else if (args_spec_list[1]->isa()) { + AbstractSlicePtr slice_ptr = dyn_cast(args_spec_list[1]); + shrink_axis_mask = GenerateStridedSliceParametersFromSlice(slice_ptr, shape, &begin, &end, &strides); + } else if (args_spec_list[1]->isa()) { + AbstractScalarPtr scalar_ptr = dyn_cast(args_spec_list[1]); + shrink_axis_mask = GenerateStridedSliceParametersFromNumber(scalar_ptr, shape, &begin, &end, &strides); + } else { + std::ostringstream args_info; + for (const auto& arg : args_spec_list) { + MS_EXCEPTION_IF_NULL(arg); + args_info << arg->ToString() << "\n"; + } + MS_LOG(EXCEPTION) << "TensorSlice requires to input a tensor and a slice or slice tuple, but got " + << args_info.str(); + } + + FuncGraphPtr ret_graph = std::make_shared(); + ret_graph->set_flags(FUNC_GRAPH_FLAG_CORE, true); + + AnfNodePtr tensor_node = ret_graph->add_parameter(); + (void)ret_graph->add_parameter(); + + auto PrimStridedSliceClass = prim::GetPythonOps("StridedSlice", "mindspore.ops.operations"); + auto PrimStridedSlice = ret_graph->NewCNode({NewValueNode(PrimStridedSliceClass), NewValueNode(0), NewValueNode(0), + NewValueNode(0), NewValueNode(0), NewValueNode(shrink_axis_mask)}); + ret_graph->set_output(ret_graph->NewCNode( + {PrimStridedSlice, tensor_node, NewValueNode(begin), NewValueNode(end), NewValueNode(strides)})); + return ret_graph; +} + +FuncGraphPtr UnpackCall::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + // slice a tensor + // args: tensor, slice or slice tuple + const std::string op_name = std::string("UnpackCall"); + size_t arg_length = args_spec_list.size(); + if (arg_length < 2) { + MS_LOG(EXCEPTION) << "" << op_name << " requires at least two args, but got " << arg_length << "."; + } + + (void)abstract::CheckArg(op_name, args_spec_list, 0); + FuncGraphPtr ret_graph = std::make_shared(); + ret_graph->set_flags(FUNC_GRAPH_FLAG_CORE, true); + + AnfNodePtr fnNode = ret_graph->add_parameter(); + std::vector elems; + elems.push_back(fnNode); + for (size_t index = 1; index < arg_length; index++) { + MS_EXCEPTION_IF_NULL(args_spec_list[index]); + if (args_spec_list[index]->isa()) { + AbstractTuplePtr arg_tuple = dyn_cast(args_spec_list[index]); + AnfNodePtr para_tuple = ret_graph->add_parameter(); + for (size_t i = 0; i < arg_tuple->size(); ++i) { + elems.push_back( + ret_graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), para_tuple, NewValueNode(SizeToInt(i))})); + } + } else if (args_spec_list[index]->isa()) { + AbstractDictionaryPtr arg_dict = dyn_cast(args_spec_list[index]); + AnfNodePtr para_dict = ret_graph->add_parameter(); + auto dict_elems = arg_dict->elements(); + (void)std::transform( + dict_elems.begin(), dict_elems.end(), std::back_inserter(elems), + [ret_graph, para_dict](const AbstractAttribute& item) { + return ret_graph->NewCNode( + {NewValueNode(prim::kPrimMakeKeywordArg), NewValueNode(item.first), + ret_graph->NewCNode({NewValueNode(prim::kPrimDictGetItem), para_dict, NewValueNode(item.first)})}); + }); + } else { + MS_LOG(EXCEPTION) << "" << op_name << " require args should be tuple or dict, but got " + << args_spec_list[index]->ToString(); + } + } + ret_graph->set_output(ret_graph->NewCNode(elems)); + return ret_graph; +} + +REGISTER_PYBIND_DEFINE( + TupleAdd_, ([](const py::module* m) { + (void)py::class_>(*m, "TupleAdd_").def(py::init()); + })); + +REGISTER_PYBIND_DEFINE(TupleSlice_, ([](const py::module* m) { + (void)py::class_>(*m, "TupleSlice_") + .def(py::init()); + })); + +REGISTER_PYBIND_DEFINE(TensorSlice_, ([](const py::module* m) { + (void)py::class_>(*m, "TensorSlice_") + .def(py::init()); + })); + +REGISTER_PYBIND_DEFINE(UnpackCall_, ([](const py::module* m) { + (void)py::class_>(*m, "UnpackCall_") + .def(py::init()); + })); +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/composite/composite.h b/mindspore/ccsrc/operator/composite/composite.h new file mode 100644 index 00000000000..176efb0425b --- /dev/null +++ b/mindspore/ccsrc/operator/composite/composite.h @@ -0,0 +1,225 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_COMPOSITE_H_ +#define MINDSPORE_CCSRC_OPERATOR_COMPOSITE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "operator/composite/zip_operation.h" +#include "operator/composite/list_append_operation.h" +#include "operator/composite/do_signature.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "utils/misc.h" +#include "utils/any.h" +#include "ir/dtype.h" +#include "ir/meta_func_graph.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +using AbstractSlicePtr = abstract::AbstractSlicePtr; +using AbstractScalarPtr = abstract::AbstractScalarPtr; +using AbstractTensorPtr = abstract::AbstractTensorPtr; +using ElemwiseMap = std::unordered_map; +using ArgsPairList = std::vector>; + +class MultitypeFuncGraph : public MetaFuncGraph { + public: + explicit MultitypeFuncGraph(const std::string& name); + ~MultitypeFuncGraph() override = default; + MS_DECLARE_PARENT(MultitypeFuncGraph, MetaFuncGraph) + + using specialize_fn = FuncGraph* (*)(TypePtrList); + // Register a method which specialize based on types vectors; + virtual void Register(const TypePtrList& types, specialize_fn s_fn); + virtual void Register(const TypePtrList& types, const py::function& py_fn); + virtual void Register(const std::vector& types_name, const py::function& py_fn); + virtual void PyRegister(const py::tuple& tuple, const py::function& py_fn); + + FuncGraphPtr GenerateFromTypes(const TypePtrList& types) override; + size_t GetPyFnCacheSize() const { return fn_cache_py_.size(); } + const std::unordered_map& GetPyFunctions() const { + return fn_cache_py_; + } + + private: + std::unordered_map fn_cache_; + std::unordered_map fn_cache_py_; +}; +using MultitypeFuncGraphPtr = std::shared_ptr; + +class HyperMap : public MetaFuncGraph { + public: + explicit HyperMap(const std::shared_ptr& fn_leaf = nullptr); + HyperMap(const HyperMap& h); + void Init(); + HyperMap& operator=(const HyperMap& h) { + if (this != &h) { + fn_leaf_ = h.fn_leaf_; + broadcast_ = h.broadcast_; + nonleaf_ = h.nonleaf_; + if (fn_leaf_) { + name_ = "hyper_map[" + fn_leaf_->name() + "]"; + } + } + return *this; + } + ~HyperMap() override = default; + MS_DECLARE_PARENT(HyperMap, MetaFuncGraph) + + abstract::AbstractBasePtrList NormalizeArgs(const abstract::AbstractBasePtrList& args_spec_list) const override; + FuncGraphPtr GenerateFromTypes(const TypePtrList& args_spec_list) override; + MetaFuncGraphPtr GetFnLeaf() { return fn_leaf_; } + + private: + AnfNodePtr FullMake(TypePtr type, const FuncGraphPtr& func_graph, const AnfNodePtr& fn_arg, + const ArgsPairList& arg_map); + AnfNodePtr FullMake(const std::shared_ptr& type, const FuncGraphPtr& func_graph, const AnfNodePtr& fn_arg, + const ArgsPairList& arg_map); + AnfNodePtr FullMake(const std::shared_ptr& type, const FuncGraphPtr& func_graph, const AnfNodePtr& fn_arg, + const ArgsPairList& arg_map); + AnfNodePtr FullMake(const std::shared_ptr& type, const FuncGraphPtr& func_graph, const AnfNodePtr& fn_arg, + const ArgsPairList& arg_map); + AnfNodePtr Make(const FuncGraphPtr& graph, const AnfNodePtr& fn_arg, const ArgsPairList& arg_map); + ArgsPairList Harmonize(const FuncGraphPtr& graph, const ArgsPairList& args_spec_list); + + MultitypeFuncGraphPtr fn_leaf_; + bool broadcast_; + std::set nonleaf_; +}; +using HyperMapPtr = std::shared_ptr; + +class HyperMapPy : public HyperMap { + public: + explicit HyperMapPy(const std::shared_ptr& fn_leaf = nullptr) : HyperMap(fn_leaf) {} + ~HyperMapPy() override = default; + MS_DECLARE_PARENT(HyperMapPy, HyperMap) +}; +using HyperMapPyPtr = std::shared_ptr; + +extern ValuePtr kCompositeHyperMap; + +class Tail : public MetaFuncGraph { + public: + explicit Tail(const std::string& name) : MetaFuncGraph(name) {} + ~Tail() override = default; + MS_DECLARE_PARENT(Tail, MetaFuncGraph) + + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + FuncGraphPtr GenerateTupleFuncGraph(const abstract::AbstractTuplePtr& a_tuple); + FuncGraphPtr GenerateListFuncGraph(const abstract::AbstractListPtr& a_list); + + friend bool operator==(const Tail& lhs, const Tail& rhs) { return lhs.name_ == rhs.name_; } +}; +using TailPtr = std::shared_ptr; + +class MakeTupleGradient : public MetaFuncGraph { + public: + explicit MakeTupleGradient(const std::string& name) : MetaFuncGraph(name) {} + ~MakeTupleGradient() override = default; + MS_DECLARE_PARENT(MakeTupleGradient, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + friend bool operator==(const MakeTupleGradient& lhs, const MakeTupleGradient& rhs) { return lhs.name_ == rhs.name_; } +}; +using MakeTupleGradientPtr = std::shared_ptr; + +class GradOperation : public MetaFuncGraph { + public: + explicit GradOperation(const std::string& name, bool get_all = false, bool get_by_list = false, + bool sens_param = false); + ~GradOperation() override = default; + MS_DECLARE_PARENT(GradOperation, MetaFuncGraph) + + FuncGraphPtr GetGrad(AnfNodePtr ptrNode, const AnfNodePtr& weights, const std::vector& ptrParams, + bool applyJ = false); + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + + bool get_all_; + bool get_by_list_; + bool sens_param_; + + private: + void doGetGrad(const FuncGraphPtr& func_graph, AnfNodePtr ptrOut, AnfNodePtr ptrBprop, AnfNodePtr weights, + ValueNodePtr opsTupleItem); +}; +using GradOperationPtr = std::shared_ptr; + +class ListMap { + public: + explicit ListMap(const std::string& name) : name_(name) { cache_.clear(); } + ~ListMap() = default; + void MakeCond(const std::vector& lists, const FuncGraphPtr& gnext_ptr, const FuncGraphPtr& graph_ptr); + void MakeNext(const std::vector& lists, const FuncGraphPtr& gcond_ptr, const FuncGraphPtr& graph_ptr); + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list); + + private: + std::string name_; + std::map, FuncGraphPtr> cache_; +}; + +class TupleAdd : public MetaFuncGraph { + public: + explicit TupleAdd(const std::string& name) : MetaFuncGraph(name) {} + ~TupleAdd() override = default; + MS_DECLARE_PARENT(TupleAdd, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + friend bool operator==(const TupleAdd& lhs, const TupleAdd& rhs) { return lhs.name_ == rhs.name_; } +}; +using TupleAddPtr = std::shared_ptr; + +class TupleSlice : public MetaFuncGraph { + public: + explicit TupleSlice(const std::string& name) : MetaFuncGraph(name) {} + ~TupleSlice() override = default; + MS_DECLARE_PARENT(TupleSlice, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + friend bool operator==(const TupleSlice& lhs, const TupleSlice& rhs) { return lhs.name_ == rhs.name_; } +}; +using TupleSlicePtr = std::shared_ptr; + +class TensorSlice : public MetaFuncGraph { + public: + explicit TensorSlice(const std::string& name) : MetaFuncGraph(name) {} + ~TensorSlice() override = default; + MS_DECLARE_PARENT(TensorSlice, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + friend bool operator==(const TensorSlice& lhs, const TensorSlice& rhs) { return lhs.name_ == rhs.name_; } +}; +using TensorSlicePtr = std::shared_ptr; + +// Expand the tuple and dict parameters generated when parsing the function call, +// and generate positional parameters and key-value pairs for function. +class UnpackCall : public MetaFuncGraph { + public: + explicit UnpackCall(const std::string& name) : MetaFuncGraph(name) {} + ~UnpackCall() override = default; + MS_DECLARE_PARENT(UnpackCall, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + friend bool operator==(const UnpackCall& lhs, const UnpackCall& rhs) { return lhs.name_ == rhs.name_; } +}; +using UnpackCallPtr = std::shared_ptr; +} // namespace prim +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPERATOR_COMPOSITE_H_ diff --git a/mindspore/ccsrc/operator/composite/do_signature.cc b/mindspore/ccsrc/operator/composite/do_signature.cc new file mode 100644 index 00000000000..62de1c71f22 --- /dev/null +++ b/mindspore/ccsrc/operator/composite/do_signature.cc @@ -0,0 +1,210 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/composite/do_signature.h" +#include +#include + +#include "pipeline/static_analysis/abstract_value.h" +#include "ir/anf.h" +#include "pipeline/static_analysis/dshape.h" +#include "pipeline/static_analysis/param_validator.h" +#include "operator/cc_implementations.h" +#include "optimizer/opt.h" +#include "utils/symbolic.h" +#include "./common.h" +#include "pybind_api/api_register.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +namespace { +using PatternListType = std::initializer_list; + +const std::vector& GetSignature(const ValuePtr& function) { + static const auto empty = std::vector(); + if (function->isa()) { + return function->cast()->signatures(); + } else if (function->isa()) { + return function->cast()->signatures(); + } + return empty; +} + +void ProcessDefault(const std::string& func_name, const AbstractBasePtrList& args_spec_list, + const std::vector& signature, bool has_var, std::vector* op_inputs) { + std::size_t sig_size = signature.size(); + auto positional_size = sig_size; + if (has_var) { + positional_size = sig_size - 1; + } + if (args_spec_list.size() < positional_size) { + for (size_t i = args_spec_list.size(); i < sig_size; ++i) { + auto default_value = signature[i].default_value; + if (default_value == nullptr) { + MS_LOG(EXCEPTION) << "Function " << func_name << "'s input length is not equal to Signature length."; + } else { + (*op_inputs).push_back(NewValueNode(default_value)); + } + } + } +} + +// Get the largest type of index in the same SignatureEnumDType of arguments. +std::map GetMaxDtypeIndex(const std::vector& dtypes, + const abstract::AbstractBasePtrList& args_spec_list) { + // record index for signature.dtypes of the same type + // eg. [T, T1, T, T2, T, T1, T3] -> {{T:(0,2,4)}, {T1:(1,5)}, {T2:(3)}, {T3:(6)}} + std::map> type_indexs; + for (size_t i = 0; i < dtypes.size(); ++i) { + auto it = type_indexs.find(dtypes[i]); + if (it == type_indexs.end()) { + (void)type_indexs.insert(std::make_pair(dtypes[i], std::vector{i})); + } else { + it->second.push_back(i); + } + } + // example:sig_dtype:[T, T1, T, T2, T, T1, T3, T4, T4] + // and args type: [int, Tensor, Tensor, float, Tensor, int, Tensor, int, float] + // result:{{T:2},{T1:1}} + std::map dst_type; + for (auto it = type_indexs.begin(); it != type_indexs.end(); (void)++it) { + auto type = it->first; + auto indexs = it->second; + // If the number of arguments belonging to the same SignatureEnumDType is less than 2, skip it. + if (indexs.size() < 2) { + continue; + } + size_t m_index = indexs[0]; + for (size_t i = 1; i < indexs.size(); ++i) { + if (args_spec_list[indexs[i]]->isa()) { + m_index = indexs[i]; + } + } + if (args_spec_list[m_index]->isa()) { + (void)dst_type.insert(std::make_pair(type, m_index)); + } + } + return dst_type; +} + +AnfNodePtr DoCast(const AnfNodePtr& param, const AnfNodePtr& source_param, const FuncGraphPtr& graph) { + // op and module import path + auto prim_dtype = prim::GetPythonOps("dtype", "mindspore.ops.functional"); + MS_EXCEPTION_IF_NULL(prim_dtype); + // op and module import path + auto prim_cast_class = prim::GetPythonOps("Cast", "mindspore.ops.operations"); + MS_EXCEPTION_IF_NULL(prim_cast_class); + auto dtype_node = NewCNode({NewValueNode(prim_dtype), source_param}, graph); + auto cast_node = NewCNode({NewValueNode(prim_cast_class)}, graph); + return NewCNode({cast_node, param, dtype_node}, graph); +} + +void DoAutoCast(const std::vector& signature, const abstract::AbstractBasePtrList& args_spec_list, + const FuncGraphPtr& graph, std::vector* op_inputs) { + std::vector dtypes; + (void)std::transform(signature.begin(), signature.end(), std::back_inserter(dtypes), + [](const Signature& sig) { return sig.dtype; }); + int empty_dtype_count = std::count(dtypes.begin(), dtypes.end(), SignatureEnumDType::kDTypeEmptyDefaultValue); + if (dtypes.size() == 0 || static_cast(dtypes.size()) == empty_dtype_count) { + return; + } + // Stat the index of the arguments with the largest type in the same SignatureEnumDType. + std::map dst_type = GetMaxDtypeIndex(dtypes, args_spec_list); + // Identify which arg requires auto cast + for (size_t i = 0; i < args_spec_list.size(); ++i) { + auto it = dst_type.find(dtypes[i]); + if (it == dst_type.end() || it->second == i || !args_spec_list[i]->isa()) { + continue; + } + // get source node for cast + AnfNodePtr source_node = (*op_inputs)[it->second + 1]; + (*op_inputs)[i + 1] = DoCast((*op_inputs)[i + 1], source_node, graph); + } +} + +AnfNodePtr BuildNewCNode(const FuncGraphPtr& func_graph, const std::string& func_name, const ValuePtr& function, + const AbstractBasePtrList& args_spec_list, const std::vector& params_list) { + // args: original inputs + auto& signature = GetSignature(function); + std::size_t sig_size = signature.size(); + auto has_var = (sig_size > 0 && signature[sig_size - 1].kind == SignatureEnumKind::kKindVarPositional); + if (sig_size > 0) { + if (has_var) { + if (sig_size - 1 > args_spec_list.size()) { + MS_LOG(EXCEPTION) << "Function " << func_name + << "'s input length less than PositionalKeyword Signature length."; + } + } else if (args_spec_list.size() > sig_size) { + MS_LOG(EXCEPTION) << "Function " << func_name << "'s input length is not equal to Signature length."; + } + } + std::vector op_inputs; + op_inputs.push_back(NewValueNode(function)); + // Assume, the write input of op is always the first input. We check if any write op, + // and add cast op on other inputs to keep the same type with assigned parameter. + AnfNodePtr assign_source = nullptr; + for (size_t i = 0; i < args_spec_list.size(); ++i) { + AnfNodePtr param = params_list[i]; + SignatureEnumRW sig = SignatureEnumRW::kRWDefault; + // If sig_size is 0 use defalut. + if (sig_size > 0 && i < sig_size) { + sig = signature[i].rw; + } else if (has_var && i >= sig_size) { + sig = signature[sig_size - 1].rw; + } + TypePtr type = args_spec_list[i]->GetTypeTrack(); + if (type && type->type_id() == kObjectTypeRef) { + if (sig == SignatureEnumRW::kRWRead) { + param = func_graph->NewCNode({NewValueNode(prim::kPrimGetRefValue), param}); + } else if (sig == SignatureEnumRW::kRWWrite) { + assign_source = func_graph->NewCNode({NewValueNode(prim::kPrimGetRefOrigin), param}); + param = func_graph->NewCNode({NewValueNode(prim::kPrimGetRefKey), param}); + } + // If sig is SignatureEnumRW::kRWRef, not do anything. + } + // add cast op here + if (assign_source != nullptr && sig != SignatureEnumRW::kRWWrite) { + param = DoCast(param, assign_source, func_graph); + } + op_inputs.push_back(param); + } + // process default + ProcessDefault(func_name, args_spec_list, signature, has_var, &op_inputs); + DoAutoCast(signature, args_spec_list, func_graph, &op_inputs); + return func_graph->NewCNode(op_inputs); +} +} // namespace + +AnfNodePtr GenerateCNode(const FuncGraphPtr& func_graph, const std::string& func_name, const ValuePtr& function, + const AbstractBasePtrList& args_spec_list, const AnfNodePtrList& old_node_inputs) { + auto new_cnode = BuildNewCNode(func_graph, func_name, function, args_spec_list, old_node_inputs); + return new_cnode; +} + +FuncGraphPtr DoSignatureMetaFuncGraph::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + FuncGraphPtr func_graph = std::make_shared(); + + for (size_t i = 0; i < args_spec_list.size(); ++i) { + (void)func_graph->add_parameter(); + } + auto new_cnode = BuildNewCNode(func_graph, name_, function_, args_spec_list, func_graph->parameters()); + func_graph->set_output(new_cnode); + func_graph->set_flags(FUNC_GRAPH_FLAG_CORE, true); + return func_graph; +} +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/composite/do_signature.h b/mindspore/ccsrc/operator/composite/do_signature.h new file mode 100644 index 00000000000..b88053e2247 --- /dev/null +++ b/mindspore/ccsrc/operator/composite/do_signature.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_COMPOSITE_DO_SIGNATURE_H_ +#define MINDSPORE_CCSRC_OPERATOR_COMPOSITE_DO_SIGNATURE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "pipeline/static_analysis/static_analysis.h" +#include "utils/misc.h" +#include "utils/any.h" +#include "ir/dtype.h" +#include "ir/meta_func_graph.h" +#include "common/utils.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +class DoSignatureMetaFuncGraph : public MetaFuncGraph { + public: + explicit DoSignatureMetaFuncGraph(const std::string& name, const ValuePtr& function) + : MetaFuncGraph("S-" + name), function_(function) {} + + ~DoSignatureMetaFuncGraph() override = default; + + MS_DECLARE_PARENT(DoSignatureMetaFuncGraph, MetaFuncGraph) + + FuncGraphPtr GenerateFuncGraph(const abstract::AbstractBasePtrList& args_spec_list) override; + const ValuePtr function() const { return function_; } + + friend bool operator==(const DoSignatureMetaFuncGraph& lhs, const DoSignatureMetaFuncGraph& rhs) { + return &lhs == &rhs; + } + + private: + ValuePtr function_; +}; +using RWSignaturePtr = std::shared_ptr; + +AnfNodePtr GenerateCNode(const FuncGraphPtr& func_graph, const std::string& func_name, const ValuePtr& function, + const AbstractBasePtrList& args_spec_list, const AnfNodePtrList& old_node_inputs); +} // namespace prim +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPERATOR_COMPOSITE_DO_SIGNATURE_H_ diff --git a/mindspore/ccsrc/operator/composite/list_append_operation.cc b/mindspore/ccsrc/operator/composite/list_append_operation.cc new file mode 100644 index 00000000000..8621a8a8ba0 --- /dev/null +++ b/mindspore/ccsrc/operator/composite/list_append_operation.cc @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/composite/list_append_operation.h" + +#include +#include +#include + +#include "pipeline/static_analysis/param_validator.h" +#include "optimizer/opt.h" +#include "pybind_api/api_register.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +FuncGraphPtr ListAppend::GenerateFuncGraph(const abstract::AbstractBasePtrList& args_list) { + abstract::CheckArgsSize("ListAppend", args_list, 2); + + AbstractBasePtr arg0 = args_list[0]; + abstract::AbstractListPtr arg0_list = dyn_cast(arg0); + MS_EXCEPTION_IF_NULL(arg0_list); + + FuncGraphPtr ret = std::make_shared(); + ret->set_flags(FUNC_GRAPH_FLAG_CORE, true); + ret->debug_info()->set_name("append"); + AnfNodePtr arg0_node = ret->add_parameter(); + + std::vector elems; + elems.push_back(NewValueNode(prim::kPrimMakeList)); + size_t arg0_length = arg0_list->size(); + for (size_t i = 0; i < arg0_length; ++i) { + elems.push_back(ret->NewCNode({NewValueNode(prim::kPrimListGetItem), arg0_node, NewValueNode(SizeToInt(i))})); + } + AnfNodePtr arg1_node = ret->add_parameter(); + elems.push_back(arg1_node); + + ret->set_output(ret->NewCNode(elems)); + return ret; +} + +REGISTER_PYBIND_DEFINE(ListAppend_, ([](const py::module* m) { + (void)py::class_>(*m, "ListAppend_") + .def(py::init()); + })); +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/composite/list_append_operation.h b/mindspore/ccsrc/operator/composite/list_append_operation.h new file mode 100644 index 00000000000..f34b6b864ec --- /dev/null +++ b/mindspore/ccsrc/operator/composite/list_append_operation.h @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_COMPOSITE_LIST_APPEND_OPERATION_H_ +#define MINDSPORE_CCSRC_OPERATOR_COMPOSITE_LIST_APPEND_OPERATION_H_ + +#include +#include +#include + +#include "ir/meta_func_graph.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +class ListAppend : public MetaFuncGraph { + public: + explicit ListAppend(const std::string& name) : MetaFuncGraph(name) {} + ~ListAppend() override = default; + MS_DECLARE_PARENT(ListAppend, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const abstract::AbstractBasePtrList& a_list) override; + friend std::ostream& operator<<(std::ostream& os, const ListAppend& list_append) { + os << list_append.name_; + return os; + } + friend bool operator==(const ListAppend& lhs, const ListAppend& rhs) { return lhs.name_ == rhs.name_; } +}; +using ListAppendPtr = std::shared_ptr; +} // namespace prim +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPERATOR_COMPOSITE_LIST_APPEND_OPERATION_H_ diff --git a/mindspore/ccsrc/operator/composite/zip_operation.cc b/mindspore/ccsrc/operator/composite/zip_operation.cc new file mode 100644 index 00000000000..b87e19b0097 --- /dev/null +++ b/mindspore/ccsrc/operator/composite/zip_operation.cc @@ -0,0 +1,90 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/composite/zip_operation.h" +#include +#include + +#include "pipeline/static_analysis/abstract_value.h" +#include "ir/anf.h" +#include "pipeline/static_analysis/dshape.h" +#include "pipeline/static_analysis/param_validator.h" +#include "operator/cc_implementations.h" +#include "optimizer/opt.h" +#include "utils/symbolic.h" +#include "./common.h" +#include "pybind_api/api_register.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +using mindspore::abstract::AbstractBase; +using mindspore::abstract::AbstractTuple; + +FuncGraphPtr ZipOperation::GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) { + // zip operation: + // input: tuple arguments + // output: tuple of items of input iterated on every input + if (args_spec_list.size() == 0) { + MS_LOG(EXCEPTION) << "zip arguments input should not be empty"; + } + + auto is_all_tuple = std::all_of(args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr& abs) -> bool { + MS_EXCEPTION_IF_NULL(abs); + return abs->isa(); + }); + if (!is_all_tuple) { + MS_LOG(EXCEPTION) << "zip input args should be tuple"; + } + + auto min_abs = std::min_element(args_spec_list.begin(), args_spec_list.end(), + [](const AbstractBasePtr& x, const AbstractBasePtr& y) { + return (x->cast()->size() < y->cast()->size()); + }); + FuncGraphPtr ret_graph = std::make_shared(); + ret_graph->set_flags(FUNC_GRAPH_FLAG_CORE, true); + for (size_t idx = 0; idx < args_spec_list.size(); idx++) { + (void)ret_graph->add_parameter(); + } + + // generate tuple output of ziped arguments input + std::vector make_tuple_nodes; + make_tuple_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t idx = 0; idx < (*min_abs)->cast()->size(); idx++) { + std::vector make_tuple_zip_nodes; + make_tuple_zip_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t arg_idx = 0; arg_idx < args_spec_list.size(); arg_idx++) { + std::vector tuple_get_item_nodes{NewValueNode(prim::kPrimTupleGetItem), + ret_graph->parameters()[arg_idx], NewValueNode(SizeToInt(idx))}; + auto tuple_get_item_op = ret_graph->NewCNode(tuple_get_item_nodes); + make_tuple_zip_nodes.push_back(tuple_get_item_op); + } + auto make_tuple_zip_op = ret_graph->NewCNode(make_tuple_zip_nodes); + make_tuple_nodes.push_back(make_tuple_zip_op); + } + ret_graph->set_output(ret_graph->NewCNode(make_tuple_nodes)); + return ret_graph; +} + +REGISTER_PYBIND_DEFINE(ZipOperation_, ([](const py::module* m) { + (void)py::class_>(*m, + "ZipOperation_") + .def(py::init()); + })); +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/composite/zip_operation.h b/mindspore/ccsrc/operator/composite/zip_operation.h new file mode 100644 index 00000000000..e1fb8d60cf0 --- /dev/null +++ b/mindspore/ccsrc/operator/composite/zip_operation.h @@ -0,0 +1,59 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_COMPOSITE_ZIP_OPERATION_H_ +#define MINDSPORE_CCSRC_OPERATOR_COMPOSITE_ZIP_OPERATION_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "pipeline/static_analysis/static_analysis.h" +#include "utils/misc.h" +#include "utils/any.h" +#include "ir/dtype.h" +#include "ir/meta_func_graph.h" + +namespace mindspore { +// namespace to support composite operators definition +namespace prim { +using AbstractBasePtr = abstract::AbstractBasePtr; +using AbstractBasePtrList = abstract::AbstractBasePtrList; +using AbstractTuplePtr = abstract::AbstractTuplePtr; + +class ZipOperation : public MetaFuncGraph { + public: + explicit ZipOperation(const std::string& name) : MetaFuncGraph(name) {} + ~ZipOperation() override = default; + MS_DECLARE_PARENT(ZipOperation, MetaFuncGraph) + FuncGraphPtr GenerateFuncGraph(const AbstractBasePtrList& args_spec_list) override; + friend std::ostream& operator<<(std::ostream& os, const ZipOperation& op) { + os << op.name_; + return os; + } + friend bool operator==(const ZipOperation& lhs, const ZipOperation& rhs) { return lhs.name_ == rhs.name_; } +}; +using ZipOperationPtr = std::shared_ptr; +} // namespace prim +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPERATOR_COMPOSITE_ZIP_OPERATION_H_ diff --git a/mindspore/ccsrc/operator/ops.cc b/mindspore/ccsrc/operator/ops.cc new file mode 100644 index 00000000000..12e6b70a6f7 --- /dev/null +++ b/mindspore/ccsrc/operator/ops.cc @@ -0,0 +1,242 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/ops.h" +#include +#include +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/data_converter.h" + +namespace mindspore { +// namespace to support primitive operators +namespace prim { +// Arithmetic +const PrimitivePtr kPrimScalarAdd = std::make_shared("scalar_add"); +const PrimitivePtr kPrimScalarSub = std::make_shared("scalar_sub"); +const PrimitivePtr kPrimScalarMul = std::make_shared("scalar_mul"); +const PrimitivePtr kPrimScalarDiv = std::make_shared("scalar_div"); +const PrimitivePtr kPrimScalarMod = std::make_shared("scalar_mod"); +const PrimitivePtr kPrimScalarPow = std::make_shared("scalar_pow"); +const PrimitivePtr kPrimScalarTrunc = std::make_shared("scalar_trunc"); +const PrimitivePtr kPrimScalarFloor = std::make_shared("scalar_floor"); +const PrimitivePtr kPrimScalarUadd = std::make_shared("scalar_uadd"); +const PrimitivePtr kPrimScalarUsub = std::make_shared("scalar_usub"); +const PrimitivePtr kPrimScalarExp = std::make_shared("scalar_exp"); +const PrimitivePtr kPrimScalarLog = std::make_shared("scalar_log"); +const PrimitivePtr kPrimScalarSin = std::make_shared("scalar_sin"); +const PrimitivePtr kPrimScalarCos = std::make_shared("scalar_cos"); +const PrimitivePtr kPrimScalarTan = std::make_shared("scalar_tan"); + +// Comparisons +const PrimitivePtr kPrimScalarEq = std::make_shared("scalar_eq"); +const PrimitivePtr kPrimScalarLt = std::make_shared("scalar_lt"); +const PrimitivePtr kPrimScalarGt = std::make_shared("scalar_gt"); +const PrimitivePtr kPrimScalarNe = std::make_shared("scalar_ne"); +const PrimitivePtr kPrimScalarLe = std::make_shared("scalar_le"); +const PrimitivePtr kPrimScalarGe = std::make_shared("scalar_ge"); +const PrimitivePtr kPrimBoolNot = std::make_shared("bool_not"); +const PrimitivePtr kPrimBoolAnd = std::make_shared("bool_and"); +const PrimitivePtr kPrimBoolOr = std::make_shared("bool_or"); +const PrimitivePtr kPrimBoolEq = std::make_shared("bool_eq"); + +// Type introspection +const PrimitivePtr kPrimTypeOf = std::make_shared("typeof"); +const PrimitivePtr kPrimHasType = std::make_shared("hastype"); + +// Statements +const PrimitivePtr kPrimSwitch = std::make_shared("switch"); +const PrimitivePtr kPrimReturn = std::make_shared("return"); +const PrimitivePtr kPrimAssign = std::make_shared("Assign"); +const PrimitivePtr kPrimAssignAdd = std::make_shared("AssignAdd"); +const PrimitivePtr kPrimAssignSub = std::make_shared("AssignSub"); +const PrimitivePtr kPrimSelect = std::make_shared("Select"); + +const PrimitivePtr kPrimDistribute = std::make_shared("distribute"); +const PrimitivePtr kPrimDot = std::make_shared("dot"); +const PrimitivePtr kPrimIm2Col = std::make_shared("im2col"); +const PrimitivePtr kPrimCol2Im = std::make_shared("col2im"); +const PrimitivePtr kPrimIm2ColV1 = std::make_shared("im2col_v1"); +const PrimitivePtr kPrimCol2ImV1 = std::make_shared("col2im_v1"); + +const PrimitivePtr kPrimResolve = std::make_shared("resolve"); +const PrimitivePtr kPrimEmbed = std::make_shared("embed"); +const PrimitivePtr kPrimRefToEmbed = std::make_shared("RefToEmbed"); +const PrimitivePtr kPrimCreateInstance = std::make_shared("create_instance"); + +// Structure +const PrimitivePtr kPrimStringEqual = std::make_shared("string_equal"); +const PrimitivePtr kPrimMakeTuple = std::make_shared("make_tuple"); +const PrimitivePtr kPrimMakeList = std::make_shared("make_list"); +const PrimitivePtr kPrimMakeDict = std::make_shared("make_dict"); +const PrimitivePtr kPrimMakeKeywordArg = std::make_shared("make_keyword_arg"); +const PrimitivePtr kPrimExtractKeywordArg = std::make_shared("extract_keyword_arg"); +const PrimitivePtr kPrimMakeSlice = std::make_shared("make_slice"); +const PrimitivePtr kPrimMakeRecord = std::make_shared("make_record"); +const PrimitivePtr kPrimTupleGetItem = std::make_shared("tuple_getitem"); +const PrimitivePtr kPrimListGetItem = std::make_shared("list_getitem"); +const PrimitivePtr kPrimArrayGetItem = std::make_shared("array_getitem"); +const PrimitivePtr kPrimTupleSetItem = std::make_shared("tuple_setitem"); +const PrimitivePtr kPrimListSetItem = std::make_shared("list_setitem"); +const PrimitivePtr kPrimArraySetItem = std::make_shared("array_setitem"); +const PrimitivePtr kPrimDictGetItem = std::make_shared("dict_getitem"); +const PrimitivePtr kPrimDictSetItem = std::make_shared("dict_setitem"); +const PrimitivePtr kPrimListAppend = std::make_shared("list_append"); +const PrimitivePtr kPrimGetAttr = std::make_shared("getattr"); +const PrimitivePtr kPrimTupleLen = std::make_shared("tuple_len"); +const PrimitivePtr kPrimDictLen = std::make_shared("dict_len"); +const PrimitivePtr kPrimListLen = std::make_shared("list_len"); +const PrimitivePtr kPrimArrayLen = std::make_shared("array_len"); +const PrimitivePtr kPrimListMap = std::make_shared("list_map"); +const PrimitivePtr kPrimListReduce = std::make_shared("list_reduce"); +const PrimitivePtr kPrimTupleReversed = std::make_shared("tuple_reversed"); + +const PrimitivePtr kPrimTileShape = std::make_shared("tile_shape"); +const PrimitivePtr kPrimReducedShape = std::make_shared("reduced_shape"); +const PrimitivePtr kPrimTupleDiv = std::make_shared("tuple_div"); +const PrimitivePtr kPrimTupleToArray = std::make_shared("tuple_to_array"); +const PrimitivePtr kPrimShapeMul = std::make_shared("shape_mul"); +const PrimitivePtr kPrimGenerateShapeIndex = std::make_shared("generate_shape_index"); +const PrimitivePtr kPrimGenerateInverseIndex = std::make_shared("generate_inverse_index"); +const PrimitivePtr kPrimTupleEqual = std::make_shared("tuple_equal"); +const PrimitivePtr kPrimListEqual = std::make_shared("list_equal"); +const PrimitivePtr kPrimMakeRange = std::make_shared("make_range"); +const PrimitivePtr kPrimStopGradient = std::make_shared("stop_gradient"); + +// Arrays +const PrimitivePtr kPrimScalarToArray = std::make_shared("scalar_to_array"); +const PrimitivePtr kPrimArrayToScalar = std::make_shared("array_to_scalar"); +const PrimitivePtr kPrimBroadcastShape = std::make_shared("broadcast_shape"); +const PrimitivePtr kPrimArrayMap = std::make_shared("array_map"); +const PrimitivePtr kPrimArrayReduce = std::make_shared("array_reduce"); +const PrimitivePtr kPrimShape = std::make_shared("Shape"); +const PrimitivePtr kPrimCast = std::make_shared("Cast"); +const PrimitivePtr kPrimConcat = std::make_shared("Concat"); +const PrimitivePtr kPrimSqueeze = std::make_shared("Squeeze"); +const PrimitivePtr kPrimTranspose = std::make_shared("Transpose"); +const PrimitivePtr kPrimGatherV2 = std::make_shared("GatherV2"); +const PrimitivePtr kPrimSize = std::make_shared("Size"); +const PrimitivePtr kPrimArgMax = std::make_shared("Argmax"); +const PrimitivePtr kPrimPack = std::make_shared("Pack"); +const PrimitivePtr kPrimUnsortedSegmentSum = std::make_shared("UnsortedSegmentSum"); +const PrimitivePtr kPrimConcatOffset = std::make_shared("ConcatOffset"); +const PrimitivePtr kPrimReshape = std::make_shared("Reshape"); +const PrimitivePtr kPrimTile = std::make_shared("Tile"); +const PrimitivePtr kPrimAddN = std::make_shared("AddN"); +const PrimitivePtr KPrimTransData = std::make_shared("TransData"); + +// Maths +const PrimitivePtr kPrimTensorAdd = std::make_shared("TensorAdd"); +const PrimitivePtr kPrimMatMul = std::make_shared("MatMul"); +const PrimitivePtr kPrimBatchMatMul = std::make_shared("BatchMatMul"); +const PrimitivePtr kPrimMaximumGrad = std::make_shared("MaximumGrad"); +const PrimitivePtr kPrimMinimumGrad = std::make_shared("MinimumGrad"); +const PrimitivePtr kPrimReduceMean = std::make_shared("ReduceMean"); +const PrimitivePtr kPrimReduceSum = std::make_shared("ReduceSum"); +const PrimitivePtr kPrimReduceAll = std::make_shared("ReduceAll"); +const PrimitivePtr kPrimReduceMax = std::make_shared("ReduceMax"); +const PrimitivePtr kPrimReduceMin = std::make_shared("ReduceMin"); +const PrimitivePtr kPrimNeg = std::make_shared("Neg"); +const PrimitivePtr kPrimSub = std::make_shared("Sub"); +const PrimitivePtr kPrimMul = std::make_shared("Mul"); +const PrimitivePtr kPrimMinimum = std::make_shared("Minimum"); +const PrimitivePtr kPrimMaximum = std::make_shared("Maximum"); +const PrimitivePtr kPrimSquare = std::make_shared("Square"); + +// NN +const PrimitivePtr kPrimFlatten = std::make_shared("Flatten"); +const PrimitivePtr kPrimLogSoftmax = std::make_shared("LogSoftmax"); +const PrimitivePtr kPrimLogSoftmaxGrad = std::make_shared("LogSoftmaxGrad"); +const PrimitivePtr kPrimTanh = std::make_shared("Tanh"); +const PrimitivePtr kPrimTanhGrad = std::make_shared("TanhGrad"); +const PrimitivePtr kPrimPooling = std::make_shared("Pooling"); +const PrimitivePtr kPrimPoolingGrad = std::make_shared("PoolingGrad"); +const PrimitivePtr kPrimMaxPool = std::make_shared("MaxPool"); +const PrimitivePtr kPrimMaxPoolGrad = std::make_shared("MaxPoolGrad"); +const PrimitivePtr kPrimFusedBatchNorm = std::make_shared("FusedBatchNorm"); +const PrimitivePtr kPrimConv2D = std::make_shared("Conv2D"); +const PrimitivePtr kPrimFusedBatchNormGrad = std::make_shared("FusedBatchNormGrad"); +const PrimitivePtr kPrimReluGrad = std::make_shared("ReluGrad"); +const PrimitivePtr kPrimConv2DBackpropInput = std::make_shared("Conv2DBackpropInput"); +const PrimitivePtr kPrimConv2DBackpropFilter = std::make_shared("Conv2DBackpropFilter"); +const PrimitivePtr kPrimDepthwiseConv2dNative = std::make_shared("DepthwiseConv2dNative"); +const PrimitivePtr kPrimDepthwiseConv2dNativeBackpropFilter = + std::make_shared("DepthwiseConv2dNativeBackpropFilter"); +const PrimitivePtr kPrimDepthwiseConv2dNativeBackpropInput = + std::make_shared("DepthwiseConv2dNativeBackpropInput"); +const PrimitivePtr kPrimBiasAddGrad = std::make_shared("BiasAddGrad"); +const PrimitivePtr kPrimSoftmaxCrossEntropyWithLogits = std::make_shared("SoftmaxCrossEntropyWithLogits"); +const PrimitivePtr kPrimSparseSoftmaxCrossEntropyWithLogits = + std::make_shared("SparseSoftmaxCrossEntropyWithLogits"); +const PrimitivePtr kPrimMomentum = std::make_shared("Momentum"); +const PrimitivePtr kPrimApplyMomentum = std::make_shared("ApplyMomentum"); +const PrimitivePtr kPrimLayerNorm = std::make_shared("LayerNorm"); +const PrimitivePtr kPrimLayerNormGrad = std::make_shared("LayerNormGrad"); +const PrimitivePtr kPrimLayerNormXBackprop = std::make_shared("LayerNormXBackprop"); +const PrimitivePtr kPrimLayerNormBetaGammaBackprop = std::make_shared("LayerNormBetaGammaBackprop"); +const PrimitivePtr kPrimDropoutGenMask = std::make_shared("DropoutGenMask"); +const PrimitivePtr kPrimOneHot = std::make_shared("OneHot"); +const PrimitivePtr kPrimGelu = std::make_shared("Gelu"); +const PrimitivePtr kPrimGeluGrad = std::make_shared("GeluGrad"); +const PrimitivePtr kPrimRelu = std::make_shared("ReLU"); +const PrimitivePtr kPrimZerosLikeTensor = std::make_shared("zeros_like_tensor"); +const PrimitivePtr kPrimFakeBprop = std::make_shared("fake_bprop"); + +// Other miscellaneous +const PrimitivePtr kPrimIdentity = std::make_shared("identity"); +const PrimitivePtr kPrimPartial = std::make_shared("partial"); +const PrimitivePtr kPrimJ = std::make_shared("J"); +const PrimitivePtr kPrimEnvSetItem = std::make_shared("env_setitem"); +const PrimitivePtr kPrimEnvGetItem = std::make_shared("env_getitem"); +const PrimitivePtr kPrimEnvAdd = std::make_shared("env_add"); +const PrimitivePtr kPrimMakeRefKey = std::make_shared("MakeRefKey"); +const PrimitivePtr kPrimGetRefKey = std::make_shared("get_ref_key"); +const PrimitivePtr kPrimGetRefValue = std::make_shared("get_ref_value"); +const PrimitivePtr kPrimGetRefOrigin = std::make_shared("get_ref_origin"); +const PrimitivePtr kPrimInsertGradientOf = std::make_shared("InsertGradientOf"); +const PrimitivePtr kPrimPrintShapeType = std::make_shared("PrintShapeType"); +const PrimitivePtr kPrimSameTypeShape = std::make_shared("SameTypeShape"); +const PrimitivePtr kPrimPrint = std::make_shared("Print"); + +const PrimitivePtr kPrimMakeRef = std::make_shared("make_ref"); +const PrimitivePtr kPrimDepend = std::make_shared("depend"); +const PrimitivePtr kPrimStateSetItem = std::make_shared("state_setitem"); + +const PrimitivePtr kPrimBroadcastGradientArgs = std::make_shared("BroadcastGradientArgs"); +const PrimitivePtr kPrimControlDepend = std::make_shared("ControlDepend"); +const PrimitivePtr kPrimIs_ = std::make_shared("is_"); +const PrimitivePtr kPrimIsNot = std::make_shared("is_not"); + +// Comm ops +const PrimitivePtr kPrimMirror = std::make_shared("_MirrorOperator"); +const PrimitivePtr kPrimVirtualDiv = std::make_shared("_VirtualDiv"); +const PrimitivePtr kPrimVirtualDataset = std::make_shared("_VirtualDataset"); + +// Debug ops +const PrimitivePtr kPrimScalarSummary = std::make_shared("ScalarSummary"); +const PrimitivePtr kPrimImageSummary = std::make_shared("ImageSummary"); +const PrimitivePtr kPrimTensorSummary = std::make_shared("TensorSummary"); + +ValuePtr GetPythonOps(const std::string& op_name, const std::string& module_name) { + py::object obj = parse::python_adapter::GetPyFn(module_name, op_name); + ValuePtr node = nullptr; + bool succ = parse::ConvertData(obj, &node); + if (!succ) { + MS_LOG(EXCEPTION) << "get Python op " << op_name << " from " << module_name << " fail"; + } + return node; +} +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/ops.h b/mindspore/ccsrc/operator/ops.h new file mode 100644 index 00000000000..f3f5dad5f1d --- /dev/null +++ b/mindspore/ccsrc/operator/ops.h @@ -0,0 +1,252 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_OPS_H_ +#define MINDSPORE_CCSRC_OPERATOR_OPS_H_ + +#include +#include +#include +#include "ir/anf.h" +#include "ir/primitive.h" + +namespace mindspore { +// namespace to support primitive operators +namespace prim { +ValuePtr GetPythonOps(const std::string& op_name, + const std::string& module_name = "mindspore._extends.parse.standard_method"); + +// Arithmetic +extern const PrimitivePtr kPrimScalarAdd; +extern const PrimitivePtr kPrimScalarSub; +extern const PrimitivePtr kPrimScalarMul; +extern const PrimitivePtr kPrimScalarDiv; +extern const PrimitivePtr kPrimScalarMod; +extern const PrimitivePtr kPrimScalarPow; +extern const PrimitivePtr kPrimScalarTrunc; +extern const PrimitivePtr kPrimScalarFloor; +extern const PrimitivePtr kPrimScalarUadd; +extern const PrimitivePtr kPrimScalarUsub; +extern const PrimitivePtr kPrimScalarExp; +extern const PrimitivePtr kPrimScalarLog; +extern const PrimitivePtr kPrimScalarSin; +extern const PrimitivePtr kPrimScalarCos; +extern const PrimitivePtr kPrimScalarTan; + +// Comparisons +extern const PrimitivePtr kPrimScalarEq; +extern const PrimitivePtr kPrimScalarLt; +extern const PrimitivePtr kPrimScalarGt; +extern const PrimitivePtr kPrimScalarNe; +extern const PrimitivePtr kPrimScalarLe; +extern const PrimitivePtr kPrimScalarGe; +extern const PrimitivePtr kPrimBoolNot; +extern const PrimitivePtr kPrimBoolAnd; +extern const PrimitivePtr kPrimBoolOr; +extern const PrimitivePtr kPrimBoolEq; + +// Type introspection +extern const PrimitivePtr kPrimTypeOf; +extern const PrimitivePtr kPrimHasType; + +// Statements +extern const PrimitivePtr kPrimSwitch; +extern const PrimitivePtr kPrimReturn; +extern const PrimitivePtr kPrimAssign; +extern const PrimitivePtr kPrimAssignAdd; +extern const PrimitivePtr kPrimAssignSub; +extern const PrimitivePtr kPrimSelect; + +extern const PrimitivePtr kPrimDistribute; +extern const PrimitivePtr kPrimDot; +extern const PrimitivePtr kPrimIm2Col; +extern const PrimitivePtr kPrimCol2Im; +extern const PrimitivePtr kPrimIm2ColV1; +extern const PrimitivePtr kPrimCol2ImV1; + +extern const PrimitivePtr kPrimResolve; +extern const PrimitivePtr kPrimEmbed; +extern const PrimitivePtr kPrimRefToEmbed; +extern const PrimitivePtr kPrimCreateInstance; + +// Structure +extern const PrimitivePtr kPrimStringEqual; +extern const PrimitivePtr kPrimMakeTuple; +extern const PrimitivePtr kPrimMakeList; +extern const PrimitivePtr kPrimMakeDict; +extern const PrimitivePtr kPrimMakeKeywordArg; +extern const PrimitivePtr kPrimExtractKeywordArg; +extern const PrimitivePtr kPrimMakeSlice; +extern const PrimitivePtr kPrimMakeRecord; +extern const PrimitivePtr kPrimTupleGetItem; +extern const PrimitivePtr kPrimListGetItem; +extern const PrimitivePtr kPrimArrayGetItem; +extern const PrimitivePtr kPrimTupleSetItem; +extern const PrimitivePtr kPrimListSetItem; +extern const PrimitivePtr kPrimArraySetItem; +extern const PrimitivePtr kPrimDictGetItem; +extern const PrimitivePtr kPrimDictSetItem; +extern const PrimitivePtr kPrimListAppend; +extern const PrimitivePtr kPrimGetAttr; +extern const PrimitivePtr kPrimTupleLen; +extern const PrimitivePtr kPrimDictLen; +extern const PrimitivePtr kPrimListLen; +extern const PrimitivePtr kPrimArrayLen; +extern const PrimitivePtr kPrimListMap; +extern const PrimitivePtr kPrimListReduce; +extern const PrimitivePtr kPrimTupleReversed; +extern const PrimitivePtr kPrimTileShape; +extern const PrimitivePtr kPrimReducedShape; +extern const PrimitivePtr kPrimTupleDiv; +extern const PrimitivePtr kPrimTupleToArray; +extern const PrimitivePtr kPrimShapeMul; +extern const PrimitivePtr kPrimGenerateShapeIndex; +extern const PrimitivePtr kPrimGenerateInverseIndex; +extern const PrimitivePtr kPrimTupleEqual; +extern const PrimitivePtr kPrimListEqual; +extern const PrimitivePtr kPrimMakeRange; +extern const PrimitivePtr kPrimStopGradient; + +// Arrays +extern const PrimitivePtr kPrimScalarToArray; +extern const PrimitivePtr kPrimArrayToScalar; +extern const PrimitivePtr kPrimBroadcastShape; +extern const PrimitivePtr kPrimArrayMap; +extern const PrimitivePtr kPrimArrayReduce; +extern const PrimitivePtr kPrimShape; +extern const PrimitivePtr kPrimCast; +extern const PrimitivePtr kPrimConcat; +extern const PrimitivePtr kPrimSqueeze; +extern const PrimitivePtr kPrimTranspose; +extern const PrimitivePtr kPrimGatherV2; +extern const PrimitivePtr kPrimSize; +extern const PrimitivePtr kPrimArgMax; +extern const PrimitivePtr kPrimPack; +extern const PrimitivePtr kPrimUnsortedSegmentSum; +extern const PrimitivePtr kPrimConcatOffset; +extern const PrimitivePtr kPrimReshape; +extern const PrimitivePtr kPrimTile; +extern const PrimitivePtr kPrimAddN; +extern const PrimitivePtr KPrimTransData; + +// Maths +extern const PrimitivePtr kPrimTensorAdd; +extern const PrimitivePtr kPrimMatMul; +extern const PrimitivePtr kPrimBatchMatMul; +extern const PrimitivePtr kPrimMaximumGrad; +extern const PrimitivePtr kPrimMinimumGrad; +extern const PrimitivePtr kPrimReduceMean; +extern const PrimitivePtr kPrimReduceSum; +extern const PrimitivePtr kPrimReduceAll; +extern const PrimitivePtr kPrimReduceMax; +extern const PrimitivePtr kPrimReduceMin; +extern const PrimitivePtr kPrimNeg; +extern const PrimitivePtr kPrimSub; +extern const PrimitivePtr kPrimMul; +extern const PrimitivePtr kPrimMinimum; +extern const PrimitivePtr kPrimMaximum; +extern const PrimitivePtr kPrimSquare; + +// NN +extern const PrimitivePtr kPrimFlatten; +extern const PrimitivePtr kPrimLogSoftmax; +extern const PrimitivePtr kPrimLogSoftmaxGrad; +extern const PrimitivePtr kPrimTanh; +extern const PrimitivePtr kPrimTanhGrad; +extern const PrimitivePtr kPrimPooling; +extern const PrimitivePtr kPrimPoolingGrad; +extern const PrimitivePtr kPrimFusedBatchNorm; +extern const PrimitivePtr kPrimConv2D; +extern const PrimitivePtr kPrimMaxPool; +extern const PrimitivePtr kPrimMaxPoolGrad; +extern const PrimitivePtr kPrimFusedBatchNormGrad; +extern const PrimitivePtr kPrimReluGrad; +extern const PrimitivePtr kPrimConv2DBackpropInput; +extern const PrimitivePtr kPrimConv2DBackpropFilter; +extern const PrimitivePtr kPrimDepthwiseConv2dNative; +extern const PrimitivePtr kPrimDepthwiseConv2dNativeBackpropFilter; +extern const PrimitivePtr kPrimDepthwiseConv2dNativeBackpropInput; + +extern const PrimitivePtr kPrimBiasAddGrad; +extern const PrimitivePtr kPrimSoftmaxCrossEntropyWithLogits; +extern const PrimitivePtr kPrimSparseSoftmaxCrossEntropyWithLogits; +extern const PrimitivePtr kPrimMomentum; +extern const PrimitivePtr kPrimApplyMomentum; +extern const PrimitivePtr kPrimLayerNorm; +extern const PrimitivePtr kPrimLayerNormGrad; +extern const PrimitivePtr kPrimLayerNormXBackprop; +extern const PrimitivePtr kPrimLayerNormBetaGammaBackprop; +extern const PrimitivePtr kPrimDropoutGenMask; +extern const PrimitivePtr kPrimOneHot; +extern const PrimitivePtr kPrimGelu; +extern const PrimitivePtr kPrimGeluGrad; +extern const PrimitivePtr kPrimRelu; +extern const PrimitivePtr kPrimActivation; +extern const PrimitivePtr kPrimZerosLikeTensor; +extern const PrimitivePtr kPrimFakeBprop; + +// Other Miscellaneous +extern const PrimitivePtr kPrimIdentity; +extern const PrimitivePtr kPrimPartial; +extern const PrimitivePtr kPrimJ; +extern const PrimitivePtr kPrimEnvSetItem; +extern const PrimitivePtr kPrimEnvGetItem; +extern const PrimitivePtr kPrimEnvAdd; +extern const PrimitivePtr kPrimMakeRefKey; +extern const PrimitivePtr kPrimMakeRef; +extern const PrimitivePtr kPrimGetRefKey; +extern const PrimitivePtr kPrimGetRefValue; +extern const PrimitivePtr kPrimGetRefOrigin; +extern const PrimitivePtr kPrimInsertGradientOf; +extern const PrimitivePtr kPrimPrintShapeType; +extern const PrimitivePtr kPrimPrint; +extern const PrimitivePtr kPrimSameTypeShape; +extern const PrimitivePtr kPrimDepend; +extern const PrimitivePtr kPrimStateSetItem; +extern const PrimitivePtr kPrimScalarSummary; +extern const PrimitivePtr kPrimImageSummary; +extern const PrimitivePtr kPrimTensorSummary; +extern const PrimitivePtr kPrimBroadcastGradientArgs; +extern const PrimitivePtr kPrimControlDepend; +extern const PrimitivePtr kPrimIs_; +extern const PrimitivePtr kPrimIsNot; +extern const PrimitivePtr kPrimMinimumGrad; +extern const PrimitivePtr kPrimMaximumGrad; + +// Comm ops +extern const PrimitivePtr kPrimMirror; +extern const PrimitivePtr kPrimVirtualDiv; +extern const PrimitivePtr kPrimVirtualDataset; + +class DoSignaturePrimitive : public Primitive { + public: + explicit DoSignaturePrimitive(const std::string& name, const ValuePtr& function) + : Primitive("S-Prim-" + name), function_(function) {} + + ~DoSignaturePrimitive() override = default; + + MS_DECLARE_PARENT(DoSignaturePrimitive, Primitive) + + const ValuePtr function() const { return function_; } + + private: + ValuePtr function_; +}; +using DoSignaturePrimitivePtr = std::shared_ptr; +} // namespace prim +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPERATOR_OPS_H_ diff --git a/mindspore/ccsrc/operator/prim_arrays.cc b/mindspore/ccsrc/operator/prim_arrays.cc new file mode 100644 index 00000000000..237ca795eb9 --- /dev/null +++ b/mindspore/ccsrc/operator/prim_arrays.cc @@ -0,0 +1,170 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" +#include "operator/cc_implementations.h" +#include "pipeline/static_analysis/param_validator.h" + +namespace mindspore { +namespace abstract { +AbstractBasePtr InferImplScalarToArray(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a scalar. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + AbstractScalarPtr arg = CheckArg(op_name, args_spec_list, 0); + return std::make_shared(arg, std::make_shared()); +} + +AbstractBasePtr InferImplArrayToScalar(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor with 0 shape. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto arg = CheckArg(op_name, args_spec_list, 0); + auto a_shp = arg->shape(); + if (!a_shp->shape().empty()) { + MS_LOG(EXCEPTION) << "array_to_scalar requires zero size shape."; + } + return arg->element(); +} + +AbstractBasePtr InferImplBroadCastShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tuples. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + auto xs = CheckArg(op_name, args_spec_list, 0); + auto ys = CheckArg(op_name, args_spec_list, 1); + + auto value_tuple_x = xs->BuildValue()->cast(); + MS_EXCEPTION_IF_NULL(value_tuple_x); + auto shp_tuple_x = value_tuple_x->value(); + std::vector shp_x; + (void)std::transform(std::begin(shp_tuple_x), std::end(shp_tuple_x), std::back_inserter(shp_x), + [](const ValuePtr &e) -> int { return GetValue(e); }); + + auto value_tuple_y = ys->BuildValue()->cast(); + MS_EXCEPTION_IF_NULL(value_tuple_y); + auto shp_tuple_y = value_tuple_y->value(); + std::vector shp_y; + (void)std::transform(std::begin(shp_tuple_y), std::end(shp_tuple_y), std::back_inserter(shp_y), + [](const ValuePtr &e) -> int { return GetValue(e); }); + + std::vector res = prim::BroadcastShape_(shp_x, shp_y); + if (res.empty()) { + MS_LOG(EXCEPTION) << "BroadcastShape fail: " << args_spec_list[0]->ToString() << "," + << args_spec_list[1]->ToString(); + } + + AbstractBasePtrList elems; + (void)std::transform(res.begin(), res.end(), std::back_inserter(elems), [](int n) -> AbstractBasePtr { + return std::make_shared(std::make_shared(n), kInt32); + }); + + return std::make_shared(elems); +} + +AbstractBasePtr InferImplShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + AbstractTensorPtr arg = CheckArg(op_name, args_spec_list, 0); + MS_LOG(DEBUG) << "InferImplShape:" << arg->ToString(); + + AbstractBasePtrList values; + auto shp = arg->shape(); + for (int entry : shp->shape()) { + auto entry_v = MakeValue(entry); + values.push_back(std::make_shared(entry_v, entry_v->type())); + } + return std::make_shared(values); +} + +AbstractBasePtr InferImplTile(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + auto arg = CheckArg(op_name, args_spec_list, 0); + auto multiples = CheckArg(op_name, args_spec_list, 1); + + ShapePtr input_shape = arg->shape(); + (void)CheckTensorDType(arg, {kInt16, kFloat16, kInt32, kFloat32}, "Input 0 of Tile should be %s"); + + auto mul_shp_value = multiples->BuildValue(); + if (mul_shp_value->isa()) { + MS_LOG(EXCEPTION) << "shape's data field can't be anything: " << args_spec_list[1]->ToString(); + } + + std::vector mul_shp; + auto value_tuple_mul = mul_shp_value->cast(); + auto mul_shp_data = value_tuple_mul->value(); + (void)std::transform(std::begin(mul_shp_data), std::end(mul_shp_data), std::back_inserter(mul_shp), + [](const ValuePtr &e) -> int { return GetValue(e); }); + if (input_shape->shape().size() != mul_shp_data.size()) { + MS_LOG(EXCEPTION) << "Tile requires input and multiples size equal, while the input size is " + << input_shape->shape().size() << ", value size is: " << mul_shp_data.size() << "."; + } + + std::vector result_shp; + for (size_t i = 0; i < mul_shp_data.size(); ++i) { + result_shp.push_back(input_shape->shape()[i] * mul_shp[i]); + } + return std::make_shared(arg->element(), std::make_shared(result_shp)); +} + +AbstractBasePtr InferImplPack(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple of tensor. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto arg = CheckArg(op_name, args_spec_list, 0); + if (arg->elements().empty()) { + MS_LOG(EXCEPTION) << "Arg elements is empty."; + } + + size_t tuple_len = arg->elements().size(); + AbstractTensorPtr tensor_base = CheckArg(op_name, arg->elements(), 0); + int rank_base = SizeToInt(tensor_base->shape()->shape().size()); + + ValuePtr axis = primitive->GetAttr("axis"); + // Axis value should be in [-(rank_base + 1), rank_base). + int axis_value = CheckAxis(op_name, axis, -(rank_base + 1), rank_base); + // If axis is negative, add offset(rank_base + 1) to turn it to positive. + axis_value = GetPositiveAxis(axis_value, IntToSize(rank_base + 1)); + + for (size_t i = 1; i < tuple_len; ++i) { + AbstractTensorPtr tensor = CheckArg(op_name, arg->elements(), i); + (void)CheckDtypeSame(op_name, tensor_base, tensor); + (void)CheckShapeSame(op_name, tensor_base, tensor); + } + + primitive->set_attr("N", MakeValue(SizeToInt(tuple_len))); + primitive->set_attr("T", tensor_base->element()->BuildType()); + + AbstractTensorPtr ret = dyn_cast(tensor_base->Broaden()); + MS_EXCEPTION_IF_NULL(ret); + auto shape = ret->shape()->shape(); + (void)shape.insert(shape.begin() + axis_value, tuple_len); + ret->set_shape(std::make_shared(shape)); + return ret; +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_debug.cc b/mindspore/ccsrc/operator/prim_debug.cc new file mode 100644 index 00000000000..28f7e92303d --- /dev/null +++ b/mindspore/ccsrc/operator/prim_debug.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/param_validator.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" +#include "utils/symbolic.h" + +namespace mindspore { +namespace abstract { +AbstractBasePtr InferImplScalarSummary(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a scalar and a tensor or scalar. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + + // check the tag + AbstractScalarPtr descriptions = CheckArg(op_name, args_spec_list, 0); + + // check the value: scalar or shape = (1,) + auto scalar_value = dyn_cast(args_spec_list[1]); + if (scalar_value == nullptr) { + auto tensor_value = dyn_cast(args_spec_list[1]); + if (tensor_value == nullptr) { + MS_LOG(EXCEPTION) << "Input must be scalar or shape(1,)"; + } + } else { + auto item_v = scalar_value->BuildValue(); + if (item_v->isa()) { + auto value = item_v->cast()->value(); + if (value.empty()) { + MS_LOG(EXCEPTION) << "Input summary value can't be null"; + } + } + } + + // Reomve the force check to support batch set summary use 'for' loop + auto item_v = descriptions->BuildValue(); + if (!item_v->isa()) { + MS_LOG(ERROR) << "First parameter shoule be string"; + } + + return std::make_shared(kAnyValue, kBool); +} + +AbstractBasePtr InferImplTensorSummary(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a scalar(tag) and a tensor(value) + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + + // check the tag + auto descriptions = CheckArg(op_name, args_spec_list, 0); + auto tensor_value = CheckArg(op_name, args_spec_list, 1); + + int tensor_rank = SizeToInt(tensor_value->shape()->shape().size()); + if (tensor_rank == 0) { + MS_LOG(EXCEPTION) << "Tensor/Image Summary evaluator second arg should be an tensor, but got a scalar"; + } + + // Reomve the force check to support batch set summary use 'for' loop + auto item_v = descriptions->BuildValue(); + if (!item_v->isa()) { + MS_LOG(WARNING) << "Summary first parameter must be string"; + } + + return std::make_shared(kAnyValue, std::make_shared()); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_maths.cc b/mindspore/ccsrc/operator/prim_maths.cc new file mode 100644 index 00000000000..02b86603e74 --- /dev/null +++ b/mindspore/ccsrc/operator/prim_maths.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" +#include "pipeline/static_analysis/param_validator.h" +#include "common/utils.h" + +namespace mindspore { +namespace abstract { +AbstractBasePtr InferImplMinOrMaxGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: three tensors. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + auto input_x = CheckArg(op_name, args_spec_list, 0); + auto input_y = CheckArg(op_name, args_spec_list, 1); + auto dout = CheckArg(op_name, args_spec_list, 2); + (void)CheckTensorsDTypeSame({input_x, input_y, dout}, {kInt, kUInt, kFloat}, + op_name + "evaluator three inputs should be %s"); + + AbstractBasePtr dx = input_x->Broaden(); + AbstractBasePtr dy = input_y->Broaden(); + + return std::make_shared(AbstractBasePtrList({dx, dy})); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_nn.cc b/mindspore/ccsrc/operator/prim_nn.cc new file mode 100644 index 00000000000..892bf2921eb --- /dev/null +++ b/mindspore/ccsrc/operator/prim_nn.cc @@ -0,0 +1,413 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" +#include "pipeline/static_analysis/param_validator.h" + +namespace mindspore { +namespace abstract { +AbstractBasePtr InferImplPooling(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + AbstractTensorPtr input_tensor = CheckArg(op_name, args_spec_list, 0); + (void)CheckTensorDType(input_tensor, {kFloat16, kFloat32}, "Input 0 of Pooling should be %s"); + + ShapePtr input_shape = dyn_cast(input_tensor->GetShapeTrack()); // NCHW + MS_EXCEPTION_IF_NULL(input_shape); + if (input_shape->shape().size() != 4) { + MS_LOG(EXCEPTION) << "Pooling input should be a 4-D tensor."; + } + int h_input = input_shape->shape()[2]; + int w_input = input_shape->shape()[3]; + + int window = primitive->GetAttr("window")->cast()->value(); + int stride = primitive->GetAttr("stride")->cast()->value(); + int padding = primitive->GetAttr("pad")->cast()->value(); + int nan_opt = primitive->GetAttr("nan_opt")->cast()->value(); + int data_mode = primitive->GetAttr("data_mode")->cast()->value(); + int ceil_mode = primitive->GetAttr("ceil_mode")->cast()->value(); + + if (stride <= 0) { + MS_LOG(EXCEPTION) << "Invalid stride value: " << stride << ", should greater then 0"; + } + if (nan_opt != 0) { + MS_LOG(EXCEPTION) << "Invalid nan_opt value: " << nan_opt << ", should be 0"; + } + if (data_mode != 1) { + MS_LOG(EXCEPTION) << "Invalid data_mode value: " << data_mode << ", should be 1"; + } + if (ceil_mode != 0) { + MS_LOG(EXCEPTION) << "Invalid ceil_mode value: " << ceil_mode << ", should be 0"; + } + + std::set available_pad_mode{"pad", "same", "valid"}; + auto pad_mode_ptr = primitive->GetAttr("pad_mode"); + if ((pad_mode_ptr != nullptr) && pad_mode_ptr->isa()) { + auto pad_mode = pad_mode_ptr->cast()->value(); + if (available_pad_mode.find(pad_mode) == available_pad_mode.end()) { + MS_LOG(EXCEPTION) << "Unsupported pad mode: " << pad_mode << ". use pad, same, valid"; + } + if (pad_mode == "valid") { + padding = 0; + } else if (pad_mode == "same") { + padding = (window - 1) / 2; + } + } + + std::set available_mode{"max", "avg"}; + auto mode_ptr = primitive->GetAttr("mode"); + if ((mode_ptr != nullptr) && mode_ptr->isa()) { + auto mode = mode_ptr->cast()->value(); + if (available_mode.find(mode) == available_mode.end()) { + MS_LOG(EXCEPTION) << "Unsupported pooling mode: " << mode << "."; + } + } + + int h_out = ((h_input + 2 * padding - (window - 1) - 1) / stride) + 1; + int w_out = ((w_input + 2 * padding - (window - 1) - 1) / stride) + 1; + std::vector shape_out = {input_shape->shape()[0], input_shape->shape()[1], h_out, w_out}; + AbstractBasePtr ret = input_tensor->Broaden(); + ret->set_shape(std::make_shared(shape_out)); + return ret; +} + +AbstractBasePtr InferImplPoolingGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: three tensors(y, dy, x). + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + auto out_y = CheckArg(op_name, args_spec_list, 0); + auto d_out = CheckArg(op_name, args_spec_list, 1); + auto input_x = CheckArg(op_name, args_spec_list, 2); + (void)CheckTensorsDTypeSame({out_y, d_out, input_x}, {kInt, kUInt, kFloat}, + op_name + "evaluator three inputs should be %s"); + + AbstractBasePtr ret = d_out->Broaden(); + auto x_shape = dyn_cast(args_spec_list[2]->GetShapeTrack()); + MS_EXCEPTION_IF_NULL(x_shape); + + ret->set_shape(x_shape); + return ret; +} + +void FusedBatchNormCheckDim(const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list) { + // check dimension, x > 1, others equal 1 + const std::string op_name = primitive->name(); + for (std::size_t i = 0; i < args_spec_list.size(); ++i) { + AbstractTensorPtr arg = CheckArg(op_name, args_spec_list, i); + ShapePtr arg_shape = dyn_cast(arg->GetShapeTrack()); + if (arg_shape == nullptr) { + MS_LOG(EXCEPTION) << "" << op_name << " type of args[" << i << "] should be Shape, but " << arg->ToString(); + } + + if (i == 0) { + if (arg_shape->shape().size() < 2) { + MS_LOG(EXCEPTION) << "" << op_name << " shape of args[" << i + << "] should be TensorShape with dimension greater than 1, but shape: " + << arg_shape->ToString(); + } + continue; + } + + if (arg_shape->shape().size() != 1) { + MS_LOG(EXCEPTION) << "" << op_name << " shape of args[" << i + << "] should be TensorShape with dimension: 1, but shape: " << arg_shape->ToString(); + } + } +} + +AbstractBasePtr InferImplFusedBatchNorm(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: five tensors(x, gamma, beta, mean, variance). + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 5); + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + MS_LOG(DEBUG) << "InferImplFusedBatchNorm args0:" << args_spec_list[0]->ToString() + << ", arg1:" << args_spec_list[1]->ToString(); + FusedBatchNormCheckDim(primitive, args_spec_list); + + auto input = args_spec_list[0]; + auto input_shape = dyn_cast(input->GetShapeTrack()); + MS_EXCEPTION_IF_NULL(input_shape); + const auto &input_shape_list = input_shape->shape(); + if (input_shape_list.size() < 2) { + MS_LOG(EXCEPTION) << "Input shape size should >= 2."; + } + + for (size_t i = 1; i < args_spec_list.size(); ++i) { + auto arg_shape = dyn_cast(args_spec_list[i]->GetShapeTrack()); + MS_EXCEPTION_IF_NULL(arg_shape); + const auto &arg_shape_list = arg_shape->shape(); + if (arg_shape_list.size() < 1) { + MS_LOG(EXCEPTION) << "Arg shape size should >= 1."; + } + if (arg_shape_list[0] != input_shape_list[1]) { + MS_LOG(EXCEPTION) << "" << op_name << " size of tensor param[" << i << "](which is " << arg_shape_list[0] + << ") should match the second dimension of tensor" + " param[0](which is " + << input_shape_list[1] << ")."; + } + } + auto input_tensor = CheckArg(op_name, args_spec_list, 0); + (void)CheckTensorDType(input_tensor, {kFloat16, kFloat32}, "param 0 of FusedBatchNorm should be %s"); + + AbstractTensorPtrList tensorPtrList = std::vector(); + for (size_t i = 1; i < args_spec_list.size(); ++i) { + auto param = CheckArg(op_name, args_spec_list, i); + tensorPtrList.push_back(param); + } + (void)CheckTensorsDTypeSame(tensorPtrList, {kFloat16, kFloat32}, "param 1 to 4 of FusedBatchNorm should be %s"); + + // check validity; + auto epsilon_value = primitive->GetAttr("epsilon"); + auto momentum_value = primitive->GetAttr("momentum"); + MS_EXCEPTION_IF_NULL(epsilon_value); + MS_EXCEPTION_IF_NULL(momentum_value); + if (!epsilon_value->isa() || !momentum_value->isa()) { + MS_LOG(EXCEPTION) << "expect epsilon and momentum be float, but: epsilon: " << epsilon_value->ToString() + << ", momentum: " << momentum_value->ToString(); + } + + auto epsilon = epsilon_value->cast()->value(); + auto momentum = momentum_value->cast()->value(); + + if (epsilon > 1.0f || epsilon <= 0.0f) { + MS_LOG(EXCEPTION) << "expect epsilon is greater than 0 and less or equal than 1, but epsilon: " << epsilon; + } + if (momentum > 1.0f || momentum < 0.0f) { + MS_LOG(EXCEPTION) << "expect momentum is great or equal than 0 and less or equal than 1, but epsilon: " << momentum; + } + + // Outputs: y, running_mean, running_variance, save_mean, save_inv_variance. + AbstractBasePtr y = input->Broaden(); + AbstractBasePtr other = args_spec_list[1]->Broaden(); + MS_LOG(DEBUG) << "output y: " << y->ToString() << ", other: " << other->ToString(); + + AbstractBasePtrList elements = {y, other, other, other, other}; + return std::make_shared(elements); +} + +AbstractBasePtr InferImplFusedBatchNormGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: five tensors(y_backprop, x, scale, save_mean, save_inv_variance). + MS_EXCEPTION_IF_NULL(args_spec_list[1]); + MS_EXCEPTION_IF_NULL(args_spec_list[2]); + MS_EXCEPTION_IF_NULL(args_spec_list[3]); + + CheckArgsSize(primitive->name(), args_spec_list, 5); + auto dx = args_spec_list[1]->Broaden(); + auto dscale = args_spec_list[2]->Broaden(); + auto dbias = args_spec_list[3]->Broaden(); + + AbstractBasePtrList rets = {dx, dscale, dbias}; + return std::make_shared(rets); +} + +AbstractBasePtr InferImplReluGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors(y_backprop, x). + CheckArgsSize(primitive->name(), args_spec_list, 2); + return args_spec_list[1]->Broaden(); +} + +AbstractBasePtr InferImplConv2DBackpropInput(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: three tensors(doutput, input, filters). + CheckArgsSize(primitive->name(), args_spec_list, 3); + return args_spec_list[1]->Broaden(); +} + +AbstractBasePtr InferImplConv2DBackpropFilter(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: three tensors(inputs, filter, doutput). + CheckArgsSize(primitive->name(), args_spec_list, 3); + return args_spec_list[2]->Broaden(); +} + +AbstractBasePtr InferImplBiasAddGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: at least one tensor(y_backprop) + // Outputs: dbias + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << "" << primitive->name() << " evaluator at least has 1 parameters, while the input size is " + << args_spec_list.size() << "."; + } + + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + ShapePtr shape_y = dyn_cast(args_spec_list[0]->GetShapeTrack()); + MS_EXCEPTION_IF_NULL(shape_y); + std::vector y_dims = shape_y->shape(); + if (y_dims.size() < 2) { + MS_LOG(EXCEPTION) << "" << primitive->name() << " input y backprop, dim should >= 2, while " << y_dims.size() + << "."; + } + std::vector bias_dims = {y_dims[1]}; + ShapePtr ret_shape = std::make_shared(bias_dims); + AbstractBasePtr ret = args_spec_list[0]->Broaden(); + ret->set_shape(ret_shape); + return ret; +} + +AbstractBasePtr InferImplRelu(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor. + CheckArgsSize(primitive->name(), args_spec_list, 1); + return args_spec_list[0]->Broaden(); +} + +AbstractBasePtr InferImplZerosLikeTensor(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor. + CheckArgsSize(primitive->name(), args_spec_list, 1); + return args_spec_list[0]->Broaden(); +} + +AbstractBasePtr InferImplFakeBprop(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor. + CheckArgsSize(primitive->name(), args_spec_list, 1); + return args_spec_list[0]->Broaden(); +} + +AbstractBasePtr InferImplLayerNorm(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: three tensors(x, gamma, beta). + // outputs: y, mean, variance + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + auto input_x = CheckArg(op_name, args_spec_list, 0); + auto input_shape = input_x->shape(); + auto const &input_shape_list = input_shape->shape(); + const size_t input_rank = input_shape_list.size(); + if (input_rank == 0) { + MS_LOG(EXCEPTION) << "input_rank should not be zero"; + } + + // begin_norm_axis and begin_params_axis should be smaller than the size of input_x and >= -1 + ValuePtr bna_ptr = primitive->GetAttr("begin_norm_axis"); + (void)CheckAxis(op_name, bna_ptr, -1, SizeToInt(input_rank) - 1); + + ValuePtr bpa_ptr = primitive->GetAttr("begin_params_axis"); + int begin_params_axis = CheckAxis(op_name, bpa_ptr, -1, SizeToInt(input_rank) - 1); + begin_params_axis = GetPositiveAxis(begin_params_axis, input_rank); + + // the beta and gama shape should be x_shape[begin_params_axis:] + auto tensor = CheckArg(op_name, args_spec_list, 0); + auto gamma = CheckArg(op_name, args_spec_list, 1); + auto beta = CheckArg(op_name, args_spec_list, 2); + (void)CheckTensorDType(tensor, {kFloat16, kFloat32}, "input 0 of LayerNorm should be %s"); + (void)CheckTensorDType(gamma, {kFloat16, kFloat32}, "input 1 of LayerNorm should be %s"); + (void)CheckTensorDType(beta, {kFloat16, kFloat32}, "input 2 of LayerNorm should be %s"); + auto gamma_shape = dyn_cast(gamma->BuildShape()); + auto beta_shape = dyn_cast(beta->BuildShape()); + MS_EXCEPTION_IF_NULL(gamma_shape); + MS_EXCEPTION_IF_NULL(beta_shape); + + auto const &gamma_shape_list = gamma_shape->shape(); + auto const &beta_shape_list = beta_shape->shape(); + if (gamma_shape_list.empty() || beta_shape_list.empty()) { + MS_LOG(EXCEPTION) << "LayerNorm evaluator gamma or beta is a AbstractScalar that is not support."; + } + + size_t begin_params_axis_u = IntToSize(begin_params_axis); + if ((begin_params_axis_u > input_shape_list.size()) || + (gamma_shape_list.size() + begin_params_axis_u < input_shape_list.size()) || + (beta_shape_list.size() + begin_params_axis_u < input_shape_list.size())) { + MS_LOG(EXCEPTION) << "Gamma and beta shape get wrong size."; + } + for (size_t i = begin_params_axis_u; i < input_shape_list.size(); ++i) { + size_t gamma_beta_shape_dim = i - begin_params_axis_u; + if ((gamma_shape_list[gamma_beta_shape_dim] != input_shape_list[i]) || + (beta_shape_list[gamma_beta_shape_dim] != input_shape_list[i])) { + MS_LOG(EXCEPTION) << "Gamma or beta shape not match input shape, input_shape=" << input_shape->ToString() + << ", gamma_shape=" << gamma_shape->ToString() << ", beta_shape=" << beta_shape->ToString(); + } + } + + auto mean_var_shape_value = input_shape->shape(); + mean_var_shape_value[input_rank - 1] = 1; + + auto mean = input_x->Broaden(); + mean->set_shape(std::make_shared(mean_var_shape_value)); + auto var = input_x->Broaden(); + var->set_shape(std::make_shared(mean_var_shape_value)); + + AbstractBasePtrList args_list({input_x->Broaden(), mean, var}); + return std::make_shared(args_list); +} + +AbstractBasePtr InferImplLayerNormGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: five tensors(y_backprob, x, variance, mean, gamma). + // Outputs: x_backprob, gamma_backprob, beta_backprob + CheckArgsSize(primitive->name(), args_spec_list, 5); + + auto x_backprob = args_spec_list[0]->Broaden(); + auto gamma_backprob = args_spec_list[4]->Broaden(); + auto beta_backprob = args_spec_list[4]->Broaden(); + + AbstractBasePtrList args_list({x_backprob, gamma_backprob, beta_backprob}); + return std::make_shared(args_list); +} + +AbstractBasePtr InferImplDropoutGenMask(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple and a tensor. + // Outputs: mask. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractTuplePtr x_shape = CheckArg(op_name, args_spec_list, 0); + AbstractTensorPtr keep_prob = CheckArg(op_name, args_spec_list, 1); + + TypePtr prob_type = keep_prob->element()->BuildType(); + if ((prob_type->type_id() != kNumberTypeFloat16) && (prob_type->type_id() != kNumberTypeFloat32)) { + MS_LOG(EXCEPTION) << "" << op_name << " keep_prob type should be float16 or float32, but " << prob_type->ToString() + << "."; + } + + auto x_shape_data = x_shape->elements(); + int count = 1; + for (std::size_t i = 0; i < x_shape->size(); ++i) { + auto value_track = x_shape_data[i]->GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + if (!value_track->isa()) { + MS_LOG(EXCEPTION) << "DropOutGenMask input x_shape elements is not int32, but " << value_track->ToString() << "."; + } + + int e_value = GetValue(value_track); + if (e_value <= 0) { + MS_LOG(EXCEPTION) << "DropOutGenMask product of x_shape should be > 0"; + } + if (std::numeric_limits::max() / count / e_value < 1) { + MS_LOG(EXCEPTION) << "integer multiply integer overflow"; + } + count = count * e_value; + } + + // convert to bytes(8 bits) mask, using round up + int bytes_count = (count + 7) / 8; + std::vector shape_y{bytes_count}; + + primitive->set_attr("T", kInt32); + return std::make_shared(std::make_shared(kAnyValue, kUInt8), + std::make_shared(std::vector{shape_y})); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_others.cc b/mindspore/ccsrc/operator/prim_others.cc new file mode 100644 index 00000000000..84144380f8f --- /dev/null +++ b/mindspore/ccsrc/operator/prim_others.cc @@ -0,0 +1,331 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/param_validator.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" +#include "utils/symbolic.h" + +namespace mindspore { +namespace abstract { +AbstractBasePtr InferImplIdentity(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // An object of a subclass of AbstractBase + CheckArgsSize(primitive->name(), args_spec_list, 1); + return args_spec_list[0]; +} + +AbstractBasePtr InferImplJ(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // args: An object of AbstractFunction. + CheckArgsSize(primitive->name(), args_spec_list, 1); + MS_LOG(DEBUG) << "evaluate J: " << args_spec_list[0]->ToString(); + + AbstractFunctionPtr x = dyn_cast(args_spec_list[0]); + if (x == nullptr) { + return std::make_shared(args_spec_list[0]); + } + + AbstractFuncAtomPtrList jv; + auto build_jv = [&jv](const AbstractFuncAtomPtr &func) { + auto j_closure = std::make_shared(func); + jv.push_back(j_closure); + }; + x->Visit(build_jv); + + return AbstractFunction::MakeAbstractFunction(jv); +} + +AbstractBasePtr InferImplEnvGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + MS_EXCEPTION_IF_NULL(primitive); + // args: Three objects of a subclass of AbstractBase, env, key, dflt(default). + CheckArgsSize(primitive->name(), args_spec_list, 3); + auto key = args_spec_list[1]; + auto dflt = args_spec_list[2]; + TypePtr type = key->GetTypeTrack(); + MS_EXCEPTION_IF_NULL(type); + if (type->type_id() != kObjectTypeSymbolicKeyType) { + MS_LOG(EXCEPTION) << "EnvGetItem evaluator args[1] should be a SymbolicKeyInstance but: " << key->ToString(); + } + if (!key->GetValueTrack()->isa()) { + return dflt; + } + ValuePtr key_value_ptr = key->GetValueTrack(); + MS_EXCEPTION_IF_NULL(key_value_ptr); + auto key_value_track = key_value_ptr->cast(); + auto expected = key_value_track->abstract(); + MS_EXCEPTION_IF_NULL(expected); + (void)expected->Join(dflt); + return expected; +} + +AbstractBasePtr InferImplEnvSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // args: Three objects of a subclass of AbstractBase, env, key, dflt(default). + CheckArgsSize(primitive->name(), args_spec_list, 3); + + auto key = args_spec_list[1]; + auto value = args_spec_list[2]; + + ValuePtr key_value_ptr = key->GetValueTrack(); + MS_EXCEPTION_IF_NULL(key_value_ptr); + auto key_value_track = key_value_ptr->cast(); + if (key_value_track == nullptr) { + MS_LOG(EXCEPTION) << "EnvGetItem evaluator args[1] expected should be able to cast to SymbolicKeyInstancePtrbut: " + << key_value_ptr->ToString(); + } + auto expected = key_value_track->abstract(); + MS_EXCEPTION_IF_NULL(expected); + (void)expected->Join(value); + return std::make_shared(kAnyValue, std::make_shared()); +} + +AbstractBasePtr InferImplEnvAdd(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // args: Three objects of a subclass of AbstractBase, env, key, dflt(default). + CheckArgsSize(primitive->name(), args_spec_list, 2); + return std::make_shared(kAnyValue, std::make_shared()); +} + +AbstractBasePtr InferImplMakeRefKey(const AnalysisEnginePtr &, const PrimitivePtr &prim, const AbstractBasePtrList &) { + ValuePtr name_value = prim->GetAttr("tag"); + auto name = name_value->cast(); + if (name == nullptr) { + MS_LOG(EXCEPTION) << "MakeRefKey attr tag sould be a String " << name_value->ToString() << "."; + } + auto refkey = std::make_shared(name->value()); + if (refkey == nullptr) { + MS_LOG(EXCEPTION) << "MakeRefKey std::make_shared failed"; + } + return refkey->ToAbstract(); +} + +AbstractBasePtr InferImplMakeRef(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // arguments: key, value, original value + if (args_spec_list.size() != 3) { + MS_LOG(EXCEPTION) << "make_ref evaluator requires 3 parameters, while the input size is " << args_spec_list.size() + << "."; + } + TypePtr type = args_spec_list[0]->GetTypeTrack(); + if (type->type_id() != kObjectTypeRefKey) { + MS_LOG(EXCEPTION) << "First input of make_ref should be a RefKey but a " << type->ToString(); + } + return std::make_shared(args_spec_list[0], args_spec_list[1], args_spec_list[2]); +} + +AbstractBasePtr InferImplGetRefKey(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // arguments: value + if (args_spec_list.size() != 1) { + MS_LOG(EXCEPTION) << "get_ref_key requires 1 parameters, while the input size is " << args_spec_list.size() << "."; + } + TypePtr type = args_spec_list[0]->GetTypeTrack(); + if (type->type_id() != kObjectTypeRef) { + MS_LOG(EXCEPTION) << "First input of get_ref_key should be a Ref but a " << type->ToString(); + } + return args_spec_list[0]->cast()->ref(); +} + +AbstractBasePtr InferImplGetRefValue(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // arguments: value + if (args_spec_list.size() != 1) { + MS_LOG(EXCEPTION) << "get_ref_value requires 1 parameters, while the input size is " << args_spec_list.size() + << "."; + } + TypePtr type = args_spec_list[0]->GetTypeTrack(); + if (type->type_id() != kObjectTypeRef) { + MS_LOG(EXCEPTION) << "First input of get_ref_value should be a Ref but a " << type->ToString(); + } + return args_spec_list[0]->cast()->ref(); +} + +AbstractBasePtr InferImplGetRefOrigin(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // arguments: value + if (args_spec_list.size() != 1) { + MS_LOG(EXCEPTION) << "get_ref_value requires 1 parameters, while the input size is " << args_spec_list.size() + << "."; + } + TypePtr type = args_spec_list[0]->GetTypeTrack(); + if (type->type_id() != kObjectTypeRef) { + MS_LOG(EXCEPTION) << "First input of get_ref_value should be a Ref but a " << type->ToString(); + } + return args_spec_list[0]->cast()->ref_origin(); +} + +AbstractBasePtr InferImplStateSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // args: Two objects of a subclass of AbstractBase, key and value. + CheckArgsSize(primitive->name(), args_spec_list, 2); + + TypePtr type = args_spec_list[0]->GetTypeTrack(); + MS_EXCEPTION_IF_NULL(type); + if (type->type_id() != kObjectTypeRefKey && type->type_id() != kObjectTypeSymbolicKeyType) { + MS_LOG(EXCEPTION) << "First input of StateSetItem should be a RefKey or SymbolicKeyType but a " << type->ToString(); + } + return std::make_shared(kAnyValue, kBool); +} + +AbstractBasePtr InferImplDepend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << primitive->name() << " input args size should be at lest 1, but got 0"; + } + auto depends = args_spec_list[0]->Broaden(); + return depends; +} + +bool CompareShape(const std::vector &x_shape, const std::vector &y_shape) { + if (x_shape.size() != y_shape.size()) { + return false; + } + + for (size_t i = 0; i < x_shape.size(); ++i) { + if (GetValue(x_shape[i]) != GetValue(y_shape[i])) { + return false; + } + } + + return true; +} + +enum State { + SAME, + X_ONE, + Y_ONE, +}; + +void ComputeReduceIndex(const std::vector &reverse_x, const std::vector &reverse_y, + std::vector *grad_x_reduce_idx, std::vector *grad_y_reduce_idy) { + const size_t n = reverse_x.size(); + for (size_t i = 0; i < n; ++i) { + State curr; + const int32_t x_i = reverse_x[i]; + const int32_t y_i = reverse_y[i]; + const int reduce_idx = SizeToInt(n - 1 - i); + if (x_i == y_i) { + curr = SAME; + } else if (x_i == 1) { + grad_x_reduce_idx->push_back(reduce_idx); + curr = X_ONE; + } else if (y_i == 1) { + grad_y_reduce_idy->push_back(reduce_idx); + curr = Y_ONE; + } else { + MS_LOG(EXCEPTION) << "not compatible shape input for BroadcastGradientArgs"; + } + if (curr == SAME && x_i == 1) { + grad_x_reduce_idx->push_back(reduce_idx); + grad_y_reduce_idy->push_back(reduce_idx); + continue; + } + } + + std::reverse(grad_x_reduce_idx->begin(), grad_x_reduce_idx->end()); + std::reverse(grad_y_reduce_idy->begin(), grad_y_reduce_idy->end()); +} + +AbstractBasePtr BroadcastGradientArgsDiff(const std::vector &x_shape, const std::vector &y_shape) { + std::vector reverse_x; + std::vector reverse_y; + + (void)std::transform(x_shape.rbegin(), x_shape.rend(), std::back_inserter(reverse_x), + [](const ValuePtr &v) { return v->cast()->value(); }); + (void)std::transform(y_shape.rbegin(), y_shape.rend(), std::back_inserter(reverse_y), + [](const ValuePtr &v) { return v->cast()->value(); }); + + if (reverse_x.size() > reverse_y.size()) { + reverse_y.resize(reverse_x.size(), 1); + } else { + reverse_x.resize(reverse_y.size(), 1); + } + + std::vector grad_x_reduce_idx; + std::vector grad_y_reduce_idy; + ComputeReduceIndex(reverse_x, reverse_y, &grad_x_reduce_idx, &grad_y_reduce_idy); + + AbstractBasePtrList abs_list_x; + AbstractBasePtrList abs_list_y; + (void)std::transform(grad_x_reduce_idx.begin(), grad_x_reduce_idx.end(), std::back_inserter(abs_list_x), + [](int v) { return abstract::FromValue(v); }); + (void)std::transform(grad_y_reduce_idy.begin(), grad_y_reduce_idy.end(), std::back_inserter(abs_list_y), + [](int v) { return abstract::FromValue(v); }); + auto x_reduce_idx = std::make_shared(abs_list_x); + auto y_reduce_idx = std::make_shared(abs_list_y); + AbstractBasePtrList elem_list; + elem_list.push_back(x_reduce_idx); + elem_list.push_back(y_reduce_idx); + + return std::make_shared(elem_list); +} + +AbstractBasePtr InferImplBroadcastGradientArgs(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // this primitive get the index that need to reduce + // input: x's shape and y's shape, inputs should be tuple + // output: tuple of x and y 's reduce index, reduce index should be a tuple + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + auto arg_x = CheckArg(op_name, args_spec_list, 0); + auto arg_y = CheckArg(op_name, args_spec_list, 1); + + ValueTuplePtr arg_x_value = arg_x->BuildValue()->cast(); + MS_EXCEPTION_IF_NULL(arg_x_value); + + ValueTuplePtr arg_y_value = arg_y->BuildValue()->cast(); + MS_EXCEPTION_IF_NULL(arg_y_value); + + const std::vector x_shape = arg_x_value->value(); + const std::vector y_shape = arg_y_value->value(); + bool is_same_shape = CompareShape(x_shape, y_shape); + // if it is the same shape , do not need reduce , return empty tuple + if (is_same_shape) { + AbstractBasePtrList empty_list; + auto x_reduce_idx = std::make_shared(empty_list); + auto y_reduce_idx = std::make_shared(empty_list); + + AbstractBasePtrList elem_list; + elem_list.push_back(x_reduce_idx); + elem_list.push_back(y_reduce_idx); + + return std::make_shared(elem_list); + } + + return BroadcastGradientArgsDiff(x_shape, y_shape); +} + +AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // args: Two objects of a subclass of AbstractBase + CheckArgsSize(primitive->name(), args_spec_list, 2); + auto arg_src = args_spec_list[0]; + auto arg_dst = args_spec_list[1]; + // control depend can not setup tuple of ops to tuple of ops dependency relation + if (arg_src->isa() && arg_dst->isa()) { + auto src_size = arg_src->cast()->size(); + auto dst_size = arg_src->cast()->size(); + if (src_size > 1 && dst_size > 1) { + MS_LOG(EXCEPTION) << "Control depend can not setup operator dependcy relationship from tuple from tuple"; + } + } + return std::make_shared(kAnyValue, kBool); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_statement.cc b/mindspore/ccsrc/operator/prim_statement.cc new file mode 100644 index 00000000000..7d5038d4e13 --- /dev/null +++ b/mindspore/ccsrc/operator/prim_statement.cc @@ -0,0 +1,173 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/param_validator.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" +#include "utils/symbolic.h" + +namespace mindspore { +namespace abstract { +AbstractBasePtr InferImplReturn(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a pointer to an AbstractBase object + if (args_spec_list.size() != 1) { + MS_LOG(INFO) << "Return evaluator requires 1 parameter, is this the default value attached? " + "while the input size is " + << args_spec_list.size() << "."; + } + AbstractBasePtr abs_base = args_spec_list[0]; + return abs_base; +} + +AbstractBasePtr InferImplTypeof(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a pointer to an AbstractBase object + if (args_spec_list.size() != 1) { + MS_LOG(EXCEPTION) << "Typeof evaluator requires 1 parameter, while the input size is " << args_spec_list.size() + << "."; + } + AbstractBasePtr abs_base = args_spec_list[0]; + MS_EXCEPTION_IF_NULL(abs_base); + TypePtr type = abs_base->BuildType(); + return std::make_shared(type); +} + +AbstractBasePtr InferImplHasType(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a pointer to an AbstractBase object and a pointer to a Type + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractTypePtr abs_type = CheckArg(op_name, args_spec_list, 1); + + auto mode_v = abs_type->GetValueTrack(); + MS_EXCEPTION_IF_NULL(mode_v); + if (!mode_v->isa()) { + MS_LOG(EXCEPTION) << "Get the type from AbstractType value failed."; + } + + TypePtr mode_t = mode_v->cast(); + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + bool v = IsSubtype(args_spec_list[0], mode_t); + return std::make_shared(std::make_shared(v), kBool); +} + +AbstractBasePtr InferImplDot(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractTensorPtr input_x = CheckArg(op_name, args_spec_list, 0); + AbstractTensorPtr input_y = CheckArg(op_name, args_spec_list, 1); + + ShapePtr x_shp = input_x->shape(); + auto x_shp_value = x_shp->shape(); + ShapePtr y_shp = input_y->shape(); + auto y_shp_value = y_shp->shape(); + // Should be matrix which shape size is 2. + if (x_shp_value.size() != 2 || y_shp_value.size() != 2) { + MS_LOG(EXCEPTION) << "" << op_name + << " evaluator requires input two 2D tensors, while the dimensions of two tensors are " + << x_shp_value.size() << ", " << y_shp_value.size() << " "; + } + if (x_shp_value[1] != y_shp_value[0] && x_shp_value[1] != Shape::SHP_ANY && y_shp_value[0] != Shape::SHP_ANY) { + MS_LOG(EXCEPTION) << "Incompatible shapes in dot: {" << x_shp->ToString() << "} and {" << y_shp->ToString() << "}"; + } + + auto x_element = input_x->element(); + MS_EXCEPTION_IF_NULL(x_element); + (void)x_element->Join(input_y->element()); + auto param = {x_shp_value[0], y_shp_value[1]}; + + return std::make_shared(input_x->element(), std::make_shared(param)); +} + +AbstractBasePtr InferImplSwitch(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + // Inputs: condition, true branch, false branch + if (args_spec_list.size() != 3) { + MS_LOG(EXCEPTION) << "Switch evaluator requires 3 parameters, while the input size is " << args_spec_list.size() + << "."; + } + + auto cond = args_spec_list[0]; + auto tb = args_spec_list[1]; + auto fb = args_spec_list[2]; + MS_EXCEPTION_IF_NULL(cond); + + ValuePtr v = cond->GetValueTrack(); + MS_EXCEPTION_IF_NULL(v); + if (v->isa()) { + MS_EXCEPTION_IF_NULL(tb); + return tb->Join(fb); + } + + if (v->isa()) { + if (v->cast()->IsOne()) { + return tb; + } else { + return fb; + } + } + + MS_LOG(EXCEPTION) << "Invalid condition value for switch " << cond->ToString(); +} + +std::vector GetSupportedTargetValue() { + std::vector list = {kNone, MakeValue(false), MakeValue(true)}; + return list; +} + +bool SupportedIsTargetValue(const ValuePtr t) { + auto list = GetSupportedTargetValue(); + auto match = std::any_of(list.begin(), list.end(), [&t](const ValuePtr &v) { return *v == *t; }); + return match; +} + +AbstractBasePtr InferImplIs_(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // statement: x is t + // Inputs: x, t + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + ValuePtr t = args_spec_list[1]->BuildValue(); + if (!SupportedIsTargetValue(t)) { + MS_LOG(EXCEPTION) << "Not supported type:" << t->ToString() + << " for statement is, supported list is:None, False, True "; + } + ValuePtr x = args_spec_list[0]->BuildValue(); + + return std::make_shared(*t == *x); +} + +AbstractBasePtr InferImplIsNot(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // statement: x is not t + // Inputs: x, t + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + ValuePtr t = args_spec_list[1]->BuildValue(); + if (!SupportedIsTargetValue(t)) { + MS_LOG(EXCEPTION) << "Not supported type:" << t->ToString() + << " for statement is not, supported list is:None, False, True "; + } + ValuePtr x = args_spec_list[0]->BuildValue(); + + return std::make_shared(!(*t == *x)); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_structures.cc b/mindspore/ccsrc/operator/prim_structures.cc new file mode 100644 index 00000000000..88699c4d382 --- /dev/null +++ b/mindspore/ccsrc/operator/prim_structures.cc @@ -0,0 +1,684 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/prim.h" +#include "pipeline/static_analysis/utils.h" +#include "pipeline/static_analysis/param_validator.h" +#include "operator/ops.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace abstract { + +AbstractBasePtr InferImplStringEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two scalars whose value is a string. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractScalarPtr scalar_x = CheckArg(op_name, args_spec_list, 0); + AbstractScalarPtr scalar_y = CheckArg(op_name, args_spec_list, 1); + + ValuePtr value_x = scalar_x->BuildValue(); + ValuePtr value_y = scalar_y->BuildValue(); + if (!value_x->isa() || !value_y->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " requires 2 parameters are string, but got param0: " << value_x->ToString() + << ", param1: " << value_y->ToString(); + } + + bool ret = (value_x->cast()->value() == value_y->cast()->value()); + return std::make_shared(ret); +} + +AbstractBasePtr InferImplMakeTuple(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + return std::make_shared(args_spec_list); +} + +AbstractBasePtr InferImplMakeList(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + return std::make_shared(args_spec_list); +} + +AbstractBasePtr InferImplMakeDict(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tuples. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractTuplePtr keys = CheckArg(op_name, args_spec_list, 0); + AbstractTuplePtr values = CheckArg(op_name, args_spec_list, 1); + + size_t keys_size = keys->size(); + if (values->size() != keys_size) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator keys' size is not equal with values' size"; + } + + std::vector key_value; + AbstractScalarPtr key; + AbstractBasePtrList key_list = keys->elements(); + AbstractBasePtrList value_list = values->elements(); + for (size_t index = 0; index < keys_size; index++) { + key = CheckArg(op_name + "key", key_list, index); + ValuePtr keyPtr = key->BuildValue(); + MS_EXCEPTION_IF_NULL(keyPtr); + if (!keyPtr->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator keys should be string, but got " << keyPtr->ToString(); + } + std::string key_string = GetValue(keyPtr); + key_value.emplace_back(key_string, value_list[index]); + } + return std::make_shared(key_value); +} + +AbstractBasePtr InferImplMakeKwarg(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a string and an object of a subclass of AbstractBase. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 0); + + ValuePtr keyPtr = key->BuildValue(); + if (!keyPtr->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator key should be string, but got " << keyPtr->ToString(); + } + std::string key_string = GetValue(keyPtr); + return std::make_shared(key_string, args_spec_list[1]); +} + +AbstractBasePtr InferImplExtractKwarg(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a string and a keyword. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 0); + AbstractKeywordArgPtr kwarg = CheckArg(op_name, args_spec_list, 1); + + ValuePtr key_value = key->BuildValue(); + if (!key_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator key should be string, but got " << key_value->ToString(); + } + std::string key_input = GetValue(key_value); + std::string key_actual = kwarg->get_key(); + if (key_actual != key_input) { + MS_LOG(EXCEPTION) << "" << op_name + << " evaluator input key should be same as AbstractKeywordArg' key, but input is " << key_input + << ", AbstractKeywordArg' key is " << key_actual; + } + return kwarg->get_arg(); +} + +AbstractBasePtr InferImplMakeSlice(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: three scalars whose value is an int32 number. + CheckArgsSize(primitive->name(), args_spec_list, 3); + size_t args_size = args_spec_list.size(); + for (size_t index = 0; index < args_size; index++) { + MS_EXCEPTION_IF_NULL(args_spec_list[index]); + if (!args_spec_list[index]->isa() && !args_spec_list[index]->isa()) { + MS_LOG(EXCEPTION) << "MakeSlice eval " << index << " parameter is neither AbstractScalar nor AbstractNone."; + } + if (args_spec_list[index]->isa() && + !dyn_cast(args_spec_list[index])->BuildValue()->isa()) { + MS_LOG(EXCEPTION) << "MakeSlice eval " << index << " parameter is an AbstractScalar, but is not an int32 number."; + } + } + // Slice: start, end, step + return std::make_shared(args_spec_list[0], args_spec_list[1], args_spec_list[2]); +} + +// Eval the return type of make_record +AbstractBasePtr InferImplMakeRecord(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: at lease two objects of a subclass of AbstractBase. + if (args_spec_list.size() < 2) { + MS_LOG(EXCEPTION) << "Typeof evaluator requires more than 1 parameter, while the input size is " + << args_spec_list.size() << "."; + } + + // args_spec_list[0] maybe AbstractScalarPtr or AbstractTypePtr + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + TypePtr type = args_spec_list[0]->GetTypeTrack(); + MS_EXCEPTION_IF_NULL(type); + if (type->type_id() != kMetaTypeTypeType) { + MS_LOG(EXCEPTION) << "Can not make type(" << type->ToString() << ")not TypeType"; + } + + ValuePtr value_track = args_spec_list[0]->GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + TypePtr type_ptr = value_track->cast(); + if (type_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Value type error, not Me type:" << value_track->ToString(); + } + + auto cls = dyn_cast(type_ptr); + MS_EXCEPTION_IF_NULL(cls); + ClassAttrVector attributes = cls->GetAttributes(); + CheckArgsSize(primitive->name(), args_spec_list, attributes.size() + 1); + + std::vector abs_attributes; + for (size_t i = 0; i < attributes.size(); i++) { + AbstractAttribute elem(attributes[i].first, args_spec_list[i + 1]); + abs_attributes.push_back(elem); + } + + return std::make_shared(cls->tag(), abs_attributes, cls->methods()); +} + +template +AbstractBasePtr InferTupleOrListGetItem(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple or list and a scalar whose value is an int32 number. + CheckArgsSize(op_name, args_spec_list, 2); + auto queue = CheckArg(op_name, args_spec_list, 0); + AbstractScalarPtr index = CheckArg(op_name, args_spec_list, 1); + + ValuePtr index_value = index->BuildValue(); + if (!index_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator index should be an int32 number, but got " + << index_value->ToString(); + } + int idx_v = GetValue(index_value); + std::size_t nelems = queue->elements().size(); + if (idx_v >= SizeToInt(nelems) || idx_v < -SizeToInt(nelems)) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator index should be in range[-" << SizeToInt(nelems) << ", " + << SizeToInt(nelems) << "), but got " << idx_v << "."; + } + + std::size_t uidx_v = 0; + if (idx_v >= 0) { + uidx_v = IntToSize(idx_v); + } else { + uidx_v = IntToSize(idx_v + SizeToInt(nelems)); + } + return queue->elements()[uidx_v]; +} + +template +AbstractBasePtr InferTupleOrListSetItem(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple or list, a scalar whose value is an int32 number and an object of a subclass of AbstractBase. + CheckArgsSize(op_name, args_spec_list, 3); + auto queue = CheckArg(op_name, args_spec_list, 0); + AbstractScalarPtr index = CheckArg(op_name, args_spec_list, 1); + + ValuePtr index_value = index->BuildValue(); + if (!index_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator index should be an int32 number, but got " + << index_value->ToString(); + } + int idx_v = GetValue(index_value); + if (idx_v < 0) { + MS_LOG(EXCEPTION) << "The index of " << typeid(T).name() << " should be positive number, but got " << idx_v << "."; + } + + size_t uidx_v = IntToSize(idx_v); + AbstractBasePtrList elements = queue->elements(); + std::size_t nelems = elements.size(); + if (uidx_v >= nelems) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator the index: " << uidx_v << " to set out of range: " << nelems - 1 + << "."; + } + elements[uidx_v] = args_spec_list[2]; + return std::make_shared(elements); +} + +AbstractBasePtr InferImplTupleGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListGetItem(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplListGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListGetItem(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplTupleSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListSetItem(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplListSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListSetItem(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplDictGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a dict and a scalar whose value is a string. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractDictionaryPtr dict = CheckArg(op_name, args_spec_list, 0); + AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 1); + + ValuePtr key_value = key->BuildValue(); + if (!key_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator key should be string, but got " << key_value->ToString(); + } + std::string key_str = GetValue(key_value); + std::vector dict_elems = dict->elements(); + auto it = std::find_if(dict_elems.begin(), dict_elems.end(), + [key_str](AbstractAttribute &item) { return item.first == key_str; }); + + if (it == dict_elems.end()) { + MS_LOG(EXCEPTION) << "The key " << key_str << " does not exist in the dict:" << args_spec_list[0]->ToString(); + } + return it->second; +} + +AbstractBasePtr InferImplDictSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a dict and a scalar whose value is a string and an object of a subclass of AbstractBase. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + AbstractDictionaryPtr dict = CheckArg(op_name, args_spec_list, 0); + AbstractScalarPtr key = CheckArg(op_name, args_spec_list, 1); + + ValuePtr key_value = key->BuildValue(); + if (!key_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator key should be string, but got " << key_value->ToString(); + } + std::string key_str = GetValue(key_value); + std::vector dict_elems = dict->elements(); + auto it = std::find_if(dict_elems.begin(), dict_elems.end(), + [key_str](AbstractAttribute &item) { return item.first == key_str; }); + + MS_EXCEPTION_IF_NULL(args_spec_list[2]); + auto new_ele = std::make_pair(key_str, args_spec_list[2]); + if (it != dict_elems.end()) { + int index = it - dict_elems.begin(); + dict_elems[IntToSize(index)] = new_ele; + } else { + dict_elems.push_back(new_ele); + } + return std::make_shared(dict_elems); +} + +AbstractBasePtr InferImplListAppend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a list and an object of a subclass of AbstractBase. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractListPtr list = CheckArg(op_name, args_spec_list, 0); + (void)AbstractJoin(list->elements()); + return list; +} + +template +AbstractBasePtr InferTupleOrListOrDictLen(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple or list or dict. + CheckArgsSize(op_name, args_spec_list, 1); + auto arg = CheckArg(op_name, args_spec_list, 0); + return std::make_shared(SizeToInt(arg->size())); +} + +AbstractBasePtr InferImplTupleLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListOrDictLen(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplListLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListOrDictLen(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplDictLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferTupleOrListOrDictLen(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplArrayLen(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + return std::make_shared(kAnyValue, kInt32); +} + +AbstractBasePtr InferImplListMap(const AnalysisEnginePtr &engine, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: fn, list1, list2, ... + MS_EXCEPTION_IF_NULL(engine); + if (args_spec_list.size() <= 1) { + MS_LOG(EXCEPTION) << "List_map requires at least 1 list. while the input size is " << args_spec_list.size() << "."; + } + AbstractFunctionPtr fn = CheckArg(primitive->name(), args_spec_list, 0); + // check args from 1. + CheckArgsSpec(AbstractBasePtrList(args_spec_list.begin() + 1, args_spec_list.end())); + + AbstractBasePtrList subargs; + for (std::size_t i = 1; i < args_spec_list.size(); i++) { + AbstractListPtr l_ptr = dyn_cast(args_spec_list[i]); + if (l_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Argument[" << i << "] of list_map should be a list."; + } + subargs.push_back(AbstractJoin(l_ptr->elements())); + } + AbstractBasePtr engin_exc = engine->Execute(fn, subargs); + AbstractBasePtrList result; + for (std::size_t i = 1; i < args_spec_list.size(); i++) { + result.push_back(engin_exc); + } + return std::make_shared(result); +} + +AbstractBasePtr InferImplListReduce(const AnalysisEnginePtr &engine, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a fn, a list and an object of a subclass of a AbstractBase. + MS_EXCEPTION_IF_NULL(engine); + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + AbstractFunctionPtr fn = CheckArg(op_name, args_spec_list, 0); + AbstractListPtr lst = CheckArg(op_name, args_spec_list, 1); + AbstractBasePtr dflt = args_spec_list[2]; + + AbstractBasePtr list_type = AbstractJoin(lst->elements()); + auto result1 = engine->Execute(fn, lst->elements()); + auto result2 = engine->Execute(fn, {dflt, list_type}); + MS_EXCEPTION_IF_NULL(result1); + return result1->Join(result2); +} + +AbstractBasePtr InferImplTupleReversed(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + AbstractTuplePtr input = CheckArg(op_name, args_spec_list, 0); + + auto tuple_elements = input->elements(); + AbstractBasePtrList elem_list; + (void)std::transform(tuple_elements.rbegin(), tuple_elements.rend(), std::back_inserter(elem_list), + [](const AbstractBasePtr &elem) { return elem->Clone(); }); + return std::make_shared(elem_list); +} + +AbstractBasePtr DoInferReduceShape(const AbstractTuplePtr &x_shape, const ValuePtr &x_shp_value, + const ValueTuplePtr &axis_value_ptr, const PrimitivePtr &primitive) { + size_t x_rank = x_shape->size(); + std::set axis_set; + auto axis_data = axis_value_ptr->value(); + if (axis_data.empty()) { + int size = 1; + AbstractBasePtrList values(x_rank, std::make_shared(size)); + return std::make_shared(values); + } + + for (auto &elem : axis_data) { + int e_value = CheckAxis(primitive->name(), elem, -SizeToInt(x_rank), SizeToInt(x_rank) - 1); + (void)axis_set.insert(e_value); + } + + auto x_shp_data = x_shp_value->cast()->value(); + if (x_shp_data.size() < x_rank) { + MS_LOG(EXCEPTION) << "x_shape_data.size() " << x_shp_data.size() << " less than x_shape.size() " << x_rank; + } + AbstractBasePtrList values; + for (size_t i = 0; i < x_rank; i++) { + if (axis_set.count(SizeToInt(i)) || axis_set.count(SizeToInt(i) - SizeToInt(x_rank))) { + auto axis_v = MakeValue(1); + values.push_back(std::make_shared(axis_v, axis_v->type())); + } else { + int dim_value = x_shp_data[i]->cast()->value(); + auto dim = MakeValue(dim_value); + values.push_back(std::make_shared(dim, dim->type())); + } + } + + return std::make_shared(values); +} + +AbstractBasePtr InferImplReduceShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: x_shape, axis + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractTuplePtr shape_x = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(args_spec_list[1]); + + auto x_shp_value = shape_x->BuildValue(); + if (x_shp_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name + << " evaluator shape's data field can't be anything: " << args_spec_list[1]->ToString(); + } + + // Axis can be scalar, tuple or None + AbstractTuplePtr axis = nullptr; + if (args_spec_list[1]->isa()) { + MS_LOG(DEBUG) << "" << op_name << " evaluator second parameter is scalar"; + AbstractBasePtrList axis_list = {dyn_cast(args_spec_list[1])}; + axis = std::make_shared(axis_list); + } else if (args_spec_list[1]->isa()) { + MS_LOG(DEBUG) << "" << op_name << " evaluator second parameter is tuple"; + axis = args_spec_list[1]->cast(); + } else { + MS_LOG(EXCEPTION) << "" << op_name << " evaluator second parameter should be a scalar or tuple, but got " + << args_spec_list[1]->ToString(); + } + + auto axis_value = axis->BuildValue(); + if (axis_value->isa()) { + MS_LOG(EXCEPTION) << "" << op_name + << " evaluator shape's data field can't be anything: " << args_spec_list[1]->ToString(); + } + auto axis_value_ptr = axis_value->cast(); + MS_EXCEPTION_IF_NULL(axis_value_ptr); + + return DoInferReduceShape(shape_x, x_shp_value, axis_value_ptr, primitive); +} + +AbstractBasePtr InferImplTupleDiv(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tuples. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 2); + AbstractTuplePtr shape_x = CheckArg(op_name, args_spec_list, 0); + AbstractTuplePtr div_shp = CheckArg(op_name, args_spec_list, 1); + MS_LOG(INFO) << "DivShape input:" << shape_x->ToString() << ", div:" << div_shp->ToString(); + + auto div_shp_value = div_shp->BuildValue(); + if (div_shp_value->isa()) { + MS_LOG(EXCEPTION) << "shape's data field can't be anythin: " << args_spec_list[0]->ToString(); + } + + auto shpx_value = shape_x->BuildValue(); + if (shpx_value->isa()) { + MS_LOG(EXCEPTION) << "shape's data field can't be anythin: " << args_spec_list[1]->ToString(); + } + + if (div_shp->size() != shape_x->size()) { + MS_LOG(EXCEPTION) << "tileshape elems shape must the same div_shp: " << div_shp->size() + << ", shapex: " << shape_x->size() << "."; + } + + auto shpx_data = shpx_value->cast()->value(); + auto div_shp_data = div_shp_value->cast()->value(); + AbstractBasePtrList values; + + for (size_t i = 0; i < div_shp_data.size(); i++) { + if (div_shp_data[i]->cast() == nullptr) { + MS_LOG(EXCEPTION) << "div_shp_shape data should be an int32 number, but it's " << args_spec_list[1]->ToString(); + } + int shapex_value = GetValue(shpx_data[i]); + int div_value = GetValue(div_shp_data[i]); + MS_LOG(DEBUG) << "div_shp_shape data shapex_value :" << shapex_value << " div_value: " << div_value; + if (div_value == 0) { + MS_LOG(EXCEPTION) << "error: division value should not be 0!"; + } + if ((shapex_value % div_value) != 0) { + MS_LOG(EXCEPTION) << "div_shp_shape data shapex must div int:" << shapex_value << " div_value: " << div_value; + } + + int result = shapex_value / div_value; + auto result_v = MakeValue(result); + values.push_back(std::make_shared(result_v, result_v->type())); + } + + return std::make_shared(values); +} + +AbstractBasePtr InferImplTuple2Array(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + AbstractTuplePtr input = CheckArg(op_name, args_spec_list, 0); + + py::tuple data_tuple = ValuePtrToPyData(input->BuildValue()); + py::array data = py::array(data_tuple); + auto tensor = std::make_shared(data); + auto ret = tensor->ToAbstract(); + ret->set_value(tensor); + MS_LOG(DEBUG) << "Tuple2arry result AbstractTensor: " << ret->ToString(); + return ret; +} + +AbstractBasePtr InferImplShapeMul(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tuple + // example: tuple = (1, 2, 3), shape_mul(tuple) = 1*2*3 = 6 + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + AbstractTuplePtr shape_x = CheckArg(op_name, args_spec_list, 0); + + auto shpx_value = shape_x->BuildValue(); + if (shpx_value->isa()) { + MS_LOG(EXCEPTION) << "shape's data field can't be anythin: " << shape_x->ToString(); + } + + auto shpx_data = shpx_value->cast()->value(); + + int result = 1; + for (size_t i = 0; i < shpx_data.size(); i++) { + int value = GetValue(shpx_data[i]); + IntMulWithOverflowCheck(result, value, &result); + } + + auto result_v = MakeValue(result); + MS_LOG(DEBUG) << "shape mul result:" << result_v->ToString(); + return std::make_shared(result_v, result_v->type()); +} + +template +AbstractBasePtr InferImplTupleOrListEqual(const std::string &op_name, const AbstractBasePtrList &args_spec_list) { + // Inputs: two tuples or two lists. + CheckArgsSize(op_name, args_spec_list, 2); + auto input_x = CheckArg(op_name, args_spec_list, 0); + auto input_y = CheckArg(op_name, args_spec_list, 1); + + ValuePtr x_value = input_x->BuildValue(); + ValuePtr y_value = input_y->BuildValue(); + return std::make_shared(*x_value == *y_value); +} + +AbstractBasePtr InferImplTupleEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferImplTupleOrListEqual(primitive->name(), args_spec_list); +} + +AbstractBasePtr InferImplListEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + return InferImplTupleOrListEqual(primitive->name(), args_spec_list); +} + +struct SlideInfo { + int start; + int step; + int stop; +}; + +void CalcSlidePara(const AbstractBasePtrList &args_spec_list, SlideInfo *slide) { + int arg1 = 0; + int arg2 = 0; + if (!args_spec_list.empty()) { + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + auto arg_value = args_spec_list[0]->BuildValue(); + if (!arg_value->isa()) { + MS_LOG(EXCEPTION) << "Only supported input an int32 number."; + } + arg1 = GetValue(arg_value); + } + + if (args_spec_list.size() >= 2) { + MS_EXCEPTION_IF_NULL(args_spec_list[1]); + auto arg_value = args_spec_list[1]->BuildValue(); + if (!arg_value->isa()) { + MS_LOG(EXCEPTION) << "Only supported input an int32 number."; + } + arg2 = GetValue(arg_value); + } + + if (args_spec_list.size() == 3) { + MS_EXCEPTION_IF_NULL(args_spec_list[2]); + auto arg_value = args_spec_list[2]->BuildValue(); + if (!arg_value->isa()) { + MS_LOG(EXCEPTION) << "Only supported input an int32 number."; + } + slide->step = GetValue(arg_value); + slide->start = arg1; + slide->stop = arg2; + } + + if (args_spec_list.size() == 2) { + slide->start = arg1; + slide->stop = arg2; + } + + if (args_spec_list.size() == 1) { + slide->stop = arg1; + } +} + +AbstractBasePtr InferImplMakeRange(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << "Cannot make range from empty input."; + } + + if (args_spec_list.size() > 3) { + MS_LOG(EXCEPTION) << "Error args size of make range operational."; + } + + SlideInfo slide = {0, 1, 0}; + CalcSlidePara(args_spec_list, &slide); + + if (slide.step == 0) { + MS_LOG(EXCEPTION) << "Error, step value is 0."; + } + + AbstractBasePtrList args; + if (slide.start <= slide.stop) { + if (slide.step <= 0) { + MS_LOG(EXCEPTION) << "Error slice[" << slide.start << ", " << slide.stop << ", " << slide.step << "]"; + } + for (int i = slide.start; i < slide.stop; i += slide.step) { + args.push_back(abstract::FromValue(i)); + } + } else { + if (slide.step >= 0) { + MS_LOG(EXCEPTION) << "Error slice[" << slide.start << ", " << slide.stop << ", " << slide.step << "]"; + } + for (int i = slide.start; i > slide.stop; i += slide.step) { + args.push_back(abstract::FromValue(i)); + } + } + + return std::make_shared(args); +} + +AbstractBasePtr InferImplStopGradient(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor + CheckArgsSize(primitive->name(), args_spec_list, 1); + return args_spec_list[0]->Clone(); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_to_function.cc b/mindspore/ccsrc/operator/prim_to_function.cc new file mode 100644 index 00000000000..234c829d447 --- /dev/null +++ b/mindspore/ccsrc/operator/prim_to_function.cc @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator/prim_to_function.h" +#include +#include +#include + +namespace mindspore { +// namespace to support prim related definition +namespace prim { + +PrimToFunction::PrimToFunction() + : prim_func_type_map_({ + // ONE_ARG prim + {"bool_not", kPrimTypeOneArg}, + {"scalar_cos", kPrimTypeOneArg}, + {"scalar_exp", kPrimTypeOneArg}, + {"scalar_floor", kPrimTypeOneArg}, + {"scalar_log", kPrimTypeOneArg}, + {"scalar_sin", kPrimTypeOneArg}, + {"scalar_tan", kPrimTypeOneArg}, + {"scalar_trunc", kPrimTypeOneArg}, + {"typeof", kPrimTypeOneArg}, + {"scalar_uadd", kPrimTypeOneArg}, + {"scalar_usub", kPrimTypeOneArg}, + // TWO_ARGS prim + {"scalar_add", kPrimTypeTwoArgs}, + {"bool_and", kPrimTypeTwoArgs}, + {"bool_eq", kPrimTypeTwoArgs}, + {"bool_or", kPrimTypeTwoArgs}, + {"scalar_div", kPrimTypeTwoArgs}, + {"scalar_eq", kPrimTypeTwoArgs}, + {"scalar_ge", kPrimTypeTwoArgs}, + {"scalar_gt", kPrimTypeTwoArgs}, + {"scalar_le", kPrimTypeTwoArgs}, + {"scalar_lt", kPrimTypeTwoArgs}, + {"scalar_ne", kPrimTypeTwoArgs}, + {"scalar_mod", kPrimTypeTwoArgs}, + {"scalar_mul", kPrimTypeTwoArgs}, + {"scalar_pow", kPrimTypeTwoArgs}, + {"scalar_sub", kPrimTypeTwoArgs}, + }) {} + +bool PrimToFunction::GetFunction(const PrimitivePtr& prim, FunctionPtr* const func) const { + bool result = false; + + if (func != nullptr) { + int args_num = GetPrimType(prim); + std::vector one_arg{std::make_shared()}; + std::vector two_args{std::make_shared(), std::make_shared()}; + TypePtr retval = std::make_shared(); + result = true; + switch (args_num) { + case kPrimTypeOneArg: + *func = Function(one_arg, retval).DeepCopy()->cast(); + break; + case kPrimTypeTwoArgs: + *func = Function(two_args, retval).DeepCopy()->cast(); + break; + default: + result = false; + break; + } + } + + return result; +} + +int PrimToFunction::GetPrimType(const PrimitivePtr& prim) const { + MS_EXCEPTION_IF_NULL(prim); + int prim_type = static_cast(kPrimTypeUnknown); + + auto value = prim_func_type_map_.find(prim->name()); + if (value != prim_func_type_map_.end()) { + prim_type = value->second; + } + return prim_type; +} +} // namespace prim +} // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_to_function.h b/mindspore/ccsrc/operator/prim_to_function.h new file mode 100644 index 00000000000..71518e4057b --- /dev/null +++ b/mindspore/ccsrc/operator/prim_to_function.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPERATOR_PRIM_TO_FUNCTION_H_ +#define MINDSPORE_CCSRC_OPERATOR_PRIM_TO_FUNCTION_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/primitive.h" +#include "ir/dtype.h" + +namespace mindspore { +/* namespace to support prim related definition */ +namespace prim { +// Supported meta type +enum PrimType { kPrimTypeUnknown, kPrimTypeOneArg, kPrimTypeTwoArgs }; + +class PrimToFunction; + +// Get the args, return value and function handle for a primitive instance. +class PrimToFunction { + public: + // Return a thread-safe singleton instance + static PrimToFunction& GetInstance() { + static PrimToFunction instance; + return instance; + } + PrimToFunction(const PrimToFunction&) = delete; + PrimToFunction& operator=(const PrimToFunction&) = delete; + ~PrimToFunction() = default; + + // Get the args and return value for a primitive instance. + bool GetFunction(const PrimitivePtr& prim, FunctionPtr* func) const; + + private: + PrimToFunction(); + // Get the number of primitive arguments + int GetPrimType(const PrimitivePtr& prim) const; + const std::unordered_map prim_func_type_map_; +}; +} // namespace prim +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPERATOR_PRIM_TO_FUNCTION_H_ diff --git a/mindspore/ccsrc/optimizer/CMakeLists.txt b/mindspore/ccsrc/optimizer/CMakeLists.txt new file mode 100644 index 00000000000..48cbeb41ddb --- /dev/null +++ b/mindspore/ccsrc/optimizer/CMakeLists.txt @@ -0,0 +1,9 @@ +file(GLOB_RECURSE _OPTIMIZER_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_optimizer_obj OBJECT ${_OPTIMIZER_ALL_SRC_FILES}) +if(ENABLE_DUMP_PROTO) + file(GLOB_RECURSE _PROTO_SRC_LIST "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc") + target_sources(_mindspore_optimizer_obj PRIVATE ${_PROTO_SRC_LIST}) +endif() \ No newline at end of file diff --git a/mindspore/ccsrc/optimizer/ad/adjoint.cc b/mindspore/ccsrc/optimizer/ad/adjoint.cc new file mode 100644 index 00000000000..46746b3f44e --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/adjoint.cc @@ -0,0 +1,96 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/ad/adjoint.h" + +#include +#include + +#include "ir/anf.h" +#include "optimizer/ad/dfunctor.h" + +namespace mindspore { +namespace ad { +Adjoint::Adjoint(const AnfNodePtr& primal, const AnfNodePtr& k, const FuncGraphPtr& caller) + : primal_(primal), caller_(caller), dout_(nullptr) { + if (k != nullptr) { + k_ = k; + MS_LOG(DEBUG) << "Add adjoint for " << primal->ToString() << " " << k_->ToString(); + } else { + // Init k hole in a recursive case. + auto k_hole = std::make_shared("k_hole"); + (void)k_hole->AddAttr("info", MakeValue(primal->ToString())); + k_ = NewValueNode(k_hole); + MS_LOG(DEBUG) << "Add hole for " << primal->ToString() << " " << k_->ToString(); + } + + dout_hole_ = caller_->NewCNode({NewValueNode(prim::GetPythonOps("zeros_like")), k_}); + RegisterKUser(dout_hole_->cast(), 1); +} + +AnfNodePtr Adjoint::k() { return k_; } + +void Adjoint::RegisterKUser(const CNodePtr& user, size_t index) { k_user_.emplace_back(std::make_pair(user, index)); } + +void Adjoint::UpdateK(const AnfNodePtr& new_k) { + MS_EXCEPTION_IF_NULL(new_k); + MS_LOG(DEBUG) << "Replace k " << k_->ToString() << " with " << new_k->ToString(); + // In recursive case, it needs update. + for (auto& user : k_user_) { + MS_LOG(DEBUG) << "Update k user " << user.first->ToString() << " " << user.second << " input with new_k" + << new_k->ToString(); + if (user.first->input(user.second) != k_) { + MS_LOG(EXCEPTION) << "Update k user " << user.first->ToString() << " " << user.second << " input with new_k " + << new_k->ToString() << ", user relation is set wrongly"; + } + user.first->set_input(user.second, new_k); + } + k_ = new_k; +} + +AnfNodePtr Adjoint::primal() { return primal_; } + +AnfNodePtr Adjoint::dout() { return dout_hole_; } + +void Adjoint::RegisterDoutUser(const CNodePtr& user, size_t index) { + dout_user_.emplace_back(std::make_pair(user, index)); +} + +void Adjoint::AccumulateDout(const AnfNodePtr& dout_factor) { + if (dout_ != nullptr) { + MS_LOG(DEBUG) << "Update dout " << dout_->ToString() << " with dout_factor " << dout_factor->ToString(); + auto add = prim::GetPythonOps("hyper_add"); + dout_ = caller_->NewCNode({NewValueNode(add), dout_, dout_factor}); + return; + } + dout_ = dout_factor; +} + +void Adjoint::CallDoutHole() { + if (dout_ != nullptr) { + for (auto& user : dout_user_) { + MS_LOG(DEBUG) << "Update dout user " << user.first->ToString() << " " << user.second << " input with dout " + << dout_->ToString(); + if (user.first->input(user.second) != dout_hole_) { + MS_LOG(EXCEPTION) << "Update dout user " << user.first->ToString() << " " << user.second << " input with dout " + << dout_->ToString() << ", user relation is set wrongly"; + } + user.first->set_input(user.second, dout_); + } + } +} +} // namespace ad +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/ad/adjoint.h b/mindspore/ccsrc/optimizer/ad/adjoint.h new file mode 100644 index 00000000000..673928129b7 --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/adjoint.h @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_AD_ADJOINT_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_AD_ADJOINT_H_ + +#include +#include +#include + +#include "ir/anf.h" +#include "optimizer/opt.h" + +namespace mindspore { +namespace ad { +class Adjoint { + public: + Adjoint(const AnfNodePtr& primal, const AnfNodePtr& k, const FuncGraphPtr& caller); + ~Adjoint() = default; + AnfNodePtr primal(); + AnfNodePtr k(); + void UpdateK(const AnfNodePtr& k); + void RegisterKUser(const CNodePtr& user, size_t index); + AnfNodePtr dout(); + void AccumulateDout(const AnfNodePtr& dout_factor); + void RegisterDoutUser(const CNodePtr& user, size_t index); + void CallDoutHole(); + + private: + AnfNodePtr primal_; + FuncGraphPtr caller_; + // For ```def f(x): return expr```, The representation graph k is ```def kf(kx): return expr, bprop{expr}```. + AnfNodePtr k_; + std::vector> k_user_; + AnfNodePtr dout_; + AnfNodePtr dout_hole_; + std::vector> dout_user_; +}; + +using AdjointPtr = std::shared_ptr; +} // namespace ad +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_AD_ADJOINT_H_ diff --git a/mindspore/ccsrc/optimizer/ad/dfunctor.cc b/mindspore/ccsrc/optimizer/ad/dfunctor.cc new file mode 100644 index 00000000000..128e4463e6a --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/dfunctor.cc @@ -0,0 +1,550 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/ad/dfunctor.h" + +#include +#include +#include + +#include "ir/anf.h" +#include "ir/meta_func_graph.h" +#include "debug/info.h" +#include "ir/func_graph_cloner.h" +#include "ir/manager.h" +#include "pipeline/resource.h" +#include "pipeline/parse/parse.h" +#include "optimizer/ad/adjoint.h" +#include "optimizer/opt.h" +#include "operator/ops.h" +#include "operator/composite/composite.h" +#include "utils/symbolic.h" +#include "./common.h" + +namespace mindspore { +namespace ad { +std::unordered_map DFunctor::func_graph_to_functor_; +std::unordered_map DFunctor::anfnode_to_adjoin_definition_; + +DFunctor::DFunctor(const FuncGraphPtr &primal_graph, const pipeline::ResourceBasePtr &resources) + : primal_graph_(primal_graph), resources_(resources), need_cut_(false), is_top_(false) { + TraceManager::DebugTrace(std::make_shared(primal_graph->debug_info())); + k_graph_ = std::make_shared(); + TraceManager::EndTrace(); + + TraceManager::DebugTrace(std::make_shared(primal_graph->debug_info())); + tape_ = std::make_shared(); + TraceManager::EndTrace(); + + dout_ = tape_->add_parameter(); +} + +void DFunctor::Init(const DFunctorPtr &functor, bool is_top) { + func_graph_to_functor_[primal_graph_] = functor; + is_top_ = is_top; +} + +void DFunctor::Clear() { + func_graph_to_functor_.clear(); + anfnode_to_adjoin_definition_.clear(); +} + +void DFunctor::BackPropagateFv(const AnfNodePtr &fv, const AnfNodePtr &din) { + auto fv_adjoint = anfnode_to_adjoin_.find(fv); + if (fv_adjoint == anfnode_to_adjoin_.end()) { + MS_LOG(DEBUG) << "BackPropagateFv can not find adjoint in anfnode_to_adjoin_ fv " << fv->func_graph()->ToString() + << " " << fv->ToString() << "."; + fv_adjoint = anfnode_to_adjoin_indirect_fv_.find(fv); + if (fv_adjoint == anfnode_to_adjoin_indirect_fv_.end()) { + MS_LOG(DEBUG) << "BackPropagateFv can not find adjoint in anfnode_to_adjoin_indirect_fv_ fv " + << fv->func_graph()->ToString() << " " << fv->ToString() << "."; + auto parent_adjoint = FindAdjoint(fv); + AdjointPtr adjoint = nullptr; + if (parent_adjoint != nullptr) { + adjoint = std::make_shared(fv, parent_adjoint->k(), tape_); + } else { + MS_LOG(DEBUG) << "BackPropagateFv failed can not find adjoint definition fv, add a k hole " + << fv->func_graph()->ToString() << " " << fv->ToString() << "."; + adjoint = std::make_shared(fv, nullptr, tape_); + } + anfnode_to_adjoin_indirect_fv_[fv] = adjoint; + fv_adjoint = anfnode_to_adjoin_indirect_fv_.find(fv); + } + } + auto key = tape_->NewCNode({NewValueNode(prim::kPrimEmbed), fv_adjoint->second->k()}); + fv_adjoint->second->RegisterKUser(key, 1); + auto default_val = tape_->NewCNode({NewValueNode(prim::GetPythonOps("zeros_like")), fv_adjoint->second->k()}); + fv_adjoint->second->RegisterKUser(default_val, 1); + auto dfv = tape_->NewCNode({NewValueNode(prim::kPrimEnvGetItem), din, key, default_val}); + MS_LOG(DEBUG) << "BackPropagateFv find adjoint in anfnode_to_adjoin_ or anfnode_to_adjoin_indirect_fv_ fv " + << fv->func_graph()->ToString() << " " << fv->ToString() << "."; + MS_LOG(DEBUG) << "BackPropagateFv get item from " << din->ToString() << " key " << key->ToString() << "."; + fv_adjoint->second->AccumulateDout(dfv); +} + +void DFunctor::BackPropagate(const CNodePtr &cnode_morph, const CNodePtr &k_app, const AdjointPtr &node_adjoint) { + auto bprop = k_graph_->NewCNode({NewValueNode(prim::kPrimTupleGetItem), k_app, NewValueNode(1)}); + // Call with delimited continuation dout. + auto bprop_app = tape_->NewCNode({bprop, node_adjoint->dout()}); + node_adjoint->RegisterDoutUser(bprop_app, 1); + for (size_t i = 0; i < cnode_morph->size(); i++) { + auto din = tape_->NewCNode({NewValueNode(prim::kPrimTupleGetItem), bprop_app, NewValueNode(SizeToInt(i))}); + auto input = cnode_morph->input(i); + // Backprop sens wrt fvs. + if (IsValueNode(input)) { + auto func_graph = GetValueNode(input); + auto functor = func_graph_to_functor_.find(func_graph); + if (functor == func_graph_to_functor_.end()) { + MS_LOG(EXCEPTION) << "BackPropagate failed functor for subgraph does not exist input[" << i << "] " + << func_graph->ToString() << "."; + } + // Consider direct and indirect fvs. + for (auto fv : func_graph->free_variables_nodes()) { + BackPropagateFv(fv, din); + } + for (auto indirect_fv : functor->second->anfnode_to_adjoin_indirect_fv_) { + MS_LOG(DEBUG) << "BackPropagate backprop indirect fv " << func_graph->ToString() << " " + << indirect_fv.first->ToString() << "."; + BackPropagateFv(indirect_fv.first, din); + } + continue; + } + // Backprop sens wrt inputs. + auto input_adjoint = anfnode_to_adjoin_.find(input); + if (input_adjoint == anfnode_to_adjoin_.end()) { + MS_LOG(EXCEPTION) << "BackPropagate adjoint does not exist input[" << i << "] " << input->ToString() << "."; + } + input_adjoint->second->AccumulateDout(din); + } +} + +// Map a morphism. +AdjointPtr DFunctor::MapMorphism(const AnfNodePtr &morph) { + // MapMorphism All type except CNode should already be mapped by MapObject. + if (!morph->isa()) { + return nullptr; + } + ScopeGuard scope_guard(morph->scope()); + auto cnode_morph = morph->cast(); + + std::vector inputs; + std::vector param_adjoints; + for (size_t i = 0; i < cnode_morph->size(); i++) { + auto node = cnode_morph->input(i); + auto node_adjoint_iter = anfnode_to_adjoin_.find(node); + AdjointPtr node_adjoint = nullptr; + AnfNodePtr k = nullptr; + if (node_adjoint_iter != anfnode_to_adjoin_.end()) { + node_adjoint = node_adjoint_iter->second; + } else { + // Input might be a CNode that needs to be handled before hand. + node_adjoint = MapMorphism(node); + } + MS_EXCEPTION_IF_NULL(node_adjoint); + k = node_adjoint->k(); + if (k == nullptr) { + MS_LOG(EXCEPTION) << "MapMorphism adjoint node does not exist, input[" << i << "] " << node->ToString() << "."; + } + inputs.push_back(k); + param_adjoints.push_back(node_adjoint); + } + TraceManager::DebugTrace(std::make_shared(cnode_morph->debug_info())); + auto k_app = k_graph_->NewCNode(inputs); + TraceManager::EndTrace(); + for (size_t i = 0; i < param_adjoints.size(); ++i) { + param_adjoints[i]->RegisterKUser(k_app, i); + } + + // Do forward computation + auto foward_app = k_graph_->NewCNode({NewValueNode(prim::kPrimTupleGetItem), k_app, NewValueNode(0)}); + // K:: cnode -> forward_app + auto node_adjoint = std::make_shared(morph, foward_app, tape_); + UpdateAdjoint(node_adjoint); + anfnode_to_adjoin_[morph] = node_adjoint; + if (cnode_morph->stop_gradient()) { + MS_LOG(WARNING) << "MapMorphism node " << morph->ToString() << " is stopped."; + return node_adjoint; + } + + // Do sens backpropagation + BackPropagate(cnode_morph, k_app, node_adjoint); + MS_LOG(DEBUG) << "MapMorphism node " << morph->ToString() << "."; + return node_adjoint; +} + +void DFunctor::MapFreeMorphism() { + // Handle cnode not attached to output, that might be refered in other functions. + for (auto &node : primal_graph_->nodes()) { + auto adjoint = FindAdjoint(node); + if (adjoint != nullptr) { + continue; + } + if (!node->isa()) { + MS_LOG(DEBUG) << "MapFreeMorphism noncnode not mapped after MapMorphism " << node->ToString() << " " + << node->type_name() << "."; + continue; + } + if (IsPrimitiveCNode(node, prim::kPrimReturn)) { + continue; + } + MS_LOG(DEBUG) << "MapFreeMorphism map nonoutput cnode after MapMorphism " << node->ToString() << "."; + (void)MapMorphism(node); + } +} + +AnfNodePtr DFunctor::AttachFvDoutToTape(const AnfNodePtr &grad_fv) { + AnfNodePtr new_grad_fv = grad_fv; + // Add grads wrt fv. + const auto &free_variables_nodes = primal_graph_->free_variables_nodes(); + for (auto &fv : free_variables_nodes) { + auto fv_adjoint = anfnode_to_adjoin_.find(fv); + if (fv_adjoint == anfnode_to_adjoin_.end()) { + MS_LOG(EXCEPTION) << "AttachFvDoutToTape fv adjoint does not exist " << fv->ToString() << "."; + } + auto key = tape_->NewCNode({NewValueNode(prim::kPrimEmbed), fv_adjoint->second->k()}); + fv_adjoint->second->RegisterKUser(key, 1); + auto sens = fv_adjoint->second->dout(); + new_grad_fv = tape_->NewCNode({ + NewValueNode(prim::kPrimEnvSetItem), + new_grad_fv, + key, + sens, + }); + fv_adjoint->second->RegisterDoutUser(new_grad_fv->cast(), 3); + MS_LOG(DEBUG) << "AttachFvDoutToTape add fv sens " << sens->ToString() << " to " << new_grad_fv->ToString() << " " + << fv->ToString() << " " << primal_graph_->ToString() << "."; + } + return new_grad_fv; +} + +AnfNodePtr DFunctor::AttachIndirectFvDoutToTape(const AnfNodePtr &grad_fv) { + AnfNodePtr new_grad_fv = grad_fv; + // Add indirect fv bprop. + for (auto &fv_adjoint : anfnode_to_adjoin_indirect_fv_) { + MS_LOG(DEBUG) << "AttachIndirectFvDoutToTape backprop indirect fv " << fv_adjoint.first->ToString() << " " + << primal_graph_->ToString() << "."; + auto key = tape_->NewCNode({NewValueNode(prim::kPrimEmbed), fv_adjoint.second->k()}); + fv_adjoint.second->RegisterKUser(key, 1); + auto sens = fv_adjoint.second->dout(); + new_grad_fv = tape_->NewCNode({ + NewValueNode(prim::kPrimEnvSetItem), + new_grad_fv, + key, + sens, + }); + fv_adjoint.second->RegisterDoutUser(new_grad_fv->cast(), 3); + MS_LOG(DEBUG) << "AttachIndirectFvDoutToTape add indirect fv sens " << sens->ToString() << " to " + << new_grad_fv->ToString() << "."; + } + return new_grad_fv; +} + +void DFunctor::MapMorphism() { + // Set stop_gradient before MapMorphism. + BroadCastStopFlag(); + + // Handle morphism from output. + (void)MapMorphism(primal_graph_->output()); + MapFreeMorphism(); + + // Construct K for primal_graph_ + auto output_adjoint = anfnode_to_adjoin_.find(primal_graph_->output()); + // Attach dout_ parameter to output_adjoint. + output_adjoint->second->AccumulateDout(dout_); + + // Set output for tape closure. + auto grad_fv = AttachIndirectFvDoutToTape(AttachFvDoutToTape(NewValueNode(newenv))); + + std::vector inputs{NewValueNode(prim::kPrimMakeTuple), grad_fv}; + // Add grads wrt inputs. + std::vector param_adjoints; + for (auto ¶m : primal_graph_->parameters()) { + auto param_adjoint = anfnode_to_adjoin_.find(param); + inputs.push_back(param_adjoint->second->dout()); + param_adjoints.push_back(param_adjoint->second); + } + auto tape_output = tape_->NewCNode(inputs); + for (size_t i = 0; i < param_adjoints.size(); ++i) { + param_adjoints[i]->RegisterDoutUser(tape_output, i + 2); + } + tape_->set_output(tape_output); + // Set output for k_graph_, K:: cnode->forward_app. + auto forward_app = output_adjoint->second->k(); + auto output = k_graph_->NewCNode({NewValueNode(prim::kPrimMakeTuple), forward_app, NewValueNode(tape_)}); + output_adjoint->second->RegisterKUser(output, 1); + k_graph_->set_output(output); + (void)primal_graph_->transforms().insert(std::make_pair("grad", FuncGraphTransform(k_graph_))); + (void)k_graph_->transforms().insert(std::make_pair("primal", FuncGraphTransform(primal_graph_))); +} + +FuncGraphPtr DFunctor::KUserDefined(const FuncGraphPtr &primal) { + // K user defined cell bprop. + auto bprop = primal->transforms().find("bprop"); + if (bprop != primal->transforms().end()) { + FuncGraphPtr bprop_graph = bprop->second.func_graph(); + const size_t param_diff = 1; + if (bprop_graph->output()->isa() && + bprop_graph->output()->cast()->size() + param_diff != bprop_graph->parameters().size()) { + MS_LOG(EXCEPTION) << "User defined Cell bprop " << primal->ToString() << " in scope " + << primal->output()->scope()->name() + << " output must be a tuple and output number should be the same with inputs."; + } + resources_->manager()->AddFuncGraph(bprop_graph); + + if (bprop_graph->free_variables_nodes().size() != 0 || primal->free_variables_nodes().size() != 0) { + MS_LOG(EXCEPTION) << "User defined Cell bprop " << primal->ToString() << " in scope " + << primal->output()->scope()->name() << " does not support Parameter data type."; + } + auto fg = g_k_prims.KUserDefinedCellBprop(bprop_graph); + if (fg == nullptr) { + MS_LOG(EXCEPTION) << "Failed to expand user defined Cell bprop " << primal->ToString() << " in scope " + << primal->output()->scope()->name() << "."; + } + + // Cache the grad func + (void)primal->transforms().insert(std::make_pair("grad", FuncGraphTransform(fg))); + (void)fg->transforms().insert(std::make_pair("primal", FuncGraphTransform(primal))); + // Reset defer_inline to enable successive inlining + primal->set_flags(FUNC_GRAPH_FLAG_DEFER_INLINE, false); + + auto functor = std::make_shared(primal, resources_); + functor->Init(functor); + functor->k_graph_ = fg; + + return fg; + } + return nullptr; +} + +// MapToK(func) +AnfNodePtr DFunctor::MapToK(const FuncGraphPtr &primal) { + auto f = func_graph_to_functor_.find(primal); + if (f != func_graph_to_functor_.end()) { + MS_LOG(DEBUG) << "K graph functor already exist " << primal->ToString() << "."; + return NewValueNode(f->second->k_graph_); + } + + auto k_user_defined = KUserDefined(primal); + if (k_user_defined != nullptr) { + MS_LOG(DEBUG) << "K graph functor user defined bprop " << primal->ToString() << "."; + return NewValueNode(k_user_defined); + } + + auto functor = std::make_shared(primal, resources_); + functor->Init(functor); + functor->MapObject(); + functor->MapMorphism(); + + MS_LOG(DEBUG) << "K graph K function graph " << primal->ToString() << " " << functor->k_graph_->ToString() << "."; + return NewValueNode(functor->k_graph_); +} + +// Construct representation graph for given node. +AnfNodePtr DFunctor::MapToK(const AnfNodePtr &primal) { + ScopeGuard scope_guard(primal->scope()); + // MapToK(prim) + if (IsValueNode(primal)) { + auto value_node = primal->cast(); + auto prim = GetValueNode(value_node); + if (prim->Hash() == prim::kPrimStopGradient->Hash() && prim->name() == prim::kPrimStopGradient->name()) { + MS_LOG(DEBUG) << "Meet a kPrimStopGradient " << prim->ToString() << "."; + need_cut_ = true; + } + auto k_prim = g_k_prims.KPrimitive(value_node, resources_); + if (k_prim != nullptr) { + k_prim = BasicClone(k_prim); + return NewValueNode(k_prim); + } + // When failed to find k_prim, try k_meta. + auto k_meta = g_k_prims.KMetaFuncGraph(prim); + if (k_meta != nullptr) { + return NewValueNode(k_meta); + } + } + + // MapToK(func) + if (IsValueNode(primal)) { + auto func_graph = GetValueNode(primal); + auto k_func = MapToK(func_graph); + return k_func; + } + + if (primal->isa()) { + TraceManager::DebugTrace(std::make_shared(primal->debug_info())); + auto ret = k_graph_->add_parameter(); + TraceManager::EndTrace(); + return ret; + } + + if (!primal->isa()) { + MS_LOG(EXCEPTION) << "K node keeped node from primal_graph_ " << primal->ToString() << " that is not a ValueNode."; + } + return primal; +} + +void DFunctor::MapFvObject() { + // Map free variable. + const auto &free_variables_nodes = primal_graph_->free_variables_nodes(); + for (auto &node : free_variables_nodes) { + ScopeGuard scope_guard(node->scope()); + MS_LOG(DEBUG) << "MapFvObject free variable " << node->ToString() << "."; + // Find fv's K from parent. + AdjointPtr adjoint = nullptr; + auto parent_adjoint = FindAdjoint(node); + if (parent_adjoint != nullptr) { + adjoint = std::make_shared(node, parent_adjoint->k(), tape_); + } else { + if (is_top_) { + // Top graph for ad, add adjoint for free variables. + adjoint = std::make_shared(node, node, tape_); + UpdateAdjoint(adjoint); + } else { + MS_LOG(DEBUG) << "MapFvObject fail to find parent adjoint for nontop fv " << node->ToString() << "."; + adjoint = std::make_shared(node, nullptr, tape_); + } + } + if (adjoint == nullptr) { + MS_LOG(EXCEPTION) << "MapFvObject failed for free variable " << node->ToString() << "."; + } + anfnode_to_adjoin_[node] = adjoint; + } +} + +void DFunctor::MapParamObject() { + // Map parameter. + for (auto &p : primal_graph_->parameters()) { + ScopeGuard scope_guard(p->scope()); + MS_LOG(DEBUG) << "MapParamObject parameter " << p->ToString() << "."; + auto adjoint = std::make_shared(p, MapToK(p), tape_); + UpdateAdjoint(adjoint); + anfnode_to_adjoin_[p] = adjoint; + } +} + +void DFunctor::MapValueObject() { + // Map ValueNode. + auto manager = resources_->manager(); + auto &value_nodes = manager->valuenodes()[primal_graph_]; + for (const auto &value_pair : value_nodes) { + auto node = value_pair.first; + auto parent_adjoint = FindAdjoint(node); + if (parent_adjoint != nullptr) { + auto adjoint = std::make_shared(node, parent_adjoint->k(), tape_); + anfnode_to_adjoin_[node] = adjoint; + continue; + } + // Skip Return. + if (IsValueNode(node) && GetValueNode(node) == prim::kPrimReturn) { + continue; + } + MS_LOG(DEBUG) << "MapValueObject node " << node->ToString() << "."; + auto adjoint = std::make_shared(node, MapToK(node), tape_); + UpdateAdjoint(adjoint); + anfnode_to_adjoin_[node] = adjoint; + } +} + +// Skip morphism. +void DFunctor::MapObject() { + // The order does not matter + MapFvObject(); + MapParamObject(); + MapValueObject(); +} + +void DFunctor::UpdateAdjoint(const AdjointPtr &adjoint_definition) { + auto primal = adjoint_definition->primal(); + if (anfnode_to_adjoin_definition_.find(primal) != anfnode_to_adjoin_definition_.end()) { + MS_LOG(EXCEPTION) << "UpdateAdjoint adjoint definition already exists " << primal_graph_->ToString() << " " + << primal->ToString() << "."; + } + anfnode_to_adjoin_definition_[primal] = adjoint_definition; + // Update k hole for primal. + for (auto &f : func_graph_to_functor_) { + auto adjoint = f.second->anfnode_to_adjoin_.find(primal); + if (adjoint != f.second->anfnode_to_adjoin_.end()) { + adjoint->second->UpdateK(adjoint_definition->k()); + } + adjoint = f.second->anfnode_to_adjoin_indirect_fv_.find(primal); + if (adjoint != f.second->anfnode_to_adjoin_indirect_fv_.end()) { + adjoint->second->UpdateK(adjoint_definition->k()); + } + } +} + +AdjointPtr DFunctor::FindAdjoint(const AnfNodePtr &primal) { + auto adjoint = anfnode_to_adjoin_definition_.find(primal); + if (adjoint != anfnode_to_adjoin_definition_.end()) { + MS_LOG(DEBUG) << "FindAdjoint found adjoint definition for free variable " << primal->ToString() << "."; + return adjoint->second; + } + MS_LOG(DEBUG) << "FindAdjoint adjoint definition for free variable not defined yet " << primal->ToString() << "."; + return nullptr; +} + +void DFunctor::CallDoutHoleOnTape() { + // Call dout hole of all adjoint. + for (auto &f : func_graph_to_functor_) { + for (auto &adjoint : f.second->anfnode_to_adjoin_) { + adjoint.second->CallDoutHole(); + } + for (auto &adjoint : f.second->anfnode_to_adjoin_indirect_fv_) { + adjoint.second->CallDoutHole(); + } + } +} +FuncGraphPtr DFunctor::k_graph() { + CallDoutHoleOnTape(); + return k_graph_; +} + +void DFunctor::BroadCastStopFlag() { + // As stop set expanding, all directly or indirectly stopped CNode will be cut off + while (need_cut_) { + need_cut_ = false; + for (auto &node : primal_graph_->nodes()) { + if (node->isa()) { + auto cnode = node->cast(); + if (!cnode->stop_gradient()) { + // Cut off the cnode only when it's not referred any more + if (IsPrimitiveCNode(cnode, prim::kPrimStopGradient) || AllReferencesStopped(cnode)) { + MS_LOG(DEBUG) << "Set stop gradient flag for " << cnode->ToString() << "."; + cnode->set_stop_gradient(true); + // The stop set changed, more cut required + need_cut_ = true; + } + } + } + } + } +} + +bool DFunctor::AllReferencesStopped(const CNodePtr &node) { + auto &users = primal_graph_->manager()->node_users()[node]; + // Only care about stop_gradient caused cutting + if (users.empty()) { + return false; + } + for (auto &kv : users) { + auto &user = kv.first; + if (!user->isa() || !user->cast()->stop_gradient()) { + return false; + } + } + return true; +} +} // namespace ad +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/ad/dfunctor.h b/mindspore/ccsrc/optimizer/ad/dfunctor.h new file mode 100644 index 00000000000..f50f866efaf --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/dfunctor.h @@ -0,0 +1,198 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_AD_D_FUNCTOR_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_AD_D_FUNCTOR_H_ + +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/meta_func_graph.h" +#include "ir/func_graph_cloner.h" +#include "pipeline/resource.h" +#include "optimizer/ad/adjoint.h" +#include "operator/ops.h" +#include "debug/trace.h" + +namespace mindspore { +namespace ad { +using Registry = std::unordered_map; +class KPrim; +extern KPrim g_k_prims; +class DFunctor; +using DFunctorPtr = std::shared_ptr; + +// D Functor's rules to map closure object and morphisms. +class DFunctor { + public: + DFunctor(const FuncGraphPtr &primal_graph, const pipeline::ResourceBasePtr &resources); + ~DFunctor() = default; + // Map object in D category to K category. + void MapObject(); + // Map morphism in D category to K category. + void MapMorphism(); + FuncGraphPtr k_graph(); + // Construct user defined k object. + FuncGraphPtr KUserDefined(const FuncGraphPtr &primal); + // Register functor objects to form a global view. + void Init(const DFunctorPtr &functor, bool is_top = false); + // Clear resources. + static void Clear(); + + private: + // Map one morphism. + AdjointPtr MapMorphism(const AnfNodePtr &morph); + // Map morphism that's not attached to output. + void MapFreeMorphism(); + void BackPropagateFv(const AnfNodePtr &fv, const AnfNodePtr &din); + void BackPropagate(const CNodePtr &cnode_morph, const CNodePtr &k_app, const AdjointPtr &node_adjoint); + AnfNodePtr AttachFvDoutToTape(const AnfNodePtr &grad_fv); + AnfNodePtr AttachIndirectFvDoutToTape(const AnfNodePtr &grad_fv); + // Map Anfnode object from D category to K category. + AnfNodePtr MapToK(const AnfNodePtr &primal); + // Map FuncGraph object from D category to K category. + AnfNodePtr MapToK(const FuncGraphPtr &primal); + // MapObject impls. + void MapFvObject(); + void MapValueObject(); + void MapParamObject(); + // Find adjoint with its primary k. + AdjointPtr FindAdjoint(const AnfNodePtr &primal); + // Broadcast stop flags. + void BroadCastStopFlag(); + bool AllReferencesStopped(const CNodePtr &node); + // Update k hole with adjoint_definition, only applied in recursive case. + void UpdateAdjoint(const AdjointPtr &adjoint_definition); + void CallDoutHoleOnTape(); + + std::unordered_map anfnode_to_adjoin_; + // Cache for indirect fv backpropagation, K o K can only do backprop layer by layer. + std::unordered_map anfnode_to_adjoin_indirect_fv_; + FuncGraphPtr primal_graph_; + // K object for primal_graph_; + FuncGraphPtr k_graph_; + // The Backprop part of k_graph_. + FuncGraphPtr tape_; + // Dout parameter for primal_graph_. + AnfNodePtr dout_; + pipeline::ResourceBasePtr resources_; + // Cut off stopped objects in category D. + bool need_cut_; + bool is_top_; + static std::unordered_map> func_graph_to_functor_; + static std::unordered_map anfnode_to_adjoin_definition_; +}; + +// D Functor's rules to map primitive object. +class KPrim { + public: + KPrim() = default; + ~KPrim() = default; + + FuncGraphPtr KPrimitive(const ValueNodePtr &value_node, const pipeline::ResourceBasePtr &resources); + MetaFuncGraphPtr KMetaFuncGraph(const PrimitivePtr &prim); + FuncGraphPtr KUserDefinedCellBprop(FuncGraphPtr bprop); + + void clear() { + bprop_registry_meta_.clear(); + bprop_registry_.clear(); + } + + private: + FuncGraphPtr GetBprop(const PrimitivePtr &prim); + FuncGraphPtr FakeBprop(const ValueNodePtr &value_node, const pipeline::ResourceBasePtr &resources); + // Given a bprop rule, do the K mapping. + template + FuncGraphPtr BpropToK(const T &primal, const FuncGraphPtr &bprop_g); + AnfNodePtr BuildOutput(const FuncGraphPtr &bprop_fg); + void TransformArgs(const FuncGraphManagerPtr &mng, const FuncGraphPtr &bprop_fg, const FuncGraphPtr &outer, + std::vector *const transf_args); + void AddCheckTypeShapeOp(const FuncGraphPtr &bprop_fg); + + Registry bprop_registry_; + std::unordered_map bprop_registry_meta_; +}; + +template +FuncGraphPtr KPrim::BpropToK(const T &primal, const FuncGraphPtr &bprop_fg) { + MS_EXCEPTION_IF_NULL(primal); + MS_EXCEPTION_IF_NULL(bprop_fg); + + if (IsPrimitiveCNode(bprop_fg->output(), prim::kPrimMakeTuple)) { + AddCheckTypeShapeOp(bprop_fg); + } + + auto debug_info = std::make_shared(); + debug_info->set_name(primal->ToString()); + + auto cloned_bprop_fg = BasicClone(bprop_fg); + MS_EXCEPTION_IF_NULL(cloned_bprop_fg); + + cloned_bprop_fg->debug_info()->set_name(""); + cloned_bprop_fg->debug_info()->set_trace_info(std::make_shared(debug_info)); + + AnfNodePtr bout = BuildOutput(cloned_bprop_fg); + cloned_bprop_fg->set_output(bout); + + TraceManager::DebugTrace(std::make_shared(debug_info)); + auto outer = std::make_shared(); + (void)outer->transforms().emplace("primal", FuncGraphTransform(primal)); + outer->set_output(NewValueNode(kNone)); + TraceManager::EndTrace(); + + auto mng = Manage({cloned_bprop_fg, outer}, false); + + // Make sure (out, dout) provided. + if (cloned_bprop_fg->parameters().size() < 2) { + MS_LOG(EXCEPTION) << "Primitive or Cell " << primal->ToString() + << " bprop requires out and dout at least, but only got " << cloned_bprop_fg->parameters().size() + << " params. NodeInfo: " << trace::GetDebugInfo(cloned_bprop_fg->debug_info()); + } + + // In a bprop definition, the last two param should be out and dout. + auto dout = cloned_bprop_fg->parameters()[cloned_bprop_fg->parameters().size() - 1]; + auto out_param = cloned_bprop_fg->parameters()[cloned_bprop_fg->parameters().size() - 2]; + std::vector transf_args; + TransformArgs(mng, cloned_bprop_fg, outer, &transf_args); + + TraceManager::DebugTrace(std::make_shared(dout->debug_info())); + (void)transf_args.insert(transf_args.begin(), NewValueNode(primal)); + auto out_value = outer->NewCNode(transf_args); + TraceManager::EndTrace(); + + (void)mng->Replace(out_param, out_value); + + TraceManager::DebugTrace(std::make_shared(out_param->debug_info())); + auto new_dout = cloned_bprop_fg->add_parameter(); + (void)mng->Replace(dout, new_dout); + // We remove all parameters except new_dout. + std::vector newBpropParams = {new_dout}; + cloned_bprop_fg->set_parameters(newBpropParams); + TraceManager::EndTrace(); + + outer->set_output(outer->NewCNode({NewValueNode(prim::kPrimMakeTuple), out_value, NewValueNode(cloned_bprop_fg)})); + return BasicClone(outer); +} +} // namespace ad +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_AD_D_FUNCTOR_H_ diff --git a/mindspore/ccsrc/optimizer/ad/grad.cc b/mindspore/ccsrc/optimizer/ad/grad.cc new file mode 100644 index 00000000000..7e1fdb842ec --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/grad.cc @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/ad/grad.h" +#include "optimizer/ad/dfunctor.h" +#include "ir/func_graph_cloner.h" +#include "utils/context/ms_context.h" +#include "utils/symbolic.h" +#include "utils/graph_utils.h" + +namespace mindspore { +namespace ad { +FuncGraphPtr Grad(const FuncGraphPtr &func_graph, const pipeline::ResourceBasePtr &resources) { + MS_EXCEPTION_IF_NULL(func_graph); + auto gradkv = func_graph->transforms().find("grad"); + if (gradkv != func_graph->transforms().end()) { + return gradkv->second.func_graph(); + } + + auto manager_ptr = resources->manager(); + MS_EXCEPTION_IF_NULL(manager_ptr); + manager_ptr->AddFuncGraph(func_graph); + + auto multi_graph_sink = [&func_graph](const FuncGraphPtr &f) { + if (MsContext::GetInstance()->is_multi_graph_sink()) { + if (func_graph->has_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES)) { + f->set_flags(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + } + } + }; + + auto f = std::make_shared(func_graph, resources); + auto user_defined = f->KUserDefined(func_graph); + if (user_defined != nullptr) { + multi_graph_sink(user_defined); + DFunctor::Clear(); + return user_defined; + } + f->Init(f, true); + f->MapObject(); + f->MapMorphism(); + auto ret = f->k_graph(); + DFunctor::Clear(); + + multi_graph_sink(ret); + return ret; +} + +FuncGraphPtr Kprim(const ValueNodePtr &value_node, const pipeline::ResourceBasePtr &resources) { + auto fg = g_k_prims.KPrimitive(value_node, resources); + if (fg == nullptr) { + return nullptr; + } + return BasicClone(fg); +} + +MetaFuncGraphPtr Kmeta(const PrimitivePtr &prim, const pipeline::ResourceBasePtr &) { + MetaFuncGraphPtr fg = g_k_prims.KMetaFuncGraph(prim); + return fg; +} +} // namespace ad +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/ad/grad.h b/mindspore/ccsrc/optimizer/ad/grad.h new file mode 100644 index 00000000000..12826311dce --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/grad.h @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_AD_GRAD_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_AD_GRAD_H_ + +#include +#include + +#include "ir/anf.h" +#include "ir/meta_func_graph.h" +#include "pipeline/resource.h" + +namespace mindspore { +namespace ad { +using ResourcePtr = std::shared_ptr; + +FuncGraphPtr Grad(const FuncGraphPtr &func_graph, const pipeline::ResourceBasePtr &resources); +FuncGraphPtr Kprim(const ValueNodePtr &value_node, const pipeline::ResourceBasePtr &resources); +MetaFuncGraphPtr Kmeta(const PrimitivePtr &prim, const pipeline::ResourceBasePtr &); +} // namespace ad +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_AD_GRAD_H_ diff --git a/mindspore/ccsrc/optimizer/ad/kprim.cc b/mindspore/ccsrc/optimizer/ad/kprim.cc new file mode 100644 index 00000000000..4576cc1ea99 --- /dev/null +++ b/mindspore/ccsrc/optimizer/ad/kprim.cc @@ -0,0 +1,224 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "ir/anf.h" +#include "ir/meta_func_graph.h" +#include "ir/func_graph_cloner.h" +#include "ir/manager.h" +#include "pipeline/resource.h" +#include "pipeline/parse/parse.h" +#include "optimizer/ad/dfunctor.h" +#include "optimizer/opt.h" +#include "operator/ops.h" +#include "operator/composite/composite.h" +#include "utils/symbolic.h" +#include "debug/info.h" +#include "debug/trace.h" + +#include "./common.h" + +namespace mindspore { +namespace ad { +using PatternListType = std::initializer_list; +KPrim g_k_prims; + +FuncGraphPtr KPrim::GetBprop(const PrimitivePtr &prim) { + // Set a child scope named "grad'PrimitiveName'" for the bprop function, + // and add "Gradients" to the front. + static const std::string gradients_scope = "Gradients/"; + static const std::string grad_op_child_scope_prefix = "/grad"; + MS_EXCEPTION_IF_NULL(prim); + auto scope = std::make_shared(gradients_scope + ScopeManager::GetInstance().GetCurrentScope()->name() + + grad_op_child_scope_prefix + prim->name()); + ScopeGuard scope_guard(scope); + py::function fn = prim->GetBpropFunction(); + FuncGraphPtr func_graph = parse::ParsePythonCode(fn); + if (func_graph == nullptr) { + MS_LOG(WARNING) << "Fail to find bprop function for " << prim->name() << "."; + return nullptr; + } + return func_graph; +} + +MetaFuncGraphPtr KPrim::KMetaFuncGraph(const PrimitivePtr &prim) { + MS_EXCEPTION_IF_NULL(prim); + + auto iter = bprop_registry_meta_.find(prim); + if (iter != bprop_registry_meta_.end()) { + return iter->second; + } + + if (prim->name() == "make_tuple") { + MetaFuncGraphPtr meta = std::make_shared("make_tuple_gradient"); + bprop_registry_meta_[prim::kPrimMakeTuple] = meta; + return meta; + } + + MS_LOG(EXCEPTION) << "Fail to find bprop function for " << prim->name() << "."; +} + +FuncGraphPtr KPrim::KPrimitive(const ValueNodePtr &value_node, const pipeline::ResourceBasePtr &resources) { + if (!IsValueNode(value_node)) { + MS_LOG(EXCEPTION) << "Primitive node is not valid."; + } + + auto prim = value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(prim); + + auto iter = bprop_registry_.find(prim); + if (iter != bprop_registry_.end()) { + return iter->second; + } + + if (prim->name() == "make_tuple") { + return nullptr; + } + + auto bprop_fg = GetBprop(prim); + if (bprop_fg == nullptr) { + bprop_fg = FakeBprop(value_node, resources); + } + + auto expanded_fg = BpropToK(prim, bprop_fg); + if (expanded_fg == nullptr) { + MS_LOG(EXCEPTION) << "Failed convert " << prim->name() + << " prim bprop function to J expanded func graph. NodeInfo: " + << trace::GetDebugInfo(bprop_fg->debug_info()); + } + + // Set bprop_g graph cache + bprop_registry_[prim] = expanded_fg; + return expanded_fg; +} + +AnfNodePtr KPrim::BuildOutput(const FuncGraphPtr &bprop_fg) { + // bprop_fg has been checked in caller + if (IsPrimitiveCNode(bprop_fg->output(), prim::kPrimMakeTuple)) { + // Set bprop output as (env, dx, dy, dz, ...) + auto cbprop = bprop_fg->output()->cast(); + auto &inputs = cbprop->inputs(); + + std::vector args; + args.push_back(NewValueNode(prim::kPrimMakeTuple)); + args.push_back(NewValueNode(newenv)); + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + return NewCNode(args, bprop_fg); + } + + // Set bprop output as (env, dx) + std::string model_name("mindspore.ops.composite.multitype_ops.add_impl"); + std::string python_ops("_tuple_add"); + auto tuple = NewCNode({NewValueNode(prim::kPrimMakeTuple), NewValueNode(newenv)}, bprop_fg); + return NewCNode({NewValueNode(prim::GetPythonOps(python_ops, model_name)), tuple, bprop_fg->output()}, bprop_fg); +} + +void KPrim::TransformArgs(const FuncGraphManagerPtr &mng, const FuncGraphPtr &bprop_fg, const FuncGraphPtr &outer, + std::vector *const transf_args) { + MS_EXCEPTION_IF_NULL(mng); + // bprop_fg has been checked in caller + // transform except the last 2 parameters: out, dout. + for (size_t i = 0; i < bprop_fg->parameters().size() - 2; ++i) { + auto p = bprop_fg->parameters()[i]; + MS_EXCEPTION_IF_NULL(p); + + TraceManager::DebugTrace(std::make_shared(p->debug_info())); + auto transf_p = outer->add_parameter(); + TraceManager::EndTrace(); + + (void)mng->Replace(p, transf_p); + transf_args->push_back(transf_p); + } +} + +void KPrim::AddCheckTypeShapeOp(const FuncGraphPtr &bprop_fg) { + // bprop_fg has been checked in caller + auto same_type_shape = prim::GetPythonOps("same_type_shape", "mindspore.ops.functional")->cast(); + MS_EXCEPTION_IF_NULL(same_type_shape); + + std::vector bout_input; + bout_input.push_back(NewValueNode(prim::kPrimMakeTuple)); + + auto fg_out = bprop_fg->output(); + MS_EXCEPTION_IF_NULL(fg_out); + auto cnode = fg_out->cast(); + MS_EXCEPTION_IF_NULL(cnode); + + auto &inputs = cnode->inputs(); + auto params = bprop_fg->parameters(); + std::vector sub_input; + for (size_t i = 1; i < inputs.size(); ++i) { + sub_input.clear(); + sub_input.push_back(NewValueNode(same_type_shape)); + sub_input.push_back(inputs[i]); + sub_input.push_back(params[i - 1]); + bout_input.push_back(bprop_fg->NewCNode(sub_input)); + } + AnfNodePtr cbout = bprop_fg->NewCNode(bout_input); + bprop_fg->set_output(cbout); +} + +FuncGraphPtr KPrim::KUserDefinedCellBprop(const FuncGraphPtr bprop_fg) { + MS_EXCEPTION_IF_NULL(bprop_fg); + auto fprop_fg = bprop_fg->transforms().find("primal")->second.func_graph(); + auto expanded_fg = BpropToK(fprop_fg, bprop_fg); + if (expanded_fg == nullptr) { + MS_LOG(EXCEPTION) << "Failed convert " << fprop_fg->ToString() + << " Cell bprop function to K expanded func graph. NodeInfo: " + << trace::GetDebugInfo(fprop_fg->debug_info()); + } + return expanded_fg; +} + +FuncGraphPtr KPrim::FakeBprop(const ValueNodePtr &value_node, const pipeline::ResourceBasePtr &resources) { + auto prim = value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(prim); + auto &node_users = resources->manager()->node_users(); + + auto &users = node_users[value_node]; + auto cnode = std::find_if(users.begin(), users.end(), [&prim](const std::pair &user) -> bool { + return IsPrimitiveCNode(user.first, prim); + }); + if (cnode == users.end()) { + MS_LOG(EXCEPTION) << "Fail to find cnode."; + } + auto inputs_num = cnode->first->cast()->inputs().size() - 1; + + auto func_graph = std::make_shared(); + std::vector outputs; + outputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + + auto fake_bprop = std::make_shared("fake_bprop"); + (void)fake_bprop->AddAttr("info", MakeValue("Primitive " + prim->name() + "'s bprop not defined.")); + + for (size_t i = 0; i < inputs_num; ++i) { + // Mock params for inputs + auto param = func_graph->add_parameter(); + // Mock derivatives for each inputs + outputs.push_back(func_graph->NewCNode({NewValueNode(fake_bprop), param})); + } + // mock params for out and dout + (void)func_graph->add_parameter(); + (void)func_graph->add_parameter(); + func_graph->set_output(func_graph->NewCNode(outputs)); + return func_graph; +} +} // namespace ad +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/clean.cc b/mindspore/ccsrc/optimizer/clean.cc new file mode 100644 index 00000000000..9e713d34255 --- /dev/null +++ b/mindspore/ccsrc/optimizer/clean.cc @@ -0,0 +1,476 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/clean.h" +#include +#include +#include +#include +#include +#include "./common.h" +#include "debug/trace.h" + +namespace mindspore { +/* namespace to support opt */ +namespace opt { +using mindspore::abstract::AbstractAttribute; +using mindspore::abstract::AbstractClass; +using mindspore::abstract::AbstractDictionary; +using mindspore::abstract::AbstractJTagged; +using mindspore::abstract::AbstractList; +using mindspore::abstract::AbstractScalar; +using mindspore::abstract::AbstractTuple; + +static AbstractBasePtr Reabs(const AbstractBasePtr& t) { + if (t == nullptr) { + return nullptr; + } + + AbstractBasePtr res = t; + if (t->isa()) { + auto abs_class = dyn_cast(t); + AbstractBasePtrList baselist; + auto attributes = abs_class->attributes(); + (void)std::transform(attributes.begin(), attributes.end(), std::back_inserter(baselist), + [](const AbstractAttribute& item) { return item.second; }); + res = std::make_shared(baselist); + } else if (t->isa()) { + auto abs_dict = dyn_cast(t); + AbstractBasePtrList baselist; + auto elements = abs_dict->elements(); + (void)std::transform(elements.begin(), elements.end(), std::back_inserter(baselist), + [](const AbstractAttribute& item) { return item.second; }); + res = std::make_shared(baselist); + } else if (t->isa()) { + auto abs_dict = dyn_cast(t); + res = std::make_shared(abs_dict->elements()); + } + return res; +} + +AnfNodePtr ConvertGetAttrToTupleGetItem(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + const auto& inputs = node->inputs(); + // Inputs should be [getattr, data, attribute] + MS_ASSERT(inputs.size() == 3 && "GetAttr should have three inputs."); + + AnfNodePtr data = inputs[1]; + AnfNodePtr cons = inputs[2]; + MS_EXCEPTION_IF_NULL(data); + MS_EXCEPTION_IF_NULL(cons); + + auto dt = data->abstract(); + MS_EXCEPTION_IF_NULL(dt); + if (!dt->isa()) { + MS_LOG(EXCEPTION) << "First parameter of getattr is not AbstractClass, but " << dt->type_name() << "."; + } + + auto cons_is_str = IsValueNode(cons); + auto cons_str = cons_is_str ? GetValue(GetValueNode(cons)) : ""; + + auto ct = dyn_cast(dt); + const auto& cmap = ct->attributes(); + int count = 0; + for (auto& item : cmap) { + if (cons_is_str && item.first == cons_str) { + break; + } + count++; + } + + auto idx_c = NewValueNode(count); + AbstractBasePtr aptr = std::make_shared(std::make_shared(count)); + idx_c->set_abstract(aptr); + + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimTupleGetItem), data, idx_c}); +} + +AnfNodePtr ConvertDictGetItemToTupleGetItem(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + // Inputs should be [dict_getitem, dict, item] + const auto& inputs = node->inputs(); + MS_ASSERT(inputs.size() == 3 && "DictGetItem should have three inputs."); + + AnfNodePtr data = inputs[1]; + AnfNodePtr cons = inputs[2]; + MS_EXCEPTION_IF_NULL(data); + MS_EXCEPTION_IF_NULL(cons); + + auto dt = data->abstract(); + MS_EXCEPTION_IF_NULL(dt); + if (!dt->isa()) { + MS_LOG(EXCEPTION) << "first parameter of dict_getitem is not AbstractDictionary, but " << dt->type_name(); + } + auto cons_is_str = IsValueNode(cons); + auto cons_str = cons_is_str ? GetValue(GetValueNode(cons)) : ""; + + auto ct = dyn_cast(dt); + const auto& cmap = ct->elements(); + int count = 0; + for (auto& item : cmap) { + if (cons_is_str && item.first == cons_str) { + break; + } + count++; + } + + auto idx_c = NewValueNode(count); + AbstractBasePtr aptr = std::make_shared(std::make_shared(count)); + idx_c->set_abstract(aptr); + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimTupleGetItem), data, idx_c}); +} + +AnfNodePtr ConvertMakeRecordToMakeTuple(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + std::vector inputs; + inputs.emplace_back(NewValueNode(prim::kPrimMakeTuple)); + // Inputs of node should be [make_record, klass, attr1, attr2, ...], so offset by 2 to get attr; + (void)inputs.insert(inputs.end(), node->inputs().begin() + 2, node->inputs().end()); + return node->func_graph()->NewCNode(inputs); +} + +AnfNodePtr ErasePartialNode(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + const auto& inputs = node->inputs(); + // Inputs should be [partial, fn, arg1, ...], so offset by 2 to get arg; + MS_ASSERT(inputs.size() >= 2 && "Partial should have more than two inputs."); + + std::vector args(inputs.begin() + 2, inputs.end()); + auto oper = inputs[1]; + if (IsPrimitive(oper, prim::kPrimMakeRecord)) { + if (args.size() == 1) { + return NewValueNode(prim::kPrimMakeTuple); + } + + if (args.size() > 1) { + std::vector new_inputs; + new_inputs.emplace_back(NewValueNode(prim::kPrimPartial)); + new_inputs.emplace_back(NewValueNode(prim::kPrimMakeTuple)); + (void)new_inputs.insert(new_inputs.end(), args.begin() + 1, args.end()); + + MS_EXCEPTION_IF_NULL(node->func_graph()); + return node->func_graph()->NewCNode(new_inputs); + } + } + return nullptr; +} + +AnfNodePtr ConvertMakeListToMakeTuple(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + std::vector inputs; + inputs.emplace_back(NewValueNode(prim::kPrimMakeTuple)); + // Inputs of node should be [make_list, item1, item2, ...], so offset by 1 to get items; + (void)inputs.insert(inputs.end(), node->inputs().begin() + 1, node->inputs().end()); + return node->func_graph()->NewCNode(inputs); +} + +AnfNodePtr ConvertListGetItemToTupleGetItem(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + const auto& inputs = node->inputs(); + // Inputs should be [list_getitem, list, item] + if (inputs.size() < 3) { + MS_LOG(EXCEPTION) << "Node's input number < 3."; + } + + AnfNodePtr data = inputs[1]; + AnfNodePtr cons = inputs[2]; + MS_EXCEPTION_IF_NULL(data); + MS_EXCEPTION_IF_NULL(cons); + + auto cons_node = cons->cast(); + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimTupleGetItem), data, cons_node}); +} + +AnfNodePtr ConvertListSetItemToTupleSetItem(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(node->func_graph()); + + const auto& inputs = node->inputs(); + // Inputs should be [list_setitem, list, index, item] + if (inputs.size() < 4) { + MS_LOG(EXCEPTION) << "Node's input number < 4."; + } + + AnfNodePtr data = inputs[1]; + AnfNodePtr cons = inputs[2]; + AnfNodePtr value = inputs[3]; + + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimTupleSetItem), data, cons, value}); +} + +AnfNodePtr EraseMakeDictNode(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + const auto& inputs = node->inputs(); + MS_ASSERT(inputs.size() >= 3 && "MakeDict should have three inputs"); + return inputs[2]; +} + +AnfNodePtr EraseMakeKeywordArgNode(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + const auto& inputs = node->inputs(); + // Inputs should be [make_keyword_arg, key, value] + MS_ASSERT(inputs.size() == 3 && "MakeKeyword should have three inputs"); + return inputs[2]; +} + +AnfNodePtr EraseExtractKeywordArg(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + const auto& inputs = node->inputs(); + // Inputs should be [extract_keyword_arg, arg, key] + MS_ASSERT(inputs.size() == 3 && "ExtractKeyword should have three inputs"); + return inputs[2]; +} + +ValueTuplePtr ConvertValueListToValueTuple(const ValueListPtr& value_list, int depth) { + const int DEPTH_MAX = 5; + if (depth > DEPTH_MAX) { + MS_LOG(EXCEPTION) << "List nesting is not allowed more than 5 levels."; + } + std::vector elements; + for (const auto& it : value_list->value()) { + ValuePtr value = nullptr; + if (it->isa()) { + value = ConvertValueListToValueTuple(it->cast(), depth + 1); + } else { + value = it; + } + elements.push_back(value); + } + return std::make_shared(elements); +} + +AnfNodePtr ConvertValueListNodeToValueTupleNode(const ValueNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + ValuePtr value = node->value(); + auto value_list = value->cast(); + MS_EXCEPTION_IF_NULL(value_list); + int depth = 0; + return std::make_shared(ConvertValueListToValueTuple(value_list, depth)); +} + +// Convert class to Tuple +// Convert getattr to getitem +// Convert make_record to make_tuple +void SimplifyDataStructures(const FuncGraphPtr& root, const FuncGraphManagerPtr& manager) { + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(root); + + // Since `manager->Replace(...);` will modify member `all_nodes_`, so `all_node` can't be a ref var + AnfNodeSet all_node = manager->all_nodes(); + for (auto& node : all_node) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + AnfNodePtr new_node = nullptr; + if (IsValueNode(node)) { + new_node = NewValueNode(prim::kPrimMakeTuple); + } else if (IsPrimitiveCNode(node, prim::kPrimGetAttr)) { + new_node = ConvertGetAttrToTupleGetItem(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimMakeRecord)) { + new_node = ConvertMakeRecordToMakeTuple(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimPartial)) { + new_node = ErasePartialNode(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimDictGetItem)) { + new_node = ConvertDictGetItemToTupleGetItem(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimMakeDict)) { + new_node = EraseMakeDictNode(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimMakeKeywordArg)) { + new_node = EraseMakeKeywordArgNode(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimExtractKeywordArg)) { + new_node = EraseExtractKeywordArg(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimMakeList)) { + new_node = ConvertMakeListToMakeTuple(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimListGetItem)) { + new_node = ConvertListGetItemToTupleGetItem(cnode); + } else if (IsPrimitiveCNode(node, prim::kPrimListSetItem)) { + new_node = ConvertListSetItemToTupleSetItem(cnode); + } else if (IsValueNode(node)) { + new_node = ConvertValueListNodeToValueTupleNode(node->cast()); + } + + if (new_node != nullptr) { + new_node->set_abstract(node->abstract()); + (void)manager->Replace(node, new_node); + } + } + + for (auto& node : manager->all_nodes()) { + auto ret = Reabs(node->abstract()); + node->set_abstract(ret); + } +} + +// expand tuples in graph parameters +static std::vector ExpandTuplesP(const FuncGraphManagerPtr& mng, const FuncGraphPtr& func_graph, + const std::vector& params) { + MS_EXCEPTION_IF_NULL(mng); + MS_EXCEPTION_IF_NULL(func_graph); + + std::vector new_params; + for (const auto& param : params) { + MS_EXCEPTION_IF_NULL(param); + auto param_abs = param->abstract(); + MS_EXCEPTION_IF_NULL(param_abs); + + if (param_abs->isa()) { + MS_LOG(EXCEPTION) << "Not Implemented Error NodeInfo: " << trace::GetDebugInfo(param->debug_info()); + } + + if (!param_abs->isa()) { + new_params.emplace_back(param); + continue; + } + + std::vector new_param; + std::vector inputs{NewValueNode(prim::kPrimMakeTuple)}; + auto abs_tuple = dyn_cast(param_abs); + for (auto& elem : abs_tuple->elements()) { + auto np = std::make_shared(func_graph); + np->set_abstract(elem); + new_param.emplace_back(np); + } + (void)inputs.insert(inputs.end(), new_param.begin(), new_param.end()); + auto new_tuple = func_graph->NewCNode(inputs); + (void)mng->Replace(param, new_tuple); + + auto expand_param = ExpandTuplesP(mng, func_graph, new_param); + (void)new_params.insert(new_params.end(), expand_param.begin(), expand_param.end()); + } + return new_params; +} + +// expand tuples in graph applies +static std::vector ExpandTuplesC(const FuncGraphPtr& graph, const std::vector& inputs) { + MS_EXCEPTION_IF_NULL(graph); + + std::vector new_inputs; + for (const auto& input : inputs) { + MS_EXCEPTION_IF_NULL(input); + + auto input_abs = input->abstract(); + MS_EXCEPTION_IF_NULL(input_abs); + + if (input_abs->isa()) { + auto abstract_tag = dyn_cast(input_abs); + if (abstract_tag->element()->isa()) { + MS_LOG(EXCEPTION) << "Not Implemented Error JTagged NodeInfo: " << trace::GetDebugInfo(input->debug_info()); + } + } + + if (!input_abs->isa()) { + new_inputs.emplace_back(input); + continue; + } + + int idx = 0; + std::vector new_input; + auto abs_tuple = dyn_cast(input_abs); + for (auto& elem : abs_tuple->elements()) { + auto c_node = graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), input, NewValueNode(idx)}); + AbstractBasePtr aptr = std::make_shared(std::make_shared(idx)); + c_node->input(2)->set_abstract(aptr); + c_node->set_abstract(elem); + new_input.emplace_back(c_node); + idx++; + } + + auto expand_tuple = ExpandTuplesC(graph, new_input); + (void)new_inputs.insert(new_inputs.end(), expand_tuple.begin(), expand_tuple.end()); + } + + return new_inputs; +} + +// remove most uses of tuples from the graph parameters & apply inputs +// tuples that are returned will be kept +// tuples in CNode's inputs: AbstractTuple (a, b ,c) --> +// CNode("tuple_getitem", (a,b,c), 0) +// CNode("tuple_getitem", (a,b,c), 1) +// CNode("tuple_getitem", (a,b,c), 2) +// tuples in Graph's parameters: AbstractTuple (a, b, c) --> +// CNode("make_tuple", Parameter(a), Parameter(b), Parameter(c)) +// cppcheck-suppress unusedFunction +void EraseTuple(const FuncGraphPtr& root, const FuncGraphManagerPtr& manager) { + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(root); + + // NOTICE: since `manager->Replace(...);` will modify member `all_nodes_`, so `all_node` can't be a ref var + AnfNodeSet all_node = manager->all_nodes(); + for (auto& node : all_node) { + auto cnode = node->cast(); + if (cnode == nullptr) { + continue; + } + + const auto& inputs = cnode->inputs(); + + // Bypass the first input in inputs as it's fn. + if (!IsValueNode(inputs[0])) { + std::vector expand_inputs; + (void)expand_inputs.insert(expand_inputs.end(), inputs.begin() + 1, inputs.end()); + + auto new_inputs = ExpandTuplesC(cnode->func_graph(), expand_inputs); + if (new_inputs != expand_inputs) { + std::vector cnode_inputs{inputs[0]}; + (void)cnode_inputs.insert(cnode_inputs.end(), new_inputs.begin(), new_inputs.end()); + + MS_EXCEPTION_IF_NULL(node->func_graph()); + auto new_node = node->func_graph()->NewCNode(cnode_inputs); + new_node->set_abstract(node->abstract()); + + (void)manager->Replace(node, new_node); + } + // Bypass the first 2 inputs in inputs as it's [partial, fn]. + } else if (cnode->IsApply(prim::kPrimPartial) && !IsValueNode(inputs[1])) { + std::vector expand_inputs; + (void)expand_inputs.insert(expand_inputs.end(), inputs.begin() + 2, inputs.end()); + + auto new_inputs = ExpandTuplesC(cnode->func_graph(), expand_inputs); + if (new_inputs != expand_inputs) { + std::vector cnode_inputs{inputs[0], inputs[1]}; + (void)cnode_inputs.insert(cnode_inputs.end(), new_inputs.begin(), new_inputs.end()); + + MS_EXCEPTION_IF_NULL(cnode->func_graph()); + auto new_node = cnode->func_graph()->NewCNode(cnode_inputs); + new_node->set_abstract(cnode->abstract()); + + (void)manager->Replace(node, new_node); + } + } + } + + FuncGraphSet all_graph = manager->func_graphs(); + for (auto& func_graph : all_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + auto expand_p = ExpandTuplesP(manager, func_graph, func_graph->parameters()); + manager->SetParameters(func_graph, expand_p); + } +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/clean.h b/mindspore/ccsrc/optimizer/clean.h new file mode 100644 index 00000000000..01db7d363dc --- /dev/null +++ b/mindspore/ccsrc/optimizer/clean.h @@ -0,0 +1,43 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_CLEAN_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_CLEAN_H_ + +#include +#include "ir/anf.h" +#include "operator/ops.h" +#include "utils/any.h" +#include "ir/manager.h" +#include "pipeline/static_analysis/dshape.h" + +namespace mindspore { +/* namespace to support opt */ +namespace opt { + +// Remove the class type from graphs +void SimplifyDataStructures(const FuncGraphPtr &root, const FuncGraphManagerPtr &manager); + +// Remove most uses of tuples from the graph +// tuples that are returned will be kept +void EraseTuple(const FuncGraphPtr &root, const FuncGraphManagerPtr &manager); + +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_CLEAN_H_ diff --git a/mindspore/ccsrc/optimizer/control_depend.cc b/mindspore/ccsrc/optimizer/control_depend.cc new file mode 100644 index 00000000000..0b5c85b1e01 --- /dev/null +++ b/mindspore/ccsrc/optimizer/control_depend.cc @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/control_depend.h" + +#include +#include +#include +#include +#include + +#include "optimizer/optimizer.h" + +namespace mindspore { +namespace opt { +std::vector DoControlDepend(const FuncGraphPtr &graph, const CNodePtr &return_node, + const std::vector &effect_index, const std::vector &cnodes) { + std::vector depend_nodes{NewValueNode(prim::kPrimDepend), return_node->input(1)}; + std::vector make_tuple{NewValueNode(prim::kPrimMakeTuple)}; + size_t effect_size = effect_index.size(); + for (size_t i = 0; i < effect_size; i++) { + size_t pre_index = 0; + if (i > 0) { + pre_index = effect_index[i - 1] + 1; + } + size_t this_index = effect_index[i]; + size_t last_index = cnodes.size() - 2; + if (i < effect_size - 1) { + last_index = effect_index[i + 1]; + } + + if (this_index > pre_index) { + std::vector pre_segment; + for (size_t k = pre_index; k < this_index; k++) { + // Skip depend, make_tuple, and tuple_get_item, because these primitives are not real operator in GE. + if (IsPrimitiveCNode(cnodes[k], prim::kPrimDepend) || IsPrimitiveCNode(cnodes[k], prim::kPrimMakeTuple) || + IsPrimitiveCNode(cnodes[k], prim::kPrimTupleGetItem)) { + continue; + } + pre_segment.push_back(cnodes[k]); + } + auto roots = FindRoots(pre_segment); + for (auto iter = roots->begin(); iter != roots->end(); (void)iter++) { + AnfNodePtr control_depend = + graph->NewCNode({NewValueNode(prim::kPrimControlDepend), *iter, cnodes[this_index]}); + make_tuple.push_back(control_depend); + } + } + if (last_index > this_index) { + std::vector last_segment; + for (size_t k = this_index + 1; k <= last_index; k++) { + // Skip depend, make_tuple, and tuple_get_item, because these primitives are not real operator in GE. + if (IsPrimitiveCNode(cnodes[k], prim::kPrimDepend) || IsPrimitiveCNode(cnodes[k], prim::kPrimMakeTuple) || + IsPrimitiveCNode(cnodes[k], prim::kPrimTupleGetItem)) { + continue; + } + last_segment.push_back(cnodes[k]); + } + auto leaves = FindLeaves(last_segment); + for (auto iter = leaves->begin(); iter != leaves->end(); (void)iter++) { + AnfNodePtr control_depend = + graph->NewCNode({NewValueNode(prim::kPrimControlDepend), cnodes[this_index], *iter}); + make_tuple.push_back(control_depend); + } + } + } + depend_nodes.push_back(graph->NewCNode(make_tuple)); + return depend_nodes; +} + +void AddControlDepend(const FuncGraphPtr &graph) { + MS_EXCEPTION_IF_NULL(graph); + std::list orders = graph->GetOrderedCnodes(); + std::vector cnodes(orders.begin(), orders.end()); + size_t cnodes_size = cnodes.size(); + // get effect index of cnodes + std::vector effect_index{}; + for (size_t i = 0; i < cnodes_size; i++) { + if (graph->HasEffect(cnodes[i])) { + effect_index.push_back(i); + } + } + if (effect_index.empty()) { + return; + } + AnfNodePtr last_node = cnodes[cnodes_size - 1]; + CNodePtr return_node; + if (last_node->isa()) { + return_node = last_node->cast(); + } + MS_EXCEPTION_IF_NULL(return_node); + if (!IsPrimitiveCNode(return_node, prim::kPrimReturn)) { + MS_LOG(EXCEPTION) << "The last cnode after sorting, not return cnode."; + } + if (return_node->inputs().size() < 2) { + MS_LOG(EXCEPTION) << "Number of return node inputs should be great than or equal to 2."; + } + + auto depend_node_inputs = DoControlDepend(graph, return_node, effect_index, cnodes); + auto depend_cnode = graph->NewCNode(depend_node_inputs); + depend_cnode->set_abstract(depend_cnode->input(1)->abstract()); + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (!manager->Replace(return_node->input(1), depend_cnode)) { + MS_LOG(EXCEPTION) << "Depend replace node failed"; + } +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/control_depend.h b/mindspore/ccsrc/optimizer/control_depend.h new file mode 100644 index 00000000000..2a51a247180 --- /dev/null +++ b/mindspore/ccsrc/optimizer/control_depend.h @@ -0,0 +1,28 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_CONTROL_DEPEND_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_CONTROL_DEPEND_H_ + +#include "ir/anf.h" + +namespace mindspore { +namespace opt { +// Automatically adding control depend based on effect order and side effect analysis. +void AddControlDepend(const FuncGraphPtr& graph); +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_CONTROL_DEPEND_H_ diff --git a/mindspore/ccsrc/optimizer/cse.cc b/mindspore/ccsrc/optimizer/cse.cc new file mode 100644 index 00000000000..82050f61088 --- /dev/null +++ b/mindspore/ccsrc/optimizer/cse.cc @@ -0,0 +1,186 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/cse.h" +#include +#include +#include +#include "./common.h" + +namespace mindspore { +/* namespace to support opt */ +namespace opt { +using mindspore::abstract::AbstractBase; +using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractFunctionPtr; + +BasePtr AbsOf(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto node_abs = node->abstract(); + // in testcase: TestOptOpt.CSE, node->abstract() is null; + if (node_abs == nullptr) { + return kAnyValue; + } + + return node_abs; +} + +namespace { +void BuildOrderGroup(const FuncGraphManagerPtr manager, std::vector *const order_group, + std::unordered_map> *groups) { + MS_EXCEPTION_IF_NULL(order_group); + + std::unordered_map hashes; + for (FuncGraphPtr fg : manager->func_graphs()) { + MS_EXCEPTION_IF_NULL(fg); + std::vector toposet = TopoSort(fg->get_return()); + for (auto node : toposet) { + MS_EXCEPTION_IF_NULL(node); + if (hashes.find(node) != hashes.end()) { + continue; + } + + std::size_t h = 0; + if (node->isa()) { + ValueNodePtr value_node = node->cast(); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + h = hash_combine(value->hash(), (AbsOf(value_node)->hash())); + } else if (node->isa()) { + auto cnode = node->cast(); + auto &inputs = cnode->inputs(); + size_t init = 0; + h = std::accumulate(inputs.begin(), inputs.end(), init, [&hashes](std::size_t hash, const AnfNodePtr &node_in) { + return hash_combine(hash, hashes[node_in]); + }); + } else if (node->isa()) { + h = node->hash(); + } else { + MS_LOG(ERROR) << "Unknow node type"; + } + + hashes[node] = h; + if (groups->find(h) == groups->end()) { + std::vector innervec({node}); + (*groups)[h] = innervec; + order_group->emplace_back(h); + } else { + (*groups)[h].push_back(node); + } + } + } +} +} // namespace + +bool CSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const { + MS_EXCEPTION_IF_NULL(main); + MS_EXCEPTION_IF_NULL(node); + + bool replace = false; + if (main->isa() && node->isa()) { + auto main_value = GetValueNode(main); + auto node_value = GetValueNode(node); + replace = (AbsOf(main) == AbsOf(node)) && (*main_value == *node_value); + } else if (main->isa() && node->isa()) { + auto c_main = main->cast(); + auto c_node = node->cast(); + const auto &inp1 = c_main->inputs(); + const auto &inp2 = c_node->inputs(); + if (inp1.size() == inp2.size()) { + bool appsame = true; + for (size_t j = 0; j < inp1.size(); j++) { + MS_EXCEPTION_IF_NULL(inp1[j]); + MS_EXCEPTION_IF_NULL(inp2[j]); + if (!(*inp1[j] == *inp2[j])) { + // Handle the case of two different Tensor, but with the same value + if (IsValueNode(inp1[j]) && IsValueNode(inp2[j])) { + auto tensor1 = GetValueNode(inp1[j]); + auto tensor2 = GetValueNode(inp2[j]); + if (tensor1->ValueEqual(*tensor2)) { + continue; + } + } + appsame = false; + break; + } + } + replace = appsame; + } + } + return replace; +} + +bool CSE::DoReplace(const FuncGraphManagerPtr manager, const std::vector &order_group, + std::unordered_map> *groups) const { + bool changes = false; + std::set clear_set; + for (auto &h : order_group) { + std::vector &group = (*groups)[h]; + // If there are more than 2 node in that group, they may be same common expression can be eliminated. + if (group.size() > 1) { + for (size_t k = 0; k < group.size() - 1; k++) { + AnfNodePtr main = group[k]; + MS_EXCEPTION_IF_NULL(main); + + // When all node in group has been replaced + // or a valuenode node, skip compare in group + if ((k + 1 + clear_set.size() == group.size()) || (k > 0 && main->isa())) { + break; + } + + // skip node has been replaced + if (clear_set.find(k) != clear_set.end()) { + continue; + } + + // Compare with rest elements in this group. + for (size_t i = k + 1; i < group.size(); i++) { + auto node = group[i]; + MS_EXCEPTION_IF_NULL(node); + + if (clear_set.find(i) != clear_set.end()) { + continue; + } + if (main->func_graph() != node->func_graph()) { + continue; + } + if (CheckReplace(node, main)) { + changes = true; + (void)manager->Replace(node, main); + (void)clear_set.insert(i); + } + } + } + clear_set.clear(); + } + } + + return changes; +} + +bool CSE::Cse(const FuncGraphPtr root, const FuncGraphManagerPtr manager) const { + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(root); + + std::unordered_map> groups; + std::vector order_group; + BuildOrderGroup(manager, &order_group, &groups); + return DoReplace(manager, order_group, &groups); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/cse.h b/mindspore/ccsrc/optimizer/cse.h new file mode 100644 index 00000000000..823b24edb7a --- /dev/null +++ b/mindspore/ccsrc/optimizer/cse.h @@ -0,0 +1,58 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_CSE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_CSE_H_ + +#include +#include +#include +#include "ir/anf.h" +#include "ir/manager.h" +#include "optimizer/optimizer.h" + +namespace mindspore { +/* namespace to support opt */ +namespace opt { + +// Common subexpression elimination. +class CSE { + public: + explicit CSE(bool report_changes = true) : report_changes_(report_changes) {} + virtual ~CSE() = default; + + bool operator()(const FuncGraphPtr &root, const OptimizerPtr &optimizer) { + bool chg = Cse(root, optimizer->resource()->manager()); + return chg && report_changes_; + } + + virtual bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const; + + bool Cse(const FuncGraphPtr root, const FuncGraphManagerPtr manager) const; + + private: + bool DoReplace(const FuncGraphManagerPtr manager, const std::vector &order_group, + std::unordered_map> *groups) const; + bool report_changes_; +}; + +BasePtr AbsOf(const AnfNodePtr &node); +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_CSE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass.cc b/mindspore/ccsrc/optimizer/irpass.cc new file mode 100644 index 00000000000..ba78696b38e --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass.cc @@ -0,0 +1,128 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/irpass.h" + +#include + +#include "optimizer/irpass/symbol_resolver.h" +#include "optimizer/irpass/arithmetic_simplify.h" +#include "optimizer/irpass/special_op_eliminate.h" +#include "optimizer/irpass/item_tuple_eliminate.h" +#include "optimizer/irpass/env_item_eliminate.h" +#include "optimizer/irpass/tile_eliminate.h" +#include "optimizer/irpass/cast_eliminate.h" +#include "optimizer/irpass/reshape_eliminate.h" +#include "optimizer/irpass/transpose_eliminate.h" +#include "optimizer/irpass/reduce_eliminate.h" +#include "optimizer/irpass/partial_eliminate.h" +#include "optimizer/irpass/ref_eliminate.h" +#include "optimizer/irpass/merge_addn.h" +#include "optimizer/irpass/branch_culling.h" +#include "optimizer/irpass/gradient_eliminate.h" +#include "optimizer/irpass/minmax_grad.h" +#include "optimizer/irpass/inline.h" +#include "optimizer/irpass/convert.h" +#include "optimizer/irpass/specialize_transform.h" +#include "optimizer/irpass/incorporate_getitem.h" +#include "optimizer/irpass/incorporate_call.h" + +namespace mindspore { +namespace opt { +namespace irpass { +OptimizeIRPassLib::OptimizeIRPassLib() { + arithmetic_simplify_ = MakeSubstitution( + ArithmeticSimplify(), "arithmetic_simplify", + {prim::kPrimScalarAdd, prim::kPrimScalarMul, prim::kPrimTensorAdd, prim::kPrimIdentity, prim::kPrimMomentum}); + special_op_eliminate_ = MakeSubstitution(SpecialOpEliminater(), "special_op_eliminate", + {prim::kPrimInsertGradientOf, prim::kPrimPrintShapeType, + prim::kPrimGetRefKey, prim::kPrimMirror, prim::kPrimVirtualDiv}); + zero_like_fill_zero_ = MakeSubstitution(ZeroLikeFillZero(), "zero_like_fill_zero", prim::kPrimZerosLikeTensor); + + // ops eliminate + item_tuple_eliminate_ = + MakeSubstitution(ItemTupleEliminater(), "item_tuple_eliminate", {prim::kPrimTupleGetItem, prim::kPrimTupleSetItem}); + tile_eliminate_ = MakeSubstitution(TileMultiplyByOne(), "tile_eliminate", prim::kPrimTile); + cast_eliminate_ = MakeSubstitution(CastEliminater(), "cast_eliminate", prim::kPrimCast); + reshape_eliminate_ = MakeSubstitution(ReshapeEliminater(), "reshape_eliminate", prim::kPrimReshape); + transpose_eliminate_ = MakeSubstitution(TransposeSameIOEliminater(), "transpose_eliminate", prim::kPrimTranspose); + reduce_eliminate_ = MakeSubstitution( + ReduceOneEliminater(), "reduce_eliminate", + {prim::kPrimReduceMean, prim::kPrimReduceAll, prim::kPrimReduceSum, prim::kPrimReduceMax, prim::kPrimReduceMin}); + partial_eliminate_ = MakeSubstitution(PartialEliminater(), "partial_eliminate", IsCNodeDup); + same_eliminate_ = MakeSubstitution(SameEliminater(), "same_eliminate", prim::kPrimSameTypeShape); + reset_defer_inline_ = MakeSubstitution(ResetDeferInline(), "reset_defer_inline", IsValueNode); + + // Env Item Eliminate + new_env_get_item_ = MakeSubstitution(NewEnvGetItem(), "new_env_get_item", prim::kPrimEnvGetItem); + add_env_get_item_ = MakeSubstitution(AddEnvGetItem(), "add_env_get_item", prim::kPrimEnvGetItem); + env_get_set_item_ = MakeSubstitution(EnvGetSetItem(), "env_get_set_item", prim::kPrimEnvGetItem); + incorporate_env_getitem_ = + MakeSubstitution(IncorporateEnvGetitem(), "incorporate_env_get_item", prim::kPrimEnvGetItem); + incorporate_env_getitem_switch_ = + MakeSubstitution(IncorporateEnvGetitemSwitch(), "incorporate_env_getitem_switch", prim::kPrimEnvGetItem); + + // Ref eliminate + make_ref_eliminate_ = MakeSubstitution(MakeRefEliminater(), "make_ref_eliminate", prim::kPrimMakeRef); + get_make_ref_eliminate_ = + MakeSubstitution(GetMakeRefEliminater(), "get_make_ref_eliminate", {prim::kPrimGetRefKey, prim::kPrimGetRefValue}); + replace_refkey_by_param_ = MakeSubstitution(ReplaceRefkeyByParam(), "replace_refkey_by_param", IsValueNode); + + // Gradient transforms + expand_jprim_ = MakeSubstitution(ExpandJPrim(), "expand_jprim", prim::kPrimJ); + stop_gradient_eliminate_ = + MakeSubstitution(StopGradientEliminater(), "stop_gradient_eliminate", prim::kPrimStopGradient); + minmaximum_grad_ = MakeSubstitution(MinMaximumGrad(), "minmaximum_grad", prim::kPrimTupleGetItem); + + // branch culling + switch_simplify_ = MakeSubstitution(SwitchSimplify(), "switch_simplify", prim::kPrimSwitch); + float_tuple_getitem_switch_ = + MakeSubstitution(FloatTupleGetItemSwitch(), "float_tuple_getitem_switch", prim::kPrimTupleGetItem); + float_env_getitem_switch_ = + MakeSubstitution(FloatEnvGetItemSwitch(), "float_env_getitem_switch", prim::kPrimEnvGetItem); + convert_switch_replacement_ = MakeSubstitution(ConvertSwitchReplacement(), "convert_switch_replacement", IsCNodeDup); + + // Addn + merge_addn_ = MakeSubstitution(MergeAddN(), "merge_addn", prim::kPrimAddN); + addn_zero_filter_ = MakeSubstitution(AddNZeroFilter(), "addn_zero_filter", prim::kPrimAddN); + + // inline + inline_ = MakeSubstitution(Inliner(), "inline", IsCNodeGraph); + replace_applicator_ = MakeSubstitution(ReplaceApplicator(), "replace_applicator", IsValueNode); + specialize_transform_ = MakeSubstitution(SpecializeOnGraphArguments(), "specialize_transform", IsCNodeGraph); + + // Incorporation + incorporate_getitem_ = MakeSubstitution(IncorporateGetitem(), "incorporate_getitem", prim::kPrimTupleGetItem); + incorporate_getitem_switch_ = + MakeSubstitution(IncorporateGetitemSwitch(), "incorporate_getitem_switch", prim::kPrimTupleGetItem); + incorporate_call_ = MakeSubstitution(IncorporateCall(), "incorporate_call", IsCNodeDup); + incorporate_call_switch_ = MakeSubstitution(IncorporateCallSwitch(), "incorporate_call_switch", IsCNodeDup); + + // Virtual Dataset + virtual_dataset_eliminate_ = + MakeSubstitution(VirtualDatasetEliminater(), "virtual_dataset_eliminate", prim::kPrimVirtualDataset); + + // Convert + print_tuple_wrapper_ = MakeSubstitution(PrintTupleWrapper(), "print_tuple_wrapper", prim::kPrimPrint); +} + +ResolveIRPassLib::ResolveIRPassLib() { + resolver_resolve_ = MakeSubstitution(ResolverResolve(), "resolver_resolve", prim::kPrimResolve); + resolver_getattr_ = MakeSubstitution(ResolverGetattr(), "resolver_getattr", prim::kPrimGetAttr); +} +} // namespace irpass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass.h b/mindspore/ccsrc/optimizer/irpass.h new file mode 100644 index 00000000000..c2af344e326 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass.h @@ -0,0 +1,157 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_H_ + +#include + +#include "optimizer/optimizer.h" +#include "optimizer/opt.h" +#include "ir/visitor.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// the collection of irpass for optimie action +class OptimizeIRPassLib { + public: + OptimizeIRPassLib(); + ~OptimizeIRPassLib() = default; + + SubstitutionPtr arithmetic_simplify_; + SubstitutionPtr special_op_eliminate_; + SubstitutionPtr zero_like_fill_zero_; + + // ops eliminate + SubstitutionPtr item_tuple_eliminate_; + SubstitutionPtr tile_eliminate_; + SubstitutionPtr cast_eliminate_; + SubstitutionPtr reshape_eliminate_; + SubstitutionPtr transpose_eliminate_; + SubstitutionPtr reduce_eliminate_; + SubstitutionPtr partial_eliminate_; + SubstitutionPtr same_eliminate_; + SubstitutionPtr reset_defer_inline_; + + // Env Item Eliminate + SubstitutionPtr new_env_get_item_; + SubstitutionPtr add_env_get_item_; + SubstitutionPtr env_get_set_item_; + SubstitutionPtr incorporate_env_getitem_; + SubstitutionPtr incorporate_env_getitem_switch_; + + // Ref eliminate + SubstitutionPtr make_ref_eliminate_; + SubstitutionPtr get_make_ref_eliminate_; + SubstitutionPtr replace_refkey_by_param_; + + // Branch culling + SubstitutionPtr switch_simplify_; + SubstitutionPtr float_tuple_getitem_switch_; + SubstitutionPtr float_env_getitem_switch_; + SubstitutionPtr convert_switch_replacement_; + + // AddN + SubstitutionPtr merge_addn_; + SubstitutionPtr addn_zero_filter_; + + // Gradient irpasses + SubstitutionPtr expand_jprim_; + SubstitutionPtr stop_gradient_eliminate_; + SubstitutionPtr minmaximum_grad_; + + // inline + SubstitutionPtr inline_; + SubstitutionPtr replace_applicator_; + SubstitutionPtr specialize_transform_; + + // Incorporation + SubstitutionPtr incorporate_getitem_; + SubstitutionPtr incorporate_getitem_switch_; + SubstitutionPtr incorporate_call_; + SubstitutionPtr incorporate_call_switch_; + + // virtual dataset + SubstitutionPtr virtual_dataset_eliminate_; + + // Convert + SubstitutionPtr print_tuple_wrapper_; +}; + +// the collection of irpass for resolve action +class ResolveIRPassLib { + public: + ResolveIRPassLib(); + ~ResolveIRPassLib() = default; + + SubstitutionPtr resolver_resolve_; + SubstitutionPtr resolver_getattr_; +}; + +// predicate functions +inline bool IsNode(const AnfNodePtr &) { return true; } + +inline bool IsCNode(const AnfNodePtr &node) { + if (node != nullptr) { + return node->isa(); + } + return false; +} + +inline bool IsVNode(const AnfNodePtr &node) { + if (node != nullptr) { + return node->isa(); + } + return false; +} + +inline bool IsParam(const AnfNodePtr &node) { + if (node != nullptr) { + return node->isa(); + } + return false; +} + +// Check if CNode Input 0 is Func Graph +inline bool IsCNodeGraph(const AnfNodePtr &node) { + if (node == nullptr || !node->isa()) { + return false; + } + + auto inp0 = node->cast()->input(0); + if (IsValueNode(inp0)) { + return true; + } + return false; +} + +// Check if CNode Input 0 is CNode +inline bool IsCNodeDup(const AnfNodePtr &node) { + if (node == nullptr || !node->isa()) { + return false; + } + + auto inp0 = node->cast()->input(0); + if (inp0 != nullptr && inp0->isa()) { + return true; + } + return false; +} +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h new file mode 100644 index 00000000000..8c5610ed1b3 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -0,0 +1,220 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ + +#include +#include +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimScalarMul, 0, X}, {prim::kPrimScalarMul, X, 0} +// {prim::kPrimScalarMul, 1, X}, {prim::kPrimScalarMul, X, 1} +class MultiplyByZeroOrOne : public AnfVisitor { + public: + MultiplyByZeroOrOne() : zero_(MakeValue(0)), one_(MakeValue(1)) {} + ~MultiplyByZeroOrOne() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimScalarMul)(node); + + if (is_zero_) { + return NewValueNode(zero_); + } + if (is_one_) { + return x_; + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (is_one_ || node->isa()) { + x_ = node; + return; + } + + AnfVisitor::Visit(node); + if (!is_one_) { + x_ = node; + } + } + + void Visit(const ValueNodePtr &vnode) override { + auto value = vnode->value(); + if (*value == *zero_) { + is_zero_ = true; + } else if (*value == *one_) { + is_one_ = true; + } + } + + void Reset() { + x_ = nullptr; + is_one_ = false; + is_zero_ = false; + } + + private: + bool is_zero_{false}, is_one_{false}; + ValuePtr zero_, one_; + AnfNodePtr x_{nullptr}; +}; + +// {prim::kPrimScalarAdd, X, 0} +// {prim::kPrimScalarAdd, 0, X} +class AddByZero : public AnfVisitor { + public: + AddByZero() : zero_(MakeValue(0)) {} + ~AddByZero() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimScalarAdd)(node); + + if (is_zero_) { + return x_; + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (node->isa() && *GetValueNode(node) == *zero_) { + is_zero_ = true; + return; + } + + x_ = node; + } + + void Reset() { + x_ = nullptr; + is_zero_ = false; + } + + private: + bool is_zero_{false}; + ValuePtr zero_; + AnfNodePtr x_{nullptr}; +}; + +// {prim::kPrimTensorAdd, {PrimZerosLikeTensor, Y}, X}, +// {prim::kPrimTensorAdd, X, {PrimZerosLikeTensor, Y}} +class TensorAddByZero : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTensorAdd)(node); + + if (is_zero_) { + return x_; + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (IsPrimitive(node, prim::kPrimZerosLikeTensor)) { + is_zero_ = true; + return; + } + + x_ = node; + } + + void Reset() { + x_ = nullptr; + is_zero_ = false; + } + + private: + bool is_zero_{false}; + AnfNodePtr x_{nullptr}; +}; + +// {PrimMomentum, {PrimZerosLikeTensor, X}, Y, Z, Xs} -> {prim::kPrimMakeTuple, Z, Y} +class OptUpdateZeroTensor : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!IsPrimitiveCNode(node, prim::kPrimMomentum) || node->func_graph() == nullptr) { + return nullptr; + } + + // {PrimMomentum, {...}, Y, Z, Xs} + auto &inputs = node->cast()->inputs(); + if (inputs.size() < 4 || !IsPrimitiveCNode(inputs[1], prim::kPrimZerosLikeTensor)) { + return nullptr; + } + auto y = inputs[2]; + auto z = inputs[3]; + + // {PrimZerosLikeTensor, X} + if (inputs[1]->cast()->size() != 2) { + return nullptr; + } + + // {prim::kPrimMakeTuple, Z, Y} + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimMakeTuple), z, y}); + } +}; + +class ArithmeticSimplify { + public: + ArithmeticSimplify() + : multiply_by_zero_or_one_(), + add_by_zero_(), + tensor_add_by_zero_(), + identity_(prim::kPrimIdentity), + opt_update_zero_tensor_() { + eliminaters_.emplace_back(multiply_by_zero_or_one_); + eliminaters_.emplace_back(add_by_zero_); + eliminaters_.emplace_back(tensor_add_by_zero_); + eliminaters_.emplace_back(identity_); + eliminaters_.emplace_back(opt_update_zero_tensor_); + } + ~ArithmeticSimplify() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = eliminater(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; + } + + private: + MultiplyByZeroOrOne multiply_by_zero_or_one_; + AddByZero add_by_zero_; + TensorAddByZero tensor_add_by_zero_; + PrimEliminater identity_; + OptUpdateZeroTensor opt_update_zero_tensor_; + std::vector eliminaters_{}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc new file mode 100644 index 00000000000..7c4ada8b3be --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc @@ -0,0 +1,572 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/irpass/branch_culling.h" + +#include +#include +#include + +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +AnfNodePtr GenerateSwitchNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, const AnfNodePtr &data, + int switch_idx) { + auto switch_node = prim::GetPythonOps("geswitch", "mindspore.ops.functional")->cast(); + std::vector switch_nodes{NewValueNode(switch_node), data, cond}; + auto switch_apply = graph->NewCNode(switch_nodes); + std::vector tuple_getitem_nodes{NewValueNode(prim::kPrimTupleGetItem), switch_apply, + NewValueNode(MakeValue(switch_idx))}; + return graph->NewCNode(tuple_getitem_nodes); +} + +AnfNodePtr GenerateSwitchTrueNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, const AnfNodePtr &data) { + return GenerateSwitchNode(graph, cond, data, 1); +} + +AnfNodePtr GenerateSwitchFalseNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, const AnfNodePtr &data) { + return GenerateSwitchNode(graph, cond, data, 0); +} + +bool InConvertWhiteList(const AnfNodePtr &node, size_t index) { + // The CNode inputs of the following Primitive with index in std::vector should not be guarded by geswitch + // node because it is attribute or ge specific reason. + // Example : when convert CNode(kPrimReduceSum, x, axis), node of index 2 in CNode->inputs is axis which should not be + // converted to switch guarded. + std::vector>> white_list({{prim::kPrimApplyMomentum, {1, 2}}, + {prim::kPrimMomentum, {2, 3}}, + {prim::kPrimStateSetItem, {1}}, + {prim::kPrimEnvGetItem, {1}}, + {prim::kPrimEnvSetItem, {1}}, + {prim::kPrimReduceSum, {2}}, + {prim::kPrimReduceMean, {2}}, + {prim::kPrimReduceAll, {2}}, + {prim::kPrimCast, {2}}, + {prim::kPrimTranspose, {2}}, + {prim::kPrimOneHot, {2}}, + {prim::kPrimGatherV2, {3}}, + {prim::kPrimReshape, {2}}, + {prim::kPrimAssign, {1}}, + {prim::kPrimAssignAdd, {1}}, + {prim::kPrimAssignSub, {1}}, + {prim::kPrimTensorSummary, {1}}, + {prim::kPrimImageSummary, {1}}, + {prim::kPrimScalarSummary, {1}}}); + for (auto &item : white_list) { + auto matched = std::any_of(item.second.begin(), item.second.end(), [&item, &node, &index](size_t idx) { + return IsPrimitiveCNode(node, item.first) && idx == index; + }); + if (matched) { + return true; + } + } + + std::vector adapter_convert_ops = {prim::kPrimDepend, prim::kPrimControlDepend}; + for (auto &item : adapter_convert_ops) { + if (IsPrimitiveCNode(node, item)) { + return true; + } + } + return false; +} + +using NodeInputReplMap = std::unordered_map, AnfNodePtr, PairHasher>; +// replace the nodes which should be changed +void RunSwitchNodeReplace(const FuncGraphManagerPtr &manager, std::vector> nodes_changed, + std::unordered_map repl_node, NodeInputReplMap repl_node_inputs) { + for (auto &node_pair : nodes_changed) { + CNodePtr old_node = node_pair.first; + CNodePtr new_node = node_pair.second; + MS_EXCEPTION_IF_NULL(old_node); + MS_EXCEPTION_IF_NULL(new_node); + for (size_t i = 0; i < old_node->size(); i++) { + auto input = old_node->input(i); + if (repl_node.count(input) != 0) { + new_node->add_input(repl_node[input]); + } else if (repl_node_inputs.count(std::pair(old_node, i)) != 0) { + new_node->add_input(repl_node_inputs[std::pair(old_node, i)]); + } else { + new_node->add_input(input); + } + } + } + + for (auto &item : repl_node) { + if (!manager->Replace(item.first, item.second)) { + MS_LOG(EXCEPTION) << "TransformGraphDependNode replace node failed original:" << item.first->DebugString() + << " to new: " << item.second->DebugString(); + } + } +} + +// trace the node that should add switch and replace them with new nodes in the graph +FuncGraphPtr TransformGraphCondBranchNodes( + const FuncGraphPtr &graph, const AnfNodePtr &cond, + const std::function &generate_func) { + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + + // record the node that has been changed + std::vector> nodes_changed; + // record the node to be replaced + std::unordered_map repl_node; + // record the node input to be replaced + NodeInputReplMap repl_node_inputs; + const AnfNodeSet &nodes = manager->nodes()[graph]; + for (auto &node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto inputs = node->cast()->inputs(); + bool should_replace = false; + // if the apply input does not belong to graph, insert a switch node + for (size_t index = 0; index < inputs.size(); index++) { + auto input_node = inputs[index]; + MS_EXCEPTION_IF_NULL(input_node); + // for some ops input should not guard it with switch + if (InConvertWhiteList(node, index)) { + continue; + } + + // If the input for node is not the graph belonged, or it is an ValueNode. + // Bypass the Primitive node which is inputs[0]. + if ((index >= 1 && inputs[index]->func_graph() != nullptr && inputs[index]->func_graph() != graph) || + ((index >= 1 && inputs[index]->isa()))) { + input_node = generate_func(graph, cond, inputs[index]); + repl_node_inputs[std::pair(node, index)] = input_node; + should_replace = true; + } + if (input_node == nullptr) { + MS_LOG(EXCEPTION) << "generate switch node failed"; + } + } + if (should_replace) { + auto new_node = graph->NewCNode(); + repl_node[node] = new_node; + nodes_changed.emplace_back(node->cast(), new_node); + } + } + RunSwitchNodeReplace(manager, nodes_changed, repl_node, repl_node_inputs); + return graph; +} + +struct SharedOp { + tensor::TensorPtr const_data; + CNodePtr square_ops[2]; + CNodePtr merge_ops[2]; +} MergeNetOutput; + +inline tensor::TensorPtr GetConstData() { return MergeNetOutput.const_data; } +inline void SetConstData(const tensor::TensorPtr &const_value) { MergeNetOutput.const_data = const_value; } + +inline CNodePtr GetSquareOp(int switch_idx) { return MergeNetOutput.square_ops[switch_idx]; } +inline void SetSquareOp(int switch_idx, const CNodePtr &op) { MergeNetOutput.square_ops[switch_idx] = op; } + +inline CNodePtr GetMergeOp(int switch_idx) { return MergeNetOutput.merge_ops[switch_idx]; } +inline void SetMergeOp(int switch_idx, const CNodePtr &op) { MergeNetOutput.merge_ops[switch_idx] = op; } + +inline void ResetSharedOp() { + SetConstData(nullptr); + SetSquareOp(0, nullptr); + SetSquareOp(1, nullptr); + SetMergeOp(0, nullptr); + SetMergeOp(1, nullptr); +} + +tensor::TensorPtr ConstData() { + std::vector shp = {1}; + tensor::TensorPtr const_data = std::make_shared(kInt32->type_id(), shp); + auto *val = static_cast(const_data->data_c(true)); + *val = 0; + return const_data; +} + +CNodePtr SquareOp(const FuncGraphPtr &graph, const AnfNodePtr &cond, int switch_idx, + const tensor::TensorPtr &const_data) { + auto PrimSquare = prim::GetPythonOps("square", "mindspore.ops.functional")->cast(); + // for the depended node , add two const data to merge the flow ,one for depended node with same switch, + // the other use the opposite + auto ctrl_data = NewValueNode(const_data); + auto ctrl_node = GenerateSwitchNode(graph, cond, ctrl_data, switch_idx); + + std::vector square_nodes{NewValueNode(PrimSquare), ctrl_node}; + auto square_op = graph->NewCNode(square_nodes); + + return square_op; +} + +CNodePtr MergeNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, int switch_idx, + const tensor::TensorPtr &const_data, const CNodePtr &square_op) { + // for the depended node , add two const data to merge the flow ,one for depended node with same switch, + // the other use the opposite + auto oppsite_ctrl_data = NewValueNode(const_data); + auto opposite_ctrl_node = GenerateSwitchNode(graph, cond, oppsite_ctrl_data, 1 - switch_idx); + + std::vector merge_nodes; + auto PrimMerge = prim::GetPythonOps("merge", "mindspore.ops.functional")->cast(); + merge_nodes.push_back(NewValueNode(PrimMerge)); + std::vector make_tuple_nodes{NewValueNode(prim::kPrimMakeTuple), square_op, opposite_ctrl_node}; + merge_nodes.push_back(graph->NewCNode(make_tuple_nodes)); + auto merge_op = graph->NewCNode(merge_nodes); + + return merge_op; +} + +// construct a depend node with merge output node, merge(square_op(switch(ctrl_data)), switch(opposite_ctrl_data)) +// control_depend(output_node, square_op) +AnfNodePtr GenerateSwitchDependNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, const AnfNodePtr &output_node, + int switch_idx) { + tensor::TensorPtr const_data = GetConstData(); + if (const_data == nullptr) { + const_data = ConstData(); + SetConstData(const_data); + } + + CNodePtr square_op = GetSquareOp(switch_idx); + if (square_op == nullptr) { + square_op = SquareOp(graph, cond, switch_idx, const_data); + SetSquareOp(switch_idx, square_op); + } + + CNodePtr merge_op = GetMergeOp(switch_idx); + if (merge_op == nullptr) { + merge_op = MergeNode(graph, cond, switch_idx, const_data, square_op); + SetMergeOp(switch_idx, merge_op); + } + + std::vector control_depend_nodes{NewValueNode(prim::kPrimControlDepend), output_node, square_op}; + auto control_depend_op = graph->NewCNode(control_depend_nodes); + + std::vector depend_nodes{NewValueNode(prim::kPrimDepend), merge_op, control_depend_op}; + auto depend_op = graph->NewCNode(depend_nodes); + + return depend_op; +} + +// construct a merge output and add dependency with the netoutput node from control_depend +// we need to reserve the control_depend node, besides the generated merge node and control_depend node +CNodePtr GenerateSwitchControlDependNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, + const AnfNodePtr &ctrl_dep_node, const AnfNodePtr &ctrl_depend_dst, + int switch_idx) { + auto PrimMerge = prim::GetPythonOps("merge", "mindspore.ops.functional")->cast(); + auto PrimSquare = prim::GetPythonOps("square", "mindspore.ops.functional")->cast(); + std::vector shp = {1}; + tensor::TensorPtr const_data = std::make_shared(kInt32->type_id(), shp); + auto *val = static_cast(const_data->data_c(true)); + *val = 0; + // for the control_depend netoutput node , add two const data to merge the flow ,one for depended node with same + // switch the other use the opposite + auto ctrl_data = NewValueNode(const_data); + auto oppsite_ctrl_data = NewValueNode(const_data); + auto ctrl_node = GenerateSwitchNode(graph, cond, ctrl_data, switch_idx); + auto opposite_ctrl_node = GenerateSwitchNode(graph, cond, oppsite_ctrl_data, 1 - switch_idx); + + std::vector square_nodes{NewValueNode(PrimSquare), ctrl_node}; + auto square_op = graph->NewCNode(square_nodes); + + std::vector merge_nodes; + merge_nodes.push_back(NewValueNode(PrimMerge)); + std::vector make_tuple_nodes{NewValueNode(prim::kPrimMakeTuple), square_op, opposite_ctrl_node}; + merge_nodes.push_back(graph->NewCNode(make_tuple_nodes)); + auto merge_output = graph->NewCNode(merge_nodes); + + std::vector control_depend_nodes{NewValueNode(prim::kPrimControlDepend), ctrl_depend_dst, square_op}; + auto cond_dep_output = graph->NewCNode(control_depend_nodes); + + std::vector depended_make_tuple_nodes{NewValueNode(prim::kPrimMakeTuple), ctrl_dep_node, merge_output, + cond_dep_output}; + return graph->NewCNode(depended_make_tuple_nodes); +} + +// generate switch nodes for true graph node inputs +AnfNodePtr GenerateSwitchDependTrueNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, const AnfNodePtr &data) { + // for switch op ,the output is a tuple ,0-th is false_branch, 1-th is true branch + return GenerateSwitchDependNode(graph, cond, data, 1); +} + +// generate switch nodes for false graph node inputs +AnfNodePtr GenerateSwitchDependFalseNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, const AnfNodePtr &data) { + // for switch op ,the output is a tuple ,0-th is false_branch, 1-th is true branch + return GenerateSwitchDependNode(graph, cond, data, 0); +} + +// generate switch nodes for true graph node inputs +CNodePtr GenerateSwitchControlDependTrueNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, + const AnfNodePtr &con_input, const AnfNodePtr &output) { + // for switch op ,the output is a tuple ,0-th is false_branch, 1-th is true branch + return GenerateSwitchControlDependNode(graph, cond, con_input, output, 1); +} + +// generate switch nodes for false graph node inputs +CNodePtr GenerateSwitchControlDependFalseNode(const FuncGraphPtr &graph, const AnfNodePtr &cond, + const AnfNodePtr &con_input, const AnfNodePtr &output) { + // for switch op ,the output is a tuple ,0-th is false_branch, 1-th is true branch + return GenerateSwitchControlDependNode(graph, cond, con_input, output, 0); +} + +// to judge if the node used in ControlDepend is a net output node +bool IsNetOutputNode(const FuncGraphManagerPtr &manager, const AnfNodePtr &node) { + auto uses = manager->node_users()[node]; + bool is_output_node = true; + for (auto &item : uses) { + if (IsPrimitiveCNode(item.first, prim::kPrimControlDepend) || IsPrimitiveCNode(item.first, prim::kPrimDepend)) { + continue; + } + is_output_node = false; + break; + } + return is_output_node; +} + +// generate node for Depended MakeTuple +void GenerateReplNodeForDependMakeTuple( + const AnfNodePtr &depended_node, const FuncGraphPtr &graph, const AnfNodePtr &cond, + const std::shared_ptr> &repl_node, + const std::function &generate_func, + const std::function &gen_ctl_depd_func) { + MS_EXCEPTION_IF_NULL(graph->manager()); + + auto make_tuple_inputs = depended_node->cast()->inputs(); + const size_t make_tuple_begin_idx = 1; + std::vector new_make_tuple_nodes; + bool replace_make_tuple = false; + new_make_tuple_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t idx = make_tuple_begin_idx; idx < make_tuple_inputs.size(); idx++) { + auto depended_tuple_input_node = make_tuple_inputs[idx]; + if (IsPrimitiveCNode(depended_tuple_input_node->cast(), prim::kPrimDepend)) { + new_make_tuple_nodes.push_back(depended_tuple_input_node); + continue; + } + if (IsPrimitiveCNode(depended_tuple_input_node->cast(), prim::kPrimControlDepend)) { + // only when the control depend input is not square op (the op to use as merge output) + auto control_inputs = depended_tuple_input_node->cast()->inputs(); + if (control_inputs.size() != 3) { + MS_LOG(EXCEPTION) << "controldepend input size != 3, got " << control_inputs.size(); + } + // control inputs: primitive, src, dst + auto dst_node = control_inputs[2]; + if (!IsPrimitiveCNode(dst_node, prim::kPrimSquare) && IsNetOutputNode(graph->manager(), dst_node)) { + auto gen_node = gen_ctl_depd_func(graph, cond, make_tuple_inputs[idx], dst_node); + MS_EXCEPTION_IF_NULL(gen_node); + auto tuple_inputs = gen_node->inputs(); + // add depended tuple inputs to new_make_tuple directly + for (size_t i = 1; i < tuple_inputs.size(); i++) { + new_make_tuple_nodes.push_back(tuple_inputs[i]); + } + } + replace_make_tuple = true; + continue; + } + + if (graph->manager()->node_users()[depended_tuple_input_node].size() == 1) { + auto gen_node = generate_func(graph, cond, depended_tuple_input_node); + new_make_tuple_nodes.push_back(gen_node); + replace_make_tuple = true; + continue; + } + + MS_LOG(WARNING) << "depended node being used by others, "; + } + if (replace_make_tuple) { + auto make_tuple_op = graph->NewCNode(new_make_tuple_nodes); + (*repl_node)[depended_node] = make_tuple_op; + } +} + +// generate a replace depend node for a single network output node +void GenerateRepDepend( + const CNodePtr &node, const FuncGraphPtr &graph, const AnfNodePtr &cond, + const std::shared_ptr> &repl_node, + const std::function &generate_func, + const std::function &gen_ctl_depd_func) { + auto inputs = node->inputs(); + if (inputs.size() != 3) { + MS_LOG(EXCEPTION) << "Inputs should be [depend, actual_value, depended_node]."; + } + + std::vector new_depened_inputs; + // Inputs should be [depend, actual_value, depended_node] + auto depended_node = inputs[2]; + new_depened_inputs.push_back(inputs[0]); + new_depened_inputs.push_back(inputs[1]); + // depended node should be make_tuple or a single depended node + if (IsPrimitiveCNode(depended_node, prim::kPrimMakeTuple)) { + GenerateReplNodeForDependMakeTuple(depended_node, graph, cond, repl_node, generate_func, gen_ctl_depd_func); + } else if (IsPrimitiveCNode(depended_node, prim::kPrimControlDepend)) { + // only when the control depend input is not square op (the op to use as merge output) + auto control_inputs = depended_node->cast()->inputs(); + // control inputs: primitive, src, dst + if (control_inputs.size() != 3) { + MS_LOG(EXCEPTION) << "controldepend input size != 3, got " << control_inputs.size(); + } + auto dst_node = control_inputs[2]; + if (!IsPrimitiveCNode(dst_node, prim::kPrimSquare) && IsNetOutputNode(graph->manager(), dst_node)) { + auto gen_node = gen_ctl_depd_func(graph, cond, depended_node, dst_node); + (*repl_node)[depended_node] = gen_node; + } + } else { + // Check if there is only single user for depend_node. + if (graph->manager()->node_users()[depended_node].size() == 1) { + auto gen_node = generate_func(graph, cond, depended_node); + (*repl_node)[depended_node] = gen_node; + } else { + MS_LOG(WARNING) << "depended node being used by others"; + } + } +} + +// generate depend node for netoutput node, to resolve the stream synchronize problem of ge +// traverse all nodes of depend node, find the graph output node , generaete a merge node of (square, const) +// and add control_depend of graph output node and square node. +FuncGraphPtr TransformGraphDependNode( + const FuncGraphPtr &graph, const AnfNodePtr &cond, + const std::function &gen_depend_func, + const std::function &gen_ctl_depd_func) { + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + + ResetSharedOp(); + std::shared_ptr> repl_node = + std::make_shared>(); // record the node to be replaced + const AnfNodeSet &nodes = manager->nodes()[graph]; + for (auto &node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + if (IsPrimitiveCNode(node, prim::kPrimDepend)) { + auto cnode = node->cast(); + if (cnode->size() != 3) { + MS_LOG(EXCEPTION) << "Dependnode input size != 3"; + } + auto depended_node = cnode->input(2); + MS_EXCEPTION_IF_NULL(depended_node); + if (!depended_node->isa()) { + continue; + } + if (IsPrimitiveCNode(depended_node, prim::kPrimDepend)) { + continue; + } + GenerateRepDepend(cnode, graph, cond, repl_node, gen_depend_func, gen_ctl_depd_func); + } + } + ResetSharedOp(); + + for (auto &item : *repl_node) { + if (!manager->Replace(item.first, item.second)) { + MS_LOG(EXCEPTION) << "TransformGraphDependNode replace node failed"; + } + } + + return graph; +} + +FuncGraphPtr TransformGraphCondTrueBranchNodes(const FuncGraphPtr &graph, const AnfNodePtr &cond) { + (void)TransformGraphCondBranchNodes(graph, cond, GenerateSwitchTrueNode); + return TransformGraphDependNode(graph, cond, GenerateSwitchDependTrueNode, GenerateSwitchControlDependTrueNode); +} + +FuncGraphPtr TransformGraphCondFalseBranchNodes(const FuncGraphPtr &graph, const AnfNodePtr &cond) { + (void)TransformGraphCondBranchNodes(graph, cond, GenerateSwitchFalseNode); + return TransformGraphDependNode(graph, cond, GenerateSwitchDependFalseNode, GenerateSwitchControlDependFalseNode); +} + +// judge if the true and false graph output is compatible(they shall have same tuple size) +bool GraphOutputCompatible(const AbstractBasePtr &true_branch_abs, const AbstractBasePtr &false_branch_abs) { + MS_EXCEPTION_IF_NULL(true_branch_abs); + MS_EXCEPTION_IF_NULL(false_branch_abs); + + if (true_branch_abs->isa() && false_branch_abs->isa()) { + abstract::AbstractTuplePtr true_branch_tuple = true_branch_abs->cast(); + abstract::AbstractTuplePtr false_branch_tuple = false_branch_abs->cast(); + if (true_branch_tuple->elements().size() != false_branch_tuple->elements().size()) { + MS_LOG(ERROR) << "true branch size:" << true_branch_tuple->elements().size() + << ", not equal to false banch size:" << false_branch_tuple->elements().size() << " "; + return false; + } + bool all_compatible = true; + for (size_t i = 0; i < true_branch_tuple->elements().size(); i++) { + all_compatible = + all_compatible && GraphOutputCompatible(true_branch_tuple->elements()[i], false_branch_tuple->elements()[i]); + } + return all_compatible; + } + TypePtr true_branch_type = true_branch_abs->BuildType(); + TypePtr false_branch_type = false_branch_abs->BuildType(); + MS_LOG(DEBUG) << "branch output Type equal?" << (*true_branch_type == *false_branch_type) + << " true:" << true_branch_type->ToString() << " false:" << false_branch_type->ToString(); + return (*true_branch_type == *false_branch_type); +} + +AnfNodePtr GenerateMergeNodes(const AnfNodePtr &true_output_node, const AnfNodePtr &false_output_node, + const AbstractBasePtr &true_graph_output_abs, + const AbstractBasePtr &false_graph_output_abs, const AnfNodePtr &cond) { + MS_EXCEPTION_IF_NULL(true_graph_output_abs); + MS_EXCEPTION_IF_NULL(false_graph_output_abs); + MS_EXCEPTION_IF_NULL(cond); + MS_EXCEPTION_IF_NULL(cond->func_graph()); + auto PrimMerge = prim::GetPythonOps("merge", "mindspore.ops.functional")->cast(); + MS_EXCEPTION_IF_NULL(PrimMerge); + + if (!true_graph_output_abs->isa()) { + std::vector merge_nodes; + merge_nodes.push_back(NewValueNode(PrimMerge)); + std::vector make_tuple_nodes{NewValueNode(prim::kPrimMakeTuple), true_output_node, false_output_node}; + merge_nodes.push_back(cond->func_graph()->NewCNode(make_tuple_nodes)); + std::vector tuple_getitem_nodes{NewValueNode(prim::kPrimTupleGetItem), + cond->func_graph()->NewCNode(merge_nodes), NewValueNode(MakeValue(0))}; + return cond->func_graph()->NewCNode(tuple_getitem_nodes); + } else { + abstract::AbstractTuplePtr true_branch_tuple = true_graph_output_abs->cast(); + abstract::AbstractTuplePtr false_branch_tuple = false_graph_output_abs->cast(); + + std::vector make_tuple_nodes; + make_tuple_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t i = 0; i < true_branch_tuple->elements().size(); i++) { + std::vector true_getitem_nodes{NewValueNode(prim::kPrimTupleGetItem), true_output_node, + NewValueNode(MakeValue(SizeToInt(i)))}; + auto true_node = cond->func_graph()->NewCNode(true_getitem_nodes); + std::vector false_getitem_nodes{NewValueNode(prim::kPrimTupleGetItem), false_output_node, + NewValueNode(MakeValue(SizeToInt(i)))}; + auto false_node = cond->func_graph()->NewCNode(false_getitem_nodes); + + auto merge_node = GenerateMergeNodes(true_node, false_node, true_branch_tuple->elements()[i], + false_branch_tuple->elements()[i], cond); + make_tuple_nodes.push_back(merge_node); + } + return cond->func_graph()->NewCNode(make_tuple_nodes); + } +} + +AnfNodePtr TransformMergeBranches(const AnfNodePtr &true_output_node, const AnfNodePtr &false_output_node, + const AbstractBasePtr &true_graph_output_abs, + const AbstractBasePtr &false_graph_output_abs, const AnfNodePtr &cond) { + if (!GraphOutputCompatible(true_graph_output_abs, false_graph_output_abs)) { + MS_LOG(EXCEPTION) << "Switch output branch not compatible, true:" << true_graph_output_abs->ToString() + << "， false:" << false_graph_output_abs->ToString(); + } + return GenerateMergeNodes(true_output_node, false_output_node, true_graph_output_abs, false_graph_output_abs, cond); +} +} // namespace internal +} // namespace irpass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/branch_culling.h b/mindspore/ccsrc/optimizer/irpass/branch_culling.h new file mode 100644 index 00000000000..80d07625ed9 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/branch_culling.h @@ -0,0 +1,237 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_BRANCH_CULLING_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_BRANCH_CULLING_H_ + +#include +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimSwitch, true, X, Y} +// {prim::kPrimSwitch, false, X, Y} +class SwitchSimplify : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + auto getx = [this](const AnfNodePtr &node) -> bool { + this->x_ = node; + return true; + }; + auto gety = [this](const AnfNodePtr &node) -> bool { + this->y_ = node; + return true; + }; + AnfVisitor::Match(prim::kPrimSwitch, {IsValueNode, getx, gety})(node); + + // simplify the switch + if (is_match_) { + if (cond_) { + return x_; + } + return y_; + } + + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (!is_match_ && IsValueNode(node)) { + cond_ = GetValue(GetValueNode(node)); + is_match_ = true; + } + } + + void Reset() { + x_ = nullptr; + y_ = nullptr; + cond_ = false; + is_match_ = false; + } + + private: + bool is_match_{false}, cond_{false}; + AnfNodePtr x_{nullptr}, y_{nullptr}; +}; + +// {prim::kPrimTupleGetItem, {prim::kPrimSwith, X0, X1, X2}, C} => +// {prim::kPrimSwith, X0, {prim::kPrimTupleGetItem, X1, C}, {prim::kPrimTupleGetItem, X2, C}} +class FloatTupleGetItemSwitch : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsCNode, IsVNode})(node); + + auto fg = node->func_graph(); + if (Xs_.empty() || c_ == nullptr || fg == nullptr) { + return nullptr; + } + + auto true_node = fg->NewCNode({NewValueNode(prim::kPrimTupleGetItem), Xs_[1], c_}); + auto false_node = fg->NewCNode({NewValueNode(prim::kPrimTupleGetItem), Xs_[2], c_}); + + return fg->NewCNode({NewValueNode(prim::kPrimSwitch), Xs_[0], true_node, false_node}); + } + + void Visit(const CNodePtr &cnode) override { + // {prim::kPrimSwith, X1, X2, X3} + if (!IsPrimitiveCNode(cnode, prim::kPrimSwitch) || cnode->size() != 4) { + return; + } + + // copy X1, X2, X3 + auto &inputs = cnode->inputs(); + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(Xs_)); + } + + void Visit(const ValueNodePtr &vnode) override { c_ = vnode; } + + void Reset() { + Xs_.clear(); + c_ = nullptr; + } + + private: + AnfNodePtr c_{nullptr}; + std::vector Xs_{}; +}; + +// {prim::kPrimEnvGetItem, {prim::kPrimSwitch, X1, X2, X3}, X4, X5} => +// {prim::kPrimSwitch, X1, {prim::kPrimEnvGetItem, X2, X4, X5}, {prim::kPrimEnvGetItem, X3, X4, X5}} +class FloatEnvGetItemSwitch : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + is_match_ = false; + AnfVisitor::Match(prim::kPrimEnvGetItem, {IsCNode, IsNode, IsNode})(node); + if (!is_match_) { + return nullptr; + } + + // {prim::kPrimEnvGetItem, {...}, X4, X5} + auto cnode = node->cast(); + auto sw_node = cnode->input(1)->cast(); + auto x4 = cnode->input(2); + auto x5 = cnode->input(3); + + is_match_ = false; + AnfVisitor::Match(prim::kPrimSwitch, {IsNode, IsNode, IsNode})(sw_node); + if (!is_match_) { + return nullptr; + } + + // {prim::kPrimSwitch, X1, X2, X3} + auto x1 = sw_node->input(1); + auto x2 = sw_node->input(2); + auto x3 = sw_node->input(3); + + auto fg = node->func_graph(); + if (fg == nullptr) { + return nullptr; + } + + auto true_node = fg->NewCNode({NewValueNode(prim::kPrimEnvGetItem), x2, x4, x5}); + auto false_node = fg->NewCNode({NewValueNode(prim::kPrimEnvGetItem), x3, x4, x5}); + + return fg->NewCNode({NewValueNode(prim::kPrimSwitch), x1, true_node, false_node}); + } + + void Visit(const AnfNodePtr &) override { is_match_ = true; } + + private: + bool is_match_{false}; +}; + +namespace internal { +FuncGraphPtr TransformGraphCondTrueBranchNodes(const FuncGraphPtr &graph, const AnfNodePtr &cond); +FuncGraphPtr TransformGraphCondFalseBranchNodes(const FuncGraphPtr &graph, const AnfNodePtr &cond); +AnfNodePtr TransformMergeBranches(const AnfNodePtr &true_output_node, const AnfNodePtr &false_output_node, + const AbstractBasePtr &true_graph_output_abs, + const AbstractBasePtr &false_graph_output_abs, const AnfNodePtr &cond); +} // namespace internal + +// {{prim::kPrimSwitch, X, G1, G2}, Xs} +class ConvertSwitchReplacement : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!node->isa() || node->func_graph() == nullptr) { + return nullptr; + } + + Reset(); + auto cnode = node->cast(); + if (cnode->size() < 1) { + return nullptr; + } + + // {prim::kPrimSwitch, X, G1, G2} + AnfVisitor::Match(prim::kPrimSwitch, {IsNode, IsValueNode, IsValueNode})(cnode->input(0)); + if (g2_ == nullptr || g1_->output() == nullptr || g2_->output() == nullptr) { + return nullptr; + } + + auto true_output = g1_->output()->abstract(); + auto false_output = g2_->output()->abstract(); + auto trans_g1 = internal::TransformGraphCondTrueBranchNodes(g1_, x_); + auto trans_g2 = internal::TransformGraphCondFalseBranchNodes(g2_, x_); + + std::vector params; + auto fg = node->func_graph(); + auto cloned_g1 = InlineClone(trans_g1, fg, params); + auto cloned_g2 = InlineClone(trans_g2, fg, params); + + return internal::TransformMergeBranches(cloned_g1, cloned_g2, true_output, false_output, x_); + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + x_ = node; + return; + } + AnfVisitor::Visit(node); + } + + void Visit(const ValueNodePtr &vnode) override { + auto g = GetValueNode(vnode); + if (g1_ == nullptr) { + g1_ = g; + } else { + g2_ = g; + } + } + + void Reset() { + x_ = nullptr; + g1_ = nullptr; + g2_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}; + FuncGraphPtr g1_{nullptr}, g2_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_BRANCH_CULLING_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/cast_eliminate.cc b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.cc new file mode 100644 index 00000000000..c5a6b6672c0 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.cc @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/irpass/cast_eliminate.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" +#include "ir/func_graph.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/parse/python_adapter.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimCast, X, T} +AnfNodePtr CastSameTypeEliminater::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimCast, {IsNode, IsVNode})(node); + + // check pattern match + if (tgt_ == nullptr) { + return nullptr; + } + + // src type check + auto src_type = src_->Type(); + if (src_type == nullptr) { + return nullptr; + } + + if (src_type->isa()) { + src_type = src_type->cast()->element(); + } + + // tgt type check + auto tgt_type = GetValueNode(tgt_); + if (tgt_type->isa()) { + tgt_type = tgt_type->cast()->element(); + } + + if (src_type->type_id() == tgt_type->type_id()) { + return src_; + } + + return nullptr; +} + +void CastSameTypeEliminater::Visit(const AnfNodePtr &node) { + if (src_ == nullptr) { + src_ = node; + } else { + tgt_ = node; + } +} + +// {prim::kPrimCast, {prim::kPrimCast, X, Y}, T} +AnfNodePtr TwoCastEliminater::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimCast, {IsCNode, IsNode})(node); + + if (x_ != nullptr && t_ != nullptr) { + auto cast_op = parse::python_adapter::GetPyFn("mindspore.ops.operations", "Cast")(); + ValuePtr cast = parse::data_converter::PyDataToValue(cast_op); + auto cnode = NewCNode({NewValueNode(cast), x_, t_}, node->func_graph()); + cnode->set_abstract(node->abstract()); + return cnode; + } + return nullptr; +} + +void TwoCastEliminater::Visit(const AnfNodePtr &node) { + if (IsPrimitiveCNode(node, prim::kPrimCast)) { + auto cnode = node->cast(); + // {prim::kPrimCast, X, Y} + if (cnode->size() != 3) { + return; + } + x_ = cnode->input(1); + } else { + t_ = node; + } +} +} // namespace irpass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h new file mode 100644 index 00000000000..734d88cb10f --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h @@ -0,0 +1,81 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimCast, X, T} +class CastSameTypeEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; + void Visit(const AnfNodePtr &node) override; + void Reset() { + src_ = nullptr; + tgt_ = nullptr; + } + + private: + AnfNodePtr src_{nullptr}, tgt_{nullptr}; +}; + +// {prim::kPrimCast, {prim::kPrimCast, X, Y}, T} +class TwoCastEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; + void Visit(const AnfNodePtr &node) override; + void Reset() { + x_ = nullptr; + t_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}, t_{nullptr}; +}; + +class CastEliminater { + public: + CastEliminater() : cast_same_type_eliminater_(), two_cast_eliminater_() {} + ~CastEliminater() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + auto new_node = cast_same_type_eliminater_(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + + new_node = two_cast_eliminater_(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + + return nullptr; + } + + private: + CastSameTypeEliminater cast_same_type_eliminater_; + TwoCastEliminater two_cast_eliminater_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/convert.h b/mindspore/ccsrc/optimizer/irpass/convert.h new file mode 100644 index 00000000000..3049bafb1ee --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/convert.h @@ -0,0 +1,62 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CONVERT_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CONVERT_H_ + +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimPrint, Xs} -> {prim::kPrimPrint, {prim::kPrinMakeTuple, Xs}} +class PrintTupleWrapper : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!IsPrimitiveCNode(node, prim::kPrimPrint)) { + return nullptr; + } + + // already be {prim::kPrimPrint, {prim::kPrinMakeTuple, Xs}} + auto cnode = node->cast(); + if (cnode->size() == 2 && IsPrimitiveCNode(cnode->input(1), prim::kPrimMakeTuple)) { + return nullptr; + } + + std::vector args; + args.push_back(NewValueNode(prim::kPrimMakeTuple)); + + // {prim::kPrimPrint, Xs} + auto &inputs = cnode->inputs(); + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + + // {prim::kPrinMakeTuple, Xs} + auto fg = node->func_graph(); + auto tuple = NewCNode(args, fg); + auto print = GetValueNode(cnode->input(0)); + return NewCNode({NewValueNode(print), tuple}, fg); + } +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CONVERT_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h b/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h new file mode 100644 index 00000000000..ce29b32d14a --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h @@ -0,0 +1,335 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ + +#include +#include +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" +#include "utils/symbolic.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +class EnvGetitemTransform { + public: + EnvGetitemTransform() : cache_() {} + ~EnvGetitemTransform() = default; + + FuncGraphPtr operator()(const FuncGraphPtr &fg, const SymbolicKeyInstancePtr &key, const AnfNodePtr &default_node) { + if (cache_.find(fg) == cache_.end()) { + cache_[fg] = {}; + } + + auto &cache = cache_[fg]; + auto hash_key = std::make_pair(key, default_node); + if (cache.find(hash_key) == cache.end()) { + std::ostringstream ss("env", std::ostringstream::app); + if (key->node() != nullptr) { + ss << key->node()->ToString(); + } + + auto new_fg = TransformableClone(fg, std::make_shared(ss.str())); + auto env = new_fg->output(); + while (IsPrimitiveCNode(env, prim::kPrimEnvSetItem)) { + // {prim::kPrimEnvSetItem, env, symbolickey, value} + auto &inputs = env->cast()->inputs(); + if (inputs.size() != 4 || !IsValueNode(inputs[2])) { + MS_LOG(EXCEPTION) << "It should be SymbolicKeyInstance."; + } + + env = inputs[1]; + auto value = inputs[3]; + auto key2 = GetValueNode(inputs[2]); + if (*key2 == *key) { + new_fg->set_output(value); + cache[hash_key] = new_fg; + cache_[fg] = cache; + return new_fg; + } + } + new_fg->set_output(new_fg->NewCNode({NewValueNode(prim::kPrimEnvGetItem), env, NewValueNode(key), default_node})); + cache[hash_key] = new_fg; + } + + return cache[hash_key]; + } + + private: + std::unordered_map, FuncGraphPtr, PairHasher>> + cache_; +}; +} // namespace internal + +// {prim::kPrimEnvGetItem, C1, C2, Y} -> Y +class NewEnvGetItem : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + auto gety = [this](const AnfNodePtr &node) -> bool { + this->y_ = node; + return true; + }; + + AnfVisitor::Match(prim::kPrimEnvGetItem, {IsValueNode, IsVNode, gety})(node); + if (env_ != nullptr && env_->Len() == 0) { + return y_; + } + return nullptr; + } + + void Visit(const ValueNodePtr &vnode) override { + if (env_ == nullptr) { + env_ = GetValueNode(vnode); + } + } + + void Reset() { + y_ = nullptr; + env_ = nullptr; + } + + private: + AnfNodePtr y_{nullptr}; + EnvInstancePtr env_{nullptr}; +}; + +// {prim::kPrimEnvGetItem, {prim::kPrimEnvAdd, X, Y}, C, Z} -> +// {prim::GetPythonOps("hyper_add"), {prim::kPrimEnvGetItem, X, C, Z}, {prim::kPrimEnvGetItem, Y, C, Z}} +class AddEnvGetItem : public AnfVisitor { + public: + AddEnvGetItem() : PrimHyperAdd_(prim::GetPythonOps("hyper_add")) {} + ~AddEnvGetItem() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + is_match_ = false; + auto IsAddCNode = [](const AnfNodePtr &node) -> bool { + return IsPrimitiveCNode(node, prim::kPrimEnvAdd) && node->cast()->size() == 3; + }; + AnfVisitor::Match(prim::kPrimEnvGetItem, {IsAddCNode, IsVNode, IsNode})(node); + + if (!is_match_ || node->func_graph() == nullptr) { + return nullptr; + } + + // {prim::kPrimEnvGetItem, {...}, C, Z} + auto cnode = node->cast(); + auto inp1 = cnode->input(1)->cast(); + auto c = cnode->input(2); + auto z = cnode->input(3); + + // {prim::kPrimEnvAdd, X, Y} + auto x = inp1->input(1); + auto y = inp1->input(2); + + auto fg = node->func_graph(); + auto xcz = fg->NewCNode({NewValueNode(prim::kPrimEnvGetItem), x, c, z}); + auto ycz = fg->NewCNode({NewValueNode(prim::kPrimEnvGetItem), y, c, z}); + + return fg->NewCNode({NewValueNode(PrimHyperAdd_), xcz, ycz}); + } + + void Visit(const AnfNodePtr &) override { is_match_ = true; } + + private: + bool is_match_{false}; + ValuePtr PrimHyperAdd_; +}; + +// {prim::kPrimEnvGetItem, {prim::kPrimEnvSetItem, X, C1, Y}, C2, Z} +class EnvGetSetItem : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + is_match_ = false; + auto IsSetCNode = [](const AnfNodePtr &node) -> bool { + if (!IsPrimitiveCNode(node, prim::kPrimEnvSetItem)) { + return false; + } + + // {prim::kPrimEnvSetItem, X, C1, Y} + auto &inputs = node->cast()->inputs(); + if (inputs.size() != 4) { + return false; + } + + return IsValueNode(inputs[2]); + }; + AnfVisitor::Match(prim::kPrimEnvGetItem, {IsSetCNode, IsValueNode, IsNode})(node); + + if (!is_match_ || node->func_graph() == nullptr) { + return nullptr; + } + + // {prim::kPrimEnvGetItem, {...}, C2, Z} + auto cnode = node->cast(); + auto inp1 = cnode->input(1)->cast(); + auto key2 = cnode->input(2); + auto c2 = GetValueNode(key2); + auto default_v = cnode->input(3); + + // {prim::kPrimEnvSetItem, X, C1, Y} + auto env = inp1->input(1); + auto c1 = GetValueNode(inp1->input(2)); + auto last_set = inp1->input(3); + + if (*c1 == *c2) { + return last_set; + } + + while (IsPrimitiveCNode(env, prim::kPrimEnvSetItem)) { + // {prim::kPrimEnvSetItem, env, symbolickey, value} + auto &inputs = env->cast()->inputs(); + if (inputs.size() != 4 || !IsValueNode(inputs[2])) { + MS_LOG(EXCEPTION) << "Input 2 should be a SymbolicKeyInstance."; + } + + env = inputs[1]; + last_set = inputs[3]; + auto symbolic_c1 = GetValueNode(inputs[2]); + if (*symbolic_c1 == *c2) { + return last_set; + } + } + + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimEnvGetItem), env, key2, default_v}); + } + + void Visit(const AnfNodePtr &) override { is_match_ = true; } + + private: + bool is_match_{false}; +}; + +// {prim::kPrimEnvGetItem, {G, Xs}, C, Y} +class IncorporateEnvGetitem : public AnfVisitor { + public: + IncorporateEnvGetitem() : env_get_item_transform_() {} + ~IncorporateEnvGetitem() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + is_match_ = false; + auto IsGCNode = [](const AnfNodePtr &node) -> bool { + auto cnode = node->cast(); + if (cnode == nullptr || cnode->size() < 1) { + return false; + } + return IsValueNode(cnode->input(0)); + }; + AnfVisitor::Match(prim::kPrimEnvGetItem, {IsGCNode, IsValueNode, IsNode})(node); + + if (!is_match_) { + return nullptr; + } + + // {prim::kPrimEnvGetItem, {...}, C, Y} + auto cnode = node->cast(); + auto inp1 = cnode->input(1)->cast(); + auto key = GetValueNode(cnode->input(2)); + auto default_v = cnode->input(3); + + // {G, Xs} + auto inputs = inp1->inputs(); + auto fg = GetValueNode(inputs[0]); + auto new_fg = env_get_item_transform_(fg, key, default_v); + + std::vector args; + args.push_back(NewValueNode(new_fg)); + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + + return node->func_graph()->NewCNode(args); + } + + void Visit(const AnfNodePtr &) override { is_match_ = true; } + + private: + bool is_match_{false}; + internal::EnvGetitemTransform env_get_item_transform_; +}; + +// {prim::kPrimEnvGetItem, {{prim::kPrimSwitch, X, G1, G2}, Xs}, C, Y} +class IncorporateEnvGetitemSwitch : public AnfVisitor { + public: + IncorporateEnvGetitemSwitch() : env_get_item_transform_() {} + ~IncorporateEnvGetitemSwitch() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + is_match_ = false; + auto IsSwNode = [](const AnfNodePtr &node) -> bool { + auto cnode = node->cast(); + if (cnode == nullptr || cnode->size() < 1) { + return false; + } + + return IsPrimitiveCNode(cnode->input(0), prim::kPrimSwitch); + }; + AnfVisitor::Match(prim::kPrimEnvGetItem, {IsSwNode, IsValueNode, IsNode})(node); + if (!is_match_ || node->func_graph() == nullptr) { + return nullptr; + } + + // {prim::kPrimEnvGetItem, {...}, C, Y} + auto cnode = node->cast(); + auto inp1 = cnode->input(1)->cast(); + auto key = GetValueNode(cnode->input(2)); + auto default_v = cnode->input(3); + + // {{prim::kPrimSwitch, X, G1, G2}, Xs} + auto inputs = inp1->inputs(); + is_match_ = false; + AnfVisitor::Match(prim::kPrimSwitch, {IsNode, IsValueNode, IsValueNode})(inputs[0]); + if (!is_match_) { + return nullptr; + } + + // {prim::kPrimSwitch, X, G1, G2} + auto sw = inputs[0]->cast(); + auto x = sw->input(1); + auto g1 = GetValueNode(sw->input(2)); + auto g2 = GetValueNode(sw->input(3)); + auto new_g1 = env_get_item_transform_(g1, key, default_v); + auto new_g2 = env_get_item_transform_(g2, key, default_v); + + auto fg = node->func_graph(); + auto new_sw = fg->NewCNode({NewValueNode(prim::kPrimSwitch), x, NewValueNode(new_g1), NewValueNode(new_g2)}); + + std::vector args{new_sw}; + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + + return fg->NewCNode(args); + } + + void Visit(const AnfNodePtr &) override { is_match_ = true; } + + private: + bool is_match_{false}; + internal::EnvGetitemTransform env_get_item_transform_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/gradient_eliminate.cc b/mindspore/ccsrc/optimizer/irpass/gradient_eliminate.cc new file mode 100644 index 00000000000..3347fa9dc0f --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/gradient_eliminate.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/irpass/gradient_eliminate.h" + +#include + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +AnfNodePtr ExpandJPrimitive(const ValueNodePtr &vnode, const pipeline::ResourceBasePtr &resource) { + ScopeGuard scope_guard(vnode->scope()); + + auto newg = ad::Kprim(vnode, resource); + if (newg != nullptr) { + return NewValueNode(newg); + } + + // when find in J failed, try in Jmeta + auto prim = GetValueNode(vnode); + MetaFuncGraphPtr meta = ad::Kmeta(prim, resource); + if (meta != nullptr) { + return NewValueNode(meta); + } + + return nullptr; +} + +bool CheckIfEmbedJFuncGraph(const FuncGraphPtr func_graph) { + // if func graph also contain J FuncGraph, then ignore this funcgraph. ExpandJ innermost graph first; + auto func_graph_manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(func_graph_manager); + return func_graph_manager->func_graph_j_total(func_graph); +} + +AnfNodePtr ExpandJ(const ValueNodePtr &vnode, const pipeline::ResourceBasePtr &resource) { + if (IsValueNode(vnode)) { + ScopeGuard scope_guard(vnode->scope()); + + auto func_graph = GetValueNode(vnode); + MS_LOG(DEBUG) << "Node is ValueNodeGraph, graph: " << func_graph->ToString(); + + // high_order_grad begin; + // if graph also contain J Graph, then ignore this graph. ExpandJ innermost graph first; + if (CheckIfEmbedJFuncGraph(func_graph)) { + MS_LOG(DEBUG) << "Funcgraph: " << func_graph->ToString() << " contains J(funcgraph), will expandJ later"; + return nullptr; + } + // high_order_grad end; + + MS_LOG(DEBUG) << "Funcgraph: " << func_graph->ToString() << " will expandJ now"; + auto newfg = ad::Grad(func_graph, resource); + return NewValueNode(newfg); + } + + if (IsValueNode(vnode)) { + return ExpandJPrimitive(vnode, resource); + } + + return nullptr; +} +} // namespace internal +} // namespace irpass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/gradient_eliminate.h b/mindspore/ccsrc/optimizer/irpass/gradient_eliminate.h new file mode 100644 index 00000000000..651dc3a2f2c --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/gradient_eliminate.h @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_GRADIENT_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_GRADIENT_ELIMINATE_H_ + +#include +#include +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" +#include "common/utils.h" +#include "operator/ops.h" +#include "optimizer/ad/grad.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +AnfNodePtr ExpandJ(const ValueNodePtr &vnode, const pipeline::ResourceBasePtr &resource); +} // namespace internal + +// {prim::kPrimJ, C} +class ExpandJPrim : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { + x_ = nullptr; + AnfVisitor::Match(prim::kPrimJ, {IsVNode})(node); + if (x_ != nullptr) { + TraceManager::DebugTrace(std::make_shared(node->debug_info())); + auto j_node = internal::ExpandJ(x_, optimizer->resource()); + TraceManager::EndTrace(); + return j_node; + } + return nullptr; + } + + void Visit(const ValueNodePtr &node) override { x_ = node; } + + private: + ValueNodePtr x_{nullptr}; +}; + +// stop_gradient(x) ==> x +class StopGradientEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + x_ = nullptr; + AnfVisitor::Match(prim::kPrimStopGradient)(node); + return x_; + } + + void Visit(const AnfNodePtr &node) override { x_ = node; } + + private: + AnfNodePtr x_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_GRADIENT_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/incorporate_call.h b/mindspore/ccsrc/optimizer/irpass/incorporate_call.h new file mode 100644 index 00000000000..5842b7bfd69 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/incorporate_call.h @@ -0,0 +1,208 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_CALL_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_CALL_H_ + +#include +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +class CallOutputTransform { + public: + CallOutputTransform() : cache_() {} + ~CallOutputTransform() = default; + + FuncGraphPtr operator()(const FuncGraphPtr &fg, size_t nargs) { + if (cache_.find(fg) == cache_.end()) { + cache_[fg] = {}; + } + + auto &cache = cache_[fg]; + if (cache.find(nargs) == cache.end()) { + FuncGraphPtr new_fg = TransformableClone(fg, std::make_shared("call")); + + std::vector new_items; + new_items.push_back(new_fg->output()); + for (size_t i = 0; i < nargs; i++) { + new_items.push_back(new_fg->add_parameter()); + } + new_fg->set_output(new_fg->NewCNode(new_items)); + + cache[nargs] = new_fg; + } + return cache[nargs]; + } + + private: + std::unordered_map> cache_; +}; +} // namespace internal + +// {{G, Xs}, Ys} +class IncorporateCall : public AnfVisitor { + public: + IncorporateCall() : call_output_transform_() {} + ~IncorporateCall() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + if (!node->isa() || node->func_graph() == nullptr) { + return nullptr; + } + + auto &inputs = node->cast()->inputs(); + if (inputs[0] == nullptr || !inputs[0]->isa()) { + return nullptr; + } + + AnfVisitor::Visit(inputs[0]); + if (fg_ == nullptr) { + return nullptr; + } + + auto xs_size = Xs_.size(); + auto ys_size = inputs.size() - 1; + auto new_fg = call_output_transform_(fg_, ys_size); + + std::vector args; + args.push_back(NewValueNode(new_fg)); + + if (xs_size > 0) { + (void)args.insert(args.end(), Xs_.begin(), Xs_.end()); + } + + if (ys_size > 0) { + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + } + + return node->func_graph()->NewCNode(args); + } + + void Visit(const CNodePtr &cnode) override { + // {G, Xs} + if (cnode->size() < 1 || !IsValueNode(cnode->input(0))) { + return; + } + + auto &inputs = cnode->inputs(); + fg_ = GetValueNode(inputs[0]); + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(Xs_)); + } + + void Reset() { + Xs_.clear(); + fg_ = nullptr; + } + + private: + FuncGraphPtr fg_; + std::vector Xs_{}; + internal::CallOutputTransform call_output_transform_; +}; + +// {{{prim::kPrimSwitch, X, G1, G2}, Xs}, Ys} +class IncorporateCallSwitch : public AnfVisitor { + public: + IncorporateCallSwitch() : call_output_transform_() {} + ~IncorporateCallSwitch() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + if (!node->isa() || node->func_graph() == nullptr) { + return nullptr; + } + + // {{...}, Ys} + auto &inputs = node->cast()->inputs(); + if (inputs[0] == nullptr || !inputs[0]->isa()) { + return nullptr; + } + + // {{...}, Xs} + auto &inputs_x = inputs[0]->cast()->inputs(); + if (inputs_x[0] == nullptr || !inputs_x[0]->isa()) { + return nullptr; + } + + // {prim::kPrimSwitch, X, G1, G2} + AnfVisitor::Match(prim::kPrimSwitch, {IsNode, IsValueNode, IsValueNode})(inputs_x[0]); + if (g2_ == nullptr) { + return nullptr; + } + + auto fg = node->func_graph(); + auto xs_size = inputs_x.size() - 1; + auto ys_size = inputs.size() - 1; + auto new_g1 = call_output_transform_(g1_, ys_size); + auto new_g2 = call_output_transform_(g2_, ys_size); + auto sw_node = fg->NewCNode({NewValueNode(prim::kPrimSwitch), x_, NewValueNode(new_g1), NewValueNode(new_g2)}); + + std::vector args{sw_node}; + if (xs_size > 0) { + (void)args.insert(args.end(), inputs_x.begin() + 1, inputs_x.end()); + } + if (ys_size > 0) { + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + } + + return fg->NewCNode(args); + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + x_ = node; + return; + } + AnfVisitor::Visit(node); + } + + void Visit(const ValueNodePtr &vnode) override { + auto g = GetValueNode(vnode); + if (g1_ == nullptr) { + g1_ = g; + } else { + g2_ = g; + } + } + + void Reset() { + x_ = nullptr; + g1_ = nullptr; + g2_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}; + FuncGraphPtr g1_{nullptr}, g2_{nullptr}; + internal::CallOutputTransform call_output_transform_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_CALL_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h b/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h new file mode 100644 index 00000000000..9dc8e7255b3 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h @@ -0,0 +1,203 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H__ + +#include +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +class GetitemTransform { + public: + GetitemTransform() : cache_() {} + ~GetitemTransform() = default; + + FuncGraphPtr operator()(const FuncGraphPtr &fg, int idx) { + if (cache_.find(fg) == cache_.end()) { + cache_[fg] = {}; + } + + auto &cache = cache_[fg]; + if (cache.find(idx) == cache.end()) { + std::ostringstream ss("tp", std::ostringstream::app); + ss << idx; + + auto new_fg = TransformableClone(fg, std::make_shared(ss.str())); + auto output = new_fg->output(); + if (IsPrimitiveCNode(output, prim::kPrimMakeTuple)) { + auto cnode = output->cast(); + auto ids = IntToSize(idx + 1); + // Inputs should be [make_tuple, item1, item2, ...], so have to offset idx in tuple_getitem by 1. + if (ids >= cnode->size()) { + MS_LOG(EXCEPTION) << "index " << ids << " is out of inputs length " << cnode->size(); + } + new_fg->set_output(cnode->input(ids)); + } else { + new_fg->set_output(new_fg->NewCNode({NewValueNode(prim::kPrimTupleGetItem), output, NewValueNode(idx)})); + } + + cache[idx] = new_fg; + } + return cache[idx]; + } + + private: + std::unordered_map> cache_; +}; +} // namespace internal + +// {prim::kPrimTupleGetItem, {G, Xs}, C} +class IncorporateGetitem : public AnfVisitor { + public: + IncorporateGetitem() : getitem_transform_() {} + ~IncorporateGetitem() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsCNode, IsValueNode})(node); + + if (node->func_graph() != nullptr && idx_ >= 0 && fg_ != nullptr) { + auto new_fg = getitem_transform_(fg_, idx_); + (void)args_.insert(args_.begin(), NewValueNode(new_fg)); + return node->func_graph()->NewCNode(args_); + } + return nullptr; + } + + void Visit(const CNodePtr &cnode) override { + if (cnode->size() == 0 || !IsValueNode(cnode->input(0))) { + return; + } + + auto &inputs = cnode->inputs(); + fg_ = GetValueNode(inputs[0]); + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(args_)); + } + + void Visit(const ValueNodePtr &vnode) override { idx_ = GetValue(vnode->value()); } + + void Reset() { + idx_ = -1; + fg_ = nullptr; + args_.clear(); + } + + private: + int idx_{-1}; + FuncGraphPtr fg_{nullptr}; + std::vector args_{}; + internal::GetitemTransform getitem_transform_; +}; + +// {prim::kPrimTupleGetItem, {{prim::kPrimSwitch, X, G1, G2}, Xs}, C} +class IncorporateGetitemSwitch : public AnfVisitor { + public: + IncorporateGetitemSwitch() : getitem_transform_() {} + ~IncorporateGetitemSwitch() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + is_in_get_ = true; + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsCNode, IsValueNode})(node); + is_in_get_ = false; + + auto fg = node->func_graph(); + if (idx_ == -1 || switch_ == nullptr || fg == nullptr) { + return nullptr; + } + + is_in_switch_ = true; + AnfVisitor::Match(prim::kPrimSwitch, {IsNode, IsValueNode, IsValueNode})(switch_); + is_in_switch_ = false; + + if (g2_ == nullptr) { + return nullptr; + } + + auto new_g1 = getitem_transform_(g1_, idx_); + auto new_g2 = getitem_transform_(g2_, idx_); + auto sw_node = fg->NewCNode({NewValueNode(prim::kPrimSwitch), x_, NewValueNode(new_g1), NewValueNode(new_g2)}); + (void)args_.insert(args_.begin(), sw_node); + + return fg->NewCNode(args_); + } + + void Visit(const AnfNodePtr &node) override { + if (is_in_switch_ && x_ == nullptr) { + x_ = node; + return; + } + AnfVisitor::Visit(node); + } + + void Visit(const CNodePtr &cnode) override { + if (is_in_get_ && cnode->size() != 0) { + auto &inputs = cnode->inputs(); + switch_ = inputs[0]; + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(args_)); + } + } + + void Visit(const ValueNodePtr &vnode) override { + if (is_in_get_) { + idx_ = GetValue(vnode->value()); + } + + if (is_in_switch_) { + auto g = GetValueNode(vnode); + if (g1_ == nullptr) { + g1_ = g; + } else { + g2_ = g; + } + } + } + + void Reset() { + x_ = nullptr; + g1_ = nullptr; + g2_ = nullptr; + switch_ = nullptr; + args_.clear(); + is_in_get_ = false; + is_in_switch_ = false; + } + + private: + int idx_{-1}; + AnfNodePtr switch_{nullptr}, x_{nullptr}; + FuncGraphPtr g1_{nullptr}, g2_{nullptr}; + bool is_in_get_{false}, is_in_switch_{false}; + std::vector args_{}; + internal::GetitemTransform getitem_transform_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/inline.h b/mindspore/ccsrc/optimizer/irpass/inline.h new file mode 100644 index 00000000000..a7b6b975bb8 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/inline.h @@ -0,0 +1,210 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INLINE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INLINE_H_ + +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +class ReplaceApplicator : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!IsValueNode(node)) { + return nullptr; + } + + auto fg = GetValueNode(node); + if (fg->has_flag(FUNC_GRAPH_FLAG_DEFER_INLINE)) { + return nullptr; + } + + auto out = fg->output(); + MS_EXCEPTION_IF_NULL(out); + if (!out->isa()) { + return nullptr; + } + + auto &inputs = out->cast()->inputs(); + auto params = fg->parameters(); + + // Exclude first elements of inputs which is fn. + auto input_size = inputs.size(); + auto param_size = params.size(); + if ((input_size == 1 && param_size == 0) || (input_size > 1 && (input_size - 1) == param_size && + std::equal(inputs.begin() + 1, inputs.end(), params.begin()))) { + auto inner = inputs[0]; + if (IsValueNode(inner) || + (IsValueNode(inner) && GetValueNode(inner)->parent() == nullptr)) { + return inner; + } + } + + return nullptr; + } +}; + +using CriterionFuncType = std::function; + +bool IsTrivial(const FuncGraphPtr &fg, AnfNodePtr) { + auto &s = fg->nodes(); + int n_cnode = std::count_if(s.begin(), s.end(), [](const AnfNodePtr &n) { + MS_EXCEPTION_IF_NULL(n); + return n->isa(); + }); + // There is at least one CNode(return, other_node). + return n_cnode <= 2; +} + +bool IsUniqueUse(const FuncGraphPtr &fg, AnfNodePtr) { + auto &users = fg->func_graph_users(); + int n_use = + std::accumulate(users.begin(), users.end(), 0, + [](int sum, const std::pair &item) { return sum + item.second; }); + return n_use == 1; +} + +bool IsInside(FuncGraphPtr, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node->func_graph()); + auto &flags = node->func_graph()->flags(); + if (flags.find("inline_inside") != flags.end()) { + return flags["inline_inside"]; + } + return false; +} + +bool IsCore(const FuncGraphPtr &fg, AnfNodePtr) { + auto &flags = fg->flags(); + if (flags.find("core") != flags.end()) { + return flags["core"]; + } + return false; +} + +bool NoCriterion(FuncGraphPtr, AnfNodePtr) { return true; } + +// {G, Xs} +class InlinerBase : public AnfVisitor { + public: + explicit InlinerBase(std::vector> criterions) : criterions_(criterions) {} + ~InlinerBase() override = default; + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!node->isa()) { + return nullptr; + } + + auto &inputs = node->cast()->inputs(); + if (inputs.size() < 1 || !IsValueNode(inputs[0])) { + return nullptr; + } + + // G + auto fg = GetValueNode(inputs[0]); + if (fg->has_flag(FUNC_GRAPH_FLAG_DEFER_INLINE)) { + return nullptr; + } + + Reset(); + bool is_match = false; + for (auto &criterion : criterions_) { + if (!criterion.first(fg, node)) { + continue; + } + + if (criterion.second && IsRecursive(fg)) { + continue; + } + + is_match = true; + break; + } + + if (!is_match) { + return nullptr; + } + + std::vector params; + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(params)); + + if (IsUniqueUse(fg, nullptr)) { + auto mng = fg->manager(); + MS_EXCEPTION_IF_NULL(mng); + ReplaceParams(mng, params, fg); + auto out_node = fg->output(); + mng->MoveAllCNodeDropGraph(fg, node->func_graph(), inputs[0]->scope()); + return out_node; + } + + return InlineClone(fg, node->func_graph(), params, inputs[0]->scope()); + } + + void ReplaceParams(const FuncGraphManagerPtr &mng, const std::vector &new_params, + const FuncGraphPtr &fg) { + auto params = fg->parameters(); + auto old_size = params.size(); + if (old_size != new_params.size()) { + MS_LOG(EXCEPTION) << "Parameter size not match."; + } + for (size_t i = 0; i < old_size; i++) { + (void)mng->Replace(params[i], new_params[i]); + } + } + + bool IsRecursive(const FuncGraphPtr &fg) { + if (!is_checked_) { + is_checked_ = true; + is_recursive_ = fg->recursive(); + } + return is_recursive_; + } + + void Reset() { + is_checked_ = false; + is_recursive_ = false; + } + + private: + bool is_checked_{false}, is_recursive_{false}; + std::vector> criterions_; +}; + +class Inliner : public InlinerBase { + public: + Inliner() + : InlinerBase({ + {IsUniqueUse, true}, + {IsTrivial, false}, + {IsInside, false}, + {IsCore, false}, + {NoCriterion, true}, + }) {} + ~Inliner() override = default; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INLINE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h new file mode 100644 index 00000000000..2edf5298ad7 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h @@ -0,0 +1,258 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ + +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// (a, b, c, ...)[0] => a +// (a, b, c, ...)[1] => b +// {prim::kPrimTupleGetItem, {prim::kPrimMakeTuple, Xs}, C} +class GetitemEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsCNode, IsVNode})(node); + + if (is_match_) { + return tuple_->input(id_); + } + return nullptr; + } + + void Visit(const CNodePtr &cnode) override { + if (IsPrimitiveCNode(cnode, prim::kPrimMakeTuple)) { + tuple_ = cnode; + } + } + + void Visit(const ValueNodePtr &vnode) override { + if (tuple_ != nullptr && IsValueNode(vnode)) { + id_ = IntToSize(GetValue(vnode->value()) + 1); + if (tuple_->size() > id_) { + is_match_ = true; + } + } + } + + void Reset() { + id_ = 0; + tuple_ = nullptr; + is_match_ = false; + } + + private: + bool is_match_{false}; + size_t id_{0}; + CNodePtr tuple_{nullptr}; +}; + +// setitem((a, b, c, ...), 0, z) => (z, b, c, ...) +// setitem((a, b, c, ...), 1, z) => (a, z, c, ...) +// {prim::kPrimTupleSetItem, {prim::kPrimMakeTuple, Xs}, C, Z} +class SetitemEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleSetItem, {IsCNode, IsVNode, IsNode})(node); + + auto fg = node->func_graph(); + if (fg != nullptr && z_ != nullptr) { + args_[id_] = z_; + return fg->NewCNode(args_); + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (is_match_) { + z_ = node; + return; + } + + AnfVisitor::Visit(node); + } + + void Visit(const CNodePtr &cnode) override { + if (IsPrimitiveCNode(cnode, prim::kPrimMakeTuple)) { + auto &inputs = cnode->inputs(); + (void)std::copy(inputs.begin(), inputs.end(), std::back_inserter(args_)); + } + } + + void Visit(const ValueNodePtr &vnode) override { + if (args_.size() > 0 && IsValueNode(vnode)) { + id_ = IntToSize(GetValue(vnode->value()) + 1); + if (id_ < args_.size()) { + is_match_ = true; + } + } + } + + void Reset() { + id_ = 0; + z_ = nullptr; + is_match_ = false; + args_.clear(); + } + + private: + bool is_match_{false}; + size_t id_{0}; + AnfNodePtr z_{nullptr}; + std::vector args_{}; +}; + +// {prim::kPrimTupleGetItem, {prim::kPrimTupleSetItem, Y, C1, X}, C2} +class GetSetitemEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsCNode, IsVNode})(node); + + auto fg = node->func_graph(); + if (fg != nullptr && key1_ >= 0 && key2_ >= 0) { + if (key1_ == key2_) { + return last_; + } + return fg->NewCNode({op_, tuple_, c2_}); + } + return nullptr; + } + + void Visit(const CNodePtr &cnode) override { + if (IsPrimitiveCNode(cnode, prim::kPrimTupleSetItem)) { + if (cnode->size() < 4) { + return; + } + op_ = cnode->input(0); + tuple_ = cnode->input(1); + last_ = cnode->input(3); + + // key of setitem + is_in_set_ = true; + AnfVisitor::Visit(cnode->input(2)); + is_in_set_ = false; + } + } + + void Visit(const ValueNodePtr &vnode) override { + if (IsValueNode(vnode)) { + auto key = GetValue(vnode->value()); + if (is_in_set_) { + key1_ = key; + } else { + c2_ = vnode; + key2_ = key; + } + } + } + + void Reset() { + key1_ = -1; + key2_ = -1; + op_ = nullptr; + c2_ = nullptr; + last_ = nullptr; + tuple_ = nullptr; + is_in_set_ = false; + } + + private: + bool is_in_set_{false}; + int key1_{-1}, key2_{-1}; + AnfNodePtr op_{nullptr}, tuple_{nullptr}, last_{nullptr}, c2_{nullptr}; +}; + +// {prim::kPrimTupleGetItem, {prim::kPrimDepend, X, Y}, C} -> +// {prim::kPrimDepend, {prim::kPrimTupleGetItem, X, C}, Y} +class GetitemDependReorder : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsCNode, IsValueNode})(node); + if (x_ == nullptr) { + return nullptr; + } + + auto fg = node->func_graph(); + auto item_node = NewCNode({NewValueNode(prim::kPrimTupleGetItem), x_, c_}, fg); + return NewCNode({NewValueNode(prim::kPrimDepend), item_node, y_}, fg); + } + + void Visit(const CNodePtr &cnode) override { + // {prim::kPrimDepend, X, Y} + if (IsPrimitiveCNode(cnode, prim::kPrimDepend) && cnode->size() == 3) { + x_ = cnode->input(1); + y_ = cnode->input(2); + } + } + + void Visit(const ValueNodePtr &vnode) override { c_ = vnode; } + + void Reset() { + x_ = nullptr; + y_ = nullptr; + c_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}, y_{nullptr}, c_{nullptr}; +}; + +class ItemTupleEliminater { + public: + ItemTupleEliminater() + : get_item_eliminater_(), set_item_eliminater_(), get_set_item_eliminater_(), get_item_depend_reorder_() { + eliminaters_.emplace_back(get_item_eliminater_); + eliminaters_.emplace_back(set_item_eliminater_); + eliminaters_.emplace_back(get_set_item_eliminater_); + eliminaters_.emplace_back(get_item_depend_reorder_); + } + ~ItemTupleEliminater() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = eliminater(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; + } + + private: + GetitemEliminater get_item_eliminater_; + SetitemEliminater set_item_eliminater_; + GetSetitemEliminater get_set_item_eliminater_; + GetitemDependReorder get_item_depend_reorder_; + std::vector eliminaters_{}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/merge_addn.h b/mindspore/ccsrc/optimizer/irpass/merge_addn.h new file mode 100644 index 00000000000..7a7c62f6f68 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/merge_addn.h @@ -0,0 +1,202 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_MERGE_ADDN_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_MERGE_ADDN_H_ + +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {PrimAddN, {prim::kPrimMakeTuple, {PrimAddN, {prim::kPrimMakeTuple, Xs}}, Ys}} -> +// {{PrimAddNClass}, {prim::kPrimMakeTuple, Xs, Ys}} +// {PrimAddN, {prim::kPrimMakeTuple, Ys, {PrimAddN, {prim::kPrimMakeTuple, Xs}}}} -> +// {{PrimAddNClass}, {prim::kPrimMakeTuple, Ys, Xs}} +class MergeAddN : public AnfVisitor { + public: + MergeAddN() : PrimAddN_(prim::GetPythonOps("AddN", "mindspore.ops.operations")) {} + ~MergeAddN() override = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { + Reset(); + optimizer_ = optimizer; + is_outer_ = true; + AnfVisitor::Match(prim::kPrimAddN, {IsCNode})(node); + if (!is_match_ || node->func_graph() == nullptr) { + return nullptr; + } + + auto fg = node->func_graph(); + // {PrimAddNClass} + auto addn_node = fg->NewCNode({NewValueNode(PrimAddN_)}); + + // {prim::kPrimMakeTuple, Xs, Ys}, {prim::kPrimMakeTuple, Ys, Xs} + (void)args_.insert(args_.begin(), NewValueNode(prim::kPrimMakeTuple)); + auto make_node = fg->NewCNode(args_); + + return fg->NewCNode({addn_node, make_node}); + } + + void Visit(const CNodePtr &cnode) override { + if (!IsPrimitiveCNode(cnode, prim::kPrimMakeTuple)) { + return; + } + + auto &inputs = cnode->inputs(); + + if (is_outer_) { + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(Ys_)); + + is_outer_ = false; + is_inner_ = true; + + // {prim::kPrimMakeTuple, {PrimAddN, {prim::kPrimMakeTuple, Xs}}, Ys} + AnfVisitor::Match(prim::kPrimAddN, {IsCNode})(inputs[1]); + if (is_match_) { + if (!is_unique(inputs[1])) { + is_match_ = false; + return; + } + (void)Ys_.erase(Ys_.begin()); + (void)std::copy(Xs_.begin(), Xs_.end(), std::back_inserter(args_)); + (void)std::copy(Ys_.begin(), Ys_.end(), std::back_inserter(args_)); + return; + } + + // {prim::kPrimMakeTuple, Ys, {PrimAddN, {prim::kPrimMakeTuple, Xs}}} + AnfVisitor::Match(prim::kPrimAddN, {IsCNode})(inputs.back()); + if (is_match_) { + if (!is_unique(inputs.back())) { + is_match_ = false; + return; + } + Ys_.pop_back(); + (void)std::copy(Ys_.begin(), Ys_.end(), std::back_inserter(args_)); + (void)std::copy(Xs_.begin(), Xs_.end(), std::back_inserter(args_)); + return; + } + + return; + } + + if (is_inner_) { + is_match_ = true; + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(Xs_)); + } + } + + bool is_unique(const AnfNodePtr &node) { + auto mng = optimizer_->resource()->manager(); + auto &node_users = mng->node_users(); + if (node_users.find(node) == node_users.end()) { + return false; + } + + size_t n_use = node_users[node].size(); + return n_use == 1; + } + + void Reset() { + Xs_.clear(); + Ys_.clear(); + args_.clear(); + is_inner_ = false; + is_outer_ = false; + is_match_ = false; + } + + private: + ValuePtr PrimAddN_; + OptimizerPtr optimizer_{nullptr}; + std::vector Xs_{}, Ys_{}, args_{}; + bool is_inner_{false}, is_outer_{false}, is_match_{false}; +}; + +// {PrimAddN, {kPrimMakeTuple, Xs}} +class AddNZeroFilter : public AnfVisitor { + public: + AddNZeroFilter() : PrimAddN_(prim::GetPythonOps("AddN", "mindspore.ops.operations")) {} + ~AddNZeroFilter() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimAddN, {IsCNode})(node); + + if (filtered_Xs_.empty() || node->func_graph() == nullptr) { + return nullptr; + } + + // if only two node in filtered_nodes, {make_tuple, x}. return x. + if (filtered_Xs_.size() == 2) { + return filtered_Xs_[1]; + } + + // if only one node in filtered_nodes, all node is zerolike, return one of the input. + if (filtered_Xs_.size() == 1 && Xs_.size() > 0) { + return Xs_[0]; + } + + if (!has_zero_like_) { + return nullptr; + } + + auto fg = node->func_graph(); + auto addn = fg->NewCNode({NewValueNode(PrimAddN_)}); + auto make_tuple = fg->NewCNode(filtered_Xs_); + return fg->NewCNode({addn, make_tuple}); + } + + void Visit(const CNodePtr &cnode) override { + if (!IsPrimitiveCNode(cnode, prim::kPrimMakeTuple)) { + return; + } + + auto &inputs = cnode->inputs(); + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(Xs_)); + + // {kPrimMakeTuple, X1, X2, ...} + filtered_Xs_.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (auto &x : Xs_) { + if (!IsPrimitiveCNode(x, prim::kPrimZerosLikeTensor)) { + filtered_Xs_.push_back(x); + } else { + has_zero_like_ = true; + } + } + } + + void Reset() { + Xs_.clear(); + filtered_Xs_.clear(); + has_zero_like_ = false; + } + + private: + ValuePtr PrimAddN_; + std::vector filtered_Xs_{}, Xs_{}; + bool has_zero_like_{false}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_MERGE_ADDN_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/minmax_grad.h b/mindspore/ccsrc/optimizer/irpass/minmax_grad.h new file mode 100644 index 00000000000..a426a9fb9bf --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/minmax_grad.h @@ -0,0 +1,110 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_MINMAX_GRAD_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_MINMAX_GRAD_H_ + +#include +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +// check if node is MinimumGrad() or MaximumGrad() +bool IsOriginMaxMinGrad(const AnfNodePtr &node) { + if (!IsPrimitiveCNode(node, prim::kPrimMaximumGrad) && !IsPrimitiveCNode(node, prim::kPrimMinimumGrad)) { + return false; + } + + auto cnode = node->cast(); + auto prim = GetValueNode(cnode->input(0)); + auto x_v = prim->GetAttr("grad_x"); + auto y_v = prim->GetAttr("grad_y"); + if (x_v == nullptr || y_v == nullptr || !x_v->isa() || !y_v->isa()) { + return false; + } + + bool x = GetValue(x_v); + bool y = GetValue(y_v); + return x && y; +} +} // namespace internal + +// {prim::kPrimTupleGetItem, {target_grad, Xs}, C} +class MinMaximumGrad : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {internal::IsOriginMaxMinGrad, IsValueNode})(node); + if (grad_ == nullptr || idx_ < 0 || idx_ > 1 || node->func_graph() == nullptr) { + return nullptr; + } + + // check single use + auto mng = optimizer->resource()->manager(); + auto &users = mng->node_users(); + if (users.find(grad_) == users.end() || users[grad_].size() != 1) { + return nullptr; + } + + // {target_grad, Xs} + auto &inputs = grad_->inputs(); + auto prim = GetValueNode(inputs[0]); + + auto new_prim = std::make_shared(prim->name()); + new_prim->set_attr("grad_x", MakeValue(true)); + new_prim->set_attr("grad_y", MakeValue(true)); + + if (idx_ == 0) { + new_prim->set_attr("grad_y", MakeValue(false)); + } + if (idx_ == 1) { + new_prim->set_attr("grad_x", MakeValue(false)); + } + + std::vector args; + args.push_back(NewValueNode(new_prim)); + (void)args.insert(args.end(), inputs.begin() + 1, inputs.end()); + + auto fg = node->func_graph(); + auto tuple = fg->NewCNode(args); + + return fg->NewCNode({NewValueNode(prim::kPrimTupleGetItem), tuple, NewValueNode(MakeValue(idx_))}); + } + + void Visit(const CNodePtr &cnode) override { grad_ = cnode; } + + void Visit(const ValueNodePtr &vnode) override { idx_ = GetValue(vnode->value()); } + + void Reset() { + idx_ = -1; + grad_ = nullptr; + } + + private: + int idx_{-1}; + CNodePtr grad_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_MINMAX_GRAD_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/partial_eliminate.h b/mindspore/ccsrc/optimizer/irpass/partial_eliminate.h new file mode 100644 index 00000000000..bc8ef9d8f3a --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/partial_eliminate.h @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_PARTIAL_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_PARTIAL_ELIMINATE_H_ + +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {{prim::kPrimPartial, X, Xs}, Ys} -> {X, Xs, Ys} +class PartialEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!node->isa() || node->func_graph() == nullptr) { + return nullptr; + } + + Xs_.clear(); + auto &inputs = node->cast()->inputs(); + Visit(inputs[0]); + + if (Xs_.size() == 0) { + return nullptr; + } + + // {X, Xs, Ys} + std::vector args{}; + (void)std::copy(Xs_.begin(), Xs_.end(), std::back_inserter(args)); + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(args)); + TraceManager::DebugTrace(std::make_shared(node->debug_info())); + auto new_node = node->func_graph()->NewCNode(args); + TraceManager::EndTrace(); + return new_node; + } + + void Visit(const AnfNodePtr &node) override { + if (!IsPrimitiveCNode(node, prim::kPrimPartial)) { + return; + } + + auto &inputs = node->cast()->inputs(); + // {prim::kPrimPartial, X, Xs} + if (inputs.size() < 2) { + return; + } + + // fill Xs + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(Xs_)); + } + + private: + std::vector Xs_{}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_PARTIAL_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/prim_eliminate.h b/mindspore/ccsrc/optimizer/irpass/prim_eliminate.h new file mode 100644 index 00000000000..725c30a6b9d --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/prim_eliminate.h @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_PRIM_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_PRIM_ELIMINATE_H_ + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim, X} +class PrimEliminater : public AnfVisitor { + public: + explicit PrimEliminater(const PrimitivePtr &prim) : prim_(prim) {} + ~PrimEliminater() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + x_ = nullptr; + AnfVisitor::Match(prim_, {IsNode})(node); + return x_; + } + + void Visit(const AnfNodePtr &node) override { x_ = node; } + + private: + AnfNodePtr x_{nullptr}; + PrimitivePtr prim_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_PRIM_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/reduce_eliminate.h b/mindspore/ccsrc/optimizer/irpass/reduce_eliminate.h new file mode 100644 index 00000000000..73dbc152e53 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/reduce_eliminate.h @@ -0,0 +1,160 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REDUCE_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REDUCE_ELIMINATE_H_ + +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/dshape.h" + +namespace mindspore { +namespace opt { +namespace irpass { +using abstract::Shape; +using abstract::ShapePtr; + +// {ReduceLike, X, axis} +class ReduceOneEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + PrimitivePtr prim; + if (IsPrimitiveCNode(node, prim::kPrimReduceMean) || IsPrimitiveCNode(node, prim::kPrimReduceAll) || + IsPrimitiveCNode(node, prim::kPrimReduceSum) || IsPrimitiveCNode(node, prim::kPrimReduceMax) || + IsPrimitiveCNode(node, prim::kPrimReduceMin)) { + prim = GetValueNode(node->cast()->input(0)); + AnfVisitor::Match(prim, {IsNode, IsVNode})(node); + if (!is_axis_one_) { + return nullptr; + } + + // consider keep_dims + auto keep_dims = prim->GetAttr("keep_dims"); + auto is_keep_dims = GetValue(keep_dims); + // {_Reduce, X, axis} -> X + if (is_keep_dims) { + return x_; + } + + // {_Reduce, Tensor} + if (is_tensor_) { + return nullptr; + } + + // {_Reduce, X, axis} -> {Reshape, X, new_shape} + std::vector elements; + for (size_t i = 0; i < x_shape_.size(); i++) { + auto iter = find(axis_.begin(), axis_.end(), i); + if (iter == axis_.end()) { + ValuePtr s = MakeValue(x_shape_[i]); + elements.push_back(s); + } + } + auto new_shape = std::make_shared(elements); + auto reshape_op = prim::GetPythonOps("reshape", "mindspore.ops.functional")->cast(); + return node->func_graph()->NewCNode({NewValueNode(reshape_op), x_, NewValueNode(new_shape)}); + } + + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + if (IsValueNode(node)) { + is_tensor_ = true; + } + // get X's shape + auto x_shape_abs = node->abstract(); + if (x_shape_abs != nullptr) { + auto x_track = x_shape_abs->GetShapeTrack()->cast(); + if (x_track == nullptr) { + return; + } + auto x_shape = x_track->shape(); + (void)std::copy(x_shape.begin(), x_shape.end(), std::back_inserter(x_shape_)); + x_ = node; + } + return; + } + + // check axis + AnfVisitor::Visit(node); + } + + void Visit(const ValueNodePtr &vnode) override { + if (x_shape_.empty()) { + return; + } + + // axis : int + if (IsValueNode(vnode)) { + auto idx = GetValue(vnode->value()); + // axis could be negative + if (idx < 0) { + idx += SizeToInt(x_shape_.size()); + } + if (SizeToInt(x_shape_.size()) > idx && x_shape_[IntToSize(idx)] == 1) { + is_axis_one_ = true; + axis_.push_back(idx); + } + return; + } + + // axis : tuple(int), default () + if (IsValueNode(vnode)) { + auto axis = GetValue>(vnode->value()); + if (axis.empty()) { + return; + } + + auto cmp = std::all_of(axis.cbegin(), axis.cend(), [this](int idx) { + // axis could be negative + if (idx < 0) { + idx += SizeToInt(x_shape_.size()); + } + return SizeToInt(this->x_shape_.size()) > idx && this->x_shape_[IntToSize(idx)] == 1; + }); + if (cmp) { + is_axis_one_ = true; + (void)std::copy(axis.begin(), axis.end(), std::back_inserter(axis_)); + } + } + } + + void Reset() { + axis_.clear(); + x_shape_.clear(); + x_ = nullptr; + is_axis_one_ = false; + is_tensor_ = false; + } + + private: + bool is_axis_one_{false}, is_tensor_{false}; + std::vector axis_{}, x_shape_{}; + AnfNodePtr x_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REDUCE_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h new file mode 100644 index 00000000000..01bdd0906e2 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h @@ -0,0 +1,105 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REF_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REF_ELIMINATE_H_ + +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimMakeRef, X, Y, Z} -> Y +class MakeRefEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + y_ = nullptr; + auto gety = [this](const AnfNodePtr &node) -> bool { + this->y_ = node; + return true; + }; + AnfVisitor::Match(prim::kPrimMakeRef, {IsNode, gety, IsNode})(node); + return y_; + } + + void Visit(const AnfNodePtr &) override {} + + private: + AnfNodePtr y_{nullptr}; +}; + +// {prim::kPrimGetRefKey, {prim::kPrimMakeRef, X, Y, Z}} -> X +// {prim::kPrimGetRefValue, {prim::kPrimMakeRef, X, Y, Z}} -> Y +class GetMakeRefEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + auto cnode = node->cast(); + if (cnode == nullptr || cnode->size() != 2) { + return nullptr; + } + + // {prim::kPrimGetRefKey/Value, {...}} + auto ref = cnode->input(1)->cast(); + if (ref == nullptr || !ref->IsApply(prim::kPrimMakeRef) || ref->size() != 4) { + return nullptr; + } + + // {prim::kPrimMakeRef, X, Y, Z} + if (cnode->IsApply(prim::kPrimGetRefKey)) { + return ref->input(1); + } + + if (cnode->IsApply(prim::kPrimGetRefValue)) { + return ref->input(2); + } + + return nullptr; + } +}; + +// IsValueNode +class ReplaceRefkeyByParam : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { + if (!IsValueNode(node)) { + return nullptr; + } + + auto refkey = GetValueNode(node); + auto resource = std::dynamic_pointer_cast(optimizer->resource()); + MS_EXCEPTION_IF_NULL(resource); + + auto top_graph = resource->func_graph(); + MS_EXCEPTION_IF_NULL(top_graph); + + for (const auto &tnode : top_graph->parameters()) { + auto para = tnode->cast(); + if (para != nullptr && para->name() == refkey->tag()) { + return para; + } + } + return nullptr; + } +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_REF_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h b/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h new file mode 100644 index 00000000000..5e7ce7cdd73 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h @@ -0,0 +1,147 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_RESHAPE_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_RESHAPE_ELIMINATE_H_ + +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "ir/func_graph.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/dshape.h" + +namespace mindspore { +namespace opt { +namespace irpass { +using abstract::Shape; +using abstract::ShapePtr; + +// {reshape_op, X, Shape} +class ReshapeSameShapeEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimReshape, {IsNode, IsVNode})(node); + + // check pattern match + if (shape_ == nullptr) { + return nullptr; + } + + auto src_shape_abs = x_->abstract(); + if (src_shape_abs == nullptr) { + return nullptr; + } + + auto src_shape = src_shape_abs->GetShapeTrack(); + auto tgt_shape = GetValueNode(shape_); + if (src_shape != nullptr && tgt_shape != nullptr && src_shape->isa()) { + auto elements = GetValue>(tgt_shape); + auto shape = src_shape->cast(); + if (shape->shape() == elements) { + return x_; + } + } + + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + x_ = node; + } else { + shape_ = node; + } + } + + void Reset() { + x_ = nullptr; + shape_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}, shape_{nullptr}; +}; + +// {PrimReshape, {PrimReshape, X, Y}, Shape} +class TwoReshapeEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimReshape, {IsCNode, IsNode})(node); + + auto fg = node->func_graph(); + if (fg != nullptr && x_ != nullptr && shape_ != nullptr) { + return fg->NewCNode({NewValueNode(prim_), x_, shape_}); + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (IsPrimitiveCNode(node, prim::kPrimReshape)) { + auto &inputs = node->cast()->inputs(); + // {PrimReshape, X, Y} + if (inputs.size() != 3) { + return; + } + prim_ = GetValueNode(inputs[0]); + x_ = inputs[1]; + } else { + shape_ = node; + } + } + + void Reset() { + prim_ = nullptr; + x_ = nullptr; + shape_ = nullptr; + } + + private: + PrimitivePtr prim_{nullptr}; + AnfNodePtr x_{nullptr}, shape_{nullptr}; +}; + +class ReshapeEliminater { + public: + ReshapeEliminater() : reshape_same_shape_eliminater_(), two_reshape_eliminater_() {} + ~ReshapeEliminater() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + auto new_node = reshape_same_shape_eliminater_(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + + new_node = two_reshape_eliminater_(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + + return nullptr; + } + + private: + ReshapeSameShapeEliminater reshape_same_shape_eliminater_; + TwoReshapeEliminater two_reshape_eliminater_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_RESHAPE_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h new file mode 100644 index 00000000000..2dd27a89c37 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h @@ -0,0 +1,157 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIAL_OP_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIAL_OP_ELIMINATE_H_ + +#include +#include +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +class SpecialOpEliminater { + public: + SpecialOpEliminater() + : insert_gradient_of_(prim::kPrimInsertGradientOf), + print_shape_type_(prim::kPrimPrintShapeType), + get_ref_value_(prim::kPrimGetRefValue), + mirror_(prim::kPrimMirror), + virtual_div_(prim::kPrimVirtualDiv) { + eliminaters_.emplace_back(insert_gradient_of_); + eliminaters_.emplace_back(print_shape_type_); + eliminaters_.emplace_back(get_ref_value_); + eliminaters_.emplace_back(mirror_); + eliminaters_.emplace_back(virtual_div_); + } + ~SpecialOpEliminater() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = eliminater(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; + } + + private: + PrimEliminater insert_gradient_of_, print_shape_type_, get_ref_value_, mirror_, virtual_div_; + std::vector eliminaters_{}; +}; + +// {PrimVirtualDataset, X} -> X +// {PrimVirtualDataset, Xs} -> {prim::kPrimMakeTuple, Xs} +class VirtualDatasetEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!IsPrimitiveCNode(node, prim::kPrimVirtualDataset) || node->func_graph() == nullptr) { + return nullptr; + } + + auto &inputs = node->cast()->inputs(); + if (inputs.size() < 1) { + return nullptr; + } + + std::vector args; + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(args)); + if (args.size() == 1) { + return args.front(); + } + + (void)args.insert(args.begin(), NewValueNode(prim::kPrimMakeTuple)); + + return node->func_graph()->NewCNode(args); + } + + void Visit(const AnfNodePtr &) override {} +}; + +// {prim::kPrimSameTypeShape, X, Y} -> X +class SameEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + x_ = nullptr; + AnfVisitor::Match(prim::kPrimSameTypeShape, {IsNode, IsNode})(node); + return x_; + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + x_ = node; + } + } + + private: + AnfNodePtr x_{nullptr}; +}; + +// Reset defer_inline flag +class ResetDeferInline : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (IsValueNode(node)) { + auto fg = GetValueNode(node); + fg->set_flags(FUNC_GRAPH_FLAG_DEFER_INLINE, false); + } + return nullptr; + } +}; + +// {PrimZerosLikeTensor, Y} -> +// {PrimFill, {PrimDType, Y}, {PrimShape, Y}, 0} +class ZeroLikeFillZero : public AnfVisitor { + public: + ZeroLikeFillZero() + : PrimFill_(prim::GetPythonOps("fill", "mindspore.ops.functional")->cast()), + PrimShape_(prim::GetPythonOps("shape", "mindspore.ops.functional")->cast()), + PrimDType_(prim::GetPythonOps("dtype", "mindspore.ops.functional")->cast()) {} + ~ZeroLikeFillZero() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + y_ = nullptr; + AnfVisitor::Match(prim::kPrimZerosLikeTensor, {IsNode})(node); + if (y_ == nullptr || node->func_graph() == nullptr) { + return nullptr; + } + + auto fg = node->func_graph(); + auto dtype = fg->NewCNode({NewValueNode(PrimDType_), y_}); + auto shape = fg->NewCNode({NewValueNode(PrimShape_), y_}); + + return fg->NewCNode({NewValueNode(PrimFill_), dtype, shape, NewValueNode(MakeValue(0))}); + } + + void Visit(const AnfNodePtr &node) override { y_ = node; } + + private: + AnfNodePtr y_{nullptr}; + PrimitivePtr PrimFill_, PrimShape_, PrimDType_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIAL_OP_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/specialize_transform.h b/mindspore/ccsrc/optimizer/irpass/specialize_transform.h new file mode 100644 index 00000000000..905479df776 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/specialize_transform.h @@ -0,0 +1,147 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIALIZE_TRANSFORM_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIALIZE_TRANSFORM_H_ + +#include +#include +#include +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "ir/manager.h" +#include "ir/func_graph.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +namespace internal { +class SpecializeTransform { + public: + SpecializeTransform() : cache_() {} + ~SpecializeTransform() = default; + + FuncGraphPtr operator()(const FuncGraphPtr &func_graph, std::vector graph_args, + std::vector prim_args) { + if (cache_.count(func_graph) == 0) { + cache_[func_graph] = {}; + } + + auto &cache = cache_[func_graph]; + auto key = std::make_pair(graph_args, prim_args); + if (cache.count(key) == 0) { + auto mng = func_graph->manager(); + MS_EXCEPTION_IF_NULL(mng); + + FuncGraphPtr new_fg = TransformableClone(func_graph, std::make_shared("sp")); + mng->AddFuncGraph(new_fg); + + std::vector params = new_fg->parameters(); + std::vector new_params; + size_t n = graph_args.size(); + for (size_t i = 0; i < n; i++) { + if (graph_args[i] != nullptr) { + auto arg = NewValueNode(graph_args[i]); + (void)mng->Replace(params[i], arg); + continue; + } + if (prim_args[i] != nullptr) { + auto arg = NewValueNode(prim_args[i]); + (void)mng->Replace(params[i], arg); + continue; + } + new_params.push_back(params[i]); + } + + mng->SetParameters(new_fg, new_params); + cache[key] = new_fg; + } + return cache[key]; + } + + private: + std::unordered_map, std::vector>, FuncGraphPtr>> + cache_; +}; +} // namespace internal + +// {G, Xs} +class SpecializeOnGraphArguments : public AnfVisitor { + public: + SpecializeOnGraphArguments() : specialize_transform_() {} + ~SpecializeOnGraphArguments() override = default; + + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + if (!node->isa() || node->func_graph() == nullptr) { + return nullptr; + } + + auto &inputs = node->cast()->inputs(); + if (!IsValueNode(inputs[0])) { + return nullptr; + } + + auto inp0_fg = GetValueNode(inputs[0]); + if (inp0_fg->recursive()) { + return nullptr; + } + + std::vector graph_args; + std::vector prim_args; + std::vector new_xs; + bool hasVNode = false; + for (size_t i = 1; i < inputs.size(); i++) { + if (IsValueNode(inputs[i])) { + auto fg_vnode = GetValueNode(inputs[i]); + graph_args.push_back(fg_vnode); + prim_args.emplace_back(nullptr); + hasVNode = true; + } else if (IsValueNode(inputs[i])) { + auto p_vnode = GetValueNode(inputs[i]); + graph_args.emplace_back(nullptr); + prim_args.push_back(p_vnode); + hasVNode = true; + } else { + graph_args.emplace_back(nullptr); + prim_args.emplace_back(nullptr); + new_xs.push_back(inputs[i]); + } + } + + if (!hasVNode) { + return nullptr; + } + + auto new_fg = specialize_transform_(inp0_fg, graph_args, prim_args); + (void)new_xs.insert(new_xs.begin(), NewValueNode(new_fg)); + + return node->func_graph()->NewCNode(new_xs); + } + + private: + internal::SpecializeTransform specialize_transform_; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIALIZE_TRANSFORM_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/symbol_resolver.h b/mindspore/ccsrc/optimizer/irpass/symbol_resolver.h new file mode 100644 index 00000000000..7b35cf54513 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/symbol_resolver.h @@ -0,0 +1,96 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SYMBOL_RESOLVER_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SYMBOL_RESOLVER_H_ + +#include +#include + +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "ir/visitor.h" +#include "operator/ops.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/parse/python_adapter.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimResolve, Ns, Sym} +class ResolverResolve : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimResolve, {IsVNode, IsVNode})(node); + if (sym_ != nullptr) { + return parse::ResolveSymbol(optimizer->manager(), ns_, sym_, node); + } + return nullptr; + } + + void Visit(const ValueNodePtr &vnode) override { + if (IsValueNode(vnode)) { + ns_ = GetValueNode(vnode); + } else if (ns_ != nullptr && IsValueNode(vnode)) { + sym_ = GetValueNode(vnode); + } + } + + void Reset() { + ns_ = nullptr; + sym_ = nullptr; + } + + private: + parse::NameSpacePtr ns_{nullptr}; + parse::SymbolPtr sym_{nullptr}; +}; + +// {prim::kPrimGetAttr, Ns, Str} +class ResolverGetattr : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimGetAttr, {IsVNode, IsVNode})(node); + if (sym_ != nullptr) { + return parse::ResolveSymbol(optimizer->manager(), ns_, sym_, node); + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (IsValueNode(node)) { + ns_ = GetValueNode(node); + } else if (ns_ != nullptr && IsValueNode(node)) { + auto str = GetValue(GetValueNode(node)); + sym_ = std::make_shared(str); + } + } + + void Reset() { + ns_ = nullptr; + sym_ = nullptr; + } + + private: + parse::NameSpacePtr ns_{nullptr}; + parse::SymbolPtr sym_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SYMBOL_RESOLVER_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/tile_eliminate.h b/mindspore/ccsrc/optimizer/irpass/tile_eliminate.h new file mode 100644 index 00000000000..86ac5bab734 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/tile_eliminate.h @@ -0,0 +1,77 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_TILE_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_TILE_ELIMINATE_H_ + +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// check if node is value tuple and all one. e.g. (1, 1, 1) +// {PrimTile, X, MultiOne} +class TileMultiplyByOne : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTile, {IsNode, IsVNode})(node); + + // check pattern match + if (tuple_ == nullptr) { + return nullptr; + } + + auto value = GetValueNode(tuple_); + auto elements = GetValue>(value); + if (elements.empty()) { + return nullptr; + } + + auto cmp = std::all_of(elements.cbegin(), elements.cend(), [](int i) { return i == 1; }); + if (cmp) { + return x_; + } + + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + x_ = node; + } else { + tuple_ = node; + } + } + + void Reset() { + x_ = nullptr; + tuple_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}, tuple_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_TILE_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/transpose_eliminate.h b/mindspore/ccsrc/optimizer/irpass/transpose_eliminate.h new file mode 100644 index 00000000000..de196ea619f --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/transpose_eliminate.h @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_TRANSPOSE_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_TRANSPOSE_ELIMINATE_H_ + +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// check if node is value tuple and ascends one by one from zero. e.g., (0, 1, 2, 3) +// {PrimTranspose, X, AscendingNums} +class TransposeSameIOEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTranspose, {IsNode, IsVNode})(node); + + // check pattern match + if (tuple_ == nullptr) { + return nullptr; + } + + auto value = GetValueNode(tuple_); + auto elements = GetValue>(value); + if (elements.empty()) { + return nullptr; + } + + int j = 0; + bool cmp = std::all_of(elements.cbegin(), elements.cend(), [&j](int i) { return i == j++; }); + // same IO settings, eliminate this transpose + if (cmp) { + return x_; + } + + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (x_ == nullptr) { + x_ = node; + } else { + tuple_ = node; + } + } + + void Reset() { + x_ = nullptr; + tuple_ = nullptr; + } + + private: + AnfNodePtr x_{nullptr}, tuple_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_TRANSPOSE_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/opt.cc b/mindspore/ccsrc/optimizer/opt.cc new file mode 100644 index 00000000000..a0faa2bf463 --- /dev/null +++ b/mindspore/ccsrc/optimizer/opt.cc @@ -0,0 +1,170 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "optimizer/opt.h" + +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/manager.h" +#include "utils/ordered_set.h" + +#include "utils/log_adapter.h" +#include "optimizer/optimizer.h" + +namespace mindspore { +/* namespace to support opt */ +namespace opt { +SubstitutionPtr MakeSubstitution(const TransformFuncType& transform, const std::string& name, + const PrimitivePtr& prim) { + auto fn = [prim](const AnfNodePtr& node) -> bool { return IsPrimitiveCNode(node, prim); }; + return std::make_shared(transform, name, fn); +} + +SubstitutionPtr MakeSubstitution(const TransformFuncType& transform, const std::string& name, + const std::vector& prims) { + auto fn = [prims](const AnfNodePtr& node) -> bool { + if (!node->isa()) { + return false; + } + + for (auto& prim : prims) { + if (IsPrimitiveCNode(node, prim)) { + return true; + } + } + return false; + }; + + return std::make_shared(transform, name, fn); +} + +SubstitutionPtr MakeSubstitution(const TransformFuncType& transform, const std::string& name, + const PredicateFuncType& predicate) { + return std::make_shared(transform, name, predicate); +} + +AnfNodePtr Substitution::operator()(const OptimizerPtr& optimizer, const AnfNodePtr& node) const { +#ifdef ENABLE_PROFILE + double t = GetTime(); +#endif + AnfNodePtr result = transform_(optimizer, node); +#ifdef ENABLE_PROFILE + if (optimizer != nullptr) { + auto time = GetTime(); + MsProfile::StatTime("substitution." + name_, time - t); + if (result != nullptr) { + MsProfile::StatTime("match." + name_, time - t); + } + } +#endif + + return result; +} + +bool SubstitutionList::ApplyTransform(const OptimizerPtr& optimizer, const AnfNodePtr& root_node, + const SubstitutionPtr& transform) const { + FuncGraphManagerPtr manager = optimizer->manager(); + std::unordered_set seen_node; + std::deque todo{root_node}; + bool changes = false; + + while (!todo.empty()) { + AnfNodePtr node = todo.front(); + todo.pop_front(); + + // check whether this node has been matched. + if (seen_node.find(node) != seen_node.end() || !manager->all_nodes().contains(node)) { + continue; + } + (void)seen_node.insert(node); + + // select nodes that this transform can be applied. + bool is_match = transform->predicate_(node); + + // apply transform on this node + bool change = false; + if (is_match) { + auto ret = (*transform)(optimizer, node); + if (ret != nullptr && ret != node) { + change = true; +#ifdef ENABLE_PROFILE + double t = GetTime(); +#endif + (void)manager->Replace(node, ret); +#ifdef ENABLE_PROFILE + MsProfile::StatTime("replace." + transform->name_, GetTime() - t); +#endif + node = ret; + } + } + + // find success, and add them to todo list + if (IsValueNode(node)) { + todo.push_back(GetValueNode(node)->output()); + } + + if (node->isa()) { + auto& inputs = node->cast()->inputs(); + (void)std::copy(inputs.begin(), inputs.end(), std::back_inserter(todo)); + } + + auto& node_users = manager->node_users(); + if (change && node_users.find(node) != node_users.end()) { + for (auto& use : node_users[node]) { + auto use_node = use.first; + todo.push_back(use_node); + if (seen_node.find(use_node) != seen_node.end()) { + (void)seen_node.erase(use_node); + } + } + } + + changes = changes || change; + } + + return changes; +} + +bool SubstitutionList::operator()(const FuncGraphPtr& func_graph, const OptimizerPtr& optimizer) const { + MS_EXCEPTION_IF_NULL(optimizer); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = optimizer->manager(); + manager->AddFuncGraph(func_graph); + + bool loop = false; + bool changes = false; + + do { + loop = false; + for (auto const& transform : list_) { + auto change = ApplyTransform(optimizer, func_graph->output(), transform); + changes = changes || change; + loop = loop || change; + } + + if (is_once_) { + break; + } + } while (loop); + + return changes; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/opt.h b/mindspore/ccsrc/optimizer/opt.h new file mode 100644 index 00000000000..bd548645f40 --- /dev/null +++ b/mindspore/ccsrc/optimizer/opt.h @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ + +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "operator/ops.h" + +namespace mindspore { +/* namespace to support opt */ +namespace opt { +class Optimizer; + +using OptimizerPtr = std::shared_ptr; +using OptimizerWeakPtr = std::weak_ptr; + +using PredicateFuncType = std::function; +using TransformFuncType = std::function; + +class Substitution { + public: + TransformFuncType transform_{nullptr}; + std::string name_; + PredicateFuncType predicate_{nullptr}; + explicit Substitution(const TransformFuncType &transform, const std::string &name, const PredicateFuncType &predicate) + : transform_(transform), name_(name), predicate_(predicate) {} + ~Substitution() = default; + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) const; +}; + +using SubstitutionPtr = std::shared_ptr; + +SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, const PrimitivePtr &prim); +SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, + const std::vector &prims); +SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, + const PredicateFuncType &predicate); + +class SubstitutionList { + public: + explicit SubstitutionList(const std::vector &patterns, bool is_once = false) + : list_(patterns), is_once_(is_once) {} + ~SubstitutionList() = default; + + bool operator()(const FuncGraphPtr &func_graph, const OptimizerPtr &optimizer) const; + + private: + bool ApplyTransform(const OptimizerPtr &optimizer, const AnfNodePtr &node, const SubstitutionPtr &transform) const; + std::vector list_; + // a flag to mark this list of Substitution can only be executed only once + bool is_once_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ diff --git a/mindspore/ccsrc/optimizer/optimizer.h b/mindspore/ccsrc/optimizer/optimizer.h new file mode 100644 index 00000000000..d821e826cf6 --- /dev/null +++ b/mindspore/ccsrc/optimizer/optimizer.h @@ -0,0 +1,192 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_OPTIMIZER_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_OPTIMIZER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef DEBUG +#include "debug/draw.h" +#include "debug/anf_ir_dump.h" +#endif +#include "optimizer/opt.h" +#include "pipeline/resource.h" +#include "pipeline/action.h" +#include "debug/trace.h" + +namespace mindspore { +namespace opt { +using OptimizeGraphFunc = std::function; + +class OptPassConfig { + public: + explicit OptPassConfig(const OptimizeGraphFunc &func) : func_(func) {} + explicit OptPassConfig(const std::vector &list, bool is_once = false) + : list_(list), is_once_(is_once) {} + OptPassConfig(const std::initializer_list &list, bool is_once = false) + : list_(list), is_once_(is_once) {} + ~OptPassConfig() = default; + + const std::vector &list() const { return list_; } + const OptimizeGraphFunc &func() const { return func_; } + + static OptPassConfig Renormalize() { return OptPassConfig(); } + const bool is_renormalize() const { return is_renormalize_; } + + const bool is_once() const { return is_once_; } + + private: + OptPassConfig() : is_renormalize_(true) {} + + OptimizeGraphFunc func_; + std::vector list_; + bool is_renormalize_{false}; + bool is_once_{false}; +}; + +class OptPass { + public: + explicit OptPass(const OptimizeGraphFunc &func) : pass_func_(func) {} + ~OptPass() = default; + + bool operator()(const FuncGraphPtr &func_graph, const OptimizerPtr &optimizer) const { + return pass_func_(func_graph, optimizer); + } + + static OptPass Renormalize() { return OptPass(); } + const bool is_renormalize() const { return is_renormalize_; } + + private: + OptPass() : is_renormalize_(true) {} + + OptimizeGraphFunc pass_func_; + bool is_renormalize_{false}; +}; +using OptPassGroupMap = std::vector>; + +class Optimizer : public std::enable_shared_from_this { + public: + Optimizer(const std::string &name, const pipeline::ResourceBasePtr &resource_ptr) + : name_(name), resource_(resource_ptr), run_only_once_(false) {} + virtual ~Optimizer() = default; + + void Init(const OptPassGroupMap &passes, bool run_only_once) { + run_only_once_ = run_only_once; + + for (auto &iter : passes) { + const std::string &name = iter.first; + pass_names_.push_back(name); + + const OptPassConfig &config = iter.second; + if (config.is_renormalize()) { + passes_.push_back(OptPass::Renormalize()); + continue; + } + + if (config.list().size() > 0) { + OptimizeGraphFunc func = SubstitutionList(config.list(), config.is_once()); + passes_.push_back(OptPass(func)); + continue; + } + + passes_.push_back(OptPass(config.func())); + } + + if (passes_.size() == 1) { + run_only_once_ = true; + } + } + + static std::shared_ptr MakeOptimizer(const std::string &name, const pipeline::ResourceBasePtr resource_ptr, + const OptPassGroupMap &passes, bool run_only_once = false) { + OptimizerPtr optimizer = std::make_shared(name, resource_ptr); + optimizer->Init(passes, run_only_once); + return optimizer; + } + + FuncGraphPtr step(FuncGraphPtr func_graph, const abstract::AbstractBasePtrList &args_spec, bool use_profile = true) { + // Optimizer step counter; + int counter = 1; + bool changes = true; + + while (changes) { + changes = false; + auto run_runc = [&counter, &func_graph, &args_spec, &changes, use_profile, this]() { + for (size_t i = 0; i < passes_.size(); ++i) { + const OptPass &opt = passes_[i]; + auto opt_func = [&func_graph, &args_spec, &changes, &opt, this]() { + if (opt.is_renormalize()) { + auto resource_ptr = std::dynamic_pointer_cast(resource_); + if (resource_ptr != nullptr) { + func_graph = pipeline::Renormalize(resource_ptr, func_graph, args_spec); + } + } else if (opt(func_graph, shared_from_this())) { + changes = true; + } + }; + use_profile ? (WITH(MsProfile::GetProfile()->Step(pass_names_[i])) opt_func) : opt_func(); +#ifdef DEBUG + MS_LOG(DEBUG) << "" << name_ << " round " << counter << " OptPass " << pass_names_[i] << " end."; + auto fg_name = name_ + "_r" + std::to_string(counter) + "_" + std::to_string(i) + "_" + pass_names_[i]; + func_graph->DumpFuncGraph(fg_name); + DumpIR(fg_name + ".ir", func_graph); + MS_LOG(DEBUG) << "Dump " << pass_names_[i] << " func graph."; +#endif + } + }; + use_profile ? (WITH(MsProfile::GetProfile()->Lap(counter++)) run_runc) : run_runc(); + + if (run_only_once_) { + break; + } + } + + auto keep_root = [&func_graph, this]() { + std::vector func_graphs; + func_graphs.push_back(func_graph); + resource_->manager()->KeepRoots(func_graphs); + }; + use_profile ? WITH(MsProfile::GetProfile()->Step("keep_roots")) keep_root : keep_root(); + return func_graph; + } + + pipeline::ResourceBasePtr resource() const { return resource_; } + FuncGraphManagerPtr manager() const { + if (resource_ != nullptr) { + return resource_->manager(); + } + MS_LOG(EXCEPTION) << "No ResourceBase exists."; + } + + const std::string name() const { return name_; } + + private: + const std::string name_; + pipeline::ResourceBasePtr resource_; + std::vector passes_; + std::vector pass_names_; + bool run_only_once_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_OPTIMIZER_H_ diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_fusion.cc b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_fusion.cc new file mode 100644 index 00000000000..37b6eb42edc --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_fusion.cc @@ -0,0 +1,430 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/allreduce_fusion/allreduce_fusion.h" +#include +#include +#include +#include +#include "utils/log_adapter.h" +#include "parallel/status.h" +#include "ir/func_graph.h" +#include "parallel/step_parallel.h" +#include "parallel/graph_util/node_info.h" +#include "parallel/costmodel_context.h" + +namespace mindspore { +namespace parallel { +std::unordered_set FindCNodesWithPara(const AnfNodePtr& para, uint32_t recursive_times = 0) { + if (recursive_times > MAX_RECURSIVE_CALL_TIMES) { + MS_LOG(EXCEPTION) << "FindCNodesWithPara exceeds max recursive call times! Max recursive call times is " + << MAX_RECURSIVE_CALL_TIMES; + } + MS_EXCEPTION_IF_NULL(para); + MS_EXCEPTION_IF_NULL(para->func_graph()); + FuncGraphManagerPtr manager = para->func_graph()->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto node_set = manager->node_users()[para]; + std::unordered_set cnode_set; + for (auto& node_pair : node_set) { + auto cnode = node_pair.first->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + auto node_prim = GetValueNode(cnode->input(0)); + MS_EXCEPTION_IF_NULL(node_prim); + if (node_prim->name() == DEPEND && node_pair.second != 1) { + continue; + } + if (IsParallelCareNode(cnode) && cnode->operator_info() != nullptr) { + (void)cnode_set.emplace(cnode); + } else { + auto cnode_set_sub = FindCNodesWithPara(node_pair.first, recursive_times + 1); + for (auto& cnode_sub : cnode_set_sub) { + (void)cnode_set.emplace(cnode_sub); + } + } + } + return cnode_set; +} + +Status AllreduceFusion::AddNodeToGraph() { + const auto& parameters = root_graph_->parameters(); + for (auto& parameter : parameters) { + if (!ParameterRequireGrad(parameter)) { + continue; + } + auto cnode_set = FindCNodesWithPara(parameter); + if (cnode_set.empty()) { + continue; + } + for (auto& cnode : cnode_set) { + MS_LOG(DEBUG) << "AddNode " << cnode->DebugString(); + if (allreduce_graph_.AddNode(cnode, parameter) != SUCCESS) { + MS_LOG(ERROR) << "AddNode failed! cnode: " << cnode->DebugString(); + return FAILED; + } + } + } + return SUCCESS; +} + +CNodeCostMap AllreduceFusion::FindCNode(const AnfNodePtr& from, uint32_t recursive_times) const { + if (recursive_times > MAX_RECURSIVE_CALL_TIMES) { + MS_LOG(EXCEPTION) << "FindCNode exceeds max recursive call times! Max recursive call times is " + << MAX_RECURSIVE_CALL_TIMES; + } + MS_EXCEPTION_IF_NULL(from); + std::unordered_map cnode_dist; + if (!from->isa()) { + return cnode_dist; + } + auto cnode = from->cast(); + if (!IsValueNode(cnode->input(0))) { + return cnode_dist; + } + + MS_LOG(DEBUG) << "cnode " << cnode->ToString() << " IsParallelCareNode: " << IsParallelCareNode(cnode) + << " operator_info: " << (cnode->operator_info() != nullptr); + + if (IsParallelCareNode(cnode) && (cnode->operator_info() != nullptr)) { + auto cost = cnode->operator_info()->GetForwardMemoryCostFromCNode(); + MS_LOG(DEBUG) << "cnode " << cnode->DebugString() << " cost: " << cost; + + if (allreduce_graph_.NodeInGraph(cnode)) { + cnode_dist[cnode] = cost; + return cnode_dist; + } else { + auto cnode_dist_next = FindNextCNodes(cnode, recursive_times + 1); + for (auto& ele : cnode_dist_next) { + cnode_dist[ele.first] = cost + ele.second; + } + } + } else { + auto cnode_dist_next = FindNextCNodes(cnode); + for (auto& ele : cnode_dist_next) { + cnode_dist[ele.first] = ele.second; + } + } + return cnode_dist; +} + +CNodeCostMap AllreduceFusion::FindNextCNodes(const CNodePtr& from, uint32_t recursive_times) const { + if (recursive_times > MAX_RECURSIVE_CALL_TIMES) { + MS_LOG(EXCEPTION) << "FindNextCNodes exceeds max recursive call times! Max recursive call times is " + << MAX_RECURSIVE_CALL_TIMES; + } + const auto& from_inputs = from->inputs(); + std::unordered_map dist_map; + MS_LOG(DEBUG) << "from cnode " << from->DebugString() << " has " << from_inputs.size() << " inputs"; + for (auto& input_node : from_inputs) { + auto cnode_dist = FindCNode(input_node, recursive_times + 1); + for (auto& ele : cnode_dist) { + (void)dist_map.emplace(ele); + } + } + return dist_map; +} + +Status AllreduceFusion::AddEdgeToGraph() { + std::unordered_map cnode_state_map; + const auto& cnodes = allreduce_graph_.cnode_set(); + for (auto& cnode : cnodes) { + cnode_state_map[cnode] = 0; + } + const auto& head_cnode = allreduce_graph_.head_cnode(); + std::queue cnode_queue; + cnode_queue.emplace(head_cnode); + cnode_state_map[head_cnode] = 1; + + while (!cnode_queue.empty()) { + const auto cur_cnode = cnode_queue.front(); + cnode_queue.pop(); + cnode_state_map[cur_cnode] = 2; + auto next = FindNextCNodes(cur_cnode); + for (auto& ele : next) { + auto& cnode = ele.first; + auto& dist = ele.second; + if (cnode_state_map[cnode] == 0) { + cnode_queue.emplace(cnode); + cnode_state_map[cnode] = 1; + } + if (allreduce_graph_.AddEdge(cur_cnode, cnode, dist) != SUCCESS) { + MS_LOG(ERROR) << "AddEdge error"; + return FAILED; + } + MS_LOG(DEBUG) << "from " << cur_cnode->DebugString() << ", to " << cnode->DebugString() << " dist " << dist; + } + } + return SUCCESS; +} + +std::vector FindMirror(const AnfNodePtr& para, uint32_t recursive_times = 0) { + if (recursive_times > MAX_RECURSIVE_CALL_TIMES) { + MS_LOG(EXCEPTION) << "FindMirror exceeds max recursive call times! Max recursive call times is " + << MAX_RECURSIVE_CALL_TIMES; + } + MS_EXCEPTION_IF_NULL(para); + MS_EXCEPTION_IF_NULL(para->func_graph()); + FuncGraphManagerPtr manager = para->func_graph()->manager(); + MS_EXCEPTION_IF_NULL(manager); + AnfNodeIndexSet node_set = manager->node_users()[para]; + std::vector cnode_list; + for (auto& node_pair : node_set) { + auto cnode = node_pair.first->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + auto node_prim = GetValueNode(cnode->input(0)); + MS_EXCEPTION_IF_NULL(node_prim); + if (node_prim->name() == CAST) { + auto mirror_cnodes = FindMirror(node_pair.first, recursive_times + 1); + if (mirror_cnodes.empty()) { + MS_LOG(WARNING) << "mirror node after cast not found"; + continue; + } + if (mirror_cnodes.size() > 1) { + MS_LOG(EXCEPTION) << "mirror node after cast number is not 1"; + } + cnode_list.emplace_back(mirror_cnodes[0]); + } + if (node_prim->name() == MIRROR_OPERATOR) { + cnode_list.emplace_back(cnode); + } + } + return cnode_list; +} + +void SetMirrorFusion(const CNodePtr& mirror_cnode, int32_t fusion, const std::string& parameter_name) { + MS_EXCEPTION_IF_NULL(mirror_cnode); + MS_LOG(DEBUG) << "Set Mirror " << mirror_cnode->DebugString() << " fusion " << fusion; + auto node_prim = GetValueNode(mirror_cnode->input(0)); + auto old_value_ptr = node_prim->GetAttr(FUSION); + if (old_value_ptr != nullptr) { + if (old_value_ptr->isa()) { + int32_t old_value = old_value_ptr->cast()->value(); + if (old_value < fusion) { + return; + } + } + } + (void)node_prim->AddAttr(FUSION, MakeValue(std::make_shared(fusion))); + (void)node_prim->AddAttr(PARAMETER, MakeValue(std::make_shared(parameter_name))); +} + +Status FindMirrorAndSetFusion(const AnfNodePtr& para, int32_t fusion) { + auto mirror_cnodes = FindMirror(para); + if (mirror_cnodes.empty()) { + MS_LOG(WARNING) << para->ToString() << " 0 Mirror CNode found."; + return SUCCESS; + } + if (mirror_cnodes.size() > 2) { + for (auto& mirror_cnode : mirror_cnodes) { + MS_EXCEPTION_IF_NULL(mirror_cnode); + MS_LOG(INFO) << mirror_cnode->DebugString(); + } + MS_EXCEPTION_IF_NULL(para); + MS_LOG(ERROR) << para->ToString() << " FindMirror is more than 2. " << mirror_cnodes.size() + << "Mirror CNode found."; + return FAILED; + } + for (auto& mirror_cnode : mirror_cnodes) { + auto parameter_name = ParameterName(para); + SetMirrorFusion(mirror_cnode, fusion, parameter_name); + } + return SUCCESS; +} + +Status FindMirrorAndSetFusion(const std::vector& paras, int32_t fusion) { + for (auto& param_node : paras) { + if (FindMirrorAndSetFusion(param_node, fusion) != SUCCESS) { + MS_LOG(ERROR) << "FindMirrorAndSetFusion failed"; + return FAILED; + } + } + return SUCCESS; +} + +Status AllreduceFusion::SetFusion(const std::vector& cost_map) { + if (cost_map.size() < 2) { + MS_LOG(ERROR) << "cost_map must has at least 2 items, cost_map size is " << cost_map.size(); + return FAILED; + } + int32_t fusion = 1; + for (auto cost_iter = cost_map.end() - 1; cost_iter != cost_map.begin(); --cost_iter) { + auto paras = allreduce_graph_.GetParaByCost(*(cost_iter - 1), *cost_iter); + if (FindMirrorAndSetFusion(paras, fusion) != SUCCESS) { + MS_LOG(ERROR) << "FindMirrorAndSetFusion failed"; + return FAILED; + } + fusion++; + } + return SUCCESS; +} + +std::vector AllreduceFusion::GenerateCostMap(int32_t fusion_times, double tail_percent) const { + double offset = allreduce_graph_.max() * (1 - tail_percent) / (fusion_times - 1); + MS_LOG(DEBUG) << "max = " << allreduce_graph_.max() << ", offset = " << offset; + std::vector cost_map; + double begin = 0; + for (auto i = 0; i < fusion_times - 1; i++) { + cost_map.push_back(begin); + begin += offset; + } + cost_map.push_back(allreduce_graph_.max() * (1 - tail_percent)); + cost_map.push_back(allreduce_graph_.max()); + MS_LOG(DEBUG) << "cost_map = " << cost_map; + return cost_map; +} + +Status AllreduceFusion::SetFusionByBackwardCompTime() { + auto fusion_times = CostModelContext::GetInstance()->costmodel_allreduce_fusion_times(); + if (fusion_times < 2) { + MS_LOG(INFO) << "'costmodel_allreduce_fusion_times' is " << fusion_times << ". Bypass ProcessAllreduceFusion"; + return SUCCESS; + } + auto tail_percent = CostModelContext::GetInstance()->costmodel_allreduce_fusion_tail_percent(); + if (tail_percent < 0 || tail_percent >= 1) { + MS_LOG(INFO) << "'costmodel_allreduce_fusion_tail_percent' is " << tail_percent + << ". Bypass ProcessAllreduceFusion"; + return SUCCESS; + } + const auto cost_map = GenerateCostMap(fusion_times, tail_percent); + MS_LOG(DEBUG) << "AllreduceGraph GenerateCostMap succeed."; + if (SetFusion(cost_map) != SUCCESS) { + MS_LOG(ERROR) << "SetFusion failed."; + return FAILED; + } + MS_LOG(DEBUG) << "AllreduceGraph SetFusion succeed."; + return SUCCESS; +} + +Status AllreduceFusion::GetSetFusionByBackwardCompAndAllreduceTimeParams() { + tail_time_ = CostModelContext::GetInstance()->costmodel_allreduce_fusion_tail_time(); + if (tail_time_ <= 0) { + MS_LOG(INFO) << "'costmodel_allreduce_tail_time' is " << tail_time_ << ". Bypass ProcessAllreduceFusion"; + return FAILED; + } + allreduce_inherent_time_ = CostModelContext::GetInstance()->costmodel_allreduce_fusion_allreduce_inherent_time(); + if (allreduce_inherent_time_ <= 0) { + MS_LOG(INFO) << "'costmodel_allreduce_fusion_allreduce_inherent_time' is " << allreduce_inherent_time_ + << ". Bypass ProcessAllreduceFusion"; + return FAILED; + } + if (tail_time_ <= allreduce_inherent_time_) { + MS_LOG(INFO) << "'costmodel_allreduce_tail_time' is " << tail_time_ + << "'costmodel_allreduce_fusion_allreduce_inherent_time' is " << allreduce_inherent_time_ + << ".tail_time is not more than allreduce_inherent_time. Bypass ProcessAllreduceFusion"; + return FAILED; + } + allreduce_bandwidth_ = CostModelContext::GetInstance()->costmodel_allreduce_fusion_allreduce_bandwidth(); + if (allreduce_bandwidth_ <= 0) { + MS_LOG(INFO) << "'costmodel_allreduce_fusion_allreduce_bandwidth' is " << allreduce_bandwidth_ + << ". Bypass ProcessAllreduceFusion"; + return FAILED; + } + computation_time_parameter_ = + CostModelContext::GetInstance()->costmodel_allreduce_fusion_computation_time_parameter(); + if (computation_time_parameter_ <= 0) { + MS_LOG(INFO) << "'costmodel_allreduce_fusion_computation_time_parameter' is " << computation_time_parameter_ + << ". Bypass ProcessAllreduceFusion"; + return FAILED; + } + return SUCCESS; +} + +Status AllreduceFusion::SetFusionByBackwardCompAndAllreduceTime() { + if (GetSetFusionByBackwardCompAndAllreduceTimeParams() != SUCCESS) { + MS_LOG(ERROR) << "GetSetFusionByBackwardCompAndAllreduceTimeParams failed!"; + return FAILED; + } + allreduce_graph_.SortArnode(); + if (allreduce_graph_.RemoveExtraParas() != SUCCESS) { + MS_LOG(ERROR) << "RemoveExtraParas failed!"; + return FAILED; + } + double para_size = (tail_time_ - allreduce_inherent_time_) / allreduce_bandwidth_; + double to_cost = allreduce_graph_.max() + FUSION_COST_EPS; + int32_t fusion = 1; + while (to_cost != 0) { + MS_LOG(INFO) << "to_cost: " << to_cost << " para_size: " << para_size; + auto node_cost_pair = allreduce_graph_.GetParaByParaSize(to_cost, para_size); + MS_LOG(INFO) << "para size: " << node_cost_pair.first.size() << " from_cost: " << node_cost_pair.second; + auto paras = node_cost_pair.first; + if (FindMirrorAndSetFusion(paras, fusion) != SUCCESS) { + MS_LOG(ERROR) << "FindMirrorAndSetFusion failed"; + return FAILED; + } + fusion++; + para_size = ((to_cost - node_cost_pair.second) * computation_time_parameter_ - allreduce_inherent_time_) / + allreduce_bandwidth_; + to_cost = node_cost_pair.second; + } + MS_LOG(DEBUG) << "AllreduceGraph SetFusionByBackwardCompAndAllreduceTime succeed."; + return SUCCESS; +} + +Status AllreduceFusion::SetFusionByAlgorithm(int32_t algorithm) { + if (algorithm == 1) { + return SetFusionByBackwardCompTime(); + } + return SetFusionByBackwardCompAndAllreduceTime(); +} + +Status AllreduceFusion::ProcessAllreduceFusion(const CNodePtr& ret) { + if (ret == nullptr) { + MS_LOG(ERROR) << "ret is nullptr."; + return FAILED; + } + auto algorithm = CostModelContext::GetInstance()->costmodel_allreduce_fusion_algorithm(); + if (algorithm < 1 || algorithm > 2) { + MS_LOG(INFO) << "'costmodel_allreduce_fusion_algorithm' is " << algorithm << ". Bypass ProcessAllreduceFusion"; + return SUCCESS; + } + ret_ = ret; + root_graph_ = ret_->func_graph(); + MS_EXCEPTION_IF_NULL(root_graph_); + auto forward_graph = ForwardGraph(root_graph_); + MS_EXCEPTION_IF_NULL(forward_graph); + forward_ret_ = forward_graph->get_return(); + MS_EXCEPTION_IF_NULL(forward_ret_); + + if (allreduce_graph_.set_head_cnode(forward_ret_) != SUCCESS) { + MS_LOG(ERROR) << "AllreduceGraph set_head_cnode failed."; + return FAILED; + } + MS_LOG(DEBUG) << "AllreduceGraph set_head_cnode succeed."; + if (AddNodeToGraph() != SUCCESS) { + MS_LOG(ERROR) << "AddNodeToGraph failed."; + return FAILED; + } + MS_LOG(DEBUG) << "AllreduceGraph AddNodeToGraph succeed."; + if (AddEdgeToGraph() != SUCCESS) { + MS_LOG(ERROR) << "AddNodeToGraph failed."; + return FAILED; + } + MS_LOG(DEBUG) << "AllreduceGraph AddEdgeToGraph succeed."; + if (SetFusionByAlgorithm(algorithm) != SUCCESS) { + MS_LOG(ERROR) << "SetFusionByAlgorithm failed."; + return FAILED; + } + MS_LOG(DEBUG) << "AllreduceGraph SetFusionByAlgorithm succeed."; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_fusion.h b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_fusion.h new file mode 100644 index 00000000000..de2844fa51c --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_fusion.h @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_FUSION_H_ +#define MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_FUSION_H_ + +#include +#include +#include "ir/anf.h" +#include "parallel/status.h" +#include "parallel/allreduce_fusion/allreduce_graph.h" + +namespace mindspore { +namespace parallel { +using CNodeCostMap = std::unordered_map; + +constexpr int32_t DEFAULT_COST_MODEL_ALLREDUCE_FUSION_ALGORITHM = 0; +constexpr int32_t DEFAULT_COST_MODEL_ALLREDUCE_FUSION_TIMES = 0; +constexpr double DEFAULT_COST_MODEL_ALLREDUCE_FUSION_TAIL_PERCENT = 0.1; +constexpr double DEFAULT_COST_MODEL_ALLREDUCE_FUSION_TAIL_TIME = 0.1; +constexpr double DEFAULT_COST_MODEL_ALLREDUCE_FUSION_ALLREDUCE_INHERENT_TIME = 0.1; +constexpr double DEFAULT_COST_MODEL_ALLREDUCE_FUSION_ALLREDUCE_BANDWIDTH = 0.1; +constexpr double DEFAULT_COST_MODEL_ALLREDUCE_FUSION_COMPUTATION_TIME_PARAMETER = 0.1; + +constexpr char FUSION[] = "fusion"; +constexpr char PARAMETER[] = "parameter"; +const uint32_t MAX_RECURSIVE_CALL_TIMES = 100; +const double FUSION_COST_EPS = 1e-7; +class AllreduceFusion { + public: + AllreduceFusion() + : allreduce_graph_(), + ret_(nullptr), + forward_ret_(nullptr), + root_graph_(nullptr), + tail_time_(0), + allreduce_inherent_time_(0), + allreduce_bandwidth_(0), + computation_time_parameter_(0) {} + virtual ~AllreduceFusion() = default; + Status ProcessAllreduceFusion(const CNodePtr& ret); + + private: + Status AddNodeToGraph(); + CNodeCostMap FindCNode(const AnfNodePtr& from, uint32_t recursive_times = 0) const; + CNodeCostMap FindNextCNodes(const CNodePtr& from, uint32_t recursive_times = 0) const; + Status AddEdgeToGraph(); + std::vector GenerateCostMap(int32_t fusion_times, double tail_percent) const; + Status SetFusion(const std::vector& cost_map); + Status SetFusionByAlgorithm(int32_t algorithm); + Status SetFusionByBackwardCompTime(); + Status SetFusionByBackwardCompAndAllreduceTime(); + Status GetSetFusionByBackwardCompAndAllreduceTimeParams(); + + AllreduceGraph allreduce_graph_; + CNodePtr ret_; + CNodePtr forward_ret_; + FuncGraphPtr root_graph_; + double tail_time_; + double allreduce_inherent_time_; + double allreduce_bandwidth_; + double computation_time_parameter_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_FUSION_H_ diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_graph.cc b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_graph.cc new file mode 100644 index 00000000000..5c97eda8d86 --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_graph.cc @@ -0,0 +1,201 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/allreduce_fusion/allreduce_graph.h" +#include +#include +#include "utils/log_adapter.h" +#include "ir/anf.h" +#include "parallel/allreduce_fusion/allreduce_node.h" + +namespace mindspore { +namespace parallel { +Status AllreduceGraph::AddNode(const CNodePtr& node, const AnfNodePtr& para) { + auto arnode = std::make_shared(AllreduceNode()); + if (arnode->Init(node) != SUCCESS) { + MS_LOG(ERROR) << "AllreduceNode Init failed"; + return FAILED; + } + if (arnode->AddPara(para) != SUCCESS) { + MS_LOG(ERROR) << "AllreduceNode AddPara failed"; + return FAILED; + } + cnode_arnode_map_[node] = arnode; + + auto arnode_emplace_return = arnode_set_.insert(arnode); + if (!arnode_emplace_return.second) { + MS_LOG(INFO) << "node: " << node->DebugString() << "'s arnode has already been added!"; + } + auto cnode_emplace_return = cnode_set_.emplace(node); + if (!cnode_emplace_return.second) { + MS_LOG(INFO) << "node: " << node->DebugString() << " has already been added!"; + } + cnode_emplace_return = para_cnodeset_map_[para].emplace(node); + if (!cnode_emplace_return.second) { + MS_LOG(INFO) << "node: " << node->DebugString() << " already in para: " << para->fullname_with_scope() + << "'s cnodeset!"; + } + auto para_emplace_return = cnode_paraset_map_[node].emplace(para); + if (!para_emplace_return.second) { + MS_LOG(INFO) << "para: " << para->fullname_with_scope() << " already in node: " << node->DebugString() + << "'s paraset!"; + } + return SUCCESS; +} + +Status AllreduceGraph::AddEdge(const CNodePtr& from, const CNodePtr& to, double dist) { + auto from_arnode_iter = cnode_arnode_map_.find(from); + if (from_arnode_iter == cnode_arnode_map_.end()) { + MS_LOG(ERROR) << "cnode from: " << from->DebugString() << "has not been added"; + PrintCNodeSet(); + return FAILED; + } + auto to_arnode_iter = cnode_arnode_map_.find(to); + if (to_arnode_iter == cnode_arnode_map_.end()) { + MS_LOG(ERROR) << "cnode to: " << to->DebugString() << "has not been added"; + PrintCNodeSet(); + return FAILED; + } + auto from_arnode = from_arnode_iter->second; + auto to_arnode = to_arnode_iter->second; + if (from_arnode->AddNext(to_arnode) != SUCCESS) { + MS_LOG(ERROR) << "from_arnode AddNext failed"; + return FAILED; + } + if (to_arnode->AddPrev(from_arnode, dist) != SUCCESS) { + MS_LOG(ERROR) << "to_arnode AddPrev failed"; + return FAILED; + } + max_ = std::max(max_, to_arnode->depend_feat_size()); + MS_LOG(DEBUG) << "from " << from->DebugString() << ", to " << to->DebugString(); + MS_LOG(DEBUG) << "from depend_feat_size: " << from_arnode->depend_feat_size() + << ", to depend_feat_size: " << to_arnode->depend_feat_size(); + return SUCCESS; +} + +bool AllreduceGraph::NodeInGraph(const CNodePtr& node) const { + auto cnode_iter = cnode_set_.find(node); + return !(cnode_iter == cnode_set_.end()); +} + +std::vector AllreduceGraph::GetParaByCost(double from, double to) { + std::vector nodes; + for (auto& cnode_arnode : cnode_arnode_map_) { + MS_LOG(DEBUG) << "cnode: " << cnode_arnode.first->DebugString() + << ", depend_feat_size: " << cnode_arnode.second->depend_feat_size() + << " curr_para_size: " << cnode_arnode.second->curr_para_size(); + if ((cnode_arnode.second->depend_feat_size() <= to) && (cnode_arnode.second->depend_feat_size() > from)) { + (void)nodes.insert(nodes.end(), cnode_paraset_map_[cnode_arnode.first].begin(), + cnode_paraset_map_[cnode_arnode.first].end()); + } + } + return nodes; +} + +std::pair, double> AllreduceGraph::GetParaByParaSize(double to, double para_size) { + std::vector nodes; + double cur_para_size = 0; + double from = to; + for (auto& arnode : arnode_vec_) { + if (arnode.depend_feat_size() >= to) { + continue; + } + if (para_size > 0 && cur_para_size >= para_size && arnode.depend_feat_size() < from) { + return std::make_pair(nodes, from); + } + (void)nodes.insert(nodes.end(), arnode.paras().begin(), arnode.paras().end()); + cur_para_size += arnode.curr_para_size(); + from = arnode.depend_feat_size(); + } + MS_LOG(INFO) << "GetParaByParaSize has reached head node! para_size: " << para_size + << " cur_para_size: " << cur_para_size << " from: " << from; + return std::make_pair(nodes, from); +} + +void AllreduceGraph::PrintCNodeSet() const { + MS_LOG(INFO) << "CNodeSet:"; + for (auto& cnode : cnode_set_) { + MS_LOG(INFO) << cnode->DebugString(); + } +} + +void AllreduceGraph::PrintAllredueGraphInfo() const { + MS_LOG(INFO) << "max: " << max_; + for (auto& cnode_arnode : cnode_arnode_map_) { + MS_LOG(INFO) << "cnode: " << cnode_arnode.first->DebugString(); + MS_LOG(INFO) << "arnode info: "; + cnode_arnode.second->ToString(); + } +} + +void AllreduceGraph::PrintArnodeVec() const { + MS_LOG(INFO) << "ArnodeVec:"; + for (auto& arnode : arnode_vec_) { + arnode.ToString(); + } +} + +void AllreduceGraph::PrintArnodeSet() const { + MS_LOG(INFO) << "ArnodeSet:"; + for (auto& arnode : arnode_set_) { + arnode->ToString(); + } +} + +void AllreduceGraph::SortArnode() { + arnode_vec_.clear(); + for (auto& node : arnode_set_) { + arnode_vec_.emplace_back(*node); + } + std::sort(arnode_vec_.begin(), arnode_vec_.end(), std::greater<>()); +} + +Status AllreduceGraph::RemoveExtraParas() { + std::unordered_set para_map; + for (auto& node : arnode_vec_) { + for (auto& para : node.paras()) { + auto emplac_result = para_map.emplace(para); + if (!emplac_result.second) { + MS_LOG(DEBUG) << "parameter: " << para->fullname_with_scope() << "in arnode"; + if (node.RemovePara(para) != SUCCESS) { + MS_LOG(ERROR) << "remove para failed"; + return FAILED; + } + } + } + } + return SUCCESS; +} + +Status AllreduceGraph::set_head_cnode(const CNodePtr& node) { + auto arnode = std::make_shared(AllreduceNode()); + if (arnode->Init(node) != SUCCESS) { + MS_LOG(ERROR) << "AllreduceNode Init failed"; + } + head_cnode_ = node; + cnode_arnode_map_[node] = arnode; + auto arnode_emplace_return = arnode_set_.insert(arnode); + if (!arnode_emplace_return.second) { + MS_LOG(WARNING) << "node: " << node->DebugString() << "'s arnode has already been added!"; + } + auto cnode_emplace_return = cnode_set_.emplace(node); + if (!cnode_emplace_return.second) { + MS_LOG(WARNING) << "node: " << node->DebugString() << " has already been added!"; + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_graph.h b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_graph.h new file mode 100644 index 00000000000..da4cbf88005 --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_graph.h @@ -0,0 +1,85 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_GRAPH_H_ +#define MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_GRAPH_H_ + +#include +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "parallel/status.h" +#include "parallel/allreduce_fusion/allreduce_node.h" + +namespace mindspore { +namespace parallel { +class AllreduceGraph { + public: + AllreduceGraph() + : head_cnode_(nullptr), + arnode_set_(), + arnode_vec_(), + cnode_set_(), + para_cnode_map_(), + para_cnodeset_map_(), + cnode_paraset_map_(), + cnode_arnode_map_(), + max_(0) {} + virtual ~AllreduceGraph() = default; + Status AddNode(const CNodePtr& node, const AnfNodePtr& para); + Status AddEdge(const CNodePtr& from, const CNodePtr& to, double dist); + bool NodeInGraph(const CNodePtr& node) const; + std::vector GetParaByCost(double from, double to); + // Find the first several AllreduceNode whose depend_feat_size is less than to, the sum of whose parameter size is + // over para_size. + // Return the parameter AnfNodePtr vector corresponding to these AllreduceNodes and the smallest depend_feat_size. + // If the sum of left AllreduceNode's parameter size is less than para_size, the returned depend_feat_size must be 0. + std::pair, double> GetParaByParaSize(double to, double para_size); + // If one parameter is used by multiple AllreduceNode, parameter belong to the last node for backward computation + // is saved by the corresponding AllreduceNode, parameters belong to other AllreduceNode are removed. + // Called during precise optimization, not implemented temporarily. + void SortArnode(); + Status RemoveExtraParas(); + void PrintCNodeSet() const; + void PrintAllredueGraphInfo() const; + void PrintArnodeVec() const; + void PrintArnodeSet() const; + const std::unordered_set& cnode_set() const { return cnode_set_; } + CNodePtr head_cnode() const { return head_cnode_; } + Status set_head_cnode(const CNodePtr& node); + double max() const { return max_; } + + private: + CNodePtr head_cnode_; + std::set arnode_set_; + std::vector arnode_vec_; + std::unordered_set cnode_set_; + // If One ParameterPtr is used by multiple CNode, the last node for backward computation is saved. + std::unordered_map> para_cnode_map_; + // One ParameterPtr may be used by multiple CNode + std::unordered_map> para_cnodeset_map_; + // Multiple Parameter may be inputs to the same CNode + std::unordered_map> cnode_paraset_map_; + std::unordered_map cnode_arnode_map_; + double max_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_GRAPH_H_ diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_node.cc b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_node.cc new file mode 100644 index 00000000000..8bed29f2f24 --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_node.cc @@ -0,0 +1,104 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/allreduce_fusion/allreduce_node.h" +#include "parallel/tensor_layout/tensor_layout.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +Status AllreduceNode::AddNext(const AllreduceNodePtr& next_node) { + if (next_node == nullptr) { + MS_LOG(ERROR) << "next_node is nullptr!"; + return FAILED; + } + next_.emplace_back(next_node); + return SUCCESS; +} + +Status AllreduceNode::AddPrev(const AllreduceNodePtr& prev_node, double dist) { + if (prev_node == nullptr) { + MS_LOG(ERROR) << "next_node is nullptr!"; + return FAILED; + } + if (dist <= 0) { + MS_LOG(ERROR) << "dist must be positive! dist: " << dist; + return FAILED; + } + prev_.emplace_back(prev_node); + depend_feat_size_ += prev_node->depend_feat_size() + dist; + return SUCCESS; +} + +Status AllreduceNode::Init(const CNodePtr& cnode_ptr) { + if (cnode_ptr == nullptr) { + MS_LOG(ERROR) << "cnode_ptr is nullptr!"; + return FAILED; + } + cnode_ptr_ = cnode_ptr; + return SUCCESS; +} + +Status AllreduceNode::AddPara(const AnfNodePtr& node_ptr) { + if (node_ptr == nullptr) { + MS_LOG(ERROR) << "node_ptr is nullptr!"; + return FAILED; + } + if (!node_ptr->isa()) { + MS_LOG(ERROR) << "node_ptr is not a ParameterPtr!"; + return FAILED; + } + auto para_ptr = node_ptr->cast(); + MS_EXCEPTION_IF_NULL(para_ptr); + auto layout_ptr = para_ptr->tensor_layout(); + if (layout_ptr == nullptr) { + MS_LOG(ERROR) << "layout_ptr is nullptr!"; + return FAILED; + } + auto emplace_return = paras_.emplace(node_ptr); + if (emplace_return.second) { + double para_size = static_cast(layout_ptr->slice_shape().size()); + curr_para_size_ += para_size; + para_size_map_[node_ptr] = para_size; + } else { + MS_LOG(INFO) << "node already exist!"; + } + return SUCCESS; +} + +Status AllreduceNode::RemovePara(const AnfNodePtr& node_ptr) { + if (node_ptr == nullptr) { + MS_LOG(ERROR) << "node_ptr is nullptr!"; + return FAILED; + } + auto erase_num = paras_.erase(node_ptr); + if (erase_num == 0) { + MS_LOG(ERROR) << "para not find!"; + return FAILED; + } + curr_para_size_ -= para_size_map_[node_ptr]; + return SUCCESS; +} + +void AllreduceNode::ToString() const { + MS_LOG(INFO) << "cnode: " << cnode_ptr_->DebugString() << "para size: " << paras_.size(); + for (auto& para : paras_) { + MS_LOG(INFO) << "para name: " << para->fullname_with_scope() << " size: " << para_size_map_.at(para); + } + MS_LOG(INFO) << "depend_feat_size: " << depend_feat_size_ << " curr_para_size: " << curr_para_size_; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_node.h b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_node.h new file mode 100644 index 00000000000..f3eeb53ec7e --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/allreduce_node.h @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_NODE_H_ +#define MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_NODE_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { +class AllreduceNode; +using AllreduceNodePtr = std::shared_ptr; + +class AllreduceNode { + public: + AllreduceNode() + : cnode_ptr_(nullptr), prev_(), next_(), paras_(), para_size_map_(), curr_para_size_(0), depend_feat_size_(0) {} + Status Init(const CNodePtr& cnode_ptr); + Status AddPara(const AnfNodePtr& node_ptr); + Status RemovePara(const AnfNodePtr& node_ptr); + const std::unordered_set& paras() const { return paras_; } + double curr_para_size() const { return curr_para_size_; } + virtual ~AllreduceNode() = default; + Status AddPrev(const AllreduceNodePtr& prev_node, double dist); + Status AddNext(const AllreduceNodePtr& next_node); + double depend_feat_size() const { return depend_feat_size_; } + void ToString() const; + bool operator<(const AllreduceNode& node) const { return depend_feat_size_ < node.depend_feat_size(); } + bool operator>(const AllreduceNode& node) const { return depend_feat_size_ > node.depend_feat_size(); } + + private: + CNodePtr cnode_ptr_; + std::vector prev_; + std::vector next_; + std::unordered_set paras_; + std::unordered_map para_size_map_; + double curr_para_size_; + double depend_feat_size_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_ALLREDUCE_NODE_H_ diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc b/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc new file mode 100644 index 00000000000..4db7007448e --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/allreduce_fusion/step_allreduce_fusion.h" +#include +#include +#include "utils/log_adapter.h" +#include "parallel/status.h" +#include "parallel/context.h" +#include "optimizer/optimizer.h" +#include "parallel/allreduce_fusion/allreduce_fusion.h" +#include "parallel/graph_util/graph_info.h" + +namespace mindspore { +namespace parallel { +bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &optimizer) { + MS_EXCEPTION_IF_NULL(root); + MS_EXCEPTION_IF_NULL(optimizer); + MS_EXCEPTION_IF_NULL(ParallelContext::GetInstance()); + std::string parallel_mode = ParallelContext::GetInstance()->parallel_mode(); + // assume no change to graph + bool changes = false; + // control whether use model_parallel mode + if (((parallel_mode != AUTO_PARALLEL) && (parallel_mode != SEMI_AUTO_PARALLEL)) || + (root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) { + return changes; + } + + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + + MS_LOG(INFO) << "Now entering allreduce fusion"; + DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN)); + + pipeline::ResourceBasePtr res = optimizer->resource(); + MS_EXCEPTION_IF_NULL(res); + + FuncGraphManagerPtr manager = res->manager(); + MS_EXCEPTION_IF_NULL(manager); + CNodePtr ret = root->get_return(); + MS_EXCEPTION_IF_NULL(ret); + + AllreduceFusion allreduce_fusion; + if (allreduce_fusion.ProcessAllreduceFusion(ret) != SUCCESS) { + MS_LOG(EXCEPTION) << "ProcessAllreduceFusion failed"; + } + + DumpGraph(root, std::string(ALLREDUCE_FUSION_END)); + + // allreduce fusion only run once + root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true; + res->results()[pipeline::kStepParallelGraph] = root; + + (void)gettimeofday(&end_time, nullptr); + uint64_t time = 1000000 * static_cast(end_time.tv_sec - start_time.tv_sec); + time += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us"; + return changes; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.h b/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.h new file mode 100644 index 00000000000..2343a7a2fef --- /dev/null +++ b/mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_STEP_ALLREDUCE_FUSION_H_ +#define MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_STEP_ALLREDUCE_FUSION_H_ + +#include "optimizer/optimizer.h" + +namespace mindspore { +namespace parallel { +constexpr char ALLREDUCE_FUSION_RUN_ONCE_ONLY[] = "allreduce_fusion_run_once_only"; +constexpr char ALLREDUCE_FUSION_BEGIN[] = "allreduce_fusion_begin"; +constexpr char ALLREDUCE_FUSION_END[] = "allreduce_fusion_end"; + +bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &optimizer); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_ALLREDUCE_FUSION_STEP_ALLREDUCE_FUSION_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/costmodel.cc new file mode 100644 index 00000000000..618e505eba8 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/costmodel.cc @@ -0,0 +1,113 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/costmodel.h" +#include +#include +#include +#include "parallel/auto_parallel/graph_costmodel.h" + +namespace mindspore { +namespace parallel { +void Simplify(CostPtrList* clist_ptrs) { + // Sort the cost_list with the memory_cost increasing, and communication_cost decreasing order. This method + // excludes the cost with greater memory_cost and greater communication_cost. + // E.g. clist_ptrs = {<100, 20>, <200, 10>, <300, 50>}. After this method, clist_ptrs = {<200, 10>, <100, 20>} + if (!COST_MODEL_SIMPLIFY_CALCULATION) { + return; + } + MS_EXCEPTION_IF_NULL(clist_ptrs); + std::vector id(clist_ptrs->size()); + std::iota(id.begin(), id.end(), size_t(0)); + std::sort(id.begin(), id.end(), [&clist_ptrs](size_t x, size_t y) { + return clist_ptrs->at(x)->memory_cost_ < clist_ptrs->at(y)->memory_cost_; + }); + CostPtrList ret; + for (size_t i = 0; i < clist_ptrs->size(); ++i) { + if ((ret.size() == size_t(0)) || (clist_ptrs->at(id[i])->communication_cost_ < ret.back()->communication_cost_)) { + ret.emplace_back(std::move(clist_ptrs->at(id[i]))); + } + } + *clist_ptrs = std::move(ret); +} + +void SimplifyForDreasingCommunicationWithPartialPara(CostPtrList* clist_ptrs) { + // Sort the cost_list with the memory_cost increasing, and communication_with_partial_para_cost decreasing order. + // This method excludes the cost with greater memory_cost and greater communication_without_para_cost. + if (!COST_MODEL_SIMPLIFY_CALCULATION) { + return; + } + MS_EXCEPTION_IF_NULL(clist_ptrs); + std::vector id(clist_ptrs->size()); + std::iota(id.begin(), id.end(), size_t(0)); + std::sort(id.begin(), id.end(), [&clist_ptrs](size_t x, size_t y) { + return clist_ptrs->at(x)->memory_cost_ < clist_ptrs->at(y)->memory_cost_; + }); + CostPtrList ret; + for (size_t i = 0; i < clist_ptrs->size(); ++i) { + if ((ret.size() == size_t(0)) || + (clist_ptrs->at(id[i])->communication_with_partial_para_ < ret.back()->communication_with_partial_para_)) { + ret.emplace_back(std::move(clist_ptrs->at(id[i]))); + } + } + *clist_ptrs = std::move(ret); +} + +void RefineForPracticalCost(const CostPtr& origin_cost, bool is_redistribution) { + MS_EXCEPTION_IF_NULL(origin_cost); + if (is_redistribution) { + // Redistribution cost + if ((origin_cost->communication_redis_forward_ > EPS) && + (origin_cost->communication_redis_forward_ <= COST_MODEL_COMMUNI_THRESHOLD)) { + origin_cost->communication_redis_forward_ = COST_MODEL_COMMUNI_CONST; + } else if (origin_cost->communication_redis_forward_ > COST_MODEL_COMMUNI_THRESHOLD) { + origin_cost->communication_redis_forward_ += COST_MODEL_COMMUNI_BIAS; + } + if ((origin_cost->communication_redis_backward_ > EPS) && + (origin_cost->communication_redis_backward_ <= COST_MODEL_COMMUNI_THRESHOLD)) { + origin_cost->communication_redis_backward_ = COST_MODEL_COMMUNI_CONST; + } else if (origin_cost->communication_redis_backward_ > COST_MODEL_COMMUNI_THRESHOLD) { + origin_cost->communication_redis_backward_ += COST_MODEL_COMMUNI_BIAS; + } + origin_cost->communication_cost_ = + origin_cost->communication_redis_forward_ + origin_cost->communication_redis_backward_; + origin_cost->communication_without_parameter_ = origin_cost->communication_cost_; + origin_cost->communication_with_partial_para_ = origin_cost->communication_cost_; + } else { + // Operator cost + double backward = 0.0; + if (std::abs(origin_cost->communication_cost_ - origin_cost->communication_without_parameter_) > EPS) { + backward = origin_cost->communication_cost_ - origin_cost->communication_without_parameter_; + } + // forward cost + if ((origin_cost->communication_without_parameter_ > EPS) && + (origin_cost->communication_without_parameter_ <= COST_MODEL_COMMUNI_THRESHOLD)) { + origin_cost->communication_without_parameter_ = COST_MODEL_COMMUNI_CONST; + } else if (origin_cost->communication_without_parameter_ > COST_MODEL_COMMUNI_THRESHOLD) { + origin_cost->communication_without_parameter_ += COST_MODEL_COMMUNI_BIAS; + } + // total + if (origin_cost->communication_cost_ > EPS) { + origin_cost->communication_cost_ = origin_cost->communication_without_parameter_ + backward; + } + if (origin_cost->communication_with_partial_para_ > EPS) { + origin_cost->communication_with_partial_para_ = + origin_cost->communication_without_parameter_ + COST_MODEL_GAMMA * backward; + } + } +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/costmodel.h b/mindspore/ccsrc/parallel/auto_parallel/costmodel.h new file mode 100644 index 00000000000..97155fb9c3c --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/costmodel.h @@ -0,0 +1,304 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_COSTMODEL_H_ +#define MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_COSTMODEL_H_ + +#include +#include +#include +#include +#include +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_info.h" + +namespace mindspore { +namespace parallel { +struct Decision; +using OperatorName = std::string; +using Attr = std::pair; +using Param = std::pair, int32_t>; +using OperatorParams = std::vector; +using OperatorAttrs = std::vector; +// OutPutInfo.fist: true if the operator's output is a tuple +// OutPutInfo.second: elements number of the tuple output. Only meaningful if OutPutInfo.fist is true. +using OutPutInfo = std::pair; +using OutPutInfoVector = std::vector; +using OperatorArgs = std::pair; +using Operator = std::pair; +using OperatorVector = std::vector; +using RedistributionOpListPtr = std::shared_ptr>; + +struct Cost { + Cost(); + Cost(double memory, double commuication, const std::shared_ptr& decision_ = nullptr) + : memory_cost_(memory), communication_cost_(commuication), decision_ptr_(std::move(decision_)) { + communication_without_parameter_ = 0.0; + communication_with_partial_para_ = 0.0; + communication_redis_forward_ = 0.0; + communication_redis_backward_ = 0.0; + } + double memory_cost_; + // 'communication_cost_' includes communications from operators (forward and backward) and edges + double communication_cost_; + // communication_without_parameter_ = communication_cost_ - (backward communication from operators) + double communication_without_parameter_; + // communication_with_partial_para_ = + // communication_without_parameter_ + COST_MODEL_GAMMA * (communication_cost_ - communication_without_parameter_ ) + double communication_with_partial_para_; + double communication_redis_forward_; + double communication_redis_backward_; + std::shared_ptr decision_ptr_; +}; + +using CostPtr = std::shared_ptr; +using CostPtrList = std::vector>; + +class StrategyWithCost { + public: + StrategyWithCost(StrategyPtr strategy, std::vector inputs_, std::vector outputs_) + : strategy_ptr(std::move(strategy)), inputs_ptr(std::move(inputs_)), outputs_ptr(std::move(outputs_)) {} + + StrategyWithCost(const StrategyWithCost& swc) = delete; + StrategyWithCost(StrategyWithCost&& swc) + : strategy_ptr(swc.strategy_ptr), + inputs_ptr(swc.inputs_ptr), + outputs_ptr(swc.outputs_ptr), + cost_list(swc.cost_list) {} + ~StrategyWithCost() = default; + + StrategyPtr strategy_ptr; + std::vector inputs_ptr; + std::vector outputs_ptr; + CostPtrList cost_list; +}; + +enum DecisionType { + OP_ELIMINATION, + EDGE_ELIMINATION, + MERGE_ELIMINATION, + CONTRACT_ELIMINATION, + TRIANGLE_ELIMINATION, + STAR_ELIMINATION, + FINAL_TYPE, + FINAL_SINGLE +}; + +struct Decision : public Base { + ~Decision() override = default; + DecisionType type_; +}; + +// 'OpEliminationDecision' is for the Operator Elimination in DP algorithm: u --> v --> w ==> u --> w. +// This data structure records the strategy 'op_strategy_' for v, the edge cost 'left_cost_' for 'u --> v', the +// operator cost 'middle_cost_' for v, and the edge cost 'right_cost_' for 'v --> w' +struct OpEliminationDecision : public Decision { + OpEliminationDecision(StrategyPtr op_stra, CostPtr l_cost, CostPtr m_cost, CostPtr r_cost) + : op_strategy_(std::move(op_stra)), + left_cost_(std::move(l_cost)), + middle_cost_(std::move(m_cost)), + right_cost_(std::move(r_cost)) { + type_ = DecisionType::OP_ELIMINATION; + } + + StrategyPtr op_strategy_; + CostPtr left_cost_; + CostPtr middle_cost_; + CostPtr right_cost_; + MS_DECLARE_PARENT(OpEliminationDecision, Decision); +}; + +/* 'EdgeEliminationDecision' is for the Edge Elimination in DP algorithm: + ____ + / \ + u v ==> u --> v, which replace the multi-edges by a single edge. + \____/ + This data structure records the cost list for all edges 'edges_cost_list_' + */ +struct EdgeEliminationDecision : public Decision { + explicit EdgeEliminationDecision(CostPtrList cost_list) : edges_cost_list_(std::move(cost_list)) { + type_ = DecisionType::EDGE_ELIMINATION; + } + + CostPtrList edges_cost_list_; + MS_DECLARE_PARENT(EdgeEliminationDecision, Decision); +}; + +// 'MergeEliminationDecision' is for the Merge Elimination in DP algorithm: +// w +// | +// | ==> u --> v +// u --> v In the original graph, v has two alive incoming edges, w has one alive outgoing edge, +// and w has zero alive incoming edges. After the Merge Elimination, the result graph contains only 'u -- >v'. +// This data structure records the strategy 'merged_op_strategy_' for operator 'w', +// the cost 'merged_op_cost_' for operator 'w', and the edge cost 'edge_cost_' for 'w --> v'. +struct MergeEliminationDecision : public Decision { + MergeEliminationDecision(StrategyPtr op_stra, CostPtr op_cost, CostPtr edge_c, StrategyPtr tar_op_stra, + CostPtr target_op_c) + : merged_op_strategy_(std::move(op_stra)), + merged_op_cost_(std::move(op_cost)), + edge_cost_(std::move(edge_c)), + target_op_strategy_(std::move(tar_op_stra)), + target_op_cost_(std::move(target_op_c)) { + type_ = DecisionType::MERGE_ELIMINATION; + } + + StrategyPtr merged_op_strategy_; + CostPtr merged_op_cost_; + CostPtr edge_cost_; + StrategyPtr target_op_strategy_; + CostPtr target_op_cost_; + MS_DECLARE_PARENT(MergeEliminationDecision, Decision); +}; + +// 'ContractEliminationDecision' is for the Contract Elimination in DP algorithm: +// u --> v +// | +// | ==> u --> w +// w In the original graph, u has two alive outgoing edges, v has one alive incoming edge, +// and v has zero outgoing edge. After the Contract Elimination, the result graph contains only 'u --> w'. +// This data structure records the strategy 'contracted_op_strategy_' for operator 'v', the cost for +// operator 'contracted_op_cost_', and the edge cost for 'edge_cost_'. +struct ContractEliminationDecision : public Decision { + ContractEliminationDecision(StrategyPtr contra_stra, CostPtr contra_op_cost, CostPtr edge_cost, + StrategyPtr target_stra, CostPtr tar_cost) + : contracted_op_strategy_(std::move(contra_stra)), + contracted_op_cost_(std::move(contra_op_cost)), + edge_cost_(std::move(edge_cost)), + target_op_strategy_(std::move(target_stra)), + target_cost_(std::move(tar_cost)) { + type_ = DecisionType::CONTRACT_ELIMINATION; + } + + StrategyPtr contracted_op_strategy_; + CostPtr contracted_op_cost_; + CostPtr edge_cost_; + StrategyPtr target_op_strategy_; + CostPtr target_cost_; + MS_DECLARE_PARENT(ContractEliminationDecision, Decision); +}; + +/* 'TriangleEliminationDecision' is for the Triangle Elimination in DP algorithm: + * + * u + * / \ + * / \ + * v --- w ==> v --- w In the original graph, u has 2 outgoing edges, v has 1 outgoing edge, + * and w has 2 incoming edges, u can be eliminated into v. + * 'eliminated_op_strategy_' is for u, 'eliminated_op_cost_' is for u, 'eliminated_left_edge_' is for edge u --> v, + * 'eliminated_right_edge_' is for edge u --> w. + */ +struct TriangleEliminationDecision : public Decision { + TriangleEliminationDecision(StrategyPtr elimi_stra, CostPtr elimi_op_cost, CostPtr l_edge_cost, CostPtr r_edge_cost, + StrategyPtr left_stra, CostPtr l_node_cost, StrategyPtr right_stra, CostPtr r_node_cost) + : eliminated_op_strategy_(std::move(elimi_stra)), + eliminated_op_cost_(std::move(elimi_op_cost)), + left_edge_cost_(std::move(l_edge_cost)), + right_edge_cost_(std::move(r_edge_cost)), + left_node_strategy_(std::move(left_stra)), + left_node_cost_(std::move(l_node_cost)), + right_node_strategy_(std::move(right_stra)), + right_node_cost_(std::move(r_node_cost)) { + type_ = DecisionType::TRIANGLE_ELIMINATION; + } + + StrategyPtr eliminated_op_strategy_; + CostPtr eliminated_op_cost_; + CostPtr left_edge_cost_; + CostPtr right_edge_cost_; + StrategyPtr left_node_strategy_; + CostPtr left_node_cost_; + StrategyPtr right_node_strategy_; + CostPtr right_node_cost_; + MS_DECLARE_PARENT(TriangleEliminationDecision, Decision); +}; + +/* 'StarEliminationDecision' is for the Star Elimination in DP algorithm: + * + * v <--- u ---> w ==> v w In the original graph, u has 0 incoming edges, and multiple outgoing edges. + * In addition, v and w have other complicated connections, resulting in v and w can not be performed other + * eliminations. After the StarElimination, u is merged into v, and the resulting graph is splitted into multiple + * connected components. + * NOTE: this elimination MUST be performed only when the above 5 operation cannot be applied. + */ +struct StarEliminationDecision : public Decision { + StarEliminationDecision(StrategyPtr elimi_op_stra, CostPtr elimi_op_cost, CostPtrList succ_edges_clist, + std::vector succ_ops_stra_list, CostPtrList succ_ops_clist) + : eliminated_op_strategy_(std::move(elimi_op_stra)), + eliminated_op_cost_(std::move(elimi_op_cost)), + succ_edges_cost_list_(std::move(succ_edges_clist)), + succ_ops_stra_list_(std::move(succ_ops_stra_list)), + succ_ops_cost_list_(std::move(succ_ops_clist)) { + type_ = DecisionType::STAR_ELIMINATION; + } + + StrategyPtr eliminated_op_strategy_; + CostPtr eliminated_op_cost_; + CostPtrList succ_edges_cost_list_; + std::vector succ_ops_stra_list_; + CostPtrList succ_ops_cost_list_; + MS_DECLARE_PARENT(StarEliminationDecision, Decision); +}; + +// This data structure records the decision for the graph which contains two nodes: u --> v. This includes +// the strategy 'u_strategy_' for 'u', the strategy 'v_strategy_' for 'v', the cost 'left_cost_' for 'u'. +struct FinalDecision : public Decision { + FinalDecision(StrategyPtr u_stra, StrategyPtr v_stra, CostPtr l_cost, CostPtr m_cost, CostPtr r_cost) + : u_strategy_(std::move(u_stra)), + v_strategy_(std::move(v_stra)), + left_cost_(std::move(l_cost)), + middle_cost_(std::move(m_cost)), + right_cost_(std::move(r_cost)) { + type_ = DecisionType::FINAL_TYPE; + } + + StrategyPtr u_strategy_; + StrategyPtr v_strategy_; + CostPtr left_cost_; + CostPtr middle_cost_; + CostPtr right_cost_; + MS_DECLARE_PARENT(FinalDecision, Decision); +}; + +// This data structure records the final decision for the graph containing a single node: u. This includes +// the strategy 'u_strategy_' for 'u', the cost 'u_cost_' for 'u'. +struct FinalSingleDecision : public Decision { + FinalSingleDecision(StrategyPtr u_stra, CostPtr u_cost) : u_strategy_(std::move(u_stra)), u_cost_(std::move(u_cost)) { + type_ = DecisionType::FINAL_SINGLE; + } + + StrategyPtr u_strategy_; + CostPtr u_cost_; + MS_DECLARE_PARENT(FinalSingleDecision, Decision); +}; + +using DecisionPtr = std::shared_ptr; +using OpEliminationDecisionPtr = std::shared_ptr; +using EdgeEliminationDecisionPtr = std::shared_ptr; +using MergeEliminationDecisionPtr = std::shared_ptr; +using ContractEliminationDecisionPtr = std::shared_ptr; +using TriangleEliminationDecisionPtr = std::shared_ptr; +using StarEliminationDecisionPtr = std::shared_ptr; +using FinalDecisionPtr = std::shared_ptr; +using FinalSingleDecisionPtr = std::shared_ptr; + +void Simplify(CostPtrList* clist); +void SimplifyForDreasingCommunicationWithPartialPara(CostPtrList* clist); +void RefineForPracticalCost(const CostPtr&, bool is_redistribution); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_COSTMODEL_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/dp_algo_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/dp_algo_costmodel.cc new file mode 100644 index 00000000000..8e042eadabd --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/dp_algo_costmodel.cc @@ -0,0 +1,220 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/dp_algo_costmodel.h" + +#include +#include +#include + +namespace mindspore { +namespace parallel { +Status GetStrategy(const CostGraphPtr& graph) { + MS_LOG(INFO) << "Searching strategies begins."; + MS_EXCEPTION_IF_NULL(graph); + std::vector eliminations; + bool flag = true; + + // Phase 1: Shrink the CostGraph using 6 operations, and record them in the order. + // Note: the checking and applying of the 6 operations MUST in current order. + while (flag) { + flag = false; + auto node = graph->CheckOpElimination(); + if (node != nullptr) { + // Applying the Operator Elimination + flag = true; + auto l_edge = node->GetAlivePrevEdges()[0]; + auto r_edge = node->GetAliveSuccEdges()[0]; + auto n_edge = graph->EliminationOp(node); + auto elimi = std::make_shared(n_edge, l_edge, node, r_edge); + eliminations.emplace_back(std::move(elimi)); + } + auto edges = graph->CheckEdgeElimination(); + if ((!flag) && (!edges.empty())) { + // Applying the Edge Elimination + flag = true; + auto n_edge = graph->EliminationEdges(edges); + auto elimi = std::make_shared(n_edge, edges); + eliminations.emplace_back(std::move(elimi)); + } + auto merge_node = graph->CheckMergeElimination(); + if ((!flag) && (merge_node != nullptr)) { + // Applying the Merge Elimination + flag = true; + auto succ_edge = merge_node->GetAliveSuccEdges()[0]; + auto target_node = graph->EliminationMerge(merge_node); + auto elimi = std::make_shared(merge_node, succ_edge, target_node); + eliminations.emplace_back(std::move(elimi)); + } + auto contracted_node = graph->CheckContractElimination(); + if ((!flag) && (contracted_node != nullptr)) { + // Applying the Contract Elimination + flag = true; + auto prev_edge = contracted_node->GetAlivePrevEdges()[0]; + auto target_node = graph->EliminationContract(contracted_node); + auto elimi = std::make_shared(target_node, prev_edge, contracted_node); + eliminations.emplace_back(std::move(elimi)); + } + auto triangle_pair = graph->CheckTriangleElimination(); + if ((!flag) && (triangle_pair.first != nullptr)) { + // Applying the Triangle Elimination + flag = true; + auto eliminated_node = triangle_pair.first; + auto l_r_edge = triangle_pair.second; + + auto left_node = l_r_edge->prev_operator(); + auto right_node = l_r_edge->next_operator(); + auto left_edge = eliminated_node->GetAliveSuccEdges()[0]; + auto right_edge = eliminated_node->GetAliveSuccEdges()[1]; + MS_EXCEPTION_IF_NULL(left_edge); + if (left_edge->next_operator() != left_node) { + auto tmp = left_edge; + left_edge = right_edge; + right_edge = tmp; + } + auto left_node_cpy = graph->EliminationTriangle(eliminated_node, l_r_edge); + auto elimi = + std::make_shared(eliminated_node, left_edge, left_node_cpy, right_edge, right_node); + eliminations.emplace_back(std::move(elimi)); + } + auto star_center = graph->CheckStarElimination(); + if ((!flag) && (star_center != nullptr)) { + // Applying the Star Elimination + flag = true; + auto succ_edges = graph->EliminationStar(star_center); + std::vector succ_nodes; + for (size_t i = 0; i < succ_edges.size(); ++i) { + MS_EXCEPTION_IF_NULL(succ_edges[i]); + succ_nodes.push_back(succ_edges[i]->next_operator()); + } + auto elimi = std::make_shared(star_center, succ_edges, succ_nodes); + eliminations.emplace_back(std::move(elimi)); + } + } + + // Phase 2: Search the cost_list in the final graph, and determine the optimal one + if (graph->SearchStrategy() != SUCCESS) { + MS_LOG(ERROR) << "Searching strategy for the final failed."; + return FAILED; + } + + // Phase 3: Recover the original CostGraph, the determine strategy for each operator + if (RecoverStrategy(eliminations) == SUCCESS) { + MS_LOG(INFO) << "Searching strategies ends."; + return SUCCESS; + } else { + MS_LOG(EXCEPTION) << "Searching strategies failed."; + } +} + +Status RecoverStrategy(std::vector eliminations) { + std::vector::reverse_iterator rit; + + for (rit = eliminations.rbegin(); rit != eliminations.rend(); ++rit) { + if ((*rit)->isa()) { + auto elimination = (*rit)->cast(); + auto e = elimination->new_edge_; + auto w = elimination->op_; + MS_EXCEPTION_IF_NULL(e); + MS_EXCEPTION_IF_NULL(w); + auto left_edge = elimination->left_edge_; + auto right_edge = elimination->right_edge_; + MS_EXCEPTION_IF_NULL(left_edge); + MS_EXCEPTION_IF_NULL(right_edge); + auto decision = e->selected_cost()->decision_ptr_->cast(); + w->SetSelectedStrategyAndCost(decision->op_strategy_, decision->middle_cost_); + left_edge->set_selected_cost(decision->left_cost_); + right_edge->set_selected_cost(decision->right_cost_); + MS_LOG(INFO) << "Recover opElimination succeeded."; + } else if ((*rit)->isa()) { + auto elimination = (*rit)->cast(); + auto new_edge = elimination->new_edge_; + MS_EXCEPTION_IF_NULL(new_edge); + auto& edges = elimination->edges_; + auto decision = new_edge->selected_cost()->decision_ptr_->cast(); + for (size_t j = 0; j < edges.size(); ++j) { + MS_EXCEPTION_IF_NULL(edges[j]); + edges[j]->set_selected_cost(decision->edges_cost_list_[j]); + } + MS_LOG(INFO) << "Recover edgeElimination succeeded."; + } else if ((*rit)->isa()) { + auto elimination = (*rit)->cast(); + auto target_node = elimination->target_node_; + MS_EXCEPTION_IF_NULL(target_node); + auto merged_node = elimination->merged_node_; + MS_EXCEPTION_IF_NULL(merged_node); + auto merged_edge = elimination->dir_edge_; + MS_EXCEPTION_IF_NULL(merged_edge); + MS_EXCEPTION_IF_NULL(target_node->selected_cost()); + MS_EXCEPTION_IF_NULL(target_node->selected_cost()->decision_ptr_); + auto decision = target_node->selected_cost()->decision_ptr_->cast(); + merged_node->SetSelectedStrategyAndCost(decision->merged_op_strategy_, decision->merged_op_cost_); + merged_edge->set_selected_cost(decision->edge_cost_); + target_node->SetSelectedStrategyAndCost(decision->target_op_strategy_, decision->target_op_cost_); + + MS_LOG(INFO) << "Recover mergeElimination succeeded."; + } else if ((*rit)->isa()) { + auto elimination = (*rit)->cast(); + auto target_node = elimination->target_node_; + auto contracted_node = elimination->contracted_node_; + auto contracted_edge = elimination->dir_edge_; + auto decision = target_node->selected_cost()->decision_ptr_->cast(); + + contracted_node->SetSelectedStrategyAndCost(decision->contracted_op_strategy_, decision->contracted_op_cost_); + contracted_edge->set_selected_cost(decision->edge_cost_); + target_node->SetSelectedStrategyAndCost(decision->target_op_strategy_, decision->target_cost_); + MS_LOG(INFO) << "Recover contractElimination succeeded."; + } else if ((*rit)->isa()) { + auto elimination = (*rit)->cast(); + auto left_node = elimination->left_node_; + auto left_edge = elimination->left_edge_; + auto eliminated_node = elimination->eliminated_node_; + auto right_edge = elimination->right_edge_; + auto right_node = elimination->right_node_; + auto decision = left_node->selected_cost()->decision_ptr_->cast(); + + eliminated_node->SetSelectedStrategyAndCost(decision->eliminated_op_strategy_, decision->eliminated_op_cost_); + left_edge->set_selected_cost(decision->left_edge_cost_); + right_edge->set_selected_cost(decision->right_edge_cost_); + left_node->SetSelectedStrategyAndCost(decision->left_node_strategy_, decision->left_node_cost_); + right_node->SetSelectedStrategyAndCost(decision->right_node_strategy_, decision->right_node_cost_); + MS_LOG(INFO) << "Recover triangleElimination succeeded."; + } else if ((*rit)->isa()) { + auto elimination = (*rit)->cast(); + auto merged_node = elimination->eliminated_node_; + auto succ_edges = elimination->succ_edges_; + auto succ_nodes = elimination->succ_ops_; + // decision is hided in succ_nodes[0] + auto decision = succ_nodes[0]->selected_cost()->decision_ptr_->cast(); + + merged_node->SetSelectedStrategyAndCost(decision->eliminated_op_strategy_, decision->eliminated_op_cost_); + for (size_t i = 0; i < succ_edges.size(); ++i) { + succ_edges[i]->set_selected_cost(decision->succ_edges_cost_list_[i]); + } + for (size_t j = 0; j < succ_nodes.size(); ++j) { + succ_nodes[j]->SetSelectedStrategyAndCost(decision->succ_ops_stra_list_[j], decision->succ_ops_cost_list_[j]); + } + MS_LOG(INFO) << "Recover starElimination succeeded."; + } else { + MS_LOG(ERROR) << "Unknown Elimination type."; + return FAILED; + } + } + + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/dp_algo_costmodel.h b/mindspore/ccsrc/parallel/auto_parallel/dp_algo_costmodel.h new file mode 100644 index 00000000000..de8ebc6fcad --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/dp_algo_costmodel.h @@ -0,0 +1,152 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_DP_ALGO_COSTMODEL_H_ +#define MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_DP_ALGO_COSTMODEL_H_ + +#include +#include +#include +#include "ir/value.h" +#include "parallel/auto_parallel/edge_costmodel.h" +#include "parallel/auto_parallel/graph_costmodel.h" + +namespace mindspore { +namespace parallel { +// There are 3 meta phases of the Dynamic Programming (DP) algorithm. The input is a CostGraph, and the goal +// is to compute the strategy for each operator in the CostGraph. +// +// Phase 1: Shrink the CostGraph using 6 operations, and record them in the order +// Using for operations: Operator Elimination, Edge Elimination, Merge Elimination, and Contract Elimination, +// each connected component in the CostGraph can be shrunk in to the final graph: u --> v. See the +// interpretation of 6 operations in costmodel.h. +// Phase 2: Search the cost_list in the final graph, and determine the optimal one +// Create the cost_list for the final graph, and choose the optimal one: one the minimum quantity +// COST_MODEL_ALPHA * memory_cost + COST_MODEL_BETA * communication_cost +// Phase 3: Recover the original CostGraph, the determine strategy for each operator +// After determining the optimal cost for the final graph, the algorithm recovers the original graph by applying +// the 4 operations in the reverse order in the Phase 1. Because each operation decision contains the strategy, +// the operators' strategies can be all determined. + +struct Elimination : public Base { + enum EliminationType { OPERA, EDGE, MERGE, CONTRACT, TRIANGLE, STAR }; + Elimination(EdgePtr n_edge, EliminationType ty) : new_edge_(std::move(n_edge)), type_(ty) {} + + EdgePtr new_edge_; + EliminationType type_; +}; + +// Operator Elimination +struct OpElimination : public Elimination { + OpElimination(EdgePtr n_edge, EdgePtr l_edge, OperatorInfoPtr op_info, EdgePtr r_edge) + : Elimination(std::move(n_edge), Elimination::EliminationType::OPERA), + left_edge_(std::move(l_edge)), + op_(std::move(op_info)), + right_edge_(std::move(r_edge)) {} + + EdgePtr left_edge_; + OperatorInfoPtr op_; + EdgePtr right_edge_; + MS_DECLARE_PARENT(OpElimination, Elimination); +}; + +// Edge Elimination +struct EdgeElimination : public Elimination { + EdgeElimination(const EdgePtr& n_edge, std::vector eds) + : Elimination(n_edge, Elimination::EliminationType::EDGE), edges_(std::move(eds)) {} + + std::vector edges_; + MS_DECLARE_PARENT(EdgeElimination, Elimination); +}; + +// Merge Elimination +struct MergeElimination : public Elimination { + MergeElimination(OperatorInfoPtr u_info, EdgePtr merged_target_edge, OperatorInfoPtr v_info) + : Elimination(nullptr, Elimination::EliminationType::MERGE), + merged_node_(std::move(u_info)), + dir_edge_(std::move(merged_target_edge)), + target_node_(std::move(v_info)) {} + + OperatorInfoPtr merged_node_; + EdgePtr dir_edge_; + OperatorInfoPtr target_node_; + MS_DECLARE_PARENT(MergeElimination, Elimination); +}; + +// Contract Elimination +struct ContractElimination : public Elimination { + ContractElimination(OperatorInfoPtr tar_info, EdgePtr tar_con_edge, OperatorInfoPtr con_info) + : Elimination(nullptr, Elimination::EliminationType::CONTRACT), + contracted_node_(std::move(con_info)), + dir_edge_(std::move(tar_con_edge)), + target_node_(std::move(tar_info)) {} + + OperatorInfoPtr contracted_node_; + EdgePtr dir_edge_; + OperatorInfoPtr target_node_; + MS_DECLARE_PARENT(ContractElimination, Elimination); +}; + +// Triangle Elimination +struct TriangleElimination : public Elimination { + TriangleElimination(OperatorInfoPtr elim_node, EdgePtr l_edge, OperatorInfoPtr l_node, EdgePtr r_edge, + OperatorInfoPtr r_node) + : Elimination(nullptr, Elimination::EliminationType::TRIANGLE), + eliminated_node_(std::move(elim_node)), + left_edge_(std::move(l_edge)), + left_node_(std::move(l_node)), + right_edge_(std::move(r_edge)), + right_node_(std::move(r_node)) {} + + OperatorInfoPtr eliminated_node_; + EdgePtr left_edge_; + OperatorInfoPtr left_node_; + EdgePtr right_edge_; + OperatorInfoPtr right_node_; + MS_DECLARE_PARENT(TriangleElimination, Elimination); +}; + +// Star Elimination +struct StarElimination : public Elimination { + StarElimination(OperatorInfoPtr elimi_node, std::vector s_edges, std::vector s_ops) + : Elimination(nullptr, Elimination::EliminationType::STAR), + eliminated_node_(std::move(elimi_node)), + succ_edges_(std::move(s_edges)), + succ_ops_(std::move(s_ops)) {} + + OperatorInfoPtr eliminated_node_; + std::vector succ_edges_; + std::vector succ_ops_; + MS_DECLARE_PARENT(StarElimination, Elimination); +}; + +using EliminationPtr = std::shared_ptr; +using OpEliminationPtr = std::shared_ptr; +using EdgeEliminationPtr = std::shared_ptr; +using MergeEliminationPtr = std::shared_ptr; +using ContractEliminationPtr = std::shared_ptr; +using TriangleEliminationPtr = std::shared_ptr; +using StarEliminationPtr = std::shared_ptr; + +// Phase 1 and Phase 2 +Status GetStrategy(const CostGraphPtr& graph); + +// Phase 3 +Status RecoverStrategy(std::vector eliminations); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_DP_ALGO_COSTMODEL_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/edge_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/edge_costmodel.cc new file mode 100644 index 00000000000..30fa90457ea --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/edge_costmodel.cc @@ -0,0 +1,262 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/edge_costmodel.h" + +#include +#include +#include +#include +#include "parallel/auto_parallel/costmodel.h" +#include "parallel/tensor_layout/tensor_redistribution.h" +#include "parallel/auto_parallel/graph_costmodel.h" + +namespace mindspore { +namespace parallel { +Status Edge::InitEdgeCost() { + bool has_available_cost = false; + for (auto& swc : prev_op_->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(swc); + pre_op_output_.emplace_back(std::make_pair(swc->strategy_ptr, swc->outputs_ptr)); + } + for (auto& swc : next_op_->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(swc); + next_op_input_.emplace_back(std::make_pair(swc->strategy_ptr, swc->inputs_ptr)); + } + if (is_identity_edge) { + for (auto& target_output : pre_op_output_) { + auto target_output_lyt = target_output.second[prev_op_output_index_].tensor_layout(); + auto target_output_str = target_output.first; + for (auto& target_input : next_op_input_) { + auto target_input_lyt = target_input.second[next_op_input_index_].tensor_layout(); + auto target_input_str = target_input.first; + if (target_output_lyt == target_input_lyt) { + CostPtrKey ck = {target_output_str, target_input_str}; + CostPtr cost = std::make_shared(0.0, 0.0); + MS_EXCEPTION_IF_NULL(cost); + cost->communication_without_parameter_ = 0.0; + cost->communication_with_partial_para_ = 0.0; + CostPtrList cl; + cl.push_back(cost); + (void)cost_map_.emplace(std::make_pair(ck, cl)); + has_available_cost = true; + } + } + } + } else { + for (auto& target_output : pre_op_output_) { + auto target_output_lyt = target_output.second[prev_op_output_index_].tensor_layout(); + auto target_output_str = target_output.first; + auto type_length = prev_op_->GetOutputTypeLengths()[prev_op_output_index_]; + for (auto& target_input : next_op_input_) { + auto target_input_lyt = target_input.second[next_op_input_index_].tensor_layout(); + auto target_input_str = target_input.first; + CostPtr cost; + if (GetRedistributionCost(target_output_lyt, target_input_lyt, type_length, &cost) != SUCCESS) { + MS_LOG(EXCEPTION) << "Failure: redistribution cost calculation failed"; + } + MS_EXCEPTION_IF_NULL(cost); + MS_LOG(DEBUG) << "The redistribution cost: memory_cost: " << cost->memory_cost_ + << ", communication_cost: " << cost->communication_cost_ + << ", communication_without_parameter_: " << cost->communication_without_parameter_ + << ", communication_with_partial_para_: " << cost->communication_with_partial_para_ << "."; + // refine communication cost calculation for practice + RefineForPracticalCost(cost, true); + CostPtrKey ck = {target_output_str, target_input_str}; + CostPtrList cl; + cl.push_back(cost); + (void)cost_map_.emplace(std::make_pair(ck, cl)); + has_available_cost = true; + } + } + } + if (!has_available_cost) { + if (!NOT_FULLY_USE_DEVICES) { + MS_LOG(EXCEPTION) << "Generating cost for edge: " << edge_name_ + << " failed, it may be caused by setting 'not_fully_use_devices' false. Try to set " + "'not_fully_use_devices' true."; + } else if (ELEMENTWISE_OP_STRA_FOLLOW) { + MS_LOG(EXCEPTION) << "Generating cost for edge: " << edge_name_ + << " failed, it may be caused by setting 'elementwise_op_strategy_follow' true. " + "Try to set 'elementwise_op_strategy_follow' false."; + } + MS_LOG(EXCEPTION) << "Generating cost for edge: " << edge_name_ << " failed."; + } + return Status::SUCCESS; +} + +Status Edge::GetRedistributionCost(const TensorLayout& prev_op_output_layout, const TensorLayout& next_op_input_layout, + size_t type_length, CostPtr* cost) { + MS_EXCEPTION_IF_NULL(prev_op_); + MS_EXCEPTION_IF_NULL(cost); + RankList dev_list = prev_op_->global_device_list(); + TensorRedistribution tensor_redistribution(false); + + // Init TensorRedistribution + if (tensor_redistribution.Init(prev_op_output_layout, next_op_input_layout, dev_list) == FAILED) { + MS_LOG(EXCEPTION) << "Failure: tensor_redistribution init failed."; + } + + if (tensor_redistribution.ComputeCost() == FAILED) { + MS_LOG(EXCEPTION) << "Failure: tensor_redistribution ComputeCost failed."; + } + + double comm_cost = tensor_redistribution.comm_cost(); + double forward_comm_cost = tensor_redistribution.forward_comm_cost(); + double backward_comm_cost = tensor_redistribution.backward_comm_cost(); + double mem_cost = tensor_redistribution.mem_cost(); + + *cost = std::make_shared(type_length * mem_cost, type_length * comm_cost); + (*cost)->communication_without_parameter_ = type_length * comm_cost; + (*cost)->communication_with_partial_para_ = + (*cost)->communication_without_parameter_ + + COST_MODEL_GAMMA * ((*cost)->communication_cost_ - (*cost)->communication_without_parameter_); + (*cost)->communication_redis_forward_ = type_length * forward_comm_cost; + (*cost)->communication_redis_backward_ = type_length * backward_comm_cost; + return Status::SUCCESS; +} + +CostPtrList Edge::GetCostList(StrategyPtr output_str, StrategyPtr input_str) { + CostPtrKey ck = {output_str, input_str}; + CostPtrList result; + if (cost_map_.find(ck) != cost_map_.end()) { + return cost_map_.at(ck); + } + return result; +} + +CostPtrList Edge::CreateEdgeEliminationCostList(const StrategyPtr& output_st_ptr, const std::vector& edges, + const StrategyPtr& input_st_ptr) { + std::function LocalGetCostList = [&](const EdgePtr& edge) { + MS_EXCEPTION_IF_NULL(edge); + return edge->GetCostList(output_st_ptr, input_st_ptr); + }; + CostPtrList result; + std::vector all_cost_list; + all_cost_list.resize(edges.size()); + (void)std::transform(edges.begin(), edges.end(), all_cost_list.begin(), LocalGetCostList); + + CostPtrList selected_cost_list(all_cost_list.size(), nullptr); + std::function recursive = [&](size_t k, double memory, double communication, + double communication_without_para) { + if (k == edges.size()) { + auto decision = std::make_shared(selected_cost_list); + CostPtr new_cost = std::make_shared(memory, communication); + MS_EXCEPTION_IF_NULL(new_cost); + new_cost->communication_without_parameter_ = communication_without_para; + new_cost->communication_with_partial_para_ = + communication_without_para + COST_MODEL_GAMMA * (communication - communication_without_para); + new_cost->decision_ptr_ = decision; + result.push_back(new_cost); + return; + } + for (auto& c : all_cost_list[k]) { + MS_EXCEPTION_IF_NULL(c); + selected_cost_list[k] = c; + recursive(k + 1, memory + c->memory_cost_, communication + c->communication_cost_, + communication_without_para + c->communication_without_parameter_); + } + }; + recursive(0, 0, 0, 0); + SimplifyForDreasingCommunicationWithPartialPara(&result); + return result; +} + +void Edge::EdgeEliminationSetNewCost(OperatorInfoPtr, const std::vector& edges, OperatorInfoPtr) { + bool valid = false; + for (const auto& output_pair : pre_op_output_) { + StrategyPtr output_st_ptr = output_pair.first; + for (const auto& input_pair : next_op_input_) { + StrategyPtr input_st_ptr = input_pair.first; + CostPtrList clist = CreateEdgeEliminationCostList(output_st_ptr, edges, input_st_ptr); + CostPtrKey key = {output_st_ptr, input_st_ptr}; + cost_map_[key] = clist; + if ((!valid) && (!clist.empty())) { + valid = true; + } + } + } + if (!valid) { + MS_LOG(EXCEPTION) << "Creating edge: " << edge_name_ << " failed."; + } +} + +void Edge::CreateOpEliminationSubCostList(StrategyPtr op_strategy, const CostPtrList& left_cost_list, + const CostPtrList& middle_cost_list, const CostPtrList& right_cost_list, + CostPtrList* ret_cost_list) { + for (auto& left_cost : left_cost_list) { + MS_EXCEPTION_IF_NULL(left_cost); + for (auto& middle_cost : middle_cost_list) { + MS_EXCEPTION_IF_NULL(middle_cost); + for (auto& right_cost : right_cost_list) { + MS_EXCEPTION_IF_NULL(right_cost); + double memory = left_cost->memory_cost_ + middle_cost->memory_cost_ + right_cost->memory_cost_; + double communication = + left_cost->communication_cost_ + middle_cost->communication_cost_ + right_cost->communication_cost_; + double communication_without_para = left_cost->communication_without_parameter_ + + middle_cost->communication_without_parameter_ + + right_cost->communication_without_parameter_; + + auto decision = std::make_shared(op_strategy, left_cost, middle_cost, right_cost); + auto cost = std::make_shared(memory, communication, decision); + MS_EXCEPTION_IF_NULL(cost); + cost->communication_without_parameter_ = communication_without_para; + cost->communication_with_partial_para_ = + communication_without_para + COST_MODEL_GAMMA * (communication - communication_without_para); + ret_cost_list->emplace_back(std::move(cost)); + } + } + } +} + +CostPtrList Edge::CreateOpEliminationCostList(const EdgePtr& e1, const StrategyPtr& output_st_ptr, + const OperatorInfoPtr& op, const EdgePtr& e2, + const StrategyPtr& input_st_ptr) { + MS_EXCEPTION_IF_NULL(op); + MS_EXCEPTION_IF_NULL(e1); + MS_EXCEPTION_IF_NULL(e2); + CostPtrList result; + for (const auto& op_strategy : op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(op_strategy); + auto middle_strategy = op_strategy->strategy_ptr; + CreateOpEliminationSubCostList(middle_strategy, e1->GetCostList(output_st_ptr, middle_strategy), + op_strategy->cost_list, e2->GetCostList(middle_strategy, input_st_ptr), &result); + } + SimplifyForDreasingCommunicationWithPartialPara(&result); + return result; +} + +void Edge::OpEliminationSetNewCost(const EdgePtr& e1, const OperatorInfoPtr& op, const EdgePtr& e2) { + bool valid = false; + for (const auto& output_pair : pre_op_output_) { + StrategyPtr output_st_ptr = output_pair.first; + for (const auto& input_pair : next_op_input_) { + StrategyPtr input_st_ptr = input_pair.first; + + CostPtrList clist = CreateOpEliminationCostList(e1, output_st_ptr, op, e2, input_st_ptr); + CostPtrKey key = {output_st_ptr, input_st_ptr}; + cost_map_[key] = clist; + if ((!valid) && (!clist.empty())) { + valid = true; + } + } + } + if (!valid) { + MS_LOG(EXCEPTION) << "Creating edge: " << edge_name_ << " failed."; + } +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/edge_costmodel.h b/mindspore/ccsrc/parallel/auto_parallel/edge_costmodel.h new file mode 100644 index 00000000000..d2bad602158 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/edge_costmodel.h @@ -0,0 +1,163 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_EDGE_COSTMODEL_H_ +#define PARALLEL_AUTO_PARALLEL_EDGE_COSTMODEL_H_ + +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/tensor_layout/tensor_info.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/costmodel.h" + +namespace mindspore { +namespace parallel { +using CostPtrKey = std::pair; +using OperatorInfoPtr = std::shared_ptr; +using EdgePtr = std::shared_ptr; + +class Edge { + // An 'Edge' connects two Operators in the CostGraph. + public: + Edge(const std::string& edge_name, const std::shared_ptr& prev_op, + const std::shared_ptr& next_op, const size_t& output_index_, const size_t& input_index_, + const bool& is_com) + : edge_name_(edge_name), + prev_op_(prev_op), + next_op_(next_op), + prev_op_output_index_(output_index_), + next_op_input_index_(input_index_), + is_combined_(is_com) { + is_identity_edge = false; + } + + Edge(const std::string& edge_name, const std::shared_ptr& prev_op, + const std::shared_ptr& next_op, const size_t& output_index_, const size_t& input_index_, + const bool& is_com, const bool& is_iden) + : edge_name_(edge_name), + prev_op_(prev_op), + next_op_(next_op), + prev_op_output_index_(output_index_), + next_op_input_index_(input_index_), + is_combined_(is_com), + is_identity_edge(is_iden) {} + + Edge(const std::string& edge_name, const std::shared_ptr& prev_op, + const std::shared_ptr& next_op, const std::vector& output_indexs_, + const std::vector& input_indexs_, const bool& is_com) + : edge_name_(edge_name), + prev_op_(prev_op), + next_op_(next_op), + pre_op_output_indexs_(output_indexs_), + next_op_input_indexs_(input_indexs_), + is_combined_(is_com) { + prev_op_output_index_ = 0; + next_op_input_index_ = 0; + is_identity_edge = false; + } + + ~Edge() = default; + std::shared_ptr prev_operator() const { return prev_op_; } + std::shared_ptr next_operator() const { return next_op_; } + std::string edge_name() const { return edge_name_; } + // Init cost_map_: for each output layout and input layout, calculate the cost + Status InitEdgeCost(); + // For two operators u--->v, given the output tensor layout of u, + // and the input tensor layout of v, return the redistribution cost, + // and the op_list to carry out the redistribution. + Status GetRedistributionCost(const TensorLayout& prev_op_output_layout, const TensorLayout& next_op_input_layout, + size_t, CostPtr* cost); + + void set_pre_op_output(const std::vector, std::vector>>& output_set) { + pre_op_output_ = output_set; + } + void set_next_op_input(const std::vector, std::vector>>& input_set) { + next_op_input_ = input_set; + } + + // Given a pair of output strategy and input strategy, return the corresponding costlist + CostPtrList GetCostList(StrategyPtr output_str, StrategyPtr input_str); + + std::vector, std::vector>> prev_op_output() const { + return pre_op_output_; + } + std::vector, std::vector>> next_op_input() const { + return next_op_input_; + } + + bool is_combined() const { return is_combined_; } + size_t prev_op_output_index() const { return prev_op_output_index_; } + size_t next_op_input_index() const { return next_op_input_index_; } + std::vector prev_op_output_indexs() const { return pre_op_output_indexs_; } + std::vector next_op_input_indexs() const { return next_op_input_indexs_; } + + CostPtrList CreateEdgeEliminationCostList(const StrategyPtr& output_st_ptr, + const std::vector>& edges, + const StrategyPtr& input_st_ptr); + // In the Edge Elimination operation in DP algorithm, 'edges' is replaced by a new edge. This method is used to + // set cost for this new edge + void EdgeEliminationSetNewCost(std::shared_ptr u, const std::vector>& edges, + std::shared_ptr v); + void CreateOpEliminationSubCostList(StrategyPtr op_strategy, const CostPtrList& left_cost_list, + const CostPtrList& middle_cost_list, const CostPtrList& right_cost_list, + CostPtrList* ret_cost_list); + + CostPtrList CreateOpEliminationCostList(const std::shared_ptr& e1, const StrategyPtr& output_st_ptr, + const std::shared_ptr& op, const std::shared_ptr& e2, + const StrategyPtr& input_st_ptr); + // In the Operation Elimination operation in DP algorithm, 'op', 'e1' and 'e2' are replaced by a new edge. + // This method is used to set cost for this new edge + void OpEliminationSetNewCost(const std::shared_ptr& e1, const std::shared_ptr& op, + const std::shared_ptr& e2); + + void set_selected_cost(const CostPtr& cost) { selected_cost_ = cost; } + const CostPtr& selected_cost() const { return selected_cost_; } + void set_parameter_involve(int para_invol) { is_output_parameter_involve_ = para_invol; } + // When the input of a operator contains WEIGHT or a output from other operators involving WEIGHT, then these input + // should stay in memory until it is used in the backward phase, which is kept in memory at the end of forward phase. + Status CorrectStrategyCostForMemoryReuse() const { return SUCCESS; } + + private: + std::string edge_name_; + std::shared_ptr prev_op_, next_op_; + std::map cost_map_; + // pre_op_output_ + std::vector, std::vector>> pre_op_output_; + std::vector, std::vector>> next_op_input_; + // the index of outputs of prev_op, and the index of inputs of next_op + size_t prev_op_output_index_, next_op_input_index_; + + // pre_op_output_indexs_ and next_op_input_indexs_ store the indexs of inputs and outputs if is_combined = true + std::vector pre_op_output_indexs_; + std::vector next_op_input_indexs_; + // is this edge constructed by combining multiple edges? If is is, then is_combined = true, else is_combined = false + bool is_combined_; + // When a Parameter in the ANF graph being used by multiple operators, we include the Parameter in the costgraph by + // replace the Parameter by a TmpIdentity operator, and connecting this TmpIdentity operator with subsequent + // operators. The resulting edges are different from those normal edges, thus this Bool variable distinguishes them. + // If it is true, then we should guarantee that the strategy for output tensor consistent with the input tensor. + bool is_identity_edge; + CostPtr selected_cost_; + int is_output_parameter_involve_ = -1; // -1: unset; 0: not parameter_involved; 1: parameter_involved +}; +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_EDGE_COSTMODEL_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc new file mode 100644 index 00000000000..d30522c2fe1 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc @@ -0,0 +1,1364 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/graph_costmodel.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace mindspore { +namespace parallel { +CostGraphPtr entire_costgraph = nullptr; +size_t TOTAL_OPS = 0; +double COST_MODEL_GAMMA = DEFAULT_COST_MODEL_GAMMA; +bool COST_MODEL_SIMPLIFY_CALCULATION = DEFAULT_COST_MODEL_SIMPLIFY_CALCULATION; +double DEVICE_MEMORY_CAPACITY = DEFAULT_DEVICE_MEMORY_CAPACITY; +double COST_MODEL_COMMUNI_THRESHOLD = DEFAULT_COST_MODEL_COMMUNI_THRESHOLD; +double COST_MODEL_COMMUNI_CONST = DEFAULT_COST_MODEL_COMMUNI_CONST; +double COST_MODEL_COMMUNI_BIAS = DEFAULT_COST_MODEL_COMMUNI_BIAS; +bool TENSOR_SLICE_ALIGNMENT_ENABLE = DEFAULT_TENSOR_SLICE_ALIGNMENT_ENABLE; +size_t TENSOR_SLICE_ALIGNMENT_SIZE = DEFAULT_TENSOR_SLICE_ALIGNMENT_SIZE; +bool NOT_FULLY_USE_DEVICES = DEFAULT_NOT_FULLY_USE_DEVICES; +bool ELEMENTWISE_OP_STRA_FOLLOW = DEFAULT_ELEMENTWISE_OP_STRA_FOLLOW; + +void CostGraph::SetDeviceMemoryAndCostParameter() { + MS_EXCEPTION_IF_NULL(CostModelContext::GetInstance()); + + // DEVICE_MEMORY_CAPACITY + auto device_memory = CostModelContext::GetInstance()->device_memory_capacity(); + if (device_memory <= 0) { + MS_LOG(EXCEPTION) << "'device_memory_capacity' must be positive."; + } + dev_memory_ = device_memory; + DEVICE_MEMORY_CAPACITY = device_memory; + MS_LOG(INFO) << "device_memory_capacity: " << DEVICE_MEMORY_CAPACITY << "."; + + // COST_MODEL_ALPHA + auto alpha = CostModelContext::GetInstance()->costmodel_alpha(); + if (alpha <= 0) { + MS_LOG(EXCEPTION) << "'costmodel_alpha' must be positive."; + } + costmodel_alpha_ = alpha; + MS_LOG(INFO) << "costmodel_alpha: " << costmodel_alpha_ << "."; + + // COST_MODEL_BETA + auto beta = CostModelContext::GetInstance()->costmodel_beta(); + if (beta <= 0) { + MS_LOG(EXCEPTION) << "'costmodel_beta' must be positive."; + } + costmodel_beta_ = beta; + MS_LOG(INFO) << "costmodel_beta: " << costmodel_beta_ << "."; + + // COST_MODEL_GAMMA + auto gamma = CostModelContext::GetInstance()->costmodel_gamma(); + if ((gamma < 0) || (gamma > 1)) { + MS_LOG(EXCEPTION) << "'costmodel_gamma' must in [0, 1]."; + } + COST_MODEL_GAMMA = gamma; + MS_LOG(INFO) << "costmodel_gamma: " << COST_MODEL_GAMMA << "."; + + // COST_MODEL_SIMPLIFY_CALCULATION + auto simplify = CostModelContext::GetInstance()->costmodel_simplify_cal(); + COST_MODEL_SIMPLIFY_CALCULATION = simplify; + if (COST_MODEL_SIMPLIFY_CALCULATION) { + MS_LOG(INFO) << "costmodel_simplify_cal: true."; + } else { + MS_LOG(INFO) << "costmodel_simplify_cal: false."; + } + + // COST_MODEL_COMMUNI_THRESHOLD + auto communi_threshold = CostModelContext::GetInstance()->costmodel_communi_threshold(); + if (communi_threshold < 0) { + MS_LOG(EXCEPTION) << "'costmodel_communi_threshold' must be non-zero."; + } + COST_MODEL_COMMUNI_THRESHOLD = communi_threshold; + MS_LOG(INFO) << "costmodel_communi_threshold: " << COST_MODEL_COMMUNI_THRESHOLD << "."; + + // COST_MODEL_COMMUNI_CONST + auto communi_const = CostModelContext::GetInstance()->costmodel_communi_const(); + if (communi_const < 0) { + MS_LOG(EXCEPTION) << "'costmodel_communi_const' must be non-zero."; + } + COST_MODEL_COMMUNI_CONST = communi_const; + MS_LOG(INFO) << "costmodel_communi_const: " << COST_MODEL_COMMUNI_CONST << "."; + + // COST_MODEL_COMMUNI_BIAS + auto communi_bias = CostModelContext::GetInstance()->costmodel_communi_bias(); + if (communi_bias < 0) { + MS_LOG(EXCEPTION) << "'costmodel_communi_bias' must be non-zero."; + } + COST_MODEL_COMMUNI_BIAS = communi_bias; + MS_LOG(INFO) << "costmodel_communi_bias: " << COST_MODEL_COMMUNI_BIAS << "."; + + // TENSOR_SLICE_ALIGNMENT_ENABLE + auto align_enable = CostModelContext::GetInstance()->tensor_slice_alignment_enable(); + TENSOR_SLICE_ALIGNMENT_ENABLE = align_enable; + if (TENSOR_SLICE_ALIGNMENT_ENABLE) { + MS_LOG(INFO) << "tensor_slice_align_enable: true."; + } else { + MS_LOG(INFO) << "tensor_slice_align_enable: false."; + } + + // TENSOR_SLICE_ALIGNMENT_SIZE + auto align_size = CostModelContext::GetInstance()->tensor_slice_alignment_size(); + if (align_size == 0) { + MS_LOG(EXCEPTION) << "'tensor_slice_align_size' must be positive."; + } + TENSOR_SLICE_ALIGNMENT_SIZE = align_size; + MS_LOG(INFO) << "tensor_slice_align_size: " << TENSOR_SLICE_ALIGNMENT_SIZE << "."; + + // NOT_FULLY_USE_DEVICES + auto not_fully_devices = CostModelContext::GetInstance()->not_fully_use_device(); + NOT_FULLY_USE_DEVICES = not_fully_devices; + if (NOT_FULLY_USE_DEVICES) { + MS_LOG(INFO) << "not_fully_use_devices: true."; + } else { + MS_LOG(INFO) << "not_fully_use_devices: false."; + } + + // ELEMENTWISE_OP_STRA_FOLLOW + auto is_ele_op_follow = CostModelContext::GetInstance()->elementwise_stra_follow(); + ELEMENTWISE_OP_STRA_FOLLOW = is_ele_op_follow; + if (ELEMENTWISE_OP_STRA_FOLLOW) { + MS_LOG(INFO) << "elementwise_op_strategy_follow: true."; + } else { + MS_LOG(INFO) << "elementwise_op_strategy_follow: false."; + } +} + +void CostGraph::RemoveOperator(const OperatorInfoPtr& op) { + for (auto it = ops_.begin(); it != ops_.end();) { + if ((*it) == op) { + it = ops_.erase(it); + } else { + ++it; + } + } +} + +bool CostGraph::IsOperatorInCostGraph(const OperatorInfoPtr& op_test) { + struct IsInGraph { + const OperatorInfoPtr test_; + explicit IsInGraph(const OperatorInfoPtr& n) : test_(n) {} + bool operator()(const OperatorInfoPtr& in) const { return (test_ == in); } + }; + return std::any_of(ops_.begin(), ops_.end(), IsInGraph(op_test)); +} + +bool CostGraph::IsEdgeInCostGraph(const std::string& test_edge_name, size_t output_index, size_t input_index) { + for (auto& edge_pair : edges_) { + auto edges = edge_pair.second; + for (auto& edge : edges) { + MS_EXCEPTION_IF_NULL(edge); + bool bool_result = (edge->edge_name() == test_edge_name) && (edge->prev_op_output_index() == output_index) && + (edge->next_op_input_index() == input_index); + if (bool_result) { + return true; + } + } + } + return false; +} + +std::vector> CostGraph::ConstructConnectedComponents( + std::vector alive_ops) { + std::map visited; + + for (auto& op : alive_ops) { + visited[op] = false; + } + + MS_LOG(INFO) << "visited: " << visited.size() << "."; + for (auto& op : alive_ops) { + if ((!visited[op]) && op->is_alive()) { + std::shared_ptr new_component = std::make_shared(); + MS_EXCEPTION_IF_NULL(new_component); + new_component->SetDeviceMemoryAndCostParameter(); + DFS(op, &visited, new_component); + connected_compoents_.push_back(new_component); + } + } + return connected_compoents_; +} + +void CostGraph::DFS(const OperatorInfoPtr& current_op, std::map* visited, + const std::shared_ptr& component) { + MS_EXCEPTION_IF_NULL(visited); + MS_EXCEPTION_IF_NULL(component); + visited->at(current_op) = true; + component->AddOperator(current_op); + + for (auto& edge : current_op->succ_edges()) { + bool bool_test = (visited->find(edge->next_operator()) != visited->end()) && + (!visited->at(edge->next_operator())) && edge->next_operator()->is_alive(); + if (bool_test) { + component->AddEdge(current_op, edge->next_operator(), edge); + DFS(edge->next_operator(), visited, component); + } + } + + for (auto& edge : current_op->prev_edges()) { + bool bool_test = (visited->find(edge->prev_operator()) != visited->end()) && + (!visited->at(edge->prev_operator())) && edge->prev_operator()->is_alive(); + if (bool_test) { + component->AddEdge(edge->prev_operator(), current_op, edge); + DFS(edge->prev_operator(), visited, component); + } + } +} + +// Create final cost list for the graph: u --> v +CostPtrList CostGraph::CreateFinalCostList(const OperatorInfoPtr& u, const std::shared_ptr& e, + const OperatorInfoPtr& v) { + MS_EXCEPTION_IF_NULL(u); + MS_EXCEPTION_IF_NULL(v); + MS_EXCEPTION_IF_NULL(e); + CostPtrList ret; + for (const auto& u_strategy : u->GetStrategyCost()) { + for (const auto& v_strategy : v->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(u_strategy); + MS_EXCEPTION_IF_NULL(v_strategy); + auto u_strategy_ptr = u_strategy->strategy_ptr; + auto v_strategy_ptr = v_strategy->strategy_ptr; + CostPtrList clist1 = u_strategy->cost_list; + CostPtrList clist2 = e->GetCostList(u_strategy_ptr, v_strategy_ptr); + CostPtrList clist3 = v_strategy->cost_list; + for (const auto& cost1 : clist1) { + for (const auto& cost2 : clist2) { + for (const auto& cost3 : clist3) { + MS_EXCEPTION_IF_NULL(cost1); + MS_EXCEPTION_IF_NULL(cost2); + MS_EXCEPTION_IF_NULL(cost3); + double memory = cost1->memory_cost_ + cost2->memory_cost_ + cost3->memory_cost_; + double commmunication = + cost1->communication_cost_ + cost2->communication_cost_ + cost3->communication_cost_; + double communication_without_para = cost1->communication_without_parameter_ + + cost2->communication_without_parameter_ + + cost3->communication_without_parameter_; + auto decision = + std::make_shared(u_strategy->strategy_ptr, v_strategy->strategy_ptr, cost1, cost2, cost3); + auto cost = std::make_shared(memory, commmunication, decision); + MS_EXCEPTION_IF_NULL(cost); + cost->communication_without_parameter_ = communication_without_para; + cost->communication_with_partial_para_ = + communication_without_para + COST_MODEL_GAMMA * (commmunication - communication_without_para); + ret.push_back(cost); + } + } + } + } + } + + SimplifyForDreasingCommunicationWithPartialPara(&ret); + return ret; +} + +// Create final cost list for the graph containing a signle node: u +CostPtrList CostGraph::CreateFinalSingleCostList(const OperatorInfoPtr& u) { + MS_EXCEPTION_IF_NULL(u); + CostPtrList ret; + for (const auto& u_strategy : u->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(u_strategy); + auto u_strategy_ptr = u_strategy->strategy_ptr; + CostPtrList clist1 = u_strategy->cost_list; + for (const auto& cost1 : clist1) { + MS_EXCEPTION_IF_NULL(cost1); + auto decision = std::make_shared(u_strategy_ptr, cost1); + auto new_cost = std::make_shared(cost1->memory_cost_, cost1->communication_cost_, decision); + MS_EXCEPTION_IF_NULL(new_cost); + new_cost->communication_without_parameter_ = cost1->communication_without_parameter_; + new_cost->communication_with_partial_para_ = + cost1->communication_without_parameter_ + + COST_MODEL_GAMMA * (cost1->communication_cost_ - cost1->communication_without_parameter_); + ret.push_back(new_cost); + } + } + + SimplifyForDreasingCommunicationWithPartialPara(&ret); + return ret; +} + +CostPtr CostGraph::SelectCostWithMemoryConstraint(const CostPtrList& cost_list, double memory) { + if (cost_list.empty() || cost_list[0]->memory_cost_ >= memory) { + return nullptr; + } + std::function LocalCompare = [&](CostPtr init, const CostPtr& cost_x) { + MS_EXCEPTION_IF_NULL(cost_x); + if (init == nullptr || cost_x->memory_cost_ < memory) { + init = cost_x; + } + return init; + }; + CostPtr ret = nullptr; + return std::accumulate(cost_list.begin(), cost_list.end(), ret, LocalCompare); +} + +CostPtr CostGraph::SelectCostWithMinTrainingTime(const CostPtrList& cost_list, double memory) { + // Select the cost with minimum training time. Currently, the training time is modeled as = + // costmodel_alpha_ * memory_cost + costmodel_beta_ * communication_with_partial_para_ + if (cost_list.empty()) { + MS_LOG(ERROR) << "Final cost list is null."; + return nullptr; + } + CostPtr ret = cost_list[0]; + MS_EXCEPTION_IF_NULL(ret); + if (ret->memory_cost_ >= memory) { + MS_LOG(ERROR) << "No available cost; the minimum cost is " << ret->memory_cost_ + << ", the memory capacity is: " << memory << "."; + return nullptr; + } + double minimum = costmodel_alpha_ * ret->memory_cost_ + costmodel_beta_ * ret->communication_with_partial_para_; + MS_LOG(INFO) << "minimum: " << minimum << ", memory_cost_: " << ret->memory_cost_ + << ", communication_with_partial_para_: " << ret->communication_with_partial_para_ + << ", communication_cost_: " << ret->communication_cost_ + << ", communication_without_parameter_: " << ret->communication_without_parameter_ << "."; + for (size_t i = 1; i < cost_list.size(); ++i) { + MS_EXCEPTION_IF_NULL(cost_list[i]); + if (cost_list[i]->memory_cost_ >= memory) { + MS_LOG(INFO) << "cost_list " << i << " memory_cost_: " << cost_list[i]->memory_cost_ + << ", is larger than the memory capacity: " << memory << "."; + break; + } + MS_LOG(INFO) << "cost_list " << i << " memory_cost_: " << cost_list[i]->memory_cost_ + << ", communication_with_partial_para_: " << cost_list[i]->communication_with_partial_para_ + << ", communication_cost_: " << cost_list[i]->communication_cost_ + << ", communication_without_parameter_: " << cost_list[i]->communication_without_parameter_ << "."; + auto tmp = + costmodel_alpha_ * cost_list[i]->memory_cost_ + costmodel_beta_ * cost_list[i]->communication_with_partial_para_; + MS_LOG(INFO) << "tmp: " << tmp; + if (minimum > tmp) { + minimum = tmp; + ret = cost_list[i]; + MS_LOG(INFO) << "selected: " << i; + } + } + return ret; +} + +CostPtrList CostGraph::SelectCostListWithMinTrainingTimeMultiple(const std::vector& all_cost_list, + double available_memory) { + CostPtrList selected_cost_list(all_cost_list.size(), nullptr); + double minimum = 0.0, total_memory = 0.0; + CostPtrList ret(all_cost_list.size(), nullptr); + for (size_t i = 0; i < all_cost_list.size(); ++i) { + if (all_cost_list[i][0] == nullptr) { + MS_LOG(ERROR) << "The cost list " << i << " is empty."; + return ret; + } else { + total_memory += all_cost_list[i][0]->memory_cost_; + minimum += costmodel_alpha_ * all_cost_list[i][0]->memory_cost_ + + costmodel_beta_ * all_cost_list[i][0]->communication_with_partial_para_; + ret[i] = all_cost_list[i][0]; + } + } + if (total_memory >= available_memory) { + MS_LOG(ERROR) << "No strategy can be found under current memory: " << available_memory + << ", minimum strategy cost: " << total_memory << "."; + return selected_cost_list; + } + + std::function recursive = [&all_cost_list, &selected_cost_list, &minimum, &ret, &recursive, + &available_memory, this](size_t k) { + if (k == all_cost_list.size()) { + double tmp_memory = 0.0, tmp_minimum = 0.0; + for (size_t i = 0; i < selected_cost_list.size(); ++i) { + MS_EXCEPTION_IF_NULL(selected_cost_list[i]); + tmp_memory += selected_cost_list[i]->memory_cost_; + tmp_minimum += costmodel_alpha_ * selected_cost_list[i]->memory_cost_ + + costmodel_beta_ * selected_cost_list[i]->communication_with_partial_para_; + } + MS_LOG(INFO) << "tmp_memory: " << tmp_memory << ", tmp_minimum: " << tmp_minimum << ", minimum: " << minimum + << "."; + if (tmp_memory < available_memory && tmp_minimum < minimum) { + ret = selected_cost_list; + minimum = tmp_minimum; + MS_LOG(INFO) << "selected tmp_memory: " << tmp_memory << ", tmp_minimum: " << tmp_minimum << "."; + } + return; + } + MS_LOG(DEBUG) << "The value minimum: " << minimum << ", available_memory: " << available_memory << "."; + for (auto& c : all_cost_list[k]) { + selected_cost_list[k] = c; + recursive(k + 1); + } + }; + recursive(0); + return ret; +} + +Status CostGraph::SearchStrategyForMultiNodeFinalGraph(const std::vector& alive_ops) { + MS_LOG(INFO) << "There are " << alive_ops.size() << " nodes in the final graph."; + auto connected_components = ConstructConnectedComponents(alive_ops); + MS_LOG(INFO) << "There are " << connected_components.size() << " components in the final graph."; + std::vector all_list; + for (size_t j = 0; j < connected_components.size(); ++j) { + auto one_component = connected_components[j]; + MS_EXCEPTION_IF_NULL(one_component); + if (one_component->GetOperators().size() == 1) { + MS_LOG(INFO) << "There are 1 operator in a component in the final graph."; + auto cost_list = one_component->CreateFinalSingleCostList(one_component->GetOperators()[0]); + all_list.push_back(cost_list); + } else if (one_component->GetOperators().size() == 2) { + MS_LOG(INFO) << "There are 2 operators in a component in the final graph."; + OperatorInfoPtr u, v; + auto first_op = one_component->GetOperators()[0]; + auto second_op = one_component->GetOperators()[1]; + MS_EXCEPTION_IF_NULL(first_op); + MS_EXCEPTION_IF_NULL(second_op); + if (!first_op->GetAliveSuccEdges().empty() && + first_op->GetAliveSuccEdges()[0]->next_operator().get() == second_op.get()) { + u = first_op; + v = second_op; + } else if (!second_op->GetAliveSuccEdges().empty() && + second_op->GetAliveSuccEdges()[0]->next_operator().get() == first_op.get()) { + u = second_op; + v = first_op; + } else { + MS_LOG(EXCEPTION) << "The final graph is not the case of u --> v, " << first_op->GetAliveSuccEdges().size() + << ", " << second_op->GetAliveSuccEdges().size() << "."; + } + MS_EXCEPTION_IF_NULL(u); + auto e = u->GetAliveSuccEdges()[0]; + auto cost_list = one_component->CreateFinalCostList(u, e, v); + all_list.push_back(cost_list); + } else { + MS_LOG(EXCEPTION) << "There are " << one_component->GetOperators().size() + << " operators in a component in the final graph."; + } + } + // + auto selected_cost_list = SelectCostListWithMinTrainingTimeMultiple(all_list, dev_memory_); + for (size_t k = 0; k < selected_cost_list.size(); ++k) { + auto selected_cost = selected_cost_list[k]; + if (selected_cost == nullptr) { + MS_LOG(ERROR) << "No vaild strategy can be found under the current device memory: " << dev_memory_ << "."; + return FAILED; + } + MS_EXCEPTION_IF_NULL(connected_components[k]); + if (connected_components[k]->GetOperators().size() == 1) { + auto u = connected_components[k]->GetOperators()[0]; + auto decision = selected_cost->decision_ptr_->cast(); + u->SetSelectedStrategyAndCost(decision->u_strategy_, decision->u_cost_); + MS_LOG(INFO) << "Searching the strategy for the component " << k << " final graph ended."; + } else if (connected_components[k]->GetOperators().size() == 2) { + OperatorInfoPtr u = nullptr, v = nullptr; + auto first_op = connected_components[k]->GetOperators()[0]; + auto second_op = connected_components[k]->GetOperators()[1]; + MS_EXCEPTION_IF_NULL(first_op); + MS_EXCEPTION_IF_NULL(second_op); + if (!first_op->GetAliveSuccEdges().empty() && + first_op->GetAliveSuccEdges()[0]->next_operator().get() == second_op.get()) { + u = first_op; + v = second_op; + } else if (!second_op->GetAliveSuccEdges().empty() && + second_op->GetAliveSuccEdges()[0]->next_operator().get() == first_op.get()) { + u = second_op; + v = first_op; + } + MS_EXCEPTION_IF_NULL(u); + auto e = u->GetAliveSuccEdges()[0]; + MS_EXCEPTION_IF_NULL(v); + MS_EXCEPTION_IF_NULL(e); + MS_EXCEPTION_IF_NULL(selected_cost->decision_ptr_); + auto decision = selected_cost->decision_ptr_->cast(); + MS_EXCEPTION_IF_NULL(decision); + u->SetSelectedStrategyAndCost(decision->u_strategy_, decision->left_cost_); + v->SetSelectedStrategyAndCost(decision->v_strategy_, decision->right_cost_); + e->set_selected_cost(decision->middle_cost_); + MS_LOG(INFO) << "Searching the strategy for the component " << k << " final graph ended."; + } + } + return SUCCESS; +} + +// searching the strategy for the final eliminated graph +Status CostGraph::SearchStrategy() { + MS_LOG(INFO) << "Searching the strategy for the eliminated final graph began."; + std::vector alive_ops; + (void)std::for_each(ops_.begin(), ops_.end(), [&alive_ops](const OperatorInfoPtr& op) { + MS_EXCEPTION_IF_NULL(op); + if (op->is_alive()) { + alive_ops.push_back(op); + } + }); + + if (alive_ops.size() > 2) { + return SearchStrategyForMultiNodeFinalGraph(alive_ops); + } else if (alive_ops.size() == 1) { + MS_LOG(INFO) << "There are 1 single node in the final graph."; + OperatorInfoPtr u = alive_ops[0]; + auto cost_list = CreateFinalSingleCostList(u); + auto cost = SelectCostWithMinTrainingTime(cost_list, dev_memory_); + if (cost == nullptr) { + MS_LOG(ERROR) << "No vaild strategy can be found under the current device memory: " << dev_memory_ << "."; + return FAILED; + } + MS_EXCEPTION_IF_NULL(u); + MS_EXCEPTION_IF_NULL(cost->decision_ptr_); + auto decision = cost->decision_ptr_->cast(); + MS_EXCEPTION_IF_NULL(decision); + u->SetSelectedStrategyAndCost(decision->u_strategy_, decision->u_cost_); + MS_LOG(INFO) << "Searching the strategy for the eliminated final graph ended."; + return SUCCESS; + } else { + // In this case, the final graph should contains exactly 2 nodes. + if (alive_ops.empty()) { + MS_LOG(INFO) << "0 Operator in the final graph."; + return SUCCESS; + } + OperatorInfoPtr u, v; + MS_EXCEPTION_IF_NULL(alive_ops[0]); + MS_EXCEPTION_IF_NULL(alive_ops[1]); + if (!alive_ops[0]->GetAliveSuccEdges().empty() && + alive_ops[0]->GetAliveSuccEdges()[0]->next_operator().get() == alive_ops[1].get()) { + u = alive_ops[0]; + v = alive_ops[1]; + } else if (!alive_ops[1]->GetAliveSuccEdges().empty() && + alive_ops[1]->GetAliveSuccEdges()[0]->next_operator().get() == alive_ops[0].get()) { + u = alive_ops[1]; + v = alive_ops[0]; + } else { + if (!alive_ops[0]->GetAliveSuccEdges().empty() || !alive_ops[1]->GetAliveSuccEdges().empty()) { + MS_LOG(EXCEPTION) << "The final graph is not the case of u --> v, " << alive_ops[0]->GetAliveSuccEdges().size() + << ", " << alive_ops[1]->GetAliveSuccEdges().size() << "."; + } else { + // In this case, the final graph consists of two single nodes + MS_LOG(INFO) << "There are 2 single nodes in the final graph."; + std::vector all_list; + auto connected_components = ConstructConnectedComponents(alive_ops); + MS_LOG(INFO) << "There are " << connected_components.size() << " components in the final graph."; + for (size_t i = 0; i < connected_components.size(); ++i) { + MS_LOG(INFO) << "There are 1 operator in a component in the final graph."; + auto one_component = connected_components[i]; + MS_EXCEPTION_IF_NULL(one_component); + auto cost_list = one_component->CreateFinalSingleCostList(one_component->GetOperators()[0]); + all_list.push_back(cost_list); + } + auto selected_cost_list = SelectCostListWithMinTrainingTimeMultiple(all_list, dev_memory_); + for (size_t k = 0; k < selected_cost_list.size(); ++k) { + auto selected_cost = selected_cost_list[k]; + if (selected_cost == nullptr) { + MS_LOG(ERROR) << "No vaild strategy can be found under the current device memory: " << dev_memory_ << "."; + return FAILED; + } + MS_EXCEPTION_IF_NULL(connected_components[k]); + auto one_operator = connected_components[k]->GetOperators()[0]; + MS_EXCEPTION_IF_NULL(selected_cost->decision_ptr_); + auto decision = selected_cost->decision_ptr_->cast(); + MS_EXCEPTION_IF_NULL(decision); + one_operator->SetSelectedStrategyAndCost(decision->u_strategy_, decision->u_cost_); + MS_LOG(INFO) << "Searching the strategy for the component " << k << " final graph ended."; + } + + return SUCCESS; + } + } + MS_LOG(INFO) << "There are 2 nodes in the final graph."; + // In this case, the finale graph is exactly of the form: u --> v + MS_EXCEPTION_IF_NULL(u); + MS_EXCEPTION_IF_NULL(v); + auto e = u->GetAliveSuccEdges()[0]; + MS_EXCEPTION_IF_NULL(e); + auto cost_list = CreateFinalCostList(u, e, v); + auto cost = SelectCostWithMinTrainingTime(cost_list, dev_memory_); + if (cost == nullptr) { + MS_LOG(ERROR) << "No vaild strategy can be found under the current device memory: " << dev_memory_ << "."; + return FAILED; + } + MS_EXCEPTION_IF_NULL(cost->decision_ptr_); + auto decision = cost->decision_ptr_->cast(); + MS_EXCEPTION_IF_NULL(decision); + u->SetSelectedStrategyAndCost(decision->u_strategy_, decision->left_cost_); + v->SetSelectedStrategyAndCost(decision->v_strategy_, decision->right_cost_); + e->set_selected_cost(decision->middle_cost_); + MS_LOG(INFO) << "Searching the strategy for the eliminated final graph ended."; + return SUCCESS; + } +} + +// Given a graph which contains the following subgraph: u --> v --> w, the node v can be eliminated +// return the v and the edge u --> v +OperatorInfoPtr CostGraph::CheckOpElimination() const { + for (auto& op : ops_) { + bool bool_test = op->is_alive() && op->GetAliveSuccEdges().size() == 1 && op->GetAlivePrevEdges().size() == 1; + if (bool_test) { + if ((op->GetAliveSuccEdges()[0]->next_operator() != op) && (op->GetAlivePrevEdges()[0]->prev_operator() != op)) { + return op; + } + } + } + return nullptr; +} + +// Check the graph whether an EdgeElimination can be performed +std::vector> CostGraph::CheckEdgeElimination() const { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + if (!op->is_alive()) continue; + std::map count; + for (auto& edge : op->GetAliveSuccEdges()) { + MS_EXCEPTION_IF_NULL(edge); + auto v = edge->next_operator(); + count[v.get()]++; + } + for (auto& pair : count) { + auto* op_ptr = pair.first; + int op_count = pair.second; + if (op_count > 1) { + std::vector> ret; + for (auto& edge : op->GetAliveSuccEdges()) { + MS_EXCEPTION_IF_NULL(edge); + if (edge->next_operator().get() == op_ptr) { + ret.push_back(edge); + } + } + return ret; + } + } + } + return {}; +} + +// Check the graph whether a MergeElimination can be performed +OperatorInfoPtr CostGraph::CheckMergeElimination() const { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + bool bool_test = op->is_alive() && op->GetAlivePrevEdges().empty() && op->GetAliveSuccEdges().size() == 1; + if (bool_test) { + auto next_op = op->GetAliveSuccEdges()[0]->next_operator(); + MS_EXCEPTION_IF_NULL(next_op); + if (!next_op->GetAlivePrevEdges().empty()) { + return op; + } + } + } + return nullptr; +} + +// Check the graph whether a ContractElimination can be performed +OperatorInfoPtr CostGraph::CheckContractElimination() const { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + bool bool_test = op->is_alive() && op->GetAlivePrevEdges().size() == 1 && op->GetAliveSuccEdges().empty(); + if (bool_test) { + auto edge = op->GetAlivePrevEdges()[0]; + MS_EXCEPTION_IF_NULL(edge); + auto prev_op = edge->prev_operator(); + MS_EXCEPTION_IF_NULL(prev_op); + if (!prev_op->GetAliveSuccEdges().empty()) { + return op; + } + } + } + return nullptr; +} + +// Check the graph whether a TriangleElimination can be performed +std::pair> CostGraph::CheckTriangleElimination() const { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + bool bool_test = (op->is_alive()) && (op->GetAlivePrevEdges().empty()) && (op->GetAliveSuccEdges().size() == 2); + if (bool_test) { + auto edge1 = op->GetAliveSuccEdges()[0]; + auto edge2 = op->GetAliveSuccEdges()[1]; + MS_EXCEPTION_IF_NULL(edge1); + MS_EXCEPTION_IF_NULL(edge2); + auto first_op = edge1->next_operator(); + auto second_op = edge2->next_operator(); + MS_EXCEPTION_IF_NULL(first_op); + for (auto& first_op_succ_edge : first_op->GetAliveSuccEdges()) { + if (first_op_succ_edge->next_operator() == second_op) { + return {op, first_op_succ_edge}; + } + } + MS_EXCEPTION_IF_NULL(second_op); + for (auto& second_op_succ_edge : second_op->GetAliveSuccEdges()) { + if (second_op_succ_edge->next_operator() == first_op) { + return {op, second_op_succ_edge}; + } + } + } + } + return {nullptr, nullptr}; +} + +// Check the graph whether a StarElimination can be performed. +// NOTE: this elimination MUST be performed only when the above 5 operation cannot be applied. +OperatorInfoPtr CostGraph::CheckStarElimination() const { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + bool bool_test = (op->is_alive()) && (op->GetAlivePrevEdges().empty()) && (op->GetAliveSuccEdges().size() > 1); + if (bool_test) { + return op; + } + } + return nullptr; +} + +// This method is for 'eliminating operator' operation in the DP algorithm. It creates a new edge to replace +// 'lefe_edge', 'op' and 'right_edge'. As a consequence, it creates new costlist for the new edge. +std::shared_ptr CostGraph::EliminationOp(const OperatorInfoPtr& op) { + // in this case, the operators are organised in the form of u-->op-->v, and the goal + // is to eliminate 'op'. + MS_EXCEPTION_IF_NULL(op); + MS_LOG(INFO) << "Now eliminating node: " << op->name() << "."; + auto edge_u_op = op->GetAlivePrevEdges()[0]; + auto edge_op_v = op->GetAliveSuccEdges()[0]; + MS_EXCEPTION_IF_NULL(edge_u_op); + MS_EXCEPTION_IF_NULL(edge_op_v); + auto u = edge_u_op->prev_operator(); + auto v = edge_op_v->next_operator(); + std::vector output_indexs, input_indexs; + size_t output_index, input_index; + MS_EXCEPTION_IF_NULL(u); + MS_EXCEPTION_IF_NULL(v); + std::string new_edge_name = u->name() + OPERATOR_TO_OPERATOR_CONNECTOR + v->name(); + std::shared_ptr new_edge; + if (edge_u_op->is_combined()) { + output_indexs = edge_u_op->prev_op_output_indexs(); + } else { + output_index = edge_u_op->prev_op_output_index(); + output_indexs.push_back(output_index); + } + if (edge_op_v->is_combined()) { + input_indexs = edge_op_v->next_op_input_indexs(); + } else { + input_index = edge_op_v->next_op_input_index(); + input_indexs.push_back(input_index); + } + + if (!edge_u_op->is_combined() && !edge_op_v->is_combined()) { + new_edge = std::make_shared(new_edge_name, u, v, output_index, input_index, false); + } else { + new_edge = std::make_shared(new_edge_name, u, v, output_indexs, input_indexs, true); + } + MS_EXCEPTION_IF_NULL(new_edge); + new_edge->set_pre_op_output(edge_u_op->prev_op_output()); + new_edge->set_next_op_input(edge_op_v->next_op_input()); + new_edge->OpEliminationSetNewCost(edge_u_op, op, edge_op_v); + u->ReplaceSuccEdge(op, new_edge); + v->ReplacePreEdge(op, new_edge); + op->SetNotAlive(); + MS_LOG(INFO) << "Eliminating node: " << op->name() << " succeeded."; + return new_edge; +} + +// This method is for 'eliminating edges' operation in the DP algorithm. It creates a new edge to replace the 'edges', +// and sets new costlist for the new edge. +std::shared_ptr CostGraph::EliminationEdges(const std::vector>& edges) { + MS_LOG(INFO) << "Now eliminating " << edges.size() << " edges."; + MS_EXCEPTION_IF_NULL(edges[0]); + auto u = edges[0]->prev_operator(); + auto v = edges[0]->next_operator(); + MS_EXCEPTION_IF_NULL(u); + MS_EXCEPTION_IF_NULL(v); + std::string new_edge_name = u->name() + OPERATOR_TO_OPERATOR_CONNECTOR + v->name(); + std::vector output_indexs, input_indexs; + + for (auto& edge : edges) { + MS_EXCEPTION_IF_NULL(edge); + if (edge->is_combined()) { + auto from_output_indexs = edge->prev_op_output_indexs(); + auto from_input_indexs = edge->next_op_input_indexs(); + (void)std::copy(from_output_indexs.begin(), from_output_indexs.end(), std::back_inserter(output_indexs)); + (void)std::copy(from_input_indexs.begin(), from_input_indexs.end(), std::back_inserter(input_indexs)); + } else { + output_indexs.push_back(edge->prev_op_output_index()); + input_indexs.push_back(edge->next_op_input_index()); + } + } + + std::shared_ptr new_edge = std::make_shared(new_edge_name, u, v, output_indexs, input_indexs, true); + MS_EXCEPTION_IF_NULL(new_edge); + new_edge->set_pre_op_output(edges[0]->prev_op_output()); + new_edge->set_next_op_input(edges[0]->next_op_input()); + + new_edge->EdgeEliminationSetNewCost(u, edges, v); + + u->ReplaceSuccEdges(v, new_edge); + v->ReplacePreEdges(u, new_edge); + MS_LOG(INFO) << "Eliminating " << edges.size() << " edges succeeded."; + return new_edge; +} + +// Given 'op_cost_list', 'edge_cost_list', and 'tar_cost_list', this method is to create 'tar_cost_list_new' +// for this contract under the strategy 'op_strategy' +void CostGraph::CreateMergeEliminationSubCostList(StrategyPtr op_strategy, const CostPtrList& op_cost_list, + const CostPtrList& edge_cost_list, StrategyPtr tar_op_strategy, + const CostPtrList& tar_cost_list, + CostPtrList* const tar_cost_list_new) { + for (size_t i = 0; i < op_cost_list.size(); ++i) { + auto& op_cost = op_cost_list[i]; + MS_EXCEPTION_IF_NULL(op_cost); + for (size_t j = 0; j < edge_cost_list.size(); ++j) { + auto& edge_cost = edge_cost_list[j]; + MS_EXCEPTION_IF_NULL(edge_cost); + for (size_t k = 0; k < tar_cost_list.size(); ++k) { + auto& tar_cost = tar_cost_list[k]; + MS_EXCEPTION_IF_NULL(tar_cost); + double memory = op_cost->memory_cost_ + edge_cost->memory_cost_ + tar_cost->memory_cost_; + double communication = + op_cost->communication_cost_ + edge_cost->communication_cost_ + tar_cost->communication_cost_; + double communication_without_para = op_cost->communication_without_parameter_ + + edge_cost->communication_without_parameter_ + + tar_cost->communication_without_parameter_; + + auto decision = + std::make_shared(op_strategy, op_cost, edge_cost, tar_op_strategy, tar_cost); + auto new_cost = std::make_shared(memory, communication, decision); + MS_EXCEPTION_IF_NULL(new_cost); + new_cost->communication_without_parameter_ = communication_without_para; + new_cost->communication_with_partial_para_ = + communication_without_para + COST_MODEL_GAMMA * (communication - communication_without_para); + MS_EXCEPTION_IF_NULL(tar_cost_list_new); + tar_cost_list_new->emplace_back(std::move(new_cost)); + } + } + } +} + +// This method is for the 'Merge' operation in DP algorithm. It creates new costlist for each strategy in the +// target_op +OperatorInfoPtr CostGraph::EliminationMerge(const OperatorInfoPtr& op) { + MS_EXCEPTION_IF_NULL(op); + auto target_op = op->GetAliveSuccEdges()[0]->next_operator(); + auto edge_ptr = op->GetAliveSuccEdges()[0]; + MS_EXCEPTION_IF_NULL(target_op); + MS_EXCEPTION_IF_NULL(edge_ptr); + MS_LOG(INFO) << "Now merging " << op->name() << " into " << target_op->name() << "."; + bool valid = false; + + for (auto& tar_stra_cost : target_op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(tar_stra_cost); + auto tar_stra = tar_stra_cost->strategy_ptr; + auto tar_clist_origin = tar_stra_cost->cost_list; + CostPtrList tar_clist_new; + + for (auto& op_stra_cost : op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(op_stra_cost); + auto op_stra = op_stra_cost->strategy_ptr; + auto op_clist = op_stra_cost->cost_list; + auto edge_clist = edge_ptr->GetCostList(op_stra, tar_stra); + + CreateMergeEliminationSubCostList(op_stra, op_clist, edge_clist, tar_stra, tar_clist_origin, &tar_clist_new); + } + SimplifyForDreasingCommunicationWithPartialPara(&tar_clist_new); + // Set the new costlist w.r.t the strategy + tar_stra_cost->cost_list = tar_clist_new; + if ((!valid) && (!tar_clist_new.empty())) { + valid = true; + } + } + + if (!valid) { + MS_LOG(EXCEPTION) << "Merging " << op->name() << " into " << target_op->name() << " failed."; + } + op->SetNotAlive(); + MS_LOG(INFO) << "Merging " << op->name() << " into " << target_op->name() << " succeeded."; + return target_op; +} + +// Given 'contract_op_cost_list', 'edge_cost_list', and 'tar_cost_list', this method is to create 'tar_cost_list_new' +// for this contract under the strategy 'contract_op_stra' +void CostGraph::CreateContractEliminationSubCostList(StrategyPtr contract_op_stra, + const CostPtrList& contract_op_cost_list, + const CostPtrList& edge_cost_list, StrategyPtr target_op_stra, + const CostPtrList& tar_cost_list, CostPtrList* tar_cost_list_new) { + for (size_t i = 0; i < contract_op_cost_list.size(); ++i) { + auto& contract_op_cost = contract_op_cost_list[i]; + MS_EXCEPTION_IF_NULL(contract_op_cost); + for (size_t j = 0; j < edge_cost_list.size(); ++j) { + auto& edge_cost = edge_cost_list[j]; + MS_EXCEPTION_IF_NULL(edge_cost); + for (size_t k = 0; k < tar_cost_list.size(); ++k) { + auto& tar_cost = tar_cost_list[k]; + MS_EXCEPTION_IF_NULL(tar_cost); + double memory = contract_op_cost->memory_cost_ + edge_cost->memory_cost_ + tar_cost->memory_cost_; + double communication = + contract_op_cost->communication_cost_ + edge_cost->communication_cost_ + tar_cost->communication_cost_; + double communication_without_para = contract_op_cost->communication_without_parameter_ + + edge_cost->communication_without_parameter_ + + tar_cost->communication_without_parameter_; + + auto decision = std::make_shared(contract_op_stra, contract_op_cost, edge_cost, + target_op_stra, tar_cost); + auto new_cost = std::make_shared(memory, communication, decision); + new_cost->communication_without_parameter_ = communication_without_para; + new_cost->communication_with_partial_para_ = + communication_without_para + COST_MODEL_GAMMA * (communication - communication_without_para); + tar_cost_list_new->emplace_back(std::move(new_cost)); + } + } + } +} + +// This method is for the 'Contract' operation in DP algorithm. It creates new costlist for each strategy in the +// target_op +OperatorInfoPtr CostGraph::EliminationContract(const OperatorInfoPtr& op) { + MS_EXCEPTION_IF_NULL(op); + auto target_op = op->GetAlivePrevEdges()[0]->prev_operator(); + auto edge_ptr = op->GetAlivePrevEdges()[0]; + MS_LOG(INFO) << "Now contracting " << op->name() << " into " << target_op->name() << "."; + bool valid = false; + + for (auto& tar_stra_cost : target_op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(tar_stra_cost); + auto tar_stra = tar_stra_cost->strategy_ptr; + auto tar_clist_origin = tar_stra_cost->cost_list; + CostPtrList tar_clist_new; + + for (auto& op_stra_cost : op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(op_stra_cost); + auto op_stra = op_stra_cost->strategy_ptr; + auto op_clist = op_stra_cost->cost_list; + auto edge_clist = edge_ptr->GetCostList(tar_stra, op_stra); + + CreateContractEliminationSubCostList(op_stra, op_clist, edge_clist, tar_stra, tar_clist_origin, &tar_clist_new); + } + SimplifyForDreasingCommunicationWithPartialPara(&tar_clist_new); + // Set the new costlist w.r.t the strategy + tar_stra_cost->cost_list = tar_clist_new; + if ((!valid) && (!tar_clist_new.empty())) { + valid = true; + } + } + if (!valid) { + MS_LOG(EXCEPTION) << "Contracting " << op->name() << " into " << target_op->name() << " failed."; + } + op->SetNotAlive(); + MS_LOG(INFO) << "Contracting " << op->name() << " into " << target_op->name() << " succeeded."; + return target_op; +} + +void CostGraph::CreateTriangleEliminationSubCostListForIdentity( + StrategyPtr elimi_op_stra, StrategyPtr left_op_stra, StrategyPtr right_op_stra, const CostPtr& right_op_cost, + const CostPtrList& elimi_op_clist, const CostPtrList& left_edge_clist, const CostPtr& right_edge_cost, + const CostPtrList& left_node_clist_origin, CostPtrList* left_node_clist_new) { + MS_EXCEPTION_IF_NULL(right_edge_cost); + MS_EXCEPTION_IF_NULL(right_op_cost); + MS_EXCEPTION_IF_NULL(left_node_clist_new); + for (auto& elimi_op_cost : elimi_op_clist) { + MS_EXCEPTION_IF_NULL(elimi_op_cost); + for (auto& left_edge_cost : left_edge_clist) { + MS_EXCEPTION_IF_NULL(left_edge_cost); + for (auto& left_node_cost : left_node_clist_origin) { + MS_EXCEPTION_IF_NULL(left_node_cost); + double new_memory_cost = elimi_op_cost->memory_cost_ + left_edge_cost->memory_cost_ + + left_node_cost->memory_cost_ + right_edge_cost->memory_cost_ + + right_op_cost->memory_cost_; + double new_commu_cost = elimi_op_cost->communication_cost_ + left_edge_cost->communication_cost_ + + left_node_cost->communication_cost_ + right_edge_cost->communication_cost_ + + right_op_cost->communication_cost_; + double new_commu_without = + elimi_op_cost->communication_without_parameter_ + left_edge_cost->communication_without_parameter_ + + left_node_cost->communication_without_parameter_ + right_edge_cost->communication_without_parameter_ + + right_op_cost->communication_without_parameter_; + + auto decision = + std::make_shared(elimi_op_stra, elimi_op_cost, left_edge_cost, right_edge_cost, + left_op_stra, left_node_cost, right_op_stra, right_op_cost); + auto new_cost = std::make_shared(new_memory_cost, new_commu_cost, decision); + new_cost->communication_without_parameter_ = new_commu_without; + new_cost->communication_with_partial_para_ = + new_commu_without + COST_MODEL_GAMMA * (new_commu_cost - new_commu_without); + left_node_clist_new->emplace_back(std::move(new_cost)); + } + } + } +} + +void CostGraph::CreateTriangleEliminationSubCostListForOthers( + StrategyPtr elimi_op_stra, StrategyPtr left_node_stra, StrategyPtr right_node_stra, const CostPtr& right_op_cost, + const CostPtrList& elimi_op_clist, const CostPtrList& left_edge_clist, const CostPtr& right_edge_cost, + const CostPtrList& left_node_clist_origin, CostPtrList* left_node_clist_new) { + CostPtr elimi_op_determined = nullptr, left_edge_determined = nullptr, init_ele = nullptr; + std::function LocalCompare = [&](CostPtr init, const CostPtr& cost_x) { + MS_EXCEPTION_IF_NULL(cost_x); + if ((init == nullptr) || (cost_x->memory_cost_ < DEVICE_MEMORY_CAPACITY)) { + init = cost_x; + } + return init; + }; + + // Find a feasible elimi_op_clist + elimi_op_determined = std::accumulate(elimi_op_clist.begin(), elimi_op_clist.end(), init_ele, LocalCompare); + init_ele = nullptr; + // Find a feasible left_edge_cost + left_edge_determined = std::accumulate(left_edge_clist.begin(), left_edge_clist.end(), init_ele, LocalCompare); + if ((elimi_op_determined == nullptr) || (left_edge_determined == nullptr)) { + return; + } + if ((elimi_op_determined->memory_cost_ >= DEVICE_MEMORY_CAPACITY) || + (left_edge_determined->memory_cost_ >= DEVICE_MEMORY_CAPACITY)) { + return; + } + + for (auto& left_node_cost : left_node_clist_origin) { + MS_EXCEPTION_IF_NULL(left_node_cost); + MS_EXCEPTION_IF_NULL(right_op_cost); + double new_memory_cost = left_node_cost->memory_cost_ + elimi_op_determined->memory_cost_ + + left_edge_determined->memory_cost_ + right_edge_cost->memory_cost_ + + right_op_cost->memory_cost_; + double commu_cost = left_node_cost->communication_cost_ + elimi_op_determined->communication_cost_ + + left_edge_determined->communication_cost_ + right_edge_cost->communication_cost_ + + right_op_cost->communication_cost_; + double commu_without = + left_node_cost->communication_without_parameter_ + elimi_op_determined->communication_without_parameter_ + + left_edge_determined->communication_without_parameter_ + right_edge_cost->communication_without_parameter_ + + right_op_cost->communication_without_parameter_; + auto decision = std::make_shared(elimi_op_stra, elimi_op_determined, + left_edge_determined, right_edge_cost, left_node_stra, + left_node_cost, right_node_stra, right_op_cost); + + auto new_cost = std::make_shared(new_memory_cost, commu_cost, decision); + new_cost->communication_without_parameter_ = commu_without; + new_cost->communication_with_partial_para_ = commu_without + COST_MODEL_GAMMA * (commu_cost - commu_without); + left_node_clist_new->emplace_back(std::move(new_cost)); + } +} + +void CostGraph::CreateTriangleEliminationCostList(const OperatorInfoPtr& elimi_op, const CostPtrList& right_node_clist, + const CostPtrList& right_edge_clist, const StrategyPtr& elimi_op_stra, + const StrategyPtr& left_node_stra, const StrategyPtr& right_node_stra, + const CostPtrList& elimi_op_clist, const CostPtrList& left_edge_clist, + const CostPtrList& left_node_clist_origin, + CostPtrList* left_node_clist_new) { + // The reason for separately dealing with when the 'elimi_op' is 'TMPIDENTITY_INFO' or others is that + // when 'elimi_op' is TMPIDENTITY_INFO, the computation is limited, while 'elimi_op' is others, the computation + // may be huge + MS_EXCEPTION_IF_NULL(elimi_op); + if (elimi_op->name().find(TMPIDENTITY_INFO_NAME) != std::string::npos) { + for (auto& right_node_cost : right_node_clist) { + MS_EXCEPTION_IF_NULL(right_node_cost); + for (auto& right_edge_cost : right_edge_clist) { + MS_EXCEPTION_IF_NULL(right_edge_cost); + if ((right_node_cost->memory_cost_ < DEVICE_MEMORY_CAPACITY) && + (right_edge_cost->memory_cost_ < DEVICE_MEMORY_CAPACITY)) { + // Exact computation for TMPIDENTITY_INFO_NAME case + CreateTriangleEliminationSubCostListForIdentity(elimi_op_stra, left_node_stra, right_node_stra, + right_node_cost, elimi_op_clist, left_edge_clist, + right_edge_cost, left_node_clist_origin, left_node_clist_new); + } + } + } + } else { + for (auto& right_node_cost : right_node_clist) { + MS_EXCEPTION_IF_NULL(right_node_cost); + for (auto& right_edge_cost : right_edge_clist) { + MS_EXCEPTION_IF_NULL(right_edge_cost); + if ((right_node_cost->memory_cost_ < DEVICE_MEMORY_CAPACITY) && + (right_edge_cost->memory_cost_ < DEVICE_MEMORY_CAPACITY)) { + // Approximate computation for other case + CreateTriangleEliminationSubCostListForOthers(elimi_op_stra, left_node_stra, right_node_stra, right_node_cost, + elimi_op_clist, left_edge_clist, right_edge_cost, + left_node_clist_origin, left_node_clist_new); + } + } + } + } +} + +OperatorInfoPtr CostGraph::EliminationTriangle(const OperatorInfoPtr& elimi_op, + const std::shared_ptr& edge_left_right) { + MS_EXCEPTION_IF_NULL(edge_left_right); + MS_EXCEPTION_IF_NULL(elimi_op); + MS_LOG(INFO) << "Now eliminating triangle: " << elimi_op->name() << "."; + auto left_node = edge_left_right->prev_operator(); + auto right_node = edge_left_right->next_operator(); + auto left_edge = elimi_op->GetAliveSuccEdges()[0]; + auto right_edge = elimi_op->GetAliveSuccEdges()[1]; + MS_EXCEPTION_IF_NULL(left_node); + MS_EXCEPTION_IF_NULL(right_node); + MS_EXCEPTION_IF_NULL(left_edge); + MS_EXCEPTION_IF_NULL(right_edge); + MS_LOG(INFO) << "The left operator is: " << left_node->name() << "."; + MS_LOG(INFO) << "The right operator is: " << right_node->name() << "."; + + if (left_edge->next_operator() != left_node) { + auto tmp = left_edge; + left_edge = right_edge; + right_edge = tmp; + } + bool valid = false; + + for (auto& left_node_stra_cost : left_node->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(left_node_stra_cost); + auto left_node_stra = left_node_stra_cost->strategy_ptr; + auto left_node_clist_origin = left_node_stra_cost->cost_list; + CostPtrList left_node_clist_new; + + for (auto& elimi_op_stra_cost : elimi_op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(elimi_op_stra_cost); + auto elimi_op_stra = elimi_op_stra_cost->strategy_ptr; + auto elimi_op_clist = elimi_op_stra_cost->cost_list; + auto left_edge_clist = left_edge->GetCostList(elimi_op_stra, left_node_stra); + + for (auto& right_node_stra_cost : right_node->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(right_node_stra_cost); + auto right_node_stra = right_node_stra_cost->strategy_ptr; + auto right_node_clist = right_node_stra_cost->cost_list; + auto right_edge_clist = right_edge->GetCostList(elimi_op_stra, right_node_stra); + + CreateTriangleEliminationCostList(elimi_op, right_node_clist, right_edge_clist, elimi_op_stra, left_node_stra, + right_node_stra, elimi_op_clist, left_edge_clist, left_node_clist_origin, + &left_node_clist_new); + } + } + SimplifyForDreasingCommunicationWithPartialPara(&left_node_clist_new); + // Set the new costlist w.r.t the strategy + left_node_stra_cost->cost_list = left_node_clist_new; + if ((!valid) && (!left_node_clist_new.empty())) { + valid = true; + } + } + + if (!valid) { + MS_LOG(EXCEPTION) << "Eliminating triangle: " << elimi_op->name() << " failed."; + } + elimi_op->SetNotAlive(); + MS_LOG(INFO) << "Eliminating triangle: " << elimi_op->name() << " succeeded."; + return left_node; +} + +void CostGraph::CreateStarEliminationSubCostList(const StrategyPtr& first_succ_node_stra, + const CostPtrList& first_succ_node_clist, + const CostPtrList& first_succ_edge_clist, + const StrategyPtr& merged_op_stra, const CostPtrList& merged_op_clist, + std::vector succ_nodes_stras, + CostPtrList& succ_edges_costs, CostPtrList& succ_nodes_costs, + CostPtrList* first_succ_node_clist_new) { + for (auto& first_succ_node_cost : first_succ_node_clist) { + for (auto& first_succ_edge_cost : first_succ_edge_clist) { + for (auto& merged_node_cost : merged_op_clist) { + MS_EXCEPTION_IF_NULL(merged_node_cost); + succ_nodes_stras[0] = first_succ_node_stra; + succ_edges_costs[0] = first_succ_edge_cost; + succ_nodes_costs[0] = first_succ_node_cost; + + double memory_cost = merged_node_cost->memory_cost_, commu_cost = merged_node_cost->communication_cost_, + commu_without = merged_node_cost->communication_without_parameter_; + for (size_t i = 0; i < succ_nodes_stras.size(); ++i) { + MS_EXCEPTION_IF_NULL(succ_edges_costs[i]); + memory_cost += succ_edges_costs[i]->memory_cost_ + succ_nodes_costs[i]->memory_cost_; + commu_cost += succ_edges_costs[i]->communication_cost_ + succ_nodes_costs[i]->communication_cost_; + commu_without += succ_edges_costs[i]->communication_without_parameter_ + + succ_nodes_costs[i]->communication_without_parameter_; + } + + auto decision = std::make_shared(merged_op_stra, merged_node_cost, succ_edges_costs, + succ_nodes_stras, succ_nodes_costs); + auto new_cost = std::make_shared(memory_cost, commu_cost, decision); + new_cost->communication_without_parameter_ = commu_without; + new_cost->communication_with_partial_para_ = commu_without + COST_MODEL_GAMMA * (commu_cost - commu_without); + first_succ_node_clist_new->emplace_back(std::move(new_cost)); + } + } + } +} + +void CostGraph::CreateStarEliminationCostList(std::vector>& succ_edges, + const StrategyPtr& first_succ_node_stra, + const CostPtrList& first_succ_node_clist, + const CostPtrList& first_succ_edge_clist, + const StrategyPtr& merged_op_stra, const CostPtrList& merged_op_clist, + CostPtrList* first_succ_node_clist_new) { + std::vector succ_nodes_stras(succ_edges.size(), nullptr); + CostPtrList succ_edges_costs(succ_edges.size(), nullptr), succ_nodes_costs(succ_edges.size(), nullptr); + std::function recursive = [&first_succ_node_stra, &first_succ_node_clist, &first_succ_edge_clist, + &merged_op_stra, &merged_op_clist, &succ_nodes_stras, &succ_edges_costs, + &succ_nodes_costs, &first_succ_node_clist_new, &succ_edges, &recursive, + this](size_t k) { + if (k == succ_edges.size()) { + CreateStarEliminationSubCostList(first_succ_node_stra, first_succ_node_clist, first_succ_edge_clist, + merged_op_stra, merged_op_clist, succ_nodes_stras, succ_edges_costs, + succ_nodes_costs, first_succ_node_clist_new); + return; + } + MS_LOG(DEBUG) << "The size of first_succ_node_clist: " << first_succ_node_clist.size() + << ", first_succ_edge_clist: " << first_succ_edge_clist.size() + << ", merged_op_clist: " << merged_op_clist.size() + << ", first_succ_node_clist_new: " << first_succ_node_clist_new->size() << "."; + auto succ_edge = succ_edges[k]; + MS_EXCEPTION_IF_NULL(succ_edge); + auto succ_node = succ_edge->next_operator(); + MS_EXCEPTION_IF_NULL(succ_node); + for (auto& succ_node_stra_cost : succ_node->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(succ_node_stra_cost); + auto succ_node_stra = succ_node_stra_cost->strategy_ptr; + auto succ_node_clist = succ_node_stra_cost->cost_list; + auto succ_edge_clist = succ_edge->GetCostList(merged_op_stra, succ_node_stra); + + for (auto& succ_node_cost : succ_node_clist) { + MS_EXCEPTION_IF_NULL(succ_node_cost); + for (auto& succ_edge_cost : succ_edge_clist) { + MS_EXCEPTION_IF_NULL(succ_edge_cost); + if ((succ_node_cost->memory_cost_ < DEVICE_MEMORY_CAPACITY) && + (succ_edge_cost->memory_cost_ < DEVICE_MEMORY_CAPACITY)) { + succ_nodes_stras[k] = succ_node_stra; + succ_edges_costs[k] = succ_edge_cost; + succ_nodes_costs[k] = succ_node_cost; + recursive(k + 1); + } + } + } + } + }; + + recursive(0); +} + +std::vector> CostGraph::EliminationStar(const OperatorInfoPtr& merged_op) { + MS_EXCEPTION_IF_NULL(merged_op); + auto succ_edges = merged_op->GetAliveSuccEdges(); + MS_LOG(INFO) << "Now eliminating star centered at: " << merged_op->name() << "."; + for (auto& succ_edge : succ_edges) { + MS_EXCEPTION_IF_NULL(succ_edge->next_operator()); + MS_LOG(INFO) << "The successive operator is: " << succ_edge->next_operator()->name() << "."; + } + + MS_EXCEPTION_IF_NULL(succ_edges[0]); + auto first_succ_node = succ_edges[0]->next_operator(); + auto first_succ_edge = succ_edges[0]; + bool valid = false; + + // 'merged_op' is merged into first_node + MS_EXCEPTION_IF_NULL(first_succ_node); + for (auto& first_succ_node_stra_cost : first_succ_node->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(first_succ_node_stra_cost); + auto first_succ_node_stra = first_succ_node_stra_cost->strategy_ptr; + auto first_succ_node_clist = first_succ_node_stra_cost->cost_list; + CostPtrList first_succ_node_clist_new; + + for (auto& merged_op_stra_cost : merged_op->GetStrategyCost()) { + MS_EXCEPTION_IF_NULL(merged_op_stra_cost); + auto merged_op_stra = merged_op_stra_cost->strategy_ptr; + auto merged_op_clist = merged_op_stra_cost->cost_list; + auto first_succ_edge_clist = first_succ_edge->GetCostList(merged_op_stra, first_succ_node_stra); + + CreateStarEliminationCostList(succ_edges, first_succ_node_stra, first_succ_node_clist, first_succ_edge_clist, + merged_op_stra, merged_op_clist, &first_succ_node_clist_new); + } + SimplifyForDreasingCommunicationWithPartialPara(&first_succ_node_clist_new); + // Set the new costlist w.r.t the strategy + first_succ_node_stra_cost->cost_list = first_succ_node_clist_new; + if ((!valid) && (!first_succ_node_clist_new.empty())) { + valid = true; + } + } + + if (!valid) { + MS_LOG(EXCEPTION) << "Eliminating star centered at: " << merged_op->name() << " failed."; + } + + merged_op->SetNotAlive(); + MS_LOG(INFO) << "Eliminating star centered at: " << merged_op->name() << " succeeded."; + return succ_edges; +} + +Status CostGraph::InitSelectedStrategy() { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + auto result = op->InitSelectedStrategy(op->selected_strategy()); + if (result != SUCCESS) { + return result; + } + } + return SUCCESS; +} + +Status CostGraph::CorrectOpsStrategyCostForMultiOutputUse() { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + if (op->GetAliveSuccEdges().size() > 1) { + // Filter out the case of a output being used by multiple operators + std::map output_count; + for (size_t i = 0; i < op->GetAliveSuccEdges().size(); ++i) { + auto output_index = op->GetAliveSuccEdges()[i]->prev_op_output_index(); + output_count[output_index]++; + } + for (size_t i = 0; i < op->GetAliveSuccEdges().size(); ++i) { + auto output_index = op->GetAliveSuccEdges()[i]->prev_op_output_index(); + if (output_count[output_index] <= 1) { + continue; + } + auto next_op = op->GetAliveSuccEdges()[i]->next_operator(); + MS_EXCEPTION_IF_NULL(next_op); + auto input_index = op->GetAliveSuccEdges()[i]->next_op_input_index(); + if (next_op->CorrectStrategyCostForMultiOutputUse(input_index) != SUCCESS) { + MS_LOG(ERROR) << "The operator name: " << op->name() << ", the next operator name: " << next_op->name() + << ", the output_index: " << output_index << ", the input_index: " << input_index << "."; + return FAILED; + } + output_count[output_index]--; + } + } + } + return SUCCESS; +} + +Status CostGraph::ComputeOpsAndEdgesParameterInvolved() { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + const auto& output_parameter = op->ComputeOpAndPrevEdgeParameterInvolved(); + if ((output_parameter != 0) && (output_parameter != 1)) { + MS_LOG(ERROR) << "Computing parameter_involved for " << op->name() << " failed."; + return FAILED; + } + } + return SUCCESS; +} + +Status CostGraph::CorrectOpsStrategyCostForMemoryReuse() { + for (auto& op : ops_) { + MS_EXCEPTION_IF_NULL(op); + if (op->CorrectStrategyCostForMemoryReuse() != SUCCESS) { + MS_LOG(ERROR) << "Correcting Operator: " << op->name() << " cost for memory reuse failed."; + return FAILED; + } + } + return SUCCESS; +} + +Status CostGraph::CorrectEdgesStrategyCostForMemoryReuse() { + for (auto& edge_pair : edges_) { + const auto& edges = edge_pair.second; + for (auto& one_edge : edges) { + if (one_edge->CorrectStrategyCostForMemoryReuse() != SUCCESS) { + MS_LOG(ERROR) << "Correcting Edge: " << one_edge->edge_name() << " cost for memory reuse failed."; + return FAILED; + } + } + } + return SUCCESS; +} + +OperatorInfoPtr CostGraph::FindTmpIdentityByParameterName(std::string& p_name) const { + for (auto one_op : ops_) { + if (one_op->name().find(IDENTITY_INFO) != std::string::npos) { + if (one_op->refkey_parameter_name() == p_name) { + return one_op; + } + } + } + return nullptr; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.h b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.h new file mode 100644 index 00000000000..4efb77055f4 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.h @@ -0,0 +1,226 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_GRAPH_COSTMODEL_H_ +#define MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_GRAPH_COSTMODEL_H_ + +#include +#include +#include +#include +#include +#include "../../common.h" +#include "common/utils.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/edge_costmodel.h" +#include "parallel/ops_info/tmp_identity_info.h" +#include "parallel/costmodel_context.h" + +namespace mindspore { +namespace parallel { +#define OPERATOR_TO_OPERATOR_CONNECTOR "-" +#define DEFAULT_DEVICE_MEMORY_CAPACITY (1024.0 * 1024.0 * 1024.0 * 16.0) +#define DEFAULT_COST_MODEL_ALPHA 1.0 +#define DEFAULT_COST_MODEL_BETA 65.0 +#define DEFAULT_COST_MODEL_GAMMA 0.02 +#define DEFAULT_COST_MODEL_SIMPLIFY_CALCULATION true +#define DEFAULT_COST_MODEL_COMMUNI_THRESHOLD 2048.0 +#define DEFAULT_COST_MODEL_COMMUNI_CONST 3072.0 +#define DEFAULT_COST_MODEL_COMMUNI_BIAS 1024.0 +#define DEFAULT_TENSOR_SLICE_ALIGNMENT_ENABLE false +#define DEFAULT_TENSOR_SLICE_ALIGNMENT_SIZE 16 +#define DEFAULT_NOT_FULLY_USE_DEVICES false +#define DEFAULT_ELEMENTWISE_OP_STRA_FOLLOW false + +class CostGraph; +using CostGraphPtr = std::shared_ptr; +extern CostGraphPtr entire_costgraph; +extern size_t TOTAL_OPS; +extern double COST_MODEL_GAMMA; +extern bool COST_MODEL_SIMPLIFY_CALCULATION; +extern double DEVICE_MEMORY_CAPACITY; +extern double COST_MODEL_COMMUNI_THRESHOLD; +extern double COST_MODEL_COMMUNI_CONST; +extern double COST_MODEL_COMMUNI_BIAS; +extern bool TENSOR_SLICE_ALIGNMENT_ENABLE; +extern size_t TENSOR_SLICE_ALIGNMENT_SIZE; +extern bool NOT_FULLY_USE_DEVICES; +extern bool ELEMENTWISE_OP_STRA_FOLLOW; + +class CostGraph { + // 'CostGraph' consists of Operators and edges between them. An edge is created between two Operators if they have + // output-input dependency relationship. + public: + CostGraph() { + dev_memory_ = DEFAULT_DEVICE_MEMORY_CAPACITY; + costmodel_alpha_ = DEFAULT_COST_MODEL_ALPHA; + costmodel_beta_ = DEFAULT_COST_MODEL_BETA; + } + ~CostGraph() = default; + void AddOperator(const OperatorInfoPtr& op) { ops_.push_back(op); } + OperatorInfoPtr FindOperatorByIndex(size_t index) { + if (index >= ops_.size()) { + MS_LOG(ERROR) << "The index: " << index << " is out of the range of ops_: " << ops_.size() << "."; + return nullptr; + } + return ops_[index]; + } + void RemoveOperator(const OperatorInfoPtr& op); + bool IsOperatorInCostGraph(const OperatorInfoPtr& op); + // the edge is in the form: u --> v + void AddEdge(OperatorInfoPtr u_node, OperatorInfoPtr v_node, const EdgePtr& edge) { + std::vector curr_edges(edges_[{u_node, v_node}]); + curr_edges.push_back(edge); + edges_[{u_node, v_node}] = curr_edges; + } + // An edge is uniquely identified by its name, and its output index and input index. + bool IsEdgeInCostGraph(const std::string&, size_t, size_t); + + void SetDeviceMemoryAndCostParameter(); + + std::vector> ConstructConnectedComponents(std::vector); + void DFS(const OperatorInfoPtr& current_op, std::map* visited, + const std::shared_ptr& component); + + CostPtrList CreateFinalCostList(const OperatorInfoPtr& u, const EdgePtr& e, const OperatorInfoPtr& v); + CostPtrList CreateFinalSingleCostList(const OperatorInfoPtr& u); + CostPtr SelectCostWithMemoryConstraint(const CostPtrList& cost_list, double memory); + CostPtr SelectCostWithMinTrainingTime(const CostPtrList& cost_list, double memory); + CostPtrList SelectCostListWithMinTrainingTimeMultiple(const std::vector& all_costlist, double memory); + Status SearchStrategyForMultiNodeFinalGraph(const std::vector&); + std::vector> GetOriginalEdgeBetweenOperators(OperatorInfoPtr u_node, OperatorInfoPtr v_node) { + return edges_[{u_node, v_node}]; + } + double GetDeviceMemory() const { return dev_memory_; } + + // Search the cost_list in the final graph, and determine the optimal one + Status SearchStrategy(); + + // Given a graph which contains the following subgraph: u --> v --> w, the node v can be eliminated + OperatorInfoPtr CheckOpElimination() const; + // Given a graph which contains the following subgraph where there are multiple edges between u and v, these edges + // can be eliminated into one + std::vector CheckEdgeElimination() const; + // Given a graph which contains the following subgraph: + // u + // | + // w --- v --- x + // where u has 0 incoming edge, u has 1 outgoing edge, and v has > 1 incoming edges, u can be merged into v. + // u is returned. + OperatorInfoPtr CheckMergeElimination() const; + // Given a graph which contains the following subgraph: + // u + // | + // v --- x + // where v has 2 outgoing edges, and u has 1 incoming edges and no outgoing edges. In this case, u can be contracted + // into v. u is returned. + OperatorInfoPtr CheckContractElimination() const; + /* Given a graph which contains the following subgraph: + * u + * / \ + * / \ + * v --- w + * where u has 2 outgoing edges, v has 1 outgoing edge, and w has 2 incoming edges, u can be eliminated into v. + * The returned value includes u and the edge >. + */ + std::pair CheckTriangleElimination() const; + /* Given a graph which contains the following subgraph: + * v <--- u ---> w + * where u has 0 incoming edges, and multiple outgoing edges. In addition, v and w have other complicated connections, + * resulting in v and w can not be performed ContractElimination. u is returned. + * NOTE: this elimination MUST be performed only when the above 5 operation cannot be applied. + */ + OperatorInfoPtr CheckStarElimination() const; + // Applying Operator Elimination in DP algorithm + EdgePtr EliminationOp(const OperatorInfoPtr& op); + // Applying Edge Elimination in DP algorithm + EdgePtr EliminationEdges(const std::vector& edges); + // Applying Merge Elimination in DP algorithm + OperatorInfoPtr EliminationMerge(const OperatorInfoPtr& op); + void CreateMergeEliminationSubCostList(StrategyPtr op_strategy, const CostPtrList& op_cost_list, + const CostPtrList& edge_cost_list, StrategyPtr tar_op_strategy, + const CostPtrList& tar_cost_list, CostPtrList* tar_cost_list_new); + // Applying Contract Elimination in DP algorithm + OperatorInfoPtr EliminationContract(const OperatorInfoPtr& op); + void CreateContractEliminationSubCostList(StrategyPtr, const CostPtrList&, const CostPtrList&, StrategyPtr, + const CostPtrList&, CostPtrList*); + + // Applying Triangle Elimination in DP algorithm. return the left_node + OperatorInfoPtr EliminationTriangle(const OperatorInfoPtr& elimi_op, const EdgePtr& edge_left_right); + void CreateTriangleEliminationCostList(const OperatorInfoPtr&, const CostPtrList&, const CostPtrList&, + const StrategyPtr&, const StrategyPtr&, const StrategyPtr&, const CostPtrList&, + const CostPtrList&, const CostPtrList&, CostPtrList*); + // Given the relevant costlist, create the TriangleElimination cost for eliminating TmpIdentityInfo + void CreateTriangleEliminationSubCostListForIdentity(StrategyPtr, StrategyPtr, StrategyPtr, const CostPtr&, + const CostPtrList&, const CostPtrList&, const CostPtr&, + const CostPtrList&, CostPtrList*); + // Given the relevant costlist, create the TriangleElimination cost for eliminating other operators + void CreateTriangleEliminationSubCostListForOthers(StrategyPtr, StrategyPtr, StrategyPtr, const CostPtr&, + const CostPtrList&, const CostPtrList&, const CostPtr&, + const CostPtrList&, CostPtrList*); + + // Applying the Star Elimination in DP algorithm. Return the successive edges of this merged_op + // NOTE: this elimination MUST be performed only when the above 5 operation cannot be applied. + std::vector EliminationStar(const OperatorInfoPtr& op); + void CreateStarEliminationCostList(std::vector&, const StrategyPtr&, const CostPtrList&, const CostPtrList&, + const StrategyPtr&, const CostPtrList&, CostPtrList*); + void CreateStarEliminationSubCostList(const StrategyPtr&, const CostPtrList&, const CostPtrList&, const StrategyPtr&, + const CostPtrList&, std::vector, CostPtrList&, CostPtrList&, + CostPtrList*); + + // When a output of a operator is being used by multiple operators, the memory cost of this part should be calculated + // only once. This method is for correcting the 'strategy_cost_' for operators + Status CorrectOpsStrategyCostForMultiOutputUse(); + // When the input of a operator is neither a WEIGHT, nor a output of a subsequent operator involving WEIGHT, then + // the memory cost can be resused. + Status CorrectOpsStrategyCostForMemoryReuse(); + // When the input of the edge is neither a WEIGHT, nor a output of a subsequent operator involving WEIGHT, then + // the memory cost can be resused. + Status CorrectEdgesStrategyCostForMemoryReuse(); + Status ComputeOpsAndEdgesParameterInvolved(); + + std::vector GetOperators() const { return ops_; } + size_t GetNumPairs() const { return edges_.size(); } + Status InitSelectedStrategy(); + OperatorInfoPtr FindTmpIdentityByParameterName(std::string&) const; + // Needed by rec_parser + void add_inputs_tensor_name(const std::vector& inputs_tensor_name) { + inputs_tensor_name_list_.push_back(inputs_tensor_name); + } + const std::vector> get_inputs_tensor_name_list() const { return inputs_tensor_name_list_; } + void add_tuple_getitem(const std::pair& tuple_getitem) { + auto ret = tuple_getitem_list_.insert(tuple_getitem); + if (ret.second == false) { + MS_LOG(EXCEPTION) << "The insert item is already exist."; + } + } + const std::map get_tuple_getitem_list() const { return tuple_getitem_list_; } + + private: + // Needed by rec_parser + std::vector> inputs_tensor_name_list_; + std::map tuple_getitem_list_; + double dev_memory_; + double costmodel_alpha_; + double costmodel_beta_; + std::vector ops_; + std::map, std::vector> edges_; + std::vector> connected_compoents_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_AUTO_PARALLEL_GRAPH_COSTMODEL_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc new file mode 100644 index 00000000000..6958932fd6d --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc @@ -0,0 +1,665 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/operator_costmodel.h" + +#include +#include "parallel/device_matrix.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +void OperatorCost::set_is_parameter(const std::vector& is_parameter) { is_parameter_ = is_parameter; } + +void OperatorCost::SetInputAndOutputTypeLength(const std::vector& input_lengths, + const std::vector& output_lengths) { + inputs_type_lengths_ = input_lengths; + outputs_type_lengths_ = output_lengths; +} + +// return the per device communication cost in the forward phase. +double MatMulCost::GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t&) const { + TensorInfo input0 = inputs[0]; + TensorInfo output0 = outputs[0]; + Shape input0_shape = input0.shape(); + Shape input0_slice_shape = input0.slice_shape(); + if (input0_shape[input0_shape.size() - 1] == input0_slice_shape[input0_slice_shape.size() - 1]) { + // If the reduced dimension has not been partitioned, then there is no communication cost. + return 0.0; + } else { + // Else, the communication cost is the size (number of bytes) of a slice of output tensor. + return ListProduct(output0.slice_shape()) * static_cast(outputs_type_lengths_[0]); + } +} + +// return the per device communication cost in the forward phase. +double MatMulCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + // In backward phase, the communication cost is incurred only when tensor B is a Parameter and tensor B does not + // fully utilize all devices + double result = 0.0; + if (is_parameter_[1]) { + TensorInfo input1 = inputs[1]; // tensor B + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape input1_shape = input1.shape(); + Shape input1_slice_shape = input1.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input1_shape.size(); ++i) { + used_device_num *= input1_shape[i] / input1_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + + return result; +} + +// Return the per device memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double MatMulCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t&) const { + // In forward phase, the memory cost = slice(A) + slice(B) + (0 or 1) allreduce(slice(C)) + double result = 0.0; + TensorInfo output0 = outputs[0]; + Shape input0_slice_shape = inputs[0].slice_shape(); + Shape input1_slice_shape = inputs[1].slice_shape(); + Shape input0_shape = inputs[0].shape(); + if (input0_shape[input0_shape.size() - 1] != input0_slice_shape[input0_slice_shape.size() - 1]) { + // If the reduced dimension has been partitioned, then there is no communication cost. + result += ListProduct(output0.slice_shape()) * static_cast(outputs_type_lengths_[0]); + } + result += ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]) + + ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + return result; +} + +// Return the per device memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double MatMulCost::GetBackwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + // In backward phase, the memory cost = (0 or 1) allreduce(slice(B)) + double result = 0.0; + if (is_parameter_[1]) { + TensorInfo input1 = inputs[1]; // tensor B + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape input1_shape = input1.shape(); + Shape input1_slice_shape = input1.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input1_shape.size(); ++i) { + used_device_num *= input1_shape[i] / input1_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + + return result; +} + +// Return the per device communication cost in the forward phase. +double ActivationCost::GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const { + // ReLU is the element-wise operator, thus it does not need communication in the forward phase + return 0.0; +} + +// Return the per device communication cost in the backward phase. +double ActivationCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + if (is_parameter_[0]) { + TensorInfo input1 = inputs[0]; + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + Shape input1_shape = input1.shape(); + Shape input1_slice_shape = input1.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input1_shape.size(); ++i) { + used_device_num *= input1_shape[i] / input1_slice_shape[i]; + } + if (total_device_num != IntToSize(used_device_num)) { + result = ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + } + return result; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double ActivationCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t&) const { + TensorInfo input0_info = inputs[0]; + Shape input0_slice_shape = input0_info.slice_shape(); + return ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double ActivationCost::GetBackwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const { + return 0.0; +} + +// Return the per device communication cost in the forward phase. +double SoftmaxCost::GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const { + // In the forward phase, the communication cost = 0 + return 0.0; +} + +// Return the per device communication cost in the backward phase. +double SoftmaxCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + if (is_parameter_[0]) { + TensorInfo input1 = inputs[0]; + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + Shape input1_shape = input1.shape(); + Shape input1_slice_shape = input1.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input1_shape.size(); ++i) { + used_device_num *= input1_shape[i] / input1_slice_shape[i]; + } + if (total_device_num != IntToSize(used_device_num)) { + result = ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + } + return result; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double SoftmaxCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t&) const { + // In the forward phase, the memory cost = slice(A) + TensorInfo input0 = inputs[0]; + Shape input0_slice_shape = input0.slice_shape(); + return ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double SoftmaxCost::GetBackwardMemoryCost(const std::vector&, + const std::vector&, const int32_t&) const { + return 0.0; +} + +// return the per device communication cost in the forward phase. +double TmpIdentityCost::GetForwardCommCost(const std::vector&, + const std::vector&, const int32_t&) const { + // Identity is the element-wise operator, thus it does not need communication in the forward phase + return 0.0; +} + +// return the per device communication cost in the backward phase. +double TmpIdentityCost::GetBackwardCommCost(const std::vector&, + const std::vector&, const int32_t&) const { + // Identity is the element-wise operator, thus it does not need communication in the backward phase + return 0.0; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double TmpIdentityCost::GetForwardMemoryCost(const std::vector& inputs, + const std::vector&, + const int32_t&) const { + TensorInfo input0_info = inputs[0]; + Shape input0_slice_shape = input0_info.slice_shape(); + return ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); +} + +// Return the per memory cost in the backward phase. The cost is calculated according to the bytes +// this operator uses +double TmpIdentityCost::GetBackwardMemoryCost(const std::vector&, + const std::vector&, + const int32_t&) const { + return 0.0; +} + +double BatchParallelCost::GetForwardMemoryCost(const std::vector& inputs, + const std::vector&, + const int32_t&) const { + double cost = 0.0; + for (size_t i = 0; i < inputs.size(); ++i) { + cost += ListProduct(inputs[i].slice_shape()) * static_cast(inputs_type_lengths_[i]); + } + return cost; +} + +double BatchParallelCost::GetBackwardMemoryCost(const std::vector&, + const std::vector&, + const int32_t&) const { + return 0.0; +} + +// return the per device communication cost in the forward phase. +double PReLUCost::GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const { + // prelu does not need communication in the forward phase + return 0.0; +} + +// return the per device communication cost in the backward phase. +double PReLUCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + if (is_parameter_[1]) { + TensorInfo input1 = inputs[1]; + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + Shape input1_shape = input1.shape(); + Shape input1_slice_shape = input1.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input1_shape.size(); ++i) { + used_device_num *= input1_shape[i] / input1_slice_shape[i]; + } + if (total_device_num != IntToSize(used_device_num)) { + result = ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + } + return result; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double PReLUCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t&) const { + // In forward phase, the memory cost = slice(A) + slice(B) + Shape input0_slice_shape = inputs[0].slice_shape(); + Shape input1_slice_shape = inputs[1].slice_shape(); + double result = ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]) + + ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + return result; +} + +// Return the per memory cost in the backward phase. The cost is calculated according to the bytes +// this operator uses +double PReLUCost::GetBackwardMemoryCost(const std::vector& inputs, + const std::vector&, + const int32_t& stage_id) const { + // In backward phase, the memory cost = (0 or 1) allreduce(slice(B)) + double result = 0.0; + if (is_parameter_[1]) { + TensorInfo input1 = inputs[1]; // tensor B + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape input1_shape = input1.shape(); + Shape input1_slice_shape = input1.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input1_shape.size(); ++i) { + used_device_num *= input1_shape[i] / input1_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) { + result += ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + } + return result; +} + +// return the per device communication cost in the forward phase. +double OneHotCost::GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const { + // onehot does not need communication in the forward phase + return 0.0; +} + +// return the per device communication cost in the backward phase. +double OneHotCost::GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const { + // onehot does not need communication in the backward phase + return 0.0; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double OneHotCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t&) const { + // In onehot's forward phase, the memory cost = slice(A) + Shape input0_slice_shape = inputs[0].slice_shape(); + return ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); +} + +// Return the per memory cost in the backward phase. The cost is calculated according to the bytes +// this operator uses +double OneHotCost::GetBackwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const { + return 0.0; +} + +// return the per device communication cost in the forward phase. +double SoftmaxCrossEntropyWithLogitsCost::GetForwardCommCost(const std::vector&, + const std::vector&, const int32_t&) const { + // SoftmaxCrossEntropyWithLogitsCost does not need communication in the forward phase + return 0.0; +} + +// return the per device communication cost in the backward phase. +double SoftmaxCrossEntropyWithLogitsCost::GetBackwardCommCost(const std::vector&, + const std::vector&, const int32_t&) const { + // SoftmaxCrossEntropyWithLogitsCost does not need communication in the backward phase + return 0.0; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double SoftmaxCrossEntropyWithLogitsCost::GetForwardMemoryCost(const std::vector& inputs, + const std::vector&, const int32_t&) const { + // In forward phase, the memory cost = slice(A) + slice(B) + Shape input0_slice_shape = inputs[0].slice_shape(); + Shape input1_slice_shape = inputs[1].slice_shape(); + double result = ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]) + + ListProduct(input1_slice_shape) * static_cast(inputs_type_lengths_[1]); + return result; +} + +// Return the per memory cost in the backward phase. The cost is calculated according to the bytes +// this operator uses +double SoftmaxCrossEntropyWithLogitsCost::GetBackwardMemoryCost(const std::vector&, + const std::vector&, const int32_t&) const { + return 0.0; +} + +// return the per device communication cost in the forward phase. +double ReshapeCost::GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const { + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + RankList dev_list = g_device_manager->GetDeviceListByStageId(stage_id); + TensorRedistribution tensor_redistribution; + if (tensor_redistribution.Init(inputs[0].tensor_layout(), outputs[0].tensor_layout(), dev_list) == FAILED) { + MS_LOG(EXCEPTION) << "Failure: tensor_redistribution init failed."; + } + if (tensor_redistribution.ComputeCost() == FAILED) { + MS_LOG(EXCEPTION) << "Failure: tensor_redistribution ComputeCost failed."; + } + return (inputs_type_lengths_[0] * tensor_redistribution.comm_cost()); +} + +// return the per device communication cost in the backward phase. +double ReshapeCost::GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const { + return 0.0; +} + +// Return the per memory cost in the forward phase. The cost is calculated according to the bytes +// this operator uses +double ReshapeCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const { + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + RankList dev_list = g_device_manager->GetDeviceListByStageId(stage_id); + TensorRedistribution tensor_redistribution; + if (tensor_redistribution.Init(inputs[0].tensor_layout(), outputs[0].tensor_layout(), dev_list) == FAILED) { + MS_LOG(EXCEPTION) << "Failure: tensor_redistribution init failed."; + } + if (tensor_redistribution.ComputeCost() == FAILED) { + MS_LOG(EXCEPTION) << "Failure: tensor_redistribution ComputeCost failed."; + } + return (inputs_type_lengths_[0] * tensor_redistribution.mem_cost()); +} + +// Return the per memory cost in the backward phase. The cost is calculated according to the bytes +// this operator uses +double ReshapeCost::GetBackwardMemoryCost(const std::vector&, + const std::vector&, const int32_t&) const { + return 0.0; +} + +double ArithmeticCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t&) const { + double result; + result = ListProduct(inputs[0].slice_shape()) * static_cast(inputs_type_lengths_[0]) + + ListProduct(inputs[1].slice_shape()) * static_cast(inputs_type_lengths_[1]); + return result; +} + +double ArithmeticCost::GetBackwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + if (is_parameter_[0]) { + TensorInfo input_a_tensor_info = inputs[0]; + Shape input_a_shape = input_a_tensor_info.shape(); + Shape input_a_slice_shape = input_a_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_a_shape.size(); ++i) { + used_device_num *= input_a_shape[i] / input_a_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_a_slice_shape) * static_cast(inputs_type_lengths_[0]); + } + + if (is_parameter_[1]) { + TensorInfo input_b_tensor_info = inputs[1]; + Shape input_b_shape = input_b_tensor_info.shape(); + Shape input_b_slice_shape = input_b_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_b_shape.size(); ++i) { + used_device_num *= input_b_shape[i] / input_b_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_b_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + return result; +} + +double ArithmeticCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + if (is_parameter_[0]) { + TensorInfo input_a_tensor_info = inputs[0]; + Shape input_a_shape = input_a_tensor_info.shape(); + Shape input_a_slice_shape = input_a_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_a_shape.size(); ++i) { + used_device_num *= input_a_shape[i] / input_a_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_a_slice_shape) * static_cast(inputs_type_lengths_[0]); + } + + if (is_parameter_[1]) { + TensorInfo input_b_tensor_info = inputs[1]; + Shape input_b_shape = input_b_tensor_info.shape(); + Shape input_b_slice_shape = input_b_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_b_shape.size(); ++i) { + used_device_num *= input_b_shape[i] / input_b_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_b_slice_shape) * static_cast(inputs_type_lengths_[1]); + } + + return result; +} + +double L2NormalizeCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + if (is_parameter_[0]) { + TensorInfo input_tensor_info = inputs[0]; + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape input_shape = input_tensor_info.shape(); + Shape input_slice_shape = input_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_shape.size(); ++i) { + used_device_num *= input_shape[i] / input_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_slice_shape) * static_cast(inputs_type_lengths_[0]); + } + + return result; +} + +double L2NormalizeCost::GetForwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t&) const { + TensorInfo input0_info = inputs[0]; + Shape input0_slice_shape = input0_info.slice_shape(); + return ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); +} + +double L2NormalizeCost::GetBackwardMemoryCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + + if (is_parameter_[0]) { + TensorInfo input_tensor_info = inputs[0]; + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape input_shape = input_tensor_info.shape(); + Shape input_slice_shape = input_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_shape.size(); ++i) { + used_device_num *= input_shape[i] / input_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_slice_shape) * static_cast(inputs_type_lengths_[0]); + } + + return result; +} + +bool IsDataParallel(const Shape& shape, const Shape& slice_shape, const int32_t& stage_id) { + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + auto strategy0 = shape[0] / slice_shape[0]; + + return (total_device_num == IntToSize(strategy0)); +} + +double ReduceMethodCost::GetForwardCommCost(const std::vector& inputs, + const std::vector& outputs, const int32_t& stage_id) const { + double result = 0.0; + TensorInfo input0 = inputs[0]; + TensorInfo output0 = outputs[0]; + Shape input0_shape = input0.shape(); + Shape input0_slice_shape = input0.slice_shape(); + if (cross_batch_ && IsDataParallel(input0_shape, input0_slice_shape, stage_id)) { + return result; + } + std::vector dim_list = input0.reduce_dim(); + std::vector::iterator pos; + pos = std::find_if(dim_list.begin(), dim_list.end(), [input0_shape, input0_slice_shape](const int32_t& index) { + return input0_shape[IntToSize(index)] != input0_slice_shape[IntToSize(index)]; + }); + if (pos != dim_list.end()) { + result += ListProduct(output0.slice_shape()) * static_cast(outputs_type_lengths_[0]); + } + + return result; +} + +double ReduceMethodCost::GetBackwardCommCost(const std::vector& inputs, const std::vector&, + const int32_t& stage_id) const { + double result = 0.0; + if (is_parameter_[0]) { + TensorInfo input_tensor_info = inputs[0]; + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape input_shape = input_tensor_info.shape(); + Shape input_slice_shape = input_tensor_info.slice_shape(); + int32_t used_device_num = 1; + for (size_t i = 0; i < input_shape.size(); ++i) { + used_device_num *= input_shape[i] / input_slice_shape[i]; + } + + if (total_device_num != IntToSize(used_device_num)) + result += ListProduct(input_slice_shape) * static_cast(inputs_type_lengths_[0]); + } + + return result; +} + +double ReduceMethodCost::GetForwardMemoryCost(const std::vector& inputs, + const std::vector& outputs, const int32_t& stage_id) const { + double result = 0.0; + TensorInfo input0 = inputs[0]; + TensorInfo output0 = outputs[0]; + std::vector dim_list = input0.reduce_dim(); + Shape input0_slice_shape = input0.slice_shape(); + Shape input0_shape = input0.shape(); + if (!cross_batch_ || !IsDataParallel(input0_shape, input0_slice_shape, stage_id)) { + std::vector::iterator pos; + pos = std::find_if(dim_list.begin(), dim_list.end(), [input0_shape, input0_slice_shape](const int32_t& index) { + return input0_shape[IntToSize(index)] != input0_slice_shape[IntToSize(index)]; + }); + if (pos != dim_list.end()) { + result += ListProduct(output0.slice_shape()) * static_cast(outputs_type_lengths_[0]); + } + } + result += ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); + + return result; +} + +double ReduceMeanCost::GetForwardMemoryCost(const std::vector& inputs, + const std::vector& outputs, const int32_t& stage_id) const { + double result = 0.0; + TensorInfo input0 = inputs[0]; + TensorInfo output0 = outputs[0]; + std::vector dim_list = input0.reduce_dim(); + Shape input0_slice_shape = input0.slice_shape(); + Shape input0_shape = input0.shape(); + if (!cross_batch_ || !IsDataParallel(input0_shape, input0_slice_shape, stage_id)) { + std::vector::iterator pos; + pos = std::find_if(dim_list.begin(), dim_list.end(), [input0_shape, input0_slice_shape](const int32_t& index) { + return input0_shape[IntToSize(index)] != input0_slice_shape[IntToSize(index)]; + }); + if (pos != dim_list.end()) { + result += ListProduct(output0.slice_shape()) * static_cast(outputs_type_lengths_[0]) * 2.0; + } + } + result += ListProduct(input0_slice_shape) * static_cast(inputs_type_lengths_[0]); + + return result; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.h b/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.h new file mode 100644 index 00000000000..ad856fc9109 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.h @@ -0,0 +1,520 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_OPERATOR_COSTMODEL_H_ +#define PARALLEL_AUTO_PARALLEL_OPERATOR_COSTMODEL_H_ + +#include +#include +#include "parallel/tensor_layout/tensor_info.h" +#include "parallel/device_manager.h" + +namespace mindspore { +namespace parallel { +#define MAXIMUM_INPUT_NUMBER 100 +#define DEFAULT_DATA_TYPE_LENGTH 4 + +class OperatorCost; +using OperatorCostPtr = std::shared_ptr; + +template +double ListProduct(std::vector vec) { + double result = 1; + for (size_t i = 0; i < vec.size(); ++i) { + result *= vec[i]; + } + return result; +} +// NOTE: Currently, the returned value in each method is bytes of memory size, which is calculated by the number of +// entries timing the length of each entry's data type +class OperatorCost { + public: + OperatorCost() { + // this is only for the case when set_is_parameter() and SetInputAndOutputTypeLength() are not invoked + for (size_t i = 0; i < MAXIMUM_INPUT_NUMBER; ++i) { + is_parameter_.push_back(false); + inputs_type_lengths_.push_back(DEFAULT_DATA_TYPE_LENGTH); + outputs_type_lengths_.push_back(DEFAULT_DATA_TYPE_LENGTH); + } + } + virtual ~OperatorCost() = default; + + void set_is_parameter(const std::vector& is_parameter); + void SetInputAndOutputTypeLength(const std::vector& input_lengths, const std::vector& output_lengths); + std::vector inputs_type_lengths() const { return inputs_type_lengths_; } + std::vector outputs_type_lengths() const { return outputs_type_lengths_; } + + // per device communication cost + virtual double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const = 0; + virtual double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const = 0; + virtual double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const = 0; + // per device computation cost + virtual double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const = 0; + virtual double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const = 0; + virtual double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const = 0; + + protected: + // for each input in 'inputs_', there is a bool variable indicating whether that the corresponding input is parameter + std::vector is_parameter_; + // for each input and output, the followings record the number of bytes of each element + std::vector inputs_type_lengths_; + std::vector outputs_type_lengths_; +}; + +class MatMulCost : public OperatorCost { + public: + MatMulCost() = default; + ~MatMulCost() override = default; + + // per device communication cost + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + // per device computation cost + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; + +using MatMulCostPtr = std::shared_ptr; + +class ActivationCost : public OperatorCost { + public: + ActivationCost() = default; + ~ActivationCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; + +using ActivationCostPtr = std::shared_ptr; + +class SoftmaxCost : public OperatorCost { + public: + SoftmaxCost() = default; + ~SoftmaxCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t&) const override; +}; + +using SoftmaxCostPtr = std::shared_ptr; + +class TmpIdentityCost : public OperatorCost { + public: + TmpIdentityCost() = default; + ~TmpIdentityCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using TmpIdentityCostPtr = std::shared_ptr; + +class BatchParallelCost : public OperatorCost { + public: + BatchParallelCost() = default; + ~BatchParallelCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using BatchParallelCostPtr = std::shared_ptr; + +class VirtualDatasetCost : public OperatorCost { + public: + VirtualDatasetCost() = default; + ~VirtualDatasetCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } +}; +using VirtualDatasetCostPtr = std::shared_ptr; + +class GeneratorBaseCost : public OperatorCost { + public: + GeneratorBaseCost() = default; + ~GeneratorBaseCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + // Inputs vector is empty for generator ops. + double GetForwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + // Generator ops don't have backward steps. + double GetBackwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } +}; +using GeneratorBaseCostPtr = std::shared_ptr; + +class PReLUCost : public OperatorCost { + public: + PReLUCost() = default; + ~PReLUCost() override = default; + + // per device communication cost + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + // per device computation cost + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using PReLUCostPtr = std::shared_ptr; + +class OneHotCost : public OperatorCost { + public: + OneHotCost() = default; + ~OneHotCost() override = default; + + // per device communication cost + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + // per device computation cost + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using OneHotCostPtr = std::shared_ptr; + +class SoftmaxCrossEntropyWithLogitsCost : public OperatorCost { + public: + SoftmaxCrossEntropyWithLogitsCost() = default; + ~SoftmaxCrossEntropyWithLogitsCost() override = default; + + // per device communication cost + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + // per device computation cost + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using SoftmaxCrossEntropyWithLogitsCostPtr = std::shared_ptr; + +class ReshapeCost : public OperatorCost { + public: + ReshapeCost() = default; + + ~ReshapeCost() override = default; + + // per device communication cost + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + + double GetForwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + // per device computation cost + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using ReshapeCostPtr = std::shared_ptr; + +class ArithmeticCost : public OperatorCost { + public: + ArithmeticCost() = default; + ~ArithmeticCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override; + + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using ArithmeticCostPtr = std::shared_ptr; + +class L2NormalizeCost : public OperatorCost { + public: + L2NormalizeCost() = default; + ~L2NormalizeCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using L2NormalizeCostPtr = std::shared_ptr; + +class ReduceMethodCost : public OperatorCost { + public: + ReduceMethodCost() = default; + ~ReduceMethodCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t& stage_id) const override; + double GetBackwardCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; + double GetBackwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + void set_cross_batch(bool cb) { cross_batch_ = cb; } + + protected: + bool cross_batch_ = false; +}; +using ReduceMethodCostPtr = std::shared_ptr; + +class ReduceMeanCost : public ReduceMethodCost { + public: + ReduceMeanCost() = default; + ~ReduceMeanCost() override = default; + + double GetForwardMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override; +}; +using ReduceMeanCostPtr = std::shared_ptr; + +class GetNextCost : public OperatorCost { + public: + GetNextCost() = default; + ~GetNextCost() override = default; + + double GetCommCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardCommCost(inputs, outputs, stage_id) + GetBackwardCommCost(inputs, outputs, stage_id); + } + double GetForwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetBackwardCommCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + double GetMemoryCost(const std::vector& inputs, const std::vector& outputs, + const int32_t& stage_id) const override { + return GetForwardMemoryCost(inputs, outputs, stage_id) + GetBackwardMemoryCost(inputs, outputs, stage_id); + } + // Inputs vector is empty for generator ops. + double GetForwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } + // Generator ops don't have backward steps. + double GetBackwardMemoryCost(const std::vector&, const std::vector&, + const int32_t&) const override { + return 0.0; + } +}; +using GetNextCostPtr = std::shared_ptr; +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_OPERATOR_COSTMODEL_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_cost.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_cost.cc new file mode 100644 index 00000000000..45407a89e6e --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_cost.cc @@ -0,0 +1,815 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/rec_core/rec_cost.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" + +namespace mindspore { +namespace parallel { +#define DOUBLE_MAX (std::numeric_limits::max)() + +// Compute redistributed cost +double CostRedis(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const std::vector> &mode, const Graph &graph) { + // Store value of cost redist + double cost_redis = 0; + + // Number of current strategies. + size_t num_strategy = node_name_to_strategy.size(); + + // Number of node-in and node-out + size_t num_node_in = node.node_in.size(); + size_t num_node_out = node.node_out.size(); + + // Set tensor edge value with original tensor shape and cutting times. + double input_tensor = node.apply.arguments[0].tensor_shape.shape_n * node.apply.arguments[0].tensor_str.str_n * + node.apply.arguments[0].tensor_shape.shape_c * node.apply.arguments[0].tensor_str.str_c * + node.apply.arguments[0].tensor_shape.shape_h * node.apply.arguments[0].tensor_str.str_h * + node.apply.arguments[0].tensor_shape.shape_w * node.apply.arguments[0].tensor_str.str_w; + + double output_tensor = node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n * + node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c * + node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_str.str_h * + node.tensor_parm.tensor_shape.shape_w * node.tensor_parm.tensor_str.str_w; + + // For each strategy candidate. + for (size_t i_strategy = 0; i_strategy < num_strategy; i_strategy++) { + // Find its forward nodes + for (size_t i_node = 0; i_node < num_node_in; i_node++) { + if (graph.nodes[node.node_in[i_node]].name == node_name_to_strategy[i_strategy].first) { + bool is_search_forward = true; + cost_redis += + CostRedisWithAdjacentNode(node_name_to_strategy, mode, i_strategy, i_node, input_tensor, is_search_forward); + } + } + + // Find its backward nodes + for (size_t i_node = 0; i_node < num_node_out; i_node++) { + if (graph.nodes[node.node_out[i_node]].name == node_name_to_strategy[i_strategy].first) { + bool is_search_forward = false; + cost_redis += + CostRedisWithAdjacentNode(node_name_to_strategy, mode, i_strategy, i_node, output_tensor, is_search_forward); + } + } + } + + return cost_redis; +} + +double CostRedisWithAdjacentNode(const std::vector> &node_name_to_strategy, + const std::vector> &mode, size_t i_strategy, size_t i_node, + double tensor_size, bool search_forward) { + double new_redis_cost = 0; + int counter = 0; + + if (search_forward) { + if (static_cast(1 / node_name_to_strategy[i_strategy].second.outputTensor.str_n) != + static_cast(1 / mode[i_node][0])) { + counter += 1; + } + if (static_cast(1 / node_name_to_strategy[i_strategy].second.outputTensor.str_c) != + static_cast(1 / mode[i_node][1])) { + counter += 1; + } + if (static_cast(1 / node_name_to_strategy[i_strategy].second.outputTensor.str_h) != + static_cast(1 / mode[i_node][2])) { + counter += 1; + } + if (static_cast(1 / node_name_to_strategy[i_strategy].second.outputTensor.str_w) != + static_cast(1 / mode[i_node][3])) { + counter += 1; + } + } else { + if (static_cast(1 / node_name_to_strategy[i_strategy].second.inputTensor[0].str_n) != + static_cast(1 / mode[2][0])) { + counter += 1; + } + if (static_cast(1 / node_name_to_strategy[i_strategy].second.inputTensor[0].str_c) != + static_cast(1 / mode[2][1])) { + counter += 1; + } + if (static_cast(1 / node_name_to_strategy[i_strategy].second.inputTensor[0].str_h) != + static_cast(1 / mode[2][2])) { + counter += 1; + } + if (static_cast(1 / node_name_to_strategy[i_strategy].second.inputTensor[0].str_w) != + static_cast(1 / mode[2][3])) { + counter += 1; + } + } + + if (counter >= 2) { + new_redis_cost = tensor_size / 4.0; + } else if (counter == 0 || counter == 1) { + new_redis_cost = 0; + } else { + MS_LOG(EXCEPTION) << "Failure: CostRedis failed."; + } + + return new_redis_cost; +} + +// Get optimal strategy for MatMul +StrategyRec CostMatMul::GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph) { + int edge_i = + static_cast(node.apply.arguments[0].tensor_shape.shape_h * node.apply.arguments[0].tensor_str.str_h); + int edge_j = + static_cast(node.apply.arguments[1].tensor_shape.shape_w * node.apply.arguments[1].tensor_str.str_w); + int edge_k = + static_cast(node.apply.arguments[0].tensor_shape.shape_w * node.apply.arguments[0].tensor_str.str_w); + + std::vector cost_op; + std::vector> mode; + + if (edge_i < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrConcatDimI(edge_j, edge_k) + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 0.5, 1}, {1, 1, 1, 1}, {1, 1, 0.5, 1}}, + graph)); + } + + if (edge_j < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrConcatDimJ(edge_i, edge_k) + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 1}, {1, 1, 1, 0.5}, {1, 1, 1, 0.5}}, + graph)); + } + + if (edge_k < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrReduceDimK(edge_i, edge_j) + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 0.5}, {1, 1, 0.5, 1}, {1, 1, 1, 1}}, + graph)); + } + + return ChoseStr(cost_op, node.apply.str); +} + +// Get weight for MatMul +double CostMatMul::GetMinCostIn(const OperatorRec &op) { + int edge_i = static_cast(op.arguments[0].tensor_shape.shape_h * op.arguments[0].tensor_str.str_h); + int edge_j = static_cast(op.arguments[1].tensor_shape.shape_w * op.arguments[1].tensor_str.str_w); + int edge_k = static_cast(op.arguments[0].tensor_shape.shape_w * op.arguments[0].tensor_str.str_w); + + std::vector cost_in; + cost_in.push_back(StrConcatDimI(edge_j, edge_k)); + cost_in.push_back(StrConcatDimJ(edge_i, edge_k)); + cost_in.push_back(StrReduceDimK(edge_i, edge_j)); + + return *min_element(cost_in.begin(), cost_in.end()); +} + +// Chose strategy for MatMul +StrategyRec CostMatMul::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_i_; + break; + + case 1: + str.inputTensor[1].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_j_; + break; + + case 2: + str.inputTensor[0].str_w /= 2.0; + str.inputTensor[1].str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_k_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure:CostMatMul failed."; + } + + return str; +} + +// Get optimal strategy for Conv +StrategyRec CostConvolution::GetOptimalStr( + const Graph::NodeType &node, const std::vector> &node_name_to_strategy, + const Graph &graph) { + const OperatorRec &op = node.apply; + + int input_tensor_h = static_cast(op.arguments[0].tensor_shape.shape_h * op.arguments[0].tensor_str.str_h); + int input_tensor_w = static_cast(op.arguments[0].tensor_shape.shape_w * op.arguments[0].tensor_str.str_w); + int input_tensor_n = static_cast(op.arguments[0].tensor_shape.shape_n * op.arguments[0].tensor_str.str_n); + int input_tensor_c = static_cast(op.arguments[0].tensor_shape.shape_c * op.arguments[0].tensor_str.str_c); + + int tensor_in = input_tensor_h * input_tensor_w * input_tensor_n * input_tensor_c; + + int tensor_filter_h = static_cast(op.arguments[1].tensor_shape.shape_h * op.arguments[1].tensor_str.str_h); + int tensor_filter_w = static_cast(op.arguments[1].tensor_shape.shape_w * op.arguments[1].tensor_str.str_w); + int tensor_filter_n = static_cast(op.arguments[1].tensor_shape.shape_n * op.arguments[1].tensor_str.str_n); + int tensor_filter_c = static_cast(op.arguments[1].tensor_shape.shape_c * op.arguments[1].tensor_str.str_c); + + int tensor_filter = tensor_filter_h * tensor_filter_w * tensor_filter_n * tensor_filter_c; + + int output_tensor_h = static_cast(node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_str.str_h); + int output_tensor_w = static_cast(node.tensor_parm.tensor_shape.shape_w * node.tensor_parm.tensor_str.str_w); + int output_tensor_n = static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n); + int output_tensor_c = static_cast(node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c); + + int tensor_out = output_tensor_h * output_tensor_w * output_tensor_n * output_tensor_c; + + std::vector cost_op; + cost_op.reserve(7); + std::vector> mode; + + if (input_tensor_n < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimB(tensor_filter) + CostRedis(node, node_name_to_strategy, + mode = {{0.5, 1, 1, 1}, {1, 1, 1, 1}, {0.5, 1, 1, 1}}, graph)); + } + + cost_op.push_back(DOUBLE_MAX); + cost_op.push_back(DOUBLE_MAX); + + if (tensor_filter < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimK(tensor_in) + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 1}, {0.5, 1, 1, 1}, {1, 0.5, 1, 1}}, graph)); + } + + cost_op.push_back(DOUBLE_MAX); + cost_op.push_back(DOUBLE_MAX); + + if (tensor_filter_c < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimQ(tensor_out) + CostRedis(node, node_name_to_strategy, + mode = {{1, 0.5, 1, 1}, {1, 0.5, 1, 1}, {1, 1, 1, 1}}, graph)); + } + + return ChoseStr(cost_op, node.apply.str); +} + +// Get weight for Conv +double CostConvolution::GetMinCostIn(const Graph::NodeType &node) { + const OperatorRec &op = node.apply; + + int tensor_in = static_cast(op.arguments[0].tensor_shape.shape_h * op.arguments[0].tensor_str.str_h) * + static_cast(op.arguments[0].tensor_shape.shape_n * op.arguments[0].tensor_str.str_n) * + static_cast(op.arguments[0].tensor_shape.shape_w * op.arguments[0].tensor_str.str_w) * + static_cast(op.arguments[0].tensor_shape.shape_c * op.arguments[0].tensor_str.str_c); + int tensor_filter = static_cast(op.arguments[1].tensor_shape.shape_h * op.arguments[1].tensor_str.str_h) * + static_cast(op.arguments[1].tensor_shape.shape_n * op.arguments[1].tensor_str.str_n) * + static_cast(op.arguments[1].tensor_shape.shape_w * op.arguments[1].tensor_str.str_w) * + static_cast(op.arguments[1].tensor_shape.shape_c * op.arguments[1].tensor_str.str_c); + int tensor_out = static_cast(node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_shape.shape_w) * + static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_shape.shape_c) * + static_cast(node.tensor_parm.tensor_str.str_h * node.tensor_parm.tensor_str.str_w) * + static_cast(node.tensor_parm.tensor_str.str_n * node.tensor_parm.tensor_str.str_c); + + std::vector cost_in; + cost_in.push_back(StrDimB(tensor_filter)); + cost_in.push_back(StrDimI(tensor_in, tensor_filter)); + cost_in.push_back(StrDimJ(tensor_in, tensor_filter)); + cost_in.push_back(StrDimK(tensor_in)); + cost_in.push_back(StrDimDI(tensor_in, tensor_out)); + cost_in.push_back(StrDimDJ(tensor_in, tensor_out)); + cost_in.push_back(StrDimQ(tensor_out)); + + return *min_element(cost_in.begin(), cost_in.end()); +} + +// Chose strategy for Conv +StrategyRec CostConvolution::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_n /= 2.0; + str.outputTensor.str_n /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_b_; + break; + + case 1: + str.inputTensor[0].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_i_; + break; + + case 2: + str.inputTensor[0].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_j_; + break; + + case 3: + str.inputTensor[1].str_n /= 2.0; + str.outputTensor.str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_k_; + break; + + case 4: + str.inputTensor[1].str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_di_; + break; + + case 5: + str.inputTensor[1].str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_dj_; + break; + + case 6: + str.inputTensor[0].str_c /= 2.0; + str.inputTensor[1].str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_q_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure: CostConvolution failed."; + } + return str; +} + +// Get optimal strategy for Pooling +StrategyRec CostPooling::GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph) { + int tensor_n = static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n); + int tensor_c = static_cast(node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c); + + std::vector cost_op; + std::vector> mode; + + if (tensor_n < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{0.5, 1, 1, 1}, {0.5, 1, 1, 1}, {0.5, 1, 1, 1}}, graph)); + } + + if (tensor_c < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 0.5, 1, 1}, {1, 0.5, 1, 1}, {1, 0.5, 1, 1}}, graph)); + } + + cost_op.push_back(DOUBLE_MAX); + cost_op.push_back(DOUBLE_MAX); + + return ChoseStr(cost_op, node.apply.str); +} + +// Chose strategy for Pooling +StrategyRec CostPooling::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_n /= 2.0; + str.outputTensor.str_n /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 1: + str.inputTensor[0].str_c /= 2.0; + str.outputTensor.str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 2: + str.inputTensor[0].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 3: + str.inputTensor[0].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure: CostPooling failed."; + } + return str; +} + +// Get optimal strategy for Add +StrategyRec CostAdd::GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph) { + int tensor_n = static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n); + int tensor_c = static_cast(node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c); + int tensor_h = static_cast(node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_str.str_h); + int tensor_w = static_cast(node.tensor_parm.tensor_shape.shape_w * node.tensor_parm.tensor_str.str_w); + + std::vector cost_op; + std::vector> mode; + + if (tensor_n < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{0.5, 1, 1, 1}, {0.5, 1, 1, 1}, {0.5, 1, 1, 1}}, graph)); + } + + if (tensor_c < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 0.5, 1, 1}, {1, 0.5, 1, 1}, {1, 0.5, 1, 1}}, graph)); + } + + if (tensor_h < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 0.5, 1}, {1, 1, 0.5, 1}, {1, 1, 0.5, 1}}, graph)); + } + + if (tensor_w < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 0.5}, {1, 1, 1, 0.5}, {1, 1, 1, 0.5}}, graph)); + } + + return ChoseStr(cost_op, node.apply.str); +} + +// Chose strategy for Add +StrategyRec CostAdd::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_n /= 2.0; + str.inputTensor[1].str_n /= 2.0; + str.outputTensor.str_n /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 1: + str.inputTensor[0].str_c /= 2.0; + str.inputTensor[1].str_c /= 2.0; + str.outputTensor.str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 2: + str.inputTensor[0].str_h /= 2.0; + str.inputTensor[1].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 3: + str.inputTensor[0].str_w /= 2.0; + str.inputTensor[1].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure: CostAdd failed."; + } + return str; +} + +// Get optimal strategy for Reshape +StrategyRec CostReshape::GetOptimalStr(const Graph::NodeType &node) const { return ChoseStr(node.apply.str); } + +StrategyRec CostReshape::ChoseStr(StrategyRec str) const { return str; } + +// Get optimal strategy for Biasadd +StrategyRec CostBiasAdd::GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph) { + int tensor_n = static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n); + int tensor_c = static_cast(node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c); + int tensor_h = static_cast(node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_str.str_h); + int tensor_w = static_cast(node.tensor_parm.tensor_shape.shape_w * node.tensor_parm.tensor_str.str_w); + + std::vector cost_op; + std::vector> mode; + + if (tensor_n < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{0.5, 1, 1, 1}, {0.5, 1, 1, 1}, {0.5, 1, 1, 1}}, graph)); + } + + if (tensor_c < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 0.5, 1, 1}, {1, 0.5, 1, 1}, {1, 0.5, 1, 1}}, graph)); + } + + if (tensor_h < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 0.5, 1}, {1, 1, 0.5, 1}, {1, 1, 0.5, 1}}, graph)); + } + + if (tensor_w < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 0.5}, {1, 1, 1, 0.5}, {1, 1, 1, 0.5}}, graph)); + } + + return ChoseStr(cost_op, node.apply.str); +} + +// Chose strategy for BiasAdd +StrategyRec CostBiasAdd::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_n /= 2.0; + str.inputTensor[1].str_n /= 2.0; + str.outputTensor.str_n /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 1: + str.inputTensor[0].str_c /= 2.0; + str.inputTensor[1].str_c /= 2.0; + str.outputTensor.str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 2: + str.inputTensor[0].str_h /= 2.0; + str.inputTensor[1].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 3: + str.inputTensor[0].str_w /= 2.0; + str.inputTensor[1].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure: CostBiasAdd failed."; + } + return str; +} + +// Get optimal strategy for Common OPs: ReLU and Softmax +StrategyRec CostCommon::GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph) { + int tensor_n = static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n); + int tensor_c = static_cast(node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c); + int tensor_h = static_cast(node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_str.str_h); + int tensor_w = static_cast(node.tensor_parm.tensor_shape.shape_w * node.tensor_parm.tensor_str.str_w); + + std::vector cost_op; + std::vector> mode; + + if (tensor_n < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{0.5, 1, 1, 1}, {0.5, 1, 1, 1}, {0.5, 1, 1, 1}}, graph)); + } + + if (tensor_c < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 0.5, 1, 1}, {1, 0.5, 1, 1}, {1, 0.5, 1, 1}}, graph)); + } + + if (tensor_h < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 0.5, 1}, {1, 1, 0.5, 1}, {1, 1, 0.5, 1}}, graph)); + } + + if (tensor_w < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(cost_in_ + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 0.5}, {1, 1, 1, 0.5}, {1, 1, 1, 0.5}}, graph)); + } + + return ChoseStr(cost_op, node.apply.str); +} + +// Chose strategy for Common op +StrategyRec CostCommon::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_n /= 2.0; + str.outputTensor.str_n /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 1: + str.inputTensor[0].str_c /= 2.0; + str.outputTensor.str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 2: + str.inputTensor[0].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + case 3: + str.inputTensor[0].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure: CostBiasAdd failed."; + } + return str; +} + +// Get optimal strategy for BN +StrategyRec CostBatchNorm::GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph) { + const OperatorRec &op = node.apply; + + int tensor_filter_n = static_cast(op.arguments[1].tensor_shape.shape_n * op.arguments[1].tensor_str.str_n); + int tensor_filter_c = static_cast(op.arguments[1].tensor_shape.shape_c * op.arguments[1].tensor_str.str_c); + int tensor_filter_h = static_cast(op.arguments[1].tensor_shape.shape_h * op.arguments[1].tensor_str.str_h); + int tensor_filter_w = static_cast(op.arguments[1].tensor_shape.shape_w * op.arguments[1].tensor_str.str_w); + + int tensor_filter = tensor_filter_h * tensor_filter_w * tensor_filter_n * tensor_filter_c; + + int output_tensor_h = static_cast(node.tensor_parm.tensor_shape.shape_h * node.tensor_parm.tensor_str.str_h); + int output_tensor_w = static_cast(node.tensor_parm.tensor_shape.shape_w * node.tensor_parm.tensor_str.str_w); + int output_tensor_n = static_cast(node.tensor_parm.tensor_shape.shape_n * node.tensor_parm.tensor_str.str_n); + int output_tensor_c = static_cast(node.tensor_parm.tensor_shape.shape_c * node.tensor_parm.tensor_str.str_c); + + std::vector cost_op; + std::vector> mode; + + if (output_tensor_n < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimB(tensor_filter) + CostRedis(node, node_name_to_strategy, + mode = {{0.5, 1, 1, 1}, {1, 1, 1, 1}, {0.5, 1, 1, 1}}, graph)); + } + + if (output_tensor_c < 2 || tensor_filter_c < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimC() + CostRedis(node, node_name_to_strategy, + mode = {{1, 0.5, 1, 1}, {1, 0.5, 1, 1}, {1, 0.5, 1, 1}}, graph)); + } + + if (output_tensor_h < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimH(tensor_filter) + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 0.5, 1}, {1, 1, 1, 1}, {1, 1, 0.5, 1}}, graph)); + } + + if (output_tensor_w < 2) { + cost_op.push_back(DOUBLE_MAX); + } else { + cost_op.push_back(StrDimW(tensor_filter) + CostRedis(node, node_name_to_strategy, + mode = {{1, 1, 1, 0.5}, {1, 1, 1, 1}, {1, 1, 1, 0.5}}, graph)); + } + + return ChoseStr(cost_op, node.apply.str); +} + +// Chose strategy for BatchNorm +StrategyRec CostBatchNorm::ChoseStr(const std::vector &cost_op, StrategyRec str) { + uint64_t min_position = min_element(cost_op.begin(), cost_op.end()) - cost_op.begin(); + if (cost_op[min_position] > (DOUBLE_MAX - 0.1)) { + return str; + } + + switch (min_position) { + case 0: + str.inputTensor[0].str_n /= 2.0; + str.outputTensor.str_n /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_b_; + break; + + case 1: + str.inputTensor[0].str_c /= 2.0; + str.inputTensor[1].str_c /= 2.0; + str.inputTensor[2].str_c /= 2.0; + str.inputTensor[3].str_c /= 2.0; + str.inputTensor[4].str_c /= 2.0; + str.outputTensor.str_c /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_c_; + break; + + case 2: + str.inputTensor[0].str_h /= 2.0; + str.outputTensor.str_h /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_h_; + break; + + case 3: + str.inputTensor[0].str_w /= 2.0; + str.outputTensor.str_w /= 2.0; + str.cut_counter += 1; + str.cost = str.cost + cost_in_w_; + break; + + default: + MS_LOG(EXCEPTION) << "Failure: CostBatchNorm failed."; + } + return str; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_cost.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_cost.h new file mode 100644 index 00000000000..7e07ff9be30 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_cost.h @@ -0,0 +1,264 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_COST_H_ +#define PARALLEL_AUTO_PARALLEL_REC_COST_H_ + +#include +#include +#include +#include +#include + +#include "parallel/auto_parallel/rec_core/rec_strategy.h" +#include "parallel/auto_parallel/rec_core/rec_graph.h" + +namespace mindspore { +namespace parallel { +double CostRedis(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const std::vector> &mode, const Graph &graph); + +double CostRedisWithAdjacentNode(const std::vector> &node_name_to_strategy, + const std::vector> &mode, size_t i_strategy, size_t i_node, + double tensor_size, bool is_search_forward); + +// class CostMatMul is used to compute the cost of MatMul operator. +class CostMatMul { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn(const OperatorRec &op); + + private: + double StrConcatDimI(int32_t a, int32_t b) { + cost_in_i_ = (static_cast(a) * static_cast(b)) / 2.0; + + return cost_in_i_; + } + + double StrConcatDimJ(int32_t a, int32_t b) { + cost_in_j_ = (static_cast(a) * static_cast(b)) / 2.0; + + return cost_in_j_; + } + + double StrReduceDimK(int32_t a, int32_t b) { + cost_in_k_ = (static_cast(a) * static_cast(b)) / 2.0; + + return cost_in_k_; + } + + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_i_ = 0; + + double cost_in_j_ = 0; + + double cost_in_k_ = 0; +}; // class CostMatMul is used to compute the cost of MatMul operator. + +// class CostConvolution is used to compute the cost of Conv operator. +class CostConvolution { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn(const Graph::NodeType &node); + + private: + double StrDimB(int32_t TensorFilter) { + cost_in_b_ = static_cast((TensorFilter) / 2.0); + + return cost_in_b_; + } + + double StrDimI(int32_t TensorIn, int32_t TensorFilter) { + cost_in_i_ = static_cast((TensorIn + TensorFilter) / 2.0); + + return cost_in_i_; + } + + double StrDimJ(int32_t TensorIn, int32_t TensorFilter) { + cost_in_j_ = static_cast((TensorIn + TensorFilter) / 2.0); + + return cost_in_j_; + } + + double StrDimK(int32_t TensorIn) { + cost_in_k_ = static_cast((TensorIn) / 2.0); + + return cost_in_k_; + } + + double StrDimDI(int32_t TensorIn, int32_t TensorOut) { + cost_in_di_ = static_cast((TensorIn + TensorOut) / 2.0); + + return cost_in_di_; + } + + double StrDimDJ(int32_t TensorIn, int32_t TensorOut) { + cost_in_dj_ = static_cast((TensorIn + TensorOut) / 2.0); + + return cost_in_dj_; + } + + double StrDimQ(int32_t TensorOut) { + cost_in_q_ = static_cast((TensorOut) / 2.0); + + return cost_in_q_; + } + + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_b_ = 0; + + double cost_in_i_ = 0; + + double cost_in_j_ = 0; + + double cost_in_k_ = 0; + + double cost_in_di_ = 0; + + double cost_in_dj_ = 0; + + double cost_in_q_ = 0; +}; // class CostConvolution is used to compute the cost of Conv operator. + +// class CostPooling is used to compute the cost of Pooling operator. +class CostPooling { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn() const { return cost_in_; } + + private: + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_ = 0; +}; // class CostPooling is used to compute the cost of Pooling operator. + +// class CostAdd is used to compute the cost of Add operator. +class CostAdd { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn() const { return cost_in_; } + + private: + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_ = 0; +}; // class CostAdd is used to compute the cost of Add operator. + +// class CostReshape is used to compute the cost of Reshape operator. +class CostReshape { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node) const; + + double GetMinCostIn() const { return cost_in_; } + + private: + StrategyRec ChoseStr(StrategyRec str) const; + + double cost_in_ = 0; +}; // class CostReshape is used to compute the cost of Reshape operator. + +// class CostBiasAdd is used to compute the cost of BiasAdd operator. +class CostBiasAdd { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn() const { return cost_in_; } + + private: + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_ = 0; +}; // class CostBiasAdd is used to compute the cost of BiasAdd operator. + +// class CostCommon is used to compute the cost of the element independent operator. +class CostCommon { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn() const { return cost_in_; } + + private: + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_ = 0; +}; // class CostCommon is used to compute the cost of Softmax & || Activation operator. + +// class BatchNorm is used to compute the cost of BatchNorm operator. +class CostBatchNorm { + public: + StrategyRec GetOptimalStr(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + const Graph &graph); + + double GetMinCostIn() const { return 0.0; } + + private: + double StrDimB(int32_t Tensor) { + cost_in_b_ = (static_cast(Tensor) * 4.0) / 2.0; + + return cost_in_b_; + } + + double StrDimC() { + cost_in_c_ = 0.0; + + return cost_in_c_; + } + + double StrDimH(int32_t Tensor) { + cost_in_h_ = (static_cast(Tensor) * 4.0) / 2.0; + + return cost_in_h_; + } + + double StrDimW(int32_t Tensor) { + cost_in_w_ = (static_cast(Tensor) * 4.0) / 2.0; + + return cost_in_w_; + } + + StrategyRec ChoseStr(const std::vector &cost_op, StrategyRec str); + + double cost_in_b_ = 0; + + double cost_in_c_ = 0; + + double cost_in_h_ = 0; + + double cost_in_w_ = 0; +}; // class BatchNorm is used to compute the cost of BatchNorm operator. +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_REC_COST_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc new file mode 100644 index 00000000000..06b7bf544a0 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc @@ -0,0 +1,212 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/rec_core/rec_generate_strategy.h" + +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/rec_core/rec_partition.h" +#include "parallel/strategy.h" +#include "ir/value.h" + +namespace mindspore { +namespace parallel { +void GenerateStrategy(const std::shared_ptr graph, std::vector> ops, + const std::shared_ptr> ops_nodes_list, + const std::shared_ptr> index_list, + const std::shared_ptr>> eli_list) { + MaskNoSupportedOps(graph); + for (size_t iter_ops = 0; iter_ops < ops.size(); iter_ops++) { + auto type = ops[iter_ops]->type(); + size_t iter_nodes = index_list->at(ops_nodes_list->at(iter_ops)); + std::vector> stra; + iter_nodes = IterNodes(ops_nodes_list, index_list, eli_list, iter_ops, iter_nodes); + for (size_t iter_op_inputs = 0; iter_op_inputs < ops[iter_ops]->inputs_tensor_info().size(); iter_op_inputs++) { + std::vector s = PrepareStrategy(graph, ops, type, iter_ops, iter_nodes, iter_op_inputs); + stra.push_back(s); + } + StrategyPtr sp = std::make_shared(0, stra); + ops[iter_ops]->SetSelectedStrategyAndCost(sp, ops[iter_ops]->selected_cost()); + } +} + +size_t IterNodes(const std::shared_ptr> ops_nodes_list, + const std::shared_ptr> index_list, + const std::shared_ptr>> eli_list, const size_t iter_ops, + size_t iter_nodes) { + if (iter_nodes > SIZE_MAX / 2) { + for (size_t iter_eli = 0; iter_eli < eli_list->size(); iter_eli++) { + if (eli_list->at(iter_eli)[0] == ops_nodes_list->at(iter_ops)) { + iter_nodes = index_list->at(eli_list->at(iter_eli)[1]); + break; + } + } + } + return iter_nodes; +} + +void PrepareMatMul(const std::shared_ptr graph, const std::vector> &ops, + const size_t iter_ops, const size_t iter_nodes, const size_t iter_op_inputs, + std::vector s) { + auto attrs = ops[iter_ops]->attrs(); + bool transpose_a = attrs[TRANSPOSE_A]->cast()->value(); + bool transpose_b = attrs[TRANSPOSE_B]->cast()->value(); + if (transpose_a && (iter_op_inputs == 0)) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_w)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_h)); + } else if (transpose_b && (iter_op_inputs == 1)) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_w)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_h)); + } else { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_w)); + } +} + +void PrepareConv2D(const std::shared_ptr graph, const size_t iter_nodes, size_t iter_op_inputs, + std::vector s) { + if (iter_op_inputs == 0) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_n)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_c)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_w)); + } else { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[1].tensor_str.str_n)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[1].tensor_str.str_c)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[1].tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[1].tensor_str.str_w)); + } +} + +void PrepareBiasAdd(const std::shared_ptr graph, const size_t iter_nodes, const size_t iter_op_inputs, + std::vector s) { + if (iter_op_inputs == 0) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_w)); + } else { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_w)); + } +} + +void PrepareBN(const std::shared_ptr graph, const size_t iter_nodes, const size_t iter_op_inputs, + std::vector s) { + if (iter_op_inputs == 0) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_n)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_c)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[0].tensor_str.str_w)); + } else { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[1].tensor_str.str_w)); + } +} + +void PrepareSparse(const size_t iter_op_inputs, std::vector s) { + if (iter_op_inputs == 0) { + s.push_back(g_device_manager->DeviceNum()); + s.push_back(1); + } else { + s.push_back(g_device_manager->DeviceNum()); + } +} + +void RefillOrigin(const std::vector> &ops, const size_t iter_ops, + const size_t iter_op_inputs, std::vector s) { + StrategyPtr origin_strategy = ops[iter_ops]->strategy(); + if (iter_op_inputs == 0) { + for (size_t j = 0; j < origin_strategy->GetInputDim()[0].size(); j++) { + s.push_back(1); + } + } else { + for (size_t k = 0; k < origin_strategy->GetInputDim()[iter_op_inputs].size(); k++) { + s.push_back(1); + } + } +} + +std::vector PrepareStrategy(const std::shared_ptr graph, + const std::vector> &ops, const std::string &type, + const size_t iter_ops, const size_t iter_nodes, const size_t iter_op_inputs) { + std::vector s; + if (type == MATMUL) { + PrepareMatMul(graph, ops, iter_ops, iter_nodes, iter_op_inputs, s); + } else if ((type == MAXPOOL) || (type == SIMPLE_MEAN) || (type == TENSOR_ADD)) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_n)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_c)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].apply.arguments[iter_op_inputs].tensor_str.str_w)); + } else if (type == CONV2D) { + PrepareConv2D(graph, iter_nodes, iter_op_inputs, s); + } else if (type == BIAS_ADD) { + PrepareBiasAdd(graph, iter_nodes, iter_op_inputs, s); + } else if (type == RESHAPE) { + s.push_back(1); + s.push_back(1); + s.push_back(1); + s.push_back(1); + } else if (type == RELU) { + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].tensor_parm.tensor_str.str_n)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].tensor_parm.tensor_str.str_c)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].tensor_parm.tensor_str.str_h)); + s.push_back(static_cast(1.0 / graph->nodes[iter_nodes].tensor_parm.tensor_str.str_w)); + } else if (type == BATCH_NORM || (type == FUSE_BATCH_NORM)) { + PrepareBN(graph, iter_nodes, iter_op_inputs, s); + } else if (type == SOFTMAX_CROSS_ENTROPY_WITH_LOGITS) { + PrepareSparse(iter_op_inputs, s); + } else { + RefillOrigin(ops, iter_ops, iter_op_inputs, s); + } + return s; +} + +void MaskNoSupportedOps(const std::shared_ptr graph) { + size_t iter_nodes = graph->nodes.size(); + for (size_t i = 0; i < iter_nodes; i++) { + if (0 == graph->nodes[i].info) { + Graph::NodeType &node = graph->nodes[i]; + + if (node.apply.op_type == 1) { // For Convolution + // cover input tensor strategy + node.apply.arguments[0].tensor_str.str_n = 1.0 / static_cast(g_device_manager->DeviceNum()); + node.apply.arguments[0].tensor_str.str_c = 1; + node.apply.arguments[0].tensor_str.str_h = 1; + node.apply.arguments[0].tensor_str.str_w = 1; + // cover filter tensor strategy + node.apply.arguments[1].tensor_str.str_n = 1; + node.apply.arguments[1].tensor_str.str_c = 1; + node.apply.arguments[1].tensor_str.str_h = 1; + node.apply.arguments[1].tensor_str.str_w = 1; + } else if (node.apply.op_type == 8) { // For BN + node.apply.arguments[0].tensor_str.str_n = 1.0 / static_cast(g_device_manager->DeviceNum()); + node.apply.arguments[0].tensor_str.str_c = 1; + node.apply.arguments[0].tensor_str.str_h = 1; + node.apply.arguments[0].tensor_str.str_w = 1; + // cover 1-d argument blobs + node.apply.arguments[1].tensor_str.str_w = 1; + node.apply.arguments[2].tensor_str.str_w = 1; + node.apply.arguments[3].tensor_str.str_w = 1; + node.apply.arguments[4].tensor_str.str_w = 1; + } else if (node.apply.op_type == 4 || node.apply.op_type == 9) { // For SparseSoftmaxCrossEntropyWithLogits + node.tensor_parm.tensor_str.str_h = 1.0 / static_cast(g_device_manager->DeviceNum()); + node.tensor_parm.tensor_str.str_w = 1; + } + } + } +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h new file mode 100644 index 00000000000..7445d3940eb --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_GENERATE_STRATEGY_H_ +#define PARALLEL_AUTO_PARALLEL_REC_GENERATE_STRATEGY_H_ + +#include +#include +#include +#include + +#include "parallel/auto_parallel/rec_core/rec_graph.h" +#include "parallel/ops_info/operator_info.h" + +namespace mindspore { +namespace parallel { +void GenerateStrategy(const std::shared_ptr graph, std::vector> ops, + const std::shared_ptr> ops_nodes_list, + const std::shared_ptr> index_list, + const std::shared_ptr>> eli_list); +void PrepareMatMul(const std::shared_ptr graph, const std::vector> &ops, + const size_t iter_ops, const size_t iter_nodes, const size_t iter_op_inputs, std::vector s); +void PrepareConv2D(const std::shared_ptr graph, const size_t iter_nodes, const size_t iter_op_inputs, + std::vector s); +void PrepareBiasAdd(const std::shared_ptr graph, const size_t iter_nodes, const size_t iter_op_inputs, + std::vector s); +void PrepareBN(const std::shared_ptr graph, const size_t iter_nodes, const size_t iter_op_inputs, + std::vector s); +void PrepareSparse(const size_t iter_op_inputs, std::vector s); +void RefillOrigin(const std::vector> &ops, const size_t iter_ops, + const size_t iter_op_inputs, std::vector s); +std::vector PrepareStrategy(const std::shared_ptr graph, + const std::vector> &ops, const std::string &type, + const size_t iter_ops, const size_t iter_nodes, const size_t iter_op_inputs); +size_t IterNodes(const std::shared_ptr> ops_nodes_list, + const std::shared_ptr> index_list, + const std::shared_ptr>> eli_list, const size_t iter_ops, + size_t iter_nodes); +void MaskNoSupportedOps(const std::shared_ptr graph); +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_REC_GENERATE_STRATEGY_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h new file mode 100644 index 00000000000..209ce6b13e7 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h @@ -0,0 +1,72 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_GRAPH_H_ +#define PARALLEL_AUTO_PARALLEL_REC_GRAPH_H_ + +#include +#include +#include + +#include "parallel/auto_parallel/rec_core/rec_tensor.h" +#include "parallel/auto_parallel/rec_core/rec_strategy.h" + +namespace mindspore { +namespace parallel { +enum OperatorType { + kRecMatMul, + kRecConvolution, + kRecPooling, + kRecAdd, + kRecSoftmax, + kRecReshape, + kRecBiasAdd, + kRecReLU, + kRecBatchNorm, + kRecSparseSoftmaxCrossEntropyWithLogits, + kRecUnkownType +}; + +enum InfoType { kApplication, kConstant }; + +struct OperatorRec { + OperatorType op_type; + TensorParam arguments[MAX_INPUT_NUM]; + StrategyRec str; +}; + +// Define simplified dataflow Graph for partitioning +class Graph { + public: + struct NodeType { + std::string name; + // Nodes that point to this node + std::vector node_in; + // Nodes that point from this node + std::vector node_out; + // Node Type Info: Application or Constant. Defined in enum . + InfoType info; + // Operator info. Defined in struct . + OperatorRec apply; + // Tensor info. Defined in tensor.h struct . + TensorParam tensor_parm; + }; + + std::vector nodes; // Nodes of the graph. Pubic. +}; // Define simplified dataflow Graph for partitioning +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_REC_GRAPH_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc new file mode 100644 index 00000000000..153ce2ea17a --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc @@ -0,0 +1,342 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/rec_core/rec_parse_graph.h" + +#include +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/rec_core/rec_tensor.h" +#include "parallel/auto_parallel/rec_core/rec_graph.h" +#include "ir/value.h" + +namespace mindspore { +namespace parallel { +const TensorParam MakeTensor(int n, int c, int h, int w) { + TensorParam new_tensor; + new_tensor.tensor_type = kFloat32; + new_tensor.tensor_shape.shape_n = n; + new_tensor.tensor_shape.shape_c = c; + new_tensor.tensor_shape.shape_h = h; + new_tensor.tensor_shape.shape_w = w; + const TensorParam& tensor = new_tensor; + return tensor; +} + +bool IsInList(const std::string& name, const std::vector& list) { + return std::find(list.begin(), list.end(), name) != list.end(); +} + +Graph::NodeType MakeNewOperator(std::vector> ops, size_t iter_ops) { + Graph::NodeType NewOp; + NewOp.name = ops[iter_ops]->cnode_name(); + NewOp.info = InfoType::kApplication; + + auto op_type = ops[iter_ops]->type(); + auto idx = DictOpType.find(op_type); + if (idx == DictOpType.end()) { + NewOp.apply.op_type = OperatorType::kRecUnkownType; + MS_LOG(INFO) << "Unknown type in rec_parse_graph::MakeNewOperator"; + } else { + NewOp.apply.op_type = DictOpType.at(op_type); + } + + if ((NewOp.apply.op_type == OperatorType::kRecMatMul) || (NewOp.apply.op_type == OperatorType::kRecBiasAdd) || + (NewOp.apply.op_type == OperatorType::kRecReshape)) { + NewOp.tensor_parm = MakeTensor(1, 1, ops[iter_ops]->outputs_tensor_info()[0].shape()[0], + ops[iter_ops]->outputs_tensor_info()[0].shape()[1]); + } else if ((NewOp.apply.op_type == OperatorType::kRecSparseSoftmaxCrossEntropyWithLogits) || + (NewOp.apply.op_type == OperatorType::kRecUnkownType)) { + NewOp.tensor_parm = MakeTensor(1, 1, 1, 1); + } else { + NewOp.tensor_parm = MakeTensor( + ops[iter_ops]->outputs_tensor_info()[0].shape()[0], ops[iter_ops]->outputs_tensor_info()[0].shape()[1], + ops[iter_ops]->outputs_tensor_info()[0].shape()[2], ops[iter_ops]->outputs_tensor_info()[0].shape()[3]); + } + + return NewOp; +} + +Graph::NodeType MakeNewTensor(std::vector> ops, const size_t iter_ops, + const std::string& input, const size_t iter_input_tensors, std::shared_ptr graph, + size_t current_op_index) { + Graph::NodeType NewTensor; + NewTensor.name = input; + NewTensor.info = InfoType::kConstant; + + if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 4) { + NewTensor.tensor_parm = MakeTensor(ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[2], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[3]); + } else if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 2) { + Fill2DTensor(ops, iter_ops, graph, iter_input_tensors, current_op_index, NewTensor); + } else if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 1) { + NewTensor.tensor_parm = MakeTensor(1, 1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0]); + } else if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 0) { + NewTensor.tensor_parm = MakeTensor(1, 1, 1, 1); + } else { + MS_LOG(ERROR) << "Tensor's shape unknown in rec_parse_graph::MakeNewTensor"; + } + return NewTensor; +} + +void Fill2DTensor(const std::vector>& ops, const size_t iter_ops, + const std::shared_ptr graph, const size_t iter_input_tensors, const size_t current_op_index, + Graph::NodeType NewTensor) { + if (graph->nodes[current_op_index].apply.op_type == OperatorType::kRecMatMul) { + auto attrs = ops[iter_ops]->attrs(); + bool transpose_a = attrs[TRANSPOSE_A]->cast()->value(); + bool transpose_b = attrs[TRANSPOSE_B]->cast()->value(); + if (transpose_a && (iter_input_tensors == 0)) { + NewTensor.tensor_parm = MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0]); + } else if (transpose_b && (iter_input_tensors == 1)) { + NewTensor.tensor_parm = MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0]); + } else { + NewTensor.tensor_parm = MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1]); + } + } else { + NewTensor.tensor_parm = MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1]); + } +} + +void CompleteOperatorInputs(std::vector> ops, size_t iter_ops, size_t iter_input_tensors, + size_t current_op_index, std::shared_ptr graph) { + if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 4) { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = + MakeTensor(ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[2], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[3]); + } else if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 2) { + Complete2DInputs(ops, iter_ops, graph, iter_input_tensors, current_op_index); + } else if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 1) { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = + MakeTensor(1, 1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0]); + } else if (ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape().size() == 0) { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = MakeTensor(1, 1, 1, 1); + } else { + MS_LOG(ERROR) << "Tensor's shape unknown in rec_parse_graph::MakeNewTensor"; + } +} + +void Complete2DInputs(const std::vector>& ops, const size_t iter_ops, + const std::shared_ptr graph, const size_t iter_input_tensors, + const size_t current_op_index) { + if (graph->nodes[current_op_index].apply.op_type == OperatorType::kRecMatMul) { + auto attrs = ops[iter_ops]->attrs(); + bool transpose_a = attrs[TRANSPOSE_A]->cast()->value(); + bool transpose_b = attrs[TRANSPOSE_B]->cast()->value(); + if (transpose_a && (iter_input_tensors == 0)) { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = + MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0]); + } else if (transpose_b && (iter_input_tensors == 1)) { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = + MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0]); + } else { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = + MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1]); + } + } else { + graph->nodes[current_op_index].apply.arguments[iter_input_tensors] = + MakeTensor(1, 1, ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[0], + ops[iter_ops]->inputs_tensor_info()[iter_input_tensors].shape()[1]); + } +} + +void MakeEdge(std::shared_ptr graph, const size_t input_index, const size_t current_op_index) { + graph->nodes[input_index].node_out.push_back(current_op_index); + graph->nodes[current_op_index].node_in.push_back(input_index); +} + +void ModifyTensorToOperator(std::shared_ptr graph, const size_t current_op_index, const size_t iter_ops, + std::vector> ops) { + graph->nodes[current_op_index].info = InfoType::kApplication; + std::string op_type = ops[iter_ops]->type(); + auto idx = DictOpType.find(op_type); + if (idx == DictOpType.end()) { + graph->nodes[current_op_index].apply.op_type = OperatorType::kRecUnkownType; + MS_LOG(INFO) << "Unknown type in rec_parse_graph::ModifyTensorToOperator"; + } else { + graph->nodes[current_op_index].apply.op_type = DictOpType.at(op_type); + } + + if ((graph->nodes[current_op_index].apply.op_type == OperatorType::kRecMatMul) || + (graph->nodes[current_op_index].apply.op_type == OperatorType::kRecBiasAdd) || + (graph->nodes[current_op_index].apply.op_type == OperatorType::kRecReshape)) { + graph->nodes[current_op_index].tensor_parm = MakeTensor(1, 1, ops[iter_ops]->outputs_tensor_info()[0].shape()[0], + ops[iter_ops]->outputs_tensor_info()[0].shape()[1]); + } else if ((graph->nodes[current_op_index].apply.op_type == OperatorType::kRecSparseSoftmaxCrossEntropyWithLogits) || + (graph->nodes[current_op_index].apply.op_type == OperatorType::kRecUnkownType)) { + graph->nodes[current_op_index].tensor_parm = MakeTensor(1, 1, 1, 1); + } else { + graph->nodes[current_op_index].tensor_parm = MakeTensor( + ops[iter_ops]->outputs_tensor_info()[0].shape()[0], ops[iter_ops]->outputs_tensor_info()[0].shape()[1], + ops[iter_ops]->outputs_tensor_info()[0].shape()[2], ops[iter_ops]->outputs_tensor_info()[0].shape()[3]); + } +} + +std::shared_ptr ParseGraph(const std::vector>& ops, + const std::vector>& input_tensor_names, + const std::shared_ptr>& ops_nodes_list) { + std::vector current_graph; + std::shared_ptr graph(new Graph); + if (ops.size() > SIZE_MAX / 2) { + MS_LOG(EXCEPTION) << "Total number of operators is bigger than " << SIZE_MAX / 2; + } + + for (size_t iter_ops = ops.size(); iter_ops > 0; iter_ops--) { + if (IsInList(ops[iter_ops - 1]->cnode_name(), current_graph)) { + size_t current_op_index = static_cast(std::distance( + current_graph.begin(), std::find(current_graph.begin(), current_graph.end(), ops[iter_ops]->cnode_name()))); + std::vector::iterator itr = ops_nodes_list->insert(ops_nodes_list->begin(), current_op_index); + if (itr != ops_nodes_list->begin()) { + MS_LOG(EXCEPTION) << "Iterator error."; + } + ModifyTensorToOperator(graph, current_op_index, iter_ops - 1, ops); + LinkOps(graph, ops, input_tensor_names, current_graph, iter_ops - 1, current_op_index); + } else { + Graph::NodeType NewOp = MakeNewOperator(ops, iter_ops - 1); + current_graph.push_back(NewOp.name); + graph->nodes.push_back(NewOp); + size_t current_op_index = graph->nodes.size() - 1; + std::vector::iterator itr = ops_nodes_list->insert(ops_nodes_list->begin(), current_op_index); + if (itr != ops_nodes_list->begin()) { + MS_LOG(EXCEPTION) << "Iterator error."; + } + LinkOps(graph, ops, input_tensor_names, current_graph, iter_ops - 1, current_op_index); + } + } + return graph; +} + +void LinkOps(std::shared_ptr graph, std::vector> ops, + const std::vector>& input_tensor_names, std::vector current_graph, + const size_t iter_ops, const size_t current_op_index) { + for (size_t iter_input_tensors = 0; + iter_input_tensors < std::min(input_tensor_names[iter_ops].size(), ops[iter_ops]->inputs_tensor_info().size()); + iter_input_tensors++) { + std::string input = input_tensor_names[iter_ops][iter_input_tensors]; + if (IsInList(input, current_graph)) { + size_t input_index = static_cast( + std::distance(current_graph.begin(), std::find(current_graph.begin(), current_graph.end(), input))); + MakeEdge(graph, input_index, current_op_index); + CompleteOperatorInputs(ops, iter_ops, iter_input_tensors, current_op_index, graph); + } else { + Graph::NodeType NewTensor = MakeNewTensor(ops, iter_ops, input, iter_input_tensors, graph, current_op_index); + current_graph.push_back(NewTensor.name); + graph->nodes.push_back(NewTensor); + size_t input_index = graph->nodes.size() - 1; + CompleteOperatorInputs(ops, iter_ops, iter_input_tensors, current_op_index, graph); + MakeEdge(graph, input_index, current_op_index); + } + + if (graph->nodes[current_op_index].apply.op_type == OperatorType::kRecBatchNorm) { + break; + } + } +} + +void Eliminate_Aux(const size_t node_index, std::shared_ptr graph, + const std::shared_ptr>> eli_list) { + if ((graph->nodes[node_index].apply.op_type == OperatorType::kRecUnkownType) || + (graph->nodes[node_index].apply.op_type == OperatorType::kRecReLU)) { + size_t input_index = (graph->nodes[node_index].node_in)[0]; + std::vector outputs = graph->nodes[node_index].node_out; + + std::vector eli; + eli.push_back(node_index); + eli.push_back(input_index); + for (size_t i = 0; i < outputs.size(); i++) { + eli.push_back(i); + } + eli_list->push_back(eli); + + for (size_t i = 1; i < (size_t)graph->nodes[node_index].node_in.size(); i++) { + std::vector tmp; + tmp.push_back(node_index); + tmp.push_back((graph->nodes[node_index].node_in)[i]); + eli_list->push_back(tmp); + } + + auto it = find(graph->nodes[input_index].node_out.begin(), graph->nodes[input_index].node_out.end(), node_index); + std::vector::iterator itr = graph->nodes[input_index].node_out.erase(it); + if (itr != it) { + MS_LOG(EXCEPTION) << "Iterator error."; + } + for (auto output : outputs) { + graph->nodes[input_index].node_out.push_back(output); + } + for (auto& output_index : outputs) { + auto itt = find(graph->nodes[output_index].node_in.begin(), graph->nodes[output_index].node_in.end(), node_index); + graph->nodes[output_index] + .node_in[static_cast(std::distance(graph->nodes[output_index].node_in.begin(), itt))] = input_index; + } + } +} + +std::shared_ptr EliminateGraph(const std::shared_ptr graph, + std::shared_ptr>> eli_list, + std::shared_ptr> index_list) { + for (size_t node_index = 0; node_index < (size_t)graph->nodes.size(); node_index++) { + if (graph->nodes[node_index].info == InfoType::kApplication) { + Eliminate_Aux(node_index, graph, eli_list); + } + } + + index_list->reserve(graph->nodes.size()); + for (size_t i = 0; i < (size_t)graph->nodes.size(); i++) { + index_list->push_back(i); + } + + for (size_t i = 0; i < (size_t)eli_list->size(); i++) { + index_list->at((eli_list->at(i)[0])) = SIZE_MAX; + for (size_t j = eli_list->at(i)[0] + 1; j < (size_t)index_list->size(); j++) { + index_list->at(j)--; + } + } + + std::shared_ptr new_graph(new Graph); + for (size_t i = 0; i < (size_t)(graph->nodes.size() - eli_list->size()); i++) { + Graph::NodeType NewOp; + new_graph->nodes.push_back(NewOp); + } + + for (size_t i = 0; i < (size_t)graph->nodes.size(); i++) { + if (index_list->at(i) > SIZE_MAX / 2) continue; + new_graph->nodes[index_list->at(i)] = graph->nodes[i]; + for (size_t j = 0; j < (size_t)new_graph->nodes[index_list->at(i)].node_in.size(); j++) { + new_graph->nodes[index_list->at(i)].node_in[j] = index_list->at(new_graph->nodes[index_list->at(i)].node_in[j]); + } + for (size_t j = 0; j < (size_t)new_graph->nodes[index_list->at(i)].node_out.size(); j++) { + new_graph->nodes[index_list->at(i)].node_out[j] = index_list->at(new_graph->nodes[index_list->at(i)].node_out[j]); + } + } + + return new_graph; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h new file mode 100644 index 00000000000..7904d260c02 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h @@ -0,0 +1,81 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_PARSE_GRAPH_H_ +#define PARALLEL_AUTO_PARALLEL_REC_PARSE_GRAPH_H_ + +#include +#include +#include +#include +#include + +#include "parallel/auto_parallel/rec_core/rec_graph.h" +#include "parallel/ops_info/operator_info.h" + +namespace mindspore { +namespace parallel { +const std::map DictOpType{ + {MATMUL, OperatorType::kRecMatMul}, + {CONV2D, OperatorType::kRecConvolution}, + {MAXPOOLV2, OperatorType::kRecPooling}, + {SIMPLE_MEAN, OperatorType::kRecPooling}, + {TENSOR_ADD, OperatorType::kRecAdd}, + {RESHAPE, OperatorType::kRecReshape}, + {BIAS_ADD, OperatorType::kRecBiasAdd}, + {RELU, OperatorType::kRecReLU}, + {BATCH_NORM, OperatorType::kRecBatchNorm}, + {SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS, OperatorType::kRecSparseSoftmaxCrossEntropyWithLogits}, +}; + +const TensorParam MakeTensor(int n, int c, int h, int w); + +bool IsInList(const std::string& name, const std::vector& list); + +Graph::NodeType MakeNewOperator(std::vector> ops, size_t iter_ops); + +Graph::NodeType MakeNewTensor(std::vector> ops, const size_t iter_ops, + const std::string& input, const size_t iter_input_tensors, std::shared_ptr graph, + size_t current_op_index); +void Fill2DTensor(const std::vector>& ops, const size_t iter_ops, + const std::shared_ptr graph, const size_t iter_input_tensors, const size_t current_op_index, + Graph::NodeType NewTensor); +void CompleteOperatorInputs(std::vector> ops, size_t iter_ops, size_t iter_input_tensors, + size_t current_op_index, std::shared_ptr graph); +void Complete2DInputs(const std::vector>& ops, const size_t iter_ops, + const std::shared_ptr graph, const size_t iter_input_tensors, + const size_t current_op_index); +void MakeEdge(std::shared_ptr graph, const size_t input_index, const size_t current_op_index); + +void ModifyTensorToOperator(std::shared_ptr graph, const size_t current_op_index, const size_t iter_ops, + std::vector> ops); + +std::shared_ptr ParseGraph(const std::vector>& ops, + const std::vector>& input_tensor_names, + const std::shared_ptr>& ops_nodes_list); + +void LinkOps(std::shared_ptr graph, std::vector> ops, + const std::vector>& input_tensor_names, std::vector current_graph, + const size_t iter_ops, const size_t current_op_index); + +std::shared_ptr EliminateGraph(const std::shared_ptr graph, + std::shared_ptr>> eli_list, + std::shared_ptr> index_list); +void Eliminate_Aux(const size_t node_index, std::shared_ptr graph, + const std::shared_ptr>> eli_list); +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_REC_PARSE_GRAPH_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc new file mode 100644 index 00000000000..a33b861f1a7 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc @@ -0,0 +1,357 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/auto_parallel/rec_core/rec_partition.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { +#define DEVICE_MEMORY 1024.0 * 1024.0 * 1024.0 // 1GB + +// Get the target node's weight for sorting. +double GetWeights(const Graph::NodeType &node) { + const OperatorRec &op = node.apply; + + if (op.op_type == 0) { + // For MatMul + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(op); + } else if (op.op_type == 1) { + // For Convolution + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(node); + } else if (op.op_type == 2) { + // For Pooling + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(); + } else if (op.op_type == 3) { + // For Add + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(); + } else if (op.op_type == 4 || op.op_type == 7 || op.op_type == 9) { + // For Softmax & || Activation + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(); + } else if (op.op_type == 5) { + // For Reshape + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(); + } else if (op.op_type == 6) { + // For BiasAdd + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(); + } else if (op.op_type == 8) { + // For BatchNorm + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetMinCostIn(); + } else { + MS_LOG(EXCEPTION) << "Failure: GetOperatorWeight failed."; + } +} + +// Sort all the nodes by their weights +std::vector SortByWeight(const std::shared_ptr graph) { + MS_EXCEPTION_IF_NULL(graph); + + std::vector> weight_to_node_index; + std::vector node_index_by_weights; + + // Get node's weight. + for (size_t i = 0; i < graph->nodes.size(); i++) { + if (graph->nodes[i].info == kApplication) { + const Graph::NodeType &node_ptr = graph->nodes[i]; + double weight = GetWeights(node_ptr); + size_t index = i; + weight_to_node_index.push_back(std::make_pair(weight, index)); + } + } + + // Do sorting. + sort(weight_to_node_index.begin(), weight_to_node_index.end()); + + // Store the result in node_index_by_weights. + uint64_t size = weight_to_node_index.size(); + for (uint64_t i = 1; i <= size; i++) { + node_index_by_weights.push_back(weight_to_node_index[size - i].second); + } + + return node_index_by_weights; +} + +// Get optimal strategy to partition the target node +StrategyRec PartitionNode(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + std::shared_ptr graph) { + MS_EXCEPTION_IF_NULL(graph); + + if (node.apply.op_type == 0) { + // For MatMul + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else if (node.apply.op_type == 1) { + // For Convolution + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else if (node.apply.op_type == 2) { + // For Pooling + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else if (node.apply.op_type == 3) { + // For Add + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else if (node.apply.op_type == 4 || node.apply.op_type == 7 || node.apply.op_type == 9) { + // For Softmax & Activation + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else if (node.apply.op_type == 5) { + // For Reshape + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node); + } else if (node.apply.op_type == 6) { + // For BiasAdd + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else if (node.apply.op_type == 8) { + // For BatchNorm + auto cost_ptr = std::make_shared(); + + return cost_ptr->GetOptimalStr(node, node_name_to_strategy, *graph); + } else { + MS_LOG(EXCEPTION) << "Failure: Partition Operator failed."; + } +} + +// Parttion graph into all devices. +Status PartitionForAllDevices(const size_t num_device, std::shared_ptr graph) { + if (num_device < 1) { + MS_LOG(EXCEPTION) << "ERROR: Number of devices can't be " << num_device << "."; + } + + if (num_device > 1024) { + MS_LOG(EXCEPTION) << "ERROR: Number of devices can't be larger than 1024."; + } + + MS_EXCEPTION_IF_NULL(graph); + + // Comopute iter times + int iter_times = static_cast(log2(num_device)); + + // N-cuts loop + for (int loop = 0; loop < iter_times; loop++) { + // Sort by weights + std::vector reorder_node_list = SortByWeight(graph); + + // get total node number + size_t iter_nodes = reorder_node_list.size(); + + // temp vector to map nodename to its strategy. + std::vector> node_name_to_strategy; + + // Loop for all the nodes + for (size_t i_node = 0; i_node < iter_nodes; i_node++) { + // get current node's index + size_t index = reorder_node_list[i_node]; + + Graph::NodeType &node_ptr = graph->nodes[index]; + + // Serch optimal strategy to cut this operator. And store the result optimal strategy in graph. + graph->nodes[index].apply.str = PartitionNode(node_ptr, node_name_to_strategy, graph); + + // Apply OP Strategy to Tensor Strategy. + graph->nodes[index] = ApplyStrToTensor(node_ptr); + + // Note down the node name and its strategy in this loop. + auto node_name_to_str = + std::pair(graph->nodes[index].name, graph->nodes[index].apply.str); + node_name_to_strategy.push_back(node_name_to_str); + } + } + + InferUndecideStrategy(graph); + if (DevicesMemoryControl(graph) != SUCCESS) { + return FAILED; + } else { + return SUCCESS; + } +} + +// Apply OP Strategy to Tensor Strategy +Graph::NodeType ApplyStrToTensor(Graph::NodeType Node) { + // Set Node's tensor_parm + Node.tensor_parm.tensor_str.str_n = Node.apply.str.outputTensor.str_n; + Node.tensor_parm.tensor_str.str_c = Node.apply.str.outputTensor.str_c; + Node.tensor_parm.tensor_str.str_h = Node.apply.str.outputTensor.str_h; + Node.tensor_parm.tensor_str.str_w = Node.apply.str.outputTensor.str_w; + + // Set input tensors' tersor_parm + for (int i = 0; i < 2; i++) { + Node.apply.arguments[i].tensor_str.str_n = Node.apply.str.inputTensor[i].str_n; + Node.apply.arguments[i].tensor_str.str_c = Node.apply.str.inputTensor[i].str_c; + Node.apply.arguments[i].tensor_str.str_h = Node.apply.str.inputTensor[i].str_h; + Node.apply.arguments[i].tensor_str.str_w = Node.apply.str.inputTensor[i].str_w; + } + return Node; +} + +// Check Strategy for the same tensor between op. +void InferUndecideStrategy(std::shared_ptr graph) { + MS_EXCEPTION_IF_NULL(graph); + + uint64_t iter_nodes = graph->nodes.size(); + + // For all the nodes in the graph + for (uint64_t i_node = 0; i_node < iter_nodes; i_node++) { + // If this target node is an operator, find it's adjecent op's strategy; + if (graph->nodes[i_node].info == 0) { + // Try to apply last op's strategy. + ApplyLastStrategy(i_node, graph); + // Try to apply next op's strategy. + ApplyNextStrategy(i_node, graph); + } + } +} + +void ApplyLastStrategy(const uint64_t node_index, std::shared_ptr graph) { + Graph::NodeType &target_node = graph->nodes[node_index]; + + // Number of node-in + size_t num_node_in = target_node.node_in.size(); + + // Find forward op and copy strategy if meets the limits. + for (size_t index = 0; index < num_node_in; index++) { + if (graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_n <= + target_node.apply.arguments[0].tensor_str.str_n && + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_c <= + target_node.apply.arguments[0].tensor_str.str_c && + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_h <= + target_node.apply.arguments[0].tensor_str.str_h && + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_w <= + target_node.apply.arguments[0].tensor_str.str_w) { + target_node.apply.arguments[0].tensor_str.str_n = + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_n; + target_node.apply.arguments[0].tensor_str.str_c = + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_c; + target_node.apply.arguments[0].tensor_str.str_h = + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_h; + target_node.apply.arguments[0].tensor_str.str_w = + graph->nodes[target_node.node_in[index]].tensor_parm.tensor_str.str_w; + } + } +} + +void ApplyNextStrategy(const uint64_t node_index, std::shared_ptr graph) { + Graph::NodeType &target_node = graph->nodes[node_index]; + + // Number of node-out + size_t num_node_out = target_node.node_out.size(); + + // Find backward op and copy strategy if meets the limits. + for (size_t index = 0; index < num_node_out; index++) { + if (graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_n <= + target_node.tensor_parm.tensor_str.str_n && + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_c <= + target_node.tensor_parm.tensor_str.str_c && + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_h <= + target_node.tensor_parm.tensor_str.str_h && + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_w <= + target_node.tensor_parm.tensor_str.str_w) { + target_node.tensor_parm.tensor_str.str_n = + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_n; + target_node.tensor_parm.tensor_str.str_c = + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_c; + target_node.tensor_parm.tensor_str.str_h = + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_h; + target_node.tensor_parm.tensor_str.str_w = + graph->nodes[target_node.node_out[index]].apply.arguments[0].tensor_str.str_w; + } + } +} + +Status DevicesMemoryControl(std::shared_ptr graph) { + MS_EXCEPTION_IF_NULL(graph); + + uint64_t iter_nodes = graph->nodes.size(); + + for (uint64_t i_node = 0; i_node < iter_nodes; i_node++) { + if (graph->nodes[i_node].info == 0) { + Graph::NodeType &Node = graph->nodes[i_node]; + double used_memory = 0.0; + + for (int index = 0; index < 2; index++) { + used_memory += Node.apply.arguments[index].tensor_str.str_n * Node.apply.arguments[index].tensor_shape.shape_n * + Node.apply.arguments[index].tensor_str.str_c * Node.apply.arguments[index].tensor_shape.shape_c * + Node.apply.arguments[index].tensor_str.str_h * Node.apply.arguments[index].tensor_shape.shape_h * + Node.apply.arguments[index].tensor_str.str_w * Node.apply.arguments[index].tensor_shape.shape_w * + GetDataTypeSize(Node.apply.arguments[index].tensor_type); + } + + used_memory += Node.tensor_parm.tensor_str.str_n * Node.tensor_parm.tensor_shape.shape_n * + Node.tensor_parm.tensor_str.str_c * Node.tensor_parm.tensor_shape.shape_c * + Node.tensor_parm.tensor_str.str_h * Node.tensor_parm.tensor_shape.shape_h * + Node.tensor_parm.tensor_str.str_w * Node.tensor_parm.tensor_shape.shape_w * + GetDataTypeSize(Node.tensor_parm.tensor_type); + if (DEVICE_MEMORY < used_memory) { + MS_LOG(EXCEPTION) << "Failure: Out of memory!"; + return FAILED; + } + } + } + + return SUCCESS; +} + +size_t GetDataTypeSize(const TensorType &type) { + switch (type) { + case kInt8: + return sizeof(int); + case kFloat16: + return sizeof(float) / 2; + case kFloat32: + return sizeof(float); + case kDouble64: + return sizeof(double); + default: + MS_LOG(EXCEPTION) << "GetDataTypeSize Failed. Unexpected type"; + } +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h new file mode 100644 index 00000000000..4e49305156e --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h @@ -0,0 +1,59 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_PARTITION_H_ +#define PARALLEL_AUTO_PARALLEL_REC_PARTITION_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "parallel/auto_parallel/rec_core/rec_graph.h" +#include "parallel/auto_parallel/rec_core/rec_strategy.h" +#include "parallel/auto_parallel/rec_core/rec_cost.h" +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { +std::vector SortByWeight(const std::shared_ptr graph); + +double GetWeights(const Graph::NodeType &node); + +StrategyRec PartitionNode(const Graph::NodeType &node, + const std::vector> &node_name_to_strategy, + std::shared_ptr graph); + +Status PartitionForAllDevices(const size_t num_device, std::shared_ptr graph); + +Graph::NodeType ApplyStrToTensor(Graph::NodeType Node); + +void InferUndecideStrategy(std::shared_ptr graph); + +void ApplyLastStrategy(const uint64_t node_index, std::shared_ptr graph); + +void ApplyNextStrategy(const uint64_t node_index, std::shared_ptr graph); + +Status DevicesMemoryControl(std::shared_ptr graph); + +size_t GetDataTypeSize(const TensorType &type); +} // namespace parallel +} // namespace mindspore + +#endif // PARALLEL_AUTO_PARALLEL_REC_PARTITION_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_strategy.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_strategy.h new file mode 100644 index 00000000000..4d8ef9c8bdb --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_strategy.h @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_STRATEGY_H_ +#define PARALLEL_AUTO_PARALLEL_REC_STRATEGY_H_ + +namespace mindspore { +namespace parallel { +#define MAX_INPUT_NUM 5 + +struct TensorStr4D { + float str_n = 1; + float str_c = 1; + float str_h = 1; + float str_w = 1; +}; + +struct StrategyRec { + TensorStr4D inputTensor[MAX_INPUT_NUM]; + TensorStr4D outputTensor; + int32_t cut_counter = 0; + double cost = 0; +}; +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_AUTO_PARALLEL_REC_STRATEGY_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_tensor.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_tensor.h new file mode 100644 index 00000000000..51ffca40230 --- /dev/null +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_tensor.h @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_AUTO_PARALLEL_REC_TENSOR_H_ +#define PARALLEL_AUTO_PARALLEL_REC_TENSOR_H_ + +#include "parallel/auto_parallel/rec_core/rec_strategy.h" + +namespace mindspore { +namespace parallel { +enum TensorType { kInt8, kFloat16, kFloat32, kDouble64 }; + +struct Shape4D { + int32_t shape_n = 1; + int32_t shape_c = 1; + int32_t shape_h = 1; + int32_t shape_w = 1; +}; + +struct TensorParam { + TensorType tensor_type = kFloat32; // default as float. + Shape4D tensor_shape; + TensorStr4D tensor_str; +}; +} // namespace parallel +} // namespace mindspore + +#endif // PARALLEL_AUTO_PARALLEL_REC_TENSOR_H_ diff --git a/mindspore/ccsrc/parallel/context.cc b/mindspore/ccsrc/parallel/context.cc new file mode 100644 index 00000000000..74ec948c491 --- /dev/null +++ b/mindspore/ccsrc/parallel/context.cc @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/context.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "parallel/device_manager.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { +std::vector PARALLEL_MODE_LIST = {STAND_ALONE, DATA_PARALLEL, HYBRID_PARALLEL, SEMI_AUTO_PARALLEL, + AUTO_PARALLEL}; +std::vector STRATEGY_SEARCH_MODE_LIST = {DYNAMIC_PROGRAMMING, RECURSIVE_PROGRAMMING}; + +std::shared_ptr ParallelContext::inst_context_ = nullptr; + +std::shared_ptr ParallelContext::GetInstance() { + if (inst_context_ == nullptr) { + inst_context_.reset(new (std::nothrow) ParallelContext()); + } + return inst_context_; +} + +ParallelContext::ParallelContext() { Reset(); } + +void ParallelContext::Reset() { + mirror_mean_ = false; + cast_before_mirror_ = true; + loss_repeated_mean_ = true; + device_num_ = 1; + global_rank_ = 0; + communication_backend_ = HCCL_BACKEND; + device_num_is_set_ = false; + global_rank_is_set_ = false; + parallel_mode_ = STAND_ALONE; + parameter_broadcast_ = false; + parameter_broadcast_is_set_ = false; +} + +void ParallelContext::set_device_num(int32_t device_num) { + device_num_ = device_num; + device_num_is_set_ = true; +} + +void ParallelContext::set_global_rank(int32_t global_rank) { + global_rank_ = global_rank; + global_rank_is_set_ = true; +} + +void ParallelContext::set_mirror_mean(bool mirror_mean) { mirror_mean_ = mirror_mean; } + +void ParallelContext::set_cast_before_mirror(bool cast_before_mirror) { cast_before_mirror_ = cast_before_mirror; } + +void ParallelContext::set_loss_repeated_mean(bool loss_repeated_mean) { loss_repeated_mean_ = loss_repeated_mean; } + +void ParallelContext::set_communication_backend(const std::string& communication_backend) { + communication_backend_ = communication_backend; +} + +bool ParallelContext::set_parallel_mode(const std::string& parallel_mode) { + auto iter = std::find(PARALLEL_MODE_LIST.begin(), PARALLEL_MODE_LIST.end(), parallel_mode); + if (iter == PARALLEL_MODE_LIST.end()) { + MS_LOG(INFO) << "Invalid parallel mode:" << parallel_mode; + return false; + } + parallel_mode_ = parallel_mode; + return true; +} + +bool ParallelContext::set_strategy_search_mode(const std::string& strategy_search_mode) { + auto iter = std::find(STRATEGY_SEARCH_MODE_LIST.begin(), STRATEGY_SEARCH_MODE_LIST.end(), strategy_search_mode); + if (iter == STRATEGY_SEARCH_MODE_LIST.end()) { + MS_LOG(INFO) << "Invalid strategy search mode mode: " << strategy_search_mode; + return false; + } + strategy_search_mode_ = strategy_search_mode; + return true; +} + +void ParallelContext::set_parameter_broadcast(bool parameter_broadcast) { + parameter_broadcast_ = parameter_broadcast; + parameter_broadcast_is_set_ = true; +} + +void ParallelContext::set_all_reduce_fusion_split_indices(const std::vector indices) { + all_reduce_fusion_split_indices_ = indices; +} + +const std::vector ParallelContext::all_reduce_fusion_split_indices() const { + return all_reduce_fusion_split_indices_; +} + +void ParallelContext::set_all_reduce_fusion_split_sizes(const std::vector sizes) { + all_reduce_fusion_split_sizes_ = sizes; +} + +const std::vector ParallelContext::all_reduce_fusion_split_sizes() const { + return all_reduce_fusion_split_sizes_; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/context.h b/mindspore/ccsrc/parallel/context.h new file mode 100644 index 00000000000..22a5f89e382 --- /dev/null +++ b/mindspore/ccsrc/parallel/context.h @@ -0,0 +1,108 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_CONTEXT_H_ +#define MINDSPORE_CCSRC_PARALLEL_CONTEXT_H_ + +#include +#include +#include +#include +#include + +#include "parallel/status.h" +#include "parallel/ops_info/ops_utils.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace parallel { +constexpr char STAND_ALONE[] = "stand_alone"; +constexpr char DATA_PARALLEL[] = "data_parallel"; +constexpr char HYBRID_PARALLEL[] = "hybrid_parallel"; +constexpr char AUTO_PARALLEL[] = "auto_parallel"; +constexpr char SEMI_AUTO_PARALLEL[] = "semi_auto_parallel"; + +constexpr char DYNAMIC_PROGRAMMING[] = "dynamic_programming"; +constexpr char RECURSIVE_PROGRAMMING[] = "recursive_programming"; + +class ParallelContext { + public: + ~ParallelContext() = default; + ParallelContext(const ParallelContext&) = delete; + ParallelContext& operator=(const ParallelContext&) = delete; + + static std::shared_ptr GetInstance(); + + void set_mirror_mean(bool mirror_mean); + bool mirror_mean() const { return mirror_mean_; } + + void set_cast_before_mirror(bool cast_before_mirror); + bool cast_before_mirror() const { return cast_before_mirror_; } + + void set_loss_repeated_mean(bool loss_repeated_mean); + bool loss_repeated_mean() const { return loss_repeated_mean_; } + + void set_device_num(int32_t device_num); + int32_t device_num() const { return device_num_; } + + void set_global_rank(int32_t global_rank); + int32_t global_rank() const { return global_rank_; } + + void set_communication_backend(const std::string& communication_backend); + std::string communication_backend() const { return communication_backend_; } + + bool set_parallel_mode(const std::string& parallel_mode); + std::string parallel_mode() const { return parallel_mode_; } + + bool set_strategy_search_mode(const std::string& strategy_search_mode); + std::string strategy_search_mode() const { return strategy_search_mode_; } + + void set_parameter_broadcast(bool parameter_broadcast); + bool parameter_broadcast() const { return parameter_broadcast_; } + + bool device_num_is_set() const { return device_num_is_set_; } + bool global_rank_is_set() const { return global_rank_is_set_; } + bool parameter_broadcast_is_set() const { return parameter_broadcast_is_set_; } + + void set_all_reduce_fusion_split_indices(const std::vector indices); + const std::vector all_reduce_fusion_split_indices() const; + void set_all_reduce_fusion_split_sizes(const std::vector sizes); + const std::vector all_reduce_fusion_split_sizes() const; + + void Reset(); + + private: + ParallelContext(); + static std::shared_ptr inst_context_; + bool mirror_mean_; + bool cast_before_mirror_; + bool loss_repeated_mean_; + int32_t device_num_; + int32_t global_rank_; + std::string communication_backend_; + std::string parallel_mode_; + std::string strategy_search_mode_; + bool parameter_broadcast_; + bool device_num_is_set_; + bool global_rank_is_set_; + bool parameter_broadcast_is_set_; + std::vector all_reduce_fusion_split_indices_; + std::vector all_reduce_fusion_split_sizes_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_CONTEXT_H_ diff --git a/mindspore/ccsrc/parallel/costmodel_context.cc b/mindspore/ccsrc/parallel/costmodel_context.cc new file mode 100644 index 00000000000..45b158bde56 --- /dev/null +++ b/mindspore/ccsrc/parallel/costmodel_context.cc @@ -0,0 +1,127 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/costmodel_context.h" + +#include + +#include "parallel/auto_parallel/graph_costmodel.h" +#include "parallel/allreduce_fusion/allreduce_fusion.h" + +namespace mindspore { +namespace parallel { +std::shared_ptr CostModelContext::cm_context_inst_ = nullptr; + +std::shared_ptr CostModelContext::GetInstance() { + if (cm_context_inst_ == nullptr) { + MS_LOG(INFO) << "Create costmodel_context"; + cm_context_inst_.reset(new (std::nothrow) CostModelContext()); + } + return cm_context_inst_; +} + +CostModelContext::CostModelContext() { + ResetCostModel(); + ResetAlgoParameters(); +} + +void CostModelContext::ResetCostModel() { + device_memory_capacity_ = DEFAULT_DEVICE_MEMORY_CAPACITY; + costmodel_alpha_ = DEFAULT_COST_MODEL_ALPHA; + costmodel_beta_ = DEFAULT_COST_MODEL_BETA; + costmodel_gamma_ = DEFAULT_COST_MODEL_GAMMA; + costmodel_communi_threshold_ = DEFAULT_COST_MODEL_COMMUNI_THRESHOLD; + costmodel_communi_const_ = DEFAULT_COST_MODEL_COMMUNI_CONST; + costmodel_communi_bias_ = DEFAULT_COST_MODEL_COMMUNI_BIAS; + costmodel_allreduce_fusion_algorithm_ = DEFAULT_COST_MODEL_ALLREDUCE_FUSION_ALGORITHM; + costmodel_allreduce_fusion_times_ = DEFAULT_COST_MODEL_ALLREDUCE_FUSION_TIMES; + costmodel_allreduce_fusion_tail_percent_ = DEFAULT_COST_MODEL_ALLREDUCE_FUSION_TAIL_PERCENT; + costmodel_allreduce_fusion_tail_time_ = DEFAULT_COST_MODEL_ALLREDUCE_FUSION_TAIL_TIME; + costmodel_allreduce_fusion_allreduce_inherent_time_ = DEFAULT_COST_MODEL_ALLREDUCE_FUSION_ALLREDUCE_INHERENT_TIME; + costmodel_allreduce_fusion_allreduce_bandwidth_ = DEFAULT_COST_MODEL_ALLREDUCE_FUSION_ALLREDUCE_BANDWIDTH; + costmodel_allreduce_fusion_computation_time_parameter_ = + DEFAULT_COST_MODEL_ALLREDUCE_FUSION_COMPUTATION_TIME_PARAMETER; +} + +void CostModelContext::ResetAlgoParameters() { + costmodel_simplify_cal_ = DEFAULT_COST_MODEL_SIMPLIFY_CALCULATION; + tensor_slice_alignment_enable_ = DEFAULT_TENSOR_SLICE_ALIGNMENT_ENABLE; + tensor_slice_alignment_size_ = DEFAULT_TENSOR_SLICE_ALIGNMENT_SIZE; + not_fully_use_device_ = DEFAULT_NOT_FULLY_USE_DEVICES; + elementwise_stra_follow_ = DEFAULT_ELEMENTWISE_OP_STRA_FOLLOW; +} + +void CostModelContext::set_device_memory_capacity(double dm_capacity) { device_memory_capacity_ = dm_capacity; } + +void CostModelContext::set_costmodel_alpha(double cm_alpha) { costmodel_alpha_ = cm_alpha; } + +void CostModelContext::set_costmodel_beta(double cm_beta) { costmodel_beta_ = cm_beta; } + +void CostModelContext::set_costmodel_gamma(double cm_gamma) { costmodel_gamma_ = cm_gamma; } + +void CostModelContext::set_costmodel_simplify_cal(bool cm_simplify) { costmodel_simplify_cal_ = cm_simplify; } + +void CostModelContext::set_costmodel_communi_threshold(double cm_communi_th) { + costmodel_communi_threshold_ = cm_communi_th; +} + +void CostModelContext::set_costmodel_communi_const(double cm_communi_const) { + costmodel_communi_const_ = cm_communi_const; +} + +void CostModelContext::set_costmodel_communi_bias(double cm_communi_bias) { costmodel_communi_bias_ = cm_communi_bias; } + +void CostModelContext::set_costmodel_allreduce_fusion_algorithm(int32_t algorithm) { + costmodel_allreduce_fusion_algorithm_ = algorithm; +} + +void CostModelContext::set_costmodel_allreduce_fusion_times(int32_t allreduce_fusion_times) { + costmodel_allreduce_fusion_times_ = allreduce_fusion_times; +} + +void CostModelContext::set_costmodel_allreduce_fusion_tail_percent(double tail_percent) { + costmodel_allreduce_fusion_tail_percent_ = tail_percent; +} + +void CostModelContext::set_costmodel_allreduce_fusion_tail_time(double tail_time) { + costmodel_allreduce_fusion_tail_time_ = tail_time; +} + +void CostModelContext::set_costmodel_allreduce_fusion_allreduce_inherent_time(double allreduce_inherent_time) { + costmodel_allreduce_fusion_allreduce_inherent_time_ = allreduce_inherent_time; +} + +void CostModelContext::set_costmodel_allreduce_fusion_allreduce_bandwidth(double allreduce_bandwidth) { + costmodel_allreduce_fusion_allreduce_bandwidth_ = allreduce_bandwidth; +} + +void CostModelContext::set_costmodel_allreduce_fusion_computation_time_parameter(double computation_time_parameter) { + costmodel_allreduce_fusion_computation_time_parameter_ = computation_time_parameter; +} + +void CostModelContext::set_tensor_slice_alignment_enable(bool ts_align) { tensor_slice_alignment_enable_ = ts_align; } + +void CostModelContext::set_tensor_slice_alignment_size(size_t ts_align_size) { + tensor_slice_alignment_size_ = ts_align_size; +} + +void CostModelContext::set_not_fully_use_device(bool not_fully_use) { not_fully_use_device_ = not_fully_use; } + +void CostModelContext::set_elementwise_stra_follow(bool elementwise_follow) { + elementwise_stra_follow_ = elementwise_follow; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/costmodel_context.h b/mindspore/ccsrc/parallel/costmodel_context.h new file mode 100644 index 00000000000..a808fc556f0 --- /dev/null +++ b/mindspore/ccsrc/parallel/costmodel_context.h @@ -0,0 +1,170 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_COSTMODEL_CONTEXT_H_ +#define MINDSPORE_CCSRC_PARALLEL_COSTMODEL_CONTEXT_H_ + +#include +#include +#include + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +class CostModelContext { + public: + ~CostModelContext() = default; + CostModelContext(const CostModelContext&) = delete; + CostModelContext& operator=(const CostModelContext&) = delete; + void ResetCostModel(); + void ResetAlgoParameters(); + + static std::shared_ptr GetInstance(); + + // DEVICE_MEMORY_CAPACITY + void set_device_memory_capacity(double); + double device_memory_capacity() const { return device_memory_capacity_; } + + // COST_MODEL_ALPHA + void set_costmodel_alpha(double); + double costmodel_alpha() const { return costmodel_alpha_; } + + // COST_MODEL_BETA + void set_costmodel_beta(double); + double costmodel_beta() const { return costmodel_beta_; } + + // COST_MODEL_GAMMA + void set_costmodel_gamma(double); + double costmodel_gamma() const { return costmodel_gamma_; } + + // COST_MODEL_SIMPLIFY_CALCULATION + void set_costmodel_simplify_cal(bool); + bool costmodel_simplify_cal() const { return costmodel_simplify_cal_; } + + // COST_MODEL_COMMUNI_THRESHOLD + void set_costmodel_communi_threshold(double); + double costmodel_communi_threshold() const { return costmodel_communi_threshold_; } + + // COST_MODEL_COMMUNI_CONST + void set_costmodel_communi_const(double); + double costmodel_communi_const() const { return costmodel_communi_const_; } + + // COST_MODEL_COMMUNI_BIAS + void set_costmodel_communi_bias(double); + double costmodel_communi_bias() const { return costmodel_communi_bias_; } + + void set_costmodel_allreduce_fusion_algorithm(int32_t); + int32_t costmodel_allreduce_fusion_algorithm() const { return costmodel_allreduce_fusion_algorithm_; } + + void set_costmodel_allreduce_fusion_times(int32_t); + int32_t costmodel_allreduce_fusion_times() const { return costmodel_allreduce_fusion_times_; } + + void set_costmodel_allreduce_fusion_tail_percent(double); + double costmodel_allreduce_fusion_tail_percent() const { return costmodel_allreduce_fusion_tail_percent_; } + + void set_costmodel_allreduce_fusion_tail_time(double); + double costmodel_allreduce_fusion_tail_time() const { return costmodel_allreduce_fusion_tail_time_; } + + void set_costmodel_allreduce_fusion_allreduce_inherent_time(double); + double costmodel_allreduce_fusion_allreduce_inherent_time() const { + return costmodel_allreduce_fusion_allreduce_inherent_time_; + } + + void set_costmodel_allreduce_fusion_allreduce_bandwidth(double); + double costmodel_allreduce_fusion_allreduce_bandwidth() const { + return costmodel_allreduce_fusion_allreduce_bandwidth_; + } + + void set_costmodel_allreduce_fusion_computation_time_parameter(double); + double costmodel_allreduce_fusion_computation_time_parameter() const { + return costmodel_allreduce_fusion_computation_time_parameter_; + } + + // TENSOR_SLICE_ALIGNMENT_ENABLE + void set_tensor_slice_alignment_enable(bool); + bool tensor_slice_alignment_enable() const { return tensor_slice_alignment_enable_; } + + // TENSOR_SLICE_ALIGNMENT_SIZE + void set_tensor_slice_alignment_size(size_t); + size_t tensor_slice_alignment_size() const { return tensor_slice_alignment_size_; } + + // NOT_FULLY_USE_DEVICES + void set_not_fully_use_device(bool); + bool not_fully_use_device() const { return not_fully_use_device_; } + + // ELEMENTWISE_OP_STRA_FOLLOW + void set_elementwise_stra_follow(bool); + bool elementwise_stra_follow() const { return elementwise_stra_follow_; } + + private: + CostModelContext(); + static std::shared_ptr cm_context_inst_; + + // DEVICE_MEMORY_CAPACITY + double device_memory_capacity_; + + // COST_MODEL_ALPHA + double costmodel_alpha_; + + // COST_MODEL_BETA + double costmodel_beta_; + + // COST_MODEL_GAMMA + double costmodel_gamma_; + + // COST_MODEL_SIMPLIFY_CALCULATION + bool costmodel_simplify_cal_; + + // COST_MODEL_COMMUNI_THRESHOLD + double costmodel_communi_threshold_; + + // COST_MODEL_COMMUNI_CONST + double costmodel_communi_const_; + + // COST_MODEL_COMMUNI_BIAS + double costmodel_communi_bias_; + + int32_t costmodel_allreduce_fusion_algorithm_; + + int32_t costmodel_allreduce_fusion_times_; + + double costmodel_allreduce_fusion_tail_percent_; + + double costmodel_allreduce_fusion_tail_time_; + + double costmodel_allreduce_fusion_allreduce_inherent_time_; + + double costmodel_allreduce_fusion_allreduce_bandwidth_; + + double costmodel_allreduce_fusion_computation_time_parameter_; + + // TENSOR_SLICE_ALIGNMENT_ENABLE + bool tensor_slice_alignment_enable_; + + // TENSOR_SLICE_ALIGNMENT_SIZE + size_t tensor_slice_alignment_size_; + + // NOT_FULLY_USE_DEVICES + bool not_fully_use_device_; + + // ELEMENTWISE_OP_STRA_FOLLOW + bool elementwise_stra_follow_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_COSTMODEL_CONTEXT_H_ diff --git a/mindspore/ccsrc/parallel/device.h b/mindspore/ccsrc/parallel/device.h new file mode 100644 index 00000000000..0373617ad59 --- /dev/null +++ b/mindspore/ccsrc/parallel/device.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_DEVICE_H_ +#define MINDSPORE_CCSRC_PARALLEL_DEVICE_H_ + +#include +#include +#include +#include + +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { +class Device { + // This class abstract the 'device' information, used in Parallel module. + public: + Device() : rank_(0) { name_.clear(); } + explicit Device(int32_t rank) : rank_(rank) { name_.clear(); } + Device(std::string name, int32_t rank) : name_(std::move(name)), rank_(rank) {} + ~Device() = default; + std::string name() const { return name_; } + int32_t rank() const { return rank_; } + + private: + std::string name_; + int32_t rank_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_DEVICE_H_ diff --git a/mindspore/ccsrc/parallel/device_manager.cc b/mindspore/ccsrc/parallel/device_manager.cc new file mode 100644 index 00000000000..999d6b44325 --- /dev/null +++ b/mindspore/ccsrc/parallel/device_manager.cc @@ -0,0 +1,374 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/device_manager.h" + +#include +#include +#include +#include +#include +#include + +#include "parallel/step_parallel.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +DeviceManagerPtr g_device_manager = nullptr; + +Stage::Stage(const std::list& devices, int num, int rank) + : devices_(devices), number_(num), rank_(rank) { + gm_ = GroupManager(); +} + +// NOTE: '-1' indicates ERROR +int Stage::global_rank(Group* g) const { return ((g == nullptr) ? rank_ : -1); } + +bool InitDevice(int32_t device_num, int32_t global_rank, const std::string& backend) { + if (device_num <= 0) { + MS_LOG(ERROR) << "'device_num' must be positive."; + return false; + } + if (global_rank < 0) { + MS_LOG(ERROR) << "'global_rank' must be nonnegative."; + return false; + } + if (device_num > MAX_DEVICE_NUM) { + MS_LOG(ERROR) << "'device_num' must be no more than " << MAX_DEVICE_NUM << "."; + return false; + } + // 'device_num_converted' must be the power of 2 + if ((IntToUint(device_num) & IntToUint(device_num - 1)) != 0) { + MS_LOG(ERROR) << "'device_num' must be the power of 2."; + return false; + } + if (global_rank >= device_num) { + MS_LOG(ERROR) << "'global_rank' must be less than 'device_num'."; + return false; + } + if ((backend != HCCL_BACKEND) && (backend != NCCL_BACKEND) && (backend != UNDEFINED_BACKEND)) { + MS_LOG(ERROR) << "Invalid backend: " << backend; + return false; + } + + RankList devices, stage_map; + for (int i = 0; i < device_num; ++i) { + devices.push_back(i); + } + + stage_map.push_back(device_num); + g_device_manager = std::make_shared(); + if (g_device_manager->Init(devices, global_rank, stage_map, backend) == SUCCESS) { + MS_LOG(INFO) << "Device initialization succeeds."; + return true; + } else { + MS_LOG(ERROR) << "Device initialization fails."; + return false; + } +} + +void CheckGlobalDeviceManager() { + if (g_device_manager == nullptr) { + MS_LOG(EXCEPTION) << "Device information has not been set!"; + } +} + +int32_t GetListMemberByIndex(size_t index, const RankList& devices) { + size_t i = 0; + int32_t result = 0; + if ((devices.empty()) || (index >= devices.size())) { + MS_LOG(EXCEPTION) << "Index is out of the list scope"; + } + auto it = devices.begin(); + for (; it != devices.end(); ++it) { + if (i == index) { + result = *it; + break; + } + ++i; + } + return result; +} + +std::shared_ptr GetListMemberByIndex(size_t index, const std::list>& device_list) { + size_t i = 0; + std::shared_ptr result; + if ((device_list.empty()) || (index >= device_list.size())) { + MS_LOG(EXCEPTION) << "Index is out of the list scope"; + } + auto it = device_list.begin(); + for (; it != device_list.end(); ++it) { + if (i == index) { + result = *it; + break; + } + ++i; + } + return result; +} + +// E.g. devices = [4, 5, 2, 1, 7, 8, 10], stage_map = [4, 3], +// therefore the stage_devices_ = [[4, 5, 2, 1], [7, 8, 10]]. +Status DeviceManager::Init(const RankList& devices, int32_t global_device_rank, const RankList& stage_map, + const std::string& backend) { + auto dev_it = devices.begin(); + auto stage_it = stage_map.begin(); + int32_t sum = 0; + + if ((backend != HCCL_BACKEND) && (backend != NCCL_BACKEND) && (backend != UNDEFINED_BACKEND)) { + MS_LOG(ERROR) << "Invalid backend: " << backend; + return Status::FAILED; + } + + for (; stage_it != stage_map.end(); ++stage_it) { + sum += (*stage_it); + } + if (IntToSize(sum) != devices.size()) { + MS_LOG(ERROR) << "The number of 'devices' in the list is not equal to the mentioned " + << "size of 'stage_map'"; + return Status::FAILED; + } + + for (; dev_it != devices.end(); ++dev_it) { + std::shared_ptr one = std::make_shared(*dev_it); + devices_.push_back(one); + } + + size_t global_index = 0; + for (stage_it = stage_map.begin(); stage_it != stage_map.end(); ++stage_it) { + int num_device = *stage_it; + if (num_device > MAX_DEVICE_NUM) { + MS_LOG(ERROR) << "The number of 'devices' in a stage must not be greater than " << MAX_DEVICE_NUM; + return Status::FAILED; + } + if (num_device <= 0) { + MS_LOG(ERROR) << "The number of 'devices' in a stage must be positive"; + return Status::FAILED; + } + RankList curr_dev_list; + for (int i = 0; i < num_device; ++i) { + curr_dev_list.push_back(GetListMemberByIndex(global_index, devices)); + global_index++; + } + stage_devices_.push_back(curr_dev_list); + } + + global_index = 0; + for (stage_it = stage_map.begin(); stage_it != stage_map.end(); ++stage_it) { + int num_device = *stage_it; + if (num_device > MAX_DEVICE_NUM) { + MS_LOG(ERROR) << "The number of 'devices' in a stage must be less than " << MAX_DEVICE_NUM; + return Status::FAILED; + } + if (num_device <= 0) { + MS_LOG(ERROR) << "The number of 'devices' in a stage must be positive"; + return Status::FAILED; + } + std::list curr_dev_list; + for (int i = 0; i < num_device; ++i) { + curr_dev_list.push_back(*GetListMemberByIndex(global_index, devices_)); + global_index++; + } + std::shared_ptr new_stage = std::make_shared(curr_dev_list); + stages_.push_back(new_stage); + } + + std::shared_ptr dev = std::make_shared(global_device_rank); + device_ = dev; + set_global_rank(global_device_rank); + backend_ = backend; + + if (backend == HCCL_BACKEND) { + gm_.set_world_group(HCCL_WORLD_GROUP); + } else if (backend_ == NCCL_BACKEND) { + gm_.set_world_group(NCCL_WORLD_GROUP); + } else { + gm_.set_world_group(UNDEFINED_WORLD_GROUP); + } + MS_LOG(INFO) << "The device num: " << devices.size() << "rank id: " << global_device_rank + << "the backend: " << backend; + return Status::SUCCESS; +} + +std::shared_ptr DeviceManager::GetStageById(int32_t stage_id) { + std::shared_ptr res; + if (IntToSize(stage_id) >= stages_.size()) { + MS_LOG(ERROR) << "the 'stage_id': " << stage_id << ", is out of the scope of 'stage_devices_': " << stages_.size(); + return res; + } + int32_t index = 0; + for (auto& stage : stages_) { + if (index == stage_id) return stage; + index++; + } + return res; +} + +RankList DeviceManager::GetDeviceListByStageId(int32_t stage_id) const { + if (IntToSize(stage_id) >= stage_devices_.size()) + MS_LOG(ERROR) << "the 'stage_id': " << stage_id + << ", is out of the scope of 'stage_devices_': " << stage_devices_.size(); + RankList res; + int32_t index = 0; + for (auto& stage : stage_devices_) { + if (index == stage_id) { + return stage; + } + index++; + } + return res; +} + +RankList DeviceManager::global_device_list(int32_t stage_id, int32_t rank, int32_t split_num) const { + RankList res; + if (split_num <= 0) { + return res; + } + if (IntToSize(stage_id) >= stage_devices_.size()) { + MS_LOG(ERROR) << "the 'stage_id': " << stage_id + << ", is out of the scope of 'stage_devices_': " << stage_devices_.size(); + return res; + } + + RankList global_list = GetDeviceListByStageId(stage_id); + if (global_list.size() % IntToSize(split_num)) { + MS_LOG(ERROR) << "dev list size(" << global_list.size() << ") can not be divisible by split num: " << stage_id; + return res; + } + + std::vector dev_list; + (void)std::copy(global_list.begin(), global_list.end(), std::back_inserter(dev_list)); + + size_t index = 0; + size_t slice_size = dev_list.size() / IntToSize(split_num); + for (int32_t i = 0; i < split_num; ++i) { + bool found = false; + index = slice_size * IntToSize(i); + for (size_t j = 0; j < slice_size; ++j) { + if (dev_list[index + j] == rank) { + found = true; + break; + } + } + + if (found) { + break; + } + } + + for (size_t k = 0; k < slice_size; ++k) { + res.push_back(dev_list[index + k]); + } + return res; +} + +Device DeviceManager::CreateNewDeviceByRank(int32_t rank) const { return Device(rank); } + +std::list DeviceManager::CreateDeviceListByRankList(RankList ranks) { + std::list dev_list; + for (auto& rank : ranks) { + Device one = CreateNewDeviceByRank(rank); + dev_list.push_back(one); + } + return dev_list; +} + +DeviceManager& DeviceManager::GetInstance() { + static DeviceManager instance = DeviceManager(); + return instance; +} + +std::string DeviceManager::FindRankListNameByHashName(const std::string& hash_name) { + std::string tmp = "WORLD_GROUP"; + if ((hash_name == HCCL_WORLD_GROUP) || (hash_name == NCCL_WORLD_GROUP)) { + return tmp; + } + auto iter = group_to_rank_.find(hash_name); + if (iter == group_to_rank_.end()) { + MS_LOG(WARNING) << "Can not find the rank list name by hash name: " << hash_name; + return tmp; + } + return iter->second; +} + +std::string HashName(const std::string& origin_name) { return std::to_string(std::hash{}(origin_name)); } + +// Group name is generated using the increasing ranks of the devices. +// E.g. the devices' ranks are '<0, 5, 3, 7, 1>', and the generated group name +// is '0-1-3-5-7'. +std::string DeviceManager::GenerateGroupNameByRanks(RankList ranks) { + std::string rank_list_name; + std::list::iterator it; + ranks.sort(); // sorted in increasing order + for (it = ranks.begin(); it != ranks.end(); ++it) { + if (it == ranks.begin()) { + rank_list_name = std::to_string(*it); + } else { + rank_list_name += "-" + std::to_string(*it); + } + } + + // hash rank-list-name and add ranks' size as prefix + std::string group_hash_name = HashName(rank_list_name); + std::string group_name = std::to_string(ranks.size()) + "-" + group_hash_name; + + if (rank_to_group_.find(rank_list_name) == rank_to_group_.end()) { + if (group_to_rank_.find(group_name) == group_to_rank_.end()) { + rank_to_group_[rank_list_name] = group_name; + group_to_rank_[group_name] = rank_list_name; + MS_LOG(INFO) << "The rank list name is " << rank_list_name << "nd group name is " << group_name; + } else { + MS_LOG(EXCEPTION) << "Hash collision, the current rank list: " << rank_list_name + << "the old rank list:" << group_to_rank_.find(group_name)->second + << "the group name: " << group_name; + } + } + return group_name; +} + +// Create the group with the given devices and the given name. The GroupManager +// gm_ will create a new group only if there does not exit a group with the same +// name. Otherwise, let the pointer g point to that group. +Group DeviceManager::CreateGroup(const std::string& group_name, const std::list& devices) { + if ((world_group() == NCCL_WORLD_GROUP) && (devices.size() != devices_.size())) { + MS_LOG(EXCEPTION) << "Do not support sub group for nccl"; + } + Group g; + (void)gm_.CreateGroup(group_name, devices, &g); + return g; +} + +// Create the group with only the given devices' ranks. +Group DeviceManager::CreateGroup(const RankList& dev_ranks) { + std::unordered_set rank_set(dev_ranks.begin(), dev_ranks.end()); + if (dev_ranks.size() != rank_set.size()) { + MS_LOG(EXCEPTION) << "Invalid dev ranks(" << dev_ranks << "), it has the Duplicate elements in list"; + } + + std::string group_name = GenerateGroupNameByRanks(dev_ranks); + std::list dev_list = CreateDeviceListByRankList(dev_ranks); + return CreateGroup(group_name, dev_list); +} + +void DeviceManager::Clear() { + devices_.clear(); + stage_devices_.clear(); + gm_.Clear(); +} + +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/device_manager.h b/mindspore/ccsrc/parallel/device_manager.h new file mode 100644 index 00000000000..80beb15f862 --- /dev/null +++ b/mindspore/ccsrc/parallel/device_manager.h @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_DEVICE_MANAGER_H_ +#define MINDSPORE_CCSRC_PARALLEL_DEVICE_MANAGER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/convert_utils.h" +#include "common/utils.h" +#include "parallel/device.h" +#include "parallel/status.h" +#include "parallel/group_manager.h" +#include "parallel/strategy.h" +#include "parallel/device_matrix.h" + +namespace mindspore { +namespace parallel { +#define MAX_DEVICE_NUM 1024 + +constexpr char HCCL_BACKEND[] = "hccl"; +constexpr char NCCL_BACKEND[] = "nccl"; +constexpr char UNDEFINED_BACKEND[] = "undefined_backend"; + +class DeviceManager; +using DeviceManagerPtr = std::shared_ptr; +// 'g_device_manager' is the globally unique manager to manage the devices. +extern DeviceManagerPtr g_device_manager; + +class Stage { + // This class is used in pipeline-parallelization. Available devices are partitioned into multiple stages. + // Currently, the function of pipeline-parallelization and this class are NOT implemented. + public: + explicit Stage(std::list devices) : devices_(std::move(devices)), number_(0), rank_(0) { + gm_ = GroupManager(); + } + Stage(const std::list& devices, int num, int rank); + ~Stage() = default; + + int GetStageNum() const { return number_; } + size_t GetDevicesNum() const { return devices_.size(); } + std::list GetDevicesList() { return devices_; } + int global_rank(Group* g) const; + + private: + std::list devices_; + int number_; + int32_t rank_; + GroupManager gm_; +}; + +// This method is used for initializing the global DeviceManager 'g_device_manager', +// arguments including 'device_num' and 'global_rank' +bool InitDevice(int32_t device_num, int32_t global_rank, const std::string& backend); + +void CheckGlobalDeviceManager(); + +std::string HashName(const std::string& rank_list_name); + +class DeviceManager { + // This class is used to manage the abstract devices, including group-related and stage-related management. + public: + DeviceManager() : local_rank_(0), global_rank_(0), stage_num_(0) { gm_ = GroupManager(); } + ~DeviceManager() = default; + + Status Init(const RankList& devices, int32_t local_device, const RankList& stage_map, const std::string& backend); + + static DeviceManager& GetInstance(); + RankList GetDeviceListByStageId(int32_t stage_id) const; + RankList global_device_list(int32_t stage_id, int32_t rank, int32_t split_num) const; + + Device CreateNewDeviceByRank(int32_t rank) const; + std::list CreateDeviceListByRankList(RankList ranks); + + std::string GenerateGroupNameByRanks(RankList dev_ranks); + Group CreateGroup(const std::string& group_name, const std::list& devices); + Group CreateGroup(const RankList& dev_ranks); + std::shared_ptr GetStageById(int32_t stage_id); + + size_t DeviceNum() const { return devices_.size(); } + + int32_t GetStageNum() const { return static_cast(stage_devices_.size()); } + + int32_t global_rank() const { return global_rank_; } + std::string backend() const { return backend_; } + void set_global_rank(int32_t global_rank) { global_rank_ = global_rank; } + void Clear(); + std::string world_group() const { return gm_.world_group(); } + std::string FindRankListNameByHashName(const std::string& hash_name); + + private: + std::list> devices_; + // each stage has a list of devices + std::list> stage_devices_; + std::shared_ptr device_; + std::list> stages_; + GroupManager gm_; + std::string backend_; + + // bimap: + std::map rank_to_group_; // the key is rank list, value is hash name + std::map group_to_rank_; // the key is hash name, value is rank list + + int32_t local_rank_; + int32_t global_rank_; + int32_t stage_num_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_DEVICE_MANAGER_H_ diff --git a/mindspore/ccsrc/parallel/device_matrix.cc b/mindspore/ccsrc/parallel/device_matrix.cc new file mode 100644 index 00000000000..5a7aa36c8e6 --- /dev/null +++ b/mindspore/ccsrc/parallel/device_matrix.cc @@ -0,0 +1,171 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/device_matrix.h" + +#include +#include +#include +#include +#include +#include + +#include "parallel/status.h" +#include "parallel/ops_info/operator_info.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { + +DeviceMatrix::DeviceMatrix(int32_t rank, RankList dev_list, Shape dev_shape) + : rank_(rank), dev_list_(std::move(dev_list)), dev_shape_(std::move(dev_shape)) { + if (!std::any_of(dev_list_.begin(), dev_list_.end(), [rank](int32_t a) { return a == rank; })) { + MS_LOG(EXCEPTION) << "Rank " << rank << " is not in the current stage!"; + } + int32_t total = std::accumulate(dev_shape_.begin(), dev_shape_.end(), 1, std::multiplies()); + if (IntToSize(total) != dev_list_.size()) { + MS_LOG(EXCEPTION) << "Device shape does not match the size of the device list!"; + } +} + +Status DeviceMatrix::CreateGroupList() { + size_t size = dev_shape_.size(); + RankList group; + for (size_t i = 0; i < size; i++) { + Status status = GetDevicesAlongDim(SizeToUint(i), &group); + group_list_.push_back(group); + if (status == Status::FAILED) { + return Status::FAILED; + } + } + return Status::SUCCESS; +} + +Status DeviceMatrix::GetDevicesAlongDim(const uint32_t& dim, RankList* devices) { + if (dim >= dev_shape_.size()) { + MS_LOG(EXCEPTION) << "The dimension " << dim << " is out of the size of the device shape!"; + } + if (dev_shape_[dim] == 1) { + *devices = {rank_}; + return Status::SUCCESS; + } + + RankList group; + std::list local_group_list; + + // lower than dim + int32_t step = 1; + for (uint32_t i = dim + 1; i < dev_shape_.size(); i++) { + step = step * dev_shape_[i]; + } + int32_t num = *dev_list_.begin(); + for (int32_t i = 0; i < dev_shape_[dim]; i++) { + group.push_back(num); + num += step; + } + + for (int32_t i = 0; i < step; i++) { + local_group_list.push_back(group); + (void)std::for_each(group.begin(), group.end(), [](int32_t& a) { a++; }); + } + + // higher than dim + step = step * dev_shape_[dim]; + int32_t len = SizeToInt(dev_list_.size()) / step; + + // search rank + int32_t target = rank_; + for (int32_t i = 0; i < len; i++) { + for (RankList& temp : local_group_list) { + if (std::any_of(temp.begin(), temp.end(), [target](int32_t a) { return a == target; })) { + *devices = temp; + return Status::SUCCESS; + } + (void)std::for_each(temp.begin(), temp.end(), [step](int32_t& a) { a = a + step; }); + } + } + MS_LOG(ERROR) << "Can't find groups for rank" << rank_ << " in device list!"; + return Status::FAILED; +} + +Shape ConvertRankToCoordinate(int32_t rank, const Shape& dev_shape) { + Shape dev_coordinate; + for (size_t i = 0; i < dev_shape.size(); ++i) { + int32_t size = dev_shape[dev_shape.size() - i - 1]; + if (size == 0) { + MS_LOG(EXCEPTION) << "Invalid dev shape: " << ShapeToString(dev_shape); + } else { + int32_t index = rank % size; + (void)dev_coordinate.insert(dev_coordinate.begin(), index); + rank = rank / size; + } + } + return dev_coordinate; +} + +Status DeviceMatrix::GetDevicesByTensorMap(const Shape& tensor_map, RankList* rank_list) { + for (auto& element : tensor_map) { + // -1 means the corresponding dimension is not split. + if (element == MAP_NONE) { + continue; + } else if ((element < 0) || (IntToSize(element) >= dev_shape_.size())) { + MS_LOG(ERROR) << "create group by tensor map: the tensor map is invalid"; + return FAILED; + } + } + + Shape current_rank_coordinate = ConvertRankToCoordinate(rank_, dev_shape_); + for (auto& tmp_rank : dev_list_) { + Shape tmp_rank_coordinate = ConvertRankToCoordinate(tmp_rank, dev_shape_); + bool matched = true; + for (auto& map : tensor_map) { + if (map == MAP_NONE) { + continue; + } + size_t index = dev_shape_.size() - IntToSize(map) - 1; + if (current_rank_coordinate[index] != tmp_rank_coordinate[index]) { + matched = false; + break; + } + } + if (matched) { + rank_list->push_back(tmp_rank); + } + } + + return SUCCESS; +} + +std::string ShapeToString(const Shape& shape) { + std::string str = "["; + for (size_t i = 0; i < shape.size(); ++i) { + str += std::to_string(shape[i]); + if (i < shape.size() - 1) { + str += ", "; + } + } + return str + "]"; +} + +std::string ListToString(const std::list& list) { + std::string str = "["; + for (auto& element : list) { + str += std::to_string(element) + ", "; + } + return str + "]"; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/device_matrix.h b/mindspore/ccsrc/parallel/device_matrix.h new file mode 100644 index 00000000000..d3b0fdcb491 --- /dev/null +++ b/mindspore/ccsrc/parallel/device_matrix.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_DEVICE_MATRIX_H_ +#define MINDSPORE_CCSRC_PARALLEL_DEVICE_MATRIX_H_ + +#include +#include +#include +#include + +#include "parallel/status.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace parallel { + +using RankList = std::list; +using Shape = std::vector; + +class DeviceMatrix { + public: + DeviceMatrix(int32_t rank, RankList devices, Shape dev_shape); + DeviceMatrix() = default; + ~DeviceMatrix() = default; + std::list group_list() const { return group_list_; } + Status CreateGroupList(); + Status GetDevicesByTensorMap(const Shape& tensor_map, RankList* rank_list); + Status GetDevicesAlongDim(const uint32_t& dim, RankList* devices); + + private: + int32_t rank_ = -1; + RankList dev_list_; + // From low dim to high dim. eg: [D0 D1 D2 D3] + Shape dev_shape_; + std::list group_list_; +}; + +std::string ShapeToString(const Shape& shape); +std::string ListToString(const std::list& list); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_DEVICE_MATRIX_H_ diff --git a/mindspore/ccsrc/parallel/dynamic_creator.h b/mindspore/ccsrc/parallel/dynamic_creator.h new file mode 100644 index 00000000000..59b8722435e --- /dev/null +++ b/mindspore/ccsrc/parallel/dynamic_creator.h @@ -0,0 +1,129 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_DYNAMIC_CREATOR_H_ +#define MINDSPORE_CCSRC_PARALLEL_DYNAMIC_CREATOR_H_ + +#include +#include +#include +#include + +#include "parallel/ops_info/ops_info_head_files.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +#define REGISTER(className) \ + OperatorInfoPtr objectCreator##className(std::string name, Shapes in, Shapes out, PrimitiveAttrs& attrs) { \ + return std::make_shared(name, in, out, attrs); \ + } \ + RegisterAction className##Register(#className, (CreatFn)objectCreator##className); + +typedef OperatorInfoPtr (*CreatFn)(const std::string& name, const Shapes& shape_in, const Shapes shape_out, + const PrimitiveAttrs& attrs); + +class DynCreator { + public: + ~DynCreator() = default; + + // creat static singleton dyn_creator instance + static DynCreator& Instance() { + static DynCreator fac = DynCreator(); + return fac; + } + // register + void Regist(std::string name, CreatFn func) { (void)Function_map_.insert(std::make_pair(name, func)); } + // creator + OperatorInfoPtr Creat(const std::string& name, const Shapes& shape_in, const Shapes& shape_out, + const PrimitiveAttrs& attrs, size_t count) { + std::string op_name = name + std::to_string(count); + auto iter = Function_map_.find(name); + if (iter == Function_map_.end()) { + MS_LOG(INFO) << name << " is not register yet"; + return nullptr; + } + return iter->second(op_name, shape_in, shape_out, attrs); + } + + private: + DynCreator() = default; + std::map Function_map_; +}; + +class RegisterAction { + public: + RegisterAction(const std::string& name, CreatFn creatfn) : name_(name) { + DynCreator::Instance().Regist(name, creatfn); + } + ~RegisterAction() = default; + + private: + std::string name_; +}; + +// operator register +REGISTER(MatMulInfo); +REGISTER(GeluInfo); +REGISTER(VirtualDatasetInfo); +REGISTER(BatchParallelInfo); +REGISTER(TanhInfo); +REGISTER(SoftmaxInfo); +REGISTER(LogSoftmaxInfo); +REGISTER(ActivationInfo); +REGISTER(SoftmaxCrossEntropyWithLogitsInfo); +REGISTER(SubInfo); +REGISTER(TensorAddInfo); +REGISTER(BiasAddInfo); +REGISTER(MulInfo); +REGISTER(DivInfo); +REGISTER(RealDivInfo); +REGISTER(PowInfo); +REGISTER(ExpInfo); +REGISTER(OneHotInfo); +REGISTER(EqualInfo); +REGISTER(NotEqualInfo); +REGISTER(LogInfo); +REGISTER(CosInfo); +REGISTER(ACosInfo); +REGISTER(LogicalNotInfo); +REGISTER(L2NormalizeInfo); +REGISTER(ReduceMaxInfo); +REGISTER(ArgMaxWithValueInfo); +REGISTER(ArgMinWithValueInfo); +REGISTER(ReduceMeanInfo); +REGISTER(ReduceSumInfo); +REGISTER(ReduceMinInfo); +REGISTER(TransposeInfo); +REGISTER(PReLUInfo); +REGISTER(DropoutDoMaskInfo); +REGISTER(DropoutGenMaskInfo) +REGISTER(ReshapeInfo); +REGISTER(FloorDivInfo); +REGISTER(MaximumInfo); +REGISTER(CastInfo); +REGISTER(GreaterInfo); +REGISTER(SparseSoftmaxCrossEntropyWithLogitsInfo); +REGISTER(AssignSubInfo); +REGISTER(ReLUInfo); +REGISTER(GatherV2Info); +REGISTER(SqrtInfo); +REGISTER(GetNextInfo); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_DYNAMIC_CREATOR_H_ diff --git a/mindspore/ccsrc/parallel/graph_util/generate_graph.cc b/mindspore/ccsrc/parallel/graph_util/generate_graph.cc new file mode 100644 index 00000000000..d0548ec6475 --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/generate_graph.cc @@ -0,0 +1,171 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/graph_util/generate_graph.h" + +#include +#include +#include +#include +#include + +using mindspore::tensor::Tensor; + +namespace mindspore { +namespace parallel { +std::string GetOpPythonPath(const OperatorName& op_name) { + // almost all ops are defined in two main paths + const std::string ops_module = OP_PATH; + py::module mod = py::module::import(common::SafeCStr(ops_module)); + if (!py::hasattr(mod, common::SafeCStr(op_name))) { + MS_LOG(EXCEPTION) << ops_module << " don't have op:" << op_name; + } + return ops_module; +} + +ValuePtr CreatOpInstance(const OperatorAttrs& attrs, const OperatorName& op_name, const std::string& instance_name) { + std::string op_path = GetOpPythonPath(op_name); + py::module mod = py::module::import(common::SafeCStr(op_path)); + if (!py::hasattr(mod, common::SafeCStr(op_name))) { + MS_LOG(ERROR) << "Failure: op_path:" << op_path << " don't have attr " << op_name; + return nullptr; + } + std::vector arg_list; + (void)std::transform(attrs.begin(), attrs.end(), std::back_inserter(arg_list), + [](const Attr& attr) { return ValuePtrToPyData(attr.second); }); + py::object obj = + parse::python_adapter::CallPyFn(GET_OP_FUNCTION_PATH, GET_OP_FUNCTION, op_name, op_path, instance_name, arg_list); + ValuePtr op_instance = nullptr; + bool succ = parse::ConvertData(obj, &op_instance); + if (!succ) { + MS_LOG(ERROR) << "Failure:get Python op " << op_path << " from " << op_name << " fail"; + return nullptr; + } + return op_instance; +} + +AnfNodePtr ValuePtrToAnfNodePtr(const ValuePtr& value_ptr) { + auto value_node = NewValueNode(value_ptr); + MS_EXCEPTION_IF_NULL(value_node); + return value_node->cast(); +} + +static std::unordered_map int_tensor_map = {}; +AnfNodePtr CreateInt32Tensor(int32_t value) { + auto it = int_tensor_map.find(value); + if (it != int_tensor_map.end()) { + return it->second; + } + mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(py::int_(value), kInt32); + ValuePtr value_ptr = MakeValue(tensor_ptr); + auto anf_node_ptr = ValuePtrToAnfNodePtr(value_ptr); + int_tensor_map[value] = anf_node_ptr; + return anf_node_ptr; +} + +AnfNodePtr CreatTypeInt(int32_t value) { + ValuePtr value_ptr = MakeValue(std::make_shared(value)); + return ValuePtrToAnfNodePtr(value_ptr); +} + +AnfNodePtr CreatInt32Imm(int32_t value) { + ValuePtr value_ptr = MakeValue(std::make_shared(value)); + return ValuePtrToAnfNodePtr(value_ptr); +} + +std::string GetInstanceNameByCNode(const CNodePtr& cnode) { + PrimitivePtr prim = GetValueNode(cnode->input(0)); + if (!prim) { + MS_LOG(EXCEPTION) << "The first input of the cnode is not a PrimitivePtr."; + } + std::string instance_name = prim->instance_name(); + return HashInstanceName(instance_name); +} + +std::string HashInstanceName(const std::string& name) { + auto using_hash_name = common::GetEnv(USING_HASH_NAME); + std::string instance_name; + if ((using_hash_name.empty()) || (using_hash_name == "on")) { + instance_name = HashName(name); + } else { + instance_name = name; + } + return instance_name; +} + +Status GenerateGraph::Init(const CNodePtr& cnode) { + if (!cnode) { + MS_LOG(ERROR) << "Init:cnode is nullptr"; + return FAILED; + } + cnode_ = cnode; + func_graph_ = cnode->func_graph(); + if (!func_graph_) { + MS_LOG(ERROR) << "Init:func_graph_ is nullptr"; + return FAILED; + } + manager_ = func_graph_->manager(); + if (!manager_) { + MS_LOG(ERROR) << "Init:manager_ is nullptr"; + return FAILED; + } + scope_ = cnode_->scope(); + if (!scope_) { + MS_LOG(ERROR) << "Init:scope_ is nullptr"; + return FAILED; + } + virtual_input_node_ = std::make_shared(nullptr); + virtual_input_node_->set_scope(scope_); + instance_name_base_ = GetInstanceNameByCNode(cnode_); + name_idx_ = 0; + return SUCCESS; +} + +AnfNodePtr GenerateGraph::PushBack(const std::vector& inputs) { + CNodePtr cnode = func_graph_->NewCNode(inputs); // using NewCNode to creat anfnode + MS_EXCEPTION_IF_NULL(cnode); + cnode->set_scope(scope_); + if (inputs.size() < 2) { + MS_LOG(EXCEPTION) << "inputs.size() must be more than 1"; + } + (void)manager_->Replace(inputs.at(1), cnode); // using Replace function to insert cnode after inputs[0] + auto new_anf_node_ptr = cnode->cast(); + MS_EXCEPTION_IF_NULL(new_anf_node_ptr); + return new_anf_node_ptr; +} + +AnfNodePtr GenerateGraph::NewOpInst(const OperatorName& op_name, const OperatorAttrs& attrs) { + name_idx_++; + ValuePtr pyop_instance = CreatOpInstance(attrs, op_name, instance_name_base_ + op_name + std::to_string(name_idx_)); + if (pyop_instance == nullptr) { + MS_LOG(EXCEPTION) << "Failure:" << op_name << " CreatOpInstance failed"; + } + auto value_node = NewValueNode(pyop_instance); + return value_node->cast(); +} + +AnfNodePtr GenerateGraph::NewOpInst(const OperatorName& op_name) { + name_idx_++; + OperatorAttrs attrs; + ValuePtr pyop_instance = CreatOpInstance(attrs, op_name, instance_name_base_ + std::to_string(name_idx_)); + if (pyop_instance == nullptr) { + MS_LOG(EXCEPTION) << "Failure:" << op_name << " CreatOpInstance failed"; + } + auto value_node = NewValueNode(pyop_instance); + return value_node->cast(); +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/graph_util/generate_graph.h b/mindspore/ccsrc/parallel/graph_util/generate_graph.h new file mode 100644 index 00000000000..713c13dfb5e --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/generate_graph.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GENERATE_GRAPH_H_ +#define MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GENERATE_GRAPH_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "./common.h" +#include "optimizer/opt.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +#define USING_HASH_NAME "USING_HASH_NAME" +// Get the operator's path where the operator has be defined +std::string GetOpPythonPath(const OperatorName& op_name); + +// Init python operator Instance +ValuePtr CreatOpInstance(const OperatorAttrs& attrs, const OperatorName& op_name, const std::string& instance_name); + +AnfNodePtr CreatTypeInt(int32_t value); +AnfNodePtr CreatInt32Imm(int32_t value); +AnfNodePtr CreateInt32Tensor(int32_t value); +std::string HashInstanceName(const std::string& name); + +class GenerateGraph { + public: + GenerateGraph() : name_idx_(0) {} + Status Init(const CNodePtr& cnode); + ~GenerateGraph() = default; + AnfNodePtr virtual_input_node() { return virtual_input_node_; } + AnfNodePtr NewOpInst(const OperatorName& op_name, const OperatorAttrs& attrs); + AnfNodePtr NewOpInst(const OperatorName& op_name); + AnfNodePtr PushBack(const std::vector& inputs); + + private: + CNodePtr cnode_; + FuncGraphManagerPtr manager_; + ScopePtr scope_; + FuncGraphPtr func_graph_; + AnfNodePtr virtual_input_node_; + std::string instance_name_base_; + int64_t name_idx_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GENERATE_GRAPH_H_ diff --git a/mindspore/ccsrc/parallel/graph_util/get_parallel_info.cc b/mindspore/ccsrc/parallel/graph_util/get_parallel_info.cc new file mode 100644 index 00000000000..6619f2cc9c2 --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/get_parallel_info.cc @@ -0,0 +1,104 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/graph_util/get_parallel_info.h" + +#include +#include +#include +#include + +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/strategy.h" +#include "ir/func_graph.h" +#include "common/utils.h" +#include "parallel/graph_util/graph_info.h" + +namespace mindspore { +namespace parallel { +py::dict GetParameterLayout(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + py::dict dict; + std::vector graph_params = graph->parameters(); + + for (auto para : graph_params) { + std::string name = std::static_pointer_cast(para)->name(); + std::shared_ptr tensor_layout = std::static_pointer_cast(para)->tensor_layout(); + if (tensor_layout == nullptr) { + MS_LOG(INFO) << "GetParameterLayout nullptr name = " << name; + } else { + auto device_arrangement = tensor_layout->device_arrangement().array(); + auto tensor_map = tensor_layout->tensor_map().array(); + std::pair, std::vector> layout(device_arrangement, tensor_map); + dict[py::str(name)] = layout; + MS_LOG(INFO) << "GetParameterLayout name = " << name << ", layout " << tensor_layout->ToString(); + } + } + return dict; +} + +py::dict GetCNodeStrategy(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + py::dict dict; + auto ret = graph->get_return(); + MS_EXCEPTION_IF_NULL(ret); + auto nodes = DeepScopedGraphSearch(ret); + + for (auto node : nodes) { + if (node->isa()) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto distributed_operation_info = cnode->operator_info(); + if (distributed_operation_info != nullptr) { + auto strategyPtr = distributed_operation_info->strategy(); + if (strategyPtr != nullptr) { + auto strategy = strategyPtr->GetInputDim(); + auto name = cnode->fullname_with_scope(); + dict[py::str(name)] = strategy; + } + } + } + } + return dict; +} + +py::dict GetAllreduceFusion(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + py::dict dict; + auto allreduce_prim_list = FindPrimtive(graph, ALL_REDUCE); + + for (auto prim : allreduce_prim_list) { + auto name_ptr = prim->GetAttr("parameter"); + auto fusion_ptr = prim->GetAttr("fusion"); + if (fusion_ptr == nullptr) { + MS_LOG(EXCEPTION) << "fusion_ptr is nullptr"; + } else if (name_ptr == nullptr) { + continue; + } + if (!name_ptr->isa()) { + MS_LOG(EXCEPTION) << "name is not StringImm"; + } + auto name = name_ptr->cast()->value(); + if (!fusion_ptr->isa()) { + MS_LOG(EXCEPTION) << "fusion is not Int32Imm"; + } + int32_t fusion = fusion_ptr->cast()->value(); + dict[py::str(name)] = fusion; + } + return dict; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/graph_util/get_parallel_info.h b/mindspore/ccsrc/parallel/graph_util/get_parallel_info.h new file mode 100644 index 00000000000..78f597b2135 --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/get_parallel_info.h @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GET_GRAPH_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GET_GRAPH_INFO_H_ + +#include "pybind11/stl.h" + +#include "ir/anf.h" + +namespace mindspore { +namespace parallel { +py::dict GetParameterLayout(const FuncGraphPtr& graph); +py::dict GetCNodeStrategy(const FuncGraphPtr& graph); +py::dict GetAllreduceFusion(const FuncGraphPtr& graph); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GET_GRAPH_INFO_H_ diff --git a/mindspore/ccsrc/parallel/graph_util/graph_info.cc b/mindspore/ccsrc/parallel/graph_util/graph_info.cc new file mode 100644 index 00000000000..be73683c576 --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/graph_info.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/graph_util/graph_info.h" +#include "ir/func_graph.h" +#include "utils/graph_utils.h" +#include "utils/context/ms_context.h" +#include "debug/draw.h" +#include "debug/anf_ir_dump.h" +#include "debug/anf_ir_utils.h" + +namespace mindspore { +namespace parallel { +std::vector FindPrimtive(const FuncGraphPtr& graph, const std::string& name) { + AnfNodePtr ret = graph->get_return(); + MS_EXCEPTION_IF_NULL(ret); + std::vector all_nodes = DeepScopedGraphSearch(ret); + std::vector prim_list; + for (auto& node : all_nodes) { + if (!IsValueNode(node)) { + continue; + } + ValueNodePtr prim_node_anf = node->cast(); + MS_EXCEPTION_IF_NULL(prim_node_anf); + PrimitivePtr node_prim = prim_node_anf->value()->cast(); + MS_EXCEPTION_IF_NULL(node_prim); + if (node_prim->name() == name) { + prim_list.emplace_back(node_prim); + } + } + return prim_list; +} + +void DumpGraph(const FuncGraphPtr& root, const std::string& name) { + if (MsContext::GetInstance()->save_graphs_flag()) { + draw::Draw(name + ".dot", root); + DumpIR(name + ".ir", root); + ExportIR(name + ".dat", "0", root); + } +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/graph_util/graph_info.h b/mindspore/ccsrc/parallel/graph_util/graph_info.h new file mode 100644 index 00000000000..96deab29064 --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/graph_info.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GRAPH_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GRAPH_INFO_H_ + +#include +#include + +#include "ir/anf.h" + +namespace mindspore { +namespace parallel { +std::vector FindPrimtive(const FuncGraphPtr& graph, const std::string& name); +void DumpGraph(const FuncGraphPtr& root, const std::string& name); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_GRAPH_INFO_H_ diff --git a/mindspore/ccsrc/parallel/graph_util/node_info.cc b/mindspore/ccsrc/parallel/graph_util/node_info.cc new file mode 100644 index 00000000000..b2ce8ba432b --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/node_info.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/graph_util/node_info.h" + +#include + +#include "ir/anf.h" +#include "pipeline/parse/python_adapter.h" + +namespace mindspore { +namespace parallel { +std::string ParameterName(const AnfNodePtr& node_ptr) { + auto para_ptr = node_ptr->cast(); + MS_EXCEPTION_IF_NULL(para_ptr); + return para_ptr->name(); +} + +bool ParameterRequireGrad(const AnfNodePtr& node_ptr) { + auto para_ptr = node_ptr->cast(); + if (para_ptr == nullptr) { + return false; + } + if (!para_ptr->has_default()) { + return false; + } + return py::cast(parse::python_adapter::GetPyObjAttr(para_ptr->default_param(), "requires_grad")); +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/graph_util/node_info.h b/mindspore/ccsrc/parallel/graph_util/node_info.h new file mode 100644 index 00000000000..f4f46d2149f --- /dev/null +++ b/mindspore/ccsrc/parallel/graph_util/node_info.h @@ -0,0 +1,31 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_NODE_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_NODE_INFO_H_ + +#include +#include "ir/base.h" + +namespace mindspore { +namespace parallel { +std::string ParameterName(const AnfNodePtr& node_ptr); + +bool ParameterRequireGrad(const AnfNodePtr& node_ptr); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_GRAPH_UTIL_NODE_INFO_H_ diff --git a/mindspore/ccsrc/parallel/group_manager.cc b/mindspore/ccsrc/parallel/group_manager.cc new file mode 100644 index 00000000000..35e81a7d83c --- /dev/null +++ b/mindspore/ccsrc/parallel/group_manager.cc @@ -0,0 +1,178 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/group_manager.h" + +#include +#include + +#include "parallel/device_manager.h" +#include "parallel/ops_info/ops_utils.h" +#include "utils/comm_manager.h" + +namespace mindspore { +namespace parallel { +Group::Group() { + name_.clear(); + devices_.clear(); +} + +Status Group::Init(const std::string &name, const std::list &devices) { + this->name_ = name; + this->devices_ = devices; + return Status::SUCCESS; +} + +std::list Group::GetDevicesList() const { return devices_; } + +bool Group::IsInThisGroup(int32_t device_rank) { + for (auto &device : devices_) { + if (device.rank() == device_rank) { + return true; + } + } + return false; +} + +// Get the position of the device in the group +Status Group::GetIndex(size_t *index) { + size_t pos = 0; + CheckGlobalDeviceManager(); + int32_t rank = g_device_manager->global_rank(); + for (auto &device : devices_) { + if (device.rank() == rank) { + *index = pos; + return Status::SUCCESS; + } else { + pos++; + } + } + MS_LOG(ERROR) << "Could not find device rank " << rank << "in this group!"; + return Status::FAILED; +} + +GroupManager::GroupManager() { groups_.clear(); } + +Status GroupManager::CreateGroup(const std::string &group_name, const std::list &devices, + mindspore::parallel::Group *const group) { + // it is simple to use size to determine whether it is a world group + uint32_t world_size = 0; + if (world_group_ != NCCL_WORLD_GROUP) { + (void)CommManager::GetInstance().GetRankSize(world_group_, &world_size); + } + + if ((world_group_ == NCCL_WORLD_GROUP) || (devices.size() == world_size)) { + auto it = groups_.find(world_group_); + if (it == groups_.end()) { + (void)group->Init(world_group_, devices); + groups_[world_group_] = *group; + } else { + *group = it->second; + } + MS_LOG(INFO) << "It is world group " << world_group_ << ", no need to create it."; + return Status::SUCCESS; + } + + auto it = groups_.find(group_name); + // If there already exits a group with the desired 'name', + // let the pointer point to the group. + if (it != groups_.end()) { + *group = it->second; + return Status::SUCCESS; + } else { + (void)group->Init(group_name, devices); + groups_[group_name] = *group; + + vector ranks; + (void)std::transform(std::begin(devices), std::end(devices), std::back_inserter(ranks), + [](const Device dev) { return (uint32_t)dev.rank(); }); + // Create group through the CommManager interface + bool ret = CommManager::GetInstance().CreateGroupSync(group_name, ranks); + if (!ret) { + MS_LOG(ERROR) << "Create group failed, group name is " << group_name; + return Status::FAILED; + } + + MS_LOG(INFO) << "Create group success, group name is " << group_name; + return Status::SUCCESS; + } +} + +Status GroupManager::DestroyGroup(mindspore::parallel::Group *const group) { + std::string name = (*group).name(); + auto it = groups_.find(name); + if (it == groups_.end()) { + MS_LOG(ERROR) << "Could not find group name :" << name; + return Status::FAILED; + } + (void)groups_.erase(it); + bool ret = CommManager::GetInstance().DestroyGroup(name); + if (!ret) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +Status GroupManager::DestroyAllGroups() { + for (auto &it : groups_) { + std::string name = it.first; + bool ret = CommManager::GetInstance().DestroyGroup(name); + if (!ret) { + return Status::FAILED; + } + } + groups_.clear(); + return Status::SUCCESS; +} + +Status GroupManager::GetRankID(const std::string &name, unsigned int *const rank_id) { + auto it = groups_.find(name); + if (it == groups_.end()) { + MS_LOG(ERROR) << "Could not find group name :" << name; + return Status::FAILED; + } + bool ret = CommManager::GetInstance().GetRankID(name, rank_id); + if (!ret) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +Status GroupManager::GetRankSize(const std::string &name, unsigned int *const rank_size) { + auto it = groups_.find(name); + if (it == groups_.end()) { + MS_LOG(ERROR) << "Could not find group name :" << name; + return Status::FAILED; + } + bool ret = CommManager::GetInstance().GetRankSize(name, rank_size); + if (!ret) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +Status GroupManager::FindGroup(const std::string &name, mindspore::parallel::Group **group) { + auto it = groups_.find(name); + if (it == groups_.end()) { + return Status::FAILED; + } + *group = &it->second; + return Status::SUCCESS; +} + +void GroupManager::Clear() { (void)DestroyAllGroups(); } +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/group_manager.h b/mindspore/ccsrc/parallel/group_manager.h new file mode 100644 index 00000000000..c3b95b99e0e --- /dev/null +++ b/mindspore/ccsrc/parallel/group_manager.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_GROUP_MANAGER_H_ +#define MINDSPORE_CCSRC_PARALLEL_GROUP_MANAGER_H_ + +#include +#include +#include +#include + +#include "parallel/device.h" +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { +constexpr char HCCL_WORLD_GROUP[] = "hccl_world_group"; +constexpr char NCCL_WORLD_GROUP[] = "nccl_world_group"; +constexpr char UNDEFINED_WORLD_GROUP[] = "undefined_world_group"; + +// Devices that need communication should in the same group. These classes are used to +// create and destroy group among devices. +class Group { + public: + Group(); + ~Group() = default; + Status Init(const std::string& name, const std::list& devices); + std::list GetDevicesList() const; + std::string name() const { return name_; } + bool IsInThisGroup(int32_t device_rank); + Status GetIndex(size_t* index); + size_t GetDevNum() const { return devices_.size(); } + + private: + std::string name_; + std::list devices_; +}; + +class GroupManager { + public: + GroupManager(); + ~GroupManager() = default; + + Status CreateGroup(const std::string& name, const std::list& devices, Group* group); + Status DestroyGroup(Group* group); + Status DestroyAllGroups(); + Status GetRankID(const std::string& name, unsigned int* rank_id); + Status GetRankSize(const std::string& name, unsigned int* rank_size); + Status FindGroup(const std::string& name, Group** group); + std::string world_group() const { return world_group_; } + void set_world_group(const std::string& name) { world_group_ = name; } + void Clear(); + + private: + // the key is group name (name_) + std::map groups_; + std::string world_group_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_GROUP_MANAGER_H_ diff --git a/mindspore/ccsrc/parallel/node_check.cc b/mindspore/ccsrc/parallel/node_check.cc new file mode 100644 index 00000000000..db53bbc59f9 --- /dev/null +++ b/mindspore/ccsrc/parallel/node_check.cc @@ -0,0 +1,86 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/node_check.h" + +#include +#include + +#include "parallel/ops_info/ops_utils.h" + +namespace mindspore { +namespace parallel { +const std::set BLACK_LIST = {TUPLE_GETITEM, + MAKE_TUPLE, + J, + LIST_GETITEM, + ARRAY_GETITEM, + TUPLE_SETITEM, + DEPEND, + LIST_SETITEM, + ARRAY_SETITEM, + DICT_GETITEM, + LIST_APPEND, + LIST_MAP, + LIST_REDUCE, + TUPLE_REVERSED, + TILE_SHAPE, + TUPLE_DIV, + TUPLE_TO_ARRAY, + MAKE_LIST, + MAKE_DICT, + MAKE_SLICE, + MAKE_RECORD, + STRING_EQUAL, + VIRTUALLOSS, + RETURN, + ENV_GETITEM, + IDENTITY, + PARTIAL, + ENVSETITEM, + ENVGETITEM, + ENVADD, + MAKEREFKEY, + MAKEREF, + GETREFKEY, + GETREFVALUE, + GETREFORIGIN, + DOT, + IM2COL, + COL2IM, + IM2COLV1, + STATESETITEM, + SCALARSUMMARY, + IMAGESUMMARY, + TENSORSUMMARY, + COL2IMV1, + RESOLVE, + BROADCASTGRADIENTARGS, + INVERTPERMUTATION, + CONTROLDEPEND, + EMBED, + CREATINSTANCE, + ZEROSLIKETENSOR, + ASSIGN, + REF_TO_EMBED, + STOP_GRADIENT}; + +bool IsInBlackList(const PrimitivePtr& prim) { + MS_EXCEPTION_IF_NULL(prim); + return (BLACK_LIST.find(prim->name()) != BLACK_LIST.end()); +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/node_check.h b/mindspore/ccsrc/parallel/node_check.h new file mode 100644 index 00000000000..6e5db37069d --- /dev/null +++ b/mindspore/ccsrc/parallel/node_check.h @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_NODE_CHECK_H_ +#define MINDSPORE_CCSRC_PARALLEL_NODE_CHECK_H_ + +#include "ir/primitive.h" + +namespace mindspore { +namespace parallel { +bool IsInBlackList(const PrimitivePtr& prim); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_NODE_CHECK_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/activation_info.cc b/mindspore/ccsrc/parallel/ops_info/activation_info.cc new file mode 100644 index 00000000000..6e581d6280f --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/activation_info.cc @@ -0,0 +1,385 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/activation_info.h" + +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/auto_parallel/costmodel.h" + +namespace mindspore { +namespace parallel { +Status Activation::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status Activation::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + + return SUCCESS; +} + +Status ActivationInfo::GetAttrs() { + if (attrs_.size() < ACTIVATION_ATTR_SIZE) { + MS_LOG(ERROR) << name_ << " : The size of attrs small than 1."; + return FAILED; + } + + if ((inputs_shape_.size() != ACTIVATION_INPUTS_SIZE) || (outputs_shape_.size() != ACTIVATION_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size(" << inputs_shape_.size() << ") or outputs shape size(" + << outputs_shape_.size() << "is wrong."; + return FAILED; + } + + auto iter = attrs_.find(ACTIVATION_TYPE); + if (iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + std::string val = iter->second->cast()->value(); + if ((val != RELU_TYPE) && (val != RELU6_TYPE) && (val != SIGMOID_TYPE)) { + MS_LOG(ERROR) << name_ << " : Activation type is wrong."; + return FAILED; + } + } else { + MS_LOG(ERROR) << name_ << " : The value of activation_type is not string."; + return FAILED; + } + } + + return SUCCESS; +} + +Status ActivationOther::GetAttrs() { + if ((inputs_shape_.size() != ACTIVATION_INPUTS_SIZE) || (outputs_shape_.size() != ACTIVATION_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size(" << inputs_shape_.size() << ") or outputs shape size(" + << outputs_shape_.size() << "is wrong."; + return FAILED; + } + return SUCCESS; +} + +Status Activation::GenerateStrategies(int32_t stage_id) { + if ((inputs_shape_.size() != ACTIVATION_INPUTS_SIZE) || (outputs_shape_.size() != ACTIVATION_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size(" << inputs_shape_.size() << ") or outputs shape size(" + << outputs_shape_.size() << "is wrong."; + return FAILED; + } + + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size(), 1); + Shapes splittable_inputs = {input0_split}; + + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Generate strategies for independent inputs() failed."; + return FAILED; + } + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << " : Successfully generated " << success << " strategy"; + PrintStrategy(sp); + } + } + return SUCCESS; +} + +Status Softmax::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + + std::vector stra = strategy->GetInputDim(); + Dimensions input_strategy = stra.at(0); + + for (auto& element : axis_) { + int32_t axis_index = element; + if (element < 0) { + size_t input_dim = inputs_shape_.at(0).size(); + axis_index = static_cast(input_dim) + element; + } + + int32_t axis_strategy = input_strategy.at(IntToSize(axis_index)); + // Dimension corresponding to axis is un-splittable + if (axis_strategy != MIN_SLICE_NUM) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : The strategy corresponding to axis dimension(" << axis_strategy << ") is not 1"; + } else { + MS_LOG(ERROR) << name_ << " : The strategy corresponding to axis dimension(" << axis_strategy << ") is not 1"; + } + return FAILED; + } + } + + return SUCCESS; +} + +Status Softmax::GetAttrs() { + if (attrs_.size() < SOFTMAX_ATTR_SIZE) { + MS_LOG(ERROR) << name_ << " : The size of attrs small than 1."; + return FAILED; + } + + auto iter = attrs_.find(AXIS); + if (iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { // the axis is a number + int32_t axis_element = iter->second->cast()->value(); + axis_.push_back(axis_element); + MS_LOG(INFO) << name_ << " : The axis is int, value is " << axis_element; + } else if (iter->second->isa()) { // the axis is a tuple + ValueTuplePtr value_tuple = iter->second->cast(); + if (value_tuple == nullptr) { + MS_LOG(ERROR) << name_ << " : The value_tuple is nullptr."; + return FAILED; + } + std::vector value_vector = value_tuple->value(); + (void)std::transform(value_vector.begin(), value_vector.end(), std::back_inserter(axis_), + [](const ValuePtr& value) { return static_cast(GetValue(value)); }); + if (axis_.empty()) { + MS_LOG(ERROR) << name_ << " : The axis tuple is empty."; + return FAILED; + } + MS_LOG(INFO) << name_ << " : The axis is tuple, value is " << ShapeToString(axis_); + } else { + MS_LOG(ERROR) << name_ << " : The value of axis is not int or tuple int."; + return FAILED; + } + } + + if ((inputs_shape_.size() != ACTIVATION_INPUTS_SIZE) || (outputs_shape_.size() != ACTIVATION_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size or outputs shape size is wrong."; + return FAILED; + } + + // for example: tensor dimension is 4, then axis range [-4, 3] + int32_t dim = SizeToInt(inputs_shape_.at(0).size()); + auto it = std::find_if(axis_.begin(), axis_.end(), + [dim](const int32_t& element) { return ((element >= dim) || (element < -dim)); }); + if (it != axis_.end()) { + MS_LOG(ERROR) << name_ << " : The axis(" << *it << ") is out of range[" << -dim << ", " << dim - 1 << "]."; + return FAILED; + } + + return SUCCESS; +} + +Status Softmax::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status Softmax::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << " : GetAttrs failed."; + return FAILED; + } + if ((inputs_shape_.size() != ACTIVATION_INPUTS_SIZE) || (outputs_shape_.size() != ACTIVATION_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size or outputs shape size is wrong."; + return FAILED; + } + + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size(), 1); + for (auto& element : axis_) { + int32_t axis_index = element; + if (element < 0) { + size_t input_dim = inputs_shape_.at(0).size(); + axis_index = static_cast(input_dim) + element; + } + input0_split[IntToSize(axis_index)] = 0; + } + Shapes splittable_inputs = {input0_split}; + + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Generate strategies for independent inputs failed."; + return FAILED; + } + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << " : Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} + +Status ActivationBase::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + + dev_matrix_shape_ = input_strategy; + + return SUCCESS; +} + +Status ActivationBase::InferMirrorOps() { + mirror_ops_.clear(); + + Shape tensor_map = inputs_tensor_map_[0]; + std::vector group; + if (CreateGroupByTensorMap(tensor_map, &group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group failed."; + return FAILED; + } + + OperatorVector mirror_op; + if (group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror ops is empty."; + return SUCCESS; + } else { + mirror_op = CreateMirrorOps(group[0].name(), group[0].GetDevNum()); + mirror_ops_.push_back(mirror_op); + std::string group_name = group[0].name(); + MS_LOG(INFO) << name_ << " : Create the mirror ops success, the group name is " << group_name; + } + + return SUCCESS; +} + +Status ActivationBase::InferForwardCommunication() { + // do nothing + return SUCCESS; +} + +Status ActivationBase::InferTensorMap() { + std::vector tensor_map_index; + size_t size = inputs_shape_.at(0).size(); + // such as 4: tensor_map_index [3,2,1,0] + for (size_t i = 0; i < size; ++i) { + tensor_map_index.push_back((int32_t)(size - i - 1)); + } + + inputs_tensor_map_.push_back(tensor_map_index); + outputs_tensor_map_.push_back(tensor_map_index); + return SUCCESS; +} + +Status ActivationBase::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = {inputs_strategy.at(0)}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_slice_shape = inputs_slice_shape.at(0); + + TensorLayout input_tensor_layout; + if (input_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], input_shape) != SUCCESS) { + return FAILED; + } + + TensorInfo input_tensor_info(input_tensor_layout, input_shape, input_slice_shape); + + inputs_tensor_info_.push_back(input_tensor_info); + outputs_tensor_info_.push_back(input_tensor_info); // the same as input + + return SUCCESS; +} + +Status ActivationBase::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status ActivationBase::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} + +Status CastInfo::InferMirrorOps() { + mirror_ops_.clear(); + + Shape tensor_map = inputs_tensor_map_[0]; + std::vector group; + if (CreateGroupByTensorMap(tensor_map, &group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group failed."; + return FAILED; + } + + OperatorVector mirror_op; + OperatorVector op_for_value; + if (group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror ops is empty."; + return SUCCESS; + } else { + mirror_op = CreateMirrorOps(group[0].name(), group[0].GetDevNum()); + mirror_ops_.push_back(mirror_op); + mirror_ops_.push_back(op_for_value); + std::string group_name = group[0].name(); + MS_LOG(INFO) << name_ << " : Create the mirror ops success, the group name is " << group_name; + } + + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/activation_info.h b/mindspore/ccsrc/parallel/ops_info/activation_info.h new file mode 100644 index 00000000000..e988ccf54c1 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/activation_info.h @@ -0,0 +1,173 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ACTIVATION_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ACTIVATION_INFO_H_ + +#include +#include +#include +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class ActivationBase : public OperatorInfo { + public: + ActivationBase(const std::string& operator_name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs) {} + ~ActivationBase() override = default; + + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + protected: + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorMap() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; +}; + +class Activation : public ActivationBase { + public: + Activation(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationBase(name, inputs_shape, outputs_shape, attrs) { + ac_cost_ptr_ = std::make_shared(); + } + ~Activation() override = default; + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return ac_cost_ptr_; } + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + + private: + ActivationCostPtr ac_cost_ptr_; +}; + +class ActivationInfo : public Activation { + public: + ActivationInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : Activation(name, inputs_shape, outputs_shape, attrs) {} + ~ActivationInfo() override = default; + + protected: + Status GetAttrs() override; // activation_type: relu, relu6, sigmoid +}; + +class ActivationOther : public Activation { + public: + ActivationOther(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : Activation(name, inputs_shape, outputs_shape, attrs) {} + ~ActivationOther() override = default; + + protected: + Status GetAttrs() override; +}; + +class GeluInfo : public ActivationOther { + public: + GeluInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~GeluInfo() override = default; +}; + +class TanhInfo : public ActivationOther { + public: + TanhInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~TanhInfo() override = default; +}; + +class Softmax : public ActivationBase { + public: + explicit Softmax(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationBase(name, inputs_shape, outputs_shape, attrs) { + sm_cost_ptr_ = std::make_shared(); + } + ~Softmax() override = default; + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return sm_cost_ptr_; } + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status GetAttrs() override; + + private: + std::vector axis_; + SoftmaxCostPtr sm_cost_ptr_; +}; + +class SoftmaxInfo : public Softmax { + public: + SoftmaxInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : Softmax(name, inputs_shape, outputs_shape, attrs) {} + ~SoftmaxInfo() override = default; +}; + +class LogSoftmaxInfo : public Softmax { + public: + LogSoftmaxInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : Softmax(name, inputs_shape, outputs_shape, attrs) {} + ~LogSoftmaxInfo() override = default; +}; + +class ReLUInfo : public ActivationOther { + public: + ReLUInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~ReLUInfo() override = default; +}; + +class CastInfo : public ActivationOther { + public: + CastInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~CastInfo() override = default; + + protected: + Status InferMirrorOps() override; +}; + +class SqrtInfo : public ActivationOther { + public: + SqrtInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~SqrtInfo() override = default; +}; +} // namespace parallel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_OPS_INFO_PARALLEL_ACTIVATION_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/arithmetic_info.cc b/mindspore/ccsrc/parallel/ops_info/arithmetic_info.cc new file mode 100644 index 00000000000..48741690ba6 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/arithmetic_info.cc @@ -0,0 +1,363 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/arithmetic_info.h" + +#include +#include +#include +#include + +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +Shape ExpendShape(const Shape &bigger_size_shape, Shape smaller_size_shape) { + size_t insert_num = bigger_size_shape.size() - smaller_size_shape.size(); + for (size_t num = 0; num < insert_num; ++num) { + (void)smaller_size_shape.insert(smaller_size_shape.begin(), 1); + } + return smaller_size_shape; +} + +Shapes ArithmeticBase::InferExpendShape() { + Shape input_a_shape = inputs_shape_.at(0); + Shape input_b_shape = inputs_shape_.at(1); + Shapes input_shapes; + size_t input_a_size = input_a_shape.size(); + size_t input_b_size = input_b_shape.size(); + if (input_a_size > input_b_size) { + input_shapes.push_back(input_a_shape); + input_shapes.push_back(ExpendShape(input_a_shape, input_b_shape)); + } else if (input_a_size < input_b_size) { + input_shapes.push_back(ExpendShape(input_b_shape, input_a_shape)); + input_shapes.push_back(input_b_shape); + } else { + input_shapes.push_back(input_a_shape); + input_shapes.push_back(input_b_shape); + } + return input_shapes; +} + +std::vector ExpendStrategy(const StrategyPtr &strategy) { + std::vector expend_strategy; + std::vector stra = strategy->GetInputDim(); + Dimensions sub_a_strategy = stra.at(0); + Dimensions sub_b_strategy = stra.at(1); + size_t input_a_size = sub_a_strategy.size(); + size_t input_b_size = sub_b_strategy.size(); + if (input_a_size > input_b_size) { + expend_strategy.push_back(sub_a_strategy); + expend_strategy.push_back(ExpendShape(sub_a_strategy, sub_b_strategy)); + } else if (input_a_size < input_b_size) { + expend_strategy.push_back(ExpendShape(sub_b_strategy, sub_a_strategy)); + expend_strategy.push_back(sub_b_strategy); + } else { + expend_strategy = stra; + } + return expend_strategy; +} + +Status ArithmeticBase::CheckStrategy(const StrategyPtr &strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + Shapes input_shapes = InferExpendShape(); + std::vector expend_strategy = ExpendStrategy(strategy); + Dimensions sub_a_strategy = expend_strategy.at(0); + Dimensions sub_b_strategy = expend_strategy.at(1); + Shape input_a_shape = input_shapes.at(0); + Shape input_b_shape = input_shapes.at(1); + + for (size_t i = 0; i < input_a_shape.size(); ++i) { + if ((sub_a_strategy[i] != sub_b_strategy[i]) && (input_a_shape[i] != 1) && (input_b_shape[i] != 1)) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + } + return SUCCESS; +} + +Status ArithmeticBase::InferDevMatrixShape() { + std::vector expend_strategy = ExpendStrategy(strategy_); + Dimensions sub_a_strategy = expend_strategy.at(0); + Dimensions sub_b_strategy = expend_strategy.at(1); + Shape dev_shape; + for (size_t i = 0; i < sub_a_strategy.size(); ++i) { + if (sub_a_strategy[i] != sub_b_strategy[i]) { + dev_shape.push_back(sub_a_strategy[i] * sub_b_strategy[i]); + } else { + dev_shape.push_back(sub_a_strategy[i]); + } + } + dev_matrix_shape_ = dev_shape; + + return SUCCESS; +} + +TensorMap SetExpendTensorMap(const Shape &strategy, const Shape &dev_matrix_shape) { + TensorMap tensor_map_index; + for (size_t i = 0; i < strategy.size(); ++i) { + if (strategy[i] == dev_matrix_shape[i]) { + tensor_map_index.push_back((int32_t)(LAST_INDEX(SizeToUint(strategy.size())) - i)); + } else { + tensor_map_index.push_back(-1); + } + } + return tensor_map_index; +} + +TensorMap SetTensorMap(const Shape &strategy_expend, const Shape &dev_matrix_shape, const Shape &strategy) { + TensorMap expend_map = SetExpendTensorMap(strategy_expend, dev_matrix_shape); + size_t dev_matrix_size = dev_matrix_shape.size(); + size_t strategy_size = strategy.size(); + if (dev_matrix_size != strategy_size) { + (void)expend_map.erase(expend_map.begin(), + expend_map.begin() + static_cast(dev_matrix_size - strategy_size)); + } + return expend_map; +} + +void ArithmeticBase::ReComputeBatchSplitFlagList() { + Shapes expend_shapes = InferExpendShape(); + Shape expend_a_shape = expend_shapes.at(0); + Shape expend_b_shape = expend_shapes.at(1); + if (expend_a_shape.size() != expend_b_shape.size()) { + MS_LOG(EXCEPTION) << name_ << " : Recompute batch split flag list is wrong."; + } + if (expend_a_shape.empty()) { + split_flag_list_[0] = false; + split_flag_list_[1] = false; + return; + } + (expend_a_shape.at(0) != 1) ? (split_flag_list_[0] = true) : (split_flag_list_[0] = false); + (expend_b_shape.at(0) != 1) ? (split_flag_list_[1] = true) : (split_flag_list_[1] = false); +} + +Status ArithmeticBase::InferTensorMap() { + std::vector tensor_map_index; + std::vector expend_strategy = ExpendStrategy(strategy_); + Dimensions sub_a_expend_strategy = expend_strategy.at(0); + Dimensions sub_b_expend_strategy = expend_strategy.at(1); + Strategys stra = strategy_->GetInputDim(); + Dimensions sub_a_strategy = stra.at(0); + Dimensions sub_b_strategy = stra.at(1); + for (size_t i = 0; i < sub_a_expend_strategy.size(); ++i) { + tensor_map_index.push_back((int32_t)(LAST_INDEX(SizeToUint(sub_a_expend_strategy.size())) - i)); + } + + Shape dev_shape; + for (size_t i = 0; i < sub_a_expend_strategy.size(); ++i) { + if (sub_a_expend_strategy[i] != sub_b_expend_strategy[i]) { + dev_shape.push_back(sub_a_expend_strategy[i] * sub_b_expend_strategy[i]); + } else { + dev_shape.push_back(sub_a_expend_strategy[i]); + } + } + inputs_tensor_map_.push_back(SetTensorMap(sub_a_expend_strategy, dev_shape, sub_a_strategy)); + inputs_tensor_map_.push_back(SetTensorMap(sub_b_expend_strategy, dev_shape, sub_b_strategy)); + outputs_tensor_map_.push_back(tensor_map_index); + + return SUCCESS; +} + +Status ArithmeticBase::InferMirrorOps() { + mirror_ops_.clear(); + Shape input_a_tensor_map = inputs_tensor_map_.at(0); + Shape input_b_tensor_map = inputs_tensor_map_.at(1); + std::vector input_a_group, input_b_group; + if (CreateGroupByTensorMap(input_a_tensor_map, &input_a_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group for input a failed."; + return FAILED; + } + if (CreateGroupByTensorMap(input_b_tensor_map, &input_b_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group for input b failed."; + return FAILED; + } + + OperatorVector op_for_input_a, op_for_input_b; + if (input_a_group.empty() && input_b_group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror group is empty."; + return SUCCESS; + } + if (!input_a_group.empty()) { + op_for_input_a = CreateMirrorOps(input_a_group[0].name(), input_a_group[0].GetDevNum()); + MS_LOG(INFO) << name_ << " : Create the mirror ops for input a success, group is " << input_a_group[0].name(); + } + if (!input_b_group.empty()) { + op_for_input_b = CreateMirrorOps(input_b_group[0].name(), input_b_group[0].GetDevNum()); + MS_LOG(INFO) << name_ << " : Create the mirror ops for input b success, group is " << input_b_group[0].name(); + } + mirror_ops_.push_back(op_for_input_a); + mirror_ops_.push_back(op_for_input_b); + + return SUCCESS; +} + +Status ArithmeticBase::InferTensorLayout(TensorLayouts *inputs_layout, TensorLayouts *outputs_layout, + const Shape &dev_matrix_array) { + if ((inputs_layout == nullptr) || (outputs_layout == nullptr)) { + MS_LOG(ERROR) << name_ << " : The layout is null."; + return FAILED; + } + TensorMap input_a_tensor_map_array = inputs_tensor_map_.at(0); + TensorMap input_b_tensor_map_array = inputs_tensor_map_.at(1); + TensorMap out_tensor_map_array = outputs_tensor_map_.at(0); + Shape input_a_shape_array = inputs_shape_.at(0); + Shape input_b_shape_array = inputs_shape_.at(1); + Shape out_shape_array = outputs_shape_.at(0); + + TensorLayout input_a_tensor_layout, input_b_tensor_layout, out_tensor_layout; + if (input_a_tensor_layout.InitFromVector(dev_matrix_array, input_a_tensor_map_array, input_a_shape_array) != + SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create tensor layout for input a failed."; + return FAILED; + } + if (input_b_tensor_layout.InitFromVector(dev_matrix_array, input_b_tensor_map_array, input_b_shape_array) != + SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create tensor layout for input b failed."; + return FAILED; + } + if (out_tensor_layout.InitFromVector(dev_matrix_array, out_tensor_map_array, out_shape_array) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create tensor layout for output failed."; + return FAILED; + } + inputs_layout->push_back(input_a_tensor_layout); + inputs_layout->push_back(input_b_tensor_layout); + outputs_layout->push_back(out_tensor_layout); + + return SUCCESS; +} + +Status ArithmeticBase::InferTensorInfo() { + // infer tensor shape + Shape input_a_shape = inputs_shape_.at(0); + Shape input_b_shape = inputs_shape_.at(1); + Shape output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + std::vector expend_strategy = ExpendStrategy(strategy_); + Dimensions sub_a_expend_strategy = expend_strategy.at(0); + Dimensions sub_b_expend_strategy = expend_strategy.at(1); + Strategys inputs_strategy = strategy_->GetInputDim(); + Shape dev_shape; + for (size_t i = 0; i < sub_a_expend_strategy.size(); ++i) { + if (sub_a_expend_strategy[i] != sub_b_expend_strategy[i]) { + dev_shape.push_back(sub_a_expend_strategy[i] * sub_b_expend_strategy[i]); + } else { + dev_shape.push_back(sub_a_expend_strategy[i]); + } + } + Strategys outputs_strategy = {dev_shape}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_a_slice_shape = inputs_slice_shape.at(0); + Shape input_b_slice_shape = inputs_slice_shape.at(1); + Shape output_slice_shape = outputs_slice_shape.at(0); + + // infer tensor layout + TensorLayouts inputs_layout, outputs_layout; + if (InferTensorLayout(&inputs_layout, &outputs_layout, dev_matrix_shape_) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Infer tensor layout failed."; + return FAILED; + } + + TensorInfo input_a_tensor_info(inputs_layout.at(0), input_a_shape, input_a_slice_shape); + TensorInfo input_b_tensor_info(inputs_layout.at(1), input_b_shape, input_b_slice_shape); + TensorInfo out_tensor_info(outputs_layout.at(0), output_shape, output_slice_shape); + + inputs_tensor_info_.push_back(input_a_tensor_info); // inputs_a + inputs_tensor_info_.push_back(input_b_tensor_info); // inputs_b + outputs_tensor_info_.push_back(out_tensor_info); // output + + return SUCCESS; +} + +Status ArithmeticBase::SetCostUnderStrategy(const StrategyPtr &strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status ArithmeticBase::GenerateStrategies(int32_t stage_id) { + Shape input0_split(inputs_shape_[0].size(), 1); + Shape input1_split(inputs_shape_[1].size(), 1); + Shapes splittable_inputs = {input0_split, input1_split}; + + std::vector sp_vector; + is_auto_parallel_ = true; + if (GenerateStrategiesWithBroadcast(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Generate strategies with broadcast failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << " : Generate strategies with broadcast success."; + + size_t success = 0; + for (auto &sp : sp_vector) { + PrintStrategy(sp); + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << " : Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} + +Status ArithmeticBase::Init(const StrategyPtr &strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status ArithmeticBase::InitForCostModel(const StrategyPtr &strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/arithmetic_info.h b/mindspore/ccsrc/parallel/ops_info/arithmetic_info.h new file mode 100644 index 00000000000..1b641152aac --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/arithmetic_info.h @@ -0,0 +1,124 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ARITHMETIC_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ARITHMETIC_INFO_H_ + +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class ArithmeticBase : public OperatorInfo { + public: + ArithmeticBase(const std::string& operator_name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs) { + arithmeticcost_ptr_ = std::make_shared(); + } + ~ArithmeticBase() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + Status GenerateStrategies(int32_t) override; + Status SetCostUnderStrategy(const StrategyPtr&) override; + OperatorCostPtr GetOperatorCost() const override { return arithmeticcost_ptr_; } + void ReComputeBatchSplitFlagList() override; + + protected: + Status GetAttrs() override { return SUCCESS; } + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override { return SUCCESS; } + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout, const Shape& dev_matrix_array); + Shapes InferExpendShape(); + ArithmeticCostPtr arithmeticcost_ptr_; +}; + +class SubInfo : public ArithmeticBase { + public: + SubInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~SubInfo() override = default; +}; + +class TensorAddInfo : public ArithmeticBase { + public: + TensorAddInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~TensorAddInfo() override = default; +}; + +class MulInfo : public ArithmeticBase { + public: + MulInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~MulInfo() override = default; +}; + +class DivInfo : public ArithmeticBase { + public: + DivInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~DivInfo() override = default; +}; + +class RealDivInfo : public ArithmeticBase { + public: + RealDivInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~RealDivInfo() override = default; +}; + +class FloorDivInfo : public ArithmeticBase { + public: + FloorDivInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~FloorDivInfo() override = default; +}; + +class GreaterInfo : public ArithmeticBase { + public: + GreaterInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~GreaterInfo() override = default; +}; + +class AssignSubInfo : public ArithmeticBase { + public: + AssignSubInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~AssignSubInfo() override = default; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_OPS_INFO_PARALLEL_ARITHMETIC_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/batch_parallel_info.cc b/mindspore/ccsrc/parallel/ops_info/batch_parallel_info.cc new file mode 100644 index 00000000000..793452b8ad2 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/batch_parallel_info.cc @@ -0,0 +1,256 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/batch_parallel_info.h" + +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_manager.h" +#include "parallel/device_matrix.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +Status BatchParallelInfo::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + + int32_t stage = strategy->GetInputStage(); + CheckGlobalDeviceManager(); + int32_t dev_num = SizeToInt(g_device_manager->GetDeviceListByStageId(stage).size()); + dev_num_ = dev_num; + + size_t strategy_size = strategy->GetInputNumber(); + std::vector stra = strategy->GetInputDim(); + for (size_t i = 0; i < strategy_size; ++i) { + Shape sub_strategy = stra.at(i); + size_t strategy_len = sub_strategy.size(); + bool flag = false; + for (size_t j = 0; j < strategy_len; ++j) { + int32_t strategy_value = sub_strategy.at(j); + if (strategy_value > 1) { + if (flag || strategy_value != dev_num_) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : It is not a valid data parallel strategy."; + } else { + MS_LOG(ERROR) << name_ << " : It is not a valid data parallel strategy."; + } + return FAILED; + } + flag = true; + } + } + } + return SUCCESS; +} + +Status BatchParallelInfo::InferDevMatrixShape() { + dev_matrix_shape_.push_back(dev_num_); + return SUCCESS; +} + +Status BatchParallelInfo::InferMirrorOps() { + mirror_ops_.clear(); + if (g_device_manager->DeviceNum() == 1) { + MS_LOG(INFO) << name_ << " : The device num is 1, no need to create mirror ops."; + return SUCCESS; + } + + MS_LOG(INFO) << name_ << " : Batch parallel input number " << strategy_->GetInputNumber(); + for (size_t i = 0; i < input_value_.size(); i++) { + MS_EXCEPTION_IF_NULL(g_device_manager); + OperatorVector op_vec = CreateMirrorOps(g_device_manager->world_group(), g_device_manager->DeviceNum()); + mirror_ops_.push_back(op_vec); + } + return SUCCESS; +} + +Status BatchParallelInfo::InferForwardCommunication() { return SUCCESS; } + +Status BatchParallelInfo::InferTensorMap() { + if (strategy_->GetInputDim()[0][0] != dev_num_) { + MS_LOG(ERROR) << name_ << " : It is not a valid data parallel strategy."; + return FAILED; + } + for (size_t i = 0; i < inputs_shape_.size(); i++) { + std::vector tensor_map_index; + for (size_t j = 0; j < inputs_shape_[i].size(); ++j) { + if (strategy_->GetInputDim()[i][j] == dev_num_ && j == 0) { + tensor_map_index.push_back(0); + } else { + tensor_map_index.push_back(MAP_NONE); + } + } + inputs_tensor_map_.push_back(tensor_map_index); + } + for (size_t i = 0; i < outputs_shape_.size(); i++) { + std::vector tensor_map_index; + for (size_t j = 0; j < outputs_shape_[i].size(); ++j) { + if (i == 0 && j == 0) { + tensor_map_index.push_back(0); + } else { + tensor_map_index.push_back(MAP_NONE); + } + } + outputs_tensor_map_.push_back(tensor_map_index); + } + return SUCCESS; +} + +Strategys BatchParallelInfo::GetOutputsStrategy() { + Strategys outputs_strategy; + + for (size_t i = 0; i < outputs_shape_.size(); ++i) { + std::vector strategy; + for (size_t j = 0; j < outputs_shape_[i].size(); ++j) { + if (i == 0 && j == 0) { + strategy.push_back(dev_num_); + } else { + strategy.push_back(1); + } + } + outputs_strategy.push_back(strategy); + } + + return outputs_strategy; +} + +Status BatchParallelInfo::InferTensorInfo() { + for (size_t i = 0; i < strategy_->GetInputNumber(); i++) { + MS_LOG(INFO) << name_ << " : The input size is " << strategy_->GetInputNumber(); + TensorLayout tensor_layout_in; + if (tensor_layout_in.InitFromVector(dev_matrix_shape_, inputs_tensor_map_.at(i), inputs_shape_.at(i)) != SUCCESS) { + return FAILED; + } + TensorInfo tensor_info_in(tensor_layout_in); + inputs_tensor_info_.push_back(tensor_info_in); + } + for (size_t i = 0; i < outputs_shape_.size(); i++) { + TensorLayout tensor_layout_out; + if (tensor_layout_out.InitFromVector(dev_matrix_shape_, outputs_tensor_map_.at(i), outputs_shape_.at(i)) != + SUCCESS) { + return FAILED; + } + TensorInfo tensor_info_out(tensor_layout_out); + outputs_tensor_info_.push_back(tensor_info_out); + } + return SUCCESS; +} + +Status BatchParallelInfo::GetAttrs() { return SUCCESS; } + +Status BatchParallelInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status BatchParallelInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} + +Status BatchParallelInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + return SUCCESS; +} + +Status BatchParallelInfo::GenerateStrategies(int32_t stage_id) { + CheckGlobalDeviceManager(); + is_auto_parallel_ = true; + size_t total_dev_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + StrategyPtr sp; + std::vector strategy; + for (size_t i = 0; i < inputs_shape_.size(); i++) { + Shape temp(inputs_shape_[i].size(), 1); + if (split_flag_list_[i]) { + temp[0] = SizeToInt(total_dev_num); + } + strategy.push_back(temp); + } + sp = std::make_shared(stage_id, strategy); + + if (SetCostUnderStrategy(sp) == SUCCESS) { + MS_LOG(INFO) << name_ << " : Successfully generated batch-parallel-strategy."; + PrintStrategy(sp); + } else { + MS_LOG(ERROR) << name_ << " : Generating batch-parallel-strategy failed."; + return FAILED; + } + return SUCCESS; +} + +void SparseSoftmaxCrossEntropyWithLogitsInfo::ReComputeBatchSplitFlagList() { + for (size_t i = 0; i < inputs_shape_.size(); i++) { + split_flag_list_[i] = true; + } +} + +void GatherV2Info::ReComputeBatchSplitFlagList() { + MS_ASSERT(inputs_shape_.size() == 2); + MS_ASSERT(input_value_.size() == 3); + MS_ASSERT(input_value_[0] == nullptr); + // the second input is the index tensor + MS_ASSERT(input_value_[1] != nullptr); + // the third input is the axis + MS_ASSERT(input_value_[2] != nullptr); + int axis = GetValue(input_value_[2]); + MS_ASSERT(axis < inputs_shape_[0].size() && axis >= 0 - inputs_shape_[0].size()); + if (axis < 0) { + axis += SizeToInt(inputs_shape_[0].size()); + } + split_flag_list_[0] = true; + // if gather axis is 0, the index's strategy is equal to device number + if (axis == 0) { + split_flag_list_[1] = true; + } +} + +Status BatchParallelInfo::InferAsLossDivisor() { + as_loss_divisor_ = 1; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/batch_parallel_info.h b/mindspore/ccsrc/parallel/ops_info/batch_parallel_info.h new file mode 100644 index 00000000000..fb08fbe9fc0 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/batch_parallel_info.h @@ -0,0 +1,82 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_BATCH_PARALLEL_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_BATCH_PARALLEL_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class BatchParallelInfo : public OperatorInfo { + public: + BatchParallelInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs), dev_num_(1) { + bp_cost_ptr_ = std::make_shared(); + } + + ~BatchParallelInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return bp_cost_ptr_; } + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status GetAttrs() override; + Strategys GetOutputsStrategy(); + Status InferAsLossDivisor() override; + + private: + int32_t dev_num_; + BatchParallelCostPtr bp_cost_ptr_; +}; + +class SparseSoftmaxCrossEntropyWithLogitsInfo : public BatchParallelInfo { + public: + SparseSoftmaxCrossEntropyWithLogitsInfo(const std::string& name, const Shapes& inputs_shape, + const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : BatchParallelInfo(name, inputs_shape, outputs_shape, attrs) {} + ~SparseSoftmaxCrossEntropyWithLogitsInfo() override = default; + void ReComputeBatchSplitFlagList() override; +}; + +class GatherV2Info : public BatchParallelInfo { + public: + GatherV2Info(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : BatchParallelInfo(name, inputs_shape, outputs_shape, attrs) {} + ~GatherV2Info() override = default; + void ReComputeBatchSplitFlagList() override; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_BATCH_PARALLEL_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/bias_add_info.cc b/mindspore/ccsrc/parallel/ops_info/bias_add_info.cc new file mode 100644 index 00000000000..f07606e4d33 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/bias_add_info.cc @@ -0,0 +1,261 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/bias_add_info.h" + +#include +#include +#include +#include + +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +Status BiasAddInfo::CheckStrategy(const StrategyPtr &strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + std::vector stra = strategy->GetInputDim(); + Dimensions sub_a_strategy = stra.at(0); + Dimensions sub_b_strategy = stra.at(1); + int32_t channel_a_strategy = sub_a_strategy.at(1); + int32_t channel_b_strategy = sub_b_strategy.at(0); + if (channel_a_strategy != channel_b_strategy) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + return SUCCESS; +} + +Status BiasAddInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions sub_a_strategy = stra.at(0); + dev_matrix_shape_ = sub_a_strategy; + return SUCCESS; +} + +void BiasAddInfo::ReComputeBatchSplitFlagList() { + split_flag_list_[0] = true; + split_flag_list_[1] = false; +} + +Status BiasAddInfo::InferTensorMap() { + TensorMap sub_a_tensor_map; + TensorMap sub_b_tensor_map; + std::vector stra = strategy_->GetInputDim(); + Dimensions sub_a_strategy = stra.at(0); + size_t sub_a_strategy_size = sub_a_strategy.size(); + for (size_t i = 0; i < sub_a_strategy_size; ++i) { + sub_a_tensor_map.push_back((int32_t)(LAST_INDEX(SizeToUint(sub_a_strategy_size)) - i)); + } + sub_b_tensor_map.push_back((int32_t)(LAST_INDEX(SizeToUint(sub_a_strategy_size)) - 1)); + + inputs_tensor_map_.push_back(sub_a_tensor_map); + inputs_tensor_map_.push_back(sub_b_tensor_map); + outputs_tensor_map_.push_back(sub_a_tensor_map); + + return SUCCESS; +} + +Status BiasAddInfo::InferMirrorOps() { + mirror_ops_.clear(); + Shape input_a_tensor_map = inputs_tensor_map_.at(0); + Shape input_b_tensor_map = inputs_tensor_map_.at(1); + std::vector input_a_group, input_b_group; + if (CreateGroupByTensorMap(input_a_tensor_map, &input_a_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group for input a failed."; + return FAILED; + } + if (CreateGroupByTensorMap(input_b_tensor_map, &input_b_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group for input b failed."; + return FAILED; + } + + OperatorVector op_for_input_a, op_for_input_b; + if (input_a_group.empty() && input_b_group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror group is empty."; + return SUCCESS; + } + if (!input_a_group.empty()) { + op_for_input_a = CreateMirrorOps(input_a_group[0].name(), input_a_group[0].GetDevNum()); + MS_LOG(INFO) << name_ << " : Create the mirror ops for input a success, group is " << input_a_group[0].name(); + } + if (!input_b_group.empty()) { + op_for_input_b = CreateMirrorOps(input_b_group[0].name(), input_b_group[0].GetDevNum()); + MS_LOG(INFO) << name_ << " : Create the mirror ops for input b success, group is " << input_b_group[0].name(); + } + mirror_ops_.push_back(op_for_input_a); + mirror_ops_.push_back(op_for_input_b); + + return SUCCESS; +} + +Status BiasAddInfo::InferTensorLayout(TensorLayouts *inputs_layout, TensorLayouts *outputs_layout, + const Shape &dev_matrix_array) { + if ((inputs_layout == nullptr) || (outputs_layout == nullptr)) { + MS_LOG(ERROR) << name_ << " : The layout is null."; + return FAILED; + } + TensorMap input_a_tensor_map_array = inputs_tensor_map_.at(0); + TensorMap input_b_tensor_map_array = inputs_tensor_map_.at(1); + TensorMap out_tensor_map_array = outputs_tensor_map_.at(0); + Shape input_a_shape_array = inputs_shape_.at(0); + Shape input_b_shape_array = inputs_shape_.at(1); + Shape out_shape_array = outputs_shape_.at(0); + + TensorLayout input_a_tensor_layout, input_b_tensor_layout, out_tensor_layout; + if (input_a_tensor_layout.InitFromVector(dev_matrix_array, input_a_tensor_map_array, input_a_shape_array) != + SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create tensor layout for input a failed."; + return FAILED; + } + if (input_b_tensor_layout.InitFromVector(dev_matrix_array, input_b_tensor_map_array, input_b_shape_array) != + SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create tensor layout for input b failed."; + return FAILED; + } + if (out_tensor_layout.InitFromVector(dev_matrix_array, out_tensor_map_array, out_shape_array) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create tensor layout for output failed."; + return FAILED; + } + inputs_layout->push_back(input_a_tensor_layout); + inputs_layout->push_back(input_b_tensor_layout); + outputs_layout->push_back(out_tensor_layout); + + return SUCCESS; +} + +Status BiasAddInfo::InferTensorInfo() { + // infer tensor shape + Shape input_a_shape = inputs_shape_.at(0); + Shape input_b_shape = inputs_shape_.at(1); + Shape output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = {inputs_strategy.at(0)}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_a_slice_shape = inputs_slice_shape.at(0); + Shape input_b_slice_shape = inputs_slice_shape.at(1); + Shape output_slice_shape = outputs_slice_shape.at(0); + + // infer tensor layout + TensorLayouts inputs_layout, outputs_layout; + if (InferTensorLayout(&inputs_layout, &outputs_layout, dev_matrix_shape_) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Infer tensor layout failed."; + return FAILED; + } + + TensorInfo input_a_tensor_info(inputs_layout.at(0), input_a_shape, input_a_slice_shape); + TensorInfo input_b_tensor_info(inputs_layout.at(1), input_b_shape, input_b_slice_shape); + TensorInfo out_tensor_info(outputs_layout.at(0), output_shape, output_slice_shape); + + inputs_tensor_info_.push_back(input_a_tensor_info); // inputs_a + inputs_tensor_info_.push_back(input_b_tensor_info); // inputs_b + outputs_tensor_info_.push_back(out_tensor_info); // output + + return SUCCESS; +} + +Status BiasAddInfo::SetCostUnderStrategy(const StrategyPtr &strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status BiasAddInfo::GenerateStrategies(int32_t stage_id) { + Shape input0_split(inputs_shape_[0].size(), 1); + Shapes splittable_inputs = {input0_split, input0_split}; + + std::vector sp_vector; + is_auto_parallel_ = true; + Shapes tmp_inputs_shape = {inputs_shape_[0], inputs_shape_[0]}; + Shapes tmp_splittable_inputs = {splittable_inputs[0], splittable_inputs[0]}; + if (GenerateStrategiesForIndependentInputs(stage_id, tmp_inputs_shape, tmp_splittable_inputs, &sp_vector) != + SUCCESS) { + return FAILED; + } + MS_LOG(INFO) << name_ << " : Generate strategies with broadcast success."; + + for (auto &sp : sp_vector) { + std::vector tmp_strategy; + Dimensions input0_strategy = sp->GetInputDim()[0]; + tmp_strategy.push_back(input0_strategy); // input0 + + Dimensions input1_strategy = {input0_strategy.at(1)}; + + // reset the strategy + tmp_strategy.push_back(input1_strategy); // input1 + sp->ResetInputs(tmp_strategy); + } + size_t success = 0; + for (auto &sp : sp_vector) { + PrintStrategy(sp); + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << " : Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} + +Status BiasAddInfo::Init(const StrategyPtr &strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status BiasAddInfo::InitForCostModel(const StrategyPtr &strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/bias_add_info.h b/mindspore/ccsrc/parallel/ops_info/bias_add_info.h new file mode 100644 index 00000000000..ac263ec91f1 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/bias_add_info.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_BIAS_ADD_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_BIAS_ADD_INFO_H_ + +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class BiasAddInfo : public OperatorInfo { + public: + BiasAddInfo(const std::string& operator_name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs) { + biasaddcost_ptr_ = std::make_shared(); + } + ~BiasAddInfo() override = default; + + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + Status GenerateStrategies(int32_t) override; + Status SetCostUnderStrategy(const StrategyPtr&) override; + OperatorCostPtr GetOperatorCost() const override { return biasaddcost_ptr_; } + void ReComputeBatchSplitFlagList() override; + + protected: + Status GetAttrs() override { return SUCCESS; } + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override { return SUCCESS; } + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout, const Shape& dev_matrix_array); + ArithmeticCostPtr biasaddcost_ptr_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_BIAS_ADD_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/comparison_function_info.h b/mindspore/ccsrc/parallel/ops_info/comparison_function_info.h new file mode 100644 index 00000000000..2762f94eb93 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/comparison_function_info.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_COMPARISON_FUNCTION_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_COMPARISON_FUNCTION_INFO_H_ + +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/arithmetic_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class EqualInfo : public ArithmeticBase { + public: + EqualInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~EqualInfo() override = default; +}; + +class NotEqualInfo : public ArithmeticBase { + public: + NotEqualInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~NotEqualInfo() override = default; +}; + +class MaximumInfo : public ArithmeticBase { + public: + MaximumInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ArithmeticBase(name, inputs_shape, outputs_shape, attrs) {} + ~MaximumInfo() override = default; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_OPS_INFO_PARALLEL_COMPARISON_FUNCTION_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/dropout_do_mask_info.cc b/mindspore/ccsrc/parallel/ops_info/dropout_do_mask_info.cc new file mode 100644 index 00000000000..1b77b913da0 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/dropout_do_mask_info.cc @@ -0,0 +1,175 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/dropout_do_mask_info.h" + +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" +#include "parallel/auto_parallel/costmodel.h" + +namespace mindspore { +namespace parallel { +Status DropoutDoMaskInfo::CheckStrategy(const StrategyPtr& strategy) { + Shapes input_shape = {inputs_shape_.at(0)}; + if (CheckStrategyValue(strategy, input_shape, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + return SUCCESS; +} + +Status DropoutDoMaskInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + + dev_matrix_shape_ = input_strategy; + + return SUCCESS; +} + +Status DropoutDoMaskInfo::InferTensorMap() { + std::vector tensor_map_index; + size_t size = inputs_shape_.at(0).size(); + // such as 4: tensor_map_index [3,2,1,0] + for (size_t i = 0; i < size; ++i) { + tensor_map_index.push_back((int32_t)(LAST_INDEX(size) - i)); + } + + TensorMap input_b_tensor_map = {MAP_NONE}; + inputs_tensor_map_.push_back(tensor_map_index); + inputs_tensor_map_.push_back(input_b_tensor_map); + outputs_tensor_map_.push_back(tensor_map_index); + return SUCCESS; +} + +Status DropoutDoMaskInfo::InferTensorInfo() { + // infer tensor shape + Shape input_a_shape = inputs_shape_.at(0); + Shape input_b_shape = inputs_shape_.at(1); + Shape output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Dimensions input_b_strategy = {1}, input_x_strategy = {}; + inputs_strategy.emplace_back(input_b_strategy); + inputs_strategy.emplace_back(input_x_strategy); + Strategys outputs_strategy = {inputs_strategy.at(0)}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_a_slice_shape = inputs_slice_shape.at(0); + Shape input_b_slice_shape = inputs_slice_shape.at(1); + Shape output_slice_shape = outputs_slice_shape.at(0); + + TensorLayout input_a_tensor_layout, input_b_tensor_layout; + TensorLayout output_tensor_layout; + if (input_a_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], input_a_shape) != SUCCESS) { + return FAILED; + } + if (input_b_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[1], input_b_shape) != SUCCESS) { + return FAILED; + } + if (output_tensor_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], output_shape) != SUCCESS) { + return FAILED; + } + TensorInfo input_a_tensor_info(input_a_tensor_layout, input_a_shape, input_a_slice_shape); + TensorInfo input_b_tensor_info(input_b_tensor_layout, input_b_shape, input_b_slice_shape); + TensorInfo output_tensor_info(output_tensor_layout, output_shape, output_slice_shape); + + inputs_tensor_info_.push_back(input_a_tensor_info); + inputs_tensor_info_.push_back(input_b_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + + return SUCCESS; +} + +Status DropoutDoMaskInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status DropoutDoMaskInfo::GenerateStrategies(int32_t stage_id) { + CheckGlobalDeviceManager(); + is_auto_parallel_ = true; + size_t dev_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + Dimensions strategy(inputs_shape_[0].size() - 1, 1); + (void)strategy.insert(strategy.begin(), SizeToInt(dev_num)); + std::vector stra = {strategy}; + StrategyPtr sp = std::make_shared(stage_id, stra); + if (SetCostUnderStrategy(sp) == SUCCESS) { + MS_LOG(INFO) << name_ << " : Successfully generated batch-parallel-strategy."; + PrintStrategy(sp); + } else { + MS_LOG(ERROR) << name_ << " : Generating batch-parallel-strategy failed."; + return FAILED; + } + return SUCCESS; +} + +std::shared_ptr>> DropoutDoMaskInfo::GenerateBatchStrategies() { + CheckGlobalDeviceManager(); + size_t dev_num = g_device_manager->GetDeviceListByStageId(0).size(); + Dimensions strategy(inputs_shape_[0].size() - 1, 1); + (void)strategy.insert(strategy.begin(), SizeToInt(dev_num)); + std::vector strategy_v = {strategy}; + return std::make_shared>>(strategy_v); +} + +Status DropoutDoMaskInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status DropoutDoMaskInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/dropout_do_mask_info.h b/mindspore/ccsrc/parallel/ops_info/dropout_do_mask_info.h new file mode 100644 index 00000000000..2fc18b521df --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/dropout_do_mask_info.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_DROPOUT_DO_MASK_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_DROPOUT_DO_MASK_INFO_H_ + +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class DropoutDoMaskInfo : public OperatorInfo { + public: + DropoutDoMaskInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + bpcost_ptr_ = std::make_shared(); + } + ~DropoutDoMaskInfo() override = default; + + Status Init(const StrategyPtr& strategy) override; + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return bpcost_ptr_; } + Status InitForCostModel(const StrategyPtr& strategy) override; + std::shared_ptr>> GenerateBatchStrategies() override; + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override { return SUCCESS; } + Status InferForwardCommunication() override { return SUCCESS; } + Status InferTensorMap() override; + Status GetAttrs() override { return SUCCESS; } + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + + private: + BatchParallelCostPtr bpcost_ptr_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_DROPOUT_DO_MASK_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/elementary_function_info.cc b/mindspore/ccsrc/parallel/ops_info/elementary_function_info.cc new file mode 100644 index 00000000000..d4f79aca652 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/elementary_function_info.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/elementary_function_info.h" + +namespace mindspore { +namespace parallel { +Status PowInfo::InferMirrorOps() { + mirror_ops_.clear(); + + Shape tensor_map = inputs_tensor_map_[0]; + std::vector group; + if (CreateGroupByTensorMap(tensor_map, &group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group failed."; + return FAILED; + } + + OperatorVector mirror_op; + OperatorVector op_for_value; + if (group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror ops is empty."; + return SUCCESS; + } else { + mirror_op = CreateMirrorOps(group[0].name(), group[0].GetDevNum()); + mirror_ops_.push_back(mirror_op); + mirror_ops_.push_back(op_for_value); + std::string group_name = group[0].name(); + MS_LOG(INFO) << name_ << " : Create the mirror ops success, the group name is " << group_name; + } + + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/elementary_function_info.h b/mindspore/ccsrc/parallel/ops_info/elementary_function_info.h new file mode 100644 index 00000000000..cf85d28fb0c --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/elementary_function_info.h @@ -0,0 +1,80 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ELEMENTARY_FUNCTION_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ELEMENTARY_FUNCTION_INFO_H_ + +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" + +namespace mindspore { +namespace parallel { +class PowInfo : public ActivationOther { + public: + PowInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~PowInfo() override = default; + + protected: + Status InferMirrorOps() override; +}; + +class ExpInfo : public ActivationOther { + public: + ExpInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~ExpInfo() override = default; +}; + +class LogInfo : public ActivationOther { + public: + LogInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~LogInfo() override = default; +}; + +class CosInfo : public ActivationOther { + public: + CosInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~CosInfo() override = default; +}; + +class ACosInfo : public ActivationOther { + public: + ACosInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~ACosInfo() override = default; +}; + +class LogicalNotInfo : public ActivationOther { + public: + LogicalNotInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : ActivationOther(name, inputs_shape, outputs_shape, attrs) {} + ~LogicalNotInfo() override = default; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ELEMENTARY_FUNCTION_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/generator_info.cc b/mindspore/ccsrc/parallel/ops_info/generator_info.cc new file mode 100644 index 00000000000..b0fd0eaa9ad --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/generator_info.cc @@ -0,0 +1,188 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/generator_info.h" + +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +Status GeneratorBase::InferTensorMap() { + TensorMap output_tensor_map = {MAP_NONE}; + outputs_tensor_map_.push_back(output_tensor_map); + return SUCCESS; +} + +Status GeneratorBase::InferTensorInfo() { + Shape output_shape = outputs_shape_.at(0); + Shape output_slice_shape = outputs_shape_.at(0); + + TensorLayout output_tensor_layout; + if (output_tensor_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], output_shape) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Creat output tensor layout failed."; + return FAILED; + } + TensorInfo output_tensor_info(output_tensor_layout, output_shape, output_slice_shape); + outputs_tensor_info_.push_back(output_tensor_info); + + return SUCCESS; +} + +Status GeneratorBase::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + + dev_matrix_shape_ = input_strategy; + + return SUCCESS; +} + +Status GeneratorBase::SetCostUnderStrategy(const StrategyPtr &strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status DropoutGenMaskInfo::GenerateStrategies(int32_t stage_id) { + if (input_value_.empty()) { + MS_LOG(ERROR) << name_ << " : Input value is empty."; + return FAILED; + } + Shape param = GetValue>(input_value_[0]); + if (param.empty()) { + MS_LOG(ERROR) << name_ << " : Input value [0] is empty."; + return FAILED; + } + // Now,only support batch parallel. + CheckGlobalDeviceManager(); + is_auto_parallel_ = true; + size_t dev_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + Dimensions strategy(param.size() - 1, 1); + (void)strategy.insert(strategy.begin(), SizeToInt(dev_num)); + std::vector stra = {strategy}; + StrategyPtr sp = std::make_shared(stage_id, stra); + if (SetCostUnderStrategy(sp) == SUCCESS) { + MS_LOG(INFO) << name_ << " : Successfully generated batch-parallel-strategy."; + PrintStrategy(sp); + } else { + MS_LOG(ERROR) << name_ << " : Generating batch-parallel-strategy failed."; + return FAILED; + } + return SUCCESS; +} + +Status DropoutGenMaskInfo::CheckStrategy(const StrategyPtr &strategy) { + if (strategy->GetInputNumber() != 1) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : The strategy is wrong."; + } else { + MS_LOG(ERROR) << name_ << " : The strategy is wrong."; + } + return FAILED; + } + + return SUCCESS; +} + +Status DropoutGenMaskInfo::InferReplaceOps(const StrategyPtr &strategy) { + Shape shape = GetValue>(input_value_[0]); + Strategys stra = strategy->GetInputDim(); + Dimensions input_strategy = stra.at(0); + int32_t dev_num = *(input_strategy.begin()); + if (dev_num <= 0) { + MS_LOG(ERROR) << name_ << " : The number of devices should not be less than 0."; + return FAILED; + } + // Batch parallel + if (shape[0] % dev_num != 0) { + MS_LOG(ERROR) << name_ << " : The shape " << shape[0] << " can't be exact divided by device number " << dev_num; + return FAILED; + } + shape[0] = shape[0] / dev_num; + ValuePtr shape_ptr = MakeValue(shape); + Attr attr_0 = std::make_pair(SEED0, attrs_[SEED0]); + Attr attr_1 = std::make_pair(SEED1, attrs_[SEED1]); + OperatorAttrs attrs = {attr_0, attr_1}; + Attr param_0 = std::make_pair(SHAPE, shape_ptr); + Attr param_1 = std::make_pair(KEEP_PROB, input_value_[1]); + OperatorParams params = {std::make_pair(param_0, 1), std::make_pair(param_1, 2)}; + OperatorArgs args = std::make_pair(attrs, params); + replace_op_ = {std::make_pair(DROPOUT_GEN_MASK, args)}; + return SUCCESS; +} + +std::shared_ptr>> DropoutGenMaskInfo::GenerateBatchStrategies() { + if (input_value_.empty()) { + MS_LOG(EXCEPTION) << name_ << " : Input value is empty."; + } + Shape param = GetValue>(input_value_[0]); + if (param.empty()) { + MS_LOG(EXCEPTION) << name_ << " : Input value [0] is empty."; + } + // Now,only support batch parallel. + CheckGlobalDeviceManager(); + size_t dev_num = g_device_manager->GetDeviceListByStageId(0).size(); + Dimensions strategy(param.size() - 1, 1); + (void)strategy.insert(strategy.begin(), SizeToInt(dev_num)); + std::vector strategy_v = {strategy}; + return std::make_shared>>(strategy_v); +} + +Status GeneratorBase::Init(const StrategyPtr &strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + + if (InferReplaceOps(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Infer replace ops failed."; + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status GeneratorBase::InitForCostModel(const StrategyPtr &strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/generator_info.h b/mindspore/ccsrc/parallel/ops_info/generator_info.h new file mode 100644 index 00000000000..0f1d7209c82 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/generator_info.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GENERATOR_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GENERATOR_INFO_H_ + +#include +#include +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class GeneratorBase : public OperatorInfo { + public: + GeneratorBase(const std::string &operator_name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs) { + generatorbasecost_ptr_ = std::make_shared(); + } + + ~GeneratorBase() override = default; + + Status Init(const StrategyPtr &strategy) override; + Status SetCostUnderStrategy(const StrategyPtr &strategy) override; + OperatorCostPtr GetOperatorCost() const override { return generatorbasecost_ptr_; } + Status InitForCostModel(const StrategyPtr &strategy) override; + + protected: + // For now, generator ops don't have attributes + Status GetAttrs() override { return Status::SUCCESS; } + Status InferTensorMap() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferMirrorOps() override { return SUCCESS; } + Status InferForwardCommunication() override { return SUCCESS; } + virtual Status InferReplaceOps(const StrategyPtr &strategy) = 0; + GeneratorBaseCostPtr generatorbasecost_ptr_; +}; + +class DropoutGenMaskInfo : public GeneratorBase { + public: + DropoutGenMaskInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : GeneratorBase(name, inputs_shape, outputs_shape, attrs) {} + ~DropoutGenMaskInfo() override = default; + Status GenerateStrategies(int32_t stage_id) override; + std::shared_ptr>> GenerateBatchStrategies() override; + + protected: + Status CheckStrategy(const StrategyPtr &strategy) override; + Status InferReplaceOps(const StrategyPtr &strategy) override; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GENERATOR_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/get_next_info.cc b/mindspore/ccsrc/parallel/ops_info/get_next_info.cc new file mode 100644 index 00000000000..f38baa1e4e4 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/get_next_info.cc @@ -0,0 +1,256 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/get_next_info.h" + +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +Status GetNextInfo::InferTensorMap() { + for (auto shp : shapes_) { + TensorMap out_tensor_map; + for (size_t i = 0; i < shp.size(); ++i) { + out_tensor_map.push_back(SizeToInt(dev_matrix_shape_.size() - i - 1)); + } + outputs_tensor_map_.push_back(out_tensor_map); + } + return SUCCESS; +} + +Status GetNextInfo::InferTensorLayout(TensorLayouts* outputs_layout) { + if (outputs_layout == nullptr) { + MS_LOG(ERROR) << name_ << " : The layout is null."; + return FAILED; + } + for (size_t i = 0; i < outputs_shape_.size(); ++i) { + TensorLayout output_layout; + if (output_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[i], outputs_shape_[i]) != SUCCESS) { + return FAILED; + } + outputs_layout->push_back(output_layout); + } + return SUCCESS; +} + +Strategys GetNextInfo::GetOutputStrategy() { + Strategys outputs_strategy; + for (auto shp : shapes_) { + Dimensions out_strategy; + out_strategy.push_back(dev_num_); + for (size_t i = 1; i < shp.size(); ++i) { + out_strategy.push_back(1); + } + outputs_strategy.push_back(out_strategy); + } + return outputs_strategy; +} + +Status GetNextInfo::InferTensorInfo() { + TensorLayouts outputs_layout; + if (InferTensorLayout(&outputs_layout) != SUCCESS) { + return FAILED; + } + for (size_t i = 0; i < outputs_shape_.size(); ++i) { + TensorInfo output_tensor_info(outputs_layout[i]); + outputs_tensor_info_.push_back(output_tensor_info); + } + return SUCCESS; +} + +Status GetNextInfo::InferDevMatrixShape() { + size_t max_shape_length = 0; + for (auto shp : shapes_) { + if (max_shape_length < shp.size()) { + max_shape_length = shp.size(); + } + } + if (max_shape_length == 0) { + MS_LOG(ERROR) << name_ << " : shape is 0"; + } + dev_matrix_shape_.push_back(dev_num_); + for (size_t i = 1; i < max_shape_length; ++i) { + dev_matrix_shape_.push_back(1); + } + return SUCCESS; +} + +Status GetNextInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed"; + return FAILED; + } + if (InferReplaceOps(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Infer replace Ops failed"; + return FAILED; + } + MS_LOG(INFO) << name_ << " : Init success"; + return SUCCESS; +} + +Status GetNextInfo::CheckStrategy(const StrategyPtr& strategy) { + std::vector stras = strategy->GetInputDim(); + for (Dimensions stra : stras) { + if (stra.size() != 0) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + } + int32_t stage = strategy->GetInputStage(); + int32_t dev_num = SizeToInt(g_device_manager->GetDeviceListByStageId(stage).size()); + dev_num_ = dev_num; + return SUCCESS; +} + +Status GetNextInfo::GetAttrTypes() { + auto iter = attrs_.find(TYPES); + if (iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + auto iter_cast = iter->second->cast(); + MS_EXCEPTION_IF_NULL(iter_cast); + auto types = iter_cast->value(); + for (auto& type : types) { + MS_EXCEPTION_IF_NULL(type); + types_.push_back(type->ToString()); + } + } else if (iter->second->isa()) { + auto iter_cast = iter->second->cast(); + MS_EXCEPTION_IF_NULL(iter_cast); + auto types = iter_cast->value(); + for (auto& type : types) { + MS_EXCEPTION_IF_NULL(type); + types_.push_back(type->ToString()); + } + } else { + MS_LOG(ERROR) << name_ << " : The value of types is not list."; + return FAILED; + } + } + return SUCCESS; +} + +Status GetNextInfo::GetAttrShapes() { + shapes_ = outputs_shape_; + if (shapes_.size() == 0) { + MS_LOG(ERROR) << name_ << " : Shape is None."; + return FAILED; + } + return SUCCESS; +} + +Status GetNextInfo::GetAttrOutPutNum() { + auto iter = attrs_.find(GETNEXT_NUM); + if (iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + output_num_ = iter->second->cast()->value(); + } else { + MS_LOG(ERROR) << name_ << " : The value of output_num is not int."; + return FAILED; + } + } + return SUCCESS; +} + +Status GetNextInfo::GetAttrs() { + if (GetAttrTypes() == FAILED || GetAttrShapes() == FAILED || GetAttrOutPutNum() == FAILED) { + return FAILED; + } + if (types_.size() != IntToSize(output_num_) || shapes_.size() != IntToSize(output_num_) || output_num_ == 0) { + MS_LOG(ERROR) << name_ << " : The output_num is not equal to shapes size."; + return FAILED; + } + return SUCCESS; +} + +Status GetNextInfo::InferReplaceOps(const StrategyPtr&) { + Shapes out_shapes = outputs_shape_; + for (size_t i = 0; i < out_shapes.size(); ++i) { + if (dev_num_ <= 0) { + MS_LOG(ERROR) << name_ << " : The dev num is 0."; + return FAILED; + } + if (out_shapes[i][0] % dev_num_ != 0) { + MS_LOG(ERROR) << name_ << " : batch num cannot floor div dev num."; + return FAILED; + } + out_shapes[i][0] = out_shapes[i][0] / dev_num_; + } + ValuePtr new_shapes = MakeValue(out_shapes); + Attr attr_types = std::make_pair(TYPES, attrs_[TYPES]); + Attr attr_shapes = std::make_pair(SHAPES, new_shapes); + Attr attr_num = std::make_pair(GETNEXT_NUM, attrs_[GETNEXT_NUM]); + Attr attr_shared_name = std::make_pair(SHARED_NAME, attrs_[SHARED_NAME]); + OperatorAttrs attrs = {attr_types, attr_shapes, attr_num, attr_shared_name}; + OperatorParams params; + OperatorArgs args = std::make_pair(attrs, params); + replace_op_ = {std::make_pair(GET_NEXT, args)}; + return SUCCESS; +} + +Status GetNextInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} + +Status GetNextInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + return SUCCESS; +} + +Status GetNextInfo::GenerateStrategies(int32_t stage_id) { + is_auto_parallel_ = true; + std::vector stra; + StrategyPtr sp = std::make_shared(stage_id, stra); + if (SetCostUnderStrategy(sp) == SUCCESS) { + MS_LOG(INFO) << name_ << " : Successfully generated strategy."; + PrintStrategy(sp); + } else { + MS_LOG(ERROR) << name_ << " : Generating strategy failed."; + return FAILED; + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/get_next_info.h b/mindspore/ccsrc/parallel/ops_info/get_next_info.h new file mode 100644 index 00000000000..1280d3b1916 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/get_next_info.h @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GETNEXT_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GETNEXT_INFO_H_ + +#include +#include +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class GetNextInfo : public OperatorInfo { + public: + GetNextInfo(const std::string &operator_name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs) { + getnextcost_ptr_ = std::make_shared(); + } + ~GetNextInfo() override = default; + + Status Init(const StrategyPtr &strategy) override; + Status SetCostUnderStrategy(const StrategyPtr &strategy) override; + OperatorCostPtr GetOperatorCost() const override { return getnextcost_ptr_; } + Status InitForCostModel(const StrategyPtr &strategy) override; + Status GenerateStrategies(int32_t stage_id) override; + + protected: + Status CheckStrategy(const StrategyPtr &strategy) override; + Status GetAttrs() override; + Status InferTensorMap() override; + Status InferTensorLayout(TensorLayouts *outputs_layout); + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferMirrorOps() override { return SUCCESS; } + Status InferForwardCommunication() override { return SUCCESS; } + Status InferReplaceOps(const StrategyPtr &strategy); + Status GetAttrTypes(); + Status GetAttrShapes(); + Status GetAttrOutPutNum(); + Strategys GetOutputStrategy(); + Status InferAsLossDivisor() override { return SUCCESS; } + + private: + int32_t dev_num_ = 1; + std::vector types_; + Shapes shapes_; + int32_t output_num_ = 0; + std::string shared_name_; + GetNextCostPtr getnextcost_ptr_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GETNEXT_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/l2_normalize_info.cc b/mindspore/ccsrc/parallel/ops_info/l2_normalize_info.cc new file mode 100644 index 00000000000..183c5dec458 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/l2_normalize_info.cc @@ -0,0 +1,124 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/l2_normalize_info.h" + +#include +#include +#include +#include + +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +Status L2NormalizeInfo::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(INFO) << name_ << " : Init success."; + } + return FAILED; + } + + std::vector stra = strategy->GetInputDim(); + Dimensions input_strategy = stra.at(0); + int32_t axis_index = axis_; + if (axis_ < 0) { + size_t input_dim = inputs_shape_.at(0).size(); + axis_index = static_cast(input_dim) + axis_; + } + + if (input_strategy[IntToSize(axis_index)] != 1) { + MS_LOG(ERROR) << name_ << " : The dim " << axis_index << " of input strategy must be 1."; + return FAILED; + } + + return SUCCESS; +} + +Status L2NormalizeInfo::GetAttrs() { + auto iter = attrs_.find(AXIS); + if (iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + axis_ = iter->second->cast()->value(); + } else { + MS_LOG(ERROR) << name_ << " : The value of axis is not int."; + return FAILED; + } + } + + return SUCCESS; +} + +Status L2NormalizeInfo::InferMirrorOps() { + mirror_ops_.clear(); + Shape input_tensor_map = inputs_tensor_map_.at(0); + std::vector input_group; + if (CreateGroupByTensorMap(input_tensor_map, &input_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Create group failed."; + return FAILED; + } + + OperatorVector op_for_weight; + if (input_group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror ops is empty."; + return SUCCESS; + } else { + op_for_weight = CreateMirrorOps(input_group[0].name(), input_group[0].GetDevNum()); + mirror_ops_.push_back(op_for_weight); + MS_LOG(INFO) << name_ << " : Create the mirror ops success, the group is " << input_group[0].name(); + } + + return SUCCESS; +} + +Status L2NormalizeInfo::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << " : GetAttrs failed."; + return FAILED; + } + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size() - 1, 1); + int32_t axis_index = axis_; + if (axis_ < 0) { + size_t input_dim = inputs_shape_.at(0).size(); + axis_index = static_cast(input_dim) + axis_; + } + (void)input0_split.insert(input0_split.begin() + axis_index, 0); + Shapes splittable_inputs = {input0_split}; + + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Generate strategies failed."; + return FAILED; + } + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << " : Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/l2_normalize_info.h b/mindspore/ccsrc/parallel/ops_info/l2_normalize_info.h new file mode 100644 index 00000000000..10bf46d8fff --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/l2_normalize_info.h @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_L2_NORMALIZE_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_L2_NORMALIZE_INFO_H_ + +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/strategy.h" +#include "parallel/auto_parallel/operator_costmodel.h" + +namespace mindspore { +namespace parallel { +class L2NormalizeInfo : public Activation { + public: + L2NormalizeInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : Activation(name, inputs_shape, outputs_shape, attrs) { + l2normalizecost_ptr_ = std::make_shared(); + } + ~L2NormalizeInfo() override = default; + Status GenerateStrategies(int32_t stage_id) override; + OperatorCostPtr GetOperatorCost() const override { return l2normalizecost_ptr_; } + + protected: + Status GetAttrs() override; + Status InferMirrorOps() override; + Status CheckStrategy(const StrategyPtr& strategy) override; + + private: + int32_t axis_ = 0; // Default value = 0 + L2NormalizeCostPtr l2normalizecost_ptr_; +}; +} // namespace parallel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_L2_NORMALIZE_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/loss_info.cc b/mindspore/ccsrc/parallel/ops_info/loss_info.cc new file mode 100644 index 00000000000..5ca383ebb54 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/loss_info.cc @@ -0,0 +1,231 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/loss_info.h" + +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { +Status SoftmaxCrossEntropyWithLogitsInfo::CheckStrategy(const mindspore::parallel::StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + + std::vector stra = strategy->GetInputDim(); + Dimensions input_strategy = stra.at(0); + Dimensions label_strategy = stra.at(1); + if (input_strategy != label_strategy) { + MS_LOG(ERROR) << name_ << " : Strategies of relevant dimensions are not equal."; + return FAILED; + } + + int32_t axis_index = axis_; + if (axis_ < 0) { + size_t input_dim = inputs_shape_.at(0).size(); + axis_index = static_cast(input_dim) + axis_; + } + + int32_t input_axis_strategy = input_strategy.at(IntToSize(axis_index)); + int32_t label_axis_strategy = label_strategy.at(IntToSize(axis_index)); + // Dimension corresponding to axis is un-splittable + if ((input_axis_strategy != MIN_SLICE_NUM) && (label_axis_strategy != MIN_SLICE_NUM)) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ + << " : The strategy corresponding to axis dimension is not 1, input: " << input_axis_strategy + << ", label: " << label_axis_strategy; + } else { + MS_LOG(ERROR) << name_ + << " : The strategy corresponding to axis dimension is not 1, input: " << input_axis_strategy + << ", label: " << label_axis_strategy; + } + return FAILED; + } + + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::GetAttrs() { + if ((inputs_shape_.size() != SoftmaxCrossEntropyWithLogitsInputsSize) || + (outputs_shape_.size() != SoftmaxCrossEntropyWithLogitsOutputsSize)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size or outputs shape size is wrong."; + return FAILED; + } + + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + dev_matrix_shape_ = input_strategy; + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::InferTensorMap() { + std::vector tensor_map_index; + size_t size = inputs_shape_[0].size(); + // such as 4: tensor_map_index [3,2,1,0] + for (size_t i = 0; i < size; ++i) { + tensor_map_index.push_back((int32_t)(size - i - 1)); + } + + std::vector first_output_tensor_map = {tensor_map_index[0]}; + inputs_tensor_map_.push_back(tensor_map_index); // input + inputs_tensor_map_.push_back(tensor_map_index); // label + outputs_tensor_map_.push_back(first_output_tensor_map); // output-0 + outputs_tensor_map_.push_back(tensor_map_index); // output-1 + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + Shape first_output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = {{inputs_strategy[0][0]}, inputs_strategy.at(0)}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_slice_shape = inputs_slice_shape.at(0); + Shape first_output_slice_shape = outputs_slice_shape.at(0); + + TensorMap input_tensor_map = inputs_tensor_map_.at(0); + TensorMap first_output_tensor_map = outputs_tensor_map_.at(0); + + TensorLayout input_tensor_layout, first_output_tensor_layout; + if ((input_tensor_layout.InitFromVector(dev_matrix_shape_, input_tensor_map, input_shape) != SUCCESS) || + (first_output_tensor_layout.InitFromVector(dev_matrix_shape_, first_output_tensor_map, first_output_shape) != + SUCCESS)) { + return FAILED; + } + TensorInfo input_tensor_info(input_tensor_layout, input_shape, input_slice_shape); + TensorInfo first_output_tensor_info(first_output_tensor_layout, first_output_shape, first_output_slice_shape); + + inputs_tensor_info_.push_back(input_tensor_info); // input + inputs_tensor_info_.push_back(input_tensor_info); // label + outputs_tensor_info_.push_back(first_output_tensor_info); // output-0 + outputs_tensor_info_.push_back(input_tensor_info); // output-1 + + return SUCCESS; +} + +// There are two outputs for SoftmaxCrossEntropyWithLogits, and outputs[1] is used for grad and overload the function. +Status SoftmaxCrossEntropyWithLogitsInfo::InferAsLossDivisor() { + if (outputs_tensor_map_.size() != 2) { + MS_LOG(ERROR) << name_ << " : The size of outputs tensor map " << outputs_tensor_map_.size() << " is error."; + return FAILED; + } + as_loss_divisor_ = ComputeRepeatDeviceNumByTensorMap(dev_matrix_shape_, outputs_tensor_map_[1]); + MS_LOG(INFO) << name_ << " : The dev matrix shape is " << ShapeToString(dev_matrix_shape_) + << ", the output tensor map is " << ShapeToString(outputs_tensor_map_[1]) << ", as_loss_divisor_ is " + << as_loss_divisor_; + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} + +void SoftmaxCrossEntropyWithLogitsInfo::ReComputeBatchSplitFlagList() { + for (size_t i = 0; i < inputs_shape_.size(); ++i) { + split_flag_list_[i] = true; + } +} + +Status SoftmaxCrossEntropyWithLogitsInfo::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << " : GetAttrs failed."; + return FAILED; + } + int32_t axis_index = axis_; + if (axis_ < 0) { + size_t input_dim = inputs_shape_[0].size(); + axis_index = static_cast(input_dim) + axis_; + } + is_auto_parallel_ = true; + + Shape input0_split(inputs_shape_[0].size(), 1); + input0_split[IntToSize(axis_index)] = 0; + Shapes splittable_inputs = {input0_split, input0_split}; + std::vector sp_vector; + if (GenerateStrategiesWithBroadcast(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Generate strategies failed."; + return FAILED; + } + + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << " : Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + + return SUCCESS; +} + +Status SoftmaxCrossEntropyWithLogitsInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + PrintStrategy(strategy); + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Set cost under strategy failed."; + } + return FAILED; + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/loss_info.h b/mindspore/ccsrc/parallel/ops_info/loss_info.h new file mode 100644 index 00000000000..70de84fda3a --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/loss_info.h @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_LOSS_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_LOSS_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +// infer shape: +// input_0 : [a, b], input_1 : [a, b] +// output_0 : [a], output_1: [a, b] +class SoftmaxCrossEntropyWithLogitsInfo : public OperatorInfo { + public: + SoftmaxCrossEntropyWithLogitsInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + softmax_loss_cost_ptr_ = std::make_shared(); + } + ~SoftmaxCrossEntropyWithLogitsInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return softmax_loss_cost_ptr_; } + void ReComputeBatchSplitFlagList() override; + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status GetAttrs() override; + Status InferMirrorOps() override { return SUCCESS; } + Status InferForwardCommunication() override { return SUCCESS; } + Status InferTensorMap() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + // There are two outputs for SoftmaxCrossEntropyWithLogits, and outputs[1] is used for grad and overload + // the InferAsLossDivisor. + Status InferAsLossDivisor() override; + SoftmaxCrossEntropyWithLogitsCostPtr softmax_loss_cost_ptr_; + + private: + int32_t axis_ = -1; // default -1 +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_LOSS_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/matmul_info.cc b/mindspore/ccsrc/parallel/ops_info/matmul_info.cc new file mode 100644 index 00000000000..2cd8e0df8f3 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/matmul_info.cc @@ -0,0 +1,622 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/matmul_info.h" + +#include +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/tensor_layout/tensor_redistribution.h" +#include "parallel/device_manager.h" +#include "parallel/auto_parallel/graph_costmodel.h" + +namespace mindspore { +namespace parallel { +void SetDevMatrixShape(const Dimensions& mat_a_strategy, const Dimensions& mat_b_strategy, bool transpose_b, + Shape* dev_matrix_shape) { + MS_EXCEPTION_IF_NULL(dev_matrix_shape); + size_t mat_a_size = mat_a_strategy.size(); + size_t mat_b_size = mat_b_strategy.size(); + if (mat_a_size >= mat_b_size) { + // for example: mat_a_strategy:[2,4,8,16], mat_b_strategy:[4,16,32] + // dev_matrix_shape:[2,4,8,16,32] (transpose_b is false) + + // [2],[4] in the example above + for (size_t i = 0; i < SECOND_FROM_END(mat_a_size); ++i) { + dev_matrix_shape->push_back(mat_a_strategy.at(i)); + } + } else { + // for example: mat_a_strategy:[8,16], mat_b_strategy:[2,4,16,32] + // dev_matrix_shape:[2,4,8,16,32] (transpose_b is false) + + // [2],[4] in the example above + for (size_t i = 0; i < SECOND_FROM_END(mat_b_size); ++i) { + dev_matrix_shape->push_back(mat_b_strategy.at(i)); + } + } + + // [8],[16] in the example above + dev_matrix_shape->push_back(mat_a_strategy.at(SECOND_FROM_END(mat_a_size))); + dev_matrix_shape->push_back(mat_a_strategy.back()); + + // [32] in the example above + if (!transpose_b) { + dev_matrix_shape->push_back(mat_b_strategy.back()); + } else { + dev_matrix_shape->push_back(mat_b_strategy.at(SECOND_FROM_END(mat_b_size))); + } +} + +Status MatMulBase::GetAttrs() { + if (attrs_.size() < MATMUL_ATTRS_SIZE) { + MS_LOG(ERROR) << name_ << " : The size of attrs small than 2."; + return FAILED; + } + + auto transpose_a_iter = attrs_.find(TRANSPOSE_A); + if (transpose_a_iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(transpose_a_iter->second); + if (transpose_a_iter->second->isa()) { + transpose_a_ = transpose_a_iter->second->cast()->value(); + } else { + MS_LOG(ERROR) << name_ << " : The value of transpose_a is not bool."; + return FAILED; + } + } + + auto transpose_b_iter = attrs_.find(TRANSPOSE_B); + if (transpose_b_iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(transpose_b_iter->second); + if (transpose_b_iter->second->isa()) { + transpose_b_ = transpose_b_iter->second->cast()->value(); + } else { + MS_LOG(ERROR) << name_ << " : The value of transpose_a is not bool."; + return FAILED; + } + } + + // infer inputs dimension size + if ((inputs_shape_.size() != MATMUL_INPUTS_SIZE) || (outputs_shape_.size() != MATMUL_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << " : Inputs shape size or outputs shape size is wrong."; + return FAILED; + } + mat_a_dimension_ = inputs_shape_.at(0).size(); + mat_b_dimension_ = inputs_shape_.at(1).size(); + + return SUCCESS; +} + +Status CheckRelevantDimension(const Dimensions& long_strategy, const Dimensions& short_strategy) { + size_t long_size = long_strategy.size(); + size_t short_size = short_strategy.size(); + if (long_size < short_size) { + MS_LOG(ERROR) << "Size error, the size of long strategy is " << long_size << ", the size of short strategy is " + << short_size; + return FAILED; + } + + size_t len_diff = long_size - short_size; + for (size_t j = 0; j < SECOND_FROM_END(short_size); ++j) { + if (long_strategy.at(len_diff + j) != short_strategy.at(j)) { + MS_LOG(ERROR) << "Strategies of relevant dimensions are not equal, long strategy is " + << ShapeToString(long_strategy) << ", short strategy is " << ShapeToString(short_strategy); + return FAILED; + } + } + + return SUCCESS; +} + +Status MatMul::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << " : Invalid strategy."; + } + return FAILED; + } + + std::vector stra = strategy->GetInputDim(); + Dimensions mat_a_strategy = stra.at(0); + Dimensions mat_b_strategy = stra.at(1); + + size_t mat_a_size = mat_a_strategy.size(); + size_t mat_b_size = mat_b_strategy.size(); + if ((mat_a_size != mat_a_dimension_) || (mat_b_size != mat_b_dimension_)) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : The dimensions of mat_a or mat_b's strategy is wrong."; + } else { + MS_LOG(ERROR) << name_ << " : The dimensions of mat_a or mat_b's strategy is wrong."; + } + return FAILED; + } + + // for example: mat_a_strategy:[2,4,8,16], mat_b_strategy:[4,16,32] + // dev_matrix_shape:[2,4,8,16,32] (transpose_b is false) + // [16] in the example above + if (!transpose_b_ && (mat_a_strategy.back() != mat_b_strategy.at(SECOND_FROM_END(mat_b_size)))) { + MS_LOG(ERROR) << name_ << " : Strategies of relevant dimensions are not equal."; + return FAILED; + } else if (transpose_b_ && (mat_a_strategy.back() != mat_b_strategy.back())) { + MS_LOG(ERROR) << name_ << " : Strategies of relevant dimensions are not equal."; + return FAILED; + } + + if (mat_a_size >= mat_b_size) { + if (CheckRelevantDimension(mat_a_strategy, mat_b_strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Strategies of relevant dimensions are not equal."; + return FAILED; + } + } else { + if (CheckRelevantDimension(mat_b_strategy, mat_a_strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Strategies of relevant dimensions are not equal."; + return FAILED; + } + } + + return SUCCESS; +} + +Status MatMulBase::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions mat_a_strategy = stra.at(0); + Dimensions mat_b_strategy = stra.at(1); + + SetDevMatrixShape(mat_a_strategy, mat_b_strategy, transpose_b_, &dev_matrix_shape_); + return SUCCESS; +} + +// all-reduce weight's grad +Status MatMulBase::InferMirrorOps() { + mirror_ops_.clear(); + + Shape mat_b_tensor_map = inputs_tensor_map_[1]; + std::vector mat_b_group; + if (CreateGroupByTensorMap(mat_b_tensor_map, &mat_b_group) != SUCCESS) { + return FAILED; + } + + OperatorVector op_for_inputs; // op_for_inputs is empty + OperatorVector op_for_weight; + + if (mat_b_group.empty()) { + MS_LOG(INFO) << name_ << " : The mirror ops is empty."; + return SUCCESS; + } else { + op_for_weight = CreateMirrorOps(mat_b_group[0].name(), mat_b_group[0].GetDevNum()); + mirror_ops_.push_back(op_for_inputs); + mirror_ops_.push_back(op_for_weight); + MS_LOG(INFO) << name_ << " : Create the mirror ops for weight success, group is " << mat_b_group[0].name(); + } + + return SUCCESS; +} + +Status MatMulBase::InferForwardCommunication() { + forward_op_.clear(); + size_t dimension = dev_matrix_shape_.size(); + size_t relevant_dimension_index = SECOND_FROM_END(dimension); + // Relevant dimension is not split and all reduce is not required + if (dev_matrix_shape_.at(relevant_dimension_index) == MIN_SLICE_NUM) { + MS_LOG(INFO) << name_ << " : Forward all reduce is not required."; + return SUCCESS; + } + + std::vector group_list; + if (CreateGroupByDim(relevant_dimension_index, &group_list) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Infer forward communication, create group failed."; + return FAILED; + } else if (group_list.empty()) { + MS_LOG(INFO) << name_ << " : Forward all reduce is not required."; + return SUCCESS; + } + + Operator op = CreateAllReduceOp(REDUCE_OP_SUM, group_list[0].name()); + forward_op_.push_back(op); + + MS_LOG(INFO) << name_ << " : The group name of forward communication is " << group_list[0].name(); + return SUCCESS; +} + +// dev_matrix_shape: [a, b, c, d, e], then output strategy: [a, b, c, e]; +Dimensions GetOutputStrategy(const Shape& dev_matrix_shape, int32_t repeated_calculation_num) { + Dimensions output_strategy = dev_matrix_shape; + if (repeated_calculation_num > 1) { + // move the first dimension(repeated_calc_num_) + (void)output_strategy.erase(output_strategy.begin()); + } + + // delete the second-to-last element + (void)output_strategy.erase(output_strategy.begin() + + static_cast(SECOND_FROM_END(output_strategy.size()))); + return output_strategy; +} + +Status MatMulBase::InferTensorMap() { + size_t size = dev_matrix_shape_.size(); + if (repeated_calc_num_ > 1) { + // move the first dimension(repeated_calc_num_), just for the convenience of tensor-map's calculation + size = dev_matrix_shape_.size() - 1; + } + + std::vector tensor_map_index; + // such as 5: tensor_map_index [4,3,2,1,0] + for (size_t i = 0; i < size; ++i) { + tensor_map_index.push_back((int32_t)(LAST_INDEX(size) - i)); + } + + // infer output tensor map: [4,3,2,0], delete the second-from-end element + TensorMap output_tensor_map = tensor_map_index; + (void)output_tensor_map.erase(output_tensor_map.begin() + static_cast(SECOND_FROM_END(size))); + + // infer mat_a tensor map + // for example: mat_a_dimension is 4, mat_a tensor map:[4,3,2,1] + TensorMap mat_a_tensor_map = tensor_map_index; + // delete last one element + mat_a_tensor_map.pop_back(); + // delete the first (dev_matrix_size - 1 - mat_a_dimension) elements + (void)mat_a_tensor_map.erase( + mat_a_tensor_map.begin(), + mat_a_tensor_map.begin() + static_cast(LAST_INDEX(size) - mat_a_dimension_)); + + // infer mat_b tensor map + TensorMap mat_b_tensor_map = tensor_map_index; + // delete the third-to-last element + (void)mat_b_tensor_map.erase(mat_b_tensor_map.begin() + static_cast(THIRD_FROM_END(size))); + // delete the first (dev_matrix_size - 1 - mat_b_dimension) elements + (void)mat_b_tensor_map.erase( + mat_b_tensor_map.begin(), + mat_b_tensor_map.begin() + static_cast(LAST_INDEX(size) - mat_b_dimension_)); + if (transpose_b_) { + // swap the last two elements + int32_t last_value = mat_b_tensor_map.back(); + mat_b_tensor_map.pop_back(); + (void)mat_b_tensor_map.insert( + mat_b_tensor_map.begin() + static_cast(LAST_INDEX(mat_b_tensor_map.size())), last_value); + } + + inputs_tensor_map_.push_back(mat_a_tensor_map); + inputs_tensor_map_.push_back(mat_b_tensor_map); + outputs_tensor_map_.push_back(output_tensor_map); + return SUCCESS; +} + +Status MatMulBase::InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout) { + TensorLayout mat_a_layout, mat_b_layout, output_layout; + if ((mat_a_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], inputs_shape_[0]) != SUCCESS) || + (mat_b_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[1], inputs_shape_[1]) != SUCCESS) || + (output_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], outputs_shape_[0]) != SUCCESS)) { + return FAILED; + } + + inputs_layout->push_back(mat_a_layout); + inputs_layout->push_back(mat_b_layout); + outputs_layout->push_back(output_layout); + return SUCCESS; +} + +Status MatMulBase::InferTensorInfo() { + // infer tensor shape + Shape mat_a_shape = inputs_shape_.at(0); + Shape mat_b_shape = inputs_shape_.at(1); + Shape output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Dimensions output_strategy = GetOutputStrategy(dev_matrix_shape_, repeated_calc_num_); + + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = {output_strategy}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape mat_a_slice_shape = inputs_slice_shape.at(0); + Shape mat_b_slice_shape = inputs_slice_shape.at(1); + Shape output_slice_shape = outputs_slice_shape.at(0); + + // infer tensor layout + TensorLayouts inputs_layout, outputs_layout; + if (InferTensorLayout(&inputs_layout, &outputs_layout) != SUCCESS) { + return FAILED; + } + + TensorLayout mat_a_layout = inputs_layout.at(0); + TensorLayout mat_b_layout = inputs_layout.at(1); + TensorLayout output_layout = outputs_layout.at(0); + TensorInfo mat_a_tensor_info(mat_a_layout, mat_a_shape, mat_a_slice_shape); + TensorInfo mat_b_tensor_info(mat_b_layout, mat_b_shape, mat_b_slice_shape); + TensorInfo output_tensor_info(output_layout, output_shape, output_slice_shape); + + inputs_tensor_info_.push_back(mat_a_tensor_info); + inputs_tensor_info_.push_back(mat_b_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + return SUCCESS; +} + +Status MatMulBase::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << " : Init failed."; + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init success."; + return SUCCESS; +} + +Status MatMulBase::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << " : Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << " : Init for cost model success."; + return SUCCESS; +} + +Status MatMulBase::SwapLastTwoElements(mindspore::parallel::Shape* const input) { + if (input->size() < 2) { + MS_LOG(ERROR) << name_ << " : The size of inputs small than 2."; + return FAILED; + } + auto last_1st_value = input->at(input->size() - 1); + auto last_2nd_value = input->at(input->size() - 2); + input->pop_back(); + input->pop_back(); + input->push_back(last_1st_value); + input->push_back(last_2nd_value); + return SUCCESS; +} + +Status MatMulBase::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << " : GetAttrs failed."; + return FAILED; + } + CheckGlobalDeviceManager(); + std::list dev_list = g_device_manager->GetDeviceListByStageId(stage_id); + size_t dev_num = dev_list.size(); + Shape input0_shape = inputs_shape_[0], input1_shape = inputs_shape_[1]; + if (transpose_a_) { + if (SwapLastTwoElements(&input0_shape) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + } + if (transpose_b_) { + if (SwapLastTwoElements(&input1_shape) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + } + // The shape of input0 (input1) + // E.g., input0 = [100, 200, 300], input1 = [300, 400] + + // Combining the input0_shape and input1_shape + // E.g., combined_shape = [100, 200, 300, 400] + is_auto_parallel_ = true; + size_t input1_shape_size = input1_shape.size(), input0_shape_size = input0_shape.size(); + Dimensions combined_partitions; + Shape combined_shape; + // In SwapLastTwoElements(), it is guaranteed that input0_shape.size() and input1_shape.size() are both larger than 2 + if (input0_shape.size() >= input1_shape.size()) { + combined_shape = input0_shape; + combined_shape.push_back(input1_shape[input1_shape.size() - 1]); + } else { + combined_shape = input1_shape; + combined_shape.push_back(input0_shape[input0_shape.size() - 2]); + } + std::function recursive = [&stage_id, &dev_num, &combined_partitions, &combined_shape, + &input1_shape_size, &recursive, &input0_shape_size, + this](uint32_t current_index, size_t n) { + // Finishing the recursive steps, if the strategy is valid, then calculate the cost + // for this operator under the strategy. + if (current_index == combined_shape.size()) { + StrategyPtr sp; + if (this->PrepareStrategy(stage_id, dev_num, combined_partitions, input0_shape_size, input1_shape_size, &sp) == + FAILED) { + return; + } + if (this->SetCostUnderStrategy(sp) == FAILED) { + MS_LOG(WARNING) << name_ << " : Calculating cost for strategy failed."; + return; + } + } else { + MS_LOG(DEBUG) << name_ << " : The value input0_shape_size: " << input0_shape_size + << ", input1_shape_size: " << input1_shape_size; + for (uint32_t i = 1; i <= n; i *= 2) { + if (n % i == 0 && IntToSize(combined_shape[current_index]) % i == 0) { + combined_partitions.push_back(i); + recursive(current_index + 1, n / i); + combined_partitions.pop_back(); + } + } + } + }; + recursive(0, dev_num); + if (strategy_cost_.empty()) { + MS_LOG(EXCEPTION) << name_ << " : No available strategy."; + } + return Status::SUCCESS; +} + +Status MatMulBase::PrepareStrategy(int32_t stage_id, size_t dev_num, + mindspore::parallel::Dimensions combined_partitions, size_t input0_shape_size, + size_t input1_shape_size, mindspore::parallel::StrategyPtr* const sp) { + int32_t product = std::accumulate(combined_partitions.begin(), combined_partitions.end(), 1, std::multiplies()); + if (NOT_FULLY_USE_DEVICES) { + if (IntToSize(product) > dev_num) { + return FAILED; + } + } else { + if (IntToSize(product) != dev_num) { + return FAILED; + } + } + Dimensions input0_partitions, input1_partitions; + if (input0_shape_size >= input1_shape_size) { + for (size_t i = 0; i < input0_shape_size; ++i) { + input0_partitions.push_back(combined_partitions[i]); + } + if (input1_shape_size == 2) { + input1_partitions.push_back(combined_partitions[combined_partitions.size() - 2]); + input1_partitions.push_back(combined_partitions[combined_partitions.size() - 1]); + } else { + // input1_shape.size() > 2 + for (size_t j = combined_partitions.size() - input1_shape_size - 1; j < combined_partitions.size(); ++j) { + if (j == combined_partitions.size() - 3) { + continue; + } + input1_partitions.push_back(combined_partitions[j]); + } + } + } else { + for (size_t i = 0; i < input1_shape_size; ++i) { + input1_partitions.push_back(combined_partitions[i]); + } + for (size_t j = combined_partitions.size() - input0_shape_size - 1; j < combined_partitions.size() - 3; ++j) { + input0_partitions.push_back(combined_partitions[j]); + } + input0_partitions.push_back(combined_partitions[combined_partitions.size() - 1]); + input0_partitions.push_back(combined_partitions[combined_partitions.size() - 3]); + } + if (transpose_a_) { + if (SwapLastTwoElements(&input0_partitions) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + } + if (transpose_b_) { + if (SwapLastTwoElements(&input1_partitions) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + } + std::vector stras; + stras.push_back(input0_partitions); + stras.push_back(input1_partitions); + (*sp) = std::make_shared(stage_id, stras); + + return SUCCESS; +} + +void MatMulBase::InitTensorInfoForCost(std::vector* relica_inputs_tensor_vector) { + TensorLayout tly; + if (transpose_a_) { + Shape replica_input0_shape(inputs_tensor_info_[0].shape()); + Shape replica_input0_slice_shape(inputs_tensor_info_[0].slice_shape()); + if (SwapLastTwoElements(&replica_input0_shape) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + if (SwapLastTwoElements(&replica_input0_slice_shape) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + + TensorInfo replica_input0_info(tly, replica_input0_shape, replica_input0_slice_shape); + relica_inputs_tensor_vector->push_back(replica_input0_info); + } else { + relica_inputs_tensor_vector->push_back(inputs_tensor_info_[0]); + } + if (transpose_b_) { + Shape replica_input1_shape(inputs_tensor_info_[1].shape()); + Shape replica_input1_slice_shape(inputs_tensor_info_[1].slice_shape()); + if (SwapLastTwoElements(&replica_input1_shape) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + if (SwapLastTwoElements(&replica_input1_slice_shape) == FAILED) { + MS_LOG(ERROR) << name_ << " : Swap last two elements failed."; + } + + TensorInfo replica_input1_info(tly, replica_input1_shape, replica_input1_slice_shape); + relica_inputs_tensor_vector->push_back(replica_input1_info); + } else { + relica_inputs_tensor_vector->push_back(inputs_tensor_info_[1]); + } +} + +Status MatMulBase::CheckForTensorSliceValid() const { + if (!TENSOR_SLICE_ALIGNMENT_ENABLE) { + return SUCCESS; + } + if (inputs_tensor_info_.empty()) { + return FAILED; + } + for (auto& one_input_tensor : inputs_tensor_info_) { + auto slice_shape = one_input_tensor.slice_shape(); + if ((IntToSize(slice_shape[LAST_INDEX(slice_shape.size())]) % TENSOR_SLICE_ALIGNMENT_SIZE != 0) || + (IntToSize(slice_shape[SECOND_FROM_END(slice_shape.size())]) % TENSOR_SLICE_ALIGNMENT_SIZE != 0)) { + return FAILED; + } + } + return SUCCESS; +} + +Status MatMulBase::SetCostUnderStrategy(const mindspore::parallel::StrategyPtr& strategy) { + if (InitForCostModel(strategy) == FAILED) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << " : Initialization under the strategy failed."; + } else { + MS_LOG(ERROR) << name_ << " : Initialization under the strategy failed."; + } + return FAILED; + } + PrintStrategy(strategy); + // Check whether the tensor slice of input_tensor_info is valid or not + if (CheckForTensorSliceValid() != SUCCESS) { + MS_LOG(INFO) << name_ << " : The tensor slice is not valid under this strategy."; + return FAILED; + } + // Here, a replicated inputs_ is constructed for the transposed TensorInfo. + std::vector relica_inputs_tensor_vector; + InitTensorInfoForCost(&relica_inputs_tensor_vector); + + int32_t stage_id = strategy->GetInputStage(); + // Here, we use the origin outputs_, because we only use the slice size of the output tensor. + // It does not matter whether the output tensor is transposed or not. + double memory_cost = + matmulcost_ptr->GetForwardMemoryCost(relica_inputs_tensor_vector, outputs_tensor_info_, stage_id); + double communication_cost = matmulcost_ptr->GetCommCost(relica_inputs_tensor_vector, outputs_tensor_info_, stage_id); + std::shared_ptr result = std::make_shared(memory_cost, communication_cost); + result->communication_without_parameter_ = + matmulcost_ptr->GetForwardCommCost(relica_inputs_tensor_vector, outputs_tensor_info_, stage_id); + result->communication_with_partial_para_ = + result->communication_without_parameter_ + + COST_MODEL_GAMMA * (communication_cost - result->communication_without_parameter_); + + // Breaking ties for preferring data parallelization + BreakingTiesForPerferringDataParallel(strategy, result); + MS_LOG(DEBUG) << name_ << " : memory_cost: " << result->memory_cost_ + << ", communication_cost: " << result->communication_cost_ + << ", communication_without_parameter_: " << result->communication_without_parameter_ + << ", communication_with_partial_para_: " << result->communication_with_partial_para_; + // refine communication cost calculation for practice + RefineForPracticalCost(result, false); + + std::shared_ptr swc = + std::make_shared(strategy, inputs_tensor_info_, outputs_tensor_info_); + swc->cost_list.push_back(result); + strategy_cost_.emplace_back(swc); + + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/matmul_info.h b/mindspore/ccsrc/parallel/ops_info/matmul_info.h new file mode 100644 index 00000000000..daf422e6d65 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/matmul_info.h @@ -0,0 +1,92 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_MATMUL_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_MATMUL_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/strategy.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { +class MatMulBase : public OperatorInfo { + public: + MatMulBase(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + matmulcost_ptr = std::make_shared(); + } + ~MatMulBase() override = default; + + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + // Generate all strategies and the corresponding cost for this MatMul operator + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + Status PrepareStrategy(int32_t stage_id, size_t dev_num, Dimensions combined_partitions, size_t input0_shape_size, + size_t input1_shape_size, StrategyPtr* sp); + + OperatorCostPtr GetOperatorCost() const override { return matmulcost_ptr; } + Status SwapLastTwoElements(Shape* shape); + + protected: + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout); + void InitTensorInfoForCost(std::vector*); + Status CheckForTensorSliceValid() const; + Status GetAttrs() override; + + bool transpose_a_ = false; + bool transpose_b_ = false; + size_t mat_a_dimension_ = 0; + size_t mat_b_dimension_ = 0; + + MatMulCostPtr matmulcost_ptr; +}; + +class MatMul : public MatMulBase { + public: + MatMul(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs) + : MatMulBase(name, inputs_shape, outputs_shape, attrs) {} + ~MatMul() override = default; + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; +}; + +class MatMulInfo : public MatMul { + public: + MatMulInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : MatMul(name, inputs_shape, outputs_shape, attrs) {} + ~MatMulInfo() override = default; +}; +} // namespace parallel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_MATMUL_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/onehot_info.cc b/mindspore/ccsrc/parallel/ops_info/onehot_info.cc new file mode 100644 index 00000000000..9b8e9071b04 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/onehot_info.cc @@ -0,0 +1,311 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/onehot_info.h" + +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/strategy.h" +#include "parallel/auto_parallel/costmodel.h" +#include "parallel/graph_util/generate_graph.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +Status OneHotInfo::GetAttrs() { + auto iter = attrs_.find(AXIS); + if (iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + axis_value_ptr_ = iter->second; + axis_ = iter->second->cast()->value(); + } else { + MS_LOG(ERROR) << name_ << ": The value of axis is not int."; + return FAILED; + } + } + + if (inputs_shape_[0].size() != 1) { + MS_LOG(ERROR) << name_ << ": Input's shape only support 1-D now."; + return FAILED; + } + + if ((axis_ > 1) || (axis_ < -1)) { + MS_LOG(ERROR) << name_ << ": Axis " << axis_ << " is out of range[-1, 1]."; + return FAILED; + } + return SUCCESS; +} + +Status OneHotInfo::CheckStrategy(const StrategyPtr& strategy) { + if (inputs_shape_.size() != 3) { + MS_LOG(ERROR) << name_ << ": inputs_shape_ size must be 3, but is " << inputs_shape_.size(); + return FAILED; + } + if (outputs_shape_.size() != 1) { + MS_LOG(ERROR) << name_ << ": outputs_shape_ size must be 1, but is " << outputs_shape_.size(); + return FAILED; + } + if (CheckStrategyValue(strategy, {outputs_shape_.at(0), inputs_shape_.at(1), inputs_shape_.at(2)}, + is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy."; + } + return FAILED; + } + + return SUCCESS; +} + +Status OneHotInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + + // Now input only support 1-D tensor, so the output is a 2-D tensor + // If input is a vector of length features, the output shape will be: + // [features, depth] if axis == -1 (or axis == 1) + // [depth, features] if axis == 0 + if (axis_ == 0) { + dev_matrix_shape_.push_back(input_strategy[1]); // the depth is un-splittable + dev_matrix_shape_.push_back(input_strategy[0]); // the features is splittable + } else { + dev_matrix_shape_.push_back(input_strategy[0]); // the features is splittable + dev_matrix_shape_.push_back(input_strategy[1]); // the depth is un-splittable + } + + return SUCCESS; +} + +Status OneHotInfo::InferTensorMap() { + std::vector input_tensor_map_index, output_tensor_map_index; + size_t size = outputs_shape_[0].size(); + // such as 2: tensor_map_index [1,0] + if (axis_ == 0) { + for (size_t i = 0; i < size; ++i) { + output_tensor_map_index.push_back((int32_t)(i)); + } + } else { + for (size_t i = 0; i < size; ++i) { + output_tensor_map_index.push_back((int32_t)(LAST_INDEX(size) - i)); + } + } + outputs_tensor_map_.push_back(output_tensor_map_index); + + // Now input only support 1-D tensor + input_tensor_map_index.push_back(1); + + inputs_tensor_map_.push_back(input_tensor_map_index); + return SUCCESS; +} + +// axis = -1 +// (0,(1,16),(),()）reid dev_matrix=(1,16) map_in=(1) map_out=(1,0) +// (0,(16,1),(),()）data parallel dev_matrix=(16,1) map_in=(1) map_out=(1,0) +// (0,(2,8),(),()）16 devices two machines，model parallel among devices in the same machine，data parallel between +// machines dev_matrix=(2,8) map_in=(1) map_out=(1,0) (0, (2,4）,(),()）16 devices dev_matrix=(2,4,2) map_in=(1) +// map_out=(1,0) +// axis = 0 +// (0, (16,1),(),()）reid dev_matrix=(1,16) map_in=(1) map_out=(0,1) +// (0, (1,16),(),()）data parallel dev_matrix=(16,1) map_in=(1) map_out=(0,1) +// (0, (8,2),(),()）16 devices two machines，model parallel among devices in the same machine，data parallel between +// machines dev_matrix=(2,8) map_in=(1) map_out=(0,1) （0，（4,2）,(),()）16 devices dev_matrix=(2,4,2) map_in=(1) +// map_out=(0,1) +Status OneHotInfo::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + Shape output_shape = outputs_shape_.at(0); + + TensorLayout input_tensor_layout, output_tensor_layout; + if ((input_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], input_shape) != SUCCESS) || + (output_tensor_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], output_shape) != SUCCESS)) { + return FAILED; + } + + TensorInfo input_tensor_info(input_tensor_layout); + TensorInfo output_tensor_info(output_tensor_layout); + + inputs_tensor_info_.push_back(input_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + + return SUCCESS; +} + +Status OneHotInfo::ExtractInputInfo() { + CheckGlobalDeviceManager(); + rank_ = g_device_manager->global_rank(); + mod_rank_ = rank_ % dev_matrix_shape_.back(); + if (!cnode_) { + MS_LOG(ERROR) << "Failure:OneHot cnode_ is nullptr"; + return FAILED; + } + if (cnode_->inputs().size() != 5) { + MS_LOG(ERROR) << "Failure:There is 5 inputs for the CNode corresponding to OneHot Primitive, real input size is " + << cnode_->inputs().size(); + return FAILED; + } + if (input_value_.size() != 4) { + MS_LOG(ERROR) << "Failure:There is 5 inputs for the CNode corresponding to OneHot Primitive, and input value size " + "must be 4, real size is " + << input_value_.size(); + return FAILED; + } + auto value_ptr = input_value_.at(1); + if (value_ptr == nullptr) { + MS_LOG(WARNING) << "Input 2 of cnode is not a value node, its type is " << cnode_->input(2)->type_name(); + return FAILED; + } + + if (value_ptr->isa()) { + total_class_number_ = value_ptr->cast()->value(); + } else { + MS_LOG(ERROR) << "OneHot Primitive depth type must be int"; + return FAILED; + } + classes_each_device_ = total_class_number_ / dev_matrix_shape_.back(); + + return SUCCESS; +} + +Status OneHotInfo::ComputeReplaceGraph(const CNodePtr& cnode) { + if (dev_matrix_shape_.back() == 1) { + replace_graph_ = nullptr; + return SUCCESS; + } + if (ExtractInputInfo() != SUCCESS) { + MS_LOG(ERROR) << "ExtractInputInfo failed"; + return FAILED; + } + GenerateGraph gen_g = GenerateGraph(); + Status status = gen_g.Init(cnode); + if (status != SUCCESS) { + MS_LOG(ERROR) << "GenerateGraph Init failed"; + return FAILED; + } + + auto floor_div = + gen_g.PushBack({gen_g.NewOpInst(FLOORDIV), gen_g.virtual_input_node(), CreateInt32Tensor(classes_each_device_)}); + auto mul1 = gen_g.PushBack({gen_g.NewOpInst(MUL), floor_div, CreateInt32Tensor(classes_each_device_)}); + auto sub1 = gen_g.PushBack({gen_g.NewOpInst(SUB), gen_g.virtual_input_node(), mul1}); + auto equal = gen_g.PushBack({gen_g.NewOpInst(EQUAL), floor_div, CreateInt32Tensor(mod_rank_)}); + auto cast = gen_g.PushBack({gen_g.NewOpInst(CAST), equal, CreatTypeInt(32)}); + auto mul2 = gen_g.PushBack({gen_g.NewOpInst(MUL), sub1, cast}); + auto tensor_add = gen_g.PushBack({gen_g.NewOpInst(TENSOR_ADD), mul2, CreateInt32Tensor(1)}); + auto mul3 = gen_g.PushBack({gen_g.NewOpInst(MUL), cast, tensor_add}); + auto sub2 = gen_g.PushBack({gen_g.NewOpInst(SUB), mul3, CreateInt32Tensor(1)}); + Attr attr_onehot_axis = std::make_pair(AXIS, axis_value_ptr_); + OperatorAttrs attrs_onehot = {attr_onehot_axis}; + auto onehot = gen_g.PushBack({gen_g.NewOpInst(ONEHOT, attrs_onehot), sub2, CreatInt32Imm(classes_each_device_), + cnode->input(3), cnode->input(4)}); + std::vector input_nodes = {floor_div, sub1}; + replace_graph_ = + std::make_shared, AnfNodePtr>>(std::make_pair(input_nodes, onehot)); + + return SUCCESS; +} + +ReplaceGraphPtr OneHotInfo::replace_graph(const CNodePtr& cnode) { + if (ComputeReplaceGraph(cnode) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": ComputeReplaceGraph failed."; + return nullptr; + } + return replace_graph_; +} + +Status OneHotInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + Status status = ComputeReplaceGraph(cnode_); + if (status != SUCCESS) { + MS_LOG(ERROR) << name_ << ": ComputeReplaceGraph failed."; + return status; + } + MS_LOG(INFO) << name_ << ": Init success."; + return SUCCESS; +} + +Status OneHotInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed."; + } + return FAILED; + } + MS_LOG(INFO) << name_ << ": Init for cost model success."; + return SUCCESS; +} + +Status OneHotInfo::GenerateStrategies(int32_t stage_id) { + Shapes splittable_inputs = {{1, 1}, {}, {}}; + std::vector sp_vector; + if (inputs_shape_.size() != 3) { + MS_LOG(ERROR) << name_ << ": inputs_shape_ size must be 3, but is " << inputs_shape_.size(); + return FAILED; + } + if (outputs_shape_.size() != 1) { + MS_LOG(ERROR) << name_ << ": outputs_shape_ size must be 1, but is " << outputs_shape_.size(); + return FAILED; + } + is_auto_parallel_ = true; + if (GenerateStrategiesForIndependentInputs(stage_id, {outputs_shape_.at(0), inputs_shape_.at(1), inputs_shape_.at(2)}, + splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategies failed."; + return FAILED; + } + + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + + return SUCCESS; +} + +Status OneHotInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + return SUCCESS; +} + +std::shared_ptr>> OneHotInfo::GenerateBatchStrategies() { + CheckGlobalDeviceManager(); + size_t dev_num = g_device_manager->GetDeviceListByStageId(0).size(); + Dimensions strategy = {SizeToInt(dev_num), 1}; + Dimensions empty_strategy; + std::vector strategy_v = {strategy, empty_strategy, empty_strategy}; + return std::make_shared>>(strategy_v); +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/onehot_info.h b/mindspore/ccsrc/parallel/ops_info/onehot_info.h new file mode 100644 index 00000000000..fd5d6be8f65 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/onehot_info.h @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ONEHOT_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_ONEHOT_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class OneHotInfo : public OperatorInfo { + public: + OneHotInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + onehot_cost_ptr_ = std::make_shared(); + } + ~OneHotInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return onehot_cost_ptr_; } + ReplaceGraphPtr replace_graph(const CNodePtr& cnode) override; + std::shared_ptr>> GenerateBatchStrategies() override; + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status GetAttrs() override; + Status InferMirrorOps() override { return SUCCESS; } + Status InferForwardCommunication() override { return SUCCESS; } + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status ExtractInputInfo(); + + private: + Status ComputeReplaceGraph(const CNodePtr& cnode); + + int axis_ = -1; + OneHotCostPtr onehot_cost_ptr_; + int32_t rank_ = 0; + int32_t total_class_number_ = 1; + int32_t classes_each_device_ = 1; + ValuePtr axis_value_ptr_; + int32_t mod_rank_ = 0; +}; +} // namespace parallel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_OPS_INFO_PARALLEL_ONEHOT_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/operator_info.cc b/mindspore/ccsrc/parallel/ops_info/operator_info.cc new file mode 100644 index 00000000000..f187d386736 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/operator_info.cc @@ -0,0 +1,1233 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/operator_info.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "ir/dtype.h" +#include "ir/meta_tensor.h" +#include "utils/context/ms_context.h" +#include "utils/log_adapter.h" +#include "parallel/context.h" +#include "parallel/auto_parallel/edge_costmodel.h" +#include "parallel/auto_parallel/graph_costmodel.h" + +namespace mindspore { +namespace parallel { +Status CheckStrategyValue(const StrategyPtr& strategy, const Shapes& inputs_shape, bool is_auto_parallel) { + if (strategy == nullptr) { + MS_LOG(ERROR) << "The strategy is null."; + return FAILED; + } + + size_t strategy_size = strategy->GetInputNumber(); + size_t inputs_shape_size = inputs_shape.size(); + if (strategy_size != inputs_shape_size) { + if (is_auto_parallel) { + MS_LOG(DEBUG) << "Strategy size: " << strategy_size << " is not equal to inputs size: " << inputs_shape_size; + } else { + MS_LOG(ERROR) << "Strategy size: " << strategy_size << " is not equal to inputs size: " << inputs_shape_size; + } + return FAILED; + } + + std::vector stra = strategy->GetInputDim(); + for (size_t i = 0; i < strategy_size; ++i) { + Shape sub_strategy = stra.at(i); + Shape sub_input_shape = inputs_shape.at(i); + size_t strategy_len = sub_strategy.size(); + size_t inputs_len = sub_input_shape.size(); + if (strategy_len != inputs_len) { + if (is_auto_parallel) { + MS_LOG(DEBUG) << "Strategy len: " << strategy_len << " is not equal to inputs len: " << inputs_len + << ", index: " << i; + } else { + MS_LOG(ERROR) << "Strategy len: " << strategy_len << " is not equal to inputs len: " << inputs_len + << ", index: " << i; + } + return FAILED; + } + + for (size_t j = 0; j < strategy_len; ++j) { + int32_t strategy_value = sub_strategy.at(j); + if (strategy_value < MIN_SLICE_NUM) { + if (is_auto_parallel) { + MS_LOG(DEBUG) << "Invalid strategy value: " << strategy_value; + } else { + MS_LOG(ERROR) << "Invalid strategy value: " << strategy_value; + } + return FAILED; + } + + if ((IntToUint(strategy_value) & IntToUint(strategy_value - 1)) != 0) { + if (is_auto_parallel) { + MS_LOG(DEBUG) << "Invalid Strategy value it is not the power of 2, " << strategy_value; + } else { + MS_LOG(ERROR) << "Invalid Strategy value it is not the power of 2, " << strategy_value; + } + return FAILED; + } + + int32_t shape_value = sub_input_shape.at(j); + if ((shape_value % strategy_value) != 0) { + if (is_auto_parallel) { + MS_LOG(DEBUG) << "Shape " << shape_value << " cannot be divisible by strategy " << strategy_value; + } else { + MS_LOG(ERROR) << "Shape " << shape_value << " cannot be divisible by strategy " << strategy_value; + } + return FAILED; + } + } + } + + return SUCCESS; +} + +void OperatorInfo::ResetQueueMember() { + inputs_tensor_info_.clear(); + outputs_tensor_info_.clear(); + inputs_tensor_map_.clear(); + outputs_tensor_map_.clear(); + dev_matrix_shape_.clear(); + forward_op_.clear(); + mirror_ops_.clear(); + replace_op_.clear(); + replace_op_info_.clear(); + virtual_div_op_.clear(); + global_device_list_.clear(); +} + +Status OperatorInfo::InferAttrs() { + if (infer_attrs_completed_) { + return SUCCESS; + } + + if (GetAttrs() != SUCCESS) { + return FAILED; + } + infer_attrs_completed_ = true; + return SUCCESS; +} + +void OperatorInfo::SetDeviceListByStrategy() { + int32_t stage = strategy_->GetInputStage(); + CheckGlobalDeviceManager(); + global_device_list_ = g_device_manager->GetDeviceListByStageId(stage); +} + +Status OperatorInfo::InferRepeatedCalcInfo() { + int32_t g_dev_list_size = SizeToInt(global_device_list_.size()); + int32_t dev_matrix_size = + std::accumulate(dev_matrix_shape_.begin(), dev_matrix_shape_.end(), 1, std::multiplies()); + if (dev_matrix_size == 0) { + MS_LOG(ERROR) << name_ << ": The dev matrix size is 0"; + return FAILED; + } + + if (g_dev_list_size == dev_matrix_size) { + repeated_calc_num_ = 1; + } else if (g_dev_list_size % dev_matrix_size == 0) { + repeated_calc_num_ = g_dev_list_size / dev_matrix_size; + } else { + MS_LOG(ERROR) << name_ << ": Dev list size " << g_dev_list_size << " can not be divisible by dev matrix size " + << dev_matrix_size; + return FAILED; + } + + CheckGlobalDeviceManager(); + int32_t rank = g_device_manager->global_rank(); + int32_t stage = strategy_->GetInputStage(); + local_device_list_ = g_device_manager->global_device_list(stage, rank, repeated_calc_num_); + + return SUCCESS; +} + +// if repeated calculation, need to set the repeated_calc_num as the first dimension of dev-matrix, +// only use for infer tensor layout +void OperatorInfo::SetRepeatedCalcDevMatrix() { + if (repeated_calc_num_ <= 1) { + return; + } + + (void)dev_matrix_shape_.insert(dev_matrix_shape_.begin(), repeated_calc_num_); +} + +// use for loss repeated calculation +Operator CreateVirtualDivOp(int32_t div_num) { + OperatorName operator_name = VIRTUAL_DIV; + ValuePtr attr0_value = MakeValue(div_num); + Attr attr0 = std::make_pair(DIVISOR, attr0_value); + OperatorAttrs operator_attrs; + operator_attrs.push_back(attr0); + + OperatorParams operator_param; + OperatorArgs operator_arg = std::make_pair(operator_attrs, operator_param); + + Operator op = std::make_pair(operator_name, operator_arg); + return op; +} + +// use for forward all reduce +Operator CreateAllReduceOp(const std::string& reduce_op, const std::string& group) { + OperatorName operator_name = ALL_REDUCE; + ValuePtr attr0_value = MakeValue(reduce_op); // ReduceOP.SUM + ValuePtr attr1_value = MakeValue(group); // group + Attr attr0 = std::make_pair(OP, attr0_value); + Attr attr1 = std::make_pair(GROUP, attr1_value); + OperatorAttrs operator_attrs; + operator_attrs.push_back(attr0); + operator_attrs.push_back(attr1); + + OperatorParams operator_param; + OperatorArgs operator_arg = std::make_pair(operator_attrs, operator_param); + + Operator op = std::make_pair(operator_name, operator_arg); + MS_LOG(INFO) << "Create all reduce op success, the reduce_op is " << reduce_op << ", the group is " << group; + return op; +} + +// use for get tensor slice +Operator CreateGetTensorSliceOp(const TensorLayout& tensor_layout) { + Shape tensor_map = tensor_layout.tensor_map().array(); + Shape dev_matrix_shape = tensor_layout.device_arrangement().array(); + OperatorName operator_name = GET_TENSOR_SLICE; + + OperatorAttrs attrs; + ValuePtr dev_mat_value = MakeValue(dev_matrix_shape); + Param dev_mat_param = std::make_pair(std::make_pair(DEV_MAT, dev_mat_value), 2); + ValuePtr tensor_map_value = MakeValue(tensor_map); + Param tensor_map_param = std::make_pair(std::make_pair(TENSOR_MAP, tensor_map_value), 3); + OperatorParams params = {dev_mat_param, tensor_map_param}; + OperatorArgs operator_arg = std::make_pair(attrs, params); + + Operator op = std::make_pair(operator_name, operator_arg); + MS_LOG(INFO) << "Create get tensor slice op success, the dev mat and tensor map is " + << ShapeToString(dev_matrix_shape) << ", " << ShapeToString(tensor_map); + return op; +} + +OperatorVector CreateMirrorOps(const std::string& group_name, size_t dev_num) { + if ((dev_num == 0) || (dev_num == 1)) { + MS_LOG(EXCEPTION) << "Invalid dev num: " << dev_num; + } + OperatorVector op_for_weight; + bool mean_flag = ParallelContext::GetInstance()->mirror_mean(); + + OperatorName operator_name = MIRROR_OPERATOR; + ValuePtr attr0_value = MakeValue(group_name); + ValuePtr attr1_value = MakeValue(dev_num); + ValuePtr attr2_value = MakeValue(mean_flag); + + Attr attr0 = std::make_pair(GROUP, attr0_value); + Attr attr1 = std::make_pair(DEV_NUM, attr1_value); + Attr attr2 = std::make_pair(MEAN_FLAG, attr2_value); + + OperatorAttrs operator_attrs; + operator_attrs.push_back(attr0); + operator_attrs.push_back(attr1); + operator_attrs.push_back(attr2); + + OperatorParams operator_param; + OperatorArgs operator_args = std::make_pair(operator_attrs, operator_param); + + Operator op = std::make_pair(operator_name, operator_args); + + op_for_weight.push_back(op); + MS_LOG(INFO) << "The group name is " << group_name << ", the dev num is " << dev_num << ", the mean flag is " + << mean_flag; + return op_for_weight; +} + +Status OperatorInfo::CreateGroupByTensorMap(const Shape& tensor_map, std::vector* group) { + if (group == nullptr) { + MS_LOG(ERROR) << "The group is null."; + return FAILED; + } + CheckGlobalDeviceManager(); + int32_t rank = g_device_manager->global_rank(); + DeviceMatrix dev_matrix(rank, global_device_list_, dev_matrix_shape_); + RankList group_devices; + if (dev_matrix.GetDevicesByTensorMap(tensor_map, &group_devices) != SUCCESS) { + return FAILED; + } + + if (group_devices.size() == 1) { + MS_LOG(INFO) << "The dev size is 1, no need to create group."; + return SUCCESS; + } + + Group g = g_device_manager->CreateGroup(group_devices); + group->push_back(g); + return SUCCESS; +} + +Status OperatorInfo::CreateGroupByDim(size_t axis, std::vector* group) { + if (group == nullptr) { + MS_LOG(ERROR) << "The group is null."; + return FAILED; + } + CheckGlobalDeviceManager(); + int32_t rank = g_device_manager->global_rank(); + DeviceMatrix dev_matrix(rank, global_device_list_, dev_matrix_shape_); + RankList group_devices; + if (dev_matrix.GetDevicesAlongDim(SizeToUint(axis), &group_devices) != SUCCESS) { + return FAILED; + } + + if (group_devices.size() == 1) { + MS_LOG(INFO) << "The dev size is 1, no need to create group."; + return SUCCESS; + } + + Group g = g_device_manager->CreateGroup(group_devices); + group->push_back(g); + return SUCCESS; +} + +Shape GetSliceShape(const Shape& tensor_shape, const Dimensions& strategy) { + Shape slice_shape; + if (std::any_of(strategy.begin(), strategy.end(), [](int32_t value) { return value <= 0; })) { + MS_LOG(ERROR) << "Invalid strategy: " << ShapeToString(strategy) << ", the element is less than or equal to 0"; + return slice_shape; + } + for (size_t i = 0; i < strategy.size(); ++i) { + slice_shape.push_back(tensor_shape.at(i) / strategy.at(i)); + } + return slice_shape; +} + +Status InferSliceShapeByStrategy(const Strategys& strategys, const Shapes& shapes, Shapes* slice_shapes) { + if (slice_shapes == nullptr) { + MS_LOG(ERROR) << "The slice_shapes is null."; + return FAILED; + } + if (strategys.size() != shapes.size()) { + MS_LOG(ERROR) << "Strategy size " << strategys.size() << " not equal to shape size " << shapes.size(); + return FAILED; + } + + for (size_t i = 0; i < strategys.size(); ++i) { + if (strategys.at(i).size() != shapes.at(i).size()) { + MS_LOG(ERROR) << "Strategy dimension " << strategys.at(i).size() << " not equal to shape dimension " + << shapes.at(i).size(); + slice_shapes->clear(); + return FAILED; + } + + for (size_t j = 0; j < shapes.at(i).size(); ++j) { + if (strategys.at(i).at(j) <= 0) { + MS_LOG(ERROR) << "Invalid strategy: " << ShapeToString(strategys[i]) + << " the element is less than or equal to 0."; + slice_shapes->clear(); + return FAILED; + } + if (shapes.at(i).at(j) % strategys.at(i).at(j) != 0) { + MS_LOG(ERROR) << "Shape cannot be divisible by strategy, " << shapes.at(i).at(j) << " : " + << strategys.at(i).at(j); + slice_shapes->clear(); + return FAILED; + } + } + Shape slice_shape = GetSliceShape(shapes.at(i), strategys.at(i)); + slice_shapes->push_back(slice_shape); + } + + return SUCCESS; +} + +Status OperatorInfo::InferSliceShape(const Strategys& inputs_strategy, const Strategys& outputs_strategy, + Shapes* inputs_slice_shape, Shapes* outputs_slice_shape) { + if (inputs_slice_shape == nullptr || outputs_slice_shape == nullptr) { + MS_LOG(ERROR) << "The slice_shape is null."; + return FAILED; + } + + if (InferSliceShapeByStrategy(inputs_strategy, inputs_shape_, inputs_slice_shape) != SUCCESS) { + MS_LOG(ERROR) << "Infer inputs slice shape error."; + return FAILED; + } + + if (InferSliceShapeByStrategy(outputs_strategy, outputs_shape_, outputs_slice_shape) != SUCCESS) { + MS_LOG(ERROR) << "Infer outputs slice shape error."; + inputs_slice_shape->clear(); + return FAILED; + } + + return SUCCESS; +} + +// method0: auto insert repeated_calculation_num for dev_matrix_shape when repeated_calculation_num > 1 +Status OperatorInfo::InitForCostModelWithAutoRepeatCalc(const StrategyPtr& strategy) { + if (strategy == nullptr) { + MS_LOG(ERROR) << name_ << ": The strategy is null."; + return FAILED; + } + + if (InferAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferAttrs failed."; + return FAILED; + } + + // must be after InferAttrs() + if (CheckStrategy(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": CheckStrategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": CheckStrategy failed."; + } + return FAILED; + } + + // need to clear queues before Init(), + // because Init() may be called multiple times by cost model + ResetQueueMember(); + + strategy_ = strategy; + SetDeviceListByStrategy(); + + if (InferDevMatrixShape() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferDevMatrixShape failed."; + return FAILED; + } + + used_devices_ = std::accumulate(dev_matrix_shape_.begin(), dev_matrix_shape_.end(), 1, std::multiplies()); + + // must be after InferDevMatrixShape + if (InferRepeatedCalcInfo() != SUCCESS) { + MS_LOG(ERROR) << ": InferRepeatedCalcInfo failed."; + return FAILED; + } + + // if repeated calculation, need to set the repeated_calc_num as the first dimension of dev-matrix for layout + SetRepeatedCalcDevMatrix(); + + if (InferTensorMap() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferTensorMap failed."; + return FAILED; + } + + if (InferTensorInfo() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferTensorInfo failed."; + return FAILED; + } + + return SUCCESS; +} + +// method1: manually insert repeated_calculation_num for dev_matrix_shape in InferDevMatrixShape +Status OperatorInfo::InitForCostModelWithManualRepeatCalc(const StrategyPtr& strategy) { + if (strategy == nullptr) { + MS_LOG(ERROR) << name_ << ": The strategy is null."; + return FAILED; + } + + if (InferAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferAttrs failed."; + return FAILED; + } + + // must be after InferAttrs() + if (CheckStrategy(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": CheckStrategy failed."; + return FAILED; + } + + // need to clear queues before Init(), + // because Init() may be called multiple times by cost model + ResetQueueMember(); + + strategy_ = strategy; + SetDeviceListByStrategy(); + + if (InferDevMatrixShape() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferDevMatrixShape failed."; + return FAILED; + } + + // must be after InferDevMatrixShape + if (InferRepeatedCalcInfo() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferRepeatedCalcInfo failed."; + return FAILED; + } + + if (InferTensorMap() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferTensorMap failed."; + return FAILED; + } + + if (InferTensorInfo() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferTensorInfo failed."; + return FAILED; + } + + return SUCCESS; +} + +Status OperatorInfo::InitWithAutoRepeatCalc(const StrategyPtr& strategy) { + if (strategy == nullptr) { + MS_LOG(ERROR) << name_ << ": The strategy is null."; + return FAILED; + } + + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + return FAILED; + } + + if (InferForwardCommunication() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferForwardCommunication failed."; + return FAILED; + } + + if (InferMirrorOps() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferMirrorOps failed."; + return FAILED; + } + + if (InferVirtualDivOps() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferVirtualDivOps failed."; + return FAILED; + } + + return SUCCESS; +} + +Status OperatorInfo::InitWithManualRepeatCalc(const StrategyPtr& strategy) { + if (strategy == nullptr) { + MS_LOG(ERROR) << name_ << ": The strategy is null."; + return FAILED; + } + + if (InitForCostModelWithManualRepeatCalc(strategy) != SUCCESS) { + return FAILED; + } + + if (InferForwardCommunication() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferForwardCommunication failed."; + return FAILED; + } + + if (InferMirrorOps() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferMirrorOps failed."; + return FAILED; + } + + if (InferVirtualDivOps() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferVirtualDivOps failed."; + return FAILED; + } + + return SUCCESS; +} + +std::vector> OperatorInfo::GetAliveSuccEdges() { + std::vector> ret; + for (auto& edge : succ_edges_) { + if ((edge->next_operator()->is_alive()) && (edge->next_operator()->name().find(RELU) != std::string::npos)) { + ret.push_back(edge); + } + } + for (auto& edge : succ_edges_) { + if ((edge->next_operator()->is_alive()) && (edge->next_operator()->name().find(RELU) == std::string::npos)) { + ret.push_back(edge); + } + } + return ret; +} + +std::vector> OperatorInfo::GetAlivePrevEdges() { + std::vector> ret; + for (auto& edge : prev_edges_) { + if (edge->prev_operator()->is_alive()) { + ret.push_back(edge); + } + } + return ret; +} + +void OperatorInfo::ReplacePreEdge(const std::shared_ptr& op, const std::shared_ptr& new_edge) { + if (op == nullptr) { + MS_LOG(ERROR) << name_ << ": ReplacePreEdge: the op is null."; + return; + } + for (auto& edge : prev_edges_) { + if (edge->prev_operator() == op) { + edge = new_edge; + return; + } + } + MS_LOG(EXCEPTION) << name_ << ": Replace edge failed: no edge has been replaced"; +} + +void OperatorInfo::ReplaceSuccEdge(const std::shared_ptr& op, const std::shared_ptr& new_edge) { + if (op == nullptr) { + MS_LOG(ERROR) << name_ << ": ReplaceSuccEdge: the op is null."; + return; + } + for (auto& edge : succ_edges_) { + if (edge->next_operator() == op) { + edge = new_edge; + return; + } + } + MS_LOG(EXCEPTION) << name_ << ": Replace edge failed: no edge has been replaced"; +} + +void OperatorInfo::ReplacePreEdges(const std::shared_ptr& op, const std::shared_ptr& new_edge) { + if (op == nullptr) { + MS_LOG(ERROR) << name_ << ": ReplacePreEdges: the op is null."; + return; + } + std::vector> new_pre_edges; + for (auto& edge : prev_edges_) { + if (edge->prev_operator() != op) { + new_pre_edges.push_back(edge); + } + } + new_pre_edges.push_back(new_edge); + prev_edges_ = new_pre_edges; +} + +void OperatorInfo::ReplaceSuccEdges(const std::shared_ptr& op, const std::shared_ptr& new_edge) { + if (op == nullptr) { + MS_LOG(ERROR) << name_ << ": ReplaceSuccEdges: the op is null"; + return; + } + std::vector> new_succ_edges; + for (auto& edge : succ_edges_) { + if (edge->next_operator() != op) { + new_succ_edges.push_back(edge); + } + } + new_succ_edges.push_back(new_edge); + succ_edges_ = new_succ_edges; +} + +std::shared_ptr>> GenerateBatchStrategiesBySplitFlag( + const Shapes& shapes, const std::vector& split_flag_list) { + if (shapes.size() != split_flag_list.size()) { + MS_LOG(ERROR) << "Split_flag_list do not have the same size as inputs shape, " << split_flag_list.size() << " : " + << shapes.size(); + return nullptr; + } + CheckGlobalDeviceManager(); + int32_t dev_num = SizeToInt(g_device_manager->GetDeviceListByStageId(0).size()); + std::vector> strategy_v; + for (size_t i = 0; i != shapes.size(); i++) { + if (shapes[i].empty()) { + MS_LOG(INFO) << "Elements of shapes is empty."; + std::vector empty_element; + strategy_v.push_back(empty_element); + } else { + std::vector element(shapes[i].size(), 1); + if (split_flag_list[i]) { + element[0] = dev_num; + } + strategy_v.push_back(element); + } + } + return std::make_shared>>(strategy_v); +} + +void OperatorInfo::ReComputeBatchSplitFlagList() { + if (!inputs_shape_.empty()) { + split_flag_list_[0] = true; + } +} + +void OperatorInfo::ComputeBatchSplitFlagList() { + split_flag_list_.clear(); + for (auto iter = inputs_shape_.begin(); iter != inputs_shape_.end(); ++iter) { + split_flag_list_.push_back(false); + } + ReComputeBatchSplitFlagList(); +} + +// This is a common method for checking whether the generated stragegy has the correct number of devuces. +Status PrepareStrategyBase(int32_t stage_id, size_t dev_num, const Shapes& inputs_partitions, StrategyPtr* const sp) { + if (sp == nullptr) { + MS_LOG(ERROR) << "The strategy is null."; + return FAILED; + } + int32_t product = 1; + + for (auto& input_partition : inputs_partitions) { + product *= std::accumulate(input_partition.begin(), input_partition.end(), 1, std::multiplies()); + } + if (NOT_FULLY_USE_DEVICES) { + if (IntToSize(product) > dev_num) { + return FAILED; + } + } else { + if ((product != 1) && (IntToSize(product) != dev_num)) { + return FAILED; + } + } + std::vector stras(inputs_partitions); + (*sp) = std::make_shared(stage_id, stras); + return SUCCESS; +} + +std::shared_ptr>> OperatorInfo::GenerateBatchStrategies() { + ComputeBatchSplitFlagList(); + return GenerateBatchStrategiesBySplitFlag(inputs_shape_, split_flag_list_); +} + +void PrintStrategy(const StrategyPtr& strategy) { + if (strategy == nullptr) { + return; + } + std::string all_strategy = ""; + for (size_t i = 0; i < strategy->GetInputNumber(); ++i) { + all_strategy += "["; + for (size_t j = 0; j < strategy->GetInputDim()[i].size(); ++j) { + all_strategy += std::to_string(strategy->GetInputDim()[i][j]); + if (j != strategy->GetInputDim()[i].size() - 1) { + all_strategy += ", "; + } + } + all_strategy += "]"; + if (i != strategy->GetInputNumber() - 1) { + all_strategy += ", "; + } + } + MS_LOG(INFO) << "The strategy is: " << all_strategy; +} + +// generate strategies for that each dimension of input0 and input1 is relevant, such as: ([a, b, c, d], [a, b, c, d]) +Status GenerateStrategiesForTwoEqualInputs(int32_t stage_id, const Shapes& inputs_shape, + const Shapes& splittable_inputs, std::vector* const sp_vector) { + if (sp_vector == nullptr) { + MS_LOG(ERROR) << "The sp_vector is null."; + return FAILED; + } + + if ((inputs_shape.size() != 2) || (splittable_inputs.size() != 2)) { + MS_LOG(ERROR) << "The inputs size is wrong."; + return FAILED; + } + + if ((inputs_shape[0].size() != inputs_shape[1].size()) || + (splittable_inputs[0].size() != splittable_inputs[1].size())) { + MS_LOG(ERROR) << "The size of two inputs are not equal."; + return FAILED; + } + + Shapes input0_shape = {inputs_shape[0]}; + Shapes input0_splittable = {splittable_inputs[0]}; + if (GenerateStrategiesForIndependentInputs(stage_id, input0_shape, input0_splittable, sp_vector) != SUCCESS) { + return FAILED; + } + + for (auto& sp : *sp_vector) { + sp->ExpandInputDimFromOneToTwo(); + } + + return SUCCESS; +} + +// generate strategies for that input0 and input1 have relevant dimensions, and input0 needs to broadcast +// such as: ([b, c, d], [a, b, c, d]) or ([1, c, d], [a, b, c, d]) +Status GenerateStrategiesForBroadcastLeft(int32_t stage_id, const Shapes& inputs_shape, const Shapes& splittable_inputs, + std::vector* const sp_vector) { + if (sp_vector == nullptr) { + MS_LOG(ERROR) << "The sp_vector is null."; + return FAILED; + } + + if (inputs_shape[0].size() >= inputs_shape[1].size()) { + MS_LOG(ERROR) << "Invalid inputs shape."; + return FAILED; + } + + // first, generate strategy for input0 the same as input1 + Shapes tmp_inputs_shape = {inputs_shape[1], inputs_shape[1]}; + Shapes tmp_splittable_inputs = {splittable_inputs[1], splittable_inputs[1]}; + if (GenerateStrategiesForTwoEqualInputs(stage_id, tmp_inputs_shape, tmp_splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForTwoEqualInputs failed."; + return FAILED; + } + + // second, get the correct strategy for input0 + for (auto& sp : *sp_vector) { + std::vector tmp_strategy; + Dimensions input0_strategy = sp->GetInputDim()[0]; + size_t size_diff = inputs_shape[1].size() - inputs_shape[0].size(); + + // erase the unnecessary part + (void)input0_strategy.erase(input0_strategy.begin(), + input0_strategy.begin() + static_cast(size_diff)); + + // handel the case likes ([1, c, d], [a, b, c, d]) + for (size_t i = 0; i < inputs_shape[0].size(); ++i) { + if (inputs_shape[0][i] == 1) { + input0_strategy[i] = 1; + } else { + break; + } + } + + // reset the strategy + tmp_strategy.push_back(input0_strategy); // input0 + tmp_strategy.push_back(sp->GetInputDim()[1]); // input1 + sp->ResetInputs(tmp_strategy); + } + return SUCCESS; +} + +// generate strategies for that input0 and input1 have relevant dimensions, and input1 needs to broadcast +// such as: ([a, b, c, d], [b, c, d]) or ([a, b, c, d], [1, c, d]) +Status GenerateStrategiesForBroadcastRight(int32_t stage_id, const Shapes& inputs_shape, + const Shapes& splittable_inputs, std::vector* const sp_vector) { + if (sp_vector == nullptr) { + MS_LOG(ERROR) << "The sp_vector is null."; + return FAILED; + } + + if (inputs_shape[0].size() <= inputs_shape[1].size()) { + MS_LOG(ERROR) << "Invalid inputs shape."; + return FAILED; + } + + // first, generate strategy for input1 the same as input0 + Shapes tmp_inputs_shape = {inputs_shape[0], inputs_shape[0]}; + Shapes tmp_splittable_inputs = {splittable_inputs[0], splittable_inputs[0]}; + if (GenerateStrategiesForTwoEqualInputs(stage_id, tmp_inputs_shape, tmp_splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForTwoEqualInputs failed."; + return FAILED; + } + + // second, get the correct strategy for input1 + for (auto& sp : *sp_vector) { + std::vector tmp_strategy; + tmp_strategy.push_back(sp->GetInputDim()[0]); // input0 + + Dimensions input1_strategy = sp->GetInputDim()[1]; + size_t size_diff = inputs_shape[0].size() - inputs_shape[1].size(); + + // erase the unnecessary part + (void)input1_strategy.erase(input1_strategy.begin(), + input1_strategy.begin() + static_cast(size_diff)); + + // handel the case likes ([a, b, c, d], [1, c, d]) + for (size_t i = 0; i < inputs_shape[1].size(); ++i) { + if (inputs_shape[1][i] == 1) { + input1_strategy[i] = 1; + } else { + break; + } + } + + // reset the strategy + tmp_strategy.push_back(input1_strategy); // input1 + sp->ResetInputs(tmp_strategy); + } + return SUCCESS; +} + +// generate strategies for that input0 and input1 have same size, and input0 or input1 needs to broadcast +// such as: ([a, 1], [1, b]) or ([a, b, c, d], [1, b, c, d]) or ([a, b, c, 1], [1, b, c, d]) +Status GenerateStrategiesForBroadcastBoth(int32_t stage_id, const Shapes& inputs_shape, const Shapes& splittable_inputs, + std::vector* const sp_vector) { + if (sp_vector == nullptr) { + MS_LOG(ERROR) << "The sp_vector is null."; + return FAILED; + } + + if (inputs_shape[0].size() != inputs_shape[1].size()) { + MS_LOG(ERROR) << "Invalid inputs shape."; + return FAILED; + } + + // step1: ([a, 1], [1, b]) -> [a, b] + Shape max_shape, splittable_vector; + for (size_t i = 0; i < inputs_shape[0].size(); ++i) { + if (inputs_shape[0][i] >= inputs_shape[1][i]) { + max_shape.push_back(inputs_shape[0][i]); + splittable_vector.push_back(splittable_inputs[0][i]); + } else { + max_shape.push_back(inputs_shape[1][i]); + splittable_vector.push_back(splittable_inputs[1][i]); + } + } + + // step2: ([a, 1], [1, b]) -> generate strategy for ([a, b], [a, b]) + Shapes tmp_inputs_shape = {max_shape, max_shape}; + Shapes tmp_splittable_inputs = {splittable_vector, splittable_vector}; + if (GenerateStrategiesForTwoEqualInputs(stage_id, tmp_inputs_shape, tmp_splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForTwoEqualInputs failed."; + return FAILED; + } + + // step3: reset the strategy if the dimension is 1 + for (auto& sp : *sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + for (size_t i = 0; i < inputs_shape[0].size(); ++i) { + if (inputs_shape[0][i] == 1) { + input0_strategy[i] = 1; + } + + if (inputs_shape[1][i] == 1) { + input1_strategy[i] = 1; + } + } + sp->ResetInputs({input0_strategy, input1_strategy}); + } + + return SUCCESS; +} + +// 'splittable_inputs' has the same dimensions as 'inputs_shape_'. '0' in 'splittable_inputs' means that +// the corresponding dimension is unsplittable, '1' in 'splittable_inputs' means that the corresponding +// dimension is splittable. 'inputs_partitions' is the result of partitions. +// NOTE: This implementation would partition all splittable dimensions in all inputs. Some operators requiring +// specific dimensions in inputs have the identical partition should have individual implementation. +Status GenerateStrategiesForIndependentInputs(int32_t stage_id, const Shapes& inputs_shape, + const Shapes& splittable_inputs, + std::vector* const sp_vector) { + if (sp_vector == nullptr) { + MS_LOG(ERROR) << "The sp_vector is null."; + return FAILED; + } + if (splittable_inputs.size() != inputs_shape.size()) { + MS_LOG(ERROR) << "Splittable_inputs do not have the same input number of inputs shape, " << splittable_inputs.size() + << " : " << inputs_shape.size(); + return FAILED; + } + CheckGlobalDeviceManager(); + size_t dev_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + + Shape combined_inputs_shape, combined_splittable_inputs, combined_partitions; + for (size_t j = 0; j < inputs_shape.size(); ++j) { + (void)combined_inputs_shape.insert(combined_inputs_shape.end(), inputs_shape[j].begin(), inputs_shape[j].end()); + (void)combined_splittable_inputs.insert(combined_splittable_inputs.end(), splittable_inputs[j].begin(), + splittable_inputs[j].end()); + } + std::function recursive = [&stage_id, &dev_num, &sp_vector, &combined_inputs_shape, + &combined_splittable_inputs, &combined_partitions, &recursive, + &inputs_shape](uint32_t current_index, size_t n) { + if (current_index == combined_inputs_shape.size()) { + MS_LOG(DEBUG) << "The value of combined_splittable_inputs.size is: " << combined_splittable_inputs.size(); + Shapes inputs_partitions; + size_t global_index = 0; + for (auto& shape : inputs_shape) { + Shape tmp_partition; + for (size_t j = 0; j < shape.size(); ++j) { + tmp_partition.push_back(combined_partitions[global_index]); + global_index++; + } + inputs_partitions.push_back(tmp_partition); + } + StrategyPtr sp; + if (PrepareStrategyBase(stage_id, dev_num, inputs_partitions, &sp) == SUCCESS) { + sp_vector->push_back(sp); + } + return; + } else { + MS_LOG(DEBUG) << "The value of sp_vector size is " << sp_vector->size(); + if (combined_splittable_inputs[current_index] == 0) { + combined_partitions.push_back(MIN_SLICE_NUM); + recursive(current_index + 1, n / MIN_SLICE_NUM); + combined_partitions.pop_back(); + } else if (combined_splittable_inputs[current_index] == 1) { + for (uint32_t i = 1; i <= n; i *= 2) { + if (n % i == 0 && IntToSize(combined_inputs_shape[current_index]) % i == 0) { + combined_partitions.push_back(i); + recursive(current_index + 1, n / i); + combined_partitions.pop_back(); + } + } + } + } + }; + recursive(0, dev_num); + if (sp_vector->empty()) { + MS_LOG(EXCEPTION) << "No available strategy for current OperatorInfo."; + } + return SUCCESS; +} + +// generate strategies for that have two inputs, and input0 or input1 maybe broadcast, +// and the corresponding dimensions that are not broadcast are all relevant dimensions +// such as: ([a, b, c, d], [a, b, c, d]) or ([b, c, d], [a, b, c, d]) or ([1, c, d], [a, b, c, d]) +// or ([a, b, c, d], [b, c, d]) or ([a, b, c, d], [1, c, d]) +// or ([a, 1], [1, b]) or ([a, b, c, d], [1, b, c, d]) or ([a, b, c, 1], [1, b, c, d]) +Status GenerateStrategiesWithBroadcast(int32_t stage_id, const Shapes& inputs_shape, const Shapes& splittable_inputs, + std::vector* const sp_vector) { + if (sp_vector == nullptr) { + MS_LOG(ERROR) << "The sp_vector is null."; + return FAILED; + } + + if ((inputs_shape.size() != 2) || (splittable_inputs.size() != 2)) { + MS_LOG(ERROR) << "The inputs' size is wrong."; + return FAILED; + } + + if (inputs_shape[0] == inputs_shape[1]) { + // element wise operation([a, b, c, d], [a, b, c, d]), so input0's strategy is equal to input1's strategy + if (GenerateStrategiesForTwoEqualInputs(stage_id, inputs_shape, splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForTwoEqualInputs failed."; + return FAILED; + } + MS_LOG(INFO) << "GenerateStrategiesForTwoEqualInputs success."; + } else if (inputs_shape[0].empty() || inputs_shape[1].empty()) { + // ([a, b, c, d], []) or ([], [a, b, c, d]) + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape, splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "Generate strategies for scalar case failed."; + return FAILED; + } + MS_LOG(INFO) << "Generate strategies for scalar case success."; + } else if (inputs_shape[0].size() > inputs_shape[1].size()) { + // ([a, b, c, d], [b, c, d]) or ([a, b, c, d], [1, c, d]) + if (GenerateStrategiesForBroadcastRight(stage_id, inputs_shape, splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForBroadcastRight failed."; + return FAILED; + } + MS_LOG(INFO) << "GenerateStrategiesForBroadcastRight success."; + } else if (inputs_shape[0].size() < inputs_shape[1].size()) { + // ([b, c, d], [a, b, c, d]) or ([1, c, d], [a, b, c, d]) + if (GenerateStrategiesForBroadcastLeft(stage_id, inputs_shape, splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForBroadcastLeft failed."; + return FAILED; + } + MS_LOG(INFO) << "GenerateStrategiesForBroadcastLeft success."; + } else { // same size, but different value + // ([a, 1], [1, b]) or ([a, b, c, d], [1, b, c, d]) or ([a, b, c, 1], [1, b, c, d]) + if (GenerateStrategiesForBroadcastBoth(stage_id, inputs_shape, splittable_inputs, sp_vector) != SUCCESS) { + MS_LOG(ERROR) << "GenerateStrategiesForBroadcastBoth failed."; + return FAILED; + } + MS_LOG(INFO) << "GenerateStrategiesForBroadcastBoth success."; + } + return SUCCESS; +} + +Status OperatorInfo::SetCostUnderStrategyBase(const StrategyPtr& strategy) { + if (InitForCostModel(strategy) == FAILED) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Initialization under the strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Initialization under the strategy failed."; + } + return FAILED; + } + int32_t stage_id = strategy->GetInputStage(); + double memory_cost = GetOperatorCost()->GetForwardMemoryCost(inputs_tensor_info_, outputs_tensor_info_, stage_id); + double communication_cost = GetOperatorCost()->GetCommCost(inputs_tensor_info_, outputs_tensor_info_, stage_id); + std::shared_ptr result = std::make_shared(memory_cost, communication_cost); + result->communication_without_parameter_ = + GetOperatorCost()->GetForwardCommCost(inputs_tensor_info_, outputs_tensor_info_, stage_id); + result->communication_with_partial_para_ = + result->communication_without_parameter_ + + COST_MODEL_GAMMA * (communication_cost - result->communication_without_parameter_); + + // Breaking ties for preferring data parallelization + BreakingTiesForPerferringDataParallel(strategy, result); + // refine communication cost calculation for practice + RefineForPracticalCost(result, false); + + std::shared_ptr swc = + std::make_shared(strategy, inputs_tensor_info_, outputs_tensor_info_); + swc->cost_list.push_back(result); + strategy_cost_.emplace_back(swc); + + return SUCCESS; +} + +Status OperatorInfo::CorrectStrategyCostForMultiOutputUse(size_t input_index) { + for (auto& swc : strategy_cost_) { + double parameter_memory_cost = ListProduct(swc->inputs_ptr[input_index].slice_shape()) * + static_cast(GetOperatorCost()->inputs_type_lengths()[input_index]); + // remove the parameter memory cost + swc->cost_list[0]->memory_cost_ -= parameter_memory_cost; + if (swc->cost_list[0]->memory_cost_ < -1) { + MS_LOG(ERROR) << "The memory cost after correction is " << swc->cost_list[0]->memory_cost_ + << ", the parameter_memory_cost is " << parameter_memory_cost; + return FAILED; + } + } + corrected_input_indices_.push_back(input_index); + return SUCCESS; +} + +int OperatorInfo::ComputeOpAndPrevEdgeParameterInvolved() { + if (is_output_parameter_involve_ != -1) { + return is_output_parameter_involve_; + } + is_parameter_involve_ = is_parameter_; + const auto& prev_edges = this->GetAlivePrevEdges(); + for (auto& p_edge : prev_edges) { + auto input_index = p_edge->next_op_input_index(); + auto prev_op_para = p_edge->prev_operator()->ComputeOpAndPrevEdgeParameterInvolved(); + if (input_index >= is_parameter_involve_.size()) { + MS_LOG(EXCEPTION) << name_ << " has input length: " << is_parameter_involve_.size() + << ", but got wrong input_index: " << input_index; + } + if (prev_op_para == 0) { + is_parameter_involve_[input_index] = false; + } else if (prev_op_para == 1) { + is_parameter_involve_[input_index] = true; + } else { + MS_LOG(EXCEPTION) << name_ << " got wrong value: " << prev_op_para << ", input_index: " << input_index; + } + p_edge->set_parameter_involve(prev_op_para); + } + if (std::any_of(is_parameter_involve_.begin(), is_parameter_involve_.end(), [](bool value) { return value; })) { + // If anyone of the input is a parameter_involved, the output is parameter_involved. + is_output_parameter_involve_ = 1; + } else { + is_output_parameter_involve_ = 0; + } + + return is_output_parameter_involve_; +} + +Status OperatorInfo::set_is_parameter(const std::vector& is_parameter) { + if (is_parameter.size() != inputs_shape_.size()) { + MS_LOG(ERROR) << "Is_parameter: " << is_parameter.size() + << " do not have the same number of inputs_shape_: " << inputs_shape_.size(); + return FAILED; + } + is_parameter_ = is_parameter; + GetOperatorCost()->set_is_parameter(is_parameter); + return SUCCESS; +} + +int32_t ComputeRepeatDeviceNumByTensorMap(const Shape& dev_matrix_shape, const Shape& tensor_map) { + int32_t ret = -1; + + // The number of repetitions is equal to the number of all devices divided by the number of devices use for + // tensor map. + int32_t device_num = std::accumulate(dev_matrix_shape.begin(), dev_matrix_shape.end(), 1, std::multiplies()); + for (auto& element : tensor_map) { + // -1 means the corresponding dimension is not split. + if (element == MAP_NONE) { + continue; + } else if ((element < 0) || (IntToSize(element) >= dev_matrix_shape.size())) { + MS_LOG(ERROR) << "Invalid tensor map: " << ShapeToString(tensor_map) << ", the dev matrix shape is " + << ShapeToString(dev_matrix_shape); + return ret; + } else { + size_t index = dev_matrix_shape.size() - IntToSize(element) - 1; + if (dev_matrix_shape[index] <= 0) { + MS_LOG(ERROR) << "Invalid dev matrix shape: " << ShapeToString(dev_matrix_shape); + return ret; + } + device_num /= dev_matrix_shape[index]; + } + } + + return device_num; +} + +Status OperatorInfo::InferAsLossDivisor() { + if (!ParallelContext::GetInstance()->loss_repeated_mean()) { + as_loss_divisor_ = 1; + return SUCCESS; + } + + if (outputs_tensor_map_.empty()) { + MS_LOG(ERROR) << name_ << ": The outputs tensor map is empty."; + return FAILED; + } + + if (outputs_tensor_map_.size() > 1) { + MS_LOG(ERROR) << name_ << ": The output size is " << outputs_tensor_map_.size() + << ", need to override this function "; + return FAILED; + } + + if (outputs_tensor_map_[0].empty()) { + as_loss_divisor_ = SizeToInt(global_device_list_.size()); + MS_LOG(INFO) << name_ << ": The output is a scalar, use the dev size " << as_loss_divisor_ << ", loss divisor."; + return SUCCESS; + } + + as_loss_divisor_ = ComputeRepeatDeviceNumByTensorMap(dev_matrix_shape_, outputs_tensor_map_[0]); + MS_LOG(INFO) << name_ << ": the dev matrix shape is " << ShapeToString(dev_matrix_shape_) + << ", the output tensor map is " << ShapeToString(outputs_tensor_map_[0]) << ", loss divisor is " + << as_loss_divisor_; + return SUCCESS; +} + +// If the operator is used as a loss, a div node is inserted for the grad of all its inputs. +Status OperatorInfo::InferVirtualDivOps() { + if (InferAsLossDivisor() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferAsLossDivisor failed."; + return FAILED; + } + + if (as_loss_divisor_ <= 0) { + MS_LOG(ERROR) << name_ << ": Invalid loss divisor: " << as_loss_divisor_; + return FAILED; + } else if (as_loss_divisor_ == 1) { + MS_LOG(INFO) << name_ << ": The loss divisor is 1, no need to create virtual div op."; + return SUCCESS; + } + + virtual_div_op_.clear(); + // if loss is repeated calculation, insert div op + Operator op = CreateVirtualDivOp(as_loss_divisor_); + virtual_div_op_.push_back(op); + return SUCCESS; +} + +Status OperatorInfo::SetInputAndOutputTypeLength(const std::vector& input_lengths, + const std::vector& output_lengths) { + if (input_lengths.size() != inputs_shape_.size()) { + MS_LOG(ERROR) << "Input_lengths: " << input_lengths.size() + << " do not have the same number of inputs shape: " << inputs_shape_.size(); + return FAILED; + } + if (output_lengths.size() != outputs_shape_.size()) { + MS_LOG(ERROR) << "Output_lengths: " << output_lengths.size() + << " do not have the same number of outputs shape: " << outputs_shape_.size(); + return FAILED; + } + inputs_type_lengths_ = input_lengths; + outputs_type_lengths_ = output_lengths; + GetOperatorCost()->SetInputAndOutputTypeLength(input_lengths, output_lengths); + return SUCCESS; +} + +void OperatorInfo::BreakingTiesForPerferringDataParallel(const StrategyPtr& stra, const CostPtr& cost) { + if (!stra->GetInputDim().empty() && !stra->GetInputDim()[0].empty()) { + CheckGlobalDeviceManager(); + auto total_device_num = g_device_manager->GetDeviceListByStageId(stra->GetInputStage()).size(); + if (IntToSize(stra->GetInputDim()[0][0]) == total_device_num) { + cost->memory_cost_ -= 1.0; + cost->communication_cost_ -= 1.0; + cost->communication_with_partial_para_ -= 1.0; + cost->communication_without_parameter_ -= 1.0; + } + } +} + +double OperatorInfo::GetForwardMemoryCostFromCNode() { + return GetOperatorCost()->GetForwardMemoryCost(inputs_tensor_info_, outputs_tensor_info_, 0); +} + +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/operator_info.h b/mindspore/ccsrc/parallel/ops_info/operator_info.h new file mode 100644 index 00000000000..9902c69ce73 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/operator_info.h @@ -0,0 +1,257 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPERATOR_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPERATOR_INFO_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/log_adapter.h" +#include "ir/base.h" +#include "common/utils.h" +#include "parallel/device_manager.h" +#include "parallel/device_matrix.h" +#include "parallel/group_manager.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_info.h" +#include "parallel/auto_parallel/costmodel.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/ops_info/ops_utils.h" + +namespace mindspore { +namespace parallel { +using ForwardOp = OperatorVector; +using MirrorOps = std::vector; +using VirtualDivOp = OperatorVector; +using TensorMaps = std::vector>; +using TensorLayouts = std::vector; +using different_type = std::vector::difference_type; +using PrimitiveAttrs = std::unordered_map; +using Strategys = std::vector; +using ReplaceGraphPtr = std::shared_ptr, AnfNodePtr>>; + +class Edge; + +class OperatorInfo { + public: + OperatorInfo(std::string name, Shapes inputs_shape, Shapes outputs_shape, PrimitiveAttrs attrs) + : name_(std::move(name)), + inputs_shape_(std::move(inputs_shape)), + outputs_shape_(std::move(outputs_shape)), + attrs_(std::move(attrs)), + is_alive_(true) { + std::vector not_parameteter(inputs_shape_.size(), false); + is_parameter_ = not_parameteter; + refkey_parameter_name_ = ""; + } + + virtual ~OperatorInfo() = default; + + Status set_is_parameter(const std::vector& is_parameter); + Status SetInputAndOutputTypeLength(const std::vector& input_lengths, + const std::vector& output_lengths); + virtual Status Init(const StrategyPtr& strategy) = 0; + virtual Status InitForCostModel(const StrategyPtr& strategy) = 0; // only init the necessary parts + + // Given the stage_id (which indicates the number of devices), + // generate all strategies for this operator + virtual Status GenerateStrategies(int32_t stage_id) = 0; + virtual OperatorCostPtr GetOperatorCost() const = 0; + virtual Status SetCostUnderStrategy(const StrategyPtr& strategy) = 0; + + virtual std::shared_ptr>> GenerateBatchStrategies(); + virtual void ReComputeBatchSplitFlagList(); + void ComputeBatchSplitFlagList(); + + double GetForwardMemoryCostFromCNode(); + // This is a common method for setting operator cost for a given strategy, in which the validity of this strategy + // is checked + Status SetCostUnderStrategyBase(const StrategyPtr& strategy); + std::vector> GetStrategyCost() { return strategy_cost_; } + // In the case of a Parameter (or a output) being used by multiple operators, the memory cost induced by + // the parameter (or a output) should be calculated only once. This method is used to + // remove this part from the 'strategy_cost_'. + Status CorrectStrategyCostForMultiOutputUse(size_t input_index); + // When the input of a operator contains WEIGHT or a output from other operators involving WEIGHT, then these input + // should stay in memory until it is used in the backward phase, which is kept in memory at the end of forward phase. + Status CorrectStrategyCostForMemoryReuse() const { return SUCCESS; } + int ComputeOpAndPrevEdgeParameterInvolved(); + + ForwardOp forward_op() const { return forward_op_; } + ForwardOp replace_op() const { return replace_op_; } + OutPutInfoVector replace_op_info() const { return replace_op_info_; } + virtual ReplaceGraphPtr replace_graph(const CNodePtr&) { return replace_graph_; } + MirrorOps mirror_ops() const { return mirror_ops_; } + VirtualDivOp virtual_div_op() const { return virtual_div_op_; } + Shape dev_matrix_shape() const { return dev_matrix_shape_; } + std::vector inputs_tensor_info() const { return inputs_tensor_info_; } + std::vector outputs_tensor_info() const { return outputs_tensor_info_; } + const std::string& name() const { return name_; } + void set_name(const std::string& name) { name_ = name; } + RankList global_device_list() const { return global_device_list_; } + + void AddSuccEdge(const std::shared_ptr& e) { succ_edges_.push_back(e); } + void AddPrevEdge(const std::shared_ptr& e) { prev_edges_.push_back(e); } + std::vector> succ_edges() const { return succ_edges_; } + std::vector> prev_edges() const { return prev_edges_; } + std::vector> GetAliveSuccEdges(); + std::vector> GetAlivePrevEdges(); + void ReplacePreEdge(const std::shared_ptr& op, const std::shared_ptr& new_edge); + void ReplaceSuccEdge(const std::shared_ptr& op, const std::shared_ptr& new_edge); + void ReplacePreEdges(const std::shared_ptr& op, const std::shared_ptr& new_edge); + void ReplaceSuccEdges(const std::shared_ptr& op, const std::shared_ptr& new_edge); + std::vector GetOutputTypeLengths() const { return GetOperatorCost()->outputs_type_lengths(); } + void SetSelectedStrategyAndCost(const StrategyPtr& s_strategy, const CostPtr& cost) { + selected_strategy_ = s_strategy; + selected_cost_ = cost; + } + StrategyPtr selected_strategy() const { return selected_strategy_; } + CostPtr selected_cost() const { return selected_cost_; } + Status InitSelectedStrategy(const StrategyPtr& s_strategy) { return Init(s_strategy); } + void set_input_value(const std::vector& input_value) { input_value_ = input_value; } + void set_outputs_dtype(const TypePtr& dtype) { outputs_dtype_ = dtype; } + void set_cnode(const CNodePtr& cnode) { cnode_ = cnode; } + bool is_alive() const { return is_alive_; } + void SetNotAlive() { is_alive_ = false; } + StrategyPtr strategy() const { return strategy_; } + void set_strategy(const StrategyPtr& strategy) { strategy_ = strategy; } + void set_refkey_parameter_name(std::string p_name) { refkey_parameter_name_ = std::move(p_name); } + const std::string& refkey_parameter_name() const { return refkey_parameter_name_; } + int used_devices() const { return used_devices_; } + // needed by rec_parser + void set_type(const std::string& type) { type_ = type; } + const std::string& type() const { return type_; } + void set_cnode_name(const std::string& cnode_name) { cnode_name_ = cnode_name; } + const std::string& cnode_name() const { return cnode_name_; } + const std::unordered_map& attrs() const { return attrs_; } + + protected: + // needed by rec_parser + std::string type_; + std::string cnode_name_; + virtual Status CheckStrategy(const StrategyPtr& strategy) = 0; + virtual Status InferTensorMap() = 0; + virtual Status InferForwardCommunication() = 0; + virtual Status InferMirrorOps() = 0; + virtual Status GetAttrs() = 0; + virtual Status InferTensorInfo() = 0; + virtual Status InferDevMatrixShape() = 0; + void SetDeviceListByStrategy(); + void SetRepeatedCalcDevMatrix(); + Status CreateGroupByTensorMap(const Shape& tensor_map, std::vector* group); + Status CreateGroupByDim(size_t axis, std::vector* group); + Status InferAttrs(); + void ResetQueueMember(); + Status InitWithAutoRepeatCalc(const StrategyPtr& strategy); + Status InitWithManualRepeatCalc(const StrategyPtr& strategy); + Status InitForCostModelWithAutoRepeatCalc(const StrategyPtr& strategy); + Status InitForCostModelWithManualRepeatCalc(const StrategyPtr& strategy); + Status InferRepeatedCalcInfo(); + Status InferVirtualDivOps(); + + // Calculate the number of repeated calculations for the output by the number of devices and the output tensor map. + // The tensor map of Outputs[0] is used by default. If there are multiple outputs, need to identify which output + // is used for grad and overload the function. If the output is a scalar, need to override the function too. + virtual Status InferAsLossDivisor(); + Status InferSliceShape(const Strategys& inputs_strategy, const Strategys& outputs_strategy, + Shapes* inputs_slice_shape, Shapes* outputs_slice_shape); + void BreakingTiesForPerferringDataParallel(const StrategyPtr&, const CostPtr&); + + std::string name_; + Shapes inputs_shape_; + Shapes outputs_shape_; + std::unordered_map attrs_; + std::vector input_value_; + TypePtr outputs_dtype_; + + StrategyPtr strategy_; + std::vector inputs_tensor_info_; + std::vector outputs_tensor_info_; + Shape dev_matrix_shape_; // if repeated calculation, it contains the repeated_calc_num as the first dimension + int32_t repeated_calc_num_ = 1; + int32_t as_loss_divisor_ = 1; + TensorMaps inputs_tensor_map_; + TensorMaps outputs_tensor_map_; + ForwardOp forward_op_; + ForwardOp replace_op_; + OutPutInfoVector replace_op_info_; + ReplaceGraphPtr replace_graph_; + MirrorOps mirror_ops_; + VirtualDivOp virtual_div_op_; + RankList global_device_list_; // the size of global_device_list equal to the size of stageID + RankList local_device_list_; // the size equal to global_device_list_.size() / repeated_calc_num_ + bool infer_attrs_completed_ = false; + + bool is_auto_parallel_ = false; // false: semi_auto_parallel; true: auto_parallel + // 'corrected_input_indices_' used to store the indices of input that have ALREADY been corrected. + std::vector corrected_input_indices_; + // Given a parallization strategy, there is a cost. + std::vector> strategy_cost_; + // For each input in 'inputs_', there is a bool variable indicating whether that the corresponding input is parameter + std::vector is_parameter_; + // For each input in 'inputs_', a bool variable is true if the corresponding one is a parameter or a output of + // pre-operator that has parameters as input. + std::vector is_parameter_involve_; + int is_output_parameter_involve_ = -1; // -1: unset; 0: not parameter_involved; 1: parameter_involved + // for each input and output, the followings record the number of bytes of each element + std::vector inputs_type_lengths_; + std::vector outputs_type_lengths_; + std::vector> prev_edges_; + std::vector> succ_edges_; + StrategyPtr selected_strategy_; + // Used in DP algorithm + bool is_alive_; + CostPtr selected_cost_; + std::vector split_flag_list_; + std::string refkey_parameter_name_; + CNodePtr cnode_; + int32_t used_devices_ = -1; +}; + +Shape GetSliceShape(const Shape& tensor_shape, const Dimensions& strategy); +Status CheckStrategyValue(const StrategyPtr& strategy, const Shapes& inputs_shape, bool); +Operator CreateVirtualDivOp(int32_t div_num); +Operator CreateAllReduceOp(const std::string& reduce_op, const std::string& group); +Operator CreateGetTensorSliceOp(const TensorLayout& tensor_layout); +OperatorVector CreateMirrorOps(const std::string& group_name, size_t dev_num); +int32_t ComputeRepeatDeviceNumByTensorMap(const Shape& dev_matrix_shape, const Shape& tensor_map); +std::shared_ptr>> GenerateBatchStrategiesBySplitFlag( + const Shapes& shapes, const std::vector& split_flag_list); + +void PrintStrategy(const StrategyPtr& strategy); +// generate strategies for that all inputs' dimensions are independent, such as: ([a, b, c, d]) +Status GenerateStrategiesForIndependentInputs(int32_t stage_id, const Shapes& inputs_shape, + const Shapes& splittable_inputs, std::vector* sp_vector); +// generate strategies for that have two inputs, and input0 or input1 maybe broadcast, +// and the corresponding dimensions that are not broadcast are all relevant dimensions +// such as: ([a, b, c, d], [a, b, c, d]) or ([b, c, d], [a, b, c, d]) or ([1, c, d], [a, b, c, d]) +// or ([a, b, c, d], [b, c, d]) or ([a, b, c, d], [1, c, d]) +// or ([a, 1], [1, b]) or ([a, b, c, d], [1, b, c, d]) or ([a, b, c, 1], [1, b, c, d]) +Status GenerateStrategiesWithBroadcast(int32_t stage_id, const Shapes& inputs_shape, const Shapes& splittable_inputs, + std::vector* sp_vector); + +Shapes GetRefKeyNodeShape(const AnfNodePtr& node, const FuncGraphPtr& func_graph); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPERATOR_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/ops_info_head_files.h b/mindspore/ccsrc/parallel/ops_info/ops_info_head_files.h new file mode 100644 index 00000000000..7eb24bd30a3 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/ops_info_head_files.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPS_INFO_HEAD_FILES_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPS_INFO_HEAD_FILES_H_ + +#include "parallel/ops_info/activation_info.h" +#include "parallel/ops_info/batch_parallel_info.h" +#include "parallel/ops_info/loss_info.h" +#include "parallel/ops_info/matmul_info.h" +#include "parallel/ops_info/onehot_info.h" +#include "parallel/ops_info/virtual_dataset_info.h" +#include "parallel/ops_info/arithmetic_info.h" +#include "parallel/ops_info/elementary_function_info.h" +#include "parallel/ops_info/comparison_function_info.h" +#include "parallel/ops_info/l2_normalize_info.h" +#include "parallel/ops_info/reduce_method_info.h" +#include "parallel/ops_info/transpose_info.h" +#include "parallel/ops_info/prelu_info.h" +#include "parallel/ops_info/reshape_info.h" +#include "parallel/ops_info/generator_info.h" +#include "parallel/ops_info/dropout_do_mask_info.h" +#include "parallel/ops_info/get_next_info.h" +#include "parallel/ops_info/bias_add_info.h" + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_HEAD_FILES_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/ops_utils.h b/mindspore/ccsrc/parallel/ops_info/ops_utils.h new file mode 100644 index 00000000000..a25200c3c18 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/ops_utils.h @@ -0,0 +1,252 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPS_UTILS_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPS_UTILS_H_ + +namespace mindspore { +namespace parallel { +constexpr size_t PRELU_INPUTS_SIZE = 2; +constexpr size_t PRELU_OUTPUTS_SIZE = 1; +constexpr size_t PRELU_SECOND_INPUT_SIZE = 1; +constexpr int32_t PRELU_CHANNEL_INDEX = 1; +constexpr int32_t PRELU_CHANNEL_STRATEGY = 1; +constexpr size_t MATMUL_ATTRS_SIZE = 2; +constexpr size_t MATMUL_INPUTS_SIZE = 2; +constexpr size_t MATMUL_OUTPUTS_SIZE = 1; +constexpr size_t ACTIVATION_ATTR_SIZE = 1; +constexpr size_t SOFTMAX_ATTR_SIZE = 1; +constexpr size_t ACTIVATION_INPUTS_SIZE = 1; +constexpr size_t ACTIVATION_OUTPUTS_SIZE = 1; +constexpr size_t SoftmaxCrossEntropyWithLogitsAttrSize = 1; +constexpr size_t SoftmaxCrossEntropyWithLogitsInputsSize = 2; +constexpr size_t SoftmaxCrossEntropyWithLogitsOutputsSize = 2; +constexpr double EPS = 1e-6; + +constexpr char AUTO_PARALLEL_RUN_ONCE_ONLY[] = "auto_parallel_run_once_only"; +constexpr char SEMI_AUTO_PARALLEL_RUN_ONCE_ONLY[] = "semi_auto_parallel_run_once_only"; +constexpr char STRATEGY[] = "strategy"; +constexpr char GEN_STRATEGY[] = "gen_strategy"; +constexpr char REDUCE_OP_SUM[] = "sum"; +constexpr char REDUCE_OP_MAX[] = "max"; +constexpr char REDUCE_OP_MIN[] = "min"; +constexpr char OP_PATH[] = "mindspore.ops.operations"; +constexpr char GET_OP_FUNCTION_PATH[] = "mindspore.parallel._utils"; +constexpr char GET_OP_FUNCTION[] = "_get_python_op"; +constexpr char KEEP_DIMS[] = "keep_dims"; +constexpr char CROSS_BATCH[] = "cross_batch"; +constexpr char STEP_PARALLEL_BEGIN[] = "step_parallel_begin"; +constexpr char STEP_PARALLEL_END[] = "step_parallel_end"; +constexpr char STEP_AUTO_PARALLEL_BEGIN[] = "step_auto_parallel_begin.dot"; + +constexpr char RELU_TYPE[] = "relu"; +constexpr char RELU6_TYPE[] = "relu6"; +constexpr char SIGMOID_TYPE[] = "sigmoid"; +constexpr char OP[] = "op"; +constexpr char IDENTITY_INFO[] = "identity_info"; +constexpr char DIVISOR[] = "divisor"; +constexpr char NONE[] = "None"; +constexpr char DEPEND[] = "depend"; +constexpr char BATCH_PARALLEL[] = "BatchParallel"; + +constexpr char ACTIVATION_TYPE[] = "activation_type"; +constexpr char TRANSPOSE_A[] = "transpose_a"; +constexpr char TRANSPOSE_B[] = "transpose_b"; +constexpr char SHAPE[] = "shape"; +constexpr char BEGIN_MASK[] = "begin_mask"; +constexpr char END_MASK[] = "end_mask"; +constexpr char ELLIPSIS_MASK[] = "ellipsis_mask"; +constexpr char NEW_AXIS_MASK[] = "new_axis_mask"; +constexpr char SHRINK_AXIS_MASK[] = "shrink_axis_mask"; +constexpr char BEGIN[] = "begin"; +constexpr char END[] = "end"; +constexpr char STRIDES[] = "strides"; +constexpr char GROUP[] = "group"; +constexpr char AXIS[] = "axis"; +constexpr char OUTPUT_NUM[] = "output_num"; +constexpr char SPLIT_COUNT[] = "split_count"; +constexpr char SPLIT_DIM[] = "split_dim"; +constexpr char CONCAT_DIM[] = "concat_dim"; +constexpr char FORWARD[] = "forward"; +constexpr char BACKWARD[] = "backward"; +constexpr char REDISTRIBUTION[] = "redistribution"; +constexpr char REPLACE[] = "replace"; +constexpr char CONNSYMBOL[] = "/"; +constexpr char INSTANCE_NAME[] = "instance_name"; +constexpr char SPLIT_SENS[] = "split_sens"; +constexpr char SPLIT_TENSOR[] = "split_tensor"; +constexpr char DEV_MAT[] = "dev_mat"; +constexpr char TENSOR_MAP[] = "tensor_map"; +constexpr char SEED0[] = "Seed0"; +constexpr char SEED1[] = "Seed1"; +constexpr char KEEP_PROB[] = "keep_prob"; +constexpr char SRC[] = "src"; +constexpr char CLONE_INFO[] = "clone_info"; +constexpr char CLONED[] = "cloned"; +constexpr char BE_CLONED[] = "be_cloned"; +constexpr char CLONED_INDEX[] = "cloned_index"; +constexpr char BE_CLONED_INDEX[] = "be_cloned_index"; +constexpr char GROUP_RANKS[] = "group_ranks"; +constexpr char IS_IN_FORWARD[] = "is_in_forward"; +constexpr char DEFAULT_INPUT[] = "default_input"; +constexpr char DTYPE[] = "dtype"; +constexpr char DEV_NUM[] = "dev_num"; +constexpr char MEAN_FLAG[] = "mean_flag"; +constexpr char TYPES[] = "types"; +constexpr char SHAPES[] = "shapes"; +constexpr char GETNEXT_NUM[] = "output_num"; +constexpr char SHARED_NAME[] = "shared_name"; +constexpr char MIRROR_OP[] = "mirror_op"; +constexpr char FORWARD_OP[] = "forward_op"; +constexpr char REDISTRIBUTION_OP[] = "redistribution_op"; + +// Operator +constexpr char VIRTUAL_DIV[] = "_VirtualDiv"; +constexpr char GET_TENSOR_SLICE[] = "_GetTensorSlice"; +constexpr char SPLIT[] = "Split"; +constexpr char ALL_TO_ALL[] = "_AlltoAll"; +constexpr char PERMUTE_BY_AXIS[] = "PermuteByAxis"; +constexpr char CONCAT_BY_AXIS[] = "ConcatByAxis"; +constexpr char SPLIT_BY_AXIS[] = "SplitByAxis"; +constexpr char ALL_REDUCE[] = "AllReduce"; +constexpr char MIRROR_OPERATOR[] = "_MirrorOperator"; +constexpr char STRIDED_SLICE[] = "StridedSlice"; +constexpr char ALL_GATHER[] = "AllGather"; +constexpr char REDUCE_SCATTER[] = "ReduceScatter"; +constexpr char CONCAT[] = "Concat"; +constexpr char SOFTMAX_CROSS_ENTROPY_WITH_LOGITS[] = "SoftmaxCrossEntropyWithLogits"; +constexpr char MATMUL[] = "MatMul"; +constexpr char GELU[] = "Gelu"; +constexpr char TANH[] = "Tanh"; +constexpr char SOFTMAX[] = "Softmax"; +constexpr char LOG_SOFTMAX[] = "LogSoftmax"; +constexpr char ACTIVATION[] = "Activation"; +constexpr char PRELU[] = "PReLU"; +constexpr char FLOORDIV[] = "FloorDiv"; +constexpr char MAXPOOL[] = "MaxPool"; +constexpr char MAXPOOLV2[] = "MaxPoolV2"; +constexpr char L2_NORMALIZE[] = "L2Normalize"; +constexpr char TRANSPOSE[] = "Transpose"; +constexpr char RESHAPE[] = "Reshape"; +constexpr char TENSOR_ADD[] = "TensorAdd"; +constexpr char BIAS_ADD[] = "BiasAdd"; +constexpr char SUB[] = "Sub"; +constexpr char MUL[] = "Mul"; +constexpr char DIV[] = "Div"; +constexpr char REAL_DIV[] = "RealDiv"; +constexpr char ASSIGN_SUB[] = "AssignSub"; +constexpr char GREATER[] = "Greater"; +constexpr char VIRTUAL_DATA_SET[] = "_VirtualDataset"; +constexpr char VIRTUAL_DATA_SET_INFO[] = "VirtualDatasetInfo"; +constexpr char SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS[] = "SparseSoftmaxCrossEntropyWithLogits"; +constexpr char RELU[] = "ReLU"; +constexpr char ONEHOT[] = "OneHot"; +constexpr char DROPOUT_DO_MASK[] = "DropoutDoMask"; +constexpr char DROPOUT_GEN_MASK[] = "DropoutGenMask"; +constexpr char REDUCE_MAX[] = "ReduceMax"; +constexpr char REDUCE_MIN[] = "ReduceMin"; +constexpr char REDUCE_SUM[] = "ReduceSum"; +constexpr char REDUCE_MEAN[] = "ReduceMean"; +constexpr char ARGMAXWITHVALUE[] = "ArgMaxWithValue"; +constexpr char ARGMINWITHVALUE[] = "ArgMinWithValue"; +constexpr char CONV2D[] = "Conv2D"; +constexpr char FUSE_BATCH_NORM[] = "FusedBatchNorm"; +constexpr char BATCH_NORM[] = "BatchNorm"; +constexpr char POOLING[] = "Pooling"; +constexpr char CAST[] = "Cast"; +constexpr char MAX_POOL_WITH_ARGMAX[] = "MaxPoolWithArgmax"; +constexpr char SIMPLE_MEAN[] = "SimpleMean"; +constexpr char FLATTEN[] = "Flatten"; +constexpr char J[] = "J"; +constexpr char TMPIDENTITY_INFO_NAME[] = "identity_info"; +constexpr char COS[] = "Cos"; +constexpr char ACOS[] = "ACos"; +constexpr char EXP[] = "Exp"; +constexpr char LOG[] = "Log"; +constexpr char SIGMOID[] = "Sigmoid"; +constexpr char POW[] = "Pow"; +constexpr char MAXIMUM[] = "Maximum"; +constexpr char EQUAL[] = "Equal"; +constexpr char LOGICALNOT[] = "LogicalNot"; +constexpr char GATHERV2[] = "GatherV2"; +constexpr char STRIDEDSLICE[] = "StridedSlice"; +constexpr char BROADCAST[] = "Broadcast"; +constexpr char SQRT[] = "Sqrt"; +constexpr char ASSIGN[] = "Assign"; +constexpr char GET_NEXT[] = "GetNext"; +constexpr char SQUEEZE[] = "Squeeze"; + +// Parallel don't care +constexpr char TUPLE_GETITEM[] = "tuple_getitem"; +constexpr char STRING_EQUAL[] = "string_equal"; +constexpr char MAKE_TUPLE[] = "make_tuple"; +constexpr char MAKE_LIST[] = "make_list"; +constexpr char MAKE_DICT[] = "make_dict"; +constexpr char MAKE_SLICE[] = "make_slice"; +constexpr char MAKE_RECORD[] = "make_record"; +constexpr char LIST_GETITEM[] = "list_getitem"; +constexpr char ARRAY_GETITEM[] = "array_getitem"; +constexpr char TUPLE_SETITEM[] = "tuple_setitem"; +constexpr char LIST_SETITEM[] = "list_setitem"; +constexpr char ARRAY_SETITEM[] = "array_setitem"; +constexpr char DICT_GETITEM[] = "dict_getitem"; +constexpr char LIST_APPEND[] = "list_append"; +constexpr char LIST_MAP[] = "list_map"; +constexpr char LIST_REDUCE[] = "list_reduce"; +constexpr char TUPLE_REVERSED[] = "tuple_reversed"; +constexpr char TILE_SHAPE[] = "tile_shape"; +constexpr char REDUCED_SHAPE[] = "reduced_shape"; +constexpr char TUPLE_DIV[] = "tuple_div"; +constexpr char TUPLE_TO_ARRAY[] = "tuple_to_array"; +constexpr char VIRTUALLOSS[] = "VirtualLoss"; +constexpr char RETURN[] = "return"; +constexpr char ENV_GETITEM[] = "env_getitem"; +constexpr char IDENTITY[] = "identity"; +constexpr char PARTIAL[] = "partial"; +constexpr char ENVSETITEM[] = "env_setitem"; +constexpr char ENVGETITEM[] = "env_getitem"; +constexpr char ENVADD[] = "env_add"; +constexpr char MAKEREFKEY[] = "MakeRefKey"; +constexpr char MAKEREF[] = "make_ref"; +constexpr char GETREFKEY[] = "get_ref_key"; +constexpr char GETREFVALUE[] = "get_ref_value"; +constexpr char GETREFORIGIN[] = "get_ref_origin"; +constexpr char STATESETITEM[] = "state_setitem"; +constexpr char SCALARSUMMARY[] = "ScalarSummary"; +constexpr char IMAGESUMMARY[] = "ImageSummary"; +constexpr char TENSORSUMMARY[] = "TensorSummary"; +constexpr char BROADCASTGRADIENTARGS[] = "BroadcastGradientArgs"; +constexpr char INVERTPERMUTATION[] = "InvertPermutation"; +constexpr char CONTROLDEPEND[] = "ControlDepend"; +constexpr char DOT[] = "dot"; +constexpr char IM2COL[] = "im2col"; +constexpr char COL2IM[] = "col2im"; +constexpr char IM2COLV1[] = "im2col_v1"; +constexpr char COL2IMV1[] = "col2im_v1"; +constexpr char RESOLVE[] = "resolve"; +constexpr char EMBED[] = "embed"; +constexpr char CREATINSTANCE[] = "create_instance"; +constexpr char ZEROSLIKETENSOR[] = "zeros_like_tensor"; +constexpr char REF_TO_EMBED[] = "RefToEmbed"; +constexpr char STOP_GRADIENT[] = "stop_gradient"; + +constexpr size_t LAST_INDEX(size_t s) { return s - 1; } +constexpr size_t SECOND_FROM_END(size_t s) { return s - 2; } +constexpr size_t THIRD_FROM_END(size_t s) { return s - 3; } +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_OPS_UTILS_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/prelu_info.cc b/mindspore/ccsrc/parallel/ops_info/prelu_info.cc new file mode 100644 index 00000000000..9aa85133317 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/prelu_info.cc @@ -0,0 +1,247 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/prelu_info.h" + +#include +#include +#include + +#include "parallel/device_manager.h" +#include "parallel/device_matrix.h" +#include "parallel/step_parallel.h" +#include "utils/convert_utils.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +/* + * prelu has 2 input + * A: A float tensor of shape [NCHW] representing the output of the preview layer. + * w: Float Tensor, w > 0: there is only two shapes are legitimate: 1, or the number of channels at input. + * the strategy of w should equal to the channel dimension of strategy of A + */ +Status PReLUInfo::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy."; + } + return FAILED; + } + std::vector stra = strategy->GetInputDim(); + if (stra[1].size() != PRELU_SECOND_INPUT_SIZE) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy size."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy size."; + } + return FAILED; + } + if ((stra[0][PRELU_CHANNEL_INDEX] != PRELU_CHANNEL_STRATEGY) || (stra[1][0] != PRELU_CHANNEL_STRATEGY)) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid channel strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid channel strategy."; + } + return FAILED; + } + return SUCCESS; +} + +/* + * device matrix is same with the strategy matrix + */ +Status PReLUInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + input_strategy_ = input_strategy; + dev_matrix_shape_ = input_strategy; + return SUCCESS; +} + +Status PReLUInfo::InferMirrorOps() { + Shape param_tensor_map = inputs_tensor_map_[1]; + std::vector param_group; + if (CreateGroupByTensorMap(param_tensor_map, ¶m_group) != SUCCESS) { + return FAILED; + } else if (param_group.empty()) { + MS_LOG(INFO) << name_ << ": The mirror ops is empty."; + return SUCCESS; + } + OperatorVector op_for_param; + op_for_param = CreateMirrorOps(param_group[0].name(), param_group[0].GetDevNum()); + // op_for_inputs is empty + OperatorVector op_for_inputs; + mirror_ops_.push_back(op_for_inputs); + mirror_ops_.push_back(op_for_param); + std::string group_name = param_group[0].name(); + MS_LOG(INFO) << name_ << ": The mirror ops group is " << group_name; + return SUCCESS; +} + +Status PReLUInfo::InferForwardCommunication() { return SUCCESS; } + +/* + * the output tensor map is the same as the input tensor map + */ +Status PReLUInfo::InferTensorMap() { + TensorMap input_tensor_map; + // such as 4: input_tensor_map [3,2,1,0] + for (size_t i = 0; i < inputs_shape_[0].size(); ++i) { + input_tensor_map.push_back((int32_t)(inputs_shape_[0].size() - i - 1)); + } + + TensorMap param_tensor_map; + param_tensor_map.push_back(input_tensor_map.at(1)); + inputs_tensor_map_.push_back(input_tensor_map); + inputs_tensor_map_.push_back(param_tensor_map); + outputs_tensor_map_.push_back(input_tensor_map); + return SUCCESS; +} + +Dimensions PReLUInfo::GetOutputStrategy() { + Dimensions output_strategy = input_strategy_; + return output_strategy; +} + +Status PReLUInfo::InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout) { + if (inputs_layout == nullptr || outputs_layout == nullptr) { + MS_LOG(ERROR) << name_ << ": InferTensorLayout: the layout is null."; + return FAILED; + } + TensorLayout input_layout, param_layout, output_layout; + if ((input_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], inputs_shape_[0]) != SUCCESS) || + (param_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[1], inputs_shape_[1]) != SUCCESS) || + (output_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], outputs_shape_[0]) != SUCCESS)) { + return FAILED; + } + inputs_layout->push_back(input_layout); + inputs_layout->push_back(param_layout); + outputs_layout->push_back(output_layout); + return SUCCESS; +} + +Status PReLUInfo::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + Shape param_shape = inputs_shape_.at(1); + Shape output_shape = outputs_shape_.at(0); + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Dimensions output_strategy = GetOutputStrategy(); + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = {output_strategy}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_slice_shape = inputs_slice_shape.at(0); + Shape param_slice_shape = inputs_slice_shape.at(1); + Shape output_slice_shape = outputs_slice_shape.at(0); + + // infer tensor layout + TensorLayouts inputs_layout, outputs_layout; + if (InferTensorLayout(&inputs_layout, &outputs_layout) != SUCCESS) { + return FAILED; + } + + TensorLayout input_layout = inputs_layout.at(0); + TensorLayout param_layout = inputs_layout.at(1); + TensorLayout output_layout = outputs_layout.at(0); + TensorInfo input_tensor_info(input_layout, input_shape, input_slice_shape); + TensorInfo param_tensor_info(param_layout, param_shape, param_slice_shape); + TensorInfo output_tensor_info(output_layout, output_shape, output_slice_shape); + + inputs_tensor_info_.push_back(input_tensor_info); + inputs_tensor_info_.push_back(param_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + return SUCCESS; +} + +Status PReLUInfo::GetAttrs() { + if ((inputs_shape_.size() != PRELU_INPUTS_SIZE) || (outputs_shape_.size() != PRELU_OUTPUTS_SIZE)) { + MS_LOG(ERROR) << name_ << ": Inputs shape size " << inputs_shape_.size() << " or outputs shape size " + << outputs_shape_.size() << " is wrong."; + return FAILED; + } + return SUCCESS; +} + +Status PReLUInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << ": Init success."; + return SUCCESS; +} + +Status PReLUInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init for cost model success."; + return SUCCESS; +} + +Status PReLUInfo::GenerateStrategies(int32_t stage_id) { + if (inputs_shape_.size() != PRELU_INPUTS_SIZE) { + return FAILED; + } + if (inputs_shape_[1].size() != PRELU_SECOND_INPUT_SIZE) { + return FAILED; + } + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size(), 1); + input0_split[1] = 0; + Shape input1_split(inputs_shape_[1].size(), 0); + Shapes splittable_inputs = {input0_split, input1_split}; + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategiesForIndependentInputs failed"; + return FAILED; + } + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} + +Status PReLUInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/prelu_info.h b/mindspore/ccsrc/parallel/ops_info/prelu_info.h new file mode 100644 index 00000000000..d7dac462919 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/prelu_info.h @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_PRELU_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_PRELU_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +/* + * parallel class for PReLU Primitive + */ +class PReLUInfo : public OperatorInfo { + public: + PReLUInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + prelucost_ptr = std::make_shared(); + } + ~PReLUInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + Status GenerateStrategies(int32_t stage_id) override; + OperatorCostPtr GetOperatorCost() const override { return prelucost_ptr; } + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout); + Status GetAttrs() override; + Dimensions GetOutputStrategy(); + + private: + Dimensions input_strategy_; + PReLUCostPtr prelucost_ptr; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_PRELU_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc new file mode 100644 index 00000000000..216357d5abc --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc @@ -0,0 +1,567 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/reduce_method_info.h" + +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "parallel/tensor_layout/tensor_redistribution.h" +#include "parallel/device_manager.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +Status ReduceMethod::CheckStrategy(const StrategyPtr &strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy."; + } + return FAILED; + } + + return SUCCESS; +} + +Status ReduceMethod::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + + dev_matrix_shape_ = input_strategy; + + return SUCCESS; +} + +std::vector ReduceMethod::reduce_dim() { + std::vector dim_list; + if (input_value_.size() < 2) { + MS_LOG(EXCEPTION) << name_ << ": Input value size is smaller than 2."; + } + if (input_value_.back() == nullptr) { + MS_LOG(EXCEPTION) << name_ << ": Input value is nullptr."; + } + MS_ASSERT(inputs_shape_.size() == 1); + auto input_dim = inputs_shape_.at(0).size(); + if (input_value_.back()->isa()) { + auto attr_axis = GetValue>(input_value_.back()); + // axis is (), reduce all dim + if (attr_axis.empty()) { + for (size_t i = 0; i < input_dim; ++i) { + dim_list.push_back(SizeToInt(i)); + } + } else { + for (auto &axis : attr_axis) { + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } + } + } else if (input_value_.back()->isa()) { + int axis = GetValue(input_value_.back()); + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } else { + MS_LOG(EXCEPTION) << "Axis type is invalid."; + } + + return dim_list; +} + +Status ReduceMethod::GetAttrs() { + // get attr cross_batch and keep_dims + auto keep_dims_iter = attrs_.find(KEEP_DIMS); + if (keep_dims_iter == attrs_.end()) { + MS_LOG(ERROR) << name_ << ": Don't have attr keep_dims."; + return FAILED; + } + + if (keep_dims_iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(keep_dims_iter->second); + if (!keep_dims_iter->second->isa()) { + MS_LOG(ERROR) << name_ << ": Keep_dims is not a bool."; + return FAILED; + } + keepdims_ = keep_dims_iter->second->cast()->value(); + } + + auto cross_batch_iter = attrs_.find(CROSS_BATCH); + if (cross_batch_iter != attrs_.end()) { + MS_EXCEPTION_IF_NULL(cross_batch_iter->second); + if (!cross_batch_iter->second->isa()) { + MS_LOG(ERROR) << name_ << ": cross_batch is not a bool."; + return FAILED; + } + cross_batch_ = cross_batch_iter->second->cast()->value(); + } + reducemethodcost_ptr_->set_cross_batch(cross_batch_); + + return SUCCESS; +} + +Status ReduceMethod::InferTensorMap() { + std::vector tensor_map_index, dim_list, output_tensor_map; + size_t size = inputs_shape_.at(0).size(); + // such as 4: tensor_map_index [3,2,1,0] + for (size_t i = 0; i < size; ++i) { + tensor_map_index.push_back((int32_t)(size - 1 - i)); + } + dim_list = reduce_dim(); + for (size_t i = 0; i < size; ++i) { + if (find(dim_list.begin(), dim_list.end(), SizeToInt(i)) != dim_list.end()) { + if (keepdims_) { + output_tensor_map.push_back(-1); + } else { + continue; + } + } else { + output_tensor_map.push_back(tensor_map_index[i]); + } + } + inputs_tensor_map_.push_back(tensor_map_index); + outputs_tensor_map_.push_back(output_tensor_map); + + return SUCCESS; +} + +bool IsDataParallelStrategy(const Dimensions &strategy) { + CheckGlobalDeviceManager(); + size_t total_dev_num = g_device_manager->GetDeviceListByStageId(0).size(); + if (strategy.empty()) { + MS_LOG(EXCEPTION) << "IsDataParallelStrategy: strategy is empty"; + } + + return (IntToSize(strategy[0]) == total_dev_num); +} + +Status ReduceMethod::InferForwardCommunication() { + Dimensions stra = strategy_->GetInputDim().at(0); + if (cross_batch_ && IsDataParallelStrategy(stra)) { + MS_LOG(INFO) << name_ << ": cross_batch is True, don't need to InferForwardCommunication"; + return SUCCESS; + } + if (cross_batch_) { + MS_LOG(INFO) << name_ << ": cross_batch is True, don't need to InferForwardCommunication"; + return SUCCESS; + } + forward_op_.clear(); + std::vector dim_list = reduce_dim(); + size_t size = stra.size(); + // judge if the reduce dim is partitioned. + Shape group_creat_map; + if (dev_matrix_shape_.size() > size) { + group_creat_map.push_back(SizeToInt(dev_matrix_shape_.size() - size_t(1))); + } + for (size_t index = 0; index < size; ++index) { + auto pos = + std::find_if(dim_list.begin(), dim_list.end(), [index](const int32_t &dim) { return SizeToInt(index) == dim; }); + if (pos != dim_list.end() && stra[index] != 1) { + continue; + } + group_creat_map.push_back(SizeToInt(size) - SizeToInt(index) - 1); + } + std::vector forward_group; + if (CreateGroupByTensorMap(group_creat_map, &forward_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferForwardCommunication group failed."; + return FAILED; + } + if (!forward_group.empty()) { + Operator op = CreateAllReduceOp(reduce_method_, forward_group[0].name()); + forward_op_.push_back(op); + std::string group_name = forward_group[0].name(); + MS_LOG(INFO) << name_ << ": Forward communication group is " << group_name; + } + + return SUCCESS; +} + +ForwardOp CreatReduceMeanForwardOp(const std::vector &forward_group, const TypePtr &dtype) { + // Creat AllReduceSum op + Operator op0 = CreateAllReduceOp(REDUCE_OP_SUM, forward_group[0].name()); + std::string group_name = forward_group[0].name(); + MS_LOG(INFO) << "The group of forward all reduce is " << group_name; + + // Creat RealDiv op + OperatorName operator1_name = REAL_DIV; + std::list device_list = forward_group[0].GetDevicesList(); + auto divisor = static_cast(device_list.size()); + py::tuple tuple = py::make_tuple(divisor); + mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tuple, dtype); + ValuePtr op1_param_value = MakeValue(tensor_ptr); + Attr op1_param = std::make_pair("divisor", op1_param_value); + OperatorParams operator1_params = {std::make_pair(op1_param, 2)}; + OperatorAttrs operator1_attrs; + OperatorArgs operator1_args = std::make_pair(operator1_attrs, operator1_params); + Operator op1 = std::make_pair(operator1_name, operator1_args); + ForwardOp forward_op = {op0, op1}; + + std::string dtype_name = dtype->ToString(); + MS_LOG(INFO) << "The divisor of Div op is " << device_list.size() << ", the dtype is " << dtype_name; + return forward_op; +} + +Status ReduceMeanInfo::InferForwardCommunication() { + Dimensions stra = strategy_->GetInputDim().at(0); + if (cross_batch_ && IsDataParallelStrategy(stra)) { + MS_LOG(INFO) << name_ << ": cross_batch is True, don't need to InferForwardCommunication"; + return SUCCESS; + } + forward_op_.clear(); + std::vector dim_list = reduce_dim(); + size_t size = stra.size(); + // judge if the reduce dim is partitioned. + Shape group_creat_map; + if (dev_matrix_shape_.size() > size) { + group_creat_map.push_back(SizeToInt(dev_matrix_shape_.size() - size_t(1))); + } + for (size_t index = 0; index < size; ++index) { + auto pos = + std::find_if(dim_list.begin(), dim_list.end(), [index](const int32_t &dim) { return SizeToInt(index) == dim; }); + if (pos != dim_list.end() && stra[index] != 1) { + continue; + } + group_creat_map.push_back(SizeToInt(size) - SizeToInt(index) - 1); + } + std::vector forward_group; + if (CreateGroupByTensorMap(group_creat_map, &forward_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferForwardCommunication group failed."; + return FAILED; + } + if (!forward_group.empty()) { + if ((outputs_dtype_ == nullptr) || !outputs_dtype_->isa()) { + MS_LOG(ERROR) << name_ << ": The dtype of output is not Array"; + return FAILED; + } + + auto element_type = outputs_dtype_->cast()->element(); + forward_op_ = CreatReduceMeanForwardOp(forward_group, element_type); + } + + return SUCCESS; +} + +Status ReduceMethod::InferMirrorOps() { + mirror_ops_.clear(); + Shape input_tensor_map = inputs_tensor_map_.at(0); + std::vector input_group; + if (CreateGroupByTensorMap(input_tensor_map, &input_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << " Infer MirrorOps failed."; + return FAILED; + } + + OperatorVector op_for_weight; + OperatorVector op_for_reduce_axis; // helper node + if (input_group.empty()) { + MS_LOG(INFO) << name_ << ": The mirror ops is empty."; + return SUCCESS; + } else { + op_for_weight = CreateMirrorOps(input_group[0].name(), input_group[0].GetDevNum()); + mirror_ops_.push_back(op_for_weight); + mirror_ops_.push_back(op_for_reduce_axis); + std::string group_name = input_group[0].name(); + MS_LOG(INFO) << name_ << ": Create the mirror ops for weight success, the group is " << group_name; + } + + return SUCCESS; +} + +Status ArgMaxWithValueInfo::InferMirrorOps() { + mirror_ops_.clear(); + Shape input_tensor_map = inputs_tensor_map_.at(0); + std::vector input_group; + if (CreateGroupByTensorMap(input_tensor_map, &input_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Infer MirrorOps failed."; + return FAILED; + } + + OperatorVector op_for_weight; + if (input_group.empty()) { + MS_LOG(INFO) << name_ << ": The mirror ops is empty."; + return SUCCESS; + } else { + op_for_weight = CreateMirrorOps(input_group[0].name(), input_group[0].GetDevNum()); + mirror_ops_.push_back(op_for_weight); + MS_LOG(INFO) << name_ << ": Create the mirror ops for weight success."; + } + + return SUCCESS; +} + +Dimensions ReduceMethod::InferOutputStrategy() { + std::vector dim_list = reduce_dim(); + Dimensions output_strategy; + Dimensions stra = strategy_->GetInputDim().at(0); + // if keepdims_ is true,then output strategy is same with input. + for (size_t i = 0; i < stra.size(); ++i) { + if (find(dim_list.begin(), dim_list.end(), SizeToInt(i)) != dim_list.end()) { + if (keepdims_) { + output_strategy.push_back(1); + } + } else { + output_strategy.push_back(stra[i]); + } + } + return output_strategy; +} + +Status ReduceMethod::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + Shape output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Dimensions output_strategy = InferOutputStrategy(); + + Strategys outputs_strategy = {output_strategy}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_slice_shape = inputs_slice_shape.at(0); + Shape output_slice_shape = outputs_slice_shape.at(0); + + TensorLayout input_tensor_layout, output_tensor_layout; + if ((input_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], input_shape) != SUCCESS) || + (output_tensor_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], output_shape) != SUCCESS)) { + return FAILED; + } + + std::vector dim_list = reduce_dim(); + TensorInfo input_tensor_info(input_tensor_layout, input_shape, input_slice_shape); + TensorInfo output_tensor_info(output_tensor_layout, output_shape, output_slice_shape); + input_tensor_info.set_reduce_dim(dim_list); + + inputs_tensor_info_.push_back(input_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + + return SUCCESS; +} + +Status ReduceMethod::SetCostUnderStrategy(const StrategyPtr &strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status ReduceMethod::GenerateStrategies(int32_t stage_id) { + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 1)) { + MS_LOG(ERROR) << name_ << ": Inputs shape size or outputs shape size is wrong, " << inputs_shape_.size() << ", " + << outputs_shape_.size(); + return FAILED; + } + + Shape input0_split(inputs_shape_[0].size(), 1); + Shapes splittable_inputs = {input0_split}; + is_auto_parallel_ = true; + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategiesForIndependentInputs failed."; + return FAILED; + } + size_t success = 0; + for (auto &sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} + +Status ReduceMethod::Init(const StrategyPtr &strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + + return SUCCESS; +} + +Status ReduceMethod::InitForCostModel(const StrategyPtr &strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed"; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed"; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init for cost model success"; + return SUCCESS; +} + +std::vector ArgMaxWithValueInfo::reduce_dim() { + std::vector dim_list; + auto iter = attrs_.find(AXIS); + if (iter == attrs_.end()) { + MS_LOG(EXCEPTION) << name_ << ": Don't have attr axis."; + } + + MS_ASSERT(inputs_shape_.size() == 1); + auto input_dim = inputs_shape_.at(0).size(); + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + auto attr_axis = GetValue>(iter->second); + if (attr_axis.empty()) { + for (size_t i = 0; i < input_dim; ++i) { + dim_list.push_back(SizeToInt(i)); + } + } else { + for (auto &axis : attr_axis) { + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } + } + } else if (iter->second->isa()) { + int axis = GetValue(iter->second); + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } else { + MS_LOG(EXCEPTION) << "Axis type is invalid."; + } + + return dim_list; +} + +Status ArgMaxWithValueInfo::CheckStrategy(const StrategyPtr &strategy) { + if (ReduceMethod::CheckStrategy(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": CheckStrategy for parent class ReduceMethod failed"; + } else { + MS_LOG(ERROR) << name_ << ": CheckStrategy for parent class ReduceMethod failed"; + } + return FAILED; + } + std::vector dim_list = reduce_dim(); + MS_ASSERT(dim_list.size() == 1); + + std::vector stra = strategy->GetInputDim(); + MS_ASSERT(stra.size() == 1); + Shape input_strategy = stra.at(0); + MS_ASSERT(dim_list.at(0) < input_strategy.size()); + if (input_strategy.at(IntToSize(dim_list.at(0))) != 1) { + MS_LOG(WARNING) + << name_ + << " CheckStrategy for ArgMaxWithValueInfo, the strategy corresponding to axis is not one, real strategy " + "is " + << input_strategy.at(IntToSize(dim_list.at(0))) + << ", the output index may be not compatible with the stand alone Primitive"; + } + return SUCCESS; +} + +Status ArgMaxWithValueInfo::InferTensorMap() { + if (ReduceMethod::InferTensorMap() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferTensorMap for parent class ReduceMethod failed"; + return FAILED; + } + MS_ASSERT(outputs_tensor_map_.size() == 1); + outputs_tensor_map_.push_back(outputs_tensor_map_[0]); + return SUCCESS; +} + +Status ArgMaxWithValueInfo::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + Shape output_shape = outputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Dimensions output_strategy = InferOutputStrategy(); + + Strategys outputs_strategy = {output_strategy, output_strategy}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_slice_shape = inputs_slice_shape.at(0); + Shape output_slice_shape = outputs_slice_shape.at(0); + + TensorLayout input_tensor_layout, output_tensor_layout; + if ((input_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], input_shape) != SUCCESS) || + (output_tensor_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], output_shape) != SUCCESS)) { + return FAILED; + } + + std::vector dim_list = reduce_dim(); + TensorInfo input_tensor_info(input_tensor_layout, input_shape, input_slice_shape); + TensorInfo output_tensor_info(output_tensor_layout, output_shape, output_slice_shape); + input_tensor_info.set_reduce_dim(dim_list); + + inputs_tensor_info_.push_back(input_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + outputs_tensor_info_.push_back(output_tensor_info); + return SUCCESS; +} + +Status ArgMaxWithValueInfo::InferAsLossDivisor() { + if (outputs_tensor_map_.empty()) { + MS_LOG(ERROR) << name_ << ": The outputs tensor map is empty."; + return FAILED; + } + + MS_LOG(INFO) << name_ << " has two outputs, use output[0] to infer"; + if (outputs_tensor_map_[0].empty()) { + as_loss_divisor_ = SizeToInt(global_device_list_.size()); + MS_LOG(INFO) << name_ << ": The output is a scalar, use the dev size" << as_loss_divisor_ << " as loss divisor."; + return SUCCESS; + } + + as_loss_divisor_ = ComputeRepeatDeviceNumByTensorMap(dev_matrix_shape_, outputs_tensor_map_[0]); + + std::string dev_matrix_shape_str = ShapeToString(dev_matrix_shape_); + std::string output_tensor_map_str = ShapeToString(outputs_tensor_map_[0]); + MS_LOG(INFO) << name_ << ": the dev matrix shape, the output tensor map, and loss divisor is " << dev_matrix_shape_str + << ", " << output_tensor_map_str << ", " << as_loss_divisor_; + return SUCCESS; +} + +Status ArgMaxWithValueInfo::GenerateStrategies(int32_t stage_id) { + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 2)) { + MS_LOG(ERROR) << name_ << ": Inputs shape size or outputs shape size is wrong, " << inputs_shape_.size() << ", " + << outputs_shape_.size(); + return FAILED; + } + Shape input0_split(inputs_shape_[0].size(), 1); + Shapes splittable_inputs = {input0_split}; + is_auto_parallel_ = true; + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategiesForIndependentInputs failed."; + return FAILED; + } + size_t success = 0; + for (auto &sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated strategy " << success; + PrintStrategy(sp); + } + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/reduce_method_info.h b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.h new file mode 100644 index 00000000000..e9f03bcdeeb --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.h @@ -0,0 +1,146 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_REDUCE_SUM_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_REDUCE_SUM_INFO_H_ + +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/strategy.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "ir/meta_tensor.h" + +namespace mindspore { +namespace parallel { +class ReduceMethod : public OperatorInfo { + public: + ReduceMethod(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + reducemethodcost_ptr_ = std::make_shared(); + } + ~ReduceMethod() override = default; + + Status Init(const StrategyPtr &strategy) override; + Status InitForCostModel(const StrategyPtr &strategy) override; + + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr &strategy) override; + OperatorCostPtr GetOperatorCost() const override { return reducemethodcost_ptr_; } + + protected: + std::string reduce_method_; + bool keepdims_ = false; + bool cross_batch_ = false; + ReduceMethodCostPtr reducemethodcost_ptr_; + Status CheckStrategy(const StrategyPtr &strategy) override; + Status GetAttrs() override; + Dimensions InferOutputStrategy(); + Status InferTensorMap() override; + Status InferTensorInfo() override; + Status InferMirrorOps() override; + virtual std::vector reduce_dim(); + Status InferForwardCommunication() override; + Status InferDevMatrixShape() override; +}; + +class ReduceMaxInfo : public ReduceMethod { + public: + ReduceMaxInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : ReduceMethod(name, inputs_shape, outputs_shape, attrs) { + reduce_method_ = REDUCE_OP_MAX; + } + + ~ReduceMaxInfo() override = default; +}; + +class ArgMaxWithValueInfo : public ReduceMethod { + public: + ArgMaxWithValueInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : ReduceMethod(name, inputs_shape, outputs_shape, attrs) { + reduce_method_ = REDUCE_OP_MAX; + } + + ~ArgMaxWithValueInfo() override = default; + + Status GenerateStrategies(int32_t stage_id) override; + + protected: + std::vector reduce_dim() override; + Status CheckStrategy(const StrategyPtr &strategy) override; + Status InferMirrorOps() override; + Status InferTensorMap() override; + Status InferTensorInfo() override; + Status InferAsLossDivisor() override; +}; + +class ArgMinWithValueInfo : public ArgMaxWithValueInfo { + public: + ArgMinWithValueInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : ArgMaxWithValueInfo(name, inputs_shape, outputs_shape, attrs) { + reduce_method_ = REDUCE_OP_MIN; + } + + ~ArgMinWithValueInfo() override = default; +}; + +class ReduceMeanInfo : public ReduceMethod { + public: + ReduceMeanInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : ReduceMethod(name, inputs_shape, outputs_shape, attrs) { + reducemethodcost_ptr_ = std::make_shared(); + } + + ~ReduceMeanInfo() override = default; + + protected: + Status InferForwardCommunication() override; +}; + +class ReduceSumInfo : public ReduceMethod { + public: + ReduceSumInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : ReduceMethod(name, inputs_shape, outputs_shape, attrs) { + reduce_method_ = REDUCE_OP_SUM; + } + + ~ReduceSumInfo() override = default; +}; + +class ReduceMinInfo : public ReduceMethod { + public: + ReduceMinInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape, + const PrimitiveAttrs &attrs) + : ReduceMethod(name, inputs_shape, outputs_shape, attrs) { + reduce_method_ = REDUCE_OP_MIN; + } + + ~ReduceMinInfo() override = default; +}; +} // namespace parallel +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_REDUCE_SUM_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/reshape_info.cc b/mindspore/ccsrc/parallel/ops_info/reshape_info.cc new file mode 100644 index 00000000000..0c95ee9c051 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/reshape_info.cc @@ -0,0 +1,435 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/reshape_info.h" + +#include +#include + +#include "parallel/device_manager.h" +#include "parallel/device_matrix.h" +#include "parallel/step_parallel.h" +#include "utils/convert_utils.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +Status ReshapeInfo::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy."; + } + return FAILED; + } + + size_t strategy_size = strategy->GetInputNumber(); + if (strategy_size != 1) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy size " << strategy_size; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy size " << strategy_size; + } + return FAILED; + } + std::vector stra = strategy->GetInputDim(); + for (size_t i = 0; i < strategy_size; ++i) { + Shape sub_strategy = stra.at(i); + size_t strategy_len = sub_strategy.size(); + bool flag = false; + for (size_t j = 0; j < strategy_len; ++j) { + int32_t strategy_value = sub_strategy.at(j); + if (strategy_value > 1) { + if (flag) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Only support batch parallel strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Only support batch parallel strategy."; + } + return FAILED; + } + flag = true; + } + } + } + return SUCCESS; +} + +/* + * support parallel degree smaller than device number, set the duplicate device dimension to the first dimension of + * device matrix + * only support batch parallel reshape operator in ReID (batch parallel degree can be smaller than device number) + */ +Status ReshapeInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + input_strategy_ = stra.at(0); + dev_matrix_shape_.push_back(input_strategy_[0]); + return SUCCESS; +} + +/* + * there is no Parameter for Reshape Primitive, so no need to do allreduce + */ +Status ReshapeInfo::InferMirrorOps() { + mirror_ops_.clear(); + Shape input_tensor_map = input_layout_.tensor_map().array(); + std::vector input_group; + if (CreateGroupByTensorMap(input_tensor_map, &input_group) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Infer MirrorOps failed."; + return FAILED; + } + + OperatorVector op_for_input; + if (input_group.empty()) { + MS_LOG(INFO) << name_ << ": The mirror ops is empty."; + return SUCCESS; + } + if (!input_group.empty()) { + op_for_input = CreateMirrorOps(input_group[0].name(), input_group[0].GetDevNum()); + std::string group_name = input_group[0].name(); + MS_LOG(INFO) << name_ << ": Create the mirror ops for input_a success, group is " << group_name; + } + mirror_ops_.push_back(op_for_input); + OperatorVector op_for_input_empty; + mirror_ops_.push_back(op_for_input_empty); + + return SUCCESS; +} + +/* + * there is no reduction dimension for forward computation of Reshape Primitive, so no need to do allreduce + */ +Status ReshapeInfo::InferForwardCommunication() { return SUCCESS; } + +/* + * get shape input of Reshape Primitive + * the result is saved in parameter_input_v_ + * not support -1 + */ +Status ReshapeInfo::GetParameterInput() { + if (input_value_[1] == nullptr) { + MS_LOG(ERROR) << name_ << ": input_value_[1] is nullptr."; + return FAILED; + } + std::vector elements; + ValueTuplePtr dim_tuple = input_value_[1]->cast(); + if (dim_tuple == nullptr) { + MS_LOG(ERROR) << name_ << ": Input_value_[1] must be ValueTuplePtr."; + return FAILED; + } + elements = dim_tuple->value(); + if (elements.size() != outputs_shape_[0].size()) { + MS_LOG(ERROR) << name_ << ": Elements size must equal to outputs shape[0] size."; + return FAILED; + } + + for (auto& element : elements) { + MS_EXCEPTION_IF_NULL(element); + if (element->isa()) { + int32_t axis = element->cast()->value(); + parameter_input_v_.push_back(axis); + } else { + MS_LOG(ERROR) << name_ << ": The value of axis must be int32."; + return FAILED; + } + } + return SUCCESS; +} + +Status ReshapeInfo::ComputeReplaceOp() { + RankList dev_list = global_device_list(); + TensorRedistribution tensor_redistribution(true, true); + if (tensor_redistribution.Init(input_layout_, output_layout_, dev_list) == FAILED) { + MS_LOG(ERROR) << name_ << ": tensor_redistribution init failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << ": input " << input_layout_.ToString(); + MS_LOG(INFO) << name_ << ": output " << output_layout_.ToString(); + MS_LOG(INFO) << name_ << ": dev_list " << dev_list.size(); + RedistributionOpListPtr redistribution_oplist_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); + if (redistribution_oplist_ptr == nullptr) { + MS_LOG(ERROR) << name_ << "InferTensorRedistribution failed."; + return FAILED; + } + replace_op_ = redistribution_oplist_ptr->first; + replace_op_info_ = redistribution_oplist_ptr->second; + MS_LOG(INFO) << name_ << ": replace op size = " << replace_op_.size(); + return SUCCESS; +} + +/* + * the first dimension of input tensor map and output tensor map is set to the last dimension of device arrangement, + * all other dimension is set to None + * only support batch parallel reshape operator in ReID (batch parallel degree can be smaller than device number) + */ +Status ReshapeInfo::InferTensorMap() { + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 1)) { + MS_LOG(ERROR) << name_ << ": inputs shape and outputs shape size must be 1. inputs shape and outputs shape are " + << inputs_shape_.size() << " and " << outputs_shape_.size(); + return FAILED; + } + + std::vector tensor_map_index_input; + tensor_map_index_input.push_back(0); + + for (size_t j = 1; j < inputs_shape_[0].size(); ++j) { + tensor_map_index_input.push_back(MAP_NONE); + } + inputs_tensor_map_.push_back(tensor_map_index_input); + + std::vector tensor_map_index_output; + tensor_map_index_output.push_back(0); + + for (size_t j = 1; j < outputs_shape_[0].size(); ++j) { + tensor_map_index_output.push_back(MAP_NONE); + } + outputs_tensor_map_.push_back(tensor_map_index_output); + return SUCCESS; +} + +/* + * the output tensor strategy is the same as input tensor strategy + * only support batch parallel reshape operator in ReID (batch parallel degree can be smaller than device number) + */ +Strategys ReshapeInfo::GetOutputsStrategy() { + Strategys outputs_strategy; + std::vector strategy; + strategy.push_back(input_strategy_[0]); + for (size_t j = 1; j < outputs_shape_[0].size(); ++j) { + strategy.push_back(1); + } + outputs_strategy.push_back(strategy); + return outputs_strategy; +} + +Status ReshapeInfo::InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout) { + if (inputs_layout == nullptr || outputs_layout == nullptr) { + MS_LOG(ERROR) << name_ << ": InferTensorLayout: the layout is null."; + return FAILED; + } + Arrangement dev_matrix; + Status status = dev_matrix.Init(dev_matrix_shape_); + if (status != Status::SUCCESS) { + return status; + } + // infer input tensor info + Shape shape_array_in = inputs_shape_.at(0); + TensorMap tensor_map_array_in = inputs_tensor_map_.at(0); + TensorLayout tensor_layout_in; + Map tensor_map_in; + status = tensor_map_in.Init(tensor_map_array_in); + if (status != Status::SUCCESS) { + return status; + } + Arrangement shape_in; + status = shape_in.Init(shape_array_in); + if (status != Status::SUCCESS) { + return status; + } + (void)tensor_layout_in.Init(dev_matrix, tensor_map_in, shape_in); + inputs_layout->push_back(tensor_layout_in); + // infer output tensor info + Shape shape_array_out = outputs_shape_.at(0); + + TensorMap tensor_map_array_out = outputs_tensor_map_.at(0); + TensorLayout tensor_layout_out; + Map tensor_map_out; + status = tensor_map_out.Init(tensor_map_array_out); + if (status != Status::SUCCESS) { + return status; + } + Arrangement shape_out; + status = shape_out.Init(shape_array_out); + if (status != Status::SUCCESS) { + return status; + } + (void)tensor_layout_out.Init(dev_matrix, tensor_map_out, shape_out); + outputs_layout->push_back(tensor_layout_out); + + input_layout_ = tensor_layout_in; + output_layout_ = tensor_layout_out; + return SUCCESS; +} + +Status ReshapeInfo::InferTensorInfo() { + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = GetOutputsStrategy(); + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + + TensorLayouts inputs_layout, outputs_layout; + if (InferTensorLayout(&inputs_layout, &outputs_layout) != SUCCESS) { + return FAILED; + } + TensorLayout tensor_layout_in = inputs_layout.at(0); + TensorLayout tensor_layout_out = outputs_layout.at(0); + Shape shape_array_in = inputs_shape_.at(0); + Shape slice_shape_in = inputs_slice_shape.at(0); + Shape shape_array_out = outputs_shape_.at(0); + Shape slice_shape_out = outputs_slice_shape.at(0); + TensorInfo tensor_info_in(tensor_layout_in, shape_array_in, slice_shape_in); + TensorInfo tensor_info_out(tensor_layout_out, shape_array_out, slice_shape_out); + inputs_tensor_info_.push_back(tensor_info_in); + outputs_tensor_info_.push_back(tensor_info_out); + return SUCCESS; +} + +void ReshapeInfo::InferTensorInfoByLayout() { + TensorInfo tensor_info_in(input_layout_); + TensorInfo tensor_info_out(output_layout_); + inputs_tensor_info_.push_back(tensor_info_in); + outputs_tensor_info_.push_back(tensor_info_out); +} + +/* + * compute parameter_input_v_ during this method + */ +Status ReshapeInfo::GetAttrs() { return GetParameterInput(); } + +void ReshapeInfo::device_number(const StrategyPtr& strategy) { + int32_t stage = 0; + if (strategy != nullptr) { + stage = strategy->GetInputStage(); + } + CheckGlobalDeviceManager(); + global_device_list_ = g_device_manager->GetDeviceListByStageId(stage); + dev_num_ = SizeToInt(global_device_list_.size()); + MS_ASSERT(dev_num_ > 0); +} + +Status ReshapeInfo::InferDefaultLayout(const Shape& shape, TensorLayout* const layout) { + std::vector tensor_map_index; + for (size_t i = 0; i < shape.size(); i++) { + tensor_map_index.push_back(MAP_NONE); + } + Status status = layout->InitFromVector({dev_num_}, tensor_map_index, shape); + if (status != Status::SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferDefaultLayout failed."; + return status; + } + return Status::SUCCESS; +} + +Status ReshapeInfo::Init(const StrategyPtr& strategy) { + ResetQueueMember(); + device_number(strategy); + if (strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + } else { + if (!input_layout_set_flag_) { + MS_ASSERT(inputs_shape_.size() == 1); + Status status = InferDefaultLayout(inputs_shape_.at(0), &input_layout_); + if (status != SUCCESS) { + MS_LOG(ERROR) << name_ << ": infer input default layout failed."; + return status; + } + } + if (!output_layout_set_flag_) { + MS_ASSERT(output_layout_.size() == 1); + Status status = InferDefaultLayout(outputs_shape_.at(0), &output_layout_); + if (status != SUCCESS) { + MS_LOG(ERROR) << name_ << ": infer output default layout failed."; + return status; + } + } + inputs_tensor_map_.push_back(input_layout_.tensor_map().array()); + outputs_tensor_map_.push_back(output_layout_.tensor_map().array()); + InferTensorInfoByLayout(); + // change dev_matrix_shape_ to input_layout_ device_arrangement before InferMirrorOps + dev_matrix_shape_ = input_layout_.device_arrangement().array(); + if (InferMirrorOps() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferMirrorOps failed."; + return FAILED; + } + // change dev_matrix_shape_ to output_layout_ device_arrangement before InferVirtualDivOps + dev_matrix_shape_ = output_layout_.device_arrangement().array(); + if (InferVirtualDivOps() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": InferVirtualDivOps failed."; + return FAILED; + } + } + Status status = ComputeReplaceOp(); + if (status != SUCCESS) { + MS_LOG(ERROR) << name_ << ": ComputeReplaceOp failed."; + return status; + } + return SUCCESS; +} + +Status ReshapeInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init for cost model success."; + return SUCCESS; +} + +Status ReshapeInfo::SetCostUnderStrategy(const mindspore::parallel::StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status ReshapeInfo::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GetAttrs failed."; + return FAILED; + } + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 1)) { + MS_LOG(ERROR) << name_ << ": Inputs shape size or outputs shape size is wrong, " << inputs_shape_.size() << ", " + << outputs_shape_.size(); + return FAILED; + } + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size(), 0); + input0_split[0] = 1; + Shapes splittable_inputs = {input0_split}; + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategiesForIndependentInputs failed."; + return FAILED; + } + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/reshape_info.h b/mindspore/ccsrc/parallel/ops_info/reshape_info.h new file mode 100644 index 00000000000..a8a93235406 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/reshape_info.h @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_RESHAPE_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_RESHAPE_INFO_H_ + +#include + +#include +#include +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +/* + * parallel class for Reshape Primitive + */ +class ReshapeInfo : public OperatorInfo { + public: + ReshapeInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs), + dev_num_(0), + input_layout_set_flag_(false), + output_layout_set_flag_(false) { + reshape_cost_ptr_ = std::make_shared(); + } + ~ReshapeInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + void SetInputLayout(const TensorLayout& input_layout) { + input_layout_ = input_layout; + input_layout_set_flag_ = true; + } + void SetOutputLayout(const TensorLayout& output_layout) { + output_layout_ = output_layout; + output_layout_set_flag_ = true; + } + Status InitForCostModel(const StrategyPtr& strategy) override; + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return reshape_cost_ptr_; } + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorMap() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout); + Status GetAttrs() override; + Strategys GetOutputsStrategy(); + ReshapeCostPtr reshape_cost_ptr_; + + private: + Status GetParameterInput(); + Status ComputeReplaceOp(); + void InferTensorInfoByLayout(); + void device_number(const StrategyPtr& strategy); + Status InferDefaultLayout(const Shape& shape, TensorLayout* const layout); + + int32_t dev_num_; + std::vector parameter_input_v_; + Dimensions input_strategy_; + TensorLayout input_layout_; + TensorLayout output_layout_; + bool input_layout_set_flag_; + bool output_layout_set_flag_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_RESHAPE_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/tmp_identity_info.cc b/mindspore/ccsrc/parallel/ops_info/tmp_identity_info.cc new file mode 100644 index 00000000000..73b18ad473a --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/tmp_identity_info.cc @@ -0,0 +1,149 @@ +/** +#include "utils/log_adapter.h" +#include "utils/log_adapter.h" +#include "utils/log_adapter.h" + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/tmp_identity_info.h" + +#include +#include + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +Status TmpIdentityInfo::CheckStrategy(const mindspore::parallel::StrategyPtr &strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": invalid strategy."; + } + return FAILED; + } + return SUCCESS; +} + +Status TmpIdentityInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions input_strategy = stra.at(0); + dev_matrix_shape_ = input_strategy; + return SUCCESS; +} + +Status TmpIdentityInfo::InferTensorMap() { + std::vector tensor_map_index; + size_t size = inputs_shape_[0].size(); + // such as 4: tensor_map_index [3,2,1,0] + for (size_t i = 0; i < size; ++i) { + tensor_map_index.push_back((int32_t)(size - 1 - i)); + } + + inputs_tensor_map_.push_back(tensor_map_index); + outputs_tensor_map_.push_back(tensor_map_index); + return SUCCESS; +} + +Status TmpIdentityInfo::InferTensorInfo() { + // infer tensor shape + Shape input_shape = inputs_shape_.at(0); + + // infer slice shape + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = {inputs_strategy.at(0)}; + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + Shape input_slice_shape = inputs_slice_shape.at(0); + + TensorLayout input_tensor_layout; + if (input_tensor_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[0], input_shape) != SUCCESS) { + return FAILED; + } + + TensorInfo input_tensor_info(input_tensor_layout, input_shape, input_slice_shape); + + inputs_tensor_info_.push_back(input_tensor_info); + outputs_tensor_info_.push_back(input_tensor_info); // the same as input + + return SUCCESS; +} + +Status TmpIdentityInfo::Init(const StrategyPtr &strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init success."; + return SUCCESS; +} + +Status TmpIdentityInfo::InitForCostModel(const StrategyPtr &strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init for cost model success."; + return SUCCESS; +} + +Status TmpIdentityInfo::SetCostUnderStrategy(const StrategyPtr &strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status TmpIdentityInfo::GenerateStrategies(int32_t stage_id) { + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 1)) { + MS_LOG(ERROR) << name_ << ": Inputs shape size or outputs shape size is wrong, " << inputs_shape_.size() << ", " + << outputs_shape_.size(); + return FAILED; + } + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size(), 1); + Shapes splittable_inputs = {input0_split}; + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategiesForIndependentInputs failed."; + return FAILED; + } + size_t success = 0; + for (auto &sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/tmp_identity_info.h b/mindspore/ccsrc/parallel/ops_info/tmp_identity_info.h new file mode 100644 index 00000000000..1ef7341f3b3 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/tmp_identity_info.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_TMP_IDENTITY_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_TMP_IDENTITY_INFO_H_ + +#include +#include +#include +#include "parallel/ops_info/operator_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class TmpIdentityInfo : public OperatorInfo { + // This operator is only used for the case of a parameter tensor being used by multiple operators, where we + // consider this parameter tensor as TmpIdentityInfo operator. TmpIdentityInfo operator tasks as input a tensor, + // and outputs the same tensor. After the transformation, subsequent operators can share the output tensor. + public: + TmpIdentityInfo(const Shapes& inputs_shape, const Shapes& outputs_shape, const PrimitiveAttrs& attrs, + const std::string& name = IDENTITY_INFO) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + id_cost_ptr_ = std::make_shared(); + } + ~TmpIdentityInfo() override = default; + + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return id_cost_ptr_; } + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status GetAttrs() override { return SUCCESS; } + Status InferMirrorOps() override { return SUCCESS; } + Status InferForwardCommunication() override { return SUCCESS; } + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + + private: + TmpIdentityCostPtr id_cost_ptr_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_TMP_IDENTITY_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/transpose_info.cc b/mindspore/ccsrc/parallel/ops_info/transpose_info.cc new file mode 100644 index 00000000000..84333a1337f --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/transpose_info.cc @@ -0,0 +1,247 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/transpose_info.h" + +#include +#include + +#include "parallel/device_manager.h" +#include "parallel/device_matrix.h" +#include "parallel/step_parallel.h" +#include "utils/convert_utils.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +Status TransposeInfo::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy."; + } + return FAILED; + } + + return SUCCESS; +} + +Status TransposeInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + input_strategy_ = stra.at(0); + for (auto& iter : input_strategy_) { + dev_matrix_shape_.push_back(iter); + } + return SUCCESS; +} + +// there is no Parameter for Transpose Primitive, so no need to do all reduce +Status TransposeInfo::InferMirrorOps() { return SUCCESS; } + +// there is no reduction dimension for forward computation of Transpose Primitive, so no need to do all reduce +Status TransposeInfo::InferForwardCommunication() { return SUCCESS; } + +/* + * get perm input of Transpose Primitive + * perm is a permutation of the dimensions of input + * the result is saved in axis_v_ + */ +Status TransposeInfo::ComputeAxis() { + if (input_value_[1] == nullptr) { + MS_LOG(ERROR) << name_ << ": input_value_[1] is nullptr."; + return FAILED; + } + std::vector elements; + ValueTuplePtr dim_tuple = input_value_[1]->cast(); + if (dim_tuple == nullptr) { + MS_LOG(ERROR) << name_ << ": input_value_[1] must be ValueTuplePtr."; + return FAILED; + } + elements = dim_tuple->value(); + if (elements.size() != inputs_shape_[0].size()) { + MS_LOG(ERROR) << name_ << ": elements size must equal to inputs shape 0 size."; + return FAILED; + } + axis_v_.clear(); + for (auto& element : elements) { + MS_EXCEPTION_IF_NULL(element); + if (element->isa()) { + int32_t axis = element->cast()->value(); + axis_v_.push_back(axis); + } else { + MS_LOG(ERROR) << name_ << ": The value of axis must be int32."; + return FAILED; + } + } + + for (int32_t i = 0; i < SizeToInt(axis_v_.size()); i++) { + auto iter = std::find(axis_v_.begin(), axis_v_.end(), i); + if (iter == axis_v_.end()) { + MS_LOG(ERROR) << name_ << ": axis_v_ must be a permutation."; + } + } + return SUCCESS; +} + +// the output tensor map is the permutation of input tensor map, the permutation is axis_v +Status TransposeInfo::InferTensorMap() { + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 1)) { + MS_LOG(ERROR) << name_ << ": inputs_shape_ and outputs_shape_ size must be 1, inputs shape and outputs shape is " + << inputs_shape_.size() << ", " << outputs_shape_.size(); + return FAILED; + } + + std::vector tensor_map_index_input; + for (size_t j = 0; j < inputs_shape_[0].size(); ++j) { + tensor_map_index_input.push_back(SizeToInt(inputs_shape_[0].size() - j - 1)); + } + inputs_tensor_map_.push_back(tensor_map_index_input); + + std::vector tensor_map_index_output = tensor_map_index_input; + for (uint32_t i = 0; i < tensor_map_index_output.size(); i++) { + tensor_map_index_output[i] = tensor_map_index_input[IntToUint(axis_v_[i])]; + } + outputs_tensor_map_.push_back(tensor_map_index_output); + return SUCCESS; +} + +// the output tensor strategy is the permutation of input tensor strategy, the permutation is axis_v +Strategys TransposeInfo::GetOutputsStrategy() { + Strategys outputs_strategy; + std::vector strategy = input_strategy_; + for (uint32_t i = 0; i < strategy.size(); i++) { + strategy[i] = input_strategy_[IntToUint(axis_v_[i])]; + } + outputs_strategy.push_back(strategy); + return outputs_strategy; +} + +Status TransposeInfo::InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout) { + if ((inputs_layout == nullptr) || (outputs_layout == nullptr)) { + MS_LOG(ERROR) << name_ << ": InferTensorLayout: the layout is null."; + return FAILED; + } + Shape shape_in = inputs_shape_.at(0); + TensorMap tensor_map_in = inputs_tensor_map_.at(0); + Shape shape_out = outputs_shape_.at(0); + TensorMap tensor_map_out = outputs_tensor_map_.at(0); + + TensorLayout tensor_layout_in, tensor_layout_out; + if ((tensor_layout_in.InitFromVector(dev_matrix_shape_, tensor_map_in, shape_in) != SUCCESS) || + (tensor_layout_out.InitFromVector(dev_matrix_shape_, tensor_map_out, shape_out) != SUCCESS)) { + return FAILED; + } + + inputs_layout->push_back(tensor_layout_in); + outputs_layout->push_back(tensor_layout_out); + return SUCCESS; +} + +Status TransposeInfo::InferTensorInfo() { + Shapes inputs_slice_shape, outputs_slice_shape; + Strategys inputs_strategy = strategy_->GetInputDim(); + Strategys outputs_strategy = GetOutputsStrategy(); + if (InferSliceShape(inputs_strategy, outputs_strategy, &inputs_slice_shape, &outputs_slice_shape) != SUCCESS) { + return FAILED; + } + + TensorLayouts inputs_layout, outputs_layout; + if (InferTensorLayout(&inputs_layout, &outputs_layout) != SUCCESS) { + return FAILED; + } + TensorLayout tensor_layout_in = inputs_layout.at(0); + TensorLayout tensor_layout_out = outputs_layout.at(0); + Shape shape_array_in = inputs_shape_.at(0); + Shape slice_shape_in = inputs_slice_shape.at(0); + Shape shape_array_out = outputs_shape_.at(0); + Shape slice_shape_out = outputs_slice_shape.at(0); + TensorInfo tensor_info_in(tensor_layout_in, shape_array_in, slice_shape_in); + TensorInfo tensor_info_out(tensor_layout_out, shape_array_out, slice_shape_out); + inputs_tensor_info_.push_back(tensor_info_in); + outputs_tensor_info_.push_back(tensor_info_out); + return SUCCESS; +} + +// compute axis_v_ during this method +Status TransposeInfo::GetAttrs() { return ComputeAxis(); } + +Status TransposeInfo::Init(const StrategyPtr& strategy) { + if (InitWithAutoRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + MS_LOG(INFO) << name_ << ": Init success."; + return SUCCESS; +} + +Status TransposeInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init for cost model success."; + return SUCCESS; +} + +Status TransposeInfo::SetCostUnderStrategy(const mindspore::parallel::StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status TransposeInfo::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GetAttrs failed."; + return FAILED; + } + if ((inputs_shape_.size() != 1) || (outputs_shape_.size() != 1)) { + MS_LOG(ERROR) << name_ << ": inputs shape size or outputs shape size is wrong, " << inputs_shape_.size() << ", " + << outputs_shape_.size(); + return FAILED; + } + is_auto_parallel_ = true; + Shape input0_split(inputs_shape_[0].size(), 1); + Shapes splittable_inputs = {input0_split}; + std::vector sp_vector; + if (GenerateStrategiesForIndependentInputs(stage_id, inputs_shape_, splittable_inputs, &sp_vector) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GenerateStrategiesForIndependentInputs failed"; + return FAILED; + } + size_t success = 0; + for (auto& sp : sp_vector) { + if (SetCostUnderStrategy(sp) == SUCCESS) { + success++; + MS_LOG(INFO) << name_ << ": Successfully generated " << success << "strategy."; + PrintStrategy(sp); + } + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/transpose_info.h b/mindspore/ccsrc/parallel/ops_info/transpose_info.h new file mode 100644 index 00000000000..6c5e98e3b9b --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/transpose_info.h @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_TRANSPOSE_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_TRANSPOSE_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +/* + * parallel class for Transpose Primitive + */ +class TransposeInfo : public OperatorInfo { + public: + TransposeInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + transpose_cost_ptr_ = std::make_shared(); + } + ~TransposeInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return transpose_cost_ptr_; } + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status InferTensorLayout(TensorLayouts* inputs_layout, TensorLayouts* outputs_layout); + Status GetAttrs() override; + Strategys GetOutputsStrategy(); + + private: + Status ComputeAxis(); + std::vector axis_v_; + Dimensions input_strategy_; + ActivationCostPtr transpose_cost_ptr_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_TRANSPOSE_INFO_H_ diff --git a/mindspore/ccsrc/parallel/ops_info/virtual_dataset_info.cc b/mindspore/ccsrc/parallel/ops_info/virtual_dataset_info.cc new file mode 100644 index 00000000000..acb39247d44 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/virtual_dataset_info.cc @@ -0,0 +1,250 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/ops_info/virtual_dataset_info.h" + +#include +#include +#include + +#include "parallel/device_manager.h" +#include "parallel/device_matrix.h" +#include "parallel/step_parallel.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { + +Status VirtualDatasetInfo::CheckStrategy(const StrategyPtr& strategy) { + if (CheckStrategyValue(strategy, inputs_shape_, is_auto_parallel_) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Invalid strategy."; + } else { + MS_LOG(ERROR) << name_ << ": Invalid strategy."; + } + return FAILED; + } + + std::vector stra = strategy->GetInputDim(); + if (stra.size() < 1) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Strategy size must be larger than 1."; + } else { + MS_LOG(ERROR) << name_ << ": Strategy size must be larger than 1."; + } + return FAILED; + } + if (stra.size() == 1) { + MS_LOG(WARNING) << name_ << ": Strategy size is 1."; + return SUCCESS; + } + Dimensions strategy_first = stra.at(1); + for (auto iter_strategy = stra.begin() + 1; iter_strategy != stra.end(); ++iter_strategy) { + if (iter_strategy->empty()) { + MS_LOG(ERROR) << name_ << ": iter_strategy size is zero."; + } + if (strategy_first.at(0) != *(iter_strategy->begin())) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": The first dimension of each strategy must be the same."; + } else { + MS_LOG(ERROR) << name_ << ": The first dimension of each strategy must be the same."; + } + return FAILED; + } + + for (auto iter_element = iter_strategy->begin() + 1; iter_element != iter_strategy->end(); ++iter_element) { + if (*iter_element != 1) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": All dimension except the first dimension of each strategy must be 1."; + } else { + MS_LOG(ERROR) << name_ << ": All dimension except the first dimension of each strategy must be 1."; + } + return FAILED; + } + } + } + return SUCCESS; +} + +Status VirtualDatasetInfo::InferDevMatrixShape() { + std::vector stra = strategy_->GetInputDim(); + Dimensions strategy_first = stra.at(0); + int32_t stage = strategy_->GetInputStage(); + CheckGlobalDeviceManager(); + int32_t dev_num = SizeToInt(g_device_manager->GetDeviceListByStageId(stage).size()); + int32_t batch_split_num = strategy_first.at(0); + dev_matrix_shape_.push_back(batch_split_num); + if (dev_num > batch_split_num) { + dev_matrix_shape_.push_back(dev_num / batch_split_num); + } + + return SUCCESS; +} + +Status VirtualDatasetInfo::InferMirrorOps() { + mirror_ops_.clear(); + + int32_t stage = strategy_->GetInputStage(); + CheckGlobalDeviceManager(); + RankList dev_list = g_device_manager->GetDeviceListByStageId(stage); + if (dev_list.empty()) { + MS_LOG(ERROR) << name_ << ": The current stage is empty!"; + return Status::FAILED; + } + if (dev_list.size() == 1) { + MS_LOG(INFO) << name_ << ": No need mirror ops."; + return Status::SUCCESS; + } + + OperatorName operator_name = BROADCAST; + ValuePtr attr0_value = MakeValue(dev_list.front()); + std::vector group_list; + if (CreateGroupByDim(dev_matrix_shape_.size() - 1, &group_list) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Infer mirror ops, create group failed."; + return FAILED; + } else if (group_list.empty()) { + MS_LOG(INFO) << name_ << ": No need mirror ops."; + return SUCCESS; + } + std::string group = group_list[0].name(); + ValuePtr attr1_value = MakeValue(group); + + Attr attr0 = std::make_pair(SRC, attr0_value); + Attr attr1 = std::make_pair(GROUP, attr1_value); + + OperatorAttrs operator_attrs = {attr0, attr1}; + + OperatorParams operator_param; + OperatorArgs operator_args = std::make_pair(operator_attrs, operator_param); + + Operator op = std::make_pair(operator_name, operator_args); + OperatorVector op_vector = {op}; + + size_t size = inputs_shape_.size(); + for (size_t i = 0; i < size; ++i) { + mirror_ops_.push_back(op_vector); + } + mirror_ops_.clear(); + return SUCCESS; +} + +Status VirtualDatasetInfo::InferForwardCommunication() { return SUCCESS; } + +Status VirtualDatasetInfo::InferTensorMap() { + for (size_t i = 0; i < strategy_->GetInputNumber(); i++) { + std::vector tensor_map_index; + tensor_map_index.push_back((int32_t)(LAST_INDEX(SizeToUint(dev_matrix_shape_.size())))); + for (size_t j = 1; j < strategy_->GetInputDim()[i].size(); ++j) { + tensor_map_index.push_back(MAP_NONE); + } + inputs_tensor_map_.push_back(tensor_map_index); + outputs_tensor_map_.push_back(tensor_map_index); + } + return SUCCESS; +} + +Status VirtualDatasetInfo::InferTensorInfo() { + for (size_t i = 0; i < strategy_->GetInputNumber(); i++) { + MS_LOG(INFO) << name_ << ": InferTensorInfo " << i << ", size " << strategy_->GetInputNumber(); + TensorLayout tensor_layout_in; + if (tensor_layout_in.InitFromVector(dev_matrix_shape_, inputs_tensor_map_.at(i), inputs_shape_.at(i)) != SUCCESS) { + return FAILED; + } + TensorInfo tensor_info_in(tensor_layout_in); + inputs_tensor_info_.push_back(tensor_info_in); + outputs_tensor_info_.push_back(tensor_info_in); + } + return SUCCESS; +} + +Status VirtualDatasetInfo::GetAttrs() { return SUCCESS; } + +Status VirtualDatasetInfo::Init(const StrategyPtr& strategy) { + if (InitWithManualRepeatCalc(strategy) != SUCCESS) { + MS_LOG(ERROR) << name_ << ": Init failed."; + return FAILED; + } + return SUCCESS; +} + +Status VirtualDatasetInfo::InitForCostModel(const StrategyPtr& strategy) { + if (InitForCostModelWithManualRepeatCalc(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Init for cost model failed."; + } else { + MS_LOG(ERROR) << name_ << ": Init for cost model failed."; + } + return FAILED; + } + + MS_LOG(INFO) << name_ << ": Init for cost model success."; + return SUCCESS; +} + +void VirtualDatasetInfo::ReComputeBatchSplitFlagList() { + for (size_t i = 0; i < inputs_shape_.size(); i++) { + split_flag_list_[i] = true; + } +} + +Status VirtualDatasetInfo::SetCostUnderStrategy(const StrategyPtr& strategy) { + if (SetCostUnderStrategyBase(strategy) != SUCCESS) { + if (is_auto_parallel_) { + MS_LOG(DEBUG) << name_ << ": Set cost under strategy failed."; + } else { + MS_LOG(ERROR) << name_ << ": Set cost under strategy failed."; + } + return FAILED; + } + + return SUCCESS; +} + +Status VirtualDatasetInfo::GenerateStrategies(int32_t stage_id) { + if (GetAttrs() != SUCCESS) { + MS_LOG(ERROR) << name_ << ": GetAttrs failed"; + return FAILED; + } + + CheckGlobalDeviceManager(); + is_auto_parallel_ = true; + size_t total_dev_num = g_device_manager->GetDeviceListByStageId(stage_id).size(); + StrategyPtr sp; + std::vector strategy; + for (auto& shape : inputs_shape_) { + Shape temp(shape.size(), 1); + temp[0] = SizeToInt(total_dev_num); + strategy.push_back(temp); + } + sp = std::make_shared(stage_id, strategy); + + if (SetCostUnderStrategy(sp) == SUCCESS) { + MS_LOG(INFO) << name_ << ": Successfully generated batch-parallel-strategy."; + PrintStrategy(sp); + } else { + MS_LOG(ERROR) << name_ << ": Generating batch-parallel-strategy failed."; + return FAILED; + } + return SUCCESS; +} + +Status VirtualDatasetInfo::InferAsLossDivisor() { + // no need to insert div op + as_loss_divisor_ = 1; + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/virtual_dataset_info.h b/mindspore/ccsrc/parallel/ops_info/virtual_dataset_info.h new file mode 100644 index 00000000000..d22565d3054 --- /dev/null +++ b/mindspore/ccsrc/parallel/ops_info/virtual_dataset_info.h @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_OPS_INFO_DATASET_INFO_H_ +#define PARALLEL_OPS_INFO_DATASET_INFO_H_ + +#include +#include +#include +#include +#include +#include "ir/value.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { +class VirtualDatasetInfo : public OperatorInfo { + public: + VirtualDatasetInfo(const std::string& name, const Shapes& inputs_shape, const Shapes& outputs_shape, + const PrimitiveAttrs& attrs) + : OperatorInfo(name, inputs_shape, outputs_shape, attrs) { + vd_cost_ptr_ = std::make_shared(); + } + ~VirtualDatasetInfo() override = default; + Status Init(const StrategyPtr& strategy) override; + Status InitForCostModel(const StrategyPtr& strategy) override; + + Status GenerateStrategies(int32_t stage_id) override; + Status SetCostUnderStrategy(const StrategyPtr& strategy) override; + OperatorCostPtr GetOperatorCost() const override { return vd_cost_ptr_; } + void ReComputeBatchSplitFlagList() override; + + protected: + Status CheckStrategy(const StrategyPtr& strategy) override; + Status InferMirrorOps() override; + Status InferForwardCommunication() override; + Status InferTensorInfo() override; + Status InferDevMatrixShape() override; + Status InferTensorMap() override; + Status GetAttrs() override; + Status InferAsLossDivisor() override; + + private: + VirtualDatasetCostPtr vd_cost_ptr_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // PARALLEL_OPS_INFO_VIRTUAL_DATASET_INFO_H_ diff --git a/mindspore/ccsrc/parallel/status.h b/mindspore/ccsrc/parallel/status.h new file mode 100644 index 00000000000..4c018c7c175 --- /dev/null +++ b/mindspore/ccsrc/parallel/status.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_STATUS_H_ +#define MINDSPORE_CCSRC_PARALLEL_STATUS_H_ + +#include +#include + +namespace mindspore { +namespace parallel { + +enum Status { + SUCCESS = 0, + FAILED, + INVALID_ARGUMENT, +}; +} +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_STATUS_H_ diff --git a/mindspore/ccsrc/parallel/step_auto_parallel.cc b/mindspore/ccsrc/parallel/step_auto_parallel.cc new file mode 100644 index 00000000000..c3e3f5893ea --- /dev/null +++ b/mindspore/ccsrc/parallel/step_auto_parallel.cc @@ -0,0 +1,964 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/step_auto_parallel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "optimizer/opt.h" +#include "optimizer/optimizer.h" +#include "pipeline/pipeline.h" +#include "pipeline/parse/python_adapter.h" +#include "parallel/auto_parallel/edge_costmodel.h" +#include "parallel/auto_parallel/graph_costmodel.h" +#include "parallel/step_parallel.h" +#include "parallel/auto_parallel/dp_algo_costmodel.h" +#include "parallel/ops_info/tmp_identity_info.h" +#include "parallel/context.h" +#include "parallel/auto_parallel/rec_core/rec_partition.h" +#include "parallel/auto_parallel/rec_core/rec_parse_graph.h" +#include "parallel/auto_parallel/rec_core/rec_generate_strategy.h" +#include "ir/meta_tensor.h" + +namespace mindspore { +namespace parallel { +// splittable_op_ will continuously be updated +std::vector splittable_op_ = {MATMUL, + GELU, + TANH, + SOFTMAX, + LOG_SOFTMAX, + ACTIVATION, + PRELU, + FLOORDIV, + L2_NORMALIZE, + TRANSPOSE, + RESHAPE, + TENSOR_ADD, + SUB, + MUL, + DIV, + GREATER, + MAXPOOL, + MAXPOOLV2, + VIRTUAL_DATA_SET, + SPARSE_SOFTMAX_CROSS_ENTROPY_WITH_LOGITS, + RELU, + ONEHOT, + DROPOUT_DO_MASK, + DROPOUT_GEN_MASK, + REDUCE_MAX, + REDUCE_MIN, + ARGMAXWITHVALUE, + ARGMINWITHVALUE, + REDUCE_SUM, + CONV2D, + FUSE_BATCH_NORM, + POOLING, + SOFTMAX_CROSS_ENTROPY_WITH_LOGITS, + MAX_POOL_WITH_ARGMAX, + SIMPLE_MEAN, + FLATTEN, + BATCH_NORM, + BIAS_ADD, + ASSIGN_SUB, + COS, + ACOS, + EXP, + LOG, + REDUCE_MEAN, + REAL_DIV, + SIGMOID, + POW, + MAXIMUM, + EQUAL, + LOGICALNOT, + GATHERV2, + STRIDEDSLICE, + SQRT, + GET_NEXT, + CAST, + SQUEEZE}; + +std::vector elementwise_op_ = {ACTIVATION, GELU, TANH, SOFTMAX, LOG_SOFTMAX, RELU, SQRT, + CAST, POW, EXP, LOG, COS, ACOS, LOGICALNOT}; + +std::vector ignore_manual_strategy_op_ = {BATCH_NORM}; + +bool StepAutoParallel(const FuncGraphPtr &root, const opt::OptimizerPtr &) { + MS_EXCEPTION_IF_NULL(root); + MS_EXCEPTION_IF_NULL(ParallelContext::GetInstance()); + std::string parallel_mode = ParallelContext::GetInstance()->parallel_mode(); + // assume no change to graph + bool changes = false; + // control whether use model_parallel mode + if ((parallel_mode != AUTO_PARALLEL) || root->flags()[AUTO_PARALLEL_RUN_ONCE_ONLY]) { + return changes; + } + // check whether strategy_search_mode is valid + std::string strategy_search_mode = ParallelContext::GetInstance()->strategy_search_mode(); + if ((strategy_search_mode != DYNAMIC_PROGRAMMING) && (strategy_search_mode != RECURSIVE_PROGRAMMING)) { + // Setting searching mode: dynanic programming as default. + strategy_search_mode = DYNAMIC_PROGRAMMING; + MS_LOG(INFO) << "Non-idicated strategy searching mode, using DP searching mode as default"; + } + + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + + if (MsContext::GetInstance()->save_graphs_flag()) { + draw::Draw(STEP_AUTO_PARALLEL_BEGIN, root); + } + MS_LOG(INFO) << "Now entering step auto parallel"; + TOTAL_OPS = 0; + AnfNodePtr ret = root->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + + if (ParallelInit() != SUCCESS) { + MS_LOG(EXCEPTION) << "Parallel init failed"; + } + + // mark the forward cnodes, parallel only care these nodes + MarkForwardCNode(root); + + if (FindCommunicationOp(all_nodes)) { + MS_LOG(EXCEPTION) << "The graph contain communication op"; + } + + // search parallelization strategy + if (strategy_search_mode == DYNAMIC_PROGRAMMING) { + if (ParallelStrategySearch(all_nodes, root) != SUCCESS) { + MS_LOG(EXCEPTION) << "Auto-parallel strategy search failed when using DP searching mode"; + } + } else if (strategy_search_mode == RECURSIVE_PROGRAMMING) { + if (ParallelStrategyRecSearch(all_nodes, root) != SUCCESS) { + MS_LOG(EXCEPTION) << "Auto-parallel strategy search failed when using RP searching mode"; + } + } else { + MS_LOG(EXCEPTION) << "Auto-parallel strategy searching mode unexpected"; + } + + (void)gettimeofday(&end_time, nullptr); + uint64_t time = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + time += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Now leaving step auto parallel, used time: " << time << " us"; + + root->flags()[AUTO_PARALLEL_RUN_ONCE_ONLY] = true; + return changes; +} + +// Given the node, return whether each input is a parameter or a output of a operator. +// The returned boolean vector should be the same order of the inputs, thus its implementation +// is closely consistent with ExtractShape() in step_parallel.cc +std::vector ExtractInputParameterByNode(const CNodePtr &node) { + std::vector is_parameter; + std::vector node_inputs{node->inputs()}; + for (size_t i = 1; i < node_inputs.size(); ++i) { + auto input = node_inputs[i]; + + if (input->isa()) { + auto input_parameter = input->cast(); + if (input_parameter->has_default()) { + bool require_grad = + py::cast(parse::python_adapter::GetPyObjAttr(input_parameter->default_param(), "requires_grad")); + is_parameter.push_back(require_grad); + } else { + is_parameter.push_back(false); + } + } else if (input->isa() || IsValueNode(input) || IsValueNode(input)) { + is_parameter.push_back(false); + } + } + return is_parameter; +} + +// Given the type, return the number of bytes to represent this type +size_t GetLengthOfDataType(const TypePtr &type) { + switch (type->type_id()) { + case kNumberTypeBool: + return sizeof(bool); + case kNumberTypeInt8: + return sizeof(int8_t); + case kNumberTypeInt16: + return sizeof(int16_t); + case kNumberTypeInt32: + return sizeof(int32_t); + case kNumberTypeInt64: + return sizeof(int64_t); + case kNumberTypeUInt8: + return sizeof(uint8_t); + case kNumberTypeUInt16: + return sizeof(uint16_t); + case kNumberTypeUInt32: + return sizeof(uint32_t); + case kNumberTypeUInt64: + return sizeof(uint64_t); + case kNumberTypeFloat16: + return sizeof(float) / 2; + case kNumberTypeFloat32: + return sizeof(float); + case kNumberTypeFloat64: + return sizeof(double); + case kNumberTypeInt: + return sizeof(int); + case kNumberTypeUInt: + return sizeof(uint); + case kNumberTypeFloat: + return sizeof(float); + default: + MS_LOG(EXCEPTION) << "Unexpected type " << type->type_name(); + } +} + +size_t GetInputsTypeLen(const AnfNodePtr &input) { + MS_EXCEPTION_IF_NULL(input); + if (!input->isa() && !input->isa() && !IsValueNode(input)) { + MS_LOG(EXCEPTION) << "The input node is not a cnode or parameter or tensor"; + } + + size_t input_type_len = 0; + auto type = input->Type(); + MS_EXCEPTION_IF_NULL(type); + if (type->isa()) { + auto input_element_type = type->cast()->element(); + input_type_len = GetLengthOfDataType(input_element_type); + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << type->type_name(); + } + return input_type_len; +} + +// Given the node, return the element length of input and output +std::vector> ExtractInputAndOutputTypeLengthByNode(const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + std::vector inputs_type_len; + std::vector outputs_type_len; + std::vector> all_types; + std::vector node_inputs{node->inputs()}; + + // extract input element length + for (auto &input : node_inputs) { + if (IsValueNode(input)) { + auto func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + std::vector parameters = FindParameterByRefKeyNode(input, func_graph); + if (parameters.size() != 1) { + MS_LOG(EXCEPTION) << "Find parameter by ref key node failed"; + } + inputs_type_len.push_back(GetInputsTypeLen(parameters[0])); + } else if (input->isa() || input->isa() || IsValueNode(input)) { + // extract input shape from parameter and apply node + inputs_type_len.push_back(GetInputsTypeLen(input)); + } + } + all_types.push_back(inputs_type_len); + + // extract output element length + auto primary_output_type = node->Type(); + MS_EXCEPTION_IF_NULL(primary_output_type); + if (primary_output_type->isa()) { + // in this case, the output is a tuple + auto tuple_output_type = primary_output_type->cast(); + auto elements = tuple_output_type->elements(); + for (auto &ele : elements) { + if (ele->isa()) { + auto ele_element_type = ele->cast()->element(); + outputs_type_len.push_back(GetLengthOfDataType(ele_element_type)); + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << primary_output_type->type_name(); + } + } + } else { + // in this case, the output is a single tensor + if (primary_output_type->isa()) { + auto element_type = primary_output_type->cast()->element(); + outputs_type_len.push_back(GetLengthOfDataType(element_type)); + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << primary_output_type->type_name(); + } + } + all_types.push_back(outputs_type_len); + + return all_types; +} + +// Be careful the argument is cnode_full_name, not the op_name +bool IsIgnoreStrategyOperator(const std::string &cnode_full_name) { + for (auto &ignore_op : ignore_manual_strategy_op_) { + if (cnode_full_name.find(ignore_op) != std::string::npos) { + return true; + } + } + return false; +} + +bool IsElementWiseOperator(const std::string &op_name) { + auto iter = std::find(elementwise_op_.begin(), elementwise_op_.end(), op_name); + return (iter != elementwise_op_.end()); +} + +bool IsSplittableOperator(const std::string &op_name) { + std::vector::iterator iter; + iter = std::find(splittable_op_.begin(), splittable_op_.end(), op_name); + return (iter != splittable_op_.end()); +} + +bool IsAutoParallelCareNode(const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(cnode); + ValueNodePtr prim_node = cnode->input(0)->cast(); + if (prim_node == nullptr) { + return false; + } + PrimitivePtr prim = GetValueNode(prim_node); + if (prim == nullptr) { + return false; + } + bool bool_result = IsParallelCareNode(cnode) && !IsSplittableOperator(prim->name()); + if (bool_result) { + MS_LOG(EXCEPTION) << "Should implementing OperatorInfo for: " << prim->name(); + } else if (prim->name() == CAST) { + return true; + } + return IsParallelCareNode(cnode) && IsSplittableOperator(prim->name()); +} + +OperatorInfoPtr CreateTheOperatorInfo(const PrimitivePtr &prim, const CNodePtr &cnode) { + auto attrs = prim->attrs(); + std::vector shape_list = ExtractShape(cnode); + if (shape_list.empty()) { + MS_LOG(EXCEPTION) << "Failure: node " << cnode->UniqueId() << " failed to extract shape"; + } + // Create an OperatorInfo instance + OperatorInfoPtr operator_info = NewOperatorInstance(prim, attrs, shape_list); + MS_EXCEPTION_IF_NULL(operator_info); + // Set the parameter information for this OperatorInfo (whether the inputs are parameters or not) + std::vector parameter_info = ExtractInputParameterByNode(cnode); + if (operator_info->set_is_parameter(parameter_info) != SUCCESS) { + MS_LOG(ERROR) << "Initializing parameter information failed for operator: " << operator_info->name(); + return nullptr; + } + // Set the data type for inputs and outputs of this OperatorInfo + std::vector> type_lengths = ExtractInputAndOutputTypeLengthByNode(cnode); + if (operator_info->SetInputAndOutputTypeLength(type_lengths[0], type_lengths[1]) != SUCCESS) { + MS_LOG(ERROR) << "Setting the lengths of inputs and outputs failed for operator: " << operator_info->name(); + return nullptr; + } + // When the 'inputs' contains numerical values for some operators, these values should be extracted from + // ANF graph + auto &inputs = cnode->inputs(); + std::vector input_value; + for (size_t index = 1; index < inputs.size(); ++index) { + if (inputs[index]->isa()) { + input_value.push_back(GetValueNode(inputs[index])); + } else { + input_value.emplace_back(nullptr); + } + } + operator_info->set_input_value(input_value); + operator_info->set_outputs_dtype(cnode->Type()); + operator_info->set_cnode(cnode); + // If no strategy has been configured for this operator, then candidate strategies are generated for + // auto-strategy searching + if (!StrategyFound(attrs)) { + // Compute split_flag_list_, indicating which input has batch dimension. This is ONLY used for preparation for + // BatchParallelInfo operator + operator_info->ComputeBatchSplitFlagList(); + if (operator_info->GenerateStrategies(0) != SUCCESS) { + MS_LOG(ERROR) << "Strategy search for Operator " << operator_info->name() << " failed."; + return nullptr; + } + } else { + // In this case, the configured strategy should be extracted to help setting cost + StrategyPtr strategyPtr = parallel::ExtractStrategy(attrs); + if (strategyPtr != nullptr) { + if (prim->name() == RESHAPE) { + MS_LOG(EXCEPTION) << "Setting strategy for Reshape goes for nothing!"; + } + // Set cost for this configured strategy + if (operator_info->SetCostUnderStrategy(strategyPtr) != SUCCESS) { + MS_LOG(EXCEPTION) << "Failure: operator " << prim->name() << " SetCostUnderStrategy failed"; + } else if (!NOT_FULLY_USE_DEVICES) { + if (!IsIgnoreStrategyOperator(cnode->fullname_with_scope())) { + // If configured to fully use devices, then checking for the user-specified strategy + int32_t used_devices = operator_info->used_devices(); + MS_EXCEPTION_IF_NULL(g_device_manager); + auto total_device_num = g_device_manager->GetDeviceListByStageId(0).size(); + // 'used_devices == -1' means that 'used_devices_' is not set + if ((used_devices == -1) || IntToSize(used_devices) != total_device_num) { + MS_LOG(EXCEPTION) << "In configuration 'NOT_FULLY_USE_DEVICES' = False, " + << "but the specified strategy uses device: " << used_devices + << ", total devices: " << total_device_num; + } + } + } + } + } + return operator_info; +} + +Status ConstructCostGraphNodes(const std::vector &all_nodes, const FuncGraphPtr &) { + MS_LOG(INFO) << "Constructing nodes for cost graph begins."; + entire_costgraph = std::make_shared(); + entire_costgraph->SetDeviceMemoryAndCostParameter(); + bool new_operator = true, first_operator = true; + std::string first_operator_cnode; + size_t current_op_index = 0; + + // Step 1 + for (auto &node : all_nodes) { + // NOTE: we only care about splittable Primitive operators + auto cnode = node->cast(); + bool bool_result = (cnode == nullptr) || (!IsValueNode(cnode->input(0))); + if (bool_result) { + continue; + } + ValueNodePtr prim_anf_node = cnode->input(0)->cast(); + if (!IsAutoParallelCareNode(cnode)) { + continue; + } + PrimitivePtr prim = GetValueNode(prim_anf_node); + MS_EXCEPTION_IF_NULL(prim); + + // When visiting the second subgraph, use the corresponding operatorInfo which already created + bool modify_new_operator = (new_operator) && (!first_operator) && (cnode->UniqueId() == first_operator_cnode); + if (modify_new_operator) { + new_operator = false; + } + if (new_operator) { + auto operator_info = CreateTheOperatorInfo(prim, cnode); + if (operator_info == nullptr) { + return FAILED; + } + // Needed by rec_parser + operator_info->set_type(prim->name()); + std::vector inputs_tensor_name = ExtractInputsTensorName(cnode); + operator_info->set_cnode_name(cnode->ToString()); + + entire_costgraph->AddOperator(operator_info); + (void)cnode->set_operator_info(operator_info); + if (first_operator) { + first_operator_cnode = cnode->UniqueId(); + first_operator = false; + } + // Needed by rec_parser + entire_costgraph->add_inputs_tensor_name(inputs_tensor_name); + } else { + auto current_op_ptr = entire_costgraph->FindOperatorByIndex(current_op_index); + if (current_op_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Find " << prim->name() << " from CostGraph failed."; + } else { + bool is_find_wrong = (current_op_ptr->name().find(VIRTUAL_DATA_SET_INFO) == std::string::npos) && + (current_op_ptr->name().find(BATCH_PARALLEL) == std::string::npos) && + (current_op_ptr->name().find(prim->name()) == std::string::npos); + + if (is_find_wrong) { + MS_LOG(EXCEPTION) << "The OperatorInfo: " << current_op_ptr->name() + << " does not match the Prim: " << prim->name(); + } + (void)cnode->set_operator_info(current_op_ptr); + current_op_index++; + } + } + } + if ((!new_operator) && (current_op_index != entire_costgraph->GetOperators().size())) { + MS_LOG(EXCEPTION) << "The second subgraph's operator number: " << current_op_index + << " does not match the first ones: " << entire_costgraph->GetOperators().size(); + } + + MS_LOG(INFO) << "Constructing nodes for cost graph ends."; + return SUCCESS; +} + +void ConstructCostGraphEdges(const std::vector &all_nodes) { + // Step 2 + MS_LOG(INFO) << "Constructing edges for cost graph begins."; + for (auto &node : all_nodes) { + auto cnode = node->cast(); + bool bool_result_cnode = (cnode == nullptr) || !IsValueNode(cnode->input(0)); + if (bool_result_cnode) { + continue; + } + auto &inputs = cnode->inputs(); + ValueNodePtr prim_anf_node = inputs[0]->cast(); + if (!IsAutoParallelCareNode(cnode)) { + continue; + } + PrimitivePtr prim = GetValueNode(prim_anf_node); + size_t edge_count = 0; + + for (size_t i = 1; i < inputs.size(); ++i) { + auto prev_cnode = inputs[i]->cast(); + bool bool_result_prev_cnode = (prev_cnode == nullptr) || (!IsValueNode(prev_cnode->input(0))); + if (bool_result_prev_cnode) { + continue; + } + ValueNodePtr prev_prim_anf_node = prev_cnode->input(0)->cast(); + PrimitivePtr prev_prim = prev_prim_anf_node->value()->cast(); + size_t output_index = 0; + + bool bool_result = + (IsAutoParallelCareNode(prev_cnode)) || (prev_prim->name() == TUPLE_GETITEM) || (prev_prim->name() == DEPEND); + while (bool_result) { + if (IsAutoParallelCareNode(prev_cnode)) { + std::string edge_name = + prev_cnode->operator_info()->name() + OPERATOR_TO_OPERATOR_CONNECTOR + cnode->operator_info()->name(); + // If the edge between these two operators already has been added, then the edge will not be added again. + if (entire_costgraph->IsEdgeInCostGraph(edge_name, output_index, i - 1)) { + break; + } + EdgePtr edge_ptr; + MS_LOG(INFO) << "Creating edge: " << edge_name; + + bool follow_strategy = ELEMENTWISE_OP_STRA_FOLLOW && IsElementWiseOperator(prev_prim->name()); + if (follow_strategy) { + // Redistribution in not allowed on the edge. + // Elementwise operators have the same strategy as their previous operators. + edge_ptr = std::make_shared(edge_name, prev_cnode->operator_info(), cnode->operator_info(), + output_index, i - 1, false, true); + } else { + edge_ptr = std::make_shared(edge_name, prev_cnode->operator_info(), cnode->operator_info(), + output_index, i - 1, false); + } + + // Init costs for this edge + if (edge_ptr->InitEdgeCost() != SUCCESS) { + MS_LOG(EXCEPTION) << "Edge cost initialization failed"; + } + cnode->operator_info()->AddPrevEdge(edge_ptr); + prev_cnode->operator_info()->AddSuccEdge(edge_ptr); + entire_costgraph->AddEdge(prev_cnode->operator_info(), cnode->operator_info(), edge_ptr); + MS_LOG(INFO) << "Successfully adding the edge between " << prev_cnode->operator_info()->name() << " and " + << cnode->operator_info()->name(); + edge_count++; + + break; + } else if (prev_prim->name() == TUPLE_GETITEM) { + // In this case, 'prev_anf_node' is 'tuple_getitem', the actual precursor node is node before + // this 'tuple_getitem' + MS_LOG(INFO) << "Jumping the 'tuple_getitem' operator."; + output_index = IntToSize(GetValue(GetValueNode(prev_cnode->input(2)))); + prev_cnode = prev_cnode->input(1)->cast(); + bool bool_result_tuple = (prev_cnode == nullptr) || (!IsValueNode(prev_cnode->input(0))); + if (bool_result_tuple) { + break; + } + prev_prim_anf_node = prev_cnode->input(0)->cast(); + prev_prim = prev_prim_anf_node->value()->cast(); + if (!IsAutoParallelCareNode(prev_cnode)) { + MS_LOG(EXCEPTION) << "Did not create OperatorInfo for : " << prev_prim->name(); + } + MS_LOG(INFO) << "Jumped the 'tuple_getitem' operator, " + << "and creating an edge between the Operator before " + << "'tuple_getitem' and the Operator after 'tuple_getitem'."; + } else if (prev_prim->name() == DEPEND) { + // In this case, 'prev_anf_node' is 'depend', the actual precursor node is node before + // this 'depend' + MS_LOG(INFO) << "Jumping the 'depend' operator."; + prev_cnode = prev_cnode->input(1)->cast(); + bool bool_result_depend = (prev_cnode == nullptr) || (!IsValueNode(prev_cnode->input(0))); + if (bool_result_depend) { + break; + } + prev_prim_anf_node = prev_cnode->input(0)->cast(); + prev_prim = prev_prim_anf_node->value()->cast(); + MS_LOG(INFO) << "Jumped the 'depend' operator, " + << "and creating an edge between the Operator before " + << "'depend' and the Operator after 'depend'."; + } + bool_result = + (IsAutoParallelCareNode(prev_cnode)) || (prev_prim->name() == TUPLE_GETITEM) || (prev_prim->name() == DEPEND); + } + } + MS_LOG(INFO) << "Successfully created " << edge_count << " edges for: " << cnode->operator_info()->name(); + } + // For the case of a output being used by multiple subsequent operators, the output induced memory cost should be + // calculated only once. This method is for correct the operators' memory cost calculation. + if (entire_costgraph->CorrectOpsStrategyCostForMultiOutputUse() != SUCCESS) { + MS_LOG(EXCEPTION) << "Correcting strategy_cost_ for operators failed."; + } else { + MS_LOG(INFO) << "Correcting strategy_cost_ for operators succeeded."; + } + MS_LOG(INFO) << "Constructing edges for cost graph ends."; +} + +std::pair> CNodeWithRefKeys(const AnfNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(cnode); + std::vector refkeys; + if (cnode->isa()) { + auto cnode_ptr = cnode->cast(); + auto inputs = cnode_ptr->inputs(); + for (auto &one_input : inputs) { + if (IsValueNode(one_input)) { + refkeys.push_back(one_input); + } + } + if (refkeys.size() >= 1) { + return std::make_pair(cnode, refkeys); + } + } + return {nullptr, refkeys}; +} + +void AugmentCostGraph(const std::vector &all_nodes) { + // Step 3 + for (auto &node : all_nodes) { + auto cnode_with_refkeys = CNodeWithRefKeys(node); + if ((!node->isa()) && (cnode_with_refkeys.first == nullptr)) { + continue; + } + std::string parameter_name; + AnfNodePtr target_parameter = nullptr; + AnfNodeIndexSet target_set; + + if (cnode_with_refkeys.first != nullptr) { + // Dealing with the RefKey case + auto refkeys = cnode_with_refkeys.second; + auto cnode = cnode_with_refkeys.first; + if (refkeys.size() > 1) { + MS_LOG(EXCEPTION) << "CNode: " << cnode->fullname_with_scope() << " 's inputs have more than 1 RefKeys."; + } + MS_EXCEPTION_IF_NULL(cnode->func_graph()); + auto cnode_func_graph = cnode->func_graph(); + MS_EXCEPTION_IF_NULL(cnode->func_graph()->manager()); + + // Find the RefKey being used + auto candidate_set_by_refkey = cnode_func_graph->manager()->node_users()[refkeys[0]]; + for (auto &candidate : candidate_set_by_refkey) { + auto candidate_node = candidate.first; + auto c = candidate_node->cast(); + if (c == nullptr || !IsValueNode(c->input(0))) { + continue; + } + if (!IsAutoParallelCareNode(c)) { + continue; + } + target_set.add(candidate); + } + + // Find the corresponding Parameter being used + std::vector parameters = FindParameterByRefKeyNode(refkeys[0], cnode_func_graph); + if (parameters.size() != 1) { + MS_LOG(EXCEPTION) << "Find parameter by ref key node failed"; + } + parameter_name = parameters[0]->cast()->name(); + target_parameter = parameters[0]; + auto candidate_set_by_para = cnode_func_graph->manager()->node_users()[parameters[0]]; + for (auto &candidate : candidate_set_by_para) { + auto candidate_node = candidate.first; + auto c = candidate_node->cast(); + if (c == nullptr || !IsValueNode(c->input(0))) { + continue; + } + if (!IsAutoParallelCareNode(c)) { + continue; + } + (void)target_set.insert(candidate); + } + } else if (node->isa()) { + // Dealing with the Parameter case + MS_EXCEPTION_IF_NULL(node->func_graph()); + MS_EXCEPTION_IF_NULL(node->func_graph()->manager()); + auto candidate_set = node->func_graph()->manager()->node_users()[node]; + for (auto &candidate : candidate_set) { + auto candidate_node = candidate.first; + auto c = candidate_node->cast(); + if (c == nullptr || !IsValueNode(c->input(0))) { + continue; + } + if (!IsAutoParallelCareNode(c)) { + continue; + } + (void)target_set.insert(candidate); + } + // In this case, node is a Parameter + parameter_name = node->cast()->name(); + target_parameter = node; + } + if (target_set.size() <= 1) { + continue; + } + + // Rule out the case when a Parameter being used by a Operator, but the Operator appears in multiple CNODEs + std::set target_without_duplicate; + for (auto &target : target_set) { + auto target_cnode = target.first->cast(); + auto input_index = target.second; + (void)target_without_duplicate.insert(std::to_string(input_index) + target_cnode->operator_info()->name()); + } + if (target_without_duplicate.size() <= 1) { + continue; + } + + // Here, it is sure that this Parameter (RefKey) is being used by multiple Operators. + OperatorInfoPtr tmp_identity_ptr; + bool new_identity = false; + std::string tmp_identity_name; + auto returned_identity = entire_costgraph->FindTmpIdentityByParameterName(parameter_name); + if (returned_identity != nullptr) { + // In this case, the TmpIdentityInfo instance has already been created + new_identity = false; + tmp_identity_ptr = returned_identity; + tmp_identity_name = tmp_identity_ptr->name(); + } else { + // In the case, the TmpIdentityInfo instance has NOT been created. Thus, a new one is created. + new_identity = true; + // 1) extract input shape from this Parameter + MS_EXCEPTION_IF_NULL(target_parameter); + AbstractBasePtr abstract = target_parameter->abstract(); + if (abstract == nullptr) { + MS_LOG(EXCEPTION) << "Failure: abstract is nullptr"; + } + auto input_shape = dyn_cast(abstract->GetShapeTrack()); + if (input_shape == nullptr) { + MS_LOG(EXCEPTION) << "Failure: input_shape is nullptr"; + } + std::vector shape_int = input_shape->shape(); + Shape shape; + (void)std::transform(shape_int.begin(), shape_int.end(), std::back_inserter(shape), + [](int sub_shape) { return static_cast(sub_shape); }); + Shapes inputs_shape = {shape}; + Shapes outputs_shape = {shape}; + // 2) init the attr + std::unordered_map attr = {}; + + // Create the TmpIdentity instance + tmp_identity_ptr = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr->set_name(tmp_identity_ptr->name() + std::to_string(TOTAL_OPS)); + TOTAL_OPS++; + tmp_identity_ptr->set_refkey_parameter_name(parameter_name); + // Set the parameter and type lengths for inputs and outputs + std::vector is_parameter; + auto casted_target_parameter = target_parameter->cast(); + MS_EXCEPTION_IF_NULL(casted_target_parameter); + if (casted_target_parameter->has_default()) { + bool require_grad = py::cast( + parse::python_adapter::GetPyObjAttr(casted_target_parameter->default_param(), "requires_grad")); + is_parameter.push_back(require_grad); + } else { + is_parameter.push_back(false); + } + if (tmp_identity_ptr->set_is_parameter(is_parameter) != SUCCESS) { + MS_LOG(EXCEPTION) << "Setting parameter for TmpIdentityInfo failed"; + } + auto node_type = target_parameter->Type(); + if (node_type->isa()) { + auto input_element_type = node_type->cast()->element(); + std::vector type_length = {GetLengthOfDataType(input_element_type)}; + if (tmp_identity_ptr->SetInputAndOutputTypeLength(type_length, type_length) != SUCCESS) { + MS_LOG(EXCEPTION) << "Setting input and output type length for TmpIdentityInfo failed"; + } + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << node_type->type_name(); + } + + // Generate strategies for this TmpIdentityInfo instance; + if (tmp_identity_ptr->GenerateStrategies(0) != SUCCESS) { + MS_LOG(EXCEPTION) << "Strategy search for Operator failed : " << tmp_identity_ptr->name(); + } + } + // A flag recording whether new edges have been created or not + bool add_identity_edge = false; + + // Create edges between this TmpIdentityInfo instance and subsequent Operator instances + for (auto &target : target_set) { + auto target_cnode = target.first->cast(); + auto prim = GetValueNode(target_cnode->input(0)); + auto input_index = target.second; + + std::string edge_name = + std::string(IDENTITY_INFO) + OPERATOR_TO_OPERATOR_CONNECTOR + target_cnode->operator_info()->name(); + // If the edge between these two operators already has been added, then the edge will not be added again. + if (entire_costgraph->IsEdgeInCostGraph(edge_name, 0, IntToSize(input_index - 1))) { + continue; + } + std::shared_ptr edge_ptr = std::make_shared( + edge_name, tmp_identity_ptr, target_cnode->operator_info(), 0, input_index - 1, false, true); + + // Correct the memory calculation for a parameter being used by multiple operators. The parameter is calculated + // only once + if (target_cnode->operator_info()->CorrectStrategyCostForMultiOutputUse(IntToSize(input_index - 1)) != SUCCESS) { + MS_LOG(EXCEPTION) << "Correcting strategy_cost_ failed : " << prim->name(); + } else { + MS_LOG(INFO) << "Correcting strategy_cost_ succeeded. " << prim->name(); + } + + if (edge_ptr->InitEdgeCost() != SUCCESS) { + MS_LOG(EXCEPTION) << "Edge cost initialization failed"; + } + target_cnode->operator_info()->AddPrevEdge(edge_ptr); + tmp_identity_ptr->AddSuccEdge(edge_ptr); + entire_costgraph->AddEdge(tmp_identity_ptr, target_cnode->operator_info(), edge_ptr); + MS_LOG(INFO) << "Successfully adding the edge between " << tmp_identity_ptr->name() << " and " + << target_cnode->operator_info()->name(); + add_identity_edge = true; + } + if (new_identity && add_identity_edge) { + // Add the TmpIdentityInfo to CostGraph if BOTH two conditions are satisfied + entire_costgraph->AddOperator(tmp_identity_ptr); + } + } +} + +Status ParallelStrategySearch(const std::vector &all_nodes, const FuncGraphPtr &root) { + // There are 4 meta-steps to determine the parallelization strategy for the ANF graph. + // Step 1: Traverse the ANF graph, and create NODEs for costgraph: + // create the OperatorInfo object for each primitive, and enumerate the parallelization strategies + // for each OperatorInfo; + // Step 2: Traverse the ANF graph, and create EDGES for costgraph: + // create the Edge object for each pair of OperatorInfo, and enumerate the parallelization strategies + // for each edge, based on the strategies of two OperatorInfos; + // Step 3: Augment the costgraph: + // taking care for the case of a single Parameter being used by multiple operators. Create a TmpIdentity + // operator for this Parameter, and add an edge for the use of this Parameter by each + // subsequent operator; + // Step 3.1: Correct the memory calculation for memory reuse + // Step 4: Run the Dynamic Programming algorithm: + // in this process, cost is calculated based on not only the operators, but also the edges. Here, the edge + // cost is caused by the redistribution of a operator's output tensor layout to the next operator's input + // tensor layout. Note that there may be several connected components in the costgraph, and the DP algorithm + // runs on each of them. + // + // OUTPUT: the determined strategy for each operator. + + // Step 1 + if (ConstructCostGraphNodes(all_nodes, root) == SUCCESS) { + MS_LOG(INFO) << "Constructing nodes for cost graph succeeded. There are " << entire_costgraph->GetOperators().size() + << " operators."; + } else { + MS_LOG(EXCEPTION) << "Constructing nodes for cost graph failed."; + } + + // Step 2 + ConstructCostGraphEdges(all_nodes); + MS_LOG(INFO) << "Constructing edges for cost graph succeeded. There are " << entire_costgraph->GetOperators().size() + << " operators, and " << entire_costgraph->GetNumPairs() << " edges.", + + // Step 3: Augment the costgraph. + AugmentCostGraph(all_nodes); + MS_LOG(INFO) << "After the augmenting procedure, there are " << entire_costgraph->GetOperators().size() + << " operators, and " << entire_costgraph->GetNumPairs() << " edges."; + + // Step 3.1: Correcting calculation for memory reuse + if (entire_costgraph->ComputeOpsAndEdgesParameterInvolved() == SUCCESS) { + // Correcting operators' memory usage + if (entire_costgraph->CorrectOpsStrategyCostForMemoryReuse() != SUCCESS) { + MS_LOG(EXCEPTION) << "Correcting operators' cost for memory reuse failed."; + } + // Correcting edges' memory usage + if (entire_costgraph->CorrectEdgesStrategyCostForMemoryReuse() != SUCCESS) { + MS_LOG(EXCEPTION) << "Correcting edges' cost for memory reuse failed."; + } + } else { + MS_LOG(EXCEPTION) << "Computing operators' parameter_involved failed."; + } + + // Step 4: run DP algorithm on the costgraph. + if (GetStrategy(entire_costgraph) != SUCCESS) { + MS_LOG(ERROR) << "Strategy search for cost-graph fails"; + return FAILED; + } + MS_LOG(INFO) << "Searching strategy succeeded."; + + if (entire_costgraph->InitSelectedStrategy() == SUCCESS) { + MS_LOG(INFO) << "Init selected strategy succeeded."; + } else { + MS_LOG(EXCEPTION) << "Init selected strategy failed."; + } + + // print the selected strategy + for (auto &op : entire_costgraph->GetOperators()) { + StrategyPtr s_strategy = op->selected_strategy(); + MS_LOG(INFO) << op->name() << " : The strategy is:"; + PrintStrategy(s_strategy); + } + + return SUCCESS; +} + +std::vector> RecInputTensorNames(const std::map::iterator &it, + std::vector> input_tensor_names) { + for (size_t j = 0; j < input_tensor_names.size(); j++) { + for (size_t k = 0; k < input_tensor_names[j].size(); k++) { + if (it->first == input_tensor_names[j][k]) { + input_tensor_names[j][k] = it->second; + break; + } + } + } + return input_tensor_names; +} + +Status ParallelStrategyRecSearch(const std::vector &all_nodes, const FuncGraphPtr &root) { + if (ConstructCostGraphNodes(all_nodes, root) == SUCCESS) { + MS_LOG(INFO) << "Constructing nodes for cost graph succeeded. There are " << entire_costgraph->GetOperators().size() + << " operators."; + } else { + MS_LOG(ERROR) << "Constructing nodes for cost graph failed."; + return FAILED; + } + auto ops = entire_costgraph->GetOperators(); + std::vector> input_tensor_names = entire_costgraph->get_inputs_tensor_name_list(); + auto tuple_getitem_list = entire_costgraph->get_tuple_getitem_list(); + for (auto it = tuple_getitem_list.begin(); it != tuple_getitem_list.end();) { + input_tensor_names = RecInputTensorNames(it++, input_tensor_names); + } + + std::shared_ptr> ops_nodes_list(new std::vector); + std::shared_ptr> index_list(new std::vector); + std::shared_ptr>> eli_list(new std::vector>); + + std::shared_ptr graph = ParseGraph(ops, input_tensor_names, ops_nodes_list); + + graph = EliminateGraph(graph, eli_list, index_list); + size_t num_device = g_device_manager->DeviceNum(); + + if (PartitionForAllDevices(num_device, graph) == SUCCESS) { + MS_LOG(INFO) << "Partition Success With " << num_device << " devices."; + } else { + MS_LOG(ERROR) << "PartitionForAllDevices failed."; + return FAILED; + } + + GenerateStrategy(graph, ops, ops_nodes_list, index_list, eli_list); + + if (entire_costgraph->InitSelectedStrategy() == SUCCESS) { + MS_LOG(INFO) << "Init selected strategy succeeded."; + } else { + MS_LOG(ERROR) << "Init selected strategy failed."; + return FAILED; + } + + // print the selected strategy + for (auto &op : entire_costgraph->GetOperators()) { + StrategyPtr s_strategy = op->selected_strategy(); + MS_LOG(INFO) << op->name() << " : The strategy is:"; + PrintStrategy(s_strategy); + } + + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/step_auto_parallel.h b/mindspore/ccsrc/parallel/step_auto_parallel.h new file mode 100644 index 00000000000..d0d603a4f43 --- /dev/null +++ b/mindspore/ccsrc/parallel/step_auto_parallel.h @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARALLEL_STEP_AUTO_PARALLEL_H_ +#define PARALLEL_STEP_AUTO_PARALLEL_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "optimizer/opt.h" +#include "pipeline/pipeline.h" +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { +bool IsSplittableOperator(const std::string &); + +bool IsAutoParallelCareNode(const CNodePtr &); + +// main step of Auto-parallel +bool StepAutoParallel(const FuncGraphPtr &func_graph, const opt::OptimizerPtr &optimizer); + +size_t GetLengthOfDataType(const TypePtr &type); + +std::vector ExtractInputParameterByNode(const CNodePtr &node); + +std::vector> ExtractInputAndOutputTypeLengthByNode(const CNodePtr &node); + +Status ConstructCostGraphNodes(const std::vector &all_nodes, const FuncGraphPtr &root); + +void ConstructCostGraphEdges(const std::vector &all_nodes); + +void AugmentCostGraph(const std::vector &all_nodes); + +Status ParallelStrategySearch(const std::vector &all_nodes, const FuncGraphPtr &root); + +Status ParallelStrategyRecSearch(const std::vector &all_nodes, const FuncGraphPtr &root); + +std::vector> RecInputTensorNames(const std::map::iterator &it, + std::vector> input_tensor_names); + +} // namespace parallel +} // namespace mindspore +#endif // PARALLEL_STEP_AUTO_PARALLEL_H_ diff --git a/mindspore/ccsrc/parallel/step_parallel.cc b/mindspore/ccsrc/parallel/step_parallel.cc new file mode 100644 index 00000000000..927acea7055 --- /dev/null +++ b/mindspore/ccsrc/parallel/step_parallel.cc @@ -0,0 +1,2197 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/step_parallel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parallel/graph_util/graph_info.h" +#include "ir/meta_tensor.h" +#include "optimizer/optimizer.h" +#include "parallel/dynamic_creator.h" +#include "parallel/ops_info/matmul_info.h" +#include "utils/symbolic.h" +#include "operator/ops.h" +#include "parallel/auto_parallel/graph_costmodel.h" +#include "parallel/device_manager.h" +#include "parallel/strategy_checkpoint/parallel_strategy_checkpoint.h" +#include "parallel/graph_util/generate_graph.h" +#include "parallel/context.h" +#include "parallel/node_check.h" +#include "utils/comm_manager.h" +#include "parallel/graph_util/node_info.h" + +using mindspore::tensor::Tensor; + +namespace mindspore { +namespace parallel { +const std::set COMMUNICATION_OPS = {ALL_REDUCE, ALL_GATHER, ALL_TO_ALL, REDUCE_SCATTER}; +const std::set INVALID_LOSS_OPS = {GET_NEXT, VIRTUALLOSS}; + +void SetCommunicationOpGroupLabel(std::vector new_node_input) { + if (new_node_input.empty()) { + return; + } + + ValueNodePtr prim_anf_node = new_node_input[0]->cast(); + PrimitivePtr prim = GetValueNode(prim_anf_node); + MS_EXCEPTION_IF_NULL(prim); + + auto attrs = prim->attrs(); + auto iter = attrs.find(GROUP); + if (iter != attrs.end()) { + auto value = iter->second; + MS_EXCEPTION_IF_NULL(value); + if (value->isa()) { + std::string hash_name = value->cast()->value(); + MS_EXCEPTION_IF_NULL(g_device_manager); + std::string rank_list_name = g_device_manager->FindRankListNameByHashName(hash_name); + (void)prim->AddAttr(GROUP_RANKS, MakeValue(rank_list_name)); + } + } +} + +std::vector CreateInput(const Operator& op, const AnfNodePtr& node, const std::string& instance_name) { + MS_EXCEPTION_IF_NULL(node); + OperatorArgs arg_forward = op.second; + ValuePtr pyop_instance = CreatOpInstance(arg_forward.first, op.first, instance_name); + MS_EXCEPTION_IF_NULL(pyop_instance); + OperatorParams params = arg_forward.second; + + std::vector new_node_input = {NewValueNode(pyop_instance), node}; + if (!params.empty()) { + for (auto& param : params) { + AnfNodePtr val = NewValueNode(param.first.second); + MS_EXCEPTION_IF_NULL(val); + int32_t position = param.second; + (void)new_node_input.insert(new_node_input.begin() + position, val); + } + } + + // if the op have 'group' attr, set the rank list name for the op + SetCommunicationOpGroupLabel(new_node_input); + return new_node_input; +} + +void InsertNode(const Operator& op, const CNodePtr& node, size_t index, const AnfNodePtr& pre_node, + const FuncGraphPtr& func_graph, const std::string& instance_name) { + // insert new node before the node + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + ScopePtr scope = node->scope(); + MS_EXCEPTION_IF_NULL(scope); + std::vector node_input = CreateInput(op, pre_node, instance_name); + CNodePtr new_node = func_graph->NewCNode(node_input); + MS_EXCEPTION_IF_NULL(new_node); + if (instance_name.find(SPLIT_SENS) == std::string::npos) { + new_node->set_in_forward_flag(true); // mark forward flag + } + auto new_node_value = node_input[0]->cast(); + MS_EXCEPTION_IF_NULL(new_node_value); + PrimitivePtr new_node_prim = new_node_value->value()->cast(); + new_node_prim->set_instance_name(instance_name); + new_node->set_scope(scope); + node_input[0]->set_scope(scope); + manager->SetEdge(node, SizeToInt(index), new_node); +} + +std::string CreateInstanceName(const CNodePtr& node, size_t index) { + MS_EXCEPTION_IF_NULL(node); + if (!IsValueNode(node->input(0))) { + MS_LOG(EXCEPTION) << "CreateInstanceName: " << node->ToString() << " doesn't have primitive"; + } + std::string name_base = node->fullname_with_scope(); + std::string name = name_base + "_" + std::to_string(index); + std::string instance_name = HashInstanceName(name); + return instance_name; +} + +void ForwardCommunication(OperatorVector forward_op, const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + // step1:get graph manager distribute_operator + FuncGraphPtr func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto uses_set = manager->node_users()[node]; + CNodePtr node_to_insert = node; + for (auto& uses_pair : uses_set) { + auto uses_cnode = uses_pair.first->cast(); + MS_EXCEPTION_IF_NULL(uses_cnode); + if (!IsValueNode(uses_cnode->input(0))) { + break; + } + PrimitivePtr value_node_prim = GetValueNode(uses_cnode->input(0)); + MS_EXCEPTION_IF_NULL(value_node_prim); + if (value_node_prim->name() == TUPLE_GETITEM) { + if (uses_set.size() > 1) { + MS_LOG(EXCEPTION) << "Now only support one output, but got " << uses_set.size(); + } + node_to_insert = uses_cnode; + } + } + MS_EXCEPTION_IF_NULL(node_to_insert); + std::reverse(forward_op.begin(), forward_op.end()); + + // step2:traverse op_list and insert node + for (size_t index = 0; index < forward_op.size(); ++index) { + std::string instance_name_base = FORWARD_OP; + std::string instance_name = instance_name_base + "_" + CreateInstanceName(node, index); + std::vector forward_input = CreateInput(forward_op[index], node_to_insert, instance_name); + CNodePtr forward_node = func_graph->NewCNode(forward_input); // using NewCNode to creat anfnode + MS_EXCEPTION_IF_NULL(forward_node); + ScopePtr scope = node->scope(); + MS_EXCEPTION_IF_NULL(scope); + forward_node->set_scope(scope); + forward_node->set_in_forward_flag(true); + forward_input[0]->set_scope(scope); + (void)manager->Replace(node_to_insert, forward_node); // using Replace function to insert node + } +} + +CNodePtr InsertMakeTuple(const AnfNodePtr& prev, uint32_t num, const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(prev); + MS_EXCEPTION_IF_NULL(func_graph); + std::vector make_tuple_inputs; + make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (uint32_t i = 0; i < num; i++) { + std::vector tuple_get_item_inputs{NewValueNode(prim::kPrimTupleGetItem), prev, + CreatInt32Imm(UintToInt(i))}; + auto tuple_get_item = func_graph->NewCNode(tuple_get_item_inputs); + MS_EXCEPTION_IF_NULL(tuple_get_item); + make_tuple_inputs.push_back(tuple_get_item); + } + auto make_tuple = func_graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(prev, make_tuple); + return make_tuple; +} + +void InsertRedistribution(const RedistributionOpListPtr& redistribution_oplist_ptr, const CNodePtr& node, + const FuncGraphPtr& func_graph, int pos, const CNodePtr& pre_node) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(pre_node); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if ((redistribution_oplist_ptr->first).size() != (redistribution_oplist_ptr->second).size()) { + MS_LOG(EXCEPTION) << "size of OperatorVector and OutPutInfoVector must be the same!"; + } + for (size_t index = 0; index < (redistribution_oplist_ptr->first).size(); ++index) { + if (pos >= SizeToInt(node->inputs().size())) { + MS_LOG(EXCEPTION) << "InsertRedistribution:pos can't be larger than node's inputs'size"; + } + // Creat new node + AnfNodePtr target_node = node->input(IntToSize(pos)); + MS_EXCEPTION_IF_NULL(target_node); + // Creat instance_name + auto op = (redistribution_oplist_ptr->first)[index]; + std::string op_name = (redistribution_oplist_ptr->first)[index].first; + std::string instance_name_base = REDISTRIBUTION_OP; + std::string instance_name = instance_name_base + "_" + CreateInstanceName(pre_node, index) + op_name; + InsertNode(op, node, IntToSize(pos), target_node, func_graph, instance_name); + if ((redistribution_oplist_ptr->second)[index].first) { + target_node = node->input(IntToSize(pos)); + MS_EXCEPTION_IF_NULL(target_node); + (void)InsertMakeTuple(target_node, (redistribution_oplist_ptr->second)[index].second, func_graph); + } + } +} + +void InsertGetTensorSliceOp(const Operator& op, const CNodePtr& node, const FuncGraphPtr& func_graph, int pos, + const std::string& instance_name) { + if (func_graph == nullptr) { + MS_LOG(EXCEPTION) << "InsertGetTensorSliceOp: the graph is null, the instance name is " << instance_name; + } + + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (pos >= SizeToInt(node->inputs().size())) { + MS_LOG(EXCEPTION) << "InsertGetTensorSliceOp: pos can't be larger than node's inputs'size, the instance name is " + << instance_name; + } + // Creat new node + AnfNodePtr pre_node = node->input(IntToSize(pos)); + MS_EXCEPTION_IF_NULL(pre_node); + InsertNode(op, node, IntToSize(pos), pre_node, func_graph, instance_name); +} + +TensorLayout GetTensorInLayout(const CNodePtr& middle_node, const PrimitivePtr& middle_prim, + const OperatorInfoPtr& distribute_operator) { + TensorInfo tensorinfo_in; + if (middle_prim->name() == TUPLE_GETITEM) { + auto value_node = middle_node->input(2)->cast(); + MS_EXCEPTION_IF_NULL(value_node); + size_t index_s = IntToSize(GetValue(value_node->value())); + if (index_s >= distribute_operator->outputs_tensor_info().size()) { + MS_LOG(EXCEPTION) << "The index out of range, index: " << index_s + << ", vector size: " << distribute_operator->outputs_tensor_info().size(); + } + tensorinfo_in = distribute_operator->outputs_tensor_info()[index_s]; + } else { + if (distribute_operator->outputs_tensor_info().empty()) { + MS_LOG(EXCEPTION) << "The outputs tensor info is empty"; + } + tensorinfo_in = distribute_operator->outputs_tensor_info()[0]; + } + return tensorinfo_in.tensor_layout(); +} + +OperatorInfoPtr GetDistributeOperator(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + if (!IsParallelCareNode(node)) { + return nullptr; + } + OperatorInfoPtr distribute_operator = node->operator_info(); + if (distribute_operator == nullptr) { + MS_LOG(EXCEPTION) << "GetDistributeOperator:distribute_operator is nullptr"; + } + return distribute_operator; +} + +void Redistribution(const std::pair& node_pair, const OperatorInfoPtr& distribute_operator, + const CNodePtr& middle_node, int index, TensorRedistribution tensor_redistribution, + const CNodePtr& pre_node) { + FuncGraphPtr func_graph = middle_node->func_graph(); + if (func_graph == nullptr) { + MS_LOG(EXCEPTION) << "Redistribution:get graph failed"; + } + CNodePtr next_node = node_pair.first->cast(); + MS_EXCEPTION_IF_NULL(next_node); + auto middle_value = middle_node->input(0)->cast(); + MS_EXCEPTION_IF_NULL(middle_value); + PrimitivePtr middle_prim = middle_value->value()->cast(); + MS_EXCEPTION_IF_NULL(middle_prim); + OperatorInfoPtr next_distribute_operator = GetDistributeOperator(next_node); + if (next_distribute_operator == nullptr) { + MS_LOG(EXCEPTION) << "Failure: " << next_node->ToString() << " GetDistributeOperator failed"; + } + RankList dev_list = distribute_operator->global_device_list(); + std::string next_prim_name = GetValueNode(next_node->input(0))->name(); + MS_LOG(DEBUG) << "Redistribution: middle_prim " << middle_prim->name() << " next_prim " << next_prim_name; + MS_LOG(DEBUG) << "Redistribution: middle_node " << middle_node->ToString() << " next_node " << next_node->ToString(); + // extract tensor layout in and out + if (distribute_operator->outputs_tensor_info().empty()) { + MS_LOG(EXCEPTION) << "Failure:pre_node's tensorinfo_in is empty"; + } + + if (IntToSize(index - 1) >= next_distribute_operator->inputs_tensor_info().size()) { + MS_LOG(EXCEPTION) << "The index is out of range, the index is " << index - 1 << ", the vector size is " + << next_distribute_operator->inputs_tensor_info().size(); + } + TensorInfo tensorinfo_out = next_distribute_operator->inputs_tensor_info()[IntToSize(index - 1)]; + TensorLayout tensorlayout_out = tensorinfo_out.tensor_layout(); + TensorLayout tensorlayout_in = GetTensorInLayout(middle_node, middle_prim, distribute_operator); + if (tensor_redistribution.Init(tensorlayout_in, tensorlayout_out, dev_list) == FAILED) { + MS_LOG(ERROR) << "Redistribution: middle_prim " << middle_prim->name() << " next_prim : " << next_prim_name; + MS_LOG(ERROR) << "Redistribution: middle_node " << middle_node->ToString() << " next_node " + << next_node->ToString(); + DumpGraph(func_graph, "redistribution_error"); + MS_LOG(EXCEPTION) << "Failure:tensor_redistribution init failed"; + } + RedistributionOpListPtr redistribution_oplist_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); + if (redistribution_oplist_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Failure:InferTensorRedistribution failed"; + } + MS_LOG(DEBUG) << "Redistribution size " << redistribution_oplist_ptr->first.size(); + if (!redistribution_oplist_ptr->first.empty()) { + // insert node before next node + InsertRedistribution(redistribution_oplist_ptr, next_node, func_graph, node_pair.second, pre_node); + } +} + +bool StrategyFound(std::unordered_map attrs) { + auto iter = attrs.find(STRATEGY); + return !((iter == attrs.end()) || (iter->second->type_name() == NONE)); +} + +bool IsCommunicationOp(const PrimitivePtr& prim) { + MS_EXCEPTION_IF_NULL(prim); + return (COMMUNICATION_OPS.find(prim->name()) != COMMUNICATION_OPS.end()); +} + +bool FindCommunicationOp(const std::vector& all_nodes) { + for (auto& node : all_nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + ValueNodePtr prim_value_node = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_value_node); + PrimitivePtr prim = GetValueNode(prim_value_node); + MS_EXCEPTION_IF_NULL(prim); + + if (IsCommunicationOp(prim) && cnode->in_forward_flag()) { + MS_EXCEPTION_IF_NULL(prim_value_node->scope()); + MS_LOG(INFO) << "The graph contain communication op: " << prim->name() << ", scope name is " + << prim_value_node->scope()->name(); + return true; + } + } + return false; +} + +bool IsParallelCareNode(const CNodePtr& cnode) { + MS_EXCEPTION_IF_NULL(cnode); + ValueNodePtr prim_node = cnode->input(0)->cast(); + if (prim_node == nullptr) { + return false; + } + PrimitivePtr prim = prim_node->value()->cast(); + if (prim == nullptr) { + return false; + } + auto attrs = prim->attrs(); + if (IsInBlackList(prim)) { + MS_LOG(INFO) << "Parallel don't care node: " << prim->name(); + return false; + } + if ((prim->name() == CAST)) { + if ((!attrs.count(STRATEGY)) && (cnode->operator_info() == nullptr)) { + return false; + } + } + + return cnode->in_forward_flag(); +} + +void StepRedistribution(const CNodePtr& node, const OperatorInfoPtr& distribute_operator, const CNodePtr& insert_node, + const TensorRedistribution& tensor_redistribution, const CNodePtr& pre_node) { + MS_EXCEPTION_IF_NULL(node->func_graph()); + FuncGraphManagerPtr manager = node->func_graph()->manager(); + MS_EXCEPTION_IF_NULL(manager); + AnfNodeIndexSet node_set = manager->node_users()[node]; + CNodePtr insert_node_new; + if (IsValueNode(node->input(0))) { + auto current_value = node->input(0)->cast(); + MS_EXCEPTION_IF_NULL(current_value); + PrimitivePtr current_prim = current_value->value()->cast(); + MS_EXCEPTION_IF_NULL(current_prim); + insert_node_new = ((current_prim->name() == TUPLE_GETITEM) ? node : insert_node); + } else { + insert_node_new = insert_node; + } + MS_EXCEPTION_IF_NULL(insert_node_new); + for (auto& node_pair : node_set) { + CNodePtr use_cnode = node_pair.first->cast(); + MS_EXCEPTION_IF_NULL(use_cnode); + if (!IsValueNode(use_cnode->input(0))) { + StepRedistribution(use_cnode, distribute_operator, insert_node_new, tensor_redistribution, pre_node); + } else { + ValueNodePtr prim_anf_node = use_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_anf_node); + PrimitivePtr node_prim = prim_anf_node->value()->cast(); + MS_EXCEPTION_IF_NULL(node_prim); + if (node_prim->name() == DEPEND && node_pair.second != 1) { + continue; + } + if (IsParallelCareNode(use_cnode) && (use_cnode->operator_info() != nullptr)) { + Redistribution(node_pair, distribute_operator, insert_node_new, node_pair.second, tensor_redistribution, + pre_node); + } else { + StepRedistribution(use_cnode, distribute_operator, insert_node_new, tensor_redistribution, pre_node); + } + } + } +} + +void SplitTensor(const AnfNodePtr& node, const CNodePtr& next_node, int index) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(next_node); + OperatorInfoPtr op_info = next_node->operator_info(); + MS_EXCEPTION_IF_NULL(op_info); + + // If the shape of tensor is [] or [1], no need to split it. + Shapes shapes = GetNodeShape(node); + if (shapes.size() != 1) { + MS_LOG(EXCEPTION) << "Split tensor for " << op_info->name() + << ": GetNodeShape for tensor_node, output size is not 1"; + } + Shape shape = shapes[0]; + std::string shape_str = ShapeToString(shape); + if (shape.empty() || ((shape.size() == 1) && (shape[0] == 1))) { + MS_LOG(INFO) << "Split tensor for " << op_info->name() << ": The shape is " << shape_str + << ", no need to split it."; + return; + } + + MS_LOG(INFO) << "Split tensor for " << op_info->name() << ": The shape of tensor is " << shape_str; + + // extract tensor layout + if (IntToSize(index - 1) >= op_info->inputs_tensor_info().size()) { + MS_LOG(EXCEPTION) << "The index is out of range, index is " << index - 1 << ", vector size is " + << op_info->inputs_tensor_info().size(); + } + TensorInfo tensor_info = op_info->inputs_tensor_info()[IntToSize(index - 1)]; + TensorLayout tensor_layout = tensor_info.tensor_layout(); + + // Use _GetTensorSlice operator to split the tensor + FuncGraphPtr func_graph = next_node->func_graph(); // only cnode can get the graph + MS_EXCEPTION_IF_NULL(func_graph); + Operator op = CreateGetTensorSliceOp(tensor_layout); + InsertGetTensorSliceOp(op, next_node, func_graph, index, SPLIT_TENSOR); +} + +void StepSplitTensor(const AnfNodePtr& node, const FuncGraphManagerPtr& manager) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(manager); + AnfNodeIndexSet node_set = manager->node_users()[node]; + for (auto& node_pair : node_set) { + CNodePtr use_cnode = node_pair.first->cast(); + if (use_cnode == nullptr || !IsValueNode(use_cnode->input(0))) { + continue; + } + ValueNodePtr prim_anf_node = use_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_anf_node); + PrimitivePtr use_cnode_prim = prim_anf_node->value()->cast(); + MS_EXCEPTION_IF_NULL(use_cnode_prim); + if (use_cnode_prim->name() == DEPEND && node_pair.second != 1) { + continue; + } + if (IsParallelCareNode(use_cnode)) { + SplitTensor(node, use_cnode, node_pair.second); + } else { + StepSplitTensor(use_cnode, manager); + } + } +} + +std::vector ReplaceOpInput(const Operator& replace_op, const std::string& instance_name, + const CNodePtr& node) { + OperatorArgs arg_replace_op = replace_op.second; + ValuePtr pyop_instance = CreatOpInstance(arg_replace_op.first, replace_op.first, instance_name); + if (pyop_instance == nullptr) { + MS_LOG(EXCEPTION) << "Failure: " << replace_op.first << " CreatOpInstance failed"; + } + OperatorParams params = arg_replace_op.second; + if (node->inputs().size() < 2) { + // GetNext operator dose not has input + if (node->inputs().size() == 1) { + return {NewValueNode(pyop_instance)}; + } + MS_LOG(EXCEPTION) << "Failure: " << node->ToString() << " size is smaller than 2"; + } + std::vector replace_input = {NewValueNode(pyop_instance), node->input(1)}; + if (!params.empty()) { + Param param_first = *(params.begin()); + int32_t first_position = param_first.second; + if (first_position == 1) { + replace_input.pop_back(); + } + for (auto& param : params) { + AnfNodePtr val = NewValueNode(param.first.second); + if (val == nullptr) { + MS_LOG(EXCEPTION) << "Failure:val is nullptr"; + } + int32_t position = param.second; + (void)replace_input.insert(replace_input.begin() + position, val); + } + } + + return replace_input; +} + +void StepReplaceOp(OperatorVector replace_op, const CNodePtr& node) { + // step1:get graph manager distribute_operator + OperatorInfoPtr distribute_operator = node->operator_info(); + if (distribute_operator == nullptr) { + MS_LOG(EXCEPTION) << "Failure:AddNode error since distribute_operator is nullptr"; + } + FuncGraphPtr func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + if (manager == nullptr) { + MS_LOG(EXCEPTION) << "Failure:AddNode error since manager is nullptr"; + } + // step2:traverse op_list and insert node + std::reverse(replace_op.begin(), replace_op.end()); + auto replace_op_info = distribute_operator->replace_op_info(); + std::reverse(replace_op_info.begin(), replace_op_info.end()); + if (!replace_op_info.empty() && replace_op_info.size() != replace_op.size()) { + MS_LOG(EXCEPTION) << "replace_op_info is not empty and size not equal to replace_op!"; + } + bool replace_op_info_flag = !replace_op_info.empty(); + for (size_t index = 0; index < replace_op.size(); ++index) { + std::string instance_name = CreateInstanceName(node, index); + std::vector replace_input; + if (index != replace_op.size() - 1) { + replace_input = CreateInput(replace_op[index], node, instance_name); + } else { + replace_input = ReplaceOpInput(replace_op[index], instance_name, node); + } + CNodePtr replace_node = func_graph->NewCNode(replace_input); + MS_EXCEPTION_IF_NULL(replace_node); + ScopePtr scope = node->scope(); + MS_EXCEPTION_IF_NULL(scope); + replace_node->set_scope(scope); + if (index == replace_op.size() - 1) { + (void)replace_node->set_operator_info(node->operator_info()); + } + replace_node->set_in_forward_flag(true); + replace_input[0]->set_scope(scope); + if (replace_op_info_flag && replace_op_info[index].first) { + auto new_cnode = InsertMakeTuple(replace_node, replace_op_info[index].second, func_graph); + (void)manager->Replace(node, new_cnode); // using Replace function to insert node + } else { + (void)manager->Replace(node, replace_node); // using Replace function to insert node + } + } + MS_LOG(INFO) << "Insert ReplaceOp success for " << distribute_operator->name(); +} + +bool IsSomePrimitive(const CNodePtr& cnode, const std::string& name) { + ValueNodePtr anf_node = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(anf_node); + PrimitivePtr prim = anf_node->value()->cast(); + return (prim->name() == name); +} + +void StepReplaceGraph(const std::shared_ptr, AnfNodePtr>>& replace_graph, + const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(replace_graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(replace_graph->second); + FuncGraphPtr func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + if (manager == nullptr) { + MS_LOG(EXCEPTION) << "Failure:AddNode error since manager is nullptr"; + } + if (!IsSomePrimitive(node, ONEHOT)) { + MS_LOG(EXCEPTION) << "Failure:Only OneHot Primitive will enter StepReplaceGraph!"; + } + if (node->inputs().size() != 5) { + MS_LOG(EXCEPTION) << "Failure:There is 5 inputs for the CNode corresponding to OneHot Primitive!"; + } + auto pre_node = node->input(1); + if (replace_graph->first.size() != 2) { + MS_LOG(EXCEPTION) << "Failure:replace_graph->first.size() must be 2 for OneHot Primitive!"; + } + for (auto& replace_input : replace_graph->first) { + MS_EXCEPTION_IF_NULL(replace_input); + manager->SetEdge(replace_input, 1, pre_node); + CNodePtr replace_input_cnode = replace_input->cast(); + MS_EXCEPTION_IF_NULL(replace_input_cnode); + (void)replace_input_cnode->set_operator_info(node->operator_info()); + replace_input_cnode->set_in_forward_flag(true); // mark this new cnode is forward node + } + // "(void)manager->Replace(replace_graph->first, pre_node);" can not be called + auto replace_output = replace_graph->second; + MS_EXCEPTION_IF_NULL(replace_output); + (void)manager->Replace(node, replace_output); + CNodePtr replace_output_cnode = replace_graph->second->cast(); + MS_EXCEPTION_IF_NULL(replace_output_cnode); + (void)replace_output_cnode->set_operator_info(node->operator_info()); + replace_output_cnode->set_in_forward_flag(true); // mark this new cnode is forward node +} + +int32_t GetTupleGetItemIndex(const CNodePtr& cnode) { + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() != 3) { + MS_LOG(EXCEPTION) << cnode->ToString() << " size( " << cnode->inputs().size() << " ) is not 3"; + } + + if (!cnode->input(2)->isa()) { + MS_LOG(EXCEPTION) << "The index of tuple getitem is not a value node"; + } + + ValuePtr tuple_index_value = GetValueNode(cnode->input(2)); + MS_EXCEPTION_IF_NULL(tuple_index_value); + if (!tuple_index_value->isa()) { + MS_LOG(EXCEPTION) << "The index of tuple getitem is not int32"; + } + return tuple_index_value->cast()->value(); +} + +// Judge whether the node is a loss, and if there are multiple outputs, +// get which output is a grad according to the tuple getitem. +// Currently, it is not supported that the sens is a tuple. +LossNodeInfo GetLossNodeInfo(const AnfNodePtr& loss_node) { + MS_EXCEPTION_IF_NULL(loss_node); + FuncGraphPtr sub_graph = loss_node->func_graph(); + MS_EXCEPTION_IF_NULL(sub_graph); + CNodePtr return_node = sub_graph->get_return(); + MS_EXCEPTION_IF_NULL(return_node); + if (return_node->inputs().size() < 2) { + MS_LOG(EXCEPTION) << "Failure: " << return_node->ToString() << " size is smaller than 2"; + } + AnfNodePtr pre_node = return_node->input(1); + MS_EXCEPTION_IF_NULL(pre_node); + + LossNodeInfo node_info; + + // return -> loss + if (pre_node == loss_node) { + node_info.has_tuple_getitem = false; + node_info.dout_index = 0; + return node_info; + } + + // return -> tuple_getitem -> loss + auto cnode = pre_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto current_value = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(current_value); + PrimitivePtr current_prim = current_value->value()->cast(); + MS_EXCEPTION_IF_NULL(current_prim); + // size of common cnode is larger than 1 + if (cnode->inputs().size() < 2) { + MS_LOG(EXCEPTION) << cnode->ToString() << " size( " << cnode->inputs().size() << " ) is smaller than 2"; + } + + if ((current_prim->name() == TUPLE_GETITEM) && (cnode->input(1) == loss_node)) { + // size of tuple_getitem cnode is 3 + auto tuple_index = GetTupleGetItemIndex(cnode); + node_info.has_tuple_getitem = true; + node_info.dout_index = tuple_index; + return node_info; + } + + MS_LOG(EXCEPTION) << "Invalid loss"; +} + +void InsertVirtualDivOp(const VirtualDivOp& virtual_div_op, const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + size_t node_size = node->inputs().size(); + FuncGraphPtr func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + + for (size_t index = 1; index < node_size; ++index) { + AnfNodePtr input = node->input(index); + MS_EXCEPTION_IF_NULL(input); + if (!input->isa() && !input->isa()) { // if it is not a tensor, continue + MS_LOG(INFO) << "insert div op: the index " << index << " is not tensor, skip"; + continue; + } + + for (size_t pos = 0; pos < virtual_div_op.size(); ++pos) { + std::string instance_name = CreateInstanceName(node, pos); + InsertNode(virtual_div_op[pos], node, index, node->input(index), func_graph, instance_name); + } + MS_LOG(INFO) << "insert div op for input index " << index << " of node"; + } +} + +std::pair FindParameter(const AnfNodePtr& node, const FuncGraphPtr& func_graph) { + if (!node->isa() && !node->isa() && !node->isa()) { + return std::make_pair(nullptr, false); + } else if (node->isa()) { + return std::make_pair(node, false); + } else if (node->isa()) { + if (IsValueNode(node)) { + std::vector param_v = FindParameterByRefKeyNode(node, func_graph); + if (param_v.size() != 1) { + MS_LOG(EXCEPTION) << "FindParameterByRefKeyNode failed, return vector size must be 1, real is " + << param_v.size(); + } + return std::make_pair(node, true); + } + return std::make_pair(nullptr, false); + } else { + CNodePtr cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + for (size_t index = 0; index < cnode->inputs().size(); ++index) { + if (!FindParameter(cnode->input(index), func_graph).first) { + continue; + } + return FindParameter(cnode->input(index), func_graph); + } + } else { + if (IsParallelCareNode(cnode)) { + return std::make_pair(nullptr, false); + } else { + ValueNodePtr prim_anf_node = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_anf_node); + for (size_t index = 0; index < cnode->inputs().size(); ++index) { + PrimitivePtr prim = prim_anf_node->value()->cast(); + MS_EXCEPTION_IF_NULL(prim); + if (prim->name() == DEPEND && index != 1) { + continue; + } + if (!FindParameter(cnode->input(index), func_graph).first) { + continue; + } + return FindParameter(cnode->input(index), func_graph); + } + } + } + } + return std::make_pair(nullptr, false); +} + +std::pair FindCNode(const AnfNodePtr& anode, const std::string& name, const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(anode); + MS_EXCEPTION_IF_NULL(anode->func_graph()); + FuncGraphManagerPtr manager = anode->func_graph()->manager(); + MS_EXCEPTION_IF_NULL(manager); + AnfNodeIndexSet node_set = manager->node_users()[anode]; + bool result = false; + CNodePtr cnode_return = nullptr; + for (auto& node_pair : node_set) { + CNodePtr use_apply = node_pair.first->cast(); + if (use_apply == nullptr || !IsValueNode(use_apply->input(0))) { + continue; + } + ValueNodePtr prim_anf_node = use_apply->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_anf_node); + PrimitivePtr node_prim = prim_anf_node->value()->cast(); + MS_EXCEPTION_IF_NULL(node_prim); + if (node_prim->name() == name && node_pair.second == 1) { + if (use_apply->func_graph() == func_graph) { + result = true; + cnode_return = use_apply; + MS_LOG(INFO) << "Find Primitive " << name << " in the same func_graph"; + continue; + } + MS_LOG(INFO) << "Find Primitive " << name << " in different func_graph"; + } + } + return std::make_pair(result, cnode_return); +} + +bool IsCastBeforMirror(const CNodePtr& node, size_t index) { + // only if cast_before_mirror is true, pre node is cast and type is not float32 return true + if (!ParallelContext::GetInstance()->cast_before_mirror()) { + return false; + } + auto pre_node = node->input(index); + MS_EXCEPTION_IF_NULL(pre_node); + auto cnode = pre_node->cast(); + if (cnode == nullptr || !IsValueNode(cnode->input(0))) { + return false; + } + auto pre_value_node = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(pre_value_node); + auto pre_prim = pre_value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(pre_prim); + if (pre_prim->name() != CAST) { + return false; + } + auto node_type = pre_node->Type(); + MS_EXCEPTION_IF_NULL(node_type); + if (!node_type->isa()) { + MS_LOG(EXCEPTION) << "Unknown type."; + } + auto input_element_type = node_type->cast()->element(); + MS_EXCEPTION_IF_NULL(input_element_type); + auto type_id = input_element_type->type_id(); + + return (type_id != kNumberTypeFloat32); +} + +void InsertMirrorOps(const MirrorOps& mirror_ops, const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + size_t node_size = node->inputs().size(); + FuncGraphPtr func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (mirror_ops.size() != node_size - 1) { + MS_LOG(EXCEPTION) << "Failure:Mirrorops's size is wrong! mirror_ops size is " << mirror_ops.size() + << ", node_size is " << node_size; + } + for (size_t index = 1; index < node_size; ++index) { + OperatorVector backward_op = mirror_ops[index - 1]; + if (backward_op.empty()) { + continue; + } + std::pair param_node_pair = FindParameter(node->input(index), func_graph); + if (!param_node_pair.first) { + continue; + } + // not a RefKey + if (!param_node_pair.second) { + auto next_cnode = FindCNode(param_node_pair.first, MIRROR_OPERATOR, func_graph); + // if there is already a MirrorOp in the same graph, use MirrorOp CNode as a input instead + if (next_cnode.first) { + MS_EXCEPTION_IF_NULL(next_cnode.second); + manager->SetEdge(node, SizeToInt(index), next_cnode.second); + continue; + } + } + // if the parameter found is a RefKey, or no MirrorOp is found in the same graph, insert a new MirrorOp + // only one MirrorOp in backward_op + if (backward_op.size() != 1) { + MS_LOG(EXCEPTION) << "backward_op size must be 1, real is " << backward_op.size(); + } + std::string instance_name = MIRROR_OP; + if (IsCastBeforMirror(node, index)) { + for (auto& op : backward_op) { + // insert new node before the node + CNodePtr cnode = node->input(index)->cast(); + MS_EXCEPTION_IF_NULL(cnode); + AnfNodePtr pre_node = cnode->input(1); + InsertNode(op, cnode, size_t(1), pre_node, func_graph, instance_name); + } + } else { + for (auto& op : backward_op) { + AnfNodePtr pre_node = node->input(index); + InsertNode(op, node, index, pre_node, func_graph, instance_name); + } + } + } +} + +void BackwardCommunication(const OperatorInfoPtr& distribute_operator, const CNodePtr& node, bool is_loss_node) { + MS_EXCEPTION_IF_NULL(distribute_operator); + MS_EXCEPTION_IF_NULL(node); + MirrorOps mirror_ops = distribute_operator->mirror_ops(); + VirtualDivOp virtual_div_op = distribute_operator->virtual_div_op(); + // insert mirror op + if (!mirror_ops.empty()) { + MS_LOG(INFO) << "insert mirror op for " << distribute_operator->name(); + InsertMirrorOps(mirror_ops, node); + } + // insert virtual div op + if (!virtual_div_op.empty() && is_loss_node) { + MS_LOG(INFO) << "insert virtual div op for " << distribute_operator->name(); + InsertVirtualDivOp(virtual_div_op, node); + } +} + +std::string GetDisOpName(const std::string& prim_name) { + std::string op_name = prim_name; + if (!prim_name.empty() && (prim_name[0] == '_')) { + op_name = prim_name.substr(1); + } + return op_name + "Info"; +} + +OperatorInfoPtr OperatorInstanceByName(const std::string& name, const PrimitiveAttrs& attrs, + const std::vector& shape_list) { + if (shape_list.size() != 2) { + MS_LOG(ERROR) << "The size of shape list is not 2"; + return nullptr; + } + if (name.length() == 0) { + MS_LOG(EXCEPTION) << "Length of name is zero!"; + } + std::string distribute_opname = GetDisOpName(name); + OperatorInfoPtr operator_ = + (OperatorInfoPtr)DynCreator::Instance().Creat(distribute_opname, shape_list[0], shape_list[1], attrs, TOTAL_OPS); + if (operator_ == nullptr) { + MS_LOG(INFO) << "Creat " << name << " failed"; + return nullptr; + } + std::string origin_name = operator_->name(); + operator_->set_name(origin_name + std::to_string(TOTAL_OPS)); + MS_LOG(INFO) << "Successfully created operator " << origin_name; + ++TOTAL_OPS; + return operator_; +} + +OperatorInfoPtr OperatorInstance(const PrimitivePtr& prim, const PrimitiveAttrs& attrs, + const std::vector& shape_list) { + MS_EXCEPTION_IF_NULL(prim); + OperatorInfoPtr operator_ = OperatorInstanceByName(prim->name(), attrs, shape_list); + if (operator_ == nullptr) { + MS_LOG(INFO) << "Creat " << prim->name() << " failed, use batch parallel"; + operator_ = OperatorInstanceByName(BATCH_PARALLEL, attrs, shape_list); + MS_EXCEPTION_IF_NULL(operator_); + } + return operator_; +} + +OperatorInfoPtr NewOperatorInstance(const PrimitivePtr& prim, const PrimitiveAttrs& attrs, + std::vector shape_list) { + OperatorInfoPtr operator_ = OperatorInstance(prim, attrs, shape_list); + for (size_t i = 0; i < shape_list[0].size(); ++i) { + MS_LOG(INFO) << "No: " << i << " input's shape: " << ShapeToString(shape_list[0][i]); + } + return operator_; +} + +StrategyPtr ExtractStrategy(std::unordered_map attrs) { + ValueTuplePtr var = attrs[STRATEGY]->cast(); + StrategyPtr strategyPtr; + MS_LOG(INFO) << "Extract information: strategy " << attrs[STRATEGY]->ToString(); + if (var == nullptr) { + MS_LOG(EXCEPTION) << "Strategy value is nullptr"; + } + if (var->size() > 0) { + std::vector elements = var->value(); + std::vector strategy; + for (uint32_t index = 0; index < elements.size(); ++index) { + Dimensions dim; + if (elements[index]->isa()) { + ValueTuplePtr value_tuple = elements[index]->cast(); + if (value_tuple == nullptr) { + MS_LOG(EXCEPTION) << "Failure:value_tuple is nullptr"; + } + + std::vector value_vector = value_tuple->value(); + (void)std::transform(value_vector.begin(), value_vector.end(), std::back_inserter(dim), + [](const ValuePtr& value) { return static_cast(GetValue(value)); }); + strategy.push_back(dim); + } else { + MS_LOG(EXCEPTION) << "Failure:Strategy's format is wrong! Need ValueSequeue"; + } + } + if (strategy.empty()) { + MS_LOG(EXCEPTION) << "ExtractStrategy:failed to extract strategy"; + } + strategyPtr = NewStrategy(0, strategy); + } + + return strategyPtr; +} + +Shapes GetNodeShape(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + Shapes shapes; + BaseShapePtr base_shape_ptr = node->Shape(); + if (node->isa()) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (IsValueNode(cnode->input(0))) { + PrimitivePtr prim = GetValueNode(cnode->input(0)); + MS_EXCEPTION_IF_NULL(prim); + if (prim->name() == MAKEREF) { + AnfNodePtr ref_node = cnode->input(1); + auto func_graph = cnode->func_graph(); + MS_EXCEPTION_IF_NULL(ref_node); + MS_EXCEPTION_IF_NULL(func_graph); + return GetRefKeyNodeShape(ref_node, func_graph); + } + } + if (cnode->input(0)->isa()) { + if (cnode->inputs().size() < 2) { + MS_LOG(EXCEPTION) << "GetNodeShape: " << node->ToString() << " size is samller than 2"; + } + base_shape_ptr = cnode->input(1)->Shape(); + } + } + if (base_shape_ptr == nullptr) { + MS_LOG(EXCEPTION) << "GetNodeShape: " << node->ToString() << " shape_ptr is nullptr, full name is " + << node->fullname_with_scope(); + } + auto tuple_shape_ptr = dyn_cast(base_shape_ptr); + if (tuple_shape_ptr != nullptr) { + auto tuple_shape = tuple_shape_ptr->shape(); + for (auto& shape : tuple_shape) { + auto each_shape = dyn_cast(shape); + MS_EXCEPTION_IF_NULL(each_shape); + shapes.push_back(each_shape->shape()); + } + } else { + auto shape_ptr = dyn_cast(base_shape_ptr); + MS_EXCEPTION_IF_NULL(shape_ptr); + shapes.push_back(shape_ptr->shape()); + } + return shapes; +} + +std::vector FindParameterByRefKeyNode(const AnfNodePtr& node, const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(func_graph); + std::vector parameters; + if (!IsValueNode(node)) { + MS_LOG(ERROR) << "The node is not a ref key"; + return parameters; + } + + auto ref_key = GetValueNode(node); + MS_EXCEPTION_IF_NULL(ref_key); + auto name = ref_key->tag(); + + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto roots = manager->roots(); + if (roots.size() != 1) { + MS_LOG(ERROR) << "The size of roots ( " << roots.size() << " ) is not 1"; + return parameters; + } + + FuncGraphPtr root_g = roots.back(); + MS_EXCEPTION_IF_NULL(root_g); + for (auto& param_node : root_g->parameters()) { + auto param = param_node->cast(); + if (param && (name == param->name())) { + parameters.push_back(param_node); + MS_LOG(INFO) << "The name of ref key is: " << name; + return parameters; + } + } + + MS_LOG(ERROR) << "The name of ref key is: " << name << ", but have not found the parameter"; + return parameters; +} + +Shapes GetRefKeyNodeShape(const AnfNodePtr& node, const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(func_graph); + + std::vector parameters = FindParameterByRefKeyNode(node, func_graph); + if (parameters.size() != 1) { + MS_LOG(EXCEPTION) << "Find parameter by ref key node failed"; + } + + Shapes input_shapes; + input_shapes = GetNodeShape(parameters[0]); + if (input_shapes.size() != 1) { + MS_LOG(EXCEPTION) << "Get input shape failed"; + } + + MS_LOG(INFO) << "The parameter shape is " << ShapeToString(input_shapes[0]); + return input_shapes; +} + +std::vector ExtractShape(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + Shapes shape_inputs, shape_outputs; + std::vector shape_all; + std::vector all_inputs = node->inputs(); + std::vector node_inputs{all_inputs.begin() + 1, all_inputs.end()}; + + for (auto& input : node_inputs) { + Shapes input_shapes; + if (IsValueNode(input)) { + auto func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + input_shapes = GetRefKeyNodeShape(input, func_graph); + } else if (IsValueNode(input) || input->isa() || input->isa()) { + input_shapes = GetNodeShape(input); + } else { + continue; + } + if (input_shapes.size() != 1) { + MS_LOG(EXCEPTION) << "ExtractShape:Get input shape failed"; + } + shape_inputs.push_back(input_shapes[0]); + } + shape_all.push_back(shape_inputs); + // extract out shape + shape_outputs = GetNodeShape(node); + shape_all.push_back(shape_outputs); + return shape_all; +} + +std::pair FindParallelCareNode(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + FuncGraphPtr func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + AnfNodeIndexSet node_set = manager->node_users()[node]; + for (auto& node_pair : node_set) { + CNodePtr cnode = node_pair.first->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + ValueNodePtr prim_node_anf = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_node_anf); + PrimitivePtr node_prim = prim_node_anf->value()->cast(); + MS_EXCEPTION_IF_NULL(node_prim); + if (node_prim->name() == DEPEND && node_pair.second != 1) { + continue; + } + if (IsParallelCareNode(cnode) && cnode->operator_info() != nullptr) { + return node_pair; + } else if (FindParallelCareNode(node_pair.first).first != nullptr) { + return FindParallelCareNode(node_pair.first); + } + } + return std::make_pair(nullptr, 0); +} + +std::pair FindSubGraph(const FuncGraphPtr& graph, const AnfNodePtr& parameter) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(parameter); + FuncGraphManagerPtr manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + std::pair prim_anf_node_pair = FindParallelCareNode(parameter); + if (prim_anf_node_pair.first != nullptr) { + return prim_anf_node_pair; + } else { + AnfNodeIndexSet param_sub_set = manager->node_users()[parameter]; + for (auto& param_pair : param_sub_set) { + CNodePtr graph_cnode = param_pair.first->cast(); + if ((graph_cnode == nullptr) || !graph_cnode->input(0)->isa()) { + continue; + } + CNodePtr graph_cnode_inp0 = graph_cnode->input(0)->cast(); + if ((graph_cnode_inp0 == nullptr) || !IsValueNode(graph_cnode_inp0->input(1))) { + continue; + } + FuncGraphPtr graph_sub = GetValueNode(graph_cnode_inp0->input(1)); + auto parameters = graph_sub->parameters(); + if (IntToSize(param_pair.second - 1) >= parameters.size()) { + MS_LOG(EXCEPTION) << "The index is out of range, index is " << param_pair.second - 1 << ", vector size is " + << parameters.size(); + } + std::pair res = FindSubGraph(graph_sub, parameters[IntToSize(param_pair.second - 1)]); + if (res.first != nullptr) { + return res; + } + } + } + return std::make_pair(nullptr, 0); +} + +void SetParallelShape(const AnfNodePtr& parameter, const std::pair& res) { + MS_EXCEPTION_IF_NULL(parameter); + AbstractBasePtr abstract = parameter->abstract(); + MS_EXCEPTION_IF_NULL(abstract); + MS_LOG(DEBUG) << "SetParallelShape " << parameter->ToString() << " shape " << parameter->Shape()->ToString(); + CNodePtr cnode = res.first->cast(); + MS_EXCEPTION_IF_NULL(cnode); + OperatorInfoPtr distribute_operator = cnode->operator_info(); + if (distribute_operator == nullptr) { + MS_LOG(EXCEPTION) << "Failure:node " << cnode->ToString() << " 's OperatorInfoPtr is nullptr"; + } + + if (IntToSize(res.second - 1) >= distribute_operator->inputs_tensor_info().size()) { + MS_LOG(EXCEPTION) << "The index is out of range, index is " << res.second - 1 << ", vector size is " + << distribute_operator->inputs_tensor_info().size(); + } + TensorInfo tensorinfo_in = distribute_operator->inputs_tensor_info()[IntToSize(res.second - 1)]; + Shape slice_shape = tensorinfo_in.slice_shape(); + MS_LOG(DEBUG) << "SetParallelShape slice_shape " << parameter->ToString() << " shape " + << MakeValue(slice_shape)->ToString(); + std::shared_ptr parallel_shape = std::make_shared(slice_shape); + MS_EXCEPTION_IF_NULL(parallel_shape); + abstract->set_shape(parallel_shape); + TensorLayout tensor_layout = tensorinfo_in.tensor_layout(); + ParameterPtr parameter_ptr = parameter->cast(); + MS_EXCEPTION_IF_NULL(parameter_ptr); + parameter_ptr->set_tensor_layout(std::make_shared(tensor_layout)); +} + +void CoverSliceShape(const FuncGraphPtr& root) { + MS_EXCEPTION_IF_NULL(root); + auto parameters = root->parameters(); + for (auto& parameter : parameters) { + MS_EXCEPTION_IF_NULL(parameter->Shape()); + std::pair res = FindSubGraph(root, parameter); + if (res.first == nullptr) { + MS_LOG(INFO) << "Parameter " << parameter->ToString() << " don't need to set parallel shape"; + } else { + SetParallelShape(parameter, res); + MS_LOG(DEBUG) << "parameter " << parameter->ToString() << " shape " << parameter->Shape()->ToString(); + } + } +} + +bool ParameterIsCloned(const FuncGraphPtr& root, const AnfNodePtr& parameter_node) { + MS_EXCEPTION_IF_NULL(root); + MS_EXCEPTION_IF_NULL(parameter_node); + FuncGraphManagerPtr manager = root->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto cloned_parameter = parameter_node->cast(); + MS_EXCEPTION_IF_NULL(cloned_parameter); + + // find the clone parameter + if (!cloned_parameter->has_default()) { + return false; + } + + py::object clone_info = parse::python_adapter::GetPyObjAttr(cloned_parameter->default_param(), CLONE_INFO); + bool cloned = py::cast(parse::python_adapter::GetPyObjAttr(clone_info, CLONED)); + if (!cloned) { + return false; + } + + MS_LOG(INFO) << "The parameter: " << cloned_parameter->name() << " is cloned"; + return true; +} + +void SetClonedTensorShapeForOptimizer(const FuncGraphPtr& root) { + MS_EXCEPTION_IF_NULL(root); + for (auto& cloned_parameter_node : root->parameters()) { + MS_EXCEPTION_IF_NULL(cloned_parameter_node); + auto cloned_parameter = cloned_parameter_node->cast(); + MS_EXCEPTION_IF_NULL(cloned_parameter); + + if (!ParameterIsCloned(root, cloned_parameter_node)) { + continue; + } + + // get the cloned index + py::object cloned_info = parse::python_adapter::GetPyObjAttr(cloned_parameter->default_param(), CLONE_INFO); + int32_t cloned_index = py::cast(parse::python_adapter::GetPyObjAttr(cloned_info, CLONED_INDEX)); + + // find the be cloned parameter + bool found_be_cloned_parameter = false; + ParameterPtr cloned_from_parameter = nullptr; + AnfNodePtr cloned_from_node = nullptr; + for (auto& be_cloned_parameter_node : root->parameters()) { + MS_EXCEPTION_IF_NULL(be_cloned_parameter_node); + auto be_cloned_parameter = be_cloned_parameter_node->cast(); + MS_EXCEPTION_IF_NULL(be_cloned_parameter); + if (!be_cloned_parameter->has_default()) { + continue; + } + + py::object be_cloned_info = parse::python_adapter::GetPyObjAttr(be_cloned_parameter->default_param(), CLONE_INFO); + if (!py::cast(parse::python_adapter::GetPyObjAttr(be_cloned_info, BE_CLONED))) { + continue; + } + + // get the be cloned index + py::list be_cloned_index = parse::python_adapter::GetPyObjAttr(be_cloned_info, BE_CLONED_INDEX); + for (auto& index : be_cloned_index) { + if (cloned_index == py::cast(index)) { + found_be_cloned_parameter = true; + cloned_from_parameter = be_cloned_parameter; + cloned_from_node = be_cloned_parameter_node; + break; + } + } + } + + if (found_be_cloned_parameter) { + // set the shape and tensor layout for cloned parameter + cloned_parameter->set_tensor_layout(cloned_from_parameter->tensor_layout()); + MS_EXCEPTION_IF_NULL(cloned_parameter_node->abstract()); + MS_EXCEPTION_IF_NULL(cloned_from_node->abstract()); + cloned_parameter_node->abstract()->set_shape(cloned_from_node->abstract()->GetShapeTrack()); + MS_LOG(INFO) << "The parameter: " << cloned_parameter->name() + << " is cloned, the be cloned parameter is: " << cloned_from_parameter->name() + << ", clone index is: " << cloned_index; + } else { + MS_LOG(EXCEPTION) << "The parameter: " << cloned_parameter->name() << " is cloned, cloned index is " + << cloned_index << ", but not found the be cloned parameter"; + } + } +} + +void SetVirtualDatasetStrategy(const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + PrimitivePtr prim = GetValueNode(node->input(0)); + MS_EXCEPTION_IF_NULL(prim); + if (prim->name() == VIRTUAL_DATA_SET) { + CheckGlobalDeviceManager(); + int32_t dev_num = SizeToInt(g_device_manager->GetDeviceListByStageId(0).size()); + auto attrs_temp = prim->attrs(); + std::vector shape_list = ExtractShape(node); + if (shape_list.empty()) { + MS_LOG(EXCEPTION) << "Failure:node " << node->ToString() << " failed to extract shape"; + } + std::vector elements; + for (size_t i = 0; i < shape_list[0].size(); i++) { + if (shape_list[0][i].empty()) { + MS_LOG(EXCEPTION) << "shape_list[ " << i << " ].size() is zero"; + } + std::vector input_strategy = {dev_num}; + for (size_t j = 1; j < shape_list[0][i].size(); j++) { + input_strategy.push_back(1); + } + elements.push_back(MakeValue(input_strategy)); + } + ValueTuplePtr strategy = std::make_shared(elements); + attrs_temp[STRATEGY] = strategy; + (void)prim->SetAttrs(attrs_temp); + } +} + +void ExtractInformation(const std::vector& all_nodes) { + for (auto& node : all_nodes) { + auto cnode = node->cast(); + if ((cnode == nullptr) || !IsValueNode(cnode->input(0))) { + continue; + } + SetVirtualDatasetStrategy(cnode); + ValueNodePtr prim_anf_node = cnode->input(0)->cast(); + PrimitivePtr prim = GetValueNode(prim_anf_node); + auto attrs = prim->attrs(); + MS_LOG(INFO) << "extract information: node: " << node->ToString() << " prim " << prim->name(); + if (IsParallelCareNode(cnode)) { + std::vector shape_list = ExtractShape(cnode); + if (shape_list.empty()) { + MS_LOG(EXCEPTION) << "Failure:node " << node->ToString() << " failed to extract shape"; + } + OperatorInfoPtr operator_ = OperatorInstance(prim, attrs, shape_list); + if (operator_ == nullptr) { + MS_LOG(EXCEPTION) << "Failure:Primitive " << prim->name() << " OperatorInstance failed"; + } + auto& inputs = cnode->inputs(); + std::vector input_value; + for (size_t index = 1; index < inputs.size(); ++index) { + if (inputs[index]->isa()) { + input_value.push_back(GetValueNode(inputs[index])); + } else { + input_value.emplace_back(nullptr); + } + } + StrategyPtr strategyPtr = nullptr; + (*operator_).set_input_value(input_value); + (*operator_).set_outputs_dtype(cnode->Type()); + (*operator_).set_cnode(cnode); + if (prim->name() == RESHAPE) { + (void)cnode->set_operator_info(operator_); + continue; + } + if (!StrategyFound(attrs)) { + MS_LOG(INFO) << "ExtractInformation: the strategy of node " << node->ToString() << " prim " << prim->name() + << " is empty, using batch parallel"; + std::shared_ptr> strategy_v_ptr = operator_->GenerateBatchStrategies(); + if (strategy_v_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Failure:Generate batch parallel strategy failed"; + } + std::vector elements; + for (size_t i = 0; i < strategy_v_ptr->size(); i++) { + elements.push_back(MakeValue((*strategy_v_ptr)[i])); + } + ValueTuplePtr strategy = std::make_shared(elements); + // display the strategy generated by batch parallel + attrs[GEN_STRATEGY] = strategy; + (void)prim->SetAttrs(attrs); + MS_LOG(INFO) << "node " << node->ToString() << " prim " << prim->name() << " batch parallel strategy is " + << attrs[GEN_STRATEGY]->ToString(); + strategyPtr = NewStrategy(0, *strategy_v_ptr); + } else { + strategyPtr = ExtractStrategy(attrs); + } + if (strategyPtr != nullptr) { + if (operator_->Init(strategyPtr) == FAILED) { + MS_LOG(EXCEPTION) << "Failure:operator " << prim->name() << " init failed"; + } + (void)cnode->set_operator_info(operator_); + } else { + MS_LOG(EXCEPTION) << "ERROR:strategy_ptr is nullptr"; + } + } + } +} + +TensorLayout GetInputLayoutFromCNode(const std::pair& node_pair) { + CNodePtr cnode = node_pair.first->cast(); + MS_EXCEPTION_IF_NULL(cnode); + OperatorInfoPtr distribute_operator = GetDistributeOperator(cnode); + MS_EXCEPTION_IF_NULL(distribute_operator); + int index = node_pair.second; + if (index > SizeToInt(distribute_operator->inputs_tensor_info().size())) { + MS_LOG(EXCEPTION) << "The index is out of range, the node_pair.second is " << index - 1 << ", the vector size is " + << distribute_operator->inputs_tensor_info().size(); + } + TensorInfo tensorinfo_in = distribute_operator->inputs_tensor_info()[IntToSize(index - 1)]; + TensorLayout tensorlayout_in = tensorinfo_in.tensor_layout(); + return tensorlayout_in; +} + +// if reshape's output connect to several primitive, return the first layout found +std::shared_ptr FindNextLayout(const CNodePtr& cnode) { + MS_EXCEPTION_IF_NULL(cnode); + MS_EXCEPTION_IF_NULL(cnode->func_graph()); + FuncGraphManagerPtr manager = cnode->func_graph()->manager(); + MS_EXCEPTION_IF_NULL(manager); + AnfNodeIndexSet node_set = manager->node_users()[cnode]; + for (auto& node_pair : node_set) { + CNodePtr use_apply = node_pair.first->cast(); + if (use_apply == nullptr || !IsValueNode(use_apply->input(0))) { + continue; + } + ValueNodePtr prim_anf_node = use_apply->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_anf_node); + PrimitivePtr node_prim = prim_anf_node->value()->cast(); + MS_EXCEPTION_IF_NULL(node_prim); + MS_LOG(INFO) << "FindNextLayout prim " << node_prim->name(); + if (node_prim->name() == DEPEND && node_pair.second != 1) { + continue; + } + if (IsParallelCareNode(use_apply) && (use_apply->operator_info() != nullptr)) { + MS_LOG(INFO) << "FindNextLayout success prim " << node_prim->name(); + auto layout = GetInputLayoutFromCNode(node_pair); + return std::make_shared(layout); + } + MS_LOG(DEBUG) << "FindNextLayout failed prim " << node_prim->name() << " " << IsParallelCareNode(use_apply) + << " " << (use_apply->operator_info() != nullptr); + + auto layout_ptr = FindNextLayout(use_apply); + if (layout_ptr) { + return layout_ptr; + } + } + MS_LOG(WARNING) << "FindNextLayout return nullptr, if reshape is not the last primitive, there must be some error"; + return nullptr; +} + +std::shared_ptr GetOutputLayoutFromCNode(const CNodePtr& cnode, size_t output_index) { + MS_EXCEPTION_IF_NULL(cnode); + OperatorInfoPtr distribute_operator = GetDistributeOperator(cnode); + MS_EXCEPTION_IF_NULL(distribute_operator); + if (distribute_operator->outputs_tensor_info().size() < output_index) { + MS_LOG(EXCEPTION) << "outputs_tensor_info size is " << distribute_operator->inputs_tensor_info().size() + << ", must be less than output_index " << output_index; + } + TensorInfo tensorinfo_out = distribute_operator->outputs_tensor_info()[output_index]; + TensorLayout tensorlayout_out = tensorinfo_out.tensor_layout(); + return std::make_shared(tensorlayout_out); +} + +std::shared_ptr FindPrevParallelCareNodeLayout(const AnfNodePtr& node, size_t output_index) { + if (!node->isa()) { + return nullptr; + } + CNodePtr cnode = node->cast(); + if (!IsValueNode(cnode->input(0))) { + return nullptr; + } + if (IsParallelCareNode(cnode) && (cnode->operator_info() != nullptr)) { + auto layout_ptr = GetOutputLayoutFromCNode(cnode, output_index); + if (!layout_ptr) { + MS_LOG(EXCEPTION) << "Failure:GetLayoutFromCNode failed"; + } + return layout_ptr; + } + return nullptr; +} + +std::shared_ptr FindPrevLayout(const AnfNodePtr& node) { + if (node->isa()) { + MS_LOG(EXCEPTION) << "Failure: parameter before reshape is not supported temporary"; + } + if (!node->isa()) { + return nullptr; + } + CNodePtr cnode = node->cast(); + if (!IsValueNode(cnode->input(0))) { + return nullptr; + } + if (IsParallelCareNode(cnode) && (cnode->operator_info() != nullptr)) { + auto layout_ptr = GetOutputLayoutFromCNode(cnode, 0); + if (!layout_ptr) { + MS_LOG(EXCEPTION) << "Failure:GetLayoutFromCNode failed"; + } + return layout_ptr; + } + ValueNodePtr prim_anf_node = cnode->input(0)->cast(); + PrimitivePtr prim = prim_anf_node->value()->cast(); + if (prim->name() == TUPLE_GETITEM) { + auto tuple_index = GetTupleGetItemIndex(cnode); + auto layout_ptr = FindPrevParallelCareNodeLayout(cnode->input(1), IntToSize(tuple_index)); + if (!layout_ptr) { + MS_LOG(EXCEPTION) + << " Failure:FindPrevLayout failed, tuple_getitem before reshape, but there does not exit a parallel care node " + "before tuple_getitem!"; + } + return layout_ptr; + } + for (size_t index = 0; index < cnode->inputs().size(); ++index) { + if (prim->name() == DEPEND && index != 1) { + continue; + } + auto layout_ptr = FindPrevLayout(cnode->inputs()[index]); + if (!layout_ptr) { + continue; + } + return layout_ptr; + } + MS_LOG(WARNING) << "FindPrevLayout return nullptr, if reshape is not the first primitive, there must be some error"; + return nullptr; +} + +void ReshapeInit(const std::vector& all_nodes) { + for (auto& node : all_nodes) { + auto cnode = node->cast(); + if ((cnode == nullptr) || !IsValueNode(cnode->input(0))) { + continue; + } + ValueNodePtr prim_anf_node = cnode->input(0)->cast(); + if (!IsParallelCareNode(cnode) || (cnode->operator_info() == nullptr)) { + continue; + } + PrimitivePtr prim = GetValueNode(prim_anf_node); + MS_EXCEPTION_IF_NULL(prim); + OperatorInfoPtr operator_info = cnode->operator_info(); + if (operator_info == nullptr) { + MS_LOG(EXCEPTION) << "Failure:Primitive " << prim->ToString() << " OperatorInstance is nullptr"; + } + if (prim->name() != RESHAPE) { + continue; + } + auto attrs = prim->attrs(); + if (StrategyFound(attrs)) { + MS_LOG(EXCEPTION) << "Setting strategy for Reshape goes for nothing!"; + } + MS_ASSERT(cnode->inputs().size() == 3); + auto prev_layout_ptr = FindPrevLayout(cnode->input(1)); + if (prev_layout_ptr) { + auto reshape_info_ptr = std::dynamic_pointer_cast(operator_info); + reshape_info_ptr->SetInputLayout(*prev_layout_ptr); + } + auto next_layout_ptr = FindNextLayout(cnode); + if (next_layout_ptr) { + auto reshape_info_ptr = std::dynamic_pointer_cast(operator_info); + reshape_info_ptr->SetOutputLayout(*next_layout_ptr); + } + if (operator_info->Init(nullptr) == FAILED) { + MS_LOG(EXCEPTION) << "Failure:operator " << prim->ToString() << " init failed"; + } + } +} + +// Sens node satisfies the following conditions: cnode(sens)-->cnode(tuple_getitem)-->cnode-->cnode(J) +bool IsGradSensNode(const AnfNodePtr& node) { + if (!node->isa()) { + return false; + } + + // cnode(sens)-->cnode(tuple_getitem) + auto cnode = node->cast(); + AnfNodePtr expect_tuple_getitem = cnode->input(0); + MS_EXCEPTION_IF_NULL(expect_tuple_getitem); + if (!expect_tuple_getitem->isa()) { + return false; + } + auto expect_tuple_getitem_cnode = expect_tuple_getitem->cast(); + MS_EXCEPTION_IF_NULL(expect_tuple_getitem_cnode); + if (!IsValueNode(expect_tuple_getitem_cnode->input(0))) { + return false; + } + ValueNodePtr expect_tuple_getitem_value_node = expect_tuple_getitem_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(expect_tuple_getitem_value_node); + PrimitivePtr expect_tuple_getitem_prim = expect_tuple_getitem_value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(expect_tuple_getitem_prim); + if (expect_tuple_getitem_prim->name() != TUPLE_GETITEM) { + return false; + } + + // cnode(sens)-->cnode(tuple_getitem)-->cnode + AnfNodePtr expect_anonymous = expect_tuple_getitem_cnode->input(1); + MS_EXCEPTION_IF_NULL(expect_anonymous); + if (!expect_anonymous->isa()) { + return false; + } + + // cnode(sens)-->cnode(tuple_getitem)-->cnode-->cnode(J) + auto expect_anonymous_cnode = expect_anonymous->cast(); + MS_EXCEPTION_IF_NULL(expect_anonymous_cnode); + AnfNodePtr expect_j = expect_anonymous_cnode->input(0); + MS_EXCEPTION_IF_NULL(expect_j); + if (!expect_j->isa()) { + return false; + } + auto expect_j_cnode = expect_j->cast(); + MS_EXCEPTION_IF_NULL(expect_j_cnode); + if (!IsValueNode(expect_j_cnode->input(0))) { + return false; + } + ValueNodePtr expect_j_value_node = expect_j_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(expect_j_value_node); + PrimitivePtr expect_j_prim = expect_j_value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(expect_j_prim); + return (expect_j_prim->name() == J); +} + +TensorLayouts GetLossNodeGradOutputLayout(const CNodePtr& loss_cnode) { + MS_EXCEPTION_IF_NULL(loss_cnode); + AnfNodePtr node = loss_cnode->cast(); + MS_EXCEPTION_IF_NULL(node); + + LossNodeInfo node_info = GetLossNodeInfo(node); + + ValueNodePtr prim_anf_node = loss_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(prim_anf_node); + PrimitivePtr prim = prim_anf_node->value()->cast(); + MS_EXCEPTION_IF_NULL(prim); + + TensorLayouts ret; + if (INVALID_LOSS_OPS.find(prim->name()) != INVALID_LOSS_OPS.end()) { + MS_LOG(WARNING) << "The loss name is: " << prim->name() << ", do nothing for split sens now"; + return ret; + } + + OperatorInfoPtr operator_info = loss_cnode->operator_info(); + MS_EXCEPTION_IF_NULL(operator_info); + + TensorInfo loss_grad_tensor_info; + size_t op_output_size = operator_info->outputs_tensor_info().size(); + MS_LOG(INFO) << "The loss name is " << operator_info->name() << ", the has tuple item is " + << node_info.has_tuple_getitem << ", the output size is " << op_output_size << ", the dout_index is " + << node_info.dout_index; + + if ((op_output_size == 0) || (op_output_size <= IntToSize(node_info.dout_index))) { + MS_LOG(EXCEPTION) << "The index is " << node_info.dout_index << ", but the size of outputs is " << op_output_size; + } + + if (!node_info.has_tuple_getitem && (op_output_size > 1)) { + MS_LOG(EXCEPTION) << "Currently, it is not supported that the sens is a tuple."; + } + + loss_grad_tensor_info = operator_info->outputs_tensor_info()[IntToSize(node_info.dout_index)]; + ret.push_back(loss_grad_tensor_info.tensor_layout()); + return ret; +} + +void SplitSens(const AnfNodePtr& grad_sens_node, const TensorLayout& loss_grad_layout) { + MS_EXCEPTION_IF_NULL(grad_sens_node); + + auto cnode = grad_sens_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + AnfNodePtr sens_tensor_node = cnode->input(1); + MS_EXCEPTION_IF_NULL(sens_tensor_node); + Shapes sens_shapes = GetNodeShape(sens_tensor_node); + if (sens_shapes.size() != 1) { + MS_LOG(EXCEPTION) << "SplitSens: GetNodeShape for sens_tensor_node, output size is not 1"; + } + // If the shape of sens tensor is [] or [1], no need to split it. + Shape sens_shape = sens_shapes[0]; + if (sens_shape.empty() || ((sens_shape.size() == 1) && (sens_shape[0] == 1))) { + if (sens_tensor_node->isa()) { + auto sens_tensor_param = sens_tensor_node->cast(); + MS_LOG(DEBUG) << "loss layout " << loss_grad_layout.ToString(); + sens_tensor_param->set_tensor_layout(std::make_shared(loss_grad_layout)); + } + MS_LOG(INFO) << "The shape of sens is " << ShapeToString(sens_shape) << ", no need to split sens"; + return; + } + auto loss_shape = loss_grad_layout.tensor_shape().array(); + if (loss_shape != sens_shape) { + MS_LOG(EXCEPTION) << "The shape of sens is not equal to loss output, it is unsupported now. Sens shape is " + << ShapeToString(sens_shape) << ", loss shape is " << ShapeToString(loss_shape); + } + MS_LOG(INFO) << "The shape of sens is " << ShapeToString(sens_shape) << ", split it."; + + if (!IsValueNode(sens_tensor_node)) { + if (sens_tensor_node->isa()) { + MS_LOG(DEBUG) << "loss layout " << loss_grad_layout.ToString(); + AbstractBasePtr abstract = sens_tensor_node->abstract(); + MS_EXCEPTION_IF_NULL(abstract); + auto slice_shape = loss_grad_layout.slice_shape().array(); + std::shared_ptr parallel_shape = std::make_shared(slice_shape); + MS_EXCEPTION_IF_NULL(parallel_shape); + abstract->set_shape(parallel_shape); + auto sens_tensor_param = sens_tensor_node->cast(); + sens_tensor_param->set_tensor_layout(std::make_shared(loss_grad_layout)); + return; + } + MS_LOG(EXCEPTION) << "SplitSens: the type of sens node is not Tensor or Parameter, it is unsupported now."; + } + + // Use _GetTensorSlice operator to split the sens tensor + FuncGraphPtr func_graph = cnode->func_graph(); // only cnode can get the graph + MS_EXCEPTION_IF_NULL(func_graph); + Operator op = CreateGetTensorSliceOp(loss_grad_layout); + InsertGetTensorSliceOp(op, cnode, func_graph, 1, SPLIT_SENS); +} + +void InsertForwardOps(const OperatorInfoPtr& distribute_operator, const CNodePtr& cnode) { + MS_EXCEPTION_IF_NULL(distribute_operator); + MS_EXCEPTION_IF_NULL(cnode); + OperatorVector forward_op = distribute_operator->forward_op(); + if (!forward_op.empty()) { + MS_LOG(INFO) << "Insert forward op for " << distribute_operator->name(); + ForwardCommunication(forward_op, cnode); + } +} + +void StepReplace(const OperatorInfoPtr& distribute_operator, const CNodePtr& cnode) { + MS_EXCEPTION_IF_NULL(distribute_operator); + MS_EXCEPTION_IF_NULL(cnode); + // StepReplaceOp + OperatorVector replace_op = distribute_operator->replace_op(); + if (!replace_op.empty()) { + MS_LOG(INFO) << "StepReplaceOp " << cnode->ToString(); + StepReplaceOp(replace_op, cnode); + } + + // StepReplaceGraph: after calling StepReplaceGraph, cnode can not be used anymore. + ReplaceGraphPtr replace_graph = distribute_operator->replace_graph(cnode); + if (!replace_op.empty() && replace_graph) { + MS_LOG(EXCEPTION) << "Only one of replace_op or replace_op can be used"; + } + if (replace_graph) { + MS_LOG(INFO) << "StepReplaceGraph " << cnode->ToString(); + StepReplaceGraph(replace_graph, cnode); + } +} + +void ParallelCommunication(const FuncGraphPtr& root, const std::vector& all_nodes, + const FuncGraphManagerPtr& manager) { + MS_EXCEPTION_IF_NULL(root); + MS_EXCEPTION_IF_NULL(manager); + TensorRedistribution tensor_redistribution; + AnfNodePtr grad_sens_node = nullptr; + + CNodePtr loss_cnode = FindLossCNodeFromRoot(root); + MS_EXCEPTION_IF_NULL(loss_cnode); + // get output layout of loss must before inserting the operators below + TensorLayouts loss_layout = GetLossNodeGradOutputLayout(loss_cnode); + + for (auto& node : all_nodes) { + // find sens node + if ((grad_sens_node == nullptr) && IsGradSensNode(node)) { + grad_sens_node = node; + MS_LOG(INFO) << "Find the sens node success"; + } + + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + OperatorInfoPtr distribute_operator = GetDistributeOperator(cnode); + if (distribute_operator == nullptr) { + continue; + } + + bool is_loss_cnode = false; + if (cnode == loss_cnode) { + is_loss_cnode = true; + } + + // insert forward ops + InsertForwardOps(distribute_operator, cnode); + + // insert redistribution ops + StepRedistribution(cnode, distribute_operator, cnode, tensor_redistribution, cnode); + + // insert backward ops + BackwardCommunication(distribute_operator, cnode, is_loss_cnode); + + // StepReplace + StepReplace(distribute_operator, cnode); + } else if (IsValueNode(node)) { + StepSplitTensor(node, manager); + } + } + + // If the shape of grad-sens tensor is not [] or [1], use get tensor slice to handel it. + // If the type of sens node is not Tensor, it is unsupported now, do nothing default. + if (grad_sens_node && !loss_layout.empty()) { + SplitSens(grad_sens_node, loss_layout[0]); + } +} + +namespace { +void RevertSymbolicKeyInstance(const FuncGraphPtr& root, const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(root); + MS_EXCEPTION_IF_NULL(node); + auto symbolic_key = GetValueNode(node); + MS_EXCEPTION_IF_NULL(symbolic_key); + auto all_upstream_node = root->manager()->node_users()[node]; + for (auto& upstream_node : all_upstream_node) { + FuncGraphPtr fg = upstream_node.first->func_graph(); + if (symbolic_key->node()->isa()) { + for (auto& param : root->parameters()) { + if (*param == *symbolic_key->node()) { + AnfNodePtr reverted_node = root->NewCNode({NewValueNode(prim::kPrimEmbed), param}); + MS_EXCEPTION_IF_NULL(reverted_node); + MS_LOG(DEBUG) << "before replace " << node->ToString() << " to node " << reverted_node->DebugString(); + (void)fg->manager()->Replace(node, reverted_node); + MS_LOG(DEBUG) << "revert node " << node->ToString() << " to node " << reverted_node->DebugString(); + } + } + } + } +} +} // namespace + +void HandleSymbolicKeyInstance(const FuncGraphPtr& root, const std::vector& all_nodes) { + MS_EXCEPTION_IF_NULL(root); + for (auto& node : all_nodes) { + // revert back SymbolicKeyInstance to embed() primitive + if (IsValueNode(node)) { + RevertSymbolicKeyInstance(root, node); + continue; + } + } +} + +void CheckpointStrategy(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_LOG(INFO) << "Save strategy to checkpoint begin"; + StrategyMap straMap; + auto ret = func_graph->get_return(); + auto all_nodes = DeepScopedGraphSearch(ret); + for (auto& node : all_nodes) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + if ((cnode == nullptr) || !IsValueNode(cnode->input(0))) { + continue; + } + PrimitivePtr prim = GetValueNode(cnode->input(0)); + MS_EXCEPTION_IF_NULL(prim); + OperatorInfoPtr operator_info = cnode->operator_info(); + if (operator_info) { + if (prim->instance_name().empty()) { + continue; + } + std::string instance_name = prim->instance_name(); + StrategyPtr strategyPtr = operator_info->strategy(); + MS_EXCEPTION_IF_NULL(node->scope()); + std::string node_name = node->scope()->name() + std::string(CONNSYMBOL) + instance_name; + straMap[node_name] = strategyPtr; + } + } + if (StrategyCheckpoint::GetInstance().Save(straMap) != SUCCESS) { + MS_LOG(EXCEPTION) << "Save strategy checkpoint failed"; + } +} + +void RestoreStrategy(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_LOG(INFO) << "Extract strategy from checkpoint begin"; + StrategyMap straMap; + if (StrategyCheckpoint::GetInstance().Load(&straMap) != SUCCESS) { + MS_LOG(EXCEPTION) << "Load strategy checkpoint failed"; + } + if (StrategyCheckpoint::GetInstance().RemoveCheckPoint() != SUCCESS) { + MS_LOG(EXCEPTION) << "Remove strategy checkpoint failed"; + } + auto ret = func_graph->get_return(); + auto all_nodes = DeepScopedGraphSearch(ret); + for (auto& node : all_nodes) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + if ((cnode == nullptr) || !IsValueNode(cnode->input(0))) { + continue; + } + PrimitivePtr prim = GetValueNode(cnode->input(0)); + MS_EXCEPTION_IF_NULL(prim); + OperatorInfoPtr operator_info = cnode->operator_info(); + if (operator_info) { + if (prim->instance_name().empty()) { + continue; + } + std::string instance_name = prim->instance_name(); + MS_EXCEPTION_IF_NULL(node->scope()); + std::string node_name = node->scope()->name() + std::string(CONNSYMBOL) + instance_name; + MS_LOG(INFO) << "Node name is " << node_name; + if (straMap.find(node_name) != straMap.end()) { + StrategyPtr strategyPtr = straMap[node_name]; + operator_info->set_strategy(strategyPtr); + } + } + } +} + +void SetForwardFlag(const std::vector& all_nodes) { + for (auto& node : all_nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + + // CNode is globally unique. + MS_LOG(DEBUG) << "Set forward flag " << cnode->DebugString() << "."; + cnode->set_in_forward_flag(true); + } +} + +void SetForwardFlag(const AnfNodeSet& all_nodes) { + for (auto& node : all_nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsValueNode(cnode->input(0))) { + continue; + } + + // CNode is globally unique. + cnode->set_in_forward_flag(true); + } +} + +CNodePtr FindLossCNode(const FuncGraphPtr& func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + CNodePtr return_node = func_graph->get_return(); + MS_EXCEPTION_IF_NULL(return_node); + if (return_node->inputs().size() < 2) { + MS_LOG(EXCEPTION) << "Failure: " << return_node->ToString() << " size is smaller than 2"; + } + AnfNodePtr pre_node = return_node->input(1); + MS_EXCEPTION_IF_NULL(pre_node); + + auto pre_cnode = pre_node->cast(); + MS_EXCEPTION_IF_NULL(pre_cnode); + auto current_value = pre_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(current_value); + PrimitivePtr current_prim = current_value->value()->cast(); + MS_EXCEPTION_IF_NULL(current_prim); + // notice: the GetNext op has not input + if (INVALID_LOSS_OPS.find(current_prim->name()) != INVALID_LOSS_OPS.end()) { + MS_LOG(INFO) << "The loss is: " << current_prim->name(); + return pre_cnode; + } + + // size of common cnode is larger than 1 + if (pre_cnode->inputs().size() < 2) { + MS_LOG(EXCEPTION) << pre_cnode->ToString() << " size( " << pre_cnode->inputs().size() << " ) is smaller than 2"; + } + + // return -> tuple_getitem -> loss + if (current_prim->name() == TUPLE_GETITEM) { + AnfNodePtr pre_pre_node = pre_cnode->input(1); + MS_EXCEPTION_IF_NULL(pre_pre_node); + + auto pre_pre_cnode = pre_pre_node->cast(); + auto value = pre_pre_cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(value); + PrimitivePtr prim = value->value()->cast(); + MS_EXCEPTION_IF_NULL(prim); + MS_LOG(INFO) << "The loss name is " << prim->name(); + return pre_pre_cnode; + } else if (current_prim->name() == MAKE_TUPLE) { + MS_LOG(EXCEPTION) << "The loss have make_tuple, it is not supported"; + } + + // return -> loss + MS_LOG(INFO) << "The loss name is " << current_prim->name(); + return pre_cnode; +} + +FuncGraphPtr FindForwardGraphByRootNodes(const AnfNodeSet& root_all_nodes) { + for (auto& node : root_all_nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if ((cnode->inputs().size() < 2) || !IsValueNode(cnode->input(0))) { + continue; + } + ValueNodePtr expect_j_value_node = cnode->input(0)->cast(); + MS_EXCEPTION_IF_NULL(expect_j_value_node); + PrimitivePtr expect_j_prim = expect_j_value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(expect_j_prim); + if (expect_j_prim->name() != J) { + continue; + } + MS_LOG(DEBUG) << "Find J prim: " << expect_j_value_node->DebugString() << "."; + if (IsValueNode(cnode->input(1))) { + auto graph = GetValueNode(cnode->input(1)); + MS_LOG(INFO) << "Find the forward graph success"; + return graph; + } + } + return nullptr; +} + +CNodePtr FindLossCNodeFromRoot(const FuncGraphPtr& root) { + MS_EXCEPTION_IF_NULL(root); + AnfNodePtr root_return_node = root->get_return(); + MS_EXCEPTION_IF_NULL(root_return_node); + const auto& all_nodes = root->nodes(); + FuncGraphPtr func_graph = FindForwardGraphByRootNodes(all_nodes); + + if (func_graph == nullptr) { + return FindLossCNode(root); + } else { + return FindLossCNode(func_graph); + } +} + +FuncGraphPtr ForwardGraph(const FuncGraphPtr& root) { + FuncGraphPtr forward_graph = root; + MS_EXCEPTION_IF_NULL(root); + AnfNodePtr root_return_node = root->get_return(); + MS_EXCEPTION_IF_NULL(root_return_node); + const auto& all_nodes = root->nodes(); + FuncGraphPtr func_graph = FindForwardGraphByRootNodes(all_nodes); + + if (func_graph != nullptr) { + forward_graph = func_graph; + } + return forward_graph; +} + +void MarkForwardCNode(const FuncGraphPtr& root) { + MS_EXCEPTION_IF_NULL(root); + AnfNodePtr root_return_node = root->get_return(); + MS_EXCEPTION_IF_NULL(root_return_node); + auto& all_nodes = root->nodes(); + FuncGraphPtr func_graph = FindForwardGraphByRootNodes(all_nodes); + + if (func_graph == nullptr) { + // Can not find the forward graph, so the ops in root graph are forward. + MS_LOG(INFO) << "Can not find the forward graph, so mark the ops in root graph"; + SetForwardFlag(all_nodes); + } else { + MS_LOG(INFO) << "The sub graph size of root is " << root->func_graphs_used().size(); + AnfNodePtr return_node = func_graph->get_return(); + MS_EXCEPTION_IF_NULL(return_node); + std::vector all_dfs_nodes = DeepLinkedGraphSearch(return_node); + SetForwardFlag(all_dfs_nodes); + } +} + +Status ParallelInit() { + MS_EXCEPTION_IF_NULL(ParallelContext::GetInstance()); + int32_t device_num = ParallelContext::GetInstance()->device_num(); + int32_t global_rank = ParallelContext::GetInstance()->global_rank(); + std::string backend = ParallelContext::GetInstance()->communication_backend(); + std::string world_group; + + if (backend == HCCL_BACKEND) { + world_group = HCCL_WORLD_GROUP; + } else if (backend == NCCL_BACKEND) { + world_group = NCCL_WORLD_GROUP; + } else { + MS_LOG(EXCEPTION) << "Invalid communication backend: " << backend; + } + + uint32_t world_rank_size = 0; + if (!ParallelContext::GetInstance()->device_num_is_set()) { + if (!CommManager::GetInstance().GetRankSize(world_group, &world_rank_size)) { + MS_LOG(EXCEPTION) << "Get rank size failed"; + } + device_num = UintToInt(world_rank_size); + MS_LOG(INFO) << "Get device num from communication model, the device num is " << device_num; + } + + uint32_t rank_id = 0; + if (!ParallelContext::GetInstance()->global_rank_is_set()) { + if (!CommManager::GetInstance().GetRankID(world_group, &rank_id)) { + MS_LOG(EXCEPTION) << "Get rank id failed"; + } + global_rank = UintToInt(rank_id); + MS_LOG(INFO) << "Get global rank from communication model, the global rank is " << global_rank; + } + + if (!InitDevice(device_num, global_rank, backend)) { + MS_LOG(ERROR) << "Init device failed"; + return FAILED; + } + + MS_LOG(INFO) << "The parallel context: dev num: " << device_num << ", global rank: " << global_rank + << ", backend: " << backend << ", mirror_mean: " << ParallelContext::GetInstance()->mirror_mean() + << ", cast_before_mirror: " << ParallelContext::GetInstance()->cast_before_mirror(); + return SUCCESS; +} + +bool StepParallel(const FuncGraphPtr& root, const opt::OptimizerPtr& optimizer) { + MS_EXCEPTION_IF_NULL(root); + MS_EXCEPTION_IF_NULL(optimizer); + MS_EXCEPTION_IF_NULL(ParallelContext::GetInstance()); + std::string parallel_mode = ParallelContext::GetInstance()->parallel_mode(); + // assume no change to graph + bool changes = false; + // control whether use model_parallel mode + if (((parallel_mode != AUTO_PARALLEL) && (parallel_mode != SEMI_AUTO_PARALLEL)) || + (root->has_flag(SEMI_AUTO_PARALLEL_RUN_ONCE_ONLY))) { + return changes; + } + + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + + MS_LOG(INFO) << "Now entering step parallel"; + DumpGraph(root, std::string(STEP_PARALLEL_BEGIN)); + + pipeline::ResourceBasePtr res = optimizer->resource(); + MS_EXCEPTION_IF_NULL(res); + + FuncGraphManagerPtr manager = res->manager(); + MS_EXCEPTION_IF_NULL(manager); + AnfNodePtr ret = root->get_return(); + MS_EXCEPTION_IF_NULL(ret); + std::vector all_nodes = DeepScopedGraphSearch(ret); + std::reverse(all_nodes.begin(), all_nodes.end()); + if (parallel_mode != AUTO_PARALLEL) { + TOTAL_OPS = 0; + if (ParallelInit() != SUCCESS) { + MS_LOG(EXCEPTION) << "Parallel init failed"; + } + + // mark the forward cnodes, parallel only care these nodes + MarkForwardCNode(root); + + if (FindCommunicationOp(all_nodes)) { + MS_LOG(EXCEPTION) << "The graph contain communication op"; + } + + // extract shape and strategy, set operator_info + ExtractInformation(all_nodes); + ReshapeInit(all_nodes); + // extract strategy from checkpoint for multi-train + if (StrategyCheckpoint::GetInstance().CheckPointOn() && StrategyCheckpoint::GetInstance().CheckPointExit()) { + RestoreStrategy(root); + } + } + // save strategy as checkpoint for multi-train + if (StrategyCheckpoint::GetInstance().CheckPointOn() && + StrategyCheckpoint::GetInstance().GetCurrentTrainTime() < StrategyCheckpoint::GetInstance().GetTrainTimes()) { + CheckpointStrategy(root); + } + + HandleSymbolicKeyInstance(root, all_nodes); + + // cover Parallel shape + CoverSliceShape(root); + + // set the shape for optimizer's clone tensor + SetClonedTensorShapeForOptimizer(root); + + // ForwardCommunication BackwardCommunication TensorRedistribution + ParallelCommunication(root, all_nodes, manager); + + DumpGraph(root, std::string(STEP_PARALLEL_END)); + + // step parallel only run once + root->flags()[SEMI_AUTO_PARALLEL_RUN_ONCE_ONLY] = true; + res->results()[pipeline::kStepParallelGraph] = root; + + (void)gettimeofday(&end_time, nullptr); + uint64_t time = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + time += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Now leaving step parallel, used time: " << time << " us"; + return changes; +} + +// Needed by rec_parser +std::vector ExtractInputsTensorName(const CNodePtr& node) { + std::vector name_inputs; + std::vector all_inputs = node->inputs(); + std::vector node_inputs{all_inputs.begin() + 1, all_inputs.end()}; + + for (auto& input : node_inputs) { + std::string name; + if (IsValueNode(input) || input->isa() || input->isa()) { + name = input->ToString(); + } else { + continue; + } + name_inputs.push_back(name); + } + + return name_inputs; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/step_parallel.h b/mindspore/ccsrc/parallel/step_parallel.h new file mode 100644 index 00000000000..ac7376a09d0 --- /dev/null +++ b/mindspore/ccsrc/parallel/step_parallel.h @@ -0,0 +1,155 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_STEP_PARALLEL_H_ +#define MINDSPORE_CCSRC_PARALLEL_STEP_PARALLEL_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "./common.h" +#include "optimizer/opt.h" +#include "parallel/strategy.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +using OperatorInfoPtr = std::shared_ptr; + +namespace mindspore { +namespace parallel { +const uint64_t kUSecondInSecond = 1000000; + +struct LossNodeInfo { + bool has_tuple_getitem = false; + int dout_index = 0; // now don't support the sens is a tuple +}; + +std::vector CreateInput(const Operator& op, const AnfNodePtr& node, const std::string& instance_name); +std::string CreateInstanceName(const CNodePtr& node, size_t index); +void ForwardCommunication(OperatorVector forward_op, const CNodePtr& node); + +void InsertRedistribution(const RedistributionOpListPtr& redistribution_oplist_ptr, const CNodePtr& node, + const FuncGraphPtr& func_graph, int pos, const CNodePtr& pre_node); + +TensorLayout GetTensorInLayout(const CNodePtr& pre_node, const PrimitivePtr& pre_prim, + const OperatorInfoPtr& distribute_operator_pre); + +OperatorInfoPtr GetDistributeOperator(const CNodePtr& node); + +void Redistribution(const std::pair& node_pair, const OperatorInfoPtr& distribute_operator, + const CNodePtr& middle_node, int index, TensorRedistribution tensor_redistribution, + const CNodePtr& pre_node); + +bool StrategyFound(std::unordered_map attrs); + +bool IsParallelCareNode(const CNodePtr& cnode); + +void MarkForwardCNode(const FuncGraphPtr& root); + +bool FindCommunicationOp(const std::vector& all_nodes); + +void StepRedistribution(const CNodePtr& node, const OperatorInfoPtr& distribute_operator, const CNodePtr& insert_node, + const TensorRedistribution& tensor_redistribution, const CNodePtr& pre_node); + +std::vector ReplaceOpInput(const Operator& replace_op, const std::string& instance_name, + const CNodePtr& node); + +void StepReplaceOp(OperatorVector replace_op, const CNodePtr& node); + +void InsertVirtualDivOp(const VirtualDivOp& virtual_div_op, const CNodePtr& node); + +std::pair FindParameter(const AnfNodePtr& node, const FuncGraphPtr& func_graph); + +std::pair FindCNode(const AnfNodePtr& anode, const std::string& name, const FuncGraphPtr& func_graph); + +void InsertMirrorOps(const MirrorOps& mirror_ops, const CNodePtr& node); + +void BackwardCommunication(const OperatorInfoPtr& distribute_operator, const CNodePtr& node, bool is_loss_node); + +// Generate and init parallel operator +OperatorInfoPtr OperatorInstance(const PrimitivePtr& prim, const PrimitiveAttrs& attrs, + const std::vector& shape_list); + +// Generate without initing parallel operator +OperatorInfoPtr NewOperatorInstance(const PrimitivePtr& prim, const PrimitiveAttrs& attrs, + std::vector shape_list); + +// Extract strategy from attr +StrategyPtr ExtractStrategy(std::unordered_map attrs); + +Shapes GetNodeShape(const AnfNodePtr& node); + +std::vector FindParameterByRefKeyNode(const AnfNodePtr& node, const FuncGraphPtr& func_graph); + +// Extract shape from anfnode +std::vector ExtractShape(const CNodePtr& node); + +std::pair FindParallelCareNode(const AnfNodePtr& node); + +// Find finally sub graph +std::pair FindSubGraph(const FuncGraphPtr& func_graph, const AnfNodePtr& parameter); + +// Set distribute shape for parameters abstract +void SetParallelShape(const AnfNodePtr& parameter, const std::pair& res); + +// change parameters'shape in resource +void CoverSliceShape(const FuncGraphPtr& root); + +void SetVirtualDatasetStrategy(const CNodePtr& node); + +// Creat parallel operator for primitive node(has strategy) +void ExtractInformation(const std::vector& all_nodes); + +TensorLayout GetInputLayoutFromCNode(const std::pair& node_pair); + +std::shared_ptr FindNextLayout(const CNodePtr& node); + +std::shared_ptr GetOutputLayoutFromCNode(const CNodePtr& cnode, size_t output_index); + +std::shared_ptr FindPrevParallelCareNodeLayout(const AnfNodePtr& node, size_t output_index); + +std::shared_ptr FindPrevLayout(const AnfNodePtr& node); + +void ReshapeInit(const std::vector& all_nodes); + +// Add node for whole graph +void ParallelCommunication(const FuncGraphPtr& root, const std::vector& all_nodes, + const FuncGraphManagerPtr& manager); + +void RestoreStrategy(const FuncGraphPtr& func_graph); + +void CheckpointStrategy(const FuncGraphPtr& func_graph); + +// main step of Parallel +bool StepParallel(const FuncGraphPtr& func_graph, const opt::OptimizerPtr& optimizer); + +int32_t GetTupleGetItemIndex(const CNodePtr& cnode); + +CNodePtr FindLossCNodeFromRoot(const FuncGraphPtr& root); + +Status ParallelInit(); + +std::vector ExtractInputsTensorName(const CNodePtr& node); + +FuncGraphPtr ForwardGraph(const FuncGraphPtr& root); +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_STEP_PARALLEL_H_ diff --git a/mindspore/ccsrc/parallel/strategy.h b/mindspore/ccsrc/parallel/strategy.h new file mode 100644 index 00000000000..24879c12c1b --- /dev/null +++ b/mindspore/ccsrc/parallel/strategy.h @@ -0,0 +1,66 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_STRATEGY_H_ +#define MINDSPORE_CCSRC_PARALLEL_STRATEGY_H_ + +#include +#include +#include +#include +#include +#include + +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { + +#define MIN_SLICE_NUM 1 + +using Dimensions = std::vector; + +class Strategy; +using StrategyPtr = std::shared_ptr; + +class Strategy { + public: + Strategy(int32_t stage, std::vector inputs) : stage_(stage), inputs_(std::move(inputs)) {} + ~Strategy() = default; + size_t GetInputNumber() const { return inputs_.size(); } + std::vector GetInputDim() const { return inputs_; } + int32_t GetInputStage() const { return stage_; } + void ExpandInputDimFromOneToTwo() { + if (inputs_.size() == 1) { + inputs_.push_back(inputs_[0]); + } + } + void ResetInputs(const std::vector& input) { inputs_ = input; } + + private: + const int32_t stage_; + + // The size of Dimensions must equal to inputs_ tensor dimension. + std::vector inputs_; +}; + +inline StrategyPtr NewStrategy(const int32_t stage, const std::vector& inputs) { + return std::make_shared(stage, inputs); +} +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_STRATEGY_H_ diff --git a/mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc b/mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc new file mode 100644 index 00000000000..4e008feee11 --- /dev/null +++ b/mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc @@ -0,0 +1,111 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/strategy_checkpoint/parallel_strategy_checkpoint.h" + +#include +#include +#include + +#include "utils/log_adapter.h" +#include "utils/node_strategy.pb.h" +#include "utils/convert_utils.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { +StrategyCheckpoint& StrategyCheckpoint::GetInstance() { + static StrategyCheckpoint instance = StrategyCheckpoint(); + return instance; +} + +bool StrategyCheckpoint::CheckPointExit() const { + std::ifstream fin(path_); + if (fin) { + return true; + } + return false; +} + +Status StrategyCheckpoint::RemoveCheckPoint() const { + if (std::remove(common::SafeCStr(path_)) == 0) { + return SUCCESS; + } + return FAILED; +} + +Status StrategyCheckpoint::Load(StrategyMap* strategy_map) { + if (strategy_map == nullptr) { + MS_LOG(EXCEPTION) << "Failure:strategy_map is nullptr"; + } + straspb::ParallelStrategyMap parallel_strategy_map; + std::fstream input(path_, std::ios::in | std::ios::binary); + if (!parallel_strategy_map.ParseFromIstream(&input)) { + MS_LOG(ERROR) << "Load strategy file failed"; + return FAILED; + } + size_t node_num = IntToSize(parallel_strategy_map.parallel_strategy_item_size()); + for (size_t i = 0; i < node_num; i++) { + straspb::ParallelStrategyItem parallel_strategy_item = parallel_strategy_map.parallel_strategy_item(SizeToInt(i)); + std::string node_name = parallel_strategy_item.node_name(); + straspb::ParallelStrategys parallel_strategys = parallel_strategy_item.parallel_strategys(); + auto stage = (int32_t)parallel_strategys.stage(); + size_t strategys_num = IntToSize(parallel_strategys.parallel_strategy_size()); + std::vector> strategy_inputs; + for (size_t j = 0; j < strategys_num; j++) { + straspb::ParallelStrategy parallel_strategy = parallel_strategys.parallel_strategy(SizeToInt(j)); + std::vector dimension; + size_t dim_num = IntToSize(parallel_strategy.dim_size()); + for (size_t k = 0; k < dim_num; k++) { + dimension.push_back(parallel_strategy.dim(SizeToInt(k))); + } + strategy_inputs.push_back(dimension); + } + + StrategyPtr strategy = NewStrategy(stage, strategy_inputs); + (*strategy_map)[node_name] = strategy; + current_train_time_ = (int32_t)parallel_strategy_map.train_time(); + } + return SUCCESS; +} + +Status StrategyCheckpoint::Save(const StrategyMap& strategy_map) { + straspb::ParallelStrategyMap parallel_strategy_map; + parallel_strategy_map.set_train_time(IntToUint(++current_train_time_)); + for (auto& node_stra : strategy_map) { + straspb::ParallelStrategyItem* parallel_strategy_item = parallel_strategy_map.add_parallel_strategy_item(); + MS_EXCEPTION_IF_NULL(parallel_strategy_item); + parallel_strategy_item->set_node_name(node_stra.first); + straspb::ParallelStrategys* parallel_strategys = parallel_strategy_item->mutable_parallel_strategys(); + MS_EXCEPTION_IF_NULL(parallel_strategys); + parallel_strategys->set_stage(IntToUint(node_stra.second->GetInputStage())); + for (auto& dims : node_stra.second->GetInputDim()) { + straspb::ParallelStrategy* parallel_strategy = parallel_strategys->add_parallel_strategy(); + MS_EXCEPTION_IF_NULL(parallel_strategy); + for (auto dim : dims) { + parallel_strategy->add_dim(IntToUint(dim)); + } + } + } + std::fstream output(path_, std::ios::out | std::ios::trunc | std::ios::binary); + if (!parallel_strategy_map.SerializeToOstream(&output)) { + MS_LOG(ERROR) << "Save strategy file failed"; + return FAILED; + } + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.h b/mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.h new file mode 100644 index 00000000000..3cbb116b421 --- /dev/null +++ b/mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_STRATEGY_CHEKCPOINT_PARALLEL_STRATEGY_CHECKPOINT_H_ +#define MINDSPORE_CCSRC_PARALLEL_STRATEGY_CHEKCPOINT_PARALLEL_STRATEGY_CHECKPOINT_H_ + +#include +#include +#include "parallel/strategy.h" +#include "parallel/ops_info/ops_utils.h" + +namespace mindspore { +namespace parallel { +constexpr char DEFAULT_CHECKPOINT_PATH[] = "./strategys.ckpt"; + +using StrategyMap = std::unordered_map; +class StrategyCheckpoint { + public: + StrategyCheckpoint() : path_(DEFAULT_CHECKPOINT_PATH), current_train_time_(1) { + train_times_ = 1; + checkpoint_on_ = false; + const char* train_times_str = std::getenv("PARALLEL_TRAIN_TIMES"); + if (train_times_str != nullptr && std::stoi(train_times_str) > 0) { + train_times_ = std::stoi(train_times_str); + } + const char* checkpoint_on_str = std::getenv("PARALLEL_CHECKPOINT_ON"); + if (checkpoint_on_str != nullptr) { + checkpoint_on_ = (std::string(checkpoint_on_str) == "on"); + } + } + ~StrategyCheckpoint() = default; + bool CheckPointExit() const; + Status RemoveCheckPoint() const; + Status Load(StrategyMap* strategy_map); + Status Save(const StrategyMap& strategy_map); + + static StrategyCheckpoint& GetInstance(); + int32_t GetTrainTimes() const { return train_times_; } + int32_t GetCurrentTrainTime() const { return current_train_time_; } + bool CheckPointOn() const { return checkpoint_on_; } + + private: + std::string path_; + bool checkpoint_on_; + // total train times for a train, get from Environmental variable:TRAIN_TIME, please export it + int32_t train_times_; + int32_t current_train_time_; +}; +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_STRATEGY_CHEKCPOINT_PARALLEL_STRATEGY_CHECKPOINT_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/arrangement.cc b/mindspore/ccsrc/parallel/tensor_layout/arrangement.cc new file mode 100644 index 00000000000..fea7e4ba65c --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/arrangement.cc @@ -0,0 +1,249 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/arrangement.h" +#include +#include +#include +#include "parallel/status.h" +#include "utils/log_adapter.h" +#include "parallel/tensor_layout/shape_util.h" +#include "utils/convert_utils.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { + +Status Arrangement::Init(const std::vector& array) { + Status status = Array::Init(array); + if (status != Status::SUCCESS) { + return Status::FAILED; + } + if (!IsValidArrangement()) { + MS_LOG(ERROR) << "invalid arrangement " << this->ToString(); + return Status::FAILED; + } + ComputeSize(); + return Status::SUCCESS; +} + +bool Arrangement::IsValidArrangement() { + return !std::any_of(array_.begin(), array_.end(), [](int32_t value) { return value <= 0; }); +} + +void Arrangement::ComputeSize() { + size_ = 1; + for (auto& value : array_) { + size_ *= value; + } +} + +/* + * if GetDimSize() = 0, return [] + * if value <= array_[0], return [value] + * if array_[0] < value <= size_[i], return [shape[0], shape[1], ..., shape[i-1], value/size_[i-1]], + * where size_[i-1] = shape[0] * shape[1] * ... * shape[i-1], + * if value > size_, return [] + */ +std::vector Arrangement::GetFrontElementByValue(int32_t value) const { + std::vector out; + if (GetDimSize() == 0) { + return out; + } + if (value <= size_) { + int32_t size = 1; + uint32_t shape_list_idx = 0; + while (size < value) { + size *= array_[shape_list_idx]; + if (size <= value) { + out.push_back(array_[shape_list_idx]); + } else { + if (size == 0) { + MS_LOG(ERROR) << "The size is 0"; + out.clear(); + return out; + } + out.push_back(value * array_[shape_list_idx] / size); + } + shape_list_idx++; + } + } + return out; +} + +std::shared_ptr Arrangement::GetExpandedShapeByExpandListRemoveLeft( + const std::vector& expand_list) const { + if (expand_list.size() != GetDimSize()) { + return nullptr; + } + std::vector new_shape; + for (uint32_t i = 0; i < expand_list.size(); i++) { + std::vector expand_shape = expand_list[i].GetFrontElementByValue(GetDimByIdx(i)); + if (expand_shape.empty()) { + new_shape.push_back(GetDimByIdx(i)); + } else { + (void)new_shape.insert(new_shape.end(), expand_shape.begin(), expand_shape.end()); + } + } + Arrangement arrangement_new; + (void)arrangement_new.Init(new_shape); + return std::make_shared(arrangement_new); +} + +/* + * example: + * expand_shape = [4, 2, 2, 2] + * array_ = [8, 4], + * arrangement_list = [[4, 2], [2, 2]] + */ +std::shared_ptr> Arrangement::GetExpandShapeList(const Arrangement& expand_shape) const { + int32_t size = 1; + uint32_t ind = 0; + std::vector arrangement_list; + std::vector shape; + for (uint32_t i = 0; i < expand_shape.GetDimSize(); i++) { + size *= expand_shape.GetDimByIdx(i); + if (size > GetDimByIdx(ind)) { + MS_LOG(ERROR) << "invalid expand_shape"; + return nullptr; + } else if (size < GetDimByIdx(ind)) { + shape.push_back(expand_shape.GetDimByIdx(i)); + continue; + } else { + shape.push_back(expand_shape.GetDimByIdx(i)); + Arrangement arrangement; + (void)arrangement.Init(shape); + arrangement_list.push_back(arrangement); + shape.clear(); + ind++; + size = 1; + } + } + if (ind != GetDimSize()) { + MS_LOG(ERROR) << "invalid expand_shape"; + return nullptr; + } + auto arrangement_new = std::make_shared>(arrangement_list); + return arrangement_new; +} + +std::shared_ptr, Arrangement>> Arrangement::GetExpandShapeListPair( + const Arrangement& expand_shape) const { + std::shared_ptr> expand_shape_list_ptr = GetExpandShapeList(expand_shape); + if (expand_shape_list_ptr == nullptr) { + return nullptr; + } + std::vector expand_num_list_shape; + (void)std::transform(expand_shape_list_ptr->begin(), expand_shape_list_ptr->end(), + std::back_inserter(expand_num_list_shape), + [](const Arrangement& arr) { return SizeToInt(arr.GetDimSize()); }); + Arrangement expand_num_list; + Status status = expand_num_list.Init(expand_num_list_shape); + if (status != Status::SUCCESS) { + return nullptr; + } + auto out_value = std::make_pair(*expand_shape_list_ptr, expand_num_list); + return std::make_shared, Arrangement>>(out_value); +} + +std::vector Arrangement::ComputeReverseAccumulateSumInReverseOrder() const { + std::vector shape_accum; + int32_t size = 0; + for (auto iter = array_.end() - 1; iter >= array_.begin(); --iter) { + shape_accum.push_back(size); + size += *iter; + } + return shape_accum; +} + +std::shared_ptr Arrangement::GetExpandedShapeByExpandListReserveLeft( + const std::vector& expand_list) const { + if (expand_list.size() != GetDimSize()) { + return nullptr; + } + std::vector new_shape; + for (uint32_t i = 0; i < expand_list.size(); i++) { + if (expand_list[i].GetDimSize() >= 1) { + int32_t size = 1; + for (uint32_t k = 0; k < expand_list[i].GetDimSize() - 1; k++) { + new_shape.push_back(expand_list[i].GetDimByIdx(k)); + size *= expand_list[i].GetDimByIdx(k); + } + new_shape.push_back(GetDimByIdx(i) / size); + } else { + new_shape.push_back(GetDimByIdx(i)); + } + } + Arrangement arrangement_new; + (void)arrangement_new.Init(new_shape); + return std::make_shared(arrangement_new); +} + +std::shared_ptr Arrangement::GetUnifiedShape(const Arrangement& in2) const { + std::vector in1_accum; + Status status = ShapeToAccumulateProduct(array_, &in1_accum); + if (status != Status::SUCCESS) { + return nullptr; + } + std::vector in2_accum; + status = ShapeToAccumulateProduct(in2.array(), &in2_accum); + if (status != Status::SUCCESS) { + return nullptr; + } + std::vector out_accum; + status = UnifyAccumulateProduct(in1_accum, in2_accum, &out_accum); + if (status != Status::SUCCESS) { + return nullptr; + } + std::vector out_shape; + status = AccumulateProductToShape(out_accum, &out_shape); + if (status != Status::SUCCESS) { + return nullptr; + } + Arrangement out; + status = out.Init(out_shape); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(out); +} + +std::vector Arrangement::GetSqueezeIdx() const { + std::vector out; + for (size_t i = 0; i < GetDimSize(); i++) { + if (GetDimByIdx(SizeToUint(i)) == 1) { + out.push_back(i); + } + } + return out; +} + +Arrangement Arrangement::GetSqueezeArrangement() const { + std::vector out_shape(array_.size()); + auto it = std::copy_if(array_.begin(), array_.end(), out_shape.begin(), [](int32_t value) { return value != 1; }); + out_shape.resize(LongToSize(std::distance(out_shape.begin(), it))); + + // if all elements are 1, out_shape = {1} + if (out_shape.empty()) { + MS_LOG(ERROR) << "out_shape size is 0, this may not happen under current situation"; + out_shape.push_back(1); + } + Arrangement out; + (void)out.Init(out_shape); + return out; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/arrangement.h b/mindspore/ccsrc/parallel/tensor_layout/arrangement.h new file mode 100644 index 00000000000..582beeaff2a --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/arrangement.h @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_ARRANGEMENT_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_ARRANGEMENT_H_ + +#include +#include +#include +#include +#include +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/array.h" + +namespace mindspore { +namespace parallel { + +class Arrangement : public Array { + public: + Arrangement() : size_(1) {} + ~Arrangement() override = default; + Status Init(const std::vector& array) override; + int32_t size() const { return size_; } + std::vector GetFrontElementByValue(int32_t value) const; + std::shared_ptr> GetExpandShapeList(const Arrangement& expand_shape) const; + std::vector ComputeReverseAccumulateSumInReverseOrder() const; + std::shared_ptr GetExpandedShapeByExpandListReserveLeft( + const std::vector& expand_list) const; + std::shared_ptr GetExpandedShapeByExpandListRemoveLeft( + const std::vector& expand_list) const; + std::shared_ptr, Arrangement>> GetExpandShapeListPair( + const Arrangement& expand_shape) const; + std::shared_ptr GetUnifiedShape(const Arrangement& in2) const; + std::vector GetSqueezeIdx() const; + Arrangement GetSqueezeArrangement() const; + + private: + bool IsValidArrangement(); + void ComputeSize(); + int32_t size_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_ARRANGEMENT_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/array.cc b/mindspore/ccsrc/parallel/tensor_layout/array.cc new file mode 100644 index 00000000000..e073c3905c1 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/array.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/array.h" +#include +#include "utils/log_adapter.h" +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { + +std::string Array::ToString() const { + std::ostringstream buffer; + buffer << "[ "; + for (auto& element : array_) { + buffer << std::to_string(element) + " "; + } + buffer << "]"; + return buffer.str(); +} + +Status Array::Init(const std::vector& array) { + array_ = array; + return IsvalidArray() ? Status::SUCCESS : Status::FAILED; +} + +bool Array::IsvalidArray() const { return true; } + +int32_t Array::GetDimByIdx(uint32_t idx) const { + size_t mod_idx = idx; + if (idx >= GetDimSize()) { + MS_LOG(EXCEPTION) << "idx is " << idx << ", but array size is " << GetDimSize(); + } + return array_[mod_idx]; +} + +int32_t Array::GetDimByReverseIdx(uint32_t idx) const { + size_t mod_idx = idx; + if (idx >= GetDimSize()) { + MS_LOG(EXCEPTION) << "idx is " << idx << " but array size is " << GetDimSize(); + } + return array_[GetDimSize() - 1 - mod_idx]; +} + +bool Array::operator==(const Array& shape) const { + if (GetDimSize() != shape.GetDimSize()) { + return false; + } + for (uint32_t i = 0; i < GetDimSize(); i++) { + if (GetDimByIdx(i) != shape.GetDimByIdx(i)) { + return false; + } + } + return true; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/array.h b/mindspore/ccsrc/parallel/tensor_layout/array.h new file mode 100644 index 00000000000..3a47f0d8187 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/array.h @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_ARRAY_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_ARRAY_H_ + +#include +#include +#include +#include +#include +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { + +class Array { + public: + Array() = default; + virtual ~Array() = default; + std::string ToString() const; + virtual Status Init(const std::vector& array); + bool IsvalidArray() const; + std::vector array() const { return array_; } + size_t GetDimSize() const { return array_.size(); } + int32_t GetDimByIdx(uint32_t idx) const; + int32_t GetDimByReverseIdx(uint32_t idx) const; + bool operator==(const Array& a1) const; + + protected: + std::vector array_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_ARRAY_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/construct_operator.cc b/mindspore/ccsrc/parallel/tensor_layout/construct_operator.cc new file mode 100644 index 00000000000..829c056fc2c --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/construct_operator.cc @@ -0,0 +1,254 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/construct_operator.h" + +#include +#include + +namespace mindspore { +namespace parallel { +Status ConstructOperator::Init(const RankList& dev_list, const Shape& dev_matrix_shape) { + dev_size_ = dev_matrix_shape.size(); + dev_matrix_shape_ = dev_matrix_shape; + dev_list_ = dev_list; + return Status::SUCCESS; +} + +Status ConstructOperator::ReshapeOP(Shape shape) { + int32_t prod = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + int32_t prod_expect = std::accumulate(tensor_shape_.begin(), tensor_shape_.end(), 1, std::multiplies()); + if (prod != prod_expect) { + ValuePtr ptr = MakeValue(shape); + MS_EXCEPTION_IF_NULL(ptr); + MS_LOG(ERROR) << "Invalid tensor shape " << ptr->ToString() << "when construct Reshape operator!"; + return Status::INVALID_ARGUMENT; + } + OperatorAttrs attrs; + ValuePtr param_value = MakeValue(shape); + Attr param = std::make_pair(SHAPE, param_value); + OperatorParams params = {std::make_pair(param, 2)}; + OperatorArgs args = std::make_pair(attrs, params); + op_ = std::make_pair(RESHAPE, args); + return Status::SUCCESS; +} + +Operator CreateStridedSliceOp(int32_t value, const Shape& begin, const Shape& end, const Shape& strides) { + ValuePtr attr_value = MakeValue(value); + Attr attr_begin_mask = std::make_pair(BEGIN_MASK, attr_value); + Attr attr_end_mask = std::make_pair(END_MASK, attr_value); + Attr attr_ellipsis_mask = std::make_pair(ELLIPSIS_MASK, attr_value); + Attr attr_new_axis_mask = std::make_pair(NEW_AXIS_MASK, attr_value); + Attr attr_shrink_axis_mask = std::make_pair(SHRINK_AXIS_MASK, attr_value); + OperatorAttrs attrs = {attr_begin_mask, attr_end_mask, attr_ellipsis_mask, attr_new_axis_mask, attr_shrink_axis_mask}; + + ValuePtr param_begin_value = MakeValue(begin); + Param param_begin = std::make_pair(std::make_pair(BEGIN, param_begin_value), 2); + ValuePtr param_end_value = MakeValue(end); + Param param_end = std::make_pair(std::make_pair(END, param_end_value), 3); + + ValuePtr param_strides_value = MakeValue(strides); + Param param_strides = std::make_pair(std::make_pair(STRIDES, param_strides_value), 4); + OperatorParams params = {param_begin, param_end, param_strides}; + OperatorArgs op_args = std::make_pair(attrs, params); + + return std::make_pair(STRIDED_SLICE, op_args); +} + +Status ConstructOperator::StridedSliceOP(Args args) { + if (args.size() < 3) { + MS_LOG(ERROR) << "args size should not be less than 3!"; + return Status::FAILED; + } + int32_t split_count = args[0]; + if (split_count <= 0) { + MS_LOG(ERROR) << "split_count should not be less than 0!"; + return Status::FAILED; + } + int32_t split_dim = args[1]; + int32_t dev_dim = args[2]; + std::vector group_list; + + if (CreateGroupByDim(dev_size_ - IntToSize(dev_dim) - 1, &group_list) != SUCCESS) { + MS_LOG(ERROR) << "stride slice op: create group failed"; + return FAILED; + } else if (group_list.empty()) { // this group only has one device, don't need do StridedSlice + MS_LOG(INFO) << "no need stride slice op"; + return SUCCESS; + } + + Group group = group_list[0]; + size_t rank; + if (group.GetIndex(&rank) == Status::FAILED) { + return Status::FAILED; + } + size_t size = tensor_shape_.size(); + Shape begin(size); + Shape end(size); + Shape strides(size, 1); + size_t index = 0; + for (auto num : tensor_shape_) { + if (index != IntToSize(split_dim)) { + begin[index] = 0; + end[index] = num; + } else { + if (num % split_count != 0) { + MS_LOG(ERROR) << "Tensor can not be split into " << split_count << " slices in the dimension " << split_dim + << "! when construct StridedSlice operator"; + return Status::INVALID_ARGUMENT; + } + int32_t count = num / split_count; + begin[index] = SizeToInt(rank) * count; + end[index] = (SizeToInt(rank) + 1) * count; + } + index++; + } + + op_ = CreateStridedSliceOp(DEFAULT, begin, end, strides); + + return Status::SUCCESS; +} + +Status ConstructOperator::AllGatherOP(int32_t dev_dim) { + if ((IntToSize(dev_dim) >= dev_size_) || (dev_dim < 0)) { + MS_LOG(ERROR) << "Invalid device dimension " << dev_dim << " when construct AllGather operator!"; + return Status::INVALID_ARGUMENT; + } + + std::vector group_list; + if (CreateGroupByDim(dev_size_ - IntToSize(dev_dim) - 1, &group_list) != SUCCESS) { + MS_LOG(ERROR) << "AllGather op: create group failed"; + return FAILED; + } else if (group_list.empty()) { // this group only has one device, don't need do allgather + MS_LOG(INFO) << "no need all gather op"; + return SUCCESS; + } + + std::string group_name = group_list[0].name(); + ValuePtr attr_value = MakeValue(group_name); + Attr attr = std::make_pair(GROUP, attr_value); + OperatorAttrs attrs = {attr}; + OperatorParams params; + OperatorArgs args = std::make_pair(attrs, params); + op_ = std::make_pair(ALL_GATHER, args); + return Status::SUCCESS; +} + +Status ConstructOperator::ConcatOP(int32_t concat_dim) { + if (IntToSize(concat_dim) >= tensor_shape_.size()) { + MS_LOG(ERROR) << "Invalid tensor dimension " << concat_dim << " when construct Concat operator!"; + return Status::INVALID_ARGUMENT; + } + ValuePtr attr_value = MakeValue(concat_dim); + Attr attr = std::make_pair(AXIS, attr_value); + OperatorAttrs attrs = {attr}; + OperatorParams params; + OperatorArgs args = std::make_pair(attrs, params); + op_ = std::make_pair(CONCAT, args); + return Status::SUCCESS; +} + +Status ConstructOperator::SplitOP(int32_t split_count) { + if (split_count <= 0) { + MS_LOG(ERROR) << "Invalid split count when construct Split operator!"; + return Status::FAILED; + } + OperatorAttrs attrs; + ValuePtr attr_value_axis = MakeValue(DEFAULT); + Attr attr_axis = std::make_pair(AXIS, attr_value_axis); + ValuePtr attr_value_split = MakeValue(split_count); + Attr attr_split = std::make_pair(OUTPUT_NUM, attr_value_split); + attrs = {attr_axis, attr_split}; + OperatorParams params; + OperatorArgs args = std::make_pair(attrs, params); + op_ = std::make_pair(SPLIT, args); + return Status::SUCCESS; +} + +Status ConstructOperator::AlltoAllOP(Args args) { + if (args.size() < 4) { + MS_LOG(ERROR) << "args size should not be less than 4!"; + return Status::FAILED; + } + int32_t split_count = args[0]; + int32_t split_dim = args[1]; + int32_t concat_dim = args[2]; + int32_t dev_dim = args[3]; + if (split_count <= 0) { + MS_LOG(ERROR) << "Invalid split count when construct AlltoAll operator!"; + return Status::FAILED; + } + if (tensor_shape_[IntToSize(split_dim)] % split_count != 0) { + MS_LOG(ERROR) << "Tensor can not be split into " << split_count << " slices in the dimension " << split_dim + << "when construct AlltoAll operator!"; + return Status::INVALID_ARGUMENT; + } + if (IntToSize(concat_dim) >= tensor_shape_.size()) { + MS_LOG(ERROR) << "Invalid split count " << split_count << " when construct AlltoAll operator!"; + return Status::INVALID_ARGUMENT; + } + if ((IntToSize(dev_dim) >= dev_size_) || (dev_dim < 0)) { + MS_LOG(ERROR) << "Invalid device dimension " << dev_dim << " when construct AlltoAll operator!"; + return Status::INVALID_ARGUMENT; + } + + std::vector group_list; + if (CreateGroupByDim(dev_size_ - IntToSize(dev_dim) - 1, &group_list) != SUCCESS) { + MS_LOG(ERROR) << "AlltoAll op: create group failed"; + return FAILED; + } else if (group_list.empty()) { // this group only has one device, don't need do alltoall + MS_LOG(INFO) << "no need all to all op"; + return SUCCESS; + } + + std::string group_name = group_list[0].name(); + ValuePtr attr_value_group = MakeValue(group_name); + Attr attr_group = std::make_pair(GROUP, attr_value_group); + ValuePtr attr_value_split_count = MakeValue(split_count); + Attr attr_split_count = std::make_pair(SPLIT_COUNT, attr_value_split_count); + ValuePtr attr_value_split_dim = MakeValue(split_dim); + Attr attr_split_dim = std::make_pair(SPLIT_DIM, attr_value_split_dim); + ValuePtr attr_value_concat_dim = MakeValue(concat_dim); + Attr attr_concat_dim = std::make_pair(CONCAT_DIM, attr_value_concat_dim); + OperatorAttrs attrs = {attr_split_count, attr_split_dim, attr_concat_dim, attr_group}; + OperatorParams params; + OperatorArgs op_args = std::make_pair(attrs, params); + op_ = std::make_pair(ALL_TO_ALL, op_args); + return Status::SUCCESS; +} + +Status ConstructOperator::CreateGroupByDim(size_t axis, std::vector* group) { + MS_EXCEPTION_IF_NULL(group); + CheckGlobalDeviceManager(); + MS_EXCEPTION_IF_NULL(g_device_manager); + int32_t rank = g_device_manager->global_rank(); + DeviceMatrix dev_matrix(rank, dev_list_, dev_matrix_shape_); + RankList group_devices; + if (dev_matrix.GetDevicesAlongDim(SizeToUint(axis), &group_devices) != SUCCESS) { + return FAILED; + } + // this group only has one device, don't need create the group + if (group_devices.size() == 1) { + MS_LOG(INFO) << "the group is empty"; + return SUCCESS; + } + + Group g = g_device_manager->CreateGroup(group_devices); + group->push_back(g); + return SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/construct_operator.h b/mindspore/ccsrc/parallel/tensor_layout/construct_operator.h new file mode 100644 index 00000000000..641b0975dd1 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/construct_operator.h @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_CONSTRUCT_OPERATOR_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_CONSTRUCT_OPERATOR_H_ + +#include +#include +#include + +#include "parallel/ops_info/operator_info.h" +#include "parallel/status.h" +#include "ir/value.h" + +namespace mindspore { +namespace parallel { +using Args = std::vector; + +class ConstructOperator { + public: + const int32_t DEFAULT = 0; + ConstructOperator() : dev_size_(0) {} + ~ConstructOperator() = default; + Status Init(const RankList& dev_list, const Shape& dev_matrix_shape); + Status ReshapeOP(Shape shape); + Status StridedSliceOP(Args args); + Status AllGatherOP(int32_t dev_dim); + Status SplitOP(int32_t split_count); + Status ConcatOP(int32_t concat_dim); + Status AlltoAllOP(Args args); + Operator GetOperator() const { return op_; } + void UpdateTensorShape(const Shape& tensor_shape) { tensor_shape_ = tensor_shape; } + + private: + Operator op_; + size_t dev_size_; + Shape tensor_shape_; + RankList dev_list_; + Shape dev_matrix_shape_; + Status CreateGroupByDim(size_t axis, std::vector* group); +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_CONSTRUCT_OPERATOR_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/layout_transfer.cc b/mindspore/ccsrc/parallel/tensor_layout/layout_transfer.cc new file mode 100644 index 00000000000..3321d751bbd --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/layout_transfer.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/layout_transfer.h" +#include "parallel/status.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { + +std::string LayoutTransfer::ToString() const { + std::ostringstream buffer; + buffer << std::endl << std::string("from_in_ tensor layout:" + from_in_.ToString()); + buffer << std::endl << std::string("to_in_ tensor layout:" + to_in_.ToString()); + return buffer.str(); +} + +LayoutTransfer::~LayoutTransfer() = default; + +Status LayoutTransfer::Init(const TensorLayout& from_in, const TensorLayout& to_in) { + from_in_ = from_in; + to_in_ = to_in; + MS_LOG(DEBUG) << "LayoutTransfer " << this->ToString(); + Status status = CheckValidTransfer(); + return status; +} + +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/layout_transfer.h b/mindspore/ccsrc/parallel/tensor_layout/layout_transfer.h new file mode 100644 index 00000000000..b892a87d305 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/layout_transfer.h @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_LAYOUT_TRANSFER_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_LAYOUT_TRANSFER_H_ + +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/tensor_layout.h" + +namespace mindspore { +namespace parallel { + +class LayoutTransfer { + public: + LayoutTransfer() = default; + virtual ~LayoutTransfer() = 0; + std::string ToString() const; + Status Init(const TensorLayout& from_in, const TensorLayout& to_in); + TensorLayout from_in() const { return from_in_; } + TensorLayout to_in() const { return to_in_; } + + protected: + bool IsSameTensorShape() const { return from_in_.IsSameTensorShape(to_in_); } + bool IsSameDeviceArrangement() const { return from_in_.IsSameDeviceArrangement(to_in_); } + + TensorLayout from_in_; + TensorLayout to_in_; + + private: + virtual Status CheckValidTransfer() = 0; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_LAYOUT_TRANSFER_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/map.cc b/mindspore/ccsrc/parallel/tensor_layout/map.cc new file mode 100644 index 00000000000..46c066b250b --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/map.cc @@ -0,0 +1,172 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/map.h" +#include +#include +#include +#include "parallel/status.h" +#include "utils/log_adapter.h" +#include "parallel/tensor_layout/shape_util.h" +#include "utils/convert_utils.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { + +Status Map::Init(const std::vector& array) { + Status status = Array::Init(array); + if (status != Status::SUCCESS) { + return Status::FAILED; + } + if (!IsValidMap()) { + MS_LOG(ERROR) << "invalid map " << this->ToString(); + return Status::FAILED; + } + return Status::SUCCESS; +} + +bool Map::IsValidMap() { + if (std::any_of(array_.begin(), array_.end(), [](int32_t value) { return ((value < 0) && (value != MAP_NONE)); })) { + return false; + } + // check that all none -1 value in array_ is different + std::vector sorted_array = array_; + std::sort(sorted_array.begin(), sorted_array.end()); + int32_t value = MAP_NONE; + for (auto& element : sorted_array) { + if (element == MAP_NONE) { + continue; + } + if (element == value) { + return false; + } + value = element; + } + return true; +} + +int32_t Map::GetMaxItem() const { + if (!array_.empty()) { + return *std::max_element(array_.begin(), array_.end()); + } else { + return MAP_NONE; + } +} + +int32_t Map::GetIndexByValue(int32_t value) const { + auto iter = find(array_.begin(), array_.end(), value); + if (iter != array_.end()) { + return static_cast(std::distance(array_.begin(), iter)); + } else { + return MAP_NONE; + } +} + +/* + * expand.size() should be equal to array_.size() + */ +std::shared_ptr Map::ExpandMapByNone(const Arrangement& expand_num_list) const { + if (expand_num_list.GetDimSize() != GetDimSize()) { + return nullptr; + } + std::vector new_shape; + for (uint32_t i = 0; i != GetDimSize(); i++) { + if (GetDimByIdx(i) == MAP_NONE) { + for (int32_t j = 0; j < expand_num_list.GetDimByIdx(i); j++) { + new_shape.push_back(MAP_NONE); + } + } else { + new_shape.push_back(GetDimByIdx(i)); + int32_t j = 1; + while (j < expand_num_list.GetDimByIdx(i)) { + new_shape.push_back(MAP_NONE); + j++; + } + } + } + auto map_new = std::make_shared(); + (void)map_new->Init(new_shape); + return map_new; +} + +/* + * expand.size() should be equal to array_.size() + */ +std::shared_ptr Map::ExpandMapByDecreaseNumber(const Arrangement& expand_num_list) const { + if (GetMaxItem() >= static_cast(expand_num_list.GetDimSize())) { + return nullptr; + } + std::vector new_shape; + for (uint32_t i = 0; i < GetDimSize(); i++) { + if (GetDimByIdx(i) == MAP_NONE) { + new_shape.push_back(MAP_NONE); + } else { + int32_t start_map = + expand_num_list.ComputeReverseAccumulateSumInReverseOrder()[static_cast(GetDimByIdx(i))]; + for (int32_t k = expand_num_list.GetDimByReverseIdx(static_cast(GetDimByIdx(i))) - 1; k >= 0; k--) { + new_shape.push_back(k + start_map); + } + } + } + auto map_new = std::make_shared(); + (void)map_new->Init(new_shape); + return map_new; +} + +std::shared_ptr> Map::ReMapVector(const std::vector& input_vector) const { + if (GetMaxItem() >= static_cast(input_vector.size())) { + return nullptr; + } + std::vector out; + Arrangement empty_arrangement; + for (uint32_t i = 0; i < GetDimSize(); i++) { + if (GetDimByIdx(i) == MAP_NONE) { + out.push_back(empty_arrangement); + } else { + out.push_back(input_vector[IntToUint(SizeToInt(input_vector.size()) - 1 - GetDimByIdx(i))]); + } + } + return std::make_shared>(out); +} + +bool Map::CheckNoneByIdxList(std::vector idx_list) const { + for (auto& value : idx_list) { + if (GetDimByIdx(SizeToUint(value)) != MAP_NONE) { + return false; + } + } + return true; +} + +Map Map::SqueezeMapByIdxList(std::vector idx_list) const { + std::vector out_shape; + for (size_t i = 0; i < GetDimSize(); i++) { + auto it = std::find(idx_list.begin(), idx_list.end(), i); + if (it == idx_list.end()) { + out_shape.push_back(GetDimByIdx(SizeToUint(i))); + } + } + if (out_shape.empty()) { + MS_LOG(ERROR) << "out_shape size is 0, this may not happen under current situation"; + out_shape.push_back(MAP_NONE); + } + Map out; + (void)out.Init(out_shape); + return out; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/map.h b/mindspore/ccsrc/parallel/tensor_layout/map.h new file mode 100644 index 00000000000..55ed3e55775 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/map.h @@ -0,0 +1,53 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_MAP_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_MAP_H_ + +#include +#include +#include +#include +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/array.h" +#include "parallel/tensor_layout/arrangement.h" + +namespace mindspore { +namespace parallel { +constexpr int32_t MAP_NONE = -1; + +class Map : public Array { + public: + Map() = default; + ~Map() override = default; + Status Init(const std::vector& array) override; + int32_t GetMaxItem() const; + int32_t GetIndexByValue(int32_t value) const; + std::shared_ptr ExpandMapByNone(const Arrangement& expand_num_list) const; + std::shared_ptr ExpandMapByDecreaseNumber(const Arrangement& expand_num_list) const; + std::shared_ptr> ReMapVector(const std::vector& input_vector) const; + bool CheckNoneByIdxList(std::vector idx_list) const; + Map SqueezeMapByIdxList(std::vector idx_list) const; + + private: + bool IsValidMap(); +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_MAP_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/redistribution_layout_transfer.cc b/mindspore/ccsrc/parallel/tensor_layout/redistribution_layout_transfer.cc new file mode 100644 index 00000000000..dea44ee60e2 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/redistribution_layout_transfer.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/redistribution_layout_transfer.h" +#include "parallel/status.h" +#include "parallel/tensor_layout/shape_util.h" +#include "parallel/tensor_layout/reshape_layout_transfer.h" + +namespace mindspore { +namespace parallel { + +Status RedistributionLayoutTransfer::CheckValidTransfer() { return Status::SUCCESS; } + +/* + * unify device arrangement between in_layout and out_layout + * after this function is called, + * in_step1_layout.device_arrangement and out_step1_layout.device_arrangement will be the same + */ +std::shared_ptr RedistributionLayoutTransfer::UnifyDeviceArrangement() const { + Arrangement in_arrangement; + Arrangement out_arrangement; + in_arrangement = from_in_.device_arrangement(); + out_arrangement = to_in_.device_arrangement(); + std::shared_ptr unify_arrangement_ptr = in_arrangement.GetUnifiedShape(out_arrangement); + if (unify_arrangement_ptr == nullptr) { + return nullptr; + } + std::shared_ptr from_out_ptr = from_in_.ExpandDeviceArrangement(*unify_arrangement_ptr); + if (from_out_ptr == nullptr) { + return nullptr; + } + std::shared_ptr to_out_ptr = to_in_.ExpandDeviceArrangement(*unify_arrangement_ptr); + if (to_out_ptr == nullptr) { + return nullptr; + } + ReshapeLayoutTransfer out; + Status status = out.Init(*from_out_ptr, *to_out_ptr); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(out); +} + +/* + * unify tensor shape between in_step1_layout.tensor_shape and out_step1_layout.tensor_shape + * after this function is called, + * in_step2_layout.tensor_shape and out_step2_layout.tensor_shape will be the same + */ +std::shared_ptr RedistributionLayoutTransfer::UnifyDeviceArrangementAndTensorShape() const { + std::shared_ptr unified_device_arrangement_ptr = UnifyDeviceArrangement(); + if (unified_device_arrangement_ptr == nullptr) { + return nullptr; + } + return unified_device_arrangement_ptr->UnifyDeviceArrangementAndTensorShape(); +} + +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/redistribution_layout_transfer.h b/mindspore/ccsrc/parallel/tensor_layout/redistribution_layout_transfer.h new file mode 100644 index 00000000000..6522b7f8c2f --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/redistribution_layout_transfer.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_REDISTRIBUTION_LAYOUT_TRANSFER_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_REDISTRIBUTION_LAYOUT_TRANSFER_H_ + +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/layout_transfer.h" +#include "parallel/tensor_layout/reshape_layout_transfer.h" + +namespace mindspore { +namespace parallel { + +class RedistributionLayoutTransfer : public LayoutTransfer { + public: + RedistributionLayoutTransfer() = default; + ~RedistributionLayoutTransfer() override = default; + std::shared_ptr UnifyDeviceArrangementAndTensorShape() const; + + private: + Status CheckValidTransfer() override; + std::shared_ptr UnifyDeviceArrangement() const; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_REDISTRIBUTION_LAYOUT_TRANSFER_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/redistribution_operator_infer.cc b/mindspore/ccsrc/parallel/tensor_layout/redistribution_operator_infer.cc new file mode 100644 index 00000000000..028fb5874aa --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/redistribution_operator_infer.cc @@ -0,0 +1,278 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/redistribution_operator_infer.h" + +#include + +#include "parallel/device_manager.h" + +namespace mindspore { +namespace parallel { + +Status RedistributionOperatorInfer::Init(const TensorLayout& tensor_layout, const Map& out_tensor_map, + RankList dev_list) { + in_tensor_map_ = tensor_layout.tensor_map(); + dev_mat_ = tensor_layout.device_arrangement(); + + if (in_tensor_map_.GetDimSize() == 0 || out_tensor_map.GetDimSize() != in_tensor_map_.GetDimSize()) { + MS_LOG(ERROR) << "Invalid input when initialize RedistributionOperatorInfer!"; + return Status::FAILED; + } + + cur_tensor_layout_ = tensor_layout; + out_tensor_map_ = out_tensor_map; + dev_list_ = std::move(dev_list); + + operator_list_.clear(); + operator_vector_.clear(); + output_info_vector_.clear(); + + if (constructor_.Init(dev_list_, dev_mat_.array()) != Status::SUCCESS) { + MS_LOG(ERROR) << "Init constructor failed"; + return Status::FAILED; + } + constructor_.UpdateTensorShape(cur_tensor_layout_.slice_shape().array()); + + size_t key = 0; + std::vector map = in_tensor_map_.array(); + for (int32_t item : map) { + map_[key++] = item; + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::InferRedistributionOperator() { + while (!map_.empty()) { + size_t len_global = operator_list_.size(); + + while (!map_.empty()) { + size_t len_split_by_axis = operator_list_.size(); + // split_by_axis operation + if (InferSplitByAxis() == Status::FAILED) { + return Status::FAILED; + } + // permute_by_axis operation + while (!map_.empty()) { + size_t len_permute_by_axis = operator_list_.size(); + if (InferPermuteByAxis() == Status::FAILED) { + return Status::FAILED; + } + if (len_permute_by_axis == operator_list_.size()) break; + } + if (len_split_by_axis == operator_list_.size()) break; + } + // concat_by_axis operation + if (InferConcatByAxis() == Status::FAILED) { + return Status::FAILED; + } + // break loop structure with concat_by_axis + if (len_global == operator_list_.size() && !map_.empty()) { + size_t index = map_.begin()->first; + int32_t in_dim = map_[index]; + map_[index] = NONE; + Args args = {SizeToInt(index), in_dim, dev_mat_.GetDimByReverseIdx(IntToUint(in_dim))}; + if (InsertOperator(CONCAT_BY_AXIS, args) == Status::FAILED) { + return Status::FAILED; + } + } + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::InferSplitByAxis() { + for (auto iter = map_.begin(); iter != map_.end();) { + uint32_t index = iter->first; + int32_t in_dim = iter->second; + int32_t out_dim = out_tensor_map_.GetDimByIdx(index); + if (in_dim == out_dim) { + (void)map_.erase(iter++); + continue; + } + if (in_dim == NONE && + !std::any_of(map_.begin(), map_.end(), + [out_dim](const RedistributionOperatorMap::value_type& a) { return a.second == out_dim; })) { + Args args = {dev_mat_.GetDimByReverseIdx(IntToUint(out_dim)), UintToInt(index), out_dim}; + if (InsertOperator(SPLIT_BY_AXIS, args) == Status::FAILED) { + MS_LOG(ERROR) << "Insert SplitByAxis Error!"; + return Status::FAILED; + } + (void)map_.erase(iter++); + } else { + (void)++iter; + } + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::InferPermuteByAxis() { + for (auto iter = map_.begin(); iter != map_.end();) { + uint32_t index = iter->first; + int32_t in_dim = map_[index]; + int32_t out_dim = out_tensor_map_.GetDimByIdx(index); + if (in_dim == out_dim) { + (void)map_.erase(iter++); + continue; + } + if (in_dim == NONE && + std::any_of(map_.begin(), map_.end(), + [out_dim](const RedistributionOperatorMap::value_type& a) { return a.second == out_dim; })) { + int32_t cat_dim = in_tensor_map_.GetIndexByValue(out_dim); + Args args_allconcat = {cat_dim, out_dim, dev_mat_.GetDimByReverseIdx(IntToUint(out_dim))}; + Args args_allsplit = {dev_mat_.GetDimByReverseIdx(IntToUint(out_dim)), UintToInt(index), out_dim}; + if (InsertOperator(CONCAT_BY_AXIS, args_allconcat) == Status::FAILED) { + MS_LOG(ERROR) << "Insert ConcatByAxis Error!"; + return Status::FAILED; + } + if (InsertOperator(SPLIT_BY_AXIS, args_allsplit) == Status::FAILED) { + MS_LOG(ERROR) << "Insert SplitByAxis Error!"; + return Status::FAILED; + } + (void)map_.erase(iter++); + map_[IntToSize(cat_dim)] = NONE; + } else { + (void)++iter; + } + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::InferConcatByAxis() { + for (auto iter = map_.begin(); iter != map_.end();) { + uint32_t index = iter->first; + int32_t in_dim = map_[index]; + int32_t out_dim = out_tensor_map_.GetDimByIdx(index); + if (in_dim != NONE && out_tensor_map_.GetIndexByValue(in_dim) == NONE) { + Args args = {SizeToInt(index), in_dim, dev_mat_.GetDimByReverseIdx(IntToUint(in_dim))}; + if (InsertOperator(CONCAT_BY_AXIS, args) == Status::FAILED) { + MS_LOG(ERROR) << "Insert ConcatByAxis Error!"; + return Status::FAILED; + } + if (out_dim == NONE) { + (void)map_.erase(iter++); + } else { + map_[index] = NONE; + (void)++iter; + } + } else { + (void)++iter; + } + } + return Status::SUCCESS; +} + +// Transfer communicative operators into primitives and insert them into vector +Status RedistributionOperatorInfer::InsertOperator(OperatorName name, Args args) { + OperatorR op = std::make_pair(name, args); + OperatorC op_cost = std::make_pair(op, cur_tensor_layout_.slice_shape().array()); + operator_list_.push_back(op_cost); + if (construct_op_flag_) { + if (name == SPLIT_BY_AXIS) { + if (TransferSplitByAxis(args) == Status::FAILED) { + return Status::FAILED; + } + } else if (name == PERMUTE_BY_AXIS) { + if (TransferPermuteByAxis(args) == Status::FAILED) { + return Status::FAILED; + } + } else { + if (TransferConcatByAxis(args) == Status::FAILED) { + return Status::FAILED; + } + } + constructor_.UpdateTensorShape(cur_tensor_layout_.slice_shape().array()); + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::TransferSplitByAxis(Args args) { + if (args.size() < 3) { + MS_LOG(ERROR) << "args size should not be less than 3!"; + return Status::FAILED; + } + uint32_t index = IntToUint(args[1]); + if (constructor_.StridedSliceOP(args) != Status::SUCCESS) { + return Status::FAILED; + } else { + operator_vector_.push_back(constructor_.GetOperator()); + output_info_vector_.push_back(std::make_pair(false, 0)); + } + if (cur_tensor_layout_.UpdateTensorMap(index, args[2]) == Status::FAILED) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::TransferPermuteByAxis(Args args) { + if (args.size() < 3) { + MS_LOG(ERROR) << "args size should not be less than 3!"; + return Status::FAILED; + } + if (constructor_.AlltoAllOP(args) != Status::SUCCESS) { + return Status::FAILED; + } else { + operator_vector_.push_back(constructor_.GetOperator()); + output_info_vector_.push_back(std::make_pair(false, 0)); + } + uint32_t index = IntToUint(args[1]); + int32_t val = args[2]; + int32_t out_dim = out_tensor_map_.GetDimByIdx(index); + + if (cur_tensor_layout_.UpdateTensorMap(IntToUint(val), NONE) == Status::FAILED) { + return Status::FAILED; + } + if (cur_tensor_layout_.UpdateTensorMap(index, out_dim) == Status::FAILED) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +Status RedistributionOperatorInfer::TransferConcatByAxis(Args args) { + if (args.size() < 3) { + MS_LOG(ERROR) << "args size should not be less than 3!"; + return Status::FAILED; + } + int32_t tensor_dim = args[0]; + int32_t dev_dim = args[1]; + int32_t split_count = args[2]; + if (constructor_.AllGatherOP(dev_dim) != Status::SUCCESS) { + return Status::FAILED; + } else { + operator_vector_.push_back(constructor_.GetOperator()); + output_info_vector_.push_back(std::make_pair(false, 0)); + } + if (tensor_dim != 0) { + if (constructor_.SplitOP(split_count) != Status::SUCCESS) { + return Status::FAILED; + } else { + operator_vector_.push_back(constructor_.GetOperator()); + output_info_vector_.push_back(std::make_pair(true, split_count)); + } + if (constructor_.ConcatOP(tensor_dim) != Status::SUCCESS) { + return Status::FAILED; + } else { + operator_vector_.push_back(constructor_.GetOperator()); + output_info_vector_.push_back(std::make_pair(false, 0)); + } + } + if (cur_tensor_layout_.UpdateTensorMap(IntToUint(tensor_dim), NONE) == Status::FAILED) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/redistribution_operator_infer.h b/mindspore/ccsrc/parallel/tensor_layout/redistribution_operator_infer.h new file mode 100644 index 00000000000..77955b5d1a2 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/redistribution_operator_infer.h @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_REDISTRIBUTION_OPERATOR_INFER_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_REDISTRIBUTION_OPERATOR_INFER_H_ + +#include +#include +#include +#include +#include +#include + +#include "parallel/tensor_layout/redistribution_layout_transfer.h" +#include "parallel/tensor_layout/construct_operator.h" +#include "utils/convert_utils.h" +namespace mindspore { +namespace parallel { + +using DeviceArrangement = std::vector; +using TensorMap = std::vector; +using TensorShape = std::vector; +using RedistributionOperatorMap = std::unordered_map; +using OperatorR = std::pair; +using OperatorC = std::pair; +using OperatorList = std::vector; + +class RedistributionOperatorInfer { + public: + const int NONE = -1; + explicit RedistributionOperatorInfer(bool construct_op_flag = true) : construct_op_flag_(construct_op_flag) {} + Status Init(const TensorLayout& tensor_layout, const Map& out_tensor_map, RankList dev_list); + ~RedistributionOperatorInfer() = default; + OperatorList operator_list() const { return operator_list_; } + OperatorVector operator_vector() const { return operator_vector_; } + OutPutInfoVector output_info_vector() const { return output_info_vector_; } + Status InferRedistributionOperator(); + + private: + Status InferSplitByAxis(); + Status InferPermuteByAxis(); + Status InferConcatByAxis(); + Status TransferSplitByAxis(Args args); + Status TransferPermuteByAxis(Args args); + Status TransferConcatByAxis(Args args); + Status InsertOperator(OperatorName name, Args args); + + OperatorList operator_list_; + OperatorVector operator_vector_; + OutPutInfoVector output_info_vector_; + Arrangement dev_mat_; + RedistributionOperatorMap map_; + Map in_tensor_map_; + Map out_tensor_map_; + TensorLayout cur_tensor_layout_; + ConstructOperator constructor_; + RankList dev_list_; + bool construct_op_flag_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_REDISTRIBUTION_OPERATOR_INFER_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/reshape_layout_transfer.cc b/mindspore/ccsrc/parallel/tensor_layout/reshape_layout_transfer.cc new file mode 100644 index 00000000000..1d56aa22203 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/reshape_layout_transfer.cc @@ -0,0 +1,131 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/reshape_layout_transfer.h" +#include "parallel/status.h" +#include "parallel/tensor_layout/shape_util.h" + +namespace mindspore { +namespace parallel { + +Status ReshapeLayoutTransfer::CheckValidTransfer() { + if (!IsSameDeviceArrangement()) { + return Status::FAILED; + } + return Status::SUCCESS; +} + +std::shared_ptr ReshapeLayoutTransfer::UnifyDeviceArrangementAndTensorShape() const { + bool is_unified = IsSameTensorShape(); + std::shared_ptr out_layout_ptr = std::make_shared(*this); + while (!is_unified) { + std::shared_ptr temp_layout_ptr = out_layout_ptr->ExtendFromTensorShapeByTo(); + out_layout_ptr = temp_layout_ptr->ExtendToTensorShapeByFrom(); + is_unified = out_layout_ptr->IsSameTensorShape(); + } + return out_layout_ptr; +} + +std::shared_ptr ReshapeLayoutTransfer::ExtendFromTensorShapeByTo() const { + std::shared_ptr out_ptr = std::make_shared(*this); + bool is_expanded = FromTensorShapeCanBeExpandByTo(); + while (!is_expanded) { + out_ptr = out_ptr->ExtendFromTensorShapeByExpandedTensorShape(); + if (out_ptr == nullptr) { + return nullptr; + } + is_expanded = out_ptr->FromTensorShapeCanBeExpandByTo(); + } + return out_ptr; +} + +std::shared_ptr ReshapeLayoutTransfer::ExtendToTensorShapeByFrom() const { + std::shared_ptr out_ptr = std::make_shared(*this); + bool is_expanded = ToTensorShapeCanBeExpandByFrom(); + while (!is_expanded) { + out_ptr = out_ptr->ExtendToTensorShapeByExpandedTensorShape(); + if (out_ptr == nullptr) { + return nullptr; + } + is_expanded = out_ptr->ToTensorShapeCanBeExpandByFrom(); + } + return out_ptr; +} + +bool ReshapeLayoutTransfer::FromTensorShapeCanBeExpandByTo() const { + return from_in_.TensorShapeCanBeExpanded(to_in_.tensor_shape()); +} + +bool ReshapeLayoutTransfer::ToTensorShapeCanBeExpandByFrom() const { + return to_in_.TensorShapeCanBeExpanded(from_in_.tensor_shape()); +} + +std::shared_ptr ReshapeLayoutTransfer::ExtendFromTensorShapeByExpandedTensorShape() const { + std::shared_ptr expanded_shape_ptr = ComputeExpandedFromTensorShapeByTo(); + if (expanded_shape_ptr == nullptr) { + return nullptr; + } + return ExpandFromTensorShapeAndExpandToDeviceArrangement(*expanded_shape_ptr); +} + +std::shared_ptr ReshapeLayoutTransfer::ExtendToTensorShapeByExpandedTensorShape() const { + std::shared_ptr exchanged_from_and_to_ptr = ExchangeFromAndTo(); + if (exchanged_from_and_to_ptr == nullptr) { + return nullptr; + } + std::shared_ptr expanded_shape_ptr = exchanged_from_and_to_ptr->ComputeExpandedFromTensorShapeByTo(); + if (expanded_shape_ptr == nullptr) { + return nullptr; + } + std::shared_ptr exchanged_out = + exchanged_from_and_to_ptr->ExpandFromTensorShapeAndExpandToDeviceArrangement(*expanded_shape_ptr); + return exchanged_out->ExchangeFromAndTo(); +} + +std::shared_ptr ReshapeLayoutTransfer::ExchangeFromAndTo() const { + ReshapeLayoutTransfer out; + Status status = out.Init(to_in_, from_in_); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(out); +} + +std::shared_ptr ReshapeLayoutTransfer::ExpandFromTensorShapeAndExpandToDeviceArrangement( + const Arrangement& expand_shape) const { + std::shared_ptr extend_tensor_shape_from_ptr = from_in_.ExpandTensorShape(expand_shape); + if (extend_tensor_shape_from_ptr == nullptr) { + return nullptr; + } + Arrangement unified_device_arrangement = extend_tensor_shape_from_ptr->device_arrangement(); + std::shared_ptr extend_device_arrangement_to_ptr = + to_in_.ExpandDeviceArrangement(unified_device_arrangement); + if (extend_device_arrangement_to_ptr == nullptr) { + return nullptr; + } + ReshapeLayoutTransfer out; + Status status = out.Init(*extend_tensor_shape_from_ptr, *extend_device_arrangement_to_ptr); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(out); +} + +std::shared_ptr ReshapeLayoutTransfer::ComputeExpandedFromTensorShapeByTo() const { + return from_in_.ComputeExpandedTensorShape(to_in_.tensor_shape()); +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/reshape_layout_transfer.h b/mindspore/ccsrc/parallel/tensor_layout/reshape_layout_transfer.h new file mode 100644 index 00000000000..9ad8e676359 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/reshape_layout_transfer.h @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_RESHAPE_LAYOUT_TRANSFER_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_RESHAPE_LAYOUT_TRANSFER_H_ + +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/layout_transfer.h" + +namespace mindspore { +namespace parallel { + +class ReshapeLayoutTransfer : public LayoutTransfer { + public: + ReshapeLayoutTransfer() = default; + ~ReshapeLayoutTransfer() override = default; + std::shared_ptr UnifyDeviceArrangementAndTensorShape() const; + std::shared_ptr ExtendFromTensorShapeByTo() const; + std::shared_ptr ExtendToTensorShapeByFrom() const; + std::shared_ptr ExtendFromTensorShapeByExpandedTensorShape() const; + std::shared_ptr ExtendToTensorShapeByExpandedTensorShape() const; + std::shared_ptr ExpandFromTensorShapeAndExpandToDeviceArrangement( + const Arrangement& expand_shape) const; + std::shared_ptr ExchangeFromAndTo() const; + + private: + Status CheckValidTransfer() override; + std::shared_ptr ComputeExpandedFromTensorShapeByTo() const; + bool FromTensorShapeCanBeExpandByTo() const; + bool ToTensorShapeCanBeExpandByFrom() const; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_RESHAPE_LAYOUT_TRANSFER_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/shape_util.cc b/mindspore/ccsrc/parallel/tensor_layout/shape_util.cc new file mode 100644 index 00000000000..54bb976032a --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/shape_util.cc @@ -0,0 +1,265 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/shape_util.h" +#include +#include "parallel/status.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { + +/* + * example: + * shape = [2, 8, 32] + * shape_accum = [2, 2 * 8, 2 * 8 * 32] + */ +Status ShapeToAccumulateProduct(const std::vector& shape, std::vector* shape_accum) { + MS_EXCEPTION_IF_NULL(shape_accum); + shape_accum->clear(); + int64_t size = 1; + for (auto iter = shape.begin(); iter < shape.end(); ++iter) { + size *= *iter; + if (size <= 0) { + MS_LOG(ERROR) << "element of shape should not be zero"; + return Status::FAILED; + } + shape_accum->push_back(size); + } + return Status::SUCCESS; +} + +/* + * example: + * shape = [2, 8, 32] + * shape_accum = [2 * 8 * 32, 8 * 32, 32] + * + */ +Status ShapeToAccumulateProductReverse(const std::vector& shape, std::vector* shape_accum) { + MS_EXCEPTION_IF_NULL(shape_accum); + shape_accum->clear(); + int64_t size = 1; + for (auto iter = shape.end() - 1; iter >= shape.begin(); --iter) { + size *= *iter; + if (size <= 0) { + MS_LOG(ERROR) << "element of shape should not be zero"; + return Status::FAILED; + } + (void)shape_accum->insert(shape_accum->begin(), size); + } + return Status::SUCCESS; +} + +/* + * example: + * shape_accum = [2, 2 * 8, 2 * 8 * 32] + * shape = [2, 8, 32] + * + */ +Status AccumulateProductToShape(const std::vector& shape_accum, std::vector* shape) { + MS_EXCEPTION_IF_NULL(shape); + shape->clear(); + int64_t value = 1; + for (auto iter = shape_accum.begin(); iter < shape_accum.end(); ++iter) { + if ((*iter) == 0) { + MS_LOG(ERROR) << "element of shape_accum should not be zero"; + return Status::FAILED; + } + if ((*iter) % value != 0) { + MS_LOG(ERROR) << "shape_accum is not a accumulate product in ascending order"; + return Status::FAILED; + } + shape->push_back(static_cast((*iter) / value)); + value = (*iter); + } + return Status::SUCCESS; +} + +/* + * example: + * shape_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * shape = [2, 8, 32] + */ +Status AccumulateProductReverseToShape(const std::vector& shape_accum_reverse, std::vector* shape) { + MS_EXCEPTION_IF_NULL(shape); + shape->clear(); + int64_t value = 1; + for (auto iter = shape_accum_reverse.end() - 1; iter >= shape_accum_reverse.begin(); --iter) { + if (*iter == 0) { + MS_LOG(ERROR) << "element of shape_accum should not be zero"; + return Status::FAILED; + } + if ((*iter) % value != 0) { + MS_LOG(ERROR) << "shape_accum is not a accumulate product in ascending order"; + return Status::FAILED; + } + (void)shape->insert(shape->begin(), static_cast((*iter) / value)); + value = *iter; + } + return Status::SUCCESS; +} + +/* + * example1: + * in1 = [2, 8] + * in2 = [4, 8] + * *out = [2, 4, 8] + * + * example2: + * in1 = [2, 4, 16] + * in2 = [8, 16] + * *out = [2, 4, 8, 16] + */ +Status UnifyAccumulateProduct(const std::vector& in1_accum, const std::vector& in2_accum, + std::vector* out_accum) { + MS_EXCEPTION_IF_NULL(out_accum); + out_accum->clear(); + auto in1_iter = in1_accum.begin(); + auto in2_iter = in2_accum.begin(); + while ((in1_iter < in1_accum.end()) || (in2_iter < in2_accum.end())) { + if ((*in1_iter <= 0) || (*in2_iter <= 0)) { + MS_LOG(ERROR) << "element of in1 and in2 must be larger than zero"; + return Status::FAILED; + } + if (*in1_iter < *in2_iter) { + out_accum->push_back(*in1_iter); + ++in1_iter; + continue; + } else if (*in1_iter == *in2_iter) { + out_accum->push_back(*in1_iter); + ++in1_iter; + ++in2_iter; + } else { + out_accum->push_back(*in2_iter); + ++in2_iter; + } + } + if ((in1_iter != in1_accum.end()) || (in2_iter != in2_accum.end())) { + MS_LOG(ERROR) << "last element of in1 and in2 must be equal"; + return Status::FAILED; + } + return Status::SUCCESS; +} + +/* + * example: + * in1 = [8, 4] + * in2 = [2, 16] + * out = [2, 4, 4] + */ +Status UnifyShape(const std::vector& in1, const std::vector& in2, std::vector* out) { + MS_EXCEPTION_IF_NULL(out); + std::vector in1_accum; + Status status = ShapeToAccumulateProduct(in1, &in1_accum); + if (status != Status::SUCCESS) { + return status; + } + std::vector in2_accum; + status = ShapeToAccumulateProduct(in2, &in2_accum); + if (status != Status::SUCCESS) { + return status; + } + std::vector out_accum; + status = UnifyAccumulateProduct(in1_accum, in2_accum, &out_accum); + if (status != Status::SUCCESS) { + return status; + } + status = AccumulateProductToShape(out_accum, out); + if (status != Status::SUCCESS) { + return status; + } + return status; +} + +/* + * example1: + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_accum_reverse = [2 * 8 * 32, 32, 8] + * out_accum_reverse = [2 * 8 * 4 * 8, 8 * 4 * 8, 4 * 8, 8] + * + * example2: + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_accum_reverse = [2 * 4 * 8, 4 * 8, 8] + * out_accum_reverse = [2 * 4 * 2 * 4 * 8, 4 * 2 * 4 * 8, 2 * 4 * 8, 4 * 8, 8] + */ +Status ExpandAccumulateProduct(const std::vector& in_accum_reverse, + const std::vector& expand_accum_reverse, + std::vector* out_accum_reverse) { + MS_EXCEPTION_IF_NULL(out_accum_reverse); + out_accum_reverse->clear(); + auto in_riter = in_accum_reverse.rbegin(); + auto expand_riter = expand_accum_reverse.rbegin(); + while (expand_riter != expand_accum_reverse.rend()) { + if (in_riter == in_accum_reverse.rend()) { + MS_LOG(ERROR) << "invalid ExpandAccumProd inputs"; + return Status::FAILED; + } + if (*in_riter > *expand_riter) { + (void)out_accum_reverse->insert(out_accum_reverse->begin(), *expand_riter); + ++expand_riter; + } else if (*in_riter == *expand_riter) { + (void)out_accum_reverse->insert(out_accum_reverse->begin(), *expand_riter); + ++in_riter; + ++expand_riter; + } else { + (void)out_accum_reverse->insert(out_accum_reverse->begin(), *in_riter); + ++in_riter; + } + } + while (in_riter != in_accum_reverse.rend()) { + (void)out_accum_reverse->insert(out_accum_reverse->begin(), *in_riter); + ++in_riter; + } + return Status::SUCCESS; +} + +/* + * example1: + * in = [2, 8, 32] + * expand = [16, 4, 8] + * out = [2, 8, 4, 8] + * + * example2: + * in = [2, 8, 32] + * expand = [2, 4, 8] + * out = [2, 4, 2, 4, 8] + */ +Status ExpandShape(const std::vector& in, const std::vector& expand, std::vector* out) { + MS_EXCEPTION_IF_NULL(out); + std::vector in_accum_reverse; + Status status = ShapeToAccumulateProductReverse(in, &in_accum_reverse); + if (status != Status::SUCCESS) { + return status; + } + std::vector expand_accum_reverse; + status = ShapeToAccumulateProductReverse(expand, &expand_accum_reverse); + if (status != Status::SUCCESS) { + return status; + } + std::vector out_accum_reverse; + status = ExpandAccumulateProduct(in_accum_reverse, expand_accum_reverse, &out_accum_reverse); + if (status != Status::SUCCESS) { + return status; + } + status = AccumulateProductReverseToShape(out_accum_reverse, out); + if (status != Status::SUCCESS) { + return status; + } + return status; +} + +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/shape_util.h b/mindspore/ccsrc/parallel/tensor_layout/shape_util.h new file mode 100644 index 00000000000..5451af063ee --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/shape_util.h @@ -0,0 +1,174 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_SHAPE_UTIL_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_SHAPE_UTIL_H_ + +#include +#include +#include +#include +#include + +#include "parallel/status.h" + +namespace mindspore { +namespace parallel { + +/* + * compute the accumulating product of all the values in shape from left to right, + * the accumulating results are saved in shape_accum from left to right + * + * given a shape = [d_n-1, d_n-2, ..., d_0](d_i > 0, i=0,1,...,n-1, elements of shape must be larger than zero), + * then *shape_accum = [d_n-1, d_n-1 * d_n-2, d_n-1 * d_n-2 * d_n-3, ..., d_n-1 * d_n-2 * ... *d_0] + * + * example: + * shape = [2, 8, 32] + * shape_accum = [2, 2 * 8, 2 * 8 * 32] + * + */ +Status ShapeToAccumulateProduct(const std::vector& shape, std::vector* shape_accum); + +/* + * compute the accumulating product of all the values in shape from right to left, + * the accumulating results are saved in shape_accum from right to left + * + * given a shape = [d_n-1, d_n-2, ..., d_0](d_i > 0, i=0,1,...,n-1, elements of shape must be larger than zero), + * then *shape_accum = [d_n-1 * d_n-2 * ... *d_0, d_n-2 * d_n-3 * ... *d_0, ..., d_0] + * + * example: + * shape = [2, 8, 32] + * shape_accum = [2 * 8 * 32, 8 * 32, 32] + * + */ +Status ShapeToAccumulateProductReverse(const std::vector& shape, std::vector* shape_accum); + +/* + * compute the original shape from the accumulating product shape_accum, + * elements of shape_accum is saved from left to right, + * given shape_accum = [accum_n-1, accum_n-2, accum_n-3, ..., accum_0] + * (accum_i > 0, i=0,1,...,n-1, elements of shape_accum must be larger than zero), + * (accum_i-1 % accum_i == 0, i=1,...,n-1) + * then *shape = [accum_n-2/accum_n-1, accum_n-3/accum_n-2, ..., accum_0/accum_1] + * + * example: + * shape_accum = [2, 2 * 8, 2 * 8 * 32] + * shape = [2, 8, 32] + * + */ +Status AccumulateProductToShape(const std::vector& shape_accum, std::vector* shape); + +/* + * compute the original shape from the accumulating product shape_accum, + * elements of shape_accum is saved from right to left, + * given shape_accum_reverse = [accum_n-1, accum_n-2, accum_n-3, ..., accum_0] + * (accum_i > 0, i=0,1,...,n-1, elements of shape_accum must be larger than zero), + * (accum_i % accum_i-1 == 0, i=1,...,n-1) + * then *shape = [accum_n-1/accum_n-2, accum_n-2/accum_n-1, ..., accum_1/accum_0] + * + * example: + * shape_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * shape = [2, 8, 32] + * + */ +Status AccumulateProductReverseToShape(const std::vector& shape_accum_reverse, std::vector* shape); + +/* + * given two accumulate product in1_accum and in2_accum, compute the union of in1_accum and in2_accum, + * results are saved in out. + * i.e. *out_accum = in1_accum U in2_accum + * elements of out are saved in increasing order + * + * example1: + * in1_accum = [2, 8] + * in2_accum = [4, 8] + * out_accum = [2, 4, 8] + * + * example2: + * in1_accum = [2, 4, 16] + * in2_accum = [8, 16] + * out_accum = [2, 4, 8, 16] + */ +Status UnifyAccumulateProduct(const std::vector& in1_accum, const std::vector& in2_accum, + std::vector* out_accum); + +/* + * given two shape in1 = [din1_n-1, din1_n-2, ..., din1_0] and in2 = [din2_m-1, din2_m-2, ..., din2_m] + * size = din1_n-1 * din1n-2 * ... * din1_0 = din2_m-1 * din2_m-2 * ... * din2_0 + * find *out = [dout_k-1, dout_k-2, ..., dout_0], s.t. dout_k-1 * dout_k-2 * ... * dout_0 = size and + * suppose in1_accum, in2_accum, and *out_accum is the ShapeToAccumulateProduct result of in1, in2, and *out + * then for each din1_i in in1_accum, din1_i is in *out_accumulate, + * for each din2_i in in2_accum, din2_i is in *out_accumulate + * + * example: + * in1 = [8, 4] + * in2 = [2, 16] + * out = [2, 4, 4] + */ +Status UnifyShape(const std::vector& in1, const std::vector& in2, std::vector* out); + +/* + * given two accumulate product in reverse order of in and expand, + * in_accum_reverse = [din_n-1, din_n-2, ..., din_0] and expand_pos_reverse = [dexp_n-1, dexp_n-2, ..., dexp_0], + * i.e. in_accum_reverse is the ShapeToAccumulateProductReverse result of a shape in, + * expand_accum_reverse is the ShapeToAccumulateProductReverse result of a shape expand, + * compute the accumulate product in reverse order out_accum_reverse = [dout_k-1, dout_k-2, ..., dout_0], + * s.t. elements in out_accum_reverse are union of elements in in_accum_reverse and expand_accum_reverse + * (out_accum_reverse = in_accum_reverse U expand_accum_reverse), and + * out_accum_reverse is the ShapeToAccumulateProductReverse result of shape expand, + * i.e. dout_i > 0, i=0,1,...,k-1, elements of out_accum_reverse must be larger than zero, + * dout_i-1 % dout_i == 0, i=1,...,k-1 + * + * example1: + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_accum_reverse = [2 * 8 * 32, 32, 8] + * out_accum_reverse = [2 * 8 * 4 * 8, 8 * 4 * 8, 4 * 8, 8] + * + * example2: + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_accum_reverse = [2 * 4 * 8, 4 * 8, 8] + * out_accum_reverse = [2 * 4 * 2 * 4 * 8, 4 * 2 * 4 * 8, 2 * 4 * 8, 4 * 8, 8] + */ +Status ExpandAccumulateProduct(const std::vector& in_accum_reverse, + const std::vector& expand_accum_reverse, + std::vector* out_accum_reverse); + +/* + * given a shape in = [din_n-1, din_n-2, ..., d_0], and the expand shape expand= [dexp_m-1, dexp_m-2, ..., dexp_0], + * compute the expended shape out = [dout_k-1, dout_k-2, ..., dout_0], + * s.t. dout_k-1 * dout_k-2 * ...* dout_0 = din_n-1 * din_n-2 * ... * d_0 + * suppose in_accum_reverse is the ShapeToAccumulateProductReverse result of in, + * expand_accum_reverse is the ShapeToAccumulateProductReverse result of expand, + * out_accum_reverse is the ShapeToAccumulateProductReverse result of out, + * then out_accum_reverse is the union of in_accum_reverse and expand_accum_reverse + * (out_accum_reverse = in_accum_reverse U expand_accum_reverse) + * + * example1: + * in = [2, 8, 32] + * expand = [16, 4, 8] + * out = [2, 8, 4, 8] + * + * example2: + * in = [2, 8, 32] + * expand = [2, 4, 8] + * out = [2, 4, 2, 4, 8] + */ +Status ExpandShape(const std::vector& in, const std::vector& expand, std::vector* out); + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_SHAPE_UTIL_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/tensor_info.h b/mindspore/ccsrc/parallel/tensor_layout/tensor_info.h new file mode 100644 index 00000000000..b24df6bbf2c --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/tensor_info.h @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_INFO_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_INFO_H_ + +#include +#include +#include +#include + +#include "parallel/status.h" +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/device_matrix.h" + +namespace mindspore { +namespace parallel { + +using Shapes = std::vector; + +class TensorInfo { + public: + TensorInfo(const TensorLayout& tensor_layout, Shape shape, Shape slice_shape) + : tensor_layout_(tensor_layout), shape_(std::move(shape)), slice_shape_(std::move(slice_shape)) {} + explicit TensorInfo(const TensorLayout& tensor_layout) : tensor_layout_(tensor_layout) { + shape_ = tensor_layout.tensor_shape().array(); + slice_shape_ = tensor_layout.slice_shape().array(); + } + // trivial default constructor will not initialize c language types. + TensorInfo() = default; + ~TensorInfo() = default; + TensorLayout tensor_layout() const { return tensor_layout_; } + Shape slice_shape() const { return slice_shape_; } + Shape shape() const { return shape_; } + void set_reduce_dim(const std::vector& dim) { reduce_dim_ = dim; } + std::vector reduce_dim() const { return reduce_dim_; } + + private: + TensorLayout tensor_layout_; + Shape shape_; + Shape slice_shape_; + // reduce method's reduce dim + std::vector reduce_dim_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_INFO_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/tensor_layout.cc b/mindspore/ccsrc/parallel/tensor_layout/tensor_layout.cc new file mode 100644 index 00000000000..9463b99ce63 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/tensor_layout.cc @@ -0,0 +1,395 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/tensor_layout.h" +#include +#include +#include "parallel/status.h" +#include "utils/log_adapter.h" +#include "parallel/tensor_layout/shape_util.h" +#include "parallel/tensor_layout/array.h" +#include "ir/value.h" +#include "parallel/device_matrix.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { + +std::string TensorLayout::ToString() const { return StandardToString() + OriginToString(); } + +std::string TensorLayout::StandardToString() const { + std::ostringstream buffer; + buffer << std::endl << std::string("device arrangement = " + device_arrangement_.ToString()); + buffer << std::endl << std::string("tensor map = " + tensor_map_.ToString()); + buffer << std::endl << std::string("tensor shape = " + tensor_shape_.ToString()); + return buffer.str(); +} + +std::string TensorLayout::OriginToString() const { + std::ostringstream buffer; + buffer << std::endl << std::string("device arrangement origin = " + device_arrangement_origin_.ToString()); + buffer << std::endl << std::string("tensor map origin = " + tensor_map_origin_.ToString()); + buffer << std::endl << std::string("tensor shape origin = " + tensor_shape_origin_.ToString()); + return buffer.str(); +} + +Status TensorLayout::Init(const Arrangement& device_arrangement, const Map& tensor_map, + const Arrangement& tensor_shape) { + device_arrangement_origin_ = device_arrangement; + tensor_map_origin_ = tensor_map; + tensor_shape_origin_ = tensor_shape; + device_arrangement_ = device_arrangement; + tensor_map_ = tensor_map; + tensor_shape_ = tensor_shape; + if (IsValidTensorLayout()) { + MS_LOG(DEBUG) << "valid origin tensor layout " << this->OriginToString(); + RemoveElementEqualToOneInDeviceArrangement(); + MS_LOG(DEBUG) << "standard tensor layout " << this->StandardToString(); + return Status::SUCCESS; + } else { + MS_LOG(ERROR) << "invalid origin tensor layout " << this->OriginToString(); + return Status::FAILED; + } +} + +Status TensorLayout::InitFromVector(const std::vector& device_arrangement, + const std::vector& tensor_map, const std::vector& tensor_shape) { + if (device_arrangement_origin_.Init(device_arrangement) != SUCCESS) { + return FAILED; + } + if (tensor_map_origin_.Init(tensor_map) != SUCCESS) { + return FAILED; + } + if (tensor_shape_origin_.Init(tensor_shape) != SUCCESS) { + return FAILED; + } + if (Init(device_arrangement_origin_, tensor_map_origin_, tensor_shape_origin_) != SUCCESS) { + return FAILED; + } + return SUCCESS; +} + +bool TensorLayout::IsValidTensorLayout() const { + if (tensor_map_origin_.GetMaxItem() >= static_cast(device_arrangement_origin_.GetDimSize())) { + MS_LOG(ERROR) << "the max element in tensor_map_origin_ must be smaller than device_arrangement_origin_ size!"; + return false; + } + if (tensor_map_origin_.GetDimSize() != tensor_shape_origin_.GetDimSize()) { + MS_LOG(ERROR) << "tensor_map_origin_ size must be equal to tensor_shape_origin_ size!"; + return false; + } + if (!TensorShapeDimensionIsDividedBySplitDeviceDimension()) { + MS_LOG(ERROR) << "TensorShapeDimensionIsDividedBySplitDeviceDimension failed!"; + return false; + } + return true; +} + +bool TensorLayout::TensorShapeDimensionIsDividedBySplitDeviceDimension() const { + for (uint32_t i = 0; i < tensor_map_.GetDimSize(); i++) { + if (tensor_map_.GetDimByIdx(i) != -1) { + int32_t divisor = GetSliceNumByTensorDimensionIndex(i); + if (divisor == 0) { + MS_LOG(ERROR) << "GetSliceNumByTensorDimensionIndex is 0"; + return false; + } + if (tensor_shape_.GetDimByIdx(i) % divisor != 0) { + return false; + } + } + } + return true; +} + +void TensorLayout::RemoveElementEqualToOneInDeviceArrangement() { + std::vector device_arrangement_shape; + std::vector tensor_map_shape = tensor_map_origin_.array(); + uint32_t dev_num = SizeToUint(device_arrangement_origin_.GetDimSize()); + int32_t dev_num_left = SizeToInt(device_arrangement_origin_.GetDimSize()); + for (uint32_t i = 0; i < dev_num; i++) { + if (device_arrangement_origin_.GetDimByIdx(i) == 1) { + int32_t idx = GetTensorDimensionIndexByDeviceDimensionIndex(static_cast(dev_num - 1 - i)); + if (idx != -1) { + tensor_map_shape[static_cast(idx)] = -1; + } + for (auto& value : tensor_map_shape) { + if (value >= dev_num_left - 1 - static_cast(i)) { + value--; + } + } + continue; + } + device_arrangement_shape.push_back(device_arrangement_origin_.GetDimByIdx(i)); + } + (void)device_arrangement_.Init(device_arrangement_shape); + (void)tensor_map_.Init(tensor_map_shape); + tensor_shape_ = tensor_shape_origin_; +} + +// if idx is not in tensor_map, return -1 +int32_t TensorLayout::GetTensorDimensionIndexByDeviceDimensionIndex(int32_t idx) const { + return tensor_map_.GetIndexByValue(idx); +} + +// tensor_map_.GetDimByIdx(idx) should not be -1 +int32_t TensorLayout::GetSliceDeviceDimensionByTensorDimensionIndex(uint32_t idx) const { + return static_cast(device_arrangement_.GetDimSize()) - 1 - tensor_map_.GetDimByIdx(idx); +} + +// tensor_map_.GetDimByIdx(idx) should not be -1 +int32_t TensorLayout::GetSliceNumByTensorDimensionIndex(uint32_t idx) const { + return device_arrangement_.GetDimByIdx(static_cast(GetSliceDeviceDimensionByTensorDimensionIndex(idx))); +} + +std::shared_ptr TensorLayout::ExpandTensorShape(const Arrangement& expanded_shape) const { + std::shared_ptr expanded_arrangement_ptr = ComputeArrangementByExpandedShape(expanded_shape); + if (expanded_arrangement_ptr == nullptr) { + return nullptr; + } + std::shared_ptr temp_tensor_layout_ptr = ExpandDeviceArrangement(*expanded_arrangement_ptr); + if (temp_tensor_layout_ptr == nullptr) { + return nullptr; + } + return temp_tensor_layout_ptr->ExpandTensorShapeWithoutExtendDeviceArrangement(expanded_shape); +} + +/* + * example1: + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_tensor_shape = [128, 4, 2, 512], + * => + * out_device_arrangement = [8, 2, 2] + */ +std::shared_ptr TensorLayout::ComputeArrangementByExpandedShape(const Arrangement& tensor_shape) const { + std::shared_ptr> expand_list_ptr = tensor_shape_.GetExpandShapeList(tensor_shape); + if (expand_list_ptr == nullptr) { + return nullptr; + } + std::vector re_map_expand_list; + Arrangement empty_arrangement; + for (int32_t i = static_cast(device_arrangement_.GetDimSize()) - 1; i >= 0; i--) { + if (tensor_map_.GetIndexByValue(i) < 0) { + re_map_expand_list.push_back(empty_arrangement); + } else { + re_map_expand_list.push_back((*expand_list_ptr)[IntToUint(tensor_map_.GetIndexByValue(i))]); + } + } + std::shared_ptr new_arrangement_ptr = + device_arrangement_.GetExpandedShapeByExpandListRemoveLeft(re_map_expand_list); + return new_arrangement_ptr; +} + +/* + * example1: + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_tensor_shape = [8, 64, 4, 256] + * => + * out_device_arrangement = [8, 4], + * out_tensor_map = [1, -1, 0, -1], + */ +std::shared_ptr TensorLayout::ExpandTensorShapeWithoutExtendDeviceArrangement( + const Arrangement& expanded_shape) const { + std::shared_ptr, Arrangement>> expand_list_pair_ptr = + tensor_shape_.GetExpandShapeListPair(expanded_shape); + if (expand_list_pair_ptr == nullptr) { + return nullptr; + } + std::shared_ptr tensor_map_new_ptr = tensor_map_.ExpandMapByNone(expand_list_pair_ptr->second); + if (tensor_map_new_ptr == nullptr) { + return nullptr; + } + TensorLayout tensor_layout_new; + Status status = tensor_layout_new.Init(device_arrangement_, *tensor_map_new_ptr, expanded_shape); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(tensor_layout_new); +} + +/* + * example1: + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 2, 2] + * => + * out_tensor_map = [3, 2, 1, 0], + * out_tensor_shape = [4, 128, 2, 512] + * + * example2: + * in_device_arrangement = [8, 4], + * in_tensor_map = [0, 1], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 2, 2] + * => + * out_tensor_map = [1, 0, 3, 2], + * out_tensor_shape = [2, 256, 4, 256] + * + * example3: + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, -1], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 2, 2] + * => + * out_tensor_map = [3, 2, -1], + * out_tensor_shape = [4, 128, 1024] + * + * example4: + * in_device_arrangement = [8, 4], + * in_tensor_map = [0, 1], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 4] + * => + * out_tensor_map = [0, 2, 1], + * out_tensor_shape = [512, 4, 256] + */ +std::shared_ptr TensorLayout::ExpandDeviceArrangement(const Arrangement& expanded_arrangement) const { + std::shared_ptr, Arrangement>> expand_list_pair_ptr = + device_arrangement_.GetExpandShapeListPair(expanded_arrangement); + if (expand_list_pair_ptr == nullptr) { + return nullptr; + } + std::shared_ptr tensor_map_new_ptr = tensor_map_.ExpandMapByDecreaseNumber(expand_list_pair_ptr->second); + if (tensor_map_new_ptr == nullptr) { + return nullptr; + } + std::shared_ptr> re_map_shape_list_ptr = + tensor_map_.ReMapVector(expand_list_pair_ptr->first); + if (re_map_shape_list_ptr == nullptr) { + return nullptr; + } + std::shared_ptr tensor_shape_new_ptr = + tensor_shape_.GetExpandedShapeByExpandListReserveLeft(*re_map_shape_list_ptr); + if (tensor_shape_new_ptr == nullptr) { + return nullptr; + } + TensorLayout tensor_layout_new; + Status status = tensor_layout_new.Init(expanded_arrangement, *tensor_map_new_ptr, *tensor_shape_new_ptr); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(tensor_layout_new); +} + +bool TensorLayout::TensorShapeCanBeExpanded(const Arrangement& expand_shape) const { + std::vector in_expand_shape_shape; + Status status = ExpandShape(tensor_shape_.array(), expand_shape.array(), &in_expand_shape_shape); + if (status != Status::SUCCESS) { + return false; + } + return (in_expand_shape_shape == tensor_shape_.array()); +} + +std::shared_ptr TensorLayout::ComputeExpandedTensorShape(const Arrangement& expand_shape) const { + std::vector in_expand_shape_shape; + Status status = ExpandShape(tensor_shape_.array(), expand_shape.array(), &in_expand_shape_shape); + if (status != Status::SUCCESS) { + return nullptr; + } + Arrangement expanded_shape; + status = expanded_shape.Init(in_expand_shape_shape); + if (status != Status::SUCCESS) { + return nullptr; + } + return std::make_shared(expanded_shape); +} + +Arrangement TensorLayout::slice_shape() const { + std::vector shape; + for (uint32_t index = 0; index < tensor_map_.GetDimSize(); index++) { + int32_t dim = tensor_map_.GetDimByIdx(index); + int32_t num = tensor_shape_.GetDimByIdx(index); + if (dim == -1) { + shape.push_back(num); + } else { + int32_t divisor = device_arrangement_.GetDimByReverseIdx(IntToUint(dim)); + shape.push_back(num / divisor); + } + } + Arrangement new_tensor_shape; + if (new_tensor_shape.Init(shape) == Status::FAILED) { + ValuePtr ptr = MakeValue(shape); + MS_LOG(EXCEPTION) << "Can't get slice shape when initialize a new shape " << ptr->ToString(); + } else { + return new_tensor_shape; + } +} + +Status TensorLayout::UpdateTensorMap(uint32_t index, int32_t value) { + if (index >= tensor_map_.GetDimSize()) { + MS_LOG(ERROR) << "Index is out of the size of the tensor map!"; + return Status::FAILED; + } + Shape shape = tensor_map_.array(); + shape[index] = value; + if (tensor_map_.Init(shape) == Status::FAILED) { + MS_LOG(ERROR) << "Update tensor map failed!"; + return Status::FAILED; + } + return Status::SUCCESS; +} + +bool TensorLayout::operator==(const TensorLayout& t1) const { + return (IsSameDeviceArrangement(t1) && IsSameTensorMap(t1) && IsSameTensorShape(t1)); +} + +/* + * remove elements equal to 1 in tensor_shape, if all elements are 1, squeeze the tensor_shape to [ 1 ] + * example 1: + * original tensor layout: + * device arrangement = [ 8 ] + * tensor map = [ 0 -1 -1 -1 ] + * tensor shape = [ 128 64 1 1 ] + * return tensor layout: + * device arrangement = [ 8 ] + * tensor map = [ 0 -1 ] + * tensor shape = [ 128 64 ] + * + * example 2: + * device arrangement = [ 8 ] + * tensor map = [ -1 -1 -1 -1 ] + * tensor shape = [ 1 1 1 1 ] + * return tensor layout: + * device arrangement = [ 8 ] + * tensor map = [ -1 ] + * tensor shape = [ 1 ] + */ +TensorLayout TensorLayout::SqueezeShape() const { + TensorLayout out; + Map out_map; + Arrangement out_shape; + if (tensor_shape_.size() == 1) { + (void)out_map.Init({MAP_NONE}); + (void)out_shape.Init({1}); + (void)out.Init(device_arrangement_, out_map, out_shape); + return out; + } + std::vector squeeze_list = tensor_shape_.GetSqueezeIdx(); + if (!tensor_map_.CheckNoneByIdxList(squeeze_list)) { + MS_LOG(ERROR) << "CheckNoneByIdxList failed, this may not happen under current situation"; + return *this; + } + out_shape = tensor_shape_.GetSqueezeArrangement(); + out_map = tensor_map_.SqueezeMapByIdxList(squeeze_list); + (void)out.Init(device_arrangement_, out_map, out_shape); + return out; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/tensor_layout.h b/mindspore/ccsrc/parallel/tensor_layout/tensor_layout.h new file mode 100644 index 00000000000..0db356b8b10 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/tensor_layout.h @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_LAYOUT_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_LAYOUT_H_ + +#include +#include +#include +#include +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/arrangement.h" +#include "parallel/tensor_layout/map.h" +#include "utils/convert_utils.h" +#include "parallel/device_manager.h" + +namespace mindspore { +namespace parallel { + +class TensorLayout { + public: + TensorLayout() = default; + ~TensorLayout() = default; + std::string ToString() const; + std::string StandardToString() const; + std::string OriginToString() const; + Status Init(const Arrangement& device_arrangement, const Map& tensor_map, const Arrangement& tensor_shape); + Status InitFromVector(const std::vector& device_arrangement, const std::vector& tensor_map, + const std::vector& tensor_shape); + + Arrangement device_arrangement() const { return device_arrangement_; } + + Map tensor_map() const { return tensor_map_; } + + Arrangement tensor_shape() const { return tensor_shape_; } + + Map origin_tensor_map() const { return tensor_map_origin_; } + + std::shared_ptr ExpandTensorShape(const Arrangement& expanded_shape) const; + + std::shared_ptr ExpandDeviceArrangement(const Arrangement& expanded_arrangement) const; + + bool IsSameTensorShape(const TensorLayout& tensor_layout) const { + return (tensor_shape_ == tensor_layout.tensor_shape()); + } + + bool IsSameDeviceArrangement(const TensorLayout& tensor_layout) const { + return (device_arrangement_ == tensor_layout.device_arrangement()); + } + + bool IsSameTensorMap(const TensorLayout& tensor_layout) const { return (tensor_map_ == tensor_layout.tensor_map()); } + + bool operator==(const TensorLayout& t1) const; + + bool TensorShapeCanBeExpanded(const Arrangement& expanded_shape) const; + + std::shared_ptr ComputeExpandedTensorShape(const Arrangement& expand_shape) const; + + Arrangement slice_shape() const; + + Status UpdateTensorMap(uint32_t index, int32_t value); + + TensorLayout SqueezeShape() const; + + private: + std::shared_ptr ExpandTensorShapeWithoutExtendDeviceArrangement( + const Arrangement& expanded_shape) const; + std::shared_ptr ComputeArrangementByExpandedShape(const Arrangement& tensor_shape) const; + bool IsValidTensorLayout() const; + void RemoveElementEqualToOneInDeviceArrangement(); + int32_t GetSliceDeviceDimensionByTensorDimensionIndex(uint32_t idx) const; + int32_t GetSliceNumByTensorDimensionIndex(uint32_t idx) const; + bool TensorShapeDimensionIsDividedBySplitDeviceDimension() const; + int32_t GetTensorDimensionIndexByDeviceDimensionIndex(int32_t idx) const; + + Arrangement device_arrangement_origin_; + Map tensor_map_origin_; + Arrangement tensor_shape_origin_; + Arrangement device_arrangement_; + Map tensor_map_; + Arrangement tensor_shape_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_LAYOUT_H_ diff --git a/mindspore/ccsrc/parallel/tensor_layout/tensor_redistribution.cc b/mindspore/ccsrc/parallel/tensor_layout/tensor_redistribution.cc new file mode 100644 index 00000000000..365a2be1cb0 --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/tensor_redistribution.cc @@ -0,0 +1,196 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "parallel/tensor_layout/tensor_redistribution.h" +#include +#include +#include +#include "parallel/status.h" +#include "parallel/tensor_layout/shape_util.h" +#include "common/utils.h" + +namespace mindspore { +namespace parallel { + +Status TensorRedistribution::Init(const TensorLayout& from, const TensorLayout& to, const RankList& dev_list) { + from_origin_ = from; + to_origin_ = to; + if (from_origin_.tensor_shape().size() != to_origin_.tensor_shape().size()) { + MS_LOG(ERROR) << "from shape size must be equal to to shape size!"; + MS_LOG(ERROR) << "reshape from_origin_ " << from_origin_.ToString(); + MS_LOG(ERROR) << "reshape to_origin_ " << to_origin_.ToString(); + return Status::FAILED; + } + + dev_list_ = dev_list; + from_ = from_origin_.SqueezeShape(); + to_ = to_origin_.SqueezeShape(); + return Status::SUCCESS; +} + +RedistributionOpListPtr TensorRedistribution::InferTensorRedistributionOperatorList() { + // Step 1: Match device arrangement between from_ and to_ + RedistributionLayoutTransfer layout_transfer; + Status status = layout_transfer.Init(from_, to_); + if (status != Status::SUCCESS) { + return nullptr; + } + std::shared_ptr ptr = layout_transfer.UnifyDeviceArrangementAndTensorShape(); + if (ptr == nullptr) { + MS_LOG(ERROR) << "Infer tensor layout return nullptr!"; + return nullptr; + } + TensorLayout from_layout = ptr->from_in(); + TensorLayout to_layout = ptr->to_in(); + MS_LOG(DEBUG) << "reshape from_layout " << from_layout.ToString(); + MS_LOG(DEBUG) << "reshape to_layout " << to_layout.ToString(); + MS_LOG(DEBUG) << "reshape from_origin_ " << from_origin_.ToString(); + MS_LOG(DEBUG) << "reshape to_origin_ " << to_origin_.ToString(); + MS_LOG(DEBUG) << "reshape from_ " << from_.ToString(); + MS_LOG(DEBUG) << "reshape to_ " << to_.ToString(); + // Step 2: Infer redistribution and insert operators + RedistributionOperatorInfer operator_infer(construct_op_flag_); + if (operator_infer.Init(from_layout, to_layout.tensor_map(), dev_list_) == Status::FAILED) { + MS_LOG(ERROR) << "Init operatorInfer failed!"; + return nullptr; + } + OperatorVector operator_vector; + OutPutInfoVector output_info_vector; + if (operator_infer.InferRedistributionOperator() != Status::SUCCESS) { + MS_LOG(ERROR) << "Infer redistribution failed!"; + return nullptr; + } else { + operator_vector = operator_infer.operator_vector(); + output_info_vector = operator_infer.output_info_vector(); + operator_list_ = operator_infer.operator_list(); + } + + // Step 3: Infer reshape and insert operators + if (InferReshape(from_layout, to_layout, &operator_vector, &output_info_vector) != Status::SUCCESS) { + MS_LOG(ERROR) << "Construct Reshape operator failed!"; + return nullptr; + } + + return std::make_shared>( + std::make_pair(operator_vector, output_info_vector)); +} + +Status TensorRedistribution::InferReshape(const TensorLayout& from_layout, const TensorLayout& to_layout, + OperatorVector* const operator_vector, + OutPutInfoVector* const output_info_vector) { + MS_EXCEPTION_IF_NULL(operator_vector); + MS_EXCEPTION_IF_NULL(output_info_vector); + ConstructOperator constructor; + if (operator_list_.empty()) { + if (from_origin_.slice_shape().array() != to_origin_.slice_shape().array() || keep_reshape_) { + reshape_flag_ = true; + constructor.UpdateTensorShape(from_origin_.slice_shape().array()); + Arrangement shape = to_origin_.slice_shape(); + MS_LOG(DEBUG) << "reshape " << shape.ToString(); + if (constructor.ReshapeOP(shape.array()) == Status::FAILED) { + return Status::FAILED; + } else { + (void)operator_vector->insert(operator_vector->begin(), constructor.GetOperator()); + (void)output_info_vector->insert(output_info_vector->begin(), std::make_pair(false, 0)); + } + } + return Status::SUCCESS; + } + + if (from_origin_.slice_shape().array() != from_layout.slice_shape().array()) { + reshape_flag_ = true; + constructor.UpdateTensorShape(from_origin_.slice_shape().array()); + Arrangement shape = from_layout.slice_shape(); + MS_LOG(DEBUG) << "reshape " << shape.ToString(); + if (constructor.ReshapeOP(shape.array()) == Status::FAILED) { + return Status::FAILED; + } else { + (void)operator_vector->insert(operator_vector->begin(), constructor.GetOperator()); + (void)output_info_vector->insert(output_info_vector->begin(), std::make_pair(false, 0)); + } + } + + if (to_origin_.slice_shape().array() != to_layout.slice_shape().array()) { + reshape_flag_ = true; + constructor.UpdateTensorShape(to_layout.slice_shape().array()); + Arrangement shape = to_origin_.slice_shape(); + MS_LOG(DEBUG) << "step_parallel to reshape " << shape.ToString(); + if (constructor.ReshapeOP(shape.array()) == Status::FAILED) { + return Status::FAILED; + } else { + (void)operator_vector->insert(operator_vector->end(), constructor.GetOperator()); + (void)output_info_vector->insert(output_info_vector->end(), std::make_pair(false, 0)); + } + } + return Status::SUCCESS; +} + +Status TensorRedistribution::ComputeCost() { + RedistributionOpListPtr redistribution_oplist_ptr = InferTensorRedistributionOperatorList(); + if (redistribution_oplist_ptr == nullptr) { + MS_LOG(ERROR) << "Failure: InferTensorRedistribution failed"; + return Status::FAILED; + } + // Compute redistribution communication cost and memory cost + for (auto& op_cost : operator_list_) { + OperatorR op = op_cost.first; + Shape slice_shape = op_cost.second; + double prod = + std::accumulate(slice_shape.begin(), slice_shape.end(), static_cast(1.0), std::multiplies()); + std::string str = op.first; + if (str == PERMUTE_BY_AXIS) { + // The shape does not change after PermuteByAxis operation. + // communication cost = all_to_all + all_to_all = 2 * slice_shape + // memory cost = slice_shape + forward_comm_cost_ += prod; + backward_comm_cost_ += prod; + comm_cost_ += 2.0 * prod; + mem_cost_ += prod; + } else if (str == CONCAT_BY_AXIS) { + // communication cost = all_gather + reduce_scatter = before_slice_shape + after_slice_shape + // memory cost = before_slice_shape + if (op.second.size() < 3) { + MS_LOG(ERROR) << "op.second size should not be less than 3!"; + return Status::FAILED; + } + double dev_num = op.second[2]; + // here, communication cost = all_gather + reduce_scatter + forward_comm_cost_ += prod * dev_num; + backward_comm_cost_ += prod; + comm_cost_ += prod * (dev_num + 1.0); + int32_t concat_dim = op.second[0]; + if (concat_dim == 0) { + // memory cost = all_gather + mem_cost_ += prod; + } else { + // memory cost = all_gather + split + concat + mem_cost_ += (prod + prod * dev_num + prod * dev_num); + } + } else { + // There is only memory cost in SplitByAxis. + // memory cost = before_slice_shape + mem_cost_ += prod; + } + } + if (reshape_flag()) { + Shape prev_slice_shape = from_.slice_shape().array(); + double prev_prod = std::accumulate(prev_slice_shape.begin(), prev_slice_shape.end(), 1, std::multiplies()); + mem_cost_ += 2.0 * prev_prod; + } + return Status::SUCCESS; +} +} // namespace parallel +} // namespace mindspore diff --git a/mindspore/ccsrc/parallel/tensor_layout/tensor_redistribution.h b/mindspore/ccsrc/parallel/tensor_layout/tensor_redistribution.h new file mode 100644 index 00000000000..daf474b3d7c --- /dev/null +++ b/mindspore/ccsrc/parallel/tensor_layout/tensor_redistribution.h @@ -0,0 +1,80 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_REDISTRIBUTION_H_ +#define MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_REDISTRIBUTION_H_ + +#include +#include +#include +#include +#include +#include + +#include "ir/value.h" +#include "parallel/status.h" +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/tensor_layout/construct_operator.h" +#include "parallel/tensor_layout/redistribution_operator_infer.h" + +namespace mindspore { +namespace parallel { + +class TensorRedistribution { + public: + explicit TensorRedistribution(bool construct_op_flag = true, bool keep_reshape = false) + : reshape_flag_(false), + comm_cost_(0.0), + forward_comm_cost_(0.0), + backward_comm_cost_(0.0), + mem_cost_(0.0), + construct_op_flag_(construct_op_flag), + keep_reshape_(keep_reshape) {} + Status Init(const TensorLayout& from, const TensorLayout& to, const RankList& dev_list); + ~TensorRedistribution() = default; + RedistributionOpListPtr InferTensorRedistributionOperatorList(); + OperatorList operator_list() const { return operator_list_; } + bool reshape_flag() const { return reshape_flag_; } + Status ComputeCost(); + double comm_cost() const { return comm_cost_; } + double mem_cost() const { return mem_cost_; } + double forward_comm_cost() const { return forward_comm_cost_; } + double backward_comm_cost() const { return backward_comm_cost_; } + + private: + Status InferReshape(const TensorLayout& from_layout, const TensorLayout& to_layout, + OperatorVector* const operator_vector, OutPutInfoVector* const output_info_vector); + + TensorLayout from_origin_; + TensorLayout to_origin_; + TensorLayout from_; + TensorLayout to_; + RankList dev_list_; + OperatorList operator_list_; + bool reshape_flag_; + double comm_cost_; + double forward_comm_cost_; + double backward_comm_cost_; + double mem_cost_; + bool construct_op_flag_; + bool keep_reshape_; +}; + +} // namespace parallel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PARALLEL_TENSOR_LAYOUT_TENSOR_REDISTRIBUTION_H_ diff --git a/mindspore/ccsrc/pipeline/CMakeLists.txt b/mindspore/ccsrc/pipeline/CMakeLists.txt new file mode 100644 index 00000000000..4aadbcce582 --- /dev/null +++ b/mindspore/ccsrc/pipeline/CMakeLists.txt @@ -0,0 +1,12 @@ +file(GLOB_RECURSE _PIPELINE_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "pipeline.cc" + "resource.cc" + "pass.cc" + "action.cc" + "validator.cc" + "remove_value_node_dup.cc" + "parse/*.cc" + "static_analysis/*.cc" + ) + +add_library(_mindspore_pipeline_obj OBJECT ${_PIPELINE_ALL_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/pipeline/action.cc b/mindspore/ccsrc/pipeline/action.cc new file mode 100644 index 00000000000..f3742ab6541 --- /dev/null +++ b/mindspore/ccsrc/pipeline/action.cc @@ -0,0 +1,339 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/action.h" + +#include +#include +#include +#include +#include +#include + +#include "ir/func_graph_cloner.h" +#include "pipeline/pass.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "pipeline/static_analysis/program_specialize.h" +#include "pipeline/resource.h" +#include "pipeline/remove_value_node_dup.h" +#include "optimizer/optimizer.h" +#include "vm/transform.h" + +namespace mindspore { +namespace pipeline { +using CompileGraphs = compile::CompileGraphs; +using abstract::AnalysisResult; +using mindspore::abstract::AnalysisContextPtr; + +abstract::AnalysisResult AbstractAnalyze(const ResourcePtr& res, const FuncGraphPtr& func_graph, + const abstract::AbstractBasePtrList& args_spec, bool clear) { + MS_LOG(DEBUG) << "AbstractAnalyze start"; + auto engine = res->engine(); + MS_EXCEPTION_IF_NULL(engine); + if (clear) { + auto manager = res->manager(); + MS_EXCEPTION_IF_NULL(manager); + engine->Clear(); + for (auto& node : manager->all_nodes()) { + MS_EXCEPTION_IF_NULL(node); + const AbstractBasePtr& prev_inferred = node->abstract(); + // Keep previous inferred value for ValueNode if the inferred value is not AbstractFunction. + if (!node->isa() || (prev_inferred != nullptr && prev_inferred->isa())) { + node->set_abstract(nullptr); + MS_LOG(DEBUG) << "Abstract of node " << node->ToString() << " is set to nullptr"; + } + } + } + auto ret = engine->Run(func_graph, args_spec); + MS_LOG(DEBUG) << "AbstractAnalyze end"; + return ret; +} + +FuncGraphPtr ProgramSpecialize(const ResourcePtr& res, const FuncGraphPtr& func_graph, + const abstract::AnalysisContextPtr& context) { + MS_LOG(DEBUG) << "ProgramSpecialize start"; + abstract::ProgramSpecializer spc(res->engine()); + FuncGraphPtr result = spc.Run(func_graph, context); + auto manager = res->manager(); + MS_EXCEPTION_IF_NULL(manager); + manager->KeepRoots({result}); + MS_LOG(DEBUG) << "ProgramSpecialize end"; + return result; +} + +FuncGraphPtr Renormalize(const ResourcePtr& res, const FuncGraphPtr& func_graph, + const abstract::AbstractBasePtrList& args_spec) { + MS_LOG(DEBUG) << "Renormalize start"; +#ifdef ENABLE_PROFILE + double t1 = GetTime(); +#endif + abstract::AnalysisResult result = AbstractAnalyze(res, func_graph, args_spec, true); +#ifdef ENABLE_PROFILE + double t2 = GetTime(); +#endif + auto ret = ProgramSpecialize(res, func_graph, result.context); +#ifdef ENABLE_PROFILE + double t3 = GetTime(); + MsProfile::StatTime("renormalize.infer", t2 - t1); + MsProfile::StatTime("renormalize.specialize", t3 - t2); +#endif + MS_LOG(DEBUG) << "Renormalize end"; + return ret; +} + +bool ParseAction(const ResourcePtr& res) { + if (!res->input()) { + MS_LOG(EXCEPTION) << "Parse error"; + } + + py::object input = res->input(); + parse::Parser::InitParserEnvironment(input); + py::module path = py::module::import("os.path"); + std::string dir = path.attr("dirname")(py::globals()["__file__"]).cast(); + + parse::python_adapter::set_python_env_flag(true); + parse::python_adapter::SetPythonPath(dir); + + FuncGraphPtr fg = parse::ConvertToFuncGraph(input); + if (fg == nullptr) { + MS_LOG(EXCEPTION) << "Parse error."; + } + res->set_func_graph(fg); + + FuncGraphManagerPtr manager = res->manager(); + if (manager == nullptr) { + MS_LOG(EXCEPTION) << "Manager is nullptr."; + } + manager->AddFuncGraph(fg); + return true; +} + +// obj_map's graphs have the same construct, these graphs can be optimized to one graph. +// This step do this optimize: graph1(x){xx(fv1),xxx(fv2)}, graph2(x){xxx(fv3),xxx(fv4)}-> +// graph1(x){base_graph(x, fv1, fv2)}, graph1(x){base_graph(x, fv3, fv4)}, base_graph(x, fv...){xxx,xxx} +// all obj_map's graph shared base_graph +bool CombineLikeGraphs(const ResourcePtr&) { + auto& obj_map = parse::data_converter::GetObjGraphs(); + + for (auto it : obj_map) { + auto& graphs = it.second; + MS_LOG(DEBUG) << "Start combine like graph:" << it.first << ", size:" << graphs.size(); + auto fg = graphs[0]; + FuncGraphPtrList func_graphs = {fg}; + ClonerPtr cloner = std::make_shared(func_graphs, false, false, true, std::make_shared(), + std::make_shared()); + cloner->Run(); + auto base_graph = cloner->cloned_func_graph()[fg]; + MS_LOG(DEBUG) << "Basegraph:" << base_graph->ToString(); + + if (fg->paramter_obj_nodes().size() == 0 || graphs.size() <= 1) { + continue; + } + auto mng = Manage(base_graph, false); + for (auto& fv : fg->paramter_obj_nodes()) { + TraceManager::DebugTrace(std::make_shared(fv->debug_info())); + auto param = base_graph->add_parameter(); + TraceManager::EndTrace(); + auto repl_node = (*cloner->cloned_node())[fv]; + (void)mng->Replace(repl_node, param); + } + MS_LOG(DEBUG) << "Fg0 paramter_obj_nodes size :" << fg->paramter_obj_nodes().size(); + + for (auto& g : graphs) { + auto fvs = g->paramter_obj_nodes(); + std::vector new_node_inputs; + new_node_inputs.push_back(NewValueNode(base_graph)); + for (auto& p : g->parameters()) { + AnfNodePtr para_after_cast = parse::GetMixedPrecisionCastHelp(g, p); + new_node_inputs.push_back(para_after_cast); + } + (void)new_node_inputs.insert(new_node_inputs.end(), fvs.begin(), fvs.end()); + AnfNodePtr out = g->NewCNode(new_node_inputs); + g->set_output(out); + MS_LOG(DEBUG) << "Combine graph newout:" << out->DebugString(4); + } + MS_LOG(DEBUG) << "End combine graph:" << it.first; + } + return true; +} + +bool SymbolResolveAction(const ResourcePtr& res) { + if (res->manager() == nullptr) { + MS_LOG(EXCEPTION) << "Resolve error."; + } + if (res->func_graph() == nullptr) { + MS_LOG(EXCEPTION) << "Resolve error"; + } + FuncGraphPtr func_graph = res->func_graph(); + auto succ = parse::ResolveFuncGraph(func_graph, res); + + // Remove usued nodes in cnode order list. + func_graph->EraseUnusedNodeInOrder(); + func_graph->ReleaseFullOrderToEffectOrder(); + for (auto fg : func_graph->func_graphs_used_total()) { + MS_EXCEPTION_IF_NULL(fg); + fg->EraseUnusedNodeInOrder(); + fg->ReleaseFullOrderToEffectOrder(); + } + return succ; +} + +bool AbstractSpecializeAction(const ResourcePtr& res) { + if (res->func_graph() == nullptr) { + MS_LOG(EXCEPTION) << "AbstractSpecialize error"; + } + + FuncGraphPtr func_graph = res->func_graph(); + abstract::AbstractBasePtrList args_spec = res->args_spec(); + + // suppose that there is not KeywordArgument for the top graph + // get the hyper parameter + for (const auto& param : func_graph->parameters()) { + auto param_node = std::static_pointer_cast(param); + if (param_node->has_default()) { + AbstractBasePtr ptr = + abstract::FromValue(parse::data_converter::PyDataToValue(param_node->default_param()), true); + args_spec.push_back(ptr); + } + } + // Analyze + AnalysisResult result = AbstractAnalyze(res, func_graph, args_spec); + // The top graph may be replaced by infer, update the top graph when the infer is done + parse::Parser::UpdateTopFuncGraph(result.context->func_graph()); + + // Specialize + FuncGraphPtr new_fg = ProgramSpecialize(res, result.context->func_graph(), result.context); + res->set_func_graph(new_fg); + + MS_LOG(DEBUG) << "End graph: " << new_fg->ToString() << ", return: " << new_fg->get_return()->DebugString(true); + return true; +} + +bool OptimizeAction(const ResourcePtr& res, const std::vector& passes) { + for (auto& pass : passes) { + WITH(MsProfile::GetProfile()->Step(pass.first))[&pass, &res]() { + MS_LOG(DEBUG) << "Pass " << pass.first << " start ..."; + auto result = pass.second(res); + if (!result) { + MS_LOG(EXCEPTION) << "Pass running to end, failed in pass:" << pass.first; + } + MS_LOG(DEBUG) << "Pass " << pass.first << " end."; + }; + } + + return true; +} + +bool GeOptimizeAction(const ResourcePtr& res) { return OptimizeAction(res, kGePasses); } + +bool VmOptimizeAction(const ResourcePtr& res) { return OptimizeAction(res, kVmPasses); } + +bool TaskEmitAction(const ResourcePtr& res) { + if (res->func_graph() == nullptr) { + MS_LOG(EXCEPTION) << "TaskEmit args error"; + } + FuncGraphPtr func_graph = res->func_graph(); + + auto bc_ptr = res->results()[kBackend].cast(); + std::vector cut_list = compile::nonlinear_ops; + if (bc_ptr->name() == kMsConvert) { + cut_list = compile::ms_nonlinear_ops; + } + std::shared_ptr compile = std::make_shared(bc_ptr, cut_list); + res->results()[kOutput] = compile->CompileAndLink(func_graph); + return true; +} + +bool ExecuteAction(const ResourcePtr& res) { + if (res->results().count(kOutput) == 0 || !res->results()[kOutput].is()) { + MS_LOG(EXCEPTION) << "Execute args error"; + } + + compile::FinalVMPtr vm = res->results()[kOutput].cast(); + if (vm == nullptr) { + MS_LOG(INFO) << "Call GE to Run the func_graph instead of VM"; + return true; + } + compile::VmEvalFuncPtr run = + std::make_shared(std::bind(&compile::FinalVM::Eval, vm, std::placeholders::_1)); + res->results()[kOutput] = run; + return true; +} + +bool RemoveValueNodeDuplicationsAction(const ResourcePtr& res) { + if (res->func_graph() == nullptr) { + MS_LOG(EXCEPTION) << "Remove value node duplications error."; + } + FuncGraphPtr func_graph = res->func_graph(); + auto manager = res->manager(); + // Remove duplicated value nodes, due to replace operation, can't use reference. + auto value_nodes = manager->valuenodes()[func_graph]; + HashCache hash_cache; + HashValue hashes; + for (const auto& value_pair : value_nodes) { + TryToDoReplace(manager.get(), value_pair.first, &hash_cache, &hashes); + } + return true; +} + +bool ValidateAction(const ResourcePtr& res) { return ValidatePass(res); } + +static std::vector CommonPipeline() { + std::vector actions; + + // Parse the python ast to ANF graph + actions.emplace_back(std::make_pair("parse", ParseAction)); + + // Resolve the python func + actions.emplace_back(std::make_pair("symbol_resolve", SymbolResolveAction)); + actions.emplace_back(std::make_pair("combine_like_graphs", CombineLikeGraphs)); + + // Evaluate type and shape, and specialize + actions.emplace_back(std::make_pair("abstract_specialize", AbstractSpecializeAction)); + + return actions; +} + +std::vector GePipeline() { + auto actions = CommonPipeline(); + // optimize + actions.emplace_back(std::make_pair("optimize", GeOptimizeAction)); + actions.emplace_back(std::make_pair("remove_value_node_duplications", RemoveValueNodeDuplicationsAction)); + actions.emplace_back(std::make_pair("validate", ValidateAction)); + return actions; +} + +std::vector VmPipeline() { + auto actions = CommonPipeline(); + + // optimize + actions.emplace_back(std::make_pair("optimize", VmOptimizeAction)); + + actions.emplace_back(std::make_pair("validate", ValidateAction)); + + // compile the ANF graph + actions.emplace_back(std::make_pair("task_emit", TaskEmitAction)); + + // to execute the graph + actions.emplace_back(std::make_pair("execute", ExecuteAction)); + + return actions; +} +} // namespace pipeline +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/action.h b/mindspore/ccsrc/pipeline/action.h new file mode 100644 index 00000000000..159e494a964 --- /dev/null +++ b/mindspore/ccsrc/pipeline/action.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PIPELINE_ACTION_H_ +#define MINDSPORE_CCSRC_PIPELINE_ACTION_H_ + +#include +#include +#include +#include +#include "pipeline/resource.h" +#include "vm/segment_runner.h" + +namespace mindspore { +extern const char kMsConvert[]; + +namespace pipeline { +using ActionItem = std::pair>; + +bool ParseAction(const ResourcePtr& res); +bool SymbolResolveAction(const ResourcePtr& res); +bool AbstractSpecializeAction(const ResourcePtr& res); +bool GeOptimizeAction(const ResourcePtr& res); +bool VmOptimizeAction(const ResourcePtr& res); +bool TaskEmitAction(const ResourcePtr& res); +bool ExecuteAction(const ResourcePtr& res); + +std::vector GePipeline(); +std::vector VmPipeline(); +abstract::AnalysisResult AbstractAnalyze(const ResourcePtr& res, const FuncGraphPtr& func_graph, + const abstract::AbstractBasePtrList& args_spec, bool clear = false); +FuncGraphPtr ProgramSpecialize(const ResourcePtr& res, const FuncGraphPtr& func_graph, + const abstract::AnalysisContextPtr& context); +FuncGraphPtr Renormalize(const ResourcePtr& res, const FuncGraphPtr& func_graph, + const abstract::AbstractBasePtrList& args_spec); +} // namespace pipeline +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PIPELINE_ACTION_H_ diff --git a/mindspore/ccsrc/pipeline/init.cc b/mindspore/ccsrc/pipeline/init.cc new file mode 100644 index 00000000000..f42ae0cf7b2 --- /dev/null +++ b/mindspore/ccsrc/pipeline/init.cc @@ -0,0 +1,307 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "kernel/oplib/oplib.h" +#include "pipeline/pipeline.h" +#include "operator/composite/composite.h" +#include "ir/signature.h" +#include "pynative/pynative_execute.h" +#include "utils/symbolic.h" +#include "pybind_api/api_register.h" +#include "pipeline/parse/python_adapter.h" +#include "utils/summary/event_writer.h" +#include "utils/config_manager.h" +#include "parallel/context.h" +#include "parallel/device_manager.h" +#include "parallel/costmodel_context.h" +#include "device/gpu/distribution/collective_init.h" + +namespace py = pybind11; + +using FuncGraph = mindspore::FuncGraph; +using EnvInstance = mindspore::EnvInstance; +using ExecutorPy = mindspore::pipeline::ExecutorPy; +using Pipeline = mindspore::pipeline::Pipeline; +using PrimitivePy = mindspore::PrimitivePy; +using MetaFuncGraph = mindspore::MetaFuncGraph; +using EventWriter = mindspore::summary::EventWriter; +using OpLib = mindspore::kernel::OpLib; +using ParallelContext = mindspore::parallel::ParallelContext; +using CostModelContext = mindspore::parallel::CostModelContext; + +// Interface with python +PYBIND11_MODULE(_c_expression, m) { + m.doc() = "MindSpore c plugin"; + + (void)py::class_>(*m, "MetaFuncGraph_") + .def_readonly(mindspore::PYTHON_METAFUNCGRAPH_FLAG, &mindspore::MetaFuncGraph::parse_info_) + .def(py::init()); + + auto fns = mindspore::PybindDefineRegister::AllFuncs(); + for (auto& item : fns) { + item.second(&m); + } + + // Class Pipeline interface + (void)py::class_>(m, "Executor_") + .def_static("get_instance", &ExecutorPy::GetInstance, "Executor get_instance.") + .def("__call__", &ExecutorPy::Run, py::arg("args"), py::arg("phase") = py::str(""), "Executor run function.") + .def("del_net_res", &ExecutorPy::DelNetRes, py::arg("network_id") = py::str(""), "Delete network resource.") + .def("get_func_graph", &ExecutorPy::GetFuncGraph, py::arg("phase") = py::str(""), "Get graph pointer.") + .def("get_func_graph_proto", &ExecutorPy::GetFuncGraphProto, py::arg("phase") = py::str(""), + py::arg("type") = py::str("onnx_ir"), "Get graph proto string by specifying ir type.") + .def("compile", &ExecutorPy::Compile, py::arg("obj"), py::arg("args"), py::arg("phase") = py::str(""), + py::arg("use_vm") = py::bool_(false), "Compile obj by executor.") + .def("get_parameter_layout", &ExecutorPy::GetParameterLayout, py::arg("phase") = py::str("train"), + "Get Parameter Tensor Layout Dictionary.") + .def("get_strategy", &ExecutorPy::GetCNodeStrategy, py::arg("phase") = py::str("train"), + "Get CNode Strategy Dictionary.") + .def("get_allreduce_fusion", &ExecutorPy::GetAllreduceFusion, py::arg("phase") = py::str("train"), + "Get Allreduce Fusion Dictionary.") + .def("build_data_graph", &ExecutorPy::BuildDFGraph, py::arg("build_params"), py::arg("phase") = py::str("train"), + py::arg("broadcast_params") = py::dict(), "Build data graph.") + .def("has_compiled", &ExecutorPy::HasCompiled, py::arg("phase") = py::str(""), "get if cell compiled.") + .def("run_init_graph", &ExecutorPy::RunInitGraph, "Run init Graph."); + // Class Graph interface + (void)py::class_(m, "FuncGraph").def(py::init()); + + (void)py::class_>(m, "EnvInstance_") + .def_readonly(mindspore::PYTHON_ENVINSTANCE_FLAG, &mindspore::EnvInstance::parse_info_) + .def(py::init()); + + (void)m.def("generate_key", &mindspore::pipeline::GenerateKey, "Generate the function graph key."); + (void)m.def("real_run_op", &mindspore::pynative::RunOp, "Run op pynatively."); + (void)m.def("initialize_distribute", &mindspore::pipeline::InitDistribute, "Initialize for Distribute.") + .def("init_ge", &mindspore::pipeline::InitGe, "Init GE"); + (void)m.def("reset_op_id", &mindspore::pipeline::ResetOpId, "Reset Operator Id"); + (void)m.def("init_hccl", &mindspore::pipeline::InitHccl, "Init Hccl"); + (void)m.def("finalize_ge", &mindspore::pipeline::FinalizeGe, "Finalize Ge"); + (void)m.def("finalize_hccl", &mindspore::pipeline::FinalizeHccl, "Finalize Hccl"); + (void)m.def("set_ge_option", &mindspore::pipeline::SetGeOption, "API for set ge option."); + (void)m.def("verify_inputs_signature", &mindspore::pipeline::VerifyInputSignature, "Verify input signature."); + (void)m.def("init_exec_dataset", &mindspore::pipeline::InitExecDataset, py::arg("queue_name"), py::arg("size"), + py::arg("batch_size"), py::arg("types"), py::arg("shapes"), py::arg("input_indexs"), + py::arg("phase") = py::str("dataset"), "Init and exec dataset."); + (void)m.def("_set_dataset_mode_config", &mindspore::ConfigManager::SetDatasetModeConfig, "API for set dataset mode."); + (void)m.def("export_graph", &mindspore::pipeline::ExportDFGraph, "Export Graph."); + + (void)py::class_>(m, "MSContext") + .def_static("get_instance", &mindspore::MsContext::GetInstance, "Get ms context instance.") + .def("get_backend_policy", &mindspore::MsContext::backend_policy, "Get backend policy.") + .def("set_backend_policy", &mindspore::MsContext::set_backend_policy, "Set backend policy.") + .def("get_execution_mode", &mindspore::MsContext::execution_mode, "Get execution mode.") + .def("set_execution_mode", &mindspore::MsContext::set_execution_mode, "Set execution mode.") + .def("set_precompile_only", &mindspore::MsContext::set_precompile_only, "Set enable precompile only.") + .def("get_precompile_only", &mindspore::MsContext::precompile_only, "Get enable precompile only.") + .def("get_device_target", &mindspore::MsContext::device_target, "Get device target.") + .def("set_device_target", &mindspore::MsContext::set_device_target, "Set device target.") + .def("get_device_id", &mindspore::MsContext::device_id, "Get device id.") + .def("set_device_id", &mindspore::MsContext::set_device_id, "Set device id.") + .def("open_tsd", &mindspore::MsContext::OpenTsd, "Open tdt dataset client.") + .def("close_tsd", &mindspore::MsContext::CloseTsd, "Close tdt dataset client.") + .def("set_hccl_flag", &mindspore::MsContext::set_enable_hccl, "Set enable hccl.") + .def("get_hccl_flag", &mindspore::MsContext::enable_hccl, "Get whether to enable hccl.") + .def("set_task_sink_flag", &mindspore::MsContext::set_enable_task_sink, "Set enable task sink.") + .def("get_task_sink_flag", &mindspore::MsContext::enable_task_sink, "Get whether to enable task sink.") + .def("get_save_graphs_flag", &mindspore::MsContext::save_graphs_flag, "Get whether to save graphs.") + .def("set_save_graphs_flag", &mindspore::MsContext::set_save_graphs_flag, "Set whether to save graphs.") + .def("get_ir_fusion_flag", &mindspore::MsContext::ir_fusion_flag, "Get whether to enable ir fusion.") + .def("set_ir_fusion_flag", &mindspore::MsContext::set_ir_fusion_flag, "Set whether to enable ir fusion.") + .def("get_auto_mixed_precision_flag", &mindspore::MsContext::auto_mixed_precision_flag, + "Get whether to enable auto mixed precision.") + .def("set_auto_mixed_precision_flag", &mindspore::MsContext::set_auto_mixed_precision_flag, + "Set whether to enable auto mixed precision.") + .def("get_enable_reduce_precision_flag", &mindspore::MsContext::enable_reduce_precision, + "Get whether to enable reduce precision.") + .def("set_enable_reduce_precision_flag", &mindspore::MsContext::set_enable_reduce_precision, + "Set whether to enable reduce precision.") + .def("get_save_graphs_path", &mindspore::MsContext::save_graphs_path, "Get save graphs path.") + .def("set_save_graphs_path", &mindspore::MsContext::set_save_graphs_path, "Set save graphs path.") + .def("get_loop_sink_flag", &mindspore::MsContext::loop_sink_flag, "Get whether to enable loop sink.") + .def("set_loop_sink_flag", &mindspore::MsContext::set_loop_sink_flag, "Set whether to enable loop sink.") + .def("get_enable_mem_reuse", &mindspore::MsContext::enable_mem_reuse, "Get whether to enable mem reuse.") + .def("set_enable_mem_reuse", &mindspore::MsContext::set_enable_mem_reuse, "Set whether to enable mem reuse.") + .def("get_save_ms_model_flag", &mindspore::MsContext::save_ms_model_flag, "Get whether to save ms model.") + .def("set_save_ms_model_flag", &mindspore::MsContext::set_save_ms_model_flag, "Set whether to save ms model.") + .def("get_save_ms_model_path", &mindspore::MsContext::save_ms_model_path, "Get path to save ms model.") + .def("set_save_ms_model_path", &mindspore::MsContext::set_save_ms_model_path, "Set path to save ms model") + .def("get_enable_gpu_summary", &mindspore::MsContext::enable_gpu_summary, "Get whether to enable gpu summary.") + .def("set_enable_gpu_summary", &mindspore::MsContext::set_enable_gpu_summary, "Set whether to enable gpu summary.") + .def("get_enable_dump", &mindspore::MsContext::enable_dump, "Get whether to enable dump.") + .def("set_enable_dump", &mindspore::MsContext::set_enable_dump, "Set whether to enable dump.") + .def("get_save_dump_path", &mindspore::MsContext::save_dump_path, "Get path to dump.") + .def("set_save_dump_path", &mindspore::MsContext::set_save_dump_path, "Set path to dump.") + .def("get_enable_dynamic_mem_pool", &mindspore::MsContext::enable_dynamic_mem_pool, + "Get whether to enable dynamic mem pool.") + .def("set_enable_dynamic_mem_pool", &mindspore::MsContext::set_enable_dynamic_mem_pool, + "Set whether to enable dynamic mem pool.") + .def("set_graph_memory_max_size", &mindspore::MsContext::set_graph_memory_max_size, "set graph memory max size.") + .def("set_variable_memory_max_size", &mindspore::MsContext::set_variable_memory_max_size, + "set variable memory max size"); + + (void)py::class_>(m, "AutoParallelContext") + .def_static("get_instance", &ParallelContext::GetInstance, "Get auto parallel context instance.") + .def("get_device_num", &ParallelContext::device_num, "Get device num.") + .def("set_device_num", &ParallelContext::set_device_num, "Set device num.") + .def("get_device_num_is_set", &ParallelContext::device_num_is_set, "Get device num is set.") + .def("get_global_rank", &ParallelContext::global_rank, "Get global rank.") + .def("set_global_rank", &ParallelContext::set_global_rank, "Set global rank.") + .def("get_global_rank_is_set", &ParallelContext::global_rank_is_set, "Get global rank is set.") + .def("get_mirror_mean", &ParallelContext::mirror_mean, "Get mirror mean.") + .def("set_mirror_mean", &ParallelContext::set_mirror_mean, "Set mirror mean.") + .def("get_cast_before_mirror", &ParallelContext::cast_before_mirror, "Get cast before mirror.") + .def("set_cast_before_mirror", &ParallelContext::set_cast_before_mirror, "Set cast before mirror.") + .def("get_loss_repeated_mean", &ParallelContext::loss_repeated_mean, "Get loss repeated mean.") + .def("set_loss_repeated_mean", &ParallelContext::set_loss_repeated_mean, "Set loss repeated mean.") + .def("get_communication_backend", &ParallelContext::communication_backend, "Get communication backend.") + .def("set_communication_backend", &ParallelContext::set_communication_backend, "Set communication backend.") + .def("get_parallel_mode", &ParallelContext::parallel_mode, "Get parallel mode.") + .def("set_parallel_mode", &ParallelContext::set_parallel_mode, "Set parallel mode.") + .def("get_strategy_search_mode", &ParallelContext::strategy_search_mode, "Get strategy search mode.") + .def("set_strategy_search_mode", &ParallelContext::set_strategy_search_mode, "Set strategy search mode.") + .def("set_all_reduce_fusion_split_indices", &ParallelContext::set_all_reduce_fusion_split_indices, + "Set all reduce fusion split indices.") + .def("get_all_reduce_fusion_split_indices", &ParallelContext::all_reduce_fusion_split_indices, + "Get all reduce fusion split indices.") + .def("set_all_reduce_fusion_split_sizes", &ParallelContext::set_all_reduce_fusion_split_sizes, + "Set all reduce fusion split sizes.") + .def("get_all_reduce_fusion_split_sizes", &ParallelContext::all_reduce_fusion_split_sizes, + "Get all reduce fusion split sizes.") + .def("get_parameter_broadcast", &ParallelContext::parameter_broadcast, "Get parameter broadcast.") + .def("get_parameter_broadcast_is_set", &ParallelContext::parameter_broadcast_is_set, + "Get parameter broadcast is set.") + .def("set_parameter_broadcast", &ParallelContext::set_parameter_broadcast, "Set parameter broadcast.") + .def("reset", &ParallelContext::Reset, "Reset auto parallel context."); + + (void)py::class_>(m, "CostModelContext") + .def_static("get_instance", &CostModelContext::GetInstance, "Get cost_model context instance.") + .def("set_device_memory_capacity", &CostModelContext::set_device_memory_capacity, + "Set the capacity of device memory.") + .def("get_device_memory_capacity", &CostModelContext::device_memory_capacity, "Get the capacity of device memory.") + .def("set_costmodel_alpha", &CostModelContext::set_costmodel_alpha, + "Set the parameter cost_model_alpha of the DP algorithm.") + .def("get_costmodel_alpha", &CostModelContext::costmodel_alpha, + "Get the parameter cost_model_alpha of the DP algorithm.") + .def("set_costmodel_beta", &CostModelContext::set_costmodel_beta, + "Set the parameter cost_model_beta of the DP algorithm.") + .def("get_costmodel_beta", &CostModelContext::costmodel_beta, + "Get the parameter cost_model_beta of the DP algorithm.") + .def("set_costmodel_gamma", &CostModelContext::set_costmodel_gamma, + "Set the parameter cost_model_gamma of the DP algorithm") + .def("get_costmodel_gamma", &CostModelContext::costmodel_gamma, + "Get the parameter cost_model_gamma of the DP algorithm.") + .def("set_simplify_cal", &CostModelContext::set_costmodel_simplify_cal, + "Set the parameter cost_model_simplify_cal of the DP algorithm.") + .def("get_simplify_cal", &CostModelContext::costmodel_simplify_cal, + "Get the parameter cost_model_simplify_cal of the DP algorithm.") + .def("set_costmodel_communi_threshold", &CostModelContext::set_costmodel_communi_threshold, + "Set the parameter cost_model_communi_threshold of the DP algorithm.") + .def("get_costmodel_communi_threshold", &CostModelContext::costmodel_communi_threshold, + "Get the parameter cost_model_communi_threshold of the DP algorithm.") + .def("set_costmodel_communi_const", &CostModelContext::set_costmodel_communi_const, + "Set the parameter cost_model_communi_const of the DP algorithm.") + .def("get_costmodel_communi_const", &CostModelContext::costmodel_communi_const, + "Get the parameter cost_model_communi_const of the DP algorithm.") + .def("set_costmodel_communi_bias", &CostModelContext::set_costmodel_communi_bias, + "Set the parameter cost_model_communi_bias of the DP algorithm.") + .def("get_costmodel_communi_bias", &CostModelContext::costmodel_communi_bias, + "Get the parameter cost_model_communi_bias of the DP algorithm.") + .def("set_costmodel_allreduce_fusion_algorithm", &CostModelContext::set_costmodel_allreduce_fusion_algorithm, + "Set the parameter gradient AllReduce fusion algorithm.") + .def("get_costmodel_allreduce_fusion_algorithm", &CostModelContext::costmodel_allreduce_fusion_algorithm, + "Get the parameter gradient AllReduce fusion algorithm.") + .def("set_costmodel_allreduce_fusion_times", &CostModelContext::set_costmodel_allreduce_fusion_times, + "Set the parameter gradient AllReduce times.") + .def("get_costmodel_allreduce_fusion_times", &CostModelContext::costmodel_allreduce_fusion_times, + "Get the parameter gradient AllReduce times.") + .def("set_costmodel_allreduce_fusion_tail_percent", &CostModelContext::set_costmodel_allreduce_fusion_tail_percent, + "Set the parameter gradient AllReduce fusion tail percent.") + .def("get_costmodel_allreduce_fusion_tail_percent", &CostModelContext::costmodel_allreduce_fusion_tail_percent, + "Get the parameter gradient AllReduce fusion tail percent.") + .def("set_costmodel_allreduce_fusion_tail_time", &CostModelContext::set_costmodel_allreduce_fusion_tail_time, + "Set the parameter gradient AllReduce fusion tail time.") + .def("get_costmodel_allreduce_fusion_tail_time", &CostModelContext::costmodel_allreduce_fusion_tail_time, + "Get the parameter gradient AllReduce fusion tail time.") + .def("set_costmodel_allreduce_fusion_allreduce_inherent_time", + &CostModelContext::set_costmodel_allreduce_fusion_allreduce_inherent_time, + "Set the parameter gradient AllReduce fusion allreduce inherent time.") + .def("get_costmodel_allreduce_fusion_allreduce_inherent_time", + &CostModelContext::costmodel_allreduce_fusion_allreduce_inherent_time, + "Get the parameter gradient AllReduce fusion allreduce inherent time.") + .def("set_costmodel_allreduce_fusion_allreduce_bandwidth", + &CostModelContext::set_costmodel_allreduce_fusion_allreduce_bandwidth, + "Set the parameter gradient AllReduce fusion allreduce bandwidth.") + .def("get_costmodel_allreduce_fusion_allreduce_bandwidth", + &CostModelContext::costmodel_allreduce_fusion_allreduce_bandwidth, + "Get the parameter gradient AllReduce fusion allreduce bandwidth.") + .def("set_costmodel_allreduce_fusion_computation_time_parameter", + &CostModelContext::set_costmodel_allreduce_fusion_computation_time_parameter, + "Set the parameter gradient AllReduce fusion computation time parameter.") + .def("get_costmodel_allreduce_fusion_computation_time_parameter", + &CostModelContext::costmodel_allreduce_fusion_computation_time_parameter, + "Get the parameter gradient AllReduce fusion computation time parameter.") + .def("set_tensor_slice_align_enable", &CostModelContext::set_tensor_slice_alignment_enable, + "Set the parameter tensor_slice_align_enable in strategy generation.") + .def("get_tensor_slice_align_enable", &CostModelContext::tensor_slice_alignment_enable, + "Get the parameter tensor_slice_align_enable in strategy generation.") + .def("set_tensor_slice_align_size", &CostModelContext::set_tensor_slice_alignment_size, + "Set the parameter tensor_slice_size in strategy generation.") + .def("get_tensor_slice_align_size", &CostModelContext::tensor_slice_alignment_size, + "Get the parameter tensor_slice_size in strategy generation.") + .def("set_not_fully_use_devices", &CostModelContext::set_not_fully_use_device, + "Set the parameter not_fully_use_devices in the DP algorithm.") + .def("get_not_fully_use_devices", &CostModelContext::not_fully_use_device, + "Get the parameter not_fully_use_devices in the DP algorithm.") + .def("set_elementwise_op_strategy_follow", &CostModelContext::set_elementwise_stra_follow, + "Set the parameter elementwise_op_strategy_follow in the DP algorithm.") + .def("get_elementwise_op_strategy_follow", &CostModelContext::elementwise_stra_follow, + "Get the parameter elementwise_op_strategy_follow in the DP algorithm.") + .def("reset_cost_model", &CostModelContext::ResetCostModel, "Reset the CostModelContext.") + .def("reset_algo_parameters", &CostModelContext::ResetAlgoParameters, "Reset the AlgoParameters."); + + (void)py::module::import("atexit").attr("register")(py::cpp_function{[&]() -> void { + // only in case that c++ calling python interface, ClearResAtexit should be called. + if (mindspore::parse::python_adapter::IsPythonEnv()) { + mindspore::pipeline::ClearResAtexit(); + +#ifdef ENABLE_MINDDATA + py::module iterators = py::module::import("mindspore.dataset.engine.iterators"); + (void)iterators.attr("_cleanup")(); +#endif + } + }}); + + (void)py::class_>(m, "EventWriter_") + .def(py::init()) + .def("GetFileName", &EventWriter::GetFileName, "Get the file name.") + .def("Open", &EventWriter::Open, "Open the write file.") + .def("Write", &EventWriter::Write, "Write the serialize event.") + .def("EventCount", &EventWriter::GetWriteEventCount, "Write event count.") + .def("Flush", &EventWriter::Flush, "Flush the event.") + .def("Close", &EventWriter::Close, "Close the write.") + .def("Shut", &EventWriter::Shut, "Final close the write."); + + (void)py::class_>(m, "Oplib") + .def(py::init()) + .def("reg_op", &OpLib::RegOp, "Register op info."); + + (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective, + "Init gpu collective communication mode."); + (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective, + "Finalize gpu collective communication mode."); +} diff --git a/mindspore/ccsrc/pipeline/parse/data_converter.cc b/mindspore/ccsrc/pipeline/parse/data_converter.cc new file mode 100644 index 00000000000..aee7c35ba6c --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/data_converter.cc @@ -0,0 +1,471 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/parse/data_converter.h" +#include +#include +#include +#include +#include +#include +#include +#include "pipeline/parse/resolve.h" +#include "pipeline/parse/python_adapter.h" +#include "operator/ops.h" +#include "operator/composite/composite.h" +#include "ir/func_graph_cloner.h" +#include "utils/symbolic.h" +#include "debug/trace.h" + +namespace mindspore { +namespace parse { +using Tensor = mindspore::tensor::Tensor; +using TensorPtr = mindspore::tensor::TensorPtr; + +namespace { +bool ConvertTuple(const py::object& obj, ValuePtr* const data, bool use_signature) { + MS_LOG(DEBUG) << "converting python tuple"; + py::tuple tuple = obj.cast(); + std::vector value_list; + for (size_t it = 0; it < tuple.size(); ++it) { + ValuePtr out = nullptr; + bool success = ConvertData(tuple[it], &out, use_signature); + if (!success) { + return false; + } + value_list.push_back(out); + } + *data = std::make_shared(value_list); + + return true; +} + +bool ConvertList(const py::object& obj, ValuePtr* const data, bool use_signature) { + MS_LOG(DEBUG) << "converting python list"; + + py::list list = obj.cast(); + std::vector value_list; + for (size_t it = 0; it < list.size(); ++it) { + ValuePtr out = nullptr; + bool success = ConvertData(list[it], &out, use_signature); + if (!success) { + return false; + } + value_list.push_back(out); + } + *data = std::make_shared(value_list); + return true; +} + +bool ConvertCellList(const py::object& obj, ValuePtr* const data, bool use_signature) { + MS_LOG(DEBUG) << "converting cell list"; + py::sequence list = obj; + std::vector value_list; + for (size_t it = 0; it < list.size(); ++it) { + ValuePtr out = nullptr; + bool success = ConvertData(list[it], &out, use_signature); + if (!success) { + return false; + } + value_list.push_back(out); + } + *data = std::make_shared(value_list); + return true; +} + +bool ConvertDict(const py::object& obj, ValuePtr* data, bool use_signature) { + MS_LOG(DEBUG) << "converting python dict"; + + py::dict dict_values = obj.cast(); + std::vector> key_values; + for (auto item : dict_values) { + if (!py::isinstance(item.first)) { + MS_LOG(EXCEPTION) << "The key of dict is only support str."; + } + std::string key = py::str(item.first); + ValuePtr out = nullptr; + bool success = ConvertData(dict_values[item.first], &out, use_signature); + if (!success) { + return false; + } + key_values.emplace_back(key, out); + } + *data = std::make_shared(key_values); + return true; +} + +void ConvertNameSpace(const py::object& obj, ValuePtr* const data) { + MS_LOG(DEBUG) << "converting python module"; + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + py::object module_namespace = python_adapter::CallPyModFn(mod, PYTHON_MOD_GET_MODULE_NAMESPACE, obj); + *data = std::make_shared(RESOLVE_NAMESPACE_NAME_MODULE, py::cast(module_namespace)); +} + +void ConvertDataClass(py::object obj, ValuePtr* const data) { + MS_LOG(DEBUG) << "converting dataclass"; + // Maybe the obj is dataclass define + auto desc = py::cast(python_adapter::CallPyObjMethod(obj, PYTHON_GET_OBJ_DESC, obj)); + // desc has format "", strip the '<' and '>' by offset 1; + *data = std::make_shared(obj, std::string(desc.begin() + 1, desc.end() - 1)); +} + +bool ConvertPrimitive(py::object obj, ValuePtr* const data, bool use_signature = false) { + MS_LOG(DEBUG) << "Converting primitive object"; + + // need check the primitive is class type or instance + auto obj_type = data_converter::GetObjType(obj); + if (obj_type == RESOLVE_TYPE_CLASS_TYPE) { + auto desc = py::cast(python_adapter::CallPyObjMethod(obj, PYTHON_GET_OBJ_DESC, obj)); + // desc has format "", strip the '<' and '>' by offset 1; + *data = std::make_shared(obj, std::string(desc.begin() + 1, desc.end() - 1)); + } else { + auto primitive = obj.cast(); + if (primitive == nullptr) { + MS_LOG(ERROR) << "Resolve Primitive error, get ptr is null"; + return false; + } + if (py::hasattr(obj, "__setattr_flag__")) { + if (py::hasattr(obj, "_clone")) { + auto clone_fn = obj.attr("_clone"); + py::object new_obj = clone_fn(); + primitive = new_obj.cast(); + } + } + if (use_signature) { + *data = std::make_shared(primitive->name(), primitive); + } else { + *data = primitive; + } + } + return true; +} + +bool ConvertMetaFuncGraph(const py::object& obj, ValuePtr* const data, bool use_signature = false) { + MS_LOG(DEBUG) << "Converting MetaFuncGraph object"; + auto meta = obj.cast(); + if (meta == nullptr) { + MS_LOG(ERROR) << "Resolve MetaFuncGraph error, get ptr is null"; + return false; + } + if (use_signature) { + *data = std::make_shared(meta->name(), meta); + } else { + *data = meta; + } + return true; +} + +bool ConvertDataType(const py::object& obj, ValuePtr* const data) { + MS_LOG(DEBUG) << "Converting type object"; + auto typeptr = obj.cast(); + if (typeptr == nullptr) { + MS_LOG(ERROR) << "Resolve TypePtr error, get ptr is null"; + return false; + } + *data = typeptr; + return true; +} + +bool ConvertTensor(const py::object& obj, ValuePtr* const data) { + MS_LOG(DEBUG) << "Converting tensor object"; + + auto m_tensor = obj.cast(); + if (m_tensor == nullptr) { + MS_LOG(ERROR) << "Resolve Tensor error, get ptr is null"; + return false; + } + *data = m_tensor; + return true; +} + +bool ConvertOtherObj(py::object obj, ValuePtr* const data) { + auto obj_type = data_converter::GetObjType(obj); + MS_LOG(DEBUG) << "Converting the object(" << ((std::string)py::str(obj)) << ") detail type: " << obj_type << " "; + if (obj_type == RESOLVE_TYPE_CLASS_TYPE) { + MS_LOG(DEBUG) << "Resolve the class type, need create class instance."; + std::string desc = py::str(obj); + // desc has format "", strip the '<' and '>' by offset 1; + *data = std::make_shared(obj, std::string(desc.begin() + 1, desc.end() - 1)); + return true; + } + if (obj_type == RESOLVE_TYPE_FUNCTION || obj_type == RESOLVE_TYPE_METHOD) { + MS_LOG(DEBUG) << "Convert the obj to func graph, type is " << obj_type; + FuncGraphPtr func_graph = ConvertToFuncGraph(obj); + if (func_graph == nullptr) { + MS_LOG(ERROR) << "Parse resolve function error."; + return false; + } + *data = func_graph; + return true; + } + if (obj_type == RESOLVE_TYPE_CLASS_INSTANCE) { + // Create the namespace for common class instance + // When the obj is Cell, default parse the 'construct' + if (data_converter::IsCellInstance(obj)) { + FuncGraphPtr func_graph = ConvertToFuncGraph(obj); + if (func_graph == nullptr) { + MS_LOG(ERROR) << "Parse resolve function error."; + return false; + } + // if the cell object has specified bprop, it has user-defined bprop function parse and record it + if (py::hasattr(obj, "bprop")) { + FuncGraphPtr bprop_graph = ConvertToFuncGraph(obj, PYTHON_MOD_GET_BPROP_METHOD); + if (bprop_graph != nullptr) { + (void)func_graph->transforms().insert(std::make_pair("bprop", FuncGraphTransform(bprop_graph))); + (void)bprop_graph->transforms().insert(std::make_pair("primal", FuncGraphTransform(func_graph))); + func_graph->set_flags(FUNC_GRAPH_FLAG_DEFER_INLINE, true); + } + } + *data = func_graph; + } else { + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + py::object namespace_var = python_adapter::CallPyModFn(mod, PYTHON_MOD_GET_MEMBER_NAMESPACE_SYMBOL, obj); + *data = std::make_shared(RESOLVE_NAMESPACE_NAME_CLASS_MEMBER, namespace_var); + } + return true; + } + MS_LOG(ERROR) << "Resolve type is invalid " << ((std::string)py::str(obj)); + return false; +} +} // namespace + +bool ConvertData(const py::object& obj, ValuePtr* const data, bool use_signature) { + // check parameter valid + if (data == nullptr) { + MS_LOG(ERROR) << " data is null pointer"; + return false; + } + + bool ret = true; + ValuePtr converted = nullptr; + if (py::isinstance(obj)) { + converted = kNone; + } else if (py::isinstance(obj)) { + converted = std::make_shared(py::cast(obj)); + } else if (py::isinstance(obj)) { + converted = std::make_shared(py::cast(obj)); + } else if (py::isinstance(obj)) { + converted = std::make_shared(py::cast(obj)); + } else if (py::isinstance(obj)) { + converted = std::make_shared(py::cast(obj)); + } else if (py::isinstance(obj)) { + ret = ConvertDict(obj, &converted, use_signature); + } else if (py::isinstance(obj)) { + ret = ConvertTuple(obj, &converted, use_signature); + } else if (py::hasattr(obj, PYTHON_CELL_AS_LIST)) { + ret = ConvertCellList(obj, &converted, use_signature); + } else if (py::isinstance(obj)) { + ret = ConvertList(obj, &converted, use_signature); + } else if (py::isinstance(obj)) { + ConvertNameSpace(obj, &converted); + } else if (py::hasattr(obj, PYTHON_DATACLASS_FIELDS)) { + ConvertDataClass(obj, &converted); + } else if (py::hasattr(obj, PYTHON_PRIMITIVE_FLAG)) { + ret = ConvertPrimitive(obj, &converted, use_signature); + } else if (py::hasattr(obj, PYTHON_METAFUNCGRAPH_FLAG)) { + ret = ConvertMetaFuncGraph(obj, &converted, use_signature); + } else if (py::hasattr(obj, PYTHON_DTYPE_FLAG)) { + ret = ConvertDataType(obj, &converted); + } else if (py::hasattr(obj, PYTHON_TENSOR_FLAG)) { + ret = ConvertTensor(obj, &converted); + } else if (py::hasattr(obj, PYTHON_ENVINSTANCE_FLAG)) { + std::shared_ptr env = obj.cast>(); + converted = env; + } else { + ret = ConvertOtherObj(obj, &converted); + } + + *data = converted; + return ret; +} + +// convert data to graph +FuncGraphPtr ConvertToFuncGraph(const py::object& obj, const std::string& python_mod_get_parse_method) { + std::vector results = data_converter::GetObjKey(obj); + std::string obj_id = results[0] + python_mod_get_parse_method; + std::string obj_key = results[1]; + FuncGraphPtr func_graph = nullptr; + Any value = Any(); + bool is_cache = data_converter::GetObjectValue(obj_id, &value); + if (is_cache) { + if (value.is()) { + MS_LOG(DEBUG) << "Get the cache data, obj = " << obj_id; + func_graph = value.cast(); + return func_graph; + } + } + + func_graph = ParsePythonCode(obj, python_mod_get_parse_method); + if (func_graph == nullptr) { + MS_LOG(ERROR) << "Parse resolve function error."; + return nullptr; + } + + data_converter::MakeProperNameToFuncGraph(func_graph, obj_id); + data_converter::CacheObjectValue(obj_id, func_graph); + if (obj_key != "") { + MS_LOG(DEBUG) << "Add graph:" << obj_key << ", func_graph:" << func_graph->ToString(); + data_converter::SetObjGraphValue(obj_key, func_graph); + } + + return func_graph; +} +namespace data_converter { +static std::unordered_map object_map_ = std::unordered_map(); + +static std::unordered_map> object_graphs_map_ = + std::unordered_map>(); + +void SetObjGraphValue(const std::string& obj_key, const FuncGraphPtr& data) { + object_graphs_map_[obj_key].push_back(data); + MS_LOG(DEBUG) << "Set func graph size:" << object_graphs_map_.size(); +} + +const std::unordered_map>& GetObjGraphs() { + MS_LOG(DEBUG) << "Obj size:" << object_graphs_map_.size(); + return object_graphs_map_; +} + +void CacheObjectValue(const std::string& obj_key, const Any& data) { object_map_[obj_key] = data; } +bool GetObjectValue(const std::string& obj_key, Any* const data) { + if (object_map_.count(obj_key)) { + *data = object_map_[obj_key]; + return true; + } + return false; +} +std::vector GetObjKey(const py::object& obj) { + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + py::tuple obj_tuple = python_adapter::CallPyModFn(mod, PYTHON_MOD_RESOLVE_GET_OBJ_KEY, obj); + if (obj_tuple.size() != 2) { + MS_LOG(EXCEPTION) << "Get_obj_key must return 2 elements"; + } + return {py::cast(obj_tuple[0]), py::cast(obj_tuple[1])}; +} + +// get obj detail type +ResolveTypeDef GetObjType(const py::object& obj) { + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + auto obj_type = + ResolveTypeDef(python_adapter::CallPyModFn(mod, PYTHON_MOD_RESOLVE_GET_OBJ_TYPE, obj).cast()); + return obj_type; +} + +// get class instance detail type +ClassInstanceTypeDef GetClassInstanceType(const py::object& obj) { + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + auto class_type = + ClassInstanceTypeDef(python_adapter::CallPyModFn(mod, PYTHON_MOD_GET_CLASS_INSTANCE_TYPE, obj).cast()); + return class_type; +} + +// check the object is Cell Instance +bool IsCellInstance(const py::object& obj) { + auto class_type = GetClassInstanceType(obj); + bool isCell = (class_type == CLASS_INSTANCE_TYPE_CELL); + return isCell; +} + +// create the python class instance +py::object CreatePythonObject(const py::object& type, const py::tuple& params) { + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + py::object obj; + if (params.size() == 0) { + obj = python_adapter::CallPyModFn(mod, PYTHON_MOD_CRETAE_OBJ_INSTANCE, type); + } else { + obj = python_adapter::CallPyModFn(mod, PYTHON_MOD_CRETAE_OBJ_INSTANCE, type, params); + } + return obj; +} + +// Generate an appropriate name and set to graph debuginfo +// character <> can not used in the dot file, so change to another symbol +void MakeProperNameToFuncGraph(const FuncGraphPtr& func_graph, std::string name) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(func_graph->debug_info()); + // set detail name info of function + std::ostringstream oss; + for (size_t i = 0; i < name.size(); i++) { + if (name[i] == '<') { + oss << "「"; + } else if (name[i] == '>') { + oss << "」"; + } else { + oss << name[i]; + } + } + func_graph->debug_info()->set_full_name(oss.str()); +} + +ValuePtr PyDataToValue(const py::object& obj) { + py::object to_convert = obj; + if (py::hasattr(obj, "__parameter__")) { + to_convert = py::cast(python_adapter::GetPyObjAttr(obj, "default_input")); + } + ValuePtr value = nullptr; + (void)ConvertData(to_convert, &value); + return value; +} +void ClearObjectCache() { + object_map_.clear(); + object_graphs_map_.clear(); +} +} // namespace data_converter + +static std::unordered_map g_dataClassToClass = {}; + +// parse dataclass to mindspore Class type +ClassPtr ParseDataClass(const py::object& cls_obj) { + std::string cls_name = py::cast(python_adapter::GetPyObjAttr(cls_obj, "__name__")); + std::string cls_module = py::cast(python_adapter::GetPyObjAttr(cls_obj, "__module__")); + std::string cls = cls_module + "." + cls_name; + auto iterator = g_dataClassToClass.find(cls); + if (iterator != g_dataClassToClass.end()) { + return iterator->second; + } + + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + ClassAttrVector attributes; + py::dict names = python_adapter::CallPyModFn(mod, PYTHON_MOD_GET_DATACLASS_ATTRS, cls_obj); + for (auto& item : names) { + TypePtr type_value = item.second.cast(); + MS_EXCEPTION_IF_NULL(type_value); + MS_LOG(DEBUG) << "(name: " << py::cast(item.first) << ", type: " << type_value->ToString() << ")"; + attributes.push_back(std::make_pair(py::cast(item.first), type_value)); + } + + std::unordered_map methods_map; + py::dict methods = python_adapter::CallPyModFn(mod, PYTHON_MOD_GET_DATACLASS_METHODS, cls_obj); + for (auto& item : methods) { + std::string fun_name = item.first.cast(); + py::object obj = py::cast(item.second); + std::shared_ptr method_obj = std::make_shared(obj, fun_name); + methods_map[fun_name] = method_obj; + } + + std::shared_ptr me_class = std::make_shared(Named(cls_name), attributes, methods_map); + // static Variable for cache + // cppcheck-suppress unreadVariable + g_dataClassToClass[cls] = me_class; + + return me_class; +} + +void CleanDataClassToClassMap() { g_dataClassToClass.clear(); } +} // namespace parse +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/parse/data_converter.h b/mindspore/ccsrc/pipeline/parse/data_converter.h new file mode 100644 index 00000000000..658360bcee5 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/data_converter.h @@ -0,0 +1,60 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_PARSE_DATA_CONVERTER_H_ +#define PIPELINE_PARSE_DATA_CONVERTER_H_ + +#include +#include +#include +#include +#include +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/python_adapter.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parse { +// data convert for parse +namespace data_converter { +void CacheObjectValue(const std::string& obj_key, const Any& data); +bool GetObjectValue(const std::string& obj_key, Any* const data); + +void SetObjGraphValue(const std::string& obj_key, const FuncGraphPtr& data); + +const std::unordered_map>& GetObjGraphs(); + +std::vector GetObjKey(const py::object& obj); +ResolveTypeDef GetObjType(const py::object& obj); +ClassInstanceTypeDef GetClassInstanceType(const py::object& obj); + +bool IsCellInstance(const py::object& obj); +py::object CreatePythonObject(const py::object& type, const py::tuple& params); +void MakeProperNameToFuncGraph(const FuncGraphPtr& func_graph, std::string name); +ValuePtr PyDataToValue(const py::object& obj); +void ClearObjectCache(); +} // namespace data_converter + +ClassPtr ParseDataClass(const py::object& cls_obj); + +void CleanDataClassToClassMap(); + +} // namespace parse +} // namespace mindspore + +#endif // PIPELINE_PARSE_DATA_CONVERTER_H_ diff --git a/mindspore/ccsrc/pipeline/parse/function_block.cc b/mindspore/ccsrc/pipeline/parse/function_block.cc new file mode 100644 index 00000000000..25cc3ab4d8d --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/function_block.cc @@ -0,0 +1,362 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/parse/function_block.h" +#include +#include +#include +#include "pipeline/parse/resolve.h" +#include "pipeline/parse/parse.h" +#include "operator/ops.h" +#include "debug/info.h" +#include "debug/trace.h" + +namespace mindspore { +namespace parse { +FunctionBlock::FunctionBlock(const Parser& parser) : parser_(parser) { + func_graph_ = std::make_shared(); + matured_ = false; +} + +void FunctionBlock::AddPrevBlock(const FunctionBlockPtr& block) { prev_blocks_.push_back(block.get()); } + +// write variable records the variable name to corresponding node +void FunctionBlock::WriteVariable(const std::string& var_name, const AnfNodePtr& node) { + MS_LOG(DEBUG) << "" << func_graph_->ToString() << " write var " << var_name << " with node " << node->DebugString(); + vars_[var_name] = node; +} + +// read variable from predecessors +AnfNodePtr FunctionBlock::ReadVariable(const std::string& var) { + // get var node if it is found + if (vars_.count(var)) { + AnfNodePtr node = vars_[var]; + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + return NewValueNode(GetValueNode(node)); + } else { + return node; + } + } + // get var from predecessor block ,if can't get the make a resolve node to it + if (matured_) { + // If only one predecessor block, read the definition of var from it. + if (prev_blocks_.size() == 1) { + auto block = prev_blocks_[0]; + MS_EXCEPTION_IF_NULL(block); + return block->ReadVariable(var); + } else if (prev_blocks_.empty()) { + // get namespace and make Reslove + return MakeResolveSymbol(var); + } + } + // If have more than one predecessor blocks then build a phi node. + auto debug_info = std::make_shared(); + debug_info->set_name(var); + TraceManager::DebugTrace(std::make_shared(debug_info)); + ParameterPtr phi_param = std::make_shared(func_graph()); + TraceManager::EndTrace(); + MS_LOG(DEBUG) << "" << func_graph_->ToString() << " generate phi node " << phi_param->ToString() << " for " << var; + func_graph()->add_parameter(phi_param); + phi_nodes_[phi_param] = var; + WriteVariable(var, phi_param); + if (matured_) { + SetPhiArgument(phi_param); + } + return phi_param; +} + +// Resolve Ast operator node +AnfNodePtr FunctionBlock::MakeResolveAstOp(const py::object& op) { + auto ast = parser_.ast(); + MS_EXCEPTION_IF_NULL(ast); + TraceGuard trace_guard(parser_.GetLocation(op)); + py::tuple namespace_var = ast->CallParserObjMethod(PYTHON_PARSE_GET_AST_NAMESPACE_SYMBOL, op); + if (namespace_var.size() != 2) { + MS_LOG(EXCEPTION) << "Resolve ast op failed, get namespace tuple size=" << namespace_var.size(); + } + NameSpacePtr name_space = std::make_shared(RESOLVE_NAMESPACE_NAME_AST, namespace_var[0]); + SymbolPtr symbol = std::make_shared(namespace_var[1].cast()); + return MakeResolve(name_space, symbol); +} + +// Resolve class member, two possible: method, member variable +AnfNodePtr FunctionBlock::MakeResolveClassMember(std::string attr) { + py::object namespace_var = + parser_.ast()->CallParseModFunction(PYTHON_MOD_GET_MEMBER_NAMESPACE_SYMBOL, parser_.ast()->obj()); + NameSpacePtr name_space = std::make_shared(RESOLVE_NAMESPACE_NAME_CLASS_MEMBER, namespace_var); + SymbolPtr symbol = std::make_shared(attr); + return MakeResolve(name_space, symbol); +} + +// Make a resolve node for symbol string +AnfNodePtr FunctionBlock::MakeResolveSymbol(const std::string& value) { + if (value.compare(0, strlen("self."), "self.") == 0) { + auto start = value.find_first_of('.') + 1; + if (start >= value.size()) { + MS_LOG(ERROR) << "Find invalid resolve symbol str: " << value; + return nullptr; + } + auto bits_str = value.substr(start); + return MakeResolveClassMember(bits_str); + } + py::tuple namespace_var = parser_.ast()->CallParserObjMethod(PYTHON_PARSE_GET_NAMESPACE_SYMBOL, value); + + NameSpacePtr name_space = std::make_shared(RESOLVE_NAMESPACE_NAME_SYMBOL_STR, namespace_var[0]); + SymbolPtr symbol = std::make_shared(namespace_var[1].cast()); + return MakeResolve(name_space, symbol); +} + +AnfNodePtr FunctionBlock::MakeResolveOperation(const std::string& value) { + py::tuple namespace_var = parser_.ast()->CallParserObjMethod(PYTHON_PARSE_GET_OPERATION_NAMESPACE_SYMBOL, value); + NameSpacePtr name_space = std::make_shared(RESOLVE_NAMESPACE_NAME_COMMON_OPS, namespace_var[0]); + SymbolPtr symbol = std::make_shared(namespace_var[1].cast()); + return MakeResolve(name_space, symbol); +} + +AnfNodePtr FunctionBlock::MakeResolve(const NameSpacePtr& name_space, const SymbolPtr& resolve_symbol) { + MS_LOG(DEBUG) << "MakeResolve for " << ((std::string)py::str(name_space->obj())) << " , " + << ((std::string)resolve_symbol->symbol()); + ValueNodePtr module_node = NewValueNode(name_space); + ValueNodePtr symbol_node = NewValueNode(resolve_symbol); + auto node = func_graph()->NewCNode({NewValueNode(prim::kPrimResolve), module_node, symbol_node}); + return node; +} + +// add input for the block's phi parameter +void FunctionBlock::SetPhiArgument(const ParameterPtr& phi) { + std::string var = phi_nodes_[phi]; + MS_LOG(DEBUG) << "graph " << func_graph_->ToString() << " set phi " << phi->ToString() << " for var " << var; + for (auto& pred : prev_blocks_) { + MS_EXCEPTION_IF_NULL(pred); + MS_LOG(DEBUG) << "graph " << func_graph_->ToString() << " pred_blocks_ " << pred->func_graph_->ToString(); + AnfNodePtr arg_node = pred->ReadVariable(var); + CNodePtr jump = pred->jumps_[this]; + jump->add_input(arg_node); + } + // If the phi node in the body part of a for/while loop is being removed, + // then the closure convert phase will generate a cycle in graph if the + // loop is kept after specialization. This should be investigate further. + // Just now user has to set a flag on a function to indicate the for loop + // will definitely can be unroll as the sequence in for statement is fixed + // size in compile time. + if (parser_.func_graph()->has_flag(GRAPH_FLAG_LOOP_CAN_UNROLL) || + parser_.func_graph()->has_flag(GRAPH_FLAG_HAS_EFFECT)) { + CollectRemovablePhi(phi); + } +} + +AnfNodePtr FunctionBlock::SearchReplaceNode(const std::string& var, const ParameterPtr& phi) { + AnfNodePtr arg_node = nullptr; + for (auto& prev : prev_blocks_) { + MS_EXCEPTION_IF_NULL(prev); + AnfNodePtr temp_node = prev->ReadVariable(var); + MS_LOG(DEBUG) << "graph " << prev->func_graph_->ToString() << " phi " << phi->ToString() << " for var " << var + << " is " << temp_node->DebugString(); + if (temp_node != phi) { + if (arg_node == nullptr) { + arg_node = temp_node; + MS_LOG(DEBUG) << "graph " << prev->func_graph_->ToString() << " phi " << phi->ToString() + << " may be replaced by node " << arg_node->DebugString(); + } else if (temp_node == arg_node) { + MS_LOG(DEBUG) << "graph " << prev->func_graph_->ToString() << " phi " << phi->ToString() << " is same as node " + << arg_node->DebugString(); + } else { + MS_LOG(DEBUG) << "phi " << phi->ToString() + << " cannot be removed as it assigns to different node. node1: " << arg_node->DebugString() + << ", node2: " << temp_node->DebugString(); + return nullptr; + } + } + } + return arg_node; +} + +// Check if there is removable unnecessary phi node in this graph. +// as per the FIRM TR 3.2, a phi node can be remove if: +// +// If all arguments of a φ-function are the same value s or the φfunction itself, +// then we remove the φ-function and let all users directly uses. We call such a +// φ-function obviously unnecessary. +// When we removed a φ-function p, then we recursively try to apply this simplification +// rule with all (former) users of p, because they may have become obviously unnecessary +// due to the removal of p +// +// phi node in graph will be removed after the whole function is parsed in a DFS visit +// of that graph.The reason is : +// 1. when this function is called, not all usage of this phi node had bound to the +// graph of this function block, some may stay in vars_ in other blocks. +// 2. it's costly to iterate the graph to replace the phi for each phi. +// Args : +// phi : This parameter node is functioning as a phi node. +void FunctionBlock::CollectRemovablePhi(const ParameterPtr& phi) { + MS_EXCEPTION_IF_NULL(phi); + std::string var = phi_nodes_[phi]; + MS_LOG(DEBUG) << "check phi " << phi->ToString() << " for " << var << " in graph " << func_graph_->ToString(); + if (prev_blocks_.size() == 0) { + MS_LOG(DEBUG) << "no phi " << phi->ToString() << " for var " << var << " in graph " << func_graph_->ToString(); + return; + } + AnfNodePtr arg_node = SearchReplaceNode(var, phi); + if (arg_node != nullptr) { + MS_LOG(DEBUG) << "graph " << func_graph_->ToString() << " phi " << phi->ToString() << " can be replaced with " + << arg_node->DebugString(); + // replace var with new one. This equal to statement in TR "v0 is immediately replaced by v1." + WriteVariable(var, arg_node); + removable_phis_[phi] = arg_node; + // The following equal to statement "The φ-function defining v1, which now reads φ(v2, v1), is optimized + // recursively". check if phi1 is assigned with this phi before, then phi1 can be replaced with arg_node. + for (auto& prev : prev_blocks_) { + MS_EXCEPTION_IF_NULL(prev); + if (!prev->matured_) { + continue; + } + for (auto& phi_iter : prev->removable_phis_) { + MS_EXCEPTION_IF_NULL(phi_iter.second); + if (phi_iter.second->isa()) { + const auto& param = phi_iter.second->cast(); + if (param == phi) { + MS_LOG(DEBUG) << "graph " << prev->func_graph_->ToString() << " var " << phi_iter.first->DebugString() + << " can be replaced from " << param->DebugString() << " with " << arg_node->DebugString(); + prev->removable_phis_[phi_iter.first] = arg_node; + } + } + } + } + } +} + +// A block should be marked matured if its predecessor blocks have been processed +void FunctionBlock::Mature() { + const auto& graphParamVec = func_graph_->parameters(); + for (auto& paramItr : graphParamVec) { + MS_EXCEPTION_IF_NULL(paramItr); + ParameterPtr param = paramItr->cast(); + if (phi_nodes_.find(param) != phi_nodes_.cend()) { + SetPhiArgument(param); + } + } + matured_ = true; +} + +// Force the conditon node to bool using bool operation +CNodePtr FunctionBlock::ForceToBoolNode(const AnfNodePtr& cond) { + TraceManager::DebugTrace(std::make_shared(cond->debug_info())); + CNodePtr op_apply_node = func_graph()->NewCNode({MakeResolveOperation(NAMED_PRIMITIVE_BOOL), cond}); + TraceManager::EndTrace(); + return op_apply_node; +} + +// Perform a jump from this block to target block +void FunctionBlock::Jump(const FunctionBlockPtr& target_block, AnfNodePtr node) { + if (func_graph()->get_return() != nullptr) { + MS_LOG(EXCEPTION) << "Failure: have return node! NodeInfo: " + << trace::GetDebugInfo(func_graph()->get_return()->debug_info()); + } + std::vector input_nodes; + input_nodes.emplace_back(NewValueNode(target_block->func_graph())); + if (node != nullptr) { + input_nodes.emplace_back(node); + } + + CNodePtr jump = func_graph()->NewCNode(input_nodes); + jumps_[target_block.get()] = jump; + target_block->AddPrevBlock(shared_from_this()); + func_graph()->set_output(jump); + InsertDependItemsBeforeReturn(); +} + +// Perform a conditional jump using switch operation. +// The first CNode select graph with condition, and than execute this graph +void FunctionBlock::ConditionalJump(AnfNodePtr condNode, const FunctionBlockPtr& true_block, + const FunctionBlockPtr& false_block) { + if (func_graph()->get_return() != nullptr) { + MS_LOG(EXCEPTION) << "Failure: have return node! NodeInfo: " + << trace::GetDebugInfo(func_graph()->get_return()->debug_info()); + } + CNodePtr switch_app = + func_graph()->NewCNode({NewValueNode(prim::kPrimSwitch), condNode, NewValueNode(true_block->func_graph()), + NewValueNode(false_block->func_graph())}); + CNodePtr switch_app_new = func_graph()->NewCNode({switch_app}); + func_graph()->set_output(switch_app_new); + InsertDependItemsBeforeReturn(); +} + +void FunctionBlock::SetStateAssgin(const AnfNodePtr& target, const std::string& readid) { + state_assign_[target] = readid; +} + +void FunctionBlock::AddAutoDepend(const AnfNodePtr& target) { auto_depends_.push_back(target); } + +void FunctionBlock::InsertDependItemsBeforeReturn() { + if (!prev_blocks_.empty()) { + for (auto& prev_block : prev_blocks_) { + MS_LOG(DEBUG) << "Has prev_block " << prev_block->func_graph()->debug_info().get(); + } + } + + ValueNodePtr make_tuple_op = NewValueNode(prim::kPrimMakeTuple); + ValueNodePtr depend_op = NewValueNode(prim::kPrimDepend); + ValueNodePtr get_refkey_op = NewValueNode(prim::kPrimGetRefKey); + ValueNodePtr stop_gradient_op = NewValueNode(prim::kPrimStopGradient); + const std::string primitive_name("assign"); + const std::string module_name("mindspore.ops.functional"); + ValueNodePtr assign_op = NewValueNode(prim::GetPythonOps(primitive_name, module_name)); + + if (state_assign_.size() == 0 && auto_depends_.size() == 0) { + return; + } + AnfNodePtr state = nullptr; + std::vector vec_states; + vec_states.emplace_back(make_tuple_op); + for (auto& item : state_assign_) { + auto source = ReadVariable(item.second); + auto refkey = func_graph()->NewCNode({get_refkey_op, item.first}); + auto assign = func_graph()->NewCNode({assign_op, refkey, source}); + MS_LOG(INFO) << "SetState read " << item.first->ToString() << ", " << item.second; + vec_states.emplace_back(assign); + } + for (auto& item : auto_depends_) { + MS_LOG(DEBUG) << "auto_depends " << item->ToString(); + vec_states.emplace_back(item); + } + // if there are only make_tuple_op and another node in vec_states(the vec_states size is 2) + // do not need to make_tuple, just use the node. + if (vec_states.size() == 2) { + state = vec_states[1]; + } else { + state = func_graph()->NewCNode(vec_states); + } + + AnfNodePtr old_ret = nullptr; + auto return_node = func_graph()->get_return(); + if (return_node) { + if (return_node->inputs().size() < 1) { + MS_LOG(EXCEPTION) << "length of inputs of output node is less than 2"; + } + old_ret = return_node->input(1); + } else { + old_ret = NewValueNode(kNone); + } + AnfNodePtr stopped = func_graph()->NewCNode({stop_gradient_op, state}); + AnfNodePtr ret = func_graph()->NewCNode({depend_op, old_ret, stopped}); + func_graph()->set_output(ret, true); + state_assign_.clear(); +} +} // namespace parse +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/parse/function_block.h b/mindspore/ccsrc/pipeline/parse/function_block.h new file mode 100644 index 00000000000..0be6e472f84 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/function_block.h @@ -0,0 +1,115 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_PARSE_FUNCTION_BLOCK_H_ +#define PIPELINE_PARSE_FUNCTION_BLOCK_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "pipeline/parse/parse_base.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parse { + +class Parser; +class NameSpace; +class Symbol; +class FunctionBlock; +using FunctionBlockPtr = std::shared_ptr; + +// A function block is a straight-line code sequence with no branches, every block has one one exit point +// which is return. When parsing function, loop or branch , we use function block to track the structure of +// the original source code. +class FunctionBlock : public std::enable_shared_from_this { + public: + explicit FunctionBlock(const Parser& parser); + virtual ~FunctionBlock() {} + + FuncGraphPtr func_graph() { return func_graph_; } + void WriteVariable(const std::string& var_name, const AnfNodePtr& node); + AnfNodePtr ReadVariable(const std::string& var_name); + void AddPrevBlock(const FunctionBlockPtr& block); + void SetPhiArgument(const ParameterPtr& phi); + void CollectRemovablePhi(const ParameterPtr& phi); + // A block is matured if all its predecessors is generated + void Mature(); + CNodePtr ForceToBoolNode(const AnfNodePtr& cond); + void Jump(const FunctionBlockPtr& block, AnfNodePtr node); + AnfNodePtr SearchReplaceNode(const std::string& var, const ParameterPtr& phi); + void ConditionalJump(AnfNodePtr condNode, const FunctionBlockPtr& trueBlock, const FunctionBlockPtr& falseBlock); + // record the assign statement of self.xx weight parameter ,which will use state_setitem op + void SetStateAssgin(const AnfNodePtr& target, const std::string& readid); + void AddAutoDepend(const AnfNodePtr& target); + void InsertDependItemsBeforeReturn(); + void AddGlobalVar(const std::string& var_name) { (void)global_vars_.insert(var_name); } + bool IsGlobalVar(const std::string& var_name) { return global_vars_.find(var_name) != global_vars_.end(); } + AnfNodePtr MakeResolveAstOp(const py::object& op); + AnfNodePtr MakeResolveClassMember(std::string attr); + AnfNodePtr MakeResolveSymbol(const std::string& value); + AnfNodePtr MakeResolveOperation(const std::string& value); + AnfNodePtr MakeResolve(const std::shared_ptr& name_space, const std::shared_ptr& resolve_symbol); + const std::unordered_map& removable_phis() const { return removable_phis_; } + + private: + // block graph + FuncGraphPtr func_graph_; + + // the block's parser + const Parser& parser_; + + // A block is matured if all its prev_blocks is processed + bool matured_; + + // store the nest-level block + // refer to comments in Parser::func_block_list_; + std::vector prev_blocks_; + + // store args and variable's node + std::map vars_; + + // phi_nodes map the parameter node to variable, it can be resolved if the block's predecessors are processed + std::map phi_nodes_; + + // jumps map the successor block and the function call that perform jump + // refer to comments in Parser::func_block_list_ that how to break the cyclic reference + std::map jumps_; + + // keeps all removable phis which will be removed in one pass. + std::unordered_map removable_phis_; + + // set state nodes need to insert before function return nodes. + std::unordered_map state_assign_; + + // hold declared global variables in function + std::set global_vars_; + + // other depend need to insert before function return nodes. + // summary or some other node + std::vector auto_depends_; +}; + +} // namespace parse +} // namespace mindspore + +#endif // PIPELINE_PARSE_FUNCTION_BLOCK_H_ diff --git a/mindspore/ccsrc/pipeline/parse/parse.cc b/mindspore/ccsrc/pipeline/parse/parse.cc new file mode 100644 index 00000000000..60cc00a3070 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/parse.cc @@ -0,0 +1,1357 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/parse/parse.h" +#include +#include +#include +#include +#include +#include "operator/ops.h" +#include "pipeline/parse/data_converter.h" +#include "operator/composite/composite.h" +#include "utils/context/ms_context.h" +#include "debug/trace.h" + +namespace mindspore { +namespace parse { + +FuncGraphPtr ParsePythonCode(const py::object &obj, const std::string &python_mod_get_parse_method) { + (void)python_adapter::set_python_scoped(); + + if (obj == nullptr || py::isinstance(obj)) { + MS_LOG(ERROR) << "Parse the python code failed, obj is nullptr or none"; + return nullptr; + } + + auto ast = std::make_shared(obj); + bool success = ast->InitParseAstInfo(python_mod_get_parse_method); + if (!success) { + MS_LOG(ERROR) << "Parse code to ast tree failed."; + return nullptr; + } + + auto parser = std::make_shared(ast); + + FuncGraphPtr func_graph = parser->ParseFuncGraph(); + if (func_graph == nullptr) { + MS_LOG(ERROR) << "Parse python code failed, errcode = " << parser->errcode(); + return nullptr; + } + + return func_graph; +} + +// if any mixed precision flag add a cast node after the parameter node. +AnfNodePtr GetMixedPrecisionCastHelp(const FuncGraphPtr &func_graph, const AnfNodePtr ¶m) { + TypePtr dst_type; + if (func_graph->has_flag(GRAPH_FLAG_MIX_PRECISION_FP32)) { + dst_type = kFloat32; + } else if (func_graph->has_flag(GRAPH_FLAG_MIX_PRECISION_FP16)) { + dst_type = kFloat16; + } else { + return param; + } + auto cast_helper = prim::GetPythonOps("_mp_cast_helper", "mindspore.ops.composite.base"); + auto partial = + func_graph->NewCNode({NewValueNode(prim::kPrimPartial), NewValueNode(cast_helper), NewValueNode(dst_type)}); + auto cast = func_graph->NewCNode({NewValueNode(prim::kCompositeHyperMap), partial, param}); + return cast; +} + +FuncGraphWeakPtr Parser::top_func_graph_ = FuncGraphWeakPtr(); + +Parser::Parser(const std::shared_ptr &ast) : ast_(ast) { + errcode_ = PARSE_SUCCESS; + BuildMethodMap(); +} + +void Parser::BuildMethodMap() { + stmt_method_map_["Return"] = &Parser::ParseReturn; + stmt_method_map_["Expr"] = &Parser::ParseExpr; + stmt_method_map_["If"] = &Parser::ParseIf; + stmt_method_map_["Assign"] = &Parser::ParseAssign; + stmt_method_map_["While"] = &Parser::ParseWhile; + stmt_method_map_["For"] = &Parser::ParseFor; + stmt_method_map_["FunctionDef"] = &Parser::ParseFunctionDef; + stmt_method_map_["AugAssign"] = &Parser::ParseAugAssign; + stmt_method_map_["Global"] = &Parser::ParseGlobal; + expr_method_map_["NoneType"] = &Parser::ParseNone; + expr_method_map_["BinOp"] = &Parser::ParseBinOp; + expr_method_map_["Name"] = &Parser::ParseName; + expr_method_map_["Num"] = &Parser::ParseNum; + expr_method_map_["Str"] = &Parser::ParseStr; + expr_method_map_["NameConstant"] = &Parser::ParseNameConstant; + expr_method_map_["Call"] = &Parser::ParseCall; + expr_method_map_["IfExp"] = &Parser::ParseIfExp; + expr_method_map_["Attribute"] = &Parser::ParseAttribute; + expr_method_map_["Compare"] = &Parser::ParseCompare; + expr_method_map_["BoolOp"] = &Parser::ParseBoolOp; + expr_method_map_["Lambda"] = &Parser::ParseLambda; + expr_method_map_["Tuple"] = &Parser::ParseTuple; + expr_method_map_["List"] = &Parser::ParseList; + expr_method_map_["Subscript"] = &Parser::ParseSubscript; + expr_method_map_["Slice"] = &Parser::ParseSlice; + expr_method_map_["ExtSlice"] = &Parser::ParseExtSlice; + expr_method_map_["Index"] = &Parser::ParseIndex; + expr_method_map_["UnaryOp"] = &Parser::ParseUnaryOp; + expr_method_map_["Dict"] = &Parser::ParseDict; +} + +void Parser::UpdateTopFuncGraph(const FuncGraphPtr &func_graph) { top_func_graph_ = FuncGraphWeakPtr(func_graph); } + +void Parser::InitParserEnvironment(const py::object &obj) { + Parser::top_func_graph_ = FuncGraphWeakPtr(); + ScopeManager::GetInstance().ClearScope(); + (void)python_adapter::CallPyFn(PYTHON_MOD_PARSE_MODULE, PYTHON_PARSE_GENERATE_SCOPE, obj); +} + +void Parser::CleanParserResource() { + Parser::top_func_graph_ = FuncGraphWeakPtr(); + ScopeManager::GetInstance().ClearScope(); +} + +FuncGraphPtr Parser::ParseFuncGraph() { + // get ast FunctionDef node + py::object node = ast_->GetAstNode(); + FunctionBlockPtr pFnBlock = ParseFunction(node); + if (errcode() != PARSE_SUCCESS) { + MS_LOG(ERROR) << "Parse function error, code is " << errcode(); + return nullptr; + } + + RemoveUnnecessaryPhis(); + + MS_EXCEPTION_IF_NULL(pFnBlock); + return pFnBlock->func_graph(); +} + +void Parser::GenerateArgsNodeForFunction(const FunctionBlockPtr &block, const py::object &fn_node) { + py::object func_args = python_adapter::GetPyObjAttr(fn_node, "args"); + py::object var_arg_node = python_adapter::GetPyObjAttr(func_args, "vararg"); + block->func_graph()->set_has_vararg(!py::isinstance(var_arg_node)); + + py::object kw_arg_node = python_adapter::GetPyObjAttr(func_args, "kwarg"); + block->func_graph()->set_has_kwarg(!py::isinstance(kw_arg_node)); + + py::list kwonly_args = python_adapter::GetPyObjAttr(func_args, "kwonlyargs"); + block->func_graph()->set_kwonlyargs_count(SizeToInt(kwonly_args.size())); + + MS_EXCEPTION_IF_NULL(ast_); + py::list args = ast_->GetArgs(fn_node); + for (std::size_t i = 0; i < args.size(); i++) { + std::string arg_name = py::cast(args[i].attr("arg")); + if (ast()->target_type() == PARSE_TARGET_OBJECT_INSTANCE) { + if (arg_name == "self") { + continue; + } + } + TraceManager::DebugTrace(GetLocation(args[i])); + auto para_node = std::make_shared(block->func_graph()); + MS_EXCEPTION_IF_NULL(para_node); + TraceManager::EndTrace(); + para_node->set_name(arg_name); + para_node->debug_info()->set_name(arg_name); + block->func_graph()->add_parameter(para_node); + AnfNodePtr para_after_cast = GetMixedPrecisionCastHelp(block->func_graph(), para_node); + block->WriteVariable(arg_name, para_after_cast); + MS_LOG(DEBUG) << "The arg[" << i << "] is " << arg_name; + } +} + +void Parser::GenerateArgsDefaultValueForFunction(const FunctionBlockPtr &block, const py::object &fn_node) { + py::list defaults = ast_->GetArgsDefaultValues(fn_node); + py::list args = ast_->GetArgs(fn_node); + std::vector namelist_for_default_value; + std::vector default_values; + for (std::size_t i = 0; i < args.size(); i++) { + std::string arg_name = py::cast(args[i].attr("arg")); + if (ast()->target_type() == PARSE_TARGET_OBJECT_INSTANCE) { + if (arg_name == "self") { + continue; + } + } + + namelist_for_default_value.push_back(arg_name); + if (py::isinstance(defaults[i])) { + default_values.push_back(NewValueNode(kNullObj)); + } else { + default_values.push_back(ParseExprNode(block, defaults[i])); + } + } + block->func_graph()->SetDefaultValues(namelist_for_default_value, default_values); +} + +ScopePtr Parser::GetScopeForParseFunction() { + ScopePtr scope = ScopeManager::GetInstance().GetCurrentScope(); + if (ast()->target_type() == PARSE_TARGET_OBJECT_INSTANCE) { + py::object scope_str = python_adapter::CallPyFn(PYTHON_MOD_PARSE_MODULE, PYTHON_PARSE_GET_SCOPE_NAME, ast_->obj()); + if (!py::isinstance(scope_str)) { + auto scope_name = py::cast(scope_str); + scope = std::make_shared(scope_name); + } + } + return scope; +} + +FunctionBlockPtr Parser::ParseFunction(const py::object &node, const FunctionBlockPtr &block) { + ScopePtr scope = GetScopeForParseFunction(); + // the node created in the parsefunction context, will inherit the scope created using scope_guard + ScopeGuard scope_guard(scope); + TraceGuard trace_guard(data_converter::GetObjKey(ast()->obj())[0], GetLocation(node)); + FunctionBlockPtr pFunBlock = MakeFunctionBlock(*this); + if (block != nullptr) { + pFunBlock->AddPrevBlock(block); + } else { + func_graph_ = pFunBlock->func_graph(); + } + pFunBlock->Mature(); + auto current_fg = pFunBlock->func_graph(); + auto function_name = py::cast(python_adapter::GetPyObjAttr(node, "name")); + MS_LOG(DEBUG) << "The function name is " << function_name; + + current_fg->debug_info()->set_name(function_name); + MS_EXCEPTION_IF_NULL(ast_); + py::list deco_list = node.attr("decorator_list"); + if (deco_list.size() > 0) { + current_fg->debug_info()->set_deco_location(GetLocation(deco_list)); + } + + bool set_flag = ast_->UpdateFuncGraphFlags(current_fg); + if (!set_flag) { + MS_LOG(ERROR) << "Set flags failed"; + return nullptr; + } + GenerateArgsNodeForFunction(pFunBlock, node); + + // when parsing the top graph of construct, save the top graph + if (GetTopFuncGraph() == nullptr) { + UpdateTopFuncGraph(pFunBlock->func_graph()); + } + + // save the function node to block + pFunBlock->WriteVariable(function_name, NewValueNode(current_fg)); + + py::object funcObj = python_adapter::GetPyObjAttr(node, "body"); + (void)ParseStatements(pFunBlock, funcObj); + + if (current_fg->get_return() == nullptr) { + MS_LOG(ERROR) << "Graph return node is null, loc:" << GetLocation(node)->ToString(); + errcode_ = PARSE_NO_RETURN; + return pFunBlock; + } + GenerateArgsDefaultValueForFunction(pFunBlock, node); + return pFunBlock; +} + +FunctionBlockPtr Parser::ParseStatements(FunctionBlockPtr fn_block, const py::object &nodes) { + py::int_ pcount = python_adapter::CallPyObjMethod(nodes, "__len__"); + size_t count = IntToSize(pcount); + MS_LOG(DEBUG) << "The nodes count is " << count; + for (size_t i = 0; i < count; i++) { + auto node = py::cast(nodes)[i]; + TraceManager::DebugTrace(GetLocation(node)); + fn_block = ParseStatement(fn_block, node); + TraceManager::EndTrace(); + // insert appropriate depended items for the function block if it has a return node + if (fn_block->func_graph()->get_return() != nullptr) { + fn_block->InsertDependItemsBeforeReturn(); + } + } + return fn_block; +} + +FunctionBlockPtr Parser::ParseStatement(const FunctionBlockPtr &block, const py::object &node) { + auto node_type = ast_->GetNodeType(node); + + // check the node type + AstMainType nodeType = node_type->main_type(); + if (nodeType != AST_MAIN_TYPE_STMT) { + MS_LOG(INFO) << "Node type is error : " << nodeType; + return block; + } + // call the process function + std::string node_name = node_type->node_name(); + MS_LOG(DEBUG) << "Ast node is " << node_name; + if (stmt_method_map_.count(node_name)) { + TraceManager::DebugTrace(GetLocation(node)); + auto stmt_block = (this->*stmt_method_map_[node_name])(block, node); + TraceManager::EndTrace(); + return stmt_block; + } else { + errcode_ = PARSE_NODE_METHOD_UNSUPPORT; + py::list location = ast_->CallParserObjMethod(PYTHON_PARSE_GET_LOCATION, node); + if (location.size() < 2) { + MS_LOG(EXCEPTION) << "List size should not be less than 2."; + } + auto filename = location[0].cast(); + auto line_no = location[1].cast(); + MS_LOG(EXCEPTION) << "unsupported syntax '" << node_name << "' at " << filename << ":" << line_no; + } +} + +AnfNodePtr Parser::ParseExprNode(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast expr"; + auto node_type = ast_->GetNodeType(node); + // check the node type + AstMainType node_main_type = node_type->main_type(); + if (node_main_type != AST_MAIN_TYPE_EXPR) { + MS_LOG(ERROR) << "Node type is error : " << node_main_type; + errcode_ = PARSE_NODE_TYPE_NO_MATCH; + return nullptr; + } + // call the process function + std::string node_name = node_type->node_name(); + MS_LOG(DEBUG) << "Ast node is " << node_name; + if (expr_method_map_.count(node_name)) { + TraceManager::DebugTrace(GetLocation(node)); + auto expr_node = (this->*expr_method_map_[node_name])(block, node); + TraceManager::EndTrace(); + return expr_node; + } else { + errcode_ = PARSE_NODE_METHOD_UNSUPPORT; + py::list ret = ast_->CallParserObjMethod(PYTHON_PARSE_GET_LOCATION, node); + auto filename = ret[0].cast(); + auto line_no = ret[1].cast(); + MS_LOG(EXCEPTION) << "unsupported syntax '" << node_name << "' at " << filename << ":" << line_no; + } +} + +// process the expr statement and expand it +// eg: x.append(y) -> x = x.append(y) +FunctionBlockPtr Parser::ParseExpr(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Expr"; + // Expr only have value , no target + py::tuple expand_info = ast_->CallParserObjMethod(PYTHON_PARSE_EXPAND_EXPR_STATEMENT, node); + + // refer pypthon function expand_expr_statement, expand_info is one of the following: + // True, expr.value, x + // True, expr.value + // False, None, None + // check the expand info result + auto is_expand = py::cast(expand_info[0]); + if (is_expand) { + // process the expr statement + py::object value_object = expand_info[1]; + AnfNodePtr value_node = ParseExprNode(block, value_object); + + if (py::len(expand_info) == 2) { + // add to depend list and insert before output + block->AddAutoDepend(value_node); + } else { + // expand the assign statement + py::object target_node = expand_info[2]; + WriteAssignVars(block, target_node, value_node); + } + } + return block; +} + +LocationPtr Parser::GetLocation(const py::object &node) const { + MS_EXCEPTION_IF_NULL(ast_); + py::list ret = ast_->CallParserObjMethod(PYTHON_PARSE_GET_LOCATION, node); + if (ret.size() < 5) { + MS_LOG(EXCEPTION) << "List size should not be less than 5."; + } + // refer to Location::Location() for each member of ret: line, column, line_end, column_end. + auto location = std::make_shared(ret[0].cast(), ret[1].cast(), ret[2].cast(), + ret[3].cast(), ret[4].cast()); + return location; +} + +void Parser::MakeConditionBlocks(const FunctionBlockPtr &pre_block, const FunctionBlockPtr &true_block, + const FunctionBlockPtr &false_block) { + true_block->AddPrevBlock(pre_block); + true_block->Mature(); + + false_block->AddPrevBlock(pre_block); + false_block->Mature(); +} + +FunctionBlockPtr Parser::ParseReturn(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast return"; + MS_EXCEPTION_IF_NULL(block); + // create return valuenode + AnfNodePtr pReturnValueNode = NewValueNode(prim::kPrimReturn); + // parse the return Statements value + py::object value = python_adapter::GetPyObjAttr(node, "value"); + AnfNodePtr pReturnStatementNode = ParseExprNode(block, value); + // Create the cnode + CNodePtr pReturnCNode = block->func_graph()->NewCNode({pReturnValueNode, pReturnStatementNode}); + + block->func_graph()->set_return(pReturnCNode); + + return block; +} + +// Process binary operators,eg: `a + b`, `a | b`, etc. +AnfNodePtr Parser::ParseBinOp(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast BinOP"; + + py::object left = python_adapter::GetPyObjAttr(node, "left"); + py::object right = python_adapter::GetPyObjAttr(node, "right"); + py::object op = python_adapter::GetPyObjAttr(node, "op"); + // create left and right ANF node + AnfNodePtr left_node = ParseExprNode(block, left); + if (left_node == nullptr) { + MS_LOG(WARNING) << "DoBinOp process left node failed: " << errcode(); + return nullptr; + } + AnfNodePtr right_node = ParseExprNode(block, right); + if (right_node == nullptr) { + MS_LOG(WARNING) << "DoBinOp process right node failed:" << errcode(); + return nullptr; + } + // resolve the op + AnfNodePtr op_node = block->MakeResolveAstOp(op); + // create apply node + return block->func_graph()->NewCNode({op_node, left_node, right_node}); +} + +AnfNodePtr Parser::ParseName(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Name"; + auto name_id = py::cast(python_adapter::GetPyObjAttr(node, "id")); + MS_LOG(DEBUG) << "The Name id is " << name_id; + TraceGuard trace_guard(GetLocation(node)); + if (block->IsGlobalVar(name_id)) { + return block->MakeResolveSymbol(name_id); + } + return block->ReadVariable(name_id); +} + +AnfNodePtr Parser::ParseNone(const FunctionBlockPtr &, const py::object &) { + MS_LOG(DEBUG) << "Process ast NoneType"; + return NewValueNode(kNone); +} + +AnfNodePtr Parser::ParseNum(const FunctionBlockPtr &, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Num"; + py::object obj = python_adapter::GetPyObjAttr(node, "n"); + TraceGuard trace_guard(GetLocation(node)); + if (py::isinstance(obj)) { + MS_LOG(INFO) << "The Num is int:" << (std::string)py::str(obj); + auto data = py::cast(obj); + return NewValueNode(data); + } else if (py::isinstance(obj)) { + MS_LOG(INFO) << "The Num is float:" << (std::string)py::str(obj); + auto data = py::cast(obj); + return NewValueNode(data); + } else { + // no else actually + MS_LOG(ERROR) << "unsupported Num type : " << (std::string)py::str(obj) << GetLocation(node)->ToString(); + errcode_ = PARSE_NODE_TYPE_UNKONW; + return nullptr; + } +} + +AnfNodePtr Parser::ParseStr(const FunctionBlockPtr &, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Str"; + auto str_s = py::cast(python_adapter::GetPyObjAttr(node, "s")); + return NewValueNode(str_s); +} + +AnfNodePtr Parser::ParseNameConstant(const FunctionBlockPtr &, const py::object &node) { + MS_LOG(DEBUG) << "Process ast NameConstant"; + py::object obj = python_adapter::GetPyObjAttr(node, "value"); + TraceGuard trace_guard(GetLocation(node)); + if (py::isinstance(obj)) { + MS_LOG(INFO) << "The NameConstant is bool:" << (std::string)py::str(obj); + auto data = py::cast(obj); + return NewValueNode(data); + } else if (py::isinstance(obj)) { + MS_LOG(INFO) << "The NameConstant is none:" << (std::string)py::str(obj); + return NewValueNode(kNone); + } else { + // no else actually + MS_LOG(ERROR) << "unsupported NameConstant type: " << (std::string)py::str(obj) << GetLocation(node)->ToString(); + errcode_ = PARSE_NODE_TYPE_UNKONW; + return nullptr; + } +} +AnfNodePtr Parser::GenerateMakeTuple(const FunctionBlockPtr &block, const std::vector &element_nodes) { + AnfNodePtr make_tuple_op = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKETUPLE); + std::vector make_tuple_nodes; + make_tuple_nodes.push_back(make_tuple_op); + (void)std::transform(element_nodes.begin(), element_nodes.end(), std::back_inserter(make_tuple_nodes), + [](AnfNodePtr arg) -> AnfNodePtr { return arg; }); + return block->func_graph()->NewCNode(make_tuple_nodes); +} +// process function call, eg : f1(x, y) ... +AnfNodePtr Parser::ParseCall(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Call"; + // process function call + py::object function_ast_node = python_adapter::GetPyObjAttr(node, "func"); + AnfNodePtr call_function_anf_node = ParseExprNode(block, function_ast_node); + // function call arguments should be passed in as groups and upacked later using unpack call + py::list args = python_adapter::GetPyObjAttr(node, "args"); + std::vector packed_arguments; + std::vector group_arguments; + + bool need_unpack_args = ParseArgsInCall(block, args, &packed_arguments, &group_arguments); + bool need_unpack_keywords = ParseKeywordsInCall(block, node, &packed_arguments); + // if there is stared or keyword argument, unpack may be needed + bool need_unpack = need_unpack_args || need_unpack_keywords; + + return GenerateAnfNodeForCall(block, call_function_anf_node, packed_arguments, group_arguments, need_unpack); +} + +AnfNodePtr Parser::GenerateAnfNodeForCall(const FunctionBlockPtr &block, const AnfNodePtr &call_function_anf_node, + const std::vector &packed_arguments, + const std::vector &group_arguments, bool need_unpack) const { + // if there is keyword arguments or starred, using an unpack_call op to unpack the argument + if (need_unpack) { + std::vector unpack_call_nodes; + auto unpack_call_op = NewValueNode(std::make_shared(NAMED_METAGRAPH_UNPACKCALL)); + unpack_call_nodes.push_back(unpack_call_op); + unpack_call_nodes.push_back(call_function_anf_node); + (void)std::transform(packed_arguments.begin(), packed_arguments.end(), std::back_inserter(unpack_call_nodes), + [](AnfNodePtr node) -> AnfNodePtr { return node; }); + CNodePtr unpack_call = block->func_graph()->NewCNode(unpack_call_nodes); + return unpack_call; + } + // else there is no keyword arguments and starred, parsed as normal arguments without unpack + std::vector func_call_nodes; + func_call_nodes.push_back(call_function_anf_node); + (void)std::transform(group_arguments.begin(), group_arguments.end(), std::back_inserter(func_call_nodes), + [](AnfNodePtr node) -> AnfNodePtr { return node; }); + CNodePtr call_anf_node = block->func_graph()->NewCNode(func_call_nodes); + return call_anf_node; +} + +bool Parser::ParseArgsInCall(const FunctionBlockPtr &block, const py::list &args, + std::vector *packed_arguments, std::vector *group_arguments) { + bool need_unpack = false; + for (size_t i = 0; i < args.size(); i++) { + auto arg_node = AstSubType(py::cast(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, args[i]))); + if (arg_node == AST_SUB_TYPE_STARRED) { + if (!group_arguments->empty()) { + packed_arguments->push_back(GenerateMakeTuple(block, *group_arguments)); + } + packed_arguments->push_back(ParseExprNode(block, python_adapter::GetPyObjAttr(args[i], "value"))); + group_arguments->clear(); + need_unpack = true; + } else { + group_arguments->push_back(ParseExprNode(block, args[i])); + } + } + if (!group_arguments->empty()) { + packed_arguments->push_back(GenerateMakeTuple(block, *group_arguments)); + } + return need_unpack; +} + +bool Parser::ParseKeywordsInCall(const FunctionBlockPtr &block, const py::object &node, + std::vector *packed_arguments) { + bool need_unpack = false; + py::list keywords = python_adapter::GetPyObjAttr(node, "keywords"); + if (!keywords.empty()) { + need_unpack = true; + std::vector keys; + std::vector values; + for (size_t index = 0; index < keywords.size(); index++) { + auto kw_key = python_adapter::GetPyObjAttr(keywords[index], "arg"); + auto kw_value = python_adapter::GetPyObjAttr(keywords[index], "value"); + if (py::isinstance(kw_key)) { + packed_arguments->push_back(ParseExprNode(block, kw_value)); + } else { + auto kw_key_c = kw_key.cast(); + keys.push_back(NewValueNode(kw_key_c)); + values.push_back(ParseExprNode(block, kw_value)); + } + } + auto keys_tuple = GenerateMakeTuple(block, keys); + auto values_tuple = GenerateMakeTuple(block, values); + auto make_dict_op = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKEDICT); + std::vector make_dict_nodes; + make_dict_nodes.push_back(make_dict_op); + make_dict_nodes.push_back(keys_tuple); + make_dict_nodes.push_back(values_tuple); + packed_arguments->push_back(block->func_graph()->NewCNode(make_dict_nodes)); + } + return need_unpack; +} + +// process call attributes of class type define, eg: x.y() +AnfNodePtr Parser::ParseAttribute(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Attribute"; + + // process class value,eg: self.xx + if (ast()->target_type() == PARSE_TARGET_OBJECT_INSTANCE) { + if (ast_->IsClassMember(node)) { + std::string var_name = "self."; + std::string attr_name = node.attr("attr").cast(); + (void)var_name.append(attr_name); + if (py::hasattr(ast()->obj(), attr_name.c_str()) && + py::hasattr(ast()->obj().attr(attr_name.c_str()), PYTHON_PRIMITIVE_FLAG)) { + return block->MakeResolveSymbol(var_name); + } else { + return block->ReadVariable(var_name); + } + } + } + + // process the get attr + // Use the Primitive replace the operation resolve node (getattr) + // because the getattr will eventually be converted to Primitive node + AnfNodePtr op_node = NewValueNode(prim::kPrimGetAttr); + + // process the attr body + py::object value_body = python_adapter::GetPyObjAttr(node, "value"); + AnfNodePtr value_node = ParseExprNode(block, value_body); + if (value_node == nullptr) { + MS_LOG(WARNING) << "Parse Attribut failed"; + return nullptr; + } + + // process the node attr + auto attr_str = python_adapter::GetPyObjAttr(node, "attr").cast(); + MS_LOG(DEBUG) << "Attr = " << attr_str; + TraceManager::DebugTrace(GetLocation(python_adapter::GetPyObjAttr(node, "attr"))); + AnfNodePtr attr_node = NewValueNode(attr_str); + TraceManager::EndTrace(); + + // create the apply node + return block->func_graph()->NewCNode({op_node, value_node, attr_node}); +} + +// Process comparison expression : a == b. a > b etc. +AnfNodePtr Parser::ParseCompare(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Compare"; + + // for python comparison ,there may be if x>y>5 , + // which there is two ops , but we only support one now + py::list ops = python_adapter::GetPyObjAttr(node, "ops"); + if (ops.size() > MAX_COMPARISON_OPS_SUPPORTED) { + MS_LOG(ERROR) << "mindspore does not support comparison with operators more than one now, ops size =" << ops.size(); + return nullptr; + } + + py::object left = python_adapter::GetPyObjAttr(node, "left"); + py::list comparators = python_adapter::GetPyObjAttr(node, "comparators"); + AnfNodePtr left_node = ParseExprNode(block, left); + AnfNodePtr right_node = ParseExprNode(block, comparators[0]); + + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr op_node = block->MakeResolveAstOp(ops[0]); + + return block->func_graph()->NewCNode({op_node, left_node, right_node}); +} + +AnfNodePtr Parser::ProcessBoolOpValueList(const FunctionBlockPtr &block, const py::list &value_list, + const py::object &op) { + // if there is only one bool op now + if (value_list.size() == 1) { + AnfNodePtr first_node = ParseExprNode(block, value_list[0]); + return first_node; + } else { + py::object first = value_list[0]; + py::list rest; + for (size_t i = 1; i < value_list.size(); i++) { + rest.append(value_list[i]); + } + + AnfNodePtr first_node = ParseExprNode(block, first); + AnfNodePtr rest_node = ProcessBoolOpValueList(block, rest, op); + auto op_node = block->MakeResolveAstOp(op); + return block->func_graph()->NewCNode({op_node, first_node, rest_node}); + } +} + +// Process comparison expression : a and b. a or b . +AnfNodePtr Parser::ParseBoolOp(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast BoolOp"; + py::object op_node = python_adapter::GetPyObjAttr(node, "op"); + py::list op_values = python_adapter::GetPyObjAttr(node, "values"); + return ProcessBoolOpValueList(block, op_values, op_node); +} + +// Process a function def +FunctionBlockPtr Parser::ParseFunctionDef(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast FunctionDef"; + FunctionBlockPtr function_block = ParseFunction(node, block); + MS_EXCEPTION_IF_NULL(function_block); + + // get function name + py::str name = python_adapter::GetPyObjAttr(node, "name"); + std::string function_name = name; + ValueNodePtr valuenode_graph = NewValueNode(function_block->func_graph()); + block->WriteVariable(function_name, valuenode_graph); + return block; +} + +// Process a lambda expression . like lambda x,y: x + y +AnfNodePtr Parser::ParseLambda(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Lambda"; + FunctionBlockPtr func_block = MakeFunctionBlock(*this); + func_block->AddPrevBlock(block); + func_block->Mature(); + + // get lambda args + py::list args = ast_->GetArgs(node); + for (std::size_t i = 0; i < args.size(); i++) { + std::string arg = py::cast(args[i].attr("arg")); + TraceManager::DebugTrace(GetLocation(args[i])); + auto para_node = std::make_shared(func_block->func_graph()); + TraceManager::EndTrace(); + para_node->debug_info()->set_name(arg); + func_block->func_graph()->add_parameter(para_node); + func_block->WriteVariable(arg, para_node); + MS_LOG(DEBUG) << "The arg[" << i << "] is " << arg; + } + + py::object body_node = python_adapter::GetPyObjAttr(node, "body"); + AnfNodePtr lambda_body_node = ParseExprNode(func_block, body_node); + func_block->func_graph()->set_output(lambda_body_node); + ValueNodePtr const_graph = NewValueNode(func_block->func_graph()); + return const_graph; +} + +// process a tuple +AnfNodePtr Parser::ParseTuple(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Tuple"; + MS_EXCEPTION_IF_NULL(block); + py::tuple elts = python_adapter::GetPyObjAttr(node, "elts"); + if (elts.size() == 0) { + auto empty_tuple = std::vector(); + return NewValueNode(std::make_shared(empty_tuple)); + } + + std::vector tuple_vec; + AnfNodePtr make_tuple_op = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKETUPLE); + tuple_vec.emplace_back(make_tuple_op); + for (size_t i = 0; i < elts.size(); i++) { + AnfNodePtr node_ptr = ParseExprNode(block, elts[i]); + tuple_vec.emplace_back(node_ptr); + } + CNodePtr tuple_app = block->func_graph()->NewCNode(tuple_vec); + return tuple_app; +} + +// process a list +AnfNodePtr Parser::ParseList(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast List"; + MS_EXCEPTION_IF_NULL(block); + py::tuple elts = python_adapter::GetPyObjAttr(node, "elts"); + if (elts.size() == 0) { + auto empty_list = std::vector(); + return NewValueNode(std::make_shared(empty_list)); + } + + std::vector list_vec; + AnfNodePtr make_list_op = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKELIST); + list_vec.emplace_back(make_list_op); + for (size_t i = 0; i < elts.size(); i++) { + AnfNodePtr node_ptr = ParseExprNode(block, elts[i]); + list_vec.emplace_back(node_ptr); + } + CNodePtr list_app = block->func_graph()->NewCNode(list_vec); + return list_app; +} + +// process a subscript, such as x[y] , node expressed as value[slice] +AnfNodePtr Parser::ParseSubscript(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Subscript"; + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr op_getitem = block->MakeResolveOperation(NAMED_PRIMITIVE_GETITEM); + py::object value_node = python_adapter::GetPyObjAttr(node, "value"); + py::object slice_node = python_adapter::GetPyObjAttr(node, "slice"); + AnfNodePtr value = ParseExprNode(block, value_node); + AnfNodePtr slice = ParseExprNode(block, slice_node); + + return block->func_graph()->NewCNode({op_getitem, value, slice}); +} + +// process a slice, get the slice value +AnfNodePtr Parser::ParseSlice(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Slice"; + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr op_makeslice = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKESLICE); + py::object start = python_adapter::GetPyObjAttr(node, "lower"); + py::object stop = python_adapter::GetPyObjAttr(node, "upper"); + py::object step = python_adapter::GetPyObjAttr(node, "step"); + AnfNodePtr start_node = ParseExprNode(block, start); + AnfNodePtr stop_node = ParseExprNode(block, stop); + AnfNodePtr step_node = ParseExprNode(block, step); + + return block->func_graph()->NewCNode({op_makeslice, start_node, stop_node, step_node}); +} + +// process a extslice +AnfNodePtr Parser::ParseExtSlice(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast ExtSlice"; + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr make_tuple_op = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKETUPLE); + py::tuple slice_tuple = python_adapter::GetPyObjAttr(node, "dims"); + + std::vector node_vec; + node_vec.emplace_back(make_tuple_op); + for (size_t i = 0; i < slice_tuple.size(); i++) { + AnfNodePtr node_ptr = ParseExprNode(block, slice_tuple[i]); + node_vec.emplace_back(node_ptr); + } + CNodePtr tuple_conde = block->func_graph()->NewCNode(node_vec); + return tuple_conde; +} + +// process a index, get the index number +AnfNodePtr Parser::ParseIndex(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast Index"; + py::object value_node = python_adapter::GetPyObjAttr(node, "value"); + return ParseExprNode(block, value_node); +} + +// process a UnaryOp, +a, -b +AnfNodePtr Parser::ParseUnaryOp(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast UnaryOp"; + py::object op = python_adapter::GetPyObjAttr(node, "op"); + + MS_EXCEPTION_IF_NULL(block); + // resolve the op + AnfNodePtr op_node = block->MakeResolveAstOp(op); + + py::object operand = python_adapter::GetPyObjAttr(node, "operand"); + AnfNodePtr operand_node = ParseExprNode(block, operand); + return block->func_graph()->NewCNode({op_node, operand_node}); +} + +// process a dict ast node expression +AnfNodePtr Parser::ParseDict(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast Dict"; + py::list keys = node.attr("keys"); + py::list values = node.attr("values"); + std::vector key_nodes; + std::vector value_nodes; + for (size_t i = 0; i < keys.size(); i++) { + key_nodes.push_back(ParseExprNode(block, keys[i])); + value_nodes.push_back(ParseExprNode(block, values[i])); + } + auto keys_tuple = GenerateMakeTuple(block, key_nodes); + auto values_tuple = GenerateMakeTuple(block, value_nodes); + MS_EXCEPTION_IF_NULL(block); + auto make_dict_op = block->MakeResolveOperation(NAMED_PRIMITIVE_MAKEDICT); + return block->func_graph()->NewCNode({make_dict_op, keys_tuple, values_tuple}); +} + +// process a augment assign such as a += b; +FunctionBlockPtr Parser::ParseAugAssign(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast AugAssign"; + py::object op = python_adapter::GetPyObjAttr(node, "op"); + + MS_EXCEPTION_IF_NULL(block); + // resolve the op + AnfNodePtr op_node = block->MakeResolveAstOp(op); + py::object target_node = python_adapter::GetPyObjAttr(node, "target"); + MS_EXCEPTION_IF_NULL(ast_); + auto ast_type = AstSubType(py::cast(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, target_node))); + AnfNodePtr read_node = nullptr; + if (ast_type == AST_SUB_TYPE_NAME) { + read_node = ParseName(block, target_node); + } else if (ast_->IsClassMember(target_node)) { + read_node = ParseAttribute(block, target_node); + } else { + MS_LOG(EXCEPTION) << "not supported augassign"; + } + if (read_node == nullptr) { + MS_LOG(EXCEPTION) << "can not get target node "; + } + + py::object value = python_adapter::GetPyObjAttr(node, "value"); + AnfNodePtr value_node = ParseExprNode(block, value); + CNodePtr augassign_app = block->func_graph()->NewCNode({op_node, read_node, value_node}); + WriteAssignVars(block, target_node, augassign_app); + return block; +} + +// process global declaration such as 'global x'; +FunctionBlockPtr Parser::ParseGlobal(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast Global"; + MS_EXCEPTION_IF_NULL(block); + py::list vars = python_adapter::GetPyObjAttr(node, "names"); + for (auto &item : vars) { + block->AddGlobalVar(py::cast(item)); + } + return block; +} + +// process a if statement +FunctionBlockPtr Parser::ParseIf(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast If"; + py::object test_node = python_adapter::GetPyObjAttr(node, "test"); + AnfNodePtr condition_node = ParseExprNode(block, test_node); + MS_EXCEPTION_IF_NULL(block); + CNodePtr bool_node = block->ForceToBoolNode(condition_node); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr true_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr false_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + MakeConditionBlocks(block, true_block, false_block); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr after_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + // process the if-true branch + py::object bodyNode = python_adapter::GetPyObjAttr(node, "body"); + FunctionBlockPtr true_end = ParseStatements(true_block, bodyNode); + + // if the return_ is set ,it has its own continuation block + if (true_end->func_graph()->get_return() == nullptr) { + true_end->Jump(after_block, nullptr); + } + + // process the orelse branch + py::object orelseNode = python_adapter::GetPyObjAttr(node, "orelse"); + FunctionBlockPtr false_end = ParseStatements(false_block, orelseNode); + + // if the return_ is set ,it has its own continuation block + if (false_end->func_graph()->get_return() == nullptr) { + false_end->Jump(after_block, nullptr); + } + + block->ConditionalJump(bool_node, true_block, false_block); + after_block->Mature(); + return after_block; +} + +FunctionBlockPtr Parser::ParseWhile(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast While"; + MS_EXCEPTION_IF_NULL(block); + MS_LOG(INFO) << "Parse while statement"; + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr header_block = MakeFunctionBlock(*this); + if (MsContext::GetInstance()->is_multi_graph_sink()) { + header_block->func_graph()->set_flags(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + } + TraceManager::EndTrace(); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr body_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr after_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + body_block->AddPrevBlock(header_block); + after_block->AddPrevBlock(header_block); + block->Jump(header_block, nullptr); + + py::object test_node = python_adapter::GetPyObjAttr(node, "test"); + AnfNodePtr condition_node = ParseExprNode(header_block, test_node); + body_block->Mature(); + header_block->ConditionalJump(condition_node, body_block, after_block); + + py::object body_node = python_adapter::GetPyObjAttr(node, "body"); + FunctionBlockPtr after_body = ParseStatements(body_block, body_node); + if (after_body->func_graph()->get_return() == nullptr) { + after_body->Jump(header_block, nullptr); + } + header_block->Mature(); + after_block->Mature(); + return after_block; +} + +CNodePtr Parser::GenerateIteratorInFor(const FunctionBlockPtr &block, const py::object &node, + const AnfNodePtr &op_iter) { + py::object iter_node = python_adapter::GetPyObjAttr(node, "iter"); + AnfNodePtr iter_anf_node = ParseExprNode(block, iter_node); + return block->func_graph()->NewCNode({op_iter, iter_anf_node}); +} +CNodePtr Parser::GenerateCondInFor(const ParameterPtr &iter_param, const FunctionBlockPtr &header_block, + const AnfNodePtr &op_hasnext) { + MS_EXCEPTION_IF_NULL(header_block); + return header_block->func_graph()->NewCNode({op_hasnext, iter_param}); +} + +FunctionBlockPtr Parser::GenerateBlockInFor(const TraceInfoPtr &trace_info) { + TraceManager::DebugTrace(trace_info); + FunctionBlockPtr body_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + return body_block; +} + +// A for loop will generate 3 functions :the test, the body, and the continuation +// for x in xs: +// body +// it compiled to be following statement +// it = iter(xs) +// while hastnext(it) +// x, it = next(it) +// body +FunctionBlockPtr Parser::ParseFor(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast For"; + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr op_iter = block->MakeResolveOperation(NAMED_PRIMITIVE_ITER); + AnfNodePtr op_next = block->MakeResolveOperation(NAMED_PRIMITIVE_NEXT); + AnfNodePtr op_getitem = block->MakeResolveOperation(NAMED_PRIMITIVE_GETITEM); + AnfNodePtr op_hasnext = block->MakeResolveOperation(NAMED_PRIMITIVE_HASNEXT); + // generate the iterator apply + CNodePtr iter_apply = GenerateIteratorInFor(block, node, op_iter); + MS_EXCEPTION_IF_NULL(iter_apply); + FunctionBlockPtr header_block = + GenerateBlockInFor(std::make_shared(block->func_graph()->debug_info())); + MS_EXCEPTION_IF_NULL(header_block); + // generate the hasnext apply which is a condition + ParameterPtr iter_param = header_block->func_graph()->add_parameter(); + CNodePtr cond_apply = GenerateCondInFor(iter_param, header_block, op_hasnext); + // generate the body of the for statement + FunctionBlockPtr body_block = GenerateBlockInFor(std::make_shared(block->func_graph()->debug_info())); + MS_EXCEPTION_IF_NULL(body_block); + body_block->AddPrevBlock(header_block); + // generate the iterator next apply + // process as following: `app = next(it); target = app[0]; it = app[1];` + CNodePtr app = body_block->func_graph()->NewCNode({op_next, iter_param}); + CNodePtr target_app = body_block->func_graph()->NewCNode({op_getitem, app, NewValueNode(0)}); + py::object target_node = python_adapter::GetPyObjAttr(node, "target"); + auto name_id = py::cast(python_adapter::GetPyObjAttr(target_node, "id")); + target_app->debug_info()->set_name(name_id); + + CNodePtr iter2_app = body_block->func_graph()->NewCNode({op_getitem, app, NewValueNode(1)}); + body_block->WriteVariable(name_id, target_app); + // link the variable name with the target + auto it_info = std::make_shared(target_app->debug_info()); + iter_param->debug_info()->set_trace_info(it_info); + iter2_app->debug_info()->set_trace_info(it_info); + iter_apply->debug_info()->set_trace_info(it_info); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr after_block = MakeFunctionBlock(*this); + MS_EXCEPTION_IF_NULL(after_block); + TraceManager::EndTrace(); + after_block->AddPrevBlock(header_block); + + block->Jump(header_block, iter_apply); + body_block->Mature(); + header_block->ConditionalJump(cond_apply, body_block, after_block); + + py::object body_node = python_adapter::GetPyObjAttr(node, "body"); + FunctionBlockPtr after_body_block = ParseStatements(body_block, body_node); + if (after_body_block->func_graph()->get_return() == nullptr) { + after_body_block->Jump(header_block, iter2_app); + } + header_block->Mature(); + after_block->Mature(); + return after_block; +} +AnfNodePtr Parser::ParseIfExp(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "process ast IfExp"; + MS_EXCEPTION_IF_NULL(block); + py::object test_node = python_adapter::GetPyObjAttr(node, "test"); + AnfNodePtr condition_node = ParseExprNode(block, test_node); + CNodePtr bool_node = block->ForceToBoolNode(condition_node); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr true_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + TraceManager::DebugTrace(std::make_shared(block->func_graph()->debug_info())); + FunctionBlockPtr false_block = MakeFunctionBlock(*this); + TraceManager::EndTrace(); + + MakeConditionBlocks(block, true_block, false_block); + + // process the if-true branch + py::object bodyNode = python_adapter::GetPyObjAttr(node, "body"); + true_block->func_graph()->debug_info()->set_location(GetLocation(bodyNode)); + AnfNodePtr true_node = ParseExprNode(true_block, bodyNode); + + // process the orelse branch + py::object orelseNode = python_adapter::GetPyObjAttr(node, "orelse"); + false_block->func_graph()->debug_info()->set_location(GetLocation(orelseNode)); + AnfNodePtr false_node = ParseExprNode(false_block, orelseNode); + + true_block->func_graph()->set_output(true_node); + false_block->func_graph()->set_output(false_node); + + // Use the Primitive replace the operation resolve node (switch) + // because the switch will eventually be converted to Primitive node + CNodePtr switch_app = + block->func_graph()->NewCNode({NewValueNode(prim::kPrimSwitch), bool_node, NewValueNode(true_block->func_graph()), + NewValueNode(false_block->func_graph())}); + + std::vector call_graph_nodes{switch_app}; + CNodePtr switch_app_call = block->func_graph()->NewCNode(call_graph_nodes); + return switch_app_call; +} + +void Parser::HandleAssignName(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &assigned_node) { + MS_EXCEPTION_IF_NULL(block); + MS_EXCEPTION_IF_NULL(assigned_node); + py::str name = python_adapter::GetPyObjAttr(targ, "id"); + std::string name_id = name; + assigned_node->debug_info()->set_name(name_id); + // set the debug name of the constant graph + if (IsValueNode(assigned_node)) { + // the value should be graph + auto fg = GetValueNode(assigned_node); + if (fg->debug_info()->name().empty()) { + fg->debug_info()->set_name(name_id); + } + } + block->WriteVariable(name_id, assigned_node); +} + +void Parser::HandleAssignTuple(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &assigned_node) { + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr op_getitem = block->MakeResolveOperation(NAMED_PRIMITIVE_GETITEM); + py::list items = python_adapter::GetPyObjAttr(targ, "elts"); + for (size_t i = 0; i < items.size(); i++) { + // Use the Primitive replace the operation resolve node (getitem) + // because the getitem will eventually be converted to Primitive node + CNodePtr item_apply = block->func_graph()->NewCNode({op_getitem, assigned_node, NewValueNode(static_cast(i))}); + + py::object elt = items[i]; + WriteAssignVars(block, elt, item_apply); + } +} + +void Parser::HandleAssignClassMember(const FunctionBlockPtr &block, const py::object &targ, + const AnfNodePtr &assigned_node) { + // Now only support the self.xx = xxxxx, can't support x.y = xxxx + AnfNodePtr target_node = ParseExprNode(block, targ); + MS_EXCEPTION_IF_NULL(target_node); + + std::string var_name = "self."; + (void)var_name.append(targ.attr("attr").cast()); + MS_LOG(DEBUG) << "assign " << var_name; + + MS_EXCEPTION_IF_NULL(block); + block->WriteVariable(var_name, assigned_node); + MS_LOG(DEBUG) << "SetState write " << var_name << " : " << target_node->ToString(); + block->SetStateAssgin(target_node, var_name); +} + +void Parser::HandleAssignSubscript(const FunctionBlockPtr &block, const py::object &targ, + const AnfNodePtr &assigned_node) { + MS_EXCEPTION_IF_NULL(block); + AnfNodePtr op_setitem = block->MakeResolveOperation(NAMED_PRIMITIVE_SETITEM); + py::object value_obj = python_adapter::GetPyObjAttr(targ, "value"); + py::object slice_obj = python_adapter::GetPyObjAttr(targ, "slice"); + AnfNodePtr value_node = ParseExprNode(block, value_obj); + AnfNodePtr slice_node = ParseExprNode(block, slice_obj); + CNodePtr setitem_app = block->func_graph()->NewCNode({op_setitem, value_node, slice_node, assigned_node}); + // getitem apply should return the sequence data structure itself + std::string var_name = ""; + if (ast_->IsClassMember(value_obj)) { + var_name = "self."; + (void)var_name.append(value_obj.attr("attr").cast()); + } else { + var_name = value_obj.attr("id").cast(); + } + block->WriteVariable(var_name, setitem_app); +} + +void Parser::WriteAssignVars(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &value_node) { + MS_EXCEPTION_IF_NULL(value_node); + MS_LOG(DEBUG) << "process WriteAssignVars"; + auto ast_type = AstSubType(py::cast(ast_->CallParserObjMethod(PYTHON_PARSE_GET_AST_TYPE, targ))); + if (ast_type == AST_SUB_TYPE_NAME) { + HandleAssignName(block, targ, value_node); + } else if (ast_type == AST_SUB_TYPE_TUPLE) { + HandleAssignTuple(block, targ, value_node); + } else if (ast_type == AST_SUB_TYPE_SUBSCRIPT) { + HandleAssignSubscript(block, targ, value_node); + } else if (ast_->IsClassMember(targ)) { + HandleAssignClassMember(block, targ, value_node); + } else { + MS_LOG(EXCEPTION) << "not supported assign type: " << ast_type + << " NodeInfo: " << trace::GetDebugInfo(value_node->debug_info()); + } +} + +// process a assign statement , such as a =b, a,b = tup +FunctionBlockPtr Parser::ParseAssign(const FunctionBlockPtr &block, const py::object &node) { + MS_LOG(DEBUG) << "Process ast assgin"; + py::object value_object = python_adapter::GetPyObjAttr(node, "value"); + AnfNodePtr value_node = ParseExprNode(block, value_object); + py::object targets_object = python_adapter::GetPyObjAttr(node, "targets"); + py::int_ pcount = python_adapter::CallPyObjMethod(targets_object, "__len__"); + size_t count = IntToSize(pcount); + MS_LOG(DEBUG) << "The nodes count is " << count; + for (size_t i = 0; i < count; i++) { + auto target_node = py::cast(targets_object)[i]; + WriteAssignVars(block, target_node, value_node); + } + + return block; +} + +void Parser::RemoveUnnecessaryPhis() { + // merge all removable phis to one map; + std::unordered_map removable_phis; + for (FunctionBlockPtr &block : func_block_list_) { + MS_EXCEPTION_IF_NULL(block); + removable_phis.insert(block->removable_phis().begin(), block->removable_phis().end()); + } + + if (removable_phis.size() == 0) { + return; + } + for (auto &node : DeepUsedGraphSearch(func_graph_->get_return())) { + if (node->isa()) { + const auto &cnode = node->cast(); + auto &inputs = cnode->inputs(); + for (std::size_t i = 0; i < inputs.size(); i++) { + if (inputs[i]->isa()) { + const auto &inp = inputs[i]->cast(); + const auto &iter = removable_phis.find(inp); + if (iter == removable_phis.end()) { + continue; + } + auto &argNode = iter->second; + MS_LOG(DEBUG) << "graph " << cnode->func_graph()->ToString() << " replace phi " << inp->ToString() << " in " + << cnode->DebugString() << " with " << argNode->DebugString(); + cnode->set_input(i, argNode); + } + } + } + } +} + +// ParseAst class code +bool ParseAst::InitParseAstInfo(const std::string &python_mod_get_parse_method) { + // init the type + target_type_ = PARSE_TARGET_UNKNOW; + + // call python parse, get the parser fn + module_ = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + py::object parse_method = python_adapter::GetPyObjAttr(obj_, PYTHON_EXTERN_PARSE_METHOD); + + // get the obj type + auto type = data_converter::GetObjType(obj_); + if (type == RESOLVE_TYPE_FUNCTION) { + target_type_ = PARSE_TARGET_FUNCTION; + function_ = obj_; + } else if (type == RESOLVE_TYPE_METHOD) { + // process the method ,need get the method's self obj + target_type_ = PARSE_TARGET_METHOD; + py::object method_object = python_adapter::GetPyObjAttr(obj_, PYTHON_GET_METHOD_SELF_CLASS); + if (py::isinstance(method_object)) { + MS_LOG(ERROR) << "Get method's self object instance failed."; + return false; + } + target_type_ = PARSE_TARGET_OBJECT_INSTANCE; + function_ = obj_; + obj_ = method_object; + } else if (type == RESOLVE_TYPE_CLASS_INSTANCE) { + // obj is class instance, get the method to parse. + function_ = python_adapter::CallPyModFn(module_, python_mod_get_parse_method, obj_, parse_method); + if (py::isinstance(function_)) { + MS_LOG(ERROR) << "Get obj method function failed."; + return false; + } + target_type_ = PARSE_TARGET_OBJECT_INSTANCE; + // check the fn is method + auto obj_type = data_converter::GetObjType(function_); + if (obj_type != RESOLVE_TYPE_METHOD) { + MS_LOG(WARNING) << "Parse method function is invalid."; + return false; + } + } else { + MS_LOG(WARNING) << "Parse obj is invalid, only can parse function and obj, type = " << type; + return false; + } + + // call python parse get ast tree + parser_ = python_adapter::CallPyModFn(module_, PYTHON_MOD_PARSE_OBJECT_FUNCTION, function_, parse_method); + ast_tree_ = python_adapter::CallPyObjMethod(parser_, "parse"); + + // get fn name and module + function_module_ = py::cast(python_adapter::GetPyObjAttr(parser_, "function_module")); + function_name_ = py::cast(python_adapter::GetPyObjAttr(parser_, "function_name")); + function_filename_ = py::cast(python_adapter::GetPyObjAttr(parser_, "filename")); + function_line_offset_ = py::cast(python_adapter::GetPyObjAttr(parser_, "line_offset")); + + return true; +} + +// Get ast tree node : is the tree bode list[0] +py::object ParseAst::GetAstNode() { + py::list tree_body = python_adapter::GetPyObjAttr(ast_tree_, "body"); + py::object ast_node = tree_body[0]; + return ast_node; +} + +py::list ParseAst::GetArgs(const py::object &func_node) { + py::list ret = python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_ARGS, func_node); + return ret; +} + +py::list ParseAst::GetArgsDefaultValues(const py::object &func_node) { + py::list ret = python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_ARGS_DEFAULT_VALUES, func_node); + return ret; +} + +AstNodeTypePtr ParseAst::GetNodeType(const py::object &node) { + py::list list_value = python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_NODE_TYPE, node); + if (list_value.size() < 2) { + MS_LOG(ERROR) << "The node of python method must has 2 values."; + return nullptr; + } + auto node_name = py::cast(list_value[0]); + auto type = AstMainType(py::cast(list_value[1])); + return std::make_shared(node, node_name, type); +} + +AstSubType ParseAst::GetOpType(const py::object &node) { + auto op_type = AstSubType(python_adapter::CallPyObjMethod(parser_, PYTHON_PARSE_GET_AST_TYPE, node).cast()); + return op_type; +} + +bool ParseAst::IsClassMember(const py::object &node) { + py::object ret = CallParseModFunction(PYTHON_MOD_PARSE_CHECK_IS_CLASS_MEMBER, node); + if (!py::isinstance(ret)) { + MS_LOG(ERROR) << "The result of mod function parse, should be bool type."; + return false; + } + return ret.cast(); +} + +bool ParseAst::UpdateFuncGraphFlags(const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + MS_LOG(ERROR) << "FuncGraph is null"; + return false; + } + + if (!py::hasattr(obj_, PYTHON_EXTERN_MINDSPORE_FLAG)) { + MS_LOG(DEBUG) << "No flags"; + return true; + } + py::dict flags = python_adapter::GetPyObjAttr(obj_, PYTHON_EXTERN_MINDSPORE_FLAG); + for (auto &item : flags) { + if (!py::isinstance(item.first) || !py::isinstance(item.second)) { + MS_LOG(ERROR) << "type error in flags dict convert"; + return false; + } + auto name = py::cast(item.first); + auto value = py::cast(item.second); + MS_LOG(DEBUG) << "Flag name: " << name << ". Value: " << value; + + func_graph->set_flags(name, value); + } + + return true; +} + +} // namespace parse +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/parse/parse.h b/mindspore/ccsrc/pipeline/parse/parse.h new file mode 100644 index 00000000000..3e891e47dd4 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/parse.h @@ -0,0 +1,320 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_PARSE_PARSE_H_ +#define PIPELINE_PARSE_PARSE_H_ + +#include +#include +#include +#include +#include +#include "utils/misc.h" +#include "ir/anf.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/function_block.h" + +namespace mindspore { +namespace parse { + +// Parse status define +enum ParseStatusCode : int { + PARSE_SUCCESS = 0, + PARSE_FUNCTION_IS_NULL, // python function is null + PARSE_PARAMETER_INVALID, // parameter is invalid + PARSE_NO_RETURN, // function no return node + PARSE_NODE_TYPE_NO_MATCH, // ast node type is error + PARSE_NODE_TYPE_UNKONW, // node type is unkonw + PARSE_NODE_METHOD_UNSUPPORT, // no method to parse the node + PARSE_DONT_RESOLVE_SYMBOL, // can't resolve the string + PARSE_NOT_SUPPORTED_COMPARE_EXPR, // the comparison is not supported + PARSE_FAILURE = 0xFF +}; + +class AstNodeType; +class ParseAst; + +// Parser to parse python function +class Parser { + public: + explicit Parser(const std::shared_ptr &ast); + + ~Parser() {} + FuncGraphPtr ParseFuncGraph(); + FuncGraphPtr func_graph() const { return func_graph_; } + ParseStatusCode errcode() const { return errcode_; } + std::shared_ptr ast() const { return ast_; } + // get location info from the ast node + LocationPtr GetLocation(const py::object &node) const; + static void InitParserEnvironment(const py::object &obj); + static void CleanParserResource(); + static FuncGraphPtr GetTopFuncGraph() { return top_func_graph_.lock(); } + static void UpdateTopFuncGraph(const FuncGraphPtr &func_graph); + + private: + // process the stmt node method list + FunctionBlockPtr ParseReturn(const FunctionBlockPtr &block, const py::object &node); + // parse expression + FunctionBlockPtr ParseExpr(const FunctionBlockPtr &block, const py::object &node); + // process a if statement + FunctionBlockPtr ParseIf(const FunctionBlockPtr &block, const py::object &node); + // process a while statement + FunctionBlockPtr ParseWhile(const FunctionBlockPtr &block, const py::object &node); + // process a for statement + FunctionBlockPtr ParseFor(const FunctionBlockPtr &block, const py::object &node); + // process a function def statement + FunctionBlockPtr ParseFunctionDef(const FunctionBlockPtr &block, const py::object &node); + // process a augment assign + FunctionBlockPtr ParseAugAssign(const FunctionBlockPtr &block, const py::object &node); + // process a global declaration + FunctionBlockPtr ParseGlobal(const FunctionBlockPtr &block, const py::object &node); + // process assign statement + FunctionBlockPtr ParseAssign(const FunctionBlockPtr &block, const py::object &node); + // process the expr and slice node method list + AnfNodePtr ParseBinOp(const FunctionBlockPtr &block, const py::object &node); + // process a variable name + AnfNodePtr ParseName(const FunctionBlockPtr &block, const py::object &node); + // process NoneType + AnfNodePtr ParseNone(const FunctionBlockPtr &block, const py::object &node); + // process a integer or float number + AnfNodePtr ParseNum(const FunctionBlockPtr &block, const py::object &node); + // process a string variable + AnfNodePtr ParseStr(const FunctionBlockPtr &block, const py::object &node); + // process a name + AnfNodePtr ParseNameConstant(const FunctionBlockPtr &block, const py::object &node); + // process a function call + AnfNodePtr ParseCall(const FunctionBlockPtr &block, const py::object &node); + // process the if expression + AnfNodePtr ParseIfExp(const FunctionBlockPtr &block, const py::object &node); + // process calss type define + AnfNodePtr ParseAttribute(const FunctionBlockPtr &block, const py::object &node); + // process a compare expression + AnfNodePtr ParseCompare(const FunctionBlockPtr &block, const py::object &node); + // process a bool operation + AnfNodePtr ParseBoolOp(const FunctionBlockPtr &block, const py::object &node); + // process a lambda operation + AnfNodePtr ParseLambda(const FunctionBlockPtr &block, const py::object &node); + // process a tuple + AnfNodePtr ParseTuple(const FunctionBlockPtr &block, const py::object &node); + // process a tuple + AnfNodePtr ParseList(const FunctionBlockPtr &block, const py::object &node); + // process a tuple + AnfNodePtr ParseSubscript(const FunctionBlockPtr &block, const py::object &node); + // process a slice + AnfNodePtr ParseSlice(const FunctionBlockPtr &block, const py::object &node); + + // process a extslice + AnfNodePtr ParseExtSlice(const FunctionBlockPtr &block, const py::object &node); + + // process a tuple + AnfNodePtr ParseIndex(const FunctionBlockPtr &block, const py::object &node); + + // process a unaryop + AnfNodePtr ParseUnaryOp(const FunctionBlockPtr &block, const py::object &node); + + // process a dict ast node expression + AnfNodePtr ParseDict(const FunctionBlockPtr &block, const py::object &node); + // generate argument nodes for ast function node + void GenerateArgsNodeForFunction(const FunctionBlockPtr &block, const py::object &function_node); + // generate argument default value for ast function node + void GenerateArgsDefaultValueForFunction(const FunctionBlockPtr &block, const py::object &function_node); + // parse ast function node + FunctionBlockPtr ParseFunction(const py::object &function_node, const FunctionBlockPtr &block = nullptr); + // parse ast statements + FunctionBlockPtr ParseStatements(FunctionBlockPtr block, const py::object &stmt_node); + // parse one ast statement node + FunctionBlockPtr ParseStatement(const FunctionBlockPtr &block, const py::object &node); + // parse an ast expresion node + AnfNodePtr ParseExprNode(const FunctionBlockPtr &block, const py::object &node); + + void MakeConditionBlocks(const FunctionBlockPtr &block, const FunctionBlockPtr &trueBlock, + const FunctionBlockPtr &falseBlock); + void RemoveUnnecessaryPhis(); + // write a new var + void WriteAssignVars(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &value_node); + + // assign value to single variable name + void HandleAssignName(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &assigned_node); + + // assign value to tuple + void HandleAssignTuple(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &assigned_node); + + // assign value to class member + void HandleAssignClassMember(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &assigned_node); + + // assign value to subscript + void HandleAssignSubscript(const FunctionBlockPtr &block, const py::object &targ, const AnfNodePtr &assigned_node); + + // process a bool operation value list + AnfNodePtr ProcessBoolOpValueList(const FunctionBlockPtr &block, const py::list &value_list, const py::object &op); + + CNodePtr GenerateIteratorInFor(const FunctionBlockPtr &block, const pybind11::object &node, + const AnfNodePtr &op_iter); + + CNodePtr GenerateCondInFor(const ParameterPtr &iter_param, const FunctionBlockPtr &header_block, + const AnfNodePtr &op_hasnext); + + FunctionBlockPtr GenerateBlockInFor(const TraceInfoPtr &trace_info); + + bool ParseKeywordsInCall(const FunctionBlockPtr &block, const py::object &node, + std::vector *packed_arguments); + + bool ParseArgsInCall(const FunctionBlockPtr &block, const py::list &args, std::vector *packed_arguments, + std::vector *group_arguments); + + AnfNodePtr GenerateAnfNodeForCall(const FunctionBlockPtr &block, const AnfNodePtr &call_function_anf_node, + const std::vector &packed_arguments, + const std::vector &group_arguments, bool need_unpack) const; + ScopePtr GetScopeForParseFunction(); + void BuildMethodMap(); + FunctionBlockPtr MakeFunctionBlock(const Parser &parse) { + FunctionBlockPtr block = std::make_shared(parse); + // In order to keep effect order in the sub-graphs which generated by control flow. + // We copy the flags from the top graph to the sub-graphs. + if (func_graph_ && !func_graph_->flags().empty()) { + block->func_graph()->set_flags(func_graph_->flags()); + } + func_block_list_.push_back(block); + return block; + } + // return a make tuple for input elements list + AnfNodePtr GenerateMakeTuple(const FunctionBlockPtr &block, const std::vector &element_nodes); + + // shared_ptr will be hold by GraphManager, so just hold a weak ref here. + static FuncGraphWeakPtr top_func_graph_; + // Python function id, used to indicate whether two CNodes come from the same Python function + const std::shared_ptr &ast_; + FuncGraphPtr func_graph_; + // error code setwhen parsing ast tree + ParseStatusCode errcode_; + + // hold all reference for FunctionBlock in this round of parsing, + // so in FunctionBlock class we can use FunctionBlock* in member + // pre_blocks_ and jumps_ to break reference cycle. + std::vector func_block_list_; + using pStmtFunc = FunctionBlockPtr (Parser::*)(const FunctionBlockPtr &block, const py::object &node); + using pExprFunc = AnfNodePtr (Parser::*)(const FunctionBlockPtr &block, const py::object &node); + // define the function map to parse ast Statement + std::map stmt_method_map_; + // define the function map to parse ast expression + std::map expr_method_map_; +}; + +// AST node type define code to ast +class AstNodeType { + public: + AstNodeType(const py::object &node, const std::string &name, AstMainType type) + : node_(node), node_name_(name), main_type_(type) {} + + ~AstNodeType() {} + + std::string node_name() const { return node_name_; } + + py::object node() const { return node_; } + + AstMainType main_type() const { return main_type_; } + + private: + const py::object &node_; + const std::string node_name_; + AstMainType main_type_; +}; + +using AstNodeTypePtr = std::shared_ptr; + +// A helper class to parse python function +class ParseAst { + public: + explicit ParseAst(const py::object &obj) : obj_(obj), target_type_(PARSE_TARGET_UNKNOW), function_line_offset_(-1) {} + + ~ParseAst() = default; + + bool InitParseAstInfo(const std::string &python_mod_get_parse_method = PYTHON_MOD_GET_PARSE_METHOD); + + py::object GetAstNode(); + + py::list GetArgs(const py::object &func_node); + + py::list GetArgsDefaultValues(const py::object &func_node); + + AstNodeTypePtr GetNodeType(const py::object &node); + + AstSubType GetOpType(const py::object &node); + + template + py::object CallParserObjMethod(const std::string &method, const T &... args) { + return python_adapter::CallPyObjMethod(parser_, method, args...); + } + + template + py::object CallParseModFunction(const std::string &function, const T &... args) { + return python_adapter::CallPyModFn(module_, function, args...); + } + + const std::string &function_name() const { return function_name_; } + + const std::string &function_module() const { return function_module_; } + + const std::string &function_filename() const { return function_filename_; } + + int function_line_offset() const { return function_line_offset_; } + + py::function function() { return function_; } + + ParseTargetTypeDef target_type() const { return target_type_; } + + py::object obj() { return obj_; } + + py::object parser() { return parser_; } + + py::object module() { return module_; } + + py::object ast_tree() { return ast_tree_; } + + bool IsClassMember(const py::object &node); + + // update the graph flags + bool UpdateFuncGraphFlags(const FuncGraphPtr &func_graph); + + private: + // save obj,eg: class instance or function + py::object obj_; + + // function or class method. + py::function function_; + + py::object ast_tree_; + py::object parser_; + py::module module_; + + // Is function or method + ParseTargetTypeDef target_type_; + + std::string function_name_; + std::string function_module_; + std::string function_filename_; + int function_line_offset_; +}; + +AnfNodePtr GetMixedPrecisionCastHelp(const FuncGraphPtr &func_graph, const AnfNodePtr ¶m); + +} // namespace parse +} // namespace mindspore + +#endif // PIPELINE_PARSE_PARSE_H_ diff --git a/mindspore/ccsrc/pipeline/parse/parse_base.h b/mindspore/ccsrc/pipeline/parse/parse_base.h new file mode 100644 index 00000000000..9f92687b6fe --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/parse_base.h @@ -0,0 +1,143 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_PARSE_PARSE_BASE_H_ +#define PIPELINE_PARSE_PARSE_BASE_H_ +#include +#include +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/manager.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +namespace parse { +// define the node type +enum AstMainType : int { + AST_MAIN_TYPE_STMT = 0, // ast.Stmt + AST_MAIN_TYPE_EXPR = 1, // ast.Expr + AST_MAIN_TYPE_SLICE = 2, // ast.Slice + AST_MAIN_TYPE_UNKNOWN = 0xFF // Error +}; + +enum AstSubType : int { + AST_SUB_TYPE_AND = 3, // ast.And + AST_SUB_TYPE_OR = 4, // ast.Or + AST_SUB_TYPE_NAME = 5, // ast.Name + AST_SUB_TYPE_TUPLE = 6, // ast.Tuple + AST_SUB_TYPE_SUBSCRIPT = 7, // ast.Subscript + AST_SUB_TYPE_STARRED = 8, // ast.Starred + AST_SUB_TYPE_UNKNOWN = 0xFF // Error +}; + +// define the parse target type +enum ParseTargetTypeDef { + PARSE_TARGET_FUNCTION = 0, // funciton + PARSE_TARGET_METHOD = 1, // method + PARSE_TARGET_OBJECT_INSTANCE = 2, // object instance + PARSE_TARGET_UNKNOW = 0xFF // ERROR TYPE +}; + +// define python module name +const char PYTHON_MOD_PARSE_MODULE[] = "mindspore._extends.parse"; +const char PYTHON_MOD_PARSE_OBJECT_FUNCTION[] = "parse_cb"; +const char PYTHON_MOD_RESOLVE_FUNCTION[] = "resolve_symbol"; +const char PYTHON_MOD_RESOLVE_GET_OBJ_KEY[] = "get_object_key"; +const char PYTHON_MOD_PARSE_CHECK_IS_CLASS_MEMBER[] = "is_class_member"; +const char PYTHON_MOD_RESOLVE_GET_OBJ_TYPE[] = "get_obj_type"; +const char PYTHON_MOD_GET_CLASS_INSTANCE_TYPE[] = "get_class_instance_type"; +const char PYTHON_MOD_CRETAE_OBJ_INSTANCE[] = "create_obj_instance"; +const char PYTHON_MOD_GET_DATACLASS_ATTRS[] = "get_dataclass_attributes"; +const char PYTHON_MOD_GET_DATACLASS_METHODS[] = "get_dataclass_methods"; +const char PYTHON_MOD_GET_MODULE_NAMESPACE[] = "get_module_namespace"; +const char PYTHON_MOD_GET_MEMBER_NAMESPACE_SYMBOL[] = "get_class_member_namespace_symbol"; +const char PYTHON_MOD_GET_PARSE_METHOD[] = "get_parse_method_of_class"; +const char PYTHON_MOD_GET_BPROP_METHOD[] = "get_bprop_method_of_class"; + +const char PYTHON_PARSE_GET_ARGS[] = "get_args"; +const char PYTHON_PARSE_GET_ARGS_DEFAULT_VALUES[] = "get_args_default_values"; +const char PYTHON_PARSE_GET_NODE_TYPE[] = "get_node_type"; +const char PYTHON_PARSE_GET_AST_TYPE[] = "get_ast_type"; +const char PYTHON_PARSE_GET_NAMESPACE_SYMBOL[] = "get_namespace_symbol"; +const char PYTHON_PARSE_GET_AST_NAMESPACE_SYMBOL[] = "get_ast_namespace_symbol"; +const char PYTHON_PARSE_GET_OPERATION_NAMESPACE_SYMBOL[] = "get_operation_namespace_symbol"; +const char PYTHON_PARSE_GET_LOCATION[] = "get_location"; +const char PYTHON_PARSE_EXPAND_EXPR_STATEMENT[] = "expand_expr_statement"; +const char PYTHON_PARSE_GENERATE_SCOPE[] = "generate_scope"; +const char PYTHON_PARSE_GET_SCOPE_NAME[] = "get_scope_name"; + +// define the common name +const char NAMED_PRIMITIVE_ITER[] = "iter"; +const char NAMED_PRIMITIVE_NEXT[] = "next"; +const char NAMED_PRIMITIVE_GETITEM[] = "getitem"; +const char NAMED_PRIMITIVE_SETITEM[] = "setitem"; +const char NAMED_PRIMITIVE_HASNEXT[] = "hasnext"; +const char NAMED_PRIMITIVE_BOOL[] = "bool"; // bool: P.identity +const char NAMED_PRIMITIVE_MAKETUPLE[] = "make_tuple"; +const char NAMED_PRIMITIVE_MAKELIST[] = "make_list"; +const char NAMED_PRIMITIVE_MAKESLICE[] = "make_slice"; +const char NAMED_PRIMITIVE_MAKEDICT[] = "make_dict"; +const char NAMED_METAGRAPH_UNPACKCALL[] = "unpack_call"; + +// define NAMED_PRIMITIVE_GETATTR "getattr" +// define python inline attr +const char PYTHON_GET_METHOD_SELF_CLASS[] = "__self__"; +const char PYTHON_GET_OBJ_DESC[] = "__str__"; + +const char PYTHON_EXTERN_PARSE_METHOD[] = "__parse_method__"; +const char PYTHON_EXTERN_MINDSPORE_FLAG[] = "_mindspore_flags"; + +// define the parse constant +const int MAX_COMPARISON_OPS_SUPPORTED = 1; + +// define the Namespace name +const char RESOLVE_NAMESPACE_NAME_AST[] = "Ast"; // for ast type namespace +const char RESOLVE_NAMESPACE_NAME_CLASS_MEMBER[] = "ClassMember"; // for class member namespace +const char RESOLVE_NAMESPACE_NAME_SYMBOL_STR[] = "SymbolStr"; // for symbol str namespace +const char RESOLVE_NAMESPACE_NAME_COMMON_OPS[] = "CommonOPS"; // for common ops, eg: hasnext, next +const char RESOLVE_NAMESPACE_NAME_MODULE[] = "Module"; // fro Module namespace + +// define Resolve type +enum ResolveTypeDef : int { + RESOLVE_TYPE_NONE = 0, // resolve None + RESOLVE_TYPE_FUNCTION = 1, // reslove function + RESOLVE_TYPE_METHOD = 2, // resolve class method + RESOLVE_TYPE_CLASS_TYPE = 3, // resolve class type + RESOLVE_TYPE_CLASS_INSTANCE = 4, // resolve the class instance of common class + RESOLVE_TYPE_INVALID = 0xFF // resolve invalid +}; + +// define the class instance detail type When the type is RESOLVE_TYPE_CLASS_INSTANCE +enum ClassInstanceTypeDef { + CLASS_INSTANCE_TYPE_CELL = 0, // class instance type is Cell + CLASS_INSTANCE_TYPE_PRIMITIVE = 1, // class instance type is Primitive + CLASS_INSTANCE_TYPE_INVALID = 0xFF +}; + +// Convert python object to ValuePtr +bool ConvertData(const py::object& obj, ValuePtr* data, bool use_signature = false); + +// Convert python obj to graph +FuncGraphPtr ConvertToFuncGraph(const py::object& obj, + const std::string& python_mod_get_parse_method = PYTHON_MOD_GET_PARSE_METHOD); + +// Parse the python object to graph +FuncGraphPtr ParsePythonCode(const py::object& obj, + const std::string& python_mod_get_parse_method = PYTHON_MOD_GET_PARSE_METHOD); +} // namespace parse +} // namespace mindspore + +#endif // PIPELINE_PARSE_PARSE_BASE_H_ diff --git a/mindspore/ccsrc/pipeline/parse/python_adapter.cc b/mindspore/ccsrc/pipeline/parse/python_adapter.cc new file mode 100644 index 00000000000..776e33235e6 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/python_adapter.cc @@ -0,0 +1,93 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/parse/python_adapter.h" +#include +#include +#include + +namespace mindspore { +namespace parse { +namespace python_adapter { +// python scoped env, should only have one scoped_ instance +static std::shared_ptr scoped_ = nullptr; +// true: start process from python, false: start process from c++ +static bool python_env_ = false; +static bool use_signature_in_resolve_ = true; +void set_use_signature_in_resolve(bool use_signature) noexcept { use_signature_in_resolve_ = use_signature; } +bool UseSignatureInResolve() { return use_signature_in_resolve_; } +void set_python_env_flag(bool python_env) noexcept { python_env_ = python_env; } +bool IsPythonEnv() { return python_env_; } +void SetPythonPath(const std::string& path) { + // load the python module path + (void)python_adapter::set_python_scoped(); + py::module sys = py::module::import("sys"); + py::list sys_path = sys.attr("path"); + + // check the path is exist? + bool is_exist = false; + for (size_t i = 0; i < sys_path.size(); i++) { + std::string path_str = py::cast(sys_path[i]); + if (path_str == path) { + is_exist = true; + } + } + if (!is_exist) { + (void)sys_path.attr("append")(path.c_str()); + } +} +std::shared_ptr set_python_scoped() { + // if start process from python, no need set the python scope. + if (!python_env_) { + if ((Py_IsInitialized() == 0) && (scoped_ == nullptr)) { + scoped_ = std::make_shared(); + } + } + return scoped_; +} + +// return the module of python +py::module GetPyModule(const std::string& module) { + if (!module.empty()) { + return py::module::import(module.c_str()); + } else { + return py::none(); + } +} + +// Get the obj of attr +py::object GetPyObjAttr(const py::object& obj, const std::string& attr) { + if (!attr.empty() && !py::isinstance(obj)) { + if (py::hasattr(obj, attr.c_str())) { + return obj.attr(attr.c_str()); + } + MS_LOG(DEBUG) << "Obj have not the attr: " << attr; + } + return py::none(); +} +py::object GetPyFn(const std::string& module, const std::string& name) { + (void)python_adapter::set_python_scoped(); + if (!module.empty() && !name.empty()) { + py::module mod = py::module::import(module.c_str()); + py::object fn = mod.attr(name.c_str()); + return fn; + } + return py::none(); +} + +} // namespace python_adapter +} // namespace parse +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/parse/python_adapter.h b/mindspore/ccsrc/pipeline/parse/python_adapter.h new file mode 100644 index 00000000000..4b9cbff251d --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/python_adapter.h @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_PARSE_PYTHON_ADAPTER_H_ +#define PIPELINE_PARSE_PYTHON_ADAPTER_H_ +#include +#include +#include + +#include "pybind11/embed.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include "pipeline/parse/parse_base.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parse { +// A utility to call python interface +namespace python_adapter { +py::module GetPyModule(const std::string& module); +py::object GetPyObjAttr(const py::object& obj, const std::string& attr); +template +py::object CallPyObjMethod(const py::object& obj, const std::string& method, T... args) { + if (!method.empty() && !py::isinstance(obj)) { + return obj.attr(method.c_str())(args...); + } + return py::none(); +} + +// call python function of module +template +py::object CallPyModFn(const py::module& mod, const std::string& function, T... args) { + if (!function.empty() && !py::isinstance(mod)) { + return mod.attr(function.c_str())(args...); + } + return py::none(); +} + +// turn off the signature when ut use parser to construct a graph. +void set_use_signature_in_resolve(bool use_signature) noexcept; +bool UseSignatureInResolve(); + +std::shared_ptr set_python_scoped(); +bool IsPythonEnv(); +void SetPythonPath(const std::string& path); +void set_python_env_flag(bool python_env) noexcept; +py::object GetPyFn(const std::string& module, const std::string& name); +// Call the python function +template +py::object CallPyFn(const std::string& module, const std::string& name, T... args) { + (void)set_python_scoped(); + if (!module.empty() && !name.empty()) { + py::module mod = py::module::import(module.c_str()); + py::object fn = mod.attr(name.c_str())(args...); + return fn; + } + return py::none(); +} +} // namespace python_adapter +} // namespace parse +} // namespace mindspore + +#endif // PIPELINE_PARSE_PYTHON_ADAPTER_H_ diff --git a/mindspore/ccsrc/pipeline/parse/resolve.cc b/mindspore/ccsrc/pipeline/parse/resolve.cc new file mode 100644 index 00000000000..976c474aa45 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/resolve.cc @@ -0,0 +1,304 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/parse/resolve.h" + +#include +#include +#include +#include + +#include "pipeline/parse/data_converter.h" +#include "pipeline/parse/parse.h" +#include "pipeline/parse/python_adapter.h" +#include "utils/any.h" +#include "operator/ops.h" +#include "optimizer/opt.h" +#include "optimizer/irpass.h" +#include "./common.h" + +namespace mindspore { +namespace parse { +abstract::AbstractBasePtr ClassObject::ToAbstract() { + ClassPtr cls_ptr = ParseDataClass(obj()); + auto abs_scalar = std::make_shared(); + abs_scalar->set_type(std::make_shared()); + abs_scalar->set_value(cls_ptr); + + AbstractBasePtrList args_spec_list = {abs_scalar}; + auto func_ptr = std::make_shared(prim::kPrimMakeRecord); + return std::make_shared(func_ptr, args_spec_list); +} + +abstract::AbstractBasePtr ClassType::ToAbstract() { + auto abs_scalar = + std::make_shared(shared_from_base(), std::make_shared()); + AbstractBasePtrList args_spec_list = {abs_scalar}; + + auto func_ptr = std::make_shared(prim::kPrimCreateInstance); + auto ret_val = std::make_shared(func_ptr, args_spec_list); + ret_val->set_value_desc(ToString()); + return ret_val; +} +// call python PYTHON_MOD_RESOLVE_FUNCTION interface to resolve the symbol in corresponding namespace +bool SymbolResolver::Resolve() { + py::module mod = python_adapter::GetPyModule(PYTHON_MOD_PARSE_MODULE); + + py::object obj = namespace_->obj(); + std::string symbol = symbol_->symbol(); + if (py::isinstance(obj)) { + MS_LOG(ERROR) << "Unresolved symbol: " << symbol; + return false; + } + result_ = python_adapter::CallPyModFn(mod, PYTHON_MOD_RESOLVE_FUNCTION, obj, common::SafeCStr(symbol)); + return true; +} + +namespace { +// argument obj should be python Parameter object +// it will be converted to Parameter node here +AnfNodePtr ResolveParameterObj(const FuncGraphPtr& func_graph, const py::object& obj) { + MS_EXCEPTION_IF_NULL(func_graph); + + // parameter object should not be none + if (py::isinstance(obj)) { + MS_LOG(EXCEPTION) << "Resolve class Parameter error because obj is null."; + } + + if (!py::hasattr(obj, "name")) { + MS_LOG(EXCEPTION) << "Resolve class Parameter error: cannot find name attr for obj"; + } + + // get the parameter name from parameter object + auto name_attr = python_adapter::GetPyObjAttr(obj, "name"); + if (py::isinstance(name_attr)) { + MS_LOG(EXCEPTION) << "Parameter object should have name attribute"; + } + + std::string param_name = py::cast(name_attr); + auto top_graph = Parser::GetTopFuncGraph(); + // if the parameter node has been created , return it + AnfNodePtr para_node = nullptr; + for (auto param : top_graph->parameters()) { + auto param_node = dyn_cast(param); + if (param_node != nullptr && param_node->name() == param_name) { + para_node = param; + break; + } + } + if (para_node == nullptr) { + ParameterPtr node = top_graph->AddWeightParameter(param_name); + node->set_default_param(obj); + para_node = node; + } + auto iter = func_graph->make_ref_params().find(para_node); + if (iter == func_graph->make_ref_params().end()) { + AnfNodePtr value = GetMixedPrecisionCastHelp(func_graph, para_node); + + AnfNodePtr make_ref = NewValueNode(prim::kPrimMakeRef); + AnfNodePtr ref_key = NewValueNode(std::make_shared(param_name)); + AnfNodePtr ref_node = func_graph->NewCNode({make_ref, ref_key, value, para_node}); + func_graph->make_ref_params()[para_node] = ref_node; + func_graph->add_parameter_obj_node(ref_node); + return ref_node; + } else { + return iter->second; + } +} + +bool ResolveObjectToNode(const FuncGraphPtr& func_graph, const py::object& obj, AnfNodePtr* const node) { + AnfNodePtr output = nullptr; + if (py::hasattr(obj, "__parameter__")) { + auto param = ResolveParameterObj(func_graph, obj); + if (param == nullptr) { + MS_LOG(ERROR) << "Resolve parameter object failed, got nullptr"; + return false; + } + MS_LOG(DEBUG) << "add param graph:" << func_graph->ToString() << ", " << param->DebugString(); + + output = param; + } else if (py::hasattr(obj, "__parameter_tuple__")) { + auto tuple = obj.cast(); + std::vector args; + args.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t it = 0; it < tuple.size(); ++it) { + AnfNodePtr out = nullptr; + bool success = ResolveObjectToNode(func_graph, tuple[it], &out); + if (!success) { + MS_LOG(ERROR) << "Resolve object to node failed"; + return false; + } + args.push_back(out); + } + output = NewCNode(args, func_graph); + } else { + ValuePtr convert_result = nullptr; + bool converted = ConvertData(obj, &convert_result, parse::python_adapter::UseSignatureInResolve()); + if (!converted) { + MS_LOG(ERROR) << "Convert data failed"; + return false; + } + MS_EXCEPTION_IF_NULL(convert_result); + output = NewValueNode(convert_result); + if (convert_result->isa()) { + output = GetMixedPrecisionCastHelp(func_graph, output); + } + } + *node = output; + return true; +} +// transform the ValueTuple or ValueList of graph node to make tuple of const graph node +bool TransformVectorGraphValueNode(const FuncGraphManagerPtr& manager, const AnfNodePtr& node, + const ValueNodePtr& value_node, AnfNodePtr* const transformed) { + MS_EXCEPTION_IF_NULL(value_node); + const auto& value_vec = GetValue>(value_node->value()); + bool has_graph_in_list = false; + for (auto& elemv : value_vec) { + MS_EXCEPTION_IF_NULL(elemv); + if (elemv->isa()) { + FuncGraphPtr new_fg = elemv->cast(); + manager->AddFuncGraph(new_fg); + has_graph_in_list = true; + continue; + } + if (has_graph_in_list) { + MS_LOG(EXCEPTION) << "list has graph in it , but not all is graph"; + } + } + // The celllist or ordered_cell will be parsed as valuetuple of const graph in it, + // So if has graph in list, try to replace the node with make tuple of graph value node. + if (has_graph_in_list) { + // change the vector of graph to be make_list of graph value node + std::vector list_vec; + auto make_list_op = NewValueNode(prim::kPrimMakeTuple); + list_vec.emplace_back(make_list_op); + (void)std::transform(std::begin(value_vec), std::end(value_vec), std::back_inserter(list_vec), + [](const ValuePtr& value) { return NewValueNode(value); }); + FuncGraphPtr cnode_graph = nullptr; + auto users = manager->node_users()[node]; + for (auto& use : users) { + auto use_node = use.first; + MS_EXCEPTION_IF_NULL(use_node); + if (use_node->isa()) { + cnode_graph = use_node->func_graph(); + } + } + + if (cnode_graph) { + CNodePtr list_app = cnode_graph->NewCNode(list_vec); + // replace the ret ptr to be make_list of graph value node + *transformed = list_app; + } else { + MS_LOG(EXCEPTION) << "Can not find apply for node use when replacing node of vector of graph"; + } + } + + return true; +} +} // namespace + +AnfNodePtr ResolveSymbol(const FuncGraphManagerPtr& manager, const NameSpacePtr& name_space, const SymbolPtr& symbol, + const AnfNodePtr& node) { + if (node->func_graph() == nullptr || manager == nullptr) { + MS_LOG(EXCEPTION) << "Node " << node->DebugString() << " graph or manager is nullptr"; + } + SymbolResolver symbol_resolver(name_space, symbol, node); + if (!symbol_resolver.Resolve()) { + MS_LOG(EXCEPTION) << "Parse Resolve node failed NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } + + py::object obj = symbol_resolver.result(); + ScopeGuard scope_guard(node->scope()); + AnfNodePtr resolved_node = nullptr; + TraceManager::DebugTrace(std::make_shared(node->debug_info())); + bool success = ResolveObjectToNode(node->func_graph(), obj, &resolved_node); + if (!success) { + MS_LOG(EXCEPTION) << "Parse Resolve covert failed NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } + if (IsValueNode(resolved_node)) { + auto new_fg = GetValueNode(resolved_node); + manager->AddFuncGraph(new_fg); + } + + // if the constant node is constant of vector of graph ,add graph to manager + if (IsValueNode(resolved_node) || IsValueNode(resolved_node)) { + (void)TransformVectorGraphValueNode(manager, node, resolved_node->cast(), &resolved_node); + } + + TraceManager::EndTrace(); + return resolved_node; +} + +namespace { +opt::OptPassGroupMap GetOptResolvePasses(const opt::irpass::ResolveIRPassLib& irpass) { + opt::OptPassGroupMap map({ + {"resolve", + { + // for resolve and getattr primitive; + irpass.resolver_resolve_, + irpass.resolver_getattr_, + }}, + }); + return map; +} +} // namespace + +bool ResolveFuncGraph(const FuncGraphPtr& func_graph, const pipeline::ResourceBasePtr& res, bool use_profile) { + if (func_graph == nullptr || res == nullptr) { + MS_LOG(ERROR) << "func_graph or resource is null"; + return false; + } + opt::irpass::ResolveIRPassLib irpass; + opt::OptimizerPtr opt_resolve = opt::Optimizer::MakeOptimizer("opt_resolve", res, GetOptResolvePasses(irpass)); + + (void)parse::python_adapter::set_python_scoped(); + + abstract::AbstractBasePtrList args_spec; + MS_EXCEPTION_IF_NULL(opt_resolve); + (void)opt_resolve->step(func_graph, args_spec, use_profile); + return true; +} + +bool ResolveAll(const FuncGraphManagerPtr& manager) { + if (manager == nullptr) { + MS_LOG(ERROR) << "func graph manager is null"; + return false; + } + + if (manager->roots().size() > 1) { + MS_LOG(WARNING) + << "After call ResolveAll, only one graph will be kept in GraphManager. ResolveAll can resolve graphs" + "called from root graph, so it's not necessary to pass all graphs as roots. " + "Please ensure your usage."; + } + // should not use pipeline::Resource as Resource::Clean will clean some + // global variable such as ScopeManager, it will cause JExpandedGraphs::GetBprop + // fail as valid scope has been cleaned. + auto res = std::make_shared(); + res->set_manager(manager); + + auto roots = manager->roots(); + for (auto& fg : roots) { + bool ret = ResolveFuncGraph(fg, res, false); + if (!ret) { + MS_EXCEPTION_IF_NULL(fg); + MS_LOG(ERROR) << "Resolve fg " << fg->ToString() << " failed"; + } + } + return true; +} +} // namespace parse +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/parse/resolve.h b/mindspore/ccsrc/pipeline/parse/resolve.h new file mode 100644 index 00000000000..ccc22c72dc3 --- /dev/null +++ b/mindspore/ccsrc/pipeline/parse/resolve.h @@ -0,0 +1,157 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_PARSE_RESOLVE_H_ +#define PIPELINE_PARSE_RESOLVE_H_ + +#include +#include +#include "ir/anf.h" +#include "ir/manager.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/parse_base.h" +#include "utils/log_adapter.h" + +// forward declaration of ResourceBase +namespace mindspore { +namespace pipeline { +class ResourceBase; +using ResourceBasePtr = std::shared_ptr; +} // namespace pipeline +} // namespace mindspore + +namespace mindspore { +namespace parse { + +// NameSpace class for resolving python code. +class NameSpace : public Named { + public: + NameSpace(const std::string& module, const py::object& obj) : Named(module), module_(module), obj_(obj) {} + ~NameSpace() override = default; + MS_DECLARE_PARENT(NameSpace, Named); + + py::object obj() { return obj_; } + std::string module() { return module_; } + abstract::AbstractBasePtr ToAbstract() override { + return std::make_shared(shared_from_base(), std::make_shared()); + } + + private: + // namespace of the module + std::string module_; + // namespace object + py::object obj_; +}; +using NameSpacePtr = std::shared_ptr; + +// Symbol in NameSpace or Class which shall be resolved. +class Symbol : public Named { + public: + explicit Symbol(const std::string& symbol) : Named(symbol), symbol_(symbol) {} + explicit Symbol(const std::string& symbol, const std::string& name) : Named(name), symbol_(symbol) {} + + ~Symbol() override = default; + MS_DECLARE_PARENT(Symbol, Named); + + std::string symbol() { return symbol_; } + abstract::AbstractBasePtr ToAbstract() override { + return std::make_shared(shared_from_base(), std::make_shared()); + } + + private: + std::string symbol_; +}; +using SymbolPtr = std::shared_ptr; + +// PyObjectWrapper class wrappers resolved python object for further processing. +class PyObjectWrapper : public Named { + public: + explicit PyObjectWrapper(const py::object& obj, const std::string name = "Python object") : Named(name), obj_(obj) {} + ~PyObjectWrapper() override = default; + MS_DECLARE_PARENT(PyObjectWrapper, Named); + py::object obj() { return obj_; } + + private: + // the object that needs to be resolved + py::object obj_; +}; + +// ClassObject class wrappers dataclass +class ClassObject : public PyObjectWrapper { + public: + explicit ClassObject(const py::object& obj, const std::string name = "Python dataclass") + : PyObjectWrapper(obj, name) {} + ~ClassObject() override = default; + MS_DECLARE_PARENT(ClassObject, PyObjectWrapper); + abstract::AbstractBasePtr ToAbstract() override; +}; + +// ClassType class wrappers class name in python +class ClassType : public PyObjectWrapper { + public: + explicit ClassType(const py::object& obj, const std::string name = "Python class type") + : PyObjectWrapper(obj, name) {} + ~ClassType() override = default; + MS_DECLARE_PARENT(ClassType, PyObjectWrapper); + abstract::AbstractBasePtr ToAbstract() override; +}; + +// SymbolResolver class for resolving symbol extracted from AnfNode. +class SymbolResolver { + public: + SymbolResolver(const NameSpacePtr& name_space, const SymbolPtr& symbol, const AnfNodePtr& node) + : namespace_(name_space), symbol_(symbol), resolved_node_(node) {} + + ~SymbolResolver() = default; + + // resolve symbol in namespace and save it in result_; + bool Resolve(); + + NameSpacePtr get_namespace() { return namespace_; } + + SymbolPtr symbol() { return symbol_; } + + py::object& result() { return result_; } + + AnfNodePtr resolved_node() { return resolved_node_; } + + // Resolve result + py::object result_; + + private: + // namespace where the symbol locates + NameSpacePtr namespace_; + // the symbol that needs to be resovled + SymbolPtr symbol_; + // the node that has been resolved + AnfNodePtr resolved_node_; +}; +using SymbolResolverPtr = std::shared_ptr; +// Resolve symbol in namespace. +AnfNodePtr ResolveSymbol(const FuncGraphManagerPtr& manager, const NameSpacePtr& name_space, const SymbolPtr& symbol, + const AnfNodePtr& node); + +// Resolve one graph which normally is the root graph. FuncGraph shall be managed by res->manager(). +bool ResolveFuncGraph(const FuncGraphPtr& func_graph, const pipeline::ResourceBasePtr& res, bool use_profile = true); + +// Resolve all graphs in manager which is defined outside of pipeline::Resource. +// Mainly used for test cases or resolve graphs which will not be managed by manager. +bool ResolveAll(const FuncGraphManagerPtr& manager); + +} // namespace parse +} // namespace mindspore + +#endif // PIPELINE_PARSE_RESOLVE_H_ diff --git a/mindspore/ccsrc/pipeline/pass.cc b/mindspore/ccsrc/pipeline/pass.cc new file mode 100644 index 00000000000..02e8c5277b2 --- /dev/null +++ b/mindspore/ccsrc/pipeline/pass.cc @@ -0,0 +1,256 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/pass.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/func_graph_cloner.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/resource.h" +#include "pipeline/validator.h" +#include "optimizer/optimizer.h" +#include "optimizer/cse.h" +#include "optimizer/clean.h" +#include "optimizer/irpass.h" +#include "optimizer/control_depend.h" +#include "parallel/step_parallel.h" +#include "parallel/step_auto_parallel.h" +#include "parallel/allreduce_fusion/step_allreduce_fusion.h" +#include "utils/any.h" + +namespace mindspore { +namespace pipeline { +using OptPassGroupMap = opt::OptPassGroupMap; +using Optimizer = opt::Optimizer; +using CompileGraphs = compile::CompileGraphs; +using abstract::AnalysisResult; +using mindspore::abstract::AnalysisContextPtr; +using mindspore::validator::Validate; + +bool SimplifyDataStructuresPass(const ResourcePtr& res) { + MS_EXCEPTION_IF_NULL(res->func_graph()); + + FuncGraphPtr func_graph = res->func_graph(); + opt::SimplifyDataStructures(func_graph, res->manager()); + + abstract::AbstractBasePtrList args_spec; + auto parameters = func_graph->parameters(); + (void)std::transform(parameters.begin(), parameters.end(), std::back_inserter(args_spec), + [](const AnfNodePtr& p) -> AbstractBasePtr { return p->abstract(); }); + FuncGraphPtr new_fg = Renormalize(res, func_graph, args_spec); + res->set_func_graph(new_fg); + res->set_args_spec(args_spec); + return true; +} + +namespace { +OptPassGroupMap GetOptPassesA(const opt::irpass::OptimizeIRPassLib& irpass) { + opt::OptPassConfig a_1 = opt::OptPassConfig({ + irpass.switch_simplify_, + + // Safe inlining + irpass.inline_, + irpass.partial_eliminate_, + irpass.replace_applicator_, + + // Specialization + irpass.specialize_transform_, + + // Arithmetic simplifications + irpass.arithmetic_simplify_, + irpass.addn_zero_filter_, + + // Miscellaneous + irpass.item_tuple_eliminate_, + irpass.env_get_set_item_, + irpass.new_env_get_item_, + irpass.add_env_get_item_, + irpass.cast_eliminate_, + irpass.reshape_eliminate_, + irpass.reduce_eliminate_, + irpass.tile_eliminate_, + irpass.transpose_eliminate_, + irpass.minmaximum_grad_, + irpass.get_make_ref_eliminate_, + }); + opt::OptPassConfig a_2 = opt::OptPassConfig({ + irpass.merge_addn_, + irpass.float_tuple_getitem_switch_, + irpass.float_env_getitem_switch_, + irpass.incorporate_getitem_, + irpass.incorporate_getitem_switch_, + irpass.incorporate_call_, + irpass.incorporate_call_switch_, + irpass.incorporate_env_getitem_, + irpass.incorporate_env_getitem_switch_, + irpass.new_env_get_item_, + }); + opt::OptPassConfig a_3 = opt::OptPassConfig({ + irpass.same_eliminate_, + irpass.replace_applicator_, + }); + opt::OptPassConfig virtual_dataset = opt::OptPassConfig({irpass.virtual_dataset_eliminate_}); + opt::OptPassConfig grad = opt::OptPassConfig({irpass.inline_, irpass.expand_jprim_}, true); + + OptPassGroupMap map_a({{"a_1", a_1}, + {"a_2", a_2}, + {"auto_parallel", opt::OptPassConfig(parallel::StepAutoParallel)}, + {"parallel", opt::OptPassConfig(parallel::StepParallel)}, + {"allreduce_fusion", opt::OptPassConfig(parallel::StepAllreduceFusion)}, + {"virtual_dataset", virtual_dataset}, + {"grad", grad}, + {"renormalize", opt::OptPassConfig::Renormalize()}, + {"cse", opt::OptPassConfig(opt::CSE(false))}, + {"a_3", a_3}}); + + return map_a; +} + +OptPassGroupMap GetOptPassesB(const opt::irpass::OptimizeIRPassLib& irpass) { + opt::OptPassConfig b_1 = opt::OptPassConfig({ + irpass.zero_like_fill_zero_, + irpass.item_tuple_eliminate_, + irpass.float_tuple_getitem_switch_, + irpass.reset_defer_inline_, + irpass.inline_, + irpass.special_op_eliminate_, + irpass.stop_gradient_eliminate_, + irpass.get_make_ref_eliminate_, + }); + opt::OptPassConfig b_2 = opt::OptPassConfig({ + irpass.replace_refkey_by_param_, + irpass.make_ref_eliminate_, + }); + OptPassGroupMap map({ + {"b_1", b_1}, + {"b_2", b_2}, + {"renormalize", opt::OptPassConfig::Renormalize()}, + {"cse", opt::OptPassConfig(opt::CSE(false))}, + }); + return map; +} + +OptPassGroupMap GetControlPhases(const opt::irpass::OptimizeIRPassLib& irpass) { + opt::OptPassConfig control_group = opt::OptPassConfig({irpass.convert_switch_replacement_}); + OptPassGroupMap map({ + {"control_group", control_group}, + {"renormalize", opt::OptPassConfig::Renormalize()}, + }); + return map; +} + +OptPassGroupMap GetPreparePhases(const opt::irpass::OptimizeIRPassLib& irpass) { + opt::OptPassConfig prepare_group = opt::OptPassConfig({irpass.print_tuple_wrapper_}); + OptPassGroupMap map({{"prepare_group", prepare_group}}); + return map; +} + +static std::unordered_map> g_pass_opts = {}; + +void InitOpt(const ResourcePtr& res) { + if (g_pass_opts.size() == 0) { + opt::irpass::OptimizeIRPassLib irpass; + g_pass_opts["opt_a"] = Optimizer::MakeOptimizer("opt_a", res, GetOptPassesA(irpass)); + g_pass_opts["opt_b"] = Optimizer::MakeOptimizer("opt_b", res, GetOptPassesB(irpass)); + g_pass_opts["opt_control"] = Optimizer::MakeOptimizer("opt_control", res, GetControlPhases(irpass)); + g_pass_opts["opt_prepare"] = Optimizer::MakeOptimizer("opt_prepare", res, GetPreparePhases(irpass)); + } +} +} // namespace + +void ReclaimOptimizer() { + for (auto& opt : g_pass_opts) { + opt.second = nullptr; + } + g_pass_opts.clear(); +} + +bool OptPassGroup(const ResourcePtr& res, const std::string& name) { + if (res->func_graph() == nullptr) { + MS_LOG(ERROR) << "opt passes int error"; + return false; + } + + abstract::AbstractBasePtrList args = res->args_spec(); + FuncGraphPtr func_graph = res->func_graph(); + MS_LOG(DEBUG) << "start " << name << " func graph:" << func_graph->ToString() << ", " + << func_graph->get_return()->DebugString(true); + InitOpt(res); + if (g_pass_opts.find(name) != g_pass_opts.end()) { + res->set_func_graph(g_pass_opts[name]->step(func_graph, args)); + } + return true; +} + +bool OptPassAGroup(const ResourcePtr& res) { return OptPassGroup(res, "opt_a"); } +bool OptPassBGroup(const ResourcePtr& res) { return OptPassGroup(res, "opt_b"); } +bool ControlGroup(const ResourcePtr& res) { return OptPassGroup(res, "opt_control"); } +bool PrepareGroup(const ResourcePtr& res) { return OptPassGroup(res, "opt_prepare"); } + +bool AddControlDependPass(const ResourcePtr& res) { + FuncGraphPtr func_graph = res->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + + if (func_graph->has_flag(GRAPH_FLAG_EFFECT_PATIAL_ORDER)) { + opt::AddControlDepend(func_graph); + } + for (auto fg : func_graph->func_graphs_used_total()) { + MS_EXCEPTION_IF_NULL(fg); + if (fg->has_flag(GRAPH_FLAG_EFFECT_PATIAL_ORDER)) { + opt::AddControlDepend(fg); + } + } + return true; +} + +bool CconvPass(const ResourcePtr& res) { + MS_EXCEPTION_IF_NULL(res->func_graph()); + FuncGraphPtr func_graph = res->func_graph(); + FuncGraphPtr new_fg = LiftingClone(func_graph); + res->set_func_graph(new_fg); + return true; +} + +bool ValidatePass(const ResourcePtr& res) { + MS_EXCEPTION_IF_NULL(res->func_graph()); + FuncGraphPtr func_graph = res->func_graph(); + Validate(func_graph); + return true; +} + +std::vector kVmPasses = {{"simplify_data_structures", SimplifyDataStructuresPass}, + {"opt_a", OptPassAGroup}, + {"opt_b", OptPassBGroup}, + {"add_control_depend", AddControlDependPass}, + {"cconv", CconvPass}}; + +std::vector kGePasses = {{"simplify_data_structures", SimplifyDataStructuresPass}, + {"opt_a", OptPassAGroup}, + {"opt_b", OptPassBGroup}, + {"add_control_depend", AddControlDependPass}, + {"opt_control", ControlGroup}, + {"opt_prepare", PrepareGroup}, + {"cconv", CconvPass}}; +} // namespace pipeline +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/pass.h b/mindspore/ccsrc/pipeline/pass.h new file mode 100644 index 00000000000..03ed8eb3708 --- /dev/null +++ b/mindspore/ccsrc/pipeline/pass.h @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PIPELINE_PASS_H_ +#define MINDSPORE_CCSRC_PIPELINE_PASS_H_ + +#include +#include +#include +#include +#include "pipeline/resource.h" + +namespace mindspore { +namespace pipeline { +using PassItem = std::pair>; + +extern std::vector kGePasses; +extern std::vector kVmPasses; + +bool CconvPass(const ResourcePtr& res); +bool ValidatePass(const ResourcePtr& res); +bool ConvertPrepareAdapt(const ResourcePtr& res); +bool AddControlDependPass(const ResourcePtr& res); + +void ReclaimOptimizer(); +} // namespace pipeline +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PIPELINE_PASS_H_ diff --git a/mindspore/ccsrc/pipeline/pipeline.cc b/mindspore/ccsrc/pipeline/pipeline.cc new file mode 100644 index 00000000000..35336e975b8 --- /dev/null +++ b/mindspore/ccsrc/pipeline/pipeline.cc @@ -0,0 +1,1298 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/pipeline.h" + +#include +#include +#include +#include +#include + +#include "pipeline/pass.h" +#include "pipeline/parse/data_converter.h" +#include "optimizer/ad/dfunctor.h" +#include "ir/meta_tensor.h" +#include "transform/convert.h" +#include "transform/df_graph_manager.h" +#include "transform/graph_builder.h" +#include "transform/graph_runner.h" +#include "debug/anf_ir_dump.h" +#include "debug/anf_ir_utils.h" +#include "utils/config_manager.h" +#include "utils/convert_utils.h" +#include "utils/utils.h" +#include "utils/base_ref.h" +#include "vm/segment_runner.h" +#include "parallel/context.h" +#include "parallel/graph_util/get_parallel_info.h" +#include "device/kernel_runtime_manager.h" +#include "debug/trace.h" + +namespace mindspore { +// namespace to support intermediate representation definition +namespace pipeline { +using Tensor = mindspore::tensor::Tensor; +using MetaTensor = mindspore::tensor::MetaTensor; +using TensorOrderMap = std::map>; +using mindspore::abstract::AbstractTensor; +using mindspore::abstract::AbstractTensorPtr; +using mindspore::abstract::AbstractTuple; +using mindspore::abstract::AbstractTuplePtr; +using mindspore::transform::DfGraphConvertor; +using mindspore::transform::DfGraphManager; +using mindspore::transform::GeTensorPtr; +using mindspore::transform::MeTensorPtr; +using mindspore::transform::Status; +using mindspore::transform::TransformUtil; + +const char IR_TYPE_ANF[] = "anf_ir"; +const char IR_TYPE_ONNX[] = "onnx_ir"; + +ExecutorPyPtr ExecutorPy::executor_ = nullptr; +std::mutex ExecutorPy::instance_lock_; + +std::unordered_map + g_args_cache; + +namespace { +std::string GetBaseNameForIR(int stage_idx, const std::string& action_name) { + std::ostringstream oss; + auto ms_context = MsContext::GetInstance(); + if (ms_context == nullptr) { + MS_LOG(EXCEPTION) << "ms_context is nullptr"; + } + auto save_graphs_path = ms_context->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + oss << save_graphs_path << "/" << stage_idx << "_" << action_name; + return oss.str(); +} + +std::string GetFilePathName(const std::string& file_name) { + std::ostringstream oss; + auto ms_context = MsContext::GetInstance(); + if (ms_context == nullptr) { + MS_LOG(EXCEPTION) << "ms_context is nullptr"; + } + auto save_graphs_path = ms_context->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + oss << save_graphs_path << "/" << file_name; + return oss.str(); +} +} // namespace + +// We will not execute graph when output is constant or just input itself. +static bool IsGraphOutputValueNodeOrParameter(const AnfNodePtr& output, const py::tuple& args, + const std::shared_ptr& ret_val) { + if (output->isa()) { + MS_LOG(INFO) << "Graph's output is a constant. No need to execute."; + ValuePtr value = GetValueNode(output); + *ret_val = ValuePtrToPyData(value); + return true; + } + + // Adapter will transform values in __init__() and construct() to parameters, this could cause + // inputs (a.k.a args in current function) size less than parameters'. + if (output->isa()) { + MS_LOG(INFO) << "Graph's output is a parameter. If all params are inputs, no need to execute."; + if (args.empty()) { + MS_LOG(EXCEPTION) << "Inputs size is 0, let graph to be executed."; + } + // Find the right parameter as ret_val. + auto func_graph = output->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + auto params = func_graph->parameters(); + if (params.empty()) { + MS_EXCEPTION(UnknownError) << "Graph's parameters size is 0"; + } + if (args.size() != params.size()) { + MS_LOG(EXCEPTION) << "Input size " << args.size() << " not equal to params size " << params.size() + << ", let graph to be executed."; + } + + auto it = std::find(params.begin(), params.end(), output); + if (it == params.end()) { + MS_EXCEPTION(UnknownError) << "When graph output is Parameter, it should be found in graph parameters"; + } + size_t index = it - params.cbegin(); + if (index >= args.size()) { + MS_EXCEPTION(UnknownError) << "Index " << index << " equal or larger than args size " << args.size() << "."; + } + *ret_val = args[index]; + return true; + } + return false; +} + +py::tuple GenerateKey(const std::string& name, const std::unordered_map& defaults) { + MS_LOG(DEBUG) << "GenerateKey args size:" << defaults.size(); + abstract::AbstractBasePtrList args_spec; + + for (auto arg : defaults) { + if (py::isinstance(arg.second)) { + MS_LOG(EXCEPTION) << "GenerateKey failed, argument input should not be py::module"; + } + ValuePtr converted = nullptr; + if (!parse::ConvertData(arg.second, &converted)) { + MS_LOG(EXCEPTION) << "GenerateKey convert arg failed"; + } + args_spec.push_back(abstract::FromValue(converted, true)); + } + if (g_args_cache.count(args_spec) == 0) { + static int key = 0; + MS_LOG(INFO) << "start new args and compile key:" << key; + g_args_cache[args_spec] = key++; + } + py::tuple argSpec = py::tuple(2); + argSpec[0] = name; + argSpec[1] = g_args_cache[args_spec]; + return argSpec; +} + +py::bool_ VerifyInputSignature(const py::list input_signature, const py::tuple inputs) { + MS_LOG(DEBUG) << "Verify args size:" << inputs.size(); + if (inputs.size() != input_signature.size()) { + MS_LOG(ERROR) << "signature size not equal to args size"; + return false; + } + + size_t count = 0; + for (auto arg_obj : inputs) { + if (py::hasattr(arg_obj, PYTHON_TENSOR_FLAG)) { + MS_LOG(DEBUG) << "Verify Tensor"; + std::shared_ptr m_tensor = arg_obj.cast>(); + if (m_tensor == nullptr) { + MS_LOG(ERROR) << "Verify Tensor error, get ptr is null"; + return false; + } + std::shared_ptr sig = input_signature[count].cast>(); + std::vector sig_shape = sig->shape(); + TypePtr sig_type = sig->Dtype(); + + std::vector tensor_shape = m_tensor->shape_c(); + if (tensor_shape != sig_shape) { + MS_LOG(ERROR) << "Python input shape is incompatible with input_signature"; + return false; + } + + if (*m_tensor->Dtype() != *sig_type) { + MS_LOG(ERROR) << "Python input type(" << m_tensor->Dtype()->ToString() << ") incompatible with input_signature(" + << sig_type->ToString() << ")"; + return false; + } + } + count++; + } + + return true; +} + +ExecutorPy::ExecutorPy() { + // because Ge only support one Session exist at the same time ,so we delete the old one + DfGraphManager::GetInstance().DeleteGraphRunner(); + DfGraphManager::GetInstance().DeleteGeSession(); +} + +ResourcePtr ExecutorPy::GetResource(const std::string& phase) { + MS_LOG(DEBUG) << "phase size:" << info_.size(); + if (info_.count(phase) == 0) { + return nullptr; + } + return info_[phase]->resource; +} + +std::string GetPhasePrefix(const std::string& phase) { + auto pos = phase.find('.'); + if (pos == std::string::npos) { + MS_LOG(EXCEPTION) << "phase has no . for prefix" << phase; + } + return phase.substr(0, pos); +} + +FuncGraphPtr ExecutorPy::GetFuncGraph(const std::string& phase) { + if (info_.count(phase) == 0) { + MS_LOG(EXCEPTION) << "no phase in executor:" << GetPhasePrefix(phase); + } + return info_[phase]->func_graph; +} + +std::size_t ExecutorPy::ArgListSize(const std::string& phase) { + if (info_.count(phase) == 0) { + MS_LOG(EXCEPTION) << "no phase in executor:" << GetPhasePrefix(phase); + } + return info_[phase]->arg_list_size; +} + +compile::VmEvalFuncPtr ExecutorPy::GetVmEvalFunc(const std::string& phase) { + ResourcePtr res = GetResource(phase); + MS_EXCEPTION_IF_NULL(res); + if (res->results().find(kOutput) != res->results().end() && res->results()[kOutput].is()) { + return res->results()[kOutput].cast(); + } + MS_LOG(ERROR) << "GetVmEvalFunc vm model can't find kOutput:" << kOutput; + return nullptr; +} + +bool ExecutorPy::HasCompiled(const std::string& phase) const { + if (info_.count(phase) == 0) { + return false; + } + return true; +} + +py::bytes ExecutorPy::GetFuncGraphProto(const std::string& phase, const std::string& ir_type) { + FuncGraphPtr fg_ptr = GetFuncGraph(phase); + if (fg_ptr == nullptr) { + for (auto& item : info_) { + MS_LOG(DEBUG) << "Phase key is: " << item.first; + } + MS_LOG(EXCEPTION) << "Can not find func graph " << phase; + } + + if (ir_type == IR_TYPE_ANF) { + std::string proto_str = GetFuncGraphProtoString(fg_ptr); + if (proto_str.empty()) { + MS_LOG(EXCEPTION) << "Graph proto is empty."; + } + return proto_str; + } + + if (ir_type == IR_TYPE_ONNX) { + std::string proto_str = GetOnnxProtoString(fg_ptr); + if (proto_str.empty()) { + MS_LOG(EXCEPTION) << "Graph proto is empty."; + } + return proto_str; + } + + MS_LOG(EXCEPTION) << "Unknown ir type: " << ir_type; +} + +py::dict ExecutorPy::GetParameterLayout(const std::string& phase) { + MS_LOG(DEBUG) << "GetParameterLayout!"; + std::string layout_graph = phase + kStepParallelGraph; + auto graph = GetFuncGraph(layout_graph); + return mindspore::parallel::GetParameterLayout(graph); +} + +py::dict ExecutorPy::GetCNodeStrategy(const std::string& phase) { + MS_LOG(DEBUG) << "GetCNodeStrategy!"; + std::string layout_graph = phase + kStepParallelGraph; + auto graph = GetFuncGraph(layout_graph); + return mindspore::parallel::GetCNodeStrategy(graph); +} + +py::dict ExecutorPy::GetAllreduceFusion(const std::string& phase) { + MS_LOG(INFO) << "GetAllreduceFusion!"; + auto graph = GetFuncGraph(phase); + return mindspore::parallel::GetAllreduceFusion(graph); +} + +void ExecutorPy::DelNetRes(const std::string& id) { +#ifdef ENABLE_GE + FinalizeGe(); +#endif + if (executor_ != nullptr) { + bool flag = false; + auto tmp_info = info_; + for (auto& item : tmp_info) { + if (item.first.find(id) != string::npos) { + MS_LOG(INFO) << "delete network res:" << item.first; + (void)info_.erase(item.first); + flag = true; + } + } + + if (flag && info_.size() == 0) { + DfGraphManager::GetInstance().DeleteGraphRunner(); + DfGraphManager::GetInstance().EraseAnfGraph(); + DfGraphManager::GetInstance().DeleteGeSession(); + } + } +} + +void ExecutorPy::ClearRes() { + MS_LOG(INFO) << "clean executor Resrouce!"; + executor_ = nullptr; +} + +ExecutorPy::~ExecutorPy() { + MS_LOG(INFO) << "Release Executor!"; + ConfigManager::GetInstance().ResetConfig(); +} + +void ExecutorPy::SaveCompiledGraph(const std::string& phase_s) { + // save the graph to ExecutorPy + FuncGraphPtr func_graph = info_[phase_s]->resource->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(parallel::ParallelContext::GetInstance()); + std::string parallel_mode = parallel::ParallelContext::GetInstance()->parallel_mode(); + + MS_LOG(INFO) << "save compiled func graph(" << func_graph->ToString() << ") phase(" << phase_s << ")!"; + info_[phase_s]->func_graph = func_graph; + if ((func_graph != nullptr) && + ((parallel_mode == parallel::AUTO_PARALLEL) || (parallel_mode == parallel::SEMI_AUTO_PARALLEL))) { + MS_LOG(DEBUG) << "save model parallel parameter layout graph!"; + func_graph = info_[phase_s]->resource->results()[kStepParallelGraph].cast(); + ExecutorInfoPtr excutor_info = std::make_shared(); + std::string layout_graph = phase_s + kStepParallelGraph; + excutor_info->func_graph = func_graph; + info_[layout_graph] = excutor_info; + } else { + MS_LOG(DEBUG) << "save model parallel parameter layout graph null!"; + } + MS_LOG(INFO) << "end save compiled func graph!"; +} + +bool ExecutorPy::ChangeExportGeirUseVmFlag(bool use_vm, const std::string& phase_s) const { + std::string phase_prefix = GetPhasePrefix(phase_s); + + if (use_vm && phase_prefix == "export") { + MS_LOG(INFO) << "use ge backend to export geir"; + use_vm = false; + } + return use_vm; +} + +void ExecutorPy::GetGeBackendPolicy() const { + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + std::string backend = ms_context->backend_policy(); + if (backend != "ge") { + MS_LOG(EXCEPTION) << "" << backend << " backend policy is not supported under ge backend!"; + } +} + +bool ExecutorPy::CompileInner(const py::object& obj, const py::tuple& args, const py::object& phase, bool use_vm) { + MS_LOG(DEBUG) << "Start ExecutorPy compile!"; + if ((!py::isinstance(phase))) { + MS_LOG(ERROR) << "arg phase must be string."; + return false; + } + // check the arg valid? + if (py::isinstance(obj)) { + MS_LOG(ERROR) << "Find error: parse obj is None."; + return false; + } +#ifdef ENABLE_GE + GetGeBackendPolicy(); +#endif + ExecutorInfoPtr excutor_info = std::make_shared(); + std::string phase_s = py::cast(phase); + MS_LOG(INFO) << "ExecutorPy compile phase:" << phase_s << "!"; + ResourcePtr resource = std::make_shared(obj); + std::vector p_actions; + + use_vm = ChangeExportGeirUseVmFlag(use_vm, phase_s); + + if (use_vm) { + // Create backend and session + resource->results()[kBackend] = compile::CreateBackend(); + p_actions = VmPipeline(); + } else { + p_actions = GePipeline(); + } + + std::shared_ptr pip = std::make_shared(resource, p_actions); + + // get the parameters items and add the value to args_spec + abstract::AbstractBasePtrList args_spec; + std::size_t size = args.size(); + for (std::size_t i = 0; i < size; i++) { + ValuePtr converted = nullptr; + bool succ = parse::ConvertData(args[i], &converted); + if (!succ) { + MS_LOG(EXCEPTION) << "args convert error"; + } + bool broaden = true; + args_spec.push_back(abstract::FromValue(converted, broaden)); + } + + resource->set_args_spec(args_spec); + excutor_info->arg_list_size = size; + excutor_info->resource = resource; + info_[phase_s] = excutor_info; + pip->Run(); + + // save the run graph func to MsPipeLine + SaveCompiledGraph(phase_s); + + resource->Clean(); + // Reclaim all resource used by optimizer; + ReclaimOptimizer(); + + MS_LOG(INFO) << "End ExecutorPy compile!"; + return true; +} + +void ExecutorPy::ReleaseResource(const py::object& phase) { + ResourcePtr res = GetResource(py::cast(phase)); + if (res != nullptr) { + res->Clean(); + } + // Reclaim all resource used by optimizer; + ReclaimOptimizer(); +} + +static std::string PrintArgs(const py::tuple& args) { + py::print(args); + return ""; +} + +bool ExecutorPy::Compile(const py::object& obj, const py::tuple& args, const py::object& phase, bool use_vm) { + bool ret_value = false; + + try { + MS_LOG(DEBUG) << PrintArgs(args); + ret_value = CompileInner(obj, args, phase, use_vm); + } catch (const py::error_already_set& ex) { + // print function call stack info before release + std::ostringstream oss; + trace::TraceGraphInfer(); + trace::GetInferStackInfo(oss); + // call py::print to output function call stack to STDOUT, in case of output the log to file, the user can see + // these info from screen, no need to open log file to find these info + py::print(oss.str()); + MS_LOG(ERROR) << oss.str(); + ReleaseResource(phase); + + // re-throw this exception to Python interpreter to handle it + throw(py::error_already_set(ex)); + } catch (const py::type_error& ex) { + ReleaseResource(phase); + throw py::type_error(ex); + } catch (const py::value_error& ex) { + ReleaseResource(phase); + throw py::value_error(ex); + } catch (const std::exception& ex) { + ReleaseResource(phase); + // re-throw this exception to Python interpreter to handle it + throw(std::runtime_error(ex.what())); + } catch (...) { + ReleaseResource(phase); + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(EXCEPTION) << "Error occurred when compile graph. Exception name: " << exName; + } + + return ret_value; +} + +void SetGeOption(const std::map& options) { + ConfigManager::GetInstance().set_ge_initialize_options(options); +} + +bool InitDistribute(const std::map& options) { + ConfigManager::GetInstance().set_parallel_strategy(ParallelStrategy::DISTRIBUTION); + MS_LOG(INFO) << "ME run in DISTRIBUTION strategy mode"; + + SetGeOption(options); +#ifdef ENABLE_GE + auto ge_options = ConfigManager::GetInstance().ge_initialize_options(); + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + if (ge::GEInitialize(ge_options) != ge::GRAPH_SUCCESS) { + MS_LOG(ERROR) << "Initialize GE failed!"; + return false; + } + } +#endif + MS_LOG(DEBUG) << "Initialize Ge success"; + return true; +} + +#ifdef ENABLE_LOAD_ANF_IR +// get MindSpore Intermediate Representation File +std::string GetMsIrFile(void) { + std::string file; + const char* path = getenv("MS_IR_FILE"); + if (path == nullptr) { + return file; + } + + char real_path[PATH_MAX] = {0}; + if (realpath(path, real_path) == nullptr) { + MS_LOG(ERROR) << "MS IR Path error, " << path; + return file; + } + file = real_path; + return file; +} + +void RunPipelineAction(const ActionItem& action, pipeline::ResourcePtr resource, bool* result) { + MS_EXCEPTION_IF_NULL(resource); + MS_EXCEPTION_IF_NULL(result); + + std::string ir_file = GetMsIrFile(); + (void)parse::python_adapter::set_python_scoped(); + if (ir_file.empty()) { + *result = action.second(resource); + return; + } + + // when in loading anf ir mode, action `parse` do nothing + if (action.first == "parse") { + parse::PythonAdapter::SetPythonEnvFlag(true); + return; + } + + // load MindSpore IR from file + if (action.first == "symbol_resolve") { + MS_LOG(DEBUG) << "" << action.first << " read ir file: " << ir_file; + std::vector graphs = ImportIR(ir_file); + if (graphs.size() == 0) { + MS_LOG(EXCEPTION) << "" << action.first << " read ir file " << ir_file << " failed as no graph found"; + } + auto manager = resource->manager(); + MS_EXCEPTION_IF_NULL(manager); + for (auto& graph : graphs) { + manager->AddFuncGraph(graph); + } + resource->set_func_graph(graphs[0]); + return; + } + + // do normal action when not in `parse` and `symbol_resolve` stage + *result = action.second(resource); +} +#endif + +void Pipeline::Run() { + MS_LOG(INFO) << "pipeline run"; + MS_EXCEPTION_IF_NULL(resource_); + FuncGraphPtr user_graph = nullptr; + + WITH(MsProfile::GetProfile())[&user_graph, this]() { + int i = 0; + for (auto& action : actions_) { +#ifdef ENABLE_TIMELINE + DumpTime& dump_time = DumpTime::GetInstance(); + dump_time.Record(action.first, GetTime(), true); +#endif + bool result = true; + WITH(MsProfile::GetProfile()->Step(action.first))[&result, &action, this]() { + MS_LOG(DEBUG) << "Action " << action.first << " start ..."; +#ifdef ENABLE_LOAD_ANF_IR + RunPipelineAction(action, resource_, &result); +#else + result = action.second(resource_); +#endif + MS_LOG(DEBUG) << "Action " << action.first << " end."; + }; + if (!result) { + MS_LOG(EXCEPTION) << "pipeline running to end, failed in step:" << action.first; + } + if (MsContext::GetInstance()->save_graphs_flag() && resource_->func_graph() != nullptr) { + auto graph = resource_->func_graph(); + if (graph != nullptr) { + user_graph = graph; + std::string base_name = GetBaseNameForIR(i, action.first); + + // generate IR file in dot format, which can be converted to svg file using graphviz dot command + draw::Draw(base_name + ".dot", graph); + // generate IR file in human readable format + DumpIR(base_name + ".ir", graph); + // generate IR file in a heavily commented format, which can also be reloaded + if (action.first != "parse") { + ExportIR(base_name + ".dat", std::to_string(i), graph); + } + } +#ifdef MS_DEBUG + // Dump graph cnode list + MS_LOG(INFO) << "Show CNode list after " << action.first; + graph->DumpCNodeList(); +#endif + } + if (resource_->func_graph() != nullptr) { + auto func_graph = resource_->func_graph(); + if (func_graph->has_flag(GRAPH_FLAG_HAS_EFFECT)) { + func_graph->EraseUnusedNodeInOrder(); + func_graph->CheckOrder(); + for (auto fg : func_graph->func_graphs_used_total()) { + MS_LOG(DEBUG) << "Check order graph " << fg->ToString() << "."; + fg->EraseUnusedNodeInOrder(); + fg->CheckOrder(); + } + } + } + i++; +#ifdef ENABLE_TIMELINE + dump_time.Record(action.first, GetTime(), false); +#endif + } + }; +#ifdef ENABLE_PROFILE + MsProfile::Print(); + MsProfile::Reset(); +#endif + + if (MsContext::GetInstance()->save_graphs_flag() && (user_graph != nullptr)) { + std::string user_graph_file = GetFilePathName("ModelDigraph.dot"); + MS_LOG(DEBUG) << "save user graph to: " << user_graph_file; + draw::DrawUserFuncGraph(user_graph_file, user_graph); + +#ifdef ENABLE_DUMP_IR + std::string filename = GetFilePathName("ms_output.pb"); + ChangeFileMode(filename, S_IRWXU); + std::ofstream ofs(filename); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file '" << filename << "' failed!"; + return; + } + ofs << GetFuncGraphProtoString(user_graph); + ofs.close(); + // set file mode to read only by user + ChangeFileMode(filename, S_IRUSR); +#endif + } + MS_LOG(INFO) << "end"; +} + +void ExecutorPy::ProcessVmArg(const py::tuple& args, const std::string& phase, VectorRef* arg_list) { + std::size_t size = args.size(); + + for (std::size_t i = 0; i < size; i++) { + py::object arg = args[i]; + auto ms_context = MsContext::GetInstance(); + if (ms_context->backend_policy() == kMsConvert && py::isinstance(arg)) { + MS_LOG(EXCEPTION) << "args[" << i << "] is numpy array, not tensor"; + } + (*arg_list).push_back(arg); + } + + ResourcePtr res = GetResource(phase); + MS_EXCEPTION_IF_NULL(res); + auto graph = res->func_graph(); + MS_EXCEPTION_IF_NULL(graph); + std::vector graph_params = graph->parameters(); + std::size_t graph_params_size = graph_params.size(); + if ((*arg_list).size() != graph_params_size) { + // maybe some default parameter + for (std::size_t i = (*arg_list).size(); i < graph_params_size; i++) { + MS_EXCEPTION_IF_NULL(graph_params[i]); + py::object obj = dyn_cast(graph_params[i])->default_param(); + py::object p_value = py::cast(parse::python_adapter::GetPyObjAttr(obj, "default_input")); + (*arg_list).push_back(p_value); + } + } +} + +py::object ExecutorPy::Run(const py::tuple& args, const py::object& phase) { + std::size_t size = args.size(); + if (!py::isinstance(phase)) { + MS_LOG(EXCEPTION) << "Run failed, phase input is not a str"; + } + auto phase_s = py::cast(phase); + std::string backend = MsContext::GetInstance()->backend_policy(); + if (backend == "ge") { + return ExecDFGraph(args, phase_s); + } + std::size_t full_arg_size = ArgListSize(phase_s); + if (size > full_arg_size) { + MS_LOG(WARNING) << "The arg num : size = " << size << ". full_arg_size = " << full_arg_size; + } + VectorRef arg_list; + ProcessVmArg(args, phase_s, &arg_list); + + compile::VmEvalFuncPtr run = GetVmEvalFunc(phase_s); + if (run == nullptr) { + MS_LOG(EXCEPTION) << "Can't find run graph func for " << phase_s; + } + + MS_LOG(DEBUG) << "eval run"; + BaseRef value = (*run)(arg_list); + MS_LOG(DEBUG) << "run end"; + return BaseRefToPyData(value); +} + +py::object StructureOutput(const AbstractBasePtr& output, const py::tuple& data, size_t* count) { + MS_EXCEPTION_IF_NULL(output); + + if (!output->isa()) { + ValuePtr value = output->BuildValue(); + if (value != kAnyValue) { + return ValuePtrToPyData(value); + } + if (!output->isa()) { + MS_LOG(EXCEPTION) << "Output can only be tensor except for constants, but got " + << output->BuildValue()->ToString() << "."; + } + if (*count >= data.size()) { + MS_LOG(EXCEPTION) << "The number of elements in the outputs : " << data.size() + << " less than the number of elements required. "; + } + auto shape = output->BuildShape(); + auto shape_act = shape->cast()->shape(); + Tensor tensor_exp = py::cast(data[*count]); + if (shape_act != tensor_exp.shape()) { + MS_LOG(EXCEPTION) << "The shape of the tensor returned from GE is not the same as " + "the shape of the tensor derived from ME."; + } + return data[(*count)++]; + } + + auto tuple_output = output->cast(); + AbstractBasePtrList elements = tuple_output->elements(); + size_t size = elements.size(); + py::tuple tp = py::tuple(size); + for (size_t i = 0; i < size; i++) { + tp[i] = StructureOutput(elements[i], data, count); + } + return std::move(tp); +} + +std::shared_ptr DoExecGraph(const FuncGraphPtr& graph, const std::vector& inputs, + const std::string& phase) { + std::vector ge_tensors = TransformUtil::ConvertInputTensors(inputs, kOpFormat_NCHW); + if (ge_tensors.size() != inputs.size()) { + MS_LOG(ERROR) << "args convert to ge tensor error"; + return nullptr; + } + + std::vector ge_outputs; + transform::RunOptions run_options; + + run_options.name = phase; + + auto graph_runner = DfGraphManager::GetInstance().GetGraphRunner(); + + if (graph_runner == nullptr) { + MS_LOG(ERROR) << "Can not found GraphRunner"; + return nullptr; + } + + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + MS_LOG(DEBUG) << "Run graph begin, inputs size is: " << inputs.size(); + Status ret = graph_runner->RunGraph(run_options, ge_tensors, &ge_outputs); + MS_LOG(DEBUG) << "Run graph finish, outputs size is: " << ge_outputs.size(); + if (ret != Status::SUCCESS) { + MS_LOG(ERROR) << "Exec graph failed"; + return nullptr; + } + } + + std::vector me_outputs = TransformUtil::ConvertGeTensors(ge_outputs); + if (me_outputs.size() != ge_outputs.size()) { + MS_LOG(ERROR) << "Convert output Ge tensor to Me tensor failed"; + } + + py::tuple outputs(me_outputs.size()); + for (std::size_t i = 0; i < outputs.size(); i++) { + outputs[i] = *me_outputs[i]; + } + + std::shared_ptr ret = nullptr; + +#ifdef ENABLE_GE + AnfNodePtr root = graph->get_return(); + MS_EXCEPTION_IF_NULL(root); + AbstractBasePtr output = root->abstract(); + size_t count = 0; + py::object oj = StructureOutput(output, outputs, &count); + ret = std::make_shared(oj); +#else + if (outputs.size() == 1) { + ret = std::make_shared(outputs[0]); + } else { + ret = std::make_shared(outputs); + } +#endif + + return ret; +} + +void DoExecNonInputGraph(const std::string& phase) { + std::vector ge_tensors; + std::vector ge_outputs; + transform::RunOptions run_options; + run_options.name = phase; + auto graph_runner = DfGraphManager::GetInstance().GetGraphRunner(); + + if (graph_runner == nullptr) { + MS_LOG(ERROR) << "Can not found GraphRunner"; + return; + } + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + Status ret = graph_runner->RunGraph(run_options, ge_tensors, &ge_outputs); + if (ret != Status::SUCCESS) { + MS_LOG(ERROR) << "Exec graph:" << run_options.name << " failed"; + return; + } + } +} + +void ExecutorPy::ProcessGeArg(const py::tuple& args, const std::string& phase, std::vector* inputs) { + // check the arg and use the ExecutorPy args + std::size_t size = args.size(); + if (size != ArgListSize(phase)) { + MS_LOG(EXCEPTION) << "The real arg num : size = " << size << ". graph_arg_size = " << ArgListSize(phase); + } + + // process the first args of tensor + // only in Dataset Feed Mode, fp_bp graph need input tensors + if (ConfigManager::GetInstance().dataset_mode() == DS_FEED_MODE) { + for (std::size_t i = 0; i < size; i++) { + ValuePtr converted = nullptr; + bool succ = parse::ConvertData(args[i], &converted); + if (!succ) { + MS_LOG(EXCEPTION) << "args convert error"; + } + if (converted->isa()) { + (*inputs).push_back(converted->cast()); + } else { + MS_LOG(EXCEPTION) << "args, " << converted->ToString() << " is not tensor"; + } + } + } +} + +py::object ExecutorPy::ExecDFGraph(const py::tuple& args, const std::string& phase) { + std::string phase_prefix = GetPhasePrefix(phase); + + if (phase_prefix == "save") { + DoExecNonInputGraph(phase); + ConfigManager::GetInstance().ResetConfig(); + return py::none(); + } + + if (info_.count(phase) == 0) { + MS_LOG(EXCEPTION) << "has no phase:" << phase; + } + +#if (!defined ENABLE_GE) || (defined ENABLE_INFER) + // Now don't use the graph because the exec ge function don't take effect + MS_EXCEPTION_IF_NULL(info_[phase]->func_graph); + if (ENABLE_TRAIN != info_[phase]->func_graph->flags()["training"]) { + MS_LOG(ERROR) << "Graph training mode mismatch mode of libraries"; + ConfigManager::GetInstance().ResetConfig(); + return py::none(); + } +#endif + + std::shared_ptr ret_val = std::make_shared(); + if (IsGraphOutputValueNodeOrParameter(info_[phase]->func_graph->output(), args, ret_val)) { + ConfigManager::GetInstance().ResetConfig(); + return *ret_val; + } + + std::vector inputs; + ProcessGeArg(args, phase, &inputs); + + std::shared_ptr ret = DoExecGraph(GetFuncGraph(phase), inputs, phase); + ConfigManager::GetInstance().ResetConfig(); + if (ret != nullptr) { + return *ret; + } else { + MS_LOG(EXCEPTION) << "exec graph failed"; + } +} + +void ExecutorPy::RunInitGraph(const py::dict& init_params, const std::string& phase) { + MS_LOG(DEBUG) << "ExecInitGraph start."; + TensorOrderMap inputs_with_name{}; + ConvertObjectToTensors(init_params, &inputs_with_name); + std::vector inputs; + (void)std::transform(inputs_with_name.begin(), inputs_with_name.end(), std::back_inserter(inputs), + [](const std::pair& item) { return item.second; }); + + std::vector ge_tensors = TransformUtil::ConvertInputTensors(inputs, kOpFormat_NCHW); + if (ge_tensors.size() != inputs.size()) { + MS_LOG(ERROR) << "Args convert to ge tensor error."; + return; + } + MS_LOG(DEBUG) << "Run graph begin, inputs size is: " << inputs.size() << "."; + + std::vector ge_outputs; + transform::RunOptions run_options; + + run_options.name = phase; + if (DfGraphManager::GetInstance().GetGraphByName(phase) == nullptr) { + MS_LOG(WARNING) << "Can not find " << phase << " sub graph, don't need data init subgraph in INFER mode."; + return; + } + auto graph_runner = DfGraphManager::GetInstance().GetGraphRunner(); + if (graph_runner == nullptr) { + MS_LOG(EXCEPTION) << "Can not found GraphRunner."; + } + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + Status ret = graph_runner->RunGraph(run_options, ge_tensors, &ge_outputs); + if (ret != Status::SUCCESS) { + MS_LOG(EXCEPTION) << "Exec " << phase << " graph failed."; + } + + MS_LOG(INFO) << "Exec " << phase << " graph success."; + + if ((ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::DISTRIBUTION) && + (DfGraphManager::GetInstance().GetGraphByName(BROADCAST_GRAPH_NAME) != nullptr)) { + run_options.name = BROADCAST_GRAPH_NAME; + ret = graph_runner->RunGraph(run_options, ge_tensors, &ge_outputs); + if (ret != Status::SUCCESS) { + MS_LOG(EXCEPTION) << "Exec BROADCAST_GRAPH_NAME failed."; + } + MS_LOG(INFO) << "Exec broadcast graph success."; + } + } +} + +Status CreateSessionAndGraphRunner(bool is_training = true) { + std::shared_ptr sess = DfGraphManager::GetInstance().GetGeSession(); + if (sess == nullptr) { + transform::SessionOptions options; + if (is_training) { + options["ge.trainFlag"] = "1"; + options["ge.streamNum"] = "100"; + options["ge.enabledLocalFmkop"] = "1"; + options["ge.hcomParallel"] = "1"; + } else { + options["ge.trainFlag"] = "0"; + } + + options["ge.enablePrintOpPass"] = "0"; + sess = transform::GraphRunner::NewSession(options); + if (sess == nullptr) { + MS_LOG(ERROR) << "Init data graph failed, because of create Ge session failed"; + return Status::FAILED; + } else { + DfGraphManager::GetInstance().SetGeSession(sess); + } + } + + transform::GraphRunnerOptions options; + options.sess_ptr = sess; + auto graph_runner = std::make_shared(options); + if (graph_runner == nullptr) { + MS_LOG(ERROR) << "Create new graph runner failed"; + return Status::FAILED; + } else { + DfGraphManager::GetInstance().SetGraphRunner(graph_runner); + } + + return Status::SUCCESS; +} + +void ExecutorPy::ConvertObjectToTensors(const py::dict& dict, TensorOrderMap* const tensors) { + for (auto item : dict) { + if ((!py::isinstance(item.first))) { + MS_LOG(WARNING) << "Type of key of py_dict is not string, ignore it."; + continue; + } + std::shared_ptr tensor; + std::string name = py::cast(item.first); + if (py::isinstance(item.second.attr("default_input"))) { + // convert float to tensor with shape([1]) + tensor = std::make_shared(kNumberTypeFloat32, std::vector({1})); + *(static_cast(tensor->data_c(true))) = py::cast(item.second.attr("default_input")); + } else if (py::isinstance(item.second.attr("default_input"))) { + // convert int to tensor with shape([1]) + tensor = std::make_shared(kNumberTypeInt32, std::vector({1})); + *(static_cast(tensor->data_c(true))) = py::cast(item.second.attr("default_input")); + } else if (py::hasattr(item.second.attr("default_input"), PYTHON_TENSOR_FLAG)) { + // cast tensor + tensor = py::cast>(item.second.attr("default_input")); + } + + if (tensor == nullptr) { + MS_LOG(EXCEPTION) << "Get default value for " << name << " failed"; + } + (void)tensors->emplace(name, tensor); + } +} + +bool ExecutorPy::AddDFGraph(const py::dict& init_params, const std::string& phase, const py::object& broadcast_params) { + FuncGraphPtr anf_graph = info_[phase]->func_graph; + DfGraphConvertor convertor(anf_graph); + + size_t pos = phase.find('.'); + std::string net_id = ((pos == std::string::npos || pos == phase.size() - 1) ? phase : phase.substr(pos + 1)); + std::string phase_prefix = phase.substr(0, pos); + + if (phase_prefix == "export") { + MS_LOG(INFO) << "Set DfGraphConvertor training : false"; + convertor.set_training(false); + } + + TensorOrderMap init_tensors{}; + ConvertObjectToTensors(init_params, &init_tensors); + (void)convertor.ConvertAllNode().InitParam(init_tensors).BuildGraph(); + + if (broadcast_params != py::none()) { + if (!py::isinstance(broadcast_params)) { + MS_LOG(ERROR) << "Invalid broadcast params, it must be py::dict type"; + return false; + } + py::dict broadcast = broadcast_params.cast(); + if (broadcast.empty()) { + (void)convertor.GenerateBroadcastGraph(init_tensors); + } else { + TensorOrderMap broadcast_tensors{}; + ConvertObjectToTensors(broadcast, &broadcast_tensors); + (void)convertor.GenerateBroadcastGraph(broadcast_tensors); + } + MS_LOG(INFO) << "Generate broadcast graph with params and broadcast_empty is " << broadcast.empty(); + } + + (void)convertor.GenerateCheckpointGraph(); + if (convertor.ErrCode() != 0) { + DfGraphManager::GetInstance().ClearGraph(); + MS_LOG(ERROR) << "convert df graph failed, err:" << convertor.ErrCode(); + return false; + } + + if (MsContext::GetInstance()->save_graphs_flag()) { + convertor.DrawComputeGraph(GetFilePathName("ge_graph.dot")); // for debug + convertor.DrawInitGraph(GetFilePathName("init_graph.dot")); // for debug + convertor.DrawSaveCheckpointGraph(GetFilePathName("save_checkpoint_graph.dot")); // for debug + } + std::string init_graph = "init_subgraph." + net_id; + std::string checkpoint_name = "save." + net_id; + if (phase == "train") { + (void)DfGraphManager::GetInstance().AddGraph(phase, convertor.GetComputeGraph(), {{"ge.exec.variable_acc", "1"}}); + } else { + (void)DfGraphManager::GetInstance().AddGraph(phase, convertor.GetComputeGraph()); + } + (void)DfGraphManager::GetInstance().AddGraph(init_graph, convertor.GetInitGraph()); + (void)DfGraphManager::GetInstance().AddGraph(checkpoint_name, convertor.GetSaveCheckpointGraph()); + (void)DfGraphManager::GetInstance().AddGraph(BROADCAST_GRAPH_NAME, convertor.GetBroadcastGraph()); + + DfGraphManager::GetInstance().SetAnfGraph(checkpoint_name, anf_graph); + + return true; +} + +FuncGraphPtr ExecutorPy::BuildDFGraph(const py::dict& init_params, const std::string& phase, + const py::object& broadcast_params) { + if (info_.count(phase) == 0) { + MS_LOG(EXCEPTION) << "no phase in executor:" << GetPhasePrefix(phase); + } + FuncGraphPtr anf_graph = info_[phase]->func_graph; + + if (MsContext::GetInstance()->save_graphs_flag()) { + draw::Draw(GetFilePathName("anf_graph.dot"), anf_graph); // for debug + DumpIR(GetFilePathName("anf_graph.ir"), anf_graph, true); + } + + if (!AddDFGraph(init_params, phase, broadcast_params)) { + MS_LOG(ERROR) << "GenConvertor failed"; + return nullptr; + } + +#if ENABLE_TRAIN + (void)setenv("GE_TRAIN", "1", 1); +#else + (void)setenv("GE_TRAIN", "0", 1); +#endif + + if (CreateSessionAndGraphRunner(static_cast(ENABLE_TRAIN)) != Status::SUCCESS) { + MS_LOG(ERROR) << "Create GE Session or GraphRunner failed."; + return nullptr; + } + + return anf_graph; +} + +bool InitExecDataset(const std::string& queue_name, int64_t iter_num, int64_t batch_size, + const std::vector& types, const std::vector>& shapes, + const std::vector& input_indexes, const std::string& phase) { + std::string name = MsContext::GetInstance()->backend_policy(); + if (name == kMsConvert || name == kMsVm) { + return InitExecDatasetVm(queue_name, iter_num, batch_size, types, shapes, input_indexes); + } else { + return InitExecDatasetGe(queue_name, iter_num, batch_size, types, shapes, input_indexes, phase); + } +} + +bool InitExecDatasetGe(const std::string& queue_name, int64_t size, int64_t batch_size, + const std::vector& types, const std::vector>& shapes, + const std::vector& input_indexes, const std::string& phase) { + // Convert types to GE types and TF types + std::vector ge_types; + (void)std::transform(types.begin(), types.end(), std::back_inserter(ge_types), [](const TypePtr& i) -> int64_t { + return transform::TransformUtil::ConvertDataType(i->type_id()); + }); + + ConfigManager::GetInstance().set_dataset_mode(DatasetMode::DS_GRAPH_MODE); + ConfigManager::GetInstance().set_iter_num(size); + ConfigManager::GetInstance().set_dataset_phase(phase); + + DatasetGraphParam param(queue_name, size, batch_size, ge_types, shapes, input_indexes); + ConfigManager::GetInstance().set_dataset_param(param); + + if (transform::BuildDatasetGraph(param, phase) != transform::SUCCESS) { + MS_LOG(ERROR) << "Build dateset graph failed."; + return false; + } + +#if ENABLE_TRAIN + (void)setenv("GE_TRAIN", "1", 1); +#else + (void)setenv("GE_TRAIN", "0", 1); +#endif + + if (CreateSessionAndGraphRunner(static_cast(ENABLE_TRAIN)) != Status::SUCCESS) { + MS_LOG(ERROR) << "Create GE Session or GraphRunner failed."; + return false; + } + + MS_LOG(INFO) << "DoExecNonInputGraph:" << phase; + DoExecNonInputGraph(phase); + + return true; +} + +bool InitExecDatasetVm(const std::string& queue_name, int64_t size, int64_t batch_size, + const std::vector& types, const std::vector>& shapes, + const std::vector& input_indexes) { + MS_LOG(INFO) << "Start InitDataSet Entry"; + std::vector int_input_indexes; + (void)std::transform(input_indexes.begin(), input_indexes.end(), std::back_inserter(int_input_indexes), + [](int64_t item) { return static_cast(item); }); + std::vector> int_shapes; + (void)std::transform(shapes.begin(), shapes.end(), std::back_inserter(int_shapes), + [](const std::vector& item) { + std::vector vector_item; + (void)std::transform(item.begin(), item.end(), std::back_inserter(vector_item), + [](int64_t inner_item) { return static_cast(inner_item); }); + return vector_item; + }); + auto p_init = std::make_shared("InitDataSetQueue"); + p_init->set_attr("queue_name", MakeValue(queue_name)); + p_init->set_attr("size", MakeValue(static_cast(size))); + p_init->set_attr("batch_size", MakeValue(static_cast(batch_size))); + p_init->set_attr("types", MakeValue(types)); + p_init->set_attr("shapes", MakeValue(int_shapes)); + p_init->set_attr("input_indexes", MakeValue(int_input_indexes)); + + const std::vector emply_str_list; + p_init->set_attr("input_names", MakeValue(emply_str_list)); + p_init->set_attr("output_names", MakeValue(emply_str_list)); + + FuncGraphPtr func_graph = std::make_shared(); + auto app_init = std::make_shared(AnfNodePtrList{NewValueNode(p_init)}, func_graph); + func_graph->set_output(app_init); + auto manager = MakeManager(); + manager->AddFuncGraph(func_graph); + + // AbstractNone indicates there is no output for this apply node. + auto abstract_none = std::make_shared(); + app_init->set_abstract(abstract_none); + + auto backend = compile::CreateBackend(); + MS_EXCEPTION_IF_NULL(backend); + auto convert_fn = backend->convert_fn(); + MS_EXCEPTION_IF_NULL(convert_fn); + // Convert CNodeList to LinConvertResult. + ConfigManager::GetInstance().set_iter_num(1); + auto runner = convert_fn({app_init}); + backend->Link(runner.graph_id); + ConfigManager::GetInstance().set_iter_num(size); + + if (!(*runner.run)) { + // empty function + MS_LOG(EXCEPTION) << "Backend " << backend->name() << " unsupports tdt dataset."; + } + + // launch init dataset runner without inputs and outputs + VectorRef args; + auto fn = runner.run; + (void)(*fn)(args); + MS_LOG(DEBUG) << "InitDataSetVm End."; + return true; +} + +void InitGe() { + // set python env flag + mindspore::parse::python_adapter::set_python_env_flag(true); + // open tsd before ge initialize + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (!ms_context->OpenTsd()) { + MS_LOG(EXCEPTION) << "open tsd failed"; + } + (void)ms_context->InitGe(); +} + +void FinalizeGe() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + (void)context_ptr->FinalizeGe(); + (void)context_ptr->CloseTsd(); +} + +void ResetOpId() { mindspore::id_generator::reset_id(); } + +void InitHccl() { +#ifdef ENABLE_GE + (void)InitGe(); +#else + mindspore::parse::python_adapter::set_python_env_flag(true); + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + (void)ms_context->OpenTsd(); + uint32_t device_id = ms_context->device_id(); + std::string device_name = ms_context->device_target(); + + if (ms_context->backend_policy() == "ms" && ms_context->device_target() == kAscendDevice) { + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(device_name, device_id); + MS_EXCEPTION_IF_NULL(runtime_instance); + if (!runtime_instance->Init()) { + MS_LOG(ERROR) << "kernel runtime init error."; + return; + } + } +#endif +} + +void FinalizeHccl() { +#ifdef ENABLE_GE + (void)FinalizeGe(); +#else + device::KernelRuntimeManager::Instance().ClearRuntimeResource(); +#endif +} +void ExportDFGraph(const std::string& file_name, const std::string&, const std::string& phase) { + MS_LOG(DEBUG) << "ExportGraph Begin"; + transform::DfGraphWrapperPtr wrap_ptr = DfGraphManager::GetInstance().GetGraphByName(phase); + if (wrap_ptr == nullptr) { + MS_LOG(ERROR) << "Get graph form DfGraphManager failed!"; + return; + } + + transform::DfGraphPtr ge_graph = wrap_ptr->graph_ptr_; + if (nullptr == ge_graph) { + MS_LOG(ERROR) << "The export graph is null"; + return; + } + + (void)ge_graph->SaveToFile(file_name); + + MS_LOG(DEBUG) << "ExportGraph End"; +} + +} // namespace pipeline +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/pipeline.h b/mindspore/ccsrc/pipeline/pipeline.h new file mode 100644 index 00000000000..b0753066820 --- /dev/null +++ b/mindspore/ccsrc/pipeline/pipeline.h @@ -0,0 +1,154 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PIPELINE_PIPELINE_H_ +#define MINDSPORE_CCSRC_PIPELINE_PIPELINE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "debug/draw.h" +#include "ir/anf.h" +#include "ir/meta_tensor.h" +#include "pipeline/action.h" +#include "vm/segment_runner.h" +#include "vm/transform.h" + +namespace mindspore { +extern const char kMsConvert[]; +extern const char kMsVm[]; + +// namespace to support pipeline structures definition +namespace pipeline { + +namespace py = pybind11; + +class Pipeline { + public: + Pipeline(const ResourcePtr& res, const std::vector& actions) : resource_(res), actions_(actions) {} + + ~Pipeline() = default; + + void Run(); + + ResourcePtr resource() { return resource_; } + + private: + ResourcePtr resource_; + std::vector actions_; +}; + +struct ExecutorInfo { + FuncGraphPtr func_graph; + ResourcePtr resource; + std::size_t arg_list_size; +}; + +using ExecutorInfoPtr = std::shared_ptr; + +// A function pipeline. +class ExecutorPy : public std::enable_shared_from_this { + public: + static std::shared_ptr GetInstance() { + std::lock_guard i_lock(instance_lock_); + if (executor_ == nullptr) { + executor_ = std::shared_ptr(new (std::nothrow) ExecutorPy()); + } + return executor_; + } + + ~ExecutorPy(); + + void SaveCompiledGraph(const std::string& phase_s); + bool CompileInner(const py::object& obj, const py::tuple& args, const py::object& phase, bool use_vm); + bool Compile(const py::object& obj, const py::tuple& args, const py::object& phase, bool use_vm); + + // for graph mode + py::object ExecDFGraph(const py::tuple& args, const std::string& phase = "train"); + + void ProcessVmArg(const py::tuple& args, const std::string& phase, VectorRef* arg_list); + void ProcessGeArg(const py::tuple& args, const std::string& phase, std::vector* inputs); + + // for pynative mode when use_vm is on + py::object Run(const py::tuple& args, const py::object& phase); + ResourcePtr GetResource(const std::string& phase); + FuncGraphPtr GetFuncGraph(const std::string& phase); + py::bytes GetFuncGraphProto(const std::string& phase, const std::string& type); + std::size_t ArgListSize(const std::string& phase); + compile::VmEvalFuncPtr GetVmEvalFunc(const std::string& phase); + bool HasCompiled(const std::string& phase) const; + + bool AddDFGraph(const py::dict& init_params, const std::string& phase, const py::object& broadcast_params); + FuncGraphPtr BuildDFGraph(const py::dict& init_params, const std::string& phase, + const py::object& broadcast_params = {}); + void RunInitGraph(const py::dict& init_params, const std::string& phase); + py::dict GetParameterLayout(const std::string& phase); + py::dict GetCNodeStrategy(const std::string& phase); + py::dict GetAllreduceFusion(const std::string& phase); + void DelNetRes(const std::string& id); + void ReleaseResource(const py::object& phase); + static void ClearRes(); + + private: + ExecutorPy(); + void ConvertObjectToTensors(const py::dict& dict, std::map* tensors); + bool ChangeExportGeirUseVmFlag(bool use_vm, const std::string& phase_s) const; + void GetGeBackendPolicy() const; + + std::map info_; + static std::shared_ptr executor_; + static std::mutex instance_lock_; +}; +using ExecutorPyPtr = std::shared_ptr; + +// Generate a key for mapping function graph +py::tuple GenerateKey(const std::string& name, const std::unordered_map& defaults); +py::bool_ VerifyInputSignature(const py::list input_signature, const py::tuple inputs); + +void SetGeOption(const std::map& options); +bool InitDistribute(const std::map& options); + +void ResetOpId(); +void InitGe(); +void FinalizeGe(); +void InitHccl(); +void FinalizeHccl(); + +// init and exec dataset sub graph +bool InitExecDataset(const std::string& queue_name, int64_t iter_num, int64_t batch_size, + const std::vector& types, const std::vector>& shapes, + const std::vector& input_indexes, const std::string& phase); + +// init and exec dataset sub graph for GE backend +bool InitExecDatasetGe(const std::string& queue_name, int64_t size, int64_t batch_size, + const std::vector& types, const std::vector>& shapes, + const std::vector& input_indexes, const std::string& phase); + +// Build and run dataset subgraph for ms backend +bool InitExecDatasetVm(const std::string& queue_name, int64_t size, int64_t batch_size, + const std::vector& types, const std::vector>& shapes, + const std::vector& input_indexes); + +void ExportDFGraph(const std::string& file_name, const std::string&, const std::string& phase); + +} // namespace pipeline +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PIPELINE_PIPELINE_H_ diff --git a/mindspore/ccsrc/pipeline/remove_value_node_dup.cc b/mindspore/ccsrc/pipeline/remove_value_node_dup.cc new file mode 100644 index 00000000000..7937c3e55f3 --- /dev/null +++ b/mindspore/ccsrc/pipeline/remove_value_node_dup.cc @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/remove_value_node_dup.h" +#include "ir/anf.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "optimizer/cse.h" +#include "utils/log_adapter.h" +#include "utils/hashing.h" + +namespace mindspore { +namespace pipeline { +void TryToDoReplace(FuncGraphManager* const manager, const AnfNodePtr& node, HashCache* const hash_cache, + HashValue* const hash_value) { + const auto& to_check_value = GetValueNode(node); + MS_EXCEPTION_IF_NULL(to_check_value); + + // Calculate hash value. + size_t h; + auto hash_iter = hash_value->find(node); + if (hash_iter == hash_value->end()) { + h = hash_combine(to_check_value->hash(), (opt::AbsOf(node)->hash())); + (*hash_value)[node] = h; + } else { + h = hash_iter->second; + } + + auto bucket_iter = hash_cache->find(h); + if (bucket_iter == hash_cache->end()) { + // Meet for the first time, add bucket. + (*hash_cache)[h] = {node}; + return; + } + + auto& bucket = bucket_iter->second; + // Check if need to replace node with value node already met. + for (const auto& v : bucket) { + // Already met and cached. + if (v == node) { + return; + } + const auto& existed_value = GetValueNode(v); + MS_EXCEPTION_IF_NULL(existed_value); + auto equal = [&]() -> bool { + if (existed_value->isa() && to_check_value->isa()) { + return existed_value->cast()->ValueEqual(*(to_check_value->cast())); + } + return *existed_value == *to_check_value; + }; + if (equal()) { + (void)manager->Replace(node, v); + return; + } + } + + // Meet for the first time, append node to bucket. + bucket.emplace_back(node); +} +} // namespace pipeline +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/remove_value_node_dup.h b/mindspore/ccsrc/pipeline/remove_value_node_dup.h new file mode 100644 index 00000000000..8fbb3f27557 --- /dev/null +++ b/mindspore/ccsrc/pipeline/remove_value_node_dup.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PIPELINE_REMOVE_VALUE_NODE_DUP_H_ +#define MINDSPORE_CCSRC_PIPELINE_REMOVE_VALUE_NODE_DUP_H_ + +#include +#include +#include "ir/base.h" +#include "ir/manager.h" + +namespace mindspore { +namespace pipeline { +using HashCache = std::unordered_map>; +using HashValue = std::unordered_map; + +void TryToDoReplace(FuncGraphManager* manager, const AnfNodePtr& node, HashCache* hash_cache, HashValue* hash_value); +} // namespace pipeline +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PIPELINE_REMOVE_VALUE_NODE_DUP_H_ diff --git a/mindspore/ccsrc/pipeline/resource.cc b/mindspore/ccsrc/pipeline/resource.cc new file mode 100644 index 00000000000..2998ff1dbb7 --- /dev/null +++ b/mindspore/ccsrc/pipeline/resource.cc @@ -0,0 +1,282 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/resource.h" +#include "pipeline/pipeline.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "debug/draw.h" +#include "debug/trace.h" +#include "ir/dtype.h" +#include "pipeline/parse/data_converter.h" +#include "operator/ops.h" +#include "utils/graph_utils.h" +#include "transform/convert.h" +#include "optimizer/ad/dfunctor.h" +#include "vm/segment_runner.h" +#include "utils/context/ms_context.h" +#include "transform/df_graph_manager.h" +#include "device/kernel_runtime_manager.h" + +namespace mindspore { +// namespace to support opmap definition +namespace pipeline { + +using MethodMap = std::unordered_map>; + +MethodMap& GetMethodMap() { + static MethodMap method_map = {{kObjectTypeString, + { + {"__bool__", std::string("str_bool")} // C.str_bool + }}, + {kMetaTypeNone, + { + {"__bool__", std::string("none_bool")} // C.none_bool + }}, + {kNumberTypeBool, + { + {"__and__", prim::kPrimBoolAnd}, // P.bool_and + {"__or__", prim::kPrimBoolOr}, // P.bool_or + {"__eq__", prim::kPrimBoolEq}, // P.bool_eq + {"__ne__", std::string("bool_ne")}, // C.bool_ne + {"__bool__", prim::kPrimIdentity} // P.identity + }}, + {kNumberTypeInt, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul + {"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv + {"__truediv__", std::string("int_truediv")}, // C.int_truediv + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow + {"__floor__", prim::kPrimIdentity}, // P.identity + {"__trunc__", prim::kPrimIdentity}, // P.identity + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt + {"__le__", prim::kPrimScalarLe}, // P.scalar_le + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge + {"__bool__", std::string("int_bool")}, // C.int_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array + }}, + {kNumberTypeUInt, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, + {"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div, + {"__truediv__", std::string("int_truediv")}, // C.int_truediv + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, + {"__floor__", prim::kPrimIdentity}, // P.identity, + {"__trunc__", prim::kPrimIdentity}, // P.identity, + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, + {"__le__", prim::kPrimScalarLe}, // P.scalar_le, + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, + {"__bool__", std::string("int_bool")}, // C.int_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, + }}, + {kNumberTypeFloat, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, + {"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv + {"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div, + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, + {"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor, + {"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc, + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, + {"__le__", prim::kPrimScalarLe}, // P.scalar_le, + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, + {"__bool__", std::string("float_bool")}, // C.float_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, + }}, + {kObjectTypeTuple, + { + {"__len__", prim::kPrimTupleLen}, // P.tuple_len, + {"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem, + {"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem, + {"__ms_iter__", prim::kPrimIdentity}, // P.identity, + {"__ms_next__", std::string("tuple_next")}, // C.tuple_next, + {"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext + {"__bool__", std::string("tuple_bool")} // C.tuple_bool + }}, + {kObjectTypeList, + { + {"__len__", prim::kPrimListLen}, // P.list_len, + {"__getitem__", prim::kPrimListGetItem}, // P.list_getitem, + {"__setitem__", prim::kPrimListSetItem}, // P.list_setitem, + {"__ms_iter__", prim::kPrimIdentity}, // P.identity + {"__ms_next__", std::string("list_next")}, // C.list_next + {"append", std::string("list_append")}, // C.list_next + {"__bool__", std::string("list_bool")}, // C.list_bool + {"__ms_hasnext__", std::string("list_hasnext")}, + }}, + {kObjectTypeDictionary, + { + {"__len__", prim::kPrimDictLen}, // P.dict_len + {"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem + {"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem, + {"__bool__", std::string("dict_bool")} // C.dict_bool + }}, + {kObjectTypeTensorType, + { + {"__add__", std::string("add")}, // C.add + {"__sub__", std::string("sub")}, // C.sub + {"__mul__", std::string("mul")}, // C.mul + {"__truediv__", std::string("truediv")}, // C.truediv + {"__floordiv__", std::string("floordiv")}, // C.floordiv + {"__mod__", std::string("mod")}, // C.mod + {"__pow__", std::string("pow_")}, // C.pow + {"__floor__", std::string("array_floor")}, // C.array_floor + {"__trunc__", std::string("array_trunc")}, // C.array_trunc + {"__pos__", std::string("array_uadd")}, // C.array_uadd + {"__neg__", std::string("array_usub")}, // C.array_usub + {"__eq__", std::string("eq")}, // C.eq + {"__ne__", std::string("ne")}, // C.ne + {"__lt__", std::string("lt")}, // C.lt + {"__gt__", std::string("gt")}, // C.gt + {"__le__", std::string("le")}, // C.le + {"__ge__", std::string("ge")}, // C.ge + {"__matmul__", prim::kPrimDot}, // P.dot, + {"__len__", prim::kPrimArrayLen}, // P.array_len, + {"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem, + {"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem, + {"__ms_iter__", std::string("array_iter")}, // C.array_iter + {"__ms_to_array__", prim::kPrimIdentity}, // P.identity, + {"item", prim::kPrimArrayToScalar}, // P.array_to_scalar, + {"transpose", std::string("transpose")}, // P.transpose + {"__bool__", std::string("tensor_bool")}, // C.tensor_bool + }}, + {kObjectTypeJTagged, {}}, + {kObjectTypeSymbolicKeyType, {}}, + {kObjectTypeEnvType, {}}}; + return method_map; +} + +Resource::Resource(const py::object& obj) + : engine_(std::make_shared(abstract::GetPrimEvaluatorConstructors(), manager_)), + input_(obj), + is_cleaned_(false) {} + +Resource::~Resource() { + MS_LOG(DEBUG) << "Resource clear"; + + // If exit normally, these global variables will be cleaned + // in Resource::Clean call by MsPipeline::Compile, but if exit with MS_LOGEXCEPTION, + // these global variables may not being cleaned, it may + // cause segmentfault when free python object inside these global varaibles + // after python interpreter got freed, so these global variables + // are cleaned here. + // So if exit normally, these global variable will be cleaned twice, + // care be taken to prevent double free in the following functions. + if (!is_cleaned_) { + try { + Clean(); + } catch (const std::exception& e) { + MS_LOG(ERROR) << "Exception when cleaning resource. Error info " << e.what(); + } catch (...) { + MS_LOG(ERROR) << "Exception when cleaning resource."; + } + } +} + +bool Resource::IsTypeInMethodMap(const TypeId& type) { + TypeId type_id = NormalizeTypeId(type); + const MethodMap& method_map = GetMethodMap(); + auto iter = method_map.find(static_cast(type_id)); + if (iter != method_map.end()) { + return true; + } + return false; +} + +Any Resource::GetMethodPtr(const TypeId& type, const std::string& name) { + TypeId type_id = NormalizeTypeId(type); + const MethodMap& method_map = GetMethodMap(); + auto iter = method_map.find(static_cast(type_id)); + if (iter == method_map.end()) { + MS_LOG(WARNING) << "Object type: " << type_id << " not in the method_map"; + return Any(); + } + + auto iter_map = iter->second.find(name); + if (iter_map == iter->second.end()) { + MS_LOG(WARNING) << "Object type: " << type_id << " have no method: " << name; + return Any(); + } + return iter_map->second; +} + +void Resource::Clean() { + // AbstractTensor->elements() will be saved in AbstractBasePtrList + args_spec_.clear(); + input_ = py::none(); + // Context with AbstractBasePtrList may be saved in GraphEvaluator + // some Evaluator like ResolveEvaluator may save Python object in cache, + // it should be cleaned before Python Interpreter destructed. + MS_EXCEPTION_IF_NULL(engine_); + engine_->ClearEvaluatorCache(); + // clean static variable to prevent from crash. As static variable is released after + // Python threads is released. + parse::data_converter::ClearObjectCache(); + parse::Parser::CleanParserResource(); + parse::CleanDataClassToClassMap(); + trace::ClearTraceStack(); + is_cleaned_ = true; +} + +void ReleaseGeTsd() { + auto context_ptr = MsContext::GetInstance(); + if (context_ptr != nullptr) { + (void)context_ptr->FinalizeGe(true); + (void)context_ptr->CloseTsd(true); + } +} + +void ClearResAtexit() { + MS_LOG(DEBUG) << "pipeline clear all resource"; + device::KernelRuntimeManager::Instance().ClearRuntimeResource(); + transform::DfGraphManager::GetInstance().ClearGraph(); + ad::g_k_prims.clear(); + + abstract::ClearPrimEvaluatorMap(); + compile::ClearConvertCache(); + transform::DfGraphConvertor::get_adpt_map().clear(); + pipeline::GetMethodMap().clear(); + pipeline::ExecutorPy::ClearRes(); + + ReleaseGeTsd(); +} +} // namespace pipeline +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/resource.h b/mindspore/ccsrc/pipeline/resource.h new file mode 100644 index 00000000000..43159ddbdd3 --- /dev/null +++ b/mindspore/ccsrc/pipeline/resource.h @@ -0,0 +1,119 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PIPELINE_RESOURCE_H_ +#define MINDSPORE_CCSRC_PIPELINE_RESOURCE_H_ + +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include "utils/any.h" +#include "utils/profile.h" +#include "ir/manager.h" +#include "pipeline/static_analysis/prim.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "./common.h" + +namespace mindspore { +namespace pipeline { + +namespace py = pybind11; + +const char kBackend[] = "backend"; +const char kStepParallelGraph[] = "step_parallel"; +const char kOutput[] = "output"; + +class InferenceResource; + +class ResourceBase { + public: + ResourceBase() { manager_ = MakeManager(); } + + virtual ~ResourceBase() = default; + + FuncGraphManagerPtr manager() { return manager_; } + // set a manager defined outside which will not manage the graphs. + void set_manager(const FuncGraphManagerPtr& manager) { manager_ = manager; } + + std::unordered_map& results() { return results_; } + + void SetResult(const std::string& key, const Any& value) { results_[key] = value; } + + Any GetResult(const std::string& key) { + if (results_.count(key) == 0) { + MS_LOG(EXCEPTION) << "this key is not in resource list:" << key; + } + return results_[key]; + } + + bool HasResult(const std::string& key) const { return results_.count(key) != 0; } + + std::unordered_map results_; + + protected: + FuncGraphManagerPtr manager_; +}; + +using ResourceBasePtr = std::shared_ptr; + +class Resource : public ResourceBase { + public: + explicit Resource(const py::object& obj = py::none()); + + ~Resource() override; + + abstract::AnalysisEnginePtr engine() { return engine_; } + + static bool IsTypeInMethodMap(const TypeId& type); + + static Any GetMethodPtr(const TypeId& type, const std::string& name); + + const py::object& input() const { return input_; } + + FuncGraphPtr func_graph() const { return func_graph_; } + void set_func_graph(const FuncGraphPtr& func_graph) { func_graph_ = func_graph; } + + const abstract::AbstractBasePtrList& args_spec() const { return args_spec_; } + void set_args_spec(const abstract::AbstractBasePtrList& args_spec) { args_spec_ = args_spec; } + + // Reclaim resource and clear the cache. + // ExecutorPy::Compile() can be called multiple times, so cache + // should be cleared. + void Clean(); + + private: + abstract::AnalysisEnginePtr engine_; + FuncGraphPtr func_graph_; + abstract::AbstractBasePtrList args_spec_; + py::object input_; + bool is_cleaned_; +}; + +using ResourcePtr = std::shared_ptr; + +void ClearResAtexit(); +void ReleaseGeTsd(); + +} // namespace pipeline +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PIPELINE_RESOURCE_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_function.cc b/mindspore/ccsrc/pipeline/static_analysis/abstract_function.cc new file mode 100644 index 00000000000..98d9b49a791 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_function.cc @@ -0,0 +1,362 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/abstract_function.h" + +#include + +#include "pipeline/static_analysis/analysis_context.h" +#include "pipeline/static_analysis/static_analysis.h" + +namespace mindspore { +namespace abstract { +class Evaluator; +class AnalysisEngine; + +AbstractFunctionPtr AbstractFunction::MakeAbstractFunction(const AbstractFuncAtomPtrList &func_list) { + if (func_list.size() == 1) { + return func_list[0]; + } + return std::make_shared(func_list); +} + +AbstractFunctionPtr AbstractFuncAtom::Join(const AbstractFunctionPtr &other) { + auto this_func = shared_from_base(); + if (other->isa()) { + if (*this_func == *other) { + return this_func; + } + return std::make_shared(this_func, other); + } + auto other_union = dyn_cast(other); + if (other_union->IsSuperSet(this_func)) { + return other; + } + return std::make_shared(this_func, other); +} + +void AbstractFuncAtom::Visit(std::function visit_func) const { + visit_func(const_cast(this)->shared_from_base()); +} + +bool AbstractFuncAtom::operator==(const AbstractFunction &other) const { return this == &other; } + +AbstractFuncUnion::AbstractFuncUnion(const AbstractFuncAtomPtrList &func_list) { func_list_ = func_list; } + +AbstractFuncUnion::AbstractFuncUnion(const AbstractFunctionPtr &first, const AbstractFunctionPtr &second) { + AbstractFuncAtomPtrList new_func_list; + auto build_func_list = [&new_func_list](const AbstractFuncAtomPtr &func) { new_func_list.push_back(func); }; + + first->Visit(build_func_list); + second->Visit(build_func_list); + func_list_ = new_func_list; +} + +std::string AbstractFuncUnion::ToString() const { + std::ostringstream buffer; + buffer << "AbstractFuncUnion({"; + int i = 0; + for (const auto &func : func_list_) { + MS_EXCEPTION_IF_NULL(func); + buffer << "[" << i << "]: " << func->ToString() << ", "; + i++; + } + buffer << "})"; + return buffer.str(); +} + +bool AbstractFuncUnion::IsSuperSet(const AbstractFunctionPtr &other) { + MS_EXCEPTION_IF_NULL(other); + std::vector is_in_list; + auto build_in_list = [this, &is_in_list](const AbstractFuncAtomPtr &func) { + auto iter = find(func_list_.begin(), func_list_.end(), func); + if (iter == func_list_.end()) { + is_in_list.push_back(false); + } + return true; + }; + other->Visit(build_in_list); + return std::all_of(is_in_list.begin(), is_in_list.end(), [](bool is_in) { return is_in; }); +} + +AbstractFunctionPtr AbstractFuncUnion::Join(const AbstractFunctionPtr &other) { + auto this_func = shared_from_base(); + if (other->isa()) { + if (IsSuperSet(other)) { + return this_func; + } + return std::make_shared(this_func, other); + } + auto other_union = dyn_cast(other); + if (other_union->IsSuperSet(this_func)) { + return other; + } + return std::make_shared(this_func, other); +} + +void AbstractFuncUnion::Visit(std::function visit_func) const { + for (AbstractFuncAtomPtr poss : func_list_) { + visit_func(poss); + } +} + +bool AbstractFuncUnion::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_union = static_cast(&other); + if (func_list_.size() != other_union->func_list_.size()) { + return false; + } + if (func_list_ == other_union->func_list_) { + return true; + } + return false; +} + +std::size_t AbstractFuncUnion::hash() const { + std::size_t hash_sum = 0; + for (auto f : func_list_) { + hash_sum = hash_combine(hash_sum, f->hash()); + } + return hash_sum; +} + +EvaluatorPtr PrimitiveAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + return engine->_GetEvaluatorFor(shared_from_base()); +} + +bool PrimitiveAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_prim = static_cast(&other); + if (prim_ == other_prim->prim_ && tracking_id() == other_prim->tracking_id()) { + return true; + } + return false; +} + +std::size_t PrimitiveAbstractClosure::hash() const { return hash_combine(tid(), prim_->hash()); } + +EvaluatorPtr FuncGraphAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + return engine->_GetEvaluatorFor(shared_from_base()); +} + +bool FuncGraphAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_fg = static_cast(&other); + if (func_graph_ == other_fg->func_graph_ && context_ == other_fg->context_) { + return true; + } + return false; +} + +std::size_t FuncGraphAbstractClosure::hash() const { + auto hash_value = hash_combine(tid(), func_graph_->hash()); + hash_value = hash_combine(hash_value, context_->hash()); + return hash_value; +} + +std::string FuncGraphAbstractClosure::ToString() const { + std::stringstream ss; + ss << "FuncGraphAbstractClosure: " << this << "FuncGraph: " << func_graph_.get() << ", " << func_graph_->ToString() + << "; Context: " << context_.get() << context_->ToString(); + return ss.str(); +} + +EvaluatorPtr MetaFuncGraphAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + return engine->_GetEvaluatorFor(shared_from_base()); +} + +bool MetaFuncGraphAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_meta_fg = static_cast(&other); + if (meta_func_graph_ == other_meta_fg->meta_func_graph_) { + return true; + } + return false; +} + +std::size_t MetaFuncGraphAbstractClosure::hash() const { + auto hash_value = hash_combine(tid(), meta_func_graph_->hash()); + return hash_value; +} + +std::string MetaFuncGraphAbstractClosure::ToString() const { + return "MetaFuncGraphAbstractClosure: " + meta_func_graph_->name(); +} + +bool PartialAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_partial = static_cast(&other); + if (fn_ != other_partial->fn_) { + return false; + } + if (args_spec_list_.size() != other_partial->args_spec_list_.size()) { + return false; + } + if (args_spec_list_ == other_partial->args_spec_list_) { + return true; + } + return false; +} + +std::size_t PartialAbstractClosure::hash() const { + auto hash_value = hash_combine(tid(), fn_->hash()); + hash_value = hash_combine(hash_value, AbstractBasePtrListHash(args_spec_list_)); + return hash_value; +} + +EvaluatorPtr PartialAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + return engine->_GetEvaluatorFor(shared_from_base()); +} + +std::string PartialAbstractClosure::ToString() const { + std::ostringstream buffer; + buffer << "PartialAbstractClosure(" << fn_->ToString() << "("; + for (auto arg : args_spec_list_) { + buffer << arg->ToString() << ", "; + } + buffer << "))"; + return buffer.str(); +} + +EvaluatorPtr JTransformedAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + return engine->_GetEvaluatorFor(shared_from_base()); +} + +bool JTransformedAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_transformed = static_cast(&other); + if (fn_ == other_transformed->fn_) { + return true; + } + return false; +} + +std::size_t JTransformedAbstractClosure::hash() const { + auto hash_value = hash_combine(tid(), fn_->hash()); + return hash_value; +} + +EvaluatorPtr VirtualAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + return engine->_GetEvaluatorFor(shared_from_base()); +} + +bool VirtualAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_virtual = static_cast(&other); + if (output_ != other_virtual->output_) { + return false; + } + if (args_spec_list_.size() != other_virtual->args_spec_list_.size()) { + return false; + } + if (args_spec_list_ == other_virtual->args_spec_list_) { + return true; + } + return false; +} + +std::size_t VirtualAbstractClosure::hash() const { + auto hash_value = hash_combine(tid(), output_->hash()); + hash_value = hash_combine(hash_value, AbstractBasePtrListHash(args_spec_list_)); + return hash_value; +} + +std::string VirtualAbstractClosure::ToString() const { + std::ostringstream buffer; + buffer << "VirtualAbstractClosure(args: {"; + int i = 0; + for (const auto &arg : args_spec_list_) { + MS_EXCEPTION_IF_NULL(arg); + buffer << "[" << i << "]: " << arg->ToString() << ", "; + i++; + } + buffer << "}, output: " << output_->ToString() << ")"; + return buffer.str(); +} + +EvaluatorPtr TypedPrimitiveAbstractClosure::GetEvaluator(AnalysisEnginePtr engine) { + MS_EXCEPTION_IF_NULL(engine); + + return engine->_GetEvaluatorFor(shared_from_base()); +} + +bool TypedPrimitiveAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + auto other_typed = static_cast(&other); + if (output_ != other_typed->output_) { + return false; + } + if (prim_ != other_typed->prim_) { + return false; + } + if (args_spec_list_.size() != other_typed->args_spec_list_.size()) { + return false; + } + if (args_spec_list_ == other_typed->args_spec_list_) { + return true; + } + return false; +} + +std::size_t TypedPrimitiveAbstractClosure::hash() const { + auto hash_value = hash_combine(tid(), prim_->hash()); + hash_value = hash_combine(hash_value, AbstractBasePtrListHash(args_spec_list_)); + return hash_value; +} + +std::string TypedPrimitiveAbstractClosure::ToString() const { + std::ostringstream buffer; + buffer << "TypedPrimitiveAbstractClosure: primitive: " << prim_->name() << "(args: {"; + int i = 0; + for (const auto &arg : args_spec_list_) { + MS_EXCEPTION_IF_NULL(arg); + buffer << "[" << i << "]: " << arg->ToString() << ", "; + i++; + } + buffer << "}, output: " << output_->ToString() << ")"; + return buffer.str(); +} + +bool DummyAbstractClosure::operator==(const AbstractFunction &other) const { + if (!other.isa()) { + return false; + } + return true; +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_function.h b/mindspore/ccsrc/pipeline/static_analysis/abstract_function.h new file mode 100644 index 00000000000..3acb22d8296 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_function.h @@ -0,0 +1,295 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_ABSTRACT_FUNCTION_H_ +#define PIPELINE_STATIC_ANALYSIS_ABSTRACT_FUNCTION_H_ + +#include +#include + +#include "pipeline/static_analysis/abstract_value.h" +#include "pipeline/static_analysis/analysis_context.h" +#include "ir/meta_func_graph.h" + +namespace mindspore { +namespace abstract { +class AbstractFuncAtom : public AbstractFunction { + public: + AbstractFuncAtom() = default; + ~AbstractFuncAtom() override = default; + MS_DECLARE_PARENT(AbstractFuncAtom, AbstractFunction) + + AbstractFunctionPtr GetUnique() override { return shared_from_base(); } + EvaluatorPtr GetEvaluator(AnalysisEnginePtr) override { + MS_LOG(EXCEPTION) << "Cannot GetEvaluator from AbstractFuncAtom"; + } + + AbstractFunctionPtr Join(const AbstractFunctionPtr &other) final; + void Visit(std::function) const final; + bool operator==(const AbstractFunction &other) const; + + std::size_t hash() const override { return tid(); } +}; + +class AbstractFuncUnion : public AbstractFunction { + public: + explicit AbstractFuncUnion(const AbstractFuncAtomPtrList &func_list); + AbstractFuncUnion(const AbstractFunctionPtr &first, const AbstractFunctionPtr &second); + ~AbstractFuncUnion() override = default; + MS_DECLARE_PARENT(AbstractFuncUnion, AbstractFunction) + + std::string ToString() const override; + + AbstractFunctionPtr GetUnique() override { MS_LOG(EXCEPTION) << "Cannot get unique from AbstractFuncUnion"; } + EvaluatorPtr GetEvaluator(AnalysisEnginePtr) override { + MS_LOG(EXCEPTION) << "Cannot GetEvaluator from AbstractFuncUnion"; + } + bool IsSuperSet(const AbstractFunctionPtr &other); + AbstractFunctionPtr Join(const AbstractFunctionPtr &other) final; + void Visit(std::function) const final; + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + AbstractFunctionPtr Copy() const override { MS_LOG(EXCEPTION) << "Cannot Copy from AbstractFuncUnion"; } + + private: + AbstractFuncAtomPtrList func_list_; +}; + +class PrimitiveAbstractClosure : public AbstractFuncAtom { + public: + // Represents a Primitive. + // prim: The primitive + // tracking_id: Identifies different uses of the same primitive. + explicit PrimitiveAbstractClosure(const PrimitivePtr &prim, const AnfNodePtr &tracking_id = nullptr) + : prim_(prim), tracking_id_(AnfNodeWeakPtr(tracking_id)) {} + ~PrimitiveAbstractClosure() override = default; + MS_DECLARE_PARENT(PrimitiveAbstractClosure, AbstractFuncAtom) + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + PrimitivePtr prim() { return prim_; } + + AnfNodePtr tracking_id() const override { return tracking_id_.lock(); } + + void set_tracking_id(AnfNodePtr node) override { tracking_id_ = AnfNodeWeakPtr(node); } + + AbstractFunctionPtr Copy() const override { return std::make_shared(prim_, tracking_id()); } + + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override { return "Prim: " + prim_->name(); } + + private: + PrimitivePtr prim_; + // store it as weak_ptr to break reference cycle. + // one reference cycle example is Graph::set_output() input0 local variable. + AnfNodeWeakPtr tracking_id_; +}; + +class FuncGraphAbstractClosure : public AbstractFuncAtom { + public: + // Represents a Graph in a certain Context. + // context: The context, or Context.empty() + FuncGraphAbstractClosure(const FuncGraphPtr &func_graph, const AnalysisContextPtr &context) + : func_graph_(func_graph), context_(context) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(context); + } + ~FuncGraphAbstractClosure() override = default; + MS_DECLARE_PARENT(FuncGraphAbstractClosure, AbstractFuncAtom) + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + FuncGraphPtr func_graph() { return func_graph_; } + + AnalysisContextPtr context() const override { return context_; } + + AbstractFunctionPtr Copy() const override { + return std::make_shared(func_graph_, context_); + } + + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override; + + private: + FuncGraphPtr func_graph_; + AnalysisContextPtr context_; +}; + +class MetaFuncGraphAbstractClosure : public AbstractFuncAtom { + public: + explicit MetaFuncGraphAbstractClosure(const MetaFuncGraphPtr &meta_func_graph, const ScopePtr &scope = kDefaultScope) + : meta_func_graph_(meta_func_graph), scope_(scope) {} + ~MetaFuncGraphAbstractClosure() override = default; + MS_DECLARE_PARENT(MetaFuncGraphAbstractClosure, AbstractFuncAtom) + + MetaFuncGraphPtr meta_func_graph() { return meta_func_graph_; } + + AnalysisContextPtr context() const override { return kDummyAnalysisContext; } + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + ScopePtr GetScope() { return scope_; } + + AbstractFunctionPtr Copy() const override { return std::make_shared(meta_func_graph_); } + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override; + + private: + MetaFuncGraphPtr meta_func_graph_; + ScopePtr scope_; +}; +using MetaFuncGraphAbstractClosurePtr = std::shared_ptr; + +class PartialAbstractClosure : public AbstractFuncAtom { + public: + // Represents a partial application. + // args_spec_list: The first few arguments of that function + PartialAbstractClosure(const AbstractFuncAtomPtr &fn, const AbstractBasePtrList &args_spec_list) + : fn_(fn), args_spec_list_(args_spec_list) {} + ~PartialAbstractClosure() override = default; + MS_DECLARE_PARENT(PartialAbstractClosure, AbstractFuncAtom) + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + AbstractFunctionPtr fn() { return fn_; } + AbstractBasePtrList args() { return args_spec_list_; } + AbstractFunctionPtr Copy() const override { return std::make_shared(fn_, args_spec_list_); } + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override; + + private: + AbstractFuncAtomPtr fn_; + AbstractBasePtrList args_spec_list_; +}; + +class JTransformedAbstractClosure : public AbstractFuncAtom { + public: + // Represents a Function transformed through the application of J. + explicit JTransformedAbstractClosure(const AbstractFuncAtomPtr &fn) : fn_(fn) {} + ~JTransformedAbstractClosure() override = default; + MS_DECLARE_PARENT(JTransformedAbstractClosure, AbstractFuncAtom) + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + AbstractFuncAtomPtr fn() { return fn_; } + AbstractFunctionPtr Copy() const override { return std::make_shared(fn_); } + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override { return "J(" + fn_->ToString() + ")"; } + + private: + AbstractFuncAtomPtr fn_; +}; + +class VirtualAbstractClosure : public AbstractFuncAtom { + public: + // Represents some function with an explicitly fixed type signature. + // args_spec_list: The arguments as abstract value given to the function + // output: The output which is abstract value. + VirtualAbstractClosure(const AbstractBasePtrList &args_spec_list, const AbstractBasePtr &output_spec) + : args_spec_list_(args_spec_list), output_(output_spec) {} + VirtualAbstractClosure(const AbstractBasePtr &args_spec, const AbstractBasePtr &output_spec) + : args_spec_list_({args_spec}), output_(output_spec) {} + ~VirtualAbstractClosure() override = default; + MS_DECLARE_PARENT(VirtualAbstractClosure, AbstractFuncAtom) + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + AbstractBasePtrList args_spec_list() { return args_spec_list_; } + + AbstractBasePtr output() { return output_; } + AbstractFunctionPtr Copy() const override { + return std::make_shared(args_spec_list_, output_); + } + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override; + + private: + AbstractBasePtrList args_spec_list_; + AbstractBasePtr output_; +}; +using VirtualAbstractClosurePtr = std::shared_ptr; + +class TypedPrimitiveAbstractClosure : public AbstractFuncAtom { + public: + // Represents a Primitive with an explicitly fixed type signature. + // args_spec_list: The arguments as abstract value given to the Primitive + // output: The output which is abstract value. + TypedPrimitiveAbstractClosure(const PrimitivePtr prim, const AbstractBasePtrList &args_spec_list, + const AbstractBasePtr &output_spec) + : prim_(prim), args_spec_list_(args_spec_list), output_(output_spec) {} + ~TypedPrimitiveAbstractClosure() override = default; + MS_DECLARE_PARENT(TypedPrimitiveAbstractClosure, AbstractFuncAtom) + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) override; + + PrimitivePtr prim() { return prim_; } + AbstractBasePtrList args_spec_list() { return args_spec_list_; } + AbstractBasePtr output() { return output_; } + AbstractFunctionPtr Copy() const override { + return std::make_shared(prim_, args_spec_list_, output_); + } + bool operator==(const AbstractFunction &other) const override; + std::size_t hash() const override; + + std::string ToString() const override; + + private: + PrimitivePtr prim_; + AbstractBasePtrList args_spec_list_; + AbstractBasePtr output_; +}; + +// Represents a function that can't be called. +class DummyAbstractClosure : public AbstractFuncAtom { + public: + DummyAbstractClosure() = default; + ~DummyAbstractClosure() = default; + MS_DECLARE_PARENT(DummyAbstractClosure, AbstractFuncAtom) + + EvaluatorPtr GetEvaluator(AnalysisEnginePtr) override { MS_LOG(EXCEPTION) << "A dummy function cannot eval."; } + + AbstractFunctionPtr Copy() const override { return std::make_shared(); } + bool operator==(const AbstractFunction &other) const override; + + std::string ToString() const override { return "DummyAbstractClosure()"; } +}; + +struct AbstractFunctionHasher { + std::size_t operator()(const AbstractFunctionPtr &t) const { + std::size_t hash = t->hash(); + return hash; + } +}; + +struct AbstractFunctionEqual { + bool operator()(const AbstractFunctionPtr &lhs, const AbstractFunctionPtr &rhs) const { return *lhs == *rhs; } +}; +} // namespace abstract +} // namespace mindspore +#endif // PIPELINE_STATIC_ANALYSIS_ABSTRACT_FUNCTION_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc new file mode 100644 index 00000000000..eef4e8b4adc --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc @@ -0,0 +1,998 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/abstract_value.h" + +#include + +#include "utils/symbolic.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "pipeline/static_analysis/utils.h" + +namespace mindspore { +namespace abstract { +bool AbstractBase::operator==(const AbstractBase &other) const { + if (tid() != other.tid()) { + return false; + } + if (value_ == nullptr || other.value_ == nullptr) { + MS_LOG(EXCEPTION) << "If value_ is nullptr, AbstractBase::operator== should not be called. this: " + << this->ToString() << ", other: " << other.ToString(); + } + + bool value_equal = *value_ == *other.value_; + bool type_equal = *type_ == *other.type_; + bool shape_equal = *shape_ == *other.shape_; + return value_equal && type_equal && shape_equal; +} + +ValuePtr AbstractBase::BuildValue() const { + if (value_ == nullptr) { + return RealBuildValue(); + } + return value_; +} + +AbstractBasePtr AbstractBase::Broaden() const { + AbstractBasePtr clone = Clone(); + clone->set_value(kAnyValue); + return clone; +} + +std::string AbstractBase::ToString() const { + std::ostringstream buffer; + std::string value = std::string("value is null"); + if (value_ != nullptr) { + value = value_->ToString(); + } + MS_EXCEPTION_IF_NULL(type_); + MS_EXCEPTION_IF_NULL(shape_); + buffer << type_name() << "(" + << "Type: " << type_->ToString() << " Value: " << value << " Shape: " << shape_->ToString() << ")"; + return buffer.str(); +} + +AbstractBasePtr AbstractScalar::Broaden() const { + AbstractBasePtr clone = Clone(); + MS_EXCEPTION_IF_NULL(clone); + auto value_track = clone->GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + if (value_track->isa()) { + return clone; + } + return AbstractBase::Broaden(); +} + +AbstractBasePtr AbstractScalar::Join(const AbstractBasePtr &other) { + MS_EXCEPTION_IF_NULL(other); + if (*this == *other) { + return shared_from_base(); + } + auto value_self = GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_self); + ValuePtr res_value = ValueJoin(value_self, other->GetValueTrack()); + TypePtr res_type = TypeJoin(GetTypeTrack(), other->GetTypeTrack()); + if (res_value == value_self) { + return shared_from_base(); + } + return std::make_shared(res_value, res_type); +} + +AbstractBasePtr AbstractType::Clone() const { + ValuePtr value_self = GetValueTrack(); + if (value_self == nullptr || !value_self->isa()) { + return nullptr; + } + TypePtr type_self = value_self->cast(); + return std::make_shared(type_self->Clone()); +} + +bool AbstractType::operator==(const AbstractBase &other) const { + if (tid() != other.tid()) { + return false; + } + // Have to compare TypePtr with value; + ValuePtr value_self = GetValueTrack(); + ValuePtr value_other = other.GetValueTrack(); + if (value_self == nullptr || value_other == nullptr) { + MS_LOG(EXCEPTION) << "AbstractType value should not be nullptr. this: " << this->ToString() + << ", other: " << other.ToString(); + } + if (!value_self->isa() || !value_other->isa()) { + return false; + } + TypePtr type_self = value_self->cast(); + TypePtr type_other = value_other->cast(); + bool value_equal = *type_self == *type_other; + return value_equal; +} + +std::string AbstractType::ToString() const { + std::ostringstream buffer; + ValuePtr value_self = GetValueTrack(); + if (value_self == nullptr) { + buffer << "AbstractType value: nullptr"; + return buffer.str(); + } + if (!value_self->isa()) { + buffer << type_name() << "(Value: nullptr)"; + return buffer.str(); + } + TypePtr type_self = value_self->cast(); + MS_EXCEPTION_IF_NULL(type_self); + buffer << type_name() << "(" + << "Value: " << type_self->ToString() << ")"; + return buffer.str(); +} + +std::string AbstractError::ToString() const { + std::ostringstream buffer; + auto value_track = GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + buffer << type_name() << "(" + << "Value: " << value_track->ToString() << ", Node: " << node_->DebugString() << ")"; + return buffer.str(); +} + +AbstractBasePtr AbstractFunction::Join(const AbstractBasePtr &other) { + MS_EXCEPTION_IF_NULL(other); + auto other_func = dyn_cast(other); + if (other_func == nullptr) { + MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); + } + return Join(other_func); +} + +bool AbstractFunction::operator==(const AbstractBase &other) const { + if (!other.isa()) { + return false; + } + const auto &other_func = static_cast(other); + bool value_equal = (*this == other_func); + return value_equal; +} + +const AbstractBasePtr AbstractSequeue::operator[](const std::size_t &dim) const { + if (dim >= size()) { + MS_LOG(EXCEPTION) << "Index [" << dim << "] Out of the size [" << size() << "] of the list."; + } + return elements_[dim]; +} + +std::string AbstractSequeue::ToString() const { + std::ostringstream buffer; + int i = 0; + for (const auto &ele : elements_) { + MS_EXCEPTION_IF_NULL(ele); + buffer << "element[" << i << "]: " << ele->ToString() << ","; + i++; + } + return buffer.str(); +} + +TypePtrList AbstractSequeue::ElementsType() const { + TypePtrList element_type_list; + for (const auto &ele : elements_) { + MS_EXCEPTION_IF_NULL(ele); + TypePtr element_type = ele->BuildType(); + element_type_list.push_back(element_type); + } + return element_type_list; +} + +BaseShapePtrList AbstractSequeue::ElementsShape() const { + BaseShapePtrList element_shape_list; + for (const auto &ele : elements_) { + MS_EXCEPTION_IF_NULL(ele); + BaseShapePtr element_shape = ele->BuildShape(); + element_shape_list.push_back(element_shape); + } + return element_shape_list; +} + +AbstractBasePtrList AbstractSequeue::ElementsClone() const { + AbstractBasePtrList ele_list; + for (const auto &ele : elements_) { + MS_EXCEPTION_IF_NULL(ele); + AbstractBasePtr clone = ele->Clone(); + ele_list.push_back(clone); + } + return ele_list; +} + +AbstractBasePtrList AbstractSequeue::ElementsBroaden() const { + AbstractBasePtrList ele_list; + for (const auto &ele : elements_) { + MS_EXCEPTION_IF_NULL(ele); + AbstractBasePtr broadend = ele->Broaden(); + ele_list.push_back(broadend); + } + return ele_list; +} + +template +ValuePtr AbstractSequeue::ElementsBuildValue() const { + std::vector element_value_list; + for (const auto &ele : elements_) { + ValuePtr element_value = ele->BuildValue(); + if (element_value->isa()) { + return kAnyValue; + } + element_value_list.push_back(element_value); + } + return std::make_shared(element_value_list); +} +template ValuePtr AbstractSequeue::ElementsBuildValue() const; +template ValuePtr AbstractSequeue::ElementsBuildValue() const; + +template +AbstractBasePtr AbstractSequeue::ElementsJoin(const AbstractBasePtr &other) { + auto other_sequeue = dyn_cast(other); + if (other_sequeue == nullptr) { + MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); + } + auto joined_list = AbstractJoin(elements_, other_sequeue->elements_); + bool changes = false; + for (std::size_t i = 0; i < elements_.size(); i++) { + if (elements_[i] != joined_list[i]) { + changes = true; + break; + } + } + if (!changes) { + return shared_from_base(); + } + return std::make_shared(joined_list); +} +template AbstractBasePtr AbstractSequeue::ElementsJoin(const AbstractBasePtr &); +template AbstractBasePtr AbstractSequeue::ElementsJoin(const AbstractBasePtr &); + +std::size_t AbstractSequeue::hash() const { + std::size_t hash_sum = hash_combine(tid(), std::hash{}(elements_.size())); + // Hashing all elements is costly, so only take at most 4 elements into account based on + // some experiments. + for (size_t i = 0; (i < elements_.size()) && (i < 4); i++) { + hash_sum = hash_combine(hash_sum, elements_[i]->hash()); + } + return hash_sum; +} + +bool AbstractTuple::operator==(const AbstractTuple &other) const { + if (&other == this) { + return true; + } + + if (elements_.size() != other.elements_.size()) { + return false; + } + for (size_t i = 0; i < elements_.size(); i++) { + if (!(*(elements_[i]) == *(other.elements_[i]))) { + return false; + } + } + return true; +} + +bool AbstractTuple::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + + if (other.isa()) { + auto other_tuple = static_cast(&other); + return *this == *other_tuple; + } + + return false; +} + +bool AbstractList::operator==(const AbstractList &other) const { + if (&other == this) { + return true; + } + + if (elements_.size() != other.elements_.size()) { + return false; + } + for (size_t i = 0; i < elements_.size(); i++) { + if (!(*(elements_[i]) == *(other.elements_[i]))) { + return false; + } + } + return true; +} + +bool AbstractList::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + + if (other.isa()) { + auto other_list = static_cast(&other); + return *this == *other_list; + } + return false; +} + +TypePtr AbstractSlice::BuildType() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + TypePtr start = start_->BuildType(); + TypePtr stop = stop_->BuildType(); + TypePtr step = step_->BuildType(); + return std::make_shared(start, stop, step); +} + +bool AbstractSlice::operator==(const AbstractSlice &other) const { + if (&other == this) { + return true; + } + return (*start_ == *other.start_ && *stop_ == *other.stop_ && *step_ == *other.step_); +} + +bool AbstractSlice::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + if (!other.isa()) { + return false; + } + auto other_slice = static_cast(&other); + return *this == *other_slice; +} + +AbstractBasePtr AbstractSlice::Clone() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + AbstractBasePtr start = start_->Clone(); + AbstractBasePtr stop = stop_->Clone(); + AbstractBasePtr step = step_->Clone(); + return std::make_shared(start, stop, step); +} + +AbstractBasePtr AbstractSlice::Broaden() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + AbstractBasePtr start = start_->Broaden(); + AbstractBasePtr stop = stop_->Broaden(); + AbstractBasePtr step = step_->Broaden(); + return std::make_shared(start, stop, step); +} + +std::string AbstractSlice::ToString() const { + std::ostringstream buffer; + buffer << type_name() << "["; + MS_EXCEPTION_IF_NULL(start_); + buffer << start_->ToString() << " : "; + MS_EXCEPTION_IF_NULL(stop_); + buffer << stop_->ToString() << " : "; + MS_EXCEPTION_IF_NULL(step_); + buffer << step_->ToString(); + buffer << "]"; + return buffer.str(); +} + +ValuePtr AbstractSlice::RealBuildValue() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + ValuePtr start = start_->BuildValue(); + ValuePtr stop = stop_->BuildValue(); + ValuePtr step = step_->BuildValue(); + if (start->isa() || stop->isa() || step->isa()) { + return kAnyValue; + } + return std::make_shared(start, stop, step); +} + +std::size_t AbstractSlice::hash() const { + MS_EXCEPTION_IF_NULL(start_); + MS_EXCEPTION_IF_NULL(stop_); + MS_EXCEPTION_IF_NULL(step_); + return hash_combine({tid(), start_->hash(), stop_->hash(), step_->hash()}); +} + +TypePtr AbstractTensor::BuildType() const { + MS_EXCEPTION_IF_NULL(element_); + TypePtr element_type = element_->BuildType(); + return std::make_shared(element_type); +} + +BaseShapePtr AbstractTensor::BuildShape() const { + auto shape = GetShapeTrack(); + // Guard from using set_shape(nullptr) + if (shape == nullptr) { + return kNoShape; + } + return shape; +} + +AbstractBasePtr AbstractTensor::Join(const AbstractBasePtr &other) { + auto other_tensor = dyn_cast(other); + if (other_tensor == nullptr) { + MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); + } + auto element = element_->Join(other_tensor->element_); + auto shape = ShapeJoin(this->shape(), other_tensor->shape()); + return std::make_shared(element, shape); +} + +bool AbstractTensor::operator==(const AbstractTensor &other) const { + if (&other == this) { + return true; + } + + auto v1 = GetValueTrack(); + auto v2 = other.GetValueTrack(); + if (v1 == nullptr || v2 == nullptr) { + MS_LOG(EXCEPTION) << "the value of AbstractTensor is nullptr"; + } + + bool is_value_equal = (v1 == v2); + if (v1->isa() && v2->isa()) { + is_value_equal = true; + } + return (*element_ == *other.element_) && (*shape() == *other.shape()) && is_value_equal; +} + +bool AbstractTensor::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + + if (other.isa()) { + auto other_tensor = static_cast(&other); + return *this == *other_tensor; + } else { + return false; + } +} + +AbstractBasePtr AbstractTensor::Clone() const { + MS_EXCEPTION_IF_NULL(element_); + auto clone = std::make_shared(element_->Clone()); + ShapePtr shp = shape(); + clone->set_shape(shp->Clone()); + clone->set_value(GetValueTrack()); + return clone; +} + +AbstractBasePtr AbstractTensor::Broaden() const { + MS_EXCEPTION_IF_NULL(element_); + auto broaden = std::make_shared(element_->Broaden()); + auto shp = shape(); + broaden->set_shape(shp->Clone()); + broaden->set_value(kAnyValue); + return broaden; +} + +AbstractBasePtr AbstractTensor::BroadenWithShape() const { + MS_EXCEPTION_IF_NULL(element_); + auto broaden = std::make_shared(element_->Broaden()); + auto shp = shape()->Clone(); + shp->Broaden(); + broaden->set_shape(shp); + broaden->set_value(kAnyValue); + return broaden; +} + +ShapePtr AbstractTensor::shape() const { + auto shp = dyn_cast(GetShapeTrack()); + if (shp == nullptr) { + MS_LOG(EXCEPTION) << "Tensor should have a shape."; + } + return shp; +} + +std::string AbstractTensor::ToString() const { + std::ostringstream buffer; + BaseShapePtr shape_track = GetShapeTrack(); + MS_EXCEPTION_IF_NULL(shape_track); + MS_EXCEPTION_IF_NULL(element_); + auto value_track = GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + buffer << type_name() << "(" + << "shape: " << shape_track->ToString() << ", element: " << element_->ToString() + << ", value_ptr: " << value_track << ", value: " << value_track->ToString() << ")"; + return buffer.str(); +} + +TypePtr AbstractDictionary::BuildType() const { + std::vector> key_values; + for (const auto &item : key_values_) { + MS_EXCEPTION_IF_NULL(item.second); + TypePtr type = item.second->BuildType(); + key_values.emplace_back(item.first, type); + } + return std::make_shared(key_values); +} + +bool AbstractDictionary::operator==(const AbstractDictionary &other) const { + if (key_values_.size() != other.key_values_.size()) { + return false; + } + + for (size_t index = 0; index < key_values_.size(); index++) { + if (key_values_[index].first != other.key_values_[index].first) { + return false; + } + if (!(*key_values_[index].second == *other.key_values_[index].second)) { + return false; + } + } + return true; +} + +bool AbstractDictionary::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + if (other.isa()) { + auto other_class = static_cast(&other); + return *this == *other_class; + } + return false; +} + +AbstractBasePtr AbstractDictionary::Clone() const { + std::vector kv; + (void)std::transform(key_values_.begin(), key_values_.end(), std::back_inserter(kv), + [](const AbstractAttribute &item) { + MS_EXCEPTION_IF_NULL(item.second); + return std::make_pair(item.first, item.second->Clone()); + }); + return std::make_shared(kv); +} + +AbstractBasePtr AbstractDictionary::Broaden() const { + std::vector kv; + (void)std::transform(key_values_.begin(), key_values_.end(), std::back_inserter(kv), + [](const AbstractAttribute &item) { + MS_EXCEPTION_IF_NULL(item.second); + return std::make_pair(item.first, item.second->Broaden()); + }); + return std::make_shared(kv); +} + +std::string AbstractDictionary::ToString() const { + std::ostringstream buffer; + buffer << type_name() << "{ "; + for (const auto &kv : key_values_) { + MS_EXCEPTION_IF_NULL(kv.second); + buffer << "(" << kv.first << ": " << kv.second->ToString() << ") "; + } + buffer << "}"; + return buffer.str(); +} + +std::size_t AbstractDictionary::hash() const { + std::size_t hash_sum = std::accumulate(key_values_.begin(), key_values_.end(), tid(), + [](std::size_t hash_sum, const AbstractAttribute &item) { + hash_sum = hash_combine(hash_sum, std::hash()(item.first)); + MS_EXCEPTION_IF_NULL(item.second); + hash_sum = hash_combine(hash_sum, item.second->hash()); + return hash_sum; + }); + return hash_sum; +} + +ValuePtr AbstractDictionary::RealBuildValue() const { + std::vector> key_values; + for (const auto &item : key_values_) { + MS_EXCEPTION_IF_NULL(item.second); + auto element_value = item.second->BuildValue(); + MS_EXCEPTION_IF_NULL(element_value); + if (element_value->isa()) { + return kAnyValue; + } + key_values.emplace_back(item.first, element_value); + } + return std::make_shared(key_values); +} + +TypePtr AbstractClass::BuildType() const { + ClassAttrVector attributes_type; + for (auto attr : attributes_) { + MS_EXCEPTION_IF_NULL(attr.second); + TypePtr type = attr.second->BuildType(); + std::pair elem(attr.first, type); + attributes_type.push_back(elem); + } + + return std::make_shared(tag_, attributes_type, methods_); +} + +bool AbstractClass::operator==(const AbstractClass &other) const { + if (!(tag_ == other.tag_)) { + return false; + } + if (attributes_.size() != other.attributes_.size()) { + return false; + } + for (size_t i = 0; i < attributes_.size(); i++) { + MS_EXCEPTION_IF_NULL(attributes_[i].second); + MS_EXCEPTION_IF_NULL(other.attributes_[i].second); + if (!(*attributes_[i].second == *other.attributes_[i].second)) { + MS_LOG(DEBUG) << "attr " << attributes_[i].first << " not equal, arg1:" << attributes_[i].second->ToString() + << " arg2:" << other.attributes_[i].second->ToString(); + return false; + } + } + // method compare; + if (methods_.size() != other.methods_.size()) { + return false; + } + for (const auto &iter : methods_) { + auto iter_other = other.methods_.find(iter.first); + if (iter_other == other.methods_.end()) { + return false; + } + if (!(*iter.second == *iter_other->second)) { + return false; + } + } + return true; +} + +bool AbstractClass::operator==(const AbstractBase &other) const { + if (other.isa()) { + auto other_class = static_cast(&other); + return *this == *other_class; + } + return false; +} + +AbstractBasePtr AbstractClass::GetAttribute(const std::string &name) { + auto it = std::find_if(attributes_.begin(), attributes_.end(), + [name](const AbstractAttribute &pair) -> bool { return pair.first == name; }); + if (it != attributes_.end()) { + return it->second; + } + return nullptr; +} + +ValuePtr AbstractClass::GetMethod(const std::string &name) { + auto method_pair = methods_.find(name); + if (method_pair != methods_.end()) { + return method_pair->second; + } + return kAnyValue; +} + +AbstractBasePtr AbstractClass::Clone() const { + std::vector attributes_clone; + for (auto attr : attributes_) { + MS_EXCEPTION_IF_NULL(attr.second); + AbstractBasePtr clone = attr.second->Clone(); + AbstractAttribute elem(attr.first, clone); + attributes_clone.push_back(elem); + } + return std::make_shared(tag_, attributes_clone, methods_); +} + +AbstractBasePtr AbstractClass::Broaden() const { + std::vector attributes_clone; + for (auto attr : attributes_) { + MS_EXCEPTION_IF_NULL(attr.second); + AbstractBasePtr clone = attr.second->Broaden(); + AbstractAttribute elem(attr.first, clone); + attributes_clone.push_back(elem); + } + return std::make_shared(tag_, attributes_clone, methods_); +} + +std::string AbstractClass::ToString() const { + std::ostringstream buffer; + buffer << type_name() << "(tag: " << tag_ << ") attrs:("; + bool append_comma = false; + for (const auto &attr : attributes_) { + if (append_comma) { + buffer << ", "; + } else { + append_comma = true; + } + MS_EXCEPTION_IF_NULL(attr.second); + buffer << attr.first << ":" << attr.second->ToString(); + } + buffer << ") method:("; + append_comma = false; + for (const auto &iter : methods_) { + if (append_comma) { + buffer << ", "; + } else { + append_comma = true; + } + MS_EXCEPTION_IF_NULL(iter.second); + buffer << iter.first << ":" << iter.second->ToString(); + } + buffer << ")"; + return buffer.str(); +} + +std::size_t AbstractClass::hash() const { + std::size_t hash_sum = std::accumulate(attributes_.begin(), attributes_.end(), hash_combine(tid(), tag_.hash()), + [](std::size_t hash_sum, const AbstractAttribute &item) { + MS_EXCEPTION_IF_NULL(item.second); + return hash_combine(hash_sum, item.second->hash()); + }); + + return hash_sum; +} + +ValuePtr AbstractClass::RealBuildValue() const { + auto cls = BuildType()->cast(); + std::unordered_map attributes_value_map; + for (const auto &attr : attributes_) { + MS_EXCEPTION_IF_NULL(attr.second); + ValuePtr _value = attr.second->BuildValue(); + if (_value->isa()) { + return kAnyValue; + } + attributes_value_map[attr.first] = _value; + } + cls->set_value(attributes_value_map); + return cls; +} + +TypePtr AbstractJTagged::BuildType() const { + MS_EXCEPTION_IF_NULL(element_); + TypePtr subtype = element_->BuildType(); + return std::make_shared(subtype); +} + +AbstractBasePtr AbstractJTagged::Join(const AbstractBasePtr &other) { + auto other_jtagged = dyn_cast(other); + if (other_jtagged == nullptr) { + MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); + } + auto joined_elem = element_->Join(other_jtagged->element_); + return std::make_shared(joined_elem); +} + +bool AbstractJTagged::operator==(const AbstractJTagged &other) const { + MS_EXCEPTION_IF_NULL(element_); + MS_EXCEPTION_IF_NULL(other.element_); + return (*element_ == *other.element_); +} + +bool AbstractJTagged::operator==(const AbstractBase &other) const { + if (other.isa()) { + auto other_jtagged = static_cast(&other); + return *this == *other_jtagged; + } + return false; +} + +std::string AbstractJTagged::ToString() const { + std::ostringstream buffer; + MS_EXCEPTION_IF_NULL(element_); + buffer << type_name() << "(" + << "element: " << element_->ToString() << ")"; + return buffer.str(); +} + +TypePtr AbstractRef::BuildType() const { + TypePtr subtype = ref_->BuildType(); + TypePtr subtype_origin = ref_origin_->BuildType(); + return std::make_shared(subtype, subtype_origin); +} + +bool AbstractRef::operator==(const AbstractRef &other) const { + return (*ref_ == *other.ref_) && (*ref_key_ == *other.ref_key_) && (*ref_origin_ == *other.ref_origin_); +} + +bool AbstractRef::operator==(const AbstractBase &other) const { + if (other.isa()) { + auto other_conf = static_cast(&other); + return *this == *other_conf; + } + return false; +} + +std::string AbstractRef::ToString() const { + std::ostringstream buffer; + buffer << type_name() << "(" + << "key: " << ref_key_->ToString() << "ref_value: " << ref_->ToString() + << "origin_value: " << ref_origin_->ToString(); + auto value = GetValueTrack(); + if (value) { + buffer << ", value: " << value->ToString(); + } + buffer << ")"; + return buffer.str(); +} + +bool AbstractNone::operator==(const AbstractNone &) const { return true; } + +bool AbstractNone::operator==(const AbstractBase &other) const { + if (other.isa()) { + auto other_none = static_cast(&other); + return *this == *other_none; + } + return false; +} + +std::string AbstractNone::ToString() const { + std::ostringstream buffer; + buffer << type_name() << "(Value: None)"; + return buffer.str(); +} + +ValuePtr AbstractNone::RealBuildValue() const { return kNone; } + +bool AbstractRefKey::operator==(const AbstractRefKey &other) const { + ValuePtr value_self = GetValueTrack(); + ValuePtr value_other = other.GetValueTrack(); + if (value_self != nullptr && value_other != nullptr) { + if (value_self->isa() && value_other->isa()) { + return true; + } + if (!value_self->isa() || !value_other->isa()) { + return false; + } + RefKeyPtr type_self = value_self->cast(); + RefKeyPtr type_other = value_other->cast(); + return *type_self == *type_other; + } else if (value_self != nullptr || value_other != nullptr) { + return false; + } + return true; +} + +bool AbstractRefKey::operator==(const AbstractBase &other) const { + if (other.isa()) { + auto other_confkey = static_cast(&other); + return *this == *other_confkey; + } else { + return false; + } +} + +std::string AbstractRefKey::ToString() const { + std::ostringstream buffer; + buffer << type_name(); + auto value = GetValueTrack(); + if (value) { + buffer << "(value: " << value->ToString() << ")"; + } + return buffer.str(); +} + +bool AbstractNull::operator==(const AbstractNull &) const { return true; } + +bool AbstractNull::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + if (other.isa()) { + auto other_none = static_cast(&other); + return *this == *other_none; + } else { + return false; + } +} + +std::string AbstractNull::ToString() const { + std::ostringstream buffer; + buffer << type_name() << "(" + << "Value: " + << "Null" + << ")"; + return buffer.str(); +} + +TypePtr AbstractKeywordArg::BuildType() const { + MS_EXCEPTION_IF_NULL(arg_value_); + TypePtr type = arg_value_->BuildType(); + return std::make_shared(arg_name_, type); +} + +AbstractBasePtr AbstractKeywordArg::Clone() const { + MS_EXCEPTION_IF_NULL(arg_value_); + return std::make_shared(arg_name_, arg_value_->Clone()); +} + +AbstractBasePtr AbstractKeywordArg::Broaden() const { + MS_EXCEPTION_IF_NULL(arg_value_); + return std::make_shared(arg_name_, arg_value_->Broaden()); +} + +std::size_t AbstractKeywordArg::hash() const { + MS_EXCEPTION_IF_NULL(arg_value_); + return hash_combine({tid(), std::hash{}(arg_name_), arg_value_->hash()}); +} + +std::string AbstractKeywordArg::ToString() const { + std::ostringstream buffer; + MS_EXCEPTION_IF_NULL(arg_value_); + buffer << type_name() << "("; + buffer << "key : " << arg_name_; + buffer << "value : " << arg_value_->ToString(); + buffer << ")"; + return buffer.str(); +} + +bool AbstractKeywordArg::operator==(const AbstractBase &other) const { + if (&other == this) { + return true; + } + + if (other.isa()) { + auto other_tuple = static_cast(&other); + return *this == *other_tuple; + } + return false; +} + +bool AbstractKeywordArg::operator==(const AbstractKeywordArg &other) const { + if (&other == this) { + return true; + } + MS_EXCEPTION_IF_NULL(arg_value_); + MS_EXCEPTION_IF_NULL(other.arg_value_); + return other.arg_name_ == arg_name_ && *other.arg_value_ == *arg_value_; +} + +ValuePtr AbstractKeywordArg::RealBuildValue() const { + MS_EXCEPTION_IF_NULL(arg_value_); + ValuePtr value = arg_value_->BuildValue(); + MS_EXCEPTION_IF_NULL(value); + if (value->isa()) { + return kAnyValue; + } + return std::make_shared(arg_name_, value); +} + +std::size_t AbstractBasePtrListHash(const AbstractBasePtrList &args_spec_list) { + std::size_t hash_value = 0; + // Hashing all elements is costly, so only take at most 4 elements into account based on + // some experiments. + for (size_t i = 0; (i < args_spec_list.size()) && (i < 4); i++) { + MS_EXCEPTION_IF_NULL(args_spec_list[i]); + hash_value = hash_combine(hash_value, args_spec_list[i]->hash()); + } + return hash_value; +} + +bool AbstractBasePtrListDeepEqual(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + std::size_t size = lhs.size(); + for (std::size_t i = 0; i < size; i++) { + MS_EXCEPTION_IF_NULL(lhs[i]); + MS_EXCEPTION_IF_NULL(rhs[i]); + if (!(*lhs[i] == *rhs[i])) { + return false; + } + } + return true; +} + +std::size_t AbstractBasePtrListHasher::operator()(const AbstractBasePtrList &args_spec_list) const { + return AbstractBasePtrListHash(args_spec_list); +} + +bool AbstractBasePtrListEqual::operator()(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs) const { + return AbstractBasePtrListDeepEqual(lhs, rhs); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h new file mode 100644 index 00000000000..9d9585bba3c --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h @@ -0,0 +1,573 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_ABSTRACT_VALUE_H_ +#define PIPELINE_STATIC_ANALYSIS_ABSTRACT_VALUE_H_ + +#include +#include +#include +#include +#include + +#include "utils/log_adapter.h" +#include "utils/hashing.h" +#include "ir/base.h" +#include "ir/dtype.h" +#include "ir/value.h" +#include "ir/meta_tensor.h" +#include "pipeline/static_analysis/dshape.h" + +namespace mindspore { +namespace abstract { +class AbstractBase; +using AbstractBasePtrList = std::vector; + +// The base class for abstract value. The abstract value is used in inferring +// to express the type, shape, and value of the real value. +class AbstractBase : public Base { + public: + explicit AbstractBase(const ValuePtr &value = nullptr, const TypePtr &type = kAnyType, + const BaseShapePtr &shape = kNoShape) + : value_(value), type_(type), shape_(shape) {} + ~AbstractBase() override = default; + MS_DECLARE_PARENT(AbstractBase, Base) + + std::size_t hash() const override { return tid(); } + std::string ToString() const override; + + virtual bool operator==(const AbstractBase &other) const; + void set_value(const ValuePtr &value) { value_ = value; } + void set_type(const TypePtr &type) { type_ = type; } + void set_shape(const BaseShapePtr &shape) { shape_ = shape; } + void set_value_desc(const std::string &desc) { value_desc_ = desc; } + const std::string &value_desc() const { return value_desc_; } + ValuePtr GetValueTrack() const { return value_; } + TypePtr GetTypeTrack() const { return type_; } + BaseShapePtr GetShapeTrack() const { return shape_; } + + // Try build a real value from an abstract value. If the value cannot be built, + // a default value (AnyValue) is returned. + ValuePtr BuildValue() const; + + virtual TypePtr BuildType() const = 0; + virtual BaseShapePtr BuildShape() const { return kNoShape; } + virtual AbstractBasePtr Clone() const = 0; + virtual AbstractBasePtr Broaden() const; + virtual AbstractBasePtr Join(const AbstractBasePtr &) { return shared_from_base(); } + + friend std::ostream &operator<<(std::ostream &os, const std::shared_ptr &a) { + os << a->ToString(); + return os; + } + + protected: + // default implementation, it can be overrided by subclass; + virtual ValuePtr RealBuildValue() const { return kAnyValue; } + + private: + ValuePtr value_; + TypePtr type_; + BaseShapePtr shape_; + std::string value_desc_; // store initial value description for error report +}; + +class AbstractScalar : public AbstractBase { + public: + AbstractScalar() : AbstractBase(kAnyValue, kAnyType) {} + explicit AbstractScalar(const ValuePtr &value, const TypePtr &type) : AbstractBase(value, type) {} + explicit AbstractScalar(const ValuePtr &value) : AbstractBase(value, value->type()) {} + explicit AbstractScalar(int value) : AbstractBase(MakeValue(value), kInt32) {} + explicit AbstractScalar(float value) : AbstractBase(MakeValue(value), kFloat32) {} + explicit AbstractScalar(double value) : AbstractBase(MakeValue(value), kFloat64) {} + explicit AbstractScalar(bool value) : AbstractBase(MakeValue(value), kBool) {} + explicit AbstractScalar(const std::string &value) : AbstractBase(MakeValue(value), kString) {} + explicit AbstractScalar(const TypePtr &type) : AbstractBase(kAnyValue, type) {} + ~AbstractScalar() override = default; + MS_DECLARE_PARENT(AbstractScalar, AbstractBase) + + std::size_t hash() const override { return hash_combine({tid(), GetValueTrack()->hash(), GetTypeTrack()->hash()}); } + + TypePtr BuildType() const override { return GetTypeTrack(); } + AbstractBasePtr Clone() const override { + return std::make_shared(GetValueTrack(), GetTypeTrack()->Clone()); + } + AbstractBasePtr Broaden() const override; + AbstractBasePtr Join(const AbstractBasePtr &other) override; +}; +using AbstractScalarPtr = std::shared_ptr; + +class AbstractType : public AbstractBase { + public: + explicit AbstractType(const TypePtr &type) : AbstractBase(type, kTypeType) { + if (type == nullptr) { + MS_LOG(EXCEPTION) << "type is nullptr"; + } + } + ~AbstractType() override = default; + MS_DECLARE_PARENT(AbstractType, AbstractBase) + + std::string ToString() const override; + bool operator==(const AbstractBase &other) const override; + + TypePtr BuildType() const override { return std::make_shared(); } + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override { return Clone(); } +}; +using AbstractTypePtr = std::shared_ptr; + +class AbstractError : public AbstractBase { + public: + explicit AbstractError(const StringImmPtr &err, const AnfNodePtr &node) : AbstractBase(err), node_(node) { + if (err == nullptr || node == nullptr) { + MS_LOG(EXCEPTION) << "err or node is nullptr"; + } + } + ~AbstractError() override = default; + MS_DECLARE_PARENT(AbstractError, AbstractBase) + + TypePtr BuildType() const override { return std::make_shared(); } + AbstractBasePtr Broaden() const override { return Clone(); } + + AbstractBasePtr Clone() const override { + return std::make_shared(GetValueTrack()->cast(), node_); + } + + std::string ToString() const override; + + private: + // Origin node been specialized to AbstractError, for debug purpose only. + const AnfNodePtr node_; +}; + +class Evaluator; +using EvaluatorPtr = std::shared_ptr; +class AnalysisEngine; +using AnalysisEnginePtr = std::shared_ptr; + +class AbstractFunction; +using AbstractFunctionPtr = std::shared_ptr; +class AbstractFuncAtom; +using AbstractFuncAtomPtr = std::shared_ptr; +using AbstractFuncAtomPtrList = std::vector; + +class AbstractFunction : public AbstractBase { + public: + AbstractFunction() = default; + ~AbstractFunction() override = default; + MS_DECLARE_PARENT(AbstractFunction, AbstractBase) + + // If there is exactly one possible function, return it. Otherwise, raise an Exception. + // Caller should ensure the uniqueness. + virtual AbstractFunctionPtr GetUnique() = 0; + + TypePtr BuildType() const override { return std::make_shared(); } + AbstractBasePtr Clone() const override { return Copy(); } + // For Function, no need to broaden. + AbstractBasePtr Broaden() const override { + return const_cast(this)->shared_from_base(); + } + virtual AbstractFunctionPtr Copy() const = 0; + + AbstractBasePtr Join(const AbstractBasePtr &other) final; + virtual AbstractFunctionPtr Join(const AbstractFunctionPtr &other) = 0; + + virtual void Visit(std::function) const = 0; + bool operator==(const AbstractBase &other) const final; + virtual bool operator==(const AbstractFunction &other) const = 0; + + static AbstractFunctionPtr MakeAbstractFunction(const AbstractFuncAtomPtrList &func_list); + + virtual EvaluatorPtr GetEvaluator(AnalysisEnginePtr engine) = 0; + virtual AnfNodePtr tracking_id() const { return nullptr; } + virtual void set_tracking_id(AnfNodePtr) {} + virtual AnalysisContextPtr context() const { return nullptr; } +}; +using AbstractFunctionPtrList = std::vector; + +// Represents a key-value pair used in function's parameters. +class AbstractKeywordArg : public AbstractBase { + public: + AbstractKeywordArg(const std::string &key, const AbstractBasePtr &argument) : arg_name_(key), arg_value_(argument) {} + ~AbstractKeywordArg() override = default; + MS_DECLARE_PARENT(AbstractKeywordArg, AbstractBase) + + TypePtr BuildType() const override; + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + std::size_t hash() const override; + + bool operator==(const AbstractKeywordArg &other) const; + bool operator==(const AbstractBase &other) const override; + std::string get_key() const { return arg_name_; } + AbstractBasePtr get_arg() const { return arg_value_; } + + std::string ToString() const override; + + protected: + ValuePtr RealBuildValue() const override; + + private: + std::string arg_name_; + AbstractBasePtr arg_value_; +}; +using AbstractKeywordArgPtr = std::shared_ptr; + +class AbstractTensor : public AbstractBase { + public: + // only element_ and value, shape track are valid member, type track are unknown. + explicit AbstractTensor(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + : AbstractBase(kAnyValue), element_(element) { + if (element == nullptr) { + MS_LOG(EXCEPTION) << "element is nullptr"; + } + if (element->isa()) { + MS_LOG(EXCEPTION) << "element type error"; + } + set_shape(shape); + } + AbstractTensor(const TypePtr &element_type, const std::vector &shape) + : AbstractBase(kAnyValue), element_(std::make_shared(kAnyValue, element_type)) { + if (element_type == nullptr) { + MS_LOG(EXCEPTION) << "element_type is nullptr"; + } + set_shape(std::make_shared(shape)); + } + explicit AbstractTensor(const tensor::TensorPtr &tensor) + : AbstractBase(tensor), element_(std::make_shared(kAnyValue, tensor->Dtype())) { + if (tensor == nullptr) { + MS_LOG(EXCEPTION) << "tensor is nullptr"; + } + set_shape(std::make_shared(tensor->shape())); + } + ~AbstractTensor() override = default; + MS_DECLARE_PARENT(AbstractTensor, AbstractBase) + + TypePtr BuildType() const override; + BaseShapePtr BuildShape() const override; + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + AbstractBasePtr BroadenWithShape() const; + AbstractBasePtr Join(const AbstractBasePtr &other) final; + + bool operator==(const AbstractTensor &other) const; + bool operator==(const AbstractBase &other) const override; + + ShapePtr shape() const; + std::string ToString() const override; + const AbstractBasePtr element() const { return element_; } + std::size_t hash() const override { + auto value = GetValueTrack(); + auto hash_sum = hash_combine(tid(), element_->hash()); + if (value != nullptr) { + auto tensor = value->cast(); + if (tensor != nullptr) { + hash_sum = hash_combine(hash_sum, IntToSize(tensor->DataSize())); + } + } + return hash_sum; + } + + private: + AbstractBasePtr element_; +}; +using AbstractTensorPtr = std::shared_ptr; +using AbstractTensorPtrList = std::vector; + +class AbstractSequeue : public AbstractBase { + public: + explicit AbstractSequeue(const AbstractBasePtrList &elements) : elements_(elements) {} + ~AbstractSequeue() override = default; + MS_DECLARE_PARENT(AbstractSequeue, AbstractBase) + + TypePtrList ElementsType() const; + BaseShapePtrList ElementsShape() const; + AbstractBasePtrList ElementsClone() const; + AbstractBasePtrList ElementsBroaden() const; + + template + ValuePtr ElementsBuildValue() const; + + template + AbstractBasePtr ElementsJoin(const AbstractBasePtr &other); + + std::size_t size() const { return elements_.size(); } + const AbstractBasePtrList &elements() const { return elements_; } + + std::size_t hash() const override; + std::string ToString() const override; + const AbstractBasePtr operator[](const std::size_t &dim) const; + + protected: + AbstractBasePtrList elements_; +}; +using AbstractSequeuePtr = std::shared_ptr; + +class AbstractTuple : public AbstractSequeue { + public: + explicit AbstractTuple(const AbstractBasePtrList &elements) : AbstractSequeue(elements) {} + + ~AbstractTuple() override = default; + MS_DECLARE_PARENT(AbstractTuple, AbstractSequeue) + + TypePtr BuildType() const override { return std::make_shared(ElementsType()); } + + BaseShapePtr BuildShape() const override { return std::make_shared(ElementsShape()); } + + AbstractBasePtr Clone() const override { return std::make_shared(ElementsClone()); } + + AbstractBasePtr Broaden() const override { return std::make_shared(ElementsBroaden()); } + + AbstractBasePtr Join(const AbstractBasePtr &other) override { return ElementsJoin(other); } + + std::string ToString() const override { return type_name() + "(" + AbstractSequeue::ToString() + ")"; } + + bool operator==(const AbstractTuple &other) const; + bool operator==(const AbstractBase &other) const override; + + protected: + ValuePtr RealBuildValue() const override { return ElementsBuildValue(); } +}; +using AbstractTuplePtr = std::shared_ptr; + +class AbstractList : public AbstractSequeue { + public: + explicit AbstractList(const AbstractBasePtrList &elements) : AbstractSequeue(elements) {} + + ~AbstractList() override = default; + MS_DECLARE_PARENT(AbstractList, AbstractSequeue) + + TypePtr BuildType() const override { return std::make_shared(ElementsType()); } + + BaseShapePtr BuildShape() const override { return std::make_shared(ElementsShape()); } + + AbstractBasePtr Clone() const override { return std::make_shared(ElementsClone()); } + + AbstractBasePtr Broaden() const override { return std::make_shared(ElementsBroaden()); } + + AbstractBasePtr Join(const AbstractBasePtr &other) override { return ElementsJoin(other); } + + std::string ToString() const override { return type_name() + "[" + AbstractSequeue::ToString() + "]"; } + + bool operator==(const AbstractList &other) const; + bool operator==(const AbstractBase &other) const override; + + protected: + ValuePtr RealBuildValue() const override { return ElementsBuildValue(); } +}; +using AbstractListPtr = std::shared_ptr; + +class AbstractClass : public AbstractBase { + public: + AbstractClass(const Named &tag, const std::vector &attributes, + const std::unordered_map &methods) + : attributes_(attributes), tag_(tag), methods_(methods) {} + + ~AbstractClass() override = default; + MS_DECLARE_PARENT(AbstractClass, AbstractBase) + + TypePtr BuildType() const override; + bool operator==(const AbstractClass &other) const; + bool operator==(const AbstractBase &other) const override; + const std::vector &attributes() const { return attributes_; } + std::unordered_map methods() { return methods_; } + AbstractBasePtr GetAttribute(const std::string &name); + ValuePtr GetMethod(const std::string &name); + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + std::string ToString() const override; + Named tag() const { return tag_; } + std::size_t hash() const override; + + protected: + ValuePtr RealBuildValue() const override; + + private: + std::vector attributes_; + Named tag_; + std::unordered_map methods_; +}; +using AbstractClassPtr = std::shared_ptr; + +class AbstractDictionary : public AbstractBase { + public: + explicit AbstractDictionary(const std::vector &key_values) : key_values_(key_values) {} + ~AbstractDictionary() override = default; + MS_DECLARE_PARENT(AbstractDictionary, AbstractBase) + + TypePtr BuildType() const override; + bool operator==(const AbstractDictionary &other) const; + bool operator==(const AbstractBase &other) const override; + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + std::string ToString() const override; + std::size_t hash() const override; + std::size_t size() const { return key_values_.size(); } + const std::vector &elements() const { return key_values_; } + + std::vector key_values_; + + protected: + ValuePtr RealBuildValue() const override; +}; +using AbstractDictionaryPtr = std::shared_ptr; + +class AbstractSlice : public AbstractBase { + public: + AbstractSlice(const AbstractBasePtr &start, const AbstractBasePtr &stop, const AbstractBasePtr &step) + : start_(start), stop_(stop), step_(step) {} + ~AbstractSlice() override = default; + MS_DECLARE_PARENT(AbstractSlice, AbstractBase) + + TypePtr BuildType() const override; + bool operator==(const AbstractSlice &other) const; + bool operator==(const AbstractBase &other) const override; + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + std::string ToString() const override; + std::size_t hash() const override; + AbstractBasePtr start() const { return start_; } + AbstractBasePtr stop() const { return stop_; } + AbstractBasePtr step() const { return step_; } + + protected: + ValuePtr RealBuildValue() const override; + + private: + AbstractBasePtr start_; + AbstractBasePtr stop_; + AbstractBasePtr step_; +}; +using AbstractSlicePtr = std::shared_ptr; + +class AbstractJTagged : public AbstractBase { + public: + explicit AbstractJTagged(const AbstractBasePtr &element) : element_(element) {} + + ~AbstractJTagged() override = default; + MS_DECLARE_PARENT(AbstractJTagged, AbstractBase) + + TypePtr BuildType() const override; + AbstractBasePtr Clone() const override { return std::make_shared(element_->Clone()); } + AbstractBasePtr Broaden() const override { return std::make_shared(element_->Broaden()); } + AbstractBasePtr Join(const AbstractBasePtr &other) override; + + bool operator==(const AbstractJTagged &other) const; + bool operator==(const AbstractBase &other) const override; + std::string ToString() const override; + AbstractBasePtr element() { return element_; } + std::size_t hash() const override { return hash_combine(tid(), element_->hash()); } + + private: + AbstractBasePtr element_; +}; +using AbstractJTaggedPtr = std::shared_ptr; + +class AbstractNone : public AbstractBase { + public: + AbstractNone() : AbstractBase() { set_type(std::make_shared()); } + ~AbstractNone() override = default; + MS_DECLARE_PARENT(AbstractNone, AbstractBase) + + TypePtr BuildType() const override { return std::make_shared(); } + bool operator==(const AbstractNone &other) const; + bool operator==(const AbstractBase &other) const override; + AbstractBasePtr Clone() const override { return std::make_shared(); } + std::string ToString() const override; + + protected: + ValuePtr RealBuildValue() const override; +}; +using AbstractNonePtr = std::shared_ptr; + +// the un assgined state value for variable, which means the variable is not assigned +class AbstractNull : public AbstractBase { + public: + AbstractNull() : AbstractBase(kNullObj) { set_type(std::make_shared()); } + ~AbstractNull() override = default; + MS_DECLARE_PARENT(AbstractNull, AbstractBase) + + TypePtr BuildType() const override { return std::make_shared(); } + bool operator==(const AbstractNull &other) const; + bool operator==(const AbstractBase &other) const override; + AbstractBasePtr Clone() const override { return std::make_shared(); } + std::string ToString() const override; +}; +using AbstractNullPtr = std::shared_ptr; + +class AbstractRefKey : public AbstractBase { + public: + AbstractRefKey() : AbstractBase() { set_type(std::make_shared()); } + ~AbstractRefKey() override = default; + MS_DECLARE_PARENT(AbstractRefKey, AbstractBase) + + TypePtr BuildType() const override { return std::make_shared(); } + bool operator==(const AbstractRefKey &other) const; + bool operator==(const AbstractBase &other) const override; + AbstractBasePtr Clone() const override { return std::make_shared(); } + std::string ToString() const override; +}; +using AbstractRefKeyPtr = std::shared_ptr; + +class AbstractRef : public AbstractBase { + public: + AbstractRef(const AbstractBasePtr &ref_key, const AbstractBasePtr &ref_value, const AbstractBasePtr &ref_origin) + : ref_key_(ref_key), ref_(ref_value), ref_origin_(ref_origin) { + set_type(std::make_shared()); + } + + ~AbstractRef() override = default; + MS_DECLARE_PARENT(AbstractRef, AbstractBase) + + TypePtr BuildType() const override; + bool operator==(const AbstractRef &other) const; + bool operator==(const AbstractBase &other) const override; + AbstractBasePtr Clone() const override { + return std::make_shared(ref_key_->Clone(), ref_->Clone(), ref_origin_->Clone()); + } + std::string ToString() const override; + AbstractBasePtr ref() { return ref_; } + AbstractBasePtr ref_origin() { return ref_origin_; } + AbstractBasePtr ref_key() { return ref_key_; } + AbstractBasePtr Broaden() const override { + return std::make_shared(ref_key_->Broaden(), ref_->Broaden(), ref_origin_->Broaden()); + } + std::size_t hash() const override { + return ref_key_->hash() ^ ref_->hash() ^ ref_origin_->hash() ^ (std::hash{}(this->tid()) << 1); + } + + private: + AbstractBasePtr ref_key_; + AbstractBasePtr ref_; + AbstractBasePtr ref_origin_; +}; +using AbstractRefPtr = std::shared_ptr; + +struct AbstractBasePtrListHasher { + std::size_t operator()(const AbstractBasePtrList &args_spec_list) const; +}; + +struct AbstractBasePtrListEqual { + bool operator()(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs) const; +}; + +std::size_t AbstractBasePtrListHash(const AbstractBasePtrList &args_spec_list); +bool AbstractBasePtrListDeepEqual(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs); +} // namespace abstract +} // namespace mindspore +#endif // PIPELINE_STATIC_ANALYSIS_ABSTRACT_VALUE_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/analysis_context.cc b/mindspore/ccsrc/pipeline/static_analysis/analysis_context.cc new file mode 100644 index 00000000000..9326ded2d54 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/analysis_context.cc @@ -0,0 +1,192 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/analysis_context.h" + +#include + +#include "utils/symbolic.h" +#include "debug/trace.h" + +namespace mindspore { +namespace abstract { +AnalysisContextPtr AnalysisContext::NewFuncGraphContext(const FuncGraphPtr &func_graph, + const AbstractBasePtrList &args_spec_list) { + FuncGraphPtr graph_parent = func_graph->parent(); + auto iter = parent_cache_.find(graph_parent); + AnalysisContextPtr parent_context = nullptr; + if (iter != parent_cache_.end()) { + parent_context = iter->second.lock(); + } + // if this happen, it will be bug in code. but we raise exception to keep the scene. + if (parent_context == nullptr) { + std::ostringstream oss; + oss << "BUG: cannot found parent_context in current context: " << this->ToString() + << ", func_graph: " << func_graph->ToString() << ", graph_parent: "; + if (graph_parent != nullptr) { + oss << graph_parent->ToString(); + } else { + oss << "nullptr"; + } + MS_LOG(EXCEPTION) << "" << oss.str() << " NodeInfo: " << trace::GetDebugInfo(func_graph->debug_info()); + } + return NewContext(parent_context, func_graph, args_spec_list); +} + +AnalysisContextPtr AnalysisContext::Filter(const FuncGraphPtr &func_graph) { + auto p_iter = parent_cache_.find(func_graph); + AnalysisContextPtr parent_context = nullptr; + if (p_iter != parent_cache_.end()) { + parent_context = p_iter->second.lock(); + } else { + auto iter_parent = parent_cache_.find(func_graph->parent()); + if (iter_parent != parent_cache_.end()) { + parent_context = iter_parent->second.lock(); + } + } + // if this happen, it will be bug in code. but we raise exception to keep the scene. + if (parent_context == nullptr) { + std::ostringstream oss; + oss << "BUG: Filter graph failed: " << func_graph->ToString() << ", graph_parent: "; + if (func_graph->parent() != nullptr) { + oss << func_graph->parent()->ToString(); + } else { + oss << "nullptr"; + } + oss << " parent_cache_: {"; + for (auto iter : parent_cache_) { + if (iter.first == nullptr) { + oss << " [graph: nullptr"; + } else { + oss << " [graph: " << iter.first->ToString(); + } + // iter.second cannot be nullptr even iter.first is nullptr as it will + // always be a Context() object. + oss << ", context: " << iter.second.lock()->ToString() << "]"; + } + oss << "}"; + MS_LOG(EXCEPTION) << "" << oss.str() << " NodeInfo: " << trace::GetDebugInfo(func_graph->debug_info()); + } + return parent_context; +} + +AnalysisContextPtr AnalysisContext::DummyContext() { + AnalysisContextPtr dummy_context = std::make_shared(nullptr, nullptr, AbstractBasePtrList()); + dummy_context->parent_cache_[nullptr] = std::weak_ptr(dummy_context); + return dummy_context; +} + +const AnalysisContextPtr kDummyAnalysisContext = + std::make_shared(nullptr, nullptr, AbstractBasePtrList()); + +bool AnalysisContext::operator==(const AnalysisContext &other) const { + if (func_graph_ != other.func_graph_) { + return false; + } + + if (args_spec_list_.size() != other.args_spec_list_.size()) { + return false; + } + + if (((parent_ == nullptr) && (other.parent_ != nullptr)) || ((parent_ != nullptr) && (other.parent_ == nullptr))) { + return false; + } + // Compare parent with content. + bool is_parent_equal = false; + if (parent_ == other.parent_) { + is_parent_equal = true; + } else if (*parent_ == *other.parent_) { + is_parent_equal = true; + } else { + return false; + } + for (std::size_t i = 0; i < args_spec_list_.size(); i++) { + if (!(*args_spec_list_[i] == *other.args_spec_list_[i])) { + return false; + } + } + return is_parent_equal; +} + +// brief The key which controls the graph cloning in Specialize. +// +// Originally, specialize use context directly as the key for cloning graph. The graph will be cloned multiple times +// for different context, which means the graph is called from different node with different arguments and different +// free values. In order to decrease the number of cloned graphs, we add this `SpecializeKey` method to control what +// graph can be reused. +// The graph called with different SymbolicKey will be reused. The abstract of SymbolicKey parameter will be joined +// and stored in the intermediate_abstract. The joined SymbolicKey would cause Poly Code in infer, thus the reused +// graph with SymbolicKey parameter should be inlined in `opt` pipeline before the next renormalize. +// The graph called with different shape should not be reused, because the combination of `shape` and `Fill` relies +// on correct shape to specialize a tensor constant. +AnalysisContextPtr AnalysisContext::SpecializeKey() const { + AbstractBasePtrList args_broad_shp; + (void)std::transform(args_spec_list_.begin(), args_spec_list_.end(), std::back_inserter(args_broad_shp), + [](const AbstractBasePtr &arg) -> AbstractBasePtr { + if (arg->isa()) { + auto val = arg->GetValueTrack(); + if (val->isa()) { + auto scalar_spec = dyn_cast(arg); + auto ret_spec = scalar_spec->Broaden(); + ret_spec->set_value(kAnyValue); + return ret_spec; + } + } + if (arg->isa()) { + MS_LOG(DEBUG) << "refkey broaden"; + auto arg_spec = dyn_cast(arg); + auto ret_spec = arg_spec->Broaden(); + return ret_spec; + } + return arg; + }); + AnalysisContextPtr context_new = std::make_shared(nullptr, func_graph_, args_broad_shp); + context_new->parent_ = parent_; + return context_new; +} + +std::size_t AnalysisContext::hash() { + std::size_t hash_value = 0; + // hash() recursion exit condition. + if (parent_ != nullptr) { + hash_value = hash_combine(hash_value, parent_->hash()); + } + if (func_graph_ != nullptr) { + hash_value = hash_combine(hash_value, func_graph_->hash()); + } + return hash_value; +} + +std::string AnalysisContext::ToString() const { + std::ostringstream buffer; + buffer << "{"; + if (func_graph_ != nullptr) { + buffer << "Func Graph: " << func_graph_->ToString(); + } + buffer << " Args: "; + int i = 0; + for (const auto &arg : args_spec_list_) { + buffer << "[" << i << "]: " << arg->ToString() << ", "; + i++; + } + if (parent_ != nullptr) { + buffer << "Parent: " << parent_->ToString(); + } + buffer << "}"; + return buffer.str(); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/analysis_context.h b/mindspore/ccsrc/pipeline/static_analysis/analysis_context.h new file mode 100644 index 00000000000..0fb043674ca --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/analysis_context.h @@ -0,0 +1,86 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_ANALYSIS_CONTEXT_H_ +#define PIPELINE_STATIC_ANALYSIS_ANALYSIS_CONTEXT_H_ + +#include +#include +#include + +#include "pipeline/static_analysis/abstract_value.h" +#include "ir/meta_func_graph.h" + +namespace mindspore { +namespace abstract { +// AnalysisContext will be stored in Config in AnalysisCache. +class AnalysisContext { + public: + AnalysisContext(const AnalysisContextPtr &parent, const FuncGraphPtr &fg, const AbstractBasePtrList &args_spec_list) + : parent_(parent), func_graph_(fg), args_spec_list_(args_spec_list) { + if (parent_ != nullptr) { + parent_cache_ = parent_->parent_cache_; + } + } + + ~AnalysisContext() = default; + + // Helper function to wrapper constructor to save shared_ptr in parent_cache. + AnalysisContextPtr NewContext(AnalysisContextPtr parent, FuncGraphPtr fg, const AbstractBasePtrList &args_spec_list) { + AnalysisContextPtr context_new = std::make_shared(parent, fg, args_spec_list); + // Reference to myself, so use weak_ptr to break reference cycle. + context_new->parent_cache_[fg] = std::weak_ptr(context_new); + return context_new; + } + + // Extend this context with values for another graph. + AnalysisContextPtr NewFuncGraphContext(const FuncGraphPtr &func_graph, const AbstractBasePtrList &args_spec_list); + + // Return a context restricted to a graph's dependencies. + AnalysisContextPtr Filter(const FuncGraphPtr &graph); + bool operator==(const AnalysisContext &other) const; + std::size_t hash(); + static AnalysisContextPtr DummyContext(); + FuncGraphPtr func_graph() const { return func_graph_; } + AnalysisContextPtr parent() const { return parent_; } + std::string ToString() const; + AnalysisContextPtr SpecializeKey() const; + AbstractBasePtrList args_spec_list() { return args_spec_list_; } + + private: + AnalysisContextPtr parent_; + FuncGraphPtr func_graph_; + AbstractBasePtrList args_spec_list_; + std::unordered_map> parent_cache_; +}; + +struct ContextHasher { + std::size_t operator()(const AnalysisContextPtr &t) const { + std::size_t hash = t->hash(); + return hash; + } +}; + +struct ContextEqual { + bool operator()(const AnalysisContextPtr &lhs, const AnalysisContextPtr &rhs) const { return *lhs == *rhs; } +}; + +extern const AnalysisContextPtr kDummyAnalysisContext; +} // namespace abstract +} // namespace mindspore +#endif // PIPELINE_STATIC_ANALYSIS_ANALYSIS_CONTEXT_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/dshape.cc b/mindspore/ccsrc/pipeline/static_analysis/dshape.cc new file mode 100644 index 00000000000..15aa71ba1e4 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/dshape.cc @@ -0,0 +1,134 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/dshape.h" + +#include +#include + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace abstract { +// used for print BaseShape content +std::ostream& operator<<(std::ostream& os, const BaseShape& bs) { + os << bs.ToString(); + return os; +} + +std::ostream& operator<<(std::ostream& os, const std::shared_ptr bs) { + MS_EXCEPTION_IF_NULL(bs); + os << bs->ToString(); + return os; +} + +bool BaseShape::operator==(const BaseShape& other) const { + if (tid() != other.tid()) { + return false; + } + return true; +} + +bool BaseShape::operator!=(const BaseShape& other) const { return !(*this == other); } + +std::string Shape::ToString() const { + std::ostringstream buffer; + bool f_begin = true; + buffer << "("; + for (auto& x : shape_) { + if (!f_begin) { + buffer << ", "; + } else { + f_begin = false; + } + buffer << x; + } + buffer << ")"; + return buffer.str(); +} + +std::string Shape::DumpText() const { + std::ostringstream buffer; + buffer << "["; + for (size_t i = 0; i < shape_.size(); i++) { + buffer << (i > 0 ? ", " : "") << shape_[i]; + } + buffer << "]"; + return buffer.str(); +} + +bool Shape::operator==(const BaseShape& other) const { + if (tid() != other.tid()) { + return false; + } + return shape_ == static_cast(other).shape_; +} + +const int Shape::SHP_ANY; +void Shape::Broaden() { + for (size_t i = 0; i < shape_.size(); i++) { + shape_[i] = SHP_ANY; + } +} + +std::string SequeueShape::ToString() const { + std::ostringstream buffer; + bool f_begin = true; + for (auto p_shp : p_shapes_) { + if (!f_begin) { + buffer << ", "; + } else { + f_begin = false; + } + MS_EXCEPTION_IF_NULL(p_shp); + buffer << p_shp->ToString(); + } + return buffer.str(); +} + +BaseShapePtrList SequeueShape::ElementsClone() const { + BaseShapePtrList ele_list; + for (auto p_shp : p_shapes_) { + MS_EXCEPTION_IF_NULL(p_shp); + ele_list.push_back(p_shp->Clone()); + } + return ele_list; +} + +template +bool SequeueShape::SequeueEqual(const BaseShape& other) const { + if (tid() != other.tid()) { + return false; + } + auto other_shapes = static_cast(other).p_shapes_; + if (other_shapes.size() != p_shapes_.size()) { + return false; + } + for (unsigned int i = 0; i < p_shapes_.size(); ++i) { + if (!(*p_shapes_[i] == *other_shapes[i])) { + return false; + } + } + return true; +} +template bool SequeueShape::SequeueEqual(const BaseShape&) const; +template bool SequeueShape::SequeueEqual(const BaseShape&) const; + +const std::shared_ptr kNoShape = std::make_shared(); +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/dshape.h b/mindspore/ccsrc/pipeline/static_analysis/dshape.h new file mode 100644 index 00000000000..6debe061c88 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/dshape.h @@ -0,0 +1,135 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_DSHAPE_H_ +#define PIPELINE_STATIC_ANALYSIS_DSHAPE_H_ + +#include +#include +#include +#include +#include +#include + +#include "utils/log_adapter.h" +#include "ir/base.h" + +namespace mindspore { +namespace abstract { +class BaseShape; +using BaseShapePtr = std::shared_ptr; +using BaseShapePtrList = std::vector; + +class BaseShape : public Base { + public: + BaseShape() = default; + ~BaseShape() override = default; + + MS_DECLARE_PARENT(BaseShape, Base) + virtual bool operator==(const BaseShape& other) const; + bool operator!=(const BaseShape& other) const; + std::size_t hash() const override { return tid(); } + + // return a deep copy + virtual BaseShapePtr Clone() const = 0; + virtual void Broaden() {} +}; + +class NoShape : public BaseShape { + public: + MS_DECLARE_PARENT(NoShape, BaseShape) + BaseShapePtr Clone() const override { return std::make_shared(); } + std::string ToString() const override { return type_name(); } +}; +extern const std::shared_ptr kNoShape; + +class Shape : public BaseShape { + public: + static const int SHP_ANY = -1; + Shape() : shape_() {} + Shape(const std::initializer_list& list) : shape_(list) {} + explicit Shape(const std::vector& list) : shape_(list) {} + ~Shape() override = default; + MS_DECLARE_PARENT(Shape, BaseShape) + std::string ToString() const override; + std::string DumpText() const override; + bool operator==(const BaseShape& other) const override; + BaseShapePtr Clone() const override { return std::make_shared(shape_); } + void Broaden() override; + std::vector& shape() { return shape_; } + + std::vector shape_; // use SHP_ANY to implement the any shape in python +}; +using ShapePtr = std::shared_ptr; +using ShapePtrList = std::vector; + +class SequeueShape : public BaseShape { + public: + SequeueShape() : p_shapes_() {} + explicit SequeueShape(const BaseShapePtrList& shapes) : p_shapes_(shapes) {} + ~SequeueShape() override = default; + MS_DECLARE_PARENT(SequeueShape, BaseShape) + + std::string ToString() const override; + BaseShapePtrList ElementsClone() const; + + template + bool SequeueEqual(const BaseShape& other) const; + + const BaseShapePtrList& shape() const { return p_shapes_; } + size_t size() const { return p_shapes_.size(); } + const BaseShapePtr operator[](std::size_t dim) const { return p_shapes_[dim]; } + + protected: + BaseShapePtrList p_shapes_; // shape list of each elements +}; +using SequeueShapePtr = std::shared_ptr; + +class TupleShape : public SequeueShape { + public: + TupleShape() : SequeueShape() {} + explicit TupleShape(const BaseShapePtrList& shapes) : SequeueShape(shapes) {} + ~TupleShape() override = default; + MS_DECLARE_PARENT(TupleShape, SequeueShape) + + std::string ToString() const override { return type_name() + "(" + SequeueShape::ToString() + ")"; } + + BaseShapePtr Clone() const override { return std::make_shared(ElementsClone()); } + + bool operator==(const BaseShape& other) const override { return SequeueEqual(other); } +}; +using TupleShapePtr = std::shared_ptr; + +class ListShape : public SequeueShape { + public: + ListShape() : SequeueShape() {} + explicit ListShape(const BaseShapePtrList& shapes) : SequeueShape(shapes) {} + ~ListShape() override = default; + MS_DECLARE_PARENT(ListShape, SequeueShape) + + std::string ToString() const override { return type_name() + "[" + SequeueShape::ToString() + "]"; } + + BaseShapePtr Clone() const override { return std::make_shared(SequeueShape::ElementsClone()); } + + bool operator==(const BaseShape& other) const override { return SequeueEqual(other); } +}; +using ListShapePtr = std::shared_ptr; +} // namespace abstract +} // namespace mindspore + +#endif // PIPELINE_STATIC_ANALYSIS_DSHAPE_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc b/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc new file mode 100644 index 00000000000..251f218145d --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc @@ -0,0 +1,346 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/evaluator.h" + +#include + +#include "ir/func_graph_cloner.h" +#include "pipeline/static_analysis/utils.h" +#include "debug/trace.h" + +namespace mindspore { +namespace abstract { +namespace { +void InferEntryLogging(const EvaluatorPtr &evaluator, const AbstractBasePtrList &arg_spec_list, + const AnfNodeConfigPtr &out_conf) { + MS_EXCEPTION_IF_NULL(evaluator); + if (out_conf != nullptr) { + MS_LOG(DEBUG) << "Evaluator " << evaluator->ToString() << " run for " << out_conf->node()->scope()->name(); + } + for (size_t i = 0; i < arg_spec_list.size(); i++) { + MS_LOG(DEBUG) << "" << evaluator->ToString() << " input[" << i + << "] abstract value: " << arg_spec_list[i]->ToString(); + } +} + +void InferFailLogging(const EvaluatorPtr &evaluator, const AbstractBasePtrList &, const AnfNodeConfigPtr &out_conf) { + MS_EXCEPTION_IF_NULL(evaluator); + if (out_conf != nullptr) { + auto node = out_conf->node(); + if (IsValueNode(node)) { + MS_LOG(ERROR) << "Evaluator " << evaluator->ToString() << " run failed for node " << node->fullname_with_scope() + << ", with debug info: " << trace::GetDebugInfo(node->debug_info()); + } else { + MS_LOG(ERROR) << "Evaluator " << evaluator->ToString() << " run failed for node " << node->DebugString() + << ", with debug info: " << trace::GetDebugInfo(node->debug_info()); + } + } +} +} // namespace + +AnalysisContextPtr BaseFuncGraphEvaluator::MakeContext(const AnalysisEnginePtr &engine, + const AbstractBasePtrList &args_spec_list) { + AbstractBasePtrList normalized_args_spec_list = NormalizeArgs(args_spec_list); + FuncGraphPtr fg = GetFuncGraph(engine, normalized_args_spec_list); + MS_EXCEPTION_IF_NULL(parent_context_); + AnalysisContextPtr context = parent_context_->NewFuncGraphContext(fg, normalized_args_spec_list); + return context; +} + +AbstractBasePtr BaseFuncGraphEvaluator::Infer(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) { + FuncGraphPtr fg = GetFuncGraph(engine, args_spec_list); + MS_EXCEPTION_IF_NULL(fg); + std::size_t nargs = fg->parameters().size(); + if (args_spec_list.size() != nargs) { + MS_LOG(EXCEPTION) << "Function " << fg->ToString() << ", The number of parameters of this function is " + << fg->parameters().size() + << "," + " but the number of provided arguments is " + << args_spec_list.size() << ". NodeInfo: " << trace::GetDebugInfo(fg->debug_info()); + } + MS_EXCEPTION_IF_NULL(parent_context_); + MS_EXCEPTION_IF_NULL(engine); + graph_context_ = parent_context_->NewFuncGraphContext(fg, args_spec_list); + const auto ¶meters = fg->parameters(); + for (size_t i = 0; i < nargs; i++) { + const auto &arg = args_spec_list[i]; + const auto &node = parameters[i]; + AnfNodeConfigPtr conf = engine->MakeConfig(node, graph_context_); + engine->cache().set_value(conf, arg); + } + const AnfNodePtr &func_node = fg->get_return(); + + MS_LOG(DEBUG) << "Analysis FuncGraph begin, func graph: " << fg->ToString() + << ", context: " << graph_context_->ToString() << ", return node: " << func_node->DebugString(); + const std::vector &all_nodes = TopoSort(func_node); + for (const auto &node : all_nodes) { + AnfNodeConfigPtr node_conf = engine->MakeConfig(node, graph_context_); + MS_LOG(DEBUG) << "Analysis node begin, func graph: " << fg->ToString() << ", node_conf: " << node_conf->ToString(); + AbstractBasePtr base = engine->GetEvaluatedValue(node_conf); + MS_LOG(DEBUG) << "Analysis node end, func graph: " << fg->ToString() << ", node_conf: " << node_conf->ToString() + << ", abstract: " << base->ToString(); + } + + AnfNodeConfigPtr ret_conf = engine->MakeConfig(func_node, graph_context_); + AbstractBasePtr base = engine->GetEvaluatedValue(ret_conf); + MS_EXCEPTION_IF_NULL(base); + MS_LOG(DEBUG) << "BaseFuncGraph " << fg->ToString() << " infer end, inferred abstract: " << base->ToString(); + return base; +} + +AbstractBasePtrList FuncGraphEvaluator::NormalizeArgs(const AbstractBasePtrList &args_spec_list) const { + MS_EXCEPTION_IF_NULL(func_graph_); + if (func_graph_->has_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES)) { + AbstractBasePtrList broaded_list; + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(broaded_list), + [](const AbstractBasePtr &arg) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(arg); + return arg->Broaden(); + }); + MS_LOG(DEBUG) << "" << func_graph_->ToString() << " original: " << mindspore::ToString(args_spec_list) + << ", broaded: " << mindspore::ToString(broaded_list); + return broaded_list; + } + + if (func_graph_->has_flag(kFuncGraphFlagUndetermin)) { + if (parent_context_) { + MS_LOG(DEBUG) << "Undeterminate FuncGraphEvaluator " << ToString() + << ", context: " << parent_context_->ToString(); + auto last_context = parent_context_->Filter(func_graph_); + if (last_context && last_context->func_graph() == func_graph_) { + MS_LOG(DEBUG) << "Find last infer context: " << last_context->ToString(); + MS_LOG(DEBUG) << "Current eval args: " << ::mindspore::ToString(args_spec_list); + MS_LOG(DEBUG) << "Last eval args: " << ::mindspore::ToString(last_context->args_spec_list()); + // Join the last eval arguments and current arguments to check if there are loop variant. + auto joined_args_spec_list = AbstractJoin(args_spec_list, last_context->args_spec_list()); + MS_LOG(DEBUG) << "Joined args: " << ::mindspore::ToString(joined_args_spec_list); + // If there is loop variant, all arguments need to be broaden to avoid wrong constant propagation. + if (!(joined_args_spec_list == args_spec_list)) { + func_graph_->set_flags(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + } + return joined_args_spec_list; + } + } + } + return args_spec_list; +} + +FuncGraphPtr FuncGraphEvaluator::GetFuncGraph(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) { + auto iter = func_graph_cache_.find(args_spec_list); + FuncGraphPtr ret = nullptr; + if (iter == func_graph_cache_.end()) { + auto fg = func_graph(); + MS_EXCEPTION_IF_NULL(fg); + TraceManager::DebugTrace(std::make_shared(fg->debug_info())); + FuncGraphPtr generated_graph = fg->GenerateGraph(args_spec_list); + TraceManager::EndTrace(); + func_graph_cache_[args_spec_list] = generated_graph; + MS_EXCEPTION_IF_NULL(engine); + engine->func_graph_manager()->AddFuncGraph(generated_graph); + ret = generated_graph; + } else { + ret = iter->second; + } + + // For the top graph, if it is replaced by generated graph, update the top graph to the new one. + if (parse::Parser::GetTopFuncGraph() == func_graph()) { + if (ret != func_graph()) { + parse::Parser::UpdateTopFuncGraph(ret); + } + } + return ret; +} + +FuncGraphPtr MetaFuncGraphEvaluator::GetFuncGraph(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) { + auto iter = func_graph_cache_.find(args_spec_list); + if (iter != func_graph_cache_.end()) { + return iter->second; + } + + MS_EXCEPTION_IF_NULL(meta_func_graph_); + FuncGraphPtr generated_func_graph = nullptr; + if (this->bound_node() != nullptr) { + TraceManager::DebugTrace(std::make_shared(bound_node()->debug_info())); + generated_func_graph = meta_func_graph_->GenerateFuncGraph(args_spec_list); + TraceManager::EndTrace(); + } else { + generated_func_graph = meta_func_graph_->GenerateFuncGraph(args_spec_list); + } + + FuncGraphPtr cloned_func_graph = BasicClone(generated_func_graph); + func_graph_cache_[args_spec_list] = cloned_func_graph; + MS_EXCEPTION_IF_NULL(engine); + engine->func_graph_manager()->AddFuncGraph(cloned_func_graph); + return cloned_func_graph; +} + +AbstractBasePtr Evaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) { + const std::string &evaluator_name = ToString(); + + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + args_spec_list = NormalizeArgs(args_spec_list); + trace::TraceGraphInferEnter(shared_from_base(), out_conf); + InferEntryLogging(shared_from_base(), args_spec_list, out_conf); + MS_EXCEPTION_IF_NULL(cache_); + auto iter = cache_->find(args_spec_list); + if (iter == cache_->end()) { + MS_LOG(DEBUG) << "" << evaluator_name << " cache miss, call Infer()."; + AbstractBasePtr ret = Infer(engine, args_spec_list); + if (ret == nullptr) { + InferFailLogging(shared_from_base(), args_spec_list, out_conf); + MS_LOG(EXCEPTION) << "Evaluator " << evaluator_name << " result is nullptr."; + } + MS_EXCEPTION_IF_NULL(ret); + MS_LOG(DEBUG) << "" << evaluator_name << " set cache. return: " << ret->ToString() << "."; + (*cache_)[args_spec_list] = ret; + trace::TraceGraphInferLeave(shared_from_base()); + return ret; + } else { + MS_EXCEPTION_IF_NULL(iter->second); + MS_LOG(DEBUG) << "" << evaluator_name << " cache hit. return: " << iter->second->ToString() << "."; + trace::TraceGraphInferLeave(shared_from_base()); + return iter->second; + } +} + +AbstractBasePtr TrivialPrimEvaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr) { + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + AbstractBasePtr ret = EvalPrim(engine, args_spec_list); + (*cache_)[args_spec_list] = ret; + return ret; +} + +AbstractBasePtr TransitionPrimEvaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) { + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + if (args_conf_list.size() == 0) { + MS_LOG(EXCEPTION) << "Size should greater than 0"; + } + AbstractBasePtr ret = EvalPrim(engine, args_spec_list, args_conf_list[0], out_conf); + // No need to cache. + return ret; +} + +AbstractBasePtr SymbolicPrimEvaluator::Run(AnalysisEnginePtr, const ConfigPtrList &args_conf_list, AnfNodeConfigPtr) { + AbstractBasePtr ret = EvalPrim(args_conf_list); + return ret; +} + +AbstractBasePtr TrackedEvaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) { + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + AbstractBasePtr ret = sub_evaluator_->Run(engine, args_conf_list, out_conf); + // Don't lookup from cache, as different out_conf with same node but different context + // may add different entry to anfnode_config_map_, like getattr primitive. + (*cache_)[args_spec_list] = ret; + return ret; +} + +AbstractBasePtr PartialAppEvaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) { + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + MS_EXCEPTION_IF_NULL(cache_); + auto iter = cache_->find(args_spec_list); + if (iter != cache_->end()) { + return iter->second; + } + + ConfigPtrList partial_args_conf_list; + // Join arguments in partial and the rest arguments from args_conf_list. + (void)std::transform(args_spec_list_.begin(), args_spec_list_.end(), std::back_inserter(partial_args_conf_list), + [](const AbstractBasePtr &arg) -> ConfigPtr { return std::make_shared(arg); }); + + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(partial_args_conf_list), + [](const AbstractBasePtr &arg) -> ConfigPtr { return std::make_shared(arg); }); + AbstractBasePtr ret = evaluator_->Run(engine, partial_args_conf_list, out_conf); + (*cache_)[args_spec_list] = ret; + return ret; +} + +AbstractBasePtr JEvaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, AnfNodeConfigPtr) { + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + MS_EXCEPTION_IF_NULL(cache_); + auto iter = cache_->find(args_spec_list); + if (iter != cache_->end()) { + return iter->second; + } + + // Call the original evaluator, get the result: y = f(x) + AbstractBasePtr result = evaluator_->Run(engine, args_conf_list, nullptr); + // Build a virtual function: bprop_f which use sense of y as input, return sense of function free variable and input + // parameters. (sense_f, sense_x, ...)(*bpro_f) (sense_y) + AbstractBasePtrList bparams; + bparams.push_back(SensitivityTransform(orig_func_)); + (void)std::transform( + args_spec_list.begin(), args_spec_list.end(), std::back_inserter(bparams), + [](const AbstractBasePtr &arg_spec) -> AbstractBasePtr { return SensitivityTransform(arg_spec); }); + AbstractBasePtr bparams_final = std::make_shared(bparams); + AbstractFunctionPtr bprop = std::make_shared(SensitivityTransform(result), bparams_final); + + // J(f)(J(x)) return a tuple (y, bprop_f) + AbstractBasePtrList jargs = {result, bprop}; + AbstractBasePtr jtuple = std::make_shared(jargs); + (*cache_)[args_spec_list] = jtuple; + return jtuple; +} + +AbstractBasePtr VirtualEvaluator::Infer(AnalysisEnginePtr, const AbstractBasePtrList &args_spec_list) { + if (args_spec_list.size() != args_spec_list_.size()) { + MS_LOG(EXCEPTION) << "Arguments mismatch, parameters no: " << args_spec_list_.size() + << ", arguments no: " << args_spec_list.size(); + } + // Check each parameter and argument match; + for (std::size_t i = 0; i < args_spec_list.size(); i++) { + MS_EXCEPTION_IF_NULL(args_spec_list[i]); + (void)args_spec_list[i]->Join(args_spec_list_[i]); + } + return output_; +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/evaluator.h b/mindspore/ccsrc/pipeline/static_analysis/evaluator.h new file mode 100644 index 00000000000..c745bca1e93 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/evaluator.h @@ -0,0 +1,295 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_EVALUATOR_H_ +#define PIPELINE_STATIC_ANALYSIS_EVALUATOR_H_ + +#include +#include +#include +#include + +#include "pipeline/static_analysis/static_analysis.h" + +namespace mindspore { +namespace abstract { +using EvaluatorCacheMap = + std::unordered_map; +using EvaluatorCacheMapPtr = std::shared_ptr; + +class Evaluator : public Base { + public: + explicit Evaluator(const std::string &id) : cache_(std::make_shared()), identifier_(id) {} + ~Evaluator() override = default; + MS_DECLARE_PARENT(Evaluator, Base); + + // difference between Run() and Infer(): + // Run() will be called with ConfigPtrList, but Infer() will be called with AbstractBasePtr. + // Run() will modify cache_ member, so it cannot marked as const; + virtual AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, AnfNodeConfigPtr out_conf); + + virtual AbstractBasePtr Infer(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) = 0; + + virtual AbstractBasePtrList NormalizeArgs(const AbstractBasePtrList &args_spec_list) const { return args_spec_list; } + + std::string ToString() const override { return identifier_; } + + virtual AnfNodePtr bound_node() const { return bound_node_.lock(); } + + virtual void set_bound_node(const AnfNodePtr &node) { bound_node_ = AnfNodeWeakPtr(node); } + + EvaluatorCacheMapPtr &cache() { return cache_; } + + EvaluatorCacheMapPtr cache_; + + std::string identifier_; + + AnfNodeWeakPtr bound_node_; +}; + +class PrimEvaluator : public Evaluator { + public: + explicit PrimEvaluator(const std::string &id) : Evaluator(id) {} + ~PrimEvaluator() override = default; + MS_DECLARE_PARENT(PrimEvaluator, Evaluator); + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) final { + MS_LOG(EXCEPTION) << "Infer() should not be called, Run() method should be called"; + } +}; + +class TrivialPrimEvaluator : public PrimEvaluator { + public: + explicit TrivialPrimEvaluator(const std::string &id) : PrimEvaluator(id) {} + ~TrivialPrimEvaluator() override = default; + MS_DECLARE_PARENT(TrivialPrimEvaluator, PrimEvaluator); + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, AnfNodeConfigPtr out_conf) final; + virtual AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list) = 0; +}; + +class TransitionPrimEvaluator : public PrimEvaluator { + public: + explicit TransitionPrimEvaluator(const std::string &id) : PrimEvaluator(id) {} + ~TransitionPrimEvaluator() override = default; + MS_DECLARE_PARENT(TransitionPrimEvaluator, PrimEvaluator); + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, AnfNodeConfigPtr out_conf) final; + // Parameter in_conf0 : the first element in args_conf_list; + virtual AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, + const ConfigPtr &in_conf0, const AnfNodeConfigPtr &out_conf) = 0; +}; + +class SymbolicPrimEvaluator : public PrimEvaluator { + public: + explicit SymbolicPrimEvaluator(const std::string &id) : PrimEvaluator(id) {} + ~SymbolicPrimEvaluator() override = default; + MS_DECLARE_PARENT(SymbolicPrimEvaluator, PrimEvaluator); + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, AnfNodeConfigPtr out_conf) final; + virtual AbstractBasePtr EvalPrim(const ConfigPtrList &args_conf_list) = 0; +}; + +// Evaluator will be stored in AnalysisEngine.constructors_ +using EvaluatorPtrList = std::vector; + +class DummyEvaluator : public Evaluator { + public: + DummyEvaluator() : Evaluator("dummy") {} + ~DummyEvaluator() override = default; + MS_DECLARE_PARENT(DummyEvaluator, Evaluator); + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) override { return nullptr; } +}; + +// Wrap another evaluator to track a subset of uses. +// A TrackedEvaluator has its own cache that maps possible calls to +// their results, but is ultimately backed by a different evaluator. +// Multiple TrackedEvaluators can be backed by the same Evaluator. +class TrackedEvaluator : public Evaluator { + public: + explicit TrackedEvaluator(const EvaluatorPtr &subinf) : Evaluator("TrackedEvaluator"), sub_evaluator_(subinf) {} + ~TrackedEvaluator() override = default; + MS_DECLARE_PARENT(TrackedEvaluator, Evaluator); + AnfNodePtr bound_node() const override { + if (sub_evaluator_ != nullptr) { + return sub_evaluator_->bound_node(); + } + return bound_node_.lock(); + } + + void set_bound_node(const AnfNodePtr &node) override { + if (sub_evaluator_ != nullptr) { + sub_evaluator_->set_bound_node(node); + } + bound_node_ = AnfNodeWeakPtr(node); + } + + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) override { + MS_LOG(EXCEPTION) << "Infer() should not be called, Run() method should be called"; + } + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) override; + std::string ToString() const override { return identifier_ + "_" + sub_evaluator_->ToString(); } + + private: + EvaluatorPtr sub_evaluator_; +}; + +class BaseFuncGraphEvaluator : public Evaluator { + public: + explicit BaseFuncGraphEvaluator(const AnalysisContextPtr &context) + : Evaluator("basegraph"), parent_context_(context) {} + + ~BaseFuncGraphEvaluator() override = default; + MS_DECLARE_PARENT(BaseFuncGraphEvaluator, Evaluator); + + AbstractBasePtr Infer(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) override; + + virtual FuncGraphPtr GetFuncGraph(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) = 0; + + AnalysisContextPtr MakeContext(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list); + AnalysisContextPtr graph_context() const { return graph_context_; } + + protected: + AnalysisContextPtr parent_context_; + + private: + AnalysisContextPtr graph_context_; +}; + +class FuncGraphEvaluator : public BaseFuncGraphEvaluator { + public: + FuncGraphEvaluator(const FuncGraphPtr &func_graph, const AnalysisContextPtr &context) + : BaseFuncGraphEvaluator(context->Filter(func_graph)), func_graph_(func_graph) {} + + ~FuncGraphEvaluator() override = default; + MS_DECLARE_PARENT(FuncGraphEvaluator, BaseFuncGraphEvaluator); + + FuncGraphPtr GetFuncGraph(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) override; + + FuncGraphPtr func_graph() { return func_graph_; } + + AbstractBasePtrList NormalizeArgs(const AbstractBasePtrList &args_spec_list) const override; + std::string ToString() const override { return identifier_ + "_" + func_graph_->ToString(); } + + private: + FuncGraphPtr func_graph_; + std::unordered_map + func_graph_cache_; +}; +using FuncGraphEvaluatorPtr = std::shared_ptr; + +class MetaFuncGraphEvaluator : public BaseFuncGraphEvaluator { + public: + // Note: context parameter is not used; + MetaFuncGraphEvaluator(const MetaFuncGraphPtr &meta_func_graph, AnalysisContextPtr, const ScopePtr &scope) + : BaseFuncGraphEvaluator(AnalysisContext::DummyContext()), meta_func_graph_(meta_func_graph), scope_(scope) {} + ~MetaFuncGraphEvaluator() override = default; + MS_DECLARE_PARENT(MetaFuncGraphEvaluator, BaseFuncGraphEvaluator); + + FuncGraphPtr GetFuncGraph(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) override; + + // Return normalized versions of the arguments. + AbstractBasePtrList NormalizeArgs(const AbstractBasePtrList &args_spec_list) const override { + return meta_func_graph_->NormalizeArgs(args_spec_list); + } + std::string ToString() const override { return identifier_ + "_" + meta_func_graph_->ToString(); } + + private: + MetaFuncGraphPtr meta_func_graph_; + std::unordered_map + func_graph_cache_; + ScopePtr scope_; +}; + +class PartialAppEvaluator : public Evaluator { + public: + PartialAppEvaluator(const EvaluatorPtr &evaluator, const AbstractBasePtrList &args) + : Evaluator("PartialAppEvaluator"), evaluator_(evaluator), args_spec_list_(args) {} + ~PartialAppEvaluator() override = default; + MS_DECLARE_PARENT(PartialAppEvaluator, Evaluator); + AnfNodePtr bound_node() const override { + if (evaluator_ != nullptr) { + return evaluator_->bound_node(); + } + return bound_node_.lock(); + } + + void set_bound_node(const AnfNodePtr &node) override { + if (evaluator_ != nullptr) { + evaluator_->set_bound_node(node); + } + bound_node_ = AnfNodeWeakPtr(node); + } + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) override { + MS_LOG(EXCEPTION) << "Should not be called, Run() method should be called"; + } + + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) override; + std::string ToString() const override { return identifier_ + "_" + evaluator_->ToString(); } + + private: + EvaluatorPtr evaluator_; + AbstractBasePtrList args_spec_list_; +}; + +class VirtualEvaluator : public Evaluator { + public: + VirtualEvaluator(const AbstractBasePtrList &args_spec_list, const AbstractBasePtr &output) + : Evaluator("virtual"), args_spec_list_(args_spec_list), output_(output) {} + ~VirtualEvaluator() override = default; + MS_DECLARE_PARENT(VirtualEvaluator, Evaluator); + + AbstractBasePtr Infer(AnalysisEnginePtr engine, const AbstractBasePtrList &args_spec_list) override; + std::string ToString() const override { return identifier_; } + + private: + AbstractBasePtrList args_spec_list_; + AbstractBasePtr output_; +}; + +class JEvaluator : public Evaluator { + public: + JEvaluator(const EvaluatorPtr &evaluator, const AbstractFunctionPtr &orig_func) + : Evaluator("JEvaluator"), evaluator_(evaluator), orig_func_(orig_func) {} + ~JEvaluator() override = default; + MS_DECLARE_PARENT(JEvaluator, Evaluator); + AnfNodePtr bound_node() const override { + if (evaluator_ != nullptr) { + return evaluator_->bound_node(); + } + return bound_node_.lock(); + } + + void set_bound_node(const AnfNodePtr &node) override { + if (evaluator_ != nullptr) { + evaluator_->set_bound_node(node); + } + bound_node_ = AnfNodeWeakPtr(node); + } + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) override { + MS_LOG(EXCEPTION) << "Should not be called, Run() method should be called"; + } + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) override; + std::string ToString() const override { return identifier_ + "_" + evaluator_->ToString(); } + + private: + EvaluatorPtr evaluator_; + AbstractFunctionPtr orig_func_; +}; +} // namespace abstract +} // namespace mindspore +#endif // PIPELINE_STATIC_ANALYSIS_EVALUATOR_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/param_validator.cc b/mindspore/ccsrc/pipeline/static_analysis/param_validator.cc new file mode 100644 index 00000000000..1b70e2fe22b --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/param_validator.cc @@ -0,0 +1,148 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/param_validator.h" + +#include +#include +#include +#include "utils/symbolic.h" +#include "pipeline/static_analysis/utils.h" + +namespace mindspore { +namespace abstract { +#define ABSTRACT_REPORT_NAME_DEC(abstract) constexpr char ReportNameTraits::name[]; + +ABSTRACT_REPORT_NAME_DEC(Tensor) +ABSTRACT_REPORT_NAME_DEC(Tuple) +ABSTRACT_REPORT_NAME_DEC(Scalar) +ABSTRACT_REPORT_NAME_DEC(List) +ABSTRACT_REPORT_NAME_DEC(Dictionary) +ABSTRACT_REPORT_NAME_DEC(Slice) +ABSTRACT_REPORT_NAME_DEC(Function) +ABSTRACT_REPORT_NAME_DEC(Type) +ABSTRACT_REPORT_NAME_DEC(KeywordArg) +ABSTRACT_REPORT_NAME_DEC(Class) + +TypePtr CheckType(TypePtr type, const TypePtrList &accepts, const std::string &error_message_prefix) { + bool ok = std::any_of(accepts.begin(), accepts.end(), + [type](const TypePtr &accept) -> bool { return IsIdentidityOrSubclass(type, accept); }); + if (ok) { + return type; + } else { + MS_LOG(EXCEPTION) << error_message_prefix << accepts << " but is " << type->ToString(); + } +} + +TypePtr CheckTensorDType(const AbstractTensorPtr &tensor, const TypePtrList &accepts, + const std::string &error_message_prefix) { + MS_EXCEPTION_IF_NULL(tensor); + TypePtr type = tensor->BuildType(); + if (!type->isa()) { + MS_LOG(EXCEPTION) << error_message_prefix << "requires Tensor but got " << type->ToString(); + } + TypePtr ele_type = tensor->element()->BuildType(); + if (ele_type == nullptr) { + MS_LOG(EXCEPTION) << "abstract tensor element type nullptr"; + } + return CheckType(ele_type, accepts, error_message_prefix); +} + +TypePtr CheckTensorsDTypeSame(const AbstractTensorPtrList &tensor_list, const TypePtrList &accepts, + const std::string &error_message_prefix) { + if (tensor_list.empty()) { + MS_LOG(EXCEPTION) << "array list is empty"; + } + + auto sample_tensor = tensor_list[0]; + MS_EXCEPTION_IF_NULL(sample_tensor); + TypePtr sample_type = sample_tensor->element()->BuildType(); + std::ostringstream loginfoBuffer; + loginfoBuffer << "same type, got"; + // Check if other elements have the same type with the first element. + for (size_t index = 1; index < tensor_list.size(); ++index) { + MS_EXCEPTION_IF_NULL(tensor_list[index]); + auto aType = tensor_list[index]->element()->BuildType(); + loginfoBuffer << " " << aType->ToString(); + if (sample_type->type_id() != aType->type_id()) { + MS_LOG(EXCEPTION) << "expected type " << sample_type->ToString() << ", but got " << aType->ToString() + << ", index " << index; + } + } + MS_LOG(DEBUG) << error_message_prefix << loginfoBuffer.str(); + return CheckTensorDType(sample_tensor, accepts, error_message_prefix); +} + +TypePtr CheckScalarType(const AbstractScalarPtr &scalar, const TypePtrList &accepts, + const std::string &error_message_prefix) { + if (scalar == nullptr) { + MS_LOG(EXCEPTION) << "scalar nullptr"; + } + auto type = scalar->BuildType(); + if (type == nullptr) { + MS_LOG(EXCEPTION) << "scalar value nullptr"; + } + + return CheckType(type, accepts, error_message_prefix); +} + +ShapePtr CheckShapeSame(const std::string &op, const AbstractTensorPtr &tensor_base, const AbstractTensorPtr &tensor) { + ShapePtr shape_base = tensor_base->shape(); + ShapePtr shape = tensor->shape(); + if (*shape != *shape_base) { + MS_LOG(EXCEPTION) << "" << op << " evaluator first arg shape " << tensor->shape()->ToString() + << " are not consistent with second arg shape " << tensor_base->shape()->ToString(); + } + return shape_base; +} + +TypePtr CheckDtypeSame(const std::string &op, const AbstractTensorPtr &tensor_base, const AbstractTensorPtr &tensor) { + TypePtr type_base = tensor_base->element()->BuildType(); + TypePtr type = tensor->element()->BuildType(); + if (*type != *type_base) { + MS_LOG(EXCEPTION) << "" << op << " evaluator first arg dtype " << type_base->ToString() + << " are not consistent with second arg dtype " << type->ToString(); + } + return type_base; +} + +int CheckAxis(const std::string &op, const ValuePtr &axis, int minimum, int max) { + if (axis == nullptr) { + MS_LOG(EXCEPTION) << "" << op << " evaluator axis is null"; + } + if (!axis->isa()) { + MS_LOG(EXCEPTION) << "" << op << " evaluator axis should be int, but got " << axis->type_name(); + } + int axis_value = GetValue(axis); + if (axis_value > max || axis_value < minimum) { + MS_LOG(EXCEPTION) << "" << op << " evaluator axis value should be in the range [" << minimum << ", " << max + << "], but get " << axis_value; + } + return axis_value; +} +void CheckArgsSize(const std::string &op, const mindspore::abstract::AbstractBasePtrList &args_spec_list, + size_t size_expect) { + if (args_spec_list.size() != size_expect) { + MS_LOG(EXCEPTION) << "" << op << " input args size should be " << size_expect << ", but got " + << args_spec_list.size(); + } + + for (size_t i = 0; i < size_expect; i++) { + MS_EXCEPTION_IF_NULL(args_spec_list[i]); + } +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h new file mode 100644 index 00000000000..5904c7e67a1 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h @@ -0,0 +1,98 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_PARAM_VALIDATOR_H_ +#define PIPELINE_STATIC_ANALYSIS_PARAM_VALIDATOR_H_ + +#include +#include +#include +#include +#include "pipeline/static_analysis/abstract_value.h" +#include "pipeline/static_analysis/utils.h" +#include "utils/any.h" +#include "ir/primitive.h" + +namespace mindspore { +namespace abstract { +// check if variable's type is an instance of any of accepts or of a subclass of it. +TypePtr CheckType(TypePtr type, const TypePtrList &accepts, const std::string &error_message_prefix); + +TypePtr CheckTensorDType(const AbstractTensorPtr &tensor, const TypePtrList &accepts, + const std::string &error_message_prefix); + +TypePtr CheckTensorsDTypeSame(const AbstractTensorPtrList &tensor_list, const TypePtrList &accepts, + const std::string &error_message_prefix); + +TypePtr CheckScalarType(const AbstractScalarPtr &scalar, const TypePtrList &accepts, + const std::string &error_message_prefix); + +ShapePtr CheckShapeSame(const std::string &op, const AbstractTensorPtr &tensor_base, const AbstractTensorPtr &tensor); + +TypePtr CheckDtypeSame(const std::string &op, const AbstractTensorPtr &tensor_base, const AbstractTensorPtr &tensor); + +int CheckAxis(const std::string &op, const ValuePtr &axis, int min, int max); + +void CheckArgsSize(const std::string &op, const AbstractBasePtrList &args_spec_list, size_t size_expect); + +template +struct ReportNameTraits {}; + +#define ABSTRACT_REPORT_NAME_TRAITS(abstract) \ + template <> \ + struct ReportNameTraits { \ + static constexpr char name[] = #abstract; \ + }; +ABSTRACT_REPORT_NAME_TRAITS(Tensor) +ABSTRACT_REPORT_NAME_TRAITS(Tuple) +ABSTRACT_REPORT_NAME_TRAITS(Scalar) +ABSTRACT_REPORT_NAME_TRAITS(List) +ABSTRACT_REPORT_NAME_TRAITS(Dictionary) +ABSTRACT_REPORT_NAME_TRAITS(Slice) +ABSTRACT_REPORT_NAME_TRAITS(Function) +ABSTRACT_REPORT_NAME_TRAITS(Type) +ABSTRACT_REPORT_NAME_TRAITS(KeywordArg) +ABSTRACT_REPORT_NAME_TRAITS(Class) + +template +std::shared_ptr CheckArg(const std::string &op, const AbstractBasePtrList &args_spec_list, size_t index) { + if (index >= args_spec_list.size()) { + MS_EXCEPTION(ValueError) << "" << op << " evaluator args list index out of bound, size " << args_spec_list.size() + << ", index " << index; + } + auto arg = dyn_cast(args_spec_list[index]); + if (arg == nullptr) { + MS_EXCEPTION(TypeError) << "Operator " << op << " input[" << index << "] should be " << ReportNameTraits::name + << ", but got " << args_spec_list[index]->BuildType()->ToString() << "."; + } + return arg; +} + +// check if each element in args_spec is type T, and can be joined. +template +void CheckArgsSpec(const AbstractBasePtrList &args_list) { + for (const auto &arg : args_list) { + if (!arg->isa()) { + MS_EXCEPTION(TypeError) << "Expected type " << ReportNameTraits::name << ", but got " + << arg->BuildType()->ToString() << "."; + } + } + (void)AbstractJoin(args_list); +} +} // namespace abstract +} // namespace mindspore + +#endif // PIPELINE_STATIC_ANALYSIS_PARAM_VALIDATOR_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.cc b/mindspore/ccsrc/pipeline/static_analysis/prim.cc new file mode 100644 index 00000000000..98d82de5d59 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.cc @@ -0,0 +1,1156 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/prim.h" + +#include +#include +#include +#include +#include +#include + +#include "operator/cc_implementations.h" +#include "operator/ops.h" +#include "operator/composite/do_signature.h" +#include "operator/prim_to_function.h" +#include "pipeline/static_analysis/utils.h" +#include "utils/symbolic.h" +#include "./common.h" +#include "pipeline/resource.h" +#include "pipeline/parse/resolve.h" +#include "ir/meta_tensor.h" +#include "utils/convert_utils.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/static_analysis/param_validator.h" +#include "common/utils.h" + +namespace mindspore { +namespace abstract { +PrimitiveEvalImplMap PrimitiveToInferImplMap = { + // Statements + {prim::kPrimReturn, {InferImplReturn, true}}, + {prim::kPrimTypeOf, {InferImplTypeof, false}}, + {prim::kPrimHasType, {InferImplHasType, false}}, + {prim::kPrimDot, {InferImplDot, true}}, + {prim::kPrimSwitch, {InferImplSwitch, true}}, + {prim::kPrimIs_, {InferImplIs_, true}}, + {prim::kPrimIsNot, {InferImplIsNot, true}}, + // Maths + {prim::kPrimMaximumGrad, {InferImplMinOrMaxGrad, true}}, + {prim::kPrimMinimumGrad, {InferImplMinOrMaxGrad, true}}, + // Array + {prim::kPrimScalarToArray, {InferImplScalarToArray, true}}, + {prim::kPrimArrayToScalar, {InferImplArrayToScalar, true}}, + {prim::kPrimBroadcastShape, {InferImplBroadCastShape, true}}, + {prim::kPrimShape, {InferImplShape, true}}, + {prim::kPrimPack, {InferImplPack, true}}, + // Structure + {prim::kPrimMakeTuple, {InferImplMakeTuple, true}}, + {prim::kPrimMakeList, {InferImplMakeList, true}}, + {prim::kPrimMakeDict, {InferImplMakeDict, true}}, + {prim::kPrimMakeSlice, {InferImplMakeSlice, true}}, + {prim::kPrimMakeKeywordArg, {InferImplMakeKwarg, true}}, + {prim::kPrimExtractKeywordArg, {InferImplExtractKwarg, true}}, + {prim::kPrimMakeRecord, {InferImplMakeRecord, false}}, + {prim::kPrimTupleGetItem, {InferImplTupleGetItem, true}}, + {prim::kPrimListGetItem, {InferImplListGetItem, true}}, + {prim::kPrimTupleSetItem, {InferImplTupleSetItem, true}}, + {prim::kPrimListSetItem, {InferImplListSetItem, true}}, + {prim::kPrimDictGetItem, {InferImplDictGetItem, true}}, + {prim::kPrimDictSetItem, {InferImplDictSetItem, true}}, + {prim::kPrimListAppend, {InferImplListAppend, true}}, + {prim::kPrimTupleLen, {InferImplTupleLen, true}}, + {prim::kPrimListLen, {InferImplListLen, true}}, + {prim::kPrimArrayLen, {InferImplArrayLen, true}}, + {prim::kPrimListMap, {InferImplListMap, false}}, + {prim::kPrimListReduce, {InferImplListReduce, false}}, + {prim::kPrimTupleReversed, {InferImplTupleReversed, false}}, + {prim::kPrimReducedShape, {InferImplReduceShape, false}}, + {prim::kPrimTupleDiv, {InferImplTupleDiv, false}}, + {prim::kPrimTupleToArray, {InferImplTuple2Array, false}}, + {prim::kPrimShapeMul, {InferImplShapeMul, false}}, + {prim::kPrimTupleEqual, {InferImplTupleEqual, false}}, + {prim::kPrimListEqual, {InferImplListEqual, false}}, + {prim::kPrimMakeRange, {InferImplMakeRange, false}}, + {prim::kPrimStopGradient, {InferImplStopGradient, false}}, + {prim::kPrimStringEqual, {InferImplStringEqual, false}}, + {prim::kPrimDictLen, {InferImplDictLen, false}}, + // NN + {prim::kPrimPooling, {InferImplPooling, true}}, + {prim::kPrimPoolingGrad, {InferImplPoolingGrad, true}}, + {prim::kPrimFusedBatchNorm, {InferImplFusedBatchNorm, true}}, + {prim::kPrimFusedBatchNormGrad, {InferImplFusedBatchNormGrad, true}}, + {prim::kPrimReluGrad, {InferImplReluGrad, true}}, + {prim::kPrimConv2DBackpropInput, {InferImplConv2DBackpropInput, true}}, + {prim::kPrimConv2DBackpropFilter, {InferImplConv2DBackpropFilter, true}}, + {prim::kPrimBiasAddGrad, {InferImplBiasAddGrad, true}}, + {prim::kPrimRelu, {InferImplRelu, true}}, + {prim::kPrimZerosLikeTensor, {InferImplZerosLikeTensor, true}}, + {prim::kPrimFakeBprop, {InferImplFakeBprop, false}}, + {prim::kPrimLayerNorm, {InferImplLayerNorm, true}}, + {prim::kPrimLayerNormGrad, {InferImplLayerNormGrad, true}}, + {prim::kPrimDropoutGenMask, {InferImplDropoutGenMask, true}}, + // Others + {prim::kPrimIdentity, {InferImplIdentity, true}}, + // Set impl to null as it will use PartialEvaluator; + {prim::kPrimPartial, {nullptr, true}}, + {prim::kPrimJ, {InferImplJ, false}}, + {prim::kPrimEnvGetItem, {InferImplEnvGetItem, true}}, + {prim::kPrimEnvSetItem, {InferImplEnvSetItem, true}}, + {prim::kPrimEnvAdd, {InferImplEnvAdd, true}}, + {prim::kPrimMakeRefKey, {InferImplMakeRefKey, true}}, + {prim::kPrimMakeRef, {InferImplMakeRef, true}}, + {prim::kPrimGetRefKey, {InferImplGetRefKey, true}}, + {prim::kPrimGetRefValue, {InferImplGetRefValue, true}}, + {prim::kPrimGetRefOrigin, {InferImplGetRefOrigin, true}}, + {prim::kPrimStateSetItem, {InferImplStateSetItem, true}}, + {prim::kPrimDepend, {InferImplDepend, true}}, + {prim::kPrimBroadcastGradientArgs, {InferImplBroadcastGradientArgs, false}}, + {prim::kPrimControlDepend, {InferImplControlDepend, true}}, + // Debug + {prim::kPrimScalarSummary, {InferImplScalarSummary, true}}, + {prim::kPrimImageSummary, {InferImplTensorSummary, true}}, + {prim::kPrimTensorSummary, {InferImplTensorSummary, true}}, +}; + +using mindspore::parse::PyObjectWrapper; + +AbstractBasePtr StandardPrimEvaluator::EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args) { + AbstractBasePtr abs_base = eval_impl_(engine, prim_, args); + return abs_base; +} + +AbstractBasePtr DoSignatureEvaluator::Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf) { + AbstractBasePtrList args_spec_list; + if (!prim_->isa()) { + MS_LOG(EXCEPTION) << "Primitive should be DoSignature, but " << prim_->ToString(); + } + if (out_conf->node() == nullptr || !out_conf->node()->isa()) { + MS_LOG(EXCEPTION) << "Node of out_conf should be CNode"; + } + + auto do_signature = dyn_cast(prim_); + auto out_node = dyn_cast(out_conf->node()); + const auto &out_node_inputs = out_node->inputs(); + if (out_node->inputs().size() == 0 || (out_node_inputs.size() - 1) != args_conf_list.size()) { + MS_LOG(EXCEPTION) << "Op: " << do_signature->function()->ToString() + << " args size should equal to inputs size minus 1, but args size " << args_conf_list.size() + << ", inputs size " << out_node_inputs.size(); + } + AnfNodePtrList args_inputs{out_node_inputs.begin() + 1, out_node_inputs.end()}; + + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &ref) -> AbstractBasePtr { return ref->GetEvaluatedValue(); }); + + ScopePtr scope = kDefaultScope; + if (out_conf != nullptr) { + scope = out_conf->node()->scope(); + } + ScopeGuard scope_guard(scope); + + AnfNodePtr new_cnode = nullptr; + if (bound_node() != nullptr) { + TraceManager::DebugTrace(std::make_shared(bound_node()->debug_info())); + new_cnode = prim::GenerateCNode(out_node->func_graph(), prim_->ToString(), do_signature->function(), args_spec_list, + args_inputs); + TraceManager::EndTrace(); + } else { + new_cnode = prim::GenerateCNode(out_node->func_graph(), prim_->ToString(), do_signature->function(), args_spec_list, + args_inputs); + } + AnfNodeConfigPtr fn_conf = engine->MakeConfig(new_cnode, out_conf->context()); + + return engine->ForwardConfig(out_conf, fn_conf); +} + +namespace { +py::object BuildValue(const ValuePtr &value_ptr) { + if (value_ptr == nullptr) { + return py::none(); + } else { + return ValuePtrToPyData(value_ptr); + } +} +} // end anonymous namespace + +py::dict ConvertAbstractToPython(const AbstractBasePtr &abs_base) { + MS_EXCEPTION_IF_NULL(abs_base); + py::dict dic; + if (abs_base->isa()) { + auto arg_tensor = dyn_cast(abs_base); + dic["shape"] = arg_tensor->shape()->shape(); + dic["dtype"] = arg_tensor->BuildType(); + dic["value"] = BuildValue(arg_tensor->BuildValue()); + } else if (abs_base->isa() || abs_base->isa() || abs_base->isa()) { + std::vector shape; + dic["shape"] = shape; + dic["dtype"] = abs_base->BuildType(); + dic["value"] = BuildValue(abs_base->BuildValue()); + } else if (abs_base->isa()) { + auto arg_tuple = dyn_cast(abs_base); + size_t len = arg_tuple->size(); + py::tuple shape_tuple(len); + py::tuple dtype_tuple(len); + + for (size_t i = 0; i < len; i++) { + py::dict out = ConvertAbstractToPython(arg_tuple->elements()[i]); + shape_tuple[i] = out["shape"]; + dtype_tuple[i] = out["dtype"]; + } + dic["shape"] = shape_tuple; + dic["dtype"] = dtype_tuple; + dic["value"] = BuildValue(arg_tuple->BuildValue()); + } else if (abs_base->isa()) { + auto arg_list = dyn_cast(abs_base); + size_t len = arg_list->size(); + py::list shape_list(len); + py::list dtype_list(len); + + for (size_t i = 0; i < len; i++) { + py::dict out = ConvertAbstractToPython(arg_list->elements()[i]); + shape_list[i] = out["shape"]; + dtype_list[i] = out["dtype"]; + } + dic["shape"] = shape_list; + dic["dtype"] = dtype_list; + dic["value"] = BuildValue(arg_list->BuildValue()); + } else if (abs_base->isa()) { + dic["shape"] = py::none(); + dic["dtype"] = py::none(); + dic["value"] = py::none(); + } else { + auto value = abs_base->BuildValue(); + if ((*value == *kAnyValue)) { + auto value_desc = abs_base->value_desc(); + MS_EXCEPTION(TypeError) << "Unsupported parameter " << (value_desc.empty() ? "type" : value_desc) + << " for python primitive."; + } + MS_EXCEPTION(TypeError) << "Unsupported parameter type for python primitive, the parameter value is " + << value->ToString(); + } + return dic; +} + +namespace { +py::tuple PreparePyInputs(const PrimitivePyPtr &prim_py, const AbstractBasePtrList &args) { + const AbstractBasePtrList *args_ptr; + + if (prim_py->is_tuple_input_) { + if (args.empty()) { + MS_LOG(EXCEPTION) << "Primitive args is empty"; + } + if (args[0] == nullptr || !args[0]->isa()) { + MS_LOG(EXCEPTION) << "Custom Primitive inputs should be packed into a Tuple after converting" + "prim convert pass for GE."; + } + args_ptr = &(args[0]->cast()->elements()); + } else { + args_ptr = &args; + } + + py::tuple py_args(args_ptr->size()); + for (size_t i = 0; i < args_ptr->size(); i++) { + auto arg_i = (*args_ptr)[i]; + py_args[i] = ConvertAbstractToPython(arg_i); + } + return py_args; +} + +AbstractBasePtr PyInferRes2Abstract(const PrimitivePyPtr &prim_py, const py::dict &output) { + // Convert to AbstractValue based on type and shape + if (output["value"].is_none()) { + auto out_shape = output["shape"]; + auto out_dtype = output["dtype"]; + return PyListDtype2AbstractTensor(out_shape, out_dtype); + } + // Convert pyobject to Value, then to AbstractValue + ValuePtr converted_ret = nullptr; + bool converted = parse::ConvertData(output["value"], &converted_ret); + if (!converted) { + MS_LOG(EXCEPTION) << "Convert data failed"; + } + auto res_spec = FromValue(converted_ret); + MS_EXCEPTION_IF_NULL(res_spec); + if (res_spec->isa()) { + // Replace to tensor constant node in specialize + auto res_tensor = res_spec->cast(); + res_tensor->set_value(converted_ret); + } + if (prim_py->IsCustomPrim()) { + // Raise error if output_num is not match the infer result. + int output_num = GetValue(prim_py->GetAttr("output_num")); + if (res_spec->isa() && output_num != 1) { + MS_LOG(EXCEPTION) << "Custom primitive " << prim_py->ToString() << " output_num " << output_num + << " not matches the infer result."; + } else if (res_spec->isa() && + (res_spec->cast()->size() != IntToSize(output_num))) { + MS_LOG(EXCEPTION) << "Custom primitive " << prim_py->ToString() << " output_num " << output_num + << " not matches the infer result."; + } + } + return res_spec; +} +} // end anonymous namespace + +AbstractBasePtr PythonPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const AbstractBasePtrList &args) { + MS_LOG(DEBUG) << "Eval for:" << prim_py_->ToString(); + + auto py_args = PreparePyInputs(prim_py_, args); + + auto pyobj = prim_py_->GetPyObj(); + if (pyobj == nullptr) { + MS_LOG(EXCEPTION) << "[" << prim_py_->ToString() << "]: pyobj is empty"; + } + auto infer_fuc = pyobj.attr("__infer__"); + + py::dict output = infer_fuc(*py_args); + MS_LOG(DEBUG) << "Output type is " << (std::string)py::str(output); + auto res_spec = PyInferRes2Abstract(prim_py_, output); + + MS_LOG(DEBUG) << "Python InferTensor result spec: " << res_spec->ToString() << "."; + return res_spec; +} + +AbstractBasePtr UniformPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const AbstractBasePtrList &args) { + // if func_desc_.retval type is super class of parameter type, then make the retval type as parameter type. + if (nargs_ != args.size()) { + MS_LOG(ERROR) << "UniformPrimEvaluator expect " << nargs_ << " args, but got " << args.size() << " inputs"; + return nullptr; + } + TypePtr ret_value_type = return_value_type_; + ValuePtrList value_list; + for (const auto &arg : args) { + // Check if all arguments are scalar type. + MS_EXCEPTION_IF_NULL(arg); + if (arg->isa()) { + auto arg_scalar = dyn_cast(arg); + auto arg_value = arg_scalar->GetValueTrack(); + value_list.push_back(arg_value); + } else { + // Raise TypeError Expected Scalar. + MS_LOG(EXCEPTION) << "Expect scalar arguments for uniform primitives."; + } + } + for (const auto &item : type_map_) { + TypePtrList selections; + MS_EXCEPTION_IF_NULL(item.second); + (void)std::transform(item.second->begin(), item.second->end(), std::back_inserter(selections), + [&args](size_t arg_idx) -> TypePtr { return args[arg_idx]->GetTypeTrack(); }); + TypePtr res = CheckTypeList(item.first, selections); + if (*return_value_type_ == *(item.first)) { + ret_value_type = res; + } + } + + ValuePtr inferred_value = RunImpl(value_list); + // for comparison primitives , return type shall have be specified to be bool. + if (specify_out_type_ != nullptr) { + ret_value_type = specify_out_type_; + } + + AbstractScalarPtr abs_base = std::make_shared(inferred_value, ret_value_type); + return abs_base; +} + +ValuePtr UniformPrimEvaluator::RunImpl(const ValuePtrList &args) const { + if (!eval_value_) { + return kAnyValue; + } else { + if (std::any_of(args.begin(), args.end(), [](const ValuePtr &arg) { + MS_EXCEPTION_IF_NULL(arg); + return arg->isa(); + })) { + return kAnyValue; + } + return impl_(args); + } +} + +// Primitive implementation +// static function start +namespace { +EvaluatorPtr InitStandardPrimEvaluator(PrimitivePtr primitive, const StandardPrimitiveEvalImpl eval_impl) { + EvaluatorPtr prim_evaluator = std::make_shared(primitive, eval_impl); + return prim_evaluator; +} + +EvaluatorPtr InitUniformPrimEvaluator(const PrimitivePtr &primitive, PrimitiveImpl prim_impl, bool eval_value, + const TypePtr &specify_out_type) { + FunctionPtr func = nullptr; + (void)prim::PrimToFunction::GetInstance().GetFunction(primitive, &func); + MS_EXCEPTION_IF_NULL(func); + + EvaluatorPtr uniform_primitive_evaluator = + std::make_shared(func, prim_impl, eval_value, specify_out_type); + return uniform_primitive_evaluator; +} + +const int kResolveCaseUserDefineClass = 1; +const int kResolveCaseBuildinTypeMethod = 2; +const int kResolveCaseFunction = 3; +int GetResolveCase(const TypePtr &data_type) { + MS_EXCEPTION_IF_NULL(data_type); + if (data_type->type_id() == kObjectTypeClass) { + return kResolveCaseUserDefineClass; + } + + // try method map, if not in method map, the data_type should be External type. + if (pipeline::Resource::IsTypeInMethodMap(data_type->type_id())) { + return kResolveCaseBuildinTypeMethod; + } + + return kResolveCaseFunction; +} + +FuncGraphPtr PyObjToGraph(const AnalysisEnginePtr &engine, const ValuePtr &method) { + MS_EXCEPTION_IF_NULL(engine); + MS_EXCEPTION_IF_NULL(method); + if (!method->isa()) { + MS_LOG(EXCEPTION) << "Method type error: " << method->ToString(); + } + + std::shared_ptr obj = method->cast>(); + FuncGraphPtr func_graph = mindspore::parse::ConvertToFuncGraph(obj->obj()); + if (func_graph == nullptr) { + MS_LOG(EXCEPTION) << "Parse python object: " << method->ToString() << " failed"; + } + + FuncGraphManagerPtr manager = engine->func_graph_manager(); + manager->AddFuncGraph(func_graph); + return func_graph; +} + +inline void AddToManager(const AnalysisEnginePtr &engine, const FuncGraphPtr func_graph) { + MS_EXCEPTION_IF_NULL(engine); + FuncGraphManagerPtr manager = engine->func_graph_manager(); + manager->AddFuncGraph(func_graph); +} + +AbstractBasePtr StaticGetterInferred(const ValuePtr &value, const ConfigPtr &data_conf, + const AnfNodeConfigPtr &old_conf) { + MS_EXCEPTION_IF_NULL(old_conf); + + AbstractBasePtr abs_ptr = ToAbstract(value, AnalysisContext::DummyContext(), old_conf); + AbstractFunctionPtr abs_func = dyn_cast(abs_ptr); + MS_EXCEPTION_IF_NULL(abs_func); + + // Create new cnode + std::vector input = {NewValueNode(prim::kPrimPartial)}; + auto func_graph_func = dyn_cast(abs_func); + if (func_graph_func != nullptr) { + FuncGraphPtr fg = func_graph_func->func_graph(); + input.push_back(NewValueNode(fg)); + } else { + auto prim_func = dyn_cast(abs_func); + MS_EXCEPTION_IF_NULL(prim_func); + PrimitivePtr prim = prim_func->prim(); + input.push_back(NewValueNode(prim)); + } + + AnfNodeConfigPtr conf = dyn_cast(data_conf); + MS_EXCEPTION_IF_NULL(conf); + input.push_back(conf->node()); + MS_EXCEPTION_IF_NULL(old_conf); + FuncGraphPtr func_graph = old_conf->node()->func_graph(); + CNodePtr new_cnode = func_graph->NewCNode(input); + AnalysisEnginePtr eng = old_conf->engine(); + AnfNodeConfigPtr fn_conf = eng->MakeConfig(new_cnode, old_conf->context()); + return eng->ForwardConfig(old_conf, fn_conf); +} + +AbstractBasePtr GenerateResolveAbstract(const AnfNodeConfigPtr &out_conf, const py::object &obj, + const ValuePtr &converted_ret) { + if (py::hasattr(obj, PYTHON_DATACLASS_FIELDS)) { + TypePtr cls_ptr = parse::ParseDataClass(converted_ret->cast>()->obj()); + + std::vector input = {NewValueNode(prim::kPrimPartial), NewValueNode(prim::kPrimMakeRecord), + NewValueNode(cls_ptr)}; + MS_EXCEPTION_IF_NULL(out_conf); + FuncGraphPtr func_graph = out_conf->node()->func_graph(); + CNodePtr new_cnode = func_graph->NewCNode(input); + AnalysisEnginePtr eng = out_conf->engine(); + AnfNodeConfigPtr fn_conf = eng->MakeConfig(new_cnode, out_conf->context()); + return eng->ForwardConfig(out_conf, fn_conf); + } else { + return ToAbstract(converted_ret, AnalysisContext::DummyContext(), out_conf); + } +} + +AbstractBasePtr GetEvaluatedValueForNameSpaceString(const AnalysisEnginePtr &engine, + const AbstractBasePtrList &args_spec_list, + const AnfNodeConfigPtr &out_conf) { + // args_spec_list: same as StaticGetter + if (args_spec_list.size() < 2) { + MS_LOG(EXCEPTION) << "Size of args_spec_list is less than 2"; + } + MS_EXCEPTION_IF_NULL(out_conf); + // An external type. + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + MS_EXCEPTION_IF_NULL(args_spec_list[1]); + MS_LOG(DEBUG) << "Args[0]: " << args_spec_list[0]->ToString(); + MS_LOG(DEBUG) << "Args[1]: " << args_spec_list[1]->ToString(); + auto data_v = args_spec_list[0]->BuildValue(); + if (!data_v->isa()) { + MS_LOG(EXCEPTION) << "Data is not NameSpace : " << data_v->ToString(); + } + + auto item_v = args_spec_list[1]->BuildValue(); + if (item_v->isa()) { + item_v = std::make_shared(item_v->cast()->value()); + } + + if (!item_v->isa()) { + MS_LOG(EXCEPTION) << "The value of the attribute could not be inferred: " << item_v->ToString(); + } + + // item_name to func addr from obj_map + parse::SymbolPtr symbol = item_v->cast(); + parse::NameSpacePtr name_space = data_v->cast(); + + parse::SymbolResolverPtr symbol_resolver = + std::make_shared(name_space, symbol, out_conf->node()); + if (!symbol_resolver->Resolve()) { + MS_LOG(EXCEPTION) << "Resolve node failed"; + } + + py::object obj = symbol_resolver->result(); + ValuePtr converted_ret = nullptr; + bool converted = parse::ConvertData(obj, &converted_ret, true); + if (!converted) { + MS_LOG(EXCEPTION) << "Convert data failed"; + } + if (converted_ret->isa()) { + AddToManager(engine, converted_ret->cast()); + } + return GenerateResolveAbstract(out_conf, obj, converted_ret); +} + +AbstractBasePtr GetEvaluatedValueForClassAttrOrMethod(const AnalysisEnginePtr &engine, + const AbstractBasePtrList &args_spec_list, const ValuePtr &item_v, + const ConfigPtr &data_conf, const AnfNodeConfigPtr &out_conf) { + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << "args_spec_list is empty"; + } + AbstractClassPtr cls = CheckArg("__FUNC__", args_spec_list, 0); + + // If item_v is an attribute, get abstract value from AbstractClass + MS_EXCEPTION_IF_NULL(item_v); + if (!item_v->isa()) { + MS_LOG(EXCEPTION) << "Attribute type error"; + } + std::string item_name = item_v->cast()->value(); + MS_LOG(DEBUG) << "Resovle name: " << cls->tag().name(); + MS_LOG(DEBUG) << "Resovle item: " << item_name; + + AbstractBasePtr attr = cls->GetAttribute(item_name); + if (attr != nullptr) { + return attr; + } + + ValuePtr method = cls->GetMethod(item_name); + if (method->isa()) { + MS_LOG(EXCEPTION) << "Unknown field, data type: " << args_spec_list[0]->BuildType()->ToString() + << ", item value: " << item_v->ToString(); + } + + // Infer class method + ValuePtr converted_v = PyObjToGraph(engine, method); + return StaticGetterInferred(converted_v, data_conf, out_conf); +} + +AbstractBasePtr GetEvaluatedValueForBuiltinTypeMethod(const AnalysisEnginePtr &engine, const ValuePtr &item_v, + const TypePtr &data_type, const ConfigPtr &data_conf, + const AnfNodeConfigPtr &out_conf) { + MS_EXCEPTION_IF_NULL(item_v); + MS_EXCEPTION_IF_NULL(data_type); + // The method maybe a Primitive or Composite + if (!item_v->isa()) { + MS_LOG(EXCEPTION) << "Error item is not string"; + } + + std::string item_name = item_v->cast()->value(); + Any method = pipeline::Resource::GetMethodPtr(data_type->type_id(), item_name); + if (method.empty()) { + MS_LOG(EXCEPTION) << "Object type: " << data_type->ToString() << " has no method: " << item_name; + } + + ValuePtr converted_v = nullptr; + if (method.is()) { + // composite registered in standard_method_map go to this branch + converted_v = prim::GetPythonOps(method.cast()); + AddToManager(engine, converted_v->cast()); + } else if (method.is()) { + converted_v = method.cast(); + } else { + MS_LOG(EXCEPTION) << "Expect to get string or PrimitivePtr from method map, but got " << method.ToString(); + } + return StaticGetterInferred(converted_v, data_conf, out_conf); +} + +AbstractBasePtr StaticGetter(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, + const ConfigPtr &data_conf, const AnfNodeConfigPtr &out_conf) { + // Inputs: namespace and its static function; or class and its member function + CheckArgsSize("StaticGetter", args_spec_list, 2); + + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + MS_EXCEPTION_IF_NULL(args_spec_list[1]); + TypePtr data_type = args_spec_list[0]->BuildType(); + ValuePtr item_value = args_spec_list[1]->BuildValue(); + ScopePtr scope = kDefaultScope; + if (out_conf != nullptr) { + scope = out_conf->node()->scope(); + } + ScopeGuard scope_guard(scope); + if (item_value->isa()) { + MS_LOG(EXCEPTION) << "The value of the attribute could not be inferred: " << item_value->ToString(); + } + + int case_v = GetResolveCase(data_type); + if (case_v == kResolveCaseUserDefineClass) { + return GetEvaluatedValueForClassAttrOrMethod(engine, args_spec_list, item_value, data_conf, out_conf); + } else if (case_v == kResolveCaseBuildinTypeMethod) { + return GetEvaluatedValueForBuiltinTypeMethod(engine, item_value, data_type, data_conf, out_conf); + } else { + return GetEvaluatedValueForNameSpaceString(engine, args_spec_list, out_conf); + } +} +} // end anonymous namespace + +// static variable start; +namespace { +class EmbedEvaluator : public SymbolicPrimEvaluator { + public: + EmbedEvaluator() : SymbolicPrimEvaluator("EmbedEvaluator") {} + ~EmbedEvaluator() override = default; + MS_DECLARE_PARENT(EmbedEvaluator, SymbolicPrimEvaluator); + AbstractBasePtr EvalPrim(const ConfigPtrList &args_conf_list) override { + // arg: free variable to be embeded + if (args_conf_list.size() != 1) { + MS_LOG(EXCEPTION) << "EmbedEvaluator requires 1 parameter, but got " << args_conf_list.size(); + } + AnfNodeConfigPtr node_conf = dyn_cast(args_conf_list[0]); + MS_EXCEPTION_IF_NULL(node_conf); + + AbstractBasePtr x = node_conf->GetEvaluatedValue(); + x = SensitivityTransform(x); + SymbolicKeyInstancePtr key = std::make_shared(node_conf->node(), x); + AbstractScalarPtr abs_scalar = std::make_shared(key, std::make_shared()); + return abs_scalar; + } +}; + +static AnfNodePtr FindParameterNodeByString(const FuncGraphManagerPtr &manager, const std::string &name) { + auto root_g_set = manager->roots(); + if (root_g_set.size() != 1) { + return nullptr; + } + const FuncGraphPtr &root_g = root_g_set.back(); + + for (auto ¶m_node : root_g->parameters()) { + auto param = param_node->cast(); + if (param && name == param->name()) { + return param; + } + } + return nullptr; +} + +class RefToEmbedEvaluator : public SymbolicPrimEvaluator { + public: + RefToEmbedEvaluator() : SymbolicPrimEvaluator("RefToEmbedEvaluator") {} + ~RefToEmbedEvaluator() override = default; + MS_DECLARE_PARENT(RefToEmbedEvaluator, SymbolicPrimEvaluator); + AbstractBasePtr EvalPrim(const ConfigPtrList &args_conf_list) override { + if (args_conf_list.size() != 1) { + MS_LOG(ERROR) << "Requires 1 parameter, but has: " << args_conf_list.size(); + return nullptr; + } + static TypePtr type = std::make_shared(); + auto node_conf = dyn_cast(args_conf_list[0]); + if (node_conf == nullptr) { + MS_LOG(ERROR) << "Conf should be AnfNodeConfig"; + return nullptr; + } + AbstractBasePtr abs = node_conf->GetEvaluatedValue(); + AbstractRefPtr ref_abs = abs->cast(); + if (ref_abs == nullptr) { + MS_LOG(ERROR) << "The first parameter of RefToEmbed should be Ref."; + return nullptr; + } + auto key_abs = ref_abs->ref_key(); + if (key_abs == nullptr) { + MS_LOG(ERROR) << "RefToEmbed input Ref key is nullptr."; + return nullptr; + } + auto key_value = key_abs->BuildValue(); + if (key_value == nullptr) { + MS_LOG(ERROR) << "RefToEmbed input Ref key value is nullptr."; + return nullptr; + } + auto refkey = key_value->cast(); + if (refkey == nullptr) { + return std::make_shared(type); + } + + std::string name = refkey->tag(); + const auto &manager = node_conf->node()->func_graph()->manager(); + auto node = FindParameterNodeByString(manager, name); + if (node == nullptr) { + MS_LOG(ERROR) << "RefToEmbed input can't find parameter \"" << name << "\" in graph."; + return nullptr; + } + AbstractBasePtr x = ref_abs->ref(); + x = SensitivityTransform(x); + std::shared_ptr key = std::make_shared(node, x); + std::shared_ptr abs_scalar = std::make_shared(key, type); + return abs_scalar; + } +}; + +class GetAttrEvaluator : public TransitionPrimEvaluator { + public: + GetAttrEvaluator() : TransitionPrimEvaluator("GetAttrEvaluator") {} + ~GetAttrEvaluator() override = default; + MS_DECLARE_PARENT(GetAttrEvaluator, TransitionPrimEvaluator); + AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, + const ConfigPtr &in_conf0, const AnfNodeConfigPtr &out_conf) override { + // Inputs: data, item + if (args_spec_list.size() != 2) { + MS_LOG(EXCEPTION) << "Expected args_spec_list size = 2, but has size:" << args_spec_list.size(); + } + AbstractBasePtr ret = nullptr; + if (bound_node() != nullptr) { + TraceManager::DebugTrace(std::make_shared(bound_node()->debug_info())); + ret = StaticGetter(engine, args_spec_list, in_conf0, out_conf); + TraceManager::EndTrace(); + } else { + ret = StaticGetter(engine, args_spec_list, in_conf0, out_conf); + } + // don't lookup from cache, as different out_conf with same node but different context + // may add different entry to anfnode_config_map, like getattr primitive; + (*cache_)[args_spec_list] = ret; + return ret; + } +}; + +class ResolveEvaluator : public TransitionPrimEvaluator { + public: + ResolveEvaluator() : TransitionPrimEvaluator("ResolveEvaluator") {} + ~ResolveEvaluator() override = default; + MS_DECLARE_PARENT(ResolveEvaluator, TransitionPrimEvaluator); + AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, + const ConfigPtr &in_conf0, const AnfNodeConfigPtr &out_conf) override { + // Inputs: namespace, symbol + if (args_spec_list.size() != 2) { + MS_LOG(EXCEPTION) << "Expected args_spec_list size = 2, but has size:" << args_spec_list.size(); + } + AbstractBasePtr ret = nullptr; + if (bound_node() != nullptr) { + TraceManager::DebugTrace(std::make_shared(bound_node()->debug_info())); + ret = StaticGetter(engine, args_spec_list, in_conf0, out_conf); + TraceManager::EndTrace(); + } else { + ret = StaticGetter(engine, args_spec_list, in_conf0, out_conf); + } + return ret; + } +}; + +class CreateInstanceEvaluator : public TransitionPrimEvaluator { + public: + CreateInstanceEvaluator() : TransitionPrimEvaluator("CreateInstanceEvaluator") {} + ~CreateInstanceEvaluator() override = default; + MS_DECLARE_PARENT(CreateInstanceEvaluator, TransitionPrimEvaluator); + AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, + const ConfigPtr &, const AnfNodeConfigPtr &out_conf) override { + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << "'args_spec_list' should not be empty"; + } + + // get the type parameter + MS_EXCEPTION_IF_NULL(args_spec_list[0]); + TypePtr type = args_spec_list[0]->GetTypeTrack(); + if (type->type_id() != kMetaTypeTypeType) { + MS_LOG(EXCEPTION) << "CreateInstanceEvaluator require first parameter should be an object of TypeType, but got " + << type->ToString(); + } + + ValuePtr value_track = args_spec_list[0]->GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + + std::shared_ptr type_obj = dyn_cast(value_track); + if (type_obj == nullptr) { + MS_LOG(EXCEPTION) << "Cast value failed, not PyObjectWrapper:" << value_track->ToString() << "."; + } + + if (!type_obj->isa()) { + MS_LOG(EXCEPTION) << "CreateInstanceEvaluator the type_obj should be an object of ClassType, but got " + << type_obj->ToString() << "."; + } + + auto class_type = type_obj->obj(); + MS_LOG(DEBUG) << "Get class type is " << type_obj->ToString() << "."; + + // get the create instance obj's parameters + pybind11::tuple params = GetParameters(args_spec_list); + + // create class instance + auto obj = parse::data_converter::CreatePythonObject(class_type, params); + if (py::isinstance(obj)) { + MS_LOG(EXCEPTION) << "Create python object failed, only support Cell and Primitive type"; + } + + // process the object + ValuePtr converted_ret = nullptr; + bool converted = parse::ConvertData(obj, &converted_ret, true); + if (!converted) { + MS_LOG(EXCEPTION) << "Convert the python object failed"; + } + MS_EXCEPTION_IF_NULL(converted_ret); + + if (converted_ret->isa()) { + AddToManager(engine, converted_ret->cast()); + } + + AbstractBasePtr ret = ToAbstract(converted_ret, AnalysisContext::DummyContext(), out_conf); + (*cache_)[args_spec_list] = ret; + return ret; + } + + pybind11::tuple GetParameters(const AbstractBasePtrList &args_spec_list) const { + // Exclude class type by minus 1; + std::size_t params_size = args_spec_list.size() - 1; + auto params = py::tuple(params_size); + if (params_size > 0) { + for (size_t i = 0; i < params_size; i++) { + // Only support the Scalar parameters type. Bypass class type by offset with 1. + auto arg = args_spec_list[i + 1]; + MS_EXCEPTION_IF_NULL(arg); + // Because the Tensor's AbstractTensor can't get value from GetValueTrack. + ValuePtr param_value = arg->BuildValue(); + py::object param = ValuePtrToPyData(param_value); + params[i] = param; + } + } + return params; + } +}; + +class PartialEvaluator : public Evaluator { + public: + PartialEvaluator() : Evaluator("PartialEvaluator") {} + ~PartialEvaluator() override = default; + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &args_conf_list, + AnfNodeConfigPtr out_conf = nullptr) override { + if (args_conf_list.size() == 0) { + MS_LOG(EXCEPTION) << "args size should be greater than 0"; + } + auto arg0_value = args_conf_list[0]->GetEvaluatedValue(); + AbstractBasePtrList args_spec_list{arg0_value}; + auto func = CheckArg("partial", args_spec_list, 0); + // Sometimes, node[0] in out_conf becomes phi0; + if (func->isa()) { + auto prim_func = dyn_cast(func); + if (prim_func->prim()->isa()) { + prim::DoSignaturePrimitivePtr do_signature_prim = dyn_cast(prim_func->prim()); + return HandleDoSignature(engine, do_signature_prim->function(), out_conf); + } + } + (void)std::transform(args_conf_list.begin() + 1, args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &ref) -> AbstractBasePtr { return ref->GetEvaluatedValue(); }); + + AbstractBasePtrList args(args_spec_list.begin() + 1, args_spec_list.end()); + + AbstractFuncAtomPtrList partialPtrList; + auto build_partial = [args, &partialPtrList](const AbstractFuncAtomPtr &atom_func) { + auto new_func = std::make_shared(atom_func, args); + partialPtrList.push_back(new_func); + }; + func->Visit(build_partial); + + auto ret = AbstractFunction::MakeAbstractFunction(partialPtrList); + (*cache_)[args_spec_list] = ret; + return ret; + } + + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) override { + MS_LOG(EXCEPTION) << "Infer() should not be called, Run() method should be called"; + } + + AbstractBasePtr HandleDoSignature(const AnalysisEnginePtr &engine, const ValuePtr &signature_value, + const AnfNodeConfigPtr &out_conf = nullptr) const { + MS_EXCEPTION_IF_NULL(out_conf); + MS_EXCEPTION_IF_NULL(out_conf->node()); + auto cnode = out_conf->node()->cast(); + if (cnode == nullptr) { + MS_LOG(EXCEPTION) << "Cnode is nullptr"; + } + std::vector new_nodes_inputs = cnode->inputs(); + auto new_signature_value = std::make_shared("signature", signature_value); + new_nodes_inputs[1] = NewValueNode(new_signature_value); + FuncGraphPtr func_graph = cnode->func_graph(); + + ScopePtr scope = kDefaultScope; + if (out_conf != nullptr) { + scope = out_conf->node()->scope(); + } + ScopeGuard scope_guard(scope); + + CNodePtr new_cnode = func_graph->NewCNode(new_nodes_inputs); + AnfNodeConfigPtr fn_conf = engine->MakeConfig(new_cnode, out_conf->context()); + return engine->ForwardConfig(out_conf, fn_conf); + } +}; + +struct PrimitiveImplInferValue { + PrimitiveImpl impl_; // implement function of primitive + bool eval_value_; // whether evaluate value + TypePtr specify_out_type_; // whether specify return type + bool in_white_list_; // true if this Primitive in white list, else false. +}; + +using PrimitiveToImplMap = std::unordered_map; + +PrimitiveToImplMap UniformPrimitiveToImplMapValue = { + {prim::kPrimScalarAdd, {prim::ScalarAdd, true, nullptr, true}}, + {prim::kPrimScalarSub, {prim::ScalarSub, true, nullptr, true}}, + {prim::kPrimScalarMul, {prim::ScalarMul, true, nullptr, true}}, + {prim::kPrimScalarDiv, {prim::ScalarDiv, true, nullptr, true}}, + {prim::kPrimScalarMod, {prim::ScalarMod, true, nullptr, true}}, + {prim::kPrimScalarUadd, {prim::ScalarUAdd, true, nullptr, true}}, + {prim::kPrimScalarUsub, {prim::ScalarUSub, true, nullptr, true}}, + {prim::kPrimScalarLog, {prim::ScalarLog, true, nullptr, true}}, + {prim::kPrimScalarEq, {prim::ScalarEq, true, std::make_shared(), true}}, + {prim::kPrimScalarLt, {prim::ScalarLt, true, std::make_shared(), true}}, + {prim::kPrimScalarGt, {prim::ScalarGt, true, std::make_shared(), true}}, + {prim::kPrimScalarNe, {prim::ScalarNe, true, std::make_shared(), true}}, + {prim::kPrimScalarLe, {prim::ScalarLe, true, std::make_shared(), true}}, + {prim::kPrimScalarGe, {prim::ScalarGe, true, std::make_shared(), true}}, + {prim::kPrimBoolNot, {prim::BoolNot, true, std::make_shared(), true}}, + {prim::kPrimBoolAnd, {prim::BoolAnd, true, std::make_shared(), true}}, + {prim::kPrimBoolEq, {prim::BoolEq, true, std::make_shared(), true}}, + {prim::kPrimBoolOr, {prim::BoolOr, true, std::make_shared(), true}}, +}; + +PrimEvaluatorMap PrimEvaluatorConstructors = PrimEvaluatorMap(); +std::mutex PrimEvaluatorConstructorMutex; + +void InitPrimEvaluatorConstructors(const PrimitiveEvalImplMap &prim_eval_impl_map) { + PrimEvaluatorMap &constructor = PrimEvaluatorConstructors; + + for (const auto &iter : prim_eval_impl_map) { + constructor[iter.first] = InitStandardPrimEvaluator(iter.first, iter.second.impl_); + } + + for (const auto &iter : UniformPrimitiveToImplMapValue) { + constructor[iter.first] = + InitUniformPrimEvaluator(iter.first, iter.second.impl_, iter.second.eval_value_, iter.second.specify_out_type_); + } + constructor[prim::kPrimEmbed] = std::make_shared(); + constructor[prim::kPrimRefToEmbed] = std::make_shared(); + constructor[prim::kPrimGetAttr] = std::make_shared(); + constructor[prim::kPrimResolve] = std::make_shared(); + constructor[prim::kPrimCreateInstance] = std::make_shared(); + constructor[prim::kPrimPartial] = std::make_shared(); +} +} // namespace + +void ClearPrimEvaluatorMap() { + PrimEvaluatorConstructors.clear(); + PrimitiveToInferImplMap.clear(); + UniformPrimitiveToImplMapValue.clear(); +} + +bool IsInWhiteList(const PrimitivePtr primitive) { + MS_EXCEPTION_IF_NULL(primitive); + + auto iter = PrimitiveToInferImplMap.find(primitive); + if (iter != PrimitiveToInferImplMap.end()) { + return iter->second.in_white_list_; + } + + auto uni_iter = UniformPrimitiveToImplMapValue.find(primitive); + if (uni_iter != UniformPrimitiveToImplMapValue.end()) { + return uni_iter->second.in_white_list_; + } + + return false; +} + +StandardPrimitiveEvalImpl GetPrimitiveInferImpl(const PrimitivePtr &primitive) { + MS_EXCEPTION_IF_NULL(primitive); + auto iter = PrimitiveToInferImplMap.find(primitive); + if (iter == PrimitiveToInferImplMap.end()) { + return nullptr; + } + return iter->second.impl_; +} + +PrimEvaluatorMap &GetPrimEvaluatorConstructors() { + PrimEvaluatorMap &constructor = PrimEvaluatorConstructors; + if (!constructor.empty()) { + return constructor; + } + std::lock_guard initLock(PrimEvaluatorConstructorMutex); + if (constructor.empty()) { + InitPrimEvaluatorConstructors(PrimitiveToInferImplMap); + } + + return constructor; +} + +namespace { +bool IsSubtypeTuple(const AbstractBasePtr x, const TypePtr model) { + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(model); + auto x_tuple = dyn_cast(x); + auto model_tuple = dyn_cast(model); + + if (x_tuple == nullptr || model_tuple == nullptr) { + return false; + } + + if (model->IsGeneric()) { + return true; + } + + if (x_tuple->size() != model_tuple->size()) { + return false; + } + + for (size_t i = 0; i < x_tuple->size(); i++) { + bool is_subtype = IsSubtype((*x_tuple)[i], (*model_tuple)[i]); + if (!is_subtype) { + return false; + } + } + return true; +} + +bool IsSubtypeArray(const AbstractBasePtr x, const TypePtr model) { + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(model); + auto x_tensor = dyn_cast(x); + auto model_tensor = dyn_cast(model); + + if (x_tensor == nullptr || model_tensor == nullptr) { + return false; + } + + if (model->IsGeneric()) { + return true; + } + + return IsSubtype(x_tensor->element(), model_tensor->element()); +} + +bool IsSubtypeList(const AbstractBasePtr x, const TypePtr model) { + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(model); + auto x_list = dyn_cast(x); + auto model_list = dyn_cast(model); + + if (x_list == nullptr || model_list == nullptr) { + return false; + } + + if (model->IsGeneric()) { + return true; + } + + if (x_list->size() != model_list->size()) { + return false; + } + + bool is_subtype = true; + for (size_t i = 0; i < x_list->size(); i++) { + is_subtype = IsSubtype((*x_list)[i], (*model_list)[i]); + if (!is_subtype) { + return false; + } + } + return is_subtype; +} + +bool IsSubtypeClass(const AbstractBasePtr x, const TypePtr model) { + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(model); + auto x_class = dyn_cast(x); + auto model_class = dyn_cast(model); + if (x_class == nullptr) { + return false; + } + if (model->IsGeneric()) { + return true; + } + + if (x_class->tag() == model_class->tag()) { + auto m_attributes = model_class->GetAttributes(); + auto x_attributes = x_class->attributes(); + if (m_attributes.size() != x_attributes.size()) { + return false; + } + + for (size_t i = 0; i < m_attributes.size(); i++) { + if (!IsSubtype(x_attributes[i].second, m_attributes[i].second)) { + return false; + } + } + return true; + } + + return false; +} + +inline bool IsSubtypeScalar(const AbstractBasePtr x, const TypePtr model) { + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(model); + if (dyn_cast(x) == nullptr) { + return false; + } + TypePtr x_type = x->GetTypeTrack(); + return IsSubType(x_type, model); +} +} // namespace + +bool IsSubtype(const AbstractBasePtr x, const TypePtr model) { + MS_EXCEPTION_IF_NULL(x); + MS_EXCEPTION_IF_NULL(model); + TypeId model_typeid = model->type_id(); + switch (model_typeid) { + case kMetaTypeObject: + return true; + case kObjectTypeTuple: + return IsSubtypeTuple(x, model); + case kObjectTypeTensorType: + return IsSubtypeArray(x, model); + case kObjectTypeList: + return IsSubtypeList(x, model); + case kObjectTypeClass: + return IsSubtypeClass(x, model); + default: + if (IsSubType(model, std::make_shared())) { + return IsSubtypeScalar(x, model); + } + MS_LOG(EXCEPTION) << "Invalid model type: " << model->ToString() << "."; + } +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.h b/mindspore/ccsrc/pipeline/static_analysis/prim.h new file mode 100644 index 00000000000..9dae576a4c2 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.h @@ -0,0 +1,314 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_PRIM_H_ +#define PIPELINE_STATIC_ANALYSIS_PRIM_H_ + +#include +#include +#include +#include +#include + +#include "pipeline/static_analysis/evaluator.h" + +namespace mindspore { +namespace abstract { +using StandardPrimitiveEvalImpl = AbstractBasePtr (*)(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &); +struct StandartPrimitiveImplReg { + StandardPrimitiveEvalImpl impl_; // Implement function of Primitive. + bool in_white_list_; // true if this Primitive in white list, else false. +}; + +using PrimitiveEvalImplMap = + std::unordered_map; + +class StandardPrimEvaluator : public TrivialPrimEvaluator { + public: + StandardPrimEvaluator(const PrimitivePtr primitive, StandardPrimitiveEvalImpl eval_impl) + : TrivialPrimEvaluator("StandardPrimEvaluator"), prim_(primitive), eval_impl_(eval_impl) {} + ~StandardPrimEvaluator() override = default; + MS_DECLARE_PARENT(StandardPrimEvaluator, TrivialPrimEvaluator); + AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args) override; + PrimitivePtr prim() { return prim_; } + + std::string ToString() const override { return identifier_ + prim_->name(); } + + private: + PrimitivePtr prim_; + const StandardPrimitiveEvalImpl eval_impl_; +}; + +using StandardPrimEvaluatorPtr = std::shared_ptr; + +class PythonPrimEvaluator : public TrivialPrimEvaluator { + public: + explicit PythonPrimEvaluator(const PrimitivePyPtr primitive) + : TrivialPrimEvaluator("PythonPrimEvaluator"), prim_py_(primitive) {} + ~PythonPrimEvaluator() override = default; + MS_DECLARE_PARENT(PythonPrimEvaluator, TrivialPrimEvaluator); + AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args) override; + PrimitivePtr prim() { return dyn_cast(prim_py_); } + + std::string ToString() const override { return identifier_ + prim_py_->name(); } + + private: + PrimitivePyPtr prim_py_; +}; + +class DoSignatureEvaluator : public Evaluator { + public: + explicit DoSignatureEvaluator(const PrimitivePtr primitive) : Evaluator("DoSignatureEvaluator"), prim_(primitive) {} + ~DoSignatureEvaluator() override = default; + AbstractBasePtr Run(AnalysisEnginePtr engine, const ConfigPtrList &argrefs, + AnfNodeConfigPtr out_config = nullptr) override; + + AbstractBasePtr Infer(AnalysisEnginePtr, const AbstractBasePtrList &) override { + MS_LOG(EXCEPTION) << "Infer() should not be called, Run() method should be called"; + } + + private: + PrimitivePtr prim_; +}; + +bool IsInWhiteList(PrimitivePtr primitive); +StandardPrimitiveEvalImpl GetPrimitiveInferImpl(const PrimitivePtr &primitive); + +using ValuePtrList = std::vector; +using PrimitiveImpl = ValuePtr (*)(const ValuePtrList &); + +class UniformPrimEvaluator : public TrivialPrimEvaluator { + public: + UniformPrimEvaluator(const FunctionPtr func_desc, PrimitiveImpl impl, bool eval_value, const TypePtr specify_out_type) + : TrivialPrimEvaluator("UniformPrimEvaluator"), + impl_(impl), + eval_value_(eval_value), + func_desc_(func_desc), + nargs_(func_desc_->args().size()), + return_value_type_(func_desc_->retval()), + specify_out_type_(specify_out_type) { + for (size_t i = 0; i < nargs_; ++i) { + TypePtr type = func_desc_->args()[i]; + if (type_map_[type]) { + type_map_[type]->push_back(i); + } else { + type_map_[type] = std::make_shared>(); + type_map_[type]->push_back(i); + } + } + } + ~UniformPrimEvaluator() override = default; + MS_DECLARE_PARENT(UniformPrimEvaluator, TrivialPrimEvaluator); + + AbstractBasePtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args) override; + ValuePtr RunImpl(const ValuePtrList &args) const; + + // If eval_value_ is False, return broadened arguments. + AbstractBasePtrList NormalizeArgs(const AbstractBasePtrList &args_spec_list) const override { + if (!eval_value_) { + AbstractBasePtrList broadened_args_spec_list; + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(broadened_args_spec_list), + [](const AbstractBasePtr &arg) -> AbstractBasePtr { return arg->Broaden(); }); + return broadened_args_spec_list; + } + return args_spec_list; + } + + private: + PrimitiveImpl impl_; + bool eval_value_; + const FunctionPtr func_desc_; + const std::size_t nargs_; + const TypePtr return_value_type_; + const TypePtr specify_out_type_; + std::unordered_map>, TypeHasher, TypeEqual> type_map_; +}; + +PrimEvaluatorMap &GetPrimEvaluatorConstructors(); + +// Check whether type x is a subtype of model. +bool IsSubtype(const AbstractBasePtr x, const TypePtr model); + +void ClearPrimEvaluatorMap(); + +py::dict ConvertAbstractToPython(const AbstractBasePtr &abs_base); + +AbstractBasePtr InferImplReturn(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTypeof(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplHasType(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplDot(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplSwitch(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIs_(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIsNot(const AnalysisEnginePtr &, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplPooling(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplPoolingGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplFusedBatchNorm(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplFusedBatchNormGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplReluGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplConv2DBackpropInput(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplConv2DBackpropFilter(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplBiasAddGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGelu(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGeluGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplRelu(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplZerosLikeTensor(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplFakeBprop(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplLayerNorm(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplLayerNormGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplDropoutGenMask(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplMinOrMaxGrad(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplScalarToArray(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplArrayToScalar(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplBroadCastShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplPack(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplMakeTuple(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeList(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeDict(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeSlice(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeKwarg(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplExtractKwarg(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeRecord(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTupleGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTupleSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplDictGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplDictSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListAppend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTupleLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplArrayLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListMap(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListReduce(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTupleReversed(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplReduceShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTupleDiv(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTuple2Array(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplShapeMul(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGenShapeIndex(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGenInverseIndex(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTupleEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplListEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeRange(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplStopGradient(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplStringEqual(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplDictLen(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplIdentity(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplJ(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplEnvGetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplEnvSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplEnvAdd(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeRefKey(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplMakeRef(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGetRefKey(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGetRefValue(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplGetRefOrigin(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplStateSetItem(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplDepend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplBroadcastGradientArgs(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplScalarSummary(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplTensorSummary(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +} // namespace abstract +} // namespace mindspore + +#endif // PIPELINE_STATIC_ANALYSIS_PRIM_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/program_specialize.cc b/mindspore/ccsrc/pipeline/static_analysis/program_specialize.cc new file mode 100644 index 00000000000..bfa1e43ceb0 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/program_specialize.cc @@ -0,0 +1,607 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/program_specialize.h" + +#include +#include +#include "./common.h" +#include "operator/ops.h" +#include "operator/composite/do_signature.h" +#include "utils/graph_utils.h" +#include "utils/profile.h" +#include "debug/trace.h" + +namespace mindspore { +namespace abstract { +namespace { +inline AbstractBasePtr GetEvaluatedValueWrap(const AnfNodeConfigPtr &conf) { + if (conf->node()->intermediate_abstract()) { + return conf->node()->intermediate_abstract(); + } + return conf->GetEvaluatedValue(); +} + +AnfNodePtr BuildValueNode(const ValuePtr &v, const AbstractBasePtr &abs_base) { + AnfNodePtr value_node = NewValueNode(v); + value_node->set_abstract(abs_base); + MS_LOG(DEBUG) << "Create ValueNode: " << value_node->ToString() << ", with abstract: " << abs_base->ToString(); + return value_node; +} + +bool IsVisible(FuncGraphPtr fg, const FuncGraphPtr &parent) { + while (fg != nullptr && fg != parent) { + fg = fg->parent(); + } + return fg == parent; +} +} // namespace + +FuncGraphPtr ProgramSpecializer::Run(const FuncGraphPtr &fg, const AnalysisContextPtr &context) { + MS_EXCEPTION_IF_NULL(fg); + MS_EXCEPTION_IF_NULL(context); + MS_LOG(DEBUG) << "Specialize topmost function graph: " << context->func_graph()->ToString(); + return SpecializeFuncGraph(fg, context); +} + +FuncGraphPtr ProgramSpecializer::SpecializeFuncGraph(const FuncGraphPtr &fg, const AnalysisContextPtr &context) { + MS_EXCEPTION_IF_NULL(fg); + MS_EXCEPTION_IF_NULL(context); + auto iter = specializations_.find(context->SpecializeKey()); + if (iter != specializations_.end()) { + return iter->second->specialized_func_graph(); + } + + std::shared_ptr fg_spec = std::make_shared(this, fg, context); + FuncGraphPtr fg2 = fg_spec->specialized_func_graph(); + specializations_[context->SpecializeKey()] = fg_spec; + fg_spec->Run(); + return fg2; +} + +std::shared_ptr ProgramSpecializer::GetFuncGraphSpecializer(const AnalysisContextPtr &context) { + MS_EXCEPTION_IF_NULL(context); + auto iter = specializations_.find(context->SpecializeKey()); + if (iter != specializations_.end()) { + return iter->second; + } + return nullptr; +} + +std::string GetNextCounter() { + static int g_CloneCounter = 1; + std::string str_count = std::to_string(g_CloneCounter); + g_CloneCounter++; + return str_count; +} + +FuncGraphSpecializer::FuncGraphSpecializer(ProgramSpecializer *const s, const FuncGraphPtr &fg, + const AnalysisContextPtr &context) + : specializer_(s), func_graph_(fg), context_(context) { + parent_ = s->GetFuncGraphSpecializer(context->parent()); + engine_ = s->engine(); + cloner_ = SpecializerClone(fg, std::make_shared(GetNextCounter())); + repl_node_ = cloner_->cloned_node(); + specialized_func_graph_ = cloner_->cloned_func_graph()[fg]; + todo_.push_back(fg->get_return()); + auto ps = fg->parameters(); + (void)todo_.insert(todo_.end(), ps.begin(), ps.end()); +} + +AnfNodePtr FuncGraphSpecializer::ReplicateDisconnectedNode(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + FuncGraphPtr fg = node->func_graph(); + + if (node->isa()) { + return node; + } + std::shared_ptr specializer = shared_from_this(); + while (fg != nullptr && fg != specializer->func_graph_) { + specializer = specializer->parent_; + } + // If had replicated, just return that. + auto iter = specializer->repl_node_->find(node); + if (iter != specializer->repl_node_->end()) { + return iter->second; + } + + auto new_node = specializer->cloner_->CloneDisconnected(node); + if (node->isa()) { + if (!new_node->isa()) { + MS_LOG(EXCEPTION) << "new_node must be a CNode, but is " << new_node->DebugString() << "."; + } + auto c_node = node->cast(); + MS_EXCEPTION_IF_NULL(c_node); + auto inputs = c_node->inputs(); + std::vector new_inputs; + (void)std::transform(inputs.begin(), inputs.end(), std::back_inserter(new_inputs), + [this](const AnfNodePtr &inp) -> AnfNodePtr { + if (inp->isa()) { + return inp; + } + return ReplicateDisconnectedNode(inp); + }); + auto c_new_node = new_node->cast(); + MS_EXCEPTION_IF_NULL(c_new_node); + c_new_node->set_inputs(new_inputs); + } + + iter = specializer->repl_node_->find(node); + if (iter != specializer->repl_node_->end()) { + if (iter->second == node) { + MS_LOG(EXCEPTION) << "Replicated is same as original node, node: " << node->ToString(); + } + } else { + MS_LOG(EXCEPTION) << "Replicate node failed, node: " << node->ToString(); + } + return new_node; +} + +AnfNodePtr FuncGraphSpecializer::GetReplicatedNode(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + FuncGraphPtr fg = node->func_graph(); + + std::shared_ptr specializer = shared_from_this(); + while (fg != nullptr && fg != specializer->func_graph_) { + specializer = specializer->parent_; + } + + MS_EXCEPTION_IF_NULL(specializer->repl_node_); + auto iter = specializer->repl_node_->find(node); + if (iter != specializer->repl_node_->end()) { + return iter->second; + } + return node; +} + +void FuncGraphSpecializer::Run() { + MS_LOG(DEBUG) << "Before run, origin func graph name: " << func_graph_->ToString() + << ", cloned func graph name: " << specialized_func_graph_->ToString() + << ", func graph: " << func_graph_->get_return()->DebugString(); + FirstPass(); + SecondPass(); + MS_LOG(DEBUG) << "After run, origin func graph name: " << func_graph_->ToString() + << ", cloned func graph name: " << specialized_func_graph_->ToString() + << ", new func graph: " << specialized_func_graph_->get_return()->DebugString(); +} + +void FuncGraphSpecializer::FirstPass() { + while (todo_.size()) { + AnfNodePtr node = todo_.back(); + todo_.pop_back(); + if (node->func_graph() == nullptr) { + // do nothing for ValueNode + continue; + } + if (node->func_graph() != func_graph_) { + if (parent_ == nullptr) { + MS_LOG(EXCEPTION) << "parent must not null NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } + parent_->AddTodoItem(node); + parent_->FirstPass(); + AnfNodePtr new_node = parent_->GetReplicatedNode(node); + if (node->isa()) { + parent_->ProcessCNode(new_node->cast()); + } + continue; + } + if (marked_.count(node) > 0) { + continue; + } + (void)marked_.insert(node); + ProcessNode(node); + } +} + +// Specialize CNode in func graphs +void FuncGraphSpecializer::SecondPass() { + for (auto &node : DeepLinkedGraphSearch(specialized_func_graph_->get_return())) { + if (node->isa()) { + ProcessCNode(node->cast()); + } + } +} + +void FuncGraphSpecializer::ProcessNode(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + ScopeGuard scope_guard(node->scope()); + AnfNodeConfigPtr conf = MakeConfig(node); + AnfNodePtr new_node = GetReplicatedNode(node); + MS_EXCEPTION_IF_NULL(new_node); + + if (new_node->func_graph() != specialized_func_graph_) { + MS_LOG(EXCEPTION) << "Error in specializer [A] node: " << node->DebugString() + << ", new_node: " << new_node->DebugString() + << ", new_node->func_graph(): " << new_node->func_graph()->ToString() + << ", specialized_func_graph_: " << specialized_func_graph_->ToString(); + return; + } + new_node->set_abstract(GetEvaluatedValueWrap(conf)); + MS_LOG(DEBUG) << "Set new_node: " << new_node->ToString() << ", abstract as: " << new_node->abstract()->ToString(); + + if (node->isa()) { + auto c_old = node->cast(); + auto c_new = new_node->cast(); + auto new_inputs = c_new->inputs(); + auto old_inputs = c_old->inputs(); + for (size_t i = 0; i < old_inputs.size(); ++i) { + auto node_input = old_inputs[i]; + AnfNodeConfigPtr iconf = MakeConfig(node_input); + AbstractBasePtr ival = GetEvaluatedValueWrap(iconf); + // First try to check if node_input can be replaced by a ValueNode. If cannot, then try to check if + // can be replaced by another CNode from anfnode_config_map, otherwise use the replicated node. + AnfNodePtr replace_node = BuildPossibleValueNode(iconf->node(), ival); + if (replace_node == nullptr) { + replace_node = BuildReplacedNode(iconf); + MS_EXCEPTION_IF_NULL(replace_node); + replace_node->set_abstract(ival); + MS_LOG(DEBUG) << "Set replaced: " << replace_node->ToString() << ", to abstract: " << ival->ToString(); + } else { + MS_LOG(DEBUG) << "Build possible value node for node: " << node_input->DebugString() + << ", ival: " << ival->ToString() << ", replace_node: " << replace_node->ToString(); + } + if (new_inputs[i] != replace_node) { + new_inputs[i] = replace_node; + MS_LOG(DEBUG) << "Set new_input[" << i << "] = " << replace_node->DebugString(); + } + } + c_new->set_inputs(new_inputs); + } +} + +AnfNodePtr FuncGraphSpecializer::BuildReplacedNode(const AnfNodeConfigPtr &conf) { + MS_EXCEPTION_IF_NULL(conf); + + auto conf_iter = engine_->anfnode_config_map().find(conf); + AnfNodeConfigPtr new_conf = conf; + while (conf_iter != engine_->anfnode_config_map().end()) { + MS_LOG(DEBUG) << "Origin conf: graph(" << new_conf->node()->func_graph()->ToString() << ", node(" + << new_conf->node()->DebugString() << ")"; + new_conf = conf_iter->second; + MS_EXCEPTION_IF_NULL(new_conf); + MS_LOG(DEBUG) << "Replaced conf: graph(" << conf->node()->func_graph()->ToString() << ", node(" + << conf->node()->DebugString() << ")"; + (void)ReplicateDisconnectedNode(new_conf->node()); + conf_iter = engine_->anfnode_config_map().find(new_conf); + } + todo_.push_back(new_conf->node()); + auto repl = GetReplicatedNode(new_conf->node()); + if (repl->func_graph()) { + MS_LOG(DEBUG) << "Set repl: graph(" << repl->func_graph()->ToString() << "), node:" << repl->DebugString() + << ") to replace origin:" << new_conf->node()->DebugString(); + } else { + MS_LOG(DEBUG) << "Set repl: graph(nullptr), node(" << repl->DebugString() + << ") to replace origin: " << new_conf->node()->DebugString(); + } + return repl; +} + +namespace { +const StringImmPtr kDeadNode = std::make_shared("Dead Node"); +const StringImmPtr kPolyNode = std::make_shared("Poly Node"); + +inline bool CanSpecializeNode(const AnfNodePtr &node) { + if (IsValueNode(node) || IsValueNode(node) || IsValueNode(node)) { + return true; + } + return false; +} +} // namespace + +AnfNodePtr FuncGraphSpecializer::BuildSpecializedNode(const AnfNodePtr &node, const AbstractBasePtr &abs, + const AbstractBasePtrList &argvals) { + MS_EXCEPTION_IF_NULL(abs); + AbstractFunctionPtr real_a = dyn_cast(abs); + MS_EXCEPTION_IF_NULL(real_a); + + AbstractFunctionPtr func = real_a->GetUnique(); + SpecializeStatusCode errcode; + ScopeGuard scope_guard(node->scope()); + AnfNodePtr repl = BuildSpecializedNodeInner(abs, func, argvals, &errcode); + if (repl == nullptr) { + if (errcode == kSpecializeFindUniqueArgvalDead) { + const auto error_dead_node = std::make_shared(kDeadNode, node); + repl = BuildValueNode(kDeadNode, error_dead_node); + MS_LOG(DEBUG) << "DEAD for node: " << node->DebugString() << ", abstract: " << abs->ToString(); + } else if (errcode == kSpecializeFindUniqueArgvalPoly) { + const auto error_poly_node = std::make_shared(kPolyNode, node); + repl = BuildValueNode(kPolyNode, error_poly_node); + MS_LOG(DEBUG) << "POLY for node: " << node->DebugString() << ", abstract: " << abs->ToString(); + } else { + MS_LOG(EXCEPTION) << "Failed to build specialized node, node: " << node->DebugString() + << ", abstract: " << abs->ToString(); + } + } + + return repl; +} + +AnfNodePtr FuncGraphSpecializer::BuildSpecializedNodeInner(const AbstractBasePtr &abs, const AbstractFunctionPtr &func, + const AbstractBasePtrList &args, + SpecializeStatusCode *errcode) { + MS_EXCEPTION_IF_NULL(abs); + MS_EXCEPTION_IF_NULL(func); + MS_EXCEPTION_IF_NULL(errcode); + *errcode = kSpecializeSuccess; + + auto real_func = dyn_cast(func); + if (real_func != nullptr) { + return BuildValueNode(real_func->prim(), abs); + } + + EvaluatorPtr eval; + eval = engine_->GetEvaluatorFor(func); + MS_EXCEPTION_IF_NULL(eval); + AbstractBasePtrList argvals = eval->NormalizeArgs(args); + + std::pair result; + SpecializeStatusCode status = FindUniqueArgvals(func, eval, argvals, &result); + if (status != kSpecializeSuccess) { + *errcode = status; + return nullptr; + } + argvals = result.first; + AbstractBasePtr unique_output = result.second; + + auto prim_func = dyn_cast(func); + if (prim_func != nullptr) { + auto type_func = std::make_shared(prim_func->prim(), argvals, unique_output); + return BuildValueNode(prim_func->prim(), type_func); + } + + if (!eval->isa()) { + MS_LOG(EXCEPTION) << "eval is not BaseGraphEvaluator, but " << eval->ToString(); + } + auto real_eval = dyn_cast(eval); + + if (func->context() != nullptr) { + if (!IsVisible(func_graph_, func->context()->func_graph())) { + MS_LOG(EXCEPTION) << "func is not visible NodeInfo: " << trace::GetDebugInfo(func_graph_->debug_info()); + } + } else { + MS_LOG(EXCEPTION) << "func context is nullptr NodeInfo: " << trace::GetDebugInfo(func_graph_->debug_info()); + } + AnalysisContextPtr context = real_eval->MakeContext(engine_, argvals); + MS_LOG(DEBUG) << "Specialize function graph: " << context->func_graph()->ToString() << ", args: " << argvals.size() + << ", graph: " << context->func_graph()->get_return()->DebugString(); + FuncGraphPtr v = specializer_->SpecializeFuncGraph(context->func_graph(), context); + return BuildValueNode(v, abs); +} + +const EvaluatorCacheMapPtr &FuncGraphSpecializer::GetEvalCache(const EvaluatorPtr &eval) { + auto cache_iter = evalcaches_.find(eval); + if (cache_iter == evalcaches_.end()) { + evalcaches_[eval] = eval->cache(); + return eval->cache(); + } + return cache_iter->second; +} + +std::pair FuncGraphSpecializer::BuildFromBroadedArgsVal( + const EvaluatorPtr &eval) { + MS_EXCEPTION_IF_NULL(eval); + std::unordered_set choices; + AbstractBasePtr ret = nullptr; + AbstractBasePtrList broaded_argvals; + for (auto &argvals_map : *evalcaches_[eval]) { + auto argvals = argvals_map.first; + broaded_argvals.clear(); + + (void)std::transform(argvals.begin(), argvals.end(), std::back_inserter(broaded_argvals), + [](const AbstractBasePtr &arg) -> AbstractBasePtr { return arg->Broaden(); }); + (void)choices.insert(broaded_argvals); + MS_LOG(DEBUG) << "Broaded_argvals: " << broaded_argvals.size() << ", " << ::mindspore::ToString(broaded_argvals); + } + + if (1 == choices.size()) { + ConfigPtrList args_conf_list; + (void)std::transform(broaded_argvals.begin(), broaded_argvals.end(), std::back_inserter(args_conf_list), + [](AbstractBasePtr v) -> ConfigPtr { return std::make_shared(v); }); + + // if broaden return null + ret = eval->Run(engine_, args_conf_list, nullptr); + EvaluatorCacheMapPtr real = std::make_shared(); + + (*real)[broaded_argvals] = ret; + evalcaches_[eval] = real; + return std::make_pair(broaded_argvals, ret); + } else { + MS_LOG(DEBUG) << "Choices.size: " << choices.size(); + return std::make_pair(AbstractBasePtrList(), nullptr); + } +} + +void FuncGraphSpecializer::ProcessCNode(const CNodePtr &new_node) { + MS_EXCEPTION_IF_NULL(new_node); + if (specializer_->seen().count(new_node) > 0) { + return; + } + specializer_->AddSeen(new_node); + + auto new_inputs = new_node->inputs(); + if (new_inputs.empty()) { + MS_LOG(EXCEPTION) << "Inputs of CNode is empty"; + } + AnfNodePtr func = new_inputs[0]; + MS_EXCEPTION_IF_NULL(func); + + // First element is func so arg start from 1 + std::vector args(new_inputs.begin() + 1, new_inputs.end()); + // CNode(CNode(Partial, f, arg1), arg2, ...) --> CNode(f, arg1, arg2, ...) + while (IsPrimitiveCNode(func, prim::kPrimPartial)) { + std::vector inputs = func->cast()->inputs(); + // First element is partial, second is func so arg is start from 2 + (void)args.insert(args.begin(), inputs.begin() + 2, inputs.end()); + func = inputs[1]; + new_inputs = args; + (void)new_inputs.insert(new_inputs.begin(), func); + } + + AbstractBasePtrList argvals; + MS_EXCEPTION_IF_NULL(new_inputs[0]); + AbstractBasePtr fnval = new_inputs[0]->abstract(); + MS_LOG(DEBUG) << "The new_inputs[0] node: pointer: " << new_inputs[0]->ToString() << ", " + << new_inputs[0]->DebugString() << ", abstract: " << new_inputs[0]->abstract()->ToString(); + + // First element is func so function arguments start from 1 + for (size_t i = 1; i < new_inputs.size(); ++i) { + argvals.push_back(new_inputs[i]->abstract()); + MS_LOG(DEBUG) << "The new_inputs[" << i << "] node: pointer: " << new_inputs[i]->ToString() << ", " + << new_inputs[i]->DebugString() << ", abstract: " << new_inputs[i]->abstract()->ToString(); + } + + if (CanSpecializeNode(func)) { + new_inputs[0] = BuildSpecializedNode(func, fnval, argvals); + } + + for (size_t i = 0; i < argvals.size();) { + size_t next = i + 1; + if (CanSpecializeNode(args[i])) { + new_inputs[next] = BuildSpecializedNode(args[i], argvals[i], std::vector{}); + } + // support for partial(Multitype) which Multitype should not be inferred to POLY. + // after one or more times clone, Multitype metafuncgraph evaluator will specialized to one type only, + // so even with partial parameter, it will specialize to that graph. + // Maybe a better idea should inline graph with partial node first, then it will have full + // parameter list to infer and specialize. + MS_EXCEPTION_IF_NULL(new_inputs[next]); + if (new_inputs[next]->isa() && (GetValueNode(new_inputs[next]) == kPolyNode) && + IsPrimitive(func, prim::kPrimPartial)) { + new_inputs[next] = args[i]; + } + i = next; + } + + new_node->set_inputs(new_inputs); +} + +namespace { +void DumpEvaluatorCache(const EvaluatorCacheMap &evaluator_cache_map, const AbstractBasePtrList &argvals) { + MS_LOG(DEBUG) << "Find unique argvals failed: " << argvals.size() << ", " << argvals << ". Check cache all items."; + int i = 0; + for (const auto &item : evaluator_cache_map) { + MS_LOG(DEBUG) << "evaluator_cache_map[" << i++ << "]: " << item.first; + } +} + +bool IsPolyFunc(const AbstractFunctionPtr &func, const AbstractBasePtrList &argvals) { + if (func->isa() && argvals.empty()) { + MS_LOG(DEBUG) << "High order primitive return POLY."; + return true; + } + if (func->isa() && argvals.empty()) { + auto meta_func_graph_wrapper = dyn_cast(func); + auto meta_func_graph = meta_func_graph_wrapper->meta_func_graph(); + if (meta_func_graph != nullptr && meta_func_graph->isa()) { + auto do_signature = dyn_cast(meta_func_graph); + if (do_signature != nullptr && do_signature->function()->isa()) { + MS_LOG(DEBUG) << "High order primitive " << do_signature->function()->ToString() << " return POLY."; + return true; + } + } + } + return false; +} +} // end anonymous namespace + +SpecializeStatusCode FuncGraphSpecializer::FindUniqueArgvals(const AbstractFunctionPtr &func, const EvaluatorPtr &eval, + const AbstractBasePtrList &argvals, + std::pair *result) { + MS_EXCEPTION_IF_NULL(func); + MS_EXCEPTION_IF_NULL(eval); + MS_EXCEPTION_IF_NULL(result); + + EvaluatorCacheMap evaluator_cache_map = *eval->cache(); + if (evaluator_cache_map.find(argvals) != evaluator_cache_map.end()) { + *result = std::make_pair(argvals, evaluator_cache_map[argvals]); + return kSpecializeSuccess; + } + DumpEvaluatorCache(evaluator_cache_map, argvals); + + const EvaluatorCacheMapPtr &choices = GetEvalCache(eval); + MS_EXCEPTION_IF_NULL(choices); + + if (choices->count(argvals)) { + *result = std::make_pair(argvals, (*choices)[argvals]); + return kSpecializeSuccess; + } else if (choices->size() == 1) { + MS_LOG(DEBUG) << "Evaluator cache has a single item, just use it."; + *result = std::make_pair(choices->begin()->first, choices->begin()->second); + return kSpecializeSuccess; + } else if (choices->empty()) { + MS_LOG(DEBUG) << "Find DEAD code, it may be optimized in later phase."; + return kSpecializeFindUniqueArgvalDead; + } else { + if (IsPolyFunc(func, argvals)) { + return kSpecializeFindUniqueArgvalPoly; + } + + MS_LOG(DEBUG) << "Try to find generalized argvals."; + *result = BuildFromBroadedArgsVal(eval); + if (!result->first.empty()) { + return kSpecializeSuccess; + } + MS_LOG(DEBUG) << "Find POLY code, it may be unused code or unresoved polymorphism."; + return kSpecializeFindUniqueArgvalPoly; + } +} + +AnfNodePtr FuncGraphSpecializer::BuildPossibleValueNode(const AnfNodePtr &origin_node, const AbstractBasePtr &ival) { + MS_EXCEPTION_IF_NULL(origin_node); + MS_EXCEPTION_IF_NULL(ival); + + AbstractFunctionPtr abs = dyn_cast(ival); + if (abs != nullptr) { + // Cannot build a determinstic ValueNode if there are multiple possible AbstractFunction. + if (abs->isa()) { + return nullptr; + } + ValuePtr value = nullptr; + if (abs->isa()) { + auto real_fn = dyn_cast(abs); + value = real_fn->prim(); + } else if (abs->isa()) { + auto real_fn = dyn_cast(abs); + value = real_fn->meta_func_graph(); + } else if (abs->isa()) { + auto real_fn = dyn_cast(abs); + value = real_fn->func_graph(); + } else { + return nullptr; + } + if (!value->isa() || value->cast()->parent() == nullptr || + (IsValueNode(origin_node) && IsVisible(func_graph_, value->cast()->parent()))) { + return BuildValueNode(value, ival); + } else { + return nullptr; + } + } else { + ValuePtr val = ival->BuildValue(); + if (val->isa()) { + return nullptr; + } else { + return BuildValueNode(val, ival); + } + } +} + +AnfNodeConfigPtr FuncGraphSpecializer::MakeConfig(const AnfNodePtr &node) { + return engine_->MakeConfig(node, context_); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/program_specialize.h b/mindspore/ccsrc/pipeline/static_analysis/program_specialize.h new file mode 100644 index 00000000000..ea3a3007d4b --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/program_specialize.h @@ -0,0 +1,131 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_SPECIALIZE_H_ +#define PIPELINE_STATIC_ANALYSIS_SPECIALIZE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph_cloner.h" +#include "pipeline/static_analysis/evaluator.h" + +namespace mindspore { +namespace abstract { +enum SpecializeStatusCode { + kSpecializeSuccess = 0, + kSpecializeFindUniqueArgvalDead = 1, // Dead Node + kSpecializeFindUniqueArgvalPoly = 2, // Poly Node + kSpecializeFailure = 0xFF +}; + +class FuncGraphSpecializer; + +// Specialize a func graph using analyzed abstract values. +class ProgramSpecializer { + public: + explicit ProgramSpecializer(const std::shared_ptr &engine) : engine_(engine) { + mng_ = engine_->func_graph_manager(); + } + ~ProgramSpecializer() = default; + // Run the program specializer on the topmost graph in the given context. + FuncGraphPtr Run(const FuncGraphPtr &fg, const AnalysisContextPtr &context); + const std::unordered_set &seen() const { return seen_; } + void AddSeen(const AnfNodePtr &node) { (void)seen_.insert(node); } + + std::shared_ptr GetFuncGraphSpecializer(const AnalysisContextPtr &context); + // Specialze one FuncGraph in a given context. + FuncGraphPtr SpecializeFuncGraph(const FuncGraphPtr &fg, const AnalysisContextPtr &context); + + std::shared_ptr engine() { return engine_; } + + private: + std::shared_ptr engine_; + std::unordered_set seen_; + FuncGraphManagerPtr mng_; + std::unordered_map, ContextHasher, ContextEqual> + specializations_; +}; + +class FuncGraphSpecializer : public std::enable_shared_from_this { + public: + FuncGraphSpecializer(ProgramSpecializer *const s, const FuncGraphPtr &fg, const AnalysisContextPtr &context); + virtual ~FuncGraphSpecializer() { + specializer_ = nullptr; + repl_node_ = nullptr; + } + void Run(); + FuncGraphPtr specialized_func_graph() { return specialized_func_graph_; } + + private: + ProgramSpecializer *specializer_; + FuncGraphPtr func_graph_; + FuncGraphPtr specialized_func_graph_; + AnalysisContextPtr context_; + std::shared_ptr parent_; + std::shared_ptr engine_; + ClonerPtr cloner_; + // ProcessNode-> [cloner_->CloneDisconnected] will clone AnfNode again. + // So, repl_node_ should pointer to GraphCloner->repl_node_ other than a copy of that. + std::unordered_map *repl_node_; + std::vector todo_; + std::unordered_set marked_; + std::unordered_map evalcaches_; + + void FirstPass(); + void SecondPass(); + void ProcessNode(const AnfNodePtr &node); + void ProcessCNode(const CNodePtr &new_node); + + AnfNodeConfigPtr MakeConfig(const AnfNodePtr &node); + inline void AddTodoItem(const AnfNodePtr &node) { todo_.push_back(node); } + // Get node replicated by Cloner. + AnfNodePtr GetReplicatedNode(const AnfNodePtr &node); + // Replicated node which is not used directly by a func graph, so it's not searchable from it's return node + // (disconnected). + AnfNodePtr ReplicateDisconnectedNode(const AnfNodePtr &node); + + // Build a value node if ival is constant and not any-value + AnfNodePtr BuildPossibleValueNode(const AnfNodePtr &origin_node, const AbstractBasePtr &ival); + // Build a replacable node for iconf->node; it may be a replicated forwared CNode in static analysis or just a + // replicated node. + AnfNodePtr BuildReplacedNode(const AnfNodeConfigPtr &conf); + // Build a specialized node from given argvals; + AnfNodePtr BuildSpecializedNode(const AnfNodePtr &node, const AbstractBasePtr &abs, + const AbstractBasePtrList &argvals); + AnfNodePtr BuildSpecializedNodeInner(const AbstractBasePtr &abs, const AbstractFunctionPtr &func, + const AbstractBasePtrList &args, SpecializeStatusCode *errcode); + + // Find the unique argument values which can be used to specialize a primitive or graph function. + SpecializeStatusCode FindUniqueArgvals(const AbstractFunctionPtr &fn, const EvaluatorPtr &eval, + const AbstractBasePtrList &argvals, + std::pair *result); + // Get cache, it may be eval's cache or cache built from broaded argument values. + const EvaluatorCacheMapPtr &GetEvalCache(const EvaluatorPtr &eval); + // Try to build unique argvals from the broaded arg vals if it is unique. + std::pair BuildFromBroadedArgsVal(const EvaluatorPtr &eval); +}; +} // namespace abstract +} // namespace mindspore +#endif // PIPELINE_STATIC_ANALYSIS_SPECIALIZE_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc new file mode 100644 index 00000000000..0bfba265db0 --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc @@ -0,0 +1,539 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/static_analysis.h" + +#include + +#include "pipeline/static_analysis/utils.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "utils/symbolic.h" +#include "ir/meta_tensor.h" +#include "ir/func_graph_cloner.h" +#include "./common.h" +#include "pipeline/parse/data_converter.h" +#include "debug/draw.h" +#include "pipeline/static_analysis/evaluator.h" +#include "debug/trace.h" + +namespace mindspore { +namespace abstract { +bool IsIntermediateAbstract(const AbstractBasePtr &arg_spec) { + if (dyn_cast(arg_spec)) { + auto v = arg_spec->GetValueTrack(); + if (v->isa()) { + return true; + } else { + return false; + } + } else { + return false; + } +} + +AbstractBasePtr IntermediateJoin(const AbstractBasePtr &arg1, const AbstractBasePtr &arg2) { + if (dyn_cast(arg1) && dyn_cast(arg2)) { + return arg1->Join(arg2); + } + return nullptr; +} + +void AnalysisCache::set_value(const AnfNodeConfigPtr &conf, const AbstractBasePtr &arg) { + MS_LOG(DEBUG) << "AnalysisCache set for NodeConfig: " << conf->node()->DebugString() + << ", Context: " << conf->context()->ToString() << ", Value: " << arg->ToString() + << ", Pointer: " << arg.get(); + cache_[conf] = arg; + + // Set intermediate abstract value. + if (IsIntermediateAbstract(arg)) { + if (conf->node()->intermediate_abstract() == nullptr) { + conf->node()->set_intermediate_abstract(arg); + MS_LOG(DEBUG) << "Set intermediate abstract: " << arg->ToString(); + } else { + auto old_spec = conf->node()->intermediate_abstract(); + auto joined_spec = IntermediateJoin(arg, old_spec); + conf->node()->set_intermediate_abstract(joined_spec); + MS_LOG(DEBUG) << "Set joined intermediate abstract:\nold_spec:\t\t" << old_spec->ToString() << "\nnew_spec:\t\t" + << arg->ToString() << "\njoined_spec:\t" + << (joined_spec != nullptr ? joined_spec->ToString() : "nullptr"); + } + } +} + +AbstractBasePtr AnalysisCache::GetValue(const AnfNodeConfigPtr &conf) { + auto value = cache_.find(conf); + if (value == cache_.end()) { + return nullptr; + } + return value->second; +} + +std::size_t AnfNodeConfigHasher::operator()(const AnfNodeConfigPtr conf) const { + MS_EXCEPTION_IF_NULL(conf); + MS_EXCEPTION_IF_NULL(conf->node()); + std::size_t hash_value = hash_combine(conf->node()->hash(), conf->context()->hash()); + if (conf->context() != nullptr && conf->context()->func_graph() != nullptr) { + MS_LOG(DEBUG) << "NodeConfgHasher Node: " << conf->node()->DebugString() + << ", Graph: " << conf->context()->func_graph()->ToString() << " ### , hash value: " << hash_value; + } else { + MS_LOG(DEBUG) << "NodeConfigHasher Node: " << conf->node()->DebugString() << " ### , hash value: " << hash_value; + } + return hash_value; +} + +bool AnfNodeConfigEqual::operator()(const AnfNodeConfigPtr lhs, const AnfNodeConfigPtr rhs) const { + if (lhs == nullptr || rhs == nullptr) { + return false; + } + if (lhs == rhs) { + return true; + } + return (*lhs == *rhs); +} + +AnalysisResult AnalysisEngine::Run(const FuncGraphPtr &func_graph, const AbstractBasePtrList &args_spec_list) { + ConfigPtrList args_conf_list; + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(args_conf_list), + [](const AbstractBasePtr &arg) -> ConfigPtr { return std::make_shared(arg); }); + MS_EXCEPTION_IF_NULL(func_graph_manager_); + func_graph_manager_->AddFuncGraph(func_graph); + + AnalysisContextPtr empty_context = AnalysisContext::DummyContext(); + + // Running the analyzer. + AnalysisContextPtr root_context = Run(func_graph, empty_context, args_conf_list); + MS_EXCEPTION_IF_NULL(root_context); + MS_EXCEPTION_IF_NULL(root_context->func_graph()); + AnfNodeConfigPtr output_conf = MakeConfig(root_context->func_graph()->get_return(), root_context); + MS_EXCEPTION_IF_NULL(func_graph); + MS_LOG(INFO) << "" << func_graph->ToString() << ": Run finished."; + + AnalysisResult result; + MS_EXCEPTION_IF_NULL(output_conf); + result.inferred = output_conf->GetEvaluatedValue(); + result.context = root_context; + return result; +} + +AnalysisContextPtr AnalysisEngine::Run(const FuncGraphPtr &func_graph, const AnalysisContextPtr &context, + const ConfigPtrList &args_conf_list) { + std::shared_ptr eval = std::make_shared(func_graph, context); + (void)eval->Run(shared_from_this(), args_conf_list, nullptr); + return eval->graph_context(); +} + +AbstractBasePtr AnalysisEngine::GetEvaluatedValue(const AnfNodeConfigPtr &conf) { + MS_EXCEPTION_IF_NULL(conf); + auto value = cache_.GetValue(conf); + if (value != nullptr) { + MS_LOG(DEBUG) << "Evaluate cache hit for NodeConfig: " << conf->ToString() << ", Value: " << value.get() << ", " + << value->ToString(); + return value; + } + + MS_LOG(DEBUG) << "Evaluate cache miss for NodeConfig: " << conf->ToString(); + value = Eval(conf); + if (value == nullptr) { + MS_LOG(EXCEPTION) << "Evaluate for NodeConfig " << conf->ToString() << " get nullptr"; + } + cache_.set_value(conf, value); + return value; +} + +AbstractBasePtr AnalysisEngine::Eval(const AnfNodeConfigPtr &conf) { + MS_EXCEPTION_IF_NULL(conf); + AnfNodePtr node = conf->node(); + AbstractBasePtr ret_abstract = nullptr; +#ifdef DEBUG + compute_conf_stack_.push_back(node); + std::ostringstream buffer; + buffer << "Compute Config Begin:"; + for (auto iter : compute_conf_stack_) { + buffer << " -> " << iter->DebugString(); + } + MS_LOG(DEBUG) << "" << buffer.str(); +#endif + MS_LOG(DEBUG) << "Begin Eval NodeConfig " << conf->ToString(); + MS_EXCEPTION_IF_NULL(node); + if (node->abstract() != nullptr) { + MS_LOG(DEBUG) << "Return old abstract: " << node->DebugString(); + ret_abstract = node->abstract(); + } else if (node->isa()) { + auto value_node = node->cast(); + ret_abstract = EvalValueNode(value_node, conf); + } else if (node->isa()) { + auto cnode = node->cast(); + trace::TraceInferCNodeEnter(conf); + ret_abstract = InferCNode(cnode, conf); + trace::TraceInferCNodeLeave(); + } else { + MS_LOG(EXCEPTION) << "Illegal AnfNode for inferring, " << node->DebugString() + << ". NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } + +#ifdef DEBUG + compute_conf_stack_.pop_back(); + if (ret_abstract == nullptr) { + MS_LOG(EXCEPTION) << "Compute Config failed, node: " << node->DebugString() + << " NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } +#endif + MS_LOG(DEBUG) << "End Eval NodeConfig " << conf->ToString() << ", res: " << ret_abstract->ToString(); + return ret_abstract; +} + +AbstractBasePtr AnalysisEngine::EvalValueNode(const ValueNodePtr &value_node, const AnfNodeConfigPtr &conf) { + MS_EXCEPTION_IF_NULL(conf); + MS_EXCEPTION_IF_NULL(value_node); + return ToAbstract(value_node->value(), conf->context(), conf); +} + +AbstractBasePtr AnalysisEngine::InferCNode(const CNodePtr &cnode, const AnfNodeConfigPtr &conf) { + MS_EXCEPTION_IF_NULL(conf); + MS_EXCEPTION_IF_NULL(cnode); + auto &inputs = cnode->inputs(); + if (inputs.empty()) { + MS_LOG(EXCEPTION) << "CNode->inputs() is empty, CNode: " << cnode->DebugString(); + } + + AnfNodePtr func_node = inputs[0]; + MS_EXCEPTION_IF_NULL(func_node); + MS_LOG(DEBUG) << "Current CNode function: " << func_node->DebugString(); + AnalysisContextPtr context = conf->context(); + AnfNodeConfigPtr func_conf = MakeConfig(func_node, context); + MS_EXCEPTION_IF_NULL(func_conf); + // Keep it in a local variable, otherwise smart pointer will free it. + AbstractBasePtr maybe_func = func_conf->GetEvaluatedValue(); + if (maybe_func == nullptr) { + MS_LOG(EXCEPTION) << "func_conf.GetEvaluatedValue() return null, func_conf: " << func_conf->ToString() + << " NodeInfo: " << trace::GetDebugInfo(cnode->debug_info()); + } + AbstractFunctionPtr func = dyn_cast(maybe_func); + if (func == nullptr) { + MS_LOG(EXCEPTION) << "func_conf.GetEvaluatedValue() return not AbstractFunction: " << maybe_func->ToString() + << ", func_conf: " << func_conf->ToString() + << " NodeInfo: " << trace::GetDebugInfo(cnode->debug_info()); + } + + ConfigPtrList args_conf_list; + // ignore the first node which is function name + for (std::size_t i = 1; i < inputs.size(); i++) { + const AnfNodePtr &node = inputs[i]; + args_conf_list.push_back(MakeConfig(node, context)); + MS_LOG(DEBUG) << "Current CNode args_conf_list[" << i << "] node: " << node->DebugString(); + } + std::vector infs; + + auto build_evaluator = [this, &infs, &cnode](const AbstractFuncAtomPtr &poss) { + auto evaluator = this->GetEvaluatorFor(poss); + evaluator->set_bound_node(cnode); + infs.push_back(evaluator); + }; + func->Visit(build_evaluator); + + return ExecuteEvaluators(infs, conf, args_conf_list); +} + +AbstractBasePtr AnalysisEngine::Execute(const AbstractFunctionPtr &func, const AbstractBasePtrList &args_spec_list) { + ConfigPtrList args_conf_list; + (void)std::transform(args_spec_list.begin(), args_spec_list.end(), std::back_inserter(args_conf_list), + [](const AbstractBasePtr &arg) -> ConfigPtr { return std::make_shared(arg); }); + std::vector infs; + MS_EXCEPTION_IF_NULL(func); + auto build_evaluator = [this, &infs](const AbstractFuncAtomPtr &poss) { + auto evaluator = this->GetEvaluatorFor(poss); + infs.push_back(evaluator); + }; + func->Visit(build_evaluator); + return ExecuteEvaluators(infs, nullptr, args_conf_list); +} + +void AnalysisEngine::ClearEvaluatorCache() { + for (std::pair element : constructors_) { + EvaluatorPtr evaluator = element.second; + MS_EXCEPTION_IF_NULL(evaluator); + MS_EXCEPTION_IF_NULL(evaluator->cache()); + evaluator->cache()->clear(); + } +} + +void AnalysisEngine::Clear() { + cache_.Clear(); + anfnode_config_map_.clear(); + eval_trace_.clear(); + constructors_.clear(); +} + +namespace { +EvaluatorPtr GetPrimEvaluator(const PrimitivePtr &prim, const AnalysisEnginePtr &engine) { + // Custom Primitive with python infer_shape, infer_type + EvaluatorPtr evaluator = nullptr; + MS_EXCEPTION_IF_NULL(prim); + if (prim->isa()) { + evaluator = std::make_shared(prim); + return evaluator; + } + if (prim->HasPyEvaluator()) { + auto prim_py = dyn_cast(prim); + if (prim_py != nullptr) { + evaluator = std::make_shared(prim_py); + } else { + MS_LOG(EXCEPTION) << "The primitive with python evaluator should be a python primitive."; + } + } else if (prim->isa() || prim->HasAttr()) { + // If a primitive may have attr, try to create a new evaluator. + StandardPrimitiveEvalImpl eval_impl = GetPrimitiveInferImpl(prim); + if (eval_impl != nullptr) { + std::shared_ptr standard_evaluator = + std::make_shared(prim, eval_impl); + evaluator = standard_evaluator; + } + } + if (evaluator == nullptr) { + if (engine == nullptr) { + // If engine is nullptr, get constructor from default. + const PrimEvaluatorMap &prim_evaluator_map = GetPrimEvaluatorConstructors(); + auto iter = prim_evaluator_map.find(prim); + if (iter == prim_evaluator_map.end()) { + evaluator = nullptr; + } else { + evaluator = iter->second; + } + } else { + // If engine is given, get constructor from engine resource. + const PrimEvaluatorMap &prim_evaluator_map = engine->PrimConstructors(); + auto iter = prim_evaluator_map.find(prim); + if (iter == prim_evaluator_map.end()) { + evaluator = nullptr; + } else { + evaluator = iter->second; + } + } + } + if (evaluator == nullptr) { + MS_LOG(EXCEPTION) << "The evaluator of the primitive is not defined (" << prim->name() << ")."; + } + return evaluator; +} +} // namespace + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &func) { + auto inf_pair = constructors_.find(func); + if (inf_pair != constructors_.end()) { + return inf_pair->second; + } + MS_EXCEPTION_IF_NULL(func); + auto primitive = func->prim(); + auto evaluator = GetPrimEvaluator(primitive, shared_from_this()); + constructors_[func] = evaluator; + return evaluator; +} + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &func) { + auto inf_pair = constructors_.find(func); + if (inf_pair != constructors_.end()) { + return inf_pair->second; + } + MS_EXCEPTION_IF_NULL(func); + std::shared_ptr func_graph_evaluator = + std::make_shared(func->func_graph(), func->context()); + constructors_[func] = func_graph_evaluator; + return func_graph_evaluator; +} + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &func) { + auto inf_pair = constructors_.find(func); + if (inf_pair != constructors_.end()) { + return inf_pair->second; + } + MS_EXCEPTION_IF_NULL(func); + std::shared_ptr evaluator = + std::make_shared(func->meta_func_graph(), func->context(), func->GetScope()); + constructors_[func] = evaluator; + return evaluator; +} + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &func) { + MS_EXCEPTION_IF_NULL(func); + AbstractFunctionPtr func_orig = func->fn(); + EvaluatorPtr evaluator_orig = GetEvaluatorFor(func_orig); + auto jevaluator = std::make_shared(evaluator_orig, func_orig); + return jevaluator; +} + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &func) { + MS_EXCEPTION_IF_NULL(func); + std::shared_ptr virtual_evaluator = + std::make_shared(func->args_spec_list(), func->output()); + return virtual_evaluator; +} + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &func) { + MS_EXCEPTION_IF_NULL(func); + AbstractFunctionPtr func_orig = func->fn(); + EvaluatorPtr evaluator_orig = GetEvaluatorFor(func_orig); + std::shared_ptr partial_evaluator = + std::make_shared(evaluator_orig, func->args()); + return partial_evaluator; +} + +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const std::shared_ptr &) { + MS_LOG(EXCEPTION) << "Should not be called "; +} + +// Forward to specific subclass of FunctionWrapper. +EvaluatorPtr AnalysisEngine::_GetEvaluatorFor(const AbstractFunctionPtr &func) { + MS_EXCEPTION_IF_NULL(func); + EvaluatorPtr evaluator = func->GetEvaluator(shared_from_this()); + return evaluator; +} + +EvaluatorPtr AnalysisEngine::GetEvaluatorFor(const AbstractFunctionPtr &func) { + MS_LOG(DEBUG) << "The func value: " << func->ToString(); + if (func->tracking_id() != nullptr) { + MS_LOG(DEBUG) << "The tracking_id: " << func->tracking_id()->DebugString(); + } + MS_EXCEPTION_IF_NULL(func); + if (func->tracking_id() == nullptr) { + EvaluatorPtr evaluator = _GetEvaluatorFor(func); + return evaluator; + } + auto inf_pair = constructors_.find(func); + if (inf_pair != constructors_.end()) { + return inf_pair->second; + } + + AbstractFunctionPtr func_generic = func->Copy(); + func_generic->set_tracking_id(nullptr); + EvaluatorPtr eval = _GetEvaluatorFor(func_generic); + auto tracked_eval = std::make_shared(eval); + constructors_[func] = tracked_eval; + + return tracked_eval; +} + +AbstractBasePtr AnalysisEngine::ExecuteEvaluators(const std::vector &evaluators, + const AnfNodeConfigPtr &out_conf, + const ConfigPtrList &args_conf_list) { + if (evaluators.size() == 1) { + EvaluatorPtr eval = evaluators[0]; + MS_EXCEPTION_IF_NULL(eval); + return eval->Run(shared_from_this(), args_conf_list, out_conf); + } + return ExecuteMultipleEvaluators(evaluators, out_conf, args_conf_list); +} + +AbstractBasePtr AnalysisEngine::ExecuteMultipleEvaluators(const std::vector &evaluators, + const AnfNodeConfigPtr &out_conf, + const ConfigPtrList &args_conf_list) { + AbstractBasePtrList out_specs; + AbstractBasePtrList args_spec_list; + (void)std::transform(args_conf_list.begin(), args_conf_list.end(), std::back_inserter(args_spec_list), + [](const ConfigPtr &conf) -> AbstractBasePtr { + MS_EXCEPTION_IF_NULL(conf); + return conf->GetEvaluatedValue(); + }); + for (auto eval : evaluators) { + auto fg_eval = eval->cast(); + if (fg_eval) { + auto undetermin_fgs = fg_eval->func_graph()->recursive_graphs(); + if (undetermin_fgs) { + for (auto undetermin_fg : *undetermin_fgs) { + MS_LOG(DEBUG) << "Set graph undetermin: " << undetermin_fg->ToString(); + // As the current evaluator has multiple possibles, all the func_graphs which + // are recursive with the current func_graph are undetermined in control flow. + undetermin_fg->set_flags(kFuncGraphFlagUndetermin, true); + } + } + } + + auto current_inf = std::make_pair(eval, args_spec_list); + // If current evaluator is under tracing, then skip current evaluator to avoid recursively inferring. + auto it = std::find(eval_trace_.begin(), eval_trace_.end(), current_inf); + if (it == eval_trace_.end()) { + eval_trace_.push_back(current_inf); + MS_EXCEPTION_IF_NULL(eval); + auto out_spec = eval->Run(shared_from_this(), args_conf_list, out_conf); + MS_EXCEPTION_IF_NULL(out_spec); + MS_LOG(DEBUG) << "Evaluator " << eval->ToString() << " return out_spec: " << out_spec->ToString(); + out_specs.push_back(out_spec); + eval_trace_.pop_back(); + } + } + if (out_specs.size() == 0) { + MS_LOG(EXCEPTION) << "There is an endless loop for evaluator."; + } + + if (out_specs.size() == 1) { + MS_EXCEPTION_IF_NULL(out_specs[0]); + // If only one result derived, then broaden it to avoid wrong constant propagation. + return out_specs[0]->Broaden(); + } + auto joined_spec = AbstractJoin(out_specs); + MS_EXCEPTION_IF_NULL(joined_spec); + MS_LOG(DEBUG) << "Multiple evaluators joined: " << joined_spec->ToString(); + return joined_spec; +} + +AbstractBasePtr AnfNodeConfig::GetEvaluatedValue() { + AnfNodeConfigPtr self = shared_from_base(); + return engine_.lock()->GetEvaluatedValue(self); +} + +AbstractBasePtr ToAbstract(const ValuePtr &value, const AnalysisContextPtr &context, const AnfNodeConfigPtr &conf) { + if (value->isa()) { + auto func_graph = value->cast(); + return func_graph->MakeAbstractClosure(context); + } + AnfNodePtr anf_node = nullptr; + if (conf != nullptr) { + anf_node = conf->node(); + } + if (value->isa()) { + auto meta_func_graph = value->cast(); + return meta_func_graph->MakeAbstractClosure(anf_node); + } + if (value->isa()) { + auto prim = value->cast(); + return prim->ToPrimAbstract(anf_node); + } + return value->ToAbstract(); +} + +AbstractBasePtr FromValueInside(const ValuePtr &value, bool broaden) { + AbstractBasePtr a = ToAbstract(value, nullptr, nullptr); + if (broaden) { + a = a->Broaden(); + } + return a; +} + +AbstractBasePtr InferOnePrim(const PrimitivePtr &primitive, const AbstractBasePtrList &arg_specs) { + auto evaluator = GetPrimEvaluator(primitive, nullptr); + MS_EXCEPTION_IF_NULL(evaluator); + if (!evaluator->isa()) { + MS_LOG(EXCEPTION) << "Prim " << primitive->ToString() << " should build a TrivialPrimEvaluator, but " + << evaluator->ToString(); + } + auto trivial_evaluator = dyn_cast(evaluator); + auto res_spec = trivial_evaluator->EvalPrim(nullptr, arg_specs); + return res_spec; +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/static_analysis.h b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.h new file mode 100644 index 00000000000..ef4f78e619e --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.h @@ -0,0 +1,245 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_STATIC_ANALYSIS_H_ +#define PIPELINE_STATIC_ANALYSIS_STATIC_ANALYSIS_H_ + +#include +#include +#include +#include +#include +#include + +#ifdef DEBUG +#include +#endif + +#include "utils/log_adapter.h" +#include "ir/anf.h" +#include "ir/primitive.h" +#include "pipeline/static_analysis/analysis_context.h" +#include "pipeline/static_analysis/abstract_function.h" +#include "pipeline/parse/parse.h" + +namespace mindspore { +namespace abstract { +// Superclass for AnfNodeConfig and VirtualConfig. +class Config : public Base { + public: + Config() = default; + ~Config() override = default; + MS_DECLARE_PARENT(Config, Base); + virtual AbstractBasePtr GetEvaluatedValue() = 0; +}; + +// Config will be stored in AnalysisCache +using ConfigPtr = std::shared_ptr; +using ConfigPtrList = std::vector; + +// Config to a certain node in a certain context. +class AnfNodeConfig : public Config { + public: + AnfNodeConfig(const AnalysisEnginePtr &engine, const AnfNodePtr &node, const AnalysisContextPtr &context) + : Config(), engine_(std::weak_ptr(engine)), node_(node) { + FuncGraphPtr fg; + if (IsValueNode(node)) { + auto v = node->cast(); + fg = v->value()->cast(); + } else { + fg = node->func_graph(); + } + context_ = nullptr; + if (context != nullptr) { + context_ = context->Filter(fg); + } + } + + ~AnfNodeConfig() override = default; + MS_DECLARE_PARENT(AnfNodeConfig, Config); + + AbstractBasePtr GetEvaluatedValue() override; + + AnalysisContextPtr context() const { return context_; } + + AnfNodePtr node() const { return node_; } + + AnalysisEnginePtr engine() const { return engine_.lock(); } + + // used by unordered_map; + bool operator==(const AnfNodeConfig &other) const { + // compare node with pointer, context with content; + // context should not be nullptr; + return (node_ == other.node_) && (*context_ == *other.context_); + } + + std::string ToString() const override { + std::ostringstream buffer; + buffer << "Node: " << node_->DebugString() << ", Context: " << context_->ToString(); + return buffer.str(); + } + + private: + // AnalysisEngine is global. + // As AnfNodeConfig is cached in AnalysisEngine.AnalysisCache, use + // weak_ptr to break Config cycle. + std::weak_ptr engine_; + AnfNodePtr node_; + AnalysisContextPtr context_; +}; + +using AnfNodeConfigPtr = std::shared_ptr; + +struct AnfNodeConfigHasher { + std::size_t operator()(const AnfNodeConfigPtr conf) const; +}; + +struct AnfNodeConfigEqual { + bool operator()(const AnfNodeConfigPtr lhs, const AnfNodeConfigPtr rhs) const; +}; + +class VirtualConfig : public Config { + public: + explicit VirtualConfig(const AbstractBasePtr &abstract) : Config(), abstract_(abstract) {} + + ~VirtualConfig() override = default; + MS_DECLARE_PARENT(VirtualConfig, Config); + AbstractBasePtr GetEvaluatedValue() override { return abstract_; } + + private: + AbstractBasePtr abstract_; +}; + +// AnalysisCache +class AnalysisCache { + public: + AnalysisCache() = default; + ~AnalysisCache() = default; + void Clear() { cache_.clear(); } + void set_value(const AnfNodeConfigPtr &conf, const AbstractBasePtr &arg); + AbstractBasePtr GetValue(const AnfNodeConfigPtr &conf); + + private: + std::unordered_map cache_; +}; + +using PrimEvaluatorMap = std::unordered_map; +using AnfNodeConfigMap = + std::unordered_map; + +struct AnalysisResult { + AbstractBasePtr inferred; + AnalysisContextPtr context; +}; + +class AnalysisEngine : public std::enable_shared_from_this { + public: + AnalysisEngine(const PrimEvaluatorMap &prim_evaluator_map, const FuncGraphManagerPtr &func_graph_manager) + : cache_(AnalysisCache()), prim_constructors_(prim_evaluator_map), func_graph_manager_(func_graph_manager) {} + ~AnalysisEngine() = default; + + // func_graph: The func_graph to analyze. + // args_spec_list: The abstracted arguments for the func_graph. Must be a tuple of AbstractBase. + AnalysisResult Run(const FuncGraphPtr &func_graph, const AbstractBasePtrList &args_spec_list); + AbstractBasePtr GetEvaluatedValue(const AnfNodeConfigPtr &conf); + // Return the Evaluator for the given function. + EvaluatorPtr GetEvaluatorFor(const AbstractFunctionPtr &fn); + + AbstractBasePtr EvalValueNode(const ValueNodePtr &value_node, const AnfNodeConfigPtr &conf); + AbstractBasePtr InferCNode(const CNodePtr &cnode, const AnfNodeConfigPtr &conf); + // Infer the result of fn(args). + AbstractBasePtr Execute(const AbstractFunctionPtr &fn, const AbstractBasePtrList &args_spec_list); + void Clear(); + void ClearEvaluatorCache(); + AnalysisCache &cache() { return cache_; } + AnfNodeConfigPtr MakeConfig(const AnfNodePtr &node, const AnalysisContextPtr &context) { + return std::make_shared(shared_from_this(), node, context); + } + // Overloaded function. + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &fn); + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &fn); + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &fn); + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &fn); + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &fn); + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &); + EvaluatorPtr _GetEvaluatorFor(const std::shared_ptr &fn); + + FuncGraphManagerPtr func_graph_manager() { return func_graph_manager_; } + const AnfNodeConfigMap &anfnode_config_map() const { return anfnode_config_map_; } + + // Set the analysis result for orig to the result for new. + // This sets an entry in anfnode_config_map from orig to new. + AbstractBasePtr ForwardConfig(const AnfNodeConfigPtr &orig_conf, const AnfNodeConfigPtr new_conf) { + // Use anfnode_config_map_[orig_conf] = new_conf will require AnfNodeConfig provide copy constructor. + (void)anfnode_config_map_.emplace(orig_conf, new_conf); + MS_LOG(DEBUG) << "Forward orig_conf: " << orig_conf->node()->DebugString() + << ", to new_conf: " << new_conf->node()->DebugString(); + return GetEvaluatedValue(new_conf); + } + const PrimEvaluatorMap &PrimConstructors() const { return prim_constructors_; } + + AnalysisCache cache_; + + private: + const PrimEvaluatorMap &prim_constructors_; + FuncGraphManagerPtr func_graph_manager_; + std::unordered_map constructors_; + AnfNodeConfigMap anfnode_config_map_; + // Use a list to trace multiple evaluators. + std::list> eval_trace_; + + AnalysisContextPtr Run(const FuncGraphPtr &func_graph, const AnalysisContextPtr &context, + const ConfigPtrList &args_conf_list); + AbstractBasePtr Eval(const AnfNodeConfigPtr &conf); + EvaluatorPtr _GetEvaluatorFor(const AbstractFunctionPtr &fn); + AbstractBasePtr ExecuteEvaluators(const std::vector &evaluators, const AnfNodeConfigPtr &out_conf, + const ConfigPtrList &args_conf_list); + AbstractBasePtr ExecuteMultipleEvaluators(const std::vector &evaluators, + const AnfNodeConfigPtr &out_conf, const ConfigPtrList &args_conf_list); + +#ifdef DEBUG + std::vector compute_conf_stack_; +#endif +}; + +// Translate the value to an abstract value. +// Arguments: +// value: The value to convert. +// context: The context in which the value was found, used if the value is a Graph. +// conf: The Config to the valuenode we are converting, if there is one, +// so that we can generate a tracking_id. +AbstractBasePtr ToAbstract(const ValuePtr &value, const AnalysisContextPtr &context = nullptr, + const AnfNodeConfigPtr &conf = nullptr); + +// Convert a value to an abstract value. +// Arguments: +// v: The value to convert. +// broaden: If True, concrete values will be made more abstract, so e.g. +// the value 1234 would become ANYTHING. +AbstractBasePtr FromValueInside(const ValuePtr &value, bool broaden = false); + +template +AbstractBasePtr FromValue(const T &value, bool broaden = false) { + return FromValueInside(MakeValue(value), broaden); +} + +AbstractBasePtr InferOnePrim(const PrimitivePtr &p, const AbstractBasePtrList &arg_specs); +} // namespace abstract +} // namespace mindspore + +#endif // PIPELINE_STATIC_ANALYSIS_STATIC_ANALYSIS_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/utils.cc b/mindspore/ccsrc/pipeline/static_analysis/utils.cc new file mode 100644 index 00000000000..997a089301f --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/utils.cc @@ -0,0 +1,201 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/utils.h" + +#include +#include +#include +#include "utils/symbolic.h" +#include "pipeline/static_analysis/param_validator.h" + +namespace mindspore { +namespace abstract { +ValuePtr ValueJoin(const ValuePtr &value1, const ValuePtr &value2) { + MS_EXCEPTION_IF_NULL(value1); + MS_EXCEPTION_IF_NULL(value2); + if (*value1 == *value2) { + return value1; + } + return kAnyValue; +} + +TypePtr TypeJoin(const TypePtr &type1, const TypePtr &type2) { + MS_EXCEPTION_IF_NULL(type1); + MS_EXCEPTION_IF_NULL(type2); + if (*type1 == *type2) { + return type1; + } + return kAnyType; +} + +ShapePtr ShapeJoin(const ShapePtr &shape1, const ShapePtr &shape2) { + MS_EXCEPTION_IF_NULL(shape1); + MS_EXCEPTION_IF_NULL(shape2); + if (*shape1 == *shape2) { + return shape1; + } + if (shape1->shape().size() != shape2->shape().size()) { + MS_LOG(WARNING) << "Unsupported shape join. shape1 = " << shape1->ToString() << ", shape2 = " << shape2->ToString(); + return shape1; + } + std::vector dims; + dims.resize(shape1->shape().size()); + for (std::size_t i = 0; i < shape1->shape().size(); i++) { + if (shape1->shape()[i] == shape2->shape()[i]) { + dims[i] = shape1->shape()[i]; + } else { + dims[i] = Shape::SHP_ANY; + } + } + return std::make_shared(dims); +} + +AbstractBasePtr AbstractJoin(const AbstractBasePtrList &args_spec_list) { + if (args_spec_list.size() < 1) { + MS_LOG(EXCEPTION) << "AbstractJoin requires at least 1 params, while the input size is " << args_spec_list.size() + << "."; + } + AbstractBasePtr arg_spec_tmp = args_spec_list[0]; + MS_EXCEPTION_IF_NULL(arg_spec_tmp); + for (auto arg_spec : args_spec_list) { + arg_spec_tmp = arg_spec_tmp->Join(arg_spec); + MS_EXCEPTION_IF_NULL(arg_spec_tmp); + } + return arg_spec_tmp; +} + +AbstractBasePtrList AbstractJoin(const AbstractBasePtrList &spec1, const AbstractBasePtrList &spec2) { + if (spec1.size() != spec2.size()) { + MS_LOG(EXCEPTION) << "Join failed as list don't have the same size. spec1: " << ::mindspore::ToString(spec1) + << ", spec2: " << ::mindspore::ToString(spec2); + } + AbstractBasePtrList joined_list; + bool changes = false; + for (std::size_t i = 0; i < spec1.size(); i++) { + auto joined_elem = spec1[i]->Join(spec2[i]); + if (joined_elem != spec1[i]) { + changes = true; + } + joined_list.push_back(joined_elem); + } + if (!changes) { + return spec1; + } + return joined_list; +} + +AbstractBasePtr SensitivityTransform(const AbstractBasePtr &spec) { + AbstractFunctionPtr f_spec = dyn_cast(spec); + if (f_spec != nullptr) { + return std::make_shared(kAnyValue, std::make_shared()); + } + return spec->Clone(); +} + +namespace { +// Join all types in args_type_list; +TypePtr TypeJoin(const TypePtrList &args_type_list) { + if (args_type_list.empty()) { + MS_LOG(EXCEPTION) << "args_type_list is empty"; + } + + TypePtr type_tmp = args_type_list[0]; + for (std::size_t i = 1; i < args_type_list.size(); i++) { + type_tmp = abstract::TypeJoin(type_tmp, args_type_list[i]); + } + return type_tmp; +} +} // namespace + +bool CheckType(const TypePtr &expected_type, const TypePtr &x) { + // As x and predicate both are mindspore type staticly, here we only to judge whether + // x is predicate or is a subclass of predicate. + return IsIdentidityOrSubclass(x, expected_type); +} + +TypePtr CheckTypeList(const TypePtr &predicate, const TypePtrList &args_type_list) { + MS_EXCEPTION_IF_NULL(predicate); + for (auto arg_type : args_type_list) { + MS_EXCEPTION_IF_NULL(arg_type); + if (!CheckType(predicate, arg_type)) { + MS_LOG(EXCEPTION) << "The expected is " << predicate->ToString() << ", not " << arg_type->ToString(); + } + } + return TypeJoin(args_type_list); +} + +int GetPositiveAxis(int axis_value, size_t increment) { + if (axis_value < 0) { + axis_value = axis_value + SizeToInt(increment); + } + + if (axis_value < 0) { + MS_LOG(EXCEPTION) << "axis_value should not still <0"; + } + + return axis_value; +} + +// Return if two shapes can be broadcast. +// Broadcast shape is placed in broadcast_output_shape. +std::vector RealBroadcast(const std::string &op, std::vector x_shape, std::vector y_shape) { + std::reverse(x_shape.begin(), x_shape.end()); + std::reverse(y_shape.begin(), y_shape.end()); + // Fill a placeholder value 1 which will be replaced later. + size_t std_len = x_shape.size() > y_shape.size() ? x_shape.size() : y_shape.size(); + y_shape.resize(std_len, 1); + x_shape.resize(std_len, 1); + + std::vector broadcast_shape; + for (size_t i = 0; i < std_len; i++) { + int x_i = x_shape[i]; // i-th dimension of x + int y_i = y_shape[i]; // i-th dimension of y + int output_i = 0; // i-th dimension of the output + if (x_i == y_i) { + output_i = x_i; + } else if (x_i == 1) { + output_i = y_i; + } else if (y_i == 1) { + output_i = x_i; + } else { + MS_LOG(EXCEPTION) + << "" << op + << " evaluator the shape of first tensor and the shape of second tensor do not meet the broadcasting " + "requirements"; + } + broadcast_shape.push_back(output_i); + } + std::reverse(broadcast_shape.begin(), broadcast_shape.end()); + return broadcast_shape; +} + +ShapePtr GetBroadcastShape(const std::string &op, const AbstractTensorPtr &tensor_x, + const AbstractTensorPtr &tensor_y) { + mindspore::abstract::ShapePtr tensor_x_shape = tensor_x->shape(); + mindspore::abstract::ShapePtr tensor_y_shape = tensor_y->shape(); + // if is the same shape ,just return the x_shape + if (*tensor_x_shape == *tensor_y_shape) { + return tensor_x_shape; + } + auto x_shape = tensor_x_shape->shape(); + auto y_shape = tensor_y_shape->shape(); + return std::make_shared(RealBroadcast(op, x_shape, y_shape)); +} +} // namespace abstract +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/utils.h b/mindspore/ccsrc/pipeline/static_analysis/utils.h new file mode 100644 index 00000000000..6a709ea99cd --- /dev/null +++ b/mindspore/ccsrc/pipeline/static_analysis/utils.h @@ -0,0 +1,57 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PIPELINE_STATIC_ANALYSIS_UTILS_H_ +#define PIPELINE_STATIC_ANALYSIS_UTILS_H_ + +#include +#include +#include +#include +#include "pipeline/static_analysis/abstract_value.h" +#include "utils/any.h" +#include "utils/misc.h" +#include "utils/convert_utils.h" +#include "ir/primitive.h" + +namespace mindspore { +namespace abstract { +ValuePtr ValueJoin(const ValuePtr &value1, const ValuePtr &value2); +TypePtr TypeJoin(const TypePtr &type1, const TypePtr &type2); +ShapePtr ShapeJoin(const ShapePtr &shape1, const ShapePtr &shape2); + +AbstractBasePtr AbstractJoin(const AbstractBasePtrList &args_spec_list); +AbstractBasePtrList AbstractJoin(const AbstractBasePtrList &spec1, const AbstractBasePtrList &spec2); + +// Return an abstract value for the sensitivity of x. +// The sensitivity of a function is an Env +// The sensitivity of J(x) is x +// else self.Clone; +AbstractBasePtr SensitivityTransform(const AbstractBasePtr &spec); + +TypePtr CheckTypeList(const TypePtr &predicate, const TypePtrList &args_type_list); + +bool CheckType(const TypePtr &expected_type, const TypePtr &x); + +int GetPositiveAxis(int axis_value, size_t increment); + +// Get broadcasted shape for binary element-wise operation +ShapePtr GetBroadcastShape(const std::string &op, const AbstractTensorPtr &tensor_x, const AbstractTensorPtr &tensor_y); +} // namespace abstract +} // namespace mindspore +#endif // PIPELINE_STATIC_ANALYSIS_UTILS_H_ diff --git a/mindspore/ccsrc/pipeline/validator.cc b/mindspore/ccsrc/pipeline/validator.cc new file mode 100644 index 00000000000..0fe32188131 --- /dev/null +++ b/mindspore/ccsrc/pipeline/validator.cc @@ -0,0 +1,118 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/validator.h" + +#include +#include + +#include "ir/manager.h" +#include "ir/dtype.h" +#include "./common.h" +#include "pipeline/static_analysis/prim.h" + +namespace mindspore { +namespace validator { +using mindspore::abstract::AbstractBase; +using mindspore::abstract::AbstractClass; +using mindspore::abstract::AbstractError; +using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractJTagged; +using mindspore::abstract::AbstractList; +using mindspore::abstract::AbstractScalar; +using mindspore::abstract::AbstractTensor; +using mindspore::abstract::AbstractTuple; +using mindspore::abstract::AbstractType; + +void ValidateOperation(const AnfNodePtr& node) { + if (!IsValueNode(node)) { + return; + } + + // Primitive must in whitelist + PrimitivePtr prim = GetValueNode(node); + if (abstract::IsInWhiteList(prim)) { + return; + } + if (prim->HasPyEvaluator()) { + MS_LOG(DEBUG) << "Primitive " << prim->name() << " has python evaluator."; + return; + } + if (prim->name() == "fake_bprop") { + MS_LOG(EXCEPTION) << "Illegal primitive: " << GetValue(prim->GetAttr("info")); + } + + MS_LOG(EXCEPTION) << "Illegal primitive: " << prim->name(); +} + +void ValidateAbstract(const AnfNodePtr& node) { + if (node == nullptr) { + MS_LOG(WARNING) << "Node to validate is invalid"; + return; + } + AbstractBasePtr ptrBase = node->abstract(); + if (ptrBase == nullptr) { + MS_LOG(WARNING) << "Abstract is null in node: " << node->DebugString(); + return; + } + if (ptrBase->isa() || ptrBase->isa()) { + // Validate a type. + MS_LOG(EXCEPTION) << "Illegal type in the graph: " << ptrBase->ToString(); + } + if (ptrBase->isa()) { + TypePtr ptrType = ptrBase->GetTypeTrack(); + MS_EXCEPTION_IF_NULL(ptrType); + if (ptrType->isa() || ptrType->isa()) { + // only send string in external + if (!IsValueNode(node)) { + // Validate a type. + MS_LOG(EXCEPTION) << "Illegal type in the graph: " << ptrBase->ToString(); + } + } + return; + } + if (ptrBase->isa()) { + // NOTICE: validate dead code? + MS_LOG(WARNING) << "AbstractError in the graph: " << ptrBase->ToString(); + return; + } + + if (ptrBase->isa() || ptrBase->isa() || ptrBase->isa() || + ptrBase->isa() || ptrBase->isa() || ptrBase->isa()) { + return; + } + + if (ptrBase->isa()) { + return; + } + + // Other types show exception + MS_LOG(EXCEPTION) << "Illegal type in the graph: " << ptrBase->ToString(); +} + +void Validate(const FuncGraphPtr& fg) { + FuncGraphManagerPtr mgr = Manage(fg, false); + MS_EXCEPTION_IF_NULL(mgr); + AnfNodeSet& all_nodes = mgr->all_nodes(); + for (const auto& anf_node : all_nodes) { + ValidateOperation(anf_node); + ValidateAbstract(anf_node); + } +} +} // namespace validator +} // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/validator.h b/mindspore/ccsrc/pipeline/validator.h new file mode 100644 index 00000000000..9944078e6c7 --- /dev/null +++ b/mindspore/ccsrc/pipeline/validator.h @@ -0,0 +1,38 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PIPELINE_VALIDATOR_H_ +#define MINDSPORE_CCSRC_PIPELINE_VALIDATOR_H_ + +#include +#include +#include +#include +#include "operator/ops.h" +#include "ir/anf.h" +#include "utils/misc.h" + +namespace mindspore { +namespace validator { +void Validate(const FuncGraphPtr& func_graph); +void ValidateAbstract(const AnfNodePtr& node); +void ValidateOperation(const AnfNodePtr& node); +} // namespace validator +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PIPELINE_VALIDATOR_H__ diff --git a/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc new file mode 100644 index 00000000000..9883443910f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc @@ -0,0 +1,252 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ascend_backend_optimization.h" +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fission/bn_split.h" +#include "pre_activate/ascend/ir_fission/bn_grad_split.h" +#include "pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.h" +#include "pre_activate/ascend/ir_fission/layer_norm_grad_split.h" +#include "pre_activate/ascend/ir_fusion/allreduce_fusion.h" +#include "pre_activate/ascend/ir_fusion/square_sum_fusion.h" +#include "pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.h" +#include "pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.h" +#include "pre_activate/ascend/ir_fusion/clip_by_value_fusion.h" +#include "pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_rule.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_right_rule.h" +#include "pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.h" +#include "pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.h" +#include "pre_activate/ascend/ir_fusion/reshape_transpose_fusion.h" +#include "pre_activate/ascend/ir_fusion/transpose_reshape_fusion.h" +#include "pre_activate/ascend/ir_fusion/adam_apply_one_fusion.h" +#include "pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.h" +#include "pre_activate/ascend/ir_fusion/transpose_transdata_fusion.h" +#include "pre_activate/ascend/ir_fusion/transdata_split.h" +#include "pre_activate/ascend/ir_fission/topk_split.h" +#include "pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.h" +#include "pre_activate/ascend/ir_fusion/mul_add_fusion.h" +#include "pre_activate/ascend/ir_fusion/mul_addn_fusion.h" +#include "pre_activate/ascend/format_type/insert_trans_op.h" +#include "pre_activate/pass/getitem_tuple.h" +#include "pre_activate/pass/optimize_dependence.h" +#include "pre_activate/pass/erase_visit_attr.h" +#include "pre_activate/ascend/format_type/insert_cast.h" +#include "pre_activate/pass/eliminate_redundant_op.h" +#include "pre_activate/pass/common_subexpression_elimination.h" +#include "pre_activate/ascend/format_type/merge_cast_to_op.h" +#include "pre_activate/ascend/format_type/check_consistency.h" +#include "pre_activate/ascend/buffer_fusion/buffer_fusion.h" +#include "pre_activate/ascend/format_type/deal_ref_trans_and_cast.h" +#include "pre_activate/ascend/ir_fission/add_memcpy_async.h" +#include "pre_activate/ascend/format_type/insert_cast_for_runop.h" +#include "pre_activate/ascend/format_type/insert_transdata_for_runop.h" +#include "utils/context/ms_context.h" +#include "debug/anf_ir_dump.h" +#include "debug/anf_ir_utils.h" + +namespace mindspore { +namespace opt { +void RunOpAscendDataLayout(const std::shared_ptr &kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto optimizer = std::make_shared(); + auto data_layout_pm = std::make_shared("pynative_transop_pm"); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + optimizer->AddPassManager(data_layout_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void RunOpAscendMixPrecision(const std::shared_ptr &kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto optimizer = std::make_shared(); + auto mixed_precision_pm = std::make_shared("pynative_transop_pm"); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + optimizer->AddPassManager(mixed_precision_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void AscendDataLayout(const std::shared_ptr &kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto optimizer = std::make_shared(); + auto data_layout_pm = std::make_shared("transop_pm"); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + data_layout_pm->AddPass(std::make_shared()); + optimizer->AddPassManager(data_layout_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void AscendMixPrecision(const std::shared_ptr &kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto optimizer = std::make_shared(); + auto mixed_precision_pm = std::make_shared("cast_pm"); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + mixed_precision_pm->AddPass(std::make_shared()); + optimizer->AddPassManager(mixed_precision_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void AscendBackendIRFusionOptimization(const std::shared_ptr &kernel_graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->save_graphs_flag(); + auto save_graphs_path = context_ptr->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_d_ir_fusion_before.ir"; + DumpIR(file_path, kernel_graph); + DumpIRProto(kernel_graph, "before_hwopt"); + } + auto optimizer = std::make_shared(); + auto ir_fusion_pm = std::make_shared("ir_fusion_pm"); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + if (context_ptr->ir_fusion_flag()) { + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + } + optimizer->AddPassManager(ir_fusion_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_d_ir_fusion_after.ir"; + DumpIR(file_path, kernel_graph); + } +} + +void RunOpAscendBackendIRFusionOptimization(const std::shared_ptr &kernel_graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (!context_ptr->ir_fusion_flag()) { + MS_LOG(INFO) << "IRFusion is not enable, skip"; + return; + } + bool save_graphs = context_ptr->save_graphs_flag(); + auto save_graphs_path = context_ptr->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_d_ir_fusion_before.ir"; + DumpIR(file_path, kernel_graph); + } + auto optimizer = std::make_shared(); + auto ir_fusion_pm = std::make_shared("ir_fusion_pm"); + ir_fusion_pm->AddPass(std::make_shared()); + + optimizer->AddPassManager(ir_fusion_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_d_ir_fusion_after.ir"; + DumpIR(file_path, kernel_graph); + } +} + +void AscendBackendOptimization(const std::shared_ptr &kernel_graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->save_graphs_flag(); + auto save_graphs_path = context_ptr->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_d_before.ir"; + DumpIR(file_path, kernel_graph); + } + // data layout optimization + AscendDataLayout(kernel_graph); + // mixed precision optimization + AscendMixPrecision(kernel_graph); + // buffer fusion + // other optimization + auto optimizer = std::make_shared(); + auto other_pm = std::make_shared("other_pm"); + other_pm->AddPass(std::make_shared()); + other_pm->AddPass(std::make_shared()); + other_pm->AddPass(std::make_shared()); + other_pm->AddPass(std::make_shared()); + other_pm->AddPass(std::make_shared()); + optimizer->AddPassManager(other_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_d_end.ir"; + DumpIR(file_path, kernel_graph); + DumpIRProto(kernel_graph, "after_hwopt"); + } +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.h b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.h new file mode 100644 index 00000000000..fcd9c15c58a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_ASCEND_BACKEND_OPTIMIZATION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_ASCEND_BACKEND_OPTIMIZATION_H_ +#include +#include "session/kernel_graph.h" +namespace mindspore { +namespace opt { +void RunOpAscendDataLayout(const std::shared_ptr &kernel_graph); +void RunOpAscendMixPrecision(const std::shared_ptr &kernel_graph); +void RunOpAscendBackendIRFusionOptimization(const std::shared_ptr &kernel_graph); +void AscendDataLayout(const std::shared_ptr &kernel_graph); +void AscendMixPrecision(const std::shared_ptr &kernel_graph); +void AscendBackendOptimization(const std::shared_ptr &kernel_graph); +void AscendBackendIRFusionOptimization(const std::shared_ptr &kernel_graph); +void RunOpAscendBackendIRFusionOptimization(const std::shared_ptr &kernel_graph); +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_ASCEND_BACKEND_OPTIMIZATION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ascend_helper.cc b/mindspore/ccsrc/pre_activate/ascend/ascend_helper.cc new file mode 100644 index 00000000000..6fff8ccd2a8 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ascend_helper.cc @@ -0,0 +1,427 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ascend_helper.h" +#include +#include "common/trans.h" +#include "common/utils.h" +#include "device/kernel_info.h" +#include "kernel/oplib/oplib.h" +#include "operator/ops.h" +#include "session/anf_runtime_algorithm.h" +#include "session/kernel_graph.h" +#include "utils/context/ms_context.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +namespace { +kernel::KernelBuildInfoPtr CreateKernelBuildInfo(const std::string &input_format, const std::string &output_format, + const AnfNodePtr &node, const kernel::KernelBuildInfo ori_build_info) { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({input_format}); + builder.SetOutputsFormat({output_format}); + builder.SetInputsDeviceType({ori_build_info.GetInputDeviceType(0)}); + builder.SetOutputsDeviceType({ori_build_info.GetOutputDeviceType(0)}); + builder.SetKernelType(ori_build_info.kernel_type()); + builder.SetFusionType(ori_build_info.fusion_type()); + builder.SetProcessor(ori_build_info.processor()); + return builder.Build(); +} + +CNodePtr NewTransOpNode(const FuncGraphPtr &func_graph, const AnfNodePtr &input, const KernelSelectPtr &kernel_select, + const bool need_padding, const std::string &op_name) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(input); + std::vector trans_inputs; + auto prim = std::make_shared(op_name); + trans_inputs.push_back(NewValueNode(prim)); + trans_inputs.push_back(input); + CNodePtr trans_node = func_graph->NewCNode(trans_inputs); + MS_EXCEPTION_IF_NULL(trans_node); + if (need_padding) { + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(input, 0)}, + {trans::TransShapeTo4d(AnfAlgo::GetOutputInferShape(input, 0))}, + trans_node.get()); + } else { + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(input, 0)}, + {AnfAlgo::GetOutputInferShape(input, 0)}, trans_node.get()); + } + // special handle for ut + if (trans_node->kernel_info() == nullptr) { + auto kernel_info = std::make_shared(); + trans_node->set_kernel_info(kernel_info); + } + MS_EXCEPTION_IF_NULL(kernel_select); + kernel_select->SelectKernel(trans_node); + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), trans_node); + MS_EXCEPTION_IF_NULL(trans_node); + trans_node->set_scope(input->scope()); + return trans_node; +} + +AnfNodePtr CreateReshapeNode(const FuncGraphPtr &func_graph, const AnfNodePtr &input_node, + const KernelSelectPtr &kernel_select, const std::vector &dst_shape) { + std::vector trans_inputs; + auto prim = std::make_shared(prim::kPrimReshape->name()); + trans_inputs.emplace_back(NewValueNode(prim)); + trans_inputs.emplace_back(input_node); + auto reshape = func_graph->NewCNode(trans_inputs); + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(input_node, 0)}, {dst_shape}, reshape.get()); + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), reshape); + AnfAlgo::SetNodeAttr(kAttrShape, MakeValue(dst_shape), reshape); + reshape->set_scope(input_node->scope()); + kernel_select->SelectKernel(reshape); + return reshape; +} + +AnfNodePtr GetTransInputNodePtr(const FuncGraphPtr &func_graph, const CNodePtr &node, size_t index, + const KernelSelectPtr &kernel_select) { + MS_EXCEPTION_IF_NULL(node); + bool padding_flag = false; + auto input_node = AnfAlgo::GetInputNode(node, index); + if (!AnfAlgo::IsFeatureMapInput(node, index)) { + input_node = InsertTransOpForOutput(func_graph, input_node, kernel_select); + MS_EXCEPTION_IF_NULL(input_node); + AnfAlgo::SetNodeInput(node, input_node, index); + } + if (AnfAlgo::GetInputFormat(node, index) == kOpFormat_NC1KHKWHWC0) { + MS_LOG(EXCEPTION) << "got the format " << AnfAlgo::GetInputFormat(node, index) + << "when inserting the transdata node " << node->DebugString(); + } + std::vector origin_shape = AnfAlgo::GetPrevNodeOutputInferShape(node, index); + std::string origin_format = kOpFormat_DEFAULT; + std::string dest_format = AnfAlgo::GetInputFormat(node, index); + if (dest_format == kOpFormat_C1HWNCoC0) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_input = AddTransOpNodeToGraph(func_graph, node, kernel_select, index, padding_flag, + origin_format, dest_format, kTransDataOpName, true); + MS_EXCEPTION_IF_NULL(replace_input); + return replace_input; + } + if (dest_format == kOpFormat_NC1HWC0 && origin_shape.size() > 1) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_input = AddTransOpNodeToGraph(func_graph, node, kernel_select, index, padding_flag, + origin_format, dest_format, kTransDataOpName, true); + MS_EXCEPTION_IF_NULL(replace_input); + MS_LOG(DEBUG) << "Inserted Translate45, index: " << index; + return replace_input; + } else if (dest_format == kOpFormat_FRAC_NZ) { + AnfNodePtr replace_input = AddTransOpNodeToGraph(func_graph, node, kernel_select, index, padding_flag, + origin_format, dest_format, kTransDataOpName, true); + MS_EXCEPTION_IF_NULL(replace_input); + MS_LOG(DEBUG) << "inserted translate " << AnfAlgo::GetInputFormat(node, index) << " To default, index: " << index; + return replace_input; + } else if (dest_format == kOpFormat_FRAC_Z && !origin_shape.empty()) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_input = AddTransOpNodeToGraph(func_graph, node, kernel_select, index, padding_flag, + origin_format, dest_format, kTransDataOpName, true); + MS_EXCEPTION_IF_NULL(replace_input); + MS_LOG(DEBUG) << "Inserted Translate45, index: " << index; + return replace_input; + } + return input_node; +} + +AnfNodePtr InsertTransOpForSingleOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select) { + MS_EXCEPTION_IF_NULL(node); + bool padding_flag = false; + std::string output_format; + std::vector origin_shape; + if (!AnfAlgo::IsRealKernel(node)) { + output_format = AnfAlgo::GetPrevNodeOutputFormat(node, 0); + origin_shape = AnfAlgo::GetPrevNodeOutputInferShape(node, 0); + } else { + output_format = AnfAlgo::GetOutputFormat(node, 0); + origin_shape = AnfAlgo::GetOutputInferShape(node, 0); + } + if (output_format == kOpFormat_NC1KHKWHWC0) { + MS_LOG(EXCEPTION) << "got the hw format " << output_format << "when insert the transdata node " + << node->DebugString(); + } + std::string origin_format = output_format; + std::string dest_format = kOpFormat_DEFAULT; + if (output_format == kOpFormat_C1HWNCoC0) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_input = AddTransOpNodeToGraph(func_graph, node, kernel_select, 0, padding_flag, origin_format, + dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_input); + return replace_input; + } + if (output_format == kOpFormat_NC1HWC0 && origin_shape.size() > 1) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_output = AddTransOpNodeToGraph(func_graph, node, kernel_select, 0, padding_flag, origin_format, + dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_output); + MS_LOG(DEBUG) << "Inserted Trans54"; + return replace_output; + } else if (output_format == kOpFormat_FRAC_NZ) { + AnfNodePtr replace_output = AddTransOpNodeToGraph(func_graph, node, kernel_select, 0, padding_flag, origin_format, + dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_output); + MS_LOG(DEBUG) << "Inserted Translate " << output_format << " To default, index: 0"; + return replace_output; + } else if (output_format == kOpFormat_FRAC_Z && !origin_shape.empty()) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_output = AddTransOpNodeToGraph(func_graph, node, kernel_select, 0, padding_flag, origin_format, + dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_output); + MS_LOG(DEBUG) << "Inserted Trans54"; + return replace_output; + } + return node; +} + +void GetTransDataInputFormat(const AnfNodePtr &node, std::string *input_format) { + MS_EXCEPTION_IF_NULL(input_format); + if (AnfAlgo::IsRealKernel(node)) { + *input_format = AnfAlgo::GetOutputFormat(node, 0); + } else { + *input_format = AnfAlgo::GetPrevNodeOutputFormat(node, 0); + } +} + +AnfNodePtr InsertTransOpForMultipleOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + std::vector make_tuple_inputs; + make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t output_idx = 0; output_idx < AnfAlgo::GetOutputTensorNum(node); ++output_idx) { + bool padding_flag = false; + + std::string output_format; + GetTransDataInputFormat(node, &output_format); + if (output_format == kOpFormat_NC1KHKWHWC0) { + MS_LOG(EXCEPTION) << "got the hw format" << output_format << " when insert the transdata node " + << node->DebugString(); + } + auto tuple_getitem = CreatTupleGetItemNode(func_graph, node, output_idx); + std::vector origin_shape = AnfAlgo::GetOutputInferShape(node, output_idx); + std::string origin_format = output_format; + std::string dest_format = kOpFormat_DEFAULT; + if (output_format == kOpFormat_C1HWNCoC0) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_input = AddTransOpNodeToGraph(func_graph, tuple_getitem, kernel_select, 0, padding_flag, + origin_format, dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_input); + return replace_input; + } + if (output_format == kOpFormat_NC1HWC0 && origin_shape.size() > 1) { + padding_flag = (origin_shape.size() != kShape4dDims); + // Insert a 5to4 trans op. + AnfNodePtr replace_output = AddTransOpNodeToGraph(func_graph, tuple_getitem, kernel_select, 0, padding_flag, + origin_format, dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_output); + MS_LOG(DEBUG) << "Inserted Translate54"; + make_tuple_inputs.push_back(replace_output); + } else if (output_format == kOpFormat_FRAC_NZ) { + AnfNodePtr replace_output = AddTransOpNodeToGraph(func_graph, tuple_getitem, kernel_select, 0, padding_flag, + origin_format, dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_output); + MS_LOG(DEBUG) << "Inserted Translate " << output_format << " To default, index: " << output_idx; + make_tuple_inputs.push_back(replace_output); + } else if (output_format == kOpFormat_FRAC_Z && !origin_shape.empty()) { + padding_flag = (origin_shape.size() != kShape4dDims); + AnfNodePtr replace_output = AddTransOpNodeToGraph(func_graph, tuple_getitem, kernel_select, 0, padding_flag, + origin_format, dest_format, kTransDataOpName, false); + MS_EXCEPTION_IF_NULL(replace_output); + MS_LOG(DEBUG) << "Inserted Translate54"; + make_tuple_inputs.push_back(replace_output); + } else { + // No need insert trans op. + make_tuple_inputs.push_back(tuple_getitem); + } + } + AnfNodePtr make_tuple = func_graph->NewCNode(make_tuple_inputs); + return make_tuple; +} +} // namespace +AnfNodePtr AddTransOpNodeToGraph(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select, size_t insert_index, const bool padding_flag, + const std::string &origin_format, const std::string &dest_format, + const std::string &op_name, bool is_insert_input) { + AnfNodePtr trans_node = nullptr; + AnfNodePtr input_node = nullptr; + AnfNodePtr trans_data = nullptr; + MS_EXCEPTION_IF_NULL(node); + if (origin_format.empty() || dest_format.empty()) { + MS_LOG(EXCEPTION) << "trans op format is error, origin = " << origin_format << ", dest " << origin_format; + } + if (is_insert_input) { + if (!node->isa()) { + MS_LOG(EXCEPTION) << "cannot insert a transdata node to a node's input which the node is not a cnode"; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + input_node = AnfAlgo::GetInputNode(cnode, insert_index); + if (padding_flag) { + auto padd_shape = trans::TransShapeTo4d(AnfAlgo::GetOutputInferShape(input_node, 0)); + auto reshape_node = CreateReshapeNode(func_graph, input_node, kernel_select, padd_shape); + trans_data = NewTransOpNode(func_graph, reshape_node, kernel_select, padding_flag, op_name); + } else { + trans_data = NewTransOpNode(func_graph, input_node, kernel_select, padding_flag, op_name); + } + trans_node = trans_data; + } else { + input_node = node; + trans_data = NewTransOpNode(func_graph, input_node, kernel_select, padding_flag, op_name); + if (padding_flag) { + auto reshape_node = + CreateReshapeNode(func_graph, trans_data, kernel_select, AnfAlgo::GetOutputInferShape(input_node, 0)); + trans_node = reshape_node; + } else { + trans_node = trans_data; + } + } + MS_EXCEPTION_IF_NULL(trans_data); + MS_EXCEPTION_IF_NULL(trans_data->kernel_info()); + auto trans_ori_build_info = trans_data->kernel_info()->select_kernel_build_info(); + auto kernel_build_info = CreateKernelBuildInfo(origin_format, dest_format, input_node, *trans_ori_build_info); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info, trans_data.get()); + return trans_node; +} + +AnfNodePtr AddCastOpNodeToGraph(const FuncGraphPtr &func_graph, const AnfNodePtr &input, const std::string &format, + const TypeId &input_type, const TypeId &output_type, + const std::vector &origin_shape, const TypeId &origin_type) { + MS_EXCEPTION_IF_NULL(func_graph); + std::string input_format = format; + std::string output_format = format; + std::vector new_cast_inputs; + auto prim = std::make_shared(prim::kPrimCast->name()); + new_cast_inputs.push_back(NewValueNode(prim)); + new_cast_inputs.push_back(input); + CNodePtr cast = func_graph->NewCNode(new_cast_inputs); + MS_EXCEPTION_IF_NULL(cast); + // set kernel build info + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({input_format}); + builder.SetOutputsFormat({output_format}); + builder.SetInputsDeviceType({input_type}); + builder.SetOutputsDeviceType({output_type}); + builder.SetFusionType(kernel::FusionType::OPAQUE); + builder.SetProcessor(kernel::Processor::AICORE); + if (kernel::OpLib::FindOp(prim::kPrimCast->name(), kernel::kTBE) != nullptr) { + builder.SetKernelType(KernelType::TBE_KERNEL); + } else { + builder.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + } + // if kernel info is null , it remarks this function is running ut + if (cast->kernel_info() == nullptr) { + auto kernel_info = std::make_shared(); + cast->set_kernel_info(kernel_info); + } + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cast.get()); + AnfAlgo::SetOutputInferTypeAndShape({origin_type}, {origin_shape}, cast.get()); + return cast; +} + +AnfNodePtr InsertTransOpForOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select) { + size_t outputs_num = AnfAlgo::GetOutputTensorNum(node); + if (outputs_num == 0) { + return node; + } + // Single output + if (outputs_num == 1 && (!AnfAlgo::IsTupleOutput(node))) { + return InsertTransOpForSingleOutput(func_graph, node, kernel_select); + } + // Multiple output + return InsertTransOpForMultipleOutput(func_graph, node, kernel_select); +} + +AnfNodePtr InsertTransOpForInput(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + std::vector new_inputs = {AnfAlgo::GetCNodePrimitiveNode(cnode)}; + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(cnode); ++input_index) { + AnfNodePtr input_node = GetTransInputNodePtr(func_graph, cnode, input_index, kernel_select); + MS_EXCEPTION_IF_NULL(input_node); + new_inputs.push_back(input_node); + } + CNodePtr new_cnode = nullptr; + // cnode changed so make a new cnode to differ from original one. + auto kernel_graph = func_graph->cast>(); + if (kernel_graph == nullptr) { + new_cnode = std::make_shared(*cnode); + } else { + new_cnode = kernel_graph->NewCNode(cnode); + } + MS_EXCEPTION_IF_NULL(new_cnode); + new_cnode->set_inputs(new_inputs); + return new_cnode; +} + +CNodePtr InsertCastForInput(const FuncGraphPtr &func_graph, const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(cnode); + std::vector new_inputs = {AnfAlgo::GetCNodePrimitiveNode(cnode)}; + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(cnode); ++input_index) { + TypeId origin_type; + auto cur_input = AnfAlgo::GetInputNode(cnode, input_index); + if (!AnfAlgo::IsFeatureMapInput(cnode, input_index)) { + // weight + origin_type = AnfAlgo::GetPrevNodeOutputDeviceDataType(cnode, input_index); + } else { + // feature map + origin_type = AnfAlgo::GetPrevNodeOutputInferDataType(cnode, input_index); + } + const std::string dev_fmt = AnfAlgo::GetInputFormat(cnode, input_index); + const std::vector origin_shape = AnfAlgo::GetPrevNodeOutputInferShape(cnode, input_index); + const TypeId device_type = AnfAlgo::GetInputDeviceDataType(cnode, input_index); + if (origin_type != device_type) { + auto cast = + AddCastOpNodeToGraph(func_graph, cur_input, dev_fmt, origin_type, device_type, origin_shape, origin_type); + MS_EXCEPTION_IF_NULL(cast); + cast->set_scope(cnode->scope()); + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), cast); + new_inputs.push_back(cast); + } else { + new_inputs.push_back(cur_input); + } + } + auto kernel_graph = func_graph->cast>(); + CNodePtr new_node = nullptr; + if (kernel_graph == nullptr) { + new_node = std::make_shared(*cnode); + } else { + new_node = kernel_graph->NewCNode(cnode); + } + MS_EXCEPTION_IF_NULL(new_node); + new_node->set_inputs(new_inputs); + return new_node; +} + +AnfNodePtr CreatTupleGetItemNode(const FuncGraphPtr &func_graph, const AnfNodePtr &node, size_t output_idx) { + auto idx = NewValueNode(SizeToInt(output_idx)); + MS_EXCEPTION_IF_NULL(idx); + auto imm = std::make_shared(SizeToInt(output_idx)); + auto abstract_scalar = std::make_shared(imm); + idx->set_abstract(abstract_scalar); + AnfNodePtr tuple_getitem = func_graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), node, idx}); + MS_EXCEPTION_IF_NULL(tuple_getitem); + tuple_getitem->set_scope(node->scope()); + std::vector origin_shape = AnfAlgo::GetOutputInferShape(node, output_idx); + TypeId origin_type = AnfAlgo::GetOutputInferDataType(node, output_idx); + AnfAlgo::SetOutputInferTypeAndShape({origin_type}, {origin_shape}, tuple_getitem.get()); + return tuple_getitem; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ascend_helper.h b/mindspore/ccsrc/pre_activate/ascend/ascend_helper.h new file mode 100644 index 00000000000..b605d700c3a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ascend_helper.h @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_ASCEND_HELPER_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_ASCEND_HELPER_H_ + +#include +#include +#include +#include "device/ascend/kernel_select_ascend.h" +#include "kernel/kernel_query.h" + +namespace mindspore { +namespace opt { +class KernelSelect { + public: + KernelSelect() = default; + virtual ~KernelSelect() = default; + virtual void SelectKernel(const CNodePtr &cnode) { device::ascend::SelectKernelInfo(cnode); } + virtual bool CheckKernelAccuracySupported(const CNodePtr &kernel_node, + const kernel::KernelBuildInfoPtr &new_kernel_build_info) { + return device::ascend::CheckKernelAccuracySupported(kernel_node, new_kernel_build_info); + } +}; +using KernelSelectPtr = std::shared_ptr; + +class KernelQuery { + public: + KernelQuery() = default; + virtual ~KernelQuery() = default; + virtual void Query(const CNodePtr &kernel_node, + std::vector> *kernel_info_list) { + kernel::KernelQuery(kernel_node, kernel_info_list); + } +}; +using KernelQueryPtr = std::shared_ptr; + +AnfNodePtr AddTransOpNodeToGraph(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select, size_t insert_index, bool padding_flag, + const std::string &origin_format, const std::string &dest_format, + const std::string &op_name, bool is_insert_input); + +AnfNodePtr AddCastOpNodeToGraph(const FuncGraphPtr &func_graph, const AnfNodePtr &input, const std::string &format, + const TypeId &input_type, const TypeId &output_type, + const std::vector &origin_shape, const TypeId &origin_type); + +AnfNodePtr InsertTransOpForInput(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select); + +AnfNodePtr InsertTransOpForOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const KernelSelectPtr &kernel_select); + +CNodePtr InsertCastForInput(const FuncGraphPtr &func_graph, const CNodePtr &cnode); + +AnfNodePtr CreatTupleGetItemNode(const FuncGraphPtr &func_graph, const AnfNodePtr &node, size_t output_idx); +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_ASCEND_HELPER_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/buffer_fusion/buffer_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/buffer_fusion/buffer_fusion.cc new file mode 100644 index 00000000000..49c5e89641a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/buffer_fusion/buffer_fusion.cc @@ -0,0 +1,676 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/buffer_fusion/buffer_fusion.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel/kernel_fusion.h" +#include "debug/anf_ir_dump.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" +#include "device/kernel_info.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace opt { +namespace { +const int8_t MAX_PATTERN_SIZE = 7; +const int8_t MIN_PATTERN_SIZE = 2; +const int8_t ELTWISE_INPUT_SIZE = 2; +const int8_t ELTWISE_USE = 1; +const int8_t MULTI_ELTWISE_USE = 2; +const int8_t MAX_MULTI_ELTWISE_SIZE = 4; +const int8_t MAX_PURE_BUFFER_SUCC_SIZE = 3; +constexpr auto kOpAttrFusionId = "fusion_id"; + +#ifdef DEBUG +std::string GetFusionTypeName(const kernel::FusionType &type) { + switch (type) { + case kernel::FusionType::COMMREDUCE: + return "COMMREDUCE"; + case kernel::FusionType::SEGMENT: + return "SEGMENT"; + case kernel::FusionType::ELEMWISE: + return "ELEMWISE"; + case kernel::FusionType::CONVLUTION: + return "CONVLUTION"; + case kernel::FusionType::OPAQUE: + return "OPAQUE"; + default: + return "OPAQUE"; + } +} + +void DumpFusionScopeInfo(const kernel::FusionScopeInfo &info) { + MS_LOG(INFO) << "=== Dump FusionScopeInfo start id: " << info.scope_id; + for (auto &node : info.input_nodes) { + MS_LOG(INFO) << "=== Input: " << node->DebugString(); + } + for (auto &node : info.output_nodes) { + MS_LOG(INFO) << "=== Output: " << node->DebugString(); + } + for (auto &node : info.compute_nodes) { + MS_LOG(INFO) << "=== Compute: (" << node->DebugString() << ")-(" << GetFusionTypeName(AnfAlgo::GetFusionType(node)) + << ")"; + } + MS_LOG(INFO) << "=== Dump FusionScopeInfo end"; +} +#endif + +void SetAnfNodeFusionId(const FusedNodeRecord &record_node) { + MS_LOG(DEBUG) << "Size of opt vector to be fused is " << record_node.size(); + int32_t id = 1; + for (auto &record : record_node) { + MS_LOG(DEBUG) << "No" << id << ", opt vector to be fused contain " << record.size() << " opt."; + for (const auto &candidate : record) { + ValuePtr fusion_id_v = MakeValue(id); + AnfAlgo::SetNodeAttr(kOpAttrFusionId, fusion_id_v, candidate); + MS_LOG(DEBUG) << "No " << id << ": " << candidate->DebugString(); + } + id++; + } +} + +bool CheckEltWiseNode(FuncGraphManager *manager, std::unordered_set *record, const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(manager); + MS_EXCEPTION_IF_NULL(record); + auto user_nodes = manager->node_users()[node]; + return (AnfAlgo::GetKernelType(node) == KernelType::TBE_KERNEL && + AnfAlgo::GetFusionType(node) == kernel::FusionType::ELEMWISE && + (user_nodes.size() <= ELTWISE_USE || record->size() == 0)); +} + +// Common method to check for predecessors and successors in a fusion pattern +std::tuple FindPredAndSuccEltWiseNodes(const int8_t &max_size, FuncGraphManager *manager, + std::unordered_set *visited_set, + std::deque *todo, + std::unordered_set *record, const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(manager); + MS_EXCEPTION_IF_NULL(visited_set); + MS_EXCEPTION_IF_NULL(todo); + MS_EXCEPTION_IF_NULL(record); + MS_EXCEPTION_IF_NULL(node); + + CNodePtr new_node = node; + if (new_node->inputs().size() < ELTWISE_INPUT_SIZE) { + return std::make_tuple(false, new_node); + } + int8_t index = 1; + auto &users = manager->node_users(); + while (CheckEltWiseNode(manager, record, new_node)) { + (void)record->insert(new_node); + (void)visited_set->insert(new_node); + (void)todo->insert(todo->end(), new_node->inputs().begin() + 1, new_node->inputs().end()); + + auto cnode = new_node->input(1); + MS_EXCEPTION_IF_NULL(cnode); + if (!cnode->isa()) { + return std::make_tuple(false, new_node); + } + new_node = cnode->cast(); + MS_EXCEPTION_IF_NULL(new_node); + + if (!AnfAlgo::IsRealKernel(new_node) || new_node->inputs().size() < ELTWISE_INPUT_SIZE || + users[(new_node)].size() >= MULTI_ELTWISE_USE || visited_set->find(new_node) != visited_set->end()) { + return std::make_tuple(false, new_node); + } + + if (index >= max_size) { + break; + } + index++; + } + return std::make_tuple(true, new_node); +} + +std::tuple MatchGeneralPattern(FuncGraphManager *manager, std::unordered_set *record, + std::unordered_set *visited_set, + std::deque *todo, const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(manager); + MS_EXCEPTION_IF_NULL(record); + MS_EXCEPTION_IF_NULL(visited_set); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(todo); + CNodePtr new_node = node; + auto &users = manager->node_users(); + if (users[(new_node)].size() >= MULTI_ELTWISE_USE) { + return std::make_tuple(false, new_node); + } + + (void)record->insert(node); + (void)visited_set->insert(node); + (void)todo->insert(todo->end(), new_node->inputs().begin() + 1, new_node->inputs().end()); + + if (node->inputs().size() < 2) { + return std::make_tuple(false, new_node); + } + // only check the first real input, will check all + auto cnode = node->input(1); + MS_EXCEPTION_IF_NULL(cnode); + if (!cnode->isa()) { + return std::make_tuple(false, new_node); + } + new_node = cnode->cast(); + MS_EXCEPTION_IF_NULL(new_node); + + if (!AnfAlgo::IsRealKernel(new_node) || users[(new_node)].size() >= MULTI_ELTWISE_USE || + visited_set->find(new_node) != visited_set->end()) { + return std::make_tuple(false, new_node); + } + return std::make_tuple(true, new_node); +} + +CNodePtr FindFusionAnfNode(FuncGraphManager *manager, std::unordered_set *visited_set, + std::unordered_set *record, std::deque *todo, const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(manager); + MS_EXCEPTION_IF_NULL(visited_set); + MS_EXCEPTION_IF_NULL(record); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(todo); + // find fusion pattern predecessor nodes + auto ret = FindPredAndSuccEltWiseNodes(MAX_MULTI_ELTWISE_SIZE, manager, visited_set, todo, record, node); + auto new_node = std::get<1>(ret); + auto node_use_size = manager->node_users()[new_node].size(); + if (!std::get<0>(ret) || (record->size() > 1 && node_use_size > 1) || record->size() >= MAX_MULTI_ELTWISE_SIZE || + AnfAlgo::GetKernelType(new_node) != KernelType::TBE_KERNEL) { + return new_node; + } + + // key of fusion precessor + auto node_fusion_type = AnfAlgo::GetFusionType(new_node); + switch (node_fusion_type) { + case kernel::FusionType::COMMREDUCE: + case kernel::FusionType::SEGMENT: + ret = MatchGeneralPattern(manager, record, visited_set, todo, new_node); + new_node = std::get<1>(ret); + if (!std::get<0>(ret)) { + return new_node; + } + break; + case kernel::FusionType::ELEMWISE: + return new_node; + // -fallthrough to default and return + case kernel::FusionType::CONVLUTION: + (void)record->insert(new_node); + default: + (void)visited_set->insert(new_node); + if (new_node != nullptr) { + (void)todo->insert(todo->end(), new_node->inputs().begin() + 1, new_node->inputs().end()); + } + return new_node; + } + // find fusion pattern successor nodes + ret = FindPredAndSuccEltWiseNodes(MAX_PURE_BUFFER_SUCC_SIZE, manager, visited_set, todo, record, new_node); + return std::get<1>(ret); +} + +CNodePtr CreateFusionOp(const std::vector &inputs_list, const std::vector &outputs_list, + const std::vector &anf_nodes, session::KernelGraph *kernel_graph) { + MS_LOG(DEBUG) << "Start Create FusionOp Kernel"; + MS_EXCEPTION_IF_NULL(kernel_graph); + std::string fusion_op_name = "FusionOp"; + for (auto node : anf_nodes) { + fusion_op_name += '_' + AnfAlgo::GetCNodeName(node); + } + auto fusion_op = std::make_shared(fusion_op_name); + MS_EXCEPTION_IF_NULL(fusion_op); + + std::vector input_names; + for (uint8_t i = 0; i < inputs_list.size(); i++) { + input_names.emplace_back("input" + to_string(i)); + } + std::vector output_names; + for (uint8_t i = 0; i < outputs_list.size(); i++) { + output_names.emplace_back("output" + to_string(i)); + } + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + fusion_op->set_attr("input_names", input_names_v); + fusion_op->set_attr("output_names", output_names_v); + std::vector fusion_inputs_list = inputs_list; + auto value_node = std::make_shared(fusion_op); + (void)fusion_inputs_list.insert(fusion_inputs_list.begin(), value_node); + auto buffer_fusion_kernel = kernel_graph->NewCNode(fusion_inputs_list); + if (buffer_fusion_kernel == nullptr) { + MS_LOG(EXCEPTION) << "New FusionOp kernel failed!"; + } + buffer_fusion_kernel->set_scope((anf_nodes.back())->scope()); + + return buffer_fusion_kernel; +} + +kernel::KernelBuildInfoPtr CreateFusionOpKernelInfo(const std::vector &inputs_list_in, + const std::vector &inputs_list, + const std::vector &outputs_list) { + MS_LOG(DEBUG) << "Start Create Kernel Info"; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + // inputs format and data type + std::vector inputs_format; + std::vector inputs_data_type; + for (auto node : inputs_list_in) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto &inputs = cnode->inputs(); + for (size_t input_index = 1; input_index < inputs.size(); ++input_index) { + if (std::find(inputs_list.begin(), inputs_list.end(), inputs[input_index]) != inputs_list.end()) { + inputs_format.push_back(AnfAlgo::GetInputFormat(node, input_index - 1)); + inputs_data_type.push_back(AnfAlgo::GetInputDeviceDataType(node, input_index - 1)); + } + } + } + // outputs format and data type + std::vector outputs_format; + std::vector outputs_data_type; + for (size_t index = 0; index < outputs_list.size(); ++index) { + for (size_t idx = 0; idx < AnfAlgo::GetOutputTensorNum(outputs_list[index]); ++idx) { + auto kernel_with_index = AnfAlgo::VisitKernel(outputs_list[index], idx); + outputs_format.push_back(AnfAlgo::GetOutputFormat(kernel_with_index.first, kernel_with_index.second)); + outputs_data_type.push_back(AnfAlgo::GetOutputDeviceDataType(kernel_with_index.first, kernel_with_index.second)); + } + } + builder.SetInputsFormat(inputs_format); + builder.SetInputsDeviceType(inputs_data_type); + builder.SetOutputsFormat(outputs_format); + builder.SetOutputsDeviceType(outputs_data_type); + builder.SetKernelType(KernelType::TBE_KERNEL); + return builder.Build(); +} + +AnfNodePtr CreateTupleGetItem(const AnfNodePtr &buffer_fusion_kernel, session::KernelGraph *kernel_graph, + size_t output_index) { + MS_EXCEPTION_IF_NULL(kernel_graph); + std::vector tuple_getitem_inputs_list; + auto value = std::make_shared(prim::kPrimTupleGetItem); + MS_EXCEPTION_IF_NULL(value); + auto idx = NewValueNode(SizeToInt(output_index)); + MS_EXCEPTION_IF_NULL(idx); + int temp = SizeToInt(output_index); + auto imm = std::make_shared(temp); + auto abstract_scalar = std::make_shared(imm); + idx->set_abstract(abstract_scalar); + tuple_getitem_inputs_list.push_back(value); + tuple_getitem_inputs_list.push_back(buffer_fusion_kernel); + tuple_getitem_inputs_list.push_back(idx); + auto tuple_item = kernel_graph->NewCNode(tuple_getitem_inputs_list); + MS_EXCEPTION_IF_NULL(tuple_item); + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(buffer_fusion_kernel, output_index)}, + {AnfAlgo::GetOutputInferShape(buffer_fusion_kernel, output_index)}, + tuple_item.get()); + return tuple_item; +} + +void ReplaceOldNode(const std::vector &outputs_list, const AnfNodePtr &buffer_fusion_kernel, + session::KernelGraph *kernel_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto manager = kernel_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (outputs_list.size() == 1) { // single output + (void)manager->Replace(outputs_list[0], buffer_fusion_kernel); + } else { // multiple output + size_t real_idx = 0; + for (size_t index = 0; index < outputs_list.size(); ++index) { + if (AnfAlgo::GetOutputTensorNum(outputs_list[index]) == 1) { + auto tuple_item = CreateTupleGetItem(buffer_fusion_kernel, kernel_graph, real_idx++); + (void)manager->Replace(outputs_list[index], tuple_item); + } else { + std::vector make_tuple_inputs; + AbstractBasePtrList abstract_list; + make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t idx = 0; idx < AnfAlgo::GetOutputTensorNum(outputs_list[index]); ++idx) { + auto tuple_item = CreateTupleGetItem(buffer_fusion_kernel, kernel_graph, real_idx++); + abstract_list.push_back(tuple_item->abstract()); + make_tuple_inputs.push_back(tuple_item); + } + AnfNodePtr make_tuple = kernel_graph->NewCNode(make_tuple_inputs); + make_tuple->set_abstract(std::make_shared(abstract_list)); + (void)manager->Replace(outputs_list[index], make_tuple); + } + } + } +} + +void GetInputList(const CNodePtr &node, const int32_t cur_fusion_id, std::vector *inputs_list) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(inputs_list); + auto &inputs = node->inputs(); + for (size_t input_index = 1; input_index < inputs.size(); ++input_index) { + auto input = inputs[input_index]; + if (AnfAlgo::IsRealCNodeKernel(input)) { + if (AnfAlgo::HasNodeAttr(kOpAttrFusionId, input)) { + auto fusion_id = AnfAlgo::GetNodeAttr(input, kOpAttrFusionId); + if (fusion_id != cur_fusion_id) { + inputs_list->push_back(input); + } + } else { + inputs_list->push_back(input); + } + } else if (input->isa()) { + for (auto &input_in : input->cast()->inputs()) { + if (AnfAlgo::IsRealCNodeKernel(input_in)) { + if (AnfAlgo::HasNodeAttr(kOpAttrFusionId, input_in)) { + auto fusion_id = AnfAlgo::GetNodeAttr(input_in, kOpAttrFusionId); + if (fusion_id != cur_fusion_id) { + inputs_list->push_back(input); + } + } else { + inputs_list->push_back(input); + } + } + } + } else { + inputs_list->push_back(input); + } + } +} + +void CheckCurrentNodeIsInput(const CNodePtr &node, const int32_t &cur_fusion_id, + std::unordered_map *buffer_fusion_infos) { + MS_EXCEPTION_IF_NULL(buffer_fusion_infos); + if ((*buffer_fusion_infos).find(cur_fusion_id) == (*buffer_fusion_infos).end()) { + BufferFusionInfo_t buffer_fusion_info; + (*buffer_fusion_infos)[cur_fusion_id] = buffer_fusion_info; + } + std::vector inputs_list; + GetInputList(node, cur_fusion_id, &inputs_list); + if (!inputs_list.empty()) { + if (!(*buffer_fusion_infos)[cur_fusion_id].inputs_list.empty()) { + (void)(*buffer_fusion_infos)[cur_fusion_id].inputs_list.insert( + (*buffer_fusion_infos)[cur_fusion_id].inputs_list.end(), inputs_list.begin(), inputs_list.end()); + (void)(*buffer_fusion_infos)[cur_fusion_id].inputs_list_in.insert( + (*buffer_fusion_infos)[cur_fusion_id].inputs_list_in.end(), node); + } else { + (*buffer_fusion_infos)[cur_fusion_id].inputs_list = inputs_list; + (*buffer_fusion_infos)[cur_fusion_id].inputs_list_in.push_back(node); + } + } +} + +void InsertNode(const AnfNodePtr &node, std::vector *list) { + MS_EXCEPTION_IF_NULL(list); + if (std::find(list->begin(), list->end(), node) == list->end()) { + (void)list->insert(list->end(), node); + } +} + +void CheckCurrentNodeIsOutput(const CNodePtr &node, const int32_t &cur_fusion_id, + std::unordered_map *buffer_fusion_infos) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(buffer_fusion_infos); + for (auto &input : node->inputs()) { + MS_EXCEPTION_IF_NULL(input); + if (AnfAlgo::IsRealCNodeKernel(input) && AnfAlgo::HasNodeAttr(kOpAttrFusionId, input)) { + auto fusion_id = AnfAlgo::GetNodeAttr(input, kOpAttrFusionId); + if (buffer_fusion_infos->find(fusion_id) == buffer_fusion_infos->end()) { + BufferFusionInfo_t buffer_fusion_info; + (*buffer_fusion_infos)[fusion_id] = buffer_fusion_info; + } + if (fusion_id != cur_fusion_id) { + InsertNode(input, &((*buffer_fusion_infos)[fusion_id].outputs_list)); + } + } else if (input->isa()) { + for (auto &input_in : input->cast()->inputs()) { + if (AnfAlgo::IsRealCNodeKernel(input_in) && AnfAlgo::HasNodeAttr(kOpAttrFusionId, input_in)) { + auto fusion_id = AnfAlgo::GetNodeAttr(input_in, kOpAttrFusionId); + if (buffer_fusion_infos->find(fusion_id) == buffer_fusion_infos->end()) { + BufferFusionInfo_t buffer_fusion_info; + (*buffer_fusion_infos)[fusion_id] = buffer_fusion_info; + } + if (fusion_id != cur_fusion_id) { + InsertNode(input_in, &((*buffer_fusion_infos)[fusion_id].outputs_list)); + } + } + } + } + } +} + +void GetFusionScopeNodeList(const session::KernelGraph &kernel_graph, + std::unordered_map *buffer_fusion_infos) { + MS_EXCEPTION_IF_NULL(buffer_fusion_infos); + auto nodes = TopoSort(kernel_graph.get_return()); + for (auto &node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (AnfAlgo::IsRealCNodeKernel(node) && AnfAlgo::HasNodeAttr(kOpAttrFusionId, node)) { + auto fusion_id = AnfAlgo::GetNodeAttr(node, kOpAttrFusionId); + (*buffer_fusion_infos)[fusion_id].anf_nodes.push_back(node); + } + } +} + +void MatchOpNamePattern(const session::KernelGraph &kernel_graph, std::unordered_set *fused_set, + FusedNodeRecord *candidate_fusion) { + MS_EXCEPTION_IF_NULL(fused_set); + MS_EXCEPTION_IF_NULL(candidate_fusion); + std::vector node_list = TopoSort(kernel_graph.get_return()); + for (auto &node : node_list) { + if (!AnfAlgo::IsRealCNodeKernel(node) || fused_set->find(node) != fused_set->end()) { + continue; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (AnfAlgo::GetCNodeName(cnode) == kBNTrainingReduceOpName) { + auto conv = cnode->input(1); + if (conv->isa() && AnfAlgo::GetCNodeName(conv) == prim::kPrimConv2D->name()) { + auto manager = kernel_graph.manager(); + MS_EXCEPTION_IF_NULL(manager); + auto &users = manager->node_users(); + AnfAlgo::SetNodeAttr(kAttrOutputUsedNum, MakeValue(users[conv].size()), conv); + std::unordered_set record({cnode, conv}); + candidate_fusion->push_back(record); + fused_set->insert(record.begin(), record.end()); + } + } + } +} + +void MatchFusionTypePattern(const session::KernelGraph &kernel_graph, std::unordered_set *fused_set, + FusedNodeRecord *candidate_fusion) { + auto manager = kernel_graph.manager(); + MS_EXCEPTION_IF_NULL(manager); + MS_EXCEPTION_IF_NULL(fused_set); + MS_EXCEPTION_IF_NULL(candidate_fusion); + + auto return_node = kernel_graph.get_return(); + MS_EXCEPTION_IF_NULL(return_node); + if (return_node->inputs().size() <= 1) { + return; + } + std::deque todo; + todo.push_back(return_node->input(1)); + std::unordered_set visited_set; + + while (!todo.empty()) { + auto node = todo.front(); + MS_EXCEPTION_IF_NULL(node); + todo.pop_front(); + std::unordered_set record; + if (visited_set.find(node) != visited_set.end() || fused_set->find(node) != fused_set->end()) { + continue; + } + // Only fuse real cnode + if (!AnfAlgo::IsRealCNodeKernel(node)) { + auto cnode = node->cast(); + if (cnode != nullptr) { + (void)todo.insert(todo.end(), cnode->inputs().begin() + 1, cnode->inputs().end()); + } + continue; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + // cnode maybe updated + cnode = FindFusionAnfNode(manager.get(), &visited_set, &record, &todo, cnode); + if (record.size() >= MIN_PATTERN_SIZE && record.size() <= MAX_PATTERN_SIZE) { + candidate_fusion->push_back(record); + fused_set->insert(record.begin(), record.end()); + } + if (record.find(cnode) == record.end()) { + todo.push_back(cnode); + } + // no node matched + if (record.size() == 0) { + (void)visited_set.insert(node); + } + (void)todo.insert(todo.end(), cnode->inputs().begin() + 1, cnode->inputs().end()); + } +} +} // namespace + +void BufferFusion::GetBufferFusionInfo(const session::KernelGraph &kernel_graph, + std::unordered_map *buffer_fusion_infos) const { + MS_EXCEPTION_IF_NULL(buffer_fusion_infos); + std::vector node_list = TopoSort(kernel_graph.get_return()); + for (auto &node : node_list) { + if (!AnfAlgo::IsRealCNodeKernel(node)) { + continue; + } + + int32_t cur_fusion_id = -1; + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (AnfAlgo::HasNodeAttr(kOpAttrFusionId, cnode)) { + cur_fusion_id = AnfAlgo::GetNodeAttr(cnode, kOpAttrFusionId); + CheckCurrentNodeIsInput(cnode, cur_fusion_id, buffer_fusion_infos); + } + // Check if current node is output + CheckCurrentNodeIsOutput(cnode, cur_fusion_id, buffer_fusion_infos); + } + + GetFusionScopeNodeList(kernel_graph, buffer_fusion_infos); + for (auto &buffer_fusion_info : *buffer_fusion_infos) { + buffer_fusion_info.second.kernel_build_info = + CreateFusionOpKernelInfo(buffer_fusion_info.second.inputs_list_in, buffer_fusion_info.second.inputs_list, + buffer_fusion_info.second.outputs_list); + } +} + +bool BufferFusion::FuseBufferFusionPattern(session::KernelGraph *kernel_graph) const { + MS_EXCEPTION_IF_NULL(kernel_graph); + bool change = false; + std::unordered_map buffer_fusion_infos; + buffer_fusion_infos.clear(); + GetBufferFusionInfo(*kernel_graph, &buffer_fusion_infos); + + std::vector fusion_scope_infos; + for (auto &buffer_fusion_info : buffer_fusion_infos) { + mindspore::kernel::FusionScopeInfo fusion_scope_info; + fusion_scope_info.scope_id = buffer_fusion_info.first; + fusion_scope_info.input_nodes = buffer_fusion_info.second.inputs_list; + fusion_scope_info.compute_nodes = buffer_fusion_info.second.anf_nodes; + fusion_scope_info.output_nodes = buffer_fusion_info.second.outputs_list; + fusion_scope_infos.push_back(fusion_scope_info); +#ifdef DEBUG + DumpFusionScopeInfo(fusion_scope_info); +#endif + } + auto kernel_mods = mindspore::kernel::KernelFusion(fusion_scope_infos); + + std::vector fusion_ids; + for (auto &buffer_fusion_info : buffer_fusion_infos) { + MS_LOG(DEBUG) << "anf node size: " << buffer_fusion_info.second.anf_nodes.size() + << ", inputs_list size: " << buffer_fusion_info.second.inputs_list.size() + << ", outputs list size: " << buffer_fusion_info.second.outputs_list.size(); + fusion_ids.push_back(buffer_fusion_info.first); + } + // Replace fusion op from return to head + std::sort(fusion_ids.begin(), fusion_ids.end()); + for (auto &fusion_id : fusion_ids) { + // Get kernel mod when supporting tbe + if (kernel_mods.find(fusion_id) == kernel_mods.end() || kernel_mods[fusion_id] == nullptr) { + MS_LOG(DEBUG) << "fusion id: " << fusion_id << ", fusion op compiling failed"; + continue; + } + change = ReplaceFusionOp(buffer_fusion_infos[fusion_id], kernel_mods[fusion_id], kernel_graph); + } + MS_LOG(DEBUG) << "End Buffer Fusion"; + return change; +} + +bool BufferFusion::MatchBufferFusionPattern(const session::KernelGraph &kernel_graph) const { + auto manager = kernel_graph.manager(); + MS_EXCEPTION_IF_NULL(manager); + auto return_node = kernel_graph.get_return(); + MS_EXCEPTION_IF_NULL(return_node); + if (return_node->inputs().size() <= 1) { + return false; + } + MS_LOG(DEBUG) << "MatchBufferFusionPattern start..."; + FusedNodeRecord candidate_fusion; + std::unordered_set fused_set; + + MatchOpNamePattern(kernel_graph, &fused_set, &candidate_fusion); + MatchFusionTypePattern(kernel_graph, &fused_set, &candidate_fusion); + + if (!candidate_fusion.empty()) { + SetAnfNodeFusionId(candidate_fusion); + } else { + return false; + } + MS_LOG(DEBUG) << "MatchBufferFusionPattern Success..."; + return true; +} + +bool BufferFusion::ReplaceFusionOp(const BufferFusionInfo_t &buffer_fusion_info, const kernel::KernelModPtr &kernel_ptr, + session::KernelGraph *kernel_graph) const { + auto buffer_fusion = CreateFusionOp(buffer_fusion_info.inputs_list, buffer_fusion_info.outputs_list, + buffer_fusion_info.anf_nodes, kernel_graph); + AnfAlgo::SetSelectKernelBuildInfo(buffer_fusion_info.kernel_build_info, buffer_fusion.get()); + // Set abstract of fusion_op node + std::vector types; + std::vector> shapes; + for (const auto &out_node : buffer_fusion_info.outputs_list) { + for (size_t idx = 0; idx < AnfAlgo::GetOutputTensorNum(out_node); ++idx) { + types.push_back(AnfAlgo::GetOutputInferDataType(out_node, idx)); + shapes.push_back(AnfAlgo::GetOutputInferShape(out_node, idx)); + } + } + if (types.empty() || shapes.empty()) { + MS_LOG(WARNING) << "buffer_fusion_info.outputs_list is empty"; + return false; + } + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, buffer_fusion.get()); + AnfAlgo::SetKernelMod(kernel_ptr, buffer_fusion.get()); + // replace node + ReplaceOldNode(buffer_fusion_info.outputs_list, buffer_fusion, kernel_graph); + return true; +} + +bool BufferFusion::Run(const FuncGraphPtr &graph) { + bool changed = false; + MS_EXCEPTION_IF_NULL(graph); + auto kernel_graph = graph->cast>(); + MS_EXCEPTION_IF_NULL(kernel_graph); + + if (MatchBufferFusionPattern(*kernel_graph)) { + changed = FuseBufferFusionPattern(kernel_graph.get()); + } + // clear fusion_id attr + for (auto &node : graph->nodes()) { + if (node != nullptr && node->isa()) { + AnfAlgo::EraseNodeAttr(kOpAttrFusionId, node); + } + } + return changed; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/buffer_fusion/buffer_fusion.h b/mindspore/ccsrc/pre_activate/ascend/buffer_fusion/buffer_fusion.h new file mode 100644 index 00000000000..c54fd0cd97d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/buffer_fusion/buffer_fusion.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_BUFFER_FUSION_BUFFER_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_BUFFER_FUSION_BUFFER_FUSION_H_ +#include +#include +#include + +#include "ir/anf.h" +#include "pre_activate/common/pass.h" +#include "device/kernel_info.h" +#include "kernel/kernel.h" +#include "session/kernel_graph.h" + +namespace mindspore { +namespace opt { +struct BufferFusionInfo_t { + std::vector anf_nodes; + std::vector inputs_list; + std::vector inputs_list_in; + std::vector outputs_list; + kernel::KernelBuildInfoPtr kernel_build_info; +}; + +using FusedNodeRecord = std::vector>; + +class BufferFusion : public Pass { + public: + BufferFusion() : Pass("buffer_fusion") {} + ~BufferFusion() override = default; + bool Run(const FuncGraphPtr &graph) override; + + private: + void GetBufferFusionInfo(const session::KernelGraph &kernel_graph, + std::unordered_map *buffer_fusion_infos) const; + bool ReplaceFusionOp(const BufferFusionInfo_t &buffer_fusion_info, const kernel::KernelModPtr &kernel_ptr, + session::KernelGraph *kernel_graph) const; + bool MatchBufferFusionPattern(const session::KernelGraph &kernel_graph) const; + bool FuseBufferFusionPattern(session::KernelGraph *kernel_graph) const; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_BUFFER_FUSION_BUFFER_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/check_consistency.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/check_consistency.cc new file mode 100644 index 00000000000..d2557a4bb70 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/check_consistency.cc @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/format_type/check_consistency.h" + +#include +#include + +#include "utils/utils.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +namespace { +bool CheckFormatForConsistency(const CNodePtr &node, const size_t input_index) { + MS_EXCEPTION_IF_NULL(node); + // get prior node's device output format + string pre_output_format = AnfAlgo::GetPrevNodeOutputFormat(node, input_index); + string selected_input_format = AnfAlgo::GetInputFormat(node, input_index); + if (pre_output_format == selected_input_format) { + return true; + } + auto input_origin_shape = AnfAlgo::GetPrevNodeOutputInferShape(node, input_index); + if (pre_output_format == kOpFormat_DEFAULT || selected_input_format == kOpFormat_DEFAULT) { + string checking_format = (pre_output_format == kOpFormat_DEFAULT) ? selected_input_format : pre_output_format; + // when input shape size is 1D, default format and NC1HWC0 are compatible + if (input_origin_shape.size() == 1 && checking_format == kOpFormat_NC1HWC0) { + return true; + } + if (kDefaultCompatibleFormat.find(checking_format) != kDefaultCompatibleFormat.end()) { + return true; + } + } + if (input_origin_shape.size() == 0) { + return true; + } + MS_LOG(ERROR) << "Found inconsistent format! input format " << input_index << ": " << pre_output_format + << ", selected input format: " << selected_input_format; + return false; +} + +bool CheckDataTypeForConsistency(const CNodePtr &node, const size_t input_index) { + MS_EXCEPTION_IF_NULL(node); + TypeId input_data_type = AnfAlgo::GetPrevNodeOutputDeviceDataType(node, input_index); + TypeId selected_data_type = AnfAlgo::GetInputDeviceDataType(node, input_index); + if (input_data_type == selected_data_type) { + return true; + } + MS_LOG(ERROR) << "Found inconsistent dtype! input dtype " << input_index << ": " << TypeIdLabel(input_data_type) + << ", selected dtype: " << TypeIdLabel(selected_data_type); + return false; +} +} // namespace + +const BaseRef CheckConsistency::DefinePattern() const { + VarPtr X = std::make_shared(); + VarPtr Xs = std::make_shared(); + return VectorRef({X, Xs}); +} + +const AnfNodePtr CheckConsistency::Process(const FuncGraphPtr &, const AnfNodePtr &node, const EquivPtr &) const { + if (node == nullptr || !node->isa() || !AnfAlgo::IsRealKernel(node)) { + return nullptr; + } + CNodePtr cnode = node->cast(); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(cnode); i++) { + if (!CheckFormatForConsistency(cnode, i) || !CheckDataTypeForConsistency(cnode, i)) { + MS_LOG(EXCEPTION) << "Found inconsistent format or data type! Op: " << AnfAlgo::GetCNodeName(node) << "[" + << node->DebugString() << "]"; + } + } + return nullptr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/check_consistency.h b/mindspore/ccsrc/pre_activate/ascend/format_type/check_consistency.h new file mode 100644 index 00000000000..e134547dc85 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/check_consistency.h @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_CHECK_CONSISTENCY_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_CHECK_CONSISTENCY_H_ + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class CheckConsistency : public PatternProcessPass { + public: + explicit CheckConsistency(bool multigraph = true) : PatternProcessPass("check_consistency", multigraph) {} + ~CheckConsistency() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_CHECK_CONSISTENCY_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/deal_ref_trans_and_cast.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/deal_ref_trans_and_cast.cc new file mode 100644 index 00000000000..fd206114150 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/deal_ref_trans_and_cast.cc @@ -0,0 +1,196 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/format_type/deal_ref_trans_and_cast.h" +#include +#include +#include +#include +#include "kernel/oplib/oplib.h" +#include "session/anf_runtime_algorithm.h" +#include "session/kernel_graph.h" + +namespace mindspore { +namespace opt { +namespace { +session::KernelWithIndex FindRefOriginNode(const AnfNodePtr &node) { + session::KernelWithIndex kernel_with_index = AnfAlgo::VisitKernel(node, 0); + AnfNodePtr cur_node = kernel_with_index.first; + size_t cur_out_index = kernel_with_index.second; + if (cur_node->isa()) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + std::string op_name = AnfAlgo::GetCNodeName(cnode); + auto op_info = mindspore::kernel::OpLib::FindOp(op_name, kernel::kTBE); + // deal ref op + if (op_info->is_ref()) { + auto ref_infos = op_info->ref_infos(); + if (ref_infos.count(cur_out_index) != 0) { + auto in_index = ref_infos.at(cur_out_index); + if (in_index > cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "ref op has wrong inputs: op inputs num is " << cnode->inputs().size() + << ", ref info is " << cur_out_index; + } + AnfNodePtr next_node = cnode->input(in_index + 1); + return FindRefOriginNode(next_node); + } + } + + // deal special (trans,cast,reshape) op + if (op_name == prim::kPrimCast->name() || op_name == prim::kPrimTranspose->name() || + op_name == prim::kPrimReshape->name() || op_name == kTransDataOpName) { + AnfNodePtr next_node = cnode->input(1); + return FindRefOriginNode(next_node); + } + } + + return kernel_with_index; +} + +void AddRefPairToKernelGraph(const FuncGraphPtr &func_graph, const CNodePtr &cnode, const AnfNodePtr &get_item, + const AnfNodePtr &final_node, size_t final_index, + const session::KernelWithIndex &origin_pair) { + // record the ref_pair + auto kernel_graph = func_graph->cast(); + MS_EXCEPTION_IF_NULL(kernel_graph); + // if the final node is get item, means no trans or cast op is added, the final node is itself + // so add the pair for itself, because the get item will removed later + auto final_ref = (final_node == get_item ? cnode : final_node); + session::AnfWithOutIndex final_pair = std::make_pair(final_ref, final_index); + if (kernel_graph->IsInRefOutputMap(final_pair)) { + MS_LOG(EXCEPTION) << "ref_pair is already in ref map, node is " << final_ref->DebugString() << ", index is " + << final_index; + } + MS_LOG(DEBUG) << "Add Ref pair, final {node ptr " << final_pair.first.get() << " , info is " + << final_pair.first->DebugString() << " , index is " << final_pair.second << "}, origin {node ptr " + << origin_pair.first.get() << ", info is " << origin_pair.first->DebugString() << " : index " + << origin_pair.second << "}"; + kernel_graph->AddRefCorrespondPairs(final_pair, origin_pair); +} + +// if get_item is nullptr, the additional node will link to the cnode +// else the additional node will link to the get_item node (the get_item node link to cnode) +AnfNodePtr AddAdditionalToRefOutput(const FuncGraphPtr &func_graph, const CNodePtr &cnode, size_t output_index, + size_t input_index, const AnfNodePtr &get_item) { + AnfNodePtr final_node = (get_item == nullptr ? cnode : get_item); + size_t final_index = output_index; + AnfNodePtr input_node = cnode->input(input_index + 1); + session::KernelWithIndex origin_pair; + origin_pair = FindRefOriginNode(input_node); + MS_EXCEPTION_IF_NULL(origin_pair.first); + if (!origin_pair.first->isa()) { + MS_LOG(EXCEPTION) << "ref op origin node is not parameter"; + } + MS_LOG(DEBUG) << "DealRefTransAndCast the node input index " << input_index << ", find origin op is " + << origin_pair.first->DebugString() << ", index is " << origin_pair.second; + auto origin_format = AnfAlgo::GetOutputFormat(origin_pair.first, origin_pair.second); + auto origin_type = AnfAlgo::GetOutputDeviceDataType(origin_pair.first, origin_pair.second); + auto cur_format = AnfAlgo::GetOutputFormat(cnode, output_index); + auto cur_type = AnfAlgo::GetOutputDeviceDataType(cnode, output_index); + auto cur_shape = AnfAlgo::GetOutputInferShape(cnode, 0); + // insert trans + if (origin_format != cur_format) { + auto kernel_select = std::make_shared(); + bool need_padding = + (cur_format == kOpFormat_NC1HWC0 && AnfAlgo::GetOutputInferShape(final_node, 0).size() != kShape4dDims); + final_node = AddTransOpNodeToGraph(func_graph, final_node, kernel_select, 0, need_padding, cur_format, + origin_format, kTransDataOpName, false); + final_index = 0; + MS_EXCEPTION_IF_NULL(final_node); + MS_LOG(INFO) << "DealRefTransAndCast add trans op, op debug info is " << final_node->DebugString(); + } + // insert cast + if (origin_type != cur_type) { + final_node = + AddCastOpNodeToGraph(func_graph, final_node, origin_format, cur_type, origin_type, cur_shape, cur_type); + MS_EXCEPTION_IF_NULL(final_node); + final_node->set_scope(cnode->scope()); + final_index = 0; + MS_LOG(INFO) << "DealRefTransAndCast add cast op, op debug info is " << final_node->DebugString(); + } + // add ref pair + AddRefPairToKernelGraph(func_graph, cnode, get_item, final_node, final_index, origin_pair); + // insert depend + if (origin_format != cur_format || origin_type != cur_type) { + std::vector depend_nodes{NewValueNode(prim::kPrimDepend), cnode, final_node}; + final_node = func_graph->NewCNode(depend_nodes); + MS_LOG(INFO) << "DealRefTransAndCast add denpend, op debug info is " << final_node->DebugString(); + } + + return final_node; +} +AnfNodePtr DealRefForMultipleOutput(const FuncGraphPtr &func_graph, const CNodePtr &cnode, + const std::shared_ptr &op_info) { + auto ref_infos = op_info->ref_infos(); + std::vector make_tuple_inputs; + AbstractBasePtrList abstract_list; + make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(cnode); ++output_index) { + AnfNodePtr final_node = CreatTupleGetItemNode(func_graph, cnode, output_index); + // deal with ref output + if (ref_infos.count(output_index) != 0) { + auto input_index = ref_infos.at(output_index); + final_node = AddAdditionalToRefOutput(func_graph, cnode, output_index, input_index, final_node); + } + abstract_list.push_back(final_node->abstract()); + make_tuple_inputs.push_back(final_node); + } + AnfNodePtr make_tuple = func_graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + make_tuple->set_abstract(std::make_shared(abstract_list)); + return make_tuple; +} + +AnfNodePtr DealRefSigleOutput(const FuncGraphPtr &func_graph, const CNodePtr &cnode, + const std::shared_ptr &op_info) { + auto ref_infos = op_info->ref_infos(); + for (const auto &ref_info : ref_infos) { + if (ref_info.second > cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "ref op has wrong inputs: op inputs num is " << cnode->inputs().size() << ", ref info is " + << ref_info.second; + } + return AddAdditionalToRefOutput(func_graph, cnode, ref_info.first, ref_info.second, nullptr); + } + return nullptr; +} +} // namespace + +const AnfNodePtr DealRefTransAndCast::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || !node->isa()) { + return nullptr; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!AnfAlgo::IsRealCNodeKernel(cnode)) { + return nullptr; + } + auto op_name = AnfAlgo::GetCNodeName(cnode); + auto op_info = mindspore::kernel::OpLib::FindOp(op_name, kernel::kTBE); + if (op_info == nullptr || !op_info->is_ref()) { + return nullptr; + } + if (op_info->is_ref()) { + if (!cnode->Type()->isa()) { + return DealRefSigleOutput(graph, cnode, op_info); + } else { + return DealRefForMultipleOutput(graph, cnode, op_info); + } + } + return nullptr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/deal_ref_trans_and_cast.h b/mindspore/ccsrc/pre_activate/ascend/format_type/deal_ref_trans_and_cast.h new file mode 100644 index 00000000000..9ed55d8b297 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/deal_ref_trans_and_cast.h @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_DEAL_REF_TRANS_AND_CAST_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_DEAL_REF_TRANS_AND_CAST_H_ + +#include "ir/anf.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class DealRefTransAndCast : public PatternProcessPass { + public: + explicit DealRefTransAndCast(bool multigraph = true) : PatternProcessPass("deal_ref_trans_and_cast", multigraph) {} + ~DealRefTransAndCast() override = default; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_DEAL_REF_TRANS_AND_CAST_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast.cc new file mode 100644 index 00000000000..0fefab10d0c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast.cc @@ -0,0 +1,118 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/format_type/insert_cast.h" + +#include +#include +#include + +#include "device/kernel_info.h" +#include "pre_activate/ascend/ascend_helper.h" +#include "pre_activate/common/helper.h" +#include "kernel/kernel_build_info.h" +#include "kernel/oplib/oplib.h" +#include "session/anf_runtime_algorithm.h" +#include "session/kernel_graph.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +namespace { +AnfNodePtr InsertCastForMultipleOutput(const FuncGraphPtr &func_graph, const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(cnode); + std::vector make_tuple_inputs; + AbstractBasePtrList abstract_list; + make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + for (size_t output_idx = 0; output_idx < AnfAlgo::GetOutputTensorNum(cnode); ++output_idx) { + const std::string dev_fmt = AnfAlgo::GetOutputFormat(cnode, output_idx); + const std::vector origin_shape = AnfAlgo::GetOutputInferShape(cnode, output_idx); + const TypeId origin_type = AnfAlgo::GetOutputInferDataType(cnode, output_idx); + const TypeId device_type = AnfAlgo::GetOutputDeviceDataType(cnode, output_idx); + auto idx = NewValueNode(SizeToInt(output_idx)); + MS_EXCEPTION_IF_NULL(idx); + auto imm = std::make_shared(output_idx); + idx->set_abstract(std::make_shared(imm)); + auto getitem = func_graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), cnode, idx}); + AnfAlgo::SetOutputInferTypeAndShape({origin_type}, {origin_shape}, getitem.get()); + AnfNodePtr replace_node = nullptr; + if (origin_type != device_type) { + replace_node = + AddCastOpNodeToGraph(func_graph, getitem, dev_fmt, device_type, origin_type, origin_shape, origin_type); + MS_EXCEPTION_IF_NULL(replace_node); + replace_node->set_scope(cnode->scope()); + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), replace_node); + } else { + replace_node = getitem; + } + abstract_list.push_back(replace_node->abstract()); + make_tuple_inputs.push_back(replace_node); + } + AnfNodePtr make_tuple = func_graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + make_tuple->set_abstract(std::make_shared(abstract_list)); + return make_tuple; +} + +AnfNodePtr InsertCastForOutput(const FuncGraphPtr &func_graph, const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(cnode); + if (AnfAlgo::GetOutputTensorNum(cnode) == 0) { + return cnode; + } + MS_EXCEPTION_IF_NULL(cnode->Type()); + // Single output + if (!cnode->Type()->isa()) { + const std::string dev_fmt = AnfAlgo::GetOutputFormat(cnode, 0); + std::vector origin_shape = AnfAlgo::GetOutputInferShape(cnode, 0); + const TypeId origin_type = AnfAlgo::GetOutputInferDataType(cnode, 0); + const TypeId device_type = AnfAlgo::GetOutputDeviceDataType(cnode, 0); + AnfNodePtr replace_node = cnode; + if (origin_type != device_type) { + replace_node = + AddCastOpNodeToGraph(func_graph, cnode, dev_fmt, device_type, origin_type, origin_shape, origin_type); + MS_EXCEPTION_IF_NULL(replace_node); + replace_node->set_scope(cnode->scope()); + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), replace_node); + } + return replace_node; + } + // Multiple output + return InsertCastForMultipleOutput(func_graph, cnode); +} +} // namespace + +const BaseRef InsertCast::DefinePattern() const { + VarPtr V = std::make_shared(UnVisited); + VarPtr Xs = std::make_shared(); + return VectorRef({V, Xs}); +} + +const AnfNodePtr InsertCast::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(node); + if (!AnfAlgo::IsRealCNodeKernel(node) || func_graph == nullptr) { + return nullptr; + } + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), node); + // process input + CNodePtr cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto new_node = InsertCastForInput(func_graph, cnode); + // process output + return InsertCastForOutput(func_graph, new_node); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast.h b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast.h new file mode 100644 index 00000000000..a7f93ec8f37 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast.h @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_INSERT_CAST_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_INSERT_CAST_H_ +#include + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pattern_engine.h" +#include "ir/anf.h" + +namespace mindspore { +namespace opt { +class InsertCast : public PatternProcessPass { + public: + explicit InsertCast(bool multigraph = true) : PatternProcessPass("insert_cast", multigraph) {} + ~InsertCast() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_INSERT_CAST_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast_for_runop.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast_for_runop.cc new file mode 100644 index 00000000000..7647b86c172 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast_for_runop.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/format_type/insert_cast_for_runop.h" + +#include + +#include "device/kernel_info.h" +#include "pre_activate/ascend/ascend_helper.h" +#include "pre_activate/common/helper.h" +#include "kernel/oplib/oplib.h" +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +const BaseRef RunOpInsertCast::DefinePattern() const { + VarPtr V = std::make_shared(UnVisited); + VarPtr Xs = std::make_shared(); + return VectorRef({V, Xs}); +} + +const AnfNodePtr RunOpInsertCast::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(node); + if (!AnfAlgo::IsRealCNodeKernel(node) || func_graph == nullptr) { + return nullptr; + } + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), node); + // process input + CNodePtr cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + return InsertCastForInput(func_graph, cnode); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast_for_runop.h b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast_for_runop.h new file mode 100644 index 00000000000..8bc42eb26a4 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_cast_for_runop.h @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEVICE_OPTIMIZER_FORMAT_TYPE_PASS_INSERT_CAST_FOR_RUNOP_H_ +#define MINDSPORE_CCSRC_DEVICE_OPTIMIZER_FORMAT_TYPE_PASS_INSERT_CAST_FOR_RUNOP_H_ +#include + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pattern_engine.h" +#include "ir/anf.h" +namespace mindspore { +namespace opt { +class RunOpInsertCast : public PatternProcessPass { + public: + explicit RunOpInsertCast(bool multigraph = true) : PatternProcessPass("insert_cast_for_runop", multigraph) {} + ~RunOpInsertCast() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_OPTIMIZER_FORMAT_TYPE_PASS_INSERT_CAST_FOR_RUNOP_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc new file mode 100644 index 00000000000..97244e40c68 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/format_type/insert_trans_op.h" +#include +#include "utils/utils.h" +#include "pre_activate/ascend/ascend_helper.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" +#include "kernel/oplib/oplib.h" + +namespace mindspore { +namespace opt { +const BaseRef InsertTransOp::DefinePattern() const { + std::shared_ptr V = std::make_shared(UnVisited); + std::shared_ptr Xs = std::make_shared(); + return VectorRef({V, Xs}); +} + +const AnfNodePtr InsertTransOp::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || !AnfAlgo::IsRealKernel(node)) { + return nullptr; + } + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), node); + MS_LOG(DEBUG) << "====process op: " << node->DebugString(); + AnfNodePtr new_node = InsertTransOpForInput(func_graph, node, kernel_select_); + return InsertTransOpForOutput(func_graph, new_node, kernel_select_); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.h b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.h new file mode 100644 index 00000000000..eb6cfa95427 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.h @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_INSERT_TRANS_OP_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_INSERT_TRANS_OP_H_ + +#include +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class InsertTransOp : public PatternProcessPass { + public: + explicit InsertTransOp(bool multigraph = true) + : PatternProcessPass("insert_trans_op", multigraph), kernel_select_(std::make_shared()) {} + ~InsertTransOp() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + KernelSelectPtr kernel_select_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_INSERT_TRANS_OP_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_transdata_for_runop.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_transdata_for_runop.cc new file mode 100644 index 00000000000..3df513a19f2 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_transdata_for_runop.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/format_type/insert_transdata_for_runop.h" +#include +#include "utils/utils.h" +#include "pre_activate/ascend/ascend_helper.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" +#include "kernel/oplib/oplib.h" + +namespace mindspore { +namespace opt { +const BaseRef RunOpInsertTransData::DefinePattern() const { + std::shared_ptr V = std::make_shared(UnVisited); + MS_EXCEPTION_IF_NULL(V); + std::shared_ptr Xs = std::make_shared(); + MS_EXCEPTION_IF_NULL(Xs); + return VectorRef({V, Xs}); +} + +const AnfNodePtr RunOpInsertTransData::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || !AnfAlgo::IsRealKernel(node)) { + return nullptr; + } + AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), node); + MS_LOG(DEBUG) << "====process op: " << node->DebugString(); + return InsertTransOpForInput(func_graph, node, kernel_select_); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_transdata_for_runop.h b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_transdata_for_runop.h new file mode 100644 index 00000000000..298a3deda9b --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_transdata_for_runop.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DEVICE_OPTIMIZER_FORMAT_TYPE_PASS_INSERT_TRANSDATA_FOR_RUNOP_H_ +#define MINDSPORE_CCSRC_DEVICE_OPTIMIZER_FORMAT_TYPE_PASS_INSERT_TRANSDATA_FOR_RUNOP_H_ + +#include +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class RunOpInsertTransData : public PatternProcessPass { + public: + explicit RunOpInsertTransData(bool multigraph = true) + : PatternProcessPass("insert_transdata_for_runop", multigraph), + kernel_select_(std::make_shared()) {} + ~RunOpInsertTransData() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + KernelSelectPtr kernel_select_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEVICE_OPTIMIZER_FORMAT_TYPE_PASS_INSERT_TRANSDATA_FOR_RUNOP_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/merge_cast_to_op.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/merge_cast_to_op.cc new file mode 100644 index 00000000000..bc34ae14133 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/merge_cast_to_op.cc @@ -0,0 +1,246 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/format_type/merge_cast_to_op.h" + +#include +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace { +const size_t kCastInputNum = 2; +const size_t kTupleGetitemInputNum = 3; +bool AlternativeKernelInfoForInput(const CNodePtr &node, const TypeId dst_type, const size_t change_idx, + const std::shared_ptr &candidate_kernel_info) { + if (node == nullptr || node->kernel_info() == nullptr || candidate_kernel_info == nullptr) { + return false; + } + + // checkout inputs' fmt and dtype except index equal change_idx + for (size_t i = 0; i < candidate_kernel_info->GetInputNum(); i++) { + if (i == change_idx) { + if (candidate_kernel_info->GetInputDeviceType(i) != dst_type || + candidate_kernel_info->GetInputFormat(i) != AnfAlgo::GetInputFormat(node, i)) { + return false; + } + } else if (candidate_kernel_info->GetInputDeviceType(i) != AnfAlgo::GetInputDeviceDataType(node, i) || + candidate_kernel_info->GetInputFormat(i) != AnfAlgo::GetInputFormat(node, i)) { + return false; + } + } + + // check outputs's fmt and dtype + for (size_t i = 0; i < candidate_kernel_info->GetOutputNum(); i++) { + if (candidate_kernel_info->GetOutputDeviceType(i) != AnfAlgo::GetOutputDeviceDataType(node, i) || + candidate_kernel_info->GetOutputFormat(i) != AnfAlgo::GetOutputFormat(node, i)) { + return false; + } + } + return true; +} + +bool GetNextNodeAndCastIndex(const FuncGraphPtr &graph, const AnfNodePtr &node, AnfNodePtr *next_node, + size_t *cast_index) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + // Check whether the cast node is used for input by only one another node. + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(node) == manager->node_users().end() || manager->node_users()[node].size() != 1) { + return false; + } + *next_node = manager->node_users()[node].begin()->first; + *cast_index = IntToSize(manager->node_users()[node].begin()->second - 1); + return true; +} + +bool CheckInputs(const CNodePtr &node, const std::shared_ptr &kernel_info) { + MS_EXCEPTION_IF_NULL(kernel_info); + if (AnfAlgo::GetInputTensorNum(node) != kernel_info->GetInputNum()) { + return false; + } + + for (size_t index = 0; index < kernel_info->GetInputNum(); ++index) { + if (AnfAlgo::GetInputFormat(node, index) != kernel_info->GetInputFormat(index) || + AnfAlgo::GetInputDeviceDataType(node, index) != kernel_info->GetInputDeviceType(index)) { + return false; + } + } + return true; +} + +bool CheckOtherOutputs(const CNodePtr &node, const std::shared_ptr &kernel_info, + const size_t idx) { + MS_EXCEPTION_IF_NULL(kernel_info); + if (AnfAlgo::GetOutputTensorNum(node) != kernel_info->GetOutputNum()) { + return false; + } + for (size_t index = 0; index < kernel_info->GetOutputNum(); ++index) { + if (idx == index) { + continue; + } + if (AnfAlgo::GetOutputFormat(node, index) != kernel_info->GetOutputFormat(index) || + AnfAlgo::GetOutputDeviceDataType(node, index) != kernel_info->GetOutputDeviceType(index)) { + return false; + } + } + return true; +} + +bool CheckIndexOutput(const CNodePtr &node, const std::shared_ptr &kernel_info, size_t index) { + if (kernel_info == nullptr) { + return false; + } + + if (AnfAlgo::GetOutputDeviceDataType(node, 0) != kernel_info->GetOutputDeviceType(index)) { + return false; + } + if (AnfAlgo::GetOutputInferShape(node, 0).size() == 4 && AnfAlgo::GetOutputFormat(node, 0) == kOpFormat_NCHW && + kernel_info->GetOutputFormat(index) == kOpFormat_DEFAULT) { + return true; + } + return AnfAlgo::GetOutputFormat(node, 0) == kernel_info->GetOutputFormat(index); +} + +AnfNodePtr MergeCastToNextOp(const FuncGraphPtr &graph, const CNodePtr &node, const KernelQueryPtr kernel_query) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(kernel_query); + AnfNodePtr next_node = nullptr; + size_t cast_index = 0; + if (!GetNextNodeAndCastIndex(graph, node, &next_node, &cast_index)) { + return nullptr; + } + MS_EXCEPTION_IF_NULL(next_node); + if (!next_node->isa() || !AnfAlgo::IsRealKernel(next_node)) { + return nullptr; + } + auto next_cnode = next_node->cast(); + auto next_op_name = AnfAlgo::GetCNodeName(next_node); + std::vector> kernel_info_list; + kernel_query->Query(next_cnode, &kernel_info_list); + + auto dst_type_id = AnfAlgo::GetInputDeviceDataType(node, 0); + auto alternative_kernel_info = std::find_if( + kernel_info_list.begin(), kernel_info_list.end(), + [&next_cnode, &dst_type_id, &cast_index](const std::shared_ptr &candidate_kernel_info) { + return AlternativeKernelInfoForInput(next_cnode, dst_type_id, cast_index, candidate_kernel_info); + }); + if (alternative_kernel_info == kernel_info_list.end()) { + return nullptr; + } + MS_LOG(INFO) << "Found alternative kernel info for current anf kernel " << next_op_name; + AnfAlgo::SetSelectKernelBuildInfo(*alternative_kernel_info, next_cnode.get()); + if (node->inputs().size() < kCastInputNum) { + auto op_name = AnfAlgo::GetCNodeName(node); + MS_LOG(EXCEPTION) << "op[" << op_name << "] has wrong input num:"; + } + return node->input(1); +} + +bool GetPriorOp(const AnfNodePtr &x_node, CNodePtr *prior_op, bool *single_output, size_t *output_idx) { + MS_EXCEPTION_IF_NULL(x_node); + if (x_node->isa()) { + auto x_cnode = x_node->cast(); + *prior_op = x_cnode; + // when x_node is tuple_getitem + if (AnfAlgo::GetCNodeName(x_node) == prim::kPrimTupleGetItem->name()) { + if (x_cnode->inputs().size() < kTupleGetitemInputNum) { + MS_LOG(EXCEPTION) << "tuple getitem node has wrong input num" << x_cnode->inputs().size(); + } + MS_EXCEPTION_IF_NULL(output_idx); + AnfNodePtr input1 = x_cnode->input(1); + MS_EXCEPTION_IF_NULL(input1); + *prior_op = input1->cast(); + MS_EXCEPTION_IF_NULL(*prior_op); + AnfNodePtr input2 = x_cnode->input(2); + MS_EXCEPTION_IF_NULL(input2); + auto value_ptr = input2->cast(); + MS_EXCEPTION_IF_NULL(value_ptr); + *output_idx = IntToSize(GetValue(value_ptr->value())); + *single_output = false; + } + return AnfAlgo::IsRealKernel(*prior_op); + } + return false; +} + +AnfNodePtr MergeCastToPriorOp(const FuncGraphPtr &graph, const CNodePtr &cur_node, const KernelQueryPtr kernel_query) { + MS_EXCEPTION_IF_NULL(cur_node); + MS_EXCEPTION_IF_NULL(kernel_query); + if (cur_node->inputs().size() < kCastInputNum) { + MS_LOG(EXCEPTION) << "op[Cast] has wrong input num:"; + } + AnfNodePtr x_node = cur_node->input(1); + if (IsUsedByOthers(graph, x_node)) { + return nullptr; + } + + CNodePtr prior_op = nullptr; + bool single_output = true; + size_t output_idx = 0; + if (!GetPriorOp(x_node, &prior_op, &single_output, &output_idx)) { + return nullptr; + } + MS_EXCEPTION_IF_NULL(prior_op); + + std::vector> kernel_info_list; + kernel_query->Query(prior_op, &kernel_info_list); + auto kernel_info_it = std::find_if( + kernel_info_list.begin(), kernel_info_list.end(), + [&prior_op, &cur_node, &output_idx](const std::shared_ptr &item_kernel_info) { + return CheckInputs(prior_op, item_kernel_info) && CheckOtherOutputs(prior_op, item_kernel_info, output_idx) && + CheckIndexOutput(cur_node, item_kernel_info, output_idx); + }); + if (kernel_info_it == kernel_info_list.end()) { + return nullptr; + } + AnfAlgo::SetSelectKernelBuildInfo(*kernel_info_it, prior_op.get()); + + auto prior_name = AnfAlgo::GetCNodeName(prior_op); + if (prior_name == kFive2FourOpName) { + AnfAlgo::CopyNodeAttr("dst_type", "dstType", cur_node, prior_op); + } else if (prior_name == kFour2FiveOpName) { + AnfAlgo::CopyNodeAttr("dst_type", cur_node, prior_op); + } + return single_output ? prior_op : x_node; +} +} // namespace + +const BaseRef MergeCastToOp::DefinePattern() const { + VarPtr X = std::make_shared(); + return VectorRef({prim::kPrimCast, X}); +} + +const AnfNodePtr MergeCastToOp::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, const EquivPtr &) const { + if (node == nullptr || !node->isa()) { + return nullptr; + } + auto cnode = node->cast(); + auto new_node = MergeCastToNextOp(graph, cnode, kernel_query_); + if (new_node == nullptr) { + new_node = MergeCastToPriorOp(graph, cnode, kernel_query_); + } + return new_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/merge_cast_to_op.h b/mindspore/ccsrc/pre_activate/ascend/format_type/merge_cast_to_op.h new file mode 100644 index 00000000000..7e05c8a02af --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/merge_cast_to_op.h @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_MERGE_CAST_TO_OP_H +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_MERGE_CAST_TO_OP_H + +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class MergeCastToOp : public PatternProcessPass { + public: + explicit MergeCastToOp(bool multigraph = true) + : PatternProcessPass("merge_cast_to_op", multigraph), kernel_query_(std::make_shared()) {} + ~MergeCastToOp() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + KernelQueryPtr kernel_query_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_FORMAT_TYPE_MERGE_CAST_TO_OP_H diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/add_memcpy_async.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/add_memcpy_async.cc new file mode 100644 index 00000000000..2ab11b60327 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/add_memcpy_async.cc @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fission/add_memcpy_async.h" +#include +#include "utils/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "optimizer/opt.h" + +namespace mindspore { +namespace opt { +namespace { +AnfNodePtr CreateMemcpyAsyncOp(const FuncGraphPtr &graph, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + auto prim = std::make_shared(kMemCpyAsyncOpName); + std::vector new_node_inputs = {NewValueNode(prim), node}; + auto new_node = graph->NewCNode(new_node_inputs); + MS_EXCEPTION_IF_NULL(new_node); + new_node->set_abstract(node->abstract()); + new_node->set_scope(node->scope()); + return new_node; +} + +const AnfNodePtr AddMemcpyAsyncIfInputIsUsedByOthers(const FuncGraphPtr &graph, const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + const std::vector &inputs = node->inputs(); + bool replace = false; + if (inputs.empty()) { + MS_LOG(EXCEPTION) << "node[" + AnfAlgo::GetCNodeName(node) + "]'s inputs is empty"; + } + std::vector new_inputs = {inputs[0]}; + for (size_t i = 1; i < inputs.size(); ++i) { + auto input = node->input(i); + if (manager->node_users().find(input) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "node has no output in manager"; + } + if (manager->node_users()[input].size() > 1) { + replace = true; + new_inputs.push_back(CreateMemcpyAsyncOp(graph, input)); + } else { + new_inputs.push_back(input); + } + } + + CNodePtr new_node = std::make_shared(*node); + new_node->set_inputs(new_inputs); + return replace ? new_node : nullptr; +} +} // namespace + +const AnfNodePtr AddMemcpyAsync::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (func_graph == nullptr || node == nullptr || !node->isa()) { + return nullptr; + } + auto cnode = node->cast(); + auto op_name = AnfAlgo::GetCNodeName(cnode); + if (op_name != kAllReduceOpName && op_name != kAllGatherOpName && op_name != kReduceScatterOpName) { + return nullptr; + } + return AddMemcpyAsyncIfInputIsUsedByOthers(func_graph, cnode); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/add_memcpy_async.h b/mindspore/ccsrc/pre_activate/ascend/ir_fission/add_memcpy_async.h new file mode 100644 index 00000000000..227fc74fed9 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/add_memcpy_async.h @@ -0,0 +1,31 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_ADD_MEMCPY_ASYNC_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_ADD_MEMCPY_ASYNC_H_ + +#include +#include "pre_activate/common/optimizer.h" +namespace mindspore { +namespace opt { +class AddMemcpyAsync : public PatternProcessPass { + public: + explicit AddMemcpyAsync(bool multigraph = true) : PatternProcessPass("add_memcpy_async", multigraph) {} + ~AddMemcpyAsync() override = default; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_ADD_MEMCPY_ASYNC_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_grad_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_grad_split.cc new file mode 100644 index 00000000000..6282ed4f760 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_grad_split.cc @@ -0,0 +1,123 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fission/bn_grad_split.h" + +#include +#include +#include + +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "common/utils.h" +#include "pre_activate/common/helper.h" +#include "device/kernel_info.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +namespace { +void CreateOutputsOfUpdateGrad(const FuncGraphPtr &graph, const CNodePtr &bn_grad_node, + std::vector *bn_update_grad_outputs) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(bn_grad_node); + auto bn_grad_inputs = bn_grad_node->inputs(); + if (bn_grad_inputs.size() != kBNGradInputNum) { + MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"; + } + std::vector bn_update_grad_inputs = { + NewValueNode(std::make_shared(kBNTrainingUpdateGradOpName)), bn_grad_inputs[1], bn_grad_inputs[2], + bn_grad_inputs[4], bn_grad_inputs[5]}; + auto bn_update_grad = graph->NewCNode(bn_update_grad_inputs); + MS_EXCEPTION_IF_NULL(bn_update_grad); + bn_update_grad->set_kernel_info(std::make_shared()); + bn_update_grad->set_scope(bn_grad_node->scope()); + + auto types = {AnfAlgo::GetOutputInferDataType(bn_grad_node, 1), AnfAlgo::GetOutputInferDataType(bn_grad_node, 2)}; + auto shapes = {AnfAlgo::GetOutputInferShape(bn_grad_node, 1), AnfAlgo::GetOutputInferShape(bn_grad_node, 2)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, bn_update_grad.get()); + + AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn_grad_node, bn_update_grad); + CreateMultipleOutputsOfAnfNode(graph, bn_update_grad, kBNTrainingUpdateGradOutputNum, bn_update_grad_outputs); +} + +void CreateOutputsOfReduceGrad(const FuncGraphPtr &graph, const CNodePtr &bn_grad_node, + const std::vector &bn_update_grad_outputs, + std::vector *bn_reduce_grad_outputs) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(bn_grad_node); + auto bn_grad_inputs = bn_grad_node->inputs(); + if (bn_grad_inputs.size() != kBNGradInputNum) { + MS_LOG(EXCEPTION) << "BNGrad has wrong inputs size"; + } + if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { + MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"; + } + std::vector bn_reduce_grad_inputs = { + NewValueNode(std::make_shared(kBNTrainingReduceGradOpName)), + bn_grad_inputs[1], + bn_grad_inputs[2], + bn_update_grad_outputs[0], + bn_update_grad_outputs[1], + bn_grad_inputs[3], + bn_grad_inputs[4], + bn_grad_inputs[5]}; + auto bn_reduce_grad = graph->NewCNode(bn_reduce_grad_inputs); + MS_EXCEPTION_IF_NULL(bn_reduce_grad); + bn_reduce_grad->set_kernel_info(std::make_shared()); + bn_reduce_grad->set_scope(bn_grad_node->scope()); + + auto types = {AnfAlgo::GetOutputInferDataType(bn_grad_node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(bn_grad_node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, bn_reduce_grad.get()); + + AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn_grad_node, bn_reduce_grad); + (*bn_reduce_grad_outputs).push_back(bn_reduce_grad); +} + +CNodePtr BNGradSplitForTBE(const FuncGraphPtr &func_graph, const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(func_graph); + std::vector bn_update_grad_outputs; + CreateOutputsOfUpdateGrad(func_graph, cnode, &bn_update_grad_outputs); + if (bn_update_grad_outputs.size() != kBNTrainingUpdateGradOutputNum) { + MS_LOG(EXCEPTION) << "bn_update_grad_outputs has wrong size"; + } + + std::vector bn_reduce_grad_outputs; + CreateOutputsOfReduceGrad(func_graph, cnode, bn_update_grad_outputs, &bn_reduce_grad_outputs); + if (bn_reduce_grad_outputs.size() != 1) { + MS_LOG(EXCEPTION) << "bn_reduce_grad_outputs has wrong size"; + } + + std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), bn_reduce_grad_outputs[0], + bn_update_grad_outputs[0], bn_update_grad_outputs[1]}; + auto make_tuple = func_graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + return make_tuple; +} +} // namespace + +const BaseRef BnGradSplit::DefinePattern() const { + VarPtr Xs = std::make_shared(); + return VectorRef({prim::kPrimFusedBatchNormGrad, Xs}); +} + +const AnfNodePtr BnGradSplit::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + return BNGradSplitForTBE(func_graph, cnode); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_grad_split.h b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_grad_split.h new file mode 100644 index 00000000000..17e1f9b98e7 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_grad_split.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_BN_GRAD_SPLIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_BN_GRAD_SPLIT_H_ + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class BnGradSplit : public PatternProcessPass { + public: + explicit BnGradSplit(bool multigraph = true) : PatternProcessPass("bn_grad_split", multigraph) {} + ~BnGradSplit() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_BN_GRAD_SPLIT_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_split.cc new file mode 100644 index 00000000000..c8d92f7200f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_split.cc @@ -0,0 +1,126 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fission/bn_split.h" + +#include +#include +#include + +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "pre_activate/common/helper.h" +#include "device/kernel_info.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +namespace { +void CreateOutputsOfBNTrainingReduce(const FuncGraphPtr &graph, const CNodePtr &bn_cnode, + std::vector *bn_training_reduce_outputs) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(bn_cnode); + if (bn_cnode->inputs().size() != kBnInputNum) { + MS_LOG(EXCEPTION) << "BN node has wrong input size"; + } + // All the inputs of BNTrainingReduce are from the inputs of BN + std::vector bn_training_reduce_inputs = { + NewValueNode(std::make_shared(kBNTrainingReduceOpName))}; + bn_training_reduce_inputs.push_back(bn_cnode->input(1)); + auto bn_training_reduce = graph->NewCNode(bn_training_reduce_inputs); + MS_EXCEPTION_IF_NULL(bn_training_reduce); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + bn_training_reduce->set_kernel_info(kernel_info); + std::vector bn_shape_i0 = AnfAlgo::GetPrevNodeOutputInferShape(bn_cnode, 0); + if (bn_shape_i0.size() != kShape4dDims) { + MS_LOG(EXCEPTION) << "Get shape of FusedBatchNorm fail"; + } + std::vector bn_training_reduce_shape = {bn_shape_i0[1]}; + auto types = {kNumberTypeFloat32, kNumberTypeFloat32}; + auto shapes = {bn_training_reduce_shape, bn_training_reduce_shape}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, bn_training_reduce.get()); + bn_training_reduce->set_scope(bn_cnode->scope()); + AnfAlgo::CopyNodeAttrs(bn_cnode, bn_training_reduce); + + CreateMultipleOutputsOfAnfNode(graph, bn_training_reduce, kBNTrainingReduceOutputNum, bn_training_reduce_outputs); +} + +AnfNodePtr CreateOutputsOfBNTrainingUpdate(const FuncGraphPtr &graph, const CNodePtr &bn_cnode, + const std::vector &bn_training_reduce_outputs) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(bn_cnode); + if (bn_cnode->inputs().size() != kBnInputNum) { + MS_LOG(EXCEPTION) << "BN node has wrong input size"; + } + if (bn_training_reduce_outputs.size() != kBNTrainingReduceOutputNum) { + MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size"; + } + // the inputs of BNTrainingUpdate are from the outputs of BNTrainingReduce and the inputs of BN + std::vector bn_training_update_inputs = { + NewValueNode(std::make_shared(kBNTrainingUpdateOpName))}; + bn_training_update_inputs.push_back(bn_cnode->input(1)); + bn_training_update_inputs.push_back(bn_training_reduce_outputs[0]); + bn_training_update_inputs.push_back(bn_training_reduce_outputs[1]); + bn_training_update_inputs.push_back(bn_cnode->input(2)); + bn_training_update_inputs.push_back(bn_cnode->input(3)); + bn_training_update_inputs.push_back(bn_cnode->input(4)); + bn_training_update_inputs.push_back(bn_cnode->input(5)); + auto bn_training_update = graph->NewCNode(bn_training_update_inputs); + MS_EXCEPTION_IF_NULL(bn_training_update); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + bn_training_update->set_kernel_info(kernel_info); + bn_training_update->set_abstract(bn_cnode->abstract()); + bn_training_update->set_scope(bn_cnode->scope()); + auto factor = AnfAlgo::GetNodeAttr(bn_cnode, kAttrMomentum); + AnfAlgo::SetNodeAttr(kAttrFactor, MakeValue(factor), bn_training_update); + AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn_cnode, bn_training_update); + AnfAlgo::SetNodeAttr(kAttrIsRef, MakeValue(true), bn_training_update); + return bn_training_update; +} + +AnfNodePtr SplitFusedBatchNormForTBE(const FuncGraphPtr &func_graph, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() < kBnInputNum) { + MS_LOG(EXCEPTION) << "op[FusedBatchNorm] has less than " << kBnInputNum << " inputs."; + } + // Create BNTrainingReduce node and get outputs of BNTrainingReduce + std::vector bn_training_reduce_outputs; + CreateOutputsOfBNTrainingReduce(func_graph, cnode, &bn_training_reduce_outputs); + if (bn_training_reduce_outputs.size() != kBN1OutputNum) { + MS_LOG(EXCEPTION) << "make outputs of op BNTrainingReduce fail"; + } + + // Create BNTrainingUpdate node + return CreateOutputsOfBNTrainingUpdate(func_graph, cnode, bn_training_reduce_outputs); +} +} // namespace + +const BaseRef BnSplit::DefinePattern() const { + VarPtr Xs = std::make_shared(); + MS_EXCEPTION_IF_NULL(Xs); + return VectorRef({prim::kPrimFusedBatchNorm, Xs}); +} + +const AnfNodePtr BnSplit::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, const EquivPtr &) const { + return SplitFusedBatchNormForTBE(func_graph, node); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_split.h b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_split.h new file mode 100644 index 00000000000..bc5975af175 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/bn_split.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_BN_SPLIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_BN_SPLIT_H_ + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class BnSplit : public PatternProcessPass { + public: + explicit BnSplit(bool multigraph = true) : PatternProcessPass("bn_split", multigraph) {} + ~BnSplit() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_BN_SPLIT_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/layer_norm_grad_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/layer_norm_grad_split.cc new file mode 100644 index 00000000000..cc1356c7247 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/layer_norm_grad_split.cc @@ -0,0 +1,121 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fission/layer_norm_grad_split.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" +#include "ir/primitive.h" +#include "common/utils.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +void LayerNormGradSplit::CreateOutputsOfLayerNormXBackprop( + const FuncGraphPtr &graph, const CNodePtr &layer_norm_grad, + std::vector *layer_norm_x_backprop_outputs) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(layer_norm_grad); + MS_EXCEPTION_IF_NULL(kernel_select_); + auto prim = std::make_shared(kLayerNormXBackpropOpName); + std::vector layer_norm_x_backprop_inputs = {NewValueNode(prim)}; + for (size_t i = 1; i < layer_norm_grad->inputs().size(); ++i) { + layer_norm_x_backprop_inputs.push_back(layer_norm_grad->input(i)); + } + auto layer_norm_x_backprop = graph->NewCNode(layer_norm_x_backprop_inputs); + MS_EXCEPTION_IF_NULL(layer_norm_x_backprop); + layer_norm_x_backprop->set_scope(layer_norm_grad->scope()); + + auto types = {AnfAlgo::GetOutputInferDataType(layer_norm_grad, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(layer_norm_grad, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, layer_norm_x_backprop.get()); + + kernel_select_->SelectKernel(layer_norm_x_backprop); + (*layer_norm_x_backprop_outputs).push_back(layer_norm_x_backprop); +} + +void LayerNormGradSplit::CreateOutputsOfLayerNormBetaGammaBackprop( + const FuncGraphPtr &graph, const CNodePtr &layer_norm_grad, + std::vector *layer_norm_beta_gamma_backprop_outputs) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(layer_norm_grad); + MS_EXCEPTION_IF_NULL(kernel_select_); + auto prim = std::make_shared(kLayerNormBetaGammaBackpropOpName); + std::vector layer_norm_beta_gamma_backprop_inputs = {NewValueNode(prim)}; + for (size_t i = 1; i < layer_norm_grad->inputs().size() - 1; ++i) { + layer_norm_beta_gamma_backprop_inputs.push_back(layer_norm_grad->input(i)); + } + auto layer_norm_beta_gamma_backprop = graph->NewCNode(layer_norm_beta_gamma_backprop_inputs); + MS_EXCEPTION_IF_NULL(layer_norm_beta_gamma_backprop); + auto kernel_info = std::make_shared(); + layer_norm_beta_gamma_backprop->set_kernel_info(kernel_info); + layer_norm_beta_gamma_backprop->set_scope(layer_norm_grad->scope()); + + auto types = {AnfAlgo::GetOutputInferDataType(layer_norm_grad, 1), + AnfAlgo::GetOutputInferDataType(layer_norm_grad, 2)}; + auto shapes = {AnfAlgo::GetOutputInferShape(layer_norm_grad, 1), AnfAlgo::GetOutputInferShape(layer_norm_grad, 2)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, layer_norm_beta_gamma_backprop.get()); + + // get device shape of LayerNormGrad's 5th Input, and convert it to attr + std::vector shape_gamma = AnfAlgo::GetInputDeviceShape(layer_norm_grad, 4); + AnfAlgo::SetNodeAttr(kAttrShapeGamma, MakeValue(opt::Convert2Int(shape_gamma)), layer_norm_beta_gamma_backprop); + + kernel_select_->SelectKernel(layer_norm_beta_gamma_backprop); + CreateMultipleOutputsOfAnfNode(graph, layer_norm_beta_gamma_backprop, kLayerNormBetaGammaBackpropOutputNum, + layer_norm_beta_gamma_backprop_outputs); +} + +const BaseRef LayerNormGradSplit::DefinePattern() const { + VarPtr Xs = std::make_shared(); + VectorRef pattern({prim::kPrimLayerNormGrad, Xs}); + return pattern; +} + +const AnfNodePtr LayerNormGradSplit::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + if (cnode->inputs().size() != kLayerNormGradInputNum) { + return nullptr; + } + + // create layer_norm_x_backprop + std::vector layer_norm_x_backprop_outputs; + CreateOutputsOfLayerNormXBackprop(graph, cnode, &layer_norm_x_backprop_outputs); + if (layer_norm_x_backprop_outputs.size() != kSingleOutputNum) { + MS_LOG(EXCEPTION) << "layer_norm_grad_outputs has wrong size"; + } + + // create layer_norm_beta_gamma_backprop + std::vector layer_norm_beta_gamma_backprop_outputs; + CreateOutputsOfLayerNormBetaGammaBackprop(graph, cnode, &layer_norm_beta_gamma_backprop_outputs); + if (layer_norm_beta_gamma_backprop_outputs.size() != kLayerNormBetaGammaBackpropOutputNum) { + MS_LOG(EXCEPTION) << "layer_norm_beta_gamma_outputs has wrong size"; + } + + std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), layer_norm_x_backprop_outputs[0], + layer_norm_beta_gamma_backprop_outputs[0], + layer_norm_beta_gamma_backprop_outputs[1]}; + auto make_tuple = graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + return make_tuple; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/layer_norm_grad_split.h b/mindspore/ccsrc/pre_activate/ascend/ir_fission/layer_norm_grad_split.h new file mode 100644 index 00000000000..f25c2e98381 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/layer_norm_grad_split.h @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_LAYER_NORM_GRAD_SPLIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_LAYER_NORM_GRAD_SPLIT_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class LayerNormGradSplit : public PatternProcessPass { + public: + explicit LayerNormGradSplit(bool multigraph = true) + : PatternProcessPass("layer_norm_grad_split", multigraph), kernel_select_(std::make_shared()) {} + ~LayerNormGradSplit() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + void CreateOutputsOfLayerNormXBackprop(const FuncGraphPtr &graph, const CNodePtr &layer_norm_grad, + std::vector *layer_norm_grad_outputs) const; + void CreateOutputsOfLayerNormBetaGammaBackprop(const FuncGraphPtr &graph, const CNodePtr &layer_norm_grad, + std::vector *layer_norm_beta_gamma_outputs) const; + KernelSelectPtr kernel_select_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_LAYER_NORM_GRAD_SPLIT_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc new file mode 100644 index 00000000000..5924f6cd1cb --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc @@ -0,0 +1,107 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fission/topk_split.h" +#include +#include +#include "utils/utils.h" +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace opt { +constexpr size_t kFloat16Len = 2; // size of float16; +namespace { +tensor::TensorPtr CreateTensor(const AnfNodePtr &node) { + // 1 create tensor + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(node, 0); + auto last_dim = shape[shape.size() - 1]; + std::vector indices_shape = {SizeToInt(last_dim)}; + TensorTypePtr tensor_type = std::make_shared(kFloat16); + MS_EXCEPTION_IF_NULL(tensor_type); + tensor::DeviceInfo device_info{kOpFormat_DEFAULT, tensor_type}; + tensor::TensorPtr indices_tensor = std::make_shared(kFloat16->type_id(), indices_shape); + MS_EXCEPTION_IF_NULL(indices_tensor); + indices_tensor->set_device_info(device_info); + + // 2 set value of tensor + auto data_ptr = indices_tensor->data_c(true); + MS_EXCEPTION_IF_NULL(data_ptr); + std::vector half_data; + for (size_t i = 0; i < last_dim; ++i) { + half_data.emplace_back(Eigen::half(static_cast(i))); + } + auto elem_num = last_dim * kFloat16Len; + auto ret_code = memcpy_s(data_ptr, static_cast(indices_tensor->data().nbytes()), half_data.data(), elem_num); + if (ret_code != 0) { + MS_LOG(ERROR) << "Failed to copy data into Tensor."; + return nullptr; + } + return indices_tensor; +} + +ValueNodePtr CreateValueNode(const AnfNodePtr &node) { + tensor::TensorPtr indices_tensor = CreateTensor(node); + MS_EXCEPTION_IF_NULL(indices_tensor); + auto indices_const = std::make_shared(indices_tensor); + MS_EXCEPTION_IF_NULL(indices_const); + auto indices_abstract = indices_tensor->ToAbstract(); + indices_const->set_abstract(indices_abstract); + auto indices_kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(indices_kernel_info); + indices_const->set_kernel_info(indices_kernel_info); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetOutputsFormat({kOpFormat_DEFAULT}); + builder1.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), indices_const.get()); + return indices_const; +} +} // namespace + +const BaseRef TopKSplit::DefinePattern() const { + VarPtr X = std::make_shared(); + MS_EXCEPTION_IF_NULL(X); + auto prim = std::make_shared(kTopKOpName); + MS_EXCEPTION_IF_NULL(prim); + return VectorRef({prim, X}); +} + +const AnfNodePtr TopKSplit::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + auto kernel_graph = func_graph->cast(); + auto indices_const = CreateValueNode(node); + // set value node as topk's input + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + MS_LOG(INFO) << "already has input size: " << cnode->inputs().size(); + cnode->add_input(indices_const); + if (kernel_graph != nullptr) { + kernel_graph->AddValueNodeToGraph(indices_const); + } + + CNodePtr new_cnode = nullptr; + if (kernel_graph == nullptr) { + new_cnode = std::make_shared(*cnode); + } else { + new_cnode = kernel_graph->NewCNode(cnode); + } + MS_EXCEPTION_IF_NULL(new_cnode); + return new_cnode; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.h b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.h new file mode 100644 index 00000000000..8fcbbac4755 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.h @@ -0,0 +1,31 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_TOPK_SPLIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_TOPK_SPLIT_H_ + +#include "pre_activate/common/optimizer.h" +namespace mindspore { +namespace opt { +class TopKSplit : public PatternProcessPass { + public: + explicit TopKSplit(bool multigraph = true) : PatternProcessPass("topk_split", multigraph) {} + ~TopKSplit() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FISSION_TOPK_SPLIT_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_fusion.cc new file mode 100644 index 00000000000..1ecf4bbd06d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_fusion.cc @@ -0,0 +1,123 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/adam_apply_one_fusion.h" +#include "pre_activate/common/helper.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +namespace { +void GetAdd0AndAdd1(const AnfNodePtr &sub0, AnfNodePtr *add0, AnfNodePtr *add1) { + MS_EXCEPTION_IF_NULL(sub0); + MS_EXCEPTION_IF_NULL(add0); + MS_EXCEPTION_IF_NULL(add1); + auto sub0_cnode = sub0->cast(); + MS_EXCEPTION_IF_NULL(sub0_cnode); + CheckCNodeInputSize(sub0_cnode, kSubInputNum); + AnfNodePtr mul4 = sub0_cnode->input(2); + MS_EXCEPTION_IF_NULL(mul4); + auto mul4_cnode = mul4->cast(); + MS_EXCEPTION_IF_NULL(mul4_cnode); + CheckCNodeInputSize(mul4_cnode, kMulInputNum); + AnfNodePtr true_div0 = mul4_cnode->input(2); + MS_EXCEPTION_IF_NULL(true_div0); + auto true_div0_cnode = true_div0->cast(); + MS_EXCEPTION_IF_NULL(true_div0_cnode); + CheckCNodeInputSize(true_div0_cnode, kRealDivInputNum); + *add0 = true_div0_cnode->input(1); + AnfNodePtr add2 = true_div0_cnode->input(2); + MS_EXCEPTION_IF_NULL(add2); + auto add2_cnode = add2->cast(); + MS_EXCEPTION_IF_NULL(add2_cnode); + CheckCNodeInputSize(add2_cnode, kAddInputNum); + AnfNodePtr sqrt0 = add2_cnode->input(1); + MS_EXCEPTION_IF_NULL(sqrt0); + auto sqrt0_cnode = sqrt0->cast(); + MS_EXCEPTION_IF_NULL(sqrt0_cnode); + CheckCNodeInputSize(sqrt0_cnode, kSqrtInputNum); + *add1 = sqrt0_cnode->input(1); +} +} // namespace + +AnfNodePtr AdamApplyOneFusion::CreateAdamApplyOneNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + auto prim = std::make_shared(kAdamApplyOneOpName); + std::vector new_node_inputs = {NewValueNode(prim)}; + for (const auto &input_var : input_vars_) { + auto input_node = utils::cast((*equiv)[input_var]); + MS_EXCEPTION_IF_NULL(input_node); + new_node_inputs.push_back(input_node); + } + for (const auto &mul_x_input_var : mul_x_input_vars_) { + auto mul_x_input_node = utils::cast((*equiv)[mul_x_input_var]); + MS_EXCEPTION_IF_NULL(mul_x_input_node); + new_node_inputs.push_back(mul_x_input_node); + } + auto add2_y_node = utils::cast((*equiv)[add2_y_]); + MS_EXCEPTION_IF_NULL(add2_y_node); + new_node_inputs.push_back(add2_y_node); + auto new_node = func_graph->NewCNode(new_node_inputs); + return new_node; +} + +const BaseRef AdamApplyOneFusion::DefinePattern() const { + const auto prim_sqrt = std::make_shared(kSqrtOpName); + const auto prim_deal_div = std::make_shared(kRealDivOpName); + VectorRef mul2 = VectorRef({prim::kPrimMul, mul_x_input_vars_[2], input_vars_[1]}); + VectorRef mul3 = VectorRef({prim::kPrimMul, mul_x_input_vars_[3], VectorRef({prim::kPrimSquare, input_vars_[0]})}); + VectorRef sqrt0 = VectorRef({prim_sqrt, VectorRef({prim::kPrimTensorAdd, mul2, mul3})}); + VectorRef mul1 = VectorRef({prim::kPrimMul, mul_x_input_vars_[1], input_vars_[0]}); + VectorRef mul0 = VectorRef({prim::kPrimMul, mul_x_input_vars_[0], input_vars_[2]}); + VectorRef add0 = VectorRef({prim::kPrimTensorAdd, mul0, mul1}); + VectorRef true_div0 = VectorRef({prim_deal_div, add0, VectorRef({prim::kPrimTensorAdd, sqrt0, add2_y_})}); + return VectorRef({prim::kPrimSub, input_vars_[3], VectorRef({prim::kPrimMul, input_vars_[4], true_div0})}); +} + +const AnfNodePtr AdamApplyOneFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + auto new_node = CreateAdamApplyOneNode(func_graph, equiv); + MS_EXCEPTION_IF_NULL(new_node); + new_node->set_scope(node->scope()); + // Set abstract of new node + AbstractBasePtrList new_node_abstract_list; + AnfNodePtr add0 = nullptr; + AnfNodePtr add1 = nullptr; + GetAdd0AndAdd1(node, &add0, &add1); + MS_EXCEPTION_IF_NULL(add0); + MS_EXCEPTION_IF_NULL(add1); + new_node_abstract_list.push_back(add1->abstract()); + new_node_abstract_list.push_back(add0->abstract()); + new_node_abstract_list.push_back(node->abstract()); + auto abstract_tuple = std::make_shared(new_node_abstract_list); + new_node->set_abstract(abstract_tuple); + // Create tuple_getitem node for outputs + std::vector new_node_outputs; + CreateMultipleOutputsOfAnfNode(func_graph, new_node, kAdamApplyOneOutputNum, &new_node_outputs); + if (new_node_outputs.size() != kAdamApplyOneOutputNum) { + MS_LOG(EXCEPTION) << "The output size of node " << new_node->DebugString() << " should be " + << kAdamApplyOneOutputNum; + } + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(add1, new_node_outputs[0]); + (void)manager->Replace(add0, new_node_outputs[1]); + return new_node_outputs[2]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_fusion.h new file mode 100644 index 00000000000..6642561b076 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_fusion.h @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ADAM_APPLY_ONE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ADAM_APPLY_ONE_FUSION_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +constexpr size_t kAdamApplyOneInputNum = 5; +constexpr size_t kAdamApplyOneMulInputNum = 4; + +class AdamApplyOneFusion : public PatternProcessPass { + public: + explicit AdamApplyOneFusion(bool multigraph = true) : PatternProcessPass("adam_apply_one_fusion", multigraph) { + for (size_t i = 0; i < kAdamApplyOneInputNum; ++i) { + input_vars_.push_back(std::make_shared()); + } + for (size_t i = 0; i < kAdamApplyOneMulInputNum; ++i) { + mul_x_input_vars_.push_back(std::make_shared()); + } + add2_y_ = std::make_shared(); + } + + ~AdamApplyOneFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + AnfNodePtr CreateAdamApplyOneNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const; + std::vector input_vars_; + std::vector mul_x_input_vars_; + VarPtr add2_y_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ADAM_APPLY_ONE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc new file mode 100644 index 00000000000..442aa64217a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.cc @@ -0,0 +1,133 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +std::tuple GetAdd0Add1Node(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto sub0 = node->cast(); + MS_EXCEPTION_IF_NULL(sub0); + auto mul5_anf = sub0->input(2); + MS_EXCEPTION_IF_NULL(mul5_anf); + auto mul5 = mul5_anf->cast(); + MS_EXCEPTION_IF_NULL(mul5); + auto add3_anf = mul5->input(2); + MS_EXCEPTION_IF_NULL(add3_anf); + auto add3 = add3_anf->cast(); + MS_EXCEPTION_IF_NULL(add3); + auto real_div0_anf = add3->input(1); + MS_EXCEPTION_IF_NULL(real_div0_anf); + auto real_div0 = real_div0_anf->cast(); + MS_EXCEPTION_IF_NULL(real_div0); + auto add0_anf = real_div0->input(1); + MS_EXCEPTION_IF_NULL(add0_anf); + auto add2_anf = real_div0->input(2); + MS_EXCEPTION_IF_NULL(add2_anf); + auto add2 = add2_anf->cast(); + MS_EXCEPTION_IF_NULL(add2); + auto sqrt0_anf = add2->input(1); + MS_EXCEPTION_IF_NULL(sqrt0_anf); + auto sqrt0 = sqrt0_anf->cast(); + MS_EXCEPTION_IF_NULL(sqrt0); + auto add1_anf = sqrt0->input(1); + MS_EXCEPTION_IF_NULL(add1_anf); + return std::make_tuple(add0_anf, add1_anf); +} +} // namespace + +std::vector AdamApplyOneWithDecayRule::GetFusionNodeInputs(const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(equiv); + auto input0 = utils::cast((*equiv)[input0_]); + auto input1 = utils::cast((*equiv)[input1_]); + auto input2 = utils::cast((*equiv)[input2_]); + auto input3 = utils::cast((*equiv)[input3_]); + auto input4 = utils::cast((*equiv)[input4_]); + auto mul0_x = utils::cast((*equiv)[mul0_x_]); + auto mul1_x = utils::cast((*equiv)[mul1_x_]); + auto mul2_x = utils::cast((*equiv)[mul2_x_]); + auto mul3_x = utils::cast((*equiv)[mul3_x_]); + auto mul4_x = utils::cast((*equiv)[mul4_x_]); + auto add2_y = utils::cast((*equiv)[add2_y_]); + auto prim = std::make_shared(kAdamApplyOneWithDecayOpName); + return {NewValueNode(prim), input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, mul4_x, add2_y}; +} + +const BaseRef AdamApplyOneWithDecayRule::DefinePattern() const { + auto sqrt = std::make_shared(kSqrtOpName); + auto real_div = std::make_shared(kRealDivOpName); + VectorRef mul0_pattern({prim::kPrimMul, mul0_x_, input2_}); + VectorRef mul1_pattern({prim::kPrimMul, mul1_x_, input0_}); + VectorRef square0_pattern({prim::kPrimSquare, input0_}); + VectorRef add0_pattern({prim::kPrimTensorAdd, mul0_pattern, mul1_pattern}); + VectorRef mul2_pattern({prim::kPrimMul, mul2_x_, input1_}); + VectorRef mul3_pattern({prim::kPrimMul, mul3_x_, square0_pattern}); + VectorRef add1_pattern({prim::kPrimTensorAdd, mul2_pattern, mul3_pattern}); + VectorRef sqrt0_pattern({sqrt, add1_pattern}); + VectorRef add2_pattern({prim::kPrimTensorAdd, sqrt0_pattern, add2_y_}); + VectorRef mul4_pattern({prim::kPrimMul, mul4_x_, input3_}); + VectorRef real_div_pattern({real_div, add0_pattern, add2_pattern}); + VectorRef add3_pattern({prim::kPrimTensorAdd, real_div_pattern, mul4_pattern}); + VectorRef mul5_pattern({prim::kPrimMul, input4_, add3_pattern}); + VectorRef sub0_pattern({prim::kPrimSub, input3_, mul5_pattern}); + return sub0_pattern; +} + +const AnfNodePtr AdamApplyOneWithDecayRule::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + if (graph == nullptr || node == nullptr || equiv == nullptr) { + return nullptr; + } + + std::vector inputs = GetFusionNodeInputs(equiv); + auto fusion_node = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(fusion_node); + fusion_node->set_scope(node->scope()); + + AnfNodePtr add0 = nullptr; + AnfNodePtr add1 = nullptr; + std::tie(add0, add1) = GetAdd0Add1Node(node); + auto types = {AnfAlgo::GetOutputInferDataType(add1, 0), AnfAlgo::GetOutputInferDataType(add0, 0), + AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(add1, 0), AnfAlgo::GetOutputInferShape(add0, 0), + AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, fusion_node.get()); + + std::vector fusion_node_outputs; + CreateMultipleOutputsOfAnfNode(graph, fusion_node, kAdamApplyOneWithDecayOutputNum, &fusion_node_outputs); + if (fusion_node_outputs.size() != kAdamApplyOneWithDecayOutputNum) { + MS_LOG(ERROR) << "create multiple outputs for fusion node fail!"; + return nullptr; + } + + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(add1, fusion_node_outputs[0]); + (void)manager->Replace(add0, fusion_node_outputs[1]); + return fusion_node_outputs[2]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.h new file mode 100644 index 00000000000..a6bab48770b --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.h @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ADAM_APPLY_ONE_WITH_DECAY_RULE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ADAM_APPLY_ONE_WITH_DECAY_RULE_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" +namespace mindspore { +namespace opt { +class AdamApplyOneWithDecayRule : public PatternProcessPass { + public: + explicit AdamApplyOneWithDecayRule(bool multigraph = true) + : PatternProcessPass("adam_apply_one_with_decay_rule", multigraph) { + input0_ = std::make_shared(); + input1_ = std::make_shared(); + input2_ = std::make_shared(); + input3_ = std::make_shared(); + input4_ = std::make_shared(); + mul0_x_ = std::make_shared(); + mul1_x_ = std::make_shared(); + mul2_x_ = std::make_shared(); + mul3_x_ = std::make_shared(); + mul4_x_ = std::make_shared(); + add2_y_ = std::make_shared(); + } + ~AdamApplyOneWithDecayRule() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + std::vector GetFusionNodeInputs(const EquivPtr &equiv) const; + VarPtr input0_; + VarPtr input1_; + VarPtr input2_; + VarPtr input3_; + VarPtr input4_; + VarPtr mul0_x_; + VarPtr mul1_x_; + VarPtr mul2_x_; + VarPtr mul3_x_; + VarPtr mul4_x_; + VarPtr add2_y_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ADAM_APPLY_ONE_WITH_DECAY_RULE_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/allreduce_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/allreduce_fusion.cc new file mode 100644 index 00000000000..7b862b21b9a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/allreduce_fusion.cc @@ -0,0 +1,245 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/allreduce_fusion.h" + +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/graph_utils.h" +#include "operator/ops.h" +#include "device/kernel_info.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/kernel_build_info.h" +#include "parallel/context.h" + +namespace mindspore { +namespace opt { +namespace { +kernel::KernelBuildInfoPtr GenerateKernelBuildInfo(const AllReduceInfo_t &allreduce_node_info, size_t start_index, + size_t end_index) { + if (end_index >= allreduce_node_info.allreduce_node.size()) { + MS_LOG(EXCEPTION) << "end index out of vector size"; + } + std::vector inputs_device_format; + std::vector outputs_device_format; + std::vector inputs_device_type; + std::vector outputs_device_type; + std::vector> outputs_shape; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + for (size_t idx = start_index; idx <= end_index; ++idx) { + auto cnode = allreduce_node_info.allreduce_node[idx]; + MS_EXCEPTION_IF_NULL(cnode); + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(cnode); ++input_index) { + inputs_device_format.push_back(AnfAlgo::GetInputFormat(cnode, input_index)); + inputs_device_type.push_back(AnfAlgo::GetInputDeviceDataType(cnode, input_index)); + } + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(cnode); ++output_index) { + outputs_device_format.push_back(AnfAlgo::GetOutputFormat(cnode, output_index)); + outputs_device_type.push_back(AnfAlgo::GetOutputDeviceDataType(cnode, output_index)); + outputs_shape.push_back(AnfAlgo::GetOutputInferShape(cnode, output_index)); + } + builder.SetFusionType(AnfAlgo::GetFusionType(cnode)); + builder.SetProcessor(AnfAlgo::GetProcessor(cnode)); + builder.SetKernelType(AnfAlgo::GetKernelType(cnode)); + } + builder.SetInputsFormat(inputs_device_format); + builder.SetOutputsFormat(outputs_device_format); + builder.SetInputsDeviceType(inputs_device_type); + builder.SetOutputsDeviceType(outputs_device_type); + return builder.Build(); +} +} // namespace + +bool AllReduceFusion::GetSplitSegments(const AllReduceInfo_t &allreduce_node_info, size_t *segment_num, + std::vector *segment_index) const { + MS_EXCEPTION_IF_NULL(segment_num); + MS_EXCEPTION_IF_NULL(segment_index); + size_t allreduce_node_size = allreduce_node_info.allreduce_node.size(); + MS_LOG(INFO) << "graph all reduce node size " << allreduce_node_size; + + auto parallel_context = parallel::ParallelContext::GetInstance(); + MS_EXCEPTION_IF_NULL(parallel_context); + const std::vector split_indices = parallel_context->all_reduce_fusion_split_indices(); + + size_t segments = 0; + if (split_indices.size() != 0) { + uint32_t last_index = 0; + for (size_t i = 0; i < split_indices.size(); ++i) { + uint32_t index = split_indices[i]; + if (index <= last_index || index >= allreduce_node_size) { + MS_LOG(EXCEPTION) << "invalid allreduce split index " << i << " " << index; + } + segment_index->push_back(index); + last_index = index; + segments++; + } + if (last_index != allreduce_node_size - 1) { + segment_index->push_back(allreduce_node_size - 1); + segments++; + } + } else { + segments = groups_; + for (size_t i = 0; i < segments - 1; ++i) { + segment_index->push_back((i + 1) * (allreduce_node_size / segments) - 1); + } + segment_index->push_back(allreduce_node_size - 1); + } + + if (segments >= allreduce_node_size) { + MS_LOG(INFO) << "fusion not changed: segment_num=" << segments << ", allreduce_node_size=" << allreduce_node_size; + return false; + } + if (segment_index->at(segments - 1) != allreduce_node_size - 1) { + MS_LOG(EXCEPTION) << "the last segment index is invalid."; + } + for (size_t i = 0; i < segments - 1; ++i) { + if (segment_index->at(i) > segment_index->at(i + 1)) { + MS_LOG(EXCEPTION) << "illegal split: segment_index[" << i << "]=" << segment_index->at(i) << ", segment_index[ " + << i + 1 << "]=" << segment_index->at(i + 1); + } + } + *segment_num = segments; + return true; +} + +AnfNodePtr AllReduceFusion::CreateFusedAllReduce(const FuncGraphPtr &func_graph, + const AllReduceInfo_t &allreduce_node_info, size_t start_index, + size_t end_index) const { + MS_EXCEPTION_IF_NULL(func_graph); + auto prim = std::make_shared(kAllReduceOpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector fusion_inputs = {NewValueNode(prim)}; + // get all inputs of current segment + if (end_index >= allreduce_node_info.allreduce_node.size()) { + MS_LOG(EXCEPTION) << "end index out of vector size"; + } + for (size_t idx = start_index; idx <= end_index; ++idx) { + auto cnode = allreduce_node_info.allreduce_node[idx]; + MS_EXCEPTION_IF_NULL(cnode); + fusion_inputs.insert(fusion_inputs.end(), cnode->inputs().begin() + 1, cnode->inputs().end()); + } + AnfNodePtr fused_node = func_graph->NewCNode(fusion_inputs); + MS_EXCEPTION_IF_NULL(fused_node); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + fused_node->set_kernel_info(kernel_info); + AbstractBasePtrList abstract_list; + for (size_t idx = start_index; idx <= end_index; ++idx) { + auto cnode = allreduce_node_info.allreduce_node[idx]; + MS_EXCEPTION_IF_NULL(cnode); + AnfAlgo::CopyNodeAttr("fusion", cnode, fused_node); + AnfAlgo::CopyNodeAttr("op", cnode, fused_node); + AnfAlgo::CopyNodeAttr("group", cnode, fused_node); + abstract_list.push_back(cnode->abstract()); + } + auto kernel_build_info = GenerateKernelBuildInfo(allreduce_node_info, start_index, end_index); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info, fused_node.get()); + auto abstract_tuple = std::make_shared(abstract_list); + MS_EXCEPTION_IF_NULL(abstract_tuple); + fused_node->set_abstract(abstract_tuple); + return fused_node; +} + +bool AllReduceFusion::DoFusion(const FuncGraphPtr &func_graph, const AllReduceInfo_t &allreduce_node_info, + size_t segment_num, const std::vector &segment_index) const { + MS_EXCEPTION_IF_NULL(func_graph); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + bool changed = false; + size_t start_index = 0; + for (size_t segment_idx = 0; segment_idx < segment_num; ++segment_idx) { + size_t end_index = segment_index.at(segment_idx); + if (end_index - start_index < 1) { + start_index = end_index + 1; + continue; + } + AnfNodePtr new_allreduce = CreateFusedAllReduce(func_graph, allreduce_node_info, start_index, end_index); + // replace old allreduce with new allreduce + for (auto idx = start_index; idx <= end_index; ++idx) { + std::vector tuple_getitem_input; + tuple_getitem_input.push_back(NewValueNode(prim::kPrimTupleGetItem)); + tuple_getitem_input.push_back(new_allreduce); + auto index = NewValueNode(SizeToInt(idx - start_index)); + MS_EXCEPTION_IF_NULL(index); + auto imm = std::make_shared(idx - start_index); + MS_EXCEPTION_IF_NULL(imm); + auto abstract_scalar = std::make_shared(); + MS_EXCEPTION_IF_NULL(abstract_scalar); + index->set_abstract(abstract_scalar); + tuple_getitem_input.push_back(index); + AnfNodePtr tuple_getitem = func_graph->NewCNode(tuple_getitem_input); + MS_EXCEPTION_IF_NULL(tuple_getitem); + auto allreduce_node_item = allreduce_node_info.allreduce_node.at(idx); + MS_EXCEPTION_IF_NULL(allreduce_node_item); + tuple_getitem->set_abstract(allreduce_node_item->abstract()); + if (!manager->Replace(allreduce_node_item, tuple_getitem)) { + MS_LOG(EXCEPTION) << "manager replace node failed"; + } + } + start_index = end_index + 1; + changed = true; + } + return changed; +} + +bool AllReduceFusion::Run(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + const float input_grad_size_num = 0.0; + const float input_grad_time_num = 0.0; + // divide candidate fusion groups with same (group,op,fusion) attrs, fusion==0 means not fusion + std::unordered_map candidate_groups; + std::vector node_list = TopoSort(func_graph->get_return()); + for (auto &node : node_list) { + if (node != nullptr && node->isa() && AnfAlgo::GetCNodeName(node) == kAllReduceOpName) { + auto primitive = AnfAlgo::GetCNodePrimitive(node); + MS_EXCEPTION_IF_NULL(primitive); + int fusion = GetValue(primitive->GetAttr("fusion")); + if (fusion == 0) { + continue; + } + std::string group = GetValue(primitive->GetAttr("group")); + std::string op = GetValue(primitive->GetAttr("op")); + std::string key = group + op + std::to_string(fusion); + if (candidate_groups.find(key) == candidate_groups.end()) { + AllReduceInfo_t allreduce_node_info; + candidate_groups[key] = allreduce_node_info; + } + candidate_groups[key].allreduce_node.push_back(node->cast()); + candidate_groups[key].input_grad_size.push_back(input_grad_size_num); + candidate_groups[key].input_grad_time.push_back(input_grad_time_num); + } + } + // split candidate group to segments according to _group class member + bool changed = false; + for (auto &it : candidate_groups) { + if (it.second.allreduce_node.size() <= 1) { + continue; + } + size_t segment_num = 0; + std::vector segment_index; + if (GetSplitSegments(it.second, &segment_num, &segment_index)) { + if (DoFusion(func_graph, it.second, segment_num, segment_index)) { + changed = true; + } + } + } + return changed; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/allreduce_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/allreduce_fusion.h new file mode 100644 index 00000000000..c26dbc20d97 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/allreduce_fusion.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ALLREDUCE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ALLREDUCE_FUSION_H_ +#include + +#include "pre_activate/common/pass.h" +#include "ir/func_graph.h" +#include "ir/anf.h" + +namespace mindspore { +namespace opt { +struct AllReduceInfo_t { + std::vector allreduce_node; + std::vector input_grad_size; + std::vector input_grad_time; +}; + +class AllReduceFusion : public Pass { + public: + explicit AllReduceFusion(size_t groups = 1) : Pass("all_reduce_fusion"), groups_(groups) {} + ~AllReduceFusion() override = default; + bool Run(const FuncGraphPtr &graph) override; + + private: + bool DoFusion(const FuncGraphPtr &func_graph, const AllReduceInfo_t &allreduce_node_info, size_t segment_num, + const std::vector &segment_index) const; + AnfNodePtr CreateFusedAllReduce(const FuncGraphPtr &func_graph, const AllReduceInfo_t &allreduce_node_info, + size_t start_index, size_t end_index) const; + bool GetSplitSegments(const AllReduceInfo_t &allreduce_node_info, size_t *segment_num, + std::vector *segment_index) const; + size_t groups_ = 1; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_ALLREDUCE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.cc new file mode 100644 index 00000000000..2af3afbf195 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.cc @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "common/utils.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +const BaseRef ClipByNormNoDivSquareSumFusion::DefinePattern() const { + auto greater = std::make_shared(kGreaterOpName); + MS_EXCEPTION_IF_NULL(greater); + auto sqrt = std::make_shared(kSqrtOpName); + MS_EXCEPTION_IF_NULL(sqrt); + + VectorRef greater_pattern({greater, input_, constant_greater_}); + VectorRef pattern( + {prim::kPrimMaximum, + VectorRef({prim::kPrimSelect, greater_pattern, + VectorRef({sqrt, VectorRef({prim::kPrimSelect, greater_pattern, input_, constant_select_})}), input_}), + constant_maximum_}); + return pattern; +} + +const AnfNodePtr ClipByNormNoDivSquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + BaseRef &input_gnode = (*equiv)[input_]; + BaseRef &constant_select_gnode = (*equiv)[constant_select_]; + BaseRef &constant_greater_gnode = (*equiv)[constant_greater_]; + BaseRef &constant_maximum_gnode = (*equiv)[constant_maximum_]; + auto input = utils::cast(input_gnode); + auto constant_select = utils::cast(constant_select_gnode); + auto constant_greater = utils::cast(constant_greater_gnode); + auto constant_maximum = utils::cast(constant_maximum_gnode); + MS_EXCEPTION_IF_NULL(input); + MS_EXCEPTION_IF_NULL(constant_select); + MS_EXCEPTION_IF_NULL(constant_greater); + MS_EXCEPTION_IF_NULL(constant_maximum); + + auto prim = std::make_shared(kClipByNormNoDivSumOpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = {NewValueNode(prim), input, constant_select, constant_greater, constant_maximum}; + auto fusion_node = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(fusion_node); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, fusion_node.get()); + fusion_node->set_scope(node->scope()); + return fusion_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.h new file mode 100644 index 00000000000..126480603e8 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.h @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CLIP_BY_NORM_NO_DIV_SQUARE_SUM_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CLIP_BY_NORM_NO_DIV_SQUARE_SUM_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +constexpr auto kInputVarName = "input"; +constexpr auto kConstantSelectVarName = "constant_select"; +constexpr auto kConstantGreaterVarName = "constant_greater"; +constexpr auto kConstantMaximumVarName = "constant_maximum"; + +class ClipByNormNoDivSquareSumFusion : public PatternProcessPass { + public: + explicit ClipByNormNoDivSquareSumFusion(bool multigraph = true) + : PatternProcessPass("clip_by_norm_no_div_square_sum_fusion", multigraph) { + input_ = std::make_shared(kInputVarName); + constant_select_ = std::make_shared(kConstantSelectVarName); + constant_greater_ = std::make_shared(kConstantGreaterVarName); + constant_maximum_ = std::make_shared(kConstantMaximumVarName); + } + ~ClipByNormNoDivSquareSumFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr input_; + VarPtr constant_select_; + VarPtr constant_greater_; + VarPtr constant_maximum_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CLIP_BY_NORM_NO_DIV_SQUARE_SUM_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_value_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_value_fusion.cc new file mode 100644 index 00000000000..df94e897ec0 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_value_fusion.cc @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/clip_by_value_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +bool GetMinimumOp(const AnfNodePtr &input0, const AnfNodePtr &input1, CNodePtr *minimum, bool *is_first_input) { + MS_EXCEPTION_IF_NULL(input0); + MS_EXCEPTION_IF_NULL(input1); + + CNodePtr cnode = nullptr; + if (input0->isa() && !input1->isa()) { + cnode = input0->cast(); + *is_first_input = true; + } else if (!input0->isa() && input1->isa()) { + cnode = input1->cast(); + *is_first_input = false; + } else if (input0->isa() && input1->isa()) { + if (AnfAlgo::GetCNodeName(input0) == prim::kPrimMinimum->name()) { + cnode = input0->cast(); + *is_first_input = true; + } else { + cnode = input1->cast(); + *is_first_input = false; + } + } else { + return false; + } + + if (AnfAlgo::GetCNodeName(cnode) != prim::kPrimMinimum->name()) { + return false; + } + *minimum = cnode; + return true; +} +} // namespace + +const BaseRef ClipByValueFusion::DefinePattern() const { + VectorRef pattern({prim::kPrimMaximum, maximum_input0_, maximum_input1_}); + return pattern; +} + +const AnfNodePtr ClipByValueFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto maximum_input0 = utils::cast((*equiv)[maximum_input0_]); + auto maximum_input1 = utils::cast((*equiv)[maximum_input1_]); + MS_EXCEPTION_IF_NULL(maximum_input0); + MS_EXCEPTION_IF_NULL(maximum_input1); + + CNodePtr minimum = nullptr; + bool is_first_input = true; + if (!GetMinimumOp(maximum_input0, maximum_input1, &minimum, &is_first_input)) { + return nullptr; + } + MS_EXCEPTION_IF_NULL(minimum); + if (minimum->inputs().size() != kMinimumInputNum) { + return nullptr; + } + + auto prim = std::make_shared(kClipByValueOpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = {NewValueNode(prim), minimum->input(1), + is_first_input ? maximum_input1 : maximum_input0, minimum->input(2)}; + auto clip_by_value = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(clip_by_value); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, clip_by_value.get()); + clip_by_value->set_scope(node->scope()); + return clip_by_value; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_value_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_value_fusion.h new file mode 100644 index 00000000000..309b7cedd00 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/clip_by_value_fusion.h @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CLIP_BY_VALUE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CLIP_BY_VALUE_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class ClipByValueFusion : public PatternProcessPass { + public: + explicit ClipByValueFusion(bool multigraph = true) : PatternProcessPass("clip_by_value_fusion", multigraph) { + maximum_input0_ = std::make_shared(); + maximum_input1_ = std::make_shared(); + } + ~ClipByValueFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr maximum_input0_; + VarPtr maximum_input1_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CLIP_BY_VALUE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.cc new file mode 100644 index 00000000000..1270ae77c19 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.h" + +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +const BaseRef ConfusionSoftmaxGradRule::DefinePattern() const { + return VectorRef( + {prim::kPrimSub, input0_, VectorRef({prim::kPrimReduceSum, VectorRef({prim::kPrimMul, input0_, input1_})})}); +} + +const AnfNodePtr ConfusionSoftmaxGradRule::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto input0 = utils::cast((*equiv)[input0_]); + auto input1 = utils::cast((*equiv)[input1_]); + MS_EXCEPTION_IF_NULL(input0); + MS_EXCEPTION_IF_NULL(input1); + + auto prim = std::make_shared(kConfusionSoftmaxGradOpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = {NewValueNode(prim), input0, input1}; + auto confusion_softmax_grad = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(confusion_softmax_grad); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, confusion_softmax_grad.get()); + confusion_softmax_grad->set_scope(node->scope()); + return confusion_softmax_grad; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.h new file mode 100644 index 00000000000..58722e586f9 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.h @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONFUSION_SOFTMAX_GRAD_RULE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONFUSION_SOFTMAX_GRAD_RULE_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class ConfusionSoftmaxGradRule : public PatternProcessPass { + public: + explicit ConfusionSoftmaxGradRule(bool multigraph = true) + : PatternProcessPass("confusion_softmax_grad_rule", multigraph), + input0_(std::make_shared()), + input1_(std::make_shared()) {} + ~ConfusionSoftmaxGradRule() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr input0_; + VarPtr input1_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONFUSION_SOFTMAX_GRAD_RULE_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.cc new file mode 100644 index 00000000000..efee8c0effd --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.cc @@ -0,0 +1,157 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.h" +#include +#include +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" + +namespace mindspore { +namespace opt { +namespace { +constexpr size_t kBn2AddReluOutputNum = 4; +enum Bn2AddReluOutput { + kBn2AddReluOutput = 0, + kBn2AddReluRunningMean, + kBn2AddReluRunningVariance, + kBn2AddReluSaveInvVariance, +}; + +std::tuple GetUsedCNode(const AnfNodePtr &node) { + auto relu_cnode = CheckAnfNodeIfCNodeAndInputSize(node, kReluInputNum); + MS_EXCEPTION_IF_NULL(relu_cnode); + auto add_cnode = CheckAnfNodeIfCNodeAndInputSize(relu_cnode->input(1), kAddInputNum); + MS_EXCEPTION_IF_NULL(add_cnode); + auto add_input1_cnode = CheckAnfNodeIfCNodeAndInputSize(add_cnode->input(1), kTupleGetitemInputNum); + MS_EXCEPTION_IF_NULL(add_input1_cnode); + auto bn_cnode = CheckAnfNodeIfCNodeAndInputSize(add_input1_cnode->input(1), kBnInputNum); + MS_EXCEPTION_IF_NULL(bn_cnode); + auto conv_cnode = CheckAnfNodeIfCNodeAndInputSize(bn_cnode->input(kX), kConvInputNum); + + return std::make_tuple(conv_cnode, bn_cnode, add_cnode, relu_cnode); +} + +void CreateOutputsOfBn2AddRelu(const FuncGraphPtr &func_graph, const std::vector &conv_bn1_outputs, + const CNodePtr &bn_node, const CNodePtr &add_node, const CNodePtr &relu_node, + std::vector *bn2_add_relu_outputs) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(add_node); + MS_EXCEPTION_IF_NULL(relu_node); + MS_EXCEPTION_IF_NULL(bn_node); + auto prim = std::make_shared(kBN2AddReluOpName); + std::vector bn2_add_relu_inputs = {NewValueNode(prim)}; + // The inputs of bn2_add_relu are from the outputs of conv_bn1, the 2nd input of add, and the 2nd to 5th inputs of bn + (void)std::copy(conv_bn1_outputs.begin(), conv_bn1_outputs.end(), std::back_inserter(bn2_add_relu_inputs)); + bn2_add_relu_inputs.push_back(add_node->input(2)); + for (size_t i = kX + 1; i <= kVariance; i++) { + bn2_add_relu_inputs.push_back(bn_node->input(i)); + } + auto bn2_add_relu_cnode = func_graph->NewCNode(bn2_add_relu_inputs); + MS_EXCEPTION_IF_NULL(bn2_add_relu_cnode); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + bn2_add_relu_cnode->set_kernel_info(kernel_info); + + // Set attr for bn2_add_relu + AnfAlgo::CopyNodeAttrs(bn_node, bn2_add_relu_cnode); + AnfAlgo::CopyNodeAttr("epsilon", "eps", bn_node, bn2_add_relu_cnode); + + // Set abstract of bn2_add_relu + auto bn_abstract_tuple = dyn_cast(bn_node->abstract()); + MS_EXCEPTION_IF_NULL(bn_abstract_tuple); + if (bn_abstract_tuple->elements().size() != kBnOutputNum) { + MS_LOG(EXCEPTION) << "Abstract tuple size of FusedBatchNorm must be " << kBnOutputNum << ", but it is " + << bn_abstract_tuple->elements().size(); + } + auto relu_abstract = relu_node->abstract(); + MS_EXCEPTION_IF_NULL(relu_abstract); + // The abstracts of node bn2_add_relu are from the some abstracts of bn and relu nodes. + AbstractBasePtrList bn2_add_relu_abstract_list{relu_abstract, bn_abstract_tuple->elements()[kRunningMean], + bn_abstract_tuple->elements()[kRunningVariance], + bn_abstract_tuple->elements()[kSaveInvVariance]}; + auto abstract_tuple = std::make_shared(bn2_add_relu_abstract_list); + MS_EXCEPTION_IF_NULL(abstract_tuple); + bn2_add_relu_cnode->set_abstract(abstract_tuple); + + CreateMultipleOutputsOfAnfNode(func_graph, bn2_add_relu_cnode, kBn2AddReluOutputNum, bn2_add_relu_outputs); +} +} // namespace + +const BaseRef ConvBnAddReluFusion::DefinePattern() const { + VarPtr X = std::make_shared(); + MS_EXCEPTION_IF_NULL(X); + VarPtr W = std::make_shared(); + MS_EXCEPTION_IF_NULL(W); + VarPtr Ys = std::make_shared(); + MS_EXCEPTION_IF_NULL(Ys); + VarPtr Zs = std::make_shared(); + MS_EXCEPTION_IF_NULL(Zs); + + return VectorRef( + {prim::kPrimRelu, + PatternListType( + {prim::kPrimTensorAdd, + PatternListType({prim::kPrimTupleGetItem, + PatternListType({prim::kPrimFusedBatchNorm, PatternListType({prim::kPrimConv2D, Ys}), Zs}), + W}), + X})}); +} + +const AnfNodePtr ConvBnAddReluFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(func_graph); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + CNodePtr conv_cnode = nullptr; + CNodePtr bn_cnode = nullptr; + CNodePtr add_cnode = nullptr; + CNodePtr relu_cnode = nullptr; + std::tie(conv_cnode, bn_cnode, add_cnode, relu_cnode) = GetUsedCNode(node); + // Create conv_bn1 node and get outputs of conv_bn1 + std::vector conv_bn1_outputs; + CreateOutputsOfConvBn1(func_graph, conv_cnode, bn_cnode, &conv_bn1_outputs); + if (conv_bn1_outputs.size() != kConvBn1OutputNum) { + MS_LOG(EXCEPTION) << "The output size of node conv_bn1 must be " << kConvBn1OutputNum << ", but it is " + << conv_bn1_outputs.size(); + } + // Replace conv_node with the output 0 of conv_bn1 directly because the conv node may be used as input by others + (void)manager->Replace(conv_cnode, conv_bn1_outputs[kData]); + + // Create bn2_add_relu node and get outputs of bn2_add_relu + std::vector bn2_add_relu_outputs; + CreateOutputsOfBn2AddRelu(func_graph, conv_bn1_outputs, bn_cnode, add_cnode, relu_cnode, &bn2_add_relu_outputs); + if (bn2_add_relu_outputs.size() != kBn2AddReluOutputNum) { + MS_LOG(EXCEPTION) << "The output size of node bn2_add_relu must be " << kBn2AddReluOutputNum << ", but it is " + << bn2_add_relu_outputs.size(); + } + + // Create a make_tuple to replace the bn node here, the outputs are from node bn2_add_relu and conv_bn1. + std::vector make_tuple_inputs{NewValueNode(prim::kPrimMakeTuple), + bn2_add_relu_outputs[kBn2AddReluOutput], + bn2_add_relu_outputs[kBn2AddReluRunningMean], + bn2_add_relu_outputs[kBn2AddReluRunningVariance], + conv_bn1_outputs[kMean], + bn2_add_relu_outputs[kBn2AddReluSaveInvVariance]}; + auto make_tuple = func_graph->NewCNode(make_tuple_inputs); + (void)manager->Replace(bn_cnode, make_tuple); + return bn2_add_relu_outputs[kBn2AddReluOutput]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.h new file mode 100644 index 00000000000..eb7cc730b5f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_add_relu_fusion.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_ADD_RELU_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_ADD_RELU_FUSION_H_ + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class ConvBnAddReluFusion : public PatternProcessPass { + public: + explicit ConvBnAddReluFusion(bool multigraph = true) : PatternProcessPass("conv_bn_add_relu_fusion", multigraph) {} + ~ConvBnAddReluFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_ADD_RELU_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_fusion.cc new file mode 100644 index 00000000000..70a7b53809c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_fusion.cc @@ -0,0 +1,93 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/conv_bn_fusion.h" +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" + +namespace mindspore { +namespace opt { +const BaseRef ConvBnFusion::DefinePattern() const { + VarPtr Xs = std::make_shared(); + MS_EXCEPTION_IF_NULL(Xs); + VarPtr Ys = std::make_shared(); + MS_EXCEPTION_IF_NULL(Ys); + return VectorRef({prim::kPrimFusedBatchNorm, PatternListType({prim::kPrimConv2D, Xs}), Ys}); +} + +const AnfNodePtr ConvBnFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "The bn node is expected to be a cnode"; + } + auto bn_cnode = node->cast(); + MS_EXCEPTION_IF_NULL(bn_cnode); + if (bn_cnode->inputs().size() < kVariance + 1) { + auto op_name = AnfAlgo::GetCNodeName(bn_cnode); + MS_LOG(EXCEPTION) << "op[" << op_name << "] has less than " << kVariance + 1 << " inputs."; + } + AnfNodePtr conv_node = bn_cnode->input(kX); + MS_EXCEPTION_IF_NULL(conv_node); + if (!conv_node->isa()) { + MS_LOG(EXCEPTION) << "The conv node is expected to be a cnode"; + } + auto conv_cnode = conv_node->cast(); + MS_EXCEPTION_IF_NULL(conv_cnode); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + // Create conv_bn1 node and get outputs of conv_bn1 + std::vector conv_bn1_outputs; + CreateOutputsOfConvBn1(func_graph, conv_cnode, bn_cnode, &conv_bn1_outputs); + if (conv_bn1_outputs.size() != kConvBn1OutputNum) { + MS_LOG(EXCEPTION) << "The output size of node conv_bn1 must be " << kConvBn1OutputNum << ", but it is " + << conv_bn1_outputs.size(); + } + // Replace conv_node with the output 0 of conv_bn1 directly because the conv node may be used as input by other + (void)manager->Replace(conv_node, conv_bn1_outputs[kData]); + + // Create bn2 node and get outputs of bn2 + std::vector bn2_outputs; + std::vector bn1_outputs = {conv_bn1_outputs[2], conv_bn1_outputs[1]}; + CreateOutputsOfFusedBn2(func_graph, bn1_outputs, bn_cnode, &bn2_outputs); + if (bn2_outputs.size() != kBN2OutputNum) { + MS_LOG(EXCEPTION) << "The output size of node fusedbn2 must be " << kBN2OutputNum << ", but it is " + << bn2_outputs.size(); + } + + // Create bn3 node and get outputs of bn3 + std::vector bn3_outputs; + CreateOutputsOfFusedBn3(func_graph, conv_bn1_outputs[0], bn1_outputs, bn2_outputs, bn_cnode, &bn3_outputs); + + if (bn3_outputs.size() != kBN3OutputNum) { + MS_LOG(EXCEPTION) << "The output size of node fusedbn3 must be " << kBN3OutputNum << ", but it is " + << bn3_outputs.size(); + } + + // Return a make_tuple to replace the bn node here, the outputs are from node bn2 and conv_bn1. + std::vector make_tuple_inputs{NewValueNode(prim::kPrimMakeTuple), + bn3_outputs[0], + bn2_outputs[1], + bn2_outputs[2], + conv_bn1_outputs[2], + bn2_outputs[0]}; + + return func_graph->NewCNode(make_tuple_inputs); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_fusion.h new file mode 100644 index 00000000000..892e6053cf8 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_fusion.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_FUSION_H_ + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class ConvBnFusion : public PatternProcessPass { + public: + explicit ConvBnFusion(bool multigraph = true) : PatternProcessPass("conv_bn_fusion", multigraph) {} + ~ConvBnFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.cc new file mode 100644 index 00000000000..c5cea86b7f7 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.cc @@ -0,0 +1,140 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.h" + +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" +#include "device/kernel_info.h" + +namespace mindspore { +namespace opt { +namespace { +std::tuple GetPrevNodes(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto relu_node = node->cast(); + MS_EXCEPTION_IF_NULL(relu_node); + if (relu_node->inputs().size() < kReluInputNum) { + MS_LOG(EXCEPTION) << "relu has wrong input size"; + } + auto tuple_getitem_anf = relu_node->input(1); + MS_EXCEPTION_IF_NULL(tuple_getitem_anf); + auto tuple_getitem = tuple_getitem_anf->cast(); + MS_EXCEPTION_IF_NULL(tuple_getitem); + if (tuple_getitem->inputs().size() < kTupleGetitemInputNum) { + MS_LOG(EXCEPTION) << "tuple getitem has wrong input size"; + } + auto bn_node_anf = tuple_getitem->input(1); + MS_EXCEPTION_IF_NULL(bn_node_anf); + auto bn_node = bn_node_anf->cast(); + MS_EXCEPTION_IF_NULL(bn_node); + if (bn_node->inputs().size() < kBnInputNum) { + MS_LOG(EXCEPTION) << "bn_node has wrong input size"; + } + auto conv_node_anf = bn_node->input(1); + MS_EXCEPTION_IF_NULL(conv_node_anf); + CNodePtr conv_node = conv_node_anf->cast(); + MS_EXCEPTION_IF_NULL(conv_node); + return std::make_tuple(bn_node, bn_node, conv_node); +} + +void CreateOutputsOfBn2Relu(const FuncGraphPtr &func_graph, const std::vector &conv_bn1_outputs, + const CNodePtr &bn_node, const CNodePtr &relu_node, + std::vector *bn2_relu_outputs) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(bn_node); + MS_EXCEPTION_IF_NULL(relu_node); + // The inputs of bn2_relu are from the outputs of conv_bn1 and the 2nd to 5th inputs of bn + std::vector bn2_relu_inputs = {NewValueNode(std::make_shared(kBN2ReLUOpName))}; + (void)std::copy(conv_bn1_outputs.begin(), conv_bn1_outputs.end(), std::back_inserter(bn2_relu_inputs)); + for (size_t i = 2; i <= 5; i++) { + bn2_relu_inputs.push_back(bn_node->input(i)); + } + auto bn2_relu = func_graph->NewCNode(bn2_relu_inputs); + MS_EXCEPTION_IF_NULL(bn2_relu); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + bn2_relu->set_kernel_info(kernel_info); + auto types = {AnfAlgo::GetOutputInferDataType(relu_node, 0), AnfAlgo::GetOutputInferDataType(bn_node, 1), + AnfAlgo::GetOutputInferDataType(bn_node, 2), AnfAlgo::GetOutputInferDataType(bn_node, 4)}; + auto shapes = {AnfAlgo::GetOutputInferShape(relu_node, 0), AnfAlgo::GetOutputInferShape(bn_node, 1), + AnfAlgo::GetOutputInferShape(bn_node, 2), AnfAlgo::GetOutputInferShape(bn_node, 4)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, bn2_relu.get()); + // Set attr for bn2_add_relu + AnfAlgo::CopyNodeAttrs(bn_node, bn2_relu); + AnfAlgo::CopyNodeAttr("epsilon", "eps", bn_node, bn2_relu); + + CreateMultipleOutputsOfAnfNode(func_graph, bn2_relu, kBn2ReluOutputNum, bn2_relu_outputs); +} +} // namespace + +const BaseRef ConvBnReluFusion::DefinePattern() const { + VarPtr Xs = std::make_shared(); + VarPtr Ys = std::make_shared(); + VarPtr Z = std::make_shared(); + MS_EXCEPTION_IF_NULL(Xs); + MS_EXCEPTION_IF_NULL(Ys); + MS_EXCEPTION_IF_NULL(Z); + return VectorRef( + {prim::kPrimRelu, + PatternListType({prim::kPrimTupleGetItem, + PatternListType({prim::kPrimFusedBatchNorm, PatternListType({prim::kPrimConv2D, Xs}), Ys}), Z})}); +} + +const AnfNodePtr ConvBnReluFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + + CNodePtr relu_node = nullptr; + CNodePtr bn_node = nullptr; + CNodePtr conv_node = nullptr; + std::tie(relu_node, bn_node, conv_node) = GetPrevNodes(node); + + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + + std::vector conv_bn1_outputs; + CreateOutputsOfConvBn1(func_graph, conv_node, bn_node, &conv_bn1_outputs); + if (conv_bn1_outputs.size() != kConvBn1OutputNum) { + MS_LOG(EXCEPTION) << "conv_bn1 outputs has wrong size: " << conv_bn1_outputs.size(); + } + (void)manager->Replace(conv_node, conv_bn1_outputs[0]); + + std::vector bn2_relu_outputs; + CreateOutputsOfBn2Relu(func_graph, conv_bn1_outputs, bn_node, relu_node, &bn2_relu_outputs); + if (bn2_relu_outputs.size() != kBn2ReluOutputNum) { + MS_LOG(EXCEPTION) << "bn2_relu outputs has wrong size: " << bn2_relu_outputs.size(); + } + std::vector make_tuple_inputs{NewValueNode(prim::kPrimMakeTuple), + bn2_relu_outputs[0], + bn2_relu_outputs[1], + bn2_relu_outputs[2], + conv_bn1_outputs[2], + bn2_relu_outputs[3]}; + auto make_tuple = func_graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + (void)manager->Replace(bn_node, make_tuple); + return bn2_relu_outputs[0]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.h new file mode 100644 index 00000000000..ea415564aee --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_RELU_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_RELU_FUSION_H_ + +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class ConvBnReluFusion : public PatternProcessPass { + public: + explicit ConvBnReluFusion(bool multigraph = true) : PatternProcessPass("conv_bn_relu_fusion", multigraph) {} + ~ConvBnReluFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_CONV_BN_RELU_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.cc new file mode 100644 index 00000000000..12f2684b3b6 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.cc @@ -0,0 +1,210 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.h" +#include +#include +#include "pre_activate/common/helper.h" +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +namespace { +bool IsC(const BaseRef &n) { + if (utils::isa(n)) { + AnfNodePtr in = utils::cast(n); + MS_EXCEPTION_IF_NULL(in); + return in->isa(); + } + return false; +} + +AnfNodePtr GetBatchNormNode(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto depend_cnode = node->cast(); + MS_EXCEPTION_IF_NULL(depend_cnode); + CheckCNodeInputSize(depend_cnode, kDependInputNum); + AnfNodePtr assign_sub = depend_cnode->input(2); + MS_EXCEPTION_IF_NULL(assign_sub); + auto assign_sub_cnode = assign_sub->cast(); + MS_EXCEPTION_IF_NULL(assign_sub_cnode); + CheckCNodeInputSize(assign_sub_cnode, kAssignSubInputNum); + AnfNodePtr mul = assign_sub_cnode->input(2); + MS_EXCEPTION_IF_NULL(mul); + auto mul_cnode = mul->cast(); + MS_EXCEPTION_IF_NULL(mul_cnode); + CheckCNodeInputSize(mul_cnode, kMulInputNum); + AnfNodePtr sub = mul_cnode->input(1); + MS_EXCEPTION_IF_NULL(sub); + auto sub_cnode = sub->cast(); + MS_EXCEPTION_IF_NULL(sub_cnode); + CheckCNodeInputSize(sub_cnode, kSubInputNum); + AnfNodePtr tuple_getitem = sub_cnode->input(2); + MS_EXCEPTION_IF_NULL(tuple_getitem); + auto tuple_getitem_cnode = tuple_getitem->cast(); + MS_EXCEPTION_IF_NULL(tuple_getitem_cnode); + CheckCNodeInputSize(tuple_getitem_cnode, kTupleGetitemInputNum); + return tuple_getitem_cnode->input(1); +} + +bool CompareTupleGetitem(const AnfNodePtr &n1, const AnfNodePtr &n2) { + MS_EXCEPTION_IF_NULL(n1); + MS_EXCEPTION_IF_NULL(n2); + auto n1_cnode = n1->cast(); + auto n2_cnode = n2->cast(); + MS_EXCEPTION_IF_NULL(n1_cnode); + MS_EXCEPTION_IF_NULL(n2_cnode); + auto index_input1 = n1_cnode->input(kInputNodeOutputIndexInTupleGetItem); + MS_EXCEPTION_IF_NULL(index_input1); + auto value_node1 = index_input1->cast(); + MS_EXCEPTION_IF_NULL(value_node1); + auto index_input2 = n2_cnode->input(kInputNodeOutputIndexInTupleGetItem); + MS_EXCEPTION_IF_NULL(index_input2); + auto value_node2 = index_input2->cast(); + MS_EXCEPTION_IF_NULL(value_node2); + return GetValue(value_node1->value()) < GetValue(value_node2->value()); +} + +void GetBNOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &bn, std::vector *bn_outputs) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(bn); + MS_EXCEPTION_IF_NULL(bn_outputs); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(bn) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "The bn node " << bn->DebugString() << " should has some outputs"; + } + for (const auto &node_index : manager->node_users()[bn]) { + AnfNodePtr output = node_index.first; + MS_EXCEPTION_IF_NULL(output); + bn_outputs->push_back(output); + } + sort(bn_outputs->begin(), bn_outputs->end(), CompareTupleGetitem); +} +} // namespace + +const BaseRef FusedBatchNormFusion::DefinePattern() const { + const auto prim_batch_norm = std::make_shared(kBatchNormOpName); + std::shared_ptr Xs = std::make_shared(); + VarPtr index0 = std::make_shared(IsC); + VarPtr index1 = std::make_shared(IsC); + VarPtr index2 = std::make_shared(IsC); + VectorRef batch_norm = VectorRef({prim_batch_norm, data_input_var0_, data_input_var1_, data_input_var2_, Xs}); + VectorRef tuple_getitem0 = VectorRef({prim::kPrimTupleGetItem, batch_norm, index0}); + VectorRef tuple_getitem1 = VectorRef({prim::kPrimTupleGetItem, batch_norm, index1}); + VectorRef tuple_getitem2 = VectorRef({prim::kPrimTupleGetItem, batch_norm, index2}); + VectorRef sub0 = VectorRef({prim::kPrimSub, variable_input_var0_, tuple_getitem1}); + VectorRef sub1 = VectorRef({prim::kPrimSub, variable_input_var1_, tuple_getitem2}); + VectorRef mul0 = VectorRef({prim::kPrimMul, sub0, constant_input_var0_}); + VectorRef mul1 = VectorRef({prim::kPrimMul, sub1, constant_input_var1_}); + VectorRef assign_sub0 = VectorRef({prim::kPrimAssignSub, variable_input_var0_, mul0}); + VectorRef assign_sub1 = VectorRef({prim::kPrimAssignSub, variable_input_var1_, mul1}); + VectorRef depend0 = VectorRef({prim::kPrimDepend, tuple_getitem0, assign_sub0}); + return VectorRef({prim::kPrimDepend, depend0, assign_sub1}); +} + +abstract::AbstractTuplePtr FusedBatchNormFusion::CreateAbstractOfFusedBatchNorm(const EquivPtr &equiv, + const AnfNodePtr &bn) const { + MS_EXCEPTION_IF_NULL(equiv); + MS_EXCEPTION_IF_NULL(bn); + auto variable_input0 = utils::cast((*equiv)[variable_input_var0_]); + MS_EXCEPTION_IF_NULL(variable_input0); + auto variable_input1 = utils::cast((*equiv)[variable_input_var1_]); + MS_EXCEPTION_IF_NULL(variable_input1); + auto bn_abstract_tuple = dyn_cast(bn->abstract()); + MS_EXCEPTION_IF_NULL(bn_abstract_tuple); + if (bn_abstract_tuple->elements().size() != kBnOutputNum) { + MS_LOG(EXCEPTION) << "The abstract size of node bn must be " << kBnOutputNum << ", but it is " + << bn_abstract_tuple->elements().size(); + } + AbstractBasePtrList fused_bn_abstract_list{bn_abstract_tuple->elements()[0], variable_input0->abstract(), + variable_input1->abstract(), bn_abstract_tuple->elements()[3], + bn_abstract_tuple->elements()[4]}; + auto abstract_tuple = std::make_shared(fused_bn_abstract_list); + return abstract_tuple; +} + +ValuePtr FusedBatchNormFusion::GetFactor(const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(equiv); + auto constant_input = utils::cast((*equiv)[constant_input_var0_]); + MS_EXCEPTION_IF_NULL(constant_input); + if (!constant_input->isa()) { + return nullptr; + } + auto value_node = constant_input->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + if (!value->isa()) { + return nullptr; + } + auto tensor_ptr = value->cast(); + MS_EXCEPTION_IF_NULL(tensor_ptr); + auto *tensor_data = static_cast(tensor_ptr->data_c()); + MS_EXCEPTION_IF_NULL(tensor_data); + return MakeValue(tensor_data[0]); +} + +const AnfNodePtr FusedBatchNormFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + // Set inputs + auto data_input0 = utils::cast((*equiv)[data_input_var0_]); + MS_EXCEPTION_IF_NULL(data_input0); + auto data_input1 = utils::cast((*equiv)[data_input_var1_]); + MS_EXCEPTION_IF_NULL(data_input1); + auto data_input2 = utils::cast((*equiv)[data_input_var2_]); + MS_EXCEPTION_IF_NULL(data_input2); + auto variable_input0 = utils::cast((*equiv)[variable_input_var0_]); + MS_EXCEPTION_IF_NULL(variable_input0); + auto variable_input1 = utils::cast((*equiv)[variable_input_var1_]); + MS_EXCEPTION_IF_NULL(variable_input1); + std::vector fused_bn_inputs = { + NewValueNode(prim::kPrimFusedBatchNorm), data_input0, data_input1, data_input2, variable_input0, variable_input1}; + auto fused_bn = func_graph->NewCNode(fused_bn_inputs); + fused_bn->set_scope(node->scope()); + MS_EXCEPTION_IF_NULL(fused_bn); + // Set abstract + AnfNodePtr bn = GetBatchNormNode(node); + fused_bn->set_abstract(CreateAbstractOfFusedBatchNorm(equiv, bn)); + // Set attr + AnfAlgo::CopyNodeAttr(kAttrEpsilon, bn, fused_bn); + ValuePtr factor = GetFactor(equiv); + if (factor == nullptr) { + return nullptr; + } + AnfAlgo::SetNodeAttr(kAttrMomentum, factor, fused_bn); + // Replace old nodes with outputs of fused_bn + std::vector fused_bn_outputs; + CreateMultipleOutputsOfAnfNode(func_graph, fused_bn, kBnOutputNum, &fused_bn_outputs); + if (fused_bn_outputs.size() != kBnOutputNum) { + MS_LOG(EXCEPTION) << "The output size of node bn must be " << kBnOutputNum << ", but it is " + << fused_bn_outputs.size(); + } + std::vector bn_outputs; + GetBNOutput(func_graph, bn, &bn_outputs); + if (bn_outputs.size() != kBnOutputNum) { + MS_LOG(EXCEPTION) << "The output size of node bn must be " << kBnOutputNum << ", but it is " << bn_outputs.size(); + } + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(bn_outputs[3], fused_bn_outputs[3]); + (void)manager->Replace(bn_outputs[4], fused_bn_outputs[4]); + return fused_bn_outputs[0]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.h new file mode 100644 index 00000000000..db25e4f9f53 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/fused_batch_norm_fusion.h @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_FUSED_BATCH_NORM_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_FUSED_BATCH_NORM_FUSION_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class FusedBatchNormFusion : public PatternProcessPass { + public: + explicit FusedBatchNormFusion(bool multigraph = true) + : PatternProcessPass("fused_batch_norm_fusion", multigraph), + data_input_var0_(std::make_shared()), + data_input_var1_(std::make_shared()), + data_input_var2_(std::make_shared()), + variable_input_var0_(std::make_shared()), + variable_input_var1_(std::make_shared()), + constant_input_var0_(std::make_shared()), + constant_input_var1_(std::make_shared()) {} + ~FusedBatchNormFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + abstract::AbstractTuplePtr CreateAbstractOfFusedBatchNorm(const EquivPtr &equiv, const AnfNodePtr &bn) const; + + ValuePtr GetFactor(const EquivPtr &equiv) const; + + VarPtr data_input_var0_; + VarPtr data_input_var1_; + VarPtr data_input_var2_; + VarPtr variable_input_var0_; + VarPtr variable_input_var1_; + VarPtr constant_input_var0_; + VarPtr constant_input_var1_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_FUSED_BATCH_NORM_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_rule.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_rule.cc new file mode 100644 index 00000000000..f0a55c6ff49 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_rule.cc @@ -0,0 +1,175 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_rule.h" +#include +#include +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace { +std::tuple GetSharedNodesByPattern(const AnfNodePtr &node) { + auto add3_cnode = CheckAnfNodeIfCNodeAndInputSize(node, kAddInputNum); + MS_EXCEPTION_IF_NULL(add3_cnode); + auto real_div2_cnode = CheckAnfNodeIfCNodeAndInputSize(add3_cnode->input(1), kMulInputNum); + MS_EXCEPTION_IF_NULL(real_div2_cnode); + auto real_div0_cnode = CheckAnfNodeIfCNodeAndInputSize(real_div2_cnode->input(1), kRealDivInputNum); + MS_EXCEPTION_IF_NULL(real_div0_cnode); + auto sqrt0_cnode = CheckAnfNodeIfCNodeAndInputSize(real_div2_cnode->input(2), kSqrtInputNum); + MS_EXCEPTION_IF_NULL(sqrt0_cnode); + auto add2_cnode = CheckAnfNodeIfCNodeAndInputSize(sqrt0_cnode->input(1), kAddInputNum); + MS_EXCEPTION_IF_NULL(add2_cnode); + auto real_div1_cnode = CheckAnfNodeIfCNodeAndInputSize(add2_cnode->input(1), kRealDivInputNum); + auto constant_add2_y = add2_cnode->input(2); + + return std::make_tuple(real_div0_cnode, real_div1_cnode, constant_add2_y); +} + +bool MatchRealDiv4(const AnfNodePtr &real_div4, const AnfNodePtr &real_div1, const AnfNodePtr &constant_add2_y) { + if (real_div4 == nullptr || !real_div4->isa()) { + return false; + } + auto real_div4_cnode = real_div4->cast(); + MS_EXCEPTION_IF_NULL(real_div4_cnode); + if (AnfAlgo::GetCNodeName(real_div4_cnode) != kRealDivOpName || real_div4_cnode->inputs().size() < kRealDivInputNum) { + return false; + } + + CNodePtr add4_cnode = nullptr; + if (!CheckIfCNodeAndInputSize(real_div4_cnode->input(2), kAddInputNum, &add4_cnode) || + AnfAlgo::GetCNodeName(add4_cnode) != prim::kPrimTensorAdd->name()) { + return false; + } + CNodePtr sqrt1_cnode = nullptr; + if (!CheckIfCNodeAndInputSize(add4_cnode->input(1), kSqrtInputNum, &sqrt1_cnode) || + AnfAlgo::GetCNodeName(sqrt1_cnode) != kSqrtOpName) { + return false; + } + + MS_EXCEPTION_IF_NULL(add4_cnode->input(2)); + MS_EXCEPTION_IF_NULL(constant_add2_y); + return sqrt1_cnode->input(1) == real_div1 && *(add4_cnode->input(2)) == *constant_add2_y; +} +} // namespace + +const BaseRef LambNextMVRule::DefinePattern() const { + const auto prim_rsqrt = std::make_shared(kRsqrtOpName); + MS_EXCEPTION_IF_NULL(prim_rsqrt); + const auto prim_deal_div = std::make_shared(kRealDivOpName); + MS_EXCEPTION_IF_NULL(prim_deal_div); + + auto mul0 = VectorRef({prim::kPrimMul, input_varptr_[7], input_varptr_[4]}); + auto mul1 = VectorRef({prim::kPrimMul, input_varptr_[8], input_varptr_[3]}); + auto mul2 = VectorRef({prim::kPrimMul, input_varptr_[9], input_varptr_[1]}); + auto mul3 = VectorRef({prim::kPrimMul, input_varptr_[10], input_varptr_[0]}); + auto mul4 = VectorRef({prim::kPrimMul, input_varptr_[11], input_varptr_[6]}); + auto add0 = VectorRef({prim::kPrimTensorAdd, mul0, mul1}); + auto add1 = VectorRef({prim::kPrimTensorAdd, mul2, mul3}); + + auto real_div0 = VectorRef({prim_deal_div, add0, input_varptr_[5]}); + auto real_div1 = VectorRef({prim_deal_div, add1, input_varptr_[2]}); + + auto add2 = VectorRef({prim::kPrimTensorAdd, real_div1, input_varptr_[12]}); + auto sqrt0 = VectorRef({prim_rsqrt, add2}); + auto real_div2 = VectorRef({prim::kPrimMul, real_div0, sqrt0}); + + return VectorRef({prim::kPrimTensorAdd, real_div2, mul4}); +} + +bool LambNextMVRule::IsRuleMatched(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + std::vector *old_pattern_outputs) const { + MS_EXCEPTION_IF_NULL(func_graph); + CNodePtr real_div0 = nullptr; + CNodePtr real_div1 = nullptr; + AnfNodePtr constant_add2_y = nullptr; + std::tie(real_div0, real_div1, constant_add2_y) = GetSharedNodesByPattern(node); + + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto &users = manager->node_users(); + if (users.find(real_div0) == users.end() || users[real_div0].size() < 2) { + return false; + } + AnfNodeIndexSet real_div0_outputs = users[real_div0]; + auto iter = std::find_if(real_div0_outputs.begin(), real_div0_outputs.end(), + [&node, &real_div1, &constant_add2_y](const std::pair &node_index) { + return node_index.first != node && node_index.second == 1 && + MatchRealDiv4(node_index.first, real_div1, constant_add2_y); + }); + if (iter == real_div0_outputs.end()) { + return false; + } + + auto add0_cnode = CheckAnfNodeIfCNodeAndInputSize(real_div0->input(1), kAddInputNum); + auto add1_cnode = CheckAnfNodeIfCNodeAndInputSize(real_div1->input(1), kAddInputNum); + (*old_pattern_outputs).push_back(node); + (*old_pattern_outputs).push_back(add0_cnode); + (*old_pattern_outputs).push_back(add1_cnode); + (*old_pattern_outputs).push_back(iter->first); + + return true; +} + +AnfNodePtr LambNextMVRule::CreateLambNextMVNode(const FuncGraphPtr &func_graph, + const std::vector &old_pattern_outputs, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + auto prim = std::make_shared(kLambNextMVOpName); + std::vector lamb_next_mv_rule_inputs = {NewValueNode(prim)}; + (void)std::transform(input_varptr_.begin(), input_varptr_.end(), std::back_inserter(lamb_next_mv_rule_inputs), + [&equiv](const VarPtr &in) { return utils::cast((*equiv)[in]); }); + auto lamb_next_mv_rule = func_graph->NewCNode(lamb_next_mv_rule_inputs); + MS_EXCEPTION_IF_NULL(lamb_next_mv_rule); + + // Set abstract of new node + AbstractBasePtrList new_abstracts; + (void)std::transform(old_pattern_outputs.begin(), old_pattern_outputs.end(), std::back_inserter(new_abstracts), + [](const AnfNodePtr &out) { return out->abstract(); }); + auto abstract_tuple = std::make_shared(new_abstracts); + MS_EXCEPTION_IF_NULL(abstract_tuple); + lamb_next_mv_rule->set_abstract(abstract_tuple); + + // Create tuple_getitem node for outputs + std::vector lamb_next_mv_rule_outputs; + CreateMultipleOutputsOfAnfNode(func_graph, lamb_next_mv_rule, kLambNextMVRuleOutputNum, &lamb_next_mv_rule_outputs); + + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(old_pattern_outputs[1], lamb_next_mv_rule_outputs[1]); + (void)manager->Replace(old_pattern_outputs[2], lamb_next_mv_rule_outputs[2]); + (void)manager->Replace(old_pattern_outputs[3], lamb_next_mv_rule_outputs[3]); + + return lamb_next_mv_rule_outputs[0]; +} + +const AnfNodePtr LambNextMVRule::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + std::vector old_pattern_outputs; + if (!IsRuleMatched(func_graph, node, &old_pattern_outputs)) { + return nullptr; + } + + return CreateLambNextMVNode(func_graph, old_pattern_outputs, equiv); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_rule.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_rule.h new file mode 100644 index 00000000000..33fb41662d2 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_rule.h @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_RULE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_RULE_H_ + +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class LambNextMVRule : public PatternProcessPass { + public: + explicit LambNextMVRule(bool multigraph = true) : PatternProcessPass("lamb_next_mv_rule", multigraph) { + for (size_t i = 0; i < kLambNextMVRuleInputNum - 1; ++i) { + input_varptr_.push_back(std::make_shared()); + } + } + ~LambNextMVRule() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + std::vector input_varptr_; + bool IsRuleMatched(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + std::vector *old_pattern_outputs) const; + AnfNodePtr CreateLambNextMVNode(const FuncGraphPtr &func_graph, const std::vector &old_pattern_outputs, + const EquivPtr &equiv) const; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_RULE_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc new file mode 100644 index 00000000000..58efbaf7100 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.cc @@ -0,0 +1,220 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.h" +#include +#include "session/anf_runtime_algorithm.h" +#include "optimizer/opt.h" + +namespace mindspore { +namespace opt { +namespace { +AnfNodePtr GetLambNextMVWithDecayOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &new_node, + const AnfNodePtr &add3, const AnfNodePtr &add5, const AnfNodePtr &real_div0, + const AnfNodePtr &real_div1) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(new_node); + MS_EXCEPTION_IF_NULL(add3); + MS_EXCEPTION_IF_NULL(real_div0); + MS_EXCEPTION_IF_NULL(real_div1); + MS_EXCEPTION_IF_NULL(add5); + // Set abstract of new node + AbstractBasePtrList new_node_list; + new_node_list.push_back(add3->abstract()); + auto real_div0_cnode = real_div0->cast(); + MS_EXCEPTION_IF_NULL(real_div0_cnode); + AnfNodePtr add0 = real_div0_cnode->input(1); + MS_EXCEPTION_IF_NULL(add0); + new_node_list.push_back(add0->abstract()); + auto real_div1_cnode = real_div1->cast(); + MS_EXCEPTION_IF_NULL(real_div1_cnode); + AnfNodePtr add1 = real_div1_cnode->input(1); + MS_EXCEPTION_IF_NULL(add1); + new_node_list.push_back(add1->abstract()); + new_node_list.push_back(add5->abstract()); + auto abstract_tuple = std::make_shared(new_node_list); + MS_EXCEPTION_IF_NULL(abstract_tuple); + new_node->set_abstract(abstract_tuple); + // Create tuple_getitem node for outputs + std::vector new_node_outputs; + CreateMultipleOutputsOfAnfNode(func_graph, new_node, kLambNextMVWithDecayOutputNum, &new_node_outputs); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(add3, new_node_outputs[0]); + (void)manager->Replace(add0, new_node_outputs[1]); + (void)manager->Replace(add1, new_node_outputs[2]); + return new_node_outputs[3]; +} + +void GetSharedInputNodesByAdd5(const AnfNodePtr &node, AnfNodePtr *mul4, AnfNodePtr *real_div0, AnfNodePtr *real_div1, + AnfNodePtr *constant_add2_y_input) { + MS_EXCEPTION_IF_NULL(node); + auto add5_cnode = node->cast(); + MS_EXCEPTION_IF_NULL(add5_cnode); + if (add5_cnode->inputs().size() < kAddInputNum) { + MS_LOG(EXCEPTION) << "The input size of Add5 is less than " << kAddInputNum; + } + *mul4 = add5_cnode->input(2); + + AnfNodePtr real_div4 = add5_cnode->input(1); + MS_EXCEPTION_IF_NULL(real_div4); + auto real_div4_cnode = real_div4->cast(); + MS_EXCEPTION_IF_NULL(real_div4_cnode); + if (real_div4_cnode->inputs().size() < kRealDivInputNum) { + MS_LOG(EXCEPTION) << "The input size of RealDiv4 is less than " << kRealDivInputNum; + } + *real_div0 = real_div4_cnode->input(1); + + AnfNodePtr add4 = real_div4_cnode->input(2); + MS_EXCEPTION_IF_NULL(add4); + auto add4_cnode = add4->cast(); + MS_EXCEPTION_IF_NULL(add4_cnode); + if (add4_cnode->inputs().size() < kAddInputNum) { + MS_LOG(EXCEPTION) << "The input size of Add4 is less than " << kAddInputNum; + } + AnfNodePtr sqrt1 = add4_cnode->input(1); + MS_EXCEPTION_IF_NULL(sqrt1); + auto sqrt1_cnode = sqrt1->cast(); + MS_EXCEPTION_IF_NULL(sqrt1_cnode); + if (sqrt1_cnode->inputs().size() < kSqrtInputNum) { + MS_LOG(EXCEPTION) << "The input size of Sqrt1 is less than " << kSqrtInputNum; + } + *real_div1 = sqrt1_cnode->input(1); + *constant_add2_y_input = add4_cnode->input(2); +} + +bool MatchAdd3(const AnfNodePtr &add3, const AnfNodePtr &mul4, const AnfNodePtr &real_div0, const AnfNodePtr &real_div1, + const AnfNodePtr &constant_add2_y) { + if (add3 == nullptr || !add3->isa()) { + return false; + } + auto add3_cnode = add3->cast(); + MS_EXCEPTION_IF_NULL(add3_cnode); + if (AnfAlgo::GetCNodeName(add3_cnode) != prim::kPrimTensorAdd->name() || + add3_cnode->inputs().size() != kAddInputNum) { + return false; + } + // Check the shared input nodes. + if (add3_cnode->input(2) != mul4) { + return false; + } + AnfNodePtr real_div2 = add3_cnode->input(1); + MS_EXCEPTION_IF_NULL(real_div2); + auto real_div2_cnode = real_div2->cast(); + MS_EXCEPTION_IF_NULL(real_div2_cnode); + if (AnfAlgo::GetCNodeName(real_div2_cnode) != prim::kPrimMul->name() || + real_div2_cnode->inputs().size() != kMulInputNum) { + return false; + } + if (real_div2_cnode->input(1) != real_div0) { + return false; + } + AnfNodePtr sqrt0 = real_div2_cnode->input(2); + MS_EXCEPTION_IF_NULL(sqrt0); + auto sqrt0_cnode = sqrt0->cast(); + MS_EXCEPTION_IF_NULL(sqrt0_cnode); + if (AnfAlgo::GetCNodeName(sqrt0_cnode) != kRsqrtOpName || sqrt0_cnode->inputs().size() != kRsqrtInputNum) { + return false; + } + AnfNodePtr add2 = sqrt0_cnode->input(1); + MS_EXCEPTION_IF_NULL(add2); + auto add2_cnode = add2->cast(); + MS_EXCEPTION_IF_NULL(add2_cnode); + if (AnfAlgo::GetCNodeName(add2_cnode) != prim::kPrimTensorAdd->name() || + add2_cnode->inputs().size() != kAddInputNum) { + return false; + } + MS_EXCEPTION_IF_NULL(add2_cnode->input(2)); + MS_EXCEPTION_IF_NULL(constant_add2_y); + return add2_cnode->input(1) == real_div1 && *(add2_cnode->input(2)) == *constant_add2_y; +} +} // namespace + +AnfNodePtr LambNextMVWithDecayRule::CreateLambNextMVWithDecayNode(const FuncGraphPtr &func_graph, + const AnfNodePtr &add3, const AnfNodePtr &add5, + const AnfNodePtr &real_div0, + const AnfNodePtr &real_div1, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(add3); + MS_EXCEPTION_IF_NULL(equiv); + // Create new node with all the inputs + auto prim = std::make_shared(kLambNextMVWithDecayOpName); + std::vector new_node_inputs = {NewValueNode(prim)}; + for (size_t i = 0; i < kLambNextMVWithDecayInputNum; ++i) { + auto input_node = utils::cast((*equiv)[input_vars_[i]]); + MS_EXCEPTION_IF_NULL(input_node); + new_node_inputs.push_back(input_node); + } + for (size_t i = 0; i < kLambNextMVWithDecayConstantMulInputNum; ++i) { + auto constant_mul_input_node = utils::cast((*equiv)[constant_mul_input_vars_[i]]); + MS_EXCEPTION_IF_NULL(constant_mul_input_node); + new_node_inputs.push_back(constant_mul_input_node); + } + auto constant_add2_y_node = utils::cast((*equiv)[constant_add2_y_]); + MS_EXCEPTION_IF_NULL(constant_add2_y_node); + new_node_inputs.push_back(constant_add2_y_node); + auto new_node = func_graph->NewCNode(new_node_inputs); + return GetLambNextMVWithDecayOutput(func_graph, new_node, add3, add5, real_div0, real_div1); +} + +const BaseRef LambNextMVWithDecayRule::DefinePattern() const { + const auto prim_sqrt = std::make_shared(kSqrtOpName); + MS_EXCEPTION_IF_NULL(prim_sqrt); + const auto prim_deal_div = std::make_shared(kRealDivOpName); + MS_EXCEPTION_IF_NULL(prim_deal_div); + VectorRef mul4 = VectorRef({prim::kPrimMul, constant_mul_input_vars_[4], input_vars_[6]}); + VectorRef add0 = + VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, constant_mul_input_vars_[0], input_vars_[4]}), + VectorRef({prim::kPrimMul, constant_mul_input_vars_[1], input_vars_[3]})}); + VectorRef real_div0 = VectorRef({prim_deal_div, add0, input_vars_[5]}); + VectorRef add1 = + VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, constant_mul_input_vars_[2], input_vars_[1]}), + VectorRef({prim::kPrimMul, constant_mul_input_vars_[3], input_vars_[0]})}); + VectorRef real_div1 = VectorRef({prim_deal_div, add1, input_vars_[2]}); + VectorRef real_div4 = VectorRef( + {prim_deal_div, real_div0, VectorRef({prim::kPrimTensorAdd, VectorRef({prim_sqrt, real_div1}), constant_add2_y_})}); + return VectorRef({prim::kPrimTensorAdd, real_div4, mul4}); +} + +const AnfNodePtr LambNextMVWithDecayRule::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + // Get the shared input nodes in patterns of add5 and add3 + AnfNodePtr mul4 = nullptr; + AnfNodePtr real_div0 = nullptr; + AnfNodePtr real_div1 = nullptr; + AnfNodePtr constant_add2_y_input = nullptr; + GetSharedInputNodesByAdd5(node, &mul4, &real_div0, &real_div1, &constant_add2_y_input); + // Get add3 and try to match the add3 pattern + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(mul4) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"; + } + AnfNodeIndexSet mul4_output_node_index_set = manager->node_users()[mul4]; + auto iter = std::find_if( + mul4_output_node_index_set.begin(), mul4_output_node_index_set.end(), + [&node, &mul4, &real_div0, &real_div1, &constant_add2_y_input](const std::pair &node_index) { + return node_index.first != node && MatchAdd3(node_index.first, mul4, real_div0, real_div1, constant_add2_y_input); + }); + if (iter != mul4_output_node_index_set.end()) { + return CreateLambNextMVWithDecayNode(func_graph, iter->first, node, real_div0, real_div1, equiv); + } + return nullptr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.h new file mode 100644 index 00000000000..161ce4e9569 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.h @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_WITH_DECAY_RULE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_WITH_DECAY_RULE_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class LambNextMVWithDecayRule : public PatternProcessPass { + public: + explicit LambNextMVWithDecayRule(bool multigraph = true) + : PatternProcessPass("lamb_next_mv_with_decay_rule", multigraph) { + for (size_t i = 0; i < kLambNextMVWithDecayInputNum; ++i) { + input_vars_.push_back(std::make_shared()); + } + for (size_t i = 0; i < kLambNextMVWithDecayConstantMulInputNum; ++i) { + constant_mul_input_vars_.push_back(std::make_shared()); + } + constant_add2_y_ = std::make_shared(); + } + + ~LambNextMVWithDecayRule() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + AnfNodePtr CreateLambNextMVWithDecayNode(const FuncGraphPtr &func_graph, const AnfNodePtr &add3, + const AnfNodePtr &add5, const AnfNodePtr &real_div0, + const AnfNodePtr &real_div1, const EquivPtr &equiv) const; + + std::vector input_vars_; + std::vector constant_mul_input_vars_; + VarPtr constant_add2_y_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_WITH_DECAY_RULE_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc new file mode 100644 index 00000000000..9efd5033639 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.cc @@ -0,0 +1,205 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.h" + +#include +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "optimizer/opt.h" + +namespace mindspore { +namespace opt { +namespace { +std::tuple GetSharedNodes(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto add3 = node->cast(); + MS_EXCEPTION_IF_NULL(add3); + if (add3->inputs().size() < kAddInputNum) { + MS_LOG(EXCEPTION) << "The input size of Add3 is less than " << kAddInputNum; + } + auto real_div2_anf = add3->input(1); + MS_EXCEPTION_IF_NULL(real_div2_anf); + auto real_div2 = real_div2_anf->cast(); + MS_EXCEPTION_IF_NULL(real_div2); + if (real_div2->inputs().size() < kRealDivInputNum) { + MS_LOG(EXCEPTION) << "The input size of RealDiv2 is less than " << kRealDivInputNum; + } + auto sqrt0_anf = real_div2->input(2); + MS_EXCEPTION_IF_NULL(sqrt0_anf); + auto sqrt0 = sqrt0_anf->cast(); + MS_EXCEPTION_IF_NULL(sqrt0); + if (sqrt0->inputs().size() < kRsqrtInputNum) { + MS_LOG(EXCEPTION) << "The input size of Sqrt0 is less than " << kSqrtInputNum; + } + auto add2_anf = sqrt0->input(1); + MS_EXCEPTION_IF_NULL(add2_anf); + auto add2 = add2_anf->cast(); + if (add2->inputs().size() < kAddInputNum) { + MS_LOG(EXCEPTION) << "The input size of Add2 is less than " << kAddInputNum; + } + return std::make_tuple(add3->input(2), real_div2->input(1), add2->input(1), add2->input(2)); +} + +bool MatchAdd5Pattern(const AnfNodePtr &node, const AnfNodePtr &mul4, const AnfNodePtr &real_div0, + const AnfNodePtr &real_div1, const AnfNodePtr &add2_y) { + if (node == nullptr || !node->isa()) { + return false; + } + auto add5 = node->cast(); + if (AnfAlgo::GetCNodeName(add5) != prim::kPrimTensorAdd->name() || add5->inputs().size() != kAddInputNum) { + return false; + } + auto real_div4_anf = add5->input(1); + if (real_div4_anf == nullptr || !real_div4_anf->isa()) { + return false; + } + auto real_div4 = real_div4_anf->cast(); + if (AnfAlgo::GetCNodeName(real_div4) != kRealDivOpName || real_div4->inputs().size() != kRealDivInputNum) { + return false; + } + auto add4_anf = real_div4->input(2); + if (add4_anf == nullptr || !add4_anf->isa()) { + return false; + } + auto add4 = add4_anf->cast(); + if (AnfAlgo::GetCNodeName(add4) != prim::kPrimTensorAdd->name() || add4->inputs().size() != kAddInputNum) { + return false; + } + auto sqrt1_anf = add4->input(1); + if (sqrt1_anf == nullptr || !sqrt1_anf->isa()) { + return false; + } + auto sqrt1 = sqrt1_anf->cast(); + if (AnfAlgo::GetCNodeName(sqrt1) != kSqrtOpName || sqrt1->inputs().size() != kSqrtInputNum) { + return false; + } + return add5->input(2) == mul4 && real_div4->input(1) == real_div0 && sqrt1->input(1) == real_div1 && + *add4->input(2) == *add2_y; +} + +std::tuple GetAdd0Add1Nodes(const AnfNodePtr &real_div0_anf, const AnfNodePtr &real_div1_anf) { + MS_EXCEPTION_IF_NULL(real_div0_anf); + MS_EXCEPTION_IF_NULL(real_div1_anf); + auto real_div0 = real_div0_anf->cast(); + auto real_div1 = real_div1_anf->cast(); + MS_EXCEPTION_IF_NULL(real_div0); + MS_EXCEPTION_IF_NULL(real_div1); + if (real_div0->inputs().size() != kRealDivInputNum) { + MS_LOG(EXCEPTION) << "RealDiv0 has wrong input size"; + } + if (real_div1->inputs().size() != kRealDivInputNum) { + MS_LOG(EXCEPTION) << "RealDiv1 has wrong input size"; + } + return std::make_tuple(real_div0->input(1), real_div1->input(1)); +} +} // namespace + +std::vector LambNextMVWithDecayV1Rule::GetFusionNodeInputs(const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(equiv); + auto i0 = utils::cast((*equiv)[input0_]); + auto i1 = utils::cast((*equiv)[input1_]); + auto i2 = utils::cast((*equiv)[input2_]); + auto i3 = utils::cast((*equiv)[input3_]); + auto i4 = utils::cast((*equiv)[input4_]); + auto i5 = utils::cast((*equiv)[input5_]); + auto i6 = utils::cast((*equiv)[input6_]); + auto i7 = utils::cast((*equiv)[mul0_x_]); + auto i8 = utils::cast((*equiv)[mul1_sub_]); + auto i9 = utils::cast((*equiv)[mul2_x_]); + auto i10 = utils::cast((*equiv)[mul3_sub1_]); + auto i11 = utils::cast((*equiv)[mul4_x_]); + auto i12 = utils::cast((*equiv)[add2_y_]); + auto prim = std::make_shared(kLambNextMVWithDecayV1OpName); + return {NewValueNode(prim), i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12}; +} + +const BaseRef LambNextMVWithDecayV1Rule::DefinePattern() const { + const auto prim_rsqrt = std::make_shared(kRsqrtOpName); + const auto prim_real_div = std::make_shared(kRealDivOpName); + VectorRef mul3({prim::kPrimMul, mul3_sub1_, input0_}); + VectorRef mul2({prim::kPrimMul, mul2_x_, input1_}); + VectorRef add1({prim::kPrimTensorAdd, mul2, mul3}); + VectorRef real_div1({prim_real_div, add1, input2_}); + VectorRef add2({prim::kPrimTensorAdd, real_div1, add2_y_}); + VectorRef mul0({prim::kPrimMul, mul0_x_, input4_}); + VectorRef mul1({prim::kPrimMul, mul1_sub_, input3_}); + VectorRef sqrt0({prim_rsqrt, add2}); + VectorRef add0({prim::kPrimTensorAdd, mul0, mul1}); + VectorRef real_div0({prim_real_div, add0, input5_}); + VectorRef real_div2({prim::kPrimMul, real_div0, sqrt0}); + VectorRef mul4({prim::kPrimMul, mul4_x_, input6_}); + VectorRef add3({prim::kPrimTensorAdd, real_div2, mul4}); + return add3; +} + +const AnfNodePtr LambNextMVWithDecayV1Rule::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + if (func_graph == nullptr || node == nullptr || equiv == nullptr) { + return nullptr; + } + AnfNodePtr mul4 = nullptr; + AnfNodePtr real_div0 = nullptr; + AnfNodePtr real_div1 = nullptr; + AnfNodePtr add2_y = nullptr; + std::tie(mul4, real_div0, real_div1, add2_y) = GetSharedNodes(node); + + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(mul4) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "The Mul4 should be used by at least another node input"; + } + AnfNodeIndexSet mul4_output_node_index_set = manager->node_users()[mul4]; + auto iter = std::find_if( + mul4_output_node_index_set.begin(), mul4_output_node_index_set.end(), + [&node, &mul4, &real_div0, &real_div1, &add2_y](const std::pair &node_index) { + return node_index.first != node && MatchAdd5Pattern(node_index.first, mul4, real_div0, real_div1, add2_y); + }); + if (iter == mul4_output_node_index_set.end()) { + return nullptr; + } + + std::vector inputs = GetFusionNodeInputs(equiv); + auto fusion_node = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(fusion_node); + fusion_node->set_scope(node->scope()); + + AnfNodePtr add0 = nullptr; + AnfNodePtr add1 = nullptr; + AnfNodePtr add5 = iter->first; + std::tie(add0, add1) = GetAdd0Add1Nodes(real_div0, real_div1); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0), AnfAlgo::GetOutputInferDataType(add0, 0), + AnfAlgo::GetOutputInferDataType(add1, 0), AnfAlgo::GetOutputInferDataType(add5, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0), AnfAlgo::GetOutputInferShape(add0, 0), + AnfAlgo::GetOutputInferShape(add1, 0), AnfAlgo::GetOutputInferShape(add5, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, fusion_node.get()); + + std::vector fusion_node_outputs; + CreateMultipleOutputsOfAnfNode(func_graph, fusion_node, kLambNextMVWithDecayV1OutputNum, &fusion_node_outputs); + if (fusion_node_outputs.size() != kLambNextMVWithDecayV1OutputNum) { + MS_LOG(ERROR) << "create multiple outputs for fusion node fail!"; + return nullptr; + } + + (void)manager->Replace(add0, fusion_node_outputs[1]); + (void)manager->Replace(add1, fusion_node_outputs[2]); + (void)manager->Replace(add5, fusion_node_outputs[3]); + return fusion_node_outputs[0]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.h new file mode 100644 index 00000000000..ff14a253ddc --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.h @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_WITH_DECAY_V1_RULE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_WITH_DECAY_V1_RULE_H_ + +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +class LambNextMVWithDecayV1Rule : public PatternProcessPass { + public: + explicit LambNextMVWithDecayV1Rule(bool multigraph = true) + : PatternProcessPass("lamb_next_mv_with_decay_v1_rule", multigraph) { + input0_ = std::make_shared(); + input1_ = std::make_shared(); + input2_ = std::make_shared(); + input3_ = std::make_shared(); + input4_ = std::make_shared(); + input5_ = std::make_shared(); + input6_ = std::make_shared(); + mul0_x_ = std::make_shared(); + mul1_sub_ = std::make_shared(); + mul2_x_ = std::make_shared(); + mul3_sub1_ = std::make_shared(); + mul4_x_ = std::make_shared(); + add2_y_ = std::make_shared(); + } + + ~LambNextMVWithDecayV1Rule() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + std::vector GetFusionNodeInputs(const EquivPtr &equiv) const; + VarPtr input0_; + VarPtr input1_; + VarPtr input2_; + VarPtr input3_; + VarPtr input4_; + VarPtr input5_; + VarPtr input6_; + VarPtr mul0_x_; + VarPtr mul1_sub_; + VarPtr mul2_x_; + VarPtr mul3_sub1_; + VarPtr mul4_x_; + VarPtr add2_y_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_MV_WITH_DECAY_V1_RULE_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_right_rule.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_right_rule.cc new file mode 100644 index 00000000000..ca9c90f4e5b --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_right_rule.cc @@ -0,0 +1,111 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/lamb_next_right_rule.h" +#include +#include "pre_activate/common/helper.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +namespace { +AnfNodePtr GetAdd1Node(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto add2_cnode = node->cast(); + MS_EXCEPTION_IF_NULL(add2_cnode); + if (add2_cnode->inputs().size() != kAddInputNum) { + MS_LOG(ERROR) << "The input size of Add2 is not equal to " << kAddInputNum; + } + AnfNodePtr sqrt0 = add2_cnode->input(1); + MS_EXCEPTION_IF_NULL(sqrt0); + auto sqrt0_cnode = sqrt0->cast(); + MS_EXCEPTION_IF_NULL(sqrt0_cnode); + if (sqrt0_cnode->inputs().size() != kSqrtInputNum) { + MS_LOG(ERROR) << "The input size of Sqrt0 is not equal to " << kSqrtInputNum; + } + AnfNodePtr real_div1 = sqrt0_cnode->input(1); + MS_EXCEPTION_IF_NULL(real_div1); + auto real_div1_cnode = real_div1->cast(); + MS_EXCEPTION_IF_NULL(real_div1_cnode); + if (real_div1_cnode->inputs().size() != kMulInputNum) { + MS_LOG(ERROR) << "The input size of RealDiv1 is not equal to " << kMulInputNum; + } + return real_div1_cnode->input(1); +} +} // namespace + +AnfNodePtr LambNextRightRule::CreateLambNextRightNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + std::vector new_node_inputs; + auto prim = std::make_shared(kLambNextRightOpName); + MS_EXCEPTION_IF_NULL(prim); + new_node_inputs.push_back(NewValueNode(prim)); + auto input0 = utils::cast((*equiv)[input0_]); + MS_EXCEPTION_IF_NULL(input0); + new_node_inputs.push_back(input0); + auto input1 = utils::cast((*equiv)[input1_]); + MS_EXCEPTION_IF_NULL(input1); + new_node_inputs.push_back(input1); + auto mul2_x = utils::cast((*equiv)[mul2_x_]); + MS_EXCEPTION_IF_NULL(mul2_x); + new_node_inputs.push_back(mul2_x); + auto mul3_x = utils::cast((*equiv)[mul3_x_]); + MS_EXCEPTION_IF_NULL(mul3_x); + new_node_inputs.push_back(mul3_x); + auto true_div1_recip = utils::cast((*equiv)[true_div1_recip_]); + MS_EXCEPTION_IF_NULL(true_div1_recip); + new_node_inputs.push_back(true_div1_recip); + auto add2_y = utils::cast((*equiv)[add2_y_]); + MS_EXCEPTION_IF_NULL(add2_y); + new_node_inputs.push_back(add2_y); + auto new_node = func_graph->NewCNode(new_node_inputs); + return new_node; +} + +const BaseRef LambNextRightRule::DefinePattern() const { + const auto prim_sqrt = std::make_shared(kSqrtOpName); + MS_EXCEPTION_IF_NULL(prim_sqrt); + VectorRef mul3 = VectorRef({prim::kPrimMul, mul3_x_, VectorRef({prim::kPrimSquare, input0_})}); + VectorRef add1 = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, mul2_x_, input1_}), mul3}); + return VectorRef( + {prim::kPrimTensorAdd, VectorRef({prim_sqrt, VectorRef({prim::kPrimMul, add1, true_div1_recip_})}), add2_y_}); +} + +const AnfNodePtr LambNextRightRule::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + auto new_node = CreateLambNextRightNode(func_graph, equiv); + MS_EXCEPTION_IF_NULL(new_node); + // Set abstract of new node + AnfNodePtr add1 = GetAdd1Node(node); + MS_EXCEPTION_IF_NULL(add1); + AbstractBasePtrList new_node_abstract_list; + new_node_abstract_list.push_back(add1->abstract()); + new_node_abstract_list.push_back(node->abstract()); + auto abstract_tuple = std::make_shared(new_node_abstract_list); + MS_EXCEPTION_IF_NULL(abstract_tuple); + new_node->set_abstract(abstract_tuple); + // Create tuple_getitem node for outputs + std::vector new_node_outputs; + CreateMultipleOutputsOfAnfNode(func_graph, new_node, kLambNextRightOutputNum, &new_node_outputs); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + (void)manager->Replace(add1, new_node_outputs[0]); + return new_node_outputs[1]; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_right_rule.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_right_rule.h new file mode 100644 index 00000000000..f78be7460bb --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_next_right_rule.h @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_RIGHT_RULE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_RIGHT_RULE_H_ + +#include +#include "pre_activate/common/optimizer.h" +namespace mindspore { +namespace opt { +class LambNextRightRule : public PatternProcessPass { + public: + explicit LambNextRightRule(bool multigraph = true) + : PatternProcessPass("lamb_next_right_rule", multigraph), + input0_(std::make_shared()), + input1_(std::make_shared()), + mul2_x_(std::make_shared()), + mul3_x_(std::make_shared()), + true_div1_recip_(std::make_shared()), + add2_y_(std::make_shared()) {} + + ~LambNextRightRule() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + AnfNodePtr CreateLambNextRightNode(const FuncGraphPtr &func_graph, const EquivPtr &equiv) const; + + VarPtr input0_; + VarPtr input1_; + VarPtr mul2_x_; + VarPtr mul3_x_; + VarPtr true_div1_recip_; + VarPtr add2_y_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_NEXT_RIGHT_RULE_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.cc new file mode 100644 index 00000000000..16a43e20720 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.cc @@ -0,0 +1,77 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "common/utils.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +const BaseRef LambUpdateWithLRRuleFusion::DefinePattern() const { + auto real_div = std::make_shared(kRealDivOpName); + MS_EXCEPTION_IF_NULL(real_div); + auto greater = std::make_shared(kGreaterOpName); + MS_EXCEPTION_IF_NULL(greater); + + VectorRef pattern_real_div0({real_div, input1_, input2_}); + VectorRef pattern_greater0({greater, input0_, constant_greater_max_}); + VectorRef pattern_greater1({greater, input1_, constant_greater_max_}); + VectorRef pattern_select0({prim::kPrimSelect, pattern_greater0, pattern_real_div0, constant_select_}); + VectorRef pattern_select1({prim::kPrimSelect, pattern_greater1, pattern_select0, constant_select_}); + VectorRef pattern_minimum0({prim::kPrimMinimum, pattern_select1, constant_minimum_}); + VectorRef pattern_maximum0({prim::kPrimMaximum, pattern_minimum0, constant_greater_max_}); + VectorRef pattern_mul0({prim::kPrimMul, pattern_maximum0, input3_}); + VectorRef pattern_mul1({prim::kPrimMul, pattern_mul0, input4_}); + VectorRef pattern({prim::kPrimSub, input5_, pattern_mul1}); + return pattern; +} + +const AnfNodePtr LambUpdateWithLRRuleFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto input0 = utils::cast((*equiv)[input0_]); + auto input1 = utils::cast((*equiv)[input1_]); + auto input2 = utils::cast((*equiv)[input2_]); + auto input3 = utils::cast((*equiv)[input3_]); + auto input4 = utils::cast((*equiv)[input4_]); + auto input5 = utils::cast((*equiv)[input5_]); + auto input6 = utils::cast((*equiv)[constant_greater_max_]); + auto input7 = utils::cast((*equiv)[constant_select_]); + auto input8 = utils::cast((*equiv)[constant_minimum_]); + + auto prim = std::make_shared(kLambUpdateWithLROpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = { + NewValueNode(prim), input0, input1, input2, input3, input4, input5, input6, input7, input8}; + auto lamb_update_with_lr = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(lamb_update_with_lr); + + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, lamb_update_with_lr.get()); + lamb_update_with_lr->set_scope(node->scope()); + return lamb_update_with_lr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.h new file mode 100644 index 00000000000..cb3939549ff --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.h @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_UPDATE_WITH_LR_RULE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_UPDATE_WITH_LR_RULE_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class LambUpdateWithLRRuleFusion : public PatternProcessPass { + public: + explicit LambUpdateWithLRRuleFusion(bool multigraph = true) + : PatternProcessPass("lamb_update_with_lr_rule_fusion", multigraph) { + input0_ = std::make_shared(); + input1_ = std::make_shared(); + input2_ = std::make_shared(); + input3_ = std::make_shared(); + input4_ = std::make_shared(); + input5_ = std::make_shared(); + constant_greater_max_ = std::make_shared(); + constant_select_ = std::make_shared(); + constant_minimum_ = std::make_shared(); + } + ~LambUpdateWithLRRuleFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr input0_; + VarPtr input1_; + VarPtr input2_; + VarPtr input3_; + VarPtr input4_; + VarPtr input5_; + VarPtr constant_greater_max_; + VarPtr constant_select_; + VarPtr constant_minimum_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_UPDATE_WITH_LR_RULE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.cc new file mode 100644 index 00000000000..069581b6e4a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.h" +#include +#include +#include +#include "utils/utils.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +const BaseRef LambUpdateWithLrV2::DefinePattern() const { + const auto prim_greater = std::make_shared(kGreaterOpName); + const auto prim_deal_div = std::make_shared(kRealDivOpName); + + VectorRef greater0({prim_greater, input_varptr_[0], input_varptr_[5]}); + VectorRef greater1({prim_greater, input_varptr_[1], input_varptr_[5]}); + VectorRef real_div0({prim_deal_div, input_varptr_[0], input_varptr_[1]}); + VectorRef select0({prim::kPrimSelect, greater1, real_div0, input_varptr_[6]}); + VectorRef select1({prim::kPrimSelect, greater0, select0, input_varptr_[6]}); + VectorRef mul0({prim::kPrimMul, select1, input_varptr_[2]}); + VectorRef mul1({prim::kPrimMul, mul0, input_varptr_[3]}); + + return VectorRef({prim::kPrimSub, input_varptr_[4], mul1}); +} + +const AnfNodePtr LambUpdateWithLrV2::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + auto prim = std::make_shared(kLambUpdateWithLrV2OpName); + std::vector inputs = {NewValueNode(prim)}; + (void)std::transform(input_varptr_.begin(), input_varptr_.end(), std::back_inserter(inputs), + [&equiv](const VarPtr &in) { return utils::cast((*equiv)[in]); }); + auto lamb_update_with_lr_v2 = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(lamb_update_with_lr_v2); + lamb_update_with_lr_v2->set_abstract(node->abstract()); + + return lamb_update_with_lr_v2; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.h new file mode 100644 index 00000000000..ea614d3d2d9 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.h @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_UPDATE_WITH_LR_V2_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_UPDATE_WITH_LR_V2_H_ + +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class LambUpdateWithLrV2 : public PatternProcessPass { + public: + explicit LambUpdateWithLrV2(bool multigraph = true) : PatternProcessPass("lamb_update_with_lr_v2", multigraph) { + for (size_t i = 0; i < kLambUpdateWithLrV2InputNum - 1; ++i) { + input_varptr_.push_back(std::make_shared()); + } + } + ~LambUpdateWithLrV2() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + std::vector input_varptr_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAMB_UPDATE_WITH_LR_V2_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc new file mode 100644 index 00000000000..fba1ab40af9 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.cc @@ -0,0 +1,159 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.h" +#include +#include +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +using common::SafeCStr; +namespace { +void GetOutputCastNodes(const FuncGraphPtr &func_graph, const AnfNodePtr &node, std::vector *cast_nodes) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(node) == manager->node_users().end()) { + return; + } + for (const auto &node_index : manager->node_users()[node]) { + AnfNodePtr output = node_index.first; + auto output_cnode = output->cast(); + MS_EXCEPTION_IF_NULL(output_cnode); + if (AnfAlgo::GetCNodeName(output_cnode) != prim::kPrimTupleGetItem->name()) { + MS_LOG(EXCEPTION) << "The output of node " << node->DebugString() << " should be " + << prim::kPrimTupleGetItem->name(); + } + if (manager->node_users().find(output) == manager->node_users().end() || + manager->node_users()[output].size() != 1) { + continue; + } + AnfNodePtr transitive_output = manager->node_users()[output].begin()->first; + MS_EXCEPTION_IF_NULL(transitive_output); + auto transitive_output_cnode = transitive_output->cast(); + MS_EXCEPTION_IF_NULL(transitive_output_cnode); + if (AnfAlgo::GetCNodeName(transitive_output_cnode) == prim::kPrimCast->name()) { + cast_nodes->push_back(transitive_output_cnode); + } + } +} + +bool CheckKernelBuildInfo(const CNodePtr &cnode, const kernel::KernelBuildInfoPtr &kernel_info) { + MS_EXCEPTION_IF_NULL(cnode); + MS_EXCEPTION_IF_NULL(kernel_info); + for (size_t i = 0; i < kernel_info->GetInputNum(); ++i) { + if (kernel_info->GetInputDeviceType(i) != kNumberTypeFloat16 || + kernel_info->GetInputFormat(i) != AnfAlgo::GetInputFormat(cnode, i)) { + return false; + } + } + for (size_t i = 0; i < kernel_info->GetOutputNum(); ++i) { + if (kernel_info->GetOutputDeviceType(i) != kNumberTypeFloat32 || + kernel_info->GetOutputFormat(i) != AnfAlgo::GetOutputFormat(cnode, i)) { + return false; + } + } + return true; +} + +bool CheckLayernormBetaGammaBackprop(const FuncGraphPtr &func_graph, const CNodePtr &cnode, + std::vector *cast_nodes) { + MS_EXCEPTION_IF_NULL(cnode); + if (!AnfAlgo::HasNodeAttr(kAttrShapeGamma, cnode)) { + MS_LOG(INFO) << "The node " << cnode->DebugString() << " has no " << kAttrShapeGamma << " attr"; + return false; + } + if (cnode->inputs().size() != kLayerNormBetaGammaBackpropInputNum) { + MS_LOG(INFO) << "The node " << cnode->DebugString() << " inputs num is not equal to " + << kLayerNormBetaGammaBackpropInputNum; + return false; + } + if (AnfAlgo::GetOutputTensorNum(cnode) != kLayerNormBetaGammaBackpropOutputNum) { + MS_LOG(INFO) << "The node " << cnode->DebugString() << " outputs num is not equal to " + << kLayerNormBetaGammaBackpropOutputNum; + return false; + } + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(cnode); ++i) { + if (AnfAlgo::GetInputDeviceDataType(cnode, i) != kNumberTypeFloat16) { + MS_LOG(INFO) << "The data type of node " << cnode->DebugString() << " input " << i << " is not float16"; + return false; + } + } + GetOutputCastNodes(func_graph, cnode, cast_nodes); + if (cast_nodes->size() != kLayerNormBetaGammaBackpropOutputNum) { + MS_LOG(INFO) << "The num of cast node in node " << cnode->DebugString() << " outputs is not equal to " + << kLayerNormBetaGammaBackpropOutputNum; + return false; + } + for (const auto &cast : *cast_nodes) { + if (AnfAlgo::GetInputDeviceDataType(cast, 0) != kNumberTypeFloat16 || + AnfAlgo::GetOutputDeviceDataType(cast, 0) != kNumberTypeFloat32) { + MS_LOG(INFO) << "The cast " << cast->DebugString() << " should be fp16->fp32"; + return false; + } + } + return true; +} +} // namespace + +const BaseRef LayerNormBetaGammaBackpropFusion::DefinePattern() const { + std::shared_ptr Xs = std::make_shared(); + const auto prim = std::make_shared(kLayerNormBetaGammaBackpropOpName); + return VectorRef({prim, Xs}); +} + +const AnfNodePtr LayerNormBetaGammaBackpropFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || !node->isa()) { + return nullptr; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + std::vector cast_nodes; + if (!CheckLayernormBetaGammaBackprop(func_graph, cnode, &cast_nodes)) { + return nullptr; + } + std::vector> kernel_info_list; + MS_EXCEPTION_IF_NULL(kernel_query_); + kernel_query_->Query(cnode, &kernel_info_list); + auto alternative_kernel_build_info = + std::find_if(kernel_info_list.begin(), kernel_info_list.end(), + [&cnode](const kernel::KernelBuildInfoPtr &candidate_kernel_build_info) { + return CheckKernelBuildInfo(cnode, candidate_kernel_build_info); + }); + if (alternative_kernel_build_info == kernel_info_list.end()) { + MS_LOG(INFO) << "Can not find alternative kernel build info for node " << node->DebugString(); + return nullptr; + } + AnfAlgo::SetSelectKernelBuildInfo(*alternative_kernel_build_info, cnode.get()); + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + // The cast_nodes size has been checked above. + MS_EXCEPTION_IF_NULL(cast_nodes[0]); + MS_EXCEPTION_IF_NULL(cast_nodes[1]); + if (cast_nodes[0]->inputs().size() != kCastInputNum) { + MS_LOG(EXCEPTION) << "The cast0 " << cast_nodes[0]->DebugString() << " input size should be " << kCastInputNum; + } + (void)manager->Replace(cast_nodes[0], cast_nodes[0]->input(1)); + if (cast_nodes[1]->inputs().size() != kCastInputNum) { + MS_LOG(EXCEPTION) << "The cast1 " << cast_nodes[1]->DebugString() << " input size should be " << kCastInputNum; + } + (void)manager->Replace(cast_nodes[1], cast_nodes[1]->input(1)); + return nullptr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.h new file mode 100644 index 00000000000..2655c0f14de --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.h @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAYER_NORM_BETA_GAMMA_BACKPROP_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAYER_NORM_BETA_GAMMA_BACKPROP_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class LayerNormBetaGammaBackpropFusion : public PatternProcessPass { + public: + explicit LayerNormBetaGammaBackpropFusion(bool multigraph = true) + : PatternProcessPass("layer_norm_beta_gamma_backprop_fusion", multigraph), + kernel_query_(std::make_shared()) {} + + ~LayerNormBetaGammaBackpropFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + KernelQueryPtr kernel_query_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_LAYER_NORM_BETA_GAMMA_BACKPROP_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.cc new file mode 100644 index 00000000000..11fd02d2d89 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.cc @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.h" +#include +#include +#include +#include "pre_activate/common/helper.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +namespace { +bool CheckValueNodeInputOfMul(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return false; + } + std::vector mul_input_shape = AnfAlgo::GetOutputInferShape(node, 0); + return mul_input_shape.empty() || (mul_input_shape.size() == 1 && mul_input_shape[0] == 1); +} +} // namespace +const BaseRef MomentumLossscaleFusion::DefinePattern() const { + VarPtr Xs = std::make_shared(); + VarPtr X0 = std::make_shared(); + VarPtr X1 = std::make_shared(); + VarPtr X2 = std::make_shared(); + VarPtr X4 = std::make_shared(); + return VectorRef({prim::kPrimApplyMomentum, X0, X1, X2, VectorRef({prim::kPrimMul, Xs}), X4}); +} + +const AnfNodePtr MomentumLossscaleFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + CheckCNodeInputSize(cnode, kApplyMomentumInputNum); + AnfNodePtr mul = cnode->input(4); + MS_EXCEPTION_IF_NULL(mul); + auto mul_cnode = mul->cast(); + MS_EXCEPTION_IF_NULL(mul_cnode); + CheckCNodeInputSize(mul_cnode, kMulInputNum); + size_t value_node_index = 0; + for (size_t i = 1; i < kMulInputNum; ++i) { + if (CheckValueNodeInputOfMul(mul_cnode->input(i))) { + value_node_index = i; + break; + } + } + if (value_node_index == 0) { + MS_LOG(DEBUG) << "The Mul " << mul->DebugString() << " to be fused must has a scalar constant input"; + return nullptr; + } + auto new_prim = std::make_shared(kFusedMulApplyMomentumOpName); + std::vector new_node_inputs{NewValueNode(new_prim), + cnode->input(1), + cnode->input(2), + cnode->input(3), + mul_cnode->input(kMulInputNum - value_node_index), + cnode->input(5), + mul_cnode->input(value_node_index)}; + auto new_node = func_graph->NewCNode(new_node_inputs); + MS_EXCEPTION_IF_NULL(new_node); + AnfAlgo::CopyNodeAttrs(node, new_node); + auto input_names_value = AnfAlgo::GetNodeAttr>(new_node, kAttrInputNames); + input_names_value[3] = "x1"; + input_names_value.emplace_back("x2"); + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(input_names_value), new_node); + new_node->set_abstract(node->abstract()); + new_node->set_scope(node->scope()); + return new_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.h new file mode 100644 index 00000000000..c092e0ca22f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.h @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_MOMENTUM_LOSSSCALE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_MOMENTUM_LOSSSCALE_FUSION_H_ + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class MomentumLossscaleFusion : public PatternProcessPass { + public: + explicit MomentumLossscaleFusion(bool multigraph = true) + : PatternProcessPass("momentum_lossscale_fusion", multigraph) {} + + ~MomentumLossscaleFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_MOMENTUM_LOSSSCALE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_add_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_add_fusion.cc new file mode 100644 index 00000000000..711ffff612c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_add_fusion.cc @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/mul_add_fusion.h" +#include +#include +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "optimizer/opt.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +const BaseRef MulAddFusion::DefinePattern() const { + VarPtr mul_x_ = std::make_shared(); + VarPtr mul_y_ = std::make_shared(); + VarPtr add_y_ = std::make_shared(); + + VectorRef mul({prim::kPrimMul, mul_x_, mul_y_}); + VectorRef add({prim::kPrimTensorAdd, mul, add_y_}); + return add; +} + +const AnfNodePtr MulAddFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, const EquivPtr &equiv) const { + if (graph == nullptr || node == nullptr || equiv == nullptr) { + return nullptr; + } + auto add = node->cast(); + if (add == nullptr || add->inputs().size() != kAddInputNum) { + return nullptr; + } + auto mul_anf = add->input(1); + if (mul_anf == nullptr) { + return nullptr; + } + auto mul = mul_anf->cast(); + if (mul == nullptr || mul->inputs().size() != kMulInputNum) { + return nullptr; + } + if (IsUsedByOthers(graph, mul)) { + MS_LOG(DEBUG) << "Mul is used by more then two nodes, cannot fuse"; + return nullptr; + } + + auto prim = std::make_shared(kFusedMulAddOpName); + std::vector inputs = {NewValueNode(prim), mul->input(1), mul->input(2), add->input(2)}; + auto fusion_node = graph->NewCNode(inputs); + fusion_node->set_scope(add->scope()); + fusion_node->set_abstract(add->abstract()); + return fusion_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_add_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_add_fusion.h new file mode 100644 index 00000000000..4b4db2b3124 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_add_fusion.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_MUL_ADD_FUSION_H +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_MUL_ADD_FUSION_H + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class MulAddFusion : public PatternProcessPass { + public: + explicit MulAddFusion(bool multigraph = true) : PatternProcessPass("mul_add_fusion", multigraph) {} + ~MulAddFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_MUL_ADD_FUSION_H diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.cc new file mode 100644 index 00000000000..83c58ab5471 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.cc @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/mul_addn_fusion.h" +#include +#include +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "optimizer/opt.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +CNodePtr CreateFusionNode(const FuncGraphPtr &graph, const CNodePtr &mul, const CNodePtr &addn, + const size_t &lossscale_input_index) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(mul); + MS_EXCEPTION_IF_NULL(addn); + auto prim = std::make_shared(kFusedMulAddNOpName); + std::vector inputs = {NewValueNode(prim)}; + inputs.push_back(mul->input(kMulInputNum - lossscale_input_index)); + inputs.push_back(addn->input(1)); + // scalar input should be 3rd input + inputs.push_back(mul->input(lossscale_input_index)); + auto fusion_node = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(fusion_node); + fusion_node->set_scope(addn->scope()); + fusion_node->set_abstract(addn->abstract()); + return fusion_node; +} +} // namespace + +const BaseRef MulAddNFusion::DefinePattern() const { + VarPtr X = std::make_shared(); + VarPtr Y = std::make_shared(); + VarPtr Z = std::make_shared(); + + VectorRef mul({prim::kPrimMul, X, Z}); + VectorRef addn({prim::kPrimAddN, Y, mul}); + return addn; +} + +const AnfNodePtr MulAddNFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + if (graph == nullptr || node == nullptr || equiv == nullptr) { + return nullptr; + } + + auto addn = node->cast(); + if (addn == nullptr || addn->inputs().size() != kAddNInputNum) { + return nullptr; + } + auto mul_anf = addn->input(2); + if (mul_anf == nullptr) { + return nullptr; + } + auto mul = mul_anf->cast(); + if (mul == nullptr || mul->inputs().size() != kMulInputNum) { + return nullptr; + } + if (IsUsedByOthers(graph, mul)) { + MS_LOG(DEBUG) << "Mul is used by more then two nodes, cannot fuse"; + return nullptr; + } + + size_t lossscale_input_index = 1; + for (size_t index = 1; index < mul->inputs().size(); ++index) { + auto input_node = mul->input(index); + MS_EXCEPTION_IF_NULL(input_node); + if (input_node->isa()) { + lossscale_input_index = index; + break; + } + } + auto constant_shape = AnfAlgo::GetOutputInferShape(mul->input(lossscale_input_index), 0); + if (!(constant_shape.size() == 0 || (constant_shape.size() == 1 && constant_shape[0] == 1))) { + MS_LOG(DEBUG) << "The const input of Mul node must be scalar or shape=(1,), but shape size is " + << constant_shape.size() << " and shape[0] is " << constant_shape[0]; + return nullptr; + } + + return CreateFusionNode(graph, mul, addn, lossscale_input_index); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.h new file mode 100644 index 00000000000..d03309bf73c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_PASS_MUL_ADDN_FUSION_H +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_PASS_MUL_ADDN_FUSION_H + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class MulAddNFusion : public PatternProcessPass { + public: + explicit MulAddNFusion(bool multigraph = true) : PatternProcessPass("mul_addn_fusion", multigraph) {} + ~MulAddNFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_PASS_MUL_ADDN_FUSION_H diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc new file mode 100644 index 00000000000..06f911c6be2 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/reshape_transpose_fusion.h" +#include +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +const BaseRef ReshapeTransposeFusion::DefinePattern() const { + const auto prim_reshape = std::make_shared(prim::kPrimReshape->name()); + VectorRef reshape({prim_reshape, input_varptr_}); + + return VectorRef({prim::kPrimTranspose, reshape}); +} + +const AnfNodePtr ReshapeTransposeFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + auto transpose_cnode = CheckAnfNodeIfCNodeAndInputSize(node, kBackendReshapeInputNum); + MS_EXCEPTION_IF_NULL(transpose_cnode); + auto reshape_cnode = CheckAnfNodeIfCNodeAndInputSize(transpose_cnode->input(1), kBackendReshapeInputNum); + MS_EXCEPTION_IF_NULL(reshape_cnode); + auto prim = std::make_shared(kConfusionTransposeDOpName); + std::vector inputs = {NewValueNode(prim), utils::cast((*equiv)[input_varptr_])}; + auto new_node = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(new_node); + new_node->set_abstract(node->abstract()); + + AnfAlgo::CopyNodeAttrs(reshape_cnode, new_node); + AnfAlgo::CopyNodeAttr(kAttrPerm, transpose_cnode, new_node); + AnfAlgo::SetNodeAttr(kAttrTransposeFirst, MakeValue(false), new_node); + + return new_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.h new file mode 100644 index 00000000000..5abf3e0d531 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.h @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_RESHAPE_TRANSPOSE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_RESHAPE_TRANSPOSE_FUSION_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class ReshapeTransposeFusion : public PatternProcessPass { + public: + explicit ReshapeTransposeFusion(bool multigraph = true) : PatternProcessPass("reshape_transpose_fusion", multigraph) { + input_varptr_ = std::make_shared(); + } + ~ReshapeTransposeFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr input_varptr_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_RESHAPE_TRANSPOSE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/square_sum_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/square_sum_fusion.cc new file mode 100644 index 00000000000..c3884ff70ac --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/square_sum_fusion.cc @@ -0,0 +1,129 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/square_sum_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" +#include "utils/utils.h" +#include "operator/ops.h" +#include "pre_activate/common/helper.h" +#include "device/kernel_info.h" + +namespace mindspore { +namespace opt { +namespace { +CNodePtr GenerateSquareSumV1(const FuncGraphPtr &graph, const CNodePtr &square, const CNodePtr &sum) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(square); + MS_EXCEPTION_IF_NULL(sum); + if (square->inputs().size() != kSquareNodeInputNum) { + MS_LOG(EXCEPTION) << "Square node has wrong input size"; + } + auto prim = std::make_shared(kSquareSumV1OpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector square_sumv1_inputs = {NewValueNode(prim), square->input(1)}; + auto square_sumv1 = graph->NewCNode(square_sumv1_inputs); + MS_EXCEPTION_IF_NULL(square_sumv1); + auto kernel_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(kernel_info); + square_sumv1->set_kernel_info(kernel_info); + auto types = {AnfAlgo::GetOutputInferDataType(sum, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(sum, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, square_sumv1.get()); + square_sumv1->set_scope(sum->scope()); + AnfAlgo::CopyNodeAttr(kAttrAxis, sum, square_sumv1); + AnfAlgo::CopyNodeAttr(kAttrKeepDims, sum, square_sumv1); + return square_sumv1; +} + +CNodePtr GenerateSquareSumV2(const FuncGraphPtr &graph, const CNodePtr &square, const CNodePtr &sum) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(square); + MS_EXCEPTION_IF_NULL(sum); + if (square->inputs().size() != kSquareNodeInputNum) { + MS_LOG(EXCEPTION) << "Square node has wrong input size"; + } + auto prim = std::make_shared(kSquareSumV2OpName); + MS_EXCEPTION_IF_NULL(prim); + std::vector square_sumv2_inputs = {NewValueNode(prim), square->input(1)}; + auto square_sumv2 = graph->NewCNode(square_sumv2_inputs); + MS_EXCEPTION_IF_NULL(square_sumv2); + auto types = {AnfAlgo::GetOutputInferDataType(sum, 0), AnfAlgo::GetOutputInferDataType(square, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(sum, 0), AnfAlgo::GetOutputInferShape(square, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, square_sumv2.get()); + square_sumv2->set_scope(sum->scope()); + AnfAlgo::CopyNodeAttr(kAttrAxis, sum, square_sumv2); + AnfAlgo::CopyNodeAttr(kAttrKeepDims, sum, square_sumv2); + return square_sumv2; +} + +std::tuple GetPrevNodes(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto sum = node->cast(); + MS_EXCEPTION_IF_NULL(sum); + if (sum->inputs().size() != kSumNodeInputNum) { + MS_LOG(EXCEPTION) << "ReduceSumD node has wrong input size"; + } + auto square_anf = sum->input(1); + MS_EXCEPTION_IF_NULL(square_anf); + auto square = square_anf->cast(); + MS_EXCEPTION_IF_NULL(square); + + return std::make_tuple(sum, square_anf, square); +} +} // namespace + +const BaseRef SquareSumFusion::DefinePattern() const { + VarPtr X = std::make_shared(); + MS_EXCEPTION_IF_NULL(X); + return VectorRef({prim::kPrimReduceSum, VectorRef({prim::kPrimSquare, X})}); +} + +const AnfNodePtr SquareSumFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + CNodePtr sum = nullptr; + AnfNodePtr square_anf = nullptr; + CNodePtr square = nullptr; + std::tie(sum, square_anf, square) = GetPrevNodes(node); + + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(square_anf) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "Square node has no output in NodeUsersMap"; + } + AnfNodePtr ret_node = nullptr; + if (manager->node_users()[square_anf].size() == 1) { + ret_node = GenerateSquareSumV1(graph, square, sum); + } else if (manager->node_users()[square_anf].size() == 2) { + auto square_sumv2 = GenerateSquareSumV2(graph, square, sum); + + std::vector square_sumv2_outputs; + CreateMultipleOutputsOfAnfNode(graph, square_sumv2, kSquareSumv2OutputNum, &square_sumv2_outputs); + if (square_sumv2_outputs.size() != kSquareSumv2OutputNum) { + MS_LOG(EXCEPTION) << "make SquareSumV2 outputs fail"; + } + (void)manager->Replace(square, square_sumv2_outputs[1]); + ret_node = square_sumv2_outputs[0]; + } + return ret_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/square_sum_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/square_sum_fusion.h new file mode 100644 index 00000000000..5a694a55858 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/square_sum_fusion.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_SQUARE_SUM_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_SQUARE_SUM_FUSION_H_ + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class SquareSumFusion : public PatternProcessPass { + public: + explicit SquareSumFusion(bool multigraph = true) : PatternProcessPass("square_sum_fusion", multigraph) {} + ~SquareSumFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_SQUARE_SUM_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transdata_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transdata_split.cc new file mode 100644 index 00000000000..faef2775992 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transdata_split.cc @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/ascend/ir_fusion/transdata_split.h" +#include +#include "pre_activate/ascend/ascend_helper.h" +#include "session/anf_runtime_algorithm.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +const std::set> invalid_formats_pair = {{kOpFormat_C1HWNCoC0, kOpFormat_NCHW}, + {kOpFormat_NCHW, kOpFormat_C1HWNCoC0}, + {kOpFormat_C1HWNCoC0, kOpFormat_DEFAULT}, + {kOpFormat_DEFAULT, kOpFormat_C1HWNCoC0}}; + +bool TransDataSplit::Run(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + bool changed = false; + std::vector node_list = TopoSort(func_graph->get_return()); + for (auto &node : node_list) { + if (node != nullptr && node->isa() && AnfAlgo::GetCNodeName(node) == kTransDataOpName) { + CheckCNodeInputSize(node->cast(), kBackendTransDataInputNum); + if (IsFormatInvaild(node)) { + changed = DoSplit(func_graph, node); + } + } + } + return changed; +} +bool TransDataSplit::IsFormatInvaild(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto input_format = AnfAlgo::GetInputFormat(node, 0); + auto output_format = AnfAlgo::GetOutputFormat(node, 0); + auto format_pair = std::make_pair(input_format, output_format); + + return invalid_formats_pair.find(format_pair) != invalid_formats_pair.end(); +} +// transdata cannot support frac_z to nchw need split transdata(frac_z-HWCN) and transpose(HWCN-NCHW) +bool TransDataSplit::DoSplit(const FuncGraphPtr &func_graph, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto input_node = node->cast()->input(1); + MS_EXCEPTION_IF_NULL(input_node); + + auto input_format = AnfAlgo::GetInputFormat(node, 0); + auto output_format = AnfAlgo::GetOutputFormat(node, 0); + AnfNodePtr new_transdata_node = nullptr; + AnfNodePtr new_transpose_node = nullptr; + AnfNodePtr new_replace_node = nullptr; + // if output_format=default transdata need split transdata->transpose else transpose->transdata + if (output_format == kOpFormat_DEFAULT || output_format == kOpFormat_NCHW) { + // trans input_format to hwcn + new_transdata_node = AddTransOpNodeToGraph(func_graph, node, kernel_select_, 0, false, input_format, kOpFormat_HWCN, + kTransDataOpName, true); + // trans hwcn to default_format + new_transpose_node = AddTransOpNodeToGraph(func_graph, new_transdata_node, kernel_select_, 0, false, kOpFormat_HWCN, + output_format, prim::kPrimTranspose->name(), false); + AnfAlgo::SetNodeAttr(kAttrPerm, MakeValue(std::vector{3, 2, 0, 1}), new_transpose_node); + new_replace_node = new_transpose_node; + } else { + // trans default to hwcn + new_transpose_node = AddTransOpNodeToGraph(func_graph, node, kernel_select_, 0, false, input_format, kOpFormat_HWCN, + prim::kPrimTranspose->name(), true); + AnfAlgo::SetNodeAttr(kAttrPerm, MakeValue(std::vector{2, 3, 1, 0}), new_transpose_node); + + // trans hwcn to output_format + new_transdata_node = AddTransOpNodeToGraph(func_graph, new_transpose_node, kernel_select_, 0, false, kOpFormat_HWCN, + output_format, kTransDataOpName, false); + new_replace_node = new_transdata_node; + } + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(func_graph); + + if (!manager->Replace(node, new_replace_node)) { + MS_LOG(EXCEPTION) << "manager replace node failed"; + } + MS_LOG(INFO) << "transdata node:" << cnode->DebugString() << "split success."; + return true; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transdata_split.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transdata_split.h new file mode 100644 index 00000000000..0e84c232563 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transdata_split.h @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSDATA_SPLIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSDATA_SPLIT_H_ +#include +#include +#include +#include + +#include "pre_activate/common/pass.h" +#include "ir/func_graph.h" +#include "ir/anf.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class TransDataSplit : public Pass { + public: + TransDataSplit() : Pass("trans_data_split"), kernel_select_(std::make_shared()) {} + ~TransDataSplit() override = default; + bool Run(const FuncGraphPtr &graph) override; + + private: + bool DoSplit(const FuncGraphPtr &func_graph, const AnfNodePtr &node); + bool IsFormatInvaild(const AnfNodePtr &node); + KernelSelectPtr kernel_select_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSDATA_SPLIT_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_reshape_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_reshape_fusion.cc new file mode 100644 index 00000000000..d991a1cd4a3 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_reshape_fusion.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/transpose_reshape_fusion.h" +#include +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +const BaseRef TransposeReshapeFusion::DefinePattern() const { + const auto prim_reshape = std::make_shared(prim::kPrimReshape->name()); + VectorRef transpose({prim::kPrimTranspose, input_varptr_}); + + return VectorRef({prim_reshape, transpose}); +} + +const AnfNodePtr TransposeReshapeFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + auto reshape_cnode = CheckAnfNodeIfCNodeAndInputSize(node, kBackendReshapeInputNum); + MS_EXCEPTION_IF_NULL(reshape_cnode); + auto transpose_cnode = CheckAnfNodeIfCNodeAndInputSize(reshape_cnode->input(1), kBackendReshapeInputNum); + MS_EXCEPTION_IF_NULL(transpose_cnode); + auto prim = std::make_shared(kConfusionTransposeDOpName); + std::vector inputs = {NewValueNode(prim), utils::cast((*equiv)[input_varptr_])}; + auto new_node = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(new_node); + + new_node->set_abstract(node->abstract()); + AnfAlgo::CopyNodeAttrs(reshape_cnode, new_node); + AnfAlgo::CopyNodeAttr(kAttrPerm, transpose_cnode, new_node); + AnfAlgo::SetNodeAttr(kAttrTransposeFirst, MakeValue(true), new_node); + + return new_node; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_reshape_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_reshape_fusion.h new file mode 100644 index 00000000000..8b979f869d7 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_reshape_fusion.h @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSPOSE_RESHAPE_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSPOSE_RESHAPE_FUSION_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class TransposeReshapeFusion : public PatternProcessPass { + public: + explicit TransposeReshapeFusion(bool multigraph = true) : PatternProcessPass("transpose_reshape_fusion", multigraph) { + input_varptr_ = std::make_shared(); + } + ~TransposeReshapeFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr input_varptr_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSPOSE_RESHAPE_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_transdata_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_transdata_fusion.cc new file mode 100644 index 00000000000..1386005d1b1 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_transdata_fusion.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/transpose_transdata_fusion.h" +#include +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +const BaseRef TransposeTransDataFusion::DefinePattern() const { + const auto prim_transdata = std::make_shared(prim::KPrimTransData->name()); + VectorRef transpose({prim::kPrimTranspose, input_varptr_}); + + return VectorRef({prim_transdata, transpose}); +} + +const AnfNodePtr TransposeTransDataFusion::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(equiv); + auto transdata_cnode = CheckAnfNodeIfCNodeAndInputSize(node, kBackendTransposeInputNum); + MS_EXCEPTION_IF_NULL(transdata_cnode); + auto transpose_cnode = CheckAnfNodeIfCNodeAndInputSize(transdata_cnode->input(1), kBackendTransDataInputNum); + MS_EXCEPTION_IF_NULL(transpose_cnode); + auto transpose_kernel_build_info = AnfAlgo::GetSelectKernelBuildInfo(transpose_cnode); + auto transdata_kernel_build_info = AnfAlgo::GetSelectKernelBuildInfo(transdata_cnode); + MS_EXCEPTION_IF_NULL(transpose_kernel_build_info); + MS_EXCEPTION_IF_NULL(transdata_kernel_build_info); + + auto new_transdata_builder = std::make_shared(); + auto transpose_input_formats = transpose_kernel_build_info->GetAllInputFormats(); + new_transdata_builder->SetInputsFormat(transpose_input_formats); + new_transdata_builder->SetOutputsFormat(transdata_kernel_build_info->GetAllOutputFormats()); + new_transdata_builder->SetInputsDeviceType(transdata_kernel_build_info->GetAllInputDeviceTypes()); + new_transdata_builder->SetOutputsDeviceType(transdata_kernel_build_info->GetAllOutputDeviceTypes()); + new_transdata_builder->SetKernelType(transdata_kernel_build_info->kernel_type()); + new_transdata_builder->SetFusionType(transdata_kernel_build_info->fusion_type()); + new_transdata_builder->SetProcessor(transdata_kernel_build_info->processor()); + + auto new_fusion_transdata = std::make_shared(kTransDataOpName); + if (kernel_select_->CheckKernelAccuracySupported(transdata_cnode, new_transdata_builder->Build())) { + std::vector inputs = {NewValueNode(new_fusion_transdata), + utils::cast((*equiv)[input_varptr_])}; + auto new_node = func_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(new_node); + new_node->set_abstract(node->abstract()); + AnfAlgo::CopyNodeAttrs(transdata_cnode, new_node); + AnfAlgo::SetNodeAttr(kAttrSrcFormat, MakeValue(transpose_input_formats[0]), new_node); + AnfAlgo::SetSelectKernelBuildInfo(new_transdata_builder->Build(), new_node.get()); + MS_LOG(INFO) << "transpose transdata fusion node:" << node->fullname_with_scope() << " success"; + return new_node; + } else { + MS_LOG(INFO) << "transpose transdata fusion node:" << node->fullname_with_scope() << " failed"; + return node; + } +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_transdata_fusion.h b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_transdata_fusion.h new file mode 100644 index 00000000000..bc9f17f340d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/transpose_transdata_fusion.h @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSPOSE_TRANSDATA_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSPOSE_TRANSDATA_FUSION_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/helper.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ascend_helper.h" + +namespace mindspore { +namespace opt { +class TransposeTransDataFusion : public PatternProcessPass { + public: + explicit TransposeTransDataFusion(bool multigraph = true) + : PatternProcessPass("transpose_transdata_fusion", multigraph) { + input_varptr_ = std::make_shared(); + kernel_select_ = std::make_shared(); + } + ~TransposeTransDataFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr input_varptr_; + KernelSelectPtr kernel_select_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_ASCEND_IR_FUSION_TRANSPOSE_TRANSDATA_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/common/common_backend_optimization.cc b/mindspore/ccsrc/pre_activate/common/common_backend_optimization.cc new file mode 100644 index 00000000000..c3fd292aa9f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/common_backend_optimization.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/common/common_backend_optimization.h" +#include +#include +#include "pre_activate/common/optimizer.h" +#include "pre_activate/pass/convert_const_input_to_attr.h" +#include "pre_activate/pass/convert_const_input_to_tensor_input.h" +#include "pre_activate/pass/convert_tuple_input_to_dynamic_input.h" +#include "utils/context/ms_context.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +void BackendCommonOptimization(const std::shared_ptr &kernel_graph) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->save_graphs_flag(); + auto save_graphs_path = context_ptr->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_common_before.ir"; + DumpIR(file_path, kernel_graph); + } + auto optimizer = std::make_shared(); + auto common_pm = std::make_shared("common_pm"); + common_pm->AddPass(std::make_shared()); + common_pm->AddPass(std::make_shared()); + common_pm->AddPass(std::make_shared()); + optimizer->AddPassManager(common_pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "hwopt_common_after.ir"; + DumpIR(file_path, kernel_graph); + } +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/common_backend_optimization.h b/mindspore/ccsrc/pre_activate/common/common_backend_optimization.h new file mode 100644 index 00000000000..6ce92da0dcd --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/common_backend_optimization.h @@ -0,0 +1,26 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_COMMON_BACKEND_OPTIMIZATION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_COMMON_BACKEND_OPTIMIZATION_H_ +#include +#include "session/kernel_graph.h" +namespace mindspore { +namespace opt { +void BackendCommonOptimization(const std::shared_ptr &kernel_graph); +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_COMMON_BACKEND_OPTIMIZATION_H_ diff --git a/mindspore/ccsrc/pre_activate/common/helper.cc b/mindspore/ccsrc/pre_activate/common/helper.cc new file mode 100644 index 00000000000..e5f0fafbe0d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/helper.cc @@ -0,0 +1,358 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/common/helper.h" +#include +#include +#include +#include "utils/utils.h" +#include "utils/base_ref.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" +#include "common/utils.h" +#include "device/kernel_info.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace opt { +std::vector Convert2Int(const std::vector &v) { + std::vector result; + (void)std::transform(v.begin(), v.end(), std::back_inserter(result), SizeToInt); + return result; +} + +bool UnVisited(const BaseRef &n) { + if (utils::isa(n)) { + AnfNodePtr in = utils::cast(n); + MS_EXCEPTION_IF_NULL(in); + if (IsValueNode(in)) { + auto value_node = in->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + auto prim_py = value->cast(); + MS_EXCEPTION_IF_NULL(prim_py); + return !prim_py->HasAttr(kAttrVisited); + } else { + return false; + } + } + return false; +} + +bool CheckIfCNodeAndInputSize(const AnfNodePtr &node, int input_size, CNodePtr *cnode) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(ERROR) << "The node is expected to be a cnode"; + return false; + } + *cnode = node->cast(); + if (*cnode == nullptr) { + return false; + } + if ((*cnode)->inputs().size() < IntToSize(input_size)) { + auto op_name = AnfAlgo::GetCNodeName(*cnode); + MS_LOG(ERROR) << "op[" + op_name + "] has less than " << input_size << " inputs."; + return false; + } + return true; +} + +CNodePtr CheckAnfNodeIfCNodeAndInputSize(const AnfNodePtr &node, int input_size) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "The node is expected to be a cnode"; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() != IntToSize(input_size)) { + auto op_name = AnfAlgo::GetCNodeName(cnode); + MS_LOG(EXCEPTION) << "op[" + op_name + "] has less than " << input_size << " inputs."; + } + return cnode; +} + +void CheckCNodeInputSize(const CNodePtr &cnode, size_t input_size) { + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() != input_size) { + MS_LOG(EXCEPTION) << "The input size of node " + cnode->DebugString() + " is not equal to " << input_size; + } +} + +bool HasSymmetricalKernelInfo(const AnfNodePtr &node_x, const AnfNodePtr &node_y) { + MS_EXCEPTION_IF_NULL(node_x); + MS_EXCEPTION_IF_NULL(node_y); + return (AnfAlgo::GetInputDeviceDataType(node_x, 0) == AnfAlgo::GetOutputDeviceDataType(node_y, 0) && + AnfAlgo::GetOutputDeviceDataType(node_x, 0) == AnfAlgo::GetInputDeviceDataType(node_y, 0)); +} + +const AnfNodePtr EliminateDependTransop(const FuncGraphPtr &func_graph, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(func_graph); + + auto transop_cnode = CheckAnfNodeIfCNodeAndInputSize(node, kTransOpInputNum); + auto depend_cnode = CheckAnfNodeIfCNodeAndInputSize(transop_cnode->input(kCastInputNum - 1), kDependInputNum); + auto prev_transop_cnode = CheckAnfNodeIfCNodeAndInputSize(depend_cnode->input(1), kTransOpInputNum); + MS_EXCEPTION_IF_NULL(depend_cnode->input(kDependInputNum - 1)); + MS_EXCEPTION_IF_NULL(prev_transop_cnode->input(kTransOpInputNum - 1)); + auto transed_node = prev_transop_cnode->input(kTransOpInputNum - 1); + MS_EXCEPTION_IF_NULL(transed_node); + + std::vector replace_depend_inputs{NewValueNode(prim::kPrimDepend), transed_node, + depend_cnode->input(kDependInputNum - 1)}; + AnfNodePtr replace_depend = func_graph->NewCNode(replace_depend_inputs); + MS_EXCEPTION_IF_NULL(replace_depend); + auto transed_abstract = transed_node->abstract(); + replace_depend->set_abstract(transed_abstract); + return replace_depend; +} + +bool Visited(const BaseRef &n) { + if (utils::isa(n)) { + AnfNodePtr in = utils::cast(n); + MS_EXCEPTION_IF_NULL(in); + if (IsValueNode(in)) { + auto value_node = in->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + auto prim_py = value->cast(); + MS_EXCEPTION_IF_NULL(prim_py); + return prim_py->HasAttr(kAttrVisited); + } else { + return false; + } + } + return false; +} + +void CreateOutputsOfConvBn1(const FuncGraphPtr &func_graph, const CNodePtr &conv_cnode, const CNodePtr &bn_cnode, + std::vector *conv_bn1_outputs) { + auto prim = std::make_shared(kConvBN1OpName); + std::vector conv_bn1_inputs = {NewValueNode(prim)}; + MS_EXCEPTION_IF_NULL(conv_cnode); + // All the inputs of conv_bn1 are from the inputs of conv + for (size_t i = 1; i < conv_cnode->inputs().size(); i++) { + conv_bn1_inputs.push_back(conv_cnode->input(i)); + } + MS_EXCEPTION_IF_NULL(func_graph); + CNodePtr conv_bn1_cnode = func_graph->NewCNode(conv_bn1_inputs); + MS_EXCEPTION_IF_NULL(conv_bn1_cnode); + auto kernel_info = std::make_shared(); + conv_bn1_cnode->set_kernel_info(kernel_info); + // Set attr for conv_bn1 + AnfAlgo::CopyNodeAttrs(conv_cnode, conv_bn1_cnode); + // Set abstract of conv_bn1 + MS_EXCEPTION_IF_NULL(bn_cnode); + auto bn_abstract_tuple = dyn_cast(bn_cnode->abstract()); + MS_EXCEPTION_IF_NULL(bn_abstract_tuple); + AbstractBasePtrList conv_bn1_abstract_list; + conv_bn1_abstract_list.push_back(conv_cnode->abstract()); + auto abstract_tensor = std::make_shared( + kFloat32, Convert2Int(AnfAlgo::GetPrevNodeOutputInferShape(bn_cnode, kVariance - 1))); + conv_bn1_abstract_list.push_back(abstract_tensor); + conv_bn1_abstract_list.push_back(bn_abstract_tuple->elements()[kSaveMean]); + auto abstract_tuple = std::make_shared(conv_bn1_abstract_list); + conv_bn1_cnode->set_abstract(abstract_tuple); + + CreateMultipleOutputsOfAnfNode(func_graph, conv_bn1_cnode, kConvBn1OutputNum, conv_bn1_outputs); +} + +void CreateOutputsOfFusedBn2(const FuncGraphPtr &graph, const std::vector &fused_bn1_outputs, + const CNodePtr &bn_node, std::vector *fused_bn2_outputs) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(bn_node); + MS_EXCEPTION_IF_NULL(fused_bn2_outputs); + if (bn_node->inputs().size() != kBnInputNum) { + MS_LOG(EXCEPTION) << "BN node has wrong input size"; + } + if (fused_bn1_outputs.size() != kBN1OutputNum) { + MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size"; + } + + // the inputs of fused_bn2 are from the outputs of fused_bn1 and the inputs of bn + std::vector fused_bn2_inputs = {NewValueNode(std::make_shared(kFusedBN2OpName))}; + fused_bn2_inputs.push_back(fused_bn1_outputs[0]); + fused_bn2_inputs.push_back(fused_bn1_outputs[1]); + fused_bn2_inputs.push_back(bn_node->input(4)); + fused_bn2_inputs.push_back(bn_node->input(5)); + auto fused_bn2 = graph->NewCNode(fused_bn2_inputs); + MS_EXCEPTION_IF_NULL(fused_bn2); + auto kernel_info = std::make_shared(); + fused_bn2->set_kernel_info(kernel_info); + auto types = {AnfAlgo::GetOutputInferDataType(bn_node, 4), AnfAlgo::GetOutputInferDataType(bn_node, 1), + AnfAlgo::GetOutputInferDataType(bn_node, 2)}; + auto shapes = {AnfAlgo::GetOutputInferShape(bn_node, 4), AnfAlgo::GetOutputInferShape(bn_node, 1), + AnfAlgo::GetOutputInferShape(bn_node, 2)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, fused_bn2.get()); + fused_bn2->set_scope(bn_node->scope()); + AnfAlgo::CopyNodeAttr(kAttrMomentum, bn_node, fused_bn2); + + CreateMultipleOutputsOfAnfNode(graph, fused_bn2, kBN2OutputNum, fused_bn2_outputs); +} + +void CreateOutputsOfFusedBn3(const FuncGraphPtr &graph, const AnfNodePtr &data_input, + const std::vector &fused_bn1_outputs, + const std::vector &fused_bn2_outputs, const CNodePtr &bn_node, + std::vector *fused_bn3_outputs) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(data_input); + MS_EXCEPTION_IF_NULL(bn_node); + MS_EXCEPTION_IF_NULL(fused_bn3_outputs); + if (bn_node->inputs().size() != kBnInputNum) { + MS_LOG(EXCEPTION) << "BN node has wrong input size"; + } + + if (fused_bn1_outputs.size() != kBN1OutputNum) { + MS_LOG(EXCEPTION) << "BN1 outputs has wrong input size"; + } + + if (fused_bn2_outputs.size() != kBN2OutputNum) { + MS_LOG(EXCEPTION) << "BN2 outputs has wrong input size"; + } + + // the inputs of fused_bn3 are from the outputs of fused_bn1 and the inputs of bn + std::vector fused_bn3_inputs = {NewValueNode(std::make_shared(kFusedBN3OpName))}; + fused_bn3_inputs.push_back(data_input); + fused_bn3_inputs.push_back(fused_bn1_outputs[0]); + fused_bn3_inputs.push_back(fused_bn2_outputs[0]); + fused_bn3_inputs.push_back(bn_node->input(2)); + fused_bn3_inputs.push_back(bn_node->input(3)); + auto fused_bn3 = graph->NewCNode(fused_bn3_inputs); + MS_EXCEPTION_IF_NULL(fused_bn3); + auto kernel_info = std::make_shared(); + fused_bn3->set_kernel_info(kernel_info); + auto types = {AnfAlgo::GetOutputInferDataType(bn_node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(bn_node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, fused_bn3.get()); + + fused_bn3->set_scope(bn_node->scope()); + AnfAlgo::CopyNodeAttr(kAttrEpsilon, kAttrEps, bn_node, fused_bn3); + + (*fused_bn3_outputs).push_back(fused_bn3); +} + +void CreateMultipleOutputsOfAnfNode(const FuncGraphPtr &func_graph, const AnfNodePtr &node, size_t output_num, + std::vector *outputs) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(outputs); + for (size_t i = 0; i < output_num; i++) { + auto idx = NewValueNode(SizeToInt(i)); + MS_EXCEPTION_IF_NULL(idx); + int temp = SizeToInt(i); + auto imm = std::make_shared(temp); + auto abstract_scalar = std::make_shared(imm); + idx->set_abstract(abstract_scalar); + auto tuple_getitem = func_graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), node, idx}); + MS_EXCEPTION_IF_NULL(tuple_getitem); + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(node, i)}, + {AnfAlgo::GetOutputInferShape(node, i)}, tuple_getitem.get()); + (*outputs).push_back(tuple_getitem); + } +} + +bool IsNopNode(const AnfNodePtr &node) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->device_target() != kAscendDevice) { + return false; + } + static std::unordered_set nop_nodes = {prim::kPrimReshape->name(), kExpandDimsOpName, + prim::kPrimSqueeze->name(), prim::kPrimFlatten->name()}; + if (node == nullptr || !node->isa()) { + return false; + } + CNodePtr cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (nop_nodes.find(AnfAlgo::GetCNodeName(cnode)) == nop_nodes.end()) { + return false; + } + if (cnode->inputs().size() != 2) { + MS_LOG(EXCEPTION) << "Nop node(" + cnode->DebugString() + ") should have only 1 input, but it has " + << cnode->inputs().size() - 1 << " inputs."; + } + return true; +} + +void HideNopNode(session::KernelGraph *const graph) { + MS_EXCEPTION_IF_NULL(graph); + auto execution_order = graph->execution_order(); + MS_LOG(INFO) << "nop node info (Before Remove) size: " << execution_order.size(); + std::vector new_nodes; + for (auto &cnode : execution_order) { + MS_EXCEPTION_IF_NULL(cnode); + if (!IsNopNode(cnode)) { + new_nodes.push_back(cnode); + } + } + graph->set_execution_order(new_nodes); + MS_LOG(INFO) << "nop node info (After Remove) size: " << graph->execution_order().size(); +} + +void RemoveNopNode(session::KernelGraph *const graph) { + MS_EXCEPTION_IF_NULL(graph); + bool changed = true; + while (changed) { + changed = false; + std::vector new_nodes; + for (auto &cnode : graph->execution_order()) { + MS_EXCEPTION_IF_NULL(cnode); + // ignore nop node itself + if (IsNopNode(cnode)) { + continue; + } + // Replace the input which is nop node + std::vector new_inputs; + new_inputs.push_back(cnode->input(0)); + bool need_update = false; + for (size_t i = 1; i < cnode->inputs().size(); ++i) { + auto input = cnode->input(i); + MS_EXCEPTION_IF_NULL(input); + auto cinput = input->cast(); + if (cinput == nullptr || !IsNopNode(cinput)) { + new_inputs.push_back(input); + continue; + } + if (cinput->inputs().size() == 2) { + new_inputs.push_back(cinput->input(1)); + need_update = true; + changed = true; + } else { + new_inputs.push_back(input); + } + } + if (need_update) { + cnode->set_inputs(new_inputs); + } + // push into new execution list + new_nodes.push_back(cnode); + } + graph->set_execution_order(new_nodes); + } +} + +bool IsUsedByOthers(const FuncGraphPtr &graph, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(node) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "node has no output in manager"; + } + return manager->node_users()[node].size() > 1; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/helper.h b/mindspore/ccsrc/pre_activate/common/helper.h new file mode 100644 index 00000000000..ba8db8bd29a --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/helper.h @@ -0,0 +1,146 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_HELPER_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_HELPER_H_ + +#include +#include +#include +#include "ir/func_graph.h" +#include "session/kernel_graph.h" +#include "common/utils.h" + +namespace mindspore { +namespace opt { +constexpr size_t kTransOpInputNum = 2; +constexpr size_t kCastInputNum = 2; +constexpr size_t kDependInputNum = 3; +constexpr size_t kReluInputNum = 2; +constexpr size_t kAddInputNum = 3; +constexpr size_t kAddNInputNum = 3; +constexpr size_t kTupleGetitemInputNum = 3; +constexpr size_t kConvInputNum = 3; +constexpr size_t kRealDivInputNum = 3; +constexpr size_t kSqrtInputNum = 2; +constexpr size_t kMulInputNum = 3; +constexpr size_t kRsqrtInputNum = 2; +constexpr size_t kSubInputNum = 3; +constexpr size_t kAssignSubInputNum = 3; + +constexpr size_t kConvBn1OutputNum = 3; +constexpr size_t kBn2ReluOutputNum = 4; + +constexpr size_t kBnInputNum = 6; +constexpr size_t kBnOutputNum = 5; + +constexpr size_t kBN1OutputNum = 2; +constexpr size_t kBN2OutputNum = 3; +constexpr size_t kBN3OutputNum = 1; + +constexpr size_t kBNGradInputNum = 6; +constexpr size_t kBNGradOutputNum = 3; + +constexpr size_t kBNGrad1OutputNum = 3; +constexpr size_t kBNGrad2OutputNum = 5; +constexpr size_t kBNGrad3OutputNum = 1; + +constexpr size_t kBNTrainingReduceOutputNum = 2; +constexpr size_t kBNTrainingUpdateOutputNum = 5; +constexpr size_t kBNTrainingUpdateGradOutputNum = 2; + +constexpr size_t kSingleOutputNum = 1; +constexpr size_t kSumNodeInputNum = 2; +constexpr size_t kSquareNodeInputNum = 2; +constexpr size_t kSquareSumv2OutputNum = 2; +constexpr size_t kMinimumInputNum = 3; + +constexpr size_t kLambNextMVWithDecayInputNum = 7; +constexpr size_t kLambNextMVWithDecayConstantMulInputNum = 5; +constexpr size_t kLambNextMVWithDecayOutputNum = 4; +constexpr size_t kLambNextMVWithDecayV1OutputNum = 4; +constexpr size_t kLambNextRightOutputNum = 2; +constexpr size_t kLambUpdateWithLrV2InputNum = 8; +constexpr size_t kLambNextMVRuleInputNum = 14; +constexpr size_t kLambNextMVRuleOutputNum = 4; +constexpr size_t kBackendReshapeInputNum = 2; +constexpr size_t kBackendTransposeInputNum = 2; +constexpr size_t kAdamApplyOneWithDecayOutputNum = 3; +constexpr size_t kLayerNormBetaGammaBackpropInputNum = 5; +constexpr size_t kLayerNormBetaGammaBackpropOutputNum = 2; +constexpr size_t kLayerNormGradInputNum = 6; +constexpr size_t kAdamApplyOneOutputNum = 3; +constexpr size_t kBackendTransDataInputNum = 2; +constexpr size_t kApplyMomentumInputNum = 6; + +enum FusedBatchNormInput { + kX = 1, + kVariance = 5, +}; +enum FusedBatchNormOutput { + kY = 0, + kRunningMean, + kRunningVariance, + kSaveMean, + kSaveInvVariance, +}; +enum ConvBn1Output { + kData = 0, + kVarPart, + kMean, +}; + +std::vector Convert2Int(const std::vector &v); + +bool UnVisited(const BaseRef &n); + +bool Visited(const BaseRef &n); + +// check if the input node is CNode, then check it's input_size, if meet condition above, return true, otherwise return +// false. cnode can only be used when return true. +bool CheckIfCNodeAndInputSize(const AnfNodePtr &node, int input_size, CNodePtr *cnode); + +// check if the input node is CNode, then check it's input_size, return CNodePtr if check success. +CNodePtr CheckAnfNodeIfCNodeAndInputSize(const AnfNodePtr &node, int input_size); + +void CheckCNodeInputSize(const CNodePtr &cnode, size_t input_size); + +bool HasSymmetricalKernelInfo(const AnfNodePtr &node_x, const AnfNodePtr &node_y); + +const AnfNodePtr EliminateDependTransop(const FuncGraphPtr &func_graph, const AnfNodePtr &node); + +void CreateOutputsOfConvBn1(const FuncGraphPtr &func_graph, const CNodePtr &conv_cnode, const CNodePtr &bn_cnode, + std::vector *conv_bn1_outputs); + +void CreateOutputsOfFusedBn2(const FuncGraphPtr &graph, const std::vector &fused_bn1_outputs, + const CNodePtr &bn_node, std::vector *fused_bn2_outputs); +void CreateOutputsOfFusedBn3(const FuncGraphPtr &graph, const AnfNodePtr &data_input, + const std::vector &fused_bn1_outputs, + const std::vector &fused_bn2_outputs, const CNodePtr &bn_node, + std::vector *fused_bn3_outputs); + +void CreateMultipleOutputsOfAnfNode(const FuncGraphPtr &kernel_graph, const AnfNodePtr &anf_node_ptr, size_t output_num, + std::vector *outputs); + +bool IsNopNode(const AnfNodePtr &node); + +void HideNopNode(session::KernelGraph *const graph); + +void RemoveNopNode(session::KernelGraph *const graph); + +bool IsUsedByOthers(const FuncGraphPtr &graph, const AnfNodePtr &node); +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_HELPER_H_ diff --git a/mindspore/ccsrc/pre_activate/common/node_pass.cc b/mindspore/ccsrc/pre_activate/common/node_pass.cc new file mode 100644 index 00000000000..cd213f8263d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/node_pass.cc @@ -0,0 +1,66 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/common/node_pass.h" + +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/manager.h" + +namespace mindspore { +namespace opt { +bool NodePass::Run(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(func_graph); + + std::unordered_set seen_node; + std::deque todo{func_graph->output()}; + bool changes = false; + while (!todo.empty()) { + AnfNodePtr node = todo.front(); + todo.pop_front(); + if (seen_node.count(node) > 0 || !manager->all_nodes().contains(node)) { + continue; + } + (void)seen_node.insert(node); + AnfNodePtr new_node = Run(func_graph, node); + bool change = (new_node != nullptr); + if (new_node != nullptr && new_node != node) { + (void)manager->Replace(node, new_node); + } else if (new_node == nullptr) { + new_node = node; + } + if (new_node && IsValueNode(new_node)) { + auto const_func_graph = GetValueNode(new_node); + MS_EXCEPTION_IF_NULL(const_func_graph); + todo.push_back(const_func_graph->output()); + } else if (new_node && new_node->isa()) { + auto cnode = new_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto inputs = cnode->inputs(); + (void)todo.insert(todo.end(), inputs.begin(), inputs.end()); + } + changes = changes || change; + } + return changes; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/node_pass.h b/mindspore/ccsrc/pre_activate/common/node_pass.h new file mode 100644 index 00000000000..7750a59e598 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/node_pass.h @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_NODE_PASS_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_NODE_PASS_H_ +#include +#include + +#include "pre_activate/common/pass.h" + +namespace mindspore { +namespace opt { +// @brief ANF Node level optimization base pass +class NodePass : public Pass { + public: + explicit NodePass(const std::string &name) : Pass(name) {} + ~NodePass() override = default; + bool Run(const FuncGraphPtr &func_graph) final; + virtual AnfNodePtr Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node) = 0; +}; +using NodePassPtr = std::shared_ptr; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_NODE_PASS_H_ diff --git a/mindspore/ccsrc/pre_activate/common/optimizer.cc b/mindspore/ccsrc/pre_activate/common/optimizer.cc new file mode 100644 index 00000000000..62cff76be01 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/optimizer.cc @@ -0,0 +1,231 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/common/optimizer.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "pre_activate/common/pass_manager.h" +#include "session/anf_runtime_algorithm.h" +#include "ir/manager.h" + +namespace mindspore { +namespace opt { +namespace { +AnfNodePtr HandleSexpVector(const BaseRef &sexp, const BaseRef &graph, bool multigraph); + +ValueNodePtr CreateValueNodeWithSexp(const BaseRef &sexp) { + if (utils::isa(sexp)) { + return NewValueNode(utils::cast(sexp)); + } + if (utils::isa(sexp)) { + return NewValueNode(utils::cast(sexp)); + } + if (utils::isa(sexp)) { + return NewValueNode(utils::cast(sexp)); + } + if (utils::isa(sexp)) { + return NewValueNode(utils::cast(sexp)); + } + return nullptr; +} + +CNodePtr CreateCNodeWithGraph(const std::vector &input_nodes, const BaseRef &graph) { + if (utils::isa(graph)) { + return std::make_shared(input_nodes, utils::cast(graph)); + } + if (utils::isa(graph)) { + return std::make_shared(input_nodes, utils::cast(graph)); + } + return nullptr; +} + +VarNodePtr CreateVarNodeWithSexp(const BaseRef &sexp, const BaseRef &graph) { + if (utils::isa(graph)) { + MS_LOG(DEBUG) << "make VarPtr " + graph.ToString(); + return std::make_shared(utils::cast(sexp), nullptr); + } + if (utils::isa(graph)) { + MS_LOG(DEBUG) << "VarNode, should input a Var in graph. It's GraphPtr: " + graph.ToString(); + return std::make_shared(utils::cast(sexp), utils::cast(graph)); + } + MS_LOG(ERROR) << "VarNode, should input a Var in graph. It's " + graph.ToString(); + return nullptr; +} + +AnfNodePtr SexpToNode(const BaseRef &sexp, const BaseRef &graph, bool multigraph = false) { + MS_LOG(DEBUG) << "SexpToNode sexp: " + sexp.ToString() + ", graph " + graph.ToString(); + if (utils::isa(sexp)) { + return HandleSexpVector(sexp, graph, multigraph); + } + if (utils::isa(sexp)) { + return CreateVarNodeWithSexp(sexp, graph); + } + if (utils::isa(sexp)) { + return utils::cast(sexp); + } + auto value_node = CreateValueNodeWithSexp(sexp); + if (value_node == nullptr) { + MS_LOG(EXCEPTION) << "sexp cannot converted. sexp: " + sexp.ToString(); + } + return value_node; +} + +AnfNodePtr HandleSexpVector(const BaseRef &sexp, const BaseRef &graph, bool multigraph) { + MS_LOG(DEBUG) << "HandleSexpVector sexp: " + sexp.ToString() + ", graph " + graph.ToString(); + std::vector input_nodes; + const auto &tuple = utils::cast(sexp); + if (multigraph && utils::isa(graph)) { + for (auto &x : tuple) { + AnfNodePtr node = SexpToNode(x, std::make_shared("G"), true); + input_nodes.push_back(node); + } + VarPtr var_ptr = utils::cast(graph); + return std::make_shared(input_nodes, var_ptr); + } + + for (auto &x : tuple) { + AnfNodePtr node = SexpToNode(x, graph, multigraph); + input_nodes.push_back(node); + } + return CreateCNodeWithGraph(input_nodes, graph); +} +} // namespace + +static bool AnfEqual(const BaseRef &a, const BaseRef &b) { + if (utils::isa(a) && utils::isa(b)) { + auto a_node = utils::cast(a); + auto b_node = utils::cast(b); + if (IsValueNode(a_node) && IsValueNode(b_node)) { + auto a_value_node = a_node->cast(); + auto a_value = a_value_node->value(); + auto a_prim = a_value->cast(); + + auto b_value_node = b_node->cast(); + auto b_value = b_value_node->value(); + auto b_prim = b_value->cast(); + + return a_prim->name() == b_prim->name(); + } else if (a_node->isa() && b_node->isa()) { + auto a_value_node_ptr = a_node->cast(); + if (a_value_node_ptr == nullptr) { + MS_LOG(EXCEPTION) << "cast value node ptr fail"; + } + auto a_value_ptr = a_value_node_ptr->value(); + if (a_value_ptr == nullptr) { + MS_LOG(EXCEPTION) << "value ptr is nullptr"; + } + + auto b_value_node_ptr = b_node->cast(); + if (b_value_node_ptr == nullptr) { + MS_LOG(EXCEPTION) << "cast value node ptr fail"; + } + auto b_value_ptr = b_value_node_ptr->value(); + if (b_value_ptr == nullptr) { + MS_LOG(EXCEPTION) << "value ptr is nullptr"; + } + + return (*a_value_ptr) == (*b_value_ptr); + } + MS_LOG(DEBUG) << "check AnfNodePtr equal"; + } + if (utils::isa(a) && utils::isa(b)) { + MS_LOG(DEBUG) << "check GraphPtr equal"; + } + return a == b; +} + +static bool CNodeTypeEqual(const BaseRef &a, const BaseRef &b) { + // To matchCNode and Kernel's type + if (utils::isa(a) && utils::isa(b)) { + return true; + } + return a.type() == b.type(); +} + +PatternProcessPass::PatternProcessPass(const std::string &name, bool multigraph) + : NodePass(name), + multigraph_(multigraph), + pattern_engine_(PatternEngine(std::make_shared(), + std::function(AnfEqual), + std::function(CNodeTypeEqual))) {} + +const BaseRef PatternProcessPass::DefinePattern() const { + VarPtr X = std::make_shared(); + return BaseRef({X}); +} + +void PatternProcessPass::Build() { + VarPtr fg = std::make_shared("RootG"); + BaseRef pattern = std::move(DefinePattern()); + pattern_ = SexpToNode(pattern, fg, multigraph_); +} + +AnfNodePtr PatternProcessPass::Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node) { + if (pattern_ == nullptr) { + Build(); + } + + auto empty_equiv = std::make_shared(); + EquivPtr equiv = pattern_engine_.Match(pattern_, node, empty_equiv); + if (equiv != nullptr && !equiv->empty()) { + return Process(func_graph, node, equiv); + } + return nullptr; +} + +void GraphOptimizer::AddPassManager(const PassManagerPtr &pass_manager) { + if (pass_manager != nullptr) { + pass_managers_.push_back(pass_manager); + } +} + +FuncGraphPtr GraphOptimizer::Optimize(const FuncGraphPtr &func_graph, bool run_only_once) { + MS_EXCEPTION_IF_NULL(func_graph); + run_only_once_ = (pass_managers_.size() == 1) ? true : run_only_once; + auto manager = func_graph->manager(); + if (manager == nullptr) { + manager = Manage(func_graph, false); + func_graph->set_manager(manager); + } + + bool changed = true; + while (changed) { + changed = false; + for (size_t i = 0; i < pass_managers_.size(); ++i) { + const PassManagerPtr &pm = pass_managers_[i]; + if (pm != nullptr && pm->Run(func_graph)) { + changed = true; + } + } + if (run_only_once_) { + break; + } + } + + std::vector func_graphs; + func_graphs.push_back(func_graph); + manager->KeepRoots(func_graphs); + (void)TopoSort(func_graph->get_return()); + return func_graph; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/optimizer.h b/mindspore/ccsrc/pre_activate/common/optimizer.h new file mode 100644 index 00000000000..8ef0b6dc340 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/optimizer.h @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_OPTIMIZER_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_OPTIMIZER_H_ + +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/primitive.h" +#include "pre_activate/common/pass_manager.h" +#include "pre_activate/common/pattern_engine.h" +#include "utils/graph_utils.h" +#include "common/utils.h" + +namespace mindspore { +namespace opt { +using PatternListType = std::initializer_list; + +class PatternProcessPass : public NodePass { + public: + explicit PatternProcessPass(const std::string &name = "", bool multigraph = true); + ~PatternProcessPass() override = default; + virtual const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const = 0; + virtual const BaseRef DefinePattern() const; + AnfNodePtr Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node) override; + + private: + void Build(); + + AnfNodePtr pattern_ = nullptr; + bool multigraph_ = true; + PatternEngine pattern_engine_; +}; + +class GraphOptimizer { + public: + explicit GraphOptimizer(const std::string &name = "graph_optimizer") : name_(name) {} + virtual ~GraphOptimizer() = default; + + void AddPassManager(const PassManagerPtr &pass_manager); + FuncGraphPtr Optimize(const FuncGraphPtr &func_graph, bool run_only_once = true); + + private: + const std::string name_ = "graph_optimizer"; + std::vector pass_managers_{}; + bool run_only_once_ = true; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_OPTIMIZER_H_ diff --git a/mindspore/ccsrc/pre_activate/common/pass.h b/mindspore/ccsrc/pre_activate/common/pass.h new file mode 100644 index 00000000000..3d2468cddb0 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/pass.h @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PASS_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PASS_H_ +#include +#include + +#include "ir/anf.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +// @brief ANF Graph level optimization base pass +class Pass { + public: + explicit Pass(const std::string &name = "pass") : name_(name) {} + virtual ~Pass() = default; + virtual bool Run(const FuncGraphPtr &func_graph) = 0; + virtual std::string name() const { return name_; } + + private: + const std::string name_; +}; +using PassPtr = std::shared_ptr; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PASS_H_ diff --git a/mindspore/ccsrc/pre_activate/common/pass_manager.cc b/mindspore/ccsrc/pre_activate/common/pass_manager.cc new file mode 100644 index 00000000000..0dfe4c763fd --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/pass_manager.cc @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/common/pass_manager.h" + +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/manager.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +const std::vector &PassManager::Passes() const { return passes_; } + +void PassManager::AddPass(const PassPtr &pass) { + if (pass != nullptr) { + passes_.push_back(pass); + } +} + +bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector &passes) const { + if (func_graph == nullptr) { + return false; + } + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->save_graphs_flag(); + auto save_graphs_path = context_ptr->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + bool changed = false; + size_t num = 0; + for (const auto &pass : passes) { + if (pass != nullptr) { + struct timeval start_time {}; + struct timeval end_time {}; + (void)gettimeofday(&start_time, nullptr); + if (pass->Run(func_graph)) { + changed = true; + } + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us"; + if (save_graphs) { + auto dump_file_path = + save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; + DumpIR(dump_file_path, func_graph); + } + num++; + } + } + return changed; +} + +bool PassManager::Run(const FuncGraphPtr &func_graph) const { + bool changed = false; + // run all passes + bool change = true; + while (change) { + change = Run(func_graph, passes_); + changed = change || changed; + if (run_only_once_) { + break; + } + } + return changed; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/pass_manager.h b/mindspore/ccsrc/pre_activate/common/pass_manager.h new file mode 100644 index 00000000000..38fe49b94c7 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/pass_manager.h @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PASS_MANAGER_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PASS_MANAGER_H_ + +#include +#include +#include +#include + +#include "pre_activate/common/pass.h" +#include "pre_activate/common/node_pass.h" + +namespace mindspore { +namespace opt { +// @brief For optimization passes management +class PassManager { + public: + explicit PassManager(const std::string &name = "pm", bool run_only_once = true) + : name_(name), passes_{}, run_only_once_(run_only_once) {} + virtual ~PassManager() = default; + // Get all the passes added by AddPass + const std::vector &Passes() const; + // Add graph pass, the pass object will be freed when pass manager freed. + void AddPass(const PassPtr &pass); + // Run passes added in pass manager on the input graph + // @param [inout] graph The graph to be optimized + // @return true, graph changed + // @return false, graph not changed + bool Run(const FuncGraphPtr &func_graph) const; + // Run the given graph passes on the input graph + // @param [inout] graph The graph to be optimized + // @param [in] passes The given graph passes + // @return true, graph changed + // @return false, graph not changed + bool Run(const FuncGraphPtr &func_graph, const std::vector &passes) const; + std::string name() const { return name_; } + + private: + const std::string name_; + std::vector passes_; + bool run_only_once_; +}; +using PassManagerPtr = std::shared_ptr; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PASS_MANAGER_H_ diff --git a/mindspore/ccsrc/pre_activate/common/pattern_engine.cc b/mindspore/ccsrc/pre_activate/common/pattern_engine.cc new file mode 100644 index 00000000000..e2ff321a894 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/pattern_engine.cc @@ -0,0 +1,328 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/common/pattern_engine.h" + +#include +#include +#include +#include + +#include "optimizer/opt.h" + +#include "ir/anf.h" +#include "utils/overload.h" + +namespace mindspore { +static int GetNextTag() { + static int kID = 0; + return kID++; +} + +void Var::EnsureTag() { + if (tag_.length() == 0) { + std::ostringstream buffer; + buffer << "_" << GetNextTag(); + tag_ = buffer.str(); + } +} + +bool operator==(const VarPtr& lhs, const VarPtr& rhs) { + if (lhs->isa() && rhs->isa()) { + CondVarPtr v1 = dyn_cast(lhs); + CondVarPtr v2 = dyn_cast(rhs); + return *v1 == *v2; + } + + if (lhs->isa() && rhs->isa()) { + SVarPtr v1 = dyn_cast(lhs); + SVarPtr v2 = dyn_cast(rhs); + return *v1 == *v2; + } + return (*lhs == *rhs); +} + +std::string SeqVar::ToString() const { + std::ostringstream buffer; + buffer << "SeqVar(" << tag() << ", " << subvar_->ToString() << ")"; + return buffer.str(); +} + +std::ostream& operator<<(std::ostream& os, const VarPtr& var) { + if (var == nullptr) { + os << ""; + } else { + os << var->ToString(); + } + return os; +} + +template <> +std::ostream& operator<<(std::ostream& os, const Equiv& equiv) { + os << "[Equiv]" + << "\n"; + for (auto& equiv_item : equiv) { + auto k = equiv_item.first; + os << k << ":"; + BaseRef x = equiv_item.second; + if (utils::isa(x)) { + auto node = utils::cast(x); + os << "TypeString[" << node->type_name() << "]"; + if (IsValueNode(node)) { + os << "IsValueNodeGraph "; + } + os << "type " << node->type_name(); + if (node->isa()) { + os << " value " << GetValueNode(node); + } + os << " addr: " << node; + } else if (utils::isa(x)) { + os << "Named " << x.ToString().c_str(); + } else if (utils::isa(x)) { + os << "TypeString[Var]"; + os << utils::cast(x); + } else if (utils::isa(x)) { + os << "TypeString[Graph]"; + } + os << "\n"; + } + return os; +} + +static BaseRef GetVar(const BaseRef& x) { + MS_LOG(DEBUG) << "getVar start :%s" + x.ToString(); + if (utils::isa(x)) { + auto node = utils::cast(x); + MS_LOG(DEBUG) << "TypeString [" + node->type_name() + "]"; + if (node->isa()) { + MS_LOG(DEBUG) << "IsVarNode " + node->cast()->var_->ToString(); + return node->cast()->var_; + } + if (node->isa()) { + MS_LOG(DEBUG) << "value " + GetValueNode(node)->ToString() + " addr: " + node->ToString(); + } else { + MS_LOG(DEBUG) << "type " + node->type_name(); + } + } else if (utils::isa(x)) { + MS_LOG(DEBUG) << "Named " + x.ToString(); + } else if (utils::isa(x)) { + MS_LOG(DEBUG) << "VectorRef"; + } else if (utils::isa(x)) { + MS_LOG(DEBUG) << "TypeString[Var] " + x.ToString(); + } + MS_LOG(DEBUG) << "GetVar end: " + x.ToString(); + return x; +} + +EquivPtr MatchOnVar(const BaseRef& pattern, const BaseRef& expr, EquivPtr equiv) { + MS_LOG(DEBUG) << "MatchOnVar pattern " + pattern.ToString() + " expr: " + expr.ToString(); + MS_EXCEPTION_IF_NULL(equiv); + if (utils::isa(pattern)) { + VarPtr var = utils::cast(pattern); + if (var->matches(expr)) { + (*equiv)[var] = expr; + MS_LOG(DEBUG) << "pattern is var match: " + pattern.ToString() + ", " + expr.ToString(); + return equiv; + } + } + + return nullptr; +} + +bool PatternEngine::ToVector(const VectorRef& pattern_ref, const VectorRef& expr_ref, VectorRef* const values_pattern, + VectorRef* const values_expr) const { + MS_EXCEPTION_IF_NULL(values_expr); + if (utils::isa(pattern_ref)) { + *values_pattern = pattern_ref; + *values_expr = expr_ref; + return true; + } + return false; +} + +bool PatternEngine::ToVector(const BaseRef& pattern_ref, const BaseRef& expr_ref, VectorRef* const values_pattern, + VectorRef* const values_expr) const { + MS_EXCEPTION_IF_NULL(values_expr); + // visitor to visite the list + auto appender_pattern = [](VectorRef& values) { + std::function fn = [&](const BaseRef& u) { + values.push_back(GetVar(u)); + return u; + }; + return fn; + }; + + visitor_->SetFn(appender_pattern(*values_pattern)); + MS_LOG(DEBUG) << "visit pattern_ref"; + bool success = visitor_->Visit(pattern_ref, nullptr); + if (!success) { + return false; + } + + auto appender_expr = [](VectorRef& values) { + std::function fn = [&](const BaseRef& u) { + values.push_back(u); + return u; + }; + return fn; + }; + + visitor_->SetFn(appender_expr(*values_expr)); + MS_LOG(DEBUG) << "visit expr_ref"; + return visitor_->Visit(expr_ref, nullptr); +} + +static int GetSVarStartIndex(const VectorRef& values) { + int index = -1; + int count = 0; + for (auto& value : values) { + if (utils::isa(value) && utils::cast(value)->isa()) { + if (index != -1) { + MS_LOG(DEBUG) << "Multiple SVars in sequence"; + return kInvalidVarIndex; + } + index = count; + } + count++; + } + return index; +} + +EquivPtr PatternEngine::AlignSVar(const VectorRef& values_pattern, const VectorRef& values_expr, EquivPtr equiv) const { + int svar_index = GetSVarStartIndex(values_pattern); + if (svar_index == kInvalidVarIndex) { + return nullptr; + } + + size_t values_pattern_len = values_pattern.size(); + size_t values_expr_len = values_expr.size(); + + if (svar_index == -1) { + if (values_pattern_len != values_expr_len) { + MS_LOG(DEBUG) << "Structures of differing size: pattern len " << values_pattern_len << ", expr len " + << values_expr_len; + return nullptr; + } + } + if (values_expr_len < values_pattern_len - 1) { + MS_LOG(DEBUG) << "invalid size: pattern len " << values_pattern_len << ", expr len " << values_expr_len; + return nullptr; + } + size_t diff = values_expr_len - values_pattern_len + 1; + for (size_t i = 0; i < values_pattern_len; i++) { + size_t expr_i = i; + if (svar_index != -1 && i == IntToSize(svar_index)) { + auto seq = + std::vector(values_expr.begin() + svar_index, values_expr.begin() + svar_index + SizeToInt(diff)); + equiv = Match(values_pattern[svar_index], seq, equiv); + } else { + if (svar_index != -1 && i > IntToSize(svar_index)) { + expr_i = i + diff - 1; + } + equiv = Match(values_pattern[i], values_expr[expr_i], equiv); + } + if (equiv == nullptr) { + return nullptr; + } + } + return equiv; +} + +EquivPtr PatternEngine::Match(const BaseRef& pattern, const BaseRef& expr, EquivPtr equiv) const { + MS_LOG(DEBUG) << "-----[in Match]"; + MS_LOG(DEBUG) << "GetVar w"; + BaseRef pattern_ref = GetVar(pattern); + MS_LOG(DEBUG) << "GetVar v"; + BaseRef expr_ref = expr; + + if (equiv == nullptr) { + MS_LOG(EXCEPTION) << "Equiv pointer is null"; + } + + MS_LOG(DEBUG) << "Pattern ref " + pattern_ref.ToString() + ", expr ref" + expr_ref.ToString(); + // 1. if pattern_ref is var and already in equiv, replace it. + if (utils::isa(pattern_ref)) { + VarPtr var = utils::cast(pattern_ref); + auto iter = equiv->find(var); + if (iter != equiv->end()) { + pattern_ref = iter->second; + } + } + + // 2. check equal + if (eq_(pattern_ref, expr_ref)) { + return equiv; + } + + // 3. match var + EquivPtr ret_equiv = MatchOnVar(pattern_ref, expr_ref, equiv); + if (ret_equiv) { + return ret_equiv; + } + + // 4. here the type can be std:vector, std:list, + // or cnode. + if (!type_eq_(pattern_ref, expr_ref)) { + MS_LOG(DEBUG) << "Type mismatch"; + return nullptr; + } + + // 5. transfer the Containers by visitor to std::vector + VectorRef values_pattern; + VectorRef values_expr; + if (!ToVector(pattern_ref, expr_ref, &values_pattern, &values_expr)) { + return nullptr; + } + + // 6. if any svar in both side, find the SeqVar index, + // try to pack the Var s in std::vector to a Seq and match elements one by one. + // check svar + return AlignSVar(values_pattern, values_expr, equiv); +} + +BaseRef PatternEngine::Replace(const BaseRef& pattern, const EquivPtr& equiv) const { + MS_EXCEPTION_IF_NULL(equiv); + MS_LOG(DEBUG) << "-----[in Replace]"; + BaseRef ref = GetVar(pattern); + BaseRef out; + bool is_match = false; + + // w is var + if (utils::isa(ref)) { + const VarPtr& var = utils::cast(ref); + auto iter = equiv->find(var); + if (iter != equiv->end()) { + out = iter->second; + is_match = true; + } + } + if (is_match) { + return out; + } + + // visitor to visit the list + std::function fn = [&, this, equiv](const BaseRef& u) { return Replace(u, equiv); }; + + visitor_->SetFn(fn); + BaseRef visit_out; + if (!visitor_->Visit(pattern, &visit_out)) { + return pattern; + } + return visit_out; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/pattern_engine.h b/mindspore/ccsrc/pre_activate/common/pattern_engine.h new file mode 100644 index 00000000000..432746332ff --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/pattern_engine.h @@ -0,0 +1,193 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PATTERN_ENGINE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PATTERN_ENGINE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pre_activate/common/visit.h" +#include "ir/base.h" +#include "utils/log_adapter.h" +#include "utils/base_ref.h" + +namespace mindspore { +class CondVar; +class SeqVar; +using CondVarPtr = std::shared_ptr; +using SVarPtr = std::shared_ptr; +const int kInvalidVarIndex = -2; + +using ConditionFunc = std::function; + +// Base wildcard variable which could match any anf node. +class Var : public Base { + friend class VarHasher; + + public: + explicit Var(const std::string& tag = "") : tag_(tag) { EnsureTag(); } + Var(const Var& other) : Base(other), tag_(other.tag_) {} + virtual Var& operator=(const Var& other) { + if (&other == this) { + return *this; + } + this->tag_ = other.tag_; + return *this; + } + ~Var() override = default; + MS_DECLARE_PARENT(Var, Base); + + virtual bool matches(const BaseRef&) { return true; } + + virtual bool operator==(const Var& other) const { return tag_ == other.tag_; } + bool operator!=(const Var& other) const { return !(&other == this); } + + std::string tag() const { return tag_; } + std::string ToString() const override { + std::ostringstream buffer; + buffer << "Var(" << tag_ << ")"; + return buffer.str(); + } + std::size_t hash() const override { return std::hash()(tag_); } + + protected: + void EnsureTag(); + + std::string tag_; +}; + +// VarNode means variable node, a subclass of AnfNode +class VarNode : public AnfNode { + public: + VarNode(const VarPtr& value, const FuncGraphPtr& func_graph) : AnfNode(func_graph), var_(value) {} + ~VarNode() override = default; + MS_DECLARE_PARENT(VarNode, AnfNode); + + const VarPtr var_; +}; +using VarNodePtr = std::shared_ptr; + +class VarHasher { + public: + std::size_t operator()(const Var& var) const { return var.hash(); } +}; + +// Condition Var, match an anf node when condition function return true. +class CondVar : public Var { + public: + explicit CondVar(const ConditionFunc& cond) : cond_fn_(cond) {} + ~CondVar() override = default; + MS_DECLARE_PARENT(CondVar, Var); + bool matches(const BaseRef& value) override { + MS_LOG(DEBUG) << "CondVarPtr match: " + value.ToString(); + if (utils::isa(value)) { + return false; + } + return cond_fn_(value); + } + ConditionFunc cond_fn_; +}; + +using Seq = VectorRef; +using SeqPtr = std::shared_ptr; + +// Sequence Var which could match multiple consecutive input nodes of a CNode. +class SeqVar : public Var { + public: + SeqVar() { subvar_ = std::make_shared(); } + ~SeqVar() override = default; + MS_DECLARE_PARENT(SeqVar, Var); + explicit SeqVar(const VarPtr subvar) : subvar_(nullptr) { subvar_ = subvar; } + bool matches(const BaseRef& value) override { + // match Seq. + if (utils::isa(value)) { + const Seq& seq = utils::cast(value); + return std::all_of(seq.begin(), seq.end(), [this](const BaseRef& v) { + auto eq = subvar_->matches(v); + return eq; + }); + } + return false; + } + bool operator==(const SeqVar& other) const { return *subvar_ == *other.subvar_; } + std::string ToString() const override; + + private: + VarPtr subvar_; +}; + +bool operator==(const VarPtr& lhs, const VarPtr& rhs); + +inline bool operator!=(const VarPtr& lhs, const VarPtr& rhs) { return !(lhs == rhs); } + +std::ostream& operator<<(std::ostream& os, const VarPtr& var); + +using Equiv = std::map; +using EquivPtr = std::shared_ptr; + +inline bool DefaultTypeEq(const BaseRef& x, const BaseRef& y) { return x.type() == y.type(); } + +class PatternEngine { + public: + PatternEngine(const std::shared_ptr& visitor, const std::function& eq, + const std::function& type_eq = DefaultTypeEq) + : visitor_(visitor), eq_(eq), type_eq_(type_eq) {} + ~PatternEngine() = default; + + EquivPtr Match(const BaseRef& pattern, const BaseRef& expr, EquivPtr equiv) const; + // Replace pattern with equivalent + BaseRef Replace(const BaseRef& pattern, const EquivPtr& equiv) const; + + private: + EquivPtr AlignSVar(const VectorRef& values_pattern, const VectorRef& values_expr, EquivPtr equiv) const; + bool ToVector(const BaseRef& pattern, const BaseRef& expr, VectorRef* const values_pattern, + VectorRef* const values_expr) const; + bool ToVector(const VectorRef& pattern_ref, const VectorRef& expr_ref, VectorRef* const values_pattern, + VectorRef* const values_expr) const; + std::shared_ptr visitor_; + std::function eq_; + std::function type_eq_; +}; +} // namespace mindspore +namespace std { +using mindspore::ERROR; +using mindspore::LogStream; +using mindspore::NoExceptionType; +template <> +struct hash { + std::size_t operator()(const mindspore::VarPtr var) const { + if (var == nullptr) { + MS_LOG(ERROR) << "Invalid var ptr"; + return 0; + } + return std::hash{}(var->tag()); + } +}; +} // namespace std +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_PATTERN_ENGINE_H_ diff --git a/mindspore/ccsrc/pre_activate/common/visit.cc b/mindspore/ccsrc/pre_activate/common/visit.cc new file mode 100644 index 00000000000..179177dd67e --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/visit.cc @@ -0,0 +1,166 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/common/visit.h" + +#include +#include +#include +#include + +#include "pre_activate/common/pattern_engine.h" +#include "utils/any.h" +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "utils/log_adapter.h" + +/* namespace to support utils definition */ +namespace mindspore { +bool CheckIfNeedExpand(const std::vector &list) { + return std::any_of(list.begin(), list.end(), [](const BaseRef &any) { return utils::isa(any); }); +} + +std::shared_ptr ExpandList(const std::vector &list) { + std::shared_ptr new_list = std::make_shared(); + for (auto &item : list) { + if (utils::isa(item)) { + const Seq &seq = utils::cast(item); + new_list->insert(new_list->end(), seq.begin(), seq.end()); + } else { + new_list->push_back(item); + } + } + return new_list; +} + +bool DefaultVisitor::Visit(const VectorRef &v_any, BaseRef *const visit_out) const { + std::vector out; + (void)std::transform(v_any.begin(), v_any.end(), std::back_inserter(out), + [this](const BaseRef &item) { return fn_(item); }); + if (visit_out != nullptr) { + *visit_out = ExpandList(out); + } + return true; +} + +bool DefaultVisitor::Visit(const BaseRef &any, BaseRef *const visit_out) const { + if (utils::isa(any)) { + return Visit(utils::cast(any), visit_out); + } else if (utils::isa(any)) { + auto nodeptr = utils::cast(any); + AnfNodePtr output; + AnfNodePtr *p_output = &output; + if (visit_out == nullptr) { + p_output = nullptr; + } + Visit(nodeptr, fn_, p_output); + if (visit_out != nullptr) { + *visit_out = output; + } + return true; + } + MS_LOG(DEBUG) << "VisitError, not support type to Visit: " + any.ToString(); + return false; +} + +void DefaultVisitor::Visit(const AnfNodePtr &node, const VisitFn &fn, AnfNodePtr *output) const { + if (node->isa()) { + Visit(node->cast(), fn, output); + return; + } + + if (node->isa()) { + Visit(node->cast(), fn, output); + return; + } + + if (output != nullptr) { + *output = node; + } +} + +void DefaultVisitor::Visit(const CNodePtr &cnode, const VisitFn &fn, AnfNodePtr *output) const { + // if output is nullptr, it's not required to make the new CNode node. + if (output == nullptr) { + for (auto &inp : cnode->inputs()) { + (void)fn(inp); + } + + if (cnode->func_graph() != nullptr) { + (void)fn(cnode->func_graph()); + } else { + (void)fn(cnode->func_graph_as_var()); + } + return; + } + + std::vector new_inputs; + std::vector after_cnode_fn; + std::shared_ptr out; + (void)std::transform(cnode->inputs().begin(), cnode->inputs().end(), std::back_inserter(after_cnode_fn), fn); + if (CheckIfNeedExpand(after_cnode_fn)) { + out = ExpandList(after_cnode_fn); + } + + std::vector &outs = after_cnode_fn; + if (out != nullptr) { + outs = out->elements(); + } + + for (auto &any_item : outs) { + if (!utils::isa(any_item)) { + MS_LOG(EXCEPTION) << "VisitError, fn not return the same type AnfNodePtr"; + } + new_inputs.push_back(utils::cast(any_item)); + } + + BaseRef any_fg; + AnfNodePtr new_cnode = nullptr; + if (cnode->func_graph() != nullptr) { + any_fg = fn(cnode->func_graph()); + if (!utils::isa(any_fg)) { + MS_LOG(EXCEPTION) << "VisitError, fn not return the same type FuncGraphPtr"; + } + new_cnode = std::make_shared(new_inputs, utils::cast(any_fg)); + } else { + any_fg = fn(cnode->func_graph_as_var()); + if (utils::isa(any_fg)) { + new_cnode = std::make_shared(new_inputs, utils::cast(any_fg)); + } else if (utils::isa(any_fg)) { + new_cnode = std::make_shared(new_inputs, utils::cast(any_fg)); + } else { + MS_LOG(EXCEPTION) << "VisitError, fn not return VarPtr or FuncGraphPtr"; + } + } + new_cnode->set_abstract(cnode->abstract()); + *output = new_cnode; +} + +void DefaultVisitor::Visit(const ValueNodePtr &vnode, const VisitFn &fn, AnfNodePtr *output) const { + const BaseRef &value = utils::cast(fn(vnode->value())); + if (utils::isa(value)) { + if (output != nullptr) { + auto ct = NewValueNode(utils::cast(value)); + ct->set_abstract(vnode->abstract()); + *output = ct; + } + return; + } + MS_LOG(EXCEPTION) << "Visit result is not ValuePtr."; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/common/visit.h b/mindspore/ccsrc/pre_activate/common/visit.h new file mode 100644 index 00000000000..2017b03b2fe --- /dev/null +++ b/mindspore/ccsrc/pre_activate/common/visit.h @@ -0,0 +1,61 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_VISIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_VISIT_H_ + +#include +#include +#include +#include +#include +#include + +#include "ir/base.h" +#include "utils/base_ref.h" + +// namespace to support utils definition +namespace mindspore { +using VisitFn = std::function; + +class Visitor { + public: + virtual void SetFn(VisitFn fn) = 0; + virtual bool Visit(const BaseRef &e, BaseRef *out) const = 0; + virtual bool Visit(const VectorRef &e, BaseRef *out) const = 0; + virtual ~Visitor() = default; +}; + +class DefaultVisitor : public Visitor { + public: + DefaultVisitor() : fn_(nullptr) {} + ~DefaultVisitor() override = default; + void SetFn(VisitFn fn) override { fn_ = fn; }; + bool Visit(const VectorRef &e, BaseRef *out) const override; + bool Visit(const BaseRef &e, BaseRef *out) const override; + void Visit(const AnfNodePtr &node, const VisitFn &fn, AnfNodePtr *output) const; + void Visit(const CNodePtr &cnode, const VisitFn &fn, AnfNodePtr *output) const; + void Visit(const ValueNodePtr &vnode, const VisitFn &fn, AnfNodePtr *output) const; + + VisitFn fn_; +}; + +std::shared_ptr ExpandList(const std::vector &list); +bool CheckIfNeedExpand(const std::vector &list); +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_COMMON_VISIT_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/kernel_refcount.cc b/mindspore/ccsrc/pre_activate/mem_reuse/kernel_refcount.cc new file mode 100644 index 00000000000..b471550b9db --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/kernel_refcount.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/mem_reuse/kernel_refcount.h" +#include +#include "utils/log_adapter.h" +namespace mindspore { +namespace memreuse { +/** + * Add some set && get function + */ +void KernelRefCount::SetKernelRefCountInfo(int index, size_t size, RefCountType reftype) { + index_ = index; + size_ = size; + reftype_ = reftype; +} + +std::vector KernelDef::GetInputRefIndexs() const { + std::vector input_ref_indexs; + if (input_refs_.empty()) { + return input_ref_indexs; + } + (void)std::transform(input_refs_.begin(), input_refs_.end(), std::back_inserter(input_ref_indexs), + [](const KernelRefCountPtr &ref_info) { return ref_info->index_; }); + return input_ref_indexs; +} + +std::vector KernelDef::GetOutputRefIndexs() const { + std::vector output_ref_indexs; + if (output_refs_.empty()) { + return output_ref_indexs; + } + (void)std::transform(output_refs_.begin(), output_refs_.end(), std::back_inserter(output_ref_indexs), + [](const KernelRefCountPtr &ref_info) { return ref_info->index_; }); + return output_ref_indexs; +} + +std::vector KernelDef::GetWkRefIndexs() const { + std::vector wk_ref_indexs; + if (wk_space_.empty()) { + return wk_ref_indexs; + } + // only one key + auto wk_refs_iter = wk_space_.begin(); + auto wk_refs = wk_refs_iter->second; + (void)std::transform(wk_refs.begin(), wk_refs.end(), std::back_inserter(wk_ref_indexs), + [](const KernelRefCountPtr &ref_info) { return ref_info->index_; }); + return wk_ref_indexs; +} +} // namespace memreuse +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/kernel_refcount.h b/mindspore/ccsrc/pre_activate/mem_reuse/kernel_refcount.h new file mode 100644 index 00000000000..b8cf3a83a38 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/kernel_refcount.h @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_KERNEL_REFCOUNT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_KERNEL_REFCOUNT_H_ +#include +#include +#include +#include + +namespace mindspore { +namespace memreuse { +enum RefCountType { kDynamicRefCount, kStaticRefCount }; +enum NodeType { NORMAL, SPECIAL }; +static constexpr int kInitIndex = -1; +class KernelRefCount { + public: + uint32_t stream_id_; + int ref_count_; + // used by dynamic memory pool, it will be reseted by ref_count_ when one minibatch end + int ref_count_dynamic_use_; + size_t offset_; + size_t size_; + int index_; + // remember to reset offset + KernelRefCount() + : stream_id_(0), + ref_count_(0), + ref_count_dynamic_use_(0), + offset_(0), + size_(0), + index_(kInitIndex), + reftype_(kStaticRefCount) {} + ~KernelRefCount() = default; + void SetKernelRefCountInfo(int index, size_t size, RefCountType reftype); + void set_reftype(RefCountType reftype) { reftype_ = reftype; } + RefCountType reftype() const { return reftype_; } + uint32_t stream_id() const { return stream_id_; } + + private: + RefCountType reftype_; +}; +using KernelRefCountPtr = std::shared_ptr; +using KernelRefCountPtrList = std::vector; +// the ptr of every kernel to be key +using KernelKey = void *; +using KernelMap = std::map>; + +class KernelDef { + public: + KernelMap inputs_; + KernelMap outputs_; + KernelMap wk_space_; + NodeType dirty = NORMAL; + KernelDef() = default; + ~KernelDef() = default; + void set_input_refs(const KernelRefCountPtrList &kernelRefPtrList) { input_refs_ = kernelRefPtrList; } + void set_output_refs(const KernelRefCountPtrList &kernelRefPtrList) { output_refs_ = kernelRefPtrList; } + KernelRefCountPtrList input_refs() const { return input_refs_; } + KernelRefCountPtrList output_refs() const { return output_refs_; } + std::vector GetInputRefIndexs() const; + std::vector GetOutputRefIndexs() const; + std::vector GetWkRefIndexs() const; + void set_stream_id(uint32_t stream_id) { stream_id_ = stream_id; } + uint32_t stream_id() const { return stream_id_; } + void set_kernel_name(const std::string &kernel_name) { kernel_name_ = kernel_name; } + std::string kernel_name() const { return kernel_name_; } + void set_scope_full_name(const std::string &scop_name) { scop_full_name_ = scop_name; } + std::string scope_full_name() const { return scop_full_name_; } + + private: + std::string scop_full_name_; + std::string kernel_name_; + uint32_t stream_id_{0}; + KernelRefCountPtrList input_refs_; + KernelRefCountPtrList output_refs_; +}; +using KernelDefPtr = std::shared_ptr; +} // namespace memreuse +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_KERNEL_REFCOUNT_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_dynamic_allocator.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_dynamic_allocator.cc new file mode 100644 index 00000000000..f0077ef6cdf --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_dynamic_allocator.cc @@ -0,0 +1,289 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/mem_reuse/mem_dynamic_allocator.h" +#include "common/utils.h" +#include "utils/convert_utils.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace device { +DynamicMemPoolBestFit::~DynamicMemPoolBestFit() { + global_mem_block_list_.clear(); + global_idle_mem_buf_map_.clear(); +} + +DeviceMemPtr DynamicMemPoolBestFit::AllocTensorMem(size_t size) { + size_t align_size = AlignMemorySize(size); + // Find the idle memory buf by tensor size, if not find, then add new memory block and memory buf. + DeviceMemPtr device_addr = FindIdleMemBuf(align_size); + if (!device_addr) { + device_addr = AddMemBlockAndMemBuf(align_size); + } + return device_addr; +} + +size_t DynamicMemPoolBestFit::AlignMemorySize(size_t size) const { + if (size == 0) { + return DYNAMIC_MEM_ALIGN_SIZE; + } + return ((size + DYNAMIC_MEM_ALIGN_SIZE - 1) / DYNAMIC_MEM_ALIGN_SIZE) * DYNAMIC_MEM_ALIGN_SIZE; +} + +DeviceMemPtr DynamicMemPoolBestFit::FindIdleMemBuf(size_t size) { + auto iter = global_idle_mem_buf_map_.lower_bound(size); + if (iter != global_idle_mem_buf_map_.end()) { + auto mem_buf = iter->second; + MS_EXCEPTION_IF_NULL(mem_buf); + if (mem_buf->status_ != kMemBufIdle) { + MS_LOG(EXCEPTION) << "Find the mem_buf is not idle, alloc_size[" << size << "] mem_buf_size[" << mem_buf->size_ + << "] mem_buf_address[" << mem_buf->device_addr_ << "]."; + } + mem_buf->status_ = kMemBufUsed; + // Remove map of old idle memory buf + (void)global_idle_mem_buf_map_.erase(iter); + // Divide memory buf + if (IsDivide(size, mem_buf->size_)) { + DivideMemBuf(size, mem_buf); + } + // Memory statistics + total_used_mem_statistics_ += mem_buf->size_; + if (total_used_mem_statistics_ > used_mem_peak_statistics_) { + used_mem_peak_statistics_ = total_used_mem_statistics_; + } + return mem_buf->device_addr_; + } + return nullptr; +} + +DeviceMemPtr DynamicMemPoolBestFit::AddMemBlockAndMemBuf(size_t size) { + size_t alloc_mem_size = CalMemBlockAllocSize(size); + + // Add new memory block + DeviceMemPtr device_addr = nullptr; + auto real_alloc_size = AllocDeviceMem(alloc_mem_size, &device_addr); + if (real_alloc_size < size) { + MS_LOG(EXCEPTION) << "Memory not enough: alloc size[" << real_alloc_size << "] is smaller than required size[" + << size << "]."; + } + auto mem_block = std::make_shared(device_addr, real_alloc_size); + MS_EXCEPTION_IF_NULL(mem_block); + auto iter = std::upper_bound(global_mem_block_list_.begin(), global_mem_block_list_.end(), device_addr, CmpMemBlock); + (void)global_mem_block_list_.insert(iter, mem_block); + // Add new memory buf + auto mem_buf = std::make_shared(device_addr, kMemBufUsed, real_alloc_size); + MS_EXCEPTION_IF_NULL(mem_buf); + // Add map of new memory buf in the block + (void)mem_block->block_all_mem_buf_map_.emplace(device_addr, mem_buf); + // Divide memory buf + if (IsDivide(size, mem_buf->size_)) { + DivideMemBuf(size, mem_buf); + } + // Memory statistics + total_mem_statistics_ += real_alloc_size; + total_used_mem_statistics_ += mem_buf->size_; + if (total_used_mem_statistics_ > used_mem_peak_statistics_) { + used_mem_peak_statistics_ = total_used_mem_statistics_; + } + return mem_buf->device_addr_; +} + +size_t DynamicMemPoolBestFit::CalMemBlockAllocSize(size_t size) { + auto device_free_mem_size = free_mem_size(); + if (device_free_mem_size < size) { + MS_LOG(EXCEPTION) << "Memory not enough: current free memory size[" << device_free_mem_size + << "] is smaller than required size[" << size << "]."; + } + + auto alloc_mem_size = mem_alloc_unit_size(); + // Growing at twice of alloc size + while (alloc_mem_size < size) { + alloc_mem_size = alloc_mem_size * 2; + } + alloc_mem_size = std::min(alloc_mem_size, device_free_mem_size); + return AlignMemorySize(alloc_mem_size); +} + +bool DynamicMemPoolBestFit::IsDivide(size_t tensor_size, size_t mem_buf_size) const { + return mem_buf_size - tensor_size >= DYNAMIC_MEM_ALIGN_SIZE; +} + +void DynamicMemPoolBestFit::DivideMemBuf(size_t size, const DynamicMemBufPtr& mem_buf) { + MS_EXCEPTION_IF_NULL(mem_buf); + auto mem_block = FindMemBlock(mem_buf->device_addr_); + MS_EXCEPTION_IF_NULL(mem_block); + + // Divide new memory buf + size_t newbuf_size = mem_buf->size_ - size; + mem_buf->size_ = size; + DeviceMemPtr newbuf_addr = AddressOffset(mem_buf->device_addr_, size); + auto new_mem_buf = std::make_shared(newbuf_addr, kMemBufIdle, newbuf_size); + // Add map of new memory buf in the block + (void)mem_block->block_all_mem_buf_map_.emplace(newbuf_addr, new_mem_buf); + // Add map of new idle memory buf + (void)global_idle_mem_buf_map_.emplace(newbuf_size, new_mem_buf); +} + +bool DynamicMemPoolBestFit::CmpMemBlock(const DeviceMemPtr device_addr, const DynamicMemBlockPtr mem_block) { + MS_EXCEPTION_IF_NULL(device_addr); + MS_EXCEPTION_IF_NULL(mem_block); + return device_addr < mem_block->device_addr(); +} + +DynamicMemBlockPtr DynamicMemPoolBestFit::FindMemBlock(const DeviceMemPtr device_addr) { + MS_EXCEPTION_IF_NULL(device_addr); + auto iter = std::upper_bound(global_mem_block_list_.begin(), global_mem_block_list_.end(), device_addr, CmpMemBlock); + if (iter != global_mem_block_list_.begin()) { + return *(--iter); + } + MS_LOG(ERROR) << "Can't find the mem_block of the device address[" << device_addr << "]."; + return nullptr; +} + +void DynamicMemPoolBestFit::FreeTensorMem(const DeviceMemPtr device_addr) { + MS_EXCEPTION_IF_NULL(device_addr); + auto mem_block = FindMemBlock(device_addr); + MS_EXCEPTION_IF_NULL(mem_block); + CombineMemBuf(mem_block, device_addr); +} + +void DynamicMemPoolBestFit::CombineMemBuf(const DynamicMemBlockPtr& mem_block, const DeviceMemPtr device_addr) { + MS_EXCEPTION_IF_NULL(mem_block); + MS_EXCEPTION_IF_NULL(device_addr); + auto iter = mem_block->block_all_mem_buf_map_.find(device_addr); + if (iter == mem_block->block_all_mem_buf_map_.end()) { + MS_LOG(EXCEPTION) << "Can't find the device address[" << device_addr << "]."; + } + auto mem_buf = iter->second; + MS_EXCEPTION_IF_NULL(mem_buf); + if (mem_buf->status_ != kMemBufUsed) { + MS_LOG(EXCEPTION) << "Find the mem_buf is not used, mem_buf_address[" << mem_buf->device_addr_ << "]."; + } + mem_buf->status_ = kMemBufIdle; + total_used_mem_statistics_ -= mem_buf->size_; + // Combine backward(combine the next_mem_buf to mem_buf) + auto next_iter = iter; + (void)next_iter++; + if (next_iter != mem_block->block_all_mem_buf_map_.end()) { + auto next_mem_buf = next_iter->second; + MS_EXCEPTION_IF_NULL(next_mem_buf); + if (next_mem_buf->status_ == kMemBufIdle) { + mem_buf->size_ += next_mem_buf->size_; + EraseIdleMemBuf(next_mem_buf->size_, next_mem_buf->device_addr_); + (void)mem_block->block_all_mem_buf_map_.erase(next_iter); + } + } + // Combine forward(combine the mem_buf to prev_mem_buf) + bool forward_combine = false; + DynamicMemBufPtr prev_mem_buf; + if (iter != mem_block->block_all_mem_buf_map_.begin()) { + auto prev_iter = iter; + (void)prev_iter--; + prev_mem_buf = prev_iter->second; + MS_EXCEPTION_IF_NULL(prev_mem_buf); + if (prev_mem_buf->status_ == kMemBufIdle) { + EraseIdleMemBuf(prev_mem_buf->size_, prev_mem_buf->device_addr_); + prev_mem_buf->size_ += mem_buf->size_; + (void)mem_block->block_all_mem_buf_map_.erase(iter); + forward_combine = true; + } + } + // Add map of new idle memory + if (forward_combine) { + (void)global_idle_mem_buf_map_.emplace(prev_mem_buf->size_, prev_mem_buf); + } else { + (void)global_idle_mem_buf_map_.emplace(mem_buf->size_, mem_buf); + } +} + +void DynamicMemPoolBestFit::EraseIdleMemBuf(size_t size, const DeviceMemPtr device_addr) { + MS_EXCEPTION_IF_NULL(device_addr); + auto iter = global_idle_mem_buf_map_.equal_range(size); + while (iter.first != iter.second) { + MS_EXCEPTION_IF_NULL(iter.first->second); + // Remove map of the idle memory buf by size and device address + if (iter.first->second->device_addr_ == device_addr) { + (void)global_idle_mem_buf_map_.erase(iter.first); + return; + } + (void)iter.first++; + } + MS_LOG(ERROR) << "Can't find the size[" << size << "] and device address[" << device_addr << "] in the idle mem_buf."; +} + +void DynamicMemPoolBestFit::ReleaseDeviceRes() { + MS_LOG(INFO) << "The dynamic memmory pool total size is " << total_mem_statistics_ << ", total used size is " + << total_used_mem_statistics_ << ", used peak size is " << used_mem_peak_statistics_ << "."; + for (auto iter = global_mem_block_list_.begin(); iter != global_mem_block_list_.end(); ++iter) { + auto device_addr = (*iter)->device_addr(); + if (device_addr != nullptr) { + if (!FreeDeviceMem(device_addr)) { + MS_LOG(EXCEPTION) << "Free device memory[" << device_addr << "] error."; + } + } + } +} + +void DynamicMemPoolBestFit::DumpDynamicMemPoolInfo() { + MS_LOG(INFO) << "Start dump dynamic memory pool info."; + DeviceAddrMapMemBuf mem_block_map; + DynamicMemBufPtr mem_buf; + size_t total_mem = 0; + size_t total_used_mem = 0; + size_t total_idle_mem1 = 0; + size_t total_idle_mem2 = 0; + // Dump the memory block info and memory buf info + MS_LOG(INFO) << "Dump all mem_block info: counts[" << global_mem_block_list_.size() << "]."; + for (auto iter = global_mem_block_list_.begin(); iter != global_mem_block_list_.end(); ++iter) { + total_mem += (*iter)->size(); + mem_block_map = (*iter)->block_all_mem_buf_map_; + MS_LOG(INFO) << "MemBlock info: number[" << iter - global_mem_block_list_.begin() << "] mem_buf_counts[" + << mem_block_map.size() << "] base_address[" << (*iter)->device_addr() << "] block_size[" + << (*iter)->size() << "]."; + for (auto iter_mem_buf = mem_block_map.begin(); iter_mem_buf != mem_block_map.end(); ++iter_mem_buf) { + mem_buf = iter_mem_buf->second; + MS_EXCEPTION_IF_NULL(mem_buf); + if (mem_buf->status_ == kMemBufIdle) { + total_idle_mem1 += mem_buf->size_; + } else { + total_used_mem += mem_buf->size_; + } + MS_LOG(INFO) << "MemBuf info: address[" << mem_buf->device_addr_ << "] size[" << mem_buf->size_ << "] status[" + << mem_buf->status_ << "]."; + } + } + // Dump all the idle memory buf info + MS_LOG(INFO) << "Dump all idle mem_buf info: counts[" << global_idle_mem_buf_map_.size() << "]."; + for (auto iter_idle = global_idle_mem_buf_map_.begin(); iter_idle != global_idle_mem_buf_map_.end(); ++iter_idle) { + mem_buf = iter_idle->second; + MS_EXCEPTION_IF_NULL(mem_buf); + total_idle_mem2 += mem_buf->size_; + MS_LOG(INFO) << "Idle mem_buf info: size[" << mem_buf->size_ << "] address[" << mem_buf->device_addr_ << "] status[" + << mem_buf->status_ << "]."; + } + // Dump the memory statistical info + MS_LOG(INFO) << "Total allocated memory[" << total_mem << "], used memory[" << total_used_mem << "], idle memory[" + << total_idle_mem1 << "]."; + if (total_idle_mem1 != total_idle_mem2) { + MS_LOG(ERROR) << "Check error: the idle memory in the mem_block is not equal the global idle memory."; + } + if (total_mem != total_used_mem + total_idle_mem1) { + MS_LOG(ERROR) << "Check error: the the total memory is not equal the sum of used memory and idle memory."; + } + MS_LOG(INFO) << "Finish dump dynamic memory pool info."; +} +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_dynamic_allocator.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_dynamic_allocator.h new file mode 100644 index 00000000000..dcf735814ca --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_dynamic_allocator.h @@ -0,0 +1,140 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_DYNAMIC_ALLOCATOR_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_DYNAMIC_ALLOCATOR_H_ + +#include +#include +#include +#include +#include + +namespace mindspore { +namespace device { +using DeviceMemPtr = void(*); + +// The status of memory buf. +enum DynamicMemBufStatus : int { kMemBufIdle, kMemBufUsed }; + +// Alloc memory aligned according to 512 bytes. +static const size_t DYNAMIC_MEM_ALIGN_SIZE = 512; + +// The minimum unit size (500M) of memory block used for dynamic extend. +static const size_t DYNAMIC_MEM_ALLOC_UNIT_SIZE = 500 << 20; + +// The Comparator of device address from small to large. +struct DeviceAddrCmp { + bool operator()(const DeviceMemPtr addr1, const DeviceMemPtr addr2) const { return addr1 < addr2; } +}; + +// Memory buf is the smallest operation object of dynamic memory pool. +struct DynamicMemBuf { + DynamicMemBuf(DeviceMemPtr addr, DynamicMemBufStatus status, size_t size) + : device_addr_(addr), status_(status), size_(size) {} + DeviceMemPtr device_addr_; + DynamicMemBufStatus status_; + size_t size_; +}; +using DynamicMemBufPtr = std::shared_ptr; +// Multimap key is the tensor size, for finding the idle memory buf by tensor size. +using SizeMapMemBuf = std::multimap; +// Map key is the device address, for finding the used memory buf in memory block by device address. +using DeviceAddrMapMemBuf = std::map; + +// Memory block is composed of memory buf. +class DynamicMemBlock { + public: + DynamicMemBlock() = default; + DynamicMemBlock(DeviceMemPtr addr_base, size_t size) : device_addr_base_(addr_base), mem_block_size_(size) {} + ~DynamicMemBlock() { block_all_mem_buf_map_.clear(); } + const DeviceMemPtr& device_addr() const { return device_addr_base_; } + size_t size() const { return mem_block_size_; } + // The map of all memory buf in this memory block by device address. + DeviceAddrMapMemBuf block_all_mem_buf_map_; + + private: + DeviceMemPtr device_addr_base_{nullptr}; + size_t mem_block_size_{0}; +}; +using DynamicMemBlockPtr = std::shared_ptr; + +// The main class of dynamic memory pool. +class DynamicMemPoolBestFit { + public: + DynamicMemPoolBestFit() = default; + virtual ~DynamicMemPoolBestFit(); + // The main program entry of memory alloc. + DeviceMemPtr AllocTensorMem(size_t size); + // The main program entry of memory free. + void FreeTensorMem(const DeviceMemPtr device_addr); + // Release the real device memory. + void ReleaseDeviceRes(); + // Display the information of memory block and memory buf. + void DumpDynamicMemPoolInfo(); + + // Get the related memory statistics information. + size_t total_mem_statistics() const { return total_mem_statistics_; } + size_t used_mem_statistics() const { return total_used_mem_statistics_; } + size_t used_mem_peak_statistics() const { return used_mem_peak_statistics_; } + + // The related interface of device memory real operation, needs override by device type. + virtual size_t AllocDeviceMem(size_t size, DeviceMemPtr* addr) = 0; + virtual bool FreeDeviceMem(const DeviceMemPtr& addr) = 0; + virtual size_t free_mem_size() = 0; + virtual size_t total_mem_size() = 0; + + protected: + // The real size by memory alloc aligned. + virtual size_t AlignMemorySize(size_t size) const; + // Get the minimum memory unit size using for dynamic extend. + virtual size_t mem_alloc_unit_size() const { return DYNAMIC_MEM_ALLOC_UNIT_SIZE; } + + private: + // Find the idle memory buf by aligned size when memory alloc. + DeviceMemPtr FindIdleMemBuf(size_t size); + // Add the memory block and memory buf when memory alloc not find the idle memory buf. + DeviceMemPtr AddMemBlockAndMemBuf(size_t size); + // Calculate memory block required alloc size when adding the memory block. + size_t CalMemBlockAllocSize(size_t size); + // Judge whether need divide the memory buf by alloc size and memory buf size. + bool IsDivide(size_t tensor_size, size_t mem_buf_size) const; + // Divide the memory buf by alloc size. + void DivideMemBuf(size_t size, const DynamicMemBufPtr& mem_buf); + // Find the memory block by deivce address. + DynamicMemBlockPtr FindMemBlock(const DeviceMemPtr device_addr); + // The Comparator of memory block by device address, because memory blocks are arranged in order by device address. + static bool CmpMemBlock(const DeviceMemPtr device_addr, const DynamicMemBlockPtr mem_block); + + // Combine the memory buf when memory free, to avoid the memory fragmentation. + void CombineMemBuf(const DynamicMemBlockPtr& mem_block, const DeviceMemPtr device_addr); + // Erase the idle memory buf by size and device address when idle memory buf is combined. + void EraseIdleMemBuf(size_t size, const DeviceMemPtr device_addr); + + // The global memory block list which is arranged in order by base device address of memory block. + std::vector global_mem_block_list_; + // The map of all idle memory buf by size. + SizeMapMemBuf global_idle_mem_buf_map_; + + // The related memory statistics information. + size_t total_mem_statistics_{0}; + size_t total_used_mem_statistics_{0}; + size_t used_mem_peak_statistics_{0}; +}; +} // namespace device +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_DYNAMIC_ALLOCATOR_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc new file mode 100644 index 00000000000..0db3c35196c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc @@ -0,0 +1,320 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/mem_reuse/mem_reuse.h" +#include +#include +#include "pre_activate/mem_reuse/mem_reuse_checker.h" +namespace mindspore { +namespace memreuse { +bool MemReuseUtil::InitDynamicOutputKernelRef() { + int index = util_index_; + auto kernel_cnodes = graph_->execution_order(); + if (kernel_cnodes.empty()) { + return true; + } + int kernel_out_ref_num = 0; + for (auto &kernel_cnode : kernel_cnodes) { +#ifdef MEM_REUSE_DEBUG + MemReuseChecker::GetInstance().CheckSignalOps(kernel_cnode); +#endif + if (kernel_cnode == nullptr) { + return false; + } + auto kernel_mod = AnfAlgo::GetKernelMod(kernel_cnode); + if (kernel_mod == nullptr) { + return false; + } + auto key = kernel_cnode.get(); + // for every apply_kernel to set new output + auto iter = kernel_output_refs_.find(key); + if (iter == kernel_output_refs_.end()) { + auto output_sizes = kernel_mod->GetOutputSizeList(); + KernelRefCountPtrList kernel_refs; + for (auto size : output_sizes) { + total_dy_size_ += size; + // do not MallocDynamicMem just record this + KernelRefCountPtr kernel_ref = std::make_shared(); + index++; + auto curr_stream_id = AnfAlgo::GetStreamId(kernel_cnode); + kernel_ref->stream_id_ = curr_stream_id; + kernel_ref->SetKernelRefCountInfo(index, size, kDynamicRefCount); + kernel_refs.push_back(kernel_ref); + kernel_out_ref_num++; + total_refs_list_.push_back(kernel_ref); + } + if (!kernel_refs.empty()) { + kernel_output_refs_[key] = kernel_refs; + } + } + } + return true; +} + +bool MemReuseUtil::InitDynamicWorkspaceKernelRef() { + int WkIndex = util_index_; + auto kernel_cnodes = graph_->execution_order(); + if (kernel_cnodes.empty()) { + return true; + } + for (auto &kernel_cnode : kernel_cnodes) { + if (kernel_cnode == nullptr) { + return false; + } + auto kernel_mod = AnfAlgo::GetKernelMod(kernel_cnode); + if (kernel_mod == nullptr) { + return false; + } + auto key = kernel_cnode.get(); + auto workspace_sizes = kernel_mod->GetWorkspaceSizeList(); + KernelRefCountPtrList workspace_kernel_refs; + for (auto size : workspace_sizes) { + total_workspace_size_ += size; + ++WkIndex; + KernelRefCountPtr workspace_ref = std::make_shared(); + workspace_ref->SetKernelRefCountInfo(WkIndex, size, kDynamicRefCount); + workspace_kernel_refs.push_back(workspace_ref); + // total wk ref + total_wk_ref_list_.push_back(workspace_ref); + } + if (!workspace_kernel_refs.empty()) { + // every key index wk_refs + kernel_workspace_refs_[key] = workspace_kernel_refs; + } + } + return true; +} + +bool MemReuseUtil::InitDynamicKernelRef(const KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(graph); + graph_ = graph; + if (!InitDynamicOutputKernelRef()) { + MS_LOG(INFO) << "InitDynamicOutputKernelRef fail"; + return false; + } + if (!InitDynamicWorkspaceKernelRef()) { + MS_LOG(INFO) << "InitDynamicWorkspaceKernelRef fail"; + return false; + } + return true; +} + +// set longest worspace list && largest workspace sizes +void MemReuseUtil::SetWorkSpaceList() { + int max_list_size = 0; + std::vector total_sizes; + std::vector max_list; + auto kernel_cnodes = graph_->execution_order(); + for (auto &kernel_cnode : kernel_cnodes) { + MS_EXCEPTION_IF_NULL(kernel_cnode); + auto cnode_key = kernel_cnode.get(); + auto cnode_iter = kernel_workspace_refs_.find(cnode_key); + if (cnode_iter != kernel_workspace_refs_.end()) { + auto kernel_refs = cnode_iter->second; + std::vector current_list; + for (size_t i = 0; i < kernel_refs.size(); ++i) { + auto size = kernel_refs[i]->size_; + current_list.push_back(size); + } + if (max_list_size < SizeToInt(current_list.size())) { + max_list_size = SizeToInt(current_list.size()); + } + (void)std::copy(current_list.begin(), current_list.end(), std::back_inserter(total_sizes)); + } + } + sort(total_sizes.rbegin(), total_sizes.rend()); + max_list.resize(IntToSize(max_list_size)); + if (SizeToInt(total_sizes.size()) < max_list_size) { + MS_LOG(EXCEPTION) << "total workspace size is less than required max list size"; + } + max_list.assign(total_sizes.begin(), total_sizes.begin() + max_list_size); + for (auto &ma : max_list) { + total_reuseworkspace_size_ += ma; + } + max_workspace_size_ = max_list_size; + max_workspace_list_ = max_list; +} + +void MemReuseUtil::SetInputMap(const CNodePtr &kernel, KernelDef *kernel_def_ptr) { + MS_EXCEPTION_IF_NULL(kernel); + MS_EXCEPTION_IF_NULL(kernel_def_ptr); + auto key = kernel.get(); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(kernel); ++i) { + auto ref_ptr = GetKernelInputRef(kernel, i); + if (ref_ptr != nullptr) { + if (ref_ptr->reftype() == kStaticRefCount) { + continue; + } else if (ref_ptr->reftype() == kDynamicRefCount) { + auto iter = kernel_def_ptr->inputs_.find(key); + if (iter == kernel_def_ptr->inputs_.end()) { + kernel_def_ptr->inputs_[key].push_back(ref_ptr); + } else { + if (std::any_of(iter->second.begin(), iter->second.end(), + [ref_ptr](const KernelRefCountPtr &it) { return (it.get() == ref_ptr.get()); })) { + break; + } + iter->second.push_back(ref_ptr); + } + } + } + } +} + +void MemReuseUtil::SetOutputMap(const CNodePtr &kernel, KernelDef *kernel_def_ptr) { + MS_EXCEPTION_IF_NULL(kernel); + MS_EXCEPTION_IF_NULL(kernel_def_ptr); + auto key = kernel.get(); + auto iter = kernel_def_ptr->outputs_.find(key); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (size_t k = 0; k < kernel_mod->GetOutputSizeList().size(); ++k) { + KernelRefCountPtr kernel_ref = kernel_output_refs_[key][k]; + if (iter == kernel_def_ptr->outputs_.end()) { + kernel_def_ptr->outputs_[key].push_back(kernel_ref); + } else { + if (std::any_of(iter->second.begin(), iter->second.end(), + [kernel_ref](const KernelRefCountPtr &it) { return (it == kernel_ref); })) { + break; + } + iter->second.push_back(kernel_ref); + } + } +} + +void MemReuseUtil::SetWkMap(const CNodePtr &kernel, KernelDef *kernel_def_ptr) { + MS_EXCEPTION_IF_NULL(kernel); + MS_EXCEPTION_IF_NULL(kernel_def_ptr); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + auto key = kernel.get(); + for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + if (kernel_workspace_refs_.find(key) != kernel_workspace_refs_.end()) { + auto wk_refs = kernel_workspace_refs_[key]; + if (i < wk_refs.size()) { + auto wk_ref = wk_refs[i]; + kernel_def_ptr->wk_space_[key].push_back(wk_ref); + } else { + MS_LOG(EXCEPTION) << "current index: " << i << " larger than wk_refs size " << wk_refs.size(); + } + } else { + MS_LOG(EXCEPTION) << "kernel_workspace_refs_ init error"; + } + } +} + +KernelRefCountPtr MemReuseUtil::GetRef(const AnfNodePtr &node, int output_idx) { + if (node == nullptr) { + MS_LOG(EXCEPTION) << "The node pointer is a nullptr."; + } + if (node->isa()) { + auto ak_node = node->cast(); + auto key = ak_node.get(); + MemReuseChecker::GetInstance().CheckOutRef(kernel_output_refs_, ak_node, IntToSize(output_idx)); + return kernel_output_refs_[key][IntToSize(output_idx)]; + } + return nullptr; +} + +KernelRefCountPtr MemReuseUtil::GetKernelInputRef(const CNodePtr &kernel, size_t input_idx) { + if (input_idx >= AnfAlgo::GetInputTensorNum(kernel)) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " + << AnfAlgo::GetInputTensorNum(kernel); + } + auto input_node = kernel->input(input_idx + 1); + auto kernel_input = AnfAlgo::VisitKernel(input_node, 0); + auto result = GetRef(kernel_input.first, SizeToInt(kernel_input.second)); + return result; +} + +void MemReuseUtil::SetKernelDefMap() { + auto kernel_cnodes = graph_->execution_order(); + for (auto &kernel : kernel_cnodes) { + KernelDefPtr kernel_def_ptr = std::make_shared(); + kernel_def_ptr->set_kernel_name(AnfAlgo::GetCNodeName(kernel)); + kernel_def_ptr->set_scope_full_name(kernel->fullname_with_scope()); + kernel_def_ptr->set_stream_id(AnfAlgo::GetStreamId(kernel)); + SetInputMap(kernel, kernel_def_ptr.get()); + SetOutputMap(kernel, kernel_def_ptr.get()); + SetWkMap(kernel, kernel_def_ptr.get()); + auto key = kernel.get(); + kernel_def_ptr->set_input_refs(kernel_def_ptr->inputs_[key]); + kernel_def_ptr->set_output_refs(kernel_def_ptr->outputs_[key]); + kernel_def_ptr_list_.push_back(kernel_def_ptr); + } +} + +void MemReuseUtil::SetReuseRefCount() { + auto kernels = graph_->execution_order(); + for (auto &kernel : kernels) { + auto key = kernel.get(); + for (auto &def : kernel_def_ptr_list_) { + auto iter = def->inputs_.find(key); + if (iter != def->inputs_.end()) { + for (auto &input : iter->second) { + input->ref_count_++; + input->ref_count_dynamic_use_++; + } + } + } + } +} + +void MemReuseUtil::SetGraphOutputRefCount() { + for (const auto &output : graph_->outputs()) { + MS_EXCEPTION_IF_NULL(output); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(output); ++i) { + if (!(output->isa())) { + continue; + } + auto cnode = output->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto input_node = cnode->input(i + 1); + MS_EXCEPTION_IF_NULL(input_node); + auto kernel_input = AnfAlgo::VisitKernel(input_node, 0); + MS_EXCEPTION_IF_NULL(kernel_input.first); + if (!(kernel_input.first->isa())) { + continue; + } + auto ak_node = kernel_input.first->cast(); + auto key = ak_node.get(); + auto iter = kernel_output_refs_.find(key); + if ((iter != kernel_output_refs_.end()) && (kernel_input.second < iter->second.size())) { + auto kernel_ref_count_ptr = kernel_output_refs_[key][kernel_input.second]; + MS_EXCEPTION_IF_NULL(kernel_ref_count_ptr); + kernel_ref_count_ptr->ref_count_ = kMaxRefCount; + kernel_ref_count_ptr->ref_count_dynamic_use_ = kMaxRefCount; + } + } + } +#ifdef MEM_REUSE_DEBUG + auto graph = *graph_; + MemReuseChecker::GetInstance().CheckMemReuseIR(total_refs_list_, kernel_def_ptr_list_, &graph); +#endif +} + +void MemReuseUtil::SetAllInfo(KernelGraph *graph) { + if (!InitDynamicKernelRef(graph)) { + MS_LOG(EXCEPTION) << "Init ReuseAssignDynamicMemory Fault"; + } + SetKernelDefMap(); + SetReuseRefCount(); + SetWorkSpaceList(); +#ifdef MEM_REUSE_DEBUG + MemReuseChecker::GetInstance().CheckMemReuseIR(total_refs_list_, kernel_def_ptr_list_, graph); +#endif +} +} // namespace memreuse +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h new file mode 100644 index 00000000000..6ecd222688c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h @@ -0,0 +1,96 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_H_ +#include +#include +#include +#include "pre_activate/mem_reuse/kernel_refcount.h" +#include "session/anf_runtime_algorithm.h" +#include "session/kernel_graph.h" +#include "kernel/tbe/tbe_utils.h" +using mindspore::kernel::tbe::TbeUtils; +namespace mindspore { +namespace memreuse { +static constexpr int kMaxRefCount = 9999; +static constexpr size_t kDefaultMemAlignSize = 512; +static constexpr size_t kAttAlignSize = 31; +static constexpr int kInvalidIndex = -2; + +using KernelDefPtrMaps = std::vector; +using KernelRefs = std::map; + +using KernelGraph = mindspore::session::KernelGraph; + +class MemReuseUtil { + public: + KernelRefs kernel_output_refs_; + KernelRefCountPtrList total_refs_list_; + KernelRefCountPtrList total_wk_ref_list_; + KernelRefs kernel_workspace_refs_; + MemReuseUtil() : util_index_(kInitIndex), graph_(nullptr) {} + ~MemReuseUtil() { + if (graph_ != nullptr) { + graph_ = nullptr; + } + MS_LOG(INFO) << "Total Dynamic Memory Size: " << total_dy_size_; + MS_LOG(INFO) << "Total WorkSpace Memory Size: " << total_workspace_size_; + MS_LOG(INFO) << "Total Reused WorkSpafce Memory Size: " << total_reuseworkspace_size_; + } + + void SetAllInfo(KernelGraph *graph); + bool InitDynamicOutputKernelRef(); + bool InitDynamicWorkspaceKernelRef(); + bool InitDynamicKernelRef(const KernelGraph *graph); + void SetWorkSpaceList(); + void SetKernelDefMap(); + void SetInputMap(const CNodePtr &kernel, KernelDef *kernel_def_ptr); + void SetOutputMap(const CNodePtr &kernel, KernelDef *kernel_def_ptr); + void SetWkMap(const CNodePtr &kernel, KernelDef *kernel_def_ptr); + void SetReuseRefCount(); + // Set the reference count of graph output specially. + void SetGraphOutputRefCount(); + + KernelRefCountPtr GetRef(const AnfNodePtr &node, int output_idx); + KernelRefCountPtr GetKernelInputRef(const CNodePtr &kernel, size_t input_idx); + KernelRefCountPtrList total_refs_list() const { return total_refs_list_; } + KernelRefCountPtrList total_wk_ref_list() const { return total_wk_ref_list_; } + KernelDefPtrMaps kernel_def_ptr_list() const { return kernel_def_ptr_list_; } + int max_workspace_size() const { return max_workspace_size_; } + std::vector max_workspace_list() const { return max_workspace_list_; } + void set_total_refs_list(const KernelRefCountPtrList &total_refs_list) { total_refs_list_ = total_refs_list; } + void set_kernel_def_ptr_list(const KernelDefPtrMaps &kernel_def_ptr_list) { + kernel_def_ptr_list_ = kernel_def_ptr_list; + } + + private: + int util_index_; + const KernelGraph *graph_; + KernelRefCountPtrList ref_list_; + KernelDefPtrMaps kernel_def_ptr_list_; + KernelRefCountPtrList last_ref_list_; + int max_workspace_size_ = 0; + std::vector max_workspace_list_; + size_t total_dy_size_ = 0; + size_t total_workspace_size_ = 0; + size_t total_reuseworkspace_size_ = 0; +}; +using MemReuseUtilPtr = std::shared_ptr; +} // namespace memreuse +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_allocator.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_allocator.cc new file mode 100644 index 00000000000..1cecd170d30 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_allocator.cc @@ -0,0 +1,450 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/mem_reuse/mem_reuse_allocator.h" +#include +#include +#include +#include "pre_activate/mem_reuse/mem_reuse.h" +#include "pre_activate/mem_reuse/mem_reuse_checker.h" + +namespace mindspore { +namespace memreuse { +void BestFitMemReuse::InitMemReuseInfo(const MemReuseUtil *mem_reuse_util_ptr) { + MS_EXCEPTION_IF_NULL(mem_reuse_util_ptr); + tensor_ptr_list_ = mem_reuse_util_ptr->total_refs_list(); + wk_tensor_list_ = mem_reuse_util_ptr->total_wk_ref_list(); + op_ptr_list_ = mem_reuse_util_ptr->kernel_def_ptr_list(); + // check info Correctness + for (auto &tensor : tensor_ptr_list_) { + tensor->size_ = AlignMemorySize(tensor->size_); + } + // align wk size to 512 && refcount == 1 + for (auto &wk : wk_tensor_list_) { + wk->size_ = AlignMemorySize(wk->size_); + wk->ref_count_ = 1; + } + auto stream_reuse = std::make_shared(); + stream_reuse->SetStreamReuseResource(); + parallel_streams_map_ = stream_reuse->parallel_streams_map(); +} + +bool BestFitMemReuse::CheckMembufIndx(const std::vector &membuf_ptr_list, size_t check_idx) const { + return check_idx < membuf_ptr_list.size(); +} + +bool BestFitMemReuse::IsMembufListEmpty(const std::vector &membuf_ptr_list) const { + return membuf_ptr_list.empty(); +} + +int BestFitMemReuse::GetFacIdx(size_t real_idx, int flag) const { + if (flag == kDyFac) { + return SizeToInt(real_idx); + } else if (flag == kWkFac) { + auto wk_fac_idx = kWkIndexFactor * SizeToInt(real_idx + 1); + return wk_fac_idx; + } else { + MS_LOG(EXCEPTION) << "flag " << flag << " is invalid"; + } +} + +int BestFitMemReuse::GetRealIdx(int fac_idx, int flag) const { + // membuf index maybe invalid_index + if (fac_idx == kInvalidIndex) { + MS_LOG(EXCEPTION) << "this membuf index is invalid"; + } + if (flag == kDyFac) { + return fac_idx; + } else if (flag == kWkFac) { + if (fac_idx % 10 == 0) { + auto wk_fac_idx = fac_idx / kWkIndexFactor + 1; + return wk_fac_idx; + } else { + MS_LOG(EXCEPTION) << "fac_idx: " << fac_idx << "is invalid"; + } + } else { + MS_LOG(EXCEPTION) << "flag: " << flag << " is invalid"; + } +} + +void BestFitMemReuse::AssignNodeOutputOffset(const KernelDef *kernel_def_ptr) { + MS_EXCEPTION_IF_NULL(kernel_def_ptr); + for (auto &tensor_idx : kernel_def_ptr->GetOutputRefIndexs()) { + CheckTensorIndex(tensor_idx); + auto tensor_desc = tensor_ptr_list_[IntToSize(tensor_idx)]; + MS_EXCEPTION_IF_NULL(tensor_desc); + auto reusable_membuf_map = GetReusableMembufMap(tensor_desc->size_); + if (!reusable_membuf_map.empty()) { + auto membuf_index = reusable_membuf_map.begin()->second; + // find the best suitable membuf in membuf list, and reuse it + ReuseExistMembuf(tensor_desc.get(), membuf_index, kDyFac); + } else { + // no membuf can reuse, add new membuf after the membuf_ptr_list + AddNewMembufPtr(tensor_desc.get(), kDyFac); +#ifdef MEM_REUSE_DEBUG + MemReuseChecker::GetInstance().IsAddNewMembuf_ = true; +#endif + } + } +} + +void BestFitMemReuse::AssignNodeWkOffset(const KernelDef *kernel_def_ptr) { + MS_EXCEPTION_IF_NULL(kernel_def_ptr); + for (auto &wk_idx : kernel_def_ptr->GetWkRefIndexs()) { + if (IntToSize(wk_idx) >= wk_tensor_list_.size()) { + MS_LOG(EXCEPTION) << "wk_idx: " << wk_idx << " is invalid"; + } + auto wk_ref = wk_tensor_list_[IntToSize(wk_idx)]; + MS_EXCEPTION_IF_NULL(wk_ref); + auto re_wk_membuf_map = GetReusableMembufMap(wk_ref->size_); + if (!re_wk_membuf_map.empty()) { + auto membuf_index = re_wk_membuf_map.begin()->second; + ReuseExistMembuf(wk_ref.get(), membuf_index, kWkFac); + } else { + AddNewMembufPtr(wk_ref.get(), kWkFac); + } + } +} +// releas pre node wk +void BestFitMemReuse::ReleasePreNodeWkSpace(const KernelDef *kernel_def_ptr) { + for (auto &wk_idx : kernel_def_ptr->GetWkRefIndexs()) { + auto wk_index = IntToSize(wk_idx); + if (wk_index >= wk_tensor_list_.size()) { + MS_LOG(EXCEPTION) << "wk_index: " << wk_index << " is larger than wk_tensor_list size" << wk_tensor_list_.size(); + } + auto wk_tensor = wk_tensor_list_[wk_index]; + wk_tensor->ref_count_--; + if (wk_tensor->ref_count_ == 0) { + ReleaseMembuf(wk_index, kWkFac); + } + } +} + +void BestFitMemReuse::ReuseExistMembuf(KernelRefCount *tensor_desc, size_t membuf_index, int flag) { + MS_EXCEPTION_IF_NULL(tensor_desc); + if (!CheckMembufIndx(membuf_ptr_list_, membuf_index)) { + return; + } + auto membuf = membuf_ptr_list_[membuf_index]; + MS_EXCEPTION_IF_NULL(membuf); + // first to split && then update membuf_info + if (IsSplit(tensor_desc->size_, membuf->size_)) { + // split the membuf, and insert a new membuf after this membuf + SplitMembuf(tensor_desc, membuf_index); + } + // update membuf status, and set tensor offset + UpdateMembufInfo(tensor_desc, membuf.get(), flag); +} + +std::map BestFitMemReuse::GetReusableMembufMap(size_t tensor_size) { + std::map size_map; + for (size_t i = 0; i < membuf_ptr_list_.size(); ++i) { + auto membuf = membuf_ptr_list_[i]; + auto called_ids = membuf->called_stream_ids_; + auto index = i; + bool IsMembufOk = membuf->status_ == kUnused && membuf->size_ >= tensor_size; + bool has_parallel_id = HasParallelId(called_ids, current_stream_id_); + if (IsMembufOk && !has_parallel_id) { + (void)size_map.insert(std::make_pair(membuf->size_, index)); + break; + } + } + return size_map; +} + +void BestFitMemReuse::UpdateMembufInfo(KernelRefCount *tensor_desc, Membuf *membuf, int flag) { + MS_EXCEPTION_IF_NULL(tensor_desc); + MS_EXCEPTION_IF_NULL(membuf); + auto fac_idx = GetFacIdx(IntToSize(tensor_desc->index_), flag); + membuf->status_ = kReused; + membuf->stream_id_ = current_stream_id_; + // clear before called_ids + membuf->called_stream_ids_.clear(); + (void)membuf->called_stream_ids_.insert(current_stream_id_); + membuf->index_ = fac_idx; + tensor_desc->offset_ = membuf->offset_; +} + +bool BestFitMemReuse::IsSplit(size_t tensor_size, size_t membuf_size) const { return tensor_size < membuf_size; } + +void BestFitMemReuse::SplitMembuf(const KernelRefCount *tensor_desc, size_t membuf_index) { + MS_EXCEPTION_IF_NULL(tensor_desc); + if (!CheckMembufIndx(membuf_ptr_list_, membuf_index)) { + return; + } + auto membuf = membuf_ptr_list_[membuf_index]; + MS_EXCEPTION_IF_NULL(membuf); + auto bias = membuf->size_ - tensor_desc->size_; + membuf->size_ = tensor_desc->size_; + // to check if spilt membuf can be merge + auto new_membuf = + std::make_shared(current_stream_id_, kUnused, bias, membuf->offset_ + membuf->size_, kInvalidIndex); + (void)membuf_ptr_list_.insert(membuf_ptr_list_.begin() + SizeToInt(membuf_index + 1), new_membuf); + MergeCalledIds(membuf.get(), new_membuf.get()); +} + +void BestFitMemReuse::AddNewMembufPtr(KernelRefCount *tensor_desc, int flag) { + MS_EXCEPTION_IF_NULL(tensor_desc); + size_t membuf_offset = std::accumulate(membuf_ptr_list_.begin(), membuf_ptr_list_.end(), IntToSize(0), + [](size_t sum, MembufPtr &membuf) { return sum + membuf->size_; }); + size_t membuf_size = tensor_desc->size_; + auto fac_idx = GetFacIdx(IntToSize(tensor_desc->index_), flag); + auto membuf = std::make_shared(current_stream_id_, kReused, membuf_size, membuf_offset, fac_idx); + membuf_ptr_list_.push_back(membuf); + tensor_desc->offset_ = membuf_offset; + (void)membuf->called_stream_ids_.insert(current_stream_id_); +} + +void BestFitMemReuse::UpdateNodeInputAndMembuf(const KernelDef *kernel_def_ptr) { + // process node input tensor + for (const auto &tensor_idx : kernel_def_ptr->GetInputRefIndexs()) { + auto tensor_index = IntToSize(tensor_idx); + CheckTensorIndex(tensor_idx); + auto tensor_desc = tensor_ptr_list_[tensor_index]; + auto fac_idx = GetFacIdx(tensor_index, kDyFac); + MS_EXCEPTION_IF_NULL(tensor_desc); + tensor_desc->ref_count_--; + // find tensor_index -> membuf update it's called_ids + for (size_t i = 0; i < membuf_ptr_list_.size(); ++i) { + auto membuf = membuf_ptr_list_[i]; + // find it + if (membuf->index_ == fac_idx) { + (void)membuf->called_stream_ids_.insert(current_stream_id_); + break; + } + } + if (tensor_desc->ref_count_ == 0) { + ReleaseMembuf(tensor_index, kDyFac); + } else if (tensor_desc->ref_count_ < 0) { + MS_LOG(EXCEPTION) << "tensor: " << tensor_desc->index_ << " refcount: " << tensor_desc->ref_count_ + << " check error"; + } + } +} + +void BestFitMemReuse::ReleaseNodeUnusedOutput(const KernelDef *kernel_def_ptr) { + for (auto &tensor_idx : kernel_def_ptr->GetOutputRefIndexs()) { + auto tensor_index = IntToSize(tensor_idx); + CheckTensorIndex(tensor_idx); + auto tensor_desc = tensor_ptr_list_[tensor_index]; + MS_EXCEPTION_IF_NULL(tensor_desc); + if (tensor_desc->ref_count_ == 0) { + ReleaseMembuf(tensor_index, kDyFac); + } else if (tensor_desc->ref_count_ < 0) { + MS_LOG(EXCEPTION) << "tensor: " << tensor_desc->index_ << " refcount: " << tensor_desc->ref_count_ + << " check error"; + } + } +} + +size_t BestFitMemReuse::FindIndx(const std::vector &membuf_ptr_list, int fac_idx) const { + size_t membuf_index = 0; + for (size_t n = 0; n < membuf_ptr_list.size(); ++n) { + auto membuf = membuf_ptr_list[n]; + if (membuf->index_ == fac_idx) { + membuf_index = n; + break; + } + } + return membuf_index; +} + +void BestFitMemReuse::ReleaseMembuf(size_t tensor_index, int flag) { + auto fac_idex = GetFacIdx(tensor_index, flag); + auto membuf_index = FindIndx(membuf_ptr_list_, fac_idex); + if (!CheckMembufIndx(membuf_ptr_list_, membuf_index)) { + return; + } + auto membuf = membuf_ptr_list_[membuf_index]; + MS_EXCEPTION_IF_NULL(membuf); + membuf->status_ = kUnused; + if (membuf_index != (membuf_ptr_list_.size() - 1)) { + auto membuf_next = membuf_ptr_list_[membuf_index + 1]; + MS_EXCEPTION_IF_NULL(membuf_next); + bool has_parallel_id = false; + for (auto &cal_id : membuf->called_stream_ids_) { + has_parallel_id = HasParallelId(membuf_next->called_stream_ids_, cal_id); + if (has_parallel_id) { + break; + } + } + if (membuf_next->status_ == kUnused && !has_parallel_id) { + membuf->size_ += membuf_next->size_; + MergeCalledIds(membuf_next.get(), membuf.get()); + auto it = membuf_ptr_list_.begin() + SizeToInt(membuf_index + 1); + (void)membuf_ptr_list_.erase(it); + } + } + if (membuf_index != 0) { + if (!CheckMembufIndx(membuf_ptr_list_, membuf_index - 1)) { + return; + } + auto membuf_prev = membuf_ptr_list_[membuf_index - 1]; + MS_EXCEPTION_IF_NULL(membuf_prev); + bool has_parallel_id = false; + for (auto &cal_id : membuf->called_stream_ids_) { + has_parallel_id = HasParallelId(membuf_prev->called_stream_ids_, cal_id); + if (has_parallel_id) { + break; + } + } + if (membuf_prev->status_ == kUnused && !has_parallel_id) { + membuf->size_ += membuf_prev->size_; + membuf->offset_ = membuf_prev->offset_; + MergeCalledIds(membuf_prev.get(), membuf.get()); + auto it = membuf_ptr_list_.begin() + SizeToInt(membuf_index - 1); + (void)membuf_ptr_list_.erase(it); + } + } +} + +bool BestFitMemReuse::HasParallelId(const std::set &called_ids, uint32_t curr_id) { + if (called_ids.empty()) { + MS_LOG(EXCEPTION) << "There is a invalid WkMembuf,called_ids is empty"; + } + for (auto item : called_ids) { + if (!IsReusableStream(curr_id, item)) { + return true; + } + } + return false; +} + +void BestFitMemReuse::MergeCalledIds(const Membuf *membuf_target, Membuf *membuf) { + MS_EXCEPTION_IF_NULL(membuf_target); + MS_EXCEPTION_IF_NULL(membuf); + for (auto target : membuf_target->called_stream_ids_) { + (void)membuf->called_stream_ids_.insert(target); + } +} + +void BestFitMemReuse::ReleaseParallStream() { + std::vector target_relea_idxs; + for (size_t i = 0; i < membuf_ptr_list_.size(); ++i) { + auto membuf = membuf_ptr_list_[i]; + if (membuf->status_ == kReused) { + continue; + } + // for begin to end, so no need merge pre_membuf + if (i != (membuf_ptr_list_.size() - 1)) { + auto membuf_next = membuf_ptr_list_[i + 1]; + if (membuf_next->status_ == kReused) { + continue; + } + MS_EXCEPTION_IF_NULL(membuf_next); + // judge current id no parallel fro membuf && membuf_next + bool has_parallel_id_crr = HasParallelId(membuf->called_stream_ids_, current_stream_id_); + bool has_parallel_id_next = HasParallelId(membuf_next->called_stream_ids_, current_stream_id_); + if (membuf->status_ == kUnused && membuf_next->status_ == kUnused && !has_parallel_id_crr && + !has_parallel_id_next) { + membuf->size_ += membuf_next->size_; + MergeCalledIds(membuf_next.get(), membuf.get()); + target_relea_idxs.push_back(i + 1); + } + } + } + // erase all target membuf + std::vector membuf_ptr_list_tmp; + for (size_t j = 0; j < membuf_ptr_list_.size(); ++j) { + for (auto idx : target_relea_idxs) { + if (j != idx) { + membuf_ptr_list_tmp.push_back(membuf_ptr_list_[j]); + } + } + } + membuf_ptr_list_.clear(); + (void)std::copy(membuf_ptr_list_tmp.begin(), membuf_ptr_list_tmp.end(), back_inserter(membuf_ptr_list_)); +} + +size_t BestFitMemReuse::AlignMemorySize(size_t size) const { + // memory size 512 align + return (size + kDefaultMemAlignSize + kAttAlignSize) / kDefaultMemAlignSize * kDefaultMemAlignSize; +} + +size_t BestFitMemReuse::GetAllocatedSize() { + size_t AllocatedSize = kTotalSize; + if (membuf_ptr_list_.empty()) { + return AllocatedSize; + } + AllocatedSize = (*membuf_ptr_list_.rbegin())->offset_ + (*membuf_ptr_list_.rbegin())->size_; + MS_LOG(INFO) << "MemReuse Allocated Dynamic Size: " << AllocatedSize; + return AllocatedSize; +} + +/** + * parallel_streams_map: key, current_stream_id; value, streams parallel to current stream + * @param curr_stream_id + * @param target_stream_id + * @return bool, if the target stream can be reused by current stream + */ +bool BestFitMemReuse::IsReusableStream(uint32_t curr_stream_id, uint32_t target_stream_id) { + auto iter_parall = parallel_streams_map_.find(curr_stream_id); + if (parallel_streams_map_.empty() || (iter_parall == parallel_streams_map_.end())) { + // no parallel stream exists + return true; + } + auto curr_parallel_set = iter_parall->second; + return curr_parallel_set.find(target_stream_id) == curr_parallel_set.end(); +} + +void BestFitMemReuse::CheckTensorIndex(int tensor_index) const { + if (tensor_index < 0) { + MS_LOG(EXCEPTION) << "warning, please check tensor info."; + } + if (IntToSize(tensor_index) >= tensor_ptr_list_.size()) { + MS_LOG(EXCEPTION) << "invalid tensor index"; + } +} + +void BestFitMemReuse::Reuse(const MemReuseUtil *mem_reuse_util_ptr) { + MS_EXCEPTION_IF_NULL(mem_reuse_util_ptr); + InitMemReuseInfo(mem_reuse_util_ptr); + KernelDefPtr pre_op = nullptr; +#ifdef MEM_REUSE_DEBUG + size_t op_num = 0; +#endif + for (const auto &op_def_ptr : op_ptr_list_) { + current_stream_id_ = op_def_ptr->stream_id(); + // releas pre_op_def + if (pre_op != nullptr) { + ReleasePreNodeWkSpace(pre_op.get()); + } + MemReuseChecker::GetInstance().IsAddNewMembuf_ = false; + // process node output tensor + AssignNodeOutputOffset(op_def_ptr.get()); +#ifdef MEM_REUSE_DEBUG + if (MemReuseChecker::GetInstance().IsAddNewMembuf_) { + MemReuseChecker::GetInstance().SetAddNewMembuInfos(op_def_ptr.get(), membuf_ptr_list_, op_num); + } +#endif + // deal with current op'workspace + AssignNodeWkOffset(op_def_ptr.get()); + pre_op = op_def_ptr; + // update node input tensor refcount, and membuf list status + UpdateNodeInputAndMembuf(op_def_ptr.get()); + // check node output tensor which refcount is equal to zero +#ifdef MEM_REUSE_DEBUG + MemReuseChecker::GetInstance().SetMembuInfos(op_def_ptr.get(), membuf_ptr_list_); + ++op_num; +#endif + } +#ifdef MEM_REUSE_DEBUG + MemReuseChecker::GetInstance().ExportMembufInfoIR(); + MemReuseChecker::GetInstance().ExportAddNewMmebufIR(); +#endif +} +} // namespace memreuse +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_allocator.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_allocator.h new file mode 100644 index 00000000000..e41c20d6201 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_allocator.h @@ -0,0 +1,124 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_ALLOCATOR_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_ALLOCATOR_H_ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pre_activate/mem_reuse/kernel_refcount.h" +#include "pre_activate/mem_reuse/mem_reuse.h" +#include "pre_activate/mem_reuse/stream_reuse.h" + +namespace mindspore { +namespace memreuse { +static constexpr int kWkIndexFactor = -1000; +static constexpr int kDyFac = -1; +static constexpr int kWkFac = 1; +static constexpr size_t kTotalSize = 0; +enum Status { kUnused, kReused }; +class Membuf { + public: + Membuf() = default; + Membuf(uint32_t stream_id, Status status, size_t size, size_t offset, int index) + : stream_id_(stream_id), status_(status), size_(size), offset_(offset), index_(index) {} + ~Membuf() = default; + // Memory block status flags + std::set called_stream_ids_; + uint32_t stream_id_{0}; + Status status_ = kUnused; + size_t size_{0}; + size_t offset_{0}; + // Store the tensor index stored in this memory block at a certain moment + int index_{0}; +}; +using MembufPtr = std::shared_ptr; + +class BestFitMemReuse { + public: + BestFitMemReuse() = default; + ~BestFitMemReuse() { membuf_ptr_list_.clear(); } + // Init all information need by memory reuse + void InitMemReuseInfo(const MemReuseUtil *mem_reuse_util_ptr); + bool CheckMembufIndx(const std::vector &membuf_ptr_list, size_t check_idx) const; + bool IsMembufListEmpty(const std::vector &membuf_ptr_list) const; + void AssignNodeWkOffset(const KernelDef *kernel_def_ptr); + void ReleasePreNodeWkSpace(const KernelDef *kernel_def_ptr); + // void assign node output tensor memory offset + void AssignNodeOutputOffset(const KernelDef *kernel_def_ptr); + void ReleaseParallStream(); + // update node input tensor refcount, and membuf list status + void UpdateNodeInputAndMembuf(const KernelDef *kernel_def_ptr); + // check node output tensor which refcount is equal to zero + void ReleaseNodeUnusedOutput(const KernelDef *kernel_def_ptr); + // If there are memory blocks that can be reused + void ReuseExistMembuf(KernelRefCount *tensor_desc, size_t membuf_index, int flag); + // Save memory blocks that can be reused to the map + std::map GetReusableMembufMap(size_t tensor_size); + // Update the status of the reused memory block + void UpdateMembufInfo(KernelRefCount *tensor_desc, Membuf *membuf, int flag); + // If the size of the memory block is greater than the size of the tensor, split the extra memory + void SplitMembuf(const KernelRefCount *tensor_desc, size_t membuf_index); + // Determine if the memory block needs to be split + bool IsSplit(size_t tensor_size, size_t membuf_size) const; + // If there is no memory block that can be reused, add a new memory block at the end + void AddNewMembufPtr(KernelRefCount *tensor_desc, int flag); + // Merge unused membuf + void ReleaseMembuf(size_t tensor_index, int flag); + bool HasParallelId(const std::set &called_ids, uint32_t curr_id); + void MergeCalledIds(const Membuf *membuf_target, Membuf *membuf); + // Memory address alignment 512 + size_t AlignMemorySize(size_t size) const; + int GetFacIdx(size_t real_idx, int flag = kDyFac) const; + int GetRealIdx(int fac_idx, int flag = kDyFac) const; + size_t FindIndx(const std::vector &membuf_ptr_list, int fac_idx) const; + void CheckTensorIndex(int tensor_index) const; + // Memory reuse main program entry + void Reuse(const MemReuseUtil *mem_reuse_util_ptr); + // Get the total memory that needs to be applied eventually + size_t GetAllocatedSize(); + // If the target stream can be reused by current stream + bool IsReusableStream(uint32_t curr_stream_id, uint32_t target_stream_id); + // set tensor_def and op_def + void set_tensor_ptr_list(const std::vector &tensor_ptr_list) { + tensor_ptr_list_ = tensor_ptr_list; + } + void set_op_ptr_list(const std::vector &op_ptr_list) { op_ptr_list_ = op_ptr_list; } + + private: + uint32_t current_stream_id_{0}; + // Save all tensor information + std::vector tensor_ptr_list_; + std::vector wk_tensor_list_; + // Save all op information, including input and output tensor index + std::vector op_ptr_list_; + // Memory block information sequence, temporary variables + std::vector membuf_ptr_list_; + std::unordered_map> parallel_streams_map_; +}; +} // namespace memreuse +} // namespace mindspore +#endif // #define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_ALLOCATOR_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_checker.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_checker.cc new file mode 100644 index 00000000000..6825da24dad --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_checker.cc @@ -0,0 +1,517 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/mem_reuse/mem_reuse_checker.h" +#include +#include +#include +#include +#include +#include + +namespace mindspore { +namespace memreuse { +MemReuseChecker &MemReuseChecker::GetInstance() { + static MemReuseChecker instance; + return instance; +} + +void MemReuseChecker::CheckSignalOps(const CNodePtr &c_node) { + std::string node_name = AnfAlgo::GetCNodeName(c_node); + if (node_name == kSend || node_name == kRecv) { + MS_LOG(INFO) << "MemReuseChecker check op_name of Send or Send"; + // get op's info && check + MS_LOG(INFO) << "op: " << node_name << " in_num: " << AnfAlgo::GetInputTensorNum(c_node) + << " out_num: " << AnfAlgo::GetOutputTensorNum(c_node); + } +} + +void MemReuseChecker::CheckWorkSpace(const std::vector &max_list) { + for (auto &ma : max_list) { + total_re_wkspe_size_checker_ += ma; + } +} + +void MemReuseChecker::CheckOutRef(const KernelRefs &kernel_refs, const CNodePtr &c_node, size_t output_idx) { + auto key = c_node.get(); + auto iter = kernel_refs.find(key); + auto node_name = AnfAlgo::GetCNodeName(c_node); + if (iter == kernel_refs.end()) { + MS_LOG(EXCEPTION) << "kernel [" << node_name << "] has no output tensor"; + } + if (output_idx >= iter->second.size()) { + MS_LOG(INFO) << "invalid cnode: " << c_node->fullname_with_scope().c_str(); + MS_LOG(EXCEPTION) << "The index: " << output_idx + << " is out of the size of kernel_output_refs_:" << iter->second.size(); + } +} + +int64_t MemReuseChecker::CalculOriInput(const KernelGraph *graph) const { + MS_EXCEPTION_IF_NULL(graph); + int64_t static_input_size = 0; + for (auto &item : graph->inputs()) { + if (!item->isa()) { + continue; + } + auto output_size = AnfAlgo::GetOutputTensorNum(item); + for (size_t index = 0; index < output_size; index++) { + TypeId ou_type = AnfAlgo::GetOutputDeviceDataType(item, index); + // parameter has not init by a cnode + if (ou_type == kTypeUnknown) { + ou_type = AnfAlgo::GetOutputInferDataType(item, index); + } + size_t type_size = GetTypeByte(TypeIdToType(ou_type)); + std::vector shape = AnfAlgo::GetOutputDeviceShape(item, index); + size_t tensor_size = + shape.empty() ? type_size : std::accumulate(shape.begin(), shape.end(), type_size, std::multiplies()); + auto checker_size = SizeToLong(tensor_size); + static_input_size += checker_size; + } + } + return static_input_size; +} + +int64_t MemReuseChecker::CalculOriValue(KernelGraph *graph) const { + MS_EXCEPTION_IF_NULL(graph); + int64_t static_value_size = 0; + for (auto &value_node : graph->graph_value_nodes()) { + MS_EXCEPTION_IF_NULL(value_node); + auto &node_value = value_node->value(); + MS_EXCEPTION_IF_NULL(node_value); + auto tensor = node_value->cast(); + if (tensor == nullptr) { + continue; + } + size_t tensor_size = tensor->data().nbytes(); + auto checker_size = SizeToLong(tensor_size); + static_value_size += checker_size; + } + return static_value_size; +} + +int64_t MemReuseChecker::CalculOriStatic(KernelGraph *graph) const { + // cal static inputs + auto static_input_size = CalculOriInput(graph); + // do not calcul outpput size + auto statica_value_size = CalculOriValue(graph); + auto total_ori_static_size = static_input_size + statica_value_size; + return total_ori_static_size; +} + +int64_t MemReuseChecker::CalculOriDy(const KernelGraph *graph) const { + MS_EXCEPTION_IF_NULL(graph); + int64_t ori_dy_size = 0; + auto kerenls = graph->execution_order(); + for (auto &kernel : kerenls) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (auto &dy_size : kernel_mod->GetOutputSizeList()) { + auto checker_size = SizeToLong(dy_size); + ori_dy_size += checker_size; + } + } + return ori_dy_size; +} + +int64_t MemReuseChecker::CalculOriWk(const KernelGraph *graph) const { + MS_EXCEPTION_IF_NULL(graph); + int64_t ori_wk_size = 0; + auto kerenls = graph->execution_order(); + for (auto &kernel : kerenls) { + auto kernel_mod = AnfAlgo::GetKernelMod(kernel); + MS_EXCEPTION_IF_NULL(kernel_mod); + for (auto &wk_size : kernel_mod->GetWorkspaceSizeList()) { + auto checker_size = SizeToLong(wk_size); + ori_wk_size += checker_size; + } + } + return ori_wk_size; +} + +std::string MemReuseChecker::GetSplitName(const std::string &scope_name) const { + auto indx = scope_name.rfind(kSplitC); + if (indx == std::string::npos) { + return scope_name; + } else { + if (indx < scope_name.size() - 1) { + auto split_name = scope_name.substr(indx + 1); + return split_name; + } + return scope_name; + } +} + +void MemReuseChecker::CheckMemReuseIR(const KernelRefCountPtrList &total_refs_list, + const KernelDefPtrMaps &kernel_def_ptr_list, KernelGraph *graph) { + total_ori_static_size_ = CalculOriStatic(graph); + total_ori_input_size_ = CalculOriInput(graph); + total_ori_value_size_ = CalculOriValue(graph); + total_ori_dy_size_ = CalculOriDy(graph); + total_ori_wkspace_size_ = CalculOriWk(graph); + std::string filename = "./memreuse.ir"; + std::ofstream ofs(filename); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file [" << filename << "] failed!"; + return; + } + ofs << "all_tensor_refs:\n"; + ofs << "index:" + << "\tsize:" + << "\trefcount:\n"; + for (auto &ref : total_refs_list) { + ofs << "%" << ref->index_ << "T" + << "\t" + << "#" << ref->size_ << "S" + << "\t" << ref->ref_count_ << "C" + << "\n"; + } + ofs << "kernel_def exc_order:\n"; + int def_idx = 0; + for (auto &def : kernel_def_ptr_list) { + ExportMemOpIr(def.get(), ofs, def_idx); + def_idx++; + } + ofs.close(); +} + +bool MemReuseChecker::CheckGraphOutputAssigned(const session::KernelGraph *graph) { + // set real graph output node to be special who's refcount equal kMaxRefCount + for (const auto &output : graph->outputs()) { + MS_EXCEPTION_IF_NULL(output); + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(output); ++i) { + if (output->isa()) { + auto cnode = output->cast(); + auto input_node = cnode->input(i + 1); + auto kernel_input_with_idx = AnfAlgo::VisitKernel(input_node, 0); + auto kernel_input = kernel_input_with_idx.first; + MS_EXCEPTION_IF_NULL(kernel_input); + auto kernel_mod = AnfAlgo::GetKernelMod(kernel_input); + if (kernel_mod == nullptr) { + continue; + } + auto output_sizes = kernel_mod->GetOutputSizeList(); + if (output_sizes.empty()) { + continue; + } + for (size_t j = 0; j < output_sizes.size(); ++j) { + if (!AnfAlgo::OutputAddrExist(kernel_input, j)) { + return false; + } + } + } + } + } + return true; +} + +void MemReuseChecker::ExportMemOpIr(const KernelDef *def, std::ofstream &ofs, int def_idx) { + auto scope_name = def->scope_full_name(); + std::string split_name = GetSplitName(scope_name); + ofs << "$" << def_idx << "\t" << split_name << "\t"; + ofs << "inputs["; + for (auto &in : def->inputs_) { + for (auto &in_ref : in.second) { + ofs << "%" << in_ref->index_ << "T" + << ","; + } + } + ofs << "]"; + ofs << "\toutpus["; + for (auto &ou : def->outputs_) { + for (auto &ou_ref : ou.second) { + ofs << "%" << ou_ref->index_ << "T" + << ","; + } + } + ofs << "]"; + ofs << "\tstreamID[" + << "@" << def->stream_id() << "]\n"; +} + +void MemReuseChecker::ExportNormalTensorIR(std::ofstream &ofs) { + ofs << "all_tensor_refs:\n"; + ofs << "index:" + << "\tsize:" + << "\trefcount:\n"; + size_t ou_idx = 0; + for (auto &ou : nor_output_tensors_) { + ofs << "%" << ou_idx << "T" + << "\t" + << "#" << nor_tensor_sizes_[ou_idx] << "S" + << "\t"; + auto iter_ref = ptr_refs_.find(ou); + if (iter_ref != ptr_refs_.end()) { + ofs << iter_ref->second << "C" + << "\n"; + } else { + MS_LOG(EXCEPTION) << "can not find refs for output"; + } + ou_idx++; + } + ofs << "kernel_def exc_order:\n"; +} + +int MemReuseChecker::GetTensorIdx(const void *in) const { + auto iter = ptr_idx_.find(in); + if (iter == ptr_idx_.end()) { + return kInvalidIndex; + } else { + return SizeToInt(iter->second); + } +} + +void MemReuseChecker::ExportNormalOpIr(const std::vector &cnodes) { + std::ofstream ofs("./normal_mem.ir"); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file failed!"; + return; + } + ExportNormalTensorIR(ofs); + size_t node_idx = 0; + for (const auto &node : cnodes) { + MS_EXCEPTION_IF_NULL(node); + ofs << "$" << node_idx << "\t" << GetSplitName(node->fullname_with_scope()) << "\t"; + std::vector in_idx; + auto iter = node_ins_.find(node.get()); + if (iter != node_ins_.end()) { + for (auto &in : iter->second) { + if (GetTensorIdx(in) != kInvalidIndex) { + in_idx.push_back(GetTensorIdx(in)); + } + } + } + std::vector ou_idx; + iter = node_ous_.find(node.get()); + if (iter != node_ous_.end()) { + for (auto &ou : iter->second) { + if (GetTensorIdx(ou) != kInvalidIndex) { + ou_idx.push_back(GetTensorIdx(ou)); + } + } + } + ofs << "inputs["; + for (auto idx : in_idx) { + bool has_in_ou = std::any_of(ou_idx.begin(), ou_idx.end(), [idx](int odx) { return idx == odx; }); + if (!has_in_ou) { + ofs << "%" << idx << "T,"; + } + } + ofs << "]\toutpus["; + for (auto odx : ou_idx) { + ofs << "%" << odx << "T,"; + } + ofs << "]\tstreamID[@" << AnfAlgo::GetStreamId(node) << "]\n"; + node_idx++; + } + ofs.close(); +} + +void MemReuseChecker::SetTesnorFromAndToInfo(const KernelDef *op_def) { + auto split_name = GetSplitName(op_def->scope_full_name()); + for (auto &in : op_def->inputs_) { + auto in_tensors = in.second; + for (auto &tensor : in_tensors) { + auto indx = tensor->index_; + tensor_to_[indx].push_back(split_name); + } + } + for (auto &ou : op_def->outputs_) { + auto ou_tensors = ou.second; + for (auto &tensor : ou_tensors) { + auto indx = tensor->index_; + tensor_from_[indx].push_back(split_name); + } + } +} + +void MemReuseChecker::CheckNormalIR(const session::KernelGraph *graph) { + const auto &cnodes = graph->execution_order(); + for (const auto &node : cnodes) { + std::vector curr_ous; + for (size_t i = 0; i < AnfAlgo::GetOutputTensorNum(node); ++i) { + auto it = AnfAlgo::GetOutputAddr(node, i); + MS_EXCEPTION_IF_NULL(it); + auto ptr = it->GetPtr(); + nor_output_tensors_.push_back(ptr); + nor_tensor_sizes_.push_back(it->GetSize()); + curr_ous.push_back(it->GetPtr()); + } + (void)node_ous_.insert(std::make_pair(node.get(), curr_ous)); + std::vector curr_ins; + for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(node); ++i) { + if (i + 1 >= node->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index: " << i + << " is larger than input number: " << AnfAlgo::GetInputTensorNum(node); + } + auto real_input_index = AnfAlgo::GetRealInputIndex(node, i); + auto input = node->input(real_input_index + 1); + MS_EXCEPTION_IF_NULL(input); + auto kernel_with_index = AnfAlgo::VisitKernel(input, 0); + if (kernel_with_index.first->isa()) { + continue; + } + auto device_address = AnfAlgo::GetPrevNodeOutputAddr(node, real_input_index); + MS_EXCEPTION_IF_NULL(device_address); + nor_input_tensors_.push_back(device_address->GetPtr()); + curr_ins.push_back(device_address->GetPtr()); + } + (void)node_ins_.insert(std::make_pair(node.get(), curr_ins)); + } + size_t ou_idx = 0; + for (const auto &ou : nor_output_tensors_) { + (void)ptr_idx_.insert(std::make_pair(ou, ou_idx)); + (void)ptr_refs_.insert(std::make_pair(ou, 0)); + ou_idx++; + } + for (const auto &in : nor_input_tensors_) { + if (ptr_idx_.find(in) != ptr_idx_.end()) { + if (ptr_refs_.find(in) != ptr_refs_.end()) { + auto iter = ptr_refs_.find(in); + (iter->second)++; + } else { + MS_LOG(EXCEPTION) << "ptr_refs is not equal to ptr_idx"; + } + } + } + ExportNormalOpIr(cnodes); +} + +void MemReuseChecker::SetMembuInfos(const KernelDef *op_def, const std::vector &membuf_ptr_list) { + std::vector curr_mem_infos; + for (const auto &mem : membuf_ptr_list) { + auto mem_checker = std::make_shared(mem->stream_id_, mem->status_, mem->size_, mem->offset_, mem->index_); + curr_mem_infos.push_back(mem_checker); + } + membuf_all_infos_.push_back(curr_mem_infos); + auto split_name = GetSplitName(op_def->scope_full_name()); + all_split_names_.push_back(split_name); + SetTesnorFromAndToInfo(op_def); +} + +void MemReuseChecker::SetAddNewMembuInfos(const KernelDef *op_def, const std::vector &membuf_ptr_list, + size_t op_idx) { + std::vector add_new_curr_mem; + + for (const auto &mem : membuf_ptr_list) { + auto mem_checker = std::make_shared(mem->stream_id_, mem->status_, mem->size_, mem->offset_, mem->index_); + add_new_curr_mem.push_back(mem_checker); + } + add_new_mem_infos_.push_back(add_new_curr_mem); + auto split_name = GetSplitName(op_def->scope_full_name()); + add_new_names_.push_back(split_name); + add_new_op_indxs_.push_back(op_idx); + add_new_stream_ids_.push_back(op_def->stream_id()); +} + +void MemReuseChecker::ExportMembufInfoIR() { + std::string ir_file_name = "./mem_buf_info.ir"; + std::ofstream ofs(ir_file_name); + int64_t total_reuse_size = 0; + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file [" << ir_file_name << "] failed!"; + } + ofs << "total_ori_static_size:" << total_ori_static_size_ << "\n"; + ofs << "total_ori_weight_size:" << total_ori_input_size_ << "\n"; + ofs << "total_ori_constant_size:" << total_ori_value_size_ << "\n"; + ofs << "total_ori_dy_size:" << total_ori_dy_size_ << "\n"; + ofs << "total_ori_wkspace_size:" << total_ori_wkspace_size_ << "\n"; + // get last membuf_list + if (membuf_all_infos_.empty()) { + return; + } + auto last_membuf_list = membuf_all_infos_.back(); + for (const auto &membuf : last_membuf_list) { + auto checker_size = SizeToLong(membuf->size_); + total_reuse_size += checker_size; + } + ofs << "total_reuse_size:" << total_reuse_size << "\n"; + size_t i = 0; + for (const auto &curr_membuf_list : membuf_all_infos_) { + ofs << all_split_names_.at(i) << "\n"; + ++i; + ofs << "mem_num\t" + << "stream_id\t" + << "status\t" + << "tensor_idex\t" + << "mem_size\t" + << "mem_head\t" + << "mem_tail\n"; + for (size_t j = 0; j < curr_membuf_list.size(); ++j) { + auto membuf = curr_membuf_list.at(j); + ofs << "&" << j << "\t" + << "streamID[@" << membuf->stream_id_ << "]" + << "\t" + << "#" << static_cast(membuf->status_) << "\t%" << membuf->index_ << "T" + << "\t" << membuf->size_ << "\t" << membuf->offset_ << "\t" << membuf->offset_ + membuf->size_ << "\n"; + } + ofs << "\n\n"; + } + ofs.close(); +} + +void MemReuseChecker::ExportAddNewMmebufIR() { + std::string ir_file_name = "./AddNewMembuf.ir"; + std::ofstream ofs(ir_file_name); + if (!ofs.is_open()) { + MS_LOG(ERROR) << "Open file [" << ir_file_name << "] failed!"; + } + auto check_idx = add_new_mem_infos_.size(); + if (check_idx == add_new_op_indxs_.size() && check_idx == add_new_names_.size() && + check_idx == add_new_stream_ids_.size()) { + size_t i = 0; + for (const auto &curr_membuf_list : add_new_mem_infos_) { + ofs << "op_idx:$" << add_new_op_indxs_.at(i) << "\t" << add_new_names_.at(i) << "\t"; + ofs << "streamID[@" << add_new_stream_ids_.at(i) << "]" + << "\n"; + i++; + ofs << "mem_num\t" + << "stream_id\t" + << "status\t" + << "tensor_idex\t" + << "mem_size\t" + << "mem_head\t" + << "mem_tail\t" + << "FromOp\t" + << "ToOp\n"; + for (size_t j = 0; j < curr_membuf_list.size(); ++j) { + auto membuf = curr_membuf_list.at(j); + ofs << "&" << j << "\t" + << "streamID[@" << membuf->stream_id_ << "]" + << "\t" + << "#" << static_cast(membuf->status_) << "\t%" << membuf->index_ << "T" + << "\t" << membuf->size_ << "\t" << membuf->offset_ << "\t" << membuf->offset_ + membuf->size_ << "\t"; + auto in_idx_iter = tensor_from_.find(membuf->index_); + if (in_idx_iter != tensor_from_.end()) { + for (auto &in_name : in_idx_iter->second) { + ofs << in_name << ","; + } + ofs << "\t"; + } + auto ou_idx_iter = tensor_to_.find(membuf->index_); + if (ou_idx_iter != tensor_to_.end()) { + for (auto &ou_name : ou_idx_iter->second) { + ofs << ou_name << ","; + } + ofs << "\n"; + } + } + ofs << "\n"; + } + } + ofs.close(); +} +} // namespace memreuse +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_checker.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_checker.h new file mode 100644 index 00000000000..d225da94b67 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse_checker.h @@ -0,0 +1,90 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_CHECKER_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_CHECKER_H_ +#include +#include +#include +#include +#include +#include "mindspore/ccsrc/ir/anf.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/mem_reuse/mem_reuse.h" +#include "kernel/common_utils.h" +#include "pre_activate/mem_reuse/mem_reuse_allocator.h" +namespace mindspore { +namespace memreuse { +constexpr auto kSend = "Send"; +constexpr auto kRecv = "Recv"; +constexpr auto kSplitC = '/'; +class MemReuseChecker { + public: + bool IsAddNewMembuf_ = false; + static MemReuseChecker &GetInstance(); + MemReuseChecker(const MemReuseChecker &) = delete; + MemReuseChecker &operator=(const MemReuseChecker &) = delete; + void CheckSignalOps(const CNodePtr &c_node); + void CheckWorkSpace(const std::vector &max_list); + void CheckOutRef(const KernelRefs &kernel_refs, const CNodePtr &c_node, size_t output_idx); + bool CheckGraphOutputAssigned(const session::KernelGraph *graph); + void CheckMemReuseIR(const KernelRefCountPtrList &total_refs_list, const KernelDefPtrMaps &kernel_def_ptr_list, + KernelGraph *graph); + int64_t CalculOriStatic(KernelGraph *graph) const; + int64_t CalculOriInput(const KernelGraph *graph) const; + int64_t CalculOriValue(KernelGraph *graph) const; + int64_t CalculOriDy(const KernelGraph *graph) const; + int64_t CalculOriWk(const KernelGraph *graph) const; + std::string GetSplitName(const std::string &scope_name) const; + int GetTensorIdx(const void *in) const; + void SetMembuInfos(const KernelDef *op_def, const std::vector &membuf_ptr_list); + void SetTesnorFromAndToInfo(const KernelDef *op_def); + void ExportMemOpIr(const KernelDef *def, std::ofstream &ofs, int def_idx); + void ExportNormalOpIr(const std::vector &cnodes); + void ExportNormalTensorIR(std::ofstream &ofs); + void CheckNormalIR(const session::KernelGraph *graph); + void ExportMembufInfoIR(); + void SetAddNewMembuInfos(const KernelDef *op_def, const std::vector &membuf_ptr_list, size_t op_idx); + void ExportAddNewMmebufIR(); + + private: + MemReuseChecker() = default; + ~MemReuseChecker() { MS_LOG(INFO) << "Total reused workspace size: " << total_re_wkspe_size_checker_; } + size_t total_re_wkspe_size_checker_{0}; + std::vector> membuf_all_infos_; + std::vector nor_output_tensors_; + std::vector nor_tensor_sizes_; + std::vector nor_input_tensors_; + std::map ptr_idx_; + std::map ptr_refs_; + std::map> node_ins_; + std::map> node_ous_; + std::vector> add_new_mem_infos_; + std::vector add_new_names_; + std::vector add_new_op_indxs_; + std::vector add_new_stream_ids_; + std::vector all_split_names_; + std::map> tensor_from_; + std::map> tensor_to_; + int64_t total_ori_static_size_ = 0; + int64_t total_ori_input_size_ = 0; + int64_t total_ori_value_size_ = 0; + int64_t total_ori_dy_size_ = 0; + int64_t total_ori_wkspace_size_ = 0; +}; +} // namespace memreuse +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_MEM_REUSE_CHECKER_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/stream_reuse.cc b/mindspore/ccsrc/pre_activate/mem_reuse/stream_reuse.cc new file mode 100644 index 00000000000..d1409cdedda --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/stream_reuse.cc @@ -0,0 +1,102 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/mem_reuse/stream_reuse.h" + +namespace mindspore { +namespace memreuse { +void StreamReuse::SetStreamReuseResource() { +#ifdef ENABLE_D + auto logic_physic_map = device::ascend::AscendStreamAssign::GetInstance().GetPhysicMap(); + auto logic_independent_map = device::ascend::AscendStreamAssign::GetInstance().GetIndependentMap(); + MS_LOG(INFO) << "stream mem reuse for Davici"; + if (!logic_independent_map.empty() && !logic_physic_map.empty()) { + set_logic_physic_map(logic_physic_map); + set_logic_independent_map(logic_independent_map); + InitReusableStreamMap(); + } else { + MS_LOG(INFO) << "Non task sink or No Parallel stream exists"; + } +#endif + MS_LOG(INFO) << "no need to set stream mem reuse resource"; +} + +std::vector> StreamReuse::SortLogicPhysicMapToList() { + std::vector> logic_physic_list; + (void)std::transform(logic_physic_map_.begin(), logic_physic_map_.end(), std::back_inserter(logic_physic_list), + [](std::pair log_phy) { return log_phy; }); + std::sort( + logic_physic_list.begin(), logic_physic_list.end(), + [](const std::pair &logic_phyic_pair1, const std::pair &logic_phyic_pair2) { + return logic_phyic_pair1.second < logic_phyic_pair2.second; + }); + return logic_physic_list; +} + +std::unordered_map> StreamReuse::GetLogicPhysicsStreamMap() { + auto logic_physic_list = SortLogicPhysicMapToList(); + std::unordered_map> logic_phyics_map; + for (size_t i = 0; i < logic_physic_list.size() - IntToSize(1); ++i) { + auto curr_logic_physic = logic_physic_list.at(i); + auto next_logic_physic = logic_physic_list.at(i + 1); + for (auto j = curr_logic_physic.second; j < next_logic_physic.second; ++j) { + (void)logic_phyics_map[curr_logic_physic.first].insert(j); + } + } + // sort the logic independ map by value + std::map temp_map; + for (const auto &logic_independ : logic_independent_map_) { + (void)temp_map.insert(std::make_pair(logic_independ.second, logic_independ.first)); + } + auto first_independent_stream_id = (*temp_map.begin()).first; + auto last_physic_logic_stream_id = (*logic_physic_list.rbegin()).second; + for (auto i = last_physic_logic_stream_id; i < first_independent_stream_id; ++i) { + (void)logic_phyics_map[(*logic_physic_list.rbegin()).first].insert(i); + } + return logic_phyics_map; +} + +void StreamReuse::InitReusableStreamMap() { + // logic_phyics_map, key, logic_stream_id; value, physic_strema_ids included in that logic stream + auto logic_phyics_map = GetLogicPhysicsStreamMap(); + // parallel_streams_map: key, current_stream_id; value, streams parallel to current stream + for (const auto &logic_to_phyics : logic_phyics_map) { + auto logic_stream_id = logic_to_phyics.first; + auto iter_inde = logic_independent_map_.find(logic_stream_id); + if (iter_inde != logic_independent_map_.end()) { + // exist independent steam parallel to these logic streams + auto independent_stream_id = iter_inde->second; + auto physics_stream_id = logic_to_phyics.second; + for (const auto &physic : physics_stream_id) { + (void)parallel_streams_map_[physic].insert(independent_stream_id); + } + } + } + for (const auto &logic_to_independent : logic_independent_map_) { + auto logic_stream_id = logic_to_independent.first; + auto independent_stream_id = logic_to_independent.second; + auto iter_physics = logic_phyics_map.find(logic_stream_id); + if (iter_physics != logic_phyics_map.end()) { + // exist logic steam parallel to these independent streams, default + auto physics_set = iter_physics->second; + for (const auto &physic : physics_set) { + (void)parallel_streams_map_[independent_stream_id].insert(physic); + } + } + } +} +} // namespace memreuse +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/stream_reuse.h b/mindspore/ccsrc/pre_activate/mem_reuse/stream_reuse.h new file mode 100644 index 00000000000..cc972336500 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/mem_reuse/stream_reuse.h @@ -0,0 +1,63 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_STREAM_REUSE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_STREAM_REUSE_H_ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/mem_reuse/kernel_refcount.h" + +#ifdef ENABLE_D +#include "device/ascend/ascend_stream_assign.h" +#endif + +namespace mindspore { +namespace memreuse { +class StreamReuse { + public: + StreamReuse() = default; + ~StreamReuse() = default; + void SetStreamReuseResource(); + void InitReusableStreamMap(); + std::vector> SortLogicPhysicMapToList(); + std::unordered_map> GetLogicPhysicsStreamMap(); + void set_logic_physic_map(const std::unordered_map &logic_physic_map) { + logic_physic_map_ = logic_physic_map; + } + void set_logic_independent_map(const std::unordered_map &logic_independent_map) { + logic_independent_map_ = logic_independent_map; + } + std::unordered_map> parallel_streams_map() { return parallel_streams_map_; } + + private: + std::unordered_map> parallel_streams_map_; + std::unordered_map logic_physic_map_; + std::unordered_map logic_independent_map_; +}; +} // namespace memreuse +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_MEM_REUSE_STREAM_REUSE_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc new file mode 100644 index 00000000000..f8604d7638c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/common_subexpression_elimination.h" +#include +#include "device/kernel_info.h" + +namespace mindspore { +namespace opt { +namespace { +bool CheckEqualKernelBuildInfo(const AnfNodePtr &main, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(main); + MS_EXCEPTION_IF_NULL(node); + auto main_kernel_info = main->kernel_info(); + auto node_kernel_info = node->kernel_info(); + if (main_kernel_info == nullptr && node_kernel_info == nullptr) { + return true; + } + if (main_kernel_info != nullptr && node_kernel_info != nullptr) { + return *main_kernel_info == *node_kernel_info; + } + return false; +} +} // namespace + +bool BackendCSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const { + MS_EXCEPTION_IF_NULL(main); + MS_EXCEPTION_IF_NULL(node); + + bool replace = false; + if (main->isa() && node->isa()) { + auto main_value = GetValueNode(main); + auto node_value = GetValueNode(node); + if (main_value->isa() && node_value->isa()) { + replace = false; + } else { + replace = (AbsOf(main) == AbsOf(node)) && (*main_value == *node_value); + } + } else if (main->isa() && node->isa()) { + if (!CheckEqualKernelBuildInfo(main, node)) { + replace = false; + } else { + auto c_main = main->cast(); + MS_EXCEPTION_IF_NULL(c_main); + auto c_node = node->cast(); + MS_EXCEPTION_IF_NULL(c_node); + const auto &inp1 = c_main->inputs(); + const auto &inp2 = c_node->inputs(); + if (inp1.size() == inp2.size()) { + bool appsame = true; + for (size_t j = 0; j < inp1.size(); j++) { + MS_EXCEPTION_IF_NULL(inp1[j]); + MS_EXCEPTION_IF_NULL(inp2[j]); + if (!(*inp1[j] == *inp2[j])) { + appsame = false; + break; + } + } + replace = appsame; + } + } + } + return replace; +} + +bool CommonSubexpressionElimination::Run(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + auto backend_cse = std::make_shared(); + return backend_cse->Cse(func_graph, func_graph->manager()); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h new file mode 100644 index 00000000000..8e1768ea99c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_COMMON_SUBEXPRESSION_ELIMINATION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_COMMON_SUBEXPRESSION_ELIMINATION_H_ +#include "pre_activate/common/pass.h" +#include "optimizer/cse.h" + +namespace mindspore { +namespace opt { +class CommonSubexpressionElimination : public Pass { + public: + CommonSubexpressionElimination() : Pass("cse") {} + ~CommonSubexpressionElimination() override = default; + bool Run(const FuncGraphPtr &func_graph) override; +}; + +class BackendCSE : public CSE { + public: + BackendCSE() = default; + ~BackendCSE() override = default; + bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_COMMON_SUBEXPRESSION_ELIMINATION_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/const_input_to_attr_registry.cc b/mindspore/ccsrc/pre_activate/pass/const_input_to_attr_registry.cc new file mode 100644 index 00000000000..42d373392c6 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/const_input_to_attr_registry.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/const_input_to_attr_registry.h" + +#include + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace opt { +ConstInputToAttrInfoRegistry &ConstInputToAttrInfoRegistry::Instance() { + static ConstInputToAttrInfoRegistry instance; + return instance; +} + +void ConstInputToAttrInfoRegistry::Register(const ConstInputToAttrInfoRegister ®) { + auto op_name = reg.GetOpName(); + if (op_input_to_attr_map_.find(op_name) == op_input_to_attr_map_.end()) { + (void)op_input_to_attr_map_.insert(make_pair(op_name, reg)); + MS_LOG(DEBUG) << op_name << " const2attr register successfully!"; + } +} + +void ConstInputToAttrInfoRegistry::Register(const std::string &op_name, + const std::unordered_set &input_attr_set) { + if (op_input_to_attr_map_.find(op_name) == op_input_to_attr_map_.end()) { + ConstInputToAttrInfoRegister reg(op_name); + (void)reg.SetConstInputToAttr(input_attr_set); + (void)op_input_to_attr_map_.insert(make_pair(op_name, reg)); + MS_LOG(DEBUG) << op_name << " const2attr register successfully!"; + } +} + +bool ConstInputToAttrInfoRegistry::GetRegisterByOpName(const std::string &op_name, + ConstInputToAttrInfoRegister *reg) const { + if (op_input_to_attr_map_.find(op_name) != op_input_to_attr_map_.end()) { + *reg = op_input_to_attr_map_.at(op_name); + MS_LOG(DEBUG) << op_name << " const2attr find in registery."; + return true; + } + return false; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/const_input_to_attr_registry.h b/mindspore/ccsrc/pre_activate/pass/const_input_to_attr_registry.h new file mode 100644 index 00000000000..48007929fbe --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/const_input_to_attr_registry.h @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONST_INPUT_TO_ATTR_REGISTRY_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONST_INPUT_TO_ATTR_REGISTRY_H_ +#include +#include +#include + +#include "common/utils.h" + +namespace mindspore { +namespace opt { +class ConstInputToAttrInfoRegister { + public: + explicit ConstInputToAttrInfoRegister(const std::string &op_name = "") : op_name_(op_name) {} + virtual ~ConstInputToAttrInfoRegister() = default; + + ConstInputToAttrInfoRegister &SetConstInputToAttr(size_t input_index) { + (void)input_attr_set_.insert(input_index); + return *this; + } + + ConstInputToAttrInfoRegister &SetConstInputToAttr(const std::unordered_set &input_attr_set) { + (void)input_attr_set_.insert(input_attr_set.begin(), input_attr_set.end()); + return *this; + } + + const std::unordered_set &GetConstInputAttrInfo() const { return input_attr_set_; } + const std::string &GetOpName() const { return op_name_; } + + private: + std::string op_name_; + std::unordered_set input_attr_set_; +}; + +class ConstInputToAttrInfoRegistry { + public: + static ConstInputToAttrInfoRegistry &Instance(); + void Register(const ConstInputToAttrInfoRegister ®); + void Register(const std::string &op_name, const std::unordered_set &input_attr_set); + bool GetRegisterByOpName(const std::string &op_name, ConstInputToAttrInfoRegister *reg) const; + + private: + ConstInputToAttrInfoRegistry() = default; + ~ConstInputToAttrInfoRegistry() = default; + DISABLE_COPY_AND_ASSIGN(ConstInputToAttrInfoRegistry) + std::unordered_map op_input_to_attr_map_; +}; + +struct ConstInputToAttrInfoReceiver { + // Note: This is implicitly converting constructor + ConstInputToAttrInfoReceiver(const ConstInputToAttrInfoRegister ®) { // NOLINT(runtime/explicit) + ConstInputToAttrInfoRegistry::Instance().Register(reg); + } +}; +} // namespace opt + +#define REG_CONST_INPUT_TO_ATTR(op_name) REG_CONST_INPUT_TO_ATTR_UNIQ_HELPER(__COUNTER__, op_name) +#define REG_CONST_INPUT_TO_ATTR_UNIQ_HELPER(ctr, op_name) REG_CONST_INPUT_TO_ATTR_UNIQ(ctr, op_name) +#define REG_CONST_INPUT_TO_ATTR_UNIQ(ctr, op_name) \ + static ::mindspore::opt::ConstInputToAttrInfoReceiver g_const_input_to_attr_register_##ctr __attribute__((unused)) = \ + ::mindspore::opt::ConstInputToAttrInfoRegister(op_name) +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONST_INPUT_TO_ATTR_REGISTRY_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_attr.cc b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_attr.cc new file mode 100644 index 00000000000..2bef0d36ca1 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_attr.cc @@ -0,0 +1,123 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/convert_const_input_to_attr.h" + +#include +#include +#include +#include +#include + +#include "pre_activate/pass/const_input_to_attr_registry.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "operator/ops.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +namespace { +void ConstInputToAttr(const CNodePtr &cnode, const std::unordered_set &input_attrs) { + MS_EXCEPTION_IF_NULL(cnode); + std::vector new_inputs; + std::vector new_input_names; + auto primitive = AnfAlgo::GetCNodePrimitive(cnode); + MS_EXCEPTION_IF_NULL(primitive); + auto input_names = primitive->GetAttr(kAttrInputNames); + if (input_names == nullptr) { + MS_LOG(DEBUG) << "input_names are nullptr in cnode[" + cnode->DebugString() + "]"; + return; + } + auto input_names_vec = GetValue>(input_names); + auto inputs = cnode->inputs(); + new_inputs.push_back(inputs[0]); + bool need_update = false; + for (size_t i = 0; i < inputs.size() - 1; ++i) { + auto input_node = inputs[i + 1]; + MS_EXCEPTION_IF_NULL(input_node); + if (input_attrs.find(i) != input_attrs.end() && input_node->isa()) { + auto value_node = input_node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + MS_LOG(DEBUG) << "start erase input[" << i << "] of cnode[" + cnode->DebugString() + "]"; + if (i >= input_names_vec.size()) { + MS_LOG(EXCEPTION) << "index " << i << " is larger than input names size [" << input_names_vec.size() << "]"; + } + primitive->set_attr(input_names_vec[i], value_node->value()); + need_update = true; + } else { + new_inputs.push_back(input_node); + if (i < input_names_vec.size()) { + new_input_names.push_back(input_names_vec[i]); + } + } + } + if (need_update) { + // Update cnode's inputs + cnode->set_inputs(new_inputs); + // Update cnode's input_names attr + primitive->set_attr(kAttrInputNames, MakeValue(new_input_names)); + } +} +} // namespace + +const AnfNodePtr ConvertConstInputToAttr::Process(const FuncGraphPtr &, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || !AnfAlgo::IsRealCNodeKernel(node)) { + return nullptr; + } + CNodePtr cnode = node->cast(); + + ConstInputToAttrInfoRegister reg; + if (!ConstInputToAttrInfoRegistry::Instance().GetRegisterByOpName(AnfAlgo::GetCNodeName(cnode), ®)) { + return nullptr; + } + ConstInputToAttr(cnode, reg.GetConstInputAttrInfo()); + return cnode; +} + +void ConvertConstInputToAttr::Init() { + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimCast->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimConv2DBackpropInput->name(), {2}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimConv2DBackpropFilter->name(), {2}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimReshape->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimReduceMax->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimReduceMin->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimReduceSum->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimReduceMean->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimGatherV2->name(), {2}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimTranspose->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimUnsortedSegmentSum->name(), {2}); + ConstInputToAttrInfoRegistry::Instance().Register(prim::kPrimOneHot->name(), {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kUnsortedSegmentProdOpName, {2}); + ConstInputToAttrInfoRegistry::Instance().Register(kUnsortedSegmentMinOpName, {2}); + ConstInputToAttrInfoRegistry::Instance().Register(kSimpleMeanGradOpName, {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kMeanGradOpName, {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kSliceOpName, {1, 2}); + ConstInputToAttrInfoRegistry::Instance().Register(kSliceGradOpName, {2, 3}); + ConstInputToAttrInfoRegistry::Instance().Register(kTileOpName, {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kScatterNdOpName, {2}); + ConstInputToAttrInfoRegistry::Instance().Register(kStridedSliceAssignOpName, {1, 2, 3}); + ConstInputToAttrInfoRegistry::Instance().Register(kStridedSliceOpName, {1, 2, 3}); + ConstInputToAttrInfoRegistry::Instance().Register(kStridedSliceGradOpName, {1, 2, 3, 4}); + ConstInputToAttrInfoRegistry::Instance().Register(kFlattenGradOpName, {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kExpandDimsOpName, {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kSplitOpName, {0}); + ConstInputToAttrInfoRegistry::Instance().Register(kTopKOpName, {1}); + ConstInputToAttrInfoRegistry::Instance().Register(kSparseApplyAdagradOpName, {2}); + ConstInputToAttrInfoRegistry::Instance().Register(kResizeNearestNeighborGrad, {1}); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_attr.h b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_attr.h new file mode 100644 index 00000000000..54caa1633cb --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_attr.h @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_CONST_INPUT_TO_ATTR_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_CONST_INPUT_TO_ATTR_H_ +#include +#include +#include + +#include "ir/anf.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class ConvertConstInputToAttr : public PatternProcessPass { + public: + explicit ConvertConstInputToAttr(bool multigraph = true) + : PatternProcessPass("convert_const_input_to_attr", multigraph) { + Init(); + } + ~ConvertConstInputToAttr() override = default; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + void Init(); + std::unordered_map> op_input_attr_map_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_CONST_INPUT_TO_ATTR_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_tensor_input.cc b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_tensor_input.cc new file mode 100644 index 00000000000..431a67792de --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_tensor_input.cc @@ -0,0 +1,159 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/convert_const_input_to_tensor_input.h" + +#include +#include + +#include "utils/graph_utils.h" +#include "session/anf_runtime_algorithm.h" +#include "session/kernel_graph.h" + +namespace mindspore { +namespace opt { +namespace { +constexpr size_t kType32Len = 4; +template +tensor::TensorPtr CreateTensorWithValueTuple(const ValueTuplePtr &value_tuple_ptr, const TypePtr &type_ptr, + size_t data_length) { + MS_EXCEPTION_IF_NULL(value_tuple_ptr); + MS_EXCEPTION_IF_NULL(type_ptr); + std::vector values; + for (const auto &v : value_tuple_ptr->value()) { + MS_EXCEPTION_IF_NULL(v); + if (v->isa()) { + ScalarPtr scalar = v->cast(); + values.push_back(GetValue(scalar)); + } else { + MS_LOG(WARNING) << "The value " << v << "of tuple is not a scalar"; + return nullptr; + } + } + std::vector tensor_shape = {SizeToInt(values.size())}; + tensor::TensorPtr tensor = std::make_shared(type_ptr->type_id(), tensor_shape); + MS_EXCEPTION_IF_NULL(tensor); + tensor::DeviceInfo device_info{kOpFormat_DEFAULT, type_ptr}; + tensor->set_device_info(device_info); + auto data_ptr = tensor->data_c(true); + MS_EXCEPTION_IF_NULL(data_ptr); + auto elem_num = values.size() * data_length; + auto ret_code = memcpy_s(data_ptr, static_cast(tensor->data().nbytes()), values.data(), elem_num); + if (ret_code != 0) { + MS_LOG(EXCEPTION) << "Failed to copy data into Tensor."; + } + return tensor; +} + +tensor::TensorPtr CreateTupleTensor(const ValueTuplePtr &value_tuple) { + MS_EXCEPTION_IF_NULL(value_tuple); + tensor::TensorPtr tensor = nullptr; + ValuePtr v = *(value_tuple->value().begin()); + MS_EXCEPTION_IF_NULL(v); + // Currently we only deal with the scalar tuple + if (!v->isa()) { + MS_LOG(WARNING) << "The value " << v << "of tuple is not a scalar"; + return nullptr; + } + ScalarPtr scalar = v->cast(); + MS_EXCEPTION_IF_NULL(scalar); + if (scalar->isa()) { + tensor = CreateTensorWithValueTuple(value_tuple, kInt32, kType32Len); + } else if (scalar->isa()) { + tensor = CreateTensorWithValueTuple(value_tuple, kFloat32, kType32Len); + } else { + auto type = scalar->type(); + auto type_str = (type == nullptr) ? "nullptr" : type->ToString(); + MS_LOG(ERROR) << "Invalid scalar type: " << type_str; + return nullptr; + } + return tensor; +} + +AnfNodePtr CreateTensorInput(const KernelGraphPtr &kernel_graph, const AnfNodePtr &input_node) { + MS_EXCEPTION_IF_NULL(input_node); + auto value_node = input_node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + tensor::TensorPtr tensor_ptr = nullptr; + if (value->isa()) { + tensor_ptr = ScalarToTensor(value->cast()); + } else if (value->isa()) { + tensor_ptr = CreateTupleTensor(value->cast()); + } else { + MS_LOG(EXCEPTION) << "The value should be a scalar or value tuple"; + } + if (tensor_ptr == nullptr) { + MS_LOG(WARNING) << "Create tensor failed"; + return nullptr; + } + auto tensor_input = std::make_shared(tensor_ptr); + MS_EXCEPTION_IF_NULL(tensor_input); + tensor_input->set_abstract(tensor_ptr->ToAbstract()); + if (kernel_graph != nullptr) { + tensor_input = kernel_graph->NewValueNode(tensor_input); + kernel_graph->AddValueNodeToGraph(tensor_input); + } + tensor_input->set_scope(input_node->scope()); + return tensor_input; +} + +AnfNodePtr ConstInputToTensorInput(const FuncGraphPtr &func_graph, const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(cnode); + std::vector new_inputs; + auto kernel_graph = func_graph->cast>(); + auto inputs = cnode->inputs(); + new_inputs.push_back(inputs[0]); + bool need_update = false; + // the first input is primitive node which is not the real input + for (size_t i = 0; i < inputs.size() - 1; ++i) { + auto input_node = inputs[i + 1]; + if (IsValueNode(input_node) || IsValueNode(input_node)) { + auto tensor_input = CreateTensorInput(kernel_graph, input_node); + if (tensor_input == nullptr) { + new_inputs.push_back(input_node); + continue; + } + new_inputs.push_back(tensor_input); + need_update = true; + } else { + new_inputs.push_back(input_node); + } + } + if (need_update) { + MS_EXCEPTION_IF_NULL(func_graph); + auto new_cnode = func_graph->NewCNode(new_inputs); + MS_EXCEPTION_IF_NULL(new_cnode); + new_cnode->set_abstract(cnode->abstract()); + new_cnode->set_scope(cnode->scope()); + AnfAlgo::CopyNodeAttrs(cnode, new_cnode); + return new_cnode; + } + return nullptr; +} +} // namespace + +const AnfNodePtr ConvertConstInputToTensorInput::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || func_graph == nullptr || !AnfAlgo::IsRealCNodeKernel(node)) { + return nullptr; + } + CNodePtr cnode = node->cast(); + return ConstInputToTensorInput(func_graph, cnode); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_tensor_input.h b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_tensor_input.h new file mode 100644 index 00000000000..1cc2bdf0ecf --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/convert_const_input_to_tensor_input.h @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_CONST_INPUT_TO_TENSOR_INPUT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_CONST_INPUT_TO_TENSOR_INPUT_H_ +#include + +#include "ir/anf.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class ConvertConstInputToTensorInput : public PatternProcessPass { + public: + explicit ConvertConstInputToTensorInput(bool multigraph = true) + : PatternProcessPass("convert_const_input_to_tensor_input", multigraph) {} + ~ConvertConstInputToTensorInput() override = default; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_CONST_INPUT_TO_TENSOR_INPUT_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/convert_tuple_input_to_dynamic_input.cc b/mindspore/ccsrc/pre_activate/pass/convert_tuple_input_to_dynamic_input.cc new file mode 100644 index 00000000000..92579111f69 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/convert_tuple_input_to_dynamic_input.cc @@ -0,0 +1,107 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/convert_tuple_input_to_dynamic_input.h" + +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "session/kernel_graph.h" + +namespace mindspore { +namespace opt { +namespace { +void ConvertTupleOuputToPlantInputs(const FuncGraphPtr &graph, const AnfNodePtr &input_node, + std::vector *plant_inputs, std::vector *dyn_input_sizes) { + MS_EXCEPTION_IF_NULL(plant_inputs); + MS_EXCEPTION_IF_NULL(dyn_input_sizes); + MS_EXCEPTION_IF_NULL(graph); + auto output_size = AnfAlgo::GetOutputTensorNum(input_node); + dyn_input_sizes->push_back(output_size); + std::vector convert_inputs; + auto kernel_graph = graph->cast(); + MS_EXCEPTION_IF_NULL(kernel_graph); + if (input_node->isa()) { + auto value_node = input_node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + convert_inputs = kernel_graph->SplitTupleValueNodeToNodeList(value_node); + } else { + for (size_t index = 0; index < output_size; ++index) { + auto idx = NewValueNode(SizeToInt(index)); + MS_EXCEPTION_IF_NULL(idx); + auto imm = std::make_shared(SizeToInt(index)); + auto abstract_scalar = std::make_shared(imm); + idx->set_abstract(abstract_scalar); + auto tuple_get_item = + graph->NewCNode(std::vector{NewValueNode(prim::kPrimTupleGetItem), input_node, idx}); + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(input_node, index)}, + {AnfAlgo::GetOutputInferShape(input_node, index)}, tuple_get_item.get()); + convert_inputs.emplace_back(tuple_get_item); + } + } + (void)std::copy(convert_inputs.begin(), convert_inputs.end(), std::back_inserter(*plant_inputs)); +} + +CNodePtr ConvertMakeTupleInputToPlantInputs(const FuncGraphPtr &graph, const CNodePtr &cnode_ptr) { + MS_EXCEPTION_IF_NULL(cnode_ptr); + MS_EXCEPTION_IF_NULL(graph); + auto &ori_args = cnode_ptr->inputs(); + if (ori_args.size() < 1) { + return nullptr; + } + std::vector plant_inputs; + std::vector dyn_input_sizes; + plant_inputs.push_back(ori_args[kAnfPrimitiveIndex]); + for (size_t i = 1; i < ori_args.size(); ++i) { + auto input_node = ori_args[i]; + if (IsPrimitiveCNode(input_node, prim::kPrimMakeTuple)) { + auto input_size = AnfAlgo::GetOutputTensorNum(input_node); + dyn_input_sizes.push_back(input_size); + auto cnode = input_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto inputs = cnode->inputs(); + (void)std::copy(inputs.begin() + 1, inputs.end(), std::back_inserter(plant_inputs)); + } else if (AnfAlgo::IsTupleOutput(input_node)) { + ConvertTupleOuputToPlantInputs(graph, input_node, &plant_inputs, &dyn_input_sizes); + } else { + dyn_input_sizes.push_back(-1); + plant_inputs.push_back(input_node); + } + } + // If there is dynamic input, set the dyn_input_sizes as an attribute and update the inputs. + if (std::any_of(dyn_input_sizes.begin(), dyn_input_sizes.end(), [](int s) { return s >= 0; })) { + AnfAlgo::SetNodeAttr(kAttrDynInputSizes, MakeValue(dyn_input_sizes), cnode_ptr); + cnode_ptr->set_inputs(plant_inputs); + } + return cnode_ptr; +} +} // namespace + +const BaseRef ConvertTupleInputToDynamicInput::DefinePattern() const { + VarPtr V = std::make_shared(); + VarPtr Xs = std::make_shared(); + return VectorRef({V, Xs}); +} + +const AnfNodePtr ConvertTupleInputToDynamicInput::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + if (node == nullptr || !node->isa() || !AnfAlgo::IsRealKernel(node)) { + return nullptr; + } + return ConvertMakeTupleInputToPlantInputs(func_graph, node->cast()); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/convert_tuple_input_to_dynamic_input.h b/mindspore/ccsrc/pre_activate/pass/convert_tuple_input_to_dynamic_input.h new file mode 100644 index 00000000000..b3d8e25d6ef --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/convert_tuple_input_to_dynamic_input.h @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_TUPLE_INPUT_TO_DYNAMIC_INPUT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_TUPLE_INPUT_TO_DYNAMIC_INPUT_H_ + +#include +#include + +#include "ir/anf.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class ConvertTupleInputToDynamicInput : public PatternProcessPass { + public: + explicit ConvertTupleInputToDynamicInput(bool multigraph = true) + : PatternProcessPass("convert_tuple_input_to_dynamic_input", multigraph) {} + + ~ConvertTupleInputToDynamicInput() override = default; + + const BaseRef DefinePattern() const override; + + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_CONVERT_TUPLE_INPUT_TO_DYNAMIC_INPUT_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/eliminate_redundant_op.cc b/mindspore/ccsrc/pre_activate/pass/eliminate_redundant_op.cc new file mode 100644 index 00000000000..2fc971881d6 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/eliminate_redundant_op.cc @@ -0,0 +1,164 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/pass/eliminate_redundant_op.h" +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +using KernelWithIndex = std::pair; +namespace { +CNodePtr GetRealPrevCNode(const AnfNodePtr &node, size_t index, std::vector *pass_vector) { + MS_EXCEPTION_IF_NULL(pass_vector); + if (node == nullptr || !node->isa()) { + return nullptr; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (AnfAlgo::IsRealCNodeKernel(cnode)) { + pass_vector->push_back(make_pair(cnode, IntToSize(1))); + return cnode; + } + + auto input0 = cnode->input(0); + MS_EXCEPTION_IF_NULL(input0); + if (IsPrimitive(input0, prim::kPrimMakeTuple)) { + auto temp_node = cnode->input(index + IntToSize(1)); + MS_EXCEPTION_IF_NULL(temp_node); + pass_vector->push_back(make_pair(cnode, index + IntToSize(1))); + return GetRealPrevCNode(temp_node, 0, pass_vector); + } else if (IsPrimitive(input0, prim::kPrimTupleGetItem)) { + auto input2 = cnode->input(2); + MS_EXCEPTION_IF_NULL(input2); + auto value_node = input2->cast(); + MS_EXCEPTION_IF_NULL(value_node); + int item_idx = GetValue(value_node->value()); + pass_vector->push_back(make_pair(cnode, IntToSize(1))); + return GetRealPrevCNode(cnode->input(1), IntToSize(item_idx), pass_vector); + } else if (IsPrimitive(input0, prim::kPrimDepend) || IsPrimitive(input0, prim::kPrimControlDepend)) { + pass_vector->push_back(make_pair(cnode, IntToSize(1))); + return GetRealPrevCNode(cnode->input(1), 0, pass_vector); + } else { + return nullptr; + } +} + +bool TransOpEliminateCondition(const CNodePtr &, const CNodePtr &) { return true; } + +bool CastEliminateCondition(const CNodePtr &node1, const CNodePtr &node2) { + return HasSymmetricalKernelInfo(node1, node2); +} + +bool TransDataOpEliminateCondition(const CNodePtr &node1, const CNodePtr &node2) { + return AnfAlgo::GetInputFormat(node1, 0) == AnfAlgo::GetOutputFormat(node2, 0) && + AnfAlgo::GetOutputFormat(node1, 0) == AnfAlgo::GetInputFormat(node2, 0); +} + +const AnfNodePtr ProcessMatchedNodes(const FuncGraphPtr &func_graph, const CNodePtr &cnode, const CNodePtr &prev_cnode, + std::vector *pass_vector) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(pass_vector); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + + bool has_depend_node = false; + bool has_node_used_more_than_once = false; + auto &users = manager->node_users(); + + auto pass_size = pass_vector->size(); + for (size_t idx = 1; idx <= pass_size - 1; ++idx) { + auto nd = (*pass_vector)[idx].first; + if (AnfAlgo::CheckPrimitiveType(nd, prim::kPrimDepend) || + AnfAlgo::CheckPrimitiveType(nd, prim::kPrimControlDepend)) { + has_depend_node = true; + } + if (users[nd].size() >= 2) { + has_node_used_more_than_once = true; + } + } + + // when no depend node and no node used more than once, no need to rebuild the pass nodes + if (!has_depend_node) { + return prev_cnode->input(1); + } else if (!has_node_used_more_than_once) { + (void)manager->Replace(prev_cnode, prev_cnode->input(1)); + return cnode->input(1); + } else { // rebuild the pass nodes + for (size_t idx = pass_size - 2; idx > 0; --idx) { + auto new_node = func_graph->NewCNode((*pass_vector)[idx].first->inputs()); + new_node->set_input((*pass_vector)[idx].second, + (*pass_vector)[idx + 1].first->input((*pass_vector)[idx + 1].second)); + (*pass_vector)[idx].first = new_node; + } + return (*pass_vector)[1].first; + } +} +} // namespace + +void EliminateRedundantOp::Init() { + (void)redundant_process_map_.emplace(std::pair( + kFour2FiveOpName, std::pair(kFive2FourOpName, TransOpEliminateCondition))); + (void)redundant_process_map_.emplace(std::pair( + kFive2FourOpName, std::pair(kFour2FiveOpName, TransOpEliminateCondition))); + (void)redundant_process_map_.emplace(std::pair( + prim::kPrimCast->name(), std::pair(prim::kPrimCast->name(), CastEliminateCondition))); + (void)redundant_process_map_.emplace(std::pair( + kTransDataOpName, std::pair(kTransDataOpName, TransDataOpEliminateCondition))); +} + +const AnfNodePtr EliminateRedundantOp::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + if (cnode == nullptr || func_graph == nullptr) { + return nullptr; + } + // match the first name + auto name1 = AnfAlgo::GetCNodeName(cnode); + auto it = redundant_process_map_.find(name1); + if (it == redundant_process_map_.end()) { + return nullptr; + } + std::vector pass_vector; + pass_vector.push_back(make_pair(cnode, 1)); + auto prev_cnode = GetRealPrevCNode(cnode->input(1), 0, &pass_vector); + if (prev_cnode == nullptr) { + return nullptr; + } + // match the second name + auto name2 = AnfAlgo::GetCNodeName(prev_cnode); + if (name2 != it->second.first) { + return nullptr; + } + // match condition + auto condition_func = it->second.second; + if (condition_func == nullptr) { + return nullptr; + } + if (!condition_func(cnode, prev_cnode)) { + return nullptr; + } + + return ProcessMatchedNodes(func_graph, cnode, prev_cnode, &pass_vector); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/eliminate_redundant_op.h b/mindspore/ccsrc/pre_activate/pass/eliminate_redundant_op.h new file mode 100644 index 00000000000..9e0dacecb18 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/eliminate_redundant_op.h @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_ELIMINATE_REDUNDANT_OP_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_ELIMINATE_REDUNDANT_OP_H_ + +#include +#include +#include +#include +#include "ir/anf.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +using ConditionFunc = std::function; +using RedundantOpPair = std::pair; + +class EliminateRedundantOp : public PatternProcessPass { + public: + explicit EliminateRedundantOp(bool multigraph = true) : PatternProcessPass("eliminate_redundant_op", multigraph) { + Init(); + } + ~EliminateRedundantOp() override = default; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + void Init(); + std::unordered_map redundant_process_map_; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_ELIMINATE_REDUNDANT_OP_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/erase_visit_attr.cc b/mindspore/ccsrc/pre_activate/pass/erase_visit_attr.cc new file mode 100644 index 00000000000..4ea817df852 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/erase_visit_attr.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/pass/erase_visit_attr.h" +#include +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +const BaseRef EraseVisitAttr::DefinePattern() const { + std::shared_ptr V = std::make_shared(Visited); + std::shared_ptr Xs = std::make_shared(); + return VectorRef({V, Xs}); +} + +const AnfNodePtr EraseVisitAttr::Process(const FuncGraphPtr &, const AnfNodePtr &node, const EquivPtr &) const { + AnfAlgo::EraseNodeAttr(kAttrVisited, node); + return nullptr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/erase_visit_attr.h b/mindspore/ccsrc/pre_activate/pass/erase_visit_attr.h new file mode 100644 index 00000000000..a986aad83a1 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/erase_visit_attr.h @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_ERASE_VISIT_ATTR_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_ERASE_VISIT_ATTR_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class EraseVisitAttr : public PatternProcessPass { + public: + explicit EraseVisitAttr(bool multigraph = true) : PatternProcessPass("erase_visit_attr", multigraph) {} + ~EraseVisitAttr() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_ERASE_VISIT_ATTR_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/getitem_tuple.cc b/mindspore/ccsrc/pre_activate/pass/getitem_tuple.cc new file mode 100644 index 00000000000..af16017a7c0 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/getitem_tuple.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/getitem_tuple.h" + +#include +#include "operator/ops.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +bool IsC(const BaseRef &n) { + MS_EXCEPTION_IF_NULL(n); + if (utils::isa(n)) { + AnfNodePtr in = utils::cast(n); + MS_EXCEPTION_IF_NULL(in); + return in->isa(); + } else { + return false; + } +} +} // namespace + +const BaseRef GetitemTuple::DefinePattern() const { + VarPtr Xs = std::make_shared(); + VarPtr C = std::make_shared(IsC); + return VectorRef({prim::kPrimTupleGetItem, VectorRef({prim::kPrimMakeTuple, Xs}), C}); +} + +const AnfNodePtr GetitemTuple::Process(const FuncGraphPtr &, const AnfNodePtr &node, const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(node); + CNodePtr tuple_getitem = node->cast(); + MS_EXCEPTION_IF_NULL(tuple_getitem); + if (tuple_getitem->inputs().size() < kTupleGetitemInputNum) { + MS_LOG(EXCEPTION) << "tuple getitem's input num is wrong"; + } + AnfNodePtr make_tuple_anf = tuple_getitem->input(kRealInputNodeIndexInTupleGetItem); + MS_EXCEPTION_IF_NULL(make_tuple_anf); + AnfNodePtr index_node = tuple_getitem->input(kInputNodeOutputIndexInTupleGetItem); + MS_EXCEPTION_IF_NULL(index_node); + if (IsValueNode(index_node)) { + ValueNodePtr value_node = index_node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + int index = GetValue(value_node->value()); + CNodePtr make_tuple = make_tuple_anf->cast(); + MS_EXCEPTION_IF_NULL(make_tuple); + if (make_tuple->inputs().size() > IntToSize(index + 1)) { + auto ret = make_tuple->input(IntToSize(index + 1)); + MS_EXCEPTION_IF_NULL(ret); + return ret; + } + } + return nullptr; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/getitem_tuple.h b/mindspore/ccsrc/pre_activate/pass/getitem_tuple.h new file mode 100644 index 00000000000..0fc42a15dc9 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/getitem_tuple.h @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_GETITEM_TUPLE_SPLIT_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_GETITEM_TUPLE_SPLIT_H_ + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class GetitemTuple : public PatternProcessPass { + public: + explicit GetitemTuple(bool multigraph = true) : PatternProcessPass("getitem_tuple", multigraph) {} + ~GetitemTuple() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_GETITEM_TUPLE_SPLIT_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/optimize_dependence.cc b/mindspore/ccsrc/pre_activate/pass/optimize_dependence.cc new file mode 100644 index 00000000000..6ae3f3be367 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/optimize_dependence.cc @@ -0,0 +1,89 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/pass/optimize_dependence.h" +#include +#include +#include +#include "pre_activate/common/helper.h" +#include "operator/ops.h" +#include "utils/utils.h" +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" + +namespace mindspore { +namespace opt { +constexpr auto kSingleInputIndex = 1; +const BaseRef OptimizeDependence::DefinePattern() const { + VarPtr X = std::make_shared("X"); + MS_EXCEPTION_IF_NULL(X); + VarPtr Y = std::make_shared("Y"); + MS_EXCEPTION_IF_NULL(Y); + return VectorRef({prim::kPrimDepend, X, Y}); +} + +const AnfNodePtr OptimizeDependence::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const EquivPtr &) const { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return nullptr; + } + auto depend_cnode = node->cast(); + if (depend_cnode->inputs().size() < kDependInputNum) { + return nullptr; + } + auto replacing_node = depend_cnode->input(kDependInputNum - 1); + MS_EXCEPTION_IF_NULL(replacing_node); + if (!replacing_node->isa()) { + return nullptr; + } + auto replacing_cnode = replacing_node->cast(); + MS_EXCEPTION_IF_NULL(replacing_cnode); + // Currently we only optimize transdata or cast nodes. + string replacing_cnode_op_name = AnfAlgo::GetCNodeName(replacing_cnode); + if (replacing_cnode_op_name != kTransDataOpName && replacing_cnode_op_name != prim::kPrimCast->name()) { + return nullptr; + } + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + // Check whether the replacing node has only one input and one output. + if (replacing_cnode->inputs().size() != kSingleInputIndex + 1) { + return nullptr; + } + if (manager->node_users().find(replacing_node) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "The node should be used by at least another node input"; + } + if (manager->node_users()[replacing_node].size() > 1) { + return nullptr; + } + std::vector new_depend_inputs = {depend_cnode->input(kAnfPrimitiveIndex), + depend_cnode->input(kRealInputIndexInDepend), + replacing_cnode->input(kSingleInputIndex)}; + auto kernel_graph = func_graph->cast>(); + CNodePtr new_depend; + if (kernel_graph == nullptr) { + new_depend = func_graph->NewCNode(new_depend_inputs); + } else { + new_depend = kernel_graph->NewCNode(depend_cnode); + MS_EXCEPTION_IF_NULL(new_depend); + new_depend->set_inputs(new_depend_inputs); + } + new_depend->set_abstract(node->abstract()); + return new_depend; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/optimize_dependence.h b/mindspore/ccsrc/pre_activate/pass/optimize_dependence.h new file mode 100644 index 00000000000..d2995cdd302 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/optimize_dependence.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_OPTIMIZE_DEPENDENCE_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_OPTIMIZE_DEPENDENCE_H_ + +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class OptimizeDependence : public PatternProcessPass { + public: + explicit OptimizeDependence(bool multigraph = true) : PatternProcessPass("optimize_dependence", multigraph) {} + ~OptimizeDependence() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_OPTIMIZE_DEPENDENCE_H_ diff --git a/mindspore/ccsrc/pre_activate/pass/remove_nop_nodes.cc b/mindspore/ccsrc/pre_activate/pass/remove_nop_nodes.cc new file mode 100644 index 00000000000..8215fdff90c --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/remove_nop_nodes.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/pass/remove_nop_nodes.h" + +#include "common/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +const AnfNodePtr RemoveNopNodes::Process(const FuncGraphPtr &, const AnfNodePtr &node, const EquivPtr &) const { + if (node == nullptr || !node->isa()) { + return nullptr; + } + CNodePtr cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!IsNopNode(node)) { + return nullptr; + } + return cnode->input(1); +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/remove_nop_nodes.h b/mindspore/ccsrc/pre_activate/pass/remove_nop_nodes.h new file mode 100644 index 00000000000..f0b68642e9e --- /dev/null +++ b/mindspore/ccsrc/pre_activate/pass/remove_nop_nodes.h @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_REMOVE_NOP_NODES_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_REMOVE_NOP_NODES_H_ +#include "ir/anf.h" +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class RemoveNopNodes : public PatternProcessPass { + public: + explicit RemoveNopNodes(bool multigraph = true) : PatternProcessPass("remove_nop_nodes", multigraph) {} + ~RemoveNopNodes() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; +}; +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_PASS_REMOVE_NOP_NODES_H_ diff --git a/mindspore/ccsrc/predict/CMakeLists.txt b/mindspore/ccsrc/predict/CMakeLists.txt new file mode 100644 index 00000000000..d88cf5cd834 --- /dev/null +++ b/mindspore/ccsrc/predict/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE _PRE_ACTIVE_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "ascend/*.cc" + "common/*.cc" + "pass/*.cc" + "gpu/*.cc" + ) + +add_library(_mindspore_pre_active_obj OBJECT ${_PRE_ACTIVE_ALL_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/predict/converter/attr_utils/convert_util.cc b/mindspore/ccsrc/predict/converter/attr_utils/convert_util.cc new file mode 100644 index 00000000000..ff2e7bab0e1 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/attr_utils/convert_util.cc @@ -0,0 +1,229 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/attr_utils/convert_util.h" + +namespace mindspore { +namespace predict { +namespace utils { +TypePtr GetTypePtr(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + TypePtr type_ptr = anf_node->Type(); + MS_EXCEPTION_IF_NULL(type_ptr); + if (type_ptr->isa()) { + auto tensor_ptr = type_ptr->cast(); + MS_EXCEPTION_IF_NULL(tensor_ptr); + TypePtr elem = tensor_ptr->element(); + return elem; + } else if (type_ptr->isa()) { + auto tuple_ptr = type_ptr->cast(); + MS_EXCEPTION_IF_NULL(tuple_ptr); + auto tuple_i = (*tuple_ptr)[0]; + MS_EXCEPTION_IF_NULL(tuple_i); + if (tuple_i->isa()) { + auto tensor_ptr = tuple_i->cast(); + MS_EXCEPTION_IF_NULL(tensor_ptr); + TypePtr elem = tensor_ptr->element(); + MS_EXCEPTION_IF_NULL(elem); + return elem; + } else if (tuple_i->isa()) { + return type_ptr; + } else { + MS_LOG(EXCEPTION) << "unsupported type: " << type_ptr->ToString(); + } + } else if (type_ptr->isa()) { + return type_ptr; + } + std::string type_name = type_ptr->ToString(); + MS_LOG(EXCEPTION) + << "The output type of node should be a tensor type a number or a tuple of tensor type, but this is: " + << type_name; +} + +MsDataType GetMSDataType(TypeId ori_data_type) { + MsDataType dst_data_type; + switch (ori_data_type) { + case kNumberTypeFloat16: + dst_data_type = mindspore::predict::DataType_DT_FLOAT16; + return dst_data_type; + case kNumberTypeFloat32: + dst_data_type = mindspore::predict::DataType_DT_FLOAT; + return dst_data_type; + case kNumberTypeInt8: + dst_data_type = mindspore::predict::DataType_DT_INT8; + return dst_data_type; + case kNumberTypeInt32: + dst_data_type = mindspore::predict::DataType_DT_INT32; + return dst_data_type; + case kNumberTypeUInt8: + dst_data_type = mindspore::predict::DataType_DT_UINT8; + return dst_data_type; + case kNumberTypeUInt32: + dst_data_type = mindspore::predict::DataType_DT_UINT32; + return dst_data_type; + case kTypeUnknown: + dst_data_type = mindspore::predict::DataType_DT_UNDEFINED; + return dst_data_type; + default: + MS_LOG(EXCEPTION) << "Ms don't support this DataType"; + } +} + +MsFormat GetMsFormat(const std::string &format_str) { + if (format_str == kOpFormat_DEFAULT) { + MsFormat ms_format = predict::Format_NCHW; + return ms_format; + } else { + // all middle format default to NCHW + return predict::Format_NCHW; + } +} + +TensorPtr GetParaAscendTensor(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + if (!anf_node->isa()) { + return nullptr; + } + auto device_type_id = AnfAlgo::GetOutputDeviceDataType(anf_node, 0); + // device type_ptr + auto device_type_ptr = GetTypePtr(anf_node); + // device shape + auto shape = AnfAlgo::GetOutputDeviceShape(anf_node, 0); + std::vector tensor_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(tensor_shape), SizeToInt); + // device format + auto format = AnfAlgo::GetOutputFormat(anf_node, 0); + // device tensor + TensorPtr device_tensor = std::make_shared(device_type_id, tensor_shape); + // device info + device_tensor->SetDeviceInfo(format, device_type_ptr); + return device_tensor; +} + +TensorPtr GetParaCpuTensor(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + if (!(anf_node->isa())) { + return nullptr; + } else { + auto ori_type_id = AnfAlgo::GetOutputInferDataType(anf_node, 0); + auto ori_type_ptr = GetTypePtr(anf_node); + auto ori_shape = AnfAlgo::GetOutputInferShape(anf_node, 0); + std::vector tensor_shape; + (void)std::transform(ori_shape.begin(), ori_shape.end(), std::back_inserter(tensor_shape), SizeToInt); + auto ori_format = AnfAlgo::GetOutputFormat(anf_node, 0); + TensorPtr cpu_tensor = std::make_shared(ori_type_id, tensor_shape); + cpu_tensor->SetDeviceInfo(ori_format, ori_type_ptr); + return cpu_tensor; + } +} + +TensorPtr GetValueTensor(const ValueNodePtr &const_node) { + MS_EXCEPTION_IF_NULL(const_node); + auto value_ptr = const_node->value(); + MS_EXCEPTION_IF_NULL(value_ptr); + if (!value_ptr->isa()) { + return nullptr; + } + TensorPtr tensor = value_ptr->cast(); + MS_EXCEPTION_IF_NULL(tensor); + auto data_type = tensor->Dtype(); + MS_EXCEPTION_IF_NULL(data_type); + auto type_id = data_type->type_id(); + auto shape = tensor->shape(); + TensorPtr tensor_constant = std::make_shared(type_id, shape); + tensor_constant->SetDeviceInfo(tensor->device_info().format_, tensor->device_info().data_type_); + return tensor_constant; +} + +TensorPtr GetKernelCpuTensor(const CNodePtr &c_node_ptr, size_t inx) { + if (c_node_ptr == nullptr || inx >= AnfAlgo::GetOutputTensorNum(c_node_ptr)) { + MS_LOG(ERROR) << "GetKernelCpuTensor failed"; + return nullptr; + } + auto ori_shape = AnfAlgo::GetOutputInferShape(c_node_ptr, inx); + auto ori_type_id = AnfAlgo::GetOutputInferDataType(c_node_ptr, inx); + std::vector tensor_shape; + (void)std::transform(ori_shape.begin(), ori_shape.end(), std::back_inserter(tensor_shape), SizeToInt); + auto ori_output_type = GetTypePtr(c_node_ptr); + TensorPtr device_tensor = std::make_shared(ori_type_id, tensor_shape); + auto format = AnfAlgo::GetOutputFormat(c_node_ptr, inx); + device_tensor->SetDeviceInfo(format, ori_output_type); + return device_tensor; +} + +TensorPtr GetKernelAscendTensor(const CNodePtr &c_node_ptr, size_t inx) { + if (c_node_ptr == nullptr || inx >= AnfAlgo::GetOutputTensorNum(c_node_ptr)) { + MS_LOG(ERROR) << "GetKernelAscendTensor failed"; + return nullptr; + } + auto shape = AnfAlgo::GetOutputDeviceShape(c_node_ptr, inx); + std::vector tensor_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(tensor_shape), SizeToInt); + auto format = AnfAlgo::GetOutputFormat(c_node_ptr, inx); + auto type_id = AnfAlgo::GetOutputDeviceDataType(c_node_ptr, inx); + auto output_type_ptr = GetTypePtr(c_node_ptr); + TensorPtr device_tensor = std::make_shared(type_id, tensor_shape); + device_tensor->SetDeviceInfo(format, output_type_ptr); + return device_tensor; +} + +TensorPtr GetOutputTensor(const AnfNodePtr &out_node, size_t inx) { + MS_EXCEPTION_IF_NULL(out_node); + auto shape = AnfAlgo::GetOutputInferShape(out_node, inx); + std::vector tensor_shape; + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(tensor_shape), SizeToInt); + auto type_id = AnfAlgo::GetOutputInferDataType(out_node, inx); + auto output_type_ptr = GetTypePtr(out_node); + auto format = AnfAlgo::GetOutputFormat(out_node, inx); + TensorPtr output_tensor = std::make_shared(type_id, tensor_shape); + output_tensor->SetDeviceInfo(format, output_type_ptr); + return output_tensor; +} + +bool FindNodeInMap(const std::unordered_map &node_map, const AnfNodePtr &node) { + return std::any_of(node_map.begin(), node_map.end(), + [node](const std::pair &kernel_key) { return kernel_key.first == node.get(); }); +} + +bool SaveDeviceModelUtil(const std::shared_ptr &new_ms_graph_ptr, const std::string &save_path_name, + SubGraphDefT *sub_graph) { + MS_EXCEPTION_IF_NULL(new_ms_graph_ptr); + MS_EXCEPTION_IF_NULL(sub_graph); + // save mindspore schema to file + new_ms_graph_ptr->name = "default_graph"; + std::unique_ptr sub_graph_ptr(sub_graph); + new_ms_graph_ptr->subgraphs.emplace_back(std::move(sub_graph_ptr)); + // get flatbuffer builder + flatbuffers::FlatBufferBuilder builder(1024); + auto offset = mindspore::predict::GraphDef::Pack(builder, new_ms_graph_ptr.get()); + builder.Finish(offset); + auto size = builder.GetSize(); + if (size == 0) { + MS_LOG(ERROR) << "builder has no size"; + return false; + } + auto content = builder.GetBufferPointer(); + std::ofstream output(save_path_name); + if (!output.is_open()) { + MS_LOG(EXCEPTION) << "mindspore.mindspoire output failed"; + } + (void)output.write((const char *)content, size); + output.close(); + return true; +} +} // namespace utils +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/attr_utils/convert_util.h b/mindspore/ccsrc/predict/converter/attr_utils/convert_util.h new file mode 100644 index 00000000000..36164f1916d --- /dev/null +++ b/mindspore/ccsrc/predict/converter/attr_utils/convert_util.h @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_ATTR_UTILS_CONVERT_UTIL_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_ATTR_UTILS_CONVERT_UTIL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "ir/meta_tensor.h" +#include "session/anf_runtime_algorithm.h" +#include "predict/schema/inner/ms_generated.h" + +using TensorPtr = mindspore::tensor::TensorPtr; +using TensorPtrList = std::vector; +using AllOutputTensors = std::unordered_map; +using OpDefT = mindspore::predict::OpDefT; +using GraphDefT = mindspore::predict::GraphDefT; +using TensorDefT = mindspore::predict::TensorDefT; +using SubGraphDefT = mindspore::predict::SubGraphDefT; +using SubGraphPtr = std::unique_ptr; +using NodeDef = mindspore::predict::NodeDefT; +using MsDataType = mindspore::predict::DataType; +using MsFormat = mindspore::predict::Format; +using MsKernelKey = void *; +namespace mindspore { +namespace predict { +namespace utils { +TypePtr GetTypePtr(const AnfNodePtr &anf_node); +MsDataType GetMSDataType(TypeId ori_data_type); +MsFormat GetMsFormat(const std::string &format_str); +TensorPtr GetParaAscendTensor(const AnfNodePtr &anf_node); +TensorPtr GetParaCpuTensor(const AnfNodePtr &anf_node); +TensorPtr GetValueTensor(const ValueNodePtr &const_node); +TensorPtr GetKernelCpuTensor(const CNodePtr &c_node_ptr, size_t inx); +TensorPtr GetKernelAscendTensor(const CNodePtr &c_node_ptr, size_t inx); +TensorPtr GetOutputTensor(const AnfNodePtr &out_node, size_t inx); +bool FindNodeInMap(const std::unordered_map &Nodemap, const AnfNodePtr &node); +bool SaveDeviceModelUtil(const std::shared_ptr &new_ms_graph_ptr, const std::string &save_path_name, + SubGraphDefT *sub_graph_def_t); +} // namespace utils +} // namespace predict +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_ATTR_UTILS_CONVERT_UTIL_H_ diff --git a/mindspore/ccsrc/predict/converter/attr_utils/op_attr_type.h b/mindspore/ccsrc/predict/converter/attr_utils/op_attr_type.h new file mode 100644 index 00000000000..49504cd3c87 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/attr_utils/op_attr_type.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_CPU_ATTR_UTILS_OP_ATTR_TYPE_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_CPU_ATTR_UTILS_OP_ATTR_TYPE_H_ +namespace mindspore { +namespace predict { +namespace convert { +typedef enum CpuOpType { + CPU_OP_PAD = 0, + CPU_OP_MAXIMUM, + CPU_OP_CONCAT, + CPU_OP_SOFTMAX, + CPU_OP_ACTIVATION, + CPU_OP_CONV2D, + CPU_OP_FUSEDBATCHNORM, + CPU_OP_CAFFEBATCHNORM, + CPU_OP_SQUEEZE, + CPU_OP_BIASADD, + CPU_OP_POOLING, + CPU_OP_DEPTHWISECONV2D, + CPU_OP_DEDEPTHWISECONV2D, + CPU_OP_RESIZE, + CPU_OP_DETECTIONPOSTPROCESS, + CPU_OP_FULLCONNECTION, + CPU_OP_MEAN, + CPU_OP_DECONV2D, + CPU_OP_SCALE, + CPU_OP_ELTWISE, + CPU_OP_ADD, + CPU_OP_SLICE, + CPU_OP_MUL, + CPU_OP_EXP, + CPU_OP_RESHAPE, + CPU_OP_POWER, + CPU_OP_ARGMAX, + CPU_OP_ARGMAX_NETOUTPUT, + CPU_OP_MATMUL, + CPU_OP_CAFFEPRELU, + CPU_OP_STRIDEDSLICE, + CPU_OP_STACK, + CPU_OP_RANGE, + CPU_OP_EXPANDDIMS, + CPU_OP_TILE, + CPU_OP_CAST, + CPU_OP_CAFFECROP, + CPU_OP_PRESERVEED = 37 +} CpuOpType_t; +} // namespace convert +} // namespace predict +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_CPU_ATTR_UTILS_OP_ATTR_TYPE_H_ diff --git a/mindspore/ccsrc/predict/converter/executor_tensor.cc b/mindspore/ccsrc/predict/converter/executor_tensor.cc new file mode 100644 index 00000000000..b51496a9b4f --- /dev/null +++ b/mindspore/ccsrc/predict/converter/executor_tensor.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/executor_tensor.h" + +namespace mindspore { +namespace executor { +int TensorCache::addExTensor(int tensor_key, const TensorPtr &tensor, int refCount, const std::vector &host_shape, + ExTensorType stable, bool inc) { + MS_EXCEPTION_IF_NULL(tensor); + TensorPtr tmp_tensor = tensor; + ExTensorPtr ex_tensor_ptr = + std::make_shared(tensor_key, tmp_tensor, refCount, nodeIndex, host_shape, stable); + int pre_index = ex_tensor_ptr->index_; + if (inc) { + nodeIndex++; + } + // no need to judge,just add to map directly + tensors[tensor_key].push_back(ex_tensor_ptr); + return pre_index; +} + +std::vector TensorCache::findTensor(int key) { + std::vector ex_tensors; + auto iter = tensors.find(key); + if (iter != tensors.end()) { + return iter->second; + } else { + MS_LOG(INFO) << "can not find any tensorlist"; + return ex_tensors; + } +} + +void TensorCache::deleteTensor(int key) { (void)tensors.erase(key); } +} // namespace executor +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/executor_tensor.h b/mindspore/ccsrc/predict/converter/executor_tensor.h new file mode 100644 index 00000000000..a3fd5d628f9 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/executor_tensor.h @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_EXECUTOR_TENSOR_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_EXECUTOR_TENSOR_H_ + +#include +#include +#include +#include +#include "ir/meta_tensor.h" + +namespace mindspore { +namespace executor { +using TensorPtr = tensor::TensorPtr; +static constexpr int MS_MAX_REFCOUNT = 999; +enum ExTensorType { INPUTDATA, WEIGHTS, CONSTANT, KERNEL, OUTPUT }; +class ExTensor { + public: + int key_; + TensorPtr device_tensor_ptr_; + int ref_count_; + int index_; + std::vector host_shape_; + ExTensorType stable_; + ExTensor(int key, TensorPtr tensor_ptr, int ref_count, int index, std::vector host_shape, + ExTensorType ex_tensor_type) + : key_(key), + device_tensor_ptr_(std::move(tensor_ptr)), + ref_count_(ref_count), + index_(index), + host_shape_(std::move(host_shape)), + stable_(ex_tensor_type) {} + ~ExTensor() { host_shape_.clear(); } +}; +using ExTensorPtr = std::shared_ptr; +class TensorCache { + public: + TensorCache() = default; + + ~TensorCache() { tensors.clear(); } + + int addExTensor(int tensor_key, const TensorPtr &tensor, int refCount, const std::vector &host_shape, + ExTensorType stable, bool inc = true); + // just adjust for dynamic tensor + std::vector findTensor(int key); + void deleteTensor(int key); + const std::unordered_map> &GetCachedTensor() const { return tensors; } + + private: + std::unordered_map> tensors; + int nodeIndex = 0; +}; +using TensorCachePtr = std::shared_ptr; +} // namespace executor +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_EXECUTOR_TENSOR_H_ diff --git a/mindspore/ccsrc/predict/converter/kernel2ms.cc b/mindspore/ccsrc/predict/converter/kernel2ms.cc new file mode 100644 index 00000000000..30b1960e419 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/kernel2ms.cc @@ -0,0 +1,564 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/kernel2ms.h" +#include +#include "transform/convert.h" +#include "predict/converter/lite_model/op_attr_packer.h" +#include "mindspore/ccsrc/operator/ops.h" + +namespace mindspore { +namespace executor { +Kernel2Ms &Kernel2Ms::GetInstance() { + static Kernel2Ms instance; + return instance; +} + +bool Kernel2Ms::SetMemResue() const { + MS_LOG(INFO) << "MemResue start"; + return true; +} + +bool Kernel2Ms::SetAllTensors(const TensorCachePtr &tensor_cache, SubGraphDefT *ms_graph) { + if (tensor_cache == nullptr || ms_graph == nullptr) { + return false; + } + const std::unordered_map> &cachedTensors = tensor_cache->GetCachedTensor(); + size_t total_size = 0; + if (cachedTensors.empty()) { + return false; + } + for (auto &iter : cachedTensors) { + auto ex_tensors = iter.second; + total_size += ex_tensors.size(); + } + ms_graph->allTensors.resize(total_size); + for (auto &iter : cachedTensors) { + for (auto &ex_tensor : iter.second) { + std::unique_ptr ms_tensor(new TensorDefT()); + auto device_tensor_tmp = ex_tensor->device_tensor_ptr_; + auto device_d_type = device_tensor_tmp->data_type(); + ms_tensor->dataType = predict::utils::GetMSDataType(device_d_type); + auto device_shape = device_tensor_tmp->shape(); + ms_tensor->dims.clear(); + if (device_shape.empty()) { + ms_tensor->dims.push_back(1); + } else { + ms_tensor->dims.assign(device_shape.begin(), device_shape.end()); + } + std::string format_str = device_tensor_tmp->device_info().format_; + ms_tensor->format = predict::utils::GetMsFormat(format_str); + ms_tensor->offset = 0; + auto stable = ex_tensor->stable_; + if (stable == INPUTDATA || stable == CONSTANT || stable == WEIGHTS) { + ms_tensor->refCount = MS_MAX_REFCOUNT; + } else { + ms_tensor->refCount = 0; + } + ms_graph->allTensors[IntToSize(ex_tensor->index_)] = std::move(ms_tensor); + } + } + return true; +} + +bool Kernel2Ms::SetGraphOutputIdx(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache, + SubGraphDefT *ms_graph, AllOutputTensors *all_output_tensors) { + MS_EXCEPTION_IF_NULL(tensor_cache); + MS_EXCEPTION_IF_NULL(ms_graph); + MS_EXCEPTION_IF_NULL(all_output_tensors); + auto out_nodes = kernel_graph_ptr->outputs(); + if (out_nodes.empty()) { + return false; + } + // maybe need to judge out_nodes is real && output must be CNode + for (size_t i = 0; i < out_nodes.size(); ++i) { + std::vector real_inputs_link; + std::vector real_output_idx_link; + GetRealInpoutsPtr(out_nodes[i], &real_inputs_link, &real_output_idx_link); + if (real_inputs_link.empty()) { + MS_LOG(INFO) << "this graph output node is vitural node, has no real input"; + continue; + } + for (size_t k = 0; k < real_inputs_link.size(); ++k) { + int key = node_indexs_[out_nodes[i].get()]; + auto ex_tensor_list = tensor_cache->findTensor(key); + if (ex_tensor_list.empty()) { + MS_LOG(INFO) << "SetGraphOutputIdx do not add Extensor "; + continue; + } + auto ex_tensor = ex_tensor_list[real_output_idx_link[k]]; + ex_tensor_list.clear(); + ms_graph->outputIndex.push_back(ex_tensor->index_); + } + } + return true; +} + +bool Kernel2Ms::SetOpOutputIdx(const CNodePtr &c_node_ptr, const TensorPtr &output_tensor, + const TensorCachePtr &tensor_cache, int ref_count, size_t order_index, + NodeDef *ms_node) { + MS_EXCEPTION_IF_NULL(c_node_ptr); + MS_EXCEPTION_IF_NULL(output_tensor); + MS_EXCEPTION_IF_NULL(ms_node); + MS_EXCEPTION_IF_NULL(tensor_cache); + if (!predict::utils::FindNodeInMap(node_indexs_, c_node_ptr)) { + MS_LOG(ERROR) << "can not find any pk_key in inited node_indexs map"; + return false; + } + int tensor_key = node_indexs_[c_node_ptr.get()]; + auto host_shape = AnfAlgo::GetOutputInferShape(c_node_ptr, order_index); + std::vector tensor_shape; + (void)std::transform(host_shape.begin(), host_shape.end(), std::back_inserter(tensor_shape), SizeToInt); + int outputIndex = tensor_cache->addExTensor(tensor_key, output_tensor, ref_count, tensor_shape, KERNEL); + ms_node->opDef->outputIndex.push_back(outputIndex); + return true; +} + +void Kernel2Ms::GetRealInpoutsPtr(const AnfNodePtr &node, std::vector *real_inputs, + std::vector *real_output_idx) { + MS_EXCEPTION_IF_NULL(real_inputs); + MS_EXCEPTION_IF_NULL(real_output_idx); + size_t default_idx = 0; + if (node->isa()) { + auto c_node = node->cast(); + MS_EXCEPTION_IF_NULL(c_node); + std::string c_node_name = transform::GetCNodeFuncName(c_node); + if (c_node_name == prim::kPrimTupleGetItem->name()) { + auto v_node = c_node->inputs()[kTupleGetItemIndex]->cast(); + MS_EXCEPTION_IF_NULL(v_node); + default_idx = IntToSize(GetValue(v_node->value())); + real_inputs->push_back(c_node->inputs()[1]); + real_output_idx->push_back(default_idx); + return; + } else if (c_node_name == prim::kPrimDepend->name()) { + GetRealInpoutsPtr(c_node->inputs()[1], real_inputs, real_output_idx); + return; + } else if (c_node_name == prim::kPrimMakeTuple->name()) { + for (auto &in : c_node->inputs()) { + GetRealInpoutsPtr(in, real_inputs, real_output_idx); + } + return; + } else { + real_inputs->push_back(node); + real_output_idx->push_back(default_idx); + } + } else if (node->isa()) { + real_inputs->push_back(node); + real_output_idx->push_back(default_idx); + } else if (node->isa()) { + real_inputs->push_back(node); + real_output_idx->push_back(default_idx); + } +} + +bool Kernel2Ms::SetOpInputIdx(const CNodePtr &c_node_ptr, const TensorCachePtr &tensor_cache, NodeDef *ms_node) { + MS_EXCEPTION_IF_NULL(c_node_ptr); + MS_EXCEPTION_IF_NULL(tensor_cache); + MS_EXCEPTION_IF_NULL(ms_node); + for (size_t i = 1; i < c_node_ptr->inputs().size(); ++i) { + std::vector real_inputs; + std::vector real_output_idx; + GetRealInpoutsPtr(c_node_ptr->inputs()[i], &real_inputs, &real_output_idx); + if (real_inputs.empty()) { + MS_LOG(INFO) << "kernel has no inputs: " << c_node_ptr.get() << " input size[%lu]" << c_node_ptr->inputs().size(); + continue; + } + for (size_t j = 0; j < real_inputs.size(); ++j) { + int key = node_indexs_[real_inputs[j].get()]; + std::vector ex_tensor_list = tensor_cache->findTensor(key); + if (ex_tensor_list.empty()) { + continue; + } + ExTensorPtr ex_tensor_ptr = ex_tensor_list[real_output_idx[j]]; + ex_tensor_list.clear(); + ms_node->opDef->inputIndex.push_back(ex_tensor_ptr->index_); + } + } + return true; +} + +void Kernel2Ms::TransformGraphIndx() { + // transform index && anfnodeptr + if (node_indexs_.empty()) { + MS_LOG(EXCEPTION) << "node_indexs_ not ininted"; + } + for (auto &item : node_indexs_) { + index_nodes_[item.second] = item.first; + } +} + +bool Kernel2Ms::InitGraphInputsIndx(const KernelGraphPtr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + auto input_nodes = kernel_graph_ptr->inputs(); + if (input_nodes.empty()) { + return false; + } + for (const auto &input_node : input_nodes) { + if (input_node->isa()) { + if (!predict::utils::FindNodeInMap(node_indexs_, input_node)) { + // init every parameter node + node_indexs_[input_node.get()] = graph_index_; + graph_index_++; + } + } else { + MS_LOG(INFO) << "This node is anfnode, no need to handle, continue. node info: " << input_node->ToString(); + continue; + } + } + MS_LOG(DEBUG) << "inputs GraphIndex: " << graph_index_; + return true; +} + +bool Kernel2Ms::InitGraphValueNodesIndx(const KernelGraphPtr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + if (kernel_graph_ptr->value_nodes().empty()) { + return false; + } + for (auto &item : kernel_graph_ptr->value_nodes()) { + if (item.first->isa()) { + auto value_node = item.first->cast(); + MS_EXCEPTION_IF_NULL(value_node); + if (value_node == nullptr) { + MS_LOG(WARNING) << "value_node is nullptr"; + return false; + } + if (value_node->value() == nullptr) { + MS_LOG(ERROR) << "Constant value is null."; + return false; + } + if (!value_node->value()->isa()) { + continue; + } + if (!predict::utils::FindNodeInMap(node_indexs_, item.first)) { + // init node + auto node_ptr = item.first; + node_indexs_[node_ptr.get()] = graph_index_; + graph_index_++; + } + } + } + return true; +} + +bool Kernel2Ms::InitGraphOpsIndx(const KernelGraphPtr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + auto kernels = kernel_graph_ptr->execution_order(); + if (kernels.empty()) { + MS_LOG(WARNING) << "this graph has no kernel"; + return false; + } + for (size_t i = 0; i < kernels.size(); ++i) { + // for each kernel's inputs foreach real_input + if (kernels[i]->isa()) { + if (!predict::utils::FindNodeInMap(node_indexs_, kernels[i])) { + // init node + node_indexs_[kernels[i].get()] = graph_index_; + graph_index_++; + } + } + } + return true; +} + +bool Kernel2Ms::InitGraphOutputsIndx(const KernelGraphPtr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + // graph output && their inputs should link together + auto out_nodes = kernel_graph_ptr->outputs(); + if (out_nodes.empty()) { + MS_LOG(ERROR) << "this graph has no outputs"; + return false; + } + for (auto &item : out_nodes) { + if (!predict::utils::FindNodeInMap(node_indexs_, item)) { + node_indexs_[item.get()] = graph_index_; + graph_index_++; + } + } + return true; +} + +bool Kernel2Ms::InitGraphIndx(const KernelGraphPtr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + // only parameter + if (!InitGraphInputsIndx(kernel_graph_ptr)) { + return false; + } + // init value node + if (!InitGraphValueNodesIndx(kernel_graph_ptr)) { + return false; + } + // init op + if (!InitGraphOpsIndx(kernel_graph_ptr)) { + return false; + } + // init Graphoutput attention: out_put nodes have inputs + return InitGraphOutputsIndx(kernel_graph_ptr); +} + +bool Kernel2Ms::SetGraphInputTensors(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache, + SubGraphDefT *ms_graph) { + MS_EXCEPTION_IF_NULL(tensor_cache); + MS_EXCEPTION_IF_NULL(ms_graph); + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + if (convert_mode_ == kConvertUnused) { + return false; + } + if (kernel_graph_ptr->inputs().empty()) { + return false; + } + for (const auto &input_node : kernel_graph_ptr->inputs()) { + if (input_node->isa()) { + ParameterPtr pk_node = dynamic_pointer_cast(input_node); + TensorPtr device_tensor; + if (convert_mode_ == kConvertCpuMode) { + device_tensor = predict::utils::GetParaCpuTensor(input_node); + } else { + device_tensor = predict::utils::GetParaAscendTensor(input_node); + } + if (device_tensor == nullptr) { + return false; + } + ExTensorType node_type; + if (AnfAlgo::IsParameterWeight(pk_node)) { + node_type = WEIGHTS; + } else { + node_type = INPUTDATA; + } + if (!predict::utils::FindNodeInMap(node_indexs_, input_node)) { + MS_LOG(WARNING) << "can not find any pk_key in inited node_indexs map"; + return false; + } + auto pk_key = node_indexs_[input_node.get()]; + all_output_tensors_[pk_key].push_back(device_tensor); + int nodeRefCount = SizeToInt(AnfAlgo::GetOutputTensorNum(input_node)); + int nodeInputIdx = + tensor_cache->addExTensor(pk_key, device_tensor, nodeRefCount, device_tensor->shape(), node_type); + if (!AnfAlgo::IsParameterWeight(pk_node)) { + ms_graph->inputIndex.push_back(nodeInputIdx); + all_input_idxs_.push_back(nodeInputIdx); + } else { + input_weight_idxs_.push_back(nodeInputIdx); + all_input_idxs_.push_back(nodeInputIdx); + } + } + } + return true; +} + +bool Kernel2Ms::SetGraphValueTensors(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + MS_EXCEPTION_IF_NULL(tensor_cache); + for (auto &item : kernel_graph_ptr->value_nodes()) { + if (item.first->isa()) { + auto const_node = item.first->cast(); + auto tensor_constant = predict::utils::GetValueTensor(const_node); + if (tensor_constant == nullptr) { + continue; + } + if (!predict::utils::FindNodeInMap(node_indexs_, item.first)) { + MS_LOG(WARNING) << "can not find any pk_key in inited node_indexs map"; + return false; + } + int constant_key = node_indexs_[(item.first).get()]; + all_output_tensors_[constant_key].push_back(tensor_constant); + auto shape = tensor_constant->shape(); + (void)tensor_cache->addExTensor(constant_key, tensor_constant, 0, shape, CONSTANT); + } + } + return true; +} + +bool Kernel2Ms::SetGraphOpTensors(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache, + SubGraphDefT *ms_graph) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + MS_EXCEPTION_IF_NULL(tensor_cache); + MS_EXCEPTION_IF_NULL(ms_graph); + auto kernels = kernel_graph_ptr->execution_order(); + if (kernels.empty()) { + MS_LOG(ERROR) << "this graph has no kernels"; + return false; + } + for (auto &kernel : kernels) { + if (!predict::utils::FindNodeInMap(node_indexs_, kernel)) { + MS_LOG(ERROR) << "can not find any pk_key in inited node_indexs map"; + return false; + } + auto kernel_key = node_indexs_[kernel.get()]; + std::unique_ptr ms_node(new NodeDef); + ms_node->fmkType = mindspore::predict::FmkType_CAFFE; + std::unique_ptr ms_op(new OpDefT()); + auto c_name = AnfAlgo::GetCNodeName(kernel); + auto fun = predict::convert::OpAttrFactory::GetInstance()->GetPackFun(c_name); + if (fun == nullptr) { + MS_LOG(ERROR) << "get node [" << kernel->fullname_with_scope() << "] attr failed."; + return false; + } else if (!fun(kernel, ms_op.get())) { + MS_LOG(ERROR) << "set node [" << kernel->fullname_with_scope() << "] attr failed."; + return false; + } + ms_node->opDef = std::move(ms_op); + auto output_size = AnfAlgo::GetOutputTensorNum(kernel); + int nodeRefCount = SizeToInt(output_size); + for (size_t j = 0; j < output_size; ++j) { + TensorPtr device_tensor; + if (convert_mode_ == kConvertCpuMode) { + device_tensor = predict::utils::GetKernelCpuTensor(kernel, j); + } else if (convert_mode_ == kConvertAscendMode) { + device_tensor = predict::utils::GetKernelAscendTensor(kernel, j); + } + if (device_tensor == nullptr) { + return false; + } + all_output_tensors_[kernel_key].push_back(device_tensor); + if (!SetOpOutputIdx(kernel, device_tensor, tensor_cache, nodeRefCount, j, ms_node.get())) { + return false; + } + } + tmp_op_nodes_.emplace_back(ms_node.release()); + } + return true; +} + +bool Kernel2Ms::KernelGraph2MsGraph(const KernelGraphPtr &kernel_graph_ptr) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + graph_index_ = 0; + all_output_tensors_.clear(); + node_indexs_.clear(); + index_nodes_.clear(); + std::unique_ptr sub_ms_graph(new SubGraphDefT()); + if (!InitGraphIndx(kernel_graph_ptr)) { + return false; + } + TransformGraphIndx(); + tensor_cache_ptr_ = std::make_shared(); + // foreach node to init it's real output tensor + if (!SetGraphInputTensors(kernel_graph_ptr, tensor_cache_ptr_, sub_ms_graph.get())) { + return false; + } + // Get KernelGraph value node + if (!SetGraphValueTensors(kernel_graph_ptr, tensor_cache_ptr_)) { + return false; + } + // Get KernelGraph apply_kernel && add opNode + if (!SetGraphOpTensors(kernel_graph_ptr, tensor_cache_ptr_, sub_ms_graph.get())) { + return false; + } + // Get KernelGraph outputs + if (!SetGraphOutputIdx(kernel_graph_ptr, tensor_cache_ptr_, sub_ms_graph.get(), &all_output_tensors_)) { + return false; + } + auto kernels = kernel_graph_ptr->execution_order(); + for (size_t i = 0; i < kernels.size(); ++i) { + auto ms_node = tmp_op_nodes_[i]; + if (!SetOpInputIdx(kernels[i], tensor_cache_ptr_, ms_node)) { + return false; + } + std::unique_ptr ms_node_tmp(ms_node); + sub_ms_graph->nodes.emplace_back(std::move(ms_node_tmp)); + } + if (!SetAllTensors(tensor_cache_ptr_, sub_ms_graph.get())) { + return false; + } + if (!SetMemResue()) { + return false; + } + sub_ms_graph_ = std::move(sub_ms_graph); + sub_ms_graph_->name = "default_sub_graph"; + return true; +} + +bool Kernel2Ms::CheckInputSizes(const std::vector &input_tensors, + const std::vector &all_input_idxs) { + if (input_tensors.size() != all_input_idxs.size()) { + MS_LOG(EXCEPTION) << "real input tensors size:" << input_tensors.size() + << "not equal converted tesnors size:" << all_input_idxs.size() << "the graph has changed"; + } + for (auto in : all_input_idxs) { + if (in < sub_ms_graph_->allTensors.size()) { + auto real_tensor = input_tensors[in]; + auto convert_dims = sub_ms_graph_->allTensors[in]->dims; + auto real_dims = real_tensor->shape(); + if (real_dims.size() != convert_dims.size()) { + return false; + } else { + for (size_t i = 0; i < convert_dims.size(); ++i) { + if (convert_dims[i] != real_dims[i]) { + return false; + } + } + } + } else { + MS_LOG(EXCEPTION) << "index: " << in << "in all_input_idxs is valid"; + } + } + return true; +} + +void Kernel2Ms::ReleaseContextRes() { + tmp_op_nodes_.clear(); + node_indexs_.clear(); + index_nodes_.clear(); + tensor_cache_ptr_ = nullptr; + all_output_tensors_.clear(); +} + +bool Kernel2Ms::KernelInput2MS(const std::vector &input_tensors) { + const std::unordered_map> &cache_tensors = tensor_cache_ptr_->GetCachedTensor(); + if (cache_tensors.empty()) { + return false; + } + auto all_weights_idxs = GetAllInputWeightIdxs(); + auto all_input_idxs = GetAllInputIdxs(); + auto real_input_size = input_tensors.size(); + // check tensor size + bool ret = CheckInputSizes(input_tensors, all_input_idxs); + std::vector match_to_rel_idxs; + // indx order not matched,macth to it + if (!ret) { + for (auto idx : all_weights_idxs) { + auto macth_idx = real_input_size - idx; + match_to_rel_idxs.push_back(macth_idx); + } + } else { + match_to_rel_idxs = all_weights_idxs; + } + if (match_to_rel_idxs.size() == all_weights_idxs.size()) { + for (size_t j = 0; j < all_weights_idxs.size(); ++j) { + auto cache_idx = all_weights_idxs[j]; + auto match_idx = match_to_rel_idxs[j]; + auto real_tensor = input_tensors[match_idx]; + auto real_size = LongToSize(real_tensor->data().nbytes()); + auto real_data = real_tensor->data_c(false); + MS_EXCEPTION_IF_NULL(real_data); + if (sub_ms_graph_->allTensors[cache_idx] != nullptr) { + sub_ms_graph_->allTensors[cache_idx]->data.resize(real_size); + } + if (memcpy_s(sub_ms_graph_->allTensors[cache_idx]->data.data(), real_size, real_data, real_size) != 0) { + MS_LOG(ERROR) << "KernelInput2MS memcpy_s failed"; + return false; + } + } + } + ReleaseContextRes(); + return true; +} + +bool Kernel2Ms::SaveDeviceModel(const std::shared_ptr &new_ms_graph_ptr, const std::string &save_path_name) { + MS_EXCEPTION_IF_NULL(new_ms_graph_ptr); + return predict::utils::SaveDeviceModelUtil(new_ms_graph_ptr, save_path_name, sub_ms_graph_.release()); +} +} // namespace executor +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/kernel2ms.h b/mindspore/ccsrc/predict/converter/kernel2ms.h new file mode 100644 index 00000000000..f991ecc94a3 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/kernel2ms.h @@ -0,0 +1,118 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_KERNEL_TO_MS_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_KERNEL_TO_MS_H_ + +#include +#include +#include +#include +#include +#include "session/kernel_graph.h" +#include "predict/converter/executor_tensor.h" +#include "predict/schema/inner/ms_generated.h" +#include "predict/converter/attr_utils/convert_util.h" + +static constexpr size_t kTupleGetItemIndex = 2; +namespace mindspore { +namespace executor { +using KernelGraphPtr = std::shared_ptr; +enum ConvertMode { kConvertCpuMode, kConvertAscendMode, kConvertUnused }; +enum TargetMode { kCPUTarget, kGPUTarget, kUnknowTarget }; +class Kernel2Ms { + public: + static Kernel2Ms &GetInstance(); + + Kernel2Ms(const Kernel2Ms &) = delete; + + Kernel2Ms &operator=(const Kernel2Ms &) = delete; + + bool KernelGraph2MsGraph(const KernelGraphPtr &kernel_graph_ptr); + + bool KernelInput2MS(const std::vector &input_tensors); + + ConvertMode convert_mode() const { return convert_mode_; } + + void set_convert_mode(ConvertMode convert_mode) { convert_mode_ = convert_mode; } + + TargetMode device_target() const { return device_target_; } + + void set_device_target(TargetMode device_target) { device_target_ = device_target; } + + bool SaveDeviceModel(const std::shared_ptr &new_ms_graph_ptr, const std::string &save_path_name); + + private: + Kernel2Ms() : graph_index_(0) {} + + void ReleaseContextRes(); + + ~Kernel2Ms() = default; + + bool SetAllTensors(const TensorCachePtr &tensor_cache, SubGraphDefT *sub_graph_def_t); + + bool SetOpInputIdx(const CNodePtr &c_node_ptr, const TensorCachePtr &tensor_cache, NodeDef *ms_node); + + bool SetOpOutputIdx(const CNodePtr &c_node_ptr, const TensorPtr &output_tensor, const TensorCachePtr &tensor_cache, + int ref_count, size_t order_index, NodeDef *ms_node); + + bool SetGraphOutputIdx(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache, + SubGraphDefT *sub_graph_def_t, AllOutputTensors *all_output_tensors); + + void TransformGraphIndx(); + + void GetRealInpoutsPtr(const AnfNodePtr &node, std::vector *real_inputs, + std::vector *real_output_idx); + + bool InitGraphIndx(const KernelGraphPtr &kernel_graph_ptr); + + bool InitGraphInputsIndx(const KernelGraphPtr &kernel_graph_ptr); + + bool InitGraphValueNodesIndx(const KernelGraphPtr &kernel_graph_ptr); + + bool InitGraphOpsIndx(const KernelGraphPtr &kernel_graph_ptr); + + bool InitGraphOutputsIndx(const KernelGraphPtr &kernel_graph_ptr); + + bool SetGraphInputTensors(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache, + SubGraphDefT *sub_graph_def_t); + + bool SetGraphValueTensors(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache); + + bool SetGraphOpTensors(const KernelGraphPtr &kernel_graph_ptr, const TensorCachePtr &tensor_cache, + SubGraphDefT *sub_graph_def_t); + std::vector GetAllInputWeightIdxs() const { return input_weight_idxs_; } + std::vector GetAllInputIdxs() const { return all_input_idxs_; } + + bool CheckInputSizes(const std::vector &input_tensors, const std::vector &all_input_idxs); + + bool SetMemResue() const; + SubGraphPtr sub_ms_graph_; + AllOutputTensors all_output_tensors_; + std::vector tmp_op_nodes_; + std::unordered_map node_indexs_; + std::unordered_map index_nodes_; + int graph_index_ = 0; + TensorCachePtr tensor_cache_ptr_ = nullptr; + ConvertMode convert_mode_ = kConvertCpuMode; + TargetMode device_target_ = kCPUTarget; + std::vector input_weight_idxs_; + std::vector all_input_idxs_; +}; +using Kernel2MsPtr = std::shared_ptr; +} // namespace executor +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_KERNEL_TO_MS_H_ diff --git a/mindspore/ccsrc/predict/converter/lite_model/op_attr_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/op_attr_packer.cc new file mode 100644 index 00000000000..f186758de5a --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/op_attr_packer.cc @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" +#include "./securec.h" + +namespace mindspore { +namespace predict { +namespace convert { +// forward declare +bool Conv2dPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool MatMulPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool BiasAddPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool ReshapePacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool ActivationPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool PoolingPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool FusedBatchNormPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool AddPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool CastPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool MeanPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool SoftmaxPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); +bool ScalePacker(const CNodePtr &c_node_ptr, OpDefT *ms_op); + +OpAttrFactory::OpAttrFactory() { + pack_funs_ = {{"Conv2D", Conv2dPacker}, + {"MatMul", MatMulPacker}, + {"BiasAdd", BiasAddPacker}, + {"Reshape", ReshapePacker}, + {"Activation", ActivationPacker}, + {"ReLU", ActivationPacker}, + {"ReLU6", ActivationPacker}, + {"EReLU", ActivationPacker}, + {"LeakyReLU", ActivationPacker}, + {"Sigmoid", ActivationPacker}, + {"Softsign", ActivationPacker}, + {"Softplus", ActivationPacker}, + {"Tanh", ActivationPacker}, + {"Hswish", ActivationPacker}, + {"Hsigmoid", ActivationPacker}, + {"MaxPool", PoolingPacker}, + {"MaxPool2D", PoolingPacker}, + {"MeanPool", PoolingPacker}, + {"GlobalPool", PoolingPacker}, + {"FusedBatchNorm", FusedBatchNormPacker}, + {"FusedBatchNormGrad", FusedBatchNormPacker}, + {"Cast", CastPacker}, + {"TensorAdd", AddPacker}, + {"SoftMax", SoftmaxPacker}, + {"SimpleMean", MeanPacker}, + {"Scale", ScalePacker}}; +} +OpAttrPackFun OpAttrFactory::GetPackFun(const std::string &opType) { + if (pack_funs_.find(opType) == pack_funs_.end()) { + MS_LOG(ERROR) << "Op Attr pack fun [\" << opType << \"] not found."; + return nullptr; + } + return pack_funs_[opType]; +} + +mindspore::predict::DataFormatType GetAttrFormat(const std::string &format) { + if (format == kOpFormat_NCHW) { + return predict::DataFormatType::DataFormatType_NCHW; + } else if (format == kOpFormat_NHWC) { + return predict::DataFormatType::DataFormatType_NHWC; + } else { + return predict::DataFormatType::DataFormatType_UNKNOW; + } +} + +mindspore::predict::PadMode GetAttrPadMode(const std::string &pad_mode) { + if (pad_mode == "same") { + return mindspore::predict::PadMode::PadMode_SAME; + } else if (pad_mode == "valid") { + return mindspore::predict::PadMode::PadMode_VALID; + } else { + return mindspore::predict::PadMode::PadMode_NOTSET; + } +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/op_attr_packer.h b/mindspore/ccsrc/predict/converter/lite_model/op_attr_packer.h new file mode 100644 index 00000000000..83d0f9287b9 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/op_attr_packer.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_OP_ATTR_PACKER_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_OP_ATTR_PACKER_H_ + +#include +#include +#include +#include "session/anf_runtime_algorithm.h" +#include "predict/schema/inner/ms_generated.h" + +static constexpr size_t kNIndex = 0; +static constexpr size_t kCIndex = 1; +static constexpr size_t kHIndex = 2; +static constexpr size_t kWIndex = 3; +static constexpr size_t kNCHWSize = 4; +namespace mindspore { +namespace predict { +namespace convert { +using OpAttrPackFun = bool (*)(const CNodePtr &c_node_ptr, OpDefT *ms_op); +class OpAttrFactory { + public: + static OpAttrFactory *GetInstance() { + static OpAttrFactory instance; + return &instance; + } + OpAttrFactory(const OpAttrFactory &) = delete; + OpAttrFactory &operator=(const OpAttrFactory &) = delete; + OpAttrPackFun GetPackFun(const std::string &op_type); + ~OpAttrFactory() { pack_funs_.clear(); } + OpAttrFactory(); + + private: + std::unordered_map pack_funs_; +}; + +mindspore::predict::DataFormatType GetAttrFormat(const std::string &format); + +mindspore::predict::PadMode GetAttrPadMode(const std::string &pad_mode); +} // namespace convert +} // namespace predict +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_CONVERTER_CPU_OP_INFO_OP_ATTR_FACTORY_H_ diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/activation_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/activation_packer.cc new file mode 100644 index 00000000000..3dc09f70b40 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/activation_packer.cc @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool ActivationPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new ActivationT()); + MS_EXCEPTION_IF_NULL(attr); + if (AnfAlgo::GetCNodeName(c_node_ptr) == "ReLU") { + attr->type = predict::ActivationType::ActivationType_RELU; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "Sigmoid") { + attr->type = predict::ActivationType::ActivationType_SIGMOID; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "ReLU6") { + attr->type = predict::ActivationType::ActivationType_RELU6; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "ELU") { + attr->type = predict::ActivationType::ActivationType_ELU; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "Leaky_ReLU") { + attr->type = predict::ActivationType::ActivationType_LEAKY_RELU; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "ABS") { + attr->type = predict::ActivationType::ActivationType_ABS; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "ReLU1") { + attr->type = predict::ActivationType::ActivationType_RELU1; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "Softsign") { + attr->type = predict::ActivationType::ActivationType_SOFTSIGN; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "Softplus") { + attr->type = predict::ActivationType::ActivationType_SOFTPLUS; + } else if (AnfAlgo::GetCNodeName(c_node_ptr) == "Tanh") { + attr->type = predict::ActivationType::ActivationType_TANH; + } else { + attr->type = predict::ActivationType::ActivationType_UNKNOW; + MS_LOG(WARNING) << "unknow Activation"; + } + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Activation; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/add_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/add_packer.cc new file mode 100644 index 00000000000..81a2d3a9af1 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/add_packer.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool AddPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new AddT()); + MS_EXCEPTION_IF_NULL(attr); + attr->format = predict::DataFormatType::DataFormatType_NCHW; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Add; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/biasadd_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/biasadd_packer.cc new file mode 100644 index 00000000000..6fe32c1f6b4 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/biasadd_packer.cc @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool BiasAddPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new BiasAddT()); + MS_EXCEPTION_IF_NULL(attr); + attr->axis = {1}; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_BiasAdd; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/cast_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/cast_packer.cc new file mode 100644 index 00000000000..d0f3f80f6c8 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/cast_packer.cc @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool CastPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new CastT()); + MS_EXCEPTION_IF_NULL(attr); + attr->srcT = 0; + attr->dstT = 0; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Cast; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/conv2d_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/conv2d_packer.cc new file mode 100644 index 00000000000..0fefb89c591 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/conv2d_packer.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool Conv2dPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + int kernel_group_value = AnfAlgo::GetNodeAttr(c_node_ptr, "group"); + int kernel_channel_value = AnfAlgo::GetNodeAttr(c_node_ptr, "out_channel"); + std::vector kernel_size_value = AnfAlgo::GetNodeAttr>(c_node_ptr, "kernel_size"); + std::string kernel_pad_mode_value = AnfAlgo::GetNodeAttr(c_node_ptr, "pad_mode"); + int kernel_pad_value = AnfAlgo::GetNodeAttr(c_node_ptr, "pad"); + int kernel_stride_value = AnfAlgo::GetNodeAttr(c_node_ptr, "stride"); + int kernel_dilation_value = AnfAlgo::GetNodeAttr(c_node_ptr, "dilation"); + std::string kernel_data_format_value = AnfAlgo::GetNodeAttr(c_node_ptr, "data_format"); + std::unique_ptr attr(new Conv2DT()); + MS_EXCEPTION_IF_NULL(attr); + attr->format = GetAttrFormat(kernel_data_format_value); + attr->group = kernel_group_value; + auto in_shape = AnfAlgo::GetPrevNodeOutputInferShape(c_node_ptr, 1); + if (in_shape.size() != kNCHWSize) { + return false; + } + attr->channelIn = SizeToInt(in_shape[1]); + attr->channelOut = kernel_channel_value; + attr->kernelW = kernel_size_value[0]; + attr->kernelH = kernel_size_value[1]; + attr->strideW = kernel_stride_value; + attr->strideH = kernel_stride_value; + attr->padMode = GetAttrPadMode(kernel_pad_mode_value); + attr->padUp = kernel_pad_value; + attr->padDown = kernel_pad_value; + attr->padLeft = kernel_pad_value; + attr->padRight = kernel_pad_value; + attr->dilateW = kernel_dilation_value; + attr->dilateH = kernel_dilation_value; + attr->hasBias = false; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Conv2D; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/fusedbatchnorm_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/fusedbatchnorm_packer.cc new file mode 100644 index 00000000000..e0092820c23 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/fusedbatchnorm_packer.cc @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool FusedBatchNormPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new FusedBatchNormT()); + MS_EXCEPTION_IF_NULL(attr); + auto kernel_epsilon = AnfAlgo::GetNodeAttr(c_node_ptr, "epsilon"); + attr->epsilon = kernel_epsilon; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_FusedBatchNorm; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/matmul_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/matmul_packer.cc new file mode 100644 index 00000000000..a0f82810a7f --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/matmul_packer.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool MatMulPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + bool kernel_transpore_a = AnfAlgo::GetNodeAttr(c_node_ptr, "transpose_a"); + bool kernel_transpore_b = AnfAlgo::GetNodeAttr(c_node_ptr, "transpose_b"); + std::unique_ptr attr(new MatMulT()); + MS_EXCEPTION_IF_NULL(attr); + attr->transposeA = kernel_transpore_a; + attr->transposeB = kernel_transpore_b; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_MatMul; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/mean_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/mean_packer.cc new file mode 100644 index 00000000000..eac3fa88f10 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/mean_packer.cc @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool MeanPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new MeanT()); + MS_EXCEPTION_IF_NULL(attr); + attr->axis = {1}; + attr->keepDims = false; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Mean; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/pooling_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/pooling_packer.cc new file mode 100644 index 00000000000..4eeb643817c --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/pooling_packer.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool PoolingPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new PoolingT()); + MS_EXCEPTION_IF_NULL(attr); + std::string kernel_format_value = AnfAlgo::GetNodeAttr(c_node_ptr, "data_format"); + attr->format = GetAttrFormat(kernel_format_value); + auto c_name = AnfAlgo::GetCNodeName(c_node_ptr); + if (c_name == "MaxPool") { + ms_op->name = c_node_ptr->fullname_with_scope(); + attr->poolingMode = mindspore::predict::PoolMode::PoolMode_MAX_POOLING; + } else if (c_name == "MeanPool") { + ms_op->name = c_node_ptr->fullname_with_scope(); + attr->poolingMode = mindspore::predict::PoolMode::PoolMode_MEAN_POOLING; + } else if (c_name == "GlobalPool") { + ms_op->name = c_node_ptr->fullname_with_scope(); + attr->poolingMode = mindspore::predict::PoolMode::PoolMode_GLOBAL_POOING; + } else { + MS_LOG(ERROR) << "unknowed pooling type."; + return false; + } + std::vector kernel_ksize = AnfAlgo::GetNodeAttr>(c_node_ptr, "ksize"); + attr->windowW = kernel_ksize[kHIndex]; + attr->windowH = kernel_ksize[kWIndex]; + std::vector kernel_strides = AnfAlgo::GetNodeAttr>(c_node_ptr, "strides"); + attr->strideW = kernel_strides[kHIndex]; + attr->strideH = kernel_strides[kWIndex]; + std::string kernel_pad_mode_value = AnfAlgo::GetNodeAttr(c_node_ptr, "padding"); + attr->padMode = GetAttrPadMode(kernel_pad_mode_value); + attr->padUp = 0; + attr->padDown = 0; + attr->padLeft = 0; + attr->padRight = 0; + attr->caffeMode = false; + ms_op->attr.type = OpT_Pooling; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/reshape_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/reshape_packer.cc new file mode 100644 index 00000000000..cd8b72a8ac6 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/reshape_packer.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool ReshapePacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new ReshapeT()); + MS_EXCEPTION_IF_NULL(attr); + attr->format = predict::DataFormatType::DataFormatType_NCHW; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Reshape; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/scale_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/scale_packer.cc new file mode 100644 index 00000000000..7b4f6f62831 --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/scale_packer.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool ScalePacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new ScaleT()); + MS_EXCEPTION_IF_NULL(attr); + attr->format = predict::DataFormatType::DataFormatType_NCHW; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_Scale; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/converter/lite_model/operations/softmax_packer.cc b/mindspore/ccsrc/predict/converter/lite_model/operations/softmax_packer.cc new file mode 100644 index 00000000000..fe96bae451f --- /dev/null +++ b/mindspore/ccsrc/predict/converter/lite_model/operations/softmax_packer.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/converter/lite_model/op_attr_packer.h" + +namespace mindspore { +namespace predict { +namespace convert { +bool SoftmaxPacker(const CNodePtr &c_node_ptr, OpDefT *ms_op) { + if (c_node_ptr == nullptr || ms_op == nullptr) { + return false; + } + std::unique_ptr attr(new SoftMaxT()); + MS_EXCEPTION_IF_NULL(attr); + attr->axis = {1}; + ms_op->name = c_node_ptr->fullname_with_scope(); + ms_op->attr.type = OpT_SoftMax; + ms_op->attr.value = attr.release(); + return true; +} +} // namespace convert +} // namespace predict +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/generator/ir/ir_model.cc b/mindspore/ccsrc/predict/generator/ir/ir_model.cc new file mode 100644 index 00000000000..ff465245771 --- /dev/null +++ b/mindspore/ccsrc/predict/generator/ir/ir_model.cc @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/generator/ir/ir_model.h" + +#include +#include + +#include "utils/log_adapter.h" + +namespace mindspore { +namespace generator { +IRModel::~IRModel() { ir_tasks_.clear(); } +void IRModel::SetIrTaskInfos(const std::vector &ir_tasks) { + (void)std::copy(ir_tasks.begin(), ir_tasks.end(), std::back_inserter(ir_tasks_)); +} +} // namespace generator +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/generator/ir/ir_model.h b/mindspore/ccsrc/predict/generator/ir/ir_model.h new file mode 100644 index 00000000000..bf1c057b5ff --- /dev/null +++ b/mindspore/ccsrc/predict/generator/ir/ir_model.h @@ -0,0 +1,37 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_MINDSPORE_CCSRC_EXECUTOR_GENERATOR_IR_IR_MODEL_H_ +#define MINDSPORE_MINDSPORE_CCSRC_EXECUTOR_GENERATOR_IR_IR_MODEL_H_ +#include +#include +#include +#include "predict/generator/ir/ir_task_info.h" +namespace mindspore { +namespace generator { +class IRModel { + public: + void SetIrTaskInfos(const std::vector& ir_tasks); + IRModel() = default; + ~IRModel(); + + private: + std::vector ir_tasks_; +}; +using IrModelPtr = std::shared_ptr; +} // namespace generator +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_EXECUTOR_GENERATOR_IR_IR_MODEL_H_ diff --git a/mindspore/ccsrc/predict/generator/ir/ir_task_info.cc b/mindspore/ccsrc/predict/generator/ir/ir_task_info.cc new file mode 100644 index 00000000000..1c275ea8ed4 --- /dev/null +++ b/mindspore/ccsrc/predict/generator/ir/ir_task_info.cc @@ -0,0 +1,244 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/generator/ir/ir_task_info.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace generator { +bool CceIRTaskInfo::SerializeIRToProto() { + auto cce_task_def_ptr = std::unique_ptr(); + auto kernel_context_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(cce_task_def_ptr); + MS_EXCEPTION_IF_NULL(kernel_context_ptr); + kernel_context_ptr->set_kernel_type(k_ctx_.kernel_type); + kernel_context_ptr->set_op_id(k_ctx_.op_id); + kernel_context_ptr->set_kernel_func_id(k_ctx_.kernel_func_id); + kernel_context_ptr->set_op_index(k_ctx_.op_index); + kernel_context_ptr->set_is_flowtable(k_ctx_.is_flowtable); + kernel_context_ptr->set_args_count(k_ctx_.args_count); + for (unsigned int i : k_ctx_.origin_op_index) { + kernel_context_ptr->add_origin_op_index(i); + } + void *tmp_args_offset = static_cast((k_ctx_.args_offset).data()); + if (tmp_args_offset == nullptr) { + MS_LOG(WARNING) << "tmp_args_offset have no data"; + return false; + } + kernel_context_ptr->set_args_offset(tmp_args_offset, k_ctx_.args_offset.size()); + cce_task_def_ptr->set_allocated_kernel_context(std::move(kernel_context_ptr).get()); + cce_task_def_ptr->set_stub_func(stub_func_); + cce_task_def_ptr->set_block_dim(block_dim_); + cce_task_def_ptr->set_args_size(args_size_); + void *tmp_sm_desc = static_cast(sm_desc_.data()); + if (tmp_sm_desc == nullptr) { + MS_LOG(WARNING) << "tmp_sm_desc have no data"; + return false; + } + cce_task_def_ptr->set_sm_desc(tmp_sm_desc, sm_desc_.size()); + + void *tmp_flow_table = static_cast(flow_table_.data()); + if (tmp_flow_table == nullptr) { + MS_LOG(WARNING) << "tmp_flow_table have no data"; + return false; + } + cce_task_def_ptr->set_flow_table(tmp_flow_table, flow_table_.size()); + return true; +} + +CceIRTaskInfo::~CceIRTaskInfo() { + args_.clear(); + sm_desc_.clear(); + flow_table_.clear(); +} + +bool TbeIRTaskInfo::SerializeIRToProto() { + auto tbe_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(tbe_task_def_ptr); + tbe_task_def_ptr->set_stub_func(stub_func_); + tbe_task_def_ptr->set_block_dim(block_dim_); + tbe_task_def_ptr->set_args_size(args_size_); + void *tmp_args = static_cast(args_.data()); + if (tmp_args == nullptr) { + MS_LOG(WARNING) << "tmp_args have no data"; + return false; + } + tbe_task_def_ptr->set_args(tmp_args, args_.size()); + void *tmp_sm_desc = static_cast(sm_desc_.data()); + if (tmp_sm_desc == nullptr) { + MS_LOG(WARNING) << "tmp_sm_desc have no data"; + return false; + } + tbe_task_def_ptr->set_sm_desc(tmp_sm_desc, sm_desc_.size()); + void *tmp_meta_data = static_cast(meta_data_.data()); + if (tmp_meta_data == nullptr) { + MS_LOG(WARNING) << "tmp_meta_data have no data"; + return false; + } + tbe_task_def_ptr->set_meta_data(tmp_meta_data, meta_data_.size()); + for (auto &in : input_data_addrs_) { + tbe_task_def_ptr->add_input_addrs(in); + } + for (auto &ou : output_data_addrs_) { + tbe_task_def_ptr->add_output_addrs(ou); + } + for (auto &wk : workspace_addrs_) { + tbe_task_def_ptr->add_workspace_addrs(wk); + } + return true; +} + +TbeIRTaskInfo::~TbeIRTaskInfo() { + args_.clear(); + sm_desc_.clear(); + meta_data_.clear(); + input_data_addrs_.clear(); + output_data_addrs_.clear(); + workspace_addrs_.clear(); +} + +bool AicpuIRTaskInfo::SerializeIRToProto() { + auto aicpu_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(aicpu_task_def_ptr); + aicpu_task_def_ptr->set_op_type(op_type_); + aicpu_task_def_ptr->set_flag(flag_); + for (auto &shape : input_data_shapes_) { + auto in_shape_ptr = aicpu_task_def_ptr->add_input_shapes(); + for (auto &in_sh : shape) { + in_shape_ptr->add_shape(static_cast(in_sh)); + } + } + for (auto &shape : output_data_shapes_) { + auto ou_shape_ptr = aicpu_task_def_ptr->add_output_shapes(); + for (auto &ou_sh : shape) { + ou_shape_ptr->add_shape(static_cast(ou_sh)); + } + } + for (auto &in_type : input_data_types_) { + aicpu_task_def_ptr->add_input_types(in_type); + } + for (auto &ou_type : output_data_types_) { + aicpu_task_def_ptr->add_output_types(ou_type); + } + for (auto &in_addr : input_data_addrs_) { + aicpu_task_def_ptr->add_input_addrs(in_addr); + } + for (auto &ou_addr : output_data_addrs_) { + aicpu_task_def_ptr->add_output_addrs(ou_addr); + } + void *tmp_node_def = static_cast(node_def_.data()); + if (tmp_node_def == nullptr) { + MS_LOG(WARNING) << "tmp_node_def have no data"; + return false; + } + aicpu_task_def_ptr->set_node_def(tmp_node_def, node_def_.size()); + void *tmp_func_def = static_cast(func_def_.data()); + if (tmp_func_def == nullptr) { + MS_LOG(WARNING) << "tmp_func_def have no data"; + return false; + } + aicpu_task_def_ptr->set_func_def(tmp_func_def, func_def_.size()); + return true; +} + +AicpuIRTaskInfo::~AicpuIRTaskInfo() { + input_data_types_.clear(); + input_data_shapes_.clear(); + input_data_addrs_.clear(); + output_data_types_.clear(); + output_data_shapes_.clear(); + output_data_addrs_.clear(); + node_def_.clear(); + func_def_.clear(); +} + +bool LabelIRTaskInfo::SerializeIRToProto() { + auto label_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(label_task_def_ptr); + label_task_def_ptr->set_label_id(label_id_); + return true; +} + +bool EventIRTaskInfo::SerializeIRToProto() { + auto event_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(event_task_def_ptr); + event_task_def_ptr->set_event_id(event_id_); + return true; +} + +bool HcclIRTaskInfo::SerializeIRToProto() { + auto hccl_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(hccl_task_def_ptr); + hccl_task_def_ptr->set_hccl_type(hccl_type_); + hccl_task_def_ptr->set_input_addr(input_data_addr_); + hccl_task_def_ptr->set_output_addr(output_data_addr_); + auto tmp_wk = static_cast(workspace_.data()); + hccl_task_def_ptr->set_workspace(tmp_wk, workspace_.size()); + hccl_task_def_ptr->set_workspace_num(workspace_num_); + auto tmp_pri_def = static_cast(private_def_.data()); + hccl_task_def_ptr->set_private_def(tmp_pri_def, private_def_.size()); + hccl_task_def_ptr->set_ops_kernel_store(ops_kernel_store_); + hccl_task_def_ptr->set_count(count_); + hccl_task_def_ptr->set_root_id(root_id_); + hccl_task_def_ptr->set_op_type(op_type_); + hccl_task_def_ptr->set_data_type(data_type_); + return true; +} + +HcclIRTaskInfo::~HcclIRTaskInfo() { + workspace_.clear(); + private_def_.clear(); +} + +bool ProfilerIRTaskInfo::SerializeIRToProto() { + auto profiler_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(profiler_task_def_ptr); + profiler_task_def_ptr->set_log_id(log_id_); + profiler_task_def_ptr->set_flat(flat_); + profiler_task_def_ptr->set_notify(notify_); + return true; +} + +bool MemcpyAsyncIRTaskInfo::SerializeIRToProto() { + auto mem_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(mem_task_def_ptr); + mem_task_def_ptr->set_dst(dst_); + mem_task_def_ptr->set_dst_max(dst_max_); + mem_task_def_ptr->set_src(src_); + mem_task_def_ptr->set_count(count_); + mem_task_def_ptr->set_kind(kind_); + return true; +} + +bool StreamSwitchIRTaskInfo::SerializeIRToProto() { + auto stream_switch_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(stream_switch_task_def_ptr); + stream_switch_task_def_ptr->set_true_stream_id(true_stream_id_); + stream_switch_task_def_ptr->set_input_addr(input_addr_); + stream_switch_task_def_ptr->set_value_addr(value_addr_); + stream_switch_task_def_ptr->set_cond(cond_); + stream_switch_task_def_ptr->set_data_type(data_type_); + return true; +} + +bool StreamActiveIRTaskInfo::SerializeIRToProto() { + auto stream_active_task_def_ptr = std::unique_ptr(); + MS_EXCEPTION_IF_NULL(stream_active_task_def_ptr); + stream_active_task_def_ptr->set_active_stream_id(active_stream_id_); + return true; +} +} // namespace generator +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/generator/ir/ir_task_info.h b/mindspore/ccsrc/predict/generator/ir/ir_task_info.h new file mode 100644 index 00000000000..8e80cdddbee --- /dev/null +++ b/mindspore/ccsrc/predict/generator/ir/ir_task_info.h @@ -0,0 +1,295 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_EXECUTOR_GENERATOR_IR_IR_TASK_H_ +#define MINDSPORE_MINDSPORE_CCSRC_EXECUTOR_GENERATOR_IR_IR_TASK_H_ +#include +#include +#include +#include +#include +#include "predict/proto/ge_runtime_taskinfo.pb.h" + +namespace mindspore { +namespace generator { +using TaskType = ::ge::model_runner::TaskDef_TaskType; +enum TaskTmpType { + CCE_TMP_DEF = 0, + TBE_TMP_DEF = 1, + AICPU_TMP_DEF = 2, + LABEL_TMP_DEF = 3, + EVENT_TMP_DEF = 4, + HCCL_TMP_DEF = 5, + PROFILER_TRACE_TMP_DEF = 6, + MEMCPY_ASYNC_TMP_DEF = 7, + STREAM_SWITCH_TMP_DEF = 8, + STREAM_ACTIVE_TMP_DEF = 9 +}; + +struct KernelContext { + uint32_t kernel_type = 0; + uint32_t op_id = 0; + uint32_t kernel_func_id = 0; + uint32_t op_index = 0; + bool is_flowtable = false; + std::vector args_offset; + uint32_t args_count = 0; + std::vector origin_op_index; +}; + +class IRtaskInfo { + public: + virtual ~IRtaskInfo() = default; + virtual bool SerializeIRToProto() = 0; + + protected: + IRtaskInfo(TaskType task_type, TaskTmpType task_tmp_type, uint64_t stream_id) + : task_type_(task_type), task_tmp_type_(task_tmp_type), stream_id_(stream_id) {} + + public: + uint64_t GetStreamId() const { return stream_id_; } + TaskType GetTaskType() const { return task_type_; } + TaskTmpType GetTaskTmpType() const { return task_tmp_type_; } + + private: + TaskType task_type_; + TaskTmpType task_tmp_type_; + uint64_t stream_id_ = 0; +}; + +using IRtaskInfoPtr = std::shared_ptr; + +class CceIRTaskInfo : public IRtaskInfo { + public: + CceIRTaskInfo(TaskType task_type, uint64_t stream_id, KernelContext k_ctx, std::string stub_func, uint32_t block_dim, + std::vector args, uint32_t args_size, std::vector sm_desc, + std::vector flow_table) + : IRtaskInfo(task_type, CCE_TMP_DEF, stream_id), + k_ctx_(std::move(k_ctx)), + stub_func_(std::move(stub_func)), + block_dim_(block_dim), + args_(std::move(args)), + args_size_(args_size), + sm_desc_(std::move(sm_desc)), + flow_table_(std::move(flow_table)) {} + ~CceIRTaskInfo() override; + bool SerializeIRToProto() override; + + private: + KernelContext k_ctx_; + std::string stub_func_; + uint32_t block_dim_ = 0; + std::vector args_; + // uintptr_t args_addr_; + uint32_t args_size_ = 0; + std::vector sm_desc_; + std::vector flow_table_; +}; + +class TbeIRTaskInfo : public IRtaskInfo { + public: + TbeIRTaskInfo(TaskType task_type, uint64_t stream_id, std::string stub_func, uint32_t block_dim, + std::vector args, uint32_t args_size, std::vector sm_desc, + std::vector meta_data, std::vector input_data_addrs, + std::vector output_data_addrs, std::vector workspace_addrs) + : IRtaskInfo(task_type, TBE_TMP_DEF, stream_id), + stub_func_(std::move(stub_func)), + block_dim_(block_dim), + args_(std::move(args)), + args_size_(args_size), + sm_desc_(std::move(sm_desc)), + meta_data_(std::move(meta_data)), + input_data_addrs_(std::move(input_data_addrs)), + output_data_addrs_(std::move(output_data_addrs)), + workspace_addrs_(std::move(workspace_addrs)) {} + ~TbeIRTaskInfo() override; + bool SerializeIRToProto() override; + + private: + std::string stub_func_; + uint32_t block_dim_ = 0; + std::vector args_; + uint32_t args_size_ = 0; + std::vector sm_desc_; + // uintptr_t binary_; + // uint32_t binary_size_; + std::vector meta_data_; + std::vector input_data_addrs_; + std::vector output_data_addrs_; + std::vector workspace_addrs_; + // std::vector flow_table_; +}; + +class AicpuIRTaskInfo : public IRtaskInfo { + public: + AicpuIRTaskInfo(TaskType task_type, uint64_t stream_id, std::string op_type, uint32_t flag, + std::vector input_data_types, std::vector> input_data_shapes, + std::vector input_data_addrs, std::vector output_data_types, + std::vector> output_data_shapes, std::vector output_data_addrs, + std::vector node_def, std::vector func_def) + : IRtaskInfo(task_type, AICPU_TMP_DEF, stream_id), + op_type_(std::move(op_type)), + flag_(flag), + input_data_types_(std::move(input_data_types)), + input_data_shapes_(std::move(input_data_shapes)), + input_data_addrs_(std::move(input_data_addrs)), + output_data_types_(std::move(output_data_types)), + output_data_shapes_(std::move(output_data_shapes)), + output_data_addrs_(std::move(output_data_addrs)), + node_def_(std::move(node_def)), + func_def_(std::move(func_def)) {} + ~AicpuIRTaskInfo() override; + bool SerializeIRToProto() override; + + private: + std::string op_type_; + uint32_t flag_ = 0; + std::vector input_data_types_; + std::vector> input_data_shapes_; + std::vector input_data_addrs_; + std::vector output_data_types_; + std::vector> output_data_shapes_; + std::vector output_data_addrs_; + std::vector node_def_; + std::vector func_def_; +}; + +class LabelIRTaskInfo : public IRtaskInfo { + public: + LabelIRTaskInfo(TaskType task_type, uint64_t stream_id, uint32_t label_id) + : IRtaskInfo(task_type, LABEL_TMP_DEF, stream_id), label_id_(label_id) {} + ~LabelIRTaskInfo() override {} + bool SerializeIRToProto() override; + + private: + uint32_t label_id_ = 0; +}; + +class EventIRTaskInfo : public IRtaskInfo { + public: + EventIRTaskInfo(TaskType task_type, uint64_t stream_id, uint32_t event_id) + : IRtaskInfo(task_type, EVENT_TMP_DEF, stream_id), event_id_(event_id) {} + ~EventIRTaskInfo() override {} + bool SerializeIRToProto() override; + + private: + uint32_t event_id_ = 0; +}; + +class HcclIRTaskInfo : public IRtaskInfo { + public: + HcclIRTaskInfo(TaskType task_type, uint64_t stream_id, std::string hccl_type, uintptr_t input_data_addr, + uintptr_t output_data_addr, std::vector workspace, int64_t workspace_num, + std::vector private_def, uintptr_t ops_kernel_store, int32_t count, int64_t root_id, + int64_t op_type, int64_t data_type) + : IRtaskInfo(task_type, HCCL_TMP_DEF, stream_id), + hccl_type_(std::move(hccl_type)), + input_data_addr_(input_data_addr), + output_data_addr_(output_data_addr), + workspace_(std::move(workspace)), + workspace_num_(workspace_num), + private_def_(std::move(private_def)), + ops_kernel_store_(ops_kernel_store), + count_(count), + root_id_(root_id), + op_type_(op_type), + data_type_(data_type) {} + ~HcclIRTaskInfo() override; + bool SerializeIRToProto() override; + + private: + std::string hccl_type_; + uintptr_t input_data_addr_ = 0; + uintptr_t output_data_addr_ = 0; + std::vector workspace_; + int64_t workspace_num_ = 0; + std::vector private_def_; + uintptr_t ops_kernel_store_ = 0; + int32_t count_ = 0; + int64_t root_id_ = 0; + int64_t op_type_ = 0; + int64_t data_type_ = 0; +}; + +class ProfilerIRTaskInfo : public IRtaskInfo { + public: + ProfilerIRTaskInfo(TaskType task_type, uint64_t stream_id, uint64_t log_id, bool notify, uint32_t flat) + : IRtaskInfo(task_type, PROFILER_TRACE_TMP_DEF, stream_id), log_id_(log_id), notify_(notify), flat_(flat) {} + ~ProfilerIRTaskInfo() override {} + bool SerializeIRToProto() override; + + private: + uint64_t log_id_ = 0; + bool notify_ = false; + uint32_t flat_ = 0; +}; + +class MemcpyAsyncIRTaskInfo : public IRtaskInfo { + public: + MemcpyAsyncIRTaskInfo(TaskType task_type, uint32_t stream_id, uint64_t dst, uint64_t dst_max, uint64_t src, + uint64_t count, int64_t kind) + : IRtaskInfo(task_type, MEMCPY_ASYNC_TMP_DEF, stream_id), + dst_(dst), + dst_max_(dst_max), + src_(src), + count_(count), + kind_(kind) {} + ~MemcpyAsyncIRTaskInfo() override {} + bool SerializeIRToProto() override; + + private: + uint64_t dst_ = 0; + uint64_t dst_max_ = 0; + uint64_t src_ = 0; + uint64_t count_ = 0; + uint32_t kind_ = 0; +}; + +class StreamSwitchIRTaskInfo : public IRtaskInfo { + public: + StreamSwitchIRTaskInfo(TaskType task_type, uint64_t stream_id, uint32_t true_stream_id, uintptr_t input_addr, + uintptr_t value_addr, uint32_t cond, int64_t data_type) + : IRtaskInfo(task_type, STREAM_SWITCH_TMP_DEF, stream_id), + true_stream_id_(true_stream_id), + input_addr_(input_addr), + value_addr_(value_addr), + cond_(cond), + data_type_(data_type) {} + ~StreamSwitchIRTaskInfo() override {} + bool SerializeIRToProto() override; + + private: + uint32_t true_stream_id_ = 0; + uintptr_t input_addr_ = 0; + uintptr_t value_addr_ = 0; + uint32_t cond_ = 0; + int64_t data_type_ = 0; +}; + +class StreamActiveIRTaskInfo : public IRtaskInfo { + public: + StreamActiveIRTaskInfo(TaskType task_type, uint64_t stream_id, uint32_t active_stream_id) + : IRtaskInfo(task_type, STREAM_ACTIVE_TMP_DEF, stream_id), active_stream_id_(active_stream_id) {} + ~StreamActiveIRTaskInfo() override {} + bool SerializeIRToProto() override; + + private: + uint32_t active_stream_id_ = 0; +}; +}; // namespace generator +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_EXECUTOR_GENERATOR_IR_IR_TASK_H_ diff --git a/mindspore/ccsrc/predict/generator/utils/ir_model_util.cc b/mindspore/ccsrc/predict/generator/utils/ir_model_util.cc new file mode 100644 index 00000000000..8128009472f --- /dev/null +++ b/mindspore/ccsrc/predict/generator/utils/ir_model_util.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/generator/utils/ir_model_util.h" +namespace mindspore { +namespace generator { +IRModelUtil &IRModelUtil::GetInstance() { + static IRModelUtil instance; + return instance; +} + +void IRModelUtil::Init() { + MS_LOG(INFO) << "IRModel init success"; + version_ = "defaultVersion"; + stream_num_ = 0; + event_num_ = 0; + batch_num_ = 0; + memory_size_ = 0; + weight_size_ = 0; + var_size_ = 0; + logic_mem_base_ = 0; + logic_var_base_ = 0; + logic_var_base_ = 0; + priority_ = 0; + is_enable_save_model_ = false; + min_static_offset_ = 0; + max_dynamic_offset_ = 0; +} +} // namespace generator +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/generator/utils/ir_model_util.h b/mindspore/ccsrc/predict/generator/utils/ir_model_util.h new file mode 100644 index 00000000000..a654cb980f8 --- /dev/null +++ b/mindspore/ccsrc/predict/generator/utils/ir_model_util.h @@ -0,0 +1,92 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_GENERATOR_IR_IR_MODEL_UTIL_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_GENERATOR_IR_IR_MODEL_UTIL_H_ +#include +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { +namespace generator { +class IRModelUtil { + public: + static IRModelUtil &GetInstance(); + IRModelUtil(const IRModelUtil &) = delete; + IRModelUtil &operator=(const IRModelUtil &) = delete; + void Init(); + + void set_version(const std::string &version) { version_ = version; } + void set_stream_num(uint32_t stream_num) { stream_num_ = stream_num; } + void set_event_num(uint32_t event_num) { event_num_ = event_num; } + void set_batch_num(uint32_t batch_num) { batch_num_ = batch_num; } + void set_memory_size(uint32_t memory_size) { memory_size_ = memory_size; } + void set_weight_size(uint32_t weight_size) { weight_size_ = weight_size; } + void set_var_size(uint32_t var_size) { var_size_ = var_size; } + void set_logic_mem_base(uint32_t logic_mem_base) { logic_mem_base_ = logic_mem_base; } + void set_logic_weight_base(uint32_t logic_weight_base) { logic_weight_base_ = logic_weight_base; } + void set_logic_var_base(uint32_t logic_var_base) { logic_var_base_ = logic_var_base; } + void set_priority(uint32_t priority) { priority_ = priority; } + void set_is_enable_save_model(bool is_enable_save_model) { is_enable_save_model_ = is_enable_save_model; } + void set_min_static_offset(uint64_t min_static_offset) { min_static_offset_ = min_static_offset; } + void set_max_dynamic_offset(uint64_t max_dynamic_offset) { max_dynamic_offset_ = max_dynamic_offset; } + void set_max_mem_size(uint64_t max_mem_size) { max_mem_size_ = max_mem_size; } + void set_irmodel_mem_base(uint8_t irmodel_mem_base) { irmodel_mem_base_ = irmodel_mem_base; } + + std::string version() const { return version_; } + uint32_t stream_num() const { return stream_num_; } + uint32_t event_num() const { return event_num_; } + uint32_t batch_num() const { return batch_num_; } + uint64_t memory_size() const { return memory_size_; } + uint64_t weight_size() const { return weight_size_; } + uint64_t var_size() const { return var_size_; } + uint64_t logic_mem_base() const { return logic_mem_base_; } + uint64_t logic_weight_base() const { return logic_weight_base_; } + uint64_t logic_var_base() const { return logic_var_base_; } + uint32_t priority() const { return priority_; } + bool is_enable_save_model() const { return is_enable_save_model_; } + uint64_t min_static_offset() const { return min_static_offset_; } + uint64_t max_dynamic_offset() const { return max_dynamic_offset_; } + uint64_t max_mem_size() const { return max_mem_size_; } + uint8_t irmodel_mem_base() const { return irmodel_mem_base_; } + + private: + IRModelUtil() = default; + ~IRModelUtil() = default; + std::string version_; + uint32_t stream_num_ = 0; + uint32_t event_num_ = 0; + uint32_t batch_num_ = 0; + uint64_t memory_size_ = 0; + uint64_t weight_size_ = 0; + uint64_t var_size_ = 0; + uint64_t logic_mem_base_ = 0; + uint64_t logic_weight_base_ = 0; + uint64_t logic_var_base_ = 0; + uint32_t priority_ = 0; + bool is_enable_save_model_ = false; + uint64_t min_static_offset_ = 0; + uint64_t max_dynamic_offset_ = 0; + uint64_t max_mem_size_ = 0; + uint8_t irmodel_mem_base_ = 0; +}; +} // namespace generator +} // namespace mindspore + +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_GENERATOR_IR_IR_MODEL_UTIL_H_ diff --git a/mindspore/ccsrc/predict/predict.cc b/mindspore/ccsrc/predict/predict.cc new file mode 100644 index 00000000000..d81dcd33216 --- /dev/null +++ b/mindspore/ccsrc/predict/predict.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "predict/predict.h" + +#include +#include +#include + +namespace mindspore { +namespace predictmodel { +void StepConvertGraph(const KernelGraphPtrNew &kernel_graph_ptr) { + MS_LOG(INFO) << "start convert_graph step"; + // get kernel_graph. this graph can be origin or device, depends on which steps to persistence + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + bool save_ms_model = MsContext::GetInstance()->save_ms_model_flag(); + if (save_ms_model) { + // set convert_mode: convert cpu info or convert Davnici + executor::Kernel2Ms::GetInstance().set_convert_mode(executor::kConvertCpuMode); + // convert kernel_graph to sub_ms_graph + bool ret = executor::Kernel2Ms::GetInstance().KernelGraph2MsGraph(kernel_graph_ptr); + if (!ret) { + MS_LOG(WARNING) << "convert to mindsporeGraph failed"; + } else { + MS_LOG(INFO) << "convert to Graph success"; + } + } +} + +void StepConvertWeight(const std::vector &inputs) { + MS_LOG(INFO) << "start convert_input step"; + // get all inputs tensor + bool save_ms_model = MsContext::GetInstance()->save_ms_model_flag(); + std::string save_path = MsContext::GetInstance()->save_ms_model_path(); + if (save_ms_model) { + MS_LOG(INFO) << "save ms model is true to path " << save_path; + if (!executor::Kernel2Ms::GetInstance().KernelInput2MS(inputs)) { + MS_LOG(WARNING) << "convert mindspore kernel input failed"; + } + auto new_ms_graph_ptr = std::make_shared(); + bool ret = executor::Kernel2Ms::GetInstance().SaveDeviceModel(new_ms_graph_ptr, save_path); + if (!ret) { + MS_LOG(WARNING) << "convert to mindsporeGraph failed"; + } else { + MS_LOG(INFO) << "save ms model success"; + } + } +} + +executor::TargetMode GetDeviceTarget(const std::string &device_target) { + if (device_target == "GPU") { + return executor::kGPUTarget; + } else if (device_target == "Ascend") { + return executor::kCPUTarget; + } else { + return executor::kUnknowTarget; + } +} +} // namespace predictmodel +} // namespace mindspore diff --git a/mindspore/ccsrc/predict/predict.h b/mindspore/ccsrc/predict/predict.h new file mode 100644 index 00000000000..04184fe77c1 --- /dev/null +++ b/mindspore/ccsrc/predict/predict.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_PREDICT_H_ +#define MINDSPORE_MINDSPORE_CCSRC_PREDICT_H_ + +#include +#include +#include +#include "session/session_basic.h" +#include "predict/converter/kernel2ms.h" + +namespace mindspore { +namespace predictmodel { +using KernelGraphPtrNew = std::shared_ptr; +void StepConvertGraph(const KernelGraphPtrNew &kernel_graph_ptr); +void StepConvertWeight(const std::vector &inputs); +executor::TargetMode GetDeviceTarget(const std::string &device_target); +} // namespace predictmodel +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_PREDICT_H_ diff --git a/mindspore/ccsrc/predict/proto/DModel_ir.proto b/mindspore/ccsrc/predict/proto/DModel_ir.proto new file mode 100644 index 00000000000..02bfa94df3d --- /dev/null +++ b/mindspore/ccsrc/predict/proto/DModel_ir.proto @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +import public "Graph_ir.proto"; +import public "ge_runtime_taskinfo.proto"; +package ge.model_runner; +option cc_enable_arenas = true; + +message ModelTaskDef { + + string version = 1; + + repeated TaskDef task = 10; + + uint32 stream_num = 11; + uint32 event_num = 12; + uint32 batch_num_ = 13; + + uint64 memory_size = 14; + uint64 weight_size = 15; + uint64 var_size_ = 16; + + uint64 logic_mem_base_ = 17; + uint64 logic_weight_base_ = 18; + uint64 logic_var_base_ = 19; + + uint32 priority_ = 20; +} diff --git a/mindspore/ccsrc/predict/proto/Graph_ir.proto b/mindspore/ccsrc/predict/proto/Graph_ir.proto new file mode 100644 index 00000000000..af91ec0917a --- /dev/null +++ b/mindspore/ccsrc/predict/proto/Graph_ir.proto @@ -0,0 +1,125 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package mindspore; + +// Data type definition +enum DataType { + DT_UNDEFINED = 0; + // Basic types. + DT_BOOL = 1; // bool + + DT_INT8 = 2; // int8_t + DT_INT16 = 3; // int16_t + DT_INT32 = 4; // int32_t + DT_INT64 = 5; // int64_t + + DT_UINT8 = 6; // uint8_t + DT_UINT16 = 7; // uint16_t + DT_UINT32 = 8; // uint32_t + DT_UINT64 = 9; // uint64_t + + DT_FLOAT16 = 10; // float 16 + DT_FLOAT32 = 11; // float 32 + DT_FLOAT64 = 12; // float 64 + + DT_STRING = 13; // string + DT_TENSOR = 14; // tensor + DT_GRAPH = 15; // graph + + // list type + DT_BOOLS = 16; // list of bool + + DT_INTS8 = 17; // list of int8_t + DT_INTS16 = 18; // list of int16_t + DT_INTS32 = 19; // list of int32_t + DT_INTS64 = 20; // list of int64_t + + DT_UINTS8 = 21; // list of uint8_t + DT_UINTS16 = 22; // list of uint16_t + DT_UINTS32 = 23; // list of uint32_t + DT_UINTS64 = 24; // list of uint64_t + + DT_FLOATS16 = 25; // list of float16 + DT_FLOATS32 = 26; // list of float32 + DT_FLOATS64 = 27; // list of float64 + + DT_STRINGS = 28; // list of string + DT_TENSORS = 29; // list of tensor + DT_GRAPHS = 30; // list of graph + + DT_TUPLE = 31; // tuple + DT_LIST = 32; // list + DT_DICT = 33; // dictionary + + // other types + DT_NONE = 34; // None + DT_SYM_INST = 35; // Symbolic Key Instance + + // type related type + DT_BASE_INT = 36; // type generic int + DT_BASE_UINT = 37; // type generate unsigned int + DT_BASE_FLOAT = 38; // type generate float + DT_TYPE = 39; // type type + DT_ANYTHING = 40; // type anything +}; + +enum MSConst { + DEFAULT_REFCOUNT = 0; + WEIGHT_REFCOUNT = 999; +}; + +message TensorDef { + DataType data_type = 1; + + repeated int64 dims = 2; + + string format = 3; + string layout = 4; + uint32 refCount = 5; + uint64 offset = 6; + uint64 size = 7; + uint64 weight_size = 8; + bytes data = 9; +} + +message OpDef { + string name = 1; + string type = 2; + + string fwk_type = 3; + string opAttr = 4; + repeated int64 input_index = 5; + repeated int64 output_index = 6; +} + +message GraphDef { + string name = 1; + + repeated int64 input_index = 2; + + repeated int64 output_index = 3; + uint64 mempool_size = 4; + + repeated OpDef opdefs = 5; + + repeated TensorDef alltensors = 6; +} + + + diff --git a/mindspore/ccsrc/predict/proto/ge_runtime_taskinfo.proto b/mindspore/ccsrc/predict/proto/ge_runtime_taskinfo.proto new file mode 100644 index 00000000000..3429d065443 --- /dev/null +++ b/mindspore/ccsrc/predict/proto/ge_runtime_taskinfo.proto @@ -0,0 +1,155 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package ge.model_runner; +option cc_enable_arenas = true; + +message TaskDef { + enum TaskType { + CCE = 0; + TBE = 1; + AICPU = 2; + LABEL_SET = 3; + LABEL_SWITCH = 4; + LABEL_GOTO = 5; + EVENT_RECORD = 6; + EVENT_WAIT = 7; + FUSION_START = 8; + FUSION_END = 9; + HCCL = 10; + PROFILER_TRACE = 11; + MEMCPY_ASYNC = 12; + STREAM_SWITCH = 13; + STREAM_ACTIVE = 14; + // insert new task type here + REVSERVED = 23; + }; + + TaskType task_type = 1; + uint64 stream_id = 2; + oneof subclass { + CceTaskDef cce_task_def = 3; + TbeTaskDef tbe_task_def = 4; + AicpuTaskDef aicpu_task_def = 5; + LabelTaskDef label_task_def = 6; + EventTaskDef event_task_def = 7; + HcclTaskDef hccl_task_def = 8; + ProfilerTaskDef profiler_task_def = 9; + MemcpyAsyncTaskDef memcpy_async_task_def = 10; + StreamSwitchTaskDef stream_switch_task_def = 11; + StreamActiveTaskDef stream_active_task_def = 12; + } +} + +message CceTaskDef { + KernelContext kernel_context = 1; + string stub_func = 2; + uint32 block_dim = 3; + bytes args = 4; + uint32 args_size = 5; + bytes sm_desc = 6; + bytes flow_table = 7; +} + +message TbeTaskDef { + string stub_func = 1; + uint32 block_dim = 2; + bytes args = 3; + uint32 args_size = 4; + bytes sm_desc = 5; + bytes meta_data = 8; + repeated uint64 input_addrs = 9; + repeated uint64 output_addrs = 10; + repeated uint64 workspace_addrs = 11; +} + +message AicpuTaskDef { + string op_type = 1; + uint32 flag = 2; + repeated uint32 input_types = 3; + repeated Shape input_shapes = 4; + repeated uint64 input_addrs = 5; + repeated uint32 output_types = 6; + repeated Shape output_shapes = 7; + repeated uint64 output_addrs = 8; + bytes node_def = 9; + bytes func_def = 10; +} + +message Shape { + repeated uint32 shape = 1; +} + +message LabelTaskDef { + uint32 label_id = 1; +} + +message EventTaskDef { + uint32 event_id = 1; +} + +message HcclTaskDef { + string hccl_type = 1; + uint64 input_addr = 2; + uint64 output_addr = 3; + bytes workspace = 4; + int64 workspace_num = 5; + bytes private_def = 6; + uint64 ops_kernel_store = 7; + int32 count = 8; + int64 root_id = 9; + int64 op_type = 10; + int64 data_type = 11; +} + +message ProfilerTaskDef { + uint64 log_id = 1; + bool notify = 2; + uint32 flat = 3; +} + +message MemcpyAsyncTaskDef { + uint64 dst = 1; + uint64 dst_max = 2; + uint64 src = 3; + uint64 count = 4; + uint32 kind = 5; +} + +message StreamSwitchTaskDef { + uint32 true_stream_id = 1; + uint64 input_addr = 2; + uint64 value_addr = 3; + int64 cond = 4; + int64 data_type = 5; +} + +message StreamActiveTaskDef { + uint32 active_stream_id = 1; +} + +message KernelContext { + uint32 kernel_type = 1; + uint32 op_id = 2; + uint32 kernel_func_id = 3; + uint32 op_index = 4; + bool is_flowtable = 5; + bytes args_offset = 6; + uint32 args_count = 7; + repeated uint32 origin_op_index = 8; +} diff --git a/mindspore/ccsrc/predict/readme.txt b/mindspore/ccsrc/predict/readme.txt new file mode 100644 index 00000000000..d75abf257be --- /dev/null +++ b/mindspore/ccsrc/predict/readme.txt @@ -0,0 +1,17 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +this is a dictory for predict including saving model &&& saving taskinfos. diff --git a/mindspore/ccsrc/predict/schema/inner/readme.txt b/mindspore/ccsrc/predict/schema/inner/readme.txt new file mode 100644 index 00000000000..774f71f602c --- /dev/null +++ b/mindspore/ccsrc/predict/schema/inner/readme.txt @@ -0,0 +1 @@ +this is a dictory for predict to gen fbs headers \ No newline at end of file diff --git a/mindspore/ccsrc/predict/schema/ms.fbs b/mindspore/ccsrc/predict/schema/ms.fbs new file mode 100644 index 00000000000..a114fc444ef --- /dev/null +++ b/mindspore/ccsrc/predict/schema/ms.fbs @@ -0,0 +1,153 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +include "op.fbs"; + +namespace mindspore.predict; + +enum DataType : int { + DT_FLOAT = 0, + DT_FLOAT16 = 1, + DT_INT8 = 2, + DT_INT32 = 3, + DT_UINT8 = 4, + DT_UINT32 = 8, + DT_UNDEFINED = 16 +} + +enum Format : int { + NCHW = 0, + NHWC, + NC4HW4 = 100, + NUM_OF_FORMAT +} + +enum MSConst: int { + WEIGHT_REFCOUNT = 999 +} + +table QuantizationDef { + // Quantized value q, corresponding float value r: + // r = scale * (q - zero_point), where scale = (rmax - rmin) / (qmax - qmin) + min: [float]; + max: [float]; + scale: [float]; + zero_point: [long]; + + // Tensor shape of the specifies dimension. + dimension: int; +} + +table TensorDef { + // data type + dataType: DataType; + // shape + dims: [int]; + format: Format; + refCount: int; + offset: int; + data: [ubyte]; + quantization: QuantizationDef; +} + +union OpT { + Concat, + SoftMax, + Activation, + Conv2D, + FusedBatchNorm, + CaffeBatchNorm, + Squeeze, + BiasAdd, + Pooling, + DepthwiseConv2D, + DeDepthwiseConv2D, + Resize, + DetectionPostProcess, + FullConnection, + Mean, + DeConv2D, + Scale, + Reshape, + Eltwise, + NetOutput, + Add, + MatMul, + StridedSlice, + Power, + Slice, + Stack, + Mul, + Pad, + Maximum, + CaffePReLU, + ArgMax, + Exp, + CaffeCrop, + Range, + ExpandDims, + Tile, + Cast +// Split +} + +enum QuantType: int { + QUANT_NONE, + QUANT_INT8 +} + +table OpDef { + name: string; + attr: OpT; + inputIndex: [uint]; + outputIndex: [uint]; + isLastConv: bool; + quantType: QuantType = QUANT_NONE; +} + + +enum FmkType: int { + TF, + CAFFE +} + +table NodeDef { + fmkType: FmkType; + opDef: OpDef; +} + + +table SubGraphDef { + name: string; + inputIndex: [uint]; + outputIndex: [uint]; + mempoolSize: uint; + nodes: [NodeDef]; + allTensors: [TensorDef]; // weight + input + output +} + +table MempoolCfg { + size: uint; + shiftFactor: uint; +} + +table GraphDef { + name: string; + mempoolCfg: MempoolCfg; + subgraphs: [SubGraphDef]; +} + +root_type GraphDef; diff --git a/mindspore/ccsrc/predict/schema/op.fbs b/mindspore/ccsrc/predict/schema/op.fbs new file mode 100644 index 00000000000..d48f11b4d1e --- /dev/null +++ b/mindspore/ccsrc/predict/schema/op.fbs @@ -0,0 +1,347 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace mindspore.predict; + +enum ResizeMethod: byte { + UNKNOW = -1, + BILINEAR = 0, + NEAREST_NEIGHBOR = 1 +} + +enum DataFormatType : byte { // todo combine with mslite.h::Format + UNKNOW = -1, + NCHW = 0, + NHWC = 1, + HWC = 2, // for input image or resize + CHW = 3, // for input image or resize +} + +enum ActivationType : byte { + NO_ACTIVATION = 0, + RELU = 1, + SIGMOID = 2, + RELU6 = 3, + ELU = 4, + LEAKY_RELU = 5, + ABS = 6, + RELU1 = 7, + SOFTSIGN = 8, + SOFTPLUS = 9, + TANH = 10, + UNKNOW = 11 +} + +enum PoolMode : byte { + MAX_POOLING = 0, + MEAN_POOLING = 1, + GLOBAL_POOING = 2 +} + +enum EltwiseMode : byte { + PROD = 0, + SUM = 1, + MAXIMUM = 2 +} + +enum PadMode : byte { + NOTSET=0, + SAME=1, + VALID=2, + CAFFE_CEIL_NEW=4 +} + +enum PaddingMode : byte { + CONSTANT = 0, + REFLECT = 1, + SYMMETRIC = 2, + MODE_RESERVED = 3 +} + +table Pad { + paddingmode: PaddingMode; + paddings: [int]; +} + +table Maximum { + format: DataFormatType = 0; +} + +table Concat { + axis: int; + n: int; +} + +table SoftMax { + axis: [int]; +} + +table Activation { + type: ActivationType = 0; +} + +table Conv2D { + format: DataFormatType = 0; + group: int; + channelIn: int; + channelOut: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + +table FusedBatchNorm { + epsilon: float; // eg. epsilon=0.001 +} + +table CaffeBatchNorm { + epsilon: float; // eg. epsilon=0.001 +} + +table Squeeze { + axis: [int]; +} + +table BiasAdd { + axis: [int]; +} + +table Pooling { + format: DataFormatType = 0; + poolingMode: PoolMode; + windowW: int; + windowH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + // todo replace with padValueMode in convolution pooling and so on + caffeMode: bool = false; +} + +table DepthwiseConv2D { + format: DataFormatType = 0; + channelIn: int; + channelMultiplier: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + +table DeDepthwiseConv2D { + format: DataFormatType = 0; + channelIn: int; + channelMultiplier: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + + +table Resize { + format: DataFormatType = 0; + method: ResizeMethod; + newHeight: long; + newWidth: long; + alignCorners: bool = false; + preserveAspectRatio: bool = false; +} + +table DetectionPostProcess { + format: DataFormatType = 0; + inputSize: int; + hScale: float; + wScale: float; + xScale: float; + yScale: float; + NmsIouThreshold: float; + NmsScoreThreshold: float; + MaxDetections: long; + DetectionsPreClass: long; + MaxClassesPreDetection: long; + NumClasses: long; + UseRegularNms: bool; +} + +table FullConnection { + format: DataFormatType = 0; + hasBias: bool; +} + +// Mean(input_tensor, axis, keep_dims) +table Mean { + axis: [int]; + keepDims: bool = false; +} + +table DeConv2D { + format: DataFormatType = 0; + group: int; + channelIn: int; + channelOut: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + +table Scale { + format: DataFormatType = 0; +} + +table Eltwise { + format: DataFormatType = 0; + mode: EltwiseMode; + // todo repeat coeff (default 1) +} + +table Add { + format: DataFormatType = 0; +} + +table Slice { + format: DataFormatType = 0; + begin: [int]; + end: [int]; + stride: [int]; +} + +table Mul { +} + +table Exp { +} + +table Reshape { + format: DataFormatType = 0; +} + +table Power { + power: float; + scale: float; + shift: float; +} + +table ArgMax { + axis: int; + outMaxValue: bool; + topK: int; + keepDims: bool; + axisType: int; +} + +table NetOutput { + format: DataFormatType = 0; +} + +table MatMul { + transposeA : bool = false; + transposeB : bool = false; +} + +table CaffePReLU { + channelShared : bool = false; +} + +table StridedSlice { + beginMask: int; + endMask: int; + ellipsisMask: int; + newAxisMask: int; + shrinkAxisMask: int; + begin: [int]; + end: [int]; + stride: [int]; + isScale: [int]; +} + +table Stack { + axis: int; + n: int; + isScale: [int]; +} + +table Range { + start: int; + limit: int; + delta: int; +} + +table ExpandDims { + dim: int; +} + +table Tile { + multiples: [int]; +} + +table Cast { + srcT: int; + dstT: int; +} + +//table Split { +// numberSplit: int; +// sizeSplits: [int]; +// splitDim: int; +//} + +table CaffeCrop { + axis : long; + offsets : [long]; +} diff --git a/mindspore/ccsrc/pybind_api/CMakeLists.txt b/mindspore/ccsrc/pybind_api/CMakeLists.txt new file mode 100644 index 00000000000..adcb5ddda19 --- /dev/null +++ b/mindspore/ccsrc/pybind_api/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _PYNATIVE_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_pynative_obj OBJECT ${_PYNATIVE_ALL_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/pybind_api/api_register.cc b/mindspore/ccsrc/pybind_api/api_register.cc new file mode 100644 index 00000000000..0c8c47d6589 --- /dev/null +++ b/mindspore/ccsrc/pybind_api/api_register.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pybind_api/api_register.h" + +#include + +namespace mindspore { + +PybindDefineRegister &PybindDefineRegister::GetSingleton() { + static PybindDefineRegister instance; + return instance; +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/pybind_api/api_register.h b/mindspore/ccsrc/pybind_api/api_register.h new file mode 100644 index 00000000000..2c1b622f31f --- /dev/null +++ b/mindspore/ccsrc/pybind_api/api_register.h @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYBIND_API_API_REGISTER_H_ +#define PYBIND_API_API_REGISTER_H_ + +#include +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +namespace py = pybind11; + +namespace mindspore { + +using PybindDefineFunc = std::function; + +class PybindDefineRegister { + public: + static void Register(const std::string& name, const PybindDefineFunc& fn) { + return GetSingleton().RegisterFn(name, fn); + } + + PybindDefineRegister(const PybindDefineRegister&) = delete; + + PybindDefineRegister& operator=(const PybindDefineRegister&) = delete; + + static std::map& AllFuncs() { return GetSingleton().fns_; } + + std::map fns_; + + protected: + PybindDefineRegister() = default; + + virtual ~PybindDefineRegister() = default; + + static PybindDefineRegister& GetSingleton(); + + void RegisterFn(const std::string& name, const PybindDefineFunc& fn) { fns_[name] = fn; } +}; + +class PybindDefineRegisterer { + public: + PybindDefineRegisterer(const std::string& name, const PybindDefineFunc& fn) { + PybindDefineRegister::Register(name, fn); + } + ~PybindDefineRegisterer() = default; +}; + +#define REGISTER_PYBIND_DEFINE(name, define) PybindDefineRegisterer g_pybind_define_f_##name(#name, define) + +} // namespace mindspore + +#endif // PYBIND_API_API_REGISTER_H_ diff --git a/mindspore/ccsrc/pybind_api/export_flags.cc b/mindspore/ccsrc/pybind_api/export_flags.cc new file mode 100644 index 00000000000..351279ccddf --- /dev/null +++ b/mindspore/ccsrc/pybind_api/export_flags.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pybind_api/export_flags.h" +namespace mindspore { + +const char PYTHON_PRIMITIVE_FLAG[] = "__primitive_flag__"; +const char PYTHON_METAFUNCGRAPH_FLAG[] = "__metafuncgraph_flag__"; +const char PYTHON_TENSOR_FLAG[] = "__tensor_flag__"; +const char PYTHON_ENVINSTANCE_FLAG[] = "__envinstance_flag__"; +const char PYTHON_DTYPE_FLAG[] = "__dtype_flag__"; +const char PYTHON_CELL_AS_LIST[] = "__cell_as_list__"; +const char PYTHON_DATACLASS_FIELDS[] = "__dataclass_fields__"; + +// flag names +const char GRAPH_FLAG_MIX_PRECISION_FP16[] = "fp16"; +const char GRAPH_FLAG_MIX_PRECISION_FP32[] = "fp32"; +const char GRAPH_FLAG_LOOP_CAN_UNROLL[] = "loop_can_unroll"; +const char GRAPH_FLAG_HAS_EFFECT[] = "has_effect"; +const char GRAPH_FLAG_EFFECT_PATIAL_ORDER[] = "_effect_patial_order"; + +} // namespace mindspore diff --git a/mindspore/ccsrc/pybind_api/export_flags.h b/mindspore/ccsrc/pybind_api/export_flags.h new file mode 100644 index 00000000000..b27357a52ee --- /dev/null +++ b/mindspore/ccsrc/pybind_api/export_flags.h @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYBIND_API_EXPORT_FLAGS_H_ +#define PYBIND_API_EXPORT_FLAGS_H_ + +namespace mindspore { + +extern const char PYTHON_PRIMITIVE_FLAG[]; +extern const char PYTHON_METAFUNCGRAPH_FLAG[]; +extern const char PYTHON_TENSOR_FLAG[]; +extern const char PYTHON_ENVINSTANCE_FLAG[]; +extern const char PYTHON_DTYPE_FLAG[]; +extern const char PYTHON_CELL_AS_LIST[]; +extern const char PYTHON_DATACLASS_FIELDS[]; + +extern const char GRAPH_FLAG_MIX_PRECISION_FP16[]; +extern const char GRAPH_FLAG_MIX_PRECISION_FP32[]; +extern const char GRAPH_FLAG_LOOP_CAN_UNROLL[]; +extern const char GRAPH_FLAG_HAS_EFFECT[]; +extern const char GRAPH_FLAG_EFFECT_PATIAL_ORDER[]; + +} // namespace mindspore + +#endif // PYBIND_API_EXPORT_FLAGS_H_ diff --git a/mindspore/ccsrc/pynative/CMakeLists.txt b/mindspore/ccsrc/pynative/CMakeLists.txt new file mode 100644 index 00000000000..adcb5ddda19 --- /dev/null +++ b/mindspore/ccsrc/pynative/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _PYNATIVE_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_pynative_obj OBJECT ${_PYNATIVE_ALL_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/pynative/pynative_execute.cc b/mindspore/ccsrc/pynative/pynative_execute.cc new file mode 100644 index 00000000000..27cfd89106c --- /dev/null +++ b/mindspore/ccsrc/pynative/pynative_execute.cc @@ -0,0 +1,518 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pynative/pynative_execute.h" + +#include +#include +#include +#include + +#include "utils/any.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "operator/ops.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/static_analysis/prim.h" +#include "session/session_factory.h" + +const char SINGLE_OP_GRAPH[] = "single_op_graph"; +// primitive unable to infer value for constant input in pynative mode +const std::unordered_set ignore_infer_prim = {"partial"}; +const std::unordered_set vm_operators = {"partial", "depend"}; + +namespace mindspore { +namespace pynative { +using transform::GraphRunner; +using transform::GraphRunnerOptions; +using transform::OperatorPtr; +inline ValuePtr PyAttrValue(const py::object& obj) { + ValuePtr converted_ret = nullptr; + bool converted = parse::ConvertData(obj, &converted_ret); + if (!converted) { + MS_LOG(EXCEPTION) << "attribute convert error with type:" << std::string(py::str(obj)); + } + return converted_ret; +} + +MeTensorPtr ConvertPyObjToTensor(const py::object& obj) { + MeTensorPtr me_tensor_ptr = nullptr; + if (py::isinstance(obj)) { + me_tensor_ptr = py::cast(obj); + } else if (py::isinstance(obj)) { + me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + } else if (py::isinstance(obj)) { + me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + } else if (py::isinstance(obj)) { + me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + } else if (py::isinstance(obj)) { + me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + } else if (py::isinstance(obj)) { + me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + } else { + MS_LOG(EXCEPTION) << "run op inputs type is invalid!"; + } + return me_tensor_ptr; +} + +void PynativeInfer(const PrimitivePyPtr& prim, const py::tuple& py_args, OpExecInfo* const op_exec_info) { + size_t size = py_args.size(); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < size; i++) { + ValuePtr input_value = PyAttrValue(py_args[i]); + if (py::isinstance(py_args[i])) { + args_spec_list.emplace_back(abstract::FromValueInside(input_value, true)); + } else { + args_spec_list.emplace_back(abstract::FromValueInside(input_value, false)); + } + } + AbstractBasePtr infer_res = InferOnePrim(prim, args_spec_list); + op_exec_info->abstract = infer_res; +} + +OpExecInfoPtr GenerateOpExecInfo(const py::args& args) { + if (args.size() != PY_ARGS_NUM) { + MS_LOG(ERROR) << "four args are needed by RunOp"; + return nullptr; + } + auto op_exec_info = std::make_shared(); + MS_EXCEPTION_IF_NULL(op_exec_info); + op_exec_info->op_name = py::cast(args[PY_NAME]); + if (py::isinstance(args[PY_PRIM])) { + py::module ops_mod = py::module::import("mindspore.ops.operations"); + py::object py_primitive = ops_mod.attr(op_exec_info->op_name.c_str())(); + op_exec_info->py_primitive = py::cast(py_primitive); + py::dict none_attrs = py::dict(); + op_exec_info->op_attrs = none_attrs; + } else { + PrimitivePyPtr prim = py::cast(args[PY_PRIM]); + auto pyobj = prim->GetPyObj(); + if (pyobj == nullptr) { + MS_LOG(EXCEPTION) << "pyobj is empty"; + } + py::tuple py_args = args[PY_INPUTS]; + // use python infer method + if (ignore_infer_prim.find(op_exec_info->op_name) == ignore_infer_prim.end()) { + PynativeInfer(prim, py_args, op_exec_info.get()); + } + op_exec_info->py_primitive = prim; + op_exec_info->op_attrs = py::getattr(args[PY_PRIM], "attrs"); + } + op_exec_info->op_inputs = args[PY_INPUTS]; + op_exec_info->inputs_mask = args[PY_INPUT_MASK]; + if (op_exec_info->op_inputs.size() != op_exec_info->inputs_mask.size()) { + MS_LOG(ERROR) << "" << op_exec_info->op_name << " op_inputs size not equal op_mask"; + return nullptr; + } + return op_exec_info; +} + +std::string GetSingleOpGraphInfo(const OpExecInfoPtr& op_exec_info) { + MS_EXCEPTION_IF_NULL(op_exec_info); + std::string graph_info; + MS_EXCEPTION_IF_NULL(op_exec_info->abstract); + // get input tensor info + size_t input_num = op_exec_info->op_inputs.size(); + for (size_t index = 0; index < input_num; ++index) { + if (py::isinstance(op_exec_info->op_inputs[index])) { + auto tensor_ptr = py::cast(op_exec_info->op_inputs[index]); + MS_EXCEPTION_IF_NULL(tensor_ptr); + (void)graph_info.append(tensor_ptr->GetShapeAndDataTypeInfo() + "_"); + } + } + // get prim and abstract info + (void)graph_info.append(std::to_string((uintptr_t)(op_exec_info->py_primitive.get())) + "_" + + op_exec_info->abstract->ToString()); + MS_LOG(INFO) << "graph info [" << graph_info << "]"; + return graph_info; +} + +bool SetInputsForSingleOpGraph(const OpExecInfoPtr& op_exec_info, const std::vector& inputs, + const OperatorPtr& op, std::vector* graph_input_nodes) { + MS_EXCEPTION_IF_NULL(op_exec_info); + MS_EXCEPTION_IF_NULL(graph_input_nodes); + auto op_inputs = op_exec_info->op_inputs; + std::string op_name = op_exec_info->op_name; + transform::OpAdapterPtr adapter = transform::DfGraphConvertor::FindAdapter(op_name, true); + if (adapter == nullptr) { + return false; + } + + int op_input_idx = 1; + size_t size = inputs.size(); + for (size_t i = 0; i < size; i++) { + if (inputs[i] == nullptr) { + continue; + } + auto const_op = std::make_shared(); + MS_EXCEPTION_IF_NULL(const_op); + (void)const_op->set_attr_value(*inputs[i]); + MeTensorPtr me_tensor_ptr = ConvertPyObjToTensor(op_inputs[i]); + MS_EXCEPTION_IF_NULL(me_tensor_ptr); + auto const_op_desc = + transform::TransformUtil::GetGeTensorDesc(me_tensor_ptr->shape_c(), me_tensor_ptr->data_type(), kOpFormat_NCHW); + if (const_op_desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << op_name << " ouptut descriptor failed!"; + return false; + } + auto pointer_cast_const_op = std::static_pointer_cast(const_op); + MS_EXCEPTION_IF_NULL(pointer_cast_const_op); + (void)pointer_cast_const_op->update_output_desc_y(*const_op_desc); + auto& input_map = adapter->getInputMap(); + if (input_map.find(op_input_idx) == input_map.end()) { + continue; + } + if (adapter->setInput(op, op_input_idx++, const_op)) { + MS_LOG(ERROR) << "fail to set params, index is " << op_input_idx; + return false; + } + graph_input_nodes->push_back(*const_op); + } + return true; +} + +bool BuildSingleOpGraph(const OpExecInfoPtr& op_exec_info, const std::vector& inputs, + const std::unordered_map& attrs, const GeGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(op_exec_info); + std::string op_name = op_exec_info->op_name; + auto op_inputs = op_exec_info->op_inputs; + transform::OpAdapterPtr adapter = transform::DfGraphConvertor::FindAdapter(op_name, true); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Unable to find Adapter for " << ((std::string)py::str(op_name)); + return false; + } + OperatorPtr op = adapter->generate(op_name); + MS_EXCEPTION_IF_NULL(op); + + std::vector graph_input_nodes; + // hold param nodes after setting input and output for the graph + // set input + if (!SetInputsForSingleOpGraph(op_exec_info, inputs, op, &graph_input_nodes)) { + return false; + } + // set attributes + for (auto attr : attrs) { + (void)adapter->setAttr(op, attr.first, attr.second); + } + // set default attributes + auto extra_attrs = adapter->GetExtraAttr(); + for (auto attr : extra_attrs) { + (void)adapter->setAttr(op, attr.first, attr.second); + } + // set input attributes + auto& input_attr_map = adapter->getInputAttrMap(); + for (auto& it : input_attr_map) { + if (op_inputs.size() < it.first) { + continue; + } + auto const_value = PyAttrValue(op_inputs[it.first - 1]); + if (const_value->isa()) { + continue; + } + it.second.set_attr(op, const_value); + } + // construct output data nodes + std::vector graph_outputs{*op}; + // set input and output nodes for the graph + MS_EXCEPTION_IF_NULL(graph); + (void)graph->SetInputs(graph_input_nodes).SetOutputs(graph_outputs); + MS_LOG(INFO) << "BuildSingleOpGraph done"; + return true; +} + +void ToTensorPtr(const OpExecInfoPtr op_exec_info, std::vector* const inputs) { + MS_EXCEPTION_IF_NULL(inputs); + MS_EXCEPTION_IF_NULL(op_exec_info); + auto op_inputs = op_exec_info->op_inputs; + size_t size = op_inputs.size(); + for (size_t i = 0; i < size; i++) { + if (py::isinstance(op_inputs[i])) { + inputs->emplace_back(nullptr); + continue; + } + MeTensorPtr me_tensor_ptr = ConvertPyObjToTensor(op_inputs[i]); + auto ge_tensor_ptr = transform::TransformUtil::ConvertTensor(me_tensor_ptr, kOpFormat_NCHW); + if (ge_tensor_ptr == nullptr) { + MS_LOG(EXCEPTION) << "convert inputs to GE tensor failed in op " << op_exec_info->op_name << "."; + } + // set inputs for operator to build single node graph + inputs->push_back(ge_tensor_ptr); + } +} + +PynativeStatusCode ConvertAttributes(const OpExecInfoPtr& op_exec_info, const std::vector& inputs) { + MS_EXCEPTION_IF_NULL(op_exec_info); + auto op_attrs = op_exec_info->op_attrs; + std::unordered_map attrs{}; + + for (auto& item : op_attrs) { + if (!py::isinstance(item.first)) { + MS_LOG(ERROR) << "type error in py dict convert"; + return PYNATIVE_OP_ATTRS_ERR; + } + std::string name = py::cast(item.first); + auto attr_value = PyAttrValue(py::cast(item.second)); + (void)attrs.emplace(name, attr_value); + } + + // build graph + GeGraphPtr graph = std::make_shared(op_exec_info->op_name); + if (BuildSingleOpGraph(op_exec_info, inputs, attrs, graph) == false) { + MS_LOG(ERROR) << "Fail to BuildSingleOpGraph"; + return PYNATIVE_GRAPH_GE_BUILD_ERR; + } + + // add the single op graph into the graph manager, which will be iterated by session. + transform::Status ret = + transform::DfGraphManager::GetInstance().AddGraph(SINGLE_OP_GRAPH, std::shared_ptr(graph)); + if (ret != transform::SUCCESS) { + MS_LOG(ERROR) << "Fail to AddGraph into graph manager"; + return PYNATIVE_GRAPH_MANAGER_ERR; + } + + return PYNATIVE_SUCCESS; +} + +std::vector ConvertOutputTensors(const OpExecInfoPtr& op_exec_info, + const std::vector& ge_tensors) { + std::vector outputs; + AbstractBasePtr abs_base = op_exec_info->abstract; + std::vector> shapes; + if (abs_base != nullptr && abs_base->isa()) { + auto arg_tensor = dyn_cast(abs_base); + shapes.emplace_back(arg_tensor->shape()->shape()); + outputs = transform::TransformUtil::ConvertGeTensors(ge_tensors, shapes); + return outputs; + } + if (abs_base != nullptr && abs_base->isa()) { + auto arg_tuple = dyn_cast(abs_base); + size_t len = arg_tuple->size(); + + for (size_t i = 0; i < len; i++) { + if (arg_tuple->elements()[i]->isa()) { + auto arg_tensor = dyn_cast(arg_tuple->elements()[i]); + shapes.emplace_back(arg_tensor->shape()->shape()); + } + } + outputs = transform::TransformUtil::ConvertGeTensors(ge_tensors, shapes); + return outputs; + } + for (auto& it : ge_tensors) { + auto tensor = transform::TransformUtil::ConvertGeTensor(it); + if (tensor != nullptr) { + outputs.emplace_back(tensor); + } + } + return outputs; +} + +py::object RunOpInGE(const OpExecInfoPtr& op_exec_info, PynativeStatusCode* status) { + MS_LOG(INFO) << "RunOpInGe start"; + MS_EXCEPTION_IF_NULL(op_exec_info); + MS_EXCEPTION_IF_NULL(status); + + // returns a null py::tuple on error + py::tuple err_ret(0); + auto op_name = op_exec_info->op_name; + transform::OpAdapterPtr adapter = transform::DfGraphConvertor::FindAdapter(op_name, true); + if (adapter == nullptr) { + MS_LOG(ERROR) << "Unable to find GE Adapter for " << ((std::string)py::str(op_name)); + *status = PYNATIVE_OP_NOT_IMPLEMENTED_ERR; + return std::move(err_ret); + } + + std::vector inputs{}; + ToTensorPtr(op_exec_info, &inputs); + // convert me attr to ge AttrValue + PynativeStatusCode ret = ConvertAttributes(op_exec_info, inputs); + if (ret != PYNATIVE_SUCCESS) { + *status = ret; + return std::move(err_ret); + } + // run graph + transform::RunOptions run_options; + run_options.name = SINGLE_OP_GRAPH; + std::vector ge_inputs; + std::vector ge_outputs; + transform::GraphRunnerOptions graph_runner_options; + graph_runner_options.options["ge.trainFlag"] = "1"; + auto graph_runner = std::make_shared(graph_runner_options); + transform::Status run_ret; + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + run_ret = graph_runner->RunGraph(run_options, ge_inputs, &ge_outputs); + } + if (run_ret != transform::Status::SUCCESS) { + MS_LOG(ERROR) << "GraphRunner Fails to Run Graph"; + *status = PYNATIVE_GRAPH_GE_RUN_ERR; + return std::move(err_ret); + } + + std::vector graph_outputs = ConvertOutputTensors(op_exec_info, ge_outputs); + size_t output_size = graph_outputs.size(); + py::tuple result(output_size); + for (size_t i = 0; i < output_size; i++) { + MS_EXCEPTION_IF_NULL(graph_outputs[i]); + result[i] = *graph_outputs[i]; + } + + *status = PYNATIVE_SUCCESS; + MS_LOG(INFO) << "RunOpInGe end"; + return std::move(result); +} + +py::object RunOpInVM(const OpExecInfoPtr& op_exec_info, PynativeStatusCode* status) { + MS_LOG(INFO) << "RunOpInVM start"; + + MS_EXCEPTION_IF_NULL(status); + MS_EXCEPTION_IF_NULL(op_exec_info); + MS_EXCEPTION_IF_NULL(op_exec_info->py_primitive); + auto func = op_exec_info->py_primitive->GetComputeFunction(); + if (py::isinstance(func)) { + MS_LOG(ERROR) << "VM failed to get func"; + *status = PYNATIVE_OP_NOT_IMPLEMENTED_ERR; + py::tuple err_ret(0); + return std::move(err_ret); + } + + // execute op + py::tuple result = py::make_tuple(func(*op_exec_info->op_inputs)); + *status = PYNATIVE_SUCCESS; + MS_LOG(INFO) << "RunOpInVM end"; + return std::move(result); +} + +py::object RunOpInMs(const OpExecInfoPtr& op_exec_info, PynativeStatusCode* status) { + MS_EXCEPTION_IF_NULL(op_exec_info); + MS_LOG(INFO) << "start run op[" << op_exec_info->op_name << "] with backend policy ms"; + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + ms_context->set_enable_pynative_infer(true); + std::string device_target = ms_context->device_target(); + if (device_target != kAscendDevice && device_target != kGPUDevice) { + MS_EXCEPTION(ArgumentError) << "device target [" << device_target << "] is not supported in Pynative mode"; + } + std::shared_ptr session = session::SessionFactory::Get().Create(device_target); + MS_EXCEPTION_IF_NULL(session); + session->Init(ms_context->device_id()); + + std::string graph_info = GetSingleOpGraphInfo(op_exec_info); + session->BuildOp(*op_exec_info, graph_info); + py::tuple result = session->RunOp(*op_exec_info, graph_info); + ms_context->set_enable_pynative_infer(false); + *status = PYNATIVE_SUCCESS; + return result; +} + +py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecInfoPtr op_exec_info, + PynativeStatusCode* const status) { + MS_EXCEPTION_IF_NULL(status); + py::object result; + switch (backend_policy) { + case kMsBackendGeOnly: { + // use GE only + MS_LOG(INFO) << "RunOp use GE only backend"; + result = RunOpInGE(op_exec_info, status); + break; + } + case kMsBackendVmOnly: { + // use vm only + MS_LOG(INFO) << "RunOp use VM only backend"; + result = RunOpInVM(op_exec_info, status); + break; + } + case kMsBackendGePrior: { + // use GE first, use vm when GE fails + MS_LOG(INFO) << "RunOp use GE first backend"; + result = RunOpInGE(op_exec_info, status); + if (*status != PYNATIVE_SUCCESS) { + result = RunOpInVM(op_exec_info, status); + } + break; + } + case kMsBackendVmPrior: { + // GE_VM_SILENT + // (should not use this policy) use vm first, use GE when vm fails + MS_LOG(INFO) << "RunOp use VM first backend"; + result = RunOpInVM(op_exec_info, status); + if (*status != PYNATIVE_SUCCESS) { + result = RunOpInGE(op_exec_info, status); + } + break; + } + case kMsBackendMsPrior: { + // use Ms fisrt,use others when ms failed + MS_LOG(INFO) << "RunOp use Ms first backend"; + result = RunOpInMs(op_exec_info, status); + if (*status != PYNATIVE_SUCCESS) { + MS_LOG(ERROR) << "RunOp use Ms backend failed!!!"; + } + break; + } + default: + MS_LOG(ERROR) << "No backend configed for run op"; + } + return result; +} + +py::tuple RunOp(const py::args& args) { + py::object result; + // returns a null py::tuple on error + py::tuple err_ret(0); + PynativeStatusCode status = PYNATIVE_UNKNOWN_STATE; + + OpExecInfoPtr op_exec_info = GenerateOpExecInfo(args); + MS_EXCEPTION_IF_NULL(op_exec_info); + if (op_exec_info->abstract != nullptr) { + py::dict output = abstract::ConvertAbstractToPython(op_exec_info->abstract); + if (!output["value"].is_none()) { + py::tuple value_ret(1); + value_ret[0] = output["value"]; + return value_ret; + } + } + MS_LOG(INFO) << "RunOp start, op name is: " << op_exec_info->op_name; + mindspore::parse::python_adapter::set_python_env_flag(true); + MsBackendPolicy backend_policy; +#if (!defined ENABLE_GE) + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ms") { + backend_policy = kMsBackendMsPrior; + } else { + backend_policy = kMsBackendVmOnly; + } +#else + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + ms_context->PynativeInitGe(); + backend_policy = kMsBackendGeOnly; +#endif + if (vm_operators.find(op_exec_info->op_name) != vm_operators.end()) { + backend_policy = kMsBackendVmOnly; + } + result = RunOpWithBackendPolicy(backend_policy, op_exec_info, &status); + if (status != PYNATIVE_SUCCESS) { + MS_LOG(ERROR) << "Fail to run " << op_exec_info->op_name; + return err_ret; + } + + MS_LOG(INFO) << "RunOp end"; + return result; +} +} // namespace pynative +} // namespace mindspore diff --git a/mindspore/ccsrc/pynative/pynative_execute.h b/mindspore/ccsrc/pynative/pynative_execute.h new file mode 100644 index 00000000000..e3d7649106a --- /dev/null +++ b/mindspore/ccsrc/pynative/pynative_execute.h @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_PYNATIVE_PYNATIVE_EXECUTE_H_ +#define MINDSPORE_CCSRC_PYNATIVE_PYNATIVE_EXECUTE_H_ + +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" + +#include "transform/convert.h" +#include "transform/graph_runner.h" +#include "transform/types.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace pynative { + +using MeTensor = mindspore::tensor::Tensor; +using MeTensorPtr = mindspore::tensor::TensorPtr; +using GeTensor = ge::Tensor; +using GeTensorPtr = std::shared_ptr; +using GeGraph = ge::Graph; +using GeGraphPtr = std::shared_ptr; +using GeOperator = ge::Operator; +using GeOperatorPtr = std::shared_ptr; + +namespace py = pybind11; + +enum PynativeStatusCode { + PYNATIVE_SUCCESS = 0, + PYNATIVE_OP_NOT_IMPLEMENTED_ERR = 1, + PYNATIVE_OP_INPUTS_ERR = 2, + PYNATIVE_OP_PARAMS_ERR = 3, + PYNATIVE_OP_ATTRS_ERR = 4, + PYNATIVE_GRAPH_MANAGER_ERR = 5, + PYNATIVE_GRAPH_GE_BUILD_ERR = 6, + PYNATIVE_GRAPH_GE_RUN_ERR = 7, + PYNATIVE_UNKNOWN_STATE = 0XFF +}; + +enum RunOpArgsEnum { PY_PRIM = 0, PY_NAME, PY_INPUTS, PY_INPUT_MASK, PY_ARGS_NUM }; + +struct OpExecInfo { + PrimitivePyPtr py_primitive; + std::string op_name; + AbstractBasePtr abstract; + + py::tuple op_inputs; + py::tuple inputs_mask; + py::dict op_attrs; +}; +using OpExecInfoPtr = std::shared_ptr; +OpExecInfoPtr GenerateOpExecInfo(const py::args& args); +bool BuildSingleOpGraph(const OpExecInfoPtr& op_exec_info, const std::vector& inputs, + const std::unordered_map& attrs, const GeGraphPtr& graph); + +py::object RunOpInGE(const OpExecInfoPtr& op_exec_info, PynativeStatusCode* status); + +py::object RunOpInVM(const OpExecInfoPtr& op_exec_info, PynativeStatusCode* status); + +py::tuple RunOp(const py::args& args); +} // namespace pynative +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_PYNATIVE_PYNATIVE_EXECUTE_H_ diff --git a/mindspore/ccsrc/session/CMakeLists.txt b/mindspore/ccsrc/session/CMakeLists.txt new file mode 100644 index 00000000000..2e685b04f4e --- /dev/null +++ b/mindspore/ccsrc/session/CMakeLists.txt @@ -0,0 +1,22 @@ +file(GLOB_RECURSE _SESSION_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "kernel_graph.cc" + "session_basic.cc" + "session_factory.cc" + "anf_runtime_algorithm.cc" + ) +#TODO : Not include session_context.cc +add_library(_mindspore_session_obj OBJECT ${_SESSION_ALL_SRC_FILES}) + +if(ENABLE_D) + file(GLOB_RECURSE _D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "ascend_session.cc" + ) + add_library(_mindspore_session_obj OBJECT ${_D_SRC_LIST}) +endif() + +if(ENABLE_GPU) + file(GLOB_RECURSE _C_EXPRESSION_GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "gpu_session.cc" + ) + add_library(_c_expression_gpu_session_obj OBJECT ${_C_EXPRESSION_GPU_SRC_LIST}) +endif() \ No newline at end of file diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.cc b/mindspore/ccsrc/session/anf_runtime_algorithm.cc new file mode 100644 index 00000000000..c0dca4522aa --- /dev/null +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.cc @@ -0,0 +1,861 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/anf_runtime_algorithm.h" +#include +#include +#include +#include +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "operator/ops.h" +#include "utils/utils.h" +#include "device/kernel_info.h" +#include "device/device_address.h" +#include "pre_activate/common/helper.h" +#include "kernel/kernel.h" +#include "kernel/kernel_build_info.h" +#include "common/utils.h" +#include "common/trans.h" + +namespace mindspore { +namespace session { +using abstract::AbstractTensor; +using abstract::AbstractTuple; +using device::KernelInfo; +using device::ascend::AscendDeviceAddress; +using kernel::KernelBuildInfoPtr; +using kernel::KernelMod; +using kernel::KernelModPtr; +namespace { +std::vector TransShapeToSizet(const abstract::ShapePtr &shape) { + MS_EXCEPTION_IF_NULL(shape); + std::vector shape_size_t; + std::transform(shape->shape().begin(), shape->shape().end(), std::back_inserter(shape_size_t), IntToSize); + return shape_size_t; +} +} // namespace + +KernelWithIndex AnfRuntimeAlgorithm::VisitKernel(const AnfNodePtr &anf_node, size_t index) { + MS_EXCEPTION_IF_NULL(anf_node); + if (anf_node->isa()) { + return std::make_pair(anf_node, 0); + } else if (anf_node->isa()) { + return std::make_pair(anf_node, 0); + } else if (anf_node->isa()) { + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto input0 = cnode->input(0); + MS_EXCEPTION_IF_NULL(input0); + if (IsPrimitive(input0, prim::kPrimMakeTuple)) { + auto node = cnode->input(index + IntToSize(1)); + MS_EXCEPTION_IF_NULL(node); + return VisitKernel(node, 0); + } else if (IsPrimitive(input0, prim::kPrimTupleGetItem)) { + if (cnode->inputs().size() != kTupleGetItemInputSize) { + MS_LOG(EXCEPTION) << "the node tuple_get_item must have 2 inputs!"; + } + auto input2 = cnode->input(kInputNodeOutputIndexInTupleGetItem); + MS_EXCEPTION_IF_NULL(input2); + auto value_node = input2->cast(); + MS_EXCEPTION_IF_NULL(value_node); + int item_idx = GetValue(value_node->value()); + return VisitKernel(cnode->input(kRealInputNodeIndexInTupleGetItem), IntToSize(item_idx)); + } else if (IsPrimitive(input0, prim::kPrimDepend) || IsPrimitive(input0, prim::kPrimControlDepend)) { + return VisitKernel(cnode->input(kRealInputIndexInDepend), 0); + } else { + return std::make_pair(anf_node, index); + } + } else { + MS_LOG(EXCEPTION) << "The input is invalid"; + } +} + +KernelWithIndex AnfRuntimeAlgorithm::VisitKernelWithReturnType(const AnfNodePtr &anf_node, size_t index, + const std::vector &return_types) { + MS_EXCEPTION_IF_NULL(anf_node); + for (const auto &prim_type : return_types) { + if (CheckPrimitiveType(anf_node, prim_type)) { + return std::make_pair(anf_node, index); + } + } + if (anf_node->isa()) { + return std::make_pair(anf_node, 0); + } else if (anf_node->isa()) { + return std::make_pair(anf_node, 0); + } else if (anf_node->isa()) { + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto input0 = cnode->input(0); + MS_EXCEPTION_IF_NULL(input0); + if (IsPrimitive(input0, prim::kPrimTupleGetItem)) { + if (cnode->inputs().size() != kTupleGetItemInputSize) { + MS_LOG(EXCEPTION) << "the node tuple_get_item must have 2 inputs!"; + } + auto input2 = cnode->input(kInputNodeOutputIndexInTupleGetItem); + MS_EXCEPTION_IF_NULL(input2); + auto value_node = input2->cast(); + MS_EXCEPTION_IF_NULL(value_node); + int item_idx = GetValue(value_node->value()); + return VisitKernelWithReturnType(cnode->input(kRealInputNodeIndexInTupleGetItem), IntToSize(item_idx)); + } else if (IsPrimitive(input0, prim::kPrimDepend) || IsPrimitive(input0, prim::kPrimControlDepend)) { + return VisitKernelWithReturnType(cnode->input(kRealInputIndexInDepend), 0); + } else { + return std::make_pair(anf_node, index); + } + } else { + MS_LOG(EXCEPTION) << "The input is invalid"; + } +} + +std::vector AnfRuntimeAlgorithm::GetAllOutput(const AnfNodePtr &node, + const std::vector &return_types) { + std::vector ret; + auto return_prim_type = return_types; + // if visited make_tuple should return back + return_prim_type.push_back(prim::kPrimMakeTuple); + auto item_with_index = AnfAlgo::VisitKernelWithReturnType(node, 0, return_prim_type); + if (AnfAlgo::CheckPrimitiveType(item_with_index.first, prim::kPrimMakeTuple)) { + MS_EXCEPTION_IF_NULL(item_with_index.first); + auto make_tuple = item_with_index.first->cast(); + MS_EXCEPTION_IF_NULL(make_tuple); + for (size_t i = 1; i < make_tuple->inputs().size(); i++) { + auto input_i_vector = GetAllOutput(make_tuple->input(i), return_types); + (void)std::copy(input_i_vector.begin(), input_i_vector.end(), std::back_inserter(ret)); + } + return ret; + } + ret.push_back(item_with_index.first); + return ret; +} + +AnfNodePtr AnfRuntimeAlgorithm::GetCNodePrimitiveNode(const CNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + return node->input(kAnfPrimitiveIndex); +} + +PrimitivePtr AnfRuntimeAlgorithm::GetCNodePrimitive(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto attr_input = GetCNodePrimitiveNode(cnode); + MS_EXCEPTION_IF_NULL(attr_input); + auto value_node = attr_input->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + auto primitive = value->cast(); + return primitive; +} + +bool AnfRuntimeAlgorithm::CheckPrimitiveType(const AnfNodePtr &node, const PrimitivePtr &primitive_type) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return false; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + return IsPrimitive(cnode->input(kAnfPrimitiveIndex), primitive_type); +} + +std::string AnfRuntimeAlgorithm::GetCNodeName(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + auto primitive = AnfAlgo::GetCNodePrimitive(node); + MS_EXCEPTION_IF_NULL(primitive); + return primitive->name(); + } + MS_LOG(EXCEPTION) << "Unknown anf node type " << node->DebugString(); +} + +std::string AnfRuntimeAlgorithm::GetNodeDebugString(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + return node->DebugString(); +} + +void AnfRuntimeAlgorithm::SetNodeAttr(const std::string &key, const ValuePtr &value, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "only cnode has attr,but this anf is " << node->DebugString(); + } + auto primitive = AnfAlgo::GetCNodePrimitive(node); + MS_EXCEPTION_IF_NULL(primitive); + primitive->set_attr(key, value); +} + +void AnfRuntimeAlgorithm::CopyNodeAttr(const std::string &key, const AnfNodePtr &from, const AnfNodePtr &to) { + CopyNodeAttr(key, key, from, to); +} + +void AnfRuntimeAlgorithm::CopyNodeAttr(const std::string &old_key, const std::string &new_key, const AnfNodePtr &from, + const AnfNodePtr &to) { + MS_EXCEPTION_IF_NULL(from); + MS_EXCEPTION_IF_NULL(to); + if (!from->isa() || !to->isa()) { + MS_LOG(EXCEPTION) << "only cnode has attr,but this from_anf is " << from->DebugString() << " ,to_node is " + << to->DebugString(); + } + auto from_primitive = AnfAlgo::GetCNodePrimitive(from); + MS_EXCEPTION_IF_NULL(from_primitive); + auto to_primitive = AnfAlgo::GetCNodePrimitive(to); + MS_EXCEPTION_IF_NULL(to_primitive); + to_primitive->set_attr(new_key, from_primitive->GetAttr(old_key)); +} + +void AnfRuntimeAlgorithm::CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr &to) { + MS_EXCEPTION_IF_NULL(from); + MS_EXCEPTION_IF_NULL(to); + if (!from->isa() || !to->isa()) { + MS_LOG(EXCEPTION) << "only cnode has attr,but this from_anf is " << from->DebugString() << ",to_node is " + << from->DebugString(); + } + auto from_primitive = AnfAlgo::GetCNodePrimitive(from); + MS_EXCEPTION_IF_NULL(from_primitive); + auto to_primitive = AnfAlgo::GetCNodePrimitive(to); + MS_EXCEPTION_IF_NULL(to_primitive); + (void)to_primitive->SetAttrs(from_primitive->attrs()); +} + +void AnfRuntimeAlgorithm::EraseNodeAttr(const std::string &key, const AnfNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "only cnode has attr,but this anf is " << node->DebugString(); + } + auto primitive = AnfAlgo::GetCNodePrimitive(node); + MS_EXCEPTION_IF_NULL(primitive); + primitive->EraseAttr(key); +} + +bool AnfRuntimeAlgorithm::HasNodeAttr(const std::string &key, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(WARNING) << "only cnode has attr,but this anf is " << node->DebugString(); + return false; + } + auto primitive = AnfAlgo::GetCNodePrimitive(node); + MS_EXCEPTION_IF_NULL(primitive); + return primitive->HasAttr(key); +} + +size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "only cnode has real input,but this anf is " << node->DebugString(); + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + size_t input_num = cnode->inputs().size(); + if (input_num == 0) { + MS_LOG(EXCEPTION) << "cnode inputs size can't be zero"; + } + // exclude intputs[0],which is value_node storing attr,inputs left are real input + return input_num - 1; +} + +size_t AnfRuntimeAlgorithm::GetOutputTensorNum(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + TypePtr type = node->Type(); + MS_EXCEPTION_IF_NULL(type); + if (type->isa()) { + auto tuple_type = type->cast(); + MS_EXCEPTION_IF_NULL(tuple_type); + return tuple_type->size(); + } else if (type->isa() || type->isa()) { + return 1; + } else if (type->isa()) { + return 0; + } else { + return 1; + } +} + +std::string AnfRuntimeAlgorithm::GetOutputFormat(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->GetOutputFormat(output_idx); +} + +std::string AnfRuntimeAlgorithm::GetInputFormat(const AnfNodePtr &node, size_t input_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->GetInputFormat(input_idx); +} + +std::string AnfRuntimeAlgorithm::GetPrevNodeOutputFormat(const AnfNodePtr &anf_node, size_t input_idx) { + MS_EXCEPTION_IF_NULL(anf_node); + if (!anf_node->isa()) { + MS_LOG(EXCEPTION) << "anf_node is not CNode."; + } + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode) + << "."; + } + auto node = cnode->input(input_idx + 1); + MS_EXCEPTION_IF_NULL(node); + KernelWithIndex kernel_with_index = VisitKernel(node, 0); + return AnfRuntimeAlgorithm::GetOutputFormat(kernel_with_index.first, kernel_with_index.second); +} + +std::vector AnfRuntimeAlgorithm::GetOutputInferShape(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + abstract::BaseShapePtr base_shape = node->Shape(); + MS_EXCEPTION_IF_NULL(base_shape); + if (base_shape->isa() && output_idx == 0) { + return TransShapeToSizet(base_shape->cast()); + } else if (base_shape->isa()) { + auto tuple_shape = base_shape->cast(); + MS_EXCEPTION_IF_NULL(tuple_shape); + if (output_idx >= tuple_shape->size()) { + MS_LOG(EXCEPTION) << "Output index " << output_idx << "is larger than output number " << tuple_shape->size() + << "."; + } + auto b_shp = (*tuple_shape)[output_idx]; + if (b_shp->isa()) { + return TransShapeToSizet(b_shp->cast()); + } else if (b_shp->isa()) { + return std::vector(); + } else { + MS_LOG(EXCEPTION) << "The output type of ApplyKernel should be a NoShape , ArrayShape or a TupleShape, but it is " + << base_shape->ToString(); + } + } else if (base_shape->isa()) { + return std::vector(); + } + MS_LOG(EXCEPTION) << "The output type of ApplyKernel should be a NoShape , ArrayShape or a TupleShape, but it is " + << base_shape->ToString(); +} + +std::vector AnfRuntimeAlgorithm::GetPrevNodeOutputInferShape(const AnfNodePtr &node, size_t input_idx) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "anf_node is not CNode."; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode) + << "."; + } + auto input_node = cnode->input(input_idx + 1); + KernelWithIndex kernel_with_index = VisitKernel(input_node, 0); + return AnfRuntimeAlgorithm::GetOutputInferShape(kernel_with_index.first, kernel_with_index.second); +} + +std::vector AnfRuntimeAlgorithm::GetOutputDeviceShape(const AnfNodePtr &node, size_t output_idx) { + auto format = GetOutputFormat(node, output_idx); + auto infer_shape = GetOutputInferShape(node, output_idx); + // if format is default_format or NC1KHKWHWC0,device shape = original shape + if (format == kOpFormat_DEFAULT || format == kOpFormat_NC1KHKWHWC0) { + return infer_shape; + } + // scalar shape + if (infer_shape.empty()) { + return infer_shape; + } + if (format == kOpFormat_FRAC_NZ) { + return trans::TransShapeToDevice(infer_shape, format); + } + // else trans infer shape to 4d and then calculate device shape + return trans::TransShapeToDevice(trans::TransShapeTo4d(infer_shape), format); +} + +std::vector AnfRuntimeAlgorithm::GetInputDeviceShape(const AnfNodePtr &node, size_t input_idx) { + auto format = GetInputFormat(node, input_idx); + auto infer_shape = GetPrevNodeOutputInferShape(node, input_idx); + // if format is default_format or NC1KHKWHWC0,device shape = original shape + if (format == kOpFormat_DEFAULT || format == kOpFormat_NC1KHKWHWC0) { + return infer_shape; + } + if (infer_shape.empty()) { + return infer_shape; + } + if (format == kOpFormat_FRAC_NZ) { + return trans::TransShapeToDevice(infer_shape, format); + } + // else trans infer shape to 4d and then calculate device shape + return trans::TransShapeToDevice(trans::TransShapeTo4d(infer_shape), format); +} + +std::vector AnfRuntimeAlgorithm::GetInputReshapeType(const AnfNodePtr &node, size_t input_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + std::vector result; + if (!build_info->GetInputReshapeType(input_idx, &result)) { + MS_LOG(EXCEPTION) << "filed to ge the node's[ " << node->DebugString() << "] reshape type !"; + } + return result; +} + +std::vector AnfRuntimeAlgorithm::GetOutputReshapeType(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + std::vector result; + if (!build_info->GetOutputReshapeType(output_idx, &result)) { + MS_LOG(EXCEPTION) << "filed to ge the node's[ " << node->DebugString() << "] reshape type !"; + } + return result; +} + +TypeId AnfRuntimeAlgorithm::GetOutputInferDataType(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + TypePtr type_ptr = node->Type(); + MS_EXCEPTION_IF_NULL(type_ptr); + if (type_ptr->isa() && output_idx == 0) { + auto tensor_ptr = type_ptr->cast(); + MS_EXCEPTION_IF_NULL(tensor_ptr); + TypePtr elem = tensor_ptr->element(); + MS_EXCEPTION_IF_NULL(elem); + return elem->type_id(); + } else if (type_ptr->isa()) { + auto tuple_ptr = type_ptr->cast(); + MS_EXCEPTION_IF_NULL(tuple_ptr); + if (output_idx >= tuple_ptr->size()) { + MS_LOG(EXCEPTION) << "Output index " << output_idx << " must be less than output number " << tuple_ptr->size(); + } + auto tuple_i = (*tuple_ptr)[output_idx]; + MS_EXCEPTION_IF_NULL(tuple_i); + if (tuple_i->isa()) { + auto tensor_ptr = tuple_i->cast(); + MS_EXCEPTION_IF_NULL(tensor_ptr); + TypePtr elem = tensor_ptr->element(); + MS_EXCEPTION_IF_NULL(elem); + return elem->type_id(); + } else if (tuple_i->isa()) { + return tuple_i->type_id(); + } else { + MS_LOG(EXCEPTION) << "Not support type " << tuple_i->ToString(); + return tuple_i->type_id(); + } + } else if (type_ptr->isa()) { + return type_ptr->type_id(); + } + return type_ptr->type_id(); +} + +TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputInferDataType(const AnfNodePtr &node, size_t input_idx) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + MS_LOG(EXCEPTION) << node->DebugString() << "is not a CNode"; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode); + } + auto input_node = cnode->input(input_idx + 1); + KernelWithIndex kernel_with_index = VisitKernel(input_node, 0); + return AnfRuntimeAlgorithm::GetOutputInferDataType(kernel_with_index.first, kernel_with_index.second); +} + +TypeId AnfRuntimeAlgorithm::GetOutputDeviceDataType(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->GetOutputDeviceType(output_idx); +} + +TypeId AnfRuntimeAlgorithm::GetInputDeviceDataType(const AnfNodePtr &node, size_t input_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->GetInputDeviceType(input_idx); +} + +TypeId AnfRuntimeAlgorithm::GetPrevNodeOutputDeviceDataType(const AnfNodePtr &anf_node, size_t input_idx) { + if (!anf_node->isa()) { + MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode."; + } + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode); + } + auto node = cnode->input(input_idx + 1); + MS_EXCEPTION_IF_NULL(node); + KernelWithIndex kernel_with_index = VisitKernel(node, 0); + return AnfRuntimeAlgorithm::GetOutputDeviceDataType(kernel_with_index.first, kernel_with_index.second); +} + +// get output device addr of anf_node +const DeviceAddress *AnfRuntimeAlgorithm::GetOutputAddr(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + if (opt::IsNopNode(node)) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() == 2) { + return AnfRuntimeAlgorithm::GetPrevNodeOutputAddr(cnode, 0); + } else { + MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node"; + } + } + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto addr = kernel_info->GetOutputAddr(output_idx); + if (addr == nullptr) { + MS_LOG(EXCEPTION) << "output_idx " << output_idx << " of node " << node->DebugString() + << " output addr is not exist"; + } + return addr; +} + +DeviceAddressPtr AnfRuntimeAlgorithm::GetMutableOutputAddr(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + if (opt::IsNopNode(node)) { + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() == 2) { + return AnfRuntimeAlgorithm::GetPrevNodeMutableOutputAddr(cnode, 0); + } else { + MS_LOG(EXCEPTION) << node->DebugString() << "Invalid nop node."; + } + } + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto addr = kernel_info->GetMutableOutputAddr(output_idx); + if (addr == nullptr) { + MS_LOG(EXCEPTION) << "output_idx" << output_idx << " of node " << node->DebugString() + << " output addr is not exist"; + } + return addr; +} + +// get output device addr of anf_node +bool AnfRuntimeAlgorithm::OutputAddrExist(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->OutputAddrExist(output_idx); +} + +const DeviceAddress *AnfRuntimeAlgorithm::GetPrevNodeOutputAddr(const AnfNodePtr &anf_node, size_t input_idx) { + if (!anf_node->isa()) { + MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf node is not a CNode"; + } + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode); + } + auto node = cnode->input(input_idx + 1); + MS_EXCEPTION_IF_NULL(node); + KernelWithIndex kernel_with_index = VisitKernel(node, 0); + return AnfRuntimeAlgorithm::GetOutputAddr(kernel_with_index.first, kernel_with_index.second); +} + +DeviceAddressPtr AnfRuntimeAlgorithm::GetPrevNodeMutableOutputAddr(const AnfNodePtr &anf_node, size_t input_idx) { + if (!anf_node->isa()) { + MS_LOG(EXCEPTION) << anf_node->DebugString() << "anf_node is not CNode."; + } + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (input_idx + 1 >= cnode->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << GetInputTensorNum(cnode); + } + auto node = cnode->input(input_idx + 1); + MS_EXCEPTION_IF_NULL(node); + KernelWithIndex kernel_with_index = VisitKernel(node, 0); + return AnfRuntimeAlgorithm::GetMutableOutputAddr(kernel_with_index.first, kernel_with_index.second); +} + +// set output device addr of anf_node +void AnfRuntimeAlgorithm::SetOutputAddr(const DeviceAddressPtr &addr, size_t output_idx, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + if (!kernel_info->SetOutputAddr(addr, output_idx)) { + MS_LOG(EXCEPTION) << "node " << node->DebugString() << "set adr" << output_idx << " fail"; + } +} + +// set workspace device addr of anf_node +void AnfRuntimeAlgorithm::SetWorkspaceAddr(const DeviceAddressPtr &addr, size_t output_idx, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + if (!kernel_info->SetWorkspaceAddr(addr, output_idx)) { + MS_LOG(EXCEPTION) << "node " << node->DebugString() << "set adr" << output_idx << " fail"; + } +} + +// get workspace device addr of anf_node +DeviceAddress *AnfRuntimeAlgorithm::GetWorkspaceAddr(const AnfNodePtr &node, size_t output_idx) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto addr = kernel_info->GetWorkspaceAddr(output_idx); + if (addr == nullptr) { + MS_LOG(EXCEPTION) << "output_idx " << output_idx << " of node " << node->DebugString() + << "] workspace addr is not exist"; + } + return addr; +} + +// set infer shapes and types of anf node +void AnfRuntimeAlgorithm::SetOutputInferTypeAndShape(const std::vector &types, + const std::vector> &shapes, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + if (types.size() != shapes.size()) { + MS_LOG(EXCEPTION) << "types size " << types.size() << "should be same with shapes size " << shapes.size(); + } + if (shapes.empty()) { + MS_LOG(EXCEPTION) << "Illegal empty output_types_shapes"; + } else if (shapes.size() == 1) { + // single output handle + std::vector shape_int; + std::transform(shapes[0].begin(), shapes[0].end(), std::back_inserter(shape_int), SizeToInt); + auto abstract = std::make_shared(TypeIdToType(types[0]), shape_int); + node->set_abstract(abstract); + } else { + // mutiple output handle + std::vector abstract_list; + for (size_t i = 0; i < types.size(); ++i) { + std::vector shape_int; + std::transform(shapes[i].begin(), shapes[i].end(), std::back_inserter(shape_int), SizeToInt); + abstract_list.push_back(std::make_shared(TypeIdToType(types[i]), shape_int)); + } + auto abstract_tuple = std::make_shared(abstract_list); + node->set_abstract(abstract_tuple); + } +} +// copy a abstract of a node to another node +void AnfRuntimeAlgorithm::CopyAbstract(const AnfNodePtr &from_node, AnfNode *to_node) { + to_node->set_abstract(from_node->abstract()); +} + +// get KernelBuildType of node ,such as ATT,RT,FWK and so on +KernelType AnfRuntimeAlgorithm::GetKernelType(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + // select_kernel_build_info() has checked whether return pointer is null + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->kernel_type(); +} + +kernel::Processor AnfRuntimeAlgorithm::GetProcessor(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->processor(); +} + +kernel::FusionType AnfRuntimeAlgorithm::GetFusionType(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + auto build_info = kernel_info->select_kernel_build_info(); + MS_EXCEPTION_IF_NULL(build_info); + return build_info->fusion_type(); +} + +// set select kernel_build_info +void AnfRuntimeAlgorithm::SetSelectKernelBuildInfo(const KernelBuildInfoPtr &select_kernel_build_info, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->set_select_kernel_build_info(select_kernel_build_info); +} + +// get select kernel_build_info +KernelBuildInfoPtr AnfRuntimeAlgorithm::GetSelectKernelBuildInfo(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->GetMutableSelectKernelBuildInfo(); +} + +// get kernelMode +KernelMod *AnfRuntimeAlgorithm::GetKernelMod(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->MutableKernelMod(); +} + +// set kernel mod +void AnfRuntimeAlgorithm::SetKernelMod(const KernelModPtr &kernel_mod, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + kernel_info->set_kernel_mod(kernel_mod); +} + +bool AnfRuntimeAlgorithm::IsRealKernel(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + // parameter and value node is not a real kernel too + if (!node->isa()) { + return true; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().empty()) { + MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << node->DebugString(); + } + auto input = cnode->inputs()[0]; + bool is_virtual_node = IsPrimitive(input, prim::kPrimImageSummary) || IsPrimitive(input, prim::kPrimScalarSummary) || + IsPrimitive(input, prim::kPrimTensorSummary) || IsPrimitive(input, prim::kPrimMakeTuple) || + IsPrimitive(input, prim::kPrimStateSetItem) || IsPrimitive(input, prim::kPrimDepend) || + IsPrimitive(input, prim::kPrimTupleGetItem) || IsPrimitive(input, prim::kPrimControlDepend) || + IsPrimitive(input, prim::kPrimReturn); + return !is_virtual_node; +} + +bool AnfRuntimeAlgorithm::IsRealCNodeKernel(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + // parameter and value node is not a real cnode kernel + if (!node->isa()) { + return false; + } + // return considered as a real node + if (CheckPrimitiveType(node, prim::kPrimReturn)) { + return true; + } + return IsRealKernel(node); +} + +bool AnfRuntimeAlgorithm::IsParameterWeight(const ParameterPtr &node) { + MS_EXCEPTION_IF_NULL(node); + return node->has_default(); +} + +void AnfRuntimeAlgorithm::SetStreamId(uint32_t stream_id, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + kernel_info->set_stream_id(stream_id); +} + +uint32_t AnfRuntimeAlgorithm::GetStreamId(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->stream_id(); +} + +void AnfRuntimeAlgorithm::SetStreamDistinctionLabel(uint32_t stream_label, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + kernel_info->set_stream_distinction_label(stream_label); +} + +uint32_t AnfRuntimeAlgorithm::GetStreamDistinctionLabel(const AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->stream_distinction_label(); +} + +void AnfRuntimeAlgorithm::SetGraphId(uint32_t graph_id, AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + kernel_info->set_graph_id(graph_id); +} + +uint32_t AnfRuntimeAlgorithm::GetGraphId(const AnfNode *node) { + MS_EXCEPTION_IF_NULL(node); + auto kernel_info = node->kernel_info(); + MS_EXCEPTION_IF_NULL(kernel_info); + return kernel_info->graph_id(); +} + +bool AnfRuntimeAlgorithm::IsTupleOutput(const AnfNodePtr &anf) { + MS_EXCEPTION_IF_NULL(anf); + TypePtr type = anf->Type(); + MS_EXCEPTION_IF_NULL(type); + return type->isa(); +} + +AnfNodePtr AnfRuntimeAlgorithm::GetInputNode(const CNodePtr &node, size_t index) { + MS_EXCEPTION_IF_NULL(node); + auto get_input_index = index + 1; + if (index + 1 > node->inputs().size()) { + MS_LOG(EXCEPTION) << "Input index size " << get_input_index << "but the node input size just" + << node->inputs().size(); + } + // input 0 is primitive node + return node->input(get_input_index); +} + +bool AnfRuntimeAlgorithm::IsFeatureMapInput(const AnfNodePtr &node, size_t input_index) { + if (!node->isa()) { + MS_LOG(EXCEPTION) << "Cannot input a parameter or a valuenode to charge it's input if is a feature"; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto input_node = cnode->input(input_index + 1); + auto node_with_index = VisitKernel(input_node, 0); + MS_EXCEPTION_IF_NULL(node_with_index.first); + if (node_with_index.first->isa()) { + return false; + } + if (node_with_index.first->isa()) { + return !AnfAlgo::IsParameterWeight(node_with_index.first->cast()); + } + return true; +} + +size_t AnfRuntimeAlgorithm::GetRealInputIndex(const mindspore::AnfNodePtr &anf_node, const size_t cur_index) { + MS_EXCEPTION_IF_NULL(anf_node); + static std::map> spec_node_list = { + {prim::kPrimConv2DBackpropInput->name(), {{0, 1}, {1, 0}}}, + {prim::kPrimConv2DBackpropFilter->name(), {{0, 1}, {1, 0}}}, + {prim::kPrimLogSoftmaxGrad->name(), {{0, 1}, {1, 0}}}, + {prim::kPrimLayerNormGrad->name(), {{0, 1}, {1, 0}, {2, 2}, {3, 3}, {4, 4}}}, + {prim::kPrimLayerNormBetaGammaBackprop->name(), {{0, 1}, {1, 0}, {2, 2}, {3, 3}}}, + {prim::kPrimLayerNormXBackprop->name(), {{0, 1}, {1, 0}, {2, 2}, {3, 3}, {4, 4}}}, + {prim::kPrimMinimumGrad->name(), {{0, 2}, {1, 0}, {2, 1}}}, + {prim::kPrimMaximumGrad->name(), {{0, 2}, {1, 0}, {2, 1}}}}; + size_t ret = cur_index; + auto node_name = AnfAlgo::GetCNodeName(anf_node); + if (AnfAlgo::GetKernelType(anf_node) == TBE_KERNEL) { + auto find = spec_node_list.find(node_name); + if (find != spec_node_list.end()) { + ret = find->second[cur_index]; + MS_LOG(INFO) << "real input index change to" << ret << ", node name:" << node_name; + } + } + return ret; +} + +void AnfRuntimeAlgorithm::SetNodeInput(const CNodePtr &node, const AnfNodePtr &input_node, size_t index) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(input_node); + node->set_input(index + 1, input_node); +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.h b/mindspore/ccsrc/session/anf_runtime_algorithm.h new file mode 100644 index 00000000000..60d373d5ad7 --- /dev/null +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.h @@ -0,0 +1,173 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_SESSION_ANF_RUNTIME_ALGORITHM_H +#define MINDSPORE_CCSRC_SESSION_ANF_RUNTIME_ALGORITHM_H +#include +#include +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "ir/dtype.h" +#include "ir/base.h" +#include "ir/primitive.h" +#include "device/device_address.h" +#include "kernel/kernel.h" +#include "kernel/kernel_build_info.h" +#include "operator/ops.h" + +namespace mindspore { +namespace session { +using AnfVisitFuncion = std::function; +using KernelWithIndex = std::pair; +class AnfRuntimeAlgorithm { + public: + // get input_anf_node's real kernel by recurse + static KernelWithIndex VisitKernel(const AnfNodePtr &input_anf_node, size_t output_index); + static KernelWithIndex VisitKernelWithReturnType(const AnfNodePtr &input_anf_node, size_t output_index, + const std::vector &return_types = { + prim::kPrimMakeTuple}); + static std::vector GetAllOutput(const AnfNodePtr &node, + const std::vector &return_types = {}); + // get cnode primitive + static AnfNodePtr GetCNodePrimitiveNode(const CNodePtr &node); + static void SetNodeInput(const CNodePtr &node, const AnfNodePtr &input_node, size_t index); + static PrimitivePtr GetCNodePrimitive(const AnfNodePtr &node); + // check whether anf node is a node of 'primitive_type',such as make_tuple is a cnode of kPrimMakeTuple + static bool CheckPrimitiveType(const AnfNodePtr &node, const PrimitivePtr &primitive_type); + // get kernel_name of anf node + static std::string GetCNodeName(const AnfNodePtr &node); + // get detail info of anf node + static std::string GetNodeDebugString(const AnfNodePtr &node); + // get attr of anf node + template + static T GetNodeAttr(const AnfNodePtr &node, const std::string &key) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + std::string node_debug_log = node->DebugString(); + MS_LOG(EXCEPTION) << "only cnode has attr,but this anf is " << node_debug_log.c_str(); + } + auto primitive = GetCNodePrimitive(node); + MS_EXCEPTION_IF_NULL(primitive); + return GetValue(primitive->GetAttr(key)); + } + static bool IsTupleOutput(const AnfNodePtr &anf); + // set attr of anf node + static void SetNodeAttr(const std::string &key, const ValuePtr &value, const AnfNodePtr &node); + // set attr of key from 'from' node to 'to' node + static void CopyNodeAttr(const std::string &key, const AnfNodePtr &from, const AnfNodePtr &to); + // set a new key for attr from 'from' node to 'to' node + static void CopyNodeAttr(const std::string &old_key, const std::string &new_key, const AnfNodePtr &from, + const AnfNodePtr &to); + // set all attrs from 'from' node to 'to' node + static void CopyNodeAttrs(const AnfNodePtr &from, const AnfNodePtr &to); + // check whether a cnode has the specified attr. + static bool HasNodeAttr(const std::string &key, const AnfNodePtr &node); + // delete attr of anf node + static void EraseNodeAttr(const std::string &key, AnfNodePtr node); + // get the num of input real_kernel(which can be build and run in device) + static size_t GetInputTensorNum(const AnfNodePtr &node); + // get the num of output real_kernel(which can be build and run in device) + static size_t GetOutputTensorNum(const AnfNodePtr &node); + // get output format select of anf node + static std::string GetOutputFormat(const AnfNodePtr &node, size_t output_idx); + // get input format select of anf node + static std::string GetInputFormat(const AnfNodePtr &node, size_t input_idx); + // get output format from prev node,input_index is the input index of current node related to prev node + static std::string GetPrevNodeOutputFormat(const AnfNodePtr &node, size_t input_idx); + // get output shapes inferred by ME from input nodes. + static std::vector GetOutputInferShape(const AnfNodePtr &node, size_t output_idx); + // get input shapes inferred by ME from input nodes. + static std::vector GetPrevNodeOutputInferShape(const AnfNodePtr &node, size_t input_idx); + // get output shapes which will built and run in device + static std::vector GetOutputDeviceShape(const AnfNodePtr &node, size_t output_idx); + // get input shapes which will built and run in device + static std::vector GetInputDeviceShape(const AnfNodePtr &node, size_t input_idx); + static std::vector GetInputReshapeType(const AnfNodePtr &node, size_t output_idx); + static std::vector GetOutputReshapeType(const AnfNodePtr &node, size_t output_idx); + // get output data type inferred by ME of anf node + static TypeId GetOutputInferDataType(const AnfNodePtr &node, size_t output_idx); + // get output original data type from prev node,input_index is the input index of current node related to prev node + static TypeId GetPrevNodeOutputInferDataType(const AnfNodePtr &node, size_t input_idx); + // get output select data typpe of anf node + static TypeId GetOutputDeviceDataType(const AnfNodePtr &node, size_t output_idx); + // get input select data type of anf node + static TypeId GetInputDeviceDataType(const AnfNodePtr &node, size_t input_idx); + // get output select data type from prev node,input_index is the input index of current node related to prev node + static TypeId GetPrevNodeOutputDeviceDataType(const AnfNodePtr &node, size_t input_idx); + // get output device addr of anf_node + static const DeviceAddress *GetOutputAddr(const AnfNodePtr &node, size_t output_idx); + // get mutable output device addr of anf_node + static DeviceAddressPtr GetMutableOutputAddr(const AnfNodePtr &node, size_t output_idx); + // check whether output addr is exist or not + static bool OutputAddrExist(const AnfNodePtr &node, size_t output_idx); + // get address from prev node,input_index is the input index of current node related to prev node + static const DeviceAddress *GetPrevNodeOutputAddr(const AnfNodePtr &node, size_t input_idx); + static DeviceAddressPtr GetPrevNodeMutableOutputAddr(const AnfNodePtr &anf_node, size_t input_idx); + // set output device addr of anf_node + static void SetOutputAddr(const DeviceAddressPtr &addr, size_t output_idx, AnfNode *node); + // set workspace device addr of anf_node + static void SetWorkspaceAddr(const DeviceAddressPtr &addr, size_t output_idx, AnfNode *node); + // get workspace device addr of anf_node + static DeviceAddress *GetWorkspaceAddr(const AnfNodePtr &node, size_t output_idx); + // set infer shapes and types of anf node + static void SetOutputInferTypeAndShape(const std::vector &types, + const std::vector> &shapes, AnfNode *node); + static void CopyAbstract(const AnfNodePtr &from_node, AnfNode *to_node); + // get KernelBuildType of node ,such as ATT,RT,FWK and so on + static KernelType GetKernelType(const AnfNodePtr &node); + // get processor type:AICORE,AICPU... + static kernel::Processor GetProcessor(const AnfNodePtr &node); + // get fusion type:AICORE,AICPU... + static kernel::FusionType GetFusionType(const AnfNodePtr &node); + // set select kernel_build_info + static void SetSelectKernelBuildInfo(const kernel::KernelBuildInfoPtr &select_kernel_build_info, AnfNode *node); + // get select kernel_build_info + static kernel::KernelBuildInfoPtr GetSelectKernelBuildInfo(const AnfNodePtr &node); + // get kernelMode + static kernel::KernelMod *GetKernelMod(const AnfNodePtr &node); + // set kernel mod + static void SetKernelMod(const kernel::KernelModPtr &kernel_mod, AnfNode *node); + // checkout whether the anf node is a real kernel that can run on device,parameter and constant is real kernel too + static bool IsRealKernel(const AnfNodePtr &node); + // checkout whether the anf node is a real kernel that is a cnode and can run on device + static bool IsRealCNodeKernel(const AnfNodePtr &node); + // check parameter is weight or data + static bool IsParameterWeight(const ParameterPtr &node); + // set stream id of kernel,which will be set in stream assign and be used in stream generate + static void SetStreamId(uint32_t stream_id, AnfNode *node); + // get stream id + static uint32_t GetStreamId(const AnfNodePtr &node); + // set stream distinction label to distinguish different ops in different streams + static void SetStreamDistinctionLabel(uint32_t stream_label, AnfNode *node); + // get stream distinction label + static uint32_t GetStreamDistinctionLabel(const AnfNode *node); + // set graph id + static void SetGraphId(uint32_t graph_id, AnfNode *node); + // get graph id + static uint32_t GetGraphId(const AnfNode *node); + static AnfNodePtr GetInputNode(const CNodePtr &node, size_t index); + static bool IsFeatureMapInput(const AnfNodePtr &node, size_t input_index); + // get real input index for some tbe ops which input order is different between me and tbe impl + static size_t GetRealInputIndex(const AnfNodePtr &anf_node, const size_t cur_index); +}; +} // namespace session +using AnfAlgo = session::AnfRuntimeAlgorithm; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_SESSION_ANF_RUNTIME_ALGORITHM_H diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc new file mode 100644 index 00000000000..1a29450313d --- /dev/null +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -0,0 +1,938 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/ascend_session.h" +#include +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/anf.h" +#include "device/kernel_runtime.h" +#include "device/ascend/kernel_select_ascend.h" +#include "device/ascend/kernel_build_ascend.h" +#include "device/ascend/ascend_kernel_runtime.h" +#include "device/ascend/ascend_device_address.h" +#include "pre_activate/ascend/ascend_backend_optimization.h" +#include "device/kernel_adjust.h" +#include "device/ascend/ascend_stream_assign.h" +#include "predict/predict.h" +#include "session/anf_runtime_algorithm.h" +#include "ir/scalar.h" +#include "debug/anf_ir_dump.h" +#include "debug/anf_ir_utils.h" +#include "common/utils.h" +#include "pre_activate/common/helper.h" +#include "device/kernel_runtime_manager.h" +#include "kernel/tbe/tbe_python_funcs.h" + +namespace mindspore { +namespace session { +const size_t kInvalidIndex = SIZE_MAX; +namespace { +void DumpGraphExeOrder(const std::vector &execution_order) { + MS_LOG(INFO) << "Dump execution_order size " << execution_order.size(); + MS_LOG(INFO) << "[index][stream_label][graph_id][node string]"; + int i = 0; + for (auto &cnode : execution_order) { + MS_EXCEPTION_IF_NULL(cnode); + MS_LOG(INFO) << "[ " << i << "]" + << "[" << AnfAlgo::GetStreamDistinctionLabel(cnode.get()) << "]" + << "[" << AnfAlgo::GetGraphId(cnode.get()) << "]" + << "[" << cnode->DebugString() << "]"; + i++; + } +} + +void DumpGraphInputArgs(const VectorRef &args) { + MS_LOG(INFO) << "args size[%lu]" << args.size(); + for (size_t i = 0; i < args.size(); i++) { + if (utils::isa(args[i])) { + auto anf = utils::cast(args[i]); + MS_EXCEPTION_IF_NULL(anf); + MS_LOG(INFO) << "Parameter arg" << i << " = [%s]" << anf->DebugString(); + } else if (utils::isa(args[i])) { + auto value = utils::cast(args[i]); + MS_EXCEPTION_IF_NULL(value); + MS_LOG(INFO) << "Tensor arg" << i << " = " << value->ToString(); + } else { + MS_LOG(INFO) << "Unknonwn arg" << i << " = " << args[i].ToString(); + } + } +} + +void SetStreamDistinctionLabel(const KernelGraphPtr &graph, uint32_t label, bool is_override) { + MS_EXCEPTION_IF_NULL(graph); + for (auto &node : graph->execution_order()) { + if (is_override || AnfAlgo::GetStreamDistinctionLabel(node.get()) == kInvalidDistincLabel) { + MS_EXCEPTION_IF_NULL(node); + AnfAlgo::SetStreamDistinctionLabel(label, node.get()); + } + } +} + +GraphId GetDistinctionLabel(const KernelGraphPtr &graph) { + MS_EXCEPTION_IF_NULL(graph); + // if graph is empty,use graph id as distinction label + if (graph->execution_order().empty()) { + return graph->graph_id(); + } + // else use first node of execution order as label + return AnfAlgo::GetStreamDistinctionLabel(graph->execution_order()[0].get()); +} +} // namespace + +GraphId AscendSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) { + MS_LOG(INFO) << "start"; + auto graph_id = graph_sum_; + // construct graph,if construct successs,graph_sum_ + 1 + auto graph = ConstructKernelGraph(lst, outputs); + MS_EXCEPTION_IF_NULL(graph); + opt::AscendBackendIRFusionOptimization(graph); + // select kernel build info + SelectKernel(*graph); + // convert kernel Graph to model + predictmodel::StepConvertGraph(graph); + // optimize graph + HardwareOptimize(graph); + // init runtime resource + InitRuntimeResource(); + // assign static memory of parameters + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + runtime_instance->AssignStaticMemoryInput(graph.get()); + MS_LOG(INFO) << "Compile graph " << graph_id << " success"; + return graph_id; +} + +void AscendSession::BuildGraph(GraphId graph_id) { + MS_LOG(INFO) << "start"; + auto graph = GetGraph(graph_id); + MS_EXCEPTION_IF_NULL(graph); + // multiple graph handle + if (graph_id == final_graph_id_) { + if (!graph->executable()) { + return; + } + // merge child graph + MergeGraphExecOrder(); + } else { + // set the distinciton label of single graph + SetStreamDistinctionLabel(GetGraph(graph_id), graph_id, false); + } + // adjust execution order because merge child graph and other special operations + AdjustKernel(graph); + // Assign streams for control sink and hccl and so on + AssignStream(graph); + + device::KernelAdjust::GetInstance().Profiling(graph); + // build kernel if node is cnode + BuildKernel(graph); + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->precompile_only()) { + MS_LOG(INFO) << "Precompile only, stop in build kernel step"; + } else { + // alloc memeory,include static memory and dynamic memory + MemoryAlloc(graph.get()); + // generate task info for task sink mode + GenerateTaskInfo(graph); + // load task info to device if it is sink mode + LoadTask(graph); + } + MS_LOG(INFO) << "end"; +} + +void AscendSession::RunGraph(const GraphId &graph_id, const std::vector &inputs, + VectorRef *const outputs) { + MS_LOG(INFO) << "start"; + auto kernel_graph = GetGraph(graph_id); + MS_EXCEPTION_IF_NULL(kernel_graph); + // if no child graph exist and no anf output exist + if (!kernel_graph->executable()) { + MS_LOG(INFO) << "no child graph but has anf output"; + UpdateOutputs(kernel_graph, outputs, inputs); + return; + } + // load input data from user input + LoadInputData(kernel_graph, inputs); + // convert inputs to model + predictmodel::StepConvertWeight(inputs); + { + py::gil_scoped_release release; + // run task on device + ExecTask(kernel_graph); + } + // get result from device + UpdateOutputs(kernel_graph, outputs, inputs); + // summary + Summary(kernel_graph.get()); + // dump used for debug + Dump(kernel_graph); + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::RunOpHardwareOptimize(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + // data layout optimization + opt::AscendDataLayout(kernel_graph); + // mixed precision optimization + opt::AscendMixPrecision(kernel_graph); + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::RunOpExecTask(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + bool ret_ok = runtime_instance->LaunchKernel(kernel_graph.get()); + if (!ret_ok) { + MS_LOG(EXCEPTION) << "run task error!"; + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::BuildOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) { + MS_LOG(INFO) << "Build op " << op_run_info.op_name << " start !"; + // construct graph include one op + auto graph = ConstructSingleOpGraph(op_run_info); + MS_EXCEPTION_IF_NULL(graph); + opt::RunOpAscendBackendIRFusionOptimization(graph); + // kernel select + SelectKernel(*graph); + // optimize + RunOpHardwareOptimize(graph); + // init runtime resource + InitRuntimeResource(); + // build kernel + RunOpAdjustKernel(graph); + BuildKernel(graph); + run_op_graphs_[graph_info] = graph; +} + +py::tuple AscendSession::RunOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) { + auto graph = run_op_graphs_[graph_info]; + MS_EXCEPTION_IF_NULL(graph); + MS_LOG(INFO) << "Run op " << op_run_info.op_name << " start!"; + // malloc mem + std::vector input_tensors = {}; + std::vector tensors_mask = {}; + ToTensorPtr(op_run_info, &input_tensors, &tensors_mask); + RunOpMemoryAlloc(input_tensors, graph.get()); + // load input data to device + LoadInputData(graph, input_tensors); + // run op + RunOpExecTask(graph); + // get output + VectorRef outputs; + UpdateOutputs(graph, &outputs, input_tensors); + // trans output to tuple + auto output_tensors = TransformBaseRefListToTuple(outputs); + if (!utils::isa(output_tensors) || + !py::isinstance(utils::cast(output_tensors).object_)) { + MS_LOG(EXCEPTION) << "The output tensors should be a tuple !"; + } + py::object tuple_obj = utils::cast(output_tensors).object_; + py::tuple tuple_tensors = py::cast(tuple_obj); + run_op_graphs_.clear(); + MS_LOG(INFO) << "Run op " << op_run_info.op_name << " finish!"; + return tuple_tensors; +} + +// compile graph steps +void AscendSession::SelectKernel(const KernelGraph &kernel_graph) const { + MS_LOG(INFO) << "start !"; + for (const auto &cnode : kernel_graph.execution_order()) { + device::ascend::SelectKernelInfo(cnode); + MS_LOG(INFO) << "select ApplyKernel: " << cnode->DebugString(); + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::InitRuntimeResource() { + MS_LOG(INFO) << "start !"; + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + if (!runtime_instance->Init()) { + MS_LOG(EXCEPTION) << "kernel runtime init error."; + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::HardwareOptimize(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "HardwareOptimize start !"; + opt::AscendBackendOptimization(kernel_graph); + MS_EXCEPTION_IF_NULL(kernel_graph); + kernel_graph->SetExecOrderByDefault(); + MS_LOG(INFO) << "HardwareOptimize Finish!"; +} + +void AscendSession::AdjustKernel(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + device::KernelAdjust::GetInstance().Reorder(kernel_graph); + opt::HideNopNode(kernel_graph.get()); + // Insert CLearZero op + // prepare for next step from json get atomic info + BuildKernel(kernel_graph); + device::ascend::KernelBuildPreprocess(kernel_graph.get()); + device::KernelAdjust::GetInstance().InsertSwitchLoop(kernel_graph); + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + bool save_graphs = context_ptr->save_graphs_flag(); + auto save_graphs_path = context_ptr->save_graphs_path(); + if (save_graphs_path.empty()) { + save_graphs_path = "."; + } + if (save_graphs) { + std::string file_path = save_graphs_path + "/" + "after_adjust_kernel.ir"; + DumpIR(file_path, kernel_graph); + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::RunOpAdjustKernel(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + opt::HideNopNode(kernel_graph.get()); + // Insert CLearZero op + // prepare for next step from json get atomic info + BuildKernel(kernel_graph); + device::ascend::KernelBuildPreprocess(kernel_graph.get()); + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::AssignStream(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + device::ascend::AscendStreamAssign::GetInstance().AssignStreamNew(kernel_graph); + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::BuildKernel(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + auto ret = device::ascend::KernelBuild(kernel_graph.get()); + if (!ret) { + MS_LOG(EXCEPTION) << "kernel build error."; + } + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "KernelBuild run in " << PRIu64 << " us " << cost; + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::MemoryAlloc(KernelGraph *kernel_graph) const { + MS_LOG(INFO) << "start !"; + MS_EXCEPTION_IF_NULL(kernel_graph); + opt::RemoveNopNode(kernel_graph); + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + runtime_instance->AssignMemory(kernel_graph); + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::RunOpMemoryAlloc(const std::vector &input_tensors, + KernelGraph *kernel_graph) const { + MS_LOG(INFO) << "start memory alloc!"; + MS_EXCEPTION_IF_NULL(kernel_graph); + opt::RemoveNopNode(kernel_graph); + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + runtime_instance->RunOpAssignMemory(input_tensors, kernel_graph); + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::GenerateTaskInfo(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + (void)device::KernelAdjust::GetInstance().StepLoadCtrlInputs(context_, kernel_graph); + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + bool ret_ok = runtime_instance->GenTask(kernel_graph.get()); + if (!ret_ok) { + MS_LOG(EXCEPTION) << "generate task error!"; + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::LoadTask(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + bool ret_ok = runtime_instance->LoadTask(kernel_graph.get()); + if (!ret_ok) { + MS_LOG(EXCEPTION) << "load task error!"; + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::ExecTask(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + bool ret_ok = runtime_instance->Run(kernel_graph.get()); + if (!ret_ok) { + MS_LOG(EXCEPTION) << "run task error!"; + } + MS_LOG(INFO) << "Finish!"; +} + +void AscendSession::Dump(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "start !"; + MS_EXCEPTION_IF_NULL(kernel_graph); + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + (void)runtime_instance->DumpData(kernel_graph.get()); + MS_LOG(INFO) << "Finish!"; +} + +GraphId AscendSession::SetFinalGraphInput(const std::vector &args) { + MS_LOG(INFO) << "start!args size " << args.size(); + auto final_graph = std::make_shared(); + final_graph_id_ = graph_sum_++; + graphs_[final_graph_id_] = final_graph; + final_graph->set_graph_id(final_graph_id_); + MS_LOG(INFO) << "Create a new final graph" << final_graph_id_ << "success"; + // init private variables and bind them with final_graph_id + graph_execute_orders_[final_graph_id_] = std::vector(); + graph_order_types_[final_graph_id_] = std::vector(); + for (const auto ¶meter : args) { + MS_EXCEPTION_IF_NULL(parameter); + if (!parameter->isa()) { + MS_LOG(EXCEPTION) << parameter->DebugString() << " is not a parameter type!"; + } + AnfNodePtr parameter_backend = nullptr; + // if function return UINT_MAX,the parameter is not exist in child graph + auto parameter_belong_graph_id = GetGraphIdByNode(parameter); + if (parameter_belong_graph_id == kInvalidGraphId) { + parameter_backend = final_graph->NewParameter(parameter->cast()); + final_graph->FrontBackendlMapAdd(parameter, parameter_backend); + MS_LOG(INFO) << "new parameter" << parameter->DebugString() << "in final_graph"; + } else { + // parametr is a parameter of child graph + auto graph = GetGraph(parameter_belong_graph_id); + MS_EXCEPTION_IF_NULL(graph); + MS_LOG(INFO) << "reuse parameter [" << parameter->DebugString() << "] of child graph [" + << parameter_belong_graph_id << "]"; + parameter_backend = graph->GetBackendAnfByFrontAnf(parameter); + } + MS_EXCEPTION_IF_NULL(parameter_backend); + MS_LOG(INFO) << "parameter backend " << parameter_backend->DebugString() << " belong_graph_id " + << AnfAlgo::GetGraphId(parameter_backend.get()); + // add parameter in backend to final graph inputs + auto final_graph_inputs = final_graph->MutableInputs(); + MS_EXCEPTION_IF_NULL(final_graph_inputs); + final_graph_inputs->push_back(parameter_backend); + } + MS_LOG(INFO) << "end final_graph_id " << final_graph_id_; + return final_graph_id_; +} + +void AscendSession::SetFinalGraphOutput(const BaseRef &output) { + auto final_graph = GetGraph(final_graph_id_); + MS_EXCEPTION_IF_NULL(final_graph); + if (!utils::isa(output)) { + if (!utils::isa(output)) { + MS_LOG(EXCEPTION) << "Unknown output type:" << output.ToString(); + } + auto value_ptr = utils::cast(output); + auto value_node = NewValueNode(value_ptr); + MS_EXCEPTION_IF_NULL(value_node); + auto kernel_info = std::make_shared(); + value_node->set_kernel_info(kernel_info); + value_node->set_abstract(abstract::FromValue(value_ptr)); + final_graph->set_output(final_graph->NewCNode({NewValueNode(prim::kPrimMakeTuple), value_node})); + final_graph->set_executable(false); + MS_LOG(INFO) << "not anf output[" << output.ToString() << "]"; + return; + } + // get the backend anf node related to the output node of front + auto output_anf_node = utils::cast(output); + auto output_from_graph_id = GetGraphIdByNode(output_anf_node); + auto output_from_graph = GetGraph(output_from_graph_id); + MS_EXCEPTION_IF_NULL(output_anf_node); + MS_LOG(INFO) << "set the output[" << output_anf_node->DebugString() << "] of graph[" << output_from_graph_id + << "] to final graph"; + MS_EXCEPTION_IF_NULL(output_from_graph); + // if output is from final graph,it remarks no child graph exist + if (final_graph_id_ == output_from_graph_id) { + MS_LOG(INFO) << "no child graph,output is " << output_anf_node->DebugString(); + final_graph->set_output(ConstructOutput({output_anf_node}, final_graph)); + final_graph->set_executable(false); + return; + } + final_graph->set_output(output_from_graph->output()); +} + +KernelGraphPtr AscendSession::GetGraph(mindspore::GraphId graph_id) { + auto it = graphs_.find(graph_id); + if (it == graphs_.end()) { + MS_LOG(WARNING) << "can't find graph " << graph_id; + return nullptr; + } + return it->second; +} + +void AscendSession::InsertSwitchToGraph(GraphId condition_graph_id, GraphId true_graph_id) { + MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "condition graph id[" << condition_graph_id << "],true graph id[" << true_graph_id << "]"; + auto condition_graph = GetGraph(condition_graph_id); + MS_EXCEPTION_IF_NULL(condition_graph); + tensor::TensorPtr tensor = std::make_shared(kNumberTypeInt32, std::vector{1}); + int32_t *val = nullptr; + val = static_cast(tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(val); + *val = 0; + auto value_node = std::make_shared(tensor); + value_node->set_abstract(abstract::FromValue(tensor, false)); + auto counter_const = condition_graph->NewValueNode(value_node); + condition_graph->AddValueNodeToGraph(counter_const); + // create a new switch op + auto switch_primitive = std::make_shared("StreamSwitch"); + auto kernel_build_info_builder = std::make_shared(); + kernel_build_info_builder->SetOutputsFormat(std::vector{kOpFormat_DEFAULT}); + kernel_build_info_builder->SetOutputsDeviceType(std::vector{kNumberTypeInt32}); + kernel_build_info_builder->SetFusionType(kernel::FusionType::OPAQUE); + kernel_build_info_builder->SetProcessor(kernel::Processor::AICORE); + kernel_build_info_builder->SetKernelType(KernelType::RT_KERNEL); + // condition graph's output must be single output + if (condition_graph->outputs().size() != 1) { + MS_LOG(EXCEPTION) << "condition_graph output num " << condition_graph_id << " should be 1"; + } + AnfNodePtr cond_output_kernel = condition_graph->outputs()[0]; + std::vector inputs = {NewValueNode(switch_primitive), cond_output_kernel, counter_const}; + CNodePtr switch_node = condition_graph->NewCNode(inputs); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), switch_node.get()); + MS_EXCEPTION_IF_NULL(switch_node); + switch_node->set_abstract(std::make_shared()); + AnfAlgo::SetGraphId(condition_graph_id, switch_node.get()); + AnfAlgo::SetStreamDistinctionLabel(GetDistinctionLabel(GetGraph(condition_graph_id)), switch_node.get()); + // set attr: cond_ RT_GREATER + AnfAlgo::SetNodeAttr(kAttrSwitchCondition, MakeValue(static_cast(RT_GREATER)), switch_node); + // set attr:data_type + AnfAlgo::SetNodeAttr(kAttrDataType, MakeValue(static_cast(RT_SWITCH_INT64)), switch_node); + // set attr:true branch graph id ,which is same to stream distinction label + AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(true_graph_id), switch_node); + // append switch at the end of condition graph + std::vector exec_order = condition_graph->execution_order(); + exec_order.push_back(switch_node); + condition_graph->set_execution_order(exec_order); + MS_LOG(INFO) << "end"; +} + +void AscendSession::CopyOutputOfIf(GraphId false_graph_id) { + auto &graph_execute_order = GetGraphOrder(final_graph_id_); + auto &graph_order_type = GetGraphOrderType(final_graph_id_); + auto false_index = ExecOrderOfChildGraph(final_graph_id_, false_graph_id); + if (false_index == kInvalidIndex || false_index == 0) { + return; + } + for (int i = SizeToInt(false_index) - 1; i >= 0; i--) { + size_t graph_index = IntToSize(i); + if (graph_index >= graph_execute_order.size()) { + MS_LOG(EXCEPTION) << "graph index[" << graph_index << "] out of range[" << graph_execute_order.size() << "]"; + } + if (graph_order_type[graph_index] == COMMON_GRAPH) { + auto true_last_id = graph_execute_order[graph_index]; + MS_LOG(INFO) << "the last graph of if true branch is " << true_last_id; + auto true_last = GetGraph(true_last_id); + auto final_graph = GetGraph(final_graph_id_); + MS_EXCEPTION_IF_NULL(final_graph); + auto false_last_id = AnfAlgo::GetGraphId(final_graph->output().get()); + auto false_last = GetGraph(false_last_id); + MS_EXCEPTION_IF_NULL(true_last); + MS_EXCEPTION_IF_NULL(false_last); + MS_LOG(INFO) << "the last graph of false branch is " << false_last_id; + // now only consider the single output + InsertMultipleAssignToGraph(true_last_id, true_last->output(), false_last->output()); + // insert stream acitve for loop sink + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_task_sink() && context_ptr->loop_sink_flag() && + ConfigManager::GetInstance().iter_num() > 1) { + // insert active in true graph, another active will be inserted in kernel adjust + InsertStreamActiveToGraph(true_last_id, kInvalidDistincLabel - 1); + } + break; + } + } +} + +void AscendSession::SwitchCompile(GraphId cond_graph_id, GraphId true_graph_id, GraphId false_graph_id) { + if (switchs_.find(cond_graph_id) != switchs_.end()) { + MS_LOG(WARNING) << "condition graph" << cond_graph_id << " has been set before "; + return; + } + switchs_[cond_graph_id] = std::pair(true_graph_id, false_graph_id); + MS_LOG(INFO) << "new switch compile " << cond_graph_id << " " << true_graph_id << " " << false_graph_id; + // set the type of condtion graph + auto cond_graph_index = ExecOrderOfChildGraph(final_graph_id_, cond_graph_id); + auto &graph_order_type = GetGraphOrderType(final_graph_id_); + if (cond_graph_index >= graph_order_type.size()) { + MS_LOG(EXCEPTION) << "cond_graph_index " << cond_graph_index << " out of range " << graph_order_types_.size(); + } + graph_order_type[cond_graph_index] = CONDITION_GRAPH; + // update disinction label of false graph,update before merge to sure the distinction + if (false_graph_id != kInvalidGraphId) { + // false graph and condition in graph same stream + auto conditon_graph = GetGraph(cond_graph_id); + SetStreamDistinctionLabel(GetGraph(false_graph_id), GetDistinctionLabel(conditon_graph), true); + // if false graph is a condition graph and has been switch compiled before,it's false should be updated again + auto cond_it = switchs_.find(false_graph_id); + while (cond_it != switchs_.end() && cond_it->second.second != kInvalidGraphId) { + cond_graph_id = cond_it->first; + false_graph_id = cond_it->second.second; + conditon_graph = GetGraph(cond_graph_id); + SetStreamDistinctionLabel(GetGraph(false_graph_id), GetDistinctionLabel(conditon_graph), true); + cond_it = switchs_.find(false_graph_id); + } + } +} // namespace session + +void AscendSession::MergeSwitchCompile() { + auto graph_execute_order = GetGraphOrder(final_graph_id_); + auto &graph_order_type = GetGraphOrderType(final_graph_id_); + for (auto switch_compile : switchs_) { + auto cond_graph_id = switch_compile.first; + auto true_graph_id = switch_compile.second.first; + auto false_graph_id = switch_compile.second.second; + MS_LOG(INFO) << "switch compile: " << cond_graph_id << " " << true_graph_id << " " << false_graph_id; + auto condition_graph = GetGraph(cond_graph_id); + auto final_graph = GetGraph(final_graph_id_); + MS_EXCEPTION_IF_NULL(condition_graph); + MS_EXCEPTION_IF_NULL(final_graph); + // insert switch to condition graph + InsertSwitchToGraph(cond_graph_id, true_graph_id); + auto cond_graph_index = ExecOrderOfChildGraph(final_graph_id_, cond_graph_id); + auto prev_graph_id = kInvalidGraphId; + // if condition graph is the first graph and final graph has assign op,then the final graph is the common graph + if (cond_graph_index == 0 && !final_graph->execution_order().empty()) { + prev_graph_id = final_graph_id_; + // set the distinction label of final graph + SetStreamDistinctionLabel(final_graph, final_graph_id_, true); + // if condition graph is not the first graph + } else if ((cond_graph_index - 1 < graph_execute_order.size()) && + (graph_order_type[cond_graph_index - 1] == COMMON_GRAPH)) { + prev_graph_id = graph_execute_order[cond_graph_index - 1]; + } + // insert stream active to common graph + if (prev_graph_id != kInvalidGraphId) { + InsertStreamActiveToGraph(prev_graph_id, GetDistinctionLabel(condition_graph)); + } + // if this is a 'if' condition + auto it = while_condtion_graphs_.find(cond_graph_id); + if (it == while_condtion_graphs_.end()) { + CopyOutputOfIf(false_graph_id); + } else { + // if it is a while,insert a stream active to true graph + GraphId from_graph = it->second; + InsertStreamActiveToGraph(from_graph, GetDistinctionLabel(condition_graph)); + } + } + MS_LOG(INFO) << "end"; +} + +// insert active to graph +void AscendSession::SetActive(GraphId from, GraphId to) { + if (while_condtion_graphs_.find(to) != while_condtion_graphs_.end()) { + MS_LOG(WARNING) << " to " << to << " has been exits in map,from " << from << ",exist from " + << while_condtion_graphs_[to]; + return; + } + MS_LOG(INFO) << "from " << from << " to " << to; + auto &graph_order = GetGraphOrder(final_graph_id_); + auto &graph_type = GetGraphOrderType(final_graph_id_); + std::vector graph_order_new; + std::vector graph_type_new; + for (size_t i = 0; i < graph_order.size(); i++) { + auto graph_id = graph_order[i]; + graph_order_new.push_back(graph_id); + graph_type_new.push_back(graph_type[i]); + if (from == graph_id) { + graph_order_new.push_back(kInvalidGraphId); + graph_type_new.push_back(BRANCH_END); + } + } + graph_order = graph_order_new; + graph_type = graph_type_new; + // set the graph type of condition graph + graph_type[ExecOrderOfChildGraph(final_graph_id_, to)] = CONDITION_GRAPH; + // record the condition graph into while condition set + while_condtion_graphs_[to] = from; +} + +void AscendSession::SetChildGraphParameter(const AnfNodePtr &front_anf, const AnfNodePtr &backend_parameter) { + MS_LOG(INFO) << "start"; + MS_EXCEPTION_IF_NULL(backend_parameter); + MS_EXCEPTION_IF_NULL(front_anf); + if (!backend_parameter->isa()) { + MS_LOG(EXCEPTION) << "backend parameter's type is not a parameter,but is " << backend_parameter->ToString(); + } + auto from_graph_id = GetGraphIdByNode(front_anf); + auto from_graph = GetGraph(from_graph_id); + MS_EXCEPTION_IF_NULL(from_graph); + + MS_LOG(INFO) << "set node[" << front_anf->DebugString() << "] of graph[" << from_graph_id << "]to node[" + << backend_parameter->DebugString() << "] of graph[" << AnfAlgo::GetGraphId(backend_parameter.get()) + << "]"; + // a node should not assign to itself + auto backend_arg = from_graph->GetBackendAnfByFrontAnf(front_anf); + if (backend_arg.get() == backend_parameter.get()) { + return; + } + // if arg is the the parameter of child graph,it is parameter of final graph too + if (front_anf->isa()) { + MS_EXCEPTION_IF_NULL(backend_arg); + if (!AnfAlgo::OutputAddrExist(backend_arg, 0)) { + // set parameter's addr in child graph to parameter in final graph + AnfAlgo::SetOutputAddr(AnfAlgo::GetMutableOutputAddr(backend_parameter, 0), 0, backend_arg.get()); + MS_LOG(INFO) << "assign mem of node" << backend_parameter->DebugString() << " of graph " + << AnfAlgo::GetGraphId(backend_parameter.get()) << " to node" << backend_arg->DebugString() + << "of graph " << AnfAlgo::GetGraphId(backend_arg.get()); + return; + } + } + InsertMultipleAssignToGraph(from_graph_id, backend_arg, backend_parameter); + // if front anf is a parameter,we can assign the value back,because backend_parameter won't be change in it's graph + // unless it's a weigth.If backend_parameter is a weight,we do should assign the value back + auto to_graph_id = AnfAlgo::GetGraphId(backend_parameter.get()); + auto to_graph = GetGraph(to_graph_id); + MS_EXCEPTION_IF_NULL(to_graph); + if (backend_arg->isa() && !to_graph->execution_order().empty()) { + InsertMultipleAssignToGraph(to_graph_id, backend_parameter, backend_arg); + } + MS_LOG(INFO) << "end"; +} + +void AscendSession::SetChildGraphParameter(const tensor::TensorPtr &front_tensor, const AnfNodePtr &backend_parameter) { + MS_LOG(INFO) << "start"; + // sync data from host to device + MS_EXCEPTION_IF_NULL(front_tensor); + size_t tensor_size = front_tensor->data().nbytes(); + auto addr = AnfAlgo::GetOutputAddr(backend_parameter, 0); + MS_EXCEPTION_IF_NULL(addr); + if (!addr->SyncHostToDevice(front_tensor->shape(), tensor_size, front_tensor->data_type(), + front_tensor->data_c(false))) { + MS_LOG(EXCEPTION) << "tensor SyncHostToDevice fail!"; + } + MS_LOG(INFO) << "end"; +} + +void AscendSession::UpdateGraphOrder(GraphId to_graph_id) { + MS_LOG(INFO) << "to_graph_id " << to_graph_id; + auto &graph_order = GetGraphOrder(final_graph_id_); + auto &graph_type = GetGraphOrderType(final_graph_id_); + for (size_t i = 0; i < graph_order.size(); i++) { + if (graph_order[i] == to_graph_id) { + return; + } + } + // if graph is not in graph order,add it to graph order + SetStreamDistinctionLabel(GetGraph(to_graph_id), to_graph_id, false); + graph_order.push_back(to_graph_id); + graph_type.push_back(COMMON_GRAPH); + for (size_t i = 0; i < graph_order.size(); i++) { + MS_LOG(INFO) << "index " << i << ",graph_id " << graph_order[i] << ",graph_type" << graph_type[i]; + } +} + +void AscendSession::SetChildGraphInput(GraphId g, const VectorRef &args) { + MS_LOG(INFO) << "Set input of graph " << g; + auto to_graph = GetGraph(g); + MS_EXCEPTION_IF_NULL(to_graph); + DumpGraphInputArgs(args); + UpdateGraphOrder(g); + std::vector graph_inputs = to_graph->inputs(); + size_t input_index = 0; + for (size_t i = 0; i < args.size(); i++) { + if (input_index >= graph_inputs.size()) { + MS_LOG(EXCEPTION) << "input_index " << input_index << " out of range size " << graph_inputs.size(); + } + if (utils::isa(args[i])) { + // arg is a anf node + for (const auto &real_arg : AnfAlgo::GetAllOutput(utils::cast(args[i]), {prim::kPrimTupleGetItem})) { + SetChildGraphParameter(real_arg, graph_inputs[input_index]); + input_index++; + } + } else if (utils::isa(args[i])) { + auto value = utils::cast(args[i]); + MS_EXCEPTION_IF_NULL(value); + // arg is a tensor + if (!value->isa()) { + MS_LOG(EXCEPTION) << "Value Node should be a tensor, unexpected value: " << value->ToString(); + } + SetChildGraphParameter(value->cast(), graph_inputs[input_index]); + input_index++; + } else { + MS_LOG(EXCEPTION) << "Unxpected arg type " << args[i].ToString(); + } + } + MS_LOG(INFO) << "end"; +} + +GraphId AscendSession::GetGraphIdByNode(const AnfNodePtr &front_anf) const { + for (const auto &graph_item : graphs_) { + auto graph = graph_item.second; + MS_EXCEPTION_IF_NULL(graph); + // if front_anf is a parameter,the backend parameter may have two + if (graph->GetBackendAnfByFrontAnf(front_anf) != nullptr) { + return graph_item.first; + } + } + MS_EXCEPTION_IF_NULL(front_anf); + MS_LOG(WARNING) << "front_anf " << front_anf->DebugString() << " is not exist in any graph"; + return kInvalidGraphId; +} + +void AscendSession::MergeGraphExecOrder() { + MS_LOG(INFO) << "start"; + // insert switch to graph + MergeSwitchCompile(); + // merge graph order + auto &graph_order = GetGraphOrder(final_graph_id_); + auto &graph_type = GetGraphOrderType(final_graph_id_); + auto final_graph = GetGraph(final_graph_id_); + MS_EXCEPTION_IF_NULL(final_graph); + if (graph_order.empty()) { + MS_LOG(WARNING) << "graph output is a lonely variable not linked to any op!"; + return; + } + // if first graph is common,the final graph has no label,then set the stream of final graph same with the first graph + SetStreamDistinctionLabel(final_graph, graph_order[0], false); + std::vector final_exec_order = final_graph->execution_order(); + KernelGraphPtr last_graph = nullptr; + for (size_t i = 0; i < graph_order.size(); i++) { + auto graph_id = graph_order[i]; + if (graph_type[i] == BRANCH_END || graph_type[i] == BRANCH_START) { + continue; + } + auto child_graph = GetGraph(graph_id); + last_graph = child_graph; + MS_EXCEPTION_IF_NULL(child_graph); + auto exec_order = child_graph->execution_order(); + MS_LOG(INFO) << "merge graph,graph_id " << graph_id; + (void)std::copy(exec_order.begin(), exec_order.end(), std::back_inserter(final_exec_order)); + // add all value nodes of child graphs to final graph + for (auto &value_node : child_graph->graph_value_nodes()) { + final_graph->AddValueNodeToGraph(value_node); + } + // copy ref map to final graph + auto child_ref_map = child_graph->GetRefMap(); + for (auto &item : child_ref_map) { + if (final_graph->IsInRefOutputMap(item.first)) { + MS_LOG(EXCEPTION) << "The ref pair is already in final graph!"; + } + final_graph->AddRefCorrespondPairs(item.first, item.second); + } + } + // set final_exec_order into final graph + MS_EXCEPTION_IF_NULL(final_graph); + DumpGraphExeOrder(final_exec_order); + final_graph->set_execution_order(final_exec_order); +} + +void AscendSession::InsertAssignToGraph(GraphId graph_id, const AnfNodePtr &from, const AnfNodePtr &to) { + MS_EXCEPTION_IF_NULL(from); + MS_EXCEPTION_IF_NULL(to); + if (AnfAlgo::OutputAddrExist(from, 0) && AnfAlgo::OutputAddrExist(to, 0) && + AnfAlgo::GetOutputAddr(from, 0) == AnfAlgo::GetOutputAddr(to, 0)) { + return; + } + if (from.get() == to.get()) { + return; + } + MS_LOG(INFO) << "Insert assign to graph " << graph_id << " from " << from->DebugString() << " to " + << to->DebugString(); + auto graph = graphs_[graph_id]; + MS_EXCEPTION_IF_NULL(graph); + // config inputs of assign node + std::vector inputs = {NewValueNode(std::make_shared("Assign")), to, from}; + // generate a new cnode + auto assign_node = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(assign_node); + assign_node->set_abstract(std::make_shared()); + auto kernel_build_info_builder = std::make_shared(); + kernel_build_info_builder->SetKernelType(KernelType::RT_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), assign_node.get()); + AnfAlgo::SetStreamDistinctionLabel(GetDistinctionLabel(graph), assign_node.get()); + // append the assign at the end of from graph + auto exec_order = graph->execution_order(); + exec_order.push_back(assign_node); + graph->set_execution_order(exec_order); +} + +void AscendSession::InsertMultipleAssignToGraph(GraphId graph_id, const AnfNodePtr &from, const AnfNodePtr &to) { + std::vector from_outputs = AnfAlgo::GetAllOutput(from, {prim::kPrimTupleGetItem}); + std::vector to_outputs = AnfAlgo::GetAllOutput(to, {prim::kPrimTupleGetItem}); + MS_LOG(INFO) << "insert assigns from [" << AnfAlgo::GetGraphId(from.get()) << "] to [" + << AnfAlgo::GetGraphId(to.get()) << "]"; + if (from_outputs.size() != to_outputs.size()) { + MS_LOG(INFO) << "from[" << from->DebugString(5) << "] to[" << to->DebugString(5) << "]"; + MS_LOG(EXCEPTION) << "from outputs size[" << from_outputs.size() << "] is not equal to to outputs size[" + << to_outputs.size() << "]"; + } + for (size_t i = 0; i < from_outputs.size(); i++) { + InsertAssignToGraph(graph_id, from_outputs[i], to_outputs[i]); + } +} + +void AscendSession::InsertStreamActiveToGraph(GraphId graph_id, uint32_t actived_stream) { + MS_LOG(INFO) << "Insert stream_active from " << graph_id << " to " << actived_stream; + auto from_graph = graphs_[graph_id]; + MS_EXCEPTION_IF_NULL(from_graph); + std::vector inputs = {NewValueNode(std::make_shared("StreamActive"))}; + auto active_node = from_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(active_node); + active_node->set_abstract(std::make_shared()); + auto kernel_build_info_builder = std::make_shared(); + kernel_build_info_builder->SetKernelType(KernelType::RT_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), active_node.get()); + // set the actived stream id into the attr of active node + std::vector active_index_value = {}; + active_index_value.push_back(actived_stream); + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_index_value), active_node); + AnfAlgo::SetStreamDistinctionLabel(GetDistinctionLabel(from_graph), active_node.get()); + // append the active node at the end of from graph + auto exec_order = from_graph->execution_order(); + exec_order.push_back(active_node); + from_graph->set_execution_order(exec_order); +} + +size_t AscendSession::ExecOrderOfChildGraph(GraphId final_graph, GraphId child_graph) { + auto &graph_order = GetGraphOrder(final_graph); + for (size_t i = 0; i < graph_order.size(); i++) { + if (child_graph == graph_order[i]) { + return i; + } + } + return kInvalidIndex; +} + +std::vector &AscendSession::GetGraphOrder(GraphId final_graph_id) { + auto graph_order_iter = graph_execute_orders_.find(final_graph_id); + if (graph_order_iter == graph_execute_orders_.end()) { + MS_LOG(EXCEPTION) << "final graph" << final_graph_id << "has no child graph"; + } + return graph_order_iter->second; +} + +// get graph order type vector by graph id +std::vector &AscendSession::GetGraphOrderType(GraphId final_graph_id) { + auto graph_type_iter = graph_order_types_.find(final_graph_id); + if (graph_type_iter == graph_order_types_.end()) { + MS_LOG(EXCEPTION) << "final graph" << final_graph_id << "has no graph_order_types_"; + } + return graph_type_iter->second; +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/ascend_session.h b/mindspore/ccsrc/session/ascend_session.h new file mode 100644 index 00000000000..e5cfb52f2a6 --- /dev/null +++ b/mindspore/ccsrc/session/ascend_session.h @@ -0,0 +1,124 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_ASCEND_SESSION_H +#define MINDSPORE_CCSRC_SESSION_ASCEND_SESSION_H +#include +#include +#include +#include +#include +#include +#include "session/session_basic.h" +#include "session/kernel_graph.h" +#include "kernel/kernel.h" +#include "session/session_factory.h" + +namespace mindspore { +namespace session { +enum GraphType : int { COMMON_GRAPH = 0, CONDITION_GRAPH = 1, BRANCH_START = 2, BRANCH_END = 3 }; + +class AscendSession : public SessionBasic { + public: + AscendSession() { final_graph_id_ = kInvalidGraphId; } + ~AscendSession() override = default; + void Init(uint32_t device_id) override { + SessionBasic::Init(device_id); + context_ = std::make_shared(kAscendDevice, device_id); + } + GraphId CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) override; + void RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *outputs) override; + void BuildGraph(GraphId) override; + void BuildOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) override; + py::tuple RunOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) override; + + // set parameters of final graph + GraphId SetFinalGraphInput(const std::vector &args) override; + // set output of final graph + void SetFinalGraphOutput(const BaseRef &output) override; + // insert switch and set the relative acitve ops + void SwitchCompile(GraphId cond_g, GraphId true_g, GraphId false_g) override; + // set args of child graph.the arg maybe come from a output of other child graphs,or from final graph's parameter + void SetChildGraphInput(GraphId g, const VectorRef &args) override; + // get graph id in child graphs by ME front anf node pointer + GraphId GetGraphIdByNode(const AnfNodePtr &front_anf) const override; + // get grpah id of final graph + GraphId GetFinalRunGraph() const override { return final_graph_id_; } + // insert active to graph + void SetActive(GraphId, GraphId) override; + + private: + void InitRuntimeResource(); + void SelectKernel(const KernelGraph &kernel_graph) const; + void HardwareOptimize(const std::shared_ptr &kernel_graph) const; + void AdjustKernel(const std::shared_ptr &kernel_graph) const; + void RunOpAdjustKernel(const std::shared_ptr &kernel_graph) const; + void AssignStream(const std::shared_ptr &kernel_graph) const; + void BuildKernel(const std::shared_ptr &kernel_graph) const; + void MemoryAlloc(KernelGraph *kernel_graph) const; + void RunOpMemoryAlloc(const std::vector &input_tensors, KernelGraph *kernel_graph) const; + void GenerateTaskInfo(const std::shared_ptr &kernel_graph) const; + void LoadTask(const std::shared_ptr &kernel_graph) const; + void ExecTask(const std::shared_ptr &kernel_graph) const; + void Dump(const std::shared_ptr &kernel_graph) const; + // below functions are used for run op + void RunOpHardwareOptimize(const std::shared_ptr &kernel_graph) const; + void RunOpExecTask(const std::shared_ptr &kernel_graph) const; + + // merge execution order list of child graphs + void MergeGraphExecOrder(); + // insert assion op to sync data bettween different graphs + void InsertAssignToGraph(GraphId graph_id, const AnfNodePtr &from, const AnfNodePtr &to); + // insert mutiple assigns to graph + void InsertMultipleAssignToGraph(GraphId graph_id, const AnfNodePtr &from, const AnfNodePtr &to); + // insert active op to graph + void InsertStreamActiveToGraph(GraphId graph_id, uint32_t actived_stream); + // get execute index of graph + size_t ExecOrderOfChildGraph(GraphId final_graph, GraphId child_graph); + // handle condition graph from vm + void InsertSwitchToGraph(GraphId condition_graph_id, GraphId true_graph_id); + // Get graph by graph id ,if not exist return null ptr + KernelGraphPtr GetGraph(GraphId graph_id); + // set child graph parameter if front arg is a anf + void SetChildGraphParameter(const AnfNodePtr &front_anf, const AnfNodePtr &backend_parameter); + // set child graph parameter if front arg is a tensor + void SetChildGraphParameter(const tensor::TensorPtr &front_tensor, const AnfNodePtr &backend_parameter); + // update the execution order of all child graphs + void UpdateGraphOrder(GraphId to_graph); + // handle switch when merge + void MergeSwitchCompile(); + // get graph order vector by graph id + std::vector &GetGraphOrder(GraphId final_graph_id); + // get graph order type vector by graph id + std::vector &GetGraphOrderType(GraphId final_graph_id); + // copy output of if and else + void CopyOutputOfIf(GraphId false_graph_id); + + // member variables + // key is final_graph_id,value is child graph execute order of final graph + std::unordered_map> graph_execute_orders_; + // key is final_graph_id,value is the graph types of child graphs + std::unordered_map> graph_order_types_; + // record condition graph of while + std::unordered_map while_condtion_graphs_; + // record all conditons + std::unordered_map> switchs_; + // final_graph_id is used in every root graph has it's own session situation + GraphId final_graph_id_; +}; +MS_REG_SESSION(kAscendDevice, AscendSession); +} // namespace session +} // namespace mindspore +#endif // MINDSPORE_CCSRC_SESSION_ASCEND_SESSION_H diff --git a/mindspore/ccsrc/session/gpu_session.cc b/mindspore/ccsrc/session/gpu_session.cc new file mode 100644 index 00000000000..196a2f300f9 --- /dev/null +++ b/mindspore/ccsrc/session/gpu_session.cc @@ -0,0 +1,167 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/gpu_session.h" +#include "device/gpu/kernel_info_setter.h" +#include "device/gpu/gpu_kernel_build.h" +#include "device/gpu/gpu_kernel_runtime.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "pre_activate/ascend/ir_fusion/allreduce_fusion.h" +#include "device/kernel_runtime_manager.h" +#include "predict/predict.h" +#include "common/utils.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace session { +namespace gpu { +using AnfAlgo = mindspore::session::AnfRuntimeAlgorithm; + +void GPUSession::SelectKernel(const std::shared_ptr &kernel_graph) const { + MS_EXCEPTION_IF_NULL(kernel_graph); + for (const auto &kernel_node : kernel_graph->execution_order()) { + MS_EXCEPTION_IF_NULL(kernel_node); + device::gpu::SetKernelInfo(kernel_node); + } +} + +void GPUSession::StartKernelRT() const { + auto runtime_instance = device::KernelRuntimeManager::Instance().GetSingleKernelRuntime(kGPUDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + if (!runtime_instance->Init()) { + MS_LOG(EXCEPTION) << "GPU start kernel runtime failed"; + } +} + +void GPUSession::Optimize(const std::shared_ptr &kernel_graph) { + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void GPUSession::BuildKernel(const std::shared_ptr &kernel_graph) const { + device::gpu::GpuBuild(kernel_graph); +} + +void GPUSession::AllocateMemory(KernelGraph *kernel_graph) const { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto runtime_instance = device::KernelRuntimeManager::Instance().GetSingleKernelRuntime(kGPUDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + runtime_instance->AssignMemory(kernel_graph); +} + +void GPUSession::RunOpAllocateMemory(const std::vector &input_tensors, + KernelGraph *kernel_graph) const { + MS_EXCEPTION_IF_NULL(kernel_graph); + auto runtime_instance = device::KernelRuntimeManager::Instance().GetSingleKernelRuntime(kGPUDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + runtime_instance->RunOpAssignMemory(input_tensors, kernel_graph); +} + +void GPUSession::Execute(const std::shared_ptr &kernel_graph) const { + auto runtime_instance = device::KernelRuntimeManager::Instance().GetSingleKernelRuntime(kGPUDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + if (!runtime_instance->Run(kernel_graph.get())) { + MS_LOG(EXCEPTION) << "GPU execute graph failed!"; + } +} + +GraphId GPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) { + // Construct graph, if construct successs, graph_sum_ + 1 + auto graph_id = graph_sum_; + auto graph = ConstructKernelGraph(lst, outputs); + // Select kernel build info + SelectKernel(graph); + // Convert kernel Graph to model + predictmodel::StepConvertGraph(graph); + // Start gpu kernel runtime + StartKernelRT(); + // AllReduce Optimize + Optimize(graph); + // Build kernel if node is cnode + BuildKernel(graph); + // Set graph execution order before memory alloc, ensure that memory alloc is according to the reorder graph + auto execution_order = graph->execution_order(); + Reorder(&execution_order); + graph->set_execution_order(execution_order); + // Alloc memeory, include static memory and dynamic memory + AllocateMemory(graph.get()); + // Reset memory resource + auto runtime_instance = device::KernelRuntimeManager::Instance().GetSingleKernelRuntime(kGPUDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + runtime_instance->FreeHostMemory(); + return graph_id; +} + +void GPUSession::RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *outputs) { + auto &kernel_graph = graphs_[graph_id]; + // Load input data from user input + LoadInputData(kernel_graph, inputs); + MS_EXCEPTION_IF_NULL(kernel_graph); + // Convert inputs to model + predictmodel::StepConvertWeight(inputs); + // Run graph on GPU + Execute(kernel_graph); + // Get result from GPU + UpdateOutputs(kernel_graph, outputs, inputs); + // Summary + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + if (context_ptr->enable_gpu_summary()) { + Summary(kernel_graph.get()); + } +} + +void GPUSession::BuildOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) { + // Prepare the graph + auto kernel_graph = ConstructSingleOpGraph(op_run_info); + MS_EXCEPTION_IF_NULL(kernel_graph); + SelectKernel(kernel_graph); + StartKernelRT(); + BuildKernel(kernel_graph); + run_op_graphs_[graph_info] = kernel_graph; +} + +py::tuple GPUSession::RunOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) { + auto kernel_graph = run_op_graphs_[graph_info]; + MS_EXCEPTION_IF_NULL(kernel_graph); + std::vector input_tensors = {}; + std::vector tensors_mask = {}; + ToTensorPtr(op_run_info, &input_tensors, &tensors_mask); + RunOpAllocateMemory(input_tensors, kernel_graph.get()); + // Execute the computation + LoadInputData(kernel_graph, input_tensors); + Execute(kernel_graph); + // Fetch outputs + VectorRef outputs; + UpdateOutputs(kernel_graph, &outputs, input_tensors); + // Trans output to tuple + auto output_tensors = TransformBaseRefListToTuple(outputs); + if (!utils::isa(output_tensors) || + !py::isinstance(utils::cast(output_tensors).object_)) { + MS_EXCEPTION(NotSupportError) << "The output tensors should be a tuple !"; + } + py::object tuple_obj = utils::cast(output_tensors).object_; + py::tuple tuple_tensors = py::cast(tuple_obj); + run_op_graphs_.clear(); + return tuple_tensors; +} +} // namespace gpu +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/gpu_session.h b/mindspore/ccsrc/session/gpu_session.h new file mode 100644 index 00000000000..e443c1e701b --- /dev/null +++ b/mindspore/ccsrc/session/gpu_session.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_GPU_SESSION_H +#define MINDSPORE_CCSRC_SESSION_GPU_SESSION_H + +#include +#include +#include "session/session_basic.h" +#include "session/kernel_graph.h" +#include "session/session_factory.h" +using KernelGraph = mindspore::session::KernelGraph; + +namespace mindspore { +namespace session { +namespace gpu { +class GPUSession : public SessionBasic { + public: + GPUSession() = default; + ~GPUSession() override = default; + + void Init(uint32_t device_id) override { + SessionBasic::Init(device_id); + context_ = std::make_shared(kGPUDevice, device_id); + } + + GraphId CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) override; + + void RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *outputs) override; + void BuildOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) override; + py::tuple RunOp(const OpRunInfo &op_run_info, const GraphInfo &graph_info) override; + + private: + void SelectKernel(const std::shared_ptr &kernel_graph) const; + + void StartKernelRT() const; + + void Optimize(const std::shared_ptr &kernel_graph); + + void BuildKernel(const std::shared_ptr &kernel_graph) const; + + void AllocateMemory(KernelGraph *kernel_graph) const; + + void RunOpAllocateMemory(const std::vector &input_tensors, KernelGraph *kernel_graph) const; + + void Execute(const std::shared_ptr &kernel_graph) const; +}; +using GPUSessionPtr = std::shared_ptr; +MS_REG_SESSION(kGPUDevice, GPUSession); +} // namespace gpu +} // namespace session +} // namespace mindspore +#endif // MINDSPORE_CCSRC_SESSION_GPU_SESSION_H diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc new file mode 100644 index 00000000000..b07840aa984 --- /dev/null +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -0,0 +1,483 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/kernel_graph.h" +#include +#include +#include +#include +#include "common/utils.h" +#include "operator/ops.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" +#include "kernel/kernel_build_info.h" + +namespace mindspore { +namespace session { +namespace { +void PushNoVisitedNode(const AnfNodePtr &node, std::queue *que, + std::unordered_set *visited_nodes) { + MS_EXCEPTION_IF_NULL(que); + MS_EXCEPTION_IF_NULL(visited_nodes); + if (visited_nodes->find(node) == visited_nodes->end()) { + que->push(node); + (void)visited_nodes->insert(node); + MS_LOG(DEBUG) << "push que:" << node->DebugString(); + } +} +} // namespace +std::vector KernelGraph::outputs() const { + MS_EXCEPTION_IF_NULL(output()); + if (IsPrimitiveCNode(output(), prim::kPrimMakeTuple)) { + auto make_tuple = output()->cast(); + MS_EXCEPTION_IF_NULL(make_tuple); + auto &inputs = make_tuple->inputs(); + return std::vector(inputs.begin() + 1, inputs.end()); + } + return std::vector(); +} + +void KernelGraph::SetExecOrderByDefault() { + BfsToUpdateNodeOutput(); + execution_order_.clear(); + std::queue allreduce_nodes; + std::queue zero_output_nodes; + std::unordered_set visited_nodes; + auto clear_output = [&zero_output_nodes, &allreduce_nodes, &visited_nodes, this](const AnfNodePtr &input) -> void { + if (node_output_num_[input] == 0 && visited_nodes.find(input) == visited_nodes.end()) { + MS_EXCEPTION_IF_NULL(input); + MS_LOG(DEBUG) << "clear output num:" << input->DebugString(); + (void)visited_nodes.insert(input); + if (input->isa() && AnfAlgo::GetCNodeName(input) == kAllReduceOpName) { + allreduce_nodes.push(input); + } else { + zero_output_nodes.push(input); + } + } + }; + zero_output_nodes.emplace(get_return()); + while (!zero_output_nodes.empty() || !allreduce_nodes.empty()) { + AnfNodePtr node; + if (!zero_output_nodes.empty()) { + node = zero_output_nodes.front(); + zero_output_nodes.pop(); + } else { + node = allreduce_nodes.front(); + allreduce_nodes.pop(); + } + MS_EXCEPTION_IF_NULL(node); + if (node->isa() && AnfAlgo::IsRealKernel(node)) { + execution_order_.push_back(node->cast()); + } + auto it = node_input_edges_.find(node); + if (it == node_input_edges_.end()) { + // value node and parameter has no input,no need to print log + if (node->isa()) { + MS_LOG(DEBUG) << "can not find node [" << node->DebugString() << "]"; + } + continue; + } + for (const auto &input_edge : it->second) { + if (node_output_num_.find(input_edge.first) == node_output_num_.end()) { + MS_EXCEPTION_IF_NULL(input_edge.first); + MS_LOG(EXCEPTION) << "can't find node[" << input_edge.first->DebugString() << "]"; + } + MS_EXCEPTION_IF_NULL(input_edge.first); + MS_LOG(DEBUG) << "decrese input:" << input_edge.first->DebugString() << ",node:" << node->DebugString() + << ",num: " << node_output_num_[input_edge.first] << ",decrease num:" << input_edge.second; + if (node_output_num_[input_edge.first] < input_edge.second) { + MS_LOG(EXCEPTION) << "input node:" << input_edge.first->DebugString() << ",node_output_num" + << node_output_num_[input_edge.first] << "depend edege:" << input_edge.second; + } + node_output_num_[input_edge.first] = node_output_num_[input_edge.first] - input_edge.second; + clear_output(input_edge.first); + } + } + CheckLoop(); + std::reverse(execution_order_.begin(), execution_order_.end()); +} + +void KernelGraph::CheckLoop() { + std::map none_zero_output; + if (node_output_edges_.size() != node_output_num_.size()) { + MS_LOG(EXCEPTION) << "node_output_edges_ size :" << node_output_edges_.size() + << "not equal to node_output_num_ size:" << node_output_num_.size(); + } + for (auto &it : node_output_num_) { + MS_EXCEPTION_IF_NULL(it.first); + string str; + auto node_output_it = node_output_edges_.find(it.first); + if (node_output_it == node_output_edges_.end()) { + MS_LOG(EXCEPTION) << "can't find node [" << it.first->DebugString() << "]"; + } + for (const auto &output_edge : node_output_edges_[it.first]) { + MS_EXCEPTION_IF_NULL(output_edge.first); + str = str.append(output_edge.first->DebugString()).append("|"); + } + if (it.second != 0) { + MS_LOG(WARNING) << "node:" << it.first->DebugString() << ",outputs:" << str << ",output num:" << it.second; + none_zero_output[it.first] = it.second; + } + } + // if don't consider control depend and loop exit,a exception will be throw + if (!none_zero_output.empty()) { + MS_LOG(EXCEPTION) << "nodes have loop,left node num:" << none_zero_output.size(); + } +} + +CNodePtr KernelGraph::NewCNode(const std::vector &inputs) { + auto cnode = FuncGraph::NewCNode(inputs); + MS_EXCEPTION_IF_NULL(cnode); + cnode->set_abstract(std::make_shared()); + // create kernel_info from new parameter + auto kernel_info = std::make_shared(); + cnode->set_kernel_info(kernel_info); + AnfAlgo::SetGraphId(graph_id_, cnode.get()); + return cnode; +} + +CNodePtr KernelGraph::NewCNode(const CNodePtr &cnode) { + MS_EXCEPTION_IF_NULL(cnode); + auto new_cnode = std::make_shared(*cnode); + // if a cnode is created not from front,this cnode won't be in map,so when replace it,we shouldn't update map + if (BakcendNodeExistInFrontBackendMap(cnode)) { + FrontBackendlMapUpdate(cnode, new_cnode); + } + AnfAlgo::SetGraphId(graph_id_, cnode.get()); + return new_cnode; +} + +ParameterPtr KernelGraph::NewParameter(const ParameterPtr ¶meter) { + ParameterPtr new_parameter = add_parameter(); + MS_EXCEPTION_IF_NULL(new_parameter); + size_t output_tensor_num = 1; + // if use default parameter = nullptr,it remarks create a new parameter from no parameter + if (parameter == nullptr) { + new_parameter->set_abstract(std::make_shared()); + } else { + // if don't use default parameter = nullptr,it remarks create a new parameter from a old parameter + new_parameter->set_abstract(parameter->abstract()); + new_parameter->set_name(parameter->name()); + if (parameter->has_default()) { + new_parameter->set_default_param(parameter->default_param()); + } + // if output is a tuple tensor,now can use for loop to handle tuple tensor + output_tensor_num = AnfAlgo::GetOutputTensorNum(parameter); + } + // create kernel_info form new parameter + auto kernel_info = std::make_shared(); + new_parameter->set_kernel_info(kernel_info); + // create kernel_build_info for new parameter + auto kernel_build_info_builder = std::make_shared(); + // create init data type, + std::vector init_data_type = {}; + for (size_t i = 0; i < output_tensor_num; i++) { + TypeId infer_data_type = AnfAlgo::GetOutputInferDataType(new_parameter, i); + init_data_type.push_back(AnfAlgo::IsParameterWeight(new_parameter) ? kTypeUnknown : infer_data_type); + } + // set the format of parameter to DEFAULT_FORMAT + kernel_build_info_builder->SetOutputsFormat(std::vector(output_tensor_num, kOpFormat_DEFAULT)); + // set parameter initaial device data type + kernel_build_info_builder->SetOutputsDeviceType(init_data_type); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), new_parameter.get()); + AnfAlgo::SetGraphId(graph_id_, new_parameter.get()); + return new_parameter; +} + +std::vector KernelGraph::SplitTupleValueNodeToNodeList(const ValueNodePtr &value_node) { + MS_EXCEPTION_IF_NULL(value_node); + auto node_value = value_node->value(); + auto output_size = AnfAlgo::GetOutputTensorNum(value_node); + std::vector convert_inputs; + if (!node_value->isa()) { + MS_LOG(EXCEPTION) << "multiple output valuenode's value must be a value tuple but got " << node_value->ToString(); + } + auto value_tuple = node_value->cast(); + MS_EXCEPTION_IF_NULL(value_tuple); + if (value_tuple->size() != output_size) { + MS_LOG(EXCEPTION) << "value tuple size" << value_tuple->size() + << " is not mathced with the value node's output size" << output_size; + } + for (size_t index = 0; index < value_tuple->value().size(); ++index) { + auto new_value_node = std::make_shared(value_tuple->value()[index]); + AnfAlgo::SetOutputInferTypeAndShape({AnfAlgo::GetOutputInferDataType(value_node, index)}, + {AnfAlgo::GetOutputInferShape(value_node, index)}, new_value_node.get()); + AddValueNodeToGraph(new_value_node); + auto kernel_info = std::make_shared(); + new_value_node->set_kernel_info(kernel_info); + // create kernel_build_info for new value node + auto kernel_build_info_builder = std::make_shared(); + // set the format of value_node to DEFAULT_FORMAT + kernel_build_info_builder->SetOutputsFormat({kOpFormat_DEFAULT}); + // set value node initial device data type = infer data type + kernel_build_info_builder->SetOutputsDeviceType({kTypeUnknown}); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), new_value_node.get()); + AnfAlgo::SetGraphId(graph_id_, new_value_node.get()); + AddValueNodeToGraph(new_value_node); + convert_inputs.emplace_back(new_value_node); + } + if (RemoveValueNodeFromGraph(value_node)) { + MS_LOG(WARNING) << "failed to remove the value_node " << value_node->DebugString(); + } + return convert_inputs; +} + +ValueNodePtr KernelGraph::NewValueNode(const ValueNodePtr &value_node) { + MS_EXCEPTION_IF_NULL(value_node); + ValueNodePtr new_value_node = std::make_shared(value_node->value()); + new_value_node->set_abstract(value_node->abstract()); + // create kernel_info fo new value node + auto kernel_info = std::make_shared(); + new_value_node->set_kernel_info(kernel_info); + // create kernel_build_info for new value node + auto kernel_build_info_builder = std::make_shared(); + // set the format of value_node to DEFAULT_FORMAT + kernel_build_info_builder->SetOutputsFormat(std::vector{kOpFormat_DEFAULT}); + // set value node initial device data type = infer data type + std::vector types; + for (size_t index = 0; index < AnfAlgo::GetOutputTensorNum(value_node); ++index) { + types.push_back(kTypeUnknown); + } + kernel_build_info_builder->SetOutputsDeviceType(types); + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), new_value_node.get()); + AnfAlgo::SetGraphId(graph_id_, new_value_node.get()); + return new_value_node; +} + +const std::vector &KernelGraph::inputs() const { + MS_EXCEPTION_IF_NULL(inputs_); + return *inputs_; +} + +void KernelGraph::FrontBackendlMapAdd(const AnfNodePtr &front_anf, const AnfNodePtr &backend_anf) { + MS_EXCEPTION_IF_NULL(front_anf); + MS_EXCEPTION_IF_NULL(backend_anf); + if (front_backend_anf_map_.find(front_anf) != front_backend_anf_map_.end()) { + MS_LOG(EXCEPTION) << "anf " << front_anf->DebugString() << " has been exist in the front_backend_anf_map_"; + } + if (backend_front_anf_map_.find(backend_anf) != backend_front_anf_map_.end()) { + MS_LOG(EXCEPTION) << "kernel " << backend_anf->DebugString() << "has been exist in the backend_front_anf_map_"; + } + front_backend_anf_map_[front_anf] = backend_anf; + backend_front_anf_map_[backend_anf] = front_anf; +} + +void KernelGraph::FrontBackendlMapUpdate(const AnfNodePtr &old_backend_anf, const AnfNodePtr &new_backend_anf) { + MS_EXCEPTION_IF_NULL(old_backend_anf); + MS_EXCEPTION_IF_NULL(new_backend_anf); + if (old_backend_anf.get() == new_backend_anf.get()) { + MS_LOG(EXCEPTION) << "old can't be same with new"; + } + if (backend_front_anf_map_.find(old_backend_anf) == backend_front_anf_map_.end()) { + MS_LOG(EXCEPTION) << "old_backend_anf " << old_backend_anf->DebugString() << " is not exist in the map"; + } + if (front_backend_anf_map_.find(backend_front_anf_map_[old_backend_anf]) == front_backend_anf_map_.end()) { + MS_LOG(EXCEPTION) << "anf is not exist in the mape ,old " << old_backend_anf->DebugString(); + } + front_backend_anf_map_[backend_front_anf_map_[old_backend_anf]] = new_backend_anf; + backend_front_anf_map_[new_backend_anf] = backend_front_anf_map_[old_backend_anf]; + // delete old kernel + (void)backend_front_anf_map_.erase(old_backend_anf); +} +// get kernel by anf +AnfNodePtr KernelGraph::GetBackendAnfByFrontAnf(const AnfNodePtr &front_anf) { + if (front_backend_anf_map_.find(front_anf) == front_backend_anf_map_.end()) { + return nullptr; + } + return front_backend_anf_map_[front_anf]; +} + +bool KernelGraph::BakcendNodeExistInFrontBackendMap(const AnfNodePtr &backend_anf) { + return backend_front_anf_map_.find(backend_anf) != backend_front_anf_map_.end(); +} + +ValueNodePtr KernelGraph::GetValueNodeByTensor(const mindspore::tensor::TensorPtr &tensor) { + if (tensor_to_value_node_map_.find(tensor) == tensor_to_value_node_map_.end()) { + return nullptr; + } + return tensor_to_value_node_map_[tensor]; +} + +void KernelGraph::TensorValueNodeMapAdd(const tensor::TensorPtr &tensor, const ValueNodePtr &value_node) { + MS_EXCEPTION_IF_NULL(tensor); + MS_EXCEPTION_IF_NULL(value_node); + tensor_to_value_node_map_[tensor] = value_node; +} + +void KernelGraph::AddDependEdge(const AnfNodePtr &node, const AnfNodePtr &input, size_t depend_edge_num) { + MS_LOG(DEBUG) << "input:" << input->DebugString() << ", node:" << node->DebugString() << ",num:" << depend_edge_num; + auto output_depend_edge = std::pair(node, depend_edge_num); + // add output depend eddge of input + auto output_it = node_output_edges_.find(input); + if (output_it == node_output_edges_.end()) { + node_output_edges_[input] = std::vector>{output_depend_edge}; + } else { + output_it->second.push_back(output_depend_edge); + } + // add input depend edge of output + auto input_depend_edge = std::pair(input, depend_edge_num); + auto input_it = node_input_edges_.find(node); + if (input_it == node_input_edges_.end()) { + node_input_edges_[node] = std::vector>{input_depend_edge}; + } else { + input_it->second.push_back(input_depend_edge); + } + // add the depend sum of node + auto depend_it = node_output_num_.find(input); + if (depend_it == node_output_num_.end()) { + node_output_num_[input] = 0; + } + node_output_num_[input] += depend_edge_num; +} + +std::vector KernelGraph::GetOutputNodes(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + auto it = node_output_edges_.find(node); + if (it == node_output_edges_.end()) { + MS_LOG(EXCEPTION) << "can'f find node[" << node->DebugString() << "]"; + } + std::vector output_nodes; + auto trans = [](const std::pair &pair) -> AnfNodePtr { return pair.first; }; + (void)std::transform(it->second.begin(), it->second.end(), std::back_inserter(output_nodes), trans); + return output_nodes; +} + +// update the depend relations of control depend +void KernelGraph::UpdateControlDependRelations(const std::vector &depends) { + for (const auto &node : depends) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!AnfAlgo::CheckPrimitiveType(node, prim::kPrimControlDepend)) { + MS_LOG(EXCEPTION) << node->DebugString() << " is not a control depend"; + } + auto prior_node = cnode->input(kControlDependPriorIndex); + auto depend_node = cnode->input(kControlDependBehindIndex); + MS_EXCEPTION_IF_NULL(prior_node); + MS_EXCEPTION_IF_NULL(depend_node); + std::vector prior_nodes = {prior_node}; + std::vector depend_nodes = {depend_node}; + MS_LOG(INFO) << "prior node[" << prior_node->DebugString() << "],depend node[" << depend_node->DebugString() + << "],depend_mode=[" << AnfAlgo::GetNodeAttr(cnode, "depend_mode") << "]"; + if (prior_node->isa()) { + prior_nodes = GetOutputNodes(prior_node); + } + if (depend_node->isa()) { + depend_nodes = GetOutputNodes(depend_node); + } + for (auto &first_node : prior_nodes) { + for (auto &second_node : depend_nodes) { + MS_EXCEPTION_IF_NULL(first_node); + MS_EXCEPTION_IF_NULL(second_node); + MS_LOG(INFO) << "add first node:" << first_node->DebugString() << ",second node:" << second_node->DebugString(); + AddDependEdge(second_node, first_node, 1); + } + } + } +} + +bool KernelGraph::HandleControlDependNode(const AnfNodePtr &node, std::queue *que, + std::unordered_set *visited_nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return false; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (!AnfAlgo::CheckPrimitiveType(node, prim::kPrimControlDepend)) { + return false; + } + // set the control depend visited but don't push it into the que + if (visited_nodes->find(node) != visited_nodes->end()) { + MS_LOG(EXCEPTION) << "control depend[" << node->DebugString() << "] has been handled before"; + } + (void)visited_nodes->insert(cnode); + // add a 0 depend num to keep the link relations to prepare for finding zero output nodes + auto prior_node = cnode->input(kControlDependPriorIndex); + auto depend_node = cnode->input(kControlDependBehindIndex); + for (const auto &input : cnode->inputs()) { + AddDependEdge(node, input, 0); + } + PushNoVisitedNode(depend_node, que, visited_nodes); + PushNoVisitedNode(prior_node, que, visited_nodes); + return true; +} + +void KernelGraph::BfsToUpdateNodeOutput() { + node_output_edges_.clear(); + node_output_num_.clear(); + node_input_edges_.clear(); + std::vector control_depends; + std::unordered_set visited_nodes; + std::queue que; + que.push(get_return()); + while (!que.empty()) { + auto node = que.front(); + que.pop(); + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + // handle data links + for (const auto &input : cnode->inputs()) { + size_t dpend_edge_num = 1; + // handle control depend,all inputs of control depend has no depend edge + if (HandleControlDependNode(input, &que, &visited_nodes)) { + control_depends.push_back(input); + dpend_edge_num = 0; + } + // the 2rd input of depend is no depend edge + if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimDepend) && input == cnode->input(kDependAttachNodeIndex)) { + dpend_edge_num = 0; + } + PushNoVisitedNode(input, &que, &visited_nodes); + AddDependEdge(node, input, dpend_edge_num); + } + } + UpdateControlDependRelations(control_depends); +} + +void KernelGraph::AddValueNodeToGraph(const ValueNodePtr &value_node) { (void)graph_value_nodes_.insert(value_node); } + +bool KernelGraph::IsInRefOutputMap(const AnfWithOutIndex &pair) const { return ref_out_in_map_.count(pair) != 0; } + +AnfWithOutIndex KernelGraph::GetRefCorrespondOutput(const AnfWithOutIndex &out_pair) const { + if (!IsInRefOutputMap(out_pair)) { + MS_LOG(EXCEPTION) << "out_pair is not in RefOutputMap"; + } + return ref_out_in_map_.at(out_pair); +} + +void KernelGraph::AddRefCorrespondPairs(const AnfWithOutIndex &final_pair, const AnfWithOutIndex &origin_pair) { + if (IsInRefOutputMap(final_pair)) { + MS_LOG(EXCEPTION) << "out_pair is already in RefOutputMap"; + } + (void)ref_out_in_map_.insert(std::make_pair(final_pair, origin_pair)); +} + +bool KernelGraph::RemoveValueNodeFromGraph(const ValueNodePtr &value_node) { + if (graph_value_nodes_.find(value_node) != graph_value_nodes_.end()) { + (void)graph_value_nodes_.erase(value_node); + return true; + } + return false; +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/kernel_graph.h b/mindspore/ccsrc/session/kernel_graph.h new file mode 100644 index 00000000000..d94638aa995 --- /dev/null +++ b/mindspore/ccsrc/session/kernel_graph.h @@ -0,0 +1,125 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_KERNEL_GRAPH_H +#define MINDSPORE_CCSRC_SESSION_KERNEL_GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ir/func_graph.h" +#include "ir/anf.h" +#include "utils/graph_utils.h" + +namespace mindspore { +namespace session { +using AnfWithOutIndex = std::pair; +class KernelGraph : public FuncGraph { + public: + KernelGraph() : graph_id_(0) { + inputs_ = std::make_shared>(); + execution_order_ = {}; + executable_ = true; + } + ~KernelGraph() override = default; + + MS_DECLARE_PARENT(KernelGraph, FuncGraph); + + const std::vector &inputs() const; + std::vector *MutableInputs() const { return inputs_.get(); } + std::vector outputs() const; + CNodePtr NewCNode(const std::vector &inputs) override; + CNodePtr NewCNode(const CNodePtr &cnode); + ParameterPtr NewParameter(const ParameterPtr ¶meter = nullptr); + ValueNodePtr NewValueNode(const ValueNodePtr &value_node = nullptr); + std::vector SplitTupleValueNodeToNodeList(const ValueNodePtr &value_node); + void set_execution_order(const std::vector &order) { execution_order_ = order; } + const std::vector &execution_order() const { return execution_order_; } + void SetExecOrderByDefault(); + uint32_t graph_id() const { return graph_id_; } + void set_graph_id(uint32_t graph_id) { graph_id_ = graph_id; } + + // and a new front to backend anf relation to maop + void FrontBackendlMapAdd(const AnfNodePtr &front_anf, const AnfNodePtr &backend_anf); + // replace old backend anf with new backend anf + void FrontBackendlMapUpdate(const AnfNodePtr &old_backend_anf, const AnfNodePtr &new_backend_anf); + // get backend anf by front anf + AnfNodePtr GetBackendAnfByFrontAnf(const AnfNodePtr &front_anf); + // check backend node wheteher exist in map + bool BakcendNodeExistInFrontBackendMap(const AnfNodePtr &backend_anf); + // get value node by tensor + ValueNodePtr GetValueNodeByTensor(const tensor::TensorPtr &tensor); + // add value node tensor relation map + void TensorValueNodeMapAdd(const tensor::TensorPtr &tensor, const ValueNodePtr &value_node); + // get all value nodes of graph + std::unordered_set graph_value_nodes() { return graph_value_nodes_; } + // add value node to graph + void AddValueNodeToGraph(const ValueNodePtr &value_node); + // ref output is in map + bool IsInRefOutputMap(const AnfWithOutIndex &pair) const; + // get ref correspond pairs + AnfWithOutIndex GetRefCorrespondOutput(const AnfWithOutIndex &out_pair) const; + // add ref correspond pairs + void AddRefCorrespondPairs(const AnfWithOutIndex &final_pair, const AnfWithOutIndex &origin_pair); + // get map + std::map GetRefMap() const { return ref_out_in_map_; } + // checkout whether loop exist in graph + void CheckLoop(); + // check whether graph is executable + bool executable() const { return executable_; } + // set executable of graph + void set_executable(bool executable) { executable_ = executable; } + + private: + // remove value node form graph + bool RemoveValueNodeFromGraph(const ValueNodePtr &value_node); + // BFS to update all nodes' output + void BfsToUpdateNodeOutput(); + // add node depend edge by data edge or control depend + void AddDependEdge(const AnfNodePtr &node, const AnfNodePtr &input, size_t depend_edge_num); + // handle control depend + std::vector GetOutputNodes(const AnfNodePtr &node); + bool HandleControlDependNode(const AnfNodePtr &node, std::queue *que, + std::unordered_set *visited_nodes); + void UpdateControlDependRelations(const std::vector &depends); + + std::shared_ptr> inputs_; + std::vector execution_order_; + uint32_t graph_id_; + + // record map bettween front anf and backend anf,use two map implement bidirectional map + std::unordered_map front_backend_anf_map_; + std::unordered_map backend_front_anf_map_; + // there may be a tensor from ME backend ,a value ndoe will be create according the tensor,map record + std::unordered_map tensor_to_value_node_map_; + // include all value nodes + std::unordered_set graph_value_nodes_; + std::unordered_map node_output_num_; + std::unordered_map>> node_input_edges_; + // record map between ref final output anf with index and ref origin input with index + std::map ref_out_in_map_; + std::unordered_map>> node_output_edges_; + // graph needn't execute + bool executable_; +}; +} // namespace session +using KernelGraphPtr = std::shared_ptr; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_SESSION_KERNEL_GRAPH_H diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc new file mode 100644 index 00000000000..1d786708a2a --- /dev/null +++ b/mindspore/ccsrc/session/session_basic.cc @@ -0,0 +1,748 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/session_basic.h" +#include +#include +#include +#include "pipeline/parse/data_converter.h" +#include "ir/manager.h" +#include "operator/ops.h" +#include "utils/context/ms_context.h" +#include "utils/config_manager.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/oplib/oplib.h" +#include "pre_activate/common/common_backend_optimization.h" +#include "pre_activate/common/helper.h" +#include "common/utils.h" +#include "ir/dtype.h" + +namespace mindspore { +namespace session { +namespace { +const int kSummaryGetItem = 2; +void GetSummaryNodes(const KernelGraph *graph, std::unordered_map> *summary) { + MS_LOG(DEBUG) << "Update summary Start"; + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(summary); + summary->clear(); + auto apply_list = TopoSort(graph->get_return()); + for (auto &n : apply_list) { + MS_EXCEPTION_IF_NULL(n); + if (IsPrimitiveCNode(n, prim::kPrimScalarSummary) || IsPrimitiveCNode(n, prim::kPrimTensorSummary) || + IsPrimitiveCNode(n, prim::kPrimImageSummary)) { + int index = 0; + auto cnode = n->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().size() <= kSummaryGetItem) { + MS_LOG(EXCEPTION) << "the node Summary should have 2 inputs at least!"; + } + auto node = cnode->input(kSummaryGetItem); + MS_EXCEPTION_IF_NULL(node); + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + auto c = node->cast(); + MS_EXCEPTION_IF_NULL(c); + if (c->inputs().size() != kTupleGetItemInputSize) { + MS_LOG(EXCEPTION) << "the node tuple_get_item must have 2 inputs!"; + } + MS_EXCEPTION_IF_NULL(c->input(kInputNodeOutputIndexInTupleGetItem)); + auto value_node = c->input(kInputNodeOutputIndexInTupleGetItem)->cast(); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + Int32ImmPtr int_imm_ptr = value->cast(); + MS_EXCEPTION_IF_NULL(int_imm_ptr); + index = int_imm_ptr->value(); + node = c->input(kRealInputNodeIndexInTupleGetItem); + } + std::pair output_pair(node, index); + // get full name with scope will add scalar or tensor or image summary tag. + (*summary)[n->fullname_with_scope()] = output_pair; + } + } + MS_LOG(DEBUG) << "Update summary end size: " << (*summary).size(); +} + +bool ExistSummaryNode(const KernelGraph *graph) { + auto ret = graph->get_return(); + MS_EXCEPTION_IF_NULL(ret); + auto all_nodes = DeepLinkedGraphSearch(ret); + for (auto &n : all_nodes) { + if (IsPrimitiveCNode(n, prim::kPrimScalarSummary) || IsPrimitiveCNode(n, prim::kPrimTensorSummary) || + IsPrimitiveCNode(n, prim::kPrimImageSummary)) { + return true; + } + } + return false; +} + +BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const KernelGraph &graph, + const std::vector &input_tensors) { + MS_EXCEPTION_IF_NULL(node); + MS_LOG(INFO) << "create tensor for output[" << node->DebugString() << "] index[" << output_index << "]"; + // if node is a value node, no need sync addr from device to host + if (!AnfAlgo::OutputAddrExist(node, output_index)) { + if (node->isa()) { + auto value_node = node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + return value_node->value(); + } + if (node->isa()) { + for (size_t input_idx = 0; input_idx < graph.inputs().size(); input_idx++) { + if (input_idx > input_tensors.size()) { + MS_LOG(EXCEPTION) << "input idx:" << input_idx << "out of range:" << input_tensors.size(); + } + if (graph.inputs()[input_idx] == node) { + return input_tensors[input_idx]; + } + } + MS_LOG(EXCEPTION) << "parameter : " << node->DebugString() << "has no output addr"; + } + } + // if proccess reach here,it remarks item_with_index is a real node(Parameter,or executable CNode) + auto address = AnfAlgo::GetOutputAddr(node, output_index); + MS_EXCEPTION_IF_NULL(address); + auto shape = AnfAlgo::GetOutputInferShape(node, output_index); + TypeId type_id = kNumberTypeFloat32; + type_id = AnfAlgo::GetOutputInferDataType(node, output_index); + std::vector temp_shape; + (void)std::copy(shape.begin(), shape.end(), std::back_inserter(temp_shape)); + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + // if in paynative mode,data only copyed to host when user want to print data + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->enable_pynative_infer()) { + tensor->set_device_address(AnfAlgo::GetMutableOutputAddr(node, output_index)); + } else if (!address->SyncDeviceToHost(tensor->shape(), LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c(true))) { + MS_LOG(INFO) << "output sync device to host error!!!"; + tensor->set_dirty(false); + } + return tensor; +} + +BaseRef CreatTensorForOutput(const AnfNodePtr &anf, const KernelGraph &graph, + const std::vector &input_tensors) { + MS_EXCEPTION_IF_NULL(anf); + MS_LOG(INFO) << "create tensor for output[" << anf->DebugString() << "]"; + auto item_with_index = AnfAlgo::VisitKernelWithReturnType(anf, 0); + MS_EXCEPTION_IF_NULL(item_with_index.first); + // special handle for maketuple + if (AnfAlgo::CheckPrimitiveType(item_with_index.first, prim::kPrimMakeTuple)) { + auto cnode = item_with_index.first->cast(); + MS_EXCEPTION_IF_NULL(cnode); + VectorRef ret; + for (size_t i = 1; i < cnode->inputs().size(); ++i) { + auto out = CreatTensorForOutput(cnode->input(i), graph, input_tensors); + ret.push_back(out); + } + return ret; + } + // if is graph return nothing ,the function should return a null anylist + size_t size = AnfAlgo::GetOutputTensorNum(item_with_index.first); + if (size == 0) { + return VectorRef(); + } + return CreateOneTensor(item_with_index.first, item_with_index.second, graph, input_tensors); +} + +BaseRef CreatTupleForOutput(const AnfNodePtr &anf, const KernelGraph &graph, + const std::vector &input_tensors) { + MS_EXCEPTION_IF_NULL(anf); + if (!AnfAlgo::IsRealKernel(anf)) { + MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] should be a executable kernel"; + } + if (anf->isa()) { + return CreateOneTensor(anf, 0, graph, input_tensors); + } + VectorRef ret; + if (anf->isa() && AnfAlgo::GetCNodeName(anf) != prim::kPrimMakeTuple->name()) { + for (size_t i = 0; i < AnfAlgo::GetOutputTensorNum(anf); ++i) { + auto out = CreateOneTensor(anf, i, graph, input_tensors); + ret.emplace_back(out); + } + } + return ret; +} + +std::string FindOpInputParameterType(const std::string &op_name, kernel::OpImplyType implyType, size_t index) { + std::string para_type; + auto op_info = kernel::OpLib::FindOp(op_name, implyType); + if (op_info == nullptr) { + return para_type; + } + auto op_inputs_info_vec = op_info->inputs_ptr(); + if (index >= op_inputs_info_vec.size()) { + return para_type; + } + auto op_io_info = op_inputs_info_vec[index]; + MS_EXCEPTION_IF_NULL(op_io_info); + para_type = op_io_info->param_type(); + return para_type; +} + +void RunOpConvertConstInputToAttr(const OpRunInfo &op_run_info, const std::shared_ptr &cnode) { + MS_EXCEPTION_IF_NULL(cnode); + auto op_inputs = op_run_info.op_inputs; + // get input names vector from attrs + auto primitive = AnfAlgo::GetCNodePrimitive(cnode); + MS_EXCEPTION_IF_NULL(primitive); + auto input_names_value = primitive->GetAttr(kAttrInputNames); + if (input_names_value == nullptr) { + return; + } + auto input_names_vec = GetValue>(input_names_value); + // convert const input to attr + size_t input_num = op_inputs.size(); + if (input_num != input_names_vec.size()) { + MS_LOG(EXCEPTION) << "input name number " << input_names_vec.size() << "is not equal to input value number " + << input_num; + } + for (size_t index = 0; index < input_num; ++index) { + // skip tensor + if (py::isinstance(op_inputs[index])) { + continue; + } + // convert to attr + auto para_type = FindOpInputParameterType(op_run_info.op_name, kernel::OpImplyType::kTBE, index); + if (!para_type.empty() && para_type == kAttrDynInput) { + auto tuple_inputs = py::cast(op_inputs[index]); + primitive->set_attr(kAttrDynInputSizes, MakeValue(std::vector{SizeToInt(tuple_inputs.size())})); + continue; + } + ValuePtr value = parse::data_converter::PyDataToValue(op_inputs[index]); + MS_EXCEPTION_IF_NULL(value); + auto input_name = input_names_vec[index]; + // set the input node as attr of the cnode, key is name of input node,value is input node's value + primitive->set_attr(input_name, value); + } +} + +ValueNodePtr CreateNewValueNode(const AnfNodePtr &anf, KernelGraph *graph) { + auto value_node = anf->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + MS_EXCEPTION_IF_NULL(value); + if (value->isa()) { + return nullptr; + } + auto new_value_node = graph->NewValueNode(value_node); + graph->FrontBackendlMapAdd(anf, new_value_node); + graph->AddValueNodeToGraph(new_value_node); + return new_value_node; +} + +ParameterPtr CreateNewParameterFromParameter(const AnfNodePtr &anf, KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(anf); + if (!anf->isa()) { + MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] is not a parameter"; + } + auto graph_inputs = graph->MutableInputs(); + MS_EXCEPTION_IF_NULL(graph_inputs); + ParameterPtr new_parameter = graph->NewParameter(anf->cast()); + graph->FrontBackendlMapAdd(anf, new_parameter); + graph_inputs->push_back(new_parameter); + return new_parameter; +} + +std::vector CreateParameterFromTuple(const AnfNodePtr &node, KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(graph); + std::vector parameters; + std::vector pre_graph_out = AnfAlgo::GetAllOutput(node, {prim::kPrimTupleGetItem}); + auto create_parameter = [&](const AbstractBasePtr &abstract) -> void { + auto parameter = graph->NewParameter(); + MS_EXCEPTION_IF_NULL(parameter); + parameter->set_abstract(abstract); + parameters.push_back(graph->NewParameter(parameter)); + }; + for (const auto &out_node : pre_graph_out) { + MS_EXCEPTION_IF_NULL(out_node); + auto abstract = out_node->abstract(); + MS_EXCEPTION_IF_NULL(abstract); + // create multiple parameters if is a tuple output real kernel + if (abstract->isa() && !AnfAlgo::CheckPrimitiveType(out_node, prim::kPrimTupleGetItem)) { + auto tuple_abstract = abstract->cast(); + MS_EXCEPTION_IF_NULL(tuple_abstract); + MS_LOG(INFO) << "tuple_size [" << tuple_abstract->size() << "]"; + for (size_t output_idx = 0; output_idx < tuple_abstract->size(); output_idx++) { + create_parameter((*tuple_abstract)[output_idx]); + } + continue; + } + // creata single parameter if is a abstract real kernel + create_parameter(out_node->abstract()); + } + return parameters; +} + +AnfNodePtr CreateNewParameterFromCNode(const AnfNodePtr &anf, KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(anf); + if (!anf->isa()) { + MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] is not a cnode"; + } + MS_LOG(INFO) << "create a new parameter from cnode[" << anf->DebugString() << "]"; + auto parameters = CreateParameterFromTuple(anf, graph); + auto graph_inputs = graph->MutableInputs(); + MS_EXCEPTION_IF_NULL(graph_inputs); + (void)std::copy(parameters.begin(), parameters.end(), std::back_inserter(*graph_inputs)); + if (parameters.empty()) { + MS_LOG(EXCEPTION) << "no parameter exist!!"; + } + if (parameters.size() == 1) { + return parameters[0]; + } + std::vector make_tuple_input = {NewValueNode(prim::kPrimMakeTuple)}; + (void)std::copy(parameters.begin(), parameters.end(), std::back_inserter(make_tuple_input)); + auto make_tuple = graph->NewCNode(make_tuple_input); + MS_EXCEPTION_IF_NULL(make_tuple); + MS_LOG(INFO) << "new make tuple [" << make_tuple->DebugString() << "] of parameters"; + return make_tuple; +} + +bool NeedInsertSwitch() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + return (context_ptr->enable_task_sink() && context_ptr->loop_sink_flag() && + ConfigManager::GetInstance().iter_num() > 1); +} + +size_t LoadCtrlInputTensor(const std::shared_ptr &context, std::vector *inputs) { + MS_EXCEPTION_IF_NULL(context); + if (!NeedInsertSwitch()) { + (void)context->results_.erase(kInputCtrlTensors); + return 0; + } + MS_LOG(INFO) << "Load kInputCtrlTensors"; + auto inputs_params = + context->GetResult(kInputCtrlTensors).cast>>(); + MS_EXCEPTION_IF_NULL(inputs_params); + if (inputs_params->empty()) { + MS_LOG(EXCEPTION) << "Illegal empty inputs_params"; + } + auto tensor = (*inputs_params)[0]; + MS_EXCEPTION_IF_NULL(tensor); + auto *val = static_cast(tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(val); + *val = 0; + tensor->set_dirty(true); + // set loop_count to zero + MS_EXCEPTION_IF_NULL(inputs); + inputs->push_back(tensor); + return inputs_params->size(); +} + +ParameterPtr ConstructRunOpParameter(const std::shared_ptr &graph, const tensor::TensorPtr &input_tensor, + bool is_weight) { + auto param = graph->NewParameter(); + MS_EXCEPTION_IF_NULL(param); + if (is_weight) { + py::object obj; + param->set_default_param(obj); + } + // set the kernel info of parameter + auto kernel_build_info_builder = std::make_shared(); + MS_EXCEPTION_IF_NULL(input_tensor); + if (input_tensor->device_address().get() == nullptr) { + kernel_build_info_builder->SetOutputsFormat(std::vector{kOpFormat_DEFAULT}); + TypeId param_init_data_type = AnfAlgo::IsParameterWeight(param) ? kTypeUnknown : input_tensor->data_type(); + kernel_build_info_builder->SetOutputsDeviceType(std::vector{param_init_data_type}); + } else { + kernel_build_info_builder->SetOutputsFormat(std::vector{input_tensor->device_address()->format()}); + kernel_build_info_builder->SetOutputsDeviceType(std::vector{input_tensor->device_address()->type_id()}); + } + AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), param.get()); + // construct abstract of parameter + auto abstract = std::make_shared(input_tensor); + param->set_abstract(abstract); + return param; +} + +void DumpGraphOutput(const Any &any, size_t recurse_level = 0) { + MS_LOG(INFO) << "graph outputs:"; + const size_t max_deep = 10; + if (recurse_level > max_deep) { + MS_LOG(INFO) << "recurse too deep"; + return; + } + std::string tab_str; + for (size_t i = 0; i < recurse_level; i++) { + tab_str = tab_str.append(" "); + } + if (any.is()) { + (void)tab_str.append("{"); + MS_LOG(INFO) << tab_str; + auto any_list = any.cast(); + for (auto &it : any_list) { + DumpGraphOutput(it, recurse_level + 1); + } + (void)tab_str.append("}"); + MS_LOG(INFO) << tab_str; + } + (void)tab_str.append(any.ToString()); + MS_LOG(INFO) << tab_str; +} +} // namespace + +GraphId SessionBasic::graph_sum_ = 0; + +CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(cnode); + MS_EXCEPTION_IF_NULL(graph); + // get primitive of old node + auto prim = AnfAlgo::GetCNodePrimitive(cnode); + MS_EXCEPTION_IF_NULL(prim); + // push attr to inputs[0] of new cnode + std::vector cnode_inputs = {std::make_shared(std::make_shared(*prim))}; + for (size_t input_idx = 1; input_idx < cnode->inputs().size(); input_idx++) { + auto anf = cnode->inputs()[input_idx]; + MS_EXCEPTION_IF_NULL(anf); + // anf has been created before + if (graph->GetBackendAnfByFrontAnf(anf) != nullptr) { + cnode_inputs.emplace_back(graph->GetBackendAnfByFrontAnf(anf)); + continue; + } else if (anf->isa() && !IsValueNode(anf)) { + // if input is a value ndoe, + auto new_value_node = CreateNewValueNode(anf, graph); + if (new_value_node != nullptr) { + cnode_inputs.emplace_back(new_value_node); + } + continue; + } else if (anf->isa()) { + // if anf is a parameter + cnode_inputs.emplace_back(CreateNewParameterFromParameter(anf, graph)); + continue; + } else if (anf->isa()) { + // the input node is a cnode from other graph + cnode_inputs.emplace_back(CreateNewParameterFromCNode(anf, graph)); + continue; + } + MS_LOG(EXCEPTION) << "unexpected input[" << anf->DebugString() << "]"; + } + return graph->NewCNode(cnode_inputs); +} + +KernelGraphPtr SessionBasic::ConstructKernelGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) { + auto graph = std::make_shared(); + graph->set_graph_id(graph_sum_); + for (const auto &node : lst) { + MS_EXCEPTION_IF_NULL(node); + MS_LOG(DEBUG) << "start create new cnode,node = " << node->DebugString(); + if (!node->isa()) { + MS_LOG(EXCEPTION) << "Inst node " << node->DebugString() << " is not CNode"; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + TraceManager::DebugTrace(std::make_shared(cnode->debug_info())); + // create a new cnode object + auto new_cnode = CreateNewCNode(cnode, graph.get()); + MS_EXCEPTION_IF_NULL(new_cnode); + new_cnode->set_abstract(cnode->abstract()); + new_cnode->set_scope(cnode->scope()); + // record map relations between anf from ME and new anf node used in backend + graph->FrontBackendlMapAdd(node, new_cnode); + TraceManager::EndTrace(); + } + // add a make_tuple at the end of graph as output + graph->set_output(ConstructOutput(outputs, graph)); + MS_EXCEPTION_IF_NULL(context_); + FuncGraphManagerPtr manager = context_->manager(); + if (manager) { + manager->AddFuncGraph(graph); + graph->set_manager(manager); + } + graph->SetExecOrderByDefault(); + opt::BackendCommonOptimization(graph); + graphs_[graph_sum_++] = graph; + return graph; +} + +// run graph steps +void SessionBasic::LoadInputData(const std::shared_ptr &kernel_graph, + const std::vector &inputs_const) const { + MS_EXCEPTION_IF_NULL(kernel_graph); + std::vector inputs(inputs_const); + size_t input_ctrl_size = 1; + MS_EXCEPTION_IF_NULL(context_); + if (context_->HasResult(kInputCtrlTensors)) { + input_ctrl_size = LoadCtrlInputTensor(context_, &inputs); + } + MS_EXCEPTION_IF_NULL(kernel_graph); + auto input_nodes = kernel_graph->inputs(); + if ((inputs.size() + input_ctrl_size) - 1 != input_nodes.size()) { + MS_LOG(EXCEPTION) << "tensor input size:" << inputs.size() + << " is not equal graph inputs size:" << input_nodes.size() + << ", input_ctrl_size:" << input_ctrl_size; + } + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + for (size_t i = 0; i < inputs.size(); ++i) { + auto tensor = inputs[i]; + MS_EXCEPTION_IF_NULL(tensor); + auto input_node = input_nodes[i]; + MS_EXCEPTION_IF_NULL(input_node); + if (input_node->isa() && AnfAlgo::OutputAddrExist(input_node, 0)) { + auto pk_node = input_node->cast(); + auto device_address = AnfAlgo::GetMutableOutputAddr(pk_node, 0); + bool need_sync = false; + if (ms_context->enable_pynative_infer()) { + if (tensor->device_address().get() == nullptr) { + need_sync = true; + } + } else { + if (tensor->is_dirty() || !AnfAlgo::IsParameterWeight(pk_node)) { + need_sync = true; + } else if (tensor->device_address() != device_address) { + (void)tensor->data_sync(); + need_sync = true; + } + } + if (need_sync) { + tensor->set_device_address(device_address); + MS_EXCEPTION_IF_NULL(device_address); + if (!device_address->SyncHostToDevice(tensor->shape(), LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c(false))) { + MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; + } + } + } + tensor->set_dirty(false); + } +} + +void SessionBasic::UpdateOutputs(const std::shared_ptr &kernel_graph, VectorRef *const outputs, + const std::vector &input_tensors) const { + MS_EXCEPTION_IF_NULL(kernel_graph); + MS_EXCEPTION_IF_NULL(outputs); + auto anf_outputs = kernel_graph->outputs(); + for (auto &item : anf_outputs) { + MS_LOG(INFO) << "update output[" << item->DebugString() << "]"; + MS_EXCEPTION_IF_NULL(item); + if (AnfAlgo::IsTupleOutput(item) && AnfAlgo::IsRealKernel(item)) { + outputs->emplace_back(CreatTupleForOutput(item, *kernel_graph, input_tensors)); + continue; + } + outputs->emplace_back(CreatTensorForOutput(item, *kernel_graph, input_tensors)); + } +} + +void SessionBasic::RegisterSummaryCallBackFunc(const CallBackFunc &callback) { + MS_EXCEPTION_IF_NULL(callback); + summary_callback_ = callback; +} + +void SessionBasic::Reorder(std::vector *node_list) { + MS_EXCEPTION_IF_NULL(node_list); + std::vector all_opt_list; + std::vector non_opt_list; + + for (const auto &node : *node_list) { + MS_EXCEPTION_IF_NULL(node); + if (kOptOpeatorSet.find(AnfAlgo::GetCNodeName(node)) != kOptOpeatorSet.end()) { + all_opt_list.emplace_back(node); + } else { + non_opt_list.emplace_back(node); + } + } + node_list->clear(); + (void)std::copy(non_opt_list.begin(), non_opt_list.end(), std::back_inserter(*node_list)); + (void)std::copy(all_opt_list.begin(), all_opt_list.end(), std::back_inserter(*node_list)); +} + +void SessionBasic::Summary(KernelGraph *graph) { + if (summary_callback_ == nullptr) { + return; + } + MS_EXCEPTION_IF_NULL(graph); + bool exist_summary = ExistSummaryNode(graph); + if (!exist_summary) { + return; + } + std::unordered_map> summary_outputs; + GetSummaryNodes(graph, &summary_outputs); + std::map params_list; + // fetch outputs apply kernel in session & run callback functions + for (auto &output_item : summary_outputs) { + auto node = output_item.second.first; + size_t index = IntToSize(output_item.second.second); + auto address = AnfAlgo::GetOutputAddr(node, index); + auto shape = AnfAlgo::GetOutputInferShape(node, index); + TypeId type_id = AnfAlgo::GetOutputInferDataType(node, index); + std::vector temp_shape; + (void)std::copy(shape.begin(), shape.end(), std::back_inserter(temp_shape)); + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + MS_EXCEPTION_IF_NULL(address); + if (!address->SyncDeviceToHost(tensor->shape(), LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c(true))) { + MS_LOG(ERROR) << "Failed to sync output from device to host."; + } + tensor->set_dirty(false); + params_list[output_item.first] = tensor; + } + // call callback function here + summary_callback_(0, params_list); +} + +void SessionBasic::ToTensorPtr(const OpRunInfo &op_run_info, std::vector *inputs, + std::vector *tensor_mask) { + MS_EXCEPTION_IF_NULL(inputs); + MS_EXCEPTION_IF_NULL(tensor_mask); + if (op_run_info.op_inputs.size() != op_run_info.inputs_mask.size()) { + MS_LOG(EXCEPTION) << "op input size " << op_run_info.op_inputs.size() << " should be equal to op input mask size " + << op_run_info.inputs_mask.size(); + } + size_t input_num = op_run_info.op_inputs.size(); + // get tensors from op_inputs + for (size_t i = 0; i < input_num; ++i) { + tensor::TensorPtr tensor_ptr = nullptr; + auto param_type = FindOpInputParameterType(op_run_info.op_name, kernel::OpImplyType::kTBE, i); + if (py::isinstance(op_run_info.op_inputs[i])) { + tensor_ptr = py::cast(op_run_info.op_inputs[i]); + } else if (!param_type.empty() && param_type == kAttrDynInput) { + auto tuple_inputs = py::cast(op_run_info.op_inputs[i]); + for (auto &&tuple_input : tuple_inputs) { + tensor_ptr = py::cast(tuple_input); + MS_EXCEPTION_IF_NULL(tensor_ptr); + inputs->push_back(tensor_ptr); + tensor_mask->push_back(py::cast(op_run_info.inputs_mask[i])); + } + continue; + } else if (op_run_info.op_name == kApplyMomentumOpName && py::isinstance(op_run_info.op_inputs[i])) { + tensor_ptr = std::make_shared(py::cast(op_run_info.op_inputs[i]), kFloat32); + } + if (tensor_ptr != nullptr) { + inputs->push_back(tensor_ptr); + tensor_mask->push_back(py::cast(op_run_info.inputs_mask[i])); + } + } +} + +CNodePtr SessionBasic::ConstructOutput(const AnfNodePtrList &outputs, const std::shared_ptr &graph) { + MS_EXCEPTION_IF_NULL(graph); + std::vector output_args; + auto FindEqu = [graph](const AnfNodePtr &out) -> AnfNodePtr { + auto backend_anf = graph->GetBackendAnfByFrontAnf(out); + if (backend_anf != nullptr) { + return backend_anf; + } + MS_LOG(EXCEPTION) << "did not find the node in the equiv map!"; + }; + output_args.push_back(NewValueNode(prim::kPrimMakeTuple)); + (void)std::transform(outputs.begin(), outputs.end(), std::back_inserter(output_args), + [&](const AnfNodePtr &out) -> AnfNodePtr { return FindEqu(out); }); + return graph->NewCNode(output_args); +} + +void SessionBasic::CreateOutputNode(const CNodePtr &cnode, const std::shared_ptr &graph) { + MS_LOG(INFO) << "start"; + std::vector make_tuple_inputs; + make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + if (AnfRuntimeAlgorithm::GetOutputTensorNum(cnode) > 1) { + for (size_t output_index = 0; output_index < AnfRuntimeAlgorithm::GetOutputTensorNum(cnode); output_index++) { + auto idx = NewValueNode(SizeToInt(output_index)); + MS_EXCEPTION_IF_NULL(idx); + auto imm = std::make_shared(output_index); + idx->set_abstract(std::make_shared(imm)); + MS_EXCEPTION_IF_NULL(graph); + auto getitem = graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), cnode, idx}); + std::vector types = {AnfAlgo::GetOutputInferDataType(cnode, output_index)}; + std::vector> shapes = {AnfAlgo::GetOutputInferShape(cnode, output_index)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, getitem.get()); + make_tuple_inputs.push_back(getitem); + } + } else { + make_tuple_inputs.push_back(cnode); + } + // create output + auto g_output = graph->NewCNode(make_tuple_inputs); + graph->set_output(g_output); + // set graph manager,which now is only used to get valuendoes and hardware optimizing + MS_EXCEPTION_IF_NULL(context_); + FuncGraphManagerPtr manager = context_->manager(); + if (manager != nullptr) { + manager->AddFuncGraph(graph); + graph->set_manager(manager); + } + MS_LOG(INFO) << "end"; +} + +std::shared_ptr SessionBasic::ConstructSingleOpGraph(const OpRunInfo &op_run_info) { + auto graph = std::make_shared(); + std::vector inputs; + if (op_run_info.op_inputs.size() != op_run_info.inputs_mask.size()) { + MS_LOG(EXCEPTION) << "op_run_info inputs.size" << op_run_info.op_inputs.size() + << " should be equal to parameter_mask.size " << op_run_info.inputs_mask.size(); + } + // set input[0] + if (op_run_info.py_primitive == nullptr) { + inputs.push_back(std::make_shared(std::make_shared(op_run_info.op_name))); + } else { + inputs.push_back(std::make_shared(op_run_info.py_primitive)); + } + // set input parameter + std::vector input_tensors; + std::vector tensors_mask; + ToTensorPtr(op_run_info, &input_tensors, &tensors_mask); + MS_LOG(INFO) << "input tensor size" << input_tensors.size(); + if (input_tensors.size() != tensors_mask.size()) { + MS_LOG(EXCEPTION) << "input tensors size " << input_tensors.size() << " should be equal to tensors mask size " + << tensors_mask.size(); + } + for (size_t i = 0; i < input_tensors.size(); ++i) { + auto parameter = ConstructRunOpParameter(graph, input_tensors[i], tensors_mask[i]); + inputs.push_back(parameter); + graph->MutableInputs()->push_back(parameter); + } + // set execution order + auto cnode = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(cnode); + // set abstract,which include inferred shapes and types + cnode->set_abstract(op_run_info.abstract); + // set const input to attr if value is not a tensor,such as scalar or tuple + RunOpConvertConstInputToAttr(op_run_info, cnode); + // set exectuion order + std::vector exe_order = {cnode}; + graph->set_execution_order(exe_order); + // set output + CreateOutputNode(cnode, graph); + return graph; +} + +BaseRef SessionBasic::TransformBaseRefListToTuple(const BaseRef &base_ref) { + if (utils::isa(base_ref)) { + auto ref_list = utils::cast(base_ref); + py::tuple output_tensors(ref_list.size()); + for (size_t i = 0; i < ref_list.size(); ++i) { + auto output = TransformBaseRefListToTuple(ref_list[i]); // use pyObjectRef + if (utils::isa(output)) { + auto tensor_ptr = utils::cast(output); + MS_EXCEPTION_IF_NULL(tensor_ptr); + output_tensors[i] = tensor_ptr; + } else if (utils::isa(output)) { + py::object obj = utils::cast(output).object_; + py::tuple tensor_tuple = py::cast(obj); + output_tensors[i] = tensor_tuple; + } else { + MS_LOG(EXCEPTION) << "The output is not a base ref list or a tensor !"; + } + } + return output_tensors; // turn tuple to py::object and store in PyObjectRef + } else if (utils::isa(base_ref)) { + return base_ref; + } else { + MS_LOG(EXCEPTION) << "The output is not a base ref list or a tensor !"; + } +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/session_basic.h b/mindspore/ccsrc/session/session_basic.h new file mode 100644 index 00000000000..f80e69bd9ff --- /dev/null +++ b/mindspore/ccsrc/session/session_basic.h @@ -0,0 +1,116 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_SESSION_BASIC_H +#define MINDSPORE_CCSRC_SESSION_SESSION_BASIC_H + +#include +#include +#include +#include +#include +#include +#include "session/session_context.h" +#include "session/kernel_graph.h" +#include "ir/anf.h" +#include "ir/meta_tensor.h" +#include "utils/any.h" +#include "utils/base_ref.h" +#include "pynative/pynative_execute.h" +#include "device/kernel_info.h" + +namespace mindspore { +using GraphId = uint32_t; +using GraphInfo = std::string; +namespace session { +using CallBackFunc = uint32_t (*)(uint32_t graph_id, + const std::map ¶ms_list); +using AnyList = std::vector; +using AnyListPtr = std::shared_ptr; + +using OpRunInfo = pynative::OpExecInfo; +using OpRunInfoPtr = std::shared_ptr; + +class SessionBasic { + public: + SessionBasic() : device_id_(0) { + graphs_ = {}; + run_op_graphs_ = {}; + summary_callback_ = nullptr; + } + + virtual void Init(uint32_t device_id) { device_id_ = device_id; } + + virtual ~SessionBasic() { summary_callback_ = nullptr; } + + virtual GraphId CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) = 0; + // build graph ,used to handle mupltiple child graphs + virtual void BuildGraph(GraphId) {} + + virtual void RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *outputs) = 0; + + virtual void BuildOp(const OpRunInfo &, const GraphInfo &) {} + + virtual py::tuple RunOp(const OpRunInfo &, const GraphInfo &) { return py::tuple(); } + + virtual void RegisterSummaryCallBackFunc(const CallBackFunc &callback); + + std::shared_ptr ConstructKernelGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs); + + CNodePtr CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph); + + // set parameters of final graph + virtual GraphId SetFinalGraphInput(const std::vector &) { return kInvalidGraphId; } + // set output of final graph + virtual void SetFinalGraphOutput(const BaseRef &) {} + // insert switch and set the relative acitve ops + virtual void SwitchCompile(GraphId, GraphId, GraphId) {} + // set args of child graph.the arg maybe come from a output of other child graphs,or from final graph's parameter + virtual void SetChildGraphInput(GraphId, const VectorRef &) {} + // get graph id in child graphs by ME front anf node pointer + virtual GraphId GetGraphIdByNode(const AnfNodePtr &) const { return kInvalidGraphId; } + virtual GraphId GetFinalRunGraph() const { return kInvalidGraphId; } + virtual void SetActive(GraphId, GraphId) {} + + protected: + virtual void LoadInputData(const std::shared_ptr &kernel_graph, + const std::vector &inputs_const) const; + void UpdateOutputs(const std::shared_ptr &kernel_graph, VectorRef *const outputs, + const std::vector &input_tensors) const; + void Reorder(std::vector *node_list); + void Summary(KernelGraph *graph); + // create graph output for RunOp + void CreateOutputNode(const CNodePtr &cnode, const std::shared_ptr &graph); + CNodePtr ConstructOutput(const AnfNodePtrList &outputs, const std::shared_ptr &graph); + // create a single run op graph + std::shared_ptr ConstructSingleOpGraph(const OpRunInfo &op_run_info); + // get tensors from op inputs + void ToTensorPtr(const OpRunInfo &op_run_info, std::vector *inputs, + std::vector *tensor_mask); + // trans BaseRef list to py::tuple + BaseRef TransformBaseRefListToTuple(const BaseRef &base_ref); + + std::unordered_map> graphs_; + std::unordered_map> run_op_graphs_; + std::shared_ptr context_; + CallBackFunc summary_callback_; + static GraphId graph_sum_; + uint32_t device_id_; +}; + +using SessionPtr = std::shared_ptr; +} // namespace session +} // namespace mindspore +#endif // MINDSPORE_CCSRC_SESSION_SESSION_BASIC_H diff --git a/mindspore/ccsrc/session/session_context.cc b/mindspore/ccsrc/session/session_context.cc new file mode 100644 index 00000000000..2b6ebf6b84f --- /dev/null +++ b/mindspore/ccsrc/session/session_context.cc @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/session_context.h" +namespace mindspore { +namespace session { +std::shared_ptr Context::GetInstance() { + static std::shared_ptr context_singleton = std::make_shared(); + return context_singleton; +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/session_context.h b/mindspore/ccsrc/session/session_context.h new file mode 100644 index 00000000000..ccf3faf55f2 --- /dev/null +++ b/mindspore/ccsrc/session/session_context.h @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_SESSION_CONTEXT_H +#define MINDSPORE_CCSRC_SESSION_SESSION_CONTEXT_H +#include +#include +#include +#include +#include +#include + +#include "ir/meta_tensor.h" +#include "pipeline/resource.h" +#include "utils/context/ms_context.h" +namespace mindspore { +namespace session { +const char kInputCtrlTensors[] = "input_ctrl_tensors"; + +class Context : public pipeline::ResourceBase { + public: + explicit Context(std::string target = kAscendDevice, uint32_t device_id = 0) + : target_(std::move(target)), device_id_(device_id) {} + ~Context() override = default; + + uint32_t device_id() const { return device_id_; } + static std::shared_ptr GetInstance(); + + private: + std::string target_; + uint32_t device_id_; +}; +} // namespace session +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_SESSION_SESSION_CONTEXT_H diff --git a/mindspore/ccsrc/session/session_factory.cc b/mindspore/ccsrc/session/session_factory.cc new file mode 100644 index 00000000000..4cd0481f8cb --- /dev/null +++ b/mindspore/ccsrc/session/session_factory.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "session/session_factory.h" +#include +#include +#include +namespace mindspore { +namespace session { +SessionFactory &SessionFactory::Get() { + static SessionFactory instance; + return instance; +} + +void SessionFactory::Register(const std::string &device_name, SessionCreator &&session_creator) { + if (session_creators_.end() == session_creators_.find(device_name)) { + (void)session_creators_.emplace(device_name, session_creator); + } +} + +std::shared_ptr SessionFactory::Create(const std::string &device_name) { + auto iter = session_creators_.find(device_name); + if (session_creators_.end() != iter) { + MS_EXCEPTION_IF_NULL(iter->second); + return (iter->second)(); + } + return nullptr; +} +} // namespace session +} // namespace mindspore diff --git a/mindspore/ccsrc/session/session_factory.h b/mindspore/ccsrc/session/session_factory.h new file mode 100644 index 00000000000..476d9ff4a13 --- /dev/null +++ b/mindspore/ccsrc/session/session_factory.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_SESSION_SESSION_FACTORY_H_ +#define MINDSPORE_CCSRC_SESSION_SESSION_FACTORY_H_ + +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "device/cpu/cpu_kernel.h" +#include "session/session_basic.h" +namespace mindspore { +namespace session { +using SessionCreator = std::function()>; +class SessionFactory { + public: + static SessionFactory &Get(); + void Register(const std::string &device_name, SessionCreator &&session_creator); + std::shared_ptr Create(const std::string &device_name); + + private: + SessionFactory() = default; + ~SessionFactory() = default; + DISABLE_COPY_AND_ASSIGN(SessionFactory) + std::map session_creators_; +}; + +class SessionRegistrar { + public: + SessionRegistrar(const std::string &device_name, SessionCreator &&session_creator) { + SessionFactory::Get().Register(device_name, std::move(session_creator)); + } + ~SessionRegistrar() = default; +}; + +#define MS_REG_SESSION(DEVICE_NAME, SESSION_CLASS) \ + static const SessionRegistrar g_session_registrar__##DEVICE_NAME##_##_reg( \ + DEVICE_NAME, []() { return std::make_shared(); }); +} // namespace session +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_SESSION_SESSION_FACTORY_H_ diff --git a/mindspore/ccsrc/transform/CMakeLists.txt b/mindspore/ccsrc/transform/CMakeLists.txt new file mode 100644 index 00000000000..718f53f6276 --- /dev/null +++ b/mindspore/ccsrc/transform/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _TRANSFORM_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_transform_obj OBJECT ${_TRANSFORM_ALL_SRC_FILES}) diff --git a/mindspore/ccsrc/transform/all_ops.h b/mindspore/ccsrc/transform/all_ops.h new file mode 100644 index 00000000000..1206b3c7a8a --- /dev/null +++ b/mindspore/ccsrc/transform/all_ops.h @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_ALL_OPS_H_ +#define TRANSFORM_ALL_OPS_H_ + +// old +#include "ops/all_ops.h" +#endif // TRANSFORM_ALL_OPS_H_ diff --git a/mindspore/ccsrc/transform/convert.cc b/mindspore/ccsrc/transform/convert.cc new file mode 100755 index 00000000000..74b0695cff8 --- /dev/null +++ b/mindspore/ccsrc/transform/convert.cc @@ -0,0 +1,1835 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/convert.h" + +#include +#include +#include +#include "utils/utils.h" + +#include "operator/ops.h" +#include "utils/log_adapter.h" +#include "utils/graph_utils.h" +#include "utils/symbolic.h" +#include "utils/config_manager.h" +#include "utils/convert_utils.h" +#include "./common.h" + +namespace mindspore { +namespace transform { +using std::endl; + +#define ADPT_DESC_ONE(T) std::make_shared(std::make_shared>()) +#define ADPT_DESC_TWO(T, I) \ + std::make_shared(std::make_shared>(), std::make_shared>()) +#define GET_MACRO(_1, _2, DESC, ...) DESC +#define ADPT_DESC(...) GET_MACRO(__VA_ARGS__, ADPT_DESC_TWO, ADPT_DESC_ONE, ...)(__VA_ARGS__) + +using ge::Operator; +using mindspore::kAnyValue; +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; + +const char kNameCustomOp[] = "CustomOp"; +const char kNameConst[] = "Const"; +const char kNameParam[] = "parameter"; +const char kNameRandomUniform[] = "RandomUniform"; +const char kNameSimpleMean[] = "SimpleMean"; +const char kNameSimpleMeanGrad[] = "SimpleMeanGrad"; +const char kNameAllReduce[] = "AllReduce"; +const char kNameBroadcast[] = "Broadcast"; +const char kNameAllgather[] = "AllGather"; +const char kNameReduceScatter[] = "ReduceScatter"; +const char kNameReduceSum[] = "ReduceSum"; +const char kNameIsFinite[] = "isFinite"; +const char kNameReciprocal[] = "Reciprocal"; +const char kNameRsqrt[] = "Rsqrt"; +const char kNameRsqrtGrad[] = "RsqrtGrad"; +const char kNameSqrt[] = "Sqrt"; +const char kNameSquare[] = "Square"; +const char kNameSquaredDifference[] = "SquaredDifference"; +const char kNamePow[] = "Pow"; +const char kNameBatchMatMul[] = "BatchMatMul"; +const char kNameStridedSlice[] = "StridedSlice"; +const char kNameStridedSliceGrad[] = "StridedSliceGrad"; +const char kNameExpandDims[] = "ExpandDims"; +const char kNameLog[] = "Log"; +const char kNameLogicalAnd[] = "LogicalAnd"; +const char kNameLogicalNot[] = "LogicalNot"; +const char kNameLogicalOr[] = "LogicalOr"; +const char kNameExp[] = "Exp"; +const char kNameLessEqual[] = "LessEqual"; +const char kNameGreaterEqual[] = "GreaterEqual"; +const char kNameEqual[] = "Equal"; +const char kNameNotEqual[] = "NotEqual"; +const char kNameFlattenGrad[] = "FlattenGrad"; +const char kNameConvolution[] = "Convolution"; +const char kNameBiasAdd[] = "BiasAdd"; +const char kNameMaxPoolGrad[] = "MaxPoolGrad"; +const char kNameAvgPoolGrad[] = "AvgPoolGrad"; +const char kNameMaxPoolGradWithArgmax[] = "MaxPoolGradWithArgmax"; +const char kNameApplyMomentum[] = "ApplyMomentum"; +const char kNameDropoutDoMask[] = "DropoutDoMask"; +const char kNameResizeBilinear[] = "ResizeBilinear"; +const char kNameResizeBilinearGrad[] = "ResizeBilinearGrad"; +const char kNameZerosLike[] = "ZerosLike"; +const char kNameOnesLike[] = "OnesLike"; +const char kNameTruncatedNormal[] = "TruncatedNormal"; +const char kNameSpaceToBatchNd[] = "SpaceToBatchNd"; +const char kNameConfusionMatrix[] = "ConfusionMatrix"; +const char kNameResizeNearestNeighborD[] = "ResizeNearestNeighbor"; +const char kNameResizeNearestNeighborGrad[] = "ResizeNearestNeighborGrad"; +const char kNameApplyAdam[] = "Adam"; +const char kNameReLU6[] = "ReLU6"; +const char kNameReLU6Grad[] = "ReLU6Grad"; +const char kNameElu[] = "Elu"; +const char kNameEluGrad[] = "EluGrad"; +const char kNameScatterNdUpdate[] = "ScatterNdUpdate"; +const char kNameNMSWithMask[] = "NMSWithMask"; +const char kNameCheckValid[] = "CheckValid"; +const char kNameSmoothL1Loss[] = "SmoothL1Loss"; +const char kNameSmoothL1LossGrad[] = "SmoothL1LossGrad"; +const char kNameSGD[] = "SGD"; +const char kNameSigmoidCrossEntropyWithLogits[] = "SigmoidCrossEntropyWithLogits"; +const char kNameSigmoidCrossEntropyWithLogitsGrad[] = "SigmoidCrossEntropyWithLogitsGrad"; +const char kNameScatterNdD[] = "ScatterNd"; +const char kNamePadD[] = "Pad"; +const char kNameGatherNd[] = "GatherNd"; +const char kNameArgmax[] = "Argmax"; +const char kNameArgmin[] = "Argmin"; +const char kNameArgMaxWithValue[] = "ArgMaxWithValue"; +const char kNameArgMinWithValue[] = "ArgMinWithValue"; +const char kNameReduceProd[] = "ReduceProd"; +const char kNameCumProd[] = "CumProd"; +const char kNameDiagpart[] = "Diagpart"; +const char kNameSplitD[] = "Split"; +const char kNameBatchToSpaceNd[] = "BatchToSpaceNd"; +const char kNameFloor[] = "Floor"; +const char kNameNPUGetFloatStatus[] = "NPUGetFloatStatus"; +const char kNameAssignAdd[] = "AssignAdd"; +const char kNameAssignSub[] = "AssignSub"; +const char kNameNPUAllocFloatStatus[] = "NPUAllocFloatStatus"; +const char kNameNPUClearFloatStatus[] = "NPUClearFloatStatus"; +const char kNameReshape[] = "Reshape"; +const char kNameRealDiv[] = "RealDiv"; +const char kNameTile[] = "Tile"; +const char kNameCos[] = "Cos"; +const char kNameACos[] = "ACos"; +const char kNameACosGrad[] = "ACosGrad"; +const char kNameFloorDiv[] = "FloorDiv"; +const char kNameSin[] = "Sin"; +const char kNamePrelu[] = "PReLU"; +const char kNamePreluGrad[] = "PReLUGrad"; +const char kNameSigmoid[] = "Sigmoid"; +const char kNameSigmoidGrad[] = "SigmoidGrad"; +const char kNameL2Normalize[] = "L2Normalize"; +const char kNameL2NormalizeGrad[] = "L2NormalizeGrad"; +const char kNameSoftmax[] = "Softmax"; +const char kNameIOU[] = "IOU"; +const char kNameBoundingBoxDecode[] = "BoundingBoxDecode"; +const char kNameBoundingBoxEncode[] = "BoundingBoxEncode"; +const char kNameSlice[] = "Slice"; +const char kNameAddN[] = "AddN"; +const char kNameLess[] = "Less"; +const char kNameGreater[] = "Greater"; +const char kNamePack[] = "Stack"; +const char kNameMerge[] = "Merge"; +const char kNameGeSwitch[] = "GeSwitch"; + +const char kNameHuberLoss[] = "HuberLoss"; +const char kNameCumSum[] = "CumSum"; +const char kNameHuberLossGrad[] = "HuberLossGrad"; +const char kNameSparseSoftmaxCrossEntropy[] = "SparseSoftmaxCrossEntropy"; +const char kNameSparseSoftmaxCrossEntropyGrad[] = "SparseSoftmaxCrossEntropyGrad"; +const char kNameTopK[] = "TopK"; +const char kNameSoftmaxGrad[] = "SoftmaxGrad"; +const char kNameMaxPool[] = "MaxPool"; +const char kNameAvgPool[] = "AvgPool"; +const char kNameBatchNorm[] = "BatchNorm"; +const char kNameBatchNormGrad[] = "BatchNormGrad"; +const char kNameROIAlign[] = "ROIAlign"; +const char kNameROIAlignGrad[] = "ROIAlignGrad"; +const char kNameRandomChoiceWithMask[] = "RandomChoiceWithMask"; +const char kNameAbs[] = "Abs"; +const char kNameAbsGrad[] = "AbsGrad"; +const char kNameBinaryCrossEntropy[] = "BinaryCrossEntropy"; +const char kNameBinaryCrossEntropyGrad[] = "BinaryCrossEntropyGrad"; +const char kNameSparseApplyAdagrad[] = "SparseApplyAdagrad"; +const char kNameSpaceToDepth[] = "SpaceToDepth"; +const char kNameDepthToSpace[] = "DepthToSpace"; +const char kNameSign[] = "Sign"; +const char kNameLARSUpdate[] = "LARSUpdate"; +const char kNameRound[] = "Round"; +const char kNamePrint[] = "Print"; +const char kNameApplyFtrl[] = "ApplyFtrl"; + +// -----------------OpAdapter initialization-------------- +std::unordered_map &DfGraphConvertor::get_adpt_map() { + static std::unordered_map adpt_map = { + {string(kNameCustomOp), ADPT_DESC(Operator)}, + {string(kNameIOU), ADPT_DESC(Iou)}, + {string(kNameGreaterEqual), ADPT_DESC(GreaterEqual)}, + {string(kNameSlice), ADPT_DESC(SliceD)}, + {string(kNameApplyMomentum), ADPT_DESC(ApplyMomentum)}, + {string(kNameMaxPool), ADPT_DESC(MaxPool)}, + {string(kNameAvgPool), ADPT_DESC(AvgPool)}, + {string(kNameTopK), ADPT_DESC(TopKV2)}, + {string(kNamePack), ADPT_DESC(Pack)}, + {string(kNameSplitD), ADPT_DESC(SplitD)}, + {string(kNameAllReduce), ADPT_DESC(HcomAllReduce)}, + {string(kNameBroadcast), ADPT_DESC(HcomBroadcast)}, + {string(kNameAllgather), ADPT_DESC(HcomAllGather)}, + {string(kNameReduceScatter), ADPT_DESC(HcomReduceScatter)}, + {string(kNameMaxPoolGrad), ADPT_DESC(MaxPoolGrad)}, + {string(kNameAvgPoolGrad), ADPT_DESC(AvgPoolGrad)}, + {string(kNameMaxPoolGradWithArgmax), ADPT_DESC(MaxPoolGradWithArgmax)}, + {prim::kPrimAssign->name(), ADPT_DESC(Assign)}, + {prim::kPrimStateSetItem->name(), ADPT_DESC(Assign)}, + {prim::kPrimReluGrad->name(), ADPT_DESC(ReluGrad)}, + {prim::kPrimFusedBatchNormGrad->name(), ADPT_DESC(FusedBatchNormGrad)}, + {prim::kPrimBiasAddGrad->name(), ADPT_DESC(BiasAddGrad)}, + {prim::kPrimConv2D->name(), ADPT_DESC(Conv2D)}, + {prim::kPrimConv2DBackpropInput->name(), ADPT_DESC(Conv2DBackpropInputD)}, + {prim::kPrimConv2DBackpropFilter->name(), ADPT_DESC(Conv2DBackpropFilterD)}, + {prim::kPrimDepthwiseConv2dNative->name(), ADPT_DESC(DepthwiseConv2D)}, + {prim::kPrimDepthwiseConv2dNativeBackpropFilter->name(), ADPT_DESC(DepthwiseConv2DBackpropFilterD)}, + {prim::kPrimDepthwiseConv2dNativeBackpropInput->name(), ADPT_DESC(DepthwiseConv2DBackpropInputD)}, + {prim::kPrimFusedBatchNorm->name(), ADPT_DESC(FusedBatchNorm, BatchNorm)}, + {string(kNameBatchNorm), ADPT_DESC(BatchNorm)}, + {string(kNameBatchNormGrad), ADPT_DESC(BatchNormGrad)}, + {string(kNameReshape), ADPT_DESC(Reshape)}, + {string(kNameFlattenGrad), ADPT_DESC(Reshape)}, + {prim::kPrimFlatten->name(), ADPT_DESC(Flatten)}, + {string(kNameAddN), ADPT_DESC(AddN)}, + {string(kNameLess), ADPT_DESC(Less)}, + {string(kNameSqrt), ADPT_DESC(Sqrt)}, + {string(kNameRsqrt), ADPT_DESC(Rsqrt)}, + {string(kNameSquare), ADPT_DESC(Square)}, + {prim::kPrimTanh->name(), ADPT_DESC(Tanh)}, + {prim::kPrimTanhGrad->name(), ADPT_DESC(TanhGrad)}, + {string(kNameResizeNearestNeighborD), ADPT_DESC(ResizeNearestNeighborD)}, + {string(kNameResizeNearestNeighborGrad), ADPT_DESC(ResizeNearestNeighborGrad)}, + {string(kNameApplyAdam), ADPT_DESC(ApplyAdam)}, + {string(kNameReLU6), ADPT_DESC(Relu6)}, + {string(kNameReLU6Grad), ADPT_DESC(Relu6Grad)}, + {string(kNameElu), ADPT_DESC(Elu)}, + {string(kNameEluGrad), ADPT_DESC(EluGrad)}, + {string(kNameResizeBilinearGrad), ADPT_DESC(ResizeBilinearGrad)}, + {string(kNameResizeBilinear), ADPT_DESC(ResizeBilinearD)}, + {string(kNameZerosLike), ADPT_DESC(ZerosLike)}, + {string(kNameOnesLike), ADPT_DESC(OnesLike)}, + {string(kNameScatterNdUpdate), ADPT_DESC(ScatterNdUpdate)}, + {string(kNameNMSWithMask), ADPT_DESC(NMSWithMask)}, + {string(kNameCheckValid), ADPT_DESC(CheckValid)}, + {string(kNameSmoothL1Loss), ADPT_DESC(SmoothL1Loss)}, + {string(kNameSmoothL1LossGrad), ADPT_DESC(SmoothL1LossGrad)}, + {string(kNameSigmoidCrossEntropyWithLogits), ADPT_DESC(SigmoidCrossEntropyWithLogits)}, + {string(kNameSigmoidCrossEntropyWithLogitsGrad), ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad)}, + {string(kNameScatterNdD), ADPT_DESC(ScatterNdD)}, + {string(kNamePadD), ADPT_DESC(PadD)}, + {string(kNameGatherNd), ADPT_DESC(GatherNd)}, + {string(kNameArgmax), ADPT_DESC(ArgMaxD)}, + {string(kNameArgmin), ADPT_DESC(ArgMinD)}, + {string(kNameArgMaxWithValue), ADPT_DESC(ArgMaxWithValue)}, + {string(kNameArgMinWithValue), ADPT_DESC(ArgMinWithValue)}, + {prim::kPrimReduceSum->name(), ADPT_DESC(ReduceSumD)}, + {prim::kPrimReduceMean->name(), ADPT_DESC(ReduceMeanD)}, + {prim::kPrimReduceAll->name(), ADPT_DESC(ReduceAll)}, + {prim::kPrimReduceMin->name(), ADPT_DESC(ReduceMinD)}, + {prim::kPrimReduceMax->name(), ADPT_DESC(ReduceMaxD)}, + {string(kNameLARSUpdate), ADPT_DESC(LarsV2Update)}, + {string(kNameReduceProd), ADPT_DESC(ReduceProdD)}, + {string(kNameCumProd), ADPT_DESC(CumprodD)}, + {string(kNameMerge), ADPT_DESC(Merge)}, + {string(kNameGeSwitch), ADPT_DESC(Switch)}, + {string(kNameCumSum), ADPT_DESC(CumsumD)}, + + {prim::kPrimMul->name(), ADPT_DESC(Mul)}, + {string(kNameTile), ADPT_DESC(TileD)}, + {prim::kPrimOneHot->name(), ADPT_DESC(OneHot)}, + + {prim::kPrimGatherV2->name(), ADPT_DESC(GatherV2D)}, + {string(kNameCos), ADPT_DESC(Cos)}, + {string(kNameACos), ADPT_DESC(Acos)}, + {string(kNameACosGrad), ADPT_DESC(AcosGrad)}, + {string(kNameFloor), ADPT_DESC(Floor)}, + {string(kNameFloorDiv), ADPT_DESC(FloorDiv)}, + {string(kNameSin), ADPT_DESC(Sin)}, + {string(kNameExp), ADPT_DESC(Exp)}, + {string(kNameBoundingBoxEncode), ADPT_DESC(BoundingBoxEncode)}, + {string(kNameBoundingBoxDecode), ADPT_DESC(BoundingBoxDecode)}, + + {prim::kPrimCast->name(), ADPT_DESC(Cast)}, + {string(kNameRealDiv), ADPT_DESC(RealDiv)}, + {prim::kPrimNeg->name(), ADPT_DESC(Neg)}, + {prim::kPrimTranspose->name(), ADPT_DESC(TransposeD)}, + {prim::kPrimSub->name(), ADPT_DESC(Sub)}, + {string(kNameReciprocal), ADPT_DESC(Reciprocal)}, + {prim::kPrimDropoutGenMask->name(), ADPT_DESC(DropOutGenMask)}, + {string(kNameAssignAdd), ADPT_DESC(AssignAdd)}, + {string(kNameAssignSub), ADPT_DESC(AssignSub)}, + {prim::kPrimConcat->name(), ADPT_DESC(ConcatD)}, + {string(kNamePow), ADPT_DESC(Pow)}, + {string(kNameExp), ADPT_DESC(Exp)}, + {string(kNameEqual), ADPT_DESC(Equal)}, + {string(kNameNotEqual), ADPT_DESC(NotEqual)}, + {string(kNameLog), ADPT_DESC(Log)}, + {string(kNameLogicalAnd), ADPT_DESC(LogicalAnd)}, + {string(kNameLogicalNot), ADPT_DESC(LogicalNot)}, + {string(kNameLogicalOr), ADPT_DESC(LogicalOr)}, + {string(kNameGreater), ADPT_DESC(Greater)}, + {prim::kPrimMaximum->name(), ADPT_DESC(Maximum)}, + {prim::kPrimRelu->name(), ADPT_DESC(Relu)}, + {string(kNamePrelu), ADPT_DESC(PRelu)}, + {string(kNamePreluGrad), ADPT_DESC(PReluGrad)}, + {string(kNameSigmoid), ADPT_DESC(Sigmoid)}, + {string(kNameSigmoidGrad), ADPT_DESC(SigmoidGrad)}, + {string(kNameSGD), ADPT_DESC(SGD)}, + {prim::kPrimLogSoftmaxGrad->name(), ADPT_DESC(LogSoftmaxGrad)}, + {prim::kPrimMaximumGrad->name(), ADPT_DESC(MaximumGrad)}, + {prim::kPrimMinimumGrad->name(), ADPT_DESC(MinimumGrad)}, + {string(kNameL2Normalize), ADPT_DESC(L2Normalize)}, + {string(kNameL2NormalizeGrad), ADPT_DESC(L2NormalizeGrad)}, + + {prim::kPrimMinimum->name(), ADPT_DESC(Minimum)}, + {prim::kPrimSelect->name(), ADPT_DESC(Select)}, + {string(kNameLessEqual), ADPT_DESC(LessEqual)}, + {prim::kPrimLogSoftmax->name(), ADPT_DESC(LogSoftmax)}, + {string(kNameTruncatedNormal), ADPT_DESC(TruncatedNormal)}, + {string(kNameStridedSliceGrad), ADPT_DESC(StridedSliceGrad)}, + {prim::kPrimGelu->name(), ADPT_DESC(Gelu)}, + {prim::kPrimGeluGrad->name(), ADPT_DESC(GeluGrad)}, + {string(kNameStridedSlice), ADPT_DESC(StridedSlice)}, + {prim::kPrimUnsortedSegmentSum->name(), ADPT_DESC(UnsortedSegmentSumD)}, + {string(kNameExpandDims), ADPT_DESC(ExpandDims)}, + {prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)}, + {prim::kPrimLayerNorm->name(), ADPT_DESC(LayerNorm)}, + {prim::kPrimLayerNormGrad->name(), ADPT_DESC(LayerNormGrad)}, + {string(kNameBatchMatMul), ADPT_DESC(BatchMatMul)}, + {string(kNameDropoutDoMask), ADPT_DESC(DropOutDoMask)}, + + {string(kNameNPUGetFloatStatus), ADPT_DESC(NPUGetFloatStatus)}, + {string(kNameNPUAllocFloatStatus), ADPT_DESC(NPUAllocFloatStatus)}, + {string(kNameNPUClearFloatStatus), ADPT_DESC(NPUClearFloatStatus)}, + + {string(kNameRandomChoiceWithMask), ADPT_DESC(RandomChoiceWithMask)}, + {prim::kPrimSoftmaxCrossEntropyWithLogits->name(), ADPT_DESC(SoftmaxCrossEntropyWithLogits)}, + + {prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)}, + {prim::kPrimImageSummary->name(), ADPT_DESC(Summary)}, + {prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)}, + {prim::kPrimTensorAdd->name(), + std::make_shared(std::make_shared>(ExtraAttr({{"mode", MakeValue(1)}})), + std::make_shared>(ExtraAttr({{"mode", MakeValue(1)}})))}, + {string(kNameBiasAdd), ADPT_DESC(BiasAdd)}, + {prim::kPrimRelu->name(), ADPT_DESC(Relu)}, + + {prim::kPrimMatMul->name(), ADPT_DESC(MatMul)}, + + {string(kNameConst), ADPT_DESC(Constant, Const)}, + {string(kNameSoftmax), ADPT_DESC(Softmax)}, + {string(kNameSoftmaxGrad), ADPT_DESC(SoftmaxGrad)}, + {string(kNameParam), ADPT_DESC(Data)}, + {string(kNameROIAlign), ADPT_DESC(ROIAlign)}, + {string(kNameROIAlignGrad), ADPT_DESC(ROIAlignGrad)}, + {string(kNameAbs), ADPT_DESC(Abs)}, + {string(kNameAbsGrad), ADPT_DESC(AbsGrad)}, + {string(kNameBinaryCrossEntropy), ADPT_DESC(BinaryCrossEntropy)}, + {string(kNameBinaryCrossEntropyGrad), ADPT_DESC(BinaryCrossEntropyGrad)}, + {string(kNameSparseApplyAdagrad), ADPT_DESC(SparseApplyAdagradD)}, + {string(kNameSpaceToDepth), ADPT_DESC(SpaceToDepth)}, + {string(kNameDepthToSpace), ADPT_DESC(DepthToSpace)}, + {string(kNameSign), ADPT_DESC(Sign)}, + {string(kNameRound), ADPT_DESC(Round)}, + {string(kNameApplyFtrl), ADPT_DESC(ApplyFtrl)}}; +#ifdef ENABLE_GE + adpt_map[string(kNamePrint)] = ADPT_DESC(Print); +#endif + return adpt_map; +} + +// ---------------implement of DfGraphConvertor------------- +std::string GetCNodeFuncName(const CNodePtr cnode) { + if (cnode->inputs().empty()) { + return ""; + } + + AnfNodePtr valuenode = cnode->input(0); + if (valuenode->isa()) { + auto value = GetValueNode(valuenode); + // check whether the valuenode is primitive + if (value->isa()) { + return value->cast()->name(); + } else { + return value->ToString(); + } + } + return ""; +} + +PrimType GetCNodeFuncType(const CNodePtr cnode) { + if (cnode->inputs().empty()) { + return kPrimTypeUnknown; + } + + AnfNodePtr valuenode = cnode->input(0); + if (IsValueNode(valuenode)) { + // check whether the valuenode is primitive + return GetValueNode(valuenode)->prim_type(); + } + return kPrimTypeUnknown; +} + +OpAdapterPtr DfGraphConvertor::FindAdapter(const AnfNodePtr node, bool train) { + if (node->isa()) { + auto cnode = node->cast(); + + std::string name = kNameCustomOp; + if (!IsCustomCNode(cnode)) { + name = GetCNodeFuncName(cnode); + } + + auto it_adpt = get_adpt_map().find(name); + if (it_adpt != get_adpt_map().end()) { + return it_adpt->second->Get(train); + } else { + MS_LOG(ERROR) << "Can't find OpAdapter for " << name; + } + } + + if (node->isa()) { + return get_adpt_map()[kNameConst]->Get(train); + } + if (node->isa()) { + return get_adpt_map()[kNameParam]->Get(train); + } + return OpAdapterPtr(nullptr); +} + +void DfGraphConvertor::InitLoopVar(std::vector *init_input) { + if (this->training_) { + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64); + auto var_iter_num = std::make_shared("npu_runconfig/iterations_per_loop"); + auto var_loop_cond = std::make_shared("npu_runconfig/loop_cond"); + auto var_one = std::make_shared("npu_runconfig/one"); + auto var_zero = std::make_shared("npu_runconfig/zero"); + (void)var_iter_num->update_output_desc_y(desc); + (void)var_loop_cond->update_output_desc_y(desc); + (void)var_one->update_output_desc_y(desc); + (void)var_zero->update_output_desc_y(desc); + vars_["npu_runconfig/iterations_per_loop"] = var_iter_num; + vars_["npu_runconfig/loop_cond"] = var_loop_cond; + vars_["npu_runconfig/one"] = var_one; + vars_["npu_runconfig/zero"] = var_zero; + + int64_t value = 0; + auto const_iter_num = std::make_shared("const/npu_runconfig/iterations_per_loop"); + if (ConfigManager::GetInstance().dataset_mode() == DS_GRAPH_MODE) { + value = ConfigManager::GetInstance().iter_num(); + } else { + MS_LOG(INFO) << "Run with feed mode, the iterator number will always be 1"; + value = 1; + ConfigManager::GetInstance().set_iter_num(value); + } + value -= 1; // iteration start from 0, the max iteration number for n loop should be n-1 + (void)const_iter_num->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_loop_cond = std::make_shared("const/npu_runconfig/loop_cond"); + value = 0; + (void)const_loop_cond->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_one = std::make_shared("const/npu_runconfig/one"); + value = 1; + (void)const_one->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_zero = std::make_shared("const/npu_runconfig/zero"); + value = 0; + (void)const_zero->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + (void)const_iter_num->update_output_desc_y(desc); + (void)const_loop_cond->update_output_desc_y(desc); + (void)const_one->update_output_desc_y(desc); + (void)const_zero->update_output_desc_y(desc); + + auto assign_iter_num = std::make_shared("assign/npu_runconfig/iterations_per_loop"); + (void)assign_iter_num->set_input_ref(*var_iter_num).set_input_value(*const_iter_num); + auto assign_loop_cond = std::make_shared("assign/npu_runconfig/loop_cond"); + (void)assign_loop_cond->set_input_ref(*var_loop_cond).set_input_value(*const_loop_cond); + auto assign_one = std::make_shared("assign/npu_runconfig/one"); + (void)assign_one->set_input_ref(*var_one).set_input_value(*const_one); + auto assign_zero = std::make_shared("assign/npu_runconfig/zero"); + (void)assign_zero->set_input_ref(*var_zero).set_input_value(*const_zero); + + init_input->push_back(*var_iter_num); + init_input->push_back(*var_loop_cond); + init_input->push_back(*var_one); + init_input->push_back(*var_zero); + init_ops_.push_back(var_iter_num); + init_ops_.push_back(var_loop_cond); + init_ops_.push_back(var_one); + init_ops_.push_back(var_zero); + init_ops_.push_back(const_iter_num); + init_ops_.push_back(const_loop_cond); + init_ops_.push_back(const_one); + init_ops_.push_back(const_zero); + init_ops_.push_back(assign_iter_num); + init_ops_.push_back(assign_loop_cond); + init_ops_.push_back(assign_one); + init_ops_.push_back(assign_zero); + } +} + +OpAdapterPtr DfGraphConvertor::FindAdapter(const std::string &name, bool train) { + auto it = get_adpt_map().find(name); + if (it != get_adpt_map().end()) { + return it->second->Get(train); + } + MS_LOG(ERROR) << "Can't find OpAdapter for " << name; + return transform::OpAdapterPtr(nullptr); +} + +void DfGraphConvertor::DrawParamInitSubGraph(const std::string &name, const AnfNodePtr &it) { + // draw init subgraph + init_sout_ << "op_assign" << it.get() << "[label=<"; + init_sout_ << "" << endl; + init_sout_ << ""; + init_sout_ << ""; + init_sout_ << ""; + init_sout_ << "" << endl; + init_sout_ << "" << endl; + init_sout_ << "
resourcevalue
" + << "\"assign_" << name << "\"
> shape=plaintext]" << endl; + init_sout_ << "param" << it.get() << "[shape=octagon, label=\"" << name << "\"]" << endl; + init_sout_ << "const" << it.get() << "[label= \"" << name << "_const" + << "\" shape=ellipse]" << endl; + init_sout_ << "param" << it.get() << "->" + << "op_assign" << it.get() << ":1" << endl; + init_sout_ << "const" << it.get() << "->" + << "op_assign" << it.get() << ":2" << endl; +} + +void DfGraphConvertor::SetupParamInitSubGraph(const TensorOrderMap &tensors, std::vector *init_input) { + DfGraphPtr init_graph = std::make_shared("init"); + std::vector nodes = TopoSort(anf_graph_->get_return()); + + for (auto &it : nodes) { + if (it->isa()) { + if (IsValueNode(it)) { + auto symbolic = GetValueNode(it); + auto name = std::static_pointer_cast(symbolic->node())->name(); + auto iter = vars_.find(name); // get correspoding varaible op + if (iter != vars_.end()) { + op_cache_[it.get()] = iter->second; + // #ifdef DRAW_GE_GRAPH + compute_sout_ << op_draw_name_[params_[name].get()] << " -> " << op_draw_name_[it.get()] + << "[style=\"dotted\"]" << endl; + // #endif + } + } else if (IsValueNode(it)) { + auto refkey = GetValueNode(it); + auto name = refkey->tag(); + auto iter = vars_.find(name); // get correspoding varaible op + if (iter != vars_.end()) { + op_cache_[it.get()] = iter->second; + compute_sout_ << op_draw_name_[params_[name].get()] << " -> " << op_draw_name_[it.get()] + << "[style=\"dotted\"]" << endl; + } + } + } + } + + for (auto &it : tensors) { + if (vars_.find(it.first) == vars_.end()) { + MS_LOG(WARNING) << "Init parameter " << it.first << " didn't appear in graph."; + vars_[it.first] = nullptr; + } + } + + // set up init sub graph + if (init_input->size()) { + // init sub graph needs no input + MS_LOG(INFO) << "Build data init subgraph."; + (void)init_graph->SetInputs(*init_input); + this->init_graph_ = init_graph; + } else { + this->init_graph_ = nullptr; + } +} + +void DfGraphConvertor::MakeDatasetHandler(const std::string &name, const size_t &input_idx, const AnfNodePtr &it) { + MS_LOG(INFO) << "The " << name << " is the " << input_idx << "(st/nd/th) input"; + if (ConfigManager::GetInstance().dataset_mode() == DS_GRAPH_MODE) { + auto getnext_idx = static_cast(input_idx); + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + if (!param.input_indexes().empty() && input_idx <= param.input_indexes().size()) { + getnext_idx = param.input_indexes()[input_idx] - 1; // input_idx start from 0. + MS_LOG(INFO) << "remap input_index:" << input_idx << " to getnext_index:" << getnext_idx << "."; + } + // use iterator_getnext op with output_name instead of data op in BuildGraph. + out_handle_cache_[it.get()] = OutHandler(dataset_iter_getnext_, "y" + std::to_string(getnext_idx)); + } +} + +void DfGraphConvertor::SetupBroadcast(const std::shared_ptr &broadcast, + const std::vector &broadcast_desc, + const DfGraphPtr &broadcast_graph, std::vector broadcast_input) { + MS_LOG(INFO) << "build broadcast subgraph"; + if (broadcast_desc.size() != broadcast_input.size()) { + MS_LOG(EXCEPTION) << "Desc number of BroadCast is not equal to number of Input"; + } + (void)broadcast->create_dynamic_input_x(static_cast(broadcast_input.size())); + (void)broadcast->create_dynamic_output_y(static_cast(broadcast_desc.size())); + for (unsigned int i = 0; i < broadcast_input.size(); i++) { + (void)broadcast->set_dynamic_input_x(i, broadcast_input[i]); + (void)broadcast->update_dynamic_output_desc_y(i, broadcast_desc[i]); + } + (void)broadcast_graph->SetInputs(broadcast_input); + this->broadcast_graph_ = broadcast_graph; +} + +void DfGraphConvertor::InitParamWithData(const TensorOrderMap &tensors) { + int index = 0; + std::vector init_input; + for (auto it : tensors) { + std::string name = it.first; + auto node_itor = params_.find(name); + // if name not in params_, create a node in graph + if (node_itor == params_.end()) { + MS_LOG(WARNING) << "" << name << " is not in params, and create a new node."; + ParameterPtr param = anf_graph_->add_parameter(); + name = name + "_temp"; + param->set_name(name); + (void)ConvertParameter(param); + node_itor = params_.find(name); + } + auto node = node_itor->second; + auto op_itor = op_cache_.find(node.get()); + if (op_itor == op_cache_.end()) { + MS_LOG(EXCEPTION) << "Can not find op for node " << node->ToString() << "."; + } + auto adpt = FindAdapter(kNameParam, training_); + if (adpt == nullptr) continue; + auto param_op = adpt->generate(name + "_data"); + MS_LOG(INFO) << "Add parameter " << name << " as input, index " << index << "."; + (void)std::static_pointer_cast(param_op)->set_attr_index(index++); + + if (!training_) { + auto adpt_const = FindAdapter(kNameConst, training_); + if (adpt_const == nullptr) continue; + auto const_op = adpt_const->generate(name + "_const"); + (void)adpt_const->setAttr(const_op, "value", it.second); + + auto const_op_desc = TransformUtil::GetGeTensorDesc(it.second->shape_c(), it.second->data_type(), kOpFormat_NCHW); + if (const_op_desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " ouptut descriptor failed!"; + continue; + } + (void)std::static_pointer_cast(const_op)->update_output_desc_y(*const_op_desc); + + vars_[name] = const_op; + op_itor->second = const_op; + continue; + } + + // create tensor descriptor for output descriptor + auto desc = TransformUtil::GetGeTensorDesc(it.second->shape_c(), it.second->data_type(), kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " ouptut descriptor failed!"; + continue; + } + + // we need three variable ops for each graph with same name + // build init subgraph + auto init_var = std::make_shared(name); + auto assign_op = std::make_shared("assign_" + name); + (void)init_var->update_output_desc_y(*desc); + (void)assign_op->set_input_ref(*init_var).set_input_value(*param_op); + init_input.push_back(*init_var); + init_ops_.push_back(param_op); + init_ops_.push_back(assign_op); + init_ops_.push_back(init_var); + + auto variable = std::make_shared(name); + (void)variable->update_output_desc_y(*desc); + // do not use read variable while variable sink + MS_LOG(DEBUG) << "InitParam, op_name = " << name << ", var = " << variable->GetName() << "."; + op_itor->second = variable; // replace parameter with variable + vars_[name] = variable; // prevent the variable operator from being freed + DrawParamInitSubGraph(name, node); + } + InitLoopVar(&init_input); + SetupParamInitSubGraph(tensors, &init_input); +} + +// convert all parameter need initialize to variable +DfGraphConvertor &DfGraphConvertor::InitParam(const TensorOrderMap &tensors) { + size_t input_idx = 0; + if (error_ != 0) { + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + error_ = INVALID_ARGUMENT; + MS_LOG(ERROR) << "Invalid AnfGraph in InitParam."; + return *this; + } + + // Processing input with MakeDatasetHandler + for (auto &it : anf_graph_->parameters()) { + auto op_itor = op_cache_.find(it.get()); // converted node + if (it->isa() && op_itor != op_cache_.end()) { + string name = std::static_pointer_cast(it)->name(); + auto tensor_itor = tensors.find(name); // in init value map + if (tensor_itor == tensors.end()) { + DfGraphConvertor::MakeDatasetHandler(name, input_idx, it); + input_idx++; + } + } + } + InitParamWithData(tensors); + init_sout_ << "}" << endl; + return *this; +} + +#if (defined ENABLE_GE) +void DfGraphConvertor::BuildSaveCheckpointGraph() { + std::vector graph_inputs; + ge::op::Save save_op("save_parms"); + int save_op_is_active = 0; + size_t index = 0; + string name; + + int32_t count_size = std::count_if(vars_.begin(), vars_.end(), [](const std::pair &it) { + return (it.second == nullptr || it.first.find("/") != std::string::npos); + }); + + (void)save_op.create_dynamic_input_tensors(vars_.size() - static_cast(count_size)); + + // for each "parameter" in anf graph excluding "input" + for (const auto &it : vars_) { + name = it.first; + if (it.second == nullptr || name.find("/") != std::string::npos) continue; + Variable variable(name); + (void)variable.update_output_desc_y(it.second->GetOutputDesc(0)); + (void)save_op.set_dynamic_input_tensors(index++, variable); + + graph_inputs.push_back(variable); + + if (save_op_is_active == 0) { + checkpoint_sout_ << "op_save" << &save_op << "[label=<"; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "
tensor
" + << "\"saveop" + << "\"
> shape=plaintext]" << endl; + } + + checkpoint_sout_ << "param" << it.second << "[shape=octagon, label=\"" << name << "\"]" << endl; + + checkpoint_sout_ << "param" << it.second << "->" + << "op_save" << &save_op << ":1" << endl; + save_op_is_active = 1; + } + if (save_op_is_active) { + std::vector graph_output; + graph_output.emplace_back(save_op); + DfGraphPtr checkpoint_graph = std::make_shared("checkpoint"); + (void)checkpoint_graph->SetInputs(graph_inputs); + (void)checkpoint_graph->SetOutputs(graph_output); + this->save_ckp_graph_ = checkpoint_graph; + } else { + this->save_ckp_graph_ = nullptr; + } + + checkpoint_sout_ << "}" << endl; + return; +} +#endif + +DfGraphConvertor &DfGraphConvertor::GenerateBroadcastGraph(const TensorOrderMap &tensors) { + if (error_ != 0) { + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + error_ = INVALID_ARGUMENT; + MS_LOG(ERROR) << "Invalid AnfGraph in generate broadcast graph"; + return *this; + } + + DfGraphPtr broadcast_graph = std::make_shared("broadcast"); + // collect the operators create for broadcast sub graph, in order to avoid auto release + std::vector broadcast_input; + std::vector broadcast_desc; + auto broadcast = std::make_shared("broadcast_parameter"); + (void)broadcast->set_attr_root_rank(0); + (void)broadcast->set_attr_group("hccl_world_group"); + broadcast_ops_.push_back(broadcast); + + // find every parameter, build broadcast subgraph (or initialize the parameter with constant) + for (auto &it : anf_graph_->parameters()) { + auto op_itor = op_cache_.find(it.get()); // converted node + if (it->isa() && op_itor != op_cache_.end()) { + string name = std::static_pointer_cast(it)->name(); + auto tensor_itor = tensors.find(name); // in init tensor map + if (tensor_itor != tensors.end()) { + auto tensor = tensor_itor->second; + auto shape_ge = tensor->shape_c(); + + // create tensor descriptor for output descriptor + auto desc = TransformUtil::GetGeTensorDesc(shape_ge, tensor->data_type(), kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " ouptut descriptor failed!"; + continue; + } + + // build broadcast subgraph + if (distribute_) { + auto broadcast_var = std::make_shared(name); + (void)broadcast_var->update_output_desc_y(*desc); + broadcast_input.push_back(*broadcast_var); + broadcast_desc.push_back(*desc); + broadcast_ops_.push_back(broadcast_var); + } + } + } + } + + // set up broadcast sub graph + if (!broadcast_input.empty()) { + DfGraphConvertor::SetupBroadcast(broadcast, broadcast_desc, broadcast_graph, broadcast_input); + } else { + this->broadcast_graph_ = nullptr; + } + return *this; +} + +DfGraphConvertor &DfGraphConvertor::GenerateCheckpointGraph() { + if (error_ != 0) { + MS_LOG(ERROR) << "Generate checkpoint graph failed, found error code " << error_ << "."; + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + error_ = INVALID_ARGUMENT; + MS_LOG(ERROR) << "Invalid AnfGraph in GenerateCheckpointGraph"; + return *this; + } +#if (defined ENABLE_GE) + BuildSaveCheckpointGraph(); + // Restoring from checkpoint file is done by pyfront, not in graph now. +#endif + return *this; +} + +DfGraphConvertor &DfGraphConvertor::ConvertAllNode() { + if (error_ != 0) { + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + MS_LOG(ERROR) << "Invalid AnfGraph"; + error_ = FAILED; + return *this; + } + + compute_sout_.clear(); + compute_sout_ << "digraph {" << endl; + init_sout_.clear(); + init_sout_ << "digraph {" << endl; + checkpoint_sout_.clear(); + checkpoint_sout_ << "digraph {" << endl; + restore_checkpoint_sout_.clear(); + restore_checkpoint_sout_ << "digraph {" << endl; + + // Convert all anf node to Operator + MS_LOG(DEBUG) << "convert all node"; + std::vector nodes = TopoSort(anf_graph_->get_return()); + for (auto &it : nodes) { + (void)Convert(it); + if (this->error_ != 0) { + MS_LOG(ERROR) << "failed to convert node: " << it->DebugString() << "."; + } + } + + // Create dataset iterator and iterator_getnext node + if (ConfigManager::GetInstance().dataset_mode() == DS_GRAPH_MODE) { + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + MS_LOG(INFO) << "Dataset param is " << param.ToString() << "."; + // GetNext + auto iter_getnext_op = make_shared("get_next_tmp"); + (void)iter_getnext_op->set_attr_output_types(param.ge_types()); + (void)iter_getnext_op->set_attr_output_shapes(param.shapes()); + (void)iter_getnext_op->set_attr_channel_name(param.queue_name()); + + // save iter_getnext_op for later use + dataset_iter_getnext_ = iter_getnext_op; + } + + // return the data flow graph + return *this; +} + +void DfGraphConvertor::TraceOutputFromTupleGetItem(const AnfNodePtr &anf_out) { + auto it = out_handle_cache_.find(anf_out.get()); + if (it != out_handle_cache_.end()) { + OutHandler handle = it->second; + auto op = handle.op; + if (op != nullptr) { + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType() << ", out_name: " << handle.out; + graph_outputs_.emplace_back(std::make_pair(*op, handle.out)); + } else { + MS_LOG(EXCEPTION) << "tuple_getitem: " << anf_out->fullname_with_scope() << " is not converted"; + } + } else { + // invalid tuple_getitem e.g. tuple_getitem(tuple_getitem())/tuple_getitem(depend())/tuple_getitem(make_tuple()) + MS_LOG(WARNING) << "Invalid tuple_getitem: " << anf_out->fullname_with_scope(); + } +} + +void DfGraphConvertor::TraceOutput(const AnfNodePtr node) { + AnfNodePtr anf_out = node; + AnfNodePtr pre_node = nullptr; + + // trace Parameter node + TraceOutputFromParameter(anf_out); + // then trace cnode + if (!node->isa()) { + return; + } + + // trace tuple_getitem + while (anf_out->isa() && IsPrimitiveCNode(anf_out, prim::kPrimTupleGetItem)) { + pre_node = anf_out; + anf_out = anf_out->cast()->input(1); + } + // trace every element of make_tuple + auto c = anf_out->cast(); + std::string name = ""; + if (anf_out->isa()) { + name = GetCNodeFuncName(c); + } + + if (name == "make_tuple") { + for (unsigned int i = 1; i < c->inputs().size(); i++) { + TraceOutput(c->input(i)); + } + } else if (name == "depend") { + if (c->inputs().size() < 3) { // "depend" primitive have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs is " << c->inputs().size() << ", which is less than 3"; + } + TraceOutput(c->input(1)); + } else if (name == "tuple_getitem") { + TraceOutputFromTupleGetItem(anf_out); + } else { + // add outputs; + auto op = Convert(anf_out); + std::string index; + if (op != nullptr) { + if ((pre_node != nullptr) && IsPrimitiveCNode(pre_node, prim::kPrimTupleGetItem)) { + auto item = out_handle_cache_.find(pre_node.get()); + if (item != out_handle_cache_.end()) { + index = item->second.out; + } else { + MS_LOG(WARNING) << "Can't get operater: " << anf_out->fullname_with_scope() << " 's output item"; + } + } + MS_LOG(INFO) << "Add graph output: " << anf_out->fullname_with_scope() << ":" << index; + graph_outputs_.emplace_back(make_pair(*op, index)); + } + } +} + +void DfGraphConvertor::TraceOutputFromParameter(const AnfNodePtr &anf_out) { + if (anf_out->isa()) { + MS_LOG(INFO) << "Add graph output: " << anf_out->fullname_with_scope(); + auto it = out_handle_cache_.find(anf_out.get()); + if (it != out_handle_cache_.end()) { + // For dataset graph mode, input parameter is converted to a "iterator_get_next:yn" OutHandler. + OutHandler handle = it->second; + auto op = handle.op; + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType() << ", out_name: " << handle.out; + graph_outputs_.emplace_back(make_pair(*op, handle.out)); + } else { + // common parameter case + auto op = Convert(anf_out); + if (op != nullptr) { + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType(); + graph_outputs_.emplace_back(std::make_pair(*op, "")); + } + } + } +} + +void SetupDatasetIterGetNextNode(const OperatorPtr &op) { + if (ConfigManager::GetInstance().dataset_mode() == DS_GRAPH_MODE) { + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + size_t output_num = param.ge_types().size(); + MS_LOG(INFO) << "Set iterator_getnext op's output num = " << output_num << "."; + // set iterator_getnext op's output num + shared_ptr iter_getnext = std::static_pointer_cast(op); + (void)iter_getnext->create_dynamic_output_y(static_cast(output_num)); + + for (uint32_t i = 0; i < output_num; i++) { + ge::TensorDesc desc(GeShape(param.shapes()[i]), ge::FORMAT_NCHW, (ge::DataType)param.ge_types()[i]); + // we don't SetRealDimCnt here since GE do not use this output's real-dim + (void)iter_getnext->update_dynamic_output_desc_y((i), desc); + } + } + return; +} + +DfGraphConvertor &DfGraphConvertor::BuildGraph() { + SetupDatasetIterGetNextNode(dataset_iter_getnext_); + + if (error_ != 0) { + return *this; + } + + // update tuple_out_handle_cache_ + for (auto it : tuple_out_handle_cache_) { + std::size_t len = it.second->size(); + for (std::size_t i = 0; i < len; i++) { + OutHandler handle = (*it.second)[i]; + if (handle.op) { + string name = handle.op->GetName(); + if (vars_.count(name)) { + OperatorPtr new_op = vars_[name]; + if (new_op != nullptr) { + MS_LOG(INFO) << "update tuple_out_handle_cache_ " << name; + (*it.second)[i] = OutHandler(new_op, handle.out); + } + } + } + } + } + + // set up dependices + MS_LOG(DEBUG) << "set up dependices"; + std::vector nodes = ::mindspore::TopoSort(anf_graph_->get_return()); + for (auto &it : nodes) { + SetNodeInput(it); + SetOpControlInput(it); + UpdateOpDesc(it); + } + + if (error_ == 0) { + df_graph_ = make_shared(anf_graph_->ToString()); + } else { + return *this; + } + + // set graph input according to the order from anf graph + std::vector inputs; + if (ConfigManager::GetInstance().dataset_mode() == DS_GRAPH_MODE) { + inputs.push_back(*dataset_iter_getnext_); + } else { + auto params = anf_graph_->parameters(); + int index = 0; + for (auto &it : params) { + auto name = std::static_pointer_cast(it)->name(); + // the parameters which has not been converted to var + if (vars_.find(name) == vars_.end()) { + auto op = Convert(it); + MS_EXCEPTION_IF_NULL(op); + MS_LOG(INFO) << "add not var input " << it->ToString() << ", index " << index; + if (op == nullptr) { + MS_LOG(ERROR) << "Convert graph failed!"; + return *this; + } + UpdateDataOpDesc(it, op); + + MS_LOG(INFO) << "add input " << it->ToString() << ", index " << index; + (void)std::static_pointer_cast(op)->set_attr_index(index++); + inputs.push_back(*op); + } else if (vars_[name] != nullptr) { + MS_LOG(INFO) << "add var input " << it->ToString(); + auto op = Convert(it); + MS_EXCEPTION_IF_NULL(op); + inputs.push_back(*op); + } + } + } + + // Add const nodes as graph input for some operator work with constant + std::transform(graph_const_inputs_.begin(), graph_const_inputs_.end(), std::back_inserter(inputs), + [](OperatorPtr x) { return *x; }); + + MS_LOG(INFO) << "set graph input num: " << inputs.size(); + (void)df_graph_->SetInputs(inputs); + + // set graph output + // set the value of finale return apply node as the output of dataflow graph + MS_LOG(DEBUG) << "set output"; + graph_outputs_.clear(); + TraceOutput(anf_graph_->get_return()->input(1)); + MS_LOG(INFO) << "set graph output num: " << graph_outputs_.size(); + (void)df_graph_->SetOutputs(graph_outputs_); + + compute_sout_ << "}" << endl; + // For the graph(e.g. eval_subgraph) whose IterNum is 1, donot set NeedIteration flag. + if (ConfigManager::GetInstance().iter_num() > 1) { + df_graph_->SetNeedIteration(true); + } + return *this; +} + +void DfGraphConvertor::UpdateDataOpDesc(const AnfNodePtr &it, const OperatorPtr &op) const { + auto node = std::static_pointer_cast(it); + if (node == nullptr) { + MS_LOG(ERROR) << "Update data op descriptor failed! Invalid node."; + return; + } + auto normal_shape_ptr = dyn_cast(node->Shape()); + vector shape; + if (normal_shape_ptr == nullptr) { + MS_LOG(ERROR) << "Update data op descriptor failed! Invalid shape."; + return; + } + shape = normal_shape_ptr->shape(); + if (node->Type() == nullptr) { + MS_LOG(ERROR) << "Update data op descriptor failed! Invalid type."; + return; + } + TypeId me_type = node->Type()->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(node->Type())->element()->type_id(); + } + std::ostringstream buf; + buf << "[" << shape << "]"; + MS_LOG(INFO) << "input shape is " << buf.str() << ", type is " << me_type; + auto desc = TransformUtil::GetGeTensorDesc(shape, me_type, "NCHW"); + if (desc == nullptr) { + MS_LOG(ERROR) << "Update data op descriptor failed! TensorDesc is null."; + } else { + (void)std::static_pointer_cast(op)->update_input_desc_data(*desc); + (void)std::static_pointer_cast(op)->update_output_desc_out(*desc); + } +} + +DfGraphPtr DfGraphConvertor::GetComputeGraph() { return df_graph_; } + +DfGraphPtr DfGraphConvertor::GetInitGraph() { return init_graph_; } + +DfGraphPtr DfGraphConvertor::GetSaveCheckpointGraph() { return save_ckp_graph_; } + +DfGraphPtr DfGraphConvertor::GetBroadcastGraph() { return broadcast_graph_; } + +void DfGraphConvertor::SetOpControlInput(const AnfNodePtr node) { + if (control_depend_cache_.find(node.get()) == control_depend_cache_.end()) { + return; + } + + std::vector control_edges = control_depend_cache_[node.get()]; + if ((control_edges.empty())) { + MS_LOG(ERROR) << "Get control depend node's src or dest operator failed"; + return; + } + + for (auto &item : control_edges) { + (void)item.dest_op->AddControlInput(*item.src_op); + } +} + +void DfGraphConvertor::SetOpInput(const OpAdapterPtr &adpt, const CNodePtr &node) { + OperatorPtr src = Convert(node); + auto &inputs = node->inputs(); + for (size_t i = 1; i < inputs.size(); i++) { + auto pred = inputs[i]; + while (pred->isa() && GetCNodeFuncName(pred->cast()) == "depend") { + pred = pred->cast()->input(1); + } + // skip the None input + if (IsValueNode(pred)) { + continue; + } + // find in out_hadnle_cache_ first + auto it = out_handle_cache_.find(pred.get()); + if (it != out_handle_cache_.end()) { + int ret = adpt->setInput(src, SizeToInt(i), it->second); + if (ret == 0) { + if (pred->isa() && GetCNodeFuncName(pred->cast()) == "tuple_getitem") { + compute_sout_ << op_draw_name_[pred->cast()->input(1).get()] << " -> " << op_draw_name_[node.get()] + << ":" << i << endl; + } else if (pred->isa()) { + compute_sout_ << op_draw_name_[pred.get()] << " -> " << op_draw_name_[node.get()] << ":" << i << endl; + } else { + // don't draw anything. + MS_LOG(INFO) << "DRAW_GE_GRAPH: Shouldn't have this case."; + } + AddGraphConstInput(it->second.op); + } + } else if (tuple_out_handle_cache_.find(pred.get()) != tuple_out_handle_cache_.end()) { + std::shared_ptr> handler_vec = tuple_out_handle_cache_[pred.get()]; + int ret = adpt->setInput(src, SizeToInt(i), handler_vec); + if ((ret == 0) && pred->isa() && (pred->cast()->inputs().size() == handler_vec->size() + 1)) { + for (unsigned int j = 0; j < handler_vec->size(); j++) { + compute_sout_ << op_draw_name_[pred->cast()->input(j + 1).get()] << " -> " + << op_draw_name_[node.get()] << ":" << i << endl; + AddGraphConstInput(handler_vec->at(j).op); + } + } else { + MS_LOG(WARNING) << "Convert tuple node setInput failed : " << node->ToString(); + } + } else { + auto op = Convert(pred); + int ret = adpt->setInput(src, SizeToInt(i), op); + if (ret == 0) { + compute_sout_ << op_draw_name_[pred.get()] << " -> " << op_draw_name_[node.get()] << ":" << i << endl; + AddGraphConstInput(op); + } + } + } +} + +void DfGraphConvertor::AddGraphConstInput(const OperatorPtr &op) { + if (op->GetOpType() == "Constant") { + graph_const_inputs_.push_back(op); + } +} + +void DfGraphConvertor::SetNodeInput(const AnfNodePtr node) { + if (!node->isa()) { + return; + } + if (op_cache_.find(node.get()) == op_cache_.end()) { + return; + } + auto cnode = node->cast(); + OpAdapterPtr adpt = FindAdapter(cnode, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return; + } + + // get Operator from op_cache_, use adapter to set Inputs + DfGraphConvertor::SetOpInput(adpt, cnode); +} + +// Update GE op's shape and type info +void DfGraphConvertor::UpdateOpDesc(const AnfNodePtr node) { + if (nullptr == node || !node->isa()) { + return; + } + + if (op_cache_.find(node.get()) == op_cache_.end()) { + return; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return; + } + + // get Operator from op_cache_ + OperatorPtr op = Convert(node); + + adpt->updateOutputDesc(op, node->Shape(), node->Type(), node); +} + +OperatorPtr DfGraphConvertor::Convert(const AnfNodePtr node) { + if (node == nullptr) { + MS_LOG(ERROR) << "node is nullptr"; + error_ = NOT_FOUND; + return nullptr; + } + // find in cache + if (op_cache_.count(node.get())) { + return op_cache_[node.get()]; + } + + // do not convert primitive node + if (IsValueNode(node)) { + return nullptr; + } + + // convert a new one + if (node->isa()) { + return ConvertCNode(node->cast()); + } + if (node->isa()) { + return ConvertParameter(node); + } + if (node->isa()) { + return ConvertValueNode(node->cast()); + } + + MS_LOG(ERROR) << "Invalide AnfNode"; + error_ = INVALID_ARGUMENT; + return nullptr; +} + +void DfGraphConvertor::ConvertMakeTuple(const CNodePtr node) { + std::shared_ptr> tuple_items = std::make_shared>(); + // convert each tuple item to a OutHandler + for (size_t i = 1; i < node->inputs().size(); i++) { + AnfNodePtr item = node->input(i); + OperatorPtr op = Convert(item); + if (op != nullptr) { + tuple_items->emplace_back(OutHandler(op, "")); + } else if (out_handle_cache_.find(item.get()) != out_handle_cache_.end()) { + tuple_items->push_back(out_handle_cache_[item.get()]); + } else { + MS_LOG(WARNING) << "This anf node is not supported as a tuple item : " << item->ToString(); + return; + } + } + + tuple_out_handle_cache_[node.get()] = tuple_items; +} + +AnfNodePtr DfGraphConvertor::TraceTupleGetItem(const CNodePtr &node, unsigned int *index) { + const int TUPLE_GET_ITEM_INDEX = 2; + if (node->inputs().size() < 3) { // "tuple_getitem" primitive must have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs of TupleGetItem is less than 3"; + } + auto index_node = node->inputs()[TUPLE_GET_ITEM_INDEX]; + if (!index_node->isa()) { + error_ = INVALID_ARGUMENT; + MS_LOG(EXCEPTION) << "can't convert get item with non-constant index"; + } + *index = IntToUint(GetValue(GetValueNode(index_node))); + return node->inputs()[1]; +} + +AnfNodePtr DfGraphConvertor::TraceDepend(const CNodePtr &node) { + auto cnode = node->cast(); + if (cnode->inputs().size() < 3) { // "depend" primitive have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs of depend is less than 3"; + } + return cnode->inputs()[1]; +} + +AnfNodePtr DfGraphConvertor::TraceMakeTuple(const CNodePtr &node, unsigned int index) { + if (index + 1 >= node->inputs().size()) { + MS_LOG(EXCEPTION) << "length of make_tuple is less than index: " << index; + } + return node->inputs()[index + 1]; +} + +OutHandler DfGraphConvertor::GetHandler(const AnfNodePtr &node, const std::stack &index_stack, + AnfNode *const draw_index) { + if (node == nullptr) { + MS_LOG(ERROR) << "Get nullptr while trace real op"; + return OutHandler(nullptr, ""); + } + std::ostringstream ss; + ss << "op" << node.get(); + if (index_stack.empty()) { + op_draw_name_[draw_index] = ss.str(); + return OutHandler(Convert(node), ""); + } else { + OpAdapterPtr adpt = FindAdapter(node, training_); + if (nullptr == adpt) { + MS_LOG(ERROR) << "Can not get node output as adpt is nullptr!"; + error_ = NOT_FOUND; + return OutHandler(nullptr, ""); + } + OperatorPtr op = Convert(node); + if (op == nullptr) { + error_ = NOT_FOUND; + MS_LOG(ERROR) << "Can not convert node for trace real op"; + return OutHandler(nullptr, ""); + } + op_draw_name_[draw_index] = ss.str(); + return adpt->getOutput(Convert(node), UintToInt(index_stack.top())); + } +} + +// get the real operator through maketuple tuple_getitem depend +OutHandler DfGraphConvertor::TraceRealOp(AnfNodePtr node) { + bool flag = IsPrimitiveCNode(node, prim::kPrimTupleGetItem) || IsPrimitiveCNode(node, prim::kPrimMakeTuple) || + IsPrimitiveCNode(node, prim::kPrimDepend); + std::stack index_stack; + auto draw_index = node.get(); + while (flag) { + flag = false; + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + unsigned int index; + node = TraceTupleGetItem(node->cast(), &index); + index_stack.push(index); + flag = true; + } else if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + if (index_stack.empty()) { + MS_LOG(ERROR) << "TraceRealOp find a make_tuple node"; + return OutHandler(nullptr, ""); + } else { + node = TraceMakeTuple(node->cast(), index_stack.top()); + index_stack.pop(); + flag = true; + } + } else if (IsPrimitiveCNode(node, prim::kPrimDepend)) { + node = TraceDepend(node->cast()); + flag = true; + } + } + return GetHandler(node, index_stack, draw_index); +} + +void DfGraphConvertor::ConvertTupleGetItem(const CNodePtr node) { + auto handle = TraceRealOp(node); + if (handle.op == nullptr) { + MS_LOG(ERROR) << "Failed to trace tuple get item"; + return; + } + out_handle_cache_[node.get()] = handle; +} + +// Get the real op for tuple_getitem through make tuple, or depend +AnfNodePtr DfGraphConvertor::GetRealOpNode(AnfNodePtr node) { + const int TUPLE_GET_ITEM_INDEX = 2; + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + auto node_inputs = node->cast()->inputs(); + if (node_inputs.size() != 3) { // "tuple_getitem" primitive must have 3 inputs + MS_LOG(ERROR) << "tuple get item node not correct!"; + error_ = FAILED; + return node; + } + MS_EXCEPTION_IF_NULL(node_inputs[TUPLE_GET_ITEM_INDEX]); + if (!node_inputs[TUPLE_GET_ITEM_INDEX]->isa()) { + error_ = INVALID_ARGUMENT; + MS_LOG(EXCEPTION) << "can't convert get item with non-constant index"; + } + auto value_ptr = GetValueNode(node_inputs[TUPLE_GET_ITEM_INDEX])->cast(); + if (value_ptr == nullptr) { + MS_LOG(ERROR) << "Can not convert get item as value is nullptr!"; + error_ = FAILED; + return node; + } + int index = value_ptr->value(); + + // make_tuple apply inputs:make_tuple, [tuple_items,] + if (IsPrimitiveCNode(node_inputs[1], prim::kPrimMakeTuple)) { + auto tuple_inputs = node->cast()->inputs(); + if (tuple_inputs.size() < IntToSize(index + 1)) { + MS_LOG(ERROR) << "make tuple input items node not correct! size:" << tuple_inputs.size() + << ", item index:" << index; + error_ = FAILED; + return node; + } + return GetRealOpNode(tuple_inputs[IntToSize(index + 1)]); + } + return GetRealOpNode(node_inputs[1]); + } + + // depend apply inputs: depend,output,depended_node + if (IsPrimitiveCNode(node, prim::kPrimDepend)) { + auto depend_inputs = node->cast()->inputs(); + if (depend_inputs.size() != 3) { // "depend" primitive have 3 inputs + MS_LOG(ERROR) << "depend input items not correct"; + error_ = FAILED; + return node; + } + return GetRealOpNode(depend_inputs[1]); + } + return node; +} + +// convert the anf node to corresponding operator list +std::vector DfGraphConvertor::ConvertDependNode(const AnfNodePtr node) { + if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + std::vector op_lists; + auto node_inputs = node->cast()->inputs(); + for (size_t index = 1; index < node_inputs.size(); index++) { + auto op = Convert(GetRealOpNode(node_inputs[index])); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert control depend node to operator failed"; + error_ = FAILED; + return std::vector({}); + } + op_lists.push_back(op); + } + return op_lists; + } + + auto op = Convert(GetRealOpNode(node)); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert control depend node to operator failed"; + error_ = FAILED; + return std::vector({}); + } + return std::vector({op}); +} + +// get the anf node list for depend +std::vector DfGraphConvertor::GetDependNodes(const AnfNodePtr &node) { + std::vector nodes; + // for make tuple, should control depend on the tuple items + if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + auto node_inputs = node->cast()->inputs(); + for (size_t index = 1; index < node_inputs.size(); index++) { + nodes.push_back(GetRealOpNode(node_inputs[index])); + } + return nodes; + } + + // for parameter ,find the apply that used the parameter as the control depended node + if (node->isa()) { + auto uses = node->func_graph()->manager()->node_users()[node]; + for (auto &use : uses) { + auto use_node = use.first; + if ((use_node->isa()) && (!IsPrimitiveCNode(use_node, prim::kPrimControlDepend))) { + nodes.push_back(GetRealOpNode(use_node)); + } + } + return nodes; + } + nodes.push_back(GetRealOpNode(node)); + return nodes; +} + +void DfGraphConvertor::DrawControlDepend(const AnfNodePtr &src_node, const AnfNodePtr &dest_node) { +#ifdef DRAW_GE_GRAPH + auto src_depend_nodes = GetDependNodes(src_node); + auto dst_depend_nodes = GetDependNodes(dest_node); + if (src_depend_nodes.size() == 1 && dst_depend_nodes.size() > 1) { + for (auto &item : dst_depend_nodes) { + compute_sout_ << op_draw_name_[src_depend_nodes[0].get()] << " -> " << op_draw_name_[item.get()] + << "[style=\"dotted\"]" << endl; + } + } else if (src_depend_nodes.size() > 1 && dst_depend_nodes.size() == 1) { + for (auto &item : src_depend_nodes) { + compute_sout_ << op_draw_name_[item.get()] << " -> " << op_draw_name_[dst_depend_nodes[0].get()] + << "[style=\"dotted\"]" << endl; + } + } else if (src_depend_nodes.size() == 1 && dst_depend_nodes.size() == 1) { + compute_sout_ << op_draw_name_[src_depend_nodes[0].get()] << " -> " << op_draw_name_[dst_depend_nodes[0].get()] + << "[style=\"dotted\"]" << endl; + } +#endif +} + +void DfGraphConvertor::GetDependOnParameterUse(const CNodePtr &node, const AnfNodePtr &src_node, + const AnfNodePtr &dest_node, + const std::shared_ptr> &src_ops_list, + const std::shared_ptr> &dst_ops_list) { + if (src_node->isa()) { + auto uses = node->func_graph()->manager()->node_users()[src_node]; + for (auto &use : uses) { + auto use_node = use.first; + if ((use_node->isa()) && (!IsPrimitiveCNode(use_node, prim::kPrimControlDepend)) && + (!IsPrimitiveCNode(use_node, prim::kPrimMakeTuple))) { + auto converted_list = ConvertDependNode(use_node); + src_ops_list->insert(src_ops_list->end(), converted_list.begin(), converted_list.end()); + } + } + } + + if (dest_node->isa()) { + auto uses = node->func_graph()->manager()->node_users()[dest_node]; + for (auto &use : uses) { + auto use_node = use.first; + if ((use_node->isa()) && (!IsPrimitiveCNode(use_node, prim::kPrimControlDepend)) && + (!IsPrimitiveCNode(use_node, prim::kPrimMakeTuple))) { + auto converted_list = ConvertDependNode(use_node); + dst_ops_list->insert(dst_ops_list->end(), converted_list.begin(), converted_list.end()); + } + } + } +} + +bool DfGraphConvertor::GetControlDependList(const CNodePtr &node, + const std::shared_ptr> &src_ops_list, + const std::shared_ptr> &dst_ops_list) { + const int CONTROL_DEPEND_INDEX = 0; + const int SRC_NODE_INDEX = 1; + const int DEST_NODE_INDEX = 2; + const int DEPEND_MODE_NORMAL_USE = 0; + const int DEPEND_MODE_ON_PARAMETER_USE = 1; + + auto node_inputs = node->inputs(); + if (node_inputs.size() <= DEST_NODE_INDEX) { + MS_LOG(WARNING) << "Control depend node input size error"; + return false; + } + auto src_node = node_inputs[SRC_NODE_INDEX]; + auto dest_node = node_inputs[DEST_NODE_INDEX]; + if ((src_node == nullptr) || (dest_node == nullptr)) { + MS_LOG(ERROR) << "Control depend node miss src or dest node"; + error_ = FAILED; + return false; + } + AnfNodePtr fn = node_inputs[CONTROL_DEPEND_INDEX]; + PrimitivePtr prim_ptr = GetValueNode(fn); + ValuePtr mode_ptr = prim_ptr->GetAttr("depend_mode"); + int depend_mode = DEPEND_MODE_NORMAL_USE; + if (mode_ptr != nullptr) { + auto mode_int = mode_ptr->cast(); + MS_EXCEPTION_IF_NULL(mode_int); + depend_mode = mode_int->value(); + MS_LOG(DEBUG) << "depend_mode = " << depend_mode; + } + if (depend_mode == DEPEND_MODE_ON_PARAMETER_USE) { + GetDependOnParameterUse(node, src_node, dest_node, src_ops_list, dst_ops_list); + } + + if (src_node->isa()) { + auto converted_list = ConvertDependNode(src_node); + src_ops_list->insert(src_ops_list->end(), converted_list.begin(), converted_list.end()); + } + + if (dest_node->isa()) { + auto converted_list = ConvertDependNode(dest_node); + dst_ops_list->insert(dst_ops_list->end(), converted_list.begin(), converted_list.end()); + } + if (src_ops_list->empty() || dst_ops_list->empty()) { + MS_LOG(WARNING) << "Control depend node's src or dest node is not a apply node, ignore it"; + error_ = SUCCESS; + } + return true; +} + +void DfGraphConvertor::ConvertControlDependNode(const CNodePtr node) { + const int SRC_NODE_INDEX = 1; + const int DEST_NODE_INDEX = 2; + if (control_depend_cache_.find(node.get()) != control_depend_cache_.end()) { + return; + } + auto node_inputs = node->inputs(); + if (node_inputs.size() <= DEST_NODE_INDEX) { + MS_LOG(WARNING) << "Control depend node input size error"; + return; + } + auto src_node = node_inputs[SRC_NODE_INDEX]; + auto dest_node = node_inputs[DEST_NODE_INDEX]; + if ((src_node == nullptr) || (dest_node == nullptr)) { + MS_LOG(ERROR) << "Control depend node miss src or dest node"; + error_ = FAILED; + return; + } + std::shared_ptr> src_ops_list = std::make_shared>(); + std::shared_ptr> dst_ops_list = std::make_shared>(); + if (!GetControlDependList(node, src_ops_list, dst_ops_list)) { + MS_LOG(ERROR) << "Get depend list failed"; + error_ = FAILED; + return; + } + std::vector control_edges; + if (src_ops_list->size() == 1 && dst_ops_list->size() > 1) { + (void)std::transform(dst_ops_list->begin(), dst_ops_list->end(), std::back_inserter(control_edges), + [src_ops_list](const OperatorPtr &op) -> ControlEdge { + return {(*src_ops_list)[0], op}; + }); + } else if (src_ops_list->size() > 1 && dst_ops_list->size() == 1) { + (void)std::transform(src_ops_list->begin(), src_ops_list->end(), std::back_inserter(control_edges), + [dst_ops_list](const OperatorPtr &op) -> ControlEdge { + return {op, (*dst_ops_list)[0]}; + }); + } else if (src_ops_list->size() == 1 && dst_ops_list->size() == 1) { + control_edges.push_back({(*src_ops_list)[0], (*dst_ops_list)[0]}); + } else { + MS_LOG(ERROR) << "Convert control depend node to operator failed, depend src:" << src_ops_list->size() + << " -> dst:" << dst_ops_list->size(); + error_ = FAILED; + return; + } + control_depend_cache_[node.get()] = control_edges; + +#ifdef DRAW_GE_GRAPH + DrawControlDepend(src_node, dest_node); +#endif +} + +bool DfGraphConvertor::CheckCNode(const std::string &name, const CNodePtr node) { + // ignore apply node of return + if (name == "return" || name == "depend") { + return false; + } + + // make_tuple is used for a dynamic_input, convert it to a vector of OutHandlers + if (name == "make_tuple") { + ConvertMakeTuple(node); + return false; + } + + // As for nodes with multi outputs, convert tuple_getitem to OutHandle + if (name == "tuple_getitem") { + ConvertTupleGetItem(node); + return false; + } + + if (name == "ControlDepend") { + ConvertControlDependNode(node); + return false; + } + + return true; +} + +OperatorPtr DfGraphConvertor::ConvertCNode(const CNodePtr node) { + std::string name = GetCNodeFuncName(node); + if (!CheckCNode(name, node)) { + return nullptr; + } + + // get corresponding OpAdapter + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return nullptr; + } + + // get operator + OperatorPtr op = nullptr; + auto it_op = op_cache_.find(node.get()); + if (it_op != op_cache_.end()) { + op = it_op->second; + } else { + op = adpt->generate(node); + } + + // set attribute for primitive + (void)adpt->setAttr(op, node); + + // add into cache + (void)op_cache_.insert(std::make_pair(node.get(), op)); + + DrawCNode(node, adpt); + + return op_cache_[node.get()]; +} + +OperatorPtr DfGraphConvertor::ConvertParameter(const AnfNodePtr node) { + // convert Parameter in ANF to variable in DataFlow + auto op = FindAdapter(node, training_)->generate(node); + op_cache_[node.get()] = op; + + // build index for parameter using name + std::string name = std::static_pointer_cast(node)->name(); + params_[name] = node; + + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + compute_sout_ << ss.str() << "[shape=octagon, label=\"" << name << "\"]" << endl; + return op_cache_[node.get()]; +} + +Status DfGraphConvertor::TryConvertValueNodeToMultiConst(const ValueNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + ValuePtr value = node->value(); + MS_EXCEPTION_IF_NULL(value); + if (!value->isa() && !value->isa()) { + return FAILED; + } + + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + if (vec.empty()) { + return FAILED; + } + + std::shared_ptr> tuple_items = std::make_shared>(); + for (size_t i = 0; i < vec.size(); i++) { + MS_EXCEPTION_IF_NULL(vec[i]); + if (vec[i]->isa()) { + GeTensorPtr ge_tensor = transform::TransformUtil::ConvertTensor(vec[i]->cast(), kOpFormat_NCHW); + auto const_op = std::make_shared(node->fullname_with_scope() + "/const/inputs/" + std::to_string(i)); + (void)const_op->set_attr_value(*ge_tensor); + (void)const_op->update_output_desc_y(ge_tensor->GetTensorDesc()); + tuple_items->emplace_back(OutHandler(const_op, "")); + } else { + return FAILED; + } + } + if (tuple_items->empty()) { + return FAILED; + } + + tuple_out_handle_cache_[node.get()] = tuple_items; + return SUCCESS; +} + +OperatorPtr DfGraphConvertor::ConvertValueNode(const ValueNodePtr node) { + // convert valuenode in ANF to Const in DataFlow + // find paramerte referenced by SymbolicKeyInstance of valuenode + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + compute_sout_ << ss.str() << "[label= \"" << node->value()->ToString() << "\" shape=ellipse]" << endl; + + if (TryConvertValueNodeToMultiConst(node) == SUCCESS) { + MS_LOG(INFO) << "Convert value node to multi Constant OP success"; + return nullptr; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return nullptr; + } + auto op = adpt->generate(node); + // set const's attrs + if (adpt->setAttr(op, "value", node->value()) != 0) { + MS_LOG(WARNING) << "set attr value for const failed"; + } + +#if (defined ENABLE_GE) + auto const_op = std::static_pointer_cast(op); + if (const_op == nullptr) { + MS_LOG(ERROR) << "Get Constant operator failed"; + return nullptr; + } + auto ge_tensor = const_op->get_attr_value(); + auto ge_desc = ge_tensor.GetTensorDesc(); + (void)const_op->update_output_desc_y(ge_desc); +#endif + + op_cache_[node.get()] = op; + return op_cache_[node.get()]; +} + +void DfGraphConvertor::DrawCNode(const CNodePtr node, const OpAdapterPtr adpt) { + if (nullptr == adpt || nullptr == node) { + MS_LOG(ERROR) << "Failed to draw apply node as adpt or node is nullptr!"; + return; + } + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + + compute_sout_ << ss.str() << "[label=<"; + compute_sout_ << "" << endl; + + auto input_map = adpt->getInputMap(); + auto dyn_input_map = adpt->getDynInputMap(); + if (input_map.size() + dyn_input_map.size() > 0) { + compute_sout_ << ""; + for (auto &it : input_map) { + compute_sout_ << ""; + } + for (auto &it : dyn_input_map) { + compute_sout_ << ""; + } + compute_sout_ << "" << endl; + } + + compute_sout_ << "" << endl; + + // print attrs' values + auto atts = adpt->GetAttrsFromDrawGraph(); + for (auto &it : atts) { + compute_sout_ << ""; + } + + adpt->clearAttrVect(); + + compute_sout_ << "
" << it.second.name << "" << it.second.name << "
\"" << node->ToString() + << ":" << GetCNodeFuncName(node) << "\"
\"" << it + << "\"
> shape=plaintext]" << endl; +} +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/convert.h b/mindspore/ccsrc/transform/convert.h new file mode 100644 index 00000000000..e38b0b2b3a8 --- /dev/null +++ b/mindspore/ccsrc/transform/convert.h @@ -0,0 +1,260 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_CONVERT_H_ +#define MINDSPORE_CCSRC_TRANSFORM_CONVERT_H_ + +#define DRAW_GE_GRAPH + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "transform/util.h" +#include "ir/meta_tensor.h" +#include "transform/df_graph_manager.h" +#include "utils/config_manager.h" +#include "transform/op_declare.h" +#include "graph/operator_reg.h" +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif +#include "graph/tensor.h" +#include "ops/all_ops.h" + +namespace mindspore { +namespace transform { +class OpAdapterDesc { + public: + OpAdapterDesc() : train_(nullptr), infer_(nullptr) {} + + OpAdapterDesc(const OpAdapterPtr& train, const OpAdapterPtr& infer) : train_(train), infer_(infer) {} + + explicit OpAdapterDesc(const OpAdapterPtr& common) : train_(common), infer_(common) {} + + OpAdapterDesc(const OpAdapterDesc& desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + } + + OpAdapterDesc(OpAdapterDesc&& desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + desc.train_ = nullptr; + desc.infer_ = nullptr; + } + + ~OpAdapterDesc() = default; + + OpAdapterPtr Get(bool train) const { return train ? train_ : infer_; } + + OpAdapterDesc& operator=(const OpAdapterDesc& desc) { + if (this != &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + } + return *this; + } + + OpAdapterDesc& operator=(OpAdapterDesc&& desc) { + if (this != &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + desc.train_ = nullptr; + desc.infer_ = nullptr; + } + return *this; + } + + private: + OpAdapterPtr train_; + OpAdapterPtr infer_; +}; + +using OpAdapterDescPtr = std::shared_ptr; +using TensorOrderMap = std::map>; + +class DfGraphConvertor { + public: + explicit DfGraphConvertor(const AnfGraphPtr& anf_graph) + : anf_graph_(anf_graph), df_graph_(std::make_shared(anf_graph_->ToString())) { +#if (!defined ENABLE_GE) || (defined ENABLE_INFER) + auto it_training = anf_graph->flags().find("training"); + if (it_training != anf_graph->flags().end()) { + training_ = it_training->second; + } else { + training_ = false; + } +#else + training_ = ENABLE_TRAIN; +#endif + auto it_distribute = anf_graph->flags().find("broadcast_flag"); + if (it_distribute != anf_graph->flags().end()) { + ConfigManager::GetInstance().set_parallel_strategy(ParallelStrategy::DISTRIBUTION); + distribute_ = it_distribute->second; + } else { + ConfigManager::GetInstance().set_parallel_strategy(ParallelStrategy::ONE_DEVICE); + distribute_ = false; + } + + MS_LOG(INFO) << "Create DfGraphConvertor with training: " << training_ << ", distribute: " << distribute_; + } + + ~DfGraphConvertor() {} + + static void RegisterAdapter(const std::string& name, OpAdapterPtr adpt) { + get_adpt_map()[name] = std::make_shared(adpt); + } + static void RegisterAdapter(const std::string& name, OpAdapterPtr train_adpt, OpAdapterPtr infer_adpt) { + get_adpt_map()[name] = std::make_shared(train_adpt, infer_adpt); + } + + void DrawComputeGraph(const std::string& name) { + std::ofstream fout(name); + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open file '" << name << "' failed!"; + return; + } + fout << compute_sout_.str(); + fout.close(); + } + void DrawInitGraph(const std::string& name) { + std::ofstream fout(name); + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open file '" << name << "' failed!"; + return; + } + fout << init_sout_.str(); + fout.close(); + } + void DrawSaveCheckpointGraph(const std::string& name) { + std::ofstream fout(name); + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open file '" << name << "' failed!"; + return; + } + fout << checkpoint_sout_.str(); + fout.close(); + } + + DfGraphConvertor& ConvertAllNode(); + DfGraphConvertor& BuildGraph(); + DfGraphConvertor& InitParam(const TensorOrderMap& tensors); + DfGraphConvertor& GenerateCheckpointGraph(); + DfGraphConvertor& GenerateBroadcastGraph(const TensorOrderMap& tensors); + void InitParamWithData(const TensorOrderMap& tensors); + void SetOpInput(const OpAdapterPtr& adpt, const CNodePtr& node); + void SetupBroadcast(const std::shared_ptr& broadcast, const std::vector& broadcast_desc, + const DfGraphPtr& broadcast_graph, std::vector broadcast_input); + void MakeDatasetHandler(const std::string& name, const size_t& input_idx, const AnfNodePtr& it); + void SetupParamInitSubGraph(const TensorOrderMap& tensors, std::vector* init_input); + void DrawParamInitSubGraph(const std::string& name, const AnfNodePtr& it); + + DfGraphPtr GetComputeGraph(); + DfGraphPtr GetInitGraph(); + DfGraphPtr GetSaveCheckpointGraph(); + DfGraphPtr GetBroadcastGraph(); + static OpAdapterPtr FindAdapter(const std::string& op_name, bool train = false); + static OpAdapterPtr FindAdapter(AnfNodePtr node, bool train = false); + int ErrCode() const { return static_cast(error_); } + + static std::unordered_map& get_adpt_map(); + bool is_training() const { return training_; } + void set_training(bool is_training) { training_ = is_training; } + + protected: + void InitLoopVar(std::vector* init_input); + + private: + std::ostringstream compute_sout_; + std::ostringstream init_sout_; + std::ostringstream checkpoint_sout_; + std::ostringstream restore_checkpoint_sout_; + std::unordered_map op_draw_name_; + + AnfNodePtr TraceTupleGetItem(const CNodePtr& node, unsigned int* index); + AnfNodePtr TraceMakeTuple(const CNodePtr& node, unsigned int index); + AnfNodePtr TraceDepend(const CNodePtr& node); + OutHandler TraceRealOp(AnfNodePtr node); + OutHandler GetHandler(const AnfNodePtr& node, const std::stack& index_stack, AnfNode* const draw_index); + OperatorPtr Convert(AnfNodePtr node); + OperatorPtr ConvertCNode(CNodePtr node); + std::vector ConvertDependNode(AnfNodePtr node); + AnfNodePtr GetRealOpNode(AnfNodePtr node); + std::vector GetDependNodes(const AnfNodePtr& node); + OperatorPtr ConvertParameter(AnfNodePtr node); + Status TryConvertValueNodeToMultiConst(const ValueNodePtr node); + OperatorPtr ConvertValueNode(ValueNodePtr node); + void ConvertTupleGetItem(const CNodePtr node); + void GetDependOnParameterUse(const CNodePtr& node, const AnfNodePtr& src_node, const AnfNodePtr& dest_node, + const std::shared_ptr>& src_ops_list, + const std::shared_ptr>& dst_ops_list); + bool GetControlDependList(const CNodePtr& node, const std::shared_ptr>& src_ops_list, + const std::shared_ptr>& dst_ops_list); + void DrawControlDepend(const AnfNodePtr& src_node, const AnfNodePtr& dest_node); + void ConvertControlDependNode(const CNodePtr node); + void ConvertMakeTuple(const CNodePtr node); + bool CheckCNode(const std::string& name, const CNodePtr node); + void TraceOutput(AnfNodePtr node); + void TraceOutputFromParameter(const AnfNodePtr& anf_out); + void TraceOutputFromTupleGetItem(const AnfNodePtr& anf_out); + void SetNodeInput(AnfNodePtr node); + void SetOpControlInput(const AnfNodePtr node); + void UpdateOpDesc(AnfNodePtr node); + void BuildSaveCheckpointGraph(); + void DrawCNode(const CNodePtr node, const OpAdapterPtr adpt); + void UpdateDataOpDesc(const AnfNodePtr& it, const OperatorPtr& op) const; + void AddGraphConstInput(const OperatorPtr& op); + + std::shared_ptr anf_graph_{nullptr}; + std::shared_ptr df_graph_{nullptr}; + std::shared_ptr init_graph_{nullptr}; + std::shared_ptr save_ckp_graph_{nullptr}; + std::shared_ptr restore_ckp_graph_{nullptr}; + std::shared_ptr broadcast_graph_{nullptr}; + std::unordered_map op_cache_; + std::unordered_map> control_depend_cache_; + /* record "tuple_getitem"<->"out_handler" mapping */ + std::unordered_map out_handle_cache_; + /* record "make_tuple"<->"out_handler vector" mapping */ + std::unordered_map>> tuple_out_handle_cache_; + std::unordered_map params_; + std::unordered_map vars_; + std::vector> graph_outputs_; + std::vector graph_const_inputs_; + std::vector init_ops_; + std::vector broadcast_ops_; + OperatorPtr dataset_iter_getnext_; + Status error_ = SUCCESS; + bool training_ = false; + bool distribute_ = false; +}; + +extern std::string GetCNodeFuncName(CNodePtr cnode); +} // namespace transform +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_TRANSFORM_CONVERT_H_ diff --git a/mindspore/ccsrc/transform/df_graph_manager.cc b/mindspore/ccsrc/transform/df_graph_manager.cc new file mode 100644 index 00000000000..3339f431451 --- /dev/null +++ b/mindspore/ccsrc/transform/df_graph_manager.cc @@ -0,0 +1,214 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/df_graph_manager.h" + +#include +#include +#include +#include + +#include "securec/include/securec.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/pipeline.h" +#include "utils/config_manager.h" +#ifndef NO_DLIB +#include "tdt/tsd_client.h" +#endif + +namespace mindspore { +namespace transform { +DfGraphWrapper::DfGraphWrapper(const std::string& name, const int& id, const DfGraphPtr& graph_ptr, + const OptionMap& options) + : name_(name), id_(id), graph_ptr_(graph_ptr), options_(options) {} + +DfGraphManager::DfGraphManager() { + graph_id_ = 0; + graph_runner_ptr_ = nullptr; + sess_ptr_ = nullptr; +} + +DfGraphManager::~DfGraphManager() { + // in python fisrt destroy after atexit but in c++ destoy before atexit + DeleteGraphRunner(); + DeleteGeSession(); + ClearGraph(); + parse::python_adapter::set_python_env_flag(false); +} + +DfGraphManager& DfGraphManager::GetInstance() { + static DfGraphManager instance; + return instance; +} + +int DfGraphManager::GenerateId() { + graph_id_++; + if (graph_id_ <= 0) { + graph_id_ = 1; + } + MS_LOG(INFO) << "Generate graph Id : " << graph_id_; + return graph_id_; +} + +Status DfGraphManager::AddGraph(const std::string& name, const DfGraphPtr& graph_ptr, const OptionMap& options) { + std::lock_guard lg(lock_); + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null, add graph failed"; + return Status::INVALID_ARGUMENT; + } + + if (graph_ptr == nullptr) { + MS_LOG(WARNING) << "The new graph {" << name << "}'s pointer is null, add graph failed"; + return Status::INVALID_ARGUMENT; + } + + int id = GenerateId(); + DfGraphWrapperPtr wrap_ptr = std::make_shared(name, id, graph_ptr, options); + auto ret = graphs_.emplace(name, wrap_ptr); + if (ret.second == false) { + MS_LOG(WARNING) << "The graph name:{ " << name << " }is already exists! The old graph will be overwritten!!"; + ret.first->second = wrap_ptr; + } + MS_LOG(INFO) << "Add graph " << name << " to GraphManager success!"; + return Status::SUCCESS; +} + +std::vector DfGraphManager::GetAllGraphs() { + std::lock_guard lg(lock_); + std::vector ret; + std::stringstream ss; + ss << "{ "; + for (auto it = graphs_.begin(); it != graphs_.end(); ++it) { + ss << it->first << ", "; + ret.emplace_back(it->second); + } + ss << "}"; + MS_LOG(INFO) << "Return graphs: " << ss.str(); + return ret; +} +std::set DfGraphManager::GetSavedGraphs() { return saved_graphs_; } + +void DfGraphManager::AddSavedGraphs(const std::string& id) { saved_graphs_.insert(id); } + +DfGraphWrapperPtr DfGraphManager::GetGraphByName(const std::string& name) { + std::lock_guard lg(lock_); + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null"; + return nullptr; + } + + auto it = graphs_.find(name); + if (it == graphs_.end()) { + MS_LOG(ERROR) << "Can't found graph name: " << name; + return nullptr; + } + MS_LOG(INFO) << "Return graph: " << name; + return it->second; +} + +void DfGraphManager::ClearGraph() noexcept { + std::lock_guard lg(lock_); + graphs_.clear(); + anf_graphs_.clear(); + MS_LOG(INFO) << "Remove all graphs in GraphManager"; +} + +void DfGraphManager::SetAnfGraph(const std::string& name, const AnfGraphPtr& anf_graph_ptr) { + DfGraphWrapperPtr df_graph = GetGraphByName(name); + if (df_graph == nullptr) { + MS_LOG(ERROR) << "Can't found graph name: " << name; + return; + } + std::lock_guard lg(lock_); + anf_graphs_[df_graph->id_] = anf_graph_ptr; +} + +AnfGraphPtr DfGraphManager::GetAnfGraph(uint32_t graph_id) { + std::lock_guard lg(lock_); + auto iter = anf_graphs_.find(graph_id); + if (iter == anf_graphs_.end()) { + MS_LOG(ERROR) << "Can't found anf graph, graph_id = " << graph_id; + return nullptr; + } + + return iter->second; +} + +void DfGraphManager::EraseAnfGraph() { + std::lock_guard lg(lock_); + anf_graphs_.clear(); +} + +void DfGraphManager::SetGeSession(const std::shared_ptr& sess_ptr) { + std::lock_guard lg(lock_); + if (sess_ptr == nullptr) { + MS_LOG(WARNING) << "You are adding a empty Ge Session"; + } + + if (sess_ptr_ == nullptr) { + MS_LOG(INFO) << "Add a new Ge Session success"; + } else { + MS_LOG(INFO) << "Add a new Ge Session success, the old Ge Session will be overwritten!!"; + } + sess_ptr_ = sess_ptr; +} + +std::shared_ptr DfGraphManager::GetGeSession() { + std::lock_guard lg(lock_); + return sess_ptr_; +} + +void DfGraphManager::DeleteGeSession() noexcept { + std::lock_guard lg(lock_); + if (sess_ptr_ == nullptr) { + MS_LOG(INFO) << "Ge Session is not exist"; + } else { + sess_ptr_ = nullptr; + saved_graphs_.clear(); + MS_LOG(INFO) << "Delete Ge Session success"; + } +} + +void DfGraphManager::SetGraphRunner(const std::shared_ptr& graph_runner_ptr) noexcept { + std::lock_guard lg(lock_); + if (graph_runner_ptr == nullptr) { + MS_LOG(WARNING) << "You are adding a empty GraphRunner"; + } + + if (graph_runner_ptr_ == nullptr) { + MS_LOG(INFO) << "Add a new GraphRunner success"; + } else { + MS_LOG(INFO) << "Add a new GraphRunner success, the old GraphRunner will be overwritten!!"; + } + graph_runner_ptr_ = graph_runner_ptr; +} + +std::shared_ptr DfGraphManager::GetGraphRunner() { + std::lock_guard lg(lock_); + return graph_runner_ptr_; +} + +void DfGraphManager::DeleteGraphRunner() noexcept { + std::lock_guard lg(lock_); + if (graph_runner_ptr_ == nullptr) { + MS_LOG(INFO) << "GraphRunner is not exist"; + } else { + graph_runner_ptr_ = nullptr; + MS_LOG(INFO) << "Delete GraphRunner success"; + } +} +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/df_graph_manager.h b/mindspore/ccsrc/transform/df_graph_manager.h new file mode 100644 index 00000000000..97137ae94bb --- /dev/null +++ b/mindspore/ccsrc/transform/df_graph_manager.h @@ -0,0 +1,86 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_DF_GRAPH_MANAGER_H_ +#define TRANSFORM_DF_GRAPH_MANAGER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "transform/types.h" +#include "ir/anf.h" + +namespace mindspore { +const char BROADCAST_GRAPH_NAME[] = "broadcast_subgraph"; + +namespace transform { +class GraphRunner; +using OptionMap = std::map; + +struct DfGraphWrapper { + public: + DfGraphWrapper(const std::string& name, const int& id, const DfGraphPtr& graph_ptr, const OptionMap& options); + ~DfGraphWrapper() {} + + std::string name_; + int id_; + DfGraphPtr graph_ptr_; + OptionMap options_ = {}; +}; + +using DfGraphWrapperPtr = std::shared_ptr; + +class DfGraphManager { + public: + ~DfGraphManager(); + void ClearGraph() noexcept; + + static DfGraphManager& GetInstance(); + Status AddGraph(const std::string& name, const DfGraphPtr& graph, const OptionMap& options = {}); + std::vector GetAllGraphs(); + std::set GetSavedGraphs(); + void AddSavedGraphs(const std::string& id); + DfGraphWrapperPtr GetGraphByName(const std::string& name); + DfGraphManager(const DfGraphManager&) = delete; + void SetAnfGraph(const std::string& name, const AnfGraphPtr& anf_graph_ptr); + AnfGraphPtr GetAnfGraph(uint32_t graph_id); + std::shared_ptr GetGraphRunner(); + void SetGraphRunner(const std::shared_ptr& graph_runner_ptr) noexcept; + void DeleteGraphRunner() noexcept; + void SetGeSession(const std::shared_ptr& sess_ptr); + std::shared_ptr GetGeSession(); + void DeleteGeSession() noexcept; + void EraseAnfGraph(); + + private: + DfGraphManager(); + int GenerateId(); + + std::mutex lock_; + std::map graphs_; + std::set saved_graphs_; + int graph_id_; + std::map anf_graphs_; + std::shared_ptr graph_runner_ptr_; + std::shared_ptr sess_ptr_; +}; +} // namespace transform +} // namespace mindspore + +#endif // TRANSFORM_DF_GRAPH_MANAGER_H_ diff --git a/mindspore/ccsrc/transform/graph_builder.cc b/mindspore/ccsrc/transform/graph_builder.cc new file mode 100644 index 00000000000..9c05969fb03 --- /dev/null +++ b/mindspore/ccsrc/transform/graph_builder.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_builder.h" + +#include +#include + +namespace mindspore { +namespace transform { +DfGraphPtr BuildMDDatasetGraph(const DatasetGraphParam& param) { + MS_LOG(INFO) << "BuildMDDatasetGraph."; + + // InitData + auto d = ge::op::InitData("init_data_tmp").set_attr_channel_name(param.queue_name()); + + // set graph inputs & outputs + std::vector inputs{d}; + std::vector outputs{d}; + DfGraphPtr dataset_graph = std::make_shared("dataset"); + (void)dataset_graph->SetInputs(inputs); + (void)dataset_graph->SetOutputs(outputs); + + return dataset_graph; +} + +Status BuildDatasetGraph(const DatasetGraphParam& param, const std::string& phase) { + Status ret; + std::string graph_name = phase; + + MS_LOG(INFO) << "BuildDatasetGraph begin. phase is " << phase; + MS_LOG(INFO) << "param is " << param.ToString() << "."; + + DfGraphPtr dataset_graph = BuildMDDatasetGraph(param); + ret = DfGraphManager::GetInstance().AddGraph(graph_name, dataset_graph); + if (ret != Status::SUCCESS) { + MS_LOG(ERROR) << "BuildDatasetGraph failed."; + } else { + MS_LOG(INFO) << "BuildDatasetGraph end."; + } + return ret; +} +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/graph_builder.h b/mindspore/ccsrc/transform/graph_builder.h new file mode 100644 index 00000000000..30b891460bb --- /dev/null +++ b/mindspore/ccsrc/transform/graph_builder.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_GRAPH_BUILDER_H_ +#define TRANSFORM_GRAPH_BUILDER_H_ + +#include +#include +#include +#include +#include +#include "transform/types.h" +#include "transform/convert.h" + +namespace mindspore { +namespace transform { +Status BuildDatasetGraph(const DatasetGraphParam& param, const std::string& phase = "dataset"); +} // namespace transform +} // namespace mindspore + +#endif // TRANSFORM_GRAPH_BUILDER_H_ diff --git a/mindspore/ccsrc/transform/graph_runner.cc b/mindspore/ccsrc/transform/graph_runner.cc new file mode 100644 index 00000000000..e77b1bcd736 --- /dev/null +++ b/mindspore/ccsrc/transform/graph_runner.cc @@ -0,0 +1,203 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + */ + +#include "transform/graph_runner.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "utils/config_manager.h" +#include "sys/time.h" +#include "utils/callbacks.h" +#include "utils/utils.h" +#include "./common.h" + +#ifdef NO_GE_CLIENT +namespace ge { +Session::Session(const std::map& options) { + if (options.empty()) { + MS_LOG(ERROR) << "session input options is empty"; + } + sessionId_ = 0; +} +Session::~Session() {} +} // namespace ge +#endif + +namespace mindspore { +namespace transform { +std::shared_ptr GraphRunner::NewSession(const SessionOptions& sess_options) { + std::shared_ptr ret = std::make_shared(sess_options); + if (ret == nullptr) { + MS_LOG(ERROR) << "Create GE session failed"; + return nullptr; + } + MS_LOG(INFO) << "Create new GE session success"; + return ret; +} + +GraphRunner::GraphRunner(const GraphRunnerOptions& options) + : options_(options), graph_manager_(DfGraphManager::GetInstance()) { + if (ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::ONE_DEVICE) { + MS_LOG(INFO) << "ME run in ONE_DEVICE strategy mode"; + } + + if (options.sess_ptr != nullptr) { + sess_ = options.sess_ptr; + } else { + sess_ = NewSession(options.options); + if (sess_ == nullptr) { + MS_LOG(EXCEPTION) << "GraphRunner initialize failed!!"; + return; + } + } + +#if (defined ENABLE_GE) + // register the callback function + if (sess_->RegisterCallBackFunc(callbacks::kCheckPoint, callbacks::CheckpointSaveCallback) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "register callback failed!"; + return; + } + + if (sess_->RegisterCallBackFunc(callbacks::kSummary, callbacks::SummarySaveCallback) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "register summary callback failed!"; + return; + } +#endif + + std::vector wrappers = graph_manager_.GetAllGraphs(); + if (wrappers.empty()) { + MS_LOG(INFO) << "The GraphManager is empty!!"; + return; + } + +#ifdef ENABLE_GE + for (auto& it : wrappers) { + std::set saved_graph = graph_manager_.GetSavedGraphs(); + auto iter_find = saved_graph.find(std::to_string(it->id_)); + if (iter_find != saved_graph.end()) { + continue; + } + MS_LOG(INFO) << "Add the graph " << (*it).name_ << " to GE, it's id is: " << (*it).id_; + graph_manager_.AddSavedGraphs(std::to_string(it->id_)); + (void)sess_->AddGraph(it->id_, *(it->graph_ptr_), it->options_); + } +#endif +} + +Status GraphRunner::RunGraph(const RunOptions& options, const std::vector& inputs, + std::vector* outputs) { + std::string name = options.name; + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null"; + return Status::INVALID_ARGUMENT; + } + + DfGraphWrapperPtr wrap_ptr = graph_manager_.GetGraphByName(name); + if (wrap_ptr == nullptr) { + MS_LOG(ERROR) << "Get graph form DfGraphManager failed!"; + return Status::NOT_FOUND; + } + + if (wrap_ptr->graph_ptr_ == nullptr) { + MS_LOG(WARNING) << "The graph is null"; + return Status::NOT_FOUND; + } + + // call ge::RunGraph() to exec a graph; + std::vector ge_inputs; + std::vector ge_outputs; + + (void)std::transform(inputs.begin(), inputs.end(), std::back_inserter(ge_inputs), + [](const GeTensorPtr& i) { return *i; }); + + MS_LOG(INFO) << "Run the graph in GE with " << ge_inputs.size() << " inputs"; + + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + +#ifdef ENABLE_GE + if (sess_ == nullptr) { + MS_LOG(ERROR) << "The GE session is null, can't run the graph!"; + return Status::FAILED; + } + + ge::Status ret = sess_->RunGraph(wrap_ptr->id_, ge_inputs, ge_outputs); + if (ret != ge::GRAPH_SUCCESS) { + MS_LOG(ERROR) << "Call GE RunGraph Failed, ret is: " << ret; + return Status::FAILED; + } +#else + ge_outputs.swap(ge_inputs); +#endif + + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Call GE RunGraph Success in " << cost << " us, the GE outputs num is: " << ge_outputs.size(); + + (void)std::transform(ge_outputs.begin(), ge_outputs.end(), std::back_inserter(*outputs), + [](const GeTensor& ge_tensor) { return std::make_shared(ge_tensor); }); + + return Status::SUCCESS; +} + +Status GraphRunner::RunGraph(const RunOptions& options, const std::vector& inputs, + std::vector* const outputs) { + std::vector ge_inputs; + for (auto it : inputs) { + MS_LOG(INFO) << "inputs tensor's data size is: " << (*it).DataSize(); + auto shape = (*it).shape(); + std::string shape_str; + for (const auto& elem : shape) { + shape_str += std::to_string(elem); + shape_str += " "; + } + MS_LOG(INFO) << "inputs tensor's shape is: { " << shape_str << "}"; + + auto ge_tensor_ptr = TransformUtil::ConvertTensor(it, kOpFormat_NCHW); + if (ge_tensor_ptr != nullptr) { + ge_inputs.emplace_back(ge_tensor_ptr); + } else { + MS_LOG(INFO) << "Convert input Me tensor to Ge tensor failed. Abort this graph"; + return Status::FAILED; + } + } + + std::vector ge_outputs; + Status ret; + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + ret = RunGraph(options, ge_inputs, &ge_outputs); + } + if (ret != Status::SUCCESS) { + return ret; + } else { + // conver GeTensor to MeTensor + for (auto& it : ge_outputs) { + auto tensor = TransformUtil::ConvertGeTensor(it); + if (tensor != nullptr) { + outputs->emplace_back(tensor); + } + } + MS_LOG(INFO) << "Return Me tensor outputs num is: " << outputs->size(); + return Status::SUCCESS; + } +} +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/graph_runner.h b/mindspore/ccsrc/transform/graph_runner.h new file mode 100644 index 00000000000..a9aa9fbc59a --- /dev/null +++ b/mindspore/ccsrc/transform/graph_runner.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_GRAPH_RUNNER_H_ +#define TRANSFORM_GRAPH_RUNNER_H_ + +#include +#include +#include +#include +#include + +#include "transform/types.h" +#include "transform/util.h" +#include "ir/meta_tensor.h" +#include "transform/df_graph_manager.h" + +namespace mindspore { +namespace transform { +using SessionOptions = std::map; + +struct GraphRunnerOptions { + std::string target{"default_graph_runner"}; + SessionOptions options; + // if sess_ptr is nullptr, GraphRunner will create a new ge session + std::shared_ptr sess_ptr{nullptr}; +}; + +struct RunOptions { + // graph's name + std::string name; +}; + +class GraphRunner { + public: + explicit GraphRunner(const GraphRunnerOptions& options); + ~GraphRunner() { sess_ = nullptr; } + Status RunGraph(const RunOptions& options, const std::vector& inputs, std::vector* outputs); + Status RunGraph(const RunOptions& options, const std::vector& inputs, std::vector* outputs); + static std::shared_ptr NewSession(const SessionOptions& sess_options); + + private: + std::shared_ptr sess_; + transform::GraphRunnerOptions options_; + DfGraphManager& graph_manager_; +}; +} // namespace transform +} // namespace mindspore + +#endif // TRANSFORM_GRAPH_RUNNER_H_ diff --git a/mindspore/ccsrc/transform/op_adapter.h b/mindspore/ccsrc/transform/op_adapter.h new file mode 100644 index 00000000000..aa466adbb8a --- /dev/null +++ b/mindspore/ccsrc/transform/op_adapter.h @@ -0,0 +1,846 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_OP_ADAPTER_H_ +#define TRANSFORM_OP_ADAPTER_H_ + +#include +#include +#include +#include + +#include "transform/op_adapter_util.h" +#include "utils/utils.h" +namespace mindspore { +namespace transform { +static uint32_t CustomInferFunc(const Operator&) { return 0; } + +template +class OpAdapter : public BaseOpAdapter { + public: + using OpType = T; + OpAdapter() {} + explicit OpAdapter(const ExtraAttr& extra_attr) : extra_attr_(extra_attr) {} + ~OpAdapter() override {} + + bool IsCustomOp(const OperatorPtr& op) { + MS_EXCEPTION_IF_NULL(op); + auto it = cus_input_map_.find(op->GetOpType()); + if (it == cus_input_map_.end()) { + return false; + } + return true; + } + + Status GenerateCustomOpInputMap(const CusOperatorPtr& op, const PrimitivePtr& prim) { + MS_EXCEPTION_IF_NULL(op); + MS_EXCEPTION_IF_NULL(prim); + // Create the map of custom op from input index to input name. + std::unordered_map input_map; + auto value = prim->GetAttr("input_names"); + if (value == nullptr) { + cus_output_map_[prim->name()] = input_map; + return NOT_FOUND; + } + + auto input_names = GetValue>(value); + for (size_t i = 0; i < input_names.size(); ++i) { + // input_map begin form 1 + input_map[i + 1] = input_names[i]; + op->CustomInputRegister(input_names[i]); + } + + if (cus_input_map_.find(prim->name()) == cus_input_map_.end()) { + cus_input_map_[prim->name()] = input_map; + } + return SUCCESS; + } + + Status GenerateCustomOpOutputMap(const CusOperatorPtr& op, const PrimitivePtr& prim) { + MS_EXCEPTION_IF_NULL(op); + MS_EXCEPTION_IF_NULL(prim); + // Create the map of custom op from output index to output name. + std::unordered_map output_map; + auto value = prim->GetAttr("output_names"); + if (value == nullptr) { + // generate a empty output_map for it + cus_output_map_[prim->name()] = output_map; + return NOT_FOUND; + } + + auto output_names = GetValue>(value); + for (size_t i = 0; i < output_names.size(); ++i) { + // output_map begin form 0 + output_map[i] = output_names[i]; + op->CustomOutputRegister(output_names[i]); + } + + if (cus_output_map_.find(prim->name()) == cus_output_map_.end()) { + cus_output_map_[prim->name()] = output_map; + } + return SUCCESS; + } + + // Convert ME UserCustom AnfNode to GE CustomOp. And set it's attrs. + OperatorPtr GenerateCustomOp(const AnfNodePtr anf) { + MS_EXCEPTION_IF_NULL(anf); + auto node = anf->cast(); + if (node == nullptr) { + return nullptr; + } + + if (node->inputs().empty()) { + MS_LOG(EXCEPTION) << "length of node inputs is empty"; + } + + auto prim = GetValueNode(node->inputs()[0]); + MS_EXCEPTION_IF_NULL(prim); + auto op = std::make_shared(node->fullname_with_scope(), prim->name()); + if (GenerateCustomOpInputMap(op, prim) != SUCCESS) { + MS_LOG(WARNING) << "Custom op node has no input_names, op[" << prim->name() << "]."; + } + + if (GenerateCustomOpOutputMap(op, prim) != SUCCESS) { + MS_LOG(WARNING) << "Custom op node has no output_names, op[" << prim->name() << "]."; + } + + op->CustomInferFuncRegister(CustomInferFunc); + + return op; + } + + OperatorPtr GenerateNormalOp(const AnfNodePtr& anf) { + OperatorPtr op = nullptr; + // There are duplicate names in ANF graph, do not assign ANF node name to GE + // GE will generate unique name automatically + if (anf != nullptr && anf->fullname_with_scope() != "") { + MS_LOG(DEBUG) << anf->fullname_with_scope(); + op = std::make_shared(anf->fullname_with_scope()); + } else { + MS_LOG(DEBUG) << "no fullname_with_scope"; + op = std::make_shared(); + } + + // set dynamic output num if op use DYNAMIC_OUTPUT + if ((op != nullptr) && (!dyn_output_map_.empty()) && (anf != nullptr)) { + TypePtr type = anf->Type(); + if (type == nullptr) { + MS_LOG(EXCEPTION) << "Dynamic output node:" << op->GetName() << "'s Type is a nullptr!"; + } + size_t num = type->isa() ? (type->cast>()->size()) : 1; + MS_LOG(INFO) << "create_dyn_output for node:" << anf->ToString() << ", type:" << type->ToString() + << ", num:" << num; + dyn_output_map_.begin()->second.create_dyn_output(op, static_cast(num)); + } + return op; + } + + OperatorPtr generate(const AnfNodePtr& anf) override { + OperatorPtr op = nullptr; + if (IsCustomCNode(anf)) { + op = GenerateCustomOp(anf); + } else { + op = GenerateNormalOp(anf); + } + return op; + } + + OperatorPtr generate(const std::string& op_name) override { return std::make_shared(op_name); } + + const std::unordered_map& getInputMap() override { return input_map_; } + const std::unordered_map& getInputAttrMap() override { return input_attr_map_; } + const std::unordered_map& getDynInputMap() override { return dyn_input_map_; } + const std::unordered_map& getOutputMap() override { return output_map_; } + + Status SetCustomOpInput(const CusOperatorPtr& op, int index, const OperatorPtr& input) { + MS_EXCEPTION_IF_NULL(op); + MS_EXCEPTION_IF_NULL(input); + auto it = cus_input_map_.find(op->GetOpType()); + if (it == cus_input_map_.end()) { + return NOT_FOUND; + } + std::unordered_map& input_map = it->second; + + if ((input_map.find(index) != input_map.end())) { + MS_LOG(DEBUG) << "Link op " << input->GetName() << " to " << op->GetName() << ":" << input_map[index]; + (void)op->SetInput(input_map[index], *input); + return SUCCESS; + } + return NOT_FOUND; + } + + Status SetNormalOpInput(const OperatorPtr& op, int index, const OperatorPtr& input) { + MS_EXCEPTION_IF_NULL(op); + auto it = input_map_.find(index); + if (it != input_map_.end()) { + MS_EXCEPTION_IF_NULL(input); + MS_LOG(DEBUG) << "Link op " << input->GetName() << " to " << op->GetName() << ":" << it->second.name; + it->second.set_op(op, input); + return SUCCESS; + } + return NOT_FOUND; + } + + int setInput(const OperatorPtr& op, int index, const OperatorPtr& input) override { + if (IsCustomOp(op)) { + auto cus_op = std::dynamic_pointer_cast(op); + return static_cast(SetCustomOpInput(cus_op, index, input)); + } else { + return static_cast(SetNormalOpInput(op, index, input)); + } + } + + Status SetCustomOpInput(const CusOperatorPtr& op, int index, const OutHandler& handle) { + MS_EXCEPTION_IF_NULL(op); + auto it = cus_input_map_.find(op->GetOpType()); + if (it == cus_input_map_.end()) { + return NOT_FOUND; + } + + std::unordered_map& input_map = it->second; + if ((handle.op != nullptr) && (input_map.find(index) != input_map.end())) { + if (handle.out.empty()) { + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << " to " << op->GetName() << ":" << input_map[index]; + (void)op->SetInput(input_map[index], *(handle.op)); + } else { + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << ":" << handle.out << " to " << op->GetName() << ":" + << input_map[index]; + (void)op->SetInput(input_map[index], *(handle.op), handle.out); + } + return SUCCESS; + } + return NOT_FOUND; + } + + Status SetNormalOpInput(const OperatorPtr& op, int index, const OutHandler& handle) { + MS_EXCEPTION_IF_NULL(op); + auto it = input_map_.find(index); + if ((handle.op != nullptr) && (it != input_map_.end())) { + if (handle.out.empty()) { + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << " to " << op->GetName() << ":" << it->second.name; + it->second.set_op(op, handle.op); + } else { + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << ":" << handle.out << " to " << op->GetName() << ":" + << it->second.name; + it->second.set_handle(op, handle); + } + return SUCCESS; + } + return NOT_FOUND; + } + + int setInput(const OperatorPtr& op, int index, const OutHandler& handle) override { + if (IsCustomOp(op)) { + auto cus_op = std::dynamic_pointer_cast(op); + return static_cast(SetCustomOpInput(cus_op, index, handle)); + } else { + return static_cast(SetNormalOpInput(op, index, handle)); + } + } + + int setInput(const OperatorPtr& op, int index, const std::shared_ptr>& handler_vec) override { + MS_EXCEPTION_IF_NULL(handler_vec); + if (IsCustomOp(op)) { + MS_LOG(ERROR) << "Custom Op do not support dynamic input"; + return static_cast(FAILED); + } + MS_EXCEPTION_IF_NULL(op); + auto it = dyn_input_map_.find(index); + if (it != dyn_input_map_.end()) { + it->second.create_dyn_input(op, static_cast(handler_vec->size())); + for (unsigned int i = 0; i < handler_vec->size(); ++i) { + OutHandler h = (*handler_vec)[i]; + MS_EXCEPTION_IF_NULL(h.op); + if (h.out.empty()) { + MS_LOG(DEBUG) << "Link op " << h.op->GetName() << " to " << op->GetName() << ":" << it->second.name; + it->second.set_op(op, (i) /* index start from 0 */, h.op); + } else { + MS_LOG(DEBUG) << "Link op " << h.op->GetName() << ":" << h.out << " to " << op->GetName() << ":" + << it->second.name; + it->second.set_handle(op, i, h); + } + } + return 0; + } + return static_cast(NOT_FOUND); + } + + OutHandler getOutput(const OperatorPtr& op, int index) override { + MS_EXCEPTION_IF_NULL(op); + if (!dyn_output_map_.empty() && !output_map_.empty()) { + MS_LOG(ERROR) << "OpAdpator(" << op->GetName() << ") has both OUTPUT and DYN_OUTPUT is not supported!"; + return OutHandler(); + } + auto it = output_map_.find(index); + if (it != output_map_.end()) { + return OutHandler(op, it->second.name); + } else if (!dyn_output_map_.empty()) { + return OutHandler(op, dyn_output_map_.begin()->second.name + std::to_string(index)); + } else { + MS_LOG(ERROR) << "OpAdpator(" << op->GetName() << ") has no OUTPUT and DYN_OUTPUT index(" << index << ")!"; + return OutHandler(); + } + } + + Status UpdateSingleOutputDesc(const OperatorPtr& op, const abstract::BaseShapePtr& shp, const TypePtr& type) { + MS_EXCEPTION_IF_NULL(type); + TypeId me_type = type->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(type)->element()->type_id(); + } + + std::vector shape; + auto normal_shape_ptr = dyn_cast(shp); + if (nullptr != normal_shape_ptr) { + shape = normal_shape_ptr->shape(); + } + + auto desc = TransformUtil::GetGeTensorDesc(shape, me_type, "NCHW"); + if (desc == nullptr) { + MS_LOG(ERROR) << "Update output descriptor failed!"; + return FAILED; + } + + if (IsCustomOp(op)) { + if (cus_output_map_.find(op->GetOpType()) == cus_output_map_.end() || + (cus_output_map_[op->GetOpType()].empty())) { + MS_LOG(ERROR) << "This op does not create custom output map"; + return FAILED; + } + auto cus_op = std::dynamic_pointer_cast(op); + MS_EXCEPTION_IF_NULL(cus_op); + std::unordered_map output_map = cus_output_map_[op->GetOpType()]; + (void)cus_op->UpdateOutputDesc(output_map[0], *desc); + } else { + if (output_map_.empty()) { + MS_LOG(INFO) << "This op does not have output map"; + return FAILED; + } + output_map_.begin()->second.update_out_desc(op, *desc); + } + return SUCCESS; + } + + size_t GetCustomOpOutputSize(const CusOperatorPtr& cus_op) { + MS_EXCEPTION_IF_NULL(cus_op); + if (cus_output_map_.find(cus_op->GetOpType()) == cus_output_map_.end()) { + MS_LOG(ERROR) << "This op does not create custom output map"; + return 0; + } + size_t output_size = cus_output_map_[cus_op->GetOpType()].size(); + return output_size; + } + + std::shared_ptr CreateOutputDesc(const abstract::ShapePtr& shape_ptr, const TypePtr& type, + const std::string& format) { + if (shape_ptr == nullptr) { + MS_LOG(ERROR) << "Shape ptr is nullptr"; + return nullptr; + } + + if (type == nullptr) { + MS_LOG(ERROR) << "Type ptr is nullptr"; + return nullptr; + } + + TypeId me_type = type->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(type)->element()->type_id(); + } + auto desc = TransformUtil::GetGeTensorDesc(shape_ptr->shape(), me_type, format); + return desc; + } + + Status UpdateMultiOutputDesc(const OperatorPtr& op, const abstract::BaseShapePtr& shp, const TypePtr& type) { + auto tuple_shp = dyn_cast(shp); + MS_EXCEPTION_IF_NULL(tuple_shp); + + size_t output_size = 0; + bool is_custom_op = IsCustomOp(op); + if (is_custom_op) { + output_size = GetCustomOpOutputSize(std::dynamic_pointer_cast(op)); + } else { + output_size = output_map_.size(); + } + + if (output_size == 0) { + MS_LOG(INFO) << "This op does not have output map"; + return FAILED; + } + + if (output_size != tuple_shp->shape().size()) { + MS_LOG(ERROR) << "output_map is not equal tuple_shape size"; + return FAILED; + } + for (size_t i = 0; i < tuple_shp->shape().size(); ++i) { + auto tuple_type = dyn_cast(type); + MS_EXCEPTION_IF_NULL(tuple_type); + TypePtr type_elem = tuple_type->elements()[i]; + std::string format = "NCHW"; + if (op->GetOpType() == kTopKOpName) { + format = "NHWC"; + } + auto desc = CreateOutputDesc(dyn_cast(tuple_shp->shape()[i]), type_elem, format); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create output descriptor failed!"; + return FAILED; + } + + if (is_custom_op) { + (void)std::dynamic_pointer_cast(op)->UpdateOutputDesc(cus_output_map_[op->GetOpType()][i], + *desc); + } else { + auto it = output_map_.find(i); + if (it != output_map_.end()) { + it->second.update_out_desc(op, *desc); + } + } + } + return SUCCESS; + } + + std::shared_ptr CreateNodeDesc(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + TypeId me_type = node->Type()->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(node->Type())->element()->type_id(); + } + if (me_type <= kNumberTypeBegin || me_type >= kNumberTypeEnd) { + return nullptr; + } + + std::vector shape; + auto shape_ptr = dyn_cast(node->Shape()); + if (nullptr != shape_ptr) { + shape = shape_ptr->shape(); + } + + auto desc = TransformUtil::GetGeTensorDesc(shape, me_type, "NCHW"); + if (desc == nullptr) { + MS_LOG(ERROR) << "Update output descriptor failed!"; + return nullptr; + } + return desc; + } + + void UpdateNormalOpInputDesc(const OperatorPtr& op, const AnfNodePtr node) { + if (op == nullptr) { + MS_LOG(ERROR) << "op is nullptr"; + return; + } + MS_EXCEPTION_IF_NULL(node); + + auto inputs = node->cast()->inputs(); + for (size_t i = 1; i < inputs.size(); ++i) { + auto it = input_map_.find(i); + if (it != input_map_.end()) { + auto desc = CreateNodeDesc(inputs[i]); + if (desc == nullptr) { + continue; + } + it->second.update_input_desc(op, *desc); + } + } + } + + void UpdateCustomOpInputDesc(const CusOperatorPtr& op, const AnfNodePtr& node) { + if (op == nullptr) { + MS_LOG(ERROR) << "op is nullptr"; + return; + } + MS_EXCEPTION_IF_NULL(node); + + if (cus_input_map_.find(op->GetOpType()) == cus_input_map_.end() || (cus_input_map_[op->GetOpType()].empty())) { + MS_LOG(ERROR) << "This op does not create custom input map"; + return; + } + + std::unordered_map& input_map = cus_input_map_[op->GetOpType()]; + auto inputs = node->cast()->inputs(); + for (size_t i = 1; i < inputs.size(); ++i) { + if (input_map.find(i) != input_map.end()) { + auto desc = CreateNodeDesc(inputs[i]); + if (desc == nullptr) { + continue; + } + (void)op->UpdateInputDesc(input_map[i], *desc); + } + } + } + + void updateInputDesc(const OperatorPtr& op, const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(op); + MS_EXCEPTION_IF_NULL(node); + if (IsCustomOp(op)) { + auto cus_op = std::dynamic_pointer_cast(op); + UpdateCustomOpInputDesc(cus_op, node); + } else { + UpdateNormalOpInputDesc(op, node); + } + } + + void updateOutputDesc(const OperatorPtr& op, const abstract::BaseShapePtr& shp, const TypePtr& type, + const AnfNodePtr& node) override { + if (op == nullptr) { + MS_LOG(ERROR) << "op is nullptr"; + return; + } + MS_EXCEPTION_IF_NULL(node); + MS_LOG(INFO) << "Op name is " << op->GetName(); + + auto normal_shape_ptr = dyn_cast(shp); + auto no_shape_ptr = dyn_cast(shp); + + if ((nullptr != normal_shape_ptr) || (nullptr != no_shape_ptr)) { + if (UpdateSingleOutputDesc(op, shp, type) != SUCCESS) { + return; + } + } else if (nullptr != dyn_cast(shp)) { + if (UpdateMultiOutputDesc(op, shp, type) != SUCCESS) { + return; + } + } else { + MS_LOG(ERROR) << "Update output desc failed, unknow output shape type"; + return; + } + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return; + } + + // Need to update input_desc while the output_desc is updated + updateInputDesc(op, node); + } + + int setAttr(const OperatorPtr& op, const std::string& attrKey, const ValuePtr& attrValue) override { + auto it = attr_map_.find(attrKey); + if (it != attr_map_.end()) { + // switch case for each avalilable attribute type + MS_LOG(INFO) << "Set attr: " << attrKey << "(" << it->second.name << "), value: " << attrValue->ToString(); + AddAttrToDrawGraph(attrKey + std::string("=") + attrValue->ToString()); + it->second.set_attr(op, attrValue); + return 0; + } + return static_cast(NOT_FOUND); + } + + int SetCustomOpAttr(const CusOperatorPtr& op, const PrimitivePtr& prim) { + enum ValueType { + SINGLE_VALUE = 0, + SEQUEUE_VALUE, + UNKNOWN_VALUE, + }; + + MS_EXCEPTION_IF_NULL(prim); + MS_EXCEPTION_IF_NULL(op); + + ValueType value_type = SINGLE_VALUE; + for (auto item : prim->attrs()) { + if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + value_type = SEQUEUE_VALUE; + auto val_seq = item.second->cast(); + if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else { + MS_LOG(EXCEPTION) << "Unsupported custom attribute type in adaptor, prim name: " << prim->name() + << ", attr name: " << item.first << ", value: " << item.second->ToString(); + } + } else { + value_type = UNKNOWN_VALUE; + MS_LOG(WARNING) << "Unsupported custom attribute type in adaptor, prim name: " << prim->name() + << ", attr name: " << item.first << ", value: " << item.second->ToString(); + return static_cast(NOT_FOUND); + } + + if (value_type == SINGLE_VALUE) { + AddAttrToDrawGraph(item.first + std::string("=") + item.second->ToString()); + } else if (value_type == SEQUEUE_VALUE) { + AddAttrToDrawGraph(item.first + std::string("=") + "[...]"); + } + } + return 0; + } + + int SetNormalOpAttr(const OperatorPtr& op, const PrimitivePtr& prim) { + int ret = 0; + MS_EXCEPTION_IF_NULL(prim); + MS_EXCEPTION_IF_NULL(op); + for (auto& it : attr_map_) { + auto value = prim->GetAttr(it.first); + if (value != nullptr) { + // set attr from primitive + ret = setAttr(op, it.first, value); + if (ret) { + return ret; + } + } else { + // set attr from extra_attr + auto it_extra = extra_attr_.find(it.first); + if (it_extra != extra_attr_.end()) { + ret = setAttr(op, it.first, it_extra->second); + if (ret) { + return ret; + } + } + } + } + return 0; + } + + int setAttr(const OperatorPtr& op, const PrimitivePtr& prim) override { + int ret = 0; + if (IsCustomPrim(prim)) { + auto cus_op = std::dynamic_pointer_cast(op); + ret = SetCustomOpAttr(cus_op, prim); + } else { + ret = SetNormalOpAttr(op, prim); + } + return ret; + } + + int setAttr(const OperatorPtr& op, const AnfNodePtr& node) override { + // no attribute for lonely node + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + return 0; + } + + auto cnode = node->cast(); + if (cnode == nullptr) { + return 0; + } + + auto& inputs = cnode->inputs(); + if (inputs.empty()) { + return 0; + } + + // get Attr T from abstract of anfnode first, + // if attr "T" appears in primitive, the primitive T will cover this one + if (attr_map_.find("T") != attr_map_.end()) { + // get dtype from inputs[1], if the node has no inputs, set the attr T with output dtype + TypePtr type; + if (inputs.size() > 1) { + type = inputs[1]->Type(); + } else { + type = node->Type(); + } + if (type != nullptr) { + (void)setAttr(op, "T", MakeValue(type)); + } + } + + // set attr from primitive and ExtraAttr + if (IsValueNode(inputs[0])) { + // set attr from primitive + PrimitivePtr prim = GetValueNode(inputs[0]); + int ret = setAttr(op, prim); + if (ret != 0) { + return ret; + } + } + + // set attr from const input + for (auto& it : input_attr_map_) { + if (inputs.size() <= it.first || !inputs[it.first]->isa()) { + continue; + } + auto const_value = GetValueNode(inputs[it.first]); + MS_LOG(INFO) << "Set attr: input_" << it.first << "(" << it.second.name + << "), value: " << const_value->ToString(); + if (const_value->isa()) { + continue; + } + AddAttrToDrawGraph(it.second.name + std::string("=") + const_value->ToString()); + it.second.set_attr(op, const_value); + } + return 0; + } + + std::unordered_map GetExtraAttr() override { return extra_attr_; } + + private: + template + static S ConvertAny(const ValuePtr& value, const AnyTraits&) { + return GetValue(value); + } + + // specialization for reverse bool + static bool ConvertAny(const ValuePtr& value, const AnyTraits&, bool reverse) { + return reverse != GetValue(value); + } + + template + static Q ConvertAny(const ValuePtr& value, const AnyTraits

& traits_from, const AnyTraits& traits_to) { + return ConvertAnyUtil(value, traits_from, traits_to); + } + + // specialization for tensor + static GeTensor ConvertAny(const ValuePtr& value, const AnyTraits& traits) { + // To-DO the format may read from ME tensor + return ConvertAnyUtil(value, traits); + } + + // specialization for int + static int64_t ConvertAny(const ValuePtr& value, const AnyTraits) { + return static_cast(GetValue(value)); + } + + // specialization for int to Vector + static std::vector ConvertAny(const ValuePtr& value, const std::string& name, + const AnyTraits> anyTraitsInt) { + return ConvertAnyUtil(value, name, anyTraitsInt); + } + + static std::vector> ConvertAny(const ValuePtr& value, + const AnyTraits>>) { + MS_EXCEPTION_IF_NULL(value); + MS_LOG(INFO) << "Value: " << value->type_name(); + std::vector> list; + if (!value->isa()) { + MS_LOG(EXCEPTION) << "Value should be ValueTuple, but got " << value->type_name(); + } + auto vec = value->cast(); + MS_EXCEPTION_IF_NULL(vec); + for (auto& it : vec->value()) { + MS_EXCEPTION_IF_NULL(it); + if (!it->isa()) { + MS_LOG(EXCEPTION) << "It should be ValueTuple, but got " << it->type_name(); + } + auto sub_vector = it->cast(); + std::vector sublist; + for (auto& item : sub_vector->value()) { + sublist.push_back(static_cast(GetValue(item))); + } + list.push_back(sublist); + } + return list; + } + + static std::vector ConvertAny(const ValuePtr& value, const AnyTraits>, + const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); + MS_LOG(INFO) << "Value: " << value->type_name(); + std::vector list; + if (value->isa()) { + auto vec = value->cast(); + MS_EXCEPTION_IF_NULL(vec); + for (auto& it : vec->value()) { + list.push_back(static_cast(GetValue(it))); + } + return list; + } + if (value->isa()) { + list.push_back(static_cast(GetValue(value))); + return list; + } + MS_LOG(EXCEPTION) << "Value should be ValueTuple or Scalar, but got " << value->type_name(); + } + + static std::string ConvertAny(const ValuePtr& value, const AnyTraits> anyTraitsVec, + const AnyTraits anyTraitsStr) { + return ConvertAnyUtil(value, anyTraitsVec, anyTraitsStr); + } + + static std::vector ConvertAny(const ValuePtr& value, const AnyTraits> anyTraitsVec, + const AnyTraits anyTraitsFlo) { + return ConvertAnyUtil(value, anyTraitsVec, anyTraitsFlo); + } + + static std::vector ConvertAny(const ValuePtr& value, const std::string& format, + const AnyTraits> anyTraitsVec, + const AnyTraits anyTraitsInt) { + return ConvertAnyUtil(value, format, anyTraitsVec, anyTraitsInt); + } + + // convert value list for value tuple to vector + template + static std::vector ConvertAny(const ValuePtr& value, const AnyTraits

& anyTraitsP, + const AnyTraits> anyTraitsQ) { + return ConvertAnyUtil(value, anyTraitsP, anyTraitsQ); + } + + static int64_t ConvertAny(const ValuePtr& value, const AnyTraits) { + auto name = GetValue(value); + auto it = enum_map_.find(name); + int v = 0; + if (it != enum_map_.end()) { + v = it->second; + } + return v; + } + + static GeDataType ConvertAny(const ValuePtr& value, const AnyTraits anyTraitsGE) { + return ConvertAnyUtil(value, anyTraitsGE); + } + + // convert any value to tensor + static GeTensor ConvertAny(const ValuePtr& value, const AnyTraits anyTraitsValue) { + return ConvertAnyUtil(value, anyTraitsValue); + } + + static const std::unordered_map input_map_; + static const std::unordered_map dyn_input_map_; + static const std::unordered_map output_map_; + static const std::unordered_map dyn_output_map_; + static const std::unordered_map attr_map_; + static const std::unordered_map enum_map_; + // convert input from anf graph to Attr in Operators + static const std::unordered_map input_attr_map_; + static std::unordered_map> cus_input_map_; + static std::unordered_map> cus_output_map_; + std::unordered_map extra_attr_; + std::unordered_map name_counts_; +}; + +template +const std::unordered_map OpAdapter::input_map_; +template +const std::unordered_map OpAdapter::dyn_input_map_; +template +const std::unordered_map OpAdapter::output_map_; +template +const std::unordered_map OpAdapter::dyn_output_map_; +template +const std::unordered_map OpAdapter::attr_map_; +template +const std::unordered_map OpAdapter::enum_map_; +template +const std::unordered_map OpAdapter::input_attr_map_; +template +std::unordered_map> OpAdapter::cus_input_map_; +template +std::unordered_map> OpAdapter::cus_output_map_; + +// specialization for method +} // namespace transform +} // namespace mindspore + +#endif // TRANSFORM_OP_ADAPTER_H_ diff --git a/mindspore/ccsrc/transform/op_adapter_base.h b/mindspore/ccsrc/transform/op_adapter_base.h new file mode 100644 index 00000000000..99106b87611 --- /dev/null +++ b/mindspore/ccsrc/transform/op_adapter_base.h @@ -0,0 +1,187 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_OP_ADAPTER_BASE_H_ +#define TRANSFORM_OP_ADAPTER_BASE_H_ + +#include +#include +#include +#include +#include +#include + +#include "transform/util.h" +#include "ir/anf.h" +#include "ir/primitive.h" +#include "ir/value.h" +#include "transform/types.h" + +#ifdef ENABLE_GE +#ifdef OPEN_SOURCE +#include "graph/types.h" +#endif +#endif + +#include "graph/operator_reg.h" +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif +#include "graph/tensor.h" +#include "transform/all_ops.h" + +namespace ge { +class CustomOperator : public Operator { + public: + CustomOperator(const string& name, const string& type) : Operator(name, type) {} + + ~CustomOperator() override{}; + + void CustomInputRegister(const string& name) { Operator::InputRegister(name); } + + void CustomOutputRegister(const string& name) { Operator::OutputRegister(name); } + + void CustomInferFuncRegister(const std::function& func) { Operator::InferFuncRegister(func); } +}; +} // namespace ge + +namespace mindspore { +namespace transform { +using CusOperatorPtr = std::shared_ptr; +using CustomOperator = ge::CustomOperator; + +struct OutHandler { + OperatorPtr op; + std::string out; + OutHandler() : op(nullptr), out("") {} + OutHandler(const OperatorPtr& op, const std::string out) : op(op), out(out) {} +}; + +struct ControlEdge { + OperatorPtr src_op; + OperatorPtr dest_op; +}; + +using AttrFunc = std::function; +using OutputFunc = std::function; +using InputOpFunc = std::function; +using InputHandleFunc = std::function; +using CreateDynInputOpFunc = std::function; +using DynInputOpFunc = std::function; +using DynInputHandleFunc = std::function; +using UpdateOutputDescFunc = std::function; +using CreateDynOutputOpFunc = std::function; + +struct AttrDesc { + std::string name; + AttrFunc set_attr; +}; + +struct InputDesc { + std::string name; + InputOpFunc set_op; + InputHandleFunc set_handle; + UpdateOutputDescFunc update_input_desc; +}; + +struct DynInputDesc { + std::string name; + CreateDynInputOpFunc create_dyn_input; + DynInputOpFunc set_op; + DynInputHandleFunc set_handle; +}; + +struct OutputDesc { + std::string name; + UpdateOutputDescFunc update_out_desc; +}; + +struct DynOutputDesc { + std::string name; + CreateDynOutputOpFunc create_dyn_output; +}; + +class BaseOpAdapter { + public: + virtual ~BaseOpAdapter() {} + virtual OperatorPtr generate(const AnfNodePtr& anf) = 0; + virtual OperatorPtr generate(const std::string& type) { return std::make_shared(type); } + virtual int setInput(const OperatorPtr& op, int index, const OperatorPtr& input) = 0; + virtual int setInput(const OperatorPtr& op, int index, const OutHandler& handle) = 0; + virtual int setInput(const OperatorPtr& op, int index, + const std::shared_ptr>& handler_vec) = 0; + virtual int setAttr(const OperatorPtr& op, const std::string& attrKey, const ValuePtr& attrValue) = 0; + virtual int setAttr(const OperatorPtr& op, const PrimitivePtr& prim) = 0; + virtual int setAttr(const OperatorPtr& op, const AnfNodePtr& node) = 0; + virtual std::unordered_map GetExtraAttr() = 0; + template ::value>::type> + int setAttr(const OperatorPtr& op, const std::string& attrKey, const std::shared_ptr& attrValue) { + return setAttr(op, attrKey, MakeValue(attrValue)); + } + template ::value>::type> + int setAttr(const OperatorPtr& op, const std::string& attrKey, const T& attrValue) { + return setAttr(op, attrKey, MakeValue(attrValue)); + } + virtual OutHandler getOutput(const OperatorPtr& op, int index) = 0; + virtual void updateOutputDesc(const OperatorPtr& op, const abstract::BaseShapePtr& shp, const TypePtr& type, + const AnfNodePtr& node) = 0; + virtual const std::unordered_map& getInputMap() = 0; + virtual const std::unordered_map& getInputAttrMap() = 0; + virtual const std::unordered_map& getDynInputMap() = 0; + virtual const std::unordered_map& getOutputMap() = 0; + void AddAttrToDrawGraph(const std::string& attr_str) { attrs_vec_.push_back(attr_str); } + const std::vector& GetAttrsFromDrawGraph() const { return attrs_vec_; } + void clearAttrVect() { attrs_vec_.clear(); } + + private: + std::vector attrs_vec_; +}; + +using OpAdapterPtr = std::shared_ptr; + +enum AttrType { + ATTR_INT = 0, + ATTR_FLOAT, + ATTR_DOUBLE, + ATTR_STRING, + ATTR_TENSOR, + ATTR_BOOL, + ATTR_LIST_INT, + ATTR_LIST_ANY_INT, + ATTR_ENUM +}; + +struct GeEnum {}; +struct TFType {}; +struct GEType {}; + +// declare Any type +template +struct AnyTraits { + using type = T; +}; + +template <> +struct AnyTraits { + using type = int64_t; +}; + +using ExtraAttr = std::unordered_map; +} // namespace transform +} // namespace mindspore +#endif // TRANSFORM_OP_ADAPTER_BASE_H_ diff --git a/mindspore/ccsrc/transform/op_adapter_util.cc b/mindspore/ccsrc/transform/op_adapter_util.cc new file mode 100755 index 00000000000..49b87148370 --- /dev/null +++ b/mindspore/ccsrc/transform/op_adapter_util.cc @@ -0,0 +1,252 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/op_adapter_util.h" + +#include +#include +#include + +#include "utils/utils.h" +#include "transform/op_adapter_base.h" + +namespace mindspore { +namespace transform { +GeTensor ConvertAnyUtil(const ValuePtr& value, const AnyTraits&) { + // To-DO the format may read from ME tensor + MS_EXCEPTION_IF_NULL(value); + auto me_tensor = value->cast(); + auto ge_tensor = TransformUtil::ConvertTensor(me_tensor, kOpFormat_NCHW); + return ge_tensor == nullptr ? GeTensor() : *ge_tensor; +} + +std::vector ConvertAnyUtil(const ValuePtr& value, const std::string& name, + const AnyTraits>) { + int64_t data = GetValue(value); + std::vector list; + int size = 2; // 2 int in list + if (name == "pad") { + size = 4; // 4 int in list + list = TransformUtil::ConvertIntToList(data, size); + list[0] = 1; + list[1] = 1; + } else { + list = TransformUtil::ConvertIntToList(data, size); + } + + return list; +} + +std::string ConvertAnyUtil(const ValuePtr& value, const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); + if (nullptr == vec) { + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::ostringstream buffer; + int i = 0; + for (auto& it : vec->value()) { + if (i != 0) { + buffer << ","; + } + buffer << GetValue(it); + i++; + } + return buffer.str(); +} + +std::vector ConvertAnyUtil(const ValuePtr& value, const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); + if (nullptr == vec) { + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::vector list; + list.resize(vec->value().size()); + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin(), + [](const ValuePtr& val) { return static_cast(GetValue(val)); }); + return list; +} + +std::vector ConvertAnyUtil(const ValuePtr& value, const std::string& format, + const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); + if (nullptr == vec) { + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::vector list; + list.resize(vec->value().size()); + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin(), + [](const ValuePtr& val) { return static_cast(GetValue(val)); }); + if (format == kOpFormat_NHWC) { + if (list.size() < 4) { + MS_LOG(EXCEPTION) << "The size of list is less than 4"; + } else { + int64_t temp = list[1]; + list[1] = list[2]; + list[2] = list[3]; + list[3] = temp; + } + } + return list; +} + +GeDataType ConvertAnyUtil(const ValuePtr& value, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + if (!value->isa()) { + MS_LOG(EXCEPTION) << "error convert Value to TypePtr for value: " << value->ToString() + << ", type: " << value->type_name() << ", value should be a Typeptr"; + } + auto type = value->cast(); + MS_EXCEPTION_IF_NULL(type); + TypeId me_type = type->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(type)->element()->type_id(); + } + return TransformUtil::ConvertDataType(me_type); +} + +GeTensor VectorToTensorUtil(const ValuePtr& value) { + // convert tuple or list to ge tensor, only supported one dim for now + MS_EXCEPTION_IF_NULL(value); + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + if (vec.empty()) { + MS_LOG(WARNING) << "Convert a none tuple to an empty ge tensor"; + return GeTensor(); + } + MS_EXCEPTION_IF_NULL(vec[0]); + if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Int32"; + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeInt32, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(int32_t)); + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Float32"; + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeFloat32, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(float)); + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Bool"; + // We use uint8_t to save bool type data + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeBool, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, static_cast(data.data()), data.size() * sizeof(uint8_t)); + } else { + MS_LOG(EXCEPTION) << "Unsupported data type of tuple or list elements: " << vec[0]->type_name(); + } + + return GeTensor(); +} + +GeTensor ConvertAnyUtil(const ValuePtr& value, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + if (value->isa()) { + // convert me tensor to ge tensor + return ConvertAnyUtil(value, AnyTraits()); + } else if (value->isa() || value->isa()) { + return VectorToTensorUtil(value); + } else if (value->isa()) { + // convert scalar Int to GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Int32"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT32); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(int32_t)); + } else if (value->isa()) { + // convert scalar Int64 to GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Int64"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(int64_t)); + } else if (value->isa()) { + // convert scalar FP32 to GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = FP32"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(float)); + } else if (value->isa()) { + // convert scalar FP32 to GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Bool"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_BOOL); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(bool)); + } else if (value->isa()) { + // convert String to GeTensor + MS_LOG(INFO) << "convert string to tensor with data type = String"; + std::string v = GetValue(value); + std::vector ge_shape; + GeShape shape(ge_shape); + GeTensorDesc desc(shape, ge::FORMAT_NCHW, ge::DT_STRING); + GeTensor str_tensor(desc); + str_tensor.SetData(v); + return str_tensor; + } else { + MS_LOG(WARNING) << "Unsupported value type: " << value->type_name() + << " to convert to tensor. Value: " << value->ToString(); + } + return GeTensor(); +} + +bool IsCustomPrim(const PrimitivePtr& prim) { + if (prim == nullptr) { + return false; + } + + ValuePtr flag = prim->GetAttr("_custom_op_flag"); + if (flag == nullptr) { + return false; + } + + return GetValue(flag); +} + +bool IsCustomCNode(const AnfNodePtr& anf) { + if (anf == nullptr) { + return false; + } + auto node = anf->cast(); + if (node == nullptr) { + return false; + } + if (node->inputs().empty()) { + MS_LOG(EXCEPTION) << "length of node inputs is empty"; + } + MS_EXCEPTION_IF_NULL(node->inputs()[0]); + if (!node->inputs()[0]->isa()) { + return false; + } + auto cus_prim = GetValueNode(node->inputs()[0]); + if (cus_prim == nullptr) { + return false; + } + + return IsCustomPrim(cus_prim); +} +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/op_adapter_util.h b/mindspore/ccsrc/transform/op_adapter_util.h new file mode 100644 index 00000000000..0cb6c763b2f --- /dev/null +++ b/mindspore/ccsrc/transform/op_adapter_util.h @@ -0,0 +1,66 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_OP_ADAPTER_UTIL_H_ +#define TRANSFORM_OP_ADAPTER_UTIL_H_ + +#include +#include + +#include "transform/op_adapter_base.h" + +namespace mindspore { +namespace transform { +template +static Q ConvertAnyUtil(const ValuePtr& value, const AnyTraits

&, const AnyTraits&) { + return static_cast(GetValue

(value)); +} + +GeTensor ConvertAnyUtil(const ValuePtr& value, const AnyTraits& traits); + +std::vector ConvertAnyUtil(const ValuePtr& value, const std::string& name, + const AnyTraits>); + +std::string ConvertAnyUtil(const ValuePtr& value, const AnyTraits>, const AnyTraits); + +std::vector ConvertAnyUtil(const ValuePtr& value, const AnyTraits>, const AnyTraits); + +std::vector ConvertAnyUtil(const ValuePtr& value, const std::string& format, + const AnyTraits>, const AnyTraits); + +GeDataType ConvertAnyUtil(const ValuePtr& value, const AnyTraits); + +template +std::vector ConvertAnyUtil(const ValuePtr& value, AnyTraits

, const AnyTraits>) { + if (!value->isa() && !value->isa()) { + MS_LOG(EXCEPTION) << "error convert Value to vector for value: " << value->ToString() + << ", type: " << value->type_name() << ", value should be a tuple or list"; + } + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + std::vector data; + for (auto& it : vec) { + data.push_back(ConvertAnyUtil(it, AnyTraits

(), AnyTraits())); + } + return data; +} + +GeTensor ConvertAnyUtil(const ValuePtr& value, const AnyTraits); + +bool IsCustomPrim(const PrimitivePtr& prim); +bool IsCustomCNode(const AnfNodePtr& node); +} // namespace transform +} // namespace mindspore +#endif // TRANSFORM_OP_ADAPTER_UTIL_H_ diff --git a/mindspore/ccsrc/transform/op_declare.cc b/mindspore/ccsrc/transform/op_declare.cc new file mode 100755 index 00000000000..78b949c5255 --- /dev/null +++ b/mindspore/ccsrc/transform/op_declare.cc @@ -0,0 +1,1183 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/op_declare.h" + +#include + +#include "transform/all_ops.h" +#include "utils/utils.h" + +namespace mindspore { +namespace transform { +#define INPUT_MAP(T) \ + template <> \ + const std::unordered_map OpAdapter::input_map_ +#define EMPTY_INPUT_MAP std::unordered_map() +#define INPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, const OperatorPtr input) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_input_##name(*input); \ + }, \ + [](const OperatorPtr op, const OutHandler& handle) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_input_##name(*(handle.op), handle.out); \ + }, \ + [](const OperatorPtr op, const GeTensorDesc desc) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->update_input_desc_##name(desc); \ + } \ + } + +#define DYN_INPUT_MAP(T) \ + template <> \ + const std::unordered_map OpAdapter::dyn_input_map_ +#define DYN_INPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_input_##name(num); \ + }, \ + [](const OperatorPtr op, unsigned int index, const OperatorPtr input) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_input_##name(index, *input); \ + }, \ + [](const OperatorPtr op, unsigned int index, const OutHandler& handle) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_input_##name(index, *(handle.op), handle.out); \ + } \ + } + +#define ATTR_MAP(T) \ + template <> \ + const std::unordered_map OpAdapter::attr_map_ +#define EMPTY_ATTR_MAP std::unordered_map() +#define ATTR_DESC(name, ...) \ + { \ +#name, \ + [](const OperatorPtr op, const ValuePtr& value) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_attr_##name(ConvertAny(value, __VA_ARGS__)); \ + } \ + } + +#define INPUT_ATTR_MAP(T) \ + template <> \ + const std::unordered_map OpAdapter::input_attr_map_ + +#define OUTPUT_MAP(T) \ + template <> \ + const std::unordered_map OpAdapter::output_map_ +#define OUTPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, const GeTensorDesc desc) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->update_output_desc_##name(desc); \ + } \ + } + +#define DYN_OUTPUT_MAP(T) \ + template <> \ + const std::unordered_map OpAdapter::dyn_output_map_ + +#define DYN_OUTPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_output_##name(num); \ + } \ + } + +template <> +std::unordered_map> OpAdapter::cus_input_map_{}; +template <> +std::unordered_map> OpAdapter::cus_output_map_{}; + +// --------------specialization for each operator---------- +// const +INPUT_MAP(Const) = EMPTY_INPUT_MAP; +ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}}; +OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}}; + +// Assign +INPUT_MAP(Assign) = {{1, INPUT_DESC(ref)}, {2, INPUT_DESC(value)}}; +ATTR_MAP(Assign) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Assign) = {{0, OUTPUT_DESC(ref)}}; + +// Constant +INPUT_MAP(Constant) = EMPTY_INPUT_MAP; +ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; +OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; + +// ApplyMomentum +INPUT_MAP(ApplyMomentum) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}, {5, INPUT_DESC(momentum)}}; +ATTR_MAP(ApplyMomentum) = {{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ApplyMomentum) = {{0, OUTPUT_DESC(var)}}; + +// ScalarSummary +INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; +ATTR_MAP(Summary) = EMPTY_ATTR_MAP; + +// data +INPUT_MAP(Data) = EMPTY_INPUT_MAP; +ATTR_MAP(Data) = EMPTY_ATTR_MAP; + +// resnet ops in ge +// BatchNorm +INPUT_MAP(BatchNorm) = {{1, INPUT_DESC(x)}, + {2, INPUT_DESC(scale)}, + {3, INPUT_DESC(offset)}, + {4, INPUT_DESC(mean)}, + {5, INPUT_DESC(variance)}}; +ATTR_MAP(BatchNorm) = {{"data_format", ATTR_DESC(data_format, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +OUTPUT_MAP(BatchNorm) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(batch_mean)}, + {2, OUTPUT_DESC(batch_variance)}, + {3, OUTPUT_DESC(reserve_space_1)}, + {4, OUTPUT_DESC(reserve_space_2)}, + {5, OUTPUT_DESC(reserve_space_3)}}; + +// BatchNormGrad +INPUT_MAP(BatchNormGrad) = {{1, INPUT_DESC(y_backprop)}, {2, INPUT_DESC(x)}, + {3, INPUT_DESC(scale)}, {4, INPUT_DESC(reserve_space_1)}, + {5, INPUT_DESC(reserve_space_2)}, {6, INPUT_DESC(reserve_space_3)}}; +ATTR_MAP(BatchNormGrad) = {{"data_format", ATTR_DESC(data_format, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +OUTPUT_MAP(BatchNormGrad) = {{0, OUTPUT_DESC(x_backprop)}, + {1, OUTPUT_DESC(scale_backprop)}, + {2, OUTPUT_DESC(offset_backprop)}, + {3, OUTPUT_DESC(reserve_space_4)}, + {4, OUTPUT_DESC(reserve_space_5)}}; + +// Relu +INPUT_MAP(Relu) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Relu) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Relu) = {{0, OUTPUT_DESC(y)}}; + +// Elu +INPUT_MAP(Elu) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Elu) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}}; +OUTPUT_MAP(Elu) = {{0, OUTPUT_DESC(y)}}; + +// EluGrad +INPUT_MAP(EluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(activations)}}; +ATTR_MAP(EluGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(EluGrad) = {{0, OUTPUT_DESC(y)}}; + +// PRelu +INPUT_MAP(PRelu) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(weight)}}; +ATTR_MAP(PRelu) = EMPTY_ATTR_MAP; +OUTPUT_MAP(PRelu) = {{0, OUTPUT_DESC(y)}}; + +// PReluGrad +INPUT_MAP(PReluGrad) = { + {1, INPUT_DESC(input_gradients)}, {2, INPUT_DESC(input_features)}, {3, INPUT_DESC(input_weights)}}; +ATTR_MAP(PReluGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(PReluGrad) = {{0, OUTPUT_DESC(output_backprops_dx)}, {1, OUTPUT_DESC(output_backprops_da)}}; + +// Sigmoid +INPUT_MAP(Sigmoid) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sigmoid) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sigmoid) = {{0, OUTPUT_DESC(y)}}; + +// SigmoidGrad +INPUT_MAP(SigmoidGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(SigmoidGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SigmoidGrad) = {{0, OUTPUT_DESC(z)}}; + +// L2NormalizeGrad +INPUT_MAP(L2NormalizeGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(dy)}}; +ATTR_MAP(L2NormalizeGrad) = { + {"axis", ATTR_DESC(dim, AnyTraits>(), AnyTraits>())}, + {"epsilon", ATTR_DESC(eps, AnyTraits())}}; +OUTPUT_MAP(L2NormalizeGrad) = {{0, OUTPUT_DESC(dx)}}; + +// LarsV2Update +INPUT_MAP(LarsV2Update) = {{1, INPUT_DESC(w)}, + {2, INPUT_DESC(g)}, + {3, INPUT_DESC(w_square_sum)}, + {4, INPUT_DESC(g_square_sum)}, + {5, INPUT_DESC(weight_decay)}, + {6, INPUT_DESC(learning_rate)}}; +ATTR_MAP(LarsV2Update) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"hyperpara", ATTR_DESC(hyperpara, AnyTraits())}, + {"use_clip", ATTR_DESC(use_clip, AnyTraits())}}; +OUTPUT_MAP(LarsV2Update) = {{0, OUTPUT_DESC(g_new)}}; + +// L2Normalize +INPUT_MAP(L2Normalize) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(L2Normalize) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}, + {"epsilon", ATTR_DESC(eps, AnyTraits())}}; +OUTPUT_MAP(L2Normalize) = {{0, OUTPUT_DESC(y)}}; + +// CumsumD +INPUT_MAP(CumsumD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(CumsumD) = {{2, ATTR_DESC(axis, AnyTraits())}}; +ATTR_MAP(CumsumD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits())}, + {"reverse", ATTR_DESC(reverse, AnyTraits())}}; +OUTPUT_MAP(CumsumD) = {{0, OUTPUT_DESC(y)}}; + +// softmax +INPUT_MAP(Softmax) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Softmax) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}, +}; +OUTPUT_MAP(Softmax) = {{0, OUTPUT_DESC(y)}}; + +// SoftmaxGrad +INPUT_MAP(SoftmaxGrad) = {{1, INPUT_DESC(softmax)}, {2, INPUT_DESC(grad_softmax)}}; +OUTPUT_MAP(SoftmaxGrad) = {{0, OUTPUT_DESC(grad_x)}}; +ATTR_MAP(SoftmaxGrad) = EMPTY_ATTR_MAP; + +// Flatten +INPUT_MAP(Flatten) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Flatten) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Flatten) = {{0, OUTPUT_DESC(y)}}; + +// add +INPUT_MAP(Add) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Add) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Add) = {{0, OUTPUT_DESC(y)}}; + +// GatherV2 +INPUT_MAP(GatherV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(axis)}}; +ATTR_MAP(GatherV2) = EMPTY_ATTR_MAP; +OUTPUT_MAP(GatherV2) = {{0, OUTPUT_DESC(y)}}; + +// ReduceSum +INPUT_MAP(ReduceSum) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ReduceSum) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceSum) = {{0, OUTPUT_DESC(y)}}; + +// ReduceSumD +INPUT_MAP(ReduceSumD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(ReduceSumD) = { + {2, ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(ReduceSumD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceSumD) = {{0, OUTPUT_DESC(y)}}; + +// ReduceProdD +INPUT_MAP(ReduceProdD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(ReduceProdD) = { + {2, ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(ReduceProdD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceProdD) = {{0, OUTPUT_DESC(y)}}; + +// CumprodD +INPUT_MAP(CumprodD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(CumprodD) = {{2, ATTR_DESC(axis, AnyTraits())}}; +ATTR_MAP(CumprodD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits())}, + {"reverse", ATTR_DESC(reverse, AnyTraits())}}; +OUTPUT_MAP(CumprodD) = {{0, OUTPUT_DESC(y)}}; + +// SoftmaxCrossEntropyWithLogits/ +INPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(labels)}}; +ATTR_MAP(SoftmaxCrossEntropyWithLogits) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(backprop)}}; + +// MeanGrad +INPUT_MAP(MeanGrad) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(MeanGrad) = {{2, ATTR_DESC(mean_grad_output_shape_value, kOpFormat_NHWC, + AnyTraits>(), AnyTraits())}}; +ATTR_MAP(MeanGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; + +INPUT_MAP(SliceD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(SliceD) = {{2, ATTR_DESC(begin, AnyTraits(), AnyTraits>())}, + {3, ATTR_DESC(size, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(SliceD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SliceD) = {{0, OUTPUT_DESC(y)}}; + +// MaxPool +INPUT_MAP(MaxPool) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(MaxPool) = {{"ksize", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding", ATTR_DESC(padding, AnyTraits())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(MaxPool) = {{0, OUTPUT_DESC(y)}}; + +// AvgPool +INPUT_MAP(AvgPool) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(AvgPool) = {{"ksize", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding", ATTR_DESC(padding, AnyTraits())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(AvgPool) = {{0, OUTPUT_DESC(y)}}; + +// GreaterEqual +INPUT_MAP(GreaterEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(GreaterEqual) = EMPTY_ATTR_MAP; +OUTPUT_MAP(GreaterEqual) = {{0, OUTPUT_DESC(y)}}; + +// AssignAdd +INPUT_MAP(AssignAdd) = {{1, INPUT_DESC(ref)}, {2, INPUT_DESC(value)}}; +ATTR_MAP(AssignAdd) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AssignAdd) = {{0, OUTPUT_DESC(ref)}}; + +// AssignSub +INPUT_MAP(AssignSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(value)}}; +ATTR_MAP(AssignSub) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AssignSub) = {{0, OUTPUT_DESC(var)}}; + +// Cos +INPUT_MAP(Cos) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Cos) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Cos) = {{0, OUTPUT_DESC(y)}}; + +// Acos +INPUT_MAP(Acos) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Acos) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Acos) = {{0, OUTPUT_DESC(y)}}; + +// AcosGrad +INPUT_MAP(AcosGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AcosGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AcosGrad) = {{0, OUTPUT_DESC(z)}}; + +// Floor +INPUT_MAP(Floor) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Floor) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Floor) = {{0, OUTPUT_DESC(y)}}; + +// FloorDiv +INPUT_MAP(FloorDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(FloorDiv) = EMPTY_ATTR_MAP; +OUTPUT_MAP(FloorDiv) = {{0, OUTPUT_DESC(y)}}; + +// Sin +INPUT_MAP(Sin) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sin) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sin) = {{0, OUTPUT_DESC(y)}}; + +// Exp +INPUT_MAP(Exp) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Exp) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Exp) = {{0, OUTPUT_DESC(y)}}; + +// BoundingBoxEncode +INPUT_MAP(BoundingBoxEncode) = { + {1, INPUT_DESC(anchor_box)}, + {2, INPUT_DESC(ground_truth_box)}, +}; +ATTR_MAP(BoundingBoxEncode) = { + {"means", ATTR_DESC(means, AnyTraits>(), AnyTraits())}, + {"stds", ATTR_DESC(stds, AnyTraits>(), AnyTraits())}, +}; +OUTPUT_MAP(BoundingBoxEncode) = {{0, OUTPUT_DESC(delats)}}; + +// BoundingBoxDecode +INPUT_MAP(BoundingBoxDecode) = { + {1, INPUT_DESC(rois)}, + {2, INPUT_DESC(deltas)}, +}; +ATTR_MAP(BoundingBoxDecode) = { + {"means", ATTR_DESC(means, AnyTraits>(), AnyTraits())}, + {"stds", ATTR_DESC(stds, AnyTraits>(), AnyTraits())}, + {"max_shape", ATTR_DESC(max_shape, AnyTraits>(), AnyTraits>())}, + {"wh_ratio_clip", ATTR_DESC(wh_ratio_clip, AnyTraits())}, +}; +OUTPUT_MAP(BoundingBoxDecode) = {{0, OUTPUT_DESC(bboxes)}}; + +#ifdef VALID_CODE + +// Less +INPUT_MAP(Less) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}}; +ATTR_MAP(Less) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Less) = {{0, OUTPUT_DESC(z)}}; + +// Cast +INPUT_MAP(Cast) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(Cast) = {{2, ATTR_DESC(dst_type, AnyTraits())}}; +ATTR_MAP(Cast) = {{"Truncate", ATTR_DESC(truncate, AnyTraits())}}; +OUTPUT_MAP(Cast) = {{0, OUTPUT_DESC(y)}}; + +// Minimum +INPUT_MAP(Minimum) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}}; +ATTR_MAP(Minimum) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}, {"beta", ATTR_DESC(beta, AnyTraits())}}; +OUTPUT_MAP(Minimum) = {{0, OUTPUT_DESC(z)}}; + +// Sub +INPUT_MAP(Sub) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Sub) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}, {"beta", ATTR_DESC(beta, AnyTraits())}}; + +#endif + +// TopKV2 +INPUT_MAP(TopKV2) = { + {1, INPUT_DESC(input)}, + {2, INPUT_DESC(k)}, +}; + +ATTR_MAP(TopKV2) = {{"T", ATTR_DESC(T, AnyTraits())}, {"sorted", ATTR_DESC(sorted, AnyTraits())}}; + +OUTPUT_MAP(TopKV2) = { + {0, OUTPUT_DESC(values)}, + {1, OUTPUT_DESC(indices)}, +}; + +// Multiply +INPUT_MAP(Multiply) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}}; +ATTR_MAP(Multiply) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Multiply) = {{0, OUTPUT_DESC(z)}}; + +// TileD +INPUT_MAP(TileD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TileD) = {{2, ATTR_DESC(multiples, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TileD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TileD) = {{0, OUTPUT_DESC(y)}}; + +// OneHot +INPUT_MAP(OneHot) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(depth)}, {3, INPUT_DESC(on_value)}, {4, INPUT_DESC(off_value)}}; +ATTR_MAP(OneHot) = {{"axis", ATTR_DESC(axis, AnyTraits())}}; +OUTPUT_MAP(OneHot) = {{0, OUTPUT_DESC(y)}}; + +// GatherV2D +INPUT_MAP(GatherV2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}}; +INPUT_ATTR_MAP(GatherV2D) = {{3, ATTR_DESC(axis, AnyTraits())}}; +ATTR_MAP(GatherV2D) = EMPTY_ATTR_MAP; +OUTPUT_MAP(GatherV2D) = {{0, OUTPUT_DESC(y)}}; + +// Reshape +INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; +ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; + +// BiasAdd +INPUT_MAP(BiasAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(bias)}}; +ATTR_MAP(BiasAdd) = {{"data_format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(BiasAdd) = {{0, OUTPUT_DESC(y)}}; + +// Iou +INPUT_MAP(Iou) = {{1, INPUT_DESC(bboxes)}, {2, INPUT_DESC(gtboxes)}}; +ATTR_MAP(Iou) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(Iou) = {{0, OUTPUT_DESC(overlap)}}; + +// ResizeNearestNeighborD +INPUT_MAP(ResizeNearestNeighborD) = {{1, INPUT_DESC(images)}}; +ATTR_MAP(ResizeNearestNeighborD) = { + {"size", ATTR_DESC(size, AnyTraits>(), AnyTraits>())}, + {"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeNearestNeighborD) = {{0, OUTPUT_DESC(y)}}; + +// ResizeNearestNeighborGrad +INPUT_MAP(ResizeNearestNeighborGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(size)}}; +ATTR_MAP(ResizeNearestNeighborGrad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeNearestNeighborGrad) = {{0, OUTPUT_DESC(y)}}; + +// ApplyAdam +INPUT_MAP(ApplyAdam) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)}, + {10, INPUT_DESC(grad)}}; +ATTR_MAP(ApplyAdam) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}}; +#ifdef ENABLE_GE +OUTPUT_MAP(ApplyAdam) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}}; +#else +OUTPUT_MAP(ApplyAdam) = {{0, OUTPUT_DESC(var)}}; +#endif + +// Relu6 +INPUT_MAP(Relu6) = {{1, INPUT_DESC(features)}}; +ATTR_MAP(Relu6) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Relu6) = {{0, OUTPUT_DESC(activations)}}; + +// Relu6Grad +INPUT_MAP(Relu6Grad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(y)}}; +ATTR_MAP(Relu6Grad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Relu6Grad) = {{0, OUTPUT_DESC(z)}}; + +// ResizeBilinearGrad +INPUT_MAP(ResizeBilinearGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(original_image)}}; +ATTR_MAP(ResizeBilinearGrad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeBilinearGrad) = {{0, OUTPUT_DESC(y)}}; + +// ResizeBilinear +INPUT_MAP(ResizeBilinearD) = {{1, INPUT_DESC(images)}}; +ATTR_MAP(ResizeBilinearD) = { + {"size", ATTR_DESC(size, AnyTraits>(), AnyTraits>())}, + {"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeBilinearD) = {{0, OUTPUT_DESC(y)}}; + +// ZerosLike +INPUT_MAP(ZerosLike) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ZerosLike) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ZerosLike) = {{0, OUTPUT_DESC(y)}}; + +// OnesLike +INPUT_MAP(OnesLike) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(OnesLike) = EMPTY_ATTR_MAP; +OUTPUT_MAP(OnesLike) = {{0, OUTPUT_DESC(y)}}; + +// NMSWithMask +INPUT_MAP(NMSWithMask) = {{1, INPUT_DESC(box_scores)}}; +ATTR_MAP(NMSWithMask) = {{"iou_threshold", ATTR_DESC(iou_threshold, AnyTraits())}}; +OUTPUT_MAP(NMSWithMask) = { + {0, OUTPUT_DESC(selected_boxes)}, {1, OUTPUT_DESC(selected_idx)}, {2, OUTPUT_DESC(selected_mask)}}; + +// Unpack +INPUT_MAP(Unpack) = {{1, INPUT_DESC(value)}}; +ATTR_MAP(Unpack) = {{"axis", ATTR_DESC(axis, AnyTraits())}, {"num", ATTR_DESC(num, AnyTraits())}}; +DYN_OUTPUT_MAP(Unpack) = {{0, DYN_OUTPUT_DESC(output)}}; + +// ScatterNdUpdate +INPUT_MAP(ScatterNdUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdUpdate) = {{0, OUTPUT_DESC(var)}}; + +// CheckValid +INPUT_MAP(CheckValid) = {{1, INPUT_DESC(bbox_tensor)}, {2, INPUT_DESC(img_metas)}}; +ATTR_MAP(CheckValid) = EMPTY_ATTR_MAP; +OUTPUT_MAP(CheckValid) = {{0, OUTPUT_DESC(valid_tensor)}}; + +// SmoothL1Loss +INPUT_MAP(SmoothL1Loss) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}}; +ATTR_MAP(SmoothL1Loss) = {{"sigma", ATTR_DESC(sigma, AnyTraits())}}; +OUTPUT_MAP(SmoothL1Loss) = {{0, OUTPUT_DESC(loss)}}; + +// SmoothL1LossGrad +INPUT_MAP(SmoothL1LossGrad) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}, {3, INPUT_DESC(dout)}}; +ATTR_MAP(SmoothL1LossGrad) = {{"sigma", ATTR_DESC(sigma, AnyTraits())}}; +OUTPUT_MAP(SmoothL1LossGrad) = {{0, OUTPUT_DESC(gradient)}}; + +// SigmoidCrossEntropyWithLogits +INPUT_MAP(SigmoidCrossEntropyWithLogits) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}}; +ATTR_MAP(SigmoidCrossEntropyWithLogits) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SigmoidCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}}; + +// SigmoidCrossEntropyWithLogitsGrad +INPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = { + {1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(dout)}}; +ATTR_MAP(SigmoidCrossEntropyWithLogitsGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = {{0, OUTPUT_DESC(gradient)}}; + +// ScatterNd +INPUT_MAP(ScatterNdD) = {{1, INPUT_DESC(indices)}, {2, INPUT_DESC(updates)}}; +INPUT_ATTR_MAP(ScatterNdD) = { + {3, ATTR_DESC(shape, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(ScatterNdD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ScatterNdD) = {{0, OUTPUT_DESC(y)}}; + +// PadD +INPUT_MAP(PadD) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(PadD) = {{"paddings", ATTR_DESC(paddings, AnyTraits>>())}}; +OUTPUT_MAP(PadD) = {{0, OUTPUT_DESC(y)}}; + +// GatherNd +INPUT_MAP(GatherNd) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(GatherNd) = EMPTY_ATTR_MAP; +OUTPUT_MAP(GatherNd) = {{0, OUTPUT_DESC(y)}}; + +// ROIAlign +INPUT_MAP(ROIAlign) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(rois)}}; +OUTPUT_MAP(ROIAlign) = {{0, OUTPUT_DESC(output)}}; +ATTR_MAP(ROIAlign) = {{"pooled_height", ATTR_DESC(pooled_height, AnyTraits())}, + {"pooled_width", ATTR_DESC(pooled_width, AnyTraits())}, + {"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits())}, + {"sample_num", ATTR_DESC(sample_num, AnyTraits())}}; + +// ROIAlignGrad +INPUT_MAP(ROIAlignGrad) = {{1, INPUT_DESC(ydiff)}, {2, INPUT_DESC(rois)}}; +OUTPUT_MAP(ROIAlignGrad) = {{0, OUTPUT_DESC(xdiff)}}; +ATTR_MAP(ROIAlignGrad) = { + {"xdiff_shape", ATTR_DESC(xdiff_shape, AnyTraits>(), AnyTraits>())}, + {"pooled_height", ATTR_DESC(pooled_height, AnyTraits())}, + {"pooled_width", ATTR_DESC(pooled_width, AnyTraits())}, + {"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits())}, + {"sample_num", ATTR_DESC(sample_num, AnyTraits())}}; + +// ArgMaxD +INPUT_MAP(ArgMaxD) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMaxD) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"output_type", ATTR_DESC(output_type, AnyTraits())}}; +OUTPUT_MAP(ArgMaxD) = {{0, OUTPUT_DESC(y)}}; + +// ArgMinD +INPUT_MAP(ArgMinD) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMinD) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"output_type", ATTR_DESC(output_type, AnyTraits())}}; +OUTPUT_MAP(ArgMinD) = {{0, OUTPUT_DESC(y)}}; + +// ArgMaxWithValue +INPUT_MAP(ArgMaxWithValue) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMaxWithValue) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ArgMaxWithValue) = {{0, OUTPUT_DESC(indice)}, {1, OUTPUT_DESC(values)}}; + +// ArgMinWithValue +INPUT_MAP(ArgMinWithValue) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMinWithValue) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ArgMinWithValue) = {{0, OUTPUT_DESC(indice)}, {1, OUTPUT_DESC(values)}}; + +// ReduceAll +INPUT_MAP(ReduceAll) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ReduceAll) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceAll) = {{0, OUTPUT_DESC(y)}}; + +// ReduceMeanD +INPUT_MAP(ReduceMeanD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(ReduceMeanD) = { + {2, ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(ReduceMeanD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceMeanD) = {{0, OUTPUT_DESC(y)}}; + +// HCOMAllreduce +INPUT_MAP(HcomAllReduce) = {{1, INPUT_DESC(x)}}; +OUTPUT_MAP(HcomAllReduce) = {{0, OUTPUT_DESC(y)}}; +ATTR_MAP(HcomAllReduce) = {{"op", ATTR_DESC(reduction, AnyTraits())}, + {"group", ATTR_DESC(group, AnyTraits())}, + {"fusion", ATTR_DESC(fusion, AnyTraits())}}; + +// HCOMBraodcast +INPUT_MAP(HcomBroadcast) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(HcomBroadcast) = {{1, DYN_INPUT_DESC(x)}}; +DYN_OUTPUT_MAP(HcomBroadcast) = {{0, DYN_OUTPUT_DESC(y)}}; +ATTR_MAP(HcomBroadcast) = {{"root_rank", ATTR_DESC(root_rank, AnyTraits())}, + {"group", ATTR_DESC(group, AnyTraits())}}; + +// HCOMAllreduce +INPUT_MAP(HcomAllGather) = {{1, INPUT_DESC(x)}}; +OUTPUT_MAP(HcomAllGather) = {{0, OUTPUT_DESC(y)}}; +ATTR_MAP(HcomAllGather) = {{"group", ATTR_DESC(group, AnyTraits())}, + {"rank_size", ATTR_DESC(rank_size, AnyTraits())}}; + +// HCOMReduceScatter +INPUT_MAP(HcomReduceScatter) = {{1, INPUT_DESC(x)}}; +OUTPUT_MAP(HcomReduceScatter) = {{0, OUTPUT_DESC(y)}}; +ATTR_MAP(HcomReduceScatter) = {{"group", ATTR_DESC(group, AnyTraits())}, + {"op", ATTR_DESC(reduction, AnyTraits())}, + {"rank_size", ATTR_DESC(rank_size, AnyTraits())}}; + +// Variable +INPUT_MAP(Variable) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Variable) = EMPTY_ATTR_MAP; + +// ReluGrad +INPUT_MAP(ReluGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +ATTR_MAP(ReluGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ReluGrad) = {{0, OUTPUT_DESC(backprops)}}; + +// FusedBatchNorm +INPUT_MAP(FusedBatchNorm) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(scale)}, {3, INPUT_DESC(b)}, {4, INPUT_DESC(mean)}, {5, INPUT_DESC(variance)}}; +ATTR_MAP(FusedBatchNorm) = {{"mode", ATTR_DESC(mode, AnyTraits())}, + {"momentum", ATTR_DESC(moving_average_fraction, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}}; +OUTPUT_MAP(FusedBatchNorm) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(running_mean)}, + {2, OUTPUT_DESC(running_variance)}, + {3, OUTPUT_DESC(save_mean)}, + {4, OUTPUT_DESC(save_inv_variance)}}; + +// FusedBatchNromGrad +INPUT_MAP(FusedBatchNormGrad) = {{1, INPUT_DESC(dy)}, + {2, INPUT_DESC(x)}, + {3, INPUT_DESC(scale)}, + {4, INPUT_DESC(save_mean)}, + {5, INPUT_DESC(save_inv_variance)}}; +ATTR_MAP(FusedBatchNormGrad) = {{"momentum", ATTR_DESC(momentum, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}}; +OUTPUT_MAP(FusedBatchNormGrad) = {{0, OUTPUT_DESC(dx)}, {1, OUTPUT_DESC(bn_scale)}, {2, OUTPUT_DESC(bn_bias)}}; + +// BiasAddGrad +INPUT_MAP(BiasAddGrad) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(BiasAddGrad) = {{"data_format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(BiasAddGrad) = {{0, OUTPUT_DESC(y)}}; + +// maxpoolgrad +INPUT_MAP(MaxPoolGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}}; +ATTR_MAP(MaxPoolGrad) = {{"ksize", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding", ATTR_DESC(padding, AnyTraits())}}; +OUTPUT_MAP(MaxPoolGrad) = {{0, OUTPUT_DESC(y)}}; + +// avgpoolgrad +INPUT_MAP(AvgPoolGrad) = {{1, INPUT_DESC(orig_input_shape)}, {2, INPUT_DESC(input_grad)}}; +ATTR_MAP(AvgPoolGrad) = {{"ksize", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding", ATTR_DESC(padding, AnyTraits())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(AvgPoolGrad) = {{0, OUTPUT_DESC(out_grad)}}; + +// MaxPoolWithArgmax +INPUT_MAP(MaxPoolGradWithArgmax) = { + {1, INPUT_DESC(x)}, + {2, INPUT_DESC(argmax)}, + {3, INPUT_DESC(grad)}, +}; +ATTR_MAP(MaxPoolGradWithArgmax) = {{"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"window", ATTR_DESC(ksize, "window", AnyTraits>())}, + {"stride", ATTR_DESC(strides, "stride", AnyTraits>())}}; + +// Conv2D +INPUT_MAP(Conv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}}; +ATTR_MAP(Conv2D) = { + {"stride", ATTR_DESC(strides, "pad", AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, "pad", AnyTraits>())}, +}; +OUTPUT_MAP(Conv2D) = {{0, OUTPUT_DESC(y)}}; + +// Conv2DBackpropInputD +INPUT_MAP(Conv2DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filters)}}; +INPUT_ATTR_MAP(Conv2DBackpropInputD) = { + {3, ATTR_DESC(input_sizes, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv2DBackpropInputD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, "strides", AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, "pad", AnyTraits>())}, +}; +OUTPUT_MAP(Conv2DBackpropInputD) = {{0, OUTPUT_DESC(y)}}; + +// Conv2DBackpropFilterD +INPUT_MAP(Conv2DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(Conv2DBackpropFilterD) = { + {3, ATTR_DESC(filter_sizes, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv2DBackpropFilterD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, "strides", AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, "pad", AnyTraits>())}, +}; +OUTPUT_MAP(Conv2DBackpropFilterD) = {{0, OUTPUT_DESC(y)}}; + +// DepthwiseConv2D +INPUT_MAP(DepthwiseConv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}}; +ATTR_MAP(DepthwiseConv2D) = { + {"stride", ATTR_DESC(strides, "pad", AnyTraits>())}, + {"pads", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, "pad", AnyTraits>())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}, +}; +OUTPUT_MAP(DepthwiseConv2D) = {{0, OUTPUT_DESC(y)}}; + +// DepthwiseConv2DBackpropInputD +INPUT_MAP(DepthwiseConv2DBackpropInputD) = {{2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}}; +INPUT_ATTR_MAP(DepthwiseConv2DBackpropInputD) = { + {1, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(DepthwiseConv2DBackpropInputD) = { + {"stride", ATTR_DESC(strides, "pad", AnyTraits>())}, + {"pads", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, "pad", AnyTraits>())}, +}; +OUTPUT_MAP(DepthwiseConv2DBackpropInputD) = {{0, OUTPUT_DESC(input_grad)}}; + +// DepthwiseConv2DBackpropFilterD +INPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{1, INPUT_DESC(input)}, {3, INPUT_DESC(out_backprop)}}; +INPUT_ATTR_MAP(DepthwiseConv2DBackpropFilterD) = { + {2, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(DepthwiseConv2DBackpropFilterD) = { + {"stride", ATTR_DESC(strides, "pad", AnyTraits>())}, + {"pads", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, "pad", AnyTraits>())}, +}; +OUTPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{0, OUTPUT_DESC(filter_grad)}}; + +// MatMul +INPUT_MAP(MatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(MatMul) = {{"transpose_a", ATTR_DESC(transpose_a, AnyTraits())}, + {"transpose_b", ATTR_DESC(transpose_b, AnyTraits())}}; +OUTPUT_MAP(MatMul) = {{0, OUTPUT_DESC(y)}}; + +// Merge +INPUT_MAP(Merge) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(Merge) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(Merge) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Merge) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(value_index)}}; + +// Switch +INPUT_MAP(Switch) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(pred)}}; +OUTPUT_MAP(Switch) = {{0, OUTPUT_DESC(output_false)}, {1, OUTPUT_DESC(output_true)}}; +ATTR_MAP(Switch) = EMPTY_ATTR_MAP; + +// AddN +INPUT_MAP(AddN) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(AddN) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(AddN) = {{"n", ATTR_DESC(N, AnyTraits())}}; +OUTPUT_MAP(AddN) = {{0, OUTPUT_DESC(y)}}; + +// Mul +INPUT_MAP(Mul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Mul) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Mul) = {{0, OUTPUT_DESC(y)}}; + +// RealDiv +INPUT_MAP(RealDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(RealDiv) = EMPTY_ATTR_MAP; +OUTPUT_MAP(RealDiv) = {{0, OUTPUT_DESC(y)}}; + +// Cast +INPUT_MAP(Cast) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(Cast) = {{2, ATTR_DESC(dst_type, AnyTraits())}}; +ATTR_MAP(Cast) = {{"Truncate", ATTR_DESC(truncate, AnyTraits())}}; +OUTPUT_MAP(Cast) = {{0, OUTPUT_DESC(y)}}; + +// Reciprocal +INPUT_MAP(Reciprocal) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Reciprocal) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reciprocal) = {{0, OUTPUT_DESC(y)}}; + +// Sub +INPUT_MAP(Sub) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Sub) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sub) = {{0, OUTPUT_DESC(y)}}; + +// SplitD +INPUT_MAP(SplitD) = {{1, INPUT_DESC(value)}}; +ATTR_MAP(SplitD) = {{"axis", ATTR_DESC(split_dim, AnyTraits())}, + {"output_num", ATTR_DESC(num_split, AnyTraits())}}; +DYN_OUTPUT_MAP(SplitD) = {{0, DYN_OUTPUT_DESC(output)}}; + +// Neg +INPUT_MAP(Neg) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Neg) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Neg) = {{0, OUTPUT_DESC(y)}}; + +// Transpose +INPUT_MAP(TransposeD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransposeD) = {{2, ATTR_DESC(perm, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransposeD) = EMPTY_ATTR_MAP; +// Do not set Transpose operator output descriptor + +// DropOutGenMask +INPUT_MAP(DropOutGenMask) = {{1, INPUT_DESC(shape)}, {2, INPUT_DESC(prob)}}; +ATTR_MAP(DropOutGenMask) = {{"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}}; +OUTPUT_MAP(DropOutGenMask) = {{0, OUTPUT_DESC(y)}}; + +// Pack +INPUT_MAP(Pack) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(Pack) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(Pack) = {{"num", ATTR_DESC(N, AnyTraits())}, {"axis", ATTR_DESC(axis, AnyTraits())}}; +OUTPUT_MAP(Pack) = {{0, OUTPUT_DESC(y)}}; + +// ConcatD +INPUT_MAP(ConcatD) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(ConcatD) = {{1, DYN_INPUT_DESC(input_values)}}; +ATTR_MAP(ConcatD) = { + {"axis", ATTR_DESC(concat_dim, AnyTraits())}, + {"inputNums", ATTR_DESC(N, AnyTraits())}, +}; +OUTPUT_MAP(ConcatD) = {{0, OUTPUT_DESC(output_data)}}; + +// Less +INPUT_MAP(Less) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Less) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Less) = {{0, OUTPUT_DESC(y)}}; + +// Rsqrt +INPUT_MAP(Rsqrt) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Rsqrt) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Rsqrt) = {{0, OUTPUT_DESC(y)}}; + +// Sqrt +INPUT_MAP(Sqrt) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sqrt) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sqrt) = {{0, OUTPUT_DESC(y)}}; + +// Square +INPUT_MAP(Square) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Square) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Square) = {{0, OUTPUT_DESC(y)}}; + +// Tanh +INPUT_MAP(Tanh) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Tanh) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Tanh) = {{0, OUTPUT_DESC(y)}}; + +// TanhGrad +INPUT_MAP(TanhGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(TanhGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TanhGrad) = {{0, OUTPUT_DESC(z)}}; + +// ReduceMinD +INPUT_MAP(ReduceMinD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(ReduceMinD) = { + {2, ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(ReduceMinD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceMinD) = {{0, OUTPUT_DESC(y)}}; + +// ReduceMaxD +INPUT_MAP(ReduceMaxD) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(ReduceMaxD) = { + {2, ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(ReduceMaxD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ReduceMaxD) = {{0, OUTPUT_DESC(y)}}; + +// Maximum +INPUT_MAP(Maximum) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Maximum) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Maximum) = {{0, OUTPUT_DESC(y)}}; + +// Minimum +INPUT_MAP(Minimum) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Minimum) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Minimum) = {{0, OUTPUT_DESC(y)}}; + +// MaximumGrad +INPUT_MAP(MaximumGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grads)}}; +ATTR_MAP(MaximumGrad) = {{"grad_x", ATTR_DESC(grad_x, AnyTraits())}, + {"grad_y", ATTR_DESC(grad_y, AnyTraits())}}; +OUTPUT_MAP(MaximumGrad) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; + +// MinimumGrad +INPUT_MAP(MinimumGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grads)}}; +ATTR_MAP(MinimumGrad) = {{"grad_x", ATTR_DESC(grad_x, AnyTraits())}, + {"grad_y", ATTR_DESC(grad_y, AnyTraits())}}; +OUTPUT_MAP(MinimumGrad) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; + +// Pow +INPUT_MAP(Pow) = { + {1, INPUT_DESC(x1)}, + {2, INPUT_DESC(x2)}, +}; +ATTR_MAP(Pow) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Pow) = {{0, OUTPUT_DESC(y)}}; + +// Equal +INPUT_MAP(Equal) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Equal) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Equal) = {{0, OUTPUT_DESC(y)}}; + +// NotEqual +INPUT_MAP(NotEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(NotEqual) = EMPTY_ATTR_MAP; +OUTPUT_MAP(NotEqual) = {{0, OUTPUT_DESC(y)}}; + +// Log +INPUT_MAP(Log) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Log) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Log) = {{0, OUTPUT_DESC(y)}}; + +// LogicalAnd +INPUT_MAP(LogicalAnd) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(LogicalAnd) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LogicalAnd) = {{0, OUTPUT_DESC(y)}}; + +// LogicalOr +INPUT_MAP(LogicalOr) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(LogicalOr) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LogicalOr) = {{0, OUTPUT_DESC(y)}}; + +// LogicalNot +INPUT_MAP(LogicalNot) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(LogicalNot) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LogicalNot) = {{0, OUTPUT_DESC(y)}}; + +// Greater +INPUT_MAP(Greater) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Greater) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Greater) = {{0, OUTPUT_DESC(y)}}; + +// LogSoftmaxGrad +INPUT_MAP(LogSoftmaxGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}}; +ATTR_MAP(LogSoftmaxGrad) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +OUTPUT_MAP(LogSoftmaxGrad) = {{0, OUTPUT_DESC(y)}}; + +// Select +INPUT_MAP(Select) = {{1, INPUT_DESC(condition)}, {2, INPUT_DESC(x1)}, {3, INPUT_DESC(x2)}}; +ATTR_MAP(Select) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Select) = {{0, OUTPUT_DESC(y)}}; + +// LessEqual +INPUT_MAP(LessEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(LessEqual) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LessEqual) = {{0, OUTPUT_DESC(y)}}; + +// LogSoftmax +INPUT_MAP(LogSoftmax) = {{1, INPUT_DESC(logits)}}; +ATTR_MAP(LogSoftmax) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +OUTPUT_MAP(LogSoftmax) = {{0, OUTPUT_DESC(logsoftmax)}}; + +// RandomChoiceWithMask +INPUT_MAP(RandomChoiceWithMask) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(RandomChoiceWithMask) = {{"count", ATTR_DESC(count, AnyTraits())}, + {"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}}; +OUTPUT_MAP(RandomChoiceWithMask) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}}; + +// TruncatedNormal +INPUT_MAP(TruncatedNormal) = {{1, INPUT_DESC(shape)}}; +ATTR_MAP(TruncatedNormal) = {{"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}}; +OUTPUT_MAP(TruncatedNormal) = {{0, OUTPUT_DESC(y)}}; + +// StridedSliceGrad +INPUT_MAP(StridedSliceGrad) = { + {1, INPUT_DESC(dy)}, {2, INPUT_DESC(shape)}, {3, INPUT_DESC(begin)}, {4, INPUT_DESC(end)}, {5, INPUT_DESC(strides)}}; +ATTR_MAP(StridedSliceGrad) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}}; +OUTPUT_MAP(StridedSliceGrad) = {{0, OUTPUT_DESC(output)}}; + +// Gelu +INPUT_MAP(Gelu) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Gelu) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Gelu) = {{0, OUTPUT_DESC(y)}}; + +// GeluGrad +INPUT_MAP(GeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}}; +ATTR_MAP(GeluGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(GeluGrad) = {{0, OUTPUT_DESC(z)}}; + +// StridedSlice +INPUT_MAP(StridedSlice) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(strides)}}; +ATTR_MAP(StridedSlice) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}}; +OUTPUT_MAP(StridedSlice) = {{0, OUTPUT_DESC(y)}}; + +// UnsortedSegmentSum +INPUT_MAP(UnsortedSegmentSumD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +INPUT_ATTR_MAP(UnsortedSegmentSumD) = {{3, ATTR_DESC(num_segments, AnyTraits())}}; +ATTR_MAP(UnsortedSegmentSumD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(UnsortedSegmentSumD) = {{0, OUTPUT_DESC(y)}}; + +// ExpandDims +INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; + +// Squeeze +INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; + +// SGD +INPUT_MAP(SGD) = {{1, INPUT_DESC(parameters)}, {2, INPUT_DESC(gradient)}, {3, INPUT_DESC(learning_rate)}, + {4, INPUT_DESC(accum)}, {5, INPUT_DESC(momentum)}, {6, INPUT_DESC(stat)}}; +ATTR_MAP(SGD) = {{"dampening", ATTR_DESC(dampening, AnyTraits())}, + {"weight_decay", ATTR_DESC(weight_decay, AnyTraits())}, + {"nesterov", ATTR_DESC(nesterov, AnyTraits())}}; +OUTPUT_MAP(SGD) = {{0, OUTPUT_DESC(parameters)}}; + +// LayerNorm +INPUT_MAP(LayerNorm) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(gamma)}, {3, INPUT_DESC(beta)}}; +ATTR_MAP(LayerNorm) = {{"begin_norm_axis", ATTR_DESC(begin_norm_axis, AnyTraits())}, + {"begin_params_axis", ATTR_DESC(begin_params_axis, AnyTraits())}}; +OUTPUT_MAP(LayerNorm) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mean)}, {2, OUTPUT_DESC(variance)}}; + +// LayerNormGrad +INPUT_MAP(LayerNormGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(dy)}, {3, INPUT_DESC(variance)}, {4, INPUT_DESC(mean)}, {5, INPUT_DESC(gamma)}}; +ATTR_MAP(LayerNormGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LayerNormGrad) = {{0, OUTPUT_DESC(pd_x)}, {1, OUTPUT_DESC(pd_gamma)}, {2, OUTPUT_DESC(pd_beta)}}; + +// BatchMatMul +INPUT_MAP(BatchMatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BatchMatMul) = {{"transpose_x1", ATTR_DESC(adj_x, AnyTraits())}, + {"transpose_x2", ATTR_DESC(adj_y, AnyTraits())}}; +OUTPUT_MAP(BatchMatMul) = {{0, OUTPUT_DESC(y)}}; + +// DropoutDoMask +INPUT_MAP(DropOutDoMask) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mask)}, {3, INPUT_DESC(keep_prob)}}; +ATTR_MAP(DropOutDoMask) = EMPTY_ATTR_MAP; +OUTPUT_MAP(DropOutDoMask) = {{0, OUTPUT_DESC(y)}}; + +// NPUGetFloatStatus +INPUT_MAP(NPUGetFloatStatus) = {{1, INPUT_DESC(addr)}}; +OUTPUT_MAP(NPUGetFloatStatus) = {{0, OUTPUT_DESC(data)}}; +ATTR_MAP(NPUGetFloatStatus) = EMPTY_ATTR_MAP; + +// NPUAllocFloatStatus +INPUT_MAP(NPUAllocFloatStatus) = EMPTY_INPUT_MAP; +ATTR_MAP(NPUAllocFloatStatus) = EMPTY_ATTR_MAP; +OUTPUT_MAP(NPUAllocFloatStatus) = {{0, OUTPUT_DESC(data)}}; + +// NPUClearFloatStatus +INPUT_MAP(NPUClearFloatStatus) = {{1, INPUT_DESC(addr)}}; +OUTPUT_MAP(NPUClearFloatStatus) = {{0, OUTPUT_DESC(data)}}; +ATTR_MAP(NPUClearFloatStatus) = EMPTY_ATTR_MAP; + +// Abs +INPUT_MAP(Abs) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Abs) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Abs) = {{0, OUTPUT_DESC(y)}}; + +// AbsGrad +INPUT_MAP(AbsGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AbsGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AbsGrad) = {{0, OUTPUT_DESC(z)}}; + +// BinaryCrossEntropy +INPUT_MAP(BinaryCrossEntropy) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(weight)}}; +ATTR_MAP(BinaryCrossEntropy) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(BinaryCrossEntropy) = {{0, OUTPUT_DESC(output)}}; + +// BinaryCrossEntropyGrad +INPUT_MAP(BinaryCrossEntropyGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(grad_output)}, {4, INPUT_DESC(weight)}}; +ATTR_MAP(BinaryCrossEntropyGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(BinaryCrossEntropyGrad) = {{0, OUTPUT_DESC(output)}}; + +// SparseApplyAdagradD +INPUT_MAP(SparseApplyAdagradD) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}}; +ATTR_MAP(SparseApplyAdagradD) = {{"lr", ATTR_DESC(lr, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(SparseApplyAdagradD) = {{0, OUTPUT_DESC(var)}}; + +// SpaceToDepth +INPUT_MAP(SpaceToDepth) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(SpaceToDepth) = {{"block_size", ATTR_DESC(block_size, AnyTraits())}}; +OUTPUT_MAP(SpaceToDepth) = {{0, OUTPUT_DESC(y)}}; + +// DepthToSpace +INPUT_MAP(DepthToSpace) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(DepthToSpace) = {{"block_size", ATTR_DESC(block_size, AnyTraits())}}; +OUTPUT_MAP(DepthToSpace) = {{0, OUTPUT_DESC(y)}}; + +// Sign +INPUT_MAP(Sign) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sign) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sign) = {{0, OUTPUT_DESC(y)}}; + +// Round +INPUT_MAP(Round) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Round) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Round) = {{0, OUTPUT_DESC(y)}}; + +// ApplyFtrl +INPUT_MAP(ApplyFtrl) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(l1)}, + {7, INPUT_DESC(l2)}, {8, INPUT_DESC(lr_power)}}; +ATTR_MAP(ApplyFtrl) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ApplyFtrl) = {{0, OUTPUT_DESC(var)}}; + +#ifdef ENABLE_GE +// Print +INPUT_MAP(Print) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(Print) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(Print) = EMPTY_ATTR_MAP; +#endif +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/op_declare.h b/mindspore/ccsrc/transform/op_declare.h new file mode 100755 index 00000000000..03463b978fe --- /dev/null +++ b/mindspore/ccsrc/transform/op_declare.h @@ -0,0 +1,444 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_OP_DECLARE_H_ +#define TRANSFORM_OP_DECLARE_H_ + +#include +#include +#include "transform/op_adapter.h" + +namespace mindspore { +namespace transform { +#define DECLARE_OP_ADAPTER(T) \ + using T = ge::op::T; \ + template <> \ + const std::unordered_map OpAdapter::input_map_; \ + template <> \ + const std::unordered_map OpAdapter::attr_map_; + +#define DECLARE_OP_USE_OUTPUT(T) \ + template <> \ + const std::unordered_map OpAdapter::output_map_; + +#define DECLARE_OP_USE_ENUM(T) \ + template <> \ + const std::unordered_map OpAdapter::enum_map_; + +#define DECLARE_OP_USE_INPUT_ATTR(T) \ + template <> \ + const std::unordered_map OpAdapter::input_attr_map_; + +#define DECLARE_OP_USE_DYN_INPUT(T) \ + template <> \ + const std::unordered_map OpAdapter::dyn_input_map_; + +#define DECLARE_OP_USE_DYN_OUTPUT(T) \ + template <> \ + const std::unordered_map OpAdapter::dyn_output_map_; + +template <> +std::unordered_map> OpAdapter::cus_input_map_; +template <> +std::unordered_map> OpAdapter::cus_output_map_; + +DECLARE_OP_ADAPTER(GreaterEqual) +DECLARE_OP_USE_OUTPUT(GreaterEqual) +DECLARE_OP_ADAPTER(SliceD) +DECLARE_OP_USE_INPUT_ATTR(SliceD) +DECLARE_OP_USE_OUTPUT(SliceD) +DECLARE_OP_ADAPTER(AssignAdd) +DECLARE_OP_USE_OUTPUT(AssignAdd) +DECLARE_OP_ADAPTER(AssignSub) +DECLARE_OP_USE_OUTPUT(AssignSub) + +DECLARE_OP_ADAPTER(ReduceMean) +DECLARE_OP_ADAPTER(Multiply) +DECLARE_OP_USE_OUTPUT(Multiply) + +// ** Distributed Operations ** +DECLARE_OP_ADAPTER(HcomReduceScatter) +DECLARE_OP_USE_OUTPUT(HcomReduceScatter) +DECLARE_OP_ADAPTER(HcomBroadcast) +DECLARE_OP_USE_DYN_INPUT(HcomBroadcast) +DECLARE_OP_USE_DYN_OUTPUT(HcomBroadcast) +DECLARE_OP_ADAPTER(HcomAllReduce) +DECLARE_OP_USE_OUTPUT(HcomAllReduce) +DECLARE_OP_ADAPTER(HcomAllGather) +DECLARE_OP_USE_OUTPUT(HcomAllGather) +DECLARE_OP_ADAPTER(Variable) +DECLARE_OP_ADAPTER(ReluGrad) +DECLARE_OP_USE_OUTPUT(ReluGrad) +DECLARE_OP_ADAPTER(FusedBatchNorm) +DECLARE_OP_USE_OUTPUT(FusedBatchNorm) +DECLARE_OP_ADAPTER(FusedBatchNormGrad) +DECLARE_OP_USE_OUTPUT(FusedBatchNormGrad) +DECLARE_OP_ADAPTER(BiasAddGrad) +DECLARE_OP_USE_OUTPUT(BiasAddGrad) +DECLARE_OP_ADAPTER(MaxPoolGradWithArgmax) +DECLARE_OP_USE_ENUM(MaxPoolGradWithArgmax) +DECLARE_OP_ADAPTER(Conv2D) +DECLARE_OP_USE_ENUM(Conv2D) +DECLARE_OP_USE_OUTPUT(Conv2D) +DECLARE_OP_ADAPTER(Conv2DBackpropInputD) +DECLARE_OP_USE_ENUM(Conv2DBackpropInputD) +DECLARE_OP_USE_INPUT_ATTR(Conv2DBackpropInputD) +DECLARE_OP_USE_OUTPUT(Conv2DBackpropInputD) +DECLARE_OP_ADAPTER(Conv2DBackpropFilterD) +DECLARE_OP_USE_ENUM(Conv2DBackpropFilterD) +DECLARE_OP_USE_INPUT_ATTR(Conv2DBackpropFilterD) +DECLARE_OP_USE_OUTPUT(Conv2DBackpropFilterD) +DECLARE_OP_ADAPTER(DepthwiseConv2D) +DECLARE_OP_USE_ENUM(DepthwiseConv2D) +DECLARE_OP_USE_OUTPUT(DepthwiseConv2D) +DECLARE_OP_ADAPTER(DepthwiseConv2DBackpropFilterD) +DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropFilterD) +DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropFilterD) +DECLARE_OP_ADAPTER(DepthwiseConv2DBackpropInputD) +DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropInputD) +DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropInputD) +DECLARE_OP_ADAPTER(Reshape) +DECLARE_OP_USE_OUTPUT(Reshape) +DECLARE_OP_ADAPTER(Iou) +DECLARE_OP_USE_OUTPUT(Iou) +DECLARE_OP_ADAPTER(ResizeNearestNeighborD) +DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborD) +DECLARE_OP_ADAPTER(ResizeNearestNeighborGrad) +DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborGrad) +DECLARE_OP_ADAPTER(ApplyAdam) +DECLARE_OP_USE_OUTPUT(ApplyAdam) +DECLARE_OP_ADAPTER(Relu6) +DECLARE_OP_USE_OUTPUT(Relu6) +DECLARE_OP_ADAPTER(Relu6Grad) +DECLARE_OP_USE_OUTPUT(Relu6Grad) +DECLARE_OP_ADAPTER(ResizeBilinearD) +DECLARE_OP_USE_OUTPUT(ResizeBilinearD) +DECLARE_OP_ADAPTER(ResizeBilinearGrad) +DECLARE_OP_USE_OUTPUT(ResizeBilinearGrad) +DECLARE_OP_ADAPTER(ZerosLike) +DECLARE_OP_USE_OUTPUT(ZerosLike) +DECLARE_OP_ADAPTER(OnesLike) +DECLARE_OP_USE_OUTPUT(OnesLike) +DECLARE_OP_ADAPTER(ScatterNdUpdate) +DECLARE_OP_USE_OUTPUT(ScatterNdUpdate) +DECLARE_OP_ADAPTER(NMSWithMask) +DECLARE_OP_USE_OUTPUT(NMSWithMask) +DECLARE_OP_ADAPTER(Unpack) +DECLARE_OP_USE_DYN_OUTPUT(Unpack) +DECLARE_OP_ADAPTER(CheckValid) +DECLARE_OP_USE_OUTPUT(CheckValid) +DECLARE_OP_ADAPTER(SmoothL1Loss) +DECLARE_OP_USE_OUTPUT(SmoothL1Loss) +DECLARE_OP_ADAPTER(SmoothL1LossGrad) +DECLARE_OP_USE_OUTPUT(SmoothL1LossGrad) +DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogits) +DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogits) +DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogitsGrad) +DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogitsGrad) +DECLARE_OP_ADAPTER(ScatterNdD) +DECLARE_OP_USE_INPUT_ATTR(ScatterNdD) +DECLARE_OP_USE_OUTPUT(ScatterNdD) +DECLARE_OP_ADAPTER(PadD) +DECLARE_OP_USE_OUTPUT(PadD) +DECLARE_OP_ADAPTER(BoundingBoxEncode) +DECLARE_OP_USE_OUTPUT(BoundingBoxEncode) +DECLARE_OP_ADAPTER(BoundingBoxDecode) +DECLARE_OP_USE_OUTPUT(BoundingBoxDecode) +DECLARE_OP_ADAPTER(GatherNd) +DECLARE_OP_USE_OUTPUT(GatherNd) +DECLARE_OP_ADAPTER(ArgMaxD) +DECLARE_OP_USE_OUTPUT(ArgMaxD) +DECLARE_OP_ADAPTER(ArgMinD) +DECLARE_OP_USE_OUTPUT(ArgMinD) +DECLARE_OP_ADAPTER(ArgMaxWithValue) +DECLARE_OP_USE_OUTPUT(ArgMaxWithValue) +DECLARE_OP_ADAPTER(ArgMinWithValue) +DECLARE_OP_USE_OUTPUT(ArgMinWithValue) +DECLARE_OP_ADAPTER(Mul) +DECLARE_OP_USE_OUTPUT(Mul) +DECLARE_OP_ADAPTER(AddN) +DECLARE_OP_USE_DYN_INPUT(AddN) +DECLARE_OP_USE_OUTPUT(AddN) +DECLARE_OP_ADAPTER(Less) +DECLARE_OP_USE_OUTPUT(Less) +DECLARE_OP_ADAPTER(Rsqrt) +DECLARE_OP_USE_OUTPUT(Rsqrt) +DECLARE_OP_ADAPTER(Sqrt) +DECLARE_OP_USE_OUTPUT(Sqrt) +DECLARE_OP_ADAPTER(Square) +DECLARE_OP_USE_OUTPUT(Square) +DECLARE_OP_ADAPTER(SplitD) +DECLARE_OP_USE_DYN_OUTPUT(SplitD) +DECLARE_OP_ADAPTER(SGD) +DECLARE_OP_USE_OUTPUT(SGD) + +DECLARE_OP_ADAPTER(Tanh) +DECLARE_OP_USE_OUTPUT(Tanh) +DECLARE_OP_ADAPTER(TanhGrad) +DECLARE_OP_USE_OUTPUT(TanhGrad) +DECLARE_OP_ADAPTER(Maximum) +DECLARE_OP_USE_OUTPUT(Maximum) +DECLARE_OP_ADAPTER(Minimum) +DECLARE_OP_USE_OUTPUT(Minimum) +DECLARE_OP_ADAPTER(MaximumGrad) +DECLARE_OP_USE_OUTPUT(MaximumGrad) +DECLARE_OP_ADAPTER(MinimumGrad) +DECLARE_OP_USE_OUTPUT(MinimumGrad) +DECLARE_OP_ADAPTER(ReduceMinD) +DECLARE_OP_USE_INPUT_ATTR(ReduceMinD) +DECLARE_OP_USE_OUTPUT(ReduceMinD) +DECLARE_OP_ADAPTER(ReduceMaxD) +DECLARE_OP_USE_INPUT_ATTR(ReduceMaxD) +DECLARE_OP_USE_OUTPUT(ReduceMaxD) +DECLARE_OP_ADAPTER(Merge) +DECLARE_OP_USE_DYN_INPUT(Merge) +DECLARE_OP_USE_OUTPUT(Merge) +DECLARE_OP_ADAPTER(Switch) +DECLARE_OP_USE_OUTPUT(Switch) + +DECLARE_OP_ADAPTER(TopKV2) +DECLARE_OP_USE_OUTPUT(TopKV2) + +DECLARE_OP_ADAPTER(RealDiv) +DECLARE_OP_USE_OUTPUT(RealDiv) + +DECLARE_OP_ADAPTER(Cast) +DECLARE_OP_USE_INPUT_ATTR(Cast) +DECLARE_OP_USE_OUTPUT(Cast) +DECLARE_OP_ADAPTER(Reciprocal) +DECLARE_OP_USE_OUTPUT(Reciprocal) +DECLARE_OP_ADAPTER(Neg) +DECLARE_OP_USE_OUTPUT(Neg) +DECLARE_OP_ADAPTER(TransposeD) +DECLARE_OP_USE_INPUT_ATTR(TransposeD) +// Do not set Transpose operator output descriptor +DECLARE_OP_ADAPTER(Sub) +DECLARE_OP_USE_OUTPUT(Sub) +DECLARE_OP_ADAPTER(DropOutGenMask) +DECLARE_OP_USE_OUTPUT(DropOutGenMask) +DECLARE_OP_ADAPTER(ConcatD) +DECLARE_OP_USE_DYN_INPUT(ConcatD) +DECLARE_OP_USE_OUTPUT(ConcatD) +DECLARE_OP_ADAPTER(Pack) +DECLARE_OP_USE_DYN_INPUT(Pack) +DECLARE_OP_USE_OUTPUT(Pack) + +DECLARE_OP_ADAPTER(Pow) +DECLARE_OP_USE_OUTPUT(Pow) +DECLARE_OP_ADAPTER(Equal) +DECLARE_OP_USE_OUTPUT(Equal) +DECLARE_OP_ADAPTER(NotEqual) +DECLARE_OP_USE_OUTPUT(NotEqual) +DECLARE_OP_ADAPTER(Log) +DECLARE_OP_USE_OUTPUT(Log) +DECLARE_OP_ADAPTER(LogicalAnd) +DECLARE_OP_USE_OUTPUT(LogicalAnd) +DECLARE_OP_ADAPTER(LogicalOr) +DECLARE_OP_USE_OUTPUT(LogicalOr) +DECLARE_OP_ADAPTER(LogicalNot) +DECLARE_OP_USE_OUTPUT(LogicalNot) +DECLARE_OP_ADAPTER(LogSoftmaxGrad) +DECLARE_OP_USE_OUTPUT(LogSoftmaxGrad) + +DECLARE_OP_ADAPTER(RandomChoiceWithMask) +DECLARE_OP_USE_OUTPUT(RandomChoiceWithMask) + +DECLARE_OP_ADAPTER(Select) +DECLARE_OP_USE_OUTPUT(Select) +DECLARE_OP_ADAPTER(LessEqual) +DECLARE_OP_USE_OUTPUT(LessEqual) +DECLARE_OP_ADAPTER(LogSoftmax) +DECLARE_OP_USE_OUTPUT(LogSoftmax) +DECLARE_OP_ADAPTER(TruncatedNormal) +DECLARE_OP_USE_OUTPUT(TruncatedNormal) +DECLARE_OP_ADAPTER(StridedSliceGrad) +DECLARE_OP_USE_OUTPUT(StridedSliceGrad) +DECLARE_OP_ADAPTER(Gelu) +DECLARE_OP_USE_OUTPUT(Gelu) +DECLARE_OP_ADAPTER(GeluGrad) +DECLARE_OP_USE_OUTPUT(GeluGrad) +DECLARE_OP_ADAPTER(StridedSlice) +DECLARE_OP_USE_OUTPUT(StridedSlice) +DECLARE_OP_ADAPTER(UnsortedSegmentSumD) +DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentSumD) +DECLARE_OP_USE_OUTPUT(UnsortedSegmentSumD) +DECLARE_OP_ADAPTER(ExpandDims) +DECLARE_OP_USE_OUTPUT(ExpandDims) +DECLARE_OP_ADAPTER(Squeeze) +DECLARE_OP_USE_OUTPUT(Squeeze) +DECLARE_OP_ADAPTER(LayerNorm) +DECLARE_OP_USE_OUTPUT(LayerNorm) +DECLARE_OP_ADAPTER(LayerNormGrad) +DECLARE_OP_USE_OUTPUT(LayerNormGrad) +DECLARE_OP_ADAPTER(BatchMatMul) +DECLARE_OP_USE_OUTPUT(BatchMatMul) +DECLARE_OP_ADAPTER(DropOutDoMask) +DECLARE_OP_USE_OUTPUT(DropOutDoMask) +// ** Mix-precision Operations ** +DECLARE_OP_ADAPTER(NPUGetFloatStatus) +DECLARE_OP_USE_OUTPUT(NPUGetFloatStatus) +DECLARE_OP_ADAPTER(NPUAllocFloatStatus) +DECLARE_OP_USE_OUTPUT(NPUAllocFloatStatus) +DECLARE_OP_ADAPTER(NPUClearFloatStatus) +DECLARE_OP_USE_OUTPUT(NPUClearFloatStatus) +DECLARE_OP_ADAPTER(MatMul) +DECLARE_OP_USE_OUTPUT(MatMul) + +DECLARE_OP_ADAPTER(SoftmaxCrossEntropyWithLogits) +DECLARE_OP_USE_OUTPUT(SoftmaxCrossEntropyWithLogits) + +DECLARE_OP_ADAPTER(MeanGrad) +DECLARE_OP_USE_INPUT_ATTR(MeanGrad) + +DECLARE_OP_ADAPTER(Assign) +DECLARE_OP_USE_OUTPUT(Assign) +DECLARE_OP_ADAPTER(Constant) +DECLARE_OP_USE_OUTPUT(Constant) +DECLARE_OP_ADAPTER(ApplyMomentum) +DECLARE_OP_USE_OUTPUT(ApplyMomentum) +// ** Summary Operations ** +DECLARE_OP_ADAPTER(Summary) + +// fully supported +DECLARE_OP_ADAPTER(Add) +DECLARE_OP_USE_OUTPUT(Add) +DECLARE_OP_ADAPTER(Const) +DECLARE_OP_USE_OUTPUT(Const) +DECLARE_OP_ADAPTER(Cos) +DECLARE_OP_USE_OUTPUT(Cos) +DECLARE_OP_ADAPTER(Acos) +DECLARE_OP_USE_OUTPUT(Acos) + +DECLARE_OP_ADAPTER(AcosGrad) +DECLARE_OP_USE_OUTPUT(AcosGrad) + +DECLARE_OP_ADAPTER(Floor) +DECLARE_OP_USE_OUTPUT(Floor) +DECLARE_OP_ADAPTER(FloorDiv) +DECLARE_OP_USE_OUTPUT(FloorDiv) +DECLARE_OP_ADAPTER(Sin) +DECLARE_OP_USE_OUTPUT(Sin) +DECLARE_OP_ADAPTER(Exp) +DECLARE_OP_USE_OUTPUT(Exp) + +DECLARE_OP_ADAPTER(ReduceAll) +DECLARE_OP_USE_OUTPUT(ReduceAll) +DECLARE_OP_ADAPTER(ReduceSum) +DECLARE_OP_USE_OUTPUT(ReduceSum) +DECLARE_OP_ADAPTER(ReduceSumD) +DECLARE_OP_USE_INPUT_ATTR(ReduceSumD) +DECLARE_OP_USE_OUTPUT(ReduceSumD) +DECLARE_OP_ADAPTER(ReduceMeanD) +DECLARE_OP_USE_INPUT_ATTR(ReduceMeanD) +DECLARE_OP_USE_OUTPUT(ReduceMeanD) +DECLARE_OP_ADAPTER(ReduceProdD) +DECLARE_OP_USE_INPUT_ATTR(ReduceProdD) +DECLARE_OP_USE_OUTPUT(ReduceProdD) +DECLARE_OP_ADAPTER(CumprodD) +DECLARE_OP_USE_INPUT_ATTR(CumprodD) +DECLARE_OP_USE_OUTPUT(CumprodD) + +DECLARE_OP_ADAPTER(TileD) +DECLARE_OP_USE_INPUT_ATTR(TileD) +DECLARE_OP_USE_OUTPUT(TileD) +DECLARE_OP_ADAPTER(OneHot) +DECLARE_OP_USE_OUTPUT(OneHot) +DECLARE_OP_ADAPTER(GatherV2D) +DECLARE_OP_USE_INPUT_ATTR(GatherV2D) +DECLARE_OP_USE_OUTPUT(GatherV2D) + +DECLARE_OP_ADAPTER(Data) +DECLARE_OP_ADAPTER(BiasAdd) +DECLARE_OP_USE_OUTPUT(BiasAdd) +DECLARE_OP_ADAPTER(BatchNorm) +DECLARE_OP_USE_OUTPUT(BatchNorm) +DECLARE_OP_ADAPTER(BatchNormGrad) +DECLARE_OP_USE_OUTPUT(BatchNormGrad) +DECLARE_OP_ADAPTER(Relu) +DECLARE_OP_USE_OUTPUT(Relu) +DECLARE_OP_ADAPTER(PRelu) +DECLARE_OP_USE_OUTPUT(PRelu) +DECLARE_OP_ADAPTER(Elu) +DECLARE_OP_USE_OUTPUT(Elu) + +DECLARE_OP_ADAPTER(EluGrad) +DECLARE_OP_USE_OUTPUT(EluGrad) +DECLARE_OP_ADAPTER(PReluGrad) +DECLARE_OP_USE_OUTPUT(PReluGrad) + +DECLARE_OP_ADAPTER(L2Normalize) +DECLARE_OP_USE_OUTPUT(L2Normalize) + +DECLARE_OP_ADAPTER(CumsumD) +DECLARE_OP_USE_INPUT_ATTR(CumsumD) +DECLARE_OP_USE_OUTPUT(CumsumD) +DECLARE_OP_ADAPTER(L2NormalizeGrad) +DECLARE_OP_USE_OUTPUT(L2NormalizeGrad) +DECLARE_OP_ADAPTER(Sigmoid) +DECLARE_OP_USE_OUTPUT(Sigmoid) +DECLARE_OP_ADAPTER(SigmoidGrad) +DECLARE_OP_USE_OUTPUT(SigmoidGrad) +DECLARE_OP_ADAPTER(Softmax) +DECLARE_OP_USE_OUTPUT(Softmax) +DECLARE_OP_ADAPTER(SoftmaxGrad) +DECLARE_OP_USE_OUTPUT(SoftmaxGrad) +DECLARE_OP_ADAPTER(Greater) +DECLARE_OP_USE_OUTPUT(Greater) +DECLARE_OP_ADAPTER(Flatten) +DECLARE_OP_USE_OUTPUT(Flatten) +DECLARE_OP_ADAPTER(GatherV2) +DECLARE_OP_USE_OUTPUT(GatherV2) +DECLARE_OP_ADAPTER(MaxPool) +DECLARE_OP_USE_OUTPUT(MaxPool) +DECLARE_OP_ADAPTER(MaxPoolGrad) +DECLARE_OP_USE_OUTPUT(MaxPoolGrad) +DECLARE_OP_ADAPTER(AvgPool) +DECLARE_OP_USE_OUTPUT(AvgPool) +DECLARE_OP_ADAPTER(AvgPoolGrad) +DECLARE_OP_USE_OUTPUT(AvgPoolGrad) +DECLARE_OP_ADAPTER(ROIAlign) +DECLARE_OP_USE_OUTPUT(ROIAlign) +DECLARE_OP_ADAPTER(ROIAlignGrad) +DECLARE_OP_USE_OUTPUT(ROIAlignGrad) +DECLARE_OP_ADAPTER(Abs) +DECLARE_OP_USE_OUTPUT(Abs) +DECLARE_OP_ADAPTER(AbsGrad) +DECLARE_OP_USE_OUTPUT(AbsGrad) +DECLARE_OP_ADAPTER(BinaryCrossEntropy) +DECLARE_OP_USE_OUTPUT(BinaryCrossEntropy) +DECLARE_OP_ADAPTER(BinaryCrossEntropyGrad) +DECLARE_OP_USE_OUTPUT(BinaryCrossEntropyGrad) +DECLARE_OP_ADAPTER(SparseApplyAdagradD) +DECLARE_OP_USE_OUTPUT(SparseApplyAdagradD) +DECLARE_OP_ADAPTER(SpaceToDepth) +DECLARE_OP_USE_OUTPUT(SpaceToDepth) +DECLARE_OP_ADAPTER(DepthToSpace) +DECLARE_OP_USE_OUTPUT(DepthToSpace) +DECLARE_OP_ADAPTER(Sign) +DECLARE_OP_USE_OUTPUT(Sign) +DECLARE_OP_ADAPTER(LarsV2Update) +DECLARE_OP_USE_OUTPUT(LarsV2Update) +DECLARE_OP_ADAPTER(Round) +DECLARE_OP_USE_OUTPUT(Round) +DECLARE_OP_ADAPTER(ApplyFtrl) +DECLARE_OP_USE_OUTPUT(ApplyFtrl) +#ifdef ENABLE_GE +DECLARE_OP_ADAPTER(Print) +DECLARE_OP_USE_DYN_INPUT(Print) +#endif +} // namespace transform +} // namespace mindspore +#endif // TRANSFORM_OP_DECLARE_H_ diff --git a/mindspore/ccsrc/transform/types.h b/mindspore/ccsrc/transform/types.h new file mode 100644 index 00000000000..1ce492fed2f --- /dev/null +++ b/mindspore/ccsrc/transform/types.h @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_TYPES_H_ +#define TRANSFORM_TYPES_H_ + +#include +#include +#include +#include +#include +#include "ir/anf.h" +#include "ir/dtype.h" +#include "ir/meta_tensor.h" + +#include "graph/tensor.h" +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif +using GeTensor = ge::Tensor; + +namespace mindspore { +namespace transform { +enum Status : int { SUCCESS = 0, FAILED, INVALID_ARGUMENT, ALREADY_EXISTS, NOT_FOUND }; + +using MeTensor = mindspore::tensor::Tensor; +using MeTensorPtr = std::shared_ptr; +using MeDataType = mindspore::TypeId; +using GeDataType = ge::DataType; +using GeFormat = ge::Format; +using GeShape = ge::Shape; +using GeTensorPtr = std::shared_ptr; +using GeTensorDesc = ge::TensorDesc; +using AnfGraph = FuncGraph; +using AnfGraphPtr = FuncGraphPtr; +using Operator = ge::Operator; +using OperatorPtr = std::shared_ptr; +using DfGraph = ge::Graph; +using DfGraphPtr = std::shared_ptr; +using TensorMap = std::unordered_map>; +} // namespace transform +} // namespace mindspore + +#endif // TRANSFORM_TYPES_H_ diff --git a/mindspore/ccsrc/transform/util.cc b/mindspore/ccsrc/transform/util.cc new file mode 100644 index 00000000000..a106a20ad80 --- /dev/null +++ b/mindspore/ccsrc/transform/util.cc @@ -0,0 +1,456 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/util.h" + +#include +#include +#include + +#include "securec/include/securec.h" +#include "utils/convert_utils.h" +#include "utils/utils.h" + +namespace mindspore { +namespace transform { +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; + +const size_t kErrorSize = 0; + +vector TransformUtil::ConvertIntToList(int64_t data, int size) { + vector list{}; + if (size <= 0) { + MS_LOG(WARNING) << "size <= 0"; + return list; + } + for (int i = 0; i < size; ++i) { + list.push_back(data); + } + return list; +} + +static std::map datatype_trans_map = { + {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, + {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, + {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, + {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, + {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, + {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; + +GeDataType TransformUtil::ConvertDataType(const MeDataType& type) { + MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; + if (datatype_trans_map.find(type) != datatype_trans_map.end()) { + return datatype_trans_map[type]; + } else { + return GeDataType::DT_UNDEFINED; + } +} + +static std::map datatype_size_map = { + {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float + {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, + {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, + {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, + {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, + {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; + +size_t TransformUtil::GetDataTypeSize(const MeDataType& type) { + if (datatype_size_map.find(type) != datatype_size_map.end()) { + return datatype_size_map[type]; + } else { + MS_LOG(ERROR) << "Illegal tensor data type!"; + return kErrorSize; + } +} + +GeFormat TransformUtil::ConvertFormat(const string& format) { + if (format == kOpFormat_NCHW) { + return GeFormat::FORMAT_NCHW; + } else if (format == kOpFormat_NC1HWC0) { + return GeFormat::FORMAT_NC1HWC0; + } else if (format == kOpFormat_NHWC) { + return GeFormat::FORMAT_NHWC; + } else if (format == kOpFormat_HWCN) { + return GeFormat::FORMAT_HWCN; + } else { + return GeFormat::FORMAT_ND; + } +} + +static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } + +std::shared_ptr TransformUtil::GetGeTensorDesc(const std::vector& me_shape, + const MeDataType& me_type, const std::string& format) { + // convert me shape to ge shape + std::vector ge_shape; + + if (me_shape.size() == 1) { + ge_shape.push_back(static_cast(me_shape[0])); + } else { + ge_shape.resize(me_shape.size()); + (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); + } + + GeShape shape(ge_shape); + if (shape.GetDimNum() == 0) { + MS_LOG(INFO) << "The dims size of Ge tensor is zero"; + } + // convert me format to ge format + GeFormat ge_format = ConvertFormat(format); + if (ge_format == GeFormat::FORMAT_ND) { + MS_LOG(ERROR) << "undefined data format : " << static_cast(ge_format); + return nullptr; + } + // convert me datatype to ge datatype + GeDataType data_type = ConvertDataType(me_type); + if (data_type == GeDataType::DT_UNDEFINED) { + MS_LOG(ERROR) << "undefined data type :" << me_type; + return nullptr; + } + + auto desc = std::make_shared(shape, ge_format, data_type); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create GeTensorDesc failed!"; + return nullptr; + } + MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); + desc->SetRealDimCnt(SizeToInt(me_shape.size())); + return desc; +} + +// if failed, return empty vector. +std::vector TransformUtil::ConvertInputTensors(const std::vector& me_tensors, + const std::string& format) { + std::vector ge_tensors; + + for (size_t index = 0; index < me_tensors.size(); index++) { + MS_EXCEPTION_IF_NULL(me_tensors[index]); + MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); + auto shape = me_tensors[index]->shape(); + std::string shape_str; + for (size_t i = 0; i < shape.size(); i++) { + shape_str += std::to_string(shape[i]); + shape_str += " "; + } + MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; + MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); + + auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); + if (ge_tensor_ptr != nullptr) { + ge_tensors.emplace_back(ge_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; + ge_tensors.clear(); + return ge_tensors; + } + } + return ge_tensors; +} + +GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr& tensor, const std::string& format) { + // get tensor data type size + MS_EXCEPTION_IF_NULL(tensor); + size_t type_size = GetDataTypeSize(tensor->data_type()); + if (type_size == kErrorSize) { + MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; + return nullptr; + } + // get tensor buff size + size_t data_buff_size = 0; + size_t elements_num = IntToSize(tensor->ElementsNum()); + if (elements_num > 0 && type_size > 0 && UINT_MAX / type_size >= elements_num) { + data_buff_size = elements_num * type_size; + } + if (data_buff_size == 0) { + if (elements_num > 0 && type_size > 0 && UINT_MAX / type_size < elements_num) { + MS_LOG(ERROR) << "The required Me Tensor data buff size " << elements_num << " x " << type_size + << " overflowed UINT_MAX: " << UINT_MAX << "."; + } else { + MS_LOG(ERROR) << "The Me Tensor data buff size is 0."; + } + return nullptr; + } + // create ge tensor + auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + if (desc == nullptr) { + MS_LOG(ERROR) << "Failed to get Tensor Desc"; + return nullptr; + } + GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); + if (tensor_ptr != nullptr) { + MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; + } + return tensor_ptr; +} + +std::vector TransformUtil::ConvertGeTensors(const std::vector& ge_tensors, + const std::vector>& request_dims) { + std::vector outputs; + + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = nullptr; + if (index < request_dims.size()) { + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); + } else { + std::vector empty_shape; + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); + } + + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} + +std::vector TransformUtil::ConvertGeTensors(const std::vector& ge_tensors) { + std::vector outputs; + + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} + +MeDataType TransformUtil::ConvertGeDataType(const GeDataType& type) { + switch (type) { + case GeDataType::DT_FLOAT16: + return MeDataType::kNumberTypeFloat16; + case GeDataType::DT_FLOAT: + return MeDataType::kNumberTypeFloat32; + case GeDataType::DT_DOUBLE: + return MeDataType::kNumberTypeFloat64; + case GeDataType::DT_INT64: + return MeDataType::kNumberTypeInt64; + case GeDataType::DT_INT32: + return MeDataType::kNumberTypeInt32; + case GeDataType::DT_INT16: + return MeDataType::kNumberTypeInt16; + case GeDataType::DT_INT8: + return MeDataType::kNumberTypeInt8; + case GeDataType::DT_BOOL: + return MeDataType::kNumberTypeBool; + case GeDataType::DT_UINT8: + return MeDataType::kNumberTypeUInt8; + case GeDataType::DT_UINT16: + return MeDataType::kNumberTypeUInt16; + case GeDataType::DT_UINT32: + return MeDataType::kNumberTypeUInt32; + case GeDataType::DT_UINT64: + return MeDataType::kNumberTypeUInt64; + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL_SUB_UINT8: + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_DUAL: + return MeDataType::kTypeUnknown; + default: + return MeDataType::kTypeUnknown; + } +} + +namespace { +bool IsGeShapeCompatible(const GeShape& ge_shape, const std::vector& request_dims) { + MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); + MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); + + const int GE_DIMS = 4; + std::vector ge_dims = ge_shape.GetDims(); + if (request_dims.size() > ge_dims.size()) { + MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; + return false; + } + + // convert NHWC to NCHW + if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && + (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; + } + + std::string::size_type i = 0; + for (; i < request_dims.size(); i++) { + if (ge_dims[i] != request_dims[i]) { + MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; + return false; + } + } + + for (; i < ge_dims.size(); i++) { + if (ge_dims[i] != 1) { + MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; + return false; + } + } + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; +} +} // namespace + +GeShape TransformUtil::ConvertMeShape(const std::vector& me_dims) { + std::vector ge_dims; + (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); + return GeShape(ge_dims); +} + +std::vector TransformUtil::ConvertGeShape(const GeShape& ge_shape) { + std::vector me_dims; + std::vector ge_dims = ge_shape.GetDims(); + (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); + return me_dims; +} + +std::vector TransformUtil::ConvertGeShape(const GeShape& ge_shape, const std::vector& request_dims) { + vector ret; + if (ge_shape.GetDimNum() == 0) { + MS_LOG(DEBUG) << "GeTensor's shape is scalar"; + return ret; + } + + if (IsGeShapeCompatible(ge_shape, request_dims) == true) { + ret = request_dims; + } else { + MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; + ret = ConvertGeShape(ge_shape); + } + return ret; +} + +MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr& ge_tensor, const std::vector& me_dims, + const TypeId& me_type) { + MeTensor me_tensor(me_type, me_dims); + + // Get the writable data pointer of the tensor and cast it to its data type + auto me_data_ptr = reinterpret_cast(me_tensor.data_c(true)); + size_t me_data_size = static_cast(me_tensor.data().nbytes()); + MS_EXCEPTION_IF_NULL(me_data_ptr); + MS_EXCEPTION_IF_NULL(ge_tensor); + if (me_data_size < ge_tensor->GetSize()) { + MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" + << ge_tensor->GetSize() << " bytes]"; + return nullptr; + } + + // Copy or use the writable data pointer of the ME tensor + MS_EXCEPTION_IF_NULL(ge_tensor->GetData()); + if (ge_tensor->GetSize() == 0) { + MS_LOG(ERROR) << "GE tensor data size is zero!"; + return nullptr; + } + errno_t ret = memcpy_s(me_data_ptr, me_data_size, ge_tensor->GetData(), ge_tensor->GetSize()); + if (ret != EOK) { + MS_LOG(INFO) << "GE tensor data size is " << ge_tensor->GetSize() << " bytes"; + MS_LOG(ERROR) << "Copy GE tensor data to me tensor failed"; + return nullptr; + } + return make_shared(me_tensor); +} + +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr& ge_tensor) { + MS_EXCEPTION_IF_NULL(ge_tensor); + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape(); + vector me_dims = ConvertGeShape(ge_shape); + + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType()); + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} + +// if request_dims is empty, use ge tensor's shape,otherwise convert to request shape +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const std::vector& request_dims) { + MS_EXCEPTION_IF_NULL(ge_tensor); + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape(); + vector me_dims = ConvertGeShape(ge_shape, request_dims); + MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + // Create a tensor with wanted data type and shape + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType()); + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} + +std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { + std::string ret; + if (ge_tensor == nullptr) { + MS_LOG(ERROR) << "Input ge tensor is nullptr"; + return ret; + } + + MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + switch (ge_tensor->GetTensorDesc().GetDataType()) { + case GeDataType::DT_UINT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DOUBLE: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_INT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT8: + case GeDataType::DT_DUAL_SUB_UINT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT16: + case GeDataType::DT_BOOL: + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL: + default: + MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) + << " ge tensor"; + break; + } + return ret; +} +} // namespace transform +} // namespace mindspore diff --git a/mindspore/ccsrc/transform/util.h b/mindspore/ccsrc/transform/util.h new file mode 100644 index 00000000000..9bcd8dc115d --- /dev/null +++ b/mindspore/ccsrc/transform/util.h @@ -0,0 +1,241 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRANSFORM_UTIL_H_ +#define TRANSFORM_UTIL_H_ + +#include +#include +#include +#include +#include "securec/include/securec.h" +#include "ir/anf.h" +#include "ir/dtype.h" +#include "ir/meta_tensor.h" +#include "transform/types.h" + +#include "graph/tensor.h" + +namespace mindspore { +namespace transform { +class TransformUtil { + public: + /* + * Parameters: + * type: [MeDataType] the data type for ME tensor + * Return： + * [GeDataType] the data type for ge tensor + * */ + static std::vector ConvertIntToList(int64_t data, int size); + + /* + * Parameters: + * type: [MeDataType] the data type for ME tensor + * Return： + * [GeDataType] the data type for ge tensor + * */ + static GeDataType ConvertDataType(const MeDataType& type); + + /* + * Parameters: + * type: [string] the data format in ME op + * Return： + * [GeFormat] the data format for ge tensor + * */ + static GeFormat ConvertFormat(const std::string& format); + + /* + * Parameters: + * type: [MeDataType] the data type for ME tensor + * Return： + * [size_t] the buff size for the type in ME + * */ + static size_t GetDataTypeSize(const MeDataType& type); + + /* + * Parameters: + * tensor: [MeTensorPtr] the me tensor to get description from + * format: [string] the data format in ME + * is_input: [bool] whether the tensor is used as input, default:false + * Return： + * [shared_ptr] the shared pointer of ge tensor description + * */ + static std::shared_ptr GetGeTensorDesc(const std::vector& shape, const MeDataType& me_type, + const std::string& format); + + /* + * Parameters: + * tensor: [MeTensor] the data tensor in ME + * format: [string] the data format in ME op + * is_input: [bool] whether the tensor is used as input, default:false + * Return： + * [GeTensor] the data tensor in GE + * */ + static GeTensorPtr ConvertTensor(const MeTensorPtr& tensor, const std::string& format); + + /* + * Parameters: + * me_tensors: [vector] the data tensors in ME + * format: [string] the data format in ME op + * Return： + * [std::vector] the data tensors in GE + * */ + static std::vector ConvertInputTensors(const std::vector& me_tensors, + const std::string& format); + + /* + * Parameters: + * tensor: [GeTensor] the data tensor in GE + * Return： + * [MeTensor] the data tensor in ME + * */ + static MeTensorPtr ConvertGeTensor(const GeTensorPtr& tensor); + + /* + * Parameters: + * tensor: [GeTensor] the data tensor in GE + * request_dims [std::vector] the output Me tensors must adjust to this shapes + * Return： + * [MeTensor] the data tensor in ME + * */ + static MeTensorPtr ConvertGeTensor(GeTensorPtr ge_tensor, const std::vector& request_dims); + /* + * Parameters: + * ge_tensors: [std::vector] the data tensor in GE + * request_dims [std::vector>] the output Me tensors must adjust to this shapes + * Return： + * [std::vector] the data tensor in ME + * */ + static std::vector ConvertGeTensors(const std::vector& ge_tensors, + const std::vector>& request_dims); + /* + * Parameters: + * ge_tensors: [std::vector] the data tensor in GE + * Return： + * [std::vector] the data tensor in ME + * */ + static std::vector ConvertGeTensors(const std::vector& ge_tensors); + /* + * Parameters: + * ge_tensor: [GeTensor] the data tensor in GE + * me_dims: [std::vector] the shape of created Me tensor + * me_type: [TypeId] the type of created Me tensor + * Return： + * [MeTensor] the data tensor in ME + * */ + static MeTensorPtr GenerateMeTensor(const GeTensorPtr& ge_tensor, const std::vector& me_dims, + const TypeId& me_type); + /* + * Parameters: + * type: [GeDataType] the ge tensor data type + * Return： + * [MeDataType] the me tensor data type + * */ + static MeDataType ConvertGeDataType(const GeDataType& type); + + /* + * Parameters: + * me_dims: [std::vector] the me shape + * Return： + * [GeShape] the ge shape + * */ + static GeShape ConvertMeShape(const std::vector& me_dims); + + /* + * Parameters: + * ge_shape: [GeShape] the ge shape + * Return： + * [vector] the me shape + * */ + static std::vector ConvertGeShape(const GeShape& ge_shape); + + /* Function: + * Convert GeShape to Me request shape, Support pattern: + * {1, x, 1, 1} --> {x} + * {x, 1, 1, 1} --> {x} + * {x, x, 1, 1} --> {x, x} + * {x, x, x, 1} --> {x, x, x} + * {x, x, x, x} --> {x, x, x, x} + * If unmatch upon patterns, return original ge dims + * Parameters: + * ge_shape: [GeShape] the ge shape + * request_dims: [vector] request dims + * Return： + * [vector] the me shape + * */ + static std::vector ConvertGeShape(const GeShape& ge_shape, const std::vector& request_dims); + + /* + * Parameters: + * vec: [std::vector] the vector to print + * Return： + * [string] value string + * */ + template ::value>::type> + static std::string PrintVector(const std::vector& vec) { + const int MAX_PRINT_NUM = 100; + std::stringstream ss; + ss << "{ "; + int i = 0; + for (auto it = vec.begin(); it != vec.end(); ++it) { + ss << std::to_string(*it) << ", "; + i++; + if (i >= MAX_PRINT_NUM) { + break; + } + } + + if (i >= MAX_PRINT_NUM) { + ss << "... to be continue}"; + } else { + ss << "}"; + } + return ss.str(); + } + + /* + * Parameters: + * ge_tensor: [GeTensorPtr] the ge tensor + * Return： + * [stringstream] value string + * */ + static std::string PrintGeTensor(const GeTensorPtr ge_tensor); + + /* + * Parameters: + * data: [uint8_t *] the ge tensor data pointer + * size: [size_t] the ge tensor data bytes + * Return： + * [shared_ptr] vector pointer + * */ + template ::value>::type> + static std::vector MakeVector(const uint8_t* const data, size_t size) { + auto dest = std::vector(size / sizeof(T)); + if (data == nullptr) { + return dest; + } + + errno_t ret = memcpy_s(dest.data(), dest.size() * sizeof(T), data, size); + if (EOK != ret) { + return std::vector(); + } + return dest; + } +}; +} // namespace transform +} // namespace mindspore + +#endif // TRANSFORM_UTIL_H_ diff --git a/mindspore/ccsrc/utils/CMakeLists.txt b/mindspore/ccsrc/utils/CMakeLists.txt new file mode 100644 index 00000000000..1fcf5e0944c --- /dev/null +++ b/mindspore/ccsrc/utils/CMakeLists.txt @@ -0,0 +1,3 @@ +file(GLOB_RECURSE _UTILS_ALL_SRC_FILES *.cc) +#TODO : "utils/node_utils.cc" +add_library(_mindspore_utils_obj OBJECT ${_UTILS_ALL_SRC_FILES}) diff --git a/mindspore/ccsrc/utils/anf_ir.proto b/mindspore/ccsrc/utils/anf_ir.proto new file mode 100644 index 00000000000..145751e7f06 --- /dev/null +++ b/mindspore/ccsrc/utils/anf_ir.proto @@ -0,0 +1,332 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package mindspore.irpb; + + +// Versioning +enum Version { + // unknown version + UNKNOWWN_VERSION = 0; + + // Initial version (IR VERSION 1), published on Sep 23, 2019 + IR_VERSION = 0x0000000000000001; +} + +// Data type definition +enum DataType { + DT_UNDEFINED = 0; + // Basic types. + DT_BOOL = 1; // bool + + DT_INT8 = 2; // int8_t + DT_INT16 = 3; // int16_t + DT_INT32 = 4; // int32_t + DT_INT64 = 5; // int64_t + + DT_UINT8 = 6; // uint8_t + DT_UINT16 = 7; // uint16_t + DT_UINT32 = 8; // uint32_t + DT_UINT64 = 9; // uint64_t + + DT_FLOAT16 = 10; // float 16 + DT_FLOAT32 = 11; // float 32 + DT_FLOAT64 = 12; // float 64 + + DT_STRING = 13; // string + DT_TENSOR = 14; // tensor + DT_GRAPH = 15; // graph + + // list type + DT_BOOLS = 16; // list of bool + + DT_INTS8 = 17; // list of int8_t + DT_INTS16 = 18; // list of int16_t + DT_INTS32 = 19; // list of int32_t + DT_INTS64 = 20; // list of int64_t + + DT_UINTS8 = 21; // list of uint8_t + DT_UINTS16 = 22; // list of uint16_t + DT_UINTS32 = 23; // list of uint32_t + DT_UINTS64 = 24; // list of uint64_t + + DT_FLOATS16 = 25; // list of float16 + DT_FLOATS32 = 26; // list of float32 + DT_FLOATS64 = 27; // list of float64 + + DT_STRINGS = 28; // list of string + DT_TENSORS = 29; // list of tensor + DT_GRAPHS = 30; // list of graph + + DT_TUPLE = 31; // tuple + DT_LIST = 32; // list + DT_DICT = 33; // dictionary + + // other types + DT_NONE = 34; // None + DT_SYM_INST = 35; // Symbolic Key Instance + + // type related type + DT_BASE_INT = 36; // type generic int + DT_BASE_UINT = 37; // type generate unsigned int + DT_BASE_FLOAT = 38; // type generate float + DT_TYPE = 39; // type type + DT_ANYTHING = 40; // type anything + DT_REFKEY = 41; // type refkey + DT_REF = 42; // type ref +} + +// Value definition for attribute value or parameter default value +message ValueProto { + // data type of value + optional DataType dtype = 1; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present for this version of the IR + optional bool bool_val = 2; // bool + optional int64 int_val = 3; // int + optional uint64 uint_val = 4; // uint + optional float float_val = 5; // float + optional double double_val = 6; // double + optional string str_val = 7; // string + optional TensorProto tensor_val = 8; // tensor value + optional GraphProto graph = 9; // graph + + repeated bool bool_vals = 10; // list of bool + repeated int64 int_vals = 11; // list of int + repeated uint64 uint_vals = 12; // list of uint + repeated float float_vals = 13; // list of float + repeated double double_vals = 14; // list of double + repeated string str_vals = 15; // list of string + repeated TensorProto tensor_vals = 16; // list of tensor value + repeated GraphProto graphs = 17; // list of graph + + // tuple or list + repeated ValueProto values = 18; // tuple, list of value + + // dictionary + repeated NamedValueProto dict_val = 19; // dictionary info + + // filed for type type + optional TypeProto type_val = 20; // type type info +} + +message AttributeProto { + optional string name = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +message NamedValueProto { + optional string key = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +// Defines a tensor shape. +message TensorShapeProto { + // One dimension of the tensor. + message Dimension { + // Size of the tensor in that dimension. + // This value must be >= -1, but values of -1 are reserved for "unknown" + // shapes (values of -1 mean "unknown" dimension). + optional int64 size = 1; + + // Optional name of the tensor dimension. + optional string name = 2; + }; + + repeated Dimension dim = 1; +} + +// Types for graph input(parameter) and output +message TypeProto { + + message Tensor { + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType elem_type = 1; + optional TensorShapeProto shape = 2; // for scalar, this field is not set + } + + // tuple type + message Sequence { + // The type and optional shape of elements of the tuple. + repeated TypeProto elem_types = 1; + }; + + // data type + optional DataType data_type = 1; + + oneof value { + // The type of a tensor. + Tensor tensor_type = 2; + + // The type of a tuple. + Sequence sequence_type = 3; + } +} + +// Defines information on graph parameters, including the name, the type, and +// the default value of parameter if exists. +message ParameterProto { + optional string name = 1; // parameter name + optional TypeProto type = 2; // parameter type + optional ValueProto default_val = 3; // default value of parameter if exists +} + +// Defines graph output information +message OutputProto { + optional string name = 1; // output node name + optional TypeProto type = 2; // output node type +} + +// Define node input information +message InputProto { + enum EdgeType { + DATA_EDGE = 0; // data edge + CONTROL_EDGE = 1; // control edge + } + + optional string name = 1; + optional EdgeType type = 2; +} + +// Nodes +// +// Computation graphs are made up of a DAG of nodes, which represent what is +// commonly called a "layer" or "pipeline stage" in machine learning frameworks. +// +// For example, it can be a node of type "Conv" that takes in an image, a filter +// tensor and a bias tensor, and produces the convolved output. +message NodeProto { + repeated InputProto input = 1; // namespace Value + optional string name = 2; // namespace Value + + // The symbolic identifier of the Operator to execute. + optional string op_type = 3; // namespace Operator + // The domain of the OperatorSet that specifies the operator named by op_type. + optional string scope = 4; // namespace Domain + + // Additional named attributes. + repeated AttributeProto attribute = 5; + + // Optional type info of this node + optional TypeProto output_type = 6; + + // other fields for debug + optional uint64 output_i = 7; +} + +// Models +// +// ModelProto is a top-level file/container format for bundling a ML model and +// associating its computation graph with metadata. +// +// The semantics of the model are described by the associated GraphProto. +message ModelProto { + // ir version + optional int64 ir_version = 1; + + // Domain name of the model. + // We use reverse domain names as name space indicators. For example: + // `com.facebook.fair` or `com.microsoft.cognitiveservices` + // + // Together with `model_version` and GraphProto.name, this forms the unique identity of + // the graph. + optional string domain = 2; + + // The version of the graph encoded. See Version enum below. + optional int64 model_version = 3; + + // The parameterized graph that is evaluated to execute the model. + optional GraphProto graph = 4; + + // metadata info of opeartors + optional OperatorSetProto metadata_operators = 5; +}; + +message OperatorProto { + optional string name = 1; // used as key, must be distinct + optional bytes config = 2; // operator config info + optional bytes obj_info = 3; // operator related object info, e.g. content of operator binary or name +}; + +message OperatorSetProto { + repeated OperatorProto operators = 1; + optional string summary = 2; // summary info of operators, e.g. file position of operators file +} + +// Graphs +// +// A graph defines the computational logic of a model and is comprised of a parameterized +// list of nodes that form a directed acyclic graph based on their inputs and outputs. +// This is the equivalent of the "network" or "graph" in many deep learning +// frameworks. +message GraphProto { + // The nodes in the graph, sorted topologically. + repeated NodeProto node = 1; + + // The name of the graph. + optional string name = 2; // namespace Graph + + // The parameters(inputs) and outputs of the graph. + repeated ParameterProto parameters = 3; + repeated OutputProto outputs = 4; + + // Constants used in this graph + repeated NamedValueProto const_vals = 5; +} + +// Tensors +// +// A serialized tensor value. +message TensorProto { + // The shape of the tensor. + repeated int64 dims = 1; + + // The data type of the tensor. + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType data_type = 2; + + // Tensor content must be organized in row-major order. + // + // Depending on the data_type field, exactly one of the fields below with + // name ending in _data is used to store the elements of the tensor. + + // For float values + repeated float float_data = 3 [packed = true]; + + // For int32, uint8, int8, uint16, int16, and bool values + // When this field is present, the data_type field MUST be + // INT32, INT16, INT8, UINT16, UINT8, or BOOL + repeated int32 int32_data = 4 [packed = true]; + + // For int64. + // When this field is present, the data_type field MUST be INT64 + repeated int64 int64_data = 5 [packed = true]; + + // For double + // When this field is present, the data_type field MUST be DOUBLE + repeated double double_data = 6 [packed = true]; + + // For uint64 and uint32 values + // When this field is present, the data_type field MUST be + // UINT32 or UINT64 + repeated uint64 uint64_data = 7 [packed = true]; + + // Store raw tensor content. When this raw_data field is used to store tensor value, + // elements MUST be stored in as fixed-width, little-endian order. + optional bytes raw_data = 8; +} diff --git a/mindspore/ccsrc/utils/any.cc b/mindspore/ccsrc/utils/any.cc new file mode 100644 index 00000000000..31ee1fd3021 --- /dev/null +++ b/mindspore/ccsrc/utils/any.cc @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/any.h" +#include +#include + +namespace mindspore { + +// only support (int, float, bool) as Literal +bool AnyIsLiteral(const Any& any) { + static const std::type_index typeid_int = std::type_index(typeid(int)); + static const std::type_index typeid_float = std::type_index(typeid(float)); + static const std::type_index typeid_bool = std::type_index(typeid(bool)); + + std::type_index typeid_any = std::type_index(any.type()); + return typeid_int == typeid_any || typeid_float == typeid_any || typeid_bool == typeid_any; +} + +std::ostream& operator<<(std::ostream& os, const pybind11::object& obj) { + os << "[py::object]"; + return os; +} + +Any& Any::operator=(const Any& other) { + if (m_ptr == other.m_ptr || &other == this) { + return *this; + } + m_ptr = other.clone(); + m_tpIndex = other.m_tpIndex; + return *this; +} + +bool Any::operator<(const Any& other) const { return this < &other; } + +Any& Any::operator=(Any&& other) { + if (this != &other) { + if (m_ptr == other.m_ptr || &other == this) { + return *this; + } + + m_ptr = std::move(other.m_ptr); + m_tpIndex = std::move(other.m_tpIndex); + other.m_ptr = nullptr; + } + return *this; +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/any.h b/mindspore/ccsrc/utils/any.h new file mode 100644 index 00000000000..ce691f1c122 --- /dev/null +++ b/mindspore/ccsrc/utils/any.h @@ -0,0 +1,218 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_UTILS_ANY_H_ +#define MINDSPORE_CCSRC_UTILS_ANY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" + +#include "utils/overload.h" +#include "utils/log_adapter.h" +#include "utils/misc.h" + +namespace mindspore { +// usage:AnyPtr sp = std::make_shared(aname); +template +std::string type(const T& t) { + return demangle(typeid(t).name()); +} + +std::ostream& operator<<(std::ostream& os, const pybind11::object& obj); + +class Any { + public: + // constructors + Any() : m_ptr(nullptr), m_tpIndex(std::type_index(typeid(void))) {} + Any(const Any& other) : m_ptr(other.clone()), m_tpIndex(other.m_tpIndex) {} + Any(Any&& other) : m_ptr(std::move(other.m_ptr)), m_tpIndex(std::move(other.m_tpIndex)) {} + + Any& operator=(Any&& other); + // right reference constructor + template ::type, Any>::value, T>::type> + Any(T&& t) : m_tpIndex(typeid(typename std::decay::type)) { // NOLINT + BasePtr new_val(new Derived::type>(std::forward(t))); + std::swap(m_ptr, new_val); + } + + ~Any() = default; + + // judge whether is empty + bool empty() const { return m_ptr == nullptr; } + + // judge the is relation + template + bool is() const { + return m_tpIndex == std::type_index(typeid(T)); + } + + const std::type_info& type() const { return m_ptr ? m_ptr->type() : typeid(void); } + + std::size_t Hash() const { + std::stringstream buffer; + buffer << m_tpIndex.name(); + if (m_ptr != nullptr) { + buffer << m_ptr->GetString(); + } + return std::hash()(buffer.str()); + } + + template + bool Apply(const std::function& fn) { + if (type() == typeid(T)) { + T x = cast(); + fn(x); + return true; + } + return false; + } + + std::string GetString() const { + if (m_ptr != nullptr) { + return m_ptr->GetString(); + } else { + return std::string(""); + } + } + + friend std::ostream& operator<<(std::ostream& os, const Any& any) { + os << any.GetString(); + return os; + } + + // type cast + template + T& cast() const { + if (!is() || !m_ptr) { + // Use MS_LOGFATAL replace throw std::bad_cast() + MS_LOG(EXCEPTION) << "can not cast " << m_tpIndex.name() << " to " << typeid(T).name(); + } + auto ptr = static_cast*>(m_ptr.get()); + return ptr->m_value; + } + + bool operator==(const Any& other) const { + if (m_tpIndex != other.m_tpIndex) { + return false; + } + if (m_ptr == nullptr && other.m_ptr == nullptr) { + return true; + } + if (m_ptr == nullptr || other.m_ptr == nullptr) { + return false; + } + return *m_ptr == *other.m_ptr; + } + + bool operator!=(const Any& other) const { return !(operator==(other)); } + + Any& operator=(const Any& other); + + bool operator<(const Any& other) const; + + std::string ToString() const { + std::ostringstream buffer; + if (m_tpIndex == typeid(float)) { + buffer << " " << cast(); + } else if (m_tpIndex == typeid(double)) { + buffer << " " << cast(); + } else if (m_tpIndex == typeid(int)) { + buffer << " " << cast(); + } else if (m_tpIndex == typeid(bool)) { + buffer << " " << cast(); + } else { + buffer << "<" << demangle(m_tpIndex.name()) << "> " << m_ptr->GetString(); + } + return buffer.str(); + } + __attribute__((used)) void dump() const { std::cout << ToString() << std::endl; } + + private: + struct Base; + using BasePtr = std::unique_ptr; + + // type base definition + struct Base { + virtual const std::type_info& type() const = 0; + virtual BasePtr clone() const = 0; + virtual ~Base() = default; + virtual bool operator==(const Base& other) const = 0; + virtual std::string GetString() = 0; + }; + + template + struct Derived : public Base { + template + explicit Derived(Args&&... args) : m_value(std::forward(args)...), serialize_cache_("") {} + + bool operator==(const Base& other) const override { + if (typeid(*this) != typeid(other)) { + return false; + } + return m_value == static_cast&>(other).m_value; + } + + const std::type_info& type() const override { return typeid(T); } + + BasePtr clone() const override { return BasePtr(new Derived(m_value)); } + + ~Derived() override {} + + std::string GetString() override { + std::stringstream buffer; + buffer << m_value; + return buffer.str(); + } + + T m_value; + std::string serialize_cache_; + }; + + // clone method + BasePtr clone() const { + if (m_ptr != nullptr) { + return m_ptr->clone(); + } + return nullptr; + } + + BasePtr m_ptr; // point to real data + std::type_index m_tpIndex; // type info of data +}; + +using AnyPtr = std::shared_ptr; + +struct AnyHash { + std::size_t operator()(const Any& c) const { return c.Hash(); } +}; + +struct AnyLess { + bool operator()(const Any& a, const Any& b) const { return a.Hash() < b.Hash(); } +}; + +bool AnyIsLiteral(const Any& any); + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_ANY_H_ diff --git a/mindspore/ccsrc/utils/base_ref.cc b/mindspore/ccsrc/utils/base_ref.cc new file mode 100644 index 00000000000..e50f0003b8f --- /dev/null +++ b/mindspore/ccsrc/utils/base_ref.cc @@ -0,0 +1,197 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/base_ref.h" + +namespace mindspore { +iterator ConstIteratorCast(std::vector* v, const const_iterator iter) { + return std::next(v->begin(), std::distance(v->cbegin(), iter)); +} + +BaseRef::BaseRef(const BaseRef& other) : Base(other), m_ptr(other.m_ptr) { + if (!m_ptr) { + m_ptr = other.copy(); + } +} + +bool BaseRef::operator==(const BaseRef& other) const { + if (m_ptr == other.m_ptr) { + return true; + } + if (m_ptr == nullptr && other.m_ptr == nullptr) { + return *this == other; + } + if (m_ptr == nullptr || other.m_ptr == nullptr) { + return false; + } + if (type() != other.type()) { + MS_LOG(DEBUG) << "Type mismatch"; + return false; + } + if (m_ptr->isa()) { + return *(m_ptr->cast()) == *(other.m_ptr->cast()); + } + + // for noderef equal + if (m_ptr->isa()) { + return *std::static_pointer_cast(m_ptr) == *std::static_pointer_cast(other.m_ptr); + } + + // for node equal + return *m_ptr == *other.m_ptr; +} + +// left reference +BaseRef& BaseRef::operator=(const BaseRef& other) { + if ((m_ptr != nullptr && m_ptr == other.m_ptr) || this == &other) { + return *this; + } + m_ptr = other.copy(); + return *this; +} + +// right reference +BaseRef& BaseRef::operator=(BaseRef&& other) { + if ((m_ptr != nullptr && m_ptr == other.m_ptr) || this == &other) { + return *this; + } + m_ptr = other.copy(); + other.m_ptr = nullptr; + return *this; +} + +std::string BaseRef::ToString() const { + if (m_ptr != nullptr) { + return std::string(m_ptr->type_name()) + std::string(" value:") + m_ptr->ToString(); + } + return std::string(); +} + +uint32_t BaseRef::type() const { + if (m_ptr != nullptr) { + return m_ptr->tid(); + } + return tid(); +} + +// left reference +SetRef& SetRef::operator=(const SetRef& other) { + if (elements_ == other.elements_ || this == &other) { + return *this; + } + elements_ = other.elements_; + return *this; +} + +std::string SetRef::ToString() const { + std::ostringstream buffer; + bool begin = true; + buffer << "set["; + for (auto& attr : elements_) { + if (!begin) { + buffer << ", "; + } else { + begin = false; + } + buffer << attr.ToString(); + } + buffer << "]"; + return buffer.str(); +} + +// left reference +VectorRef& VectorRef::operator=(const VectorRef& other) { + if (elements_ == other.elements_ || this == &other) { + return *this; + } + elements_ = other.elements_; + return *this; +} + +std::string VectorRef::ToString() const { + std::ostringstream buffer; + bool begin = true; + buffer << "vector["; + for (auto& attr : elements_) { + if (!begin) { + buffer << ", "; + } else { + begin = false; + } + buffer << attr.ToString(); + } + buffer << "]"; + return buffer.str(); +} + +bool VectorRef::operator==(const BaseRef& other) const { + if (!utils::isa(other)) { + return false; + } + return *this == utils::cast(other); +} + +bool VectorRef::operator==(const VectorRef& other) const { + if (elements_.size() != other.elements_.size()) { + return false; + } + for (size_t i = 0; i < elements_.size(); ++i) { + if (elements_[i] != other.elements_[i]) { + return false; + } + } + return true; +} + +bool SetRef::operator==(const BaseRef& other) const { + if (!utils::isa(other)) { + return false; + } + return *this == utils::cast(other); +} + +bool SetRef::operator==(const SetRef& other) const { + if (elements_.size() != other.elements_.size()) { + return false; + } + auto iter = elements_.begin(); + auto oth_iter = other.elements_.begin(); + for (; iter != elements_.end(); iter++, oth_iter++) { + if (*iter != *oth_iter) { + return false; + } + } + return true; +} + +bool RunFunctionRef::operator==(const BaseRef& other) const { + if (!utils::isa(other)) { + return false; + } + return *this == utils::cast(other); +} + +bool RunFunctionRef::operator==(const RunFunctionRef& other) const { return func_ == other.func_; } + +bool PyObjectRef::operator==(const BaseRef& other) const { + if (!utils::isa(other)) { + return false; + } + return *this == utils::cast(other); +} + +bool PyObjectRef::operator==(const PyObjectRef& other) const { return object_ == other.object_; } +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/base_ref.h b/mindspore/ccsrc/utils/base_ref.h new file mode 100644 index 00000000000..ed00d8280c2 --- /dev/null +++ b/mindspore/ccsrc/utils/base_ref.h @@ -0,0 +1,394 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_UTILS_BASE_REF_H_ +#define MINDSPORE_CCSRC_UTILS_BASE_REF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/value.h" + +namespace mindspore { +class BaseRef; +class VectorRef; +class SetRef; +class PyObjectRef; +class RunFunctionRef; + +using iterator = std::vector::iterator; + +using const_iterator = std::vector::const_iterator; +using const_reverse_iterator = std::vector::const_reverse_iterator; + +using RunFunc = std::function; +using RunFuncPtr = std::shared_ptr; + +template +using remove_reference_t = typename std::remove_reference::type; +template +using remove_const_t = typename std::remove_const::type; +template +using is_base = std::is_base_of>; +template +using is_value = std::is_base_of>; +template +using is_base_ref = std::is_base_of>; + +iterator ConstIteratorCast(std::vector* v, const_iterator iter); + +inline std::shared_ptr MakeNode(const std::vector& elements) { + return std::make_shared(elements); +} + +inline std::shared_ptr MakeNode(std::initializer_list elements) { + return std::make_shared(elements); +} + +// Anfnode, Funcgraph and some not value node class +template >::value && is_base::value, + int>::type = 0> +inline BasePtr MakeNode(const T& v) { + return v; +} + +template >::value && !is_base_ref::value, int>::type = 0> +inline BasePtr MakeNode(const T& v) { + return MakeValue(v); +} + +inline std::shared_ptr MakeNode(const VectorRef& a) { return std::make_shared(std::move(a)); } +inline std::shared_ptr MakeNode(const AnfNodePtrList& a) { + std::vector ret; + (void)std::transform(a.begin(), a.end(), std::back_inserter(ret), [](const AnfNodePtr& v) { return v; }); + return std::make_shared(ret); +} +inline std::shared_ptr MakeNode(const SetRef& a) { return std::make_shared(std::move(a)); } +inline std::shared_ptr MakeNode(const RunFuncPtr& a) { return std::make_shared(a); } +inline std::shared_ptr MakeNode(const py::object& a) { return std::make_shared(a); } +inline std::shared_ptr MakeNode(const py::tuple& a) { return std::make_shared(a); } + +class BaseRef : public Base { + public: + BaseRef() : m_ptr(nullptr) {} + BaseRef(const BaseRef& other); + virtual std::shared_ptr copy() const { return m_ptr; } + + BaseRef(BaseRef&& other) : Base(other) { + m_ptr = other.m_ptr; + other.m_ptr = nullptr; + } + + // right reference constructor + template ::type, BaseRef>::value, T>::type> + BaseRef(T&& t) { // NOLINT + m_ptr = MakeNode(t); + } + + ~BaseRef() override { m_ptr = nullptr; } + + MS_DECLARE_PARENT(BaseRef, Base) + + bool operator!=(const BaseRef& other) const { return !(operator==(other)); } + + virtual bool operator==(const BaseRef& other) const; + + // left reference + virtual BaseRef& operator=(const BaseRef& other); + // right reference + virtual BaseRef& operator=(BaseRef&& other); + + std::size_t hash() const override { + if (m_ptr == nullptr) { + MS_LOG(ERROR) << "Invalid m_ptr"; + return 0; + } + return m_ptr->hash(); + } + + std::string ToString() const override; + + bool is_null() const { return m_ptr == nullptr; } + + virtual uint32_t type() const; + + BasePtr m_ptr; // point to real data +}; +using BaseRefPtr = std::shared_ptr; + +struct BaseRefHash { + std::size_t operator()(const BaseRef& c) const { return c.hash(); } +}; + +struct BaseRefLess { + bool operator()(const BaseRef& a, const BaseRef& b) const { return a.hash() < b.hash(); } +}; + +namespace utils { +// judge isa relation +// examples: isa(handle), isa(handle) +template ::value && !is_base_ref::value, int>::type = 0> +bool isa(const BaseRef& handle) { + if (!handle.m_ptr) { + return false; + } + return handle.m_ptr->isa(); +} + +// noderef isa ptr isa(x) or isa() +template ::value, typename T::element_type>::type, + typename std::enable_if::value || is_base_ref::value, int>::type = 0> +bool isa(const BaseRef& handle) { + if (handle.m_ptr == nullptr) { + return typeid(handle.m_ptr) == typeid(T); + } + + if (handle.m_ptr->isa()) { + return true; + } + + // constptr isa can be true + return std::dynamic_pointer_cast(handle.m_ptr) != nullptr; +} + +// isa(handle) +template ::type::element_type> +bool isa(const BaseRef& handle) { + if (handle.m_ptr == nullptr) { + return false; + } + return handle.m_ptr->isa(); +} + +// isa(handle), judge reference or ptr +template ::value, int>::type = 0> +bool isa(const BaseRef& handle) { + static const uint32_t tid = Base::GetTypeId(typeid(T).name()); + return handle.IsFromTypeId(tid) || (handle.m_ptr && handle.m_ptr->isa()); +} + +// valueref -> C++ type +// cast(handle) +template ::value && !is_shared_ptr::value, int>::type = 0> +T cast(const BaseRef& handle) { + T ret = GetValue(std::static_pointer_cast(handle.m_ptr)); + return std::move(ret); +} + +// valueref -> valueref type +// cast(handle) +template ::value, int>::type = 0> +const T& cast(const BaseRef& handle) { + if (handle.m_ptr) { + return static_cast(*handle.m_ptr); + } + + return std::move(static_cast(handle)); +} + +// valueref -> nodeptr type +// cast(handle) +template ::value, typename T::element_type>::type, + typename std::enable_if::value && std::is_base_of::value, + int>::type = 0> +T cast(const BaseRef& handle) { + if (!handle.m_ptr) { + MS_LOG(EXCEPTION) << "Can not cast to " << typeid(T).name() << ", pointer is null"; + } + + auto m = handle.m_ptr->cast(); + if (nullptr != m) { + return m; + } + return std::static_pointer_cast(handle.m_ptr); +} +} // namespace utils + +class VectorRef : public BaseRef { + public: + VectorRef() {} + explicit VectorRef(const std::vector& elements) : elements_(elements) {} + VectorRef(const const_iterator& begin, const const_iterator& end) : elements_(begin, end) {} + + // left reference + virtual VectorRef& operator=(const VectorRef& other); + + ~VectorRef() override = default; + + std::shared_ptr copy() const override { return std::make_shared(elements_); } + + bool empty() const { return (elements_.size() == 0); } + + std::size_t size() const { return elements_.size(); } + MS_DECLARE_PARENT(VectorRef, BaseRef) + + const BaseRef& operator[](const std::size_t& dim) const { + if (dim >= size()) { + MS_LOG(EXCEPTION) << "Out of the size of the tuple."; + } + return elements_[dim]; + } + + uint32_t type() const override { return tid(); } + std::string ToString() const override; + std::vector& elements() { return elements_; } + void clear() { elements_.clear(); } + + bool operator==(const BaseRef& other) const override; + bool operator==(const VectorRef& other) const; + + void push_back(const BaseRef& value) { elements_.push_back(value); } + void push_back(BaseRef&& value) { elements_.push_back(value); } + + void emplace_back(const BaseRef& value) { elements_.emplace_back(value); } + void emplace_back(BaseRef&& value) { elements_.emplace_back(value); } + + template + void insert(const iterator pos, const InputIt first, const InputIt last) { + (void)elements_.insert(pos, first, last); + } + + template + void insert(const const_iterator cpos, const InputIt first, const InputIt last) { + auto pos = ConstIteratorCast(&elements_, cpos); + (void)elements_.insert(pos, first, last); + } + + const_iterator begin() const { return elements_.begin(); } + const_iterator end() const { return elements_.end(); } + + const_reverse_iterator rbegin() const { return elements_.rbegin(); } + const_reverse_iterator rend() const { return elements_.rend(); } + + iterator erase(const const_iterator cpos) { + auto pos = ConstIteratorCast(&elements_, cpos); + return elements_.erase(pos); + } + + iterator erase(const const_iterator cfirst, const const_iterator clast) { + auto first = ConstIteratorCast(&elements_, cfirst); + auto last = ConstIteratorCast(&elements_, clast); + return elements_.erase(first, last); + } + + std::size_t hash() const override { + std::stringstream buffer; + buffer << ToString(); + return std::hash()(buffer.str()); + } + + std::vector elements_; +}; + +using VectorRefPtr = std::shared_ptr; + +using set_iterator = std::set::iterator; +using const_set_iterator = std::set::const_iterator; + +struct VectorRefHash { + std::size_t operator()(const VectorRef& c) const { return c.hash(); } +}; + +class SetRef : public BaseRef { + public: + SetRef() {} + explicit SetRef(const std::set& elements) : elements_(elements) {} + SetRef(const std::initializer_list elements) : elements_(elements.begin(), elements.end()) {} + SetRef(const const_set_iterator& begin, const const_set_iterator& end) : elements_(begin, end) {} + + // left reference + virtual SetRef& operator=(const SetRef& other); + + bool operator==(const BaseRef& other) const override; + bool operator==(const SetRef& other) const; + + ~SetRef() override = default; + + std::shared_ptr copy() const override { return std::make_shared(elements_); } + + bool empty() const { return (elements_.size() == 0); } + + std::size_t size() const { return elements_.size(); } + MS_DECLARE_PARENT(SetRef, BaseRef) + + uint32_t type() const override { return tid(); } + std::string ToString() const override; + std::set& elements() { return elements_; } + void clear() { elements_.clear(); } + + void insert(const BaseRef& elem) { (void)elements_.insert(elem); } + + const_set_iterator begin() const { return elements_.begin(); } + const_set_iterator end() const { return elements_.end(); } + + template + void insert(const InputIt first, const InputIt last) { + (void)elements_.insert(first, last); + } + + std::size_t count(const BaseRef& elem) const { return elements_.count(elem); } + const_set_iterator find(const BaseRef& elem) const { return elements_.find(elem); } + + std::set elements_; +}; + +using SetRefPtr = std::shared_ptr; + +class PyObjectRef : public BaseRef { + public: + explicit PyObjectRef(const py::object& py_object) : object_(py_object) {} + explicit PyObjectRef(const py::tuple& tuple_obj) : object_(tuple_obj) {} + + ~PyObjectRef() override = default; + + std::shared_ptr copy() const override { return std::make_shared(object_); } + MS_DECLARE_PARENT(PyObjectRef, BaseRef) + + uint32_t type() const override { return tid(); } + std::string ToString() const override { return py::str(object_); } + bool operator==(const BaseRef& other) const override; + bool operator==(const PyObjectRef& other) const; + + py::object object_; +}; + +class RunFunctionRef : public BaseRef { + public: + RunFunctionRef() {} + explicit RunFunctionRef(const RunFuncPtr& ref_func) : func_(ref_func) {} + + ~RunFunctionRef() override = default; + MS_DECLARE_PARENT(RunFunctionRef, BaseRef) + + uint32_t type() const override { return tid(); } + std::string ToString() const override { return std::string("RunFunctionRef"); } + bool operator==(const BaseRef& other) const override; + bool operator==(const RunFunctionRef& other) const; + + RunFuncPtr func_; +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_NODE_REF_H_ diff --git a/mindspore/ccsrc/utils/callbacks.cc b/mindspore/ccsrc/utils/callbacks.cc new file mode 100644 index 00000000000..ab3f7d883b7 --- /dev/null +++ b/mindspore/ccsrc/utils/callbacks.cc @@ -0,0 +1,217 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/callbacks.h" +#include +#include +#include +#include +#include "pybind11/pybind11.h" +#include "transform/df_graph_manager.h" +#include "transform/util.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/parse/python_adapter.h" +#include "utils/visible.h" + +namespace mindspore { +namespace callbacks { +using mindspore::transform::Status; +using mindspore::transform::TransformUtil; + +const char PYTHON_MOD_CALLBACK_MODULE[] = "mindspore.train.callback"; +const char PYTHON_FUN_PROCESS_CHECKPOINT[] = "_checkpoint_cb_for_save_op"; +const char PYTHON_FUN_PROCESS_SUMMARY[] = "_summary_cb_for_save_op"; +const char kSummary[] = "Summary"; +const char kCheckPoint[] = "Save"; +const int ONE_SHAPE = 1; + +bool GetParameterShape(const FuncGraphPtr& graph, const std::string& param_name, + const std::shared_ptr>& shape) { + if (graph == nullptr) { + MS_LOG(ERROR) << "Graph is null, can not get graph parameter"; + return false; + } + + auto parameter_nodes = graph->parameters(); + for (auto& node : parameter_nodes) { + ParameterPtr param_node = std::static_pointer_cast(node); + if (param_node == nullptr) { + MS_LOG(ERROR) << "Parameter node is null, can not get graph parameter"; + return false; + } + if (param_node->name() == param_name) { + py::object parameter = param_node->default_param(); + ValuePtr value = parse::data_converter::PyDataToValue(parameter); + TensorPtr tensor = std::dynamic_pointer_cast(value); + if (tensor == nullptr) { + shape->push_back(ONE_SHAPE); + } else { + *shape = tensor->shape(); + } + return true; + } + } + MS_LOG(ERROR) << "Can not find parameter of name:" << param_name; + return false; +} + +static TensorPtr GetMeTensorTransformed(uint32_t graph_id, const std::string& parameter_name, + const std::shared_ptr& ge_tensor_ptr) { + FuncGraphPtr anf_graph = transform::DfGraphManager::GetInstance().GetAnfGraph(graph_id); + if (anf_graph == nullptr) { + MS_LOG(ERROR) << "Get anf graph failed during callback"; + return nullptr; + } + + std::shared_ptr> parameter_shape_ptr = std::make_shared>(); + if (!GetParameterShape(anf_graph, parameter_name, parameter_shape_ptr)) { + MS_LOG(ERROR) << "Can not get parameter shape during callback"; + return nullptr; + } + + return TransformUtil::ConvertGeTensor(ge_tensor_ptr, *parameter_shape_ptr); +} + +uint32_t CheckpointSaveCallback(uint32_t graph_id, const std::map& params_list) { + // Acquire GIL before calling Python code + py::gil_scoped_acquire acquire; + + MS_LOG(DEBUG) << "Start the checkpoint save callback function in checkpoint save process."; + py::list parameter_list = py::list(); + for (auto& item : params_list) { + std::string name = item.first; + std::shared_ptr ge_tensor_ptr = std::make_shared(item.second); + TensorPtr tensor_ptr = GetMeTensorTransformed(graph_id, name, ge_tensor_ptr); + if (tensor_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Transform ge tensor to me tensor failed"; + } + py::dict param_dict; + param_dict["name"] = name; + param_dict["data"] = tensor_ptr; + parameter_list.append(param_dict); + } + py::bool_ ret = + parse::python_adapter::CallPyFn(PYTHON_MOD_CALLBACK_MODULE, PYTHON_FUN_PROCESS_CHECKPOINT, parameter_list); + auto bool_ret = py::cast(ret); + + uint32_t status = Status::SUCCESS; + if (!bool_ret) { + status = Status::FAILED; + MS_LOG(ERROR) << "python checkpoint return false during callback"; + } + return status; +} + +static TensorPtr GetMeTensorForSummary(const std::string& name, const std::shared_ptr& ge_tensor_ptr) { + // confirm the type by name + // Format: xxx[:Scalar] xxx[:Image] xxx[:Tensor] + if (name.empty()) { + MS_LOG(EXCEPTION) << "The summary name is empty."; + } + auto bpos = name.rfind("[:"); + if (bpos >= name.size()) { + MS_LOG(EXCEPTION) << "The summary name(" << name << ") is invalid."; + } + auto tname = name.substr(bpos); + if (tname == "[:Scalar]") { + MS_LOG(DEBUG) << "The summary(" << name << ") is Scalar"; + // process the scalar type summary + // Because the ge tensor is dim = 4, so set the (1,1,1,1)-->(1,) + // We do the (1,) shape is scalar + auto shape = std::vector({ONE_SHAPE}); + return TransformUtil::ConvertGeTensor(ge_tensor_ptr, shape); + } + if (tname == "[:Tensor]") { + MS_LOG(DEBUG) << "The summary(" << name << ") is Tensor"; + // process the tensor summary + // Now we can't get the real shape, so we keep same shape with GE + return TransformUtil::ConvertGeTensor(ge_tensor_ptr); + } + if (tname == "[:Image]") { + MS_LOG(DEBUG) << "The summary(" << name << ") is Image"; + // process the Image summary + // Image dim = 4, is same with ge, so we keep same shape with GE + return TransformUtil::ConvertGeTensor(ge_tensor_ptr); + } + + MS_LOG(EXCEPTION) << "The summary name(" << name << ") is invalid."; +} + +// Cache the summary callback data +// Output Format: [{"name": tag_name, "data": tensor}, {"name": tag_name, "data": tensor},...] +uint32_t MS_EXPORT SummarySaveCallback(uint32_t graph_id, const std::map& params_list) { + // Acquire GIL before calling Python code + py::gil_scoped_acquire acquire; + + MS_LOG(DEBUG) << "Start the summary save callback function for graph " << graph_id << "."; + py::list summary_list = py::list(); + MS_LOG(DEBUG) << "Param list size = " << params_list.size(); + for (auto& item : params_list) { + std::string tag_name = item.first; + std::shared_ptr ge_tensor_ptr = std::make_shared(item.second); + TensorPtr tensor_ptr = GetMeTensorForSummary(tag_name, ge_tensor_ptr); + if (tensor_ptr == nullptr) { + MS_LOG(EXCEPTION) << "ConvertGeTensor return tensor is null"; + } + py::dict summary_value_dict; + summary_value_dict["name"] = tag_name; + summary_value_dict["data"] = tensor_ptr; + summary_list.append(summary_value_dict); + } + + py::bool_ ret = parse::python_adapter::CallPyFn(PYTHON_MOD_CALLBACK_MODULE, PYTHON_FUN_PROCESS_SUMMARY, summary_list); + auto bool_ret = py::cast(ret); + if (!bool_ret) { + MS_LOG(ERROR) << "Python checkpoint return false during callback"; + return Status::FAILED; + } + MS_LOG(DEBUG) << "End the summary save callback function."; + return Status::SUCCESS; +} + +// Cache the summary callback data from ME session +// Remove the GE module on new architecture +// Output Format: [{"name": tag_name, "data": tensor}, {"name": tag_name, "data": tensor},...] +uint32_t MS_EXPORT SummarySaveCallback(uint32_t graph_id, const std::map& params_list) { + // Acquire GIL before calling Python code + py::gil_scoped_acquire acquire; + py::list summary_list = py::list(); + + MS_LOG(INFO) << "The Summary save callback function for graph " << graph_id + << ", Param list size = " << params_list.size() << "."; + for (auto& item : params_list) { + std::string tag_name = item.first; + auto tensor_ptr = item.second; + if (tensor_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Summary tensor is null"; + } + py::dict summary_value_dict; + summary_value_dict["name"] = tag_name; + summary_value_dict["data"] = tensor_ptr; + summary_list.append(summary_value_dict); + } + + py::bool_ ret = parse::python_adapter::CallPyFn(PYTHON_MOD_CALLBACK_MODULE, PYTHON_FUN_PROCESS_SUMMARY, summary_list); + auto bool_ret = py::cast(ret); + if (!bool_ret) { + MS_LOG(ERROR) << "Python checkpoint return false during callback"; + return Status::FAILED; + } + MS_LOG(DEBUG) << "End the summary save callback function."; + return Status::SUCCESS; +} +} // namespace callbacks +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/callbacks.h b/mindspore/ccsrc/utils/callbacks.h new file mode 100644 index 00000000000..2a18b21b167 --- /dev/null +++ b/mindspore/ccsrc/utils/callbacks.h @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_UTILS_CALLBACKS_H_ +#define MINDSPORE_CCSRC_UTILS_CALLBACKS_H_ + +#include +#include +#include +#include +#include "transform/types.h" +#include "transform/util.h" + +namespace mindspore { +namespace callbacks { + +using mindspore::tensor::TensorPtr; +// define the python callback module and function +extern const char PYTHON_MOD_CALLBACK_MODULE[]; +extern const char PYTHON_FUN_PROCESS_CHECKPOINT[]; +extern const char PYTHON_FUN_PROCESS_SUMMARY[]; + +extern const char kSummary[]; +extern const char kCheckPoint[]; +extern const std::string kPythonCheckpointModuleName; +extern const std::string kPythonCheckpointFuncName; +bool GetParameterShape(const FuncGraphPtr& anf_graph, const std::string& param_name, + const std::shared_ptr>& shape); +uint32_t CheckpointSaveCallback(uint32_t, const std::map&); +uint32_t SummarySaveCallback(uint32_t, const std::map&); +uint32_t SummarySaveCallback(uint32_t, const std::map&); + +} // namespace callbacks +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_CALLBACKS_H_ diff --git a/mindspore/ccsrc/utils/checkpoint.proto b/mindspore/ccsrc/utils/checkpoint.proto new file mode 100644 index 00000000000..31c7cd8004f --- /dev/null +++ b/mindspore/ccsrc/utils/checkpoint.proto @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +message Checkpoint { + message Value { + required string tag = 1; + required TensorProto tensor = 2; + } + repeated Value value = 1; +} + + +message TensorProto { + // The shape of the tensor. + repeated int64 dims = 1; + // The type of the tensor. + required string tensor_type = 2; + // The data of the tensor. + required bytes tensor_content = 3; +} diff --git a/mindspore/ccsrc/utils/comm_manager.cc b/mindspore/ccsrc/utils/comm_manager.cc new file mode 100644 index 00000000000..70adfb7467f --- /dev/null +++ b/mindspore/ccsrc/utils/comm_manager.cc @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/comm_manager.h" +#include "utils/convert_utils.h" +#ifndef NO_DLIB +#include "hccl/hcom.h" +#endif + +namespace mindspore { +CommManager &CommManager::GetInstance() noexcept { + static CommManager instance("hccl"); + return instance; +} + +#ifndef NO_DLIB +#define HCCL_RUN_CHECK(op_name, group, op) \ + do { \ + auto hccl_result = (op); \ + if (hccl_result != tagHcclResult::HCCL_SUCCESS) { \ + MS_LOG(ERROR) << op_name << " failed: #" << group << "#"; \ + return false; \ + } \ + } while (0) + +#define HCCL_GROUP_CHECK_EMPTY(group) \ + do { \ + if (group.length() == 0) { \ + MS_LOG(ERROR) << "The length of group name should not be 0"; \ + return false; \ + } \ + } while (0) + +#define HCCL_GROUP_CHECK_IS_WORLD(group) \ + do { \ + if (group == "hccl_world_group") { \ + MS_LOG(ERROR) << "The group name should not be hccl_world_group"; \ + return false; \ + } \ + } while (0) + +bool CommManager::CreateGroupSync(const string &group, const vector &rank_id_list) const { + auto rank_size = rank_id_list.size(); + HCCL_GROUP_CHECK_EMPTY(group); + HCCL_GROUP_CHECK_IS_WORLD(group); + HCCL_RUN_CHECK(string("create communicate group"), group, + hcom_create_group(group.c_str(), UlongToUint(rank_size), vector(rank_id_list).data())); + return true; +} + +bool CommManager::GetRankID(const string &group, unsigned int *rank_id) const { + HCCL_GROUP_CHECK_EMPTY(group); + HCCL_RUN_CHECK(string("get rank_id"), group, hcom_get_rank_id(group.c_str(), rank_id)); + return true; +} + +bool CommManager::GetRankSize(const string &group, unsigned int *rank_size) const { + HCCL_GROUP_CHECK_EMPTY(group); + HCCL_RUN_CHECK(string("get rank size"), group, hcom_get_rank_size(group.c_str(), rank_size)); + return true; +} + +bool CommManager::DestroyGroup(const string &group) const { + HCCL_GROUP_CHECK_EMPTY(group); + HCCL_GROUP_CHECK_IS_WORLD(group); + HCCL_RUN_CHECK(string("destroy communicate group"), group, hcom_destroy_group(group.c_str())); + return true; +} +#else +bool CommManager::CreateGroupSync(const string &, const vector &) const { return true; } + +bool CommManager::GetRankID(const string &group, unsigned int *rank_id) const { return true; } + +bool CommManager::GetRankSize(const string &group, unsigned int *rank_size) const { + *rank_size = NO_COMM_DLIB_RANK_SIZE; + return true; +} + +bool CommManager::DestroyGroup(const string &group) const { return true; } +#endif +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/comm_manager.h b/mindspore/ccsrc/utils/comm_manager.h new file mode 100644 index 00000000000..002d0c35e9c --- /dev/null +++ b/mindspore/ccsrc/utils/comm_manager.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_COMM_MANAGER_H +#define MINDSPORE_CCSRC_UTILS_COMM_MANAGER_H + +#include +#include +#include +#include "utils/log_adapter.h" + +using std::string; +using std::vector; + +namespace mindspore { +constexpr unsigned int NO_COMM_DLIB_RANK_SIZE = 2048; + +class CommManager { + public: + static CommManager &GetInstance() noexcept; + bool CreateGroupSync(const string &group, const vector &rank_id_list) const; + bool DestroyGroup(const string &group) const; + bool GetRankID(const string &group, unsigned int *rank_id) const; + bool GetRankSize(const string &group, unsigned int *rank_size) const; + ~CommManager() = default; + + CommManager(const CommManager &) = delete; + + private: + explicit CommManager(string backend) : backend_(std::move(backend)) {} + string backend_; +}; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_COMMUNICATION_MANAGER_H diff --git a/mindspore/ccsrc/utils/config_manager.cc b/mindspore/ccsrc/utils/config_manager.cc new file mode 100644 index 00000000000..ac8a965878d --- /dev/null +++ b/mindspore/ccsrc/utils/config_manager.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/config_manager.h" + +#include + +#include "utils/log_adapter.h" + +namespace mindspore { + +ConfigManager& ConfigManager::GetInstance() noexcept { + static ConfigManager instance; + return instance; +} + +void ConfigManager::SetDatasetModeConfig(const std::string& mode) { + static const std::map mode_map = {{"feed", DS_FEED_MODE}, {"graph", DS_GRAPH_MODE}}; + if (mode_map.find(mode) == mode_map.end()) { + MS_LOG(ERROR) << "Invalid dataset mode:" << mode; + return; + } + GetInstance().dataset_mode_ = mode_map.at(mode); +} + +void ConfigManager::ResetConfig() noexcept { + parallel_strategy_ = ONE_DEVICE; + dataset_mode_ = DS_FEED_MODE; + dataset_param_ = DatasetGraphParam("", 0, 0, {}, {}, {}); + iter_num_ = 1; +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/config_manager.h b/mindspore/ccsrc/utils/config_manager.h new file mode 100644 index 00000000000..31137f62439 --- /dev/null +++ b/mindspore/ccsrc/utils/config_manager.h @@ -0,0 +1,117 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_CONFIG_MANAGER_H_ +#define MINDSPORE_CCSRC_UTILS_CONFIG_MANAGER_H_ + +#include +#include +#include +#include +#include +#include + +#include "utils/overload.h" + +namespace mindspore { + +enum ParallelStrategy { + ONE_DEVICE = 0, + DISTRIBUTION, +}; + +enum DatasetMode { DS_FEED_MODE = 0, DS_GRAPH_MODE }; + +class DatasetGraphParam { + public: + DatasetGraphParam(const std::string& name, int64_t size, int64_t batch_size, const std::vector& ge_types, + const std::vector>& shapes, const std::vector& input_indexes) + : queue_name_(name), + loop_size_(size), + batch_size_(batch_size), + ge_types_(ge_types), + shapes_(shapes), + input_indexes_(input_indexes) {} + + ~DatasetGraphParam() = default; + + std::string ToString() const { + std::ostringstream buffer; + buffer << "DatasetGraphParam: queue_name=" << queue_name_ << " size=" << loop_size_ << " batch_size=" << batch_size_ + << " ge_types=" << ge_types_ << " shapes=" << shapes_ << " input_indexes=" << input_indexes_; + return buffer.str(); + } + std::string queue_name() const { return queue_name_; } + int64_t loop_size() const { return loop_size_; } + int64_t batch_size() const { return batch_size_; } + std::vector ge_types() const { return ge_types_; } + std::vector> shapes() const { return shapes_; } + std::vector input_indexes() const { return input_indexes_; } + + private: + std::string queue_name_; + int64_t loop_size_; + int64_t batch_size_; + std::vector ge_types_; + std::vector> shapes_; + std::vector input_indexes_; +}; + +class ConfigManager { + public: + ConfigManager(const ConfigManager&) = delete; + ConfigManager& operator=(const ConfigManager&) = delete; + static ConfigManager& GetInstance() noexcept; + + ParallelStrategy parallel_strategy() const { return parallel_strategy_; } + void set_parallel_strategy(ParallelStrategy strategy) { parallel_strategy_ = strategy; } + + const std::map& ge_initialize_options() const { return ge_initialize_options_; } + void set_ge_initialize_options(const std::map& options) { + ge_initialize_options_ = options; + } + + DatasetMode dataset_mode() const { return dataset_mode_; } + void set_dataset_mode(DatasetMode mode) { dataset_mode_ = mode; } + int64_t iter_num() const { return iter_num_; } + void set_iter_num(const int64_t num) { iter_num_ = num; } + + std::string dataset_phase() const { return dataset_phase_; } + void set_dataset_phase(const std::string& phase) { dataset_phase_ = phase; } + + DatasetGraphParam dataset_param() const { return dataset_param_; } + void set_dataset_param(const DatasetGraphParam& param) { dataset_param_ = param; } + + static void SetDatasetModeConfig(const std::string& mode); + + void ResetConfig() noexcept; + + std::map ge_initialize_options_; + + private: + ConfigManager() = default; + ~ConfigManager() = default; + + ParallelStrategy parallel_strategy_{ONE_DEVICE}; + DatasetMode dataset_mode_{DS_FEED_MODE}; + DatasetGraphParam dataset_param_{"", 0, 0, {}, {}, {}}; + int64_t iter_num_{1}; + std::string dataset_phase_{""}; +}; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_CONFIG_MANAGER_H_ diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc new file mode 100644 index 00000000000..bf05af98582 --- /dev/null +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -0,0 +1,436 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/context/ms_context.h" +#include +#include +#include +#include "./common.h" +#include "utils/convert_utils.h" +#include "utils/tensorprint_utils.h" +#ifndef NO_DLIB +#include "tdt/tsd_client.h" +#include "tdt/tdt_host_interface.h" +#include "tdt/data_common.h" +#endif +#include "transform/df_graph_manager.h" +#include "ir/meta_tensor.h" + +namespace mindspore { +using mindspore::transform::DfGraphManager; +using transform::GraphRunner; +using transform::GraphRunnerOptions; + +std::atomic thread_1_must_end(false); + +std::shared_ptr MsContext::inst_context_ = nullptr; +std::map MsContext::policy_map_ = {{"ge", kMsBackendGePrior}, + {"vm", kMsBackendVmOnly}, + {"ms", kMsBackendMsPrior}, + {"ge_only", kMsBackendGeOnly}, + {"vm_prior", kMsBackendVmPrior}}; + +MsContext::MsContext(const std::string& policy, const std::string& target) { + save_graphs_flag_ = false; + save_graphs_path_ = "."; + save_ms_model_flag_ = false; + save_ms_model_path_ = "./model.ms"; + enable_dump_ = false; + save_dump_path_ = "."; + tsd_ref_ = 0; + ge_ref_ = 0; + is_multi_graph_sink_ = false; + is_pynative_ge_init_ = false; + enable_reduce_precision_ = true; + auto env_device = common::GetEnv("DEVICE_ID"); + if (!env_device.empty()) { + device_id_ = UlongToUint(std::stoul(env_device.c_str())); + } else { + device_id_ = 0; + } + backend_policy_ = policy_map_[policy]; + device_target_ = target; + execution_mode_ = kGraphMode; + enable_task_sink_ = true; + ir_fusion_flag_ = true; + enable_hccl_ = false; + enable_loop_sink_ = false; + enable_mem_reuse_ = true; + enable_gpu_summary_ = true; + precompile_only_ = false; + auto_mixed_precision_flag_ = true; + enable_pynative_infer_ = false; + enable_dynamic_mem_pool_ = false; + graph_memory_max_size_ = "0"; + variable_memory_max_size_ = "0"; + MS_LOG(INFO) << "Create context with backend policy:" << policy << ", device target:" << target << "."; +} + +std::shared_ptr MsContext::GetInstance() { + if (inst_context_ == nullptr) { +#ifdef ENABLE_GE + inst_context_.reset(new (std::nothrow) MsContext("ge", kAscendDevice)); +#elif defined(ENABLE_D) + inst_context_.reset(new (std::nothrow) MsContext("ms", kAscendDevice)); +#elif defined(ENABLE_GPU) + inst_context_.reset(new (std::nothrow) MsContext("ms", kGPUDevice)); +#else + inst_context_.reset(new (std::nothrow) MsContext("vm", kCPUDevice)); +#endif + } + return inst_context_; +} + +bool MsContext::set_backend_policy(const std::string& policy) { + if (policy_map_.find(policy) == policy_map_.end()) { + MS_LOG(ERROR) << "invalid backend policy name: " << policy; + return false; + } + backend_policy_ = policy_map_[policy]; + MS_LOG(INFO) << "ms set context backend policy:" << policy; + return true; +} + +std::string MsContext::backend_policy() const { + auto res = std::find_if( + policy_map_.begin(), policy_map_.end(), + [&, this](const std::pair& item) { return item.second == backend_policy_; }); + if (res != policy_map_.end()) { + return res->first; + } + return "unknown"; +} + +void MsContext::set_execution_mode(int execution_mode) { + if (execution_mode != kGraphMode && execution_mode != kPynativeMode) { + MS_LOG(EXCEPTION) << "The execution mode is invalid!"; + } + execution_mode_ = execution_mode; +} + +bool MsContext::set_device_target(const std::string& target) { + if (kTargetSet.find(target) == kTargetSet.end()) { + MS_LOG(ERROR) << "invalid device target name: " << target; + return false; + } + if (target == kDavinciDevice) { + device_target_ = kAscendDevice; + } else { + device_target_ = target; + } + MS_LOG(INFO) << "ms set context device target:" << target; + return true; +} + +bool MsContext::set_device_id(uint32_t device_id) { + device_id_ = device_id; + MS_LOG(INFO) << "ms set context device id:" << device_id; + return true; +} + +#ifndef NO_DLIB +// Open tdt dataset +bool MsContext::OpenTsd() { + if (is_pynative_ge_init_) { + return true; + } + + if (tsd_ref_) { + MS_LOG(DEBUG) << "TDT Dataset client is already opened."; + tsd_ref_++; + return true; + } + + unsigned int device_id; + unsigned int rank_size = 1; + + device_id = device_id_; + + auto rank_size_env = common::GetEnv("RANK_SIZE"); + if (rank_size_env.empty()) { + MS_LOG(INFO) << "Should config rank size."; + rank_size = 1; + } else { + int rank_env = std::stoi(rank_size_env); + if (rank_env <= 0) { + MS_LOG(EXCEPTION) << "Error rank size " << rank_env << "."; + } + rank_size = IntToUint(rank_env); + } + + MS_LOG(INFO) << "Device id = " << device_id << ", rank size = " << rank_size << "."; + + TDT_StatusT status = tdt::TsdClient::GetInstance()->Open(device_id, rank_size); + if (status != TDT_OK) { + MS_LOG(EXCEPTION) << "Device " << device_id << " is occupied, open tsd failed, status = " << status << "."; + return false; + } + tsd_ref_++; +#ifdef ENABLE_TDTQUE + int32_t initStatus = tdt::TdtHostInit(device_id); + if (initStatus != TDT_OK_CODE) { + MS_LOG(EXCEPTION) << "Init tsd failed, status = " << initStatus << "."; + return false; + } + tdt_print_ = std::thread(TensorPrint()); +#endif + MS_LOG(INFO) << "Open and init tsd successful, tsd reference = " << tsd_ref_ << "."; + return true; +} + +bool MsContext::CloseTsd(bool force) { + if (tsd_ref_ == 0) { + return true; + } + tsd_ref_--; + if (force || tsd_ref_ == 0) { + tsd_ref_ = 0; +#ifdef ENABLE_TDTQUE + int32_t stopStatus = tdt::TdtHostStop(KNpuLog); + if (stopStatus != TDT_OK_CODE) { + MS_LOG(EXCEPTION) << "Stop tsd failed, status = " << stopStatus << "."; + return false; + } + py::gil_scoped_release gil_release; + int32_t destroyStatus = tdt::TdtHostDestroy(); + if (destroyStatus != TDT_OK_CODE) { + MS_LOG(EXCEPTION) << "Destroy tsd failed, status = " << destroyStatus << "."; + return false; + } + try { + if (tdt_print_.joinable()) { + MS_LOG(INFO) << "join tdt host receive process"; + tdt_print_.join(); + } + } catch (const std::exception& e) { + MS_LOG(ERROR) << "tdt thread join failed: " << e.what(); + } +#endif + TDT_StatusT status = tdt::TsdClient::GetInstance()->Close(); + if (status != TDT_OK) { + MS_LOG(EXCEPTION) << "Close tsd failed, status = " << status << "."; + return false; + } + is_pynative_ge_init_ = false; + MS_LOG(INFO) << "Destroy and close tsd successful, status = " << status << "."; + } else { + MS_LOG(DEBUG) << "TDT Dataset client is used, no need to close, tsd reference = " << tsd_ref_ << "."; + } + + return true; +} +#else +bool MsContext::OpenTsd() { return true; } + +bool MsContext::CloseTsd(bool) { return true; } +#endif + +void MsContext::SetHcclOptions(std::map* ge_options) const { + auto env_table_file = common::GetEnv("RANK_TABLE_FILE"); + auto env_rank_id = common::GetEnv("RANK_ID"); + auto env_device_id = std::to_string(device_id_); + if (!(env_table_file.empty() || env_rank_id.empty())) { + MS_LOG(INFO) << "Initialize Ge for distribute parameter"; + MS_LOG(INFO) << "Use hccl, make sure hccl lib is set in OPTION_EXEC_EXTERN_PLUGIN_PATH."; + auto env_hccl_flag = common::GetEnv("HCCL_FLAG"); + if (!env_hccl_flag.empty()) { + (*ge_options)["ge.exec.hcclFlag"] = env_hccl_flag; + } + (*ge_options)["ge.exec.isUseHcom"] = "1"; + (*ge_options)["ge.exec.deviceId"] = env_device_id; + (*ge_options)["ge.exec.rankId"] = env_rank_id; + (*ge_options)["ge.exec.podName"] = env_rank_id; + (*ge_options)["ge.exec.rankTableFile"] = env_table_file; + (*ge_options)["ge.graphRunMode"] = "1"; + } else { + // device id is still needed for non-distribute case + (*ge_options)["ge.exec.deviceId"] = env_device_id; + MS_LOG(INFO) << "No hccl mode. " + "If use hccl, make sure [RANK_TABLE_FILE,RANK_ID,DEVICE_ID,DEPLOY_MODE] all be set in ENV."; + } + + auto env_deploy_mode = common::GetEnv("DEPLOY_MODE"); + if (!env_deploy_mode.empty()) { + (*ge_options)["ge.exec.deployMode"] = env_deploy_mode; + } else { + (*ge_options)["ge.exec.deployMode"] = "0"; + MS_LOG(WARNING) << "DEPLOY_MODE is not set in ENV. Now set to default value 0"; + } +} + +void MsContext::GetGeOptions(std::map* ge_options) const { +#ifdef ENABLE_GE + (*ge_options)["device_id"] = "0"; + (*ge_options)["ge.exec.enableDump"] = enable_dump_; + (*ge_options)["ge.exec.dumpPath"] = save_dump_path_; + // only not supported in ge + auto tbe_plugin_path = common::GetEnv("ME_TBE_PLUGIN_PATH"); + if (!tbe_plugin_path.empty()) { + char real_path[PATH_MAX] = {0}; + if (nullptr == realpath(tbe_plugin_path.c_str(), real_path)) { + MS_LOG(ERROR) << "Ms tbe plugin Path error, " << tbe_plugin_path; + } else { + tbe_plugin_path = real_path; + (*ge_options)["ge.TBE_plugin_path"] = tbe_plugin_path; + } + } else { + MS_LOG(ERROR) << "Set TBE plugin path failed!"; + } + (*ge_options)["rank_table_file"] = ""; + auto env_ddk_version = common::GetEnv("DDK_VERSION"); + if (!env_ddk_version.empty()) { + (*ge_options)["ge.DDK_version"] = env_ddk_version; + } else { + (*ge_options)["ge.DDK_version"] = "1.60.T17.B830"; + } + (*ge_options)["graphType"] = "1"; + + if (graph_memory_max_size_ != "0") { + (*ge_options)["ge.graphMemoryMaxSize"] = graph_memory_max_size_; + } + + if (variable_memory_max_size_ != "0") { + (*ge_options)["ge.variableMemoryMaxSize"] = variable_memory_max_size_; + } + +#if ENABLE_TRAIN == 1 + (*ge_options)["ge.graphRunMode"] = "1"; +#endif + SetDisableReuseMemoryFlag(ge_options); + SetHcclOptions(ge_options); + + auto env_job_id = common::GetEnv("JOB_ID"); + if (!env_job_id.empty()) { + (*ge_options)["ge.exec.jobId"] = env_job_id; + } else { + (*ge_options)["ge.exec.jobId"] = "0"; + MS_LOG(WARNING) << "JOB_ID is not set in ENV. Now set to default value 0"; + } + + auto env_fe_flag = common::GetEnv("FE_FLAG"); + if (!env_fe_flag.empty()) { + (*ge_options)["ge.feFlag"] = env_fe_flag; + MS_LOG(INFO) << "Use FE, make sure fe lib is set in OPTION_EXEC_EXTERN_PLUGIN_PATH."; + } + + auto env_aicpu_flag = common::GetEnv("AICPU_FLAG"); + if (!env_aicpu_flag.empty()) { + (*ge_options)["ge.aicpuFlag"] = env_aicpu_flag; + MS_LOG(INFO) << "Use AICPU, make sure aicpu lib is set in OPTION_EXEC_EXTERN_PLUGIN_PATH."; + } + + // all libs are set in same env variable "OPTION_EXEC_EXTERN_PLUGIN_PATH", such as FE, HCCL, AICPU, etc + auto load_path = common::GetEnv("OPTION_EXEC_EXTERN_PLUGIN_PATH"); + if (!load_path.empty()) { + char real_path[PATH_MAX] = {0}; + if (realpath(load_path.c_str(), real_path)) { + load_path = real_path; + (*ge_options)["ge.soLoadPath"] = load_path; + } + } else { + MS_LOG(ERROR) << "Set lib load path failed!"; + } + + auto proto_lib_path = common::GetEnv("OPTION_PROTO_LIB_PATH"); + if (!proto_lib_path.empty()) { + char real_path[PATH_MAX] = {0}; + if (realpath(proto_lib_path.c_str(), real_path)) { + proto_lib_path = real_path; + (*ge_options)["ge.opsProtoLibPath"] = proto_lib_path; + } + } else { + MS_LOG(ERROR) << "Set proto lib path failed!"; + } + + // Disbale the global variable acc, only enable it whlie adding training graph in pipeline + (*ge_options)["ge.exec.variable_acc"] = "0"; +#endif +} + +void MsContext::SetDisableReuseMemoryFlag(std::map* ge_options) const { + auto env_disable_reuse_memory = common::GetEnv("DISABLE_REUSE_MEMORY"); + if (!env_disable_reuse_memory.empty()) { + (*ge_options)["ge.exec.disableReuseMemory"] = env_disable_reuse_memory; + } else { + (*ge_options)["ge.exec.disableReuseMemory"] = "0"; + MS_LOG(WARNING) << "DISABLE_REUSE_MEMORY is not set in ENV. Now set to default value 0"; + } +} + +bool MsContext::InitGe() { +#ifdef ENABLE_GE + if (is_pynative_ge_init_) { + return true; + } + + if (ge_ref_) { + ge_ref_++; + return true; + } + + std::map ge_options; + GetGeOptions(&ge_options); + { + // Release GIL before calling into (potentially long-running) C++ code + py::gil_scoped_release release; + if (ge::GEInitialize(ge_options) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "Initialize GE failed!"; + } + } + ge_ref_++; + MS_LOG(INFO) << "Init ge successful, ge reference = " << ge_ref_ << "."; +#endif + return true; +} + +bool MsContext::FinalizeGe(bool force) { +#ifdef ENABLE_GE + if (ge_ref_ == 0) { + return true; + } + ge_ref_--; + if (force || ge_ref_ == 0) { + ge_ref_ = 0; + try { + DfGraphManager::GetInstance().DeleteGraphRunner(); + DfGraphManager::GetInstance().DeleteGeSession(); + } catch (const std::exception& e) { + MS_LOG(ERROR) << "Error occurred when deleting GE graph runner and session fail. Error: " << e.what(); + } catch (...) { + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(ERROR) << "Error occurred when deleting GE graph runner and session fail. Exception name: " << exName; + } + if (ge::GEFinalize() != ge::GRAPH_SUCCESS) { + MS_LOG(WARNING) << "Finalize GE failed!"; + } + is_pynative_ge_init_ = false; + } else { + MS_LOG(INFO) << "Ge is used, no need to finalize, tsd reference = " << ge_ref_ << "."; + } +#endif + return true; +} + +bool MsContext::PynativeInitGe() { + if (is_pynative_ge_init_ || ge_ref_ || tsd_ref_) { + return true; + } + (void)OpenTsd(); + (void)InitGe(); + is_pynative_ge_init_ = true; + return true; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/context/ms_context.h b/mindspore/ccsrc/utils/context/ms_context.h new file mode 100644 index 00000000000..e7d8dc769f7 --- /dev/null +++ b/mindspore/ccsrc/utils/context/ms_context.h @@ -0,0 +1,180 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_CONTEXT_MS_CONTEXT_H_ +#define MINDSPORE_CCSRC_UTILS_CONTEXT_MS_CONTEXT_H_ +#include +#include +#include +#include +#include +#include +#include +#include "transform/graph_runner.h" +#include "utils/log_adapter.h" + +namespace mindspore { + +enum MsBackendPolicy { + kMsBackendGeOnly = 0, + kMsBackendVmOnly = 1, + kMsBackendGePrior = 2, + kMsBackendVmPrior = 3, + kMsBackendMsPrior = 4, + kMsBackendUnknown = 5, +}; + +const int kGraphMode = 0; +const int kPynativeMode = 1; +const char kCPUDevice[] = "CPU"; +const char kGPUDevice[] = "GPU"; +const char kAscendDevice[] = "Ascend"; +const char kDavinciDevice[] = "Davinci"; +const char KNpuLog[] = "_npu_log"; +const std::set kTargetSet = {kCPUDevice, kGPUDevice, kAscendDevice, kDavinciDevice}; + +class MsContext { + public: + ~MsContext() = default; + MsContext(const MsContext&) = delete; + MsContext& operator=(const MsContext&) = delete; + + static std::shared_ptr GetInstance(); + + std::string backend_policy() const; + bool set_backend_policy(const std::string& policy); + + int execution_mode() const { return execution_mode_; } + void set_execution_mode(int execution_mode); + + bool enable_pynative_infer() const { return enable_pynative_infer_; } + void set_enable_pynative_infer(bool enable_pynative_infer) { enable_pynative_infer_ = enable_pynative_infer; } + + void set_enable_task_sink(bool enable_task_sink) { enable_task_sink_ = enable_task_sink; } + bool enable_task_sink() const { return enable_task_sink_; } + + void set_precompile_only(bool precompile_only) { precompile_only_ = precompile_only; } + bool precompile_only() const { return precompile_only_; } + + std::string device_target() const { return device_target_; } + bool set_device_target(const std::string& target); + + uint32_t device_id() const { return device_id_; } + bool set_device_id(uint32_t device_id); + + bool save_graphs_flag() const { return save_graphs_flag_; } + void set_save_graphs_flag(bool save_graphs_flag) { save_graphs_flag_ = save_graphs_flag; } + + std::string save_graphs_path() const { return save_graphs_path_; } + void set_save_graphs_path(const std::string& save_paths) { save_graphs_path_ = save_paths; } + + bool OpenTsd(); + bool CloseTsd(bool force = false); + bool InitGe(); + bool FinalizeGe(bool force = false); + void set_enable_hccl(bool enable_hccl) { enable_hccl_ = enable_hccl; } + bool enable_hccl() const { return enable_hccl_; } + bool PynativeInitGe(); + + void set_ir_fusion_flag(bool ir_fusion_flag) { ir_fusion_flag_ = ir_fusion_flag; } + bool ir_fusion_flag() const { return ir_fusion_flag_; } + + void set_loop_sink_flag(bool loop_sink_flag) { enable_loop_sink_ = loop_sink_flag; } + bool loop_sink_flag() const { return enable_loop_sink_; } + + void set_enable_mem_reuse(bool enable_mem_reuse) { enable_mem_reuse_ = enable_mem_reuse; } + bool enable_mem_reuse() const { return enable_mem_reuse_; } + + bool save_ms_model_flag() const { return save_ms_model_flag_; } + void set_save_ms_model_flag(bool save_ms_model_flag) { save_ms_model_flag_ = save_ms_model_flag; } + + std::string save_ms_model_path() const { return save_ms_model_path_; } + void set_save_ms_model_path(const std::string& save_ms_model_path) { save_ms_model_path_ = save_ms_model_path; } + + void set_enable_gpu_summary(bool enable_gpu_summary) { enable_gpu_summary_ = enable_gpu_summary; } + bool enable_gpu_summary() const { return enable_gpu_summary_; } + + void set_auto_mixed_precision_flag(bool auto_mixed_precision_flag) { + auto_mixed_precision_flag_ = auto_mixed_precision_flag; + } + bool auto_mixed_precision_flag() const { return auto_mixed_precision_flag_; } + + void set_enable_reduce_precision(bool flag) { enable_reduce_precision_ = flag; } + bool enable_reduce_precision() const { return enable_reduce_precision_; } + + void set_enable_dump(bool flag) { enable_dump_ = flag; } + bool enable_dump() const { return enable_dump_; } + + void set_save_dump_path(const std::string& path) { save_dump_path_ = path; } + std::string save_dump_path() const { return save_dump_path_; } + + bool IsTsdOpened() const { return tsd_ref_ > 0; } + + bool is_multi_graph_sink() const { return is_multi_graph_sink_; } + void set_is_multi_graph_sink(bool flag) { is_multi_graph_sink_ = flag; } + + void set_enable_dynamic_mem_pool(bool enable_dynamic_mem_pool) { enable_dynamic_mem_pool_ = enable_dynamic_mem_pool; } + bool enable_dynamic_mem_pool() const { return enable_dynamic_mem_pool_; } + + void set_graph_memory_max_size(const std::string& graph_memory_max_size) { + graph_memory_max_size_ = graph_memory_max_size; + } + + void set_variable_memory_max_size(const std::string& variable_memory_max_size) { + variable_memory_max_size_ = variable_memory_max_size; + } + + private: + MsContext(const std::string& backend_policy, const std::string& target); + void GetGeOptions(std::map* ge_options) const; + void SetDisableReuseMemoryFlag(std::map* ge_options) const; + void SetHcclOptions(std::map* ge_options) const; + + static std::shared_ptr inst_context_; + static std::map policy_map_; + MsBackendPolicy backend_policy_; + std::string device_target_; + uint32_t device_id_; + int execution_mode_; + bool enable_pynative_infer_; + bool save_graphs_flag_; + std::string save_graphs_path_; + uint32_t tsd_ref_; + uint32_t ge_ref_; + bool enable_task_sink_; + bool enable_hccl_; + bool precompile_only_; + bool ir_fusion_flag_; + bool auto_mixed_precision_flag_; + bool enable_reduce_precision_; + bool enable_loop_sink_; + bool enable_mem_reuse_; + std::string save_ms_model_path_; + bool save_ms_model_flag_; + bool enable_gpu_summary_; + bool enable_dump_; + std::string save_dump_path_; + bool is_multi_graph_sink_; + bool is_pynative_ge_init_; + bool enable_dynamic_mem_pool_; + std::string graph_memory_max_size_; + std::string variable_memory_max_size_; + std::thread tdt_print_; +}; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_CONTEXT_MS_CONTEXT_H_ diff --git a/mindspore/ccsrc/utils/contract.h b/mindspore/ccsrc/utils/contract.h new file mode 100644 index 00000000000..fc257b3e242 --- /dev/null +++ b/mindspore/ccsrc/utils/contract.h @@ -0,0 +1,107 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_CONTRACT_H +#define MINDSPORE_CCSRC_UTILS_CONTRACT_H +#include +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { +class ContractError : public std::logic_error { + public: + explicit ContractError(const std::string &msg) : std::logic_error(msg) {} + explicit ContractError(const char *msg) : std::logic_error(msg) {} +}; + +struct Signatory { + LocationInfo location_info; + const char *extra_info; +}; + +struct NotNullRule { + template + constexpr static bool Check(const T &val) { + return val != nullptr; + } + constexpr static const char *Desc() { return " must not be null"; } +}; +template +class EnsuresAccess {}; + +template +using RemoveCVR = std::remove_cv_t>; + +template +class Ensures : public EnsuresAccess { + public: + Ensures(T v, const Signatory &signatory) : value_(v) { + if (!R::Check(value_)) { + LogStream contract_stream; + contract_stream << "contract error: " << signatory.extra_info << R::Desc(); + LogWriter(signatory.location_info, EXCEPTION, ArgumentError) ^ contract_stream; + } + } + template >> + Ensures(const Ensures &other) : value_(other.get()) {} + + T get() const { return value_; } + T &get() { return value_; } + + operator T() const { return value_; } + + private: + T value_; +}; + +template +class EnsuresAccess>>> { + public: + T operator->() const { + auto ptr = static_cast *>(this)->get(); + return ptr; + } +}; + +template +struct IsSharedPtr : std::false_type {}; +template +struct IsSharedPtr> : std::true_type {}; +template +struct IsSharedPtr &> : std::true_type {}; + +template +class EnsuresAccess::value>> { + using element_type = typename std::remove_cv_t>::element_type; + + public: + element_type *operator->() const { + auto ptr = static_cast *>(this)->get(); + return ptr.get(); + } +}; + +template +using Expects = Ensures; +template +using NotNull = Ensures; +#define ENSURE(_v, _rule) Ensures(_v, {{__FILE__, __LINE__, __FUNCTION__}, #_v}) +#define NOT_NULL(_v) ENSURE(_v, NotNullRule) +} // namespace mindspore +#endif // MINDSPORE_CCSRC_UTILS_CONTRACT_H diff --git a/mindspore/ccsrc/utils/convert_utils.cc b/mindspore/ccsrc/utils/convert_utils.cc new file mode 100644 index 00000000000..ccd21f6801a --- /dev/null +++ b/mindspore/ccsrc/utils/convert_utils.cc @@ -0,0 +1,376 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/convert_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "pybind11/pybind11.h" + +#include "ir/meta_tensor.h" +#include "pipeline/parse/parse.h" +#include "ir/value.h" + +namespace mindspore { +py::object BuiltinsToPyData(const Any &value); +py::object BuiltinsToPyData(const BaseRef &value); +py::object VectorToPyData(const Any &value); +py::object VectorRefToPyData(const VectorRef &value); + +py::object ValuePtrToPyData(const ValuePtr &value) { + if (value == nullptr) { + MS_LOG(EXCEPTION) << "value is null"; + } + py::object ret; + if (value->isa()) { + MS_LOG(DEBUG) << "int"; + py::int_ v = value->cast()->value(); + ret = v; + } else if (value->isa()) { + MS_LOG(DEBUG) << "uint64"; + py::int_ v = value->cast()->value(); + ret = v; + } else if (value->isa()) { + MS_LOG(DEBUG) << "bool"; + py::bool_ v = value->cast()->value(); + ret = v; + } else if (value->isa()) { + MS_LOG(DEBUG) << "double"; + py::float_ v = value->cast()->value(); + ret = v; + } else if (value->isa()) { + MS_LOG(DEBUG) << "float"; + py::float_ v = value->cast()->value(); + ret = v; + } else if (value->isa()) { + MS_LOG(DEBUG) << "String"; + py::str v = value->cast()->value(); + ret = v; + } else if (value->isa()) { + MS_LOG(DEBUG) << "tensor"; + py::tuple v(1); + v[0] = value->cast(); + ret = v[0]; + } else if (value->isa()) { + MS_LOG(DEBUG) << "RefKey"; + py::tuple v(1); + v[0] = value->cast(); + ret = v[0]; + } else if (value->isa()) { + MS_LOG(DEBUG) << "tuple"; + auto value_tuple = value->cast()->value(); + py::tuple rets(value_tuple.size()); + + size_t i = 0; + for (auto &v : value_tuple) { + rets[i] = ValuePtrToPyData(v); + i++; + } + ret = rets; + } else if (value->isa()) { + MS_LOG(DEBUG) << "list"; + auto value_list = value->cast()->value(); + py::list rets(value_list.size()); + + size_t i = 0; + for (auto &v : value_list) { + rets[i] = ValuePtrToPyData(v); + i++; + } + ret = rets; + } else if (value->isa()) { + py::tuple v(1); + v[0] = value->cast(); + ret = v[0]; + } else if (value->isa()) { + ret = py::none(); + } else if (value->isa()) { + ret = py::none(); + } else { + MS_LOG(EXCEPTION) << "Unsupported convert value: " << value->ToString() << " to a PyData."; + } + return ret; +} + +py::object AnyToPyData(const Any &value) { + py::object ret; + MS_LOG(DEBUG) << "AnyToPyData " << value.GetString(); + if (value.is() || value.is() || value.is() || value.is()) { + ret = BuiltinsToPyData(value); + } else if (value.is()) { + MS_LOG(DEBUG) << "ValuePtr"; + ValuePtr v = value.cast(); + ret = ValuePtrToPyData(v); + } else if (value.is()) { + MS_LOG(DEBUG) << "tensor"; + py::tuple v(1); + v[0] = value.cast(); + ret = v[0]; + } else if (value.is()) { + MS_LOG(DEBUG) << "py obj"; + ret = value.cast(); + } else if (value.is>() || value.is>()) { + ret = VectorToPyData(value); + } else if (value.is>()) { + MS_LOG(DEBUG) << "list_any"; + auto value_list = value.cast>(); + py::list rets = py::list(); + for (auto &v : value_list) { + rets.append(AnyToPyData(v)); + } + ret = rets; + } else if (value.is>()) { + auto value_list = value.cast>(); + py::tuple rets(value_list.size()); + for (size_t i = 0; i < value_list.size(); i++) { + rets[i] = AnyToPyData(value_list[i]); + } + ret = rets; + } else if (value.is()) { + py::tuple v(1); + v[0] = value.cast(); + ret = v[0]; + } else { + MS_LOG(EXCEPTION) << "value is not support type"; + } + return ret; +} + +py::object BaseRefToPyData(const BaseRef &value) { + py::object ret; + MS_LOG(DEBUG) << "BaseRefToPyData " << common::SafeCStr(value.ToString()); + if (utils::isa(value) || utils::isa(value) || utils::isa(value) || utils::isa(value)) { + ret = BuiltinsToPyData(value); + } else if (utils::isa(value)) { + MS_LOG(DEBUG) << "ValuePtr"; + ValuePtr v = utils::cast(value); + ret = ValuePtrToPyData(v); + } else if (utils::isa(value)) { + MS_LOG(DEBUG) << "tensor"; + py::tuple v(1); + v[0] = utils::cast(value); + ret = v[0]; + } else if (utils::isa(value)) { + MS_LOG(DEBUG) << "py obj"; + PyObjectRef py_ref = utils::cast(value); + ret = py_ref.object_; + } else if (utils::isa(value)) { + auto vec_ref = utils::cast(value); + ret = VectorRefToPyData(vec_ref); + } else if (utils::isa(value)) { + py::tuple v(1); + v[0] = utils::cast(value); + ret = v[0]; + } else { + MS_LOG(EXCEPTION) << "value is not support type"; + } + return ret; +} + +bool ValueToBool(const ValuePtr &v, bool *value) { + MS_EXCEPTION_IF_NULL(v); + if (v->isa()) { + *value = v->cast()->value(); + } else if (v->isa()) { + *value = v->cast()->value() == 0 ? false : true; + } else if (v->isa()) { + *value = v->cast()->value() == 0 ? false : true; + } else if (v->isa()) { + *value = v->cast()->value() == 0 ? false : true; + } else if (v->isa()) { + *value = v->cast()->value() == 0 ? false : true; + } else if (v->isa()) { + auto tensor = v->cast(); + MS_EXCEPTION_IF_NULL(tensor); + + bool *tensor_data = static_cast(tensor->data_c()); + // maybe need to support if tensor is a bool array + auto vb = tensor_data[0]; + *value = vb; + } else { + MS_LOG(WARNING) << "value is not supported to cast to be bool"; + return false; + } + return true; +} + +bool BaseRefToBool(const BaseRef &v, bool *value) { + if (utils::isa(v)) { + return ValueToBool(utils::cast(v), value); + } else if (utils::isa(v)) { + auto vb = utils::cast(v); + if (vb == true) { + *value = true; + } else { + *value = false; + } + } else if (utils::isa(v)) { + auto vb = utils::cast(v); + if (vb == 0) { + *value = false; + } else { + *value = true; + } + } else if (utils::isa(v)) { + auto vb = utils::cast(v); + if (vb == 0) { + *value = false; + } else { + *value = true; + } + } else if (utils::isa(v)) { + auto vb = utils::cast(v); + if (vb >= -FLT_EPSILON && vb <= FLT_EPSILON) { + *value = false; + } else { + *value = true; + } + } else if (utils::isa(v)) { + auto vb = utils::cast(v); + if (vb >= -DBL_EPSILON && vb <= DBL_EPSILON) { + *value = false; + } else { + *value = true; + } + } else { + MS_LOG(DEBUG) << "value is not supported to cast to be bool"; + return false; + } + return true; +} + +py::object BuiltinsToPyData(const Any &value) { + if (value.is()) { + MS_LOG(DEBUG) << "int"; + py::int_ ret = value.cast(); + return std::move(ret); + } else if (value.is()) { + MS_LOG(DEBUG) << "float"; + py::float_ ret = value.cast(); + return std::move(ret); + } else if (value.is()) { + MS_LOG(DEBUG) << "double"; + py::float_ ret = value.cast(); + return std::move(ret); + } else { + MS_LOG(DEBUG) << "bool"; + py::bool_ ret = value.cast(); + return std::move(ret); + } +} + +py::object BuiltinsToPyData(const BaseRef &value) { + if (utils::isa(value)) { + MS_LOG(DEBUG) << "int"; + py::int_ ret = utils::cast(value); + return std::move(ret); + } else if (utils::isa(value)) { + MS_LOG(DEBUG) << "float"; + py::float_ ret = utils::cast(value); + return std::move(ret); + } else if (utils::isa(value)) { + MS_LOG(DEBUG) << "double"; + py::float_ ret = utils::cast(value); + return std::move(ret); + } else { + MS_LOG(DEBUG) << "bool"; + py::bool_ ret = utils::cast(value); + return std::move(ret); + } +} + +py::object VectorToPyData(const Any &value) { + py::object ret; + if (value.is>()) { + MS_LOG(DEBUG) << "vector_tensor"; + std::vector outputs; + outputs = value.cast>(); + py::tuple tensor_tuple(outputs.size()); + for (std::size_t i = 0; i < outputs.size(); ++i) { + tensor_tuple[i] = *outputs[i]; + } + ret = tensor_tuple; + } else { + MS_LOG(DEBUG) << "vector_any"; + auto value_list = value.cast>(); + py::tuple any_tuple = py::tuple(value_list.size()); + size_t i = 0; + for (auto &v : value_list) { + any_tuple[i] = AnyToPyData(v); + i++; + } + ret = any_tuple; + } + return ret; +} + +py::object VectorRefToPyData(const VectorRef &value_list) { + py::object ret; + MS_LOG(DEBUG) << "vector_ref"; + size_t value_size = value_list.size(); + py::tuple ref_tuple = py::tuple(value_size); + for (size_t i = 0; i < value_size; i++) { + ref_tuple[i] = BaseRefToPyData(value_list[i]); + } + ret = ref_tuple; + return ret; +} + +AbstractBasePtr PyListDtype2AbstractTensor(const py::object &shape_obj, const py::object &type_obj) { + if ((py::isinstance(shape_obj) || py::isinstance(shape_obj)) && + py::hasattr(type_obj, PYTHON_DTYPE_FLAG)) { + auto ret_vec = shape_obj.cast>(); + auto ret_dtype = type_obj.cast(); + MS_EXCEPTION_IF_NULL(ret_dtype); + // if the size of shape list is empty, return an scalar abstract + if (ret_vec.empty() && (!ret_dtype->isa())) { + abstract::AbstractScalarPtr abs_scalar = std::make_shared(kAnyValue, ret_dtype); + return abs_scalar; + } + AbstractBasePtr tensor = nullptr; + if (ret_dtype->isa()) { + auto tensor_type = type_obj.cast(); + MS_EXCEPTION_IF_NULL(tensor_type); + tensor = std::make_shared(tensor_type->element(), ret_vec); + } else { + tensor = std::make_shared(ret_dtype, ret_vec); + } + return tensor; + } else if (py::isinstance(shape_obj) && py::isinstance(type_obj)) { + py::tuple shape_tuple = shape_obj.cast(); + py::tuple typeid_tuple = type_obj.cast(); + AbstractBasePtrList ptr_list; + for (size_t it = 0; it < shape_tuple.size(); ++it) { + auto tensor_it = PyListDtype2AbstractTensor(shape_tuple[it], typeid_tuple[it]); + ptr_list.push_back(tensor_it); + } + auto tuple = std::make_shared(ptr_list); + return tuple; + } else if (shape_obj.is_none() && type_obj.is_none()) { + // AbstractNone indicates there is no output for this CNode node. + auto abstract_none = std::make_shared(); + return abstract_none; + } else { + MS_LOG(EXCEPTION) << "Python evaluator return invalid shape or type. " << (std::string)py::str(type_obj); + } +} +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/convert_utils.h b/mindspore/ccsrc/utils/convert_utils.h new file mode 100644 index 00000000000..f190f98e6ab --- /dev/null +++ b/mindspore/ccsrc/utils/convert_utils.h @@ -0,0 +1,125 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_CONVERT_UTILS_H_ +#define MINDSPORE_CCSRC_UTILS_CONVERT_UTILS_H_ + +#include +#include "pybind11/pybind11.h" + +#include "utils/any.h" +#include "ir/base.h" +#include "ir/anf.h" +#include "utils/base_ref.h" + +namespace py = pybind11; + +namespace mindspore { + +py::object AnyToPyData(const Any &value); +py::object BaseRefToPyData(const BaseRef &value); +bool BaseRefToBool(const BaseRef &in, bool *out); +bool ValueToBool(const ValuePtr &in, bool *out); +py::object ValuePtrToPyData(const ValuePtr &value); + +inline int SizeToInt(size_t u) { + if (u > static_cast((std::numeric_limits::max)())) { + MS_LOG(EXCEPTION) << "The size_t value(" << u << ") exceeds the maximum value of int."; + } + return static_cast(u); +} + +inline uint32_t SizeToUint(size_t u) { + if (u > static_cast((std::numeric_limits::max)())) { + MS_LOG(EXCEPTION) << "The size_t value(" << u << ") exceeds the maximum value of uint32_t."; + } + return static_cast(u); +} + +inline int64_t SizeToLong(size_t u) { + if (u > static_cast((std::numeric_limits::max)())) { + MS_LOG(EXCEPTION) << "The size_t value(" << u << ") exceeds the maximum value of int64_t."; + } + return static_cast(u); +} + +inline size_t IntToSize(int u) { + if (u < 0) { + MS_LOG(EXCEPTION) << "The int value(" << u << ") is less than 0."; + } + return static_cast(u); +} + +inline size_t LongToSize(int64_t u) { + if (u < 0) { + MS_LOG(EXCEPTION) << "The int64_t value(" << u << ") is less than 0."; + } + return static_cast(u); +} + +inline size_t FloatToSize(float u) { + if (u < 0) { + MS_LOG(EXCEPTION) << "The float value(" << u << ") is less than 0."; + } + + if (u > static_cast((std::numeric_limits::max)())) { + MS_LOG(EXCEPTION) << "The float value(" << u << ") exceeds the maximum value of size_t."; + } + return static_cast(u); +} + +inline uint32_t IntToUint(int32_t u) { + if (u < 0) { + MS_LOG(EXCEPTION) << "The int32_t value(" << u << ") is less than 0."; + } + return static_cast(u); +} + +inline int32_t UintToInt(uint32_t u) { + if (u > static_cast((std::numeric_limits::max)())) { + MS_LOG(EXCEPTION) << "The uint32_t value(" << u << ") exceeds the maximum value of int32_t."; + } + return static_cast(u); +} + +inline unsigned int UlongToUint(size_t u) { + if (u > static_cast((std::numeric_limits::max)())) { + MS_LOG(EXCEPTION) << "The size_t value(" << u << ") exceeds the maximum value of unsigned int."; + } + return static_cast(u); +} + +inline void IntMulWithOverflowCheck(int a, int b, int *c) { + int out = a * b; + if (a != 0) { + bool ok = ((out / a) != b); + if (ok) { + MS_LOG(EXCEPTION) << "Mul: a(" << a << ") * b(" << b << ") result is overflow"; + } + } + *c = out; +} + +inline uint8_t *AddressOffset(void *address, size_t offset) { + MS_EXCEPTION_IF_NULL(address); + return static_cast(address) + offset; +} + +AbstractBasePtr PyListDtype2AbstractTensor(const py::object &shape_obj, const py::object &type_obj); + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_CONVERT_UTILS_H_ diff --git a/mindspore/ccsrc/utils/counter.h b/mindspore/ccsrc/utils/counter.h new file mode 100644 index 00000000000..891f9c7a35a --- /dev/null +++ b/mindspore/ccsrc/utils/counter.h @@ -0,0 +1,102 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_COUNTER_H_ +#define MINDSPORE_CCSRC_UTILS_COUNTER_H_ +#include +#include "utils/ordered_map.h" + +namespace mindspore { + +template , class Equal = std::equal_to> +class Counter { + using counter_type = Counter; + + public: + Counter() = default; + ~Counter() = default; + + Counter(const Counter& other) { data = other.data; } + Counter& operator=(const Counter& other) { + if (this != &other) { + data = other.data; + } + return *this; + } + + int& operator[](const T& t) { return data[t]; } + + counter_type operator-(const counter_type& other) { + counter_type new_counter; + for (auto iter = begin(); iter != end(); ++iter) { + auto key = iter->first; + int value = iter->second; + auto item = other.data.find(key); + if (item != other.data.end()) { + int o_value = item->second; + if (value - o_value > 0) { + new_counter[key] = value - o_value; + } + } else { + new_counter[key] = value; + } + } + + return new_counter; + } + + counter_type operator+(const counter_type& other) { + counter_type new_counter; + for (auto iter = begin(); iter != end(); ++iter) { + auto key = iter->first; + int value = iter->second; + auto item = other.data.find(key); + if (item != other.data.end()) { + new_counter[key] = iter->second + item->second; + } else { + new_counter[key] = value; + } + } + + for (auto iter = other.cbegin(); iter != other.cend(); ++iter) { + auto key = iter->first; + int value = iter->second; + if (!new_counter.contains(key)) { + new_counter[key] = value; + } + } + + return new_counter; + } + + std::size_t size() const { return data.size(); } + + bool contains(const T& t) const { return data.find(t) != data.end(); } + + typename OrderedMap::iterator begin() { return data.begin(); } + + typename OrderedMap::iterator end() { return data.end(); } + + typename OrderedMap::const_iterator cbegin() const { return data.cbegin(); } + + typename OrderedMap::const_iterator cend() const { return data.cend(); } + + private: + OrderedMap data; +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_COUNTER_H_ diff --git a/mindspore/ccsrc/utils/error_code.h b/mindspore/ccsrc/utils/error_code.h new file mode 100644 index 00000000000..f6d70666c41 --- /dev/null +++ b/mindspore/ccsrc/utils/error_code.h @@ -0,0 +1,114 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ +#define MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ + +#include +#include + +namespace DataEngineBase { +// system ID +const int SYSID_MD = 20; + +// Runtime location +enum LogRuntime { + RT_HOST = 0b01, + RT_DEVICE = 0b10, +}; + +// sub model +enum SubModuleId { + COMMON_MODULE = 0, + DATAGET_MODULE, + MINDRECORD_MODULE, +}; + +// error code type +enum ErrorCodeType { + ERROR_CODE = 0b01, + EXCEPTION_CODE = 0b10, +}; + +// error level +enum ErrorLevel { + COMMON_LEVEL = 0b000, + SUGGESTION_LEVEL = 0b001, + MINOR_LEVEL = 0b010, + MAJOR_LEVEL = 0b011, + CRITICAL_LEVEL = 0b100, +}; + +// code compose(4 byte), runtime: 2 bit, type: 2 bit, level: 3 bit, sysid: 8 bit, modid: 5 bit, value: 12 bit +#define DE_ERRORNO(runtime, type, level, sysid, submodid, name, value, desc) \ + constexpr DataEngineBase::Status name = ((0xFF & ((uint8_t)runtime)) << 30) | ((0xFF & ((uint8_t)type)) << 28) | \ + ((0xFF & ((uint8_t)level)) << 25) | ((0xFF & ((uint8_t)sysid)) << 17) | \ + ((0xFF & ((uint8_t)submodid)) << 12) | (0x0FFF & ((uint16_t)value)); \ + const DataEngineBase::ErrorNoRegisterar g_##name##_errorno(name, desc); + +// each module defines error codes using the following macros +#define DE_ERRORNO_COMMON(name, value, desc) \ + DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ + DataEngineBase::SYSID_MD, DataEngineBase::COMMON_MODULE, name, value, desc) + +#define DE_ERRORNO_DATASET(name, value, desc) \ + DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ + DataEngineBase::SYSID_MD, DataEngineBase::DATAGET_MODULE, name, value, desc) + +#define DE_ERRORNO_MINDRECORD(name, value, desc) \ + DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ + DataEngineBase::SYSID_MD, DataEngineBase::MINDRECORD_MODULE, name, value, desc) + +// get error code description +#define DE_GET_ERRORNO_STR(value) DataEngineBase::StatusFactory::Instance()->GetErrDesc(value) + +class StatusFactory { + public: + static StatusFactory *Instance() { + static StatusFactory instance; + return &instance; + } + + void RegisterErrorNo(uint32_t err, const std::string &desc) { + if (err_desc_.find(err) != err_desc_.end()) return; + err_desc_[err] = desc; + } + + std::string GetErrDesc(uint32_t err) { + auto iter_find = err_desc_.find(err); + if (iter_find == err_desc_.end()) return ""; + return iter_find->second; + } + + protected: + StatusFactory() = default; + + ~StatusFactory() = default; + + private: + std::map err_desc_; +}; + +class ErrorNoRegisterar { + public: + ErrorNoRegisterar(uint32_t err, const std::string &desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); } + + ~ErrorNoRegisterar() = default; +}; + +using Status = uint32_t; +} // namespace DataEngineBase +#endif // MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ diff --git a/mindspore/ccsrc/utils/graph_utils.cc b/mindspore/ccsrc/utils/graph_utils.cc new file mode 100644 index 00000000000..938df2c291c --- /dev/null +++ b/mindspore/ccsrc/utils/graph_utils.cc @@ -0,0 +1,531 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/graph_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/visitor.h" +#include "utils/log_adapter.h" +#include "common/utils.h" +#include "pipeline/parse/function_block.h" +#include "pipeline/parse/python_adapter.h" + +namespace mindspore { +using SymbolicKeyTypePtr = std::shared_ptr; + +namespace { +class DeepFirstSearcher : public AnfVisitor { + public: + explicit DeepFirstSearcher(const IncludeFunc& include) : include_(include) {} + ~DeepFirstSearcher() override = default; + + std::vector Search(const AnfNodePtr& root) { + if (root == nullptr) { + return res_; + } + Visit(root); + return res_; + } + + void Visit(const AnfNodePtr& node) override { + MS_EXCEPTION_IF_NULL(node); + if (seen_.count(node) != 0) { + return; + } + + (void)seen_.insert(node); + + auto incl = include_(node); + if (incl == EXCLUDE) { + return; + } + + res_.push_back(node); + if (incl == FOLLOW) { + AnfVisitor::Visit(node); + } + } + + private: + IncludeFunc include_; + std::vector res_{}; + std::set seen_{}; +}; + +class DeepScopedGraphSearcher : public DeepFirstSearcher { + public: + explicit DeepScopedGraphSearcher(const IncludeFunc& include) : DeepFirstSearcher(include) {} + ~DeepScopedGraphSearcher() override = default; + + void Visit(const CNodePtr& cnode) override { + if (cnode->func_graph() == nullptr) { + return; + } + + AnfNodePtr ret = cnode->func_graph()->get_return(); + if (ret != nullptr) { + DeepFirstSearcher::Visit(ret); + } + + auto& inputs = cnode->inputs(); + for (auto iter = inputs.rbegin(); iter != inputs.rend(); ++iter) { + DeepFirstSearcher::Visit(*iter); + } + } + + void Visit(const ValueNodePtr& vnode) override { + if (!IsValueNode(vnode)) { + return; + } + + auto graph = GetValueNode(vnode); + AnfNodePtr ret = graph->get_return(); + if (ret != nullptr) { + DeepFirstSearcher::Visit(ret); + } + } + + void Visit(const ParameterPtr& param) override { + if (param->func_graph() == nullptr) { + return; + } + + AnfNodePtr ret = param->func_graph()->get_return(); + if (ret != nullptr) { + DeepFirstSearcher::Visit(ret); + } + } +}; + +class DeepUsedGraphSearcher : public DeepFirstSearcher { + public: + explicit DeepUsedGraphSearcher(const IncludeFunc& include) : DeepFirstSearcher(include) {} + ~DeepUsedGraphSearcher() override = default; + + void Visit(const CNodePtr& cnode) override { + auto& inputs = cnode->inputs(); + for (auto iter = inputs.rbegin(); iter != inputs.rend(); ++iter) { + DeepFirstSearcher::Visit(*iter); + } + } + + void Visit(const ValueNodePtr& vnode) override { + if (!IsValueNode(vnode)) { + return; + } + + auto graph = GetValueNode(vnode); + AnfNodePtr ret = graph->get_return(); + if (ret != nullptr) { + DeepFirstSearcher::Visit(ret); + } + } +}; + +class DeepLinkedGraphSearcher : public DeepFirstSearcher { + public: + explicit DeepLinkedGraphSearcher(const IncludeFunc& include) : DeepFirstSearcher(include) {} + ~DeepLinkedGraphSearcher() override = default; + + void Visit(const CNodePtr& cnode) override { + auto& inputs = cnode->inputs(); + for (auto iter = inputs.rbegin(); iter != inputs.rend(); ++iter) { + DeepFirstSearcher::Visit(*iter); + } + } + + void Visit(const ValueNodePtr&) override {} +}; +} // namespace + +std::vector DeepScopedGraphSearch(const AnfNodePtr& root, const IncludeFunc& include) { + return DeepScopedGraphSearcher(include).Search(root); +} + +std::vector DeepUsedGraphSearch(const AnfNodePtr& root, const IncludeFunc& include) { + return DeepUsedGraphSearcher(include).Search(root); +} + +std::vector DeepLinkedGraphSearch(const AnfNodePtr& root, const IncludeFunc& include) { + return DeepLinkedGraphSearcher(include).Search(root); +} + +std::vector TopoSort(const AnfNodePtr& root, const SuccFunc& succ, const IncludeFunc& include) { + std::unordered_set done; + std::list todo(1, root); + std::unordered_map rank; + std::vector res; + + while (!todo.empty()) { + AnfNodePtr node = todo.back(); + if (done.find(node) != done.end()) { + todo.pop_back(); + continue; + } + if (rank.find(node) != rank.end() && rank[node] != todo.size()) { + MS_LOG(EXCEPTION) << "Graph exists cycle, node " << node->DebugString(); + } + rank[node] = todo.size(); + bool cont = false; + auto incl = include(node); + if (incl == FOLLOW) { + auto succs = succ(node); + for (const auto i : succs) { + if ((done.find(i) == done.end()) + // Handle the case for 2 subgraphs calls each other. + // If the ValueNodeGraph's return is already in the todo list, do not follow it. + && !((std::find(todo.begin(), todo.end(), i) != todo.end()) && (i->func_graph() != nullptr) && + (i->func_graph()->get_return() == i))) { + todo.push_back(i); + cont = true; + } + } + } else if (incl == NOFOLLOW) { + // do nothing + } else if (incl == EXCLUDE) { + (void)done.insert(node); + todo.pop_back(); + continue; + } else { + MS_LOG(EXCEPTION) << "include(node) must return one of: \"follow\", \"nofollow\", \"exclude\""; + } + if (cont) { + continue; + } + (void)done.insert(node); + res.push_back(node); + todo.pop_back(); + } + return res; +} + +std::vector SuccDeeper(const AnfNodePtr& node) { + std::vector vecs; + if (node == nullptr) { + return vecs; + } + + if (IsValueNode(node)) { + auto graph = GetValueNode(node); + auto ret = graph->get_return(); + if (ret != nullptr) { + vecs.push_back(ret); + } + return vecs; + } else if (node->func_graph() != nullptr) { + if (node->isa()) { + auto& inputs = node->cast()->inputs(); + (void)vecs.insert(vecs.end(), inputs.begin(), inputs.end()); + } + auto graph = node->func_graph(); + if (graph->get_return() != nullptr) { + vecs.push_back(graph->get_return()); + } + return vecs; + } + + return vecs; +} + +std::vector SuccDeeperSimple(const AnfNodePtr& node) { + std::vector vecs; + if (node == nullptr) { + return vecs; + } + + if (IsValueNode(node)) { + auto graph = GetValueNode(node); + auto ret = graph->get_return(); + if (ret != nullptr) { + vecs.push_back(ret); + } + return vecs; + } else { + if (node->isa()) { + auto& inputs = node->cast()->inputs(); + (void)vecs.insert(vecs.end(), inputs.begin(), inputs.end()); + } + return vecs; + } +} + +std::vector SuccIncoming(const AnfNodePtr& node) { + std::vector vecs; + if (node == nullptr) { + return vecs; + } + + if (node->isa()) { + auto& inputs = node->cast()->inputs(); + (void)vecs.insert(vecs.end(), inputs.begin(), inputs.end()); + } + return vecs; +} + +std::vector SuccIncludeFV(const FuncGraphPtr& fg, const AnfNodePtr& node) { + std::vector vecs; + if (node == nullptr) { + return vecs; + } + if (node->isa()) { + auto cnode = node->cast(); + auto& inputs = cnode->inputs(); + // Check if free variables used. + for (const auto& input : inputs) { + auto input_fg = GetValueNode(input); + if (input_fg) { + for (auto& fv : input_fg->free_variables_nodes()) { + if (fv->func_graph() == fg && fg->nodes().contains(fv)) { + vecs.push_back(fv); + } + } + } + } + (void)vecs.insert(vecs.end(), inputs.begin(), inputs.end()); + } + return vecs; +} + +IncludeType AlwaysInclude(const AnfNodePtr&) { return FOLLOW; } + +IncludeType IncludeBelongGraph(const FuncGraphPtr& fg, const AnfNodePtr& node) { + if (node->func_graph() == fg) { + return FOLLOW; + } else { + return EXCLUDE; + } +} + +FuncGraphIndex::FuncGraphIndex(const FuncGraphPtr& fg, const SearchFunc& search, const IncludeFunc& include) { + MS_EXCEPTION_IF_NULL(fg); + Acquire(fg); + + auto vec = search(fg->get_return(), include); + for (auto& node : vec) { + MS_EXCEPTION_IF_NULL(node); + Acquire(node); + if (node->func_graph() != nullptr) { + Acquire(node->func_graph()); + } + } +} + +std::set FuncGraphIndex::GetFuncGraphs(const std::string& key) { + std::set func_graphs; + if (index_func_graph_.find(key) != index_func_graph_.end()) { + func_graphs = index_func_graph_[key]; + } + return func_graphs; +} + +std::set FuncGraphIndex::GetNodes(const std::string& key) { + if (index_node_.find(key) != index_node_.end()) { + return index_node_[key]; + } + + return std::set(); +} + +FuncGraphPtr FuncGraphIndex::GetFirstFuncGraph(const std::string& key) { + if (GetFuncGraphs(key).empty()) { + return nullptr; + } + + auto fg = *GetFuncGraphs(key).begin(); + return fg; +} + +AnfNodePtr FuncGraphIndex::GetFirstNode(const std::string& key) { + if (GetNodes(key).empty()) { + return nullptr; + } + + auto node = *GetNodes(key).begin(); + return node; +} + +void FuncGraphIndex::Acquire(const FuncGraphPtr& key) { + std::string name = label_manage::Label(key->debug_info()); + if (!name.empty()) { + (void)index_func_graph_[name].insert(key); + } +} + +void FuncGraphIndex::Acquire(const AnfNodePtr& key) { + std::string name = label_manage::Label(key->debug_info()); + if (!name.empty()) { + (void)index_node_[name].insert(key); + } +} + +// Isomorphism +static bool SameNodeShallow(const AnfNodePtr& node1, const AnfNodePtr& node2, FuncGraphPairMapEquiv* equiv_func_graph, + NodeMapEquiv* const equiv_node) { + if (equiv_node == nullptr) { + MS_LOG(ERROR) << "Invalid equiv_node"; + return false; + } + if (equiv_node->count(node1) > 0 && (*equiv_node)[node1] == node2) { + return true; + } + if (IsValueNode(node1) && IsValueNode(node2)) { + return Isomorphic(GetValueNode(node1), GetValueNode(node2), equiv_func_graph, + equiv_node); + } + if (node1->isa() && node2->isa()) { + auto a1 = GetValueNode(node1); + auto a2 = GetValueNode(node2); + if (a1->isa() && a2->isa()) { + return a1->cast()->name() == a2->cast()->name(); + } else { + return *a1 == *a2; + } + } + if (node1->isa() && node2->isa()) { + auto para1 = node1->cast(); + auto para2 = node2->cast(); + if (para1->name() == para2->name()) { + return true; + } + MS_LOG(DEBUG) << "two parameters are not equal."; + return false; + } + MS_LOG(ERROR) << "type error"; + return false; +} + +static bool SameNode(const AnfNodePtr& node1, const AnfNodePtr& node2, FuncGraphPairMapEquiv* equiv_func_graph, + NodeMapEquiv* const equiv_node) { + MS_EXCEPTION_IF_NULL(node1); + MS_EXCEPTION_IF_NULL(node2); + if (node1->isa() && node2->isa()) { + auto& inputs1 = node1->cast()->inputs(); + auto& inputs2 = node2->cast()->inputs(); + for (std::size_t i = 0; i < inputs1.size(); ++i) { + if (!SameNodeShallow(inputs1[i], inputs2[i], equiv_func_graph, equiv_node)) { + return false; + } + } + return true; + } + return SameNodeShallow(node1, node2, equiv_func_graph, equiv_node); +} + +static bool SameSubgraph(AnfNodePtr root1, AnfNodePtr root2, FuncGraphPairMapEquiv* equiv_func_graph, + NodeMapEquiv* const equiv_node) { + std::unordered_set done; + std::stack> todo; + + todo.push(std::make_pair(root1, root2)); + while (todo.size() > 0) { + AnfNodePtr node1 = todo.top().first; + if (done.count(node1) > 0) { + todo.pop(); + continue; + } + AnfNodePtr node2 = todo.top().second; + + bool condition = false; + std::vector s1 = SuccIncoming(node1); + std::vector s2 = SuccIncoming(node2); + + if (s1.size() != s2.size()) { + return false; + } + for (std::size_t i = 0; i < s1.size(); ++i) { + if (done.count(s1[i]) == 0) { + todo.push(std::make_pair(s1[i], s2[i])); + condition = true; + } + } + if (condition) { + continue; + } + (void)done.insert(node1); + + auto res = SameNode(node1, node2, equiv_func_graph, equiv_node); + if (res) { + (*equiv_node)[node1] = node2; + } else { + return false; + } + todo.pop(); + } + return true; +} + +bool Isomorphic(FuncGraphPtr fg1, FuncGraphPtr fg2, FuncGraphPairMapEquiv* equiv_func_graph, + NodeMapEquiv* const equiv_node) { + auto fg1_fg2 = std::make_pair(fg1, fg2); + if (equiv_func_graph == nullptr) { + MS_LOG(ERROR) << "equiv_func_graph not init"; + return false; + } + if (equiv_func_graph->find(fg1_fg2) != equiv_func_graph->end()) { + return (*equiv_func_graph)[fg1_fg2] != kNotEquiv; + } + if (fg1 == nullptr || fg2 == nullptr) { + MS_LOG(ERROR) << "Invalid function graph"; + return false; + } + if (fg1->parameters().size() != fg2->parameters().size()) { + MS_LOG(DEBUG) << "parameters size not match"; + return false; + } + if (equiv_node != nullptr) { + for (std::size_t i = 0; i < fg1->parameters().size(); ++i) { + (*equiv_node)[fg1->parameters()[i]] = fg2->parameters()[i]; + } + (*equiv_func_graph)[fg1_fg2] = kPending; + auto result = SameSubgraph(fg1->get_return(), fg2->get_return(), equiv_func_graph, equiv_node); + (*equiv_func_graph)[fg1_fg2] = EquivState(result); + return result; + } + + MS_LOG(ERROR) << "equiv_node not init"; + return false; +} + +tensor::TensorPtr ScalarToTensor(const ScalarPtr& scalar) { + if (scalar == nullptr) { + MS_EXCEPTION(ArgumentError) << "Nullptr Error!"; + } + tensor::TensorPtr tensor = nullptr; + if (scalar->isa()) { + tensor = std::make_shared(py::float_(GetValue(scalar)), kFloat32); + } else if (scalar->isa()) { + tensor = std::make_shared(py::int_(GetValue(scalar)), kInt32); + } else if (scalar->isa()) { + tensor = std::make_shared(py::array(py::bool_(GetValue(scalar))), kBool); + } else { + auto type = scalar->type(); + auto type_str = (type == nullptr) ? "nullptr" : type->ToString(); + MS_LOG(EXCEPTION) << "Invalid scalar type: " << type_str; + } + MS_EXCEPTION_IF_NULL(tensor); + return tensor; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/graph_utils.h b/mindspore/ccsrc/utils/graph_utils.h new file mode 100644 index 00000000000..57bc0e42fcf --- /dev/null +++ b/mindspore/ccsrc/utils/graph_utils.h @@ -0,0 +1,103 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_GRAPH_UTILS_H_ +#define MINDSPORE_CCSRC_UTILS_GRAPH_UTILS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/primitive.h" +#include "ir/scalar.h" +#include "ir/meta_tensor.h" +#include "debug/label.h" + +namespace mindspore { + +enum IncludeType { FOLLOW, NOFOLLOW, EXCLUDE }; + +using IncludeFunc = std::function; +using SuccFunc = std::function(AnfNodePtr)>; +using SearchFunc = std::function(const AnfNodePtr&, const IncludeFunc&)>; + +std::vector SuccDeeper(const AnfNodePtr& node); +std::vector SuccDeeperSimple(const AnfNodePtr& node); +std::vector SuccIncoming(const AnfNodePtr& node); +std::vector SuccIncludeFV(const FuncGraphPtr& fg, const AnfNodePtr& node); + +IncludeType AlwaysInclude(const AnfNodePtr& node); +IncludeType IncludeBelongGraph(const FuncGraphPtr& fg, const AnfNodePtr& node); + +std::vector DeepScopedGraphSearch(const AnfNodePtr& root, const IncludeFunc& include = AlwaysInclude); +std::vector DeepUsedGraphSearch(const AnfNodePtr& root, const IncludeFunc& include = AlwaysInclude); +std::vector DeepLinkedGraphSearch(const AnfNodePtr& root, const IncludeFunc& include = AlwaysInclude); + +std::vector TopoSort(const AnfNodePtr& root, const SuccFunc& succ = SuccIncoming, + const IncludeFunc& include = AlwaysInclude); + +class FuncGraphIndex { + public: + explicit FuncGraphIndex(const FuncGraphPtr& fg, const SearchFunc& search = DeepScopedGraphSearch, + const IncludeFunc& include = AlwaysInclude); + FuncGraphIndex(const FuncGraphIndex&) = delete; + FuncGraphIndex& operator=(const FuncGraphIndex&) = delete; + + virtual ~FuncGraphIndex() {} + + std::set GetFuncGraphs(const std::string& key); + std::set GetNodes(const std::string& key); + FuncGraphPtr GetFirstFuncGraph(const std::string& key); + AnfNodePtr GetFirstNode(const std::string& key); + + private: + void Acquire(const FuncGraphPtr& key); + void Acquire(const AnfNodePtr& key); + + std::map> index_func_graph_; + std::map> index_node_; +}; + +// Isomorphism + +struct PairHasher { + template + std::size_t operator()(const std::pair& p) const { + auto h1 = std::hash{}(p.first); + auto h2 = std::hash{}(p.second); + return h1 ^ h2; + } +}; + +enum EquivState { kNotEquiv = 0, kEquiv = 1, kPending = 2 }; + +using FuncGraphPairMapEquiv = std::unordered_map, EquivState, PairHasher>; +using NodeMapEquiv = std::unordered_map; + +bool Isomorphic(FuncGraphPtr g1, FuncGraphPtr g2, FuncGraphPairMapEquiv* equiv_func_graph, NodeMapEquiv* equiv_node); + +tensor::TensorPtr ScalarToTensor(const ScalarPtr& scalar); +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_GRAPH_UTILS_H_ diff --git a/mindspore/ccsrc/utils/hashing.h b/mindspore/ccsrc/utils/hashing.h new file mode 100644 index 00000000000..730657ce7a9 --- /dev/null +++ b/mindspore/ccsrc/utils/hashing.h @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_HASHING_H_ +#define MINDSPORE_CCSRC_UTILS_HASHING_H_ + +#include + +namespace mindspore { +inline std::size_t hash_combine(std::size_t hash_sum, std::size_t hash_val) { + // Reference from http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0814r0.pdf + return ((hash_sum << 6) + (hash_sum >> 2) + 0x9e3779b9 + hash_val) ^ hash_sum; +} + +inline std::size_t hash_combine(const std::initializer_list& hash_vals) { + std::size_t hash_sum = 0; + for (auto hash_val : hash_vals) { + hash_sum = hash_combine(hash_sum, hash_val); + } + return hash_sum; +} +} // namespace mindspore +#endif // MINDSPORE_CCSRC_UTILS_HASHING_H_ diff --git a/mindspore/ccsrc/utils/log_adapter.cc b/mindspore/ccsrc/utils/log_adapter.cc new file mode 100644 index 00000000000..19482ec1939 --- /dev/null +++ b/mindspore/ccsrc/utils/log_adapter.cc @@ -0,0 +1,238 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/log_adapter.h" + +#include +#include "pybind11/pybind11.h" + +#include "debug/trace.h" + +// namespace to support utils module definition +namespace mindspore { +#ifdef USE_GLOG +static std::string GetTime() { +#define BUFLEN 80 + struct timeval cur_time; + (void)gettimeofday(&cur_time, NULL); + + struct tm now; + (void)localtime_r(&cur_time.tv_sec, &now); + static char buf[BUFLEN]; + (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time + // set micro-second + buf[27] = '\0'; + int idx = 26; + auto num = cur_time.tv_usec; + for (int i = 5; i >= 0; i--) { + buf[idx--] = static_cast(num % 10 + '0'); + num /= 10; + if (i % 3 == 0) { + buf[idx--] = '.'; + } + } + return std::string(buf); +} + +static std::string GetProcName() { +#if defined(__APPLE__) || defined(__FreeBSD__) + const char *appname = getprogname(); +#elif defined(_GNU_SOURCE) + const char *appname = program_invocation_name; +#else + const char *appname = "?"; +#endif + // some times, the appname is an absolute path, its too long + std::string app_name(appname); + std::size_t pos = app_name.rfind("/"); + if (pos == std::string::npos) { + return app_name; + } + if (pos + 1 >= app_name.size()) { + return app_name; + } + return app_name.substr(pos + 1); +} + +static std::string GetLogLevel(MsLogLevel level) { +#define _TO_STRING(x) #x + static const char *const level_names[] = { + _TO_STRING(DEBUG), + _TO_STRING(INFO), + _TO_STRING(WARNING), + _TO_STRING(ERROR), + }; +#undef _TO_STRING + if (level > ERROR) { + level = ERROR; + } + return std::string(level_names[level]); +} + +// convert MsLogLevel to corresponding glog level +static int GetGlogLevel(MsLogLevel level) { + switch (level) { + case DEBUG: + case INFO: + return google::GLOG_INFO; + case WARNING: + return google::GLOG_WARNING; + case ERROR: + default: + return google::GLOG_ERROR; + } +} +#else +// convert MsLogLevel to corresponding slog level +static int GetSlogLevel(MsLogLevel level) { + switch (level) { + case DEBUG: + return DLOG_DEBUG; + case INFO: + return DLOG_INFO; + case WARNING: + return DLOG_WARN; + case ERROR: + default: + return DLOG_ERROR; + } +} +#endif + +static std::string ExceptionTypeToString(ExceptionType type) { +#define _TO_STRING(x) #x + // clang-format off + static const char *const type_names[] = { + _TO_STRING(NoExceptionType), + _TO_STRING(UnknownError), + _TO_STRING(ArgumentError), + _TO_STRING(NotSupportError), + _TO_STRING(NotExistsError), + _TO_STRING(AlreadyExistsError), + _TO_STRING(UnavailableError), + _TO_STRING(DeviceProcessError), + _TO_STRING(AbortedError), + _TO_STRING(TimeOutError), + _TO_STRING(ResourceUnavailable), + _TO_STRING(NoPermissionError), + _TO_STRING(ValueError), + _TO_STRING(TypeError), + }; + // clang-format on +#undef _TO_STRING + if (type < UnknownError || type > TypeError) { + type = UnknownError; + } + return std::string(type_names[type]); +} + +void LogWriter::OutputLog(const std::ostringstream &msg) const { +#ifdef USE_GLOG + google::LogMessage("", 0, GetGlogLevel(log_level_)).stream() + << "[" << GetLogLevel(log_level_) << "] ME(" << getpid() << "," << GetProcName() << "):" << GetTime() << " " + << "[" << location_.file_ << ":" << location_.line_ << "] " << location_.func_ << "] " << msg.str() << std::endl; +#else + auto str_msg = msg.str(); + Dlog(static_cast(ME), GetSlogLevel(log_level_), "[%s:%d] %s] %s", location_.file_, location_.line_, + location_.func_, str_msg.c_str()); +#endif +} + +void LogWriter::operator<(const LogStream &stream) const noexcept { + std::ostringstream msg; + msg << stream.sstream_->rdbuf(); + OutputLog(msg); +} + +void LogWriter::operator^(const LogStream &stream) const { + std::ostringstream msg; + msg << stream.sstream_->rdbuf(); + OutputLog(msg); + + std::ostringstream oss; + oss << location_.file_ << ":" << location_.line_ << " " << location_.func_ << "] "; + if (exception_type_ != NoExceptionType) { + oss << ExceptionTypeToString(exception_type_) << " "; + } + oss << msg.str(); + + trace::TraceGraphInfer(); + trace::GetInferStackInfo(oss); + + if (exception_type_ == ValueError) { + throw pybind11::value_error(oss.str()); + } + if (exception_type_ == TypeError) { + throw pybind11::type_error(oss.str()); + } + pybind11::pybind11_fail(oss.str()); +} + +static std::string GetEnv(const std::string &envvar) { + const char *value = ::getenv(envvar.c_str()); + + if (value == nullptr) { + return std::string(); + } + + return std::string(value); +} + +#ifndef USE_GLOG +// set default log warning to WARNING +int g_mslog_level = WARNING; + +static void InitMsLogLevel() { + int log_level = WARNING; // set default log level to WARNING + auto str_level = GetEnv("GLOG_v"); + if (str_level.size() == 1) { + int ch = str_level.c_str()[0]; + ch = ch - '0'; // substract ASCII code of '0', which is 48 + if (ch >= DEBUG && ch <= ERROR) { + log_level = ch; + } + } + g_mslog_level = log_level; +} + +#endif + +} // namespace mindspore + +extern "C" { +// shared lib init hook +void mindspore_log_init(void) { +#ifdef USE_GLOG + // do not use glog predefined log prefix + FLAGS_log_prefix = false; + static bool is_glog_initialzed = false; + if (!is_glog_initialzed) { + google::InitGoogleLogging("mindspore"); + is_glog_initialzed = true; + } + // set default log level to WARNING + if (mindspore::GetEnv("GLOG_v").empty()) { + FLAGS_v = mindspore::WARNING; + } + // default print log to screen + if (mindspore::GetEnv("GLOG_logtostderr").empty()) { + FLAGS_logtostderr = true; + } +#else + mindspore::InitMsLogLevel(); +#endif +} +} diff --git a/mindspore/ccsrc/utils/log_adapter.h b/mindspore/ccsrc/utils/log_adapter.h new file mode 100644 index 00000000000..61c253782e4 --- /dev/null +++ b/mindspore/ccsrc/utils/log_adapter.h @@ -0,0 +1,159 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_LOG_ADAPTER_H_ +#define MINDSPORE_CCSRC_UTILS_LOG_ADAPTER_H_ + +#include +#include +#include +#include +#include +#include "./securec.h" +#ifdef USE_GLOG +#include "glog/logging.h" +#else +#include "toolchain/slog.h" +#endif +// NOTICE: when relative path of 'log_adapter.h' changed, macro 'LOG_HDR_FILE_REL_PATH' must be changed +#define LOG_HDR_FILE_REL_PATH "mindspore/ccsrc/utils/log_adapter.h" + +// Get start index of file relative path in __FILE__ +static constexpr int GetRelPathPos() noexcept { + return sizeof(__FILE__) > sizeof(LOG_HDR_FILE_REL_PATH) ? sizeof(__FILE__) - sizeof(LOG_HDR_FILE_REL_PATH) : 0; +} + +namespace mindspore { +#define FILE_NAME \ + (sizeof(__FILE__) > GetRelPathPos() ? static_cast(__FILE__) + GetRelPathPos() \ + : static_cast(__FILE__)) +enum ExceptionType { + NoExceptionType = 0, + UnknownError, + ArgumentError, + NotSupportError, + NotExistsError, + AlreadyExistsError, + UnavailableError, + DeviceProcessError, + AbortedError, + TimeOutError, + ResourceUnavailable, + NoPermissionError, + ValueError, + TypeError, +}; + +struct LocationInfo { + LocationInfo(const char *file, int line, const char *func) : file_(file), line_(line), func_(func) {} + ~LocationInfo() = default; + + const char *file_; + int line_; + const char *func_; +}; + +class LogStream { + public: + LogStream() { sstream_ = std::make_shared(); } + ~LogStream() = default; + + template + LogStream &operator<<(const T &val) noexcept { + (*sstream_) << val; + return *this; + } + + LogStream &operator<<(std::ostream &func(std::ostream &os)) noexcept { + (*sstream_) << func; + return *this; + } + + friend class LogWriter; + + private: + std::shared_ptr sstream_; +}; + +template ::value, int>::type = 0> +constexpr std::ostream &operator<<(std::ostream &stream, const T &value) { + return stream << static_cast::type>(value); +} + +enum MsLogLevel : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION }; + +#ifndef USE_GLOG +extern int g_mslog_level; +#endif + +class LogWriter { + public: + LogWriter(const LocationInfo &location, MsLogLevel log_level, ExceptionType excp_type = NoExceptionType) + : location_(location), log_level_(log_level), exception_type_(excp_type) {} + ~LogWriter() = default; + + void operator<(const LogStream &stream) const noexcept __attribute__((visibility("default"))); + void operator^(const LogStream &stream) const __attribute__((noreturn, visibility("default"))); + + private: + void OutputLog(const std::ostringstream &msg) const; + + LocationInfo location_; + MsLogLevel log_level_; + ExceptionType exception_type_; +}; + +#define MSLOG_IF(level, condition, excp_type) \ + static_cast(0), !(condition) \ + ? void(0) \ + : mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), level, \ + excp_type) < mindspore::LogStream() +#define MSLOG_THROW(excp_type) \ + mindspore::LogWriter(mindspore::LocationInfo(FILE_NAME, __LINE__, __FUNCTION__), mindspore::EXCEPTION, excp_type) ^ \ + mindspore::LogStream() + +#ifdef USE_GLOG +#define IS_OUTPUT_ON(level) (level) >= FLAGS_v +#else +#define IS_OUTPUT_ON(level) (level) >= mindspore::g_mslog_level +#endif + +#define MS_LOG(level) MS_LOG_##level + +#define MS_LOG_DEBUG MSLOG_IF(mindspore::DEBUG, IS_OUTPUT_ON(mindspore::DEBUG), mindspore::NoExceptionType) +#define MS_LOG_INFO MSLOG_IF(mindspore::INFO, IS_OUTPUT_ON(mindspore::INFO), mindspore::NoExceptionType) +#define MS_LOG_WARNING MSLOG_IF(mindspore::WARNING, IS_OUTPUT_ON(mindspore::WARNING), mindspore::NoExceptionType) +#define MS_LOG_ERROR MSLOG_IF(mindspore::ERROR, IS_OUTPUT_ON(mindspore::ERROR), mindspore::NoExceptionType) + +#define MS_LOG_EXCEPTION MSLOG_THROW(mindspore::NoExceptionType) +#define MS_EXCEPTION(type) MSLOG_THROW(type) +} // namespace mindspore + +#define MS_EXCEPTION_IF_NULL(ptr) \ + do { \ + if ((ptr) == nullptr) { \ + MS_LOG(EXCEPTION) << ": The pointer[" << #ptr << "] is null."; \ + } \ + } while (0) + +#ifdef DEBUG +#include +#define MS_ASSERT(f) assert(f) +#else +#define MS_ASSERT(f) ((void)0) +#endif + +#endif // MINDSPORE_CCSRC_UTILS_LOG_ADAPTER_H_ diff --git a/mindspore/ccsrc/utils/misc.cc b/mindspore/ccsrc/utils/misc.cc new file mode 100644 index 00000000000..47e675a3413 --- /dev/null +++ b/mindspore/ccsrc/utils/misc.cc @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/misc.h" + +namespace mindspore { + +const int RET_SUCCESS = 0; +const int RET_FAILED = 1; +const int RET_CONTINUE = 2; +const int RET_BREAK = 3; + +std::string demangle(const char* name) { + int status = -1; + std::unique_ptr res{abi::__cxa_demangle(name, nullptr, nullptr, &status), std::free}; + return (status == 0) ? res.get() : name; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/misc.h b/mindspore/ccsrc/utils/misc.h new file mode 100644 index 00000000000..66e8937f9cc --- /dev/null +++ b/mindspore/ccsrc/utils/misc.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_MISC_H_ +#define MINDSPORE_CCSRC_UTILS_MISC_H_ + +#include +#include +#include +#include +#include + +#include "utils/log_adapter.h" + +namespace mindspore { + +extern const int RET_SUCCESS; +extern const int RET_FAILED; +extern const int RET_CONTINUE; +extern const int RET_BREAK; + +// demangle the name to make it human reablable. +extern std::string demangle(const char* name); + +} // namespace mindspore +#endif // MINDSPORE_CCSRC_UTILS_MISC_H_ diff --git a/mindspore/ccsrc/utils/node_strategy.proto b/mindspore/ccsrc/utils/node_strategy.proto new file mode 100644 index 00000000000..dc06482ba1d --- /dev/null +++ b/mindspore/ccsrc/utils/node_strategy.proto @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package mindspore.straspb; + +message ParallelStrategy { + repeated uint32 dim = 1; +} + +message ParallelStrategys { + required uint32 stage = 1; + repeated ParallelStrategy parallel_strategy = 2; +} + +message ParallelStrategyItem { + required string node_name = 1; + required ParallelStrategys parallel_strategys = 2; +} + +message ParallelStrategyMap { + required uint32 train_time = 1; + repeated ParallelStrategyItem parallel_strategy_item = 2; +} \ No newline at end of file diff --git a/mindspore/ccsrc/utils/ordered_map.h b/mindspore/ccsrc/utils/ordered_map.h new file mode 100644 index 00000000000..48aa36df31f --- /dev/null +++ b/mindspore/ccsrc/utils/ordered_map.h @@ -0,0 +1,199 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_ORDERED_MAP_H_ +#define MINDSPORE_CCSRC_UTILS_ORDERED_MAP_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { +// Implementation of OrderedMap that keeps insertion order +// using unordered_map to improve the performance of find/erase, and use list to keep insertion order +template , class Equal = std::equal_to> +class OrderedMap { + public: + using key_t = KeyT; + using value_t = ValueT; + using hasher = Hash; + using equal = Equal; + using pair_type = std::pair; + using sequential_type = std::list; + using iterator = typename sequential_type::iterator; + using const_iterator = typename sequential_type::const_iterator; + using reverse_iterator = typename sequential_type::reverse_iterator; + using const_reverse_iterator = typename sequential_type::const_reverse_iterator; + using map_type = std::unordered_map; + using value_type = typename sequential_type::value_type; + using size_type = typename sequential_type::size_type; + + iterator begin() { return sequential_data_.begin(); } + iterator end() { return sequential_data_.end(); } + const_iterator begin() const { return sequential_data_.cbegin(); } + const_iterator end() const { return sequential_data_.cend(); } + const_iterator cbegin() const { return sequential_data_.cbegin(); } + const_iterator cend() const { return sequential_data_.cend(); } + + reverse_iterator rbegin() { return sequential_data_.rbegin(); } + reverse_iterator rend() { return sequential_data_.rend(); } + const_reverse_iterator rbegin() const { return sequential_data_.rbegin(); } + const_reverse_iterator rend() const { return sequential_data_.rend(); } + + pair_type &front() { return sequential_data_.front(); } + const pair_type &front() const { return sequential_data_.front(); } + pair_type &back() { return sequential_data_.back(); } + const pair_type &back() const { return sequential_data_.back(); } + + OrderedMap() = default; + ~OrderedMap() = default; + OrderedMap(const OrderedMap &os) { + for (auto &item : os.sequential_data_) { + (void)insert(pair_type(item.first, item.second)); + } + } + + // Explicitly construct OrderedMap use sequential_type + explicit OrderedMap(const sequential_type &other) { + for (auto &item : other) { + (void)insert(pair_type(item.first, item.second)); + } + } + + OrderedMap &operator=(const OrderedMap &os) { + if (this != &os) { + for (auto &item : os.sequential_data_) { + (void)insert(pair_type(item.first, item.second)); + } + } + return *this; + } + + void clear() { + map_data_.clear(); + sequential_data_.clear(); + } + + void swap(OrderedMap &rhs) { + std::swap(map_data_, rhs.map_data_); + std::swap(sequential_data_, rhs.sequential_data_); + } + + void reserve(size_type num_entries) { + map_data_.reserve(num_entries); + sequential_data_.reserve(num_entries); + } + + std::pair add(const key_t &key) { + iterator empty_itr; + std::pair map_pair = std::make_pair(key, empty_itr); + std::pair result = map_data_.insert(map_pair); + auto &seq_itr = result.first->second; + if (result.second) { + auto it = sequential_data_.insert(sequential_data_.end(), std::make_pair(key, ValueT())); + seq_itr = it; + } + return std::pair(seq_itr, result.second); + } + + ValueT &operator[](const key_t &key) { + auto result = add(key); + return (*result.first).second; + } + + std::pair insert(const pair_type &kv) { + auto result = add(kv.first); + if (result.second) { + *(result.first) = kv.second; + return std::make_pair(std::prev(end()), true); + } + return std::make_pair(result.first, false); + } + + std::pair insert(pair_type &&kv) { + iterator empty_itr; + std::pair map_pair = std::make_pair(kv.first, empty_itr); + std::pair result = map_data_.insert(map_pair); + auto &seq_itr = result.first->second; + if (result.second) { + auto it = sequential_data_.insert(sequential_data_.end(), std::move(kv)); + seq_itr = it; + return std::make_pair(std::prev(end()), true); + } + return std::make_pair(seq_itr, false); + } + + bool empty() const { return sequential_data_.empty(); } + + size_type size() const { return sequential_data_.size(); } + + size_type count(const key_t &key) const { + auto pos = map_data_.find(key); + return pos == map_data_.end() ? 0 : 1; + } + + iterator find(const key_t &key) { + typename map_type::const_iterator pos = map_data_.find(key); + return pos == map_data_.end() ? sequential_data_.end() : (pos->second); + } + + const_iterator find(const key_t &key) const { + auto pos = map_data_.find(key); + return pos == map_data_.end() ? sequential_data_.end() : (pos->second); + } + + // Remove the last element from the sequential_data_. + void pop_back() { + typename map_type::iterator pos = map_data_.find(sequential_data_.back().first); + map_data_.erase(pos); + sequential_data_.pop_back(); + } + + // Remove the first element from the sequential_data_. + void pop_front() { + typename map_type::iterator pos = map_data_.find(sequential_data_.first().first); + map_data_.erase(pos); + sequential_data_.pop_front(); + } + + // Remove the element given by Iterator. + typename sequential_type::iterator erase(const typename sequential_type::iterator &itr) { + (void)map_data_.erase(itr->first); + auto next = sequential_data_.erase(itr); + if (next == sequential_data_.end()) return next; + return next; + } + + // Remove the element with the given key + size_type erase(const key_t &key) { + auto itr = find(key); + if (itr == end()) return 0; + (void)erase(itr); + return 1; + } + + private: + map_type map_data_; + sequential_type sequential_data_; +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_ORDERED_MAP_H_ diff --git a/mindspore/ccsrc/utils/ordered_set.h b/mindspore/ccsrc/utils/ordered_set.h new file mode 100644 index 00000000000..b22053f196c --- /dev/null +++ b/mindspore/ccsrc/utils/ordered_set.h @@ -0,0 +1,281 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_ORDERED_SET_H_ +#define MINDSPORE_CCSRC_UTILS_ORDERED_SET_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { + +// Implementation of OrderedSet that keeps insertion order +// using map as set, and use list as a sequential container to record elements to keep insertion order +template , class KeyEqual = std::equal_to> +class OrderedSet { + public: + using element_type = T; + using hasher = Hash; + using equal = KeyEqual; + using sequential_type = std::list; + using vector_type = std::vector; + using iterator = typename sequential_type::iterator; + using const_iterator = typename sequential_type::const_iterator; + using reverse_iterator = typename sequential_type::reverse_iterator; + using const_reverse_iterator = typename sequential_type::const_reverse_iterator; + using map_type = std::unordered_map; + using ordered_set_type = OrderedSet; + + OrderedSet() = default; + ~OrderedSet() = default; + // OrderedSet use an iterator to list as mapped value to improve the performance of insertion and deletion, + // So copy of OrderedSet should re-build value of the map key to make it pointer to the new list,, thus we use + // traversal to build elements. + OrderedSet(const OrderedSet& os) { + for (auto& item : os.ordered_data_) { + add(item); + } + } + + explicit OrderedSet(const sequential_type& other) { + for (auto& item : other) { + add(item); + } + } + + // Explicitly construct an OrderedSet use vector + explicit OrderedSet(const vector_type& other) { + for (auto& item : other) { + add(item); + } + } + + OrderedSet& operator=(const OrderedSet& os) { + if (this != &os) { + for (auto& item : os.ordered_data_) { + add(item); + } + } + return *this; + } + + // Add an element to the OrderedSet, without judging return value + void add(const element_type& e) { (void)insert(e); } + + // insert an element to the OrderedSet + std::pair insert(const element_type& e) { + iterator empty_itr; + std::pair map_pair = std::make_pair(e, empty_itr); + auto result = mapped_data_.insert(map_pair); + auto& seq_idx = result.first->second; + // if insert success; + if (result.second) { + auto it = ordered_data_.insert(ordered_data_.end(), e); + seq_idx = it; + } + return std::pair(seq_idx, result.second); + } + + // Remove an element, if removed return true, otherwise return false + bool erase(const element_type& e) { + auto pos = mapped_data_.find(e); + if (pos == mapped_data_.end()) { + return false; + } + // erase the sequential data first + (void)ordered_data_.erase(pos->second); + (void)mapped_data_.erase(pos); + return true; + } + + // Return the container size + std::size_t size() const { return mapped_data_.size(); } + + bool empty() const { return mapped_data_.size() == 0; } + + // Return the string contents in orderset, using ordered_data + std::string toString() { + std::ostringstream res; + res << "orderset content:\n"; + for (auto& item : ordered_data_) { + res << std::to_string(reinterpret_cast(item.get())) << " "; + } + return res.str(); + } + + // Clear the elements + void clear() { + mapped_data_.clear(); + ordered_data_.clear(); + } + + // Compare two orderedset, if the order is not equal shall return false + bool operator==(const OrderedSet& other) const { return ordered_data_ == other.ordered_data_; } + + // Remove and return the first element in the OrderedSet + T pop() { + if (ordered_data_.size() != 0) { + T res = ordered_data_.front(); + (void)mapped_data_.erase(res); + (void)ordered_data_.erase(ordered_data_.begin()); + return res; + } + MS_LOG(EXCEPTION) << "pop() on empty OrderedSet"; + } + + T back() { + if (ordered_data_.size() != 0) { + return ordered_data_.back(); + } + MS_LOG(EXCEPTION) << "back() on empty OrderedSet"; + } + + // Return true if there are no common elements + bool is_disjoint(const OrderedSet& other) { + for (auto& item : other.ordered_data_) { + if (mapped_data_.find(item) != mapped_data_.end()) { + return false; + } + } + return true; + } + + // Test whether this is subset of other + bool is_subset(const OrderedSet& other) { + for (auto& item : ordered_data_) { + if (other.mapped_data_.find(item) == other.mapped_data_.end()) { + return false; + } + } + return true; + } + + // Add elements in other to this orderedset + void update(const OrderedSet& other) { + for (auto& item : other.ordered_data_) { + add(item); + } + } + + void update(const std::shared_ptr& other) { update(*other); } + + void update(const sequential_type& other) { + for (auto& item : other) { + add(item); + } + } + + void update(const vector_type& other) { + for (auto& item : other) { + add(item); + } + } + + ordered_set_type get_union(const OrderedSet& other) { + ordered_set_type res(ordered_data_); + res.update(other); + return res; + } + + // Get the union with other set, this operator may cost time because of copy + ordered_set_type operator|(const OrderedSet& other) { return get_union(other); } + + // Return the intersection of two sets + ordered_set_type intersection(const OrderedSet& other) { + ordered_set_type res(ordered_data_); + for (auto& item : ordered_data_) { + if (other.mapped_data_.find(item) == other.mapped_data_.end()) { + (void)res.erase(item); + } + } + return res; + } + ordered_set_type operator&(const OrderedSet& other) { return intersection(other); } + + // Return the symmetric difference of two sets + ordered_set_type symmetric_difference(const OrderedSet& other) { + ordered_set_type res(ordered_data_); + for (auto& item : other.ordered_data_) { + if (mapped_data_.find(item) != mapped_data_.end()) { + (void)res.erase(item); + } else { + res.add(item); + } + } + return res; + } + + ordered_set_type operator^(const OrderedSet& other) { return symmetric_difference(other); } + + // Remove elements which is also in others. + void difference_update(const OrderedSet& other) { + // use vector traversal, to keep ordrer + for (auto& item : other.ordered_data_) { + (void)erase(item); + } + } + + void difference_update(const sequential_type& other) { + for (auto& item : other) { + (void)erase(item); + } + } + + void difference_update(const vector_type& other) { + for (auto& item : other) { + (void)erase(item); + } + } + + // Return the set with elements that are not in the others + ordered_set_type difference(const OrderedSet& other) { + ordered_set_type res(ordered_data_); + res.difference_update(other); + return res; + } + ordered_set_type operator-(const OrderedSet& other) { return difference(other); } + + bool contains(const element_type& e) const { return (mapped_data_.find(e) != mapped_data_.end()); } + + // Return the count of an element in set + std::size_t count(const element_type& e) const { return mapped_data_.count(e); } + + iterator begin() { return ordered_data_.begin(); } + iterator end() { return ordered_data_.end(); } + + const_iterator begin() const { return ordered_data_.cbegin(); } + const_iterator end() const { return ordered_data_.cend(); } + + const_iterator cbegin() const { return ordered_data_.cbegin(); } + const_iterator cend() const { return ordered_data_.cend(); } + + private: + map_type mapped_data_; + sequential_type ordered_data_; +}; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_ORDERED_SET_H_ diff --git a/mindspore/ccsrc/utils/overload.h b/mindspore/ccsrc/utils/overload.h new file mode 100644 index 00000000000..a95e285fc72 --- /dev/null +++ b/mindspore/ccsrc/utils/overload.h @@ -0,0 +1,140 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_OVERLOAD_H_ +#define MINDSPORE_CCSRC_UTILS_OVERLOAD_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mindspore { + +template +std::ostream &operator<<(std::ostream &out, const std::vector &v) { + out << "[const vector]["; + size_t last = v.size() - 1; + for (size_t i = 0; i < v.size(); ++i) { + out << v[i]; + if (i != last) out << ", "; + } + out << "]"; + return out; +} + +template +std::ostream &operator<<(std::ostream &os, const std::list &vec) { + bool begin = true; + os << "[const list]["; + for (auto &item : vec) { + if (!begin) { + os << ", "; + } else { + begin = false; + } + os << item; + } + os << "]"; + + return os; +} + +template +std::ostream &operator<<(std::ostream &os, const std::initializer_list &vec) { + bool begin = true; + os << "["; + for (auto &item : vec) { + if (!begin) { + os << ", "; + } else { + begin = false; + } + os << item; + } + os << "]"; + + return os; +} + +template +bool operator==(const std::initializer_list &lhs, const std::initializer_list &rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + auto lit = lhs.begin(); + auto rit = rhs.begin(); + while (lit != lhs.end()) { + if (!(*lit == *rit)) { + return false; + } + lit++; + rit++; + } + return true; +} + +template +std::ostream &operator<<(std::ostream &os, const std::pair &pair) { + os << "[const pair]"; + + return os; +} + +template +std::ostream &operator<<(std::ostream &os, const std::unordered_map &map) { + os << "[const unordered_map]"; + return os; +} + +template +std::ostream &operator<<(std::ostream &os, const std::map &map) { + os << "[const map]"; + return os; +} + +template +std::string ToString(const std::vector &vec) { + std::ostringstream buffer; + + buffer << vec; + return buffer.str(); +} + +template +std::string ToString(const std::unordered_map &map) { + std::ostringstream buffer; + + buffer << map; + return buffer.str(); +} + +template +std::string ToString(const std::map &map) { + std::ostringstream buffer; + + buffer << map; + return buffer.str(); +} + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_OVERLOAD_H_ diff --git a/mindspore/ccsrc/utils/profile.cc b/mindspore/ccsrc/utils/profile.cc new file mode 100644 index 00000000000..7a2bb2aa666 --- /dev/null +++ b/mindspore/ccsrc/utils/profile.cc @@ -0,0 +1,347 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/profile.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { + +namespace { +const char KEY_PROF_TOTAL[] = "__total__"; + +void PrintProfile(std::ostringstream& oss, const TimeInfo& time_info, int indent = 0, + std::map* sums = nullptr, const std::string& prefix = ""); + +void PrintTimeInfoMap(std::ostringstream& oss, const TimeInfoMap& dict, int indent = 0, + std::map* sums = nullptr, const std::string& prefix = "") { + for (auto iter = dict.begin(); iter != dict.end(); ++iter) { + if (iter->second == nullptr) { + continue; + } + // indent by multiples of 4 spaces. + oss << std::setw(indent * 4) << "" + << "[" << iter->first << "]: " << iter->second->time_; + if (iter->second->dict_ != nullptr) { + oss << ", [" << iter->second->dict_->size() << "]"; + } + oss << "\n"; + + std::string newPrefix = prefix; + if (iter->first.find("Cycle ") != 0) { + newPrefix = prefix.empty() ? iter->first : prefix + "." + iter->first; + } + PrintProfile(oss, *iter->second, indent + 1, sums, newPrefix); + if (iter->second->dict_ == nullptr) { + (*sums)[newPrefix] += iter->second->time_; + } + } +} + +void PrintProfile(std::ostringstream& oss, const TimeInfo& time_info, int indent, std::map* sums, + const std::string& prefix) { + bool need_free = false; + if (sums == nullptr) { + sums = new (std::nothrow) std::map(); + if (sums == nullptr) { + MS_LOG(ERROR) << "memory allocation failed"; + return; + } + need_free = true; + } + + // indent by multiples of 4 spaces. + if (indent == 0) { + oss << "TotalTime = " << time_info.time_; + if (time_info.dict_ != nullptr) { + oss << ", [" << time_info.dict_->size() << "]"; + } + oss << "\n"; + } + + if (time_info.dict_ != nullptr) { + PrintTimeInfoMap(oss, *time_info.dict_, indent, sums, prefix); + } + + // print time percentage info + if (need_free) { + double total = 0.0; + for (auto iter = sums->begin(); iter != sums->end(); ++iter) { + total += iter->second; + } + oss << "Sums\n"; + if (total >= 0.0 + DBL_EPSILON) { + for (auto& iter : *sums) { + oss << " " << std::left << std::setw(36) << iter.first << " : " << std::right << std::setw(12) << std::fixed + << std::setprecision(6) << iter.second << "s : " << std::right << std::setw(5) << std::fixed + << std::setprecision(2) << iter.second / total * 100 << "%\n"; + } + } + delete sums; + } +} +} // namespace + +double GetTime(void) { + struct timeval tv = {0, 0}; + (void)gettimeofday(&tv, nullptr); + return tv.tv_sec + tv.tv_usec * 1.0e-6; +} + +TimeInfo::~TimeInfo() { + if (dict_ == nullptr) { + return; + } + for (auto iter = dict_->begin(); iter != dict_->end(); ++iter) { + delete iter->second; + iter->second = nullptr; + } + delete dict_; + dict_ = nullptr; +} + +ProfileBase::ProfileBase() : context_("", this) { + ctx_ptr_ = &context_; + context_.parent_ = nullptr; +} + +ProfileBase::~ProfileBase() { + context_.parent_ = nullptr; + if (context_.time_info_ != nullptr) { + delete context_.time_info_; + context_.time_info_ = nullptr; + } + ctx_ptr_ = nullptr; +} + +void Profile::Print(void) { + if (ctx_ptr_ == nullptr || ctx_ptr_->time_info_ == nullptr) { + return; + } + std::ostringstream oss; + PrintProfile(oss, *ctx_ptr_->time_info_); + std::string text = oss.str(); + // the length of text is too long to use MS_LOGINFO, use printf to print it + (void)printf("%s", text.c_str()); + (void)fflush(stdout); +} + +// Start a step in the current context with the given name. +// Nomes must be unique otherwise the previous record will be overwritten. +ProfContext* Profile::Step(const std::string& name) { + ctx_ptr_ = new (std::nothrow) ProfContext(name, this); + if (ctx_ptr_ == nullptr) { + MS_LOG(ERROR) << "memory allocation failed"; + return nullptr; + } + return ctx_ptr_; +} + +// Creates subcontext for a repeated action. +// Count should be monotonically increasing. +ProfContext* Profile::Lap(int count) { + std::ostringstream oss; + oss << "Cycle " << count; + ctx_ptr_ = new (std::nothrow) ProfContext(oss.str(), this); + if (ctx_ptr_ == nullptr) { + MS_LOG(ERROR) << "memory allocation failed"; + return nullptr; + } + return ctx_ptr_; +} + +void Profile::Pop(void) noexcept { + if (ctx_ptr_ == nullptr) { + return; + } + ctx_ptr_ = ctx_ptr_->parent_; +} + +ProfContext::ProfContext(const std::string& name, ProfileBase* const prof) : name_(name), prof_(prof) { + // Initialize a subcontext. + time_info_ = nullptr; + if (prof == nullptr || IsTopContext()) { + parent_ = nullptr; + } else { + parent_ = prof->ctx_ptr_; + } +} + +ProfContext::~ProfContext() { + // top level context + if (parent_ == nullptr || IsTopContext()) { + if (time_info_ != nullptr) { + delete time_info_; + } + } else { + parent_->Insert(name_, time_info_); + if (prof_ != nullptr) { + prof_->Pop(); + } + } + + time_info_ = nullptr; + prof_ = nullptr; + parent_ = nullptr; +} + +void ProfContext::SetTime(double time) noexcept { + if (time_info_ == nullptr) { + time_info_ = new (std::nothrow) TimeInfo(time); + if (time_info_ == nullptr) { + MS_LOG(ERROR) << "memory allocation failed"; + return; + } + } + time_info_->time_ = time; +} + +void ProfContext::Insert(const std::string& name, const TimeInfo* time) noexcept { + if (time_info_ == nullptr) { + time_info_ = new (std::nothrow) TimeInfo(); + if (time_info_ == nullptr) { + MS_LOG(ERROR) << "memory allocation failed"; + delete time; + time = nullptr; + return; + } + } + + if (time_info_->dict_ == nullptr) { + time_info_->dict_ = new (std::nothrow) TimeInfoMap(); + if (time_info_->dict_ == nullptr) { + MS_LOG(ERROR) << "memory allocation failed"; + delete time; + time = nullptr; + delete time_info_; + time_info_ = nullptr; + return; + } + } + + auto iter = time_info_->dict_->find(name); + // if contains item with same name, delete it + if (iter != time_info_->dict_->end()) { + delete iter->second; + iter->second = nullptr; + (void)time_info_->dict_->erase(iter); + } + (*time_info_->dict_)[name] = time; +} + +bool ProfContext::IsTopContext() const noexcept { return (prof_ != nullptr) && (this == &prof_->context_); } + +ProfTransaction::ProfTransaction(const ProfileBase* prof) { ctx_ = (prof != nullptr ? prof->ctx_ptr_ : nullptr); } + +ProfTransaction::~ProfTransaction() { + if (ctx_ != nullptr && !ctx_->IsTopContext()) { + delete ctx_; + } + ctx_ = nullptr; +} + +void DumpTime::Record(const std::string& step_name, const double time, const bool is_start) { + file_ss_ << " {" << std::endl; + file_ss_ << " \"name\": " + << "\"" << step_name << "\"," << std::endl; + file_ss_ << " \"cat\": " + << "\"FUNCTION\"," << std::endl; + if (is_start) { + file_ss_ << " \"ph\": " + << "\"B\"," << std::endl; + } else { + file_ss_ << " \"ph\": " + << "\"E\"," << std::endl; + } + file_ss_ << " \"ts\": " << std::setprecision(16) << time * 1000000 << "," << std::endl; + file_ss_ << " \"pid\": " + << "1" << std::endl; + file_ss_ << " }" << std::endl; + file_ss_ << " ," << std::endl; +} + +void DumpTime::Save() { + try { + file_out_.open(file_path_, std::ios::trunc | std::ios::out); + } catch (const std::exception& e) { + MS_LOG(EXCEPTION) << "Cannot open file in " << (file_path_); + } + file_out_ << "{\n"; + file_out_ << " \"traceEvents\": [" << std::endl; + file_ss_ >> file_out_.rdbuf(); + (void)file_out_.seekp(-7, std::ios::end); + file_out_ << " ]" << std::endl << " ,\n"; + file_out_ << " \"displayTimeUnit\": \"ms\"" << std::endl; + file_out_ << "}"; + file_out_.close(); +} + +struct TimeInfoGroup { + double total_time = 0.0; + int total_count = 0; + std::list::const_iterator> items; +}; + +static void PrintTimeStat(std::ostringstream& oss, const TimeInfoGroup& group, const std::string& prefix) { + oss << "------[" << prefix << "] " << std::setw(10) << std::fixed << std::setprecision(6) << group.total_time + << std::setw(6) << group.total_count << "\n"; + for (const auto& iter : group.items) { + oss << std::setw(5) << std::fixed << std::setprecision(2) << iter->second.time_ / group.total_time * 100 + << "% : " << std::setw(12) << std::fixed << std::setprecision(6) << iter->second.time_ << "s : " << std::setw(6) + << iter->second.count_ << ": " << iter->first << "\n"; + } +} + +void MsProfile::Print() { + GetProfile()->Print(); + std::vector items = {"substitution.", "renormalize.", "replace.", "match.", + "func_graph_cloner_run.", "meta_graph.", "manager."}; + std::vector groups(items.size() + 1); + const auto& stat = GetSingleton().time_stat_; + // group all time infos + for (auto iter = stat.cbegin(); iter != stat.cend(); ++iter) { + auto matched_idx = items.size(); + for (size_t i = 0; i < items.size(); ++i) { + if (iter->first.find(items[i]) != std::string::npos) { + matched_idx = i; + break; + } + } + groups[matched_idx].total_time += iter->second.time_; + groups[matched_idx].total_count += iter->second.count_; + groups[matched_idx].items.push_back(iter); + } + std::ostringstream oss; + for (size_t i = 0; i < groups.size(); ++i) { + std::string prefix = (i < items.size() ? items[i] : std::string("others.")); + PrintTimeStat(oss, groups[i], prefix); + } + std::string text = oss.str(); + // the length of text is too long to use MS_LOGINFO, use printf to print it + (void)printf("\nTime group info:\n%s", text.c_str()); + (void)fflush(stdout); +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/profile.h b/mindspore/ccsrc/utils/profile.h new file mode 100644 index 00000000000..4824f1f6abe --- /dev/null +++ b/mindspore/ccsrc/utils/profile.h @@ -0,0 +1,233 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_PROFILE_H_ +#define MINDSPORE_CCSRC_UTILS_PROFILE_H_ + +#include +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace mindspore { + +struct TimeInfo; +using TimeInfoMap = std::map; + +extern double GetTime(); + +class ProfileBase; + +struct TimeInfo { + explicit TimeInfo(double time = -1.0) : time_(time), dict_(nullptr) {} + TimeInfo(const TimeInfo&) = delete; + ~TimeInfo(); + + double time_; + TimeInfoMap* dict_; +}; + +// Utility class for Profile. +class ProfContext { + friend class Profile; + friend class ProfileBase; + friend class ProfTransaction; + + public: + ProfContext(const std::string& name, ProfileBase* prof); + ~ProfContext(); + + ProfContext(const ProfContext&) = delete; + ProfContext& operator=(const ProfContext&) = delete; + + void SetTime(double time) noexcept; + void Insert(const std::string& name, const TimeInfo* time) noexcept; + bool IsTopContext() const noexcept; + + private: + std::string name_; + ProfileBase* prof_; + ProfContext* parent_; + TimeInfo* time_info_; +}; + +class ProfileBase { + friend class ProfContext; + friend class ProfTransaction; + + public: + ProfileBase(); + virtual ~ProfileBase(); + + virtual void Print(void) {} + virtual ProfContext* Step(const std::string&) { return nullptr; } + virtual ProfContext* Lap(int) { return nullptr; } + virtual void Pop(void) {} + + // top level profile context + ProfContext context_; + // profile context pointer, act as a stack pointer + ProfContext* ctx_ptr_ = nullptr; +}; + +class Profile : public ProfileBase { + public: + Profile() = default; + ~Profile() override = default; + Profile(const Profile&) = delete; + Profile& operator=(const Profile&) = delete; + + void Print(void) override; + ProfContext* Step(const std::string& name) override; + ProfContext* Lap(int count) override; + void Pop(void) noexcept override; +}; + +class ProfTransaction { + public: + explicit ProfTransaction(const ProfileBase* prof); + explicit ProfTransaction(ProfContext* const ctx) : ctx_(ctx) {} + ProfTransaction(const ProfTransaction&) = delete; + ~ProfTransaction(); + + template + void operator-(const Function& func) { + double start_time = GetTime(); + func(); + double end_time = GetTime(); + if (ctx_ != nullptr) { + ctx_->SetTime(end_time - start_time); + } + } + + private: + ProfContext* ctx_ = nullptr; +}; + +class NoProfTransaction { + public: + explicit NoProfTransaction(ProfileBase* prof) {} + explicit NoProfTransaction(ProfContext* ctx) {} + ~NoProfTransaction() = default; + + template + void operator-(const Function& func) { + func(); + } +}; + +class DumpTime { + public: + ~DumpTime() { + try { + Save(); + } catch (const std::exception& e) { + MS_LOG(ERROR) << "Cannot save file by profile::DumpTime::save"; + } catch (...) { + MS_LOG(ERROR) << "Uncaught exception"; + } + } + DumpTime(const DumpTime&) = delete; + DumpTime& operator=(const DumpTime&) = delete; + static DumpTime& GetInstance() { + static DumpTime instance; + return instance; + } + void set_file_path(const std::string& save_path) { file_path_ = save_path; } + void Record(const std::string& name, const double time, const bool is_start); + void Save(); + + private: + DumpTime() = default; + std::stringstream file_ss_; + std::ofstream file_out_; + std::string file_path_ = "./timeline.json"; +}; + +struct TimeStat { + TimeStat() { + time_ = 0.0; + count_ = 0; + } + ~TimeStat() = default; + + void operator+=(double t) { + time_ += t; + count_ += 1; + } + + TimeStat operator+(double t) { + TimeStat ts = *this; + ts += t; + return ts; + } + + double time_; + int count_; +}; + +class MsProfile { + public: + ~MsProfile() { Clear(); } + + static void Reset() { GetSingleton().Clear(); } + + static ProfileBase* GetProfile() { + MsProfile& ms_prof = GetSingleton(); + if (ms_prof.profile_ == nullptr) { +#ifdef ENABLE_PROFILE + ms_prof.profile_ = new Profile(); +#else + ms_prof.profile_ = new ProfileBase(); +#endif + } + return ms_prof.profile_; + } + static void StatTime(const std::string& id, double time) { GetSingleton().time_stat_[id] += time; } + + static void Print(); + + private: + MsProfile() = default; + + static MsProfile& GetSingleton() { + static MsProfile profile; + return profile; + } + + void Clear() { + time_stat_.clear(); + if (profile_ != nullptr) { + delete profile_; + profile_ = nullptr; + } + } + + std::map time_stat_; // record time and count info from some activity + ProfileBase* profile_ = nullptr; // record hierarchical profile info +}; + +} // namespace mindspore + +#ifdef ENABLE_PROFILE +#define WITH(x) ProfTransaction(x) - +#else +#define WITH(x) NoProfTransaction(x) - +#endif + +#endif // MINDSPORE_CCSRC_UTILS_PROFILE_H_ diff --git a/mindspore/ccsrc/utils/signal.h b/mindspore/ccsrc/utils/signal.h new file mode 100644 index 00000000000..af7b36a8b5d --- /dev/null +++ b/mindspore/ccsrc/utils/signal.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_SIGNAL_H_ +#define MINDSPORE_CCSRC_UTILS_SIGNAL_H_ + +#include +#include +#include +#include + +namespace mindspore { +template +std::function bind_member(Type* instance, Return (Type::*method)(Args...)) { + return [=](Args&&... args) -> Return { return (instance->*method)(std::forward(args)...); }; +} + +template +class Slot { + public: + explicit Slot(const std::function& callback) : callback(callback) {} + + ~Slot() {} + + std::function callback = nullptr; +}; + +template +class Signal { + public: + template + void operator()(Args&&... args) { + for (auto& slot : slots_) { + if (slot->callback != nullptr) { + slot->callback(std::forward(args)...); + } + } + } + + void add_slot(const std::function& func) { + auto slot = std::make_shared>(func); + slots_.push_back(slot); + } + + // signal connect to a class member func + template + void connect(InstanceType instance, MemberFuncType func) { + add_slot(bind_member(instance, func)); + } + + private: + std::vector>> slots_; +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_EVENT_H_ diff --git a/mindspore/ccsrc/utils/summary.proto b/mindspore/ccsrc/utils/summary.proto new file mode 100644 index 00000000000..f7fb7335970 --- /dev/null +++ b/mindspore/ccsrc/utils/summary.proto @@ -0,0 +1,78 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package mindspore.irpb; +option cc_enable_arenas = true; + +// The ANF IR define, include the tensor and graph define +import "anf_ir.proto"; + +// Event Protocol buffer, Top define +message Event { + // Timestamp + required double wall_time = 1; + + // The step of train. + optional int64 step = 2; + + oneof what { + // An event file was started, with the specified version. + // Now version is "Mindspore.Event:1" + string version = 3; + + // GraphDef. + GraphProto graph_def = 4; + + // Summary data + Summary summary = 5; + } +} + +// A Summary is a set of named values that be produced regularly during training +message Summary { + message Image { + // Dimensions of the image. + required int32 height = 1; + required int32 width = 2; + // Valid colorspace values are: + // 1 - grayscale type + // 2 - grayscale + alpha type + // 3 - RGB type + // 4 - RGBA type + // 5 - DIGITAL_YUV type + // 6 - BGRA type + required int32 colorspace = 3; + // Image data in encoded format. Now only support the RGB. + required bytes encoded_image = 4; + } + + message Value { + // Tag name for the data. + required string tag = 1; + + // Value associated with the tag. + oneof value { + float scalar_value = 3; + Image image = 4; + TensorProto tensor = 8; + } + } + + // Set of values for the summary. + repeated Value value = 1; +} diff --git a/mindspore/ccsrc/utils/summary/event_writer.cc b/mindspore/ccsrc/utils/summary/event_writer.cc new file mode 100644 index 00000000000..b540ecd58b6 --- /dev/null +++ b/mindspore/ccsrc/utils/summary/event_writer.cc @@ -0,0 +1,189 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/summary/event_writer.h" +#include +#include +#include +#include "utils/log_adapter.h" +#include "utils/convert_utils.h" + +namespace mindspore { +namespace summary { + +// implement the EventWriter +EventWriter::EventWriter(const std::string &file_full_name) : filename_(file_full_name), events_write_count_(0) { + fs_ = system::Env::GetFileSystem(); + if (fs_ == nullptr) { + MS_LOG(EXCEPTION) << "Get the file system failed."; + } + event_file_ = fs_->CreateWriteFile(filename_); + if (event_file_ == nullptr) { + MS_LOG(EXCEPTION) << "Create the event file(" << file_full_name << ") failed."; + } + // set the event writer status + status_ = true; +} + +EventWriter::~EventWriter() { + if (event_file_ != nullptr) { + bool result = Close(); + if (!result) { + MS_LOG(ERROR) << "Close file(" << filename_ << ") failed."; + } + } +} + +// get the write event count +int32_t EventWriter::GetWriteEventCount() const { return events_write_count_; } + +// Open the file +bool EventWriter::Open() { + if (event_file_ == nullptr) { + MS_LOG(ERROR) << "Open the file(" << filename_ << ") failed."; + return false; + } + bool result = event_file_->Open(); + if (!result) { + MS_LOG(ERROR) << "Open the file(" << filename_ << ") failed."; + } + return result; +} + +// write the event serialization string to file +void EventWriter::Write(const std::string &event_str) { + if (event_file_ == nullptr) { + MS_LOG(ERROR) << "Write failed because file could not be opened."; + return; + } + events_write_count_++; + bool result = WriteRecord(event_str); + if (!result) { + MS_LOG(ERROR) << "Event write failed."; + } +} + +bool EventWriter::Flush() { + // Confirm the event file is exist? + if (!fs_->FileExist(filename_)) { + MS_LOG(ERROR) << "Failed to flush to file(" << filename_ << ") because the file not exist."; + return false; + } + if (event_file_ == nullptr) { + MS_LOG(ERROR) << "Can't flush because the event file is null."; + return false; + } + // Sync the file + if (!event_file_->Flush()) { + MS_LOG(ERROR) << "Failed to sync to file(" << filename_ << "), the event count(" << events_write_count_ << ")."; + return false; + } + MS_LOG(DEBUG) << "Flush " << events_write_count_ << " events to disk file(" << filename_ << ")."; + return true; +} + +bool EventWriter::Close() noexcept { + MS_LOG(DEBUG) << "Close the event writer."; + bool result = true; + if (!status_) { + MS_LOG(INFO) << "The event writer is closed."; + return result; + } + if (event_file_ != nullptr) { + result = event_file_->Close(); + if (!result) { + MS_LOG(ERROR) << "Close the file(" << filename_ << ") failed."; + } + } + return result; +} + +bool EventWriter::Shut() noexcept { + MS_LOG(DEBUG) << "ShutDown the event writer."; + if (!status_) { + MS_LOG(INFO) << "The event writer is closed."; + return true; + } + bool result = Flush(); + if (!result) { + MS_LOG(ERROR) << "Flush failed when close the file."; + } + if (event_file_ != nullptr) { + bool _close = event_file_->Close(); + if (!_close) { + MS_LOG(ERROR) << "Close the file(" << filename_ << ") failed."; + result = _close; + } + } + events_write_count_ = 0; + status_ = false; + return result; +} + +// Summary Record Format: +// 1 uint64 : data length +// 2 uint32 : mask crc value of data length +// 3 bytes : data +// 4 uint32 : mask crc value of data +bool EventWriter::WriteRecord(const std::string &data) { + if (event_file_ == nullptr) { + MS_LOG(ERROR) << "Writer not initialized or previously closed."; + return false; + } + // Write the data to event file + const unsigned int kArrayLen = sizeof(uint64_t); + char data_len_array[kArrayLen]; + char crc_array[sizeof(uint32_t)]; + + // step 1: write the data length + system::EncodeFixed64(data_len_array, kArrayLen, static_cast(data.size())); + bool result = event_file_->Write(string(data_len_array, sizeof(data_len_array))); + if (!result) { + MS_LOG(ERROR) << "Write the Summary data length failed."; + return false; + } + + // step 2: write the crc of data length + system::EncodeFixed64(data_len_array, kArrayLen, SizeToInt(data.size())); + uint32_t crc = system::Crc32c::GetMaskCrc32cValue(data_len_array, sizeof(data_len_array)); + system::EncodeFixed32(crc_array, crc); + result = event_file_->Write(string(crc_array, sizeof(crc_array))); + if (!result) { + MS_LOG(ERROR) << "Write the Summary data length crc failed."; + return false; + } + + // step 3: write the data + result = event_file_->Write(data); + if (!result) { + MS_LOG(ERROR) << "Write the Summary data failed."; + return false; + } + + // step 4: write data crc + crc = system::Crc32c::GetMaskCrc32cValue(data.data(), data.size()); + system::EncodeFixed32(crc_array, crc); + result = event_file_->Write(string(crc_array, sizeof(crc_array))); + if (!result) { + MS_LOG(ERROR) << "Write the Summary footer failed."; + return false; + } + + return true; +} + +} // namespace summary +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/summary/event_writer.h b/mindspore/ccsrc/utils/summary/event_writer.h new file mode 100644 index 00000000000..921119ffcb4 --- /dev/null +++ b/mindspore/ccsrc/utils/summary/event_writer.h @@ -0,0 +1,86 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SUMMARY_EVENT_WRITER_H_ +#define SUMMARY_EVENT_WRITER_H_ + +#include +#include + +#include "pybind11/pybind11.h" +#include "securec/include/securec.h" +#include "utils/system/base.h" +#include "utils/system/file_system.h" +#include "utils/system/crc32c.h" +#include "utils/system/env.h" + +namespace mindspore { +namespace summary { +namespace py = pybind11; +using string = std::string; +using Env = system::Env; +using WriteFile = system::WriteFile; +using WriteFilePtr = std::shared_ptr; +using FileSystem = system::FileSystem; + +class EventWriter { + public: + // The file name = path + file_name + explicit EventWriter(const std::string &file_full_name); + + ~EventWriter(); + + // return the file name + std::string GetFileName() const { return filename_; } + + // return the count of write event + int32_t GetWriteEventCount() const; + + // Open the file + bool Open(); + + // write the Serialized "event_str" to file + void Write(const std::string &event_str); + + // Flush the cache to disk + bool Flush(); + + // close the file + bool Close() noexcept; + + // Final close: flush and close the event writer and clean + bool Shut() noexcept; + + // Summary Record Format: + // 1 uint64 : data length + // 2 uint32 : mask crc value of data length + // 3 bytes : data + // 4 uint32 : mask crc value of data + bool WriteRecord(const std::string &data); + + private: + // True: valid / False: closed + bool status_ = false; + std::shared_ptr fs_; + std::string filename_; + WriteFilePtr event_file_; + int32_t events_write_count_ = 0; +}; + +} // namespace summary +} // namespace mindspore + +#endif // SUMMARY_EVENT_WRITER_H_ diff --git a/mindspore/ccsrc/utils/symbolic.cc b/mindspore/ccsrc/utils/symbolic.cc new file mode 100644 index 00000000000..8764678288d --- /dev/null +++ b/mindspore/ccsrc/utils/symbolic.cc @@ -0,0 +1,54 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/symbolic.h" + +#include + +namespace mindspore { + +std::ostream& operator<<(std::ostream& out, const std::shared_ptr& objPtr) { + out << "("; + MS_EXCEPTION_IF_NULL(objPtr); + for (auto& iter : objPtr->contents_) { + out << iter.first << ":" << iter.second << ";"; + } + out << ")"; + return out; +} + +bool EnvInstance::operator==(const EnvInstance& other) const { + if (Len() != other.Len()) { + return false; + } + bool equal = std::all_of(contents_.begin(), contents_.end(), + [&other](const std::pair& item) -> bool { + return other.contents_.find(item.first) != other.contents_.end(); + }); + return equal; +} +bool EnvInstance::operator==(const Value& other) const { + if (other.isa()) { + auto other_env_inst = static_cast(&other); + return *this == *other_env_inst; + } + return false; +} +std::shared_ptr newenv = std::make_shared(); + +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/symbolic.h b/mindspore/ccsrc/utils/symbolic.h new file mode 100644 index 00000000000..3c712483ee8 --- /dev/null +++ b/mindspore/ccsrc/utils/symbolic.h @@ -0,0 +1,176 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_SYMBOLIC_H_ +#define MINDSPORE_CCSRC_UTILS_SYMBOLIC_H_ + +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { + +class SymbolicKeyInstance : public Value { + public: + SymbolicKeyInstance(const AnfNodePtr& node, const abstract::AbstractBasePtr& abstract) + : node_(node), abstract_(abstract) {} + ~SymbolicKeyInstance() override = default; + MS_DECLARE_PARENT(SymbolicKeyInstance, Value); + AnfNodePtr node() const { return node_; } + abstract::AbstractBasePtr abstract() const { return abstract_; } + bool operator==(const SymbolicKeyInstance& other) const { + return (*node_ == *other.node_) && (*abstract_ == *other.abstract_); + } + + std::size_t hash() const override { return std::hash{}(node_); } + friend std::ostream& operator<<(std::ostream& os, const std::shared_ptr& inst) { + if (inst == nullptr) { + os << "[Key][" + << "Invalid symbolic key instance" + << "]"; + } else { + os << "[Key][" << inst->node_->type_name() << "]" << inst->node_->ToString(); + } + return os; + } + std::string ToString() const override { + return node_ == nullptr ? "Invalid node" : "[Key][" + node_->type_name() + "]" + node_->ToString(); + } + bool operator==(const Value& other) const override { + if (other.isa()) { + auto other_ = static_cast(other); + return *this == other_; + } else { + return false; + } + } + abstract::AbstractBasePtr ToAbstract() override { + return std::make_shared(shared_from_base(), + std::make_shared()); + } + + private: + AnfNodePtr node_; + abstract::AbstractBasePtr abstract_; +}; + +using SymbolicKeyInstancePtr = std::shared_ptr; + +struct SymbolicKeyInstanceHash { + std::size_t operator()(const SymbolicKeyInstancePtr s) const { + if (s == nullptr) { + return 0; + } + return s->abstract()->hash(); + } +}; + +struct SymbolicKeyInstanceEqual { + bool operator()(const SymbolicKeyInstancePtr lhs, const SymbolicKeyInstancePtr rhs) const { + if (lhs == nullptr || rhs == nullptr) { + return false; + } + MS_EXCEPTION_IF_NULL(lhs->node()); + MS_EXCEPTION_IF_NULL(rhs->node()); + MS_EXCEPTION_IF_NULL(lhs->abstract()); + MS_EXCEPTION_IF_NULL(rhs->abstract()); + return (*lhs->node() == *rhs->node()) && (*lhs->abstract() == *rhs->abstract()); + } +}; + +using EnvInstanceContentsMap = + std::unordered_map; + +// Environment mapping keys to values. +// Keys are SymbolicKeyInstances, which represent nodes in the graph along +// with inferred properties. +class EnvInstance : public Value { + public: + friend std::ostream& operator<<(std::ostream& out, const std::shared_ptr& env); + + explicit EnvInstance(const EnvInstanceContentsMap& contents = {}) : contents_(contents) {} + ~EnvInstance() override = default; + MS_DECLARE_PARENT(EnvInstance, Value); + abstract::AbstractBasePtr ToAbstract() override { + return std::make_shared(shared_from_base(), std::make_shared()); + } + bool operator==(const EnvInstance& other) const; + bool operator==(const Value& other) const override; + EnvInstance(const EnvInstance& v) : Value(v), contents_(v.contents_) {} + EnvInstance(EnvInstance&& v) = default; + EnvInstance& operator=(EnvInstance&& src) noexcept { + if (&src != this) { + contents_ = src.contents_; + } + return *this; + }; + + // Get the sensitivity list for the given key + const Any& Get(const SymbolicKeyInstancePtr& key, const Any& def) const { + auto iterator = contents_.find(key); + if (iterator != contents_.end()) { + return iterator->second; + } + return def; + } + + // Set a value for the given key. + EnvInstance Set(const SymbolicKeyInstancePtr& key, const Any& value) const { + EnvInstance rval(contents_); + rval.contents_[key] = value; + return rval; + } + + // Add two EnvInstances. + EnvInstance Add(const EnvInstance& other) const { + EnvInstance rval(contents_); + for (auto iter_other : other.contents_) { + auto item_self = contents_.find(iter_other.first); + if (item_self != contents_.end()) { + MS_LOG(DEBUG) << "Need to use add"; + } else { + rval.contents_[iter_other.first] = iter_other.second; + } + } + return rval; + } + + size_t Len() const { return contents_.size(); } + std::size_t hash() const override { + // deterministic characteristic of member variables. + return Len(); + } + + const bool parse_info_ = true; + + private: + EnvInstanceContentsMap contents_; +}; + +using EnvInstancePtr = std::shared_ptr; + +extern std::shared_ptr newenv; + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_SYMBOLIC_H_ diff --git a/mindspore/ccsrc/utils/system/base.h b/mindspore/ccsrc/utils/system/base.h new file mode 100644 index 00000000000..dace2e71786 --- /dev/null +++ b/mindspore/ccsrc/utils/system/base.h @@ -0,0 +1,138 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_SYSTEM_BASE_H_ +#define MINDSPORE_CCSRC_UTILS_SYSTEM_BASE_H_ + +#include +#include +#include "securec/include/securec.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace system { + +using string = std::string; + +using int8 = int8_t; +using int16 = int16_t; +using int32 = int32_t; +using int64 = int64_t; + +using uint8 = uint8_t; +using uint16 = uint16_t; +using uint32 = uint32_t; +using uint64 = uint64_t; + +#define MS_MALLOC(size) malloc(size) +#define MS_FREE(ptr) free(ptr) + +// Use the macro to confirm the system env +#if defined(ANDROID) || defined(__ANDROID__) + +#define SYSTEM_ENV_POSIX_ANDROID + +#elif defined(__APPLE__) + +#define SYSTEM_ENV_POSIX + +#elif defined(_WIN32) + +#define SYSTEM_ENV_WINDOWS + +#elif defined(__arm__) + +#define SYSTEM_ENV_POSIX + +#else // default set the POSIX + +#define SYSTEM_ENV_POSIX + +#endif + +// define the platform +enum PlatformDefine { + kPlatformPosix = 0, // Posix platform + kPlatformPosixAndroid, // Android of posix + kPlatformWindows, // Windows system + kPlatformUnknow = 0xFF // Error +}; + +class Platform { + public: + Platform() { + platform_ = kPlatformUnknow; +#if defined(SYSTEM_ENV_POSIX) + platform_ = kPlatformPosix; +#elif defined(SYSTEM_ENV_POSIX_ANDROID) + platform_ = kPlatformPosixAndroid +#elif defined(SYSTEM_ENV_WINDOWS) + platform_ = kPlatformWindows; +#endif + } + + ~Platform() = default; + + static PlatformDefine get_platform() { + static const auto sys_env = std::make_shared(); + return sys_env->platform_; + } + + private: + PlatformDefine platform_; +}; + +// check the null point +#define MS_EXCEPT_CHECK_NULL(value) \ + do { \ + if (value == nullptr) { \ + MS_LOG(EXCEPTION) << "The value is null."; \ + } \ + } while (0) + +// define the big or little endian type +constexpr bool kLittleEndian = true; + +// implement common define function +// Get the 32 bits align value +inline uint32 DecodeFixed32(const char* ptr) { + uint32 result; + if (EOK != memcpy_s(&result, sizeof(result), ptr, sizeof(result))) { + MS_LOG(EXCEPTION) << "Call DecodeFixed32 memcpy value failure."; + } + return result; +} +// Used to fetch a naturally-aligned 32-bit word in little endian byte-order +inline uint32 LE_LOAD32(const uint8_t* p) { return DecodeFixed32(reinterpret_cast(p)); } +// Encode the data to buffer +inline void EncodeFixed32(char* buf, uint32 value) { + if (EOK != memcpy_s(buf, sizeof(value), &value, sizeof(value))) { + MS_LOG(EXCEPTION) << "Call EncodeFixed32 memcpy value failure."; + } +} +inline void EncodeFixed64(char* buf, const unsigned int array_len, int64 value) { + if (sizeof(value) > array_len) { + MS_LOG(EXCEPTION) << "Buffer overflow, real size is " << array_len << ", but required " << sizeof(value) << "."; + } + if (EOK != memcpy_s(buf, sizeof(value), &value, sizeof(value))) { + MS_LOG(EXCEPTION) << "Call EncodeFixed64 memcpy value failure."; + } +} + +} // namespace system +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_SYSTEM_BASE_H_ diff --git a/mindspore/ccsrc/utils/system/crc32c.cc b/mindspore/ccsrc/utils/system/crc32c.cc new file mode 100644 index 00000000000..9bb3c4f3e63 --- /dev/null +++ b/mindspore/ccsrc/utils/system/crc32c.cc @@ -0,0 +1,322 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/system/crc32c.h" +#include + +namespace mindspore { +namespace system { + +const unsigned int CRC_TABLE_SIZE = 256; + +static const uint32_t crc_table_o32[CRC_TABLE_SIZE] = { + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, + 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C, + 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, + 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, + 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, + 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, + 0x6EF07595, 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, 0x5125DAD3, + 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F, + 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, 0x61C69362, 0x93AD1061, 0x80FDE395, + 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, + 0x44694011, 0x5739B3E5, 0xA55230E6, 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, + 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, + 0x563C5F93, 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, 0x1871A4D8, + 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5, + 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, + 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, + 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, + 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, + 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, + 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6, + 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, 0xF36E6F75, 0x0105EC76, 0x12551F82, + 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351}; + +static const uint32_t crc_table_o40[CRC_TABLE_SIZE] = { + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, 0x9D14C3B8, + 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, 0x3FC5F181, 0x2C6769F6, + 0x1880C16F, 0x0B225918, 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, 0xA2D13239, 0xB173AA4E, 0x859402D7, + 0x96369AA0, 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, + 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, 0xAC154166, + 0xBFB7D911, 0x8B507188, 0x98F2E9FF, 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, 0x0EC4735F, 0x1D66EB28, + 0x298143B1, 0x3A23DBC6, 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, 0x93D0B0E7, 0x80722890, 0xB4958009, + 0xA737187E, 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, 0xC0D23785, + 0xD370AFF2, 0xE797076B, 0xF4359F1C, 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, 0x5DC6F43D, 0x4E646C4A, + 0x7A83C4D3, 0x69215CA4, 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, 0x809C2506, 0x933EBD71, 0xA7D915E8, + 0xB47B8D9F, 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, + 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, 0xF1D3B55B, + 0xE2712D2C, 0xD69685B5, 0xC5341DC2, 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, 0x6CC776E3, 0x7F65EE94, + 0x4B82460D, 0x5820DE7A, 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, 0xB5499B25, 0xA6EB0352, 0x920CABCB, + 0x81AE33BC, 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, 0x5912C8C0, + 0x4AB050B7, 0x7E57F82E, 0x6DF56059, 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, 0x844819FB, 0x97EA818C, + 0xA30D2915, 0xB0AFB162, 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, + 0x2DBB72DA, 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, + 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, 0x68134A1E, + 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, 0x4A5E5D21, 0x59FCC556, + 0x6D1B6DCF, 0x7EB9F5B8, 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, + 0xE3AD3600, 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, 0x7B5FDFFF, + 0x68FD4788, 0x5C1AEF11, 0x4FB87766, 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, 0xE64B1C47, 0xF5E98430, + 0xC10E2CA9, 0xD2ACB4DE, 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, 0x449A2E7E, 0x5738B609, 0x63DF1E90, + 0x707D86E7, 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, + 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483}; + +static const uint32_t crc_table_o48[CRC_TABLE_SIZE] = { + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, 0x38513EC5, + 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, 0x70A27D8A, 0xD5E3EFF4, + 0x3FCD2F87, 0x9A8CBDF9, 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, 0x48F3434F, 0xEDB2D131, 0x079C1142, + 0xA2DD833C, 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, + 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, 0x47CB61CB, + 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, 0x0F382284, 0xAA79B0FA, + 0x40577089, 0xE516E2F7, 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, 0x37691C41, 0x92288E3F, 0x78064E4C, + 0xDD47DC32, 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, 0xB7C7FD53, + 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, 0x8F96C396, 0x2AD751E8, + 0xC0F9919B, 0x65B803E5, 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, 0x26217BCD, 0x8360E9B3, 0x694E29C0, + 0xCC0FBBBE, 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, + 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, 0xC85DA25D, + 0x6D1C3023, 0x8732F050, 0x2273622E, 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, 0xF00C9C98, 0x554D0EE6, + 0xBF63CE95, 0x1A225CEB, 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, 0x15F9D359, 0xB0B84127, 0x5A968154, + 0xFFD7132A, 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, 0xC3D4340C, + 0x6695A672, 0x8CBB6601, 0x29FAF47F, 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, 0x6A638C57, 0xCF221E29, + 0x250CDE5A, 0x804D4C24, 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, 0x5232B292, 0xF77320EC, 0x1D5DE09F, + 0xB81C72E1, 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, + 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, 0xBC4E6B02, + 0x190FF97C, 0xF321390F, 0x5660AB71, 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, 0xD29C5380, 0x77DDC1FE, + 0x9DF3018D, 0x38B293F3, 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, + 0x00E3AD36, 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, 0xAD060C8E, + 0x08479EF0, 0xE2695E83, 0x4728CCFD, 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, 0x9557324B, 0x3016A035, + 0xDA386046, 0x7F79F238, 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, 0xDDA47104, 0x78E5E37A, 0x92CB2309, + 0x378AB177, 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, + 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8}; + +static const uint32_t crc_table_o56[CRC_TABLE_SIZE] = { + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, 0xF64463E6, + 0x2B01C95E, 0x49234067, 0x9466EADF, 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, 0xE964B13D, 0x34211B85, + 0x560392BC, 0x8B463804, 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, 0x1F20D2DB, 0xC2657863, 0xA047F15A, + 0x7D025BE2, 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, + 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, 0x5A43469E, + 0x8706EC26, 0xE524651F, 0x3861CFA7, 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, 0x45639445, 0x98263EFD, + 0xFA04B7C4, 0x27411D7C, 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, 0xB327F7A3, 0x6E625D1B, 0x0C40D422, + 0xD1057E9A, 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, 0x42C2EEDA, + 0x9F874462, 0xFDA5CD5B, 0x20E067E3, 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, 0xB4868D3C, 0x69C32784, + 0x0BE1AEBD, 0xD6A40405, 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, + 0x1EA1C255, 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, + 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, 0xEEC5CBA2, + 0x3380611A, 0x51A2E823, 0x8CE7429B, 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, 0x1881A844, 0xC5C402FC, + 0xA7E68BC5, 0x7AA3217D, 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, + 0x4BA071F5, 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, 0x4D801BE4, + 0x90C5B15C, 0xF2E73865, 0x2FA292DD, 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, 0x8585DDB4, 0x58C0770C, + 0x3AE2FE35, 0xE7A7548D, 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, + 0x11E3376B, 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, + 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, 0xE1873E9C, + 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, 0x8224A72B, 0x5F610D93, + 0x3D4384AA, 0xE0062E12, 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, 0x7460C4CD, 0xA9256E75, 0xCB07E74C, + 0x16424DF4, 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, 0x2E238253, + 0xF36628EB, 0x9144A1D2, 0x4C010B6A, 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, 0xD867E1B5, 0x05224B0D, + 0x6700C234, 0xBA45688C, 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, 0xC747336E, 0x1A0299D6, 0x782010EF, + 0xA565BA57, 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, + 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842}; + +static const uint32_t crc_table_o64[CRC_TABLE_SIZE] = { + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, 0xC5670B91, + 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, 0x8F2261D3, 0xB7330E7F, + 0xFF00BE8B, 0xC711D127, 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, 0x4A456A42, 0x725405EE, 0x3A67B51A, + 0x0276DAB6, 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, + 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, 0x3E8A0076, + 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, 0x74CF6A34, 0x4CDE0598, + 0x04EDB56C, 0x3CFCDAC0, 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, + 0xF99BD151, 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, 0xB8730B7D, + 0x806264D1, 0xC851D425, 0xF040BB89, 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, 0x7D1400EC, 0x45056F40, + 0x0D36DFB4, 0x3527B018, 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, + 0x64CA6F0D, 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, + 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, 0x439E009A, + 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, 0x86F90B0B, 0xBEE864A7, + 0xF6DBD453, 0xCECABBFF, 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, + 0xC6D4DB18, 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, 0x24E7BF1E, + 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, 0x750A600B, 0x4D1B0FA7, + 0x0528BF53, 0x3D39D0FF, 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, + 0xF85EDB6E, 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, + 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, 0xDF0AB4F9, + 0xE71BDB55, 0xAF286BA1, 0x9739040D, 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, 0xB9B60142, 0x81A76EEE, + 0xC994DE1A, 0xF185B1B6, 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, + 0x34E2BA27, 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, 0x425B0AA5, + 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, 0x873C0134, 0xBF2D6E98, + 0xF71EDE6C, 0xCF0FB1C0, 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, 0xCD796B76, 0xF56804DA, 0xBD5BB42E, + 0x854ADB82, 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, + 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3}; + +static const uint32_t crc_table_o72[CRC_TABLE_SIZE] = { + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, 0x6006181F, + 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, 0xC00C303E, 0x2F3C5B27, + 0x1B8090FD, 0xF4B0FBE4, 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, 0xA00A2821, 0x4F3A4338, 0x7B8688E2, + 0x94B6E3FB, 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, + 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, 0x570739E5, + 0xB83752FC, 0x8C8B9926, 0x63BBF23F, 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, 0xF70D11C4, 0x183D7ADD, + 0x2C81B107, 0xC3B1DA1E, 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, 0x970B09DB, 0x783B62C2, 0x4C87A918, + 0xA3B7C201, 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, 0xCE086BD5, + 0x213800CC, 0x1584CB16, 0xFAB4A00F, 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, 0xAE0E73CA, 0x413E18D3, + 0x7582D309, 0x9AB2B810, 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, 0x8BF04D66, 0x64C0267F, 0x507CEDA5, + 0xBF4C86BC, 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, + 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, 0xF9094A2F, + 0x16392136, 0x2285EAEC, 0xCDB581F5, 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, 0x990F5230, 0x763F3929, + 0x4283F2F3, 0xADB399EA, 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, 0xAEFD80A1, 0x41CDEBB8, 0x75712062, + 0x9A414B7B, 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, 0xBC029FF7, + 0x5332F4EE, 0x678E3F34, 0x88BE542D, 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, 0x99FCA15B, 0x76CCCA42, + 0x42700198, 0xAD406A81, 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, 0xF9FAB944, 0x16CAD25D, 0x22761987, + 0xCD46729E, 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, + 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, 0x8B03BE0D, + 0x6433D514, 0x508F1ECE, 0xBFBF75D7, 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, 0xA0F9DB4A, 0x4FC9B053, + 0x7B757B89, 0x94451090, 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, 0xC0FFC355, 0x2FCFA84C, 0x1B736396, + 0xF443088F, 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, 0x97F8FAB0, + 0x78C891A9, 0x4C745A73, 0xA344316A, 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, 0xF7FEE2AF, 0x18CE89B6, + 0x2C72426C, 0xC3422975, 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, + 0x63480154, 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, + 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C}; + +static const uint32_t crc_table_o80[CRC_TABLE_SIZE] = { + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, 0x4E2DFD53, + 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, 0x9C5BFAA6, 0xF458D66E, + 0x4C5DA336, 0x245E8FFE, 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, 0xD27607F5, 0xBA752B3D, 0x02705E65, + 0x6A7372AD, 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, + 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, 0xD696BB3F, + 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, 0x04E0BCCA, 0x6CE39002, + 0xD4E6E55A, 0xBCE5C992, 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, + 0xF2C834C1, 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, 0xE6ECFDDC, + 0x8EEFD114, 0x36EAA44C, 0x5EE98884, 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, 0xA8C1008F, 0xC0C22C47, + 0x78C7591F, 0x10C475D7, 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, + 0xFFE9F19F, 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, + 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, 0x7E57BBB0, + 0x16549778, 0xAE51E220, 0xC652CEE8, 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, 0x307A46E3, 0x58796A2B, + 0xE07C1F73, 0x887F33BB, 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, 0x508ECB25, 0x388DE7ED, 0x808892B5, + 0xE88BBE7D, 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, 0x27180901, + 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, 0xC8358D49, 0xA036A181, + 0x1833D4D9, 0x7030F811, 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, 0x8618701A, 0xEE1B5CD2, 0x561E298A, + 0x3E1D0542, 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, + 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, 0xBFA34F6D, + 0xD7A063A5, 0x6FA516FD, 0x07A63A35, 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, 0x2A39CC5F, 0x423AE097, + 0xFA3F95CF, 0x923CB907, 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, 0x6414310C, 0x0C171DC4, 0xB412689C, + 0xDC114454, 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, 0xB2828A33, + 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, 0xFCAF7760, 0x94AC5BA8, + 0x2CA92EF0, 0x44AA0238, 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, + 0x96DC05CD, 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, + 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F}; + +static const uint32_t crc_table_o88[CRC_TABLE_SIZE] = { + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, 0x423B04DA, + 0x0B0779FD, 0xD043FE94, 0x997F83B3, 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, 0x847609B4, 0xCD4A7493, + 0x160EF3FA, 0x5F328EDD, 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, 0xC64D0D6E, 0x8F717049, 0x5435F720, + 0x1D098A07, 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, + 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, 0x6E26E32E, + 0x271A9E09, 0xFC5E1960, 0xB5626447, 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, 0xA86BEE40, 0xE1579367, + 0x3A13140E, 0x732F6929, 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, 0xEA50EA9A, 0xA36C97BD, 0x782810D4, + 0x31146DF3, 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, 0x9E76C286, + 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, 0xDC4DC65C, 0x9571BB7B, + 0x4E353C12, 0x07094135, 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, 0x1700AEAB, 0x5E3CD38C, 0x857854E5, + 0xCC4429C2, 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, + 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, 0xB26B2572, + 0xFB575855, 0x2013DF3C, 0x692FA21B, 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, 0xF05021A8, 0xB96C5C8F, + 0x6228DBE6, 0x2B14A6C1, 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, 0x151C1409, 0x5C20692E, 0x8764EE47, + 0xCE589360, 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, 0xF24C9B0A, + 0xBB70E62D, 0x60346144, 0x29081C63, 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, 0x3901F3FD, 0x703D8EDA, + 0xAB7909B3, 0xE2457494, 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, 0x7B3AF727, 0x32068A00, 0xE9420D69, + 0xA07E704E, 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, + 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, 0xDE517CFE, + 0x976D01D9, 0x4C2986B0, 0x0515FB97, 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, 0x0F1CDF3B, 0x4620A21C, + 0x9D642575, 0xD4585852, 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, + 0x96635C88, 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, 0x230138CF, + 0x6A3D45E8, 0xB179C281, 0xF845BFA6, 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, 0x613A3C15, 0x28064132, + 0xF342C65B, 0xBA7EBB7C, 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, 0xA777317B, 0xEE4B4C5C, 0x350FCB35, + 0x7C33B612, 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, + 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5}; + +// Use the 8 table to calc crc32c value +static inline void CRC32T8(uint32 *crc, const uint8 **p) { + auto c = static_cast(*crc ^ LE_LOAD32(*p)); + *p += 4; + *crc = crc_table_o88[c & 0xff] ^ crc_table_o80[(c >> 8) & 0xff] ^ crc_table_o72[(c >> 16) & 0xff] ^ + crc_table_o64[(c >> 24) & 0xff]; + c = static_cast(LE_LOAD32(*p)); + *crc = (*crc) ^ crc_table_o56[c & 0xff] ^ crc_table_o48[(c >> 8) & 0xff] ^ crc_table_o40[(c >> 16) & 0xff] ^ + crc_table_o32[(c >> 24) & 0xff]; + *p += 4; +} + +// calc the crc32c value +uint32 Crc32c::MakeCrc32c(uint32 init_crc, const char *data, size_t size) { + MS_EXCEPT_CHECK_NULL(data); + uint32_t crc = init_crc ^ 0xffffffffu; + const unsigned int OFFSET = 8; + + // Get the origin begin and end address(not aligment) + auto *bp = reinterpret_cast(data); + const uint8_t *ep = bp + size; + + // Get the alignment address + // Make x point to the first 4-byte aligned byte in the string. + // This may just exceed the length of the string. + auto pval = reinterpret_cast(bp); + auto *bp_align = reinterpret_cast(MEM_ALIGN(pval, 2)); + + // process the not aligment bits when size < 4 byte + if (bp_align <= ep) { + // Process bytes until complete or p-align 4 bytes. + while (bp != bp_align) { + crc = crc_table_o32[(crc & 0xff) ^ (*bp++)] ^ (crc >> 8); + } + } + + // Process bytes 8 at a time use the 8 table + while ((ep - bp) >= OFFSET) { + CRC32T8(&crc, &bp); + } + + // Process the last not alignment bytes + while (bp < ep) { + crc = crc_table_o32[(crc & 0xff) ^ (*bp++)] ^ (crc >> 8); + } + return crc ^ 0xffffffffu; +} + +} // namespace system +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/system/crc32c.h b/mindspore/ccsrc/utils/system/crc32c.h new file mode 100644 index 00000000000..4411423babe --- /dev/null +++ b/mindspore/ccsrc/utils/system/crc32c.h @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_SYSTEM_CRC32C_H_ +#define MINDSPORE_CCSRC_UTILS_SYSTEM_CRC32C_H_ + +#include +#include +#include "utils/system/base.h" +#include "utils/system/env.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace system { + +// Align n to (1 << m) byte boundary +#define MEM_ALIGN(n, m) ((n + ((1 << (m)) - 1)) & ~((1 << (m)) - 1)) + +// Masked for crc. +static constexpr uint32 kMaskDelta = 0xa282ead8ul; +static const int kRightShift = 15; +static const int kLeftShift = (32 - kRightShift); +// Provide the Crc32c function +class Crc32c { + public: + Crc32c() = default; + ~Crc32c() = default; + + // Calculate the crc32c value, use the 8 table method + static uint32 MakeCrc32c(uint32 init_crc, const char* data, size_t size); + + // retrun the crc32c value(need mask) + static uint32 GetMaskCrc32cValue(const char* data, size_t n) { + auto crc = MakeCrc32c(0, data, n); + // Rotate right by kRightShift bits and add kMaskDelta(a constant). + return ((crc >> kRightShift) | (crc << kLeftShift)) + kMaskDelta; + } +}; + +} // namespace system +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_SYSTEM_CRC32C_H_ diff --git a/mindspore/ccsrc/utils/system/env.h b/mindspore/ccsrc/utils/system/env.h new file mode 100644 index 00000000000..fd1ded04192 --- /dev/null +++ b/mindspore/ccsrc/utils/system/env.h @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_SYSTEM_ENV_H_ +#define MINDSPORE_CCSRC_UTILS_SYSTEM_ENV_H_ + +#include +#include "utils/system/base.h" +#include "utils/log_adapter.h" +#include "utils/system/file_system.h" + +namespace mindspore { +namespace system { +// Confirm the system env and create the filesystem, time... +class Env { + public: + Env() { platform_ = Platform::get_platform(); } + virtual ~Env() = default; + + static std::shared_ptr GetFileSystem() { +#if defined(SYSTEM_ENV_POSIX) + auto fs = std::make_shared(); + return fs; +#else + MS_LOG(EXCEPTION) << "Now not support the platform."; +#endif + } + + private: + PlatformDefine platform_; +}; + +} // namespace system +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_SYSTEM_ENV_H_ diff --git a/mindspore/ccsrc/utils/system/file_system.cc b/mindspore/ccsrc/utils/system/file_system.cc new file mode 100644 index 00000000000..aee89d4b7bb --- /dev/null +++ b/mindspore/ccsrc/utils/system/file_system.cc @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/system/file_system.h" +#include +#include +#include +#include + +namespace mindspore { +namespace system { + +#if defined(SYSTEM_ENV_POSIX) +// Implement the Posix file systen +WriteFilePtr PosixFileSystem::CreateWriteFile(const string& file_name) { + if (file_name.empty()) { + MS_LOG(ERROR) << "Create write file failed because the file name is null."; + return nullptr; + } + auto fp = std::make_shared(file_name); + if (fp == nullptr) { + MS_LOG(ERROR) << "Create write file(" << file_name << ") failed."; + return nullptr; + } + bool result = fp->Open(); + if (!result) { + MS_LOG(ERROR) << "Open the write file(" << file_name << ") failed."; + return nullptr; + } + return fp; +} + +bool PosixFileSystem::FileExist(const string& file_name) { + if (file_name.empty()) { + MS_LOG(WARNING) << "The file name is null."; + return false; + } + auto result = access(file_name.c_str(), F_OK); + if (result != 0) { + MS_LOG(INFO) << "The file(" << file_name << ") not exist."; + return false; + } + return true; +} + +bool PosixFileSystem::DeleteFile(const string& file_name) { + if (file_name.empty()) { + MS_LOG(WARNING) << "The file name is null."; + return false; + } + auto result = unlink(file_name.c_str()); + if (result != 0) { + MS_LOG(ERROR) << "Delete the file(" << file_name << ") is falire, error(" << errno << ")."; + return false; + } + return true; +} + +static const int DEFAULT_MKDIR_MODE = 0700; +bool PosixFileSystem::CreateDir(const string& dir_name) { + if (dir_name.empty()) { + MS_LOG(WARNING) << "The directory name is null."; + return false; + } + auto result = mkdir(dir_name.c_str(), DEFAULT_MKDIR_MODE); + if (result != 0) { + MS_LOG(ERROR) << "Create the dir(" << dir_name << ") is falire, error(" << errno << ")."; + return false; + } + return true; +} + +bool PosixFileSystem::DeleteDir(const string& dir_name) { + if (dir_name.empty()) { + MS_LOG(WARNING) << "The directory name is null."; + return false; + } + auto result = rmdir(dir_name.c_str()); + if (result != 0) { + MS_LOG(ERROR) << "Delete the dir(" << dir_name << ") is falire, error(" << errno << ")."; + return false; + } + return true; +} +#endif + +} // namespace system +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/system/file_system.h b/mindspore/ccsrc/utils/system/file_system.h new file mode 100644 index 00000000000..ef0cf885be1 --- /dev/null +++ b/mindspore/ccsrc/utils/system/file_system.h @@ -0,0 +1,204 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_SYSTEM_FILE_SYSTEM_H_ +#define MINDSPORE_CCSRC_UTILS_SYSTEM_FILE_SYSTEM_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "utils/system/base.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace system { + +class WriteFile; +class PosixWriteFile; +using WriteFilePtr = std::shared_ptr; +using PosixWriteFilePtr = std::shared_ptr; + +// File system of create or delete directory +class FileSystem { + public: + FileSystem() = default; + + virtual ~FileSystem() = default; + + // Create a new read/write file + virtual WriteFilePtr CreateWriteFile(const string& file_name) = 0; + + // Check the file is exist? + virtual bool FileExist(const string& file_name) = 0; + + // Delete the file + virtual bool DeleteFile(const string& file_name) = 0; + + // Create a directory + virtual bool CreateDir(const string& dir_name) = 0; + + // Delete the specified directory + virtual bool DeleteDir(const string& dir_name) = 0; +}; + +// A file that can be read and write +class WriteFile { + public: + explicit WriteFile(const string& file_name) : file_name_(file_name) {} + + virtual ~WriteFile() = default; + + // Open the file + virtual bool Open() = 0; + + // append the content to file + virtual bool Write(const std::string& data) { + MS_LOG(WARNING) << "Attention: Maybe not call the function."; + return true; + } + + // name: return the file name + string get_file_name() { return file_name_; } + + // flush: flush local buffer data to filesystem. + virtual bool Flush() = 0; + + // sync: sync the content to disk + virtual bool Sync() = 0; + + // close the file + virtual bool Close() = 0; + + protected: + string file_name_; +}; + +#if defined(SYSTEM_ENV_POSIX) +// File system of create or delete directory for posix system +class PosixFileSystem : public FileSystem { + public: + PosixFileSystem() = default; + + ~PosixFileSystem() override = default; + + // create a new write file + WriteFilePtr CreateWriteFile(const string& file_name) override; + + // check the file is exist? + bool FileExist(const string& file_name) override; + + // delete the file + bool DeleteFile(const string& file_name) override; + + // Create a Directory + bool CreateDir(const string& dir_name) override; + + // Delete the specified directory. + bool DeleteDir(const string& dir_name) override; +}; + +// A file that can be read and write for posix +class PosixWriteFile : public WriteFile { + public: + explicit PosixWriteFile(const string& file_name) : WriteFile(file_name), file_(nullptr) {} + PosixWriteFile(const PosixWriteFile&); + PosixWriteFile& operator=(const PosixWriteFile&); + + ~PosixWriteFile() override { + try { + if (file_ != nullptr) { + (void)fclose(file_); + file_ = nullptr; + } + } catch (const std::exception& e) { + MS_LOG(ERROR) << "Exception when closing file."; + } catch (...) { + MS_LOG(ERROR) << "Non standard exception when closing file."; + } + } + + bool Open() override { + if (file_ != nullptr) { + MS_LOG(WARNING) << "The File(" << file_name_ << ") already open."; + return true; + } + // check the path + if (nullptr == file_name_.c_str()) { + MS_LOG(EXCEPTION) << "The file path is null."; + } + char path[PATH_MAX + 1] = {0x00}; + if (file_name_.size() > PATH_MAX || nullptr == realpath(file_name_.c_str(), path)) { + MS_LOG(EXCEPTION) << "Convert to real path fail, file name is " << file_name_ << "."; + } + + // open the file + file_ = fopen(path, "w+"); + if (file_ == nullptr) { + MS_LOG(ERROR) << "File(" << path << ") IO ERROR: " << errno << "."; + return false; + } + return true; + } + + bool Write(const std::string& data) override { + MS_LOG(DEBUG) << "Write data(" << data.size() << ") to file(" << this->file_name_ << ")."; + size_t r = fwrite(data.data(), 1, data.size(), file_); + if (r != data.size()) { + MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << "."; + return false; + } + return true; + } + + bool Close() override { + if (file_ == nullptr) { + MS_LOG(INFO) << "File(" << file_name_ << ") already close."; + return true; + } + bool result = true; + if (fclose(file_) != 0) { + MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << errno << "."; + result = false; + } + file_ = nullptr; + return result; + } + + bool Flush() override { + if (fflush(file_) != 0) { + MS_LOG(ERROR) << "File(" << file_name_ << ") IO ERROR: " << EBADF << "."; + return false; + } + return true; + } + + bool Sync() override { return Flush(); } + + private: + FILE* file_; +}; +#endif + +} // namespace system +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_SYSTEM_FILE_SYSTEM_H_ diff --git a/mindspore/ccsrc/utils/tensorprint_utils.cc b/mindspore/ccsrc/utils/tensorprint_utils.cc new file mode 100644 index 00000000000..ee58ed418cd --- /dev/null +++ b/mindspore/ccsrc/utils/tensorprint_utils.cc @@ -0,0 +1,137 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "utils/tensorprint_utils.h" +#include +#include +#include +#include +#include +#include +#include "ir/meta_tensor.h" +#include "device/convert_tensor_utils.h" +#include "./securec.h" +#ifndef NO_DLIB +#include "tdt/tsd_client.h" +#include "tdt/tdt_host_interface.h" +#include "tdt/data_common.h" +#endif + +namespace mindspore { +const char kShapeSeperator[] = ","; +static std::map print_type_map = { + {"int8_t", TypeId::kNumberTypeInt8}, {"uint8_t", TypeId::kNumberTypeUInt8}, + {"int16_t", TypeId::kNumberTypeInt16}, {"uint16_t", TypeId::kNumberTypeUInt16}, + {"int32_t", TypeId::kNumberTypeInt32}, {"uint32_t", TypeId::kNumberTypeUInt32}, + {"int64_t", TypeId::kNumberTypeInt64}, {"uint64_t", TypeId::kNumberTypeUInt64}, + {"float16", TypeId::kNumberTypeFloat16}, {"float", TypeId::kNumberTypeFloat32}, + {"double", TypeId::kNumberTypeFloat64}}; + +static std::map type_size_map = { + {"int8_t", sizeof(int8_t)}, {"uint8_t", sizeof(uint8_t)}, {"int16_t", sizeof(int16_t)}, + {"uint16_t", sizeof(uint16_t)}, {"int32_t", sizeof(int32_t)}, {"uint32_t", sizeof(uint32_t)}, + {"int64_t", sizeof(int64_t)}, {"uint64_t", sizeof(uint64_t)}, {"float16", sizeof(float) / 2}, + {"float", sizeof(float)}, {"double", sizeof(double)}}; + +bool ParseTensorShape(const std::string &input_shape_str, std::vector *const tensor_shape, size_t *dims) { + if (tensor_shape == nullptr) { + return false; + } + std::string shape_str = input_shape_str; + if (shape_str.size() <= 2) { + return false; + } + (void)shape_str.erase(shape_str.begin()); + shape_str.pop_back(); + shape_str += kShapeSeperator; + string::size_type pos_begin = 0; + string::size_type pos_end = shape_str.find(kShapeSeperator); + while (pos_end != std::string::npos) { + string dim_str = shape_str.substr(pos_begin, pos_end - pos_begin); + tensor_shape->emplace_back(std::stoi(dim_str)); + (*dims) = (*dims) * std::stoul(dim_str); + pos_begin = pos_end + sizeof(kShapeSeperator) - 1; + pos_end = shape_str.find(kShapeSeperator, pos_begin); + } + return true; +} + +bool PrintTensorToString(const char *str_data_ptr, mindspore::tensor::Tensor *const print_tensor, + const size_t &memory_size) { + auto *tensor_data_ptr = static_cast(print_tensor->data_c(true)); + MS_EXCEPTION_IF_NULL(tensor_data_ptr); + auto cp_ret = + memcpy_s(tensor_data_ptr, static_cast(print_tensor->data().nbytes()), str_data_ptr, memory_size); + if (cp_ret != EOK) { + MS_LOG(ERROR) << "Print op Failed to copy the memory to py::tensor " << cp_ret; + return false; + } + return true; +} + +#ifndef NO_DLIB +bool ConvertDataItem2Tensor(const std::vector &items) { + // Acquire Python GIL + py::gil_scoped_acquire gil_acquire; + std::ostringstream buf; + bool ret_end_sequence = false; + for (auto &item : items) { + if (item.dataType_ == tdt::TDT_END_OF_SEQUENCE) { + ret_end_sequence = true; + break; + } + std::vector tensor_shape; + size_t totaldims = 1; + if (!ParseTensorShape(item.tensorShape_, &tensor_shape, &totaldims)) { + MS_LOG(ERROR) << "Tensor print can not parse tensor shape, receive info" << item.tensorShape_; + continue; + } + std::shared_ptr str_data_ptr = std::static_pointer_cast(item.dataPtr_); + MS_EXCEPTION_IF_NULL(str_data_ptr); + + if (item.tensorType_ == "string") { + std::string data(reinterpret_cast(str_data_ptr->c_str()), item.dataLen_); + buf << data << std::endl; + } else { + auto type_iter = print_type_map.find(item.tensorType_); + if (type_iter == print_type_map.end()) { + MS_LOG(ERROR) << "type of tensor need to print is not soupport" << item.tensorType_; + continue; + } + auto type_id = type_iter->second; + mindspore::tensor::Tensor print_tensor(type_id, tensor_shape); + auto memory_size = totaldims * type_size_map[item.tensorType_]; + if (PrintTensorToString(str_data_ptr->data(), &print_tensor, memory_size)) { + buf << print_tensor.ToStringRepr() << std::endl; + } + } + } + std::cout << buf.str() << std::endl; + return ret_end_sequence; +} + +void TensorPrint::operator()() { + while (true) { + std::vector bundle; + if (tdt::TdtHostPopData("_npu_log", bundle) != 0) { + break; + } + if (ConvertDataItem2Tensor(bundle)) { + break; + } + } +} +#endif +} // namespace mindspore diff --git a/mindspore/ccsrc/utils/tensorprint_utils.h b/mindspore/ccsrc/utils/tensorprint_utils.h new file mode 100644 index 00000000000..c8442e62913 --- /dev/null +++ b/mindspore/ccsrc/utils/tensorprint_utils.h @@ -0,0 +1,38 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_TENSORPRINT_UTILS_H_ +#define MINDSPORE_CCSRC_UTILS_TENSORPRINT_UTILS_H_ + +#include +#include "ir/dtype/type.h" +#ifndef NO_DLIB +#include "tdt/tsd_client.h" +#include "tdt/tdt_host_interface.h" +#include "tdt/data_common.h" +#endif +namespace mindspore { +class TensorPrint { + public: + TensorPrint() {} + ~TensorPrint() = default; +#ifndef NO_DLIB + void operator()(); +#endif +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_TENSOR_PRINT_H_ diff --git a/mindspore/ccsrc/utils/utils.h b/mindspore/ccsrc/utils/utils.h new file mode 100644 index 00000000000..3328480d00c --- /dev/null +++ b/mindspore/ccsrc/utils/utils.h @@ -0,0 +1,206 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_MINDSPORE_CCSRC_UTILS_UTILS_H_ +#define MINDSPORE_MINDSPORE_CCSRC_UTILS_UTILS_H_ + +#include +#include +#include +#include +#include +#include + +#include "utils/log_adapter.h" + +namespace mindspore { +// op name. Op which not exists in operator/ops.h, so define it's name here +constexpr auto kFour2FiveOpName = "Four2Five"; +constexpr auto kFive2FourOpName = "Five2Four"; +constexpr auto kConvBN1OpName = "ConvBN1"; +constexpr auto kBN2AddReluOpName = "BN2AddRelu"; +constexpr auto kBN2ReLUOpName = "BN2Relu"; +constexpr auto kBN2OpName = "BN2"; +constexpr auto kFusedBN1OpName = "FusedBN1"; +constexpr auto kFusedBN2OpName = "FusedBN2"; +constexpr auto kFusedBN3OpName = "FusedBN3"; +constexpr auto kBNGrad1OpName = "BNGrad1"; +constexpr auto kBNGrad2OpName = "BNGrad2"; +constexpr auto kBNGrad3OpName = "BNGrad3"; +constexpr auto kClearZeroOpName = "ClearZero"; +constexpr auto kAtomicAddrCleanOpName = "AtomicAddrClean"; +constexpr auto kAllReduceOpName = "AllReduce"; +constexpr auto kAllGatherOpName = "AllGather"; +constexpr auto kBroadcastOpName = "Broadcast"; +constexpr auto kReduceScatterOpName = "ReduceScatter"; +constexpr auto kMemCpyAsyncOpName = "memcpy_async"; +constexpr auto kTopKOpName = "TopK"; +constexpr auto kBNTrainingReduceOpName = "BNTrainingReduce"; +constexpr auto kBNTrainingUpdateOpName = "BNTrainingUpdate"; +constexpr auto kSimpleMeanGradOpName = "SimpleMeanGrad"; +constexpr auto kMeanGradOpName = "MeanGrad"; +constexpr auto kSliceOpName = "Slice"; +constexpr auto kSliceGradOpName = "SliceGrad"; +constexpr auto kTileOpName = "Tile"; +constexpr auto kScatterNdOpName = "ScatterNd"; +constexpr auto kStridedSliceAssignOpName = "StridedSliceAssign"; +constexpr auto kStridedSliceOpName = "StridedSlice"; +constexpr auto kStridedSliceGradOpName = "StridedSliceGrad"; +constexpr auto kUnsortedSegmentProdOpName = "UnsortedSegmentProd"; +constexpr auto kUnsortedSegmentMinOpName = "UnsortedSegmentMin"; +constexpr auto kFlattenGradOpName = "FlattenGrad"; +constexpr auto kExpandDimsOpName = "ExpandDims"; +constexpr auto kSplitOpName = "Split"; +constexpr auto kSparseApplyAdagradOpName = "SparseApplyAdagrad"; +constexpr auto kMomentumOpName = "Momentum"; +constexpr auto kApplyMomentumOpName = "ApplyMomentum"; +constexpr auto kApplyAdadeltaOpName = "ApplyAdadelta"; +constexpr auto kApplyAdagradOpName = "ApplyAdagrad"; +constexpr auto kApplyAdagradDAName = "ApplyAdagradDA"; +constexpr auto kApplyAdamOpName = "ApplyAdam"; +constexpr auto kApplyAdaMaxOpName = "ApplyAdaMax"; +constexpr auto kApplyAddSignOpName = "ApplyAddSign"; +constexpr auto kApplyCenteredRMSPOpName = "ApplyCenteredRMSP"; +constexpr auto kApplyFtrlOpName = "ApplyFtrl"; +constexpr auto kApplyFtrlV2OpName = "ApplyFtrlV2"; +constexpr auto kApplyGradientDescentOpName = "ApplyGradientDescent"; +constexpr auto kApplyPowerSignOpName = "ApplyPowerSign"; +constexpr auto kApplyProximalAdagradOpName = "ApplyProximalAdagrad "; +constexpr auto kApplyProximalGradientDescentOpName = "ApplyProximalGradientDescent"; +constexpr auto kApplyRMSPropOpName = "ApplyRMSProp"; +constexpr auto kTransDataOpName = "TransData"; +constexpr auto kBNTrainingUpdateGradOpName = "BNTrainingUpdateGrad"; +constexpr auto kBNTrainingReduceGradOpName = "BNTrainingReduceGrad"; +constexpr auto kSquareSumV1OpName = "SquareSumV1"; +constexpr auto kSquareSumV2OpName = "SquareSumV2"; +constexpr auto kClipByNormNoDivSumOpName = "ClipByNormNoDivSum"; +constexpr auto kGreaterOpName = "Greater"; +constexpr auto kSqrtOpName = "Sqrt"; +constexpr auto kRsqrtOpName = "Rsqrt"; +constexpr auto kRealDivOpName = "RealDiv"; +constexpr auto kLambUpdateWithLROpName = "LambUpdateWithLR"; +constexpr auto kLambNextMVWithDecayOpName = "LambNextMVWithDecay"; +constexpr auto kLambNextMVWithDecayV1OpName = "LambNextMVWithDecayV1"; +constexpr auto kClipByValueOpName = "ClipByValue"; +constexpr auto kLambNextRightOpName = "LambNextRight"; +constexpr auto kConfusionSoftmaxGradOpName = "ConfusionSoftmaxGrad"; +constexpr auto kLambUpdateWithLrV2OpName = "LambUpdateWithLrV2"; +constexpr auto kLayerNormXBackpropOpName = "LayerNormXBackprop"; +constexpr auto kLayerNormBetaGammaBackpropOpName = "LayerNormBetaGammaBackprop"; +constexpr auto kLambNextMVOpName = "LambNextMV"; +constexpr auto kConfusionTransposeDOpName = "ConfusionTransposeD"; +constexpr auto kAdamApplyOneWithDecayOpName = "AdamApplyOneWithDecay"; +constexpr auto kBatchNormOpName = "BatchNorm"; +constexpr auto kAdamApplyOneOpName = "AdamApplyOne"; +constexpr auto kDropoutGenMask = "DropoutGenMask"; +constexpr auto kResizeNearestNeighborGrad = "ResizeNearestNeighborGrad"; +constexpr auto kFusedMulAddOpName = "FusedMulAdd"; +constexpr auto kFusedMulAddNOpName = "FusedMulAddN"; +constexpr auto kFusedMulApplyMomentumOpName = "FusedMulApplyMomentum"; + +// attr key name +constexpr auto kAttrInputNames = "input_names"; +constexpr auto kAttrOutputNames = "output_names"; +constexpr auto kAttrVisited = "visited"; +constexpr auto kAttrShape = "shape"; +constexpr auto kAttrMomentum = "momentum"; +constexpr auto kAttrEps = "eps"; +constexpr auto kAttrEpsilon = "epsilon"; +constexpr auto kAttrFactor = "factor"; +constexpr auto kAttrIsRef = "isRef"; +constexpr auto kAttrDataShape = "data_shape"; +constexpr auto kAttrAxis = "axis"; +constexpr auto kAttrKeepDims = "keep_dims"; +constexpr auto kAttrShapeGamma = "shape_gamma"; +constexpr auto kAttrPerm = "perm"; +constexpr auto kAttrTransposeFirst = "transpose_first"; +constexpr auto kAttrAutomicAddMemSize = "automic_add_mem_size"; +constexpr auto kAttrAutomicOutputIndexs = "atomic_output_clean_indexs"; +constexpr auto kAttrAutomicWorkspaceSize = "atomic_workspace_clean_size"; +constexpr auto kAttrSwitchCondition = "switch_condition"; +constexpr auto kAttrDataType = "data_type"; +constexpr auto kAttrActiveTarget = "active_target"; +constexpr auto kAttrActiveStreamList = "active_stream_list"; +constexpr auto kAttrTrueBranchStream = "true_branch_stream"; +constexpr auto kAttrEventId = "event_id"; +constexpr auto kAttrDynInput = "dynamic"; +constexpr auto kAttrDynInputSizes = "dyn_input_sizes"; +constexpr auto kAttrSrcFormat = "src_format"; +constexpr auto kAttrOutputUsedNum = "output_used_num"; + +// attr value +constexpr auto kValueTargetSwitch = "target_switch"; +constexpr auto kValueTargetOther = "target_other"; + +// some size +const size_t kShape4dDims = 4; +const size_t kShape5dDims = 5; +const size_t kCubeSize = 16; +const size_t kMemAlignSize = 512; + +// define special index in special node +constexpr auto kAnfPrimitiveIndex = 0; +constexpr auto kRealInputNodeIndexInTupleGetItem = 1; +constexpr auto kInputNodeOutputIndexInTupleGetItem = 2; +constexpr auto kTupleGetItemInputSize = 3; +// index define of control depend +constexpr auto kControlDependPriorIndex = 1; +constexpr auto kControlDependBehindIndex = 2; +// index define of depend +constexpr auto kRealInputIndexInDepend = 1; +constexpr auto kDependAttachNodeIndex = 2; + +// format +constexpr auto kOpFormat_DEFAULT = "DefaultFormat"; +constexpr auto kOpFormat_NC1KHKWHWC0 = "NC1KHKWHWC0"; +constexpr auto kOpFormat_ND = "ND"; +constexpr auto kOpFormat_NCHW = "NCHW"; +constexpr auto kOpFormat_NHWC = "NHWC"; +constexpr auto kOpFormat_HWCN = "HWCN"; +constexpr auto kOpFormat_NC1HWC0 = "NC1HWC0"; +constexpr auto kOpFormat_FRAC_Z = "FracZ"; +constexpr auto kOpFormat_FRAC_NZ = "FRACTAL_NZ"; +constexpr auto kOpFormat_C1HWNCoC0 = "C1HWNCoC0"; +const std::set k1DSupportFormat = {kOpFormat_DEFAULT, kOpFormat_NCHW, kOpFormat_NHWC, + kOpFormat_FRAC_Z, kOpFormat_NC1KHKWHWC0, kOpFormat_NC1HWC0}; +const std::set k2DSupportFormat = {kOpFormat_DEFAULT, kOpFormat_NCHW, kOpFormat_NHWC, kOpFormat_FRAC_Z, + kOpFormat_NC1KHKWHWC0}; +const std::set k3DSupportFormat = {kOpFormat_DEFAULT, kOpFormat_NC1KHKWHWC0}; +const std::set k4DSupportFormat = k1DSupportFormat; +const std::vector> kShapeSupportFormatMap = {k1DSupportFormat, k2DSupportFormat, k3DSupportFormat, + k4DSupportFormat}; +const std::set kDefaultCompatibleFormat = {kOpFormat_ND, kOpFormat_NCHW, kOpFormat_NHWC, kOpFormat_HWCN}; + +const std::set kOptOpeatorSet = { + kMomentumOpName, kApplyMomentumOpName, kApplyAdadeltaOpName, + kApplyAdagradOpName, kApplyAdagradDAName, kApplyAdamOpName, + kApplyAdaMaxOpName, kApplyAddSignOpName, kApplyCenteredRMSPOpName, + kApplyFtrlOpName, kApplyFtrlV2OpName, kApplyGradientDescentOpName, + kApplyPowerSignOpName, kApplyProximalAdagradOpName, kApplyProximalGradientDescentOpName, + kApplyRMSPropOpName, +}; + +static inline void ChangeFileMode(const std::string& file_name, mode_t mode) { + if (access(file_name.c_str(), F_OK) != 0) { + MS_LOG(DEBUG) << "File `" << file_name << "` does not exist."; + return; + } + if (chmod(file_name.c_str(), mode) != 0) { + MS_LOG(WARNING) << "Change file `" << file_name << "` to mode " << std::oct << mode << " fail."; + } +} +} // namespace mindspore +#endif // MINDSPORE_MINDSPORE_CCSRC_UTILS_UTILS_H_ diff --git a/mindspore/ccsrc/utils/visible.h b/mindspore/ccsrc/utils/visible.h new file mode 100644 index 00000000000..96395230f93 --- /dev/null +++ b/mindspore/ccsrc/utils/visible.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_UTILS_VISIBLE_H_ +#define MINDSPORE_CCSRC_UTILS_VISIBLE_H_ + +namespace mindspore { +// refer to https://gcc.gnu.org/wiki/Visibility +#if defined _WIN32 || defined __CYGWIN__ +#ifdef BUILDING_DLL +#ifdef __GNUC__ +#define MS_EXPORT __attribute__((dllexport)) +#else +#define MS_EXPORT __declspec(dllexport) // Note: actually gcc seems to also supports this syntax. +#endif +#else +#ifdef __GNUC__ +#define MS_EXPORT __attribute__((dllimport)) +#else +#define MS_EXPORT __declspec(dllimport) // Note: actually gcc seems to also supports this syntax. +#endif +#endif +#define MS_LOCAL +#else +#define MS_EXPORT __attribute__((visibility("default"))) +#define MS_LOCAL __attribute__((visibility("hidden"))) +#endif + +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_UTILS_VISIBLE_H_ diff --git a/mindspore/ccsrc/vm/CMakeLists.txt b/mindspore/ccsrc/vm/CMakeLists.txt new file mode 100644 index 00000000000..2df984a29fc --- /dev/null +++ b/mindspore/ccsrc/vm/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE _VM_ALL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "*.cc" + ) + +add_library(_mindspore_vm_obj OBJECT ${_VM_ALL_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc new file mode 100644 index 00000000000..28609abfa9f --- /dev/null +++ b/mindspore/ccsrc/vm/backend.cc @@ -0,0 +1,280 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "vm/backend.h" + +#include +#include + +#include "utils/log_adapter.h" +#include "ir/anf.h" +#include "utils/callbacks.h" +#include "utils/graph_utils.h" +#include "session/session_factory.h" +#include "common/utils.h" + +namespace mindspore { +namespace compile { +bool Backend::GetCond(const BaseRef &c, bool *const value) { return BaseRefToBool(c, value); } + +LinConvertResult MsBackend::GetMultiGraphRun(const FuncGraphPtr &g) { + // multi_graph merge to one, big graph have paramters in begin and only have one output + MS_LOG(DEBUG) << "graph:" << g->ToString() << " parameter size:" << g->parameters().size(); + multi_result_.inputs = g->parameters(); + final_output_ = NewValueNode("fake_output"); + multi_result_.outputs = {final_output_}; + GraphId final_g = sess_->GetFinalRunGraph(); + + multi_result_.run = std::make_shared( + [final_g, this](const VectorRef &args) -> VectorRef { return MsRunGraph(final_g, args); }); + return multi_result_; +} + +LinConvertResult MsBackend::MsConvert(const AnfNodePtrList &lst) { + MS_LOG(DEBUG) << "MsConvert"; + MS_EXCEPTION_IF_NULL(MsContext::GetInstance()); + auto cached = g_ConvertCache.find(lst); + if (cached != g_ConvertCache.end()) { + return cached->second; + } + + LinConvertResult result; + + FuncGraphPtr fg; + AnfNodePtrList inputs; + AnfNodePtrList outputs; + + std::tie(fg, inputs, outputs) = TransformSegmentToAnfGraph(lst); + result.inputs = inputs; + result.outputs = outputs; + result.graph_id = kInvalidGraphId; + auto graph_id = sess_->CompileGraph(lst, outputs); + if (MsContext::GetInstance()->precompile_only()) { + MS_LOG(INFO) << "PrecompileOnly, stop run graph"; + return result; + } + + result.run = std::make_shared( + [graph_id, this](const VectorRef &args) -> VectorRef { return MsRunGraph(graph_id, args); }); + MS_EXCEPTION_IF_NULL(result.run); + + result.simu_run = std::make_shared( + [graph_id, this](const VectorRef &args) -> VectorRef { return MsSimuRunGraph(graph_id, args); }); + MS_EXCEPTION_IF_NULL(result.simu_run); + result.graph_id = graph_id; + + graph_id_map_[graph_id] = result; + (void)g_ConvertCache.emplace(lst, result); + return result; +} + +void MsBackend::SetSwitchActive(const BaseRef &c, bool cond) { + GraphId active_g = simu_cond_map_[c].cond_graph_map[cond]; + + GraphId cond_g = kInvalidGraphId; + if (utils::isa(c)) { + cond_g = sess_->GetGraphIdByNode(utils::cast(c)); + } else { + MS_LOG(EXCEPTION) << "cond not a anf node:" << c.ToString(); + } + auto before_cond = curr_switch_; + if (curr_switch_.hash() != c.hash()) { + // invoke while false->before true call + if (simu_cond_map_[before_cond].cond_graph_map.count(false)) { + active_g = simu_cond_map_[before_cond].cond_graph_map[false]; + } else { + active_g = kInvalidGraphId; + } + // while x < y: + // z = y + 1 + // while z < c2: + // out = out + 1 + // z = z + 1 + if (active_g == cond_g) { + active_g = kInvalidGraphId; + simu_cond_map_[before_cond].cond_graph_map[false] = kInvalidGraphId; + } + MS_LOG(DEBUG) << "invoke set active:" << active_g; + } + MS_LOG(DEBUG) << "switch set active:" << active_g << ", " << cond_g; + sess_->SetActive(active_g, cond_g); +} + +void MsBackend::SetSwitchGraph() { + MS_LOG(DEBUG) << "SetSwitchGraph curr_switch:" << curr_switch_.ToString(); + + if (is_switch_call_) { + GraphId false_g = kInvalidGraphId; + GraphId true_g = kInvalidGraphId; + MS_LOG(DEBUG) << "start SetSwitchGraph"; + true_g = simu_cond_map_[curr_switch_].cond_graph_map[true]; + bool curr_cond = simu_cond_map_[curr_switch_].curr_cond; + if (!curr_cond) { + if (simu_cond_map_[curr_switch_].cond_graph_map.count(curr_cond)) { + // has false branch + false_g = simu_cond_map_[curr_switch_].cond_graph_map[false]; + } + GraphId cond_g = kInvalidGraphId; + if (utils::isa(curr_switch_)) { + cond_g = sess_->GetGraphIdByNode(utils::cast(curr_switch_)); + } else { + MS_LOG(EXCEPTION) << "cond not a anf node:" << curr_switch_.ToString(); + } + MS_LOG(DEBUG) << "switch compile:" << cond_g << ", " << true_g << ", " << false_g; + sess_->SwitchCompile(cond_g, true_g, false_g); + } + is_switch_call_ = false; + MS_LOG(DEBUG) << "end SetSwitchGraph:" << curr_cond << ", " << is_switch_call_; + } +} + +// compile set input output +VectorRef MsBackend::MsSimuRunGraph(const GraphId &g, const VectorRef &args) { + MS_LOG(DEBUG) << "set graph input:" << g; + // switch maybe twice + sess_->SetChildGraphInput(g, args); + + if (is_switch_call_) { + bool curr_cond = simu_cond_map_[curr_switch_].curr_cond; + MS_LOG(DEBUG) << "switch call MsSimuRunGraph:" << curr_cond; + if (0 == simu_cond_map_[curr_switch_].cond_graph_map.count(curr_cond)) { + MS_LOG(DEBUG) << "switch call MsSimuRunGraph:" << curr_cond << ", " << g; + simu_cond_map_[curr_switch_].cond_graph_map[curr_cond] = g; + SetSwitchGraph(); + } + } + + std::vector outputs; + (void)std::transform(graph_id_map_[g].outputs.begin(), graph_id_map_[g].outputs.end(), std::back_inserter(outputs), + [](const AnfNodePtr &v) { return v; }); + return VectorRef(outputs); +} + +VectorRef MsBackend::MsRunGraph(const GraphId &g, const VectorRef &args) { + MS_LOG(DEBUG) << "start ms graph run:" << args.size() << ", g:" << g; + // Run graph + std::vector inputs; + for (const auto &arg : args) { + if (utils::isa(arg)) { + auto value = utils::cast(arg); + inputs.push_back(value); + } else if (utils::isa(arg)) { + auto value = utils::cast(arg); + if (value->isa()) { + (void)std::transform(value->cast()->value().begin(), value->cast()->value().end(), + std::back_inserter(inputs), + [](const ValuePtr &v) { return v->cast(); }); + } else if (value->isa()) { + tensor::TensorPtr scalar_tensor = ScalarToTensor(value->cast()); + MS_EXCEPTION_IF_NULL(scalar_tensor); + inputs.push_back(scalar_tensor); + } else { + inputs.push_back(value->cast()); + } + } else if (utils::isa(arg)) { + auto value = utils::cast(arg).object_; + inputs.push_back(py::cast(value)); + } + } + + VectorRef outputs; + // call ms rungraph (graphId, input ,output) + sess_->RunGraph(g, inputs, &outputs); + MS_LOG(DEBUG) << "RunGraph finished:" << outputs.size(); + return outputs; +} + +void MsBackend::SetSimuCondFlag(const BaseRef &c, int flag) { + MS_LOG(DEBUG) << "while set cond :" << c.ToString() << ", " << simu_cond_map_.size(); + + if (simu_cond_map_.find(c) == simu_cond_map_.end()) { + MS_LOG(EXCEPTION) << "error c not find"; + } + simu_cond_map_[c].flag = flag; +} + +int MsBackend::GetSimuCondFlag(const BaseRef &c) { + BaseRef cond = c; + if (cond.is_null()) { + MS_LOG(DEBUG) << "get curr_switch"; + cond = curr_switch_; + } + if (simu_cond_map_.find(cond) == simu_cond_map_.end()) { + MS_LOG(ERROR) << "error c not find"; + return -1; + } + return simu_cond_map_[cond].flag; +} + +SwitchCondStatus MsBackend::SetSimuCond(const BaseRef &c, bool value) { + MS_LOG(DEBUG) << "set cond :" << c.ToString() << ", " << simu_cond_map_.size(); + + CondGraph cond_graph; + cond_graph.curr_cond = value; + if (simu_cond_map_.find(c) == simu_cond_map_.end()) { + cond_graph.flag = 0; + simu_cond_map_[c] = cond_graph; + } + + if (simu_cond_map_[c].cond_graph_map.count(value)) { + if (value == true) { + return kCondAlreadyRun; + } + } + simu_cond_map_[c].curr_cond = value; + MS_LOG(DEBUG) << "end set cond "; + return kCondOk; +} + +void MsBackend::SimulateRun(FinalVMPtr rt, FuncGraphPtr root) { + MS_LOG(DEBUG) << "Simulate run,root:" << root->ToString() << ", " << root->parameters().size(); + std::vector args; + auto parameters = root->parameters(); + (void)std::transform(parameters.begin(), parameters.end(), std::back_inserter(args), + [](const AnfNodePtr &v) { return v; }); + MS_LOG(DEBUG) << "Simulate start"; + (void)sess_->SetFinalGraphInput(parameters); + BaseRef output = rt->Eval(VectorRef(args)); + sess_->SetFinalGraphOutput(output); + MS_LOG(DEBUG) << "Simulate Eval end"; +} + +void MsBackend::Link(GraphId graph_id) { + if (graph_id == kInvalidGraphId) { + graph_id = sess_->GetFinalRunGraph(); + } + sess_->BuildGraph(graph_id); +} + +Backend::Backend(const std::string &name) : name_(name) { + MS_LOG(DEBUG) << "select backend:" << name; + convert_fn_ = backends[name_]; + is_switch_call_ = false; + is_multi_graph_sink_ = false; + simu_flag_ = false; +} + +MsBackend::MsBackend(const std::string &name, const std::string &target, uint32_t device_id) : Backend(name) { + convert_fn_ = std::bind(&MsBackend::MsConvert, this, std::placeholders::_1); + sess_ = session::SessionFactory::Get().Create(target); + if (sess_ == nullptr) { + MS_LOG(EXCEPTION) << "Session create failed!, please make sure target device:" << target << " is available."; + } + sess_->Init(device_id); + sess_->RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback); +} + +} // namespace compile +} // namespace mindspore diff --git a/mindspore/ccsrc/vm/backend.h b/mindspore/ccsrc/vm/backend.h new file mode 100644 index 00000000000..b950e7adcbf --- /dev/null +++ b/mindspore/ccsrc/vm/backend.h @@ -0,0 +1,112 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_VM_BACKEND_H_ +#define MINDSPORE_CCSRC_VM_BACKEND_H_ + +#include +#include +#include + +#include "ir/anf.h" +#include "vm/segment_runner.h" +#include "vm/vm.h" +#include "session/session_basic.h" + +namespace mindspore { +namespace compile { +enum SwitchCondStatus { + kCondOk = 0, + kCondAlreadyRun, +}; + +class Backend { + public: + explicit Backend(const std::string &name); + + virtual ~Backend() = default; + + LinkFuncType convert_fn() { return convert_fn_; } + std::string name() { return name_; } + virtual void SimulateRun(FinalVMPtr, FuncGraphPtr) {} + virtual SwitchCondStatus SetSimuCond(const BaseRef &, bool) { return kCondOk; } + virtual bool GetCond(const BaseRef &c, bool *value); + virtual void SetSwitchGraph() {} + virtual void SetSwitchActive(const BaseRef &, bool) {} + + void set_curr_switch(const BaseRef &value) { + curr_switch_ = value; + is_switch_call_ = true; + } + + BaseRef curr_switch() { return curr_switch_; } + virtual void Link(GraphId) {} + virtual LinConvertResult GetMultiGraphRun(const FuncGraphPtr &) { return LinConvertResult(); } + virtual void SetSimuCondFlag(const BaseRef &, int) {} + virtual int GetSimuCondFlag(const BaseRef &) { return 0; } + + LinConvertResult multi_result() { return multi_result_; } + void set_multi_result(const LinConvertResult &value) { multi_result_ = value; } + AnfNodePtr final_output() const { return final_output_; } + bool is_multi_graph_sink() const { return is_multi_graph_sink_; } + void set_is_multi_graph_sink(bool flag) { is_multi_graph_sink_ = flag; } + bool simu_flag() const { return simu_flag_; } + bool is_switch_call() const { return is_switch_call_; } + void set_simu_flag(bool simu) { simu_flag_ = simu; } + + protected: + std::string name_; + LinkFuncType convert_fn_; + BaseRef curr_switch_; // curr switch node + bool is_multi_graph_sink_; + bool is_switch_call_; + bool simu_flag_; + LinConvertResult multi_result_; + AnfNodePtr final_output_; +}; + +struct CondGraph { + bool curr_cond; + int flag; + std::unordered_map cond_graph_map; +}; + +class MsBackend : public Backend { + public: + MsBackend(const std::string &name, const std::string &target, uint32_t device_id); + ~MsBackend() override = default; + + LinConvertResult MsConvert(const AnfNodePtrList &lst); + VectorRef MsRunGraph(const GraphId &g, const VectorRef &args); + + VectorRef MsSimuRunGraph(const GraphId &g, const VectorRef &args); + void SimulateRun(FinalVMPtr rt, FuncGraphPtr root) override; + SwitchCondStatus SetSimuCond(const BaseRef &c, bool value) override; + + void SetSwitchGraph() override; + void SetSwitchActive(const BaseRef &c, bool cond) override; + void Link(GraphId) override; + LinConvertResult GetMultiGraphRun(const FuncGraphPtr &g) override; + void SetSimuCondFlag(const BaseRef &c, int flag) override; + int GetSimuCondFlag(const BaseRef &c) override; + + private: + session::SessionPtr sess_; + std::unordered_map simu_cond_map_; + std::unordered_map graph_id_map_; +}; +} // namespace compile +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/vm/segment_runner.cc b/mindspore/ccsrc/vm/segment_runner.cc new file mode 100644 index 00000000000..82a61c010dd --- /dev/null +++ b/mindspore/ccsrc/vm/segment_runner.cc @@ -0,0 +1,192 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF gNY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "vm/segment_runner.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/log_adapter.h" +#include "ir/manager.h" +#include "ir/func_graph_cloner.h" +#include "operator/ops.h" + +namespace mindspore { +const char kMsConvert[] = "ms"; +const char kMsVm[] = "vm"; +const char kGeVm[] = "ge"; + +namespace compile { +// cached conversion +ConvertCache g_ConvertCache; +void ClearConvertCache() { g_ConvertCache.clear(); } + +// Return the list of nodes whose values are required beyond this segment. +// Arguments: +// lst: list of nodes (the segment) +// users: dict mapping each node to its users (globally) +// seen: set of nodes that are part of the segment +AnfNodePtrList GetOutput(const AnfNodePtrList& lst, const NodeUsersMap& users, const std::vector& seen) { + AnfNodePtrList output; + if (users.size() == 0) { + return output; + } + + (void)std::transform( + std::begin(lst), std::end(lst), std::back_inserter(output), [&users, &seen](AnfNodePtr n) -> AnfNodePtr { + auto usersn = users.find(n); + bool is_referred_out_of_segment = std::any_of( + std::begin(usersn->second), std::end(usersn->second), [&seen](const std::pair& u) -> bool { + return std::find(std::begin(seen), std::end(seen), u.first) == std::end(seen); + }); + if (n->isa() && is_referred_out_of_segment) { + return n; + } + return nullptr; + }); + + // remove nullptr + for (auto it = output.begin(); it != output.end();) { + if (*it == nullptr) { + it = output.erase(it); + } else { + ++it; + } + } + + return output; +} + +std::tuple TransformSegmentToAnfGraph(const AnfNodePtrList& lst) { + auto fg = std::make_shared(); + AnfNodePtrList inputs; + AnfNodePtrToAnfNodePtrMap eqv; + if (lst.empty()) { + MS_LOG(EXCEPTION) << "Input anf node list is empty"; + } + + auto ref = [&eqv, &inputs, &fg](const AnfNodePtr& a) -> AnfNodePtr { + if (a->isa() && !IsValueNode(a)) { + eqv[a] = a; + } else if (eqv.find(a) == eqv.end()) { + inputs.push_back(a); + eqv[a] = fg->add_parameter(); + } + + return eqv[a]; + }; + + // Merge CNodes into a AnfGraph that represents a linear instruction segment + for (auto n : lst) { + if (!n->isa()) { + MS_LOG(EXCEPTION) << "Inst is not CNode"; + } + auto& inps = n->cast()->inputs(); + + if (inps.empty()) { + MS_LOG(EXCEPTION) << "Input is empty"; + } + if (!IsValueNode(inps[0])) { + MS_LOG(EXCEPTION) << "Input[0] Must be a Primitive valuenode"; + } + auto fn = inps[0]; + + std::vector args{fn}; + (void)std::transform(std::begin(inps) + 1, std::end(inps), std::back_inserter(args), ref); + + eqv[n] = fg->NewCNode(args); + } + + std::vector eqv_keys; + (void)std::transform(std::begin(eqv), std::end(eqv), std::back_inserter(eqv_keys), + [](const std::pair& elem) -> AnfNodePtr { return elem.first; }); + + auto outputs = GetOutput(lst, lst[0]->func_graph()->manager()->node_users(), eqv_keys); + std::vector output_args; + output_args.push_back(NewValueNode(prim::kPrimMakeTuple)); + (void)std::transform(std::begin(outputs), std::end(outputs), std::back_inserter(output_args), + [&eqv](const AnfNodePtr& o) -> AnfNodePtr { return eqv[o]; }); + + // Set output for AnfGraph + auto fg_output = fg->NewCNode(output_args); + fg->set_output(fg_output); + + return std::make_tuple(fg, inputs, outputs); +} + +// Converts the list of nodes to a runnable form. +// All the nodes in the list must represent linear flow (no calls, branches, ...) +// Returns: +// (fn, inputs, outputs): +// - fn: A callable function +// - inputs: the list of inputs nodes whose values should be +// provided to the function +// - outputs: the list of output nodes corresponding to the +// outputs of the function +// Notes: +// This implementation will convert the nodes into a subgraph +// that will run using the MsVM. +template +LinConvertResult Convert(const AnfNodePtrList& lst) { + auto cached = g_ConvertCache.find(lst); + if (cached != g_ConvertCache.end()) { + return cached->second; + } + + LinConvertResult result; + + FuncGraphPtr fg = nullptr; + AnfNodePtrList inputs; + AnfNodePtrList outputs; + + std::tie(fg, inputs, outputs) = TransformSegmentToAnfGraph(lst); + + // Clone in case g contains subgraphs that have a different manager + fg = BasicClone(fg); + + std::shared_ptr vm = std::make_shared(); + + result.run = + std::make_shared([fg, vm](const VectorRef& args) -> VectorRef { return vm->RunGraph(fg, args); }); + result.inputs = inputs; + result.outputs = outputs; + result.graph_id = UINT32_MAX; + + (void)g_ConvertCache.emplace(lst, result); + return result; +} + +LinkFuncType MsVmConvert = Convert; +LinkFuncType GeVmConvert = Convert; + +std::unordered_map backends = {{kMsVm, MsVmConvert}, {kGeVm, GeVmConvert}}; + +std::set backend_list = { + kMsConvert, + kMsVm, + kGeVm, +}; + +} // namespace compile +} // namespace mindspore diff --git a/mindspore/ccsrc/vm/segment_runner.h b/mindspore/ccsrc/vm/segment_runner.h new file mode 100644 index 00000000000..112a770de8d --- /dev/null +++ b/mindspore/ccsrc/vm/segment_runner.h @@ -0,0 +1,60 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_VM_SEGMENT_RUNNER_H_ +#define MINDSPORE_CCSRC_VM_SEGMENT_RUNNER_H_ + +#include +#include +#include +#include +#include + +#include "ir/anf.h" +#include "vm/vmimpl.h" + +namespace mindspore { +extern const char kMsVm[]; +extern const char kGeVm[]; +extern const char kMsConvert[]; + +namespace compile { + +struct LinConvertResult { + RunFuncPtr run; + RunFuncPtr simu_run; + std::vector inputs; + std::vector outputs; + uint32_t graph_id; +}; + +using LinkFuncType = std::function; +using ConvertCache = std::unordered_map; +extern LinkFuncType MsVmConvert; +extern LinkFuncType GeVmConvert; +extern std::unordered_map backends; +extern ConvertCache g_ConvertCache; +extern std::set backend_list; + +void ClearConvertCache(); + +std::tuple TransformSegmentToAnfGraph(const AnfNodePtrList& lst); +} // namespace compile +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_VM_SEGMENT_RUNNER_H_ diff --git a/mindspore/ccsrc/vm/transform.cc b/mindspore/ccsrc/vm/transform.cc new file mode 100644 index 00000000000..d5933db1ab9 --- /dev/null +++ b/mindspore/ccsrc/vm/transform.cc @@ -0,0 +1,644 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "vm/transform.h" + +#include +#include +#include +#include + +#include "pipeline/static_analysis/abstract_value.h" +#include "transform/convert.h" +#include "utils/graph_utils.h" +#include "utils/context/ms_context.h" +#include "debug/trace.h" + +namespace mindspore { +namespace compile { +using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractFunctionPtr; +using PrimTypePair = std::pair; +using MapPrimTypeFuncGraph = std::map; +using TypedPrimitiveAbstractClosurePtr = std::shared_ptr; + +std::vector nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch, + prim::kPrimMakeTuple}; + +std::vector ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch}; + +CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector& cut_list) + : backend_(backend), cut_list_(cut_list) { + MS_EXCEPTION_IF_NULL(backend_); + lin_convert_ = backend_->convert_fn(); + if (lin_convert_ == nullptr) { + MS_LOG(EXCEPTION) << "Attribute 'lin_convert' is null.: " << backend->name(); + } + + is_gevm_convert_ = false; + if (backend->name() == kGeVm) { + MS_LOG(INFO) << "Attribute 'is_gevm_convert' is true"; + is_gevm_convert_ = true; + } + is_graph_cut = false; +} + +bool CompileGraph::IsCut(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + auto cnode = node->cast(); + auto& inputs = cnode->inputs(); + if (inputs.empty()) { + MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; + } + + AnfNodePtr fn = inputs[0]; + if (!IsValueNode(fn)) { + return true; + } + + PrimitivePtr node_prim = GetValueNode(fn); + for (auto& prim : cut_list_) { + MS_EXCEPTION_IF_NULL(prim); + if (prim->name() == node_prim->name()) { + return true; + } + } + + if (is_gevm_convert_) { + auto name = transform::GetCNodeFuncName(cnode); + auto adpt = transform::DfGraphConvertor::FindAdapter(name); + if (adpt == nullptr) { + is_graph_cut = true; + } + return true; + } + } + + return false; +} + +VectorRef CompileGraph::SplitNodes(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + VectorRef splits; + VectorRef split; + std::vector nodes = TopoSort(graph->get_return()); + + MS_LOG(DEBUG) << "Split all nodes size:" << nodes.size(); + for (auto& node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (IsCut(node)) { + MS_LOG(DEBUG) << "Cut node:" << node->DebugString(10) << ", size:" << split.size(); + if (split.size() != 0) { + splits.push_back(split); + } + splits.push_back(node); + split.clear(); + } else if (!(node->isa() || node->isa())) { + split.push_back(node); + MS_LOG(DEBUG) << "Insert node:" << node->DebugString(10) << ", size:" << split.size(); + } + } + MS_LOG(DEBUG) << "Split node size :" << splits.size(); + return splits; +} + +// Push the value node on the stack. +void CompileGraph::Push(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + if (slots_.count(node) > 0) { + MS_LOG(EXCEPTION) << "Push failed node in slots:" << node->DebugString() + << " NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } + MS_LOG(DEBUG) << "Push node: " << node->DebugString(true) << " height_: " << height_ + << " is parameter: " << node->isa(); + slots_[node] = height_; + set_height(height_ + 1); +} + +void CompileGraph::AddInst(const Instruction& inst, const int& arg) { + VectorRef args; + args.push_back(arg); + AddInst(inst, args); +} + +void CompileGraph::AddInst(const Instruction& inst, const ValuePtr& arg) { + VectorRef args; + args.push_back(arg); + AddInst(inst, args); +} + +void CompileGraph::AddInst(const Instruction& inst, const VectorRef& args) { + inst_.push_back(std::make_pair(inst, args)); +} + +// Gets the stack reference for the node value. If the node is a constant, +// it may actually cause the push in to not be mentioned before. +int CompileGraph::Ref(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_LOG(DEBUG) << "Start Ref node " << node->DebugString(true) << " height_: " << height_; + if (slots_.count(node) == 0 && node->isa()) { + if (IsValueNode(node)) { + MS_LOG(DEBUG) << "Push graph."; + AddInst(Instruction::kGraph, GetValueNode(node)); + } else { + MS_LOG(DEBUG) << "Push."; + if (IsValueNode(node)) { + MS_LOG(EXCEPTION) << "must not be primitive in here NodeInfo: " << trace::GetDebugInfo(node->debug_info()); + } else { + AddInst(Instruction::kPush, GetValueNode(node)); + } + } + Push(node); + } + MS_LOG(DEBUG) << "End Ref node end height_: " << height_ << ", slots: " << slots_[node] + << ", return: " << slots_[node] - height_; + return slots_[node] - height_; +} + +// Make sure the value of node is at the top of the stack. +void CompileGraph::AddInput(const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + if (slots_.count(node) == 0) { + MS_LOG(DEBUG) << "Input node is null " << node->DebugString(true); + (void)Ref(node); + return; + } + AddInst(Instruction::kInput, Ref(node)); + set_height(height_ + 1); +} + +// Call back effect in stack +void CompileGraph::Ret(int nargs) { set_height(height_ - nargs); } + +void CompileGraph::PushParameters(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + std::vector parameters = graph->parameters(); + for (size_t i = parameters.size(); i != 0; i--) { + Push(parameters[i - 1]); + MS_LOG(DEBUG) << "Push parameter " << i - 1 << ": " << parameters[i - 1]->DebugString(true); + } +} + +int CompileGraph::LinConvert(const FuncGraphPtr& graph, const AnfNodePtrList& node_list) { + MS_LOG(DEBUG) << "LinConvert start"; + LinConvertResult result; + + if (backend_->simu_flag()) { + result = backend_->GetMultiGraphRun(graph); + } else { + result = lin_convert_(node_list); + } + + if (result.run == nullptr) { + MS_LOG(ERROR) << "LinConvert failed"; + return RET_FAILED; + } + + if (!(*result.run)) { + if (result.inputs.size() != result.outputs.size()) { + MS_EXCEPTION_IF_NULL(graph); + MS_LOG(EXCEPTION) << "must inputs equal outputs NodeInfo: " << trace::GetDebugInfo(graph->debug_info()); + } else { + size_t size = result.inputs.size(); + for (size_t i = 0; i < size; i++) { + Tie(result.inputs[i], result.outputs[i]); + } + return RET_CONTINUE; + } + } + AddExternal(result); + for (auto& o : result.outputs) { + Push(o); + } + + return RET_SUCCESS; +} + +void CompileGraph::AddSinkSwitch(const CNodePtr& node) { + MS_LOG(DEBUG) << "AddSinkSwitch:" << node->ToString(); + if (backend_->is_multi_graph_sink()) { + VectorRef args; + args.emplace_back(-1); + MS_LOG(DEBUG) << "call::" << height_; + AddInst(Instruction::kCall, args); + + args.clear(); + args.emplace_back(true); + AddInst(Instruction::kSwitchReturn, args); + + args.clear(); + args.emplace_back(false); + args.emplace_back(Ref(node->input(1))); + args.emplace_back(Ref(node->input(2))); + args.emplace_back(Ref(node->input(3))); + AddInst(Instruction::kSwitch, args); + } +} + +int CompileGraph::InterpretNode(const FuncGraphPtr& graph, const CNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + MS_LOG(DEBUG) << "Interpret node: " << node->DebugString(true); + std::vector node_inputs = node->inputs(); + if (node_inputs.empty()) { + MS_LOG(EXCEPTION) << "The node->inputs() is empty"; + } + AnfNodePtr fn = node_inputs[0]; + if (IsValueNode(fn)) { + PrimitivePtr value = GetValueNode(fn); + MS_LOG(DEBUG) << "The fn is primitive " << (*value).name(); + for (size_t i = node_inputs.size() - 1; i > 0; i--) { + AddInput(node->input(i)); + } + if (IsPrimitive(fn, prim::kPrimReturn)) { + AddReturn(node); + return RET_BREAK; + } + if (IsPrimitive(fn, prim::kPrimPartial)) { + AddPartial(node); + } else if (IsPrimitive(fn, prim::kPrimSwitch)) { + AddSwitch(node); + AddSinkSwitch(node); + } else if (IsPrimitive(fn, prim::kPrimMakeTuple)) { + AddMakeTuple(node); + } else { + AddPrimitive(node, value); + } + } else { + int ret = AddCall(graph, node); + if (ret == RET_BREAK) { + return ret; + } + } + Push(node); + return RET_SUCCESS; +} + +void CompileGraph::GenMultiGraphsRun(const FuncGraphPtr& graph) { + auto ret = LinConvert(graph, {}); + if (ret == RET_FAILED) { + MS_LOG(EXCEPTION) << "MultiGraphRun failed."; + } + AddReturn(nullptr); +} + +bool CompileGraph::SplitGraph(const FuncGraphPtr& graph) { + MS_LOG(DEBUG) << "Start split graph"; + MS_EXCEPTION_IF_NULL(graph); + VectorRef splits = SplitNodes(graph); + + MS_LOG(DEBUG) << "Split nodes size:" << splits.size(); + for (auto& split : splits) { + int ret = RET_SUCCESS; + if (utils::isa(split)) { + MS_LOG(DEBUG) << "Start a extern LinConvert"; + std::vector args; + auto vec_ref = utils::cast(split); + (void)std::transform(vec_ref.begin(), vec_ref.end(), std::back_inserter(args), + [](const BaseRef& v) { return utils::cast(v); }); + ret = LinConvert(graph, args); + MS_LOG(DEBUG) << "End a extern LinConvert"; + if (ret == RET_FAILED) { + return false; + } + if (ret == RET_CONTINUE) { + continue; + } + } else { + MS_LOG(DEBUG) << "Start a cut node"; + if (!(utils::isa(split) && utils::cast(split)->isa())) { + MS_LOG(EXCEPTION) << "must be anfnode here NodeInfo: " << trace::GetDebugInfo(graph->debug_info()); + } + CNodePtr node = utils::cast(split)->cast(); + ret = InterpretNode(graph, node); + MS_LOG(DEBUG) << "End a cut node"; + if (ret == RET_BREAK) { + break; + } + } + } + MS_LOG(DEBUG) << "End split graph"; + return true; +} + +InstSet CompileGraph::GenMultiGraphsSinkInst(const FuncGraphPtr& graph) { + InstSet inst = Run(graph); + return inst; +} + +InstSet CompileGraph::Run(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + MS_LOG(DEBUG) << "Compile start graph: " << graph->ToString(); + + Reset(); + PushParameters(graph); + int param_height = height_; + MS_LOG(DEBUG) << "'param_height': " << height_ << " to split graph: " << graph->get_return()->DebugString(true); + + if (backend_->simu_flag()) { + GenMultiGraphsRun(graph); + } else { + if (!SplitGraph(graph)) { + return inst_; + } + } + + AddPadStack(param_height); + auto ret = inst_; + Reset(); + return ret; +} + +void CompileGraph::AddPadStack(int param_height) { + int stack_sizes = max_height_ - param_height; + MS_LOG(DEBUG) << "Pad stack max_height_:" << max_height_ << " param:" << param_height + << " need_stack:" << stack_sizes; + if (stack_sizes > 0) { + VectorRef need_stacks({stack_sizes}); + (void)inst_.insert(inst_.begin(), std::make_pair(Instruction::kPadStack, need_stacks)); + } +} + +void CompileGraph::AddTailCall(const AnfNodePtr& fn, size_t size) { + VectorRef args; + args.emplace_back(Ref(fn)); + args.emplace_back(height_); + args.emplace_back(static_cast(size - 1)); + MS_LOG(DEBUG) << "Tail call:" << Ref(fn) << ", " << height_ << ", " << size - 1; + AddInst(Instruction::kTailCall, args); +} + +void CompileGraph::AddPartial(const CNodePtr& node) { + auto inputs = node->inputs(); + VectorRef args; + for (size_t i = 1; i < inputs.size(); i++) { + args.emplace_back(Ref(inputs[i])); + } + AddInst(Instruction::kPartial, args); +} + +void CompileGraph::AddMakeTuple(const CNodePtr& node) { + auto inputs = node->inputs(); + VectorRef args; + for (size_t i = 1; i < inputs.size(); i++) { + args.emplace_back(Ref(inputs[i])); + } + AddInst(Instruction::kTuple, args); +} + +void CompileGraph::AddSwitch(const CNodePtr& node) { + auto inputs = node->inputs(); + if (inputs.size() < 4) { + MS_LOG(EXCEPTION) << "Length of inputs of primitive " << prim::kPrimSwitch->name() << " is less than 4"; + } + VectorRef args; + if (backend_->is_multi_graph_sink()) { + args.emplace_back(true); + } + args.emplace_back(Ref(inputs[1])); + args.emplace_back(Ref(inputs[2])); + args.emplace_back(Ref(inputs[3])); + AddInst(Instruction::kSwitch, args); +} + +void CompileGraph::AddReturn(const CNodePtr& node) { + VectorRef args; + if (backend_->simu_flag()) { + args.emplace_back(Ref(backend_->final_output())); + } else { + args.emplace_back(Ref(node->input(1))); + } + args.emplace_back(height_); + AddInst(Instruction::kReturn, args); +} + +void CompileGraph::AddPrimitive(const CNodePtr& node, const PrimitivePtr& prim) { + auto inputs = node->inputs(); + VectorRef args; + args.push_back(prim); + for (size_t i = 1; i < inputs.size(); i++) { + args.emplace_back(Ref(inputs[i])); + } + AddInst(Instruction::kPrim, args); +} + +int CompileGraph::AddCall(const FuncGraphPtr& graph, const CNodePtr& node) { + auto node_inputs = node->inputs(); + AnfNodePtr fn = node_inputs[0]; + (void)Ref(fn); + size_t size = node_inputs.size(); + for (size_t i = size - 1; i > 0; i--) { + AddInput(node_inputs[i]); + } + if (node == graph->output()) { + AddTailCall(fn, size); + return RET_BREAK; + } + MS_LOG(DEBUG) << "Call:" << Ref(fn) << ", " << height_ << ", " << size - 1; + AddInst(Instruction::kCall, Ref(fn)); + Ret(static_cast(size - 1)); + return RET_SUCCESS; +} + +void CompileGraph::AddExternal(const LinConvertResult& result) { + VectorRef args; + args.push_back(result.run); + args.push_back(result.simu_run); + size_t size = result.inputs.size(); + for (size_t i = 0; i < size; i++) { + args.emplace_back(Ref(result.inputs[i])); + } + AddInst(Instruction::kExternal, args); +} + +void TraverseGraphMap( + const FuncGraphManagerPtr& manager_ptr, FuncGraphTransaction* const tr, const FuncGraphToAnfNodeCounterMap& cts, + const std::function(const PrimitivePtr, const AbstractFunctionPtr)>& get_prim_graph) { + MS_EXCEPTION_IF_NULL(manager_ptr); + MS_EXCEPTION_IF_NULL(tr); + for (const auto& ct_graphs : cts) { + for (const auto& ct_any : ct_graphs.second) { + AnfNodePtr const_primitive_node = ct_any.first; + if (const_primitive_node != nullptr && IsValueNode(const_primitive_node)) { + auto users = manager_ptr->node_users()[const_primitive_node]; + for (auto& use : users) { + CNodePtr node = use.first->cast(); + MS_EXCEPTION_IF_NULL(node); + int key = use.second; + if (key != 0) { + MS_EXCEPTION_IF_NULL(node->input(0)); + bool key_is_const = node->input(0)->isa(); + PrimitivePtr value = GetValueNode(node->input(0)); + bool is_prim_array_map = !(prim::kPrimArrayMap->name().compare(value->name())); + bool is_prim_array_reduce = !(prim::kPrimArrayReduce->name().compare(value->name())); + if (key == 1 && key_is_const && (is_prim_array_map || is_prim_array_reduce)) { + continue; + } + FuncGraphPtr g = get_prim_graph(GetValueNode(const_primitive_node), + dyn_cast(const_primitive_node->abstract())); + tr->SetEdge(node, key, NewValueNode(g)); + } + } + } + } + } +} + +FuncGraphPtr WrapPrimitives(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + FuncGraphManagerPtr manager_ptr = graph->manager(); + MS_EXCEPTION_IF_NULL(manager_ptr); + MapPrimTypeFuncGraph prim_graphs; + auto get_prim_graph = [&](const PrimitivePtr& prim, const AbstractFunctionPtr& type) { + PrimTypePair prim_type = std::make_pair(prim, type); + if (prim_graphs.end() == prim_graphs.find(prim_type)) { + FuncGraphPtr g = std::make_shared(); + std::vector args; + ValueNodePtr prim_ct = NewValueNode(prim); + MS_EXCEPTION_IF_NULL(prim_ct); + prim_ct->set_abstract(type); + args.push_back(prim_ct); + MS_EXCEPTION_IF_NULL(type); + TypedPrimitiveAbstractClosurePtr tp = dyn_cast(type->GetUnique()); + MS_EXCEPTION_IF_NULL(tp); + MS_EXCEPTION_IF_NULL(g); + for (auto t : tp->args_spec_list()) { + ParameterPtr p = g->add_parameter(); + p->set_abstract(t); + args.push_back(p); + } + AnfNodePtr out = g->NewCNode(args); + out->set_abstract(tp->output()); + g->set_output(out); + prim_graphs[prim_type] = g; + } + + return prim_graphs[prim_type]; + }; + + FuncGraphTransaction tr = manager_ptr->Transact(); + auto& cts = manager_ptr->valuenodes(); + TraverseGraphMap(manager_ptr, &tr, cts, get_prim_graph); + + return graph; +} + +CompileGraphs::CompileGraphs(const BackendPtr& backend, const std::vector& cut_list) : backend_(backend) { + MS_EXCEPTION_IF_NULL(backend); + MS_LOG(DEBUG) << "Start vm: " << backend->name(); + transform_ = std::make_shared(backend, cut_list); + Reset(); +} + +// Convert graphs to unlinked instructions. +void CompileGraphs::Compile(const FuncGraphPtr& graph) { + MS_LOG(DEBUG) << "Start"; + auto graph_manager = graph->manager(); + MS_EXCEPTION_IF_NULL(graph_manager); + FuncGraphSet graphs = graph_manager->func_graphs(); + for (auto& g : graphs) { + mapping_[g] = static_cast(insts_.size()); + if (transform_ != nullptr) { + InstSet insts = transform_->Run(g); + if (!insts.empty()) { + (void)insts_.insert(insts_.end(), insts.begin(), insts.end()); + } + } + } + MS_LOG(DEBUG) << "End"; +} + +// Link instructions from multiple function graphs together. +FinalVMPtr CompileGraphs::Link(const FuncGraphPtr& graph) { + MS_LOG(DEBUG) << "Start"; + for (std::size_t i = 0; i < insts_.size(); i++) { + InstType inst = insts_[i]; + MS_LOG(DEBUG) << "Link point:" << inst_str[inst.first]; + if (Instruction::kGraph == inst.first) { + if (inst.second.empty()) { + MS_LOG(EXCEPTION) << "The second element of inst is empty"; + } + FuncGraphPtr func_graph = utils::cast(inst.second[0])->cast(); + MS_LOG(DEBUG) << "Link graph:" << func_graph->ToString(); + insts_[i] = std::make_pair(Instruction::kPush, VectorRef(std::vector{mapping_[func_graph]})); + } + } + + FinalVMPtr rt = std::make_shared(insts_, backend_); + if (backend_->is_multi_graph_sink()) { + backend_->set_simu_flag(true); + MS_LOG(DEBUG) << "Start simulate"; + backend_->SimulateRun(rt, graph); + MS_LOG(DEBUG) << "Link graphs"; + insts_ = transform_->GenMultiGraphsSinkInst(graph); + rt->set_insts(insts_); + backend_->set_simu_flag(false); + MS_LOG(DEBUG) << "End start simulate"; + backend_->Link(kInvalidGraphId); + } + MS_LOG(DEBUG) << "End"; + return rt; +} + +// Convert all graphs to unlinked instructions and link them. +FinalVMPtr CompileGraphs::CompileAndLink(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + MS_LOG(DEBUG) << "Start"; + Reset(); + MS_LOG(DEBUG) << "Begin parameter:" << graph->parameters().size(); + + (void)WrapPrimitives(graph); + Compile(graph); + +#ifdef ENABLE_GE + if (!transform_->IsGraphCut()) { + return nullptr; + } +#endif + + FinalVMPtr rt = Link(graph); + Reset(); + MS_LOG(DEBUG) << "End"; + return rt; +} + +BackendPtr CreateBackend() { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + std::string name = context_ptr->backend_policy(); + MS_LOG(INFO) << "CreateBackend is: " << name; + if (backend_list.count(name) == 0) { + MS_LOG(EXCEPTION) << "Backend is error: " << name; + } + + if (name == kMsConvert) { + std::string target = context_ptr->device_target(); + uint32_t device_id = context_ptr->device_id(); + auto backend = std::make_shared(name, target, device_id); + std::string device_target = MsContext::GetInstance()->device_target(); + if (device_target == kAscendDevice) { + backend->set_is_multi_graph_sink(true); + context_ptr->set_is_multi_graph_sink(true); + } + return backend; + } + + return std::make_shared(name); +} +} // namespace compile +} // namespace mindspore diff --git a/mindspore/ccsrc/vm/transform.h b/mindspore/ccsrc/vm/transform.h new file mode 100644 index 00000000000..206fd00431e --- /dev/null +++ b/mindspore/ccsrc/vm/transform.h @@ -0,0 +1,142 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_VM_TRANSFORM_H_ +#define MINDSPORE_CCSRC_VM_TRANSFORM_H_ + +#include +#include +#include +#include +#include +#include + +#include "vm/vm.h" +#include "ir/anf.h" +#include "operator/ops.h" +#include "vm/segment_runner.h" +#include "vm/backend.h" + +// mindspore namespace is the top level namespace of Mindsporeession project. +// Other namespace should be a sub namespace of mindspore namespace in the ME project. +namespace mindspore { +extern const char kMsVm[]; +extern const char kGeVm[]; + +// compile namespace +// A sub namespace in ME to support compile related definition. +namespace compile { +extern std::vector nonlinear_ops; +extern std::vector ms_nonlinear_ops; + +using VmEvalFunc = std::function; +using VmEvalFuncPtr = std::shared_ptr>; + +class CompileGraph { + public: + explicit CompileGraph(const BackendPtr& backend, const std::vector& cut_list = nonlinear_ops); + + ~CompileGraph() = default; + + InstSet Run(const FuncGraphPtr& func_graph); + InstSet GenMultiGraphsSinkInst(const FuncGraphPtr& graph); + bool IsGraphCut() const { return is_graph_cut; } + bool IsCut(const AnfNodePtr& node); + void Push(const AnfNodePtr& node); + void Tie(const AnfNodePtr& n1, const AnfNodePtr& n2) { slots_[n2] = slots_[n1]; } + void Ret(int nargs); + void GenMultiGraphsRun(const FuncGraphPtr& graph); + int Ref(const AnfNodePtr& node); + VectorRef SplitNodes(const FuncGraphPtr& func_graph); + + void set_height(int h) { + height_ = h; + if (height_ > max_height_) { + max_height_ = height_; + } + } + + void Reset() { + height_ = 0; + max_height_ = 0; + slots_.clear(); + inst_.clear(); + } + + private: + void PushParameters(const FuncGraphPtr& func_graph); + bool SplitGraph(const FuncGraphPtr& func_graph); + int LinConvert(const FuncGraphPtr& func_graph, const AnfNodePtrList& node_list); + int InterpretNode(const FuncGraphPtr& func_graph, const CNodePtr& node); + int AddCall(const FuncGraphPtr& graph, const CNodePtr& node); + void AddSinkSwitch(const CNodePtr& node); + void AddPadStack(int param_height); + void AddTailCall(const AnfNodePtr& fn, size_t size); + void AddPartial(const CNodePtr& node); + void AddMakeTuple(const CNodePtr& node); + void AddSwitch(const CNodePtr& node); + void AddReturn(const CNodePtr& node); + void AddPrimitive(const CNodePtr& node, const PrimitivePtr& prim); + void AddInput(const AnfNodePtr& node); + void AddExternal(const LinConvertResult& result); + void AddInst(const Instruction& inst, const int& arg); + void AddInst(const Instruction& inst, const ValuePtr& arg); + void AddInst(const Instruction& inst, const VectorRef& args); + + BackendPtr backend_; + LinkFuncType lin_convert_; + bool is_gevm_convert_; + bool is_graph_cut; + int height_{0}; + int max_height_{0}; + std::vector cut_list_; + std::unordered_map slots_; + InstSet inst_; +}; + +using CompileGraphPtr = std::shared_ptr; + +// CompileGraphs is used to Convert a graph cluster into instruction lists. +class CompileGraphs { + public: + explicit CompileGraphs(const BackendPtr& backend, const std::vector& cut_list = nonlinear_ops); + + ~CompileGraphs() = default; + + void Reset() { + insts_.clear(); + mapping_.clear(); + } + + void Compile(const FuncGraphPtr& func_graph); + FinalVMPtr Link(const FuncGraphPtr& func_graph); + FinalVMPtr CompileAndLink(const FuncGraphPtr& func_graph); + + private: + InstSet insts_; + std::unordered_map mapping_; + CompileGraphPtr transform_; + BackendPtr backend_; +}; + +BackendPtr CreateBackend(); + +} // namespace compile +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_VM_TRANSFORM_H_ diff --git a/mindspore/ccsrc/vm/vm.cc b/mindspore/ccsrc/vm/vm.cc new file mode 100644 index 00000000000..493873b0bcf --- /dev/null +++ b/mindspore/ccsrc/vm/vm.cc @@ -0,0 +1,523 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "vm/vm.h" + +#include + +#include "vm/vmimpl.h" +#include "vm/backend.h" +#include "vm/transform.h" +#include "pipeline/parse/data_converter.h" + +namespace mindspore { +namespace compile { + +// Initialize StructPartial. +// Arguments: +// fn_: Callable function. +// args_: Sequence of function args. +StructPartial::StructPartial(int fn, const VectorRef& args) : fn_(fn), args_(args) {} + +std::ostream& operator<<(std::ostream& os, const StructPartial& other) { + os << "partial(" << other.fn_ << ", " << other.args_.ToString() << ")"; + return os; +} + +bool operator==(const StructPartial& lhs, const StructPartial& rhs) { + return (lhs.fn_ == rhs.fn_ && lhs.args_ == rhs.args_); +} + +StructSimuSwitch::StructSimuSwitch(const BaseRef& fn, const BaseRef& value) : fn_(fn), value_(value) {} + +std::ostream& operator<<(std::ostream& os, const StructSimuSwitch& other) { + os << "SimulSwitch(" << other.fn_.ToString() << ", " << other.value_.ToString() << ")"; + return os; +} + +bool operator==(const StructSimuSwitch& lhs, const StructSimuSwitch& rhs) { + return (lhs.fn_ == rhs.fn_ && lhs.value_ == rhs.value_); +} + +std::ostream& operator<<(std::ostream& os, const SwitchCondStatus& other) { + os << "SwitchCondStatus(" << static_cast(other) << ")"; + return os; +} + +// Follow the specified instructions to create a VM. +// Arguments: +// insts_: std::vector> +// insts_stack_: The value stack. +// retp_: The call stack. +// pc_: program counter (next instruction) +// sp_: stack pointer (for the value stack) +FinalVM::FinalVM(const InstSet& insts, const BackendPtr& backend) : insts_(insts), pc_(0), sp_(0), backend_(backend) { + MS_LOG(DEBUG) << "InstSet size:" << insts_.size(); + insts_stack_.emplace_back(BaseRef()); + retp_.push(-1); +} + +void FinalVM::Push(const BaseRef& v) { + MS_LOG(DEBUG) << "Push " << v.ToString() << " sp_:" << sp_; + insts_stack_[IntToSize(sp_++)] = v; +} + +void FinalVM::Pop(int n) { + if (n > sp_) { + MS_LOG(EXCEPTION) << "Invalid value of n " << n << ", it should be not more than " << sp_ - 1; + } + for (int i = 0; i < n; i++) { + insts_stack_[IntToSize(sp_ - i - 1)] = BaseRef(); + } + sp_ -= n; +} + +void FinalVM::MoveStack(int nitems, int height) { + if (nitems > height || height > sp_) { + MS_LOG(EXCEPTION) << "MoveStack arg error: nitems=" << nitems << " height=" << height; + } + int n = height - nitems; + int src = sp_ - height; + int dst = sp_ - nitems; + for (int i = 0; i < nitems; i++) { + insts_stack_[IntToSize(src + i)] = insts_stack_[IntToSize(dst + i)]; + } + Pop(n); +} + +BaseRef FinalVM::Ref(int i) { + MS_LOG(DEBUG) << "Ref i:" << i << " sp_:" << sp_; + size_t sp_next = IntToSize(sp_ + i); + if (sp_next < insts_stack_.size()) { + if (utils::isa(insts_stack_[sp_next])) { + py::object value = utils::cast(insts_stack_[sp_next]).object_; + MS_LOG(DEBUG) << "VM ref python:" << py::str(value); + return parse::data_converter::PyDataToValue(value); + } + MS_LOG(DEBUG) << "Ref not python :" << insts_stack_[sp_next].ToString(); + return insts_stack_[sp_next]; + } + + MS_LOG(EXCEPTION) << "IndexError: index(" << sp_next << ") out of range [0, " << insts_stack_.size() << ")."; +} + +void FinalVM::Pushp() { retp_.push(pc_); } + +void FinalVM::Popp() { + if (retp_.empty()) { + MS_LOG(EXCEPTION) << "Stack retp_ is empty"; + } + pc_ = retp_.top(); + MS_LOG(DEBUG) << "Pop pc:" << pc_ << ", sp:" << sp_; + retp_.pop(); +} + +void FinalVM::Pushsp() { retsp_.push(sp_); } + +void FinalVM::Popsp() { + int sp = retsp_.top(); + MS_LOG(DEBUG) << "Current sp:" << sp_ << ", before sp:" << sp << ", " << sp_ - sp; + if (sp_ >= sp) { + Pop(sp_ - sp + 1); + retsp_.pop(); + } else { + MS_LOG(EXCEPTION) << "Stack point sp_:" << sp << " must biger than sp:" << sp_; + } +} + +void FinalVM::DoJmp(const BaseRef& jmp_orig) { + MS_LOG(DEBUG) << "Start"; + + BaseRef jmp = jmp_orig; + if (backend_->simu_flag()) { + if (utils::isa(jmp)) { // need to inherit from Base + MS_LOG(DEBUG) << "Start jump StructSwitch"; + auto simu_value = utils::cast>(jmp); + jmp = simu_value->fn_; + backend_->set_curr_switch(simu_value->value_); + } + } + + if (utils::isa(jmp)) { // need to inherit from Base + MS_LOG(DEBUG) << "Start jump StructPartial"; + auto new_jmp = utils::cast>(jmp); + auto args = new_jmp->args_; + InstPadStack(VectorRef(std::vector{static_cast(args.size())})); + auto iter = args.rbegin(); + for (; iter != args.rend(); ++iter) { + Push(*iter); + } + pc_ = new_jmp->fn_; + return; + } + + if (!utils::isa(jmp)) { + MS_LOG(EXCEPTION) << "Jmp inst should be a int"; + } + pc_ = utils::cast(jmp); + MS_LOG(DEBUG) << "End do jump pc_:" << pc_; +} + +BaseRef FinalVM::Eval(const VectorRef& args) { + MS_LOG(DEBUG) << "Start: " << args.size(); + insts_stack_.clear(); + insts_stack_.resize(args.size()); + std::stack().swap(retp_); + retp_.push(-1); + pc_ = 0; + sp_ = 0; + + auto riter = args.rbegin(); + for (; riter != args.rend(); ++riter) { + if (utils::isa(*riter)) { + PyObjectRef py_ref = utils::cast(*riter); + py::object value = py_ref.object_; + if (py::isinstance(value)) { + auto a = py::cast(value); + Push(static_cast(a)); + continue; + } + } + Push(*riter); + } + + while (pc_ >= 0) { + auto inst = insts_[IntToSize(pc_)]; + MS_LOG(DEBUG) << "Loop " << insts_.size() << ", pc:" << pc_ << ", inst:" << inst_str[inst.first]; + ++pc_; + auto iter = inst_function_map.find(inst.first); + if (iter != inst_function_map.end()) { + iter->second(inst.second); + } else { + MS_LOG(EXCEPTION) << "Unknown instruction {" << inst_str[inst.first] << "}"; + } + } + + MS_LOG(DEBUG) << "End"; + return insts_stack_[0]; +} + +void FinalVM::InstCall(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 1; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameter, while the input size is " + << args.size() << "."; + return; + } + + int jmp = utils::cast(args[0]); + MS_LOG(DEBUG) << "Call pushp:" << pc_ << ", jmp:" << jmp << ", sp:" << sp_; + Pushp(); + DoJmp(Ref(jmp)); + MS_LOG(DEBUG) << "Instcall end sp :" << sp_; +} + +void FinalVM::InstTailCall(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 3; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameters, while the input size is " + << args.size() << "."; + return; + } + + int jmp = utils::cast(args[0]); + int height = utils::cast(args[1]); + int nargs = utils::cast(args[2]); + + auto new_jmp = Ref(jmp); + + if (backend_->simu_flag()) { + if (backend_->GetSimuCondFlag(BaseRef()) == 2) { + MS_LOG(DEBUG) << "invoke while call tail first"; + Pop(height); + Push(1); + Popp(); + return; + } + } + MoveStack(nargs, height); + MS_LOG(DEBUG) << "TailCall pushp:" << pc_ << ", jmp:" << jmp; + DoJmp(new_jmp); + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstSwitchReturn(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + if (args.size() != 1) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires one parameter, while the input size is " << args.size() << "."; + return; + } + Pop(1); + Popsp(); +} + +void FinalVM::InstReturn(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 2; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameters, while the input size is " + << args.size() << "."; + return; + } + + int rpos = utils::cast(args[0]); + int height = utils::cast(args[1]); + + auto rv = Ref(rpos); + if (backend_->simu_flag() && backend_->is_switch_call()) { + backend_->SetSwitchGraph(); + } + + Pop(height); + Push(rv); + Popp(); + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstPartial(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 1; + if (args.size() < args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " or more parameters, while the input size is " + << args.size() << "."; + return; + } + + int fn_ = utils::cast(args[0]); + auto fn = utils::cast(Ref(fn_)); + MS_LOG(DEBUG) << "Partial argssize:" << args.size(); + std::vector outs(args.size() - 1); + + (void)std::transform(args.begin() + 1, args.end(), outs.begin(), + [&, this](const BaseRef& a) { return Ref(utils::cast(a)); }); + Push(std::make_shared(fn, VectorRef(outs))); + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstSimuSwitch(const VectorRef& args) { + const size_t args_size = 4; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameters, while the input size is " + << args.size() << "."; + return; + } + bool cond = utils::cast(args[0]); + int cond_node = utils::cast(args[1]); + int vtrue = utils::cast(args[2]); + int vfalse = utils::cast(args[3]); + + MS_LOG(DEBUG) << "Simu switch cond:" << cond; + BaseRef c = Ref(cond_node); + bool bool_value = cond; + SwitchCondStatus cond_stat = backend_->SetSimuCond(c, bool_value); + + int cond_flag = backend_->GetSimuCondFlag(c); + MS_LOG(DEBUG) << "Simu switch cond:" << cond << ", " << cond_flag << ", " << c.cast()->DebugString(); + if (cond_flag == 2) { + Popp(); + Popp(); + backend_->SetSimuCondFlag(c, 0); + return; + } + + if (cond_stat == kCondAlreadyRun) { + MS_LOG(DEBUG) << "switch alreay run bool while true jmp"; + if (cond_flag == 0) { + MS_LOG(DEBUG) << "switch second run bool while true jmp"; + backend_->SetSwitchActive(c, true); + Push(std::make_shared(Ref(vtrue), c)); + Pushsp(); + backend_->SetSimuCondFlag(c, 1); + return; + } else if (cond_flag == 1) { + MS_LOG(DEBUG) << "switch first run bool while if jmp"; + Push(std::make_shared(Ref(vfalse), c)); + (void)backend_->SetSimuCond(c, false); + backend_->SetSimuCondFlag(c, 2); + return; + } else { + MS_LOG(EXCEPTION) << "error cond not find"; + return; + } + } + if (bool_value) { + Push(std::make_shared(Ref(vtrue), c)); + Pushsp(); + } else { + Push(std::make_shared(Ref(vfalse), c)); + } +} + +void FinalVM::InstRealSwitch(const VectorRef& args) { + const size_t args_size = 3; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameters, while the input size is " + << args.size() << "."; + return; + } + + int cond = utils::cast(args[0]); + int vtrue = utils::cast(args[1]); + int vfalse = utils::cast(args[2]); + + BaseRef c = Ref(cond); + MS_LOG(DEBUG) << "" << vtrue << " false:" << vfalse << " InstSwitch: " << c.ToString(); + bool bool_value = false; + if (backend_->GetCond(c, &bool_value)) { + MS_LOG(DEBUG) << "Cond:" << bool_value; + if (bool_value) { + Push(Ref(vtrue)); + } else { + Push(Ref(vfalse)); + } + } else { + MS_LOG(EXCEPTION) << "Not supported type to be casted to bool"; + } +} + +void FinalVM::InstSwitch(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + if (backend_->is_multi_graph_sink()) { + InstSimuSwitch(args); + } else { + InstRealSwitch(args); + } +} + +void FinalVM::InstTuple(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + VectorRef tuple; + auto iter = args.begin(); + for (; iter != args.end(); ++iter) { + auto a = utils::cast(*iter); + tuple.push_back(Ref(a)); + } + Push(tuple); + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstPush(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 1; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameter, while the input size is " + << args.size() << "."; + return; + } + + auto v = args[0]; + Push(v); + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstInput(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 1; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameter, while the input size is " + << args.size() << "."; + return; + } + + int rpos = utils::cast(args[0]); + Push(Ref(rpos)); + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstPadStack(const VectorRef& args) { + MS_LOG(DEBUG) << "Start"; + const size_t args_size = 1; + if (args.size() != args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " parameter, while the input size is " + << args.size() << "."; + return; + } + + int sz = utils::cast(args[0]); + MS_LOG(DEBUG) << "" << insts_stack_.size() << " need padstack " << sz << " sp_ " << sp_; + size_t stack_size = insts_stack_.size(); + int need = sz - (static_cast(stack_size) - sp_); + if (need > 0) { + MS_LOG(DEBUG) << "InstPadStack resize: size:" << insts_stack_.size() << " need pad:" << need; + insts_stack_.resize(stack_size + IntToSize(need)); + } + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstExternal(const VectorRef& args) { + MS_LOG(DEBUG) << "Start:" << args.size(); + + if (args.empty()) { + MS_LOG(EXCEPTION) << "Args is empty!"; + } + + VectorRef tuple; + RunFunctionRef run_ref = utils::cast(args[0]); + compile::RunFuncPtr fn = run_ref.func_; + if (backend_->simu_flag()) { + MS_LOG(DEBUG) << "Simu run"; + if (args.size() == 1) { + MS_LOG(EXCEPTION) << "The number of args should be greater than 1, but got 1"; + } + auto simu_run_ref = utils::cast(args[1]); + fn = simu_run_ref.func_; + } + for (size_t i = 2; i < args.size(); ++i) { + auto index = utils::cast(args[i]); + tuple.push_back(Ref(index)); + } + + if (!fn) { + MS_LOG(EXCEPTION) << "Function not callable"; + } + + auto outs = (*fn)(tuple); + MS_LOG(DEBUG) << "'fn' out size:" << outs.size(); + for (auto& o : outs) { + MS_LOG(DEBUG) << "InstExternal value:" << o.ToString(); + Push(o); + } + MS_LOG(DEBUG) << "End"; +} + +void FinalVM::InstPushPrim(const VectorRef& args) { + MS_LOG(DEBUG) << "Start: " << args.size(); + const size_t args_size = 2; + if (args.size() < args_size) { + MS_LOG(ERROR) << "" << __FUNCTION__ << " requires " << args_size << " or more parameters, while the input size is " + << args.size() << "."; + return; + } + + VectorRef tuple; + auto prim = utils::cast(args[0]); + for (size_t i = 1; i < args.size(); ++i) { + auto index = utils::cast(args[1]); + tuple.push_back(Ref(index)); + } + + auto outs = RunOperation(prim, tuple); + Push(outs); + + MS_LOG(DEBUG) << "End"; +} + +} // namespace compile +} // namespace mindspore diff --git a/mindspore/ccsrc/vm/vm.h b/mindspore/ccsrc/vm/vm.h new file mode 100644 index 00000000000..3e1e5b5c085 --- /dev/null +++ b/mindspore/ccsrc/vm/vm.h @@ -0,0 +1,153 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_VM_VM_H_ +#define MINDSPORE_CCSRC_VM_VM_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "utils/base_ref.h" + +namespace mindspore { +namespace compile { + +class Backend; +using BackendPtr = std::shared_ptr; + +enum Instruction { + kCall = 0, + kTailCall, + kReturn, + kPartial, + kSwitch, + kSwitchReturn, + kTuple, + kInput, + kExternal, + kPush, + kPrim, + kGraph, + kPadStack +}; + +using InstType = std::pair; +using InstSet = std::vector; +using InstFunctionMap = std::map>; + +const std::vector inst_str{"call", "tail_call", "return", "partial", "switch", "switch_return", "tuple", + "input", "external", "push", "primitive", "graph", "pad_stack"}; +class StructPartial : public Base { + public: + // Initialize StructPartial. + StructPartial(int fn, const VectorRef& args); + + virtual ~StructPartial() = default; + MS_DECLARE_PARENT(StructPartial, Base) + + int fn_; + VectorRef args_; +}; + +std::ostream& operator<<(std::ostream& os, const StructPartial& other); +bool operator==(const StructPartial& lhs, const StructPartial& rhs); + +class StructSimuSwitch : public Base { + public: + StructSimuSwitch(const BaseRef& fn, const BaseRef& value); + + virtual ~StructSimuSwitch() = default; + MS_DECLARE_PARENT(StructSimuSwitch, Base) + + BaseRef fn_; + BaseRef value_; +}; + +std::ostream& operator<<(std::ostream& os, const StructSimuSwitch& other); +bool operator==(const StructSimuSwitch& lhs, const StructSimuSwitch& rhs); + +class FinalVM { + public: + // Create a VM with the specified instructions and backend. + explicit FinalVM(const InstSet& insts, const BackendPtr& backend); + + virtual ~FinalVM() = default; + + BaseRef Eval(const VectorRef& args); + void InstCall(const VectorRef& args); + void InstTailCall(const VectorRef& args); + void InstReturn(const VectorRef& args); + void InstPartial(const VectorRef& args); + void InstSwitch(const VectorRef& args); + void InstSimuSwitch(const VectorRef& args); + void InstRealSwitch(const VectorRef& args); + void InstTuple(const VectorRef& args); + void InstPush(const VectorRef& args); + void InstInput(const VectorRef& args); + void InstPadStack(const VectorRef& args); + void InstExternal(const VectorRef& args); + void InstPushPrim(const VectorRef& args); + void InstSwitchReturn(const VectorRef& args); + void set_insts(const InstSet& value) { insts_ = value; } + + protected: + BaseRef Ref(int i); + void Push(const BaseRef& v); + void Pop(int n = 1); + void MoveStack(int nitems, int height); + void Pushp(); + void Popp(); + void Pushsp(); + void Popsp(); + void DoJmp(const BaseRef& jmp); + + private: + InstSet insts_; + std::deque insts_stack_; + std::stack retp_; + std::stack retsp_; + int pc_; + int sp_; + BackendPtr backend_; + const InstFunctionMap inst_function_map = { + {Instruction::kCall, [this](const VectorRef& args) { InstCall(args); }}, + {Instruction::kTailCall, [this](const VectorRef& args) { InstTailCall(args); }}, + {Instruction::kReturn, [this](const VectorRef& args) { InstReturn(args); }}, + {Instruction::kPartial, [this](const VectorRef& args) { InstPartial(args); }}, + {Instruction::kSwitch, [this](const VectorRef& args) { InstSwitch(args); }}, + {Instruction::kTuple, [this](const VectorRef& args) { InstTuple(args); }}, + {Instruction::kPush, [this](const VectorRef& args) { InstPush(args); }}, + {Instruction::kInput, [this](const VectorRef& args) { InstInput(args); }}, + {Instruction::kPadStack, [this](const VectorRef& args) { InstPadStack(args); }}, + {Instruction::kExternal, [this](const VectorRef& args) { InstExternal(args); }}, + {Instruction::kPrim, [this](const VectorRef& args) { InstPushPrim(args); }}, + {Instruction::kSwitchReturn, [this](const VectorRef& args) { InstSwitchReturn(args); }}, + }; +}; + +using FinalVMPtr = std::shared_ptr; + +} // namespace compile +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_VM_VM_H_ diff --git a/mindspore/ccsrc/vm/vmimpl.cc b/mindspore/ccsrc/vm/vmimpl.cc new file mode 100644 index 00000000000..e64cd16fcf8 --- /dev/null +++ b/mindspore/ccsrc/vm/vmimpl.cc @@ -0,0 +1,500 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "vm/vmimpl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "transform/graph_runner.h" +#include "transform/convert.h" +#include "ir/meta_tensor.h" +#include "operator/ops.h" +#include "ir/manager.h" +#include "ir/func_graph_cloner.h" +#include "utils/convert_utils.h" +#include "debug/draw.h" + +namespace mindspore { +namespace compile { + +using PrimitivePyPtr = std::shared_ptr; + +static const char SEGMENT_GRAPH_NAME[] = "runnable_segment"; + +VectorRef GeVM::RunGraph(const FuncGraphPtr& anf_graph, const VectorRef& args) { + // Convert graph + transform::DfGraphConvertor convertor(anf_graph); + + (void)convertor.ConvertAllNode().BuildGraph(); + if (convertor.ErrCode() == 0) { + (void)transform::DfGraphManager::GetInstance().AddGraph(SEGMENT_GRAPH_NAME, convertor.GetComputeGraph()); + } else { + MS_LOG(EXCEPTION) << "convert df graph failed"; + } + + // Run graph + transform::GraphRunnerOptions options; + transform::GraphRunner graph_runner(options); + transform::RunOptions run_options; + run_options.name = SEGMENT_GRAPH_NAME; + + std::vector inputs; + (void)std::transform(std::begin(args), std::end(args), std::back_inserter(inputs), + [](const BaseRef& arg) -> tensor::TensorPtr { + auto value_ref = utils::cast(arg); + auto value = value_ref.object_; + return py::cast(value); + }); + std::vector outputs; + (void)graph_runner.RunGraph(run_options, inputs, &outputs); + std::vector ret; + (void)std::copy(outputs.begin(), outputs.end(), std::back_inserter(ret)); + return VectorRef(ret); +} + +// Indicate a call to a new frame. +struct CallWrap : public Base { + explicit CallWrap(const VMFramePtr& vm_frame) : frame(vm_frame) {} + VMFramePtr frame{nullptr}; +}; +using CallWrapPtr = std::shared_ptr; + +// Indicates a return with its value. +struct ReturnWrap : public Base { + explicit ReturnWrap(const BaseRef& r_value) : value(r_value) {} + BaseRef value{BaseRef()}; +}; +using ReturnWrapPtr = std::shared_ptr; + +VMFrame::VMFrame(const AnfNodePtrList& nodes, const AnfNodePtrToBaseRefMap& values, + const AnfNodePtrToBaseRefMap& closure) + : values_(values), todo_(nodes), closure_(closure) { + std::reverse(std::begin(todo_), std::end(todo_)); +} + +const BaseRef VMFrame::operator[](const AnfNodePtr& node) { + MS_EXCEPTION_IF_NULL(node); + auto ret = values_.find(node); + if (ret != values_.end()) { + return ret->second; + } + + ret = closure_.find(node); + if (ret != closure_.end()) { + return ret->second; + } + + if (node->isa()) { + return GetValueNode(node); + } + + MS_LOG(EXCEPTION) << "ValueError " << node->type_name(); +} + +Closure::Closure(const FuncGraphPtr& graph, const AnfNodePtrToBaseRefMap& values) + : func_graph_(graph), values_(values) {} + +BaseRef Closure::operator()(const VectorRef& args) { + MS_LOG(DEBUG) << "start closure"; + return vm_->Evaluate(func_graph_, args, values_); +} + +Partial::Partial(const BaseRef& fn, const VectorRef& args, const VMPtr& vm) : fn_(fn), args_(args), vm_(vm) {} + +BaseRef Partial::operator()(const VectorRef& nodes) { + VectorRef arglist; + (void)arglist.insert(arglist.end(), args_.begin(), args_.end()); + (void)arglist.insert(arglist.end(), nodes.begin(), nodes.end()); + return vm_->Call(fn_, arglist); +} + +SetRef VM::ComputeFvs(const FuncGraphPtr& graph) { + MS_EXCEPTION_IF_NULL(graph); + SetRef rval; + for (auto& fkv : graph->free_variables_total()) { + if (utils::isa(fkv.first)) { + // Add all value_nodes of g that refer to a fv graph + auto g = utils::cast(fkv.first); + for (auto& ctkv : g->value_nodes()) { + auto ct = ctkv.first; + if (GetValueNode(ct) == g) { + (void)rval.insert(ct); + } + } + } else { + // Add a normal fv + (void)rval.insert(fkv.first); + } + } + + return rval; +} + +void VM::AcquireGraph(const FuncGraphPtr& graph) { + // Already acquired + if (vars_.find(graph) != vars_.end()) { + return; + } + // Add g to manager + manager_->AddFuncGraph(graph); + // Compute fvs for all acquired graph + auto graphs = graph->manager()->func_graphs(); + for (auto g = graphs.begin(); g != graphs.end(); ++g) { + vars_[*g] = ComputeFvs(*g); + } +} + +VectorRef VM::ExportSequence(const VectorRef& seq) { + std::vector ret; + (void)std::transform(std::begin(seq), std::end(seq), std::back_inserter(ret), + [&, this](const BaseRef& x) -> BaseRef { return Export(x); }); + return VectorRef(ret); +} + +ClosurePtr VM::ExportClosure(const ClosurePtr& clos) { + MS_EXCEPTION_IF_NULL(clos); + clos->set_vm(shared_from_this()); + return clos; +} + +// transform graph to executable closure +ClosurePtr VM::ExportGraph(const FuncGraphPtr& g) { + auto c = std::make_shared(g, AnfNodePtrToBaseRefMap()); + MS_EXCEPTION_IF_NULL(c); + c->set_vm(shared_from_this()); + return c; +} + +BaseRef VM::ExportObj(const BaseRef& obj) const { return obj; } + +BaseRef VM::Export(const BaseRef& value) { + if (utils::isa(value) && utils::cast(value)->isa()) { + return ExportGraph(utils::cast(value)->cast()); + } + + if (utils::isa(value) && utils::cast(value)->isa()) { + return ExportPrimitive(utils::cast(value)->cast()); + } + + if (utils::isa(value)) { + return ExportGraph(utils::cast(value)); + } + + if (utils::isa(value)) { + return ExportClosure(utils::cast(value)); + } + + if (utils::isa(value)) { + return ExportPrimitive(utils::cast(value)); + } + + if (utils::isa(value)) { + return ExportSequence(utils::cast(value)); + } + + return ExportObj(value); +} + +// Run a graph. +// This will evaluate the passed-in graph and return the resulting value. +BaseRef VM::Evaluate(const FuncGraphPtr& graph, const VectorRef& args, const AnfNodePtrToBaseRefMap& closure) { + AcquireGraph(graph); + MS_LOG(DEBUG) << "evalue arg size: " << args.size(); + if (args.size() != graph->parameters().size()) { + MS_LOG(EXCEPTION) << "Call with wrong number of arguments, expect " << graph->parameters().size() << ", but got " + << args.size(); + } + + // toposort graph nodes, the order will be reversed by frame so that the dependent be computed first + auto nodes = TopoSort(graph->get_return(), SuccVm(graph)); + // mapping parameters to args + AnfNodePtrToBaseRefMap values; + for (size_t i = 0; i < args.size(); i++) { + values[graph->parameters()[i]] = args[i]; + } + // create top frame with params initialized + VMFramePtrList frames{std::make_shared(nodes, values, closure)}; + // execute frames starting from top frame + while (!frames.empty()) { + auto frame = frames[frames.size() - 1]; + auto todo = frame->todo(); + while (!todo.empty()) { + auto except = HandleNode(todo[todo.size() - 1], frame); + if (utils::isa(except)) { + if (todo.size() == 2) { + // The last element is always a return, replace the ret with call frame + frames[frames.size() - 1] = utils::cast(except)->frame; + } else { + frames.push_back(utils::cast(except)->frame); + } + break; + } + if (utils::isa(except)) { + (void)frames.erase(frames.begin() + (static_cast(frames.size()) - 1)); + if (frames.size() > 0) { + auto top = frames[frames.size() - 1]; + auto td = top->todo(); + // set value for top frame's last evaluated node + if (td.empty()) { + MS_LOG(EXCEPTION) << "The td is empty"; + } + top->values()[td[td.size() - 1]] = utils::cast(except)->value; + (void)td.erase(td.begin() + (static_cast(td.size()) - 1)); + } else { + return Export(utils::cast(except)->value); + } + break; + } + (void)todo.erase(todo.begin() + (static_cast(todo.size()) - 1)); + } + } + MS_LOG(EXCEPTION) << "VM Evaluate error"; +} + +SuccFunc VM::SuccVm(const FuncGraphPtr& graph) { + auto fn = [&, this](const AnfNodePtr& node) -> AnfNodePtrList { + MS_EXCEPTION_IF_NULL(node); + AnfNodePtrList ret; + + // Follow node.incoming + if (node->isa()) { + auto& inputs = node->cast()->inputs(); + for (auto& i : inputs) { + if (i->func_graph() == node->func_graph() || + (IsValueNode(i) && GetValueNode(i)->parent() == graph)) { + ret.push_back(i); + } + } + } + + // for subgraph input, add their fvs as succ nodes + if (IsValueNode(node) && GetValueNode(node)->parent() == graph) { + auto fvs = utils::cast(vars_[GetValueNode(node)]); + (void)std::transform(fvs.begin(), fvs.end(), std::back_inserter(ret), + [](const BaseRef& value) -> AnfNodePtr { return utils::cast(value); }); + } + + return ret; + }; + return fn; +} + +BaseRef VM::Call(const BaseRef& fn, const VectorRef& args) { + if (utils::isa(fn)) { + return RunOperation(utils::cast(fn), args); + } + + if (utils::isa(fn)) { + return Evaluate(utils::cast(fn), args); + } + + if (utils::isa(fn)) { + auto clos = utils::cast(fn); + return Evaluate(clos->func_graph(), args, clos->values()); + } + + MS_LOG(EXCEPTION) << "Can't call fn"; +} + +// make call frame for graph +BaseRef VM::_Call(const BaseRef& graph, const VectorRef& args) { + AnfNodePtrToBaseRefMap clos; + auto func_graph = graph; + if (utils::isa(func_graph)) { + clos = utils::cast(func_graph)->values(); + func_graph = utils::cast(func_graph)->func_graph(); + } + if (utils::isa(func_graph)) { + func_graph = utils::cast(func_graph)->cast(); + } + + if (!utils::isa(func_graph)) { + MS_LOG(EXCEPTION) << "Graph type error"; + } + + auto graphPtr = utils::cast(func_graph); + + if (vars_.find(graphPtr) == vars_.end()) { + AcquireGraph(graphPtr); + } + + if (args.size() != graphPtr->parameters().size()) { + MS_LOG(EXCEPTION) << "Call with wrong number of arguments, expect " << graphPtr->parameters().size() << ", but got " + << args.size(); + } + + auto nodes = TopoSort(graphPtr->get_return(), SuccVm(graphPtr)); + AnfNodePtrToBaseRefMap values; + for (size_t i = 0; i < args.size(); i++) { + values[graphPtr->parameters()[i]] = args[i]; + } + + return std::make_shared(std::make_shared(nodes, values, clos)); +} + +// make closure out of graph with fv values from frame +ClosurePtr VM::MakeClosure(const FuncGraphPtr& graph, const VMFramePtr& frame) { + MS_EXCEPTION_IF_NULL(frame); + AnfNodePtrToBaseRefMap clos; + + for (auto& v : utils::cast(vars_[graph])) { + auto anf = utils::cast(v); + clos[anf] = (*frame)[anf]; + } + + return std::make_shared(graph, clos); +} + +BaseRef VM::DispatchCall(const AnfNodePtr& node, const VMFramePtr& frame, const BaseRef& fn, const VectorRef& args) { + if (utils::isa(fn) && utils::cast(fn)->isa()) { + auto fnval = utils::cast(fn)->cast(); + MS_LOG(DEBUG) << "DispatchCall prim:" << fnval->name() << ", node:" << node->DebugString(true); + if (args.empty()) { + MS_LOG(EXCEPTION) << "args is empty"; + } + if (fnval == prim::kPrimReturn) { + MS_LOG(DEBUG) << "return args:" << args.size(); + return std::make_shared(args[0]); + } + + if (fnval == prim::kPrimMakeTuple) { + frame->values()[node] = args; + return BaseRef(); + } + + if (fnval == prim::kPrimPartial) { + VectorRef partial_args(args.begin() + 1, args.end()); + frame->values()[node] = (std::make_shared(args[0], partial_args, shared_from_this())); + return BaseRef(); + } + + // call prim implementation + frame->values()[node] = RunOperation(fnval, args); + return BaseRef(); + } + + // partial args logic + if (utils::isa(fn)) { + auto fnPtr = utils::cast(fn); + + VectorRef arglist; + (void)arglist.insert(arglist.end(), fnPtr->args().begin(), fnPtr->args().end()); + (void)arglist.insert(arglist.end(), args.begin(), args.end()); + + auto ret = DispatchCall(node, frame, fnPtr->fn(), arglist); + if (utils::isa(ret) || utils::isa(ret)) { + return ret; + } + } + + // create frame for graph and closure + if ((utils::isa(fn) && utils::cast(fn)->isa()) || utils::isa(fn)) { + auto ret = _Call(fn, args); + if (utils::isa(ret) || utils::isa(ret)) { + return ret; + } + } + + MS_LOG(EXCEPTION) << "Invalid fn to call"; +} + +BaseRef VM::HandleNode(const AnfNodePtr& node, const VMFramePtr& frame) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + // pass + return BaseRef(); + } + + if (node->isa()) { + // We only visit valuenode graphs + if (!IsValueNode(node)) { + MS_LOG(EXCEPTION) << "We only visit valuenode graphs "; + } + auto g = GetValueNode(node); + + // if g is a graph with fvs, we need to make a closure for it + auto iterG = vars_.find(g); + if (iterG != vars_.end() && utils::cast(iterG->second).size() != 0) { + frame->values()[node] = MakeClosure(g, frame); + } + + return BaseRef(); + } + + if (node->isa()) { + std::vector fnArgs; + auto& inputs = node->cast()->inputs(); + // set args' values in frame + (void)std::transform(std::begin(inputs), std::end(inputs), std::back_inserter(fnArgs), + [&](const AnfNodePtr& inp) -> BaseRef { return (*frame)[inp]; }); + if (fnArgs.empty()) { + MS_LOG(EXCEPTION) << "function arguments is empty"; + } else { + auto args = VectorRef(fnArgs.begin() + 1, fnArgs.end()); + auto except = DispatchCall(node, frame, fnArgs[0], args); + return except; + } + } + + MS_LOG(EXCEPTION) << "Unknown node type"; +} + +VectorRef VM::RunGraph(const FuncGraphPtr& g, const VectorRef& args) { + this->manager_ = Manage(g); + + auto fn = utils::cast(Export(g)); + auto result = (*fn)(args); + + if (utils::isa(result)) { + return utils::cast(result); + } else { + VectorRef ret({result}); + return ret; + } +} + +BaseRef RunOperation(const PrimitivePtr& prim, const VectorRef& args) { + PrimitivePyPtr operation = dyn_cast(prim); + + MS_LOG(DEBUG) << "operation start " << prim->name(); + auto func = operation != nullptr ? operation->GetComputeFunction() : prim->GetComputeFunction(); + if (py::isinstance(func)) { + MS_LOG(EXCEPTION) << "" << prim->name() << " 's compute function is not implemented"; + } + + py::tuple py_args = py::tuple(args.size()); + MS_LOG(DEBUG) << "input for operation:"; + size_t i = 0; + for (auto& arg : args) { + py_args[i] = BaseRefToPyData(arg); + MS_LOG(DEBUG) << "arg: " << i << ":"; + i++; + } + py::object obj = func(*py_args); + MS_LOG(DEBUG) << "result:" << py::str(obj); + return obj; +} + +} // namespace compile +} // namespace mindspore diff --git a/mindspore/ccsrc/vm/vmimpl.h b/mindspore/ccsrc/vm/vmimpl.h new file mode 100644 index 00000000000..8ff02ae9462 --- /dev/null +++ b/mindspore/ccsrc/vm/vmimpl.h @@ -0,0 +1,195 @@ +/** + * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). + * + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_VM_VMIMPL_H_ +#define MINDSPORE_CCSRC_VM_VMIMPL_H_ + +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/manager.h" +#include "ir/meta_tensor.h" +#include "utils/base_ref.h" + +namespace mindspore { +namespace compile { + +using AnfNodePtrList = std::vector; +using AnfNodePtrToBaseRefMap = std::unordered_map; +using AnfNodePtrToAnfNodePtrMap = std::unordered_map; + +using FuncGraphPtrToBaseRefMap = std::unordered_map; + +using TensorList = std::vector; + +class Closure; +using ClosurePtr = std::shared_ptr; + +class VMFrame; +using VMFramePtr = std::shared_ptr; +using VMFramePtrList = std::vector; + +class VM; +using VMPtr = std::shared_ptr; + +class Partial; +using PartialPtr = std::shared_ptr; + +using RunFunc = std::function; +using RunFuncPtr = std::shared_ptr; + +using SuccFunc = std::function; + +class VMImpl { + public: + virtual VectorRef RunGraph(const FuncGraphPtr& fg, const VectorRef& args) = 0; + virtual ~VMImpl() = default; +}; + +class GeVM : public VMImpl { + public: + VectorRef RunGraph(const FuncGraphPtr& fg, const VectorRef& args) override; + ~GeVM() override = default; +}; + +// An execution frame. +// This holds the state for an application of a graph. The nodes list +// must contain free variables of graphs encountered before the +// graph themselves. +// You can index a frame with a node to get its value in the context +// of this frame (if it has already been evaluated). +// Attributes: +// nodes: list of nodes remaining to execute +// values: Mapping of node to their values in this application +// closure: values for the closure if the current application is a closure +class VMFrame { + public: + VMFrame(const AnfNodePtrList& nodes, const AnfNodePtrToBaseRefMap& values, const AnfNodePtrToBaseRefMap& closure); + const BaseRef operator[](const AnfNodePtr& node); + const AnfNodePtrList& todo() const { return todo_; } + + AnfNodePtrToBaseRefMap& values() { return values_; } + + virtual ~VMFrame() = default; + + AnfNodePtrToBaseRefMap values_; + + private: + AnfNodePtrList todo_; + AnfNodePtrToBaseRefMap closure_; +}; + +// Representation of a closure. +class Closure : public Base { + public: + Closure(const FuncGraphPtr& func_graph, const AnfNodePtrToBaseRefMap& values); + BaseRef operator()(const VectorRef& args); + + const VMPtr& vm() const { return vm_; } + + void set_vm(const VMPtr& vm) { vm_ = vm; } + + const FuncGraphPtr& func_graph() const { return func_graph_; } + + const AnfNodePtrToBaseRefMap& values() const { return values_; } + + virtual ~Closure() = default; + + MS_DECLARE_PARENT(Closure, Base) + + private: + FuncGraphPtr func_graph_; + AnfNodePtrToBaseRefMap values_; + VMPtr vm_; +}; + +// Representation of a partial application. +class Partial : public Base { + public: + Partial(const BaseRef& fn, const VectorRef& args, const VMPtr& vm); + BaseRef operator()(const VectorRef& nodes); + const BaseRef& fn() const { return fn_; } + + const VectorRef& args() const { return args_; } + + virtual ~Partial() = default; + MS_DECLARE_PARENT(Partial, Base) + + private: + BaseRef fn_; + VectorRef args_; + VMPtr vm_; +}; + +// Virtual Machine interface. +class VM : public std::enable_shared_from_this, public VMImpl { + public: + SetRef ComputeFvs(const FuncGraphPtr& func_graph); + + void AcquireGraph(const FuncGraphPtr& func_graph); + + VectorRef ExportSequence(const VectorRef& seq); + + BaseRef ExportPrimitive(const PrimitivePtr&) const { return kAnyValue; } + + ClosurePtr ExportClosure(const ClosurePtr& clos); + + // Return an object that executes `fg` when called on arguments. + ClosurePtr ExportGraph(const FuncGraphPtr& fg); + + BaseRef ExportObj(const BaseRef& obj) const; + + BaseRef Export(const BaseRef& value); + + // Run a graph. + // This will evaluate the passed-in graph and return the + // resulting value. + BaseRef Evaluate(const FuncGraphPtr& func_graph, const VectorRef& args, + const AnfNodePtrToBaseRefMap& closure = AnfNodePtrToBaseRefMap()); + + // Return a visitor for the graph. + SuccFunc SuccVm(const FuncGraphPtr& func_graph); + + // Call the `fn` object. + // `fn` can be anything that would be valid as the first element of an apply. + BaseRef Call(const BaseRef& fn, const VectorRef& args); + + BaseRef _Call(const BaseRef& graph, const VectorRef& args); + + ClosurePtr MakeClosure(const FuncGraphPtr& func_graph, const VMFramePtr& frame); + + BaseRef DispatchCall(const AnfNodePtr& node, const VMFramePtr& frame, const BaseRef& fn, const VectorRef& args); + + BaseRef HandleNode(const AnfNodePtr& node, const VMFramePtr& frame); + + VectorRef RunGraph(const FuncGraphPtr& fg, const VectorRef& args) override; + + private: + FuncGraphManagerPtr manager_; + FuncGraphPtrToBaseRefMap vars_; +}; + +extern BaseRef RunOperation(const PrimitivePtr& prim, const VectorRef& args); + +} // namespace compile +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_VM_VMIMPL_H_ diff --git a/mindspore/common/__init__.py b/mindspore/common/__init__.py new file mode 100644 index 00000000000..ead8aee556e --- /dev/null +++ b/mindspore/common/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Top-level reference to dtype of common module.""" +from . import dtype +from .api import ms_function +from .dtype import * +from .parameter import Parameter, ParameterTuple +from .tensor import MetaTensor, Tensor + +__all__ = [ + "MetaTensor", "Tensor", # tensor + 'ms_function', # api + 'Parameter', 'ParameterTuple', # parameter + "dtype" + ] + +__all__.extend(dtype.__all__) diff --git a/mindspore/common/_register_for_tensor.py b/mindspore/common/_register_for_tensor.py new file mode 100644 index 00000000000..da183d95491 --- /dev/null +++ b/mindspore/common/_register_for_tensor.py @@ -0,0 +1,35 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Registry the relation.""" + +from collections import UserDict + + +class Registry(UserDict): + """Registry class for registry functions for tensor call primitive ops function.""" + + def register(self, obj_str, obj): + if isinstance(obj_str, str): + self[obj_str] = obj + + def get(self, obj_str): + """Get the value by str.""" + if isinstance(obj_str, str): + obj = self[obj_str] + return obj + + +tensor_operator_registry = Registry() diff --git a/mindspore/common/api.py b/mindspore/common/api.py new file mode 100644 index 00000000000..8e23e9184dd --- /dev/null +++ b/mindspore/common/api.py @@ -0,0 +1,510 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Providing interface methods.""" +import types +from collections import OrderedDict +from functools import wraps +from mindspore import context +from mindspore import log as logger +from mindspore.parallel._utils import _get_parallel_mode +from .._c_expression import generate_key, Executor_, Tensor, MetaTensor +from .._c_expression import verify_inputs_signature, init_exec_dataset, export_graph, _set_dataset_mode_config, init_ge +from .tensor import Tensor as MsTensor + +# store ms_function class compiled pipeline cache +ms_compile_cache = {} + + +def _convert_function_arguments(fn, *args): + """ + Process the fn default parameters. + + Args: + fn (Function): The function to be parsed. + args (tuple): The parameters of the function. + """ + arguments_dict = OrderedDict() + parse_method = None + if isinstance(fn, (types.FunctionType, types.MethodType)): + parse_method = fn.__name__ + index = 0 + for value in args: + arguments_dict[f'arg{index}'] = value + index = index + 1 + logger.debug("fn(%r) full parameters dict is: %r", fn, arguments_dict) + converted = True + else: + logger.warning("Find error: fn isn't function or method") + converted = False + return converted, arguments_dict, parse_method + + +def _wrap_func(fn): + """ + Wrapper function, convert return data to tensor or tuple of tensor. + + Args: + fn (Function): The function need be wrapped. + + Returns: + Function, a new function with return suitable format data. + """ + @wraps(fn) + def wrapper(*arg, **kwargs): + results = fn(*arg, **kwargs) + + def _convert_data(data): + if isinstance(data, Tensor) and not isinstance(data, MsTensor): + return MsTensor(data) + return data + + if isinstance(results, tuple): + return tuple(_convert_data(x) for x in results) + if isinstance(results, list): + return list(_convert_data(x) for x in results) + return _convert_data(results) + + return wrapper + + +def _exec_init_graph(obj, init_phase): + """Execute the parameter initializer graph.""" + inst_executor = Executor_.get_instance() + exec_init_graph = False + for param in obj.get_parameters(): + if not param.is_init: + param.is_init = True + exec_init_graph = True + + if exec_init_graph: + inst_executor.run_init_graph(obj.parameters_dict(), init_phase) + + +class _MindSporeFunction: + """ + Represents a function compiled by mind expression. + + _MindSporeFunction will compile the original function for every combination + of argument types and shapes it is given (as well as their values, optionally). + + Args: + fn (Function): The root function to compile. + input_signature (Function): User defines signature to verify input. + obj (Object): If function is a method, obj is the owner of function, + else, obj is none. + """ + def __init__(self, fn, input_signature=None, obj=None): + self.fn = fn + self.save_graphs = context.get_context("save_graphs") + self.save_graphs_path = context.get_context("save_graphs_path") + self.input_signature = input_signature + self.obj = None + self.identify_obj = None + if hasattr(obj, fn.__name__): + self.obj = obj + elif obj is not None: + self.identify_obj = obj + self._executor = Executor_.get_instance() + + def build_data_init_graph(self, graph_name): + """Build GE data graph and init graph for the given graph name.""" + if self.obj is None: + logger.warning("Make sure parameter should not be used in function") + para_dict = OrderedDict() + self._executor.build_data_graph(para_dict, graph_name) + return + self._executor.build_data_graph(self.obj.parameters_dict(), graph_name, self.obj.parameters_broadcast_dict()) + init_phase = "init_subgraph" + graph_name[graph_name.find("."):] + _exec_init_graph(self.obj, init_phase) + + def compile(self, arguments_dict, method_name): + """Returns pipline for the given args.""" + args_list = tuple(arguments_dict.values()) + arg_names = tuple(arguments_dict.keys()) + + # remove first self parameter when fn is a method + if self.obj is not None: + args_list = args_list[1:] + arg_names = arg_names[1:] + + # verify the signature for both function and method + if self.input_signature is not None: + signatures = [] + for sig_spec in self.input_signature: + if not isinstance(sig_spec, MetaTensor): + raise TypeError("Input_signature is not MetaTensor") + signatures.append(sig_spec) + is_valid_input = verify_inputs_signature(signatures, args_list) + if not is_valid_input: + raise ValueError("Inputs is incompatible with input signature!") + + dic = dict(zip(arg_names, args_list)) + generate_name = self.fn.__module__ + "." + self.fn.__name__ + self.fn.__parse_method__ = method_name + + # replace key with obj info and object ext info when fn is a method + if self.obj is not None: + self.obj.__parse_method__ = method_name + generate_name = self.obj.__module__ + "." + str(self.obj.create_time) + if self.identify_obj is not None: + generate_name = generate_name + str(id(self.identify_obj)) + + key = generate_key(generate_name, dic) + phase = str(key[1]) + generate_name + if key not in ms_compile_cache.keys(): + is_compile = False + if self.obj is None: + is_compile = self._executor.compile(self.fn, args_list, phase, True) + else: + is_compile = self._executor.compile(self.obj, args_list, phase, True) + if not is_compile: + raise RuntimeError("Executor compile failed.") + if context.get_context("enable_ge"): + self.build_data_init_graph(phase) + # since function can be redefined, we only cache class method pipeline + if self.obj is not None or self.identify_obj is not None: + ms_compile_cache[key] = phase + return phase + + return ms_compile_cache[key] + + @_wrap_func + def __call__(self, *args): + init_ge() + converted, arguments_dict, parse_method = _convert_function_arguments(self.fn, *args) + if not converted: + raise RuntimeError('Process function parameter is failure') + + args_list = tuple(arguments_dict.values()) + if self.obj is not None: + args_list = args_list[1:] + + phase = self.compile(arguments_dict, parse_method) + + if context.get_context("precompile_only"): + return None + return self._executor(args_list, phase) + + +def ms_function(fn=None, obj=None, input_signature=None): + """ + Creates a callable MindSpore graph from a python function. + + This allows the MindSpore runtime to apply optimizations based on graph. + + Args: + fn (Function): The Python function that will be run as a graph. Default: None. + obj (Object): The Python Object that provide information for identify compiled function. Default: None. + input_signature (MetaTensor): The MetaTensor to describe the input arguments. The MetaTensor specifies + the shape and dtype of the Tensor and they will be supplied to this function. If input_signature + is specified, every input to `fn` must be a `Tensor`. And the input parameters of `fn` cannot accept + `**kwargs`. The shape and dtype of actual inputs should keep same with input_signature, or TypeError + will be raised. Default: None. + + Returns: + Function, if `fn` is not None, returns a callable that will execute the compiled function; If `fn` is None, + returns a decorator and when this decorator invokes with a single `fn` argument, the callable is equal to the + case when `fn` is not None. + + Examples: + >>> def tensor_add(x, y): + >>> z = F.tensor_add(x, y) + >>> return z + >>> + >>> @ms_function + >>> def tensor_add_with_dec(x, y): + >>> z = F.tensor_add(x, y) + >>> return z + >>> + >>> @ms_function(input_signature=(MetaTensor(mstype.float32, (1, 1, 3, 3)), + >>> MetaTensor(mstype.float32, (1, 1, 3, 3)))) + >>> def tensor_add_with_sig(x, y): + >>> z = F.tensor_add(x, y) + >>> return z + >>> + >>> x = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)) + >>> y = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)) + >>> + >>> tensor_add_graph = ms_function(fn=tensor_add) + >>> out = tensor_add_graph(x, y) + >>> out = tensor_add_with_dec(x, y) + >>> out = tensor_add_with_sig(x, y) + """ + def wrap_mindspore(func): + @wraps(func) + def staging_specialize(*args): + process_obj = obj + if args and not isinstance(args[0], MsTensor) and hasattr(args[0], func.__name__): + process_obj = args[0] + args = (x.default_input if hasattr(x, 'default_input') else x for x in args) + return _MindSporeFunction(func, input_signature, process_obj)(*args) + + return staging_specialize + + if fn is not None: + return wrap_mindspore(fn) + return wrap_mindspore + + +def _generate_pip_args(obj, *args, method="construct"): + """Generate arguments for pipeline.""" + if hasattr(obj, method): + fn = getattr(obj, method) + else: + raise AttributeError('The process method is not exist') + converted, arguments_dict, parse_method = _convert_function_arguments(fn, *args) + if not converted: + raise RuntimeError('Process method parameter is failure') + args_list = tuple(arguments_dict.values()) + args_names = tuple(arguments_dict.keys()) + obj.__parse_method__ = parse_method + return args_names, args_list + + +class _Executor: + """ + An executor used to compile/manage/run graph. + + Including data_graph, train_graph, eval_graph and predict graph. + + Returns: + Graph, return the result of pipeline running. + """ + + def __init__(self): + # create needed graph by lazy mode + self.is_init = False + self._executor = Executor_.get_instance() + self.compile_cache = {} + self.phase_prefix = "" + + def init_dataset(self, queue_name, dataset_size, batch_size, dataset_types, dataset_shapes, + input_indexs, phase='dataset'): + """ + Initialization interface for calling data subgraph. + + Args: + queue_name (str): The name of tdt queue on the device. + dataset_size (int): The size of dataset. + batch_size (int): The size of batch. + dataset_types (list): The output types of element in dataset. + dataset_shapes (list): The output shapes of element in dataset. + input_indexs (list): The index of data with net. + phase (str): The name of phase, e.g., train_dataset/eval_dataset. Default: 'dataset'. + + Returns: + bool, specifies whether the data subgraph was initialized successfully. + """ + if not init_exec_dataset(queue_name=queue_name, + size=dataset_size, + batch_size=batch_size, + types=dataset_types, + shapes=dataset_shapes, + input_indexs=input_indexs, + phase=phase): + raise RuntimeError("Failure to init and dataset subgraph!") + return True + + def _build_data_graph(self, obj, params, phase): + if params is None: + self._executor.build_data_graph(obj.parameters_dict(), phase, obj.parameters_broadcast_dict()) + elif isinstance(params, OrderedDict): + self._executor.build_data_graph(params, phase) + else: + raise TypeError('Parameters need OrderedDict type, but got {}'. + format(type(params))) + + def compile(self, obj, *args, phase='predict', params=None): + """ + Compiles graph. + + Args: + obj (Function/Cell): The function or cell instance need compile. + args (tuple): Function or cell input arguments. + phase (str): The name of compile phase. Default: 'predict'. + params (OrderedDict): The parameters dictionary used for init data graph. Default: None. + + Return: + Str, the full phase of the cell. + Bool, if the graph has been compiled before, return False, else return True. + """ + obj.check_names() + args_names, args_list = _generate_pip_args(obj, *args) + dic = dict(zip(args_names, args_list)) + key = generate_key(phase, dic) + self.phase_prefix = str(key[1]) + if phase == 'export': + phase = phase + '.' + str(obj.create_time) + else: + phase = self.phase_prefix + phase + '.' + str(obj.create_time) + enable_debug_runtime = context.get_context("enable_debug_runtime") + enable_ge = context.get_context("enable_ge") + + use_vm = not enable_ge or (enable_debug_runtime and context.get_context("mode") == context.PYNATIVE_MODE) + + if phase in self.compile_cache.keys(): + logger.debug("%r graph has existed.", phase) + return phase, False + + result = self._executor.compile(obj, args_list, phase, use_vm) + self.compile_cache[phase] = phase + if not result: + raise RuntimeError("Executor compile failed.") + graph = self._executor.get_func_graph(phase) + + if graph is None: + logger.error("%r graph compile failed.", phase) + + if not enable_debug_runtime or enable_ge: + if _get_parallel_mode() in ["auto_parallel", "semi_auto_parallel"]: + obj.parameter_layout_dict = self._executor.get_parameter_layout(phase) + obj.load_parameter_slice(params) + + if _get_parallel_mode() in ["hybrid_parallel"]: + obj.parameter_layout_dict = self._build_parameter_layout(obj) + + # the following GE init process is not needed when use vm or ms backend + if enable_ge: + # decide whether to sink based on whether the inputs is virtual or not + if args_list and isinstance(args_list[0], Tensor) and args_list[0].virtual_flag: + _set_dataset_mode_config('graph') + else: + _set_dataset_mode_config('feed') + + self._build_data_graph(obj, params, phase) + + if "export" not in phase: + init_phase = "init_subgraph" + "." + str(obj.create_time) + _exec_init_graph(obj, init_phase) + elif not enable_ge and "export" in phase: + self._build_data_graph(obj, params, phase) + + return phase, True + + def _get_strategy(self, obj): + real_phase = self.phase_prefix + obj.phase + '.' + str(obj.create_time) + return self._executor.get_strategy(real_phase) + + def _get_allreduce_fusion(self, obj): + real_phase = self.phase_prefix + obj.phase + '.' + str(obj.create_time) + return self._executor.get_allreduce_fusion(real_phase) + + def has_compiled(self, phase='predict'): + """ + Specify whether have been compiled. + + Args: + phase (str): The phase name. Default: 'predict'. + + Returns: + bool, specifies whether the specific graph has been compiled. + """ + return self._executor.has_compiled(phase) + + def __call__(self, obj, *args, phase='predict'): + if context.get_context("precompile_only"): + return None + return self.run(obj, *args, phase=phase) + + @_wrap_func + def _exec_pip(self, obj, *args, phase=''): + """Execute the generated pipeline.""" + fn = obj.construct + converted, arguments_dict, parse_method = _convert_function_arguments(fn, *args) + if not converted: + raise RuntimeError('Process method parameter is failure') + args_list = tuple(arguments_dict.values()) + obj.__parse_method__ = parse_method + return self._executor(args_list, phase) + + def run(self, obj, *args, phase='predict'): + """ + Run the specific graph. + + Args: + phase (str): The phase name. Default: 'predict'. + + Returns: + Tensor/Tuple, return execute result. + """ + if phase == 'save': + return self._executor((), phase + '.' + str(obj.create_time)) + + phase_real = self.phase_prefix + phase + '.' + str(obj.create_time) + if self.has_compiled(phase_real): + return self._exec_pip(obj, *args, phase=phase_real) + raise KeyError('{} graph is not exist.'.format(phase_real)) + + def _build_parameter_layout(self, obj): + """ + Build parameter layout, for layerwise_parallel parameter. + + Args: + obj (Function or Cell): The function or cell instance need to be compiled. + + Returns: + Dictionary, parameter layout info. + """ + parameter_layout_dict = {} + layerwise_parallel_parameters = [] + for key in obj.parameters_dict(): + if obj.parameters_dict()[key].layerwise_parallel is True: + layerwise_parallel_parameters.append(key) + + if not layerwise_parallel_parameters: + return parameter_layout_dict + + from ..communication.management import get_group_size + group_size = [get_group_size()] + for key in layerwise_parallel_parameters: + tensor_map = [0] + shape = obj.parameters_dict()[key].data.shape() + for x in range(len(shape)): # dim 0 set 0, others set -1 + if x: + tensor_map.append(-1) + layout = [group_size, tensor_map] + parameter_layout_dict[key] = layout + + return parameter_layout_dict + + def del_net_res(self, net_id): + self._executor.del_net_res(net_id) + + def _get_func_graph_proto(self, exec_id, ir_type="onnx_ir", use_prefix=False): + """Get graph proto from pipeline.""" + if use_prefix: + exec_id = self.phase_prefix + exec_id + if self._executor.has_compiled(exec_id) is False: + return None + return self._executor.get_func_graph_proto(exec_id, ir_type) + + def export(self, net, file_name, file_format='GEIR'): + """ + Export graph. + + Args: + net (Cell): MindSpore network + file_name (str): File name of model to export + file_format (str): MindSpore currently support 'GEIR' and 'ONNX' format for exported model + """ + phase = 'export' + '.' + str(net.create_time) + export_graph(file_name, file_format, phase) + + +_executor = _Executor() + +__all__ = ['ms_function'] diff --git a/mindspore/common/dtype.py b/mindspore/common/dtype.py new file mode 100644 index 00000000000..702e01effb5 --- /dev/null +++ b/mindspore/common/dtype.py @@ -0,0 +1,243 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Data type for MindSpore.""" + +import numpy as np +from .._c_expression import typing, EnvInstance_ +from .._c_expression.typing import Type + +__dtype__ = [ + "int8", "byte", + "int16", "short", + "int32", "intc", + "int64", "intp", + "uint8", "ubyte", + "uint16", "ushort", + "uint32", "uintc", + "uint64", "uintp", + "float16", "half", + "float32", "single", + "float64", "double", + "bool_", "float_", + "list_", "tuple_", + "string", "type_none", + "tensor_type", + "Type" +] + +__method__ = [ + "dtype_to_nptype", "issubclass_", "dtype_to_pytype", + "pytype_to_dtype", "get_py_obj_dtype" +] + +__all__ = ["Type"] +__all__.extend(__dtype__) +__all__.extend(__method__) + +# type definition +bool_ = typing.Bool() + +int8 = typing.Int(8) +byte = int8 +int16 = typing.Int(16) +short = int16 +int32 = typing.Int(32) +intc = int32 +int64 = typing.Int(64) +intp = int64 + +uint8 = typing.UInt(8) +ubyte = uint8 +uint16 = typing.UInt(16) +ushort = uint16 +uint32 = typing.UInt(32) +uintc = uint32 +uint64 = typing.UInt(64) +uintp = uint64 + +float16 = typing.Float(16) +half = float16 +float32 = typing.Float(32) +single = float32 +float64 = typing.Float(64) +double = float64 + +int_ = typing.Int() +uint = typing.UInt() +float_ = typing.Float() +number = typing.Number() + +list_ = typing.List() +tuple_ = typing.Tuple() +tensor = typing.TensorType() +function = typing.Function() +symbolic_key = typing.SymbolicKeyType() +env_type = typing.EnvType() +type_type = typing.TypeType() +type_none = typing.TypeNone() +string = typing.String() +type_refkey = typing.RefKeyType() +tensor_type = typing.TensorType + +number_type = (int8, + int16, + int32, + int64, + uint8, + uint16, + uint32, + uint64, + float16, + float32, + float64,) + +int_type = (int8, int16, int32, int64,) +float_type = (float16, float32, float64,) + +_simple_types = { + list: list_, + tuple: tuple_, + type(None): type_none, + bool: bool_, + int: int64, + float: float64, + str: string, + np.bool_: bool_, + np.str: string, + np.int8: int8, + np.int16: int16, + np.int32: int32, + np.int64: int64, + np.uint8: uint8, + np.uint16: uint16, + np.uint32: uint32, + np.uint64: uint64, + np.float16: float16, + np.float32: float32, + np.float64: float64, + EnvInstance_: env_type, +} + + +def pytype_to_dtype(obj): + """ + Convert python type to MindSpore type. + + Args: + obj (type): A python type object. + + Returns: + Type of MindSpore type. + """ + + if isinstance(obj, np.dtype): + obj = obj.type + if isinstance(obj, typing.Type): + return obj + if isinstance(obj, type) and obj in _simple_types: + return _simple_types[obj] + raise NotImplementedError() + + +def get_py_obj_dtype(obj): + """ + Get the corresponding MindSpore data type by python type or variable. + + Args: + obj: An object of python type, or a variable in python type. + + Returns: + Type of MindSpore type. + """ + + if isinstance(obj, (typing.Type, type)): + return pytype_to_dtype(obj) + return pytype_to_dtype(type(obj)) + + +def dtype_to_nptype(type_): + """ + Get numpy data type corresponding to MindSpore dtype. + + Args: + type_ (:class:`mindspore.dtype`): MindSpore's dtype. + + Returns: + The data type of numpy. + """ + + return { + bool_: np.bool_, + int8: np.int8, + int16: np.int16, + int32: np.int32, + int64: np.int64, + uint8: np.uint8, + uint16: np.uint16, + uint32: np.uint32, + uint64: np.uint64, + float16: np.float16, + float32: np.float32, + float64: np.float64, + }[type_] + + +def dtype_to_pytype(type_): + """ + Get python type corresponding to MindSpore dtype. + + Args: + type_ (:class:`mindspore.dtype`): MindSpore's dtype. + + Returns: + Type of python. + """ + + return { + bool_: bool, + int8: int, + int16: int, + int32: int, + int64: int, + uint8: int, + uint16: int, + uint32: int, + uint64: int, + float16: float, + float32: float, + float64: float, + list_: list, + tuple_: tuple, + string: str, + type_none: type(None) + }[type_] + + +def issubclass_(type_, dtype): + """ + Determine whether `type_` is a subclass of `dtype`. + + Args: + type_ (:class:`mindspore.dtype`): Target MindSpore dtype. + dtype (:class:`mindspore.dtype`): Compare MindSpore dtype. + + Returns: + bool, True or False. + """ + if not isinstance(type_, typing.Type): + return False + return typing.is_subclass(type_, dtype) diff --git a/mindspore/common/initializer.py b/mindspore/common/initializer.py new file mode 100644 index 00000000000..99b45013072 --- /dev/null +++ b/mindspore/common/initializer.py @@ -0,0 +1,317 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Initializer for cell parameters.""" +import numbers +import math + +from functools import reduce +import numpy as np +from scipy.stats import truncnorm +from mindspore import log as logger + +from . import dtype as mstype +from .tensor import Tensor + +_INITIALIZER_ALIAS = dict() + + +class Initializer: + """ + The base class of the initializer. + + Args: + kwargs (dict): Keyword arguments for Initializer. + + Returns: + Array, assigned array. + """ + def __init__(self, **kwargs): + self._kwargs = kwargs + + def _initialize(self, *kwargs): + raise NotImplementedError('Must be overridden!') + + def __call__(self, arr): + return self._initialize(arr) + + +def _register(*aliases): + """Return the alias register.""" + def alias_reg(cls): + name = cls.__name__ + name = name.lower() + if name not in _INITIALIZER_ALIAS: + _INITIALIZER_ALIAS[name] = cls + + for alias in aliases: + if alias not in _INITIALIZER_ALIAS: + _INITIALIZER_ALIAS[alias] = cls + + return cls + + return alias_reg + + +def _assignment(arr, num): + """Assign the value of `num` to `arr`.""" + if arr.shape == (): + arr = arr.reshape((1)) + arr[:] = num + arr = arr.reshape(()) + else: + if isinstance(num, np.ndarray): + arr[:] = num[:] + else: + arr[:] = num + return arr + + +@_register('zeros') +class Zero(Initializer): + """ + Initialize the array to zero. + + Args: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + """ + def _initialize(self, arr): + _assignment(arr, 0) + + +@_register('ones') +class One(Initializer): + """ + Initialize the array to one. + + Args: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + """ + def _initialize(self, arr): + _assignment(arr, 1) + + +def _calculate_in_and_out(arr): + """ + Calculate n_in and n_out. + + Args: + arr (Array): Input array. + + Returns: + Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. + """ + dim = len(arr.shape) + if dim < 2: + raise ValueError("If initialize data with xavier uniform, the dimension of data must greater than 1.") + + n_in = arr.shape[1] + n_out = arr.shape[0] + + if dim > 2: + counter = reduce(lambda x, y: x * y, arr.shape[2:]) + n_in *= counter + n_out *= counter + return n_in, n_out + + +@_register('xavier_uniform') +class XavierUniform(Initializer): + r""" + Initialize the array with xavier uniform algorithm, and from a uniform distribution collect samples within + U[-boundary, boundary] where :math:`boundary = gain * \sqrt{\frac{6}{n_{in} + n_{out}}}`. + + Args: + gain (Array): The array to be assigned. Default: 1. + + Returns: + Array, assigned array. + """ + def __init__(self, gain=1): + super(XavierUniform, self).__init__(gain=gain) + self.gain = gain + + def _initialize(self, arr): + n_in, n_out = _calculate_in_and_out(arr) + + boundary = self.gain * math.sqrt(6.0 / (n_in + n_out)) + data = np.random.uniform(-boundary, boundary, arr.shape) + + _assignment(arr, data) + + +@_register('he_uniform') +class HeUniform(Initializer): + r""" + Initialize the array with He kaiming uniform algorithm, and from a uniform distribution collect samples within + U[-boundary, boundary] where :math:`boundary = \sqrt{\frac{6}{n_{in}}}` where :math:`n_{in}` is the number of + input units in the weight tensor. + + Args: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + """ + + def _initialize(self, arr): + n_in, _ = _calculate_in_and_out(arr) + + boundary = math.sqrt(6.0 / n_in) + data = np.random.uniform(-boundary, boundary, arr.shape) + + _assignment(arr, data) + + +class _Constant(Initializer): + """ + Initialize a constant. + + Args: + value (int or numpy.ndarray): The value to initialize. + + Returns: + Array, initialize array. + """ + def __init__(self, value): + super(_Constant, self).__init__(value=value) + self.value = value + + def _initialize(self, arr): + _assignment(arr, self.value) + + +@_register() +class Uniform(Initializer): + """ + Initialize a uniform array, and obtain values U(-scale, scale) from the uniform distribution + to fill the input tensor. + + Args: + scale (float): The scale of the array. Default: 0.07. + + Returns: + Array, uniform array. + """ + def __init__(self, scale=0.07): + super(Uniform, self).__init__(scale=scale) + self.scale = scale + + def _initialize(self, arr): + tmp = np.random.uniform(-self.scale, self.scale, arr.shape) + _assignment(arr, tmp) + + +@_register() +class Normal(Initializer): + """ + Initialize a normal array, and obtain values N(0, sigma) from the uniform distribution + to fill the input tensor. + + Args: + sigma (float): The sigma of the array. Default: 0.01. + + Returns: + Array, normal array. + """ + def __init__(self, sigma=0.01): + super(Normal, self).__init__(sigma=sigma) + self.sigma = sigma + + def _initialize(self, arr): + tmp = np.random.normal(0, self.sigma, arr.shape) + _assignment(arr, tmp) + + +@_register() +class TruncatedNormal(Initializer): + """ + Initialize a truncated normal distribution which is a bounded normal distribution within N(low, high). + + Args: + sigma (float): The sigma of the array. Default: 0.01. + + Returns: + Array, truncated normal array. + """ + def __init__(self, sigma=0.01): + super(TruncatedNormal, self).__init__(sigma=sigma) + self.sigma = sigma + + def _initialize(self, arr): + tmp = truncnorm.rvs(-2, 2, loc=0, scale=self.sigma, size=arr.shape, random_state=None) + _assignment(arr, tmp) + + +def initializer(init, shape=None, dtype=mstype.float32): + """ + Create and initialize a tensor. + + Args: + init (Union[Tensor, str, Initializer, numbers.Number]): Initialize value. + shape (Union[tuple, list, int]): A list of integers, a tuple of integers or an integer as the shape of + output. Default: None. + dtype (:class:`mindspore.dtype`): The type of data in initialized tensor. Default: mstype.float32. + + Returns: + Tensor, initialized tensor. + + Examples: + >>> tensor = initializer('ones', [1, 2, 3], mstype.float32) + """ + if not isinstance(init, (Tensor, numbers.Number, str, Initializer)): + raise TypeError('Unsupported init type.') + + if isinstance(init, Tensor): + init_shape = init.shape() + shape = shape if isinstance(shape, (tuple, list)) else [shape] + if shape is not None and init_shape != tuple(shape): + raise ValueError("The shape of init should be same as variable shape, but got the shape of init {} and " + "the variable shape {}.".format(list(init.shape()), shape)) + return init + + try: + arr = np.ndarray(shape) + except ValueError: + msg = "Error shape={}".format(shape) + logger.error(msg) + raise ValueError(msg) + + if isinstance(init, numbers.Number): + init_obj = _Constant(init) + elif isinstance(init, str): + init_obj = _INITIALIZER_ALIAS[init.lower()]() + else: + init_obj = init + + init_obj(arr) + return Tensor(arr, dtype=dtype) + + +__all__ = [ + 'Initializer', + 'initializer', + 'TruncatedNormal', + 'Normal', + 'Uniform', + 'HeUniform', + 'XavierUniform', + 'One', + 'Zero'] diff --git a/mindspore/common/parameter.py b/mindspore/common/parameter.py new file mode 100644 index 00000000000..b1992c8b825 --- /dev/null +++ b/mindspore/common/parameter.py @@ -0,0 +1,200 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Parameter for cell.""" +from copy import copy +import numpy as np +from .initializer import initializer +from .tensor import Tensor +from .._checkparam import _check_str_by_regular +from ..parallel._utils import _set_clone_info, _CloneInfo + +__all__ = ['Parameter', 'ParameterTuple'] + +PARAMETER_NAME_DEFAULT = "Parameter" +PARAMETER_NAME_PREFIX_MAX_LEN = 1024 + + +def _check_type(x): + """Check input data type""" + if not isinstance(x, Parameter): + raise ValueError("Should be `Parameter` collection.") + return True + + +class Parameter: + """ + Parameter types of cell models. + + Note: + Each parameter of Cell is represented by Parameter class. + + Args: + default_input (Tensor): A parameter tensor. + name (str): Name of the child parameter. + requires_grad (bool): True if the parameter requires gradient. Default: True. + layerwise_parallel (bool): A kind of model parallel mode. When layerwise_parallel is true in paralle mode, + broadcast and gradients communication would not be applied on parameters. Default: False. + """ + def __init__(self, default_input, name, requires_grad=True, layerwise_parallel=False): + self.set_parameter_data(default_input) + self.name = name + self.requires_grad = requires_grad + self.layerwise_parallel = layerwise_parallel + self._is_init = False + self.clone_info = _CloneInfo() + + def __repr__(self): + format_str = 'Parameter (name={name})' + return format_str.format(name=self._name) + + def __parameter__(self): + """For parse check.""" + + @property + def name(self): + """Get the name of the parameter.""" + return self._name + + @name.setter + def name(self, name_): + """ + Define a name for the parameter. + + Args: + name_ (`str` or `None`): The name of the parameter. When the parameter is None or an empty string, + the default value `PARAMETER_NAME_DEFAULT` is used. + """ + if name_ is None: + name_ = PARAMETER_NAME_DEFAULT + elif isinstance(name_, str): + name_ = name_.strip() + if name_ == '': + name_ = PARAMETER_NAME_DEFAULT + if len(name_) > PARAMETER_NAME_PREFIX_MAX_LEN: + raise ValueError("The length of the '{}' name should be less than {}.". + format(name_, PARAMETER_NAME_PREFIX_MAX_LEN)) + else: + raise ValueError("The type of the name should be `str` or `None`.") + self._name = name_ + + @property + def is_init(self): + """Get init status of the parameter.""" + return self._is_init + + @is_init.setter + def is_init(self, is_init_): + """ + Set init status of the parameter. + + Args: + is_init_ (bool): The init status of the parameter. + """ + self._is_init = is_init_ + + def clone(self, prefix, init='same'): + """ + Clone the parameter. + + Args: + prefix (str): Namespace of parameter. + init (Union[Tensor, str, Initializer, numbers.Number]): Initialize the shape of the parameter. + Default: 'same'. + + Returns: + Parameter, a new parameter. + """ + _check_str_by_regular(prefix) + x = copy(self) + x.name = prefix + '.' + x.name + x.is_init = False + if init != 'same': + shape = self.default_input.shape() + dtype = self.default_input.dtype() + x.default_input = initializer(init, shape=shape, dtype=dtype) + + x.clone_info = copy(self.clone_info) + _set_clone_info(self.clone_info, x.clone_info) + return x + + @property + def layerwise_parallel(self): + return self._layerwise_parallel + + @layerwise_parallel.setter + def layerwise_parallel(self, value=True): + if not isinstance(value, bool): + raise TypeError("`layerwise_parallel` parameter must be bool type") + self._layerwise_parallel = value + + @property + def requires_grad(self): + """Return whether the parameter requires gradient.""" + return self._requires_grad + + @requires_grad.setter + def requires_grad(self, value=True): + if not isinstance(value, bool): + raise TypeError("`requires_grad` parameter must be bool type") + self._requires_grad = value + + @property + def data(self): + return self.default_input + + def set_parameter_data(self, data): + if isinstance(data, (Tensor, list, int, float, + np.float16, np.float32, np.int32, np.int16, np.ndarray)) and not isinstance(data, bool): + if isinstance(data, Tensor): + # make a copy of Tensor to init the parameter + data = Tensor(data.asnumpy().copy()) + self.default_input = data + else: + raise ValueError("Parameter data must be tensor or number.") + + +class ParameterTuple(tuple): + """ + Class for storing tuple of parameters. + + Note: + Used to store the parameters of the network into the parameter tuple collection. + """ + def __new__(cls, iterable): + """Create instance object of ParameterTuple.""" + g = (x for x in iterable if _check_type(x)) + return tuple.__new__(ParameterTuple, g) + + def clone(self, prefix, init='same'): + """ + Clone the parameter. + + Args: + prefix (str): Namespace of parameter. + init (str): Initialize the shape of the parameter. Default: 'same'. + + Returns: + Tuple, the new Parameter tuple. + """ + _check_str_by_regular(prefix) + new = [] + for x in self: + x1 = x.clone(prefix, init) + new.append(x1) + return ParameterTuple(new) + + def __parameter_tuple__(self): + """For parse check.""" diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py new file mode 100644 index 00000000000..4c7f31921ba --- /dev/null +++ b/mindspore/common/tensor.py @@ -0,0 +1,117 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tensor implementation.""" +import numpy as np + +from .._c_expression import Tensor as Tensor_ +from .._c_expression import MetaTensor +from .._checkparam import check_type, check_typename +from . import dtype as mstype +from ._register_for_tensor import tensor_operator_registry + +__all__ = ['Tensor', 'MetaTensor'] + + +class Tensor(Tensor_): + """ + Tensor for data storage. + + Tensor inherits tensor object in C++ side, some functions are implemented + in C++ side and some functions are implemented in Python layer. + + Args: + input_data (Tensor, float, int, bool, tuple, list, numpy.ndarray): Input data of the tensor. + dtype (:class:`mindspore.dtype`): Should be None, bool or numeric type defined in `mindspore.dtype`. + The argument is used to define the data type of the output tensor. If it is None, the data type of the + output tensor will be as same as the `input_data`. Default: None. + + Outputs: + Tensor, with the same shape as `input_data`. + + Examples: + >>> # init a tensor with input data + >>> t1 = mindspore.Tensor(np.zeros([1, 2, 3]), mindspore.float32) + >>> assert isinstance(t1, mindspore.Tensor) + >>> assert t1.shape() == (1, 2, 3) + >>> assert t1.dtype() == mindspore.float32 + >>> + >>> # init a tensor with a float scalar + >>> t2 = mindspore.Tensor(0.1) + >>> assert isinstance(t2, mindspore.Tensor) + >>> assert t2.dtype() == mindspore.float64 + """ + + def __init__(self, input_data, dtype=None): + # If input_data is tuple/list/numpy.ndarray, it's support in check_type method. + check_type('tensor input_data', input_data, (Tensor_, float, int)) + if dtype is not None: + check_typename('dtype', dtype, mstype.number_type + (mstype.bool_,)) + if isinstance(input_data, np.ndarray) and (not input_data.flags['FORC']): + input_data = np.ascontiguousarray(input_data) + if dtype is None: + super(Tensor, self).__init__(input_data) + else: + super(Tensor, self).__init__(input_data, dtype) + self._virtual_flag = False + + def __repr__(self): + return str(self.__str__()) + + def __add__(self, other): + if not isinstance(other, Tensor): + raise TypeError("input_data must be a tensor") + out = tensor_operator_registry.get('__add__')(self, other) + return out + + def __mul__(self, other): + if not isinstance(other, Tensor): + raise TypeError("input_data must be a tensor") + out = tensor_operator_registry.get('__mul__')(self, other) + return out + + def __iadd__(self, other): + out = self.__add__(other) + return out + + def __imul__(self, other): + out = self.__mul__(other) + return out + + def __sub__(self, other): + if not isinstance(other, Tensor): + raise TypeError("input_data must be a tensor") + out = self.__add__(Tensor(-other.asnumpy())) + return out + + def __isub__(self, other): + out = self.__sub__(other) + return out + + def __str__(self): + if self.dtype() == mstype.type_none: + return "Unknown Tensor type!" + return str(self.asnumpy()) + + @property + def virtual_flag(self): + """Mark tensor is virtual.""" + return self._virtual_flag + + @virtual_flag.setter + def virtual_flag(self, value): + """The setter of virtual_flag.""" + if not isinstance(value, bool): + raise TypeError("virtual_flag must be bool.") + self._virtual_flag = value diff --git a/mindspore/communication/__init__.py b/mindspore/communication/__init__.py new file mode 100644 index 00000000000..65078f68203 --- /dev/null +++ b/mindspore/communication/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Collective communication interface. +""" + +from .management import GlobalComm, init, release, get_rank, get_group_size, get_world_rank_from_group_rank, \ + get_group_rank_from_world_rank, create_group, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, get_group, \ + get_local_rank, get_local_rank_size, destroy_group + + +__all__ = [ + "GlobalComm", "init", "release", "get_rank", "get_group_size", "get_world_rank_from_group_rank", + "get_group_rank_from_world_rank", "create_group", "HCCL_WORLD_COMM_GROUP", "NCCL_WORLD_COMM_GROUP", "get_group", + "get_local_rank", "get_local_rank_size", "destroy_group" +] diff --git a/mindspore/communication/_comm_helper.py b/mindspore/communication/_comm_helper.py new file mode 100644 index 00000000000..099c8cfc2d7 --- /dev/null +++ b/mindspore/communication/_comm_helper.py @@ -0,0 +1,367 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""comm_helper""" + + +from ._hccl_management import load_lib as hccl_load_lib + +_HCCL_AVAILABLE = False +_NCCL_AVAILABLE = False +try: + import mindspore._ms_mpi as mpi + _NCCL_AVAILABLE = True +except ImportError: + _NCCL_AVAILABLE = False + + +try: + hccl_load_lib() + _HCCL_AVAILABLE = True +except RuntimeError: + _HCCL_AVAILABLE = False + +if _HCCL_AVAILABLE: + from . import _hccl_management as hccl +else: + try: + import hccl_test.manage.api as hccl + _HCCL_AVAILABLE = True + except ImportError: + _HCCL_AVAILABLE = False + + +HCCL_WORLD_COMM_GROUP = "hccl_world_group" +NCCL_WORLD_COMM_GROUP = "nccl_world_group" + + +class Backend: + """ + Class for available backends. + + Note: + The backends' value should be string, e.g., "hccl". + If backend is set to Backend.UNDEFINED, it will be seen as invaliad. + + Args: + name (str): The name of backend. + + Raises: + TypeError: If name is not a string. + ValueError: If backend is invalid. + + Examples: + >>> Backend("abc") + >>> hccl = Backend("hccl") + """ + UNDEFINED = "undefined" + HCCL = "hccl" + NCCL = "nccl" + + def __new__(cls, name): + """Create instance object of Backend.""" + if not isinstance(name, str): + raise TypeError("Backend name must be a string, but got {}".format(type(name))) + value = getattr(Backend, name.upper(), Backend.UNDEFINED) + if value == Backend.UNDEFINED: + raise ValueError("Invalid backend: '{}'".format(name)) + return value + + +def is_hccl_available(): + """ + Check hccl api is available. + + Returns: + Boolean. Return whether hccl is available or not. + """ + return _HCCL_AVAILABLE + + +def is_nccl_available(): + """ + Check nccl api is available. + + Returns: + Boolean. Return whether nccl is available or not. + """ + return _NCCL_AVAILABLE + + +def check_parameter_available(func): + """ + Check parameter is available. If not available, raise Error. + + Args: + func (Function): The function to be run. + + Raises: + RuntimeError. + + Returns: + Wrapper. If not available, raise Error. + """ + def wrapper(*args, **kargs): + group = None + if "group" in kargs.keys(): + group = kargs.get("group") + if group is not None and not isinstance(group, str): + raise TypeError("Group should be str or None, " + "but got group {}".format(type(group))) + + if "backend" in kargs.keys(): + backend = kargs.get("backend") + if backend is Backend.HCCL and not is_hccl_available(): + raise RuntimeError("Distributed Communication doesn't have HCCL built in") + if backend is Backend.NCCL and not is_nccl_available(): + raise RuntimeError("Distributed Communication doesn't have NCCL built in") + + if group is None: + if backend is Backend.HCCL: + group = HCCL_WORLD_COMM_GROUP + elif backend is Backend.NCCL: + group = NCCL_WORLD_COMM_GROUP + return func(*args, **kargs) + return wrapper + + +@check_parameter_available +def _get_rank_helper(group, backend): + """ + The Helper to do get_rank_id. + + Args: + group (str): The communication group. + backend (str): The backend, like "hccl". + + Raises: + ValueError: If backend is invalid. + + Returns: + Integer. The local rank id of the calling process. + """ + rank_id = None + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + rank_id = hccl.get_rank_id() + else: + rank_id = hccl.get_rank_id(group) + elif backend == Backend.NCCL: + if group == NCCL_WORLD_COMM_GROUP: + rank_id = mpi.get_rank_id() + else: + raise RuntimeError("Nccl doesn't support get_rank_id by user group now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + return rank_id + + +@check_parameter_available +def _get_local_rank_helper(group, backend): + """ + The Helper to do get_local_rank_id. + + Args: + group (str): The communication group. + backend (str): The backend, like "hccl". + + Raises: + ValueError: If backend is invalid. + + Returns: + Integer. The local rank id of the calling process. + """ + rank_id = None + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + rank_id = hccl.get_local_rank_id() + else: + rank_id = hccl.get_local_rank_id(group) + elif backend == Backend.NCCL: + raise RuntimeError("Nccl doesn't support get_local_rank_id now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + return rank_id + + +@check_parameter_available +def _get_size_helper(group, backend): + """ + The Helper to do get_rank_size. + + Args: + group (str): The communication group. + backend (str): The backend, like "hccl". + + Raises: + ValueError: If backend is invalid. + + Returns: + Integer. The rank size of specified group. + """ + size = None + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + size = hccl.get_rank_size() + else: + size = hccl.get_rank_size(group) + elif backend == Backend.NCCL: + if group == NCCL_WORLD_COMM_GROUP: + size = mpi.get_rank_size() + else: + raise RuntimeError("Nccl doesn't support get_rank_size by user group now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + return size + + +@check_parameter_available +def _get_local_size_helper(group, backend): + """ + The Helper to do get_local_rank_size. + + Args: + group (str): The communication group. + backend (str): The backend, like "hccl". + + Raises: + ValueError: If backend is invalid. + + Returns: + Integer. The local rank size where the calling process is being within specified group. + """ + size = None + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + size = hccl.get_local_rank_size() + else: + size = hccl.get_local_rank_size(group) + elif backend == Backend.NCCL: + raise RuntimeError("Nccl doesn't support get_local_rank_size now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + return size + + +@check_parameter_available +def _get_world_rank_from_group_rank_helper(group, group_rank_id, backend): + """ + The Helper to do get_world_rank_from_group_rank. + + Args: + group (str): The user communication group. + group_rank_id (int): A rank id in user communication group. + backend (str): The backend, like "hccl". + + Raises: + TypeError: If group_rank_id is not int. + ValueError: If group is "hccl_world_group" or backend is invalid. + + Returns: + Integer. A rank id in world communication group. + """ + world_rank_id = None + if not isinstance(group_rank_id, int): + raise TypeError("group_rank_id should be int, but got type {}".format(type(group_rank_id))) + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + raise ValueError("Group cannot be 'hccl_world_group'. ") + world_rank_id = hccl.get_world_rank_from_group_rank(group, group_rank_id) + elif backend == Backend.NCCL: + raise RuntimeError("Nccl doesn't support get_world_rank_from_group_rank now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + return world_rank_id + + +@check_parameter_available +def _get_group_rank_from_world_rank_helper(world_rank_id, group, backend): + """ + The Helper to do get_group_rank_from_world_rank. + + Args: + world_rank_id (int): A rank id in world communication group. + group (str): The user communication group. + backend (str): The backend, like "hccl". + + Raises: + TypeError: If world_rank_id is not int. + ValueError: If group is 'hccl_world_group' or backend is invalid. + + Returns: + Integer. A rank id in user communication group. + """ + group_rank_id = None + if not isinstance(world_rank_id, int): + raise TypeError("world_rank_id should be int, but got type {}".format(type(world_rank_id))) + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + raise ValueError("Group cannot be 'hccl_world_group'. ") + group_rank_id = hccl.get_group_rank_from_world_rank(world_rank_id, group) + elif backend == Backend.NCCL: + raise RuntimeError("Nccl doesn't support get_group_rank_from_world_rank now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + return group_rank_id + + +@check_parameter_available +def _create_group_helper(group, rank_ids, backend): + """ + The Helper to do create_group. + + Args: + group (str): The communication group. + rank_ids (list): Rank ids in the group. + backend (str): The backend, like "hccl". + + Raises: + TypeError: If rank_ids is not a list. + ValueError: If rank_ids size is not larger than 1 or rank_ids has duplicate data or backend is invalid. + """ + if backend == Backend.HCCL: + if not isinstance(rank_ids, list): + raise TypeError("Rank_ids {} should be list".format(rank_ids)) + rank_size = len(rank_ids) + if rank_size < 2: + raise ValueError("Rank_ids size {} should be large than 1".format(rank_size)) + if len(rank_ids) - len(list(set(rank_ids))) > 0: + raise ValueError("List rank_ids in Group {} has duplicate data!".format(group)) + hccl.create_group(group, rank_size, rank_ids) + elif backend == Backend.NCCL: + raise RuntimeError("Nccl doesn't support create_group now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) + + +@check_parameter_available +def _destroy_group_helper(group, backend): + """ + The Helper to do destroy_group. + + Args: + group (str): The user communication group. + backend (str): The backend, like "hccl". + + Raises: + ValueError: If group is "hccl_world_group" or backend is invalid. + """ + if backend == Backend.HCCL: + if group == HCCL_WORLD_COMM_GROUP: + raise ValueError("The hccl_world_group does not support destruction.") + hccl.destroy_group(group) + elif backend == Backend.NCCL: + raise RuntimeError("Nccl doesn't support destroy_group now.") + else: + raise ValueError("Invalid backend: '{}'".format(backend)) diff --git a/mindspore/communication/_hccl_management.py b/mindspore/communication/_hccl_management.py new file mode 100644 index 00000000000..3c834116826 --- /dev/null +++ b/mindspore/communication/_hccl_management.py @@ -0,0 +1,254 @@ +# Copyright 2020 Huawei Technologies Co., Ltd + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""HCCL management API""" +import ctypes + +MAX_GROUP_NAME_LEN = 127 +MAX_RANK_NUM = 4096 +HCCL_LIB = 'libhccl.so' +HCCL_LIB_CTYPES = "" + + +def check_group(group): + """ + A function that check if a collection communication group is leagal. + + Returns: + None + """ + if isinstance(group, (str)): + group_len = len(group) + if group_len > MAX_GROUP_NAME_LEN or group_len == 0: + raise ValueError('Group name is invalid.') + else: + raise TypeError('Group must be a python str.') + + +def check_rank_num(rank_num): + """ + A function that check if a collection communication rank number is leagal.If not raise error. + + Returns: + None + """ + if isinstance(rank_num, (int)): + if rank_num > MAX_RANK_NUM or rank_num <= 0: + raise ValueError('Rank number is out of range.') + else: + raise TypeError('Rank number must be a python int.') + + +def check_rank_id(rank_id): + """ + A function that check if a collection communication rank id is leagal.If not raise error. + + Returns: + None + """ + if isinstance(rank_id, (int)): + if rank_id >= MAX_RANK_NUM or rank_id < 0: + raise ValueError('Rank id is out of range.') + else: + raise TypeError('Rank id must be a python int.') + + +def load_lib(): + try: + hccl_lib = ctypes.CDLL(HCCL_LIB) + except Exception: + raise RuntimeError('Get hccl lib error.') + global HCCL_LIB_CTYPES + HCCL_LIB_CTYPES = hccl_lib + + +def c_str(string): + """Convert a python string to C string.""" + if not isinstance(string, str): + string = string.decode('ascii') + return ctypes.c_char_p(string.encode('utf-8')) + + +def c_array(ctype, values): + """Create ctypes array from a python array.""" + return (ctype * len(values))(*values) + + +def create_group(group, rank_num, rank_ids): + """ + Create group. + + A function that creates a collection communication group which includes 'rank_num' + device and 'rank_ids' is the list of these ranks of devices. + + Note: + The world group can not be created. + + Returns: + None + """ + check_group(group) + check_rank_num(rank_num) + if isinstance(rank_ids, (list)): + if rank_num != len(rank_ids): + raise ValueError('Rank number is not equal to the length of rank_ids.') + for rank_id in rank_ids: + if not isinstance(rank_id, (int)) or rank_id < 0: + raise ValueError('Rank id must be unsigned integer!') + c_array_rank_ids = c_array(ctypes.c_uint, rank_ids) + c_rank_num = ctypes.c_uint(rank_num) + c_group = c_str(group) + ret = HCCL_LIB_CTYPES.hcom_create_group(c_group, c_rank_num, c_array_rank_ids) + if ret != 0: + raise RuntimeError('Create group error.') + else: + raise TypeError('Rank ids must be a python list.') + + +def destroy_group(group): + """ + A function that destroy the group which created by user. + + Note: + The world group can not be destroy. + + Returns: + None + """ + check_group(group) + c_group = c_str(group) + ret = HCCL_LIB_CTYPES.hcom_destroy_group(c_group) + if ret != 0: + raise RuntimeError('Destroy group error.') + + +def get_rank_size(group="hccl_world_group"): + """ + A function that returns the number of ranks within the given collection communication group. + + Note: + The default group is hccl_world_group. + + Returns: + An integer scalar with the num of ranks. + """ + check_group(group) + c_group = c_str(group) + c_rank_size = ctypes.c_uint() + ret = HCCL_LIB_CTYPES.hcom_get_rank_size(c_group, ctypes.byref(c_rank_size)) + if ret != 0: + raise RuntimeError('Get rank size error.') + + return c_rank_size.value + + +def get_rank_id(group="hccl_world_group"): + """ + A function that returns the rank id of the calling process, within the given collection communication group. + + Returns: + An integer scalar with the rank id of the calling process. + """ + check_group(group) + c_group = c_str(group) + c_rank_id = ctypes.c_uint() + ret = HCCL_LIB_CTYPES.hcom_get_rank_id(c_group, ctypes.byref(c_rank_id)) + if ret != 0: + raise RuntimeError('Get rank id error.') + + return c_rank_id.value + + +def get_local_rank_size(group="hccl_world_group"): + """ + A function that returns the number of local ranks within the given collection communication group. + + Note: + The default group is hccl_world_group. + + Returns: + An integer scalar with the num of local ranks. + """ + check_group(group) + c_group = c_str(group) + c_local_rank_size = ctypes.c_uint() + ret = HCCL_LIB_CTYPES.hcom_get_local_rank_size(c_group, ctypes.byref(c_local_rank_size)) + if ret != 0: + raise RuntimeError('Get local rank size error.') + + return c_local_rank_size.value + + +def get_local_rank_id(group="hccl_world_group"): + """ + Get local rank id. + + A function that returns the local rank id of the calling process, within the given collection communication group. + + Returns: + An integer scalar with the local rank id of the calling process. + """ + check_group(group) + c_group = c_str(group) + c_local_rank_id = ctypes.c_uint() + ret = HCCL_LIB_CTYPES.hcom_get_local_rank_id(c_group, ctypes.byref(c_local_rank_id)) + if ret != 0: + raise RuntimeError('Get local rank id error.') + + return c_local_rank_id.value + + +def get_world_rank_from_group_rank(group, group_rank_id): + """ + Get world rank from group rank. + + A function that returns the rank id in the world group corresponding to the + rank which id is 'group_rank_id' in the user group. + + Returns: + An integer scalar with the rank id in the world group. + """ + check_group(group) + check_rank_id(group_rank_id) + c_group = c_str(group) + c_group_rank_id = ctypes.c_uint(group_rank_id) + c_world_rank_id = ctypes.c_uint() + ret = HCCL_LIB_CTYPES.hcom_get_world_rank_from_group_rank(c_group, c_group_rank_id, ctypes.byref(c_world_rank_id)) + if ret != 0: + raise RuntimeError('Get world rank from group rank error.') + + return c_world_rank_id.value + + +def get_group_rank_from_world_rank(world_rank_id, group): + """ + Get group rank from world rank. + + A function that returns the rank id in the user group corresponding to the + rank which id is 'world_rank_id' in the world group. + + Returns: + An integer scalar with the rank id in the user group. + """ + check_group(group) + check_rank_id(world_rank_id) + c_group = c_str(group) + c_world_rank_id = ctypes.c_uint(world_rank_id) + c_group_rank_id = ctypes.c_uint() + ret = HCCL_LIB_CTYPES.hcom_get_group_rank_from_world_rank(c_world_rank_id, c_group, ctypes.byref(c_group_rank_id)) + if ret != 0: + raise RuntimeError('Get group rank from world rank error.') + + return c_group_rank_id.value diff --git a/mindspore/communication/management.py b/mindspore/communication/management.py new file mode 100755 index 00000000000..7208538a074 --- /dev/null +++ b/mindspore/communication/management.py @@ -0,0 +1,253 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Communication management API""" +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from ._comm_helper import Backend, _get_rank_helper, _get_size_helper, \ + _get_world_rank_from_group_rank_helper, _get_group_rank_from_world_rank_helper, \ + _create_group_helper, _destroy_group_helper, HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, \ + _get_local_rank_helper, _get_local_size_helper +from .._c_expression import init_hccl, finalize_hccl, init_gpu_collective + + +__all__ = ["init", "release", "get_rank", "get_local_rank", "get_group_size", "get_group", + "get_local_rank_size", "get_world_rank_from_group_rank", + "get_group_rank_from_world_rank", "create_group", "destroy_group", + "HCCL_WORLD_COMM_GROUP", "NCCL_WORLD_COMM_GROUP"] + +DEFAULT_WORLD_COMM_GROUP = HCCL_WORLD_COMM_GROUP +DEFAULT_BACKEND = Backend("hccl") + + +def get_group(group): + """Get the global world group if the group is default world comm group.""" + if group == DEFAULT_WORLD_COMM_GROUP: + return GlobalComm.WORLD_COMM_GROUP + return group + + +class GlobalComm: + """Global communication info.""" + BACKEND = DEFAULT_BACKEND + WORLD_COMM_GROUP = DEFAULT_WORLD_COMM_GROUP + + +def init(backend_name="hccl"): + """ + Init distributed backend, e.g., hccl/nccl, it is required before communication service can be used. + + Note: + The full name of hccl is Huawei Collective Communication Library. + The full name of nccl is NVIDIA Collective Communication Library. + + Args: + backend_name (str): Backend. + + Raises: + TypeError: If backend name is not a string. + RuntimeError: If backend is invalid or distributed init fails. + """ + if not isinstance(backend_name, str): + raise TypeError("Backend name must be a string, but got {}".format(type(backend_name))) + + if backend_name == "hccl": + init_hccl() + GlobalComm.BACKEND = Backend("hccl") + GlobalComm.WORLD_COMM_GROUP = HCCL_WORLD_COMM_GROUP + elif backend_name == "nccl": + init_gpu_collective() + GlobalComm.BACKEND = Backend("nccl") + GlobalComm.WORLD_COMM_GROUP = NCCL_WORLD_COMM_GROUP + else: + raise RuntimeError("Backend name {} is not supported.".format(backend_name)) + + auto_parallel_context().set_communication_backend(backend_name) + + +def release(): + """ + Release distributed resource. e.g., hccl/nccl. + + Raises: + RuntimeError: If distributed resource release fails. + """ + finalize_hccl() + + +def get_rank(group=GlobalComm.WORLD_COMM_GROUP): + """ + Gets rank ID for current device in specified collective communication group. + + Args: + group (str): ProcessGroup, the process group to work on. Default: WORLD_COMM_GROUP. + + Returns: + int, the rank ID of the calling process within the group. + + Raises: + TypeError: If group is not a string. + ValueError: If backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + return _get_rank_helper(group=get_group(group), backend=GlobalComm.BACKEND) + + +def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP): + """ + Gets local rank ID for current device in specified collective communication group. + + Note: + Nccl is not supported. + + Args: + group (str): ProcessGroup, the process group to work on. Default: WORLD_COMM_GROUP. + + Returns: + int, the local rank ID of the calling process within the group. + + Raises: + TypeError: If group is not a string. + ValueError: If backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + return _get_local_rank_helper(group=get_group(group), backend=GlobalComm.BACKEND) + + +def get_group_size(group=GlobalComm.WORLD_COMM_GROUP): + """ + Gets rank size of the specified collective communication group. + + Args: + group (str): ProcessGroup, the process group to work on. + + Returns: + int, the rank size of the group. + + Raises: + TypeError: If group is not a string. + ValueError: If backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + return _get_size_helper(group=get_group(group), backend=GlobalComm.BACKEND) + + +def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP): + """ + Gets local rank size of the specified collective communication group. + + Note: + Nccl is not supported. + + Args: + group (str): ProcessGroup, the process group to work on. + + Returns: + int, the local rank size where the calling process is being within the group. + + Raises: + TypeError: If group is not a string. + ValueError: If backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + return _get_local_size_helper(group=get_group(group), backend=GlobalComm.BACKEND) + + +def get_world_rank_from_group_rank(group, group_rank_id): + """ + Gets the rank ID in world communication group corresponding to the rank ID in specified user communication group. + + Note: + Nccl is not supported. + The parameter group should not be "hccl_world_group". + + Args: + group (str): The user communication group. + group_rank_id (int): A rank ID in user communication group. + + Returns: + int, the rank ID in world communication group. + + Raises: + TypeError: If group_rank_id is not a int or group is not a string. + ValueError: If group is 'hccl_world_group' or backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + return _get_world_rank_from_group_rank_helper(group=group, group_rank_id=group_rank_id, backend=GlobalComm.BACKEND) + + +def get_group_rank_from_world_rank(world_rank_id, group): + """ + Gets the rank ID in specified user communication group corresponding to the rank ID in world communication group. + + Note: + Nccl is not supported. + The parameter group should not be "hccl_world_group". + + Args: + world_rank_id (int): A rank ID in world communication group. + group (str): The user communication group. + + Returns: + int, the rank ID in user communication group. + + Raises: + TypeError: If world_rank_id is not a int or group is not a string. + ValueError: If group is 'hccl_world_group' or backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + return _get_group_rank_from_world_rank_helper(world_rank_id=world_rank_id, group=group, backend=GlobalComm.BACKEND) + + +def create_group(group, rank_ids): + """ + Creates user collective communication group. + + Note: + Nccl is not supported. + The size of rank_ids should be larger than 1. + Rank_ids should not have duplicate data. + + Args: + group (str): ProcessGroup, the process group to create. + rank_ids (list): List of device ID. + + Raises: + TypeError: If group is not a string or rank_ids is not a list. + ValueError: If rank_ids size is not larger than 1 or rank_ids has duplicate data or backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + Examples: + >>> group = "0-1" + >>> rank_ids = [0,1] + >>> create_group(group, rank_ids) + """ + _create_group_helper(group, rank_ids, backend=GlobalComm.BACKEND) + + +def destroy_group(group): + """ + Destroys user collective communication group. + + Note: + Nccl is not supported. + The parameter group should not be "hccl_world_group". + + Args: + group (str): ProcessGroup, the process group to destroy. + + Raises: + TypeError: If group is not a string. + ValueError: If group is "hccl_world_group" or backend is invalid. + RuntimeError: If hccl/nccl is not available or nccl not supports. + """ + _destroy_group_helper(group, backend=GlobalComm.BACKEND) diff --git a/mindspore/context.py b/mindspore/context.py new file mode 100644 index 00000000000..89365f3d1ca --- /dev/null +++ b/mindspore/context.py @@ -0,0 +1,549 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +The context of mindspore, used to configure the current execution environment, +including execution mode, execution backend and other feature switchs. +""" +import threading +import logging +from collections import namedtuple +from types import FunctionType +from mindspore._c_expression import MSContext +from mindspore._extends.pynative_helper import args_type_check +from mindspore.parallel._auto_parallel_context import _set_auto_parallel_context, _get_auto_parallel_context, \ + _reset_auto_parallel_context + +logger = logging.getLogger('Context') + + +__all__ = ['GRAPH_MODE', 'PYNATIVE_MODE', 'set_context', 'get_context', 'set_auto_parallel_context', + 'get_auto_parallel_context', 'reset_auto_parallel_context'] + +GRAPH_MODE = 0 +PYNATIVE_MODE = 1 + + +class _ThreadLocalInfo(threading.local): + """ + Thread local Info used for store thread local attributes. + """ + def __init__(self): + super(_ThreadLocalInfo, self).__init__() + self._reserve_class_name_in_scope = True + + @property + def reserve_class_name_in_scope(self): + """Gets whether to save the network class name in the scope.""" + return self._reserve_class_name_in_scope + + @reserve_class_name_in_scope.setter + def reserve_class_name_in_scope(self, reserve_class_name_in_scope): + """Sets whether to save the network class name in the scope.""" + if not isinstance(reserve_class_name_in_scope, bool): + raise ValueError("Set reserve_class_name_in_scope value must be bool!") + self._reserve_class_name_in_scope = reserve_class_name_in_scope + + +_ContextRecord = namedtuple("_ContextRecord", ["is_pynative_mode", "switch_context_fn"]) + + +class _ContextSwitchInfo(threading.local): + """ + Record of context switch information. + + Args: + is_pynative (bool): Whether to adopt the PyNative mode. + """ + def __init__(self, is_pynative): + super(_ContextSwitchInfo, self).__init__() + self.context_stack = [] + if is_pynative: + self.push(True, None) + + def push(self, is_pynative, switch_context_fn): + """ + Push a context switch record onto the stack. + + Args: + is_pynative (bool): Whether context switch to PyNative mode. + switch_context_fn (Function): A callable that executes the context switch. + """ + if isinstance(switch_context_fn, FunctionType): + switch_context_fn() + self.context_stack.append(_ContextRecord(is_pynative, switch_context_fn)) + + def pop(self): + self.context_stack.pop() + + +class _Context: + """ + _Context is the environment in which operations are executed + + Note: + Create a context through instantiating Context object is not recommended. + should use context() to get the context since Context is singleton. + """ + _instance = None + _instance_lock = threading.Lock() + + def __init__(self): + self._thread_local_info = _ThreadLocalInfo() + self._context_switches = _ContextSwitchInfo(True) + self._context_handle = MSContext.get_instance() + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance_lock.acquire() + cls._instance = object.__new__(cls) + cls._instance_lock.release() + return cls._instance + + def __getattribute__(self, attr): + value = object.__getattribute__(self, attr) + if attr == "_context_handle" and value is None: + raise ValueError("Context handle is none in context!!!") + return value + + # For Ascend task sink mode execution + @property + def enable_task_sink(self): + return self._context_handle.get_task_sink_flag() + + @enable_task_sink.setter + def enable_task_sink(self, task_sink): + self._context_handle.set_task_sink_flag(task_sink) + + @property + def mode(self): + return self._context_handle.get_execution_mode() + + @mode.setter + def mode(self, mode): + """ + Switch between Graph mode and PyNative mode. + + Args: + mode (int): GRAPH_MODE or PYNATIVE_MODE. + """ + self._context_handle.set_execution_mode(mode) + if mode == PYNATIVE_MODE: + if self.enable_debug_runtime: + self.set_backend_policy("vm") + self._context_switches.push(True, None) + else: + if self.enable_debug_runtime: + self.set_backend_policy("ge") + self._context_switches.push(False, None) + + def set_backend_policy(self, policy): + success = self._context_handle.set_backend_policy(policy) + if not success: + raise RuntimeError("Backend policy must be one of ge, vm, ms.") + + @property + def precompile_only(self): + return self._context_handle.get_precompile_only() + + @precompile_only.setter + def precompile_only(self, precompile_only): + self._context_handle.set_precompile_only(precompile_only) + + @property + def save_graphs(self): + return self._context_handle.get_save_graphs_flag() + + @save_graphs.setter + def save_graphs(self, save_graphs_flag): + self._context_handle.set_save_graphs_flag(save_graphs_flag) + + @property + def save_graphs_path(self): + return self._context_handle.get_save_graphs_path() + + @save_graphs_path.setter + def save_graphs_path(self, save_graphs_path): + self._context_handle.set_save_graphs_path(save_graphs_path) + + @property + def device_target(self): + return self._context_handle.get_device_target() + + @device_target.setter + def device_target(self, target): + success = self._context_handle.set_device_target(target) + if not success: + raise ValueError("target device name is invalid!!!") + + @property + def device_id(self): + return self._context_handle.get_device_id() + + @device_id.setter + def device_id(self, device_id): + if device_id < 0 or device_id > 4095: + raise ValueError("Device id must be in [0, 4095], but got {}".format(device_id)) + success = self._context_handle.set_device_id(device_id) + if not success: + raise RuntimeError("Device id set failed!!!") + + @property + def enable_hccl(self): + return self._context_handle.get_hccl_flag() + + @enable_hccl.setter + def enable_hccl(self, hccl): + self._context_handle.set_hccl_flag(hccl) + + @property + def enable_ir_fusion(self): + return self._context_handle.get_ir_fusion_flag() + + @enable_ir_fusion.setter + def enable_ir_fusion(self, enable_ir_fusion): + self._context_handle.set_ir_fusion_flag(enable_ir_fusion) + + @property + def enable_loop_sink(self): + return self._context_handle.get_loop_sink_flag() + + @enable_loop_sink.setter + def enable_loop_sink(self, enable_loop_sink): + self._context_handle.set_loop_sink_flag(enable_loop_sink) + + @property + def enable_mem_reuse(self): + return self._context_handle.get_enable_mem_reuse() + + @enable_mem_reuse.setter + def enable_mem_reuse(self, enable_mem_reuse): + self._context_handle.set_enable_mem_reuse(enable_mem_reuse) + + @property + def save_ms_model(self): + return self._context_handle.get_save_ms_model_flag() + + @save_ms_model.setter + def save_ms_model(self, save_ms_model_flag): + self._context_handle.set_save_ms_model_flag(save_ms_model_flag) + + @property + def save_ms_model_path(self): + return self._context_handle.get_save_ms_model_path() + + @save_ms_model_path.setter + def save_ms_model_path(self, save_ms_model_path): + self._context_handle.set_save_ms_model_path(save_ms_model_path) + + @property + def enable_gpu_summary(self): + return self._context_handle.get_enable_gpu_summary() + + @enable_gpu_summary.setter + def enable_gpu_summary(self, enable_gpu_summary): + self._context_handle.set_enable_gpu_summary(enable_gpu_summary) + + @property + def enable_auto_mixed_precision(self): + return self._context_handle.get_auto_mixed_precision_flag() + + @enable_auto_mixed_precision.setter + def enable_auto_mixed_precision(self, enable_auto_mixed_precision): + self._context_handle.set_auto_mixed_precision_flag(enable_auto_mixed_precision) + + @property + def enable_reduce_precision(self): + return self._context_handle.get_enable_reduce_precision_flag() + + @enable_reduce_precision.setter + def enable_reduce_precision(self, enable_reduce_precision): + self._context_handle.set_enable_reduce_precision_flag(enable_reduce_precision) + + @property + def enable_dump(self): + return self._context_handle.get_enable_dump() + + @enable_dump.setter + def enable_dump(self, enable_dump): + self._context_handle.set_enable_dump(enable_dump) + + @property + def save_dump_path(self): + return self._context_handle.get_save_dump_path() + + @save_dump_path.setter + def save_dump_path(self, save_dump_path): + self._context_handle.set_save_dump_path(save_dump_path) + + @property + def reserve_class_name_in_scope(self): + """Gets whether to save the network class name in the scope.""" + return self._thread_local_info.reserve_class_name_in_scope + + @reserve_class_name_in_scope.setter + def reserve_class_name_in_scope(self, reserve_class_name_in_scope): + """Sets whether to save the network class name in the scope.""" + self._thread_local_info.reserve_class_name_in_scope = reserve_class_name_in_scope + + @property + def enable_dynamic_memory(self): + return self._context_handle.get_enable_dynamic_mem_pool() + + @enable_dynamic_memory.setter + def enable_dynamic_memory(self, enable_dynamic_memory): + self._context_handle.set_enable_dynamic_mem_pool(enable_dynamic_memory) + + @property + def graph_memory_max_size(self): + return None + + @graph_memory_max_size.setter + def graph_memory_max_size(self, graph_memory_max_size): + if check_input_fotmat(graph_memory_max_size): + graph_memory_max_size_ = graph_memory_max_size[:-2] + " * 1024 * 1024 * 1024" + self._context_handle.set_graph_memory_max_size(graph_memory_max_size_) + else: + raise ValueError("Context param graph_memory_max_size should be in correct format! Such as \"26GB\"") + + @property + def variable_memory_max_size(self): + return None + + @variable_memory_max_size.setter + def variable_memory_max_size(self, variable_memory_max_size): + if check_input_fotmat(variable_memory_max_size): + variable_memory_max_size_ = variable_memory_max_size[:-2] + " * 1024 * 1024 * 1024" + self._context_handle.set_variable_memory_max_size(variable_memory_max_size_) + else: + raise ValueError("Context param variable_memory_max_size should be in correct format! Such as \"5GB\"") + + @property + def enable_ge(self): + return self._context_handle.get_backend_policy() == 'ge' + + @property + def enable_debug_runtime(self): + return self._thread_local_info.debug_runtime + + @enable_debug_runtime.setter + def enable_debug_runtime(self, enable): + thread_info = self._thread_local_info + thread_info.debug_runtime = enable + + +def check_input_fotmat(x): + import re + pattern = r'[1-9][0-9]*(\.)?[0-9]*GB|0\.[0-9]*GB' + result = re.match(pattern, x) + return result is not None + +_k_context = None + + +def _context(): + """ + Get the global _context, if context is not created, create a new one. + + Returns: + _Context, the global context in PyNative mode. + """ + global _k_context + if _k_context is None: + default_backend = 'debug' + try: + from mindspore import default_config + default_backend = default_config.__backend__ + except ImportError: + logger.error("import default config fail") + _k_context = _Context() + _k_context.enable_debug_runtime = False + if default_backend == 'debug': + _k_context.enable_debug_runtime = True + default_backend = 'vm' + _k_context.set_backend_policy(default_backend) + return _k_context + + +@args_type_check(device_num=int, global_rank=int, mirror_mean=bool, cast_before_mirror=bool, parallel_mode=str, + parameter_broadcast=bool) +def set_auto_parallel_context(**kwargs): + """ + Set auto parallel context. + + Note: + Attribute name is required for setting attributes. + + Args: + device_num (int): Available device number, the value must be in [1, 4096]. Default: 1. + global_rank (int): Global rank id, the value must be in [0, 4095]. Default: 0. + mirror_mean (bool): Whether to perform mean operator after all-reduce of mirror. Default: False. + cast_before_mirror (bool): Insert Mirror Op after the cast if this flag is True. Default: True. + parallel_mode (str): There are five kinds of parallel modes, "stand_alone", "data_parallel", + "hybrid_parallel", "semi_auto_parallel" and "auto_parallel". Default: "stand_alone". + + - stand_alone: Only one processor working. + + - data_parallel: Distributing the data across different processors. + + - hybrid_parallel: Achieving data parallelism and model parallelism manually. + + - semi_auto_parallel: Achieving data parallelism and model parallelism by + setting parallel strategies. + + - auto_parallel: Achieving parallelism automatically. + parameter_broadcast (bool): Indicating whether to broadcast parameters before training. + "stand_alone", "semi_auto_parallel" and "auto_parallel" do not support parameter + broadcast. Default: False. + + Raises: + ValueError: If input key is not attribute in auto parallel context. + + Examples: + >>> context.set_auto_parallel_context(device_num=8) + >>> context.set_auto_parallel_context(global_rank=0) + >>> context.set_auto_parallel_context(mirror_mean=True) + >>> context.set_auto_parallel_context(cast_before_mirror=False) + >>> context.set_auto_parallel_context(parallel_mode="auto_parallel") + >>> context.set_auto_parallel_context(parameter_broadcast=False) + """ + _set_auto_parallel_context(**kwargs) + + +def get_auto_parallel_context(attr_key): + """ + Gets auto parallel context attribute value according to the key. + + Args: + attr_key (str): The key of the attribute. + + Returns: + Returns attribute value according to the key. + + Raises: + ValueError: If input key is not attribute in auto parallel context. + """ + return _get_auto_parallel_context(attr_key) + + +def reset_auto_parallel_context(): + """ + Reset auto parallel context attributes to the default values: + + - device_num: 1. + - global_rank: 0. + - mirror_mean: False. + - cast_before_mirror: True. + - parallel_mode: "stand_alone". + - parameter_broadcast: False. + """ + _reset_auto_parallel_context() + + +@args_type_check(mode=int, precompile_only=bool, device_target=str, + device_id=int, enable_ir_fusion=bool, save_graphs=bool, enable_hccl=bool, + enable_task_sink=bool, save_graphs_path=str, enable_loop_sink=bool, + enable_mem_reuse=bool, save_ms_model=bool, save_ms_model_path=str, enable_gpu_summary=bool, + enable_auto_mixed_precision=bool, enable_dump=bool, save_dump_path=str, + enable_reduce_precision=bool, enable_dynamic_memory=bool, graph_memory_max_size=str, + variable_memory_max_size=str) +def set_context(**kwargs): + """ + Set context for running environment. + + Context should be configured before running your program. If there is no configuration, + the "Ascend" device target will be used by default. GRAPH_MODE or + PYNATIVE_MODE can be set by `mode` attribute and both modes support all backends, default + mode is PYNATIVE_MODE. + + When the `save_graphs` attribute is set to True, attribute of `save_graphs_path` is used to set the + intermediate compilation graph storage path. By default, the graphs are saved in the current directory. + As for other configurations and arguments, please refer to the corresponding module + description, the configuration is optional and can be enabled when needed. + + Note: + Attribute name is required for setting attributes. + If need to config graph max memory size and variable max memory size, one must make sure: + The sum of graph_memory_max_size and variable_memory_max_size should be less than total memory size of + a device, while the total memory is supposed to be no more than 256GB. + + Args: + mode (int): Running in GRAPH_MODE(0) or PYNATIVE_MODE(1). Default: PYNATIVE_MODE. + device_target (str): The target device to run, support "Ascend", "GPU", "CPU". Default: "Ascend". + device_id (int): Id of target device, the value must be in [0, device_num_per_host-1], + while device_num_per_host should no more than 4096. Default: 0. + enable_ir_fusion (bool): Whether to enable ir fusion. Default: True. + save_graphs (bool): Whether to save graphs. Default: False. + enable_hccl (bool): Whether to enable hccl. Default: False. + enable_loop_sink (bool): Whether to enable loop sink. Default: False. + enable_task_sink (bool): Whether to enable task sink. Default: True. + enable_mem_reuse (bool): Whether to enable memory reuse. Default: True. + save_ms_model (bool): Whether to save model converted by graph. Default: False. + save_ms_model_path (str): Path to save converted model. Default: "." + enable_gpu_summary (bool): Whether to enable gpu summary. Default: True. + save_graphs_path (str): Path to save graphs. Default: "." + enable_auto_mixed_precision (bool): Whether to enable auto mixed precision. Default: True. + reserve_class_name_in_scope (bool) : Whether to save the network class name in the scope. Default: True. + enable_reduce_precision (bool): Whether to enable precision reduction. Default: True. + enable_dump (bool): Whether to enable dump. Default: False. + save_dump_path (str): Set path to dump data. Default: ".". + enable_dynamic_memory (bool): Whether to enable dynamic memory. Default: False. + graph_memory_max_size (str): Set graph memory max size. Default: "26GB". + variable_memory_max_size (str): Set variable memory max size. Default: "5GB". + + Raises: + ValueError: If input key is not an attribute in context. + + Examples: + >>> context.set_context(mode=context.GRAPH_MODE) + >>> context.set_context(mode=context.PYNATIVE_MODE) + >>> context.set_context(device_target="Ascend") + >>> context.set_context(device_id=0) + >>> context.set_context(save_graphs=True, save_graphs_path="./model.ms") + >>> context.set_context(enable_task_sink=True) + >>> context.set_context(enable_mem_reuse=True) + >>> context.set_context(enable_reduce_precision=True) + >>> context.set_context(save_ms_model=True, save_ms_model_path=".") + >>> context.set_context(enable_gpu_summary=False) + >>> context.set_context(enable_dump=False, save_dump_path=".") + >>> context.set_context(reserve_class_name_in_scope=True) + >>> context.set_context(enable_dynamic_memory=True) + >>> context.set_context(graph_memory_max_size="25GB") + >>> context.set_context(variable_memory_max_size="6GB") + >>> context.set_context(mode=context.GRAPH_MODE, + >>> device_target="Ascend",device_id=0, save_graphs=True, + >>> save_graphs_path="/mindspore") + """ + for key, value in kwargs.items(): + if not hasattr(_context(), key): + raise ValueError("Set context keyword %s is not recognized!" % key) + setattr(_context(), key, value) + + +def get_context(attr_key): + """ + Gets context attribute value according to the input key. + + Args: + attr_key (str): The key of the attribute. + + Returns: + Object, The value of given attribute key. + + Raises: + ValueError: If input key is not an attribute in context. + """ + if not hasattr(_context(), attr_key): + raise ValueError("Get context keyword %s is not recognized!" % attr_key) + return getattr(_context(), attr_key) diff --git a/mindspore/dataset/__init__.py b/mindspore/dataset/__init__.py new file mode 100644 index 00000000000..479c66045fe --- /dev/null +++ b/mindspore/dataset/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module provides APIs to load and process various datasets: MNIST, +CIFAR-10, CIFAR-100, VOC, ImageNet, CelebA dataset, etc. It also supports +datasets in special format, including mindrecord, tfrecord, manifest. Users +can also create samplers with this module to sample data. +""" + +from .core.configuration import config +from .engine.datasets import StorageDataset, TFRecordDataset, ImageFolderDatasetV2, MnistDataset, MindDataset, \ + GeneratorDataset, ManifestDataset, Cifar10Dataset, Cifar100Dataset, VOCDataset, CelebADataset, Schema, \ + Shuffle, zip +from .engine.samplers import DistributedSampler, PKSampler, RandomSampler, SequentialSampler, SubsetRandomSampler, \ + WeightedRandomSampler +from .engine.serializer_deserializer import serialize, deserialize, show + +__all__ = ["config", "ImageFolderDatasetV2", "MnistDataset", "StorageDataset", + "MindDataset", "GeneratorDataset", "TFRecordDataset", + "ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", + "VOCDataset", "Schema", "DistributedSampler", "PKSampler", "RandomSampler", + "SequentialSampler", "SubsetRandomSampler", "WeightedRandomSampler", "zip"] diff --git a/mindspore/dataset/core/__init__.py b/mindspore/dataset/core/__init__.py new file mode 100644 index 00000000000..992e8ad355e --- /dev/null +++ b/mindspore/dataset/core/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mindspore/dataset/core/configuration.py b/mindspore/dataset/core/configuration.py new file mode 100644 index 00000000000..d052c138d8a --- /dev/null +++ b/mindspore/dataset/core/configuration.py @@ -0,0 +1,155 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The configuration manager. +""" + +import mindspore._c_dataengine as cde + +INT32_MAX = 2147483647 +UINT32_MAX = 4294967295 + + +class ConfigurationManager: + """The configuration manager""" + + def __init__(self): + self.config = cde.GlobalContext.config_manager() + + def set_seed(self, seed): + """ + Set the seed to be used in any random generator. This is used to produce deterministic results. + + Args: + seed(int): seed to be set + + Raises: + ValueError: If seed is invalid (< 0 or > MAX_UINT_32). + + Examples: + >>> import mindspore.dataset as ds + >>> con = ds.engine.ConfigurationManager() + >>> # sets the new seed value, now operators with a random seed will use new seed value. + >>> con.set_seed(1000) + """ + if seed < 0 or seed > UINT32_MAX: + raise ValueError("Seed given is not within the required range") + self.config.set_seed(seed) + + def get_seed(self): + """ + Get the seed + + Returns: + Int, seed. + """ + return self.config.get_seed() + + def set_prefetch_size(self, size): + """ + Set the number of rows to be prefetched. + + Args: + size: total number of rows to be prefetched. + + Raises: + ValueError: If prefetch_size is invalid (<= 0 or > MAX_INT_32). + + Examples: + >>> import mindspore.dataset as ds + >>> con = ds.engine.ConfigurationManager() + >>> # sets the new prefetch value. + >>> con.set_prefetch_size(1000) + """ + if size <= 0 or size > INT32_MAX: + raise ValueError("Prefetch size given is not within the required range") + self.config.set_op_connector_size(size) + + def get_prefetch_size(self): + """ + Get the prefetch size in number of rows. + + Returns: + Size, total number of rows to be prefetched. + """ + return self.config.get_op_connector_size() + + def set_num_parallel_workers(self, num): + """ + Set the default number of parallel workers + + Args: + num: number of parallel workers to be used as a default for each operation + + Raises: + ValueError: If num_parallel_workers is invalid (<= 0 or > MAX_INT_32). + + Examples: + >>> import mindspore.dataset as ds + >>> con = ds.engine.ConfigurationManager() + >>> # sets the new parallel_workers value, now parallel dataset operators will run with 8 workers. + >>> con.set_num_parallel_workers(8) + """ + if num <= 0 or num > INT32_MAX: + raise ValueError("Num workers given is not within the required range") + self.config.set_num_parallel_workers(num) + + def get_num_parallel_workers(self): + """ + Get the default number of parallel workers. + + Returns: + Int, number of parallel workers to be used as a default for each operation + """ + return self.config.get_num_parallel_workers() + + def __str__(self): + """ + String representation of the configurations. + + Returns: + Str, configurations. + """ + return str(self.config) + + def load(self, file): + """ + Load configuration from a file. + + Args: + file: path the config file to be loaded + + Raises: + RuntimeError: If file is invalid and parsing fails. + + Examples: + >>> import mindspore.dataset as ds + >>> con = ds.engine.ConfigurationManager() + >>> # sets the default value according to values in configuration file. + >>> con.load("path/to/config/file") + >>> # example config file: + >>> # { + >>> # "logFilePath": "/tmp", + >>> # "rowsPerBuffer": 32, + >>> # "numParallelWorkers": 4, + >>> # "workerConnectorSize": 16, + >>> # "opConnectorSize": 16, + >>> # "seed": 5489 + >>> # } + """ + self.config.load(file) + + +config = ConfigurationManager() diff --git a/mindspore/dataset/core/datatypes.py b/mindspore/dataset/core/datatypes.py new file mode 100644 index 00000000000..a8411d729a8 --- /dev/null +++ b/mindspore/dataset/core/datatypes.py @@ -0,0 +1,68 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Define the data types. +""" + +import mindspore._c_dataengine as cde +from mindspore._c_expression import typing + +import mindspore.common.dtype as mstype + + +def mstype_to_detype(type_): + """ + Get de data type corresponding to mindspore dtype. + + Args: + type_ (:class:`mindspore.dtype`): MindSpore's dtype. + + Returns: + The data type of de. + """ + if not isinstance(type_, typing.Type): + raise NotImplementedError() + + return { + mstype.bool_: cde.DataType("bool"), + mstype.int8: cde.DataType("int8"), + mstype.int16: cde.DataType("int16"), + mstype.int32: cde.DataType("int32"), + mstype.int64: cde.DataType("int64"), + mstype.uint8: cde.DataType("uint8"), + mstype.uint16: cde.DataType("uint16"), + mstype.uint32: cde.DataType("uint32"), + mstype.uint64: cde.DataType("uint64"), + mstype.float16: cde.DataType("float16"), + mstype.float32: cde.DataType("float32"), + mstype.float64: cde.DataType("float64"), + }[type_] + + +def mstypelist_to_detypelist(type_list): + """ + Get list[de type] corresponding to list[mindspore.dtype]. + + Args: + type_list (:list[mindspore.dtype]): a list of MindSpore's dtype. + + Returns: + The list of de data type. + """ + for index, _ in enumerate(type_list): + if type_list[index] is not None: + type_list[index] = mstype_to_detype(type_list[index]) + + return type_list diff --git a/mindspore/dataset/datapreprocess/__init__.py b/mindspore/dataset/datapreprocess/__init__.py new file mode 100644 index 00000000000..0c7d1aa7c77 --- /dev/null +++ b/mindspore/dataset/datapreprocess/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Preprocess of dataset. +""" + +from .preprocess_imagenet_validate_dataset import * diff --git a/mindspore/dataset/datapreprocess/imagenet_validate_dataset_2012_image_dir_map.txt b/mindspore/dataset/datapreprocess/imagenet_validate_dataset_2012_image_dir_map.txt new file mode 100644 index 00000000000..03f5357c65d --- /dev/null +++ b/mindspore/dataset/datapreprocess/imagenet_validate_dataset_2012_image_dir_map.txt @@ -0,0 +1,50000 @@ +ILSVRC2012_val_00000001.JPEG:n01751748 +ILSVRC2012_val_00000002.JPEG:n09193705 +ILSVRC2012_val_00000003.JPEG:n02105855 +ILSVRC2012_val_00000004.JPEG:n04263257 +ILSVRC2012_val_00000005.JPEG:n03125729 +ILSVRC2012_val_00000006.JPEG:n01735189 +ILSVRC2012_val_00000007.JPEG:n02346627 +ILSVRC2012_val_00000008.JPEG:n02776631 +ILSVRC2012_val_00000009.JPEG:n03794056 +ILSVRC2012_val_00000010.JPEG:n02328150 +ILSVRC2012_val_00000011.JPEG:n01917289 +ILSVRC2012_val_00000012.JPEG:n02125311 +ILSVRC2012_val_00000013.JPEG:n02484975 +ILSVRC2012_val_00000014.JPEG:n04065272 +ILSVRC2012_val_00000015.JPEG:n03496892 +ILSVRC2012_val_00000016.JPEG:n02066245 +ILSVRC2012_val_00000017.JPEG:n01914609 +ILSVRC2012_val_00000018.JPEG:n01616318 +ILSVRC2012_val_00000019.JPEG:n02971356 +ILSVRC2012_val_00000020.JPEG:n03126707 +ILSVRC2012_val_00000021.JPEG:n02346627 +ILSVRC2012_val_00000022.JPEG:n02091244 +ILSVRC2012_val_00000023.JPEG:n07742313 +ILSVRC2012_val_00000024.JPEG:n03956157 +ILSVRC2012_val_00000025.JPEG:n01616318 +ILSVRC2012_val_00000026.JPEG:n04380533 +ILSVRC2012_val_00000027.JPEG:n02114548 +ILSVRC2012_val_00000028.JPEG:n02089973 +ILSVRC2012_val_00000029.JPEG:n01729977 +ILSVRC2012_val_00000030.JPEG:n04435653 +ILSVRC2012_val_00000031.JPEG:n02280649 +ILSVRC2012_val_00000032.JPEG:n03444034 +ILSVRC2012_val_00000033.JPEG:n02077923 +ILSVRC2012_val_00000034.JPEG:n09835506 +ILSVRC2012_val_00000035.JPEG:n03478589 +ILSVRC2012_val_00000036.JPEG:n04532106 +ILSVRC2012_val_00000037.JPEG:n01644900 +ILSVRC2012_val_00000038.JPEG:n02666196 +ILSVRC2012_val_00000039.JPEG:n04141327 +ILSVRC2012_val_00000040.JPEG:n01773797 +ILSVRC2012_val_00000041.JPEG:n03125729 +ILSVRC2012_val_00000042.JPEG:n04049303 +ILSVRC2012_val_00000043.JPEG:n02006656 +ILSVRC2012_val_00000044.JPEG:n02097209 +ILSVRC2012_val_00000045.JPEG:n02111277 +ILSVRC2012_val_00000046.JPEG:n03950228 +ILSVRC2012_val_00000047.JPEG:n03393912 +ILSVRC2012_val_00000048.JPEG:n02089973 +ILSVRC2012_val_00000049.JPEG:n03930630 +ILSVRC2012_val_00000050.JPEG:n02640242 +ILSVRC2012_val_00000051.JPEG:n01828970 +ILSVRC2012_val_00000052.JPEG:n01632777 +ILSVRC2012_val_00000053.JPEG:n04372370 +ILSVRC2012_val_00000054.JPEG:n03485794 +ILSVRC2012_val_00000055.JPEG:n02443114 +ILSVRC2012_val_00000056.JPEG:n02930766 +ILSVRC2012_val_00000057.JPEG:n02112018 +ILSVRC2012_val_00000058.JPEG:n13040303 +ILSVRC2012_val_00000059.JPEG:n04485082 +ILSVRC2012_val_00000060.JPEG:n03482405 +ILSVRC2012_val_00000061.JPEG:n02963159 +ILSVRC2012_val_00000062.JPEG:n02093859 +ILSVRC2012_val_00000063.JPEG:n01910747 +ILSVRC2012_val_00000064.JPEG:n01693334 +ILSVRC2012_val_00000065.JPEG:n04371430 +ILSVRC2012_val_00000066.JPEG:n02526121 +ILSVRC2012_val_00000067.JPEG:n01871265 +ILSVRC2012_val_00000068.JPEG:n04532106 +ILSVRC2012_val_00000069.JPEG:n04482393 +ILSVRC2012_val_00000070.JPEG:n04370456 +ILSVRC2012_val_00000071.JPEG:n02927161 +ILSVRC2012_val_00000072.JPEG:n02074367 +ILSVRC2012_val_00000073.JPEG:n01608432 +ILSVRC2012_val_00000074.JPEG:n02966193 +ILSVRC2012_val_00000075.JPEG:n01795545 +ILSVRC2012_val_00000076.JPEG:n02791270 +ILSVRC2012_val_00000077.JPEG:n02087394 +ILSVRC2012_val_00000078.JPEG:n02116738 +ILSVRC2012_val_00000079.JPEG:n02091635 +ILSVRC2012_val_00000080.JPEG:n02895154 +ILSVRC2012_val_00000081.JPEG:n09193705 +ILSVRC2012_val_00000082.JPEG:n02088094 +ILSVRC2012_val_00000083.JPEG:n04200800 +ILSVRC2012_val_00000084.JPEG:n01737021 +ILSVRC2012_val_00000085.JPEG:n02974003 +ILSVRC2012_val_00000086.JPEG:n03032252 +ILSVRC2012_val_00000087.JPEG:n02483708 +ILSVRC2012_val_00000088.JPEG:n01632458 +ILSVRC2012_val_00000089.JPEG:n02992529 +ILSVRC2012_val_00000090.JPEG:n01698640 +ILSVRC2012_val_00000091.JPEG:n02114548 +ILSVRC2012_val_00000092.JPEG:n02497673 +ILSVRC2012_val_00000093.JPEG:n02480855 +ILSVRC2012_val_00000094.JPEG:n04147183 +ILSVRC2012_val_00000095.JPEG:n02487347 +ILSVRC2012_val_00000096.JPEG:n03895866 +ILSVRC2012_val_00000097.JPEG:n02325366 +ILSVRC2012_val_00000098.JPEG:n02033041 +ILSVRC2012_val_00000099.JPEG:n07745940 +ILSVRC2012_val_00000100.JPEG:n02415577 +ILSVRC2012_val_00000101.JPEG:n02951585 +ILSVRC2012_val_00000102.JPEG:n02087394 +ILSVRC2012_val_00000103.JPEG:n04485082 +ILSVRC2012_val_00000104.JPEG:n04505470 +ILSVRC2012_val_00000105.JPEG:n02097658 +ILSVRC2012_val_00000106.JPEG:n04591157 +ILSVRC2012_val_00000107.JPEG:n01770081 +ILSVRC2012_val_00000108.JPEG:n02992211 +ILSVRC2012_val_00000109.JPEG:n03691459 +ILSVRC2012_val_00000110.JPEG:n03594734 +ILSVRC2012_val_00000111.JPEG:n01983481 +ILSVRC2012_val_00000112.JPEG:n03937543 +ILSVRC2012_val_00000113.JPEG:n02105412 +ILSVRC2012_val_00000114.JPEG:n03843555 +ILSVRC2012_val_00000115.JPEG:n02091244 +ILSVRC2012_val_00000116.JPEG:n07831146 +ILSVRC2012_val_00000117.JPEG:n03710637 +ILSVRC2012_val_00000118.JPEG:n03733281 +ILSVRC2012_val_00000119.JPEG:n03782006 +ILSVRC2012_val_00000120.JPEG:n03733131 +ILSVRC2012_val_00000121.JPEG:n03933933 +ILSVRC2012_val_00000122.JPEG:n02980441 +ILSVRC2012_val_00000123.JPEG:n04409515 +ILSVRC2012_val_00000124.JPEG:n02606052 +ILSVRC2012_val_00000125.JPEG:n02226429 +ILSVRC2012_val_00000126.JPEG:n02883205 +ILSVRC2012_val_00000127.JPEG:n02422699 +ILSVRC2012_val_00000128.JPEG:n01614925 +ILSVRC2012_val_00000129.JPEG:n07697537 +ILSVRC2012_val_00000130.JPEG:n02123394 +ILSVRC2012_val_00000131.JPEG:n04252077 +ILSVRC2012_val_00000132.JPEG:n03337140 +ILSVRC2012_val_00000133.JPEG:n02117135 +ILSVRC2012_val_00000134.JPEG:n02107142 +ILSVRC2012_val_00000135.JPEG:n04037443 +ILSVRC2012_val_00000136.JPEG:n02397096 +ILSVRC2012_val_00000137.JPEG:n03187595 +ILSVRC2012_val_00000138.JPEG:n02319095 +ILSVRC2012_val_00000139.JPEG:n07932039 +ILSVRC2012_val_00000140.JPEG:n03372029 +ILSVRC2012_val_00000141.JPEG:n02088466 +ILSVRC2012_val_00000142.JPEG:n02319095 +ILSVRC2012_val_00000143.JPEG:n04125021 +ILSVRC2012_val_00000144.JPEG:n03954731 +ILSVRC2012_val_00000145.JPEG:n09421951 +ILSVRC2012_val_00000146.JPEG:n04487394 +ILSVRC2012_val_00000147.JPEG:n02113624 +ILSVRC2012_val_00000148.JPEG:n03843555 +ILSVRC2012_val_00000149.JPEG:n03485407 +ILSVRC2012_val_00000150.JPEG:n09332890 +ILSVRC2012_val_00000151.JPEG:n03642806 +ILSVRC2012_val_00000152.JPEG:n03710193 +ILSVRC2012_val_00000153.JPEG:n01677366 +ILSVRC2012_val_00000154.JPEG:n01950731 +ILSVRC2012_val_00000155.JPEG:n07714990 +ILSVRC2012_val_00000156.JPEG:n02114855 +ILSVRC2012_val_00000157.JPEG:n02119022 +ILSVRC2012_val_00000158.JPEG:n04086273 +ILSVRC2012_val_00000159.JPEG:n04201297 +ILSVRC2012_val_00000160.JPEG:n03733281 +ILSVRC2012_val_00000161.JPEG:n02100877 +ILSVRC2012_val_00000162.JPEG:n03016953 +ILSVRC2012_val_00000163.JPEG:n03733805 +ILSVRC2012_val_00000164.JPEG:n03063599 +ILSVRC2012_val_00000165.JPEG:n07714990 +ILSVRC2012_val_00000166.JPEG:n03854065 +ILSVRC2012_val_00000167.JPEG:n04149813 +ILSVRC2012_val_00000168.JPEG:n03786901 +ILSVRC2012_val_00000169.JPEG:n03467068 +ILSVRC2012_val_00000170.JPEG:n02087046 +ILSVRC2012_val_00000171.JPEG:n04326547 +ILSVRC2012_val_00000172.JPEG:n02100735 +ILSVRC2012_val_00000173.JPEG:n03775546 +ILSVRC2012_val_00000174.JPEG:n02111500 +ILSVRC2012_val_00000175.JPEG:n02814533 +ILSVRC2012_val_00000176.JPEG:n02097047 +ILSVRC2012_val_00000177.JPEG:n02027492 +ILSVRC2012_val_00000178.JPEG:n02109961 +ILSVRC2012_val_00000179.JPEG:n02389026 +ILSVRC2012_val_00000180.JPEG:n02105855 +ILSVRC2012_val_00000181.JPEG:n02445715 +ILSVRC2012_val_00000182.JPEG:n03259280 +ILSVRC2012_val_00000183.JPEG:n07711569 +ILSVRC2012_val_00000184.JPEG:n03710637 +ILSVRC2012_val_00000185.JPEG:n03670208 +ILSVRC2012_val_00000186.JPEG:n02128757 +ILSVRC2012_val_00000187.JPEG:n04467665 +ILSVRC2012_val_00000188.JPEG:n02114855 +ILSVRC2012_val_00000189.JPEG:n01873310 +ILSVRC2012_val_00000190.JPEG:n03476684 +ILSVRC2012_val_00000191.JPEG:n02093428 +ILSVRC2012_val_00000192.JPEG:n03891251 +ILSVRC2012_val_00000193.JPEG:n02859443 +ILSVRC2012_val_00000194.JPEG:n04125021 +ILSVRC2012_val_00000195.JPEG:n01978287 +ILSVRC2012_val_00000196.JPEG:n02643566 +ILSVRC2012_val_00000197.JPEG:n07697537 +ILSVRC2012_val_00000198.JPEG:n01560419 +ILSVRC2012_val_00000199.JPEG:n03290653 +ILSVRC2012_val_00000200.JPEG:n13037406 +ILSVRC2012_val_00000201.JPEG:n03891332 +ILSVRC2012_val_00000202.JPEG:n02883205 +ILSVRC2012_val_00000203.JPEG:n02106382 +ILSVRC2012_val_00000204.JPEG:n02672831 +ILSVRC2012_val_00000205.JPEG:n04330267 +ILSVRC2012_val_00000206.JPEG:n02489166 +ILSVRC2012_val_00000207.JPEG:n02058221 +ILSVRC2012_val_00000208.JPEG:n03584829 +ILSVRC2012_val_00000209.JPEG:n07565083 +ILSVRC2012_val_00000210.JPEG:n03125729 +ILSVRC2012_val_00000211.JPEG:n02123597 +ILSVRC2012_val_00000212.JPEG:n04536866 +ILSVRC2012_val_00000213.JPEG:n02965783 +ILSVRC2012_val_00000214.JPEG:n09428293 +ILSVRC2012_val_00000215.JPEG:n02965783 +ILSVRC2012_val_00000216.JPEG:n11879895 +ILSVRC2012_val_00000217.JPEG:n01560419 +ILSVRC2012_val_00000218.JPEG:n01775062 +ILSVRC2012_val_00000219.JPEG:n03595614 +ILSVRC2012_val_00000220.JPEG:n02110958 +ILSVRC2012_val_00000221.JPEG:n03709823 +ILSVRC2012_val_00000222.JPEG:n03777754 +ILSVRC2012_val_00000223.JPEG:n02951585 +ILSVRC2012_val_00000224.JPEG:n02100877 +ILSVRC2012_val_00000225.JPEG:n01629819 +ILSVRC2012_val_00000226.JPEG:n02909870 +ILSVRC2012_val_00000227.JPEG:n02101388 +ILSVRC2012_val_00000228.JPEG:n02091244 +ILSVRC2012_val_00000229.JPEG:n01667114 +ILSVRC2012_val_00000230.JPEG:n03998194 +ILSVRC2012_val_00000231.JPEG:n01986214 +ILSVRC2012_val_00000232.JPEG:n04192698 +ILSVRC2012_val_00000233.JPEG:n02128757 +ILSVRC2012_val_00000234.JPEG:n02793495 +ILSVRC2012_val_00000235.JPEG:n09256479 +ILSVRC2012_val_00000236.JPEG:n01443537 +ILSVRC2012_val_00000237.JPEG:n02089973 +ILSVRC2012_val_00000238.JPEG:n01981276 +ILSVRC2012_val_00000239.JPEG:n02837789 +ILSVRC2012_val_00000240.JPEG:n03888605 +ILSVRC2012_val_00000241.JPEG:n03201208 +ILSVRC2012_val_00000242.JPEG:n02480855 +ILSVRC2012_val_00000243.JPEG:n03814639 +ILSVRC2012_val_00000244.JPEG:n04090263 +ILSVRC2012_val_00000245.JPEG:n01986214 +ILSVRC2012_val_00000246.JPEG:n02415577 +ILSVRC2012_val_00000247.JPEG:n01534433 +ILSVRC2012_val_00000248.JPEG:n02093256 +ILSVRC2012_val_00000249.JPEG:n03134739 +ILSVRC2012_val_00000250.JPEG:n03016953 +ILSVRC2012_val_00000251.JPEG:n12620546 +ILSVRC2012_val_00000252.JPEG:n03937543 +ILSVRC2012_val_00000253.JPEG:n02815834 +ILSVRC2012_val_00000254.JPEG:n03776460 +ILSVRC2012_val_00000255.JPEG:n10565667 +ILSVRC2012_val_00000256.JPEG:n03207743 +ILSVRC2012_val_00000257.JPEG:n02992529 +ILSVRC2012_val_00000258.JPEG:n01631663 +ILSVRC2012_val_00000259.JPEG:n03729826 +ILSVRC2012_val_00000260.JPEG:n04033995 +ILSVRC2012_val_00000261.JPEG:n04462240 +ILSVRC2012_val_00000262.JPEG:n01443537 +ILSVRC2012_val_00000263.JPEG:n02091831 +ILSVRC2012_val_00000264.JPEG:n03874293 +ILSVRC2012_val_00000265.JPEG:n03874599 +ILSVRC2012_val_00000266.JPEG:n04238763 +ILSVRC2012_val_00000267.JPEG:n07584110 +ILSVRC2012_val_00000268.JPEG:n02749479 +ILSVRC2012_val_00000269.JPEG:n02110185 +ILSVRC2012_val_00000270.JPEG:n09193705 +ILSVRC2012_val_00000271.JPEG:n04311004 +ILSVRC2012_val_00000272.JPEG:n02788148 +ILSVRC2012_val_00000273.JPEG:n02445715 +ILSVRC2012_val_00000274.JPEG:n06874185 +ILSVRC2012_val_00000275.JPEG:n04074963 +ILSVRC2012_val_00000276.JPEG:n01631663 +ILSVRC2012_val_00000277.JPEG:n03803284 +ILSVRC2012_val_00000278.JPEG:n01828970 +ILSVRC2012_val_00000279.JPEG:n02096437 +ILSVRC2012_val_00000280.JPEG:n04554684 +ILSVRC2012_val_00000281.JPEG:n03599486 +ILSVRC2012_val_00000282.JPEG:n03595614 +ILSVRC2012_val_00000283.JPEG:n02123394 +ILSVRC2012_val_00000284.JPEG:n04515003 +ILSVRC2012_val_00000285.JPEG:n04591157 +ILSVRC2012_val_00000286.JPEG:n04560804 +ILSVRC2012_val_00000287.JPEG:n02794156 +ILSVRC2012_val_00000288.JPEG:n03344393 +ILSVRC2012_val_00000289.JPEG:n02687172 +ILSVRC2012_val_00000290.JPEG:n04328186 +ILSVRC2012_val_00000291.JPEG:n04479046 +ILSVRC2012_val_00000292.JPEG:n03967562 +ILSVRC2012_val_00000293.JPEG:n01440764 +ILSVRC2012_val_00000294.JPEG:n04465501 +ILSVRC2012_val_00000295.JPEG:n03457902 +ILSVRC2012_val_00000296.JPEG:n04532670 +ILSVRC2012_val_00000297.JPEG:n01688243 +ILSVRC2012_val_00000298.JPEG:n01749939 +ILSVRC2012_val_00000299.JPEG:n01768244 +ILSVRC2012_val_00000300.JPEG:n02091831 +ILSVRC2012_val_00000301.JPEG:n02321529 +ILSVRC2012_val_00000302.JPEG:n02939185 +ILSVRC2012_val_00000303.JPEG:n02129604 +ILSVRC2012_val_00000304.JPEG:n12985857 +ILSVRC2012_val_00000305.JPEG:n03485794 +ILSVRC2012_val_00000306.JPEG:n02408429 +ILSVRC2012_val_00000307.JPEG:n01443537 +ILSVRC2012_val_00000308.JPEG:n03590841 +ILSVRC2012_val_00000309.JPEG:n07697537 +ILSVRC2012_val_00000310.JPEG:n04154565 +ILSVRC2012_val_00000311.JPEG:n03443371 +ILSVRC2012_val_00000312.JPEG:n02514041 +ILSVRC2012_val_00000313.JPEG:n09468604 +ILSVRC2012_val_00000314.JPEG:n03769881 +ILSVRC2012_val_00000315.JPEG:n02787622 +ILSVRC2012_val_00000316.JPEG:n02526121 +ILSVRC2012_val_00000317.JPEG:n03888605 +ILSVRC2012_val_00000318.JPEG:n01622779 +ILSVRC2012_val_00000319.JPEG:n01872401 +ILSVRC2012_val_00000320.JPEG:n07745940 +ILSVRC2012_val_00000321.JPEG:n03085013 +ILSVRC2012_val_00000322.JPEG:n02445715 +ILSVRC2012_val_00000323.JPEG:n02120505 +ILSVRC2012_val_00000324.JPEG:n01751748 +ILSVRC2012_val_00000325.JPEG:n04141327 +ILSVRC2012_val_00000326.JPEG:n02443484 +ILSVRC2012_val_00000327.JPEG:n02089078 +ILSVRC2012_val_00000328.JPEG:n01608432 +ILSVRC2012_val_00000329.JPEG:n01514668 +ILSVRC2012_val_00000330.JPEG:n03160309 +ILSVRC2012_val_00000331.JPEG:n04070727 +ILSVRC2012_val_00000332.JPEG:n07715103 +ILSVRC2012_val_00000333.JPEG:n02110958 +ILSVRC2012_val_00000334.JPEG:n03976657 +ILSVRC2012_val_00000335.JPEG:n03902125 +ILSVRC2012_val_00000336.JPEG:n02909870 +ILSVRC2012_val_00000337.JPEG:n01740131 +ILSVRC2012_val_00000338.JPEG:n04532106 +ILSVRC2012_val_00000339.JPEG:n03197337 +ILSVRC2012_val_00000340.JPEG:n02493509 +ILSVRC2012_val_00000341.JPEG:n10148035 +ILSVRC2012_val_00000342.JPEG:n02172182 +ILSVRC2012_val_00000343.JPEG:n02437616 +ILSVRC2012_val_00000344.JPEG:n03062245 +ILSVRC2012_val_00000345.JPEG:n04286575 +ILSVRC2012_val_00000346.JPEG:n03018349 +ILSVRC2012_val_00000347.JPEG:n02951358 +ILSVRC2012_val_00000348.JPEG:n02130308 +ILSVRC2012_val_00000349.JPEG:n04277352 +ILSVRC2012_val_00000350.JPEG:n02096585 +ILSVRC2012_val_00000351.JPEG:n04589890 +ILSVRC2012_val_00000352.JPEG:n02965783 +ILSVRC2012_val_00000353.JPEG:n02978881 +ILSVRC2012_val_00000354.JPEG:n02804414 +ILSVRC2012_val_00000355.JPEG:n02112137 +ILSVRC2012_val_00000356.JPEG:n02007558 +ILSVRC2012_val_00000357.JPEG:n03670208 +ILSVRC2012_val_00000358.JPEG:n02894605 +ILSVRC2012_val_00000359.JPEG:n03657121 +ILSVRC2012_val_00000360.JPEG:n03876231 +ILSVRC2012_val_00000361.JPEG:n02165105 +ILSVRC2012_val_00000362.JPEG:n01669191 +ILSVRC2012_val_00000363.JPEG:n02011460 +ILSVRC2012_val_00000364.JPEG:n03710193 +ILSVRC2012_val_00000365.JPEG:n03796401 +ILSVRC2012_val_00000366.JPEG:n02916936 +ILSVRC2012_val_00000367.JPEG:n03492542 +ILSVRC2012_val_00000368.JPEG:n03998194 +ILSVRC2012_val_00000369.JPEG:n04552348 +ILSVRC2012_val_00000370.JPEG:n01824575 +ILSVRC2012_val_00000371.JPEG:n01917289 +ILSVRC2012_val_00000372.JPEG:n03461385 +ILSVRC2012_val_00000373.JPEG:n03874293 +ILSVRC2012_val_00000374.JPEG:n03272010 +ILSVRC2012_val_00000375.JPEG:n02099712 +ILSVRC2012_val_00000376.JPEG:n02999410 +ILSVRC2012_val_00000377.JPEG:n04179913 +ILSVRC2012_val_00000378.JPEG:n07831146 +ILSVRC2012_val_00000379.JPEG:n02096177 +ILSVRC2012_val_00000380.JPEG:n04350905 +ILSVRC2012_val_00000381.JPEG:n04507155 +ILSVRC2012_val_00000382.JPEG:n03743016 +ILSVRC2012_val_00000383.JPEG:n02105505 +ILSVRC2012_val_00000384.JPEG:n03649909 +ILSVRC2012_val_00000385.JPEG:n03680355 +ILSVRC2012_val_00000386.JPEG:n01910747 +ILSVRC2012_val_00000387.JPEG:n03529860 +ILSVRC2012_val_00000388.JPEG:n02787622 +ILSVRC2012_val_00000389.JPEG:n02012849 +ILSVRC2012_val_00000390.JPEG:n02011460 +ILSVRC2012_val_00000391.JPEG:n02094114 +ILSVRC2012_val_00000392.JPEG:n02950826 +ILSVRC2012_val_00000393.JPEG:n02105855 +ILSVRC2012_val_00000394.JPEG:n09288635 +ILSVRC2012_val_00000395.JPEG:n01773797 +ILSVRC2012_val_00000396.JPEG:n01774750 +ILSVRC2012_val_00000397.JPEG:n04409515 +ILSVRC2012_val_00000398.JPEG:n02497673 +ILSVRC2012_val_00000399.JPEG:n02113799 +ILSVRC2012_val_00000400.JPEG:n02786058 +ILSVRC2012_val_00000401.JPEG:n02443484 +ILSVRC2012_val_00000402.JPEG:n02981792 +ILSVRC2012_val_00000403.JPEG:n03095699 +ILSVRC2012_val_00000404.JPEG:n01664065 +ILSVRC2012_val_00000405.JPEG:n02092002 +ILSVRC2012_val_00000406.JPEG:n07711569 +ILSVRC2012_val_00000407.JPEG:n02219486 +ILSVRC2012_val_00000408.JPEG:n13133613 +ILSVRC2012_val_00000409.JPEG:n02114548 +ILSVRC2012_val_00000410.JPEG:n03529860 +ILSVRC2012_val_00000411.JPEG:n02097298 +ILSVRC2012_val_00000412.JPEG:n13133613 +ILSVRC2012_val_00000413.JPEG:n04355933 +ILSVRC2012_val_00000414.JPEG:n01537544 +ILSVRC2012_val_00000415.JPEG:n01847000 +ILSVRC2012_val_00000416.JPEG:n04428191 +ILSVRC2012_val_00000417.JPEG:n02666196 +ILSVRC2012_val_00000418.JPEG:n02268443 +ILSVRC2012_val_00000419.JPEG:n03291819 +ILSVRC2012_val_00000420.JPEG:n01828970 +ILSVRC2012_val_00000421.JPEG:n04099969 +ILSVRC2012_val_00000422.JPEG:n02747177 +ILSVRC2012_val_00000423.JPEG:n07720875 +ILSVRC2012_val_00000424.JPEG:n02088094 +ILSVRC2012_val_00000425.JPEG:n02113624 +ILSVRC2012_val_00000426.JPEG:n03710637 +ILSVRC2012_val_00000427.JPEG:n03637318 +ILSVRC2012_val_00000428.JPEG:n03942813 +ILSVRC2012_val_00000429.JPEG:n02093859 +ILSVRC2012_val_00000430.JPEG:n03794056 +ILSVRC2012_val_00000431.JPEG:n02930766 +ILSVRC2012_val_00000432.JPEG:n02930766 +ILSVRC2012_val_00000433.JPEG:n04525038 +ILSVRC2012_val_00000434.JPEG:n03796401 +ILSVRC2012_val_00000435.JPEG:n03709823 +ILSVRC2012_val_00000436.JPEG:n02097047 +ILSVRC2012_val_00000437.JPEG:n04604644 +ILSVRC2012_val_00000438.JPEG:n03938244 +ILSVRC2012_val_00000439.JPEG:n01560419 +ILSVRC2012_val_00000440.JPEG:n02097298 +ILSVRC2012_val_00000441.JPEG:n02091635 +ILSVRC2012_val_00000442.JPEG:n04136333 +ILSVRC2012_val_00000443.JPEG:n07718747 +ILSVRC2012_val_00000444.JPEG:n02417914 +ILSVRC2012_val_00000445.JPEG:n03355925 +ILSVRC2012_val_00000446.JPEG:n02445715 +ILSVRC2012_val_00000447.JPEG:n02445715 +ILSVRC2012_val_00000448.JPEG:n03495258 +ILSVRC2012_val_00000449.JPEG:n04447861 +ILSVRC2012_val_00000450.JPEG:n02111500 +ILSVRC2012_val_00000451.JPEG:n03584829 +ILSVRC2012_val_00000452.JPEG:n03977966 +ILSVRC2012_val_00000453.JPEG:n04116512 +ILSVRC2012_val_00000454.JPEG:n04019541 +ILSVRC2012_val_00000455.JPEG:n04200800 +ILSVRC2012_val_00000456.JPEG:n02408429 +ILSVRC2012_val_00000457.JPEG:n02085936 +ILSVRC2012_val_00000458.JPEG:n03992509 +ILSVRC2012_val_00000459.JPEG:n02769748 +ILSVRC2012_val_00000460.JPEG:n04613696 +ILSVRC2012_val_00000461.JPEG:n07716906 +ILSVRC2012_val_00000462.JPEG:n02085782 +ILSVRC2012_val_00000463.JPEG:n07718472 +ILSVRC2012_val_00000464.JPEG:n04398044 +ILSVRC2012_val_00000465.JPEG:n03920288 +ILSVRC2012_val_00000466.JPEG:n01860187 +ILSVRC2012_val_00000467.JPEG:n03272010 +ILSVRC2012_val_00000468.JPEG:n04008634 +ILSVRC2012_val_00000469.JPEG:n04090263 +ILSVRC2012_val_00000470.JPEG:n02028035 +ILSVRC2012_val_00000471.JPEG:n01677366 +ILSVRC2012_val_00000472.JPEG:n13037406 +ILSVRC2012_val_00000473.JPEG:n04067472 +ILSVRC2012_val_00000474.JPEG:n02095889 +ILSVRC2012_val_00000475.JPEG:n04532670 +ILSVRC2012_val_00000476.JPEG:n01582220 +ILSVRC2012_val_00000477.JPEG:n03476684 +ILSVRC2012_val_00000478.JPEG:n02395406 +ILSVRC2012_val_00000479.JPEG:n04487394 +ILSVRC2012_val_00000480.JPEG:n02443484 +ILSVRC2012_val_00000481.JPEG:n02510455 +ILSVRC2012_val_00000482.JPEG:n04550184 +ILSVRC2012_val_00000483.JPEG:n02814860 +ILSVRC2012_val_00000484.JPEG:n12144580 +ILSVRC2012_val_00000485.JPEG:n03126707 +ILSVRC2012_val_00000486.JPEG:n02486410 +ILSVRC2012_val_00000487.JPEG:n02125311 +ILSVRC2012_val_00000488.JPEG:n03777754 +ILSVRC2012_val_00000489.JPEG:n03924679 +ILSVRC2012_val_00000490.JPEG:n04613696 +ILSVRC2012_val_00000491.JPEG:n07875152 +ILSVRC2012_val_00000492.JPEG:n02058221 +ILSVRC2012_val_00000493.JPEG:n03188531 +ILSVRC2012_val_00000494.JPEG:n02777292 +ILSVRC2012_val_00000495.JPEG:n02489166 +ILSVRC2012_val_00000496.JPEG:n02066245 +ILSVRC2012_val_00000497.JPEG:n04579432 +ILSVRC2012_val_00000498.JPEG:n01630670 +ILSVRC2012_val_00000499.JPEG:n02666196 +ILSVRC2012_val_00000500.JPEG:n02091635 +ILSVRC2012_val_00000501.JPEG:n02114548 +ILSVRC2012_val_00000502.JPEG:n02356798 +ILSVRC2012_val_00000503.JPEG:n03201208 +ILSVRC2012_val_00000504.JPEG:n03240683 +ILSVRC2012_val_00000505.JPEG:n03590841 +ILSVRC2012_val_00000506.JPEG:n03018349 +ILSVRC2012_val_00000507.JPEG:n02104029 +ILSVRC2012_val_00000508.JPEG:n04251144 +ILSVRC2012_val_00000509.JPEG:n10148035 +ILSVRC2012_val_00000510.JPEG:n02169497 +ILSVRC2012_val_00000511.JPEG:n02089867 +ILSVRC2012_val_00000512.JPEG:n01734418 +ILSVRC2012_val_00000513.JPEG:n04476259 +ILSVRC2012_val_00000514.JPEG:n02843684 +ILSVRC2012_val_00000515.JPEG:n04008634 +ILSVRC2012_val_00000516.JPEG:n03400231 +ILSVRC2012_val_00000517.JPEG:n02119022 +ILSVRC2012_val_00000518.JPEG:n02137549 +ILSVRC2012_val_00000519.JPEG:n03761084 +ILSVRC2012_val_00000520.JPEG:n02490219 +ILSVRC2012_val_00000521.JPEG:n03840681 +ILSVRC2012_val_00000522.JPEG:n04346328 +ILSVRC2012_val_00000523.JPEG:n01677366 +ILSVRC2012_val_00000524.JPEG:n02102318 +ILSVRC2012_val_00000525.JPEG:n04458633 +ILSVRC2012_val_00000526.JPEG:n04476259 +ILSVRC2012_val_00000527.JPEG:n04209239 +ILSVRC2012_val_00000528.JPEG:n01795545 +ILSVRC2012_val_00000529.JPEG:n10565667 +ILSVRC2012_val_00000530.JPEG:n02114367 +ILSVRC2012_val_00000531.JPEG:n02107574 +ILSVRC2012_val_00000532.JPEG:n03032252 +ILSVRC2012_val_00000533.JPEG:n02104365 +ILSVRC2012_val_00000534.JPEG:n03133878 +ILSVRC2012_val_00000535.JPEG:n04336792 +ILSVRC2012_val_00000536.JPEG:n02112137 +ILSVRC2012_val_00000537.JPEG:n03000684 +ILSVRC2012_val_00000538.JPEG:n04553703 +ILSVRC2012_val_00000539.JPEG:n02102480 +ILSVRC2012_val_00000540.JPEG:n03825788 +ILSVRC2012_val_00000541.JPEG:n01695060 +ILSVRC2012_val_00000542.JPEG:n03250847 +ILSVRC2012_val_00000543.JPEG:n07860988 +ILSVRC2012_val_00000544.JPEG:n04310018 +ILSVRC2012_val_00000545.JPEG:n02071294 +ILSVRC2012_val_00000546.JPEG:n01945685 +ILSVRC2012_val_00000547.JPEG:n01855672 +ILSVRC2012_val_00000548.JPEG:n02037110 +ILSVRC2012_val_00000549.JPEG:n03868863 +ILSVRC2012_val_00000550.JPEG:n04229816 +ILSVRC2012_val_00000551.JPEG:n12057211 +ILSVRC2012_val_00000552.JPEG:n02408429 +ILSVRC2012_val_00000553.JPEG:n02481823 +ILSVRC2012_val_00000554.JPEG:n07716358 +ILSVRC2012_val_00000555.JPEG:n04487394 +ILSVRC2012_val_00000556.JPEG:n03662601 +ILSVRC2012_val_00000557.JPEG:n02979186 +ILSVRC2012_val_00000558.JPEG:n02910353 +ILSVRC2012_val_00000559.JPEG:n04266014 +ILSVRC2012_val_00000560.JPEG:n03895866 +ILSVRC2012_val_00000561.JPEG:n04443257 +ILSVRC2012_val_00000562.JPEG:n02917067 +ILSVRC2012_val_00000563.JPEG:n04149813 +ILSVRC2012_val_00000564.JPEG:n03041632 +ILSVRC2012_val_00000565.JPEG:n02364673 +ILSVRC2012_val_00000566.JPEG:n02999410 +ILSVRC2012_val_00000567.JPEG:n04435653 +ILSVRC2012_val_00000568.JPEG:n04228054 +ILSVRC2012_val_00000569.JPEG:n02814860 +ILSVRC2012_val_00000570.JPEG:n01531178 +ILSVRC2012_val_00000571.JPEG:n03662601 +ILSVRC2012_val_00000572.JPEG:n07880968 +ILSVRC2012_val_00000573.JPEG:n04487081 +ILSVRC2012_val_00000574.JPEG:n07614500 +ILSVRC2012_val_00000575.JPEG:n03532672 +ILSVRC2012_val_00000576.JPEG:n01807496 +ILSVRC2012_val_00000577.JPEG:n02011460 +ILSVRC2012_val_00000578.JPEG:n02074367 +ILSVRC2012_val_00000579.JPEG:n04462240 +ILSVRC2012_val_00000580.JPEG:n02977058 +ILSVRC2012_val_00000581.JPEG:n02281406 +ILSVRC2012_val_00000582.JPEG:n03041632 +ILSVRC2012_val_00000583.JPEG:n04350905 +ILSVRC2012_val_00000584.JPEG:n02788148 +ILSVRC2012_val_00000585.JPEG:n02137549 +ILSVRC2012_val_00000586.JPEG:n04562935 +ILSVRC2012_val_00000587.JPEG:n04590129 +ILSVRC2012_val_00000588.JPEG:n02093991 +ILSVRC2012_val_00000589.JPEG:n03995372 +ILSVRC2012_val_00000590.JPEG:n02111889 +ILSVRC2012_val_00000591.JPEG:n04081281 +ILSVRC2012_val_00000592.JPEG:n02133161 +ILSVRC2012_val_00000593.JPEG:n02006656 +ILSVRC2012_val_00000594.JPEG:n02107908 +ILSVRC2012_val_00000595.JPEG:n04347754 +ILSVRC2012_val_00000596.JPEG:n02950826 +ILSVRC2012_val_00000597.JPEG:n02504013 +ILSVRC2012_val_00000598.JPEG:n04560804 +ILSVRC2012_val_00000599.JPEG:n02088364 +ILSVRC2012_val_00000600.JPEG:n02128385 +ILSVRC2012_val_00000601.JPEG:n02860847 +ILSVRC2012_val_00000602.JPEG:n04399382 +ILSVRC2012_val_00000603.JPEG:n02105412 +ILSVRC2012_val_00000604.JPEG:n02115641 +ILSVRC2012_val_00000605.JPEG:n07753592 +ILSVRC2012_val_00000606.JPEG:n07880968 +ILSVRC2012_val_00000607.JPEG:n03598930 +ILSVRC2012_val_00000608.JPEG:n03724870 +ILSVRC2012_val_00000609.JPEG:n02066245 +ILSVRC2012_val_00000610.JPEG:n02128925 +ILSVRC2012_val_00000611.JPEG:n04465501 +ILSVRC2012_val_00000612.JPEG:n02094258 +ILSVRC2012_val_00000613.JPEG:n02086646 +ILSVRC2012_val_00000614.JPEG:n04141076 +ILSVRC2012_val_00000615.JPEG:n04136333 +ILSVRC2012_val_00000616.JPEG:n13133613 +ILSVRC2012_val_00000617.JPEG:n02342885 +ILSVRC2012_val_00000618.JPEG:n02281406 +ILSVRC2012_val_00000619.JPEG:n03443371 +ILSVRC2012_val_00000620.JPEG:n07613480 +ILSVRC2012_val_00000621.JPEG:n04008634 +ILSVRC2012_val_00000622.JPEG:n04141327 +ILSVRC2012_val_00000623.JPEG:n04347754 +ILSVRC2012_val_00000624.JPEG:n03314780 +ILSVRC2012_val_00000625.JPEG:n02165456 +ILSVRC2012_val_00000626.JPEG:n03930313 +ILSVRC2012_val_00000627.JPEG:n04392985 +ILSVRC2012_val_00000628.JPEG:n01872401 +ILSVRC2012_val_00000629.JPEG:n04204238 +ILSVRC2012_val_00000630.JPEG:n07831146 +ILSVRC2012_val_00000631.JPEG:n02690373 +ILSVRC2012_val_00000632.JPEG:n12144580 +ILSVRC2012_val_00000633.JPEG:n02776631 +ILSVRC2012_val_00000634.JPEG:n02877765 +ILSVRC2012_val_00000635.JPEG:n02108089 +ILSVRC2012_val_00000636.JPEG:n03532672 +ILSVRC2012_val_00000637.JPEG:n03126707 +ILSVRC2012_val_00000638.JPEG:n01560419 +ILSVRC2012_val_00000639.JPEG:n02268853 +ILSVRC2012_val_00000640.JPEG:n03691459 +ILSVRC2012_val_00000641.JPEG:n03404251 +ILSVRC2012_val_00000642.JPEG:n02364673 +ILSVRC2012_val_00000643.JPEG:n02101556 +ILSVRC2012_val_00000644.JPEG:n02326432 +ILSVRC2012_val_00000645.JPEG:n03954731 +ILSVRC2012_val_00000646.JPEG:n07831146 +ILSVRC2012_val_00000647.JPEG:n03584254 +ILSVRC2012_val_00000648.JPEG:n02012849 +ILSVRC2012_val_00000649.JPEG:n03804744 +ILSVRC2012_val_00000650.JPEG:n02128385 +ILSVRC2012_val_00000651.JPEG:n01530575 +ILSVRC2012_val_00000652.JPEG:n03933933 +ILSVRC2012_val_00000653.JPEG:n04409515 +ILSVRC2012_val_00000654.JPEG:n02823428 +ILSVRC2012_val_00000655.JPEG:n01877812 +ILSVRC2012_val_00000656.JPEG:n03920288 +ILSVRC2012_val_00000657.JPEG:n02510455 +ILSVRC2012_val_00000658.JPEG:n02112350 +ILSVRC2012_val_00000659.JPEG:n03594945 +ILSVRC2012_val_00000660.JPEG:n03642806 +ILSVRC2012_val_00000661.JPEG:n02395406 +ILSVRC2012_val_00000662.JPEG:n03452741 +ILSVRC2012_val_00000663.JPEG:n02860847 +ILSVRC2012_val_00000664.JPEG:n03673027 +ILSVRC2012_val_00000665.JPEG:n02102040 +ILSVRC2012_val_00000666.JPEG:n04505470 +ILSVRC2012_val_00000667.JPEG:n04086273 +ILSVRC2012_val_00000668.JPEG:n02099849 +ILSVRC2012_val_00000669.JPEG:n01990800 +ILSVRC2012_val_00000670.JPEG:n03781244 +ILSVRC2012_val_00000671.JPEG:n04461696 +ILSVRC2012_val_00000672.JPEG:n02106166 +ILSVRC2012_val_00000673.JPEG:n04141076 +ILSVRC2012_val_00000674.JPEG:n07717556 +ILSVRC2012_val_00000675.JPEG:n02361337 +ILSVRC2012_val_00000676.JPEG:n03976657 +ILSVRC2012_val_00000677.JPEG:n03832673 +ILSVRC2012_val_00000678.JPEG:n03109150 +ILSVRC2012_val_00000679.JPEG:n01776313 +ILSVRC2012_val_00000680.JPEG:n03788195 +ILSVRC2012_val_00000681.JPEG:n03884397 +ILSVRC2012_val_00000682.JPEG:n04019541 +ILSVRC2012_val_00000683.JPEG:n01693334 +ILSVRC2012_val_00000684.JPEG:n03633091 +ILSVRC2012_val_00000685.JPEG:n02325366 +ILSVRC2012_val_00000686.JPEG:n03623198 +ILSVRC2012_val_00000687.JPEG:n02795169 +ILSVRC2012_val_00000688.JPEG:n01744401 +ILSVRC2012_val_00000689.JPEG:n01955084 +ILSVRC2012_val_00000690.JPEG:n02002556 +ILSVRC2012_val_00000691.JPEG:n07754684 +ILSVRC2012_val_00000692.JPEG:n02174001 +ILSVRC2012_val_00000693.JPEG:n02793495 +ILSVRC2012_val_00000694.JPEG:n02095889 +ILSVRC2012_val_00000695.JPEG:n02484975 +ILSVRC2012_val_00000696.JPEG:n02094433 +ILSVRC2012_val_00000697.JPEG:n09229709 +ILSVRC2012_val_00000698.JPEG:n03207941 +ILSVRC2012_val_00000699.JPEG:n02655020 +ILSVRC2012_val_00000700.JPEG:n03773504 +ILSVRC2012_val_00000701.JPEG:n04367480 +ILSVRC2012_val_00000702.JPEG:n03933933 +ILSVRC2012_val_00000703.JPEG:n01955084 +ILSVRC2012_val_00000704.JPEG:n04355933 +ILSVRC2012_val_00000705.JPEG:n13040303 +ILSVRC2012_val_00000706.JPEG:n02786058 +ILSVRC2012_val_00000707.JPEG:n04090263 +ILSVRC2012_val_00000708.JPEG:n02101006 +ILSVRC2012_val_00000709.JPEG:n02124075 +ILSVRC2012_val_00000710.JPEG:n03720891 +ILSVRC2012_val_00000711.JPEG:n07749582 +ILSVRC2012_val_00000712.JPEG:n04517823 +ILSVRC2012_val_00000713.JPEG:n01534433 +ILSVRC2012_val_00000714.JPEG:n04335435 +ILSVRC2012_val_00000715.JPEG:n03661043 +ILSVRC2012_val_00000716.JPEG:n02101556 +ILSVRC2012_val_00000717.JPEG:n03785016 +ILSVRC2012_val_00000718.JPEG:n03133878 +ILSVRC2012_val_00000719.JPEG:n02113978 +ILSVRC2012_val_00000720.JPEG:n02930766 +ILSVRC2012_val_00000721.JPEG:n02783161 +ILSVRC2012_val_00000722.JPEG:n03958227 +ILSVRC2012_val_00000723.JPEG:n02441942 +ILSVRC2012_val_00000724.JPEG:n02859443 +ILSVRC2012_val_00000725.JPEG:n02096437 +ILSVRC2012_val_00000726.JPEG:n02447366 +ILSVRC2012_val_00000727.JPEG:n07742313 +ILSVRC2012_val_00000728.JPEG:n07583066 +ILSVRC2012_val_00000729.JPEG:n02110063 +ILSVRC2012_val_00000730.JPEG:n03146219 +ILSVRC2012_val_00000731.JPEG:n12998815 +ILSVRC2012_val_00000732.JPEG:n03425413 +ILSVRC2012_val_00000733.JPEG:n02123394 +ILSVRC2012_val_00000734.JPEG:n03594734 +ILSVRC2012_val_00000735.JPEG:n02006656 +ILSVRC2012_val_00000736.JPEG:n02992211 +ILSVRC2012_val_00000737.JPEG:n04442312 +ILSVRC2012_val_00000738.JPEG:n03032252 +ILSVRC2012_val_00000739.JPEG:n01608432 +ILSVRC2012_val_00000740.JPEG:n02927161 +ILSVRC2012_val_00000741.JPEG:n03485794 +ILSVRC2012_val_00000742.JPEG:n07583066 +ILSVRC2012_val_00000743.JPEG:n03347037 +ILSVRC2012_val_00000744.JPEG:n01847000 +ILSVRC2012_val_00000745.JPEG:n04557648 +ILSVRC2012_val_00000746.JPEG:n03478589 +ILSVRC2012_val_00000747.JPEG:n01530575 +ILSVRC2012_val_00000748.JPEG:n02098105 +ILSVRC2012_val_00000749.JPEG:n01755581 +ILSVRC2012_val_00000750.JPEG:n03045698 +ILSVRC2012_val_00000751.JPEG:n02028035 +ILSVRC2012_val_00000752.JPEG:n03538406 +ILSVRC2012_val_00000753.JPEG:n03956157 +ILSVRC2012_val_00000754.JPEG:n01871265 +ILSVRC2012_val_00000755.JPEG:n13044778 +ILSVRC2012_val_00000756.JPEG:n02119789 +ILSVRC2012_val_00000757.JPEG:n07875152 +ILSVRC2012_val_00000758.JPEG:n02107908 +ILSVRC2012_val_00000759.JPEG:n02791124 +ILSVRC2012_val_00000760.JPEG:n03697007 +ILSVRC2012_val_00000761.JPEG:n03207743 +ILSVRC2012_val_00000762.JPEG:n02791270 +ILSVRC2012_val_00000763.JPEG:n02865351 +ILSVRC2012_val_00000764.JPEG:n03345487 +ILSVRC2012_val_00000765.JPEG:n03976467 +ILSVRC2012_val_00000766.JPEG:n03124043 +ILSVRC2012_val_00000767.JPEG:n04252225 +ILSVRC2012_val_00000768.JPEG:n02165105 +ILSVRC2012_val_00000769.JPEG:n03314780 +ILSVRC2012_val_00000770.JPEG:n04040759 +ILSVRC2012_val_00000771.JPEG:n02730930 +ILSVRC2012_val_00000772.JPEG:n02236044 +ILSVRC2012_val_00000773.JPEG:n07873807 +ILSVRC2012_val_00000774.JPEG:n02006656 +ILSVRC2012_val_00000775.JPEG:n02514041 +ILSVRC2012_val_00000776.JPEG:n03534580 +ILSVRC2012_val_00000777.JPEG:n03179701 +ILSVRC2012_val_00000778.JPEG:n04366367 +ILSVRC2012_val_00000779.JPEG:n02138441 +ILSVRC2012_val_00000780.JPEG:n03450230 +ILSVRC2012_val_00000781.JPEG:n01943899 +ILSVRC2012_val_00000782.JPEG:n07836838 +ILSVRC2012_val_00000783.JPEG:n03691459 +ILSVRC2012_val_00000784.JPEG:n04467665 +ILSVRC2012_val_00000785.JPEG:n02115641 +ILSVRC2012_val_00000786.JPEG:n01742172 +ILSVRC2012_val_00000787.JPEG:n02795169 +ILSVRC2012_val_00000788.JPEG:n02481823 +ILSVRC2012_val_00000789.JPEG:n07583066 +ILSVRC2012_val_00000790.JPEG:n02749479 +ILSVRC2012_val_00000791.JPEG:n01665541 +ILSVRC2012_val_00000792.JPEG:n04131690 +ILSVRC2012_val_00000793.JPEG:n03769881 +ILSVRC2012_val_00000794.JPEG:n02009229 +ILSVRC2012_val_00000795.JPEG:n04487081 +ILSVRC2012_val_00000796.JPEG:n02123159 +ILSVRC2012_val_00000797.JPEG:n04542943 +ILSVRC2012_val_00000798.JPEG:n07760859 +ILSVRC2012_val_00000799.JPEG:n02097658 +ILSVRC2012_val_00000800.JPEG:n02113799 +ILSVRC2012_val_00000801.JPEG:n07932039 +ILSVRC2012_val_00000802.JPEG:n02097474 +ILSVRC2012_val_00000803.JPEG:n03793489 +ILSVRC2012_val_00000804.JPEG:n02791124 +ILSVRC2012_val_00000805.JPEG:n04591713 +ILSVRC2012_val_00000806.JPEG:n01735189 +ILSVRC2012_val_00000807.JPEG:n01631663 +ILSVRC2012_val_00000808.JPEG:n02892767 +ILSVRC2012_val_00000809.JPEG:n04458633 +ILSVRC2012_val_00000810.JPEG:n02277742 +ILSVRC2012_val_00000811.JPEG:n07697537 +ILSVRC2012_val_00000812.JPEG:n03781244 +ILSVRC2012_val_00000813.JPEG:n02791270 +ILSVRC2012_val_00000814.JPEG:n03854065 +ILSVRC2012_val_00000815.JPEG:n04356056 +ILSVRC2012_val_00000816.JPEG:n07802026 +ILSVRC2012_val_00000817.JPEG:n03733131 +ILSVRC2012_val_00000818.JPEG:n01980166 +ILSVRC2012_val_00000819.JPEG:n02174001 +ILSVRC2012_val_00000820.JPEG:n07684084 +ILSVRC2012_val_00000821.JPEG:n01981276 +ILSVRC2012_val_00000822.JPEG:n03874293 +ILSVRC2012_val_00000823.JPEG:n03146219 +ILSVRC2012_val_00000824.JPEG:n02099267 +ILSVRC2012_val_00000825.JPEG:n02018207 +ILSVRC2012_val_00000826.JPEG:n04398044 +ILSVRC2012_val_00000827.JPEG:n03832673 +ILSVRC2012_val_00000828.JPEG:n02493509 +ILSVRC2012_val_00000829.JPEG:n03478589 +ILSVRC2012_val_00000830.JPEG:n06359193 +ILSVRC2012_val_00000831.JPEG:n02971356 +ILSVRC2012_val_00000832.JPEG:n02093754 +ILSVRC2012_val_00000833.JPEG:n04487081 +ILSVRC2012_val_00000834.JPEG:n03929855 +ILSVRC2012_val_00000835.JPEG:n03485407 +ILSVRC2012_val_00000836.JPEG:n01930112 +ILSVRC2012_val_00000837.JPEG:n01592084 +ILSVRC2012_val_00000838.JPEG:n02088238 +ILSVRC2012_val_00000839.JPEG:n04613696 +ILSVRC2012_val_00000840.JPEG:n03967562 +ILSVRC2012_val_00000841.JPEG:n03814639 +ILSVRC2012_val_00000842.JPEG:n04311174 +ILSVRC2012_val_00000843.JPEG:n04286575 +ILSVRC2012_val_00000844.JPEG:n03884397 +ILSVRC2012_val_00000845.JPEG:n03534580 +ILSVRC2012_val_00000846.JPEG:n03793489 +ILSVRC2012_val_00000847.JPEG:n02106382 +ILSVRC2012_val_00000848.JPEG:n03045698 +ILSVRC2012_val_00000849.JPEG:n03661043 +ILSVRC2012_val_00000850.JPEG:n03814906 +ILSVRC2012_val_00000851.JPEG:n02669723 +ILSVRC2012_val_00000852.JPEG:n03459775 +ILSVRC2012_val_00000853.JPEG:n03785016 +ILSVRC2012_val_00000854.JPEG:n04584207 +ILSVRC2012_val_00000855.JPEG:n03657121 +ILSVRC2012_val_00000856.JPEG:n03476991 +ILSVRC2012_val_00000857.JPEG:n04243546 +ILSVRC2012_val_00000858.JPEG:n04560804 +ILSVRC2012_val_00000859.JPEG:n03788365 +ILSVRC2012_val_00000860.JPEG:n01796340 +ILSVRC2012_val_00000861.JPEG:n04019541 +ILSVRC2012_val_00000862.JPEG:n03496892 +ILSVRC2012_val_00000863.JPEG:n07711569 +ILSVRC2012_val_00000864.JPEG:n03788195 +ILSVRC2012_val_00000865.JPEG:n02133161 +ILSVRC2012_val_00000866.JPEG:n04548362 +ILSVRC2012_val_00000867.JPEG:n02113712 +ILSVRC2012_val_00000868.JPEG:n03673027 +ILSVRC2012_val_00000869.JPEG:n12144580 +ILSVRC2012_val_00000870.JPEG:n02481823 +ILSVRC2012_val_00000871.JPEG:n02132136 +ILSVRC2012_val_00000872.JPEG:n03956157 +ILSVRC2012_val_00000873.JPEG:n01532829 +ILSVRC2012_val_00000874.JPEG:n04493381 +ILSVRC2012_val_00000875.JPEG:n02094258 +ILSVRC2012_val_00000876.JPEG:n03483316 +ILSVRC2012_val_00000877.JPEG:n01770081 +ILSVRC2012_val_00000878.JPEG:n02006656 +ILSVRC2012_val_00000879.JPEG:n02871525 +ILSVRC2012_val_00000880.JPEG:n01580077 +ILSVRC2012_val_00000881.JPEG:n07730033 +ILSVRC2012_val_00000882.JPEG:n02097474 +ILSVRC2012_val_00000883.JPEG:n02093647 +ILSVRC2012_val_00000884.JPEG:n02088466 +ILSVRC2012_val_00000885.JPEG:n01795545 +ILSVRC2012_val_00000886.JPEG:n07716906 +ILSVRC2012_val_00000887.JPEG:n03481172 +ILSVRC2012_val_00000888.JPEG:n01608432 +ILSVRC2012_val_00000889.JPEG:n02097209 +ILSVRC2012_val_00000890.JPEG:n01629819 +ILSVRC2012_val_00000891.JPEG:n07695742 +ILSVRC2012_val_00000892.JPEG:n02389026 +ILSVRC2012_val_00000893.JPEG:n02977058 +ILSVRC2012_val_00000894.JPEG:n04090263 +ILSVRC2012_val_00000895.JPEG:n04522168 +ILSVRC2012_val_00000896.JPEG:n02871525 +ILSVRC2012_val_00000897.JPEG:n04258138 +ILSVRC2012_val_00000898.JPEG:n02127052 +ILSVRC2012_val_00000899.JPEG:n04476259 +ILSVRC2012_val_00000900.JPEG:n03617480 +ILSVRC2012_val_00000901.JPEG:n04273569 +ILSVRC2012_val_00000902.JPEG:n03485794 +ILSVRC2012_val_00000903.JPEG:n06794110 +ILSVRC2012_val_00000904.JPEG:n03085013 +ILSVRC2012_val_00000905.JPEG:n02974003 +ILSVRC2012_val_00000906.JPEG:n02869837 +ILSVRC2012_val_00000907.JPEG:n02086240 +ILSVRC2012_val_00000908.JPEG:n01685808 +ILSVRC2012_val_00000909.JPEG:n02088466 +ILSVRC2012_val_00000910.JPEG:n03584829 +ILSVRC2012_val_00000911.JPEG:n01514668 +ILSVRC2012_val_00000912.JPEG:n02114367 +ILSVRC2012_val_00000913.JPEG:n03447447 +ILSVRC2012_val_00000914.JPEG:n04435653 +ILSVRC2012_val_00000915.JPEG:n03065424 +ILSVRC2012_val_00000916.JPEG:n01616318 +ILSVRC2012_val_00000917.JPEG:n02841315 +ILSVRC2012_val_00000918.JPEG:n02655020 +ILSVRC2012_val_00000919.JPEG:n03496892 +ILSVRC2012_val_00000920.JPEG:n04040759 +ILSVRC2012_val_00000921.JPEG:n01496331 +ILSVRC2012_val_00000922.JPEG:n02094258 +ILSVRC2012_val_00000923.JPEG:n03787032 +ILSVRC2012_val_00000924.JPEG:n02172182 +ILSVRC2012_val_00000925.JPEG:n01693334 +ILSVRC2012_val_00000926.JPEG:n02168699 +ILSVRC2012_val_00000927.JPEG:n03793489 +ILSVRC2012_val_00000928.JPEG:n07613480 +ILSVRC2012_val_00000929.JPEG:n01824575 +ILSVRC2012_val_00000930.JPEG:n01665541 +ILSVRC2012_val_00000931.JPEG:n04065272 +ILSVRC2012_val_00000932.JPEG:n02699494 +ILSVRC2012_val_00000933.JPEG:n02526121 +ILSVRC2012_val_00000934.JPEG:n01774750 +ILSVRC2012_val_00000935.JPEG:n03126707 +ILSVRC2012_val_00000936.JPEG:n04254777 +ILSVRC2012_val_00000937.JPEG:n02325366 +ILSVRC2012_val_00000938.JPEG:n01665541 +ILSVRC2012_val_00000939.JPEG:n02007558 +ILSVRC2012_val_00000940.JPEG:n01873310 +ILSVRC2012_val_00000941.JPEG:n01734418 +ILSVRC2012_val_00000942.JPEG:n03271574 +ILSVRC2012_val_00000943.JPEG:n01776313 +ILSVRC2012_val_00000944.JPEG:n01644373 +ILSVRC2012_val_00000945.JPEG:n02486410 +ILSVRC2012_val_00000946.JPEG:n02106662 +ILSVRC2012_val_00000947.JPEG:n03125729 +ILSVRC2012_val_00000948.JPEG:n02087394 +ILSVRC2012_val_00000949.JPEG:n02094433 +ILSVRC2012_val_00000950.JPEG:n07684084 +ILSVRC2012_val_00000951.JPEG:n04532670 +ILSVRC2012_val_00000952.JPEG:n01843383 +ILSVRC2012_val_00000953.JPEG:n02835271 +ILSVRC2012_val_00000954.JPEG:n12985857 +ILSVRC2012_val_00000955.JPEG:n04485082 +ILSVRC2012_val_00000956.JPEG:n02167151 +ILSVRC2012_val_00000957.JPEG:n03394916 +ILSVRC2012_val_00000958.JPEG:n01664065 +ILSVRC2012_val_00000959.JPEG:n04286575 +ILSVRC2012_val_00000960.JPEG:n03874293 +ILSVRC2012_val_00000961.JPEG:n02699494 +ILSVRC2012_val_00000962.JPEG:n01601694 +ILSVRC2012_val_00000963.JPEG:n01582220 +ILSVRC2012_val_00000964.JPEG:n02486261 +ILSVRC2012_val_00000965.JPEG:n02268853 +ILSVRC2012_val_00000966.JPEG:n03947888 +ILSVRC2012_val_00000967.JPEG:n13040303 +ILSVRC2012_val_00000968.JPEG:n03967562 +ILSVRC2012_val_00000969.JPEG:n03602883 +ILSVRC2012_val_00000970.JPEG:n01882714 +ILSVRC2012_val_00000971.JPEG:n04505470 +ILSVRC2012_val_00000972.JPEG:n02226429 +ILSVRC2012_val_00000973.JPEG:n04522168 +ILSVRC2012_val_00000974.JPEG:n02481823 +ILSVRC2012_val_00000975.JPEG:n02108422 +ILSVRC2012_val_00000976.JPEG:n03670208 +ILSVRC2012_val_00000977.JPEG:n07718747 +ILSVRC2012_val_00000978.JPEG:n01688243 +ILSVRC2012_val_00000979.JPEG:n02747177 +ILSVRC2012_val_00000980.JPEG:n07248320 +ILSVRC2012_val_00000981.JPEG:n02328150 +ILSVRC2012_val_00000982.JPEG:n02963159 +ILSVRC2012_val_00000983.JPEG:n02117135 +ILSVRC2012_val_00000984.JPEG:n03676483 +ILSVRC2012_val_00000985.JPEG:n06596364 +ILSVRC2012_val_00000986.JPEG:n01775062 +ILSVRC2012_val_00000987.JPEG:n03724870 +ILSVRC2012_val_00000988.JPEG:n03347037 +ILSVRC2012_val_00000989.JPEG:n13133613 +ILSVRC2012_val_00000990.JPEG:n02319095 +ILSVRC2012_val_00000991.JPEG:n03944341 +ILSVRC2012_val_00000992.JPEG:n02088238 +ILSVRC2012_val_00000993.JPEG:n02110185 +ILSVRC2012_val_00000994.JPEG:n01443537 +ILSVRC2012_val_00000995.JPEG:n06794110 +ILSVRC2012_val_00000996.JPEG:n02606052 +ILSVRC2012_val_00000997.JPEG:n02113186 +ILSVRC2012_val_00000998.JPEG:n02704792 +ILSVRC2012_val_00000999.JPEG:n03692522 +ILSVRC2012_val_00001000.JPEG:n03018349 +ILSVRC2012_val_00001001.JPEG:n02095314 +ILSVRC2012_val_00001002.JPEG:n04523525 +ILSVRC2012_val_00001003.JPEG:n02356798 +ILSVRC2012_val_00001004.JPEG:n04228054 +ILSVRC2012_val_00001005.JPEG:n02108000 +ILSVRC2012_val_00001006.JPEG:n04371430 +ILSVRC2012_val_00001007.JPEG:n01770393 +ILSVRC2012_val_00001008.JPEG:n04456115 +ILSVRC2012_val_00001009.JPEG:n02110958 +ILSVRC2012_val_00001010.JPEG:n01631663 +ILSVRC2012_val_00001011.JPEG:n02708093 +ILSVRC2012_val_00001012.JPEG:n02835271 +ILSVRC2012_val_00001013.JPEG:n02807133 +ILSVRC2012_val_00001014.JPEG:n02280649 +ILSVRC2012_val_00001015.JPEG:n02277742 +ILSVRC2012_val_00001016.JPEG:n03857828 +ILSVRC2012_val_00001017.JPEG:n03452741 +ILSVRC2012_val_00001018.JPEG:n03388043 +ILSVRC2012_val_00001019.JPEG:n06596364 +ILSVRC2012_val_00001020.JPEG:n04252225 +ILSVRC2012_val_00001021.JPEG:n04458633 +ILSVRC2012_val_00001022.JPEG:n01689811 +ILSVRC2012_val_00001023.JPEG:n03935335 +ILSVRC2012_val_00001024.JPEG:n01560419 +ILSVRC2012_val_00001025.JPEG:n02500267 +ILSVRC2012_val_00001026.JPEG:n02319095 +ILSVRC2012_val_00001027.JPEG:n02412080 +ILSVRC2012_val_00001028.JPEG:n02096437 +ILSVRC2012_val_00001029.JPEG:n03814639 +ILSVRC2012_val_00001030.JPEG:n03494278 +ILSVRC2012_val_00001031.JPEG:n01518878 +ILSVRC2012_val_00001032.JPEG:n02486261 +ILSVRC2012_val_00001033.JPEG:n01629819 +ILSVRC2012_val_00001034.JPEG:n04606251 +ILSVRC2012_val_00001035.JPEG:n03787032 +ILSVRC2012_val_00001036.JPEG:n01877812 +ILSVRC2012_val_00001037.JPEG:n01773157 +ILSVRC2012_val_00001038.JPEG:n02104365 +ILSVRC2012_val_00001039.JPEG:n02113978 +ILSVRC2012_val_00001040.JPEG:n02123394 +ILSVRC2012_val_00001041.JPEG:n02966687 +ILSVRC2012_val_00001042.JPEG:n01728920 +ILSVRC2012_val_00001043.JPEG:n02916936 +ILSVRC2012_val_00001044.JPEG:n01860187 +ILSVRC2012_val_00001045.JPEG:n03255030 +ILSVRC2012_val_00001046.JPEG:n02011460 +ILSVRC2012_val_00001047.JPEG:n02087394 +ILSVRC2012_val_00001048.JPEG:n02817516 +ILSVRC2012_val_00001049.JPEG:n02085620 +ILSVRC2012_val_00001050.JPEG:n02437616 +ILSVRC2012_val_00001051.JPEG:n02606052 +ILSVRC2012_val_00001052.JPEG:n03447721 +ILSVRC2012_val_00001053.JPEG:n01773157 +ILSVRC2012_val_00001054.JPEG:n02497673 +ILSVRC2012_val_00001055.JPEG:n04380533 +ILSVRC2012_val_00001056.JPEG:n02056570 +ILSVRC2012_val_00001057.JPEG:n01917289 +ILSVRC2012_val_00001058.JPEG:n12267677 +ILSVRC2012_val_00001059.JPEG:n04325704 +ILSVRC2012_val_00001060.JPEG:n02130308 +ILSVRC2012_val_00001061.JPEG:n02730930 +ILSVRC2012_val_00001062.JPEG:n03933933 +ILSVRC2012_val_00001063.JPEG:n02981792 +ILSVRC2012_val_00001064.JPEG:n07892512 +ILSVRC2012_val_00001065.JPEG:n02112018 +ILSVRC2012_val_00001066.JPEG:n02398521 +ILSVRC2012_val_00001067.JPEG:n02009912 +ILSVRC2012_val_00001068.JPEG:n02002724 +ILSVRC2012_val_00001069.JPEG:n02086079 +ILSVRC2012_val_00001070.JPEG:n02100236 +ILSVRC2012_val_00001071.JPEG:n03085013 +ILSVRC2012_val_00001072.JPEG:n02837789 +ILSVRC2012_val_00001073.JPEG:n02018795 +ILSVRC2012_val_00001074.JPEG:n02106382 +ILSVRC2012_val_00001075.JPEG:n02489166 +ILSVRC2012_val_00001076.JPEG:n03937543 +ILSVRC2012_val_00001077.JPEG:n02910353 +ILSVRC2012_val_00001078.JPEG:n07836838 +ILSVRC2012_val_00001079.JPEG:n15075141 +ILSVRC2012_val_00001080.JPEG:n02877765 +ILSVRC2012_val_00001081.JPEG:n03602883 +ILSVRC2012_val_00001082.JPEG:n02233338 +ILSVRC2012_val_00001083.JPEG:n13037406 +ILSVRC2012_val_00001084.JPEG:n01580077 +ILSVRC2012_val_00001085.JPEG:n04069434 +ILSVRC2012_val_00001086.JPEG:n04371774 +ILSVRC2012_val_00001087.JPEG:n03938244 +ILSVRC2012_val_00001088.JPEG:n02326432 +ILSVRC2012_val_00001089.JPEG:n03085013 +ILSVRC2012_val_00001090.JPEG:n02804610 +ILSVRC2012_val_00001091.JPEG:n04141975 +ILSVRC2012_val_00001092.JPEG:n02484975 +ILSVRC2012_val_00001093.JPEG:n02930766 +ILSVRC2012_val_00001094.JPEG:n03000134 +ILSVRC2012_val_00001095.JPEG:n02488702 +ILSVRC2012_val_00001096.JPEG:n02113023 +ILSVRC2012_val_00001097.JPEG:n02088632 +ILSVRC2012_val_00001098.JPEG:n02783161 +ILSVRC2012_val_00001099.JPEG:n02490219 +ILSVRC2012_val_00001100.JPEG:n04505470 +ILSVRC2012_val_00001101.JPEG:n02123394 +ILSVRC2012_val_00001102.JPEG:n04357314 +ILSVRC2012_val_00001103.JPEG:n02825657 +ILSVRC2012_val_00001104.JPEG:n02493509 +ILSVRC2012_val_00001105.JPEG:n03720891 +ILSVRC2012_val_00001106.JPEG:n03673027 +ILSVRC2012_val_00001107.JPEG:n03492542 +ILSVRC2012_val_00001108.JPEG:n01739381 +ILSVRC2012_val_00001109.JPEG:n02105056 +ILSVRC2012_val_00001110.JPEG:n03481172 +ILSVRC2012_val_00001111.JPEG:n03947888 +ILSVRC2012_val_00001112.JPEG:n02099601 +ILSVRC2012_val_00001113.JPEG:n02105505 +ILSVRC2012_val_00001114.JPEG:n01514859 +ILSVRC2012_val_00001115.JPEG:n07871810 +ILSVRC2012_val_00001116.JPEG:n03445924 +ILSVRC2012_val_00001117.JPEG:n12267677 +ILSVRC2012_val_00001118.JPEG:n04536866 +ILSVRC2012_val_00001119.JPEG:n03314780 +ILSVRC2012_val_00001120.JPEG:n12768682 +ILSVRC2012_val_00001121.JPEG:n02028035 +ILSVRC2012_val_00001122.JPEG:n01980166 +ILSVRC2012_val_00001123.JPEG:n02099601 +ILSVRC2012_val_00001124.JPEG:n01981276 +ILSVRC2012_val_00001125.JPEG:n07730033 +ILSVRC2012_val_00001126.JPEG:n02909870 +ILSVRC2012_val_00001127.JPEG:n04179913 +ILSVRC2012_val_00001128.JPEG:n02089973 +ILSVRC2012_val_00001129.JPEG:n02111277 +ILSVRC2012_val_00001130.JPEG:n12057211 +ILSVRC2012_val_00001131.JPEG:n01632458 +ILSVRC2012_val_00001132.JPEG:n02123394 +ILSVRC2012_val_00001133.JPEG:n04350905 +ILSVRC2012_val_00001134.JPEG:n03937543 +ILSVRC2012_val_00001135.JPEG:n02730930 +ILSVRC2012_val_00001136.JPEG:n01795545 +ILSVRC2012_val_00001137.JPEG:n02091244 +ILSVRC2012_val_00001138.JPEG:n01632777 +ILSVRC2012_val_00001139.JPEG:n03584829 +ILSVRC2012_val_00001140.JPEG:n03709823 +ILSVRC2012_val_00001141.JPEG:n02086646 +ILSVRC2012_val_00001142.JPEG:n01824575 +ILSVRC2012_val_00001143.JPEG:n03977966 +ILSVRC2012_val_00001144.JPEG:n03417042 +ILSVRC2012_val_00001145.JPEG:n02892201 +ILSVRC2012_val_00001146.JPEG:n01806143 +ILSVRC2012_val_00001147.JPEG:n02105855 +ILSVRC2012_val_00001148.JPEG:n02115913 +ILSVRC2012_val_00001149.JPEG:n03902125 +ILSVRC2012_val_00001150.JPEG:n01774384 +ILSVRC2012_val_00001151.JPEG:n07880968 +ILSVRC2012_val_00001152.JPEG:n02112137 +ILSVRC2012_val_00001153.JPEG:n09428293 +ILSVRC2012_val_00001154.JPEG:n04116512 +ILSVRC2012_val_00001155.JPEG:n02486410 +ILSVRC2012_val_00001156.JPEG:n03930630 +ILSVRC2012_val_00001157.JPEG:n04090263 +ILSVRC2012_val_00001158.JPEG:n01843383 +ILSVRC2012_val_00001159.JPEG:n07802026 +ILSVRC2012_val_00001160.JPEG:n04429376 +ILSVRC2012_val_00001161.JPEG:n02317335 +ILSVRC2012_val_00001162.JPEG:n02027492 +ILSVRC2012_val_00001163.JPEG:n01818515 +ILSVRC2012_val_00001164.JPEG:n02086646 +ILSVRC2012_val_00001165.JPEG:n02018207 +ILSVRC2012_val_00001166.JPEG:n04371430 +ILSVRC2012_val_00001167.JPEG:n03347037 +ILSVRC2012_val_00001168.JPEG:n03014705 +ILSVRC2012_val_00001169.JPEG:n04125021 +ILSVRC2012_val_00001170.JPEG:n03764736 +ILSVRC2012_val_00001171.JPEG:n02981792 +ILSVRC2012_val_00001172.JPEG:n02114367 +ILSVRC2012_val_00001173.JPEG:n04192698 +ILSVRC2012_val_00001174.JPEG:n04330267 +ILSVRC2012_val_00001175.JPEG:n03729826 +ILSVRC2012_val_00001176.JPEG:n02607072 +ILSVRC2012_val_00001177.JPEG:n02504458 +ILSVRC2012_val_00001178.JPEG:n03769881 +ILSVRC2012_val_00001179.JPEG:n02018207 +ILSVRC2012_val_00001180.JPEG:n03929855 +ILSVRC2012_val_00001181.JPEG:n04591157 +ILSVRC2012_val_00001182.JPEG:n03947888 +ILSVRC2012_val_00001183.JPEG:n04317175 +ILSVRC2012_val_00001184.JPEG:n03125729 +ILSVRC2012_val_00001185.JPEG:n01749939 +ILSVRC2012_val_00001186.JPEG:n04399382 +ILSVRC2012_val_00001187.JPEG:n02276258 +ILSVRC2012_val_00001188.JPEG:n03598930 +ILSVRC2012_val_00001189.JPEG:n02606052 +ILSVRC2012_val_00001190.JPEG:n03089624 +ILSVRC2012_val_00001191.JPEG:n02099601 +ILSVRC2012_val_00001192.JPEG:n03770439 +ILSVRC2012_val_00001193.JPEG:n02655020 +ILSVRC2012_val_00001194.JPEG:n07745940 +ILSVRC2012_val_00001195.JPEG:n02095314 +ILSVRC2012_val_00001196.JPEG:n04336792 +ILSVRC2012_val_00001197.JPEG:n04033995 +ILSVRC2012_val_00001198.JPEG:n02112018 +ILSVRC2012_val_00001199.JPEG:n02132136 +ILSVRC2012_val_00001200.JPEG:n02860847 +ILSVRC2012_val_00001201.JPEG:n03100240 +ILSVRC2012_val_00001202.JPEG:n02966687 +ILSVRC2012_val_00001203.JPEG:n02111129 +ILSVRC2012_val_00001204.JPEG:n04273569 +ILSVRC2012_val_00001205.JPEG:n04149813 +ILSVRC2012_val_00001206.JPEG:n02092002 +ILSVRC2012_val_00001207.JPEG:n03769881 +ILSVRC2012_val_00001208.JPEG:n04599235 +ILSVRC2012_val_00001209.JPEG:n03825788 +ILSVRC2012_val_00001210.JPEG:n04118776 +ILSVRC2012_val_00001211.JPEG:n04336792 +ILSVRC2012_val_00001212.JPEG:n02115641 +ILSVRC2012_val_00001213.JPEG:n01622779 +ILSVRC2012_val_00001214.JPEG:n02909870 +ILSVRC2012_val_00001215.JPEG:n02276258 +ILSVRC2012_val_00001216.JPEG:n02977058 +ILSVRC2012_val_00001217.JPEG:n02326432 +ILSVRC2012_val_00001218.JPEG:n01608432 +ILSVRC2012_val_00001219.JPEG:n03347037 +ILSVRC2012_val_00001220.JPEG:n02978881 +ILSVRC2012_val_00001221.JPEG:n02787622 +ILSVRC2012_val_00001222.JPEG:n02093256 +ILSVRC2012_val_00001223.JPEG:n02101556 +ILSVRC2012_val_00001224.JPEG:n02100735 +ILSVRC2012_val_00001225.JPEG:n02085782 +ILSVRC2012_val_00001226.JPEG:n02342885 +ILSVRC2012_val_00001227.JPEG:n03733281 +ILSVRC2012_val_00001228.JPEG:n02085782 +ILSVRC2012_val_00001229.JPEG:n03706229 +ILSVRC2012_val_00001230.JPEG:n02002724 +ILSVRC2012_val_00001231.JPEG:n13037406 +ILSVRC2012_val_00001232.JPEG:n02422106 +ILSVRC2012_val_00001233.JPEG:n07614500 +ILSVRC2012_val_00001234.JPEG:n02113712 +ILSVRC2012_val_00001235.JPEG:n04336792 +ILSVRC2012_val_00001236.JPEG:n02486261 +ILSVRC2012_val_00001237.JPEG:n02356798 +ILSVRC2012_val_00001238.JPEG:n02268443 +ILSVRC2012_val_00001239.JPEG:n04179913 +ILSVRC2012_val_00001240.JPEG:n04277352 +ILSVRC2012_val_00001241.JPEG:n02346627 +ILSVRC2012_val_00001242.JPEG:n03089624 +ILSVRC2012_val_00001243.JPEG:n02835271 +ILSVRC2012_val_00001244.JPEG:n02086240 +ILSVRC2012_val_00001245.JPEG:n04579432 +ILSVRC2012_val_00001246.JPEG:n03180011 +ILSVRC2012_val_00001247.JPEG:n04285008 +ILSVRC2012_val_00001248.JPEG:n02408429 +ILSVRC2012_val_00001249.JPEG:n04392985 +ILSVRC2012_val_00001250.JPEG:n02091244 +ILSVRC2012_val_00001251.JPEG:n02815834 +ILSVRC2012_val_00001252.JPEG:n02834397 +ILSVRC2012_val_00001253.JPEG:n04009552 +ILSVRC2012_val_00001254.JPEG:n02488291 +ILSVRC2012_val_00001255.JPEG:n03290653 +ILSVRC2012_val_00001256.JPEG:n03325584 +ILSVRC2012_val_00001257.JPEG:n03637318 +ILSVRC2012_val_00001258.JPEG:n02730930 +ILSVRC2012_val_00001259.JPEG:n02865351 +ILSVRC2012_val_00001260.JPEG:n02119789 +ILSVRC2012_val_00001261.JPEG:n03929855 +ILSVRC2012_val_00001262.JPEG:n03676483 +ILSVRC2012_val_00001263.JPEG:n04423845 +ILSVRC2012_val_00001264.JPEG:n03874293 +ILSVRC2012_val_00001265.JPEG:n03908618 +ILSVRC2012_val_00001266.JPEG:n03598930 +ILSVRC2012_val_00001267.JPEG:n02090379 +ILSVRC2012_val_00001268.JPEG:n01944390 +ILSVRC2012_val_00001269.JPEG:n04152593 +ILSVRC2012_val_00001270.JPEG:n09288635 +ILSVRC2012_val_00001271.JPEG:n02066245 +ILSVRC2012_val_00001272.JPEG:n01768244 +ILSVRC2012_val_00001273.JPEG:n03272010 +ILSVRC2012_val_00001274.JPEG:n01531178 +ILSVRC2012_val_00001275.JPEG:n03255030 +ILSVRC2012_val_00001276.JPEG:n03676483 +ILSVRC2012_val_00001277.JPEG:n02002556 +ILSVRC2012_val_00001278.JPEG:n02749479 +ILSVRC2012_val_00001279.JPEG:n02415577 +ILSVRC2012_val_00001280.JPEG:n02403003 +ILSVRC2012_val_00001281.JPEG:n07565083 +ILSVRC2012_val_00001282.JPEG:n02981792 +ILSVRC2012_val_00001283.JPEG:n01776313 +ILSVRC2012_val_00001284.JPEG:n02097474 +ILSVRC2012_val_00001285.JPEG:n02667093 +ILSVRC2012_val_00001286.JPEG:n02096177 +ILSVRC2012_val_00001287.JPEG:n03255030 +ILSVRC2012_val_00001288.JPEG:n01819313 +ILSVRC2012_val_00001289.JPEG:n02791124 +ILSVRC2012_val_00001290.JPEG:n02279972 +ILSVRC2012_val_00001291.JPEG:n04090263 +ILSVRC2012_val_00001292.JPEG:n09193705 +ILSVRC2012_val_00001293.JPEG:n04335435 +ILSVRC2012_val_00001294.JPEG:n03733131 +ILSVRC2012_val_00001295.JPEG:n03250847 +ILSVRC2012_val_00001296.JPEG:n04263257 +ILSVRC2012_val_00001297.JPEG:n02096585 +ILSVRC2012_val_00001298.JPEG:n03976467 +ILSVRC2012_val_00001299.JPEG:n02963159 +ILSVRC2012_val_00001300.JPEG:n04613696 +ILSVRC2012_val_00001301.JPEG:n04310018 +ILSVRC2012_val_00001302.JPEG:n02107574 +ILSVRC2012_val_00001303.JPEG:n03724870 +ILSVRC2012_val_00001304.JPEG:n09428293 +ILSVRC2012_val_00001305.JPEG:n02101006 +ILSVRC2012_val_00001306.JPEG:n04372370 +ILSVRC2012_val_00001307.JPEG:n03930630 +ILSVRC2012_val_00001308.JPEG:n07584110 +ILSVRC2012_val_00001309.JPEG:n01735189 +ILSVRC2012_val_00001310.JPEG:n04599235 +ILSVRC2012_val_00001311.JPEG:n02835271 +ILSVRC2012_val_00001312.JPEG:n04330267 +ILSVRC2012_val_00001313.JPEG:n02108915 +ILSVRC2012_val_00001314.JPEG:n02110185 +ILSVRC2012_val_00001315.JPEG:n07684084 +ILSVRC2012_val_00001316.JPEG:n04204347 +ILSVRC2012_val_00001317.JPEG:n02672831 +ILSVRC2012_val_00001318.JPEG:n03742115 +ILSVRC2012_val_00001319.JPEG:n04131690 +ILSVRC2012_val_00001320.JPEG:n09428293 +ILSVRC2012_val_00001321.JPEG:n04487394 +ILSVRC2012_val_00001322.JPEG:n03710193 +ILSVRC2012_val_00001323.JPEG:n09332890 +ILSVRC2012_val_00001324.JPEG:n03478589 +ILSVRC2012_val_00001325.JPEG:n04486054 +ILSVRC2012_val_00001326.JPEG:n02951358 +ILSVRC2012_val_00001327.JPEG:n09428293 +ILSVRC2012_val_00001328.JPEG:n04596742 +ILSVRC2012_val_00001329.JPEG:n01872401 +ILSVRC2012_val_00001330.JPEG:n04505470 +ILSVRC2012_val_00001331.JPEG:n04154565 +ILSVRC2012_val_00001332.JPEG:n02666196 +ILSVRC2012_val_00001333.JPEG:n02437616 +ILSVRC2012_val_00001334.JPEG:n03724870 +ILSVRC2012_val_00001335.JPEG:n02120079 +ILSVRC2012_val_00001336.JPEG:n01828970 +ILSVRC2012_val_00001337.JPEG:n03141823 +ILSVRC2012_val_00001338.JPEG:n01698640 +ILSVRC2012_val_00001339.JPEG:n03095699 +ILSVRC2012_val_00001340.JPEG:n04099969 +ILSVRC2012_val_00001341.JPEG:n02123045 +ILSVRC2012_val_00001342.JPEG:n04482393 +ILSVRC2012_val_00001343.JPEG:n04026417 +ILSVRC2012_val_00001344.JPEG:n02110806 +ILSVRC2012_val_00001345.JPEG:n04033901 +ILSVRC2012_val_00001346.JPEG:n04041544 +ILSVRC2012_val_00001347.JPEG:n02869837 +ILSVRC2012_val_00001348.JPEG:n04136333 +ILSVRC2012_val_00001349.JPEG:n02112350 +ILSVRC2012_val_00001350.JPEG:n03388043 +ILSVRC2012_val_00001351.JPEG:n03065424 +ILSVRC2012_val_00001352.JPEG:n02128757 +ILSVRC2012_val_00001353.JPEG:n04330267 +ILSVRC2012_val_00001354.JPEG:n02879718 +ILSVRC2012_val_00001355.JPEG:n02859443 +ILSVRC2012_val_00001356.JPEG:n01968897 +ILSVRC2012_val_00001357.JPEG:n01847000 +ILSVRC2012_val_00001358.JPEG:n01871265 +ILSVRC2012_val_00001359.JPEG:n02129165 +ILSVRC2012_val_00001360.JPEG:n02408429 +ILSVRC2012_val_00001361.JPEG:n04263257 +ILSVRC2012_val_00001362.JPEG:n13054560 +ILSVRC2012_val_00001363.JPEG:n02090379 +ILSVRC2012_val_00001364.JPEG:n04553703 +ILSVRC2012_val_00001365.JPEG:n03929660 +ILSVRC2012_val_00001366.JPEG:n01990800 +ILSVRC2012_val_00001367.JPEG:n03494278 +ILSVRC2012_val_00001368.JPEG:n01514859 +ILSVRC2012_val_00001369.JPEG:n02804610 +ILSVRC2012_val_00001370.JPEG:n01773157 +ILSVRC2012_val_00001371.JPEG:n02087046 +ILSVRC2012_val_00001372.JPEG:n07802026 +ILSVRC2012_val_00001373.JPEG:n03777754 +ILSVRC2012_val_00001374.JPEG:n07720875 +ILSVRC2012_val_00001375.JPEG:n01694178 +ILSVRC2012_val_00001376.JPEG:n06794110 +ILSVRC2012_val_00001377.JPEG:n02795169 +ILSVRC2012_val_00001378.JPEG:n07583066 +ILSVRC2012_val_00001379.JPEG:n02094114 +ILSVRC2012_val_00001380.JPEG:n03841143 +ILSVRC2012_val_00001381.JPEG:n01985128 +ILSVRC2012_val_00001382.JPEG:n03776460 +ILSVRC2012_val_00001383.JPEG:n02859443 +ILSVRC2012_val_00001384.JPEG:n02808304 +ILSVRC2012_val_00001385.JPEG:n02092339 +ILSVRC2012_val_00001386.JPEG:n02441942 +ILSVRC2012_val_00001387.JPEG:n02002724 +ILSVRC2012_val_00001388.JPEG:n04296562 +ILSVRC2012_val_00001389.JPEG:n02086910 +ILSVRC2012_val_00001390.JPEG:n02690373 +ILSVRC2012_val_00001391.JPEG:n01616318 +ILSVRC2012_val_00001392.JPEG:n07718472 +ILSVRC2012_val_00001393.JPEG:n02086240 +ILSVRC2012_val_00001394.JPEG:n04049303 +ILSVRC2012_val_00001395.JPEG:n04235860 +ILSVRC2012_val_00001396.JPEG:n06359193 +ILSVRC2012_val_00001397.JPEG:n02110958 +ILSVRC2012_val_00001398.JPEG:n01518878 +ILSVRC2012_val_00001399.JPEG:n02950826 +ILSVRC2012_val_00001400.JPEG:n03447721 +ILSVRC2012_val_00001401.JPEG:n02111129 +ILSVRC2012_val_00001402.JPEG:n04517823 +ILSVRC2012_val_00001403.JPEG:n03769881 +ILSVRC2012_val_00001404.JPEG:n02112350 +ILSVRC2012_val_00001405.JPEG:n07693725 +ILSVRC2012_val_00001406.JPEG:n07747607 +ILSVRC2012_val_00001407.JPEG:n02444819 +ILSVRC2012_val_00001408.JPEG:n02109047 +ILSVRC2012_val_00001409.JPEG:n04485082 +ILSVRC2012_val_00001410.JPEG:n10148035 +ILSVRC2012_val_00001411.JPEG:n03127925 +ILSVRC2012_val_00001412.JPEG:n04328186 +ILSVRC2012_val_00001413.JPEG:n03347037 +ILSVRC2012_val_00001414.JPEG:n02102480 +ILSVRC2012_val_00001415.JPEG:n07614500 +ILSVRC2012_val_00001416.JPEG:n02676566 +ILSVRC2012_val_00001417.JPEG:n04599235 +ILSVRC2012_val_00001418.JPEG:n03534580 +ILSVRC2012_val_00001419.JPEG:n02093256 +ILSVRC2012_val_00001420.JPEG:n03710721 +ILSVRC2012_val_00001421.JPEG:n02167151 +ILSVRC2012_val_00001422.JPEG:n04116512 +ILSVRC2012_val_00001423.JPEG:n04141975 +ILSVRC2012_val_00001424.JPEG:n03877472 +ILSVRC2012_val_00001425.JPEG:n02092339 +ILSVRC2012_val_00001426.JPEG:n03042490 +ILSVRC2012_val_00001427.JPEG:n04604644 +ILSVRC2012_val_00001428.JPEG:n03355925 +ILSVRC2012_val_00001429.JPEG:n04009552 +ILSVRC2012_val_00001430.JPEG:n03598930 +ILSVRC2012_val_00001431.JPEG:n02672831 +ILSVRC2012_val_00001432.JPEG:n03425413 +ILSVRC2012_val_00001433.JPEG:n03649909 +ILSVRC2012_val_00001434.JPEG:n02099429 +ILSVRC2012_val_00001435.JPEG:n01819313 +ILSVRC2012_val_00001436.JPEG:n02640242 +ILSVRC2012_val_00001437.JPEG:n02978881 +ILSVRC2012_val_00001438.JPEG:n03670208 +ILSVRC2012_val_00001439.JPEG:n02342885 +ILSVRC2012_val_00001440.JPEG:n03888257 +ILSVRC2012_val_00001441.JPEG:n03729826 +ILSVRC2012_val_00001442.JPEG:n02457408 +ILSVRC2012_val_00001443.JPEG:n02860847 +ILSVRC2012_val_00001444.JPEG:n09246464 +ILSVRC2012_val_00001445.JPEG:n02097298 +ILSVRC2012_val_00001446.JPEG:n03649909 +ILSVRC2012_val_00001447.JPEG:n04228054 +ILSVRC2012_val_00001448.JPEG:n02113624 +ILSVRC2012_val_00001449.JPEG:n01978287 +ILSVRC2012_val_00001450.JPEG:n03895866 +ILSVRC2012_val_00001451.JPEG:n03393912 +ILSVRC2012_val_00001452.JPEG:n03127925 +ILSVRC2012_val_00001453.JPEG:n03720891 +ILSVRC2012_val_00001454.JPEG:n01774384 +ILSVRC2012_val_00001455.JPEG:n04065272 +ILSVRC2012_val_00001456.JPEG:n03485407 +ILSVRC2012_val_00001457.JPEG:n04033901 +ILSVRC2012_val_00001458.JPEG:n02488291 +ILSVRC2012_val_00001459.JPEG:n12057211 +ILSVRC2012_val_00001460.JPEG:n01774750 +ILSVRC2012_val_00001461.JPEG:n01798484 +ILSVRC2012_val_00001462.JPEG:n01537544 +ILSVRC2012_val_00001463.JPEG:n07720875 +ILSVRC2012_val_00001464.JPEG:n03838899 +ILSVRC2012_val_00001465.JPEG:n04120489 +ILSVRC2012_val_00001466.JPEG:n02264363 +ILSVRC2012_val_00001467.JPEG:n02113978 +ILSVRC2012_val_00001468.JPEG:n02799071 +ILSVRC2012_val_00001469.JPEG:n02114367 +ILSVRC2012_val_00001470.JPEG:n04332243 +ILSVRC2012_val_00001471.JPEG:n03062245 +ILSVRC2012_val_00001472.JPEG:n02077923 +ILSVRC2012_val_00001473.JPEG:n02398521 +ILSVRC2012_val_00001474.JPEG:n04435653 +ILSVRC2012_val_00001475.JPEG:n01692333 +ILSVRC2012_val_00001476.JPEG:n07831146 +ILSVRC2012_val_00001477.JPEG:n04523525 +ILSVRC2012_val_00001478.JPEG:n02342885 +ILSVRC2012_val_00001479.JPEG:n07753275 +ILSVRC2012_val_00001480.JPEG:n01807496 +ILSVRC2012_val_00001481.JPEG:n02098413 +ILSVRC2012_val_00001482.JPEG:n01744401 +ILSVRC2012_val_00001483.JPEG:n07836838 +ILSVRC2012_val_00001484.JPEG:n02104029 +ILSVRC2012_val_00001485.JPEG:n02092339 +ILSVRC2012_val_00001486.JPEG:n02092339 +ILSVRC2012_val_00001487.JPEG:n02115913 +ILSVRC2012_val_00001488.JPEG:n01608432 +ILSVRC2012_val_00001489.JPEG:n03325584 +ILSVRC2012_val_00001490.JPEG:n02066245 +ILSVRC2012_val_00001491.JPEG:n03345487 +ILSVRC2012_val_00001492.JPEG:n03394916 +ILSVRC2012_val_00001493.JPEG:n01773797 +ILSVRC2012_val_00001494.JPEG:n02113186 +ILSVRC2012_val_00001495.JPEG:n02667093 +ILSVRC2012_val_00001496.JPEG:n02124075 +ILSVRC2012_val_00001497.JPEG:n04118538 +ILSVRC2012_val_00001498.JPEG:n02134084 +ILSVRC2012_val_00001499.JPEG:n02317335 +ILSVRC2012_val_00001500.JPEG:n03047690 +ILSVRC2012_val_00001501.JPEG:n03938244 +ILSVRC2012_val_00001502.JPEG:n02219486 +ILSVRC2012_val_00001503.JPEG:n07718747 +ILSVRC2012_val_00001504.JPEG:n02490219 +ILSVRC2012_val_00001505.JPEG:n04326547 +ILSVRC2012_val_00001506.JPEG:n02690373 +ILSVRC2012_val_00001507.JPEG:n07717556 +ILSVRC2012_val_00001508.JPEG:n01580077 +ILSVRC2012_val_00001509.JPEG:n02443484 +ILSVRC2012_val_00001510.JPEG:n04443257 +ILSVRC2012_val_00001511.JPEG:n04033995 +ILSVRC2012_val_00001512.JPEG:n07590611 +ILSVRC2012_val_00001513.JPEG:n02403003 +ILSVRC2012_val_00001514.JPEG:n07768694 +ILSVRC2012_val_00001515.JPEG:n03803284 +ILSVRC2012_val_00001516.JPEG:n04371774 +ILSVRC2012_val_00001517.JPEG:n02802426 +ILSVRC2012_val_00001518.JPEG:n06794110 +ILSVRC2012_val_00001519.JPEG:n04483307 +ILSVRC2012_val_00001520.JPEG:n02791270 +ILSVRC2012_val_00001521.JPEG:n02028035 +ILSVRC2012_val_00001522.JPEG:n03764736 +ILSVRC2012_val_00001523.JPEG:n07860988 +ILSVRC2012_val_00001524.JPEG:n09421951 +ILSVRC2012_val_00001525.JPEG:n03773504 +ILSVRC2012_val_00001526.JPEG:n04152593 +ILSVRC2012_val_00001527.JPEG:n04367480 +ILSVRC2012_val_00001528.JPEG:n02950826 +ILSVRC2012_val_00001529.JPEG:n02168699 +ILSVRC2012_val_00001530.JPEG:n04458633 +ILSVRC2012_val_00001531.JPEG:n01983481 +ILSVRC2012_val_00001532.JPEG:n04404412 +ILSVRC2012_val_00001533.JPEG:n04252225 +ILSVRC2012_val_00001534.JPEG:n04596742 +ILSVRC2012_val_00001535.JPEG:n02480495 +ILSVRC2012_val_00001536.JPEG:n02281787 +ILSVRC2012_val_00001537.JPEG:n01795545 +ILSVRC2012_val_00001538.JPEG:n02089867 +ILSVRC2012_val_00001539.JPEG:n02169497 +ILSVRC2012_val_00001540.JPEG:n02666196 +ILSVRC2012_val_00001541.JPEG:n04311004 +ILSVRC2012_val_00001542.JPEG:n02879718 +ILSVRC2012_val_00001543.JPEG:n03457902 +ILSVRC2012_val_00001544.JPEG:n02074367 +ILSVRC2012_val_00001545.JPEG:n03297495 +ILSVRC2012_val_00001546.JPEG:n02481823 +ILSVRC2012_val_00001547.JPEG:n04485082 +ILSVRC2012_val_00001548.JPEG:n02091244 +ILSVRC2012_val_00001549.JPEG:n07718747 +ILSVRC2012_val_00001550.JPEG:n02102480 +ILSVRC2012_val_00001551.JPEG:n04147183 +ILSVRC2012_val_00001552.JPEG:n03014705 +ILSVRC2012_val_00001553.JPEG:n02814860 +ILSVRC2012_val_00001554.JPEG:n04532670 +ILSVRC2012_val_00001555.JPEG:n02094114 +ILSVRC2012_val_00001556.JPEG:n01532829 +ILSVRC2012_val_00001557.JPEG:n01664065 +ILSVRC2012_val_00001558.JPEG:n04090263 +ILSVRC2012_val_00001559.JPEG:n03995372 +ILSVRC2012_val_00001560.JPEG:n03134739 +ILSVRC2012_val_00001561.JPEG:n06596364 +ILSVRC2012_val_00001562.JPEG:n03710637 +ILSVRC2012_val_00001563.JPEG:n01807496 +ILSVRC2012_val_00001564.JPEG:n02096294 +ILSVRC2012_val_00001565.JPEG:n04026417 +ILSVRC2012_val_00001566.JPEG:n02165105 +ILSVRC2012_val_00001567.JPEG:n03998194 +ILSVRC2012_val_00001568.JPEG:n02112706 +ILSVRC2012_val_00001569.JPEG:n04366367 +ILSVRC2012_val_00001570.JPEG:n02177972 +ILSVRC2012_val_00001571.JPEG:n04152593 +ILSVRC2012_val_00001572.JPEG:n04442312 +ILSVRC2012_val_00001573.JPEG:n01697457 +ILSVRC2012_val_00001574.JPEG:n03775071 +ILSVRC2012_val_00001575.JPEG:n07892512 +ILSVRC2012_val_00001576.JPEG:n02091831 +ILSVRC2012_val_00001577.JPEG:n02101388 +ILSVRC2012_val_00001578.JPEG:n01749939 +ILSVRC2012_val_00001579.JPEG:n03384352 +ILSVRC2012_val_00001580.JPEG:n02484975 +ILSVRC2012_val_00001581.JPEG:n03868242 +ILSVRC2012_val_00001582.JPEG:n01753488 +ILSVRC2012_val_00001583.JPEG:n02687172 +ILSVRC2012_val_00001584.JPEG:n02807133 +ILSVRC2012_val_00001585.JPEG:n02231487 +ILSVRC2012_val_00001586.JPEG:n02018795 +ILSVRC2012_val_00001587.JPEG:n04270147 +ILSVRC2012_val_00001588.JPEG:n03063599 +ILSVRC2012_val_00001589.JPEG:n04591713 +ILSVRC2012_val_00001590.JPEG:n03895866 +ILSVRC2012_val_00001591.JPEG:n03481172 +ILSVRC2012_val_00001592.JPEG:n04456115 +ILSVRC2012_val_00001593.JPEG:n01755581 +ILSVRC2012_val_00001594.JPEG:n02319095 +ILSVRC2012_val_00001595.JPEG:n02526121 +ILSVRC2012_val_00001596.JPEG:n01796340 +ILSVRC2012_val_00001597.JPEG:n02094433 +ILSVRC2012_val_00001598.JPEG:n01558993 +ILSVRC2012_val_00001599.JPEG:n04238763 +ILSVRC2012_val_00001600.JPEG:n03127925 +ILSVRC2012_val_00001601.JPEG:n03017168 +ILSVRC2012_val_00001602.JPEG:n02692877 +ILSVRC2012_val_00001603.JPEG:n04179913 +ILSVRC2012_val_00001604.JPEG:n02791124 +ILSVRC2012_val_00001605.JPEG:n03494278 +ILSVRC2012_val_00001606.JPEG:n06596364 +ILSVRC2012_val_00001607.JPEG:n01751748 +ILSVRC2012_val_00001608.JPEG:n02074367 +ILSVRC2012_val_00001609.JPEG:n03249569 +ILSVRC2012_val_00001610.JPEG:n04357314 +ILSVRC2012_val_00001611.JPEG:n07579787 +ILSVRC2012_val_00001612.JPEG:n04550184 +ILSVRC2012_val_00001613.JPEG:n06596364 +ILSVRC2012_val_00001614.JPEG:n03761084 +ILSVRC2012_val_00001615.JPEG:n07718472 +ILSVRC2012_val_00001616.JPEG:n03376595 +ILSVRC2012_val_00001617.JPEG:n04428191 +ILSVRC2012_val_00001618.JPEG:n01773157 +ILSVRC2012_val_00001619.JPEG:n07248320 +ILSVRC2012_val_00001620.JPEG:n03400231 +ILSVRC2012_val_00001621.JPEG:n04447861 +ILSVRC2012_val_00001622.JPEG:n03854065 +ILSVRC2012_val_00001623.JPEG:n01694178 +ILSVRC2012_val_00001624.JPEG:n02111500 +ILSVRC2012_val_00001625.JPEG:n04111531 +ILSVRC2012_val_00001626.JPEG:n02090622 +ILSVRC2012_val_00001627.JPEG:n03450230 +ILSVRC2012_val_00001628.JPEG:n04536866 +ILSVRC2012_val_00001629.JPEG:n01817953 +ILSVRC2012_val_00001630.JPEG:n02843684 +ILSVRC2012_val_00001631.JPEG:n03776460 +ILSVRC2012_val_00001632.JPEG:n04201297 +ILSVRC2012_val_00001633.JPEG:n04204238 +ILSVRC2012_val_00001634.JPEG:n02094114 +ILSVRC2012_val_00001635.JPEG:n04238763 +ILSVRC2012_val_00001636.JPEG:n01667114 +ILSVRC2012_val_00001637.JPEG:n02116738 +ILSVRC2012_val_00001638.JPEG:n03709823 +ILSVRC2012_val_00001639.JPEG:n04153751 +ILSVRC2012_val_00001640.JPEG:n02422699 +ILSVRC2012_val_00001641.JPEG:n01796340 +ILSVRC2012_val_00001642.JPEG:n07836838 +ILSVRC2012_val_00001643.JPEG:n02027492 +ILSVRC2012_val_00001644.JPEG:n03478589 +ILSVRC2012_val_00001645.JPEG:n01689811 +ILSVRC2012_val_00001646.JPEG:n02110958 +ILSVRC2012_val_00001647.JPEG:n03538406 +ILSVRC2012_val_00001648.JPEG:n03207743 +ILSVRC2012_val_00001649.JPEG:n01669191 +ILSVRC2012_val_00001650.JPEG:n06794110 +ILSVRC2012_val_00001651.JPEG:n02087394 +ILSVRC2012_val_00001652.JPEG:n01641577 +ILSVRC2012_val_00001653.JPEG:n07873807 +ILSVRC2012_val_00001654.JPEG:n03314780 +ILSVRC2012_val_00001655.JPEG:n04591157 +ILSVRC2012_val_00001656.JPEG:n02487347 +ILSVRC2012_val_00001657.JPEG:n04277352 +ILSVRC2012_val_00001658.JPEG:n07749582 +ILSVRC2012_val_00001659.JPEG:n03792782 +ILSVRC2012_val_00001660.JPEG:n03947888 +ILSVRC2012_val_00001661.JPEG:n03792782 +ILSVRC2012_val_00001662.JPEG:n01669191 +ILSVRC2012_val_00001663.JPEG:n02102318 +ILSVRC2012_val_00001664.JPEG:n03788365 +ILSVRC2012_val_00001665.JPEG:n03899768 +ILSVRC2012_val_00001666.JPEG:n04392985 +ILSVRC2012_val_00001667.JPEG:n01629819 +ILSVRC2012_val_00001668.JPEG:n04557648 +ILSVRC2012_val_00001669.JPEG:n02640242 +ILSVRC2012_val_00001670.JPEG:n02325366 +ILSVRC2012_val_00001671.JPEG:n07749582 +ILSVRC2012_val_00001672.JPEG:n04264628 +ILSVRC2012_val_00001673.JPEG:n04487081 +ILSVRC2012_val_00001674.JPEG:n02978881 +ILSVRC2012_val_00001675.JPEG:n03720891 +ILSVRC2012_val_00001676.JPEG:n01494475 +ILSVRC2012_val_00001677.JPEG:n02951358 +ILSVRC2012_val_00001678.JPEG:n01828970 +ILSVRC2012_val_00001679.JPEG:n04286575 +ILSVRC2012_val_00001680.JPEG:n04540053 +ILSVRC2012_val_00001681.JPEG:n04332243 +ILSVRC2012_val_00001682.JPEG:n04367480 +ILSVRC2012_val_00001683.JPEG:n03840681 +ILSVRC2012_val_00001684.JPEG:n02106662 +ILSVRC2012_val_00001685.JPEG:n03376595 +ILSVRC2012_val_00001686.JPEG:n02113186 +ILSVRC2012_val_00001687.JPEG:n03085013 +ILSVRC2012_val_00001688.JPEG:n09246464 +ILSVRC2012_val_00001689.JPEG:n03127747 +ILSVRC2012_val_00001690.JPEG:n04367480 +ILSVRC2012_val_00001691.JPEG:n03290653 +ILSVRC2012_val_00001692.JPEG:n07760859 +ILSVRC2012_val_00001693.JPEG:n02102973 +ILSVRC2012_val_00001694.JPEG:n03290653 +ILSVRC2012_val_00001695.JPEG:n01751748 +ILSVRC2012_val_00001696.JPEG:n02089973 +ILSVRC2012_val_00001697.JPEG:n02086910 +ILSVRC2012_val_00001698.JPEG:n02112350 +ILSVRC2012_val_00001699.JPEG:n03272562 +ILSVRC2012_val_00001700.JPEG:n04456115 +ILSVRC2012_val_00001701.JPEG:n03785016 +ILSVRC2012_val_00001702.JPEG:n02110341 +ILSVRC2012_val_00001703.JPEG:n01728920 +ILSVRC2012_val_00001704.JPEG:n04554684 +ILSVRC2012_val_00001705.JPEG:n02417914 +ILSVRC2012_val_00001706.JPEG:n01756291 +ILSVRC2012_val_00001707.JPEG:n03590841 +ILSVRC2012_val_00001708.JPEG:n01877812 +ILSVRC2012_val_00001709.JPEG:n02113186 +ILSVRC2012_val_00001710.JPEG:n02093256 +ILSVRC2012_val_00001711.JPEG:n02099849 +ILSVRC2012_val_00001712.JPEG:n02397096 +ILSVRC2012_val_00001713.JPEG:n03642806 +ILSVRC2012_val_00001714.JPEG:n02231487 +ILSVRC2012_val_00001715.JPEG:n04179913 +ILSVRC2012_val_00001716.JPEG:n02012849 +ILSVRC2012_val_00001717.JPEG:n02279972 +ILSVRC2012_val_00001718.JPEG:n04447861 +ILSVRC2012_val_00001719.JPEG:n04355933 +ILSVRC2012_val_00001720.JPEG:n01560419 +ILSVRC2012_val_00001721.JPEG:n02445715 +ILSVRC2012_val_00001722.JPEG:n03770679 +ILSVRC2012_val_00001723.JPEG:n03929855 +ILSVRC2012_val_00001724.JPEG:n01688243 +ILSVRC2012_val_00001725.JPEG:n06596364 +ILSVRC2012_val_00001726.JPEG:n07930864 +ILSVRC2012_val_00001727.JPEG:n01945685 +ILSVRC2012_val_00001728.JPEG:n01631663 +ILSVRC2012_val_00001729.JPEG:n03216828 +ILSVRC2012_val_00001730.JPEG:n03995372 +ILSVRC2012_val_00001731.JPEG:n02782093 +ILSVRC2012_val_00001732.JPEG:n01860187 +ILSVRC2012_val_00001733.JPEG:n04443257 +ILSVRC2012_val_00001734.JPEG:n04579432 +ILSVRC2012_val_00001735.JPEG:n07745940 +ILSVRC2012_val_00001736.JPEG:n04146614 +ILSVRC2012_val_00001737.JPEG:n02177972 +ILSVRC2012_val_00001738.JPEG:n04392985 +ILSVRC2012_val_00001739.JPEG:n01644373 +ILSVRC2012_val_00001740.JPEG:n02317335 +ILSVRC2012_val_00001741.JPEG:n04553703 +ILSVRC2012_val_00001742.JPEG:n02138441 +ILSVRC2012_val_00001743.JPEG:n13040303 +ILSVRC2012_val_00001744.JPEG:n01985128 +ILSVRC2012_val_00001745.JPEG:n02134418 +ILSVRC2012_val_00001746.JPEG:n01945685 +ILSVRC2012_val_00001747.JPEG:n02526121 +ILSVRC2012_val_00001748.JPEG:n02317335 +ILSVRC2012_val_00001749.JPEG:n01820546 +ILSVRC2012_val_00001750.JPEG:n04501370 +ILSVRC2012_val_00001751.JPEG:n01560419 +ILSVRC2012_val_00001752.JPEG:n02268443 +ILSVRC2012_val_00001753.JPEG:n03796401 +ILSVRC2012_val_00001754.JPEG:n03916031 +ILSVRC2012_val_00001755.JPEG:n02992211 +ILSVRC2012_val_00001756.JPEG:n03127747 +ILSVRC2012_val_00001757.JPEG:n03180011 +ILSVRC2012_val_00001758.JPEG:n02102480 +ILSVRC2012_val_00001759.JPEG:n04277352 +ILSVRC2012_val_00001760.JPEG:n01776313 +ILSVRC2012_val_00001761.JPEG:n03017168 +ILSVRC2012_val_00001762.JPEG:n02111129 +ILSVRC2012_val_00001763.JPEG:n02190166 +ILSVRC2012_val_00001764.JPEG:n02098413 +ILSVRC2012_val_00001765.JPEG:n02090721 +ILSVRC2012_val_00001766.JPEG:n01776313 +ILSVRC2012_val_00001767.JPEG:n09421951 +ILSVRC2012_val_00001768.JPEG:n02113023 +ILSVRC2012_val_00001769.JPEG:n02672831 +ILSVRC2012_val_00001770.JPEG:n03764736 +ILSVRC2012_val_00001771.JPEG:n04146614 +ILSVRC2012_val_00001772.JPEG:n03347037 +ILSVRC2012_val_00001773.JPEG:n03868242 +ILSVRC2012_val_00001774.JPEG:n02667093 +ILSVRC2012_val_00001775.JPEG:n02093647 +ILSVRC2012_val_00001776.JPEG:n02169497 +ILSVRC2012_val_00001777.JPEG:n02089973 +ILSVRC2012_val_00001778.JPEG:n07747607 +ILSVRC2012_val_00001779.JPEG:n02085782 +ILSVRC2012_val_00001780.JPEG:n02815834 +ILSVRC2012_val_00001781.JPEG:n02105412 +ILSVRC2012_val_00001782.JPEG:n02086910 +ILSVRC2012_val_00001783.JPEG:n04204238 +ILSVRC2012_val_00001784.JPEG:n03530642 +ILSVRC2012_val_00001785.JPEG:n07583066 +ILSVRC2012_val_00001786.JPEG:n04039381 +ILSVRC2012_val_00001787.JPEG:n02965783 +ILSVRC2012_val_00001788.JPEG:n04501370 +ILSVRC2012_val_00001789.JPEG:n04086273 +ILSVRC2012_val_00001790.JPEG:n04263257 +ILSVRC2012_val_00001791.JPEG:n02443484 +ILSVRC2012_val_00001792.JPEG:n04162706 +ILSVRC2012_val_00001793.JPEG:n07613480 +ILSVRC2012_val_00001794.JPEG:n04525038 +ILSVRC2012_val_00001795.JPEG:n04266014 +ILSVRC2012_val_00001796.JPEG:n03721384 +ILSVRC2012_val_00001797.JPEG:n04467665 +ILSVRC2012_val_00001798.JPEG:n04523525 +ILSVRC2012_val_00001799.JPEG:n04162706 +ILSVRC2012_val_00001800.JPEG:n02025239 +ILSVRC2012_val_00001801.JPEG:n04146614 +ILSVRC2012_val_00001802.JPEG:n01677366 +ILSVRC2012_val_00001803.JPEG:n04179913 +ILSVRC2012_val_00001804.JPEG:n04125021 +ILSVRC2012_val_00001805.JPEG:n02917067 +ILSVRC2012_val_00001806.JPEG:n04392985 +ILSVRC2012_val_00001807.JPEG:n04550184 +ILSVRC2012_val_00001808.JPEG:n02090721 +ILSVRC2012_val_00001809.JPEG:n03796401 +ILSVRC2012_val_00001810.JPEG:n03014705 +ILSVRC2012_val_00001811.JPEG:n04344873 +ILSVRC2012_val_00001812.JPEG:n02091635 +ILSVRC2012_val_00001813.JPEG:n01608432 +ILSVRC2012_val_00001814.JPEG:n03690938 +ILSVRC2012_val_00001815.JPEG:n04141975 +ILSVRC2012_val_00001816.JPEG:n01629819 +ILSVRC2012_val_00001817.JPEG:n04523525 +ILSVRC2012_val_00001818.JPEG:n01955084 +ILSVRC2012_val_00001819.JPEG:n01756291 +ILSVRC2012_val_00001820.JPEG:n04443257 +ILSVRC2012_val_00001821.JPEG:n02927161 +ILSVRC2012_val_00001822.JPEG:n07880968 +ILSVRC2012_val_00001823.JPEG:n07836838 +ILSVRC2012_val_00001824.JPEG:n02484975 +ILSVRC2012_val_00001825.JPEG:n02091032 +ILSVRC2012_val_00001826.JPEG:n07714571 +ILSVRC2012_val_00001827.JPEG:n03535780 +ILSVRC2012_val_00001828.JPEG:n04149813 +ILSVRC2012_val_00001829.JPEG:n09468604 +ILSVRC2012_val_00001830.JPEG:n02033041 +ILSVRC2012_val_00001831.JPEG:n03584254 +ILSVRC2012_val_00001832.JPEG:n04550184 +ILSVRC2012_val_00001833.JPEG:n03887697 +ILSVRC2012_val_00001834.JPEG:n03838899 +ILSVRC2012_val_00001835.JPEG:n02174001 +ILSVRC2012_val_00001836.JPEG:n03272010 +ILSVRC2012_val_00001837.JPEG:n03297495 +ILSVRC2012_val_00001838.JPEG:n04074963 +ILSVRC2012_val_00001839.JPEG:n03649909 +ILSVRC2012_val_00001840.JPEG:n03496892 +ILSVRC2012_val_00001841.JPEG:n03467068 +ILSVRC2012_val_00001842.JPEG:n02268853 +ILSVRC2012_val_00001843.JPEG:n03400231 +ILSVRC2012_val_00001844.JPEG:n02093256 +ILSVRC2012_val_00001845.JPEG:n04367480 +ILSVRC2012_val_00001846.JPEG:n02091134 +ILSVRC2012_val_00001847.JPEG:n04118776 +ILSVRC2012_val_00001848.JPEG:n02086646 +ILSVRC2012_val_00001849.JPEG:n07753592 +ILSVRC2012_val_00001850.JPEG:n02504013 +ILSVRC2012_val_00001851.JPEG:n02104365 +ILSVRC2012_val_00001852.JPEG:n02096177 +ILSVRC2012_val_00001853.JPEG:n03961711 +ILSVRC2012_val_00001854.JPEG:n04069434 +ILSVRC2012_val_00001855.JPEG:n03376595 +ILSVRC2012_val_00001856.JPEG:n01817953 +ILSVRC2012_val_00001857.JPEG:n01955084 +ILSVRC2012_val_00001858.JPEG:n02107142 +ILSVRC2012_val_00001859.JPEG:n03344393 +ILSVRC2012_val_00001860.JPEG:n03709823 +ILSVRC2012_val_00001861.JPEG:n02974003 +ILSVRC2012_val_00001862.JPEG:n02090379 +ILSVRC2012_val_00001863.JPEG:n04332243 +ILSVRC2012_val_00001864.JPEG:n03125729 +ILSVRC2012_val_00001865.JPEG:n03935335 +ILSVRC2012_val_00001866.JPEG:n02814860 +ILSVRC2012_val_00001867.JPEG:n01860187 +ILSVRC2012_val_00001868.JPEG:n03220513 +ILSVRC2012_val_00001869.JPEG:n02094114 +ILSVRC2012_val_00001870.JPEG:n03877472 +ILSVRC2012_val_00001871.JPEG:n02009912 +ILSVRC2012_val_00001872.JPEG:n02108000 +ILSVRC2012_val_00001873.JPEG:n02229544 +ILSVRC2012_val_00001874.JPEG:n03697007 +ILSVRC2012_val_00001875.JPEG:n03124170 +ILSVRC2012_val_00001876.JPEG:n02206856 +ILSVRC2012_val_00001877.JPEG:n03841143 +ILSVRC2012_val_00001878.JPEG:n04153751 +ILSVRC2012_val_00001879.JPEG:n01742172 +ILSVRC2012_val_00001880.JPEG:n13133613 +ILSVRC2012_val_00001881.JPEG:n04525305 +ILSVRC2012_val_00001882.JPEG:n01930112 +ILSVRC2012_val_00001883.JPEG:n02795169 +ILSVRC2012_val_00001884.JPEG:n02233338 +ILSVRC2012_val_00001885.JPEG:n02417914 +ILSVRC2012_val_00001886.JPEG:n03935335 +ILSVRC2012_val_00001887.JPEG:n01770393 +ILSVRC2012_val_00001888.JPEG:n02125311 +ILSVRC2012_val_00001889.JPEG:n03482405 +ILSVRC2012_val_00001890.JPEG:n04604644 +ILSVRC2012_val_00001891.JPEG:n02009912 +ILSVRC2012_val_00001892.JPEG:n03791053 +ILSVRC2012_val_00001893.JPEG:n03223299 +ILSVRC2012_val_00001894.JPEG:n03032252 +ILSVRC2012_val_00001895.JPEG:n04501370 +ILSVRC2012_val_00001896.JPEG:n03372029 +ILSVRC2012_val_00001897.JPEG:n03485794 +ILSVRC2012_val_00001898.JPEG:n02110341 +ILSVRC2012_val_00001899.JPEG:n04200800 +ILSVRC2012_val_00001900.JPEG:n02106166 +ILSVRC2012_val_00001901.JPEG:n04592741 +ILSVRC2012_val_00001902.JPEG:n02950826 +ILSVRC2012_val_00001903.JPEG:n04041544 +ILSVRC2012_val_00001904.JPEG:n07831146 +ILSVRC2012_val_00001905.JPEG:n04116512 +ILSVRC2012_val_00001906.JPEG:n01514859 +ILSVRC2012_val_00001907.JPEG:n03868242 +ILSVRC2012_val_00001908.JPEG:n03026506 +ILSVRC2012_val_00001909.JPEG:n02443484 +ILSVRC2012_val_00001910.JPEG:n02701002 +ILSVRC2012_val_00001911.JPEG:n04116512 +ILSVRC2012_val_00001912.JPEG:n02815834 +ILSVRC2012_val_00001913.JPEG:n03929855 +ILSVRC2012_val_00001914.JPEG:n03676483 +ILSVRC2012_val_00001915.JPEG:n01534433 +ILSVRC2012_val_00001916.JPEG:n02701002 +ILSVRC2012_val_00001917.JPEG:n02113978 +ILSVRC2012_val_00001918.JPEG:n04371430 +ILSVRC2012_val_00001919.JPEG:n03991062 +ILSVRC2012_val_00001920.JPEG:n07718472 +ILSVRC2012_val_00001921.JPEG:n02268853 +ILSVRC2012_val_00001922.JPEG:n04264628 +ILSVRC2012_val_00001923.JPEG:n02098105 +ILSVRC2012_val_00001924.JPEG:n07565083 +ILSVRC2012_val_00001925.JPEG:n02112706 +ILSVRC2012_val_00001926.JPEG:n02094114 +ILSVRC2012_val_00001927.JPEG:n02093991 +ILSVRC2012_val_00001928.JPEG:n02488291 +ILSVRC2012_val_00001929.JPEG:n02093859 +ILSVRC2012_val_00001930.JPEG:n03047690 +ILSVRC2012_val_00001931.JPEG:n01682714 +ILSVRC2012_val_00001932.JPEG:n07717410 +ILSVRC2012_val_00001933.JPEG:n01883070 +ILSVRC2012_val_00001934.JPEG:n04562935 +ILSVRC2012_val_00001935.JPEG:n01498041 +ILSVRC2012_val_00001936.JPEG:n07745940 +ILSVRC2012_val_00001937.JPEG:n02109525 +ILSVRC2012_val_00001938.JPEG:n01644900 +ILSVRC2012_val_00001939.JPEG:n01694178 +ILSVRC2012_val_00001940.JPEG:n03063689 +ILSVRC2012_val_00001941.JPEG:n02894605 +ILSVRC2012_val_00001942.JPEG:n01682714 +ILSVRC2012_val_00001943.JPEG:n03544143 +ILSVRC2012_val_00001944.JPEG:n02101556 +ILSVRC2012_val_00001945.JPEG:n02966687 +ILSVRC2012_val_00001946.JPEG:n03485407 +ILSVRC2012_val_00001947.JPEG:n03657121 +ILSVRC2012_val_00001948.JPEG:n02236044 +ILSVRC2012_val_00001949.JPEG:n07860988 +ILSVRC2012_val_00001950.JPEG:n01677366 +ILSVRC2012_val_00001951.JPEG:n07718747 +ILSVRC2012_val_00001952.JPEG:n02690373 +ILSVRC2012_val_00001953.JPEG:n04099969 +ILSVRC2012_val_00001954.JPEG:n03814639 +ILSVRC2012_val_00001955.JPEG:n02098413 +ILSVRC2012_val_00001956.JPEG:n01985128 +ILSVRC2012_val_00001957.JPEG:n02093647 +ILSVRC2012_val_00001958.JPEG:n02504458 +ILSVRC2012_val_00001959.JPEG:n01944390 +ILSVRC2012_val_00001960.JPEG:n03445924 +ILSVRC2012_val_00001961.JPEG:n03866082 +ILSVRC2012_val_00001962.JPEG:n03355925 +ILSVRC2012_val_00001963.JPEG:n02105855 +ILSVRC2012_val_00001964.JPEG:n03041632 +ILSVRC2012_val_00001965.JPEG:n03791053 +ILSVRC2012_val_00001966.JPEG:n03954731 +ILSVRC2012_val_00001967.JPEG:n07695742 +ILSVRC2012_val_00001968.JPEG:n02102040 +ILSVRC2012_val_00001969.JPEG:n03956157 +ILSVRC2012_val_00001970.JPEG:n03983396 +ILSVRC2012_val_00001971.JPEG:n02105855 +ILSVRC2012_val_00001972.JPEG:n03249569 +ILSVRC2012_val_00001973.JPEG:n03976467 +ILSVRC2012_val_00001974.JPEG:n03843555 +ILSVRC2012_val_00001975.JPEG:n02641379 +ILSVRC2012_val_00001976.JPEG:n03272562 +ILSVRC2012_val_00001977.JPEG:n03658185 +ILSVRC2012_val_00001978.JPEG:n03976467 +ILSVRC2012_val_00001979.JPEG:n02398521 +ILSVRC2012_val_00001980.JPEG:n03791053 +ILSVRC2012_val_00001981.JPEG:n03065424 +ILSVRC2012_val_00001982.JPEG:n03759954 +ILSVRC2012_val_00001983.JPEG:n03216828 +ILSVRC2012_val_00001984.JPEG:n03796401 +ILSVRC2012_val_00001985.JPEG:n01980166 +ILSVRC2012_val_00001986.JPEG:n09193705 +ILSVRC2012_val_00001987.JPEG:n01773797 +ILSVRC2012_val_00001988.JPEG:n02129604 +ILSVRC2012_val_00001989.JPEG:n04009552 +ILSVRC2012_val_00001990.JPEG:n02980441 +ILSVRC2012_val_00001991.JPEG:n03188531 +ILSVRC2012_val_00001992.JPEG:n02100735 +ILSVRC2012_val_00001993.JPEG:n07860988 +ILSVRC2012_val_00001994.JPEG:n03929855 +ILSVRC2012_val_00001995.JPEG:n04037443 +ILSVRC2012_val_00001996.JPEG:n03467068 +ILSVRC2012_val_00001997.JPEG:n02094114 +ILSVRC2012_val_00001998.JPEG:n03899768 +ILSVRC2012_val_00001999.JPEG:n04525038 +ILSVRC2012_val_00002000.JPEG:n02074367 +ILSVRC2012_val_00002001.JPEG:n04033901 +ILSVRC2012_val_00002002.JPEG:n02012849 +ILSVRC2012_val_00002003.JPEG:n02009229 +ILSVRC2012_val_00002004.JPEG:n02109961 +ILSVRC2012_val_00002005.JPEG:n03804744 +ILSVRC2012_val_00002006.JPEG:n02396427 +ILSVRC2012_val_00002007.JPEG:n02233338 +ILSVRC2012_val_00002008.JPEG:n03240683 +ILSVRC2012_val_00002009.JPEG:n03393912 +ILSVRC2012_val_00002010.JPEG:n03777568 +ILSVRC2012_val_00002011.JPEG:n02494079 +ILSVRC2012_val_00002012.JPEG:n02106662 +ILSVRC2012_val_00002013.JPEG:n04033995 +ILSVRC2012_val_00002014.JPEG:n02231487 +ILSVRC2012_val_00002015.JPEG:n04355338 +ILSVRC2012_val_00002016.JPEG:n04550184 +ILSVRC2012_val_00002017.JPEG:n02699494 +ILSVRC2012_val_00002018.JPEG:n04118538 +ILSVRC2012_val_00002019.JPEG:n03388043 +ILSVRC2012_val_00002020.JPEG:n02869837 +ILSVRC2012_val_00002021.JPEG:n02097047 +ILSVRC2012_val_00002022.JPEG:n03063689 +ILSVRC2012_val_00002023.JPEG:n01530575 +ILSVRC2012_val_00002024.JPEG:n02091032 +ILSVRC2012_val_00002025.JPEG:n03042490 +ILSVRC2012_val_00002026.JPEG:n03930313 +ILSVRC2012_val_00002027.JPEG:n02264363 +ILSVRC2012_val_00002028.JPEG:n02442845 +ILSVRC2012_val_00002029.JPEG:n02325366 +ILSVRC2012_val_00002030.JPEG:n01883070 +ILSVRC2012_val_00002031.JPEG:n01614925 +ILSVRC2012_val_00002032.JPEG:n03447721 +ILSVRC2012_val_00002033.JPEG:n03444034 +ILSVRC2012_val_00002034.JPEG:n02979186 +ILSVRC2012_val_00002035.JPEG:n02815834 +ILSVRC2012_val_00002036.JPEG:n02123394 +ILSVRC2012_val_00002037.JPEG:n03250847 +ILSVRC2012_val_00002038.JPEG:n02883205 +ILSVRC2012_val_00002039.JPEG:n04554684 +ILSVRC2012_val_00002040.JPEG:n03047690 +ILSVRC2012_val_00002041.JPEG:n01773157 +ILSVRC2012_val_00002042.JPEG:n02172182 +ILSVRC2012_val_00002043.JPEG:n03249569 +ILSVRC2012_val_00002044.JPEG:n04613696 +ILSVRC2012_val_00002045.JPEG:n03692522 +ILSVRC2012_val_00002046.JPEG:n04044716 +ILSVRC2012_val_00002047.JPEG:n12985857 +ILSVRC2012_val_00002048.JPEG:n02342885 +ILSVRC2012_val_00002049.JPEG:n03425413 +ILSVRC2012_val_00002050.JPEG:n02895154 +ILSVRC2012_val_00002051.JPEG:n01704323 +ILSVRC2012_val_00002052.JPEG:n01560419 +ILSVRC2012_val_00002053.JPEG:n02974003 +ILSVRC2012_val_00002054.JPEG:n07695742 +ILSVRC2012_val_00002055.JPEG:n03016953 +ILSVRC2012_val_00002056.JPEG:n03729826 +ILSVRC2012_val_00002057.JPEG:n03250847 +ILSVRC2012_val_00002058.JPEG:n02927161 +ILSVRC2012_val_00002059.JPEG:n02091635 +ILSVRC2012_val_00002060.JPEG:n01990800 +ILSVRC2012_val_00002061.JPEG:n02980441 +ILSVRC2012_val_00002062.JPEG:n02676566 +ILSVRC2012_val_00002063.JPEG:n02114548 +ILSVRC2012_val_00002064.JPEG:n02422699 +ILSVRC2012_val_00002065.JPEG:n04208210 +ILSVRC2012_val_00002066.JPEG:n02109961 +ILSVRC2012_val_00002067.JPEG:n04332243 +ILSVRC2012_val_00002068.JPEG:n04127249 +ILSVRC2012_val_00002069.JPEG:n03871628 +ILSVRC2012_val_00002070.JPEG:n02391049 +ILSVRC2012_val_00002071.JPEG:n01537544 +ILSVRC2012_val_00002072.JPEG:n02124075 +ILSVRC2012_val_00002073.JPEG:n02422106 +ILSVRC2012_val_00002074.JPEG:n01775062 +ILSVRC2012_val_00002075.JPEG:n03188531 +ILSVRC2012_val_00002076.JPEG:n02443114 +ILSVRC2012_val_00002077.JPEG:n01694178 +ILSVRC2012_val_00002078.JPEG:n03063689 +ILSVRC2012_val_00002079.JPEG:n02088364 +ILSVRC2012_val_00002080.JPEG:n04476259 +ILSVRC2012_val_00002081.JPEG:n04442312 +ILSVRC2012_val_00002082.JPEG:n03792972 +ILSVRC2012_val_00002083.JPEG:n07831146 +ILSVRC2012_val_00002084.JPEG:n02483708 +ILSVRC2012_val_00002085.JPEG:n04346328 +ILSVRC2012_val_00002086.JPEG:n04591713 +ILSVRC2012_val_00002087.JPEG:n03794056 +ILSVRC2012_val_00002088.JPEG:n04153751 +ILSVRC2012_val_00002089.JPEG:n03782006 +ILSVRC2012_val_00002090.JPEG:n02058221 +ILSVRC2012_val_00002091.JPEG:n04162706 +ILSVRC2012_val_00002092.JPEG:n04522168 +ILSVRC2012_val_00002093.JPEG:n03673027 +ILSVRC2012_val_00002094.JPEG:n04483307 +ILSVRC2012_val_00002095.JPEG:n03691459 +ILSVRC2012_val_00002096.JPEG:n03478589 +ILSVRC2012_val_00002097.JPEG:n02102318 +ILSVRC2012_val_00002098.JPEG:n07749582 +ILSVRC2012_val_00002099.JPEG:n07730033 +ILSVRC2012_val_00002100.JPEG:n01829413 +ILSVRC2012_val_00002101.JPEG:n01729977 +ILSVRC2012_val_00002102.JPEG:n04501370 +ILSVRC2012_val_00002103.JPEG:n09472597 +ILSVRC2012_val_00002104.JPEG:n03781244 +ILSVRC2012_val_00002105.JPEG:n02134084 +ILSVRC2012_val_00002106.JPEG:n01742172 +ILSVRC2012_val_00002107.JPEG:n03782006 +ILSVRC2012_val_00002108.JPEG:n04553703 +ILSVRC2012_val_00002109.JPEG:n09835506 +ILSVRC2012_val_00002110.JPEG:n03804744 +ILSVRC2012_val_00002111.JPEG:n02088238 +ILSVRC2012_val_00002112.JPEG:n04067472 +ILSVRC2012_val_00002113.JPEG:n03764736 +ILSVRC2012_val_00002114.JPEG:n02992529 +ILSVRC2012_val_00002115.JPEG:n03874599 +ILSVRC2012_val_00002116.JPEG:n03124043 +ILSVRC2012_val_00002117.JPEG:n04065272 +ILSVRC2012_val_00002118.JPEG:n02782093 +ILSVRC2012_val_00002119.JPEG:n03788195 +ILSVRC2012_val_00002120.JPEG:n04389033 +ILSVRC2012_val_00002121.JPEG:n03673027 +ILSVRC2012_val_00002122.JPEG:n04389033 +ILSVRC2012_val_00002123.JPEG:n03775071 +ILSVRC2012_val_00002124.JPEG:n07753113 +ILSVRC2012_val_00002125.JPEG:n12144580 +ILSVRC2012_val_00002126.JPEG:n02013706 +ILSVRC2012_val_00002127.JPEG:n02190166 +ILSVRC2012_val_00002128.JPEG:n04275548 +ILSVRC2012_val_00002129.JPEG:n03250847 +ILSVRC2012_val_00002130.JPEG:n03947888 +ILSVRC2012_val_00002131.JPEG:n01729977 +ILSVRC2012_val_00002132.JPEG:n02138441 +ILSVRC2012_val_00002133.JPEG:n04264628 +ILSVRC2012_val_00002134.JPEG:n03967562 +ILSVRC2012_val_00002135.JPEG:n03445924 +ILSVRC2012_val_00002136.JPEG:n04355338 +ILSVRC2012_val_00002137.JPEG:n02640242 +ILSVRC2012_val_00002138.JPEG:n01440764 +ILSVRC2012_val_00002139.JPEG:n12267677 +ILSVRC2012_val_00002140.JPEG:n02489166 +ILSVRC2012_val_00002141.JPEG:n02165105 +ILSVRC2012_val_00002142.JPEG:n03599486 +ILSVRC2012_val_00002143.JPEG:n03272010 +ILSVRC2012_val_00002144.JPEG:n02018207 +ILSVRC2012_val_00002145.JPEG:n02747177 +ILSVRC2012_val_00002146.JPEG:n04487081 +ILSVRC2012_val_00002147.JPEG:n02119789 +ILSVRC2012_val_00002148.JPEG:n02666196 +ILSVRC2012_val_00002149.JPEG:n02606052 +ILSVRC2012_val_00002150.JPEG:n02086646 +ILSVRC2012_val_00002151.JPEG:n04040759 +ILSVRC2012_val_00002152.JPEG:n01984695 +ILSVRC2012_val_00002153.JPEG:n12998815 +ILSVRC2012_val_00002154.JPEG:n01751748 +ILSVRC2012_val_00002155.JPEG:n04584207 +ILSVRC2012_val_00002156.JPEG:n04149813 +ILSVRC2012_val_00002157.JPEG:n01981276 +ILSVRC2012_val_00002158.JPEG:n02841315 +ILSVRC2012_val_00002159.JPEG:n03777754 +ILSVRC2012_val_00002160.JPEG:n04376876 +ILSVRC2012_val_00002161.JPEG:n02859443 +ILSVRC2012_val_00002162.JPEG:n04389033 +ILSVRC2012_val_00002163.JPEG:n01665541 +ILSVRC2012_val_00002164.JPEG:n04208210 +ILSVRC2012_val_00002165.JPEG:n04041544 +ILSVRC2012_val_00002166.JPEG:n02071294 +ILSVRC2012_val_00002167.JPEG:n13052670 +ILSVRC2012_val_00002168.JPEG:n01616318 +ILSVRC2012_val_00002169.JPEG:n03871628 +ILSVRC2012_val_00002170.JPEG:n02028035 +ILSVRC2012_val_00002171.JPEG:n03110669 +ILSVRC2012_val_00002172.JPEG:n01819313 +ILSVRC2012_val_00002173.JPEG:n04229816 +ILSVRC2012_val_00002174.JPEG:n02769748 +ILSVRC2012_val_00002175.JPEG:n03832673 +ILSVRC2012_val_00002176.JPEG:n02095889 +ILSVRC2012_val_00002177.JPEG:n01806143 +ILSVRC2012_val_00002178.JPEG:n02708093 +ILSVRC2012_val_00002179.JPEG:n07753113 +ILSVRC2012_val_00002180.JPEG:n02804610 +ILSVRC2012_val_00002181.JPEG:n02879718 +ILSVRC2012_val_00002182.JPEG:n03595614 +ILSVRC2012_val_00002183.JPEG:n02769748 +ILSVRC2012_val_00002184.JPEG:n07802026 +ILSVRC2012_val_00002185.JPEG:n04357314 +ILSVRC2012_val_00002186.JPEG:n09288635 +ILSVRC2012_val_00002187.JPEG:n07753592 +ILSVRC2012_val_00002188.JPEG:n04525038 +ILSVRC2012_val_00002189.JPEG:n04590129 +ILSVRC2012_val_00002190.JPEG:n01981276 +ILSVRC2012_val_00002191.JPEG:n01530575 +ILSVRC2012_val_00002192.JPEG:n02006656 +ILSVRC2012_val_00002193.JPEG:n03903868 +ILSVRC2012_val_00002194.JPEG:n02095570 +ILSVRC2012_val_00002195.JPEG:n03602883 +ILSVRC2012_val_00002196.JPEG:n03476991 +ILSVRC2012_val_00002197.JPEG:n04328186 +ILSVRC2012_val_00002198.JPEG:n03617480 +ILSVRC2012_val_00002199.JPEG:n03272562 +ILSVRC2012_val_00002200.JPEG:n02328150 +ILSVRC2012_val_00002201.JPEG:n04536866 +ILSVRC2012_val_00002202.JPEG:n02814860 +ILSVRC2012_val_00002203.JPEG:n03710193 +ILSVRC2012_val_00002204.JPEG:n04263257 +ILSVRC2012_val_00002205.JPEG:n02699494 +ILSVRC2012_val_00002206.JPEG:n04418357 +ILSVRC2012_val_00002207.JPEG:n01496331 +ILSVRC2012_val_00002208.JPEG:n02086079 +ILSVRC2012_val_00002209.JPEG:n03495258 +ILSVRC2012_val_00002210.JPEG:n03417042 +ILSVRC2012_val_00002211.JPEG:n03065424 +ILSVRC2012_val_00002212.JPEG:n03041632 +ILSVRC2012_val_00002213.JPEG:n04467665 +ILSVRC2012_val_00002214.JPEG:n02085936 +ILSVRC2012_val_00002215.JPEG:n03956157 +ILSVRC2012_val_00002216.JPEG:n02110341 +ILSVRC2012_val_00002217.JPEG:n07760859 +ILSVRC2012_val_00002218.JPEG:n03467068 +ILSVRC2012_val_00002219.JPEG:n02825657 +ILSVRC2012_val_00002220.JPEG:n02669723 +ILSVRC2012_val_00002221.JPEG:n07579787 +ILSVRC2012_val_00002222.JPEG:n02097658 +ILSVRC2012_val_00002223.JPEG:n03717622 +ILSVRC2012_val_00002224.JPEG:n03590841 +ILSVRC2012_val_00002225.JPEG:n02268443 +ILSVRC2012_val_00002226.JPEG:n07697313 +ILSVRC2012_val_00002227.JPEG:n02859443 +ILSVRC2012_val_00002228.JPEG:n01622779 +ILSVRC2012_val_00002229.JPEG:n02999410 +ILSVRC2012_val_00002230.JPEG:n01877812 +ILSVRC2012_val_00002231.JPEG:n01744401 +ILSVRC2012_val_00002232.JPEG:n01669191 +ILSVRC2012_val_00002233.JPEG:n04507155 +ILSVRC2012_val_00002234.JPEG:n02108000 +ILSVRC2012_val_00002235.JPEG:n10148035 +ILSVRC2012_val_00002236.JPEG:n04009552 +ILSVRC2012_val_00002237.JPEG:n09421951 +ILSVRC2012_val_00002238.JPEG:n03457902 +ILSVRC2012_val_00002239.JPEG:n02091032 +ILSVRC2012_val_00002240.JPEG:n03759954 +ILSVRC2012_val_00002241.JPEG:n01443537 +ILSVRC2012_val_00002242.JPEG:n02011460 +ILSVRC2012_val_00002243.JPEG:n01984695 +ILSVRC2012_val_00002244.JPEG:n02791270 +ILSVRC2012_val_00002245.JPEG:n03617480 +ILSVRC2012_val_00002246.JPEG:n02089973 +ILSVRC2012_val_00002247.JPEG:n02105641 +ILSVRC2012_val_00002248.JPEG:n03595614 +ILSVRC2012_val_00002249.JPEG:n03207941 +ILSVRC2012_val_00002250.JPEG:n03146219 +ILSVRC2012_val_00002251.JPEG:n04367480 +ILSVRC2012_val_00002252.JPEG:n07695742 +ILSVRC2012_val_00002253.JPEG:n03376595 +ILSVRC2012_val_00002254.JPEG:n09835506 +ILSVRC2012_val_00002255.JPEG:n02342885 +ILSVRC2012_val_00002256.JPEG:n03393912 +ILSVRC2012_val_00002257.JPEG:n04311004 +ILSVRC2012_val_00002258.JPEG:n04589890 +ILSVRC2012_val_00002259.JPEG:n02114367 +ILSVRC2012_val_00002260.JPEG:n02104029 +ILSVRC2012_val_00002261.JPEG:n01945685 +ILSVRC2012_val_00002262.JPEG:n02094114 +ILSVRC2012_val_00002263.JPEG:n01824575 +ILSVRC2012_val_00002264.JPEG:n04380533 +ILSVRC2012_val_00002265.JPEG:n02025239 +ILSVRC2012_val_00002266.JPEG:n03218198 +ILSVRC2012_val_00002267.JPEG:n02110627 +ILSVRC2012_val_00002268.JPEG:n04026417 +ILSVRC2012_val_00002269.JPEG:n02749479 +ILSVRC2012_val_00002270.JPEG:n07613480 +ILSVRC2012_val_00002271.JPEG:n02437312 +ILSVRC2012_val_00002272.JPEG:n03347037 +ILSVRC2012_val_00002273.JPEG:n02403003 +ILSVRC2012_val_00002274.JPEG:n03942813 +ILSVRC2012_val_00002275.JPEG:n03450230 +ILSVRC2012_val_00002276.JPEG:n04252225 +ILSVRC2012_val_00002277.JPEG:n02108000 +ILSVRC2012_val_00002278.JPEG:n03837869 +ILSVRC2012_val_00002279.JPEG:n02165105 +ILSVRC2012_val_00002280.JPEG:n03000247 +ILSVRC2012_val_00002281.JPEG:n04344873 +ILSVRC2012_val_00002282.JPEG:n02504458 +ILSVRC2012_val_00002283.JPEG:n02110185 +ILSVRC2012_val_00002284.JPEG:n01498041 +ILSVRC2012_val_00002285.JPEG:n04270147 +ILSVRC2012_val_00002286.JPEG:n04239074 +ILSVRC2012_val_00002287.JPEG:n03924679 +ILSVRC2012_val_00002288.JPEG:n02086646 +ILSVRC2012_val_00002289.JPEG:n09835506 +ILSVRC2012_val_00002290.JPEG:n03424325 +ILSVRC2012_val_00002291.JPEG:n04370456 +ILSVRC2012_val_00002292.JPEG:n03777754 +ILSVRC2012_val_00002293.JPEG:n03529860 +ILSVRC2012_val_00002294.JPEG:n02102040 +ILSVRC2012_val_00002295.JPEG:n01688243 +ILSVRC2012_val_00002296.JPEG:n02110627 +ILSVRC2012_val_00002297.JPEG:n02100735 +ILSVRC2012_val_00002298.JPEG:n02102177 +ILSVRC2012_val_00002299.JPEG:n04086273 +ILSVRC2012_val_00002300.JPEG:n01883070 +ILSVRC2012_val_00002301.JPEG:n04366367 +ILSVRC2012_val_00002302.JPEG:n02107574 +ILSVRC2012_val_00002303.JPEG:n02102480 +ILSVRC2012_val_00002304.JPEG:n04008634 +ILSVRC2012_val_00002305.JPEG:n02169497 +ILSVRC2012_val_00002306.JPEG:n04141327 +ILSVRC2012_val_00002307.JPEG:n02442845 +ILSVRC2012_val_00002308.JPEG:n03662601 +ILSVRC2012_val_00002309.JPEG:n01855032 +ILSVRC2012_val_00002310.JPEG:n04589890 +ILSVRC2012_val_00002311.JPEG:n02018795 +ILSVRC2012_val_00002312.JPEG:n03271574 +ILSVRC2012_val_00002313.JPEG:n02097298 +ILSVRC2012_val_00002314.JPEG:n03445777 +ILSVRC2012_val_00002315.JPEG:n02102040 +ILSVRC2012_val_00002316.JPEG:n03617480 +ILSVRC2012_val_00002317.JPEG:n02108422 +ILSVRC2012_val_00002318.JPEG:n02097474 +ILSVRC2012_val_00002319.JPEG:n02109525 +ILSVRC2012_val_00002320.JPEG:n02097474 +ILSVRC2012_val_00002321.JPEG:n11879895 +ILSVRC2012_val_00002322.JPEG:n03223299 +ILSVRC2012_val_00002323.JPEG:n02100583 +ILSVRC2012_val_00002324.JPEG:n03840681 +ILSVRC2012_val_00002325.JPEG:n02091032 +ILSVRC2012_val_00002326.JPEG:n01843065 +ILSVRC2012_val_00002327.JPEG:n03769881 +ILSVRC2012_val_00002328.JPEG:n02091467 +ILSVRC2012_val_00002329.JPEG:n02134418 +ILSVRC2012_val_00002330.JPEG:n02109047 +ILSVRC2012_val_00002331.JPEG:n04456115 +ILSVRC2012_val_00002332.JPEG:n03866082 +ILSVRC2012_val_00002333.JPEG:n04239074 +ILSVRC2012_val_00002334.JPEG:n02484975 +ILSVRC2012_val_00002335.JPEG:n04259630 +ILSVRC2012_val_00002336.JPEG:n07760859 +ILSVRC2012_val_00002337.JPEG:n09246464 +ILSVRC2012_val_00002338.JPEG:n01484850 +ILSVRC2012_val_00002339.JPEG:n02443114 +ILSVRC2012_val_00002340.JPEG:n04251144 +ILSVRC2012_val_00002341.JPEG:n03843555 +ILSVRC2012_val_00002342.JPEG:n04131690 +ILSVRC2012_val_00002343.JPEG:n07716906 +ILSVRC2012_val_00002344.JPEG:n03584254 +ILSVRC2012_val_00002345.JPEG:n04033901 +ILSVRC2012_val_00002346.JPEG:n04146614 +ILSVRC2012_val_00002347.JPEG:n03633091 +ILSVRC2012_val_00002348.JPEG:n13037406 +ILSVRC2012_val_00002349.JPEG:n04254680 +ILSVRC2012_val_00002350.JPEG:n07583066 +ILSVRC2012_val_00002351.JPEG:n03483316 +ILSVRC2012_val_00002352.JPEG:n02056570 +ILSVRC2012_val_00002353.JPEG:n02102177 +ILSVRC2012_val_00002354.JPEG:n04355338 +ILSVRC2012_val_00002355.JPEG:n01669191 +ILSVRC2012_val_00002356.JPEG:n04039381 +ILSVRC2012_val_00002357.JPEG:n01532829 +ILSVRC2012_val_00002358.JPEG:n02978881 +ILSVRC2012_val_00002359.JPEG:n03691459 +ILSVRC2012_val_00002360.JPEG:n04118776 +ILSVRC2012_val_00002361.JPEG:n02672831 +ILSVRC2012_val_00002362.JPEG:n06785654 +ILSVRC2012_val_00002363.JPEG:n07749582 +ILSVRC2012_val_00002364.JPEG:n02536864 +ILSVRC2012_val_00002365.JPEG:n02116738 +ILSVRC2012_val_00002366.JPEG:n04239074 +ILSVRC2012_val_00002367.JPEG:n02483708 +ILSVRC2012_val_00002368.JPEG:n03124170 +ILSVRC2012_val_00002369.JPEG:n07930864 +ILSVRC2012_val_00002370.JPEG:n02018207 +ILSVRC2012_val_00002371.JPEG:n04074963 +ILSVRC2012_val_00002372.JPEG:n01514859 +ILSVRC2012_val_00002373.JPEG:n02089867 +ILSVRC2012_val_00002374.JPEG:n03804744 +ILSVRC2012_val_00002375.JPEG:n04116512 +ILSVRC2012_val_00002376.JPEG:n02802426 +ILSVRC2012_val_00002377.JPEG:n03627232 +ILSVRC2012_val_00002378.JPEG:n03787032 +ILSVRC2012_val_00002379.JPEG:n02281406 +ILSVRC2012_val_00002380.JPEG:n07613480 +ILSVRC2012_val_00002381.JPEG:n02526121 +ILSVRC2012_val_00002382.JPEG:n02860847 +ILSVRC2012_val_00002383.JPEG:n01806143 +ILSVRC2012_val_00002384.JPEG:n03706229 +ILSVRC2012_val_00002385.JPEG:n03982430 +ILSVRC2012_val_00002386.JPEG:n04009552 +ILSVRC2012_val_00002387.JPEG:n01616318 +ILSVRC2012_val_00002388.JPEG:n01828970 +ILSVRC2012_val_00002389.JPEG:n03920288 +ILSVRC2012_val_00002390.JPEG:n03680355 +ILSVRC2012_val_00002391.JPEG:n02727426 +ILSVRC2012_val_00002392.JPEG:n02963159 +ILSVRC2012_val_00002393.JPEG:n02102973 +ILSVRC2012_val_00002394.JPEG:n04209133 +ILSVRC2012_val_00002395.JPEG:n01798484 +ILSVRC2012_val_00002396.JPEG:n02190166 +ILSVRC2012_val_00002397.JPEG:n02091635 +ILSVRC2012_val_00002398.JPEG:n02089078 +ILSVRC2012_val_00002399.JPEG:n04371774 +ILSVRC2012_val_00002400.JPEG:n04515003 +ILSVRC2012_val_00002401.JPEG:n02655020 +ILSVRC2012_val_00002402.JPEG:n02104029 +ILSVRC2012_val_00002403.JPEG:n01877812 +ILSVRC2012_val_00002404.JPEG:n02794156 +ILSVRC2012_val_00002405.JPEG:n02974003 +ILSVRC2012_val_00002406.JPEG:n02096585 +ILSVRC2012_val_00002407.JPEG:n04525305 +ILSVRC2012_val_00002408.JPEG:n02672831 +ILSVRC2012_val_00002409.JPEG:n02113712 +ILSVRC2012_val_00002410.JPEG:n02917067 +ILSVRC2012_val_00002411.JPEG:n02096437 +ILSVRC2012_val_00002412.JPEG:n07745940 +ILSVRC2012_val_00002413.JPEG:n02326432 +ILSVRC2012_val_00002414.JPEG:n03314780 +ILSVRC2012_val_00002415.JPEG:n02236044 +ILSVRC2012_val_00002416.JPEG:n02102973 +ILSVRC2012_val_00002417.JPEG:n02093428 +ILSVRC2012_val_00002418.JPEG:n03297495 +ILSVRC2012_val_00002419.JPEG:n03676483 +ILSVRC2012_val_00002420.JPEG:n03775071 +ILSVRC2012_val_00002421.JPEG:n04536866 +ILSVRC2012_val_00002422.JPEG:n04554684 +ILSVRC2012_val_00002423.JPEG:n03400231 +ILSVRC2012_val_00002424.JPEG:n04346328 +ILSVRC2012_val_00002425.JPEG:n01530575 +ILSVRC2012_val_00002426.JPEG:n04133789 +ILSVRC2012_val_00002427.JPEG:n03160309 +ILSVRC2012_val_00002428.JPEG:n01930112 +ILSVRC2012_val_00002429.JPEG:n03494278 +ILSVRC2012_val_00002430.JPEG:n03063599 +ILSVRC2012_val_00002431.JPEG:n03891332 +ILSVRC2012_val_00002432.JPEG:n04476259 +ILSVRC2012_val_00002433.JPEG:n02410509 +ILSVRC2012_val_00002434.JPEG:n03417042 +ILSVRC2012_val_00002435.JPEG:n07753113 +ILSVRC2012_val_00002436.JPEG:n03498962 +ILSVRC2012_val_00002437.JPEG:n03991062 +ILSVRC2012_val_00002438.JPEG:n04086273 +ILSVRC2012_val_00002439.JPEG:n01739381 +ILSVRC2012_val_00002440.JPEG:n07753275 +ILSVRC2012_val_00002441.JPEG:n03065424 +ILSVRC2012_val_00002442.JPEG:n03476991 +ILSVRC2012_val_00002443.JPEG:n07565083 +ILSVRC2012_val_00002444.JPEG:n01608432 +ILSVRC2012_val_00002445.JPEG:n04258138 +ILSVRC2012_val_00002446.JPEG:n03803284 +ILSVRC2012_val_00002447.JPEG:n02120079 +ILSVRC2012_val_00002448.JPEG:n02454379 +ILSVRC2012_val_00002449.JPEG:n01537544 +ILSVRC2012_val_00002450.JPEG:n02492035 +ILSVRC2012_val_00002451.JPEG:n02219486 +ILSVRC2012_val_00002452.JPEG:n01735189 +ILSVRC2012_val_00002453.JPEG:n03594734 +ILSVRC2012_val_00002454.JPEG:n02442845 +ILSVRC2012_val_00002455.JPEG:n04485082 +ILSVRC2012_val_00002456.JPEG:n03599486 +ILSVRC2012_val_00002457.JPEG:n02086079 +ILSVRC2012_val_00002458.JPEG:n03995372 +ILSVRC2012_val_00002459.JPEG:n04501370 +ILSVRC2012_val_00002460.JPEG:n02113712 +ILSVRC2012_val_00002461.JPEG:n02102480 +ILSVRC2012_val_00002462.JPEG:n03599486 +ILSVRC2012_val_00002463.JPEG:n04162706 +ILSVRC2012_val_00002464.JPEG:n03868242 +ILSVRC2012_val_00002465.JPEG:n04209133 +ILSVRC2012_val_00002466.JPEG:n02791124 +ILSVRC2012_val_00002467.JPEG:n01819313 +ILSVRC2012_val_00002468.JPEG:n02116738 +ILSVRC2012_val_00002469.JPEG:n02894605 +ILSVRC2012_val_00002470.JPEG:n03764736 +ILSVRC2012_val_00002471.JPEG:n03476684 +ILSVRC2012_val_00002472.JPEG:n02123159 +ILSVRC2012_val_00002473.JPEG:n02325366 +ILSVRC2012_val_00002474.JPEG:n03457902 +ILSVRC2012_val_00002475.JPEG:n02123597 +ILSVRC2012_val_00002476.JPEG:n09399592 +ILSVRC2012_val_00002477.JPEG:n02488291 +ILSVRC2012_val_00002478.JPEG:n03788365 +ILSVRC2012_val_00002479.JPEG:n01770081 +ILSVRC2012_val_00002480.JPEG:n01498041 +ILSVRC2012_val_00002481.JPEG:n02110341 +ILSVRC2012_val_00002482.JPEG:n02834397 +ILSVRC2012_val_00002483.JPEG:n02391049 +ILSVRC2012_val_00002484.JPEG:n02113023 +ILSVRC2012_val_00002485.JPEG:n02099712 +ILSVRC2012_val_00002486.JPEG:n01739381 +ILSVRC2012_val_00002487.JPEG:n02980441 +ILSVRC2012_val_00002488.JPEG:n02027492 +ILSVRC2012_val_00002489.JPEG:n03208938 +ILSVRC2012_val_00002490.JPEG:n07734744 +ILSVRC2012_val_00002491.JPEG:n02027492 +ILSVRC2012_val_00002492.JPEG:n02108000 +ILSVRC2012_val_00002493.JPEG:n03902125 +ILSVRC2012_val_00002494.JPEG:n04044716 +ILSVRC2012_val_00002495.JPEG:n09428293 +ILSVRC2012_val_00002496.JPEG:n01981276 +ILSVRC2012_val_00002497.JPEG:n02869837 +ILSVRC2012_val_00002498.JPEG:n03425413 +ILSVRC2012_val_00002499.JPEG:n03085013 +ILSVRC2012_val_00002500.JPEG:n03804744 +ILSVRC2012_val_00002501.JPEG:n02443114 +ILSVRC2012_val_00002502.JPEG:n01983481 +ILSVRC2012_val_00002503.JPEG:n02088466 +ILSVRC2012_val_00002504.JPEG:n02077923 +ILSVRC2012_val_00002505.JPEG:n01740131 +ILSVRC2012_val_00002506.JPEG:n09468604 +ILSVRC2012_val_00002507.JPEG:n02783161 +ILSVRC2012_val_00002508.JPEG:n03888257 +ILSVRC2012_val_00002509.JPEG:n02797295 +ILSVRC2012_val_00002510.JPEG:n04252225 +ILSVRC2012_val_00002511.JPEG:n01622779 +ILSVRC2012_val_00002512.JPEG:n01669191 +ILSVRC2012_val_00002513.JPEG:n03710637 +ILSVRC2012_val_00002514.JPEG:n01669191 +ILSVRC2012_val_00002515.JPEG:n01983481 +ILSVRC2012_val_00002516.JPEG:n02108422 +ILSVRC2012_val_00002517.JPEG:n04111531 +ILSVRC2012_val_00002518.JPEG:n04179913 +ILSVRC2012_val_00002519.JPEG:n04204238 +ILSVRC2012_val_00002520.JPEG:n04389033 +ILSVRC2012_val_00002521.JPEG:n02087046 +ILSVRC2012_val_00002522.JPEG:n01872401 +ILSVRC2012_val_00002523.JPEG:n02692877 +ILSVRC2012_val_00002524.JPEG:n01632777 +ILSVRC2012_val_00002525.JPEG:n02640242 +ILSVRC2012_val_00002526.JPEG:n02927161 +ILSVRC2012_val_00002527.JPEG:n02814860 +ILSVRC2012_val_00002528.JPEG:n03792972 +ILSVRC2012_val_00002529.JPEG:n04039381 +ILSVRC2012_val_00002530.JPEG:n02480855 +ILSVRC2012_val_00002531.JPEG:n03599486 +ILSVRC2012_val_00002532.JPEG:n04326547 +ILSVRC2012_val_00002533.JPEG:n03691459 +ILSVRC2012_val_00002534.JPEG:n04592741 +ILSVRC2012_val_00002535.JPEG:n03014705 +ILSVRC2012_val_00002536.JPEG:n01582220 +ILSVRC2012_val_00002537.JPEG:n13052670 +ILSVRC2012_val_00002538.JPEG:n02802426 +ILSVRC2012_val_00002539.JPEG:n01797886 +ILSVRC2012_val_00002540.JPEG:n04263257 +ILSVRC2012_val_00002541.JPEG:n04350905 +ILSVRC2012_val_00002542.JPEG:n03372029 +ILSVRC2012_val_00002543.JPEG:n02484975 +ILSVRC2012_val_00002544.JPEG:n09428293 +ILSVRC2012_val_00002545.JPEG:n03887697 +ILSVRC2012_val_00002546.JPEG:n02112350 +ILSVRC2012_val_00002547.JPEG:n03110669 +ILSVRC2012_val_00002548.JPEG:n02910353 +ILSVRC2012_val_00002549.JPEG:n02096294 +ILSVRC2012_val_00002550.JPEG:n02102177 +ILSVRC2012_val_00002551.JPEG:n02115913 +ILSVRC2012_val_00002552.JPEG:n02804610 +ILSVRC2012_val_00002553.JPEG:n04239074 +ILSVRC2012_val_00002554.JPEG:n04005630 +ILSVRC2012_val_00002555.JPEG:n04118538 +ILSVRC2012_val_00002556.JPEG:n04067472 +ILSVRC2012_val_00002557.JPEG:n02128757 +ILSVRC2012_val_00002558.JPEG:n02097658 +ILSVRC2012_val_00002559.JPEG:n02099849 +ILSVRC2012_val_00002560.JPEG:n01882714 +ILSVRC2012_val_00002561.JPEG:n02494079 +ILSVRC2012_val_00002562.JPEG:n03379051 +ILSVRC2012_val_00002563.JPEG:n02808440 +ILSVRC2012_val_00002564.JPEG:n04392985 +ILSVRC2012_val_00002565.JPEG:n02114548 +ILSVRC2012_val_00002566.JPEG:n02206856 +ILSVRC2012_val_00002567.JPEG:n03976657 +ILSVRC2012_val_00002568.JPEG:n01729322 +ILSVRC2012_val_00002569.JPEG:n07831146 +ILSVRC2012_val_00002570.JPEG:n01883070 +ILSVRC2012_val_00002571.JPEG:n02361337 +ILSVRC2012_val_00002572.JPEG:n02128757 +ILSVRC2012_val_00002573.JPEG:n02097130 +ILSVRC2012_val_00002574.JPEG:n04447861 +ILSVRC2012_val_00002575.JPEG:n13052670 +ILSVRC2012_val_00002576.JPEG:n02096177 +ILSVRC2012_val_00002577.JPEG:n03691459 +ILSVRC2012_val_00002578.JPEG:n02134084 +ILSVRC2012_val_00002579.JPEG:n02494079 +ILSVRC2012_val_00002580.JPEG:n03642806 +ILSVRC2012_val_00002581.JPEG:n04136333 +ILSVRC2012_val_00002582.JPEG:n02268853 +ILSVRC2012_val_00002583.JPEG:n02417914 +ILSVRC2012_val_00002584.JPEG:n03891332 +ILSVRC2012_val_00002585.JPEG:n09246464 +ILSVRC2012_val_00002586.JPEG:n03032252 +ILSVRC2012_val_00002587.JPEG:n02825657 +ILSVRC2012_val_00002588.JPEG:n03498962 +ILSVRC2012_val_00002589.JPEG:n03160309 +ILSVRC2012_val_00002590.JPEG:n04026417 +ILSVRC2012_val_00002591.JPEG:n04296562 +ILSVRC2012_val_00002592.JPEG:n03534580 +ILSVRC2012_val_00002593.JPEG:n03216828 +ILSVRC2012_val_00002594.JPEG:n07880968 +ILSVRC2012_val_00002595.JPEG:n03393912 +ILSVRC2012_val_00002596.JPEG:n02948072 +ILSVRC2012_val_00002597.JPEG:n04560804 +ILSVRC2012_val_00002598.JPEG:n04152593 +ILSVRC2012_val_00002599.JPEG:n04509417 +ILSVRC2012_val_00002600.JPEG:n03884397 +ILSVRC2012_val_00002601.JPEG:n02129604 +ILSVRC2012_val_00002602.JPEG:n01944390 +ILSVRC2012_val_00002603.JPEG:n04310018 +ILSVRC2012_val_00002604.JPEG:n04086273 +ILSVRC2012_val_00002605.JPEG:n07584110 +ILSVRC2012_val_00002606.JPEG:n04258138 +ILSVRC2012_val_00002607.JPEG:n04264628 +ILSVRC2012_val_00002608.JPEG:n13040303 +ILSVRC2012_val_00002609.JPEG:n02109525 +ILSVRC2012_val_00002610.JPEG:n04462240 +ILSVRC2012_val_00002611.JPEG:n02791270 +ILSVRC2012_val_00002612.JPEG:n03384352 +ILSVRC2012_val_00002613.JPEG:n04070727 +ILSVRC2012_val_00002614.JPEG:n02108422 +ILSVRC2012_val_00002615.JPEG:n03485407 +ILSVRC2012_val_00002616.JPEG:n02093647 +ILSVRC2012_val_00002617.JPEG:n03000134 +ILSVRC2012_val_00002618.JPEG:n03089624 +ILSVRC2012_val_00002619.JPEG:n07615774 +ILSVRC2012_val_00002620.JPEG:n03956157 +ILSVRC2012_val_00002621.JPEG:n02776631 +ILSVRC2012_val_00002622.JPEG:n01729977 +ILSVRC2012_val_00002623.JPEG:n03868242 +ILSVRC2012_val_00002624.JPEG:n03899768 +ILSVRC2012_val_00002625.JPEG:n01871265 +ILSVRC2012_val_00002626.JPEG:n03180011 +ILSVRC2012_val_00002627.JPEG:n03630383 +ILSVRC2012_val_00002628.JPEG:n01968897 +ILSVRC2012_val_00002629.JPEG:n02939185 +ILSVRC2012_val_00002630.JPEG:n02097474 +ILSVRC2012_val_00002631.JPEG:n04154565 +ILSVRC2012_val_00002632.JPEG:n04462240 +ILSVRC2012_val_00002633.JPEG:n02028035 +ILSVRC2012_val_00002634.JPEG:n04041544 +ILSVRC2012_val_00002635.JPEG:n02111129 +ILSVRC2012_val_00002636.JPEG:n03026506 +ILSVRC2012_val_00002637.JPEG:n04389033 +ILSVRC2012_val_00002638.JPEG:n02808440 +ILSVRC2012_val_00002639.JPEG:n03124170 +ILSVRC2012_val_00002640.JPEG:n02129165 +ILSVRC2012_val_00002641.JPEG:n02776631 +ILSVRC2012_val_00002642.JPEG:n04259630 +ILSVRC2012_val_00002643.JPEG:n03902125 +ILSVRC2012_val_00002644.JPEG:n07760859 +ILSVRC2012_val_00002645.JPEG:n01744401 +ILSVRC2012_val_00002646.JPEG:n02128757 +ILSVRC2012_val_00002647.JPEG:n02843684 +ILSVRC2012_val_00002648.JPEG:n02091134 +ILSVRC2012_val_00002649.JPEG:n02256656 +ILSVRC2012_val_00002650.JPEG:n03814639 +ILSVRC2012_val_00002651.JPEG:n02666196 +ILSVRC2012_val_00002652.JPEG:n02497673 +ILSVRC2012_val_00002653.JPEG:n13054560 +ILSVRC2012_val_00002654.JPEG:n01914609 +ILSVRC2012_val_00002655.JPEG:n01580077 +ILSVRC2012_val_00002656.JPEG:n02089867 +ILSVRC2012_val_00002657.JPEG:n03630383 +ILSVRC2012_val_00002658.JPEG:n02025239 +ILSVRC2012_val_00002659.JPEG:n02123597 +ILSVRC2012_val_00002660.JPEG:n02807133 +ILSVRC2012_val_00002661.JPEG:n03673027 +ILSVRC2012_val_00002662.JPEG:n04317175 +ILSVRC2012_val_00002663.JPEG:n15075141 +ILSVRC2012_val_00002664.JPEG:n01795545 +ILSVRC2012_val_00002665.JPEG:n03888257 +ILSVRC2012_val_00002666.JPEG:n03062245 +ILSVRC2012_val_00002667.JPEG:n04209133 +ILSVRC2012_val_00002668.JPEG:n01531178 +ILSVRC2012_val_00002669.JPEG:n02410509 +ILSVRC2012_val_00002670.JPEG:n04162706 +ILSVRC2012_val_00002671.JPEG:n03814639 +ILSVRC2012_val_00002672.JPEG:n02102177 +ILSVRC2012_val_00002673.JPEG:n04399382 +ILSVRC2012_val_00002674.JPEG:n03220513 +ILSVRC2012_val_00002675.JPEG:n06874185 +ILSVRC2012_val_00002676.JPEG:n04152593 +ILSVRC2012_val_00002677.JPEG:n07880968 +ILSVRC2012_val_00002678.JPEG:n02066245 +ILSVRC2012_val_00002679.JPEG:n01735189 +ILSVRC2012_val_00002680.JPEG:n03271574 +ILSVRC2012_val_00002681.JPEG:n01592084 +ILSVRC2012_val_00002682.JPEG:n04355933 +ILSVRC2012_val_00002683.JPEG:n02085936 +ILSVRC2012_val_00002684.JPEG:n01978455 +ILSVRC2012_val_00002685.JPEG:n04597913 +ILSVRC2012_val_00002686.JPEG:n07871810 +ILSVRC2012_val_00002687.JPEG:n02093859 +ILSVRC2012_val_00002688.JPEG:n01773549 +ILSVRC2012_val_00002689.JPEG:n03126707 +ILSVRC2012_val_00002690.JPEG:n03452741 +ILSVRC2012_val_00002691.JPEG:n02027492 +ILSVRC2012_val_00002692.JPEG:n02408429 +ILSVRC2012_val_00002693.JPEG:n01985128 +ILSVRC2012_val_00002694.JPEG:n03670208 +ILSVRC2012_val_00002695.JPEG:n04458633 +ILSVRC2012_val_00002696.JPEG:n04273569 +ILSVRC2012_val_00002697.JPEG:n03785016 +ILSVRC2012_val_00002698.JPEG:n01751748 +ILSVRC2012_val_00002699.JPEG:n03188531 +ILSVRC2012_val_00002700.JPEG:n02917067 +ILSVRC2012_val_00002701.JPEG:n02086240 +ILSVRC2012_val_00002702.JPEG:n03770439 +ILSVRC2012_val_00002703.JPEG:n03240683 +ILSVRC2012_val_00002704.JPEG:n03920288 +ILSVRC2012_val_00002705.JPEG:n03954731 +ILSVRC2012_val_00002706.JPEG:n02109525 +ILSVRC2012_val_00002707.JPEG:n03016953 +ILSVRC2012_val_00002708.JPEG:n02107683 +ILSVRC2012_val_00002709.JPEG:n01665541 +ILSVRC2012_val_00002710.JPEG:n04310018 +ILSVRC2012_val_00002711.JPEG:n03485407 +ILSVRC2012_val_00002712.JPEG:n03187595 +ILSVRC2012_val_00002713.JPEG:n03814639 +ILSVRC2012_val_00002714.JPEG:n02095570 +ILSVRC2012_val_00002715.JPEG:n01968897 +ILSVRC2012_val_00002716.JPEG:n03874599 +ILSVRC2012_val_00002717.JPEG:n02493509 +ILSVRC2012_val_00002718.JPEG:n02130308 +ILSVRC2012_val_00002719.JPEG:n02749479 +ILSVRC2012_val_00002720.JPEG:n01945685 +ILSVRC2012_val_00002721.JPEG:n02536864 +ILSVRC2012_val_00002722.JPEG:n04154565 +ILSVRC2012_val_00002723.JPEG:n02328150 +ILSVRC2012_val_00002724.JPEG:n03908618 +ILSVRC2012_val_00002725.JPEG:n01737021 +ILSVRC2012_val_00002726.JPEG:n02408429 +ILSVRC2012_val_00002727.JPEG:n02231487 +ILSVRC2012_val_00002728.JPEG:n04131690 +ILSVRC2012_val_00002729.JPEG:n03970156 +ILSVRC2012_val_00002730.JPEG:n01530575 +ILSVRC2012_val_00002731.JPEG:n04336792 +ILSVRC2012_val_00002732.JPEG:n02951358 +ILSVRC2012_val_00002733.JPEG:n02879718 +ILSVRC2012_val_00002734.JPEG:n03944341 +ILSVRC2012_val_00002735.JPEG:n03788195 +ILSVRC2012_val_00002736.JPEG:n02895154 +ILSVRC2012_val_00002737.JPEG:n03838899 +ILSVRC2012_val_00002738.JPEG:n02037110 +ILSVRC2012_val_00002739.JPEG:n04009552 +ILSVRC2012_val_00002740.JPEG:n03141823 +ILSVRC2012_val_00002741.JPEG:n02102973 +ILSVRC2012_val_00002742.JPEG:n07730033 +ILSVRC2012_val_00002743.JPEG:n01984695 +ILSVRC2012_val_00002744.JPEG:n07693725 +ILSVRC2012_val_00002745.JPEG:n04065272 +ILSVRC2012_val_00002746.JPEG:n01631663 +ILSVRC2012_val_00002747.JPEG:n02699494 +ILSVRC2012_val_00002748.JPEG:n03095699 +ILSVRC2012_val_00002749.JPEG:n02112350 +ILSVRC2012_val_00002750.JPEG:n04019541 +ILSVRC2012_val_00002751.JPEG:n09835506 +ILSVRC2012_val_00002752.JPEG:n01484850 +ILSVRC2012_val_00002753.JPEG:n07697313 +ILSVRC2012_val_00002754.JPEG:n01729322 +ILSVRC2012_val_00002755.JPEG:n03085013 +ILSVRC2012_val_00002756.JPEG:n04041544 +ILSVRC2012_val_00002757.JPEG:n02396427 +ILSVRC2012_val_00002758.JPEG:n02879718 +ILSVRC2012_val_00002759.JPEG:n03891332 +ILSVRC2012_val_00002760.JPEG:n04590129 +ILSVRC2012_val_00002761.JPEG:n03271574 +ILSVRC2012_val_00002762.JPEG:n02454379 +ILSVRC2012_val_00002763.JPEG:n01944390 +ILSVRC2012_val_00002764.JPEG:n02099267 +ILSVRC2012_val_00002765.JPEG:n02097658 +ILSVRC2012_val_00002766.JPEG:n07720875 +ILSVRC2012_val_00002767.JPEG:n02484975 +ILSVRC2012_val_00002768.JPEG:n03733805 +ILSVRC2012_val_00002769.JPEG:n02086240 +ILSVRC2012_val_00002770.JPEG:n04204238 +ILSVRC2012_val_00002771.JPEG:n03483316 +ILSVRC2012_val_00002772.JPEG:n03201208 +ILSVRC2012_val_00002773.JPEG:n02095570 +ILSVRC2012_val_00002774.JPEG:n01630670 +ILSVRC2012_val_00002775.JPEG:n03201208 +ILSVRC2012_val_00002776.JPEG:n01755581 +ILSVRC2012_val_00002777.JPEG:n02879718 +ILSVRC2012_val_00002778.JPEG:n03065424 +ILSVRC2012_val_00002779.JPEG:n02037110 +ILSVRC2012_val_00002780.JPEG:n02108915 +ILSVRC2012_val_00002781.JPEG:n02807133 +ILSVRC2012_val_00002782.JPEG:n04023962 +ILSVRC2012_val_00002783.JPEG:n01669191 +ILSVRC2012_val_00002784.JPEG:n02098286 +ILSVRC2012_val_00002785.JPEG:n04252225 +ILSVRC2012_val_00002786.JPEG:n02115641 +ILSVRC2012_val_00002787.JPEG:n02281787 +ILSVRC2012_val_00002788.JPEG:n06794110 +ILSVRC2012_val_00002789.JPEG:n02391049 +ILSVRC2012_val_00002790.JPEG:n04486054 +ILSVRC2012_val_00002791.JPEG:n01817953 +ILSVRC2012_val_00002792.JPEG:n04041544 +ILSVRC2012_val_00002793.JPEG:n04277352 +ILSVRC2012_val_00002794.JPEG:n02107574 +ILSVRC2012_val_00002795.JPEG:n09193705 +ILSVRC2012_val_00002796.JPEG:n04371774 +ILSVRC2012_val_00002797.JPEG:n04372370 +ILSVRC2012_val_00002798.JPEG:n03724870 +ILSVRC2012_val_00002799.JPEG:n03388183 +ILSVRC2012_val_00002800.JPEG:n04371430 +ILSVRC2012_val_00002801.JPEG:n02788148 +ILSVRC2012_val_00002802.JPEG:n01817953 +ILSVRC2012_val_00002803.JPEG:n02699494 +ILSVRC2012_val_00002804.JPEG:n07730033 +ILSVRC2012_val_00002805.JPEG:n09468604 +ILSVRC2012_val_00002806.JPEG:n04254777 +ILSVRC2012_val_00002807.JPEG:n04501370 +ILSVRC2012_val_00002808.JPEG:n03637318 +ILSVRC2012_val_00002809.JPEG:n02782093 +ILSVRC2012_val_00002810.JPEG:n04152593 +ILSVRC2012_val_00002811.JPEG:n01882714 +ILSVRC2012_val_00002812.JPEG:n02916936 +ILSVRC2012_val_00002813.JPEG:n03661043 +ILSVRC2012_val_00002814.JPEG:n04336792 +ILSVRC2012_val_00002815.JPEG:n02422699 +ILSVRC2012_val_00002816.JPEG:n04019541 +ILSVRC2012_val_00002817.JPEG:n01664065 +ILSVRC2012_val_00002818.JPEG:n03325584 +ILSVRC2012_val_00002819.JPEG:n03976657 +ILSVRC2012_val_00002820.JPEG:n04423845 +ILSVRC2012_val_00002821.JPEG:n04404412 +ILSVRC2012_val_00002822.JPEG:n03527444 +ILSVRC2012_val_00002823.JPEG:n02123045 +ILSVRC2012_val_00002824.JPEG:n02094114 +ILSVRC2012_val_00002825.JPEG:n01558993 +ILSVRC2012_val_00002826.JPEG:n03062245 +ILSVRC2012_val_00002827.JPEG:n02113712 +ILSVRC2012_val_00002828.JPEG:n03662601 +ILSVRC2012_val_00002829.JPEG:n03065424 +ILSVRC2012_val_00002830.JPEG:n03388183 +ILSVRC2012_val_00002831.JPEG:n03447721 +ILSVRC2012_val_00002832.JPEG:n01667778 +ILSVRC2012_val_00002833.JPEG:n03584254 +ILSVRC2012_val_00002834.JPEG:n03000247 +ILSVRC2012_val_00002835.JPEG:n07718747 +ILSVRC2012_val_00002836.JPEG:n01737021 +ILSVRC2012_val_00002837.JPEG:n02676566 +ILSVRC2012_val_00002838.JPEG:n01795545 +ILSVRC2012_val_00002839.JPEG:n07860988 +ILSVRC2012_val_00002840.JPEG:n04086273 +ILSVRC2012_val_00002841.JPEG:n04332243 +ILSVRC2012_val_00002842.JPEG:n03447721 +ILSVRC2012_val_00002843.JPEG:n01829413 +ILSVRC2012_val_00002844.JPEG:n02236044 +ILSVRC2012_val_00002845.JPEG:n02165105 +ILSVRC2012_val_00002846.JPEG:n01796340 +ILSVRC2012_val_00002847.JPEG:n02092339 +ILSVRC2012_val_00002848.JPEG:n01443537 +ILSVRC2012_val_00002849.JPEG:n04370456 +ILSVRC2012_val_00002850.JPEG:n03961711 +ILSVRC2012_val_00002851.JPEG:n07579787 +ILSVRC2012_val_00002852.JPEG:n01753488 +ILSVRC2012_val_00002853.JPEG:n02708093 +ILSVRC2012_val_00002854.JPEG:n02111277 +ILSVRC2012_val_00002855.JPEG:n01774750 +ILSVRC2012_val_00002856.JPEG:n04286575 +ILSVRC2012_val_00002857.JPEG:n02483708 +ILSVRC2012_val_00002858.JPEG:n02002724 +ILSVRC2012_val_00002859.JPEG:n02536864 +ILSVRC2012_val_00002860.JPEG:n03400231 +ILSVRC2012_val_00002861.JPEG:n03485794 +ILSVRC2012_val_00002862.JPEG:n02480495 +ILSVRC2012_val_00002863.JPEG:n02509815 +ILSVRC2012_val_00002864.JPEG:n04111531 +ILSVRC2012_val_00002865.JPEG:n07716358 +ILSVRC2012_val_00002866.JPEG:n01968897 +ILSVRC2012_val_00002867.JPEG:n04579145 +ILSVRC2012_val_00002868.JPEG:n02892201 +ILSVRC2012_val_00002869.JPEG:n02091134 +ILSVRC2012_val_00002870.JPEG:n04118776 +ILSVRC2012_val_00002871.JPEG:n03249569 +ILSVRC2012_val_00002872.JPEG:n01601694 +ILSVRC2012_val_00002873.JPEG:n04522168 +ILSVRC2012_val_00002874.JPEG:n02441942 +ILSVRC2012_val_00002875.JPEG:n03271574 +ILSVRC2012_val_00002876.JPEG:n02692877 +ILSVRC2012_val_00002877.JPEG:n03930313 +ILSVRC2012_val_00002878.JPEG:n02100735 +ILSVRC2012_val_00002879.JPEG:n04428191 +ILSVRC2012_val_00002880.JPEG:n03706229 +ILSVRC2012_val_00002881.JPEG:n02119789 +ILSVRC2012_val_00002882.JPEG:n02111277 +ILSVRC2012_val_00002883.JPEG:n01629819 +ILSVRC2012_val_00002884.JPEG:n04476259 +ILSVRC2012_val_00002885.JPEG:n03958227 +ILSVRC2012_val_00002886.JPEG:n03240683 +ILSVRC2012_val_00002887.JPEG:n02504458 +ILSVRC2012_val_00002888.JPEG:n04461696 +ILSVRC2012_val_00002889.JPEG:n09229709 +ILSVRC2012_val_00002890.JPEG:n01728920 +ILSVRC2012_val_00002891.JPEG:n02422106 +ILSVRC2012_val_00002892.JPEG:n03450230 +ILSVRC2012_val_00002893.JPEG:n02268853 +ILSVRC2012_val_00002894.JPEG:n03902125 +ILSVRC2012_val_00002895.JPEG:n03868863 +ILSVRC2012_val_00002896.JPEG:n09428293 +ILSVRC2012_val_00002897.JPEG:n04482393 +ILSVRC2012_val_00002898.JPEG:n03680355 +ILSVRC2012_val_00002899.JPEG:n01744401 +ILSVRC2012_val_00002900.JPEG:n12620546 +ILSVRC2012_val_00002901.JPEG:n02002556 +ILSVRC2012_val_00002902.JPEG:n04136333 +ILSVRC2012_val_00002903.JPEG:n02447366 +ILSVRC2012_val_00002904.JPEG:n02226429 +ILSVRC2012_val_00002905.JPEG:n03249569 +ILSVRC2012_val_00002906.JPEG:n02281406 +ILSVRC2012_val_00002907.JPEG:n03721384 +ILSVRC2012_val_00002908.JPEG:n03874599 +ILSVRC2012_val_00002909.JPEG:n02951585 +ILSVRC2012_val_00002910.JPEG:n04074963 +ILSVRC2012_val_00002911.JPEG:n02480495 +ILSVRC2012_val_00002912.JPEG:n03929855 +ILSVRC2012_val_00002913.JPEG:n03016953 +ILSVRC2012_val_00002914.JPEG:n03376595 +ILSVRC2012_val_00002915.JPEG:n07747607 +ILSVRC2012_val_00002916.JPEG:n15075141 +ILSVRC2012_val_00002917.JPEG:n02085620 +ILSVRC2012_val_00002918.JPEG:n04141975 +ILSVRC2012_val_00002919.JPEG:n03733805 +ILSVRC2012_val_00002920.JPEG:n03670208 +ILSVRC2012_val_00002921.JPEG:n02085620 +ILSVRC2012_val_00002922.JPEG:n01491361 +ILSVRC2012_val_00002923.JPEG:n03803284 +ILSVRC2012_val_00002924.JPEG:n02415577 +ILSVRC2012_val_00002925.JPEG:n07714571 +ILSVRC2012_val_00002926.JPEG:n03929855 +ILSVRC2012_val_00002927.JPEG:n13037406 +ILSVRC2012_val_00002928.JPEG:n01740131 +ILSVRC2012_val_00002929.JPEG:n01580077 +ILSVRC2012_val_00002930.JPEG:n03891251 +ILSVRC2012_val_00002931.JPEG:n02128925 +ILSVRC2012_val_00002932.JPEG:n01664065 +ILSVRC2012_val_00002933.JPEG:n02090379 +ILSVRC2012_val_00002934.JPEG:n07920052 +ILSVRC2012_val_00002935.JPEG:n02279972 +ILSVRC2012_val_00002936.JPEG:n02490219 +ILSVRC2012_val_00002937.JPEG:n02906734 +ILSVRC2012_val_00002938.JPEG:n01914609 +ILSVRC2012_val_00002939.JPEG:n01704323 +ILSVRC2012_val_00002940.JPEG:n02105412 +ILSVRC2012_val_00002941.JPEG:n03492542 +ILSVRC2012_val_00002942.JPEG:n04482393 +ILSVRC2012_val_00002943.JPEG:n02788148 +ILSVRC2012_val_00002944.JPEG:n01985128 +ILSVRC2012_val_00002945.JPEG:n03388549 +ILSVRC2012_val_00002946.JPEG:n04251144 +ILSVRC2012_val_00002947.JPEG:n02939185 +ILSVRC2012_val_00002948.JPEG:n02114548 +ILSVRC2012_val_00002949.JPEG:n07836838 +ILSVRC2012_val_00002950.JPEG:n10148035 +ILSVRC2012_val_00002951.JPEG:n03976467 +ILSVRC2012_val_00002952.JPEG:n03447721 +ILSVRC2012_val_00002953.JPEG:n02006656 +ILSVRC2012_val_00002954.JPEG:n07802026 +ILSVRC2012_val_00002955.JPEG:n04370456 +ILSVRC2012_val_00002956.JPEG:n02417914 +ILSVRC2012_val_00002957.JPEG:n01776313 +ILSVRC2012_val_00002958.JPEG:n02112018 +ILSVRC2012_val_00002959.JPEG:n03938244 +ILSVRC2012_val_00002960.JPEG:n02536864 +ILSVRC2012_val_00002961.JPEG:n07802026 +ILSVRC2012_val_00002962.JPEG:n04501370 +ILSVRC2012_val_00002963.JPEG:n02963159 +ILSVRC2012_val_00002964.JPEG:n03759954 +ILSVRC2012_val_00002965.JPEG:n02028035 +ILSVRC2012_val_00002966.JPEG:n04044716 +ILSVRC2012_val_00002967.JPEG:n02123394 +ILSVRC2012_val_00002968.JPEG:n02823428 +ILSVRC2012_val_00002969.JPEG:n01491361 +ILSVRC2012_val_00002970.JPEG:n04008634 +ILSVRC2012_val_00002971.JPEG:n01877812 +ILSVRC2012_val_00002972.JPEG:n07615774 +ILSVRC2012_val_00002973.JPEG:n09256479 +ILSVRC2012_val_00002974.JPEG:n01833805 +ILSVRC2012_val_00002975.JPEG:n04127249 +ILSVRC2012_val_00002976.JPEG:n04507155 +ILSVRC2012_val_00002977.JPEG:n03673027 +ILSVRC2012_val_00002978.JPEG:n01882714 +ILSVRC2012_val_00002979.JPEG:n03697007 +ILSVRC2012_val_00002980.JPEG:n03637318 +ILSVRC2012_val_00002981.JPEG:n04332243 +ILSVRC2012_val_00002982.JPEG:n12267677 +ILSVRC2012_val_00002983.JPEG:n07714571 +ILSVRC2012_val_00002984.JPEG:n03485794 +ILSVRC2012_val_00002985.JPEG:n04004767 +ILSVRC2012_val_00002986.JPEG:n02795169 +ILSVRC2012_val_00002987.JPEG:n02120505 +ILSVRC2012_val_00002988.JPEG:n02086646 +ILSVRC2012_val_00002989.JPEG:n02107908 +ILSVRC2012_val_00002990.JPEG:n03888257 +ILSVRC2012_val_00002991.JPEG:n01795545 +ILSVRC2012_val_00002992.JPEG:n03272010 +ILSVRC2012_val_00002993.JPEG:n07714571 +ILSVRC2012_val_00002994.JPEG:n02097047 +ILSVRC2012_val_00002995.JPEG:n03874293 +ILSVRC2012_val_00002996.JPEG:n02391049 +ILSVRC2012_val_00002997.JPEG:n01855672 +ILSVRC2012_val_00002998.JPEG:n01871265 +ILSVRC2012_val_00002999.JPEG:n04208210 +ILSVRC2012_val_00003000.JPEG:n02487347 +ILSVRC2012_val_00003001.JPEG:n02013706 +ILSVRC2012_val_00003002.JPEG:n02096051 +ILSVRC2012_val_00003003.JPEG:n03598930 +ILSVRC2012_val_00003004.JPEG:n03873416 +ILSVRC2012_val_00003005.JPEG:n02871525 +ILSVRC2012_val_00003006.JPEG:n02102973 +ILSVRC2012_val_00003007.JPEG:n03710637 +ILSVRC2012_val_00003008.JPEG:n01773157 +ILSVRC2012_val_00003009.JPEG:n03208938 +ILSVRC2012_val_00003010.JPEG:n04325704 +ILSVRC2012_val_00003011.JPEG:n02002724 +ILSVRC2012_val_00003012.JPEG:n02137549 +ILSVRC2012_val_00003013.JPEG:n02125311 +ILSVRC2012_val_00003014.JPEG:n01440764 +ILSVRC2012_val_00003015.JPEG:n01806567 +ILSVRC2012_val_00003016.JPEG:n03345487 +ILSVRC2012_val_00003017.JPEG:n04209239 +ILSVRC2012_val_00003018.JPEG:n07860988 +ILSVRC2012_val_00003019.JPEG:n07802026 +ILSVRC2012_val_00003020.JPEG:n07714571 +ILSVRC2012_val_00003021.JPEG:n12768682 +ILSVRC2012_val_00003022.JPEG:n02108422 +ILSVRC2012_val_00003023.JPEG:n01770393 +ILSVRC2012_val_00003024.JPEG:n03124043 +ILSVRC2012_val_00003025.JPEG:n04023962 +ILSVRC2012_val_00003026.JPEG:n02105056 +ILSVRC2012_val_00003027.JPEG:n04476259 +ILSVRC2012_val_00003028.JPEG:n02871525 +ILSVRC2012_val_00003029.JPEG:n03598930 +ILSVRC2012_val_00003030.JPEG:n02206856 +ILSVRC2012_val_00003031.JPEG:n03223299 +ILSVRC2012_val_00003032.JPEG:n02259212 +ILSVRC2012_val_00003033.JPEG:n02607072 +ILSVRC2012_val_00003034.JPEG:n02834397 +ILSVRC2012_val_00003035.JPEG:n02364673 +ILSVRC2012_val_00003036.JPEG:n03131574 +ILSVRC2012_val_00003037.JPEG:n02802426 +ILSVRC2012_val_00003038.JPEG:n02117135 +ILSVRC2012_val_00003039.JPEG:n04370456 +ILSVRC2012_val_00003040.JPEG:n01829413 +ILSVRC2012_val_00003041.JPEG:n04033901 +ILSVRC2012_val_00003042.JPEG:n02123159 +ILSVRC2012_val_00003043.JPEG:n02794156 +ILSVRC2012_val_00003044.JPEG:n02132136 +ILSVRC2012_val_00003045.JPEG:n02883205 +ILSVRC2012_val_00003046.JPEG:n07720875 +ILSVRC2012_val_00003047.JPEG:n03920288 +ILSVRC2012_val_00003048.JPEG:n02892201 +ILSVRC2012_val_00003049.JPEG:n04285008 +ILSVRC2012_val_00003050.JPEG:n03345487 +ILSVRC2012_val_00003051.JPEG:n03661043 +ILSVRC2012_val_00003052.JPEG:n04423845 +ILSVRC2012_val_00003053.JPEG:n02013706 +ILSVRC2012_val_00003054.JPEG:n01924916 +ILSVRC2012_val_00003055.JPEG:n03095699 +ILSVRC2012_val_00003056.JPEG:n09428293 +ILSVRC2012_val_00003057.JPEG:n04153751 +ILSVRC2012_val_00003058.JPEG:n02865351 +ILSVRC2012_val_00003059.JPEG:n03384352 +ILSVRC2012_val_00003060.JPEG:n02786058 +ILSVRC2012_val_00003061.JPEG:n02099429 +ILSVRC2012_val_00003062.JPEG:n03014705 +ILSVRC2012_val_00003063.JPEG:n02113712 +ILSVRC2012_val_00003064.JPEG:n01833805 +ILSVRC2012_val_00003065.JPEG:n03924679 +ILSVRC2012_val_00003066.JPEG:n03937543 +ILSVRC2012_val_00003067.JPEG:n02892767 +ILSVRC2012_val_00003068.JPEG:n01819313 +ILSVRC2012_val_00003069.JPEG:n02109047 +ILSVRC2012_val_00003070.JPEG:n01694178 +ILSVRC2012_val_00003071.JPEG:n01729322 +ILSVRC2012_val_00003072.JPEG:n02808440 +ILSVRC2012_val_00003073.JPEG:n04266014 +ILSVRC2012_val_00003074.JPEG:n01978287 +ILSVRC2012_val_00003075.JPEG:n04111531 +ILSVRC2012_val_00003076.JPEG:n04540053 +ILSVRC2012_val_00003077.JPEG:n02100735 +ILSVRC2012_val_00003078.JPEG:n03935335 +ILSVRC2012_val_00003079.JPEG:n04372370 +ILSVRC2012_val_00003080.JPEG:n03930630 +ILSVRC2012_val_00003081.JPEG:n02443114 +ILSVRC2012_val_00003082.JPEG:n03854065 +ILSVRC2012_val_00003083.JPEG:n03724870 +ILSVRC2012_val_00003084.JPEG:n09193705 +ILSVRC2012_val_00003085.JPEG:n02640242 +ILSVRC2012_val_00003086.JPEG:n03967562 +ILSVRC2012_val_00003087.JPEG:n07711569 +ILSVRC2012_val_00003088.JPEG:n04147183 +ILSVRC2012_val_00003089.JPEG:n03710721 +ILSVRC2012_val_00003090.JPEG:n02965783 +ILSVRC2012_val_00003091.JPEG:n02951585 +ILSVRC2012_val_00003092.JPEG:n01582220 +ILSVRC2012_val_00003093.JPEG:n03014705 +ILSVRC2012_val_00003094.JPEG:n02643566 +ILSVRC2012_val_00003095.JPEG:n01739381 +ILSVRC2012_val_00003096.JPEG:n03814906 +ILSVRC2012_val_00003097.JPEG:n01882714 +ILSVRC2012_val_00003098.JPEG:n01729322 +ILSVRC2012_val_00003099.JPEG:n02860847 +ILSVRC2012_val_00003100.JPEG:n04350905 +ILSVRC2012_val_00003101.JPEG:n01697457 +ILSVRC2012_val_00003102.JPEG:n03220513 +ILSVRC2012_val_00003103.JPEG:n04311004 +ILSVRC2012_val_00003104.JPEG:n03877472 +ILSVRC2012_val_00003105.JPEG:n04209239 +ILSVRC2012_val_00003106.JPEG:n04149813 +ILSVRC2012_val_00003107.JPEG:n03770679 +ILSVRC2012_val_00003108.JPEG:n04548362 +ILSVRC2012_val_00003109.JPEG:n07930864 +ILSVRC2012_val_00003110.JPEG:n03661043 +ILSVRC2012_val_00003111.JPEG:n03400231 +ILSVRC2012_val_00003112.JPEG:n02930766 +ILSVRC2012_val_00003113.JPEG:n04613696 +ILSVRC2012_val_00003114.JPEG:n03866082 +ILSVRC2012_val_00003115.JPEG:n01990800 +ILSVRC2012_val_00003116.JPEG:n01534433 +ILSVRC2012_val_00003117.JPEG:n03947888 +ILSVRC2012_val_00003118.JPEG:n02492660 +ILSVRC2012_val_00003119.JPEG:n01985128 +ILSVRC2012_val_00003120.JPEG:n03793489 +ILSVRC2012_val_00003121.JPEG:n03977966 +ILSVRC2012_val_00003122.JPEG:n01795545 +ILSVRC2012_val_00003123.JPEG:n04086273 +ILSVRC2012_val_00003124.JPEG:n01688243 +ILSVRC2012_val_00003125.JPEG:n02423022 +ILSVRC2012_val_00003126.JPEG:n04277352 +ILSVRC2012_val_00003127.JPEG:n03877472 +ILSVRC2012_val_00003128.JPEG:n03208938 +ILSVRC2012_val_00003129.JPEG:n04476259 +ILSVRC2012_val_00003130.JPEG:n04550184 +ILSVRC2012_val_00003131.JPEG:n03063599 +ILSVRC2012_val_00003132.JPEG:n04523525 +ILSVRC2012_val_00003133.JPEG:n02123597 +ILSVRC2012_val_00003134.JPEG:n02708093 +ILSVRC2012_val_00003135.JPEG:n02134418 +ILSVRC2012_val_00003136.JPEG:n02086079 +ILSVRC2012_val_00003137.JPEG:n11879895 +ILSVRC2012_val_00003138.JPEG:n03676483 +ILSVRC2012_val_00003139.JPEG:n02107574 +ILSVRC2012_val_00003140.JPEG:n02113978 +ILSVRC2012_val_00003141.JPEG:n03764736 +ILSVRC2012_val_00003142.JPEG:n03642806 +ILSVRC2012_val_00003143.JPEG:n01748264 +ILSVRC2012_val_00003144.JPEG:n02167151 +ILSVRC2012_val_00003145.JPEG:n04612504 +ILSVRC2012_val_00003146.JPEG:n02817516 +ILSVRC2012_val_00003147.JPEG:n02051845 +ILSVRC2012_val_00003148.JPEG:n03724870 +ILSVRC2012_val_00003149.JPEG:n02077923 +ILSVRC2012_val_00003150.JPEG:n01443537 +ILSVRC2012_val_00003151.JPEG:n03065424 +ILSVRC2012_val_00003152.JPEG:n02105505 +ILSVRC2012_val_00003153.JPEG:n02051845 +ILSVRC2012_val_00003154.JPEG:n02087394 +ILSVRC2012_val_00003155.JPEG:n01735189 +ILSVRC2012_val_00003156.JPEG:n04310018 +ILSVRC2012_val_00003157.JPEG:n01632458 +ILSVRC2012_val_00003158.JPEG:n02509815 +ILSVRC2012_val_00003159.JPEG:n02093859 +ILSVRC2012_val_00003160.JPEG:n01669191 +ILSVRC2012_val_00003161.JPEG:n03868242 +ILSVRC2012_val_00003162.JPEG:n03400231 +ILSVRC2012_val_00003163.JPEG:n02423022 +ILSVRC2012_val_00003164.JPEG:n02090622 +ILSVRC2012_val_00003165.JPEG:n03146219 +ILSVRC2012_val_00003166.JPEG:n02397096 +ILSVRC2012_val_00003167.JPEG:n03532672 +ILSVRC2012_val_00003168.JPEG:n02013706 +ILSVRC2012_val_00003169.JPEG:n01622779 +ILSVRC2012_val_00003170.JPEG:n02483708 +ILSVRC2012_val_00003171.JPEG:n03187595 +ILSVRC2012_val_00003172.JPEG:n02114712 +ILSVRC2012_val_00003173.JPEG:n03131574 +ILSVRC2012_val_00003174.JPEG:n03476991 +ILSVRC2012_val_00003175.JPEG:n03838899 +ILSVRC2012_val_00003176.JPEG:n02105162 +ILSVRC2012_val_00003177.JPEG:n04604644 +ILSVRC2012_val_00003178.JPEG:n01689811 +ILSVRC2012_val_00003179.JPEG:n02113624 +ILSVRC2012_val_00003180.JPEG:n03691459 +ILSVRC2012_val_00003181.JPEG:n15075141 +ILSVRC2012_val_00003182.JPEG:n01773797 +ILSVRC2012_val_00003183.JPEG:n01491361 +ILSVRC2012_val_00003184.JPEG:n04209133 +ILSVRC2012_val_00003185.JPEG:n04476259 +ILSVRC2012_val_00003186.JPEG:n03444034 +ILSVRC2012_val_00003187.JPEG:n02488291 +ILSVRC2012_val_00003188.JPEG:n03485407 +ILSVRC2012_val_00003189.JPEG:n01630670 +ILSVRC2012_val_00003190.JPEG:n04599235 +ILSVRC2012_val_00003191.JPEG:n02174001 +ILSVRC2012_val_00003192.JPEG:n02834397 +ILSVRC2012_val_00003193.JPEG:n02509815 +ILSVRC2012_val_00003194.JPEG:n03538406 +ILSVRC2012_val_00003195.JPEG:n03535780 +ILSVRC2012_val_00003196.JPEG:n02105855 +ILSVRC2012_val_00003197.JPEG:n04501370 +ILSVRC2012_val_00003198.JPEG:n02098105 +ILSVRC2012_val_00003199.JPEG:n03763968 +ILSVRC2012_val_00003200.JPEG:n03095699 +ILSVRC2012_val_00003201.JPEG:n04591713 +ILSVRC2012_val_00003202.JPEG:n02363005 +ILSVRC2012_val_00003203.JPEG:n03599486 +ILSVRC2012_val_00003204.JPEG:n01491361 +ILSVRC2012_val_00003205.JPEG:n02090622 +ILSVRC2012_val_00003206.JPEG:n03590841 +ILSVRC2012_val_00003207.JPEG:n03832673 +ILSVRC2012_val_00003208.JPEG:n02013706 +ILSVRC2012_val_00003209.JPEG:n06874185 +ILSVRC2012_val_00003210.JPEG:n06596364 +ILSVRC2012_val_00003211.JPEG:n04074963 +ILSVRC2012_val_00003212.JPEG:n04389033 +ILSVRC2012_val_00003213.JPEG:n02447366 +ILSVRC2012_val_00003214.JPEG:n01631663 +ILSVRC2012_val_00003215.JPEG:n02841315 +ILSVRC2012_val_00003216.JPEG:n03733805 +ILSVRC2012_val_00003217.JPEG:n03146219 +ILSVRC2012_val_00003218.JPEG:n02974003 +ILSVRC2012_val_00003219.JPEG:n03947888 +ILSVRC2012_val_00003220.JPEG:n02095570 +ILSVRC2012_val_00003221.JPEG:n02422106 +ILSVRC2012_val_00003222.JPEG:n04049303 +ILSVRC2012_val_00003223.JPEG:n02396427 +ILSVRC2012_val_00003224.JPEG:n03891251 +ILSVRC2012_val_00003225.JPEG:n02422106 +ILSVRC2012_val_00003226.JPEG:n04486054 +ILSVRC2012_val_00003227.JPEG:n02091831 +ILSVRC2012_val_00003228.JPEG:n07760859 +ILSVRC2012_val_00003229.JPEG:n03179701 +ILSVRC2012_val_00003230.JPEG:n03947888 +ILSVRC2012_val_00003231.JPEG:n03692522 +ILSVRC2012_val_00003232.JPEG:n02097298 +ILSVRC2012_val_00003233.JPEG:n03602883 +ILSVRC2012_val_00003234.JPEG:n02974003 +ILSVRC2012_val_00003235.JPEG:n02951585 +ILSVRC2012_val_00003236.JPEG:n04141327 +ILSVRC2012_val_00003237.JPEG:n04357314 +ILSVRC2012_val_00003238.JPEG:n02786058 +ILSVRC2012_val_00003239.JPEG:n02268853 +ILSVRC2012_val_00003240.JPEG:n04596742 +ILSVRC2012_val_00003241.JPEG:n03788365 +ILSVRC2012_val_00003242.JPEG:n02111277 +ILSVRC2012_val_00003243.JPEG:n02104365 +ILSVRC2012_val_00003244.JPEG:n03584254 +ILSVRC2012_val_00003245.JPEG:n04509417 +ILSVRC2012_val_00003246.JPEG:n03494278 +ILSVRC2012_val_00003247.JPEG:n02939185 +ILSVRC2012_val_00003248.JPEG:n02363005 +ILSVRC2012_val_00003249.JPEG:n03047690 +ILSVRC2012_val_00003250.JPEG:n04366367 +ILSVRC2012_val_00003251.JPEG:n04409515 +ILSVRC2012_val_00003252.JPEG:n04380533 +ILSVRC2012_val_00003253.JPEG:n03187595 +ILSVRC2012_val_00003254.JPEG:n01882714 +ILSVRC2012_val_00003255.JPEG:n03680355 +ILSVRC2012_val_00003256.JPEG:n03124170 +ILSVRC2012_val_00003257.JPEG:n01986214 +ILSVRC2012_val_00003258.JPEG:n04004767 +ILSVRC2012_val_00003259.JPEG:n01833805 +ILSVRC2012_val_00003260.JPEG:n04141076 +ILSVRC2012_val_00003261.JPEG:n02033041 +ILSVRC2012_val_00003262.JPEG:n03109150 +ILSVRC2012_val_00003263.JPEG:n04560804 +ILSVRC2012_val_00003264.JPEG:n07930864 +ILSVRC2012_val_00003265.JPEG:n02114548 +ILSVRC2012_val_00003266.JPEG:n02877765 +ILSVRC2012_val_00003267.JPEG:n02093754 +ILSVRC2012_val_00003268.JPEG:n01737021 +ILSVRC2012_val_00003269.JPEG:n02093647 +ILSVRC2012_val_00003270.JPEG:n03794056 +ILSVRC2012_val_00003271.JPEG:n01843383 +ILSVRC2012_val_00003272.JPEG:n01978287 +ILSVRC2012_val_00003273.JPEG:n01669191 +ILSVRC2012_val_00003274.JPEG:n02870880 +ILSVRC2012_val_00003275.JPEG:n02071294 +ILSVRC2012_val_00003276.JPEG:n02098286 +ILSVRC2012_val_00003277.JPEG:n04120489 +ILSVRC2012_val_00003278.JPEG:n04239074 +ILSVRC2012_val_00003279.JPEG:n01537544 +ILSVRC2012_val_00003280.JPEG:n02504013 +ILSVRC2012_val_00003281.JPEG:n03929855 +ILSVRC2012_val_00003282.JPEG:n09193705 +ILSVRC2012_val_00003283.JPEG:n03534580 +ILSVRC2012_val_00003284.JPEG:n03018349 +ILSVRC2012_val_00003285.JPEG:n04179913 +ILSVRC2012_val_00003286.JPEG:n01735189 +ILSVRC2012_val_00003287.JPEG:n01665541 +ILSVRC2012_val_00003288.JPEG:n12768682 +ILSVRC2012_val_00003289.JPEG:n02669723 +ILSVRC2012_val_00003290.JPEG:n03930313 +ILSVRC2012_val_00003291.JPEG:n04200800 +ILSVRC2012_val_00003292.JPEG:n02363005 +ILSVRC2012_val_00003293.JPEG:n04552348 +ILSVRC2012_val_00003294.JPEG:n03992509 +ILSVRC2012_val_00003295.JPEG:n02123159 +ILSVRC2012_val_00003296.JPEG:n04505470 +ILSVRC2012_val_00003297.JPEG:n01518878 +ILSVRC2012_val_00003298.JPEG:n01742172 +ILSVRC2012_val_00003299.JPEG:n02445715 +ILSVRC2012_val_00003300.JPEG:n03584254 +ILSVRC2012_val_00003301.JPEG:n02101556 +ILSVRC2012_val_00003302.JPEG:n02398521 +ILSVRC2012_val_00003303.JPEG:n02106166 +ILSVRC2012_val_00003304.JPEG:n04372370 +ILSVRC2012_val_00003305.JPEG:n04346328 +ILSVRC2012_val_00003306.JPEG:n02109047 +ILSVRC2012_val_00003307.JPEG:n03498962 +ILSVRC2012_val_00003308.JPEG:n01980166 +ILSVRC2012_val_00003309.JPEG:n07753275 +ILSVRC2012_val_00003310.JPEG:n04447861 +ILSVRC2012_val_00003311.JPEG:n09332890 +ILSVRC2012_val_00003312.JPEG:n04417672 +ILSVRC2012_val_00003313.JPEG:n07248320 +ILSVRC2012_val_00003314.JPEG:n02412080 +ILSVRC2012_val_00003315.JPEG:n03218198 +ILSVRC2012_val_00003316.JPEG:n04428191 +ILSVRC2012_val_00003317.JPEG:n04447861 +ILSVRC2012_val_00003318.JPEG:n04557648 +ILSVRC2012_val_00003319.JPEG:n01677366 +ILSVRC2012_val_00003320.JPEG:n01774750 +ILSVRC2012_val_00003321.JPEG:n09399592 +ILSVRC2012_val_00003322.JPEG:n02859443 +ILSVRC2012_val_00003323.JPEG:n04456115 +ILSVRC2012_val_00003324.JPEG:n02018795 +ILSVRC2012_val_00003325.JPEG:n03935335 +ILSVRC2012_val_00003326.JPEG:n04465501 +ILSVRC2012_val_00003327.JPEG:n02112706 +ILSVRC2012_val_00003328.JPEG:n02799071 +ILSVRC2012_val_00003329.JPEG:n07684084 +ILSVRC2012_val_00003330.JPEG:n01614925 +ILSVRC2012_val_00003331.JPEG:n02167151 +ILSVRC2012_val_00003332.JPEG:n04606251 +ILSVRC2012_val_00003333.JPEG:n04317175 +ILSVRC2012_val_00003334.JPEG:n04311004 +ILSVRC2012_val_00003335.JPEG:n02077923 +ILSVRC2012_val_00003336.JPEG:n04326547 +ILSVRC2012_val_00003337.JPEG:n02483708 +ILSVRC2012_val_00003338.JPEG:n02963159 +ILSVRC2012_val_00003339.JPEG:n07565083 +ILSVRC2012_val_00003340.JPEG:n04557648 +ILSVRC2012_val_00003341.JPEG:n02397096 +ILSVRC2012_val_00003342.JPEG:n04133789 +ILSVRC2012_val_00003343.JPEG:n02229544 +ILSVRC2012_val_00003344.JPEG:n04317175 +ILSVRC2012_val_00003345.JPEG:n07749582 +ILSVRC2012_val_00003346.JPEG:n03803284 +ILSVRC2012_val_00003347.JPEG:n04456115 +ILSVRC2012_val_00003348.JPEG:n01828970 +ILSVRC2012_val_00003349.JPEG:n02408429 +ILSVRC2012_val_00003350.JPEG:n01632458 +ILSVRC2012_val_00003351.JPEG:n03028079 +ILSVRC2012_val_00003352.JPEG:n03291819 +ILSVRC2012_val_00003353.JPEG:n01773797 +ILSVRC2012_val_00003354.JPEG:n02096585 +ILSVRC2012_val_00003355.JPEG:n02110341 +ILSVRC2012_val_00003356.JPEG:n01669191 +ILSVRC2012_val_00003357.JPEG:n01986214 +ILSVRC2012_val_00003358.JPEG:n03742115 +ILSVRC2012_val_00003359.JPEG:n01910747 +ILSVRC2012_val_00003360.JPEG:n02966687 +ILSVRC2012_val_00003361.JPEG:n02025239 +ILSVRC2012_val_00003362.JPEG:n07615774 +ILSVRC2012_val_00003363.JPEG:n02090721 +ILSVRC2012_val_00003364.JPEG:n01855672 +ILSVRC2012_val_00003365.JPEG:n02965783 +ILSVRC2012_val_00003366.JPEG:n03924679 +ILSVRC2012_val_00003367.JPEG:n11879895 +ILSVRC2012_val_00003368.JPEG:n02113186 +ILSVRC2012_val_00003369.JPEG:n04270147 +ILSVRC2012_val_00003370.JPEG:n02804610 +ILSVRC2012_val_00003371.JPEG:n06359193 +ILSVRC2012_val_00003372.JPEG:n02965783 +ILSVRC2012_val_00003373.JPEG:n03777754 +ILSVRC2012_val_00003374.JPEG:n09399592 +ILSVRC2012_val_00003375.JPEG:n01693334 +ILSVRC2012_val_00003376.JPEG:n04033901 +ILSVRC2012_val_00003377.JPEG:n02098413 +ILSVRC2012_val_00003378.JPEG:n01981276 +ILSVRC2012_val_00003379.JPEG:n03657121 +ILSVRC2012_val_00003380.JPEG:n02096437 +ILSVRC2012_val_00003381.JPEG:n03841143 +ILSVRC2012_val_00003382.JPEG:n02123394 +ILSVRC2012_val_00003383.JPEG:n02447366 +ILSVRC2012_val_00003384.JPEG:n03345487 +ILSVRC2012_val_00003385.JPEG:n02963159 +ILSVRC2012_val_00003386.JPEG:n01580077 +ILSVRC2012_val_00003387.JPEG:n03481172 +ILSVRC2012_val_00003388.JPEG:n02483362 +ILSVRC2012_val_00003389.JPEG:n02894605 +ILSVRC2012_val_00003390.JPEG:n02109525 +ILSVRC2012_val_00003391.JPEG:n04525038 +ILSVRC2012_val_00003392.JPEG:n01917289 +ILSVRC2012_val_00003393.JPEG:n03983396 +ILSVRC2012_val_00003394.JPEG:n04462240 +ILSVRC2012_val_00003395.JPEG:n04153751 +ILSVRC2012_val_00003396.JPEG:n03992509 +ILSVRC2012_val_00003397.JPEG:n02906734 +ILSVRC2012_val_00003398.JPEG:n03290653 +ILSVRC2012_val_00003399.JPEG:n02017213 +ILSVRC2012_val_00003400.JPEG:n02808440 +ILSVRC2012_val_00003401.JPEG:n04515003 +ILSVRC2012_val_00003402.JPEG:n02422106 +ILSVRC2012_val_00003403.JPEG:n02115913 +ILSVRC2012_val_00003404.JPEG:n03720891 +ILSVRC2012_val_00003405.JPEG:n10148035 +ILSVRC2012_val_00003406.JPEG:n02794156 +ILSVRC2012_val_00003407.JPEG:n02096294 +ILSVRC2012_val_00003408.JPEG:n03220513 +ILSVRC2012_val_00003409.JPEG:n02437312 +ILSVRC2012_val_00003410.JPEG:n02058221 +ILSVRC2012_val_00003411.JPEG:n04540053 +ILSVRC2012_val_00003412.JPEG:n07753592 +ILSVRC2012_val_00003413.JPEG:n02105641 +ILSVRC2012_val_00003414.JPEG:n04325704 +ILSVRC2012_val_00003415.JPEG:n04447861 +ILSVRC2012_val_00003416.JPEG:n07695742 +ILSVRC2012_val_00003417.JPEG:n03666591 +ILSVRC2012_val_00003418.JPEG:n03642806 +ILSVRC2012_val_00003419.JPEG:n01910747 +ILSVRC2012_val_00003420.JPEG:n03733281 +ILSVRC2012_val_00003421.JPEG:n01768244 +ILSVRC2012_val_00003422.JPEG:n03888605 +ILSVRC2012_val_00003423.JPEG:n13133613 +ILSVRC2012_val_00003424.JPEG:n03590841 +ILSVRC2012_val_00003425.JPEG:n03127925 +ILSVRC2012_val_00003426.JPEG:n02488291 +ILSVRC2012_val_00003427.JPEG:n04208210 +ILSVRC2012_val_00003428.JPEG:n04592741 +ILSVRC2012_val_00003429.JPEG:n04557648 +ILSVRC2012_val_00003430.JPEG:n02169497 +ILSVRC2012_val_00003431.JPEG:n01773549 +ILSVRC2012_val_00003432.JPEG:n02672831 +ILSVRC2012_val_00003433.JPEG:n03742115 +ILSVRC2012_val_00003434.JPEG:n01983481 +ILSVRC2012_val_00003435.JPEG:n02113978 +ILSVRC2012_val_00003436.JPEG:n03494278 +ILSVRC2012_val_00003437.JPEG:n02490219 +ILSVRC2012_val_00003438.JPEG:n02488291 +ILSVRC2012_val_00003439.JPEG:n03062245 +ILSVRC2012_val_00003440.JPEG:n02167151 +ILSVRC2012_val_00003441.JPEG:n02676566 +ILSVRC2012_val_00003442.JPEG:n04392985 +ILSVRC2012_val_00003443.JPEG:n03877472 +ILSVRC2012_val_00003444.JPEG:n02168699 +ILSVRC2012_val_00003445.JPEG:n02488291 +ILSVRC2012_val_00003446.JPEG:n02840245 +ILSVRC2012_val_00003447.JPEG:n03014705 +ILSVRC2012_val_00003448.JPEG:n04044716 +ILSVRC2012_val_00003449.JPEG:n02119022 +ILSVRC2012_val_00003450.JPEG:n01824575 +ILSVRC2012_val_00003451.JPEG:n02840245 +ILSVRC2012_val_00003452.JPEG:n04023962 +ILSVRC2012_val_00003453.JPEG:n03032252 +ILSVRC2012_val_00003454.JPEG:n02486410 +ILSVRC2012_val_00003455.JPEG:n03197337 +ILSVRC2012_val_00003456.JPEG:n02974003 +ILSVRC2012_val_00003457.JPEG:n04086273 +ILSVRC2012_val_00003458.JPEG:n02441942 +ILSVRC2012_val_00003459.JPEG:n03496892 +ILSVRC2012_val_00003460.JPEG:n03721384 +ILSVRC2012_val_00003461.JPEG:n03538406 +ILSVRC2012_val_00003462.JPEG:n03041632 +ILSVRC2012_val_00003463.JPEG:n02927161 +ILSVRC2012_val_00003464.JPEG:n02408429 +ILSVRC2012_val_00003465.JPEG:n03759954 +ILSVRC2012_val_00003466.JPEG:n03690938 +ILSVRC2012_val_00003467.JPEG:n01930112 +ILSVRC2012_val_00003468.JPEG:n01744401 +ILSVRC2012_val_00003469.JPEG:n02992529 +ILSVRC2012_val_00003470.JPEG:n03873416 +ILSVRC2012_val_00003471.JPEG:n07615774 +ILSVRC2012_val_00003472.JPEG:n02012849 +ILSVRC2012_val_00003473.JPEG:n03777568 +ILSVRC2012_val_00003474.JPEG:n03676483 +ILSVRC2012_val_00003475.JPEG:n01968897 +ILSVRC2012_val_00003476.JPEG:n03866082 +ILSVRC2012_val_00003477.JPEG:n04005630 +ILSVRC2012_val_00003478.JPEG:n04285008 +ILSVRC2012_val_00003479.JPEG:n02841315 +ILSVRC2012_val_00003480.JPEG:n02106030 +ILSVRC2012_val_00003481.JPEG:n02276258 +ILSVRC2012_val_00003482.JPEG:n02422106 +ILSVRC2012_val_00003483.JPEG:n03649909 +ILSVRC2012_val_00003484.JPEG:n03017168 +ILSVRC2012_val_00003485.JPEG:n02097474 +ILSVRC2012_val_00003486.JPEG:n02948072 +ILSVRC2012_val_00003487.JPEG:n02256656 +ILSVRC2012_val_00003488.JPEG:n04179913 +ILSVRC2012_val_00003489.JPEG:n09835506 +ILSVRC2012_val_00003490.JPEG:n02111889 +ILSVRC2012_val_00003491.JPEG:n02988304 +ILSVRC2012_val_00003492.JPEG:n07836838 +ILSVRC2012_val_00003493.JPEG:n02051845 +ILSVRC2012_val_00003494.JPEG:n02971356 +ILSVRC2012_val_00003495.JPEG:n02640242 +ILSVRC2012_val_00003496.JPEG:n03065424 +ILSVRC2012_val_00003497.JPEG:n04201297 +ILSVRC2012_val_00003498.JPEG:n02281406 +ILSVRC2012_val_00003499.JPEG:n02134418 +ILSVRC2012_val_00003500.JPEG:n02500267 +ILSVRC2012_val_00003501.JPEG:n02895154 +ILSVRC2012_val_00003502.JPEG:n02870880 +ILSVRC2012_val_00003503.JPEG:n03617480 +ILSVRC2012_val_00003504.JPEG:n02415577 +ILSVRC2012_val_00003505.JPEG:n03733131 +ILSVRC2012_val_00003506.JPEG:n03594734 +ILSVRC2012_val_00003507.JPEG:n04152593 +ILSVRC2012_val_00003508.JPEG:n04258138 +ILSVRC2012_val_00003509.JPEG:n04286575 +ILSVRC2012_val_00003510.JPEG:n04336792 +ILSVRC2012_val_00003511.JPEG:n02484975 +ILSVRC2012_val_00003512.JPEG:n04041544 +ILSVRC2012_val_00003513.JPEG:n04081281 +ILSVRC2012_val_00003514.JPEG:n03291819 +ILSVRC2012_val_00003515.JPEG:n04584207 +ILSVRC2012_val_00003516.JPEG:n02100877 +ILSVRC2012_val_00003517.JPEG:n03459775 +ILSVRC2012_val_00003518.JPEG:n01498041 +ILSVRC2012_val_00003519.JPEG:n04429376 +ILSVRC2012_val_00003520.JPEG:n04252077 +ILSVRC2012_val_00003521.JPEG:n04515003 +ILSVRC2012_val_00003522.JPEG:n02108089 +ILSVRC2012_val_00003523.JPEG:n03876231 +ILSVRC2012_val_00003524.JPEG:n03838899 +ILSVRC2012_val_00003525.JPEG:n07716358 +ILSVRC2012_val_00003526.JPEG:n02025239 +ILSVRC2012_val_00003527.JPEG:n02965783 +ILSVRC2012_val_00003528.JPEG:n04033901 +ILSVRC2012_val_00003529.JPEG:n03841143 +ILSVRC2012_val_00003530.JPEG:n02102318 +ILSVRC2012_val_00003531.JPEG:n03888605 +ILSVRC2012_val_00003532.JPEG:n03777568 +ILSVRC2012_val_00003533.JPEG:n04350905 +ILSVRC2012_val_00003534.JPEG:n02870880 +ILSVRC2012_val_00003535.JPEG:n04277352 +ILSVRC2012_val_00003536.JPEG:n07720875 +ILSVRC2012_val_00003537.JPEG:n02317335 +ILSVRC2012_val_00003538.JPEG:n02504458 +ILSVRC2012_val_00003539.JPEG:n02488291 +ILSVRC2012_val_00003540.JPEG:n02137549 +ILSVRC2012_val_00003541.JPEG:n02490219 +ILSVRC2012_val_00003542.JPEG:n04428191 +ILSVRC2012_val_00003543.JPEG:n03662601 +ILSVRC2012_val_00003544.JPEG:n04532670 +ILSVRC2012_val_00003545.JPEG:n02105412 +ILSVRC2012_val_00003546.JPEG:n02091831 +ILSVRC2012_val_00003547.JPEG:n04154565 +ILSVRC2012_val_00003548.JPEG:n01531178 +ILSVRC2012_val_00003549.JPEG:n07753275 +ILSVRC2012_val_00003550.JPEG:n02117135 +ILSVRC2012_val_00003551.JPEG:n01882714 +ILSVRC2012_val_00003552.JPEG:n03272010 +ILSVRC2012_val_00003553.JPEG:n03759954 +ILSVRC2012_val_00003554.JPEG:n03866082 +ILSVRC2012_val_00003555.JPEG:n03992509 +ILSVRC2012_val_00003556.JPEG:n02137549 +ILSVRC2012_val_00003557.JPEG:n01537544 +ILSVRC2012_val_00003558.JPEG:n01494475 +ILSVRC2012_val_00003559.JPEG:n03179701 +ILSVRC2012_val_00003560.JPEG:n01694178 +ILSVRC2012_val_00003561.JPEG:n04554684 +ILSVRC2012_val_00003562.JPEG:n04204347 +ILSVRC2012_val_00003563.JPEG:n11879895 +ILSVRC2012_val_00003564.JPEG:n04366367 +ILSVRC2012_val_00003565.JPEG:n04371430 +ILSVRC2012_val_00003566.JPEG:n12057211 +ILSVRC2012_val_00003567.JPEG:n02730930 +ILSVRC2012_val_00003568.JPEG:n03461385 +ILSVRC2012_val_00003569.JPEG:n01728572 +ILSVRC2012_val_00003570.JPEG:n01688243 +ILSVRC2012_val_00003571.JPEG:n04141975 +ILSVRC2012_val_00003572.JPEG:n02174001 +ILSVRC2012_val_00003573.JPEG:n04310018 +ILSVRC2012_val_00003574.JPEG:n02077923 +ILSVRC2012_val_00003575.JPEG:n02105505 +ILSVRC2012_val_00003576.JPEG:n03250847 +ILSVRC2012_val_00003577.JPEG:n01776313 +ILSVRC2012_val_00003578.JPEG:n04532106 +ILSVRC2012_val_00003579.JPEG:n02346627 +ILSVRC2012_val_00003580.JPEG:n04493381 +ILSVRC2012_val_00003581.JPEG:n07742313 +ILSVRC2012_val_00003582.JPEG:n04335435 +ILSVRC2012_val_00003583.JPEG:n02112018 +ILSVRC2012_val_00003584.JPEG:n02097298 +ILSVRC2012_val_00003585.JPEG:n04254120 +ILSVRC2012_val_00003586.JPEG:n02231487 +ILSVRC2012_val_00003587.JPEG:n03394916 +ILSVRC2012_val_00003588.JPEG:n01806143 +ILSVRC2012_val_00003589.JPEG:n04311004 +ILSVRC2012_val_00003590.JPEG:n03216828 +ILSVRC2012_val_00003591.JPEG:n07615774 +ILSVRC2012_val_00003592.JPEG:n07614500 +ILSVRC2012_val_00003593.JPEG:n07768694 +ILSVRC2012_val_00003594.JPEG:n07248320 +ILSVRC2012_val_00003595.JPEG:n03594734 +ILSVRC2012_val_00003596.JPEG:n04008634 +ILSVRC2012_val_00003597.JPEG:n02091134 +ILSVRC2012_val_00003598.JPEG:n02606052 +ILSVRC2012_val_00003599.JPEG:n04310018 +ILSVRC2012_val_00003600.JPEG:n07714990 +ILSVRC2012_val_00003601.JPEG:n01945685 +ILSVRC2012_val_00003602.JPEG:n02326432 +ILSVRC2012_val_00003603.JPEG:n01704323 +ILSVRC2012_val_00003604.JPEG:n01944390 +ILSVRC2012_val_00003605.JPEG:n01514668 +ILSVRC2012_val_00003606.JPEG:n01514668 +ILSVRC2012_val_00003607.JPEG:n01740131 +ILSVRC2012_val_00003608.JPEG:n04356056 +ILSVRC2012_val_00003609.JPEG:n03492542 +ILSVRC2012_val_00003610.JPEG:n02643566 +ILSVRC2012_val_00003611.JPEG:n03759954 +ILSVRC2012_val_00003612.JPEG:n03854065 +ILSVRC2012_val_00003613.JPEG:n03781244 +ILSVRC2012_val_00003614.JPEG:n03125729 +ILSVRC2012_val_00003615.JPEG:n02087394 +ILSVRC2012_val_00003616.JPEG:n02093754 +ILSVRC2012_val_00003617.JPEG:n02802426 +ILSVRC2012_val_00003618.JPEG:n03527444 +ILSVRC2012_val_00003619.JPEG:n07747607 +ILSVRC2012_val_00003620.JPEG:n03394916 +ILSVRC2012_val_00003621.JPEG:n01644373 +ILSVRC2012_val_00003622.JPEG:n02823428 +ILSVRC2012_val_00003623.JPEG:n02106550 +ILSVRC2012_val_00003624.JPEG:n03954731 +ILSVRC2012_val_00003625.JPEG:n01944390 +ILSVRC2012_val_00003626.JPEG:n09472597 +ILSVRC2012_val_00003627.JPEG:n03126707 +ILSVRC2012_val_00003628.JPEG:n02102973 +ILSVRC2012_val_00003629.JPEG:n03443371 +ILSVRC2012_val_00003630.JPEG:n03529860 +ILSVRC2012_val_00003631.JPEG:n02489166 +ILSVRC2012_val_00003632.JPEG:n04606251 +ILSVRC2012_val_00003633.JPEG:n04371774 +ILSVRC2012_val_00003634.JPEG:n03197337 +ILSVRC2012_val_00003635.JPEG:n04252225 +ILSVRC2012_val_00003636.JPEG:n01986214 +ILSVRC2012_val_00003637.JPEG:n03841143 +ILSVRC2012_val_00003638.JPEG:n02111129 +ILSVRC2012_val_00003639.JPEG:n04251144 +ILSVRC2012_val_00003640.JPEG:n02782093 +ILSVRC2012_val_00003641.JPEG:n03786901 +ILSVRC2012_val_00003642.JPEG:n04542943 +ILSVRC2012_val_00003643.JPEG:n03196217 +ILSVRC2012_val_00003644.JPEG:n01735189 +ILSVRC2012_val_00003645.JPEG:n03125729 +ILSVRC2012_val_00003646.JPEG:n02089867 +ILSVRC2012_val_00003647.JPEG:n04009552 +ILSVRC2012_val_00003648.JPEG:n02860847 +ILSVRC2012_val_00003649.JPEG:n02229544 +ILSVRC2012_val_00003650.JPEG:n01871265 +ILSVRC2012_val_00003651.JPEG:n03930313 +ILSVRC2012_val_00003652.JPEG:n04296562 +ILSVRC2012_val_00003653.JPEG:n03388549 +ILSVRC2012_val_00003654.JPEG:n02437616 +ILSVRC2012_val_00003655.JPEG:n02423022 +ILSVRC2012_val_00003656.JPEG:n02190166 +ILSVRC2012_val_00003657.JPEG:n04522168 +ILSVRC2012_val_00003658.JPEG:n04136333 +ILSVRC2012_val_00003659.JPEG:n02009229 +ILSVRC2012_val_00003660.JPEG:n07716358 +ILSVRC2012_val_00003661.JPEG:n01798484 +ILSVRC2012_val_00003662.JPEG:n01990800 +ILSVRC2012_val_00003663.JPEG:n04525038 +ILSVRC2012_val_00003664.JPEG:n07754684 +ILSVRC2012_val_00003665.JPEG:n01582220 +ILSVRC2012_val_00003666.JPEG:n03673027 +ILSVRC2012_val_00003667.JPEG:n02977058 +ILSVRC2012_val_00003668.JPEG:n04317175 +ILSVRC2012_val_00003669.JPEG:n03495258 +ILSVRC2012_val_00003670.JPEG:n02692877 +ILSVRC2012_val_00003671.JPEG:n02089973 +ILSVRC2012_val_00003672.JPEG:n01843065 +ILSVRC2012_val_00003673.JPEG:n03584254 +ILSVRC2012_val_00003674.JPEG:n02802426 +ILSVRC2012_val_00003675.JPEG:n02364673 +ILSVRC2012_val_00003676.JPEG:n01807496 +ILSVRC2012_val_00003677.JPEG:n02172182 +ILSVRC2012_val_00003678.JPEG:n03742115 +ILSVRC2012_val_00003679.JPEG:n02687172 +ILSVRC2012_val_00003680.JPEG:n02769748 +ILSVRC2012_val_00003681.JPEG:n07716358 +ILSVRC2012_val_00003682.JPEG:n03028079 +ILSVRC2012_val_00003683.JPEG:n02107142 +ILSVRC2012_val_00003684.JPEG:n02749479 +ILSVRC2012_val_00003685.JPEG:n02417914 +ILSVRC2012_val_00003686.JPEG:n04296562 +ILSVRC2012_val_00003687.JPEG:n01829413 +ILSVRC2012_val_00003688.JPEG:n01698640 +ILSVRC2012_val_00003689.JPEG:n03935335 +ILSVRC2012_val_00003690.JPEG:n02096294 +ILSVRC2012_val_00003691.JPEG:n02112706 +ILSVRC2012_val_00003692.JPEG:n02692877 +ILSVRC2012_val_00003693.JPEG:n01740131 +ILSVRC2012_val_00003694.JPEG:n07754684 +ILSVRC2012_val_00003695.JPEG:n04136333 +ILSVRC2012_val_00003696.JPEG:n02112137 +ILSVRC2012_val_00003697.JPEG:n02326432 +ILSVRC2012_val_00003698.JPEG:n02113624 +ILSVRC2012_val_00003699.JPEG:n07715103 +ILSVRC2012_val_00003700.JPEG:n02484975 +ILSVRC2012_val_00003701.JPEG:n03781244 +ILSVRC2012_val_00003702.JPEG:n01630670 +ILSVRC2012_val_00003703.JPEG:n02701002 +ILSVRC2012_val_00003704.JPEG:n03776460 +ILSVRC2012_val_00003705.JPEG:n01978455 +ILSVRC2012_val_00003706.JPEG:n01755581 +ILSVRC2012_val_00003707.JPEG:n01819313 +ILSVRC2012_val_00003708.JPEG:n03838899 +ILSVRC2012_val_00003709.JPEG:n04146614 +ILSVRC2012_val_00003710.JPEG:n04251144 +ILSVRC2012_val_00003711.JPEG:n02113023 +ILSVRC2012_val_00003712.JPEG:n02483362 +ILSVRC2012_val_00003713.JPEG:n04456115 +ILSVRC2012_val_00003714.JPEG:n02101006 +ILSVRC2012_val_00003715.JPEG:n02992211 +ILSVRC2012_val_00003716.JPEG:n02037110 +ILSVRC2012_val_00003717.JPEG:n03045698 +ILSVRC2012_val_00003718.JPEG:n02963159 +ILSVRC2012_val_00003719.JPEG:n03249569 +ILSVRC2012_val_00003720.JPEG:n06359193 +ILSVRC2012_val_00003721.JPEG:n03196217 +ILSVRC2012_val_00003722.JPEG:n01693334 +ILSVRC2012_val_00003723.JPEG:n02085936 +ILSVRC2012_val_00003724.JPEG:n03697007 +ILSVRC2012_val_00003725.JPEG:n02092002 +ILSVRC2012_val_00003726.JPEG:n02099712 +ILSVRC2012_val_00003727.JPEG:n02793495 +ILSVRC2012_val_00003728.JPEG:n03710721 +ILSVRC2012_val_00003729.JPEG:n02102318 +ILSVRC2012_val_00003730.JPEG:n03895866 +ILSVRC2012_val_00003731.JPEG:n02097209 +ILSVRC2012_val_00003732.JPEG:n03127747 +ILSVRC2012_val_00003733.JPEG:n01950731 +ILSVRC2012_val_00003734.JPEG:n02106166 +ILSVRC2012_val_00003735.JPEG:n01443537 +ILSVRC2012_val_00003736.JPEG:n03372029 +ILSVRC2012_val_00003737.JPEG:n04229816 +ILSVRC2012_val_00003738.JPEG:n01990800 +ILSVRC2012_val_00003739.JPEG:n04258138 +ILSVRC2012_val_00003740.JPEG:n03637318 +ILSVRC2012_val_00003741.JPEG:n03633091 +ILSVRC2012_val_00003742.JPEG:n03770439 +ILSVRC2012_val_00003743.JPEG:n01818515 +ILSVRC2012_val_00003744.JPEG:n04069434 +ILSVRC2012_val_00003745.JPEG:n02110063 +ILSVRC2012_val_00003746.JPEG:n01664065 +ILSVRC2012_val_00003747.JPEG:n02504458 +ILSVRC2012_val_00003748.JPEG:n01641577 +ILSVRC2012_val_00003749.JPEG:n04562935 +ILSVRC2012_val_00003750.JPEG:n03825788 +ILSVRC2012_val_00003751.JPEG:n03873416 +ILSVRC2012_val_00003752.JPEG:n02484975 +ILSVRC2012_val_00003753.JPEG:n01984695 +ILSVRC2012_val_00003754.JPEG:n03761084 +ILSVRC2012_val_00003755.JPEG:n02892201 +ILSVRC2012_val_00003756.JPEG:n04392985 +ILSVRC2012_val_00003757.JPEG:n04357314 +ILSVRC2012_val_00003758.JPEG:n02097130 +ILSVRC2012_val_00003759.JPEG:n03394916 +ILSVRC2012_val_00003760.JPEG:n03124170 +ILSVRC2012_val_00003761.JPEG:n03938244 +ILSVRC2012_val_00003762.JPEG:n01582220 +ILSVRC2012_val_00003763.JPEG:n04133789 +ILSVRC2012_val_00003764.JPEG:n07871810 +ILSVRC2012_val_00003765.JPEG:n02114855 +ILSVRC2012_val_00003766.JPEG:n02445715 +ILSVRC2012_val_00003767.JPEG:n03017168 +ILSVRC2012_val_00003768.JPEG:n01729977 +ILSVRC2012_val_00003769.JPEG:n02101006 +ILSVRC2012_val_00003770.JPEG:n04153751 +ILSVRC2012_val_00003771.JPEG:n07730033 +ILSVRC2012_val_00003772.JPEG:n02802426 +ILSVRC2012_val_00003773.JPEG:n02130308 +ILSVRC2012_val_00003774.JPEG:n02096585 +ILSVRC2012_val_00003775.JPEG:n01860187 +ILSVRC2012_val_00003776.JPEG:n01980166 +ILSVRC2012_val_00003777.JPEG:n02825657 +ILSVRC2012_val_00003778.JPEG:n03450230 +ILSVRC2012_val_00003779.JPEG:n04037443 +ILSVRC2012_val_00003780.JPEG:n04090263 +ILSVRC2012_val_00003781.JPEG:n02361337 +ILSVRC2012_val_00003782.JPEG:n02823750 +ILSVRC2012_val_00003783.JPEG:n02843684 +ILSVRC2012_val_00003784.JPEG:n03372029 +ILSVRC2012_val_00003785.JPEG:n01749939 +ILSVRC2012_val_00003786.JPEG:n02808440 +ILSVRC2012_val_00003787.JPEG:n03384352 +ILSVRC2012_val_00003788.JPEG:n02129165 +ILSVRC2012_val_00003789.JPEG:n02095570 +ILSVRC2012_val_00003790.JPEG:n02916936 +ILSVRC2012_val_00003791.JPEG:n02098105 +ILSVRC2012_val_00003792.JPEG:n02093256 +ILSVRC2012_val_00003793.JPEG:n03445777 +ILSVRC2012_val_00003794.JPEG:n02111500 +ILSVRC2012_val_00003795.JPEG:n04553703 +ILSVRC2012_val_00003796.JPEG:n03871628 +ILSVRC2012_val_00003797.JPEG:n03876231 +ILSVRC2012_val_00003798.JPEG:n03062245 +ILSVRC2012_val_00003799.JPEG:n03207941 +ILSVRC2012_val_00003800.JPEG:n04428191 +ILSVRC2012_val_00003801.JPEG:n02408429 +ILSVRC2012_val_00003802.JPEG:n04005630 +ILSVRC2012_val_00003803.JPEG:n02777292 +ILSVRC2012_val_00003804.JPEG:n03877845 +ILSVRC2012_val_00003805.JPEG:n04599235 +ILSVRC2012_val_00003806.JPEG:n02514041 +ILSVRC2012_val_00003807.JPEG:n04081281 +ILSVRC2012_val_00003808.JPEG:n02111889 +ILSVRC2012_val_00003809.JPEG:n03208938 +ILSVRC2012_val_00003810.JPEG:n02105855 +ILSVRC2012_val_00003811.JPEG:n10565667 +ILSVRC2012_val_00003812.JPEG:n02493793 +ILSVRC2012_val_00003813.JPEG:n02676566 +ILSVRC2012_val_00003814.JPEG:n02219486 +ILSVRC2012_val_00003815.JPEG:n04147183 +ILSVRC2012_val_00003816.JPEG:n01531178 +ILSVRC2012_val_00003817.JPEG:n04542943 +ILSVRC2012_val_00003818.JPEG:n02492660 +ILSVRC2012_val_00003819.JPEG:n04235860 +ILSVRC2012_val_00003820.JPEG:n02321529 +ILSVRC2012_val_00003821.JPEG:n01687978 +ILSVRC2012_val_00003822.JPEG:n02066245 +ILSVRC2012_val_00003823.JPEG:n01818515 +ILSVRC2012_val_00003824.JPEG:n03461385 +ILSVRC2012_val_00003825.JPEG:n03710637 +ILSVRC2012_val_00003826.JPEG:n03854065 +ILSVRC2012_val_00003827.JPEG:n01872401 +ILSVRC2012_val_00003828.JPEG:n01847000 +ILSVRC2012_val_00003829.JPEG:n03690938 +ILSVRC2012_val_00003830.JPEG:n06596364 +ILSVRC2012_val_00003831.JPEG:n07932039 +ILSVRC2012_val_00003832.JPEG:n02102973 +ILSVRC2012_val_00003833.JPEG:n01806567 +ILSVRC2012_val_00003834.JPEG:n02106382 +ILSVRC2012_val_00003835.JPEG:n15075141 +ILSVRC2012_val_00003836.JPEG:n02109047 +ILSVRC2012_val_00003837.JPEG:n02087394 +ILSVRC2012_val_00003838.JPEG:n01774750 +ILSVRC2012_val_00003839.JPEG:n02128385 +ILSVRC2012_val_00003840.JPEG:n07871810 +ILSVRC2012_val_00003841.JPEG:n02086240 +ILSVRC2012_val_00003842.JPEG:n04209239 +ILSVRC2012_val_00003843.JPEG:n07749582 +ILSVRC2012_val_00003844.JPEG:n04392985 +ILSVRC2012_val_00003845.JPEG:n02058221 +ILSVRC2012_val_00003846.JPEG:n01644373 +ILSVRC2012_val_00003847.JPEG:n03127925 +ILSVRC2012_val_00003848.JPEG:n03690938 +ILSVRC2012_val_00003849.JPEG:n04485082 +ILSVRC2012_val_00003850.JPEG:n03388183 +ILSVRC2012_val_00003851.JPEG:n02110627 +ILSVRC2012_val_00003852.JPEG:n02165105 +ILSVRC2012_val_00003853.JPEG:n03785016 +ILSVRC2012_val_00003854.JPEG:n02259212 +ILSVRC2012_val_00003855.JPEG:n02108915 +ILSVRC2012_val_00003856.JPEG:n02099267 +ILSVRC2012_val_00003857.JPEG:n04044716 +ILSVRC2012_val_00003858.JPEG:n01990800 +ILSVRC2012_val_00003859.JPEG:n01986214 +ILSVRC2012_val_00003860.JPEG:n01632777 +ILSVRC2012_val_00003861.JPEG:n01580077 +ILSVRC2012_val_00003862.JPEG:n02106030 +ILSVRC2012_val_00003863.JPEG:n01632458 +ILSVRC2012_val_00003864.JPEG:n03337140 +ILSVRC2012_val_00003865.JPEG:n01695060 +ILSVRC2012_val_00003866.JPEG:n09399592 +ILSVRC2012_val_00003867.JPEG:n04116512 +ILSVRC2012_val_00003868.JPEG:n03443371 +ILSVRC2012_val_00003869.JPEG:n02097658 +ILSVRC2012_val_00003870.JPEG:n04039381 +ILSVRC2012_val_00003871.JPEG:n02422699 +ILSVRC2012_val_00003872.JPEG:n02105855 +ILSVRC2012_val_00003873.JPEG:n03792782 +ILSVRC2012_val_00003874.JPEG:n02229544 +ILSVRC2012_val_00003875.JPEG:n01950731 +ILSVRC2012_val_00003876.JPEG:n02256656 +ILSVRC2012_val_00003877.JPEG:n03916031 +ILSVRC2012_val_00003878.JPEG:n01534433 +ILSVRC2012_val_00003879.JPEG:n03791053 +ILSVRC2012_val_00003880.JPEG:n04200800 +ILSVRC2012_val_00003881.JPEG:n03314780 +ILSVRC2012_val_00003882.JPEG:n04120489 +ILSVRC2012_val_00003883.JPEG:n04584207 +ILSVRC2012_val_00003884.JPEG:n01820546 +ILSVRC2012_val_00003885.JPEG:n04125021 +ILSVRC2012_val_00003886.JPEG:n02930766 +ILSVRC2012_val_00003887.JPEG:n02093647 +ILSVRC2012_val_00003888.JPEG:n02910353 +ILSVRC2012_val_00003889.JPEG:n03452741 +ILSVRC2012_val_00003890.JPEG:n03482405 +ILSVRC2012_val_00003891.JPEG:n04380533 +ILSVRC2012_val_00003892.JPEG:n01622779 +ILSVRC2012_val_00003893.JPEG:n07768694 +ILSVRC2012_val_00003894.JPEG:n03042490 +ILSVRC2012_val_00003895.JPEG:n03461385 +ILSVRC2012_val_00003896.JPEG:n04285008 +ILSVRC2012_val_00003897.JPEG:n04540053 +ILSVRC2012_val_00003898.JPEG:n02099267 +ILSVRC2012_val_00003899.JPEG:n12057211 +ILSVRC2012_val_00003900.JPEG:n04118776 +ILSVRC2012_val_00003901.JPEG:n04162706 +ILSVRC2012_val_00003902.JPEG:n12620546 +ILSVRC2012_val_00003903.JPEG:n01534433 +ILSVRC2012_val_00003904.JPEG:n01675722 +ILSVRC2012_val_00003905.JPEG:n02089078 +ILSVRC2012_val_00003906.JPEG:n03290653 +ILSVRC2012_val_00003907.JPEG:n02883205 +ILSVRC2012_val_00003908.JPEG:n07697537 +ILSVRC2012_val_00003909.JPEG:n03393912 +ILSVRC2012_val_00003910.JPEG:n02113186 +ILSVRC2012_val_00003911.JPEG:n03014705 +ILSVRC2012_val_00003912.JPEG:n04435653 +ILSVRC2012_val_00003913.JPEG:n03590841 +ILSVRC2012_val_00003914.JPEG:n03773504 +ILSVRC2012_val_00003915.JPEG:n02782093 +ILSVRC2012_val_00003916.JPEG:n02980441 +ILSVRC2012_val_00003917.JPEG:n04239074 +ILSVRC2012_val_00003918.JPEG:n04228054 +ILSVRC2012_val_00003919.JPEG:n03877845 +ILSVRC2012_val_00003920.JPEG:n04023962 +ILSVRC2012_val_00003921.JPEG:n04404412 +ILSVRC2012_val_00003922.JPEG:n02088238 +ILSVRC2012_val_00003923.JPEG:n03617480 +ILSVRC2012_val_00003924.JPEG:n03670208 +ILSVRC2012_val_00003925.JPEG:n09229709 +ILSVRC2012_val_00003926.JPEG:n02971356 +ILSVRC2012_val_00003927.JPEG:n04553703 +ILSVRC2012_val_00003928.JPEG:n01748264 +ILSVRC2012_val_00003929.JPEG:n02091467 +ILSVRC2012_val_00003930.JPEG:n07697537 +ILSVRC2012_val_00003931.JPEG:n02113186 +ILSVRC2012_val_00003932.JPEG:n07615774 +ILSVRC2012_val_00003933.JPEG:n02328150 +ILSVRC2012_val_00003934.JPEG:n02883205 +ILSVRC2012_val_00003935.JPEG:n07579787 +ILSVRC2012_val_00003936.JPEG:n01514668 +ILSVRC2012_val_00003937.JPEG:n03877845 +ILSVRC2012_val_00003938.JPEG:n02108915 +ILSVRC2012_val_00003939.JPEG:n07760859 +ILSVRC2012_val_00003940.JPEG:n02125311 +ILSVRC2012_val_00003941.JPEG:n03899768 +ILSVRC2012_val_00003942.JPEG:n01924916 +ILSVRC2012_val_00003943.JPEG:n02487347 +ILSVRC2012_val_00003944.JPEG:n02979186 +ILSVRC2012_val_00003945.JPEG:n03594945 +ILSVRC2012_val_00003946.JPEG:n03895866 +ILSVRC2012_val_00003947.JPEG:n02441942 +ILSVRC2012_val_00003948.JPEG:n13040303 +ILSVRC2012_val_00003949.JPEG:n03710193 +ILSVRC2012_val_00003950.JPEG:n03709823 +ILSVRC2012_val_00003951.JPEG:n03544143 +ILSVRC2012_val_00003952.JPEG:n02843684 +ILSVRC2012_val_00003953.JPEG:n02085782 +ILSVRC2012_val_00003954.JPEG:n02088466 +ILSVRC2012_val_00003955.JPEG:n01910747 +ILSVRC2012_val_00003956.JPEG:n04599235 +ILSVRC2012_val_00003957.JPEG:n01847000 +ILSVRC2012_val_00003958.JPEG:n02423022 +ILSVRC2012_val_00003959.JPEG:n03476991 +ILSVRC2012_val_00003960.JPEG:n02690373 +ILSVRC2012_val_00003961.JPEG:n07730033 +ILSVRC2012_val_00003962.JPEG:n03733281 +ILSVRC2012_val_00003963.JPEG:n02129604 +ILSVRC2012_val_00003964.JPEG:n02027492 +ILSVRC2012_val_00003965.JPEG:n04443257 +ILSVRC2012_val_00003966.JPEG:n03977966 +ILSVRC2012_val_00003967.JPEG:n03992509 +ILSVRC2012_val_00003968.JPEG:n02108422 +ILSVRC2012_val_00003969.JPEG:n07875152 +ILSVRC2012_val_00003970.JPEG:n03793489 +ILSVRC2012_val_00003971.JPEG:n03127925 +ILSVRC2012_val_00003972.JPEG:n04579145 +ILSVRC2012_val_00003973.JPEG:n02395406 +ILSVRC2012_val_00003974.JPEG:n02119022 +ILSVRC2012_val_00003975.JPEG:n03706229 +ILSVRC2012_val_00003976.JPEG:n03902125 +ILSVRC2012_val_00003977.JPEG:n03777568 +ILSVRC2012_val_00003978.JPEG:n02125311 +ILSVRC2012_val_00003979.JPEG:n04458633 +ILSVRC2012_val_00003980.JPEG:n02672831 +ILSVRC2012_val_00003981.JPEG:n01784675 +ILSVRC2012_val_00003982.JPEG:n02138441 +ILSVRC2012_val_00003983.JPEG:n04328186 +ILSVRC2012_val_00003984.JPEG:n02120505 +ILSVRC2012_val_00003985.JPEG:n01644373 +ILSVRC2012_val_00003986.JPEG:n03544143 +ILSVRC2012_val_00003987.JPEG:n01818515 +ILSVRC2012_val_00003988.JPEG:n03877472 +ILSVRC2012_val_00003989.JPEG:n04044716 +ILSVRC2012_val_00003990.JPEG:n04009552 +ILSVRC2012_val_00003991.JPEG:n03220513 +ILSVRC2012_val_00003992.JPEG:n04067472 +ILSVRC2012_val_00003993.JPEG:n02172182 +ILSVRC2012_val_00003994.JPEG:n02823750 +ILSVRC2012_val_00003995.JPEG:n02317335 +ILSVRC2012_val_00003996.JPEG:n04467665 +ILSVRC2012_val_00003997.JPEG:n02229544 +ILSVRC2012_val_00003998.JPEG:n04049303 +ILSVRC2012_val_00003999.JPEG:n02116738 +ILSVRC2012_val_00004000.JPEG:n07584110 +ILSVRC2012_val_00004001.JPEG:n02018795 +ILSVRC2012_val_00004002.JPEG:n03930313 +ILSVRC2012_val_00004003.JPEG:n02480495 +ILSVRC2012_val_00004004.JPEG:n02172182 +ILSVRC2012_val_00004005.JPEG:n09399592 +ILSVRC2012_val_00004006.JPEG:n01530575 +ILSVRC2012_val_00004007.JPEG:n02971356 +ILSVRC2012_val_00004008.JPEG:n02105641 +ILSVRC2012_val_00004009.JPEG:n01698640 +ILSVRC2012_val_00004010.JPEG:n04553703 +ILSVRC2012_val_00004011.JPEG:n02280649 +ILSVRC2012_val_00004012.JPEG:n01807496 +ILSVRC2012_val_00004013.JPEG:n02504458 +ILSVRC2012_val_00004014.JPEG:n03617480 +ILSVRC2012_val_00004015.JPEG:n03884397 +ILSVRC2012_val_00004016.JPEG:n02011460 +ILSVRC2012_val_00004017.JPEG:n02704792 +ILSVRC2012_val_00004018.JPEG:n03393912 +ILSVRC2012_val_00004019.JPEG:n01667114 +ILSVRC2012_val_00004020.JPEG:n03598930 +ILSVRC2012_val_00004021.JPEG:n01775062 +ILSVRC2012_val_00004022.JPEG:n07717410 +ILSVRC2012_val_00004023.JPEG:n04118776 +ILSVRC2012_val_00004024.JPEG:n03218198 +ILSVRC2012_val_00004025.JPEG:n03255030 +ILSVRC2012_val_00004026.JPEG:n02111129 +ILSVRC2012_val_00004027.JPEG:n02892201 +ILSVRC2012_val_00004028.JPEG:n03444034 +ILSVRC2012_val_00004029.JPEG:n03692522 +ILSVRC2012_val_00004030.JPEG:n02364673 +ILSVRC2012_val_00004031.JPEG:n07718747 +ILSVRC2012_val_00004032.JPEG:n04418357 +ILSVRC2012_val_00004033.JPEG:n04235860 +ILSVRC2012_val_00004034.JPEG:n03000684 +ILSVRC2012_val_00004035.JPEG:n03929660 +ILSVRC2012_val_00004036.JPEG:n03670208 +ILSVRC2012_val_00004037.JPEG:n01560419 +ILSVRC2012_val_00004038.JPEG:n02494079 +ILSVRC2012_val_00004039.JPEG:n03197337 +ILSVRC2012_val_00004040.JPEG:n01737021 +ILSVRC2012_val_00004041.JPEG:n07697313 +ILSVRC2012_val_00004042.JPEG:n02127052 +ILSVRC2012_val_00004043.JPEG:n03764736 +ILSVRC2012_val_00004044.JPEG:n04270147 +ILSVRC2012_val_00004045.JPEG:n02097474 +ILSVRC2012_val_00004046.JPEG:n04204347 +ILSVRC2012_val_00004047.JPEG:n03291819 +ILSVRC2012_val_00004048.JPEG:n03134739 +ILSVRC2012_val_00004049.JPEG:n02086240 +ILSVRC2012_val_00004050.JPEG:n03691459 +ILSVRC2012_val_00004051.JPEG:n01924916 +ILSVRC2012_val_00004052.JPEG:n04550184 +ILSVRC2012_val_00004053.JPEG:n02093754 +ILSVRC2012_val_00004054.JPEG:n03110669 +ILSVRC2012_val_00004055.JPEG:n02643566 +ILSVRC2012_val_00004056.JPEG:n02108422 +ILSVRC2012_val_00004057.JPEG:n02795169 +ILSVRC2012_val_00004058.JPEG:n02483362 +ILSVRC2012_val_00004059.JPEG:n03983396 +ILSVRC2012_val_00004060.JPEG:n02093647 +ILSVRC2012_val_00004061.JPEG:n02815834 +ILSVRC2012_val_00004062.JPEG:n04069434 +ILSVRC2012_val_00004063.JPEG:n03930313 +ILSVRC2012_val_00004064.JPEG:n02326432 +ILSVRC2012_val_00004065.JPEG:n02086079 +ILSVRC2012_val_00004066.JPEG:n03958227 +ILSVRC2012_val_00004067.JPEG:n04258138 +ILSVRC2012_val_00004068.JPEG:n03498962 +ILSVRC2012_val_00004069.JPEG:n03697007 +ILSVRC2012_val_00004070.JPEG:n03126707 +ILSVRC2012_val_00004071.JPEG:n02980441 +ILSVRC2012_val_00004072.JPEG:n03530642 +ILSVRC2012_val_00004073.JPEG:n02086910 +ILSVRC2012_val_00004074.JPEG:n02087394 +ILSVRC2012_val_00004075.JPEG:n02280649 +ILSVRC2012_val_00004076.JPEG:n04285008 +ILSVRC2012_val_00004077.JPEG:n02093256 +ILSVRC2012_val_00004078.JPEG:n01950731 +ILSVRC2012_val_00004079.JPEG:n03733131 +ILSVRC2012_val_00004080.JPEG:n04277352 +ILSVRC2012_val_00004081.JPEG:n02086240 +ILSVRC2012_val_00004082.JPEG:n03544143 +ILSVRC2012_val_00004083.JPEG:n03782006 +ILSVRC2012_val_00004084.JPEG:n01632777 +ILSVRC2012_val_00004085.JPEG:n02086646 +ILSVRC2012_val_00004086.JPEG:n03297495 +ILSVRC2012_val_00004087.JPEG:n09246464 +ILSVRC2012_val_00004088.JPEG:n02123597 +ILSVRC2012_val_00004089.JPEG:n02687172 +ILSVRC2012_val_00004090.JPEG:n04487081 +ILSVRC2012_val_00004091.JPEG:n02236044 +ILSVRC2012_val_00004092.JPEG:n03710193 +ILSVRC2012_val_00004093.JPEG:n02607072 +ILSVRC2012_val_00004094.JPEG:n02788148 +ILSVRC2012_val_00004095.JPEG:n01776313 +ILSVRC2012_val_00004096.JPEG:n04376876 +ILSVRC2012_val_00004097.JPEG:n02102973 +ILSVRC2012_val_00004098.JPEG:n07873807 +ILSVRC2012_val_00004099.JPEG:n03372029 +ILSVRC2012_val_00004100.JPEG:n02104029 +ILSVRC2012_val_00004101.JPEG:n02669723 +ILSVRC2012_val_00004102.JPEG:n01693334 +ILSVRC2012_val_00004103.JPEG:n12985857 +ILSVRC2012_val_00004104.JPEG:n03785016 +ILSVRC2012_val_00004105.JPEG:n02066245 +ILSVRC2012_val_00004106.JPEG:n01698640 +ILSVRC2012_val_00004107.JPEG:n04086273 +ILSVRC2012_val_00004108.JPEG:n03047690 +ILSVRC2012_val_00004109.JPEG:n04026417 +ILSVRC2012_val_00004110.JPEG:n01773797 +ILSVRC2012_val_00004111.JPEG:n03742115 +ILSVRC2012_val_00004112.JPEG:n02018207 +ILSVRC2012_val_00004113.JPEG:n01978455 +ILSVRC2012_val_00004114.JPEG:n02988304 +ILSVRC2012_val_00004115.JPEG:n03595614 +ILSVRC2012_val_00004116.JPEG:n02965783 +ILSVRC2012_val_00004117.JPEG:n02992529 +ILSVRC2012_val_00004118.JPEG:n01773157 +ILSVRC2012_val_00004119.JPEG:n03417042 +ILSVRC2012_val_00004120.JPEG:n03376595 +ILSVRC2012_val_00004121.JPEG:n04435653 +ILSVRC2012_val_00004122.JPEG:n07711569 +ILSVRC2012_val_00004123.JPEG:n03970156 +ILSVRC2012_val_00004124.JPEG:n02877765 +ILSVRC2012_val_00004125.JPEG:n04111531 +ILSVRC2012_val_00004126.JPEG:n09256479 +ILSVRC2012_val_00004127.JPEG:n02641379 +ILSVRC2012_val_00004128.JPEG:n04179913 +ILSVRC2012_val_00004129.JPEG:n02113023 +ILSVRC2012_val_00004130.JPEG:n03977966 +ILSVRC2012_val_00004131.JPEG:n04525038 +ILSVRC2012_val_00004132.JPEG:n02190166 +ILSVRC2012_val_00004133.JPEG:n04070727 +ILSVRC2012_val_00004134.JPEG:n02111277 +ILSVRC2012_val_00004135.JPEG:n02128757 +ILSVRC2012_val_00004136.JPEG:n01784675 +ILSVRC2012_val_00004137.JPEG:n02412080 +ILSVRC2012_val_00004138.JPEG:n03146219 +ILSVRC2012_val_00004139.JPEG:n03485794 +ILSVRC2012_val_00004140.JPEG:n01773157 +ILSVRC2012_val_00004141.JPEG:n02119022 +ILSVRC2012_val_00004142.JPEG:n02704792 +ILSVRC2012_val_00004143.JPEG:n01737021 +ILSVRC2012_val_00004144.JPEG:n03697007 +ILSVRC2012_val_00004145.JPEG:n03450230 +ILSVRC2012_val_00004146.JPEG:n01770081 +ILSVRC2012_val_00004147.JPEG:n03792782 +ILSVRC2012_val_00004148.JPEG:n02089867 +ILSVRC2012_val_00004149.JPEG:n02817516 +ILSVRC2012_val_00004150.JPEG:n03141823 +ILSVRC2012_val_00004151.JPEG:n01773157 +ILSVRC2012_val_00004152.JPEG:n07860988 +ILSVRC2012_val_00004153.JPEG:n02317335 +ILSVRC2012_val_00004154.JPEG:n04442312 +ILSVRC2012_val_00004155.JPEG:n04428191 +ILSVRC2012_val_00004156.JPEG:n04049303 +ILSVRC2012_val_00004157.JPEG:n12620546 +ILSVRC2012_val_00004158.JPEG:n04591157 +ILSVRC2012_val_00004159.JPEG:n03980874 +ILSVRC2012_val_00004160.JPEG:n03314780 +ILSVRC2012_val_00004161.JPEG:n02514041 +ILSVRC2012_val_00004162.JPEG:n03376595 +ILSVRC2012_val_00004163.JPEG:n01774384 +ILSVRC2012_val_00004164.JPEG:n01774384 +ILSVRC2012_val_00004165.JPEG:n04579432 +ILSVRC2012_val_00004166.JPEG:n04336792 +ILSVRC2012_val_00004167.JPEG:n01872401 +ILSVRC2012_val_00004168.JPEG:n02483708 +ILSVRC2012_val_00004169.JPEG:n03127925 +ILSVRC2012_val_00004170.JPEG:n03314780 +ILSVRC2012_val_00004171.JPEG:n03843555 +ILSVRC2012_val_00004172.JPEG:n01770081 +ILSVRC2012_val_00004173.JPEG:n02480855 +ILSVRC2012_val_00004174.JPEG:n04118776 +ILSVRC2012_val_00004175.JPEG:n01910747 +ILSVRC2012_val_00004176.JPEG:n03126707 +ILSVRC2012_val_00004177.JPEG:n02233338 +ILSVRC2012_val_00004178.JPEG:n02114855 +ILSVRC2012_val_00004179.JPEG:n02808304 +ILSVRC2012_val_00004180.JPEG:n02107683 +ILSVRC2012_val_00004181.JPEG:n03590841 +ILSVRC2012_val_00004182.JPEG:n01737021 +ILSVRC2012_val_00004183.JPEG:n01514859 +ILSVRC2012_val_00004184.JPEG:n04346328 +ILSVRC2012_val_00004185.JPEG:n02102480 +ILSVRC2012_val_00004186.JPEG:n02093754 +ILSVRC2012_val_00004187.JPEG:n09472597 +ILSVRC2012_val_00004188.JPEG:n09332890 +ILSVRC2012_val_00004189.JPEG:n03630383 +ILSVRC2012_val_00004190.JPEG:n02492035 +ILSVRC2012_val_00004191.JPEG:n04026417 +ILSVRC2012_val_00004192.JPEG:n02110185 +ILSVRC2012_val_00004193.JPEG:n03125729 +ILSVRC2012_val_00004194.JPEG:n04465501 +ILSVRC2012_val_00004195.JPEG:n07695742 +ILSVRC2012_val_00004196.JPEG:n03775546 +ILSVRC2012_val_00004197.JPEG:n02930766 +ILSVRC2012_val_00004198.JPEG:n07753275 +ILSVRC2012_val_00004199.JPEG:n07684084 +ILSVRC2012_val_00004200.JPEG:n04486054 +ILSVRC2012_val_00004201.JPEG:n01677366 +ILSVRC2012_val_00004202.JPEG:n03127747 +ILSVRC2012_val_00004203.JPEG:n02917067 +ILSVRC2012_val_00004204.JPEG:n04347754 +ILSVRC2012_val_00004205.JPEG:n02704792 +ILSVRC2012_val_00004206.JPEG:n07583066 +ILSVRC2012_val_00004207.JPEG:n07714990 +ILSVRC2012_val_00004208.JPEG:n02111500 +ILSVRC2012_val_00004209.JPEG:n03085013 +ILSVRC2012_val_00004210.JPEG:n02233338 +ILSVRC2012_val_00004211.JPEG:n03977966 +ILSVRC2012_val_00004212.JPEG:n03876231 +ILSVRC2012_val_00004213.JPEG:n07760859 +ILSVRC2012_val_00004214.JPEG:n03623198 +ILSVRC2012_val_00004215.JPEG:n02268853 +ILSVRC2012_val_00004216.JPEG:n07730033 +ILSVRC2012_val_00004217.JPEG:n02097047 +ILSVRC2012_val_00004218.JPEG:n02981792 +ILSVRC2012_val_00004219.JPEG:n01984695 +ILSVRC2012_val_00004220.JPEG:n04584207 +ILSVRC2012_val_00004221.JPEG:n01665541 +ILSVRC2012_val_00004222.JPEG:n01734418 +ILSVRC2012_val_00004223.JPEG:n02100877 +ILSVRC2012_val_00004224.JPEG:n03109150 +ILSVRC2012_val_00004225.JPEG:n02099712 +ILSVRC2012_val_00004226.JPEG:n01855672 +ILSVRC2012_val_00004227.JPEG:n02486410 +ILSVRC2012_val_00004228.JPEG:n02099267 +ILSVRC2012_val_00004229.JPEG:n03804744 +ILSVRC2012_val_00004230.JPEG:n04179913 +ILSVRC2012_val_00004231.JPEG:n02091032 +ILSVRC2012_val_00004232.JPEG:n04200800 +ILSVRC2012_val_00004233.JPEG:n04127249 +ILSVRC2012_val_00004234.JPEG:n01833805 +ILSVRC2012_val_00004235.JPEG:n01855672 +ILSVRC2012_val_00004236.JPEG:n02909870 +ILSVRC2012_val_00004237.JPEG:n04423845 +ILSVRC2012_val_00004238.JPEG:n03345487 +ILSVRC2012_val_00004239.JPEG:n04456115 +ILSVRC2012_val_00004240.JPEG:n04517823 +ILSVRC2012_val_00004241.JPEG:n07714990 +ILSVRC2012_val_00004242.JPEG:n03492542 +ILSVRC2012_val_00004243.JPEG:n01531178 +ILSVRC2012_val_00004244.JPEG:n07892512 +ILSVRC2012_val_00004245.JPEG:n01534433 +ILSVRC2012_val_00004246.JPEG:n03982430 +ILSVRC2012_val_00004247.JPEG:n04116512 +ILSVRC2012_val_00004248.JPEG:n02097130 +ILSVRC2012_val_00004249.JPEG:n04612504 +ILSVRC2012_val_00004250.JPEG:n03146219 +ILSVRC2012_val_00004251.JPEG:n02097130 +ILSVRC2012_val_00004252.JPEG:n04517823 +ILSVRC2012_val_00004253.JPEG:n07684084 +ILSVRC2012_val_00004254.JPEG:n01978455 +ILSVRC2012_val_00004255.JPEG:n02236044 +ILSVRC2012_val_00004256.JPEG:n01798484 +ILSVRC2012_val_00004257.JPEG:n04200800 +ILSVRC2012_val_00004258.JPEG:n01985128 +ILSVRC2012_val_00004259.JPEG:n09468604 +ILSVRC2012_val_00004260.JPEG:n02268853 +ILSVRC2012_val_00004261.JPEG:n02090622 +ILSVRC2012_val_00004262.JPEG:n03000684 +ILSVRC2012_val_00004263.JPEG:n04447861 +ILSVRC2012_val_00004264.JPEG:n04154565 +ILSVRC2012_val_00004265.JPEG:n02840245 +ILSVRC2012_val_00004266.JPEG:n03126707 +ILSVRC2012_val_00004267.JPEG:n02391049 +ILSVRC2012_val_00004268.JPEG:n04532106 +ILSVRC2012_val_00004269.JPEG:n01728572 +ILSVRC2012_val_00004270.JPEG:n03124043 +ILSVRC2012_val_00004271.JPEG:n01773549 +ILSVRC2012_val_00004272.JPEG:n02480855 +ILSVRC2012_val_00004273.JPEG:n07860988 +ILSVRC2012_val_00004274.JPEG:n02105056 +ILSVRC2012_val_00004275.JPEG:n03888605 +ILSVRC2012_val_00004276.JPEG:n02116738 +ILSVRC2012_val_00004277.JPEG:n02804610 +ILSVRC2012_val_00004278.JPEG:n02113799 +ILSVRC2012_val_00004279.JPEG:n03899768 +ILSVRC2012_val_00004280.JPEG:n01729322 +ILSVRC2012_val_00004281.JPEG:n07873807 +ILSVRC2012_val_00004282.JPEG:n02116738 +ILSVRC2012_val_00004283.JPEG:n02795169 +ILSVRC2012_val_00004284.JPEG:n02256656 +ILSVRC2012_val_00004285.JPEG:n07720875 +ILSVRC2012_val_00004286.JPEG:n03584829 +ILSVRC2012_val_00004287.JPEG:n02097209 +ILSVRC2012_val_00004288.JPEG:n02092002 +ILSVRC2012_val_00004289.JPEG:n07614500 +ILSVRC2012_val_00004290.JPEG:n03599486 +ILSVRC2012_val_00004291.JPEG:n02825657 +ILSVRC2012_val_00004292.JPEG:n02966687 +ILSVRC2012_val_00004293.JPEG:n04428191 +ILSVRC2012_val_00004294.JPEG:n02488702 +ILSVRC2012_val_00004295.JPEG:n01774384 +ILSVRC2012_val_00004296.JPEG:n03908618 +ILSVRC2012_val_00004297.JPEG:n03814639 +ILSVRC2012_val_00004298.JPEG:n02444819 +ILSVRC2012_val_00004299.JPEG:n02825657 +ILSVRC2012_val_00004300.JPEG:n02325366 +ILSVRC2012_val_00004301.JPEG:n03394916 +ILSVRC2012_val_00004302.JPEG:n02077923 +ILSVRC2012_val_00004303.JPEG:n03709823 +ILSVRC2012_val_00004304.JPEG:n04579432 +ILSVRC2012_val_00004305.JPEG:n03967562 +ILSVRC2012_val_00004306.JPEG:n01514668 +ILSVRC2012_val_00004307.JPEG:n04548280 +ILSVRC2012_val_00004308.JPEG:n03899768 +ILSVRC2012_val_00004309.JPEG:n02892201 +ILSVRC2012_val_00004310.JPEG:n01704323 +ILSVRC2012_val_00004311.JPEG:n01484850 +ILSVRC2012_val_00004312.JPEG:n03535780 +ILSVRC2012_val_00004313.JPEG:n03775546 +ILSVRC2012_val_00004314.JPEG:n03337140 +ILSVRC2012_val_00004315.JPEG:n01514859 +ILSVRC2012_val_00004316.JPEG:n01580077 +ILSVRC2012_val_00004317.JPEG:n01580077 +ILSVRC2012_val_00004318.JPEG:n04509417 +ILSVRC2012_val_00004319.JPEG:n03977966 +ILSVRC2012_val_00004320.JPEG:n02115641 +ILSVRC2012_val_00004321.JPEG:n07697313 +ILSVRC2012_val_00004322.JPEG:n07753275 +ILSVRC2012_val_00004323.JPEG:n04542943 +ILSVRC2012_val_00004324.JPEG:n02910353 +ILSVRC2012_val_00004325.JPEG:n02087046 +ILSVRC2012_val_00004326.JPEG:n04443257 +ILSVRC2012_val_00004327.JPEG:n03788365 +ILSVRC2012_val_00004328.JPEG:n04429376 +ILSVRC2012_val_00004329.JPEG:n01484850 +ILSVRC2012_val_00004330.JPEG:n02843684 +ILSVRC2012_val_00004331.JPEG:n04479046 +ILSVRC2012_val_00004332.JPEG:n01990800 +ILSVRC2012_val_00004333.JPEG:n09193705 +ILSVRC2012_val_00004334.JPEG:n02115641 +ILSVRC2012_val_00004335.JPEG:n01773549 +ILSVRC2012_val_00004336.JPEG:n09246464 +ILSVRC2012_val_00004337.JPEG:n03956157 +ILSVRC2012_val_00004338.JPEG:n03065424 +ILSVRC2012_val_00004339.JPEG:n02174001 +ILSVRC2012_val_00004340.JPEG:n01824575 +ILSVRC2012_val_00004341.JPEG:n02099267 +ILSVRC2012_val_00004342.JPEG:n02093647 +ILSVRC2012_val_00004343.JPEG:n03133878 +ILSVRC2012_val_00004344.JPEG:n01580077 +ILSVRC2012_val_00004345.JPEG:n01622779 +ILSVRC2012_val_00004346.JPEG:n03271574 +ILSVRC2012_val_00004347.JPEG:n07768694 +ILSVRC2012_val_00004348.JPEG:n04376876 +ILSVRC2012_val_00004349.JPEG:n01877812 +ILSVRC2012_val_00004350.JPEG:n03110669 +ILSVRC2012_val_00004351.JPEG:n01728920 +ILSVRC2012_val_00004352.JPEG:n04141327 +ILSVRC2012_val_00004353.JPEG:n04389033 +ILSVRC2012_val_00004354.JPEG:n02096294 +ILSVRC2012_val_00004355.JPEG:n02492035 +ILSVRC2012_val_00004356.JPEG:n03876231 +ILSVRC2012_val_00004357.JPEG:n07716906 +ILSVRC2012_val_00004358.JPEG:n02097474 +ILSVRC2012_val_00004359.JPEG:n02086240 +ILSVRC2012_val_00004360.JPEG:n02708093 +ILSVRC2012_val_00004361.JPEG:n02105641 +ILSVRC2012_val_00004362.JPEG:n01984695 +ILSVRC2012_val_00004363.JPEG:n03125729 +ILSVRC2012_val_00004364.JPEG:n03944341 +ILSVRC2012_val_00004365.JPEG:n03450230 +ILSVRC2012_val_00004366.JPEG:n02109525 +ILSVRC2012_val_00004367.JPEG:n04389033 +ILSVRC2012_val_00004368.JPEG:n07760859 +ILSVRC2012_val_00004369.JPEG:n01704323 +ILSVRC2012_val_00004370.JPEG:n04540053 +ILSVRC2012_val_00004371.JPEG:n02823428 +ILSVRC2012_val_00004372.JPEG:n02115641 +ILSVRC2012_val_00004373.JPEG:n03733281 +ILSVRC2012_val_00004374.JPEG:n02093754 +ILSVRC2012_val_00004375.JPEG:n01532829 +ILSVRC2012_val_00004376.JPEG:n07802026 +ILSVRC2012_val_00004377.JPEG:n09472597 +ILSVRC2012_val_00004378.JPEG:n02091134 +ILSVRC2012_val_00004379.JPEG:n03041632 +ILSVRC2012_val_00004380.JPEG:n04372370 +ILSVRC2012_val_00004381.JPEG:n01608432 +ILSVRC2012_val_00004382.JPEG:n04265275 +ILSVRC2012_val_00004383.JPEG:n02804414 +ILSVRC2012_val_00004384.JPEG:n03109150 +ILSVRC2012_val_00004385.JPEG:n04328186 +ILSVRC2012_val_00004386.JPEG:n02107312 +ILSVRC2012_val_00004387.JPEG:n03100240 +ILSVRC2012_val_00004388.JPEG:n03250847 +ILSVRC2012_val_00004389.JPEG:n03393912 +ILSVRC2012_val_00004390.JPEG:n02090622 +ILSVRC2012_val_00004391.JPEG:n02840245 +ILSVRC2012_val_00004392.JPEG:n02870880 +ILSVRC2012_val_00004393.JPEG:n04562935 +ILSVRC2012_val_00004394.JPEG:n02397096 +ILSVRC2012_val_00004395.JPEG:n03995372 +ILSVRC2012_val_00004396.JPEG:n02106662 +ILSVRC2012_val_00004397.JPEG:n02096177 +ILSVRC2012_val_00004398.JPEG:n02493509 +ILSVRC2012_val_00004399.JPEG:n02965783 +ILSVRC2012_val_00004400.JPEG:n01981276 +ILSVRC2012_val_00004401.JPEG:n01990800 +ILSVRC2012_val_00004402.JPEG:n01698640 +ILSVRC2012_val_00004403.JPEG:n02088238 +ILSVRC2012_val_00004404.JPEG:n02107908 +ILSVRC2012_val_00004405.JPEG:n09399592 +ILSVRC2012_val_00004406.JPEG:n02790996 +ILSVRC2012_val_00004407.JPEG:n02091134 +ILSVRC2012_val_00004408.JPEG:n04252225 +ILSVRC2012_val_00004409.JPEG:n02447366 +ILSVRC2012_val_00004410.JPEG:n03179701 +ILSVRC2012_val_00004411.JPEG:n02123394 +ILSVRC2012_val_00004412.JPEG:n02974003 +ILSVRC2012_val_00004413.JPEG:n03124170 +ILSVRC2012_val_00004414.JPEG:n03045698 +ILSVRC2012_val_00004415.JPEG:n03271574 +ILSVRC2012_val_00004416.JPEG:n04067472 +ILSVRC2012_val_00004417.JPEG:n01494475 +ILSVRC2012_val_00004418.JPEG:n01984695 +ILSVRC2012_val_00004419.JPEG:n02321529 +ILSVRC2012_val_00004420.JPEG:n03062245 +ILSVRC2012_val_00004421.JPEG:n07892512 +ILSVRC2012_val_00004422.JPEG:n02123045 +ILSVRC2012_val_00004423.JPEG:n02099849 +ILSVRC2012_val_00004424.JPEG:n02672831 +ILSVRC2012_val_00004425.JPEG:n03854065 +ILSVRC2012_val_00004426.JPEG:n02825657 +ILSVRC2012_val_00004427.JPEG:n01644900 +ILSVRC2012_val_00004428.JPEG:n07745940 +ILSVRC2012_val_00004429.JPEG:n04366367 +ILSVRC2012_val_00004430.JPEG:n09288635 +ILSVRC2012_val_00004431.JPEG:n03447447 +ILSVRC2012_val_00004432.JPEG:n03124043 +ILSVRC2012_val_00004433.JPEG:n12267677 +ILSVRC2012_val_00004434.JPEG:n02091244 +ILSVRC2012_val_00004435.JPEG:n02111277 +ILSVRC2012_val_00004436.JPEG:n02088632 +ILSVRC2012_val_00004437.JPEG:n12985857 +ILSVRC2012_val_00004438.JPEG:n04517823 +ILSVRC2012_val_00004439.JPEG:n03594945 +ILSVRC2012_val_00004440.JPEG:n04049303 +ILSVRC2012_val_00004441.JPEG:n03908714 +ILSVRC2012_val_00004442.JPEG:n03697007 +ILSVRC2012_val_00004443.JPEG:n07714571 +ILSVRC2012_val_00004444.JPEG:n01986214 +ILSVRC2012_val_00004445.JPEG:n03014705 +ILSVRC2012_val_00004446.JPEG:n04238763 +ILSVRC2012_val_00004447.JPEG:n02950826 +ILSVRC2012_val_00004448.JPEG:n01755581 +ILSVRC2012_val_00004449.JPEG:n02108089 +ILSVRC2012_val_00004450.JPEG:n02111500 +ILSVRC2012_val_00004451.JPEG:n02028035 +ILSVRC2012_val_00004452.JPEG:n03425413 +ILSVRC2012_val_00004453.JPEG:n02276258 +ILSVRC2012_val_00004454.JPEG:n03690938 +ILSVRC2012_val_00004455.JPEG:n03478589 +ILSVRC2012_val_00004456.JPEG:n04579432 +ILSVRC2012_val_00004457.JPEG:n04209133 +ILSVRC2012_val_00004458.JPEG:n02492035 +ILSVRC2012_val_00004459.JPEG:n04479046 +ILSVRC2012_val_00004460.JPEG:n03131574 +ILSVRC2012_val_00004461.JPEG:n04026417 +ILSVRC2012_val_00004462.JPEG:n01981276 +ILSVRC2012_val_00004463.JPEG:n01514668 +ILSVRC2012_val_00004464.JPEG:n02643566 +ILSVRC2012_val_00004465.JPEG:n03791053 +ILSVRC2012_val_00004466.JPEG:n02870880 +ILSVRC2012_val_00004467.JPEG:n04235860 +ILSVRC2012_val_00004468.JPEG:n06596364 +ILSVRC2012_val_00004469.JPEG:n04019541 +ILSVRC2012_val_00004470.JPEG:n09246464 +ILSVRC2012_val_00004471.JPEG:n03065424 +ILSVRC2012_val_00004472.JPEG:n13054560 +ILSVRC2012_val_00004473.JPEG:n04597913 +ILSVRC2012_val_00004474.JPEG:n02111500 +ILSVRC2012_val_00004475.JPEG:n04252077 +ILSVRC2012_val_00004476.JPEG:n03857828 +ILSVRC2012_val_00004477.JPEG:n02100236 +ILSVRC2012_val_00004478.JPEG:n04442312 +ILSVRC2012_val_00004479.JPEG:n02363005 +ILSVRC2012_val_00004480.JPEG:n04040759 +ILSVRC2012_val_00004481.JPEG:n03127925 +ILSVRC2012_val_00004482.JPEG:n04033995 +ILSVRC2012_val_00004483.JPEG:n03662601 +ILSVRC2012_val_00004484.JPEG:n02966193 +ILSVRC2012_val_00004485.JPEG:n03761084 +ILSVRC2012_val_00004486.JPEG:n03838899 +ILSVRC2012_val_00004487.JPEG:n04081281 +ILSVRC2012_val_00004488.JPEG:n04243546 +ILSVRC2012_val_00004489.JPEG:n04252077 +ILSVRC2012_val_00004490.JPEG:n04487081 +ILSVRC2012_val_00004491.JPEG:n04417672 +ILSVRC2012_val_00004492.JPEG:n03662601 +ILSVRC2012_val_00004493.JPEG:n03476991 +ILSVRC2012_val_00004494.JPEG:n01829413 +ILSVRC2012_val_00004495.JPEG:n07614500 +ILSVRC2012_val_00004496.JPEG:n02701002 +ILSVRC2012_val_00004497.JPEG:n07754684 +ILSVRC2012_val_00004498.JPEG:n04258138 +ILSVRC2012_val_00004499.JPEG:n01744401 +ILSVRC2012_val_00004500.JPEG:n03259280 +ILSVRC2012_val_00004501.JPEG:n02676566 +ILSVRC2012_val_00004502.JPEG:n03017168 +ILSVRC2012_val_00004503.JPEG:n01817953 +ILSVRC2012_val_00004504.JPEG:n04049303 +ILSVRC2012_val_00004505.JPEG:n01692333 +ILSVRC2012_val_00004506.JPEG:n02108551 +ILSVRC2012_val_00004507.JPEG:n03134739 +ILSVRC2012_val_00004508.JPEG:n02410509 +ILSVRC2012_val_00004509.JPEG:n03871628 +ILSVRC2012_val_00004510.JPEG:n04525305 +ILSVRC2012_val_00004511.JPEG:n02093754 +ILSVRC2012_val_00004512.JPEG:n04461696 +ILSVRC2012_val_00004513.JPEG:n04523525 +ILSVRC2012_val_00004514.JPEG:n11939491 +ILSVRC2012_val_00004515.JPEG:n04612504 +ILSVRC2012_val_00004516.JPEG:n03706229 +ILSVRC2012_val_00004517.JPEG:n02167151 +ILSVRC2012_val_00004518.JPEG:n01582220 +ILSVRC2012_val_00004519.JPEG:n03692522 +ILSVRC2012_val_00004520.JPEG:n03595614 +ILSVRC2012_val_00004521.JPEG:n02823428 +ILSVRC2012_val_00004522.JPEG:n03950228 +ILSVRC2012_val_00004523.JPEG:n04399382 +ILSVRC2012_val_00004524.JPEG:n03877845 +ILSVRC2012_val_00004525.JPEG:n04596742 +ILSVRC2012_val_00004526.JPEG:n04005630 +ILSVRC2012_val_00004527.JPEG:n03724870 +ILSVRC2012_val_00004528.JPEG:n03445924 +ILSVRC2012_val_00004529.JPEG:n07614500 +ILSVRC2012_val_00004530.JPEG:n01883070 +ILSVRC2012_val_00004531.JPEG:n03710637 +ILSVRC2012_val_00004532.JPEG:n04120489 +ILSVRC2012_val_00004533.JPEG:n03127925 +ILSVRC2012_val_00004534.JPEG:n03249569 +ILSVRC2012_val_00004535.JPEG:n02879718 +ILSVRC2012_val_00004536.JPEG:n04562935 +ILSVRC2012_val_00004537.JPEG:n03630383 +ILSVRC2012_val_00004538.JPEG:n02106662 +ILSVRC2012_val_00004539.JPEG:n02097474 +ILSVRC2012_val_00004540.JPEG:n02114855 +ILSVRC2012_val_00004541.JPEG:n09332890 +ILSVRC2012_val_00004542.JPEG:n02096051 +ILSVRC2012_val_00004543.JPEG:n03995372 +ILSVRC2012_val_00004544.JPEG:n03016953 +ILSVRC2012_val_00004545.JPEG:n03447447 +ILSVRC2012_val_00004546.JPEG:n10565667 +ILSVRC2012_val_00004547.JPEG:n07579787 +ILSVRC2012_val_00004548.JPEG:n02102040 +ILSVRC2012_val_00004549.JPEG:n02097298 +ILSVRC2012_val_00004550.JPEG:n01514668 +ILSVRC2012_val_00004551.JPEG:n04332243 +ILSVRC2012_val_00004552.JPEG:n03770679 +ILSVRC2012_val_00004553.JPEG:n02102040 +ILSVRC2012_val_00004554.JPEG:n01616318 +ILSVRC2012_val_00004555.JPEG:n01694178 +ILSVRC2012_val_00004556.JPEG:n02817516 +ILSVRC2012_val_00004557.JPEG:n02086240 +ILSVRC2012_val_00004558.JPEG:n03787032 +ILSVRC2012_val_00004559.JPEG:n01582220 +ILSVRC2012_val_00004560.JPEG:n02097130 +ILSVRC2012_val_00004561.JPEG:n03690938 +ILSVRC2012_val_00004562.JPEG:n02825657 +ILSVRC2012_val_00004563.JPEG:n02106662 +ILSVRC2012_val_00004564.JPEG:n02490219 +ILSVRC2012_val_00004565.JPEG:n02514041 +ILSVRC2012_val_00004566.JPEG:n03958227 +ILSVRC2012_val_00004567.JPEG:n03658185 +ILSVRC2012_val_00004568.JPEG:n03187595 +ILSVRC2012_val_00004569.JPEG:n02107908 +ILSVRC2012_val_00004570.JPEG:n07734744 +ILSVRC2012_val_00004571.JPEG:n02093859 +ILSVRC2012_val_00004572.JPEG:n02011460 +ILSVRC2012_val_00004573.JPEG:n04447861 +ILSVRC2012_val_00004574.JPEG:n02640242 +ILSVRC2012_val_00004575.JPEG:n02793495 +ILSVRC2012_val_00004576.JPEG:n02514041 +ILSVRC2012_val_00004577.JPEG:n01534433 +ILSVRC2012_val_00004578.JPEG:n02132136 +ILSVRC2012_val_00004579.JPEG:n02108422 +ILSVRC2012_val_00004580.JPEG:n01768244 +ILSVRC2012_val_00004581.JPEG:n04399382 +ILSVRC2012_val_00004582.JPEG:n01734418 +ILSVRC2012_val_00004583.JPEG:n02037110 +ILSVRC2012_val_00004584.JPEG:n02444819 +ILSVRC2012_val_00004585.JPEG:n03272562 +ILSVRC2012_val_00004586.JPEG:n02906734 +ILSVRC2012_val_00004587.JPEG:n01740131 +ILSVRC2012_val_00004588.JPEG:n03325584 +ILSVRC2012_val_00004589.JPEG:n03598930 +ILSVRC2012_val_00004590.JPEG:n02277742 +ILSVRC2012_val_00004591.JPEG:n03443371 +ILSVRC2012_val_00004592.JPEG:n03447721 +ILSVRC2012_val_00004593.JPEG:n02097130 +ILSVRC2012_val_00004594.JPEG:n04347754 +ILSVRC2012_val_00004595.JPEG:n03903868 +ILSVRC2012_val_00004596.JPEG:n03529860 +ILSVRC2012_val_00004597.JPEG:n06785654 +ILSVRC2012_val_00004598.JPEG:n01985128 +ILSVRC2012_val_00004599.JPEG:n02892767 +ILSVRC2012_val_00004600.JPEG:n02074367 +ILSVRC2012_val_00004601.JPEG:n02445715 +ILSVRC2012_val_00004602.JPEG:n03131574 +ILSVRC2012_val_00004603.JPEG:n02892201 +ILSVRC2012_val_00004604.JPEG:n02114548 +ILSVRC2012_val_00004605.JPEG:n02096294 +ILSVRC2012_val_00004606.JPEG:n03787032 +ILSVRC2012_val_00004607.JPEG:n03776460 +ILSVRC2012_val_00004608.JPEG:n02870880 +ILSVRC2012_val_00004609.JPEG:n04347754 +ILSVRC2012_val_00004610.JPEG:n03930313 +ILSVRC2012_val_00004611.JPEG:n02095889 +ILSVRC2012_val_00004612.JPEG:n02124075 +ILSVRC2012_val_00004613.JPEG:n01641577 +ILSVRC2012_val_00004614.JPEG:n07753592 +ILSVRC2012_val_00004615.JPEG:n02100583 +ILSVRC2012_val_00004616.JPEG:n04591157 +ILSVRC2012_val_00004617.JPEG:n02488291 +ILSVRC2012_val_00004618.JPEG:n03690938 +ILSVRC2012_val_00004619.JPEG:n03791053 +ILSVRC2012_val_00004620.JPEG:n02860847 +ILSVRC2012_val_00004621.JPEG:n04612504 +ILSVRC2012_val_00004622.JPEG:n01677366 +ILSVRC2012_val_00004623.JPEG:n02112350 +ILSVRC2012_val_00004624.JPEG:n03062245 +ILSVRC2012_val_00004625.JPEG:n02909870 +ILSVRC2012_val_00004626.JPEG:n09428293 +ILSVRC2012_val_00004627.JPEG:n01860187 +ILSVRC2012_val_00004628.JPEG:n02999410 +ILSVRC2012_val_00004629.JPEG:n13044778 +ILSVRC2012_val_00004630.JPEG:n04070727 +ILSVRC2012_val_00004631.JPEG:n02105855 +ILSVRC2012_val_00004632.JPEG:n01950731 +ILSVRC2012_val_00004633.JPEG:n04443257 +ILSVRC2012_val_00004634.JPEG:n02110341 +ILSVRC2012_val_00004635.JPEG:n04265275 +ILSVRC2012_val_00004636.JPEG:n04273569 +ILSVRC2012_val_00004637.JPEG:n03000247 +ILSVRC2012_val_00004638.JPEG:n01675722 +ILSVRC2012_val_00004639.JPEG:n03838899 +ILSVRC2012_val_00004640.JPEG:n13040303 +ILSVRC2012_val_00004641.JPEG:n03016953 +ILSVRC2012_val_00004642.JPEG:n03793489 +ILSVRC2012_val_00004643.JPEG:n02119022 +ILSVRC2012_val_00004644.JPEG:n04366367 +ILSVRC2012_val_00004645.JPEG:n03388549 +ILSVRC2012_val_00004646.JPEG:n06874185 +ILSVRC2012_val_00004647.JPEG:n02980441 +ILSVRC2012_val_00004648.JPEG:n03676483 +ILSVRC2012_val_00004649.JPEG:n04065272 +ILSVRC2012_val_00004650.JPEG:n02102040 +ILSVRC2012_val_00004651.JPEG:n04501370 +ILSVRC2012_val_00004652.JPEG:n01740131 +ILSVRC2012_val_00004653.JPEG:n04162706 +ILSVRC2012_val_00004654.JPEG:n04325704 +ILSVRC2012_val_00004655.JPEG:n01443537 +ILSVRC2012_val_00004656.JPEG:n02672831 +ILSVRC2012_val_00004657.JPEG:n02101006 +ILSVRC2012_val_00004658.JPEG:n04417672 +ILSVRC2012_val_00004659.JPEG:n01990800 +ILSVRC2012_val_00004660.JPEG:n02133161 +ILSVRC2012_val_00004661.JPEG:n02264363 +ILSVRC2012_val_00004662.JPEG:n04548280 +ILSVRC2012_val_00004663.JPEG:n03935335 +ILSVRC2012_val_00004664.JPEG:n02906734 +ILSVRC2012_val_00004665.JPEG:n01985128 +ILSVRC2012_val_00004666.JPEG:n02107574 +ILSVRC2012_val_00004667.JPEG:n03125729 +ILSVRC2012_val_00004668.JPEG:n03208938 +ILSVRC2012_val_00004669.JPEG:n02074367 +ILSVRC2012_val_00004670.JPEG:n03133878 +ILSVRC2012_val_00004671.JPEG:n02085782 +ILSVRC2012_val_00004672.JPEG:n02607072 +ILSVRC2012_val_00004673.JPEG:n03388043 +ILSVRC2012_val_00004674.JPEG:n02096585 +ILSVRC2012_val_00004675.JPEG:n07693725 +ILSVRC2012_val_00004676.JPEG:n02786058 +ILSVRC2012_val_00004677.JPEG:n01443537 +ILSVRC2012_val_00004678.JPEG:n01873310 +ILSVRC2012_val_00004679.JPEG:n02791124 +ILSVRC2012_val_00004680.JPEG:n04325704 +ILSVRC2012_val_00004681.JPEG:n03530642 +ILSVRC2012_val_00004682.JPEG:n04147183 +ILSVRC2012_val_00004683.JPEG:n02484975 +ILSVRC2012_val_00004684.JPEG:n02091635 +ILSVRC2012_val_00004685.JPEG:n03100240 +ILSVRC2012_val_00004686.JPEG:n02879718 +ILSVRC2012_val_00004687.JPEG:n02093991 +ILSVRC2012_val_00004688.JPEG:n11879895 +ILSVRC2012_val_00004689.JPEG:n01737021 +ILSVRC2012_val_00004690.JPEG:n13054560 +ILSVRC2012_val_00004691.JPEG:n01945685 +ILSVRC2012_val_00004692.JPEG:n04356056 +ILSVRC2012_val_00004693.JPEG:n02342885 +ILSVRC2012_val_00004694.JPEG:n04192698 +ILSVRC2012_val_00004695.JPEG:n04536866 +ILSVRC2012_val_00004696.JPEG:n04435653 +ILSVRC2012_val_00004697.JPEG:n01829413 +ILSVRC2012_val_00004698.JPEG:n01496331 +ILSVRC2012_val_00004699.JPEG:n03887697 +ILSVRC2012_val_00004700.JPEG:n03770679 +ILSVRC2012_val_00004701.JPEG:n12057211 +ILSVRC2012_val_00004702.JPEG:n12985857 +ILSVRC2012_val_00004703.JPEG:n04266014 +ILSVRC2012_val_00004704.JPEG:n02916936 +ILSVRC2012_val_00004705.JPEG:n04429376 +ILSVRC2012_val_00004706.JPEG:n02229544 +ILSVRC2012_val_00004707.JPEG:n03763968 +ILSVRC2012_val_00004708.JPEG:n03595614 +ILSVRC2012_val_00004709.JPEG:n02837789 +ILSVRC2012_val_00004710.JPEG:n02109047 +ILSVRC2012_val_00004711.JPEG:n02106030 +ILSVRC2012_val_00004712.JPEG:n03180011 +ILSVRC2012_val_00004713.JPEG:n02102973 +ILSVRC2012_val_00004714.JPEG:n02865351 +ILSVRC2012_val_00004715.JPEG:n02074367 +ILSVRC2012_val_00004716.JPEG:n02169497 +ILSVRC2012_val_00004717.JPEG:n02087046 +ILSVRC2012_val_00004718.JPEG:n03141823 +ILSVRC2012_val_00004719.JPEG:n02124075 +ILSVRC2012_val_00004720.JPEG:n02437312 +ILSVRC2012_val_00004721.JPEG:n07892512 +ILSVRC2012_val_00004722.JPEG:n01776313 +ILSVRC2012_val_00004723.JPEG:n02641379 +ILSVRC2012_val_00004724.JPEG:n01644900 +ILSVRC2012_val_00004725.JPEG:n03042490 +ILSVRC2012_val_00004726.JPEG:n03630383 +ILSVRC2012_val_00004727.JPEG:n03785016 +ILSVRC2012_val_00004728.JPEG:n07730033 +ILSVRC2012_val_00004729.JPEG:n03544143 +ILSVRC2012_val_00004730.JPEG:n02007558 +ILSVRC2012_val_00004731.JPEG:n02109047 +ILSVRC2012_val_00004732.JPEG:n02910353 +ILSVRC2012_val_00004733.JPEG:n02107312 +ILSVRC2012_val_00004734.JPEG:n02389026 +ILSVRC2012_val_00004735.JPEG:n01698640 +ILSVRC2012_val_00004736.JPEG:n03633091 +ILSVRC2012_val_00004737.JPEG:n04442312 +ILSVRC2012_val_00004738.JPEG:n07248320 +ILSVRC2012_val_00004739.JPEG:n04525038 +ILSVRC2012_val_00004740.JPEG:n03459775 +ILSVRC2012_val_00004741.JPEG:n03297495 +ILSVRC2012_val_00004742.JPEG:n03676483 +ILSVRC2012_val_00004743.JPEG:n03476991 +ILSVRC2012_val_00004744.JPEG:n02097658 +ILSVRC2012_val_00004745.JPEG:n03888257 +ILSVRC2012_val_00004746.JPEG:n02115913 +ILSVRC2012_val_00004747.JPEG:n01532829 +ILSVRC2012_val_00004748.JPEG:n02085936 +ILSVRC2012_val_00004749.JPEG:n01532829 +ILSVRC2012_val_00004750.JPEG:n02107312 +ILSVRC2012_val_00004751.JPEG:n02403003 +ILSVRC2012_val_00004752.JPEG:n03933933 +ILSVRC2012_val_00004753.JPEG:n02483362 +ILSVRC2012_val_00004754.JPEG:n02105162 +ILSVRC2012_val_00004755.JPEG:n02066245 +ILSVRC2012_val_00004756.JPEG:n01518878 +ILSVRC2012_val_00004757.JPEG:n01685808 +ILSVRC2012_val_00004758.JPEG:n03782006 +ILSVRC2012_val_00004759.JPEG:n07695742 +ILSVRC2012_val_00004760.JPEG:n09835506 +ILSVRC2012_val_00004761.JPEG:n04141076 +ILSVRC2012_val_00004762.JPEG:n02454379 +ILSVRC2012_val_00004763.JPEG:n02107683 +ILSVRC2012_val_00004764.JPEG:n03874293 +ILSVRC2012_val_00004765.JPEG:n02177972 +ILSVRC2012_val_00004766.JPEG:n02106166 +ILSVRC2012_val_00004767.JPEG:n04590129 +ILSVRC2012_val_00004768.JPEG:n03388549 +ILSVRC2012_val_00004769.JPEG:n04399382 +ILSVRC2012_val_00004770.JPEG:n02096585 +ILSVRC2012_val_00004771.JPEG:n02093256 +ILSVRC2012_val_00004772.JPEG:n02319095 +ILSVRC2012_val_00004773.JPEG:n04560804 +ILSVRC2012_val_00004774.JPEG:n02089973 +ILSVRC2012_val_00004775.JPEG:n03223299 +ILSVRC2012_val_00004776.JPEG:n02091244 +ILSVRC2012_val_00004777.JPEG:n02089867 +ILSVRC2012_val_00004778.JPEG:n04335435 +ILSVRC2012_val_00004779.JPEG:n03825788 +ILSVRC2012_val_00004780.JPEG:n02056570 +ILSVRC2012_val_00004781.JPEG:n01669191 +ILSVRC2012_val_00004782.JPEG:n02113978 +ILSVRC2012_val_00004783.JPEG:n03141823 +ILSVRC2012_val_00004784.JPEG:n02640242 +ILSVRC2012_val_00004785.JPEG:n02841315 +ILSVRC2012_val_00004786.JPEG:n04146614 +ILSVRC2012_val_00004787.JPEG:n03400231 +ILSVRC2012_val_00004788.JPEG:n02490219 +ILSVRC2012_val_00004789.JPEG:n03791053 +ILSVRC2012_val_00004790.JPEG:n07880968 +ILSVRC2012_val_00004791.JPEG:n02025239 +ILSVRC2012_val_00004792.JPEG:n03873416 +ILSVRC2012_val_00004793.JPEG:n02437616 +ILSVRC2012_val_00004794.JPEG:n03220513 +ILSVRC2012_val_00004795.JPEG:n02089973 +ILSVRC2012_val_00004796.JPEG:n03045698 +ILSVRC2012_val_00004797.JPEG:n02100735 +ILSVRC2012_val_00004798.JPEG:n04228054 +ILSVRC2012_val_00004799.JPEG:n06785654 +ILSVRC2012_val_00004800.JPEG:n04554684 +ILSVRC2012_val_00004801.JPEG:n03595614 +ILSVRC2012_val_00004802.JPEG:n03933933 +ILSVRC2012_val_00004803.JPEG:n03954731 +ILSVRC2012_val_00004804.JPEG:n02110806 +ILSVRC2012_val_00004805.JPEG:n02056570 +ILSVRC2012_val_00004806.JPEG:n04476259 +ILSVRC2012_val_00004807.JPEG:n03032252 +ILSVRC2012_val_00004808.JPEG:n02445715 +ILSVRC2012_val_00004809.JPEG:n03895866 +ILSVRC2012_val_00004810.JPEG:n02317335 +ILSVRC2012_val_00004811.JPEG:n04479046 +ILSVRC2012_val_00004812.JPEG:n02782093 +ILSVRC2012_val_00004813.JPEG:n02172182 +ILSVRC2012_val_00004814.JPEG:n02417914 +ILSVRC2012_val_00004815.JPEG:n03041632 +ILSVRC2012_val_00004816.JPEG:n04507155 +ILSVRC2012_val_00004817.JPEG:n02672831 +ILSVRC2012_val_00004818.JPEG:n02108000 +ILSVRC2012_val_00004819.JPEG:n07714990 +ILSVRC2012_val_00004820.JPEG:n03532672 +ILSVRC2012_val_00004821.JPEG:n02123597 +ILSVRC2012_val_00004822.JPEG:n03218198 +ILSVRC2012_val_00004823.JPEG:n02091134 +ILSVRC2012_val_00004824.JPEG:n02825657 +ILSVRC2012_val_00004825.JPEG:n02916936 +ILSVRC2012_val_00004826.JPEG:n03874599 +ILSVRC2012_val_00004827.JPEG:n03876231 +ILSVRC2012_val_00004828.JPEG:n03160309 +ILSVRC2012_val_00004829.JPEG:n04118538 +ILSVRC2012_val_00004830.JPEG:n03259280 +ILSVRC2012_val_00004831.JPEG:n03670208 +ILSVRC2012_val_00004832.JPEG:n07745940 +ILSVRC2012_val_00004833.JPEG:n03733805 +ILSVRC2012_val_00004834.JPEG:n01669191 +ILSVRC2012_val_00004835.JPEG:n03404251 +ILSVRC2012_val_00004836.JPEG:n07718747 +ILSVRC2012_val_00004837.JPEG:n07831146 +ILSVRC2012_val_00004838.JPEG:n02403003 +ILSVRC2012_val_00004839.JPEG:n02883205 +ILSVRC2012_val_00004840.JPEG:n02415577 +ILSVRC2012_val_00004841.JPEG:n01784675 +ILSVRC2012_val_00004842.JPEG:n02492035 +ILSVRC2012_val_00004843.JPEG:n03599486 +ILSVRC2012_val_00004844.JPEG:n01877812 +ILSVRC2012_val_00004845.JPEG:n01877812 +ILSVRC2012_val_00004846.JPEG:n03498962 +ILSVRC2012_val_00004847.JPEG:n04355338 +ILSVRC2012_val_00004848.JPEG:n03617480 +ILSVRC2012_val_00004849.JPEG:n03404251 +ILSVRC2012_val_00004850.JPEG:n02277742 +ILSVRC2012_val_00004851.JPEG:n02169497 +ILSVRC2012_val_00004852.JPEG:n02113624 +ILSVRC2012_val_00004853.JPEG:n04067472 +ILSVRC2012_val_00004854.JPEG:n04465501 +ILSVRC2012_val_00004855.JPEG:n04335435 +ILSVRC2012_val_00004856.JPEG:n02444819 +ILSVRC2012_val_00004857.JPEG:n09421951 +ILSVRC2012_val_00004858.JPEG:n04591157 +ILSVRC2012_val_00004859.JPEG:n01622779 +ILSVRC2012_val_00004860.JPEG:n03425413 +ILSVRC2012_val_00004861.JPEG:n02346627 +ILSVRC2012_val_00004862.JPEG:n04162706 +ILSVRC2012_val_00004863.JPEG:n03874293 +ILSVRC2012_val_00004864.JPEG:n02138441 +ILSVRC2012_val_00004865.JPEG:n04005630 +ILSVRC2012_val_00004866.JPEG:n03769881 +ILSVRC2012_val_00004867.JPEG:n03942813 +ILSVRC2012_val_00004868.JPEG:n04285008 +ILSVRC2012_val_00004869.JPEG:n02114855 +ILSVRC2012_val_00004870.JPEG:n02114712 +ILSVRC2012_val_00004871.JPEG:n02708093 +ILSVRC2012_val_00004872.JPEG:n03124170 +ILSVRC2012_val_00004873.JPEG:n01498041 +ILSVRC2012_val_00004874.JPEG:n07613480 +ILSVRC2012_val_00004875.JPEG:n02363005 +ILSVRC2012_val_00004876.JPEG:n03355925 +ILSVRC2012_val_00004877.JPEG:n13054560 +ILSVRC2012_val_00004878.JPEG:n03180011 +ILSVRC2012_val_00004879.JPEG:n04552348 +ILSVRC2012_val_00004880.JPEG:n02423022 +ILSVRC2012_val_00004881.JPEG:n04525038 +ILSVRC2012_val_00004882.JPEG:n02504013 +ILSVRC2012_val_00004883.JPEG:n02107312 +ILSVRC2012_val_00004884.JPEG:n02091467 +ILSVRC2012_val_00004885.JPEG:n02101006 +ILSVRC2012_val_00004886.JPEG:n03721384 +ILSVRC2012_val_00004887.JPEG:n07695742 +ILSVRC2012_val_00004888.JPEG:n02823428 +ILSVRC2012_val_00004889.JPEG:n04589890 +ILSVRC2012_val_00004890.JPEG:n04584207 +ILSVRC2012_val_00004891.JPEG:n04111531 +ILSVRC2012_val_00004892.JPEG:n03160309 +ILSVRC2012_val_00004893.JPEG:n01531178 +ILSVRC2012_val_00004894.JPEG:n02123394 +ILSVRC2012_val_00004895.JPEG:n02777292 +ILSVRC2012_val_00004896.JPEG:n04208210 +ILSVRC2012_val_00004897.JPEG:n01667114 +ILSVRC2012_val_00004898.JPEG:n01667114 +ILSVRC2012_val_00004899.JPEG:n04597913 +ILSVRC2012_val_00004900.JPEG:n03529860 +ILSVRC2012_val_00004901.JPEG:n03450230 +ILSVRC2012_val_00004902.JPEG:n02123045 +ILSVRC2012_val_00004903.JPEG:n12768682 +ILSVRC2012_val_00004904.JPEG:n01924916 +ILSVRC2012_val_00004905.JPEG:n02536864 +ILSVRC2012_val_00004906.JPEG:n04442312 +ILSVRC2012_val_00004907.JPEG:n02747177 +ILSVRC2012_val_00004908.JPEG:n07831146 +ILSVRC2012_val_00004909.JPEG:n02951358 +ILSVRC2012_val_00004910.JPEG:n03857828 +ILSVRC2012_val_00004911.JPEG:n03482405 +ILSVRC2012_val_00004912.JPEG:n03028079 +ILSVRC2012_val_00004913.JPEG:n04040759 +ILSVRC2012_val_00004914.JPEG:n02417914 +ILSVRC2012_val_00004915.JPEG:n01689811 +ILSVRC2012_val_00004916.JPEG:n03188531 +ILSVRC2012_val_00004917.JPEG:n04070727 +ILSVRC2012_val_00004918.JPEG:n07720875 +ILSVRC2012_val_00004919.JPEG:n02168699 +ILSVRC2012_val_00004920.JPEG:n11939491 +ILSVRC2012_val_00004921.JPEG:n01704323 +ILSVRC2012_val_00004922.JPEG:n03223299 +ILSVRC2012_val_00004923.JPEG:n01930112 +ILSVRC2012_val_00004924.JPEG:n02747177 +ILSVRC2012_val_00004925.JPEG:n03903868 +ILSVRC2012_val_00004926.JPEG:n02093428 +ILSVRC2012_val_00004927.JPEG:n01728572 +ILSVRC2012_val_00004928.JPEG:n03459775 +ILSVRC2012_val_00004929.JPEG:n04409515 +ILSVRC2012_val_00004930.JPEG:n03977966 +ILSVRC2012_val_00004931.JPEG:n03220513 +ILSVRC2012_val_00004932.JPEG:n04355933 +ILSVRC2012_val_00004933.JPEG:n03662601 +ILSVRC2012_val_00004934.JPEG:n03916031 +ILSVRC2012_val_00004935.JPEG:n07836838 +ILSVRC2012_val_00004936.JPEG:n07714571 +ILSVRC2012_val_00004937.JPEG:n03891332 +ILSVRC2012_val_00004938.JPEG:n02105251 +ILSVRC2012_val_00004939.JPEG:n03028079 +ILSVRC2012_val_00004940.JPEG:n02117135 +ILSVRC2012_val_00004941.JPEG:n02096585 +ILSVRC2012_val_00004942.JPEG:n04458633 +ILSVRC2012_val_00004943.JPEG:n02883205 +ILSVRC2012_val_00004944.JPEG:n01818515 +ILSVRC2012_val_00004945.JPEG:n01641577 +ILSVRC2012_val_00004946.JPEG:n04070727 +ILSVRC2012_val_00004947.JPEG:n02093428 +ILSVRC2012_val_00004948.JPEG:n03494278 +ILSVRC2012_val_00004949.JPEG:n03255030 +ILSVRC2012_val_00004950.JPEG:n03769881 +ILSVRC2012_val_00004951.JPEG:n07716358 +ILSVRC2012_val_00004952.JPEG:n03877845 +ILSVRC2012_val_00004953.JPEG:n07760859 +ILSVRC2012_val_00004954.JPEG:n03495258 +ILSVRC2012_val_00004955.JPEG:n04370456 +ILSVRC2012_val_00004956.JPEG:n02091134 +ILSVRC2012_val_00004957.JPEG:n03874293 +ILSVRC2012_val_00004958.JPEG:n03026506 +ILSVRC2012_val_00004959.JPEG:n03259280 +ILSVRC2012_val_00004960.JPEG:n02097209 +ILSVRC2012_val_00004961.JPEG:n03873416 +ILSVRC2012_val_00004962.JPEG:n07760859 +ILSVRC2012_val_00004963.JPEG:n02108422 +ILSVRC2012_val_00004964.JPEG:n01872401 +ILSVRC2012_val_00004965.JPEG:n01981276 +ILSVRC2012_val_00004966.JPEG:n04153751 +ILSVRC2012_val_00004967.JPEG:n02110185 +ILSVRC2012_val_00004968.JPEG:n02095570 +ILSVRC2012_val_00004969.JPEG:n01496331 +ILSVRC2012_val_00004970.JPEG:n04285008 +ILSVRC2012_val_00004971.JPEG:n03075370 +ILSVRC2012_val_00004972.JPEG:n02815834 +ILSVRC2012_val_00004973.JPEG:n09256479 +ILSVRC2012_val_00004974.JPEG:n02092339 +ILSVRC2012_val_00004975.JPEG:n02808304 +ILSVRC2012_val_00004976.JPEG:n09428293 +ILSVRC2012_val_00004977.JPEG:n02101006 +ILSVRC2012_val_00004978.JPEG:n02412080 +ILSVRC2012_val_00004979.JPEG:n04285008 +ILSVRC2012_val_00004980.JPEG:n03954731 +ILSVRC2012_val_00004981.JPEG:n04311004 +ILSVRC2012_val_00004982.JPEG:n03476991 +ILSVRC2012_val_00004983.JPEG:n01518878 +ILSVRC2012_val_00004984.JPEG:n02687172 +ILSVRC2012_val_00004985.JPEG:n02342885 +ILSVRC2012_val_00004986.JPEG:n02346627 +ILSVRC2012_val_00004987.JPEG:n02883205 +ILSVRC2012_val_00004988.JPEG:n03457902 +ILSVRC2012_val_00004989.JPEG:n02097658 +ILSVRC2012_val_00004990.JPEG:n02504458 +ILSVRC2012_val_00004991.JPEG:n03930313 +ILSVRC2012_val_00004992.JPEG:n02087394 +ILSVRC2012_val_00004993.JPEG:n02802426 +ILSVRC2012_val_00004994.JPEG:n03272010 +ILSVRC2012_val_00004995.JPEG:n02102318 +ILSVRC2012_val_00004996.JPEG:n02091467 +ILSVRC2012_val_00004997.JPEG:n02099849 +ILSVRC2012_val_00004998.JPEG:n04552348 +ILSVRC2012_val_00004999.JPEG:n02443114 +ILSVRC2012_val_00005000.JPEG:n02276258 +ILSVRC2012_val_00005001.JPEG:n03642806 +ILSVRC2012_val_00005002.JPEG:n02342885 +ILSVRC2012_val_00005003.JPEG:n03916031 +ILSVRC2012_val_00005004.JPEG:n02125311 +ILSVRC2012_val_00005005.JPEG:n02837789 +ILSVRC2012_val_00005006.JPEG:n02130308 +ILSVRC2012_val_00005007.JPEG:n04509417 +ILSVRC2012_val_00005008.JPEG:n03207941 +ILSVRC2012_val_00005009.JPEG:n03877845 +ILSVRC2012_val_00005010.JPEG:n13052670 +ILSVRC2012_val_00005011.JPEG:n02317335 +ILSVRC2012_val_00005012.JPEG:n03444034 +ILSVRC2012_val_00005013.JPEG:n03179701 +ILSVRC2012_val_00005014.JPEG:n04371774 +ILSVRC2012_val_00005015.JPEG:n03924679 +ILSVRC2012_val_00005016.JPEG:n02950826 +ILSVRC2012_val_00005017.JPEG:n02110958 +ILSVRC2012_val_00005018.JPEG:n02113978 +ILSVRC2012_val_00005019.JPEG:n02109961 +ILSVRC2012_val_00005020.JPEG:n02363005 +ILSVRC2012_val_00005021.JPEG:n02090622 +ILSVRC2012_val_00005022.JPEG:n07930864 +ILSVRC2012_val_00005023.JPEG:n03857828 +ILSVRC2012_val_00005024.JPEG:n03763968 +ILSVRC2012_val_00005025.JPEG:n07684084 +ILSVRC2012_val_00005026.JPEG:n02497673 +ILSVRC2012_val_00005027.JPEG:n02102480 +ILSVRC2012_val_00005028.JPEG:n04275548 +ILSVRC2012_val_00005029.JPEG:n04264628 +ILSVRC2012_val_00005030.JPEG:n02058221 +ILSVRC2012_val_00005031.JPEG:n01687978 +ILSVRC2012_val_00005032.JPEG:n02877765 +ILSVRC2012_val_00005033.JPEG:n01748264 +ILSVRC2012_val_00005034.JPEG:n02028035 +ILSVRC2012_val_00005035.JPEG:n02909870 +ILSVRC2012_val_00005036.JPEG:n04332243 +ILSVRC2012_val_00005037.JPEG:n09835506 +ILSVRC2012_val_00005038.JPEG:n04192698 +ILSVRC2012_val_00005039.JPEG:n03877845 +ILSVRC2012_val_00005040.JPEG:n03832673 +ILSVRC2012_val_00005041.JPEG:n04179913 +ILSVRC2012_val_00005042.JPEG:n03623198 +ILSVRC2012_val_00005043.JPEG:n02107908 +ILSVRC2012_val_00005044.JPEG:n04548362 +ILSVRC2012_val_00005045.JPEG:n01641577 +ILSVRC2012_val_00005046.JPEG:n02992211 +ILSVRC2012_val_00005047.JPEG:n04326547 +ILSVRC2012_val_00005048.JPEG:n02783161 +ILSVRC2012_val_00005049.JPEG:n03743016 +ILSVRC2012_val_00005050.JPEG:n01729977 +ILSVRC2012_val_00005051.JPEG:n04146614 +ILSVRC2012_val_00005052.JPEG:n01695060 +ILSVRC2012_val_00005053.JPEG:n03649909 +ILSVRC2012_val_00005054.JPEG:n02087394 +ILSVRC2012_val_00005055.JPEG:n03424325 +ILSVRC2012_val_00005056.JPEG:n01688243 +ILSVRC2012_val_00005057.JPEG:n03223299 +ILSVRC2012_val_00005058.JPEG:n01914609 +ILSVRC2012_val_00005059.JPEG:n02091032 +ILSVRC2012_val_00005060.JPEG:n02095570 +ILSVRC2012_val_00005061.JPEG:n07720875 +ILSVRC2012_val_00005062.JPEG:n02606052 +ILSVRC2012_val_00005063.JPEG:n03584829 +ILSVRC2012_val_00005064.JPEG:n02110185 +ILSVRC2012_val_00005065.JPEG:n03220513 +ILSVRC2012_val_00005066.JPEG:n07745940 +ILSVRC2012_val_00005067.JPEG:n01824575 +ILSVRC2012_val_00005068.JPEG:n02099601 +ILSVRC2012_val_00005069.JPEG:n11939491 +ILSVRC2012_val_00005070.JPEG:n07749582 +ILSVRC2012_val_00005071.JPEG:n03457902 +ILSVRC2012_val_00005072.JPEG:n01784675 +ILSVRC2012_val_00005073.JPEG:n02112018 +ILSVRC2012_val_00005074.JPEG:n03733131 +ILSVRC2012_val_00005075.JPEG:n04328186 +ILSVRC2012_val_00005076.JPEG:n04037443 +ILSVRC2012_val_00005077.JPEG:n03717622 +ILSVRC2012_val_00005078.JPEG:n01694178 +ILSVRC2012_val_00005079.JPEG:n02871525 +ILSVRC2012_val_00005080.JPEG:n02808440 +ILSVRC2012_val_00005081.JPEG:n04560804 +ILSVRC2012_val_00005082.JPEG:n02097474 +ILSVRC2012_val_00005083.JPEG:n02137549 +ILSVRC2012_val_00005084.JPEG:n01981276 +ILSVRC2012_val_00005085.JPEG:n02443114 +ILSVRC2012_val_00005086.JPEG:n02101006 +ILSVRC2012_val_00005087.JPEG:n04550184 +ILSVRC2012_val_00005088.JPEG:n12985857 +ILSVRC2012_val_00005089.JPEG:n02236044 +ILSVRC2012_val_00005090.JPEG:n02488291 +ILSVRC2012_val_00005091.JPEG:n04532106 +ILSVRC2012_val_00005092.JPEG:n03895866 +ILSVRC2012_val_00005093.JPEG:n03617480 +ILSVRC2012_val_00005094.JPEG:n03417042 +ILSVRC2012_val_00005095.JPEG:n03903868 +ILSVRC2012_val_00005096.JPEG:n03584254 +ILSVRC2012_val_00005097.JPEG:n02389026 +ILSVRC2012_val_00005098.JPEG:n04435653 +ILSVRC2012_val_00005099.JPEG:n02492035 +ILSVRC2012_val_00005100.JPEG:n01796340 +ILSVRC2012_val_00005101.JPEG:n03447721 +ILSVRC2012_val_00005102.JPEG:n03447447 +ILSVRC2012_val_00005103.JPEG:n03595614 +ILSVRC2012_val_00005104.JPEG:n04579145 +ILSVRC2012_val_00005105.JPEG:n02777292 +ILSVRC2012_val_00005106.JPEG:n04147183 +ILSVRC2012_val_00005107.JPEG:n02006656 +ILSVRC2012_val_00005108.JPEG:n03843555 +ILSVRC2012_val_00005109.JPEG:n02504458 +ILSVRC2012_val_00005110.JPEG:n03444034 +ILSVRC2012_val_00005111.JPEG:n03673027 +ILSVRC2012_val_00005112.JPEG:n04417672 +ILSVRC2012_val_00005113.JPEG:n10148035 +ILSVRC2012_val_00005114.JPEG:n04179913 +ILSVRC2012_val_00005115.JPEG:n03792972 +ILSVRC2012_val_00005116.JPEG:n04552348 +ILSVRC2012_val_00005117.JPEG:n02281406 +ILSVRC2012_val_00005118.JPEG:n02326432 +ILSVRC2012_val_00005119.JPEG:n02493509 +ILSVRC2012_val_00005120.JPEG:n03314780 +ILSVRC2012_val_00005121.JPEG:n03485407 +ILSVRC2012_val_00005122.JPEG:n01980166 +ILSVRC2012_val_00005123.JPEG:n04442312 +ILSVRC2012_val_00005124.JPEG:n03602883 +ILSVRC2012_val_00005125.JPEG:n01986214 +ILSVRC2012_val_00005126.JPEG:n02108915 +ILSVRC2012_val_00005127.JPEG:n02492660 +ILSVRC2012_val_00005128.JPEG:n03384352 +ILSVRC2012_val_00005129.JPEG:n04367480 +ILSVRC2012_val_00005130.JPEG:n04467665 +ILSVRC2012_val_00005131.JPEG:n02814860 +ILSVRC2012_val_00005132.JPEG:n01728572 +ILSVRC2012_val_00005133.JPEG:n03733281 +ILSVRC2012_val_00005134.JPEG:n03216828 +ILSVRC2012_val_00005135.JPEG:n02494079 +ILSVRC2012_val_00005136.JPEG:n03733805 +ILSVRC2012_val_00005137.JPEG:n02279972 +ILSVRC2012_val_00005138.JPEG:n01692333 +ILSVRC2012_val_00005139.JPEG:n02091635 +ILSVRC2012_val_00005140.JPEG:n04487081 +ILSVRC2012_val_00005141.JPEG:n03866082 +ILSVRC2012_val_00005142.JPEG:n03208938 +ILSVRC2012_val_00005143.JPEG:n07714990 +ILSVRC2012_val_00005144.JPEG:n02906734 +ILSVRC2012_val_00005145.JPEG:n02807133 +ILSVRC2012_val_00005146.JPEG:n02095570 +ILSVRC2012_val_00005147.JPEG:n03594945 +ILSVRC2012_val_00005148.JPEG:n03492542 +ILSVRC2012_val_00005149.JPEG:n02442845 +ILSVRC2012_val_00005150.JPEG:n01833805 +ILSVRC2012_val_00005151.JPEG:n02395406 +ILSVRC2012_val_00005152.JPEG:n06874185 +ILSVRC2012_val_00005153.JPEG:n02490219 +ILSVRC2012_val_00005154.JPEG:n02071294 +ILSVRC2012_val_00005155.JPEG:n02447366 +ILSVRC2012_val_00005156.JPEG:n01537544 +ILSVRC2012_val_00005157.JPEG:n02281787 +ILSVRC2012_val_00005158.JPEG:n02268443 +ILSVRC2012_val_00005159.JPEG:n03775546 +ILSVRC2012_val_00005160.JPEG:n04429376 +ILSVRC2012_val_00005161.JPEG:n03832673 +ILSVRC2012_val_00005162.JPEG:n04398044 +ILSVRC2012_val_00005163.JPEG:n04370456 +ILSVRC2012_val_00005164.JPEG:n02128757 +ILSVRC2012_val_00005165.JPEG:n04162706 +ILSVRC2012_val_00005166.JPEG:n04146614 +ILSVRC2012_val_00005167.JPEG:n04482393 +ILSVRC2012_val_00005168.JPEG:n07860988 +ILSVRC2012_val_00005169.JPEG:n02167151 +ILSVRC2012_val_00005170.JPEG:n02095889 +ILSVRC2012_val_00005171.JPEG:n02487347 +ILSVRC2012_val_00005172.JPEG:n01632777 +ILSVRC2012_val_00005173.JPEG:n02992211 +ILSVRC2012_val_00005174.JPEG:n02097658 +ILSVRC2012_val_00005175.JPEG:n02107683 +ILSVRC2012_val_00005176.JPEG:n03980874 +ILSVRC2012_val_00005177.JPEG:n07753592 +ILSVRC2012_val_00005178.JPEG:n02037110 +ILSVRC2012_val_00005179.JPEG:n03388183 +ILSVRC2012_val_00005180.JPEG:n01695060 +ILSVRC2012_val_00005181.JPEG:n04258138 +ILSVRC2012_val_00005182.JPEG:n02802426 +ILSVRC2012_val_00005183.JPEG:n03425413 +ILSVRC2012_val_00005184.JPEG:n02403003 +ILSVRC2012_val_00005185.JPEG:n03868242 +ILSVRC2012_val_00005186.JPEG:n02006656 +ILSVRC2012_val_00005187.JPEG:n02667093 +ILSVRC2012_val_00005188.JPEG:n02607072 +ILSVRC2012_val_00005189.JPEG:n02093647 +ILSVRC2012_val_00005190.JPEG:n02536864 +ILSVRC2012_val_00005191.JPEG:n04591713 +ILSVRC2012_val_00005192.JPEG:n02669723 +ILSVRC2012_val_00005193.JPEG:n03733805 +ILSVRC2012_val_00005194.JPEG:n03259280 +ILSVRC2012_val_00005195.JPEG:n03709823 +ILSVRC2012_val_00005196.JPEG:n04483307 +ILSVRC2012_val_00005197.JPEG:n03877472 +ILSVRC2012_val_00005198.JPEG:n02113023 +ILSVRC2012_val_00005199.JPEG:n04133789 +ILSVRC2012_val_00005200.JPEG:n06359193 +ILSVRC2012_val_00005201.JPEG:n03903868 +ILSVRC2012_val_00005202.JPEG:n03089624 +ILSVRC2012_val_00005203.JPEG:n02013706 +ILSVRC2012_val_00005204.JPEG:n04266014 +ILSVRC2012_val_00005205.JPEG:n02504013 +ILSVRC2012_val_00005206.JPEG:n02101006 +ILSVRC2012_val_00005207.JPEG:n02124075 +ILSVRC2012_val_00005208.JPEG:n01774750 +ILSVRC2012_val_00005209.JPEG:n02112350 +ILSVRC2012_val_00005210.JPEG:n02526121 +ILSVRC2012_val_00005211.JPEG:n03485407 +ILSVRC2012_val_00005212.JPEG:n03496892 +ILSVRC2012_val_00005213.JPEG:n02655020 +ILSVRC2012_val_00005214.JPEG:n07714571 +ILSVRC2012_val_00005215.JPEG:n02087394 +ILSVRC2012_val_00005216.JPEG:n03160309 +ILSVRC2012_val_00005217.JPEG:n02091831 +ILSVRC2012_val_00005218.JPEG:n03047690 +ILSVRC2012_val_00005219.JPEG:n04612504 +ILSVRC2012_val_00005220.JPEG:n02859443 +ILSVRC2012_val_00005221.JPEG:n04033995 +ILSVRC2012_val_00005222.JPEG:n02950826 +ILSVRC2012_val_00005223.JPEG:n03187595 +ILSVRC2012_val_00005224.JPEG:n01592084 +ILSVRC2012_val_00005225.JPEG:n07892512 +ILSVRC2012_val_00005226.JPEG:n04507155 +ILSVRC2012_val_00005227.JPEG:n01692333 +ILSVRC2012_val_00005228.JPEG:n01981276 +ILSVRC2012_val_00005229.JPEG:n02823750 +ILSVRC2012_val_00005230.JPEG:n04251144 +ILSVRC2012_val_00005231.JPEG:n04548362 +ILSVRC2012_val_00005232.JPEG:n07565083 +ILSVRC2012_val_00005233.JPEG:n04209133 +ILSVRC2012_val_00005234.JPEG:n01877812 +ILSVRC2012_val_00005235.JPEG:n04486054 +ILSVRC2012_val_00005236.JPEG:n09421951 +ILSVRC2012_val_00005237.JPEG:n02231487 +ILSVRC2012_val_00005238.JPEG:n02113799 +ILSVRC2012_val_00005239.JPEG:n02098413 +ILSVRC2012_val_00005240.JPEG:n04081281 +ILSVRC2012_val_00005241.JPEG:n02999410 +ILSVRC2012_val_00005242.JPEG:n02107312 +ILSVRC2012_val_00005243.JPEG:n02346627 +ILSVRC2012_val_00005244.JPEG:n01675722 +ILSVRC2012_val_00005245.JPEG:n02795169 +ILSVRC2012_val_00005246.JPEG:n03649909 +ILSVRC2012_val_00005247.JPEG:n04090263 +ILSVRC2012_val_00005248.JPEG:n03871628 +ILSVRC2012_val_00005249.JPEG:n01877812 +ILSVRC2012_val_00005250.JPEG:n03670208 +ILSVRC2012_val_00005251.JPEG:n03866082 +ILSVRC2012_val_00005252.JPEG:n03496892 +ILSVRC2012_val_00005253.JPEG:n07248320 +ILSVRC2012_val_00005254.JPEG:n04162706 +ILSVRC2012_val_00005255.JPEG:n02098413 +ILSVRC2012_val_00005256.JPEG:n04069434 +ILSVRC2012_val_00005257.JPEG:n03938244 +ILSVRC2012_val_00005258.JPEG:n02101006 +ILSVRC2012_val_00005259.JPEG:n02325366 +ILSVRC2012_val_00005260.JPEG:n03388549 +ILSVRC2012_val_00005261.JPEG:n03393912 +ILSVRC2012_val_00005262.JPEG:n01739381 +ILSVRC2012_val_00005263.JPEG:n02108089 +ILSVRC2012_val_00005264.JPEG:n03000134 +ILSVRC2012_val_00005265.JPEG:n03124170 +ILSVRC2012_val_00005266.JPEG:n02037110 +ILSVRC2012_val_00005267.JPEG:n02098105 +ILSVRC2012_val_00005268.JPEG:n01986214 +ILSVRC2012_val_00005269.JPEG:n03314780 +ILSVRC2012_val_00005270.JPEG:n10148035 +ILSVRC2012_val_00005271.JPEG:n04200800 +ILSVRC2012_val_00005272.JPEG:n03457902 +ILSVRC2012_val_00005273.JPEG:n02091831 +ILSVRC2012_val_00005274.JPEG:n02835271 +ILSVRC2012_val_00005275.JPEG:n03642806 +ILSVRC2012_val_00005276.JPEG:n02101388 +ILSVRC2012_val_00005277.JPEG:n02128757 +ILSVRC2012_val_00005278.JPEG:n04004767 +ILSVRC2012_val_00005279.JPEG:n02091635 +ILSVRC2012_val_00005280.JPEG:n04311004 +ILSVRC2012_val_00005281.JPEG:n04328186 +ILSVRC2012_val_00005282.JPEG:n01829413 +ILSVRC2012_val_00005283.JPEG:n02108000 +ILSVRC2012_val_00005284.JPEG:n03877845 +ILSVRC2012_val_00005285.JPEG:n03935335 +ILSVRC2012_val_00005286.JPEG:n01744401 +ILSVRC2012_val_00005287.JPEG:n01531178 +ILSVRC2012_val_00005288.JPEG:n13044778 +ILSVRC2012_val_00005289.JPEG:n02699494 +ILSVRC2012_val_00005290.JPEG:n01775062 +ILSVRC2012_val_00005291.JPEG:n02088364 +ILSVRC2012_val_00005292.JPEG:n04239074 +ILSVRC2012_val_00005293.JPEG:n03781244 +ILSVRC2012_val_00005294.JPEG:n02442845 +ILSVRC2012_val_00005295.JPEG:n03028079 +ILSVRC2012_val_00005296.JPEG:n09421951 +ILSVRC2012_val_00005297.JPEG:n12768682 +ILSVRC2012_val_00005298.JPEG:n02454379 +ILSVRC2012_val_00005299.JPEG:n03065424 +ILSVRC2012_val_00005300.JPEG:n02113023 +ILSVRC2012_val_00005301.JPEG:n01873310 +ILSVRC2012_val_00005302.JPEG:n03594945 +ILSVRC2012_val_00005303.JPEG:n03792782 +ILSVRC2012_val_00005304.JPEG:n03529860 +ILSVRC2012_val_00005305.JPEG:n02174001 +ILSVRC2012_val_00005306.JPEG:n02487347 +ILSVRC2012_val_00005307.JPEG:n01692333 +ILSVRC2012_val_00005308.JPEG:n02837789 +ILSVRC2012_val_00005309.JPEG:n04487394 +ILSVRC2012_val_00005310.JPEG:n02509815 +ILSVRC2012_val_00005311.JPEG:n03970156 +ILSVRC2012_val_00005312.JPEG:n02445715 +ILSVRC2012_val_00005313.JPEG:n02666196 +ILSVRC2012_val_00005314.JPEG:n02009912 +ILSVRC2012_val_00005315.JPEG:n01797886 +ILSVRC2012_val_00005316.JPEG:n07583066 +ILSVRC2012_val_00005317.JPEG:n02111500 +ILSVRC2012_val_00005318.JPEG:n03461385 +ILSVRC2012_val_00005319.JPEG:n04371774 +ILSVRC2012_val_00005320.JPEG:n04296562 +ILSVRC2012_val_00005321.JPEG:n02978881 +ILSVRC2012_val_00005322.JPEG:n02066245 +ILSVRC2012_val_00005323.JPEG:n02129604 +ILSVRC2012_val_00005324.JPEG:n03761084 +ILSVRC2012_val_00005325.JPEG:n09229709 +ILSVRC2012_val_00005326.JPEG:n01774750 +ILSVRC2012_val_00005327.JPEG:n02108915 +ILSVRC2012_val_00005328.JPEG:n01797886 +ILSVRC2012_val_00005329.JPEG:n04482393 +ILSVRC2012_val_00005330.JPEG:n03792782 +ILSVRC2012_val_00005331.JPEG:n02095314 +ILSVRC2012_val_00005332.JPEG:n01693334 +ILSVRC2012_val_00005333.JPEG:n04560804 +ILSVRC2012_val_00005334.JPEG:n04376876 +ILSVRC2012_val_00005335.JPEG:n07718747 +ILSVRC2012_val_00005336.JPEG:n01532829 +ILSVRC2012_val_00005337.JPEG:n03888605 +ILSVRC2012_val_00005338.JPEG:n02980441 +ILSVRC2012_val_00005339.JPEG:n01494475 +ILSVRC2012_val_00005340.JPEG:n02093754 +ILSVRC2012_val_00005341.JPEG:n07802026 +ILSVRC2012_val_00005342.JPEG:n04562935 +ILSVRC2012_val_00005343.JPEG:n02165456 +ILSVRC2012_val_00005344.JPEG:n02356798 +ILSVRC2012_val_00005345.JPEG:n03977966 +ILSVRC2012_val_00005346.JPEG:n03124170 +ILSVRC2012_val_00005347.JPEG:n02797295 +ILSVRC2012_val_00005348.JPEG:n04201297 +ILSVRC2012_val_00005349.JPEG:n04392985 +ILSVRC2012_val_00005350.JPEG:n04579432 +ILSVRC2012_val_00005351.JPEG:n02106550 +ILSVRC2012_val_00005352.JPEG:n02782093 +ILSVRC2012_val_00005353.JPEG:n04252077 +ILSVRC2012_val_00005354.JPEG:n04326547 +ILSVRC2012_val_00005355.JPEG:n02454379 +ILSVRC2012_val_00005356.JPEG:n02437312 +ILSVRC2012_val_00005357.JPEG:n01729977 +ILSVRC2012_val_00005358.JPEG:n02123045 +ILSVRC2012_val_00005359.JPEG:n04229816 +ILSVRC2012_val_00005360.JPEG:n02077923 +ILSVRC2012_val_00005361.JPEG:n03788195 +ILSVRC2012_val_00005362.JPEG:n02124075 +ILSVRC2012_val_00005363.JPEG:n02051845 +ILSVRC2012_val_00005364.JPEG:n02087394 +ILSVRC2012_val_00005365.JPEG:n02096437 +ILSVRC2012_val_00005366.JPEG:n02403003 +ILSVRC2012_val_00005367.JPEG:n02769748 +ILSVRC2012_val_00005368.JPEG:n04392985 +ILSVRC2012_val_00005369.JPEG:n02134084 +ILSVRC2012_val_00005370.JPEG:n02840245 +ILSVRC2012_val_00005371.JPEG:n04273569 +ILSVRC2012_val_00005372.JPEG:n03125729 +ILSVRC2012_val_00005373.JPEG:n03967562 +ILSVRC2012_val_00005374.JPEG:n03961711 +ILSVRC2012_val_00005375.JPEG:n03961711 +ILSVRC2012_val_00005376.JPEG:n07579787 +ILSVRC2012_val_00005377.JPEG:n04270147 +ILSVRC2012_val_00005378.JPEG:n02965783 +ILSVRC2012_val_00005379.JPEG:n02006656 +ILSVRC2012_val_00005380.JPEG:n03995372 +ILSVRC2012_val_00005381.JPEG:n03444034 +ILSVRC2012_val_00005382.JPEG:n02814860 +ILSVRC2012_val_00005383.JPEG:n04070727 +ILSVRC2012_val_00005384.JPEG:n04208210 +ILSVRC2012_val_00005385.JPEG:n04486054 +ILSVRC2012_val_00005386.JPEG:n03729826 +ILSVRC2012_val_00005387.JPEG:n02120079 +ILSVRC2012_val_00005388.JPEG:n04591713 +ILSVRC2012_val_00005389.JPEG:n02808304 +ILSVRC2012_val_00005390.JPEG:n02105641 +ILSVRC2012_val_00005391.JPEG:n03770439 +ILSVRC2012_val_00005392.JPEG:n04228054 +ILSVRC2012_val_00005393.JPEG:n02094114 +ILSVRC2012_val_00005394.JPEG:n03400231 +ILSVRC2012_val_00005395.JPEG:n02106166 +ILSVRC2012_val_00005396.JPEG:n03868863 +ILSVRC2012_val_00005397.JPEG:n02089078 +ILSVRC2012_val_00005398.JPEG:n03954731 +ILSVRC2012_val_00005399.JPEG:n04355338 +ILSVRC2012_val_00005400.JPEG:n02669723 +ILSVRC2012_val_00005401.JPEG:n04200800 +ILSVRC2012_val_00005402.JPEG:n04266014 +ILSVRC2012_val_00005403.JPEG:n03929855 +ILSVRC2012_val_00005404.JPEG:n02107312 +ILSVRC2012_val_00005405.JPEG:n04023962 +ILSVRC2012_val_00005406.JPEG:n03958227 +ILSVRC2012_val_00005407.JPEG:n01677366 +ILSVRC2012_val_00005408.JPEG:n02791124 +ILSVRC2012_val_00005409.JPEG:n03485407 +ILSVRC2012_val_00005410.JPEG:n02129165 +ILSVRC2012_val_00005411.JPEG:n03075370 +ILSVRC2012_val_00005412.JPEG:n01558993 +ILSVRC2012_val_00005413.JPEG:n02988304 +ILSVRC2012_val_00005414.JPEG:n04355933 +ILSVRC2012_val_00005415.JPEG:n02134418 +ILSVRC2012_val_00005416.JPEG:n01675722 +ILSVRC2012_val_00005417.JPEG:n07920052 +ILSVRC2012_val_00005418.JPEG:n02321529 +ILSVRC2012_val_00005419.JPEG:n02018795 +ILSVRC2012_val_00005420.JPEG:n03992509 +ILSVRC2012_val_00005421.JPEG:n03868863 +ILSVRC2012_val_00005422.JPEG:n03796401 +ILSVRC2012_val_00005423.JPEG:n02892767 +ILSVRC2012_val_00005424.JPEG:n04254120 +ILSVRC2012_val_00005425.JPEG:n03785016 +ILSVRC2012_val_00005426.JPEG:n04591157 +ILSVRC2012_val_00005427.JPEG:n01518878 +ILSVRC2012_val_00005428.JPEG:n06794110 +ILSVRC2012_val_00005429.JPEG:n01930112 +ILSVRC2012_val_00005430.JPEG:n02951585 +ILSVRC2012_val_00005431.JPEG:n07711569 +ILSVRC2012_val_00005432.JPEG:n01496331 +ILSVRC2012_val_00005433.JPEG:n02788148 +ILSVRC2012_val_00005434.JPEG:n03207743 +ILSVRC2012_val_00005435.JPEG:n03794056 +ILSVRC2012_val_00005436.JPEG:n04332243 +ILSVRC2012_val_00005437.JPEG:n04356056 +ILSVRC2012_val_00005438.JPEG:n07873807 +ILSVRC2012_val_00005439.JPEG:n02667093 +ILSVRC2012_val_00005440.JPEG:n03271574 +ILSVRC2012_val_00005441.JPEG:n02794156 +ILSVRC2012_val_00005442.JPEG:n02493793 +ILSVRC2012_val_00005443.JPEG:n03527444 +ILSVRC2012_val_00005444.JPEG:n02951585 +ILSVRC2012_val_00005445.JPEG:n03240683 +ILSVRC2012_val_00005446.JPEG:n02109961 +ILSVRC2012_val_00005447.JPEG:n01795545 +ILSVRC2012_val_00005448.JPEG:n03599486 +ILSVRC2012_val_00005449.JPEG:n04599235 +ILSVRC2012_val_00005450.JPEG:n01644900 +ILSVRC2012_val_00005451.JPEG:n07880968 +ILSVRC2012_val_00005452.JPEG:n04317175 +ILSVRC2012_val_00005453.JPEG:n02840245 +ILSVRC2012_val_00005454.JPEG:n02408429 +ILSVRC2012_val_00005455.JPEG:n07248320 +ILSVRC2012_val_00005456.JPEG:n04285008 +ILSVRC2012_val_00005457.JPEG:n02096585 +ILSVRC2012_val_00005458.JPEG:n02704792 +ILSVRC2012_val_00005459.JPEG:n04560804 +ILSVRC2012_val_00005460.JPEG:n03785016 +ILSVRC2012_val_00005461.JPEG:n02927161 +ILSVRC2012_val_00005462.JPEG:n03697007 +ILSVRC2012_val_00005463.JPEG:n07930864 +ILSVRC2012_val_00005464.JPEG:n07248320 +ILSVRC2012_val_00005465.JPEG:n02028035 +ILSVRC2012_val_00005466.JPEG:n02123597 +ILSVRC2012_val_00005467.JPEG:n02676566 +ILSVRC2012_val_00005468.JPEG:n07583066 +ILSVRC2012_val_00005469.JPEG:n02871525 +ILSVRC2012_val_00005470.JPEG:n02134084 +ILSVRC2012_val_00005471.JPEG:n02091032 +ILSVRC2012_val_00005472.JPEG:n04462240 +ILSVRC2012_val_00005473.JPEG:n02117135 +ILSVRC2012_val_00005474.JPEG:n02009912 +ILSVRC2012_val_00005475.JPEG:n09193705 +ILSVRC2012_val_00005476.JPEG:n09472597 +ILSVRC2012_val_00005477.JPEG:n02834397 +ILSVRC2012_val_00005478.JPEG:n03764736 +ILSVRC2012_val_00005479.JPEG:n01753488 +ILSVRC2012_val_00005480.JPEG:n03895866 +ILSVRC2012_val_00005481.JPEG:n02112018 +ILSVRC2012_val_00005482.JPEG:n02165105 +ILSVRC2012_val_00005483.JPEG:n02837789 +ILSVRC2012_val_00005484.JPEG:n03457902 +ILSVRC2012_val_00005485.JPEG:n04522168 +ILSVRC2012_val_00005486.JPEG:n04023962 +ILSVRC2012_val_00005487.JPEG:n04536866 +ILSVRC2012_val_00005488.JPEG:n04005630 +ILSVRC2012_val_00005489.JPEG:n02110627 +ILSVRC2012_val_00005490.JPEG:n02708093 +ILSVRC2012_val_00005491.JPEG:n04554684 +ILSVRC2012_val_00005492.JPEG:n01514668 +ILSVRC2012_val_00005493.JPEG:n02090379 +ILSVRC2012_val_00005494.JPEG:n07836838 +ILSVRC2012_val_00005495.JPEG:n02108089 +ILSVRC2012_val_00005496.JPEG:n03095699 +ILSVRC2012_val_00005497.JPEG:n04366367 +ILSVRC2012_val_00005498.JPEG:n04039381 +ILSVRC2012_val_00005499.JPEG:n07802026 +ILSVRC2012_val_00005500.JPEG:n03100240 +ILSVRC2012_val_00005501.JPEG:n03255030 +ILSVRC2012_val_00005502.JPEG:n04235860 +ILSVRC2012_val_00005503.JPEG:n02980441 +ILSVRC2012_val_00005504.JPEG:n03218198 +ILSVRC2012_val_00005505.JPEG:n01514668 +ILSVRC2012_val_00005506.JPEG:n03000684 +ILSVRC2012_val_00005507.JPEG:n02088094 +ILSVRC2012_val_00005508.JPEG:n02815834 +ILSVRC2012_val_00005509.JPEG:n03657121 +ILSVRC2012_val_00005510.JPEG:n03891251 +ILSVRC2012_val_00005511.JPEG:n02808440 +ILSVRC2012_val_00005512.JPEG:n02916936 +ILSVRC2012_val_00005513.JPEG:n03661043 +ILSVRC2012_val_00005514.JPEG:n04243546 +ILSVRC2012_val_00005515.JPEG:n04065272 +ILSVRC2012_val_00005516.JPEG:n03666591 +ILSVRC2012_val_00005517.JPEG:n04604644 +ILSVRC2012_val_00005518.JPEG:n04509417 +ILSVRC2012_val_00005519.JPEG:n03937543 +ILSVRC2012_val_00005520.JPEG:n04509417 +ILSVRC2012_val_00005521.JPEG:n02109961 +ILSVRC2012_val_00005522.JPEG:n04251144 +ILSVRC2012_val_00005523.JPEG:n02869837 +ILSVRC2012_val_00005524.JPEG:n02113712 +ILSVRC2012_val_00005525.JPEG:n02492660 +ILSVRC2012_val_00005526.JPEG:n02841315 +ILSVRC2012_val_00005527.JPEG:n07734744 +ILSVRC2012_val_00005528.JPEG:n04456115 +ILSVRC2012_val_00005529.JPEG:n02640242 +ILSVRC2012_val_00005530.JPEG:n03929855 +ILSVRC2012_val_00005531.JPEG:n04266014 +ILSVRC2012_val_00005532.JPEG:n01644900 +ILSVRC2012_val_00005533.JPEG:n02807133 +ILSVRC2012_val_00005534.JPEG:n03814639 +ILSVRC2012_val_00005535.JPEG:n01514859 +ILSVRC2012_val_00005536.JPEG:n01784675 +ILSVRC2012_val_00005537.JPEG:n04023962 +ILSVRC2012_val_00005538.JPEG:n02256656 +ILSVRC2012_val_00005539.JPEG:n01695060 +ILSVRC2012_val_00005540.JPEG:n03532672 +ILSVRC2012_val_00005541.JPEG:n04070727 +ILSVRC2012_val_00005542.JPEG:n03742115 +ILSVRC2012_val_00005543.JPEG:n03482405 +ILSVRC2012_val_00005544.JPEG:n01773797 +ILSVRC2012_val_00005545.JPEG:n03388183 +ILSVRC2012_val_00005546.JPEG:n03792782 +ILSVRC2012_val_00005547.JPEG:n09246464 +ILSVRC2012_val_00005548.JPEG:n03394916 +ILSVRC2012_val_00005549.JPEG:n13052670 +ILSVRC2012_val_00005550.JPEG:n03498962 +ILSVRC2012_val_00005551.JPEG:n02356798 +ILSVRC2012_val_00005552.JPEG:n02966193 +ILSVRC2012_val_00005553.JPEG:n01798484 +ILSVRC2012_val_00005554.JPEG:n03394916 +ILSVRC2012_val_00005555.JPEG:n04476259 +ILSVRC2012_val_00005556.JPEG:n03854065 +ILSVRC2012_val_00005557.JPEG:n03950228 +ILSVRC2012_val_00005558.JPEG:n02708093 +ILSVRC2012_val_00005559.JPEG:n02206856 +ILSVRC2012_val_00005560.JPEG:n03026506 +ILSVRC2012_val_00005561.JPEG:n04004767 +ILSVRC2012_val_00005562.JPEG:n03691459 +ILSVRC2012_val_00005563.JPEG:n01682714 +ILSVRC2012_val_00005564.JPEG:n02095570 +ILSVRC2012_val_00005565.JPEG:n02480855 +ILSVRC2012_val_00005566.JPEG:n03424325 +ILSVRC2012_val_00005567.JPEG:n01531178 +ILSVRC2012_val_00005568.JPEG:n03868863 +ILSVRC2012_val_00005569.JPEG:n02883205 +ILSVRC2012_val_00005570.JPEG:n02795169 +ILSVRC2012_val_00005571.JPEG:n04399382 +ILSVRC2012_val_00005572.JPEG:n02840245 +ILSVRC2012_val_00005573.JPEG:n02808304 +ILSVRC2012_val_00005574.JPEG:n01695060 +ILSVRC2012_val_00005575.JPEG:n02110063 +ILSVRC2012_val_00005576.JPEG:n01601694 +ILSVRC2012_val_00005577.JPEG:n04229816 +ILSVRC2012_val_00005578.JPEG:n02927161 +ILSVRC2012_val_00005579.JPEG:n03187595 +ILSVRC2012_val_00005580.JPEG:n02454379 +ILSVRC2012_val_00005581.JPEG:n04483307 +ILSVRC2012_val_00005582.JPEG:n01986214 +ILSVRC2012_val_00005583.JPEG:n02104029 +ILSVRC2012_val_00005584.JPEG:n04485082 +ILSVRC2012_val_00005585.JPEG:n02808304 +ILSVRC2012_val_00005586.JPEG:n03384352 +ILSVRC2012_val_00005587.JPEG:n02107574 +ILSVRC2012_val_00005588.JPEG:n02927161 +ILSVRC2012_val_00005589.JPEG:n03924679 +ILSVRC2012_val_00005590.JPEG:n01685808 +ILSVRC2012_val_00005591.JPEG:n02364673 +ILSVRC2012_val_00005592.JPEG:n04389033 +ILSVRC2012_val_00005593.JPEG:n07718472 +ILSVRC2012_val_00005594.JPEG:n01558993 +ILSVRC2012_val_00005595.JPEG:n03047690 +ILSVRC2012_val_00005596.JPEG:n03595614 +ILSVRC2012_val_00005597.JPEG:n02071294 +ILSVRC2012_val_00005598.JPEG:n03028079 +ILSVRC2012_val_00005599.JPEG:n01806143 +ILSVRC2012_val_00005600.JPEG:n03814639 +ILSVRC2012_val_00005601.JPEG:n02007558 +ILSVRC2012_val_00005602.JPEG:n04525038 +ILSVRC2012_val_00005603.JPEG:n02128385 +ILSVRC2012_val_00005604.JPEG:n02391049 +ILSVRC2012_val_00005605.JPEG:n04372370 +ILSVRC2012_val_00005606.JPEG:n03769881 +ILSVRC2012_val_00005607.JPEG:n02100877 +ILSVRC2012_val_00005608.JPEG:n09288635 +ILSVRC2012_val_00005609.JPEG:n03950228 +ILSVRC2012_val_00005610.JPEG:n02786058 +ILSVRC2012_val_00005611.JPEG:n03788365 +ILSVRC2012_val_00005612.JPEG:n01667114 +ILSVRC2012_val_00005613.JPEG:n02119789 +ILSVRC2012_val_00005614.JPEG:n02279972 +ILSVRC2012_val_00005615.JPEG:n02033041 +ILSVRC2012_val_00005616.JPEG:n02086910 +ILSVRC2012_val_00005617.JPEG:n01749939 +ILSVRC2012_val_00005618.JPEG:n03337140 +ILSVRC2012_val_00005619.JPEG:n07693725 +ILSVRC2012_val_00005620.JPEG:n02492660 +ILSVRC2012_val_00005621.JPEG:n02442845 +ILSVRC2012_val_00005622.JPEG:n02917067 +ILSVRC2012_val_00005623.JPEG:n03733281 +ILSVRC2012_val_00005624.JPEG:n07920052 +ILSVRC2012_val_00005625.JPEG:n02490219 +ILSVRC2012_val_00005626.JPEG:n02111277 +ILSVRC2012_val_00005627.JPEG:n02123394 +ILSVRC2012_val_00005628.JPEG:n02128757 +ILSVRC2012_val_00005629.JPEG:n02992211 +ILSVRC2012_val_00005630.JPEG:n03424325 +ILSVRC2012_val_00005631.JPEG:n03942813 +ILSVRC2012_val_00005632.JPEG:n04399382 +ILSVRC2012_val_00005633.JPEG:n04417672 +ILSVRC2012_val_00005634.JPEG:n01828970 +ILSVRC2012_val_00005635.JPEG:n03854065 +ILSVRC2012_val_00005636.JPEG:n02325366 +ILSVRC2012_val_00005637.JPEG:n02492035 +ILSVRC2012_val_00005638.JPEG:n03220513 +ILSVRC2012_val_00005639.JPEG:n02087046 +ILSVRC2012_val_00005640.JPEG:n03602883 +ILSVRC2012_val_00005641.JPEG:n01983481 +ILSVRC2012_val_00005642.JPEG:n01498041 +ILSVRC2012_val_00005643.JPEG:n02834397 +ILSVRC2012_val_00005644.JPEG:n03791053 +ILSVRC2012_val_00005645.JPEG:n04604644 +ILSVRC2012_val_00005646.JPEG:n07730033 +ILSVRC2012_val_00005647.JPEG:n01675722 +ILSVRC2012_val_00005648.JPEG:n02105056 +ILSVRC2012_val_00005649.JPEG:n04039381 +ILSVRC2012_val_00005650.JPEG:n02835271 +ILSVRC2012_val_00005651.JPEG:n02787622 +ILSVRC2012_val_00005652.JPEG:n04591157 +ILSVRC2012_val_00005653.JPEG:n02484975 +ILSVRC2012_val_00005654.JPEG:n04044716 +ILSVRC2012_val_00005655.JPEG:n02977058 +ILSVRC2012_val_00005656.JPEG:n03000247 +ILSVRC2012_val_00005657.JPEG:n03602883 +ILSVRC2012_val_00005658.JPEG:n02112018 +ILSVRC2012_val_00005659.JPEG:n04584207 +ILSVRC2012_val_00005660.JPEG:n03733281 +ILSVRC2012_val_00005661.JPEG:n04209133 +ILSVRC2012_val_00005662.JPEG:n02106662 +ILSVRC2012_val_00005663.JPEG:n01740131 +ILSVRC2012_val_00005664.JPEG:n03983396 +ILSVRC2012_val_00005665.JPEG:n04141327 +ILSVRC2012_val_00005666.JPEG:n03476684 +ILSVRC2012_val_00005667.JPEG:n03337140 +ILSVRC2012_val_00005668.JPEG:n04311174 +ILSVRC2012_val_00005669.JPEG:n02510455 +ILSVRC2012_val_00005670.JPEG:n03476991 +ILSVRC2012_val_00005671.JPEG:n04456115 +ILSVRC2012_val_00005672.JPEG:n03141823 +ILSVRC2012_val_00005673.JPEG:n04009552 +ILSVRC2012_val_00005674.JPEG:n03461385 +ILSVRC2012_val_00005675.JPEG:n01797886 +ILSVRC2012_val_00005676.JPEG:n01734418 +ILSVRC2012_val_00005677.JPEG:n02108915 +ILSVRC2012_val_00005678.JPEG:n04251144 +ILSVRC2012_val_00005679.JPEG:n04192698 +ILSVRC2012_val_00005680.JPEG:n04525038 +ILSVRC2012_val_00005681.JPEG:n03995372 +ILSVRC2012_val_00005682.JPEG:n01985128 +ILSVRC2012_val_00005683.JPEG:n07930864 +ILSVRC2012_val_00005684.JPEG:n02514041 +ILSVRC2012_val_00005685.JPEG:n02098413 +ILSVRC2012_val_00005686.JPEG:n03388183 +ILSVRC2012_val_00005687.JPEG:n02095889 +ILSVRC2012_val_00005688.JPEG:n02992529 +ILSVRC2012_val_00005689.JPEG:n07920052 +ILSVRC2012_val_00005690.JPEG:n03249569 +ILSVRC2012_val_00005691.JPEG:n02667093 +ILSVRC2012_val_00005692.JPEG:n03393912 +ILSVRC2012_val_00005693.JPEG:n03743016 +ILSVRC2012_val_00005694.JPEG:n03876231 +ILSVRC2012_val_00005695.JPEG:n02138441 +ILSVRC2012_val_00005696.JPEG:n07875152 +ILSVRC2012_val_00005697.JPEG:n02099601 +ILSVRC2012_val_00005698.JPEG:n01630670 +ILSVRC2012_val_00005699.JPEG:n02099429 +ILSVRC2012_val_00005700.JPEG:n03706229 +ILSVRC2012_val_00005701.JPEG:n03992509 +ILSVRC2012_val_00005702.JPEG:n03141823 +ILSVRC2012_val_00005703.JPEG:n03109150 +ILSVRC2012_val_00005704.JPEG:n02504013 +ILSVRC2012_val_00005705.JPEG:n02992529 +ILSVRC2012_val_00005706.JPEG:n01943899 +ILSVRC2012_val_00005707.JPEG:n03796401 +ILSVRC2012_val_00005708.JPEG:n01675722 +ILSVRC2012_val_00005709.JPEG:n04141327 +ILSVRC2012_val_00005710.JPEG:n07697537 +ILSVRC2012_val_00005711.JPEG:n04141327 +ILSVRC2012_val_00005712.JPEG:n02871525 +ILSVRC2012_val_00005713.JPEG:n04254680 +ILSVRC2012_val_00005714.JPEG:n07836838 +ILSVRC2012_val_00005715.JPEG:n03133878 +ILSVRC2012_val_00005716.JPEG:n02346627 +ILSVRC2012_val_00005717.JPEG:n03649909 +ILSVRC2012_val_00005718.JPEG:n02090622 +ILSVRC2012_val_00005719.JPEG:n03124170 +ILSVRC2012_val_00005720.JPEG:n04458633 +ILSVRC2012_val_00005721.JPEG:n04525305 +ILSVRC2012_val_00005722.JPEG:n03666591 +ILSVRC2012_val_00005723.JPEG:n02699494 +ILSVRC2012_val_00005724.JPEG:n03680355 +ILSVRC2012_val_00005725.JPEG:n01692333 +ILSVRC2012_val_00005726.JPEG:n02480495 +ILSVRC2012_val_00005727.JPEG:n03109150 +ILSVRC2012_val_00005728.JPEG:n02342885 +ILSVRC2012_val_00005729.JPEG:n02776631 +ILSVRC2012_val_00005730.JPEG:n04596742 +ILSVRC2012_val_00005731.JPEG:n03018349 +ILSVRC2012_val_00005732.JPEG:n04525305 +ILSVRC2012_val_00005733.JPEG:n01824575 +ILSVRC2012_val_00005734.JPEG:n01882714 +ILSVRC2012_val_00005735.JPEG:n02115641 +ILSVRC2012_val_00005736.JPEG:n02788148 +ILSVRC2012_val_00005737.JPEG:n04335435 +ILSVRC2012_val_00005738.JPEG:n02085936 +ILSVRC2012_val_00005739.JPEG:n02782093 +ILSVRC2012_val_00005740.JPEG:n03095699 +ILSVRC2012_val_00005741.JPEG:n03127925 +ILSVRC2012_val_00005742.JPEG:n09468604 +ILSVRC2012_val_00005743.JPEG:n07717410 +ILSVRC2012_val_00005744.JPEG:n03417042 +ILSVRC2012_val_00005745.JPEG:n12998815 +ILSVRC2012_val_00005746.JPEG:n02113023 +ILSVRC2012_val_00005747.JPEG:n07742313 +ILSVRC2012_val_00005748.JPEG:n04296562 +ILSVRC2012_val_00005749.JPEG:n07714571 +ILSVRC2012_val_00005750.JPEG:n02107312 +ILSVRC2012_val_00005751.JPEG:n01806143 +ILSVRC2012_val_00005752.JPEG:n04033995 +ILSVRC2012_val_00005753.JPEG:n02025239 +ILSVRC2012_val_00005754.JPEG:n03930313 +ILSVRC2012_val_00005755.JPEG:n02641379 +ILSVRC2012_val_00005756.JPEG:n03804744 +ILSVRC2012_val_00005757.JPEG:n07745940 +ILSVRC2012_val_00005758.JPEG:n02097658 +ILSVRC2012_val_00005759.JPEG:n07930864 +ILSVRC2012_val_00005760.JPEG:n03089624 +ILSVRC2012_val_00005761.JPEG:n02492035 +ILSVRC2012_val_00005762.JPEG:n02791124 +ILSVRC2012_val_00005763.JPEG:n02172182 +ILSVRC2012_val_00005764.JPEG:n02865351 +ILSVRC2012_val_00005765.JPEG:n01739381 +ILSVRC2012_val_00005766.JPEG:n03950228 +ILSVRC2012_val_00005767.JPEG:n02099429 +ILSVRC2012_val_00005768.JPEG:n01644900 +ILSVRC2012_val_00005769.JPEG:n02788148 +ILSVRC2012_val_00005770.JPEG:n01622779 +ILSVRC2012_val_00005771.JPEG:n02027492 +ILSVRC2012_val_00005772.JPEG:n04254120 +ILSVRC2012_val_00005773.JPEG:n03929855 +ILSVRC2012_val_00005774.JPEG:n02814533 +ILSVRC2012_val_00005775.JPEG:n02226429 +ILSVRC2012_val_00005776.JPEG:n07715103 +ILSVRC2012_val_00005777.JPEG:n03840681 +ILSVRC2012_val_00005778.JPEG:n02256656 +ILSVRC2012_val_00005779.JPEG:n01833805 +ILSVRC2012_val_00005780.JPEG:n12267677 +ILSVRC2012_val_00005781.JPEG:n01687978 +ILSVRC2012_val_00005782.JPEG:n04592741 +ILSVRC2012_val_00005783.JPEG:n04592741 +ILSVRC2012_val_00005784.JPEG:n07873807 +ILSVRC2012_val_00005785.JPEG:n02110627 +ILSVRC2012_val_00005786.JPEG:n02277742 +ILSVRC2012_val_00005787.JPEG:n04266014 +ILSVRC2012_val_00005788.JPEG:n01776313 +ILSVRC2012_val_00005789.JPEG:n02794156 +ILSVRC2012_val_00005790.JPEG:n02093428 +ILSVRC2012_val_00005791.JPEG:n04311004 +ILSVRC2012_val_00005792.JPEG:n03920288 +ILSVRC2012_val_00005793.JPEG:n03047690 +ILSVRC2012_val_00005794.JPEG:n03992509 +ILSVRC2012_val_00005795.JPEG:n02112350 +ILSVRC2012_val_00005796.JPEG:n04591157 +ILSVRC2012_val_00005797.JPEG:n03017168 +ILSVRC2012_val_00005798.JPEG:n03459775 +ILSVRC2012_val_00005799.JPEG:n01667778 +ILSVRC2012_val_00005800.JPEG:n01820546 +ILSVRC2012_val_00005801.JPEG:n03485794 +ILSVRC2012_val_00005802.JPEG:n02804610 +ILSVRC2012_val_00005803.JPEG:n03602883 +ILSVRC2012_val_00005804.JPEG:n03666591 +ILSVRC2012_val_00005805.JPEG:n01872401 +ILSVRC2012_val_00005806.JPEG:n04589890 +ILSVRC2012_val_00005807.JPEG:n02730930 +ILSVRC2012_val_00005808.JPEG:n02090379 +ILSVRC2012_val_00005809.JPEG:n03670208 +ILSVRC2012_val_00005810.JPEG:n02892201 +ILSVRC2012_val_00005811.JPEG:n03372029 +ILSVRC2012_val_00005812.JPEG:n03062245 +ILSVRC2012_val_00005813.JPEG:n02486410 +ILSVRC2012_val_00005814.JPEG:n04562935 +ILSVRC2012_val_00005815.JPEG:n01697457 +ILSVRC2012_val_00005816.JPEG:n02099429 +ILSVRC2012_val_00005817.JPEG:n04111531 +ILSVRC2012_val_00005818.JPEG:n01728920 +ILSVRC2012_val_00005819.JPEG:n04153751 +ILSVRC2012_val_00005820.JPEG:n02113624 +ILSVRC2012_val_00005821.JPEG:n01770393 +ILSVRC2012_val_00005822.JPEG:n04266014 +ILSVRC2012_val_00005823.JPEG:n02017213 +ILSVRC2012_val_00005824.JPEG:n03483316 +ILSVRC2012_val_00005825.JPEG:n01742172 +ILSVRC2012_val_00005826.JPEG:n02480855 +ILSVRC2012_val_00005827.JPEG:n01739381 +ILSVRC2012_val_00005828.JPEG:n01768244 +ILSVRC2012_val_00005829.JPEG:n03908714 +ILSVRC2012_val_00005830.JPEG:n02006656 +ILSVRC2012_val_00005831.JPEG:n02089867 +ILSVRC2012_val_00005832.JPEG:n03026506 +ILSVRC2012_val_00005833.JPEG:n01558993 +ILSVRC2012_val_00005834.JPEG:n03980874 +ILSVRC2012_val_00005835.JPEG:n03775546 +ILSVRC2012_val_00005836.JPEG:n01980166 +ILSVRC2012_val_00005837.JPEG:n09399592 +ILSVRC2012_val_00005838.JPEG:n02804610 +ILSVRC2012_val_00005839.JPEG:n04336792 +ILSVRC2012_val_00005840.JPEG:n02027492 +ILSVRC2012_val_00005841.JPEG:n04251144 +ILSVRC2012_val_00005842.JPEG:n02100735 +ILSVRC2012_val_00005843.JPEG:n03788365 +ILSVRC2012_val_00005844.JPEG:n13040303 +ILSVRC2012_val_00005845.JPEG:n02328150 +ILSVRC2012_val_00005846.JPEG:n15075141 +ILSVRC2012_val_00005847.JPEG:n07802026 +ILSVRC2012_val_00005848.JPEG:n01532829 +ILSVRC2012_val_00005849.JPEG:n03594734 +ILSVRC2012_val_00005850.JPEG:n02676566 +ILSVRC2012_val_00005851.JPEG:n04404412 +ILSVRC2012_val_00005852.JPEG:n02346627 +ILSVRC2012_val_00005853.JPEG:n02843684 +ILSVRC2012_val_00005854.JPEG:n02108000 +ILSVRC2012_val_00005855.JPEG:n02871525 +ILSVRC2012_val_00005856.JPEG:n02606052 +ILSVRC2012_val_00005857.JPEG:n03982430 +ILSVRC2012_val_00005858.JPEG:n02165456 +ILSVRC2012_val_00005859.JPEG:n02823750 +ILSVRC2012_val_00005860.JPEG:n01871265 +ILSVRC2012_val_00005861.JPEG:n02730930 +ILSVRC2012_val_00005862.JPEG:n03770679 +ILSVRC2012_val_00005863.JPEG:n04505470 +ILSVRC2012_val_00005864.JPEG:n03404251 +ILSVRC2012_val_00005865.JPEG:n01883070 +ILSVRC2012_val_00005866.JPEG:n02979186 +ILSVRC2012_val_00005867.JPEG:n02093991 +ILSVRC2012_val_00005868.JPEG:n01630670 +ILSVRC2012_val_00005869.JPEG:n04120489 +ILSVRC2012_val_00005870.JPEG:n01443537 +ILSVRC2012_val_00005871.JPEG:n04371774 +ILSVRC2012_val_00005872.JPEG:n03866082 +ILSVRC2012_val_00005873.JPEG:n01833805 +ILSVRC2012_val_00005874.JPEG:n03527444 +ILSVRC2012_val_00005875.JPEG:n03998194 +ILSVRC2012_val_00005876.JPEG:n03873416 +ILSVRC2012_val_00005877.JPEG:n02930766 +ILSVRC2012_val_00005878.JPEG:n03776460 +ILSVRC2012_val_00005879.JPEG:n06596364 +ILSVRC2012_val_00005880.JPEG:n02321529 +ILSVRC2012_val_00005881.JPEG:n04392985 +ILSVRC2012_val_00005882.JPEG:n03796401 +ILSVRC2012_val_00005883.JPEG:n04483307 +ILSVRC2012_val_00005884.JPEG:n02526121 +ILSVRC2012_val_00005885.JPEG:n02396427 +ILSVRC2012_val_00005886.JPEG:n02113023 +ILSVRC2012_val_00005887.JPEG:n03443371 +ILSVRC2012_val_00005888.JPEG:n07747607 +ILSVRC2012_val_00005889.JPEG:n01980166 +ILSVRC2012_val_00005890.JPEG:n02058221 +ILSVRC2012_val_00005891.JPEG:n02167151 +ILSVRC2012_val_00005892.JPEG:n02769748 +ILSVRC2012_val_00005893.JPEG:n03127925 +ILSVRC2012_val_00005894.JPEG:n02190166 +ILSVRC2012_val_00005895.JPEG:n03272562 +ILSVRC2012_val_00005896.JPEG:n02097130 +ILSVRC2012_val_00005897.JPEG:n04560804 +ILSVRC2012_val_00005898.JPEG:n02086240 +ILSVRC2012_val_00005899.JPEG:n04326547 +ILSVRC2012_val_00005900.JPEG:n02095314 +ILSVRC2012_val_00005901.JPEG:n01843383 +ILSVRC2012_val_00005902.JPEG:n02107312 +ILSVRC2012_val_00005903.JPEG:n03954731 +ILSVRC2012_val_00005904.JPEG:n02281406 +ILSVRC2012_val_00005905.JPEG:n02105641 +ILSVRC2012_val_00005906.JPEG:n03075370 +ILSVRC2012_val_00005907.JPEG:n02883205 +ILSVRC2012_val_00005908.JPEG:n01829413 +ILSVRC2012_val_00005909.JPEG:n02099849 +ILSVRC2012_val_00005910.JPEG:n02112137 +ILSVRC2012_val_00005911.JPEG:n07684084 +ILSVRC2012_val_00005912.JPEG:n03095699 +ILSVRC2012_val_00005913.JPEG:n02408429 +ILSVRC2012_val_00005914.JPEG:n10565667 +ILSVRC2012_val_00005915.JPEG:n02641379 +ILSVRC2012_val_00005916.JPEG:n02259212 +ILSVRC2012_val_00005917.JPEG:n02128757 +ILSVRC2012_val_00005918.JPEG:n03344393 +ILSVRC2012_val_00005919.JPEG:n01665541 +ILSVRC2012_val_00005920.JPEG:n04004767 +ILSVRC2012_val_00005921.JPEG:n07734744 +ILSVRC2012_val_00005922.JPEG:n02088364 +ILSVRC2012_val_00005923.JPEG:n02100583 +ILSVRC2012_val_00005924.JPEG:n02672831 +ILSVRC2012_val_00005925.JPEG:n01820546 +ILSVRC2012_val_00005926.JPEG:n03376595 +ILSVRC2012_val_00005927.JPEG:n04070727 +ILSVRC2012_val_00005928.JPEG:n02981792 +ILSVRC2012_val_00005929.JPEG:n03709823 +ILSVRC2012_val_00005930.JPEG:n02206856 +ILSVRC2012_val_00005931.JPEG:n01537544 +ILSVRC2012_val_00005932.JPEG:n01776313 +ILSVRC2012_val_00005933.JPEG:n04579145 +ILSVRC2012_val_00005934.JPEG:n02492035 +ILSVRC2012_val_00005935.JPEG:n02804414 +ILSVRC2012_val_00005936.JPEG:n02113799 +ILSVRC2012_val_00005937.JPEG:n02104365 +ILSVRC2012_val_00005938.JPEG:n03483316 +ILSVRC2012_val_00005939.JPEG:n09256479 +ILSVRC2012_val_00005940.JPEG:n03642806 +ILSVRC2012_val_00005941.JPEG:n07590611 +ILSVRC2012_val_00005942.JPEG:n02094433 +ILSVRC2012_val_00005943.JPEG:n02089973 +ILSVRC2012_val_00005944.JPEG:n02497673 +ILSVRC2012_val_00005945.JPEG:n01968897 +ILSVRC2012_val_00005946.JPEG:n02090721 +ILSVRC2012_val_00005947.JPEG:n02167151 +ILSVRC2012_val_00005948.JPEG:n02974003 +ILSVRC2012_val_00005949.JPEG:n02514041 +ILSVRC2012_val_00005950.JPEG:n03781244 +ILSVRC2012_val_00005951.JPEG:n02408429 +ILSVRC2012_val_00005952.JPEG:n02279972 +ILSVRC2012_val_00005953.JPEG:n04311174 +ILSVRC2012_val_00005954.JPEG:n01990800 +ILSVRC2012_val_00005955.JPEG:n02804610 +ILSVRC2012_val_00005956.JPEG:n03146219 +ILSVRC2012_val_00005957.JPEG:n13040303 +ILSVRC2012_val_00005958.JPEG:n07930864 +ILSVRC2012_val_00005959.JPEG:n04423845 +ILSVRC2012_val_00005960.JPEG:n02437616 +ILSVRC2012_val_00005961.JPEG:n03388043 +ILSVRC2012_val_00005962.JPEG:n04487394 +ILSVRC2012_val_00005963.JPEG:n04201297 +ILSVRC2012_val_00005964.JPEG:n02704792 +ILSVRC2012_val_00005965.JPEG:n01729322 +ILSVRC2012_val_00005966.JPEG:n04371430 +ILSVRC2012_val_00005967.JPEG:n03937543 +ILSVRC2012_val_00005968.JPEG:n03216828 +ILSVRC2012_val_00005969.JPEG:n02486261 +ILSVRC2012_val_00005970.JPEG:n02666196 +ILSVRC2012_val_00005971.JPEG:n04612504 +ILSVRC2012_val_00005972.JPEG:n03180011 +ILSVRC2012_val_00005973.JPEG:n03240683 +ILSVRC2012_val_00005974.JPEG:n03627232 +ILSVRC2012_val_00005975.JPEG:n01877812 +ILSVRC2012_val_00005976.JPEG:n04486054 +ILSVRC2012_val_00005977.JPEG:n02782093 +ILSVRC2012_val_00005978.JPEG:n02814533 +ILSVRC2012_val_00005979.JPEG:n02119022 +ILSVRC2012_val_00005980.JPEG:n03788195 +ILSVRC2012_val_00005981.JPEG:n07720875 +ILSVRC2012_val_00005982.JPEG:n02096051 +ILSVRC2012_val_00005983.JPEG:n03903868 +ILSVRC2012_val_00005984.JPEG:n02105162 +ILSVRC2012_val_00005985.JPEG:n04125021 +ILSVRC2012_val_00005986.JPEG:n03272010 +ILSVRC2012_val_00005987.JPEG:n03794056 +ILSVRC2012_val_00005988.JPEG:n02058221 +ILSVRC2012_val_00005989.JPEG:n03457902 +ILSVRC2012_val_00005990.JPEG:n04584207 +ILSVRC2012_val_00005991.JPEG:n03785016 +ILSVRC2012_val_00005992.JPEG:n04311004 +ILSVRC2012_val_00005993.JPEG:n03837869 +ILSVRC2012_val_00005994.JPEG:n02101556 +ILSVRC2012_val_00005995.JPEG:n03840681 +ILSVRC2012_val_00005996.JPEG:n03425413 +ILSVRC2012_val_00005997.JPEG:n03496892 +ILSVRC2012_val_00005998.JPEG:n02127052 +ILSVRC2012_val_00005999.JPEG:n01980166 +ILSVRC2012_val_00006000.JPEG:n03770439 +ILSVRC2012_val_00006001.JPEG:n04398044 +ILSVRC2012_val_00006002.JPEG:n02105412 +ILSVRC2012_val_00006003.JPEG:n03032252 +ILSVRC2012_val_00006004.JPEG:n03594734 +ILSVRC2012_val_00006005.JPEG:n02096437 +ILSVRC2012_val_00006006.JPEG:n10148035 +ILSVRC2012_val_00006007.JPEG:n01443537 +ILSVRC2012_val_00006008.JPEG:n04125021 +ILSVRC2012_val_00006009.JPEG:n03649909 +ILSVRC2012_val_00006010.JPEG:n02939185 +ILSVRC2012_val_00006011.JPEG:n01737021 +ILSVRC2012_val_00006012.JPEG:n02510455 +ILSVRC2012_val_00006013.JPEG:n02398521 +ILSVRC2012_val_00006014.JPEG:n02490219 +ILSVRC2012_val_00006015.JPEG:n03595614 +ILSVRC2012_val_00006016.JPEG:n04277352 +ILSVRC2012_val_00006017.JPEG:n03649909 +ILSVRC2012_val_00006018.JPEG:n07716906 +ILSVRC2012_val_00006019.JPEG:n02808440 +ILSVRC2012_val_00006020.JPEG:n03124170 +ILSVRC2012_val_00006021.JPEG:n03538406 +ILSVRC2012_val_00006022.JPEG:n03376595 +ILSVRC2012_val_00006023.JPEG:n02860847 +ILSVRC2012_val_00006024.JPEG:n01797886 +ILSVRC2012_val_00006025.JPEG:n04243546 +ILSVRC2012_val_00006026.JPEG:n03673027 +ILSVRC2012_val_00006027.JPEG:n04462240 +ILSVRC2012_val_00006028.JPEG:n03595614 +ILSVRC2012_val_00006029.JPEG:n04579432 +ILSVRC2012_val_00006030.JPEG:n01558993 +ILSVRC2012_val_00006031.JPEG:n04081281 +ILSVRC2012_val_00006032.JPEG:n04136333 +ILSVRC2012_val_00006033.JPEG:n03223299 +ILSVRC2012_val_00006034.JPEG:n03197337 +ILSVRC2012_val_00006035.JPEG:n02094114 +ILSVRC2012_val_00006036.JPEG:n03452741 +ILSVRC2012_val_00006037.JPEG:n04392985 +ILSVRC2012_val_00006038.JPEG:n02666196 +ILSVRC2012_val_00006039.JPEG:n02786058 +ILSVRC2012_val_00006040.JPEG:n09332890 +ILSVRC2012_val_00006041.JPEG:n03759954 +ILSVRC2012_val_00006042.JPEG:n04125021 +ILSVRC2012_val_00006043.JPEG:n03000684 +ILSVRC2012_val_00006044.JPEG:n04597913 +ILSVRC2012_val_00006045.JPEG:n01768244 +ILSVRC2012_val_00006046.JPEG:n02099601 +ILSVRC2012_val_00006047.JPEG:n07716358 +ILSVRC2012_val_00006048.JPEG:n03530642 +ILSVRC2012_val_00006049.JPEG:n01860187 +ILSVRC2012_val_00006050.JPEG:n02012849 +ILSVRC2012_val_00006051.JPEG:n02814860 +ILSVRC2012_val_00006052.JPEG:n02110063 +ILSVRC2012_val_00006053.JPEG:n03160309 +ILSVRC2012_val_00006054.JPEG:n02091032 +ILSVRC2012_val_00006055.JPEG:n15075141 +ILSVRC2012_val_00006056.JPEG:n02127052 +ILSVRC2012_val_00006057.JPEG:n02699494 +ILSVRC2012_val_00006058.JPEG:n04447861 +ILSVRC2012_val_00006059.JPEG:n02109961 +ILSVRC2012_val_00006060.JPEG:n03532672 +ILSVRC2012_val_00006061.JPEG:n04099969 +ILSVRC2012_val_00006062.JPEG:n03594945 +ILSVRC2012_val_00006063.JPEG:n02101556 +ILSVRC2012_val_00006064.JPEG:n04200800 +ILSVRC2012_val_00006065.JPEG:n02100236 +ILSVRC2012_val_00006066.JPEG:n04149813 +ILSVRC2012_val_00006067.JPEG:n07920052 +ILSVRC2012_val_00006068.JPEG:n04149813 +ILSVRC2012_val_00006069.JPEG:n02097209 +ILSVRC2012_val_00006070.JPEG:n03793489 +ILSVRC2012_val_00006071.JPEG:n09428293 +ILSVRC2012_val_00006072.JPEG:n03840681 +ILSVRC2012_val_00006073.JPEG:n02799071 +ILSVRC2012_val_00006074.JPEG:n04332243 +ILSVRC2012_val_00006075.JPEG:n01807496 +ILSVRC2012_val_00006076.JPEG:n04479046 +ILSVRC2012_val_00006077.JPEG:n02101388 +ILSVRC2012_val_00006078.JPEG:n02099849 +ILSVRC2012_val_00006079.JPEG:n02085620 +ILSVRC2012_val_00006080.JPEG:n02655020 +ILSVRC2012_val_00006081.JPEG:n02802426 +ILSVRC2012_val_00006082.JPEG:n04204347 +ILSVRC2012_val_00006083.JPEG:n02094433 +ILSVRC2012_val_00006084.JPEG:n02814533 +ILSVRC2012_val_00006085.JPEG:n04398044 +ILSVRC2012_val_00006086.JPEG:n04090263 +ILSVRC2012_val_00006087.JPEG:n02051845 +ILSVRC2012_val_00006088.JPEG:n04548362 +ILSVRC2012_val_00006089.JPEG:n04259630 +ILSVRC2012_val_00006090.JPEG:n04209133 +ILSVRC2012_val_00006091.JPEG:n04596742 +ILSVRC2012_val_00006092.JPEG:n02114855 +ILSVRC2012_val_00006093.JPEG:n02091635 +ILSVRC2012_val_00006094.JPEG:n01795545 +ILSVRC2012_val_00006095.JPEG:n02231487 +ILSVRC2012_val_00006096.JPEG:n07831146 +ILSVRC2012_val_00006097.JPEG:n02110341 +ILSVRC2012_val_00006098.JPEG:n01728920 +ILSVRC2012_val_00006099.JPEG:n02802426 +ILSVRC2012_val_00006100.JPEG:n01978455 +ILSVRC2012_val_00006101.JPEG:n03388043 +ILSVRC2012_val_00006102.JPEG:n03041632 +ILSVRC2012_val_00006103.JPEG:n03976657 +ILSVRC2012_val_00006104.JPEG:n02443484 +ILSVRC2012_val_00006105.JPEG:n01735189 +ILSVRC2012_val_00006106.JPEG:n04310018 +ILSVRC2012_val_00006107.JPEG:n02009229 +ILSVRC2012_val_00006108.JPEG:n02325366 +ILSVRC2012_val_00006109.JPEG:n03075370 +ILSVRC2012_val_00006110.JPEG:n04149813 +ILSVRC2012_val_00006111.JPEG:n03891251 +ILSVRC2012_val_00006112.JPEG:n02125311 +ILSVRC2012_val_00006113.JPEG:n04074963 +ILSVRC2012_val_00006114.JPEG:n02105855 +ILSVRC2012_val_00006115.JPEG:n04525038 +ILSVRC2012_val_00006116.JPEG:n02002724 +ILSVRC2012_val_00006117.JPEG:n03924679 +ILSVRC2012_val_00006118.JPEG:n03947888 +ILSVRC2012_val_00006119.JPEG:n03544143 +ILSVRC2012_val_00006120.JPEG:n01704323 +ILSVRC2012_val_00006121.JPEG:n02177972 +ILSVRC2012_val_00006122.JPEG:n04509417 +ILSVRC2012_val_00006123.JPEG:n07754684 +ILSVRC2012_val_00006124.JPEG:n03961711 +ILSVRC2012_val_00006125.JPEG:n02364673 +ILSVRC2012_val_00006126.JPEG:n07614500 +ILSVRC2012_val_00006127.JPEG:n04239074 +ILSVRC2012_val_00006128.JPEG:n02825657 +ILSVRC2012_val_00006129.JPEG:n02391049 +ILSVRC2012_val_00006130.JPEG:n03447721 +ILSVRC2012_val_00006131.JPEG:n03042490 +ILSVRC2012_val_00006132.JPEG:n04442312 +ILSVRC2012_val_00006133.JPEG:n02098105 +ILSVRC2012_val_00006134.JPEG:n03388043 +ILSVRC2012_val_00006135.JPEG:n03692522 +ILSVRC2012_val_00006136.JPEG:n04428191 +ILSVRC2012_val_00006137.JPEG:n02100236 +ILSVRC2012_val_00006138.JPEG:n04591157 +ILSVRC2012_val_00006139.JPEG:n03729826 +ILSVRC2012_val_00006140.JPEG:n03775071 +ILSVRC2012_val_00006141.JPEG:n02480855 +ILSVRC2012_val_00006142.JPEG:n03697007 +ILSVRC2012_val_00006143.JPEG:n02088094 +ILSVRC2012_val_00006144.JPEG:n02012849 +ILSVRC2012_val_00006145.JPEG:n02119789 +ILSVRC2012_val_00006146.JPEG:n02085782 +ILSVRC2012_val_00006147.JPEG:n03424325 +ILSVRC2012_val_00006148.JPEG:n01872401 +ILSVRC2012_val_00006149.JPEG:n01631663 +ILSVRC2012_val_00006150.JPEG:n02788148 +ILSVRC2012_val_00006151.JPEG:n01698640 +ILSVRC2012_val_00006152.JPEG:n02672831 +ILSVRC2012_val_00006153.JPEG:n04162706 +ILSVRC2012_val_00006154.JPEG:n04591157 +ILSVRC2012_val_00006155.JPEG:n02128385 +ILSVRC2012_val_00006156.JPEG:n02992529 +ILSVRC2012_val_00006157.JPEG:n03443371 +ILSVRC2012_val_00006158.JPEG:n03792782 +ILSVRC2012_val_00006159.JPEG:n04200800 +ILSVRC2012_val_00006160.JPEG:n04069434 +ILSVRC2012_val_00006161.JPEG:n02490219 +ILSVRC2012_val_00006162.JPEG:n03868242 +ILSVRC2012_val_00006163.JPEG:n04277352 +ILSVRC2012_val_00006164.JPEG:n03770439 +ILSVRC2012_val_00006165.JPEG:n01773157 +ILSVRC2012_val_00006166.JPEG:n04026417 +ILSVRC2012_val_00006167.JPEG:n03492542 +ILSVRC2012_val_00006168.JPEG:n02107908 +ILSVRC2012_val_00006169.JPEG:n04548362 +ILSVRC2012_val_00006170.JPEG:n03379051 +ILSVRC2012_val_00006171.JPEG:n01582220 +ILSVRC2012_val_00006172.JPEG:n02109047 +ILSVRC2012_val_00006173.JPEG:n04579145 +ILSVRC2012_val_00006174.JPEG:n02114548 +ILSVRC2012_val_00006175.JPEG:n04152593 +ILSVRC2012_val_00006176.JPEG:n02769748 +ILSVRC2012_val_00006177.JPEG:n04296562 +ILSVRC2012_val_00006178.JPEG:n02097209 +ILSVRC2012_val_00006179.JPEG:n01983481 +ILSVRC2012_val_00006180.JPEG:n04366367 +ILSVRC2012_val_00006181.JPEG:n03657121 +ILSVRC2012_val_00006182.JPEG:n02879718 +ILSVRC2012_val_00006183.JPEG:n02119789 +ILSVRC2012_val_00006184.JPEG:n03947888 +ILSVRC2012_val_00006185.JPEG:n02342885 +ILSVRC2012_val_00006186.JPEG:n04152593 +ILSVRC2012_val_00006187.JPEG:n04370456 +ILSVRC2012_val_00006188.JPEG:n03032252 +ILSVRC2012_val_00006189.JPEG:n07880968 +ILSVRC2012_val_00006190.JPEG:n04328186 +ILSVRC2012_val_00006191.JPEG:n02107574 +ILSVRC2012_val_00006192.JPEG:n02017213 +ILSVRC2012_val_00006193.JPEG:n01945685 +ILSVRC2012_val_00006194.JPEG:n04550184 +ILSVRC2012_val_00006195.JPEG:n01514859 +ILSVRC2012_val_00006196.JPEG:n04479046 +ILSVRC2012_val_00006197.JPEG:n07695742 +ILSVRC2012_val_00006198.JPEG:n03481172 +ILSVRC2012_val_00006199.JPEG:n07747607 +ILSVRC2012_val_00006200.JPEG:n02437312 +ILSVRC2012_val_00006201.JPEG:n03742115 +ILSVRC2012_val_00006202.JPEG:n01924916 +ILSVRC2012_val_00006203.JPEG:n01608432 +ILSVRC2012_val_00006204.JPEG:n04584207 +ILSVRC2012_val_00006205.JPEG:n02825657 +ILSVRC2012_val_00006206.JPEG:n12144580 +ILSVRC2012_val_00006207.JPEG:n01689811 +ILSVRC2012_val_00006208.JPEG:n04228054 +ILSVRC2012_val_00006209.JPEG:n02113624 +ILSVRC2012_val_00006210.JPEG:n07697313 +ILSVRC2012_val_00006211.JPEG:n04367480 +ILSVRC2012_val_00006212.JPEG:n04026417 +ILSVRC2012_val_00006213.JPEG:n01616318 +ILSVRC2012_val_00006214.JPEG:n02643566 +ILSVRC2012_val_00006215.JPEG:n04228054 +ILSVRC2012_val_00006216.JPEG:n01443537 +ILSVRC2012_val_00006217.JPEG:n04252077 +ILSVRC2012_val_00006218.JPEG:n01734418 +ILSVRC2012_val_00006219.JPEG:n02490219 +ILSVRC2012_val_00006220.JPEG:n02814533 +ILSVRC2012_val_00006221.JPEG:n01796340 +ILSVRC2012_val_00006222.JPEG:n03160309 +ILSVRC2012_val_00006223.JPEG:n04355933 +ILSVRC2012_val_00006224.JPEG:n03666591 +ILSVRC2012_val_00006225.JPEG:n02443114 +ILSVRC2012_val_00006226.JPEG:n03595614 +ILSVRC2012_val_00006227.JPEG:n02948072 +ILSVRC2012_val_00006228.JPEG:n03786901 +ILSVRC2012_val_00006229.JPEG:n04380533 +ILSVRC2012_val_00006230.JPEG:n01824575 +ILSVRC2012_val_00006231.JPEG:n02018207 +ILSVRC2012_val_00006232.JPEG:n02111500 +ILSVRC2012_val_00006233.JPEG:n03188531 +ILSVRC2012_val_00006234.JPEG:n03417042 +ILSVRC2012_val_00006235.JPEG:n13037406 +ILSVRC2012_val_00006236.JPEG:n02869837 +ILSVRC2012_val_00006237.JPEG:n03627232 +ILSVRC2012_val_00006238.JPEG:n07716906 +ILSVRC2012_val_00006239.JPEG:n02130308 +ILSVRC2012_val_00006240.JPEG:n02422106 +ILSVRC2012_val_00006241.JPEG:n03544143 +ILSVRC2012_val_00006242.JPEG:n02108551 +ILSVRC2012_val_00006243.JPEG:n03314780 +ILSVRC2012_val_00006244.JPEG:n01694178 +ILSVRC2012_val_00006245.JPEG:n02437312 +ILSVRC2012_val_00006246.JPEG:n02978881 +ILSVRC2012_val_00006247.JPEG:n04243546 +ILSVRC2012_val_00006248.JPEG:n02823428 +ILSVRC2012_val_00006249.JPEG:n03916031 +ILSVRC2012_val_00006250.JPEG:n01616318 +ILSVRC2012_val_00006251.JPEG:n01496331 +ILSVRC2012_val_00006252.JPEG:n15075141 +ILSVRC2012_val_00006253.JPEG:n02071294 +ILSVRC2012_val_00006254.JPEG:n03095699 +ILSVRC2012_val_00006255.JPEG:n04525305 +ILSVRC2012_val_00006256.JPEG:n02483362 +ILSVRC2012_val_00006257.JPEG:n02109047 +ILSVRC2012_val_00006258.JPEG:n02930766 +ILSVRC2012_val_00006259.JPEG:n03792972 +ILSVRC2012_val_00006260.JPEG:n04507155 +ILSVRC2012_val_00006261.JPEG:n02091032 +ILSVRC2012_val_00006262.JPEG:n01744401 +ILSVRC2012_val_00006263.JPEG:n03929660 +ILSVRC2012_val_00006264.JPEG:n01632458 +ILSVRC2012_val_00006265.JPEG:n02090622 +ILSVRC2012_val_00006266.JPEG:n13037406 +ILSVRC2012_val_00006267.JPEG:n01580077 +ILSVRC2012_val_00006268.JPEG:n03028079 +ILSVRC2012_val_00006269.JPEG:n04366367 +ILSVRC2012_val_00006270.JPEG:n03000247 +ILSVRC2012_val_00006271.JPEG:n02088094 +ILSVRC2012_val_00006272.JPEG:n04376876 +ILSVRC2012_val_00006273.JPEG:n02110341 +ILSVRC2012_val_00006274.JPEG:n03983396 +ILSVRC2012_val_00006275.JPEG:n02791124 +ILSVRC2012_val_00006276.JPEG:n02977058 +ILSVRC2012_val_00006277.JPEG:n03384352 +ILSVRC2012_val_00006278.JPEG:n03042490 +ILSVRC2012_val_00006279.JPEG:n02643566 +ILSVRC2012_val_00006280.JPEG:n04522168 +ILSVRC2012_val_00006281.JPEG:n02804414 +ILSVRC2012_val_00006282.JPEG:n07760859 +ILSVRC2012_val_00006283.JPEG:n02445715 +ILSVRC2012_val_00006284.JPEG:n01728920 +ILSVRC2012_val_00006285.JPEG:n04285008 +ILSVRC2012_val_00006286.JPEG:n01697457 +ILSVRC2012_val_00006287.JPEG:n03961711 +ILSVRC2012_val_00006288.JPEG:n03134739 +ILSVRC2012_val_00006289.JPEG:n01882714 +ILSVRC2012_val_00006290.JPEG:n07716358 +ILSVRC2012_val_00006291.JPEG:n02364673 +ILSVRC2012_val_00006292.JPEG:n02536864 +ILSVRC2012_val_00006293.JPEG:n07880968 +ILSVRC2012_val_00006294.JPEG:n03662601 +ILSVRC2012_val_00006295.JPEG:n02699494 +ILSVRC2012_val_00006296.JPEG:n04133789 +ILSVRC2012_val_00006297.JPEG:n04141076 +ILSVRC2012_val_00006298.JPEG:n04366367 +ILSVRC2012_val_00006299.JPEG:n02892201 +ILSVRC2012_val_00006300.JPEG:n02100877 +ILSVRC2012_val_00006301.JPEG:n01695060 +ILSVRC2012_val_00006302.JPEG:n07747607 +ILSVRC2012_val_00006303.JPEG:n02971356 +ILSVRC2012_val_00006304.JPEG:n02804414 +ILSVRC2012_val_00006305.JPEG:n01665541 +ILSVRC2012_val_00006306.JPEG:n02422699 +ILSVRC2012_val_00006307.JPEG:n03065424 +ILSVRC2012_val_00006308.JPEG:n07693725 +ILSVRC2012_val_00006309.JPEG:n04336792 +ILSVRC2012_val_00006310.JPEG:n07932039 +ILSVRC2012_val_00006311.JPEG:n04311174 +ILSVRC2012_val_00006312.JPEG:n07715103 +ILSVRC2012_val_00006313.JPEG:n02268853 +ILSVRC2012_val_00006314.JPEG:n02096585 +ILSVRC2012_val_00006315.JPEG:n01981276 +ILSVRC2012_val_00006316.JPEG:n04133789 +ILSVRC2012_val_00006317.JPEG:n02814860 +ILSVRC2012_val_00006318.JPEG:n03388183 +ILSVRC2012_val_00006319.JPEG:n01631663 +ILSVRC2012_val_00006320.JPEG:n02447366 +ILSVRC2012_val_00006321.JPEG:n01560419 +ILSVRC2012_val_00006322.JPEG:n02319095 +ILSVRC2012_val_00006323.JPEG:n04370456 +ILSVRC2012_val_00006324.JPEG:n04152593 +ILSVRC2012_val_00006325.JPEG:n02939185 +ILSVRC2012_val_00006326.JPEG:n01534433 +ILSVRC2012_val_00006327.JPEG:n02909870 +ILSVRC2012_val_00006328.JPEG:n01537544 +ILSVRC2012_val_00006329.JPEG:n07565083 +ILSVRC2012_val_00006330.JPEG:n02106030 +ILSVRC2012_val_00006331.JPEG:n01630670 +ILSVRC2012_val_00006332.JPEG:n02837789 +ILSVRC2012_val_00006333.JPEG:n03633091 +ILSVRC2012_val_00006334.JPEG:n01614925 +ILSVRC2012_val_00006335.JPEG:n13052670 +ILSVRC2012_val_00006336.JPEG:n02104029 +ILSVRC2012_val_00006337.JPEG:n02877765 +ILSVRC2012_val_00006338.JPEG:n02106166 +ILSVRC2012_val_00006339.JPEG:n02011460 +ILSVRC2012_val_00006340.JPEG:n03590841 +ILSVRC2012_val_00006341.JPEG:n02130308 +ILSVRC2012_val_00006342.JPEG:n01968897 +ILSVRC2012_val_00006343.JPEG:n02397096 +ILSVRC2012_val_00006344.JPEG:n02966193 +ILSVRC2012_val_00006345.JPEG:n02129165 +ILSVRC2012_val_00006346.JPEG:n03393912 +ILSVRC2012_val_00006347.JPEG:n03133878 +ILSVRC2012_val_00006348.JPEG:n03743016 +ILSVRC2012_val_00006349.JPEG:n03947888 +ILSVRC2012_val_00006350.JPEG:n02133161 +ILSVRC2012_val_00006351.JPEG:n02102480 +ILSVRC2012_val_00006352.JPEG:n02457408 +ILSVRC2012_val_00006353.JPEG:n02111889 +ILSVRC2012_val_00006354.JPEG:n02364673 +ILSVRC2012_val_00006355.JPEG:n02980441 +ILSVRC2012_val_00006356.JPEG:n02138441 +ILSVRC2012_val_00006357.JPEG:n03908714 +ILSVRC2012_val_00006358.JPEG:n04599235 +ILSVRC2012_val_00006359.JPEG:n03220513 +ILSVRC2012_val_00006360.JPEG:n01729977 +ILSVRC2012_val_00006361.JPEG:n02808304 +ILSVRC2012_val_00006362.JPEG:n03223299 +ILSVRC2012_val_00006363.JPEG:n03444034 +ILSVRC2012_val_00006364.JPEG:n03538406 +ILSVRC2012_val_00006365.JPEG:n03384352 +ILSVRC2012_val_00006366.JPEG:n02607072 +ILSVRC2012_val_00006367.JPEG:n07684084 +ILSVRC2012_val_00006368.JPEG:n07697537 +ILSVRC2012_val_00006369.JPEG:n07565083 +ILSVRC2012_val_00006370.JPEG:n02939185 +ILSVRC2012_val_00006371.JPEG:n04483307 +ILSVRC2012_val_00006372.JPEG:n01843065 +ILSVRC2012_val_00006373.JPEG:n03272010 +ILSVRC2012_val_00006374.JPEG:n04370456 +ILSVRC2012_val_00006375.JPEG:n03627232 +ILSVRC2012_val_00006376.JPEG:n03259280 +ILSVRC2012_val_00006377.JPEG:n01698640 +ILSVRC2012_val_00006378.JPEG:n01775062 +ILSVRC2012_val_00006379.JPEG:n02769748 +ILSVRC2012_val_00006380.JPEG:n04428191 +ILSVRC2012_val_00006381.JPEG:n04326547 +ILSVRC2012_val_00006382.JPEG:n02090721 +ILSVRC2012_val_00006383.JPEG:n02051845 +ILSVRC2012_val_00006384.JPEG:n03124170 +ILSVRC2012_val_00006385.JPEG:n02422106 +ILSVRC2012_val_00006386.JPEG:n02134418 +ILSVRC2012_val_00006387.JPEG:n09399592 +ILSVRC2012_val_00006388.JPEG:n03447721 +ILSVRC2012_val_00006389.JPEG:n04090263 +ILSVRC2012_val_00006390.JPEG:n04584207 +ILSVRC2012_val_00006391.JPEG:n03884397 +ILSVRC2012_val_00006392.JPEG:n02356798 +ILSVRC2012_val_00006393.JPEG:n02105641 +ILSVRC2012_val_00006394.JPEG:n03786901 +ILSVRC2012_val_00006395.JPEG:n02835271 +ILSVRC2012_val_00006396.JPEG:n02090379 +ILSVRC2012_val_00006397.JPEG:n03379051 +ILSVRC2012_val_00006398.JPEG:n04389033 +ILSVRC2012_val_00006399.JPEG:n01847000 +ILSVRC2012_val_00006400.JPEG:n02125311 +ILSVRC2012_val_00006401.JPEG:n02089078 +ILSVRC2012_val_00006402.JPEG:n01498041 +ILSVRC2012_val_00006403.JPEG:n01749939 +ILSVRC2012_val_00006404.JPEG:n02102177 +ILSVRC2012_val_00006405.JPEG:n04023962 +ILSVRC2012_val_00006406.JPEG:n03788365 +ILSVRC2012_val_00006407.JPEG:n02127052 +ILSVRC2012_val_00006408.JPEG:n04326547 +ILSVRC2012_val_00006409.JPEG:n01641577 +ILSVRC2012_val_00006410.JPEG:n02484975 +ILSVRC2012_val_00006411.JPEG:n07768694 +ILSVRC2012_val_00006412.JPEG:n03777754 +ILSVRC2012_val_00006413.JPEG:n04487394 +ILSVRC2012_val_00006414.JPEG:n07873807 +ILSVRC2012_val_00006415.JPEG:n02089078 +ILSVRC2012_val_00006416.JPEG:n02112137 +ILSVRC2012_val_00006417.JPEG:n03733281 +ILSVRC2012_val_00006418.JPEG:n04141975 +ILSVRC2012_val_00006419.JPEG:n02105251 +ILSVRC2012_val_00006420.JPEG:n04040759 +ILSVRC2012_val_00006421.JPEG:n13052670 +ILSVRC2012_val_00006422.JPEG:n07684084 +ILSVRC2012_val_00006423.JPEG:n03179701 +ILSVRC2012_val_00006424.JPEG:n03804744 +ILSVRC2012_val_00006425.JPEG:n03127747 +ILSVRC2012_val_00006426.JPEG:n01748264 +ILSVRC2012_val_00006427.JPEG:n02408429 +ILSVRC2012_val_00006428.JPEG:n03126707 +ILSVRC2012_val_00006429.JPEG:n03595614 +ILSVRC2012_val_00006430.JPEG:n04235860 +ILSVRC2012_val_00006431.JPEG:n02117135 +ILSVRC2012_val_00006432.JPEG:n03938244 +ILSVRC2012_val_00006433.JPEG:n02497673 +ILSVRC2012_val_00006434.JPEG:n03425413 +ILSVRC2012_val_00006435.JPEG:n04192698 +ILSVRC2012_val_00006436.JPEG:n03980874 +ILSVRC2012_val_00006437.JPEG:n01774384 +ILSVRC2012_val_00006438.JPEG:n04591157 +ILSVRC2012_val_00006439.JPEG:n02403003 +ILSVRC2012_val_00006440.JPEG:n01729322 +ILSVRC2012_val_00006441.JPEG:n02834397 +ILSVRC2012_val_00006442.JPEG:n03527444 +ILSVRC2012_val_00006443.JPEG:n03763968 +ILSVRC2012_val_00006444.JPEG:n04120489 +ILSVRC2012_val_00006445.JPEG:n02100735 +ILSVRC2012_val_00006446.JPEG:n01955084 +ILSVRC2012_val_00006447.JPEG:n02483362 +ILSVRC2012_val_00006448.JPEG:n02510455 +ILSVRC2012_val_00006449.JPEG:n01817953 +ILSVRC2012_val_00006450.JPEG:n03868242 +ILSVRC2012_val_00006451.JPEG:n02483362 +ILSVRC2012_val_00006452.JPEG:n04418357 +ILSVRC2012_val_00006453.JPEG:n01968897 +ILSVRC2012_val_00006454.JPEG:n03691459 +ILSVRC2012_val_00006455.JPEG:n01882714 +ILSVRC2012_val_00006456.JPEG:n02883205 +ILSVRC2012_val_00006457.JPEG:n01829413 +ILSVRC2012_val_00006458.JPEG:n02870880 +ILSVRC2012_val_00006459.JPEG:n02396427 +ILSVRC2012_val_00006460.JPEG:n01843383 +ILSVRC2012_val_00006461.JPEG:n10148035 +ILSVRC2012_val_00006462.JPEG:n02699494 +ILSVRC2012_val_00006463.JPEG:n01580077 +ILSVRC2012_val_00006464.JPEG:n04238763 +ILSVRC2012_val_00006465.JPEG:n03496892 +ILSVRC2012_val_00006466.JPEG:n07684084 +ILSVRC2012_val_00006467.JPEG:n02950826 +ILSVRC2012_val_00006468.JPEG:n03445777 +ILSVRC2012_val_00006469.JPEG:n01798484 +ILSVRC2012_val_00006470.JPEG:n03877845 +ILSVRC2012_val_00006471.JPEG:n04239074 +ILSVRC2012_val_00006472.JPEG:n01622779 +ILSVRC2012_val_00006473.JPEG:n02099712 +ILSVRC2012_val_00006474.JPEG:n02837789 +ILSVRC2012_val_00006475.JPEG:n07730033 +ILSVRC2012_val_00006476.JPEG:n09835506 +ILSVRC2012_val_00006477.JPEG:n04532106 +ILSVRC2012_val_00006478.JPEG:n03976467 +ILSVRC2012_val_00006479.JPEG:n03854065 +ILSVRC2012_val_00006480.JPEG:n01756291 +ILSVRC2012_val_00006481.JPEG:n07892512 +ILSVRC2012_val_00006482.JPEG:n15075141 +ILSVRC2012_val_00006483.JPEG:n02971356 +ILSVRC2012_val_00006484.JPEG:n02113023 +ILSVRC2012_val_00006485.JPEG:n04023962 +ILSVRC2012_val_00006486.JPEG:n02108551 +ILSVRC2012_val_00006487.JPEG:n02002724 +ILSVRC2012_val_00006488.JPEG:n09288635 +ILSVRC2012_val_00006489.JPEG:n03457902 +ILSVRC2012_val_00006490.JPEG:n03124170 +ILSVRC2012_val_00006491.JPEG:n01484850 +ILSVRC2012_val_00006492.JPEG:n04548362 +ILSVRC2012_val_00006493.JPEG:n03201208 +ILSVRC2012_val_00006494.JPEG:n01734418 +ILSVRC2012_val_00006495.JPEG:n02090622 +ILSVRC2012_val_00006496.JPEG:n03929660 +ILSVRC2012_val_00006497.JPEG:n03868863 +ILSVRC2012_val_00006498.JPEG:n02480855 +ILSVRC2012_val_00006499.JPEG:n02028035 +ILSVRC2012_val_00006500.JPEG:n01692333 +ILSVRC2012_val_00006501.JPEG:n02206856 +ILSVRC2012_val_00006502.JPEG:n03970156 +ILSVRC2012_val_00006503.JPEG:n07768694 +ILSVRC2012_val_00006504.JPEG:n04376876 +ILSVRC2012_val_00006505.JPEG:n02089973 +ILSVRC2012_val_00006506.JPEG:n03976467 +ILSVRC2012_val_00006507.JPEG:n03134739 +ILSVRC2012_val_00006508.JPEG:n03788195 +ILSVRC2012_val_00006509.JPEG:n04399382 +ILSVRC2012_val_00006510.JPEG:n04023962 +ILSVRC2012_val_00006511.JPEG:n03393912 +ILSVRC2012_val_00006512.JPEG:n12620546 +ILSVRC2012_val_00006513.JPEG:n03085013 +ILSVRC2012_val_00006514.JPEG:n02277742 +ILSVRC2012_val_00006515.JPEG:n03272562 +ILSVRC2012_val_00006516.JPEG:n01698640 +ILSVRC2012_val_00006517.JPEG:n04039381 +ILSVRC2012_val_00006518.JPEG:n02877765 +ILSVRC2012_val_00006519.JPEG:n03680355 +ILSVRC2012_val_00006520.JPEG:n01873310 +ILSVRC2012_val_00006521.JPEG:n04039381 +ILSVRC2012_val_00006522.JPEG:n02980441 +ILSVRC2012_val_00006523.JPEG:n04376876 +ILSVRC2012_val_00006524.JPEG:n01729322 +ILSVRC2012_val_00006525.JPEG:n02795169 +ILSVRC2012_val_00006526.JPEG:n01530575 +ILSVRC2012_val_00006527.JPEG:n04515003 +ILSVRC2012_val_00006528.JPEG:n02794156 +ILSVRC2012_val_00006529.JPEG:n02165105 +ILSVRC2012_val_00006530.JPEG:n03594945 +ILSVRC2012_val_00006531.JPEG:n02093991 +ILSVRC2012_val_00006532.JPEG:n02256656 +ILSVRC2012_val_00006533.JPEG:n02105412 +ILSVRC2012_val_00006534.JPEG:n03216828 +ILSVRC2012_val_00006535.JPEG:n02110806 +ILSVRC2012_val_00006536.JPEG:n03297495 +ILSVRC2012_val_00006537.JPEG:n02112137 +ILSVRC2012_val_00006538.JPEG:n03710721 +ILSVRC2012_val_00006539.JPEG:n02110185 +ILSVRC2012_val_00006540.JPEG:n09421951 +ILSVRC2012_val_00006541.JPEG:n02480855 +ILSVRC2012_val_00006542.JPEG:n04336792 +ILSVRC2012_val_00006543.JPEG:n02510455 +ILSVRC2012_val_00006544.JPEG:n02087046 +ILSVRC2012_val_00006545.JPEG:n02110627 +ILSVRC2012_val_00006546.JPEG:n04005630 +ILSVRC2012_val_00006547.JPEG:n02536864 +ILSVRC2012_val_00006548.JPEG:n04277352 +ILSVRC2012_val_00006549.JPEG:n01774750 +ILSVRC2012_val_00006550.JPEG:n02667093 +ILSVRC2012_val_00006551.JPEG:n04554684 +ILSVRC2012_val_00006552.JPEG:n02823750 +ILSVRC2012_val_00006553.JPEG:n03196217 +ILSVRC2012_val_00006554.JPEG:n01496331 +ILSVRC2012_val_00006555.JPEG:n01855032 +ILSVRC2012_val_00006556.JPEG:n02128757 +ILSVRC2012_val_00006557.JPEG:n03764736 +ILSVRC2012_val_00006558.JPEG:n02981792 +ILSVRC2012_val_00006559.JPEG:n03876231 +ILSVRC2012_val_00006560.JPEG:n04458633 +ILSVRC2012_val_00006561.JPEG:n03888257 +ILSVRC2012_val_00006562.JPEG:n01860187 +ILSVRC2012_val_00006563.JPEG:n04326547 +ILSVRC2012_val_00006564.JPEG:n09421951 +ILSVRC2012_val_00006565.JPEG:n07880968 +ILSVRC2012_val_00006566.JPEG:n02500267 +ILSVRC2012_val_00006567.JPEG:n01770081 +ILSVRC2012_val_00006568.JPEG:n03584254 +ILSVRC2012_val_00006569.JPEG:n07711569 +ILSVRC2012_val_00006570.JPEG:n09468604 +ILSVRC2012_val_00006571.JPEG:n01614925 +ILSVRC2012_val_00006572.JPEG:n03788365 +ILSVRC2012_val_00006573.JPEG:n04560804 +ILSVRC2012_val_00006574.JPEG:n01729977 +ILSVRC2012_val_00006575.JPEG:n03717622 +ILSVRC2012_val_00006576.JPEG:n02410509 +ILSVRC2012_val_00006577.JPEG:n02437312 +ILSVRC2012_val_00006578.JPEG:n03000684 +ILSVRC2012_val_00006579.JPEG:n01632777 +ILSVRC2012_val_00006580.JPEG:n02028035 +ILSVRC2012_val_00006581.JPEG:n07873807 +ILSVRC2012_val_00006582.JPEG:n01630670 +ILSVRC2012_val_00006583.JPEG:n03388183 +ILSVRC2012_val_00006584.JPEG:n02110185 +ILSVRC2012_val_00006585.JPEG:n02098413 +ILSVRC2012_val_00006586.JPEG:n02107142 +ILSVRC2012_val_00006587.JPEG:n04209133 +ILSVRC2012_val_00006588.JPEG:n07932039 +ILSVRC2012_val_00006589.JPEG:n03992509 +ILSVRC2012_val_00006590.JPEG:n04612504 +ILSVRC2012_val_00006591.JPEG:n01986214 +ILSVRC2012_val_00006592.JPEG:n04270147 +ILSVRC2012_val_00006593.JPEG:n06874185 +ILSVRC2012_val_00006594.JPEG:n02909870 +ILSVRC2012_val_00006595.JPEG:n02168699 +ILSVRC2012_val_00006596.JPEG:n03785016 +ILSVRC2012_val_00006597.JPEG:n01532829 +ILSVRC2012_val_00006598.JPEG:n04264628 +ILSVRC2012_val_00006599.JPEG:n02484975 +ILSVRC2012_val_00006600.JPEG:n02799071 +ILSVRC2012_val_00006601.JPEG:n04209133 +ILSVRC2012_val_00006602.JPEG:n07584110 +ILSVRC2012_val_00006603.JPEG:n01560419 +ILSVRC2012_val_00006604.JPEG:n02117135 +ILSVRC2012_val_00006605.JPEG:n07684084 +ILSVRC2012_val_00006606.JPEG:n03814906 +ILSVRC2012_val_00006607.JPEG:n03908618 +ILSVRC2012_val_00006608.JPEG:n02279972 +ILSVRC2012_val_00006609.JPEG:n02098413 +ILSVRC2012_val_00006610.JPEG:n02097658 +ILSVRC2012_val_00006611.JPEG:n04154565 +ILSVRC2012_val_00006612.JPEG:n02125311 +ILSVRC2012_val_00006613.JPEG:n02018795 +ILSVRC2012_val_00006614.JPEG:n02168699 +ILSVRC2012_val_00006615.JPEG:n02096177 +ILSVRC2012_val_00006616.JPEG:n03047690 +ILSVRC2012_val_00006617.JPEG:n02747177 +ILSVRC2012_val_00006618.JPEG:n03788365 +ILSVRC2012_val_00006619.JPEG:n02128385 +ILSVRC2012_val_00006620.JPEG:n03000134 +ILSVRC2012_val_00006621.JPEG:n03775546 +ILSVRC2012_val_00006622.JPEG:n04204238 +ILSVRC2012_val_00006623.JPEG:n04604644 +ILSVRC2012_val_00006624.JPEG:n03980874 +ILSVRC2012_val_00006625.JPEG:n03598930 +ILSVRC2012_val_00006626.JPEG:n01855672 +ILSVRC2012_val_00006627.JPEG:n02090721 +ILSVRC2012_val_00006628.JPEG:n07715103 +ILSVRC2012_val_00006629.JPEG:n02443114 +ILSVRC2012_val_00006630.JPEG:n02102177 +ILSVRC2012_val_00006631.JPEG:n04258138 +ILSVRC2012_val_00006632.JPEG:n04591713 +ILSVRC2012_val_00006633.JPEG:n03297495 +ILSVRC2012_val_00006634.JPEG:n01667778 +ILSVRC2012_val_00006635.JPEG:n04350905 +ILSVRC2012_val_00006636.JPEG:n04589890 +ILSVRC2012_val_00006637.JPEG:n06794110 +ILSVRC2012_val_00006638.JPEG:n03884397 +ILSVRC2012_val_00006639.JPEG:n04367480 +ILSVRC2012_val_00006640.JPEG:n03877845 +ILSVRC2012_val_00006641.JPEG:n10148035 +ILSVRC2012_val_00006642.JPEG:n03492542 +ILSVRC2012_val_00006643.JPEG:n04116512 +ILSVRC2012_val_00006644.JPEG:n03785016 +ILSVRC2012_val_00006645.JPEG:n01968897 +ILSVRC2012_val_00006646.JPEG:n02111889 +ILSVRC2012_val_00006647.JPEG:n04579432 +ILSVRC2012_val_00006648.JPEG:n03492542 +ILSVRC2012_val_00006649.JPEG:n02111277 +ILSVRC2012_val_00006650.JPEG:n03535780 +ILSVRC2012_val_00006651.JPEG:n03786901 +ILSVRC2012_val_00006652.JPEG:n02113799 +ILSVRC2012_val_00006653.JPEG:n04347754 +ILSVRC2012_val_00006654.JPEG:n03535780 +ILSVRC2012_val_00006655.JPEG:n02963159 +ILSVRC2012_val_00006656.JPEG:n03249569 +ILSVRC2012_val_00006657.JPEG:n03617480 +ILSVRC2012_val_00006658.JPEG:n04070727 +ILSVRC2012_val_00006659.JPEG:n02108000 +ILSVRC2012_val_00006660.JPEG:n03075370 +ILSVRC2012_val_00006661.JPEG:n03355925 +ILSVRC2012_val_00006662.JPEG:n04418357 +ILSVRC2012_val_00006663.JPEG:n02783161 +ILSVRC2012_val_00006664.JPEG:n02112137 +ILSVRC2012_val_00006665.JPEG:n03179701 +ILSVRC2012_val_00006666.JPEG:n02114367 +ILSVRC2012_val_00006667.JPEG:n02098286 +ILSVRC2012_val_00006668.JPEG:n02119022 +ILSVRC2012_val_00006669.JPEG:n03000684 +ILSVRC2012_val_00006670.JPEG:n01695060 +ILSVRC2012_val_00006671.JPEG:n15075141 +ILSVRC2012_val_00006672.JPEG:n02877765 +ILSVRC2012_val_00006673.JPEG:n02107683 +ILSVRC2012_val_00006674.JPEG:n03721384 +ILSVRC2012_val_00006675.JPEG:n02107142 +ILSVRC2012_val_00006676.JPEG:n02092339 +ILSVRC2012_val_00006677.JPEG:n02687172 +ILSVRC2012_val_00006678.JPEG:n02396427 +ILSVRC2012_val_00006679.JPEG:n01629819 +ILSVRC2012_val_00006680.JPEG:n03272010 +ILSVRC2012_val_00006681.JPEG:n10148035 +ILSVRC2012_val_00006682.JPEG:n04141076 +ILSVRC2012_val_00006683.JPEG:n04044716 +ILSVRC2012_val_00006684.JPEG:n04277352 +ILSVRC2012_val_00006685.JPEG:n02364673 +ILSVRC2012_val_00006686.JPEG:n04141975 +ILSVRC2012_val_00006687.JPEG:n01819313 +ILSVRC2012_val_00006688.JPEG:n03775546 +ILSVRC2012_val_00006689.JPEG:n03379051 +ILSVRC2012_val_00006690.JPEG:n01756291 +ILSVRC2012_val_00006691.JPEG:n03785016 +ILSVRC2012_val_00006692.JPEG:n04476259 +ILSVRC2012_val_00006693.JPEG:n04612504 +ILSVRC2012_val_00006694.JPEG:n01632777 +ILSVRC2012_val_00006695.JPEG:n03838899 +ILSVRC2012_val_00006696.JPEG:n02007558 +ILSVRC2012_val_00006697.JPEG:n01440764 +ILSVRC2012_val_00006698.JPEG:n02088094 +ILSVRC2012_val_00006699.JPEG:n01735189 +ILSVRC2012_val_00006700.JPEG:n02356798 +ILSVRC2012_val_00006701.JPEG:n02095889 +ILSVRC2012_val_00006702.JPEG:n09229709 +ILSVRC2012_val_00006703.JPEG:n02132136 +ILSVRC2012_val_00006704.JPEG:n02091635 +ILSVRC2012_val_00006705.JPEG:n07754684 +ILSVRC2012_val_00006706.JPEG:n03146219 +ILSVRC2012_val_00006707.JPEG:n03467068 +ILSVRC2012_val_00006708.JPEG:n03047690 +ILSVRC2012_val_00006709.JPEG:n02408429 +ILSVRC2012_val_00006710.JPEG:n02086910 +ILSVRC2012_val_00006711.JPEG:n02012849 +ILSVRC2012_val_00006712.JPEG:n04522168 +ILSVRC2012_val_00006713.JPEG:n01943899 +ILSVRC2012_val_00006714.JPEG:n12144580 +ILSVRC2012_val_00006715.JPEG:n01820546 +ILSVRC2012_val_00006716.JPEG:n01824575 +ILSVRC2012_val_00006717.JPEG:n01677366 +ILSVRC2012_val_00006718.JPEG:n03868242 +ILSVRC2012_val_00006719.JPEG:n03814639 +ILSVRC2012_val_00006720.JPEG:n02091635 +ILSVRC2012_val_00006721.JPEG:n04033901 +ILSVRC2012_val_00006722.JPEG:n02074367 +ILSVRC2012_val_00006723.JPEG:n04597913 +ILSVRC2012_val_00006724.JPEG:n07880968 +ILSVRC2012_val_00006725.JPEG:n01871265 +ILSVRC2012_val_00006726.JPEG:n03000684 +ILSVRC2012_val_00006727.JPEG:n01983481 +ILSVRC2012_val_00006728.JPEG:n07753592 +ILSVRC2012_val_00006729.JPEG:n04235860 +ILSVRC2012_val_00006730.JPEG:n02229544 +ILSVRC2012_val_00006731.JPEG:n03814906 +ILSVRC2012_val_00006732.JPEG:n03527444 +ILSVRC2012_val_00006733.JPEG:n04532106 +ILSVRC2012_val_00006734.JPEG:n02447366 +ILSVRC2012_val_00006735.JPEG:n04179913 +ILSVRC2012_val_00006736.JPEG:n04116512 +ILSVRC2012_val_00006737.JPEG:n01631663 +ILSVRC2012_val_00006738.JPEG:n04037443 +ILSVRC2012_val_00006739.JPEG:n03947888 +ILSVRC2012_val_00006740.JPEG:n02708093 +ILSVRC2012_val_00006741.JPEG:n03874293 +ILSVRC2012_val_00006742.JPEG:n04612504 +ILSVRC2012_val_00006743.JPEG:n04589890 +ILSVRC2012_val_00006744.JPEG:n02097130 +ILSVRC2012_val_00006745.JPEG:n03089624 +ILSVRC2012_val_00006746.JPEG:n03670208 +ILSVRC2012_val_00006747.JPEG:n04579145 +ILSVRC2012_val_00006748.JPEG:n03344393 +ILSVRC2012_val_00006749.JPEG:n07614500 +ILSVRC2012_val_00006750.JPEG:n04462240 +ILSVRC2012_val_00006751.JPEG:n01751748 +ILSVRC2012_val_00006752.JPEG:n04201297 +ILSVRC2012_val_00006753.JPEG:n07802026 +ILSVRC2012_val_00006754.JPEG:n02795169 +ILSVRC2012_val_00006755.JPEG:n07613480 +ILSVRC2012_val_00006756.JPEG:n07747607 +ILSVRC2012_val_00006757.JPEG:n02115913 +ILSVRC2012_val_00006758.JPEG:n02493793 +ILSVRC2012_val_00006759.JPEG:n03770679 +ILSVRC2012_val_00006760.JPEG:n02268443 +ILSVRC2012_val_00006761.JPEG:n02009912 +ILSVRC2012_val_00006762.JPEG:n04423845 +ILSVRC2012_val_00006763.JPEG:n01530575 +ILSVRC2012_val_00006764.JPEG:n01685808 +ILSVRC2012_val_00006765.JPEG:n07715103 +ILSVRC2012_val_00006766.JPEG:n03016953 +ILSVRC2012_val_00006767.JPEG:n03355925 +ILSVRC2012_val_00006768.JPEG:n04554684 +ILSVRC2012_val_00006769.JPEG:n04366367 +ILSVRC2012_val_00006770.JPEG:n03207941 +ILSVRC2012_val_00006771.JPEG:n03887697 +ILSVRC2012_val_00006772.JPEG:n04336792 +ILSVRC2012_val_00006773.JPEG:n03759954 +ILSVRC2012_val_00006774.JPEG:n03595614 +ILSVRC2012_val_00006775.JPEG:n02480855 +ILSVRC2012_val_00006776.JPEG:n04525038 +ILSVRC2012_val_00006777.JPEG:n04355338 +ILSVRC2012_val_00006778.JPEG:n02129165 +ILSVRC2012_val_00006779.JPEG:n03255030 +ILSVRC2012_val_00006780.JPEG:n02843684 +ILSVRC2012_val_00006781.JPEG:n04493381 +ILSVRC2012_val_00006782.JPEG:n02992211 +ILSVRC2012_val_00006783.JPEG:n03814906 +ILSVRC2012_val_00006784.JPEG:n04239074 +ILSVRC2012_val_00006785.JPEG:n06794110 +ILSVRC2012_val_00006786.JPEG:n03977966 +ILSVRC2012_val_00006787.JPEG:n02979186 +ILSVRC2012_val_00006788.JPEG:n03207941 +ILSVRC2012_val_00006789.JPEG:n07875152 +ILSVRC2012_val_00006790.JPEG:n01798484 +ILSVRC2012_val_00006791.JPEG:n02484975 +ILSVRC2012_val_00006792.JPEG:n02127052 +ILSVRC2012_val_00006793.JPEG:n02133161 +ILSVRC2012_val_00006794.JPEG:n03929660 +ILSVRC2012_val_00006795.JPEG:n02966687 +ILSVRC2012_val_00006796.JPEG:n12985857 +ILSVRC2012_val_00006797.JPEG:n01873310 +ILSVRC2012_val_00006798.JPEG:n07584110 +ILSVRC2012_val_00006799.JPEG:n02088094 +ILSVRC2012_val_00006800.JPEG:n01748264 +ILSVRC2012_val_00006801.JPEG:n02101006 +ILSVRC2012_val_00006802.JPEG:n03450230 +ILSVRC2012_val_00006803.JPEG:n03657121 +ILSVRC2012_val_00006804.JPEG:n03991062 +ILSVRC2012_val_00006805.JPEG:n02013706 +ILSVRC2012_val_00006806.JPEG:n03742115 +ILSVRC2012_val_00006807.JPEG:n03595614 +ILSVRC2012_val_00006808.JPEG:n04591713 +ILSVRC2012_val_00006809.JPEG:n03891251 +ILSVRC2012_val_00006810.JPEG:n01943899 +ILSVRC2012_val_00006811.JPEG:n03065424 +ILSVRC2012_val_00006812.JPEG:n04127249 +ILSVRC2012_val_00006813.JPEG:n03584829 +ILSVRC2012_val_00006814.JPEG:n02018207 +ILSVRC2012_val_00006815.JPEG:n02089973 +ILSVRC2012_val_00006816.JPEG:n03773504 +ILSVRC2012_val_00006817.JPEG:n01751748 +ILSVRC2012_val_00006818.JPEG:n02119022 +ILSVRC2012_val_00006819.JPEG:n02276258 +ILSVRC2012_val_00006820.JPEG:n04086273 +ILSVRC2012_val_00006821.JPEG:n01877812 +ILSVRC2012_val_00006822.JPEG:n02917067 +ILSVRC2012_val_00006823.JPEG:n02168699 +ILSVRC2012_val_00006824.JPEG:n02107574 +ILSVRC2012_val_00006825.JPEG:n03954731 +ILSVRC2012_val_00006826.JPEG:n02443114 +ILSVRC2012_val_00006827.JPEG:n02101556 +ILSVRC2012_val_00006828.JPEG:n01943899 +ILSVRC2012_val_00006829.JPEG:n03457902 +ILSVRC2012_val_00006830.JPEG:n01644900 +ILSVRC2012_val_00006831.JPEG:n01770081 +ILSVRC2012_val_00006832.JPEG:n03495258 +ILSVRC2012_val_00006833.JPEG:n02606052 +ILSVRC2012_val_00006834.JPEG:n02109047 +ILSVRC2012_val_00006835.JPEG:n01532829 +ILSVRC2012_val_00006836.JPEG:n02099429 +ILSVRC2012_val_00006837.JPEG:n02100735 +ILSVRC2012_val_00006838.JPEG:n03216828 +ILSVRC2012_val_00006839.JPEG:n04204347 +ILSVRC2012_val_00006840.JPEG:n02095889 +ILSVRC2012_val_00006841.JPEG:n03794056 +ILSVRC2012_val_00006842.JPEG:n02104365 +ILSVRC2012_val_00006843.JPEG:n03595614 +ILSVRC2012_val_00006844.JPEG:n01630670 +ILSVRC2012_val_00006845.JPEG:n03223299 +ILSVRC2012_val_00006846.JPEG:n04389033 +ILSVRC2012_val_00006847.JPEG:n01796340 +ILSVRC2012_val_00006848.JPEG:n02098286 +ILSVRC2012_val_00006849.JPEG:n02109525 +ILSVRC2012_val_00006850.JPEG:n04509417 +ILSVRC2012_val_00006851.JPEG:n01580077 +ILSVRC2012_val_00006852.JPEG:n04209239 +ILSVRC2012_val_00006853.JPEG:n01675722 +ILSVRC2012_val_00006854.JPEG:n07718747 +ILSVRC2012_val_00006855.JPEG:n02787622 +ILSVRC2012_val_00006856.JPEG:n04553703 +ILSVRC2012_val_00006857.JPEG:n02100877 +ILSVRC2012_val_00006858.JPEG:n02708093 +ILSVRC2012_val_00006859.JPEG:n01687978 +ILSVRC2012_val_00006860.JPEG:n01944390 +ILSVRC2012_val_00006861.JPEG:n02807133 +ILSVRC2012_val_00006862.JPEG:n03908714 +ILSVRC2012_val_00006863.JPEG:n12620546 +ILSVRC2012_val_00006864.JPEG:n04009552 +ILSVRC2012_val_00006865.JPEG:n04591713 +ILSVRC2012_val_00006866.JPEG:n02112350 +ILSVRC2012_val_00006867.JPEG:n02168699 +ILSVRC2012_val_00006868.JPEG:n03773504 +ILSVRC2012_val_00006869.JPEG:n03127747 +ILSVRC2012_val_00006870.JPEG:n03393912 +ILSVRC2012_val_00006871.JPEG:n03617480 +ILSVRC2012_val_00006872.JPEG:n02704792 +ILSVRC2012_val_00006873.JPEG:n03590841 +ILSVRC2012_val_00006874.JPEG:n03445924 +ILSVRC2012_val_00006875.JPEG:n02486261 +ILSVRC2012_val_00006876.JPEG:n03803284 +ILSVRC2012_val_00006877.JPEG:n03954731 +ILSVRC2012_val_00006878.JPEG:n02971356 +ILSVRC2012_val_00006879.JPEG:n03000247 +ILSVRC2012_val_00006880.JPEG:n03887697 +ILSVRC2012_val_00006881.JPEG:n02894605 +ILSVRC2012_val_00006882.JPEG:n04286575 +ILSVRC2012_val_00006883.JPEG:n02172182 +ILSVRC2012_val_00006884.JPEG:n01873310 +ILSVRC2012_val_00006885.JPEG:n04118538 +ILSVRC2012_val_00006886.JPEG:n04357314 +ILSVRC2012_val_00006887.JPEG:n02113624 +ILSVRC2012_val_00006888.JPEG:n02667093 +ILSVRC2012_val_00006889.JPEG:n03141823 +ILSVRC2012_val_00006890.JPEG:n04423845 +ILSVRC2012_val_00006891.JPEG:n03742115 +ILSVRC2012_val_00006892.JPEG:n02085620 +ILSVRC2012_val_00006893.JPEG:n02727426 +ILSVRC2012_val_00006894.JPEG:n04606251 +ILSVRC2012_val_00006895.JPEG:n02088466 +ILSVRC2012_val_00006896.JPEG:n03109150 +ILSVRC2012_val_00006897.JPEG:n03134739 +ILSVRC2012_val_00006898.JPEG:n02361337 +ILSVRC2012_val_00006899.JPEG:n03832673 +ILSVRC2012_val_00006900.JPEG:n02087394 +ILSVRC2012_val_00006901.JPEG:n02177972 +ILSVRC2012_val_00006902.JPEG:n04347754 +ILSVRC2012_val_00006903.JPEG:n07718747 +ILSVRC2012_val_00006904.JPEG:n03710721 +ILSVRC2012_val_00006905.JPEG:n03970156 +ILSVRC2012_val_00006906.JPEG:n04229816 +ILSVRC2012_val_00006907.JPEG:n01601694 +ILSVRC2012_val_00006908.JPEG:n02606052 +ILSVRC2012_val_00006909.JPEG:n03425413 +ILSVRC2012_val_00006910.JPEG:n03447447 +ILSVRC2012_val_00006911.JPEG:n04336792 +ILSVRC2012_val_00006912.JPEG:n04486054 +ILSVRC2012_val_00006913.JPEG:n04201297 +ILSVRC2012_val_00006914.JPEG:n07614500 +ILSVRC2012_val_00006915.JPEG:n02226429 +ILSVRC2012_val_00006916.JPEG:n01622779 +ILSVRC2012_val_00006917.JPEG:n04435653 +ILSVRC2012_val_00006918.JPEG:n09288635 +ILSVRC2012_val_00006919.JPEG:n02790996 +ILSVRC2012_val_00006920.JPEG:n02108000 +ILSVRC2012_val_00006921.JPEG:n03961711 +ILSVRC2012_val_00006922.JPEG:n03417042 +ILSVRC2012_val_00006923.JPEG:n03017168 +ILSVRC2012_val_00006924.JPEG:n03840681 +ILSVRC2012_val_00006925.JPEG:n02509815 +ILSVRC2012_val_00006926.JPEG:n04019541 +ILSVRC2012_val_00006927.JPEG:n01692333 +ILSVRC2012_val_00006928.JPEG:n01843065 +ILSVRC2012_val_00006929.JPEG:n03461385 +ILSVRC2012_val_00006930.JPEG:n04296562 +ILSVRC2012_val_00006931.JPEG:n02493509 +ILSVRC2012_val_00006932.JPEG:n03133878 +ILSVRC2012_val_00006933.JPEG:n02110627 +ILSVRC2012_val_00006934.JPEG:n07932039 +ILSVRC2012_val_00006935.JPEG:n02091831 +ILSVRC2012_val_00006936.JPEG:n03249569 +ILSVRC2012_val_00006937.JPEG:n02091467 +ILSVRC2012_val_00006938.JPEG:n03680355 +ILSVRC2012_val_00006939.JPEG:n07714990 +ILSVRC2012_val_00006940.JPEG:n02412080 +ILSVRC2012_val_00006941.JPEG:n03250847 +ILSVRC2012_val_00006942.JPEG:n03447721 +ILSVRC2012_val_00006943.JPEG:n02916936 +ILSVRC2012_val_00006944.JPEG:n02107683 +ILSVRC2012_val_00006945.JPEG:n02492035 +ILSVRC2012_val_00006946.JPEG:n03404251 +ILSVRC2012_val_00006947.JPEG:n02102177 +ILSVRC2012_val_00006948.JPEG:n07932039 +ILSVRC2012_val_00006949.JPEG:n04557648 +ILSVRC2012_val_00006950.JPEG:n04372370 +ILSVRC2012_val_00006951.JPEG:n03891251 +ILSVRC2012_val_00006952.JPEG:n02974003 +ILSVRC2012_val_00006953.JPEG:n15075141 +ILSVRC2012_val_00006954.JPEG:n02444819 +ILSVRC2012_val_00006955.JPEG:n04462240 +ILSVRC2012_val_00006956.JPEG:n02100236 +ILSVRC2012_val_00006957.JPEG:n02108551 +ILSVRC2012_val_00006958.JPEG:n04515003 +ILSVRC2012_val_00006959.JPEG:n02002556 +ILSVRC2012_val_00006960.JPEG:n02794156 +ILSVRC2012_val_00006961.JPEG:n04204238 +ILSVRC2012_val_00006962.JPEG:n04090263 +ILSVRC2012_val_00006963.JPEG:n04584207 +ILSVRC2012_val_00006964.JPEG:n02120505 +ILSVRC2012_val_00006965.JPEG:n03773504 +ILSVRC2012_val_00006966.JPEG:n02165456 +ILSVRC2012_val_00006967.JPEG:n07684084 +ILSVRC2012_val_00006968.JPEG:n04311174 +ILSVRC2012_val_00006969.JPEG:n02002556 +ILSVRC2012_val_00006970.JPEG:n02106382 +ILSVRC2012_val_00006971.JPEG:n01695060 +ILSVRC2012_val_00006972.JPEG:n02783161 +ILSVRC2012_val_00006973.JPEG:n02422699 +ILSVRC2012_val_00006974.JPEG:n03982430 +ILSVRC2012_val_00006975.JPEG:n02397096 +ILSVRC2012_val_00006976.JPEG:n03976657 +ILSVRC2012_val_00006977.JPEG:n02692877 +ILSVRC2012_val_00006978.JPEG:n03841143 +ILSVRC2012_val_00006979.JPEG:n03710637 +ILSVRC2012_val_00006980.JPEG:n04259630 +ILSVRC2012_val_00006981.JPEG:n02099601 +ILSVRC2012_val_00006982.JPEG:n03942813 +ILSVRC2012_val_00006983.JPEG:n12998815 +ILSVRC2012_val_00006984.JPEG:n11939491 +ILSVRC2012_val_00006985.JPEG:n04399382 +ILSVRC2012_val_00006986.JPEG:n03065424 +ILSVRC2012_val_00006987.JPEG:n01644373 +ILSVRC2012_val_00006988.JPEG:n04462240 +ILSVRC2012_val_00006989.JPEG:n03992509 +ILSVRC2012_val_00006990.JPEG:n03534580 +ILSVRC2012_val_00006991.JPEG:n02398521 +ILSVRC2012_val_00006992.JPEG:n02095889 +ILSVRC2012_val_00006993.JPEG:n02808440 +ILSVRC2012_val_00006994.JPEG:n04264628 +ILSVRC2012_val_00006995.JPEG:n02786058 +ILSVRC2012_val_00006996.JPEG:n04399382 +ILSVRC2012_val_00006997.JPEG:n03933933 +ILSVRC2012_val_00006998.JPEG:n04487081 +ILSVRC2012_val_00006999.JPEG:n01873310 +ILSVRC2012_val_00007000.JPEG:n04409515 +ILSVRC2012_val_00007001.JPEG:n02108089 +ILSVRC2012_val_00007002.JPEG:n02091831 +ILSVRC2012_val_00007003.JPEG:n07734744 +ILSVRC2012_val_00007004.JPEG:n04552348 +ILSVRC2012_val_00007005.JPEG:n04162706 +ILSVRC2012_val_00007006.JPEG:n02123045 +ILSVRC2012_val_00007007.JPEG:n13040303 +ILSVRC2012_val_00007008.JPEG:n02492035 +ILSVRC2012_val_00007009.JPEG:n03657121 +ILSVRC2012_val_00007010.JPEG:n02488291 +ILSVRC2012_val_00007011.JPEG:n02027492 +ILSVRC2012_val_00007012.JPEG:n02769748 +ILSVRC2012_val_00007013.JPEG:n07753113 +ILSVRC2012_val_00007014.JPEG:n03814639 +ILSVRC2012_val_00007015.JPEG:n01704323 +ILSVRC2012_val_00007016.JPEG:n02276258 +ILSVRC2012_val_00007017.JPEG:n04557648 +ILSVRC2012_val_00007018.JPEG:n03478589 +ILSVRC2012_val_00007019.JPEG:n04435653 +ILSVRC2012_val_00007020.JPEG:n03535780 +ILSVRC2012_val_00007021.JPEG:n04371774 +ILSVRC2012_val_00007022.JPEG:n02823750 +ILSVRC2012_val_00007023.JPEG:n02124075 +ILSVRC2012_val_00007024.JPEG:n07695742 +ILSVRC2012_val_00007025.JPEG:n03337140 +ILSVRC2012_val_00007026.JPEG:n03884397 +ILSVRC2012_val_00007027.JPEG:n01917289 +ILSVRC2012_val_00007028.JPEG:n07720875 +ILSVRC2012_val_00007029.JPEG:n07742313 +ILSVRC2012_val_00007030.JPEG:n04019541 +ILSVRC2012_val_00007031.JPEG:n02130308 +ILSVRC2012_val_00007032.JPEG:n02102040 +ILSVRC2012_val_00007033.JPEG:n02104365 +ILSVRC2012_val_00007034.JPEG:n02963159 +ILSVRC2012_val_00007035.JPEG:n01687978 +ILSVRC2012_val_00007036.JPEG:n07754684 +ILSVRC2012_val_00007037.JPEG:n02328150 +ILSVRC2012_val_00007038.JPEG:n02791124 +ILSVRC2012_val_00007039.JPEG:n04286575 +ILSVRC2012_val_00007040.JPEG:n04606251 +ILSVRC2012_val_00007041.JPEG:n03814639 +ILSVRC2012_val_00007042.JPEG:n09246464 +ILSVRC2012_val_00007043.JPEG:n02009229 +ILSVRC2012_val_00007044.JPEG:n01665541 +ILSVRC2012_val_00007045.JPEG:n04399382 +ILSVRC2012_val_00007046.JPEG:n04429376 +ILSVRC2012_val_00007047.JPEG:n04033995 +ILSVRC2012_val_00007048.JPEG:n04238763 +ILSVRC2012_val_00007049.JPEG:n09256479 +ILSVRC2012_val_00007050.JPEG:n01632458 +ILSVRC2012_val_00007051.JPEG:n04004767 +ILSVRC2012_val_00007052.JPEG:n04111531 +ILSVRC2012_val_00007053.JPEG:n03710637 +ILSVRC2012_val_00007054.JPEG:n02107908 +ILSVRC2012_val_00007055.JPEG:n04008634 +ILSVRC2012_val_00007056.JPEG:n02106382 +ILSVRC2012_val_00007057.JPEG:n02086079 +ILSVRC2012_val_00007058.JPEG:n07871810 +ILSVRC2012_val_00007059.JPEG:n02105505 +ILSVRC2012_val_00007060.JPEG:n02013706 +ILSVRC2012_val_00007061.JPEG:n03733131 +ILSVRC2012_val_00007062.JPEG:n07875152 +ILSVRC2012_val_00007063.JPEG:n03376595 +ILSVRC2012_val_00007064.JPEG:n03594945 +ILSVRC2012_val_00007065.JPEG:n01776313 +ILSVRC2012_val_00007066.JPEG:n03016953 +ILSVRC2012_val_00007067.JPEG:n04243546 +ILSVRC2012_val_00007068.JPEG:n04252225 +ILSVRC2012_val_00007069.JPEG:n03709823 +ILSVRC2012_val_00007070.JPEG:n02939185 +ILSVRC2012_val_00007071.JPEG:n02107574 +ILSVRC2012_val_00007072.JPEG:n02097047 +ILSVRC2012_val_00007073.JPEG:n02109525 +ILSVRC2012_val_00007074.JPEG:n03916031 +ILSVRC2012_val_00007075.JPEG:n02116738 +ILSVRC2012_val_00007076.JPEG:n07579787 +ILSVRC2012_val_00007077.JPEG:n02018795 +ILSVRC2012_val_00007078.JPEG:n03967562 +ILSVRC2012_val_00007079.JPEG:n03075370 +ILSVRC2012_val_00007080.JPEG:n12998815 +ILSVRC2012_val_00007081.JPEG:n01818515 +ILSVRC2012_val_00007082.JPEG:n02190166 +ILSVRC2012_val_00007083.JPEG:n02701002 +ILSVRC2012_val_00007084.JPEG:n01685808 +ILSVRC2012_val_00007085.JPEG:n12267677 +ILSVRC2012_val_00007086.JPEG:n02107683 +ILSVRC2012_val_00007087.JPEG:n07695742 +ILSVRC2012_val_00007088.JPEG:n02085782 +ILSVRC2012_val_00007089.JPEG:n03692522 +ILSVRC2012_val_00007090.JPEG:n02086646 +ILSVRC2012_val_00007091.JPEG:n03623198 +ILSVRC2012_val_00007092.JPEG:n03534580 +ILSVRC2012_val_00007093.JPEG:n02133161 +ILSVRC2012_val_00007094.JPEG:n07584110 +ILSVRC2012_val_00007095.JPEG:n03980874 +ILSVRC2012_val_00007096.JPEG:n03710721 +ILSVRC2012_val_00007097.JPEG:n03838899 +ILSVRC2012_val_00007098.JPEG:n04311174 +ILSVRC2012_val_00007099.JPEG:n03976467 +ILSVRC2012_val_00007100.JPEG:n02966687 +ILSVRC2012_val_00007101.JPEG:n03785016 +ILSVRC2012_val_00007102.JPEG:n02097658 +ILSVRC2012_val_00007103.JPEG:n04442312 +ILSVRC2012_val_00007104.JPEG:n04380533 +ILSVRC2012_val_00007105.JPEG:n03042490 +ILSVRC2012_val_00007106.JPEG:n03982430 +ILSVRC2012_val_00007107.JPEG:n02510455 +ILSVRC2012_val_00007108.JPEG:n02408429 +ILSVRC2012_val_00007109.JPEG:n02093859 +ILSVRC2012_val_00007110.JPEG:n07718472 +ILSVRC2012_val_00007111.JPEG:n02086079 +ILSVRC2012_val_00007112.JPEG:n02834397 +ILSVRC2012_val_00007113.JPEG:n03670208 +ILSVRC2012_val_00007114.JPEG:n01728572 +ILSVRC2012_val_00007115.JPEG:n02444819 +ILSVRC2012_val_00007116.JPEG:n02091467 +ILSVRC2012_val_00007117.JPEG:n04325704 +ILSVRC2012_val_00007118.JPEG:n04332243 +ILSVRC2012_val_00007119.JPEG:n03223299 +ILSVRC2012_val_00007120.JPEG:n01734418 +ILSVRC2012_val_00007121.JPEG:n03496892 +ILSVRC2012_val_00007122.JPEG:n01697457 +ILSVRC2012_val_00007123.JPEG:n03884397 +ILSVRC2012_val_00007124.JPEG:n03483316 +ILSVRC2012_val_00007125.JPEG:n04285008 +ILSVRC2012_val_00007126.JPEG:n01795545 +ILSVRC2012_val_00007127.JPEG:n03220513 +ILSVRC2012_val_00007128.JPEG:n02007558 +ILSVRC2012_val_00007129.JPEG:n01532829 +ILSVRC2012_val_00007130.JPEG:n02236044 +ILSVRC2012_val_00007131.JPEG:n06596364 +ILSVRC2012_val_00007132.JPEG:n04111531 +ILSVRC2012_val_00007133.JPEG:n03032252 +ILSVRC2012_val_00007134.JPEG:n03814639 +ILSVRC2012_val_00007135.JPEG:n04317175 +ILSVRC2012_val_00007136.JPEG:n04033995 +ILSVRC2012_val_00007137.JPEG:n02086079 +ILSVRC2012_val_00007138.JPEG:n07684084 +ILSVRC2012_val_00007139.JPEG:n01829413 +ILSVRC2012_val_00007140.JPEG:n02128757 +ILSVRC2012_val_00007141.JPEG:n03983396 +ILSVRC2012_val_00007142.JPEG:n04487081 +ILSVRC2012_val_00007143.JPEG:n02190166 +ILSVRC2012_val_00007144.JPEG:n04523525 +ILSVRC2012_val_00007145.JPEG:n04328186 +ILSVRC2012_val_00007146.JPEG:n04116512 +ILSVRC2012_val_00007147.JPEG:n03450230 +ILSVRC2012_val_00007148.JPEG:n04228054 +ILSVRC2012_val_00007149.JPEG:n02102177 +ILSVRC2012_val_00007150.JPEG:n03873416 +ILSVRC2012_val_00007151.JPEG:n02488702 +ILSVRC2012_val_00007152.JPEG:n02226429 +ILSVRC2012_val_00007153.JPEG:n02018207 +ILSVRC2012_val_00007154.JPEG:n04044716 +ILSVRC2012_val_00007155.JPEG:n03394916 +ILSVRC2012_val_00007156.JPEG:n01818515 +ILSVRC2012_val_00007157.JPEG:n01910747 +ILSVRC2012_val_00007158.JPEG:n03584829 +ILSVRC2012_val_00007159.JPEG:n03240683 +ILSVRC2012_val_00007160.JPEG:n04133789 +ILSVRC2012_val_00007161.JPEG:n03095699 +ILSVRC2012_val_00007162.JPEG:n04325704 +ILSVRC2012_val_00007163.JPEG:n02606052 +ILSVRC2012_val_00007164.JPEG:n02102318 +ILSVRC2012_val_00007165.JPEG:n02106382 +ILSVRC2012_val_00007166.JPEG:n03424325 +ILSVRC2012_val_00007167.JPEG:n02906734 +ILSVRC2012_val_00007168.JPEG:n01818515 +ILSVRC2012_val_00007169.JPEG:n04548362 +ILSVRC2012_val_00007170.JPEG:n04086273 +ILSVRC2012_val_00007171.JPEG:n07590611 +ILSVRC2012_val_00007172.JPEG:n02033041 +ILSVRC2012_val_00007173.JPEG:n04501370 +ILSVRC2012_val_00007174.JPEG:n02486261 +ILSVRC2012_val_00007175.JPEG:n03793489 +ILSVRC2012_val_00007176.JPEG:n02974003 +ILSVRC2012_val_00007177.JPEG:n09428293 +ILSVRC2012_val_00007178.JPEG:n02088466 +ILSVRC2012_val_00007179.JPEG:n04355933 +ILSVRC2012_val_00007180.JPEG:n02113712 +ILSVRC2012_val_00007181.JPEG:n02777292 +ILSVRC2012_val_00007182.JPEG:n02490219 +ILSVRC2012_val_00007183.JPEG:n02105056 +ILSVRC2012_val_00007184.JPEG:n02071294 +ILSVRC2012_val_00007185.JPEG:n02655020 +ILSVRC2012_val_00007186.JPEG:n03425413 +ILSVRC2012_val_00007187.JPEG:n02808440 +ILSVRC2012_val_00007188.JPEG:n02493509 +ILSVRC2012_val_00007189.JPEG:n03384352 +ILSVRC2012_val_00007190.JPEG:n02108422 +ILSVRC2012_val_00007191.JPEG:n04350905 +ILSVRC2012_val_00007192.JPEG:n07695742 +ILSVRC2012_val_00007193.JPEG:n02077923 +ILSVRC2012_val_00007194.JPEG:n03476991 +ILSVRC2012_val_00007195.JPEG:n03857828 +ILSVRC2012_val_00007196.JPEG:n02494079 +ILSVRC2012_val_00007197.JPEG:n01440764 +ILSVRC2012_val_00007198.JPEG:n02277742 +ILSVRC2012_val_00007199.JPEG:n02509815 +ILSVRC2012_val_00007200.JPEG:n07730033 +ILSVRC2012_val_00007201.JPEG:n01774384 +ILSVRC2012_val_00007202.JPEG:n02951585 +ILSVRC2012_val_00007203.JPEG:n02892201 +ILSVRC2012_val_00007204.JPEG:n02488702 +ILSVRC2012_val_00007205.JPEG:n02782093 +ILSVRC2012_val_00007206.JPEG:n03854065 +ILSVRC2012_val_00007207.JPEG:n04517823 +ILSVRC2012_val_00007208.JPEG:n03467068 +ILSVRC2012_val_00007209.JPEG:n07920052 +ILSVRC2012_val_00007210.JPEG:n03180011 +ILSVRC2012_val_00007211.JPEG:n02111129 +ILSVRC2012_val_00007212.JPEG:n02361337 +ILSVRC2012_val_00007213.JPEG:n03544143 +ILSVRC2012_val_00007214.JPEG:n07717556 +ILSVRC2012_val_00007215.JPEG:n03291819 +ILSVRC2012_val_00007216.JPEG:n02110063 +ILSVRC2012_val_00007217.JPEG:n03825788 +ILSVRC2012_val_00007218.JPEG:n02110185 +ILSVRC2012_val_00007219.JPEG:n02108422 +ILSVRC2012_val_00007220.JPEG:n01744401 +ILSVRC2012_val_00007221.JPEG:n04204347 +ILSVRC2012_val_00007222.JPEG:n01744401 +ILSVRC2012_val_00007223.JPEG:n02086079 +ILSVRC2012_val_00007224.JPEG:n01773549 +ILSVRC2012_val_00007225.JPEG:n03498962 +ILSVRC2012_val_00007226.JPEG:n02979186 +ILSVRC2012_val_00007227.JPEG:n01694178 +ILSVRC2012_val_00007228.JPEG:n04265275 +ILSVRC2012_val_00007229.JPEG:n04371774 +ILSVRC2012_val_00007230.JPEG:n01669191 +ILSVRC2012_val_00007231.JPEG:n01582220 +ILSVRC2012_val_00007232.JPEG:n02128925 +ILSVRC2012_val_00007233.JPEG:n02747177 +ILSVRC2012_val_00007234.JPEG:n02108551 +ILSVRC2012_val_00007235.JPEG:n02105056 +ILSVRC2012_val_00007236.JPEG:n02107312 +ILSVRC2012_val_00007237.JPEG:n01532829 +ILSVRC2012_val_00007238.JPEG:n01698640 +ILSVRC2012_val_00007239.JPEG:n03661043 +ILSVRC2012_val_00007240.JPEG:n02834397 +ILSVRC2012_val_00007241.JPEG:n03956157 +ILSVRC2012_val_00007242.JPEG:n01739381 +ILSVRC2012_val_00007243.JPEG:n02500267 +ILSVRC2012_val_00007244.JPEG:n02317335 +ILSVRC2012_val_00007245.JPEG:n02951358 +ILSVRC2012_val_00007246.JPEG:n02105505 +ILSVRC2012_val_00007247.JPEG:n07718747 +ILSVRC2012_val_00007248.JPEG:n04192698 +ILSVRC2012_val_00007249.JPEG:n04536866 +ILSVRC2012_val_00007250.JPEG:n03710637 +ILSVRC2012_val_00007251.JPEG:n02346627 +ILSVRC2012_val_00007252.JPEG:n03476684 +ILSVRC2012_val_00007253.JPEG:n02086910 +ILSVRC2012_val_00007254.JPEG:n02747177 +ILSVRC2012_val_00007255.JPEG:n02096177 +ILSVRC2012_val_00007256.JPEG:n04548280 +ILSVRC2012_val_00007257.JPEG:n01630670 +ILSVRC2012_val_00007258.JPEG:n01682714 +ILSVRC2012_val_00007259.JPEG:n04275548 +ILSVRC2012_val_00007260.JPEG:n03538406 +ILSVRC2012_val_00007261.JPEG:n02113712 +ILSVRC2012_val_00007262.JPEG:n09421951 +ILSVRC2012_val_00007263.JPEG:n01560419 +ILSVRC2012_val_00007264.JPEG:n04252225 +ILSVRC2012_val_00007265.JPEG:n02423022 +ILSVRC2012_val_00007266.JPEG:n01697457 +ILSVRC2012_val_00007267.JPEG:n02389026 +ILSVRC2012_val_00007268.JPEG:n03595614 +ILSVRC2012_val_00007269.JPEG:n02415577 +ILSVRC2012_val_00007270.JPEG:n04004767 +ILSVRC2012_val_00007271.JPEG:n02672831 +ILSVRC2012_val_00007272.JPEG:n03018349 +ILSVRC2012_val_00007273.JPEG:n03998194 +ILSVRC2012_val_00007274.JPEG:n03089624 +ILSVRC2012_val_00007275.JPEG:n04273569 +ILSVRC2012_val_00007276.JPEG:n02058221 +ILSVRC2012_val_00007277.JPEG:n03544143 +ILSVRC2012_val_00007278.JPEG:n02395406 +ILSVRC2012_val_00007279.JPEG:n03535780 +ILSVRC2012_val_00007280.JPEG:n03450230 +ILSVRC2012_val_00007281.JPEG:n03888605 +ILSVRC2012_val_00007282.JPEG:n13052670 +ILSVRC2012_val_00007283.JPEG:n01910747 +ILSVRC2012_val_00007284.JPEG:n01843065 +ILSVRC2012_val_00007285.JPEG:n03982430 +ILSVRC2012_val_00007286.JPEG:n03447721 +ILSVRC2012_val_00007287.JPEG:n01955084 +ILSVRC2012_val_00007288.JPEG:n01630670 +ILSVRC2012_val_00007289.JPEG:n03803284 +ILSVRC2012_val_00007290.JPEG:n02120079 +ILSVRC2012_val_00007291.JPEG:n03372029 +ILSVRC2012_val_00007292.JPEG:n02504458 +ILSVRC2012_val_00007293.JPEG:n03874599 +ILSVRC2012_val_00007294.JPEG:n02011460 +ILSVRC2012_val_00007295.JPEG:n02108089 +ILSVRC2012_val_00007296.JPEG:n03627232 +ILSVRC2012_val_00007297.JPEG:n02492660 +ILSVRC2012_val_00007298.JPEG:n04399382 +ILSVRC2012_val_00007299.JPEG:n02412080 +ILSVRC2012_val_00007300.JPEG:n03325584 +ILSVRC2012_val_00007301.JPEG:n03706229 +ILSVRC2012_val_00007302.JPEG:n02500267 +ILSVRC2012_val_00007303.JPEG:n02123159 +ILSVRC2012_val_00007304.JPEG:n04238763 +ILSVRC2012_val_00007305.JPEG:n02883205 +ILSVRC2012_val_00007306.JPEG:n13044778 +ILSVRC2012_val_00007307.JPEG:n07836838 +ILSVRC2012_val_00007308.JPEG:n02799071 +ILSVRC2012_val_00007309.JPEG:n01917289 +ILSVRC2012_val_00007310.JPEG:n04273569 +ILSVRC2012_val_00007311.JPEG:n04552348 +ILSVRC2012_val_00007312.JPEG:n01795545 +ILSVRC2012_val_00007313.JPEG:n02011460 +ILSVRC2012_val_00007314.JPEG:n03944341 +ILSVRC2012_val_00007315.JPEG:n02356798 +ILSVRC2012_val_00007316.JPEG:n04264628 +ILSVRC2012_val_00007317.JPEG:n02859443 +ILSVRC2012_val_00007318.JPEG:n02108915 +ILSVRC2012_val_00007319.JPEG:n02108422 +ILSVRC2012_val_00007320.JPEG:n04591713 +ILSVRC2012_val_00007321.JPEG:n02099849 +ILSVRC2012_val_00007322.JPEG:n07693725 +ILSVRC2012_val_00007323.JPEG:n01795545 +ILSVRC2012_val_00007324.JPEG:n04596742 +ILSVRC2012_val_00007325.JPEG:n03868242 +ILSVRC2012_val_00007326.JPEG:n03958227 +ILSVRC2012_val_00007327.JPEG:n02093991 +ILSVRC2012_val_00007328.JPEG:n03134739 +ILSVRC2012_val_00007329.JPEG:n01917289 +ILSVRC2012_val_00007330.JPEG:n02099712 +ILSVRC2012_val_00007331.JPEG:n03314780 +ILSVRC2012_val_00007332.JPEG:n11879895 +ILSVRC2012_val_00007333.JPEG:n10148035 +ILSVRC2012_val_00007334.JPEG:n02018795 +ILSVRC2012_val_00007335.JPEG:n02747177 +ILSVRC2012_val_00007336.JPEG:n04542943 +ILSVRC2012_val_00007337.JPEG:n03141823 +ILSVRC2012_val_00007338.JPEG:n02797295 +ILSVRC2012_val_00007339.JPEG:n01704323 +ILSVRC2012_val_00007340.JPEG:n02777292 +ILSVRC2012_val_00007341.JPEG:n02769748 +ILSVRC2012_val_00007342.JPEG:n04033995 +ILSVRC2012_val_00007343.JPEG:n01860187 +ILSVRC2012_val_00007344.JPEG:n02321529 +ILSVRC2012_val_00007345.JPEG:n01917289 +ILSVRC2012_val_00007346.JPEG:n03785016 +ILSVRC2012_val_00007347.JPEG:n03956157 +ILSVRC2012_val_00007348.JPEG:n03100240 +ILSVRC2012_val_00007349.JPEG:n04041544 +ILSVRC2012_val_00007350.JPEG:n02165105 +ILSVRC2012_val_00007351.JPEG:n03947888 +ILSVRC2012_val_00007352.JPEG:n03891251 +ILSVRC2012_val_00007353.JPEG:n03709823 +ILSVRC2012_val_00007354.JPEG:n02988304 +ILSVRC2012_val_00007355.JPEG:n02106030 +ILSVRC2012_val_00007356.JPEG:n02095570 +ILSVRC2012_val_00007357.JPEG:n02814860 +ILSVRC2012_val_00007358.JPEG:n03649909 +ILSVRC2012_val_00007359.JPEG:n03110669 +ILSVRC2012_val_00007360.JPEG:n02444819 +ILSVRC2012_val_00007361.JPEG:n04044716 +ILSVRC2012_val_00007362.JPEG:n04487394 +ILSVRC2012_val_00007363.JPEG:n02422106 +ILSVRC2012_val_00007364.JPEG:n04069434 +ILSVRC2012_val_00007365.JPEG:n02165456 +ILSVRC2012_val_00007366.JPEG:n02098105 +ILSVRC2012_val_00007367.JPEG:n02106382 +ILSVRC2012_val_00007368.JPEG:n02280649 +ILSVRC2012_val_00007369.JPEG:n02002556 +ILSVRC2012_val_00007370.JPEG:n01980166 +ILSVRC2012_val_00007371.JPEG:n02091032 +ILSVRC2012_val_00007372.JPEG:n09229709 +ILSVRC2012_val_00007373.JPEG:n03642806 +ILSVRC2012_val_00007374.JPEG:n03770679 +ILSVRC2012_val_00007375.JPEG:n02172182 +ILSVRC2012_val_00007376.JPEG:n07892512 +ILSVRC2012_val_00007377.JPEG:n01944390 +ILSVRC2012_val_00007378.JPEG:n04462240 +ILSVRC2012_val_00007379.JPEG:n02114548 +ILSVRC2012_val_00007380.JPEG:n02403003 +ILSVRC2012_val_00007381.JPEG:n03899768 +ILSVRC2012_val_00007382.JPEG:n09472597 +ILSVRC2012_val_00007383.JPEG:n03530642 +ILSVRC2012_val_00007384.JPEG:n02974003 +ILSVRC2012_val_00007385.JPEG:n02777292 +ILSVRC2012_val_00007386.JPEG:n02093428 +ILSVRC2012_val_00007387.JPEG:n01829413 +ILSVRC2012_val_00007388.JPEG:n02097298 +ILSVRC2012_val_00007389.JPEG:n01882714 +ILSVRC2012_val_00007390.JPEG:n01833805 +ILSVRC2012_val_00007391.JPEG:n03481172 +ILSVRC2012_val_00007392.JPEG:n02094114 +ILSVRC2012_val_00007393.JPEG:n03218198 +ILSVRC2012_val_00007394.JPEG:n02640242 +ILSVRC2012_val_00007395.JPEG:n02422699 +ILSVRC2012_val_00007396.JPEG:n03297495 +ILSVRC2012_val_00007397.JPEG:n04592741 +ILSVRC2012_val_00007398.JPEG:n01644373 +ILSVRC2012_val_00007399.JPEG:n02066245 +ILSVRC2012_val_00007400.JPEG:n03028079 +ILSVRC2012_val_00007401.JPEG:n04399382 +ILSVRC2012_val_00007402.JPEG:n03355925 +ILSVRC2012_val_00007403.JPEG:n03187595 +ILSVRC2012_val_00007404.JPEG:n02071294 +ILSVRC2012_val_00007405.JPEG:n01494475 +ILSVRC2012_val_00007406.JPEG:n02119789 +ILSVRC2012_val_00007407.JPEG:n02963159 +ILSVRC2012_val_00007408.JPEG:n03976657 +ILSVRC2012_val_00007409.JPEG:n03759954 +ILSVRC2012_val_00007410.JPEG:n02916936 +ILSVRC2012_val_00007411.JPEG:n02120079 +ILSVRC2012_val_00007412.JPEG:n03109150 +ILSVRC2012_val_00007413.JPEG:n04370456 +ILSVRC2012_val_00007414.JPEG:n02817516 +ILSVRC2012_val_00007415.JPEG:n01734418 +ILSVRC2012_val_00007416.JPEG:n02415577 +ILSVRC2012_val_00007417.JPEG:n03691459 +ILSVRC2012_val_00007418.JPEG:n04023962 +ILSVRC2012_val_00007419.JPEG:n02114712 +ILSVRC2012_val_00007420.JPEG:n03995372 +ILSVRC2012_val_00007421.JPEG:n06359193 +ILSVRC2012_val_00007422.JPEG:n01943899 +ILSVRC2012_val_00007423.JPEG:n01860187 +ILSVRC2012_val_00007424.JPEG:n02859443 +ILSVRC2012_val_00007425.JPEG:n02268443 +ILSVRC2012_val_00007426.JPEG:n02488702 +ILSVRC2012_val_00007427.JPEG:n03110669 +ILSVRC2012_val_00007428.JPEG:n03250847 +ILSVRC2012_val_00007429.JPEG:n02165105 +ILSVRC2012_val_00007430.JPEG:n02102480 +ILSVRC2012_val_00007431.JPEG:n03026506 +ILSVRC2012_val_00007432.JPEG:n04465501 +ILSVRC2012_val_00007433.JPEG:n03733131 +ILSVRC2012_val_00007434.JPEG:n01910747 +ILSVRC2012_val_00007435.JPEG:n04277352 +ILSVRC2012_val_00007436.JPEG:n03065424 +ILSVRC2012_val_00007437.JPEG:n01644900 +ILSVRC2012_val_00007438.JPEG:n02951358 +ILSVRC2012_val_00007439.JPEG:n04399382 +ILSVRC2012_val_00007440.JPEG:n02326432 +ILSVRC2012_val_00007441.JPEG:n03529860 +ILSVRC2012_val_00007442.JPEG:n03764736 +ILSVRC2012_val_00007443.JPEG:n02444819 +ILSVRC2012_val_00007444.JPEG:n02093256 +ILSVRC2012_val_00007445.JPEG:n02091134 +ILSVRC2012_val_00007446.JPEG:n02091635 +ILSVRC2012_val_00007447.JPEG:n11879895 +ILSVRC2012_val_00007448.JPEG:n03657121 +ILSVRC2012_val_00007449.JPEG:n04613696 +ILSVRC2012_val_00007450.JPEG:n03452741 +ILSVRC2012_val_00007451.JPEG:n04596742 +ILSVRC2012_val_00007452.JPEG:n02097474 +ILSVRC2012_val_00007453.JPEG:n02672831 +ILSVRC2012_val_00007454.JPEG:n01968897 +ILSVRC2012_val_00007455.JPEG:n02486410 +ILSVRC2012_val_00007456.JPEG:n02488291 +ILSVRC2012_val_00007457.JPEG:n02356798 +ILSVRC2012_val_00007458.JPEG:n07749582 +ILSVRC2012_val_00007459.JPEG:n04033995 +ILSVRC2012_val_00007460.JPEG:n03000684 +ILSVRC2012_val_00007461.JPEG:n04428191 +ILSVRC2012_val_00007462.JPEG:n02089078 +ILSVRC2012_val_00007463.JPEG:n04005630 +ILSVRC2012_val_00007464.JPEG:n03476991 +ILSVRC2012_val_00007465.JPEG:n02817516 +ILSVRC2012_val_00007466.JPEG:n04371774 +ILSVRC2012_val_00007467.JPEG:n12144580 +ILSVRC2012_val_00007468.JPEG:n12144580 +ILSVRC2012_val_00007469.JPEG:n03950228 +ILSVRC2012_val_00007470.JPEG:n02009912 +ILSVRC2012_val_00007471.JPEG:n03425413 +ILSVRC2012_val_00007472.JPEG:n04141975 +ILSVRC2012_val_00007473.JPEG:n02790996 +ILSVRC2012_val_00007474.JPEG:n01818515 +ILSVRC2012_val_00007475.JPEG:n07583066 +ILSVRC2012_val_00007476.JPEG:n04116512 +ILSVRC2012_val_00007477.JPEG:n03417042 +ILSVRC2012_val_00007478.JPEG:n01739381 +ILSVRC2012_val_00007479.JPEG:n01944390 +ILSVRC2012_val_00007480.JPEG:n03447721 +ILSVRC2012_val_00007481.JPEG:n03891332 +ILSVRC2012_val_00007482.JPEG:n01689811 +ILSVRC2012_val_00007483.JPEG:n04081281 +ILSVRC2012_val_00007484.JPEG:n02892767 +ILSVRC2012_val_00007485.JPEG:n04590129 +ILSVRC2012_val_00007486.JPEG:n01632777 +ILSVRC2012_val_00007487.JPEG:n02086910 +ILSVRC2012_val_00007488.JPEG:n01742172 +ILSVRC2012_val_00007489.JPEG:n04579145 +ILSVRC2012_val_00007490.JPEG:n02814860 +ILSVRC2012_val_00007491.JPEG:n04458633 +ILSVRC2012_val_00007492.JPEG:n04487394 +ILSVRC2012_val_00007493.JPEG:n02088632 +ILSVRC2012_val_00007494.JPEG:n03942813 +ILSVRC2012_val_00007495.JPEG:n04162706 +ILSVRC2012_val_00007496.JPEG:n07613480 +ILSVRC2012_val_00007497.JPEG:n02098413 +ILSVRC2012_val_00007498.JPEG:n04037443 +ILSVRC2012_val_00007499.JPEG:n02457408 +ILSVRC2012_val_00007500.JPEG:n04461696 +ILSVRC2012_val_00007501.JPEG:n02110185 +ILSVRC2012_val_00007502.JPEG:n03887697 +ILSVRC2012_val_00007503.JPEG:n03344393 +ILSVRC2012_val_00007504.JPEG:n04336792 +ILSVRC2012_val_00007505.JPEG:n04209239 +ILSVRC2012_val_00007506.JPEG:n02480495 +ILSVRC2012_val_00007507.JPEG:n02102480 +ILSVRC2012_val_00007508.JPEG:n04040759 +ILSVRC2012_val_00007509.JPEG:n03372029 +ILSVRC2012_val_00007510.JPEG:n03017168 +ILSVRC2012_val_00007511.JPEG:n02087046 +ILSVRC2012_val_00007512.JPEG:n02110185 +ILSVRC2012_val_00007513.JPEG:n04131690 +ILSVRC2012_val_00007514.JPEG:n02133161 +ILSVRC2012_val_00007515.JPEG:n02749479 +ILSVRC2012_val_00007516.JPEG:n02092002 +ILSVRC2012_val_00007517.JPEG:n04612504 +ILSVRC2012_val_00007518.JPEG:n03388183 +ILSVRC2012_val_00007519.JPEG:n03417042 +ILSVRC2012_val_00007520.JPEG:n02168699 +ILSVRC2012_val_00007521.JPEG:n07248320 +ILSVRC2012_val_00007522.JPEG:n02012849 +ILSVRC2012_val_00007523.JPEG:n03791053 +ILSVRC2012_val_00007524.JPEG:n02027492 +ILSVRC2012_val_00007525.JPEG:n07768694 +ILSVRC2012_val_00007526.JPEG:n02115913 +ILSVRC2012_val_00007527.JPEG:n02093428 +ILSVRC2012_val_00007528.JPEG:n01630670 +ILSVRC2012_val_00007529.JPEG:n02226429 +ILSVRC2012_val_00007530.JPEG:n01514859 +ILSVRC2012_val_00007531.JPEG:n07716358 +ILSVRC2012_val_00007532.JPEG:n02860847 +ILSVRC2012_val_00007533.JPEG:n04041544 +ILSVRC2012_val_00007534.JPEG:n02105505 +ILSVRC2012_val_00007535.JPEG:n02107683 +ILSVRC2012_val_00007536.JPEG:n03394916 +ILSVRC2012_val_00007537.JPEG:n03384352 +ILSVRC2012_val_00007538.JPEG:n04536866 +ILSVRC2012_val_00007539.JPEG:n02107312 +ILSVRC2012_val_00007540.JPEG:n04487081 +ILSVRC2012_val_00007541.JPEG:n02447366 +ILSVRC2012_val_00007542.JPEG:n02113186 +ILSVRC2012_val_00007543.JPEG:n03777754 +ILSVRC2012_val_00007544.JPEG:n03496892 +ILSVRC2012_val_00007545.JPEG:n09421951 +ILSVRC2012_val_00007546.JPEG:n02097298 +ILSVRC2012_val_00007547.JPEG:n02112706 +ILSVRC2012_val_00007548.JPEG:n02128757 +ILSVRC2012_val_00007549.JPEG:n02169497 +ILSVRC2012_val_00007550.JPEG:n03933933 +ILSVRC2012_val_00007551.JPEG:n02109961 +ILSVRC2012_val_00007552.JPEG:n04254120 +ILSVRC2012_val_00007553.JPEG:n04562935 +ILSVRC2012_val_00007554.JPEG:n02457408 +ILSVRC2012_val_00007555.JPEG:n02093754 +ILSVRC2012_val_00007556.JPEG:n15075141 +ILSVRC2012_val_00007557.JPEG:n02788148 +ILSVRC2012_val_00007558.JPEG:n01751748 +ILSVRC2012_val_00007559.JPEG:n02837789 +ILSVRC2012_val_00007560.JPEG:n06359193 +ILSVRC2012_val_00007561.JPEG:n01630670 +ILSVRC2012_val_00007562.JPEG:n03908618 +ILSVRC2012_val_00007563.JPEG:n07754684 +ILSVRC2012_val_00007564.JPEG:n02013706 +ILSVRC2012_val_00007565.JPEG:n03680355 +ILSVRC2012_val_00007566.JPEG:n02788148 +ILSVRC2012_val_00007567.JPEG:n06794110 +ILSVRC2012_val_00007568.JPEG:n02102040 +ILSVRC2012_val_00007569.JPEG:n01496331 +ILSVRC2012_val_00007570.JPEG:n03482405 +ILSVRC2012_val_00007571.JPEG:n02107312 +ILSVRC2012_val_00007572.JPEG:n13054560 +ILSVRC2012_val_00007573.JPEG:n03843555 +ILSVRC2012_val_00007574.JPEG:n01644373 +ILSVRC2012_val_00007575.JPEG:n02894605 +ILSVRC2012_val_00007576.JPEG:n01818515 +ILSVRC2012_val_00007577.JPEG:n03899768 +ILSVRC2012_val_00007578.JPEG:n02134084 +ILSVRC2012_val_00007579.JPEG:n01692333 +ILSVRC2012_val_00007580.JPEG:n02948072 +ILSVRC2012_val_00007581.JPEG:n03743016 +ILSVRC2012_val_00007582.JPEG:n07583066 +ILSVRC2012_val_00007583.JPEG:n02279972 +ILSVRC2012_val_00007584.JPEG:n07760859 +ILSVRC2012_val_00007585.JPEG:n03868863 +ILSVRC2012_val_00007586.JPEG:n02422699 +ILSVRC2012_val_00007587.JPEG:n02825657 +ILSVRC2012_val_00007588.JPEG:n02480855 +ILSVRC2012_val_00007589.JPEG:n02226429 +ILSVRC2012_val_00007590.JPEG:n04033901 +ILSVRC2012_val_00007591.JPEG:n01817953 +ILSVRC2012_val_00007592.JPEG:n04285008 +ILSVRC2012_val_00007593.JPEG:n04550184 +ILSVRC2012_val_00007594.JPEG:n04476259 +ILSVRC2012_val_00007595.JPEG:n02100877 +ILSVRC2012_val_00007596.JPEG:n09835506 +ILSVRC2012_val_00007597.JPEG:n02410509 +ILSVRC2012_val_00007598.JPEG:n03207743 +ILSVRC2012_val_00007599.JPEG:n03877845 +ILSVRC2012_val_00007600.JPEG:n03947888 +ILSVRC2012_val_00007601.JPEG:n01774750 +ILSVRC2012_val_00007602.JPEG:n02641379 +ILSVRC2012_val_00007603.JPEG:n04584207 +ILSVRC2012_val_00007604.JPEG:n02481823 +ILSVRC2012_val_00007605.JPEG:n07768694 +ILSVRC2012_val_00007606.JPEG:n02130308 +ILSVRC2012_val_00007607.JPEG:n04147183 +ILSVRC2012_val_00007608.JPEG:n04596742 +ILSVRC2012_val_00007609.JPEG:n02395406 +ILSVRC2012_val_00007610.JPEG:n07754684 +ILSVRC2012_val_00007611.JPEG:n04252225 +ILSVRC2012_val_00007612.JPEG:n04118538 +ILSVRC2012_val_00007613.JPEG:n09256479 +ILSVRC2012_val_00007614.JPEG:n07742313 +ILSVRC2012_val_00007615.JPEG:n02769748 +ILSVRC2012_val_00007616.JPEG:n03888257 +ILSVRC2012_val_00007617.JPEG:n03658185 +ILSVRC2012_val_00007618.JPEG:n04067472 +ILSVRC2012_val_00007619.JPEG:n02481823 +ILSVRC2012_val_00007620.JPEG:n03255030 +ILSVRC2012_val_00007621.JPEG:n03903868 +ILSVRC2012_val_00007622.JPEG:n03124043 +ILSVRC2012_val_00007623.JPEG:n03874599 +ILSVRC2012_val_00007624.JPEG:n06596364 +ILSVRC2012_val_00007625.JPEG:n04355933 +ILSVRC2012_val_00007626.JPEG:n04613696 +ILSVRC2012_val_00007627.JPEG:n04357314 +ILSVRC2012_val_00007628.JPEG:n02814860 +ILSVRC2012_val_00007629.JPEG:n02099601 +ILSVRC2012_val_00007630.JPEG:n01806567 +ILSVRC2012_val_00007631.JPEG:n02396427 +ILSVRC2012_val_00007632.JPEG:n02106166 +ILSVRC2012_val_00007633.JPEG:n03769881 +ILSVRC2012_val_00007634.JPEG:n02113023 +ILSVRC2012_val_00007635.JPEG:n04146614 +ILSVRC2012_val_00007636.JPEG:n02640242 +ILSVRC2012_val_00007637.JPEG:n02966193 +ILSVRC2012_val_00007638.JPEG:n02841315 +ILSVRC2012_val_00007639.JPEG:n02481823 +ILSVRC2012_val_00007640.JPEG:n03724870 +ILSVRC2012_val_00007641.JPEG:n03998194 +ILSVRC2012_val_00007642.JPEG:n04522168 +ILSVRC2012_val_00007643.JPEG:n02747177 +ILSVRC2012_val_00007644.JPEG:n02317335 +ILSVRC2012_val_00007645.JPEG:n04067472 +ILSVRC2012_val_00007646.JPEG:n02129165 +ILSVRC2012_val_00007647.JPEG:n07714571 +ILSVRC2012_val_00007648.JPEG:n03992509 +ILSVRC2012_val_00007649.JPEG:n03379051 +ILSVRC2012_val_00007650.JPEG:n04141975 +ILSVRC2012_val_00007651.JPEG:n02028035 +ILSVRC2012_val_00007652.JPEG:n02085936 +ILSVRC2012_val_00007653.JPEG:n04540053 +ILSVRC2012_val_00007654.JPEG:n02112137 +ILSVRC2012_val_00007655.JPEG:n03977966 +ILSVRC2012_val_00007656.JPEG:n03637318 +ILSVRC2012_val_00007657.JPEG:n03887697 +ILSVRC2012_val_00007658.JPEG:n09468604 +ILSVRC2012_val_00007659.JPEG:n03424325 +ILSVRC2012_val_00007660.JPEG:n04584207 +ILSVRC2012_val_00007661.JPEG:n01917289 +ILSVRC2012_val_00007662.JPEG:n07579787 +ILSVRC2012_val_00007663.JPEG:n03325584 +ILSVRC2012_val_00007664.JPEG:n01829413 +ILSVRC2012_val_00007665.JPEG:n04540053 +ILSVRC2012_val_00007666.JPEG:n03127925 +ILSVRC2012_val_00007667.JPEG:n01558993 +ILSVRC2012_val_00007668.JPEG:n02027492 +ILSVRC2012_val_00007669.JPEG:n03424325 +ILSVRC2012_val_00007670.JPEG:n03109150 +ILSVRC2012_val_00007671.JPEG:n06794110 +ILSVRC2012_val_00007672.JPEG:n01773797 +ILSVRC2012_val_00007673.JPEG:n03188531 +ILSVRC2012_val_00007674.JPEG:n02106382 +ILSVRC2012_val_00007675.JPEG:n03788365 +ILSVRC2012_val_00007676.JPEG:n02123159 +ILSVRC2012_val_00007677.JPEG:n01773797 +ILSVRC2012_val_00007678.JPEG:n02229544 +ILSVRC2012_val_00007679.JPEG:n02727426 +ILSVRC2012_val_00007680.JPEG:n02823428 +ILSVRC2012_val_00007681.JPEG:n02454379 +ILSVRC2012_val_00007682.JPEG:n02106030 +ILSVRC2012_val_00007683.JPEG:n01924916 +ILSVRC2012_val_00007684.JPEG:n12998815 +ILSVRC2012_val_00007685.JPEG:n04179913 +ILSVRC2012_val_00007686.JPEG:n04099969 +ILSVRC2012_val_00007687.JPEG:n07684084 +ILSVRC2012_val_00007688.JPEG:n03450230 +ILSVRC2012_val_00007689.JPEG:n04435653 +ILSVRC2012_val_00007690.JPEG:n02422106 +ILSVRC2012_val_00007691.JPEG:n03637318 +ILSVRC2012_val_00007692.JPEG:n03018349 +ILSVRC2012_val_00007693.JPEG:n04429376 +ILSVRC2012_val_00007694.JPEG:n03868863 +ILSVRC2012_val_00007695.JPEG:n02110806 +ILSVRC2012_val_00007696.JPEG:n02226429 +ILSVRC2012_val_00007697.JPEG:n02006656 +ILSVRC2012_val_00007698.JPEG:n03843555 +ILSVRC2012_val_00007699.JPEG:n06359193 +ILSVRC2012_val_00007700.JPEG:n01860187 +ILSVRC2012_val_00007701.JPEG:n01694178 +ILSVRC2012_val_00007702.JPEG:n02138441 +ILSVRC2012_val_00007703.JPEG:n03630383 +ILSVRC2012_val_00007704.JPEG:n04009552 +ILSVRC2012_val_00007705.JPEG:n02101006 +ILSVRC2012_val_00007706.JPEG:n03496892 +ILSVRC2012_val_00007707.JPEG:n03447721 +ILSVRC2012_val_00007708.JPEG:n07920052 +ILSVRC2012_val_00007709.JPEG:n07873807 +ILSVRC2012_val_00007710.JPEG:n01729977 +ILSVRC2012_val_00007711.JPEG:n03220513 +ILSVRC2012_val_00007712.JPEG:n01614925 +ILSVRC2012_val_00007713.JPEG:n02134084 +ILSVRC2012_val_00007714.JPEG:n03908618 +ILSVRC2012_val_00007715.JPEG:n03763968 +ILSVRC2012_val_00007716.JPEG:n03544143 +ILSVRC2012_val_00007717.JPEG:n02797295 +ILSVRC2012_val_00007718.JPEG:n04392985 +ILSVRC2012_val_00007719.JPEG:n01728920 +ILSVRC2012_val_00007720.JPEG:n03876231 +ILSVRC2012_val_00007721.JPEG:n03259280 +ILSVRC2012_val_00007722.JPEG:n03325584 +ILSVRC2012_val_00007723.JPEG:n04296562 +ILSVRC2012_val_00007724.JPEG:n02909870 +ILSVRC2012_val_00007725.JPEG:n02493793 +ILSVRC2012_val_00007726.JPEG:n02112706 +ILSVRC2012_val_00007727.JPEG:n02776631 +ILSVRC2012_val_00007728.JPEG:n02447366 +ILSVRC2012_val_00007729.JPEG:n01514859 +ILSVRC2012_val_00007730.JPEG:n03954731 +ILSVRC2012_val_00007731.JPEG:n03344393 +ILSVRC2012_val_00007732.JPEG:n04125021 +ILSVRC2012_val_00007733.JPEG:n03930630 +ILSVRC2012_val_00007734.JPEG:n04116512 +ILSVRC2012_val_00007735.JPEG:n02441942 +ILSVRC2012_val_00007736.JPEG:n03344393 +ILSVRC2012_val_00007737.JPEG:n02125311 +ILSVRC2012_val_00007738.JPEG:n02643566 +ILSVRC2012_val_00007739.JPEG:n03840681 +ILSVRC2012_val_00007740.JPEG:n02106662 +ILSVRC2012_val_00007741.JPEG:n03325584 +ILSVRC2012_val_00007742.JPEG:n07695742 +ILSVRC2012_val_00007743.JPEG:n01491361 +ILSVRC2012_val_00007744.JPEG:n03814906 +ILSVRC2012_val_00007745.JPEG:n03075370 +ILSVRC2012_val_00007746.JPEG:n02098286 +ILSVRC2012_val_00007747.JPEG:n02666196 +ILSVRC2012_val_00007748.JPEG:n07718472 +ILSVRC2012_val_00007749.JPEG:n02948072 +ILSVRC2012_val_00007750.JPEG:n01698640 +ILSVRC2012_val_00007751.JPEG:n03777754 +ILSVRC2012_val_00007752.JPEG:n07714571 +ILSVRC2012_val_00007753.JPEG:n01945685 +ILSVRC2012_val_00007754.JPEG:n03085013 +ILSVRC2012_val_00007755.JPEG:n03445777 +ILSVRC2012_val_00007756.JPEG:n04380533 +ILSVRC2012_val_00007757.JPEG:n01986214 +ILSVRC2012_val_00007758.JPEG:n03673027 +ILSVRC2012_val_00007759.JPEG:n03710193 +ILSVRC2012_val_00007760.JPEG:n02441942 +ILSVRC2012_val_00007761.JPEG:n01734418 +ILSVRC2012_val_00007762.JPEG:n02105412 +ILSVRC2012_val_00007763.JPEG:n03447447 +ILSVRC2012_val_00007764.JPEG:n04591157 +ILSVRC2012_val_00007765.JPEG:n02727426 +ILSVRC2012_val_00007766.JPEG:n04486054 +ILSVRC2012_val_00007767.JPEG:n02510455 +ILSVRC2012_val_00007768.JPEG:n03958227 +ILSVRC2012_val_00007769.JPEG:n01978455 +ILSVRC2012_val_00007770.JPEG:n04461696 +ILSVRC2012_val_00007771.JPEG:n03908618 +ILSVRC2012_val_00007772.JPEG:n04522168 +ILSVRC2012_val_00007773.JPEG:n02107908 +ILSVRC2012_val_00007774.JPEG:n07715103 +ILSVRC2012_val_00007775.JPEG:n04009552 +ILSVRC2012_val_00007776.JPEG:n03457902 +ILSVRC2012_val_00007777.JPEG:n03447447 +ILSVRC2012_val_00007778.JPEG:n01820546 +ILSVRC2012_val_00007779.JPEG:n02692877 +ILSVRC2012_val_00007780.JPEG:n03874599 +ILSVRC2012_val_00007781.JPEG:n02101388 +ILSVRC2012_val_00007782.JPEG:n02115641 +ILSVRC2012_val_00007783.JPEG:n03532672 +ILSVRC2012_val_00007784.JPEG:n03127925 +ILSVRC2012_val_00007785.JPEG:n04081281 +ILSVRC2012_val_00007786.JPEG:n02814533 +ILSVRC2012_val_00007787.JPEG:n02916936 +ILSVRC2012_val_00007788.JPEG:n02483708 +ILSVRC2012_val_00007789.JPEG:n02791124 +ILSVRC2012_val_00007790.JPEG:n04505470 +ILSVRC2012_val_00007791.JPEG:n04417672 +ILSVRC2012_val_00007792.JPEG:n03876231 +ILSVRC2012_val_00007793.JPEG:n01829413 +ILSVRC2012_val_00007794.JPEG:n09246464 +ILSVRC2012_val_00007795.JPEG:n01728920 +ILSVRC2012_val_00007796.JPEG:n02363005 +ILSVRC2012_val_00007797.JPEG:n07754684 +ILSVRC2012_val_00007798.JPEG:n07717556 +ILSVRC2012_val_00007799.JPEG:n03000247 +ILSVRC2012_val_00007800.JPEG:n01873310 +ILSVRC2012_val_00007801.JPEG:n02091635 +ILSVRC2012_val_00007802.JPEG:n07831146 +ILSVRC2012_val_00007803.JPEG:n02794156 +ILSVRC2012_val_00007804.JPEG:n03825788 +ILSVRC2012_val_00007805.JPEG:n03476991 +ILSVRC2012_val_00007806.JPEG:n04033901 +ILSVRC2012_val_00007807.JPEG:n02607072 +ILSVRC2012_val_00007808.JPEG:n02123394 +ILSVRC2012_val_00007809.JPEG:n03534580 +ILSVRC2012_val_00007810.JPEG:n01770081 +ILSVRC2012_val_00007811.JPEG:n02011460 +ILSVRC2012_val_00007812.JPEG:n02843684 +ILSVRC2012_val_00007813.JPEG:n02109525 +ILSVRC2012_val_00007814.JPEG:n03916031 +ILSVRC2012_val_00007815.JPEG:n04418357 +ILSVRC2012_val_00007816.JPEG:n03710637 +ILSVRC2012_val_00007817.JPEG:n03075370 +ILSVRC2012_val_00007818.JPEG:n01644900 +ILSVRC2012_val_00007819.JPEG:n04254680 +ILSVRC2012_val_00007820.JPEG:n07768694 +ILSVRC2012_val_00007821.JPEG:n04228054 +ILSVRC2012_val_00007822.JPEG:n04258138 +ILSVRC2012_val_00007823.JPEG:n04357314 +ILSVRC2012_val_00007824.JPEG:n07836838 +ILSVRC2012_val_00007825.JPEG:n03000134 +ILSVRC2012_val_00007826.JPEG:n04310018 +ILSVRC2012_val_00007827.JPEG:n03000134 +ILSVRC2012_val_00007828.JPEG:n02098413 +ILSVRC2012_val_00007829.JPEG:n02108000 +ILSVRC2012_val_00007830.JPEG:n04252077 +ILSVRC2012_val_00007831.JPEG:n02457408 +ILSVRC2012_val_00007832.JPEG:n04483307 +ILSVRC2012_val_00007833.JPEG:n02105505 +ILSVRC2012_val_00007834.JPEG:n03125729 +ILSVRC2012_val_00007835.JPEG:n02091467 +ILSVRC2012_val_00007836.JPEG:n03868242 +ILSVRC2012_val_00007837.JPEG:n02106166 +ILSVRC2012_val_00007838.JPEG:n03240683 +ILSVRC2012_val_00007839.JPEG:n02917067 +ILSVRC2012_val_00007840.JPEG:n02105056 +ILSVRC2012_val_00007841.JPEG:n04525305 +ILSVRC2012_val_00007842.JPEG:n01753488 +ILSVRC2012_val_00007843.JPEG:n02978881 +ILSVRC2012_val_00007844.JPEG:n03977966 +ILSVRC2012_val_00007845.JPEG:n02486261 +ILSVRC2012_val_00007846.JPEG:n04162706 +ILSVRC2012_val_00007847.JPEG:n02120079 +ILSVRC2012_val_00007848.JPEG:n03709823 +ILSVRC2012_val_00007849.JPEG:n03127747 +ILSVRC2012_val_00007850.JPEG:n02089973 +ILSVRC2012_val_00007851.JPEG:n03089624 +ILSVRC2012_val_00007852.JPEG:n03814906 +ILSVRC2012_val_00007853.JPEG:n01534433 +ILSVRC2012_val_00007854.JPEG:n04613696 +ILSVRC2012_val_00007855.JPEG:n03325584 +ILSVRC2012_val_00007856.JPEG:n04505470 +ILSVRC2012_val_00007857.JPEG:n03325584 +ILSVRC2012_val_00007858.JPEG:n02115641 +ILSVRC2012_val_00007859.JPEG:n03630383 +ILSVRC2012_val_00007860.JPEG:n01930112 +ILSVRC2012_val_00007861.JPEG:n04204238 +ILSVRC2012_val_00007862.JPEG:n03063689 +ILSVRC2012_val_00007863.JPEG:n02233338 +ILSVRC2012_val_00007864.JPEG:n03916031 +ILSVRC2012_val_00007865.JPEG:n02786058 +ILSVRC2012_val_00007866.JPEG:n02113799 +ILSVRC2012_val_00007867.JPEG:n03935335 +ILSVRC2012_val_00007868.JPEG:n04179913 +ILSVRC2012_val_00007869.JPEG:n03690938 +ILSVRC2012_val_00007870.JPEG:n02442845 +ILSVRC2012_val_00007871.JPEG:n01819313 +ILSVRC2012_val_00007872.JPEG:n01534433 +ILSVRC2012_val_00007873.JPEG:n01753488 +ILSVRC2012_val_00007874.JPEG:n02823750 +ILSVRC2012_val_00007875.JPEG:n01491361 +ILSVRC2012_val_00007876.JPEG:n03124043 +ILSVRC2012_val_00007877.JPEG:n01749939 +ILSVRC2012_val_00007878.JPEG:n02328150 +ILSVRC2012_val_00007879.JPEG:n03272562 +ILSVRC2012_val_00007880.JPEG:n02094258 +ILSVRC2012_val_00007881.JPEG:n04597913 +ILSVRC2012_val_00007882.JPEG:n01773549 +ILSVRC2012_val_00007883.JPEG:n03724870 +ILSVRC2012_val_00007884.JPEG:n01871265 +ILSVRC2012_val_00007885.JPEG:n01751748 +ILSVRC2012_val_00007886.JPEG:n04039381 +ILSVRC2012_val_00007887.JPEG:n03733805 +ILSVRC2012_val_00007888.JPEG:n02783161 +ILSVRC2012_val_00007889.JPEG:n02948072 +ILSVRC2012_val_00007890.JPEG:n02397096 +ILSVRC2012_val_00007891.JPEG:n02233338 +ILSVRC2012_val_00007892.JPEG:n02093647 +ILSVRC2012_val_00007893.JPEG:n03016953 +ILSVRC2012_val_00007894.JPEG:n04344873 +ILSVRC2012_val_00007895.JPEG:n02640242 +ILSVRC2012_val_00007896.JPEG:n01677366 +ILSVRC2012_val_00007897.JPEG:n02106166 +ILSVRC2012_val_00007898.JPEG:n07745940 +ILSVRC2012_val_00007899.JPEG:n03710637 +ILSVRC2012_val_00007900.JPEG:n03529860 +ILSVRC2012_val_00007901.JPEG:n02988304 +ILSVRC2012_val_00007902.JPEG:n04350905 +ILSVRC2012_val_00007903.JPEG:n02105056 +ILSVRC2012_val_00007904.JPEG:n01630670 +ILSVRC2012_val_00007905.JPEG:n12998815 +ILSVRC2012_val_00007906.JPEG:n02094258 +ILSVRC2012_val_00007907.JPEG:n03481172 +ILSVRC2012_val_00007908.JPEG:n04515003 +ILSVRC2012_val_00007909.JPEG:n04418357 +ILSVRC2012_val_00007910.JPEG:n03075370 +ILSVRC2012_val_00007911.JPEG:n04273569 +ILSVRC2012_val_00007912.JPEG:n01592084 +ILSVRC2012_val_00007913.JPEG:n03290653 +ILSVRC2012_val_00007914.JPEG:n04487394 +ILSVRC2012_val_00007915.JPEG:n02109047 +ILSVRC2012_val_00007916.JPEG:n02259212 +ILSVRC2012_val_00007917.JPEG:n04604644 +ILSVRC2012_val_00007918.JPEG:n03976467 +ILSVRC2012_val_00007919.JPEG:n04023962 +ILSVRC2012_val_00007920.JPEG:n02910353 +ILSVRC2012_val_00007921.JPEG:n03394916 +ILSVRC2012_val_00007922.JPEG:n02106662 +ILSVRC2012_val_00007923.JPEG:n01882714 +ILSVRC2012_val_00007924.JPEG:n03494278 +ILSVRC2012_val_00007925.JPEG:n01770393 +ILSVRC2012_val_00007926.JPEG:n03445924 +ILSVRC2012_val_00007927.JPEG:n02102177 +ILSVRC2012_val_00007928.JPEG:n02110958 +ILSVRC2012_val_00007929.JPEG:n02089973 +ILSVRC2012_val_00007930.JPEG:n01924916 +ILSVRC2012_val_00007931.JPEG:n02113799 +ILSVRC2012_val_00007932.JPEG:n01817953 +ILSVRC2012_val_00007933.JPEG:n02091134 +ILSVRC2012_val_00007934.JPEG:n01697457 +ILSVRC2012_val_00007935.JPEG:n03443371 +ILSVRC2012_val_00007936.JPEG:n04482393 +ILSVRC2012_val_00007937.JPEG:n01749939 +ILSVRC2012_val_00007938.JPEG:n01985128 +ILSVRC2012_val_00007939.JPEG:n04116512 +ILSVRC2012_val_00007940.JPEG:n03452741 +ILSVRC2012_val_00007941.JPEG:n03220513 +ILSVRC2012_val_00007942.JPEG:n02510455 +ILSVRC2012_val_00007943.JPEG:n03761084 +ILSVRC2012_val_00007944.JPEG:n02916936 +ILSVRC2012_val_00007945.JPEG:n02089867 +ILSVRC2012_val_00007946.JPEG:n02281406 +ILSVRC2012_val_00007947.JPEG:n03445777 +ILSVRC2012_val_00007948.JPEG:n03642806 +ILSVRC2012_val_00007949.JPEG:n03255030 +ILSVRC2012_val_00007950.JPEG:n09428293 +ILSVRC2012_val_00007951.JPEG:n01774750 +ILSVRC2012_val_00007952.JPEG:n03220513 +ILSVRC2012_val_00007953.JPEG:n04254777 +ILSVRC2012_val_00007954.JPEG:n13037406 +ILSVRC2012_val_00007955.JPEG:n04235860 +ILSVRC2012_val_00007956.JPEG:n07875152 +ILSVRC2012_val_00007957.JPEG:n01877812 +ILSVRC2012_val_00007958.JPEG:n02086240 +ILSVRC2012_val_00007959.JPEG:n03876231 +ILSVRC2012_val_00007960.JPEG:n02484975 +ILSVRC2012_val_00007961.JPEG:n03595614 +ILSVRC2012_val_00007962.JPEG:n03733805 +ILSVRC2012_val_00007963.JPEG:n02099712 +ILSVRC2012_val_00007964.JPEG:n03884397 +ILSVRC2012_val_00007965.JPEG:n03016953 +ILSVRC2012_val_00007966.JPEG:n02088632 +ILSVRC2012_val_00007967.JPEG:n04086273 +ILSVRC2012_val_00007968.JPEG:n02797295 +ILSVRC2012_val_00007969.JPEG:n04392985 +ILSVRC2012_val_00007970.JPEG:n03124043 +ILSVRC2012_val_00007971.JPEG:n02102480 +ILSVRC2012_val_00007972.JPEG:n02100583 +ILSVRC2012_val_00007973.JPEG:n01855032 +ILSVRC2012_val_00007974.JPEG:n02667093 +ILSVRC2012_val_00007975.JPEG:n01945685 +ILSVRC2012_val_00007976.JPEG:n03250847 +ILSVRC2012_val_00007977.JPEG:n01644373 +ILSVRC2012_val_00007978.JPEG:n04147183 +ILSVRC2012_val_00007979.JPEG:n02641379 +ILSVRC2012_val_00007980.JPEG:n02342885 +ILSVRC2012_val_00007981.JPEG:n03666591 +ILSVRC2012_val_00007982.JPEG:n03000134 +ILSVRC2012_val_00007983.JPEG:n03197337 +ILSVRC2012_val_00007984.JPEG:n02807133 +ILSVRC2012_val_00007985.JPEG:n03394916 +ILSVRC2012_val_00007986.JPEG:n01797886 +ILSVRC2012_val_00007987.JPEG:n02443114 +ILSVRC2012_val_00007988.JPEG:n02056570 +ILSVRC2012_val_00007989.JPEG:n02916936 +ILSVRC2012_val_00007990.JPEG:n04090263 +ILSVRC2012_val_00007991.JPEG:n01756291 +ILSVRC2012_val_00007992.JPEG:n03724870 +ILSVRC2012_val_00007993.JPEG:n02747177 +ILSVRC2012_val_00007994.JPEG:n04553703 +ILSVRC2012_val_00007995.JPEG:n01983481 +ILSVRC2012_val_00007996.JPEG:n04479046 +ILSVRC2012_val_00007997.JPEG:n07920052 +ILSVRC2012_val_00007998.JPEG:n01631663 +ILSVRC2012_val_00007999.JPEG:n01981276 +ILSVRC2012_val_00008000.JPEG:n02097474 +ILSVRC2012_val_00008001.JPEG:n02268443 +ILSVRC2012_val_00008002.JPEG:n01944390 +ILSVRC2012_val_00008003.JPEG:n02108422 +ILSVRC2012_val_00008004.JPEG:n04487081 +ILSVRC2012_val_00008005.JPEG:n07734744 +ILSVRC2012_val_00008006.JPEG:n02091244 +ILSVRC2012_val_00008007.JPEG:n02835271 +ILSVRC2012_val_00008008.JPEG:n01824575 +ILSVRC2012_val_00008009.JPEG:n02056570 +ILSVRC2012_val_00008010.JPEG:n03773504 +ILSVRC2012_val_00008011.JPEG:n01688243 +ILSVRC2012_val_00008012.JPEG:n03345487 +ILSVRC2012_val_00008013.JPEG:n03345487 +ILSVRC2012_val_00008014.JPEG:n02486410 +ILSVRC2012_val_00008015.JPEG:n03271574 +ILSVRC2012_val_00008016.JPEG:n03485407 +ILSVRC2012_val_00008017.JPEG:n02483362 +ILSVRC2012_val_00008018.JPEG:n02113712 +ILSVRC2012_val_00008019.JPEG:n02786058 +ILSVRC2012_val_00008020.JPEG:n04579145 +ILSVRC2012_val_00008021.JPEG:n02948072 +ILSVRC2012_val_00008022.JPEG:n03595614 +ILSVRC2012_val_00008023.JPEG:n03594734 +ILSVRC2012_val_00008024.JPEG:n01491361 +ILSVRC2012_val_00008025.JPEG:n01729977 +ILSVRC2012_val_00008026.JPEG:n04033995 +ILSVRC2012_val_00008027.JPEG:n04597913 +ILSVRC2012_val_00008028.JPEG:n01871265 +ILSVRC2012_val_00008029.JPEG:n02992211 +ILSVRC2012_val_00008030.JPEG:n02361337 +ILSVRC2012_val_00008031.JPEG:n04070727 +ILSVRC2012_val_00008032.JPEG:n02007558 +ILSVRC2012_val_00008033.JPEG:n03110669 +ILSVRC2012_val_00008034.JPEG:n09399592 +ILSVRC2012_val_00008035.JPEG:n02009912 +ILSVRC2012_val_00008036.JPEG:n03249569 +ILSVRC2012_val_00008037.JPEG:n02415577 +ILSVRC2012_val_00008038.JPEG:n02190166 +ILSVRC2012_val_00008039.JPEG:n02701002 +ILSVRC2012_val_00008040.JPEG:n03042490 +ILSVRC2012_val_00008041.JPEG:n01871265 +ILSVRC2012_val_00008042.JPEG:n02091467 +ILSVRC2012_val_00008043.JPEG:n03208938 +ILSVRC2012_val_00008044.JPEG:n02105505 +ILSVRC2012_val_00008045.JPEG:n04589890 +ILSVRC2012_val_00008046.JPEG:n02138441 +ILSVRC2012_val_00008047.JPEG:n04591157 +ILSVRC2012_val_00008048.JPEG:n03344393 +ILSVRC2012_val_00008049.JPEG:n01622779 +ILSVRC2012_val_00008050.JPEG:n01924916 +ILSVRC2012_val_00008051.JPEG:n02137549 +ILSVRC2012_val_00008052.JPEG:n04328186 +ILSVRC2012_val_00008053.JPEG:n07590611 +ILSVRC2012_val_00008054.JPEG:n01776313 +ILSVRC2012_val_00008055.JPEG:n04389033 +ILSVRC2012_val_00008056.JPEG:n02058221 +ILSVRC2012_val_00008057.JPEG:n03786901 +ILSVRC2012_val_00008058.JPEG:n02865351 +ILSVRC2012_val_00008059.JPEG:n02536864 +ILSVRC2012_val_00008060.JPEG:n04154565 +ILSVRC2012_val_00008061.JPEG:n02108422 +ILSVRC2012_val_00008062.JPEG:n07583066 +ILSVRC2012_val_00008063.JPEG:n03770439 +ILSVRC2012_val_00008064.JPEG:n04235860 +ILSVRC2012_val_00008065.JPEG:n03594945 +ILSVRC2012_val_00008066.JPEG:n02096051 +ILSVRC2012_val_00008067.JPEG:n03590841 +ILSVRC2012_val_00008068.JPEG:n04525038 +ILSVRC2012_val_00008069.JPEG:n02264363 +ILSVRC2012_val_00008070.JPEG:n04592741 +ILSVRC2012_val_00008071.JPEG:n02364673 +ILSVRC2012_val_00008072.JPEG:n01735189 +ILSVRC2012_val_00008073.JPEG:n02977058 +ILSVRC2012_val_00008074.JPEG:n02488291 +ILSVRC2012_val_00008075.JPEG:n07871810 +ILSVRC2012_val_00008076.JPEG:n03062245 +ILSVRC2012_val_00008077.JPEG:n04557648 +ILSVRC2012_val_00008078.JPEG:n03837869 +ILSVRC2012_val_00008079.JPEG:n01770081 +ILSVRC2012_val_00008080.JPEG:n04273569 +ILSVRC2012_val_00008081.JPEG:n03290653 +ILSVRC2012_val_00008082.JPEG:n03124043 +ILSVRC2012_val_00008083.JPEG:n02971356 +ILSVRC2012_val_00008084.JPEG:n02423022 +ILSVRC2012_val_00008085.JPEG:n02094114 +ILSVRC2012_val_00008086.JPEG:n01695060 +ILSVRC2012_val_00008087.JPEG:n01917289 +ILSVRC2012_val_00008088.JPEG:n02814533 +ILSVRC2012_val_00008089.JPEG:n03250847 +ILSVRC2012_val_00008090.JPEG:n02110063 +ILSVRC2012_val_00008091.JPEG:n02666196 +ILSVRC2012_val_00008092.JPEG:n02488291 +ILSVRC2012_val_00008093.JPEG:n02504013 +ILSVRC2012_val_00008094.JPEG:n02130308 +ILSVRC2012_val_00008095.JPEG:n01695060 +ILSVRC2012_val_00008096.JPEG:n03089624 +ILSVRC2012_val_00008097.JPEG:n02906734 +ILSVRC2012_val_00008098.JPEG:n02791124 +ILSVRC2012_val_00008099.JPEG:n09835506 +ILSVRC2012_val_00008100.JPEG:n07695742 +ILSVRC2012_val_00008101.JPEG:n06874185 +ILSVRC2012_val_00008102.JPEG:n04229816 +ILSVRC2012_val_00008103.JPEG:n02408429 +ILSVRC2012_val_00008104.JPEG:n02087394 +ILSVRC2012_val_00008105.JPEG:n03297495 +ILSVRC2012_val_00008106.JPEG:n02058221 +ILSVRC2012_val_00008107.JPEG:n03763968 +ILSVRC2012_val_00008108.JPEG:n01491361 +ILSVRC2012_val_00008109.JPEG:n03781244 +ILSVRC2012_val_00008110.JPEG:n03873416 +ILSVRC2012_val_00008111.JPEG:n02111277 +ILSVRC2012_val_00008112.JPEG:n13052670 +ILSVRC2012_val_00008113.JPEG:n02119022 +ILSVRC2012_val_00008114.JPEG:n02108000 +ILSVRC2012_val_00008115.JPEG:n02791124 +ILSVRC2012_val_00008116.JPEG:n03028079 +ILSVRC2012_val_00008117.JPEG:n02906734 +ILSVRC2012_val_00008118.JPEG:n02112350 +ILSVRC2012_val_00008119.JPEG:n02102318 +ILSVRC2012_val_00008120.JPEG:n04118776 +ILSVRC2012_val_00008121.JPEG:n02823428 +ILSVRC2012_val_00008122.JPEG:n04435653 +ILSVRC2012_val_00008123.JPEG:n03786901 +ILSVRC2012_val_00008124.JPEG:n02105505 +ILSVRC2012_val_00008125.JPEG:n01514859 +ILSVRC2012_val_00008126.JPEG:n02860847 +ILSVRC2012_val_00008127.JPEG:n01871265 +ILSVRC2012_val_00008128.JPEG:n07742313 +ILSVRC2012_val_00008129.JPEG:n01695060 +ILSVRC2012_val_00008130.JPEG:n01735189 +ILSVRC2012_val_00008131.JPEG:n03141823 +ILSVRC2012_val_00008132.JPEG:n02692877 +ILSVRC2012_val_00008133.JPEG:n04254680 +ILSVRC2012_val_00008134.JPEG:n02483708 +ILSVRC2012_val_00008135.JPEG:n02011460 +ILSVRC2012_val_00008136.JPEG:n02927161 +ILSVRC2012_val_00008137.JPEG:n02113978 +ILSVRC2012_val_00008138.JPEG:n02106166 +ILSVRC2012_val_00008139.JPEG:n03770679 +ILSVRC2012_val_00008140.JPEG:n02169497 +ILSVRC2012_val_00008141.JPEG:n04482393 +ILSVRC2012_val_00008142.JPEG:n02277742 +ILSVRC2012_val_00008143.JPEG:n04485082 +ILSVRC2012_val_00008144.JPEG:n01984695 +ILSVRC2012_val_00008145.JPEG:n03658185 +ILSVRC2012_val_00008146.JPEG:n01697457 +ILSVRC2012_val_00008147.JPEG:n09428293 +ILSVRC2012_val_00008148.JPEG:n02102480 +ILSVRC2012_val_00008149.JPEG:n04501370 +ILSVRC2012_val_00008150.JPEG:n04141975 +ILSVRC2012_val_00008151.JPEG:n01614925 +ILSVRC2012_val_00008152.JPEG:n02089078 +ILSVRC2012_val_00008153.JPEG:n03935335 +ILSVRC2012_val_00008154.JPEG:n02486410 +ILSVRC2012_val_00008155.JPEG:n01843065 +ILSVRC2012_val_00008156.JPEG:n01984695 +ILSVRC2012_val_00008157.JPEG:n02363005 +ILSVRC2012_val_00008158.JPEG:n04536866 +ILSVRC2012_val_00008159.JPEG:n04141076 +ILSVRC2012_val_00008160.JPEG:n01950731 +ILSVRC2012_val_00008161.JPEG:n03445777 +ILSVRC2012_val_00008162.JPEG:n02102040 +ILSVRC2012_val_00008163.JPEG:n07715103 +ILSVRC2012_val_00008164.JPEG:n09256479 +ILSVRC2012_val_00008165.JPEG:n03781244 +ILSVRC2012_val_00008166.JPEG:n02090379 +ILSVRC2012_val_00008167.JPEG:n02129165 +ILSVRC2012_val_00008168.JPEG:n04532670 +ILSVRC2012_val_00008169.JPEG:n02939185 +ILSVRC2012_val_00008170.JPEG:n04259630 +ILSVRC2012_val_00008171.JPEG:n03788365 +ILSVRC2012_val_00008172.JPEG:n03461385 +ILSVRC2012_val_00008173.JPEG:n04606251 +ILSVRC2012_val_00008174.JPEG:n04428191 +ILSVRC2012_val_00008175.JPEG:n02488702 +ILSVRC2012_val_00008176.JPEG:n01518878 +ILSVRC2012_val_00008177.JPEG:n02107142 +ILSVRC2012_val_00008178.JPEG:n01622779 +ILSVRC2012_val_00008179.JPEG:n02483708 +ILSVRC2012_val_00008180.JPEG:n07753113 +ILSVRC2012_val_00008181.JPEG:n07930864 +ILSVRC2012_val_00008182.JPEG:n01984695 +ILSVRC2012_val_00008183.JPEG:n03476684 +ILSVRC2012_val_00008184.JPEG:n02655020 +ILSVRC2012_val_00008185.JPEG:n03376595 +ILSVRC2012_val_00008186.JPEG:n01806143 +ILSVRC2012_val_00008187.JPEG:n04286575 +ILSVRC2012_val_00008188.JPEG:n02490219 +ILSVRC2012_val_00008189.JPEG:n02640242 +ILSVRC2012_val_00008190.JPEG:n04141975 +ILSVRC2012_val_00008191.JPEG:n03938244 +ILSVRC2012_val_00008192.JPEG:n02100735 +ILSVRC2012_val_00008193.JPEG:n04041544 +ILSVRC2012_val_00008194.JPEG:n02108915 +ILSVRC2012_val_00008195.JPEG:n03769881 +ILSVRC2012_val_00008196.JPEG:n02108551 +ILSVRC2012_val_00008197.JPEG:n02110185 +ILSVRC2012_val_00008198.JPEG:n02086646 +ILSVRC2012_val_00008199.JPEG:n03388043 +ILSVRC2012_val_00008200.JPEG:n07697313 +ILSVRC2012_val_00008201.JPEG:n02098105 +ILSVRC2012_val_00008202.JPEG:n04597913 +ILSVRC2012_val_00008203.JPEG:n04090263 +ILSVRC2012_val_00008204.JPEG:n02492660 +ILSVRC2012_val_00008205.JPEG:n02795169 +ILSVRC2012_val_00008206.JPEG:n02086240 +ILSVRC2012_val_00008207.JPEG:n02097130 +ILSVRC2012_val_00008208.JPEG:n02346627 +ILSVRC2012_val_00008209.JPEG:n01622779 +ILSVRC2012_val_00008210.JPEG:n01978287 +ILSVRC2012_val_00008211.JPEG:n01924916 +ILSVRC2012_val_00008212.JPEG:n02655020 +ILSVRC2012_val_00008213.JPEG:n02787622 +ILSVRC2012_val_00008214.JPEG:n02108551 +ILSVRC2012_val_00008215.JPEG:n03717622 +ILSVRC2012_val_00008216.JPEG:n07697313 +ILSVRC2012_val_00008217.JPEG:n02105505 +ILSVRC2012_val_00008218.JPEG:n07753113 +ILSVRC2012_val_00008219.JPEG:n04204347 +ILSVRC2012_val_00008220.JPEG:n02909870 +ILSVRC2012_val_00008221.JPEG:n01828970 +ILSVRC2012_val_00008222.JPEG:n02018795 +ILSVRC2012_val_00008223.JPEG:n07836838 +ILSVRC2012_val_00008224.JPEG:n01775062 +ILSVRC2012_val_00008225.JPEG:n07716358 +ILSVRC2012_val_00008226.JPEG:n01675722 +ILSVRC2012_val_00008227.JPEG:n02807133 +ILSVRC2012_val_00008228.JPEG:n02493793 +ILSVRC2012_val_00008229.JPEG:n02091467 +ILSVRC2012_val_00008230.JPEG:n02804414 +ILSVRC2012_val_00008231.JPEG:n12144580 +ILSVRC2012_val_00008232.JPEG:n02823428 +ILSVRC2012_val_00008233.JPEG:n09229709 +ILSVRC2012_val_00008234.JPEG:n03379051 +ILSVRC2012_val_00008235.JPEG:n02791270 +ILSVRC2012_val_00008236.JPEG:n01828970 +ILSVRC2012_val_00008237.JPEG:n03832673 +ILSVRC2012_val_00008238.JPEG:n04366367 +ILSVRC2012_val_00008239.JPEG:n03877845 +ILSVRC2012_val_00008240.JPEG:n03372029 +ILSVRC2012_val_00008241.JPEG:n03961711 +ILSVRC2012_val_00008242.JPEG:n03916031 +ILSVRC2012_val_00008243.JPEG:n03788365 +ILSVRC2012_val_00008244.JPEG:n04265275 +ILSVRC2012_val_00008245.JPEG:n01806143 +ILSVRC2012_val_00008246.JPEG:n04008634 +ILSVRC2012_val_00008247.JPEG:n02794156 +ILSVRC2012_val_00008248.JPEG:n03777754 +ILSVRC2012_val_00008249.JPEG:n01630670 +ILSVRC2012_val_00008250.JPEG:n07860988 +ILSVRC2012_val_00008251.JPEG:n04239074 +ILSVRC2012_val_00008252.JPEG:n04270147 +ILSVRC2012_val_00008253.JPEG:n03761084 +ILSVRC2012_val_00008254.JPEG:n04270147 +ILSVRC2012_val_00008255.JPEG:n04487081 +ILSVRC2012_val_00008256.JPEG:n02481823 +ILSVRC2012_val_00008257.JPEG:n02395406 +ILSVRC2012_val_00008258.JPEG:n02093859 +ILSVRC2012_val_00008259.JPEG:n03991062 +ILSVRC2012_val_00008260.JPEG:n04264628 +ILSVRC2012_val_00008261.JPEG:n04258138 +ILSVRC2012_val_00008262.JPEG:n06359193 +ILSVRC2012_val_00008263.JPEG:n02074367 +ILSVRC2012_val_00008264.JPEG:n07614500 +ILSVRC2012_val_00008265.JPEG:n02865351 +ILSVRC2012_val_00008266.JPEG:n07718747 +ILSVRC2012_val_00008267.JPEG:n04074963 +ILSVRC2012_val_00008268.JPEG:n04482393 +ILSVRC2012_val_00008269.JPEG:n03347037 +ILSVRC2012_val_00008270.JPEG:n02110063 +ILSVRC2012_val_00008271.JPEG:n07836838 +ILSVRC2012_val_00008272.JPEG:n02090379 +ILSVRC2012_val_00008273.JPEG:n03595614 +ILSVRC2012_val_00008274.JPEG:n03482405 +ILSVRC2012_val_00008275.JPEG:n13052670 +ILSVRC2012_val_00008276.JPEG:n04023962 +ILSVRC2012_val_00008277.JPEG:n03991062 +ILSVRC2012_val_00008278.JPEG:n04548280 +ILSVRC2012_val_00008279.JPEG:n02056570 +ILSVRC2012_val_00008280.JPEG:n02794156 +ILSVRC2012_val_00008281.JPEG:n13133613 +ILSVRC2012_val_00008282.JPEG:n02100877 +ILSVRC2012_val_00008283.JPEG:n03272010 +ILSVRC2012_val_00008284.JPEG:n02107683 +ILSVRC2012_val_00008285.JPEG:n04149813 +ILSVRC2012_val_00008286.JPEG:n04152593 +ILSVRC2012_val_00008287.JPEG:n02002556 +ILSVRC2012_val_00008288.JPEG:n03954731 +ILSVRC2012_val_00008289.JPEG:n01968897 +ILSVRC2012_val_00008290.JPEG:n03388043 +ILSVRC2012_val_00008291.JPEG:n03764736 +ILSVRC2012_val_00008292.JPEG:n02690373 +ILSVRC2012_val_00008293.JPEG:n02966193 +ILSVRC2012_val_00008294.JPEG:n01518878 +ILSVRC2012_val_00008295.JPEG:n02128385 +ILSVRC2012_val_00008296.JPEG:n03197337 +ILSVRC2012_val_00008297.JPEG:n02092002 +ILSVRC2012_val_00008298.JPEG:n03110669 +ILSVRC2012_val_00008299.JPEG:n03478589 +ILSVRC2012_val_00008300.JPEG:n02457408 +ILSVRC2012_val_00008301.JPEG:n02870880 +ILSVRC2012_val_00008302.JPEG:n02011460 +ILSVRC2012_val_00008303.JPEG:n02093428 +ILSVRC2012_val_00008304.JPEG:n03063689 +ILSVRC2012_val_00008305.JPEG:n03337140 +ILSVRC2012_val_00008306.JPEG:n04356056 +ILSVRC2012_val_00008307.JPEG:n02963159 +ILSVRC2012_val_00008308.JPEG:n04435653 +ILSVRC2012_val_00008309.JPEG:n03871628 +ILSVRC2012_val_00008310.JPEG:n02110627 +ILSVRC2012_val_00008311.JPEG:n02088238 +ILSVRC2012_val_00008312.JPEG:n03160309 +ILSVRC2012_val_00008313.JPEG:n03983396 +ILSVRC2012_val_00008314.JPEG:n02992529 +ILSVRC2012_val_00008315.JPEG:n03843555 +ILSVRC2012_val_00008316.JPEG:n01773549 +ILSVRC2012_val_00008317.JPEG:n02389026 +ILSVRC2012_val_00008318.JPEG:n09468604 +ILSVRC2012_val_00008319.JPEG:n04505470 +ILSVRC2012_val_00008320.JPEG:n02109961 +ILSVRC2012_val_00008321.JPEG:n02794156 +ILSVRC2012_val_00008322.JPEG:n03854065 +ILSVRC2012_val_00008323.JPEG:n04355338 +ILSVRC2012_val_00008324.JPEG:n02094433 +ILSVRC2012_val_00008325.JPEG:n13133613 +ILSVRC2012_val_00008326.JPEG:n03272010 +ILSVRC2012_val_00008327.JPEG:n01667778 +ILSVRC2012_val_00008328.JPEG:n03494278 +ILSVRC2012_val_00008329.JPEG:n12768682 +ILSVRC2012_val_00008330.JPEG:n02481823 +ILSVRC2012_val_00008331.JPEG:n03085013 +ILSVRC2012_val_00008332.JPEG:n03179701 +ILSVRC2012_val_00008333.JPEG:n01667778 +ILSVRC2012_val_00008334.JPEG:n02102040 +ILSVRC2012_val_00008335.JPEG:n02112706 +ILSVRC2012_val_00008336.JPEG:n02951585 +ILSVRC2012_val_00008337.JPEG:n02108089 +ILSVRC2012_val_00008338.JPEG:n02099601 +ILSVRC2012_val_00008339.JPEG:n07860988 +ILSVRC2012_val_00008340.JPEG:n04033995 +ILSVRC2012_val_00008341.JPEG:n03388183 +ILSVRC2012_val_00008342.JPEG:n02127052 +ILSVRC2012_val_00008343.JPEG:n02107142 +ILSVRC2012_val_00008344.JPEG:n03814639 +ILSVRC2012_val_00008345.JPEG:n04004767 +ILSVRC2012_val_00008346.JPEG:n02099712 +ILSVRC2012_val_00008347.JPEG:n01582220 +ILSVRC2012_val_00008348.JPEG:n02102177 +ILSVRC2012_val_00008349.JPEG:n02100735 +ILSVRC2012_val_00008350.JPEG:n03958227 +ILSVRC2012_val_00008351.JPEG:n02481823 +ILSVRC2012_val_00008352.JPEG:n01773549 +ILSVRC2012_val_00008353.JPEG:n03131574 +ILSVRC2012_val_00008354.JPEG:n04540053 +ILSVRC2012_val_00008355.JPEG:n03424325 +ILSVRC2012_val_00008356.JPEG:n03871628 +ILSVRC2012_val_00008357.JPEG:n02116738 +ILSVRC2012_val_00008358.JPEG:n09229709 +ILSVRC2012_val_00008359.JPEG:n02797295 +ILSVRC2012_val_00008360.JPEG:n02704792 +ILSVRC2012_val_00008361.JPEG:n02825657 +ILSVRC2012_val_00008362.JPEG:n02115913 +ILSVRC2012_val_00008363.JPEG:n03888605 +ILSVRC2012_val_00008364.JPEG:n02009229 +ILSVRC2012_val_00008365.JPEG:n03063689 +ILSVRC2012_val_00008366.JPEG:n07734744 +ILSVRC2012_val_00008367.JPEG:n02669723 +ILSVRC2012_val_00008368.JPEG:n02101556 +ILSVRC2012_val_00008369.JPEG:n03045698 +ILSVRC2012_val_00008370.JPEG:n04532106 +ILSVRC2012_val_00008371.JPEG:n03961711 +ILSVRC2012_val_00008372.JPEG:n04372370 +ILSVRC2012_val_00008373.JPEG:n02655020 +ILSVRC2012_val_00008374.JPEG:n02094433 +ILSVRC2012_val_00008375.JPEG:n02088466 +ILSVRC2012_val_00008376.JPEG:n04005630 +ILSVRC2012_val_00008377.JPEG:n12144580 +ILSVRC2012_val_00008378.JPEG:n02892767 +ILSVRC2012_val_00008379.JPEG:n02091244 +ILSVRC2012_val_00008380.JPEG:n03110669 +ILSVRC2012_val_00008381.JPEG:n03759954 +ILSVRC2012_val_00008382.JPEG:n03594945 +ILSVRC2012_val_00008383.JPEG:n03594945 +ILSVRC2012_val_00008384.JPEG:n04462240 +ILSVRC2012_val_00008385.JPEG:n07711569 +ILSVRC2012_val_00008386.JPEG:n03259280 +ILSVRC2012_val_00008387.JPEG:n04482393 +ILSVRC2012_val_00008388.JPEG:n02018207 +ILSVRC2012_val_00008389.JPEG:n03134739 +ILSVRC2012_val_00008390.JPEG:n03832673 +ILSVRC2012_val_00008391.JPEG:n04467665 +ILSVRC2012_val_00008392.JPEG:n04285008 +ILSVRC2012_val_00008393.JPEG:n02169497 +ILSVRC2012_val_00008394.JPEG:n03796401 +ILSVRC2012_val_00008395.JPEG:n02099267 +ILSVRC2012_val_00008396.JPEG:n02909870 +ILSVRC2012_val_00008397.JPEG:n02105412 +ILSVRC2012_val_00008398.JPEG:n04265275 +ILSVRC2012_val_00008399.JPEG:n01728572 +ILSVRC2012_val_00008400.JPEG:n04336792 +ILSVRC2012_val_00008401.JPEG:n02834397 +ILSVRC2012_val_00008402.JPEG:n02804414 +ILSVRC2012_val_00008403.JPEG:n04548362 +ILSVRC2012_val_00008404.JPEG:n03109150 +ILSVRC2012_val_00008405.JPEG:n02895154 +ILSVRC2012_val_00008406.JPEG:n03929660 +ILSVRC2012_val_00008407.JPEG:n01685808 +ILSVRC2012_val_00008408.JPEG:n02111500 +ILSVRC2012_val_00008409.JPEG:n04033995 +ILSVRC2012_val_00008410.JPEG:n01768244 +ILSVRC2012_val_00008411.JPEG:n02002556 +ILSVRC2012_val_00008412.JPEG:n03887697 +ILSVRC2012_val_00008413.JPEG:n04069434 +ILSVRC2012_val_00008414.JPEG:n03594734 +ILSVRC2012_val_00008415.JPEG:n02500267 +ILSVRC2012_val_00008416.JPEG:n07714990 +ILSVRC2012_val_00008417.JPEG:n02137549 +ILSVRC2012_val_00008418.JPEG:n03014705 +ILSVRC2012_val_00008419.JPEG:n02447366 +ILSVRC2012_val_00008420.JPEG:n01537544 +ILSVRC2012_val_00008421.JPEG:n07802026 +ILSVRC2012_val_00008422.JPEG:n03895866 +ILSVRC2012_val_00008423.JPEG:n04330267 +ILSVRC2012_val_00008424.JPEG:n03602883 +ILSVRC2012_val_00008425.JPEG:n02795169 +ILSVRC2012_val_00008426.JPEG:n04153751 +ILSVRC2012_val_00008427.JPEG:n03782006 +ILSVRC2012_val_00008428.JPEG:n02489166 +ILSVRC2012_val_00008429.JPEG:n03447721 +ILSVRC2012_val_00008430.JPEG:n03417042 +ILSVRC2012_val_00008431.JPEG:n04550184 +ILSVRC2012_val_00008432.JPEG:n02500267 +ILSVRC2012_val_00008433.JPEG:n02112706 +ILSVRC2012_val_00008434.JPEG:n03347037 +ILSVRC2012_val_00008435.JPEG:n02088364 +ILSVRC2012_val_00008436.JPEG:n02640242 +ILSVRC2012_val_00008437.JPEG:n03983396 +ILSVRC2012_val_00008438.JPEG:n02817516 +ILSVRC2012_val_00008439.JPEG:n01695060 +ILSVRC2012_val_00008440.JPEG:n13133613 +ILSVRC2012_val_00008441.JPEG:n02095314 +ILSVRC2012_val_00008442.JPEG:n03887697 +ILSVRC2012_val_00008443.JPEG:n02892767 +ILSVRC2012_val_00008444.JPEG:n07697313 +ILSVRC2012_val_00008445.JPEG:n11939491 +ILSVRC2012_val_00008446.JPEG:n04332243 +ILSVRC2012_val_00008447.JPEG:n02667093 +ILSVRC2012_val_00008448.JPEG:n02643566 +ILSVRC2012_val_00008449.JPEG:n02493509 +ILSVRC2012_val_00008450.JPEG:n04251144 +ILSVRC2012_val_00008451.JPEG:n02730930 +ILSVRC2012_val_00008452.JPEG:n04118776 +ILSVRC2012_val_00008453.JPEG:n02097209 +ILSVRC2012_val_00008454.JPEG:n04335435 +ILSVRC2012_val_00008455.JPEG:n03016953 +ILSVRC2012_val_00008456.JPEG:n03691459 +ILSVRC2012_val_00008457.JPEG:n04037443 +ILSVRC2012_val_00008458.JPEG:n02100583 +ILSVRC2012_val_00008459.JPEG:n02104029 +ILSVRC2012_val_00008460.JPEG:n02088466 +ILSVRC2012_val_00008461.JPEG:n09193705 +ILSVRC2012_val_00008462.JPEG:n03495258 +ILSVRC2012_val_00008463.JPEG:n02095314 +ILSVRC2012_val_00008464.JPEG:n03355925 +ILSVRC2012_val_00008465.JPEG:n07613480 +ILSVRC2012_val_00008466.JPEG:n02971356 +ILSVRC2012_val_00008467.JPEG:n04153751 +ILSVRC2012_val_00008468.JPEG:n01945685 +ILSVRC2012_val_00008469.JPEG:n01697457 +ILSVRC2012_val_00008470.JPEG:n04532106 +ILSVRC2012_val_00008471.JPEG:n02895154 +ILSVRC2012_val_00008472.JPEG:n04548362 +ILSVRC2012_val_00008473.JPEG:n04485082 +ILSVRC2012_val_00008474.JPEG:n02002724 +ILSVRC2012_val_00008475.JPEG:n02999410 +ILSVRC2012_val_00008476.JPEG:n03976467 +ILSVRC2012_val_00008477.JPEG:n02951358 +ILSVRC2012_val_00008478.JPEG:n03874293 +ILSVRC2012_val_00008479.JPEG:n02442845 +ILSVRC2012_val_00008480.JPEG:n04229816 +ILSVRC2012_val_00008481.JPEG:n01614925 +ILSVRC2012_val_00008482.JPEG:n02769748 +ILSVRC2012_val_00008483.JPEG:n04461696 +ILSVRC2012_val_00008484.JPEG:n02486410 +ILSVRC2012_val_00008485.JPEG:n03916031 +ILSVRC2012_val_00008486.JPEG:n04562935 +ILSVRC2012_val_00008487.JPEG:n02098413 +ILSVRC2012_val_00008488.JPEG:n02097474 +ILSVRC2012_val_00008489.JPEG:n03584829 +ILSVRC2012_val_00008490.JPEG:n02606052 +ILSVRC2012_val_00008491.JPEG:n02123394 +ILSVRC2012_val_00008492.JPEG:n03871628 +ILSVRC2012_val_00008493.JPEG:n04311004 +ILSVRC2012_val_00008494.JPEG:n02865351 +ILSVRC2012_val_00008495.JPEG:n01601694 +ILSVRC2012_val_00008496.JPEG:n02111129 +ILSVRC2012_val_00008497.JPEG:n04509417 +ILSVRC2012_val_00008498.JPEG:n01882714 +ILSVRC2012_val_00008499.JPEG:n03908714 +ILSVRC2012_val_00008500.JPEG:n02102973 +ILSVRC2012_val_00008501.JPEG:n03983396 +ILSVRC2012_val_00008502.JPEG:n02093859 +ILSVRC2012_val_00008503.JPEG:n03775071 +ILSVRC2012_val_00008504.JPEG:n02667093 +ILSVRC2012_val_00008505.JPEG:n02906734 +ILSVRC2012_val_00008506.JPEG:n07873807 +ILSVRC2012_val_00008507.JPEG:n04277352 +ILSVRC2012_val_00008508.JPEG:n04153751 +ILSVRC2012_val_00008509.JPEG:n01675722 +ILSVRC2012_val_00008510.JPEG:n01601694 +ILSVRC2012_val_00008511.JPEG:n04263257 +ILSVRC2012_val_00008512.JPEG:n01582220 +ILSVRC2012_val_00008513.JPEG:n03000134 +ILSVRC2012_val_00008514.JPEG:n04263257 +ILSVRC2012_val_00008515.JPEG:n04286575 +ILSVRC2012_val_00008516.JPEG:n06359193 +ILSVRC2012_val_00008517.JPEG:n02445715 +ILSVRC2012_val_00008518.JPEG:n03179701 +ILSVRC2012_val_00008519.JPEG:n04275548 +ILSVRC2012_val_00008520.JPEG:n02444819 +ILSVRC2012_val_00008521.JPEG:n02002724 +ILSVRC2012_val_00008522.JPEG:n03124170 +ILSVRC2012_val_00008523.JPEG:n02018795 +ILSVRC2012_val_00008524.JPEG:n02776631 +ILSVRC2012_val_00008525.JPEG:n12144580 +ILSVRC2012_val_00008526.JPEG:n03041632 +ILSVRC2012_val_00008527.JPEG:n02101556 +ILSVRC2012_val_00008528.JPEG:n04435653 +ILSVRC2012_val_00008529.JPEG:n04254120 +ILSVRC2012_val_00008530.JPEG:n04505470 +ILSVRC2012_val_00008531.JPEG:n03297495 +ILSVRC2012_val_00008532.JPEG:n02093256 +ILSVRC2012_val_00008533.JPEG:n03529860 +ILSVRC2012_val_00008534.JPEG:n01734418 +ILSVRC2012_val_00008535.JPEG:n04462240 +ILSVRC2012_val_00008536.JPEG:n02089867 +ILSVRC2012_val_00008537.JPEG:n03259280 +ILSVRC2012_val_00008538.JPEG:n03804744 +ILSVRC2012_val_00008539.JPEG:n02484975 +ILSVRC2012_val_00008540.JPEG:n03372029 +ILSVRC2012_val_00008541.JPEG:n02992529 +ILSVRC2012_val_00008542.JPEG:n01629819 +ILSVRC2012_val_00008543.JPEG:n03814639 +ILSVRC2012_val_00008544.JPEG:n04004767 +ILSVRC2012_val_00008545.JPEG:n02280649 +ILSVRC2012_val_00008546.JPEG:n04275548 +ILSVRC2012_val_00008547.JPEG:n04023962 +ILSVRC2012_val_00008548.JPEG:n03476684 +ILSVRC2012_val_00008549.JPEG:n01843383 +ILSVRC2012_val_00008550.JPEG:n02490219 +ILSVRC2012_val_00008551.JPEG:n03450230 +ILSVRC2012_val_00008552.JPEG:n02088238 +ILSVRC2012_val_00008553.JPEG:n02129165 +ILSVRC2012_val_00008554.JPEG:n07716906 +ILSVRC2012_val_00008555.JPEG:n02006656 +ILSVRC2012_val_00008556.JPEG:n07615774 +ILSVRC2012_val_00008557.JPEG:n04033901 +ILSVRC2012_val_00008558.JPEG:n02101388 +ILSVRC2012_val_00008559.JPEG:n02412080 +ILSVRC2012_val_00008560.JPEG:n02871525 +ILSVRC2012_val_00008561.JPEG:n01689811 +ILSVRC2012_val_00008562.JPEG:n02447366 +ILSVRC2012_val_00008563.JPEG:n02951585 +ILSVRC2012_val_00008564.JPEG:n03325584 +ILSVRC2012_val_00008565.JPEG:n04238763 +ILSVRC2012_val_00008566.JPEG:n01817953 +ILSVRC2012_val_00008567.JPEG:n07753275 +ILSVRC2012_val_00008568.JPEG:n03803284 +ILSVRC2012_val_00008569.JPEG:n03724870 +ILSVRC2012_val_00008570.JPEG:n01694178 +ILSVRC2012_val_00008571.JPEG:n04613696 +ILSVRC2012_val_00008572.JPEG:n03961711 +ILSVRC2012_val_00008573.JPEG:n04553703 +ILSVRC2012_val_00008574.JPEG:n04493381 +ILSVRC2012_val_00008575.JPEG:n04507155 +ILSVRC2012_val_00008576.JPEG:n03388183 +ILSVRC2012_val_00008577.JPEG:n04483307 +ILSVRC2012_val_00008578.JPEG:n02840245 +ILSVRC2012_val_00008579.JPEG:n01739381 +ILSVRC2012_val_00008580.JPEG:n03837869 +ILSVRC2012_val_00008581.JPEG:n03980874 +ILSVRC2012_val_00008582.JPEG:n02093647 +ILSVRC2012_val_00008583.JPEG:n02992529 +ILSVRC2012_val_00008584.JPEG:n03983396 +ILSVRC2012_val_00008585.JPEG:n02110958 +ILSVRC2012_val_00008586.JPEG:n01688243 +ILSVRC2012_val_00008587.JPEG:n02100236 +ILSVRC2012_val_00008588.JPEG:n01873310 +ILSVRC2012_val_00008589.JPEG:n04525038 +ILSVRC2012_val_00008590.JPEG:n03496892 +ILSVRC2012_val_00008591.JPEG:n04350905 +ILSVRC2012_val_00008592.JPEG:n02115913 +ILSVRC2012_val_00008593.JPEG:n01824575 +ILSVRC2012_val_00008594.JPEG:n04443257 +ILSVRC2012_val_00008595.JPEG:n01729322 +ILSVRC2012_val_00008596.JPEG:n03197337 +ILSVRC2012_val_00008597.JPEG:n09421951 +ILSVRC2012_val_00008598.JPEG:n07614500 +ILSVRC2012_val_00008599.JPEG:n03445777 +ILSVRC2012_val_00008600.JPEG:n03680355 +ILSVRC2012_val_00008601.JPEG:n04579145 +ILSVRC2012_val_00008602.JPEG:n03345487 +ILSVRC2012_val_00008603.JPEG:n03062245 +ILSVRC2012_val_00008604.JPEG:n02655020 +ILSVRC2012_val_00008605.JPEG:n02769748 +ILSVRC2012_val_00008606.JPEG:n03930630 +ILSVRC2012_val_00008607.JPEG:n03956157 +ILSVRC2012_val_00008608.JPEG:n04332243 +ILSVRC2012_val_00008609.JPEG:n03690938 +ILSVRC2012_val_00008610.JPEG:n04153751 +ILSVRC2012_val_00008611.JPEG:n04456115 +ILSVRC2012_val_00008612.JPEG:n02883205 +ILSVRC2012_val_00008613.JPEG:n01631663 +ILSVRC2012_val_00008614.JPEG:n02841315 +ILSVRC2012_val_00008615.JPEG:n02480495 +ILSVRC2012_val_00008616.JPEG:n02396427 +ILSVRC2012_val_00008617.JPEG:n04357314 +ILSVRC2012_val_00008618.JPEG:n01695060 +ILSVRC2012_val_00008619.JPEG:n02101556 +ILSVRC2012_val_00008620.JPEG:n03947888 +ILSVRC2012_val_00008621.JPEG:n04367480 +ILSVRC2012_val_00008622.JPEG:n03958227 +ILSVRC2012_val_00008623.JPEG:n01924916 +ILSVRC2012_val_00008624.JPEG:n02111129 +ILSVRC2012_val_00008625.JPEG:n02939185 +ILSVRC2012_val_00008626.JPEG:n01829413 +ILSVRC2012_val_00008627.JPEG:n02108915 +ILSVRC2012_val_00008628.JPEG:n03388183 +ILSVRC2012_val_00008629.JPEG:n02410509 +ILSVRC2012_val_00008630.JPEG:n04273569 +ILSVRC2012_val_00008631.JPEG:n02119789 +ILSVRC2012_val_00008632.JPEG:n04505470 +ILSVRC2012_val_00008633.JPEG:n02094258 +ILSVRC2012_val_00008634.JPEG:n02231487 +ILSVRC2012_val_00008635.JPEG:n02916936 +ILSVRC2012_val_00008636.JPEG:n02441942 +ILSVRC2012_val_00008637.JPEG:n04039381 +ILSVRC2012_val_00008638.JPEG:n02883205 +ILSVRC2012_val_00008639.JPEG:n02098413 +ILSVRC2012_val_00008640.JPEG:n01496331 +ILSVRC2012_val_00008641.JPEG:n03534580 +ILSVRC2012_val_00008642.JPEG:n07714990 +ILSVRC2012_val_00008643.JPEG:n04286575 +ILSVRC2012_val_00008644.JPEG:n03000247 +ILSVRC2012_val_00008645.JPEG:n03691459 +ILSVRC2012_val_00008646.JPEG:n03376595 +ILSVRC2012_val_00008647.JPEG:n01729322 +ILSVRC2012_val_00008648.JPEG:n12144580 +ILSVRC2012_val_00008649.JPEG:n04192698 +ILSVRC2012_val_00008650.JPEG:n03998194 +ILSVRC2012_val_00008651.JPEG:n02979186 +ILSVRC2012_val_00008652.JPEG:n02102973 +ILSVRC2012_val_00008653.JPEG:n02110627 +ILSVRC2012_val_00008654.JPEG:n01728572 +ILSVRC2012_val_00008655.JPEG:n03272010 +ILSVRC2012_val_00008656.JPEG:n03786901 +ILSVRC2012_val_00008657.JPEG:n04033901 +ILSVRC2012_val_00008658.JPEG:n02097047 +ILSVRC2012_val_00008659.JPEG:n03947888 +ILSVRC2012_val_00008660.JPEG:n07873807 +ILSVRC2012_val_00008661.JPEG:n02097047 +ILSVRC2012_val_00008662.JPEG:n07754684 +ILSVRC2012_val_00008663.JPEG:n02276258 +ILSVRC2012_val_00008664.JPEG:n02104365 +ILSVRC2012_val_00008665.JPEG:n01734418 +ILSVRC2012_val_00008666.JPEG:n03976467 +ILSVRC2012_val_00008667.JPEG:n02825657 +ILSVRC2012_val_00008668.JPEG:n01694178 +ILSVRC2012_val_00008669.JPEG:n01682714 +ILSVRC2012_val_00008670.JPEG:n02747177 +ILSVRC2012_val_00008671.JPEG:n03710193 +ILSVRC2012_val_00008672.JPEG:n09288635 +ILSVRC2012_val_00008673.JPEG:n02510455 +ILSVRC2012_val_00008674.JPEG:n02319095 +ILSVRC2012_val_00008675.JPEG:n02088364 +ILSVRC2012_val_00008676.JPEG:n02129604 +ILSVRC2012_val_00008677.JPEG:n04326547 +ILSVRC2012_val_00008678.JPEG:n03871628 +ILSVRC2012_val_00008679.JPEG:n02096177 +ILSVRC2012_val_00008680.JPEG:n09246464 +ILSVRC2012_val_00008681.JPEG:n03127925 +ILSVRC2012_val_00008682.JPEG:n02488702 +ILSVRC2012_val_00008683.JPEG:n06785654 +ILSVRC2012_val_00008684.JPEG:n02066245 +ILSVRC2012_val_00008685.JPEG:n12998815 +ILSVRC2012_val_00008686.JPEG:n01632777 +ILSVRC2012_val_00008687.JPEG:n02091244 +ILSVRC2012_val_00008688.JPEG:n01742172 +ILSVRC2012_val_00008689.JPEG:n03908618 +ILSVRC2012_val_00008690.JPEG:n04536866 +ILSVRC2012_val_00008691.JPEG:n03841143 +ILSVRC2012_val_00008692.JPEG:n01917289 +ILSVRC2012_val_00008693.JPEG:n02276258 +ILSVRC2012_val_00008694.JPEG:n03457902 +ILSVRC2012_val_00008695.JPEG:n04041544 +ILSVRC2012_val_00008696.JPEG:n03259280 +ILSVRC2012_val_00008697.JPEG:n02236044 +ILSVRC2012_val_00008698.JPEG:n02090379 +ILSVRC2012_val_00008699.JPEG:n04127249 +ILSVRC2012_val_00008700.JPEG:n03873416 +ILSVRC2012_val_00008701.JPEG:n02415577 +ILSVRC2012_val_00008702.JPEG:n03590841 +ILSVRC2012_val_00008703.JPEG:n02094258 +ILSVRC2012_val_00008704.JPEG:n03884397 +ILSVRC2012_val_00008705.JPEG:n01978287 +ILSVRC2012_val_00008706.JPEG:n02172182 +ILSVRC2012_val_00008707.JPEG:n01990800 +ILSVRC2012_val_00008708.JPEG:n04476259 +ILSVRC2012_val_00008709.JPEG:n03871628 +ILSVRC2012_val_00008710.JPEG:n03584829 +ILSVRC2012_val_00008711.JPEG:n04118776 +ILSVRC2012_val_00008712.JPEG:n02509815 +ILSVRC2012_val_00008713.JPEG:n02102480 +ILSVRC2012_val_00008714.JPEG:n01729977 +ILSVRC2012_val_00008715.JPEG:n02776631 +ILSVRC2012_val_00008716.JPEG:n03125729 +ILSVRC2012_val_00008717.JPEG:n02948072 +ILSVRC2012_val_00008718.JPEG:n01774384 +ILSVRC2012_val_00008719.JPEG:n01695060 +ILSVRC2012_val_00008720.JPEG:n07734744 +ILSVRC2012_val_00008721.JPEG:n01990800 +ILSVRC2012_val_00008722.JPEG:n02445715 +ILSVRC2012_val_00008723.JPEG:n03017168 +ILSVRC2012_val_00008724.JPEG:n02606052 +ILSVRC2012_val_00008725.JPEG:n04612504 +ILSVRC2012_val_00008726.JPEG:n02119789 +ILSVRC2012_val_00008727.JPEG:n02113978 +ILSVRC2012_val_00008728.JPEG:n03706229 +ILSVRC2012_val_00008729.JPEG:n02115913 +ILSVRC2012_val_00008730.JPEG:n02655020 +ILSVRC2012_val_00008731.JPEG:n02640242 +ILSVRC2012_val_00008732.JPEG:n03478589 +ILSVRC2012_val_00008733.JPEG:n03891251 +ILSVRC2012_val_00008734.JPEG:n02892201 +ILSVRC2012_val_00008735.JPEG:n02676566 +ILSVRC2012_val_00008736.JPEG:n01877812 +ILSVRC2012_val_00008737.JPEG:n02037110 +ILSVRC2012_val_00008738.JPEG:n07745940 +ILSVRC2012_val_00008739.JPEG:n02090721 +ILSVRC2012_val_00008740.JPEG:n04548280 +ILSVRC2012_val_00008741.JPEG:n02971356 +ILSVRC2012_val_00008742.JPEG:n03042490 +ILSVRC2012_val_00008743.JPEG:n02865351 +ILSVRC2012_val_00008744.JPEG:n04310018 +ILSVRC2012_val_00008745.JPEG:n07802026 +ILSVRC2012_val_00008746.JPEG:n01843065 +ILSVRC2012_val_00008747.JPEG:n01944390 +ILSVRC2012_val_00008748.JPEG:n03443371 +ILSVRC2012_val_00008749.JPEG:n01496331 +ILSVRC2012_val_00008750.JPEG:n13044778 +ILSVRC2012_val_00008751.JPEG:n03196217 +ILSVRC2012_val_00008752.JPEG:n02111889 +ILSVRC2012_val_00008753.JPEG:n09288635 +ILSVRC2012_val_00008754.JPEG:n03777568 +ILSVRC2012_val_00008755.JPEG:n03970156 +ILSVRC2012_val_00008756.JPEG:n02027492 +ILSVRC2012_val_00008757.JPEG:n09332890 +ILSVRC2012_val_00008758.JPEG:n04326547 +ILSVRC2012_val_00008759.JPEG:n04458633 +ILSVRC2012_val_00008760.JPEG:n02093428 +ILSVRC2012_val_00008761.JPEG:n03992509 +ILSVRC2012_val_00008762.JPEG:n03908618 +ILSVRC2012_val_00008763.JPEG:n03290653 +ILSVRC2012_val_00008764.JPEG:n04311004 +ILSVRC2012_val_00008765.JPEG:n03764736 +ILSVRC2012_val_00008766.JPEG:n04465501 +ILSVRC2012_val_00008767.JPEG:n03345487 +ILSVRC2012_val_00008768.JPEG:n04099969 +ILSVRC2012_val_00008769.JPEG:n02843684 +ILSVRC2012_val_00008770.JPEG:n02361337 +ILSVRC2012_val_00008771.JPEG:n02066245 +ILSVRC2012_val_00008772.JPEG:n02099601 +ILSVRC2012_val_00008773.JPEG:n03259280 +ILSVRC2012_val_00008774.JPEG:n02105641 +ILSVRC2012_val_00008775.JPEG:n01755581 +ILSVRC2012_val_00008776.JPEG:n03937543 +ILSVRC2012_val_00008777.JPEG:n03249569 +ILSVRC2012_val_00008778.JPEG:n02124075 +ILSVRC2012_val_00008779.JPEG:n03761084 +ILSVRC2012_val_00008780.JPEG:n02834397 +ILSVRC2012_val_00008781.JPEG:n03891251 +ILSVRC2012_val_00008782.JPEG:n07753275 +ILSVRC2012_val_00008783.JPEG:n04389033 +ILSVRC2012_val_00008784.JPEG:n03599486 +ILSVRC2012_val_00008785.JPEG:n04392985 +ILSVRC2012_val_00008786.JPEG:n01582220 +ILSVRC2012_val_00008787.JPEG:n03642806 +ILSVRC2012_val_00008788.JPEG:n01749939 +ILSVRC2012_val_00008789.JPEG:n01944390 +ILSVRC2012_val_00008790.JPEG:n03146219 +ILSVRC2012_val_00008791.JPEG:n09428293 +ILSVRC2012_val_00008792.JPEG:n02112350 +ILSVRC2012_val_00008793.JPEG:n03249569 +ILSVRC2012_val_00008794.JPEG:n02085936 +ILSVRC2012_val_00008795.JPEG:n03240683 +ILSVRC2012_val_00008796.JPEG:n04597913 +ILSVRC2012_val_00008797.JPEG:n03249569 +ILSVRC2012_val_00008798.JPEG:n02256656 +ILSVRC2012_val_00008799.JPEG:n07248320 +ILSVRC2012_val_00008800.JPEG:n04376876 +ILSVRC2012_val_00008801.JPEG:n03089624 +ILSVRC2012_val_00008802.JPEG:n04118538 +ILSVRC2012_val_00008803.JPEG:n02966687 +ILSVRC2012_val_00008804.JPEG:n03891332 +ILSVRC2012_val_00008805.JPEG:n01773157 +ILSVRC2012_val_00008806.JPEG:n02948072 +ILSVRC2012_val_00008807.JPEG:n01685808 +ILSVRC2012_val_00008808.JPEG:n04371430 +ILSVRC2012_val_00008809.JPEG:n02107312 +ILSVRC2012_val_00008810.JPEG:n01749939 +ILSVRC2012_val_00008811.JPEG:n02085936 +ILSVRC2012_val_00008812.JPEG:n02091831 +ILSVRC2012_val_00008813.JPEG:n02098105 +ILSVRC2012_val_00008814.JPEG:n02708093 +ILSVRC2012_val_00008815.JPEG:n02120505 +ILSVRC2012_val_00008816.JPEG:n01601694 +ILSVRC2012_val_00008817.JPEG:n06874185 +ILSVRC2012_val_00008818.JPEG:n02319095 +ILSVRC2012_val_00008819.JPEG:n01616318 +ILSVRC2012_val_00008820.JPEG:n01775062 +ILSVRC2012_val_00008821.JPEG:n13040303 +ILSVRC2012_val_00008822.JPEG:n03796401 +ILSVRC2012_val_00008823.JPEG:n04482393 +ILSVRC2012_val_00008824.JPEG:n03272562 +ILSVRC2012_val_00008825.JPEG:n03478589 +ILSVRC2012_val_00008826.JPEG:n02190166 +ILSVRC2012_val_00008827.JPEG:n02910353 +ILSVRC2012_val_00008828.JPEG:n02951358 +ILSVRC2012_val_00008829.JPEG:n01749939 +ILSVRC2012_val_00008830.JPEG:n12985857 +ILSVRC2012_val_00008831.JPEG:n04254120 +ILSVRC2012_val_00008832.JPEG:n03944341 +ILSVRC2012_val_00008833.JPEG:n03743016 +ILSVRC2012_val_00008834.JPEG:n01855672 +ILSVRC2012_val_00008835.JPEG:n04228054 +ILSVRC2012_val_00008836.JPEG:n03642806 +ILSVRC2012_val_00008837.JPEG:n03956157 +ILSVRC2012_val_00008838.JPEG:n04162706 +ILSVRC2012_val_00008839.JPEG:n02992211 +ILSVRC2012_val_00008840.JPEG:n01883070 +ILSVRC2012_val_00008841.JPEG:n03045698 +ILSVRC2012_val_00008842.JPEG:n02018207 +ILSVRC2012_val_00008843.JPEG:n01872401 +ILSVRC2012_val_00008844.JPEG:n04239074 +ILSVRC2012_val_00008845.JPEG:n07932039 +ILSVRC2012_val_00008846.JPEG:n04392985 +ILSVRC2012_val_00008847.JPEG:n02641379 +ILSVRC2012_val_00008848.JPEG:n01484850 +ILSVRC2012_val_00008849.JPEG:n01742172 +ILSVRC2012_val_00008850.JPEG:n04376876 +ILSVRC2012_val_00008851.JPEG:n04550184 +ILSVRC2012_val_00008852.JPEG:n03733805 +ILSVRC2012_val_00008853.JPEG:n04371774 +ILSVRC2012_val_00008854.JPEG:n04317175 +ILSVRC2012_val_00008855.JPEG:n03873416 +ILSVRC2012_val_00008856.JPEG:n02361337 +ILSVRC2012_val_00008857.JPEG:n02002556 +ILSVRC2012_val_00008858.JPEG:n02168699 +ILSVRC2012_val_00008859.JPEG:n02098413 +ILSVRC2012_val_00008860.JPEG:n02104365 +ILSVRC2012_val_00008861.JPEG:n03841143 +ILSVRC2012_val_00008862.JPEG:n02074367 +ILSVRC2012_val_00008863.JPEG:n04344873 +ILSVRC2012_val_00008864.JPEG:n07615774 +ILSVRC2012_val_00008865.JPEG:n04149813 +ILSVRC2012_val_00008866.JPEG:n02321529 +ILSVRC2012_val_00008867.JPEG:n12144580 +ILSVRC2012_val_00008868.JPEG:n02509815 +ILSVRC2012_val_00008869.JPEG:n03938244 +ILSVRC2012_val_00008870.JPEG:n01978455 +ILSVRC2012_val_00008871.JPEG:n03047690 +ILSVRC2012_val_00008872.JPEG:n04252077 +ILSVRC2012_val_00008873.JPEG:n02487347 +ILSVRC2012_val_00008874.JPEG:n03141823 +ILSVRC2012_val_00008875.JPEG:n02666196 +ILSVRC2012_val_00008876.JPEG:n02123045 +ILSVRC2012_val_00008877.JPEG:n02486410 +ILSVRC2012_val_00008878.JPEG:n02492660 +ILSVRC2012_val_00008879.JPEG:n03796401 +ILSVRC2012_val_00008880.JPEG:n02112350 +ILSVRC2012_val_00008881.JPEG:n07730033 +ILSVRC2012_val_00008882.JPEG:n03950228 +ILSVRC2012_val_00008883.JPEG:n04162706 +ILSVRC2012_val_00008884.JPEG:n02895154 +ILSVRC2012_val_00008885.JPEG:n02105641 +ILSVRC2012_val_00008886.JPEG:n03404251 +ILSVRC2012_val_00008887.JPEG:n02007558 +ILSVRC2012_val_00008888.JPEG:n01739381 +ILSVRC2012_val_00008889.JPEG:n02481823 +ILSVRC2012_val_00008890.JPEG:n04409515 +ILSVRC2012_val_00008891.JPEG:n02443114 +ILSVRC2012_val_00008892.JPEG:n02879718 +ILSVRC2012_val_00008893.JPEG:n03345487 +ILSVRC2012_val_00008894.JPEG:n02268853 +ILSVRC2012_val_00008895.JPEG:n12620546 +ILSVRC2012_val_00008896.JPEG:n03930313 +ILSVRC2012_val_00008897.JPEG:n04380533 +ILSVRC2012_val_00008898.JPEG:n01518878 +ILSVRC2012_val_00008899.JPEG:n04596742 +ILSVRC2012_val_00008900.JPEG:n03680355 +ILSVRC2012_val_00008901.JPEG:n02074367 +ILSVRC2012_val_00008902.JPEG:n01667778 +ILSVRC2012_val_00008903.JPEG:n03376595 +ILSVRC2012_val_00008904.JPEG:n04366367 +ILSVRC2012_val_00008905.JPEG:n02097047 +ILSVRC2012_val_00008906.JPEG:n02101006 +ILSVRC2012_val_00008907.JPEG:n01873310 +ILSVRC2012_val_00008908.JPEG:n03876231 +ILSVRC2012_val_00008909.JPEG:n04507155 +ILSVRC2012_val_00008910.JPEG:n02086910 +ILSVRC2012_val_00008911.JPEG:n04370456 +ILSVRC2012_val_00008912.JPEG:n02687172 +ILSVRC2012_val_00008913.JPEG:n03724870 +ILSVRC2012_val_00008914.JPEG:n02966193 +ILSVRC2012_val_00008915.JPEG:n02776631 +ILSVRC2012_val_00008916.JPEG:n03089624 +ILSVRC2012_val_00008917.JPEG:n04456115 +ILSVRC2012_val_00008918.JPEG:n03325584 +ILSVRC2012_val_00008919.JPEG:n01770081 +ILSVRC2012_val_00008920.JPEG:n04428191 +ILSVRC2012_val_00008921.JPEG:n01667778 +ILSVRC2012_val_00008922.JPEG:n02132136 +ILSVRC2012_val_00008923.JPEG:n02105162 +ILSVRC2012_val_00008924.JPEG:n03743016 +ILSVRC2012_val_00008925.JPEG:n04367480 +ILSVRC2012_val_00008926.JPEG:n02098105 +ILSVRC2012_val_00008927.JPEG:n03000134 +ILSVRC2012_val_00008928.JPEG:n02100236 +ILSVRC2012_val_00008929.JPEG:n02011460 +ILSVRC2012_val_00008930.JPEG:n02097047 +ILSVRC2012_val_00008931.JPEG:n02177972 +ILSVRC2012_val_00008932.JPEG:n04493381 +ILSVRC2012_val_00008933.JPEG:n03874293 +ILSVRC2012_val_00008934.JPEG:n02017213 +ILSVRC2012_val_00008935.JPEG:n03908714 +ILSVRC2012_val_00008936.JPEG:n02361337 +ILSVRC2012_val_00008937.JPEG:n02669723 +ILSVRC2012_val_00008938.JPEG:n02119022 +ILSVRC2012_val_00008939.JPEG:n02105505 +ILSVRC2012_val_00008940.JPEG:n03884397 +ILSVRC2012_val_00008941.JPEG:n02190166 +ILSVRC2012_val_00008942.JPEG:n03216828 +ILSVRC2012_val_00008943.JPEG:n02410509 +ILSVRC2012_val_00008944.JPEG:n02101556 +ILSVRC2012_val_00008945.JPEG:n02098286 +ILSVRC2012_val_00008946.JPEG:n03250847 +ILSVRC2012_val_00008947.JPEG:n02117135 +ILSVRC2012_val_00008948.JPEG:n03929660 +ILSVRC2012_val_00008949.JPEG:n04332243 +ILSVRC2012_val_00008950.JPEG:n03891332 +ILSVRC2012_val_00008951.JPEG:n02018207 +ILSVRC2012_val_00008952.JPEG:n01498041 +ILSVRC2012_val_00008953.JPEG:n03977966 +ILSVRC2012_val_00008954.JPEG:n02892767 +ILSVRC2012_val_00008955.JPEG:n03781244 +ILSVRC2012_val_00008956.JPEG:n02094433 +ILSVRC2012_val_00008957.JPEG:n02112137 +ILSVRC2012_val_00008958.JPEG:n02910353 +ILSVRC2012_val_00008959.JPEG:n03791053 +ILSVRC2012_val_00008960.JPEG:n01773157 +ILSVRC2012_val_00008961.JPEG:n03599486 +ILSVRC2012_val_00008962.JPEG:n11939491 +ILSVRC2012_val_00008963.JPEG:n01496331 +ILSVRC2012_val_00008964.JPEG:n02950826 +ILSVRC2012_val_00008965.JPEG:n09246464 +ILSVRC2012_val_00008966.JPEG:n02099429 +ILSVRC2012_val_00008967.JPEG:n02108551 +ILSVRC2012_val_00008968.JPEG:n02895154 +ILSVRC2012_val_00008969.JPEG:n09229709 +ILSVRC2012_val_00008970.JPEG:n07932039 +ILSVRC2012_val_00008971.JPEG:n03721384 +ILSVRC2012_val_00008972.JPEG:n03529860 +ILSVRC2012_val_00008973.JPEG:n02113186 +ILSVRC2012_val_00008974.JPEG:n03929660 +ILSVRC2012_val_00008975.JPEG:n02086646 +ILSVRC2012_val_00008976.JPEG:n02787622 +ILSVRC2012_val_00008977.JPEG:n02676566 +ILSVRC2012_val_00008978.JPEG:n02006656 +ILSVRC2012_val_00008979.JPEG:n02104365 +ILSVRC2012_val_00008980.JPEG:n03045698 +ILSVRC2012_val_00008981.JPEG:n03100240 +ILSVRC2012_val_00008982.JPEG:n03599486 +ILSVRC2012_val_00008983.JPEG:n03924679 +ILSVRC2012_val_00008984.JPEG:n03937543 +ILSVRC2012_val_00008985.JPEG:n02869837 +ILSVRC2012_val_00008986.JPEG:n02123394 +ILSVRC2012_val_00008987.JPEG:n01980166 +ILSVRC2012_val_00008988.JPEG:n04355933 +ILSVRC2012_val_00008989.JPEG:n03133878 +ILSVRC2012_val_00008990.JPEG:n03709823 +ILSVRC2012_val_00008991.JPEG:n06794110 +ILSVRC2012_val_00008992.JPEG:n02110341 +ILSVRC2012_val_00008993.JPEG:n01796340 +ILSVRC2012_val_00008994.JPEG:n02978881 +ILSVRC2012_val_00008995.JPEG:n03495258 +ILSVRC2012_val_00008996.JPEG:n03452741 +ILSVRC2012_val_00008997.JPEG:n02091032 +ILSVRC2012_val_00008998.JPEG:n04442312 +ILSVRC2012_val_00008999.JPEG:n04118776 +ILSVRC2012_val_00009000.JPEG:n01630670 +ILSVRC2012_val_00009001.JPEG:n03662601 +ILSVRC2012_val_00009002.JPEG:n02174001 +ILSVRC2012_val_00009003.JPEG:n04606251 +ILSVRC2012_val_00009004.JPEG:n02107142 +ILSVRC2012_val_00009005.JPEG:n03814906 +ILSVRC2012_val_00009006.JPEG:n03457902 +ILSVRC2012_val_00009007.JPEG:n02085782 +ILSVRC2012_val_00009008.JPEG:n03598930 +ILSVRC2012_val_00009009.JPEG:n02094258 +ILSVRC2012_val_00009010.JPEG:n03000247 +ILSVRC2012_val_00009011.JPEG:n02966193 +ILSVRC2012_val_00009012.JPEG:n02489166 +ILSVRC2012_val_00009013.JPEG:n04367480 +ILSVRC2012_val_00009014.JPEG:n02110063 +ILSVRC2012_val_00009015.JPEG:n07753275 +ILSVRC2012_val_00009016.JPEG:n07715103 +ILSVRC2012_val_00009017.JPEG:n04485082 +ILSVRC2012_val_00009018.JPEG:n03075370 +ILSVRC2012_val_00009019.JPEG:n02098105 +ILSVRC2012_val_00009020.JPEG:n13054560 +ILSVRC2012_val_00009021.JPEG:n02730930 +ILSVRC2012_val_00009022.JPEG:n03670208 +ILSVRC2012_val_00009023.JPEG:n02281787 +ILSVRC2012_val_00009024.JPEG:n04462240 +ILSVRC2012_val_00009025.JPEG:n02510455 +ILSVRC2012_val_00009026.JPEG:n02814860 +ILSVRC2012_val_00009027.JPEG:n04482393 +ILSVRC2012_val_00009028.JPEG:n03498962 +ILSVRC2012_val_00009029.JPEG:n09229709 +ILSVRC2012_val_00009030.JPEG:n02097130 +ILSVRC2012_val_00009031.JPEG:n04265275 +ILSVRC2012_val_00009032.JPEG:n04004767 +ILSVRC2012_val_00009033.JPEG:n02093647 +ILSVRC2012_val_00009034.JPEG:n01443537 +ILSVRC2012_val_00009035.JPEG:n01704323 +ILSVRC2012_val_00009036.JPEG:n02096437 +ILSVRC2012_val_00009037.JPEG:n03394916 +ILSVRC2012_val_00009038.JPEG:n04423845 +ILSVRC2012_val_00009039.JPEG:n02108422 +ILSVRC2012_val_00009040.JPEG:n03706229 +ILSVRC2012_val_00009041.JPEG:n02869837 +ILSVRC2012_val_00009042.JPEG:n01737021 +ILSVRC2012_val_00009043.JPEG:n03930313 +ILSVRC2012_val_00009044.JPEG:n04039381 +ILSVRC2012_val_00009045.JPEG:n02113186 +ILSVRC2012_val_00009046.JPEG:n02403003 +ILSVRC2012_val_00009047.JPEG:n02037110 +ILSVRC2012_val_00009048.JPEG:n03637318 +ILSVRC2012_val_00009049.JPEG:n02823750 +ILSVRC2012_val_00009050.JPEG:n01677366 +ILSVRC2012_val_00009051.JPEG:n02093256 +ILSVRC2012_val_00009052.JPEG:n02096294 +ILSVRC2012_val_00009053.JPEG:n06596364 +ILSVRC2012_val_00009054.JPEG:n03220513 +ILSVRC2012_val_00009055.JPEG:n02106030 +ILSVRC2012_val_00009056.JPEG:n02917067 +ILSVRC2012_val_00009057.JPEG:n02090622 +ILSVRC2012_val_00009058.JPEG:n04141076 +ILSVRC2012_val_00009059.JPEG:n01749939 +ILSVRC2012_val_00009060.JPEG:n02981792 +ILSVRC2012_val_00009061.JPEG:n02111889 +ILSVRC2012_val_00009062.JPEG:n02116738 +ILSVRC2012_val_00009063.JPEG:n09246464 +ILSVRC2012_val_00009064.JPEG:n02791124 +ILSVRC2012_val_00009065.JPEG:n02091244 +ILSVRC2012_val_00009066.JPEG:n02119022 +ILSVRC2012_val_00009067.JPEG:n02445715 +ILSVRC2012_val_00009068.JPEG:n03216828 +ILSVRC2012_val_00009069.JPEG:n03095699 +ILSVRC2012_val_00009070.JPEG:n03481172 +ILSVRC2012_val_00009071.JPEG:n04442312 +ILSVRC2012_val_00009072.JPEG:n02802426 +ILSVRC2012_val_00009073.JPEG:n09428293 +ILSVRC2012_val_00009074.JPEG:n03065424 +ILSVRC2012_val_00009075.JPEG:n02363005 +ILSVRC2012_val_00009076.JPEG:n12057211 +ILSVRC2012_val_00009077.JPEG:n02422106 +ILSVRC2012_val_00009078.JPEG:n02999410 +ILSVRC2012_val_00009079.JPEG:n03207743 +ILSVRC2012_val_00009080.JPEG:n03786901 +ILSVRC2012_val_00009081.JPEG:n02363005 +ILSVRC2012_val_00009082.JPEG:n02417914 +ILSVRC2012_val_00009083.JPEG:n01698640 +ILSVRC2012_val_00009084.JPEG:n03063599 +ILSVRC2012_val_00009085.JPEG:n04409515 +ILSVRC2012_val_00009086.JPEG:n03891251 +ILSVRC2012_val_00009087.JPEG:n03794056 +ILSVRC2012_val_00009088.JPEG:n02101388 +ILSVRC2012_val_00009089.JPEG:n04044716 +ILSVRC2012_val_00009090.JPEG:n02226429 +ILSVRC2012_val_00009091.JPEG:n01818515 +ILSVRC2012_val_00009092.JPEG:n01558993 +ILSVRC2012_val_00009093.JPEG:n02110806 +ILSVRC2012_val_00009094.JPEG:n03337140 +ILSVRC2012_val_00009095.JPEG:n03627232 +ILSVRC2012_val_00009096.JPEG:n04204238 +ILSVRC2012_val_00009097.JPEG:n07873807 +ILSVRC2012_val_00009098.JPEG:n03930630 +ILSVRC2012_val_00009099.JPEG:n04311174 +ILSVRC2012_val_00009100.JPEG:n01616318 +ILSVRC2012_val_00009101.JPEG:n04330267 +ILSVRC2012_val_00009102.JPEG:n04179913 +ILSVRC2012_val_00009103.JPEG:n04501370 +ILSVRC2012_val_00009104.JPEG:n02687172 +ILSVRC2012_val_00009105.JPEG:n02086079 +ILSVRC2012_val_00009106.JPEG:n03976467 +ILSVRC2012_val_00009107.JPEG:n03950228 +ILSVRC2012_val_00009108.JPEG:n01773797 +ILSVRC2012_val_00009109.JPEG:n03197337 +ILSVRC2012_val_00009110.JPEG:n02640242 +ILSVRC2012_val_00009111.JPEG:n01440764 +ILSVRC2012_val_00009112.JPEG:n02342885 +ILSVRC2012_val_00009113.JPEG:n02389026 +ILSVRC2012_val_00009114.JPEG:n02895154 +ILSVRC2012_val_00009115.JPEG:n02056570 +ILSVRC2012_val_00009116.JPEG:n04584207 +ILSVRC2012_val_00009117.JPEG:n03042490 +ILSVRC2012_val_00009118.JPEG:n09421951 +ILSVRC2012_val_00009119.JPEG:n01616318 +ILSVRC2012_val_00009120.JPEG:n03384352 +ILSVRC2012_val_00009121.JPEG:n07248320 +ILSVRC2012_val_00009122.JPEG:n03590841 +ILSVRC2012_val_00009123.JPEG:n03903868 +ILSVRC2012_val_00009124.JPEG:n02129165 +ILSVRC2012_val_00009125.JPEG:n02123159 +ILSVRC2012_val_00009126.JPEG:n03837869 +ILSVRC2012_val_00009127.JPEG:n03630383 +ILSVRC2012_val_00009128.JPEG:n02119789 +ILSVRC2012_val_00009129.JPEG:n07768694 +ILSVRC2012_val_00009130.JPEG:n02102973 +ILSVRC2012_val_00009131.JPEG:n03788195 +ILSVRC2012_val_00009132.JPEG:n01682714 +ILSVRC2012_val_00009133.JPEG:n02130308 +ILSVRC2012_val_00009134.JPEG:n03495258 +ILSVRC2012_val_00009135.JPEG:n03770439 +ILSVRC2012_val_00009136.JPEG:n02398521 +ILSVRC2012_val_00009137.JPEG:n02965783 +ILSVRC2012_val_00009138.JPEG:n02033041 +ILSVRC2012_val_00009139.JPEG:n02088094 +ILSVRC2012_val_00009140.JPEG:n02939185 +ILSVRC2012_val_00009141.JPEG:n01914609 +ILSVRC2012_val_00009142.JPEG:n04147183 +ILSVRC2012_val_00009143.JPEG:n03720891 +ILSVRC2012_val_00009144.JPEG:n02105641 +ILSVRC2012_val_00009145.JPEG:n01843383 +ILSVRC2012_val_00009146.JPEG:n01818515 +ILSVRC2012_val_00009147.JPEG:n02730930 +ILSVRC2012_val_00009148.JPEG:n02109961 +ILSVRC2012_val_00009149.JPEG:n04398044 +ILSVRC2012_val_00009150.JPEG:n04131690 +ILSVRC2012_val_00009151.JPEG:n01914609 +ILSVRC2012_val_00009152.JPEG:n03481172 +ILSVRC2012_val_00009153.JPEG:n04317175 +ILSVRC2012_val_00009154.JPEG:n03344393 +ILSVRC2012_val_00009155.JPEG:n04557648 +ILSVRC2012_val_00009156.JPEG:n02120505 +ILSVRC2012_val_00009157.JPEG:n02109961 +ILSVRC2012_val_00009158.JPEG:n02128385 +ILSVRC2012_val_00009159.JPEG:n02391049 +ILSVRC2012_val_00009160.JPEG:n03041632 +ILSVRC2012_val_00009161.JPEG:n09246464 +ILSVRC2012_val_00009162.JPEG:n03666591 +ILSVRC2012_val_00009163.JPEG:n02111129 +ILSVRC2012_val_00009164.JPEG:n02974003 +ILSVRC2012_val_00009165.JPEG:n02643566 +ILSVRC2012_val_00009166.JPEG:n03492542 +ILSVRC2012_val_00009167.JPEG:n02090622 +ILSVRC2012_val_00009168.JPEG:n02389026 +ILSVRC2012_val_00009169.JPEG:n01735189 +ILSVRC2012_val_00009170.JPEG:n03478589 +ILSVRC2012_val_00009171.JPEG:n03785016 +ILSVRC2012_val_00009172.JPEG:n03854065 +ILSVRC2012_val_00009173.JPEG:n03207743 +ILSVRC2012_val_00009174.JPEG:n04399382 +ILSVRC2012_val_00009175.JPEG:n02108422 +ILSVRC2012_val_00009176.JPEG:n04428191 +ILSVRC2012_val_00009177.JPEG:n07760859 +ILSVRC2012_val_00009178.JPEG:n03888605 +ILSVRC2012_val_00009179.JPEG:n02704792 +ILSVRC2012_val_00009180.JPEG:n03697007 +ILSVRC2012_val_00009181.JPEG:n03657121 +ILSVRC2012_val_00009182.JPEG:n04141975 +ILSVRC2012_val_00009183.JPEG:n04008634 +ILSVRC2012_val_00009184.JPEG:n02799071 +ILSVRC2012_val_00009185.JPEG:n02018795 +ILSVRC2012_val_00009186.JPEG:n02877765 +ILSVRC2012_val_00009187.JPEG:n07613480 +ILSVRC2012_val_00009188.JPEG:n11939491 +ILSVRC2012_val_00009189.JPEG:n02108089 +ILSVRC2012_val_00009190.JPEG:n02098413 +ILSVRC2012_val_00009191.JPEG:n01440764 +ILSVRC2012_val_00009192.JPEG:n01776313 +ILSVRC2012_val_00009193.JPEG:n03804744 +ILSVRC2012_val_00009194.JPEG:n01817953 +ILSVRC2012_val_00009195.JPEG:n02788148 +ILSVRC2012_val_00009196.JPEG:n03400231 +ILSVRC2012_val_00009197.JPEG:n03899768 +ILSVRC2012_val_00009198.JPEG:n02027492 +ILSVRC2012_val_00009199.JPEG:n02028035 +ILSVRC2012_val_00009200.JPEG:n02087394 +ILSVRC2012_val_00009201.JPEG:n04392985 +ILSVRC2012_val_00009202.JPEG:n01944390 +ILSVRC2012_val_00009203.JPEG:n04204238 +ILSVRC2012_val_00009204.JPEG:n03995372 +ILSVRC2012_val_00009205.JPEG:n02437616 +ILSVRC2012_val_00009206.JPEG:n03000684 +ILSVRC2012_val_00009207.JPEG:n03146219 +ILSVRC2012_val_00009208.JPEG:n01496331 +ILSVRC2012_val_00009209.JPEG:n02128925 +ILSVRC2012_val_00009210.JPEG:n02025239 +ILSVRC2012_val_00009211.JPEG:n03903868 +ILSVRC2012_val_00009212.JPEG:n06596364 +ILSVRC2012_val_00009213.JPEG:n01990800 +ILSVRC2012_val_00009214.JPEG:n03877845 +ILSVRC2012_val_00009215.JPEG:n02704792 +ILSVRC2012_val_00009216.JPEG:n01773549 +ILSVRC2012_val_00009217.JPEG:n03271574 +ILSVRC2012_val_00009218.JPEG:n02667093 +ILSVRC2012_val_00009219.JPEG:n01514668 +ILSVRC2012_val_00009220.JPEG:n02089867 +ILSVRC2012_val_00009221.JPEG:n02410509 +ILSVRC2012_val_00009222.JPEG:n09193705 +ILSVRC2012_val_00009223.JPEG:n04204238 +ILSVRC2012_val_00009224.JPEG:n02110806 +ILSVRC2012_val_00009225.JPEG:n02823428 +ILSVRC2012_val_00009226.JPEG:n01807496 +ILSVRC2012_val_00009227.JPEG:n07753592 +ILSVRC2012_val_00009228.JPEG:n02835271 +ILSVRC2012_val_00009229.JPEG:n04579432 +ILSVRC2012_val_00009230.JPEG:n03763968 +ILSVRC2012_val_00009231.JPEG:n01667114 +ILSVRC2012_val_00009232.JPEG:n01770393 +ILSVRC2012_val_00009233.JPEG:n02364673 +ILSVRC2012_val_00009234.JPEG:n03777568 +ILSVRC2012_val_00009235.JPEG:n04204238 +ILSVRC2012_val_00009236.JPEG:n04252077 +ILSVRC2012_val_00009237.JPEG:n01496331 +ILSVRC2012_val_00009238.JPEG:n02877765 +ILSVRC2012_val_00009239.JPEG:n01532829 +ILSVRC2012_val_00009240.JPEG:n02640242 +ILSVRC2012_val_00009241.JPEG:n04483307 +ILSVRC2012_val_00009242.JPEG:n04332243 +ILSVRC2012_val_00009243.JPEG:n03197337 +ILSVRC2012_val_00009244.JPEG:n02094433 +ILSVRC2012_val_00009245.JPEG:n03995372 +ILSVRC2012_val_00009246.JPEG:n03485407 +ILSVRC2012_val_00009247.JPEG:n02085782 +ILSVRC2012_val_00009248.JPEG:n04591157 +ILSVRC2012_val_00009249.JPEG:n07930864 +ILSVRC2012_val_00009250.JPEG:n02086079 +ILSVRC2012_val_00009251.JPEG:n01983481 +ILSVRC2012_val_00009252.JPEG:n04162706 +ILSVRC2012_val_00009253.JPEG:n02981792 +ILSVRC2012_val_00009254.JPEG:n02447366 +ILSVRC2012_val_00009255.JPEG:n03733805 +ILSVRC2012_val_00009256.JPEG:n02097298 +ILSVRC2012_val_00009257.JPEG:n04120489 +ILSVRC2012_val_00009258.JPEG:n04442312 +ILSVRC2012_val_00009259.JPEG:n07714990 +ILSVRC2012_val_00009260.JPEG:n02823428 +ILSVRC2012_val_00009261.JPEG:n02788148 +ILSVRC2012_val_00009262.JPEG:n02791270 +ILSVRC2012_val_00009263.JPEG:n11879895 +ILSVRC2012_val_00009264.JPEG:n03776460 +ILSVRC2012_val_00009265.JPEG:n02834397 +ILSVRC2012_val_00009266.JPEG:n03657121 +ILSVRC2012_val_00009267.JPEG:n02423022 +ILSVRC2012_val_00009268.JPEG:n03785016 +ILSVRC2012_val_00009269.JPEG:n03888257 +ILSVRC2012_val_00009270.JPEG:n02018207 +ILSVRC2012_val_00009271.JPEG:n01742172 +ILSVRC2012_val_00009272.JPEG:n04154565 +ILSVRC2012_val_00009273.JPEG:n02536864 +ILSVRC2012_val_00009274.JPEG:n03447721 +ILSVRC2012_val_00009275.JPEG:n02229544 +ILSVRC2012_val_00009276.JPEG:n04540053 +ILSVRC2012_val_00009277.JPEG:n04266014 +ILSVRC2012_val_00009278.JPEG:n03457902 +ILSVRC2012_val_00009279.JPEG:n03425413 +ILSVRC2012_val_00009280.JPEG:n02504013 +ILSVRC2012_val_00009281.JPEG:n02107312 +ILSVRC2012_val_00009282.JPEG:n02177972 +ILSVRC2012_val_00009283.JPEG:n02489166 +ILSVRC2012_val_00009284.JPEG:n04330267 +ILSVRC2012_val_00009285.JPEG:n03791053 +ILSVRC2012_val_00009286.JPEG:n04311004 +ILSVRC2012_val_00009287.JPEG:n02422699 +ILSVRC2012_val_00009288.JPEG:n02319095 +ILSVRC2012_val_00009289.JPEG:n04606251 +ILSVRC2012_val_00009290.JPEG:n04229816 +ILSVRC2012_val_00009291.JPEG:n02101556 +ILSVRC2012_val_00009292.JPEG:n04592741 +ILSVRC2012_val_00009293.JPEG:n03666591 +ILSVRC2012_val_00009294.JPEG:n02088094 +ILSVRC2012_val_00009295.JPEG:n02017213 +ILSVRC2012_val_00009296.JPEG:n03759954 +ILSVRC2012_val_00009297.JPEG:n02128925 +ILSVRC2012_val_00009298.JPEG:n03544143 +ILSVRC2012_val_00009299.JPEG:n03188531 +ILSVRC2012_val_00009300.JPEG:n03459775 +ILSVRC2012_val_00009301.JPEG:n04254680 +ILSVRC2012_val_00009302.JPEG:n03496892 +ILSVRC2012_val_00009303.JPEG:n02483362 +ILSVRC2012_val_00009304.JPEG:n02906734 +ILSVRC2012_val_00009305.JPEG:n07753275 +ILSVRC2012_val_00009306.JPEG:n02879718 +ILSVRC2012_val_00009307.JPEG:n02641379 +ILSVRC2012_val_00009308.JPEG:n02814860 +ILSVRC2012_val_00009309.JPEG:n03400231 +ILSVRC2012_val_00009310.JPEG:n02966687 +ILSVRC2012_val_00009311.JPEG:n09246464 +ILSVRC2012_val_00009312.JPEG:n02114712 +ILSVRC2012_val_00009313.JPEG:n02087046 +ILSVRC2012_val_00009314.JPEG:n02115913 +ILSVRC2012_val_00009315.JPEG:n03424325 +ILSVRC2012_val_00009316.JPEG:n03529860 +ILSVRC2012_val_00009317.JPEG:n01943899 +ILSVRC2012_val_00009318.JPEG:n04238763 +ILSVRC2012_val_00009319.JPEG:n03146219 +ILSVRC2012_val_00009320.JPEG:n02747177 +ILSVRC2012_val_00009321.JPEG:n02233338 +ILSVRC2012_val_00009322.JPEG:n13044778 +ILSVRC2012_val_00009323.JPEG:n03109150 +ILSVRC2012_val_00009324.JPEG:n02112350 +ILSVRC2012_val_00009325.JPEG:n03180011 +ILSVRC2012_val_00009326.JPEG:n02091831 +ILSVRC2012_val_00009327.JPEG:n03134739 +ILSVRC2012_val_00009328.JPEG:n03133878 +ILSVRC2012_val_00009329.JPEG:n01740131 +ILSVRC2012_val_00009330.JPEG:n02125311 +ILSVRC2012_val_00009331.JPEG:n02398521 +ILSVRC2012_val_00009332.JPEG:n02219486 +ILSVRC2012_val_00009333.JPEG:n04086273 +ILSVRC2012_val_00009334.JPEG:n02091244 +ILSVRC2012_val_00009335.JPEG:n02099849 +ILSVRC2012_val_00009336.JPEG:n02119789 +ILSVRC2012_val_00009337.JPEG:n04039381 +ILSVRC2012_val_00009338.JPEG:n02094114 +ILSVRC2012_val_00009339.JPEG:n04562935 +ILSVRC2012_val_00009340.JPEG:n03938244 +ILSVRC2012_val_00009341.JPEG:n07693725 +ILSVRC2012_val_00009342.JPEG:n12998815 +ILSVRC2012_val_00009343.JPEG:n04542943 +ILSVRC2012_val_00009344.JPEG:n02389026 +ILSVRC2012_val_00009345.JPEG:n03417042 +ILSVRC2012_val_00009346.JPEG:n01440764 +ILSVRC2012_val_00009347.JPEG:n02095889 +ILSVRC2012_val_00009348.JPEG:n02090379 +ILSVRC2012_val_00009349.JPEG:n02493509 +ILSVRC2012_val_00009350.JPEG:n02672831 +ILSVRC2012_val_00009351.JPEG:n01534433 +ILSVRC2012_val_00009352.JPEG:n02794156 +ILSVRC2012_val_00009353.JPEG:n02396427 +ILSVRC2012_val_00009354.JPEG:n02117135 +ILSVRC2012_val_00009355.JPEG:n03782006 +ILSVRC2012_val_00009356.JPEG:n04336792 +ILSVRC2012_val_00009357.JPEG:n03042490 +ILSVRC2012_val_00009358.JPEG:n03075370 +ILSVRC2012_val_00009359.JPEG:n02488291 +ILSVRC2012_val_00009360.JPEG:n04332243 +ILSVRC2012_val_00009361.JPEG:n02708093 +ILSVRC2012_val_00009362.JPEG:n02097209 +ILSVRC2012_val_00009363.JPEG:n02356798 +ILSVRC2012_val_00009364.JPEG:n03837869 +ILSVRC2012_val_00009365.JPEG:n04355338 +ILSVRC2012_val_00009366.JPEG:n03584829 +ILSVRC2012_val_00009367.JPEG:n03041632 +ILSVRC2012_val_00009368.JPEG:n06359193 +ILSVRC2012_val_00009369.JPEG:n03041632 +ILSVRC2012_val_00009370.JPEG:n03888257 +ILSVRC2012_val_00009371.JPEG:n03717622 +ILSVRC2012_val_00009372.JPEG:n04235860 +ILSVRC2012_val_00009373.JPEG:n04275548 +ILSVRC2012_val_00009374.JPEG:n01592084 +ILSVRC2012_val_00009375.JPEG:n03388549 +ILSVRC2012_val_00009376.JPEG:n01669191 +ILSVRC2012_val_00009377.JPEG:n07760859 +ILSVRC2012_val_00009378.JPEG:n02090622 +ILSVRC2012_val_00009379.JPEG:n01440764 +ILSVRC2012_val_00009380.JPEG:n01729322 +ILSVRC2012_val_00009381.JPEG:n02480495 +ILSVRC2012_val_00009382.JPEG:n07871810 +ILSVRC2012_val_00009383.JPEG:n04505470 +ILSVRC2012_val_00009384.JPEG:n04418357 +ILSVRC2012_val_00009385.JPEG:n03404251 +ILSVRC2012_val_00009386.JPEG:n03676483 +ILSVRC2012_val_00009387.JPEG:n02165105 +ILSVRC2012_val_00009388.JPEG:n04008634 +ILSVRC2012_val_00009389.JPEG:n03958227 +ILSVRC2012_val_00009390.JPEG:n02480855 +ILSVRC2012_val_00009391.JPEG:n02823750 +ILSVRC2012_val_00009392.JPEG:n07579787 +ILSVRC2012_val_00009393.JPEG:n02009912 +ILSVRC2012_val_00009394.JPEG:n07734744 +ILSVRC2012_val_00009395.JPEG:n03372029 +ILSVRC2012_val_00009396.JPEG:n01440764 +ILSVRC2012_val_00009397.JPEG:n02102177 +ILSVRC2012_val_00009398.JPEG:n03840681 +ILSVRC2012_val_00009399.JPEG:n07753275 +ILSVRC2012_val_00009400.JPEG:n03026506 +ILSVRC2012_val_00009401.JPEG:n01601694 +ILSVRC2012_val_00009402.JPEG:n03047690 +ILSVRC2012_val_00009403.JPEG:n02086079 +ILSVRC2012_val_00009404.JPEG:n02979186 +ILSVRC2012_val_00009405.JPEG:n02089078 +ILSVRC2012_val_00009406.JPEG:n02397096 +ILSVRC2012_val_00009407.JPEG:n12985857 +ILSVRC2012_val_00009408.JPEG:n02808304 +ILSVRC2012_val_00009409.JPEG:n04118538 +ILSVRC2012_val_00009410.JPEG:n04229816 +ILSVRC2012_val_00009411.JPEG:n09428293 +ILSVRC2012_val_00009412.JPEG:n07880968 +ILSVRC2012_val_00009413.JPEG:n04548280 +ILSVRC2012_val_00009414.JPEG:n03804744 +ILSVRC2012_val_00009415.JPEG:n01622779 +ILSVRC2012_val_00009416.JPEG:n02110063 +ILSVRC2012_val_00009417.JPEG:n02814860 +ILSVRC2012_val_00009418.JPEG:n02128385 +ILSVRC2012_val_00009419.JPEG:n01824575 +ILSVRC2012_val_00009420.JPEG:n01496331 +ILSVRC2012_val_00009421.JPEG:n04286575 +ILSVRC2012_val_00009422.JPEG:n03599486 +ILSVRC2012_val_00009423.JPEG:n03857828 +ILSVRC2012_val_00009424.JPEG:n03866082 +ILSVRC2012_val_00009425.JPEG:n03495258 +ILSVRC2012_val_00009426.JPEG:n02526121 +ILSVRC2012_val_00009427.JPEG:n02098105 +ILSVRC2012_val_00009428.JPEG:n02102973 +ILSVRC2012_val_00009429.JPEG:n03124043 +ILSVRC2012_val_00009430.JPEG:n04357314 +ILSVRC2012_val_00009431.JPEG:n07768694 +ILSVRC2012_val_00009432.JPEG:n03000134 +ILSVRC2012_val_00009433.JPEG:n03970156 +ILSVRC2012_val_00009434.JPEG:n04040759 +ILSVRC2012_val_00009435.JPEG:n02112706 +ILSVRC2012_val_00009436.JPEG:n04008634 +ILSVRC2012_val_00009437.JPEG:n04040759 +ILSVRC2012_val_00009438.JPEG:n06794110 +ILSVRC2012_val_00009439.JPEG:n02086646 +ILSVRC2012_val_00009440.JPEG:n02066245 +ILSVRC2012_val_00009441.JPEG:n03884397 +ILSVRC2012_val_00009442.JPEG:n03967562 +ILSVRC2012_val_00009443.JPEG:n04125021 +ILSVRC2012_val_00009444.JPEG:n02910353 +ILSVRC2012_val_00009445.JPEG:n02236044 +ILSVRC2012_val_00009446.JPEG:n01981276 +ILSVRC2012_val_00009447.JPEG:n07871810 +ILSVRC2012_val_00009448.JPEG:n02099849 +ILSVRC2012_val_00009449.JPEG:n03146219 +ILSVRC2012_val_00009450.JPEG:n04146614 +ILSVRC2012_val_00009451.JPEG:n09193705 +ILSVRC2012_val_00009452.JPEG:n02113023 +ILSVRC2012_val_00009453.JPEG:n02100236 +ILSVRC2012_val_00009454.JPEG:n13044778 +ILSVRC2012_val_00009455.JPEG:n03584829 +ILSVRC2012_val_00009456.JPEG:n03180011 +ILSVRC2012_val_00009457.JPEG:n02027492 +ILSVRC2012_val_00009458.JPEG:n03240683 +ILSVRC2012_val_00009459.JPEG:n02526121 +ILSVRC2012_val_00009460.JPEG:n01494475 +ILSVRC2012_val_00009461.JPEG:n02492660 +ILSVRC2012_val_00009462.JPEG:n01774750 +ILSVRC2012_val_00009463.JPEG:n07768694 +ILSVRC2012_val_00009464.JPEG:n02113712 +ILSVRC2012_val_00009465.JPEG:n03666591 +ILSVRC2012_val_00009466.JPEG:n12998815 +ILSVRC2012_val_00009467.JPEG:n03657121 +ILSVRC2012_val_00009468.JPEG:n02110806 +ILSVRC2012_val_00009469.JPEG:n03717622 +ILSVRC2012_val_00009470.JPEG:n02087394 +ILSVRC2012_val_00009471.JPEG:n02692877 +ILSVRC2012_val_00009472.JPEG:n02497673 +ILSVRC2012_val_00009473.JPEG:n04507155 +ILSVRC2012_val_00009474.JPEG:n02114855 +ILSVRC2012_val_00009475.JPEG:n04332243 +ILSVRC2012_val_00009476.JPEG:n02100877 +ILSVRC2012_val_00009477.JPEG:n04332243 +ILSVRC2012_val_00009478.JPEG:n02110627 +ILSVRC2012_val_00009479.JPEG:n03424325 +ILSVRC2012_val_00009480.JPEG:n02104365 +ILSVRC2012_val_00009481.JPEG:n01943899 +ILSVRC2012_val_00009482.JPEG:n03535780 +ILSVRC2012_val_00009483.JPEG:n02883205 +ILSVRC2012_val_00009484.JPEG:n01667778 +ILSVRC2012_val_00009485.JPEG:n01986214 +ILSVRC2012_val_00009486.JPEG:n02666196 +ILSVRC2012_val_00009487.JPEG:n02966687 +ILSVRC2012_val_00009488.JPEG:n02097658 +ILSVRC2012_val_00009489.JPEG:n03866082 +ILSVRC2012_val_00009490.JPEG:n04239074 +ILSVRC2012_val_00009491.JPEG:n02488702 +ILSVRC2012_val_00009492.JPEG:n01735189 +ILSVRC2012_val_00009493.JPEG:n04090263 +ILSVRC2012_val_00009494.JPEG:n04008634 +ILSVRC2012_val_00009495.JPEG:n03742115 +ILSVRC2012_val_00009496.JPEG:n03877472 +ILSVRC2012_val_00009497.JPEG:n03788195 +ILSVRC2012_val_00009498.JPEG:n03794056 +ILSVRC2012_val_00009499.JPEG:n01768244 +ILSVRC2012_val_00009500.JPEG:n02797295 +ILSVRC2012_val_00009501.JPEG:n02009229 +ILSVRC2012_val_00009502.JPEG:n03085013 +ILSVRC2012_val_00009503.JPEG:n02119789 +ILSVRC2012_val_00009504.JPEG:n04557648 +ILSVRC2012_val_00009505.JPEG:n02099267 +ILSVRC2012_val_00009506.JPEG:n03424325 +ILSVRC2012_val_00009507.JPEG:n03666591 +ILSVRC2012_val_00009508.JPEG:n01667778 +ILSVRC2012_val_00009509.JPEG:n07875152 +ILSVRC2012_val_00009510.JPEG:n01514668 +ILSVRC2012_val_00009511.JPEG:n02492660 +ILSVRC2012_val_00009512.JPEG:n03482405 +ILSVRC2012_val_00009513.JPEG:n04033901 +ILSVRC2012_val_00009514.JPEG:n04044716 +ILSVRC2012_val_00009515.JPEG:n03290653 +ILSVRC2012_val_00009516.JPEG:n12057211 +ILSVRC2012_val_00009517.JPEG:n02981792 +ILSVRC2012_val_00009518.JPEG:n01496331 +ILSVRC2012_val_00009519.JPEG:n02483362 +ILSVRC2012_val_00009520.JPEG:n03314780 +ILSVRC2012_val_00009521.JPEG:n04099969 +ILSVRC2012_val_00009522.JPEG:n02669723 +ILSVRC2012_val_00009523.JPEG:n02113799 +ILSVRC2012_val_00009524.JPEG:n02074367 +ILSVRC2012_val_00009525.JPEG:n02094258 +ILSVRC2012_val_00009526.JPEG:n03866082 +ILSVRC2012_val_00009527.JPEG:n04540053 +ILSVRC2012_val_00009528.JPEG:n02777292 +ILSVRC2012_val_00009529.JPEG:n03782006 +ILSVRC2012_val_00009530.JPEG:n02105251 +ILSVRC2012_val_00009531.JPEG:n03761084 +ILSVRC2012_val_00009532.JPEG:n01955084 +ILSVRC2012_val_00009533.JPEG:n02643566 +ILSVRC2012_val_00009534.JPEG:n02106662 +ILSVRC2012_val_00009535.JPEG:n01580077 +ILSVRC2012_val_00009536.JPEG:n01828970 +ILSVRC2012_val_00009537.JPEG:n02690373 +ILSVRC2012_val_00009538.JPEG:n03063599 +ILSVRC2012_val_00009539.JPEG:n02114548 +ILSVRC2012_val_00009540.JPEG:n03014705 +ILSVRC2012_val_00009541.JPEG:n03724870 +ILSVRC2012_val_00009542.JPEG:n02088364 +ILSVRC2012_val_00009543.JPEG:n07716358 +ILSVRC2012_val_00009544.JPEG:n03724870 +ILSVRC2012_val_00009545.JPEG:n03937543 +ILSVRC2012_val_00009546.JPEG:n02091635 +ILSVRC2012_val_00009547.JPEG:n02106382 +ILSVRC2012_val_00009548.JPEG:n07613480 +ILSVRC2012_val_00009549.JPEG:n13133613 +ILSVRC2012_val_00009550.JPEG:n04591157 +ILSVRC2012_val_00009551.JPEG:n02396427 +ILSVRC2012_val_00009552.JPEG:n03776460 +ILSVRC2012_val_00009553.JPEG:n02108089 +ILSVRC2012_val_00009554.JPEG:n02017213 +ILSVRC2012_val_00009555.JPEG:n04350905 +ILSVRC2012_val_00009556.JPEG:n02107683 +ILSVRC2012_val_00009557.JPEG:n04228054 +ILSVRC2012_val_00009558.JPEG:n01773549 +ILSVRC2012_val_00009559.JPEG:n03888257 +ILSVRC2012_val_00009560.JPEG:n02488291 +ILSVRC2012_val_00009561.JPEG:n04493381 +ILSVRC2012_val_00009562.JPEG:n01817953 +ILSVRC2012_val_00009563.JPEG:n01641577 +ILSVRC2012_val_00009564.JPEG:n02012849 +ILSVRC2012_val_00009565.JPEG:n01797886 +ILSVRC2012_val_00009566.JPEG:n02787622 +ILSVRC2012_val_00009567.JPEG:n02910353 +ILSVRC2012_val_00009568.JPEG:n04067472 +ILSVRC2012_val_00009569.JPEG:n03100240 +ILSVRC2012_val_00009570.JPEG:n02087046 +ILSVRC2012_val_00009571.JPEG:n03733131 +ILSVRC2012_val_00009572.JPEG:n02643566 +ILSVRC2012_val_00009573.JPEG:n02916936 +ILSVRC2012_val_00009574.JPEG:n02480495 +ILSVRC2012_val_00009575.JPEG:n02815834 +ILSVRC2012_val_00009576.JPEG:n02086079 +ILSVRC2012_val_00009577.JPEG:n02814860 +ILSVRC2012_val_00009578.JPEG:n02114712 +ILSVRC2012_val_00009579.JPEG:n07742313 +ILSVRC2012_val_00009580.JPEG:n01728920 +ILSVRC2012_val_00009581.JPEG:n02356798 +ILSVRC2012_val_00009582.JPEG:n13044778 +ILSVRC2012_val_00009583.JPEG:n01798484 +ILSVRC2012_val_00009584.JPEG:n04613696 +ILSVRC2012_val_00009585.JPEG:n02108915 +ILSVRC2012_val_00009586.JPEG:n02109047 +ILSVRC2012_val_00009587.JPEG:n03272010 +ILSVRC2012_val_00009588.JPEG:n04008634 +ILSVRC2012_val_00009589.JPEG:n02097209 +ILSVRC2012_val_00009590.JPEG:n01843065 +ILSVRC2012_val_00009591.JPEG:n02999410 +ILSVRC2012_val_00009592.JPEG:n04086273 +ILSVRC2012_val_00009593.JPEG:n03888257 +ILSVRC2012_val_00009594.JPEG:n02123394 +ILSVRC2012_val_00009595.JPEG:n04356056 +ILSVRC2012_val_00009596.JPEG:n09468604 +ILSVRC2012_val_00009597.JPEG:n01601694 +ILSVRC2012_val_00009598.JPEG:n03950228 +ILSVRC2012_val_00009599.JPEG:n04344873 +ILSVRC2012_val_00009600.JPEG:n02672831 +ILSVRC2012_val_00009601.JPEG:n12768682 +ILSVRC2012_val_00009602.JPEG:n02110341 +ILSVRC2012_val_00009603.JPEG:n10148035 +ILSVRC2012_val_00009604.JPEG:n02114367 +ILSVRC2012_val_00009605.JPEG:n04409515 +ILSVRC2012_val_00009606.JPEG:n03240683 +ILSVRC2012_val_00009607.JPEG:n04285008 +ILSVRC2012_val_00009608.JPEG:n07831146 +ILSVRC2012_val_00009609.JPEG:n03584254 +ILSVRC2012_val_00009610.JPEG:n01855672 +ILSVRC2012_val_00009611.JPEG:n02489166 +ILSVRC2012_val_00009612.JPEG:n03216828 +ILSVRC2012_val_00009613.JPEG:n03297495 +ILSVRC2012_val_00009614.JPEG:n04086273 +ILSVRC2012_val_00009615.JPEG:n01514859 +ILSVRC2012_val_00009616.JPEG:n01629819 +ILSVRC2012_val_00009617.JPEG:n02643566 +ILSVRC2012_val_00009618.JPEG:n02113023 +ILSVRC2012_val_00009619.JPEG:n02791270 +ILSVRC2012_val_00009620.JPEG:n03983396 +ILSVRC2012_val_00009621.JPEG:n07880968 +ILSVRC2012_val_00009622.JPEG:n02268853 +ILSVRC2012_val_00009623.JPEG:n03970156 +ILSVRC2012_val_00009624.JPEG:n02091831 +ILSVRC2012_val_00009625.JPEG:n02268853 +ILSVRC2012_val_00009626.JPEG:n02167151 +ILSVRC2012_val_00009627.JPEG:n03742115 +ILSVRC2012_val_00009628.JPEG:n03947888 +ILSVRC2012_val_00009629.JPEG:n04591157 +ILSVRC2012_val_00009630.JPEG:n03729826 +ILSVRC2012_val_00009631.JPEG:n02988304 +ILSVRC2012_val_00009632.JPEG:n03717622 +ILSVRC2012_val_00009633.JPEG:n02391049 +ILSVRC2012_val_00009634.JPEG:n02096585 +ILSVRC2012_val_00009635.JPEG:n02219486 +ILSVRC2012_val_00009636.JPEG:n02093647 +ILSVRC2012_val_00009637.JPEG:n02002556 +ILSVRC2012_val_00009638.JPEG:n02504458 +ILSVRC2012_val_00009639.JPEG:n01665541 +ILSVRC2012_val_00009640.JPEG:n03938244 +ILSVRC2012_val_00009641.JPEG:n03776460 +ILSVRC2012_val_00009642.JPEG:n02093256 +ILSVRC2012_val_00009643.JPEG:n02056570 +ILSVRC2012_val_00009644.JPEG:n02096051 +ILSVRC2012_val_00009645.JPEG:n02488702 +ILSVRC2012_val_00009646.JPEG:n07693725 +ILSVRC2012_val_00009647.JPEG:n01796340 +ILSVRC2012_val_00009648.JPEG:n02950826 +ILSVRC2012_val_00009649.JPEG:n01828970 +ILSVRC2012_val_00009650.JPEG:n03534580 +ILSVRC2012_val_00009651.JPEG:n03394916 +ILSVRC2012_val_00009652.JPEG:n04404412 +ILSVRC2012_val_00009653.JPEG:n03895866 +ILSVRC2012_val_00009654.JPEG:n01944390 +ILSVRC2012_val_00009655.JPEG:n04554684 +ILSVRC2012_val_00009656.JPEG:n02444819 +ILSVRC2012_val_00009657.JPEG:n03623198 +ILSVRC2012_val_00009658.JPEG:n04263257 +ILSVRC2012_val_00009659.JPEG:n04099969 +ILSVRC2012_val_00009660.JPEG:n02105855 +ILSVRC2012_val_00009661.JPEG:n03584829 +ILSVRC2012_val_00009662.JPEG:n04442312 +ILSVRC2012_val_00009663.JPEG:n01514668 +ILSVRC2012_val_00009664.JPEG:n02088364 +ILSVRC2012_val_00009665.JPEG:n01943899 +ILSVRC2012_val_00009666.JPEG:n02091831 +ILSVRC2012_val_00009667.JPEG:n02071294 +ILSVRC2012_val_00009668.JPEG:n03461385 +ILSVRC2012_val_00009669.JPEG:n04485082 +ILSVRC2012_val_00009670.JPEG:n01630670 +ILSVRC2012_val_00009671.JPEG:n01873310 +ILSVRC2012_val_00009672.JPEG:n02011460 +ILSVRC2012_val_00009673.JPEG:n02113978 +ILSVRC2012_val_00009674.JPEG:n01629819 +ILSVRC2012_val_00009675.JPEG:n07711569 +ILSVRC2012_val_00009676.JPEG:n04023962 +ILSVRC2012_val_00009677.JPEG:n01631663 +ILSVRC2012_val_00009678.JPEG:n02815834 +ILSVRC2012_val_00009679.JPEG:n01797886 +ILSVRC2012_val_00009680.JPEG:n03662601 +ILSVRC2012_val_00009681.JPEG:n02704792 +ILSVRC2012_val_00009682.JPEG:n02494079 +ILSVRC2012_val_00009683.JPEG:n02124075 +ILSVRC2012_val_00009684.JPEG:n03530642 +ILSVRC2012_val_00009685.JPEG:n03424325 +ILSVRC2012_val_00009686.JPEG:n02974003 +ILSVRC2012_val_00009687.JPEG:n01685808 +ILSVRC2012_val_00009688.JPEG:n02086910 +ILSVRC2012_val_00009689.JPEG:n04004767 +ILSVRC2012_val_00009690.JPEG:n03720891 +ILSVRC2012_val_00009691.JPEG:n04200800 +ILSVRC2012_val_00009692.JPEG:n01755581 +ILSVRC2012_val_00009693.JPEG:n04118776 +ILSVRC2012_val_00009694.JPEG:n02058221 +ILSVRC2012_val_00009695.JPEG:n03124170 +ILSVRC2012_val_00009696.JPEG:n03584829 +ILSVRC2012_val_00009697.JPEG:n01978455 +ILSVRC2012_val_00009698.JPEG:n02100583 +ILSVRC2012_val_00009699.JPEG:n03131574 +ILSVRC2012_val_00009700.JPEG:n03467068 +ILSVRC2012_val_00009701.JPEG:n02490219 +ILSVRC2012_val_00009702.JPEG:n02978881 +ILSVRC2012_val_00009703.JPEG:n02096051 +ILSVRC2012_val_00009704.JPEG:n04254120 +ILSVRC2012_val_00009705.JPEG:n03028079 +ILSVRC2012_val_00009706.JPEG:n04371774 +ILSVRC2012_val_00009707.JPEG:n02105641 +ILSVRC2012_val_00009708.JPEG:n02397096 +ILSVRC2012_val_00009709.JPEG:n04258138 +ILSVRC2012_val_00009710.JPEG:n03297495 +ILSVRC2012_val_00009711.JPEG:n02108000 +ILSVRC2012_val_00009712.JPEG:n02096585 +ILSVRC2012_val_00009713.JPEG:n02090721 +ILSVRC2012_val_00009714.JPEG:n02786058 +ILSVRC2012_val_00009715.JPEG:n02025239 +ILSVRC2012_val_00009716.JPEG:n01784675 +ILSVRC2012_val_00009717.JPEG:n03393912 +ILSVRC2012_val_00009718.JPEG:n01755581 +ILSVRC2012_val_00009719.JPEG:n02437616 +ILSVRC2012_val_00009720.JPEG:n02219486 +ILSVRC2012_val_00009721.JPEG:n03388549 +ILSVRC2012_val_00009722.JPEG:n02769748 +ILSVRC2012_val_00009723.JPEG:n03384352 +ILSVRC2012_val_00009724.JPEG:n03998194 +ILSVRC2012_val_00009725.JPEG:n02699494 +ILSVRC2012_val_00009726.JPEG:n04277352 +ILSVRC2012_val_00009727.JPEG:n03637318 +ILSVRC2012_val_00009728.JPEG:n02415577 +ILSVRC2012_val_00009729.JPEG:n03788365 +ILSVRC2012_val_00009730.JPEG:n01943899 +ILSVRC2012_val_00009731.JPEG:n02009229 +ILSVRC2012_val_00009732.JPEG:n04325704 +ILSVRC2012_val_00009733.JPEG:n04532670 +ILSVRC2012_val_00009734.JPEG:n01498041 +ILSVRC2012_val_00009735.JPEG:n03793489 +ILSVRC2012_val_00009736.JPEG:n04141076 +ILSVRC2012_val_00009737.JPEG:n04525038 +ILSVRC2012_val_00009738.JPEG:n04548362 +ILSVRC2012_val_00009739.JPEG:n02012849 +ILSVRC2012_val_00009740.JPEG:n02093754 +ILSVRC2012_val_00009741.JPEG:n03534580 +ILSVRC2012_val_00009742.JPEG:n04532670 +ILSVRC2012_val_00009743.JPEG:n02859443 +ILSVRC2012_val_00009744.JPEG:n02027492 +ILSVRC2012_val_00009745.JPEG:n04070727 +ILSVRC2012_val_00009746.JPEG:n03673027 +ILSVRC2012_val_00009747.JPEG:n11879895 +ILSVRC2012_val_00009748.JPEG:n02643566 +ILSVRC2012_val_00009749.JPEG:n04606251 +ILSVRC2012_val_00009750.JPEG:n04613696 +ILSVRC2012_val_00009751.JPEG:n03680355 +ILSVRC2012_val_00009752.JPEG:n01860187 +ILSVRC2012_val_00009753.JPEG:n04251144 +ILSVRC2012_val_00009754.JPEG:n01739381 +ILSVRC2012_val_00009755.JPEG:n02098413 +ILSVRC2012_val_00009756.JPEG:n04019541 +ILSVRC2012_val_00009757.JPEG:n02101556 +ILSVRC2012_val_00009758.JPEG:n03201208 +ILSVRC2012_val_00009759.JPEG:n04532106 +ILSVRC2012_val_00009760.JPEG:n02879718 +ILSVRC2012_val_00009761.JPEG:n02951585 +ILSVRC2012_val_00009762.JPEG:n04604644 +ILSVRC2012_val_00009763.JPEG:n04275548 +ILSVRC2012_val_00009764.JPEG:n02097474 +ILSVRC2012_val_00009765.JPEG:n03482405 +ILSVRC2012_val_00009766.JPEG:n07734744 +ILSVRC2012_val_00009767.JPEG:n03868242 +ILSVRC2012_val_00009768.JPEG:n04332243 +ILSVRC2012_val_00009769.JPEG:n04589890 +ILSVRC2012_val_00009770.JPEG:n03788365 +ILSVRC2012_val_00009771.JPEG:n03649909 +ILSVRC2012_val_00009772.JPEG:n02090721 +ILSVRC2012_val_00009773.JPEG:n02672831 +ILSVRC2012_val_00009774.JPEG:n02109525 +ILSVRC2012_val_00009775.JPEG:n02112018 +ILSVRC2012_val_00009776.JPEG:n07615774 +ILSVRC2012_val_00009777.JPEG:n02102480 +ILSVRC2012_val_00009778.JPEG:n03125729 +ILSVRC2012_val_00009779.JPEG:n01632458 +ILSVRC2012_val_00009780.JPEG:n04252225 +ILSVRC2012_val_00009781.JPEG:n01824575 +ILSVRC2012_val_00009782.JPEG:n02666196 +ILSVRC2012_val_00009783.JPEG:n03832673 +ILSVRC2012_val_00009784.JPEG:n02105641 +ILSVRC2012_val_00009785.JPEG:n07768694 +ILSVRC2012_val_00009786.JPEG:n03871628 +ILSVRC2012_val_00009787.JPEG:n03127925 +ILSVRC2012_val_00009788.JPEG:n03344393 +ILSVRC2012_val_00009789.JPEG:n02096177 +ILSVRC2012_val_00009790.JPEG:n03887697 +ILSVRC2012_val_00009791.JPEG:n03424325 +ILSVRC2012_val_00009792.JPEG:n03014705 +ILSVRC2012_val_00009793.JPEG:n03796401 +ILSVRC2012_val_00009794.JPEG:n03617480 +ILSVRC2012_val_00009795.JPEG:n04065272 +ILSVRC2012_val_00009796.JPEG:n03982430 +ILSVRC2012_val_00009797.JPEG:n04479046 +ILSVRC2012_val_00009798.JPEG:n03763968 +ILSVRC2012_val_00009799.JPEG:n02486410 +ILSVRC2012_val_00009800.JPEG:n07742313 +ILSVRC2012_val_00009801.JPEG:n02687172 +ILSVRC2012_val_00009802.JPEG:n03794056 +ILSVRC2012_val_00009803.JPEG:n04254680 +ILSVRC2012_val_00009804.JPEG:n03661043 +ILSVRC2012_val_00009805.JPEG:n02837789 +ILSVRC2012_val_00009806.JPEG:n02454379 +ILSVRC2012_val_00009807.JPEG:n01560419 +ILSVRC2012_val_00009808.JPEG:n04443257 +ILSVRC2012_val_00009809.JPEG:n07613480 +ILSVRC2012_val_00009810.JPEG:n02110806 +ILSVRC2012_val_00009811.JPEG:n01818515 +ILSVRC2012_val_00009812.JPEG:n02099712 +ILSVRC2012_val_00009813.JPEG:n03384352 +ILSVRC2012_val_00009814.JPEG:n04366367 +ILSVRC2012_val_00009815.JPEG:n03676483 +ILSVRC2012_val_00009816.JPEG:n02892767 +ILSVRC2012_val_00009817.JPEG:n02110627 +ILSVRC2012_val_00009818.JPEG:n02096294 +ILSVRC2012_val_00009819.JPEG:n01667778 +ILSVRC2012_val_00009820.JPEG:n02870880 +ILSVRC2012_val_00009821.JPEG:n03425413 +ILSVRC2012_val_00009822.JPEG:n01751748 +ILSVRC2012_val_00009823.JPEG:n04275548 +ILSVRC2012_val_00009824.JPEG:n03187595 +ILSVRC2012_val_00009825.JPEG:n02437312 +ILSVRC2012_val_00009826.JPEG:n03623198 +ILSVRC2012_val_00009827.JPEG:n01796340 +ILSVRC2012_val_00009828.JPEG:n09472597 +ILSVRC2012_val_00009829.JPEG:n04523525 +ILSVRC2012_val_00009830.JPEG:n02486261 +ILSVRC2012_val_00009831.JPEG:n01531178 +ILSVRC2012_val_00009832.JPEG:n02493509 +ILSVRC2012_val_00009833.JPEG:n02979186 +ILSVRC2012_val_00009834.JPEG:n03584829 +ILSVRC2012_val_00009835.JPEG:n03924679 +ILSVRC2012_val_00009836.JPEG:n02099601 +ILSVRC2012_val_00009837.JPEG:n03259280 +ILSVRC2012_val_00009838.JPEG:n04229816 +ILSVRC2012_val_00009839.JPEG:n01872401 +ILSVRC2012_val_00009840.JPEG:n04579432 +ILSVRC2012_val_00009841.JPEG:n01855672 +ILSVRC2012_val_00009842.JPEG:n01622779 +ILSVRC2012_val_00009843.JPEG:n02509815 +ILSVRC2012_val_00009844.JPEG:n04525305 +ILSVRC2012_val_00009845.JPEG:n04131690 +ILSVRC2012_val_00009846.JPEG:n02484975 +ILSVRC2012_val_00009847.JPEG:n09193705 +ILSVRC2012_val_00009848.JPEG:n02097658 +ILSVRC2012_val_00009849.JPEG:n02877765 +ILSVRC2012_val_00009850.JPEG:n02749479 +ILSVRC2012_val_00009851.JPEG:n06596364 +ILSVRC2012_val_00009852.JPEG:n01806567 +ILSVRC2012_val_00009853.JPEG:n02093428 +ILSVRC2012_val_00009854.JPEG:n01773157 +ILSVRC2012_val_00009855.JPEG:n03207941 +ILSVRC2012_val_00009856.JPEG:n03947888 +ILSVRC2012_val_00009857.JPEG:n01818515 +ILSVRC2012_val_00009858.JPEG:n02092339 +ILSVRC2012_val_00009859.JPEG:n02276258 +ILSVRC2012_val_00009860.JPEG:n03207743 +ILSVRC2012_val_00009861.JPEG:n02794156 +ILSVRC2012_val_00009862.JPEG:n02106166 +ILSVRC2012_val_00009863.JPEG:n03529860 +ILSVRC2012_val_00009864.JPEG:n04493381 +ILSVRC2012_val_00009865.JPEG:n02086079 +ILSVRC2012_val_00009866.JPEG:n02011460 +ILSVRC2012_val_00009867.JPEG:n03961711 +ILSVRC2012_val_00009868.JPEG:n03680355 +ILSVRC2012_val_00009869.JPEG:n04263257 +ILSVRC2012_val_00009870.JPEG:n01819313 +ILSVRC2012_val_00009871.JPEG:n02102177 +ILSVRC2012_val_00009872.JPEG:n04254120 +ILSVRC2012_val_00009873.JPEG:n03888257 +ILSVRC2012_val_00009874.JPEG:n03729826 +ILSVRC2012_val_00009875.JPEG:n04136333 +ILSVRC2012_val_00009876.JPEG:n04346328 +ILSVRC2012_val_00009877.JPEG:n02107908 +ILSVRC2012_val_00009878.JPEG:n02447366 +ILSVRC2012_val_00009879.JPEG:n03125729 +ILSVRC2012_val_00009880.JPEG:n03476684 +ILSVRC2012_val_00009881.JPEG:n02443114 +ILSVRC2012_val_00009882.JPEG:n03788195 +ILSVRC2012_val_00009883.JPEG:n03710637 +ILSVRC2012_val_00009884.JPEG:n03657121 +ILSVRC2012_val_00009885.JPEG:n03633091 +ILSVRC2012_val_00009886.JPEG:n03141823 +ILSVRC2012_val_00009887.JPEG:n07802026 +ILSVRC2012_val_00009888.JPEG:n02113978 +ILSVRC2012_val_00009889.JPEG:n01665541 +ILSVRC2012_val_00009890.JPEG:n01744401 +ILSVRC2012_val_00009891.JPEG:n02834397 +ILSVRC2012_val_00009892.JPEG:n03633091 +ILSVRC2012_val_00009893.JPEG:n04335435 +ILSVRC2012_val_00009894.JPEG:n02011460 +ILSVRC2012_val_00009895.JPEG:n02099712 +ILSVRC2012_val_00009896.JPEG:n03527444 +ILSVRC2012_val_00009897.JPEG:n03180011 +ILSVRC2012_val_00009898.JPEG:n02408429 +ILSVRC2012_val_00009899.JPEG:n02123394 +ILSVRC2012_val_00009900.JPEG:n03980874 +ILSVRC2012_val_00009901.JPEG:n04070727 +ILSVRC2012_val_00009902.JPEG:n03445777 +ILSVRC2012_val_00009903.JPEG:n04465501 +ILSVRC2012_val_00009904.JPEG:n03530642 +ILSVRC2012_val_00009905.JPEG:n03291819 +ILSVRC2012_val_00009906.JPEG:n04252077 +ILSVRC2012_val_00009907.JPEG:n01689811 +ILSVRC2012_val_00009908.JPEG:n02058221 +ILSVRC2012_val_00009909.JPEG:n02112137 +ILSVRC2012_val_00009910.JPEG:n01950731 +ILSVRC2012_val_00009911.JPEG:n01682714 +ILSVRC2012_val_00009912.JPEG:n02231487 +ILSVRC2012_val_00009913.JPEG:n07684084 +ILSVRC2012_val_00009914.JPEG:n03481172 +ILSVRC2012_val_00009915.JPEG:n02963159 +ILSVRC2012_val_00009916.JPEG:n07768694 +ILSVRC2012_val_00009917.JPEG:n03977966 +ILSVRC2012_val_00009918.JPEG:n02165456 +ILSVRC2012_val_00009919.JPEG:n02939185 +ILSVRC2012_val_00009920.JPEG:n04258138 +ILSVRC2012_val_00009921.JPEG:n02123045 +ILSVRC2012_val_00009922.JPEG:n02128757 +ILSVRC2012_val_00009923.JPEG:n02037110 +ILSVRC2012_val_00009924.JPEG:n02128925 +ILSVRC2012_val_00009925.JPEG:n02483362 +ILSVRC2012_val_00009926.JPEG:n03483316 +ILSVRC2012_val_00009927.JPEG:n04273569 +ILSVRC2012_val_00009928.JPEG:n04208210 +ILSVRC2012_val_00009929.JPEG:n03942813 +ILSVRC2012_val_00009930.JPEG:n03291819 +ILSVRC2012_val_00009931.JPEG:n03467068 +ILSVRC2012_val_00009932.JPEG:n02091467 +ILSVRC2012_val_00009933.JPEG:n02113624 +ILSVRC2012_val_00009934.JPEG:n03950228 +ILSVRC2012_val_00009935.JPEG:n03786901 +ILSVRC2012_val_00009936.JPEG:n04228054 +ILSVRC2012_val_00009937.JPEG:n03649909 +ILSVRC2012_val_00009938.JPEG:n01629819 +ILSVRC2012_val_00009939.JPEG:n02104365 +ILSVRC2012_val_00009940.JPEG:n02865351 +ILSVRC2012_val_00009941.JPEG:n02097047 +ILSVRC2012_val_00009942.JPEG:n03902125 +ILSVRC2012_val_00009943.JPEG:n02231487 +ILSVRC2012_val_00009944.JPEG:n04033995 +ILSVRC2012_val_00009945.JPEG:n02172182 +ILSVRC2012_val_00009946.JPEG:n01632777 +ILSVRC2012_val_00009947.JPEG:n02494079 +ILSVRC2012_val_00009948.JPEG:n02391049 +ILSVRC2012_val_00009949.JPEG:n02093256 +ILSVRC2012_val_00009950.JPEG:n03992509 +ILSVRC2012_val_00009951.JPEG:n03710721 +ILSVRC2012_val_00009952.JPEG:n03272010 +ILSVRC2012_val_00009953.JPEG:n03124043 +ILSVRC2012_val_00009954.JPEG:n02422699 +ILSVRC2012_val_00009955.JPEG:n02492035 +ILSVRC2012_val_00009956.JPEG:n02410509 +ILSVRC2012_val_00009957.JPEG:n04120489 +ILSVRC2012_val_00009958.JPEG:n02793495 +ILSVRC2012_val_00009959.JPEG:n03594734 +ILSVRC2012_val_00009960.JPEG:n03841143 +ILSVRC2012_val_00009961.JPEG:n03124043 +ILSVRC2012_val_00009962.JPEG:n04265275 +ILSVRC2012_val_00009963.JPEG:n02088466 +ILSVRC2012_val_00009964.JPEG:n02123159 +ILSVRC2012_val_00009965.JPEG:n03461385 +ILSVRC2012_val_00009966.JPEG:n01675722 +ILSVRC2012_val_00009967.JPEG:n02965783 +ILSVRC2012_val_00009968.JPEG:n07753113 +ILSVRC2012_val_00009969.JPEG:n07614500 +ILSVRC2012_val_00009970.JPEG:n04154565 +ILSVRC2012_val_00009971.JPEG:n03590841 +ILSVRC2012_val_00009972.JPEG:n02361337 +ILSVRC2012_val_00009973.JPEG:n07720875 +ILSVRC2012_val_00009974.JPEG:n01843383 +ILSVRC2012_val_00009975.JPEG:n04162706 +ILSVRC2012_val_00009976.JPEG:n02134418 +ILSVRC2012_val_00009977.JPEG:n03271574 +ILSVRC2012_val_00009978.JPEG:n01494475 +ILSVRC2012_val_00009979.JPEG:n01729977 +ILSVRC2012_val_00009980.JPEG:n01689811 +ILSVRC2012_val_00009981.JPEG:n01582220 +ILSVRC2012_val_00009982.JPEG:n02655020 +ILSVRC2012_val_00009983.JPEG:n03594945 +ILSVRC2012_val_00009984.JPEG:n02099712 +ILSVRC2012_val_00009985.JPEG:n02110627 +ILSVRC2012_val_00009986.JPEG:n02441942 +ILSVRC2012_val_00009987.JPEG:n02791124 +ILSVRC2012_val_00009988.JPEG:n02007558 +ILSVRC2012_val_00009989.JPEG:n03891332 +ILSVRC2012_val_00009990.JPEG:n02791270 +ILSVRC2012_val_00009991.JPEG:n02037110 +ILSVRC2012_val_00009992.JPEG:n02127052 +ILSVRC2012_val_00009993.JPEG:n01910747 +ILSVRC2012_val_00009994.JPEG:n01829413 +ILSVRC2012_val_00009995.JPEG:n04523525 +ILSVRC2012_val_00009996.JPEG:n02417914 +ILSVRC2012_val_00009997.JPEG:n04465501 +ILSVRC2012_val_00009998.JPEG:n01860187 +ILSVRC2012_val_00009999.JPEG:n03935335 +ILSVRC2012_val_00010000.JPEG:n03908714 +ILSVRC2012_val_00010001.JPEG:n02018207 +ILSVRC2012_val_00010002.JPEG:n02006656 +ILSVRC2012_val_00010003.JPEG:n07802026 +ILSVRC2012_val_00010004.JPEG:n03950228 +ILSVRC2012_val_00010005.JPEG:n07590611 +ILSVRC2012_val_00010006.JPEG:n02092002 +ILSVRC2012_val_00010007.JPEG:n04423845 +ILSVRC2012_val_00010008.JPEG:n02790996 +ILSVRC2012_val_00010009.JPEG:n04252225 +ILSVRC2012_val_00010010.JPEG:n03666591 +ILSVRC2012_val_00010011.JPEG:n02109961 +ILSVRC2012_val_00010012.JPEG:n03930630 +ILSVRC2012_val_00010013.JPEG:n02860847 +ILSVRC2012_val_00010014.JPEG:n04552348 +ILSVRC2012_val_00010015.JPEG:n02092339 +ILSVRC2012_val_00010016.JPEG:n09229709 +ILSVRC2012_val_00010017.JPEG:n02791270 +ILSVRC2012_val_00010018.JPEG:n07579787 +ILSVRC2012_val_00010019.JPEG:n03196217 +ILSVRC2012_val_00010020.JPEG:n02500267 +ILSVRC2012_val_00010021.JPEG:n02790996 +ILSVRC2012_val_00010022.JPEG:n01622779 +ILSVRC2012_val_00010023.JPEG:n02484975 +ILSVRC2012_val_00010024.JPEG:n02669723 +ILSVRC2012_val_00010025.JPEG:n02280649 +ILSVRC2012_val_00010026.JPEG:n11879895 +ILSVRC2012_val_00010027.JPEG:n03769881 +ILSVRC2012_val_00010028.JPEG:n02167151 +ILSVRC2012_val_00010029.JPEG:n02403003 +ILSVRC2012_val_00010030.JPEG:n03717622 +ILSVRC2012_val_00010031.JPEG:n02093991 +ILSVRC2012_val_00010032.JPEG:n03942813 +ILSVRC2012_val_00010033.JPEG:n04254680 +ILSVRC2012_val_00010034.JPEG:n04443257 +ILSVRC2012_val_00010035.JPEG:n01860187 +ILSVRC2012_val_00010036.JPEG:n09229709 +ILSVRC2012_val_00010037.JPEG:n02028035 +ILSVRC2012_val_00010038.JPEG:n02087394 +ILSVRC2012_val_00010039.JPEG:n01986214 +ILSVRC2012_val_00010040.JPEG:n02115641 +ILSVRC2012_val_00010041.JPEG:n02640242 +ILSVRC2012_val_00010042.JPEG:n04328186 +ILSVRC2012_val_00010043.JPEG:n03908618 +ILSVRC2012_val_00010044.JPEG:n04154565 +ILSVRC2012_val_00010045.JPEG:n02797295 +ILSVRC2012_val_00010046.JPEG:n02097209 +ILSVRC2012_val_00010047.JPEG:n02125311 +ILSVRC2012_val_00010048.JPEG:n07932039 +ILSVRC2012_val_00010049.JPEG:n02102973 +ILSVRC2012_val_00010050.JPEG:n03529860 +ILSVRC2012_val_00010051.JPEG:n01980166 +ILSVRC2012_val_00010052.JPEG:n02443114 +ILSVRC2012_val_00010053.JPEG:n03733131 +ILSVRC2012_val_00010054.JPEG:n07718472 +ILSVRC2012_val_00010055.JPEG:n03255030 +ILSVRC2012_val_00010056.JPEG:n02009912 +ILSVRC2012_val_00010057.JPEG:n02087394 +ILSVRC2012_val_00010058.JPEG:n03218198 +ILSVRC2012_val_00010059.JPEG:n02106550 +ILSVRC2012_val_00010060.JPEG:n03888605 +ILSVRC2012_val_00010061.JPEG:n01704323 +ILSVRC2012_val_00010062.JPEG:n02091635 +ILSVRC2012_val_00010063.JPEG:n03710721 +ILSVRC2012_val_00010064.JPEG:n02325366 +ILSVRC2012_val_00010065.JPEG:n02112350 +ILSVRC2012_val_00010066.JPEG:n03207743 +ILSVRC2012_val_00010067.JPEG:n03980874 +ILSVRC2012_val_00010068.JPEG:n03042490 +ILSVRC2012_val_00010069.JPEG:n07590611 +ILSVRC2012_val_00010070.JPEG:n02096051 +ILSVRC2012_val_00010071.JPEG:n02408429 +ILSVRC2012_val_00010072.JPEG:n02091244 +ILSVRC2012_val_00010073.JPEG:n03773504 +ILSVRC2012_val_00010074.JPEG:n01491361 +ILSVRC2012_val_00010075.JPEG:n02120505 +ILSVRC2012_val_00010076.JPEG:n02607072 +ILSVRC2012_val_00010077.JPEG:n02487347 +ILSVRC2012_val_00010078.JPEG:n02504458 +ILSVRC2012_val_00010079.JPEG:n04204347 +ILSVRC2012_val_00010080.JPEG:n02037110 +ILSVRC2012_val_00010081.JPEG:n02790996 +ILSVRC2012_val_00010082.JPEG:n02107312 +ILSVRC2012_val_00010083.JPEG:n04044716 +ILSVRC2012_val_00010084.JPEG:n02002556 +ILSVRC2012_val_00010085.JPEG:n02727426 +ILSVRC2012_val_00010086.JPEG:n04606251 +ILSVRC2012_val_00010087.JPEG:n02091831 +ILSVRC2012_val_00010088.JPEG:n03598930 +ILSVRC2012_val_00010089.JPEG:n03089624 +ILSVRC2012_val_00010090.JPEG:n01807496 +ILSVRC2012_val_00010091.JPEG:n07613480 +ILSVRC2012_val_00010092.JPEG:n04404412 +ILSVRC2012_val_00010093.JPEG:n04542943 +ILSVRC2012_val_00010094.JPEG:n09229709 +ILSVRC2012_val_00010095.JPEG:n03467068 +ILSVRC2012_val_00010096.JPEG:n01943899 +ILSVRC2012_val_00010097.JPEG:n11939491 +ILSVRC2012_val_00010098.JPEG:n02086646 +ILSVRC2012_val_00010099.JPEG:n02095314 +ILSVRC2012_val_00010100.JPEG:n02328150 +ILSVRC2012_val_00010101.JPEG:n02992529 +ILSVRC2012_val_00010102.JPEG:n02281787 +ILSVRC2012_val_00010103.JPEG:n04008634 +ILSVRC2012_val_00010104.JPEG:n07697313 +ILSVRC2012_val_00010105.JPEG:n03347037 +ILSVRC2012_val_00010106.JPEG:n02012849 +ILSVRC2012_val_00010107.JPEG:n02099429 +ILSVRC2012_val_00010108.JPEG:n04179913 +ILSVRC2012_val_00010109.JPEG:n02106662 +ILSVRC2012_val_00010110.JPEG:n03841143 +ILSVRC2012_val_00010111.JPEG:n07768694 +ILSVRC2012_val_00010112.JPEG:n07880968 +ILSVRC2012_val_00010113.JPEG:n02111129 +ILSVRC2012_val_00010114.JPEG:n04456115 +ILSVRC2012_val_00010115.JPEG:n04330267 +ILSVRC2012_val_00010116.JPEG:n01629819 +ILSVRC2012_val_00010117.JPEG:n04146614 +ILSVRC2012_val_00010118.JPEG:n03710193 +ILSVRC2012_val_00010119.JPEG:n03250847 +ILSVRC2012_val_00010120.JPEG:n02808304 +ILSVRC2012_val_00010121.JPEG:n03018349 +ILSVRC2012_val_00010122.JPEG:n01943899 +ILSVRC2012_val_00010123.JPEG:n02398521 +ILSVRC2012_val_00010124.JPEG:n03388549 +ILSVRC2012_val_00010125.JPEG:n02097658 +ILSVRC2012_val_00010126.JPEG:n03529860 +ILSVRC2012_val_00010127.JPEG:n02782093 +ILSVRC2012_val_00010128.JPEG:n01592084 +ILSVRC2012_val_00010129.JPEG:n04311174 +ILSVRC2012_val_00010130.JPEG:n02823750 +ILSVRC2012_val_00010131.JPEG:n04067472 +ILSVRC2012_val_00010132.JPEG:n02422699 +ILSVRC2012_val_00010133.JPEG:n03832673 +ILSVRC2012_val_00010134.JPEG:n04367480 +ILSVRC2012_val_00010135.JPEG:n04557648 +ILSVRC2012_val_00010136.JPEG:n02051845 +ILSVRC2012_val_00010137.JPEG:n01882714 +ILSVRC2012_val_00010138.JPEG:n02012849 +ILSVRC2012_val_00010139.JPEG:n03796401 +ILSVRC2012_val_00010140.JPEG:n01735189 +ILSVRC2012_val_00010141.JPEG:n09256479 +ILSVRC2012_val_00010142.JPEG:n03529860 +ILSVRC2012_val_00010143.JPEG:n11939491 +ILSVRC2012_val_00010144.JPEG:n03673027 +ILSVRC2012_val_00010145.JPEG:n01669191 +ILSVRC2012_val_00010146.JPEG:n03742115 +ILSVRC2012_val_00010147.JPEG:n02692877 +ILSVRC2012_val_00010148.JPEG:n02328150 +ILSVRC2012_val_00010149.JPEG:n07715103 +ILSVRC2012_val_00010150.JPEG:n02268443 +ILSVRC2012_val_00010151.JPEG:n02268853 +ILSVRC2012_val_00010152.JPEG:n01770393 +ILSVRC2012_val_00010153.JPEG:n07718747 +ILSVRC2012_val_00010154.JPEG:n07714571 +ILSVRC2012_val_00010155.JPEG:n01695060 +ILSVRC2012_val_00010156.JPEG:n01843065 +ILSVRC2012_val_00010157.JPEG:n03404251 +ILSVRC2012_val_00010158.JPEG:n02823750 +ILSVRC2012_val_00010159.JPEG:n04264628 +ILSVRC2012_val_00010160.JPEG:n03478589 +ILSVRC2012_val_00010161.JPEG:n02643566 +ILSVRC2012_val_00010162.JPEG:n01514859 +ILSVRC2012_val_00010163.JPEG:n02086646 +ILSVRC2012_val_00010164.JPEG:n01692333 +ILSVRC2012_val_00010165.JPEG:n03841143 +ILSVRC2012_val_00010166.JPEG:n03977966 +ILSVRC2012_val_00010167.JPEG:n04136333 +ILSVRC2012_val_00010168.JPEG:n02089973 +ILSVRC2012_val_00010169.JPEG:n02097298 +ILSVRC2012_val_00010170.JPEG:n04311174 +ILSVRC2012_val_00010171.JPEG:n01677366 +ILSVRC2012_val_00010172.JPEG:n01930112 +ILSVRC2012_val_00010173.JPEG:n02128925 +ILSVRC2012_val_00010174.JPEG:n03710721 +ILSVRC2012_val_00010175.JPEG:n02909870 +ILSVRC2012_val_00010176.JPEG:n02027492 +ILSVRC2012_val_00010177.JPEG:n04252077 +ILSVRC2012_val_00010178.JPEG:n03544143 +ILSVRC2012_val_00010179.JPEG:n09332890 +ILSVRC2012_val_00010180.JPEG:n04118776 +ILSVRC2012_val_00010181.JPEG:n04553703 +ILSVRC2012_val_00010182.JPEG:n02488702 +ILSVRC2012_val_00010183.JPEG:n02109525 +ILSVRC2012_val_00010184.JPEG:n04443257 +ILSVRC2012_val_00010185.JPEG:n01728572 +ILSVRC2012_val_00010186.JPEG:n03384352 +ILSVRC2012_val_00010187.JPEG:n04136333 +ILSVRC2012_val_00010188.JPEG:n07718472 +ILSVRC2012_val_00010189.JPEG:n03773504 +ILSVRC2012_val_00010190.JPEG:n04273569 +ILSVRC2012_val_00010191.JPEG:n02730930 +ILSVRC2012_val_00010192.JPEG:n02259212 +ILSVRC2012_val_00010193.JPEG:n03125729 +ILSVRC2012_val_00010194.JPEG:n01748264 +ILSVRC2012_val_00010195.JPEG:n03095699 +ILSVRC2012_val_00010196.JPEG:n02504458 +ILSVRC2012_val_00010197.JPEG:n04579432 +ILSVRC2012_val_00010198.JPEG:n02231487 +ILSVRC2012_val_00010199.JPEG:n04442312 +ILSVRC2012_val_00010200.JPEG:n03447447 +ILSVRC2012_val_00010201.JPEG:n02939185 +ILSVRC2012_val_00010202.JPEG:n02110341 +ILSVRC2012_val_00010203.JPEG:n04458633 +ILSVRC2012_val_00010204.JPEG:n03492542 +ILSVRC2012_val_00010205.JPEG:n02841315 +ILSVRC2012_val_00010206.JPEG:n04285008 +ILSVRC2012_val_00010207.JPEG:n02787622 +ILSVRC2012_val_00010208.JPEG:n01514668 +ILSVRC2012_val_00010209.JPEG:n03877472 +ILSVRC2012_val_00010210.JPEG:n04486054 +ILSVRC2012_val_00010211.JPEG:n04238763 +ILSVRC2012_val_00010212.JPEG:n02480495 +ILSVRC2012_val_00010213.JPEG:n07871810 +ILSVRC2012_val_00010214.JPEG:n01968897 +ILSVRC2012_val_00010215.JPEG:n03954731 +ILSVRC2012_val_00010216.JPEG:n03584829 +ILSVRC2012_val_00010217.JPEG:n03379051 +ILSVRC2012_val_00010218.JPEG:n02123394 +ILSVRC2012_val_00010219.JPEG:n03259280 +ILSVRC2012_val_00010220.JPEG:n07920052 +ILSVRC2012_val_00010221.JPEG:n02113712 +ILSVRC2012_val_00010222.JPEG:n02092002 +ILSVRC2012_val_00010223.JPEG:n02727426 +ILSVRC2012_val_00010224.JPEG:n04149813 +ILSVRC2012_val_00010225.JPEG:n01775062 +ILSVRC2012_val_00010226.JPEG:n03457902 +ILSVRC2012_val_00010227.JPEG:n03791053 +ILSVRC2012_val_00010228.JPEG:n02106550 +ILSVRC2012_val_00010229.JPEG:n09288635 +ILSVRC2012_val_00010230.JPEG:n01742172 +ILSVRC2012_val_00010231.JPEG:n02219486 +ILSVRC2012_val_00010232.JPEG:n04332243 +ILSVRC2012_val_00010233.JPEG:n02490219 +ILSVRC2012_val_00010234.JPEG:n04033901 +ILSVRC2012_val_00010235.JPEG:n03590841 +ILSVRC2012_val_00010236.JPEG:n04344873 +ILSVRC2012_val_00010237.JPEG:n07753592 +ILSVRC2012_val_00010238.JPEG:n02085936 +ILSVRC2012_val_00010239.JPEG:n03447721 +ILSVRC2012_val_00010240.JPEG:n01580077 +ILSVRC2012_val_00010241.JPEG:n02120505 +ILSVRC2012_val_00010242.JPEG:n02504458 +ILSVRC2012_val_00010243.JPEG:n03633091 +ILSVRC2012_val_00010244.JPEG:n02113023 +ILSVRC2012_val_00010245.JPEG:n02109525 +ILSVRC2012_val_00010246.JPEG:n11879895 +ILSVRC2012_val_00010247.JPEG:n03445924 +ILSVRC2012_val_00010248.JPEG:n01882714 +ILSVRC2012_val_00010249.JPEG:n02089867 +ILSVRC2012_val_00010250.JPEG:n04604644 +ILSVRC2012_val_00010251.JPEG:n03697007 +ILSVRC2012_val_00010252.JPEG:n02814533 +ILSVRC2012_val_00010253.JPEG:n02094114 +ILSVRC2012_val_00010254.JPEG:n01631663 +ILSVRC2012_val_00010255.JPEG:n02105251 +ILSVRC2012_val_00010256.JPEG:n02948072 +ILSVRC2012_val_00010257.JPEG:n04200800 +ILSVRC2012_val_00010258.JPEG:n01820546 +ILSVRC2012_val_00010259.JPEG:n03125729 +ILSVRC2012_val_00010260.JPEG:n03290653 +ILSVRC2012_val_00010261.JPEG:n02102480 +ILSVRC2012_val_00010262.JPEG:n04525038 +ILSVRC2012_val_00010263.JPEG:n03347037 +ILSVRC2012_val_00010264.JPEG:n03950228 +ILSVRC2012_val_00010265.JPEG:n02319095 +ILSVRC2012_val_00010266.JPEG:n03160309 +ILSVRC2012_val_00010267.JPEG:n03787032 +ILSVRC2012_val_00010268.JPEG:n02107574 +ILSVRC2012_val_00010269.JPEG:n04487394 +ILSVRC2012_val_00010270.JPEG:n04548280 +ILSVRC2012_val_00010271.JPEG:n07697537 +ILSVRC2012_val_00010272.JPEG:n01580077 +ILSVRC2012_val_00010273.JPEG:n03599486 +ILSVRC2012_val_00010274.JPEG:n04599235 +ILSVRC2012_val_00010275.JPEG:n01735189 +ILSVRC2012_val_00010276.JPEG:n04612504 +ILSVRC2012_val_00010277.JPEG:n02786058 +ILSVRC2012_val_00010278.JPEG:n03000247 +ILSVRC2012_val_00010279.JPEG:n02906734 +ILSVRC2012_val_00010280.JPEG:n13054560 +ILSVRC2012_val_00010281.JPEG:n02132136 +ILSVRC2012_val_00010282.JPEG:n02939185 +ILSVRC2012_val_00010283.JPEG:n02101006 +ILSVRC2012_val_00010284.JPEG:n04141975 +ILSVRC2012_val_00010285.JPEG:n04127249 +ILSVRC2012_val_00010286.JPEG:n07565083 +ILSVRC2012_val_00010287.JPEG:n01641577 +ILSVRC2012_val_00010288.JPEG:n02017213 +ILSVRC2012_val_00010289.JPEG:n02095889 +ILSVRC2012_val_00010290.JPEG:n02096585 +ILSVRC2012_val_00010291.JPEG:n03461385 +ILSVRC2012_val_00010292.JPEG:n02231487 +ILSVRC2012_val_00010293.JPEG:n04493381 +ILSVRC2012_val_00010294.JPEG:n02092339 +ILSVRC2012_val_00010295.JPEG:n04332243 +ILSVRC2012_val_00010296.JPEG:n02497673 +ILSVRC2012_val_00010297.JPEG:n02119022 +ILSVRC2012_val_00010298.JPEG:n02099601 +ILSVRC2012_val_00010299.JPEG:n04311004 +ILSVRC2012_val_00010300.JPEG:n03920288 +ILSVRC2012_val_00010301.JPEG:n02704792 +ILSVRC2012_val_00010302.JPEG:n02091032 +ILSVRC2012_val_00010303.JPEG:n03240683 +ILSVRC2012_val_00010304.JPEG:n03538406 +ILSVRC2012_val_00010305.JPEG:n04560804 +ILSVRC2012_val_00010306.JPEG:n01440764 +ILSVRC2012_val_00010307.JPEG:n02776631 +ILSVRC2012_val_00010308.JPEG:n02013706 +ILSVRC2012_val_00010309.JPEG:n02099849 +ILSVRC2012_val_00010310.JPEG:n01532829 +ILSVRC2012_val_00010311.JPEG:n02110341 +ILSVRC2012_val_00010312.JPEG:n01944390 +ILSVRC2012_val_00010313.JPEG:n03218198 +ILSVRC2012_val_00010314.JPEG:n02099712 +ILSVRC2012_val_00010315.JPEG:n04429376 +ILSVRC2012_val_00010316.JPEG:n03249569 +ILSVRC2012_val_00010317.JPEG:n02422106 +ILSVRC2012_val_00010318.JPEG:n04254777 +ILSVRC2012_val_00010319.JPEG:n04009552 +ILSVRC2012_val_00010320.JPEG:n03617480 +ILSVRC2012_val_00010321.JPEG:n03337140 +ILSVRC2012_val_00010322.JPEG:n01692333 +ILSVRC2012_val_00010323.JPEG:n02493509 +ILSVRC2012_val_00010324.JPEG:n12144580 +ILSVRC2012_val_00010325.JPEG:n03095699 +ILSVRC2012_val_00010326.JPEG:n03781244 +ILSVRC2012_val_00010327.JPEG:n03782006 +ILSVRC2012_val_00010328.JPEG:n02099429 +ILSVRC2012_val_00010329.JPEG:n09428293 +ILSVRC2012_val_00010330.JPEG:n04179913 +ILSVRC2012_val_00010331.JPEG:n02105251 +ILSVRC2012_val_00010332.JPEG:n07716358 +ILSVRC2012_val_00010333.JPEG:n04357314 +ILSVRC2012_val_00010334.JPEG:n03895866 +ILSVRC2012_val_00010335.JPEG:n02948072 +ILSVRC2012_val_00010336.JPEG:n03888257 +ILSVRC2012_val_00010337.JPEG:n03447447 +ILSVRC2012_val_00010338.JPEG:n07248320 +ILSVRC2012_val_00010339.JPEG:n01537544 +ILSVRC2012_val_00010340.JPEG:n02487347 +ILSVRC2012_val_00010341.JPEG:n03982430 +ILSVRC2012_val_00010342.JPEG:n02910353 +ILSVRC2012_val_00010343.JPEG:n07892512 +ILSVRC2012_val_00010344.JPEG:n09468604 +ILSVRC2012_val_00010345.JPEG:n03857828 +ILSVRC2012_val_00010346.JPEG:n03290653 +ILSVRC2012_val_00010347.JPEG:n03388043 +ILSVRC2012_val_00010348.JPEG:n03843555 +ILSVRC2012_val_00010349.JPEG:n04423845 +ILSVRC2012_val_00010350.JPEG:n04404412 +ILSVRC2012_val_00010351.JPEG:n04347754 +ILSVRC2012_val_00010352.JPEG:n01537544 +ILSVRC2012_val_00010353.JPEG:n02992529 +ILSVRC2012_val_00010354.JPEG:n02101388 +ILSVRC2012_val_00010355.JPEG:n02056570 +ILSVRC2012_val_00010356.JPEG:n02093859 +ILSVRC2012_val_00010357.JPEG:n02105412 +ILSVRC2012_val_00010358.JPEG:n03933933 +ILSVRC2012_val_00010359.JPEG:n02704792 +ILSVRC2012_val_00010360.JPEG:n03063599 +ILSVRC2012_val_00010361.JPEG:n12267677 +ILSVRC2012_val_00010362.JPEG:n04482393 +ILSVRC2012_val_00010363.JPEG:n01443537 +ILSVRC2012_val_00010364.JPEG:n03670208 +ILSVRC2012_val_00010365.JPEG:n04590129 +ILSVRC2012_val_00010366.JPEG:n07565083 +ILSVRC2012_val_00010367.JPEG:n04111531 +ILSVRC2012_val_00010368.JPEG:n03188531 +ILSVRC2012_val_00010369.JPEG:n02114712 +ILSVRC2012_val_00010370.JPEG:n04409515 +ILSVRC2012_val_00010371.JPEG:n03272010 +ILSVRC2012_val_00010372.JPEG:n02107312 +ILSVRC2012_val_00010373.JPEG:n02112018 +ILSVRC2012_val_00010374.JPEG:n03676483 +ILSVRC2012_val_00010375.JPEG:n03770439 +ILSVRC2012_val_00010376.JPEG:n13133613 +ILSVRC2012_val_00010377.JPEG:n04259630 +ILSVRC2012_val_00010378.JPEG:n02105641 +ILSVRC2012_val_00010379.JPEG:n04049303 +ILSVRC2012_val_00010380.JPEG:n02807133 +ILSVRC2012_val_00010381.JPEG:n03249569 +ILSVRC2012_val_00010382.JPEG:n02099267 +ILSVRC2012_val_00010383.JPEG:n04065272 +ILSVRC2012_val_00010384.JPEG:n07716906 +ILSVRC2012_val_00010385.JPEG:n02087394 +ILSVRC2012_val_00010386.JPEG:n01669191 +ILSVRC2012_val_00010387.JPEG:n04376876 +ILSVRC2012_val_00010388.JPEG:n01847000 +ILSVRC2012_val_00010389.JPEG:n02123597 +ILSVRC2012_val_00010390.JPEG:n04131690 +ILSVRC2012_val_00010391.JPEG:n02033041 +ILSVRC2012_val_00010392.JPEG:n04357314 +ILSVRC2012_val_00010393.JPEG:n01530575 +ILSVRC2012_val_00010394.JPEG:n02841315 +ILSVRC2012_val_00010395.JPEG:n01698640 +ILSVRC2012_val_00010396.JPEG:n04179913 +ILSVRC2012_val_00010397.JPEG:n01824575 +ILSVRC2012_val_00010398.JPEG:n02092002 +ILSVRC2012_val_00010399.JPEG:n02058221 +ILSVRC2012_val_00010400.JPEG:n03617480 +ILSVRC2012_val_00010401.JPEG:n04146614 +ILSVRC2012_val_00010402.JPEG:n02097130 +ILSVRC2012_val_00010403.JPEG:n09399592 +ILSVRC2012_val_00010404.JPEG:n02892201 +ILSVRC2012_val_00010405.JPEG:n02116738 +ILSVRC2012_val_00010406.JPEG:n04204347 +ILSVRC2012_val_00010407.JPEG:n04522168 +ILSVRC2012_val_00010408.JPEG:n04136333 +ILSVRC2012_val_00010409.JPEG:n01531178 +ILSVRC2012_val_00010410.JPEG:n02346627 +ILSVRC2012_val_00010411.JPEG:n02168699 +ILSVRC2012_val_00010412.JPEG:n01980166 +ILSVRC2012_val_00010413.JPEG:n07711569 +ILSVRC2012_val_00010414.JPEG:n03347037 +ILSVRC2012_val_00010415.JPEG:n04208210 +ILSVRC2012_val_00010416.JPEG:n02823750 +ILSVRC2012_val_00010417.JPEG:n02124075 +ILSVRC2012_val_00010418.JPEG:n02509815 +ILSVRC2012_val_00010419.JPEG:n03404251 +ILSVRC2012_val_00010420.JPEG:n02088364 +ILSVRC2012_val_00010421.JPEG:n01798484 +ILSVRC2012_val_00010422.JPEG:n02009912 +ILSVRC2012_val_00010423.JPEG:n03814639 +ILSVRC2012_val_00010424.JPEG:n02172182 +ILSVRC2012_val_00010425.JPEG:n03840681 +ILSVRC2012_val_00010426.JPEG:n02002556 +ILSVRC2012_val_00010427.JPEG:n03888257 +ILSVRC2012_val_00010428.JPEG:n03065424 +ILSVRC2012_val_00010429.JPEG:n03325584 +ILSVRC2012_val_00010430.JPEG:n02317335 +ILSVRC2012_val_00010431.JPEG:n02281406 +ILSVRC2012_val_00010432.JPEG:n03658185 +ILSVRC2012_val_00010433.JPEG:n02095570 +ILSVRC2012_val_00010434.JPEG:n03920288 +ILSVRC2012_val_00010435.JPEG:n03710637 +ILSVRC2012_val_00010436.JPEG:n02123597 +ILSVRC2012_val_00010437.JPEG:n03877472 +ILSVRC2012_val_00010438.JPEG:n04357314 +ILSVRC2012_val_00010439.JPEG:n07802026 +ILSVRC2012_val_00010440.JPEG:n04067472 +ILSVRC2012_val_00010441.JPEG:n02437616 +ILSVRC2012_val_00010442.JPEG:n03482405 +ILSVRC2012_val_00010443.JPEG:n01532829 +ILSVRC2012_val_00010444.JPEG:n04553703 +ILSVRC2012_val_00010445.JPEG:n03065424 +ILSVRC2012_val_00010446.JPEG:n02058221 +ILSVRC2012_val_00010447.JPEG:n07718472 +ILSVRC2012_val_00010448.JPEG:n04252225 +ILSVRC2012_val_00010449.JPEG:n02096585 +ILSVRC2012_val_00010450.JPEG:n02097658 +ILSVRC2012_val_00010451.JPEG:n04525305 +ILSVRC2012_val_00010452.JPEG:n12057211 +ILSVRC2012_val_00010453.JPEG:n04259630 +ILSVRC2012_val_00010454.JPEG:n02490219 +ILSVRC2012_val_00010455.JPEG:n04285008 +ILSVRC2012_val_00010456.JPEG:n01534433 +ILSVRC2012_val_00010457.JPEG:n01622779 +ILSVRC2012_val_00010458.JPEG:n04067472 +ILSVRC2012_val_00010459.JPEG:n04557648 +ILSVRC2012_val_00010460.JPEG:n03888257 +ILSVRC2012_val_00010461.JPEG:n02096051 +ILSVRC2012_val_00010462.JPEG:n01632458 +ILSVRC2012_val_00010463.JPEG:n02808304 +ILSVRC2012_val_00010464.JPEG:n12985857 +ILSVRC2012_val_00010465.JPEG:n01756291 +ILSVRC2012_val_00010466.JPEG:n02111500 +ILSVRC2012_val_00010467.JPEG:n02963159 +ILSVRC2012_val_00010468.JPEG:n02790996 +ILSVRC2012_val_00010469.JPEG:n03630383 +ILSVRC2012_val_00010470.JPEG:n07714990 +ILSVRC2012_val_00010471.JPEG:n04589890 +ILSVRC2012_val_00010472.JPEG:n02128757 +ILSVRC2012_val_00010473.JPEG:n02786058 +ILSVRC2012_val_00010474.JPEG:n02951358 +ILSVRC2012_val_00010475.JPEG:n03763968 +ILSVRC2012_val_00010476.JPEG:n02356798 +ILSVRC2012_val_00010477.JPEG:n01818515 +ILSVRC2012_val_00010478.JPEG:n02607072 +ILSVRC2012_val_00010479.JPEG:n07717410 +ILSVRC2012_val_00010480.JPEG:n03877472 +ILSVRC2012_val_00010481.JPEG:n04069434 +ILSVRC2012_val_00010482.JPEG:n02483362 +ILSVRC2012_val_00010483.JPEG:n04479046 +ILSVRC2012_val_00010484.JPEG:n02268853 +ILSVRC2012_val_00010485.JPEG:n10148035 +ILSVRC2012_val_00010486.JPEG:n02815834 +ILSVRC2012_val_00010487.JPEG:n02116738 +ILSVRC2012_val_00010488.JPEG:n04501370 +ILSVRC2012_val_00010489.JPEG:n03131574 +ILSVRC2012_val_00010490.JPEG:n02099712 +ILSVRC2012_val_00010491.JPEG:n02108915 +ILSVRC2012_val_00010492.JPEG:n04209239 +ILSVRC2012_val_00010493.JPEG:n03770439 +ILSVRC2012_val_00010494.JPEG:n02226429 +ILSVRC2012_val_00010495.JPEG:n12144580 +ILSVRC2012_val_00010496.JPEG:n02906734 +ILSVRC2012_val_00010497.JPEG:n02783161 +ILSVRC2012_val_00010498.JPEG:n02667093 +ILSVRC2012_val_00010499.JPEG:n04239074 +ILSVRC2012_val_00010500.JPEG:n02110063 +ILSVRC2012_val_00010501.JPEG:n01582220 +ILSVRC2012_val_00010502.JPEG:n07768694 +ILSVRC2012_val_00010503.JPEG:n01774750 +ILSVRC2012_val_00010504.JPEG:n03787032 +ILSVRC2012_val_00010505.JPEG:n12057211 +ILSVRC2012_val_00010506.JPEG:n03764736 +ILSVRC2012_val_00010507.JPEG:n01795545 +ILSVRC2012_val_00010508.JPEG:n03623198 +ILSVRC2012_val_00010509.JPEG:n01443537 +ILSVRC2012_val_00010510.JPEG:n02892201 +ILSVRC2012_val_00010511.JPEG:n03868242 +ILSVRC2012_val_00010512.JPEG:n03384352 +ILSVRC2012_val_00010513.JPEG:n02403003 +ILSVRC2012_val_00010514.JPEG:n03658185 +ILSVRC2012_val_00010515.JPEG:n03485794 +ILSVRC2012_val_00010516.JPEG:n02085782 +ILSVRC2012_val_00010517.JPEG:n04328186 +ILSVRC2012_val_00010518.JPEG:n03388183 +ILSVRC2012_val_00010519.JPEG:n04344873 +ILSVRC2012_val_00010520.JPEG:n07716358 +ILSVRC2012_val_00010521.JPEG:n02097047 +ILSVRC2012_val_00010522.JPEG:n01737021 +ILSVRC2012_val_00010523.JPEG:n01695060 +ILSVRC2012_val_00010524.JPEG:n02098286 +ILSVRC2012_val_00010525.JPEG:n04258138 +ILSVRC2012_val_00010526.JPEG:n03127747 +ILSVRC2012_val_00010527.JPEG:n07565083 +ILSVRC2012_val_00010528.JPEG:n01667114 +ILSVRC2012_val_00010529.JPEG:n03929660 +ILSVRC2012_val_00010530.JPEG:n03476684 +ILSVRC2012_val_00010531.JPEG:n03785016 +ILSVRC2012_val_00010532.JPEG:n04041544 +ILSVRC2012_val_00010533.JPEG:n02100236 +ILSVRC2012_val_00010534.JPEG:n03854065 +ILSVRC2012_val_00010535.JPEG:n03529860 +ILSVRC2012_val_00010536.JPEG:n02097209 +ILSVRC2012_val_00010537.JPEG:n02100236 +ILSVRC2012_val_00010538.JPEG:n04540053 +ILSVRC2012_val_00010539.JPEG:n02002556 +ILSVRC2012_val_00010540.JPEG:n03495258 +ILSVRC2012_val_00010541.JPEG:n02834397 +ILSVRC2012_val_00010542.JPEG:n04346328 +ILSVRC2012_val_00010543.JPEG:n03485407 +ILSVRC2012_val_00010544.JPEG:n02835271 +ILSVRC2012_val_00010545.JPEG:n01729977 +ILSVRC2012_val_00010546.JPEG:n02802426 +ILSVRC2012_val_00010547.JPEG:n03781244 +ILSVRC2012_val_00010548.JPEG:n02793495 +ILSVRC2012_val_00010549.JPEG:n02892767 +ILSVRC2012_val_00010550.JPEG:n02086240 +ILSVRC2012_val_00010551.JPEG:n02490219 +ILSVRC2012_val_00010552.JPEG:n02119022 +ILSVRC2012_val_00010553.JPEG:n06359193 +ILSVRC2012_val_00010554.JPEG:n03207743 +ILSVRC2012_val_00010555.JPEG:n01980166 +ILSVRC2012_val_00010556.JPEG:n04467665 +ILSVRC2012_val_00010557.JPEG:n04332243 +ILSVRC2012_val_00010558.JPEG:n03598930 +ILSVRC2012_val_00010559.JPEG:n04523525 +ILSVRC2012_val_00010560.JPEG:n03877472 +ILSVRC2012_val_00010561.JPEG:n03976657 +ILSVRC2012_val_00010562.JPEG:n02256656 +ILSVRC2012_val_00010563.JPEG:n02097130 +ILSVRC2012_val_00010564.JPEG:n02606052 +ILSVRC2012_val_00010565.JPEG:n04037443 +ILSVRC2012_val_00010566.JPEG:n02793495 +ILSVRC2012_val_00010567.JPEG:n03929855 +ILSVRC2012_val_00010568.JPEG:n04118776 +ILSVRC2012_val_00010569.JPEG:n02727426 +ILSVRC2012_val_00010570.JPEG:n01833805 +ILSVRC2012_val_00010571.JPEG:n02536864 +ILSVRC2012_val_00010572.JPEG:n03710721 +ILSVRC2012_val_00010573.JPEG:n03459775 +ILSVRC2012_val_00010574.JPEG:n04311004 +ILSVRC2012_val_00010575.JPEG:n02113712 +ILSVRC2012_val_00010576.JPEG:n02480495 +ILSVRC2012_val_00010577.JPEG:n03041632 +ILSVRC2012_val_00010578.JPEG:n02966193 +ILSVRC2012_val_00010579.JPEG:n03476684 +ILSVRC2012_val_00010580.JPEG:n07716358 +ILSVRC2012_val_00010581.JPEG:n04310018 +ILSVRC2012_val_00010582.JPEG:n07579787 +ILSVRC2012_val_00010583.JPEG:n02493793 +ILSVRC2012_val_00010584.JPEG:n02094433 +ILSVRC2012_val_00010585.JPEG:n07734744 +ILSVRC2012_val_00010586.JPEG:n01744401 +ILSVRC2012_val_00010587.JPEG:n03770679 +ILSVRC2012_val_00010588.JPEG:n04523525 +ILSVRC2012_val_00010589.JPEG:n02364673 +ILSVRC2012_val_00010590.JPEG:n03355925 +ILSVRC2012_val_00010591.JPEG:n07715103 +ILSVRC2012_val_00010592.JPEG:n02403003 +ILSVRC2012_val_00010593.JPEG:n01644900 +ILSVRC2012_val_00010594.JPEG:n01518878 +ILSVRC2012_val_00010595.JPEG:n02815834 +ILSVRC2012_val_00010596.JPEG:n04251144 +ILSVRC2012_val_00010597.JPEG:n02690373 +ILSVRC2012_val_00010598.JPEG:n02124075 +ILSVRC2012_val_00010599.JPEG:n04553703 +ILSVRC2012_val_00010600.JPEG:n04081281 +ILSVRC2012_val_00010601.JPEG:n02408429 +ILSVRC2012_val_00010602.JPEG:n01704323 +ILSVRC2012_val_00010603.JPEG:n02640242 +ILSVRC2012_val_00010604.JPEG:n03478589 +ILSVRC2012_val_00010605.JPEG:n04447861 +ILSVRC2012_val_00010606.JPEG:n07875152 +ILSVRC2012_val_00010607.JPEG:n04209133 +ILSVRC2012_val_00010608.JPEG:n07734744 +ILSVRC2012_val_00010609.JPEG:n04487081 +ILSVRC2012_val_00010610.JPEG:n02177972 +ILSVRC2012_val_00010611.JPEG:n02892767 +ILSVRC2012_val_00010612.JPEG:n02113624 +ILSVRC2012_val_00010613.JPEG:n03016953 +ILSVRC2012_val_00010614.JPEG:n07753275 +ILSVRC2012_val_00010615.JPEG:n02319095 +ILSVRC2012_val_00010616.JPEG:n07745940 +ILSVRC2012_val_00010617.JPEG:n02108000 +ILSVRC2012_val_00010618.JPEG:n02028035 +ILSVRC2012_val_00010619.JPEG:n02504458 +ILSVRC2012_val_00010620.JPEG:n02106550 +ILSVRC2012_val_00010621.JPEG:n07754684 +ILSVRC2012_val_00010622.JPEG:n03063599 +ILSVRC2012_val_00010623.JPEG:n03787032 +ILSVRC2012_val_00010624.JPEG:n02098105 +ILSVRC2012_val_00010625.JPEG:n03467068 +ILSVRC2012_val_00010626.JPEG:n02089867 +ILSVRC2012_val_00010627.JPEG:n02093428 +ILSVRC2012_val_00010628.JPEG:n07718747 +ILSVRC2012_val_00010629.JPEG:n07831146 +ILSVRC2012_val_00010630.JPEG:n03496892 +ILSVRC2012_val_00010631.JPEG:n03961711 +ILSVRC2012_val_00010632.JPEG:n01924916 +ILSVRC2012_val_00010633.JPEG:n01883070 +ILSVRC2012_val_00010634.JPEG:n01704323 +ILSVRC2012_val_00010635.JPEG:n03733281 +ILSVRC2012_val_00010636.JPEG:n03791053 +ILSVRC2012_val_00010637.JPEG:n02930766 +ILSVRC2012_val_00010638.JPEG:n03478589 +ILSVRC2012_val_00010639.JPEG:n01980166 +ILSVRC2012_val_00010640.JPEG:n01985128 +ILSVRC2012_val_00010641.JPEG:n09472597 +ILSVRC2012_val_00010642.JPEG:n03967562 +ILSVRC2012_val_00010643.JPEG:n02087394 +ILSVRC2012_val_00010644.JPEG:n01914609 +ILSVRC2012_val_00010645.JPEG:n02497673 +ILSVRC2012_val_00010646.JPEG:n03924679 +ILSVRC2012_val_00010647.JPEG:n03706229 +ILSVRC2012_val_00010648.JPEG:n02108089 +ILSVRC2012_val_00010649.JPEG:n15075141 +ILSVRC2012_val_00010650.JPEG:n03977966 +ILSVRC2012_val_00010651.JPEG:n07715103 +ILSVRC2012_val_00010652.JPEG:n03187595 +ILSVRC2012_val_00010653.JPEG:n02236044 +ILSVRC2012_val_00010654.JPEG:n04599235 +ILSVRC2012_val_00010655.JPEG:n03529860 +ILSVRC2012_val_00010656.JPEG:n04023962 +ILSVRC2012_val_00010657.JPEG:n02092339 +ILSVRC2012_val_00010658.JPEG:n02977058 +ILSVRC2012_val_00010659.JPEG:n07584110 +ILSVRC2012_val_00010660.JPEG:n07730033 +ILSVRC2012_val_00010661.JPEG:n03272010 +ILSVRC2012_val_00010662.JPEG:n03676483 +ILSVRC2012_val_00010663.JPEG:n02493509 +ILSVRC2012_val_00010664.JPEG:n09468604 +ILSVRC2012_val_00010665.JPEG:n02091467 +ILSVRC2012_val_00010666.JPEG:n03534580 +ILSVRC2012_val_00010667.JPEG:n03125729 +ILSVRC2012_val_00010668.JPEG:n04467665 +ILSVRC2012_val_00010669.JPEG:n01665541 +ILSVRC2012_val_00010670.JPEG:n04330267 +ILSVRC2012_val_00010671.JPEG:n02917067 +ILSVRC2012_val_00010672.JPEG:n03196217 +ILSVRC2012_val_00010673.JPEG:n02009229 +ILSVRC2012_val_00010674.JPEG:n03042490 +ILSVRC2012_val_00010675.JPEG:n01632458 +ILSVRC2012_val_00010676.JPEG:n03100240 +ILSVRC2012_val_00010677.JPEG:n02965783 +ILSVRC2012_val_00010678.JPEG:n02172182 +ILSVRC2012_val_00010679.JPEG:n03920288 +ILSVRC2012_val_00010680.JPEG:n03109150 +ILSVRC2012_val_00010681.JPEG:n07747607 +ILSVRC2012_val_00010682.JPEG:n02093859 +ILSVRC2012_val_00010683.JPEG:n02655020 +ILSVRC2012_val_00010684.JPEG:n03658185 +ILSVRC2012_val_00010685.JPEG:n03584254 +ILSVRC2012_val_00010686.JPEG:n02110806 +ILSVRC2012_val_00010687.JPEG:n04596742 +ILSVRC2012_val_00010688.JPEG:n02113799 +ILSVRC2012_val_00010689.JPEG:n01530575 +ILSVRC2012_val_00010690.JPEG:n03345487 +ILSVRC2012_val_00010691.JPEG:n02917067 +ILSVRC2012_val_00010692.JPEG:n03788195 +ILSVRC2012_val_00010693.JPEG:n02105162 +ILSVRC2012_val_00010694.JPEG:n15075141 +ILSVRC2012_val_00010695.JPEG:n04317175 +ILSVRC2012_val_00010696.JPEG:n04251144 +ILSVRC2012_val_00010697.JPEG:n02112018 +ILSVRC2012_val_00010698.JPEG:n04326547 +ILSVRC2012_val_00010699.JPEG:n03838899 +ILSVRC2012_val_00010700.JPEG:n01955084 +ILSVRC2012_val_00010701.JPEG:n02417914 +ILSVRC2012_val_00010702.JPEG:n02099849 +ILSVRC2012_val_00010703.JPEG:n02317335 +ILSVRC2012_val_00010704.JPEG:n03095699 +ILSVRC2012_val_00010705.JPEG:n02699494 +ILSVRC2012_val_00010706.JPEG:n04554684 +ILSVRC2012_val_00010707.JPEG:n03729826 +ILSVRC2012_val_00010708.JPEG:n04005630 +ILSVRC2012_val_00010709.JPEG:n02108422 +ILSVRC2012_val_00010710.JPEG:n03127925 +ILSVRC2012_val_00010711.JPEG:n02123045 +ILSVRC2012_val_00010712.JPEG:n03832673 +ILSVRC2012_val_00010713.JPEG:n02504013 +ILSVRC2012_val_00010714.JPEG:n01806567 +ILSVRC2012_val_00010715.JPEG:n04069434 +ILSVRC2012_val_00010716.JPEG:n04023962 +ILSVRC2012_val_00010717.JPEG:n04111531 +ILSVRC2012_val_00010718.JPEG:n02097209 +ILSVRC2012_val_00010719.JPEG:n02105056 +ILSVRC2012_val_00010720.JPEG:n02097209 +ILSVRC2012_val_00010721.JPEG:n03376595 +ILSVRC2012_val_00010722.JPEG:n02095314 +ILSVRC2012_val_00010723.JPEG:n01756291 +ILSVRC2012_val_00010724.JPEG:n03773504 +ILSVRC2012_val_00010725.JPEG:n01980166 +ILSVRC2012_val_00010726.JPEG:n06794110 +ILSVRC2012_val_00010727.JPEG:n04074963 +ILSVRC2012_val_00010728.JPEG:n02747177 +ILSVRC2012_val_00010729.JPEG:n02108551 +ILSVRC2012_val_00010730.JPEG:n03255030 +ILSVRC2012_val_00010731.JPEG:n03891251 +ILSVRC2012_val_00010732.JPEG:n03935335 +ILSVRC2012_val_00010733.JPEG:n03673027 +ILSVRC2012_val_00010734.JPEG:n02111277 +ILSVRC2012_val_00010735.JPEG:n03188531 +ILSVRC2012_val_00010736.JPEG:n02100236 +ILSVRC2012_val_00010737.JPEG:n02992529 +ILSVRC2012_val_00010738.JPEG:n02607072 +ILSVRC2012_val_00010739.JPEG:n02095889 +ILSVRC2012_val_00010740.JPEG:n02002556 +ILSVRC2012_val_00010741.JPEG:n02834397 +ILSVRC2012_val_00010742.JPEG:n02134084 +ILSVRC2012_val_00010743.JPEG:n07716906 +ILSVRC2012_val_00010744.JPEG:n02804414 +ILSVRC2012_val_00010745.JPEG:n02134084 +ILSVRC2012_val_00010746.JPEG:n04008634 +ILSVRC2012_val_00010747.JPEG:n02509815 +ILSVRC2012_val_00010748.JPEG:n04254120 +ILSVRC2012_val_00010749.JPEG:n04147183 +ILSVRC2012_val_00010750.JPEG:n04204238 +ILSVRC2012_val_00010751.JPEG:n03908714 +ILSVRC2012_val_00010752.JPEG:n04162706 +ILSVRC2012_val_00010753.JPEG:n03197337 +ILSVRC2012_val_00010754.JPEG:n11879895 +ILSVRC2012_val_00010755.JPEG:n03787032 +ILSVRC2012_val_00010756.JPEG:n04111531 +ILSVRC2012_val_00010757.JPEG:n02978881 +ILSVRC2012_val_00010758.JPEG:n02102177 +ILSVRC2012_val_00010759.JPEG:n03379051 +ILSVRC2012_val_00010760.JPEG:n04371774 +ILSVRC2012_val_00010761.JPEG:n01704323 +ILSVRC2012_val_00010762.JPEG:n03710721 +ILSVRC2012_val_00010763.JPEG:n01518878 +ILSVRC2012_val_00010764.JPEG:n03016953 +ILSVRC2012_val_00010765.JPEG:n02106382 +ILSVRC2012_val_00010766.JPEG:n04540053 +ILSVRC2012_val_00010767.JPEG:n01558993 +ILSVRC2012_val_00010768.JPEG:n02105412 +ILSVRC2012_val_00010769.JPEG:n02981792 +ILSVRC2012_val_00010770.JPEG:n03028079 +ILSVRC2012_val_00010771.JPEG:n03782006 +ILSVRC2012_val_00010772.JPEG:n02086079 +ILSVRC2012_val_00010773.JPEG:n04192698 +ILSVRC2012_val_00010774.JPEG:n02233338 +ILSVRC2012_val_00010775.JPEG:n03649909 +ILSVRC2012_val_00010776.JPEG:n03496892 +ILSVRC2012_val_00010777.JPEG:n02276258 +ILSVRC2012_val_00010778.JPEG:n03832673 +ILSVRC2012_val_00010779.JPEG:n04070727 +ILSVRC2012_val_00010780.JPEG:n03899768 +ILSVRC2012_val_00010781.JPEG:n03017168 +ILSVRC2012_val_00010782.JPEG:n03485794 +ILSVRC2012_val_00010783.JPEG:n04591157 +ILSVRC2012_val_00010784.JPEG:n02493509 +ILSVRC2012_val_00010785.JPEG:n02093754 +ILSVRC2012_val_00010786.JPEG:n02107683 +ILSVRC2012_val_00010787.JPEG:n04208210 +ILSVRC2012_val_00010788.JPEG:n02992529 +ILSVRC2012_val_00010789.JPEG:n03124043 +ILSVRC2012_val_00010790.JPEG:n03876231 +ILSVRC2012_val_00010791.JPEG:n03691459 +ILSVRC2012_val_00010792.JPEG:n01667778 +ILSVRC2012_val_00010793.JPEG:n07730033 +ILSVRC2012_val_00010794.JPEG:n04252225 +ILSVRC2012_val_00010795.JPEG:n04208210 +ILSVRC2012_val_00010796.JPEG:n02860847 +ILSVRC2012_val_00010797.JPEG:n01742172 +ILSVRC2012_val_00010798.JPEG:n02094114 +ILSVRC2012_val_00010799.JPEG:n03000134 +ILSVRC2012_val_00010800.JPEG:n07860988 +ILSVRC2012_val_00010801.JPEG:n01775062 +ILSVRC2012_val_00010802.JPEG:n03958227 +ILSVRC2012_val_00010803.JPEG:n03045698 +ILSVRC2012_val_00010804.JPEG:n03759954 +ILSVRC2012_val_00010805.JPEG:n02086240 +ILSVRC2012_val_00010806.JPEG:n03676483 +ILSVRC2012_val_00010807.JPEG:n04532670 +ILSVRC2012_val_00010808.JPEG:n02100583 +ILSVRC2012_val_00010809.JPEG:n02793495 +ILSVRC2012_val_00010810.JPEG:n01855032 +ILSVRC2012_val_00010811.JPEG:n04275548 +ILSVRC2012_val_00010812.JPEG:n04409515 +ILSVRC2012_val_00010813.JPEG:n03733131 +ILSVRC2012_val_00010814.JPEG:n03710193 +ILSVRC2012_val_00010815.JPEG:n07760859 +ILSVRC2012_val_00010816.JPEG:n03854065 +ILSVRC2012_val_00010817.JPEG:n01629819 +ILSVRC2012_val_00010818.JPEG:n02840245 +ILSVRC2012_val_00010819.JPEG:n03691459 +ILSVRC2012_val_00010820.JPEG:n03452741 +ILSVRC2012_val_00010821.JPEG:n03297495 +ILSVRC2012_val_00010822.JPEG:n03877472 +ILSVRC2012_val_00010823.JPEG:n02125311 +ILSVRC2012_val_00010824.JPEG:n04037443 +ILSVRC2012_val_00010825.JPEG:n02526121 +ILSVRC2012_val_00010826.JPEG:n01698640 +ILSVRC2012_val_00010827.JPEG:n04591713 +ILSVRC2012_val_00010828.JPEG:n02860847 +ILSVRC2012_val_00010829.JPEG:n02412080 +ILSVRC2012_val_00010830.JPEG:n01728572 +ILSVRC2012_val_00010831.JPEG:n04152593 +ILSVRC2012_val_00010832.JPEG:n02879718 +ILSVRC2012_val_00010833.JPEG:n02699494 +ILSVRC2012_val_00010834.JPEG:n02115913 +ILSVRC2012_val_00010835.JPEG:n03000134 +ILSVRC2012_val_00010836.JPEG:n02326432 +ILSVRC2012_val_00010837.JPEG:n02966193 +ILSVRC2012_val_00010838.JPEG:n04326547 +ILSVRC2012_val_00010839.JPEG:n04049303 +ILSVRC2012_val_00010840.JPEG:n04501370 +ILSVRC2012_val_00010841.JPEG:n07590611 +ILSVRC2012_val_00010842.JPEG:n02088466 +ILSVRC2012_val_00010843.JPEG:n01665541 +ILSVRC2012_val_00010844.JPEG:n03141823 +ILSVRC2012_val_00010845.JPEG:n02037110 +ILSVRC2012_val_00010846.JPEG:n02110958 +ILSVRC2012_val_00010847.JPEG:n03481172 +ILSVRC2012_val_00010848.JPEG:n07860988 +ILSVRC2012_val_00010849.JPEG:n02509815 +ILSVRC2012_val_00010850.JPEG:n02869837 +ILSVRC2012_val_00010851.JPEG:n03930313 +ILSVRC2012_val_00010852.JPEG:n03492542 +ILSVRC2012_val_00010853.JPEG:n02480855 +ILSVRC2012_val_00010854.JPEG:n02486261 +ILSVRC2012_val_00010855.JPEG:n03495258 +ILSVRC2012_val_00010856.JPEG:n03478589 +ILSVRC2012_val_00010857.JPEG:n03063599 +ILSVRC2012_val_00010858.JPEG:n04525038 +ILSVRC2012_val_00010859.JPEG:n02109525 +ILSVRC2012_val_00010860.JPEG:n02787622 +ILSVRC2012_val_00010861.JPEG:n01592084 +ILSVRC2012_val_00010862.JPEG:n02437616 +ILSVRC2012_val_00010863.JPEG:n13040303 +ILSVRC2012_val_00010864.JPEG:n04118776 +ILSVRC2012_val_00010865.JPEG:n02104365 +ILSVRC2012_val_00010866.JPEG:n02927161 +ILSVRC2012_val_00010867.JPEG:n03532672 +ILSVRC2012_val_00010868.JPEG:n03814639 +ILSVRC2012_val_00010869.JPEG:n01910747 +ILSVRC2012_val_00010870.JPEG:n01737021 +ILSVRC2012_val_00010871.JPEG:n03877845 +ILSVRC2012_val_00010872.JPEG:n07579787 +ILSVRC2012_val_00010873.JPEG:n09288635 +ILSVRC2012_val_00010874.JPEG:n01981276 +ILSVRC2012_val_00010875.JPEG:n03133878 +ILSVRC2012_val_00010876.JPEG:n02667093 +ILSVRC2012_val_00010877.JPEG:n02747177 +ILSVRC2012_val_00010878.JPEG:n02500267 +ILSVRC2012_val_00010879.JPEG:n04370456 +ILSVRC2012_val_00010880.JPEG:n01601694 +ILSVRC2012_val_00010881.JPEG:n03769881 +ILSVRC2012_val_00010882.JPEG:n04372370 +ILSVRC2012_val_00010883.JPEG:n02114712 +ILSVRC2012_val_00010884.JPEG:n02326432 +ILSVRC2012_val_00010885.JPEG:n03134739 +ILSVRC2012_val_00010886.JPEG:n03041632 +ILSVRC2012_val_00010887.JPEG:n01685808 +ILSVRC2012_val_00010888.JPEG:n02233338 +ILSVRC2012_val_00010889.JPEG:n01614925 +ILSVRC2012_val_00010890.JPEG:n03982430 +ILSVRC2012_val_00010891.JPEG:n03929855 +ILSVRC2012_val_00010892.JPEG:n04069434 +ILSVRC2012_val_00010893.JPEG:n04367480 +ILSVRC2012_val_00010894.JPEG:n03961711 +ILSVRC2012_val_00010895.JPEG:n03201208 +ILSVRC2012_val_00010896.JPEG:n02092002 +ILSVRC2012_val_00010897.JPEG:n04370456 +ILSVRC2012_val_00010898.JPEG:n04376876 +ILSVRC2012_val_00010899.JPEG:n02395406 +ILSVRC2012_val_00010900.JPEG:n03717622 +ILSVRC2012_val_00010901.JPEG:n04317175 +ILSVRC2012_val_00010902.JPEG:n02088094 +ILSVRC2012_val_00010903.JPEG:n02950826 +ILSVRC2012_val_00010904.JPEG:n01697457 +ILSVRC2012_val_00010905.JPEG:n04591157 +ILSVRC2012_val_00010906.JPEG:n01784675 +ILSVRC2012_val_00010907.JPEG:n03930630 +ILSVRC2012_val_00010908.JPEG:n04251144 +ILSVRC2012_val_00010909.JPEG:n02802426 +ILSVRC2012_val_00010910.JPEG:n07697537 +ILSVRC2012_val_00010911.JPEG:n01689811 +ILSVRC2012_val_00010912.JPEG:n12998815 +ILSVRC2012_val_00010913.JPEG:n04550184 +ILSVRC2012_val_00010914.JPEG:n04486054 +ILSVRC2012_val_00010915.JPEG:n01667778 +ILSVRC2012_val_00010916.JPEG:n03916031 +ILSVRC2012_val_00010917.JPEG:n01795545 +ILSVRC2012_val_00010918.JPEG:n02790996 +ILSVRC2012_val_00010919.JPEG:n01910747 +ILSVRC2012_val_00010920.JPEG:n02085936 +ILSVRC2012_val_00010921.JPEG:n03938244 +ILSVRC2012_val_00010922.JPEG:n03976467 +ILSVRC2012_val_00010923.JPEG:n02325366 +ILSVRC2012_val_00010924.JPEG:n03527444 +ILSVRC2012_val_00010925.JPEG:n02268443 +ILSVRC2012_val_00010926.JPEG:n03290653 +ILSVRC2012_val_00010927.JPEG:n03444034 +ILSVRC2012_val_00010928.JPEG:n02105056 +ILSVRC2012_val_00010929.JPEG:n02096437 +ILSVRC2012_val_00010930.JPEG:n03457902 +ILSVRC2012_val_00010931.JPEG:n03843555 +ILSVRC2012_val_00010932.JPEG:n02500267 +ILSVRC2012_val_00010933.JPEG:n02088094 +ILSVRC2012_val_00010934.JPEG:n02769748 +ILSVRC2012_val_00010935.JPEG:n04525038 +ILSVRC2012_val_00010936.JPEG:n02606052 +ILSVRC2012_val_00010937.JPEG:n04487081 +ILSVRC2012_val_00010938.JPEG:n02486261 +ILSVRC2012_val_00010939.JPEG:n03492542 +ILSVRC2012_val_00010940.JPEG:n03733131 +ILSVRC2012_val_00010941.JPEG:n02120505 +ILSVRC2012_val_00010942.JPEG:n07745940 +ILSVRC2012_val_00010943.JPEG:n02112137 +ILSVRC2012_val_00010944.JPEG:n07579787 +ILSVRC2012_val_00010945.JPEG:n02105505 +ILSVRC2012_val_00010946.JPEG:n03452741 +ILSVRC2012_val_00010947.JPEG:n10148035 +ILSVRC2012_val_00010948.JPEG:n04125021 +ILSVRC2012_val_00010949.JPEG:n04026417 +ILSVRC2012_val_00010950.JPEG:n02089867 +ILSVRC2012_val_00010951.JPEG:n03995372 +ILSVRC2012_val_00010952.JPEG:n02177972 +ILSVRC2012_val_00010953.JPEG:n03903868 +ILSVRC2012_val_00010954.JPEG:n04409515 +ILSVRC2012_val_00010955.JPEG:n01943899 +ILSVRC2012_val_00010956.JPEG:n02100236 +ILSVRC2012_val_00010957.JPEG:n03124170 +ILSVRC2012_val_00010958.JPEG:n03197337 +ILSVRC2012_val_00010959.JPEG:n02361337 +ILSVRC2012_val_00010960.JPEG:n04325704 +ILSVRC2012_val_00010961.JPEG:n03920288 +ILSVRC2012_val_00010962.JPEG:n03825788 +ILSVRC2012_val_00010963.JPEG:n02101388 +ILSVRC2012_val_00010964.JPEG:n11879895 +ILSVRC2012_val_00010965.JPEG:n03443371 +ILSVRC2012_val_00010966.JPEG:n02071294 +ILSVRC2012_val_00010967.JPEG:n07880968 +ILSVRC2012_val_00010968.JPEG:n03769881 +ILSVRC2012_val_00010969.JPEG:n03902125 +ILSVRC2012_val_00010970.JPEG:n02110806 +ILSVRC2012_val_00010971.JPEG:n03637318 +ILSVRC2012_val_00010972.JPEG:n04019541 +ILSVRC2012_val_00010973.JPEG:n03840681 +ILSVRC2012_val_00010974.JPEG:n02342885 +ILSVRC2012_val_00010975.JPEG:n03476684 +ILSVRC2012_val_00010976.JPEG:n02094114 +ILSVRC2012_val_00010977.JPEG:n04023962 +ILSVRC2012_val_00010978.JPEG:n03706229 +ILSVRC2012_val_00010979.JPEG:n02730930 +ILSVRC2012_val_00010980.JPEG:n02877765 +ILSVRC2012_val_00010981.JPEG:n04548362 +ILSVRC2012_val_00010982.JPEG:n02088632 +ILSVRC2012_val_00010983.JPEG:n04285008 +ILSVRC2012_val_00010984.JPEG:n07873807 +ILSVRC2012_val_00010985.JPEG:n03903868 +ILSVRC2012_val_00010986.JPEG:n04501370 +ILSVRC2012_val_00010987.JPEG:n04118538 +ILSVRC2012_val_00010988.JPEG:n02025239 +ILSVRC2012_val_00010989.JPEG:n03530642 +ILSVRC2012_val_00010990.JPEG:n02018207 +ILSVRC2012_val_00010991.JPEG:n03476684 +ILSVRC2012_val_00010992.JPEG:n03602883 +ILSVRC2012_val_00010993.JPEG:n02948072 +ILSVRC2012_val_00010994.JPEG:n02102040 +ILSVRC2012_val_00010995.JPEG:n02123394 +ILSVRC2012_val_00010996.JPEG:n01944390 +ILSVRC2012_val_00010997.JPEG:n02268853 +ILSVRC2012_val_00010998.JPEG:n04590129 +ILSVRC2012_val_00010999.JPEG:n01530575 +ILSVRC2012_val_00011000.JPEG:n02117135 +ILSVRC2012_val_00011001.JPEG:n03691459 +ILSVRC2012_val_00011002.JPEG:n02504013 +ILSVRC2012_val_00011003.JPEG:n03179701 +ILSVRC2012_val_00011004.JPEG:n04357314 +ILSVRC2012_val_00011005.JPEG:n04399382 +ILSVRC2012_val_00011006.JPEG:n03218198 +ILSVRC2012_val_00011007.JPEG:n02865351 +ILSVRC2012_val_00011008.JPEG:n03598930 +ILSVRC2012_val_00011009.JPEG:n02113978 +ILSVRC2012_val_00011010.JPEG:n03697007 +ILSVRC2012_val_00011011.JPEG:n01843383 +ILSVRC2012_val_00011012.JPEG:n02074367 +ILSVRC2012_val_00011013.JPEG:n02264363 +ILSVRC2012_val_00011014.JPEG:n01742172 +ILSVRC2012_val_00011015.JPEG:n02123045 +ILSVRC2012_val_00011016.JPEG:n02795169 +ILSVRC2012_val_00011017.JPEG:n03721384 +ILSVRC2012_val_00011018.JPEG:n02129165 +ILSVRC2012_val_00011019.JPEG:n03544143 +ILSVRC2012_val_00011020.JPEG:n04522168 +ILSVRC2012_val_00011021.JPEG:n12985857 +ILSVRC2012_val_00011022.JPEG:n02814860 +ILSVRC2012_val_00011023.JPEG:n02110958 +ILSVRC2012_val_00011024.JPEG:n02100735 +ILSVRC2012_val_00011025.JPEG:n13044778 +ILSVRC2012_val_00011026.JPEG:n02817516 +ILSVRC2012_val_00011027.JPEG:n07730033 +ILSVRC2012_val_00011028.JPEG:n04429376 +ILSVRC2012_val_00011029.JPEG:n04033995 +ILSVRC2012_val_00011030.JPEG:n04367480 +ILSVRC2012_val_00011031.JPEG:n03729826 +ILSVRC2012_val_00011032.JPEG:n02493793 +ILSVRC2012_val_00011033.JPEG:n04141975 +ILSVRC2012_val_00011034.JPEG:n01740131 +ILSVRC2012_val_00011035.JPEG:n01914609 +ILSVRC2012_val_00011036.JPEG:n02134418 +ILSVRC2012_val_00011037.JPEG:n01739381 +ILSVRC2012_val_00011038.JPEG:n02687172 +ILSVRC2012_val_00011039.JPEG:n02483362 +ILSVRC2012_val_00011040.JPEG:n13037406 +ILSVRC2012_val_00011041.JPEG:n01742172 +ILSVRC2012_val_00011042.JPEG:n02396427 +ILSVRC2012_val_00011043.JPEG:n02397096 +ILSVRC2012_val_00011044.JPEG:n01689811 +ILSVRC2012_val_00011045.JPEG:n09399592 +ILSVRC2012_val_00011046.JPEG:n04347754 +ILSVRC2012_val_00011047.JPEG:n02865351 +ILSVRC2012_val_00011048.JPEG:n04344873 +ILSVRC2012_val_00011049.JPEG:n02111889 +ILSVRC2012_val_00011050.JPEG:n02939185 +ILSVRC2012_val_00011051.JPEG:n04033995 +ILSVRC2012_val_00011052.JPEG:n02037110 +ILSVRC2012_val_00011053.JPEG:n01773157 +ILSVRC2012_val_00011054.JPEG:n03599486 +ILSVRC2012_val_00011055.JPEG:n02093647 +ILSVRC2012_val_00011056.JPEG:n01532829 +ILSVRC2012_val_00011057.JPEG:n02097209 +ILSVRC2012_val_00011058.JPEG:n02492660 +ILSVRC2012_val_00011059.JPEG:n04009552 +ILSVRC2012_val_00011060.JPEG:n04033901 +ILSVRC2012_val_00011061.JPEG:n02099429 +ILSVRC2012_val_00011062.JPEG:n02056570 +ILSVRC2012_val_00011063.JPEG:n02098413 +ILSVRC2012_val_00011064.JPEG:n02992211 +ILSVRC2012_val_00011065.JPEG:n03788195 +ILSVRC2012_val_00011066.JPEG:n03207743 +ILSVRC2012_val_00011067.JPEG:n03444034 +ILSVRC2012_val_00011068.JPEG:n03814639 +ILSVRC2012_val_00011069.JPEG:n04485082 +ILSVRC2012_val_00011070.JPEG:n01981276 +ILSVRC2012_val_00011071.JPEG:n01978455 +ILSVRC2012_val_00011072.JPEG:n03461385 +ILSVRC2012_val_00011073.JPEG:n01688243 +ILSVRC2012_val_00011074.JPEG:n02277742 +ILSVRC2012_val_00011075.JPEG:n03388043 +ILSVRC2012_val_00011076.JPEG:n02871525 +ILSVRC2012_val_00011077.JPEG:n02101556 +ILSVRC2012_val_00011078.JPEG:n03131574 +ILSVRC2012_val_00011079.JPEG:n02236044 +ILSVRC2012_val_00011080.JPEG:n07248320 +ILSVRC2012_val_00011081.JPEG:n03041632 +ILSVRC2012_val_00011082.JPEG:n02095314 +ILSVRC2012_val_00011083.JPEG:n04344873 +ILSVRC2012_val_00011084.JPEG:n02119022 +ILSVRC2012_val_00011085.JPEG:n02172182 +ILSVRC2012_val_00011086.JPEG:n13054560 +ILSVRC2012_val_00011087.JPEG:n01978287 +ILSVRC2012_val_00011088.JPEG:n03532672 +ILSVRC2012_val_00011089.JPEG:n04536866 +ILSVRC2012_val_00011090.JPEG:n02105412 +ILSVRC2012_val_00011091.JPEG:n04118538 +ILSVRC2012_val_00011092.JPEG:n02443484 +ILSVRC2012_val_00011093.JPEG:n01695060 +ILSVRC2012_val_00011094.JPEG:n02909870 +ILSVRC2012_val_00011095.JPEG:n02441942 +ILSVRC2012_val_00011096.JPEG:n02017213 +ILSVRC2012_val_00011097.JPEG:n02799071 +ILSVRC2012_val_00011098.JPEG:n04147183 +ILSVRC2012_val_00011099.JPEG:n04589890 +ILSVRC2012_val_00011100.JPEG:n02056570 +ILSVRC2012_val_00011101.JPEG:n02486261 +ILSVRC2012_val_00011102.JPEG:n03345487 +ILSVRC2012_val_00011103.JPEG:n04328186 +ILSVRC2012_val_00011104.JPEG:n02328150 +ILSVRC2012_val_00011105.JPEG:n04476259 +ILSVRC2012_val_00011106.JPEG:n04346328 +ILSVRC2012_val_00011107.JPEG:n04273569 +ILSVRC2012_val_00011108.JPEG:n03290653 +ILSVRC2012_val_00011109.JPEG:n03627232 +ILSVRC2012_val_00011110.JPEG:n02791124 +ILSVRC2012_val_00011111.JPEG:n02012849 +ILSVRC2012_val_00011112.JPEG:n02259212 +ILSVRC2012_val_00011113.JPEG:n02090379 +ILSVRC2012_val_00011114.JPEG:n03627232 +ILSVRC2012_val_00011115.JPEG:n03764736 +ILSVRC2012_val_00011116.JPEG:n02817516 +ILSVRC2012_val_00011117.JPEG:n04326547 +ILSVRC2012_val_00011118.JPEG:n03065424 +ILSVRC2012_val_00011119.JPEG:n02909870 +ILSVRC2012_val_00011120.JPEG:n01675722 +ILSVRC2012_val_00011121.JPEG:n04522168 +ILSVRC2012_val_00011122.JPEG:n13133613 +ILSVRC2012_val_00011123.JPEG:n02655020 +ILSVRC2012_val_00011124.JPEG:n04209133 +ILSVRC2012_val_00011125.JPEG:n02783161 +ILSVRC2012_val_00011126.JPEG:n03796401 +ILSVRC2012_val_00011127.JPEG:n03250847 +ILSVRC2012_val_00011128.JPEG:n01872401 +ILSVRC2012_val_00011129.JPEG:n01682714 +ILSVRC2012_val_00011130.JPEG:n01873310 +ILSVRC2012_val_00011131.JPEG:n01631663 +ILSVRC2012_val_00011132.JPEG:n04005630 +ILSVRC2012_val_00011133.JPEG:n02843684 +ILSVRC2012_val_00011134.JPEG:n02769748 +ILSVRC2012_val_00011135.JPEG:n02804610 +ILSVRC2012_val_00011136.JPEG:n03782006 +ILSVRC2012_val_00011137.JPEG:n01978455 +ILSVRC2012_val_00011138.JPEG:n02097298 +ILSVRC2012_val_00011139.JPEG:n02787622 +ILSVRC2012_val_00011140.JPEG:n07716906 +ILSVRC2012_val_00011141.JPEG:n02111129 +ILSVRC2012_val_00011142.JPEG:n02123045 +ILSVRC2012_val_00011143.JPEG:n02279972 +ILSVRC2012_val_00011144.JPEG:n02497673 +ILSVRC2012_val_00011145.JPEG:n02980441 +ILSVRC2012_val_00011146.JPEG:n02111129 +ILSVRC2012_val_00011147.JPEG:n03297495 +ILSVRC2012_val_00011148.JPEG:n04487081 +ILSVRC2012_val_00011149.JPEG:n04370456 +ILSVRC2012_val_00011150.JPEG:n01667778 +ILSVRC2012_val_00011151.JPEG:n03710193 +ILSVRC2012_val_00011152.JPEG:n02096294 +ILSVRC2012_val_00011153.JPEG:n03876231 +ILSVRC2012_val_00011154.JPEG:n03938244 +ILSVRC2012_val_00011155.JPEG:n02950826 +ILSVRC2012_val_00011156.JPEG:n04311174 +ILSVRC2012_val_00011157.JPEG:n04081281 +ILSVRC2012_val_00011158.JPEG:n01687978 +ILSVRC2012_val_00011159.JPEG:n04371774 +ILSVRC2012_val_00011160.JPEG:n06794110 +ILSVRC2012_val_00011161.JPEG:n02281406 +ILSVRC2012_val_00011162.JPEG:n04326547 +ILSVRC2012_val_00011163.JPEG:n02395406 +ILSVRC2012_val_00011164.JPEG:n02096051 +ILSVRC2012_val_00011165.JPEG:n02113186 +ILSVRC2012_val_00011166.JPEG:n04070727 +ILSVRC2012_val_00011167.JPEG:n02206856 +ILSVRC2012_val_00011168.JPEG:n02690373 +ILSVRC2012_val_00011169.JPEG:n01729977 +ILSVRC2012_val_00011170.JPEG:n03000684 +ILSVRC2012_val_00011171.JPEG:n01514859 +ILSVRC2012_val_00011172.JPEG:n03197337 +ILSVRC2012_val_00011173.JPEG:n03445924 +ILSVRC2012_val_00011174.JPEG:n04604644 +ILSVRC2012_val_00011175.JPEG:n02280649 +ILSVRC2012_val_00011176.JPEG:n02090379 +ILSVRC2012_val_00011177.JPEG:n02012849 +ILSVRC2012_val_00011178.JPEG:n01534433 +ILSVRC2012_val_00011179.JPEG:n07734744 +ILSVRC2012_val_00011180.JPEG:n03838899 +ILSVRC2012_val_00011181.JPEG:n02177972 +ILSVRC2012_val_00011182.JPEG:n04423845 +ILSVRC2012_val_00011183.JPEG:n03899768 +ILSVRC2012_val_00011184.JPEG:n02098105 +ILSVRC2012_val_00011185.JPEG:n03633091 +ILSVRC2012_val_00011186.JPEG:n02701002 +ILSVRC2012_val_00011187.JPEG:n04371430 +ILSVRC2012_val_00011188.JPEG:n02114367 +ILSVRC2012_val_00011189.JPEG:n03947888 +ILSVRC2012_val_00011190.JPEG:n01820546 +ILSVRC2012_val_00011191.JPEG:n02088238 +ILSVRC2012_val_00011192.JPEG:n03929855 +ILSVRC2012_val_00011193.JPEG:n04612504 +ILSVRC2012_val_00011194.JPEG:n02963159 +ILSVRC2012_val_00011195.JPEG:n02966193 +ILSVRC2012_val_00011196.JPEG:n02037110 +ILSVRC2012_val_00011197.JPEG:n03982430 +ILSVRC2012_val_00011198.JPEG:n02107574 +ILSVRC2012_val_00011199.JPEG:n02966193 +ILSVRC2012_val_00011200.JPEG:n04355933 +ILSVRC2012_val_00011201.JPEG:n03372029 +ILSVRC2012_val_00011202.JPEG:n02113978 +ILSVRC2012_val_00011203.JPEG:n04398044 +ILSVRC2012_val_00011204.JPEG:n02087046 +ILSVRC2012_val_00011205.JPEG:n02106166 +ILSVRC2012_val_00011206.JPEG:n04465501 +ILSVRC2012_val_00011207.JPEG:n03179701 +ILSVRC2012_val_00011208.JPEG:n10565667 +ILSVRC2012_val_00011209.JPEG:n03492542 +ILSVRC2012_val_00011210.JPEG:n01735189 +ILSVRC2012_val_00011211.JPEG:n02120079 +ILSVRC2012_val_00011212.JPEG:n02105251 +ILSVRC2012_val_00011213.JPEG:n01873310 +ILSVRC2012_val_00011214.JPEG:n02110063 +ILSVRC2012_val_00011215.JPEG:n03388183 +ILSVRC2012_val_00011216.JPEG:n02444819 +ILSVRC2012_val_00011217.JPEG:n02687172 +ILSVRC2012_val_00011218.JPEG:n01871265 +ILSVRC2012_val_00011219.JPEG:n02445715 +ILSVRC2012_val_00011220.JPEG:n04590129 +ILSVRC2012_val_00011221.JPEG:n12985857 +ILSVRC2012_val_00011222.JPEG:n01819313 +ILSVRC2012_val_00011223.JPEG:n03938244 +ILSVRC2012_val_00011224.JPEG:n02443114 +ILSVRC2012_val_00011225.JPEG:n04380533 +ILSVRC2012_val_00011226.JPEG:n04277352 +ILSVRC2012_val_00011227.JPEG:n02444819 +ILSVRC2012_val_00011228.JPEG:n02536864 +ILSVRC2012_val_00011229.JPEG:n02111277 +ILSVRC2012_val_00011230.JPEG:n02948072 +ILSVRC2012_val_00011231.JPEG:n03938244 +ILSVRC2012_val_00011232.JPEG:n07753113 +ILSVRC2012_val_00011233.JPEG:n01440764 +ILSVRC2012_val_00011234.JPEG:n09193705 +ILSVRC2012_val_00011235.JPEG:n02509815 +ILSVRC2012_val_00011236.JPEG:n01770393 +ILSVRC2012_val_00011237.JPEG:n01828970 +ILSVRC2012_val_00011238.JPEG:n03794056 +ILSVRC2012_val_00011239.JPEG:n03902125 +ILSVRC2012_val_00011240.JPEG:n02097474 +ILSVRC2012_val_00011241.JPEG:n07714571 +ILSVRC2012_val_00011242.JPEG:n02107908 +ILSVRC2012_val_00011243.JPEG:n01698640 +ILSVRC2012_val_00011244.JPEG:n04590129 +ILSVRC2012_val_00011245.JPEG:n02481823 +ILSVRC2012_val_00011246.JPEG:n04418357 +ILSVRC2012_val_00011247.JPEG:n02504013 +ILSVRC2012_val_00011248.JPEG:n02815834 +ILSVRC2012_val_00011249.JPEG:n01530575 +ILSVRC2012_val_00011250.JPEG:n03131574 +ILSVRC2012_val_00011251.JPEG:n02104365 +ILSVRC2012_val_00011252.JPEG:n04204238 +ILSVRC2012_val_00011253.JPEG:n02454379 +ILSVRC2012_val_00011254.JPEG:n04147183 +ILSVRC2012_val_00011255.JPEG:n02077923 +ILSVRC2012_val_00011256.JPEG:n02488291 +ILSVRC2012_val_00011257.JPEG:n02342885 +ILSVRC2012_val_00011258.JPEG:n02097474 +ILSVRC2012_val_00011259.JPEG:n07716358 +ILSVRC2012_val_00011260.JPEG:n03337140 +ILSVRC2012_val_00011261.JPEG:n04417672 +ILSVRC2012_val_00011262.JPEG:n01694178 +ILSVRC2012_val_00011263.JPEG:n04311004 +ILSVRC2012_val_00011264.JPEG:n06785654 +ILSVRC2012_val_00011265.JPEG:n07768694 +ILSVRC2012_val_00011266.JPEG:n04149813 +ILSVRC2012_val_00011267.JPEG:n01560419 +ILSVRC2012_val_00011268.JPEG:n03970156 +ILSVRC2012_val_00011269.JPEG:n04125021 +ILSVRC2012_val_00011270.JPEG:n09428293 +ILSVRC2012_val_00011271.JPEG:n04258138 +ILSVRC2012_val_00011272.JPEG:n03720891 +ILSVRC2012_val_00011273.JPEG:n04086273 +ILSVRC2012_val_00011274.JPEG:n02804610 +ILSVRC2012_val_00011275.JPEG:n03642806 +ILSVRC2012_val_00011276.JPEG:n03133878 +ILSVRC2012_val_00011277.JPEG:n02974003 +ILSVRC2012_val_00011278.JPEG:n01629819 +ILSVRC2012_val_00011279.JPEG:n03983396 +ILSVRC2012_val_00011280.JPEG:n04154565 +ILSVRC2012_val_00011281.JPEG:n02483362 +ILSVRC2012_val_00011282.JPEG:n04019541 +ILSVRC2012_val_00011283.JPEG:n03065424 +ILSVRC2012_val_00011284.JPEG:n04040759 +ILSVRC2012_val_00011285.JPEG:n06596364 +ILSVRC2012_val_00011286.JPEG:n04131690 +ILSVRC2012_val_00011287.JPEG:n01770393 +ILSVRC2012_val_00011288.JPEG:n04550184 +ILSVRC2012_val_00011289.JPEG:n02120079 +ILSVRC2012_val_00011290.JPEG:n03255030 +ILSVRC2012_val_00011291.JPEG:n02326432 +ILSVRC2012_val_00011292.JPEG:n03344393 +ILSVRC2012_val_00011293.JPEG:n12985857 +ILSVRC2012_val_00011294.JPEG:n01675722 +ILSVRC2012_val_00011295.JPEG:n01729322 +ILSVRC2012_val_00011296.JPEG:n02112137 +ILSVRC2012_val_00011297.JPEG:n04398044 +ILSVRC2012_val_00011298.JPEG:n02013706 +ILSVRC2012_val_00011299.JPEG:n04162706 +ILSVRC2012_val_00011300.JPEG:n04069434 +ILSVRC2012_val_00011301.JPEG:n03630383 +ILSVRC2012_val_00011302.JPEG:n02840245 +ILSVRC2012_val_00011303.JPEG:n01644900 +ILSVRC2012_val_00011304.JPEG:n03680355 +ILSVRC2012_val_00011305.JPEG:n04229816 +ILSVRC2012_val_00011306.JPEG:n09193705 +ILSVRC2012_val_00011307.JPEG:n02788148 +ILSVRC2012_val_00011308.JPEG:n04462240 +ILSVRC2012_val_00011309.JPEG:n03775546 +ILSVRC2012_val_00011310.JPEG:n06596364 +ILSVRC2012_val_00011311.JPEG:n02090721 +ILSVRC2012_val_00011312.JPEG:n03388183 +ILSVRC2012_val_00011313.JPEG:n04252077 +ILSVRC2012_val_00011314.JPEG:n03042490 +ILSVRC2012_val_00011315.JPEG:n01843065 +ILSVRC2012_val_00011316.JPEG:n02111129 +ILSVRC2012_val_00011317.JPEG:n01616318 +ILSVRC2012_val_00011318.JPEG:n04409515 +ILSVRC2012_val_00011319.JPEG:n10148035 +ILSVRC2012_val_00011320.JPEG:n01677366 +ILSVRC2012_val_00011321.JPEG:n02655020 +ILSVRC2012_val_00011322.JPEG:n02107683 +ILSVRC2012_val_00011323.JPEG:n02105162 +ILSVRC2012_val_00011324.JPEG:n03888257 +ILSVRC2012_val_00011325.JPEG:n02128925 +ILSVRC2012_val_00011326.JPEG:n03868863 +ILSVRC2012_val_00011327.JPEG:n04069434 +ILSVRC2012_val_00011328.JPEG:n01773797 +ILSVRC2012_val_00011329.JPEG:n03792782 +ILSVRC2012_val_00011330.JPEG:n03792782 +ILSVRC2012_val_00011331.JPEG:n01560419 +ILSVRC2012_val_00011332.JPEG:n07742313 +ILSVRC2012_val_00011333.JPEG:n13054560 +ILSVRC2012_val_00011334.JPEG:n02981792 +ILSVRC2012_val_00011335.JPEG:n03916031 +ILSVRC2012_val_00011336.JPEG:n03623198 +ILSVRC2012_val_00011337.JPEG:n04146614 +ILSVRC2012_val_00011338.JPEG:n11879895 +ILSVRC2012_val_00011339.JPEG:n01675722 +ILSVRC2012_val_00011340.JPEG:n02097130 +ILSVRC2012_val_00011341.JPEG:n04423845 +ILSVRC2012_val_00011342.JPEG:n02089973 +ILSVRC2012_val_00011343.JPEG:n04592741 +ILSVRC2012_val_00011344.JPEG:n01968897 +ILSVRC2012_val_00011345.JPEG:n07718747 +ILSVRC2012_val_00011346.JPEG:n02992529 +ILSVRC2012_val_00011347.JPEG:n07753275 +ILSVRC2012_val_00011348.JPEG:n07745940 +ILSVRC2012_val_00011349.JPEG:n02108422 +ILSVRC2012_val_00011350.JPEG:n02804414 +ILSVRC2012_val_00011351.JPEG:n02342885 +ILSVRC2012_val_00011352.JPEG:n03379051 +ILSVRC2012_val_00011353.JPEG:n02457408 +ILSVRC2012_val_00011354.JPEG:n02437312 +ILSVRC2012_val_00011355.JPEG:n03787032 +ILSVRC2012_val_00011356.JPEG:n02091032 +ILSVRC2012_val_00011357.JPEG:n02002556 +ILSVRC2012_val_00011358.JPEG:n03666591 +ILSVRC2012_val_00011359.JPEG:n03717622 +ILSVRC2012_val_00011360.JPEG:n07831146 +ILSVRC2012_val_00011361.JPEG:n03208938 +ILSVRC2012_val_00011362.JPEG:n02840245 +ILSVRC2012_val_00011363.JPEG:n03891332 +ILSVRC2012_val_00011364.JPEG:n04589890 +ILSVRC2012_val_00011365.JPEG:n03887697 +ILSVRC2012_val_00011366.JPEG:n04141076 +ILSVRC2012_val_00011367.JPEG:n03770439 +ILSVRC2012_val_00011368.JPEG:n02113023 +ILSVRC2012_val_00011369.JPEG:n02009912 +ILSVRC2012_val_00011370.JPEG:n02823750 +ILSVRC2012_val_00011371.JPEG:n04252077 +ILSVRC2012_val_00011372.JPEG:n02396427 +ILSVRC2012_val_00011373.JPEG:n02099601 +ILSVRC2012_val_00011374.JPEG:n02279972 +ILSVRC2012_val_00011375.JPEG:n01843383 +ILSVRC2012_val_00011376.JPEG:n02749479 +ILSVRC2012_val_00011377.JPEG:n04228054 +ILSVRC2012_val_00011378.JPEG:n04590129 +ILSVRC2012_val_00011379.JPEG:n01773797 +ILSVRC2012_val_00011380.JPEG:n02027492 +ILSVRC2012_val_00011381.JPEG:n02093428 +ILSVRC2012_val_00011382.JPEG:n02259212 +ILSVRC2012_val_00011383.JPEG:n01910747 +ILSVRC2012_val_00011384.JPEG:n02088364 +ILSVRC2012_val_00011385.JPEG:n02093754 +ILSVRC2012_val_00011386.JPEG:n07860988 +ILSVRC2012_val_00011387.JPEG:n02093428 +ILSVRC2012_val_00011388.JPEG:n01494475 +ILSVRC2012_val_00011389.JPEG:n03888605 +ILSVRC2012_val_00011390.JPEG:n04589890 +ILSVRC2012_val_00011391.JPEG:n02092339 +ILSVRC2012_val_00011392.JPEG:n07584110 +ILSVRC2012_val_00011393.JPEG:n02190166 +ILSVRC2012_val_00011394.JPEG:n02096051 +ILSVRC2012_val_00011395.JPEG:n04023962 +ILSVRC2012_val_00011396.JPEG:n02484975 +ILSVRC2012_val_00011397.JPEG:n03980874 +ILSVRC2012_val_00011398.JPEG:n02870880 +ILSVRC2012_val_00011399.JPEG:n01807496 +ILSVRC2012_val_00011400.JPEG:n02090721 +ILSVRC2012_val_00011401.JPEG:n02011460 +ILSVRC2012_val_00011402.JPEG:n02033041 +ILSVRC2012_val_00011403.JPEG:n01514668 +ILSVRC2012_val_00011404.JPEG:n02094114 +ILSVRC2012_val_00011405.JPEG:n02687172 +ILSVRC2012_val_00011406.JPEG:n02013706 +ILSVRC2012_val_00011407.JPEG:n04523525 +ILSVRC2012_val_00011408.JPEG:n07718747 +ILSVRC2012_val_00011409.JPEG:n02361337 +ILSVRC2012_val_00011410.JPEG:n07720875 +ILSVRC2012_val_00011411.JPEG:n04005630 +ILSVRC2012_val_00011412.JPEG:n04509417 +ILSVRC2012_val_00011413.JPEG:n07613480 +ILSVRC2012_val_00011414.JPEG:n01622779 +ILSVRC2012_val_00011415.JPEG:n03131574 +ILSVRC2012_val_00011416.JPEG:n01631663 +ILSVRC2012_val_00011417.JPEG:n02701002 +ILSVRC2012_val_00011418.JPEG:n03014705 +ILSVRC2012_val_00011419.JPEG:n02607072 +ILSVRC2012_val_00011420.JPEG:n01560419 +ILSVRC2012_val_00011421.JPEG:n03197337 +ILSVRC2012_val_00011422.JPEG:n09193705 +ILSVRC2012_val_00011423.JPEG:n02099849 +ILSVRC2012_val_00011424.JPEG:n03000134 +ILSVRC2012_val_00011425.JPEG:n02480495 +ILSVRC2012_val_00011426.JPEG:n03733805 +ILSVRC2012_val_00011427.JPEG:n07802026 +ILSVRC2012_val_00011428.JPEG:n01749939 +ILSVRC2012_val_00011429.JPEG:n03956157 +ILSVRC2012_val_00011430.JPEG:n01955084 +ILSVRC2012_val_00011431.JPEG:n03445777 +ILSVRC2012_val_00011432.JPEG:n02927161 +ILSVRC2012_val_00011433.JPEG:n02105162 +ILSVRC2012_val_00011434.JPEG:n02088238 +ILSVRC2012_val_00011435.JPEG:n06794110 +ILSVRC2012_val_00011436.JPEG:n09332890 +ILSVRC2012_val_00011437.JPEG:n02823428 +ILSVRC2012_val_00011438.JPEG:n03773504 +ILSVRC2012_val_00011439.JPEG:n03657121 +ILSVRC2012_val_00011440.JPEG:n04044716 +ILSVRC2012_val_00011441.JPEG:n07760859 +ILSVRC2012_val_00011442.JPEG:n03207941 +ILSVRC2012_val_00011443.JPEG:n07717410 +ILSVRC2012_val_00011444.JPEG:n01664065 +ILSVRC2012_val_00011445.JPEG:n03291819 +ILSVRC2012_val_00011446.JPEG:n01580077 +ILSVRC2012_val_00011447.JPEG:n02132136 +ILSVRC2012_val_00011448.JPEG:n01687978 +ILSVRC2012_val_00011449.JPEG:n09332890 +ILSVRC2012_val_00011450.JPEG:n04590129 +ILSVRC2012_val_00011451.JPEG:n04487081 +ILSVRC2012_val_00011452.JPEG:n03838899 +ILSVRC2012_val_00011453.JPEG:n01981276 +ILSVRC2012_val_00011454.JPEG:n03899768 +ILSVRC2012_val_00011455.JPEG:n04004767 +ILSVRC2012_val_00011456.JPEG:n03207743 +ILSVRC2012_val_00011457.JPEG:n02106166 +ILSVRC2012_val_00011458.JPEG:n07873807 +ILSVRC2012_val_00011459.JPEG:n04039381 +ILSVRC2012_val_00011460.JPEG:n03388549 +ILSVRC2012_val_00011461.JPEG:n03977966 +ILSVRC2012_val_00011462.JPEG:n03384352 +ILSVRC2012_val_00011463.JPEG:n02114367 +ILSVRC2012_val_00011464.JPEG:n07695742 +ILSVRC2012_val_00011465.JPEG:n02105412 +ILSVRC2012_val_00011466.JPEG:n04591157 +ILSVRC2012_val_00011467.JPEG:n01729322 +ILSVRC2012_val_00011468.JPEG:n02066245 +ILSVRC2012_val_00011469.JPEG:n03938244 +ILSVRC2012_val_00011470.JPEG:n03240683 +ILSVRC2012_val_00011471.JPEG:n07880968 +ILSVRC2012_val_00011472.JPEG:n03782006 +ILSVRC2012_val_00011473.JPEG:n02086646 +ILSVRC2012_val_00011474.JPEG:n01632777 +ILSVRC2012_val_00011475.JPEG:n02793495 +ILSVRC2012_val_00011476.JPEG:n02281406 +ILSVRC2012_val_00011477.JPEG:n02443484 +ILSVRC2012_val_00011478.JPEG:n03208938 +ILSVRC2012_val_00011479.JPEG:n04350905 +ILSVRC2012_val_00011480.JPEG:n03179701 +ILSVRC2012_val_00011481.JPEG:n03658185 +ILSVRC2012_val_00011482.JPEG:n02480855 +ILSVRC2012_val_00011483.JPEG:n01737021 +ILSVRC2012_val_00011484.JPEG:n09256479 +ILSVRC2012_val_00011485.JPEG:n04357314 +ILSVRC2012_val_00011486.JPEG:n03424325 +ILSVRC2012_val_00011487.JPEG:n02807133 +ILSVRC2012_val_00011488.JPEG:n01855032 +ILSVRC2012_val_00011489.JPEG:n01828970 +ILSVRC2012_val_00011490.JPEG:n03980874 +ILSVRC2012_val_00011491.JPEG:n02107683 +ILSVRC2012_val_00011492.JPEG:n03895866 +ILSVRC2012_val_00011493.JPEG:n07768694 +ILSVRC2012_val_00011494.JPEG:n02090721 +ILSVRC2012_val_00011495.JPEG:n02110958 +ILSVRC2012_val_00011496.JPEG:n02669723 +ILSVRC2012_val_00011497.JPEG:n04599235 +ILSVRC2012_val_00011498.JPEG:n02105641 +ILSVRC2012_val_00011499.JPEG:n02692877 +ILSVRC2012_val_00011500.JPEG:n02927161 +ILSVRC2012_val_00011501.JPEG:n01582220 +ILSVRC2012_val_00011502.JPEG:n02325366 +ILSVRC2012_val_00011503.JPEG:n04039381 +ILSVRC2012_val_00011504.JPEG:n02790996 +ILSVRC2012_val_00011505.JPEG:n07760859 +ILSVRC2012_val_00011506.JPEG:n02114712 +ILSVRC2012_val_00011507.JPEG:n02099712 +ILSVRC2012_val_00011508.JPEG:n04275548 +ILSVRC2012_val_00011509.JPEG:n04366367 +ILSVRC2012_val_00011510.JPEG:n02687172 +ILSVRC2012_val_00011511.JPEG:n02113624 +ILSVRC2012_val_00011512.JPEG:n02454379 +ILSVRC2012_val_00011513.JPEG:n04120489 +ILSVRC2012_val_00011514.JPEG:n03785016 +ILSVRC2012_val_00011515.JPEG:n02279972 +ILSVRC2012_val_00011516.JPEG:n04209239 +ILSVRC2012_val_00011517.JPEG:n01677366 +ILSVRC2012_val_00011518.JPEG:n01682714 +ILSVRC2012_val_00011519.JPEG:n01601694 +ILSVRC2012_val_00011520.JPEG:n02483708 +ILSVRC2012_val_00011521.JPEG:n07718747 +ILSVRC2012_val_00011522.JPEG:n04344873 +ILSVRC2012_val_00011523.JPEG:n02483362 +ILSVRC2012_val_00011524.JPEG:n07717556 +ILSVRC2012_val_00011525.JPEG:n01981276 +ILSVRC2012_val_00011526.JPEG:n02699494 +ILSVRC2012_val_00011527.JPEG:n03160309 +ILSVRC2012_val_00011528.JPEG:n02123597 +ILSVRC2012_val_00011529.JPEG:n03970156 +ILSVRC2012_val_00011530.JPEG:n01669191 +ILSVRC2012_val_00011531.JPEG:n01756291 +ILSVRC2012_val_00011532.JPEG:n02606052 +ILSVRC2012_val_00011533.JPEG:n02795169 +ILSVRC2012_val_00011534.JPEG:n03478589 +ILSVRC2012_val_00011535.JPEG:n02259212 +ILSVRC2012_val_00011536.JPEG:n06785654 +ILSVRC2012_val_00011537.JPEG:n02114712 +ILSVRC2012_val_00011538.JPEG:n04311174 +ILSVRC2012_val_00011539.JPEG:n03891332 +ILSVRC2012_val_00011540.JPEG:n04443257 +ILSVRC2012_val_00011541.JPEG:n01687978 +ILSVRC2012_val_00011542.JPEG:n04259630 +ILSVRC2012_val_00011543.JPEG:n02128925 +ILSVRC2012_val_00011544.JPEG:n02526121 +ILSVRC2012_val_00011545.JPEG:n03447721 +ILSVRC2012_val_00011546.JPEG:n04239074 +ILSVRC2012_val_00011547.JPEG:n03877472 +ILSVRC2012_val_00011548.JPEG:n03710637 +ILSVRC2012_val_00011549.JPEG:n07711569 +ILSVRC2012_val_00011550.JPEG:n04153751 +ILSVRC2012_val_00011551.JPEG:n01682714 +ILSVRC2012_val_00011552.JPEG:n03598930 +ILSVRC2012_val_00011553.JPEG:n04131690 +ILSVRC2012_val_00011554.JPEG:n01819313 +ILSVRC2012_val_00011555.JPEG:n02085620 +ILSVRC2012_val_00011556.JPEG:n02113023 +ILSVRC2012_val_00011557.JPEG:n03133878 +ILSVRC2012_val_00011558.JPEG:n07768694 +ILSVRC2012_val_00011559.JPEG:n04579432 +ILSVRC2012_val_00011560.JPEG:n04532670 +ILSVRC2012_val_00011561.JPEG:n03976467 +ILSVRC2012_val_00011562.JPEG:n04326547 +ILSVRC2012_val_00011563.JPEG:n02951358 +ILSVRC2012_val_00011564.JPEG:n02279972 +ILSVRC2012_val_00011565.JPEG:n03000247 +ILSVRC2012_val_00011566.JPEG:n03837869 +ILSVRC2012_val_00011567.JPEG:n09288635 +ILSVRC2012_val_00011568.JPEG:n03196217 +ILSVRC2012_val_00011569.JPEG:n03733805 +ILSVRC2012_val_00011570.JPEG:n02111889 +ILSVRC2012_val_00011571.JPEG:n04286575 +ILSVRC2012_val_00011572.JPEG:n01985128 +ILSVRC2012_val_00011573.JPEG:n02105056 +ILSVRC2012_val_00011574.JPEG:n02783161 +ILSVRC2012_val_00011575.JPEG:n03902125 +ILSVRC2012_val_00011576.JPEG:n02643566 +ILSVRC2012_val_00011577.JPEG:n04553703 +ILSVRC2012_val_00011578.JPEG:n03787032 +ILSVRC2012_val_00011579.JPEG:n02799071 +ILSVRC2012_val_00011580.JPEG:n02137549 +ILSVRC2012_val_00011581.JPEG:n03445777 +ILSVRC2012_val_00011582.JPEG:n03240683 +ILSVRC2012_val_00011583.JPEG:n02093256 +ILSVRC2012_val_00011584.JPEG:n01847000 +ILSVRC2012_val_00011585.JPEG:n01978455 +ILSVRC2012_val_00011586.JPEG:n02089973 +ILSVRC2012_val_00011587.JPEG:n03482405 +ILSVRC2012_val_00011588.JPEG:n06874185 +ILSVRC2012_val_00011589.JPEG:n02280649 +ILSVRC2012_val_00011590.JPEG:n02129604 +ILSVRC2012_val_00011591.JPEG:n02892767 +ILSVRC2012_val_00011592.JPEG:n02480495 +ILSVRC2012_val_00011593.JPEG:n02106662 +ILSVRC2012_val_00011594.JPEG:n12144580 +ILSVRC2012_val_00011595.JPEG:n03599486 +ILSVRC2012_val_00011596.JPEG:n02066245 +ILSVRC2012_val_00011597.JPEG:n02454379 +ILSVRC2012_val_00011598.JPEG:n01873310 +ILSVRC2012_val_00011599.JPEG:n03690938 +ILSVRC2012_val_00011600.JPEG:n02389026 +ILSVRC2012_val_00011601.JPEG:n02264363 +ILSVRC2012_val_00011602.JPEG:n02966193 +ILSVRC2012_val_00011603.JPEG:n02500267 +ILSVRC2012_val_00011604.JPEG:n03538406 +ILSVRC2012_val_00011605.JPEG:n01843065 +ILSVRC2012_val_00011606.JPEG:n04254680 +ILSVRC2012_val_00011607.JPEG:n04346328 +ILSVRC2012_val_00011608.JPEG:n03961711 +ILSVRC2012_val_00011609.JPEG:n03970156 +ILSVRC2012_val_00011610.JPEG:n03207941 +ILSVRC2012_val_00011611.JPEG:n03791053 +ILSVRC2012_val_00011612.JPEG:n02085936 +ILSVRC2012_val_00011613.JPEG:n03954731 +ILSVRC2012_val_00011614.JPEG:n03857828 +ILSVRC2012_val_00011615.JPEG:n02807133 +ILSVRC2012_val_00011616.JPEG:n02443114 +ILSVRC2012_val_00011617.JPEG:n02219486 +ILSVRC2012_val_00011618.JPEG:n03670208 +ILSVRC2012_val_00011619.JPEG:n04263257 +ILSVRC2012_val_00011620.JPEG:n03110669 +ILSVRC2012_val_00011621.JPEG:n01795545 +ILSVRC2012_val_00011622.JPEG:n03467068 +ILSVRC2012_val_00011623.JPEG:n02115913 +ILSVRC2012_val_00011624.JPEG:n02119789 +ILSVRC2012_val_00011625.JPEG:n04487081 +ILSVRC2012_val_00011626.JPEG:n02791124 +ILSVRC2012_val_00011627.JPEG:n04201297 +ILSVRC2012_val_00011628.JPEG:n04265275 +ILSVRC2012_val_00011629.JPEG:n01784675 +ILSVRC2012_val_00011630.JPEG:n02814533 +ILSVRC2012_val_00011631.JPEG:n02417914 +ILSVRC2012_val_00011632.JPEG:n07932039 +ILSVRC2012_val_00011633.JPEG:n02606052 +ILSVRC2012_val_00011634.JPEG:n01768244 +ILSVRC2012_val_00011635.JPEG:n04311004 +ILSVRC2012_val_00011636.JPEG:n03662601 +ILSVRC2012_val_00011637.JPEG:n02607072 +ILSVRC2012_val_00011638.JPEG:n01773549 +ILSVRC2012_val_00011639.JPEG:n02085620 +ILSVRC2012_val_00011640.JPEG:n02730930 +ILSVRC2012_val_00011641.JPEG:n04347754 +ILSVRC2012_val_00011642.JPEG:n02051845 +ILSVRC2012_val_00011643.JPEG:n01914609 +ILSVRC2012_val_00011644.JPEG:n03729826 +ILSVRC2012_val_00011645.JPEG:n02129165 +ILSVRC2012_val_00011646.JPEG:n01537544 +ILSVRC2012_val_00011647.JPEG:n03888605 +ILSVRC2012_val_00011648.JPEG:n03764736 +ILSVRC2012_val_00011649.JPEG:n04579145 +ILSVRC2012_val_00011650.JPEG:n01630670 +ILSVRC2012_val_00011651.JPEG:n01950731 +ILSVRC2012_val_00011652.JPEG:n03599486 +ILSVRC2012_val_00011653.JPEG:n03786901 +ILSVRC2012_val_00011654.JPEG:n04243546 +ILSVRC2012_val_00011655.JPEG:n04040759 +ILSVRC2012_val_00011656.JPEG:n03594945 +ILSVRC2012_val_00011657.JPEG:n01632458 +ILSVRC2012_val_00011658.JPEG:n02823750 +ILSVRC2012_val_00011659.JPEG:n04442312 +ILSVRC2012_val_00011660.JPEG:n02859443 +ILSVRC2012_val_00011661.JPEG:n01629819 +ILSVRC2012_val_00011662.JPEG:n04254777 +ILSVRC2012_val_00011663.JPEG:n04039381 +ILSVRC2012_val_00011664.JPEG:n01641577 +ILSVRC2012_val_00011665.JPEG:n04553703 +ILSVRC2012_val_00011666.JPEG:n03443371 +ILSVRC2012_val_00011667.JPEG:n04467665 +ILSVRC2012_val_00011668.JPEG:n03991062 +ILSVRC2012_val_00011669.JPEG:n02219486 +ILSVRC2012_val_00011670.JPEG:n02799071 +ILSVRC2012_val_00011671.JPEG:n04026417 +ILSVRC2012_val_00011672.JPEG:n03930313 +ILSVRC2012_val_00011673.JPEG:n02096585 +ILSVRC2012_val_00011674.JPEG:n03534580 +ILSVRC2012_val_00011675.JPEG:n07753113 +ILSVRC2012_val_00011676.JPEG:n03868863 +ILSVRC2012_val_00011677.JPEG:n01773549 +ILSVRC2012_val_00011678.JPEG:n03720891 +ILSVRC2012_val_00011679.JPEG:n02727426 +ILSVRC2012_val_00011680.JPEG:n02096177 +ILSVRC2012_val_00011681.JPEG:n03272562 +ILSVRC2012_val_00011682.JPEG:n02100236 +ILSVRC2012_val_00011683.JPEG:n03450230 +ILSVRC2012_val_00011684.JPEG:n03697007 +ILSVRC2012_val_00011685.JPEG:n02927161 +ILSVRC2012_val_00011686.JPEG:n01798484 +ILSVRC2012_val_00011687.JPEG:n02865351 +ILSVRC2012_val_00011688.JPEG:n01631663 +ILSVRC2012_val_00011689.JPEG:n02100236 +ILSVRC2012_val_00011690.JPEG:n03871628 +ILSVRC2012_val_00011691.JPEG:n03394916 +ILSVRC2012_val_00011692.JPEG:n03983396 +ILSVRC2012_val_00011693.JPEG:n03908714 +ILSVRC2012_val_00011694.JPEG:n02641379 +ILSVRC2012_val_00011695.JPEG:n07892512 +ILSVRC2012_val_00011696.JPEG:n01877812 +ILSVRC2012_val_00011697.JPEG:n01824575 +ILSVRC2012_val_00011698.JPEG:n02106030 +ILSVRC2012_val_00011699.JPEG:n02100583 +ILSVRC2012_val_00011700.JPEG:n03424325 +ILSVRC2012_val_00011701.JPEG:n02106166 +ILSVRC2012_val_00011702.JPEG:n01682714 +ILSVRC2012_val_00011703.JPEG:n04456115 +ILSVRC2012_val_00011704.JPEG:n01784675 +ILSVRC2012_val_00011705.JPEG:n03868242 +ILSVRC2012_val_00011706.JPEG:n02100877 +ILSVRC2012_val_00011707.JPEG:n04033901 +ILSVRC2012_val_00011708.JPEG:n04266014 +ILSVRC2012_val_00011709.JPEG:n04332243 +ILSVRC2012_val_00011710.JPEG:n02443114 +ILSVRC2012_val_00011711.JPEG:n04487081 +ILSVRC2012_val_00011712.JPEG:n01774750 +ILSVRC2012_val_00011713.JPEG:n02129165 +ILSVRC2012_val_00011714.JPEG:n01984695 +ILSVRC2012_val_00011715.JPEG:n03769881 +ILSVRC2012_val_00011716.JPEG:n02422106 +ILSVRC2012_val_00011717.JPEG:n04328186 +ILSVRC2012_val_00011718.JPEG:n02108915 +ILSVRC2012_val_00011719.JPEG:n02088364 +ILSVRC2012_val_00011720.JPEG:n02795169 +ILSVRC2012_val_00011721.JPEG:n01773157 +ILSVRC2012_val_00011722.JPEG:n03063689 +ILSVRC2012_val_00011723.JPEG:n04326547 +ILSVRC2012_val_00011724.JPEG:n01644900 +ILSVRC2012_val_00011725.JPEG:n09229709 +ILSVRC2012_val_00011726.JPEG:n02133161 +ILSVRC2012_val_00011727.JPEG:n03016953 +ILSVRC2012_val_00011728.JPEG:n02085620 +ILSVRC2012_val_00011729.JPEG:n07565083 +ILSVRC2012_val_00011730.JPEG:n02317335 +ILSVRC2012_val_00011731.JPEG:n04485082 +ILSVRC2012_val_00011732.JPEG:n02125311 +ILSVRC2012_val_00011733.JPEG:n04591157 +ILSVRC2012_val_00011734.JPEG:n02396427 +ILSVRC2012_val_00011735.JPEG:n04347754 +ILSVRC2012_val_00011736.JPEG:n02129604 +ILSVRC2012_val_00011737.JPEG:n02422699 +ILSVRC2012_val_00011738.JPEG:n02123597 +ILSVRC2012_val_00011739.JPEG:n03388183 +ILSVRC2012_val_00011740.JPEG:n03590841 +ILSVRC2012_val_00011741.JPEG:n02807133 +ILSVRC2012_val_00011742.JPEG:n03676483 +ILSVRC2012_val_00011743.JPEG:n03255030 +ILSVRC2012_val_00011744.JPEG:n02174001 +ILSVRC2012_val_00011745.JPEG:n04536866 +ILSVRC2012_val_00011746.JPEG:n02104029 +ILSVRC2012_val_00011747.JPEG:n02817516 +ILSVRC2012_val_00011748.JPEG:n02087046 +ILSVRC2012_val_00011749.JPEG:n02085782 +ILSVRC2012_val_00011750.JPEG:n02115641 +ILSVRC2012_val_00011751.JPEG:n02086910 +ILSVRC2012_val_00011752.JPEG:n02834397 +ILSVRC2012_val_00011753.JPEG:n03201208 +ILSVRC2012_val_00011754.JPEG:n02086240 +ILSVRC2012_val_00011755.JPEG:n02454379 +ILSVRC2012_val_00011756.JPEG:n02422699 +ILSVRC2012_val_00011757.JPEG:n02106662 +ILSVRC2012_val_00011758.JPEG:n04560804 +ILSVRC2012_val_00011759.JPEG:n02699494 +ILSVRC2012_val_00011760.JPEG:n02871525 +ILSVRC2012_val_00011761.JPEG:n04591157 +ILSVRC2012_val_00011762.JPEG:n04149813 +ILSVRC2012_val_00011763.JPEG:n03920288 +ILSVRC2012_val_00011764.JPEG:n02099267 +ILSVRC2012_val_00011765.JPEG:n02105412 +ILSVRC2012_val_00011766.JPEG:n01667778 +ILSVRC2012_val_00011767.JPEG:n03535780 +ILSVRC2012_val_00011768.JPEG:n02085936 +ILSVRC2012_val_00011769.JPEG:n03344393 +ILSVRC2012_val_00011770.JPEG:n03871628 +ILSVRC2012_val_00011771.JPEG:n02268853 +ILSVRC2012_val_00011772.JPEG:n02276258 +ILSVRC2012_val_00011773.JPEG:n03773504 +ILSVRC2012_val_00011774.JPEG:n04505470 +ILSVRC2012_val_00011775.JPEG:n02895154 +ILSVRC2012_val_00011776.JPEG:n01740131 +ILSVRC2012_val_00011777.JPEG:n02101388 +ILSVRC2012_val_00011778.JPEG:n01847000 +ILSVRC2012_val_00011779.JPEG:n04111531 +ILSVRC2012_val_00011780.JPEG:n02280649 +ILSVRC2012_val_00011781.JPEG:n04509417 +ILSVRC2012_val_00011782.JPEG:n01496331 +ILSVRC2012_val_00011783.JPEG:n02264363 +ILSVRC2012_val_00011784.JPEG:n02109525 +ILSVRC2012_val_00011785.JPEG:n03372029 +ILSVRC2012_val_00011786.JPEG:n03903868 +ILSVRC2012_val_00011787.JPEG:n01796340 +ILSVRC2012_val_00011788.JPEG:n02988304 +ILSVRC2012_val_00011789.JPEG:n02486261 +ILSVRC2012_val_00011790.JPEG:n07932039 +ILSVRC2012_val_00011791.JPEG:n03841143 +ILSVRC2012_val_00011792.JPEG:n02089867 +ILSVRC2012_val_00011793.JPEG:n02099429 +ILSVRC2012_val_00011794.JPEG:n03062245 +ILSVRC2012_val_00011795.JPEG:n02799071 +ILSVRC2012_val_00011796.JPEG:n03485794 +ILSVRC2012_val_00011797.JPEG:n03944341 +ILSVRC2012_val_00011798.JPEG:n02090379 +ILSVRC2012_val_00011799.JPEG:n04370456 +ILSVRC2012_val_00011800.JPEG:n04125021 +ILSVRC2012_val_00011801.JPEG:n03929855 +ILSVRC2012_val_00011802.JPEG:n02110063 +ILSVRC2012_val_00011803.JPEG:n02794156 +ILSVRC2012_val_00011804.JPEG:n04141076 +ILSVRC2012_val_00011805.JPEG:n02085936 +ILSVRC2012_val_00011806.JPEG:n04606251 +ILSVRC2012_val_00011807.JPEG:n02099712 +ILSVRC2012_val_00011808.JPEG:n01773549 +ILSVRC2012_val_00011809.JPEG:n02992529 +ILSVRC2012_val_00011810.JPEG:n03347037 +ILSVRC2012_val_00011811.JPEG:n02120505 +ILSVRC2012_val_00011812.JPEG:n02727426 +ILSVRC2012_val_00011813.JPEG:n03483316 +ILSVRC2012_val_00011814.JPEG:n04479046 +ILSVRC2012_val_00011815.JPEG:n03544143 +ILSVRC2012_val_00011816.JPEG:n03888605 +ILSVRC2012_val_00011817.JPEG:n04548362 +ILSVRC2012_val_00011818.JPEG:n13037406 +ILSVRC2012_val_00011819.JPEG:n04044716 +ILSVRC2012_val_00011820.JPEG:n02259212 +ILSVRC2012_val_00011821.JPEG:n02835271 +ILSVRC2012_val_00011822.JPEG:n01797886 +ILSVRC2012_val_00011823.JPEG:n02823428 +ILSVRC2012_val_00011824.JPEG:n04086273 +ILSVRC2012_val_00011825.JPEG:n02127052 +ILSVRC2012_val_00011826.JPEG:n03133878 +ILSVRC2012_val_00011827.JPEG:n03733281 +ILSVRC2012_val_00011828.JPEG:n02676566 +ILSVRC2012_val_00011829.JPEG:n02667093 +ILSVRC2012_val_00011830.JPEG:n04026417 +ILSVRC2012_val_00011831.JPEG:n07932039 +ILSVRC2012_val_00011832.JPEG:n04252077 +ILSVRC2012_val_00011833.JPEG:n03976467 +ILSVRC2012_val_00011834.JPEG:n04366367 +ILSVRC2012_val_00011835.JPEG:n03443371 +ILSVRC2012_val_00011836.JPEG:n04346328 +ILSVRC2012_val_00011837.JPEG:n02112018 +ILSVRC2012_val_00011838.JPEG:n03781244 +ILSVRC2012_val_00011839.JPEG:n03459775 +ILSVRC2012_val_00011840.JPEG:n03876231 +ILSVRC2012_val_00011841.JPEG:n01534433 +ILSVRC2012_val_00011842.JPEG:n03017168 +ILSVRC2012_val_00011843.JPEG:n02808304 +ILSVRC2012_val_00011844.JPEG:n07730033 +ILSVRC2012_val_00011845.JPEG:n02169497 +ILSVRC2012_val_00011846.JPEG:n02514041 +ILSVRC2012_val_00011847.JPEG:n04458633 +ILSVRC2012_val_00011848.JPEG:n02002556 +ILSVRC2012_val_00011849.JPEG:n03980874 +ILSVRC2012_val_00011850.JPEG:n03131574 +ILSVRC2012_val_00011851.JPEG:n01807496 +ILSVRC2012_val_00011852.JPEG:n04330267 +ILSVRC2012_val_00011853.JPEG:n01773549 +ILSVRC2012_val_00011854.JPEG:n02123159 +ILSVRC2012_val_00011855.JPEG:n04204347 +ILSVRC2012_val_00011856.JPEG:n02395406 +ILSVRC2012_val_00011857.JPEG:n02321529 +ILSVRC2012_val_00011858.JPEG:n03124043 +ILSVRC2012_val_00011859.JPEG:n03617480 +ILSVRC2012_val_00011860.JPEG:n01910747 +ILSVRC2012_val_00011861.JPEG:n01784675 +ILSVRC2012_val_00011862.JPEG:n03733131 +ILSVRC2012_val_00011863.JPEG:n07875152 +ILSVRC2012_val_00011864.JPEG:n04599235 +ILSVRC2012_val_00011865.JPEG:n09428293 +ILSVRC2012_val_00011866.JPEG:n07565083 +ILSVRC2012_val_00011867.JPEG:n02206856 +ILSVRC2012_val_00011868.JPEG:n03127747 +ILSVRC2012_val_00011869.JPEG:n02086240 +ILSVRC2012_val_00011870.JPEG:n04146614 +ILSVRC2012_val_00011871.JPEG:n04532670 +ILSVRC2012_val_00011872.JPEG:n03259280 +ILSVRC2012_val_00011873.JPEG:n02104365 +ILSVRC2012_val_00011874.JPEG:n01855032 +ILSVRC2012_val_00011875.JPEG:n04366367 +ILSVRC2012_val_00011876.JPEG:n02977058 +ILSVRC2012_val_00011877.JPEG:n02444819 +ILSVRC2012_val_00011878.JPEG:n02088632 +ILSVRC2012_val_00011879.JPEG:n04562935 +ILSVRC2012_val_00011880.JPEG:n03891251 +ILSVRC2012_val_00011881.JPEG:n07718747 +ILSVRC2012_val_00011882.JPEG:n02783161 +ILSVRC2012_val_00011883.JPEG:n03929855 +ILSVRC2012_val_00011884.JPEG:n01872401 +ILSVRC2012_val_00011885.JPEG:n07693725 +ILSVRC2012_val_00011886.JPEG:n02859443 +ILSVRC2012_val_00011887.JPEG:n04370456 +ILSVRC2012_val_00011888.JPEG:n02259212 +ILSVRC2012_val_00011889.JPEG:n02231487 +ILSVRC2012_val_00011890.JPEG:n04065272 +ILSVRC2012_val_00011891.JPEG:n02361337 +ILSVRC2012_val_00011892.JPEG:n02395406 +ILSVRC2012_val_00011893.JPEG:n02094433 +ILSVRC2012_val_00011894.JPEG:n01833805 +ILSVRC2012_val_00011895.JPEG:n02097474 +ILSVRC2012_val_00011896.JPEG:n03868242 +ILSVRC2012_val_00011897.JPEG:n04041544 +ILSVRC2012_val_00011898.JPEG:n02493793 +ILSVRC2012_val_00011899.JPEG:n02174001 +ILSVRC2012_val_00011900.JPEG:n02085620 +ILSVRC2012_val_00011901.JPEG:n12620546 +ILSVRC2012_val_00011902.JPEG:n02412080 +ILSVRC2012_val_00011903.JPEG:n02808440 +ILSVRC2012_val_00011904.JPEG:n02489166 +ILSVRC2012_val_00011905.JPEG:n04069434 +ILSVRC2012_val_00011906.JPEG:n03763968 +ILSVRC2012_val_00011907.JPEG:n03721384 +ILSVRC2012_val_00011908.JPEG:n04522168 +ILSVRC2012_val_00011909.JPEG:n03527444 +ILSVRC2012_val_00011910.JPEG:n04147183 +ILSVRC2012_val_00011911.JPEG:n02277742 +ILSVRC2012_val_00011912.JPEG:n03743016 +ILSVRC2012_val_00011913.JPEG:n02490219 +ILSVRC2012_val_00011914.JPEG:n01443537 +ILSVRC2012_val_00011915.JPEG:n01534433 +ILSVRC2012_val_00011916.JPEG:n02965783 +ILSVRC2012_val_00011917.JPEG:n02106382 +ILSVRC2012_val_00011918.JPEG:n02007558 +ILSVRC2012_val_00011919.JPEG:n03908618 +ILSVRC2012_val_00011920.JPEG:n04357314 +ILSVRC2012_val_00011921.JPEG:n02108089 +ILSVRC2012_val_00011922.JPEG:n01980166 +ILSVRC2012_val_00011923.JPEG:n03642806 +ILSVRC2012_val_00011924.JPEG:n04090263 +ILSVRC2012_val_00011925.JPEG:n02093256 +ILSVRC2012_val_00011926.JPEG:n02841315 +ILSVRC2012_val_00011927.JPEG:n01695060 +ILSVRC2012_val_00011928.JPEG:n04152593 +ILSVRC2012_val_00011929.JPEG:n04532670 +ILSVRC2012_val_00011930.JPEG:n04201297 +ILSVRC2012_val_00011931.JPEG:n03476684 +ILSVRC2012_val_00011932.JPEG:n02236044 +ILSVRC2012_val_00011933.JPEG:n02769748 +ILSVRC2012_val_00011934.JPEG:n03187595 +ILSVRC2012_val_00011935.JPEG:n02841315 +ILSVRC2012_val_00011936.JPEG:n04081281 +ILSVRC2012_val_00011937.JPEG:n07873807 +ILSVRC2012_val_00011938.JPEG:n04548362 +ILSVRC2012_val_00011939.JPEG:n03595614 +ILSVRC2012_val_00011940.JPEG:n04532670 +ILSVRC2012_val_00011941.JPEG:n03047690 +ILSVRC2012_val_00011942.JPEG:n04552348 +ILSVRC2012_val_00011943.JPEG:n01806143 +ILSVRC2012_val_00011944.JPEG:n04542943 +ILSVRC2012_val_00011945.JPEG:n07717556 +ILSVRC2012_val_00011946.JPEG:n03782006 +ILSVRC2012_val_00011947.JPEG:n02107574 +ILSVRC2012_val_00011948.JPEG:n04118776 +ILSVRC2012_val_00011949.JPEG:n04523525 +ILSVRC2012_val_00011950.JPEG:n04141327 +ILSVRC2012_val_00011951.JPEG:n03000684 +ILSVRC2012_val_00011952.JPEG:n02124075 +ILSVRC2012_val_00011953.JPEG:n02667093 +ILSVRC2012_val_00011954.JPEG:n03976467 +ILSVRC2012_val_00011955.JPEG:n02965783 +ILSVRC2012_val_00011956.JPEG:n06785654 +ILSVRC2012_val_00011957.JPEG:n04548280 +ILSVRC2012_val_00011958.JPEG:n03840681 +ILSVRC2012_val_00011959.JPEG:n04243546 +ILSVRC2012_val_00011960.JPEG:n03447721 +ILSVRC2012_val_00011961.JPEG:n03720891 +ILSVRC2012_val_00011962.JPEG:n03825788 +ILSVRC2012_val_00011963.JPEG:n02791270 +ILSVRC2012_val_00011964.JPEG:n02870880 +ILSVRC2012_val_00011965.JPEG:n03535780 +ILSVRC2012_val_00011966.JPEG:n02165456 +ILSVRC2012_val_00011967.JPEG:n02132136 +ILSVRC2012_val_00011968.JPEG:n04044716 +ILSVRC2012_val_00011969.JPEG:n03970156 +ILSVRC2012_val_00011970.JPEG:n03692522 +ILSVRC2012_val_00011971.JPEG:n01744401 +ILSVRC2012_val_00011972.JPEG:n04418357 +ILSVRC2012_val_00011973.JPEG:n02167151 +ILSVRC2012_val_00011974.JPEG:n02790996 +ILSVRC2012_val_00011975.JPEG:n03903868 +ILSVRC2012_val_00011976.JPEG:n02860847 +ILSVRC2012_val_00011977.JPEG:n02417914 +ILSVRC2012_val_00011978.JPEG:n01985128 +ILSVRC2012_val_00011979.JPEG:n02281787 +ILSVRC2012_val_00011980.JPEG:n10148035 +ILSVRC2012_val_00011981.JPEG:n02974003 +ILSVRC2012_val_00011982.JPEG:n03777754 +ILSVRC2012_val_00011983.JPEG:n03445777 +ILSVRC2012_val_00011984.JPEG:n04532106 +ILSVRC2012_val_00011985.JPEG:n02085782 +ILSVRC2012_val_00011986.JPEG:n03452741 +ILSVRC2012_val_00011987.JPEG:n03670208 +ILSVRC2012_val_00011988.JPEG:n03866082 +ILSVRC2012_val_00011989.JPEG:n02105162 +ILSVRC2012_val_00011990.JPEG:n03220513 +ILSVRC2012_val_00011991.JPEG:n03529860 +ILSVRC2012_val_00011992.JPEG:n04376876 +ILSVRC2012_val_00011993.JPEG:n01440764 +ILSVRC2012_val_00011994.JPEG:n03498962 +ILSVRC2012_val_00011995.JPEG:n02687172 +ILSVRC2012_val_00011996.JPEG:n01665541 +ILSVRC2012_val_00011997.JPEG:n04344873 +ILSVRC2012_val_00011998.JPEG:n02489166 +ILSVRC2012_val_00011999.JPEG:n03384352 +ILSVRC2012_val_00012000.JPEG:n02443484 +ILSVRC2012_val_00012001.JPEG:n03976657 +ILSVRC2012_val_00012002.JPEG:n04540053 +ILSVRC2012_val_00012003.JPEG:n01817953 +ILSVRC2012_val_00012004.JPEG:n02098105 +ILSVRC2012_val_00012005.JPEG:n02655020 +ILSVRC2012_val_00012006.JPEG:n01756291 +ILSVRC2012_val_00012007.JPEG:n02099267 +ILSVRC2012_val_00012008.JPEG:n04141327 +ILSVRC2012_val_00012009.JPEG:n07734744 +ILSVRC2012_val_00012010.JPEG:n03690938 +ILSVRC2012_val_00012011.JPEG:n02133161 +ILSVRC2012_val_00012012.JPEG:n10148035 +ILSVRC2012_val_00012013.JPEG:n03461385 +ILSVRC2012_val_00012014.JPEG:n03840681 +ILSVRC2012_val_00012015.JPEG:n02099267 +ILSVRC2012_val_00012016.JPEG:n03908618 +ILSVRC2012_val_00012017.JPEG:n02483708 +ILSVRC2012_val_00012018.JPEG:n03710637 +ILSVRC2012_val_00012019.JPEG:n02804610 +ILSVRC2012_val_00012020.JPEG:n02906734 +ILSVRC2012_val_00012021.JPEG:n07836838 +ILSVRC2012_val_00012022.JPEG:n03930313 +ILSVRC2012_val_00012023.JPEG:n02786058 +ILSVRC2012_val_00012024.JPEG:n01795545 +ILSVRC2012_val_00012025.JPEG:n02804610 +ILSVRC2012_val_00012026.JPEG:n02095570 +ILSVRC2012_val_00012027.JPEG:n03447721 +ILSVRC2012_val_00012028.JPEG:n04311004 +ILSVRC2012_val_00012029.JPEG:n04229816 +ILSVRC2012_val_00012030.JPEG:n04208210 +ILSVRC2012_val_00012031.JPEG:n03710193 +ILSVRC2012_val_00012032.JPEG:n03584829 +ILSVRC2012_val_00012033.JPEG:n04355338 +ILSVRC2012_val_00012034.JPEG:n03146219 +ILSVRC2012_val_00012035.JPEG:n02085620 +ILSVRC2012_val_00012036.JPEG:n04522168 +ILSVRC2012_val_00012037.JPEG:n02106030 +ILSVRC2012_val_00012038.JPEG:n03908618 +ILSVRC2012_val_00012039.JPEG:n02113624 +ILSVRC2012_val_00012040.JPEG:n04429376 +ILSVRC2012_val_00012041.JPEG:n02100877 +ILSVRC2012_val_00012042.JPEG:n02894605 +ILSVRC2012_val_00012043.JPEG:n02088632 +ILSVRC2012_val_00012044.JPEG:n02490219 +ILSVRC2012_val_00012045.JPEG:n02264363 +ILSVRC2012_val_00012046.JPEG:n04204238 +ILSVRC2012_val_00012047.JPEG:n07717556 +ILSVRC2012_val_00012048.JPEG:n02699494 +ILSVRC2012_val_00012049.JPEG:n13040303 +ILSVRC2012_val_00012050.JPEG:n02782093 +ILSVRC2012_val_00012051.JPEG:n04238763 +ILSVRC2012_val_00012052.JPEG:n03935335 +ILSVRC2012_val_00012053.JPEG:n02111889 +ILSVRC2012_val_00012054.JPEG:n04147183 +ILSVRC2012_val_00012055.JPEG:n02089078 +ILSVRC2012_val_00012056.JPEG:n03598930 +ILSVRC2012_val_00012057.JPEG:n04131690 +ILSVRC2012_val_00012058.JPEG:n01534433 +ILSVRC2012_val_00012059.JPEG:n04039381 +ILSVRC2012_val_00012060.JPEG:n02113023 +ILSVRC2012_val_00012061.JPEG:n03649909 +ILSVRC2012_val_00012062.JPEG:n02804610 +ILSVRC2012_val_00012063.JPEG:n02950826 +ILSVRC2012_val_00012064.JPEG:n07695742 +ILSVRC2012_val_00012065.JPEG:n03899768 +ILSVRC2012_val_00012066.JPEG:n03662601 +ILSVRC2012_val_00012067.JPEG:n02100877 +ILSVRC2012_val_00012068.JPEG:n06359193 +ILSVRC2012_val_00012069.JPEG:n04270147 +ILSVRC2012_val_00012070.JPEG:n03527444 +ILSVRC2012_val_00012071.JPEG:n04023962 +ILSVRC2012_val_00012072.JPEG:n03207743 +ILSVRC2012_val_00012073.JPEG:n03691459 +ILSVRC2012_val_00012074.JPEG:n02086646 +ILSVRC2012_val_00012075.JPEG:n04456115 +ILSVRC2012_val_00012076.JPEG:n04335435 +ILSVRC2012_val_00012077.JPEG:n04493381 +ILSVRC2012_val_00012078.JPEG:n03355925 +ILSVRC2012_val_00012079.JPEG:n02128757 +ILSVRC2012_val_00012080.JPEG:n03710637 +ILSVRC2012_val_00012081.JPEG:n02749479 +ILSVRC2012_val_00012082.JPEG:n04111531 +ILSVRC2012_val_00012083.JPEG:n02669723 +ILSVRC2012_val_00012084.JPEG:n04591157 +ILSVRC2012_val_00012085.JPEG:n02106550 +ILSVRC2012_val_00012086.JPEG:n04069434 +ILSVRC2012_val_00012087.JPEG:n01669191 +ILSVRC2012_val_00012088.JPEG:n03496892 +ILSVRC2012_val_00012089.JPEG:n01855672 +ILSVRC2012_val_00012090.JPEG:n03803284 +ILSVRC2012_val_00012091.JPEG:n04371774 +ILSVRC2012_val_00012092.JPEG:n02965783 +ILSVRC2012_val_00012093.JPEG:n01955084 +ILSVRC2012_val_00012094.JPEG:n03710637 +ILSVRC2012_val_00012095.JPEG:n04147183 +ILSVRC2012_val_00012096.JPEG:n03792782 +ILSVRC2012_val_00012097.JPEG:n04597913 +ILSVRC2012_val_00012098.JPEG:n04266014 +ILSVRC2012_val_00012099.JPEG:n02790996 +ILSVRC2012_val_00012100.JPEG:n02099601 +ILSVRC2012_val_00012101.JPEG:n03627232 +ILSVRC2012_val_00012102.JPEG:n02219486 +ILSVRC2012_val_00012103.JPEG:n07760859 +ILSVRC2012_val_00012104.JPEG:n02877765 +ILSVRC2012_val_00012105.JPEG:n07715103 +ILSVRC2012_val_00012106.JPEG:n02259212 +ILSVRC2012_val_00012107.JPEG:n07747607 +ILSVRC2012_val_00012108.JPEG:n04376876 +ILSVRC2012_val_00012109.JPEG:n01748264 +ILSVRC2012_val_00012110.JPEG:n04317175 +ILSVRC2012_val_00012111.JPEG:n02687172 +ILSVRC2012_val_00012112.JPEG:n13037406 +ILSVRC2012_val_00012113.JPEG:n02321529 +ILSVRC2012_val_00012114.JPEG:n02981792 +ILSVRC2012_val_00012115.JPEG:n02992211 +ILSVRC2012_val_00012116.JPEG:n03891332 +ILSVRC2012_val_00012117.JPEG:n01944390 +ILSVRC2012_val_00012118.JPEG:n02398521 +ILSVRC2012_val_00012119.JPEG:n07753275 +ILSVRC2012_val_00012120.JPEG:n01687978 +ILSVRC2012_val_00012121.JPEG:n03325584 +ILSVRC2012_val_00012122.JPEG:n01806143 +ILSVRC2012_val_00012123.JPEG:n01795545 +ILSVRC2012_val_00012124.JPEG:n02256656 +ILSVRC2012_val_00012125.JPEG:n13133613 +ILSVRC2012_val_00012126.JPEG:n06785654 +ILSVRC2012_val_00012127.JPEG:n02236044 +ILSVRC2012_val_00012128.JPEG:n04033901 +ILSVRC2012_val_00012129.JPEG:n02892767 +ILSVRC2012_val_00012130.JPEG:n03792972 +ILSVRC2012_val_00012131.JPEG:n07753592 +ILSVRC2012_val_00012132.JPEG:n01580077 +ILSVRC2012_val_00012133.JPEG:n03535780 +ILSVRC2012_val_00012134.JPEG:n03602883 +ILSVRC2012_val_00012135.JPEG:n02423022 +ILSVRC2012_val_00012136.JPEG:n03599486 +ILSVRC2012_val_00012137.JPEG:n02279972 +ILSVRC2012_val_00012138.JPEG:n02655020 +ILSVRC2012_val_00012139.JPEG:n03637318 +ILSVRC2012_val_00012140.JPEG:n02108000 +ILSVRC2012_val_00012141.JPEG:n03355925 +ILSVRC2012_val_00012142.JPEG:n04486054 +ILSVRC2012_val_00012143.JPEG:n01986214 +ILSVRC2012_val_00012144.JPEG:n03014705 +ILSVRC2012_val_00012145.JPEG:n04599235 +ILSVRC2012_val_00012146.JPEG:n02107312 +ILSVRC2012_val_00012147.JPEG:n04522168 +ILSVRC2012_val_00012148.JPEG:n03782006 +ILSVRC2012_val_00012149.JPEG:n02091244 +ILSVRC2012_val_00012150.JPEG:n04238763 +ILSVRC2012_val_00012151.JPEG:n01641577 +ILSVRC2012_val_00012152.JPEG:n02268853 +ILSVRC2012_val_00012153.JPEG:n07711569 +ILSVRC2012_val_00012154.JPEG:n03662601 +ILSVRC2012_val_00012155.JPEG:n02102318 +ILSVRC2012_val_00012156.JPEG:n01677366 +ILSVRC2012_val_00012157.JPEG:n02097209 +ILSVRC2012_val_00012158.JPEG:n03763968 +ILSVRC2012_val_00012159.JPEG:n03786901 +ILSVRC2012_val_00012160.JPEG:n02509815 +ILSVRC2012_val_00012161.JPEG:n02086910 +ILSVRC2012_val_00012162.JPEG:n06794110 +ILSVRC2012_val_00012163.JPEG:n07920052 +ILSVRC2012_val_00012164.JPEG:n03379051 +ILSVRC2012_val_00012165.JPEG:n02346627 +ILSVRC2012_val_00012166.JPEG:n02018795 +ILSVRC2012_val_00012167.JPEG:n02480495 +ILSVRC2012_val_00012168.JPEG:n07711569 +ILSVRC2012_val_00012169.JPEG:n04532670 +ILSVRC2012_val_00012170.JPEG:n02099712 +ILSVRC2012_val_00012171.JPEG:n02110806 +ILSVRC2012_val_00012172.JPEG:n03759954 +ILSVRC2012_val_00012173.JPEG:n02123597 +ILSVRC2012_val_00012174.JPEG:n04154565 +ILSVRC2012_val_00012175.JPEG:n03347037 +ILSVRC2012_val_00012176.JPEG:n02077923 +ILSVRC2012_val_00012177.JPEG:n02514041 +ILSVRC2012_val_00012178.JPEG:n01616318 +ILSVRC2012_val_00012179.JPEG:n02641379 +ILSVRC2012_val_00012180.JPEG:n04086273 +ILSVRC2012_val_00012181.JPEG:n02097298 +ILSVRC2012_val_00012182.JPEG:n02930766 +ILSVRC2012_val_00012183.JPEG:n01983481 +ILSVRC2012_val_00012184.JPEG:n03995372 +ILSVRC2012_val_00012185.JPEG:n03891332 +ILSVRC2012_val_00012186.JPEG:n03218198 +ILSVRC2012_val_00012187.JPEG:n02058221 +ILSVRC2012_val_00012188.JPEG:n01729322 +ILSVRC2012_val_00012189.JPEG:n02799071 +ILSVRC2012_val_00012190.JPEG:n01820546 +ILSVRC2012_val_00012191.JPEG:n04127249 +ILSVRC2012_val_00012192.JPEG:n02834397 +ILSVRC2012_val_00012193.JPEG:n02097209 +ILSVRC2012_val_00012194.JPEG:n03196217 +ILSVRC2012_val_00012195.JPEG:n03216828 +ILSVRC2012_val_00012196.JPEG:n02096585 +ILSVRC2012_val_00012197.JPEG:n04229816 +ILSVRC2012_val_00012198.JPEG:n11879895 +ILSVRC2012_val_00012199.JPEG:n03977966 +ILSVRC2012_val_00012200.JPEG:n03876231 +ILSVRC2012_val_00012201.JPEG:n03908618 +ILSVRC2012_val_00012202.JPEG:n03255030 +ILSVRC2012_val_00012203.JPEG:n02106662 +ILSVRC2012_val_00012204.JPEG:n02488702 +ILSVRC2012_val_00012205.JPEG:n02978881 +ILSVRC2012_val_00012206.JPEG:n03868242 +ILSVRC2012_val_00012207.JPEG:n03710721 +ILSVRC2012_val_00012208.JPEG:n03494278 +ILSVRC2012_val_00012209.JPEG:n02363005 +ILSVRC2012_val_00012210.JPEG:n02939185 +ILSVRC2012_val_00012211.JPEG:n07768694 +ILSVRC2012_val_00012212.JPEG:n04505470 +ILSVRC2012_val_00012213.JPEG:n02028035 +ILSVRC2012_val_00012214.JPEG:n02894605 +ILSVRC2012_val_00012215.JPEG:n07717410 +ILSVRC2012_val_00012216.JPEG:n07745940 +ILSVRC2012_val_00012217.JPEG:n04429376 +ILSVRC2012_val_00012218.JPEG:n04344873 +ILSVRC2012_val_00012219.JPEG:n02727426 +ILSVRC2012_val_00012220.JPEG:n01753488 +ILSVRC2012_val_00012221.JPEG:n02110806 +ILSVRC2012_val_00012222.JPEG:n03661043 +ILSVRC2012_val_00012223.JPEG:n01806567 +ILSVRC2012_val_00012224.JPEG:n01955084 +ILSVRC2012_val_00012225.JPEG:n03467068 +ILSVRC2012_val_00012226.JPEG:n02110063 +ILSVRC2012_val_00012227.JPEG:n03902125 +ILSVRC2012_val_00012228.JPEG:n03450230 +ILSVRC2012_val_00012229.JPEG:n01692333 +ILSVRC2012_val_00012230.JPEG:n02114855 +ILSVRC2012_val_00012231.JPEG:n01644900 +ILSVRC2012_val_00012232.JPEG:n07742313 +ILSVRC2012_val_00012233.JPEG:n07565083 +ILSVRC2012_val_00012234.JPEG:n04505470 +ILSVRC2012_val_00012235.JPEG:n02088364 +ILSVRC2012_val_00012236.JPEG:n03733131 +ILSVRC2012_val_00012237.JPEG:n02105056 +ILSVRC2012_val_00012238.JPEG:n02606052 +ILSVRC2012_val_00012239.JPEG:n03179701 +ILSVRC2012_val_00012240.JPEG:n07715103 +ILSVRC2012_val_00012241.JPEG:n02641379 +ILSVRC2012_val_00012242.JPEG:n03259280 +ILSVRC2012_val_00012243.JPEG:n07873807 +ILSVRC2012_val_00012244.JPEG:n04584207 +ILSVRC2012_val_00012245.JPEG:n02110063 +ILSVRC2012_val_00012246.JPEG:n03218198 +ILSVRC2012_val_00012247.JPEG:n02494079 +ILSVRC2012_val_00012248.JPEG:n01644373 +ILSVRC2012_val_00012249.JPEG:n04332243 +ILSVRC2012_val_00012250.JPEG:n02115913 +ILSVRC2012_val_00012251.JPEG:n02120079 +ILSVRC2012_val_00012252.JPEG:n09229709 +ILSVRC2012_val_00012253.JPEG:n02481823 +ILSVRC2012_val_00012254.JPEG:n04235860 +ILSVRC2012_val_00012255.JPEG:n02113799 +ILSVRC2012_val_00012256.JPEG:n02823428 +ILSVRC2012_val_00012257.JPEG:n04371774 +ILSVRC2012_val_00012258.JPEG:n02442845 +ILSVRC2012_val_00012259.JPEG:n01498041 +ILSVRC2012_val_00012260.JPEG:n03944341 +ILSVRC2012_val_00012261.JPEG:n09332890 +ILSVRC2012_val_00012262.JPEG:n02091134 +ILSVRC2012_val_00012263.JPEG:n02690373 +ILSVRC2012_val_00012264.JPEG:n02788148 +ILSVRC2012_val_00012265.JPEG:n02869837 +ILSVRC2012_val_00012266.JPEG:n04204238 +ILSVRC2012_val_00012267.JPEG:n01675722 +ILSVRC2012_val_00012268.JPEG:n02236044 +ILSVRC2012_val_00012269.JPEG:n02280649 +ILSVRC2012_val_00012270.JPEG:n12144580 +ILSVRC2012_val_00012271.JPEG:n01882714 +ILSVRC2012_val_00012272.JPEG:n04120489 +ILSVRC2012_val_00012273.JPEG:n02999410 +ILSVRC2012_val_00012274.JPEG:n03692522 +ILSVRC2012_val_00012275.JPEG:n01729322 +ILSVRC2012_val_00012276.JPEG:n04532670 +ILSVRC2012_val_00012277.JPEG:n03337140 +ILSVRC2012_val_00012278.JPEG:n02966193 +ILSVRC2012_val_00012279.JPEG:n07742313 +ILSVRC2012_val_00012280.JPEG:n03793489 +ILSVRC2012_val_00012281.JPEG:n04355933 +ILSVRC2012_val_00012282.JPEG:n03220513 +ILSVRC2012_val_00012283.JPEG:n02445715 +ILSVRC2012_val_00012284.JPEG:n04443257 +ILSVRC2012_val_00012285.JPEG:n04026417 +ILSVRC2012_val_00012286.JPEG:n02823428 +ILSVRC2012_val_00012287.JPEG:n03976467 +ILSVRC2012_val_00012288.JPEG:n02102177 +ILSVRC2012_val_00012289.JPEG:n03773504 +ILSVRC2012_val_00012290.JPEG:n04487394 +ILSVRC2012_val_00012291.JPEG:n02085936 +ILSVRC2012_val_00012292.JPEG:n07614500 +ILSVRC2012_val_00012293.JPEG:n02089078 +ILSVRC2012_val_00012294.JPEG:n02206856 +ILSVRC2012_val_00012295.JPEG:n04147183 +ILSVRC2012_val_00012296.JPEG:n04501370 +ILSVRC2012_val_00012297.JPEG:n02422699 +ILSVRC2012_val_00012298.JPEG:n02085782 +ILSVRC2012_val_00012299.JPEG:n02097130 +ILSVRC2012_val_00012300.JPEG:n03929660 +ILSVRC2012_val_00012301.JPEG:n01751748 +ILSVRC2012_val_00012302.JPEG:n02099849 +ILSVRC2012_val_00012303.JPEG:n01924916 +ILSVRC2012_val_00012304.JPEG:n01692333 +ILSVRC2012_val_00012305.JPEG:n04275548 +ILSVRC2012_val_00012306.JPEG:n03991062 +ILSVRC2012_val_00012307.JPEG:n01824575 +ILSVRC2012_val_00012308.JPEG:n03218198 +ILSVRC2012_val_00012309.JPEG:n02018207 +ILSVRC2012_val_00012310.JPEG:n03530642 +ILSVRC2012_val_00012311.JPEG:n03782006 +ILSVRC2012_val_00012312.JPEG:n03697007 +ILSVRC2012_val_00012313.JPEG:n07734744 +ILSVRC2012_val_00012314.JPEG:n01820546 +ILSVRC2012_val_00012315.JPEG:n02280649 +ILSVRC2012_val_00012316.JPEG:n02115913 +ILSVRC2012_val_00012317.JPEG:n04325704 +ILSVRC2012_val_00012318.JPEG:n02104029 +ILSVRC2012_val_00012319.JPEG:n03250847 +ILSVRC2012_val_00012320.JPEG:n11879895 +ILSVRC2012_val_00012321.JPEG:n03709823 +ILSVRC2012_val_00012322.JPEG:n03271574 +ILSVRC2012_val_00012323.JPEG:n04483307 +ILSVRC2012_val_00012324.JPEG:n04525038 +ILSVRC2012_val_00012325.JPEG:n02835271 +ILSVRC2012_val_00012326.JPEG:n02102318 +ILSVRC2012_val_00012327.JPEG:n04285008 +ILSVRC2012_val_00012328.JPEG:n01491361 +ILSVRC2012_val_00012329.JPEG:n01742172 +ILSVRC2012_val_00012330.JPEG:n02077923 +ILSVRC2012_val_00012331.JPEG:n01728572 +ILSVRC2012_val_00012332.JPEG:n01914609 +ILSVRC2012_val_00012333.JPEG:n03388549 +ILSVRC2012_val_00012334.JPEG:n03085013 +ILSVRC2012_val_00012335.JPEG:n02395406 +ILSVRC2012_val_00012336.JPEG:n03868863 +ILSVRC2012_val_00012337.JPEG:n04033901 +ILSVRC2012_val_00012338.JPEG:n02011460 +ILSVRC2012_val_00012339.JPEG:n02123159 +ILSVRC2012_val_00012340.JPEG:n02391049 +ILSVRC2012_val_00012341.JPEG:n04039381 +ILSVRC2012_val_00012342.JPEG:n01695060 +ILSVRC2012_val_00012343.JPEG:n02129165 +ILSVRC2012_val_00012344.JPEG:n03944341 +ILSVRC2012_val_00012345.JPEG:n04462240 +ILSVRC2012_val_00012346.JPEG:n02403003 +ILSVRC2012_val_00012347.JPEG:n03920288 +ILSVRC2012_val_00012348.JPEG:n03649909 +ILSVRC2012_val_00012349.JPEG:n04515003 +ILSVRC2012_val_00012350.JPEG:n03372029 +ILSVRC2012_val_00012351.JPEG:n02091467 +ILSVRC2012_val_00012352.JPEG:n04372370 +ILSVRC2012_val_00012353.JPEG:n02129165 +ILSVRC2012_val_00012354.JPEG:n01753488 +ILSVRC2012_val_00012355.JPEG:n02113712 +ILSVRC2012_val_00012356.JPEG:n03445777 +ILSVRC2012_val_00012357.JPEG:n04525305 +ILSVRC2012_val_00012358.JPEG:n01768244 +ILSVRC2012_val_00012359.JPEG:n02493509 +ILSVRC2012_val_00012360.JPEG:n03743016 +ILSVRC2012_val_00012361.JPEG:n12998815 +ILSVRC2012_val_00012362.JPEG:n03770439 +ILSVRC2012_val_00012363.JPEG:n02777292 +ILSVRC2012_val_00012364.JPEG:n02097298 +ILSVRC2012_val_00012365.JPEG:n01687978 +ILSVRC2012_val_00012366.JPEG:n04179913 +ILSVRC2012_val_00012367.JPEG:n02749479 +ILSVRC2012_val_00012368.JPEG:n03627232 +ILSVRC2012_val_00012369.JPEG:n03207743 +ILSVRC2012_val_00012370.JPEG:n03476991 +ILSVRC2012_val_00012371.JPEG:n07745940 +ILSVRC2012_val_00012372.JPEG:n01883070 +ILSVRC2012_val_00012373.JPEG:n03792972 +ILSVRC2012_val_00012374.JPEG:n03769881 +ILSVRC2012_val_00012375.JPEG:n02011460 +ILSVRC2012_val_00012376.JPEG:n02870880 +ILSVRC2012_val_00012377.JPEG:n02123045 +ILSVRC2012_val_00012378.JPEG:n04040759 +ILSVRC2012_val_00012379.JPEG:n07684084 +ILSVRC2012_val_00012380.JPEG:n02111277 +ILSVRC2012_val_00012381.JPEG:n01877812 +ILSVRC2012_val_00012382.JPEG:n04019541 +ILSVRC2012_val_00012383.JPEG:n03197337 +ILSVRC2012_val_00012384.JPEG:n02494079 +ILSVRC2012_val_00012385.JPEG:n03187595 +ILSVRC2012_val_00012386.JPEG:n02687172 +ILSVRC2012_val_00012387.JPEG:n02883205 +ILSVRC2012_val_00012388.JPEG:n07754684 +ILSVRC2012_val_00012389.JPEG:n09399592 +ILSVRC2012_val_00012390.JPEG:n02791270 +ILSVRC2012_val_00012391.JPEG:n03063689 +ILSVRC2012_val_00012392.JPEG:n03902125 +ILSVRC2012_val_00012393.JPEG:n02415577 +ILSVRC2012_val_00012394.JPEG:n02086240 +ILSVRC2012_val_00012395.JPEG:n02093991 +ILSVRC2012_val_00012396.JPEG:n02802426 +ILSVRC2012_val_00012397.JPEG:n03782006 +ILSVRC2012_val_00012398.JPEG:n03478589 +ILSVRC2012_val_00012399.JPEG:n02128385 +ILSVRC2012_val_00012400.JPEG:n02894605 +ILSVRC2012_val_00012401.JPEG:n02115641 +ILSVRC2012_val_00012402.JPEG:n02011460 +ILSVRC2012_val_00012403.JPEG:n02951358 +ILSVRC2012_val_00012404.JPEG:n02128757 +ILSVRC2012_val_00012405.JPEG:n02871525 +ILSVRC2012_val_00012406.JPEG:n02346627 +ILSVRC2012_val_00012407.JPEG:n03450230 +ILSVRC2012_val_00012408.JPEG:n09229709 +ILSVRC2012_val_00012409.JPEG:n02417914 +ILSVRC2012_val_00012410.JPEG:n01796340 +ILSVRC2012_val_00012411.JPEG:n02128925 +ILSVRC2012_val_00012412.JPEG:n04486054 +ILSVRC2012_val_00012413.JPEG:n02749479 +ILSVRC2012_val_00012414.JPEG:n02346627 +ILSVRC2012_val_00012415.JPEG:n01930112 +ILSVRC2012_val_00012416.JPEG:n02091032 +ILSVRC2012_val_00012417.JPEG:n02963159 +ILSVRC2012_val_00012418.JPEG:n01944390 +ILSVRC2012_val_00012419.JPEG:n02793495 +ILSVRC2012_val_00012420.JPEG:n02018207 +ILSVRC2012_val_00012421.JPEG:n04153751 +ILSVRC2012_val_00012422.JPEG:n02790996 +ILSVRC2012_val_00012423.JPEG:n02129165 +ILSVRC2012_val_00012424.JPEG:n03538406 +ILSVRC2012_val_00012425.JPEG:n02965783 +ILSVRC2012_val_00012426.JPEG:n03179701 +ILSVRC2012_val_00012427.JPEG:n03160309 +ILSVRC2012_val_00012428.JPEG:n01644373 +ILSVRC2012_val_00012429.JPEG:n01770393 +ILSVRC2012_val_00012430.JPEG:n02109961 +ILSVRC2012_val_00012431.JPEG:n01873310 +ILSVRC2012_val_00012432.JPEG:n03085013 +ILSVRC2012_val_00012433.JPEG:n01735189 +ILSVRC2012_val_00012434.JPEG:n04370456 +ILSVRC2012_val_00012435.JPEG:n02018207 +ILSVRC2012_val_00012436.JPEG:n02018795 +ILSVRC2012_val_00012437.JPEG:n02110627 +ILSVRC2012_val_00012438.JPEG:n03804744 +ILSVRC2012_val_00012439.JPEG:n03534580 +ILSVRC2012_val_00012440.JPEG:n07760859 +ILSVRC2012_val_00012441.JPEG:n01631663 +ILSVRC2012_val_00012442.JPEG:n04482393 +ILSVRC2012_val_00012443.JPEG:n02917067 +ILSVRC2012_val_00012444.JPEG:n07753592 +ILSVRC2012_val_00012445.JPEG:n03447447 +ILSVRC2012_val_00012446.JPEG:n02112706 +ILSVRC2012_val_00012447.JPEG:n03947888 +ILSVRC2012_val_00012448.JPEG:n02927161 +ILSVRC2012_val_00012449.JPEG:n04228054 +ILSVRC2012_val_00012450.JPEG:n03259280 +ILSVRC2012_val_00012451.JPEG:n07753275 +ILSVRC2012_val_00012452.JPEG:n07753592 +ILSVRC2012_val_00012453.JPEG:n02948072 +ILSVRC2012_val_00012454.JPEG:n07697313 +ILSVRC2012_val_00012455.JPEG:n01984695 +ILSVRC2012_val_00012456.JPEG:n11879895 +ILSVRC2012_val_00012457.JPEG:n02125311 +ILSVRC2012_val_00012458.JPEG:n12998815 +ILSVRC2012_val_00012459.JPEG:n03976657 +ILSVRC2012_val_00012460.JPEG:n02096294 +ILSVRC2012_val_00012461.JPEG:n04264628 +ILSVRC2012_val_00012462.JPEG:n04548362 +ILSVRC2012_val_00012463.JPEG:n02276258 +ILSVRC2012_val_00012464.JPEG:n03891251 +ILSVRC2012_val_00012465.JPEG:n03127925 +ILSVRC2012_val_00012466.JPEG:n02834397 +ILSVRC2012_val_00012467.JPEG:n03854065 +ILSVRC2012_val_00012468.JPEG:n02979186 +ILSVRC2012_val_00012469.JPEG:n07920052 +ILSVRC2012_val_00012470.JPEG:n02110627 +ILSVRC2012_val_00012471.JPEG:n02095314 +ILSVRC2012_val_00012472.JPEG:n04049303 +ILSVRC2012_val_00012473.JPEG:n02965783 +ILSVRC2012_val_00012474.JPEG:n02895154 +ILSVRC2012_val_00012475.JPEG:n02013706 +ILSVRC2012_val_00012476.JPEG:n04044716 +ILSVRC2012_val_00012477.JPEG:n03709823 +ILSVRC2012_val_00012478.JPEG:n02138441 +ILSVRC2012_val_00012479.JPEG:n02777292 +ILSVRC2012_val_00012480.JPEG:n01943899 +ILSVRC2012_val_00012481.JPEG:n07892512 +ILSVRC2012_val_00012482.JPEG:n02091831 +ILSVRC2012_val_00012483.JPEG:n03743016 +ILSVRC2012_val_00012484.JPEG:n01514668 +ILSVRC2012_val_00012485.JPEG:n04243546 +ILSVRC2012_val_00012486.JPEG:n02105251 +ILSVRC2012_val_00012487.JPEG:n03032252 +ILSVRC2012_val_00012488.JPEG:n01855032 +ILSVRC2012_val_00012489.JPEG:n04612504 +ILSVRC2012_val_00012490.JPEG:n03770679 +ILSVRC2012_val_00012491.JPEG:n03866082 +ILSVRC2012_val_00012492.JPEG:n02091134 +ILSVRC2012_val_00012493.JPEG:n03443371 +ILSVRC2012_val_00012494.JPEG:n03777568 +ILSVRC2012_val_00012495.JPEG:n03773504 +ILSVRC2012_val_00012496.JPEG:n02480855 +ILSVRC2012_val_00012497.JPEG:n07745940 +ILSVRC2012_val_00012498.JPEG:n02391049 +ILSVRC2012_val_00012499.JPEG:n01910747 +ILSVRC2012_val_00012500.JPEG:n02277742 +ILSVRC2012_val_00012501.JPEG:n03938244 +ILSVRC2012_val_00012502.JPEG:n02788148 +ILSVRC2012_val_00012503.JPEG:n01440764 +ILSVRC2012_val_00012504.JPEG:n03425413 +ILSVRC2012_val_00012505.JPEG:n03895866 +ILSVRC2012_val_00012506.JPEG:n03950228 +ILSVRC2012_val_00012507.JPEG:n02133161 +ILSVRC2012_val_00012508.JPEG:n01843065 +ILSVRC2012_val_00012509.JPEG:n02992211 +ILSVRC2012_val_00012510.JPEG:n02834397 +ILSVRC2012_val_00012511.JPEG:n02066245 +ILSVRC2012_val_00012512.JPEG:n03337140 +ILSVRC2012_val_00012513.JPEG:n07716358 +ILSVRC2012_val_00012514.JPEG:n03584829 +ILSVRC2012_val_00012515.JPEG:n02095314 +ILSVRC2012_val_00012516.JPEG:n02093991 +ILSVRC2012_val_00012517.JPEG:n02974003 +ILSVRC2012_val_00012518.JPEG:n02025239 +ILSVRC2012_val_00012519.JPEG:n04596742 +ILSVRC2012_val_00012520.JPEG:n02916936 +ILSVRC2012_val_00012521.JPEG:n01768244 +ILSVRC2012_val_00012522.JPEG:n03720891 +ILSVRC2012_val_00012523.JPEG:n02056570 +ILSVRC2012_val_00012524.JPEG:n02102177 +ILSVRC2012_val_00012525.JPEG:n04557648 +ILSVRC2012_val_00012526.JPEG:n02268853 +ILSVRC2012_val_00012527.JPEG:n02098105 +ILSVRC2012_val_00012528.JPEG:n01514859 +ILSVRC2012_val_00012529.JPEG:n04141975 +ILSVRC2012_val_00012530.JPEG:n02071294 +ILSVRC2012_val_00012531.JPEG:n03188531 +ILSVRC2012_val_00012532.JPEG:n04254777 +ILSVRC2012_val_00012533.JPEG:n03709823 +ILSVRC2012_val_00012534.JPEG:n03095699 +ILSVRC2012_val_00012535.JPEG:n04517823 +ILSVRC2012_val_00012536.JPEG:n03733131 +ILSVRC2012_val_00012537.JPEG:n07693725 +ILSVRC2012_val_00012538.JPEG:n03476684 +ILSVRC2012_val_00012539.JPEG:n03724870 +ILSVRC2012_val_00012540.JPEG:n03983396 +ILSVRC2012_val_00012541.JPEG:n02342885 +ILSVRC2012_val_00012542.JPEG:n02510455 +ILSVRC2012_val_00012543.JPEG:n03874293 +ILSVRC2012_val_00012544.JPEG:n02823428 +ILSVRC2012_val_00012545.JPEG:n04356056 +ILSVRC2012_val_00012546.JPEG:n01494475 +ILSVRC2012_val_00012547.JPEG:n04251144 +ILSVRC2012_val_00012548.JPEG:n02894605 +ILSVRC2012_val_00012549.JPEG:n02097658 +ILSVRC2012_val_00012550.JPEG:n04273569 +ILSVRC2012_val_00012551.JPEG:n02123045 +ILSVRC2012_val_00012552.JPEG:n03250847 +ILSVRC2012_val_00012553.JPEG:n01687978 +ILSVRC2012_val_00012554.JPEG:n02012849 +ILSVRC2012_val_00012555.JPEG:n03733131 +ILSVRC2012_val_00012556.JPEG:n02096294 +ILSVRC2012_val_00012557.JPEG:n02279972 +ILSVRC2012_val_00012558.JPEG:n01641577 +ILSVRC2012_val_00012559.JPEG:n03804744 +ILSVRC2012_val_00012560.JPEG:n02871525 +ILSVRC2012_val_00012561.JPEG:n04479046 +ILSVRC2012_val_00012562.JPEG:n07697313 +ILSVRC2012_val_00012563.JPEG:n02786058 +ILSVRC2012_val_00012564.JPEG:n01924916 +ILSVRC2012_val_00012565.JPEG:n07932039 +ILSVRC2012_val_00012566.JPEG:n02099712 +ILSVRC2012_val_00012567.JPEG:n03271574 +ILSVRC2012_val_00012568.JPEG:n02488702 +ILSVRC2012_val_00012569.JPEG:n02927161 +ILSVRC2012_val_00012570.JPEG:n02815834 +ILSVRC2012_val_00012571.JPEG:n02877765 +ILSVRC2012_val_00012572.JPEG:n04560804 +ILSVRC2012_val_00012573.JPEG:n03297495 +ILSVRC2012_val_00012574.JPEG:n04590129 +ILSVRC2012_val_00012575.JPEG:n03944341 +ILSVRC2012_val_00012576.JPEG:n03980874 +ILSVRC2012_val_00012577.JPEG:n02105056 +ILSVRC2012_val_00012578.JPEG:n01734418 +ILSVRC2012_val_00012579.JPEG:n03947888 +ILSVRC2012_val_00012580.JPEG:n02363005 +ILSVRC2012_val_00012581.JPEG:n06596364 +ILSVRC2012_val_00012582.JPEG:n07753275 +ILSVRC2012_val_00012583.JPEG:n02930766 +ILSVRC2012_val_00012584.JPEG:n02093859 +ILSVRC2012_val_00012585.JPEG:n03207941 +ILSVRC2012_val_00012586.JPEG:n01818515 +ILSVRC2012_val_00012587.JPEG:n03657121 +ILSVRC2012_val_00012588.JPEG:n01629819 +ILSVRC2012_val_00012589.JPEG:n03063689 +ILSVRC2012_val_00012590.JPEG:n03255030 +ILSVRC2012_val_00012591.JPEG:n02808440 +ILSVRC2012_val_00012592.JPEG:n02981792 +ILSVRC2012_val_00012593.JPEG:n09246464 +ILSVRC2012_val_00012594.JPEG:n04591713 +ILSVRC2012_val_00012595.JPEG:n03492542 +ILSVRC2012_val_00012596.JPEG:n04517823 +ILSVRC2012_val_00012597.JPEG:n03240683 +ILSVRC2012_val_00012598.JPEG:n07716358 +ILSVRC2012_val_00012599.JPEG:n07717556 +ILSVRC2012_val_00012600.JPEG:n02814533 +ILSVRC2012_val_00012601.JPEG:n01843383 +ILSVRC2012_val_00012602.JPEG:n03691459 +ILSVRC2012_val_00012603.JPEG:n02134418 +ILSVRC2012_val_00012604.JPEG:n02110185 +ILSVRC2012_val_00012605.JPEG:n02093754 +ILSVRC2012_val_00012606.JPEG:n02807133 +ILSVRC2012_val_00012607.JPEG:n07684084 +ILSVRC2012_val_00012608.JPEG:n02091244 +ILSVRC2012_val_00012609.JPEG:n03873416 +ILSVRC2012_val_00012610.JPEG:n02113624 +ILSVRC2012_val_00012611.JPEG:n02094433 +ILSVRC2012_val_00012612.JPEG:n02917067 +ILSVRC2012_val_00012613.JPEG:n03450230 +ILSVRC2012_val_00012614.JPEG:n03888605 +ILSVRC2012_val_00012615.JPEG:n01616318 +ILSVRC2012_val_00012616.JPEG:n04435653 +ILSVRC2012_val_00012617.JPEG:n02111277 +ILSVRC2012_val_00012618.JPEG:n02006656 +ILSVRC2012_val_00012619.JPEG:n02363005 +ILSVRC2012_val_00012620.JPEG:n02497673 +ILSVRC2012_val_00012621.JPEG:n07753592 +ILSVRC2012_val_00012622.JPEG:n07711569 +ILSVRC2012_val_00012623.JPEG:n01693334 +ILSVRC2012_val_00012624.JPEG:n03954731 +ILSVRC2012_val_00012625.JPEG:n04033995 +ILSVRC2012_val_00012626.JPEG:n04208210 +ILSVRC2012_val_00012627.JPEG:n02817516 +ILSVRC2012_val_00012628.JPEG:n07754684 +ILSVRC2012_val_00012629.JPEG:n02256656 +ILSVRC2012_val_00012630.JPEG:n13052670 +ILSVRC2012_val_00012631.JPEG:n04417672 +ILSVRC2012_val_00012632.JPEG:n11939491 +ILSVRC2012_val_00012633.JPEG:n02443114 +ILSVRC2012_val_00012634.JPEG:n03445777 +ILSVRC2012_val_00012635.JPEG:n02093859 +ILSVRC2012_val_00012636.JPEG:n07684084 +ILSVRC2012_val_00012637.JPEG:n03026506 +ILSVRC2012_val_00012638.JPEG:n04081281 +ILSVRC2012_val_00012639.JPEG:n02002724 +ILSVRC2012_val_00012640.JPEG:n02317335 +ILSVRC2012_val_00012641.JPEG:n03584829 +ILSVRC2012_val_00012642.JPEG:n04039381 +ILSVRC2012_val_00012643.JPEG:n03062245 +ILSVRC2012_val_00012644.JPEG:n02091134 +ILSVRC2012_val_00012645.JPEG:n07745940 +ILSVRC2012_val_00012646.JPEG:n02092002 +ILSVRC2012_val_00012647.JPEG:n03991062 +ILSVRC2012_val_00012648.JPEG:n02843684 +ILSVRC2012_val_00012649.JPEG:n03961711 +ILSVRC2012_val_00012650.JPEG:n04069434 +ILSVRC2012_val_00012651.JPEG:n01558993 +ILSVRC2012_val_00012652.JPEG:n07745940 +ILSVRC2012_val_00012653.JPEG:n04486054 +ILSVRC2012_val_00012654.JPEG:n04347754 +ILSVRC2012_val_00012655.JPEG:n02011460 +ILSVRC2012_val_00012656.JPEG:n02808304 +ILSVRC2012_val_00012657.JPEG:n02109961 +ILSVRC2012_val_00012658.JPEG:n04229816 +ILSVRC2012_val_00012659.JPEG:n04409515 +ILSVRC2012_val_00012660.JPEG:n04116512 +ILSVRC2012_val_00012661.JPEG:n03857828 +ILSVRC2012_val_00012662.JPEG:n02445715 +ILSVRC2012_val_00012663.JPEG:n03920288 +ILSVRC2012_val_00012664.JPEG:n02488702 +ILSVRC2012_val_00012665.JPEG:n03126707 +ILSVRC2012_val_00012666.JPEG:n07932039 +ILSVRC2012_val_00012667.JPEG:n02835271 +ILSVRC2012_val_00012668.JPEG:n03445924 +ILSVRC2012_val_00012669.JPEG:n01797886 +ILSVRC2012_val_00012670.JPEG:n03476684 +ILSVRC2012_val_00012671.JPEG:n03658185 +ILSVRC2012_val_00012672.JPEG:n01943899 +ILSVRC2012_val_00012673.JPEG:n02951358 +ILSVRC2012_val_00012674.JPEG:n03532672 +ILSVRC2012_val_00012675.JPEG:n02966193 +ILSVRC2012_val_00012676.JPEG:n02988304 +ILSVRC2012_val_00012677.JPEG:n02229544 +ILSVRC2012_val_00012678.JPEG:n02095570 +ILSVRC2012_val_00012679.JPEG:n02841315 +ILSVRC2012_val_00012680.JPEG:n04536866 +ILSVRC2012_val_00012681.JPEG:n02268853 +ILSVRC2012_val_00012682.JPEG:n03445924 +ILSVRC2012_val_00012683.JPEG:n03803284 +ILSVRC2012_val_00012684.JPEG:n04254777 +ILSVRC2012_val_00012685.JPEG:n02443484 +ILSVRC2012_val_00012686.JPEG:n03133878 +ILSVRC2012_val_00012687.JPEG:n02799071 +ILSVRC2012_val_00012688.JPEG:n13133613 +ILSVRC2012_val_00012689.JPEG:n02102040 +ILSVRC2012_val_00012690.JPEG:n02107908 +ILSVRC2012_val_00012691.JPEG:n03947888 +ILSVRC2012_val_00012692.JPEG:n04487394 +ILSVRC2012_val_00012693.JPEG:n03599486 +ILSVRC2012_val_00012694.JPEG:n03452741 +ILSVRC2012_val_00012695.JPEG:n02097298 +ILSVRC2012_val_00012696.JPEG:n04417672 +ILSVRC2012_val_00012697.JPEG:n02493793 +ILSVRC2012_val_00012698.JPEG:n02325366 +ILSVRC2012_val_00012699.JPEG:n07747607 +ILSVRC2012_val_00012700.JPEG:n03188531 +ILSVRC2012_val_00012701.JPEG:n04482393 +ILSVRC2012_val_00012702.JPEG:n02088632 +ILSVRC2012_val_00012703.JPEG:n04461696 +ILSVRC2012_val_00012704.JPEG:n03249569 +ILSVRC2012_val_00012705.JPEG:n07693725 +ILSVRC2012_val_00012706.JPEG:n02096437 +ILSVRC2012_val_00012707.JPEG:n01773797 +ILSVRC2012_val_00012708.JPEG:n02105162 +ILSVRC2012_val_00012709.JPEG:n02843684 +ILSVRC2012_val_00012710.JPEG:n02950826 +ILSVRC2012_val_00012711.JPEG:n02492660 +ILSVRC2012_val_00012712.JPEG:n04366367 +ILSVRC2012_val_00012713.JPEG:n01981276 +ILSVRC2012_val_00012714.JPEG:n03207941 +ILSVRC2012_val_00012715.JPEG:n02966193 +ILSVRC2012_val_00012716.JPEG:n03534580 +ILSVRC2012_val_00012717.JPEG:n02112018 +ILSVRC2012_val_00012718.JPEG:n01688243 +ILSVRC2012_val_00012719.JPEG:n04584207 +ILSVRC2012_val_00012720.JPEG:n02415577 +ILSVRC2012_val_00012721.JPEG:n01847000 +ILSVRC2012_val_00012722.JPEG:n02514041 +ILSVRC2012_val_00012723.JPEG:n02488291 +ILSVRC2012_val_00012724.JPEG:n02749479 +ILSVRC2012_val_00012725.JPEG:n04380533 +ILSVRC2012_val_00012726.JPEG:n02510455 +ILSVRC2012_val_00012727.JPEG:n02526121 +ILSVRC2012_val_00012728.JPEG:n07745940 +ILSVRC2012_val_00012729.JPEG:n03930313 +ILSVRC2012_val_00012730.JPEG:n03877845 +ILSVRC2012_val_00012731.JPEG:n01755581 +ILSVRC2012_val_00012732.JPEG:n01667114 +ILSVRC2012_val_00012733.JPEG:n02108000 +ILSVRC2012_val_00012734.JPEG:n02699494 +ILSVRC2012_val_00012735.JPEG:n02363005 +ILSVRC2012_val_00012736.JPEG:n02100877 +ILSVRC2012_val_00012737.JPEG:n03770439 +ILSVRC2012_val_00012738.JPEG:n02114712 +ILSVRC2012_val_00012739.JPEG:n02100735 +ILSVRC2012_val_00012740.JPEG:n02108000 +ILSVRC2012_val_00012741.JPEG:n02028035 +ILSVRC2012_val_00012742.JPEG:n02108551 +ILSVRC2012_val_00012743.JPEG:n02484975 +ILSVRC2012_val_00012744.JPEG:n07718747 +ILSVRC2012_val_00012745.JPEG:n03498962 +ILSVRC2012_val_00012746.JPEG:n01665541 +ILSVRC2012_val_00012747.JPEG:n02894605 +ILSVRC2012_val_00012748.JPEG:n04118776 +ILSVRC2012_val_00012749.JPEG:n02119022 +ILSVRC2012_val_00012750.JPEG:n04258138 +ILSVRC2012_val_00012751.JPEG:n04604644 +ILSVRC2012_val_00012752.JPEG:n02115641 +ILSVRC2012_val_00012753.JPEG:n07768694 +ILSVRC2012_val_00012754.JPEG:n12267677 +ILSVRC2012_val_00012755.JPEG:n03908714 +ILSVRC2012_val_00012756.JPEG:n03876231 +ILSVRC2012_val_00012757.JPEG:n07717556 +ILSVRC2012_val_00012758.JPEG:n11879895 +ILSVRC2012_val_00012759.JPEG:n01688243 +ILSVRC2012_val_00012760.JPEG:n03208938 +ILSVRC2012_val_00012761.JPEG:n12267677 +ILSVRC2012_val_00012762.JPEG:n02669723 +ILSVRC2012_val_00012763.JPEG:n02965783 +ILSVRC2012_val_00012764.JPEG:n02276258 +ILSVRC2012_val_00012765.JPEG:n01631663 +ILSVRC2012_val_00012766.JPEG:n04487394 +ILSVRC2012_val_00012767.JPEG:n02825657 +ILSVRC2012_val_00012768.JPEG:n01749939 +ILSVRC2012_val_00012769.JPEG:n04037443 +ILSVRC2012_val_00012770.JPEG:n04041544 +ILSVRC2012_val_00012771.JPEG:n03376595 +ILSVRC2012_val_00012772.JPEG:n04532670 +ILSVRC2012_val_00012773.JPEG:n02104365 +ILSVRC2012_val_00012774.JPEG:n02233338 +ILSVRC2012_val_00012775.JPEG:n02793495 +ILSVRC2012_val_00012776.JPEG:n03770439 +ILSVRC2012_val_00012777.JPEG:n01910747 +ILSVRC2012_val_00012778.JPEG:n04154565 +ILSVRC2012_val_00012779.JPEG:n01980166 +ILSVRC2012_val_00012780.JPEG:n03793489 +ILSVRC2012_val_00012781.JPEG:n02025239 +ILSVRC2012_val_00012782.JPEG:n02480495 +ILSVRC2012_val_00012783.JPEG:n03781244 +ILSVRC2012_val_00012784.JPEG:n04399382 +ILSVRC2012_val_00012785.JPEG:n07871810 +ILSVRC2012_val_00012786.JPEG:n04065272 +ILSVRC2012_val_00012787.JPEG:n02017213 +ILSVRC2012_val_00012788.JPEG:n01943899 +ILSVRC2012_val_00012789.JPEG:n04067472 +ILSVRC2012_val_00012790.JPEG:n03761084 +ILSVRC2012_val_00012791.JPEG:n02094433 +ILSVRC2012_val_00012792.JPEG:n03538406 +ILSVRC2012_val_00012793.JPEG:n02494079 +ILSVRC2012_val_00012794.JPEG:n04147183 +ILSVRC2012_val_00012795.JPEG:n04141076 +ILSVRC2012_val_00012796.JPEG:n04589890 +ILSVRC2012_val_00012797.JPEG:n01601694 +ILSVRC2012_val_00012798.JPEG:n02123394 +ILSVRC2012_val_00012799.JPEG:n06874185 +ILSVRC2012_val_00012800.JPEG:n02114548 +ILSVRC2012_val_00012801.JPEG:n03637318 +ILSVRC2012_val_00012802.JPEG:n03710193 +ILSVRC2012_val_00012803.JPEG:n04536866 +ILSVRC2012_val_00012804.JPEG:n09399592 +ILSVRC2012_val_00012805.JPEG:n03452741 +ILSVRC2012_val_00012806.JPEG:n03594945 +ILSVRC2012_val_00012807.JPEG:n07860988 +ILSVRC2012_val_00012808.JPEG:n03085013 +ILSVRC2012_val_00012809.JPEG:n02814533 +ILSVRC2012_val_00012810.JPEG:n03461385 +ILSVRC2012_val_00012811.JPEG:n04252077 +ILSVRC2012_val_00012812.JPEG:n02859443 +ILSVRC2012_val_00012813.JPEG:n04033901 +ILSVRC2012_val_00012814.JPEG:n01530575 +ILSVRC2012_val_00012815.JPEG:n03476684 +ILSVRC2012_val_00012816.JPEG:n04069434 +ILSVRC2012_val_00012817.JPEG:n02105056 +ILSVRC2012_val_00012818.JPEG:n02128385 +ILSVRC2012_val_00012819.JPEG:n01694178 +ILSVRC2012_val_00012820.JPEG:n01688243 +ILSVRC2012_val_00012821.JPEG:n03372029 +ILSVRC2012_val_00012822.JPEG:n04465501 +ILSVRC2012_val_00012823.JPEG:n02808440 +ILSVRC2012_val_00012824.JPEG:n04235860 +ILSVRC2012_val_00012825.JPEG:n02177972 +ILSVRC2012_val_00012826.JPEG:n13044778 +ILSVRC2012_val_00012827.JPEG:n02096177 +ILSVRC2012_val_00012828.JPEG:n01770081 +ILSVRC2012_val_00012829.JPEG:n01669191 +ILSVRC2012_val_00012830.JPEG:n02481823 +ILSVRC2012_val_00012831.JPEG:n07880968 +ILSVRC2012_val_00012832.JPEG:n03888605 +ILSVRC2012_val_00012833.JPEG:n02117135 +ILSVRC2012_val_00012834.JPEG:n02096437 +ILSVRC2012_val_00012835.JPEG:n02397096 +ILSVRC2012_val_00012836.JPEG:n01592084 +ILSVRC2012_val_00012837.JPEG:n03769881 +ILSVRC2012_val_00012838.JPEG:n03026506 +ILSVRC2012_val_00012839.JPEG:n02107574 +ILSVRC2012_val_00012840.JPEG:n02114367 +ILSVRC2012_val_00012841.JPEG:n03124170 +ILSVRC2012_val_00012842.JPEG:n03733281 +ILSVRC2012_val_00012843.JPEG:n03692522 +ILSVRC2012_val_00012844.JPEG:n02037110 +ILSVRC2012_val_00012845.JPEG:n02167151 +ILSVRC2012_val_00012846.JPEG:n01930112 +ILSVRC2012_val_00012847.JPEG:n03995372 +ILSVRC2012_val_00012848.JPEG:n03355925 +ILSVRC2012_val_00012849.JPEG:n03676483 +ILSVRC2012_val_00012850.JPEG:n03000247 +ILSVRC2012_val_00012851.JPEG:n02966193 +ILSVRC2012_val_00012852.JPEG:n02910353 +ILSVRC2012_val_00012853.JPEG:n01682714 +ILSVRC2012_val_00012854.JPEG:n02910353 +ILSVRC2012_val_00012855.JPEG:n02510455 +ILSVRC2012_val_00012856.JPEG:n02106550 +ILSVRC2012_val_00012857.JPEG:n02120079 +ILSVRC2012_val_00012858.JPEG:n03841143 +ILSVRC2012_val_00012859.JPEG:n04229816 +ILSVRC2012_val_00012860.JPEG:n02447366 +ILSVRC2012_val_00012861.JPEG:n02091467 +ILSVRC2012_val_00012862.JPEG:n04456115 +ILSVRC2012_val_00012863.JPEG:n03937543 +ILSVRC2012_val_00012864.JPEG:n01818515 +ILSVRC2012_val_00012865.JPEG:n04086273 +ILSVRC2012_val_00012866.JPEG:n02865351 +ILSVRC2012_val_00012867.JPEG:n03109150 +ILSVRC2012_val_00012868.JPEG:n02808304 +ILSVRC2012_val_00012869.JPEG:n03483316 +ILSVRC2012_val_00012870.JPEG:n01560419 +ILSVRC2012_val_00012871.JPEG:n07930864 +ILSVRC2012_val_00012872.JPEG:n04392985 +ILSVRC2012_val_00012873.JPEG:n04592741 +ILSVRC2012_val_00012874.JPEG:n04192698 +ILSVRC2012_val_00012875.JPEG:n02089973 +ILSVRC2012_val_00012876.JPEG:n03485794 +ILSVRC2012_val_00012877.JPEG:n07613480 +ILSVRC2012_val_00012878.JPEG:n02951585 +ILSVRC2012_val_00012879.JPEG:n01494475 +ILSVRC2012_val_00012880.JPEG:n01443537 +ILSVRC2012_val_00012881.JPEG:n02097298 +ILSVRC2012_val_00012882.JPEG:n02877765 +ILSVRC2012_val_00012883.JPEG:n02101388 +ILSVRC2012_val_00012884.JPEG:n03271574 +ILSVRC2012_val_00012885.JPEG:n03041632 +ILSVRC2012_val_00012886.JPEG:n03895866 +ILSVRC2012_val_00012887.JPEG:n02865351 +ILSVRC2012_val_00012888.JPEG:n02091134 +ILSVRC2012_val_00012889.JPEG:n02027492 +ILSVRC2012_val_00012890.JPEG:n03201208 +ILSVRC2012_val_00012891.JPEG:n03983396 +ILSVRC2012_val_00012892.JPEG:n02364673 +ILSVRC2012_val_00012893.JPEG:n02134084 +ILSVRC2012_val_00012894.JPEG:n02165105 +ILSVRC2012_val_00012895.JPEG:n01773549 +ILSVRC2012_val_00012896.JPEG:n04127249 +ILSVRC2012_val_00012897.JPEG:n04275548 +ILSVRC2012_val_00012898.JPEG:n01883070 +ILSVRC2012_val_00012899.JPEG:n02112706 +ILSVRC2012_val_00012900.JPEG:n03776460 +ILSVRC2012_val_00012901.JPEG:n02108000 +ILSVRC2012_val_00012902.JPEG:n02397096 +ILSVRC2012_val_00012903.JPEG:n04525305 +ILSVRC2012_val_00012904.JPEG:n02113624 +ILSVRC2012_val_00012905.JPEG:n02268853 +ILSVRC2012_val_00012906.JPEG:n02091134 +ILSVRC2012_val_00012907.JPEG:n03476991 +ILSVRC2012_val_00012908.JPEG:n02815834 +ILSVRC2012_val_00012909.JPEG:n04525305 +ILSVRC2012_val_00012910.JPEG:n03857828 +ILSVRC2012_val_00012911.JPEG:n03272010 +ILSVRC2012_val_00012912.JPEG:n04523525 +ILSVRC2012_val_00012913.JPEG:n04335435 +ILSVRC2012_val_00012914.JPEG:n03595614 +ILSVRC2012_val_00012915.JPEG:n07932039 +ILSVRC2012_val_00012916.JPEG:n03345487 +ILSVRC2012_val_00012917.JPEG:n03877472 +ILSVRC2012_val_00012918.JPEG:n04485082 +ILSVRC2012_val_00012919.JPEG:n02794156 +ILSVRC2012_val_00012920.JPEG:n03877472 +ILSVRC2012_val_00012921.JPEG:n03492542 +ILSVRC2012_val_00012922.JPEG:n02114712 +ILSVRC2012_val_00012923.JPEG:n02883205 +ILSVRC2012_val_00012924.JPEG:n02106662 +ILSVRC2012_val_00012925.JPEG:n03417042 +ILSVRC2012_val_00012926.JPEG:n03617480 +ILSVRC2012_val_00012927.JPEG:n02978881 +ILSVRC2012_val_00012928.JPEG:n02101556 +ILSVRC2012_val_00012929.JPEG:n04039381 +ILSVRC2012_val_00012930.JPEG:n02105641 +ILSVRC2012_val_00012931.JPEG:n02098413 +ILSVRC2012_val_00012932.JPEG:n04552348 +ILSVRC2012_val_00012933.JPEG:n02823750 +ILSVRC2012_val_00012934.JPEG:n07753113 +ILSVRC2012_val_00012935.JPEG:n02110063 +ILSVRC2012_val_00012936.JPEG:n09332890 +ILSVRC2012_val_00012937.JPEG:n09468604 +ILSVRC2012_val_00012938.JPEG:n02457408 +ILSVRC2012_val_00012939.JPEG:n01537544 +ILSVRC2012_val_00012940.JPEG:n02497673 +ILSVRC2012_val_00012941.JPEG:n09229709 +ILSVRC2012_val_00012942.JPEG:n04311004 +ILSVRC2012_val_00012943.JPEG:n02776631 +ILSVRC2012_val_00012944.JPEG:n02692877 +ILSVRC2012_val_00012945.JPEG:n03623198 +ILSVRC2012_val_00012946.JPEG:n04328186 +ILSVRC2012_val_00012947.JPEG:n03697007 +ILSVRC2012_val_00012948.JPEG:n02102177 +ILSVRC2012_val_00012949.JPEG:n01687978 +ILSVRC2012_val_00012950.JPEG:n03207743 +ILSVRC2012_val_00012951.JPEG:n03733131 +ILSVRC2012_val_00012952.JPEG:n02099429 +ILSVRC2012_val_00012953.JPEG:n03769881 +ILSVRC2012_val_00012954.JPEG:n02099601 +ILSVRC2012_val_00012955.JPEG:n02787622 +ILSVRC2012_val_00012956.JPEG:n03000134 +ILSVRC2012_val_00012957.JPEG:n03895866 +ILSVRC2012_val_00012958.JPEG:n02127052 +ILSVRC2012_val_00012959.JPEG:n04136333 +ILSVRC2012_val_00012960.JPEG:n02106662 +ILSVRC2012_val_00012961.JPEG:n13044778 +ILSVRC2012_val_00012962.JPEG:n01981276 +ILSVRC2012_val_00012963.JPEG:n03680355 +ILSVRC2012_val_00012964.JPEG:n03372029 +ILSVRC2012_val_00012965.JPEG:n03908618 +ILSVRC2012_val_00012966.JPEG:n03877472 +ILSVRC2012_val_00012967.JPEG:n04346328 +ILSVRC2012_val_00012968.JPEG:n04557648 +ILSVRC2012_val_00012969.JPEG:n04270147 +ILSVRC2012_val_00012970.JPEG:n04428191 +ILSVRC2012_val_00012971.JPEG:n02870880 +ILSVRC2012_val_00012972.JPEG:n03297495 +ILSVRC2012_val_00012973.JPEG:n02871525 +ILSVRC2012_val_00012974.JPEG:n02391049 +ILSVRC2012_val_00012975.JPEG:n02123045 +ILSVRC2012_val_00012976.JPEG:n01871265 +ILSVRC2012_val_00012977.JPEG:n02071294 +ILSVRC2012_val_00012978.JPEG:n02119022 +ILSVRC2012_val_00012979.JPEG:n04592741 +ILSVRC2012_val_00012980.JPEG:n02509815 +ILSVRC2012_val_00012981.JPEG:n03424325 +ILSVRC2012_val_00012982.JPEG:n02514041 +ILSVRC2012_val_00012983.JPEG:n02101006 +ILSVRC2012_val_00012984.JPEG:n02747177 +ILSVRC2012_val_00012985.JPEG:n01950731 +ILSVRC2012_val_00012986.JPEG:n02172182 +ILSVRC2012_val_00012987.JPEG:n04336792 +ILSVRC2012_val_00012988.JPEG:n04356056 +ILSVRC2012_val_00012989.JPEG:n04252077 +ILSVRC2012_val_00012990.JPEG:n01740131 +ILSVRC2012_val_00012991.JPEG:n04613696 +ILSVRC2012_val_00012992.JPEG:n04023962 +ILSVRC2012_val_00012993.JPEG:n04485082 +ILSVRC2012_val_00012994.JPEG:n02128925 +ILSVRC2012_val_00012995.JPEG:n02086079 +ILSVRC2012_val_00012996.JPEG:n03983396 +ILSVRC2012_val_00012997.JPEG:n02134084 +ILSVRC2012_val_00012998.JPEG:n02133161 +ILSVRC2012_val_00012999.JPEG:n02128925 +ILSVRC2012_val_00013000.JPEG:n04517823 +ILSVRC2012_val_00013001.JPEG:n07875152 +ILSVRC2012_val_00013002.JPEG:n02128385 +ILSVRC2012_val_00013003.JPEG:n04204347 +ILSVRC2012_val_00013004.JPEG:n02077923 +ILSVRC2012_val_00013005.JPEG:n03272010 +ILSVRC2012_val_00013006.JPEG:n02840245 +ILSVRC2012_val_00013007.JPEG:n02105641 +ILSVRC2012_val_00013008.JPEG:n01817953 +ILSVRC2012_val_00013009.JPEG:n04146614 +ILSVRC2012_val_00013010.JPEG:n04554684 +ILSVRC2012_val_00013011.JPEG:n03796401 +ILSVRC2012_val_00013012.JPEG:n04039381 +ILSVRC2012_val_00013013.JPEG:n02788148 +ILSVRC2012_val_00013014.JPEG:n04483307 +ILSVRC2012_val_00013015.JPEG:n02493793 +ILSVRC2012_val_00013016.JPEG:n03692522 +ILSVRC2012_val_00013017.JPEG:n03075370 +ILSVRC2012_val_00013018.JPEG:n03733281 +ILSVRC2012_val_00013019.JPEG:n04238763 +ILSVRC2012_val_00013020.JPEG:n02815834 +ILSVRC2012_val_00013021.JPEG:n03065424 +ILSVRC2012_val_00013022.JPEG:n02672831 +ILSVRC2012_val_00013023.JPEG:n03602883 +ILSVRC2012_val_00013024.JPEG:n04346328 +ILSVRC2012_val_00013025.JPEG:n02066245 +ILSVRC2012_val_00013026.JPEG:n03444034 +ILSVRC2012_val_00013027.JPEG:n03594734 +ILSVRC2012_val_00013028.JPEG:n15075141 +ILSVRC2012_val_00013029.JPEG:n12144580 +ILSVRC2012_val_00013030.JPEG:n07579787 +ILSVRC2012_val_00013031.JPEG:n02992529 +ILSVRC2012_val_00013032.JPEG:n04515003 +ILSVRC2012_val_00013033.JPEG:n02107142 +ILSVRC2012_val_00013034.JPEG:n02117135 +ILSVRC2012_val_00013035.JPEG:n01734418 +ILSVRC2012_val_00013036.JPEG:n01693334 +ILSVRC2012_val_00013037.JPEG:n02105505 +ILSVRC2012_val_00013038.JPEG:n02992211 +ILSVRC2012_val_00013039.JPEG:n02869837 +ILSVRC2012_val_00013040.JPEG:n13133613 +ILSVRC2012_val_00013041.JPEG:n02666196 +ILSVRC2012_val_00013042.JPEG:n04041544 +ILSVRC2012_val_00013043.JPEG:n03857828 +ILSVRC2012_val_00013044.JPEG:n04418357 +ILSVRC2012_val_00013045.JPEG:n02113978 +ILSVRC2012_val_00013046.JPEG:n01744401 +ILSVRC2012_val_00013047.JPEG:n02797295 +ILSVRC2012_val_00013048.JPEG:n02699494 +ILSVRC2012_val_00013049.JPEG:n02489166 +ILSVRC2012_val_00013050.JPEG:n02098286 +ILSVRC2012_val_00013051.JPEG:n04243546 +ILSVRC2012_val_00013052.JPEG:n02134418 +ILSVRC2012_val_00013053.JPEG:n02106662 +ILSVRC2012_val_00013054.JPEG:n03670208 +ILSVRC2012_val_00013055.JPEG:n04090263 +ILSVRC2012_val_00013056.JPEG:n02692877 +ILSVRC2012_val_00013057.JPEG:n03467068 +ILSVRC2012_val_00013058.JPEG:n04238763 +ILSVRC2012_val_00013059.JPEG:n03788365 +ILSVRC2012_val_00013060.JPEG:n03657121 +ILSVRC2012_val_00013061.JPEG:n02906734 +ILSVRC2012_val_00013062.JPEG:n02326432 +ILSVRC2012_val_00013063.JPEG:n02676566 +ILSVRC2012_val_00013064.JPEG:n02607072 +ILSVRC2012_val_00013065.JPEG:n03627232 +ILSVRC2012_val_00013066.JPEG:n02894605 +ILSVRC2012_val_00013067.JPEG:n03538406 +ILSVRC2012_val_00013068.JPEG:n04136333 +ILSVRC2012_val_00013069.JPEG:n01632458 +ILSVRC2012_val_00013070.JPEG:n04125021 +ILSVRC2012_val_00013071.JPEG:n03134739 +ILSVRC2012_val_00013072.JPEG:n01697457 +ILSVRC2012_val_00013073.JPEG:n03924679 +ILSVRC2012_val_00013074.JPEG:n04243546 +ILSVRC2012_val_00013075.JPEG:n09256479 +ILSVRC2012_val_00013076.JPEG:n02493793 +ILSVRC2012_val_00013077.JPEG:n07871810 +ILSVRC2012_val_00013078.JPEG:n02177972 +ILSVRC2012_val_00013079.JPEG:n01917289 +ILSVRC2012_val_00013080.JPEG:n02088466 +ILSVRC2012_val_00013081.JPEG:n04069434 +ILSVRC2012_val_00013082.JPEG:n03891251 +ILSVRC2012_val_00013083.JPEG:n02113799 +ILSVRC2012_val_00013084.JPEG:n07711569 +ILSVRC2012_val_00013085.JPEG:n01833805 +ILSVRC2012_val_00013086.JPEG:n04270147 +ILSVRC2012_val_00013087.JPEG:n04259630 +ILSVRC2012_val_00013088.JPEG:n02859443 +ILSVRC2012_val_00013089.JPEG:n04270147 +ILSVRC2012_val_00013090.JPEG:n02110063 +ILSVRC2012_val_00013091.JPEG:n03042490 +ILSVRC2012_val_00013092.JPEG:n03290653 +ILSVRC2012_val_00013093.JPEG:n02002724 +ILSVRC2012_val_00013094.JPEG:n02100583 +ILSVRC2012_val_00013095.JPEG:n01608432 +ILSVRC2012_val_00013096.JPEG:n03710193 +ILSVRC2012_val_00013097.JPEG:n03777754 +ILSVRC2012_val_00013098.JPEG:n02971356 +ILSVRC2012_val_00013099.JPEG:n04482393 +ILSVRC2012_val_00013100.JPEG:n13037406 +ILSVRC2012_val_00013101.JPEG:n01768244 +ILSVRC2012_val_00013102.JPEG:n03929855 +ILSVRC2012_val_00013103.JPEG:n03016953 +ILSVRC2012_val_00013104.JPEG:n07584110 +ILSVRC2012_val_00013105.JPEG:n02113023 +ILSVRC2012_val_00013106.JPEG:n04447861 +ILSVRC2012_val_00013107.JPEG:n02128925 +ILSVRC2012_val_00013108.JPEG:n02988304 +ILSVRC2012_val_00013109.JPEG:n04201297 +ILSVRC2012_val_00013110.JPEG:n02006656 +ILSVRC2012_val_00013111.JPEG:n01807496 +ILSVRC2012_val_00013112.JPEG:n03658185 +ILSVRC2012_val_00013113.JPEG:n03394916 +ILSVRC2012_val_00013114.JPEG:n07716358 +ILSVRC2012_val_00013115.JPEG:n07579787 +ILSVRC2012_val_00013116.JPEG:n02102177 +ILSVRC2012_val_00013117.JPEG:n01729322 +ILSVRC2012_val_00013118.JPEG:n03775071 +ILSVRC2012_val_00013119.JPEG:n04482393 +ILSVRC2012_val_00013120.JPEG:n02415577 +ILSVRC2012_val_00013121.JPEG:n02607072 +ILSVRC2012_val_00013122.JPEG:n02909870 +ILSVRC2012_val_00013123.JPEG:n03255030 +ILSVRC2012_val_00013124.JPEG:n03344393 +ILSVRC2012_val_00013125.JPEG:n02325366 +ILSVRC2012_val_00013126.JPEG:n02102480 +ILSVRC2012_val_00013127.JPEG:n02102177 +ILSVRC2012_val_00013128.JPEG:n04423845 +ILSVRC2012_val_00013129.JPEG:n02130308 +ILSVRC2012_val_00013130.JPEG:n03785016 +ILSVRC2012_val_00013131.JPEG:n02787622 +ILSVRC2012_val_00013132.JPEG:n04200800 +ILSVRC2012_val_00013133.JPEG:n02087046 +ILSVRC2012_val_00013134.JPEG:n04487394 +ILSVRC2012_val_00013135.JPEG:n04152593 +ILSVRC2012_val_00013136.JPEG:n04065272 +ILSVRC2012_val_00013137.JPEG:n07831146 +ILSVRC2012_val_00013138.JPEG:n02843684 +ILSVRC2012_val_00013139.JPEG:n07248320 +ILSVRC2012_val_00013140.JPEG:n03498962 +ILSVRC2012_val_00013141.JPEG:n02128757 +ILSVRC2012_val_00013142.JPEG:n04523525 +ILSVRC2012_val_00013143.JPEG:n02999410 +ILSVRC2012_val_00013144.JPEG:n03697007 +ILSVRC2012_val_00013145.JPEG:n02097209 +ILSVRC2012_val_00013146.JPEG:n11939491 +ILSVRC2012_val_00013147.JPEG:n04141327 +ILSVRC2012_val_00013148.JPEG:n07248320 +ILSVRC2012_val_00013149.JPEG:n04461696 +ILSVRC2012_val_00013150.JPEG:n02110185 +ILSVRC2012_val_00013151.JPEG:n02483708 +ILSVRC2012_val_00013152.JPEG:n03902125 +ILSVRC2012_val_00013153.JPEG:n02168699 +ILSVRC2012_val_00013154.JPEG:n02834397 +ILSVRC2012_val_00013155.JPEG:n02108915 +ILSVRC2012_val_00013156.JPEG:n02963159 +ILSVRC2012_val_00013157.JPEG:n03841143 +ILSVRC2012_val_00013158.JPEG:n02120505 +ILSVRC2012_val_00013159.JPEG:n02111129 +ILSVRC2012_val_00013160.JPEG:n02112350 +ILSVRC2012_val_00013161.JPEG:n03793489 +ILSVRC2012_val_00013162.JPEG:n03649909 +ILSVRC2012_val_00013163.JPEG:n04090263 +ILSVRC2012_val_00013164.JPEG:n02727426 +ILSVRC2012_val_00013165.JPEG:n04033995 +ILSVRC2012_val_00013166.JPEG:n01608432 +ILSVRC2012_val_00013167.JPEG:n02364673 +ILSVRC2012_val_00013168.JPEG:n02895154 +ILSVRC2012_val_00013169.JPEG:n07730033 +ILSVRC2012_val_00013170.JPEG:n02423022 +ILSVRC2012_val_00013171.JPEG:n02999410 +ILSVRC2012_val_00013172.JPEG:n07579787 +ILSVRC2012_val_00013173.JPEG:n02086079 +ILSVRC2012_val_00013174.JPEG:n01631663 +ILSVRC2012_val_00013175.JPEG:n02494079 +ILSVRC2012_val_00013176.JPEG:n04118776 +ILSVRC2012_val_00013177.JPEG:n03467068 +ILSVRC2012_val_00013178.JPEG:n03476684 +ILSVRC2012_val_00013179.JPEG:n03954731 +ILSVRC2012_val_00013180.JPEG:n03775546 +ILSVRC2012_val_00013181.JPEG:n02981792 +ILSVRC2012_val_00013182.JPEG:n01873310 +ILSVRC2012_val_00013183.JPEG:n01980166 +ILSVRC2012_val_00013184.JPEG:n04049303 +ILSVRC2012_val_00013185.JPEG:n04099969 +ILSVRC2012_val_00013186.JPEG:n02965783 +ILSVRC2012_val_00013187.JPEG:n02281787 +ILSVRC2012_val_00013188.JPEG:n02823750 +ILSVRC2012_val_00013189.JPEG:n02655020 +ILSVRC2012_val_00013190.JPEG:n02403003 +ILSVRC2012_val_00013191.JPEG:n02951358 +ILSVRC2012_val_00013192.JPEG:n02028035 +ILSVRC2012_val_00013193.JPEG:n02504458 +ILSVRC2012_val_00013194.JPEG:n03814639 +ILSVRC2012_val_00013195.JPEG:n02085620 +ILSVRC2012_val_00013196.JPEG:n04486054 +ILSVRC2012_val_00013197.JPEG:n03761084 +ILSVRC2012_val_00013198.JPEG:n07930864 +ILSVRC2012_val_00013199.JPEG:n04522168 +ILSVRC2012_val_00013200.JPEG:n04347754 +ILSVRC2012_val_00013201.JPEG:n01644373 +ILSVRC2012_val_00013202.JPEG:n02992211 +ILSVRC2012_val_00013203.JPEG:n04483307 +ILSVRC2012_val_00013204.JPEG:n02102973 +ILSVRC2012_val_00013205.JPEG:n04467665 +ILSVRC2012_val_00013206.JPEG:n03026506 +ILSVRC2012_val_00013207.JPEG:n03026506 +ILSVRC2012_val_00013208.JPEG:n07697537 +ILSVRC2012_val_00013209.JPEG:n01532829 +ILSVRC2012_val_00013210.JPEG:n04442312 +ILSVRC2012_val_00013211.JPEG:n02108551 +ILSVRC2012_val_00013212.JPEG:n01824575 +ILSVRC2012_val_00013213.JPEG:n04254777 +ILSVRC2012_val_00013214.JPEG:n03109150 +ILSVRC2012_val_00013215.JPEG:n01728920 +ILSVRC2012_val_00013216.JPEG:n04380533 +ILSVRC2012_val_00013217.JPEG:n02795169 +ILSVRC2012_val_00013218.JPEG:n04493381 +ILSVRC2012_val_00013219.JPEG:n03141823 +ILSVRC2012_val_00013220.JPEG:n01817953 +ILSVRC2012_val_00013221.JPEG:n04026417 +ILSVRC2012_val_00013222.JPEG:n02909870 +ILSVRC2012_val_00013223.JPEG:n01601694 +ILSVRC2012_val_00013224.JPEG:n02834397 +ILSVRC2012_val_00013225.JPEG:n03376595 +ILSVRC2012_val_00013226.JPEG:n02909870 +ILSVRC2012_val_00013227.JPEG:n07711569 +ILSVRC2012_val_00013228.JPEG:n03891251 +ILSVRC2012_val_00013229.JPEG:n01806567 +ILSVRC2012_val_00013230.JPEG:n03854065 +ILSVRC2012_val_00013231.JPEG:n03814906 +ILSVRC2012_val_00013232.JPEG:n02808304 +ILSVRC2012_val_00013233.JPEG:n04153751 +ILSVRC2012_val_00013234.JPEG:n07768694 +ILSVRC2012_val_00013235.JPEG:n04532106 +ILSVRC2012_val_00013236.JPEG:n02102973 +ILSVRC2012_val_00013237.JPEG:n02346627 +ILSVRC2012_val_00013238.JPEG:n13133613 +ILSVRC2012_val_00013239.JPEG:n02129604 +ILSVRC2012_val_00013240.JPEG:n02443484 +ILSVRC2012_val_00013241.JPEG:n03792972 +ILSVRC2012_val_00013242.JPEG:n02804414 +ILSVRC2012_val_00013243.JPEG:n02097298 +ILSVRC2012_val_00013244.JPEG:n02708093 +ILSVRC2012_val_00013245.JPEG:n01748264 +ILSVRC2012_val_00013246.JPEG:n03992509 +ILSVRC2012_val_00013247.JPEG:n04591713 +ILSVRC2012_val_00013248.JPEG:n02105162 +ILSVRC2012_val_00013249.JPEG:n03840681 +ILSVRC2012_val_00013250.JPEG:n02276258 +ILSVRC2012_val_00013251.JPEG:n02100583 +ILSVRC2012_val_00013252.JPEG:n02408429 +ILSVRC2012_val_00013253.JPEG:n03770679 +ILSVRC2012_val_00013254.JPEG:n07717556 +ILSVRC2012_val_00013255.JPEG:n02280649 +ILSVRC2012_val_00013256.JPEG:n02006656 +ILSVRC2012_val_00013257.JPEG:n04560804 +ILSVRC2012_val_00013258.JPEG:n04285008 +ILSVRC2012_val_00013259.JPEG:n03868863 +ILSVRC2012_val_00013260.JPEG:n02088238 +ILSVRC2012_val_00013261.JPEG:n02799071 +ILSVRC2012_val_00013262.JPEG:n04560804 +ILSVRC2012_val_00013263.JPEG:n02108551 +ILSVRC2012_val_00013264.JPEG:n02487347 +ILSVRC2012_val_00013265.JPEG:n01614925 +ILSVRC2012_val_00013266.JPEG:n04505470 +ILSVRC2012_val_00013267.JPEG:n04090263 +ILSVRC2012_val_00013268.JPEG:n03661043 +ILSVRC2012_val_00013269.JPEG:n01675722 +ILSVRC2012_val_00013270.JPEG:n01531178 +ILSVRC2012_val_00013271.JPEG:n01632458 +ILSVRC2012_val_00013272.JPEG:n01695060 +ILSVRC2012_val_00013273.JPEG:n04254777 +ILSVRC2012_val_00013274.JPEG:n04355933 +ILSVRC2012_val_00013275.JPEG:n03743016 +ILSVRC2012_val_00013276.JPEG:n04259630 +ILSVRC2012_val_00013277.JPEG:n01534433 +ILSVRC2012_val_00013278.JPEG:n02110958 +ILSVRC2012_val_00013279.JPEG:n02112350 +ILSVRC2012_val_00013280.JPEG:n02488702 +ILSVRC2012_val_00013281.JPEG:n02687172 +ILSVRC2012_val_00013282.JPEG:n09246464 +ILSVRC2012_val_00013283.JPEG:n02071294 +ILSVRC2012_val_00013284.JPEG:n02497673 +ILSVRC2012_val_00013285.JPEG:n03871628 +ILSVRC2012_val_00013286.JPEG:n07717556 +ILSVRC2012_val_00013287.JPEG:n02105412 +ILSVRC2012_val_00013288.JPEG:n02999410 +ILSVRC2012_val_00013289.JPEG:n02105412 +ILSVRC2012_val_00013290.JPEG:n04208210 +ILSVRC2012_val_00013291.JPEG:n04589890 +ILSVRC2012_val_00013292.JPEG:n03379051 +ILSVRC2012_val_00013293.JPEG:n03404251 +ILSVRC2012_val_00013294.JPEG:n03014705 +ILSVRC2012_val_00013295.JPEG:n04146614 +ILSVRC2012_val_00013296.JPEG:n03938244 +ILSVRC2012_val_00013297.JPEG:n02107142 +ILSVRC2012_val_00013298.JPEG:n03452741 +ILSVRC2012_val_00013299.JPEG:n01667114 +ILSVRC2012_val_00013300.JPEG:n04311174 +ILSVRC2012_val_00013301.JPEG:n01667778 +ILSVRC2012_val_00013302.JPEG:n03127747 +ILSVRC2012_val_00013303.JPEG:n02105412 +ILSVRC2012_val_00013304.JPEG:n09399592 +ILSVRC2012_val_00013305.JPEG:n07716906 +ILSVRC2012_val_00013306.JPEG:n03673027 +ILSVRC2012_val_00013307.JPEG:n03197337 +ILSVRC2012_val_00013308.JPEG:n03450230 +ILSVRC2012_val_00013309.JPEG:n02113186 +ILSVRC2012_val_00013310.JPEG:n01775062 +ILSVRC2012_val_00013311.JPEG:n04380533 +ILSVRC2012_val_00013312.JPEG:n06359193 +ILSVRC2012_val_00013313.JPEG:n03483316 +ILSVRC2012_val_00013314.JPEG:n02172182 +ILSVRC2012_val_00013315.JPEG:n03496892 +ILSVRC2012_val_00013316.JPEG:n03843555 +ILSVRC2012_val_00013317.JPEG:n04476259 +ILSVRC2012_val_00013318.JPEG:n02110806 +ILSVRC2012_val_00013319.JPEG:n04467665 +ILSVRC2012_val_00013320.JPEG:n04548280 +ILSVRC2012_val_00013321.JPEG:n01518878 +ILSVRC2012_val_00013322.JPEG:n02281787 +ILSVRC2012_val_00013323.JPEG:n02093647 +ILSVRC2012_val_00013324.JPEG:n04404412 +ILSVRC2012_val_00013325.JPEG:n04356056 +ILSVRC2012_val_00013326.JPEG:n03840681 +ILSVRC2012_val_00013327.JPEG:n03995372 +ILSVRC2012_val_00013328.JPEG:n02326432 +ILSVRC2012_val_00013329.JPEG:n02777292 +ILSVRC2012_val_00013330.JPEG:n01776313 +ILSVRC2012_val_00013331.JPEG:n03220513 +ILSVRC2012_val_00013332.JPEG:n02795169 +ILSVRC2012_val_00013333.JPEG:n02074367 +ILSVRC2012_val_00013334.JPEG:n01968897 +ILSVRC2012_val_00013335.JPEG:n07693725 +ILSVRC2012_val_00013336.JPEG:n02906734 +ILSVRC2012_val_00013337.JPEG:n03777754 +ILSVRC2012_val_00013338.JPEG:n02497673 +ILSVRC2012_val_00013339.JPEG:n03126707 +ILSVRC2012_val_00013340.JPEG:n04259630 +ILSVRC2012_val_00013341.JPEG:n03729826 +ILSVRC2012_val_00013342.JPEG:n04026417 +ILSVRC2012_val_00013343.JPEG:n01855032 +ILSVRC2012_val_00013344.JPEG:n02808440 +ILSVRC2012_val_00013345.JPEG:n04346328 +ILSVRC2012_val_00013346.JPEG:n03930313 +ILSVRC2012_val_00013347.JPEG:n04560804 +ILSVRC2012_val_00013348.JPEG:n03127925 +ILSVRC2012_val_00013349.JPEG:n07684084 +ILSVRC2012_val_00013350.JPEG:n04417672 +ILSVRC2012_val_00013351.JPEG:n02172182 +ILSVRC2012_val_00013352.JPEG:n02325366 +ILSVRC2012_val_00013353.JPEG:n03899768 +ILSVRC2012_val_00013354.JPEG:n01644900 +ILSVRC2012_val_00013355.JPEG:n02113186 +ILSVRC2012_val_00013356.JPEG:n03710637 +ILSVRC2012_val_00013357.JPEG:n03857828 +ILSVRC2012_val_00013358.JPEG:n02114548 +ILSVRC2012_val_00013359.JPEG:n04326547 +ILSVRC2012_val_00013360.JPEG:n02643566 +ILSVRC2012_val_00013361.JPEG:n02092002 +ILSVRC2012_val_00013362.JPEG:n03124170 +ILSVRC2012_val_00013363.JPEG:n02281406 +ILSVRC2012_val_00013364.JPEG:n01806567 +ILSVRC2012_val_00013365.JPEG:n04254680 +ILSVRC2012_val_00013366.JPEG:n03344393 +ILSVRC2012_val_00013367.JPEG:n01532829 +ILSVRC2012_val_00013368.JPEG:n02116738 +ILSVRC2012_val_00013369.JPEG:n02116738 +ILSVRC2012_val_00013370.JPEG:n02094258 +ILSVRC2012_val_00013371.JPEG:n03690938 +ILSVRC2012_val_00013372.JPEG:n03272562 +ILSVRC2012_val_00013373.JPEG:n03110669 +ILSVRC2012_val_00013374.JPEG:n03786901 +ILSVRC2012_val_00013375.JPEG:n07920052 +ILSVRC2012_val_00013376.JPEG:n04355933 +ILSVRC2012_val_00013377.JPEG:n01978455 +ILSVRC2012_val_00013378.JPEG:n01806143 +ILSVRC2012_val_00013379.JPEG:n01944390 +ILSVRC2012_val_00013380.JPEG:n03450230 +ILSVRC2012_val_00013381.JPEG:n02088364 +ILSVRC2012_val_00013382.JPEG:n03956157 +ILSVRC2012_val_00013383.JPEG:n02437312 +ILSVRC2012_val_00013384.JPEG:n03590841 +ILSVRC2012_val_00013385.JPEG:n04344873 +ILSVRC2012_val_00013386.JPEG:n02277742 +ILSVRC2012_val_00013387.JPEG:n02111277 +ILSVRC2012_val_00013388.JPEG:n01784675 +ILSVRC2012_val_00013389.JPEG:n04483307 +ILSVRC2012_val_00013390.JPEG:n02132136 +ILSVRC2012_val_00013391.JPEG:n04019541 +ILSVRC2012_val_00013392.JPEG:n01693334 +ILSVRC2012_val_00013393.JPEG:n01608432 +ILSVRC2012_val_00013394.JPEG:n01667114 +ILSVRC2012_val_00013395.JPEG:n02236044 +ILSVRC2012_val_00013396.JPEG:n03775546 +ILSVRC2012_val_00013397.JPEG:n01739381 +ILSVRC2012_val_00013398.JPEG:n02100583 +ILSVRC2012_val_00013399.JPEG:n02090622 +ILSVRC2012_val_00013400.JPEG:n01729322 +ILSVRC2012_val_00013401.JPEG:n04350905 +ILSVRC2012_val_00013402.JPEG:n02056570 +ILSVRC2012_val_00013403.JPEG:n04612504 +ILSVRC2012_val_00013404.JPEG:n04505470 +ILSVRC2012_val_00013405.JPEG:n12057211 +ILSVRC2012_val_00013406.JPEG:n03837869 +ILSVRC2012_val_00013407.JPEG:n01531178 +ILSVRC2012_val_00013408.JPEG:n04376876 +ILSVRC2012_val_00013409.JPEG:n02454379 +ILSVRC2012_val_00013410.JPEG:n02124075 +ILSVRC2012_val_00013411.JPEG:n02395406 +ILSVRC2012_val_00013412.JPEG:n02114367 +ILSVRC2012_val_00013413.JPEG:n03481172 +ILSVRC2012_val_00013414.JPEG:n02109047 +ILSVRC2012_val_00013415.JPEG:n07715103 +ILSVRC2012_val_00013416.JPEG:n04154565 +ILSVRC2012_val_00013417.JPEG:n02423022 +ILSVRC2012_val_00013418.JPEG:n01756291 +ILSVRC2012_val_00013419.JPEG:n02108089 +ILSVRC2012_val_00013420.JPEG:n02493793 +ILSVRC2012_val_00013421.JPEG:n03602883 +ILSVRC2012_val_00013422.JPEG:n02168699 +ILSVRC2012_val_00013423.JPEG:n01978455 +ILSVRC2012_val_00013424.JPEG:n02097298 +ILSVRC2012_val_00013425.JPEG:n02447366 +ILSVRC2012_val_00013426.JPEG:n04229816 +ILSVRC2012_val_00013427.JPEG:n07583066 +ILSVRC2012_val_00013428.JPEG:n03207743 +ILSVRC2012_val_00013429.JPEG:n07248320 +ILSVRC2012_val_00013430.JPEG:n02100583 +ILSVRC2012_val_00013431.JPEG:n02823750 +ILSVRC2012_val_00013432.JPEG:n01608432 +ILSVRC2012_val_00013433.JPEG:n04418357 +ILSVRC2012_val_00013434.JPEG:n01833805 +ILSVRC2012_val_00013435.JPEG:n03930630 +ILSVRC2012_val_00013436.JPEG:n03425413 +ILSVRC2012_val_00013437.JPEG:n02788148 +ILSVRC2012_val_00013438.JPEG:n03637318 +ILSVRC2012_val_00013439.JPEG:n04265275 +ILSVRC2012_val_00013440.JPEG:n02281787 +ILSVRC2012_val_00013441.JPEG:n04335435 +ILSVRC2012_val_00013442.JPEG:n02093428 +ILSVRC2012_val_00013443.JPEG:n06359193 +ILSVRC2012_val_00013444.JPEG:n03944341 +ILSVRC2012_val_00013445.JPEG:n04041544 +ILSVRC2012_val_00013446.JPEG:n04515003 +ILSVRC2012_val_00013447.JPEG:n02106550 +ILSVRC2012_val_00013448.JPEG:n02097130 +ILSVRC2012_val_00013449.JPEG:n02837789 +ILSVRC2012_val_00013450.JPEG:n07753275 +ILSVRC2012_val_00013451.JPEG:n04026417 +ILSVRC2012_val_00013452.JPEG:n03673027 +ILSVRC2012_val_00013453.JPEG:n03887697 +ILSVRC2012_val_00013454.JPEG:n03110669 +ILSVRC2012_val_00013455.JPEG:n03769881 +ILSVRC2012_val_00013456.JPEG:n01532829 +ILSVRC2012_val_00013457.JPEG:n02006656 +ILSVRC2012_val_00013458.JPEG:n04296562 +ILSVRC2012_val_00013459.JPEG:n04347754 +ILSVRC2012_val_00013460.JPEG:n01828970 +ILSVRC2012_val_00013461.JPEG:n03125729 +ILSVRC2012_val_00013462.JPEG:n03877472 +ILSVRC2012_val_00013463.JPEG:n02096051 +ILSVRC2012_val_00013464.JPEG:n04483307 +ILSVRC2012_val_00013465.JPEG:n02398521 +ILSVRC2012_val_00013466.JPEG:n03770679 +ILSVRC2012_val_00013467.JPEG:n02106662 +ILSVRC2012_val_00013468.JPEG:n03775546 +ILSVRC2012_val_00013469.JPEG:n04347754 +ILSVRC2012_val_00013470.JPEG:n02676566 +ILSVRC2012_val_00013471.JPEG:n03690938 +ILSVRC2012_val_00013472.JPEG:n07831146 +ILSVRC2012_val_00013473.JPEG:n04398044 +ILSVRC2012_val_00013474.JPEG:n01985128 +ILSVRC2012_val_00013475.JPEG:n02109047 +ILSVRC2012_val_00013476.JPEG:n03785016 +ILSVRC2012_val_00013477.JPEG:n03494278 +ILSVRC2012_val_00013478.JPEG:n03792972 +ILSVRC2012_val_00013479.JPEG:n02114367 +ILSVRC2012_val_00013480.JPEG:n03777754 +ILSVRC2012_val_00013481.JPEG:n04090263 +ILSVRC2012_val_00013482.JPEG:n02132136 +ILSVRC2012_val_00013483.JPEG:n03134739 +ILSVRC2012_val_00013484.JPEG:n01491361 +ILSVRC2012_val_00013485.JPEG:n09332890 +ILSVRC2012_val_00013486.JPEG:n03803284 +ILSVRC2012_val_00013487.JPEG:n02120079 +ILSVRC2012_val_00013488.JPEG:n03075370 +ILSVRC2012_val_00013489.JPEG:n02104365 +ILSVRC2012_val_00013490.JPEG:n03884397 +ILSVRC2012_val_00013491.JPEG:n02790996 +ILSVRC2012_val_00013492.JPEG:n01751748 +ILSVRC2012_val_00013493.JPEG:n07695742 +ILSVRC2012_val_00013494.JPEG:n02123045 +ILSVRC2012_val_00013495.JPEG:n03759954 +ILSVRC2012_val_00013496.JPEG:n03733131 +ILSVRC2012_val_00013497.JPEG:n12998815 +ILSVRC2012_val_00013498.JPEG:n03223299 +ILSVRC2012_val_00013499.JPEG:n07745940 +ILSVRC2012_val_00013500.JPEG:n04532106 +ILSVRC2012_val_00013501.JPEG:n02111889 +ILSVRC2012_val_00013502.JPEG:n02708093 +ILSVRC2012_val_00013503.JPEG:n01944390 +ILSVRC2012_val_00013504.JPEG:n01534433 +ILSVRC2012_val_00013505.JPEG:n02361337 +ILSVRC2012_val_00013506.JPEG:n02113624 +ILSVRC2012_val_00013507.JPEG:n02090721 +ILSVRC2012_val_00013508.JPEG:n02093256 +ILSVRC2012_val_00013509.JPEG:n02025239 +ILSVRC2012_val_00013510.JPEG:n04355933 +ILSVRC2012_val_00013511.JPEG:n03452741 +ILSVRC2012_val_00013512.JPEG:n01530575 +ILSVRC2012_val_00013513.JPEG:n01443537 +ILSVRC2012_val_00013514.JPEG:n04209239 +ILSVRC2012_val_00013515.JPEG:n02037110 +ILSVRC2012_val_00013516.JPEG:n04154565 +ILSVRC2012_val_00013517.JPEG:n03594945 +ILSVRC2012_val_00013518.JPEG:n04465501 +ILSVRC2012_val_00013519.JPEG:n07714990 +ILSVRC2012_val_00013520.JPEG:n03868863 +ILSVRC2012_val_00013521.JPEG:n01819313 +ILSVRC2012_val_00013522.JPEG:n04026417 +ILSVRC2012_val_00013523.JPEG:n04553703 +ILSVRC2012_val_00013524.JPEG:n02112706 +ILSVRC2012_val_00013525.JPEG:n01980166 +ILSVRC2012_val_00013526.JPEG:n02797295 +ILSVRC2012_val_00013527.JPEG:n03888257 +ILSVRC2012_val_00013528.JPEG:n02342885 +ILSVRC2012_val_00013529.JPEG:n03216828 +ILSVRC2012_val_00013530.JPEG:n03388043 +ILSVRC2012_val_00013531.JPEG:n03804744 +ILSVRC2012_val_00013532.JPEG:n02138441 +ILSVRC2012_val_00013533.JPEG:n01689811 +ILSVRC2012_val_00013534.JPEG:n04553703 +ILSVRC2012_val_00013535.JPEG:n02231487 +ILSVRC2012_val_00013536.JPEG:n04208210 +ILSVRC2012_val_00013537.JPEG:n03372029 +ILSVRC2012_val_00013538.JPEG:n02096177 +ILSVRC2012_val_00013539.JPEG:n04429376 +ILSVRC2012_val_00013540.JPEG:n03272010 +ILSVRC2012_val_00013541.JPEG:n02493509 +ILSVRC2012_val_00013542.JPEG:n03127747 +ILSVRC2012_val_00013543.JPEG:n02786058 +ILSVRC2012_val_00013544.JPEG:n03777568 +ILSVRC2012_val_00013545.JPEG:n04238763 +ILSVRC2012_val_00013546.JPEG:n03535780 +ILSVRC2012_val_00013547.JPEG:n03938244 +ILSVRC2012_val_00013548.JPEG:n02408429 +ILSVRC2012_val_00013549.JPEG:n02097658 +ILSVRC2012_val_00013550.JPEG:n02123159 +ILSVRC2012_val_00013551.JPEG:n03891251 +ILSVRC2012_val_00013552.JPEG:n02165105 +ILSVRC2012_val_00013553.JPEG:n02437312 +ILSVRC2012_val_00013554.JPEG:n02114712 +ILSVRC2012_val_00013555.JPEG:n04540053 +ILSVRC2012_val_00013556.JPEG:n04270147 +ILSVRC2012_val_00013557.JPEG:n02113186 +ILSVRC2012_val_00013558.JPEG:n02281406 +ILSVRC2012_val_00013559.JPEG:n03899768 +ILSVRC2012_val_00013560.JPEG:n04442312 +ILSVRC2012_val_00013561.JPEG:n04023962 +ILSVRC2012_val_00013562.JPEG:n02963159 +ILSVRC2012_val_00013563.JPEG:n02102973 +ILSVRC2012_val_00013564.JPEG:n01860187 +ILSVRC2012_val_00013565.JPEG:n03297495 +ILSVRC2012_val_00013566.JPEG:n03733805 +ILSVRC2012_val_00013567.JPEG:n03980874 +ILSVRC2012_val_00013568.JPEG:n04336792 +ILSVRC2012_val_00013569.JPEG:n04366367 +ILSVRC2012_val_00013570.JPEG:n02412080 +ILSVRC2012_val_00013571.JPEG:n02966687 +ILSVRC2012_val_00013572.JPEG:n03763968 +ILSVRC2012_val_00013573.JPEG:n02098286 +ILSVRC2012_val_00013574.JPEG:n01756291 +ILSVRC2012_val_00013575.JPEG:n03929855 +ILSVRC2012_val_00013576.JPEG:n03944341 +ILSVRC2012_val_00013577.JPEG:n03271574 +ILSVRC2012_val_00013578.JPEG:n04026417 +ILSVRC2012_val_00013579.JPEG:n07754684 +ILSVRC2012_val_00013580.JPEG:n01985128 +ILSVRC2012_val_00013581.JPEG:n07753113 +ILSVRC2012_val_00013582.JPEG:n01675722 +ILSVRC2012_val_00013583.JPEG:n02106166 +ILSVRC2012_val_00013584.JPEG:n02116738 +ILSVRC2012_val_00013585.JPEG:n03916031 +ILSVRC2012_val_00013586.JPEG:n04065272 +ILSVRC2012_val_00013587.JPEG:n03110669 +ILSVRC2012_val_00013588.JPEG:n07747607 +ILSVRC2012_val_00013589.JPEG:n02009912 +ILSVRC2012_val_00013590.JPEG:n03950228 +ILSVRC2012_val_00013591.JPEG:n03483316 +ILSVRC2012_val_00013592.JPEG:n07716358 +ILSVRC2012_val_00013593.JPEG:n03216828 +ILSVRC2012_val_00013594.JPEG:n09835506 +ILSVRC2012_val_00013595.JPEG:n03393912 +ILSVRC2012_val_00013596.JPEG:n02526121 +ILSVRC2012_val_00013597.JPEG:n03770439 +ILSVRC2012_val_00013598.JPEG:n02002724 +ILSVRC2012_val_00013599.JPEG:n02871525 +ILSVRC2012_val_00013600.JPEG:n01776313 +ILSVRC2012_val_00013601.JPEG:n04355933 +ILSVRC2012_val_00013602.JPEG:n03450230 +ILSVRC2012_val_00013603.JPEG:n02025239 +ILSVRC2012_val_00013604.JPEG:n02107312 +ILSVRC2012_val_00013605.JPEG:n04606251 +ILSVRC2012_val_00013606.JPEG:n03063599 +ILSVRC2012_val_00013607.JPEG:n01795545 +ILSVRC2012_val_00013608.JPEG:n04254777 +ILSVRC2012_val_00013609.JPEG:n02120079 +ILSVRC2012_val_00013610.JPEG:n01833805 +ILSVRC2012_val_00013611.JPEG:n02099601 +ILSVRC2012_val_00013612.JPEG:n13052670 +ILSVRC2012_val_00013613.JPEG:n02676566 +ILSVRC2012_val_00013614.JPEG:n03457902 +ILSVRC2012_val_00013615.JPEG:n03720891 +ILSVRC2012_val_00013616.JPEG:n03793489 +ILSVRC2012_val_00013617.JPEG:n01775062 +ILSVRC2012_val_00013618.JPEG:n01978287 +ILSVRC2012_val_00013619.JPEG:n10565667 +ILSVRC2012_val_00013620.JPEG:n02916936 +ILSVRC2012_val_00013621.JPEG:n03599486 +ILSVRC2012_val_00013622.JPEG:n02110958 +ILSVRC2012_val_00013623.JPEG:n01443537 +ILSVRC2012_val_00013624.JPEG:n04204238 +ILSVRC2012_val_00013625.JPEG:n02672831 +ILSVRC2012_val_00013626.JPEG:n07717410 +ILSVRC2012_val_00013627.JPEG:n04209239 +ILSVRC2012_val_00013628.JPEG:n01491361 +ILSVRC2012_val_00013629.JPEG:n02963159 +ILSVRC2012_val_00013630.JPEG:n03424325 +ILSVRC2012_val_00013631.JPEG:n03697007 +ILSVRC2012_val_00013632.JPEG:n03344393 +ILSVRC2012_val_00013633.JPEG:n03445777 +ILSVRC2012_val_00013634.JPEG:n02999410 +ILSVRC2012_val_00013635.JPEG:n02441942 +ILSVRC2012_val_00013636.JPEG:n04525038 +ILSVRC2012_val_00013637.JPEG:n02403003 +ILSVRC2012_val_00013638.JPEG:n07684084 +ILSVRC2012_val_00013639.JPEG:n03125729 +ILSVRC2012_val_00013640.JPEG:n02095570 +ILSVRC2012_val_00013641.JPEG:n01796340 +ILSVRC2012_val_00013642.JPEG:n03599486 +ILSVRC2012_val_00013643.JPEG:n07747607 +ILSVRC2012_val_00013644.JPEG:n04507155 +ILSVRC2012_val_00013645.JPEG:n07768694 +ILSVRC2012_val_00013646.JPEG:n04501370 +ILSVRC2012_val_00013647.JPEG:n07734744 +ILSVRC2012_val_00013648.JPEG:n02676566 +ILSVRC2012_val_00013649.JPEG:n01871265 +ILSVRC2012_val_00013650.JPEG:n03680355 +ILSVRC2012_val_00013651.JPEG:n02088466 +ILSVRC2012_val_00013652.JPEG:n10565667 +ILSVRC2012_val_00013653.JPEG:n02110958 +ILSVRC2012_val_00013654.JPEG:n02096437 +ILSVRC2012_val_00013655.JPEG:n01498041 +ILSVRC2012_val_00013656.JPEG:n02130308 +ILSVRC2012_val_00013657.JPEG:n07836838 +ILSVRC2012_val_00013658.JPEG:n03884397 +ILSVRC2012_val_00013659.JPEG:n04065272 +ILSVRC2012_val_00013660.JPEG:n02033041 +ILSVRC2012_val_00013661.JPEG:n02607072 +ILSVRC2012_val_00013662.JPEG:n13040303 +ILSVRC2012_val_00013663.JPEG:n02808304 +ILSVRC2012_val_00013664.JPEG:n03095699 +ILSVRC2012_val_00013665.JPEG:n03485407 +ILSVRC2012_val_00013666.JPEG:n02395406 +ILSVRC2012_val_00013667.JPEG:n04560804 +ILSVRC2012_val_00013668.JPEG:n02676566 +ILSVRC2012_val_00013669.JPEG:n04589890 +ILSVRC2012_val_00013670.JPEG:n02110958 +ILSVRC2012_val_00013671.JPEG:n02837789 +ILSVRC2012_val_00013672.JPEG:n01669191 +ILSVRC2012_val_00013673.JPEG:n02123045 +ILSVRC2012_val_00013674.JPEG:n07579787 +ILSVRC2012_val_00013675.JPEG:n01667778 +ILSVRC2012_val_00013676.JPEG:n12998815 +ILSVRC2012_val_00013677.JPEG:n04613696 +ILSVRC2012_val_00013678.JPEG:n02951585 +ILSVRC2012_val_00013679.JPEG:n03623198 +ILSVRC2012_val_00013680.JPEG:n03764736 +ILSVRC2012_val_00013681.JPEG:n02892767 +ILSVRC2012_val_00013682.JPEG:n02102318 +ILSVRC2012_val_00013683.JPEG:n04040759 +ILSVRC2012_val_00013684.JPEG:n02123045 +ILSVRC2012_val_00013685.JPEG:n03062245 +ILSVRC2012_val_00013686.JPEG:n02701002 +ILSVRC2012_val_00013687.JPEG:n03201208 +ILSVRC2012_val_00013688.JPEG:n04266014 +ILSVRC2012_val_00013689.JPEG:n01873310 +ILSVRC2012_val_00013690.JPEG:n04597913 +ILSVRC2012_val_00013691.JPEG:n03595614 +ILSVRC2012_val_00013692.JPEG:n07716906 +ILSVRC2012_val_00013693.JPEG:n02988304 +ILSVRC2012_val_00013694.JPEG:n03445924 +ILSVRC2012_val_00013695.JPEG:n02860847 +ILSVRC2012_val_00013696.JPEG:n02095889 +ILSVRC2012_val_00013697.JPEG:n02115913 +ILSVRC2012_val_00013698.JPEG:n01756291 +ILSVRC2012_val_00013699.JPEG:n02114548 +ILSVRC2012_val_00013700.JPEG:n02457408 +ILSVRC2012_val_00013701.JPEG:n03995372 +ILSVRC2012_val_00013702.JPEG:n01614925 +ILSVRC2012_val_00013703.JPEG:n02107312 +ILSVRC2012_val_00013704.JPEG:n03930630 +ILSVRC2012_val_00013705.JPEG:n03017168 +ILSVRC2012_val_00013706.JPEG:n03535780 +ILSVRC2012_val_00013707.JPEG:n01985128 +ILSVRC2012_val_00013708.JPEG:n02177972 +ILSVRC2012_val_00013709.JPEG:n03045698 +ILSVRC2012_val_00013710.JPEG:n13133613 +ILSVRC2012_val_00013711.JPEG:n04398044 +ILSVRC2012_val_00013712.JPEG:n02099267 +ILSVRC2012_val_00013713.JPEG:n01829413 +ILSVRC2012_val_00013714.JPEG:n02114712 +ILSVRC2012_val_00013715.JPEG:n02104029 +ILSVRC2012_val_00013716.JPEG:n01440764 +ILSVRC2012_val_00013717.JPEG:n04263257 +ILSVRC2012_val_00013718.JPEG:n04251144 +ILSVRC2012_val_00013719.JPEG:n03584254 +ILSVRC2012_val_00013720.JPEG:n03874599 +ILSVRC2012_val_00013721.JPEG:n06359193 +ILSVRC2012_val_00013722.JPEG:n04070727 +ILSVRC2012_val_00013723.JPEG:n04209133 +ILSVRC2012_val_00013724.JPEG:n04065272 +ILSVRC2012_val_00013725.JPEG:n01748264 +ILSVRC2012_val_00013726.JPEG:n02980441 +ILSVRC2012_val_00013727.JPEG:n02093754 +ILSVRC2012_val_00013728.JPEG:n02097658 +ILSVRC2012_val_00013729.JPEG:n03187595 +ILSVRC2012_val_00013730.JPEG:n01742172 +ILSVRC2012_val_00013731.JPEG:n04590129 +ILSVRC2012_val_00013732.JPEG:n03188531 +ILSVRC2012_val_00013733.JPEG:n02504013 +ILSVRC2012_val_00013734.JPEG:n02017213 +ILSVRC2012_val_00013735.JPEG:n02979186 +ILSVRC2012_val_00013736.JPEG:n02843684 +ILSVRC2012_val_00013737.JPEG:n04040759 +ILSVRC2012_val_00013738.JPEG:n01667778 +ILSVRC2012_val_00013739.JPEG:n01820546 +ILSVRC2012_val_00013740.JPEG:n02116738 +ILSVRC2012_val_00013741.JPEG:n04243546 +ILSVRC2012_val_00013742.JPEG:n04090263 +ILSVRC2012_val_00013743.JPEG:n03888605 +ILSVRC2012_val_00013744.JPEG:n01985128 +ILSVRC2012_val_00013745.JPEG:n02823750 +ILSVRC2012_val_00013746.JPEG:n04141975 +ILSVRC2012_val_00013747.JPEG:n03376595 +ILSVRC2012_val_00013748.JPEG:n02108915 +ILSVRC2012_val_00013749.JPEG:n03372029 +ILSVRC2012_val_00013750.JPEG:n02423022 +ILSVRC2012_val_00013751.JPEG:n01728920 +ILSVRC2012_val_00013752.JPEG:n02102973 +ILSVRC2012_val_00013753.JPEG:n01580077 +ILSVRC2012_val_00013754.JPEG:n02492660 +ILSVRC2012_val_00013755.JPEG:n07716906 +ILSVRC2012_val_00013756.JPEG:n02096294 +ILSVRC2012_val_00013757.JPEG:n03259280 +ILSVRC2012_val_00013758.JPEG:n03884397 +ILSVRC2012_val_00013759.JPEG:n02102973 +ILSVRC2012_val_00013760.JPEG:n03666591 +ILSVRC2012_val_00013761.JPEG:n02486410 +ILSVRC2012_val_00013762.JPEG:n02102480 +ILSVRC2012_val_00013763.JPEG:n02105162 +ILSVRC2012_val_00013764.JPEG:n09246464 +ILSVRC2012_val_00013765.JPEG:n02823750 +ILSVRC2012_val_00013766.JPEG:n04152593 +ILSVRC2012_val_00013767.JPEG:n03196217 +ILSVRC2012_val_00013768.JPEG:n01818515 +ILSVRC2012_val_00013769.JPEG:n04591157 +ILSVRC2012_val_00013770.JPEG:n04328186 +ILSVRC2012_val_00013771.JPEG:n01742172 +ILSVRC2012_val_00013772.JPEG:n01753488 +ILSVRC2012_val_00013773.JPEG:n02971356 +ILSVRC2012_val_00013774.JPEG:n09428293 +ILSVRC2012_val_00013775.JPEG:n02927161 +ILSVRC2012_val_00013776.JPEG:n03180011 +ILSVRC2012_val_00013777.JPEG:n04099969 +ILSVRC2012_val_00013778.JPEG:n02795169 +ILSVRC2012_val_00013779.JPEG:n02895154 +ILSVRC2012_val_00013780.JPEG:n03929660 +ILSVRC2012_val_00013781.JPEG:n01910747 +ILSVRC2012_val_00013782.JPEG:n03854065 +ILSVRC2012_val_00013783.JPEG:n02747177 +ILSVRC2012_val_00013784.JPEG:n03803284 +ILSVRC2012_val_00013785.JPEG:n02123394 +ILSVRC2012_val_00013786.JPEG:n04264628 +ILSVRC2012_val_00013787.JPEG:n04243546 +ILSVRC2012_val_00013788.JPEG:n02123159 +ILSVRC2012_val_00013789.JPEG:n01983481 +ILSVRC2012_val_00013790.JPEG:n02526121 +ILSVRC2012_val_00013791.JPEG:n12267677 +ILSVRC2012_val_00013792.JPEG:n06785654 +ILSVRC2012_val_00013793.JPEG:n04606251 +ILSVRC2012_val_00013794.JPEG:n01855672 +ILSVRC2012_val_00013795.JPEG:n02281406 +ILSVRC2012_val_00013796.JPEG:n04296562 +ILSVRC2012_val_00013797.JPEG:n01773549 +ILSVRC2012_val_00013798.JPEG:n02127052 +ILSVRC2012_val_00013799.JPEG:n02090622 +ILSVRC2012_val_00013800.JPEG:n02088094 +ILSVRC2012_val_00013801.JPEG:n04125021 +ILSVRC2012_val_00013802.JPEG:n01728920 +ILSVRC2012_val_00013803.JPEG:n03595614 +ILSVRC2012_val_00013804.JPEG:n02090622 +ILSVRC2012_val_00013805.JPEG:n04285008 +ILSVRC2012_val_00013806.JPEG:n03874293 +ILSVRC2012_val_00013807.JPEG:n02823428 +ILSVRC2012_val_00013808.JPEG:n02028035 +ILSVRC2012_val_00013809.JPEG:n02077923 +ILSVRC2012_val_00013810.JPEG:n02017213 +ILSVRC2012_val_00013811.JPEG:n03903868 +ILSVRC2012_val_00013812.JPEG:n02127052 +ILSVRC2012_val_00013813.JPEG:n04317175 +ILSVRC2012_val_00013814.JPEG:n02107683 +ILSVRC2012_val_00013815.JPEG:n01984695 +ILSVRC2012_val_00013816.JPEG:n03995372 +ILSVRC2012_val_00013817.JPEG:n02090721 +ILSVRC2012_val_00013818.JPEG:n02089867 +ILSVRC2012_val_00013819.JPEG:n10148035 +ILSVRC2012_val_00013820.JPEG:n01737021 +ILSVRC2012_val_00013821.JPEG:n01883070 +ILSVRC2012_val_00013822.JPEG:n01819313 +ILSVRC2012_val_00013823.JPEG:n03958227 +ILSVRC2012_val_00013824.JPEG:n03841143 +ILSVRC2012_val_00013825.JPEG:n03459775 +ILSVRC2012_val_00013826.JPEG:n03777568 +ILSVRC2012_val_00013827.JPEG:n03417042 +ILSVRC2012_val_00013828.JPEG:n02110185 +ILSVRC2012_val_00013829.JPEG:n03388549 +ILSVRC2012_val_00013830.JPEG:n03924679 +ILSVRC2012_val_00013831.JPEG:n02672831 +ILSVRC2012_val_00013832.JPEG:n02165456 +ILSVRC2012_val_00013833.JPEG:n03207743 +ILSVRC2012_val_00013834.JPEG:n04136333 +ILSVRC2012_val_00013835.JPEG:n02971356 +ILSVRC2012_val_00013836.JPEG:n04039381 +ILSVRC2012_val_00013837.JPEG:n04162706 +ILSVRC2012_val_00013838.JPEG:n02791124 +ILSVRC2012_val_00013839.JPEG:n03124170 +ILSVRC2012_val_00013840.JPEG:n01843065 +ILSVRC2012_val_00013841.JPEG:n04428191 +ILSVRC2012_val_00013842.JPEG:n03874599 +ILSVRC2012_val_00013843.JPEG:n02102480 +ILSVRC2012_val_00013844.JPEG:n04487394 +ILSVRC2012_val_00013845.JPEG:n01883070 +ILSVRC2012_val_00013846.JPEG:n02966193 +ILSVRC2012_val_00013847.JPEG:n01494475 +ILSVRC2012_val_00013848.JPEG:n02110341 +ILSVRC2012_val_00013849.JPEG:n07716358 +ILSVRC2012_val_00013850.JPEG:n07248320 +ILSVRC2012_val_00013851.JPEG:n02814860 +ILSVRC2012_val_00013852.JPEG:n04133789 +ILSVRC2012_val_00013853.JPEG:n02443114 +ILSVRC2012_val_00013854.JPEG:n02110063 +ILSVRC2012_val_00013855.JPEG:n04509417 +ILSVRC2012_val_00013856.JPEG:n02108089 +ILSVRC2012_val_00013857.JPEG:n04548362 +ILSVRC2012_val_00013858.JPEG:n01748264 +ILSVRC2012_val_00013859.JPEG:n03710637 +ILSVRC2012_val_00013860.JPEG:n02091467 +ILSVRC2012_val_00013861.JPEG:n02110341 +ILSVRC2012_val_00013862.JPEG:n02113624 +ILSVRC2012_val_00013863.JPEG:n01819313 +ILSVRC2012_val_00013864.JPEG:n02939185 +ILSVRC2012_val_00013865.JPEG:n03272562 +ILSVRC2012_val_00013866.JPEG:n02787622 +ILSVRC2012_val_00013867.JPEG:n12267677 +ILSVRC2012_val_00013868.JPEG:n04141327 +ILSVRC2012_val_00013869.JPEG:n02110958 +ILSVRC2012_val_00013870.JPEG:n01687978 +ILSVRC2012_val_00013871.JPEG:n04429376 +ILSVRC2012_val_00013872.JPEG:n01729322 +ILSVRC2012_val_00013873.JPEG:n02093647 +ILSVRC2012_val_00013874.JPEG:n07920052 +ILSVRC2012_val_00013875.JPEG:n01910747 +ILSVRC2012_val_00013876.JPEG:n02107908 +ILSVRC2012_val_00013877.JPEG:n03895866 +ILSVRC2012_val_00013878.JPEG:n02086079 +ILSVRC2012_val_00013879.JPEG:n02895154 +ILSVRC2012_val_00013880.JPEG:n13037406 +ILSVRC2012_val_00013881.JPEG:n03876231 +ILSVRC2012_val_00013882.JPEG:n04590129 +ILSVRC2012_val_00013883.JPEG:n01692333 +ILSVRC2012_val_00013884.JPEG:n03717622 +ILSVRC2012_val_00013885.JPEG:n02109525 +ILSVRC2012_val_00013886.JPEG:n04355338 +ILSVRC2012_val_00013887.JPEG:n03777568 +ILSVRC2012_val_00013888.JPEG:n03314780 +ILSVRC2012_val_00013889.JPEG:n03887697 +ILSVRC2012_val_00013890.JPEG:n04141975 +ILSVRC2012_val_00013891.JPEG:n01978287 +ILSVRC2012_val_00013892.JPEG:n04597913 +ILSVRC2012_val_00013893.JPEG:n04141975 +ILSVRC2012_val_00013894.JPEG:n02782093 +ILSVRC2012_val_00013895.JPEG:n03868242 +ILSVRC2012_val_00013896.JPEG:n02002724 +ILSVRC2012_val_00013897.JPEG:n03196217 +ILSVRC2012_val_00013898.JPEG:n04153751 +ILSVRC2012_val_00013899.JPEG:n01629819 +ILSVRC2012_val_00013900.JPEG:n02808440 +ILSVRC2012_val_00013901.JPEG:n02058221 +ILSVRC2012_val_00013902.JPEG:n01531178 +ILSVRC2012_val_00013903.JPEG:n02114712 +ILSVRC2012_val_00013904.JPEG:n03494278 +ILSVRC2012_val_00013905.JPEG:n04204347 +ILSVRC2012_val_00013906.JPEG:n03793489 +ILSVRC2012_val_00013907.JPEG:n03483316 +ILSVRC2012_val_00013908.JPEG:n04209239 +ILSVRC2012_val_00013909.JPEG:n03776460 +ILSVRC2012_val_00013910.JPEG:n04336792 +ILSVRC2012_val_00013911.JPEG:n02114548 +ILSVRC2012_val_00013912.JPEG:n02667093 +ILSVRC2012_val_00013913.JPEG:n02834397 +ILSVRC2012_val_00013914.JPEG:n04456115 +ILSVRC2012_val_00013915.JPEG:n03394916 +ILSVRC2012_val_00013916.JPEG:n04346328 +ILSVRC2012_val_00013917.JPEG:n01776313 +ILSVRC2012_val_00013918.JPEG:n02124075 +ILSVRC2012_val_00013919.JPEG:n02356798 +ILSVRC2012_val_00013920.JPEG:n03895866 +ILSVRC2012_val_00013921.JPEG:n02963159 +ILSVRC2012_val_00013922.JPEG:n01883070 +ILSVRC2012_val_00013923.JPEG:n03355925 +ILSVRC2012_val_00013924.JPEG:n02226429 +ILSVRC2012_val_00013925.JPEG:n03417042 +ILSVRC2012_val_00013926.JPEG:n02106550 +ILSVRC2012_val_00013927.JPEG:n02101388 +ILSVRC2012_val_00013928.JPEG:n04200800 +ILSVRC2012_val_00013929.JPEG:n02011460 +ILSVRC2012_val_00013930.JPEG:n02112706 +ILSVRC2012_val_00013931.JPEG:n04326547 +ILSVRC2012_val_00013932.JPEG:n01985128 +ILSVRC2012_val_00013933.JPEG:n03110669 +ILSVRC2012_val_00013934.JPEG:n03804744 +ILSVRC2012_val_00013935.JPEG:n04141327 +ILSVRC2012_val_00013936.JPEG:n11939491 +ILSVRC2012_val_00013937.JPEG:n02105251 +ILSVRC2012_val_00013938.JPEG:n03201208 +ILSVRC2012_val_00013939.JPEG:n07754684 +ILSVRC2012_val_00013940.JPEG:n01632777 +ILSVRC2012_val_00013941.JPEG:n04553703 +ILSVRC2012_val_00013942.JPEG:n04149813 +ILSVRC2012_val_00013943.JPEG:n02481823 +ILSVRC2012_val_00013944.JPEG:n03947888 +ILSVRC2012_val_00013945.JPEG:n01534433 +ILSVRC2012_val_00013946.JPEG:n03457902 +ILSVRC2012_val_00013947.JPEG:n02776631 +ILSVRC2012_val_00013948.JPEG:n04209239 +ILSVRC2012_val_00013949.JPEG:n04523525 +ILSVRC2012_val_00013950.JPEG:n04074963 +ILSVRC2012_val_00013951.JPEG:n02233338 +ILSVRC2012_val_00013952.JPEG:n03930313 +ILSVRC2012_val_00013953.JPEG:n03249569 +ILSVRC2012_val_00013954.JPEG:n03884397 +ILSVRC2012_val_00013955.JPEG:n01601694 +ILSVRC2012_val_00013956.JPEG:n04560804 +ILSVRC2012_val_00013957.JPEG:n02514041 +ILSVRC2012_val_00013958.JPEG:n03417042 +ILSVRC2012_val_00013959.JPEG:n07880968 +ILSVRC2012_val_00013960.JPEG:n03594734 +ILSVRC2012_val_00013961.JPEG:n03344393 +ILSVRC2012_val_00013962.JPEG:n02088632 +ILSVRC2012_val_00013963.JPEG:n02106662 +ILSVRC2012_val_00013964.JPEG:n02108551 +ILSVRC2012_val_00013965.JPEG:n01744401 +ILSVRC2012_val_00013966.JPEG:n02483708 +ILSVRC2012_val_00013967.JPEG:n02971356 +ILSVRC2012_val_00013968.JPEG:n02909870 +ILSVRC2012_val_00013969.JPEG:n02841315 +ILSVRC2012_val_00013970.JPEG:n03496892 +ILSVRC2012_val_00013971.JPEG:n02100583 +ILSVRC2012_val_00013972.JPEG:n03476684 +ILSVRC2012_val_00013973.JPEG:n07718472 +ILSVRC2012_val_00013974.JPEG:n01641577 +ILSVRC2012_val_00013975.JPEG:n06596364 +ILSVRC2012_val_00013976.JPEG:n03954731 +ILSVRC2012_val_00013977.JPEG:n04357314 +ILSVRC2012_val_00013978.JPEG:n04259630 +ILSVRC2012_val_00013979.JPEG:n07695742 +ILSVRC2012_val_00013980.JPEG:n04423845 +ILSVRC2012_val_00013981.JPEG:n03249569 +ILSVRC2012_val_00013982.JPEG:n04111531 +ILSVRC2012_val_00013983.JPEG:n02895154 +ILSVRC2012_val_00013984.JPEG:n04149813 +ILSVRC2012_val_00013985.JPEG:n02114712 +ILSVRC2012_val_00013986.JPEG:n04252225 +ILSVRC2012_val_00013987.JPEG:n03770679 +ILSVRC2012_val_00013988.JPEG:n02837789 +ILSVRC2012_val_00013989.JPEG:n04428191 +ILSVRC2012_val_00013990.JPEG:n02361337 +ILSVRC2012_val_00013991.JPEG:n02100236 +ILSVRC2012_val_00013992.JPEG:n01728920 +ILSVRC2012_val_00013993.JPEG:n03594945 +ILSVRC2012_val_00013994.JPEG:n02268443 +ILSVRC2012_val_00013995.JPEG:n07875152 +ILSVRC2012_val_00013996.JPEG:n07695742 +ILSVRC2012_val_00013997.JPEG:n02108551 +ILSVRC2012_val_00013998.JPEG:n01531178 +ILSVRC2012_val_00013999.JPEG:n01980166 +ILSVRC2012_val_00014000.JPEG:n02106382 +ILSVRC2012_val_00014001.JPEG:n03658185 +ILSVRC2012_val_00014002.JPEG:n02988304 +ILSVRC2012_val_00014003.JPEG:n04141076 +ILSVRC2012_val_00014004.JPEG:n02906734 +ILSVRC2012_val_00014005.JPEG:n02012849 +ILSVRC2012_val_00014006.JPEG:n02786058 +ILSVRC2012_val_00014007.JPEG:n01614925 +ILSVRC2012_val_00014008.JPEG:n02206856 +ILSVRC2012_val_00014009.JPEG:n01631663 +ILSVRC2012_val_00014010.JPEG:n03100240 +ILSVRC2012_val_00014011.JPEG:n03047690 +ILSVRC2012_val_00014012.JPEG:n03180011 +ILSVRC2012_val_00014013.JPEG:n02895154 +ILSVRC2012_val_00014014.JPEG:n02782093 +ILSVRC2012_val_00014015.JPEG:n03595614 +ILSVRC2012_val_00014016.JPEG:n09332890 +ILSVRC2012_val_00014017.JPEG:n07749582 +ILSVRC2012_val_00014018.JPEG:n04258138 +ILSVRC2012_val_00014019.JPEG:n03095699 +ILSVRC2012_val_00014020.JPEG:n02096177 +ILSVRC2012_val_00014021.JPEG:n01728920 +ILSVRC2012_val_00014022.JPEG:n03538406 +ILSVRC2012_val_00014023.JPEG:n01806143 +ILSVRC2012_val_00014024.JPEG:n02088238 +ILSVRC2012_val_00014025.JPEG:n04501370 +ILSVRC2012_val_00014026.JPEG:n09229709 +ILSVRC2012_val_00014027.JPEG:n04423845 +ILSVRC2012_val_00014028.JPEG:n02397096 +ILSVRC2012_val_00014029.JPEG:n02133161 +ILSVRC2012_val_00014030.JPEG:n02088238 +ILSVRC2012_val_00014031.JPEG:n02264363 +ILSVRC2012_val_00014032.JPEG:n02101006 +ILSVRC2012_val_00014033.JPEG:n04515003 +ILSVRC2012_val_00014034.JPEG:n02870880 +ILSVRC2012_val_00014035.JPEG:n04548280 +ILSVRC2012_val_00014036.JPEG:n04461696 +ILSVRC2012_val_00014037.JPEG:n03028079 +ILSVRC2012_val_00014038.JPEG:n02268853 +ILSVRC2012_val_00014039.JPEG:n03874599 +ILSVRC2012_val_00014040.JPEG:n01877812 +ILSVRC2012_val_00014041.JPEG:n02699494 +ILSVRC2012_val_00014042.JPEG:n12985857 +ILSVRC2012_val_00014043.JPEG:n02454379 +ILSVRC2012_val_00014044.JPEG:n04326547 +ILSVRC2012_val_00014045.JPEG:n02089867 +ILSVRC2012_val_00014046.JPEG:n01560419 +ILSVRC2012_val_00014047.JPEG:n02093256 +ILSVRC2012_val_00014048.JPEG:n04204347 +ILSVRC2012_val_00014049.JPEG:n04347754 +ILSVRC2012_val_00014050.JPEG:n02086240 +ILSVRC2012_val_00014051.JPEG:n04286575 +ILSVRC2012_val_00014052.JPEG:n04482393 +ILSVRC2012_val_00014053.JPEG:n03840681 +ILSVRC2012_val_00014054.JPEG:n04065272 +ILSVRC2012_val_00014055.JPEG:n02480855 +ILSVRC2012_val_00014056.JPEG:n02749479 +ILSVRC2012_val_00014057.JPEG:n03492542 +ILSVRC2012_val_00014058.JPEG:n02096437 +ILSVRC2012_val_00014059.JPEG:n02317335 +ILSVRC2012_val_00014060.JPEG:n02174001 +ILSVRC2012_val_00014061.JPEG:n04525305 +ILSVRC2012_val_00014062.JPEG:n04039381 +ILSVRC2012_val_00014063.JPEG:n07753592 +ILSVRC2012_val_00014064.JPEG:n13037406 +ILSVRC2012_val_00014065.JPEG:n02494079 +ILSVRC2012_val_00014066.JPEG:n04258138 +ILSVRC2012_val_00014067.JPEG:n02229544 +ILSVRC2012_val_00014068.JPEG:n01843383 +ILSVRC2012_val_00014069.JPEG:n01728920 +ILSVRC2012_val_00014070.JPEG:n04330267 +ILSVRC2012_val_00014071.JPEG:n02325366 +ILSVRC2012_val_00014072.JPEG:n02808304 +ILSVRC2012_val_00014073.JPEG:n04462240 +ILSVRC2012_val_00014074.JPEG:n03874293 +ILSVRC2012_val_00014075.JPEG:n03482405 +ILSVRC2012_val_00014076.JPEG:n01629819 +ILSVRC2012_val_00014077.JPEG:n03781244 +ILSVRC2012_val_00014078.JPEG:n04392985 +ILSVRC2012_val_00014079.JPEG:n04258138 +ILSVRC2012_val_00014080.JPEG:n03160309 +ILSVRC2012_val_00014081.JPEG:n02096585 +ILSVRC2012_val_00014082.JPEG:n01614925 +ILSVRC2012_val_00014083.JPEG:n02017213 +ILSVRC2012_val_00014084.JPEG:n04133789 +ILSVRC2012_val_00014085.JPEG:n04277352 +ILSVRC2012_val_00014086.JPEG:n02106030 +ILSVRC2012_val_00014087.JPEG:n04428191 +ILSVRC2012_val_00014088.JPEG:n03400231 +ILSVRC2012_val_00014089.JPEG:n03249569 +ILSVRC2012_val_00014090.JPEG:n01514668 +ILSVRC2012_val_00014091.JPEG:n10148035 +ILSVRC2012_val_00014092.JPEG:n02397096 +ILSVRC2012_val_00014093.JPEG:n07697313 +ILSVRC2012_val_00014094.JPEG:n07802026 +ILSVRC2012_val_00014095.JPEG:n03887697 +ILSVRC2012_val_00014096.JPEG:n07248320 +ILSVRC2012_val_00014097.JPEG:n01855032 +ILSVRC2012_val_00014098.JPEG:n03908618 +ILSVRC2012_val_00014099.JPEG:n02086910 +ILSVRC2012_val_00014100.JPEG:n04254680 +ILSVRC2012_val_00014101.JPEG:n02104365 +ILSVRC2012_val_00014102.JPEG:n03445777 +ILSVRC2012_val_00014103.JPEG:n02011460 +ILSVRC2012_val_00014104.JPEG:n07695742 +ILSVRC2012_val_00014105.JPEG:n04344873 +ILSVRC2012_val_00014106.JPEG:n01667778 +ILSVRC2012_val_00014107.JPEG:n02091244 +ILSVRC2012_val_00014108.JPEG:n01534433 +ILSVRC2012_val_00014109.JPEG:n02097474 +ILSVRC2012_val_00014110.JPEG:n02701002 +ILSVRC2012_val_00014111.JPEG:n03208938 +ILSVRC2012_val_00014112.JPEG:n03676483 +ILSVRC2012_val_00014113.JPEG:n03770439 +ILSVRC2012_val_00014114.JPEG:n01755581 +ILSVRC2012_val_00014115.JPEG:n02108915 +ILSVRC2012_val_00014116.JPEG:n01753488 +ILSVRC2012_val_00014117.JPEG:n02102480 +ILSVRC2012_val_00014118.JPEG:n03633091 +ILSVRC2012_val_00014119.JPEG:n03662601 +ILSVRC2012_val_00014120.JPEG:n01770393 +ILSVRC2012_val_00014121.JPEG:n07590611 +ILSVRC2012_val_00014122.JPEG:n04264628 +ILSVRC2012_val_00014123.JPEG:n03998194 +ILSVRC2012_val_00014124.JPEG:n02396427 +ILSVRC2012_val_00014125.JPEG:n02102040 +ILSVRC2012_val_00014126.JPEG:n01770393 +ILSVRC2012_val_00014127.JPEG:n04162706 +ILSVRC2012_val_00014128.JPEG:n02281406 +ILSVRC2012_val_00014129.JPEG:n12768682 +ILSVRC2012_val_00014130.JPEG:n01945685 +ILSVRC2012_val_00014131.JPEG:n03483316 +ILSVRC2012_val_00014132.JPEG:n01978287 +ILSVRC2012_val_00014133.JPEG:n02119022 +ILSVRC2012_val_00014134.JPEG:n02169497 +ILSVRC2012_val_00014135.JPEG:n03991062 +ILSVRC2012_val_00014136.JPEG:n04465501 +ILSVRC2012_val_00014137.JPEG:n07614500 +ILSVRC2012_val_00014138.JPEG:n01990800 +ILSVRC2012_val_00014139.JPEG:n01534433 +ILSVRC2012_val_00014140.JPEG:n03770679 +ILSVRC2012_val_00014141.JPEG:n09288635 +ILSVRC2012_val_00014142.JPEG:n03188531 +ILSVRC2012_val_00014143.JPEG:n09256479 +ILSVRC2012_val_00014144.JPEG:n04259630 +ILSVRC2012_val_00014145.JPEG:n02110627 +ILSVRC2012_val_00014146.JPEG:n04560804 +ILSVRC2012_val_00014147.JPEG:n02113978 +ILSVRC2012_val_00014148.JPEG:n02095889 +ILSVRC2012_val_00014149.JPEG:n04599235 +ILSVRC2012_val_00014150.JPEG:n03259280 +ILSVRC2012_val_00014151.JPEG:n02111277 +ILSVRC2012_val_00014152.JPEG:n02794156 +ILSVRC2012_val_00014153.JPEG:n04328186 +ILSVRC2012_val_00014154.JPEG:n04254680 +ILSVRC2012_val_00014155.JPEG:n03661043 +ILSVRC2012_val_00014156.JPEG:n03599486 +ILSVRC2012_val_00014157.JPEG:n02097130 +ILSVRC2012_val_00014158.JPEG:n02033041 +ILSVRC2012_val_00014159.JPEG:n02071294 +ILSVRC2012_val_00014160.JPEG:n03937543 +ILSVRC2012_val_00014161.JPEG:n09288635 +ILSVRC2012_val_00014162.JPEG:n03709823 +ILSVRC2012_val_00014163.JPEG:n02489166 +ILSVRC2012_val_00014164.JPEG:n03673027 +ILSVRC2012_val_00014165.JPEG:n01828970 +ILSVRC2012_val_00014166.JPEG:n04532106 +ILSVRC2012_val_00014167.JPEG:n03496892 +ILSVRC2012_val_00014168.JPEG:n01924916 +ILSVRC2012_val_00014169.JPEG:n04548280 +ILSVRC2012_val_00014170.JPEG:n02319095 +ILSVRC2012_val_00014171.JPEG:n02395406 +ILSVRC2012_val_00014172.JPEG:n02782093 +ILSVRC2012_val_00014173.JPEG:n04554684 +ILSVRC2012_val_00014174.JPEG:n02086240 +ILSVRC2012_val_00014175.JPEG:n03916031 +ILSVRC2012_val_00014176.JPEG:n02791270 +ILSVRC2012_val_00014177.JPEG:n07717410 +ILSVRC2012_val_00014178.JPEG:n04238763 +ILSVRC2012_val_00014179.JPEG:n02730930 +ILSVRC2012_val_00014180.JPEG:n01514859 +ILSVRC2012_val_00014181.JPEG:n01748264 +ILSVRC2012_val_00014182.JPEG:n02988304 +ILSVRC2012_val_00014183.JPEG:n03461385 +ILSVRC2012_val_00014184.JPEG:n03272562 +ILSVRC2012_val_00014185.JPEG:n04330267 +ILSVRC2012_val_00014186.JPEG:n07860988 +ILSVRC2012_val_00014187.JPEG:n02276258 +ILSVRC2012_val_00014188.JPEG:n07871810 +ILSVRC2012_val_00014189.JPEG:n02097474 +ILSVRC2012_val_00014190.JPEG:n02999410 +ILSVRC2012_val_00014191.JPEG:n04037443 +ILSVRC2012_val_00014192.JPEG:n01614925 +ILSVRC2012_val_00014193.JPEG:n04033901 +ILSVRC2012_val_00014194.JPEG:n03944341 +ILSVRC2012_val_00014195.JPEG:n02655020 +ILSVRC2012_val_00014196.JPEG:n01608432 +ILSVRC2012_val_00014197.JPEG:n03874599 +ILSVRC2012_val_00014198.JPEG:n03594945 +ILSVRC2012_val_00014199.JPEG:n04252225 +ILSVRC2012_val_00014200.JPEG:n07892512 +ILSVRC2012_val_00014201.JPEG:n03717622 +ILSVRC2012_val_00014202.JPEG:n03763968 +ILSVRC2012_val_00014203.JPEG:n02110627 +ILSVRC2012_val_00014204.JPEG:n02795169 +ILSVRC2012_val_00014205.JPEG:n03000134 +ILSVRC2012_val_00014206.JPEG:n02494079 +ILSVRC2012_val_00014207.JPEG:n03042490 +ILSVRC2012_val_00014208.JPEG:n03100240 +ILSVRC2012_val_00014209.JPEG:n07875152 +ILSVRC2012_val_00014210.JPEG:n02802426 +ILSVRC2012_val_00014211.JPEG:n02484975 +ILSVRC2012_val_00014212.JPEG:n09229709 +ILSVRC2012_val_00014213.JPEG:n02747177 +ILSVRC2012_val_00014214.JPEG:n06596364 +ILSVRC2012_val_00014215.JPEG:n04557648 +ILSVRC2012_val_00014216.JPEG:n02123394 +ILSVRC2012_val_00014217.JPEG:n02002724 +ILSVRC2012_val_00014218.JPEG:n02167151 +ILSVRC2012_val_00014219.JPEG:n02504013 +ILSVRC2012_val_00014220.JPEG:n01616318 +ILSVRC2012_val_00014221.JPEG:n03770439 +ILSVRC2012_val_00014222.JPEG:n04428191 +ILSVRC2012_val_00014223.JPEG:n02051845 +ILSVRC2012_val_00014224.JPEG:n04579145 +ILSVRC2012_val_00014225.JPEG:n02093754 +ILSVRC2012_val_00014226.JPEG:n12267677 +ILSVRC2012_val_00014227.JPEG:n01641577 +ILSVRC2012_val_00014228.JPEG:n02963159 +ILSVRC2012_val_00014229.JPEG:n02807133 +ILSVRC2012_val_00014230.JPEG:n04590129 +ILSVRC2012_val_00014231.JPEG:n03467068 +ILSVRC2012_val_00014232.JPEG:n01629819 +ILSVRC2012_val_00014233.JPEG:n02443484 +ILSVRC2012_val_00014234.JPEG:n02088238 +ILSVRC2012_val_00014235.JPEG:n02412080 +ILSVRC2012_val_00014236.JPEG:n03532672 +ILSVRC2012_val_00014237.JPEG:n04591157 +ILSVRC2012_val_00014238.JPEG:n04486054 +ILSVRC2012_val_00014239.JPEG:n02692877 +ILSVRC2012_val_00014240.JPEG:n02727426 +ILSVRC2012_val_00014241.JPEG:n04371774 +ILSVRC2012_val_00014242.JPEG:n04273569 +ILSVRC2012_val_00014243.JPEG:n03733131 +ILSVRC2012_val_00014244.JPEG:n03544143 +ILSVRC2012_val_00014245.JPEG:n02104365 +ILSVRC2012_val_00014246.JPEG:n02109961 +ILSVRC2012_val_00014247.JPEG:n03447447 +ILSVRC2012_val_00014248.JPEG:n01872401 +ILSVRC2012_val_00014249.JPEG:n03961711 +ILSVRC2012_val_00014250.JPEG:n02116738 +ILSVRC2012_val_00014251.JPEG:n01688243 +ILSVRC2012_val_00014252.JPEG:n01749939 +ILSVRC2012_val_00014253.JPEG:n03141823 +ILSVRC2012_val_00014254.JPEG:n02509815 +ILSVRC2012_val_00014255.JPEG:n12985857 +ILSVRC2012_val_00014256.JPEG:n01829413 +ILSVRC2012_val_00014257.JPEG:n02109047 +ILSVRC2012_val_00014258.JPEG:n02526121 +ILSVRC2012_val_00014259.JPEG:n02097658 +ILSVRC2012_val_00014260.JPEG:n03216828 +ILSVRC2012_val_00014261.JPEG:n02870880 +ILSVRC2012_val_00014262.JPEG:n04266014 +ILSVRC2012_val_00014263.JPEG:n04355338 +ILSVRC2012_val_00014264.JPEG:n03633091 +ILSVRC2012_val_00014265.JPEG:n01910747 +ILSVRC2012_val_00014266.JPEG:n02006656 +ILSVRC2012_val_00014267.JPEG:n03445924 +ILSVRC2012_val_00014268.JPEG:n02906734 +ILSVRC2012_val_00014269.JPEG:n04099969 +ILSVRC2012_val_00014270.JPEG:n02099712 +ILSVRC2012_val_00014271.JPEG:n02229544 +ILSVRC2012_val_00014272.JPEG:n04443257 +ILSVRC2012_val_00014273.JPEG:n02687172 +ILSVRC2012_val_00014274.JPEG:n04273569 +ILSVRC2012_val_00014275.JPEG:n02489166 +ILSVRC2012_val_00014276.JPEG:n03924679 +ILSVRC2012_val_00014277.JPEG:n12985857 +ILSVRC2012_val_00014278.JPEG:n02167151 +ILSVRC2012_val_00014279.JPEG:n02321529 +ILSVRC2012_val_00014280.JPEG:n02102040 +ILSVRC2012_val_00014281.JPEG:n02870880 +ILSVRC2012_val_00014282.JPEG:n01693334 +ILSVRC2012_val_00014283.JPEG:n02097298 +ILSVRC2012_val_00014284.JPEG:n01882714 +ILSVRC2012_val_00014285.JPEG:n04040759 +ILSVRC2012_val_00014286.JPEG:n03791053 +ILSVRC2012_val_00014287.JPEG:n02979186 +ILSVRC2012_val_00014288.JPEG:n02454379 +ILSVRC2012_val_00014289.JPEG:n03131574 +ILSVRC2012_val_00014290.JPEG:n04141327 +ILSVRC2012_val_00014291.JPEG:n02981792 +ILSVRC2012_val_00014292.JPEG:n02974003 +ILSVRC2012_val_00014293.JPEG:n02090721 +ILSVRC2012_val_00014294.JPEG:n04131690 +ILSVRC2012_val_00014295.JPEG:n02106030 +ILSVRC2012_val_00014296.JPEG:n02493793 +ILSVRC2012_val_00014297.JPEG:n02963159 +ILSVRC2012_val_00014298.JPEG:n04596742 +ILSVRC2012_val_00014299.JPEG:n11879895 +ILSVRC2012_val_00014300.JPEG:n03457902 +ILSVRC2012_val_00014301.JPEG:n02823750 +ILSVRC2012_val_00014302.JPEG:n01774750 +ILSVRC2012_val_00014303.JPEG:n03788365 +ILSVRC2012_val_00014304.JPEG:n02389026 +ILSVRC2012_val_00014305.JPEG:n02823750 +ILSVRC2012_val_00014306.JPEG:n02493509 +ILSVRC2012_val_00014307.JPEG:n07583066 +ILSVRC2012_val_00014308.JPEG:n01682714 +ILSVRC2012_val_00014309.JPEG:n03899768 +ILSVRC2012_val_00014310.JPEG:n02279972 +ILSVRC2012_val_00014311.JPEG:n07747607 +ILSVRC2012_val_00014312.JPEG:n01692333 +ILSVRC2012_val_00014313.JPEG:n04243546 +ILSVRC2012_val_00014314.JPEG:n04317175 +ILSVRC2012_val_00014315.JPEG:n02106550 +ILSVRC2012_val_00014316.JPEG:n01664065 +ILSVRC2012_val_00014317.JPEG:n01677366 +ILSVRC2012_val_00014318.JPEG:n02093754 +ILSVRC2012_val_00014319.JPEG:n04346328 +ILSVRC2012_val_00014320.JPEG:n02106550 +ILSVRC2012_val_00014321.JPEG:n02127052 +ILSVRC2012_val_00014322.JPEG:n03666591 +ILSVRC2012_val_00014323.JPEG:n03877845 +ILSVRC2012_val_00014324.JPEG:n03125729 +ILSVRC2012_val_00014325.JPEG:n03786901 +ILSVRC2012_val_00014326.JPEG:n03775071 +ILSVRC2012_val_00014327.JPEG:n02412080 +ILSVRC2012_val_00014328.JPEG:n01518878 +ILSVRC2012_val_00014329.JPEG:n03720891 +ILSVRC2012_val_00014330.JPEG:n01735189 +ILSVRC2012_val_00014331.JPEG:n02356798 +ILSVRC2012_val_00014332.JPEG:n02110806 +ILSVRC2012_val_00014333.JPEG:n03047690 +ILSVRC2012_val_00014334.JPEG:n04462240 +ILSVRC2012_val_00014335.JPEG:n02951585 +ILSVRC2012_val_00014336.JPEG:n01558993 +ILSVRC2012_val_00014337.JPEG:n03065424 +ILSVRC2012_val_00014338.JPEG:n02860847 +ILSVRC2012_val_00014339.JPEG:n02486410 +ILSVRC2012_val_00014340.JPEG:n02398521 +ILSVRC2012_val_00014341.JPEG:n04346328 +ILSVRC2012_val_00014342.JPEG:n02106030 +ILSVRC2012_val_00014343.JPEG:n02445715 +ILSVRC2012_val_00014344.JPEG:n04153751 +ILSVRC2012_val_00014345.JPEG:n02509815 +ILSVRC2012_val_00014346.JPEG:n01828970 +ILSVRC2012_val_00014347.JPEG:n04069434 +ILSVRC2012_val_00014348.JPEG:n07714571 +ILSVRC2012_val_00014349.JPEG:n13044778 +ILSVRC2012_val_00014350.JPEG:n01955084 +ILSVRC2012_val_00014351.JPEG:n03662601 +ILSVRC2012_val_00014352.JPEG:n01664065 +ILSVRC2012_val_00014353.JPEG:n02708093 +ILSVRC2012_val_00014354.JPEG:n02408429 +ILSVRC2012_val_00014355.JPEG:n03920288 +ILSVRC2012_val_00014356.JPEG:n02190166 +ILSVRC2012_val_00014357.JPEG:n02091635 +ILSVRC2012_val_00014358.JPEG:n04229816 +ILSVRC2012_val_00014359.JPEG:n01773549 +ILSVRC2012_val_00014360.JPEG:n02106662 +ILSVRC2012_val_00014361.JPEG:n02009912 +ILSVRC2012_val_00014362.JPEG:n01558993 +ILSVRC2012_val_00014363.JPEG:n02127052 +ILSVRC2012_val_00014364.JPEG:n02843684 +ILSVRC2012_val_00014365.JPEG:n02174001 +ILSVRC2012_val_00014366.JPEG:n03345487 +ILSVRC2012_val_00014367.JPEG:n01990800 +ILSVRC2012_val_00014368.JPEG:n03584254 +ILSVRC2012_val_00014369.JPEG:n02389026 +ILSVRC2012_val_00014370.JPEG:n02389026 +ILSVRC2012_val_00014371.JPEG:n04069434 +ILSVRC2012_val_00014372.JPEG:n03032252 +ILSVRC2012_val_00014373.JPEG:n07749582 +ILSVRC2012_val_00014374.JPEG:n02110627 +ILSVRC2012_val_00014375.JPEG:n02807133 +ILSVRC2012_val_00014376.JPEG:n02012849 +ILSVRC2012_val_00014377.JPEG:n03208938 +ILSVRC2012_val_00014378.JPEG:n02107142 +ILSVRC2012_val_00014379.JPEG:n03995372 +ILSVRC2012_val_00014380.JPEG:n02927161 +ILSVRC2012_val_00014381.JPEG:n03888257 +ILSVRC2012_val_00014382.JPEG:n02802426 +ILSVRC2012_val_00014383.JPEG:n09193705 +ILSVRC2012_val_00014384.JPEG:n07716906 +ILSVRC2012_val_00014385.JPEG:n03345487 +ILSVRC2012_val_00014386.JPEG:n02088094 +ILSVRC2012_val_00014387.JPEG:n03297495 +ILSVRC2012_val_00014388.JPEG:n02871525 +ILSVRC2012_val_00014389.JPEG:n02363005 +ILSVRC2012_val_00014390.JPEG:n02206856 +ILSVRC2012_val_00014391.JPEG:n02445715 +ILSVRC2012_val_00014392.JPEG:n02783161 +ILSVRC2012_val_00014393.JPEG:n02948072 +ILSVRC2012_val_00014394.JPEG:n09421951 +ILSVRC2012_val_00014395.JPEG:n02410509 +ILSVRC2012_val_00014396.JPEG:n02808304 +ILSVRC2012_val_00014397.JPEG:n03903868 +ILSVRC2012_val_00014398.JPEG:n02110063 +ILSVRC2012_val_00014399.JPEG:n03724870 +ILSVRC2012_val_00014400.JPEG:n07836838 +ILSVRC2012_val_00014401.JPEG:n04141975 +ILSVRC2012_val_00014402.JPEG:n02487347 +ILSVRC2012_val_00014403.JPEG:n02112137 +ILSVRC2012_val_00014404.JPEG:n02804610 +ILSVRC2012_val_00014405.JPEG:n07734744 +ILSVRC2012_val_00014406.JPEG:n04462240 +ILSVRC2012_val_00014407.JPEG:n03372029 +ILSVRC2012_val_00014408.JPEG:n02177972 +ILSVRC2012_val_00014409.JPEG:n02085620 +ILSVRC2012_val_00014410.JPEG:n01917289 +ILSVRC2012_val_00014411.JPEG:n04070727 +ILSVRC2012_val_00014412.JPEG:n02823428 +ILSVRC2012_val_00014413.JPEG:n02860847 +ILSVRC2012_val_00014414.JPEG:n04392985 +ILSVRC2012_val_00014415.JPEG:n02791124 +ILSVRC2012_val_00014416.JPEG:n01847000 +ILSVRC2012_val_00014417.JPEG:n01784675 +ILSVRC2012_val_00014418.JPEG:n02093991 +ILSVRC2012_val_00014419.JPEG:n03457902 +ILSVRC2012_val_00014420.JPEG:n02939185 +ILSVRC2012_val_00014421.JPEG:n04493381 +ILSVRC2012_val_00014422.JPEG:n03271574 +ILSVRC2012_val_00014423.JPEG:n02509815 +ILSVRC2012_val_00014424.JPEG:n03793489 +ILSVRC2012_val_00014425.JPEG:n02690373 +ILSVRC2012_val_00014426.JPEG:n03983396 +ILSVRC2012_val_00014427.JPEG:n02927161 +ILSVRC2012_val_00014428.JPEG:n03018349 +ILSVRC2012_val_00014429.JPEG:n03908618 +ILSVRC2012_val_00014430.JPEG:n02110341 +ILSVRC2012_val_00014431.JPEG:n03776460 +ILSVRC2012_val_00014432.JPEG:n02124075 +ILSVRC2012_val_00014433.JPEG:n04335435 +ILSVRC2012_val_00014434.JPEG:n03127747 +ILSVRC2012_val_00014435.JPEG:n02948072 +ILSVRC2012_val_00014436.JPEG:n03085013 +ILSVRC2012_val_00014437.JPEG:n02442845 +ILSVRC2012_val_00014438.JPEG:n02916936 +ILSVRC2012_val_00014439.JPEG:n01688243 +ILSVRC2012_val_00014440.JPEG:n02879718 +ILSVRC2012_val_00014441.JPEG:n02097298 +ILSVRC2012_val_00014442.JPEG:n04589890 +ILSVRC2012_val_00014443.JPEG:n02607072 +ILSVRC2012_val_00014444.JPEG:n02948072 +ILSVRC2012_val_00014445.JPEG:n04525038 +ILSVRC2012_val_00014446.JPEG:n02100735 +ILSVRC2012_val_00014447.JPEG:n02814533 +ILSVRC2012_val_00014448.JPEG:n03000134 +ILSVRC2012_val_00014449.JPEG:n03478589 +ILSVRC2012_val_00014450.JPEG:n02037110 +ILSVRC2012_val_00014451.JPEG:n04235860 +ILSVRC2012_val_00014452.JPEG:n02112137 +ILSVRC2012_val_00014453.JPEG:n04435653 +ILSVRC2012_val_00014454.JPEG:n04273569 +ILSVRC2012_val_00014455.JPEG:n03794056 +ILSVRC2012_val_00014456.JPEG:n01910747 +ILSVRC2012_val_00014457.JPEG:n01748264 +ILSVRC2012_val_00014458.JPEG:n01883070 +ILSVRC2012_val_00014459.JPEG:n04200800 +ILSVRC2012_val_00014460.JPEG:n04590129 +ILSVRC2012_val_00014461.JPEG:n03443371 +ILSVRC2012_val_00014462.JPEG:n02791124 +ILSVRC2012_val_00014463.JPEG:n03075370 +ILSVRC2012_val_00014464.JPEG:n03673027 +ILSVRC2012_val_00014465.JPEG:n01742172 +ILSVRC2012_val_00014466.JPEG:n03476684 +ILSVRC2012_val_00014467.JPEG:n01484850 +ILSVRC2012_val_00014468.JPEG:n01675722 +ILSVRC2012_val_00014469.JPEG:n02978881 +ILSVRC2012_val_00014470.JPEG:n03938244 +ILSVRC2012_val_00014471.JPEG:n02106166 +ILSVRC2012_val_00014472.JPEG:n01729977 +ILSVRC2012_val_00014473.JPEG:n04118776 +ILSVRC2012_val_00014474.JPEG:n04209239 +ILSVRC2012_val_00014475.JPEG:n03376595 +ILSVRC2012_val_00014476.JPEG:n04008634 +ILSVRC2012_val_00014477.JPEG:n02095889 +ILSVRC2012_val_00014478.JPEG:n01855032 +ILSVRC2012_val_00014479.JPEG:n03376595 +ILSVRC2012_val_00014480.JPEG:n04456115 +ILSVRC2012_val_00014481.JPEG:n02879718 +ILSVRC2012_val_00014482.JPEG:n04238763 +ILSVRC2012_val_00014483.JPEG:n02268443 +ILSVRC2012_val_00014484.JPEG:n02794156 +ILSVRC2012_val_00014485.JPEG:n02105505 +ILSVRC2012_val_00014486.JPEG:n01914609 +ILSVRC2012_val_00014487.JPEG:n03899768 +ILSVRC2012_val_00014488.JPEG:n02676566 +ILSVRC2012_val_00014489.JPEG:n02099601 +ILSVRC2012_val_00014490.JPEG:n02106382 +ILSVRC2012_val_00014491.JPEG:n04264628 +ILSVRC2012_val_00014492.JPEG:n04501370 +ILSVRC2012_val_00014493.JPEG:n03594734 +ILSVRC2012_val_00014494.JPEG:n03895866 +ILSVRC2012_val_00014495.JPEG:n04332243 +ILSVRC2012_val_00014496.JPEG:n04008634 +ILSVRC2012_val_00014497.JPEG:n02492035 +ILSVRC2012_val_00014498.JPEG:n01773797 +ILSVRC2012_val_00014499.JPEG:n04228054 +ILSVRC2012_val_00014500.JPEG:n02110958 +ILSVRC2012_val_00014501.JPEG:n06359193 +ILSVRC2012_val_00014502.JPEG:n02403003 +ILSVRC2012_val_00014503.JPEG:n04409515 +ILSVRC2012_val_00014504.JPEG:n03337140 +ILSVRC2012_val_00014505.JPEG:n02483708 +ILSVRC2012_val_00014506.JPEG:n02106166 +ILSVRC2012_val_00014507.JPEG:n04209133 +ILSVRC2012_val_00014508.JPEG:n02114367 +ILSVRC2012_val_00014509.JPEG:n03743016 +ILSVRC2012_val_00014510.JPEG:n03201208 +ILSVRC2012_val_00014511.JPEG:n03207941 +ILSVRC2012_val_00014512.JPEG:n02804414 +ILSVRC2012_val_00014513.JPEG:n04487081 +ILSVRC2012_val_00014514.JPEG:n01945685 +ILSVRC2012_val_00014515.JPEG:n02606052 +ILSVRC2012_val_00014516.JPEG:n03388043 +ILSVRC2012_val_00014517.JPEG:n03661043 +ILSVRC2012_val_00014518.JPEG:n02804610 +ILSVRC2012_val_00014519.JPEG:n04235860 +ILSVRC2012_val_00014520.JPEG:n02795169 +ILSVRC2012_val_00014521.JPEG:n03476991 +ILSVRC2012_val_00014522.JPEG:n03444034 +ILSVRC2012_val_00014523.JPEG:n03942813 +ILSVRC2012_val_00014524.JPEG:n04026417 +ILSVRC2012_val_00014525.JPEG:n03337140 +ILSVRC2012_val_00014526.JPEG:n02108422 +ILSVRC2012_val_00014527.JPEG:n04033995 +ILSVRC2012_val_00014528.JPEG:n03041632 +ILSVRC2012_val_00014529.JPEG:n02134418 +ILSVRC2012_val_00014530.JPEG:n04554684 +ILSVRC2012_val_00014531.JPEG:n03733131 +ILSVRC2012_val_00014532.JPEG:n02116738 +ILSVRC2012_val_00014533.JPEG:n03786901 +ILSVRC2012_val_00014534.JPEG:n03937543 +ILSVRC2012_val_00014535.JPEG:n04147183 +ILSVRC2012_val_00014536.JPEG:n04131690 +ILSVRC2012_val_00014537.JPEG:n03400231 +ILSVRC2012_val_00014538.JPEG:n02125311 +ILSVRC2012_val_00014539.JPEG:n02410509 +ILSVRC2012_val_00014540.JPEG:n01775062 +ILSVRC2012_val_00014541.JPEG:n02814533 +ILSVRC2012_val_00014542.JPEG:n02110185 +ILSVRC2012_val_00014543.JPEG:n04008634 +ILSVRC2012_val_00014544.JPEG:n04597913 +ILSVRC2012_val_00014545.JPEG:n01883070 +ILSVRC2012_val_00014546.JPEG:n07714990 +ILSVRC2012_val_00014547.JPEG:n02112350 +ILSVRC2012_val_00014548.JPEG:n02437616 +ILSVRC2012_val_00014549.JPEG:n03662601 +ILSVRC2012_val_00014550.JPEG:n02074367 +ILSVRC2012_val_00014551.JPEG:n04239074 +ILSVRC2012_val_00014552.JPEG:n03063689 +ILSVRC2012_val_00014553.JPEG:n07831146 +ILSVRC2012_val_00014554.JPEG:n02869837 +ILSVRC2012_val_00014555.JPEG:n03920288 +ILSVRC2012_val_00014556.JPEG:n13052670 +ILSVRC2012_val_00014557.JPEG:n03016953 +ILSVRC2012_val_00014558.JPEG:n02788148 +ILSVRC2012_val_00014559.JPEG:n04613696 +ILSVRC2012_val_00014560.JPEG:n02113023 +ILSVRC2012_val_00014561.JPEG:n03866082 +ILSVRC2012_val_00014562.JPEG:n02992529 +ILSVRC2012_val_00014563.JPEG:n04479046 +ILSVRC2012_val_00014564.JPEG:n04467665 +ILSVRC2012_val_00014565.JPEG:n04540053 +ILSVRC2012_val_00014566.JPEG:n02927161 +ILSVRC2012_val_00014567.JPEG:n03992509 +ILSVRC2012_val_00014568.JPEG:n04347754 +ILSVRC2012_val_00014569.JPEG:n03495258 +ILSVRC2012_val_00014570.JPEG:n03633091 +ILSVRC2012_val_00014571.JPEG:n02105251 +ILSVRC2012_val_00014572.JPEG:n02231487 +ILSVRC2012_val_00014573.JPEG:n02102318 +ILSVRC2012_val_00014574.JPEG:n02667093 +ILSVRC2012_val_00014575.JPEG:n01749939 +ILSVRC2012_val_00014576.JPEG:n02133161 +ILSVRC2012_val_00014577.JPEG:n03372029 +ILSVRC2012_val_00014578.JPEG:n02486261 +ILSVRC2012_val_00014579.JPEG:n04004767 +ILSVRC2012_val_00014580.JPEG:n02088466 +ILSVRC2012_val_00014581.JPEG:n07579787 +ILSVRC2012_val_00014582.JPEG:n02791270 +ILSVRC2012_val_00014583.JPEG:n03131574 +ILSVRC2012_val_00014584.JPEG:n02391049 +ILSVRC2012_val_00014585.JPEG:n01664065 +ILSVRC2012_val_00014586.JPEG:n02099429 +ILSVRC2012_val_00014587.JPEG:n01776313 +ILSVRC2012_val_00014588.JPEG:n03920288 +ILSVRC2012_val_00014589.JPEG:n02109047 +ILSVRC2012_val_00014590.JPEG:n02317335 +ILSVRC2012_val_00014591.JPEG:n04612504 +ILSVRC2012_val_00014592.JPEG:n03584254 +ILSVRC2012_val_00014593.JPEG:n03457902 +ILSVRC2012_val_00014594.JPEG:n02051845 +ILSVRC2012_val_00014595.JPEG:n03047690 +ILSVRC2012_val_00014596.JPEG:n04507155 +ILSVRC2012_val_00014597.JPEG:n02704792 +ILSVRC2012_val_00014598.JPEG:n01748264 +ILSVRC2012_val_00014599.JPEG:n02017213 +ILSVRC2012_val_00014600.JPEG:n03450230 +ILSVRC2012_val_00014601.JPEG:n02841315 +ILSVRC2012_val_00014602.JPEG:n04070727 +ILSVRC2012_val_00014603.JPEG:n02992211 +ILSVRC2012_val_00014604.JPEG:n03404251 +ILSVRC2012_val_00014605.JPEG:n02092339 +ILSVRC2012_val_00014606.JPEG:n12768682 +ILSVRC2012_val_00014607.JPEG:n07873807 +ILSVRC2012_val_00014608.JPEG:n03041632 +ILSVRC2012_val_00014609.JPEG:n03379051 +ILSVRC2012_val_00014610.JPEG:n04435653 +ILSVRC2012_val_00014611.JPEG:n04146614 +ILSVRC2012_val_00014612.JPEG:n02012849 +ILSVRC2012_val_00014613.JPEG:n03443371 +ILSVRC2012_val_00014614.JPEG:n04152593 +ILSVRC2012_val_00014615.JPEG:n04507155 +ILSVRC2012_val_00014616.JPEG:n03447447 +ILSVRC2012_val_00014617.JPEG:n04252225 +ILSVRC2012_val_00014618.JPEG:n03770439 +ILSVRC2012_val_00014619.JPEG:n13037406 +ILSVRC2012_val_00014620.JPEG:n01748264 +ILSVRC2012_val_00014621.JPEG:n04550184 +ILSVRC2012_val_00014622.JPEG:n03207941 +ILSVRC2012_val_00014623.JPEG:n07716906 +ILSVRC2012_val_00014624.JPEG:n03595614 +ILSVRC2012_val_00014625.JPEG:n07875152 +ILSVRC2012_val_00014626.JPEG:n04560804 +ILSVRC2012_val_00014627.JPEG:n04479046 +ILSVRC2012_val_00014628.JPEG:n03127925 +ILSVRC2012_val_00014629.JPEG:n07248320 +ILSVRC2012_val_00014630.JPEG:n02342885 +ILSVRC2012_val_00014631.JPEG:n02088466 +ILSVRC2012_val_00014632.JPEG:n03485407 +ILSVRC2012_val_00014633.JPEG:n09399592 +ILSVRC2012_val_00014634.JPEG:n04039381 +ILSVRC2012_val_00014635.JPEG:n04548280 +ILSVRC2012_val_00014636.JPEG:n02099267 +ILSVRC2012_val_00014637.JPEG:n04254777 +ILSVRC2012_val_00014638.JPEG:n06785654 +ILSVRC2012_val_00014639.JPEG:n02190166 +ILSVRC2012_val_00014640.JPEG:n03868242 +ILSVRC2012_val_00014641.JPEG:n04141076 +ILSVRC2012_val_00014642.JPEG:n02980441 +ILSVRC2012_val_00014643.JPEG:n03868863 +ILSVRC2012_val_00014644.JPEG:n02437312 +ILSVRC2012_val_00014645.JPEG:n02096177 +ILSVRC2012_val_00014646.JPEG:n02701002 +ILSVRC2012_val_00014647.JPEG:n03259280 +ILSVRC2012_val_00014648.JPEG:n02834397 +ILSVRC2012_val_00014649.JPEG:n15075141 +ILSVRC2012_val_00014650.JPEG:n07880968 +ILSVRC2012_val_00014651.JPEG:n02096585 +ILSVRC2012_val_00014652.JPEG:n09256479 +ILSVRC2012_val_00014653.JPEG:n02091032 +ILSVRC2012_val_00014654.JPEG:n03457902 +ILSVRC2012_val_00014655.JPEG:n02099849 +ILSVRC2012_val_00014656.JPEG:n02398521 +ILSVRC2012_val_00014657.JPEG:n02129165 +ILSVRC2012_val_00014658.JPEG:n03404251 +ILSVRC2012_val_00014659.JPEG:n01774384 +ILSVRC2012_val_00014660.JPEG:n03977966 +ILSVRC2012_val_00014661.JPEG:n02980441 +ILSVRC2012_val_00014662.JPEG:n02137549 +ILSVRC2012_val_00014663.JPEG:n03920288 +ILSVRC2012_val_00014664.JPEG:n01770081 +ILSVRC2012_val_00014665.JPEG:n03891332 +ILSVRC2012_val_00014666.JPEG:n03196217 +ILSVRC2012_val_00014667.JPEG:n02782093 +ILSVRC2012_val_00014668.JPEG:n02510455 +ILSVRC2012_val_00014669.JPEG:n03535780 +ILSVRC2012_val_00014670.JPEG:n04263257 +ILSVRC2012_val_00014671.JPEG:n02790996 +ILSVRC2012_val_00014672.JPEG:n03146219 +ILSVRC2012_val_00014673.JPEG:n01601694 +ILSVRC2012_val_00014674.JPEG:n03379051 +ILSVRC2012_val_00014675.JPEG:n03188531 +ILSVRC2012_val_00014676.JPEG:n02790996 +ILSVRC2012_val_00014677.JPEG:n04596742 +ILSVRC2012_val_00014678.JPEG:n01560419 +ILSVRC2012_val_00014679.JPEG:n03376595 +ILSVRC2012_val_00014680.JPEG:n12768682 +ILSVRC2012_val_00014681.JPEG:n02504013 +ILSVRC2012_val_00014682.JPEG:n03388043 +ILSVRC2012_val_00014683.JPEG:n02231487 +ILSVRC2012_val_00014684.JPEG:n03134739 +ILSVRC2012_val_00014685.JPEG:n03775071 +ILSVRC2012_val_00014686.JPEG:n02509815 +ILSVRC2012_val_00014687.JPEG:n07695742 +ILSVRC2012_val_00014688.JPEG:n02325366 +ILSVRC2012_val_00014689.JPEG:n09835506 +ILSVRC2012_val_00014690.JPEG:n04418357 +ILSVRC2012_val_00014691.JPEG:n04483307 +ILSVRC2012_val_00014692.JPEG:n04069434 +ILSVRC2012_val_00014693.JPEG:n03991062 +ILSVRC2012_val_00014694.JPEG:n02487347 +ILSVRC2012_val_00014695.JPEG:n03223299 +ILSVRC2012_val_00014696.JPEG:n02817516 +ILSVRC2012_val_00014697.JPEG:n03207743 +ILSVRC2012_val_00014698.JPEG:n02110627 +ILSVRC2012_val_00014699.JPEG:n04604644 +ILSVRC2012_val_00014700.JPEG:n02112350 +ILSVRC2012_val_00014701.JPEG:n02109961 +ILSVRC2012_val_00014702.JPEG:n03534580 +ILSVRC2012_val_00014703.JPEG:n03208938 +ILSVRC2012_val_00014704.JPEG:n03125729 +ILSVRC2012_val_00014705.JPEG:n03947888 +ILSVRC2012_val_00014706.JPEG:n04154565 +ILSVRC2012_val_00014707.JPEG:n01860187 +ILSVRC2012_val_00014708.JPEG:n02328150 +ILSVRC2012_val_00014709.JPEG:n02777292 +ILSVRC2012_val_00014710.JPEG:n02112018 +ILSVRC2012_val_00014711.JPEG:n02113978 +ILSVRC2012_val_00014712.JPEG:n02033041 +ILSVRC2012_val_00014713.JPEG:n07871810 +ILSVRC2012_val_00014714.JPEG:n10148035 +ILSVRC2012_val_00014715.JPEG:n01981276 +ILSVRC2012_val_00014716.JPEG:n07860988 +ILSVRC2012_val_00014717.JPEG:n03492542 +ILSVRC2012_val_00014718.JPEG:n04005630 +ILSVRC2012_val_00014719.JPEG:n02093428 +ILSVRC2012_val_00014720.JPEG:n04355933 +ILSVRC2012_val_00014721.JPEG:n02108089 +ILSVRC2012_val_00014722.JPEG:n03841143 +ILSVRC2012_val_00014723.JPEG:n02704792 +ILSVRC2012_val_00014724.JPEG:n02277742 +ILSVRC2012_val_00014725.JPEG:n03874599 +ILSVRC2012_val_00014726.JPEG:n04371774 +ILSVRC2012_val_00014727.JPEG:n01775062 +ILSVRC2012_val_00014728.JPEG:n03461385 +ILSVRC2012_val_00014729.JPEG:n02096585 +ILSVRC2012_val_00014730.JPEG:n02093754 +ILSVRC2012_val_00014731.JPEG:n02011460 +ILSVRC2012_val_00014732.JPEG:n02814533 +ILSVRC2012_val_00014733.JPEG:n02787622 +ILSVRC2012_val_00014734.JPEG:n02114367 +ILSVRC2012_val_00014735.JPEG:n01641577 +ILSVRC2012_val_00014736.JPEG:n03992509 +ILSVRC2012_val_00014737.JPEG:n04265275 +ILSVRC2012_val_00014738.JPEG:n02096051 +ILSVRC2012_val_00014739.JPEG:n07745940 +ILSVRC2012_val_00014740.JPEG:n02422106 +ILSVRC2012_val_00014741.JPEG:n01496331 +ILSVRC2012_val_00014742.JPEG:n03188531 +ILSVRC2012_val_00014743.JPEG:n07614500 +ILSVRC2012_val_00014744.JPEG:n02101006 +ILSVRC2012_val_00014745.JPEG:n02101006 +ILSVRC2012_val_00014746.JPEG:n13040303 +ILSVRC2012_val_00014747.JPEG:n02085936 +ILSVRC2012_val_00014748.JPEG:n03961711 +ILSVRC2012_val_00014749.JPEG:n02093991 +ILSVRC2012_val_00014750.JPEG:n07714571 +ILSVRC2012_val_00014751.JPEG:n01986214 +ILSVRC2012_val_00014752.JPEG:n01669191 +ILSVRC2012_val_00014753.JPEG:n01984695 +ILSVRC2012_val_00014754.JPEG:n03297495 +ILSVRC2012_val_00014755.JPEG:n02108422 +ILSVRC2012_val_00014756.JPEG:n03249569 +ILSVRC2012_val_00014757.JPEG:n04398044 +ILSVRC2012_val_00014758.JPEG:n03775546 +ILSVRC2012_val_00014759.JPEG:n01986214 +ILSVRC2012_val_00014760.JPEG:n04579432 +ILSVRC2012_val_00014761.JPEG:n07714571 +ILSVRC2012_val_00014762.JPEG:n01945685 +ILSVRC2012_val_00014763.JPEG:n02640242 +ILSVRC2012_val_00014764.JPEG:n06785654 +ILSVRC2012_val_00014765.JPEG:n04116512 +ILSVRC2012_val_00014766.JPEG:n02099429 +ILSVRC2012_val_00014767.JPEG:n09229709 +ILSVRC2012_val_00014768.JPEG:n01682714 +ILSVRC2012_val_00014769.JPEG:n01749939 +ILSVRC2012_val_00014770.JPEG:n02007558 +ILSVRC2012_val_00014771.JPEG:n01498041 +ILSVRC2012_val_00014772.JPEG:n04507155 +ILSVRC2012_val_00014773.JPEG:n02124075 +ILSVRC2012_val_00014774.JPEG:n02101006 +ILSVRC2012_val_00014775.JPEG:n02104029 +ILSVRC2012_val_00014776.JPEG:n02676566 +ILSVRC2012_val_00014777.JPEG:n02606052 +ILSVRC2012_val_00014778.JPEG:n04238763 +ILSVRC2012_val_00014779.JPEG:n02101388 +ILSVRC2012_val_00014780.JPEG:n02107312 +ILSVRC2012_val_00014781.JPEG:n03347037 +ILSVRC2012_val_00014782.JPEG:n02493509 +ILSVRC2012_val_00014783.JPEG:n02396427 +ILSVRC2012_val_00014784.JPEG:n04065272 +ILSVRC2012_val_00014785.JPEG:n03840681 +ILSVRC2012_val_00014786.JPEG:n04515003 +ILSVRC2012_val_00014787.JPEG:n02091635 +ILSVRC2012_val_00014788.JPEG:n02325366 +ILSVRC2012_val_00014789.JPEG:n04033901 +ILSVRC2012_val_00014790.JPEG:n01675722 +ILSVRC2012_val_00014791.JPEG:n03788365 +ILSVRC2012_val_00014792.JPEG:n13037406 +ILSVRC2012_val_00014793.JPEG:n03527444 +ILSVRC2012_val_00014794.JPEG:n01695060 +ILSVRC2012_val_00014795.JPEG:n04328186 +ILSVRC2012_val_00014796.JPEG:n07590611 +ILSVRC2012_val_00014797.JPEG:n01728572 +ILSVRC2012_val_00014798.JPEG:n02119022 +ILSVRC2012_val_00014799.JPEG:n02974003 +ILSVRC2012_val_00014800.JPEG:n02410509 +ILSVRC2012_val_00014801.JPEG:n07892512 +ILSVRC2012_val_00014802.JPEG:n07730033 +ILSVRC2012_val_00014803.JPEG:n04330267 +ILSVRC2012_val_00014804.JPEG:n03868863 +ILSVRC2012_val_00014805.JPEG:n02018207 +ILSVRC2012_val_00014806.JPEG:n02500267 +ILSVRC2012_val_00014807.JPEG:n02980441 +ILSVRC2012_val_00014808.JPEG:n01843065 +ILSVRC2012_val_00014809.JPEG:n02093859 +ILSVRC2012_val_00014810.JPEG:n02094114 +ILSVRC2012_val_00014811.JPEG:n07768694 +ILSVRC2012_val_00014812.JPEG:n04154565 +ILSVRC2012_val_00014813.JPEG:n02123394 +ILSVRC2012_val_00014814.JPEG:n03843555 +ILSVRC2012_val_00014815.JPEG:n02123159 +ILSVRC2012_val_00014816.JPEG:n02107574 +ILSVRC2012_val_00014817.JPEG:n01795545 +ILSVRC2012_val_00014818.JPEG:n02917067 +ILSVRC2012_val_00014819.JPEG:n02071294 +ILSVRC2012_val_00014820.JPEG:n03895866 +ILSVRC2012_val_00014821.JPEG:n03179701 +ILSVRC2012_val_00014822.JPEG:n03950228 +ILSVRC2012_val_00014823.JPEG:n04259630 +ILSVRC2012_val_00014824.JPEG:n02165105 +ILSVRC2012_val_00014825.JPEG:n02120079 +ILSVRC2012_val_00014826.JPEG:n02804610 +ILSVRC2012_val_00014827.JPEG:n02279972 +ILSVRC2012_val_00014828.JPEG:n01728920 +ILSVRC2012_val_00014829.JPEG:n02978881 +ILSVRC2012_val_00014830.JPEG:n03710637 +ILSVRC2012_val_00014831.JPEG:n01872401 +ILSVRC2012_val_00014832.JPEG:n03160309 +ILSVRC2012_val_00014833.JPEG:n02442845 +ILSVRC2012_val_00014834.JPEG:n09256479 +ILSVRC2012_val_00014835.JPEG:n02950826 +ILSVRC2012_val_00014836.JPEG:n02841315 +ILSVRC2012_val_00014837.JPEG:n04357314 +ILSVRC2012_val_00014838.JPEG:n02865351 +ILSVRC2012_val_00014839.JPEG:n04111531 +ILSVRC2012_val_00014840.JPEG:n07747607 +ILSVRC2012_val_00014841.JPEG:n03594945 +ILSVRC2012_val_00014842.JPEG:n03763968 +ILSVRC2012_val_00014843.JPEG:n04606251 +ILSVRC2012_val_00014844.JPEG:n03895866 +ILSVRC2012_val_00014845.JPEG:n02113978 +ILSVRC2012_val_00014846.JPEG:n04554684 +ILSVRC2012_val_00014847.JPEG:n04344873 +ILSVRC2012_val_00014848.JPEG:n04254120 +ILSVRC2012_val_00014849.JPEG:n01740131 +ILSVRC2012_val_00014850.JPEG:n03976467 +ILSVRC2012_val_00014851.JPEG:n07753275 +ILSVRC2012_val_00014852.JPEG:n02443484 +ILSVRC2012_val_00014853.JPEG:n02939185 +ILSVRC2012_val_00014854.JPEG:n02977058 +ILSVRC2012_val_00014855.JPEG:n13037406 +ILSVRC2012_val_00014856.JPEG:n07747607 +ILSVRC2012_val_00014857.JPEG:n04467665 +ILSVRC2012_val_00014858.JPEG:n01784675 +ILSVRC2012_val_00014859.JPEG:n04536866 +ILSVRC2012_val_00014860.JPEG:n02123159 +ILSVRC2012_val_00014861.JPEG:n02119789 +ILSVRC2012_val_00014862.JPEG:n04548362 +ILSVRC2012_val_00014863.JPEG:n02111129 +ILSVRC2012_val_00014864.JPEG:n06794110 +ILSVRC2012_val_00014865.JPEG:n04239074 +ILSVRC2012_val_00014866.JPEG:n03733805 +ILSVRC2012_val_00014867.JPEG:n02088466 +ILSVRC2012_val_00014868.JPEG:n03764736 +ILSVRC2012_val_00014869.JPEG:n01914609 +ILSVRC2012_val_00014870.JPEG:n02105505 +ILSVRC2012_val_00014871.JPEG:n02412080 +ILSVRC2012_val_00014872.JPEG:n04254680 +ILSVRC2012_val_00014873.JPEG:n04523525 +ILSVRC2012_val_00014874.JPEG:n07697537 +ILSVRC2012_val_00014875.JPEG:n01728920 +ILSVRC2012_val_00014876.JPEG:n02794156 +ILSVRC2012_val_00014877.JPEG:n02113978 +ILSVRC2012_val_00014878.JPEG:n13040303 +ILSVRC2012_val_00014879.JPEG:n01514859 +ILSVRC2012_val_00014880.JPEG:n04398044 +ILSVRC2012_val_00014881.JPEG:n02364673 +ILSVRC2012_val_00014882.JPEG:n01924916 +ILSVRC2012_val_00014883.JPEG:n02007558 +ILSVRC2012_val_00014884.JPEG:n03803284 +ILSVRC2012_val_00014885.JPEG:n02795169 +ILSVRC2012_val_00014886.JPEG:n03916031 +ILSVRC2012_val_00014887.JPEG:n02088238 +ILSVRC2012_val_00014888.JPEG:n02086646 +ILSVRC2012_val_00014889.JPEG:n03063689 +ILSVRC2012_val_00014890.JPEG:n01806143 +ILSVRC2012_val_00014891.JPEG:n04366367 +ILSVRC2012_val_00014892.JPEG:n03109150 +ILSVRC2012_val_00014893.JPEG:n04523525 +ILSVRC2012_val_00014894.JPEG:n04208210 +ILSVRC2012_val_00014895.JPEG:n01978287 +ILSVRC2012_val_00014896.JPEG:n03272010 +ILSVRC2012_val_00014897.JPEG:n03146219 +ILSVRC2012_val_00014898.JPEG:n03933933 +ILSVRC2012_val_00014899.JPEG:n04525305 +ILSVRC2012_val_00014900.JPEG:n03124043 +ILSVRC2012_val_00014901.JPEG:n02510455 +ILSVRC2012_val_00014902.JPEG:n01687978 +ILSVRC2012_val_00014903.JPEG:n01824575 +ILSVRC2012_val_00014904.JPEG:n04613696 +ILSVRC2012_val_00014905.JPEG:n06359193 +ILSVRC2012_val_00014906.JPEG:n03110669 +ILSVRC2012_val_00014907.JPEG:n03388183 +ILSVRC2012_val_00014908.JPEG:n03691459 +ILSVRC2012_val_00014909.JPEG:n02280649 +ILSVRC2012_val_00014910.JPEG:n03133878 +ILSVRC2012_val_00014911.JPEG:n02085782 +ILSVRC2012_val_00014912.JPEG:n02087046 +ILSVRC2012_val_00014913.JPEG:n02090721 +ILSVRC2012_val_00014914.JPEG:n02497673 +ILSVRC2012_val_00014915.JPEG:n04344873 +ILSVRC2012_val_00014916.JPEG:n04330267 +ILSVRC2012_val_00014917.JPEG:n01514859 +ILSVRC2012_val_00014918.JPEG:n02488702 +ILSVRC2012_val_00014919.JPEG:n04525038 +ILSVRC2012_val_00014920.JPEG:n07711569 +ILSVRC2012_val_00014921.JPEG:n01978455 +ILSVRC2012_val_00014922.JPEG:n01768244 +ILSVRC2012_val_00014923.JPEG:n02105855 +ILSVRC2012_val_00014924.JPEG:n04604644 +ILSVRC2012_val_00014925.JPEG:n02281406 +ILSVRC2012_val_00014926.JPEG:n01739381 +ILSVRC2012_val_00014927.JPEG:n01693334 +ILSVRC2012_val_00014928.JPEG:n02113978 +ILSVRC2012_val_00014929.JPEG:n07749582 +ILSVRC2012_val_00014930.JPEG:n03786901 +ILSVRC2012_val_00014931.JPEG:n01883070 +ILSVRC2012_val_00014932.JPEG:n09246464 +ILSVRC2012_val_00014933.JPEG:n03841143 +ILSVRC2012_val_00014934.JPEG:n03482405 +ILSVRC2012_val_00014935.JPEG:n12998815 +ILSVRC2012_val_00014936.JPEG:n03938244 +ILSVRC2012_val_00014937.JPEG:n04238763 +ILSVRC2012_val_00014938.JPEG:n03929855 +ILSVRC2012_val_00014939.JPEG:n02892201 +ILSVRC2012_val_00014940.JPEG:n02486261 +ILSVRC2012_val_00014941.JPEG:n02676566 +ILSVRC2012_val_00014942.JPEG:n01843065 +ILSVRC2012_val_00014943.JPEG:n01728920 +ILSVRC2012_val_00014944.JPEG:n03379051 +ILSVRC2012_val_00014945.JPEG:n02823750 +ILSVRC2012_val_00014946.JPEG:n02776631 +ILSVRC2012_val_00014947.JPEG:n02488291 +ILSVRC2012_val_00014948.JPEG:n02317335 +ILSVRC2012_val_00014949.JPEG:n02002724 +ILSVRC2012_val_00014950.JPEG:n01755581 +ILSVRC2012_val_00014951.JPEG:n03110669 +ILSVRC2012_val_00014952.JPEG:n04019541 +ILSVRC2012_val_00014953.JPEG:n03095699 +ILSVRC2012_val_00014954.JPEG:n04004767 +ILSVRC2012_val_00014955.JPEG:n03877845 +ILSVRC2012_val_00014956.JPEG:n02120505 +ILSVRC2012_val_00014957.JPEG:n02113624 +ILSVRC2012_val_00014958.JPEG:n07695742 +ILSVRC2012_val_00014959.JPEG:n03127747 +ILSVRC2012_val_00014960.JPEG:n03041632 +ILSVRC2012_val_00014961.JPEG:n01744401 +ILSVRC2012_val_00014962.JPEG:n02098286 +ILSVRC2012_val_00014963.JPEG:n02100735 +ILSVRC2012_val_00014964.JPEG:n02264363 +ILSVRC2012_val_00014965.JPEG:n04456115 +ILSVRC2012_val_00014966.JPEG:n02219486 +ILSVRC2012_val_00014967.JPEG:n02129165 +ILSVRC2012_val_00014968.JPEG:n04275548 +ILSVRC2012_val_00014969.JPEG:n03874599 +ILSVRC2012_val_00014970.JPEG:n03706229 +ILSVRC2012_val_00014971.JPEG:n01770081 +ILSVRC2012_val_00014972.JPEG:n02988304 +ILSVRC2012_val_00014973.JPEG:n02105505 +ILSVRC2012_val_00014974.JPEG:n02130308 +ILSVRC2012_val_00014975.JPEG:n02113799 +ILSVRC2012_val_00014976.JPEG:n06596364 +ILSVRC2012_val_00014977.JPEG:n02028035 +ILSVRC2012_val_00014978.JPEG:n01784675 +ILSVRC2012_val_00014979.JPEG:n04266014 +ILSVRC2012_val_00014980.JPEG:n02422106 +ILSVRC2012_val_00014981.JPEG:n03271574 +ILSVRC2012_val_00014982.JPEG:n01622779 +ILSVRC2012_val_00014983.JPEG:n04229816 +ILSVRC2012_val_00014984.JPEG:n02988304 +ILSVRC2012_val_00014985.JPEG:n02977058 +ILSVRC2012_val_00014986.JPEG:n03594734 +ILSVRC2012_val_00014987.JPEG:n03196217 +ILSVRC2012_val_00014988.JPEG:n04008634 +ILSVRC2012_val_00014989.JPEG:n03947888 +ILSVRC2012_val_00014990.JPEG:n03032252 +ILSVRC2012_val_00014991.JPEG:n02037110 +ILSVRC2012_val_00014992.JPEG:n03424325 +ILSVRC2012_val_00014993.JPEG:n03873416 +ILSVRC2012_val_00014994.JPEG:n03379051 +ILSVRC2012_val_00014995.JPEG:n02096437 +ILSVRC2012_val_00014996.JPEG:n03887697 +ILSVRC2012_val_00014997.JPEG:n04154565 +ILSVRC2012_val_00014998.JPEG:n03803284 +ILSVRC2012_val_00014999.JPEG:n06794110 +ILSVRC2012_val_00015000.JPEG:n03956157 +ILSVRC2012_val_00015001.JPEG:n03297495 +ILSVRC2012_val_00015002.JPEG:n03444034 +ILSVRC2012_val_00015003.JPEG:n09256479 +ILSVRC2012_val_00015004.JPEG:n02317335 +ILSVRC2012_val_00015005.JPEG:n03871628 +ILSVRC2012_val_00015006.JPEG:n04192698 +ILSVRC2012_val_00015007.JPEG:n07873807 +ILSVRC2012_val_00015008.JPEG:n02793495 +ILSVRC2012_val_00015009.JPEG:n03764736 +ILSVRC2012_val_00015010.JPEG:n02483362 +ILSVRC2012_val_00015011.JPEG:n01773797 +ILSVRC2012_val_00015012.JPEG:n03788195 +ILSVRC2012_val_00015013.JPEG:n03032252 +ILSVRC2012_val_00015014.JPEG:n04311174 +ILSVRC2012_val_00015015.JPEG:n02111889 +ILSVRC2012_val_00015016.JPEG:n03970156 +ILSVRC2012_val_00015017.JPEG:n04447861 +ILSVRC2012_val_00015018.JPEG:n02018795 +ILSVRC2012_val_00015019.JPEG:n03666591 +ILSVRC2012_val_00015020.JPEG:n03314780 +ILSVRC2012_val_00015021.JPEG:n02229544 +ILSVRC2012_val_00015022.JPEG:n02172182 +ILSVRC2012_val_00015023.JPEG:n02486410 +ILSVRC2012_val_00015024.JPEG:n02607072 +ILSVRC2012_val_00015025.JPEG:n02276258 +ILSVRC2012_val_00015026.JPEG:n04254777 +ILSVRC2012_val_00015027.JPEG:n02403003 +ILSVRC2012_val_00015028.JPEG:n02094114 +ILSVRC2012_val_00015029.JPEG:n09246464 +ILSVRC2012_val_00015030.JPEG:n02114367 +ILSVRC2012_val_00015031.JPEG:n03788365 +ILSVRC2012_val_00015032.JPEG:n03297495 +ILSVRC2012_val_00015033.JPEG:n02492660 +ILSVRC2012_val_00015034.JPEG:n04326547 +ILSVRC2012_val_00015035.JPEG:n03201208 +ILSVRC2012_val_00015036.JPEG:n04286575 +ILSVRC2012_val_00015037.JPEG:n03492542 +ILSVRC2012_val_00015038.JPEG:n03877472 +ILSVRC2012_val_00015039.JPEG:n01910747 +ILSVRC2012_val_00015040.JPEG:n01608432 +ILSVRC2012_val_00015041.JPEG:n02490219 +ILSVRC2012_val_00015042.JPEG:n03710637 +ILSVRC2012_val_00015043.JPEG:n04344873 +ILSVRC2012_val_00015044.JPEG:n02951358 +ILSVRC2012_val_00015045.JPEG:n01498041 +ILSVRC2012_val_00015046.JPEG:n01729322 +ILSVRC2012_val_00015047.JPEG:n04409515 +ILSVRC2012_val_00015048.JPEG:n04146614 +ILSVRC2012_val_00015049.JPEG:n03873416 +ILSVRC2012_val_00015050.JPEG:n02090721 +ILSVRC2012_val_00015051.JPEG:n04081281 +ILSVRC2012_val_00015052.JPEG:n03976467 +ILSVRC2012_val_00015053.JPEG:n02837789 +ILSVRC2012_val_00015054.JPEG:n04409515 +ILSVRC2012_val_00015055.JPEG:n03759954 +ILSVRC2012_val_00015056.JPEG:n02168699 +ILSVRC2012_val_00015057.JPEG:n03127925 +ILSVRC2012_val_00015058.JPEG:n03970156 +ILSVRC2012_val_00015059.JPEG:n01665541 +ILSVRC2012_val_00015060.JPEG:n03160309 +ILSVRC2012_val_00015061.JPEG:n04251144 +ILSVRC2012_val_00015062.JPEG:n04311174 +ILSVRC2012_val_00015063.JPEG:n02098413 +ILSVRC2012_val_00015064.JPEG:n02480855 +ILSVRC2012_val_00015065.JPEG:n01773549 +ILSVRC2012_val_00015066.JPEG:n02489166 +ILSVRC2012_val_00015067.JPEG:n03494278 +ILSVRC2012_val_00015068.JPEG:n02229544 +ILSVRC2012_val_00015069.JPEG:n01729977 +ILSVRC2012_val_00015070.JPEG:n04552348 +ILSVRC2012_val_00015071.JPEG:n04033995 +ILSVRC2012_val_00015072.JPEG:n01882714 +ILSVRC2012_val_00015073.JPEG:n04366367 +ILSVRC2012_val_00015074.JPEG:n03271574 +ILSVRC2012_val_00015075.JPEG:n03666591 +ILSVRC2012_val_00015076.JPEG:n02093428 +ILSVRC2012_val_00015077.JPEG:n02791124 +ILSVRC2012_val_00015078.JPEG:n03384352 +ILSVRC2012_val_00015079.JPEG:n03498962 +ILSVRC2012_val_00015080.JPEG:n03709823 +ILSVRC2012_val_00015081.JPEG:n02422699 +ILSVRC2012_val_00015082.JPEG:n02085782 +ILSVRC2012_val_00015083.JPEG:n04133789 +ILSVRC2012_val_00015084.JPEG:n02486261 +ILSVRC2012_val_00015085.JPEG:n12985857 +ILSVRC2012_val_00015086.JPEG:n04372370 +ILSVRC2012_val_00015087.JPEG:n03857828 +ILSVRC2012_val_00015088.JPEG:n04367480 +ILSVRC2012_val_00015089.JPEG:n04612504 +ILSVRC2012_val_00015090.JPEG:n04399382 +ILSVRC2012_val_00015091.JPEG:n01632458 +ILSVRC2012_val_00015092.JPEG:n03717622 +ILSVRC2012_val_00015093.JPEG:n02514041 +ILSVRC2012_val_00015094.JPEG:n02018207 +ILSVRC2012_val_00015095.JPEG:n07615774 +ILSVRC2012_val_00015096.JPEG:n02098413 +ILSVRC2012_val_00015097.JPEG:n03691459 +ILSVRC2012_val_00015098.JPEG:n02108915 +ILSVRC2012_val_00015099.JPEG:n07920052 +ILSVRC2012_val_00015100.JPEG:n04228054 +ILSVRC2012_val_00015101.JPEG:n04493381 +ILSVRC2012_val_00015102.JPEG:n04081281 +ILSVRC2012_val_00015103.JPEG:n03832673 +ILSVRC2012_val_00015104.JPEG:n13052670 +ILSVRC2012_val_00015105.JPEG:n04584207 +ILSVRC2012_val_00015106.JPEG:n04252225 +ILSVRC2012_val_00015107.JPEG:n01608432 +ILSVRC2012_val_00015108.JPEG:n02708093 +ILSVRC2012_val_00015109.JPEG:n04398044 +ILSVRC2012_val_00015110.JPEG:n02087046 +ILSVRC2012_val_00015111.JPEG:n04599235 +ILSVRC2012_val_00015112.JPEG:n02177972 +ILSVRC2012_val_00015113.JPEG:n02326432 +ILSVRC2012_val_00015114.JPEG:n02490219 +ILSVRC2012_val_00015115.JPEG:n03761084 +ILSVRC2012_val_00015116.JPEG:n02101556 +ILSVRC2012_val_00015117.JPEG:n04599235 +ILSVRC2012_val_00015118.JPEG:n04467665 +ILSVRC2012_val_00015119.JPEG:n02097658 +ILSVRC2012_val_00015120.JPEG:n01978287 +ILSVRC2012_val_00015121.JPEG:n04612504 +ILSVRC2012_val_00015122.JPEG:n02397096 +ILSVRC2012_val_00015123.JPEG:n03018349 +ILSVRC2012_val_00015124.JPEG:n02391049 +ILSVRC2012_val_00015125.JPEG:n07584110 +ILSVRC2012_val_00015126.JPEG:n02457408 +ILSVRC2012_val_00015127.JPEG:n01776313 +ILSVRC2012_val_00015128.JPEG:n02120079 +ILSVRC2012_val_00015129.JPEG:n02727426 +ILSVRC2012_val_00015130.JPEG:n02791270 +ILSVRC2012_val_00015131.JPEG:n04590129 +ILSVRC2012_val_00015132.JPEG:n02058221 +ILSVRC2012_val_00015133.JPEG:n03599486 +ILSVRC2012_val_00015134.JPEG:n03788365 +ILSVRC2012_val_00015135.JPEG:n02098105 +ILSVRC2012_val_00015136.JPEG:n02097047 +ILSVRC2012_val_00015137.JPEG:n03794056 +ILSVRC2012_val_00015138.JPEG:n02966193 +ILSVRC2012_val_00015139.JPEG:n01494475 +ILSVRC2012_val_00015140.JPEG:n02514041 +ILSVRC2012_val_00015141.JPEG:n01773157 +ILSVRC2012_val_00015142.JPEG:n07613480 +ILSVRC2012_val_00015143.JPEG:n09332890 +ILSVRC2012_val_00015144.JPEG:n02086910 +ILSVRC2012_val_00015145.JPEG:n02071294 +ILSVRC2012_val_00015146.JPEG:n02105412 +ILSVRC2012_val_00015147.JPEG:n02966193 +ILSVRC2012_val_00015148.JPEG:n02481823 +ILSVRC2012_val_00015149.JPEG:n04228054 +ILSVRC2012_val_00015150.JPEG:n02825657 +ILSVRC2012_val_00015151.JPEG:n03775071 +ILSVRC2012_val_00015152.JPEG:n02096177 +ILSVRC2012_val_00015153.JPEG:n02328150 +ILSVRC2012_val_00015154.JPEG:n01768244 +ILSVRC2012_val_00015155.JPEG:n03028079 +ILSVRC2012_val_00015156.JPEG:n03534580 +ILSVRC2012_val_00015157.JPEG:n01484850 +ILSVRC2012_val_00015158.JPEG:n09428293 +ILSVRC2012_val_00015159.JPEG:n03788365 +ILSVRC2012_val_00015160.JPEG:n02106550 +ILSVRC2012_val_00015161.JPEG:n03782006 +ILSVRC2012_val_00015162.JPEG:n04258138 +ILSVRC2012_val_00015163.JPEG:n03710637 +ILSVRC2012_val_00015164.JPEG:n02097298 +ILSVRC2012_val_00015165.JPEG:n03721384 +ILSVRC2012_val_00015166.JPEG:n02391049 +ILSVRC2012_val_00015167.JPEG:n02013706 +ILSVRC2012_val_00015168.JPEG:n02840245 +ILSVRC2012_val_00015169.JPEG:n03249569 +ILSVRC2012_val_00015170.JPEG:n02454379 +ILSVRC2012_val_00015171.JPEG:n02865351 +ILSVRC2012_val_00015172.JPEG:n02206856 +ILSVRC2012_val_00015173.JPEG:n02093991 +ILSVRC2012_val_00015174.JPEG:n01877812 +ILSVRC2012_val_00015175.JPEG:n03485407 +ILSVRC2012_val_00015176.JPEG:n02101388 +ILSVRC2012_val_00015177.JPEG:n03014705 +ILSVRC2012_val_00015178.JPEG:n04456115 +ILSVRC2012_val_00015179.JPEG:n03976657 +ILSVRC2012_val_00015180.JPEG:n03188531 +ILSVRC2012_val_00015181.JPEG:n02342885 +ILSVRC2012_val_00015182.JPEG:n02096437 +ILSVRC2012_val_00015183.JPEG:n02102318 +ILSVRC2012_val_00015184.JPEG:n03376595 +ILSVRC2012_val_00015185.JPEG:n03271574 +ILSVRC2012_val_00015186.JPEG:n02177972 +ILSVRC2012_val_00015187.JPEG:n03594945 +ILSVRC2012_val_00015188.JPEG:n03126707 +ILSVRC2012_val_00015189.JPEG:n02099712 +ILSVRC2012_val_00015190.JPEG:n01692333 +ILSVRC2012_val_00015191.JPEG:n02966687 +ILSVRC2012_val_00015192.JPEG:n03930313 +ILSVRC2012_val_00015193.JPEG:n01667778 +ILSVRC2012_val_00015194.JPEG:n07716906 +ILSVRC2012_val_00015195.JPEG:n01580077 +ILSVRC2012_val_00015196.JPEG:n03804744 +ILSVRC2012_val_00015197.JPEG:n02111277 +ILSVRC2012_val_00015198.JPEG:n03100240 +ILSVRC2012_val_00015199.JPEG:n04548280 +ILSVRC2012_val_00015200.JPEG:n02814533 +ILSVRC2012_val_00015201.JPEG:n04204347 +ILSVRC2012_val_00015202.JPEG:n04141327 +ILSVRC2012_val_00015203.JPEG:n02066245 +ILSVRC2012_val_00015204.JPEG:n02096585 +ILSVRC2012_val_00015205.JPEG:n02102480 +ILSVRC2012_val_00015206.JPEG:n03125729 +ILSVRC2012_val_00015207.JPEG:n03272010 +ILSVRC2012_val_00015208.JPEG:n03980874 +ILSVRC2012_val_00015209.JPEG:n07753592 +ILSVRC2012_val_00015210.JPEG:n02105412 +ILSVRC2012_val_00015211.JPEG:n02443114 +ILSVRC2012_val_00015212.JPEG:n04579432 +ILSVRC2012_val_00015213.JPEG:n02101556 +ILSVRC2012_val_00015214.JPEG:n03995372 +ILSVRC2012_val_00015215.JPEG:n02950826 +ILSVRC2012_val_00015216.JPEG:n01534433 +ILSVRC2012_val_00015217.JPEG:n02088238 +ILSVRC2012_val_00015218.JPEG:n07715103 +ILSVRC2012_val_00015219.JPEG:n02795169 +ILSVRC2012_val_00015220.JPEG:n01484850 +ILSVRC2012_val_00015221.JPEG:n01753488 +ILSVRC2012_val_00015222.JPEG:n02607072 +ILSVRC2012_val_00015223.JPEG:n01530575 +ILSVRC2012_val_00015224.JPEG:n01692333 +ILSVRC2012_val_00015225.JPEG:n04153751 +ILSVRC2012_val_00015226.JPEG:n02111500 +ILSVRC2012_val_00015227.JPEG:n03131574 +ILSVRC2012_val_00015228.JPEG:n03803284 +ILSVRC2012_val_00015229.JPEG:n02437312 +ILSVRC2012_val_00015230.JPEG:n02974003 +ILSVRC2012_val_00015231.JPEG:n02776631 +ILSVRC2012_val_00015232.JPEG:n04125021 +ILSVRC2012_val_00015233.JPEG:n09428293 +ILSVRC2012_val_00015234.JPEG:n02843684 +ILSVRC2012_val_00015235.JPEG:n03047690 +ILSVRC2012_val_00015236.JPEG:n02417914 +ILSVRC2012_val_00015237.JPEG:n03998194 +ILSVRC2012_val_00015238.JPEG:n03110669 +ILSVRC2012_val_00015239.JPEG:n02445715 +ILSVRC2012_val_00015240.JPEG:n04525305 +ILSVRC2012_val_00015241.JPEG:n03998194 +ILSVRC2012_val_00015242.JPEG:n01514668 +ILSVRC2012_val_00015243.JPEG:n02321529 +ILSVRC2012_val_00015244.JPEG:n02088466 +ILSVRC2012_val_00015245.JPEG:n01644373 +ILSVRC2012_val_00015246.JPEG:n07714571 +ILSVRC2012_val_00015247.JPEG:n04357314 +ILSVRC2012_val_00015248.JPEG:n03991062 +ILSVRC2012_val_00015249.JPEG:n02088094 +ILSVRC2012_val_00015250.JPEG:n02687172 +ILSVRC2012_val_00015251.JPEG:n02110185 +ILSVRC2012_val_00015252.JPEG:n02089078 +ILSVRC2012_val_00015253.JPEG:n09468604 +ILSVRC2012_val_00015254.JPEG:n02408429 +ILSVRC2012_val_00015255.JPEG:n04389033 +ILSVRC2012_val_00015256.JPEG:n03706229 +ILSVRC2012_val_00015257.JPEG:n02488702 +ILSVRC2012_val_00015258.JPEG:n03992509 +ILSVRC2012_val_00015259.JPEG:n02417914 +ILSVRC2012_val_00015260.JPEG:n04086273 +ILSVRC2012_val_00015261.JPEG:n07613480 +ILSVRC2012_val_00015262.JPEG:n04270147 +ILSVRC2012_val_00015263.JPEG:n03887697 +ILSVRC2012_val_00015264.JPEG:n01601694 +ILSVRC2012_val_00015265.JPEG:n02123159 +ILSVRC2012_val_00015266.JPEG:n01518878 +ILSVRC2012_val_00015267.JPEG:n07836838 +ILSVRC2012_val_00015268.JPEG:n04443257 +ILSVRC2012_val_00015269.JPEG:n01592084 +ILSVRC2012_val_00015270.JPEG:n03109150 +ILSVRC2012_val_00015271.JPEG:n02264363 +ILSVRC2012_val_00015272.JPEG:n02808304 +ILSVRC2012_val_00015273.JPEG:n04252225 +ILSVRC2012_val_00015274.JPEG:n01630670 +ILSVRC2012_val_00015275.JPEG:n04507155 +ILSVRC2012_val_00015276.JPEG:n03047690 +ILSVRC2012_val_00015277.JPEG:n03344393 +ILSVRC2012_val_00015278.JPEG:n02981792 +ILSVRC2012_val_00015279.JPEG:n03680355 +ILSVRC2012_val_00015280.JPEG:n07579787 +ILSVRC2012_val_00015281.JPEG:n02526121 +ILSVRC2012_val_00015282.JPEG:n01984695 +ILSVRC2012_val_00015283.JPEG:n04485082 +ILSVRC2012_val_00015284.JPEG:n03814639 +ILSVRC2012_val_00015285.JPEG:n02977058 +ILSVRC2012_val_00015286.JPEG:n03866082 +ILSVRC2012_val_00015287.JPEG:n04404412 +ILSVRC2012_val_00015288.JPEG:n04116512 +ILSVRC2012_val_00015289.JPEG:n03100240 +ILSVRC2012_val_00015290.JPEG:n03127925 +ILSVRC2012_val_00015291.JPEG:n01847000 +ILSVRC2012_val_00015292.JPEG:n02051845 +ILSVRC2012_val_00015293.JPEG:n02177972 +ILSVRC2012_val_00015294.JPEG:n02106030 +ILSVRC2012_val_00015295.JPEG:n03770679 +ILSVRC2012_val_00015296.JPEG:n03535780 +ILSVRC2012_val_00015297.JPEG:n03676483 +ILSVRC2012_val_00015298.JPEG:n01843383 +ILSVRC2012_val_00015299.JPEG:n01873310 +ILSVRC2012_val_00015300.JPEG:n02085936 +ILSVRC2012_val_00015301.JPEG:n02328150 +ILSVRC2012_val_00015302.JPEG:n03089624 +ILSVRC2012_val_00015303.JPEG:n02102318 +ILSVRC2012_val_00015304.JPEG:n02500267 +ILSVRC2012_val_00015305.JPEG:n04040759 +ILSVRC2012_val_00015306.JPEG:n04552348 +ILSVRC2012_val_00015307.JPEG:n02101006 +ILSVRC2012_val_00015308.JPEG:n07749582 +ILSVRC2012_val_00015309.JPEG:n03884397 +ILSVRC2012_val_00015310.JPEG:n02111129 +ILSVRC2012_val_00015311.JPEG:n03662601 +ILSVRC2012_val_00015312.JPEG:n03250847 +ILSVRC2012_val_00015313.JPEG:n02129604 +ILSVRC2012_val_00015314.JPEG:n03461385 +ILSVRC2012_val_00015315.JPEG:n03970156 +ILSVRC2012_val_00015316.JPEG:n04317175 +ILSVRC2012_val_00015317.JPEG:n03958227 +ILSVRC2012_val_00015318.JPEG:n07714990 +ILSVRC2012_val_00015319.JPEG:n01980166 +ILSVRC2012_val_00015320.JPEG:n03929660 +ILSVRC2012_val_00015321.JPEG:n03314780 +ILSVRC2012_val_00015322.JPEG:n01855032 +ILSVRC2012_val_00015323.JPEG:n03630383 +ILSVRC2012_val_00015324.JPEG:n01817953 +ILSVRC2012_val_00015325.JPEG:n02095889 +ILSVRC2012_val_00015326.JPEG:n04505470 +ILSVRC2012_val_00015327.JPEG:n02727426 +ILSVRC2012_val_00015328.JPEG:n03598930 +ILSVRC2012_val_00015329.JPEG:n02105855 +ILSVRC2012_val_00015330.JPEG:n02115913 +ILSVRC2012_val_00015331.JPEG:n03110669 +ILSVRC2012_val_00015332.JPEG:n10148035 +ILSVRC2012_val_00015333.JPEG:n02106550 +ILSVRC2012_val_00015334.JPEG:n02086079 +ILSVRC2012_val_00015335.JPEG:n04380533 +ILSVRC2012_val_00015336.JPEG:n10565667 +ILSVRC2012_val_00015337.JPEG:n03249569 +ILSVRC2012_val_00015338.JPEG:n02095889 +ILSVRC2012_val_00015339.JPEG:n02492660 +ILSVRC2012_val_00015340.JPEG:n07873807 +ILSVRC2012_val_00015341.JPEG:n02797295 +ILSVRC2012_val_00015342.JPEG:n04209239 +ILSVRC2012_val_00015343.JPEG:n02786058 +ILSVRC2012_val_00015344.JPEG:n02837789 +ILSVRC2012_val_00015345.JPEG:n02841315 +ILSVRC2012_val_00015346.JPEG:n02704792 +ILSVRC2012_val_00015347.JPEG:n03935335 +ILSVRC2012_val_00015348.JPEG:n04562935 +ILSVRC2012_val_00015349.JPEG:n02099429 +ILSVRC2012_val_00015350.JPEG:n02112137 +ILSVRC2012_val_00015351.JPEG:n03325584 +ILSVRC2012_val_00015352.JPEG:n04442312 +ILSVRC2012_val_00015353.JPEG:n04033995 +ILSVRC2012_val_00015354.JPEG:n07614500 +ILSVRC2012_val_00015355.JPEG:n02108089 +ILSVRC2012_val_00015356.JPEG:n03710721 +ILSVRC2012_val_00015357.JPEG:n03100240 +ILSVRC2012_val_00015358.JPEG:n02093859 +ILSVRC2012_val_00015359.JPEG:n02906734 +ILSVRC2012_val_00015360.JPEG:n04254777 +ILSVRC2012_val_00015361.JPEG:n07871810 +ILSVRC2012_val_00015362.JPEG:n02422106 +ILSVRC2012_val_00015363.JPEG:n04049303 +ILSVRC2012_val_00015364.JPEG:n03961711 +ILSVRC2012_val_00015365.JPEG:n02777292 +ILSVRC2012_val_00015366.JPEG:n04443257 +ILSVRC2012_val_00015367.JPEG:n04597913 +ILSVRC2012_val_00015368.JPEG:n02927161 +ILSVRC2012_val_00015369.JPEG:n03424325 +ILSVRC2012_val_00015370.JPEG:n03032252 +ILSVRC2012_val_00015371.JPEG:n02795169 +ILSVRC2012_val_00015372.JPEG:n02123394 +ILSVRC2012_val_00015373.JPEG:n01498041 +ILSVRC2012_val_00015374.JPEG:n01751748 +ILSVRC2012_val_00015375.JPEG:n03793489 +ILSVRC2012_val_00015376.JPEG:n03345487 +ILSVRC2012_val_00015377.JPEG:n02091635 +ILSVRC2012_val_00015378.JPEG:n02123159 +ILSVRC2012_val_00015379.JPEG:n02107142 +ILSVRC2012_val_00015380.JPEG:n02484975 +ILSVRC2012_val_00015381.JPEG:n03666591 +ILSVRC2012_val_00015382.JPEG:n03085013 +ILSVRC2012_val_00015383.JPEG:n04325704 +ILSVRC2012_val_00015384.JPEG:n03208938 +ILSVRC2012_val_00015385.JPEG:n04562935 +ILSVRC2012_val_00015386.JPEG:n04152593 +ILSVRC2012_val_00015387.JPEG:n09472597 +ILSVRC2012_val_00015388.JPEG:n07875152 +ILSVRC2012_val_00015389.JPEG:n04597913 +ILSVRC2012_val_00015390.JPEG:n04099969 +ILSVRC2012_val_00015391.JPEG:n03976657 +ILSVRC2012_val_00015392.JPEG:n02028035 +ILSVRC2012_val_00015393.JPEG:n03796401 +ILSVRC2012_val_00015394.JPEG:n02917067 +ILSVRC2012_val_00015395.JPEG:n02110958 +ILSVRC2012_val_00015396.JPEG:n02730930 +ILSVRC2012_val_00015397.JPEG:n02802426 +ILSVRC2012_val_00015398.JPEG:n02917067 +ILSVRC2012_val_00015399.JPEG:n02704792 +ILSVRC2012_val_00015400.JPEG:n07760859 +ILSVRC2012_val_00015401.JPEG:n02123597 +ILSVRC2012_val_00015402.JPEG:n01981276 +ILSVRC2012_val_00015403.JPEG:n01688243 +ILSVRC2012_val_00015404.JPEG:n03400231 +ILSVRC2012_val_00015405.JPEG:n02088238 +ILSVRC2012_val_00015406.JPEG:n07753275 +ILSVRC2012_val_00015407.JPEG:n02100583 +ILSVRC2012_val_00015408.JPEG:n01955084 +ILSVRC2012_val_00015409.JPEG:n02777292 +ILSVRC2012_val_00015410.JPEG:n01534433 +ILSVRC2012_val_00015411.JPEG:n03908714 +ILSVRC2012_val_00015412.JPEG:n02120079 +ILSVRC2012_val_00015413.JPEG:n04465501 +ILSVRC2012_val_00015414.JPEG:n02641379 +ILSVRC2012_val_00015415.JPEG:n02098286 +ILSVRC2012_val_00015416.JPEG:n01534433 +ILSVRC2012_val_00015417.JPEG:n02917067 +ILSVRC2012_val_00015418.JPEG:n04371774 +ILSVRC2012_val_00015419.JPEG:n02110958 +ILSVRC2012_val_00015420.JPEG:n03538406 +ILSVRC2012_val_00015421.JPEG:n03443371 +ILSVRC2012_val_00015422.JPEG:n03902125 +ILSVRC2012_val_00015423.JPEG:n03075370 +ILSVRC2012_val_00015424.JPEG:n04336792 +ILSVRC2012_val_00015425.JPEG:n02091831 +ILSVRC2012_val_00015426.JPEG:n02510455 +ILSVRC2012_val_00015427.JPEG:n02097047 +ILSVRC2012_val_00015428.JPEG:n03908618 +ILSVRC2012_val_00015429.JPEG:n02817516 +ILSVRC2012_val_00015430.JPEG:n02111889 +ILSVRC2012_val_00015431.JPEG:n01531178 +ILSVRC2012_val_00015432.JPEG:n02481823 +ILSVRC2012_val_00015433.JPEG:n03110669 +ILSVRC2012_val_00015434.JPEG:n02095570 +ILSVRC2012_val_00015435.JPEG:n03982430 +ILSVRC2012_val_00015436.JPEG:n03444034 +ILSVRC2012_val_00015437.JPEG:n07714571 +ILSVRC2012_val_00015438.JPEG:n07932039 +ILSVRC2012_val_00015439.JPEG:n01768244 +ILSVRC2012_val_00015440.JPEG:n02837789 +ILSVRC2012_val_00015441.JPEG:n03637318 +ILSVRC2012_val_00015442.JPEG:n04141975 +ILSVRC2012_val_00015443.JPEG:n01910747 +ILSVRC2012_val_00015444.JPEG:n03873416 +ILSVRC2012_val_00015445.JPEG:n03018349 +ILSVRC2012_val_00015446.JPEG:n02114548 +ILSVRC2012_val_00015447.JPEG:n07717556 +ILSVRC2012_val_00015448.JPEG:n03494278 +ILSVRC2012_val_00015449.JPEG:n03924679 +ILSVRC2012_val_00015450.JPEG:n02012849 +ILSVRC2012_val_00015451.JPEG:n02361337 +ILSVRC2012_val_00015452.JPEG:n02398521 +ILSVRC2012_val_00015453.JPEG:n03443371 +ILSVRC2012_val_00015454.JPEG:n07615774 +ILSVRC2012_val_00015455.JPEG:n02009912 +ILSVRC2012_val_00015456.JPEG:n02395406 +ILSVRC2012_val_00015457.JPEG:n02777292 +ILSVRC2012_val_00015458.JPEG:n02783161 +ILSVRC2012_val_00015459.JPEG:n02445715 +ILSVRC2012_val_00015460.JPEG:n03743016 +ILSVRC2012_val_00015461.JPEG:n03891332 +ILSVRC2012_val_00015462.JPEG:n04542943 +ILSVRC2012_val_00015463.JPEG:n15075141 +ILSVRC2012_val_00015464.JPEG:n02091244 +ILSVRC2012_val_00015465.JPEG:n02114367 +ILSVRC2012_val_00015466.JPEG:n03404251 +ILSVRC2012_val_00015467.JPEG:n03000134 +ILSVRC2012_val_00015468.JPEG:n01667114 +ILSVRC2012_val_00015469.JPEG:n03763968 +ILSVRC2012_val_00015470.JPEG:n02233338 +ILSVRC2012_val_00015471.JPEG:n09428293 +ILSVRC2012_val_00015472.JPEG:n03793489 +ILSVRC2012_val_00015473.JPEG:n04258138 +ILSVRC2012_val_00015474.JPEG:n04023962 +ILSVRC2012_val_00015475.JPEG:n01667778 +ILSVRC2012_val_00015476.JPEG:n03899768 +ILSVRC2012_val_00015477.JPEG:n13133613 +ILSVRC2012_val_00015478.JPEG:n03599486 +ILSVRC2012_val_00015479.JPEG:n03042490 +ILSVRC2012_val_00015480.JPEG:n04467665 +ILSVRC2012_val_00015481.JPEG:n03633091 +ILSVRC2012_val_00015482.JPEG:n02437616 +ILSVRC2012_val_00015483.JPEG:n02835271 +ILSVRC2012_val_00015484.JPEG:n03791053 +ILSVRC2012_val_00015485.JPEG:n04486054 +ILSVRC2012_val_00015486.JPEG:n07717410 +ILSVRC2012_val_00015487.JPEG:n07613480 +ILSVRC2012_val_00015488.JPEG:n01728920 +ILSVRC2012_val_00015489.JPEG:n03400231 +ILSVRC2012_val_00015490.JPEG:n02790996 +ILSVRC2012_val_00015491.JPEG:n02676566 +ILSVRC2012_val_00015492.JPEG:n04562935 +ILSVRC2012_val_00015493.JPEG:n02264363 +ILSVRC2012_val_00015494.JPEG:n04141975 +ILSVRC2012_val_00015495.JPEG:n03089624 +ILSVRC2012_val_00015496.JPEG:n03954731 +ILSVRC2012_val_00015497.JPEG:n03467068 +ILSVRC2012_val_00015498.JPEG:n02690373 +ILSVRC2012_val_00015499.JPEG:n02102040 +ILSVRC2012_val_00015500.JPEG:n01985128 +ILSVRC2012_val_00015501.JPEG:n04116512 +ILSVRC2012_val_00015502.JPEG:n02497673 +ILSVRC2012_val_00015503.JPEG:n04392985 +ILSVRC2012_val_00015504.JPEG:n03937543 +ILSVRC2012_val_00015505.JPEG:n02006656 +ILSVRC2012_val_00015506.JPEG:n01773549 +ILSVRC2012_val_00015507.JPEG:n02704792 +ILSVRC2012_val_00015508.JPEG:n02999410 +ILSVRC2012_val_00015509.JPEG:n07930864 +ILSVRC2012_val_00015510.JPEG:n02011460 +ILSVRC2012_val_00015511.JPEG:n02107312 +ILSVRC2012_val_00015512.JPEG:n02910353 +ILSVRC2012_val_00015513.JPEG:n01795545 +ILSVRC2012_val_00015514.JPEG:n04111531 +ILSVRC2012_val_00015515.JPEG:n02894605 +ILSVRC2012_val_00015516.JPEG:n01614925 +ILSVRC2012_val_00015517.JPEG:n02793495 +ILSVRC2012_val_00015518.JPEG:n02100877 +ILSVRC2012_val_00015519.JPEG:n03761084 +ILSVRC2012_val_00015520.JPEG:n02504013 +ILSVRC2012_val_00015521.JPEG:n02408429 +ILSVRC2012_val_00015522.JPEG:n07583066 +ILSVRC2012_val_00015523.JPEG:n01744401 +ILSVRC2012_val_00015524.JPEG:n03447447 +ILSVRC2012_val_00015525.JPEG:n03125729 +ILSVRC2012_val_00015526.JPEG:n01978287 +ILSVRC2012_val_00015527.JPEG:n04346328 +ILSVRC2012_val_00015528.JPEG:n03742115 +ILSVRC2012_val_00015529.JPEG:n02483708 +ILSVRC2012_val_00015530.JPEG:n13054560 +ILSVRC2012_val_00015531.JPEG:n02096177 +ILSVRC2012_val_00015532.JPEG:n03920288 +ILSVRC2012_val_00015533.JPEG:n02837789 +ILSVRC2012_val_00015534.JPEG:n03877472 +ILSVRC2012_val_00015535.JPEG:n02165105 +ILSVRC2012_val_00015536.JPEG:n03937543 +ILSVRC2012_val_00015537.JPEG:n03982430 +ILSVRC2012_val_00015538.JPEG:n03787032 +ILSVRC2012_val_00015539.JPEG:n07880968 +ILSVRC2012_val_00015540.JPEG:n04371774 +ILSVRC2012_val_00015541.JPEG:n04146614 +ILSVRC2012_val_00015542.JPEG:n03394916 +ILSVRC2012_val_00015543.JPEG:n03903868 +ILSVRC2012_val_00015544.JPEG:n02687172 +ILSVRC2012_val_00015545.JPEG:n01494475 +ILSVRC2012_val_00015546.JPEG:n02536864 +ILSVRC2012_val_00015547.JPEG:n02129165 +ILSVRC2012_val_00015548.JPEG:n07920052 +ILSVRC2012_val_00015549.JPEG:n01496331 +ILSVRC2012_val_00015550.JPEG:n02009912 +ILSVRC2012_val_00015551.JPEG:n02692877 +ILSVRC2012_val_00015552.JPEG:n02101006 +ILSVRC2012_val_00015553.JPEG:n03271574 +ILSVRC2012_val_00015554.JPEG:n04371774 +ILSVRC2012_val_00015555.JPEG:n01496331 +ILSVRC2012_val_00015556.JPEG:n04557648 +ILSVRC2012_val_00015557.JPEG:n02027492 +ILSVRC2012_val_00015558.JPEG:n02125311 +ILSVRC2012_val_00015559.JPEG:n03376595 +ILSVRC2012_val_00015560.JPEG:n01872401 +ILSVRC2012_val_00015561.JPEG:n04346328 +ILSVRC2012_val_00015562.JPEG:n02091134 +ILSVRC2012_val_00015563.JPEG:n04238763 +ILSVRC2012_val_00015564.JPEG:n01776313 +ILSVRC2012_val_00015565.JPEG:n01796340 +ILSVRC2012_val_00015566.JPEG:n01770081 +ILSVRC2012_val_00015567.JPEG:n03141823 +ILSVRC2012_val_00015568.JPEG:n01665541 +ILSVRC2012_val_00015569.JPEG:n04133789 +ILSVRC2012_val_00015570.JPEG:n02096437 +ILSVRC2012_val_00015571.JPEG:n02096051 +ILSVRC2012_val_00015572.JPEG:n10565667 +ILSVRC2012_val_00015573.JPEG:n04542943 +ILSVRC2012_val_00015574.JPEG:n03447447 +ILSVRC2012_val_00015575.JPEG:n09421951 +ILSVRC2012_val_00015576.JPEG:n02113624 +ILSVRC2012_val_00015577.JPEG:n03160309 +ILSVRC2012_val_00015578.JPEG:n02504458 +ILSVRC2012_val_00015579.JPEG:n01774750 +ILSVRC2012_val_00015580.JPEG:n03871628 +ILSVRC2012_val_00015581.JPEG:n04590129 +ILSVRC2012_val_00015582.JPEG:n12057211 +ILSVRC2012_val_00015583.JPEG:n03481172 +ILSVRC2012_val_00015584.JPEG:n03000247 +ILSVRC2012_val_00015585.JPEG:n04090263 +ILSVRC2012_val_00015586.JPEG:n04141076 +ILSVRC2012_val_00015587.JPEG:n01914609 +ILSVRC2012_val_00015588.JPEG:n03775071 +ILSVRC2012_val_00015589.JPEG:n02869837 +ILSVRC2012_val_00015590.JPEG:n04509417 +ILSVRC2012_val_00015591.JPEG:n04371430 +ILSVRC2012_val_00015592.JPEG:n02097209 +ILSVRC2012_val_00015593.JPEG:n04613696 +ILSVRC2012_val_00015594.JPEG:n02669723 +ILSVRC2012_val_00015595.JPEG:n02883205 +ILSVRC2012_val_00015596.JPEG:n01748264 +ILSVRC2012_val_00015597.JPEG:n01955084 +ILSVRC2012_val_00015598.JPEG:n04204238 +ILSVRC2012_val_00015599.JPEG:n03743016 +ILSVRC2012_val_00015600.JPEG:n02177972 +ILSVRC2012_val_00015601.JPEG:n03868863 +ILSVRC2012_val_00015602.JPEG:n04133789 +ILSVRC2012_val_00015603.JPEG:n02168699 +ILSVRC2012_val_00015604.JPEG:n04041544 +ILSVRC2012_val_00015605.JPEG:n02115913 +ILSVRC2012_val_00015606.JPEG:n02259212 +ILSVRC2012_val_00015607.JPEG:n02096177 +ILSVRC2012_val_00015608.JPEG:n02277742 +ILSVRC2012_val_00015609.JPEG:n04493381 +ILSVRC2012_val_00015610.JPEG:n02093859 +ILSVRC2012_val_00015611.JPEG:n03160309 +ILSVRC2012_val_00015612.JPEG:n04120489 +ILSVRC2012_val_00015613.JPEG:n09246464 +ILSVRC2012_val_00015614.JPEG:n04005630 +ILSVRC2012_val_00015615.JPEG:n03938244 +ILSVRC2012_val_00015616.JPEG:n03208938 +ILSVRC2012_val_00015617.JPEG:n04033901 +ILSVRC2012_val_00015618.JPEG:n02835271 +ILSVRC2012_val_00015619.JPEG:n04049303 +ILSVRC2012_val_00015620.JPEG:n02951585 +ILSVRC2012_val_00015621.JPEG:n04229816 +ILSVRC2012_val_00015622.JPEG:n01755581 +ILSVRC2012_val_00015623.JPEG:n01734418 +ILSVRC2012_val_00015624.JPEG:n01843065 +ILSVRC2012_val_00015625.JPEG:n02114367 +ILSVRC2012_val_00015626.JPEG:n09288635 +ILSVRC2012_val_00015627.JPEG:n04147183 +ILSVRC2012_val_00015628.JPEG:n03196217 +ILSVRC2012_val_00015629.JPEG:n04367480 +ILSVRC2012_val_00015630.JPEG:n03467068 +ILSVRC2012_val_00015631.JPEG:n01491361 +ILSVRC2012_val_00015632.JPEG:n02091831 +ILSVRC2012_val_00015633.JPEG:n04154565 +ILSVRC2012_val_00015634.JPEG:n07875152 +ILSVRC2012_val_00015635.JPEG:n07873807 +ILSVRC2012_val_00015636.JPEG:n02690373 +ILSVRC2012_val_00015637.JPEG:n02730930 +ILSVRC2012_val_00015638.JPEG:n04389033 +ILSVRC2012_val_00015639.JPEG:n02879718 +ILSVRC2012_val_00015640.JPEG:n03223299 +ILSVRC2012_val_00015641.JPEG:n01784675 +ILSVRC2012_val_00015642.JPEG:n03447721 +ILSVRC2012_val_00015643.JPEG:n01742172 +ILSVRC2012_val_00015644.JPEG:n01728572 +ILSVRC2012_val_00015645.JPEG:n12985857 +ILSVRC2012_val_00015646.JPEG:n03376595 +ILSVRC2012_val_00015647.JPEG:n03089624 +ILSVRC2012_val_00015648.JPEG:n03887697 +ILSVRC2012_val_00015649.JPEG:n04270147 +ILSVRC2012_val_00015650.JPEG:n01930112 +ILSVRC2012_val_00015651.JPEG:n02814533 +ILSVRC2012_val_00015652.JPEG:n07802026 +ILSVRC2012_val_00015653.JPEG:n07920052 +ILSVRC2012_val_00015654.JPEG:n03425413 +ILSVRC2012_val_00015655.JPEG:n06596364 +ILSVRC2012_val_00015656.JPEG:n03134739 +ILSVRC2012_val_00015657.JPEG:n02108422 +ILSVRC2012_val_00015658.JPEG:n12998815 +ILSVRC2012_val_00015659.JPEG:n07753113 +ILSVRC2012_val_00015660.JPEG:n02056570 +ILSVRC2012_val_00015661.JPEG:n09256479 +ILSVRC2012_val_00015662.JPEG:n04238763 +ILSVRC2012_val_00015663.JPEG:n02951585 +ILSVRC2012_val_00015664.JPEG:n04033901 +ILSVRC2012_val_00015665.JPEG:n01833805 +ILSVRC2012_val_00015666.JPEG:n01737021 +ILSVRC2012_val_00015667.JPEG:n01694178 +ILSVRC2012_val_00015668.JPEG:n06785654 +ILSVRC2012_val_00015669.JPEG:n02500267 +ILSVRC2012_val_00015670.JPEG:n02085782 +ILSVRC2012_val_00015671.JPEG:n03825788 +ILSVRC2012_val_00015672.JPEG:n03899768 +ILSVRC2012_val_00015673.JPEG:n01843383 +ILSVRC2012_val_00015674.JPEG:n02782093 +ILSVRC2012_val_00015675.JPEG:n01855672 +ILSVRC2012_val_00015676.JPEG:n04239074 +ILSVRC2012_val_00015677.JPEG:n04604644 +ILSVRC2012_val_00015678.JPEG:n07583066 +ILSVRC2012_val_00015679.JPEG:n03041632 +ILSVRC2012_val_00015680.JPEG:n02777292 +ILSVRC2012_val_00015681.JPEG:n03627232 +ILSVRC2012_val_00015682.JPEG:n03884397 +ILSVRC2012_val_00015683.JPEG:n02328150 +ILSVRC2012_val_00015684.JPEG:n04005630 +ILSVRC2012_val_00015685.JPEG:n02093859 +ILSVRC2012_val_00015686.JPEG:n01749939 +ILSVRC2012_val_00015687.JPEG:n03000134 +ILSVRC2012_val_00015688.JPEG:n04037443 +ILSVRC2012_val_00015689.JPEG:n03888257 +ILSVRC2012_val_00015690.JPEG:n01824575 +ILSVRC2012_val_00015691.JPEG:n07875152 +ILSVRC2012_val_00015692.JPEG:n02526121 +ILSVRC2012_val_00015693.JPEG:n07920052 +ILSVRC2012_val_00015694.JPEG:n02102040 +ILSVRC2012_val_00015695.JPEG:n02869837 +ILSVRC2012_val_00015696.JPEG:n02099849 +ILSVRC2012_val_00015697.JPEG:n04356056 +ILSVRC2012_val_00015698.JPEG:n01749939 +ILSVRC2012_val_00015699.JPEG:n02442845 +ILSVRC2012_val_00015700.JPEG:n04487081 +ILSVRC2012_val_00015701.JPEG:n02087046 +ILSVRC2012_val_00015702.JPEG:n04201297 +ILSVRC2012_val_00015703.JPEG:n02094433 +ILSVRC2012_val_00015704.JPEG:n02480495 +ILSVRC2012_val_00015705.JPEG:n02096585 +ILSVRC2012_val_00015706.JPEG:n01518878 +ILSVRC2012_val_00015707.JPEG:n04141975 +ILSVRC2012_val_00015708.JPEG:n02981792 +ILSVRC2012_val_00015709.JPEG:n01632458 +ILSVRC2012_val_00015710.JPEG:n02093647 +ILSVRC2012_val_00015711.JPEG:n02018207 +ILSVRC2012_val_00015712.JPEG:n04040759 +ILSVRC2012_val_00015713.JPEG:n01820546 +ILSVRC2012_val_00015714.JPEG:n03840681 +ILSVRC2012_val_00015715.JPEG:n03832673 +ILSVRC2012_val_00015716.JPEG:n02051845 +ILSVRC2012_val_00015717.JPEG:n01883070 +ILSVRC2012_val_00015718.JPEG:n03534580 +ILSVRC2012_val_00015719.JPEG:n02028035 +ILSVRC2012_val_00015720.JPEG:n03857828 +ILSVRC2012_val_00015721.JPEG:n01682714 +ILSVRC2012_val_00015722.JPEG:n04049303 +ILSVRC2012_val_00015723.JPEG:n02096585 +ILSVRC2012_val_00015724.JPEG:n04254120 +ILSVRC2012_val_00015725.JPEG:n02071294 +ILSVRC2012_val_00015726.JPEG:n03868863 +ILSVRC2012_val_00015727.JPEG:n02206856 +ILSVRC2012_val_00015728.JPEG:n04086273 +ILSVRC2012_val_00015729.JPEG:n02177972 +ILSVRC2012_val_00015730.JPEG:n02085782 +ILSVRC2012_val_00015731.JPEG:n03942813 +ILSVRC2012_val_00015732.JPEG:n01496331 +ILSVRC2012_val_00015733.JPEG:n04355933 +ILSVRC2012_val_00015734.JPEG:n02790996 +ILSVRC2012_val_00015735.JPEG:n04265275 +ILSVRC2012_val_00015736.JPEG:n03976467 +ILSVRC2012_val_00015737.JPEG:n02279972 +ILSVRC2012_val_00015738.JPEG:n02086240 +ILSVRC2012_val_00015739.JPEG:n01824575 +ILSVRC2012_val_00015740.JPEG:n09421951 +ILSVRC2012_val_00015741.JPEG:n02123159 +ILSVRC2012_val_00015742.JPEG:n02086079 +ILSVRC2012_val_00015743.JPEG:n07717410 +ILSVRC2012_val_00015744.JPEG:n02422106 +ILSVRC2012_val_00015745.JPEG:n02236044 +ILSVRC2012_val_00015746.JPEG:n01608432 +ILSVRC2012_val_00015747.JPEG:n03062245 +ILSVRC2012_val_00015748.JPEG:n07734744 +ILSVRC2012_val_00015749.JPEG:n01983481 +ILSVRC2012_val_00015750.JPEG:n04542943 +ILSVRC2012_val_00015751.JPEG:n01773797 +ILSVRC2012_val_00015752.JPEG:n02526121 +ILSVRC2012_val_00015753.JPEG:n01688243 +ILSVRC2012_val_00015754.JPEG:n01990800 +ILSVRC2012_val_00015755.JPEG:n02169497 +ILSVRC2012_val_00015756.JPEG:n01768244 +ILSVRC2012_val_00015757.JPEG:n01770393 +ILSVRC2012_val_00015758.JPEG:n03977966 +ILSVRC2012_val_00015759.JPEG:n02096585 +ILSVRC2012_val_00015760.JPEG:n03532672 +ILSVRC2012_val_00015761.JPEG:n07711569 +ILSVRC2012_val_00015762.JPEG:n01734418 +ILSVRC2012_val_00015763.JPEG:n04326547 +ILSVRC2012_val_00015764.JPEG:n09332890 +ILSVRC2012_val_00015765.JPEG:n04584207 +ILSVRC2012_val_00015766.JPEG:n02114712 +ILSVRC2012_val_00015767.JPEG:n02093754 +ILSVRC2012_val_00015768.JPEG:n03495258 +ILSVRC2012_val_00015769.JPEG:n01616318 +ILSVRC2012_val_00015770.JPEG:n02326432 +ILSVRC2012_val_00015771.JPEG:n04507155 +ILSVRC2012_val_00015772.JPEG:n03527444 +ILSVRC2012_val_00015773.JPEG:n01981276 +ILSVRC2012_val_00015774.JPEG:n02097298 +ILSVRC2012_val_00015775.JPEG:n03958227 +ILSVRC2012_val_00015776.JPEG:n02165105 +ILSVRC2012_val_00015777.JPEG:n07718472 +ILSVRC2012_val_00015778.JPEG:n04591157 +ILSVRC2012_val_00015779.JPEG:n04286575 +ILSVRC2012_val_00015780.JPEG:n04208210 +ILSVRC2012_val_00015781.JPEG:n02120505 +ILSVRC2012_val_00015782.JPEG:n04265275 +ILSVRC2012_val_00015783.JPEG:n04147183 +ILSVRC2012_val_00015784.JPEG:n03271574 +ILSVRC2012_val_00015785.JPEG:n02128385 +ILSVRC2012_val_00015786.JPEG:n02110958 +ILSVRC2012_val_00015787.JPEG:n03888257 +ILSVRC2012_val_00015788.JPEG:n02730930 +ILSVRC2012_val_00015789.JPEG:n01978455 +ILSVRC2012_val_00015790.JPEG:n02843684 +ILSVRC2012_val_00015791.JPEG:n03590841 +ILSVRC2012_val_00015792.JPEG:n03065424 +ILSVRC2012_val_00015793.JPEG:n03854065 +ILSVRC2012_val_00015794.JPEG:n01739381 +ILSVRC2012_val_00015795.JPEG:n01773797 +ILSVRC2012_val_00015796.JPEG:n03976657 +ILSVRC2012_val_00015797.JPEG:n04116512 +ILSVRC2012_val_00015798.JPEG:n02092339 +ILSVRC2012_val_00015799.JPEG:n01817953 +ILSVRC2012_val_00015800.JPEG:n02119789 +ILSVRC2012_val_00015801.JPEG:n01748264 +ILSVRC2012_val_00015802.JPEG:n02169497 +ILSVRC2012_val_00015803.JPEG:n03125729 +ILSVRC2012_val_00015804.JPEG:n02091467 +ILSVRC2012_val_00015805.JPEG:n07714571 +ILSVRC2012_val_00015806.JPEG:n02704792 +ILSVRC2012_val_00015807.JPEG:n02085936 +ILSVRC2012_val_00015808.JPEG:n02108915 +ILSVRC2012_val_00015809.JPEG:n03314780 +ILSVRC2012_val_00015810.JPEG:n02086646 +ILSVRC2012_val_00015811.JPEG:n07697537 +ILSVRC2012_val_00015812.JPEG:n03584829 +ILSVRC2012_val_00015813.JPEG:n03773504 +ILSVRC2012_val_00015814.JPEG:n04204347 +ILSVRC2012_val_00015815.JPEG:n01796340 +ILSVRC2012_val_00015816.JPEG:n03930313 +ILSVRC2012_val_00015817.JPEG:n02033041 +ILSVRC2012_val_00015818.JPEG:n02236044 +ILSVRC2012_val_00015819.JPEG:n02895154 +ILSVRC2012_val_00015820.JPEG:n02708093 +ILSVRC2012_val_00015821.JPEG:n02115641 +ILSVRC2012_val_00015822.JPEG:n04209239 +ILSVRC2012_val_00015823.JPEG:n01735189 +ILSVRC2012_val_00015824.JPEG:n03201208 +ILSVRC2012_val_00015825.JPEG:n09468604 +ILSVRC2012_val_00015826.JPEG:n03047690 +ILSVRC2012_val_00015827.JPEG:n04254777 +ILSVRC2012_val_00015828.JPEG:n06596364 +ILSVRC2012_val_00015829.JPEG:n03627232 +ILSVRC2012_val_00015830.JPEG:n01532829 +ILSVRC2012_val_00015831.JPEG:n01694178 +ILSVRC2012_val_00015832.JPEG:n04081281 +ILSVRC2012_val_00015833.JPEG:n03495258 +ILSVRC2012_val_00015834.JPEG:n02788148 +ILSVRC2012_val_00015835.JPEG:n01775062 +ILSVRC2012_val_00015836.JPEG:n04355933 +ILSVRC2012_val_00015837.JPEG:n03017168 +ILSVRC2012_val_00015838.JPEG:n04599235 +ILSVRC2012_val_00015839.JPEG:n03785016 +ILSVRC2012_val_00015840.JPEG:n07871810 +ILSVRC2012_val_00015841.JPEG:n03980874 +ILSVRC2012_val_00015842.JPEG:n02071294 +ILSVRC2012_val_00015843.JPEG:n04493381 +ILSVRC2012_val_00015844.JPEG:n04372370 +ILSVRC2012_val_00015845.JPEG:n02087046 +ILSVRC2012_val_00015846.JPEG:n04584207 +ILSVRC2012_val_00015847.JPEG:n04086273 +ILSVRC2012_val_00015848.JPEG:n02092339 +ILSVRC2012_val_00015849.JPEG:n02817516 +ILSVRC2012_val_00015850.JPEG:n03240683 +ILSVRC2012_val_00015851.JPEG:n12998815 +ILSVRC2012_val_00015852.JPEG:n03075370 +ILSVRC2012_val_00015853.JPEG:n02804414 +ILSVRC2012_val_00015854.JPEG:n01833805 +ILSVRC2012_val_00015855.JPEG:n01695060 +ILSVRC2012_val_00015856.JPEG:n04596742 +ILSVRC2012_val_00015857.JPEG:n04398044 +ILSVRC2012_val_00015858.JPEG:n02106382 +ILSVRC2012_val_00015859.JPEG:n04204238 +ILSVRC2012_val_00015860.JPEG:n02219486 +ILSVRC2012_val_00015861.JPEG:n02437312 +ILSVRC2012_val_00015862.JPEG:n04335435 +ILSVRC2012_val_00015863.JPEG:n01531178 +ILSVRC2012_val_00015864.JPEG:n04201297 +ILSVRC2012_val_00015865.JPEG:n03920288 +ILSVRC2012_val_00015866.JPEG:n03759954 +ILSVRC2012_val_00015867.JPEG:n03792782 +ILSVRC2012_val_00015868.JPEG:n02412080 +ILSVRC2012_val_00015869.JPEG:n04536866 +ILSVRC2012_val_00015870.JPEG:n03874293 +ILSVRC2012_val_00015871.JPEG:n02708093 +ILSVRC2012_val_00015872.JPEG:n02437312 +ILSVRC2012_val_00015873.JPEG:n04509417 +ILSVRC2012_val_00015874.JPEG:n01990800 +ILSVRC2012_val_00015875.JPEG:n04579145 +ILSVRC2012_val_00015876.JPEG:n02480495 +ILSVRC2012_val_00015877.JPEG:n04371430 +ILSVRC2012_val_00015878.JPEG:n02105056 +ILSVRC2012_val_00015879.JPEG:n03930630 +ILSVRC2012_val_00015880.JPEG:n03481172 +ILSVRC2012_val_00015881.JPEG:n02808440 +ILSVRC2012_val_00015882.JPEG:n07932039 +ILSVRC2012_val_00015883.JPEG:n04428191 +ILSVRC2012_val_00015884.JPEG:n02971356 +ILSVRC2012_val_00015885.JPEG:n02090379 +ILSVRC2012_val_00015886.JPEG:n03857828 +ILSVRC2012_val_00015887.JPEG:n02988304 +ILSVRC2012_val_00015888.JPEG:n02115913 +ILSVRC2012_val_00015889.JPEG:n04599235 +ILSVRC2012_val_00015890.JPEG:n04033901 +ILSVRC2012_val_00015891.JPEG:n11879895 +ILSVRC2012_val_00015892.JPEG:n03014705 +ILSVRC2012_val_00015893.JPEG:n02002724 +ILSVRC2012_val_00015894.JPEG:n02445715 +ILSVRC2012_val_00015895.JPEG:n02870880 +ILSVRC2012_val_00015896.JPEG:n02951585 +ILSVRC2012_val_00015897.JPEG:n02129604 +ILSVRC2012_val_00015898.JPEG:n02123394 +ILSVRC2012_val_00015899.JPEG:n01860187 +ILSVRC2012_val_00015900.JPEG:n03788195 +ILSVRC2012_val_00015901.JPEG:n03729826 +ILSVRC2012_val_00015902.JPEG:n01665541 +ILSVRC2012_val_00015903.JPEG:n01531178 +ILSVRC2012_val_00015904.JPEG:n04442312 +ILSVRC2012_val_00015905.JPEG:n02777292 +ILSVRC2012_val_00015906.JPEG:n13044778 +ILSVRC2012_val_00015907.JPEG:n07720875 +ILSVRC2012_val_00015908.JPEG:n02027492 +ILSVRC2012_val_00015909.JPEG:n02480855 +ILSVRC2012_val_00015910.JPEG:n04447861 +ILSVRC2012_val_00015911.JPEG:n02403003 +ILSVRC2012_val_00015912.JPEG:n03874599 +ILSVRC2012_val_00015913.JPEG:n01622779 +ILSVRC2012_val_00015914.JPEG:n02860847 +ILSVRC2012_val_00015915.JPEG:n03884397 +ILSVRC2012_val_00015916.JPEG:n13040303 +ILSVRC2012_val_00015917.JPEG:n03796401 +ILSVRC2012_val_00015918.JPEG:n03388549 +ILSVRC2012_val_00015919.JPEG:n03970156 +ILSVRC2012_val_00015920.JPEG:n02112137 +ILSVRC2012_val_00015921.JPEG:n03775071 +ILSVRC2012_val_00015922.JPEG:n01601694 +ILSVRC2012_val_00015923.JPEG:n02093991 +ILSVRC2012_val_00015924.JPEG:n01664065 +ILSVRC2012_val_00015925.JPEG:n02077923 +ILSVRC2012_val_00015926.JPEG:n02487347 +ILSVRC2012_val_00015927.JPEG:n02444819 +ILSVRC2012_val_00015928.JPEG:n02480855 +ILSVRC2012_val_00015929.JPEG:n04505470 +ILSVRC2012_val_00015930.JPEG:n03980874 +ILSVRC2012_val_00015931.JPEG:n03447447 +ILSVRC2012_val_00015932.JPEG:n01955084 +ILSVRC2012_val_00015933.JPEG:n02056570 +ILSVRC2012_val_00015934.JPEG:n03127747 +ILSVRC2012_val_00015935.JPEG:n02692877 +ILSVRC2012_val_00015936.JPEG:n06596364 +ILSVRC2012_val_00015937.JPEG:n03400231 +ILSVRC2012_val_00015938.JPEG:n03482405 +ILSVRC2012_val_00015939.JPEG:n03920288 +ILSVRC2012_val_00015940.JPEG:n03871628 +ILSVRC2012_val_00015941.JPEG:n03496892 +ILSVRC2012_val_00015942.JPEG:n12267677 +ILSVRC2012_val_00015943.JPEG:n04310018 +ILSVRC2012_val_00015944.JPEG:n02865351 +ILSVRC2012_val_00015945.JPEG:n01924916 +ILSVRC2012_val_00015946.JPEG:n03000247 +ILSVRC2012_val_00015947.JPEG:n03393912 +ILSVRC2012_val_00015948.JPEG:n02825657 +ILSVRC2012_val_00015949.JPEG:n06785654 +ILSVRC2012_val_00015950.JPEG:n02097474 +ILSVRC2012_val_00015951.JPEG:n04179913 +ILSVRC2012_val_00015952.JPEG:n02112350 +ILSVRC2012_val_00015953.JPEG:n03444034 +ILSVRC2012_val_00015954.JPEG:n03133878 +ILSVRC2012_val_00015955.JPEG:n02132136 +ILSVRC2012_val_00015956.JPEG:n02843684 +ILSVRC2012_val_00015957.JPEG:n01770393 +ILSVRC2012_val_00015958.JPEG:n01871265 +ILSVRC2012_val_00015959.JPEG:n03290653 +ILSVRC2012_val_00015960.JPEG:n03207941 +ILSVRC2012_val_00015961.JPEG:n03476991 +ILSVRC2012_val_00015962.JPEG:n03481172 +ILSVRC2012_val_00015963.JPEG:n04590129 +ILSVRC2012_val_00015964.JPEG:n01532829 +ILSVRC2012_val_00015965.JPEG:n03642806 +ILSVRC2012_val_00015966.JPEG:n03388183 +ILSVRC2012_val_00015967.JPEG:n02094258 +ILSVRC2012_val_00015968.JPEG:n03496892 +ILSVRC2012_val_00015969.JPEG:n04467665 +ILSVRC2012_val_00015970.JPEG:n02963159 +ILSVRC2012_val_00015971.JPEG:n02328150 +ILSVRC2012_val_00015972.JPEG:n02101388 +ILSVRC2012_val_00015973.JPEG:n09256479 +ILSVRC2012_val_00015974.JPEG:n03777568 +ILSVRC2012_val_00015975.JPEG:n02165456 +ILSVRC2012_val_00015976.JPEG:n03042490 +ILSVRC2012_val_00015977.JPEG:n02363005 +ILSVRC2012_val_00015978.JPEG:n13054560 +ILSVRC2012_val_00015979.JPEG:n02808440 +ILSVRC2012_val_00015980.JPEG:n04532670 +ILSVRC2012_val_00015981.JPEG:n01688243 +ILSVRC2012_val_00015982.JPEG:n03602883 +ILSVRC2012_val_00015983.JPEG:n02206856 +ILSVRC2012_val_00015984.JPEG:n03400231 +ILSVRC2012_val_00015985.JPEG:n02346627 +ILSVRC2012_val_00015986.JPEG:n01871265 +ILSVRC2012_val_00015987.JPEG:n01806567 +ILSVRC2012_val_00015988.JPEG:n02727426 +ILSVRC2012_val_00015989.JPEG:n04067472 +ILSVRC2012_val_00015990.JPEG:n02088094 +ILSVRC2012_val_00015991.JPEG:n04553703 +ILSVRC2012_val_00015992.JPEG:n13037406 +ILSVRC2012_val_00015993.JPEG:n07718472 +ILSVRC2012_val_00015994.JPEG:n04252077 +ILSVRC2012_val_00015995.JPEG:n04258138 +ILSVRC2012_val_00015996.JPEG:n02808440 +ILSVRC2012_val_00015997.JPEG:n02328150 +ILSVRC2012_val_00015998.JPEG:n03325584 +ILSVRC2012_val_00015999.JPEG:n01774750 +ILSVRC2012_val_00016000.JPEG:n02123159 +ILSVRC2012_val_00016001.JPEG:n02111277 +ILSVRC2012_val_00016002.JPEG:n04591157 +ILSVRC2012_val_00016003.JPEG:n03871628 +ILSVRC2012_val_00016004.JPEG:n03775071 +ILSVRC2012_val_00016005.JPEG:n04136333 +ILSVRC2012_val_00016006.JPEG:n03976467 +ILSVRC2012_val_00016007.JPEG:n03908618 +ILSVRC2012_val_00016008.JPEG:n03483316 +ILSVRC2012_val_00016009.JPEG:n04487394 +ILSVRC2012_val_00016010.JPEG:n02769748 +ILSVRC2012_val_00016011.JPEG:n04523525 +ILSVRC2012_val_00016012.JPEG:n12998815 +ILSVRC2012_val_00016013.JPEG:n04553703 +ILSVRC2012_val_00016014.JPEG:n04152593 +ILSVRC2012_val_00016015.JPEG:n02346627 +ILSVRC2012_val_00016016.JPEG:n02007558 +ILSVRC2012_val_00016017.JPEG:n03110669 +ILSVRC2012_val_00016018.JPEG:n01440764 +ILSVRC2012_val_00016019.JPEG:n09472597 +ILSVRC2012_val_00016020.JPEG:n02730930 +ILSVRC2012_val_00016021.JPEG:n02782093 +ILSVRC2012_val_00016022.JPEG:n04483307 +ILSVRC2012_val_00016023.JPEG:n02028035 +ILSVRC2012_val_00016024.JPEG:n04040759 +ILSVRC2012_val_00016025.JPEG:n03372029 +ILSVRC2012_val_00016026.JPEG:n02808440 +ILSVRC2012_val_00016027.JPEG:n02120505 +ILSVRC2012_val_00016028.JPEG:n03141823 +ILSVRC2012_val_00016029.JPEG:n02100236 +ILSVRC2012_val_00016030.JPEG:n01770393 +ILSVRC2012_val_00016031.JPEG:n01739381 +ILSVRC2012_val_00016032.JPEG:n03208938 +ILSVRC2012_val_00016033.JPEG:n03954731 +ILSVRC2012_val_00016034.JPEG:n04536866 +ILSVRC2012_val_00016035.JPEG:n04456115 +ILSVRC2012_val_00016036.JPEG:n03000247 +ILSVRC2012_val_00016037.JPEG:n04612504 +ILSVRC2012_val_00016038.JPEG:n02837789 +ILSVRC2012_val_00016039.JPEG:n03538406 +ILSVRC2012_val_00016040.JPEG:n02699494 +ILSVRC2012_val_00016041.JPEG:n03967562 +ILSVRC2012_val_00016042.JPEG:n04398044 +ILSVRC2012_val_00016043.JPEG:n03710721 +ILSVRC2012_val_00016044.JPEG:n04356056 +ILSVRC2012_val_00016045.JPEG:n04033995 +ILSVRC2012_val_00016046.JPEG:n02415577 +ILSVRC2012_val_00016047.JPEG:n04270147 +ILSVRC2012_val_00016048.JPEG:n03866082 +ILSVRC2012_val_00016049.JPEG:n03271574 +ILSVRC2012_val_00016050.JPEG:n02133161 +ILSVRC2012_val_00016051.JPEG:n03483316 +ILSVRC2012_val_00016052.JPEG:n01514668 +ILSVRC2012_val_00016053.JPEG:n03770679 +ILSVRC2012_val_00016054.JPEG:n04532670 +ILSVRC2012_val_00016055.JPEG:n03720891 +ILSVRC2012_val_00016056.JPEG:n02096437 +ILSVRC2012_val_00016057.JPEG:n03444034 +ILSVRC2012_val_00016058.JPEG:n02088632 +ILSVRC2012_val_00016059.JPEG:n02328150 +ILSVRC2012_val_00016060.JPEG:n02787622 +ILSVRC2012_val_00016061.JPEG:n12998815 +ILSVRC2012_val_00016062.JPEG:n07716358 +ILSVRC2012_val_00016063.JPEG:n02817516 +ILSVRC2012_val_00016064.JPEG:n03961711 +ILSVRC2012_val_00016065.JPEG:n02823428 +ILSVRC2012_val_00016066.JPEG:n01753488 +ILSVRC2012_val_00016067.JPEG:n02443114 +ILSVRC2012_val_00016068.JPEG:n04370456 +ILSVRC2012_val_00016069.JPEG:n04542943 +ILSVRC2012_val_00016070.JPEG:n03876231 +ILSVRC2012_val_00016071.JPEG:n02509815 +ILSVRC2012_val_00016072.JPEG:n04371430 +ILSVRC2012_val_00016073.JPEG:n04141975 +ILSVRC2012_val_00016074.JPEG:n02112350 +ILSVRC2012_val_00016075.JPEG:n02321529 +ILSVRC2012_val_00016076.JPEG:n02097474 +ILSVRC2012_val_00016077.JPEG:n04461696 +ILSVRC2012_val_00016078.JPEG:n03804744 +ILSVRC2012_val_00016079.JPEG:n02786058 +ILSVRC2012_val_00016080.JPEG:n12768682 +ILSVRC2012_val_00016081.JPEG:n01855032 +ILSVRC2012_val_00016082.JPEG:n03992509 +ILSVRC2012_val_00016083.JPEG:n01773797 +ILSVRC2012_val_00016084.JPEG:n02443484 +ILSVRC2012_val_00016085.JPEG:n02101006 +ILSVRC2012_val_00016086.JPEG:n09421951 +ILSVRC2012_val_00016087.JPEG:n03837869 +ILSVRC2012_val_00016088.JPEG:n04356056 +ILSVRC2012_val_00016089.JPEG:n01744401 +ILSVRC2012_val_00016090.JPEG:n02701002 +ILSVRC2012_val_00016091.JPEG:n03977966 +ILSVRC2012_val_00016092.JPEG:n02105056 +ILSVRC2012_val_00016093.JPEG:n02102318 +ILSVRC2012_val_00016094.JPEG:n03095699 +ILSVRC2012_val_00016095.JPEG:n01728572 +ILSVRC2012_val_00016096.JPEG:n01873310 +ILSVRC2012_val_00016097.JPEG:n03930313 +ILSVRC2012_val_00016098.JPEG:n03930630 +ILSVRC2012_val_00016099.JPEG:n06359193 +ILSVRC2012_val_00016100.JPEG:n02033041 +ILSVRC2012_val_00016101.JPEG:n04604644 +ILSVRC2012_val_00016102.JPEG:n03781244 +ILSVRC2012_val_00016103.JPEG:n04599235 +ILSVRC2012_val_00016104.JPEG:n02114548 +ILSVRC2012_val_00016105.JPEG:n02356798 +ILSVRC2012_val_00016106.JPEG:n03271574 +ILSVRC2012_val_00016107.JPEG:n07932039 +ILSVRC2012_val_00016108.JPEG:n02100735 +ILSVRC2012_val_00016109.JPEG:n04069434 +ILSVRC2012_val_00016110.JPEG:n04346328 +ILSVRC2012_val_00016111.JPEG:n09332890 +ILSVRC2012_val_00016112.JPEG:n12768682 +ILSVRC2012_val_00016113.JPEG:n02795169 +ILSVRC2012_val_00016114.JPEG:n04049303 +ILSVRC2012_val_00016115.JPEG:n02403003 +ILSVRC2012_val_00016116.JPEG:n04239074 +ILSVRC2012_val_00016117.JPEG:n02493793 +ILSVRC2012_val_00016118.JPEG:n02127052 +ILSVRC2012_val_00016119.JPEG:n04317175 +ILSVRC2012_val_00016120.JPEG:n02363005 +ILSVRC2012_val_00016121.JPEG:n03832673 +ILSVRC2012_val_00016122.JPEG:n04296562 +ILSVRC2012_val_00016123.JPEG:n03630383 +ILSVRC2012_val_00016124.JPEG:n01739381 +ILSVRC2012_val_00016125.JPEG:n02107683 +ILSVRC2012_val_00016126.JPEG:n02012849 +ILSVRC2012_val_00016127.JPEG:n03786901 +ILSVRC2012_val_00016128.JPEG:n04033995 +ILSVRC2012_val_00016129.JPEG:n03782006 +ILSVRC2012_val_00016130.JPEG:n02113624 +ILSVRC2012_val_00016131.JPEG:n02783161 +ILSVRC2012_val_00016132.JPEG:n02134418 +ILSVRC2012_val_00016133.JPEG:n03532672 +ILSVRC2012_val_00016134.JPEG:n02012849 +ILSVRC2012_val_00016135.JPEG:n02415577 +ILSVRC2012_val_00016136.JPEG:n02096437 +ILSVRC2012_val_00016137.JPEG:n03220513 +ILSVRC2012_val_00016138.JPEG:n01945685 +ILSVRC2012_val_00016139.JPEG:n02892201 +ILSVRC2012_val_00016140.JPEG:n04044716 +ILSVRC2012_val_00016141.JPEG:n07742313 +ILSVRC2012_val_00016142.JPEG:n03376595 +ILSVRC2012_val_00016143.JPEG:n02643566 +ILSVRC2012_val_00016144.JPEG:n01735189 +ILSVRC2012_val_00016145.JPEG:n01729977 +ILSVRC2012_val_00016146.JPEG:n02105251 +ILSVRC2012_val_00016147.JPEG:n09421951 +ILSVRC2012_val_00016148.JPEG:n02099712 +ILSVRC2012_val_00016149.JPEG:n03388043 +ILSVRC2012_val_00016150.JPEG:n02174001 +ILSVRC2012_val_00016151.JPEG:n04147183 +ILSVRC2012_val_00016152.JPEG:n02013706 +ILSVRC2012_val_00016153.JPEG:n13054560 +ILSVRC2012_val_00016154.JPEG:n02978881 +ILSVRC2012_val_00016155.JPEG:n09246464 +ILSVRC2012_val_00016156.JPEG:n02699494 +ILSVRC2012_val_00016157.JPEG:n02107312 +ILSVRC2012_val_00016158.JPEG:n03017168 +ILSVRC2012_val_00016159.JPEG:n07745940 +ILSVRC2012_val_00016160.JPEG:n02233338 +ILSVRC2012_val_00016161.JPEG:n02791270 +ILSVRC2012_val_00016162.JPEG:n01950731 +ILSVRC2012_val_00016163.JPEG:n03857828 +ILSVRC2012_val_00016164.JPEG:n02025239 +ILSVRC2012_val_00016165.JPEG:n03452741 +ILSVRC2012_val_00016166.JPEG:n02101388 +ILSVRC2012_val_00016167.JPEG:n03388549 +ILSVRC2012_val_00016168.JPEG:n01484850 +ILSVRC2012_val_00016169.JPEG:n02111277 +ILSVRC2012_val_00016170.JPEG:n01950731 +ILSVRC2012_val_00016171.JPEG:n02174001 +ILSVRC2012_val_00016172.JPEG:n02105162 +ILSVRC2012_val_00016173.JPEG:n02480855 +ILSVRC2012_val_00016174.JPEG:n03325584 +ILSVRC2012_val_00016175.JPEG:n03272562 +ILSVRC2012_val_00016176.JPEG:n03876231 +ILSVRC2012_val_00016177.JPEG:n01644373 +ILSVRC2012_val_00016178.JPEG:n04380533 +ILSVRC2012_val_00016179.JPEG:n07697537 +ILSVRC2012_val_00016180.JPEG:n04380533 +ILSVRC2012_val_00016181.JPEG:n02190166 +ILSVRC2012_val_00016182.JPEG:n07753592 +ILSVRC2012_val_00016183.JPEG:n01630670 +ILSVRC2012_val_00016184.JPEG:n02730930 +ILSVRC2012_val_00016185.JPEG:n03788195 +ILSVRC2012_val_00016186.JPEG:n02669723 +ILSVRC2012_val_00016187.JPEG:n02100735 +ILSVRC2012_val_00016188.JPEG:n03271574 +ILSVRC2012_val_00016189.JPEG:n03179701 +ILSVRC2012_val_00016190.JPEG:n02486261 +ILSVRC2012_val_00016191.JPEG:n02105412 +ILSVRC2012_val_00016192.JPEG:n02417914 +ILSVRC2012_val_00016193.JPEG:n01770081 +ILSVRC2012_val_00016194.JPEG:n02123394 +ILSVRC2012_val_00016195.JPEG:n01855672 +ILSVRC2012_val_00016196.JPEG:n02480495 +ILSVRC2012_val_00016197.JPEG:n02692877 +ILSVRC2012_val_00016198.JPEG:n01532829 +ILSVRC2012_val_00016199.JPEG:n04372370 +ILSVRC2012_val_00016200.JPEG:n01910747 +ILSVRC2012_val_00016201.JPEG:n03400231 +ILSVRC2012_val_00016202.JPEG:n02444819 +ILSVRC2012_val_00016203.JPEG:n04099969 +ILSVRC2012_val_00016204.JPEG:n03498962 +ILSVRC2012_val_00016205.JPEG:n04154565 +ILSVRC2012_val_00016206.JPEG:n02783161 +ILSVRC2012_val_00016207.JPEG:n03124170 +ILSVRC2012_val_00016208.JPEG:n03417042 +ILSVRC2012_val_00016209.JPEG:n04254120 +ILSVRC2012_val_00016210.JPEG:n07717410 +ILSVRC2012_val_00016211.JPEG:n04372370 +ILSVRC2012_val_00016212.JPEG:n07565083 +ILSVRC2012_val_00016213.JPEG:n03661043 +ILSVRC2012_val_00016214.JPEG:n04074963 +ILSVRC2012_val_00016215.JPEG:n02504458 +ILSVRC2012_val_00016216.JPEG:n03720891 +ILSVRC2012_val_00016217.JPEG:n03445924 +ILSVRC2012_val_00016218.JPEG:n03873416 +ILSVRC2012_val_00016219.JPEG:n03775071 +ILSVRC2012_val_00016220.JPEG:n02443114 +ILSVRC2012_val_00016221.JPEG:n03623198 +ILSVRC2012_val_00016222.JPEG:n03000247 +ILSVRC2012_val_00016223.JPEG:n02423022 +ILSVRC2012_val_00016224.JPEG:n03929660 +ILSVRC2012_val_00016225.JPEG:n02782093 +ILSVRC2012_val_00016226.JPEG:n01930112 +ILSVRC2012_val_00016227.JPEG:n01776313 +ILSVRC2012_val_00016228.JPEG:n03388183 +ILSVRC2012_val_00016229.JPEG:n02133161 +ILSVRC2012_val_00016230.JPEG:n02782093 +ILSVRC2012_val_00016231.JPEG:n03393912 +ILSVRC2012_val_00016232.JPEG:n03794056 +ILSVRC2012_val_00016233.JPEG:n09256479 +ILSVRC2012_val_00016234.JPEG:n07920052 +ILSVRC2012_val_00016235.JPEG:n03384352 +ILSVRC2012_val_00016236.JPEG:n02666196 +ILSVRC2012_val_00016237.JPEG:n02894605 +ILSVRC2012_val_00016238.JPEG:n03476684 +ILSVRC2012_val_00016239.JPEG:n02526121 +ILSVRC2012_val_00016240.JPEG:n02123045 +ILSVRC2012_val_00016241.JPEG:n03673027 +ILSVRC2012_val_00016242.JPEG:n03197337 +ILSVRC2012_val_00016243.JPEG:n02114548 +ILSVRC2012_val_00016244.JPEG:n04599235 +ILSVRC2012_val_00016245.JPEG:n02085936 +ILSVRC2012_val_00016246.JPEG:n02963159 +ILSVRC2012_val_00016247.JPEG:n04258138 +ILSVRC2012_val_00016248.JPEG:n03983396 +ILSVRC2012_val_00016249.JPEG:n03187595 +ILSVRC2012_val_00016250.JPEG:n03290653 +ILSVRC2012_val_00016251.JPEG:n03179701 +ILSVRC2012_val_00016252.JPEG:n01531178 +ILSVRC2012_val_00016253.JPEG:n02398521 +ILSVRC2012_val_00016254.JPEG:n02119789 +ILSVRC2012_val_00016255.JPEG:n02089867 +ILSVRC2012_val_00016256.JPEG:n04548362 +ILSVRC2012_val_00016257.JPEG:n02486410 +ILSVRC2012_val_00016258.JPEG:n01704323 +ILSVRC2012_val_00016259.JPEG:n01494475 +ILSVRC2012_val_00016260.JPEG:n04141327 +ILSVRC2012_val_00016261.JPEG:n02790996 +ILSVRC2012_val_00016262.JPEG:n02056570 +ILSVRC2012_val_00016263.JPEG:n02106166 +ILSVRC2012_val_00016264.JPEG:n02018795 +ILSVRC2012_val_00016265.JPEG:n04523525 +ILSVRC2012_val_00016266.JPEG:n03598930 +ILSVRC2012_val_00016267.JPEG:n04118776 +ILSVRC2012_val_00016268.JPEG:n03662601 +ILSVRC2012_val_00016269.JPEG:n04509417 +ILSVRC2012_val_00016270.JPEG:n02606052 +ILSVRC2012_val_00016271.JPEG:n02966193 +ILSVRC2012_val_00016272.JPEG:n03775071 +ILSVRC2012_val_00016273.JPEG:n02317335 +ILSVRC2012_val_00016274.JPEG:n03146219 +ILSVRC2012_val_00016275.JPEG:n03355925 +ILSVRC2012_val_00016276.JPEG:n02229544 +ILSVRC2012_val_00016277.JPEG:n02443114 +ILSVRC2012_val_00016278.JPEG:n03355925 +ILSVRC2012_val_00016279.JPEG:n04590129 +ILSVRC2012_val_00016280.JPEG:n02804414 +ILSVRC2012_val_00016281.JPEG:n02114367 +ILSVRC2012_val_00016282.JPEG:n03379051 +ILSVRC2012_val_00016283.JPEG:n02138441 +ILSVRC2012_val_00016284.JPEG:n03461385 +ILSVRC2012_val_00016285.JPEG:n04200800 +ILSVRC2012_val_00016286.JPEG:n03584829 +ILSVRC2012_val_00016287.JPEG:n01755581 +ILSVRC2012_val_00016288.JPEG:n04335435 +ILSVRC2012_val_00016289.JPEG:n03127747 +ILSVRC2012_val_00016290.JPEG:n04263257 +ILSVRC2012_val_00016291.JPEG:n04192698 +ILSVRC2012_val_00016292.JPEG:n01622779 +ILSVRC2012_val_00016293.JPEG:n02422699 +ILSVRC2012_val_00016294.JPEG:n02107683 +ILSVRC2012_val_00016295.JPEG:n04532670 +ILSVRC2012_val_00016296.JPEG:n02906734 +ILSVRC2012_val_00016297.JPEG:n02804414 +ILSVRC2012_val_00016298.JPEG:n12768682 +ILSVRC2012_val_00016299.JPEG:n02108089 +ILSVRC2012_val_00016300.JPEG:n02909870 +ILSVRC2012_val_00016301.JPEG:n03837869 +ILSVRC2012_val_00016302.JPEG:n02113186 +ILSVRC2012_val_00016303.JPEG:n02112350 +ILSVRC2012_val_00016304.JPEG:n01677366 +ILSVRC2012_val_00016305.JPEG:n03630383 +ILSVRC2012_val_00016306.JPEG:n02526121 +ILSVRC2012_val_00016307.JPEG:n02840245 +ILSVRC2012_val_00016308.JPEG:n01687978 +ILSVRC2012_val_00016309.JPEG:n04515003 +ILSVRC2012_val_00016310.JPEG:n15075141 +ILSVRC2012_val_00016311.JPEG:n02841315 +ILSVRC2012_val_00016312.JPEG:n02422106 +ILSVRC2012_val_00016313.JPEG:n02783161 +ILSVRC2012_val_00016314.JPEG:n02814533 +ILSVRC2012_val_00016315.JPEG:n02102177 +ILSVRC2012_val_00016316.JPEG:n02415577 +ILSVRC2012_val_00016317.JPEG:n03782006 +ILSVRC2012_val_00016318.JPEG:n01770081 +ILSVRC2012_val_00016319.JPEG:n02114548 +ILSVRC2012_val_00016320.JPEG:n03958227 +ILSVRC2012_val_00016321.JPEG:n01728920 +ILSVRC2012_val_00016322.JPEG:n03494278 +ILSVRC2012_val_00016323.JPEG:n01873310 +ILSVRC2012_val_00016324.JPEG:n02894605 +ILSVRC2012_val_00016325.JPEG:n01833805 +ILSVRC2012_val_00016326.JPEG:n03160309 +ILSVRC2012_val_00016327.JPEG:n04458633 +ILSVRC2012_val_00016328.JPEG:n03223299 +ILSVRC2012_val_00016329.JPEG:n12620546 +ILSVRC2012_val_00016330.JPEG:n12998815 +ILSVRC2012_val_00016331.JPEG:n01496331 +ILSVRC2012_val_00016332.JPEG:n04461696 +ILSVRC2012_val_00016333.JPEG:n01981276 +ILSVRC2012_val_00016334.JPEG:n03595614 +ILSVRC2012_val_00016335.JPEG:n02101388 +ILSVRC2012_val_00016336.JPEG:n03937543 +ILSVRC2012_val_00016337.JPEG:n03100240 +ILSVRC2012_val_00016338.JPEG:n03791053 +ILSVRC2012_val_00016339.JPEG:n04613696 +ILSVRC2012_val_00016340.JPEG:n02134084 +ILSVRC2012_val_00016341.JPEG:n04141975 +ILSVRC2012_val_00016342.JPEG:n02093859 +ILSVRC2012_val_00016343.JPEG:n03125729 +ILSVRC2012_val_00016344.JPEG:n02326432 +ILSVRC2012_val_00016345.JPEG:n03680355 +ILSVRC2012_val_00016346.JPEG:n03998194 +ILSVRC2012_val_00016347.JPEG:n01494475 +ILSVRC2012_val_00016348.JPEG:n02342885 +ILSVRC2012_val_00016349.JPEG:n03976657 +ILSVRC2012_val_00016350.JPEG:n01819313 +ILSVRC2012_val_00016351.JPEG:n04606251 +ILSVRC2012_val_00016352.JPEG:n01740131 +ILSVRC2012_val_00016353.JPEG:n02797295 +ILSVRC2012_val_00016354.JPEG:n02123394 +ILSVRC2012_val_00016355.JPEG:n02169497 +ILSVRC2012_val_00016356.JPEG:n03630383 +ILSVRC2012_val_00016357.JPEG:n01689811 +ILSVRC2012_val_00016358.JPEG:n03950228 +ILSVRC2012_val_00016359.JPEG:n07584110 +ILSVRC2012_val_00016360.JPEG:n04591713 +ILSVRC2012_val_00016361.JPEG:n04127249 +ILSVRC2012_val_00016362.JPEG:n12144580 +ILSVRC2012_val_00016363.JPEG:n07831146 +ILSVRC2012_val_00016364.JPEG:n03791053 +ILSVRC2012_val_00016365.JPEG:n02808440 +ILSVRC2012_val_00016366.JPEG:n02793495 +ILSVRC2012_val_00016367.JPEG:n02437312 +ILSVRC2012_val_00016368.JPEG:n02138441 +ILSVRC2012_val_00016369.JPEG:n02111500 +ILSVRC2012_val_00016370.JPEG:n02109961 +ILSVRC2012_val_00016371.JPEG:n03459775 +ILSVRC2012_val_00016372.JPEG:n03126707 +ILSVRC2012_val_00016373.JPEG:n03388549 +ILSVRC2012_val_00016374.JPEG:n02096294 +ILSVRC2012_val_00016375.JPEG:n03961711 +ILSVRC2012_val_00016376.JPEG:n04209133 +ILSVRC2012_val_00016377.JPEG:n04243546 +ILSVRC2012_val_00016378.JPEG:n02791270 +ILSVRC2012_val_00016379.JPEG:n01685808 +ILSVRC2012_val_00016380.JPEG:n02965783 +ILSVRC2012_val_00016381.JPEG:n03775546 +ILSVRC2012_val_00016382.JPEG:n02074367 +ILSVRC2012_val_00016383.JPEG:n03775546 +ILSVRC2012_val_00016384.JPEG:n03584254 +ILSVRC2012_val_00016385.JPEG:n02119789 +ILSVRC2012_val_00016386.JPEG:n02437312 +ILSVRC2012_val_00016387.JPEG:n03888257 +ILSVRC2012_val_00016388.JPEG:n03187595 +ILSVRC2012_val_00016389.JPEG:n02123045 +ILSVRC2012_val_00016390.JPEG:n03937543 +ILSVRC2012_val_00016391.JPEG:n02412080 +ILSVRC2012_val_00016392.JPEG:n01729322 +ILSVRC2012_val_00016393.JPEG:n03908714 +ILSVRC2012_val_00016394.JPEG:n02125311 +ILSVRC2012_val_00016395.JPEG:n01494475 +ILSVRC2012_val_00016396.JPEG:n02894605 +ILSVRC2012_val_00016397.JPEG:n03908618 +ILSVRC2012_val_00016398.JPEG:n02114855 +ILSVRC2012_val_00016399.JPEG:n02123159 +ILSVRC2012_val_00016400.JPEG:n03598930 +ILSVRC2012_val_00016401.JPEG:n02107142 +ILSVRC2012_val_00016402.JPEG:n03290653 +ILSVRC2012_val_00016403.JPEG:n02791124 +ILSVRC2012_val_00016404.JPEG:n03803284 +ILSVRC2012_val_00016405.JPEG:n03937543 +ILSVRC2012_val_00016406.JPEG:n03388043 +ILSVRC2012_val_00016407.JPEG:n03131574 +ILSVRC2012_val_00016408.JPEG:n02788148 +ILSVRC2012_val_00016409.JPEG:n02106382 +ILSVRC2012_val_00016410.JPEG:n04467665 +ILSVRC2012_val_00016411.JPEG:n02100877 +ILSVRC2012_val_00016412.JPEG:n04330267 +ILSVRC2012_val_00016413.JPEG:n03697007 +ILSVRC2012_val_00016414.JPEG:n03710721 +ILSVRC2012_val_00016415.JPEG:n02403003 +ILSVRC2012_val_00016416.JPEG:n02108089 +ILSVRC2012_val_00016417.JPEG:n03017168 +ILSVRC2012_val_00016418.JPEG:n03733281 +ILSVRC2012_val_00016419.JPEG:n03792972 +ILSVRC2012_val_00016420.JPEG:n02105056 +ILSVRC2012_val_00016421.JPEG:n01806567 +ILSVRC2012_val_00016422.JPEG:n01630670 +ILSVRC2012_val_00016423.JPEG:n03337140 +ILSVRC2012_val_00016424.JPEG:n03467068 +ILSVRC2012_val_00016425.JPEG:n01873310 +ILSVRC2012_val_00016426.JPEG:n02398521 +ILSVRC2012_val_00016427.JPEG:n02013706 +ILSVRC2012_val_00016428.JPEG:n04120489 +ILSVRC2012_val_00016429.JPEG:n02708093 +ILSVRC2012_val_00016430.JPEG:n02110341 +ILSVRC2012_val_00016431.JPEG:n03770679 +ILSVRC2012_val_00016432.JPEG:n02480495 +ILSVRC2012_val_00016433.JPEG:n03450230 +ILSVRC2012_val_00016434.JPEG:n03584254 +ILSVRC2012_val_00016435.JPEG:n02823750 +ILSVRC2012_val_00016436.JPEG:n04127249 +ILSVRC2012_val_00016437.JPEG:n02410509 +ILSVRC2012_val_00016438.JPEG:n04562935 +ILSVRC2012_val_00016439.JPEG:n04019541 +ILSVRC2012_val_00016440.JPEG:n04613696 +ILSVRC2012_val_00016441.JPEG:n01632777 +ILSVRC2012_val_00016442.JPEG:n07836838 +ILSVRC2012_val_00016443.JPEG:n02114855 +ILSVRC2012_val_00016444.JPEG:n02100236 +ILSVRC2012_val_00016445.JPEG:n02102318 +ILSVRC2012_val_00016446.JPEG:n07831146 +ILSVRC2012_val_00016447.JPEG:n03742115 +ILSVRC2012_val_00016448.JPEG:n03662601 +ILSVRC2012_val_00016449.JPEG:n03720891 +ILSVRC2012_val_00016450.JPEG:n02804610 +ILSVRC2012_val_00016451.JPEG:n02107142 +ILSVRC2012_val_00016452.JPEG:n03733131 +ILSVRC2012_val_00016453.JPEG:n03791053 +ILSVRC2012_val_00016454.JPEG:n03991062 +ILSVRC2012_val_00016455.JPEG:n02808304 +ILSVRC2012_val_00016456.JPEG:n03594945 +ILSVRC2012_val_00016457.JPEG:n02749479 +ILSVRC2012_val_00016458.JPEG:n04562935 +ILSVRC2012_val_00016459.JPEG:n02134084 +ILSVRC2012_val_00016460.JPEG:n02342885 +ILSVRC2012_val_00016461.JPEG:n03538406 +ILSVRC2012_val_00016462.JPEG:n02107683 +ILSVRC2012_val_00016463.JPEG:n02012849 +ILSVRC2012_val_00016464.JPEG:n01682714 +ILSVRC2012_val_00016465.JPEG:n02988304 +ILSVRC2012_val_00016466.JPEG:n07932039 +ILSVRC2012_val_00016467.JPEG:n02206856 +ILSVRC2012_val_00016468.JPEG:n03447447 +ILSVRC2012_val_00016469.JPEG:n01753488 +ILSVRC2012_val_00016470.JPEG:n01755581 +ILSVRC2012_val_00016471.JPEG:n02119022 +ILSVRC2012_val_00016472.JPEG:n04597913 +ILSVRC2012_val_00016473.JPEG:n03314780 +ILSVRC2012_val_00016474.JPEG:n02865351 +ILSVRC2012_val_00016475.JPEG:n03459775 +ILSVRC2012_val_00016476.JPEG:n01530575 +ILSVRC2012_val_00016477.JPEG:n04335435 +ILSVRC2012_val_00016478.JPEG:n09288635 +ILSVRC2012_val_00016479.JPEG:n02769748 +ILSVRC2012_val_00016480.JPEG:n02256656 +ILSVRC2012_val_00016481.JPEG:n03131574 +ILSVRC2012_val_00016482.JPEG:n03770439 +ILSVRC2012_val_00016483.JPEG:n02123045 +ILSVRC2012_val_00016484.JPEG:n02096177 +ILSVRC2012_val_00016485.JPEG:n04131690 +ILSVRC2012_val_00016486.JPEG:n02397096 +ILSVRC2012_val_00016487.JPEG:n01798484 +ILSVRC2012_val_00016488.JPEG:n02107574 +ILSVRC2012_val_00016489.JPEG:n02113186 +ILSVRC2012_val_00016490.JPEG:n01855672 +ILSVRC2012_val_00016491.JPEG:n03791053 +ILSVRC2012_val_00016492.JPEG:n03770679 +ILSVRC2012_val_00016493.JPEG:n01983481 +ILSVRC2012_val_00016494.JPEG:n02093256 +ILSVRC2012_val_00016495.JPEG:n01968897 +ILSVRC2012_val_00016496.JPEG:n02692877 +ILSVRC2012_val_00016497.JPEG:n02356798 +ILSVRC2012_val_00016498.JPEG:n07875152 +ILSVRC2012_val_00016499.JPEG:n02107312 +ILSVRC2012_val_00016500.JPEG:n02837789 +ILSVRC2012_val_00016501.JPEG:n03042490 +ILSVRC2012_val_00016502.JPEG:n03188531 +ILSVRC2012_val_00016503.JPEG:n03447721 +ILSVRC2012_val_00016504.JPEG:n02825657 +ILSVRC2012_val_00016505.JPEG:n03868242 +ILSVRC2012_val_00016506.JPEG:n04552348 +ILSVRC2012_val_00016507.JPEG:n01770081 +ILSVRC2012_val_00016508.JPEG:n02095314 +ILSVRC2012_val_00016509.JPEG:n04204347 +ILSVRC2012_val_00016510.JPEG:n02087394 +ILSVRC2012_val_00016511.JPEG:n04065272 +ILSVRC2012_val_00016512.JPEG:n02132136 +ILSVRC2012_val_00016513.JPEG:n02134418 +ILSVRC2012_val_00016514.JPEG:n01632777 +ILSVRC2012_val_00016515.JPEG:n04325704 +ILSVRC2012_val_00016516.JPEG:n03776460 +ILSVRC2012_val_00016517.JPEG:n01955084 +ILSVRC2012_val_00016518.JPEG:n02129604 +ILSVRC2012_val_00016519.JPEG:n01644900 +ILSVRC2012_val_00016520.JPEG:n02101006 +ILSVRC2012_val_00016521.JPEG:n04357314 +ILSVRC2012_val_00016522.JPEG:n12985857 +ILSVRC2012_val_00016523.JPEG:n03670208 +ILSVRC2012_val_00016524.JPEG:n07760859 +ILSVRC2012_val_00016525.JPEG:n04067472 +ILSVRC2012_val_00016526.JPEG:n02099849 +ILSVRC2012_val_00016527.JPEG:n03770679 +ILSVRC2012_val_00016528.JPEG:n02978881 +ILSVRC2012_val_00016529.JPEG:n03623198 +ILSVRC2012_val_00016530.JPEG:n03717622 +ILSVRC2012_val_00016531.JPEG:n04536866 +ILSVRC2012_val_00016532.JPEG:n02835271 +ILSVRC2012_val_00016533.JPEG:n07717410 +ILSVRC2012_val_00016534.JPEG:n04429376 +ILSVRC2012_val_00016535.JPEG:n02869837 +ILSVRC2012_val_00016536.JPEG:n03124170 +ILSVRC2012_val_00016537.JPEG:n01632458 +ILSVRC2012_val_00016538.JPEG:n01531178 +ILSVRC2012_val_00016539.JPEG:n03127925 +ILSVRC2012_val_00016540.JPEG:n02097047 +ILSVRC2012_val_00016541.JPEG:n03950228 +ILSVRC2012_val_00016542.JPEG:n03028079 +ILSVRC2012_val_00016543.JPEG:n02107312 +ILSVRC2012_val_00016544.JPEG:n13052670 +ILSVRC2012_val_00016545.JPEG:n02090721 +ILSVRC2012_val_00016546.JPEG:n07711569 +ILSVRC2012_val_00016547.JPEG:n02091831 +ILSVRC2012_val_00016548.JPEG:n01530575 +ILSVRC2012_val_00016549.JPEG:n04146614 +ILSVRC2012_val_00016550.JPEG:n01667114 +ILSVRC2012_val_00016551.JPEG:n03958227 +ILSVRC2012_val_00016552.JPEG:n02098286 +ILSVRC2012_val_00016553.JPEG:n07871810 +ILSVRC2012_val_00016554.JPEG:n01980166 +ILSVRC2012_val_00016555.JPEG:n02412080 +ILSVRC2012_val_00016556.JPEG:n02500267 +ILSVRC2012_val_00016557.JPEG:n01924916 +ILSVRC2012_val_00016558.JPEG:n04254680 +ILSVRC2012_val_00016559.JPEG:n02480495 +ILSVRC2012_val_00016560.JPEG:n01774384 +ILSVRC2012_val_00016561.JPEG:n03216828 +ILSVRC2012_val_00016562.JPEG:n07711569 +ILSVRC2012_val_00016563.JPEG:n03026506 +ILSVRC2012_val_00016564.JPEG:n01749939 +ILSVRC2012_val_00016565.JPEG:n03344393 +ILSVRC2012_val_00016566.JPEG:n03938244 +ILSVRC2012_val_00016567.JPEG:n02098105 +ILSVRC2012_val_00016568.JPEG:n01986214 +ILSVRC2012_val_00016569.JPEG:n01917289 +ILSVRC2012_val_00016570.JPEG:n04418357 +ILSVRC2012_val_00016571.JPEG:n02058221 +ILSVRC2012_val_00016572.JPEG:n02106030 +ILSVRC2012_val_00016573.JPEG:n02966193 +ILSVRC2012_val_00016574.JPEG:n03032252 +ILSVRC2012_val_00016575.JPEG:n02206856 +ILSVRC2012_val_00016576.JPEG:n03063599 +ILSVRC2012_val_00016577.JPEG:n02107312 +ILSVRC2012_val_00016578.JPEG:n03843555 +ILSVRC2012_val_00016579.JPEG:n02108551 +ILSVRC2012_val_00016580.JPEG:n01855672 +ILSVRC2012_val_00016581.JPEG:n02107142 +ILSVRC2012_val_00016582.JPEG:n02102040 +ILSVRC2012_val_00016583.JPEG:n04357314 +ILSVRC2012_val_00016584.JPEG:n04505470 +ILSVRC2012_val_00016585.JPEG:n03529860 +ILSVRC2012_val_00016586.JPEG:n02437312 +ILSVRC2012_val_00016587.JPEG:n02129604 +ILSVRC2012_val_00016588.JPEG:n03773504 +ILSVRC2012_val_00016589.JPEG:n02100877 +ILSVRC2012_val_00016590.JPEG:n03877472 +ILSVRC2012_val_00016591.JPEG:n04501370 +ILSVRC2012_val_00016592.JPEG:n07880968 +ILSVRC2012_val_00016593.JPEG:n04458633 +ILSVRC2012_val_00016594.JPEG:n02167151 +ILSVRC2012_val_00016595.JPEG:n03721384 +ILSVRC2012_val_00016596.JPEG:n02102480 +ILSVRC2012_val_00016597.JPEG:n07579787 +ILSVRC2012_val_00016598.JPEG:n02123394 +ILSVRC2012_val_00016599.JPEG:n02484975 +ILSVRC2012_val_00016600.JPEG:n03942813 +ILSVRC2012_val_00016601.JPEG:n04270147 +ILSVRC2012_val_00016602.JPEG:n03777568 +ILSVRC2012_val_00016603.JPEG:n02085782 +ILSVRC2012_val_00016604.JPEG:n01729977 +ILSVRC2012_val_00016605.JPEG:n04404412 +ILSVRC2012_val_00016606.JPEG:n04311174 +ILSVRC2012_val_00016607.JPEG:n03160309 +ILSVRC2012_val_00016608.JPEG:n02454379 +ILSVRC2012_val_00016609.JPEG:n02096294 +ILSVRC2012_val_00016610.JPEG:n04065272 +ILSVRC2012_val_00016611.JPEG:n02483362 +ILSVRC2012_val_00016612.JPEG:n02364673 +ILSVRC2012_val_00016613.JPEG:n03100240 +ILSVRC2012_val_00016614.JPEG:n07873807 +ILSVRC2012_val_00016615.JPEG:n03594734 +ILSVRC2012_val_00016616.JPEG:n04344873 +ILSVRC2012_val_00016617.JPEG:n07590611 +ILSVRC2012_val_00016618.JPEG:n01883070 +ILSVRC2012_val_00016619.JPEG:n03770439 +ILSVRC2012_val_00016620.JPEG:n03141823 +ILSVRC2012_val_00016621.JPEG:n02133161 +ILSVRC2012_val_00016622.JPEG:n01689811 +ILSVRC2012_val_00016623.JPEG:n01833805 +ILSVRC2012_val_00016624.JPEG:n02814860 +ILSVRC2012_val_00016625.JPEG:n04367480 +ILSVRC2012_val_00016626.JPEG:n03710637 +ILSVRC2012_val_00016627.JPEG:n07714571 +ILSVRC2012_val_00016628.JPEG:n02071294 +ILSVRC2012_val_00016629.JPEG:n01768244 +ILSVRC2012_val_00016630.JPEG:n03388183 +ILSVRC2012_val_00016631.JPEG:n01847000 +ILSVRC2012_val_00016632.JPEG:n03325584 +ILSVRC2012_val_00016633.JPEG:n01667114 +ILSVRC2012_val_00016634.JPEG:n02236044 +ILSVRC2012_val_00016635.JPEG:n04141327 +ILSVRC2012_val_00016636.JPEG:n03467068 +ILSVRC2012_val_00016637.JPEG:n01687978 +ILSVRC2012_val_00016638.JPEG:n04285008 +ILSVRC2012_val_00016639.JPEG:n03483316 +ILSVRC2012_val_00016640.JPEG:n03447447 +ILSVRC2012_val_00016641.JPEG:n02264363 +ILSVRC2012_val_00016642.JPEG:n02097209 +ILSVRC2012_val_00016643.JPEG:n04501370 +ILSVRC2012_val_00016644.JPEG:n09468604 +ILSVRC2012_val_00016645.JPEG:n02930766 +ILSVRC2012_val_00016646.JPEG:n01917289 +ILSVRC2012_val_00016647.JPEG:n04554684 +ILSVRC2012_val_00016648.JPEG:n02979186 +ILSVRC2012_val_00016649.JPEG:n02442845 +ILSVRC2012_val_00016650.JPEG:n03345487 +ILSVRC2012_val_00016651.JPEG:n02486410 +ILSVRC2012_val_00016652.JPEG:n02841315 +ILSVRC2012_val_00016653.JPEG:n03899768 +ILSVRC2012_val_00016654.JPEG:n09399592 +ILSVRC2012_val_00016655.JPEG:n03344393 +ILSVRC2012_val_00016656.JPEG:n02088364 +ILSVRC2012_val_00016657.JPEG:n03763968 +ILSVRC2012_val_00016658.JPEG:n02105162 +ILSVRC2012_val_00016659.JPEG:n04235860 +ILSVRC2012_val_00016660.JPEG:n03903868 +ILSVRC2012_val_00016661.JPEG:n09428293 +ILSVRC2012_val_00016662.JPEG:n03661043 +ILSVRC2012_val_00016663.JPEG:n03249569 +ILSVRC2012_val_00016664.JPEG:n02268443 +ILSVRC2012_val_00016665.JPEG:n02444819 +ILSVRC2012_val_00016666.JPEG:n02116738 +ILSVRC2012_val_00016667.JPEG:n03902125 +ILSVRC2012_val_00016668.JPEG:n02093991 +ILSVRC2012_val_00016669.JPEG:n02110185 +ILSVRC2012_val_00016670.JPEG:n03832673 +ILSVRC2012_val_00016671.JPEG:n03983396 +ILSVRC2012_val_00016672.JPEG:n07716358 +ILSVRC2012_val_00016673.JPEG:n02113712 +ILSVRC2012_val_00016674.JPEG:n03887697 +ILSVRC2012_val_00016675.JPEG:n03424325 +ILSVRC2012_val_00016676.JPEG:n03958227 +ILSVRC2012_val_00016677.JPEG:n01534433 +ILSVRC2012_val_00016678.JPEG:n02086646 +ILSVRC2012_val_00016679.JPEG:n04591713 +ILSVRC2012_val_00016680.JPEG:n07753113 +ILSVRC2012_val_00016681.JPEG:n03841143 +ILSVRC2012_val_00016682.JPEG:n02790996 +ILSVRC2012_val_00016683.JPEG:n02165456 +ILSVRC2012_val_00016684.JPEG:n02009229 +ILSVRC2012_val_00016685.JPEG:n02814860 +ILSVRC2012_val_00016686.JPEG:n04462240 +ILSVRC2012_val_00016687.JPEG:n02730930 +ILSVRC2012_val_00016688.JPEG:n02085620 +ILSVRC2012_val_00016689.JPEG:n02098413 +ILSVRC2012_val_00016690.JPEG:n03337140 +ILSVRC2012_val_00016691.JPEG:n02807133 +ILSVRC2012_val_00016692.JPEG:n04263257 +ILSVRC2012_val_00016693.JPEG:n02108422 +ILSVRC2012_val_00016694.JPEG:n02138441 +ILSVRC2012_val_00016695.JPEG:n01630670 +ILSVRC2012_val_00016696.JPEG:n04008634 +ILSVRC2012_val_00016697.JPEG:n02113799 +ILSVRC2012_val_00016698.JPEG:n02643566 +ILSVRC2012_val_00016699.JPEG:n12057211 +ILSVRC2012_val_00016700.JPEG:n01665541 +ILSVRC2012_val_00016701.JPEG:n04404412 +ILSVRC2012_val_00016702.JPEG:n03691459 +ILSVRC2012_val_00016703.JPEG:n01729977 +ILSVRC2012_val_00016704.JPEG:n03290653 +ILSVRC2012_val_00016705.JPEG:n01924916 +ILSVRC2012_val_00016706.JPEG:n02486410 +ILSVRC2012_val_00016707.JPEG:n04332243 +ILSVRC2012_val_00016708.JPEG:n13052670 +ILSVRC2012_val_00016709.JPEG:n03598930 +ILSVRC2012_val_00016710.JPEG:n02437616 +ILSVRC2012_val_00016711.JPEG:n02093991 +ILSVRC2012_val_00016712.JPEG:n01729977 +ILSVRC2012_val_00016713.JPEG:n02115641 +ILSVRC2012_val_00016714.JPEG:n02825657 +ILSVRC2012_val_00016715.JPEG:n02786058 +ILSVRC2012_val_00016716.JPEG:n02788148 +ILSVRC2012_val_00016717.JPEG:n02094258 +ILSVRC2012_val_00016718.JPEG:n02793495 +ILSVRC2012_val_00016719.JPEG:n03388043 +ILSVRC2012_val_00016720.JPEG:n02128757 +ILSVRC2012_val_00016721.JPEG:n02443484 +ILSVRC2012_val_00016722.JPEG:n02088094 +ILSVRC2012_val_00016723.JPEG:n03110669 +ILSVRC2012_val_00016724.JPEG:n01985128 +ILSVRC2012_val_00016725.JPEG:n07714990 +ILSVRC2012_val_00016726.JPEG:n02869837 +ILSVRC2012_val_00016727.JPEG:n03595614 +ILSVRC2012_val_00016728.JPEG:n04592741 +ILSVRC2012_val_00016729.JPEG:n02127052 +ILSVRC2012_val_00016730.JPEG:n07880968 +ILSVRC2012_val_00016731.JPEG:n02643566 +ILSVRC2012_val_00016732.JPEG:n09256479 +ILSVRC2012_val_00016733.JPEG:n02356798 +ILSVRC2012_val_00016734.JPEG:n02509815 +ILSVRC2012_val_00016735.JPEG:n04487394 +ILSVRC2012_val_00016736.JPEG:n03721384 +ILSVRC2012_val_00016737.JPEG:n01728572 +ILSVRC2012_val_00016738.JPEG:n02992211 +ILSVRC2012_val_00016739.JPEG:n03877845 +ILSVRC2012_val_00016740.JPEG:n02231487 +ILSVRC2012_val_00016741.JPEG:n02445715 +ILSVRC2012_val_00016742.JPEG:n02095570 +ILSVRC2012_val_00016743.JPEG:n04579145 +ILSVRC2012_val_00016744.JPEG:n03706229 +ILSVRC2012_val_00016745.JPEG:n02107574 +ILSVRC2012_val_00016746.JPEG:n01833805 +ILSVRC2012_val_00016747.JPEG:n01629819 +ILSVRC2012_val_00016748.JPEG:n03445777 +ILSVRC2012_val_00016749.JPEG:n03710721 +ILSVRC2012_val_00016750.JPEG:n03014705 +ILSVRC2012_val_00016751.JPEG:n04336792 +ILSVRC2012_val_00016752.JPEG:n04311174 +ILSVRC2012_val_00016753.JPEG:n03724870 +ILSVRC2012_val_00016754.JPEG:n03920288 +ILSVRC2012_val_00016755.JPEG:n03063689 +ILSVRC2012_val_00016756.JPEG:n03908618 +ILSVRC2012_val_00016757.JPEG:n02085620 +ILSVRC2012_val_00016758.JPEG:n02699494 +ILSVRC2012_val_00016759.JPEG:n02096437 +ILSVRC2012_val_00016760.JPEG:n03804744 +ILSVRC2012_val_00016761.JPEG:n04209239 +ILSVRC2012_val_00016762.JPEG:n03249569 +ILSVRC2012_val_00016763.JPEG:n11939491 +ILSVRC2012_val_00016764.JPEG:n01882714 +ILSVRC2012_val_00016765.JPEG:n02129165 +ILSVRC2012_val_00016766.JPEG:n03773504 +ILSVRC2012_val_00016767.JPEG:n04346328 +ILSVRC2012_val_00016768.JPEG:n02102040 +ILSVRC2012_val_00016769.JPEG:n12620546 +ILSVRC2012_val_00016770.JPEG:n02177972 +ILSVRC2012_val_00016771.JPEG:n02066245 +ILSVRC2012_val_00016772.JPEG:n03492542 +ILSVRC2012_val_00016773.JPEG:n02090721 +ILSVRC2012_val_00016774.JPEG:n04482393 +ILSVRC2012_val_00016775.JPEG:n01914609 +ILSVRC2012_val_00016776.JPEG:n02174001 +ILSVRC2012_val_00016777.JPEG:n02233338 +ILSVRC2012_val_00016778.JPEG:n01693334 +ILSVRC2012_val_00016779.JPEG:n01665541 +ILSVRC2012_val_00016780.JPEG:n02280649 +ILSVRC2012_val_00016781.JPEG:n01514668 +ILSVRC2012_val_00016782.JPEG:n01641577 +ILSVRC2012_val_00016783.JPEG:n02107683 +ILSVRC2012_val_00016784.JPEG:n04040759 +ILSVRC2012_val_00016785.JPEG:n03355925 +ILSVRC2012_val_00016786.JPEG:n04579432 +ILSVRC2012_val_00016787.JPEG:n02280649 +ILSVRC2012_val_00016788.JPEG:n02361337 +ILSVRC2012_val_00016789.JPEG:n03937543 +ILSVRC2012_val_00016790.JPEG:n03891251 +ILSVRC2012_val_00016791.JPEG:n02492035 +ILSVRC2012_val_00016792.JPEG:n03759954 +ILSVRC2012_val_00016793.JPEG:n03763968 +ILSVRC2012_val_00016794.JPEG:n01582220 +ILSVRC2012_val_00016795.JPEG:n03866082 +ILSVRC2012_val_00016796.JPEG:n04086273 +ILSVRC2012_val_00016797.JPEG:n04330267 +ILSVRC2012_val_00016798.JPEG:n04476259 +ILSVRC2012_val_00016799.JPEG:n04118776 +ILSVRC2012_val_00016800.JPEG:n03180011 +ILSVRC2012_val_00016801.JPEG:n03838899 +ILSVRC2012_val_00016802.JPEG:n03627232 +ILSVRC2012_val_00016803.JPEG:n04264628 +ILSVRC2012_val_00016804.JPEG:n02101006 +ILSVRC2012_val_00016805.JPEG:n02113624 +ILSVRC2012_val_00016806.JPEG:n02395406 +ILSVRC2012_val_00016807.JPEG:n01675722 +ILSVRC2012_val_00016808.JPEG:n04090263 +ILSVRC2012_val_00016809.JPEG:n03785016 +ILSVRC2012_val_00016810.JPEG:n02137549 +ILSVRC2012_val_00016811.JPEG:n02277742 +ILSVRC2012_val_00016812.JPEG:n03642806 +ILSVRC2012_val_00016813.JPEG:n07718472 +ILSVRC2012_val_00016814.JPEG:n03447447 +ILSVRC2012_val_00016815.JPEG:n03792782 +ILSVRC2012_val_00016816.JPEG:n04008634 +ILSVRC2012_val_00016817.JPEG:n04254777 +ILSVRC2012_val_00016818.JPEG:n01631663 +ILSVRC2012_val_00016819.JPEG:n04254680 +ILSVRC2012_val_00016820.JPEG:n02074367 +ILSVRC2012_val_00016821.JPEG:n01744401 +ILSVRC2012_val_00016822.JPEG:n03127747 +ILSVRC2012_val_00016823.JPEG:n02190166 +ILSVRC2012_val_00016824.JPEG:n03623198 +ILSVRC2012_val_00016825.JPEG:n02607072 +ILSVRC2012_val_00016826.JPEG:n02877765 +ILSVRC2012_val_00016827.JPEG:n02790996 +ILSVRC2012_val_00016828.JPEG:n02992529 +ILSVRC2012_val_00016829.JPEG:n02492660 +ILSVRC2012_val_00016830.JPEG:n02117135 +ILSVRC2012_val_00016831.JPEG:n01580077 +ILSVRC2012_val_00016832.JPEG:n03028079 +ILSVRC2012_val_00016833.JPEG:n02102040 +ILSVRC2012_val_00016834.JPEG:n01494475 +ILSVRC2012_val_00016835.JPEG:n04461696 +ILSVRC2012_val_00016836.JPEG:n01917289 +ILSVRC2012_val_00016837.JPEG:n04146614 +ILSVRC2012_val_00016838.JPEG:n04004767 +ILSVRC2012_val_00016839.JPEG:n02906734 +ILSVRC2012_val_00016840.JPEG:n01560419 +ILSVRC2012_val_00016841.JPEG:n02085936 +ILSVRC2012_val_00016842.JPEG:n12267677 +ILSVRC2012_val_00016843.JPEG:n03075370 +ILSVRC2012_val_00016844.JPEG:n01682714 +ILSVRC2012_val_00016845.JPEG:n02669723 +ILSVRC2012_val_00016846.JPEG:n01751748 +ILSVRC2012_val_00016847.JPEG:n02999410 +ILSVRC2012_val_00016848.JPEG:n10148035 +ILSVRC2012_val_00016849.JPEG:n02797295 +ILSVRC2012_val_00016850.JPEG:n03958227 +ILSVRC2012_val_00016851.JPEG:n03134739 +ILSVRC2012_val_00016852.JPEG:n01860187 +ILSVRC2012_val_00016853.JPEG:n02443114 +ILSVRC2012_val_00016854.JPEG:n03028079 +ILSVRC2012_val_00016855.JPEG:n03495258 +ILSVRC2012_val_00016856.JPEG:n03787032 +ILSVRC2012_val_00016857.JPEG:n02108089 +ILSVRC2012_val_00016858.JPEG:n01687978 +ILSVRC2012_val_00016859.JPEG:n01484850 +ILSVRC2012_val_00016860.JPEG:n02098105 +ILSVRC2012_val_00016861.JPEG:n03942813 +ILSVRC2012_val_00016862.JPEG:n02109525 +ILSVRC2012_val_00016863.JPEG:n04613696 +ILSVRC2012_val_00016864.JPEG:n01631663 +ILSVRC2012_val_00016865.JPEG:n09835506 +ILSVRC2012_val_00016866.JPEG:n01784675 +ILSVRC2012_val_00016867.JPEG:n02137549 +ILSVRC2012_val_00016868.JPEG:n09472597 +ILSVRC2012_val_00016869.JPEG:n02895154 +ILSVRC2012_val_00016870.JPEG:n03676483 +ILSVRC2012_val_00016871.JPEG:n04209239 +ILSVRC2012_val_00016872.JPEG:n01784675 +ILSVRC2012_val_00016873.JPEG:n03028079 +ILSVRC2012_val_00016874.JPEG:n03355925 +ILSVRC2012_val_00016875.JPEG:n03483316 +ILSVRC2012_val_00016876.JPEG:n03337140 +ILSVRC2012_val_00016877.JPEG:n03495258 +ILSVRC2012_val_00016878.JPEG:n04311004 +ILSVRC2012_val_00016879.JPEG:n04270147 +ILSVRC2012_val_00016880.JPEG:n03791053 +ILSVRC2012_val_00016881.JPEG:n02488702 +ILSVRC2012_val_00016882.JPEG:n02895154 +ILSVRC2012_val_00016883.JPEG:n02100583 +ILSVRC2012_val_00016884.JPEG:n10565667 +ILSVRC2012_val_00016885.JPEG:n04548280 +ILSVRC2012_val_00016886.JPEG:n02091134 +ILSVRC2012_val_00016887.JPEG:n01806567 +ILSVRC2012_val_00016888.JPEG:n02264363 +ILSVRC2012_val_00016889.JPEG:n02708093 +ILSVRC2012_val_00016890.JPEG:n02111277 +ILSVRC2012_val_00016891.JPEG:n02692877 +ILSVRC2012_val_00016892.JPEG:n03837869 +ILSVRC2012_val_00016893.JPEG:n03240683 +ILSVRC2012_val_00016894.JPEG:n03773504 +ILSVRC2012_val_00016895.JPEG:n03706229 +ILSVRC2012_val_00016896.JPEG:n03742115 +ILSVRC2012_val_00016897.JPEG:n01734418 +ILSVRC2012_val_00016898.JPEG:n12998815 +ILSVRC2012_val_00016899.JPEG:n03452741 +ILSVRC2012_val_00016900.JPEG:n06596364 +ILSVRC2012_val_00016901.JPEG:n03041632 +ILSVRC2012_val_00016902.JPEG:n02096585 +ILSVRC2012_val_00016903.JPEG:n04317175 +ILSVRC2012_val_00016904.JPEG:n07892512 +ILSVRC2012_val_00016905.JPEG:n01755581 +ILSVRC2012_val_00016906.JPEG:n03777568 +ILSVRC2012_val_00016907.JPEG:n03457902 +ILSVRC2012_val_00016908.JPEG:n02106382 +ILSVRC2012_val_00016909.JPEG:n01601694 +ILSVRC2012_val_00016910.JPEG:n03691459 +ILSVRC2012_val_00016911.JPEG:n02114855 +ILSVRC2012_val_00016912.JPEG:n03461385 +ILSVRC2012_val_00016913.JPEG:n02096294 +ILSVRC2012_val_00016914.JPEG:n03498962 +ILSVRC2012_val_00016915.JPEG:n04482393 +ILSVRC2012_val_00016916.JPEG:n02412080 +ILSVRC2012_val_00016917.JPEG:n03857828 +ILSVRC2012_val_00016918.JPEG:n02124075 +ILSVRC2012_val_00016919.JPEG:n02106550 +ILSVRC2012_val_00016920.JPEG:n03950228 +ILSVRC2012_val_00016921.JPEG:n07730033 +ILSVRC2012_val_00016922.JPEG:n02093991 +ILSVRC2012_val_00016923.JPEG:n07768694 +ILSVRC2012_val_00016924.JPEG:n02870880 +ILSVRC2012_val_00016925.JPEG:n02672831 +ILSVRC2012_val_00016926.JPEG:n02268443 +ILSVRC2012_val_00016927.JPEG:n03773504 +ILSVRC2012_val_00016928.JPEG:n09332890 +ILSVRC2012_val_00016929.JPEG:n02025239 +ILSVRC2012_val_00016930.JPEG:n04562935 +ILSVRC2012_val_00016931.JPEG:n07742313 +ILSVRC2012_val_00016932.JPEG:n04192698 +ILSVRC2012_val_00016933.JPEG:n04049303 +ILSVRC2012_val_00016934.JPEG:n01644900 +ILSVRC2012_val_00016935.JPEG:n02769748 +ILSVRC2012_val_00016936.JPEG:n01774384 +ILSVRC2012_val_00016937.JPEG:n02894605 +ILSVRC2012_val_00016938.JPEG:n03127747 +ILSVRC2012_val_00016939.JPEG:n03045698 +ILSVRC2012_val_00016940.JPEG:n03388549 +ILSVRC2012_val_00016941.JPEG:n03724870 +ILSVRC2012_val_00016942.JPEG:n03706229 +ILSVRC2012_val_00016943.JPEG:n03825788 +ILSVRC2012_val_00016944.JPEG:n01775062 +ILSVRC2012_val_00016945.JPEG:n03670208 +ILSVRC2012_val_00016946.JPEG:n02492035 +ILSVRC2012_val_00016947.JPEG:n01983481 +ILSVRC2012_val_00016948.JPEG:n04435653 +ILSVRC2012_val_00016949.JPEG:n03028079 +ILSVRC2012_val_00016950.JPEG:n03445924 +ILSVRC2012_val_00016951.JPEG:n02108000 +ILSVRC2012_val_00016952.JPEG:n01882714 +ILSVRC2012_val_00016953.JPEG:n02346627 +ILSVRC2012_val_00016954.JPEG:n09399592 +ILSVRC2012_val_00016955.JPEG:n12620546 +ILSVRC2012_val_00016956.JPEG:n03047690 +ILSVRC2012_val_00016957.JPEG:n02807133 +ILSVRC2012_val_00016958.JPEG:n03630383 +ILSVRC2012_val_00016959.JPEG:n03325584 +ILSVRC2012_val_00016960.JPEG:n02110063 +ILSVRC2012_val_00016961.JPEG:n07860988 +ILSVRC2012_val_00016962.JPEG:n01443537 +ILSVRC2012_val_00016963.JPEG:n04523525 +ILSVRC2012_val_00016964.JPEG:n02112706 +ILSVRC2012_val_00016965.JPEG:n02815834 +ILSVRC2012_val_00016966.JPEG:n03720891 +ILSVRC2012_val_00016967.JPEG:n03843555 +ILSVRC2012_val_00016968.JPEG:n02992211 +ILSVRC2012_val_00016969.JPEG:n02107908 +ILSVRC2012_val_00016970.JPEG:n03662601 +ILSVRC2012_val_00016971.JPEG:n03207743 +ILSVRC2012_val_00016972.JPEG:n04507155 +ILSVRC2012_val_00016973.JPEG:n02094433 +ILSVRC2012_val_00016974.JPEG:n02791270 +ILSVRC2012_val_00016975.JPEG:n02788148 +ILSVRC2012_val_00016976.JPEG:n02094258 +ILSVRC2012_val_00016977.JPEG:n02105162 +ILSVRC2012_val_00016978.JPEG:n04179913 +ILSVRC2012_val_00016979.JPEG:n07930864 +ILSVRC2012_val_00016980.JPEG:n03873416 +ILSVRC2012_val_00016981.JPEG:n02027492 +ILSVRC2012_val_00016982.JPEG:n02790996 +ILSVRC2012_val_00016983.JPEG:n03924679 +ILSVRC2012_val_00016984.JPEG:n07753275 +ILSVRC2012_val_00016985.JPEG:n03658185 +ILSVRC2012_val_00016986.JPEG:n02444819 +ILSVRC2012_val_00016987.JPEG:n07802026 +ILSVRC2012_val_00016988.JPEG:n01484850 +ILSVRC2012_val_00016989.JPEG:n02113186 +ILSVRC2012_val_00016990.JPEG:n02110341 +ILSVRC2012_val_00016991.JPEG:n02090622 +ILSVRC2012_val_00016992.JPEG:n04366367 +ILSVRC2012_val_00016993.JPEG:n01773157 +ILSVRC2012_val_00016994.JPEG:n03792972 +ILSVRC2012_val_00016995.JPEG:n02690373 +ILSVRC2012_val_00016996.JPEG:n02090622 +ILSVRC2012_val_00016997.JPEG:n06794110 +ILSVRC2012_val_00016998.JPEG:n02101388 +ILSVRC2012_val_00016999.JPEG:n07697313 +ILSVRC2012_val_00017000.JPEG:n03297495 +ILSVRC2012_val_00017001.JPEG:n03032252 +ILSVRC2012_val_00017002.JPEG:n01688243 +ILSVRC2012_val_00017003.JPEG:n02090379 +ILSVRC2012_val_00017004.JPEG:n02017213 +ILSVRC2012_val_00017005.JPEG:n04152593 +ILSVRC2012_val_00017006.JPEG:n02108551 +ILSVRC2012_val_00017007.JPEG:n03658185 +ILSVRC2012_val_00017008.JPEG:n02643566 +ILSVRC2012_val_00017009.JPEG:n04049303 +ILSVRC2012_val_00017010.JPEG:n03544143 +ILSVRC2012_val_00017011.JPEG:n03709823 +ILSVRC2012_val_00017012.JPEG:n01632458 +ILSVRC2012_val_00017013.JPEG:n02111500 +ILSVRC2012_val_00017014.JPEG:n07717556 +ILSVRC2012_val_00017015.JPEG:n01688243 +ILSVRC2012_val_00017016.JPEG:n07747607 +ILSVRC2012_val_00017017.JPEG:n01592084 +ILSVRC2012_val_00017018.JPEG:n03485794 +ILSVRC2012_val_00017019.JPEG:n02443114 +ILSVRC2012_val_00017020.JPEG:n03888257 +ILSVRC2012_val_00017021.JPEG:n07753592 +ILSVRC2012_val_00017022.JPEG:n01930112 +ILSVRC2012_val_00017023.JPEG:n03127747 +ILSVRC2012_val_00017024.JPEG:n01580077 +ILSVRC2012_val_00017025.JPEG:n12057211 +ILSVRC2012_val_00017026.JPEG:n03344393 +ILSVRC2012_val_00017027.JPEG:n03697007 +ILSVRC2012_val_00017028.JPEG:n01601694 +ILSVRC2012_val_00017029.JPEG:n01818515 +ILSVRC2012_val_00017030.JPEG:n04517823 +ILSVRC2012_val_00017031.JPEG:n04584207 +ILSVRC2012_val_00017032.JPEG:n02002724 +ILSVRC2012_val_00017033.JPEG:n03424325 +ILSVRC2012_val_00017034.JPEG:n03895866 +ILSVRC2012_val_00017035.JPEG:n03787032 +ILSVRC2012_val_00017036.JPEG:n02100236 +ILSVRC2012_val_00017037.JPEG:n03110669 +ILSVRC2012_val_00017038.JPEG:n04523525 +ILSVRC2012_val_00017039.JPEG:n01983481 +ILSVRC2012_val_00017040.JPEG:n04465501 +ILSVRC2012_val_00017041.JPEG:n02090721 +ILSVRC2012_val_00017042.JPEG:n02980441 +ILSVRC2012_val_00017043.JPEG:n02088094 +ILSVRC2012_val_00017044.JPEG:n02492035 +ILSVRC2012_val_00017045.JPEG:n03109150 +ILSVRC2012_val_00017046.JPEG:n02091635 +ILSVRC2012_val_00017047.JPEG:n07695742 +ILSVRC2012_val_00017048.JPEG:n02074367 +ILSVRC2012_val_00017049.JPEG:n07754684 +ILSVRC2012_val_00017050.JPEG:n02783161 +ILSVRC2012_val_00017051.JPEG:n03761084 +ILSVRC2012_val_00017052.JPEG:n02096585 +ILSVRC2012_val_00017053.JPEG:n04099969 +ILSVRC2012_val_00017054.JPEG:n01930112 +ILSVRC2012_val_00017055.JPEG:n03379051 +ILSVRC2012_val_00017056.JPEG:n02105412 +ILSVRC2012_val_00017057.JPEG:n02097298 +ILSVRC2012_val_00017058.JPEG:n04026417 +ILSVRC2012_val_00017059.JPEG:n03866082 +ILSVRC2012_val_00017060.JPEG:n04004767 +ILSVRC2012_val_00017061.JPEG:n01704323 +ILSVRC2012_val_00017062.JPEG:n04286575 +ILSVRC2012_val_00017063.JPEG:n02321529 +ILSVRC2012_val_00017064.JPEG:n04417672 +ILSVRC2012_val_00017065.JPEG:n04389033 +ILSVRC2012_val_00017066.JPEG:n02909870 +ILSVRC2012_val_00017067.JPEG:n01685808 +ILSVRC2012_val_00017068.JPEG:n01806143 +ILSVRC2012_val_00017069.JPEG:n02006656 +ILSVRC2012_val_00017070.JPEG:n03832673 +ILSVRC2012_val_00017071.JPEG:n07697313 +ILSVRC2012_val_00017072.JPEG:n07932039 +ILSVRC2012_val_00017073.JPEG:n02206856 +ILSVRC2012_val_00017074.JPEG:n12144580 +ILSVRC2012_val_00017075.JPEG:n02108422 +ILSVRC2012_val_00017076.JPEG:n07753113 +ILSVRC2012_val_00017077.JPEG:n03777754 +ILSVRC2012_val_00017078.JPEG:n04259630 +ILSVRC2012_val_00017079.JPEG:n02641379 +ILSVRC2012_val_00017080.JPEG:n13052670 +ILSVRC2012_val_00017081.JPEG:n03788365 +ILSVRC2012_val_00017082.JPEG:n02870880 +ILSVRC2012_val_00017083.JPEG:n02799071 +ILSVRC2012_val_00017084.JPEG:n02137549 +ILSVRC2012_val_00017085.JPEG:n02999410 +ILSVRC2012_val_00017086.JPEG:n04317175 +ILSVRC2012_val_00017087.JPEG:n02094114 +ILSVRC2012_val_00017088.JPEG:n03529860 +ILSVRC2012_val_00017089.JPEG:n03188531 +ILSVRC2012_val_00017090.JPEG:n03160309 +ILSVRC2012_val_00017091.JPEG:n03697007 +ILSVRC2012_val_00017092.JPEG:n02091831 +ILSVRC2012_val_00017093.JPEG:n03594734 +ILSVRC2012_val_00017094.JPEG:n04389033 +ILSVRC2012_val_00017095.JPEG:n02799071 +ILSVRC2012_val_00017096.JPEG:n07747607 +ILSVRC2012_val_00017097.JPEG:n02504458 +ILSVRC2012_val_00017098.JPEG:n04277352 +ILSVRC2012_val_00017099.JPEG:n01914609 +ILSVRC2012_val_00017100.JPEG:n02281787 +ILSVRC2012_val_00017101.JPEG:n03868863 +ILSVRC2012_val_00017102.JPEG:n09421951 +ILSVRC2012_val_00017103.JPEG:n03792782 +ILSVRC2012_val_00017104.JPEG:n02102318 +ILSVRC2012_val_00017105.JPEG:n01484850 +ILSVRC2012_val_00017106.JPEG:n04192698 +ILSVRC2012_val_00017107.JPEG:n02089867 +ILSVRC2012_val_00017108.JPEG:n03584254 +ILSVRC2012_val_00017109.JPEG:n01728572 +ILSVRC2012_val_00017110.JPEG:n03062245 +ILSVRC2012_val_00017111.JPEG:n02109047 +ILSVRC2012_val_00017112.JPEG:n02108422 +ILSVRC2012_val_00017113.JPEG:n02088632 +ILSVRC2012_val_00017114.JPEG:n02447366 +ILSVRC2012_val_00017115.JPEG:n02236044 +ILSVRC2012_val_00017116.JPEG:n02910353 +ILSVRC2012_val_00017117.JPEG:n02105056 +ILSVRC2012_val_00017118.JPEG:n03498962 +ILSVRC2012_val_00017119.JPEG:n03250847 +ILSVRC2012_val_00017120.JPEG:n04120489 +ILSVRC2012_val_00017121.JPEG:n02999410 +ILSVRC2012_val_00017122.JPEG:n03467068 +ILSVRC2012_val_00017123.JPEG:n03187595 +ILSVRC2012_val_00017124.JPEG:n03255030 +ILSVRC2012_val_00017125.JPEG:n04004767 +ILSVRC2012_val_00017126.JPEG:n02091635 +ILSVRC2012_val_00017127.JPEG:n04507155 +ILSVRC2012_val_00017128.JPEG:n03782006 +ILSVRC2012_val_00017129.JPEG:n02317335 +ILSVRC2012_val_00017130.JPEG:n02165456 +ILSVRC2012_val_00017131.JPEG:n04243546 +ILSVRC2012_val_00017132.JPEG:n02099849 +ILSVRC2012_val_00017133.JPEG:n04239074 +ILSVRC2012_val_00017134.JPEG:n09246464 +ILSVRC2012_val_00017135.JPEG:n04335435 +ILSVRC2012_val_00017136.JPEG:n03770439 +ILSVRC2012_val_00017137.JPEG:n01978455 +ILSVRC2012_val_00017138.JPEG:n01644373 +ILSVRC2012_val_00017139.JPEG:n02256656 +ILSVRC2012_val_00017140.JPEG:n02509815 +ILSVRC2012_val_00017141.JPEG:n03584254 +ILSVRC2012_val_00017142.JPEG:n03710721 +ILSVRC2012_val_00017143.JPEG:n01795545 +ILSVRC2012_val_00017144.JPEG:n07753592 +ILSVRC2012_val_00017145.JPEG:n02412080 +ILSVRC2012_val_00017146.JPEG:n07892512 +ILSVRC2012_val_00017147.JPEG:n02091032 +ILSVRC2012_val_00017148.JPEG:n04074963 +ILSVRC2012_val_00017149.JPEG:n03197337 +ILSVRC2012_val_00017150.JPEG:n03075370 +ILSVRC2012_val_00017151.JPEG:n02111129 +ILSVRC2012_val_00017152.JPEG:n03930630 +ILSVRC2012_val_00017153.JPEG:n01770081 +ILSVRC2012_val_00017154.JPEG:n04235860 +ILSVRC2012_val_00017155.JPEG:n02132136 +ILSVRC2012_val_00017156.JPEG:n02100735 +ILSVRC2012_val_00017157.JPEG:n01978287 +ILSVRC2012_val_00017158.JPEG:n02097658 +ILSVRC2012_val_00017159.JPEG:n04540053 +ILSVRC2012_val_00017160.JPEG:n04149813 +ILSVRC2012_val_00017161.JPEG:n02105251 +ILSVRC2012_val_00017162.JPEG:n01984695 +ILSVRC2012_val_00017163.JPEG:n03314780 +ILSVRC2012_val_00017164.JPEG:n02115641 +ILSVRC2012_val_00017165.JPEG:n04235860 +ILSVRC2012_val_00017166.JPEG:n02843684 +ILSVRC2012_val_00017167.JPEG:n04311004 +ILSVRC2012_val_00017168.JPEG:n04118776 +ILSVRC2012_val_00017169.JPEG:n02276258 +ILSVRC2012_val_00017170.JPEG:n02909870 +ILSVRC2012_val_00017171.JPEG:n02701002 +ILSVRC2012_val_00017172.JPEG:n02051845 +ILSVRC2012_val_00017173.JPEG:n04599235 +ILSVRC2012_val_00017174.JPEG:n01689811 +ILSVRC2012_val_00017175.JPEG:n03637318 +ILSVRC2012_val_00017176.JPEG:n03344393 +ILSVRC2012_val_00017177.JPEG:n04591713 +ILSVRC2012_val_00017178.JPEG:n02018795 +ILSVRC2012_val_00017179.JPEG:n02795169 +ILSVRC2012_val_00017180.JPEG:n04462240 +ILSVRC2012_val_00017181.JPEG:n03776460 +ILSVRC2012_val_00017182.JPEG:n03404251 +ILSVRC2012_val_00017183.JPEG:n03188531 +ILSVRC2012_val_00017184.JPEG:n07749582 +ILSVRC2012_val_00017185.JPEG:n01631663 +ILSVRC2012_val_00017186.JPEG:n02123597 +ILSVRC2012_val_00017187.JPEG:n02328150 +ILSVRC2012_val_00017188.JPEG:n02110958 +ILSVRC2012_val_00017189.JPEG:n02125311 +ILSVRC2012_val_00017190.JPEG:n04023962 +ILSVRC2012_val_00017191.JPEG:n03133878 +ILSVRC2012_val_00017192.JPEG:n03131574 +ILSVRC2012_val_00017193.JPEG:n02091467 +ILSVRC2012_val_00017194.JPEG:n01484850 +ILSVRC2012_val_00017195.JPEG:n02096177 +ILSVRC2012_val_00017196.JPEG:n01496331 +ILSVRC2012_val_00017197.JPEG:n02058221 +ILSVRC2012_val_00017198.JPEG:n03028079 +ILSVRC2012_val_00017199.JPEG:n02113023 +ILSVRC2012_val_00017200.JPEG:n02480855 +ILSVRC2012_val_00017201.JPEG:n02892201 +ILSVRC2012_val_00017202.JPEG:n04418357 +ILSVRC2012_val_00017203.JPEG:n03042490 +ILSVRC2012_val_00017204.JPEG:n03124170 +ILSVRC2012_val_00017205.JPEG:n12985857 +ILSVRC2012_val_00017206.JPEG:n04141975 +ILSVRC2012_val_00017207.JPEG:n01860187 +ILSVRC2012_val_00017208.JPEG:n02130308 +ILSVRC2012_val_00017209.JPEG:n04037443 +ILSVRC2012_val_00017210.JPEG:n13052670 +ILSVRC2012_val_00017211.JPEG:n07714571 +ILSVRC2012_val_00017212.JPEG:n02391049 +ILSVRC2012_val_00017213.JPEG:n04149813 +ILSVRC2012_val_00017214.JPEG:n04099969 +ILSVRC2012_val_00017215.JPEG:n01729977 +ILSVRC2012_val_00017216.JPEG:n04243546 +ILSVRC2012_val_00017217.JPEG:n02978881 +ILSVRC2012_val_00017218.JPEG:n03131574 +ILSVRC2012_val_00017219.JPEG:n02127052 +ILSVRC2012_val_00017220.JPEG:n04366367 +ILSVRC2012_val_00017221.JPEG:n02229544 +ILSVRC2012_val_00017222.JPEG:n01669191 +ILSVRC2012_val_00017223.JPEG:n02489166 +ILSVRC2012_val_00017224.JPEG:n07716906 +ILSVRC2012_val_00017225.JPEG:n03208938 +ILSVRC2012_val_00017226.JPEG:n02088466 +ILSVRC2012_val_00017227.JPEG:n02093754 +ILSVRC2012_val_00017228.JPEG:n01632777 +ILSVRC2012_val_00017229.JPEG:n04118538 +ILSVRC2012_val_00017230.JPEG:n02363005 +ILSVRC2012_val_00017231.JPEG:n02114855 +ILSVRC2012_val_00017232.JPEG:n09256479 +ILSVRC2012_val_00017233.JPEG:n02787622 +ILSVRC2012_val_00017234.JPEG:n02105412 +ILSVRC2012_val_00017235.JPEG:n03498962 +ILSVRC2012_val_00017236.JPEG:n12768682 +ILSVRC2012_val_00017237.JPEG:n03216828 +ILSVRC2012_val_00017238.JPEG:n03598930 +ILSVRC2012_val_00017239.JPEG:n02643566 +ILSVRC2012_val_00017240.JPEG:n03837869 +ILSVRC2012_val_00017241.JPEG:n07695742 +ILSVRC2012_val_00017242.JPEG:n01817953 +ILSVRC2012_val_00017243.JPEG:n01667778 +ILSVRC2012_val_00017244.JPEG:n04251144 +ILSVRC2012_val_00017245.JPEG:n02231487 +ILSVRC2012_val_00017246.JPEG:n04005630 +ILSVRC2012_val_00017247.JPEG:n03445777 +ILSVRC2012_val_00017248.JPEG:n04597913 +ILSVRC2012_val_00017249.JPEG:n07615774 +ILSVRC2012_val_00017250.JPEG:n02769748 +ILSVRC2012_val_00017251.JPEG:n01833805 +ILSVRC2012_val_00017252.JPEG:n01828970 +ILSVRC2012_val_00017253.JPEG:n01796340 +ILSVRC2012_val_00017254.JPEG:n01694178 +ILSVRC2012_val_00017255.JPEG:n03995372 +ILSVRC2012_val_00017256.JPEG:n03494278 +ILSVRC2012_val_00017257.JPEG:n03271574 +ILSVRC2012_val_00017258.JPEG:n03014705 +ILSVRC2012_val_00017259.JPEG:n02088632 +ILSVRC2012_val_00017260.JPEG:n03788195 +ILSVRC2012_val_00017261.JPEG:n02328150 +ILSVRC2012_val_00017262.JPEG:n02992529 +ILSVRC2012_val_00017263.JPEG:n03498962 +ILSVRC2012_val_00017264.JPEG:n02169497 +ILSVRC2012_val_00017265.JPEG:n02112137 +ILSVRC2012_val_00017266.JPEG:n02483362 +ILSVRC2012_val_00017267.JPEG:n07836838 +ILSVRC2012_val_00017268.JPEG:n02086240 +ILSVRC2012_val_00017269.JPEG:n01739381 +ILSVRC2012_val_00017270.JPEG:n02325366 +ILSVRC2012_val_00017271.JPEG:n03877472 +ILSVRC2012_val_00017272.JPEG:n04589890 +ILSVRC2012_val_00017273.JPEG:n02133161 +ILSVRC2012_val_00017274.JPEG:n01632777 +ILSVRC2012_val_00017275.JPEG:n02105162 +ILSVRC2012_val_00017276.JPEG:n04019541 +ILSVRC2012_val_00017277.JPEG:n01775062 +ILSVRC2012_val_00017278.JPEG:n02107574 +ILSVRC2012_val_00017279.JPEG:n04509417 +ILSVRC2012_val_00017280.JPEG:n01860187 +ILSVRC2012_val_00017281.JPEG:n02088632 +ILSVRC2012_val_00017282.JPEG:n03459775 +ILSVRC2012_val_00017283.JPEG:n03133878 +ILSVRC2012_val_00017284.JPEG:n04254680 +ILSVRC2012_val_00017285.JPEG:n01755581 +ILSVRC2012_val_00017286.JPEG:n02939185 +ILSVRC2012_val_00017287.JPEG:n02091134 +ILSVRC2012_val_00017288.JPEG:n02114712 +ILSVRC2012_val_00017289.JPEG:n07714990 +ILSVRC2012_val_00017290.JPEG:n02484975 +ILSVRC2012_val_00017291.JPEG:n03445924 +ILSVRC2012_val_00017292.JPEG:n03018349 +ILSVRC2012_val_00017293.JPEG:n02802426 +ILSVRC2012_val_00017294.JPEG:n01774384 +ILSVRC2012_val_00017295.JPEG:n03124043 +ILSVRC2012_val_00017296.JPEG:n03355925 +ILSVRC2012_val_00017297.JPEG:n03146219 +ILSVRC2012_val_00017298.JPEG:n03388183 +ILSVRC2012_val_00017299.JPEG:n02226429 +ILSVRC2012_val_00017300.JPEG:n07860988 +ILSVRC2012_val_00017301.JPEG:n03388183 +ILSVRC2012_val_00017302.JPEG:n04009552 +ILSVRC2012_val_00017303.JPEG:n02488291 +ILSVRC2012_val_00017304.JPEG:n03899768 +ILSVRC2012_val_00017305.JPEG:n03649909 +ILSVRC2012_val_00017306.JPEG:n03393912 +ILSVRC2012_val_00017307.JPEG:n02797295 +ILSVRC2012_val_00017308.JPEG:n03014705 +ILSVRC2012_val_00017309.JPEG:n03729826 +ILSVRC2012_val_00017310.JPEG:n01560419 +ILSVRC2012_val_00017311.JPEG:n02114367 +ILSVRC2012_val_00017312.JPEG:n03637318 +ILSVRC2012_val_00017313.JPEG:n02115641 +ILSVRC2012_val_00017314.JPEG:n04517823 +ILSVRC2012_val_00017315.JPEG:n02346627 +ILSVRC2012_val_00017316.JPEG:n02033041 +ILSVRC2012_val_00017317.JPEG:n02804414 +ILSVRC2012_val_00017318.JPEG:n07714990 +ILSVRC2012_val_00017319.JPEG:n04120489 +ILSVRC2012_val_00017320.JPEG:n03481172 +ILSVRC2012_val_00017321.JPEG:n02099267 +ILSVRC2012_val_00017322.JPEG:n10565667 +ILSVRC2012_val_00017323.JPEG:n03825788 +ILSVRC2012_val_00017324.JPEG:n03240683 +ILSVRC2012_val_00017325.JPEG:n02123597 +ILSVRC2012_val_00017326.JPEG:n02097130 +ILSVRC2012_val_00017327.JPEG:n02090721 +ILSVRC2012_val_00017328.JPEG:n02094433 +ILSVRC2012_val_00017329.JPEG:n02667093 +ILSVRC2012_val_00017330.JPEG:n03461385 +ILSVRC2012_val_00017331.JPEG:n02101388 +ILSVRC2012_val_00017332.JPEG:n09399592 +ILSVRC2012_val_00017333.JPEG:n02109047 +ILSVRC2012_val_00017334.JPEG:n04153751 +ILSVRC2012_val_00017335.JPEG:n04479046 +ILSVRC2012_val_00017336.JPEG:n03223299 +ILSVRC2012_val_00017337.JPEG:n13133613 +ILSVRC2012_val_00017338.JPEG:n01688243 +ILSVRC2012_val_00017339.JPEG:n02363005 +ILSVRC2012_val_00017340.JPEG:n04493381 +ILSVRC2012_val_00017341.JPEG:n02445715 +ILSVRC2012_val_00017342.JPEG:n02280649 +ILSVRC2012_val_00017343.JPEG:n03804744 +ILSVRC2012_val_00017344.JPEG:n04596742 +ILSVRC2012_val_00017345.JPEG:n04597913 +ILSVRC2012_val_00017346.JPEG:n01729322 +ILSVRC2012_val_00017347.JPEG:n02793495 +ILSVRC2012_val_00017348.JPEG:n04604644 +ILSVRC2012_val_00017349.JPEG:n04592741 +ILSVRC2012_val_00017350.JPEG:n03425413 +ILSVRC2012_val_00017351.JPEG:n04332243 +ILSVRC2012_val_00017352.JPEG:n04562935 +ILSVRC2012_val_00017353.JPEG:n02494079 +ILSVRC2012_val_00017354.JPEG:n07693725 +ILSVRC2012_val_00017355.JPEG:n07717410 +ILSVRC2012_val_00017356.JPEG:n06874185 +ILSVRC2012_val_00017357.JPEG:n03063689 +ILSVRC2012_val_00017358.JPEG:n02389026 +ILSVRC2012_val_00017359.JPEG:n02110627 +ILSVRC2012_val_00017360.JPEG:n03930630 +ILSVRC2012_val_00017361.JPEG:n01871265 +ILSVRC2012_val_00017362.JPEG:n07716358 +ILSVRC2012_val_00017363.JPEG:n02114712 +ILSVRC2012_val_00017364.JPEG:n03216828 +ILSVRC2012_val_00017365.JPEG:n06596364 +ILSVRC2012_val_00017366.JPEG:n03494278 +ILSVRC2012_val_00017367.JPEG:n07579787 +ILSVRC2012_val_00017368.JPEG:n04548280 +ILSVRC2012_val_00017369.JPEG:n04409515 +ILSVRC2012_val_00017370.JPEG:n02102040 +ILSVRC2012_val_00017371.JPEG:n07753113 +ILSVRC2012_val_00017372.JPEG:n01632777 +ILSVRC2012_val_00017373.JPEG:n02843684 +ILSVRC2012_val_00017374.JPEG:n02395406 +ILSVRC2012_val_00017375.JPEG:n02100583 +ILSVRC2012_val_00017376.JPEG:n03481172 +ILSVRC2012_val_00017377.JPEG:n02099849 +ILSVRC2012_val_00017378.JPEG:n02708093 +ILSVRC2012_val_00017379.JPEG:n01980166 +ILSVRC2012_val_00017380.JPEG:n02096294 +ILSVRC2012_val_00017381.JPEG:n01744401 +ILSVRC2012_val_00017382.JPEG:n03291819 +ILSVRC2012_val_00017383.JPEG:n04004767 +ILSVRC2012_val_00017384.JPEG:n01534433 +ILSVRC2012_val_00017385.JPEG:n03223299 +ILSVRC2012_val_00017386.JPEG:n03773504 +ILSVRC2012_val_00017387.JPEG:n04090263 +ILSVRC2012_val_00017388.JPEG:n02002724 +ILSVRC2012_val_00017389.JPEG:n02422106 +ILSVRC2012_val_00017390.JPEG:n04325704 +ILSVRC2012_val_00017391.JPEG:n01531178 +ILSVRC2012_val_00017392.JPEG:n02948072 +ILSVRC2012_val_00017393.JPEG:n02281787 +ILSVRC2012_val_00017394.JPEG:n04239074 +ILSVRC2012_val_00017395.JPEG:n04399382 +ILSVRC2012_val_00017396.JPEG:n03400231 +ILSVRC2012_val_00017397.JPEG:n02802426 +ILSVRC2012_val_00017398.JPEG:n02165456 +ILSVRC2012_val_00017399.JPEG:n02256656 +ILSVRC2012_val_00017400.JPEG:n02104029 +ILSVRC2012_val_00017401.JPEG:n06794110 +ILSVRC2012_val_00017402.JPEG:n07932039 +ILSVRC2012_val_00017403.JPEG:n02793495 +ILSVRC2012_val_00017404.JPEG:n02093754 +ILSVRC2012_val_00017405.JPEG:n02834397 +ILSVRC2012_val_00017406.JPEG:n02165456 +ILSVRC2012_val_00017407.JPEG:n03394916 +ILSVRC2012_val_00017408.JPEG:n02138441 +ILSVRC2012_val_00017409.JPEG:n01729977 +ILSVRC2012_val_00017410.JPEG:n02138441 +ILSVRC2012_val_00017411.JPEG:n04311174 +ILSVRC2012_val_00017412.JPEG:n03388043 +ILSVRC2012_val_00017413.JPEG:n03344393 +ILSVRC2012_val_00017414.JPEG:n03445924 +ILSVRC2012_val_00017415.JPEG:n02504013 +ILSVRC2012_val_00017416.JPEG:n13040303 +ILSVRC2012_val_00017417.JPEG:n02363005 +ILSVRC2012_val_00017418.JPEG:n02206856 +ILSVRC2012_val_00017419.JPEG:n03982430 +ILSVRC2012_val_00017420.JPEG:n03661043 +ILSVRC2012_val_00017421.JPEG:n02107574 +ILSVRC2012_val_00017422.JPEG:n03785016 +ILSVRC2012_val_00017423.JPEG:n02231487 +ILSVRC2012_val_00017424.JPEG:n04487394 +ILSVRC2012_val_00017425.JPEG:n04376876 +ILSVRC2012_val_00017426.JPEG:n04277352 +ILSVRC2012_val_00017427.JPEG:n07718472 +ILSVRC2012_val_00017428.JPEG:n04118776 +ILSVRC2012_val_00017429.JPEG:n01914609 +ILSVRC2012_val_00017430.JPEG:n01798484 +ILSVRC2012_val_00017431.JPEG:n01944390 +ILSVRC2012_val_00017432.JPEG:n03355925 +ILSVRC2012_val_00017433.JPEG:n03742115 +ILSVRC2012_val_00017434.JPEG:n02108089 +ILSVRC2012_val_00017435.JPEG:n03924679 +ILSVRC2012_val_00017436.JPEG:n03134739 +ILSVRC2012_val_00017437.JPEG:n02011460 +ILSVRC2012_val_00017438.JPEG:n02974003 +ILSVRC2012_val_00017439.JPEG:n02100583 +ILSVRC2012_val_00017440.JPEG:n01496331 +ILSVRC2012_val_00017441.JPEG:n01860187 +ILSVRC2012_val_00017442.JPEG:n02100236 +ILSVRC2012_val_00017443.JPEG:n04596742 +ILSVRC2012_val_00017444.JPEG:n02119789 +ILSVRC2012_val_00017445.JPEG:n02342885 +ILSVRC2012_val_00017446.JPEG:n04044716 +ILSVRC2012_val_00017447.JPEG:n04099969 +ILSVRC2012_val_00017448.JPEG:n03602883 +ILSVRC2012_val_00017449.JPEG:n07717556 +ILSVRC2012_val_00017450.JPEG:n04548280 +ILSVRC2012_val_00017451.JPEG:n03843555 +ILSVRC2012_val_00017452.JPEG:n04409515 +ILSVRC2012_val_00017453.JPEG:n02093647 +ILSVRC2012_val_00017454.JPEG:n01797886 +ILSVRC2012_val_00017455.JPEG:n04429376 +ILSVRC2012_val_00017456.JPEG:n03063599 +ILSVRC2012_val_00017457.JPEG:n07760859 +ILSVRC2012_val_00017458.JPEG:n02487347 +ILSVRC2012_val_00017459.JPEG:n01697457 +ILSVRC2012_val_00017460.JPEG:n03706229 +ILSVRC2012_val_00017461.JPEG:n02988304 +ILSVRC2012_val_00017462.JPEG:n03134739 +ILSVRC2012_val_00017463.JPEG:n02979186 +ILSVRC2012_val_00017464.JPEG:n02892201 +ILSVRC2012_val_00017465.JPEG:n03840681 +ILSVRC2012_val_00017466.JPEG:n03425413 +ILSVRC2012_val_00017467.JPEG:n13044778 +ILSVRC2012_val_00017468.JPEG:n04330267 +ILSVRC2012_val_00017469.JPEG:n03425413 +ILSVRC2012_val_00017470.JPEG:n02099849 +ILSVRC2012_val_00017471.JPEG:n04044716 +ILSVRC2012_val_00017472.JPEG:n01440764 +ILSVRC2012_val_00017473.JPEG:n02105251 +ILSVRC2012_val_00017474.JPEG:n03599486 +ILSVRC2012_val_00017475.JPEG:n03240683 +ILSVRC2012_val_00017476.JPEG:n02097130 +ILSVRC2012_val_00017477.JPEG:n04162706 +ILSVRC2012_val_00017478.JPEG:n03443371 +ILSVRC2012_val_00017479.JPEG:n02492660 +ILSVRC2012_val_00017480.JPEG:n03793489 +ILSVRC2012_val_00017481.JPEG:n04347754 +ILSVRC2012_val_00017482.JPEG:n04296562 +ILSVRC2012_val_00017483.JPEG:n03666591 +ILSVRC2012_val_00017484.JPEG:n04584207 +ILSVRC2012_val_00017485.JPEG:n04136333 +ILSVRC2012_val_00017486.JPEG:n02123159 +ILSVRC2012_val_00017487.JPEG:n04070727 +ILSVRC2012_val_00017488.JPEG:n02981792 +ILSVRC2012_val_00017489.JPEG:n07718472 +ILSVRC2012_val_00017490.JPEG:n01694178 +ILSVRC2012_val_00017491.JPEG:n10565667 +ILSVRC2012_val_00017492.JPEG:n04532670 +ILSVRC2012_val_00017493.JPEG:n02480495 +ILSVRC2012_val_00017494.JPEG:n07590611 +ILSVRC2012_val_00017495.JPEG:n02111277 +ILSVRC2012_val_00017496.JPEG:n04554684 +ILSVRC2012_val_00017497.JPEG:n01695060 +ILSVRC2012_val_00017498.JPEG:n04311004 +ILSVRC2012_val_00017499.JPEG:n02102480 +ILSVRC2012_val_00017500.JPEG:n04447861 +ILSVRC2012_val_00017501.JPEG:n02807133 +ILSVRC2012_val_00017502.JPEG:n04398044 +ILSVRC2012_val_00017503.JPEG:n04418357 +ILSVRC2012_val_00017504.JPEG:n03690938 +ILSVRC2012_val_00017505.JPEG:n01644373 +ILSVRC2012_val_00017506.JPEG:n03837869 +ILSVRC2012_val_00017507.JPEG:n02493793 +ILSVRC2012_val_00017508.JPEG:n01796340 +ILSVRC2012_val_00017509.JPEG:n02095889 +ILSVRC2012_val_00017510.JPEG:n03781244 +ILSVRC2012_val_00017511.JPEG:n02088466 +ILSVRC2012_val_00017512.JPEG:n02906734 +ILSVRC2012_val_00017513.JPEG:n04596742 +ILSVRC2012_val_00017514.JPEG:n12057211 +ILSVRC2012_val_00017515.JPEG:n02097658 +ILSVRC2012_val_00017516.JPEG:n03954731 +ILSVRC2012_val_00017517.JPEG:n02447366 +ILSVRC2012_val_00017518.JPEG:n03223299 +ILSVRC2012_val_00017519.JPEG:n03710637 +ILSVRC2012_val_00017520.JPEG:n03459775 +ILSVRC2012_val_00017521.JPEG:n04458633 +ILSVRC2012_val_00017522.JPEG:n02397096 +ILSVRC2012_val_00017523.JPEG:n03877472 +ILSVRC2012_val_00017524.JPEG:n07584110 +ILSVRC2012_val_00017525.JPEG:n03393912 +ILSVRC2012_val_00017526.JPEG:n07716906 +ILSVRC2012_val_00017527.JPEG:n07836838 +ILSVRC2012_val_00017528.JPEG:n03720891 +ILSVRC2012_val_00017529.JPEG:n02109961 +ILSVRC2012_val_00017530.JPEG:n04326547 +ILSVRC2012_val_00017531.JPEG:n01753488 +ILSVRC2012_val_00017532.JPEG:n02389026 +ILSVRC2012_val_00017533.JPEG:n07734744 +ILSVRC2012_val_00017534.JPEG:n07745940 +ILSVRC2012_val_00017535.JPEG:n02094114 +ILSVRC2012_val_00017536.JPEG:n02981792 +ILSVRC2012_val_00017537.JPEG:n02097298 +ILSVRC2012_val_00017538.JPEG:n03930630 +ILSVRC2012_val_00017539.JPEG:n02783161 +ILSVRC2012_val_00017540.JPEG:n04346328 +ILSVRC2012_val_00017541.JPEG:n01774750 +ILSVRC2012_val_00017542.JPEG:n01829413 +ILSVRC2012_val_00017543.JPEG:n02910353 +ILSVRC2012_val_00017544.JPEG:n02894605 +ILSVRC2012_val_00017545.JPEG:n02132136 +ILSVRC2012_val_00017546.JPEG:n04372370 +ILSVRC2012_val_00017547.JPEG:n04040759 +ILSVRC2012_val_00017548.JPEG:n02493509 +ILSVRC2012_val_00017549.JPEG:n03788195 +ILSVRC2012_val_00017550.JPEG:n04357314 +ILSVRC2012_val_00017551.JPEG:n02106166 +ILSVRC2012_val_00017552.JPEG:n02168699 +ILSVRC2012_val_00017553.JPEG:n02091831 +ILSVRC2012_val_00017554.JPEG:n02105056 +ILSVRC2012_val_00017555.JPEG:n01986214 +ILSVRC2012_val_00017556.JPEG:n02268443 +ILSVRC2012_val_00017557.JPEG:n01739381 +ILSVRC2012_val_00017558.JPEG:n01774384 +ILSVRC2012_val_00017559.JPEG:n02444819 +ILSVRC2012_val_00017560.JPEG:n02105641 +ILSVRC2012_val_00017561.JPEG:n01687978 +ILSVRC2012_val_00017562.JPEG:n04606251 +ILSVRC2012_val_00017563.JPEG:n03325584 +ILSVRC2012_val_00017564.JPEG:n04596742 +ILSVRC2012_val_00017565.JPEG:n02325366 +ILSVRC2012_val_00017566.JPEG:n02950826 +ILSVRC2012_val_00017567.JPEG:n04067472 +ILSVRC2012_val_00017568.JPEG:n02086646 +ILSVRC2012_val_00017569.JPEG:n02113799 +ILSVRC2012_val_00017570.JPEG:n04557648 +ILSVRC2012_val_00017571.JPEG:n04429376 +ILSVRC2012_val_00017572.JPEG:n01704323 +ILSVRC2012_val_00017573.JPEG:n02056570 +ILSVRC2012_val_00017574.JPEG:n02488291 +ILSVRC2012_val_00017575.JPEG:n07614500 +ILSVRC2012_val_00017576.JPEG:n03089624 +ILSVRC2012_val_00017577.JPEG:n01532829 +ILSVRC2012_val_00017578.JPEG:n03160309 +ILSVRC2012_val_00017579.JPEG:n04550184 +ILSVRC2012_val_00017580.JPEG:n07730033 +ILSVRC2012_val_00017581.JPEG:n02095570 +ILSVRC2012_val_00017582.JPEG:n04367480 +ILSVRC2012_val_00017583.JPEG:n04081281 +ILSVRC2012_val_00017584.JPEG:n04254120 +ILSVRC2012_val_00017585.JPEG:n04443257 +ILSVRC2012_val_00017586.JPEG:n03777568 +ILSVRC2012_val_00017587.JPEG:n03584829 +ILSVRC2012_val_00017588.JPEG:n04201297 +ILSVRC2012_val_00017589.JPEG:n12144580 +ILSVRC2012_val_00017590.JPEG:n02834397 +ILSVRC2012_val_00017591.JPEG:n03127925 +ILSVRC2012_val_00017592.JPEG:n02100735 +ILSVRC2012_val_00017593.JPEG:n02256656 +ILSVRC2012_val_00017594.JPEG:n02092002 +ILSVRC2012_val_00017595.JPEG:n01753488 +ILSVRC2012_val_00017596.JPEG:n04259630 +ILSVRC2012_val_00017597.JPEG:n03197337 +ILSVRC2012_val_00017598.JPEG:n02510455 +ILSVRC2012_val_00017599.JPEG:n02108422 +ILSVRC2012_val_00017600.JPEG:n02013706 +ILSVRC2012_val_00017601.JPEG:n03840681 +ILSVRC2012_val_00017602.JPEG:n02108089 +ILSVRC2012_val_00017603.JPEG:n04485082 +ILSVRC2012_val_00017604.JPEG:n03584829 +ILSVRC2012_val_00017605.JPEG:n02134084 +ILSVRC2012_val_00017606.JPEG:n03814639 +ILSVRC2012_val_00017607.JPEG:n04522168 +ILSVRC2012_val_00017608.JPEG:n04589890 +ILSVRC2012_val_00017609.JPEG:n04252225 +ILSVRC2012_val_00017610.JPEG:n03188531 +ILSVRC2012_val_00017611.JPEG:n03594945 +ILSVRC2012_val_00017612.JPEG:n03691459 +ILSVRC2012_val_00017613.JPEG:n04041544 +ILSVRC2012_val_00017614.JPEG:n04033901 +ILSVRC2012_val_00017615.JPEG:n04090263 +ILSVRC2012_val_00017616.JPEG:n02486410 +ILSVRC2012_val_00017617.JPEG:n03873416 +ILSVRC2012_val_00017618.JPEG:n03871628 +ILSVRC2012_val_00017619.JPEG:n02325366 +ILSVRC2012_val_00017620.JPEG:n02841315 +ILSVRC2012_val_00017621.JPEG:n02037110 +ILSVRC2012_val_00017622.JPEG:n02909870 +ILSVRC2012_val_00017623.JPEG:n01629819 +ILSVRC2012_val_00017624.JPEG:n07565083 +ILSVRC2012_val_00017625.JPEG:n02088094 +ILSVRC2012_val_00017626.JPEG:n03954731 +ILSVRC2012_val_00017627.JPEG:n12998815 +ILSVRC2012_val_00017628.JPEG:n03661043 +ILSVRC2012_val_00017629.JPEG:n04332243 +ILSVRC2012_val_00017630.JPEG:n02167151 +ILSVRC2012_val_00017631.JPEG:n04099969 +ILSVRC2012_val_00017632.JPEG:n04266014 +ILSVRC2012_val_00017633.JPEG:n03733131 +ILSVRC2012_val_00017634.JPEG:n02033041 +ILSVRC2012_val_00017635.JPEG:n02165456 +ILSVRC2012_val_00017636.JPEG:n02109047 +ILSVRC2012_val_00017637.JPEG:n02999410 +ILSVRC2012_val_00017638.JPEG:n02177972 +ILSVRC2012_val_00017639.JPEG:n02033041 +ILSVRC2012_val_00017640.JPEG:n03899768 +ILSVRC2012_val_00017641.JPEG:n01685808 +ILSVRC2012_val_00017642.JPEG:n04023962 +ILSVRC2012_val_00017643.JPEG:n02114712 +ILSVRC2012_val_00017644.JPEG:n03775546 +ILSVRC2012_val_00017645.JPEG:n02092002 +ILSVRC2012_val_00017646.JPEG:n02107142 +ILSVRC2012_val_00017647.JPEG:n02977058 +ILSVRC2012_val_00017648.JPEG:n01582220 +ILSVRC2012_val_00017649.JPEG:n04127249 +ILSVRC2012_val_00017650.JPEG:n03814906 +ILSVRC2012_val_00017651.JPEG:n03769881 +ILSVRC2012_val_00017652.JPEG:n03393912 +ILSVRC2012_val_00017653.JPEG:n03291819 +ILSVRC2012_val_00017654.JPEG:n02497673 +ILSVRC2012_val_00017655.JPEG:n03127925 +ILSVRC2012_val_00017656.JPEG:n09193705 +ILSVRC2012_val_00017657.JPEG:n07831146 +ILSVRC2012_val_00017658.JPEG:n03980874 +ILSVRC2012_val_00017659.JPEG:n07753113 +ILSVRC2012_val_00017660.JPEG:n01558993 +ILSVRC2012_val_00017661.JPEG:n02808304 +ILSVRC2012_val_00017662.JPEG:n03854065 +ILSVRC2012_val_00017663.JPEG:n04483307 +ILSVRC2012_val_00017664.JPEG:n02102040 +ILSVRC2012_val_00017665.JPEG:n04326547 +ILSVRC2012_val_00017666.JPEG:n02443484 +ILSVRC2012_val_00017667.JPEG:n09256479 +ILSVRC2012_val_00017668.JPEG:n03961711 +ILSVRC2012_val_00017669.JPEG:n01641577 +ILSVRC2012_val_00017670.JPEG:n03733131 +ILSVRC2012_val_00017671.JPEG:n04254680 +ILSVRC2012_val_00017672.JPEG:n02099601 +ILSVRC2012_val_00017673.JPEG:n02089078 +ILSVRC2012_val_00017674.JPEG:n03016953 +ILSVRC2012_val_00017675.JPEG:n03216828 +ILSVRC2012_val_00017676.JPEG:n02101388 +ILSVRC2012_val_00017677.JPEG:n02229544 +ILSVRC2012_val_00017678.JPEG:n02606052 +ILSVRC2012_val_00017679.JPEG:n04141076 +ILSVRC2012_val_00017680.JPEG:n01694178 +ILSVRC2012_val_00017681.JPEG:n03063689 +ILSVRC2012_val_00017682.JPEG:n01774384 +ILSVRC2012_val_00017683.JPEG:n02607072 +ILSVRC2012_val_00017684.JPEG:n02091244 +ILSVRC2012_val_00017685.JPEG:n03937543 +ILSVRC2012_val_00017686.JPEG:n04328186 +ILSVRC2012_val_00017687.JPEG:n03532672 +ILSVRC2012_val_00017688.JPEG:n03485407 +ILSVRC2012_val_00017689.JPEG:n07717556 +ILSVRC2012_val_00017690.JPEG:n02006656 +ILSVRC2012_val_00017691.JPEG:n04525305 +ILSVRC2012_val_00017692.JPEG:n02123597 +ILSVRC2012_val_00017693.JPEG:n02708093 +ILSVRC2012_val_00017694.JPEG:n02137549 +ILSVRC2012_val_00017695.JPEG:n07614500 +ILSVRC2012_val_00017696.JPEG:n03947888 +ILSVRC2012_val_00017697.JPEG:n03983396 +ILSVRC2012_val_00017698.JPEG:n03544143 +ILSVRC2012_val_00017699.JPEG:n01440764 +ILSVRC2012_val_00017700.JPEG:n01440764 +ILSVRC2012_val_00017701.JPEG:n03717622 +ILSVRC2012_val_00017702.JPEG:n02085620 +ILSVRC2012_val_00017703.JPEG:n02727426 +ILSVRC2012_val_00017704.JPEG:n03485794 +ILSVRC2012_val_00017705.JPEG:n03825788 +ILSVRC2012_val_00017706.JPEG:n04259630 +ILSVRC2012_val_00017707.JPEG:n02788148 +ILSVRC2012_val_00017708.JPEG:n03930630 +ILSVRC2012_val_00017709.JPEG:n04392985 +ILSVRC2012_val_00017710.JPEG:n02454379 +ILSVRC2012_val_00017711.JPEG:n02100236 +ILSVRC2012_val_00017712.JPEG:n01534433 +ILSVRC2012_val_00017713.JPEG:n02102318 +ILSVRC2012_val_00017714.JPEG:n04044716 +ILSVRC2012_val_00017715.JPEG:n02113186 +ILSVRC2012_val_00017716.JPEG:n02066245 +ILSVRC2012_val_00017717.JPEG:n02127052 +ILSVRC2012_val_00017718.JPEG:n01950731 +ILSVRC2012_val_00017719.JPEG:n03000684 +ILSVRC2012_val_00017720.JPEG:n02843684 +ILSVRC2012_val_00017721.JPEG:n04147183 +ILSVRC2012_val_00017722.JPEG:n02110063 +ILSVRC2012_val_00017723.JPEG:n07590611 +ILSVRC2012_val_00017724.JPEG:n02113712 +ILSVRC2012_val_00017725.JPEG:n04074963 +ILSVRC2012_val_00017726.JPEG:n03871628 +ILSVRC2012_val_00017727.JPEG:n02168699 +ILSVRC2012_val_00017728.JPEG:n09246464 +ILSVRC2012_val_00017729.JPEG:n07802026 +ILSVRC2012_val_00017730.JPEG:n01693334 +ILSVRC2012_val_00017731.JPEG:n03908714 +ILSVRC2012_val_00017732.JPEG:n02130308 +ILSVRC2012_val_00017733.JPEG:n09193705 +ILSVRC2012_val_00017734.JPEG:n02091244 +ILSVRC2012_val_00017735.JPEG:n02111500 +ILSVRC2012_val_00017736.JPEG:n03642806 +ILSVRC2012_val_00017737.JPEG:n04033901 +ILSVRC2012_val_00017738.JPEG:n02999410 +ILSVRC2012_val_00017739.JPEG:n02128925 +ILSVRC2012_val_00017740.JPEG:n06359193 +ILSVRC2012_val_00017741.JPEG:n07717410 +ILSVRC2012_val_00017742.JPEG:n02102318 +ILSVRC2012_val_00017743.JPEG:n04208210 +ILSVRC2012_val_00017744.JPEG:n02086079 +ILSVRC2012_val_00017745.JPEG:n03868863 +ILSVRC2012_val_00017746.JPEG:n03743016 +ILSVRC2012_val_00017747.JPEG:n03062245 +ILSVRC2012_val_00017748.JPEG:n03717622 +ILSVRC2012_val_00017749.JPEG:n04069434 +ILSVRC2012_val_00017750.JPEG:n03598930 +ILSVRC2012_val_00017751.JPEG:n01978287 +ILSVRC2012_val_00017752.JPEG:n04026417 +ILSVRC2012_val_00017753.JPEG:n01748264 +ILSVRC2012_val_00017754.JPEG:n02096294 +ILSVRC2012_val_00017755.JPEG:n04483307 +ILSVRC2012_val_00017756.JPEG:n01592084 +ILSVRC2012_val_00017757.JPEG:n03787032 +ILSVRC2012_val_00017758.JPEG:n03742115 +ILSVRC2012_val_00017759.JPEG:n01795545 +ILSVRC2012_val_00017760.JPEG:n02807133 +ILSVRC2012_val_00017761.JPEG:n02769748 +ILSVRC2012_val_00017762.JPEG:n02108915 +ILSVRC2012_val_00017763.JPEG:n04509417 +ILSVRC2012_val_00017764.JPEG:n02093754 +ILSVRC2012_val_00017765.JPEG:n02129604 +ILSVRC2012_val_00017766.JPEG:n02090622 +ILSVRC2012_val_00017767.JPEG:n01806567 +ILSVRC2012_val_00017768.JPEG:n04579432 +ILSVRC2012_val_00017769.JPEG:n04542943 +ILSVRC2012_val_00017770.JPEG:n03400231 +ILSVRC2012_val_00017771.JPEG:n07871810 +ILSVRC2012_val_00017772.JPEG:n09399592 +ILSVRC2012_val_00017773.JPEG:n02114367 +ILSVRC2012_val_00017774.JPEG:n04049303 +ILSVRC2012_val_00017775.JPEG:n02979186 +ILSVRC2012_val_00017776.JPEG:n02494079 +ILSVRC2012_val_00017777.JPEG:n03944341 +ILSVRC2012_val_00017778.JPEG:n03535780 +ILSVRC2012_val_00017779.JPEG:n03297495 +ILSVRC2012_val_00017780.JPEG:n07831146 +ILSVRC2012_val_00017781.JPEG:n02457408 +ILSVRC2012_val_00017782.JPEG:n04254680 +ILSVRC2012_val_00017783.JPEG:n03028079 +ILSVRC2012_val_00017784.JPEG:n03498962 +ILSVRC2012_val_00017785.JPEG:n02883205 +ILSVRC2012_val_00017786.JPEG:n02077923 +ILSVRC2012_val_00017787.JPEG:n02090721 +ILSVRC2012_val_00017788.JPEG:n04005630 +ILSVRC2012_val_00017789.JPEG:n02056570 +ILSVRC2012_val_00017790.JPEG:n01775062 +ILSVRC2012_val_00017791.JPEG:n03866082 +ILSVRC2012_val_00017792.JPEG:n02087394 +ILSVRC2012_val_00017793.JPEG:n04336792 +ILSVRC2012_val_00017794.JPEG:n01917289 +ILSVRC2012_val_00017795.JPEG:n04111531 +ILSVRC2012_val_00017796.JPEG:n02007558 +ILSVRC2012_val_00017797.JPEG:n04086273 +ILSVRC2012_val_00017798.JPEG:n02843684 +ILSVRC2012_val_00017799.JPEG:n13037406 +ILSVRC2012_val_00017800.JPEG:n04200800 +ILSVRC2012_val_00017801.JPEG:n03000684 +ILSVRC2012_val_00017802.JPEG:n03991062 +ILSVRC2012_val_00017803.JPEG:n02488702 +ILSVRC2012_val_00017804.JPEG:n02808440 +ILSVRC2012_val_00017805.JPEG:n03887697 +ILSVRC2012_val_00017806.JPEG:n01784675 +ILSVRC2012_val_00017807.JPEG:n02058221 +ILSVRC2012_val_00017808.JPEG:n02841315 +ILSVRC2012_val_00017809.JPEG:n02114367 +ILSVRC2012_val_00017810.JPEG:n03657121 +ILSVRC2012_val_00017811.JPEG:n02787622 +ILSVRC2012_val_00017812.JPEG:n03095699 +ILSVRC2012_val_00017813.JPEG:n03450230 +ILSVRC2012_val_00017814.JPEG:n02123394 +ILSVRC2012_val_00017815.JPEG:n02869837 +ILSVRC2012_val_00017816.JPEG:n03793489 +ILSVRC2012_val_00017817.JPEG:n02094258 +ILSVRC2012_val_00017818.JPEG:n04380533 +ILSVRC2012_val_00017819.JPEG:n02978881 +ILSVRC2012_val_00017820.JPEG:n07584110 +ILSVRC2012_val_00017821.JPEG:n02927161 +ILSVRC2012_val_00017822.JPEG:n02930766 +ILSVRC2012_val_00017823.JPEG:n02093428 +ILSVRC2012_val_00017824.JPEG:n04507155 +ILSVRC2012_val_00017825.JPEG:n03534580 +ILSVRC2012_val_00017826.JPEG:n03857828 +ILSVRC2012_val_00017827.JPEG:n01872401 +ILSVRC2012_val_00017828.JPEG:n03337140 +ILSVRC2012_val_00017829.JPEG:n02980441 +ILSVRC2012_val_00017830.JPEG:n02102177 +ILSVRC2012_val_00017831.JPEG:n02509815 +ILSVRC2012_val_00017832.JPEG:n02097047 +ILSVRC2012_val_00017833.JPEG:n02992529 +ILSVRC2012_val_00017834.JPEG:n02797295 +ILSVRC2012_val_00017835.JPEG:n03866082 +ILSVRC2012_val_00017836.JPEG:n02279972 +ILSVRC2012_val_00017837.JPEG:n03485794 +ILSVRC2012_val_00017838.JPEG:n03530642 +ILSVRC2012_val_00017839.JPEG:n01518878 +ILSVRC2012_val_00017840.JPEG:n04483307 +ILSVRC2012_val_00017841.JPEG:n04033901 +ILSVRC2012_val_00017842.JPEG:n07749582 +ILSVRC2012_val_00017843.JPEG:n02917067 +ILSVRC2012_val_00017844.JPEG:n03623198 +ILSVRC2012_val_00017845.JPEG:n02233338 +ILSVRC2012_val_00017846.JPEG:n03623198 +ILSVRC2012_val_00017847.JPEG:n03594945 +ILSVRC2012_val_00017848.JPEG:n02256656 +ILSVRC2012_val_00017849.JPEG:n02999410 +ILSVRC2012_val_00017850.JPEG:n02093991 +ILSVRC2012_val_00017851.JPEG:n02002724 +ILSVRC2012_val_00017852.JPEG:n03788365 +ILSVRC2012_val_00017853.JPEG:n03623198 +ILSVRC2012_val_00017854.JPEG:n02110063 +ILSVRC2012_val_00017855.JPEG:n01740131 +ILSVRC2012_val_00017856.JPEG:n04346328 +ILSVRC2012_val_00017857.JPEG:n04033995 +ILSVRC2012_val_00017858.JPEG:n02095889 +ILSVRC2012_val_00017859.JPEG:n04311174 +ILSVRC2012_val_00017860.JPEG:n02445715 +ILSVRC2012_val_00017861.JPEG:n03218198 +ILSVRC2012_val_00017862.JPEG:n02640242 +ILSVRC2012_val_00017863.JPEG:n04462240 +ILSVRC2012_val_00017864.JPEG:n03180011 +ILSVRC2012_val_00017865.JPEG:n02093256 +ILSVRC2012_val_00017866.JPEG:n03425413 +ILSVRC2012_val_00017867.JPEG:n02504013 +ILSVRC2012_val_00017868.JPEG:n03877472 +ILSVRC2012_val_00017869.JPEG:n02087046 +ILSVRC2012_val_00017870.JPEG:n03976467 +ILSVRC2012_val_00017871.JPEG:n02091134 +ILSVRC2012_val_00017872.JPEG:n04044716 +ILSVRC2012_val_00017873.JPEG:n02088364 +ILSVRC2012_val_00017874.JPEG:n02009912 +ILSVRC2012_val_00017875.JPEG:n02206856 +ILSVRC2012_val_00017876.JPEG:n03297495 +ILSVRC2012_val_00017877.JPEG:n02871525 +ILSVRC2012_val_00017878.JPEG:n03633091 +ILSVRC2012_val_00017879.JPEG:n02105855 +ILSVRC2012_val_00017880.JPEG:n03075370 +ILSVRC2012_val_00017881.JPEG:n02119789 +ILSVRC2012_val_00017882.JPEG:n01644373 +ILSVRC2012_val_00017883.JPEG:n03216828 +ILSVRC2012_val_00017884.JPEG:n03478589 +ILSVRC2012_val_00017885.JPEG:n03929855 +ILSVRC2012_val_00017886.JPEG:n02939185 +ILSVRC2012_val_00017887.JPEG:n01847000 +ILSVRC2012_val_00017888.JPEG:n02317335 +ILSVRC2012_val_00017889.JPEG:n01983481 +ILSVRC2012_val_00017890.JPEG:n03657121 +ILSVRC2012_val_00017891.JPEG:n02086910 +ILSVRC2012_val_00017892.JPEG:n02088238 +ILSVRC2012_val_00017893.JPEG:n02168699 +ILSVRC2012_val_00017894.JPEG:n03976467 +ILSVRC2012_val_00017895.JPEG:n07697313 +ILSVRC2012_val_00017896.JPEG:n03743016 +ILSVRC2012_val_00017897.JPEG:n04086273 +ILSVRC2012_val_00017898.JPEG:n04200800 +ILSVRC2012_val_00017899.JPEG:n01632777 +ILSVRC2012_val_00017900.JPEG:n03529860 +ILSVRC2012_val_00017901.JPEG:n03404251 +ILSVRC2012_val_00017902.JPEG:n03255030 +ILSVRC2012_val_00017903.JPEG:n03476991 +ILSVRC2012_val_00017904.JPEG:n04311174 +ILSVRC2012_val_00017905.JPEG:n02093991 +ILSVRC2012_val_00017906.JPEG:n03924679 +ILSVRC2012_val_00017907.JPEG:n03478589 +ILSVRC2012_val_00017908.JPEG:n04258138 +ILSVRC2012_val_00017909.JPEG:n01774384 +ILSVRC2012_val_00017910.JPEG:n02277742 +ILSVRC2012_val_00017911.JPEG:n01980166 +ILSVRC2012_val_00017912.JPEG:n02951358 +ILSVRC2012_val_00017913.JPEG:n03983396 +ILSVRC2012_val_00017914.JPEG:n03482405 +ILSVRC2012_val_00017915.JPEG:n02091244 +ILSVRC2012_val_00017916.JPEG:n01592084 +ILSVRC2012_val_00017917.JPEG:n02415577 +ILSVRC2012_val_00017918.JPEG:n02125311 +ILSVRC2012_val_00017919.JPEG:n03888257 +ILSVRC2012_val_00017920.JPEG:n03871628 +ILSVRC2012_val_00017921.JPEG:n02096437 +ILSVRC2012_val_00017922.JPEG:n03743016 +ILSVRC2012_val_00017923.JPEG:n04118776 +ILSVRC2012_val_00017924.JPEG:n02526121 +ILSVRC2012_val_00017925.JPEG:n07711569 +ILSVRC2012_val_00017926.JPEG:n01694178 +ILSVRC2012_val_00017927.JPEG:n01744401 +ILSVRC2012_val_00017928.JPEG:n03424325 +ILSVRC2012_val_00017929.JPEG:n10565667 +ILSVRC2012_val_00017930.JPEG:n02007558 +ILSVRC2012_val_00017931.JPEG:n01860187 +ILSVRC2012_val_00017932.JPEG:n03127925 +ILSVRC2012_val_00017933.JPEG:n04380533 +ILSVRC2012_val_00017934.JPEG:n03637318 +ILSVRC2012_val_00017935.JPEG:n02088238 +ILSVRC2012_val_00017936.JPEG:n04118538 +ILSVRC2012_val_00017937.JPEG:n02101006 +ILSVRC2012_val_00017938.JPEG:n02110958 +ILSVRC2012_val_00017939.JPEG:n01820546 +ILSVRC2012_val_00017940.JPEG:n02106550 +ILSVRC2012_val_00017941.JPEG:n03874293 +ILSVRC2012_val_00017942.JPEG:n02229544 +ILSVRC2012_val_00017943.JPEG:n03937543 +ILSVRC2012_val_00017944.JPEG:n03838899 +ILSVRC2012_val_00017945.JPEG:n04147183 +ILSVRC2012_val_00017946.JPEG:n03697007 +ILSVRC2012_val_00017947.JPEG:n02655020 +ILSVRC2012_val_00017948.JPEG:n01677366 +ILSVRC2012_val_00017949.JPEG:n02415577 +ILSVRC2012_val_00017950.JPEG:n03891332 +ILSVRC2012_val_00017951.JPEG:n03673027 +ILSVRC2012_val_00017952.JPEG:n02328150 +ILSVRC2012_val_00017953.JPEG:n02363005 +ILSVRC2012_val_00017954.JPEG:n04209133 +ILSVRC2012_val_00017955.JPEG:n04065272 +ILSVRC2012_val_00017956.JPEG:n04399382 +ILSVRC2012_val_00017957.JPEG:n02114548 +ILSVRC2012_val_00017958.JPEG:n03724870 +ILSVRC2012_val_00017959.JPEG:n12620546 +ILSVRC2012_val_00017960.JPEG:n04277352 +ILSVRC2012_val_00017961.JPEG:n02105855 +ILSVRC2012_val_00017962.JPEG:n01704323 +ILSVRC2012_val_00017963.JPEG:n01697457 +ILSVRC2012_val_00017964.JPEG:n02094433 +ILSVRC2012_val_00017965.JPEG:n02110958 +ILSVRC2012_val_00017966.JPEG:n02092339 +ILSVRC2012_val_00017967.JPEG:n01734418 +ILSVRC2012_val_00017968.JPEG:n02108915 +ILSVRC2012_val_00017969.JPEG:n02791270 +ILSVRC2012_val_00017970.JPEG:n01534433 +ILSVRC2012_val_00017971.JPEG:n04111531 +ILSVRC2012_val_00017972.JPEG:n03476684 +ILSVRC2012_val_00017973.JPEG:n02708093 +ILSVRC2012_val_00017974.JPEG:n01955084 +ILSVRC2012_val_00017975.JPEG:n01580077 +ILSVRC2012_val_00017976.JPEG:n01592084 +ILSVRC2012_val_00017977.JPEG:n03602883 +ILSVRC2012_val_00017978.JPEG:n02871525 +ILSVRC2012_val_00017979.JPEG:n04037443 +ILSVRC2012_val_00017980.JPEG:n02086910 +ILSVRC2012_val_00017981.JPEG:n13040303 +ILSVRC2012_val_00017982.JPEG:n07749582 +ILSVRC2012_val_00017983.JPEG:n01930112 +ILSVRC2012_val_00017984.JPEG:n13037406 +ILSVRC2012_val_00017985.JPEG:n03792972 +ILSVRC2012_val_00017986.JPEG:n01775062 +ILSVRC2012_val_00017987.JPEG:n02403003 +ILSVRC2012_val_00017988.JPEG:n02974003 +ILSVRC2012_val_00017989.JPEG:n01644373 +ILSVRC2012_val_00017990.JPEG:n02966193 +ILSVRC2012_val_00017991.JPEG:n03481172 +ILSVRC2012_val_00017992.JPEG:n02095570 +ILSVRC2012_val_00017993.JPEG:n03297495 +ILSVRC2012_val_00017994.JPEG:n01614925 +ILSVRC2012_val_00017995.JPEG:n01440764 +ILSVRC2012_val_00017996.JPEG:n02879718 +ILSVRC2012_val_00017997.JPEG:n02105641 +ILSVRC2012_val_00017998.JPEG:n03125729 +ILSVRC2012_val_00017999.JPEG:n03891332 +ILSVRC2012_val_00018000.JPEG:n01697457 +ILSVRC2012_val_00018001.JPEG:n03443371 +ILSVRC2012_val_00018002.JPEG:n03794056 +ILSVRC2012_val_00018003.JPEG:n02231487 +ILSVRC2012_val_00018004.JPEG:n02395406 +ILSVRC2012_val_00018005.JPEG:n02787622 +ILSVRC2012_val_00018006.JPEG:n03425413 +ILSVRC2012_val_00018007.JPEG:n02111889 +ILSVRC2012_val_00018008.JPEG:n01632458 +ILSVRC2012_val_00018009.JPEG:n02110806 +ILSVRC2012_val_00018010.JPEG:n03584829 +ILSVRC2012_val_00018011.JPEG:n03733805 +ILSVRC2012_val_00018012.JPEG:n04613696 +ILSVRC2012_val_00018013.JPEG:n07747607 +ILSVRC2012_val_00018014.JPEG:n02687172 +ILSVRC2012_val_00018015.JPEG:n03792782 +ILSVRC2012_val_00018016.JPEG:n02492035 +ILSVRC2012_val_00018017.JPEG:n02489166 +ILSVRC2012_val_00018018.JPEG:n03393912 +ILSVRC2012_val_00018019.JPEG:n03018349 +ILSVRC2012_val_00018020.JPEG:n03843555 +ILSVRC2012_val_00018021.JPEG:n02769748 +ILSVRC2012_val_00018022.JPEG:n02168699 +ILSVRC2012_val_00018023.JPEG:n03272010 +ILSVRC2012_val_00018024.JPEG:n04532106 +ILSVRC2012_val_00018025.JPEG:n01943899 +ILSVRC2012_val_00018026.JPEG:n01882714 +ILSVRC2012_val_00018027.JPEG:n03127747 +ILSVRC2012_val_00018028.JPEG:n02088632 +ILSVRC2012_val_00018029.JPEG:n04589890 +ILSVRC2012_val_00018030.JPEG:n12768682 +ILSVRC2012_val_00018031.JPEG:n07715103 +ILSVRC2012_val_00018032.JPEG:n02410509 +ILSVRC2012_val_00018033.JPEG:n03995372 +ILSVRC2012_val_00018034.JPEG:n01728920 +ILSVRC2012_val_00018035.JPEG:n02091134 +ILSVRC2012_val_00018036.JPEG:n01820546 +ILSVRC2012_val_00018037.JPEG:n01739381 +ILSVRC2012_val_00018038.JPEG:n02917067 +ILSVRC2012_val_00018039.JPEG:n04591157 +ILSVRC2012_val_00018040.JPEG:n07697313 +ILSVRC2012_val_00018041.JPEG:n01728920 +ILSVRC2012_val_00018042.JPEG:n02835271 +ILSVRC2012_val_00018043.JPEG:n02028035 +ILSVRC2012_val_00018044.JPEG:n03908714 +ILSVRC2012_val_00018045.JPEG:n02096294 +ILSVRC2012_val_00018046.JPEG:n02106030 +ILSVRC2012_val_00018047.JPEG:n03384352 +ILSVRC2012_val_00018048.JPEG:n02174001 +ILSVRC2012_val_00018049.JPEG:n04522168 +ILSVRC2012_val_00018050.JPEG:n03866082 +ILSVRC2012_val_00018051.JPEG:n02817516 +ILSVRC2012_val_00018052.JPEG:n01978287 +ILSVRC2012_val_00018053.JPEG:n04259630 +ILSVRC2012_val_00018054.JPEG:n04399382 +ILSVRC2012_val_00018055.JPEG:n02113978 +ILSVRC2012_val_00018056.JPEG:n03447721 +ILSVRC2012_val_00018057.JPEG:n02749479 +ILSVRC2012_val_00018058.JPEG:n03188531 +ILSVRC2012_val_00018059.JPEG:n02483708 +ILSVRC2012_val_00018060.JPEG:n07693725 +ILSVRC2012_val_00018061.JPEG:n03014705 +ILSVRC2012_val_00018062.JPEG:n01622779 +ILSVRC2012_val_00018063.JPEG:n03642806 +ILSVRC2012_val_00018064.JPEG:n02018207 +ILSVRC2012_val_00018065.JPEG:n09332890 +ILSVRC2012_val_00018066.JPEG:n03670208 +ILSVRC2012_val_00018067.JPEG:n03291819 +ILSVRC2012_val_00018068.JPEG:n02017213 +ILSVRC2012_val_00018069.JPEG:n02098286 +ILSVRC2012_val_00018070.JPEG:n04141327 +ILSVRC2012_val_00018071.JPEG:n02105251 +ILSVRC2012_val_00018072.JPEG:n02447366 +ILSVRC2012_val_00018073.JPEG:n02321529 +ILSVRC2012_val_00018074.JPEG:n03792782 +ILSVRC2012_val_00018075.JPEG:n01443537 +ILSVRC2012_val_00018076.JPEG:n01943899 +ILSVRC2012_val_00018077.JPEG:n04522168 +ILSVRC2012_val_00018078.JPEG:n13133613 +ILSVRC2012_val_00018079.JPEG:n03891251 +ILSVRC2012_val_00018080.JPEG:n02106166 +ILSVRC2012_val_00018081.JPEG:n04592741 +ILSVRC2012_val_00018082.JPEG:n04179913 +ILSVRC2012_val_00018083.JPEG:n03216828 +ILSVRC2012_val_00018084.JPEG:n04467665 +ILSVRC2012_val_00018085.JPEG:n01883070 +ILSVRC2012_val_00018086.JPEG:n07614500 +ILSVRC2012_val_00018087.JPEG:n02105162 +ILSVRC2012_val_00018088.JPEG:n04456115 +ILSVRC2012_val_00018089.JPEG:n04332243 +ILSVRC2012_val_00018090.JPEG:n04049303 +ILSVRC2012_val_00018091.JPEG:n07615774 +ILSVRC2012_val_00018092.JPEG:n01616318 +ILSVRC2012_val_00018093.JPEG:n07802026 +ILSVRC2012_val_00018094.JPEG:n03291819 +ILSVRC2012_val_00018095.JPEG:n01688243 +ILSVRC2012_val_00018096.JPEG:n02396427 +ILSVRC2012_val_00018097.JPEG:n09229709 +ILSVRC2012_val_00018098.JPEG:n09399592 +ILSVRC2012_val_00018099.JPEG:n02027492 +ILSVRC2012_val_00018100.JPEG:n04517823 +ILSVRC2012_val_00018101.JPEG:n03325584 +ILSVRC2012_val_00018102.JPEG:n02165456 +ILSVRC2012_val_00018103.JPEG:n03803284 +ILSVRC2012_val_00018104.JPEG:n02802426 +ILSVRC2012_val_00018105.JPEG:n09428293 +ILSVRC2012_val_00018106.JPEG:n02168699 +ILSVRC2012_val_00018107.JPEG:n02106662 +ILSVRC2012_val_00018108.JPEG:n03259280 +ILSVRC2012_val_00018109.JPEG:n03733131 +ILSVRC2012_val_00018110.JPEG:n04258138 +ILSVRC2012_val_00018111.JPEG:n01924916 +ILSVRC2012_val_00018112.JPEG:n01945685 +ILSVRC2012_val_00018113.JPEG:n09428293 +ILSVRC2012_val_00018114.JPEG:n02871525 +ILSVRC2012_val_00018115.JPEG:n02786058 +ILSVRC2012_val_00018116.JPEG:n03721384 +ILSVRC2012_val_00018117.JPEG:n04285008 +ILSVRC2012_val_00018118.JPEG:n03485794 +ILSVRC2012_val_00018119.JPEG:n01784675 +ILSVRC2012_val_00018120.JPEG:n04428191 +ILSVRC2012_val_00018121.JPEG:n02092002 +ILSVRC2012_val_00018122.JPEG:n04372370 +ILSVRC2012_val_00018123.JPEG:n04099969 +ILSVRC2012_val_00018124.JPEG:n03026506 +ILSVRC2012_val_00018125.JPEG:n02971356 +ILSVRC2012_val_00018126.JPEG:n02106030 +ILSVRC2012_val_00018127.JPEG:n04131690 +ILSVRC2012_val_00018128.JPEG:n01847000 +ILSVRC2012_val_00018129.JPEG:n03794056 +ILSVRC2012_val_00018130.JPEG:n12985857 +ILSVRC2012_val_00018131.JPEG:n02488702 +ILSVRC2012_val_00018132.JPEG:n01872401 +ILSVRC2012_val_00018133.JPEG:n03372029 +ILSVRC2012_val_00018134.JPEG:n01806567 +ILSVRC2012_val_00018135.JPEG:n01917289 +ILSVRC2012_val_00018136.JPEG:n03444034 +ILSVRC2012_val_00018137.JPEG:n01776313 +ILSVRC2012_val_00018138.JPEG:n02814533 +ILSVRC2012_val_00018139.JPEG:n02672831 +ILSVRC2012_val_00018140.JPEG:n03637318 +ILSVRC2012_val_00018141.JPEG:n02113978 +ILSVRC2012_val_00018142.JPEG:n02165456 +ILSVRC2012_val_00018143.JPEG:n04548280 +ILSVRC2012_val_00018144.JPEG:n02917067 +ILSVRC2012_val_00018145.JPEG:n01560419 +ILSVRC2012_val_00018146.JPEG:n02825657 +ILSVRC2012_val_00018147.JPEG:n04552348 +ILSVRC2012_val_00018148.JPEG:n02999410 +ILSVRC2012_val_00018149.JPEG:n02190166 +ILSVRC2012_val_00018150.JPEG:n03065424 +ILSVRC2012_val_00018151.JPEG:n02825657 +ILSVRC2012_val_00018152.JPEG:n07716358 +ILSVRC2012_val_00018153.JPEG:n02877765 +ILSVRC2012_val_00018154.JPEG:n09421951 +ILSVRC2012_val_00018155.JPEG:n12267677 +ILSVRC2012_val_00018156.JPEG:n01819313 +ILSVRC2012_val_00018157.JPEG:n04264628 +ILSVRC2012_val_00018158.JPEG:n03344393 +ILSVRC2012_val_00018159.JPEG:n02002724 +ILSVRC2012_val_00018160.JPEG:n01641577 +ILSVRC2012_val_00018161.JPEG:n02256656 +ILSVRC2012_val_00018162.JPEG:n01532829 +ILSVRC2012_val_00018163.JPEG:n03854065 +ILSVRC2012_val_00018164.JPEG:n02791270 +ILSVRC2012_val_00018165.JPEG:n02951585 +ILSVRC2012_val_00018166.JPEG:n03014705 +ILSVRC2012_val_00018167.JPEG:n01592084 +ILSVRC2012_val_00018168.JPEG:n01728572 +ILSVRC2012_val_00018169.JPEG:n01774750 +ILSVRC2012_val_00018170.JPEG:n03868242 +ILSVRC2012_val_00018171.JPEG:n04370456 +ILSVRC2012_val_00018172.JPEG:n03337140 +ILSVRC2012_val_00018173.JPEG:n03124043 +ILSVRC2012_val_00018174.JPEG:n03290653 +ILSVRC2012_val_00018175.JPEG:n02488291 +ILSVRC2012_val_00018176.JPEG:n04505470 +ILSVRC2012_val_00018177.JPEG:n04553703 +ILSVRC2012_val_00018178.JPEG:n02107574 +ILSVRC2012_val_00018179.JPEG:n01692333 +ILSVRC2012_val_00018180.JPEG:n12620546 +ILSVRC2012_val_00018181.JPEG:n04086273 +ILSVRC2012_val_00018182.JPEG:n03657121 +ILSVRC2012_val_00018183.JPEG:n01582220 +ILSVRC2012_val_00018184.JPEG:n03485407 +ILSVRC2012_val_00018185.JPEG:n03840681 +ILSVRC2012_val_00018186.JPEG:n07768694 +ILSVRC2012_val_00018187.JPEG:n03782006 +ILSVRC2012_val_00018188.JPEG:n02114548 +ILSVRC2012_val_00018189.JPEG:n11939491 +ILSVRC2012_val_00018190.JPEG:n04552348 +ILSVRC2012_val_00018191.JPEG:n03208938 +ILSVRC2012_val_00018192.JPEG:n02006656 +ILSVRC2012_val_00018193.JPEG:n03764736 +ILSVRC2012_val_00018194.JPEG:n07695742 +ILSVRC2012_val_00018195.JPEG:n01820546 +ILSVRC2012_val_00018196.JPEG:n02326432 +ILSVRC2012_val_00018197.JPEG:n02009229 +ILSVRC2012_val_00018198.JPEG:n02408429 +ILSVRC2012_val_00018199.JPEG:n03018349 +ILSVRC2012_val_00018200.JPEG:n03018349 +ILSVRC2012_val_00018201.JPEG:n02504458 +ILSVRC2012_val_00018202.JPEG:n02089973 +ILSVRC2012_val_00018203.JPEG:n01917289 +ILSVRC2012_val_00018204.JPEG:n01739381 +ILSVRC2012_val_00018205.JPEG:n02130308 +ILSVRC2012_val_00018206.JPEG:n04099969 +ILSVRC2012_val_00018207.JPEG:n02102040 +ILSVRC2012_val_00018208.JPEG:n03788195 +ILSVRC2012_val_00018209.JPEG:n03764736 +ILSVRC2012_val_00018210.JPEG:n02422699 +ILSVRC2012_val_00018211.JPEG:n01978287 +ILSVRC2012_val_00018212.JPEG:n02860847 +ILSVRC2012_val_00018213.JPEG:n02749479 +ILSVRC2012_val_00018214.JPEG:n03877845 +ILSVRC2012_val_00018215.JPEG:n03404251 +ILSVRC2012_val_00018216.JPEG:n04209133 +ILSVRC2012_val_00018217.JPEG:n07695742 +ILSVRC2012_val_00018218.JPEG:n04090263 +ILSVRC2012_val_00018219.JPEG:n03720891 +ILSVRC2012_val_00018220.JPEG:n04311174 +ILSVRC2012_val_00018221.JPEG:n03642806 +ILSVRC2012_val_00018222.JPEG:n03933933 +ILSVRC2012_val_00018223.JPEG:n04005630 +ILSVRC2012_val_00018224.JPEG:n02093991 +ILSVRC2012_val_00018225.JPEG:n02977058 +ILSVRC2012_val_00018226.JPEG:n09835506 +ILSVRC2012_val_00018227.JPEG:n03417042 +ILSVRC2012_val_00018228.JPEG:n01742172 +ILSVRC2012_val_00018229.JPEG:n03888257 +ILSVRC2012_val_00018230.JPEG:n02782093 +ILSVRC2012_val_00018231.JPEG:n07802026 +ILSVRC2012_val_00018232.JPEG:n03208938 +ILSVRC2012_val_00018233.JPEG:n02130308 +ILSVRC2012_val_00018234.JPEG:n02090622 +ILSVRC2012_val_00018235.JPEG:n04040759 +ILSVRC2012_val_00018236.JPEG:n02422699 +ILSVRC2012_val_00018237.JPEG:n03594945 +ILSVRC2012_val_00018238.JPEG:n02437616 +ILSVRC2012_val_00018239.JPEG:n03337140 +ILSVRC2012_val_00018240.JPEG:n09399592 +ILSVRC2012_val_00018241.JPEG:n02129604 +ILSVRC2012_val_00018242.JPEG:n02488291 +ILSVRC2012_val_00018243.JPEG:n04597913 +ILSVRC2012_val_00018244.JPEG:n03089624 +ILSVRC2012_val_00018245.JPEG:n03710193 +ILSVRC2012_val_00018246.JPEG:n02930766 +ILSVRC2012_val_00018247.JPEG:n04435653 +ILSVRC2012_val_00018248.JPEG:n01806567 +ILSVRC2012_val_00018249.JPEG:n03100240 +ILSVRC2012_val_00018250.JPEG:n01582220 +ILSVRC2012_val_00018251.JPEG:n03871628 +ILSVRC2012_val_00018252.JPEG:n02422106 +ILSVRC2012_val_00018253.JPEG:n02494079 +ILSVRC2012_val_00018254.JPEG:n04372370 +ILSVRC2012_val_00018255.JPEG:n07716358 +ILSVRC2012_val_00018256.JPEG:n04277352 +ILSVRC2012_val_00018257.JPEG:n02236044 +ILSVRC2012_val_00018258.JPEG:n03891332 +ILSVRC2012_val_00018259.JPEG:n03814639 +ILSVRC2012_val_00018260.JPEG:n02396427 +ILSVRC2012_val_00018261.JPEG:n02793495 +ILSVRC2012_val_00018262.JPEG:n02096437 +ILSVRC2012_val_00018263.JPEG:n02504458 +ILSVRC2012_val_00018264.JPEG:n02085936 +ILSVRC2012_val_00018265.JPEG:n01978287 +ILSVRC2012_val_00018266.JPEG:n04239074 +ILSVRC2012_val_00018267.JPEG:n03532672 +ILSVRC2012_val_00018268.JPEG:n02869837 +ILSVRC2012_val_00018269.JPEG:n02127052 +ILSVRC2012_val_00018270.JPEG:n03680355 +ILSVRC2012_val_00018271.JPEG:n02206856 +ILSVRC2012_val_00018272.JPEG:n03602883 +ILSVRC2012_val_00018273.JPEG:n01817953 +ILSVRC2012_val_00018274.JPEG:n03733805 +ILSVRC2012_val_00018275.JPEG:n03938244 +ILSVRC2012_val_00018276.JPEG:n03450230 +ILSVRC2012_val_00018277.JPEG:n04044716 +ILSVRC2012_val_00018278.JPEG:n02965783 +ILSVRC2012_val_00018279.JPEG:n03938244 +ILSVRC2012_val_00018280.JPEG:n01592084 +ILSVRC2012_val_00018281.JPEG:n03290653 +ILSVRC2012_val_00018282.JPEG:n04479046 +ILSVRC2012_val_00018283.JPEG:n07831146 +ILSVRC2012_val_00018284.JPEG:n01735189 +ILSVRC2012_val_00018285.JPEG:n04525305 +ILSVRC2012_val_00018286.JPEG:n02870880 +ILSVRC2012_val_00018287.JPEG:n02776631 +ILSVRC2012_val_00018288.JPEG:n02172182 +ILSVRC2012_val_00018289.JPEG:n04081281 +ILSVRC2012_val_00018290.JPEG:n03876231 +ILSVRC2012_val_00018291.JPEG:n01985128 +ILSVRC2012_val_00018292.JPEG:n01917289 +ILSVRC2012_val_00018293.JPEG:n10148035 +ILSVRC2012_val_00018294.JPEG:n04286575 +ILSVRC2012_val_00018295.JPEG:n03598930 +ILSVRC2012_val_00018296.JPEG:n02085782 +ILSVRC2012_val_00018297.JPEG:n02699494 +ILSVRC2012_val_00018298.JPEG:n04009552 +ILSVRC2012_val_00018299.JPEG:n03492542 +ILSVRC2012_val_00018300.JPEG:n07749582 +ILSVRC2012_val_00018301.JPEG:n03017168 +ILSVRC2012_val_00018302.JPEG:n03494278 +ILSVRC2012_val_00018303.JPEG:n02134418 +ILSVRC2012_val_00018304.JPEG:n03792782 +ILSVRC2012_val_00018305.JPEG:n01687978 +ILSVRC2012_val_00018306.JPEG:n13040303 +ILSVRC2012_val_00018307.JPEG:n03220513 +ILSVRC2012_val_00018308.JPEG:n03347037 +ILSVRC2012_val_00018309.JPEG:n03476684 +ILSVRC2012_val_00018310.JPEG:n01828970 +ILSVRC2012_val_00018311.JPEG:n02114367 +ILSVRC2012_val_00018312.JPEG:n07715103 +ILSVRC2012_val_00018313.JPEG:n02119789 +ILSVRC2012_val_00018314.JPEG:n01749939 +ILSVRC2012_val_00018315.JPEG:n03791053 +ILSVRC2012_val_00018316.JPEG:n02457408 +ILSVRC2012_val_00018317.JPEG:n01440764 +ILSVRC2012_val_00018318.JPEG:n01824575 +ILSVRC2012_val_00018319.JPEG:n04372370 +ILSVRC2012_val_00018320.JPEG:n07802026 +ILSVRC2012_val_00018321.JPEG:n04270147 +ILSVRC2012_val_00018322.JPEG:n04033901 +ILSVRC2012_val_00018323.JPEG:n04515003 +ILSVRC2012_val_00018324.JPEG:n03950228 +ILSVRC2012_val_00018325.JPEG:n04005630 +ILSVRC2012_val_00018326.JPEG:n02091032 +ILSVRC2012_val_00018327.JPEG:n02090379 +ILSVRC2012_val_00018328.JPEG:n02486410 +ILSVRC2012_val_00018329.JPEG:n07684084 +ILSVRC2012_val_00018330.JPEG:n04592741 +ILSVRC2012_val_00018331.JPEG:n02106382 +ILSVRC2012_val_00018332.JPEG:n02165456 +ILSVRC2012_val_00018333.JPEG:n02483708 +ILSVRC2012_val_00018334.JPEG:n01737021 +ILSVRC2012_val_00018335.JPEG:n02814533 +ILSVRC2012_val_00018336.JPEG:n04081281 +ILSVRC2012_val_00018337.JPEG:n03884397 +ILSVRC2012_val_00018338.JPEG:n07749582 +ILSVRC2012_val_00018339.JPEG:n01641577 +ILSVRC2012_val_00018340.JPEG:n03929855 +ILSVRC2012_val_00018341.JPEG:n04550184 +ILSVRC2012_val_00018342.JPEG:n04467665 +ILSVRC2012_val_00018343.JPEG:n03930313 +ILSVRC2012_val_00018344.JPEG:n02951585 +ILSVRC2012_val_00018345.JPEG:n02747177 +ILSVRC2012_val_00018346.JPEG:n04487394 +ILSVRC2012_val_00018347.JPEG:n01773549 +ILSVRC2012_val_00018348.JPEG:n04228054 +ILSVRC2012_val_00018349.JPEG:n02410509 +ILSVRC2012_val_00018350.JPEG:n04596742 +ILSVRC2012_val_00018351.JPEG:n02795169 +ILSVRC2012_val_00018352.JPEG:n03496892 +ILSVRC2012_val_00018353.JPEG:n04613696 +ILSVRC2012_val_00018354.JPEG:n02398521 +ILSVRC2012_val_00018355.JPEG:n03814906 +ILSVRC2012_val_00018356.JPEG:n02823750 +ILSVRC2012_val_00018357.JPEG:n02106550 +ILSVRC2012_val_00018358.JPEG:n02128385 +ILSVRC2012_val_00018359.JPEG:n02364673 +ILSVRC2012_val_00018360.JPEG:n03770679 +ILSVRC2012_val_00018361.JPEG:n02099429 +ILSVRC2012_val_00018362.JPEG:n01669191 +ILSVRC2012_val_00018363.JPEG:n12057211 +ILSVRC2012_val_00018364.JPEG:n04476259 +ILSVRC2012_val_00018365.JPEG:n02229544 +ILSVRC2012_val_00018366.JPEG:n03781244 +ILSVRC2012_val_00018367.JPEG:n02509815 +ILSVRC2012_val_00018368.JPEG:n02807133 +ILSVRC2012_val_00018369.JPEG:n02132136 +ILSVRC2012_val_00018370.JPEG:n03447721 +ILSVRC2012_val_00018371.JPEG:n02840245 +ILSVRC2012_val_00018372.JPEG:n03743016 +ILSVRC2012_val_00018373.JPEG:n04118776 +ILSVRC2012_val_00018374.JPEG:n04356056 +ILSVRC2012_val_00018375.JPEG:n02190166 +ILSVRC2012_val_00018376.JPEG:n03424325 +ILSVRC2012_val_00018377.JPEG:n04606251 +ILSVRC2012_val_00018378.JPEG:n04146614 +ILSVRC2012_val_00018379.JPEG:n04040759 +ILSVRC2012_val_00018380.JPEG:n07754684 +ILSVRC2012_val_00018381.JPEG:n02119022 +ILSVRC2012_val_00018382.JPEG:n02454379 +ILSVRC2012_val_00018383.JPEG:n02443484 +ILSVRC2012_val_00018384.JPEG:n04310018 +ILSVRC2012_val_00018385.JPEG:n03527444 +ILSVRC2012_val_00018386.JPEG:n04399382 +ILSVRC2012_val_00018387.JPEG:n03843555 +ILSVRC2012_val_00018388.JPEG:n01740131 +ILSVRC2012_val_00018389.JPEG:n02127052 +ILSVRC2012_val_00018390.JPEG:n02749479 +ILSVRC2012_val_00018391.JPEG:n03045698 +ILSVRC2012_val_00018392.JPEG:n02086240 +ILSVRC2012_val_00018393.JPEG:n01795545 +ILSVRC2012_val_00018394.JPEG:n04592741 +ILSVRC2012_val_00018395.JPEG:n02701002 +ILSVRC2012_val_00018396.JPEG:n04149813 +ILSVRC2012_val_00018397.JPEG:n02823750 +ILSVRC2012_val_00018398.JPEG:n01728920 +ILSVRC2012_val_00018399.JPEG:n04493381 +ILSVRC2012_val_00018400.JPEG:n02894605 +ILSVRC2012_val_00018401.JPEG:n03970156 +ILSVRC2012_val_00018402.JPEG:n03838899 +ILSVRC2012_val_00018403.JPEG:n03877845 +ILSVRC2012_val_00018404.JPEG:n03534580 +ILSVRC2012_val_00018405.JPEG:n02094258 +ILSVRC2012_val_00018406.JPEG:n03047690 +ILSVRC2012_val_00018407.JPEG:n02033041 +ILSVRC2012_val_00018408.JPEG:n03208938 +ILSVRC2012_val_00018409.JPEG:n03124043 +ILSVRC2012_val_00018410.JPEG:n03000134 +ILSVRC2012_val_00018411.JPEG:n03250847 +ILSVRC2012_val_00018412.JPEG:n01817953 +ILSVRC2012_val_00018413.JPEG:n02727426 +ILSVRC2012_val_00018414.JPEG:n01669191 +ILSVRC2012_val_00018415.JPEG:n02268443 +ILSVRC2012_val_00018416.JPEG:n03770439 +ILSVRC2012_val_00018417.JPEG:n02389026 +ILSVRC2012_val_00018418.JPEG:n04550184 +ILSVRC2012_val_00018419.JPEG:n02804610 +ILSVRC2012_val_00018420.JPEG:n03461385 +ILSVRC2012_val_00018421.JPEG:n02091244 +ILSVRC2012_val_00018422.JPEG:n02363005 +ILSVRC2012_val_00018423.JPEG:n02391049 +ILSVRC2012_val_00018424.JPEG:n07717410 +ILSVRC2012_val_00018425.JPEG:n03404251 +ILSVRC2012_val_00018426.JPEG:n07695742 +ILSVRC2012_val_00018427.JPEG:n04462240 +ILSVRC2012_val_00018428.JPEG:n01817953 +ILSVRC2012_val_00018429.JPEG:n06359193 +ILSVRC2012_val_00018430.JPEG:n01685808 +ILSVRC2012_val_00018431.JPEG:n02509815 +ILSVRC2012_val_00018432.JPEG:n09835506 +ILSVRC2012_val_00018433.JPEG:n04523525 +ILSVRC2012_val_00018434.JPEG:n04398044 +ILSVRC2012_val_00018435.JPEG:n01955084 +ILSVRC2012_val_00018436.JPEG:n02423022 +ILSVRC2012_val_00018437.JPEG:n02129604 +ILSVRC2012_val_00018438.JPEG:n02066245 +ILSVRC2012_val_00018439.JPEG:n01773797 +ILSVRC2012_val_00018440.JPEG:n02859443 +ILSVRC2012_val_00018441.JPEG:n04090263 +ILSVRC2012_val_00018442.JPEG:n03617480 +ILSVRC2012_val_00018443.JPEG:n04548280 +ILSVRC2012_val_00018444.JPEG:n03929855 +ILSVRC2012_val_00018445.JPEG:n03777754 +ILSVRC2012_val_00018446.JPEG:n02791270 +ILSVRC2012_val_00018447.JPEG:n02317335 +ILSVRC2012_val_00018448.JPEG:n03791053 +ILSVRC2012_val_00018449.JPEG:n03180011 +ILSVRC2012_val_00018450.JPEG:n01677366 +ILSVRC2012_val_00018451.JPEG:n03976467 +ILSVRC2012_val_00018452.JPEG:n02497673 +ILSVRC2012_val_00018453.JPEG:n01729322 +ILSVRC2012_val_00018454.JPEG:n03297495 +ILSVRC2012_val_00018455.JPEG:n02268853 +ILSVRC2012_val_00018456.JPEG:n01742172 +ILSVRC2012_val_00018457.JPEG:n07716906 +ILSVRC2012_val_00018458.JPEG:n03630383 +ILSVRC2012_val_00018459.JPEG:n02825657 +ILSVRC2012_val_00018460.JPEG:n02094258 +ILSVRC2012_val_00018461.JPEG:n07873807 +ILSVRC2012_val_00018462.JPEG:n03776460 +ILSVRC2012_val_00018463.JPEG:n01843383 +ILSVRC2012_val_00018464.JPEG:n02840245 +ILSVRC2012_val_00018465.JPEG:n02607072 +ILSVRC2012_val_00018466.JPEG:n01491361 +ILSVRC2012_val_00018467.JPEG:n03109150 +ILSVRC2012_val_00018468.JPEG:n03908618 +ILSVRC2012_val_00018469.JPEG:n02132136 +ILSVRC2012_val_00018470.JPEG:n01950731 +ILSVRC2012_val_00018471.JPEG:n02133161 +ILSVRC2012_val_00018472.JPEG:n04070727 +ILSVRC2012_val_00018473.JPEG:n03384352 +ILSVRC2012_val_00018474.JPEG:n03594945 +ILSVRC2012_val_00018475.JPEG:n03933933 +ILSVRC2012_val_00018476.JPEG:n03891332 +ILSVRC2012_val_00018477.JPEG:n01968897 +ILSVRC2012_val_00018478.JPEG:n09229709 +ILSVRC2012_val_00018479.JPEG:n02095314 +ILSVRC2012_val_00018480.JPEG:n02088364 +ILSVRC2012_val_00018481.JPEG:n01641577 +ILSVRC2012_val_00018482.JPEG:n03124170 +ILSVRC2012_val_00018483.JPEG:n03272562 +ILSVRC2012_val_00018484.JPEG:n02817516 +ILSVRC2012_val_00018485.JPEG:n01943899 +ILSVRC2012_val_00018486.JPEG:n07590611 +ILSVRC2012_val_00018487.JPEG:n04235860 +ILSVRC2012_val_00018488.JPEG:n03991062 +ILSVRC2012_val_00018489.JPEG:n02006656 +ILSVRC2012_val_00018490.JPEG:n04026417 +ILSVRC2012_val_00018491.JPEG:n02113799 +ILSVRC2012_val_00018492.JPEG:n04311004 +ILSVRC2012_val_00018493.JPEG:n02815834 +ILSVRC2012_val_00018494.JPEG:n04008634 +ILSVRC2012_val_00018495.JPEG:n07718472 +ILSVRC2012_val_00018496.JPEG:n02437616 +ILSVRC2012_val_00018497.JPEG:n04325704 +ILSVRC2012_val_00018498.JPEG:n03676483 +ILSVRC2012_val_00018499.JPEG:n03207941 +ILSVRC2012_val_00018500.JPEG:n02066245 +ILSVRC2012_val_00018501.JPEG:n03873416 +ILSVRC2012_val_00018502.JPEG:n02489166 +ILSVRC2012_val_00018503.JPEG:n03782006 +ILSVRC2012_val_00018504.JPEG:n04523525 +ILSVRC2012_val_00018505.JPEG:n03710637 +ILSVRC2012_val_00018506.JPEG:n02791270 +ILSVRC2012_val_00018507.JPEG:n09835506 +ILSVRC2012_val_00018508.JPEG:n01768244 +ILSVRC2012_val_00018509.JPEG:n03888257 +ILSVRC2012_val_00018510.JPEG:n04325704 +ILSVRC2012_val_00018511.JPEG:n02007558 +ILSVRC2012_val_00018512.JPEG:n01641577 +ILSVRC2012_val_00018513.JPEG:n03983396 +ILSVRC2012_val_00018514.JPEG:n04179913 +ILSVRC2012_val_00018515.JPEG:n03786901 +ILSVRC2012_val_00018516.JPEG:n03425413 +ILSVRC2012_val_00018517.JPEG:n02012849 +ILSVRC2012_val_00018518.JPEG:n03876231 +ILSVRC2012_val_00018519.JPEG:n02802426 +ILSVRC2012_val_00018520.JPEG:n04067472 +ILSVRC2012_val_00018521.JPEG:n02112350 +ILSVRC2012_val_00018522.JPEG:n02797295 +ILSVRC2012_val_00018523.JPEG:n03895866 +ILSVRC2012_val_00018524.JPEG:n07753113 +ILSVRC2012_val_00018525.JPEG:n03297495 +ILSVRC2012_val_00018526.JPEG:n02091635 +ILSVRC2012_val_00018527.JPEG:n04487394 +ILSVRC2012_val_00018528.JPEG:n03729826 +ILSVRC2012_val_00018529.JPEG:n02104029 +ILSVRC2012_val_00018530.JPEG:n02102973 +ILSVRC2012_val_00018531.JPEG:n03000247 +ILSVRC2012_val_00018532.JPEG:n01871265 +ILSVRC2012_val_00018533.JPEG:n03920288 +ILSVRC2012_val_00018534.JPEG:n03627232 +ILSVRC2012_val_00018535.JPEG:n02229544 +ILSVRC2012_val_00018536.JPEG:n02092339 +ILSVRC2012_val_00018537.JPEG:n02802426 +ILSVRC2012_val_00018538.JPEG:n03018349 +ILSVRC2012_val_00018539.JPEG:n13044778 +ILSVRC2012_val_00018540.JPEG:n03014705 +ILSVRC2012_val_00018541.JPEG:n02776631 +ILSVRC2012_val_00018542.JPEG:n03109150 +ILSVRC2012_val_00018543.JPEG:n13052670 +ILSVRC2012_val_00018544.JPEG:n03218198 +ILSVRC2012_val_00018545.JPEG:n04125021 +ILSVRC2012_val_00018546.JPEG:n04550184 +ILSVRC2012_val_00018547.JPEG:n04479046 +ILSVRC2012_val_00018548.JPEG:n04443257 +ILSVRC2012_val_00018549.JPEG:n03908618 +ILSVRC2012_val_00018550.JPEG:n02094433 +ILSVRC2012_val_00018551.JPEG:n02113186 +ILSVRC2012_val_00018552.JPEG:n02105162 +ILSVRC2012_val_00018553.JPEG:n02980441 +ILSVRC2012_val_00018554.JPEG:n02971356 +ILSVRC2012_val_00018555.JPEG:n07697313 +ILSVRC2012_val_00018556.JPEG:n02102177 +ILSVRC2012_val_00018557.JPEG:n04613696 +ILSVRC2012_val_00018558.JPEG:n02095889 +ILSVRC2012_val_00018559.JPEG:n02979186 +ILSVRC2012_val_00018560.JPEG:n09472597 +ILSVRC2012_val_00018561.JPEG:n03476684 +ILSVRC2012_val_00018562.JPEG:n02692877 +ILSVRC2012_val_00018563.JPEG:n01756291 +ILSVRC2012_val_00018564.JPEG:n03976657 +ILSVRC2012_val_00018565.JPEG:n03494278 +ILSVRC2012_val_00018566.JPEG:n03026506 +ILSVRC2012_val_00018567.JPEG:n04228054 +ILSVRC2012_val_00018568.JPEG:n04146614 +ILSVRC2012_val_00018569.JPEG:n03100240 +ILSVRC2012_val_00018570.JPEG:n02018795 +ILSVRC2012_val_00018571.JPEG:n01873310 +ILSVRC2012_val_00018572.JPEG:n04026417 +ILSVRC2012_val_00018573.JPEG:n02086910 +ILSVRC2012_val_00018574.JPEG:n04192698 +ILSVRC2012_val_00018575.JPEG:n02093991 +ILSVRC2012_val_00018576.JPEG:n04116512 +ILSVRC2012_val_00018577.JPEG:n02107908 +ILSVRC2012_val_00018578.JPEG:n02066245 +ILSVRC2012_val_00018579.JPEG:n04026417 +ILSVRC2012_val_00018580.JPEG:n02444819 +ILSVRC2012_val_00018581.JPEG:n02536864 +ILSVRC2012_val_00018582.JPEG:n02361337 +ILSVRC2012_val_00018583.JPEG:n03770439 +ILSVRC2012_val_00018584.JPEG:n02086646 +ILSVRC2012_val_00018585.JPEG:n03444034 +ILSVRC2012_val_00018586.JPEG:n04008634 +ILSVRC2012_val_00018587.JPEG:n02727426 +ILSVRC2012_val_00018588.JPEG:n07615774 +ILSVRC2012_val_00018589.JPEG:n02107908 +ILSVRC2012_val_00018590.JPEG:n03637318 +ILSVRC2012_val_00018591.JPEG:n04317175 +ILSVRC2012_val_00018592.JPEG:n03662601 +ILSVRC2012_val_00018593.JPEG:n09256479 +ILSVRC2012_val_00018594.JPEG:n03933933 +ILSVRC2012_val_00018595.JPEG:n03666591 +ILSVRC2012_val_00018596.JPEG:n02102318 +ILSVRC2012_val_00018597.JPEG:n07802026 +ILSVRC2012_val_00018598.JPEG:n04467665 +ILSVRC2012_val_00018599.JPEG:n03109150 +ILSVRC2012_val_00018600.JPEG:n03710721 +ILSVRC2012_val_00018601.JPEG:n02817516 +ILSVRC2012_val_00018602.JPEG:n01855672 +ILSVRC2012_val_00018603.JPEG:n03259280 +ILSVRC2012_val_00018604.JPEG:n02108089 +ILSVRC2012_val_00018605.JPEG:n01943899 +ILSVRC2012_val_00018606.JPEG:n02655020 +ILSVRC2012_val_00018607.JPEG:n02817516 +ILSVRC2012_val_00018608.JPEG:n07871810 +ILSVRC2012_val_00018609.JPEG:n03935335 +ILSVRC2012_val_00018610.JPEG:n03250847 +ILSVRC2012_val_00018611.JPEG:n04417672 +ILSVRC2012_val_00018612.JPEG:n04252077 +ILSVRC2012_val_00018613.JPEG:n01910747 +ILSVRC2012_val_00018614.JPEG:n03950228 +ILSVRC2012_val_00018615.JPEG:n02009912 +ILSVRC2012_val_00018616.JPEG:n02690373 +ILSVRC2012_val_00018617.JPEG:n02787622 +ILSVRC2012_val_00018618.JPEG:n01685808 +ILSVRC2012_val_00018619.JPEG:n02486410 +ILSVRC2012_val_00018620.JPEG:n04326547 +ILSVRC2012_val_00018621.JPEG:n03467068 +ILSVRC2012_val_00018622.JPEG:n01742172 +ILSVRC2012_val_00018623.JPEG:n02965783 +ILSVRC2012_val_00018624.JPEG:n04209133 +ILSVRC2012_val_00018625.JPEG:n06874185 +ILSVRC2012_val_00018626.JPEG:n01797886 +ILSVRC2012_val_00018627.JPEG:n01755581 +ILSVRC2012_val_00018628.JPEG:n03942813 +ILSVRC2012_val_00018629.JPEG:n02087394 +ILSVRC2012_val_00018630.JPEG:n02137549 +ILSVRC2012_val_00018631.JPEG:n03047690 +ILSVRC2012_val_00018632.JPEG:n04447861 +ILSVRC2012_val_00018633.JPEG:n04275548 +ILSVRC2012_val_00018634.JPEG:n02229544 +ILSVRC2012_val_00018635.JPEG:n03530642 +ILSVRC2012_val_00018636.JPEG:n01930112 +ILSVRC2012_val_00018637.JPEG:n04548362 +ILSVRC2012_val_00018638.JPEG:n04552348 +ILSVRC2012_val_00018639.JPEG:n02486261 +ILSVRC2012_val_00018640.JPEG:n02328150 +ILSVRC2012_val_00018641.JPEG:n03355925 +ILSVRC2012_val_00018642.JPEG:n02096177 +ILSVRC2012_val_00018643.JPEG:n02403003 +ILSVRC2012_val_00018644.JPEG:n01817953 +ILSVRC2012_val_00018645.JPEG:n01629819 +ILSVRC2012_val_00018646.JPEG:n03983396 +ILSVRC2012_val_00018647.JPEG:n03207941 +ILSVRC2012_val_00018648.JPEG:n01806567 +ILSVRC2012_val_00018649.JPEG:n02089973 +ILSVRC2012_val_00018650.JPEG:n07714990 +ILSVRC2012_val_00018651.JPEG:n03590841 +ILSVRC2012_val_00018652.JPEG:n02086646 +ILSVRC2012_val_00018653.JPEG:n03781244 +ILSVRC2012_val_00018654.JPEG:n02090622 +ILSVRC2012_val_00018655.JPEG:n03445924 +ILSVRC2012_val_00018656.JPEG:n02051845 +ILSVRC2012_val_00018657.JPEG:n04560804 +ILSVRC2012_val_00018658.JPEG:n09288635 +ILSVRC2012_val_00018659.JPEG:n03840681 +ILSVRC2012_val_00018660.JPEG:n01622779 +ILSVRC2012_val_00018661.JPEG:n03445924 +ILSVRC2012_val_00018662.JPEG:n02058221 +ILSVRC2012_val_00018663.JPEG:n03837869 +ILSVRC2012_val_00018664.JPEG:n02125311 +ILSVRC2012_val_00018665.JPEG:n02783161 +ILSVRC2012_val_00018666.JPEG:n01698640 +ILSVRC2012_val_00018667.JPEG:n02787622 +ILSVRC2012_val_00018668.JPEG:n03706229 +ILSVRC2012_val_00018669.JPEG:n02840245 +ILSVRC2012_val_00018670.JPEG:n02808440 +ILSVRC2012_val_00018671.JPEG:n03680355 +ILSVRC2012_val_00018672.JPEG:n01560419 +ILSVRC2012_val_00018673.JPEG:n01978287 +ILSVRC2012_val_00018674.JPEG:n02422699 +ILSVRC2012_val_00018675.JPEG:n01687978 +ILSVRC2012_val_00018676.JPEG:n01537544 +ILSVRC2012_val_00018677.JPEG:n03793489 +ILSVRC2012_val_00018678.JPEG:n03016953 +ILSVRC2012_val_00018679.JPEG:n04044716 +ILSVRC2012_val_00018680.JPEG:n01560419 +ILSVRC2012_val_00018681.JPEG:n02056570 +ILSVRC2012_val_00018682.JPEG:n03179701 +ILSVRC2012_val_00018683.JPEG:n09468604 +ILSVRC2012_val_00018684.JPEG:n03623198 +ILSVRC2012_val_00018685.JPEG:n02690373 +ILSVRC2012_val_00018686.JPEG:n02454379 +ILSVRC2012_val_00018687.JPEG:n04467665 +ILSVRC2012_val_00018688.JPEG:n02112018 +ILSVRC2012_val_00018689.JPEG:n04591157 +ILSVRC2012_val_00018690.JPEG:n04243546 +ILSVRC2012_val_00018691.JPEG:n04254777 +ILSVRC2012_val_00018692.JPEG:n01558993 +ILSVRC2012_val_00018693.JPEG:n07932039 +ILSVRC2012_val_00018694.JPEG:n04258138 +ILSVRC2012_val_00018695.JPEG:n02085936 +ILSVRC2012_val_00018696.JPEG:n03240683 +ILSVRC2012_val_00018697.JPEG:n04409515 +ILSVRC2012_val_00018698.JPEG:n03661043 +ILSVRC2012_val_00018699.JPEG:n01532829 +ILSVRC2012_val_00018700.JPEG:n03930630 +ILSVRC2012_val_00018701.JPEG:n02112350 +ILSVRC2012_val_00018702.JPEG:n02837789 +ILSVRC2012_val_00018703.JPEG:n02098286 +ILSVRC2012_val_00018704.JPEG:n04485082 +ILSVRC2012_val_00018705.JPEG:n03272562 +ILSVRC2012_val_00018706.JPEG:n02105505 +ILSVRC2012_val_00018707.JPEG:n03916031 +ILSVRC2012_val_00018708.JPEG:n07742313 +ILSVRC2012_val_00018709.JPEG:n03042490 +ILSVRC2012_val_00018710.JPEG:n02105855 +ILSVRC2012_val_00018711.JPEG:n04229816 +ILSVRC2012_val_00018712.JPEG:n04447861 +ILSVRC2012_val_00018713.JPEG:n02916936 +ILSVRC2012_val_00018714.JPEG:n02120505 +ILSVRC2012_val_00018715.JPEG:n02917067 +ILSVRC2012_val_00018716.JPEG:n01984695 +ILSVRC2012_val_00018717.JPEG:n02454379 +ILSVRC2012_val_00018718.JPEG:n03529860 +ILSVRC2012_val_00018719.JPEG:n03482405 +ILSVRC2012_val_00018720.JPEG:n04049303 +ILSVRC2012_val_00018721.JPEG:n03452741 +ILSVRC2012_val_00018722.JPEG:n02113023 +ILSVRC2012_val_00018723.JPEG:n03447721 +ILSVRC2012_val_00018724.JPEG:n01728572 +ILSVRC2012_val_00018725.JPEG:n03942813 +ILSVRC2012_val_00018726.JPEG:n03929855 +ILSVRC2012_val_00018727.JPEG:n03344393 +ILSVRC2012_val_00018728.JPEG:n01692333 +ILSVRC2012_val_00018729.JPEG:n01945685 +ILSVRC2012_val_00018730.JPEG:n03929660 +ILSVRC2012_val_00018731.JPEG:n07565083 +ILSVRC2012_val_00018732.JPEG:n04579432 +ILSVRC2012_val_00018733.JPEG:n03594734 +ILSVRC2012_val_00018734.JPEG:n03793489 +ILSVRC2012_val_00018735.JPEG:n02114712 +ILSVRC2012_val_00018736.JPEG:n02111129 +ILSVRC2012_val_00018737.JPEG:n02091244 +ILSVRC2012_val_00018738.JPEG:n12057211 +ILSVRC2012_val_00018739.JPEG:n02493793 +ILSVRC2012_val_00018740.JPEG:n03404251 +ILSVRC2012_val_00018741.JPEG:n03026506 +ILSVRC2012_val_00018742.JPEG:n01817953 +ILSVRC2012_val_00018743.JPEG:n02130308 +ILSVRC2012_val_00018744.JPEG:n02930766 +ILSVRC2012_val_00018745.JPEG:n03594734 +ILSVRC2012_val_00018746.JPEG:n02777292 +ILSVRC2012_val_00018747.JPEG:n02486410 +ILSVRC2012_val_00018748.JPEG:n09468604 +ILSVRC2012_val_00018749.JPEG:n02489166 +ILSVRC2012_val_00018750.JPEG:n01981276 +ILSVRC2012_val_00018751.JPEG:n04275548 +ILSVRC2012_val_00018752.JPEG:n02865351 +ILSVRC2012_val_00018753.JPEG:n04118538 +ILSVRC2012_val_00018754.JPEG:n01641577 +ILSVRC2012_val_00018755.JPEG:n02113624 +ILSVRC2012_val_00018756.JPEG:n04008634 +ILSVRC2012_val_00018757.JPEG:n01945685 +ILSVRC2012_val_00018758.JPEG:n02692877 +ILSVRC2012_val_00018759.JPEG:n02749479 +ILSVRC2012_val_00018760.JPEG:n03891332 +ILSVRC2012_val_00018761.JPEG:n02795169 +ILSVRC2012_val_00018762.JPEG:n02105641 +ILSVRC2012_val_00018763.JPEG:n04136333 +ILSVRC2012_val_00018764.JPEG:n04417672 +ILSVRC2012_val_00018765.JPEG:n04263257 +ILSVRC2012_val_00018766.JPEG:n06596364 +ILSVRC2012_val_00018767.JPEG:n02091032 +ILSVRC2012_val_00018768.JPEG:n03770679 +ILSVRC2012_val_00018769.JPEG:n07749582 +ILSVRC2012_val_00018770.JPEG:n02977058 +ILSVRC2012_val_00018771.JPEG:n03594734 +ILSVRC2012_val_00018772.JPEG:n02317335 +ILSVRC2012_val_00018773.JPEG:n04550184 +ILSVRC2012_val_00018774.JPEG:n02437312 +ILSVRC2012_val_00018775.JPEG:n01728572 +ILSVRC2012_val_00018776.JPEG:n02395406 +ILSVRC2012_val_00018777.JPEG:n04522168 +ILSVRC2012_val_00018778.JPEG:n04209133 +ILSVRC2012_val_00018779.JPEG:n02108000 +ILSVRC2012_val_00018780.JPEG:n01843383 +ILSVRC2012_val_00018781.JPEG:n04004767 +ILSVRC2012_val_00018782.JPEG:n03804744 +ILSVRC2012_val_00018783.JPEG:n04398044 +ILSVRC2012_val_00018784.JPEG:n02643566 +ILSVRC2012_val_00018785.JPEG:n13052670 +ILSVRC2012_val_00018786.JPEG:n03443371 +ILSVRC2012_val_00018787.JPEG:n02101388 +ILSVRC2012_val_00018788.JPEG:n02133161 +ILSVRC2012_val_00018789.JPEG:n02641379 +ILSVRC2012_val_00018790.JPEG:n03814906 +ILSVRC2012_val_00018791.JPEG:n02115913 +ILSVRC2012_val_00018792.JPEG:n02108915 +ILSVRC2012_val_00018793.JPEG:n01978287 +ILSVRC2012_val_00018794.JPEG:n04277352 +ILSVRC2012_val_00018795.JPEG:n04493381 +ILSVRC2012_val_00018796.JPEG:n01608432 +ILSVRC2012_val_00018797.JPEG:n04548280 +ILSVRC2012_val_00018798.JPEG:n03379051 +ILSVRC2012_val_00018799.JPEG:n03796401 +ILSVRC2012_val_00018800.JPEG:n02051845 +ILSVRC2012_val_00018801.JPEG:n04350905 +ILSVRC2012_val_00018802.JPEG:n04612504 +ILSVRC2012_val_00018803.JPEG:n03207743 +ILSVRC2012_val_00018804.JPEG:n02097298 +ILSVRC2012_val_00018805.JPEG:n03447447 +ILSVRC2012_val_00018806.JPEG:n02804610 +ILSVRC2012_val_00018807.JPEG:n01770393 +ILSVRC2012_val_00018808.JPEG:n10148035 +ILSVRC2012_val_00018809.JPEG:n02094258 +ILSVRC2012_val_00018810.JPEG:n03720891 +ILSVRC2012_val_00018811.JPEG:n02089078 +ILSVRC2012_val_00018812.JPEG:n02130308 +ILSVRC2012_val_00018813.JPEG:n02536864 +ILSVRC2012_val_00018814.JPEG:n03942813 +ILSVRC2012_val_00018815.JPEG:n02110341 +ILSVRC2012_val_00018816.JPEG:n04579432 +ILSVRC2012_val_00018817.JPEG:n07716358 +ILSVRC2012_val_00018818.JPEG:n03095699 +ILSVRC2012_val_00018819.JPEG:n02128925 +ILSVRC2012_val_00018820.JPEG:n04141975 +ILSVRC2012_val_00018821.JPEG:n02119789 +ILSVRC2012_val_00018822.JPEG:n03481172 +ILSVRC2012_val_00018823.JPEG:n03532672 +ILSVRC2012_val_00018824.JPEG:n02655020 +ILSVRC2012_val_00018825.JPEG:n07749582 +ILSVRC2012_val_00018826.JPEG:n02109961 +ILSVRC2012_val_00018827.JPEG:n02101556 +ILSVRC2012_val_00018828.JPEG:n03662601 +ILSVRC2012_val_00018829.JPEG:n03803284 +ILSVRC2012_val_00018830.JPEG:n02641379 +ILSVRC2012_val_00018831.JPEG:n04367480 +ILSVRC2012_val_00018832.JPEG:n02101388 +ILSVRC2012_val_00018833.JPEG:n04562935 +ILSVRC2012_val_00018834.JPEG:n01694178 +ILSVRC2012_val_00018835.JPEG:n02088466 +ILSVRC2012_val_00018836.JPEG:n02536864 +ILSVRC2012_val_00018837.JPEG:n03781244 +ILSVRC2012_val_00018838.JPEG:n04192698 +ILSVRC2012_val_00018839.JPEG:n02167151 +ILSVRC2012_val_00018840.JPEG:n02089078 +ILSVRC2012_val_00018841.JPEG:n03544143 +ILSVRC2012_val_00018842.JPEG:n03026506 +ILSVRC2012_val_00018843.JPEG:n02128925 +ILSVRC2012_val_00018844.JPEG:n04251144 +ILSVRC2012_val_00018845.JPEG:n03929855 +ILSVRC2012_val_00018846.JPEG:n03085013 +ILSVRC2012_val_00018847.JPEG:n03125729 +ILSVRC2012_val_00018848.JPEG:n01677366 +ILSVRC2012_val_00018849.JPEG:n03661043 +ILSVRC2012_val_00018850.JPEG:n04584207 +ILSVRC2012_val_00018851.JPEG:n04200800 +ILSVRC2012_val_00018852.JPEG:n02487347 +ILSVRC2012_val_00018853.JPEG:n02321529 +ILSVRC2012_val_00018854.JPEG:n03814906 +ILSVRC2012_val_00018855.JPEG:n01924916 +ILSVRC2012_val_00018856.JPEG:n02802426 +ILSVRC2012_val_00018857.JPEG:n01693334 +ILSVRC2012_val_00018858.JPEG:n02169497 +ILSVRC2012_val_00018859.JPEG:n02128925 +ILSVRC2012_val_00018860.JPEG:n07717556 +ILSVRC2012_val_00018861.JPEG:n03895866 +ILSVRC2012_val_00018862.JPEG:n02099429 +ILSVRC2012_val_00018863.JPEG:n03085013 +ILSVRC2012_val_00018864.JPEG:n11939491 +ILSVRC2012_val_00018865.JPEG:n09468604 +ILSVRC2012_val_00018866.JPEG:n02109047 +ILSVRC2012_val_00018867.JPEG:n07565083 +ILSVRC2012_val_00018868.JPEG:n04310018 +ILSVRC2012_val_00018869.JPEG:n02988304 +ILSVRC2012_val_00018870.JPEG:n07754684 +ILSVRC2012_val_00018871.JPEG:n02058221 +ILSVRC2012_val_00018872.JPEG:n02114367 +ILSVRC2012_val_00018873.JPEG:n03485794 +ILSVRC2012_val_00018874.JPEG:n03424325 +ILSVRC2012_val_00018875.JPEG:n04443257 +ILSVRC2012_val_00018876.JPEG:n01697457 +ILSVRC2012_val_00018877.JPEG:n02219486 +ILSVRC2012_val_00018878.JPEG:n02877765 +ILSVRC2012_val_00018879.JPEG:n01644900 +ILSVRC2012_val_00018880.JPEG:n03775071 +ILSVRC2012_val_00018881.JPEG:n02097047 +ILSVRC2012_val_00018882.JPEG:n02085620 +ILSVRC2012_val_00018883.JPEG:n07693725 +ILSVRC2012_val_00018884.JPEG:n03160309 +ILSVRC2012_val_00018885.JPEG:n02815834 +ILSVRC2012_val_00018886.JPEG:n03110669 +ILSVRC2012_val_00018887.JPEG:n03868863 +ILSVRC2012_val_00018888.JPEG:n04008634 +ILSVRC2012_val_00018889.JPEG:n03743016 +ILSVRC2012_val_00018890.JPEG:n02094114 +ILSVRC2012_val_00018891.JPEG:n03208938 +ILSVRC2012_val_00018892.JPEG:n07590611 +ILSVRC2012_val_00018893.JPEG:n04273569 +ILSVRC2012_val_00018894.JPEG:n03706229 +ILSVRC2012_val_00018895.JPEG:n02013706 +ILSVRC2012_val_00018896.JPEG:n07753592 +ILSVRC2012_val_00018897.JPEG:n02916936 +ILSVRC2012_val_00018898.JPEG:n02112137 +ILSVRC2012_val_00018899.JPEG:n02108089 +ILSVRC2012_val_00018900.JPEG:n03841143 +ILSVRC2012_val_00018901.JPEG:n03595614 +ILSVRC2012_val_00018902.JPEG:n03125729 +ILSVRC2012_val_00018903.JPEG:n07742313 +ILSVRC2012_val_00018904.JPEG:n02487347 +ILSVRC2012_val_00018905.JPEG:n04235860 +ILSVRC2012_val_00018906.JPEG:n02782093 +ILSVRC2012_val_00018907.JPEG:n01742172 +ILSVRC2012_val_00018908.JPEG:n04604644 +ILSVRC2012_val_00018909.JPEG:n04554684 +ILSVRC2012_val_00018910.JPEG:n04086273 +ILSVRC2012_val_00018911.JPEG:n02906734 +ILSVRC2012_val_00018912.JPEG:n02091635 +ILSVRC2012_val_00018913.JPEG:n03201208 +ILSVRC2012_val_00018914.JPEG:n07693725 +ILSVRC2012_val_00018915.JPEG:n09332890 +ILSVRC2012_val_00018916.JPEG:n02088364 +ILSVRC2012_val_00018917.JPEG:n03017168 +ILSVRC2012_val_00018918.JPEG:n03729826 +ILSVRC2012_val_00018919.JPEG:n03983396 +ILSVRC2012_val_00018920.JPEG:n03676483 +ILSVRC2012_val_00018921.JPEG:n04204347 +ILSVRC2012_val_00018922.JPEG:n04251144 +ILSVRC2012_val_00018923.JPEG:n02917067 +ILSVRC2012_val_00018924.JPEG:n04081281 +ILSVRC2012_val_00018925.JPEG:n03930313 +ILSVRC2012_val_00018926.JPEG:n03494278 +ILSVRC2012_val_00018927.JPEG:n03160309 +ILSVRC2012_val_00018928.JPEG:n02389026 +ILSVRC2012_val_00018929.JPEG:n03250847 +ILSVRC2012_val_00018930.JPEG:n03133878 +ILSVRC2012_val_00018931.JPEG:n02091635 +ILSVRC2012_val_00018932.JPEG:n02389026 +ILSVRC2012_val_00018933.JPEG:n02087394 +ILSVRC2012_val_00018934.JPEG:n02113799 +ILSVRC2012_val_00018935.JPEG:n02281787 +ILSVRC2012_val_00018936.JPEG:n04548280 +ILSVRC2012_val_00018937.JPEG:n04509417 +ILSVRC2012_val_00018938.JPEG:n03384352 +ILSVRC2012_val_00018939.JPEG:n02009229 +ILSVRC2012_val_00018940.JPEG:n04370456 +ILSVRC2012_val_00018941.JPEG:n07753275 +ILSVRC2012_val_00018942.JPEG:n02102177 +ILSVRC2012_val_00018943.JPEG:n01494475 +ILSVRC2012_val_00018944.JPEG:n03459775 +ILSVRC2012_val_00018945.JPEG:n02804610 +ILSVRC2012_val_00018946.JPEG:n04456115 +ILSVRC2012_val_00018947.JPEG:n02099712 +ILSVRC2012_val_00018948.JPEG:n01494475 +ILSVRC2012_val_00018949.JPEG:n04344873 +ILSVRC2012_val_00018950.JPEG:n03788195 +ILSVRC2012_val_00018951.JPEG:n01944390 +ILSVRC2012_val_00018952.JPEG:n01910747 +ILSVRC2012_val_00018953.JPEG:n03868242 +ILSVRC2012_val_00018954.JPEG:n03452741 +ILSVRC2012_val_00018955.JPEG:n13044778 +ILSVRC2012_val_00018956.JPEG:n01883070 +ILSVRC2012_val_00018957.JPEG:n02701002 +ILSVRC2012_val_00018958.JPEG:n02793495 +ILSVRC2012_val_00018959.JPEG:n02692877 +ILSVRC2012_val_00018960.JPEG:n03220513 +ILSVRC2012_val_00018961.JPEG:n01978287 +ILSVRC2012_val_00018962.JPEG:n02483362 +ILSVRC2012_val_00018963.JPEG:n01776313 +ILSVRC2012_val_00018964.JPEG:n02808304 +ILSVRC2012_val_00018965.JPEG:n03721384 +ILSVRC2012_val_00018966.JPEG:n02012849 +ILSVRC2012_val_00018967.JPEG:n03733281 +ILSVRC2012_val_00018968.JPEG:n07920052 +ILSVRC2012_val_00018969.JPEG:n02326432 +ILSVRC2012_val_00018970.JPEG:n04192698 +ILSVRC2012_val_00018971.JPEG:n02113799 +ILSVRC2012_val_00018972.JPEG:n02106550 +ILSVRC2012_val_00018973.JPEG:n02097298 +ILSVRC2012_val_00018974.JPEG:n02509815 +ILSVRC2012_val_00018975.JPEG:n02835271 +ILSVRC2012_val_00018976.JPEG:n04548280 +ILSVRC2012_val_00018977.JPEG:n04522168 +ILSVRC2012_val_00018978.JPEG:n03950228 +ILSVRC2012_val_00018979.JPEG:n01689811 +ILSVRC2012_val_00018980.JPEG:n09428293 +ILSVRC2012_val_00018981.JPEG:n01877812 +ILSVRC2012_val_00018982.JPEG:n02100583 +ILSVRC2012_val_00018983.JPEG:n01704323 +ILSVRC2012_val_00018984.JPEG:n03680355 +ILSVRC2012_val_00018985.JPEG:n03000247 +ILSVRC2012_val_00018986.JPEG:n03742115 +ILSVRC2012_val_00018987.JPEG:n04486054 +ILSVRC2012_val_00018988.JPEG:n02097298 +ILSVRC2012_val_00018989.JPEG:n02091635 +ILSVRC2012_val_00018990.JPEG:n03680355 +ILSVRC2012_val_00018991.JPEG:n02002556 +ILSVRC2012_val_00018992.JPEG:n02101388 +ILSVRC2012_val_00018993.JPEG:n01818515 +ILSVRC2012_val_00018994.JPEG:n02454379 +ILSVRC2012_val_00018995.JPEG:n03216828 +ILSVRC2012_val_00018996.JPEG:n03933933 +ILSVRC2012_val_00018997.JPEG:n02107683 +ILSVRC2012_val_00018998.JPEG:n04252077 +ILSVRC2012_val_00018999.JPEG:n02980441 +ILSVRC2012_val_00019000.JPEG:n04039381 +ILSVRC2012_val_00019001.JPEG:n03201208 +ILSVRC2012_val_00019002.JPEG:n02102177 +ILSVRC2012_val_00019003.JPEG:n03388549 +ILSVRC2012_val_00019004.JPEG:n04523525 +ILSVRC2012_val_00019005.JPEG:n03770439 +ILSVRC2012_val_00019006.JPEG:n03710193 +ILSVRC2012_val_00019007.JPEG:n01675722 +ILSVRC2012_val_00019008.JPEG:n04501370 +ILSVRC2012_val_00019009.JPEG:n04501370 +ILSVRC2012_val_00019010.JPEG:n02092002 +ILSVRC2012_val_00019011.JPEG:n03598930 +ILSVRC2012_val_00019012.JPEG:n07932039 +ILSVRC2012_val_00019013.JPEG:n02101006 +ILSVRC2012_val_00019014.JPEG:n02268853 +ILSVRC2012_val_00019015.JPEG:n04259630 +ILSVRC2012_val_00019016.JPEG:n03871628 +ILSVRC2012_val_00019017.JPEG:n02786058 +ILSVRC2012_val_00019018.JPEG:n03485794 +ILSVRC2012_val_00019019.JPEG:n02009912 +ILSVRC2012_val_00019020.JPEG:n02091244 +ILSVRC2012_val_00019021.JPEG:n02808304 +ILSVRC2012_val_00019022.JPEG:n01860187 +ILSVRC2012_val_00019023.JPEG:n07613480 +ILSVRC2012_val_00019024.JPEG:n01843065 +ILSVRC2012_val_00019025.JPEG:n02095889 +ILSVRC2012_val_00019026.JPEG:n01943899 +ILSVRC2012_val_00019027.JPEG:n02859443 +ILSVRC2012_val_00019028.JPEG:n02112350 +ILSVRC2012_val_00019029.JPEG:n02165456 +ILSVRC2012_val_00019030.JPEG:n01773797 +ILSVRC2012_val_00019031.JPEG:n02328150 +ILSVRC2012_val_00019032.JPEG:n03485407 +ILSVRC2012_val_00019033.JPEG:n01955084 +ILSVRC2012_val_00019034.JPEG:n01601694 +ILSVRC2012_val_00019035.JPEG:n03290653 +ILSVRC2012_val_00019036.JPEG:n01796340 +ILSVRC2012_val_00019037.JPEG:n06359193 +ILSVRC2012_val_00019038.JPEG:n01558993 +ILSVRC2012_val_00019039.JPEG:n03950228 +ILSVRC2012_val_00019040.JPEG:n02096437 +ILSVRC2012_val_00019041.JPEG:n02093859 +ILSVRC2012_val_00019042.JPEG:n01773549 +ILSVRC2012_val_00019043.JPEG:n04154565 +ILSVRC2012_val_00019044.JPEG:n02437616 +ILSVRC2012_val_00019045.JPEG:n02017213 +ILSVRC2012_val_00019046.JPEG:n04146614 +ILSVRC2012_val_00019047.JPEG:n02488702 +ILSVRC2012_val_00019048.JPEG:n02137549 +ILSVRC2012_val_00019049.JPEG:n02013706 +ILSVRC2012_val_00019050.JPEG:n02100735 +ILSVRC2012_val_00019051.JPEG:n04465501 +ILSVRC2012_val_00019052.JPEG:n02727426 +ILSVRC2012_val_00019053.JPEG:n04467665 +ILSVRC2012_val_00019054.JPEG:n02095889 +ILSVRC2012_val_00019055.JPEG:n02415577 +ILSVRC2012_val_00019056.JPEG:n03075370 +ILSVRC2012_val_00019057.JPEG:n02097298 +ILSVRC2012_val_00019058.JPEG:n02027492 +ILSVRC2012_val_00019059.JPEG:n02441942 +ILSVRC2012_val_00019060.JPEG:n02104029 +ILSVRC2012_val_00019061.JPEG:n03617480 +ILSVRC2012_val_00019062.JPEG:n03623198 +ILSVRC2012_val_00019063.JPEG:n02536864 +ILSVRC2012_val_00019064.JPEG:n07875152 +ILSVRC2012_val_00019065.JPEG:n04208210 +ILSVRC2012_val_00019066.JPEG:n02423022 +ILSVRC2012_val_00019067.JPEG:n03016953 +ILSVRC2012_val_00019068.JPEG:n01669191 +ILSVRC2012_val_00019069.JPEG:n04344873 +ILSVRC2012_val_00019070.JPEG:n02526121 +ILSVRC2012_val_00019071.JPEG:n09472597 +ILSVRC2012_val_00019072.JPEG:n03873416 +ILSVRC2012_val_00019073.JPEG:n01829413 +ILSVRC2012_val_00019074.JPEG:n12057211 +ILSVRC2012_val_00019075.JPEG:n02950826 +ILSVRC2012_val_00019076.JPEG:n02786058 +ILSVRC2012_val_00019077.JPEG:n02486410 +ILSVRC2012_val_00019078.JPEG:n02486261 +ILSVRC2012_val_00019079.JPEG:n02423022 +ILSVRC2012_val_00019080.JPEG:n02107574 +ILSVRC2012_val_00019081.JPEG:n03773504 +ILSVRC2012_val_00019082.JPEG:n01558993 +ILSVRC2012_val_00019083.JPEG:n02096177 +ILSVRC2012_val_00019084.JPEG:n03961711 +ILSVRC2012_val_00019085.JPEG:n01873310 +ILSVRC2012_val_00019086.JPEG:n04118538 +ILSVRC2012_val_00019087.JPEG:n02091032 +ILSVRC2012_val_00019088.JPEG:n03483316 +ILSVRC2012_val_00019089.JPEG:n13040303 +ILSVRC2012_val_00019090.JPEG:n03180011 +ILSVRC2012_val_00019091.JPEG:n02125311 +ILSVRC2012_val_00019092.JPEG:n02172182 +ILSVRC2012_val_00019093.JPEG:n03976657 +ILSVRC2012_val_00019094.JPEG:n02094258 +ILSVRC2012_val_00019095.JPEG:n02980441 +ILSVRC2012_val_00019096.JPEG:n02107312 +ILSVRC2012_val_00019097.JPEG:n01755581 +ILSVRC2012_val_00019098.JPEG:n02776631 +ILSVRC2012_val_00019099.JPEG:n02492660 +ILSVRC2012_val_00019100.JPEG:n01664065 +ILSVRC2012_val_00019101.JPEG:n01514668 +ILSVRC2012_val_00019102.JPEG:n02966193 +ILSVRC2012_val_00019103.JPEG:n02492035 +ILSVRC2012_val_00019104.JPEG:n03482405 +ILSVRC2012_val_00019105.JPEG:n04019541 +ILSVRC2012_val_00019106.JPEG:n03954731 +ILSVRC2012_val_00019107.JPEG:n02106550 +ILSVRC2012_val_00019108.JPEG:n04404412 +ILSVRC2012_val_00019109.JPEG:n02797295 +ILSVRC2012_val_00019110.JPEG:n01955084 +ILSVRC2012_val_00019111.JPEG:n04612504 +ILSVRC2012_val_00019112.JPEG:n04069434 +ILSVRC2012_val_00019113.JPEG:n02492035 +ILSVRC2012_val_00019114.JPEG:n10565667 +ILSVRC2012_val_00019115.JPEG:n02091134 +ILSVRC2012_val_00019116.JPEG:n01631663 +ILSVRC2012_val_00019117.JPEG:n02727426 +ILSVRC2012_val_00019118.JPEG:n02071294 +ILSVRC2012_val_00019119.JPEG:n02124075 +ILSVRC2012_val_00019120.JPEG:n02092002 +ILSVRC2012_val_00019121.JPEG:n02321529 +ILSVRC2012_val_00019122.JPEG:n04208210 +ILSVRC2012_val_00019123.JPEG:n01819313 +ILSVRC2012_val_00019124.JPEG:n02087046 +ILSVRC2012_val_00019125.JPEG:n04409515 +ILSVRC2012_val_00019126.JPEG:n03485794 +ILSVRC2012_val_00019127.JPEG:n04356056 +ILSVRC2012_val_00019128.JPEG:n02087046 +ILSVRC2012_val_00019129.JPEG:n02492035 +ILSVRC2012_val_00019130.JPEG:n02085782 +ILSVRC2012_val_00019131.JPEG:n03788365 +ILSVRC2012_val_00019132.JPEG:n02483708 +ILSVRC2012_val_00019133.JPEG:n04532106 +ILSVRC2012_val_00019134.JPEG:n02106030 +ILSVRC2012_val_00019135.JPEG:n03742115 +ILSVRC2012_val_00019136.JPEG:n03868242 +ILSVRC2012_val_00019137.JPEG:n03000684 +ILSVRC2012_val_00019138.JPEG:n02100236 +ILSVRC2012_val_00019139.JPEG:n02398521 +ILSVRC2012_val_00019140.JPEG:n03976657 +ILSVRC2012_val_00019141.JPEG:n03595614 +ILSVRC2012_val_00019142.JPEG:n03884397 +ILSVRC2012_val_00019143.JPEG:n03109150 +ILSVRC2012_val_00019144.JPEG:n02978881 +ILSVRC2012_val_00019145.JPEG:n02279972 +ILSVRC2012_val_00019146.JPEG:n02391049 +ILSVRC2012_val_00019147.JPEG:n03417042 +ILSVRC2012_val_00019148.JPEG:n01734418 +ILSVRC2012_val_00019149.JPEG:n07565083 +ILSVRC2012_val_00019150.JPEG:n03970156 +ILSVRC2012_val_00019151.JPEG:n02256656 +ILSVRC2012_val_00019152.JPEG:n01689811 +ILSVRC2012_val_00019153.JPEG:n02107683 +ILSVRC2012_val_00019154.JPEG:n04591713 +ILSVRC2012_val_00019155.JPEG:n02105855 +ILSVRC2012_val_00019156.JPEG:n04099969 +ILSVRC2012_val_00019157.JPEG:n02980441 +ILSVRC2012_val_00019158.JPEG:n07720875 +ILSVRC2012_val_00019159.JPEG:n04259630 +ILSVRC2012_val_00019160.JPEG:n07920052 +ILSVRC2012_val_00019161.JPEG:n03777754 +ILSVRC2012_val_00019162.JPEG:n02099429 +ILSVRC2012_val_00019163.JPEG:n03777568 +ILSVRC2012_val_00019164.JPEG:n03759954 +ILSVRC2012_val_00019165.JPEG:n02109525 +ILSVRC2012_val_00019166.JPEG:n04264628 +ILSVRC2012_val_00019167.JPEG:n03584829 +ILSVRC2012_val_00019168.JPEG:n04525305 +ILSVRC2012_val_00019169.JPEG:n02099712 +ILSVRC2012_val_00019170.JPEG:n01689811 +ILSVRC2012_val_00019171.JPEG:n02169497 +ILSVRC2012_val_00019172.JPEG:n02011460 +ILSVRC2012_val_00019173.JPEG:n02109961 +ILSVRC2012_val_00019174.JPEG:n03814906 +ILSVRC2012_val_00019175.JPEG:n02095314 +ILSVRC2012_val_00019176.JPEG:n03866082 +ILSVRC2012_val_00019177.JPEG:n02966687 +ILSVRC2012_val_00019178.JPEG:n03710721 +ILSVRC2012_val_00019179.JPEG:n02690373 +ILSVRC2012_val_00019180.JPEG:n02514041 +ILSVRC2012_val_00019181.JPEG:n03062245 +ILSVRC2012_val_00019182.JPEG:n02797295 +ILSVRC2012_val_00019183.JPEG:n02167151 +ILSVRC2012_val_00019184.JPEG:n01518878 +ILSVRC2012_val_00019185.JPEG:n13040303 +ILSVRC2012_val_00019186.JPEG:n13044778 +ILSVRC2012_val_00019187.JPEG:n02088364 +ILSVRC2012_val_00019188.JPEG:n03045698 +ILSVRC2012_val_00019189.JPEG:n03857828 +ILSVRC2012_val_00019190.JPEG:n09288635 +ILSVRC2012_val_00019191.JPEG:n03873416 +ILSVRC2012_val_00019192.JPEG:n10148035 +ILSVRC2012_val_00019193.JPEG:n02837789 +ILSVRC2012_val_00019194.JPEG:n03388183 +ILSVRC2012_val_00019195.JPEG:n03272010 +ILSVRC2012_val_00019196.JPEG:n13054560 +ILSVRC2012_val_00019197.JPEG:n02699494 +ILSVRC2012_val_00019198.JPEG:n02051845 +ILSVRC2012_val_00019199.JPEG:n02966193 +ILSVRC2012_val_00019200.JPEG:n02437312 +ILSVRC2012_val_00019201.JPEG:n04557648 +ILSVRC2012_val_00019202.JPEG:n02177972 +ILSVRC2012_val_00019203.JPEG:n03792782 +ILSVRC2012_val_00019204.JPEG:n01751748 +ILSVRC2012_val_00019205.JPEG:n02892767 +ILSVRC2012_val_00019206.JPEG:n04344873 +ILSVRC2012_val_00019207.JPEG:n03902125 +ILSVRC2012_val_00019208.JPEG:n01558993 +ILSVRC2012_val_00019209.JPEG:n02087394 +ILSVRC2012_val_00019210.JPEG:n02006656 +ILSVRC2012_val_00019211.JPEG:n01784675 +ILSVRC2012_val_00019212.JPEG:n02099601 +ILSVRC2012_val_00019213.JPEG:n03930313 +ILSVRC2012_val_00019214.JPEG:n02980441 +ILSVRC2012_val_00019215.JPEG:n02097209 +ILSVRC2012_val_00019216.JPEG:n02091032 +ILSVRC2012_val_00019217.JPEG:n03742115 +ILSVRC2012_val_00019218.JPEG:n02606052 +ILSVRC2012_val_00019219.JPEG:n02104365 +ILSVRC2012_val_00019220.JPEG:n02097130 +ILSVRC2012_val_00019221.JPEG:n07860988 +ILSVRC2012_val_00019222.JPEG:n02120079 +ILSVRC2012_val_00019223.JPEG:n04235860 +ILSVRC2012_val_00019224.JPEG:n02883205 +ILSVRC2012_val_00019225.JPEG:n02727426 +ILSVRC2012_val_00019226.JPEG:n02099267 +ILSVRC2012_val_00019227.JPEG:n03884397 +ILSVRC2012_val_00019228.JPEG:n02992211 +ILSVRC2012_val_00019229.JPEG:n03095699 +ILSVRC2012_val_00019230.JPEG:n04254777 +ILSVRC2012_val_00019231.JPEG:n02093859 +ILSVRC2012_val_00019232.JPEG:n03146219 +ILSVRC2012_val_00019233.JPEG:n04548362 +ILSVRC2012_val_00019234.JPEG:n04335435 +ILSVRC2012_val_00019235.JPEG:n02489166 +ILSVRC2012_val_00019236.JPEG:n01531178 +ILSVRC2012_val_00019237.JPEG:n02259212 +ILSVRC2012_val_00019238.JPEG:n02894605 +ILSVRC2012_val_00019239.JPEG:n02114855 +ILSVRC2012_val_00019240.JPEG:n03188531 +ILSVRC2012_val_00019241.JPEG:n02088466 +ILSVRC2012_val_00019242.JPEG:n03956157 +ILSVRC2012_val_00019243.JPEG:n04589890 +ILSVRC2012_val_00019244.JPEG:n04525038 +ILSVRC2012_val_00019245.JPEG:n02233338 +ILSVRC2012_val_00019246.JPEG:n04612504 +ILSVRC2012_val_00019247.JPEG:n07711569 +ILSVRC2012_val_00019248.JPEG:n02437312 +ILSVRC2012_val_00019249.JPEG:n03976657 +ILSVRC2012_val_00019250.JPEG:n12144580 +ILSVRC2012_val_00019251.JPEG:n01843065 +ILSVRC2012_val_00019252.JPEG:n02120505 +ILSVRC2012_val_00019253.JPEG:n07745940 +ILSVRC2012_val_00019254.JPEG:n04552348 +ILSVRC2012_val_00019255.JPEG:n03710721 +ILSVRC2012_val_00019256.JPEG:n03425413 +ILSVRC2012_val_00019257.JPEG:n01697457 +ILSVRC2012_val_00019258.JPEG:n02396427 +ILSVRC2012_val_00019259.JPEG:n02092339 +ILSVRC2012_val_00019260.JPEG:n02493509 +ILSVRC2012_val_00019261.JPEG:n02087046 +ILSVRC2012_val_00019262.JPEG:n02123159 +ILSVRC2012_val_00019263.JPEG:n04251144 +ILSVRC2012_val_00019264.JPEG:n04259630 +ILSVRC2012_val_00019265.JPEG:n02096051 +ILSVRC2012_val_00019266.JPEG:n04507155 +ILSVRC2012_val_00019267.JPEG:n02106662 +ILSVRC2012_val_00019268.JPEG:n03445777 +ILSVRC2012_val_00019269.JPEG:n03494278 +ILSVRC2012_val_00019270.JPEG:n01756291 +ILSVRC2012_val_00019271.JPEG:n03063689 +ILSVRC2012_val_00019272.JPEG:n02105162 +ILSVRC2012_val_00019273.JPEG:n04346328 +ILSVRC2012_val_00019274.JPEG:n04591713 +ILSVRC2012_val_00019275.JPEG:n03662601 +ILSVRC2012_val_00019276.JPEG:n02093428 +ILSVRC2012_val_00019277.JPEG:n02917067 +ILSVRC2012_val_00019278.JPEG:n03710721 +ILSVRC2012_val_00019279.JPEG:n02493509 +ILSVRC2012_val_00019280.JPEG:n02794156 +ILSVRC2012_val_00019281.JPEG:n07720875 +ILSVRC2012_val_00019282.JPEG:n01669191 +ILSVRC2012_val_00019283.JPEG:n02088364 +ILSVRC2012_val_00019284.JPEG:n01873310 +ILSVRC2012_val_00019285.JPEG:n04037443 +ILSVRC2012_val_00019286.JPEG:n03598930 +ILSVRC2012_val_00019287.JPEG:n07714571 +ILSVRC2012_val_00019288.JPEG:n04069434 +ILSVRC2012_val_00019289.JPEG:n03888257 +ILSVRC2012_val_00019290.JPEG:n07718472 +ILSVRC2012_val_00019291.JPEG:n03676483 +ILSVRC2012_val_00019292.JPEG:n03929660 +ILSVRC2012_val_00019293.JPEG:n02514041 +ILSVRC2012_val_00019294.JPEG:n02105056 +ILSVRC2012_val_00019295.JPEG:n04275548 +ILSVRC2012_val_00019296.JPEG:n03534580 +ILSVRC2012_val_00019297.JPEG:n04296562 +ILSVRC2012_val_00019298.JPEG:n03770439 +ILSVRC2012_val_00019299.JPEG:n02165456 +ILSVRC2012_val_00019300.JPEG:n02704792 +ILSVRC2012_val_00019301.JPEG:n03995372 +ILSVRC2012_val_00019302.JPEG:n04344873 +ILSVRC2012_val_00019303.JPEG:n02123159 +ILSVRC2012_val_00019304.JPEG:n11879895 +ILSVRC2012_val_00019305.JPEG:n02094114 +ILSVRC2012_val_00019306.JPEG:n02514041 +ILSVRC2012_val_00019307.JPEG:n03388549 +ILSVRC2012_val_00019308.JPEG:n01629819 +ILSVRC2012_val_00019309.JPEG:n02776631 +ILSVRC2012_val_00019310.JPEG:n02963159 +ILSVRC2012_val_00019311.JPEG:n03857828 +ILSVRC2012_val_00019312.JPEG:n07768694 +ILSVRC2012_val_00019313.JPEG:n01847000 +ILSVRC2012_val_00019314.JPEG:n02229544 +ILSVRC2012_val_00019315.JPEG:n02834397 +ILSVRC2012_val_00019316.JPEG:n04380533 +ILSVRC2012_val_00019317.JPEG:n07717410 +ILSVRC2012_val_00019318.JPEG:n02112706 +ILSVRC2012_val_00019319.JPEG:n03014705 +ILSVRC2012_val_00019320.JPEG:n11939491 +ILSVRC2012_val_00019321.JPEG:n02769748 +ILSVRC2012_val_00019322.JPEG:n03075370 +ILSVRC2012_val_00019323.JPEG:n03534580 +ILSVRC2012_val_00019324.JPEG:n02116738 +ILSVRC2012_val_00019325.JPEG:n02111277 +ILSVRC2012_val_00019326.JPEG:n03482405 +ILSVRC2012_val_00019327.JPEG:n02096294 +ILSVRC2012_val_00019328.JPEG:n01819313 +ILSVRC2012_val_00019329.JPEG:n02105056 +ILSVRC2012_val_00019330.JPEG:n04540053 +ILSVRC2012_val_00019331.JPEG:n03028079 +ILSVRC2012_val_00019332.JPEG:n03467068 +ILSVRC2012_val_00019333.JPEG:n02107683 +ILSVRC2012_val_00019334.JPEG:n12768682 +ILSVRC2012_val_00019335.JPEG:n02481823 +ILSVRC2012_val_00019336.JPEG:n02447366 +ILSVRC2012_val_00019337.JPEG:n03255030 +ILSVRC2012_val_00019338.JPEG:n02977058 +ILSVRC2012_val_00019339.JPEG:n12620546 +ILSVRC2012_val_00019340.JPEG:n03131574 +ILSVRC2012_val_00019341.JPEG:n02981792 +ILSVRC2012_val_00019342.JPEG:n02110063 +ILSVRC2012_val_00019343.JPEG:n03494278 +ILSVRC2012_val_00019344.JPEG:n02415577 +ILSVRC2012_val_00019345.JPEG:n02398521 +ILSVRC2012_val_00019346.JPEG:n04554684 +ILSVRC2012_val_00019347.JPEG:n03063599 +ILSVRC2012_val_00019348.JPEG:n04579145 +ILSVRC2012_val_00019349.JPEG:n04335435 +ILSVRC2012_val_00019350.JPEG:n04264628 +ILSVRC2012_val_00019351.JPEG:n04311004 +ILSVRC2012_val_00019352.JPEG:n02457408 +ILSVRC2012_val_00019353.JPEG:n02106550 +ILSVRC2012_val_00019354.JPEG:n04483307 +ILSVRC2012_val_00019355.JPEG:n02977058 +ILSVRC2012_val_00019356.JPEG:n02091244 +ILSVRC2012_val_00019357.JPEG:n02169497 +ILSVRC2012_val_00019358.JPEG:n03041632 +ILSVRC2012_val_00019359.JPEG:n03630383 +ILSVRC2012_val_00019360.JPEG:n02669723 +ILSVRC2012_val_00019361.JPEG:n02104029 +ILSVRC2012_val_00019362.JPEG:n02364673 +ILSVRC2012_val_00019363.JPEG:n02749479 +ILSVRC2012_val_00019364.JPEG:n02107312 +ILSVRC2012_val_00019365.JPEG:n02128925 +ILSVRC2012_val_00019366.JPEG:n02091831 +ILSVRC2012_val_00019367.JPEG:n04554684 +ILSVRC2012_val_00019368.JPEG:n01978287 +ILSVRC2012_val_00019369.JPEG:n02655020 +ILSVRC2012_val_00019370.JPEG:n02125311 +ILSVRC2012_val_00019371.JPEG:n04136333 +ILSVRC2012_val_00019372.JPEG:n07753113 +ILSVRC2012_val_00019373.JPEG:n01943899 +ILSVRC2012_val_00019374.JPEG:n04204347 +ILSVRC2012_val_00019375.JPEG:n03372029 +ILSVRC2012_val_00019376.JPEG:n04418357 +ILSVRC2012_val_00019377.JPEG:n02980441 +ILSVRC2012_val_00019378.JPEG:n02859443 +ILSVRC2012_val_00019379.JPEG:n04235860 +ILSVRC2012_val_00019380.JPEG:n09472597 +ILSVRC2012_val_00019381.JPEG:n02328150 +ILSVRC2012_val_00019382.JPEG:n02017213 +ILSVRC2012_val_00019383.JPEG:n01734418 +ILSVRC2012_val_00019384.JPEG:n03930313 +ILSVRC2012_val_00019385.JPEG:n03868242 +ILSVRC2012_val_00019386.JPEG:n04355338 +ILSVRC2012_val_00019387.JPEG:n04118538 +ILSVRC2012_val_00019388.JPEG:n02804610 +ILSVRC2012_val_00019389.JPEG:n02028035 +ILSVRC2012_val_00019390.JPEG:n02835271 +ILSVRC2012_val_00019391.JPEG:n02114548 +ILSVRC2012_val_00019392.JPEG:n03710193 +ILSVRC2012_val_00019393.JPEG:n04033901 +ILSVRC2012_val_00019394.JPEG:n01984695 +ILSVRC2012_val_00019395.JPEG:n03443371 +ILSVRC2012_val_00019396.JPEG:n03956157 +ILSVRC2012_val_00019397.JPEG:n07753113 +ILSVRC2012_val_00019398.JPEG:n03532672 +ILSVRC2012_val_00019399.JPEG:n01664065 +ILSVRC2012_val_00019400.JPEG:n02786058 +ILSVRC2012_val_00019401.JPEG:n02125311 +ILSVRC2012_val_00019402.JPEG:n02085620 +ILSVRC2012_val_00019403.JPEG:n02655020 +ILSVRC2012_val_00019404.JPEG:n04235860 +ILSVRC2012_val_00019405.JPEG:n03018349 +ILSVRC2012_val_00019406.JPEG:n13040303 +ILSVRC2012_val_00019407.JPEG:n03658185 +ILSVRC2012_val_00019408.JPEG:n04254680 +ILSVRC2012_val_00019409.JPEG:n01484850 +ILSVRC2012_val_00019410.JPEG:n03594945 +ILSVRC2012_val_00019411.JPEG:n04209133 +ILSVRC2012_val_00019412.JPEG:n03877845 +ILSVRC2012_val_00019413.JPEG:n12985857 +ILSVRC2012_val_00019414.JPEG:n02102040 +ILSVRC2012_val_00019415.JPEG:n02112018 +ILSVRC2012_val_00019416.JPEG:n03467068 +ILSVRC2012_val_00019417.JPEG:n02115641 +ILSVRC2012_val_00019418.JPEG:n04562935 +ILSVRC2012_val_00019419.JPEG:n03042490 +ILSVRC2012_val_00019420.JPEG:n04429376 +ILSVRC2012_val_00019421.JPEG:n02895154 +ILSVRC2012_val_00019422.JPEG:n13052670 +ILSVRC2012_val_00019423.JPEG:n01514668 +ILSVRC2012_val_00019424.JPEG:n01491361 +ILSVRC2012_val_00019425.JPEG:n01924916 +ILSVRC2012_val_00019426.JPEG:n04039381 +ILSVRC2012_val_00019427.JPEG:n02437616 +ILSVRC2012_val_00019428.JPEG:n04065272 +ILSVRC2012_val_00019429.JPEG:n01855672 +ILSVRC2012_val_00019430.JPEG:n03733281 +ILSVRC2012_val_00019431.JPEG:n03935335 +ILSVRC2012_val_00019432.JPEG:n02492035 +ILSVRC2012_val_00019433.JPEG:n02130308 +ILSVRC2012_val_00019434.JPEG:n04131690 +ILSVRC2012_val_00019435.JPEG:n01484850 +ILSVRC2012_val_00019436.JPEG:n03197337 +ILSVRC2012_val_00019437.JPEG:n03761084 +ILSVRC2012_val_00019438.JPEG:n03899768 +ILSVRC2012_val_00019439.JPEG:n02128385 +ILSVRC2012_val_00019440.JPEG:n04604644 +ILSVRC2012_val_00019441.JPEG:n03623198 +ILSVRC2012_val_00019442.JPEG:n04152593 +ILSVRC2012_val_00019443.JPEG:n02783161 +ILSVRC2012_val_00019444.JPEG:n04252225 +ILSVRC2012_val_00019445.JPEG:n04118538 +ILSVRC2012_val_00019446.JPEG:n02412080 +ILSVRC2012_val_00019447.JPEG:n03717622 +ILSVRC2012_val_00019448.JPEG:n02480495 +ILSVRC2012_val_00019449.JPEG:n02102480 +ILSVRC2012_val_00019450.JPEG:n02676566 +ILSVRC2012_val_00019451.JPEG:n02492035 +ILSVRC2012_val_00019452.JPEG:n04265275 +ILSVRC2012_val_00019453.JPEG:n07742313 +ILSVRC2012_val_00019454.JPEG:n03483316 +ILSVRC2012_val_00019455.JPEG:n03706229 +ILSVRC2012_val_00019456.JPEG:n02129165 +ILSVRC2012_val_00019457.JPEG:n07718747 +ILSVRC2012_val_00019458.JPEG:n03967562 +ILSVRC2012_val_00019459.JPEG:n01443537 +ILSVRC2012_val_00019460.JPEG:n02190166 +ILSVRC2012_val_00019461.JPEG:n01943899 +ILSVRC2012_val_00019462.JPEG:n02089078 +ILSVRC2012_val_00019463.JPEG:n03627232 +ILSVRC2012_val_00019464.JPEG:n02110958 +ILSVRC2012_val_00019465.JPEG:n03902125 +ILSVRC2012_val_00019466.JPEG:n04081281 +ILSVRC2012_val_00019467.JPEG:n02172182 +ILSVRC2012_val_00019468.JPEG:n02099849 +ILSVRC2012_val_00019469.JPEG:n02492035 +ILSVRC2012_val_00019470.JPEG:n02999410 +ILSVRC2012_val_00019471.JPEG:n04435653 +ILSVRC2012_val_00019472.JPEG:n03127925 +ILSVRC2012_val_00019473.JPEG:n07880968 +ILSVRC2012_val_00019474.JPEG:n04243546 +ILSVRC2012_val_00019475.JPEG:n03544143 +ILSVRC2012_val_00019476.JPEG:n01877812 +ILSVRC2012_val_00019477.JPEG:n02823750 +ILSVRC2012_val_00019478.JPEG:n02814533 +ILSVRC2012_val_00019479.JPEG:n02916936 +ILSVRC2012_val_00019480.JPEG:n02120505 +ILSVRC2012_val_00019481.JPEG:n02088632 +ILSVRC2012_val_00019482.JPEG:n02977058 +ILSVRC2012_val_00019483.JPEG:n07734744 +ILSVRC2012_val_00019484.JPEG:n02676566 +ILSVRC2012_val_00019485.JPEG:n01770081 +ILSVRC2012_val_00019486.JPEG:n04116512 +ILSVRC2012_val_00019487.JPEG:n02871525 +ILSVRC2012_val_00019488.JPEG:n02091032 +ILSVRC2012_val_00019489.JPEG:n02536864 +ILSVRC2012_val_00019490.JPEG:n03223299 +ILSVRC2012_val_00019491.JPEG:n02963159 +ILSVRC2012_val_00019492.JPEG:n03180011 +ILSVRC2012_val_00019493.JPEG:n03207743 +ILSVRC2012_val_00019494.JPEG:n03496892 +ILSVRC2012_val_00019495.JPEG:n03444034 +ILSVRC2012_val_00019496.JPEG:n03100240 +ILSVRC2012_val_00019497.JPEG:n04592741 +ILSVRC2012_val_00019498.JPEG:n02091831 +ILSVRC2012_val_00019499.JPEG:n04613696 +ILSVRC2012_val_00019500.JPEG:n02097130 +ILSVRC2012_val_00019501.JPEG:n03196217 +ILSVRC2012_val_00019502.JPEG:n04523525 +ILSVRC2012_val_00019503.JPEG:n04505470 +ILSVRC2012_val_00019504.JPEG:n04153751 +ILSVRC2012_val_00019505.JPEG:n03786901 +ILSVRC2012_val_00019506.JPEG:n03220513 +ILSVRC2012_val_00019507.JPEG:n02808440 +ILSVRC2012_val_00019508.JPEG:n04399382 +ILSVRC2012_val_00019509.JPEG:n03594945 +ILSVRC2012_val_00019510.JPEG:n01978455 +ILSVRC2012_val_00019511.JPEG:n01824575 +ILSVRC2012_val_00019512.JPEG:n01986214 +ILSVRC2012_val_00019513.JPEG:n03792782 +ILSVRC2012_val_00019514.JPEG:n02730930 +ILSVRC2012_val_00019515.JPEG:n03208938 +ILSVRC2012_val_00019516.JPEG:n02641379 +ILSVRC2012_val_00019517.JPEG:n02106030 +ILSVRC2012_val_00019518.JPEG:n02106550 +ILSVRC2012_val_00019519.JPEG:n02110063 +ILSVRC2012_val_00019520.JPEG:n03786901 +ILSVRC2012_val_00019521.JPEG:n04532670 +ILSVRC2012_val_00019522.JPEG:n03595614 +ILSVRC2012_val_00019523.JPEG:n13054560 +ILSVRC2012_val_00019524.JPEG:n02233338 +ILSVRC2012_val_00019525.JPEG:n03803284 +ILSVRC2012_val_00019526.JPEG:n03355925 +ILSVRC2012_val_00019527.JPEG:n02236044 +ILSVRC2012_val_00019528.JPEG:n02951585 +ILSVRC2012_val_00019529.JPEG:n03063599 +ILSVRC2012_val_00019530.JPEG:n03047690 +ILSVRC2012_val_00019531.JPEG:n01496331 +ILSVRC2012_val_00019532.JPEG:n02708093 +ILSVRC2012_val_00019533.JPEG:n02356798 +ILSVRC2012_val_00019534.JPEG:n04442312 +ILSVRC2012_val_00019535.JPEG:n02107574 +ILSVRC2012_val_00019536.JPEG:n03459775 +ILSVRC2012_val_00019537.JPEG:n04026417 +ILSVRC2012_val_00019538.JPEG:n02860847 +ILSVRC2012_val_00019539.JPEG:n02655020 +ILSVRC2012_val_00019540.JPEG:n03983396 +ILSVRC2012_val_00019541.JPEG:n03658185 +ILSVRC2012_val_00019542.JPEG:n04589890 +ILSVRC2012_val_00019543.JPEG:n03956157 +ILSVRC2012_val_00019544.JPEG:n02093991 +ILSVRC2012_val_00019545.JPEG:n02091032 +ILSVRC2012_val_00019546.JPEG:n02977058 +ILSVRC2012_val_00019547.JPEG:n01667114 +ILSVRC2012_val_00019548.JPEG:n02500267 +ILSVRC2012_val_00019549.JPEG:n03347037 +ILSVRC2012_val_00019550.JPEG:n07716906 +ILSVRC2012_val_00019551.JPEG:n03598930 +ILSVRC2012_val_00019552.JPEG:n02841315 +ILSVRC2012_val_00019553.JPEG:n04254777 +ILSVRC2012_val_00019554.JPEG:n04049303 +ILSVRC2012_val_00019555.JPEG:n13040303 +ILSVRC2012_val_00019556.JPEG:n03495258 +ILSVRC2012_val_00019557.JPEG:n04596742 +ILSVRC2012_val_00019558.JPEG:n15075141 +ILSVRC2012_val_00019559.JPEG:n02105251 +ILSVRC2012_val_00019560.JPEG:n01667114 +ILSVRC2012_val_00019561.JPEG:n01775062 +ILSVRC2012_val_00019562.JPEG:n02002724 +ILSVRC2012_val_00019563.JPEG:n04536866 +ILSVRC2012_val_00019564.JPEG:n01768244 +ILSVRC2012_val_00019565.JPEG:n02808440 +ILSVRC2012_val_00019566.JPEG:n02087046 +ILSVRC2012_val_00019567.JPEG:n02917067 +ILSVRC2012_val_00019568.JPEG:n04111531 +ILSVRC2012_val_00019569.JPEG:n02190166 +ILSVRC2012_val_00019570.JPEG:n03690938 +ILSVRC2012_val_00019571.JPEG:n13040303 +ILSVRC2012_val_00019572.JPEG:n04133789 +ILSVRC2012_val_00019573.JPEG:n03877845 +ILSVRC2012_val_00019574.JPEG:n01985128 +ILSVRC2012_val_00019575.JPEG:n03220513 +ILSVRC2012_val_00019576.JPEG:n03970156 +ILSVRC2012_val_00019577.JPEG:n04483307 +ILSVRC2012_val_00019578.JPEG:n01641577 +ILSVRC2012_val_00019579.JPEG:n03384352 +ILSVRC2012_val_00019580.JPEG:n02823750 +ILSVRC2012_val_00019581.JPEG:n02088238 +ILSVRC2012_val_00019582.JPEG:n04346328 +ILSVRC2012_val_00019583.JPEG:n04423845 +ILSVRC2012_val_00019584.JPEG:n04356056 +ILSVRC2012_val_00019585.JPEG:n04509417 +ILSVRC2012_val_00019586.JPEG:n02606052 +ILSVRC2012_val_00019587.JPEG:n01704323 +ILSVRC2012_val_00019588.JPEG:n07831146 +ILSVRC2012_val_00019589.JPEG:n02120505 +ILSVRC2012_val_00019590.JPEG:n02099601 +ILSVRC2012_val_00019591.JPEG:n02799071 +ILSVRC2012_val_00019592.JPEG:n02233338 +ILSVRC2012_val_00019593.JPEG:n03394916 +ILSVRC2012_val_00019594.JPEG:n02865351 +ILSVRC2012_val_00019595.JPEG:n03272562 +ILSVRC2012_val_00019596.JPEG:n03843555 +ILSVRC2012_val_00019597.JPEG:n09246464 +ILSVRC2012_val_00019598.JPEG:n02825657 +ILSVRC2012_val_00019599.JPEG:n02951585 +ILSVRC2012_val_00019600.JPEG:n03692522 +ILSVRC2012_val_00019601.JPEG:n04517823 +ILSVRC2012_val_00019602.JPEG:n03803284 +ILSVRC2012_val_00019603.JPEG:n02086910 +ILSVRC2012_val_00019604.JPEG:n07613480 +ILSVRC2012_val_00019605.JPEG:n09399592 +ILSVRC2012_val_00019606.JPEG:n03775071 +ILSVRC2012_val_00019607.JPEG:n02099429 +ILSVRC2012_val_00019608.JPEG:n07695742 +ILSVRC2012_val_00019609.JPEG:n03527444 +ILSVRC2012_val_00019610.JPEG:n04330267 +ILSVRC2012_val_00019611.JPEG:n03832673 +ILSVRC2012_val_00019612.JPEG:n02894605 +ILSVRC2012_val_00019613.JPEG:n02951585 +ILSVRC2012_val_00019614.JPEG:n09332890 +ILSVRC2012_val_00019615.JPEG:n13054560 +ILSVRC2012_val_00019616.JPEG:n03623198 +ILSVRC2012_val_00019617.JPEG:n02363005 +ILSVRC2012_val_00019618.JPEG:n04275548 +ILSVRC2012_val_00019619.JPEG:n09288635 +ILSVRC2012_val_00019620.JPEG:n03902125 +ILSVRC2012_val_00019621.JPEG:n04435653 +ILSVRC2012_val_00019622.JPEG:n04398044 +ILSVRC2012_val_00019623.JPEG:n02666196 +ILSVRC2012_val_00019624.JPEG:n04147183 +ILSVRC2012_val_00019625.JPEG:n02454379 +ILSVRC2012_val_00019626.JPEG:n02107574 +ILSVRC2012_val_00019627.JPEG:n04592741 +ILSVRC2012_val_00019628.JPEG:n04200800 +ILSVRC2012_val_00019629.JPEG:n02066245 +ILSVRC2012_val_00019630.JPEG:n01629819 +ILSVRC2012_val_00019631.JPEG:n03272562 +ILSVRC2012_val_00019632.JPEG:n03877472 +ILSVRC2012_val_00019633.JPEG:n02009229 +ILSVRC2012_val_00019634.JPEG:n03532672 +ILSVRC2012_val_00019635.JPEG:n02437312 +ILSVRC2012_val_00019636.JPEG:n02089078 +ILSVRC2012_val_00019637.JPEG:n04127249 +ILSVRC2012_val_00019638.JPEG:n03443371 +ILSVRC2012_val_00019639.JPEG:n02091635 +ILSVRC2012_val_00019640.JPEG:n02667093 +ILSVRC2012_val_00019641.JPEG:n03935335 +ILSVRC2012_val_00019642.JPEG:n02364673 +ILSVRC2012_val_00019643.JPEG:n02165105 +ILSVRC2012_val_00019644.JPEG:n03770439 +ILSVRC2012_val_00019645.JPEG:n03063599 +ILSVRC2012_val_00019646.JPEG:n02363005 +ILSVRC2012_val_00019647.JPEG:n03100240 +ILSVRC2012_val_00019648.JPEG:n02815834 +ILSVRC2012_val_00019649.JPEG:n04275548 +ILSVRC2012_val_00019650.JPEG:n02791270 +ILSVRC2012_val_00019651.JPEG:n02325366 +ILSVRC2012_val_00019652.JPEG:n01695060 +ILSVRC2012_val_00019653.JPEG:n02787622 +ILSVRC2012_val_00019654.JPEG:n07753113 +ILSVRC2012_val_00019655.JPEG:n02128385 +ILSVRC2012_val_00019656.JPEG:n04125021 +ILSVRC2012_val_00019657.JPEG:n02395406 +ILSVRC2012_val_00019658.JPEG:n04371430 +ILSVRC2012_val_00019659.JPEG:n03388043 +ILSVRC2012_val_00019660.JPEG:n12620546 +ILSVRC2012_val_00019661.JPEG:n04597913 +ILSVRC2012_val_00019662.JPEG:n03967562 +ILSVRC2012_val_00019663.JPEG:n02708093 +ILSVRC2012_val_00019664.JPEG:n02280649 +ILSVRC2012_val_00019665.JPEG:n02113978 +ILSVRC2012_val_00019666.JPEG:n09288635 +ILSVRC2012_val_00019667.JPEG:n03425413 +ILSVRC2012_val_00019668.JPEG:n03207941 +ILSVRC2012_val_00019669.JPEG:n01740131 +ILSVRC2012_val_00019670.JPEG:n04120489 +ILSVRC2012_val_00019671.JPEG:n02106382 +ILSVRC2012_val_00019672.JPEG:n02536864 +ILSVRC2012_val_00019673.JPEG:n04458633 +ILSVRC2012_val_00019674.JPEG:n03633091 +ILSVRC2012_val_00019675.JPEG:n03967562 +ILSVRC2012_val_00019676.JPEG:n04371430 +ILSVRC2012_val_00019677.JPEG:n02690373 +ILSVRC2012_val_00019678.JPEG:n02113186 +ILSVRC2012_val_00019679.JPEG:n02870880 +ILSVRC2012_val_00019680.JPEG:n02114855 +ILSVRC2012_val_00019681.JPEG:n02396427 +ILSVRC2012_val_00019682.JPEG:n02132136 +ILSVRC2012_val_00019683.JPEG:n02107908 +ILSVRC2012_val_00019684.JPEG:n01950731 +ILSVRC2012_val_00019685.JPEG:n02992529 +ILSVRC2012_val_00019686.JPEG:n03814639 +ILSVRC2012_val_00019687.JPEG:n03594734 +ILSVRC2012_val_00019688.JPEG:n07613480 +ILSVRC2012_val_00019689.JPEG:n07932039 +ILSVRC2012_val_00019690.JPEG:n03721384 +ILSVRC2012_val_00019691.JPEG:n02641379 +ILSVRC2012_val_00019692.JPEG:n03721384 +ILSVRC2012_val_00019693.JPEG:n03661043 +ILSVRC2012_val_00019694.JPEG:n04509417 +ILSVRC2012_val_00019695.JPEG:n02814533 +ILSVRC2012_val_00019696.JPEG:n02437616 +ILSVRC2012_val_00019697.JPEG:n04192698 +ILSVRC2012_val_00019698.JPEG:n02002724 +ILSVRC2012_val_00019699.JPEG:n15075141 +ILSVRC2012_val_00019700.JPEG:n03670208 +ILSVRC2012_val_00019701.JPEG:n02974003 +ILSVRC2012_val_00019702.JPEG:n02094433 +ILSVRC2012_val_00019703.JPEG:n03617480 +ILSVRC2012_val_00019704.JPEG:n04486054 +ILSVRC2012_val_00019705.JPEG:n03290653 +ILSVRC2012_val_00019706.JPEG:n03255030 +ILSVRC2012_val_00019707.JPEG:n04435653 +ILSVRC2012_val_00019708.JPEG:n02916936 +ILSVRC2012_val_00019709.JPEG:n01728572 +ILSVRC2012_val_00019710.JPEG:n01632777 +ILSVRC2012_val_00019711.JPEG:n03028079 +ILSVRC2012_val_00019712.JPEG:n02106382 +ILSVRC2012_val_00019713.JPEG:n12267677 +ILSVRC2012_val_00019714.JPEG:n02279972 +ILSVRC2012_val_00019715.JPEG:n02111129 +ILSVRC2012_val_00019716.JPEG:n01820546 +ILSVRC2012_val_00019717.JPEG:n03680355 +ILSVRC2012_val_00019718.JPEG:n03991062 +ILSVRC2012_val_00019719.JPEG:n02090721 +ILSVRC2012_val_00019720.JPEG:n02879718 +ILSVRC2012_val_00019721.JPEG:n01514668 +ILSVRC2012_val_00019722.JPEG:n01728572 +ILSVRC2012_val_00019723.JPEG:n04442312 +ILSVRC2012_val_00019724.JPEG:n03379051 +ILSVRC2012_val_00019725.JPEG:n02930766 +ILSVRC2012_val_00019726.JPEG:n03982430 +ILSVRC2012_val_00019727.JPEG:n02497673 +ILSVRC2012_val_00019728.JPEG:n02115641 +ILSVRC2012_val_00019729.JPEG:n02389026 +ILSVRC2012_val_00019730.JPEG:n02793495 +ILSVRC2012_val_00019731.JPEG:n03594945 +ILSVRC2012_val_00019732.JPEG:n03661043 +ILSVRC2012_val_00019733.JPEG:n04398044 +ILSVRC2012_val_00019734.JPEG:n01773797 +ILSVRC2012_val_00019735.JPEG:n03630383 +ILSVRC2012_val_00019736.JPEG:n07892512 +ILSVRC2012_val_00019737.JPEG:n02259212 +ILSVRC2012_val_00019738.JPEG:n02128757 +ILSVRC2012_val_00019739.JPEG:n03595614 +ILSVRC2012_val_00019740.JPEG:n03126707 +ILSVRC2012_val_00019741.JPEG:n04200800 +ILSVRC2012_val_00019742.JPEG:n12620546 +ILSVRC2012_val_00019743.JPEG:n02091032 +ILSVRC2012_val_00019744.JPEG:n01531178 +ILSVRC2012_val_00019745.JPEG:n03775071 +ILSVRC2012_val_00019746.JPEG:n02346627 +ILSVRC2012_val_00019747.JPEG:n02096294 +ILSVRC2012_val_00019748.JPEG:n04204347 +ILSVRC2012_val_00019749.JPEG:n02892201 +ILSVRC2012_val_00019750.JPEG:n01807496 +ILSVRC2012_val_00019751.JPEG:n03825788 +ILSVRC2012_val_00019752.JPEG:n02342885 +ILSVRC2012_val_00019753.JPEG:n02128385 +ILSVRC2012_val_00019754.JPEG:n07745940 +ILSVRC2012_val_00019755.JPEG:n04404412 +ILSVRC2012_val_00019756.JPEG:n03720891 +ILSVRC2012_val_00019757.JPEG:n02109961 +ILSVRC2012_val_00019758.JPEG:n03976657 +ILSVRC2012_val_00019759.JPEG:n02093256 +ILSVRC2012_val_00019760.JPEG:n03787032 +ILSVRC2012_val_00019761.JPEG:n03794056 +ILSVRC2012_val_00019762.JPEG:n04136333 +ILSVRC2012_val_00019763.JPEG:n03787032 +ILSVRC2012_val_00019764.JPEG:n02105855 +ILSVRC2012_val_00019765.JPEG:n01774384 +ILSVRC2012_val_00019766.JPEG:n02974003 +ILSVRC2012_val_00019767.JPEG:n02106030 +ILSVRC2012_val_00019768.JPEG:n04023962 +ILSVRC2012_val_00019769.JPEG:n03485794 +ILSVRC2012_val_00019770.JPEG:n02086910 +ILSVRC2012_val_00019771.JPEG:n02091134 +ILSVRC2012_val_00019772.JPEG:n02727426 +ILSVRC2012_val_00019773.JPEG:n04591157 +ILSVRC2012_val_00019774.JPEG:n03804744 +ILSVRC2012_val_00019775.JPEG:n04111531 +ILSVRC2012_val_00019776.JPEG:n03733805 +ILSVRC2012_val_00019777.JPEG:n02787622 +ILSVRC2012_val_00019778.JPEG:n02980441 +ILSVRC2012_val_00019779.JPEG:n03347037 +ILSVRC2012_val_00019780.JPEG:n01630670 +ILSVRC2012_val_00019781.JPEG:n04579432 +ILSVRC2012_val_00019782.JPEG:n01944390 +ILSVRC2012_val_00019783.JPEG:n12620546 +ILSVRC2012_val_00019784.JPEG:n02114712 +ILSVRC2012_val_00019785.JPEG:n03527444 +ILSVRC2012_val_00019786.JPEG:n04239074 +ILSVRC2012_val_00019787.JPEG:n01807496 +ILSVRC2012_val_00019788.JPEG:n01592084 +ILSVRC2012_val_00019789.JPEG:n02879718 +ILSVRC2012_val_00019790.JPEG:n04429376 +ILSVRC2012_val_00019791.JPEG:n02643566 +ILSVRC2012_val_00019792.JPEG:n07871810 +ILSVRC2012_val_00019793.JPEG:n07753113 +ILSVRC2012_val_00019794.JPEG:n03042490 +ILSVRC2012_val_00019795.JPEG:n02281787 +ILSVRC2012_val_00019796.JPEG:n03179701 +ILSVRC2012_val_00019797.JPEG:n01685808 +ILSVRC2012_val_00019798.JPEG:n03814906 +ILSVRC2012_val_00019799.JPEG:n02927161 +ILSVRC2012_val_00019800.JPEG:n02346627 +ILSVRC2012_val_00019801.JPEG:n03160309 +ILSVRC2012_val_00019802.JPEG:n04037443 +ILSVRC2012_val_00019803.JPEG:n02708093 +ILSVRC2012_val_00019804.JPEG:n03590841 +ILSVRC2012_val_00019805.JPEG:n04370456 +ILSVRC2012_val_00019806.JPEG:n02948072 +ILSVRC2012_val_00019807.JPEG:n02494079 +ILSVRC2012_val_00019808.JPEG:n06785654 +ILSVRC2012_val_00019809.JPEG:n04507155 +ILSVRC2012_val_00019810.JPEG:n02011460 +ILSVRC2012_val_00019811.JPEG:n02256656 +ILSVRC2012_val_00019812.JPEG:n04037443 +ILSVRC2012_val_00019813.JPEG:n03485794 +ILSVRC2012_val_00019814.JPEG:n03271574 +ILSVRC2012_val_00019815.JPEG:n04254777 +ILSVRC2012_val_00019816.JPEG:n02128757 +ILSVRC2012_val_00019817.JPEG:n04154565 +ILSVRC2012_val_00019818.JPEG:n03461385 +ILSVRC2012_val_00019819.JPEG:n02966193 +ILSVRC2012_val_00019820.JPEG:n02226429 +ILSVRC2012_val_00019821.JPEG:n02101006 +ILSVRC2012_val_00019822.JPEG:n02112018 +ILSVRC2012_val_00019823.JPEG:n07695742 +ILSVRC2012_val_00019824.JPEG:n02110341 +ILSVRC2012_val_00019825.JPEG:n02443114 +ILSVRC2012_val_00019826.JPEG:n02110185 +ILSVRC2012_val_00019827.JPEG:n02948072 +ILSVRC2012_val_00019828.JPEG:n02840245 +ILSVRC2012_val_00019829.JPEG:n03854065 +ILSVRC2012_val_00019830.JPEG:n02096294 +ILSVRC2012_val_00019831.JPEG:n02980441 +ILSVRC2012_val_00019832.JPEG:n03062245 +ILSVRC2012_val_00019833.JPEG:n03584829 +ILSVRC2012_val_00019834.JPEG:n01644900 +ILSVRC2012_val_00019835.JPEG:n03891251 +ILSVRC2012_val_00019836.JPEG:n03599486 +ILSVRC2012_val_00019837.JPEG:n02701002 +ILSVRC2012_val_00019838.JPEG:n02172182 +ILSVRC2012_val_00019839.JPEG:n03888605 +ILSVRC2012_val_00019840.JPEG:n03642806 +ILSVRC2012_val_00019841.JPEG:n04562935 +ILSVRC2012_val_00019842.JPEG:n01930112 +ILSVRC2012_val_00019843.JPEG:n02389026 +ILSVRC2012_val_00019844.JPEG:n02783161 +ILSVRC2012_val_00019845.JPEG:n02807133 +ILSVRC2012_val_00019846.JPEG:n04099969 +ILSVRC2012_val_00019847.JPEG:n03457902 +ILSVRC2012_val_00019848.JPEG:n03633091 +ILSVRC2012_val_00019849.JPEG:n03594945 +ILSVRC2012_val_00019850.JPEG:n07695742 +ILSVRC2012_val_00019851.JPEG:n07714990 +ILSVRC2012_val_00019852.JPEG:n03208938 +ILSVRC2012_val_00019853.JPEG:n04479046 +ILSVRC2012_val_00019854.JPEG:n09835506 +ILSVRC2012_val_00019855.JPEG:n03595614 +ILSVRC2012_val_00019856.JPEG:n01983481 +ILSVRC2012_val_00019857.JPEG:n03670208 +ILSVRC2012_val_00019858.JPEG:n01734418 +ILSVRC2012_val_00019859.JPEG:n01978455 +ILSVRC2012_val_00019860.JPEG:n03721384 +ILSVRC2012_val_00019861.JPEG:n02091635 +ILSVRC2012_val_00019862.JPEG:n02133161 +ILSVRC2012_val_00019863.JPEG:n04026417 +ILSVRC2012_val_00019864.JPEG:n01734418 +ILSVRC2012_val_00019865.JPEG:n03530642 +ILSVRC2012_val_00019866.JPEG:n04209133 +ILSVRC2012_val_00019867.JPEG:n04099969 +ILSVRC2012_val_00019868.JPEG:n01616318 +ILSVRC2012_val_00019869.JPEG:n02279972 +ILSVRC2012_val_00019870.JPEG:n03676483 +ILSVRC2012_val_00019871.JPEG:n03868863 +ILSVRC2012_val_00019872.JPEG:n02666196 +ILSVRC2012_val_00019873.JPEG:n02396427 +ILSVRC2012_val_00019874.JPEG:n01768244 +ILSVRC2012_val_00019875.JPEG:n03240683 +ILSVRC2012_val_00019876.JPEG:n02112018 +ILSVRC2012_val_00019877.JPEG:n13133613 +ILSVRC2012_val_00019878.JPEG:n03032252 +ILSVRC2012_val_00019879.JPEG:n04235860 +ILSVRC2012_val_00019880.JPEG:n02110627 +ILSVRC2012_val_00019881.JPEG:n03404251 +ILSVRC2012_val_00019882.JPEG:n04350905 +ILSVRC2012_val_00019883.JPEG:n02087046 +ILSVRC2012_val_00019884.JPEG:n01843383 +ILSVRC2012_val_00019885.JPEG:n01797886 +ILSVRC2012_val_00019886.JPEG:n02992211 +ILSVRC2012_val_00019887.JPEG:n02950826 +ILSVRC2012_val_00019888.JPEG:n02268853 +ILSVRC2012_val_00019889.JPEG:n03888605 +ILSVRC2012_val_00019890.JPEG:n07248320 +ILSVRC2012_val_00019891.JPEG:n03160309 +ILSVRC2012_val_00019892.JPEG:n07248320 +ILSVRC2012_val_00019893.JPEG:n03868242 +ILSVRC2012_val_00019894.JPEG:n01704323 +ILSVRC2012_val_00019895.JPEG:n01944390 +ILSVRC2012_val_00019896.JPEG:n04462240 +ILSVRC2012_val_00019897.JPEG:n06794110 +ILSVRC2012_val_00019898.JPEG:n03032252 +ILSVRC2012_val_00019899.JPEG:n04376876 +ILSVRC2012_val_00019900.JPEG:n02281406 +ILSVRC2012_val_00019901.JPEG:n02134418 +ILSVRC2012_val_00019902.JPEG:n03584829 +ILSVRC2012_val_00019903.JPEG:n03598930 +ILSVRC2012_val_00019904.JPEG:n04254777 +ILSVRC2012_val_00019905.JPEG:n04435653 +ILSVRC2012_val_00019906.JPEG:n02017213 +ILSVRC2012_val_00019907.JPEG:n04049303 +ILSVRC2012_val_00019908.JPEG:n03180011 +ILSVRC2012_val_00019909.JPEG:n03782006 +ILSVRC2012_val_00019910.JPEG:n02749479 +ILSVRC2012_val_00019911.JPEG:n04525305 +ILSVRC2012_val_00019912.JPEG:n02791270 +ILSVRC2012_val_00019913.JPEG:n04429376 +ILSVRC2012_val_00019914.JPEG:n02102318 +ILSVRC2012_val_00019915.JPEG:n07584110 +ILSVRC2012_val_00019916.JPEG:n02966687 +ILSVRC2012_val_00019917.JPEG:n02423022 +ILSVRC2012_val_00019918.JPEG:n02107142 +ILSVRC2012_val_00019919.JPEG:n02101556 +ILSVRC2012_val_00019920.JPEG:n04179913 +ILSVRC2012_val_00019921.JPEG:n02999410 +ILSVRC2012_val_00019922.JPEG:n02091134 +ILSVRC2012_val_00019923.JPEG:n02797295 +ILSVRC2012_val_00019924.JPEG:n04560804 +ILSVRC2012_val_00019925.JPEG:n01955084 +ILSVRC2012_val_00019926.JPEG:n07583066 +ILSVRC2012_val_00019927.JPEG:n03743016 +ILSVRC2012_val_00019928.JPEG:n03623198 +ILSVRC2012_val_00019929.JPEG:n03843555 +ILSVRC2012_val_00019930.JPEG:n02134084 +ILSVRC2012_val_00019931.JPEG:n02093256 +ILSVRC2012_val_00019932.JPEG:n02105505 +ILSVRC2012_val_00019933.JPEG:n03788195 +ILSVRC2012_val_00019934.JPEG:n07716906 +ILSVRC2012_val_00019935.JPEG:n04542943 +ILSVRC2012_val_00019936.JPEG:n04296562 +ILSVRC2012_val_00019937.JPEG:n02120079 +ILSVRC2012_val_00019938.JPEG:n03920288 +ILSVRC2012_val_00019939.JPEG:n02892767 +ILSVRC2012_val_00019940.JPEG:n04311174 +ILSVRC2012_val_00019941.JPEG:n04141327 +ILSVRC2012_val_00019942.JPEG:n02117135 +ILSVRC2012_val_00019943.JPEG:n03888605 +ILSVRC2012_val_00019944.JPEG:n04557648 +ILSVRC2012_val_00019945.JPEG:n04523525 +ILSVRC2012_val_00019946.JPEG:n02281787 +ILSVRC2012_val_00019947.JPEG:n02951358 +ILSVRC2012_val_00019948.JPEG:n03680355 +ILSVRC2012_val_00019949.JPEG:n07693725 +ILSVRC2012_val_00019950.JPEG:n02870880 +ILSVRC2012_val_00019951.JPEG:n02007558 +ILSVRC2012_val_00019952.JPEG:n06596364 +ILSVRC2012_val_00019953.JPEG:n01984695 +ILSVRC2012_val_00019954.JPEG:n03345487 +ILSVRC2012_val_00019955.JPEG:n02091244 +ILSVRC2012_val_00019956.JPEG:n09256479 +ILSVRC2012_val_00019957.JPEG:n02105162 +ILSVRC2012_val_00019958.JPEG:n07693725 +ILSVRC2012_val_00019959.JPEG:n03838899 +ILSVRC2012_val_00019960.JPEG:n03534580 +ILSVRC2012_val_00019961.JPEG:n02493509 +ILSVRC2012_val_00019962.JPEG:n02096177 +ILSVRC2012_val_00019963.JPEG:n07892512 +ILSVRC2012_val_00019964.JPEG:n02018795 +ILSVRC2012_val_00019965.JPEG:n04592741 +ILSVRC2012_val_00019966.JPEG:n01728920 +ILSVRC2012_val_00019967.JPEG:n07875152 +ILSVRC2012_val_00019968.JPEG:n01773797 +ILSVRC2012_val_00019969.JPEG:n02051845 +ILSVRC2012_val_00019970.JPEG:n04273569 +ILSVRC2012_val_00019971.JPEG:n03125729 +ILSVRC2012_val_00019972.JPEG:n01773549 +ILSVRC2012_val_00019973.JPEG:n04376876 +ILSVRC2012_val_00019974.JPEG:n04336792 +ILSVRC2012_val_00019975.JPEG:n02137549 +ILSVRC2012_val_00019976.JPEG:n03633091 +ILSVRC2012_val_00019977.JPEG:n01877812 +ILSVRC2012_val_00019978.JPEG:n02128757 +ILSVRC2012_val_00019979.JPEG:n04423845 +ILSVRC2012_val_00019980.JPEG:n02981792 +ILSVRC2012_val_00019981.JPEG:n03452741 +ILSVRC2012_val_00019982.JPEG:n01735189 +ILSVRC2012_val_00019983.JPEG:n04532106 +ILSVRC2012_val_00019984.JPEG:n02268853 +ILSVRC2012_val_00019985.JPEG:n07615774 +ILSVRC2012_val_00019986.JPEG:n03538406 +ILSVRC2012_val_00019987.JPEG:n01917289 +ILSVRC2012_val_00019988.JPEG:n01496331 +ILSVRC2012_val_00019989.JPEG:n01773549 +ILSVRC2012_val_00019990.JPEG:n03788195 +ILSVRC2012_val_00019991.JPEG:n02916936 +ILSVRC2012_val_00019992.JPEG:n03045698 +ILSVRC2012_val_00019993.JPEG:n03743016 +ILSVRC2012_val_00019994.JPEG:n03868863 +ILSVRC2012_val_00019995.JPEG:n04479046 +ILSVRC2012_val_00019996.JPEG:n01882714 +ILSVRC2012_val_00019997.JPEG:n03197337 +ILSVRC2012_val_00019998.JPEG:n02013706 +ILSVRC2012_val_00019999.JPEG:n07873807 +ILSVRC2012_val_00020000.JPEG:n02480855 +ILSVRC2012_val_00020001.JPEG:n04409515 +ILSVRC2012_val_00020002.JPEG:n02930766 +ILSVRC2012_val_00020003.JPEG:n03888257 +ILSVRC2012_val_00020004.JPEG:n03127925 +ILSVRC2012_val_00020005.JPEG:n11939491 +ILSVRC2012_val_00020006.JPEG:n02328150 +ILSVRC2012_val_00020007.JPEG:n02895154 +ILSVRC2012_val_00020008.JPEG:n02408429 +ILSVRC2012_val_00020009.JPEG:n02361337 +ILSVRC2012_val_00020010.JPEG:n02092339 +ILSVRC2012_val_00020011.JPEG:n01484850 +ILSVRC2012_val_00020012.JPEG:n03065424 +ILSVRC2012_val_00020013.JPEG:n02167151 +ILSVRC2012_val_00020014.JPEG:n01798484 +ILSVRC2012_val_00020015.JPEG:n02110341 +ILSVRC2012_val_00020016.JPEG:n02085620 +ILSVRC2012_val_00020017.JPEG:n04417672 +ILSVRC2012_val_00020018.JPEG:n02097047 +ILSVRC2012_val_00020019.JPEG:n04235860 +ILSVRC2012_val_00020020.JPEG:n02692877 +ILSVRC2012_val_00020021.JPEG:n04599235 +ILSVRC2012_val_00020022.JPEG:n04201297 +ILSVRC2012_val_00020023.JPEG:n02110341 +ILSVRC2012_val_00020024.JPEG:n03776460 +ILSVRC2012_val_00020025.JPEG:n02037110 +ILSVRC2012_val_00020026.JPEG:n02174001 +ILSVRC2012_val_00020027.JPEG:n02797295 +ILSVRC2012_val_00020028.JPEG:n02939185 +ILSVRC2012_val_00020029.JPEG:n03637318 +ILSVRC2012_val_00020030.JPEG:n03710721 +ILSVRC2012_val_00020031.JPEG:n02086646 +ILSVRC2012_val_00020032.JPEG:n03657121 +ILSVRC2012_val_00020033.JPEG:n02509815 +ILSVRC2012_val_00020034.JPEG:n07836838 +ILSVRC2012_val_00020035.JPEG:n04592741 +ILSVRC2012_val_00020036.JPEG:n04264628 +ILSVRC2012_val_00020037.JPEG:n04399382 +ILSVRC2012_val_00020038.JPEG:n02814533 +ILSVRC2012_val_00020039.JPEG:n04311174 +ILSVRC2012_val_00020040.JPEG:n02137549 +ILSVRC2012_val_00020041.JPEG:n07753113 +ILSVRC2012_val_00020042.JPEG:n02704792 +ILSVRC2012_val_00020043.JPEG:n02093859 +ILSVRC2012_val_00020044.JPEG:n01694178 +ILSVRC2012_val_00020045.JPEG:n03444034 +ILSVRC2012_val_00020046.JPEG:n01784675 +ILSVRC2012_val_00020047.JPEG:n02088466 +ILSVRC2012_val_00020048.JPEG:n03692522 +ILSVRC2012_val_00020049.JPEG:n02091244 +ILSVRC2012_val_00020050.JPEG:n02133161 +ILSVRC2012_val_00020051.JPEG:n09835506 +ILSVRC2012_val_00020052.JPEG:n01614925 +ILSVRC2012_val_00020053.JPEG:n02168699 +ILSVRC2012_val_00020054.JPEG:n02113624 +ILSVRC2012_val_00020055.JPEG:n03109150 +ILSVRC2012_val_00020056.JPEG:n02190166 +ILSVRC2012_val_00020057.JPEG:n03710721 +ILSVRC2012_val_00020058.JPEG:n02092002 +ILSVRC2012_val_00020059.JPEG:n01644373 +ILSVRC2012_val_00020060.JPEG:n04357314 +ILSVRC2012_val_00020061.JPEG:n01704323 +ILSVRC2012_val_00020062.JPEG:n01882714 +ILSVRC2012_val_00020063.JPEG:n03908618 +ILSVRC2012_val_00020064.JPEG:n04592741 +ILSVRC2012_val_00020065.JPEG:n02095570 +ILSVRC2012_val_00020066.JPEG:n02870880 +ILSVRC2012_val_00020067.JPEG:n04277352 +ILSVRC2012_val_00020068.JPEG:n03666591 +ILSVRC2012_val_00020069.JPEG:n09332890 +ILSVRC2012_val_00020070.JPEG:n02090721 +ILSVRC2012_val_00020071.JPEG:n04326547 +ILSVRC2012_val_00020072.JPEG:n04251144 +ILSVRC2012_val_00020073.JPEG:n04033901 +ILSVRC2012_val_00020074.JPEG:n02977058 +ILSVRC2012_val_00020075.JPEG:n03095699 +ILSVRC2012_val_00020076.JPEG:n02114548 +ILSVRC2012_val_00020077.JPEG:n02966193 +ILSVRC2012_val_00020078.JPEG:n07717410 +ILSVRC2012_val_00020079.JPEG:n04562935 +ILSVRC2012_val_00020080.JPEG:n02814860 +ILSVRC2012_val_00020081.JPEG:n02963159 +ILSVRC2012_val_00020082.JPEG:n02090721 +ILSVRC2012_val_00020083.JPEG:n03891251 +ILSVRC2012_val_00020084.JPEG:n02325366 +ILSVRC2012_val_00020085.JPEG:n03630383 +ILSVRC2012_val_00020086.JPEG:n03742115 +ILSVRC2012_val_00020087.JPEG:n03400231 +ILSVRC2012_val_00020088.JPEG:n07753275 +ILSVRC2012_val_00020089.JPEG:n02174001 +ILSVRC2012_val_00020090.JPEG:n01877812 +ILSVRC2012_val_00020091.JPEG:n02870880 +ILSVRC2012_val_00020092.JPEG:n02892201 +ILSVRC2012_val_00020093.JPEG:n02727426 +ILSVRC2012_val_00020094.JPEG:n02115913 +ILSVRC2012_val_00020095.JPEG:n02395406 +ILSVRC2012_val_00020096.JPEG:n03956157 +ILSVRC2012_val_00020097.JPEG:n02074367 +ILSVRC2012_val_00020098.JPEG:n07760859 +ILSVRC2012_val_00020099.JPEG:n04476259 +ILSVRC2012_val_00020100.JPEG:n03018349 +ILSVRC2012_val_00020101.JPEG:n04208210 +ILSVRC2012_val_00020102.JPEG:n04560804 +ILSVRC2012_val_00020103.JPEG:n03794056 +ILSVRC2012_val_00020104.JPEG:n03803284 +ILSVRC2012_val_00020105.JPEG:n03476684 +ILSVRC2012_val_00020106.JPEG:n01514668 +ILSVRC2012_val_00020107.JPEG:n04347754 +ILSVRC2012_val_00020108.JPEG:n01773157 +ILSVRC2012_val_00020109.JPEG:n01820546 +ILSVRC2012_val_00020110.JPEG:n04443257 +ILSVRC2012_val_00020111.JPEG:n03976657 +ILSVRC2012_val_00020112.JPEG:n04146614 +ILSVRC2012_val_00020113.JPEG:n02100583 +ILSVRC2012_val_00020114.JPEG:n04476259 +ILSVRC2012_val_00020115.JPEG:n01776313 +ILSVRC2012_val_00020116.JPEG:n02095570 +ILSVRC2012_val_00020117.JPEG:n03180011 +ILSVRC2012_val_00020118.JPEG:n02110806 +ILSVRC2012_val_00020119.JPEG:n02129165 +ILSVRC2012_val_00020120.JPEG:n02504013 +ILSVRC2012_val_00020121.JPEG:n02808304 +ILSVRC2012_val_00020122.JPEG:n03854065 +ILSVRC2012_val_00020123.JPEG:n02066245 +ILSVRC2012_val_00020124.JPEG:n01685808 +ILSVRC2012_val_00020125.JPEG:n03290653 +ILSVRC2012_val_00020126.JPEG:n01924916 +ILSVRC2012_val_00020127.JPEG:n03776460 +ILSVRC2012_val_00020128.JPEG:n02102973 +ILSVRC2012_val_00020129.JPEG:n03871628 +ILSVRC2012_val_00020130.JPEG:n04266014 +ILSVRC2012_val_00020131.JPEG:n04350905 +ILSVRC2012_val_00020132.JPEG:n02104029 +ILSVRC2012_val_00020133.JPEG:n03598930 +ILSVRC2012_val_00020134.JPEG:n04344873 +ILSVRC2012_val_00020135.JPEG:n10565667 +ILSVRC2012_val_00020136.JPEG:n02123045 +ILSVRC2012_val_00020137.JPEG:n02437312 +ILSVRC2012_val_00020138.JPEG:n03759954 +ILSVRC2012_val_00020139.JPEG:n02437616 +ILSVRC2012_val_00020140.JPEG:n02123159 +ILSVRC2012_val_00020141.JPEG:n01664065 +ILSVRC2012_val_00020142.JPEG:n02916936 +ILSVRC2012_val_00020143.JPEG:n03124170 +ILSVRC2012_val_00020144.JPEG:n02504013 +ILSVRC2012_val_00020145.JPEG:n03272562 +ILSVRC2012_val_00020146.JPEG:n03617480 +ILSVRC2012_val_00020147.JPEG:n02091244 +ILSVRC2012_val_00020148.JPEG:n02051845 +ILSVRC2012_val_00020149.JPEG:n02090622 +ILSVRC2012_val_00020150.JPEG:n04376876 +ILSVRC2012_val_00020151.JPEG:n04613696 +ILSVRC2012_val_00020152.JPEG:n02108551 +ILSVRC2012_val_00020153.JPEG:n04328186 +ILSVRC2012_val_00020154.JPEG:n01682714 +ILSVRC2012_val_00020155.JPEG:n03777754 +ILSVRC2012_val_00020156.JPEG:n02095570 +ILSVRC2012_val_00020157.JPEG:n07802026 +ILSVRC2012_val_00020158.JPEG:n02437616 +ILSVRC2012_val_00020159.JPEG:n02169497 +ILSVRC2012_val_00020160.JPEG:n02100735 +ILSVRC2012_val_00020161.JPEG:n01748264 +ILSVRC2012_val_00020162.JPEG:n03942813 +ILSVRC2012_val_00020163.JPEG:n04296562 +ILSVRC2012_val_00020164.JPEG:n02264363 +ILSVRC2012_val_00020165.JPEG:n04517823 +ILSVRC2012_val_00020166.JPEG:n03207743 +ILSVRC2012_val_00020167.JPEG:n02927161 +ILSVRC2012_val_00020168.JPEG:n04332243 +ILSVRC2012_val_00020169.JPEG:n02110185 +ILSVRC2012_val_00020170.JPEG:n04409515 +ILSVRC2012_val_00020171.JPEG:n02480495 +ILSVRC2012_val_00020172.JPEG:n09468604 +ILSVRC2012_val_00020173.JPEG:n02100735 +ILSVRC2012_val_00020174.JPEG:n07716358 +ILSVRC2012_val_00020175.JPEG:n15075141 +ILSVRC2012_val_00020176.JPEG:n03814639 +ILSVRC2012_val_00020177.JPEG:n02105251 +ILSVRC2012_val_00020178.JPEG:n01537544 +ILSVRC2012_val_00020179.JPEG:n01855672 +ILSVRC2012_val_00020180.JPEG:n01644900 +ILSVRC2012_val_00020181.JPEG:n04037443 +ILSVRC2012_val_00020182.JPEG:n02870880 +ILSVRC2012_val_00020183.JPEG:n02264363 +ILSVRC2012_val_00020184.JPEG:n04336792 +ILSVRC2012_val_00020185.JPEG:n09229709 +ILSVRC2012_val_00020186.JPEG:n03146219 +ILSVRC2012_val_00020187.JPEG:n02837789 +ILSVRC2012_val_00020188.JPEG:n03733281 +ILSVRC2012_val_00020189.JPEG:n04599235 +ILSVRC2012_val_00020190.JPEG:n04008634 +ILSVRC2012_val_00020191.JPEG:n02111500 +ILSVRC2012_val_00020192.JPEG:n04560804 +ILSVRC2012_val_00020193.JPEG:n02116738 +ILSVRC2012_val_00020194.JPEG:n02009229 +ILSVRC2012_val_00020195.JPEG:n03272562 +ILSVRC2012_val_00020196.JPEG:n02106030 +ILSVRC2012_val_00020197.JPEG:n03666591 +ILSVRC2012_val_00020198.JPEG:n02356798 +ILSVRC2012_val_00020199.JPEG:n09835506 +ILSVRC2012_val_00020200.JPEG:n02727426 +ILSVRC2012_val_00020201.JPEG:n02113712 +ILSVRC2012_val_00020202.JPEG:n02397096 +ILSVRC2012_val_00020203.JPEG:n04153751 +ILSVRC2012_val_00020204.JPEG:n02808304 +ILSVRC2012_val_00020205.JPEG:n02033041 +ILSVRC2012_val_00020206.JPEG:n02992529 +ILSVRC2012_val_00020207.JPEG:n02837789 +ILSVRC2012_val_00020208.JPEG:n03355925 +ILSVRC2012_val_00020209.JPEG:n03492542 +ILSVRC2012_val_00020210.JPEG:n03991062 +ILSVRC2012_val_00020211.JPEG:n02457408 +ILSVRC2012_val_00020212.JPEG:n03085013 +ILSVRC2012_val_00020213.JPEG:n04501370 +ILSVRC2012_val_00020214.JPEG:n02843684 +ILSVRC2012_val_00020215.JPEG:n02490219 +ILSVRC2012_val_00020216.JPEG:n02106382 +ILSVRC2012_val_00020217.JPEG:n02489166 +ILSVRC2012_val_00020218.JPEG:n03670208 +ILSVRC2012_val_00020219.JPEG:n02447366 +ILSVRC2012_val_00020220.JPEG:n02655020 +ILSVRC2012_val_00020221.JPEG:n13054560 +ILSVRC2012_val_00020222.JPEG:n03445924 +ILSVRC2012_val_00020223.JPEG:n03903868 +ILSVRC2012_val_00020224.JPEG:n02099601 +ILSVRC2012_val_00020225.JPEG:n02119022 +ILSVRC2012_val_00020226.JPEG:n02422106 +ILSVRC2012_val_00020227.JPEG:n04019541 +ILSVRC2012_val_00020228.JPEG:n04355933 +ILSVRC2012_val_00020229.JPEG:n04200800 +ILSVRC2012_val_00020230.JPEG:n02123597 +ILSVRC2012_val_00020231.JPEG:n13052670 +ILSVRC2012_val_00020232.JPEG:n03250847 +ILSVRC2012_val_00020233.JPEG:n02992529 +ILSVRC2012_val_00020234.JPEG:n02951585 +ILSVRC2012_val_00020235.JPEG:n03085013 +ILSVRC2012_val_00020236.JPEG:n01768244 +ILSVRC2012_val_00020237.JPEG:n04525305 +ILSVRC2012_val_00020238.JPEG:n03187595 +ILSVRC2012_val_00020239.JPEG:n01798484 +ILSVRC2012_val_00020240.JPEG:n03467068 +ILSVRC2012_val_00020241.JPEG:n04370456 +ILSVRC2012_val_00020242.JPEG:n03832673 +ILSVRC2012_val_00020243.JPEG:n02097130 +ILSVRC2012_val_00020244.JPEG:n03240683 +ILSVRC2012_val_00020245.JPEG:n04371430 +ILSVRC2012_val_00020246.JPEG:n04579432 +ILSVRC2012_val_00020247.JPEG:n04458633 +ILSVRC2012_val_00020248.JPEG:n04483307 +ILSVRC2012_val_00020249.JPEG:n02980441 +ILSVRC2012_val_00020250.JPEG:n02102318 +ILSVRC2012_val_00020251.JPEG:n04154565 +ILSVRC2012_val_00020252.JPEG:n03452741 +ILSVRC2012_val_00020253.JPEG:n03961711 +ILSVRC2012_val_00020254.JPEG:n02808440 +ILSVRC2012_val_00020255.JPEG:n03063689 +ILSVRC2012_val_00020256.JPEG:n02114855 +ILSVRC2012_val_00020257.JPEG:n02096051 +ILSVRC2012_val_00020258.JPEG:n04461696 +ILSVRC2012_val_00020259.JPEG:n04487394 +ILSVRC2012_val_00020260.JPEG:n02113186 +ILSVRC2012_val_00020261.JPEG:n07892512 +ILSVRC2012_val_00020262.JPEG:n03223299 +ILSVRC2012_val_00020263.JPEG:n04081281 +ILSVRC2012_val_00020264.JPEG:n04371774 +ILSVRC2012_val_00020265.JPEG:n04417672 +ILSVRC2012_val_00020266.JPEG:n03249569 +ILSVRC2012_val_00020267.JPEG:n03197337 +ILSVRC2012_val_00020268.JPEG:n02101006 +ILSVRC2012_val_00020269.JPEG:n01768244 +ILSVRC2012_val_00020270.JPEG:n02113186 +ILSVRC2012_val_00020271.JPEG:n03899768 +ILSVRC2012_val_00020272.JPEG:n02783161 +ILSVRC2012_val_00020273.JPEG:n01734418 +ILSVRC2012_val_00020274.JPEG:n01728920 +ILSVRC2012_val_00020275.JPEG:n02497673 +ILSVRC2012_val_00020276.JPEG:n03063599 +ILSVRC2012_val_00020277.JPEG:n04479046 +ILSVRC2012_val_00020278.JPEG:n02895154 +ILSVRC2012_val_00020279.JPEG:n02100877 +ILSVRC2012_val_00020280.JPEG:n01983481 +ILSVRC2012_val_00020281.JPEG:n03908618 +ILSVRC2012_val_00020282.JPEG:n04507155 +ILSVRC2012_val_00020283.JPEG:n03344393 +ILSVRC2012_val_00020284.JPEG:n01829413 +ILSVRC2012_val_00020285.JPEG:n02342885 +ILSVRC2012_val_00020286.JPEG:n02190166 +ILSVRC2012_val_00020287.JPEG:n07802026 +ILSVRC2012_val_00020288.JPEG:n03991062 +ILSVRC2012_val_00020289.JPEG:n02974003 +ILSVRC2012_val_00020290.JPEG:n01698640 +ILSVRC2012_val_00020291.JPEG:n04447861 +ILSVRC2012_val_00020292.JPEG:n03623198 +ILSVRC2012_val_00020293.JPEG:n04347754 +ILSVRC2012_val_00020294.JPEG:n07614500 +ILSVRC2012_val_00020295.JPEG:n12144580 +ILSVRC2012_val_00020296.JPEG:n04254680 +ILSVRC2012_val_00020297.JPEG:n04482393 +ILSVRC2012_val_00020298.JPEG:n01943899 +ILSVRC2012_val_00020299.JPEG:n03887697 +ILSVRC2012_val_00020300.JPEG:n03598930 +ILSVRC2012_val_00020301.JPEG:n02483362 +ILSVRC2012_val_00020302.JPEG:n02120079 +ILSVRC2012_val_00020303.JPEG:n03680355 +ILSVRC2012_val_00020304.JPEG:n03485407 +ILSVRC2012_val_00020305.JPEG:n02130308 +ILSVRC2012_val_00020306.JPEG:n02894605 +ILSVRC2012_val_00020307.JPEG:n03841143 +ILSVRC2012_val_00020308.JPEG:n02172182 +ILSVRC2012_val_00020309.JPEG:n02727426 +ILSVRC2012_val_00020310.JPEG:n04418357 +ILSVRC2012_val_00020311.JPEG:n02097209 +ILSVRC2012_val_00020312.JPEG:n03495258 +ILSVRC2012_val_00020313.JPEG:n02701002 +ILSVRC2012_val_00020314.JPEG:n03481172 +ILSVRC2012_val_00020315.JPEG:n02860847 +ILSVRC2012_val_00020316.JPEG:n04435653 +ILSVRC2012_val_00020317.JPEG:n03384352 +ILSVRC2012_val_00020318.JPEG:n04131690 +ILSVRC2012_val_00020319.JPEG:n02701002 +ILSVRC2012_val_00020320.JPEG:n03868863 +ILSVRC2012_val_00020321.JPEG:n01644373 +ILSVRC2012_val_00020322.JPEG:n03000247 +ILSVRC2012_val_00020323.JPEG:n02397096 +ILSVRC2012_val_00020324.JPEG:n04118776 +ILSVRC2012_val_00020325.JPEG:n02117135 +ILSVRC2012_val_00020326.JPEG:n02051845 +ILSVRC2012_val_00020327.JPEG:n03649909 +ILSVRC2012_val_00020328.JPEG:n02869837 +ILSVRC2012_val_00020329.JPEG:n03661043 +ILSVRC2012_val_00020330.JPEG:n02090622 +ILSVRC2012_val_00020331.JPEG:n02190166 +ILSVRC2012_val_00020332.JPEG:n02134084 +ILSVRC2012_val_00020333.JPEG:n02701002 +ILSVRC2012_val_00020334.JPEG:n03496892 +ILSVRC2012_val_00020335.JPEG:n02871525 +ILSVRC2012_val_00020336.JPEG:n04277352 +ILSVRC2012_val_00020337.JPEG:n02966193 +ILSVRC2012_val_00020338.JPEG:n07697313 +ILSVRC2012_val_00020339.JPEG:n03447447 +ILSVRC2012_val_00020340.JPEG:n03388183 +ILSVRC2012_val_00020341.JPEG:n02483708 +ILSVRC2012_val_00020342.JPEG:n03623198 +ILSVRC2012_val_00020343.JPEG:n09421951 +ILSVRC2012_val_00020344.JPEG:n02128925 +ILSVRC2012_val_00020345.JPEG:n02823428 +ILSVRC2012_val_00020346.JPEG:n02410509 +ILSVRC2012_val_00020347.JPEG:n02099429 +ILSVRC2012_val_00020348.JPEG:n04162706 +ILSVRC2012_val_00020349.JPEG:n01601694 +ILSVRC2012_val_00020350.JPEG:n06794110 +ILSVRC2012_val_00020351.JPEG:n03929660 +ILSVRC2012_val_00020352.JPEG:n07920052 +ILSVRC2012_val_00020353.JPEG:n04273569 +ILSVRC2012_val_00020354.JPEG:n02259212 +ILSVRC2012_val_00020355.JPEG:n03180011 +ILSVRC2012_val_00020356.JPEG:n01685808 +ILSVRC2012_val_00020357.JPEG:n02095889 +ILSVRC2012_val_00020358.JPEG:n04204347 +ILSVRC2012_val_00020359.JPEG:n02804414 +ILSVRC2012_val_00020360.JPEG:n02236044 +ILSVRC2012_val_00020361.JPEG:n04111531 +ILSVRC2012_val_00020362.JPEG:n02132136 +ILSVRC2012_val_00020363.JPEG:n07717556 +ILSVRC2012_val_00020364.JPEG:n03388183 +ILSVRC2012_val_00020365.JPEG:n04200800 +ILSVRC2012_val_00020366.JPEG:n04154565 +ILSVRC2012_val_00020367.JPEG:n02099601 +ILSVRC2012_val_00020368.JPEG:n03065424 +ILSVRC2012_val_00020369.JPEG:n03942813 +ILSVRC2012_val_00020370.JPEG:n01930112 +ILSVRC2012_val_00020371.JPEG:n04049303 +ILSVRC2012_val_00020372.JPEG:n02965783 +ILSVRC2012_val_00020373.JPEG:n03444034 +ILSVRC2012_val_00020374.JPEG:n03131574 +ILSVRC2012_val_00020375.JPEG:n02090721 +ILSVRC2012_val_00020376.JPEG:n02281787 +ILSVRC2012_val_00020377.JPEG:n04389033 +ILSVRC2012_val_00020378.JPEG:n07615774 +ILSVRC2012_val_00020379.JPEG:n02086240 +ILSVRC2012_val_00020380.JPEG:n02105412 +ILSVRC2012_val_00020381.JPEG:n03794056 +ILSVRC2012_val_00020382.JPEG:n03977966 +ILSVRC2012_val_00020383.JPEG:n01728572 +ILSVRC2012_val_00020384.JPEG:n03218198 +ILSVRC2012_val_00020385.JPEG:n07584110 +ILSVRC2012_val_00020386.JPEG:n02134084 +ILSVRC2012_val_00020387.JPEG:n03991062 +ILSVRC2012_val_00020388.JPEG:n03124170 +ILSVRC2012_val_00020389.JPEG:n04070727 +ILSVRC2012_val_00020390.JPEG:n03908618 +ILSVRC2012_val_00020391.JPEG:n07932039 +ILSVRC2012_val_00020392.JPEG:n02110806 +ILSVRC2012_val_00020393.JPEG:n01630670 +ILSVRC2012_val_00020394.JPEG:n03598930 +ILSVRC2012_val_00020395.JPEG:n04355338 +ILSVRC2012_val_00020396.JPEG:n03014705 +ILSVRC2012_val_00020397.JPEG:n02172182 +ILSVRC2012_val_00020398.JPEG:n03721384 +ILSVRC2012_val_00020399.JPEG:n02095314 +ILSVRC2012_val_00020400.JPEG:n02979186 +ILSVRC2012_val_00020401.JPEG:n01742172 +ILSVRC2012_val_00020402.JPEG:n04409515 +ILSVRC2012_val_00020403.JPEG:n02089973 +ILSVRC2012_val_00020404.JPEG:n02422699 +ILSVRC2012_val_00020405.JPEG:n03763968 +ILSVRC2012_val_00020406.JPEG:n02492660 +ILSVRC2012_val_00020407.JPEG:n02910353 +ILSVRC2012_val_00020408.JPEG:n03743016 +ILSVRC2012_val_00020409.JPEG:n03196217 +ILSVRC2012_val_00020410.JPEG:n02840245 +ILSVRC2012_val_00020411.JPEG:n03804744 +ILSVRC2012_val_00020412.JPEG:n04532106 +ILSVRC2012_val_00020413.JPEG:n03773504 +ILSVRC2012_val_00020414.JPEG:n02100236 +ILSVRC2012_val_00020415.JPEG:n02325366 +ILSVRC2012_val_00020416.JPEG:n07753275 +ILSVRC2012_val_00020417.JPEG:n03483316 +ILSVRC2012_val_00020418.JPEG:n01494475 +ILSVRC2012_val_00020419.JPEG:n04344873 +ILSVRC2012_val_00020420.JPEG:n04259630 +ILSVRC2012_val_00020421.JPEG:n03627232 +ILSVRC2012_val_00020422.JPEG:n02280649 +ILSVRC2012_val_00020423.JPEG:n02883205 +ILSVRC2012_val_00020424.JPEG:n04404412 +ILSVRC2012_val_00020425.JPEG:n04357314 +ILSVRC2012_val_00020426.JPEG:n04286575 +ILSVRC2012_val_00020427.JPEG:n03803284 +ILSVRC2012_val_00020428.JPEG:n02098413 +ILSVRC2012_val_00020429.JPEG:n04209239 +ILSVRC2012_val_00020430.JPEG:n01632777 +ILSVRC2012_val_00020431.JPEG:n03908618 +ILSVRC2012_val_00020432.JPEG:n02110185 +ILSVRC2012_val_00020433.JPEG:n02457408 +ILSVRC2012_val_00020434.JPEG:n02788148 +ILSVRC2012_val_00020435.JPEG:n03467068 +ILSVRC2012_val_00020436.JPEG:n01443537 +ILSVRC2012_val_00020437.JPEG:n04310018 +ILSVRC2012_val_00020438.JPEG:n03325584 +ILSVRC2012_val_00020439.JPEG:n02395406 +ILSVRC2012_val_00020440.JPEG:n03133878 +ILSVRC2012_val_00020441.JPEG:n02134084 +ILSVRC2012_val_00020442.JPEG:n02089867 +ILSVRC2012_val_00020443.JPEG:n01833805 +ILSVRC2012_val_00020444.JPEG:n03443371 +ILSVRC2012_val_00020445.JPEG:n03838899 +ILSVRC2012_val_00020446.JPEG:n03216828 +ILSVRC2012_val_00020447.JPEG:n03485794 +ILSVRC2012_val_00020448.JPEG:n03761084 +ILSVRC2012_val_00020449.JPEG:n02500267 +ILSVRC2012_val_00020450.JPEG:n04435653 +ILSVRC2012_val_00020451.JPEG:n01514668 +ILSVRC2012_val_00020452.JPEG:n10565667 +ILSVRC2012_val_00020453.JPEG:n01675722 +ILSVRC2012_val_00020454.JPEG:n02233338 +ILSVRC2012_val_00020455.JPEG:n02497673 +ILSVRC2012_val_00020456.JPEG:n01784675 +ILSVRC2012_val_00020457.JPEG:n03761084 +ILSVRC2012_val_00020458.JPEG:n02279972 +ILSVRC2012_val_00020459.JPEG:n03721384 +ILSVRC2012_val_00020460.JPEG:n02088238 +ILSVRC2012_val_00020461.JPEG:n03017168 +ILSVRC2012_val_00020462.JPEG:n01770081 +ILSVRC2012_val_00020463.JPEG:n03347037 +ILSVRC2012_val_00020464.JPEG:n02231487 +ILSVRC2012_val_00020465.JPEG:n12768682 +ILSVRC2012_val_00020466.JPEG:n03877472 +ILSVRC2012_val_00020467.JPEG:n02730930 +ILSVRC2012_val_00020468.JPEG:n02088238 +ILSVRC2012_val_00020469.JPEG:n01592084 +ILSVRC2012_val_00020470.JPEG:n03998194 +ILSVRC2012_val_00020471.JPEG:n03478589 +ILSVRC2012_val_00020472.JPEG:n03776460 +ILSVRC2012_val_00020473.JPEG:n02086910 +ILSVRC2012_val_00020474.JPEG:n02113624 +ILSVRC2012_val_00020475.JPEG:n02669723 +ILSVRC2012_val_00020476.JPEG:n01930112 +ILSVRC2012_val_00020477.JPEG:n04356056 +ILSVRC2012_val_00020478.JPEG:n12768682 +ILSVRC2012_val_00020479.JPEG:n09421951 +ILSVRC2012_val_00020480.JPEG:n03908618 +ILSVRC2012_val_00020481.JPEG:n02120079 +ILSVRC2012_val_00020482.JPEG:n02133161 +ILSVRC2012_val_00020483.JPEG:n03345487 +ILSVRC2012_val_00020484.JPEG:n02087046 +ILSVRC2012_val_00020485.JPEG:n04118538 +ILSVRC2012_val_00020486.JPEG:n03344393 +ILSVRC2012_val_00020487.JPEG:n02704792 +ILSVRC2012_val_00020488.JPEG:n02112018 +ILSVRC2012_val_00020489.JPEG:n02100583 +ILSVRC2012_val_00020490.JPEG:n03196217 +ILSVRC2012_val_00020491.JPEG:n04133789 +ILSVRC2012_val_00020492.JPEG:n02640242 +ILSVRC2012_val_00020493.JPEG:n02817516 +ILSVRC2012_val_00020494.JPEG:n01740131 +ILSVRC2012_val_00020495.JPEG:n01532829 +ILSVRC2012_val_00020496.JPEG:n04548362 +ILSVRC2012_val_00020497.JPEG:n04509417 +ILSVRC2012_val_00020498.JPEG:n02364673 +ILSVRC2012_val_00020499.JPEG:n02415577 +ILSVRC2012_val_00020500.JPEG:n04204347 +ILSVRC2012_val_00020501.JPEG:n12267677 +ILSVRC2012_val_00020502.JPEG:n03445777 +ILSVRC2012_val_00020503.JPEG:n07584110 +ILSVRC2012_val_00020504.JPEG:n03544143 +ILSVRC2012_val_00020505.JPEG:n03764736 +ILSVRC2012_val_00020506.JPEG:n07892512 +ILSVRC2012_val_00020507.JPEG:n01770393 +ILSVRC2012_val_00020508.JPEG:n01688243 +ILSVRC2012_val_00020509.JPEG:n04033995 +ILSVRC2012_val_00020510.JPEG:n04590129 +ILSVRC2012_val_00020511.JPEG:n01978287 +ILSVRC2012_val_00020512.JPEG:n02113712 +ILSVRC2012_val_00020513.JPEG:n02093428 +ILSVRC2012_val_00020514.JPEG:n01819313 +ILSVRC2012_val_00020515.JPEG:n02437312 +ILSVRC2012_val_00020516.JPEG:n03706229 +ILSVRC2012_val_00020517.JPEG:n03535780 +ILSVRC2012_val_00020518.JPEG:n02112137 +ILSVRC2012_val_00020519.JPEG:n04266014 +ILSVRC2012_val_00020520.JPEG:n02137549 +ILSVRC2012_val_00020521.JPEG:n03630383 +ILSVRC2012_val_00020522.JPEG:n03089624 +ILSVRC2012_val_00020523.JPEG:n04208210 +ILSVRC2012_val_00020524.JPEG:n03100240 +ILSVRC2012_val_00020525.JPEG:n02480495 +ILSVRC2012_val_00020526.JPEG:n02860847 +ILSVRC2012_val_00020527.JPEG:n03062245 +ILSVRC2012_val_00020528.JPEG:n04409515 +ILSVRC2012_val_00020529.JPEG:n04404412 +ILSVRC2012_val_00020530.JPEG:n02687172 +ILSVRC2012_val_00020531.JPEG:n04065272 +ILSVRC2012_val_00020532.JPEG:n03770439 +ILSVRC2012_val_00020533.JPEG:n04049303 +ILSVRC2012_val_00020534.JPEG:n03249569 +ILSVRC2012_val_00020535.JPEG:n02088238 +ILSVRC2012_val_00020536.JPEG:n01978287 +ILSVRC2012_val_00020537.JPEG:n04532106 +ILSVRC2012_val_00020538.JPEG:n01687978 +ILSVRC2012_val_00020539.JPEG:n01751748 +ILSVRC2012_val_00020540.JPEG:n02981792 +ILSVRC2012_val_00020541.JPEG:n03792972 +ILSVRC2012_val_00020542.JPEG:n04326547 +ILSVRC2012_val_00020543.JPEG:n01728920 +ILSVRC2012_val_00020544.JPEG:n04612504 +ILSVRC2012_val_00020545.JPEG:n07714990 +ILSVRC2012_val_00020546.JPEG:n03764736 +ILSVRC2012_val_00020547.JPEG:n07717410 +ILSVRC2012_val_00020548.JPEG:n04141327 +ILSVRC2012_val_00020549.JPEG:n03032252 +ILSVRC2012_val_00020550.JPEG:n02107574 +ILSVRC2012_val_00020551.JPEG:n02226429 +ILSVRC2012_val_00020552.JPEG:n01820546 +ILSVRC2012_val_00020553.JPEG:n02088364 +ILSVRC2012_val_00020554.JPEG:n03961711 +ILSVRC2012_val_00020555.JPEG:n07753113 +ILSVRC2012_val_00020556.JPEG:n02094114 +ILSVRC2012_val_00020557.JPEG:n03733805 +ILSVRC2012_val_00020558.JPEG:n02607072 +ILSVRC2012_val_00020559.JPEG:n02028035 +ILSVRC2012_val_00020560.JPEG:n03857828 +ILSVRC2012_val_00020561.JPEG:n02807133 +ILSVRC2012_val_00020562.JPEG:n04456115 +ILSVRC2012_val_00020563.JPEG:n02640242 +ILSVRC2012_val_00020564.JPEG:n02206856 +ILSVRC2012_val_00020565.JPEG:n12144580 +ILSVRC2012_val_00020566.JPEG:n02115913 +ILSVRC2012_val_00020567.JPEG:n03627232 +ILSVRC2012_val_00020568.JPEG:n02699494 +ILSVRC2012_val_00020569.JPEG:n01756291 +ILSVRC2012_val_00020570.JPEG:n03630383 +ILSVRC2012_val_00020571.JPEG:n02280649 +ILSVRC2012_val_00020572.JPEG:n02799071 +ILSVRC2012_val_00020573.JPEG:n07749582 +ILSVRC2012_val_00020574.JPEG:n01773157 +ILSVRC2012_val_00020575.JPEG:n09256479 +ILSVRC2012_val_00020576.JPEG:n04235860 +ILSVRC2012_val_00020577.JPEG:n06874185 +ILSVRC2012_val_00020578.JPEG:n02002556 +ILSVRC2012_val_00020579.JPEG:n02454379 +ILSVRC2012_val_00020580.JPEG:n03775546 +ILSVRC2012_val_00020581.JPEG:n02177972 +ILSVRC2012_val_00020582.JPEG:n02009229 +ILSVRC2012_val_00020583.JPEG:n03297495 +ILSVRC2012_val_00020584.JPEG:n03895866 +ILSVRC2012_val_00020585.JPEG:n01694178 +ILSVRC2012_val_00020586.JPEG:n01698640 +ILSVRC2012_val_00020587.JPEG:n01796340 +ILSVRC2012_val_00020588.JPEG:n03124043 +ILSVRC2012_val_00020589.JPEG:n02107683 +ILSVRC2012_val_00020590.JPEG:n02981792 +ILSVRC2012_val_00020591.JPEG:n04540053 +ILSVRC2012_val_00020592.JPEG:n07695742 +ILSVRC2012_val_00020593.JPEG:n02102318 +ILSVRC2012_val_00020594.JPEG:n02123597 +ILSVRC2012_val_00020595.JPEG:n04152593 +ILSVRC2012_val_00020596.JPEG:n01695060 +ILSVRC2012_val_00020597.JPEG:n04252077 +ILSVRC2012_val_00020598.JPEG:n01689811 +ILSVRC2012_val_00020599.JPEG:n01882714 +ILSVRC2012_val_00020600.JPEG:n04141327 +ILSVRC2012_val_00020601.JPEG:n07753592 +ILSVRC2012_val_00020602.JPEG:n02793495 +ILSVRC2012_val_00020603.JPEG:n04136333 +ILSVRC2012_val_00020604.JPEG:n03876231 +ILSVRC2012_val_00020605.JPEG:n02860847 +ILSVRC2012_val_00020606.JPEG:n04591157 +ILSVRC2012_val_00020607.JPEG:n04380533 +ILSVRC2012_val_00020608.JPEG:n03259280 +ILSVRC2012_val_00020609.JPEG:n03530642 +ILSVRC2012_val_00020610.JPEG:n01558993 +ILSVRC2012_val_00020611.JPEG:n04355338 +ILSVRC2012_val_00020612.JPEG:n02017213 +ILSVRC2012_val_00020613.JPEG:n02091032 +ILSVRC2012_val_00020614.JPEG:n07615774 +ILSVRC2012_val_00020615.JPEG:n07693725 +ILSVRC2012_val_00020616.JPEG:n02319095 +ILSVRC2012_val_00020617.JPEG:n04335435 +ILSVRC2012_val_00020618.JPEG:n06794110 +ILSVRC2012_val_00020619.JPEG:n11879895 +ILSVRC2012_val_00020620.JPEG:n09332890 +ILSVRC2012_val_00020621.JPEG:n02708093 +ILSVRC2012_val_00020622.JPEG:n02643566 +ILSVRC2012_val_00020623.JPEG:n03895866 +ILSVRC2012_val_00020624.JPEG:n03838899 +ILSVRC2012_val_00020625.JPEG:n03393912 +ILSVRC2012_val_00020626.JPEG:n02112137 +ILSVRC2012_val_00020627.JPEG:n01955084 +ILSVRC2012_val_00020628.JPEG:n02094433 +ILSVRC2012_val_00020629.JPEG:n02791124 +ILSVRC2012_val_00020630.JPEG:n03877472 +ILSVRC2012_val_00020631.JPEG:n03792782 +ILSVRC2012_val_00020632.JPEG:n01756291 +ILSVRC2012_val_00020633.JPEG:n02097474 +ILSVRC2012_val_00020634.JPEG:n03259280 +ILSVRC2012_val_00020635.JPEG:n02190166 +ILSVRC2012_val_00020636.JPEG:n07715103 +ILSVRC2012_val_00020637.JPEG:n02095889 +ILSVRC2012_val_00020638.JPEG:n04532106 +ILSVRC2012_val_00020639.JPEG:n04597913 +ILSVRC2012_val_00020640.JPEG:n03743016 +ILSVRC2012_val_00020641.JPEG:n04548362 +ILSVRC2012_val_00020642.JPEG:n02481823 +ILSVRC2012_val_00020643.JPEG:n03388549 +ILSVRC2012_val_00020644.JPEG:n02319095 +ILSVRC2012_val_00020645.JPEG:n03792972 +ILSVRC2012_val_00020646.JPEG:n02823750 +ILSVRC2012_val_00020647.JPEG:n03623198 +ILSVRC2012_val_00020648.JPEG:n03933933 +ILSVRC2012_val_00020649.JPEG:n02231487 +ILSVRC2012_val_00020650.JPEG:n03476684 +ILSVRC2012_val_00020651.JPEG:n02098286 +ILSVRC2012_val_00020652.JPEG:n02169497 +ILSVRC2012_val_00020653.JPEG:n03379051 +ILSVRC2012_val_00020654.JPEG:n02457408 +ILSVRC2012_val_00020655.JPEG:n07742313 +ILSVRC2012_val_00020656.JPEG:n07615774 +ILSVRC2012_val_00020657.JPEG:n02206856 +ILSVRC2012_val_00020658.JPEG:n04239074 +ILSVRC2012_val_00020659.JPEG:n03393912 +ILSVRC2012_val_00020660.JPEG:n01592084 +ILSVRC2012_val_00020661.JPEG:n03680355 +ILSVRC2012_val_00020662.JPEG:n02837789 +ILSVRC2012_val_00020663.JPEG:n03590841 +ILSVRC2012_val_00020664.JPEG:n01986214 +ILSVRC2012_val_00020665.JPEG:n03657121 +ILSVRC2012_val_00020666.JPEG:n03697007 +ILSVRC2012_val_00020667.JPEG:n01697457 +ILSVRC2012_val_00020668.JPEG:n02447366 +ILSVRC2012_val_00020669.JPEG:n04418357 +ILSVRC2012_val_00020670.JPEG:n04367480 +ILSVRC2012_val_00020671.JPEG:n03220513 +ILSVRC2012_val_00020672.JPEG:n04479046 +ILSVRC2012_val_00020673.JPEG:n03100240 +ILSVRC2012_val_00020674.JPEG:n03000684 +ILSVRC2012_val_00020675.JPEG:n01978287 +ILSVRC2012_val_00020676.JPEG:n02105855 +ILSVRC2012_val_00020677.JPEG:n03127925 +ILSVRC2012_val_00020678.JPEG:n02105855 +ILSVRC2012_val_00020679.JPEG:n02092002 +ILSVRC2012_val_00020680.JPEG:n02028035 +ILSVRC2012_val_00020681.JPEG:n02094258 +ILSVRC2012_val_00020682.JPEG:n04204347 +ILSVRC2012_val_00020683.JPEG:n01795545 +ILSVRC2012_val_00020684.JPEG:n02125311 +ILSVRC2012_val_00020685.JPEG:n02823750 +ILSVRC2012_val_00020686.JPEG:n02112137 +ILSVRC2012_val_00020687.JPEG:n03126707 +ILSVRC2012_val_00020688.JPEG:n02123597 +ILSVRC2012_val_00020689.JPEG:n03223299 +ILSVRC2012_val_00020690.JPEG:n01798484 +ILSVRC2012_val_00020691.JPEG:n02280649 +ILSVRC2012_val_00020692.JPEG:n01776313 +ILSVRC2012_val_00020693.JPEG:n02641379 +ILSVRC2012_val_00020694.JPEG:n01608432 +ILSVRC2012_val_00020695.JPEG:n03249569 +ILSVRC2012_val_00020696.JPEG:n01630670 +ILSVRC2012_val_00020697.JPEG:n03895866 +ILSVRC2012_val_00020698.JPEG:n03888257 +ILSVRC2012_val_00020699.JPEG:n02422106 +ILSVRC2012_val_00020700.JPEG:n02093859 +ILSVRC2012_val_00020701.JPEG:n04125021 +ILSVRC2012_val_00020702.JPEG:n04065272 +ILSVRC2012_val_00020703.JPEG:n03814906 +ILSVRC2012_val_00020704.JPEG:n03992509 +ILSVRC2012_val_00020705.JPEG:n04423845 +ILSVRC2012_val_00020706.JPEG:n03393912 +ILSVRC2012_val_00020707.JPEG:n02066245 +ILSVRC2012_val_00020708.JPEG:n02114548 +ILSVRC2012_val_00020709.JPEG:n10148035 +ILSVRC2012_val_00020710.JPEG:n01608432 +ILSVRC2012_val_00020711.JPEG:n04355338 +ILSVRC2012_val_00020712.JPEG:n04277352 +ILSVRC2012_val_00020713.JPEG:n03976467 +ILSVRC2012_val_00020714.JPEG:n02859443 +ILSVRC2012_val_00020715.JPEG:n04141076 +ILSVRC2012_val_00020716.JPEG:n02127052 +ILSVRC2012_val_00020717.JPEG:n02088466 +ILSVRC2012_val_00020718.JPEG:n07880968 +ILSVRC2012_val_00020719.JPEG:n09835506 +ILSVRC2012_val_00020720.JPEG:n03874293 +ILSVRC2012_val_00020721.JPEG:n03481172 +ILSVRC2012_val_00020722.JPEG:n04355338 +ILSVRC2012_val_00020723.JPEG:n02894605 +ILSVRC2012_val_00020724.JPEG:n03544143 +ILSVRC2012_val_00020725.JPEG:n02977058 +ILSVRC2012_val_00020726.JPEG:n01773157 +ILSVRC2012_val_00020727.JPEG:n02486261 +ILSVRC2012_val_00020728.JPEG:n02112137 +ILSVRC2012_val_00020729.JPEG:n03075370 +ILSVRC2012_val_00020730.JPEG:n01601694 +ILSVRC2012_val_00020731.JPEG:n04004767 +ILSVRC2012_val_00020732.JPEG:n04273569 +ILSVRC2012_val_00020733.JPEG:n04275548 +ILSVRC2012_val_00020734.JPEG:n02966193 +ILSVRC2012_val_00020735.JPEG:n03443371 +ILSVRC2012_val_00020736.JPEG:n01755581 +ILSVRC2012_val_00020737.JPEG:n02100877 +ILSVRC2012_val_00020738.JPEG:n04325704 +ILSVRC2012_val_00020739.JPEG:n02090379 +ILSVRC2012_val_00020740.JPEG:n02088466 +ILSVRC2012_val_00020741.JPEG:n03347037 +ILSVRC2012_val_00020742.JPEG:n03691459 +ILSVRC2012_val_00020743.JPEG:n01616318 +ILSVRC2012_val_00020744.JPEG:n01820546 +ILSVRC2012_val_00020745.JPEG:n04009552 +ILSVRC2012_val_00020746.JPEG:n03637318 +ILSVRC2012_val_00020747.JPEG:n01795545 +ILSVRC2012_val_00020748.JPEG:n02108000 +ILSVRC2012_val_00020749.JPEG:n01843383 +ILSVRC2012_val_00020750.JPEG:n03908618 +ILSVRC2012_val_00020751.JPEG:n07753275 +ILSVRC2012_val_00020752.JPEG:n02950826 +ILSVRC2012_val_00020753.JPEG:n04069434 +ILSVRC2012_val_00020754.JPEG:n02701002 +ILSVRC2012_val_00020755.JPEG:n02799071 +ILSVRC2012_val_00020756.JPEG:n02786058 +ILSVRC2012_val_00020757.JPEG:n02526121 +ILSVRC2012_val_00020758.JPEG:n03459775 +ILSVRC2012_val_00020759.JPEG:n04552348 +ILSVRC2012_val_00020760.JPEG:n04462240 +ILSVRC2012_val_00020761.JPEG:n02108915 +ILSVRC2012_val_00020762.JPEG:n02088364 +ILSVRC2012_val_00020763.JPEG:n02791270 +ILSVRC2012_val_00020764.JPEG:n01682714 +ILSVRC2012_val_00020765.JPEG:n02123394 +ILSVRC2012_val_00020766.JPEG:n02101388 +ILSVRC2012_val_00020767.JPEG:n02840245 +ILSVRC2012_val_00020768.JPEG:n04493381 +ILSVRC2012_val_00020769.JPEG:n01990800 +ILSVRC2012_val_00020770.JPEG:n04162706 +ILSVRC2012_val_00020771.JPEG:n13054560 +ILSVRC2012_val_00020772.JPEG:n01632777 +ILSVRC2012_val_00020773.JPEG:n02093859 +ILSVRC2012_val_00020774.JPEG:n02025239 +ILSVRC2012_val_00020775.JPEG:n02797295 +ILSVRC2012_val_00020776.JPEG:n03179701 +ILSVRC2012_val_00020777.JPEG:n02980441 +ILSVRC2012_val_00020778.JPEG:n04596742 +ILSVRC2012_val_00020779.JPEG:n01980166 +ILSVRC2012_val_00020780.JPEG:n09835506 +ILSVRC2012_val_00020781.JPEG:n03445777 +ILSVRC2012_val_00020782.JPEG:n03110669 +ILSVRC2012_val_00020783.JPEG:n02094114 +ILSVRC2012_val_00020784.JPEG:n02086079 +ILSVRC2012_val_00020785.JPEG:n01443537 +ILSVRC2012_val_00020786.JPEG:n02110063 +ILSVRC2012_val_00020787.JPEG:n04355338 +ILSVRC2012_val_00020788.JPEG:n01560419 +ILSVRC2012_val_00020789.JPEG:n03355925 +ILSVRC2012_val_00020790.JPEG:n02119022 +ILSVRC2012_val_00020791.JPEG:n03447447 +ILSVRC2012_val_00020792.JPEG:n02219486 +ILSVRC2012_val_00020793.JPEG:n02113624 +ILSVRC2012_val_00020794.JPEG:n04523525 +ILSVRC2012_val_00020795.JPEG:n01983481 +ILSVRC2012_val_00020796.JPEG:n10565667 +ILSVRC2012_val_00020797.JPEG:n03803284 +ILSVRC2012_val_00020798.JPEG:n04367480 +ILSVRC2012_val_00020799.JPEG:n03400231 +ILSVRC2012_val_00020800.JPEG:n01980166 +ILSVRC2012_val_00020801.JPEG:n04596742 +ILSVRC2012_val_00020802.JPEG:n02417914 +ILSVRC2012_val_00020803.JPEG:n02514041 +ILSVRC2012_val_00020804.JPEG:n02033041 +ILSVRC2012_val_00020805.JPEG:n02094114 +ILSVRC2012_val_00020806.JPEG:n02134084 +ILSVRC2012_val_00020807.JPEG:n13040303 +ILSVRC2012_val_00020808.JPEG:n03763968 +ILSVRC2012_val_00020809.JPEG:n04111531 +ILSVRC2012_val_00020810.JPEG:n02090622 +ILSVRC2012_val_00020811.JPEG:n02486261 +ILSVRC2012_val_00020812.JPEG:n03452741 +ILSVRC2012_val_00020813.JPEG:n04458633 +ILSVRC2012_val_00020814.JPEG:n02094114 +ILSVRC2012_val_00020815.JPEG:n02097658 +ILSVRC2012_val_00020816.JPEG:n01978455 +ILSVRC2012_val_00020817.JPEG:n02988304 +ILSVRC2012_val_00020818.JPEG:n04229816 +ILSVRC2012_val_00020819.JPEG:n02892767 +ILSVRC2012_val_00020820.JPEG:n02804414 +ILSVRC2012_val_00020821.JPEG:n03240683 +ILSVRC2012_val_00020822.JPEG:n01443537 +ILSVRC2012_val_00020823.JPEG:n02088632 +ILSVRC2012_val_00020824.JPEG:n02172182 +ILSVRC2012_val_00020825.JPEG:n02786058 +ILSVRC2012_val_00020826.JPEG:n02701002 +ILSVRC2012_val_00020827.JPEG:n04515003 +ILSVRC2012_val_00020828.JPEG:n07693725 +ILSVRC2012_val_00020829.JPEG:n03594945 +ILSVRC2012_val_00020830.JPEG:n02100735 +ILSVRC2012_val_00020831.JPEG:n04204347 +ILSVRC2012_val_00020832.JPEG:n02093754 +ILSVRC2012_val_00020833.JPEG:n09428293 +ILSVRC2012_val_00020834.JPEG:n03958227 +ILSVRC2012_val_00020835.JPEG:n03042490 +ILSVRC2012_val_00020836.JPEG:n06359193 +ILSVRC2012_val_00020837.JPEG:n02102177 +ILSVRC2012_val_00020838.JPEG:n03445924 +ILSVRC2012_val_00020839.JPEG:n04141975 +ILSVRC2012_val_00020840.JPEG:n03690938 +ILSVRC2012_val_00020841.JPEG:n02108089 +ILSVRC2012_val_00020842.JPEG:n03075370 +ILSVRC2012_val_00020843.JPEG:n04517823 +ILSVRC2012_val_00020844.JPEG:n03208938 +ILSVRC2012_val_00020845.JPEG:n03958227 +ILSVRC2012_val_00020846.JPEG:n10148035 +ILSVRC2012_val_00020847.JPEG:n02444819 +ILSVRC2012_val_00020848.JPEG:n02092002 +ILSVRC2012_val_00020849.JPEG:n10565667 +ILSVRC2012_val_00020850.JPEG:n02437312 +ILSVRC2012_val_00020851.JPEG:n02280649 +ILSVRC2012_val_00020852.JPEG:n02909870 +ILSVRC2012_val_00020853.JPEG:n03977966 +ILSVRC2012_val_00020854.JPEG:n03110669 +ILSVRC2012_val_00020855.JPEG:n03777568 +ILSVRC2012_val_00020856.JPEG:n07930864 +ILSVRC2012_val_00020857.JPEG:n04560804 +ILSVRC2012_val_00020858.JPEG:n03888605 +ILSVRC2012_val_00020859.JPEG:n02120505 +ILSVRC2012_val_00020860.JPEG:n03014705 +ILSVRC2012_val_00020861.JPEG:n01744401 +ILSVRC2012_val_00020862.JPEG:n03770439 +ILSVRC2012_val_00020863.JPEG:n03393912 +ILSVRC2012_val_00020864.JPEG:n02727426 +ILSVRC2012_val_00020865.JPEG:n02093754 +ILSVRC2012_val_00020866.JPEG:n03379051 +ILSVRC2012_val_00020867.JPEG:n03788195 +ILSVRC2012_val_00020868.JPEG:n02099601 +ILSVRC2012_val_00020869.JPEG:n02481823 +ILSVRC2012_val_00020870.JPEG:n03291819 +ILSVRC2012_val_00020871.JPEG:n04127249 +ILSVRC2012_val_00020872.JPEG:n03803284 +ILSVRC2012_val_00020873.JPEG:n03794056 +ILSVRC2012_val_00020874.JPEG:n03478589 +ILSVRC2012_val_00020875.JPEG:n02009912 +ILSVRC2012_val_00020876.JPEG:n07579787 +ILSVRC2012_val_00020877.JPEG:n02951358 +ILSVRC2012_val_00020878.JPEG:n03297495 +ILSVRC2012_val_00020879.JPEG:n04517823 +ILSVRC2012_val_00020880.JPEG:n03794056 +ILSVRC2012_val_00020881.JPEG:n03854065 +ILSVRC2012_val_00020882.JPEG:n04325704 +ILSVRC2012_val_00020883.JPEG:n03902125 +ILSVRC2012_val_00020884.JPEG:n03207941 +ILSVRC2012_val_00020885.JPEG:n03160309 +ILSVRC2012_val_00020886.JPEG:n02727426 +ILSVRC2012_val_00020887.JPEG:n03498962 +ILSVRC2012_val_00020888.JPEG:n02056570 +ILSVRC2012_val_00020889.JPEG:n01530575 +ILSVRC2012_val_00020890.JPEG:n03290653 +ILSVRC2012_val_00020891.JPEG:n03133878 +ILSVRC2012_val_00020892.JPEG:n02099267 +ILSVRC2012_val_00020893.JPEG:n03742115 +ILSVRC2012_val_00020894.JPEG:n04273569 +ILSVRC2012_val_00020895.JPEG:n02977058 +ILSVRC2012_val_00020896.JPEG:n03724870 +ILSVRC2012_val_00020897.JPEG:n04597913 +ILSVRC2012_val_00020898.JPEG:n03763968 +ILSVRC2012_val_00020899.JPEG:n03201208 +ILSVRC2012_val_00020900.JPEG:n02672831 +ILSVRC2012_val_00020901.JPEG:n02096437 +ILSVRC2012_val_00020902.JPEG:n02916936 +ILSVRC2012_val_00020903.JPEG:n04398044 +ILSVRC2012_val_00020904.JPEG:n03110669 +ILSVRC2012_val_00020905.JPEG:n01580077 +ILSVRC2012_val_00020906.JPEG:n03775546 +ILSVRC2012_val_00020907.JPEG:n01665541 +ILSVRC2012_val_00020908.JPEG:n03109150 +ILSVRC2012_val_00020909.JPEG:n01843383 +ILSVRC2012_val_00020910.JPEG:n01751748 +ILSVRC2012_val_00020911.JPEG:n04487394 +ILSVRC2012_val_00020912.JPEG:n02804414 +ILSVRC2012_val_00020913.JPEG:n04200800 +ILSVRC2012_val_00020914.JPEG:n03661043 +ILSVRC2012_val_00020915.JPEG:n01806143 +ILSVRC2012_val_00020916.JPEG:n01641577 +ILSVRC2012_val_00020917.JPEG:n02325366 +ILSVRC2012_val_00020918.JPEG:n03976467 +ILSVRC2012_val_00020919.JPEG:n02917067 +ILSVRC2012_val_00020920.JPEG:n01819313 +ILSVRC2012_val_00020921.JPEG:n04465501 +ILSVRC2012_val_00020922.JPEG:n01955084 +ILSVRC2012_val_00020923.JPEG:n03063599 +ILSVRC2012_val_00020924.JPEG:n04099969 +ILSVRC2012_val_00020925.JPEG:n02793495 +ILSVRC2012_val_00020926.JPEG:n02086079 +ILSVRC2012_val_00020927.JPEG:n02859443 +ILSVRC2012_val_00020928.JPEG:n03690938 +ILSVRC2012_val_00020929.JPEG:n13052670 +ILSVRC2012_val_00020930.JPEG:n02088238 +ILSVRC2012_val_00020931.JPEG:n02699494 +ILSVRC2012_val_00020932.JPEG:n03721384 +ILSVRC2012_val_00020933.JPEG:n02006656 +ILSVRC2012_val_00020934.JPEG:n02415577 +ILSVRC2012_val_00020935.JPEG:n02981792 +ILSVRC2012_val_00020936.JPEG:n02492035 +ILSVRC2012_val_00020937.JPEG:n03379051 +ILSVRC2012_val_00020938.JPEG:n02280649 +ILSVRC2012_val_00020939.JPEG:n03095699 +ILSVRC2012_val_00020940.JPEG:n03720891 +ILSVRC2012_val_00020941.JPEG:n03459775 +ILSVRC2012_val_00020942.JPEG:n02422106 +ILSVRC2012_val_00020943.JPEG:n01644373 +ILSVRC2012_val_00020944.JPEG:n03347037 +ILSVRC2012_val_00020945.JPEG:n02834397 +ILSVRC2012_val_00020946.JPEG:n03218198 +ILSVRC2012_val_00020947.JPEG:n03627232 +ILSVRC2012_val_00020948.JPEG:n04557648 +ILSVRC2012_val_00020949.JPEG:n02423022 +ILSVRC2012_val_00020950.JPEG:n01784675 +ILSVRC2012_val_00020951.JPEG:n03425413 +ILSVRC2012_val_00020952.JPEG:n04579432 +ILSVRC2012_val_00020953.JPEG:n07875152 +ILSVRC2012_val_00020954.JPEG:n03461385 +ILSVRC2012_val_00020955.JPEG:n03404251 +ILSVRC2012_val_00020956.JPEG:n03658185 +ILSVRC2012_val_00020957.JPEG:n07720875 +ILSVRC2012_val_00020958.JPEG:n01943899 +ILSVRC2012_val_00020959.JPEG:n12620546 +ILSVRC2012_val_00020960.JPEG:n03967562 +ILSVRC2012_val_00020961.JPEG:n02102480 +ILSVRC2012_val_00020962.JPEG:n02500267 +ILSVRC2012_val_00020963.JPEG:n02087046 +ILSVRC2012_val_00020964.JPEG:n03595614 +ILSVRC2012_val_00020965.JPEG:n02100236 +ILSVRC2012_val_00020966.JPEG:n07892512 +ILSVRC2012_val_00020967.JPEG:n04505470 +ILSVRC2012_val_00020968.JPEG:n01986214 +ILSVRC2012_val_00020969.JPEG:n02447366 +ILSVRC2012_val_00020970.JPEG:n01978455 +ILSVRC2012_val_00020971.JPEG:n03942813 +ILSVRC2012_val_00020972.JPEG:n02917067 +ILSVRC2012_val_00020973.JPEG:n02125311 +ILSVRC2012_val_00020974.JPEG:n04275548 +ILSVRC2012_val_00020975.JPEG:n02077923 +ILSVRC2012_val_00020976.JPEG:n01829413 +ILSVRC2012_val_00020977.JPEG:n04557648 +ILSVRC2012_val_00020978.JPEG:n02483362 +ILSVRC2012_val_00020979.JPEG:n03250847 +ILSVRC2012_val_00020980.JPEG:n02454379 +ILSVRC2012_val_00020981.JPEG:n02793495 +ILSVRC2012_val_00020982.JPEG:n03891251 +ILSVRC2012_val_00020983.JPEG:n03938244 +ILSVRC2012_val_00020984.JPEG:n03467068 +ILSVRC2012_val_00020985.JPEG:n02226429 +ILSVRC2012_val_00020986.JPEG:n02106166 +ILSVRC2012_val_00020987.JPEG:n04465501 +ILSVRC2012_val_00020988.JPEG:n04423845 +ILSVRC2012_val_00020989.JPEG:n02108422 +ILSVRC2012_val_00020990.JPEG:n02776631 +ILSVRC2012_val_00020991.JPEG:n01773797 +ILSVRC2012_val_00020992.JPEG:n03250847 +ILSVRC2012_val_00020993.JPEG:n04606251 +ILSVRC2012_val_00020994.JPEG:n01664065 +ILSVRC2012_val_00020995.JPEG:n04127249 +ILSVRC2012_val_00020996.JPEG:n04254777 +ILSVRC2012_val_00020997.JPEG:n02483362 +ILSVRC2012_val_00020998.JPEG:n03041632 +ILSVRC2012_val_00020999.JPEG:n01729322 +ILSVRC2012_val_00021000.JPEG:n02093859 +ILSVRC2012_val_00021001.JPEG:n02977058 +ILSVRC2012_val_00021002.JPEG:n04252225 +ILSVRC2012_val_00021003.JPEG:n02116738 +ILSVRC2012_val_00021004.JPEG:n02950826 +ILSVRC2012_val_00021005.JPEG:n03494278 +ILSVRC2012_val_00021006.JPEG:n02130308 +ILSVRC2012_val_00021007.JPEG:n03786901 +ILSVRC2012_val_00021008.JPEG:n04462240 +ILSVRC2012_val_00021009.JPEG:n03617480 +ILSVRC2012_val_00021010.JPEG:n04418357 +ILSVRC2012_val_00021011.JPEG:n02879718 +ILSVRC2012_val_00021012.JPEG:n03018349 +ILSVRC2012_val_00021013.JPEG:n03272010 +ILSVRC2012_val_00021014.JPEG:n03379051 +ILSVRC2012_val_00021015.JPEG:n01614925 +ILSVRC2012_val_00021016.JPEG:n02102040 +ILSVRC2012_val_00021017.JPEG:n01630670 +ILSVRC2012_val_00021018.JPEG:n03627232 +ILSVRC2012_val_00021019.JPEG:n13037406 +ILSVRC2012_val_00021020.JPEG:n09288635 +ILSVRC2012_val_00021021.JPEG:n07584110 +ILSVRC2012_val_00021022.JPEG:n02102177 +ILSVRC2012_val_00021023.JPEG:n03347037 +ILSVRC2012_val_00021024.JPEG:n01632458 +ILSVRC2012_val_00021025.JPEG:n01768244 +ILSVRC2012_val_00021026.JPEG:n03584254 +ILSVRC2012_val_00021027.JPEG:n04346328 +ILSVRC2012_val_00021028.JPEG:n03599486 +ILSVRC2012_val_00021029.JPEG:n03109150 +ILSVRC2012_val_00021030.JPEG:n03692522 +ILSVRC2012_val_00021031.JPEG:n15075141 +ILSVRC2012_val_00021032.JPEG:n01742172 +ILSVRC2012_val_00021033.JPEG:n02841315 +ILSVRC2012_val_00021034.JPEG:n13040303 +ILSVRC2012_val_00021035.JPEG:n02117135 +ILSVRC2012_val_00021036.JPEG:n02107142 +ILSVRC2012_val_00021037.JPEG:n04266014 +ILSVRC2012_val_00021038.JPEG:n03724870 +ILSVRC2012_val_00021039.JPEG:n07248320 +ILSVRC2012_val_00021040.JPEG:n02704792 +ILSVRC2012_val_00021041.JPEG:n03871628 +ILSVRC2012_val_00021042.JPEG:n01990800 +ILSVRC2012_val_00021043.JPEG:n02129604 +ILSVRC2012_val_00021044.JPEG:n02119789 +ILSVRC2012_val_00021045.JPEG:n02125311 +ILSVRC2012_val_00021046.JPEG:n04606251 +ILSVRC2012_val_00021047.JPEG:n07768694 +ILSVRC2012_val_00021048.JPEG:n03187595 +ILSVRC2012_val_00021049.JPEG:n04376876 +ILSVRC2012_val_00021050.JPEG:n04483307 +ILSVRC2012_val_00021051.JPEG:n02110063 +ILSVRC2012_val_00021052.JPEG:n02107142 +ILSVRC2012_val_00021053.JPEG:n02782093 +ILSVRC2012_val_00021054.JPEG:n04487081 +ILSVRC2012_val_00021055.JPEG:n01675722 +ILSVRC2012_val_00021056.JPEG:n01608432 +ILSVRC2012_val_00021057.JPEG:n03297495 +ILSVRC2012_val_00021058.JPEG:n02098105 +ILSVRC2012_val_00021059.JPEG:n01950731 +ILSVRC2012_val_00021060.JPEG:n04238763 +ILSVRC2012_val_00021061.JPEG:n02105855 +ILSVRC2012_val_00021062.JPEG:n04552348 +ILSVRC2012_val_00021063.JPEG:n02051845 +ILSVRC2012_val_00021064.JPEG:n02128925 +ILSVRC2012_val_00021065.JPEG:n02877765 +ILSVRC2012_val_00021066.JPEG:n02128385 +ILSVRC2012_val_00021067.JPEG:n02877765 +ILSVRC2012_val_00021068.JPEG:n01872401 +ILSVRC2012_val_00021069.JPEG:n01682714 +ILSVRC2012_val_00021070.JPEG:n03481172 +ILSVRC2012_val_00021071.JPEG:n02509815 +ILSVRC2012_val_00021072.JPEG:n02236044 +ILSVRC2012_val_00021073.JPEG:n02280649 +ILSVRC2012_val_00021074.JPEG:n02488702 +ILSVRC2012_val_00021075.JPEG:n03492542 +ILSVRC2012_val_00021076.JPEG:n01749939 +ILSVRC2012_val_00021077.JPEG:n03207743 +ILSVRC2012_val_00021078.JPEG:n03179701 +ILSVRC2012_val_00021079.JPEG:n02100877 +ILSVRC2012_val_00021080.JPEG:n01981276 +ILSVRC2012_val_00021081.JPEG:n03710637 +ILSVRC2012_val_00021082.JPEG:n03223299 +ILSVRC2012_val_00021083.JPEG:n01630670 +ILSVRC2012_val_00021084.JPEG:n03877472 +ILSVRC2012_val_00021085.JPEG:n01560419 +ILSVRC2012_val_00021086.JPEG:n02259212 +ILSVRC2012_val_00021087.JPEG:n04127249 +ILSVRC2012_val_00021088.JPEG:n03796401 +ILSVRC2012_val_00021089.JPEG:n04486054 +ILSVRC2012_val_00021090.JPEG:n01807496 +ILSVRC2012_val_00021091.JPEG:n03492542 +ILSVRC2012_val_00021092.JPEG:n01694178 +ILSVRC2012_val_00021093.JPEG:n01740131 +ILSVRC2012_val_00021094.JPEG:n01985128 +ILSVRC2012_val_00021095.JPEG:n03637318 +ILSVRC2012_val_00021096.JPEG:n03584254 +ILSVRC2012_val_00021097.JPEG:n07717556 +ILSVRC2012_val_00021098.JPEG:n07753592 +ILSVRC2012_val_00021099.JPEG:n02791124 +ILSVRC2012_val_00021100.JPEG:n03786901 +ILSVRC2012_val_00021101.JPEG:n02965783 +ILSVRC2012_val_00021102.JPEG:n03733131 +ILSVRC2012_val_00021103.JPEG:n04458633 +ILSVRC2012_val_00021104.JPEG:n01614925 +ILSVRC2012_val_00021105.JPEG:n04435653 +ILSVRC2012_val_00021106.JPEG:n03534580 +ILSVRC2012_val_00021107.JPEG:n04532106 +ILSVRC2012_val_00021108.JPEG:n02276258 +ILSVRC2012_val_00021109.JPEG:n01697457 +ILSVRC2012_val_00021110.JPEG:n03187595 +ILSVRC2012_val_00021111.JPEG:n04590129 +ILSVRC2012_val_00021112.JPEG:n04004767 +ILSVRC2012_val_00021113.JPEG:n03877472 +ILSVRC2012_val_00021114.JPEG:n07248320 +ILSVRC2012_val_00021115.JPEG:n03207743 +ILSVRC2012_val_00021116.JPEG:n02892767 +ILSVRC2012_val_00021117.JPEG:n03976467 +ILSVRC2012_val_00021118.JPEG:n03133878 +ILSVRC2012_val_00021119.JPEG:n03594734 +ILSVRC2012_val_00021120.JPEG:n01877812 +ILSVRC2012_val_00021121.JPEG:n03785016 +ILSVRC2012_val_00021122.JPEG:n04613696 +ILSVRC2012_val_00021123.JPEG:n03534580 +ILSVRC2012_val_00021124.JPEG:n02013706 +ILSVRC2012_val_00021125.JPEG:n01985128 +ILSVRC2012_val_00021126.JPEG:n02110806 +ILSVRC2012_val_00021127.JPEG:n02441942 +ILSVRC2012_val_00021128.JPEG:n04554684 +ILSVRC2012_val_00021129.JPEG:n03916031 +ILSVRC2012_val_00021130.JPEG:n01748264 +ILSVRC2012_val_00021131.JPEG:n04204347 +ILSVRC2012_val_00021132.JPEG:n03450230 +ILSVRC2012_val_00021133.JPEG:n01622779 +ILSVRC2012_val_00021134.JPEG:n02799071 +ILSVRC2012_val_00021135.JPEG:n02017213 +ILSVRC2012_val_00021136.JPEG:n03201208 +ILSVRC2012_val_00021137.JPEG:n02487347 +ILSVRC2012_val_00021138.JPEG:n02497673 +ILSVRC2012_val_00021139.JPEG:n01795545 +ILSVRC2012_val_00021140.JPEG:n02487347 +ILSVRC2012_val_00021141.JPEG:n04487081 +ILSVRC2012_val_00021142.JPEG:n03710637 +ILSVRC2012_val_00021143.JPEG:n04026417 +ILSVRC2012_val_00021144.JPEG:n07747607 +ILSVRC2012_val_00021145.JPEG:n02092002 +ILSVRC2012_val_00021146.JPEG:n02701002 +ILSVRC2012_val_00021147.JPEG:n02492660 +ILSVRC2012_val_00021148.JPEG:n03995372 +ILSVRC2012_val_00021149.JPEG:n02415577 +ILSVRC2012_val_00021150.JPEG:n02091831 +ILSVRC2012_val_00021151.JPEG:n02423022 +ILSVRC2012_val_00021152.JPEG:n02165456 +ILSVRC2012_val_00021153.JPEG:n03666591 +ILSVRC2012_val_00021154.JPEG:n04604644 +ILSVRC2012_val_00021155.JPEG:n02107142 +ILSVRC2012_val_00021156.JPEG:n02951358 +ILSVRC2012_val_00021157.JPEG:n02219486 +ILSVRC2012_val_00021158.JPEG:n04542943 +ILSVRC2012_val_00021159.JPEG:n03777568 +ILSVRC2012_val_00021160.JPEG:n03787032 +ILSVRC2012_val_00021161.JPEG:n04332243 +ILSVRC2012_val_00021162.JPEG:n02927161 +ILSVRC2012_val_00021163.JPEG:n09288635 +ILSVRC2012_val_00021164.JPEG:n01704323 +ILSVRC2012_val_00021165.JPEG:n02091244 +ILSVRC2012_val_00021166.JPEG:n02894605 +ILSVRC2012_val_00021167.JPEG:n04554684 +ILSVRC2012_val_00021168.JPEG:n02085936 +ILSVRC2012_val_00021169.JPEG:n03014705 +ILSVRC2012_val_00021170.JPEG:n01871265 +ILSVRC2012_val_00021171.JPEG:n02113799 +ILSVRC2012_val_00021172.JPEG:n02107683 +ILSVRC2012_val_00021173.JPEG:n03347037 +ILSVRC2012_val_00021174.JPEG:n04296562 +ILSVRC2012_val_00021175.JPEG:n09256479 +ILSVRC2012_val_00021176.JPEG:n02110341 +ILSVRC2012_val_00021177.JPEG:n06874185 +ILSVRC2012_val_00021178.JPEG:n03967562 +ILSVRC2012_val_00021179.JPEG:n02708093 +ILSVRC2012_val_00021180.JPEG:n04344873 +ILSVRC2012_val_00021181.JPEG:n02437616 +ILSVRC2012_val_00021182.JPEG:n04523525 +ILSVRC2012_val_00021183.JPEG:n02099712 +ILSVRC2012_val_00021184.JPEG:n04404412 +ILSVRC2012_val_00021185.JPEG:n04277352 +ILSVRC2012_val_00021186.JPEG:n02948072 +ILSVRC2012_val_00021187.JPEG:n04111531 +ILSVRC2012_val_00021188.JPEG:n03452741 +ILSVRC2012_val_00021189.JPEG:n02966193 +ILSVRC2012_val_00021190.JPEG:n03452741 +ILSVRC2012_val_00021191.JPEG:n02100735 +ILSVRC2012_val_00021192.JPEG:n04597913 +ILSVRC2012_val_00021193.JPEG:n07747607 +ILSVRC2012_val_00021194.JPEG:n03764736 +ILSVRC2012_val_00021195.JPEG:n02123159 +ILSVRC2012_val_00021196.JPEG:n02107574 +ILSVRC2012_val_00021197.JPEG:n01729977 +ILSVRC2012_val_00021198.JPEG:n03976467 +ILSVRC2012_val_00021199.JPEG:n03788195 +ILSVRC2012_val_00021200.JPEG:n07717556 +ILSVRC2012_val_00021201.JPEG:n15075141 +ILSVRC2012_val_00021202.JPEG:n04596742 +ILSVRC2012_val_00021203.JPEG:n01729977 +ILSVRC2012_val_00021204.JPEG:n03042490 +ILSVRC2012_val_00021205.JPEG:n02102040 +ILSVRC2012_val_00021206.JPEG:n02093991 +ILSVRC2012_val_00021207.JPEG:n12144580 +ILSVRC2012_val_00021208.JPEG:n02107908 +ILSVRC2012_val_00021209.JPEG:n04612504 +ILSVRC2012_val_00021210.JPEG:n02981792 +ILSVRC2012_val_00021211.JPEG:n01644900 +ILSVRC2012_val_00021212.JPEG:n02128385 +ILSVRC2012_val_00021213.JPEG:n02128925 +ILSVRC2012_val_00021214.JPEG:n02110806 +ILSVRC2012_val_00021215.JPEG:n01748264 +ILSVRC2012_val_00021216.JPEG:n02777292 +ILSVRC2012_val_00021217.JPEG:n04209239 +ILSVRC2012_val_00021218.JPEG:n02112350 +ILSVRC2012_val_00021219.JPEG:n02361337 +ILSVRC2012_val_00021220.JPEG:n04141327 +ILSVRC2012_val_00021221.JPEG:n02229544 +ILSVRC2012_val_00021222.JPEG:n02281406 +ILSVRC2012_val_00021223.JPEG:n03895866 +ILSVRC2012_val_00021224.JPEG:n02108915 +ILSVRC2012_val_00021225.JPEG:n12768682 +ILSVRC2012_val_00021226.JPEG:n02106030 +ILSVRC2012_val_00021227.JPEG:n03218198 +ILSVRC2012_val_00021228.JPEG:n04133789 +ILSVRC2012_val_00021229.JPEG:n02093428 +ILSVRC2012_val_00021230.JPEG:n03461385 +ILSVRC2012_val_00021231.JPEG:n02119789 +ILSVRC2012_val_00021232.JPEG:n03444034 +ILSVRC2012_val_00021233.JPEG:n02877765 +ILSVRC2012_val_00021234.JPEG:n03724870 +ILSVRC2012_val_00021235.JPEG:n03773504 +ILSVRC2012_val_00021236.JPEG:n01698640 +ILSVRC2012_val_00021237.JPEG:n02504013 +ILSVRC2012_val_00021238.JPEG:n02231487 +ILSVRC2012_val_00021239.JPEG:n01558993 +ILSVRC2012_val_00021240.JPEG:n06785654 +ILSVRC2012_val_00021241.JPEG:n01981276 +ILSVRC2012_val_00021242.JPEG:n02389026 +ILSVRC2012_val_00021243.JPEG:n04277352 +ILSVRC2012_val_00021244.JPEG:n02687172 +ILSVRC2012_val_00021245.JPEG:n03291819 +ILSVRC2012_val_00021246.JPEG:n04447861 +ILSVRC2012_val_00021247.JPEG:n04310018 +ILSVRC2012_val_00021248.JPEG:n02486410 +ILSVRC2012_val_00021249.JPEG:n02105855 +ILSVRC2012_val_00021250.JPEG:n02948072 +ILSVRC2012_val_00021251.JPEG:n03785016 +ILSVRC2012_val_00021252.JPEG:n02002724 +ILSVRC2012_val_00021253.JPEG:n03417042 +ILSVRC2012_val_00021254.JPEG:n03188531 +ILSVRC2012_val_00021255.JPEG:n02259212 +ILSVRC2012_val_00021256.JPEG:n02776631 +ILSVRC2012_val_00021257.JPEG:n02951585 +ILSVRC2012_val_00021258.JPEG:n03337140 +ILSVRC2012_val_00021259.JPEG:n01751748 +ILSVRC2012_val_00021260.JPEG:n02879718 +ILSVRC2012_val_00021261.JPEG:n04277352 +ILSVRC2012_val_00021262.JPEG:n12057211 +ILSVRC2012_val_00021263.JPEG:n02951585 +ILSVRC2012_val_00021264.JPEG:n03967562 +ILSVRC2012_val_00021265.JPEG:n07714571 +ILSVRC2012_val_00021266.JPEG:n02085620 +ILSVRC2012_val_00021267.JPEG:n02510455 +ILSVRC2012_val_00021268.JPEG:n02869837 +ILSVRC2012_val_00021269.JPEG:n01980166 +ILSVRC2012_val_00021270.JPEG:n01756291 +ILSVRC2012_val_00021271.JPEG:n03792972 +ILSVRC2012_val_00021272.JPEG:n02112137 +ILSVRC2012_val_00021273.JPEG:n03680355 +ILSVRC2012_val_00021274.JPEG:n03841143 +ILSVRC2012_val_00021275.JPEG:n07565083 +ILSVRC2012_val_00021276.JPEG:n07693725 +ILSVRC2012_val_00021277.JPEG:n07715103 +ILSVRC2012_val_00021278.JPEG:n01820546 +ILSVRC2012_val_00021279.JPEG:n01873310 +ILSVRC2012_val_00021280.JPEG:n03777568 +ILSVRC2012_val_00021281.JPEG:n01833805 +ILSVRC2012_val_00021282.JPEG:n02676566 +ILSVRC2012_val_00021283.JPEG:n03447721 +ILSVRC2012_val_00021284.JPEG:n02500267 +ILSVRC2012_val_00021285.JPEG:n03602883 +ILSVRC2012_val_00021286.JPEG:n04239074 +ILSVRC2012_val_00021287.JPEG:n04118538 +ILSVRC2012_val_00021288.JPEG:n04536866 +ILSVRC2012_val_00021289.JPEG:n04548362 +ILSVRC2012_val_00021290.JPEG:n02776631 +ILSVRC2012_val_00021291.JPEG:n01667778 +ILSVRC2012_val_00021292.JPEG:n03825788 +ILSVRC2012_val_00021293.JPEG:n03891332 +ILSVRC2012_val_00021294.JPEG:n04258138 +ILSVRC2012_val_00021295.JPEG:n04542943 +ILSVRC2012_val_00021296.JPEG:n02099849 +ILSVRC2012_val_00021297.JPEG:n03041632 +ILSVRC2012_val_00021298.JPEG:n04179913 +ILSVRC2012_val_00021299.JPEG:n01632458 +ILSVRC2012_val_00021300.JPEG:n01537544 +ILSVRC2012_val_00021301.JPEG:n02930766 +ILSVRC2012_val_00021302.JPEG:n03814639 +ILSVRC2012_val_00021303.JPEG:n02643566 +ILSVRC2012_val_00021304.JPEG:n03498962 +ILSVRC2012_val_00021305.JPEG:n01798484 +ILSVRC2012_val_00021306.JPEG:n02692877 +ILSVRC2012_val_00021307.JPEG:n03134739 +ILSVRC2012_val_00021308.JPEG:n03314780 +ILSVRC2012_val_00021309.JPEG:n02870880 +ILSVRC2012_val_00021310.JPEG:n07768694 +ILSVRC2012_val_00021311.JPEG:n04141076 +ILSVRC2012_val_00021312.JPEG:n03786901 +ILSVRC2012_val_00021313.JPEG:n03314780 +ILSVRC2012_val_00021314.JPEG:n02172182 +ILSVRC2012_val_00021315.JPEG:n02092339 +ILSVRC2012_val_00021316.JPEG:n03259280 +ILSVRC2012_val_00021317.JPEG:n07880968 +ILSVRC2012_val_00021318.JPEG:n02115641 +ILSVRC2012_val_00021319.JPEG:n01990800 +ILSVRC2012_val_00021320.JPEG:n12768682 +ILSVRC2012_val_00021321.JPEG:n07930864 +ILSVRC2012_val_00021322.JPEG:n03527444 +ILSVRC2012_val_00021323.JPEG:n02091244 +ILSVRC2012_val_00021324.JPEG:n03769881 +ILSVRC2012_val_00021325.JPEG:n01494475 +ILSVRC2012_val_00021326.JPEG:n03249569 +ILSVRC2012_val_00021327.JPEG:n02395406 +ILSVRC2012_val_00021328.JPEG:n03776460 +ILSVRC2012_val_00021329.JPEG:n12985857 +ILSVRC2012_val_00021330.JPEG:n02056570 +ILSVRC2012_val_00021331.JPEG:n02486410 +ILSVRC2012_val_00021332.JPEG:n01737021 +ILSVRC2012_val_00021333.JPEG:n02488702 +ILSVRC2012_val_00021334.JPEG:n01978455 +ILSVRC2012_val_00021335.JPEG:n01622779 +ILSVRC2012_val_00021336.JPEG:n02510455 +ILSVRC2012_val_00021337.JPEG:n01776313 +ILSVRC2012_val_00021338.JPEG:n07831146 +ILSVRC2012_val_00021339.JPEG:n02018207 +ILSVRC2012_val_00021340.JPEG:n02808304 +ILSVRC2012_val_00021341.JPEG:n01855032 +ILSVRC2012_val_00021342.JPEG:n03803284 +ILSVRC2012_val_00021343.JPEG:n02514041 +ILSVRC2012_val_00021344.JPEG:n02099849 +ILSVRC2012_val_00021345.JPEG:n01806143 +ILSVRC2012_val_00021346.JPEG:n03837869 +ILSVRC2012_val_00021347.JPEG:n03902125 +ILSVRC2012_val_00021348.JPEG:n02895154 +ILSVRC2012_val_00021349.JPEG:n04208210 +ILSVRC2012_val_00021350.JPEG:n02107142 +ILSVRC2012_val_00021351.JPEG:n01855672 +ILSVRC2012_val_00021352.JPEG:n02480495 +ILSVRC2012_val_00021353.JPEG:n04065272 +ILSVRC2012_val_00021354.JPEG:n03761084 +ILSVRC2012_val_00021355.JPEG:n02100236 +ILSVRC2012_val_00021356.JPEG:n02111277 +ILSVRC2012_val_00021357.JPEG:n02089867 +ILSVRC2012_val_00021358.JPEG:n04552348 +ILSVRC2012_val_00021359.JPEG:n02791124 +ILSVRC2012_val_00021360.JPEG:n02101556 +ILSVRC2012_val_00021361.JPEG:n02480855 +ILSVRC2012_val_00021362.JPEG:n02097658 +ILSVRC2012_val_00021363.JPEG:n03180011 +ILSVRC2012_val_00021364.JPEG:n03899768 +ILSVRC2012_val_00021365.JPEG:n02087394 +ILSVRC2012_val_00021366.JPEG:n02236044 +ILSVRC2012_val_00021367.JPEG:n02794156 +ILSVRC2012_val_00021368.JPEG:n04550184 +ILSVRC2012_val_00021369.JPEG:n02099849 +ILSVRC2012_val_00021370.JPEG:n02111129 +ILSVRC2012_val_00021371.JPEG:n03976657 +ILSVRC2012_val_00021372.JPEG:n01847000 +ILSVRC2012_val_00021373.JPEG:n04465501 +ILSVRC2012_val_00021374.JPEG:n03063599 +ILSVRC2012_val_00021375.JPEG:n03733131 +ILSVRC2012_val_00021376.JPEG:n09332890 +ILSVRC2012_val_00021377.JPEG:n02892767 +ILSVRC2012_val_00021378.JPEG:n01978455 +ILSVRC2012_val_00021379.JPEG:n02111129 +ILSVRC2012_val_00021380.JPEG:n03832673 +ILSVRC2012_val_00021381.JPEG:n04141327 +ILSVRC2012_val_00021382.JPEG:n02276258 +ILSVRC2012_val_00021383.JPEG:n03786901 +ILSVRC2012_val_00021384.JPEG:n02672831 +ILSVRC2012_val_00021385.JPEG:n01978455 +ILSVRC2012_val_00021386.JPEG:n02807133 +ILSVRC2012_val_00021387.JPEG:n03290653 +ILSVRC2012_val_00021388.JPEG:n03297495 +ILSVRC2012_val_00021389.JPEG:n02112350 +ILSVRC2012_val_00021390.JPEG:n02894605 +ILSVRC2012_val_00021391.JPEG:n03763968 +ILSVRC2012_val_00021392.JPEG:n02776631 +ILSVRC2012_val_00021393.JPEG:n04606251 +ILSVRC2012_val_00021394.JPEG:n03498962 +ILSVRC2012_val_00021395.JPEG:n04443257 +ILSVRC2012_val_00021396.JPEG:n04355933 +ILSVRC2012_val_00021397.JPEG:n02727426 +ILSVRC2012_val_00021398.JPEG:n12057211 +ILSVRC2012_val_00021399.JPEG:n04376876 +ILSVRC2012_val_00021400.JPEG:n02403003 +ILSVRC2012_val_00021401.JPEG:n03495258 +ILSVRC2012_val_00021402.JPEG:n04584207 +ILSVRC2012_val_00021403.JPEG:n04462240 +ILSVRC2012_val_00021404.JPEG:n01729322 +ILSVRC2012_val_00021405.JPEG:n03207941 +ILSVRC2012_val_00021406.JPEG:n02483708 +ILSVRC2012_val_00021407.JPEG:n10565667 +ILSVRC2012_val_00021408.JPEG:n03866082 +ILSVRC2012_val_00021409.JPEG:n04019541 +ILSVRC2012_val_00021410.JPEG:n04154565 +ILSVRC2012_val_00021411.JPEG:n13052670 +ILSVRC2012_val_00021412.JPEG:n02992211 +ILSVRC2012_val_00021413.JPEG:n03642806 +ILSVRC2012_val_00021414.JPEG:n03372029 +ILSVRC2012_val_00021415.JPEG:n03832673 +ILSVRC2012_val_00021416.JPEG:n03617480 +ILSVRC2012_val_00021417.JPEG:n01797886 +ILSVRC2012_val_00021418.JPEG:n04591157 +ILSVRC2012_val_00021419.JPEG:n04443257 +ILSVRC2012_val_00021420.JPEG:n03045698 +ILSVRC2012_val_00021421.JPEG:n03207941 +ILSVRC2012_val_00021422.JPEG:n04081281 +ILSVRC2012_val_00021423.JPEG:n02165105 +ILSVRC2012_val_00021424.JPEG:n02105412 +ILSVRC2012_val_00021425.JPEG:n02980441 +ILSVRC2012_val_00021426.JPEG:n02097658 +ILSVRC2012_val_00021427.JPEG:n02823750 +ILSVRC2012_val_00021428.JPEG:n02397096 +ILSVRC2012_val_00021429.JPEG:n03662601 +ILSVRC2012_val_00021430.JPEG:n01514859 +ILSVRC2012_val_00021431.JPEG:n03759954 +ILSVRC2012_val_00021432.JPEG:n02859443 +ILSVRC2012_val_00021433.JPEG:n02011460 +ILSVRC2012_val_00021434.JPEG:n03467068 +ILSVRC2012_val_00021435.JPEG:n04458633 +ILSVRC2012_val_00021436.JPEG:n02111277 +ILSVRC2012_val_00021437.JPEG:n01751748 +ILSVRC2012_val_00021438.JPEG:n03127747 +ILSVRC2012_val_00021439.JPEG:n03838899 +ILSVRC2012_val_00021440.JPEG:n07715103 +ILSVRC2012_val_00021441.JPEG:n02894605 +ILSVRC2012_val_00021442.JPEG:n02793495 +ILSVRC2012_val_00021443.JPEG:n07248320 +ILSVRC2012_val_00021444.JPEG:n03995372 +ILSVRC2012_val_00021445.JPEG:n02094258 +ILSVRC2012_val_00021446.JPEG:n03937543 +ILSVRC2012_val_00021447.JPEG:n03642806 +ILSVRC2012_val_00021448.JPEG:n02607072 +ILSVRC2012_val_00021449.JPEG:n03483316 +ILSVRC2012_val_00021450.JPEG:n02090622 +ILSVRC2012_val_00021451.JPEG:n04525305 +ILSVRC2012_val_00021452.JPEG:n02085936 +ILSVRC2012_val_00021453.JPEG:n03920288 +ILSVRC2012_val_00021454.JPEG:n03063599 +ILSVRC2012_val_00021455.JPEG:n01843065 +ILSVRC2012_val_00021456.JPEG:n02099267 +ILSVRC2012_val_00021457.JPEG:n01739381 +ILSVRC2012_val_00021458.JPEG:n03793489 +ILSVRC2012_val_00021459.JPEG:n02018207 +ILSVRC2012_val_00021460.JPEG:n03775071 +ILSVRC2012_val_00021461.JPEG:n01496331 +ILSVRC2012_val_00021462.JPEG:n06785654 +ILSVRC2012_val_00021463.JPEG:n03935335 +ILSVRC2012_val_00021464.JPEG:n03887697 +ILSVRC2012_val_00021465.JPEG:n07747607 +ILSVRC2012_val_00021466.JPEG:n03773504 +ILSVRC2012_val_00021467.JPEG:n07860988 +ILSVRC2012_val_00021468.JPEG:n04456115 +ILSVRC2012_val_00021469.JPEG:n02492035 +ILSVRC2012_val_00021470.JPEG:n03874293 +ILSVRC2012_val_00021471.JPEG:n04275548 +ILSVRC2012_val_00021472.JPEG:n03063689 +ILSVRC2012_val_00021473.JPEG:n02101006 +ILSVRC2012_val_00021474.JPEG:n01807496 +ILSVRC2012_val_00021475.JPEG:n02113978 +ILSVRC2012_val_00021476.JPEG:n02655020 +ILSVRC2012_val_00021477.JPEG:n02488702 +ILSVRC2012_val_00021478.JPEG:n02174001 +ILSVRC2012_val_00021479.JPEG:n04004767 +ILSVRC2012_val_00021480.JPEG:n04579432 +ILSVRC2012_val_00021481.JPEG:n04141975 +ILSVRC2012_val_00021482.JPEG:n03584254 +ILSVRC2012_val_00021483.JPEG:n02112706 +ILSVRC2012_val_00021484.JPEG:n03127747 +ILSVRC2012_val_00021485.JPEG:n02097047 +ILSVRC2012_val_00021486.JPEG:n04458633 +ILSVRC2012_val_00021487.JPEG:n02814533 +ILSVRC2012_val_00021488.JPEG:n02510455 +ILSVRC2012_val_00021489.JPEG:n02106166 +ILSVRC2012_val_00021490.JPEG:n02492035 +ILSVRC2012_val_00021491.JPEG:n13054560 +ILSVRC2012_val_00021492.JPEG:n04090263 +ILSVRC2012_val_00021493.JPEG:n02110341 +ILSVRC2012_val_00021494.JPEG:n02965783 +ILSVRC2012_val_00021495.JPEG:n04235860 +ILSVRC2012_val_00021496.JPEG:n01735189 +ILSVRC2012_val_00021497.JPEG:n01698640 +ILSVRC2012_val_00021498.JPEG:n07697313 +ILSVRC2012_val_00021499.JPEG:n02276258 +ILSVRC2012_val_00021500.JPEG:n03868242 +ILSVRC2012_val_00021501.JPEG:n02321529 +ILSVRC2012_val_00021502.JPEG:n03042490 +ILSVRC2012_val_00021503.JPEG:n04418357 +ILSVRC2012_val_00021504.JPEG:n03814906 +ILSVRC2012_val_00021505.JPEG:n02607072 +ILSVRC2012_val_00021506.JPEG:n04517823 +ILSVRC2012_val_00021507.JPEG:n03496892 +ILSVRC2012_val_00021508.JPEG:n07717556 +ILSVRC2012_val_00021509.JPEG:n02051845 +ILSVRC2012_val_00021510.JPEG:n03291819 +ILSVRC2012_val_00021511.JPEG:n09399592 +ILSVRC2012_val_00021512.JPEG:n02791124 +ILSVRC2012_val_00021513.JPEG:n02259212 +ILSVRC2012_val_00021514.JPEG:n02233338 +ILSVRC2012_val_00021515.JPEG:n07802026 +ILSVRC2012_val_00021516.JPEG:n03047690 +ILSVRC2012_val_00021517.JPEG:n03995372 +ILSVRC2012_val_00021518.JPEG:n03530642 +ILSVRC2012_val_00021519.JPEG:n02966687 +ILSVRC2012_val_00021520.JPEG:n02492035 +ILSVRC2012_val_00021521.JPEG:n02229544 +ILSVRC2012_val_00021522.JPEG:n01689811 +ILSVRC2012_val_00021523.JPEG:n01532829 +ILSVRC2012_val_00021524.JPEG:n03733805 +ILSVRC2012_val_00021525.JPEG:n01776313 +ILSVRC2012_val_00021526.JPEG:n02112137 +ILSVRC2012_val_00021527.JPEG:n04200800 +ILSVRC2012_val_00021528.JPEG:n07747607 +ILSVRC2012_val_00021529.JPEG:n03016953 +ILSVRC2012_val_00021530.JPEG:n03729826 +ILSVRC2012_val_00021531.JPEG:n07734744 +ILSVRC2012_val_00021532.JPEG:n02088094 +ILSVRC2012_val_00021533.JPEG:n04542943 +ILSVRC2012_val_00021534.JPEG:n02667093 +ILSVRC2012_val_00021535.JPEG:n03400231 +ILSVRC2012_val_00021536.JPEG:n04355933 +ILSVRC2012_val_00021537.JPEG:n03544143 +ILSVRC2012_val_00021538.JPEG:n02128385 +ILSVRC2012_val_00021539.JPEG:n04356056 +ILSVRC2012_val_00021540.JPEG:n02112018 +ILSVRC2012_val_00021541.JPEG:n02859443 +ILSVRC2012_val_00021542.JPEG:n02128925 +ILSVRC2012_val_00021543.JPEG:n02091032 +ILSVRC2012_val_00021544.JPEG:n04004767 +ILSVRC2012_val_00021545.JPEG:n02096051 +ILSVRC2012_val_00021546.JPEG:n02113712 +ILSVRC2012_val_00021547.JPEG:n02927161 +ILSVRC2012_val_00021548.JPEG:n03476991 +ILSVRC2012_val_00021549.JPEG:n02423022 +ILSVRC2012_val_00021550.JPEG:n12144580 +ILSVRC2012_val_00021551.JPEG:n04548280 +ILSVRC2012_val_00021552.JPEG:n03724870 +ILSVRC2012_val_00021553.JPEG:n04335435 +ILSVRC2012_val_00021554.JPEG:n07583066 +ILSVRC2012_val_00021555.JPEG:n02871525 +ILSVRC2012_val_00021556.JPEG:n03272010 +ILSVRC2012_val_00021557.JPEG:n02484975 +ILSVRC2012_val_00021558.JPEG:n02786058 +ILSVRC2012_val_00021559.JPEG:n09472597 +ILSVRC2012_val_00021560.JPEG:n04209133 +ILSVRC2012_val_00021561.JPEG:n03717622 +ILSVRC2012_val_00021562.JPEG:n03598930 +ILSVRC2012_val_00021563.JPEG:n02417914 +ILSVRC2012_val_00021564.JPEG:n01824575 +ILSVRC2012_val_00021565.JPEG:n04204238 +ILSVRC2012_val_00021566.JPEG:n02999410 +ILSVRC2012_val_00021567.JPEG:n04467665 +ILSVRC2012_val_00021568.JPEG:n04239074 +ILSVRC2012_val_00021569.JPEG:n03444034 +ILSVRC2012_val_00021570.JPEG:n04263257 +ILSVRC2012_val_00021571.JPEG:n03903868 +ILSVRC2012_val_00021572.JPEG:n02492035 +ILSVRC2012_val_00021573.JPEG:n02110627 +ILSVRC2012_val_00021574.JPEG:n02007558 +ILSVRC2012_val_00021575.JPEG:n02090379 +ILSVRC2012_val_00021576.JPEG:n03995372 +ILSVRC2012_val_00021577.JPEG:n04325704 +ILSVRC2012_val_00021578.JPEG:n04277352 +ILSVRC2012_val_00021579.JPEG:n02494079 +ILSVRC2012_val_00021580.JPEG:n02321529 +ILSVRC2012_val_00021581.JPEG:n12144580 +ILSVRC2012_val_00021582.JPEG:n01687978 +ILSVRC2012_val_00021583.JPEG:n03095699 +ILSVRC2012_val_00021584.JPEG:n02074367 +ILSVRC2012_val_00021585.JPEG:n02128925 +ILSVRC2012_val_00021586.JPEG:n02363005 +ILSVRC2012_val_00021587.JPEG:n02346627 +ILSVRC2012_val_00021588.JPEG:n04579145 +ILSVRC2012_val_00021589.JPEG:n03133878 +ILSVRC2012_val_00021590.JPEG:n02776631 +ILSVRC2012_val_00021591.JPEG:n03787032 +ILSVRC2012_val_00021592.JPEG:n03127747 +ILSVRC2012_val_00021593.JPEG:n01749939 +ILSVRC2012_val_00021594.JPEG:n01860187 +ILSVRC2012_val_00021595.JPEG:n04317175 +ILSVRC2012_val_00021596.JPEG:n12768682 +ILSVRC2012_val_00021597.JPEG:n02219486 +ILSVRC2012_val_00021598.JPEG:n03630383 +ILSVRC2012_val_00021599.JPEG:n02097130 +ILSVRC2012_val_00021600.JPEG:n02859443 +ILSVRC2012_val_00021601.JPEG:n03529860 +ILSVRC2012_val_00021602.JPEG:n02229544 +ILSVRC2012_val_00021603.JPEG:n03272562 +ILSVRC2012_val_00021604.JPEG:n04116512 +ILSVRC2012_val_00021605.JPEG:n01685808 +ILSVRC2012_val_00021606.JPEG:n03902125 +ILSVRC2012_val_00021607.JPEG:n02174001 +ILSVRC2012_val_00021608.JPEG:n02112706 +ILSVRC2012_val_00021609.JPEG:n02840245 +ILSVRC2012_val_00021610.JPEG:n04141975 +ILSVRC2012_val_00021611.JPEG:n01641577 +ILSVRC2012_val_00021612.JPEG:n02326432 +ILSVRC2012_val_00021613.JPEG:n07749582 +ILSVRC2012_val_00021614.JPEG:n02797295 +ILSVRC2012_val_00021615.JPEG:n04596742 +ILSVRC2012_val_00021616.JPEG:n02974003 +ILSVRC2012_val_00021617.JPEG:n01729977 +ILSVRC2012_val_00021618.JPEG:n02504013 +ILSVRC2012_val_00021619.JPEG:n02843684 +ILSVRC2012_val_00021620.JPEG:n03825788 +ILSVRC2012_val_00021621.JPEG:n04517823 +ILSVRC2012_val_00021622.JPEG:n03216828 +ILSVRC2012_val_00021623.JPEG:n04346328 +ILSVRC2012_val_00021624.JPEG:n02408429 +ILSVRC2012_val_00021625.JPEG:n01797886 +ILSVRC2012_val_00021626.JPEG:n02493509 +ILSVRC2012_val_00021627.JPEG:n02799071 +ILSVRC2012_val_00021628.JPEG:n04204347 +ILSVRC2012_val_00021629.JPEG:n07716906 +ILSVRC2012_val_00021630.JPEG:n06874185 +ILSVRC2012_val_00021631.JPEG:n02093647 +ILSVRC2012_val_00021632.JPEG:n02111889 +ILSVRC2012_val_00021633.JPEG:n04254777 +ILSVRC2012_val_00021634.JPEG:n02966687 +ILSVRC2012_val_00021635.JPEG:n03938244 +ILSVRC2012_val_00021636.JPEG:n02321529 +ILSVRC2012_val_00021637.JPEG:n03089624 +ILSVRC2012_val_00021638.JPEG:n02096585 +ILSVRC2012_val_00021639.JPEG:n02877765 +ILSVRC2012_val_00021640.JPEG:n03259280 +ILSVRC2012_val_00021641.JPEG:n02895154 +ILSVRC2012_val_00021642.JPEG:n02107574 +ILSVRC2012_val_00021643.JPEG:n07615774 +ILSVRC2012_val_00021644.JPEG:n03131574 +ILSVRC2012_val_00021645.JPEG:n02497673 +ILSVRC2012_val_00021646.JPEG:n01688243 +ILSVRC2012_val_00021647.JPEG:n04273569 +ILSVRC2012_val_00021648.JPEG:n03873416 +ILSVRC2012_val_00021649.JPEG:n03763968 +ILSVRC2012_val_00021650.JPEG:n01534433 +ILSVRC2012_val_00021651.JPEG:n03187595 +ILSVRC2012_val_00021652.JPEG:n02786058 +ILSVRC2012_val_00021653.JPEG:n02165105 +ILSVRC2012_val_00021654.JPEG:n02099601 +ILSVRC2012_val_00021655.JPEG:n02782093 +ILSVRC2012_val_00021656.JPEG:n01601694 +ILSVRC2012_val_00021657.JPEG:n03459775 +ILSVRC2012_val_00021658.JPEG:n01770081 +ILSVRC2012_val_00021659.JPEG:n04019541 +ILSVRC2012_val_00021660.JPEG:n01742172 +ILSVRC2012_val_00021661.JPEG:n03452741 +ILSVRC2012_val_00021662.JPEG:n03891251 +ILSVRC2012_val_00021663.JPEG:n01818515 +ILSVRC2012_val_00021664.JPEG:n03825788 +ILSVRC2012_val_00021665.JPEG:n04141975 +ILSVRC2012_val_00021666.JPEG:n02087394 +ILSVRC2012_val_00021667.JPEG:n02325366 +ILSVRC2012_val_00021668.JPEG:n02092339 +ILSVRC2012_val_00021669.JPEG:n07584110 +ILSVRC2012_val_00021670.JPEG:n03649909 +ILSVRC2012_val_00021671.JPEG:n02113712 +ILSVRC2012_val_00021672.JPEG:n04579145 +ILSVRC2012_val_00021673.JPEG:n03908714 +ILSVRC2012_val_00021674.JPEG:n04392985 +ILSVRC2012_val_00021675.JPEG:n02124075 +ILSVRC2012_val_00021676.JPEG:n13040303 +ILSVRC2012_val_00021677.JPEG:n02051845 +ILSVRC2012_val_00021678.JPEG:n02231487 +ILSVRC2012_val_00021679.JPEG:n02493509 +ILSVRC2012_val_00021680.JPEG:n01748264 +ILSVRC2012_val_00021681.JPEG:n03457902 +ILSVRC2012_val_00021682.JPEG:n03146219 +ILSVRC2012_val_00021683.JPEG:n01675722 +ILSVRC2012_val_00021684.JPEG:n03787032 +ILSVRC2012_val_00021685.JPEG:n02361337 +ILSVRC2012_val_00021686.JPEG:n07579787 +ILSVRC2012_val_00021687.JPEG:n04479046 +ILSVRC2012_val_00021688.JPEG:n02168699 +ILSVRC2012_val_00021689.JPEG:n02992211 +ILSVRC2012_val_00021690.JPEG:n02113624 +ILSVRC2012_val_00021691.JPEG:n02974003 +ILSVRC2012_val_00021692.JPEG:n04357314 +ILSVRC2012_val_00021693.JPEG:n07920052 +ILSVRC2012_val_00021694.JPEG:n07615774 +ILSVRC2012_val_00021695.JPEG:n03452741 +ILSVRC2012_val_00021696.JPEG:n03534580 +ILSVRC2012_val_00021697.JPEG:n02094258 +ILSVRC2012_val_00021698.JPEG:n04505470 +ILSVRC2012_val_00021699.JPEG:n02641379 +ILSVRC2012_val_00021700.JPEG:n03868863 +ILSVRC2012_val_00021701.JPEG:n02422699 +ILSVRC2012_val_00021702.JPEG:n03249569 +ILSVRC2012_val_00021703.JPEG:n02123394 +ILSVRC2012_val_00021704.JPEG:n02106662 +ILSVRC2012_val_00021705.JPEG:n01784675 +ILSVRC2012_val_00021706.JPEG:n04371430 +ILSVRC2012_val_00021707.JPEG:n04557648 +ILSVRC2012_val_00021708.JPEG:n02514041 +ILSVRC2012_val_00021709.JPEG:n02051845 +ILSVRC2012_val_00021710.JPEG:n03916031 +ILSVRC2012_val_00021711.JPEG:n01751748 +ILSVRC2012_val_00021712.JPEG:n02504458 +ILSVRC2012_val_00021713.JPEG:n07734744 +ILSVRC2012_val_00021714.JPEG:n02494079 +ILSVRC2012_val_00021715.JPEG:n03902125 +ILSVRC2012_val_00021716.JPEG:n02930766 +ILSVRC2012_val_00021717.JPEG:n03977966 +ILSVRC2012_val_00021718.JPEG:n03724870 +ILSVRC2012_val_00021719.JPEG:n04116512 +ILSVRC2012_val_00021720.JPEG:n03272010 +ILSVRC2012_val_00021721.JPEG:n04049303 +ILSVRC2012_val_00021722.JPEG:n03590841 +ILSVRC2012_val_00021723.JPEG:n02361337 +ILSVRC2012_val_00021724.JPEG:n04044716 +ILSVRC2012_val_00021725.JPEG:n03680355 +ILSVRC2012_val_00021726.JPEG:n03637318 +ILSVRC2012_val_00021727.JPEG:n11939491 +ILSVRC2012_val_00021728.JPEG:n03866082 +ILSVRC2012_val_00021729.JPEG:n03272010 +ILSVRC2012_val_00021730.JPEG:n02119789 +ILSVRC2012_val_00021731.JPEG:n07615774 +ILSVRC2012_val_00021732.JPEG:n03602883 +ILSVRC2012_val_00021733.JPEG:n03492542 +ILSVRC2012_val_00021734.JPEG:n04310018 +ILSVRC2012_val_00021735.JPEG:n02231487 +ILSVRC2012_val_00021736.JPEG:n02110185 +ILSVRC2012_val_00021737.JPEG:n03544143 +ILSVRC2012_val_00021738.JPEG:n03995372 +ILSVRC2012_val_00021739.JPEG:n02268443 +ILSVRC2012_val_00021740.JPEG:n01440764 +ILSVRC2012_val_00021741.JPEG:n02480855 +ILSVRC2012_val_00021742.JPEG:n02317335 +ILSVRC2012_val_00021743.JPEG:n01692333 +ILSVRC2012_val_00021744.JPEG:n02109961 +ILSVRC2012_val_00021745.JPEG:n03379051 +ILSVRC2012_val_00021746.JPEG:n03075370 +ILSVRC2012_val_00021747.JPEG:n02687172 +ILSVRC2012_val_00021748.JPEG:n04442312 +ILSVRC2012_val_00021749.JPEG:n03584254 +ILSVRC2012_val_00021750.JPEG:n01729977 +ILSVRC2012_val_00021751.JPEG:n02727426 +ILSVRC2012_val_00021752.JPEG:n03134739 +ILSVRC2012_val_00021753.JPEG:n01828970 +ILSVRC2012_val_00021754.JPEG:n02093428 +ILSVRC2012_val_00021755.JPEG:n02233338 +ILSVRC2012_val_00021756.JPEG:n02091831 +ILSVRC2012_val_00021757.JPEG:n02939185 +ILSVRC2012_val_00021758.JPEG:n04579432 +ILSVRC2012_val_00021759.JPEG:n04266014 +ILSVRC2012_val_00021760.JPEG:n03291819 +ILSVRC2012_val_00021761.JPEG:n03954731 +ILSVRC2012_val_00021762.JPEG:n03838899 +ILSVRC2012_val_00021763.JPEG:n07871810 +ILSVRC2012_val_00021764.JPEG:n02077923 +ILSVRC2012_val_00021765.JPEG:n12057211 +ILSVRC2012_val_00021766.JPEG:n02415577 +ILSVRC2012_val_00021767.JPEG:n02115641 +ILSVRC2012_val_00021768.JPEG:n03781244 +ILSVRC2012_val_00021769.JPEG:n07880968 +ILSVRC2012_val_00021770.JPEG:n07711569 +ILSVRC2012_val_00021771.JPEG:n03838899 +ILSVRC2012_val_00021772.JPEG:n03180011 +ILSVRC2012_val_00021773.JPEG:n02114712 +ILSVRC2012_val_00021774.JPEG:n03887697 +ILSVRC2012_val_00021775.JPEG:n02930766 +ILSVRC2012_val_00021776.JPEG:n01644900 +ILSVRC2012_val_00021777.JPEG:n02111277 +ILSVRC2012_val_00021778.JPEG:n02999410 +ILSVRC2012_val_00021779.JPEG:n03534580 +ILSVRC2012_val_00021780.JPEG:n02497673 +ILSVRC2012_val_00021781.JPEG:n02410509 +ILSVRC2012_val_00021782.JPEG:n02777292 +ILSVRC2012_val_00021783.JPEG:n03461385 +ILSVRC2012_val_00021784.JPEG:n04086273 +ILSVRC2012_val_00021785.JPEG:n03627232 +ILSVRC2012_val_00021786.JPEG:n01689811 +ILSVRC2012_val_00021787.JPEG:n09193705 +ILSVRC2012_val_00021788.JPEG:n01955084 +ILSVRC2012_val_00021789.JPEG:n03916031 +ILSVRC2012_val_00021790.JPEG:n04355338 +ILSVRC2012_val_00021791.JPEG:n04259630 +ILSVRC2012_val_00021792.JPEG:n03617480 +ILSVRC2012_val_00021793.JPEG:n01498041 +ILSVRC2012_val_00021794.JPEG:n02169497 +ILSVRC2012_val_00021795.JPEG:n02423022 +ILSVRC2012_val_00021796.JPEG:n02422106 +ILSVRC2012_val_00021797.JPEG:n02699494 +ILSVRC2012_val_00021798.JPEG:n02494079 +ILSVRC2012_val_00021799.JPEG:n04515003 +ILSVRC2012_val_00021800.JPEG:n03724870 +ILSVRC2012_val_00021801.JPEG:n02113799 +ILSVRC2012_val_00021802.JPEG:n03930630 +ILSVRC2012_val_00021803.JPEG:n04458633 +ILSVRC2012_val_00021804.JPEG:n04065272 +ILSVRC2012_val_00021805.JPEG:n02939185 +ILSVRC2012_val_00021806.JPEG:n02281787 +ILSVRC2012_val_00021807.JPEG:n02504458 +ILSVRC2012_val_00021808.JPEG:n02190166 +ILSVRC2012_val_00021809.JPEG:n03691459 +ILSVRC2012_val_00021810.JPEG:n02408429 +ILSVRC2012_val_00021811.JPEG:n07579787 +ILSVRC2012_val_00021812.JPEG:n02114712 +ILSVRC2012_val_00021813.JPEG:n04125021 +ILSVRC2012_val_00021814.JPEG:n04461696 +ILSVRC2012_val_00021815.JPEG:n03384352 +ILSVRC2012_val_00021816.JPEG:n03388183 +ILSVRC2012_val_00021817.JPEG:n03837869 +ILSVRC2012_val_00021818.JPEG:n03485407 +ILSVRC2012_val_00021819.JPEG:n01986214 +ILSVRC2012_val_00021820.JPEG:n03255030 +ILSVRC2012_val_00021821.JPEG:n02804610 +ILSVRC2012_val_00021822.JPEG:n03255030 +ILSVRC2012_val_00021823.JPEG:n01924916 +ILSVRC2012_val_00021824.JPEG:n04398044 +ILSVRC2012_val_00021825.JPEG:n04540053 +ILSVRC2012_val_00021826.JPEG:n02667093 +ILSVRC2012_val_00021827.JPEG:n03146219 +ILSVRC2012_val_00021828.JPEG:n02483708 +ILSVRC2012_val_00021829.JPEG:n03125729 +ILSVRC2012_val_00021830.JPEG:n09256479 +ILSVRC2012_val_00021831.JPEG:n02089078 +ILSVRC2012_val_00021832.JPEG:n02607072 +ILSVRC2012_val_00021833.JPEG:n03742115 +ILSVRC2012_val_00021834.JPEG:n04067472 +ILSVRC2012_val_00021835.JPEG:n02114712 +ILSVRC2012_val_00021836.JPEG:n03196217 +ILSVRC2012_val_00021837.JPEG:n04254120 +ILSVRC2012_val_00021838.JPEG:n02105412 +ILSVRC2012_val_00021839.JPEG:n03250847 +ILSVRC2012_val_00021840.JPEG:n02111500 +ILSVRC2012_val_00021841.JPEG:n07565083 +ILSVRC2012_val_00021842.JPEG:n04162706 +ILSVRC2012_val_00021843.JPEG:n01917289 +ILSVRC2012_val_00021844.JPEG:n03018349 +ILSVRC2012_val_00021845.JPEG:n03530642 +ILSVRC2012_val_00021846.JPEG:n02107908 +ILSVRC2012_val_00021847.JPEG:n02169497 +ILSVRC2012_val_00021848.JPEG:n02018795 +ILSVRC2012_val_00021849.JPEG:n03658185 +ILSVRC2012_val_00021850.JPEG:n03424325 +ILSVRC2012_val_00021851.JPEG:n02018207 +ILSVRC2012_val_00021852.JPEG:n03630383 +ILSVRC2012_val_00021853.JPEG:n03903868 +ILSVRC2012_val_00021854.JPEG:n07745940 +ILSVRC2012_val_00021855.JPEG:n02138441 +ILSVRC2012_val_00021856.JPEG:n03372029 +ILSVRC2012_val_00021857.JPEG:n02319095 +ILSVRC2012_val_00021858.JPEG:n01855672 +ILSVRC2012_val_00021859.JPEG:n03062245 +ILSVRC2012_val_00021860.JPEG:n07753592 +ILSVRC2012_val_00021861.JPEG:n04147183 +ILSVRC2012_val_00021862.JPEG:n04254777 +ILSVRC2012_val_00021863.JPEG:n03838899 +ILSVRC2012_val_00021864.JPEG:n02219486 +ILSVRC2012_val_00021865.JPEG:n04270147 +ILSVRC2012_val_00021866.JPEG:n07871810 +ILSVRC2012_val_00021867.JPEG:n01910747 +ILSVRC2012_val_00021868.JPEG:n02999410 +ILSVRC2012_val_00021869.JPEG:n12768682 +ILSVRC2012_val_00021870.JPEG:n03649909 +ILSVRC2012_val_00021871.JPEG:n04120489 +ILSVRC2012_val_00021872.JPEG:n02002724 +ILSVRC2012_val_00021873.JPEG:n01756291 +ILSVRC2012_val_00021874.JPEG:n02445715 +ILSVRC2012_val_00021875.JPEG:n02009912 +ILSVRC2012_val_00021876.JPEG:n01798484 +ILSVRC2012_val_00021877.JPEG:n04532670 +ILSVRC2012_val_00021878.JPEG:n04604644 +ILSVRC2012_val_00021879.JPEG:n04044716 +ILSVRC2012_val_00021880.JPEG:n02169497 +ILSVRC2012_val_00021881.JPEG:n02669723 +ILSVRC2012_val_00021882.JPEG:n04461696 +ILSVRC2012_val_00021883.JPEG:n02134084 +ILSVRC2012_val_00021884.JPEG:n03743016 +ILSVRC2012_val_00021885.JPEG:n01798484 +ILSVRC2012_val_00021886.JPEG:n03404251 +ILSVRC2012_val_00021887.JPEG:n02783161 +ILSVRC2012_val_00021888.JPEG:n03201208 +ILSVRC2012_val_00021889.JPEG:n02134084 +ILSVRC2012_val_00021890.JPEG:n02607072 +ILSVRC2012_val_00021891.JPEG:n03180011 +ILSVRC2012_val_00021892.JPEG:n02094433 +ILSVRC2012_val_00021893.JPEG:n03388549 +ILSVRC2012_val_00021894.JPEG:n07590611 +ILSVRC2012_val_00021895.JPEG:n02640242 +ILSVRC2012_val_00021896.JPEG:n02085782 +ILSVRC2012_val_00021897.JPEG:n02871525 +ILSVRC2012_val_00021898.JPEG:n03967562 +ILSVRC2012_val_00021899.JPEG:n02119789 +ILSVRC2012_val_00021900.JPEG:n04507155 +ILSVRC2012_val_00021901.JPEG:n04149813 +ILSVRC2012_val_00021902.JPEG:n03492542 +ILSVRC2012_val_00021903.JPEG:n02437312 +ILSVRC2012_val_00021904.JPEG:n02098105 +ILSVRC2012_val_00021905.JPEG:n01443537 +ILSVRC2012_val_00021906.JPEG:n01632458 +ILSVRC2012_val_00021907.JPEG:n02860847 +ILSVRC2012_val_00021908.JPEG:n02113023 +ILSVRC2012_val_00021909.JPEG:n03337140 +ILSVRC2012_val_00021910.JPEG:n12620546 +ILSVRC2012_val_00021911.JPEG:n03459775 +ILSVRC2012_val_00021912.JPEG:n11879895 +ILSVRC2012_val_00021913.JPEG:n03085013 +ILSVRC2012_val_00021914.JPEG:n02096585 +ILSVRC2012_val_00021915.JPEG:n02088466 +ILSVRC2012_val_00021916.JPEG:n01751748 +ILSVRC2012_val_00021917.JPEG:n02497673 +ILSVRC2012_val_00021918.JPEG:n02236044 +ILSVRC2012_val_00021919.JPEG:n03109150 +ILSVRC2012_val_00021920.JPEG:n02130308 +ILSVRC2012_val_00021921.JPEG:n04325704 +ILSVRC2012_val_00021922.JPEG:n03676483 +ILSVRC2012_val_00021923.JPEG:n02105412 +ILSVRC2012_val_00021924.JPEG:n03180011 +ILSVRC2012_val_00021925.JPEG:n02787622 +ILSVRC2012_val_00021926.JPEG:n02025239 +ILSVRC2012_val_00021927.JPEG:n01693334 +ILSVRC2012_val_00021928.JPEG:n02325366 +ILSVRC2012_val_00021929.JPEG:n02281787 +ILSVRC2012_val_00021930.JPEG:n04597913 +ILSVRC2012_val_00021931.JPEG:n04346328 +ILSVRC2012_val_00021932.JPEG:n04404412 +ILSVRC2012_val_00021933.JPEG:n02006656 +ILSVRC2012_val_00021934.JPEG:n02107312 +ILSVRC2012_val_00021935.JPEG:n02165456 +ILSVRC2012_val_00021936.JPEG:n03042490 +ILSVRC2012_val_00021937.JPEG:n04418357 +ILSVRC2012_val_00021938.JPEG:n02093428 +ILSVRC2012_val_00021939.JPEG:n04133789 +ILSVRC2012_val_00021940.JPEG:n07754684 +ILSVRC2012_val_00021941.JPEG:n03075370 +ILSVRC2012_val_00021942.JPEG:n03916031 +ILSVRC2012_val_00021943.JPEG:n04536866 +ILSVRC2012_val_00021944.JPEG:n07711569 +ILSVRC2012_val_00021945.JPEG:n02895154 +ILSVRC2012_val_00021946.JPEG:n02105251 +ILSVRC2012_val_00021947.JPEG:n02692877 +ILSVRC2012_val_00021948.JPEG:n03344393 +ILSVRC2012_val_00021949.JPEG:n04493381 +ILSVRC2012_val_00021950.JPEG:n04579145 +ILSVRC2012_val_00021951.JPEG:n03201208 +ILSVRC2012_val_00021952.JPEG:n04243546 +ILSVRC2012_val_00021953.JPEG:n02167151 +ILSVRC2012_val_00021954.JPEG:n01797886 +ILSVRC2012_val_00021955.JPEG:n09256479 +ILSVRC2012_val_00021956.JPEG:n01582220 +ILSVRC2012_val_00021957.JPEG:n04548362 +ILSVRC2012_val_00021958.JPEG:n03476684 +ILSVRC2012_val_00021959.JPEG:n04606251 +ILSVRC2012_val_00021960.JPEG:n04579432 +ILSVRC2012_val_00021961.JPEG:n02086910 +ILSVRC2012_val_00021962.JPEG:n02134084 +ILSVRC2012_val_00021963.JPEG:n02109525 +ILSVRC2012_val_00021964.JPEG:n04238763 +ILSVRC2012_val_00021965.JPEG:n03764736 +ILSVRC2012_val_00021966.JPEG:n04044716 +ILSVRC2012_val_00021967.JPEG:n04548362 +ILSVRC2012_val_00021968.JPEG:n02692877 +ILSVRC2012_val_00021969.JPEG:n03207941 +ILSVRC2012_val_00021970.JPEG:n04229816 +ILSVRC2012_val_00021971.JPEG:n03598930 +ILSVRC2012_val_00021972.JPEG:n04591157 +ILSVRC2012_val_00021973.JPEG:n02317335 +ILSVRC2012_val_00021974.JPEG:n01734418 +ILSVRC2012_val_00021975.JPEG:n15075141 +ILSVRC2012_val_00021976.JPEG:n03825788 +ILSVRC2012_val_00021977.JPEG:n04536866 +ILSVRC2012_val_00021978.JPEG:n04254777 +ILSVRC2012_val_00021979.JPEG:n02277742 +ILSVRC2012_val_00021980.JPEG:n03877845 +ILSVRC2012_val_00021981.JPEG:n02747177 +ILSVRC2012_val_00021982.JPEG:n01667778 +ILSVRC2012_val_00021983.JPEG:n01664065 +ILSVRC2012_val_00021984.JPEG:n03180011 +ILSVRC2012_val_00021985.JPEG:n02701002 +ILSVRC2012_val_00021986.JPEG:n13040303 +ILSVRC2012_val_00021987.JPEG:n03388549 +ILSVRC2012_val_00021988.JPEG:n04591713 +ILSVRC2012_val_00021989.JPEG:n04389033 +ILSVRC2012_val_00021990.JPEG:n02699494 +ILSVRC2012_val_00021991.JPEG:n02105162 +ILSVRC2012_val_00021992.JPEG:n02280649 +ILSVRC2012_val_00021993.JPEG:n04254777 +ILSVRC2012_val_00021994.JPEG:n02607072 +ILSVRC2012_val_00021995.JPEG:n01985128 +ILSVRC2012_val_00021996.JPEG:n03045698 +ILSVRC2012_val_00021997.JPEG:n03717622 +ILSVRC2012_val_00021998.JPEG:n02086240 +ILSVRC2012_val_00021999.JPEG:n03903868 +ILSVRC2012_val_00022000.JPEG:n02326432 +ILSVRC2012_val_00022001.JPEG:n02229544 +ILSVRC2012_val_00022002.JPEG:n03530642 +ILSVRC2012_val_00022003.JPEG:n01685808 +ILSVRC2012_val_00022004.JPEG:n02091467 +ILSVRC2012_val_00022005.JPEG:n03544143 +ILSVRC2012_val_00022006.JPEG:n03902125 +ILSVRC2012_val_00022007.JPEG:n02125311 +ILSVRC2012_val_00022008.JPEG:n09399592 +ILSVRC2012_val_00022009.JPEG:n04070727 +ILSVRC2012_val_00022010.JPEG:n07730033 +ILSVRC2012_val_00022011.JPEG:n07684084 +ILSVRC2012_val_00022012.JPEG:n04398044 +ILSVRC2012_val_00022013.JPEG:n03372029 +ILSVRC2012_val_00022014.JPEG:n03483316 +ILSVRC2012_val_00022015.JPEG:n03495258 +ILSVRC2012_val_00022016.JPEG:n01728572 +ILSVRC2012_val_00022017.JPEG:n04037443 +ILSVRC2012_val_00022018.JPEG:n02395406 +ILSVRC2012_val_00022019.JPEG:n03457902 +ILSVRC2012_val_00022020.JPEG:n03761084 +ILSVRC2012_val_00022021.JPEG:n01734418 +ILSVRC2012_val_00022022.JPEG:n02090721 +ILSVRC2012_val_00022023.JPEG:n03976657 +ILSVRC2012_val_00022024.JPEG:n03785016 +ILSVRC2012_val_00022025.JPEG:n01514668 +ILSVRC2012_val_00022026.JPEG:n04357314 +ILSVRC2012_val_00022027.JPEG:n02835271 +ILSVRC2012_val_00022028.JPEG:n02504013 +ILSVRC2012_val_00022029.JPEG:n02489166 +ILSVRC2012_val_00022030.JPEG:n03530642 +ILSVRC2012_val_00022031.JPEG:n02950826 +ILSVRC2012_val_00022032.JPEG:n02111889 +ILSVRC2012_val_00022033.JPEG:n04371774 +ILSVRC2012_val_00022034.JPEG:n04560804 +ILSVRC2012_val_00022035.JPEG:n03445924 +ILSVRC2012_val_00022036.JPEG:n02091831 +ILSVRC2012_val_00022037.JPEG:n07753592 +ILSVRC2012_val_00022038.JPEG:n03447721 +ILSVRC2012_val_00022039.JPEG:n01770081 +ILSVRC2012_val_00022040.JPEG:n02487347 +ILSVRC2012_val_00022041.JPEG:n02794156 +ILSVRC2012_val_00022042.JPEG:n02097209 +ILSVRC2012_val_00022043.JPEG:n03891251 +ILSVRC2012_val_00022044.JPEG:n02790996 +ILSVRC2012_val_00022045.JPEG:n03109150 +ILSVRC2012_val_00022046.JPEG:n04380533 +ILSVRC2012_val_00022047.JPEG:n03595614 +ILSVRC2012_val_00022048.JPEG:n04153751 +ILSVRC2012_val_00022049.JPEG:n04591713 +ILSVRC2012_val_00022050.JPEG:n02108915 +ILSVRC2012_val_00022051.JPEG:n04429376 +ILSVRC2012_val_00022052.JPEG:n01641577 +ILSVRC2012_val_00022053.JPEG:n04264628 +ILSVRC2012_val_00022054.JPEG:n03271574 +ILSVRC2012_val_00022055.JPEG:n02114367 +ILSVRC2012_val_00022056.JPEG:n07930864 +ILSVRC2012_val_00022057.JPEG:n02105641 +ILSVRC2012_val_00022058.JPEG:n02104365 +ILSVRC2012_val_00022059.JPEG:n03717622 +ILSVRC2012_val_00022060.JPEG:n04423845 +ILSVRC2012_val_00022061.JPEG:n02094258 +ILSVRC2012_val_00022062.JPEG:n02116738 +ILSVRC2012_val_00022063.JPEG:n01692333 +ILSVRC2012_val_00022064.JPEG:n02909870 +ILSVRC2012_val_00022065.JPEG:n02606052 +ILSVRC2012_val_00022066.JPEG:n02099849 +ILSVRC2012_val_00022067.JPEG:n02363005 +ILSVRC2012_val_00022068.JPEG:n07734744 +ILSVRC2012_val_00022069.JPEG:n02841315 +ILSVRC2012_val_00022070.JPEG:n01860187 +ILSVRC2012_val_00022071.JPEG:n02090721 +ILSVRC2012_val_00022072.JPEG:n03841143 +ILSVRC2012_val_00022073.JPEG:n02892201 +ILSVRC2012_val_00022074.JPEG:n04125021 +ILSVRC2012_val_00022075.JPEG:n04612504 +ILSVRC2012_val_00022076.JPEG:n01537544 +ILSVRC2012_val_00022077.JPEG:n04505470 +ILSVRC2012_val_00022078.JPEG:n02281406 +ILSVRC2012_val_00022079.JPEG:n03983396 +ILSVRC2012_val_00022080.JPEG:n02123045 +ILSVRC2012_val_00022081.JPEG:n01784675 +ILSVRC2012_val_00022082.JPEG:n02493509 +ILSVRC2012_val_00022083.JPEG:n03476991 +ILSVRC2012_val_00022084.JPEG:n03534580 +ILSVRC2012_val_00022085.JPEG:n02123159 +ILSVRC2012_val_00022086.JPEG:n02808440 +ILSVRC2012_val_00022087.JPEG:n04074963 +ILSVRC2012_val_00022088.JPEG:n01616318 +ILSVRC2012_val_00022089.JPEG:n03786901 +ILSVRC2012_val_00022090.JPEG:n03721384 +ILSVRC2012_val_00022091.JPEG:n02086240 +ILSVRC2012_val_00022092.JPEG:n02488702 +ILSVRC2012_val_00022093.JPEG:n03642806 +ILSVRC2012_val_00022094.JPEG:n03160309 +ILSVRC2012_val_00022095.JPEG:n01796340 +ILSVRC2012_val_00022096.JPEG:n13044778 +ILSVRC2012_val_00022097.JPEG:n09256479 +ILSVRC2012_val_00022098.JPEG:n03089624 +ILSVRC2012_val_00022099.JPEG:n02086910 +ILSVRC2012_val_00022100.JPEG:n04604644 +ILSVRC2012_val_00022101.JPEG:n04040759 +ILSVRC2012_val_00022102.JPEG:n07584110 +ILSVRC2012_val_00022103.JPEG:n04552348 +ILSVRC2012_val_00022104.JPEG:n04149813 +ILSVRC2012_val_00022105.JPEG:n02066245 +ILSVRC2012_val_00022106.JPEG:n01580077 +ILSVRC2012_val_00022107.JPEG:n04443257 +ILSVRC2012_val_00022108.JPEG:n04336792 +ILSVRC2012_val_00022109.JPEG:n02107683 +ILSVRC2012_val_00022110.JPEG:n01797886 +ILSVRC2012_val_00022111.JPEG:n02134418 +ILSVRC2012_val_00022112.JPEG:n02134418 +ILSVRC2012_val_00022113.JPEG:n01632777 +ILSVRC2012_val_00022114.JPEG:n06359193 +ILSVRC2012_val_00022115.JPEG:n01797886 +ILSVRC2012_val_00022116.JPEG:n03485407 +ILSVRC2012_val_00022117.JPEG:n04259630 +ILSVRC2012_val_00022118.JPEG:n03992509 +ILSVRC2012_val_00022119.JPEG:n07248320 +ILSVRC2012_val_00022120.JPEG:n04486054 +ILSVRC2012_val_00022121.JPEG:n03026506 +ILSVRC2012_val_00022122.JPEG:n02088632 +ILSVRC2012_val_00022123.JPEG:n03124043 +ILSVRC2012_val_00022124.JPEG:n02442845 +ILSVRC2012_val_00022125.JPEG:n02091467 +ILSVRC2012_val_00022126.JPEG:n03376595 +ILSVRC2012_val_00022127.JPEG:n04310018 +ILSVRC2012_val_00022128.JPEG:n02966687 +ILSVRC2012_val_00022129.JPEG:n03777568 +ILSVRC2012_val_00022130.JPEG:n03100240 +ILSVRC2012_val_00022131.JPEG:n04350905 +ILSVRC2012_val_00022132.JPEG:n02843684 +ILSVRC2012_val_00022133.JPEG:n02109961 +ILSVRC2012_val_00022134.JPEG:n01631663 +ILSVRC2012_val_00022135.JPEG:n03240683 +ILSVRC2012_val_00022136.JPEG:n03141823 +ILSVRC2012_val_00022137.JPEG:n02091635 +ILSVRC2012_val_00022138.JPEG:n01443537 +ILSVRC2012_val_00022139.JPEG:n11939491 +ILSVRC2012_val_00022140.JPEG:n02002724 +ILSVRC2012_val_00022141.JPEG:n03733281 +ILSVRC2012_val_00022142.JPEG:n02106662 +ILSVRC2012_val_00022143.JPEG:n03942813 +ILSVRC2012_val_00022144.JPEG:n03337140 +ILSVRC2012_val_00022145.JPEG:n03777568 +ILSVRC2012_val_00022146.JPEG:n04251144 +ILSVRC2012_val_00022147.JPEG:n07716906 +ILSVRC2012_val_00022148.JPEG:n01820546 +ILSVRC2012_val_00022149.JPEG:n03929660 +ILSVRC2012_val_00022150.JPEG:n03478589 +ILSVRC2012_val_00022151.JPEG:n02441942 +ILSVRC2012_val_00022152.JPEG:n02364673 +ILSVRC2012_val_00022153.JPEG:n09835506 +ILSVRC2012_val_00022154.JPEG:n04515003 +ILSVRC2012_val_00022155.JPEG:n02264363 +ILSVRC2012_val_00022156.JPEG:n01773157 +ILSVRC2012_val_00022157.JPEG:n01770393 +ILSVRC2012_val_00022158.JPEG:n03777568 +ILSVRC2012_val_00022159.JPEG:n04049303 +ILSVRC2012_val_00022160.JPEG:n02219486 +ILSVRC2012_val_00022161.JPEG:n02130308 +ILSVRC2012_val_00022162.JPEG:n02437312 +ILSVRC2012_val_00022163.JPEG:n02815834 +ILSVRC2012_val_00022164.JPEG:n02093647 +ILSVRC2012_val_00022165.JPEG:n01616318 +ILSVRC2012_val_00022166.JPEG:n04332243 +ILSVRC2012_val_00022167.JPEG:n12620546 +ILSVRC2012_val_00022168.JPEG:n10148035 +ILSVRC2012_val_00022169.JPEG:n02927161 +ILSVRC2012_val_00022170.JPEG:n02128757 +ILSVRC2012_val_00022171.JPEG:n03496892 +ILSVRC2012_val_00022172.JPEG:n03417042 +ILSVRC2012_val_00022173.JPEG:n04200800 +ILSVRC2012_val_00022174.JPEG:n02484975 +ILSVRC2012_val_00022175.JPEG:n01689811 +ILSVRC2012_val_00022176.JPEG:n02107574 +ILSVRC2012_val_00022177.JPEG:n03976657 +ILSVRC2012_val_00022178.JPEG:n03998194 +ILSVRC2012_val_00022179.JPEG:n02088632 +ILSVRC2012_val_00022180.JPEG:n04243546 +ILSVRC2012_val_00022181.JPEG:n03788365 +ILSVRC2012_val_00022182.JPEG:n02087046 +ILSVRC2012_val_00022183.JPEG:n10565667 +ILSVRC2012_val_00022184.JPEG:n03832673 +ILSVRC2012_val_00022185.JPEG:n02412080 +ILSVRC2012_val_00022186.JPEG:n01558993 +ILSVRC2012_val_00022187.JPEG:n03492542 +ILSVRC2012_val_00022188.JPEG:n04540053 +ILSVRC2012_val_00022189.JPEG:n01796340 +ILSVRC2012_val_00022190.JPEG:n04376876 +ILSVRC2012_val_00022191.JPEG:n02395406 +ILSVRC2012_val_00022192.JPEG:n03075370 +ILSVRC2012_val_00022193.JPEG:n07753592 +ILSVRC2012_val_00022194.JPEG:n02481823 +ILSVRC2012_val_00022195.JPEG:n02457408 +ILSVRC2012_val_00022196.JPEG:n02110806 +ILSVRC2012_val_00022197.JPEG:n03877472 +ILSVRC2012_val_00022198.JPEG:n01667778 +ILSVRC2012_val_00022199.JPEG:n03131574 +ILSVRC2012_val_00022200.JPEG:n03956157 +ILSVRC2012_val_00022201.JPEG:n02108422 +ILSVRC2012_val_00022202.JPEG:n02114548 +ILSVRC2012_val_00022203.JPEG:n03272010 +ILSVRC2012_val_00022204.JPEG:n03394916 +ILSVRC2012_val_00022205.JPEG:n01774384 +ILSVRC2012_val_00022206.JPEG:n03623198 +ILSVRC2012_val_00022207.JPEG:n02027492 +ILSVRC2012_val_00022208.JPEG:n04099969 +ILSVRC2012_val_00022209.JPEG:n02106662 +ILSVRC2012_val_00022210.JPEG:n02951358 +ILSVRC2012_val_00022211.JPEG:n01798484 +ILSVRC2012_val_00022212.JPEG:n13133613 +ILSVRC2012_val_00022213.JPEG:n03207743 +ILSVRC2012_val_00022214.JPEG:n04560804 +ILSVRC2012_val_00022215.JPEG:n02268443 +ILSVRC2012_val_00022216.JPEG:n03775071 +ILSVRC2012_val_00022217.JPEG:n04346328 +ILSVRC2012_val_00022218.JPEG:n01930112 +ILSVRC2012_val_00022219.JPEG:n03584254 +ILSVRC2012_val_00022220.JPEG:n02790996 +ILSVRC2012_val_00022221.JPEG:n09256479 +ILSVRC2012_val_00022222.JPEG:n01985128 +ILSVRC2012_val_00022223.JPEG:n02480495 +ILSVRC2012_val_00022224.JPEG:n02268853 +ILSVRC2012_val_00022225.JPEG:n03627232 +ILSVRC2012_val_00022226.JPEG:n03180011 +ILSVRC2012_val_00022227.JPEG:n02233338 +ILSVRC2012_val_00022228.JPEG:n03982430 +ILSVRC2012_val_00022229.JPEG:n02841315 +ILSVRC2012_val_00022230.JPEG:n03649909 +ILSVRC2012_val_00022231.JPEG:n04336792 +ILSVRC2012_val_00022232.JPEG:n09468604 +ILSVRC2012_val_00022233.JPEG:n02056570 +ILSVRC2012_val_00022234.JPEG:n02787622 +ILSVRC2012_val_00022235.JPEG:n03764736 +ILSVRC2012_val_00022236.JPEG:n02442845 +ILSVRC2012_val_00022237.JPEG:n02437616 +ILSVRC2012_val_00022238.JPEG:n03445924 +ILSVRC2012_val_00022239.JPEG:n01917289 +ILSVRC2012_val_00022240.JPEG:n02107312 +ILSVRC2012_val_00022241.JPEG:n02137549 +ILSVRC2012_val_00022242.JPEG:n03599486 +ILSVRC2012_val_00022243.JPEG:n03721384 +ILSVRC2012_val_00022244.JPEG:n04041544 +ILSVRC2012_val_00022245.JPEG:n01824575 +ILSVRC2012_val_00022246.JPEG:n04285008 +ILSVRC2012_val_00022247.JPEG:n01687978 +ILSVRC2012_val_00022248.JPEG:n01514668 +ILSVRC2012_val_00022249.JPEG:n04554684 +ILSVRC2012_val_00022250.JPEG:n04209239 +ILSVRC2012_val_00022251.JPEG:n03272562 +ILSVRC2012_val_00022252.JPEG:n03425413 +ILSVRC2012_val_00022253.JPEG:n02797295 +ILSVRC2012_val_00022254.JPEG:n02106382 +ILSVRC2012_val_00022255.JPEG:n06359193 +ILSVRC2012_val_00022256.JPEG:n03642806 +ILSVRC2012_val_00022257.JPEG:n01677366 +ILSVRC2012_val_00022258.JPEG:n03134739 +ILSVRC2012_val_00022259.JPEG:n02105641 +ILSVRC2012_val_00022260.JPEG:n01985128 +ILSVRC2012_val_00022261.JPEG:n03594945 +ILSVRC2012_val_00022262.JPEG:n07583066 +ILSVRC2012_val_00022263.JPEG:n02667093 +ILSVRC2012_val_00022264.JPEG:n02086646 +ILSVRC2012_val_00022265.JPEG:n07590611 +ILSVRC2012_val_00022266.JPEG:n02111889 +ILSVRC2012_val_00022267.JPEG:n03857828 +ILSVRC2012_val_00022268.JPEG:n04259630 +ILSVRC2012_val_00022269.JPEG:n02730930 +ILSVRC2012_val_00022270.JPEG:n04285008 +ILSVRC2012_val_00022271.JPEG:n03095699 +ILSVRC2012_val_00022272.JPEG:n03761084 +ILSVRC2012_val_00022273.JPEG:n02167151 +ILSVRC2012_val_00022274.JPEG:n04404412 +ILSVRC2012_val_00022275.JPEG:n04254120 +ILSVRC2012_val_00022276.JPEG:n04461696 +ILSVRC2012_val_00022277.JPEG:n04192698 +ILSVRC2012_val_00022278.JPEG:n01873310 +ILSVRC2012_val_00022279.JPEG:n03763968 +ILSVRC2012_val_00022280.JPEG:n02804414 +ILSVRC2012_val_00022281.JPEG:n04325704 +ILSVRC2012_val_00022282.JPEG:n01682714 +ILSVRC2012_val_00022283.JPEG:n02120505 +ILSVRC2012_val_00022284.JPEG:n03584829 +ILSVRC2012_val_00022285.JPEG:n04356056 +ILSVRC2012_val_00022286.JPEG:n04476259 +ILSVRC2012_val_00022287.JPEG:n09332890 +ILSVRC2012_val_00022288.JPEG:n04399382 +ILSVRC2012_val_00022289.JPEG:n03676483 +ILSVRC2012_val_00022290.JPEG:n03961711 +ILSVRC2012_val_00022291.JPEG:n09332890 +ILSVRC2012_val_00022292.JPEG:n02096294 +ILSVRC2012_val_00022293.JPEG:n04532106 +ILSVRC2012_val_00022294.JPEG:n04149813 +ILSVRC2012_val_00022295.JPEG:n03891251 +ILSVRC2012_val_00022296.JPEG:n06874185 +ILSVRC2012_val_00022297.JPEG:n02769748 +ILSVRC2012_val_00022298.JPEG:n04485082 +ILSVRC2012_val_00022299.JPEG:n04277352 +ILSVRC2012_val_00022300.JPEG:n03793489 +ILSVRC2012_val_00022301.JPEG:n03788365 +ILSVRC2012_val_00022302.JPEG:n02389026 +ILSVRC2012_val_00022303.JPEG:n03709823 +ILSVRC2012_val_00022304.JPEG:n03032252 +ILSVRC2012_val_00022305.JPEG:n02606052 +ILSVRC2012_val_00022306.JPEG:n03271574 +ILSVRC2012_val_00022307.JPEG:n03492542 +ILSVRC2012_val_00022308.JPEG:n01665541 +ILSVRC2012_val_00022309.JPEG:n01675722 +ILSVRC2012_val_00022310.JPEG:n03691459 +ILSVRC2012_val_00022311.JPEG:n07892512 +ILSVRC2012_val_00022312.JPEG:n02799071 +ILSVRC2012_val_00022313.JPEG:n02007558 +ILSVRC2012_val_00022314.JPEG:n02510455 +ILSVRC2012_val_00022315.JPEG:n03742115 +ILSVRC2012_val_00022316.JPEG:n04136333 +ILSVRC2012_val_00022317.JPEG:n03630383 +ILSVRC2012_val_00022318.JPEG:n02910353 +ILSVRC2012_val_00022319.JPEG:n02111129 +ILSVRC2012_val_00022320.JPEG:n02488702 +ILSVRC2012_val_00022321.JPEG:n01950731 +ILSVRC2012_val_00022322.JPEG:n04204238 +ILSVRC2012_val_00022323.JPEG:n04461696 +ILSVRC2012_val_00022324.JPEG:n02102318 +ILSVRC2012_val_00022325.JPEG:n03538406 +ILSVRC2012_val_00022326.JPEG:n03916031 +ILSVRC2012_val_00022327.JPEG:n02130308 +ILSVRC2012_val_00022328.JPEG:n04311174 +ILSVRC2012_val_00022329.JPEG:n01667114 +ILSVRC2012_val_00022330.JPEG:n02115641 +ILSVRC2012_val_00022331.JPEG:n04487394 +ILSVRC2012_val_00022332.JPEG:n02233338 +ILSVRC2012_val_00022333.JPEG:n02099267 +ILSVRC2012_val_00022334.JPEG:n01797886 +ILSVRC2012_val_00022335.JPEG:n02051845 +ILSVRC2012_val_00022336.JPEG:n04428191 +ILSVRC2012_val_00022337.JPEG:n02124075 +ILSVRC2012_val_00022338.JPEG:n04532670 +ILSVRC2012_val_00022339.JPEG:n03775546 +ILSVRC2012_val_00022340.JPEG:n07892512 +ILSVRC2012_val_00022341.JPEG:n02100877 +ILSVRC2012_val_00022342.JPEG:n04398044 +ILSVRC2012_val_00022343.JPEG:n04590129 +ILSVRC2012_val_00022344.JPEG:n02101388 +ILSVRC2012_val_00022345.JPEG:n04254680 +ILSVRC2012_val_00022346.JPEG:n04485082 +ILSVRC2012_val_00022347.JPEG:n03026506 +ILSVRC2012_val_00022348.JPEG:n04111531 +ILSVRC2012_val_00022349.JPEG:n03924679 +ILSVRC2012_val_00022350.JPEG:n01667778 +ILSVRC2012_val_00022351.JPEG:n02169497 +ILSVRC2012_val_00022352.JPEG:n04311004 +ILSVRC2012_val_00022353.JPEG:n03947888 +ILSVRC2012_val_00022354.JPEG:n02093754 +ILSVRC2012_val_00022355.JPEG:n01818515 +ILSVRC2012_val_00022356.JPEG:n03763968 +ILSVRC2012_val_00022357.JPEG:n04380533 +ILSVRC2012_val_00022358.JPEG:n02077923 +ILSVRC2012_val_00022359.JPEG:n02488702 +ILSVRC2012_val_00022360.JPEG:n01770393 +ILSVRC2012_val_00022361.JPEG:n02226429 +ILSVRC2012_val_00022362.JPEG:n07932039 +ILSVRC2012_val_00022363.JPEG:n02095314 +ILSVRC2012_val_00022364.JPEG:n01847000 +ILSVRC2012_val_00022365.JPEG:n03250847 +ILSVRC2012_val_00022366.JPEG:n04296562 +ILSVRC2012_val_00022367.JPEG:n02100236 +ILSVRC2012_val_00022368.JPEG:n03045698 +ILSVRC2012_val_00022369.JPEG:n07590611 +ILSVRC2012_val_00022370.JPEG:n03787032 +ILSVRC2012_val_00022371.JPEG:n02101006 +ILSVRC2012_val_00022372.JPEG:n01873310 +ILSVRC2012_val_00022373.JPEG:n02009912 +ILSVRC2012_val_00022374.JPEG:n02096051 +ILSVRC2012_val_00022375.JPEG:n07749582 +ILSVRC2012_val_00022376.JPEG:n02112018 +ILSVRC2012_val_00022377.JPEG:n03000134 +ILSVRC2012_val_00022378.JPEG:n03447721 +ILSVRC2012_val_00022379.JPEG:n04118776 +ILSVRC2012_val_00022380.JPEG:n03970156 +ILSVRC2012_val_00022381.JPEG:n01944390 +ILSVRC2012_val_00022382.JPEG:n07613480 +ILSVRC2012_val_00022383.JPEG:n02879718 +ILSVRC2012_val_00022384.JPEG:n01873310 +ILSVRC2012_val_00022385.JPEG:n03187595 +ILSVRC2012_val_00022386.JPEG:n03325584 +ILSVRC2012_val_00022387.JPEG:n01496331 +ILSVRC2012_val_00022388.JPEG:n02097298 +ILSVRC2012_val_00022389.JPEG:n03793489 +ILSVRC2012_val_00022390.JPEG:n02111500 +ILSVRC2012_val_00022391.JPEG:n04311174 +ILSVRC2012_val_00022392.JPEG:n01739381 +ILSVRC2012_val_00022393.JPEG:n02114548 +ILSVRC2012_val_00022394.JPEG:n02165105 +ILSVRC2012_val_00022395.JPEG:n01930112 +ILSVRC2012_val_00022396.JPEG:n02823428 +ILSVRC2012_val_00022397.JPEG:n04111531 +ILSVRC2012_val_00022398.JPEG:n02137549 +ILSVRC2012_val_00022399.JPEG:n04355338 +ILSVRC2012_val_00022400.JPEG:n03916031 +ILSVRC2012_val_00022401.JPEG:n03791053 +ILSVRC2012_val_00022402.JPEG:n02113186 +ILSVRC2012_val_00022403.JPEG:n04081281 +ILSVRC2012_val_00022404.JPEG:n02104029 +ILSVRC2012_val_00022405.JPEG:n03483316 +ILSVRC2012_val_00022406.JPEG:n04579145 +ILSVRC2012_val_00022407.JPEG:n01558993 +ILSVRC2012_val_00022408.JPEG:n01748264 +ILSVRC2012_val_00022409.JPEG:n02791270 +ILSVRC2012_val_00022410.JPEG:n03929660 +ILSVRC2012_val_00022411.JPEG:n02129604 +ILSVRC2012_val_00022412.JPEG:n02102040 +ILSVRC2012_val_00022413.JPEG:n03796401 +ILSVRC2012_val_00022414.JPEG:n02007558 +ILSVRC2012_val_00022415.JPEG:n11879895 +ILSVRC2012_val_00022416.JPEG:n06794110 +ILSVRC2012_val_00022417.JPEG:n07614500 +ILSVRC2012_val_00022418.JPEG:n02006656 +ILSVRC2012_val_00022419.JPEG:n04065272 +ILSVRC2012_val_00022420.JPEG:n02486261 +ILSVRC2012_val_00022421.JPEG:n02640242 +ILSVRC2012_val_00022422.JPEG:n01806143 +ILSVRC2012_val_00022423.JPEG:n03991062 +ILSVRC2012_val_00022424.JPEG:n02788148 +ILSVRC2012_val_00022425.JPEG:n09472597 +ILSVRC2012_val_00022426.JPEG:n03935335 +ILSVRC2012_val_00022427.JPEG:n02510455 +ILSVRC2012_val_00022428.JPEG:n03958227 +ILSVRC2012_val_00022429.JPEG:n02105641 +ILSVRC2012_val_00022430.JPEG:n04428191 +ILSVRC2012_val_00022431.JPEG:n03018349 +ILSVRC2012_val_00022432.JPEG:n02116738 +ILSVRC2012_val_00022433.JPEG:n03773504 +ILSVRC2012_val_00022434.JPEG:n02087046 +ILSVRC2012_val_00022435.JPEG:n03709823 +ILSVRC2012_val_00022436.JPEG:n01749939 +ILSVRC2012_val_00022437.JPEG:n02190166 +ILSVRC2012_val_00022438.JPEG:n02085782 +ILSVRC2012_val_00022439.JPEG:n01843065 +ILSVRC2012_val_00022440.JPEG:n03743016 +ILSVRC2012_val_00022441.JPEG:n01828970 +ILSVRC2012_val_00022442.JPEG:n01828970 +ILSVRC2012_val_00022443.JPEG:n03908714 +ILSVRC2012_val_00022444.JPEG:n03937543 +ILSVRC2012_val_00022445.JPEG:n02817516 +ILSVRC2012_val_00022446.JPEG:n04592741 +ILSVRC2012_val_00022447.JPEG:n02869837 +ILSVRC2012_val_00022448.JPEG:n03874293 +ILSVRC2012_val_00022449.JPEG:n04540053 +ILSVRC2012_val_00022450.JPEG:n03250847 +ILSVRC2012_val_00022451.JPEG:n02971356 +ILSVRC2012_val_00022452.JPEG:n02114548 +ILSVRC2012_val_00022453.JPEG:n02113023 +ILSVRC2012_val_00022454.JPEG:n04081281 +ILSVRC2012_val_00022455.JPEG:n03857828 +ILSVRC2012_val_00022456.JPEG:n03450230 +ILSVRC2012_val_00022457.JPEG:n04127249 +ILSVRC2012_val_00022458.JPEG:n02108089 +ILSVRC2012_val_00022459.JPEG:n02093428 +ILSVRC2012_val_00022460.JPEG:n04392985 +ILSVRC2012_val_00022461.JPEG:n04254120 +ILSVRC2012_val_00022462.JPEG:n02782093 +ILSVRC2012_val_00022463.JPEG:n02012849 +ILSVRC2012_val_00022464.JPEG:n03179701 +ILSVRC2012_val_00022465.JPEG:n04357314 +ILSVRC2012_val_00022466.JPEG:n13133613 +ILSVRC2012_val_00022467.JPEG:n02992211 +ILSVRC2012_val_00022468.JPEG:n04243546 +ILSVRC2012_val_00022469.JPEG:n01664065 +ILSVRC2012_val_00022470.JPEG:n01695060 +ILSVRC2012_val_00022471.JPEG:n04005630 +ILSVRC2012_val_00022472.JPEG:n03400231 +ILSVRC2012_val_00022473.JPEG:n03733131 +ILSVRC2012_val_00022474.JPEG:n02107142 +ILSVRC2012_val_00022475.JPEG:n02104365 +ILSVRC2012_val_00022476.JPEG:n04597913 +ILSVRC2012_val_00022477.JPEG:n04238763 +ILSVRC2012_val_00022478.JPEG:n04371430 +ILSVRC2012_val_00022479.JPEG:n03877472 +ILSVRC2012_val_00022480.JPEG:n04589890 +ILSVRC2012_val_00022481.JPEG:n04154565 +ILSVRC2012_val_00022482.JPEG:n01734418 +ILSVRC2012_val_00022483.JPEG:n03781244 +ILSVRC2012_val_00022484.JPEG:n07745940 +ILSVRC2012_val_00022485.JPEG:n02109961 +ILSVRC2012_val_00022486.JPEG:n01755581 +ILSVRC2012_val_00022487.JPEG:n07742313 +ILSVRC2012_val_00022488.JPEG:n04118776 +ILSVRC2012_val_00022489.JPEG:n01734418 +ILSVRC2012_val_00022490.JPEG:n02085782 +ILSVRC2012_val_00022491.JPEG:n03100240 +ILSVRC2012_val_00022492.JPEG:n02013706 +ILSVRC2012_val_00022493.JPEG:n03658185 +ILSVRC2012_val_00022494.JPEG:n03290653 +ILSVRC2012_val_00022495.JPEG:n02105505 +ILSVRC2012_val_00022496.JPEG:n03888257 +ILSVRC2012_val_00022497.JPEG:n02865351 +ILSVRC2012_val_00022498.JPEG:n02277742 +ILSVRC2012_val_00022499.JPEG:n02099849 +ILSVRC2012_val_00022500.JPEG:n03131574 +ILSVRC2012_val_00022501.JPEG:n02102177 +ILSVRC2012_val_00022502.JPEG:n02093428 +ILSVRC2012_val_00022503.JPEG:n02814860 +ILSVRC2012_val_00022504.JPEG:n01734418 +ILSVRC2012_val_00022505.JPEG:n01580077 +ILSVRC2012_val_00022506.JPEG:n04136333 +ILSVRC2012_val_00022507.JPEG:n04483307 +ILSVRC2012_val_00022508.JPEG:n01774384 +ILSVRC2012_val_00022509.JPEG:n02364673 +ILSVRC2012_val_00022510.JPEG:n06874185 +ILSVRC2012_val_00022511.JPEG:n07754684 +ILSVRC2012_val_00022512.JPEG:n07734744 +ILSVRC2012_val_00022513.JPEG:n04487081 +ILSVRC2012_val_00022514.JPEG:n07802026 +ILSVRC2012_val_00022515.JPEG:n09399592 +ILSVRC2012_val_00022516.JPEG:n03602883 +ILSVRC2012_val_00022517.JPEG:n04435653 +ILSVRC2012_val_00022518.JPEG:n02096437 +ILSVRC2012_val_00022519.JPEG:n02672831 +ILSVRC2012_val_00022520.JPEG:n02107683 +ILSVRC2012_val_00022521.JPEG:n02086646 +ILSVRC2012_val_00022522.JPEG:n01698640 +ILSVRC2012_val_00022523.JPEG:n03485794 +ILSVRC2012_val_00022524.JPEG:n03967562 +ILSVRC2012_val_00022525.JPEG:n01664065 +ILSVRC2012_val_00022526.JPEG:n03837869 +ILSVRC2012_val_00022527.JPEG:n01950731 +ILSVRC2012_val_00022528.JPEG:n02909870 +ILSVRC2012_val_00022529.JPEG:n01756291 +ILSVRC2012_val_00022530.JPEG:n02091467 +ILSVRC2012_val_00022531.JPEG:n03658185 +ILSVRC2012_val_00022532.JPEG:n02690373 +ILSVRC2012_val_00022533.JPEG:n02012849 +ILSVRC2012_val_00022534.JPEG:n03709823 +ILSVRC2012_val_00022535.JPEG:n02123597 +ILSVRC2012_val_00022536.JPEG:n13044778 +ILSVRC2012_val_00022537.JPEG:n02167151 +ILSVRC2012_val_00022538.JPEG:n03425413 +ILSVRC2012_val_00022539.JPEG:n07730033 +ILSVRC2012_val_00022540.JPEG:n03721384 +ILSVRC2012_val_00022541.JPEG:n03126707 +ILSVRC2012_val_00022542.JPEG:n02883205 +ILSVRC2012_val_00022543.JPEG:n02111889 +ILSVRC2012_val_00022544.JPEG:n03866082 +ILSVRC2012_val_00022545.JPEG:n01698640 +ILSVRC2012_val_00022546.JPEG:n04584207 +ILSVRC2012_val_00022547.JPEG:n03485407 +ILSVRC2012_val_00022548.JPEG:n02105251 +ILSVRC2012_val_00022549.JPEG:n03743016 +ILSVRC2012_val_00022550.JPEG:n03314780 +ILSVRC2012_val_00022551.JPEG:n03769881 +ILSVRC2012_val_00022552.JPEG:n01494475 +ILSVRC2012_val_00022553.JPEG:n04005630 +ILSVRC2012_val_00022554.JPEG:n03291819 +ILSVRC2012_val_00022555.JPEG:n03721384 +ILSVRC2012_val_00022556.JPEG:n04118776 +ILSVRC2012_val_00022557.JPEG:n03868242 +ILSVRC2012_val_00022558.JPEG:n04265275 +ILSVRC2012_val_00022559.JPEG:n09835506 +ILSVRC2012_val_00022560.JPEG:n03443371 +ILSVRC2012_val_00022561.JPEG:n03459775 +ILSVRC2012_val_00022562.JPEG:n04501370 +ILSVRC2012_val_00022563.JPEG:n01688243 +ILSVRC2012_val_00022564.JPEG:n03494278 +ILSVRC2012_val_00022565.JPEG:n02486410 +ILSVRC2012_val_00022566.JPEG:n02105251 +ILSVRC2012_val_00022567.JPEG:n03956157 +ILSVRC2012_val_00022568.JPEG:n02410509 +ILSVRC2012_val_00022569.JPEG:n02116738 +ILSVRC2012_val_00022570.JPEG:n04532106 +ILSVRC2012_val_00022571.JPEG:n02100236 +ILSVRC2012_val_00022572.JPEG:n04591157 +ILSVRC2012_val_00022573.JPEG:n02398521 +ILSVRC2012_val_00022574.JPEG:n04131690 +ILSVRC2012_val_00022575.JPEG:n03935335 +ILSVRC2012_val_00022576.JPEG:n02098105 +ILSVRC2012_val_00022577.JPEG:n04428191 +ILSVRC2012_val_00022578.JPEG:n02110627 +ILSVRC2012_val_00022579.JPEG:n03970156 +ILSVRC2012_val_00022580.JPEG:n03950228 +ILSVRC2012_val_00022581.JPEG:n02110341 +ILSVRC2012_val_00022582.JPEG:n04201297 +ILSVRC2012_val_00022583.JPEG:n07932039 +ILSVRC2012_val_00022584.JPEG:n07920052 +ILSVRC2012_val_00022585.JPEG:n03063689 +ILSVRC2012_val_00022586.JPEG:n02137549 +ILSVRC2012_val_00022587.JPEG:n03100240 +ILSVRC2012_val_00022588.JPEG:n01665541 +ILSVRC2012_val_00022589.JPEG:n04099969 +ILSVRC2012_val_00022590.JPEG:n02106382 +ILSVRC2012_val_00022591.JPEG:n02009912 +ILSVRC2012_val_00022592.JPEG:n03223299 +ILSVRC2012_val_00022593.JPEG:n02091635 +ILSVRC2012_val_00022594.JPEG:n03982430 +ILSVRC2012_val_00022595.JPEG:n04548362 +ILSVRC2012_val_00022596.JPEG:n01978455 +ILSVRC2012_val_00022597.JPEG:n01614925 +ILSVRC2012_val_00022598.JPEG:n02841315 +ILSVRC2012_val_00022599.JPEG:n07711569 +ILSVRC2012_val_00022600.JPEG:n04335435 +ILSVRC2012_val_00022601.JPEG:n02892767 +ILSVRC2012_val_00022602.JPEG:n03345487 +ILSVRC2012_val_00022603.JPEG:n02948072 +ILSVRC2012_val_00022604.JPEG:n04127249 +ILSVRC2012_val_00022605.JPEG:n02909870 +ILSVRC2012_val_00022606.JPEG:n02099712 +ILSVRC2012_val_00022607.JPEG:n04162706 +ILSVRC2012_val_00022608.JPEG:n01981276 +ILSVRC2012_val_00022609.JPEG:n02085620 +ILSVRC2012_val_00022610.JPEG:n02917067 +ILSVRC2012_val_00022611.JPEG:n07716358 +ILSVRC2012_val_00022612.JPEG:n04332243 +ILSVRC2012_val_00022613.JPEG:n03724870 +ILSVRC2012_val_00022614.JPEG:n04074963 +ILSVRC2012_val_00022615.JPEG:n01984695 +ILSVRC2012_val_00022616.JPEG:n03794056 +ILSVRC2012_val_00022617.JPEG:n03929855 +ILSVRC2012_val_00022618.JPEG:n01773157 +ILSVRC2012_val_00022619.JPEG:n01806567 +ILSVRC2012_val_00022620.JPEG:n04350905 +ILSVRC2012_val_00022621.JPEG:n03804744 +ILSVRC2012_val_00022622.JPEG:n10565667 +ILSVRC2012_val_00022623.JPEG:n07747607 +ILSVRC2012_val_00022624.JPEG:n03218198 +ILSVRC2012_val_00022625.JPEG:n03942813 +ILSVRC2012_val_00022626.JPEG:n01877812 +ILSVRC2012_val_00022627.JPEG:n03924679 +ILSVRC2012_val_00022628.JPEG:n07753592 +ILSVRC2012_val_00022629.JPEG:n02113799 +ILSVRC2012_val_00022630.JPEG:n02086079 +ILSVRC2012_val_00022631.JPEG:n03814639 +ILSVRC2012_val_00022632.JPEG:n02834397 +ILSVRC2012_val_00022633.JPEG:n02109525 +ILSVRC2012_val_00022634.JPEG:n07720875 +ILSVRC2012_val_00022635.JPEG:n04273569 +ILSVRC2012_val_00022636.JPEG:n03018349 +ILSVRC2012_val_00022637.JPEG:n03404251 +ILSVRC2012_val_00022638.JPEG:n03888257 +ILSVRC2012_val_00022639.JPEG:n03485407 +ILSVRC2012_val_00022640.JPEG:n07730033 +ILSVRC2012_val_00022641.JPEG:n13052670 +ILSVRC2012_val_00022642.JPEG:n02095889 +ILSVRC2012_val_00022643.JPEG:n01739381 +ILSVRC2012_val_00022644.JPEG:n01514859 +ILSVRC2012_val_00022645.JPEG:n02106030 +ILSVRC2012_val_00022646.JPEG:n07860988 +ILSVRC2012_val_00022647.JPEG:n03775546 +ILSVRC2012_val_00022648.JPEG:n04263257 +ILSVRC2012_val_00022649.JPEG:n03485794 +ILSVRC2012_val_00022650.JPEG:n03924679 +ILSVRC2012_val_00022651.JPEG:n04228054 +ILSVRC2012_val_00022652.JPEG:n02319095 +ILSVRC2012_val_00022653.JPEG:n02747177 +ILSVRC2012_val_00022654.JPEG:n03770679 +ILSVRC2012_val_00022655.JPEG:n03980874 +ILSVRC2012_val_00022656.JPEG:n02097658 +ILSVRC2012_val_00022657.JPEG:n02988304 +ILSVRC2012_val_00022658.JPEG:n07579787 +ILSVRC2012_val_00022659.JPEG:n02137549 +ILSVRC2012_val_00022660.JPEG:n01644373 +ILSVRC2012_val_00022661.JPEG:n02870880 +ILSVRC2012_val_00022662.JPEG:n04069434 +ILSVRC2012_val_00022663.JPEG:n13040303 +ILSVRC2012_val_00022664.JPEG:n02106550 +ILSVRC2012_val_00022665.JPEG:n02804414 +ILSVRC2012_val_00022666.JPEG:n07565083 +ILSVRC2012_val_00022667.JPEG:n03877845 +ILSVRC2012_val_00022668.JPEG:n03187595 +ILSVRC2012_val_00022669.JPEG:n02074367 +ILSVRC2012_val_00022670.JPEG:n02099712 +ILSVRC2012_val_00022671.JPEG:n01950731 +ILSVRC2012_val_00022672.JPEG:n03884397 +ILSVRC2012_val_00022673.JPEG:n03776460 +ILSVRC2012_val_00022674.JPEG:n04209133 +ILSVRC2012_val_00022675.JPEG:n03697007 +ILSVRC2012_val_00022676.JPEG:n01978287 +ILSVRC2012_val_00022677.JPEG:n03792972 +ILSVRC2012_val_00022678.JPEG:n07716906 +ILSVRC2012_val_00022679.JPEG:n04146614 +ILSVRC2012_val_00022680.JPEG:n03887697 +ILSVRC2012_val_00022681.JPEG:n02095889 +ILSVRC2012_val_00022682.JPEG:n02096177 +ILSVRC2012_val_00022683.JPEG:n04435653 +ILSVRC2012_val_00022684.JPEG:n02091032 +ILSVRC2012_val_00022685.JPEG:n02840245 +ILSVRC2012_val_00022686.JPEG:n02097658 +ILSVRC2012_val_00022687.JPEG:n02002724 +ILSVRC2012_val_00022688.JPEG:n02058221 +ILSVRC2012_val_00022689.JPEG:n03127747 +ILSVRC2012_val_00022690.JPEG:n04501370 +ILSVRC2012_val_00022691.JPEG:n01817953 +ILSVRC2012_val_00022692.JPEG:n02113186 +ILSVRC2012_val_00022693.JPEG:n01877812 +ILSVRC2012_val_00022694.JPEG:n04004767 +ILSVRC2012_val_00022695.JPEG:n02441942 +ILSVRC2012_val_00022696.JPEG:n02408429 +ILSVRC2012_val_00022697.JPEG:n04116512 +ILSVRC2012_val_00022698.JPEG:n02134418 +ILSVRC2012_val_00022699.JPEG:n03529860 +ILSVRC2012_val_00022700.JPEG:n03041632 +ILSVRC2012_val_00022701.JPEG:n03447447 +ILSVRC2012_val_00022702.JPEG:n03188531 +ILSVRC2012_val_00022703.JPEG:n03770439 +ILSVRC2012_val_00022704.JPEG:n03633091 +ILSVRC2012_val_00022705.JPEG:n02086646 +ILSVRC2012_val_00022706.JPEG:n02011460 +ILSVRC2012_val_00022707.JPEG:n04209133 +ILSVRC2012_val_00022708.JPEG:n04229816 +ILSVRC2012_val_00022709.JPEG:n01622779 +ILSVRC2012_val_00022710.JPEG:n01667114 +ILSVRC2012_val_00022711.JPEG:n01685808 +ILSVRC2012_val_00022712.JPEG:n02113186 +ILSVRC2012_val_00022713.JPEG:n02097047 +ILSVRC2012_val_00022714.JPEG:n03876231 +ILSVRC2012_val_00022715.JPEG:n02699494 +ILSVRC2012_val_00022716.JPEG:n03961711 +ILSVRC2012_val_00022717.JPEG:n03530642 +ILSVRC2012_val_00022718.JPEG:n03452741 +ILSVRC2012_val_00022719.JPEG:n02708093 +ILSVRC2012_val_00022720.JPEG:n01985128 +ILSVRC2012_val_00022721.JPEG:n02894605 +ILSVRC2012_val_00022722.JPEG:n03124170 +ILSVRC2012_val_00022723.JPEG:n03633091 +ILSVRC2012_val_00022724.JPEG:n13054560 +ILSVRC2012_val_00022725.JPEG:n02112137 +ILSVRC2012_val_00022726.JPEG:n02120505 +ILSVRC2012_val_00022727.JPEG:n01532829 +ILSVRC2012_val_00022728.JPEG:n03929660 +ILSVRC2012_val_00022729.JPEG:n04589890 +ILSVRC2012_val_00022730.JPEG:n04507155 +ILSVRC2012_val_00022731.JPEG:n01685808 +ILSVRC2012_val_00022732.JPEG:n02077923 +ILSVRC2012_val_00022733.JPEG:n04523525 +ILSVRC2012_val_00022734.JPEG:n04592741 +ILSVRC2012_val_00022735.JPEG:n02056570 +ILSVRC2012_val_00022736.JPEG:n03841143 +ILSVRC2012_val_00022737.JPEG:n02226429 +ILSVRC2012_val_00022738.JPEG:n04243546 +ILSVRC2012_val_00022739.JPEG:n04285008 +ILSVRC2012_val_00022740.JPEG:n02483708 +ILSVRC2012_val_00022741.JPEG:n03944341 +ILSVRC2012_val_00022742.JPEG:n04553703 +ILSVRC2012_val_00022743.JPEG:n03977966 +ILSVRC2012_val_00022744.JPEG:n02441942 +ILSVRC2012_val_00022745.JPEG:n01818515 +ILSVRC2012_val_00022746.JPEG:n03871628 +ILSVRC2012_val_00022747.JPEG:n03692522 +ILSVRC2012_val_00022748.JPEG:n07768694 +ILSVRC2012_val_00022749.JPEG:n02607072 +ILSVRC2012_val_00022750.JPEG:n04456115 +ILSVRC2012_val_00022751.JPEG:n04590129 +ILSVRC2012_val_00022752.JPEG:n03476991 +ILSVRC2012_val_00022753.JPEG:n02091134 +ILSVRC2012_val_00022754.JPEG:n03394916 +ILSVRC2012_val_00022755.JPEG:n01990800 +ILSVRC2012_val_00022756.JPEG:n02066245 +ILSVRC2012_val_00022757.JPEG:n02279972 +ILSVRC2012_val_00022758.JPEG:n01944390 +ILSVRC2012_val_00022759.JPEG:n02105251 +ILSVRC2012_val_00022760.JPEG:n04273569 +ILSVRC2012_val_00022761.JPEG:n03857828 +ILSVRC2012_val_00022762.JPEG:n02110185 +ILSVRC2012_val_00022763.JPEG:n02096051 +ILSVRC2012_val_00022764.JPEG:n01770081 +ILSVRC2012_val_00022765.JPEG:n02259212 +ILSVRC2012_val_00022766.JPEG:n02799071 +ILSVRC2012_val_00022767.JPEG:n01806143 +ILSVRC2012_val_00022768.JPEG:n03476684 +ILSVRC2012_val_00022769.JPEG:n01796340 +ILSVRC2012_val_00022770.JPEG:n03100240 +ILSVRC2012_val_00022771.JPEG:n01632777 +ILSVRC2012_val_00022772.JPEG:n02190166 +ILSVRC2012_val_00022773.JPEG:n02066245 +ILSVRC2012_val_00022774.JPEG:n03976657 +ILSVRC2012_val_00022775.JPEG:n03788365 +ILSVRC2012_val_00022776.JPEG:n02108422 +ILSVRC2012_val_00022777.JPEG:n03400231 +ILSVRC2012_val_00022778.JPEG:n04589890 +ILSVRC2012_val_00022779.JPEG:n04435653 +ILSVRC2012_val_00022780.JPEG:n02326432 +ILSVRC2012_val_00022781.JPEG:n03954731 +ILSVRC2012_val_00022782.JPEG:n04591157 +ILSVRC2012_val_00022783.JPEG:n02823428 +ILSVRC2012_val_00022784.JPEG:n07716358 +ILSVRC2012_val_00022785.JPEG:n02088632 +ILSVRC2012_val_00022786.JPEG:n01824575 +ILSVRC2012_val_00022787.JPEG:n01631663 +ILSVRC2012_val_00022788.JPEG:n02086079 +ILSVRC2012_val_00022789.JPEG:n03995372 +ILSVRC2012_val_00022790.JPEG:n04517823 +ILSVRC2012_val_00022791.JPEG:n02480855 +ILSVRC2012_val_00022792.JPEG:n03445777 +ILSVRC2012_val_00022793.JPEG:n04357314 +ILSVRC2012_val_00022794.JPEG:n03884397 +ILSVRC2012_val_00022795.JPEG:n03445924 +ILSVRC2012_val_00022796.JPEG:n03777754 +ILSVRC2012_val_00022797.JPEG:n03133878 +ILSVRC2012_val_00022798.JPEG:n03873416 +ILSVRC2012_val_00022799.JPEG:n02086240 +ILSVRC2012_val_00022800.JPEG:n04553703 +ILSVRC2012_val_00022801.JPEG:n04133789 +ILSVRC2012_val_00022802.JPEG:n07693725 +ILSVRC2012_val_00022803.JPEG:n02895154 +ILSVRC2012_val_00022804.JPEG:n02317335 +ILSVRC2012_val_00022805.JPEG:n04613696 +ILSVRC2012_val_00022806.JPEG:n01819313 +ILSVRC2012_val_00022807.JPEG:n03977966 +ILSVRC2012_val_00022808.JPEG:n02109047 +ILSVRC2012_val_00022809.JPEG:n03000247 +ILSVRC2012_val_00022810.JPEG:n02443114 +ILSVRC2012_val_00022811.JPEG:n03272010 +ILSVRC2012_val_00022812.JPEG:n01697457 +ILSVRC2012_val_00022813.JPEG:n04200800 +ILSVRC2012_val_00022814.JPEG:n02109047 +ILSVRC2012_val_00022815.JPEG:n02840245 +ILSVRC2012_val_00022816.JPEG:n01739381 +ILSVRC2012_val_00022817.JPEG:n06794110 +ILSVRC2012_val_00022818.JPEG:n01756291 +ILSVRC2012_val_00022819.JPEG:n01748264 +ILSVRC2012_val_00022820.JPEG:n03950228 +ILSVRC2012_val_00022821.JPEG:n02971356 +ILSVRC2012_val_00022822.JPEG:n02123159 +ILSVRC2012_val_00022823.JPEG:n04346328 +ILSVRC2012_val_00022824.JPEG:n02092339 +ILSVRC2012_val_00022825.JPEG:n01729977 +ILSVRC2012_val_00022826.JPEG:n03187595 +ILSVRC2012_val_00022827.JPEG:n02454379 +ILSVRC2012_val_00022828.JPEG:n03794056 +ILSVRC2012_val_00022829.JPEG:n03967562 +ILSVRC2012_val_00022830.JPEG:n04039381 +ILSVRC2012_val_00022831.JPEG:n02879718 +ILSVRC2012_val_00022832.JPEG:n02441942 +ILSVRC2012_val_00022833.JPEG:n04515003 +ILSVRC2012_val_00022834.JPEG:n04311174 +ILSVRC2012_val_00022835.JPEG:n03100240 +ILSVRC2012_val_00022836.JPEG:n03868242 +ILSVRC2012_val_00022837.JPEG:n03126707 +ILSVRC2012_val_00022838.JPEG:n04461696 +ILSVRC2012_val_00022839.JPEG:n13054560 +ILSVRC2012_val_00022840.JPEG:n04398044 +ILSVRC2012_val_00022841.JPEG:n01667114 +ILSVRC2012_val_00022842.JPEG:n01664065 +ILSVRC2012_val_00022843.JPEG:n02106382 +ILSVRC2012_val_00022844.JPEG:n04613696 +ILSVRC2012_val_00022845.JPEG:n02948072 +ILSVRC2012_val_00022846.JPEG:n12144580 +ILSVRC2012_val_00022847.JPEG:n03877472 +ILSVRC2012_val_00022848.JPEG:n02096585 +ILSVRC2012_val_00022849.JPEG:n03935335 +ILSVRC2012_val_00022850.JPEG:n04429376 +ILSVRC2012_val_00022851.JPEG:n02110185 +ILSVRC2012_val_00022852.JPEG:n03207941 +ILSVRC2012_val_00022853.JPEG:n02123045 +ILSVRC2012_val_00022854.JPEG:n03788195 +ILSVRC2012_val_00022855.JPEG:n04259630 +ILSVRC2012_val_00022856.JPEG:n02097209 +ILSVRC2012_val_00022857.JPEG:n02092002 +ILSVRC2012_val_00022858.JPEG:n01877812 +ILSVRC2012_val_00022859.JPEG:n03529860 +ILSVRC2012_val_00022860.JPEG:n02966687 +ILSVRC2012_val_00022861.JPEG:n03980874 +ILSVRC2012_val_00022862.JPEG:n02013706 +ILSVRC2012_val_00022863.JPEG:n02776631 +ILSVRC2012_val_00022864.JPEG:n02445715 +ILSVRC2012_val_00022865.JPEG:n01496331 +ILSVRC2012_val_00022866.JPEG:n01807496 +ILSVRC2012_val_00022867.JPEG:n02112137 +ILSVRC2012_val_00022868.JPEG:n02086646 +ILSVRC2012_val_00022869.JPEG:n04118776 +ILSVRC2012_val_00022870.JPEG:n03658185 +ILSVRC2012_val_00022871.JPEG:n01985128 +ILSVRC2012_val_00022872.JPEG:n02504013 +ILSVRC2012_val_00022873.JPEG:n12998815 +ILSVRC2012_val_00022874.JPEG:n02233338 +ILSVRC2012_val_00022875.JPEG:n12057211 +ILSVRC2012_val_00022876.JPEG:n07875152 +ILSVRC2012_val_00022877.JPEG:n03840681 +ILSVRC2012_val_00022878.JPEG:n03721384 +ILSVRC2012_val_00022879.JPEG:n03908714 +ILSVRC2012_val_00022880.JPEG:n02412080 +ILSVRC2012_val_00022881.JPEG:n02113799 +ILSVRC2012_val_00022882.JPEG:n02096437 +ILSVRC2012_val_00022883.JPEG:n02669723 +ILSVRC2012_val_00022884.JPEG:n03775546 +ILSVRC2012_val_00022885.JPEG:n03393912 +ILSVRC2012_val_00022886.JPEG:n07718472 +ILSVRC2012_val_00022887.JPEG:n01883070 +ILSVRC2012_val_00022888.JPEG:n02120079 +ILSVRC2012_val_00022889.JPEG:n01532829 +ILSVRC2012_val_00022890.JPEG:n04443257 +ILSVRC2012_val_00022891.JPEG:n02917067 +ILSVRC2012_val_00022892.JPEG:n02877765 +ILSVRC2012_val_00022893.JPEG:n02115913 +ILSVRC2012_val_00022894.JPEG:n07920052 +ILSVRC2012_val_00022895.JPEG:n01773797 +ILSVRC2012_val_00022896.JPEG:n02123159 +ILSVRC2012_val_00022897.JPEG:n03447447 +ILSVRC2012_val_00022898.JPEG:n04613696 +ILSVRC2012_val_00022899.JPEG:n03933933 +ILSVRC2012_val_00022900.JPEG:n04380533 +ILSVRC2012_val_00022901.JPEG:n01728572 +ILSVRC2012_val_00022902.JPEG:n03535780 +ILSVRC2012_val_00022903.JPEG:n04599235 +ILSVRC2012_val_00022904.JPEG:n02877765 +ILSVRC2012_val_00022905.JPEG:n13037406 +ILSVRC2012_val_00022906.JPEG:n02971356 +ILSVRC2012_val_00022907.JPEG:n02504458 +ILSVRC2012_val_00022908.JPEG:n02101388 +ILSVRC2012_val_00022909.JPEG:n04370456 +ILSVRC2012_val_00022910.JPEG:n09229709 +ILSVRC2012_val_00022911.JPEG:n02113624 +ILSVRC2012_val_00022912.JPEG:n02492035 +ILSVRC2012_val_00022913.JPEG:n02089867 +ILSVRC2012_val_00022914.JPEG:n09421951 +ILSVRC2012_val_00022915.JPEG:n02219486 +ILSVRC2012_val_00022916.JPEG:n02494079 +ILSVRC2012_val_00022917.JPEG:n02963159 +ILSVRC2012_val_00022918.JPEG:n03930630 +ILSVRC2012_val_00022919.JPEG:n02206856 +ILSVRC2012_val_00022920.JPEG:n02091831 +ILSVRC2012_val_00022921.JPEG:n02504013 +ILSVRC2012_val_00022922.JPEG:n02097298 +ILSVRC2012_val_00022923.JPEG:n09428293 +ILSVRC2012_val_00022924.JPEG:n04596742 +ILSVRC2012_val_00022925.JPEG:n01632777 +ILSVRC2012_val_00022926.JPEG:n02018207 +ILSVRC2012_val_00022927.JPEG:n03344393 +ILSVRC2012_val_00022928.JPEG:n03388549 +ILSVRC2012_val_00022929.JPEG:n03791053 +ILSVRC2012_val_00022930.JPEG:n01729322 +ILSVRC2012_val_00022931.JPEG:n02018207 +ILSVRC2012_val_00022932.JPEG:n03599486 +ILSVRC2012_val_00022933.JPEG:n03297495 +ILSVRC2012_val_00022934.JPEG:n02093859 +ILSVRC2012_val_00022935.JPEG:n01629819 +ILSVRC2012_val_00022936.JPEG:n04037443 +ILSVRC2012_val_00022937.JPEG:n01693334 +ILSVRC2012_val_00022938.JPEG:n02058221 +ILSVRC2012_val_00022939.JPEG:n03141823 +ILSVRC2012_val_00022940.JPEG:n04252225 +ILSVRC2012_val_00022941.JPEG:n04418357 +ILSVRC2012_val_00022942.JPEG:n01774384 +ILSVRC2012_val_00022943.JPEG:n03871628 +ILSVRC2012_val_00022944.JPEG:n03598930 +ILSVRC2012_val_00022945.JPEG:n03032252 +ILSVRC2012_val_00022946.JPEG:n02321529 +ILSVRC2012_val_00022947.JPEG:n02117135 +ILSVRC2012_val_00022948.JPEG:n02206856 +ILSVRC2012_val_00022949.JPEG:n03944341 +ILSVRC2012_val_00022950.JPEG:n02111129 +ILSVRC2012_val_00022951.JPEG:n02346627 +ILSVRC2012_val_00022952.JPEG:n03404251 +ILSVRC2012_val_00022953.JPEG:n02113023 +ILSVRC2012_val_00022954.JPEG:n02009229 +ILSVRC2012_val_00022955.JPEG:n02879718 +ILSVRC2012_val_00022956.JPEG:n01748264 +ILSVRC2012_val_00022957.JPEG:n01773549 +ILSVRC2012_val_00022958.JPEG:n04252077 +ILSVRC2012_val_00022959.JPEG:n02825657 +ILSVRC2012_val_00022960.JPEG:n03476991 +ILSVRC2012_val_00022961.JPEG:n03584254 +ILSVRC2012_val_00022962.JPEG:n04350905 +ILSVRC2012_val_00022963.JPEG:n13052670 +ILSVRC2012_val_00022964.JPEG:n04141076 +ILSVRC2012_val_00022965.JPEG:n03388549 +ILSVRC2012_val_00022966.JPEG:n02415577 +ILSVRC2012_val_00022967.JPEG:n02607072 +ILSVRC2012_val_00022968.JPEG:n04346328 +ILSVRC2012_val_00022969.JPEG:n01914609 +ILSVRC2012_val_00022970.JPEG:n02641379 +ILSVRC2012_val_00022971.JPEG:n03782006 +ILSVRC2012_val_00022972.JPEG:n01601694 +ILSVRC2012_val_00022973.JPEG:n03388183 +ILSVRC2012_val_00022974.JPEG:n03803284 +ILSVRC2012_val_00022975.JPEG:n02690373 +ILSVRC2012_val_00022976.JPEG:n02106662 +ILSVRC2012_val_00022977.JPEG:n02097047 +ILSVRC2012_val_00022978.JPEG:n07892512 +ILSVRC2012_val_00022979.JPEG:n02277742 +ILSVRC2012_val_00022980.JPEG:n10148035 +ILSVRC2012_val_00022981.JPEG:n02412080 +ILSVRC2012_val_00022982.JPEG:n02091635 +ILSVRC2012_val_00022983.JPEG:n01917289 +ILSVRC2012_val_00022984.JPEG:n03742115 +ILSVRC2012_val_00022985.JPEG:n04074963 +ILSVRC2012_val_00022986.JPEG:n03124043 +ILSVRC2012_val_00022987.JPEG:n02669723 +ILSVRC2012_val_00022988.JPEG:n04507155 +ILSVRC2012_val_00022989.JPEG:n02808304 +ILSVRC2012_val_00022990.JPEG:n02111500 +ILSVRC2012_val_00022991.JPEG:n03761084 +ILSVRC2012_val_00022992.JPEG:n01797886 +ILSVRC2012_val_00022993.JPEG:n03874599 +ILSVRC2012_val_00022994.JPEG:n03476991 +ILSVRC2012_val_00022995.JPEG:n04404412 +ILSVRC2012_val_00022996.JPEG:n02108915 +ILSVRC2012_val_00022997.JPEG:n01694178 +ILSVRC2012_val_00022998.JPEG:n02802426 +ILSVRC2012_val_00022999.JPEG:n02974003 +ILSVRC2012_val_00023000.JPEG:n03028079 +ILSVRC2012_val_00023001.JPEG:n03944341 +ILSVRC2012_val_00023002.JPEG:n03742115 +ILSVRC2012_val_00023003.JPEG:n02111500 +ILSVRC2012_val_00023004.JPEG:n02117135 +ILSVRC2012_val_00023005.JPEG:n02092339 +ILSVRC2012_val_00023006.JPEG:n04133789 +ILSVRC2012_val_00023007.JPEG:n03868242 +ILSVRC2012_val_00023008.JPEG:n07714990 +ILSVRC2012_val_00023009.JPEG:n07579787 +ILSVRC2012_val_00023010.JPEG:n04252077 +ILSVRC2012_val_00023011.JPEG:n02096051 +ILSVRC2012_val_00023012.JPEG:n02102480 +ILSVRC2012_val_00023013.JPEG:n02174001 +ILSVRC2012_val_00023014.JPEG:n03085013 +ILSVRC2012_val_00023015.JPEG:n01740131 +ILSVRC2012_val_00023016.JPEG:n02107312 +ILSVRC2012_val_00023017.JPEG:n04162706 +ILSVRC2012_val_00023018.JPEG:n02869837 +ILSVRC2012_val_00023019.JPEG:n02412080 +ILSVRC2012_val_00023020.JPEG:n04612504 +ILSVRC2012_val_00023021.JPEG:n01807496 +ILSVRC2012_val_00023022.JPEG:n04041544 +ILSVRC2012_val_00023023.JPEG:n03459775 +ILSVRC2012_val_00023024.JPEG:n02017213 +ILSVRC2012_val_00023025.JPEG:n02101006 +ILSVRC2012_val_00023026.JPEG:n07749582 +ILSVRC2012_val_00023027.JPEG:n02109047 +ILSVRC2012_val_00023028.JPEG:n07718472 +ILSVRC2012_val_00023029.JPEG:n02877765 +ILSVRC2012_val_00023030.JPEG:n01622779 +ILSVRC2012_val_00023031.JPEG:n01882714 +ILSVRC2012_val_00023032.JPEG:n03781244 +ILSVRC2012_val_00023033.JPEG:n02137549 +ILSVRC2012_val_00023034.JPEG:n02342885 +ILSVRC2012_val_00023035.JPEG:n03498962 +ILSVRC2012_val_00023036.JPEG:n04127249 +ILSVRC2012_val_00023037.JPEG:n06785654 +ILSVRC2012_val_00023038.JPEG:n02105412 +ILSVRC2012_val_00023039.JPEG:n03447447 +ILSVRC2012_val_00023040.JPEG:n09193705 +ILSVRC2012_val_00023041.JPEG:n02326432 +ILSVRC2012_val_00023042.JPEG:n04590129 +ILSVRC2012_val_00023043.JPEG:n02892201 +ILSVRC2012_val_00023044.JPEG:n03425413 +ILSVRC2012_val_00023045.JPEG:n04235860 +ILSVRC2012_val_00023046.JPEG:n03000247 +ILSVRC2012_val_00023047.JPEG:n03272562 +ILSVRC2012_val_00023048.JPEG:n03598930 +ILSVRC2012_val_00023049.JPEG:n02174001 +ILSVRC2012_val_00023050.JPEG:n03347037 +ILSVRC2012_val_00023051.JPEG:n07920052 +ILSVRC2012_val_00023052.JPEG:n01784675 +ILSVRC2012_val_00023053.JPEG:n07718747 +ILSVRC2012_val_00023054.JPEG:n02279972 +ILSVRC2012_val_00023055.JPEG:n02097298 +ILSVRC2012_val_00023056.JPEG:n03394916 +ILSVRC2012_val_00023057.JPEG:n03977966 +ILSVRC2012_val_00023058.JPEG:n03692522 +ILSVRC2012_val_00023059.JPEG:n03825788 +ILSVRC2012_val_00023060.JPEG:n07717556 +ILSVRC2012_val_00023061.JPEG:n02727426 +ILSVRC2012_val_00023062.JPEG:n02396427 +ILSVRC2012_val_00023063.JPEG:n07747607 +ILSVRC2012_val_00023064.JPEG:n04330267 +ILSVRC2012_val_00023065.JPEG:n03062245 +ILSVRC2012_val_00023066.JPEG:n02389026 +ILSVRC2012_val_00023067.JPEG:n02871525 +ILSVRC2012_val_00023068.JPEG:n02107142 +ILSVRC2012_val_00023069.JPEG:n02012849 +ILSVRC2012_val_00023070.JPEG:n02077923 +ILSVRC2012_val_00023071.JPEG:n03532672 +ILSVRC2012_val_00023072.JPEG:n03216828 +ILSVRC2012_val_00023073.JPEG:n02486261 +ILSVRC2012_val_00023074.JPEG:n01494475 +ILSVRC2012_val_00023075.JPEG:n04251144 +ILSVRC2012_val_00023076.JPEG:n02109047 +ILSVRC2012_val_00023077.JPEG:n03649909 +ILSVRC2012_val_00023078.JPEG:n01873310 +ILSVRC2012_val_00023079.JPEG:n03710637 +ILSVRC2012_val_00023080.JPEG:n01632458 +ILSVRC2012_val_00023081.JPEG:n02077923 +ILSVRC2012_val_00023082.JPEG:n04263257 +ILSVRC2012_val_00023083.JPEG:n04423845 +ILSVRC2012_val_00023084.JPEG:n02279972 +ILSVRC2012_val_00023085.JPEG:n01728572 +ILSVRC2012_val_00023086.JPEG:n02128757 +ILSVRC2012_val_00023087.JPEG:n04552348 +ILSVRC2012_val_00023088.JPEG:n07747607 +ILSVRC2012_val_00023089.JPEG:n07932039 +ILSVRC2012_val_00023090.JPEG:n02071294 +ILSVRC2012_val_00023091.JPEG:n02951585 +ILSVRC2012_val_00023092.JPEG:n02123159 +ILSVRC2012_val_00023093.JPEG:n04201297 +ILSVRC2012_val_00023094.JPEG:n03680355 +ILSVRC2012_val_00023095.JPEG:n02892767 +ILSVRC2012_val_00023096.JPEG:n03930630 +ILSVRC2012_val_00023097.JPEG:n01798484 +ILSVRC2012_val_00023098.JPEG:n01729977 +ILSVRC2012_val_00023099.JPEG:n01798484 +ILSVRC2012_val_00023100.JPEG:n04371430 +ILSVRC2012_val_00023101.JPEG:n02090379 +ILSVRC2012_val_00023102.JPEG:n03347037 +ILSVRC2012_val_00023103.JPEG:n03998194 +ILSVRC2012_val_00023104.JPEG:n03947888 +ILSVRC2012_val_00023105.JPEG:n02108422 +ILSVRC2012_val_00023106.JPEG:n02837789 +ILSVRC2012_val_00023107.JPEG:n03888257 +ILSVRC2012_val_00023108.JPEG:n01739381 +ILSVRC2012_val_00023109.JPEG:n04179913 +ILSVRC2012_val_00023110.JPEG:n07590611 +ILSVRC2012_val_00023111.JPEG:n02279972 +ILSVRC2012_val_00023112.JPEG:n03063599 +ILSVRC2012_val_00023113.JPEG:n02113712 +ILSVRC2012_val_00023114.JPEG:n02444819 +ILSVRC2012_val_00023115.JPEG:n03532672 +ILSVRC2012_val_00023116.JPEG:n02687172 +ILSVRC2012_val_00023117.JPEG:n07720875 +ILSVRC2012_val_00023118.JPEG:n01819313 +ILSVRC2012_val_00023119.JPEG:n02445715 +ILSVRC2012_val_00023120.JPEG:n03793489 +ILSVRC2012_val_00023121.JPEG:n02092002 +ILSVRC2012_val_00023122.JPEG:n03899768 +ILSVRC2012_val_00023123.JPEG:n03424325 +ILSVRC2012_val_00023124.JPEG:n02978881 +ILSVRC2012_val_00023125.JPEG:n01534433 +ILSVRC2012_val_00023126.JPEG:n02999410 +ILSVRC2012_val_00023127.JPEG:n04557648 +ILSVRC2012_val_00023128.JPEG:n01608432 +ILSVRC2012_val_00023129.JPEG:n02391049 +ILSVRC2012_val_00023130.JPEG:n03929660 +ILSVRC2012_val_00023131.JPEG:n02835271 +ILSVRC2012_val_00023132.JPEG:n03876231 +ILSVRC2012_val_00023133.JPEG:n02102318 +ILSVRC2012_val_00023134.JPEG:n02777292 +ILSVRC2012_val_00023135.JPEG:n04004767 +ILSVRC2012_val_00023136.JPEG:n03933933 +ILSVRC2012_val_00023137.JPEG:n07836838 +ILSVRC2012_val_00023138.JPEG:n01751748 +ILSVRC2012_val_00023139.JPEG:n07718472 +ILSVRC2012_val_00023140.JPEG:n04254777 +ILSVRC2012_val_00023141.JPEG:n03424325 +ILSVRC2012_val_00023142.JPEG:n03063599 +ILSVRC2012_val_00023143.JPEG:n02095570 +ILSVRC2012_val_00023144.JPEG:n01824575 +ILSVRC2012_val_00023145.JPEG:n04311004 +ILSVRC2012_val_00023146.JPEG:n01677366 +ILSVRC2012_val_00023147.JPEG:n03062245 +ILSVRC2012_val_00023148.JPEG:n03627232 +ILSVRC2012_val_00023149.JPEG:n03134739 +ILSVRC2012_val_00023150.JPEG:n04372370 +ILSVRC2012_val_00023151.JPEG:n03075370 +ILSVRC2012_val_00023152.JPEG:n02802426 +ILSVRC2012_val_00023153.JPEG:n03447721 +ILSVRC2012_val_00023154.JPEG:n01829413 +ILSVRC2012_val_00023155.JPEG:n02090379 +ILSVRC2012_val_00023156.JPEG:n04192698 +ILSVRC2012_val_00023157.JPEG:n03743016 +ILSVRC2012_val_00023158.JPEG:n01692333 +ILSVRC2012_val_00023159.JPEG:n02099601 +ILSVRC2012_val_00023160.JPEG:n03720891 +ILSVRC2012_val_00023161.JPEG:n02951585 +ILSVRC2012_val_00023162.JPEG:n01532829 +ILSVRC2012_val_00023163.JPEG:n02281406 +ILSVRC2012_val_00023164.JPEG:n02096177 +ILSVRC2012_val_00023165.JPEG:n03920288 +ILSVRC2012_val_00023166.JPEG:n02927161 +ILSVRC2012_val_00023167.JPEG:n04179913 +ILSVRC2012_val_00023168.JPEG:n02100236 +ILSVRC2012_val_00023169.JPEG:n04515003 +ILSVRC2012_val_00023170.JPEG:n07802026 +ILSVRC2012_val_00023171.JPEG:n02088632 +ILSVRC2012_val_00023172.JPEG:n03950228 +ILSVRC2012_val_00023173.JPEG:n09193705 +ILSVRC2012_val_00023174.JPEG:n03841143 +ILSVRC2012_val_00023175.JPEG:n02093647 +ILSVRC2012_val_00023176.JPEG:n04336792 +ILSVRC2012_val_00023177.JPEG:n04357314 +ILSVRC2012_val_00023178.JPEG:n03929660 +ILSVRC2012_val_00023179.JPEG:n02093647 +ILSVRC2012_val_00023180.JPEG:n02093428 +ILSVRC2012_val_00023181.JPEG:n04049303 +ILSVRC2012_val_00023182.JPEG:n01873310 +ILSVRC2012_val_00023183.JPEG:n02268853 +ILSVRC2012_val_00023184.JPEG:n03838899 +ILSVRC2012_val_00023185.JPEG:n01484850 +ILSVRC2012_val_00023186.JPEG:n03337140 +ILSVRC2012_val_00023187.JPEG:n01537544 +ILSVRC2012_val_00023188.JPEG:n02174001 +ILSVRC2012_val_00023189.JPEG:n03063599 +ILSVRC2012_val_00023190.JPEG:n02640242 +ILSVRC2012_val_00023191.JPEG:n03721384 +ILSVRC2012_val_00023192.JPEG:n04596742 +ILSVRC2012_val_00023193.JPEG:n02795169 +ILSVRC2012_val_00023194.JPEG:n02492660 +ILSVRC2012_val_00023195.JPEG:n02892201 +ILSVRC2012_val_00023196.JPEG:n02361337 +ILSVRC2012_val_00023197.JPEG:n04417672 +ILSVRC2012_val_00023198.JPEG:n02113624 +ILSVRC2012_val_00023199.JPEG:n02028035 +ILSVRC2012_val_00023200.JPEG:n02999410 +ILSVRC2012_val_00023201.JPEG:n01629819 +ILSVRC2012_val_00023202.JPEG:n02115913 +ILSVRC2012_val_00023203.JPEG:n02089078 +ILSVRC2012_val_00023204.JPEG:n01768244 +ILSVRC2012_val_00023205.JPEG:n04263257 +ILSVRC2012_val_00023206.JPEG:n01944390 +ILSVRC2012_val_00023207.JPEG:n01945685 +ILSVRC2012_val_00023208.JPEG:n02071294 +ILSVRC2012_val_00023209.JPEG:n03937543 +ILSVRC2012_val_00023210.JPEG:n02391049 +ILSVRC2012_val_00023211.JPEG:n02018207 +ILSVRC2012_val_00023212.JPEG:n02129165 +ILSVRC2012_val_00023213.JPEG:n02074367 +ILSVRC2012_val_00023214.JPEG:n01518878 +ILSVRC2012_val_00023215.JPEG:n03445777 +ILSVRC2012_val_00023216.JPEG:n04149813 +ILSVRC2012_val_00023217.JPEG:n02669723 +ILSVRC2012_val_00023218.JPEG:n02097047 +ILSVRC2012_val_00023219.JPEG:n02865351 +ILSVRC2012_val_00023220.JPEG:n07753592 +ILSVRC2012_val_00023221.JPEG:n02814533 +ILSVRC2012_val_00023222.JPEG:n03874599 +ILSVRC2012_val_00023223.JPEG:n07720875 +ILSVRC2012_val_00023224.JPEG:n04116512 +ILSVRC2012_val_00023225.JPEG:n02417914 +ILSVRC2012_val_00023226.JPEG:n02027492 +ILSVRC2012_val_00023227.JPEG:n03877845 +ILSVRC2012_val_00023228.JPEG:n02123159 +ILSVRC2012_val_00023229.JPEG:n04264628 +ILSVRC2012_val_00023230.JPEG:n02236044 +ILSVRC2012_val_00023231.JPEG:n02108089 +ILSVRC2012_val_00023232.JPEG:n04133789 +ILSVRC2012_val_00023233.JPEG:n04147183 +ILSVRC2012_val_00023234.JPEG:n02085620 +ILSVRC2012_val_00023235.JPEG:n02091134 +ILSVRC2012_val_00023236.JPEG:n03944341 +ILSVRC2012_val_00023237.JPEG:n13037406 +ILSVRC2012_val_00023238.JPEG:n02422106 +ILSVRC2012_val_00023239.JPEG:n01498041 +ILSVRC2012_val_00023240.JPEG:n03775071 +ILSVRC2012_val_00023241.JPEG:n04357314 +ILSVRC2012_val_00023242.JPEG:n02102040 +ILSVRC2012_val_00023243.JPEG:n01682714 +ILSVRC2012_val_00023244.JPEG:n01775062 +ILSVRC2012_val_00023245.JPEG:n03014705 +ILSVRC2012_val_00023246.JPEG:n01693334 +ILSVRC2012_val_00023247.JPEG:n01616318 +ILSVRC2012_val_00023248.JPEG:n04604644 +ILSVRC2012_val_00023249.JPEG:n03109150 +ILSVRC2012_val_00023250.JPEG:n02088238 +ILSVRC2012_val_00023251.JPEG:n01981276 +ILSVRC2012_val_00023252.JPEG:n02422106 +ILSVRC2012_val_00023253.JPEG:n01985128 +ILSVRC2012_val_00023254.JPEG:n04026417 +ILSVRC2012_val_00023255.JPEG:n01644900 +ILSVRC2012_val_00023256.JPEG:n02095570 +ILSVRC2012_val_00023257.JPEG:n04266014 +ILSVRC2012_val_00023258.JPEG:n02236044 +ILSVRC2012_val_00023259.JPEG:n02115913 +ILSVRC2012_val_00023260.JPEG:n01883070 +ILSVRC2012_val_00023261.JPEG:n03840681 +ILSVRC2012_val_00023262.JPEG:n02481823 +ILSVRC2012_val_00023263.JPEG:n03447721 +ILSVRC2012_val_00023264.JPEG:n01981276 +ILSVRC2012_val_00023265.JPEG:n03673027 +ILSVRC2012_val_00023266.JPEG:n02835271 +ILSVRC2012_val_00023267.JPEG:n02123159 +ILSVRC2012_val_00023268.JPEG:n02113186 +ILSVRC2012_val_00023269.JPEG:n03947888 +ILSVRC2012_val_00023270.JPEG:n02100877 +ILSVRC2012_val_00023271.JPEG:n03814639 +ILSVRC2012_val_00023272.JPEG:n02510455 +ILSVRC2012_val_00023273.JPEG:n04037443 +ILSVRC2012_val_00023274.JPEG:n03929660 +ILSVRC2012_val_00023275.JPEG:n03837869 +ILSVRC2012_val_00023276.JPEG:n02791270 +ILSVRC2012_val_00023277.JPEG:n03461385 +ILSVRC2012_val_00023278.JPEG:n02951585 +ILSVRC2012_val_00023279.JPEG:n04525305 +ILSVRC2012_val_00023280.JPEG:n02788148 +ILSVRC2012_val_00023281.JPEG:n02165105 +ILSVRC2012_val_00023282.JPEG:n04592741 +ILSVRC2012_val_00023283.JPEG:n02091467 +ILSVRC2012_val_00023284.JPEG:n03188531 +ILSVRC2012_val_00023285.JPEG:n02091134 +ILSVRC2012_val_00023286.JPEG:n03617480 +ILSVRC2012_val_00023287.JPEG:n03954731 +ILSVRC2012_val_00023288.JPEG:n04328186 +ILSVRC2012_val_00023289.JPEG:n02105162 +ILSVRC2012_val_00023290.JPEG:n02870880 +ILSVRC2012_val_00023291.JPEG:n03028079 +ILSVRC2012_val_00023292.JPEG:n04596742 +ILSVRC2012_val_00023293.JPEG:n04204347 +ILSVRC2012_val_00023294.JPEG:n02108422 +ILSVRC2012_val_00023295.JPEG:n01740131 +ILSVRC2012_val_00023296.JPEG:n02363005 +ILSVRC2012_val_00023297.JPEG:n03840681 +ILSVRC2012_val_00023298.JPEG:n04116512 +ILSVRC2012_val_00023299.JPEG:n02138441 +ILSVRC2012_val_00023300.JPEG:n04367480 +ILSVRC2012_val_00023301.JPEG:n01773797 +ILSVRC2012_val_00023302.JPEG:n04350905 +ILSVRC2012_val_00023303.JPEG:n02095314 +ILSVRC2012_val_00023304.JPEG:n09229709 +ILSVRC2012_val_00023305.JPEG:n02494079 +ILSVRC2012_val_00023306.JPEG:n03788365 +ILSVRC2012_val_00023307.JPEG:n02117135 +ILSVRC2012_val_00023308.JPEG:n01641577 +ILSVRC2012_val_00023309.JPEG:n04192698 +ILSVRC2012_val_00023310.JPEG:n02087046 +ILSVRC2012_val_00023311.JPEG:n12620546 +ILSVRC2012_val_00023312.JPEG:n02410509 +ILSVRC2012_val_00023313.JPEG:n03777568 +ILSVRC2012_val_00023314.JPEG:n02948072 +ILSVRC2012_val_00023315.JPEG:n03662601 +ILSVRC2012_val_00023316.JPEG:n02690373 +ILSVRC2012_val_00023317.JPEG:n02441942 +ILSVRC2012_val_00023318.JPEG:n03127925 +ILSVRC2012_val_00023319.JPEG:n02066245 +ILSVRC2012_val_00023320.JPEG:n02097130 +ILSVRC2012_val_00023321.JPEG:n03187595 +ILSVRC2012_val_00023322.JPEG:n02977058 +ILSVRC2012_val_00023323.JPEG:n03977966 +ILSVRC2012_val_00023324.JPEG:n03291819 +ILSVRC2012_val_00023325.JPEG:n02788148 +ILSVRC2012_val_00023326.JPEG:n03482405 +ILSVRC2012_val_00023327.JPEG:n02090721 +ILSVRC2012_val_00023328.JPEG:n02105641 +ILSVRC2012_val_00023329.JPEG:n04525038 +ILSVRC2012_val_00023330.JPEG:n04328186 +ILSVRC2012_val_00023331.JPEG:n03424325 +ILSVRC2012_val_00023332.JPEG:n03498962 +ILSVRC2012_val_00023333.JPEG:n03223299 +ILSVRC2012_val_00023334.JPEG:n04552348 +ILSVRC2012_val_00023335.JPEG:n09193705 +ILSVRC2012_val_00023336.JPEG:n07697537 +ILSVRC2012_val_00023337.JPEG:n04596742 +ILSVRC2012_val_00023338.JPEG:n01797886 +ILSVRC2012_val_00023339.JPEG:n01980166 +ILSVRC2012_val_00023340.JPEG:n02093991 +ILSVRC2012_val_00023341.JPEG:n01688243 +ILSVRC2012_val_00023342.JPEG:n01817953 +ILSVRC2012_val_00023343.JPEG:n03485407 +ILSVRC2012_val_00023344.JPEG:n01795545 +ILSVRC2012_val_00023345.JPEG:n02794156 +ILSVRC2012_val_00023346.JPEG:n02102480 +ILSVRC2012_val_00023347.JPEG:n01819313 +ILSVRC2012_val_00023348.JPEG:n03188531 +ILSVRC2012_val_00023349.JPEG:n02965783 +ILSVRC2012_val_00023350.JPEG:n03534580 +ILSVRC2012_val_00023351.JPEG:n02395406 +ILSVRC2012_val_00023352.JPEG:n02033041 +ILSVRC2012_val_00023353.JPEG:n03337140 +ILSVRC2012_val_00023354.JPEG:n04200800 +ILSVRC2012_val_00023355.JPEG:n02797295 +ILSVRC2012_val_00023356.JPEG:n02804414 +ILSVRC2012_val_00023357.JPEG:n02088364 +ILSVRC2012_val_00023358.JPEG:n03000247 +ILSVRC2012_val_00023359.JPEG:n03937543 +ILSVRC2012_val_00023360.JPEG:n02389026 +ILSVRC2012_val_00023361.JPEG:n01682714 +ILSVRC2012_val_00023362.JPEG:n02101388 +ILSVRC2012_val_00023363.JPEG:n01685808 +ILSVRC2012_val_00023364.JPEG:n07880968 +ILSVRC2012_val_00023365.JPEG:n02509815 +ILSVRC2012_val_00023366.JPEG:n03938244 +ILSVRC2012_val_00023367.JPEG:n04532670 +ILSVRC2012_val_00023368.JPEG:n03967562 +ILSVRC2012_val_00023369.JPEG:n03196217 +ILSVRC2012_val_00023370.JPEG:n02892767 +ILSVRC2012_val_00023371.JPEG:n01843383 +ILSVRC2012_val_00023372.JPEG:n02978881 +ILSVRC2012_val_00023373.JPEG:n01748264 +ILSVRC2012_val_00023374.JPEG:n04423845 +ILSVRC2012_val_00023375.JPEG:n02396427 +ILSVRC2012_val_00023376.JPEG:n03388043 +ILSVRC2012_val_00023377.JPEG:n03000134 +ILSVRC2012_val_00023378.JPEG:n04429376 +ILSVRC2012_val_00023379.JPEG:n03483316 +ILSVRC2012_val_00023380.JPEG:n03485407 +ILSVRC2012_val_00023381.JPEG:n02256656 +ILSVRC2012_val_00023382.JPEG:n04086273 +ILSVRC2012_val_00023383.JPEG:n02356798 +ILSVRC2012_val_00023384.JPEG:n02747177 +ILSVRC2012_val_00023385.JPEG:n01773157 +ILSVRC2012_val_00023386.JPEG:n03297495 +ILSVRC2012_val_00023387.JPEG:n02403003 +ILSVRC2012_val_00023388.JPEG:n07718472 +ILSVRC2012_val_00023389.JPEG:n03445924 +ILSVRC2012_val_00023390.JPEG:n01843383 +ILSVRC2012_val_00023391.JPEG:n02328150 +ILSVRC2012_val_00023392.JPEG:n03447447 +ILSVRC2012_val_00023393.JPEG:n02124075 +ILSVRC2012_val_00023394.JPEG:n02098105 +ILSVRC2012_val_00023395.JPEG:n06596364 +ILSVRC2012_val_00023396.JPEG:n03388183 +ILSVRC2012_val_00023397.JPEG:n06596364 +ILSVRC2012_val_00023398.JPEG:n02504013 +ILSVRC2012_val_00023399.JPEG:n04041544 +ILSVRC2012_val_00023400.JPEG:n02009912 +ILSVRC2012_val_00023401.JPEG:n02093859 +ILSVRC2012_val_00023402.JPEG:n04350905 +ILSVRC2012_val_00023403.JPEG:n02317335 +ILSVRC2012_val_00023404.JPEG:n07871810 +ILSVRC2012_val_00023405.JPEG:n02105855 +ILSVRC2012_val_00023406.JPEG:n02607072 +ILSVRC2012_val_00023407.JPEG:n02095570 +ILSVRC2012_val_00023408.JPEG:n02389026 +ILSVRC2012_val_00023409.JPEG:n06785654 +ILSVRC2012_val_00023410.JPEG:n09421951 +ILSVRC2012_val_00023411.JPEG:n02114855 +ILSVRC2012_val_00023412.JPEG:n03216828 +ILSVRC2012_val_00023413.JPEG:n01855032 +ILSVRC2012_val_00023414.JPEG:n03095699 +ILSVRC2012_val_00023415.JPEG:n02115641 +ILSVRC2012_val_00023416.JPEG:n01955084 +ILSVRC2012_val_00023417.JPEG:n03095699 +ILSVRC2012_val_00023418.JPEG:n03133878 +ILSVRC2012_val_00023419.JPEG:n03902125 +ILSVRC2012_val_00023420.JPEG:n02395406 +ILSVRC2012_val_00023421.JPEG:n04371774 +ILSVRC2012_val_00023422.JPEG:n04525305 +ILSVRC2012_val_00023423.JPEG:n03345487 +ILSVRC2012_val_00023424.JPEG:n02108551 +ILSVRC2012_val_00023425.JPEG:n01774750 +ILSVRC2012_val_00023426.JPEG:n02480495 +ILSVRC2012_val_00023427.JPEG:n03594945 +ILSVRC2012_val_00023428.JPEG:n02091635 +ILSVRC2012_val_00023429.JPEG:n04557648 +ILSVRC2012_val_00023430.JPEG:n03388549 +ILSVRC2012_val_00023431.JPEG:n01784675 +ILSVRC2012_val_00023432.JPEG:n13040303 +ILSVRC2012_val_00023433.JPEG:n13037406 +ILSVRC2012_val_00023434.JPEG:n01776313 +ILSVRC2012_val_00023435.JPEG:n02099601 +ILSVRC2012_val_00023436.JPEG:n03134739 +ILSVRC2012_val_00023437.JPEG:n02110185 +ILSVRC2012_val_00023438.JPEG:n01537544 +ILSVRC2012_val_00023439.JPEG:n13133613 +ILSVRC2012_val_00023440.JPEG:n02102040 +ILSVRC2012_val_00023441.JPEG:n01530575 +ILSVRC2012_val_00023442.JPEG:n01735189 +ILSVRC2012_val_00023443.JPEG:n01491361 +ILSVRC2012_val_00023444.JPEG:n07583066 +ILSVRC2012_val_00023445.JPEG:n02137549 +ILSVRC2012_val_00023446.JPEG:n03908714 +ILSVRC2012_val_00023447.JPEG:n03045698 +ILSVRC2012_val_00023448.JPEG:n01914609 +ILSVRC2012_val_00023449.JPEG:n02326432 +ILSVRC2012_val_00023450.JPEG:n01631663 +ILSVRC2012_val_00023451.JPEG:n03868242 +ILSVRC2012_val_00023452.JPEG:n03920288 +ILSVRC2012_val_00023453.JPEG:n03729826 +ILSVRC2012_val_00023454.JPEG:n02002724 +ILSVRC2012_val_00023455.JPEG:n03776460 +ILSVRC2012_val_00023456.JPEG:n03535780 +ILSVRC2012_val_00023457.JPEG:n03146219 +ILSVRC2012_val_00023458.JPEG:n02094258 +ILSVRC2012_val_00023459.JPEG:n03841143 +ILSVRC2012_val_00023460.JPEG:n02797295 +ILSVRC2012_val_00023461.JPEG:n02500267 +ILSVRC2012_val_00023462.JPEG:n04392985 +ILSVRC2012_val_00023463.JPEG:n02504458 +ILSVRC2012_val_00023464.JPEG:n01773797 +ILSVRC2012_val_00023465.JPEG:n04325704 +ILSVRC2012_val_00023466.JPEG:n03920288 +ILSVRC2012_val_00023467.JPEG:n02999410 +ILSVRC2012_val_00023468.JPEG:n02655020 +ILSVRC2012_val_00023469.JPEG:n02097474 +ILSVRC2012_val_00023470.JPEG:n09472597 +ILSVRC2012_val_00023471.JPEG:n02099712 +ILSVRC2012_val_00023472.JPEG:n02980441 +ILSVRC2012_val_00023473.JPEG:n04461696 +ILSVRC2012_val_00023474.JPEG:n02814533 +ILSVRC2012_val_00023475.JPEG:n03495258 +ILSVRC2012_val_00023476.JPEG:n01784675 +ILSVRC2012_val_00023477.JPEG:n03000684 +ILSVRC2012_val_00023478.JPEG:n07760859 +ILSVRC2012_val_00023479.JPEG:n04141327 +ILSVRC2012_val_00023480.JPEG:n02641379 +ILSVRC2012_val_00023481.JPEG:n04200800 +ILSVRC2012_val_00023482.JPEG:n04141327 +ILSVRC2012_val_00023483.JPEG:n01943899 +ILSVRC2012_val_00023484.JPEG:n04037443 +ILSVRC2012_val_00023485.JPEG:n04357314 +ILSVRC2012_val_00023486.JPEG:n02097474 +ILSVRC2012_val_00023487.JPEG:n03857828 +ILSVRC2012_val_00023488.JPEG:n01630670 +ILSVRC2012_val_00023489.JPEG:n02417914 +ILSVRC2012_val_00023490.JPEG:n02747177 +ILSVRC2012_val_00023491.JPEG:n04590129 +ILSVRC2012_val_00023492.JPEG:n02037110 +ILSVRC2012_val_00023493.JPEG:n03841143 +ILSVRC2012_val_00023494.JPEG:n04204238 +ILSVRC2012_val_00023495.JPEG:n04252225 +ILSVRC2012_val_00023496.JPEG:n02791270 +ILSVRC2012_val_00023497.JPEG:n09193705 +ILSVRC2012_val_00023498.JPEG:n04376876 +ILSVRC2012_val_00023499.JPEG:n02815834 +ILSVRC2012_val_00023500.JPEG:n01817953 +ILSVRC2012_val_00023501.JPEG:n04356056 +ILSVRC2012_val_00023502.JPEG:n02007558 +ILSVRC2012_val_00023503.JPEG:n02917067 +ILSVRC2012_val_00023504.JPEG:n03544143 +ILSVRC2012_val_00023505.JPEG:n03954731 +ILSVRC2012_val_00023506.JPEG:n03372029 +ILSVRC2012_val_00023507.JPEG:n02930766 +ILSVRC2012_val_00023508.JPEG:n04310018 +ILSVRC2012_val_00023509.JPEG:n03630383 +ILSVRC2012_val_00023510.JPEG:n04009552 +ILSVRC2012_val_00023511.JPEG:n02132136 +ILSVRC2012_val_00023512.JPEG:n07745940 +ILSVRC2012_val_00023513.JPEG:n02094114 +ILSVRC2012_val_00023514.JPEG:n02480855 +ILSVRC2012_val_00023515.JPEG:n02093991 +ILSVRC2012_val_00023516.JPEG:n02113624 +ILSVRC2012_val_00023517.JPEG:n03662601 +ILSVRC2012_val_00023518.JPEG:n12144580 +ILSVRC2012_val_00023519.JPEG:n02443114 +ILSVRC2012_val_00023520.JPEG:n01914609 +ILSVRC2012_val_00023521.JPEG:n04040759 +ILSVRC2012_val_00023522.JPEG:n02834397 +ILSVRC2012_val_00023523.JPEG:n02276258 +ILSVRC2012_val_00023524.JPEG:n04557648 +ILSVRC2012_val_00023525.JPEG:n07718472 +ILSVRC2012_val_00023526.JPEG:n02108915 +ILSVRC2012_val_00023527.JPEG:n07753113 +ILSVRC2012_val_00023528.JPEG:n02093428 +ILSVRC2012_val_00023529.JPEG:n03976467 +ILSVRC2012_val_00023530.JPEG:n01984695 +ILSVRC2012_val_00023531.JPEG:n02492035 +ILSVRC2012_val_00023532.JPEG:n04275548 +ILSVRC2012_val_00023533.JPEG:n02100877 +ILSVRC2012_val_00023534.JPEG:n04254777 +ILSVRC2012_val_00023535.JPEG:n02799071 +ILSVRC2012_val_00023536.JPEG:n03908618 +ILSVRC2012_val_00023537.JPEG:n03773504 +ILSVRC2012_val_00023538.JPEG:n03347037 +ILSVRC2012_val_00023539.JPEG:n02107574 +ILSVRC2012_val_00023540.JPEG:n03529860 +ILSVRC2012_val_00023541.JPEG:n02093256 +ILSVRC2012_val_00023542.JPEG:n03291819 +ILSVRC2012_val_00023543.JPEG:n02110958 +ILSVRC2012_val_00023544.JPEG:n04275548 +ILSVRC2012_val_00023545.JPEG:n04273569 +ILSVRC2012_val_00023546.JPEG:n02113023 +ILSVRC2012_val_00023547.JPEG:n03958227 +ILSVRC2012_val_00023548.JPEG:n04417672 +ILSVRC2012_val_00023549.JPEG:n03272562 +ILSVRC2012_val_00023550.JPEG:n01980166 +ILSVRC2012_val_00023551.JPEG:n01514668 +ILSVRC2012_val_00023552.JPEG:n02002556 +ILSVRC2012_val_00023553.JPEG:n02086079 +ILSVRC2012_val_00023554.JPEG:n02104365 +ILSVRC2012_val_00023555.JPEG:n01677366 +ILSVRC2012_val_00023556.JPEG:n03770679 +ILSVRC2012_val_00023557.JPEG:n02096177 +ILSVRC2012_val_00023558.JPEG:n02094258 +ILSVRC2012_val_00023559.JPEG:n01440764 +ILSVRC2012_val_00023560.JPEG:n01943899 +ILSVRC2012_val_00023561.JPEG:n02099849 +ILSVRC2012_val_00023562.JPEG:n03899768 +ILSVRC2012_val_00023563.JPEG:n01729322 +ILSVRC2012_val_00023564.JPEG:n01776313 +ILSVRC2012_val_00023565.JPEG:n06359193 +ILSVRC2012_val_00023566.JPEG:n02447366 +ILSVRC2012_val_00023567.JPEG:n03857828 +ILSVRC2012_val_00023568.JPEG:n03384352 +ILSVRC2012_val_00023569.JPEG:n02111277 +ILSVRC2012_val_00023570.JPEG:n02226429 +ILSVRC2012_val_00023571.JPEG:n04366367 +ILSVRC2012_val_00023572.JPEG:n01737021 +ILSVRC2012_val_00023573.JPEG:n01537544 +ILSVRC2012_val_00023574.JPEG:n02951358 +ILSVRC2012_val_00023575.JPEG:n04371430 +ILSVRC2012_val_00023576.JPEG:n03196217 +ILSVRC2012_val_00023577.JPEG:n02100236 +ILSVRC2012_val_00023578.JPEG:n04443257 +ILSVRC2012_val_00023579.JPEG:n04479046 +ILSVRC2012_val_00023580.JPEG:n03983396 +ILSVRC2012_val_00023581.JPEG:n03218198 +ILSVRC2012_val_00023582.JPEG:n02105505 +ILSVRC2012_val_00023583.JPEG:n01978287 +ILSVRC2012_val_00023584.JPEG:n04286575 +ILSVRC2012_val_00023585.JPEG:n03866082 +ILSVRC2012_val_00023586.JPEG:n04208210 +ILSVRC2012_val_00023587.JPEG:n03891332 +ILSVRC2012_val_00023588.JPEG:n03857828 +ILSVRC2012_val_00023589.JPEG:n02504013 +ILSVRC2012_val_00023590.JPEG:n03982430 +ILSVRC2012_val_00023591.JPEG:n04554684 +ILSVRC2012_val_00023592.JPEG:n04317175 +ILSVRC2012_val_00023593.JPEG:n04552348 +ILSVRC2012_val_00023594.JPEG:n12057211 +ILSVRC2012_val_00023595.JPEG:n02483362 +ILSVRC2012_val_00023596.JPEG:n02097474 +ILSVRC2012_val_00023597.JPEG:n02361337 +ILSVRC2012_val_00023598.JPEG:n02120505 +ILSVRC2012_val_00023599.JPEG:n03594945 +ILSVRC2012_val_00023600.JPEG:n03498962 +ILSVRC2012_val_00023601.JPEG:n01978455 +ILSVRC2012_val_00023602.JPEG:n01829413 +ILSVRC2012_val_00023603.JPEG:n02105505 +ILSVRC2012_val_00023604.JPEG:n01978455 +ILSVRC2012_val_00023605.JPEG:n04356056 +ILSVRC2012_val_00023606.JPEG:n07718472 +ILSVRC2012_val_00023607.JPEG:n01518878 +ILSVRC2012_val_00023608.JPEG:n02795169 +ILSVRC2012_val_00023609.JPEG:n03617480 +ILSVRC2012_val_00023610.JPEG:n03372029 +ILSVRC2012_val_00023611.JPEG:n02099267 +ILSVRC2012_val_00023612.JPEG:n04229816 +ILSVRC2012_val_00023613.JPEG:n07717410 +ILSVRC2012_val_00023614.JPEG:n02895154 +ILSVRC2012_val_00023615.JPEG:n02110185 +ILSVRC2012_val_00023616.JPEG:n04149813 +ILSVRC2012_val_00023617.JPEG:n02056570 +ILSVRC2012_val_00023618.JPEG:n04404412 +ILSVRC2012_val_00023619.JPEG:n03028079 +ILSVRC2012_val_00023620.JPEG:n02110341 +ILSVRC2012_val_00023621.JPEG:n04120489 +ILSVRC2012_val_00023622.JPEG:n02804414 +ILSVRC2012_val_00023623.JPEG:n02988304 +ILSVRC2012_val_00023624.JPEG:n02167151 +ILSVRC2012_val_00023625.JPEG:n04392985 +ILSVRC2012_val_00023626.JPEG:n07747607 +ILSVRC2012_val_00023627.JPEG:n02966687 +ILSVRC2012_val_00023628.JPEG:n09399592 +ILSVRC2012_val_00023629.JPEG:n03761084 +ILSVRC2012_val_00023630.JPEG:n03400231 +ILSVRC2012_val_00023631.JPEG:n04136333 +ILSVRC2012_val_00023632.JPEG:n04423845 +ILSVRC2012_val_00023633.JPEG:n02978881 +ILSVRC2012_val_00023634.JPEG:n02099429 +ILSVRC2012_val_00023635.JPEG:n07892512 +ILSVRC2012_val_00023636.JPEG:n02137549 +ILSVRC2012_val_00023637.JPEG:n01807496 +ILSVRC2012_val_00023638.JPEG:n04033995 +ILSVRC2012_val_00023639.JPEG:n03876231 +ILSVRC2012_val_00023640.JPEG:n03063599 +ILSVRC2012_val_00023641.JPEG:n04005630 +ILSVRC2012_val_00023642.JPEG:n02489166 +ILSVRC2012_val_00023643.JPEG:n03197337 +ILSVRC2012_val_00023644.JPEG:n04456115 +ILSVRC2012_val_00023645.JPEG:n03388043 +ILSVRC2012_val_00023646.JPEG:n03062245 +ILSVRC2012_val_00023647.JPEG:n03899768 +ILSVRC2012_val_00023648.JPEG:n04371430 +ILSVRC2012_val_00023649.JPEG:n03729826 +ILSVRC2012_val_00023650.JPEG:n02165456 +ILSVRC2012_val_00023651.JPEG:n02769748 +ILSVRC2012_val_00023652.JPEG:n02412080 +ILSVRC2012_val_00023653.JPEG:n02086240 +ILSVRC2012_val_00023654.JPEG:n01665541 +ILSVRC2012_val_00023655.JPEG:n02412080 +ILSVRC2012_val_00023656.JPEG:n02445715 +ILSVRC2012_val_00023657.JPEG:n01735189 +ILSVRC2012_val_00023658.JPEG:n02086079 +ILSVRC2012_val_00023659.JPEG:n02110185 +ILSVRC2012_val_00023660.JPEG:n07697537 +ILSVRC2012_val_00023661.JPEG:n02112350 +ILSVRC2012_val_00023662.JPEG:n02137549 +ILSVRC2012_val_00023663.JPEG:n02398521 +ILSVRC2012_val_00023664.JPEG:n02971356 +ILSVRC2012_val_00023665.JPEG:n03980874 +ILSVRC2012_val_00023666.JPEG:n02106030 +ILSVRC2012_val_00023667.JPEG:n02980441 +ILSVRC2012_val_00023668.JPEG:n09193705 +ILSVRC2012_val_00023669.JPEG:n03393912 +ILSVRC2012_val_00023670.JPEG:n04562935 +ILSVRC2012_val_00023671.JPEG:n03691459 +ILSVRC2012_val_00023672.JPEG:n02870880 +ILSVRC2012_val_00023673.JPEG:n02443484 +ILSVRC2012_val_00023674.JPEG:n02979186 +ILSVRC2012_val_00023675.JPEG:n02100735 +ILSVRC2012_val_00023676.JPEG:n01682714 +ILSVRC2012_val_00023677.JPEG:n02607072 +ILSVRC2012_val_00023678.JPEG:n01688243 +ILSVRC2012_val_00023679.JPEG:n02454379 +ILSVRC2012_val_00023680.JPEG:n02443484 +ILSVRC2012_val_00023681.JPEG:n07248320 +ILSVRC2012_val_00023682.JPEG:n03814639 +ILSVRC2012_val_00023683.JPEG:n04509417 +ILSVRC2012_val_00023684.JPEG:n04019541 +ILSVRC2012_val_00023685.JPEG:n03938244 +ILSVRC2012_val_00023686.JPEG:n01667114 +ILSVRC2012_val_00023687.JPEG:n03791053 +ILSVRC2012_val_00023688.JPEG:n04442312 +ILSVRC2012_val_00023689.JPEG:n02226429 +ILSVRC2012_val_00023690.JPEG:n01693334 +ILSVRC2012_val_00023691.JPEG:n02794156 +ILSVRC2012_val_00023692.JPEG:n01773549 +ILSVRC2012_val_00023693.JPEG:n01685808 +ILSVRC2012_val_00023694.JPEG:n03598930 +ILSVRC2012_val_00023695.JPEG:n02017213 +ILSVRC2012_val_00023696.JPEG:n02124075 +ILSVRC2012_val_00023697.JPEG:n02091134 +ILSVRC2012_val_00023698.JPEG:n01530575 +ILSVRC2012_val_00023699.JPEG:n03657121 +ILSVRC2012_val_00023700.JPEG:n01768244 +ILSVRC2012_val_00023701.JPEG:n04552348 +ILSVRC2012_val_00023702.JPEG:n02106030 +ILSVRC2012_val_00023703.JPEG:n01667114 +ILSVRC2012_val_00023704.JPEG:n02790996 +ILSVRC2012_val_00023705.JPEG:n02699494 +ILSVRC2012_val_00023706.JPEG:n03291819 +ILSVRC2012_val_00023707.JPEG:n01694178 +ILSVRC2012_val_00023708.JPEG:n02423022 +ILSVRC2012_val_00023709.JPEG:n01855672 +ILSVRC2012_val_00023710.JPEG:n03459775 +ILSVRC2012_val_00023711.JPEG:n04070727 +ILSVRC2012_val_00023712.JPEG:n03770439 +ILSVRC2012_val_00023713.JPEG:n03709823 +ILSVRC2012_val_00023714.JPEG:n01924916 +ILSVRC2012_val_00023715.JPEG:n06785654 +ILSVRC2012_val_00023716.JPEG:n03272562 +ILSVRC2012_val_00023717.JPEG:n02099429 +ILSVRC2012_val_00023718.JPEG:n03100240 +ILSVRC2012_val_00023719.JPEG:n02174001 +ILSVRC2012_val_00023720.JPEG:n06794110 +ILSVRC2012_val_00023721.JPEG:n03759954 +ILSVRC2012_val_00023722.JPEG:n04357314 +ILSVRC2012_val_00023723.JPEG:n03584829 +ILSVRC2012_val_00023724.JPEG:n03345487 +ILSVRC2012_val_00023725.JPEG:n03443371 +ILSVRC2012_val_00023726.JPEG:n02100236 +ILSVRC2012_val_00023727.JPEG:n03709823 +ILSVRC2012_val_00023728.JPEG:n04350905 +ILSVRC2012_val_00023729.JPEG:n02086910 +ILSVRC2012_val_00023730.JPEG:n02977058 +ILSVRC2012_val_00023731.JPEG:n02112018 +ILSVRC2012_val_00023732.JPEG:n04409515 +ILSVRC2012_val_00023733.JPEG:n04118776 +ILSVRC2012_val_00023734.JPEG:n03376595 +ILSVRC2012_val_00023735.JPEG:n02101556 +ILSVRC2012_val_00023736.JPEG:n02776631 +ILSVRC2012_val_00023737.JPEG:n02108551 +ILSVRC2012_val_00023738.JPEG:n03291819 +ILSVRC2012_val_00023739.JPEG:n07745940 +ILSVRC2012_val_00023740.JPEG:n02109047 +ILSVRC2012_val_00023741.JPEG:n04336792 +ILSVRC2012_val_00023742.JPEG:n03494278 +ILSVRC2012_val_00023743.JPEG:n03388183 +ILSVRC2012_val_00023744.JPEG:n02398521 +ILSVRC2012_val_00023745.JPEG:n03485794 +ILSVRC2012_val_00023746.JPEG:n03018349 +ILSVRC2012_val_00023747.JPEG:n03967562 +ILSVRC2012_val_00023748.JPEG:n02116738 +ILSVRC2012_val_00023749.JPEG:n02085620 +ILSVRC2012_val_00023750.JPEG:n02108551 +ILSVRC2012_val_00023751.JPEG:n02894605 +ILSVRC2012_val_00023752.JPEG:n07695742 +ILSVRC2012_val_00023753.JPEG:n01693334 +ILSVRC2012_val_00023754.JPEG:n04356056 +ILSVRC2012_val_00023755.JPEG:n02120079 +ILSVRC2012_val_00023756.JPEG:n04540053 +ILSVRC2012_val_00023757.JPEG:n03134739 +ILSVRC2012_val_00023758.JPEG:n01644900 +ILSVRC2012_val_00023759.JPEG:n01697457 +ILSVRC2012_val_00023760.JPEG:n02108000 +ILSVRC2012_val_00023761.JPEG:n03720891 +ILSVRC2012_val_00023762.JPEG:n03733281 +ILSVRC2012_val_00023763.JPEG:n04404412 +ILSVRC2012_val_00023764.JPEG:n02098105 +ILSVRC2012_val_00023765.JPEG:n02089867 +ILSVRC2012_val_00023766.JPEG:n01530575 +ILSVRC2012_val_00023767.JPEG:n03884397 +ILSVRC2012_val_00023768.JPEG:n03602883 +ILSVRC2012_val_00023769.JPEG:n02090721 +ILSVRC2012_val_00023770.JPEG:n04228054 +ILSVRC2012_val_00023771.JPEG:n03208938 +ILSVRC2012_val_00023772.JPEG:n02483708 +ILSVRC2012_val_00023773.JPEG:n02017213 +ILSVRC2012_val_00023774.JPEG:n02097047 +ILSVRC2012_val_00023775.JPEG:n02509815 +ILSVRC2012_val_00023776.JPEG:n02447366 +ILSVRC2012_val_00023777.JPEG:n03532672 +ILSVRC2012_val_00023778.JPEG:n01518878 +ILSVRC2012_val_00023779.JPEG:n02123045 +ILSVRC2012_val_00023780.JPEG:n01847000 +ILSVRC2012_val_00023781.JPEG:n02690373 +ILSVRC2012_val_00023782.JPEG:n02092002 +ILSVRC2012_val_00023783.JPEG:n02096177 +ILSVRC2012_val_00023784.JPEG:n04487081 +ILSVRC2012_val_00023785.JPEG:n02526121 +ILSVRC2012_val_00023786.JPEG:n02124075 +ILSVRC2012_val_00023787.JPEG:n03717622 +ILSVRC2012_val_00023788.JPEG:n02106030 +ILSVRC2012_val_00023789.JPEG:n02002724 +ILSVRC2012_val_00023790.JPEG:n03240683 +ILSVRC2012_val_00023791.JPEG:n03902125 +ILSVRC2012_val_00023792.JPEG:n03709823 +ILSVRC2012_val_00023793.JPEG:n02974003 +ILSVRC2012_val_00023794.JPEG:n02100583 +ILSVRC2012_val_00023795.JPEG:n03201208 +ILSVRC2012_val_00023796.JPEG:n01833805 +ILSVRC2012_val_00023797.JPEG:n13052670 +ILSVRC2012_val_00023798.JPEG:n02219486 +ILSVRC2012_val_00023799.JPEG:n02107574 +ILSVRC2012_val_00023800.JPEG:n07742313 +ILSVRC2012_val_00023801.JPEG:n02112018 +ILSVRC2012_val_00023802.JPEG:n02489166 +ILSVRC2012_val_00023803.JPEG:n02441942 +ILSVRC2012_val_00023804.JPEG:n07753275 +ILSVRC2012_val_00023805.JPEG:n01819313 +ILSVRC2012_val_00023806.JPEG:n02643566 +ILSVRC2012_val_00023807.JPEG:n03110669 +ILSVRC2012_val_00023808.JPEG:n04482393 +ILSVRC2012_val_00023809.JPEG:n04613696 +ILSVRC2012_val_00023810.JPEG:n02129604 +ILSVRC2012_val_00023811.JPEG:n02088466 +ILSVRC2012_val_00023812.JPEG:n02134418 +ILSVRC2012_val_00023813.JPEG:n02114855 +ILSVRC2012_val_00023814.JPEG:n04591157 +ILSVRC2012_val_00023815.JPEG:n02277742 +ILSVRC2012_val_00023816.JPEG:n02112350 +ILSVRC2012_val_00023817.JPEG:n03590841 +ILSVRC2012_val_00023818.JPEG:n04476259 +ILSVRC2012_val_00023819.JPEG:n02326432 +ILSVRC2012_val_00023820.JPEG:n01755581 +ILSVRC2012_val_00023821.JPEG:n11939491 +ILSVRC2012_val_00023822.JPEG:n04264628 +ILSVRC2012_val_00023823.JPEG:n12998815 +ILSVRC2012_val_00023824.JPEG:n02101388 +ILSVRC2012_val_00023825.JPEG:n02137549 +ILSVRC2012_val_00023826.JPEG:n02236044 +ILSVRC2012_val_00023827.JPEG:n02123394 +ILSVRC2012_val_00023828.JPEG:n02909870 +ILSVRC2012_val_00023829.JPEG:n03733805 +ILSVRC2012_val_00023830.JPEG:n04120489 +ILSVRC2012_val_00023831.JPEG:n03958227 +ILSVRC2012_val_00023832.JPEG:n02100877 +ILSVRC2012_val_00023833.JPEG:n02169497 +ILSVRC2012_val_00023834.JPEG:n02168699 +ILSVRC2012_val_00023835.JPEG:n03794056 +ILSVRC2012_val_00023836.JPEG:n04146614 +ILSVRC2012_val_00023837.JPEG:n03787032 +ILSVRC2012_val_00023838.JPEG:n03937543 +ILSVRC2012_val_00023839.JPEG:n03388549 +ILSVRC2012_val_00023840.JPEG:n01978455 +ILSVRC2012_val_00023841.JPEG:n06874185 +ILSVRC2012_val_00023842.JPEG:n03717622 +ILSVRC2012_val_00023843.JPEG:n07875152 +ILSVRC2012_val_00023844.JPEG:n01820546 +ILSVRC2012_val_00023845.JPEG:n03445777 +ILSVRC2012_val_00023846.JPEG:n02109961 +ILSVRC2012_val_00023847.JPEG:n04127249 +ILSVRC2012_val_00023848.JPEG:n07716358 +ILSVRC2012_val_00023849.JPEG:n03661043 +ILSVRC2012_val_00023850.JPEG:n01534433 +ILSVRC2012_val_00023851.JPEG:n03982430 +ILSVRC2012_val_00023852.JPEG:n02490219 +ILSVRC2012_val_00023853.JPEG:n04152593 +ILSVRC2012_val_00023854.JPEG:n03062245 +ILSVRC2012_val_00023855.JPEG:n01644373 +ILSVRC2012_val_00023856.JPEG:n02951358 +ILSVRC2012_val_00023857.JPEG:n04041544 +ILSVRC2012_val_00023858.JPEG:n02974003 +ILSVRC2012_val_00023859.JPEG:n02102318 +ILSVRC2012_val_00023860.JPEG:n04127249 +ILSVRC2012_val_00023861.JPEG:n02500267 +ILSVRC2012_val_00023862.JPEG:n04548280 +ILSVRC2012_val_00023863.JPEG:n02690373 +ILSVRC2012_val_00023864.JPEG:n02125311 +ILSVRC2012_val_00023865.JPEG:n01950731 +ILSVRC2012_val_00023866.JPEG:n02007558 +ILSVRC2012_val_00023867.JPEG:n12267677 +ILSVRC2012_val_00023868.JPEG:n03045698 +ILSVRC2012_val_00023869.JPEG:n01443537 +ILSVRC2012_val_00023870.JPEG:n02447366 +ILSVRC2012_val_00023871.JPEG:n02124075 +ILSVRC2012_val_00023872.JPEG:n03916031 +ILSVRC2012_val_00023873.JPEG:n03146219 +ILSVRC2012_val_00023874.JPEG:n02843684 +ILSVRC2012_val_00023875.JPEG:n02980441 +ILSVRC2012_val_00023876.JPEG:n03187595 +ILSVRC2012_val_00023877.JPEG:n02091134 +ILSVRC2012_val_00023878.JPEG:n03124170 +ILSVRC2012_val_00023879.JPEG:n07749582 +ILSVRC2012_val_00023880.JPEG:n03594734 +ILSVRC2012_val_00023881.JPEG:n02666196 +ILSVRC2012_val_00023882.JPEG:n03782006 +ILSVRC2012_val_00023883.JPEG:n07697537 +ILSVRC2012_val_00023884.JPEG:n02111889 +ILSVRC2012_val_00023885.JPEG:n03724870 +ILSVRC2012_val_00023886.JPEG:n02085620 +ILSVRC2012_val_00023887.JPEG:n03492542 +ILSVRC2012_val_00023888.JPEG:n02102177 +ILSVRC2012_val_00023889.JPEG:n04515003 +ILSVRC2012_val_00023890.JPEG:n02167151 +ILSVRC2012_val_00023891.JPEG:n03877472 +ILSVRC2012_val_00023892.JPEG:n07720875 +ILSVRC2012_val_00023893.JPEG:n02097209 +ILSVRC2012_val_00023894.JPEG:n03208938 +ILSVRC2012_val_00023895.JPEG:n01601694 +ILSVRC2012_val_00023896.JPEG:n04067472 +ILSVRC2012_val_00023897.JPEG:n02174001 +ILSVRC2012_val_00023898.JPEG:n02123394 +ILSVRC2012_val_00023899.JPEG:n07583066 +ILSVRC2012_val_00023900.JPEG:n03599486 +ILSVRC2012_val_00023901.JPEG:n04005630 +ILSVRC2012_val_00023902.JPEG:n01698640 +ILSVRC2012_val_00023903.JPEG:n03047690 +ILSVRC2012_val_00023904.JPEG:n03793489 +ILSVRC2012_val_00023905.JPEG:n02916936 +ILSVRC2012_val_00023906.JPEG:n02124075 +ILSVRC2012_val_00023907.JPEG:n01592084 +ILSVRC2012_val_00023908.JPEG:n03127747 +ILSVRC2012_val_00023909.JPEG:n02130308 +ILSVRC2012_val_00023910.JPEG:n02094114 +ILSVRC2012_val_00023911.JPEG:n04131690 +ILSVRC2012_val_00023912.JPEG:n03063599 +ILSVRC2012_val_00023913.JPEG:n02110341 +ILSVRC2012_val_00023914.JPEG:n04008634 +ILSVRC2012_val_00023915.JPEG:n03218198 +ILSVRC2012_val_00023916.JPEG:n01496331 +ILSVRC2012_val_00023917.JPEG:n03146219 +ILSVRC2012_val_00023918.JPEG:n03496892 +ILSVRC2012_val_00023919.JPEG:n02097047 +ILSVRC2012_val_00023920.JPEG:n02397096 +ILSVRC2012_val_00023921.JPEG:n03942813 +ILSVRC2012_val_00023922.JPEG:n03787032 +ILSVRC2012_val_00023923.JPEG:n02125311 +ILSVRC2012_val_00023924.JPEG:n02119789 +ILSVRC2012_val_00023925.JPEG:n01945685 +ILSVRC2012_val_00023926.JPEG:n02105162 +ILSVRC2012_val_00023927.JPEG:n03127747 +ILSVRC2012_val_00023928.JPEG:n02107142 +ILSVRC2012_val_00023929.JPEG:n02992529 +ILSVRC2012_val_00023930.JPEG:n12620546 +ILSVRC2012_val_00023931.JPEG:n04067472 +ILSVRC2012_val_00023932.JPEG:n01630670 +ILSVRC2012_val_00023933.JPEG:n02423022 +ILSVRC2012_val_00023934.JPEG:n02948072 +ILSVRC2012_val_00023935.JPEG:n01491361 +ILSVRC2012_val_00023936.JPEG:n04067472 +ILSVRC2012_val_00023937.JPEG:n04263257 +ILSVRC2012_val_00023938.JPEG:n03223299 +ILSVRC2012_val_00023939.JPEG:n02088238 +ILSVRC2012_val_00023940.JPEG:n02231487 +ILSVRC2012_val_00023941.JPEG:n01739381 +ILSVRC2012_val_00023942.JPEG:n01532829 +ILSVRC2012_val_00023943.JPEG:n02099849 +ILSVRC2012_val_00023944.JPEG:n09256479 +ILSVRC2012_val_00023945.JPEG:n01580077 +ILSVRC2012_val_00023946.JPEG:n03895866 +ILSVRC2012_val_00023947.JPEG:n02037110 +ILSVRC2012_val_00023948.JPEG:n07742313 +ILSVRC2012_val_00023949.JPEG:n02091032 +ILSVRC2012_val_00023950.JPEG:n03841143 +ILSVRC2012_val_00023951.JPEG:n01986214 +ILSVRC2012_val_00023952.JPEG:n04356056 +ILSVRC2012_val_00023953.JPEG:n02971356 +ILSVRC2012_val_00023954.JPEG:n01774384 +ILSVRC2012_val_00023955.JPEG:n02097474 +ILSVRC2012_val_00023956.JPEG:n04019541 +ILSVRC2012_val_00023957.JPEG:n07753275 +ILSVRC2012_val_00023958.JPEG:n01944390 +ILSVRC2012_val_00023959.JPEG:n04371774 +ILSVRC2012_val_00023960.JPEG:n02120079 +ILSVRC2012_val_00023961.JPEG:n07932039 +ILSVRC2012_val_00023962.JPEG:n04033901 +ILSVRC2012_val_00023963.JPEG:n04074963 +ILSVRC2012_val_00023964.JPEG:n02843684 +ILSVRC2012_val_00023965.JPEG:n03457902 +ILSVRC2012_val_00023966.JPEG:n02089078 +ILSVRC2012_val_00023967.JPEG:n03544143 +ILSVRC2012_val_00023968.JPEG:n02088238 +ILSVRC2012_val_00023969.JPEG:n02342885 +ILSVRC2012_val_00023970.JPEG:n01753488 +ILSVRC2012_val_00023971.JPEG:n02895154 +ILSVRC2012_val_00023972.JPEG:n04009552 +ILSVRC2012_val_00023973.JPEG:n01806143 +ILSVRC2012_val_00023974.JPEG:n03794056 +ILSVRC2012_val_00023975.JPEG:n01740131 +ILSVRC2012_val_00023976.JPEG:n02423022 +ILSVRC2012_val_00023977.JPEG:n02033041 +ILSVRC2012_val_00023978.JPEG:n03942813 +ILSVRC2012_val_00023979.JPEG:n04023962 +ILSVRC2012_val_00023980.JPEG:n03630383 +ILSVRC2012_val_00023981.JPEG:n04251144 +ILSVRC2012_val_00023982.JPEG:n04376876 +ILSVRC2012_val_00023983.JPEG:n02107142 +ILSVRC2012_val_00023984.JPEG:n01740131 +ILSVRC2012_val_00023985.JPEG:n03075370 +ILSVRC2012_val_00023986.JPEG:n01494475 +ILSVRC2012_val_00023987.JPEG:n04590129 +ILSVRC2012_val_00023988.JPEG:n02786058 +ILSVRC2012_val_00023989.JPEG:n01773549 +ILSVRC2012_val_00023990.JPEG:n02028035 +ILSVRC2012_val_00023991.JPEG:n01978287 +ILSVRC2012_val_00023992.JPEG:n02966193 +ILSVRC2012_val_00023993.JPEG:n03982430 +ILSVRC2012_val_00023994.JPEG:n02442845 +ILSVRC2012_val_00023995.JPEG:n07734744 +ILSVRC2012_val_00023996.JPEG:n07615774 +ILSVRC2012_val_00023997.JPEG:n03970156 +ILSVRC2012_val_00023998.JPEG:n03000134 +ILSVRC2012_val_00023999.JPEG:n01883070 +ILSVRC2012_val_00024000.JPEG:n02124075 +ILSVRC2012_val_00024001.JPEG:n07892512 +ILSVRC2012_val_00024002.JPEG:n03970156 +ILSVRC2012_val_00024003.JPEG:n03958227 +ILSVRC2012_val_00024004.JPEG:n04532670 +ILSVRC2012_val_00024005.JPEG:n03743016 +ILSVRC2012_val_00024006.JPEG:n04479046 +ILSVRC2012_val_00024007.JPEG:n02011460 +ILSVRC2012_val_00024008.JPEG:n02391049 +ILSVRC2012_val_00024009.JPEG:n03877845 +ILSVRC2012_val_00024010.JPEG:n01981276 +ILSVRC2012_val_00024011.JPEG:n02488291 +ILSVRC2012_val_00024012.JPEG:n01592084 +ILSVRC2012_val_00024013.JPEG:n03544143 +ILSVRC2012_val_00024014.JPEG:n02168699 +ILSVRC2012_val_00024015.JPEG:n01494475 +ILSVRC2012_val_00024016.JPEG:n03887697 +ILSVRC2012_val_00024017.JPEG:n03249569 +ILSVRC2012_val_00024018.JPEG:n03777754 +ILSVRC2012_val_00024019.JPEG:n02100236 +ILSVRC2012_val_00024020.JPEG:n02017213 +ILSVRC2012_val_00024021.JPEG:n02999410 +ILSVRC2012_val_00024022.JPEG:n03590841 +ILSVRC2012_val_00024023.JPEG:n03476991 +ILSVRC2012_val_00024024.JPEG:n04192698 +ILSVRC2012_val_00024025.JPEG:n01582220 +ILSVRC2012_val_00024026.JPEG:n04604644 +ILSVRC2012_val_00024027.JPEG:n03658185 +ILSVRC2012_val_00024028.JPEG:n03773504 +ILSVRC2012_val_00024029.JPEG:n02640242 +ILSVRC2012_val_00024030.JPEG:n01819313 +ILSVRC2012_val_00024031.JPEG:n02906734 +ILSVRC2012_val_00024032.JPEG:n07697537 +ILSVRC2012_val_00024033.JPEG:n02403003 +ILSVRC2012_val_00024034.JPEG:n04270147 +ILSVRC2012_val_00024035.JPEG:n03544143 +ILSVRC2012_val_00024036.JPEG:n02859443 +ILSVRC2012_val_00024037.JPEG:n03733131 +ILSVRC2012_val_00024038.JPEG:n03733131 +ILSVRC2012_val_00024039.JPEG:n04251144 +ILSVRC2012_val_00024040.JPEG:n01806143 +ILSVRC2012_val_00024041.JPEG:n04254120 +ILSVRC2012_val_00024042.JPEG:n04350905 +ILSVRC2012_val_00024043.JPEG:n02090379 +ILSVRC2012_val_00024044.JPEG:n01582220 +ILSVRC2012_val_00024045.JPEG:n03868242 +ILSVRC2012_val_00024046.JPEG:n02088466 +ILSVRC2012_val_00024047.JPEG:n02793495 +ILSVRC2012_val_00024048.JPEG:n04136333 +ILSVRC2012_val_00024049.JPEG:n03476684 +ILSVRC2012_val_00024050.JPEG:n02129604 +ILSVRC2012_val_00024051.JPEG:n02112137 +ILSVRC2012_val_00024052.JPEG:n01622779 +ILSVRC2012_val_00024053.JPEG:n02087046 +ILSVRC2012_val_00024054.JPEG:n02114548 +ILSVRC2012_val_00024055.JPEG:n07875152 +ILSVRC2012_val_00024056.JPEG:n01773549 +ILSVRC2012_val_00024057.JPEG:n03721384 +ILSVRC2012_val_00024058.JPEG:n01843065 +ILSVRC2012_val_00024059.JPEG:n01601694 +ILSVRC2012_val_00024060.JPEG:n04254680 +ILSVRC2012_val_00024061.JPEG:n07860988 +ILSVRC2012_val_00024062.JPEG:n04523525 +ILSVRC2012_val_00024063.JPEG:n01843383 +ILSVRC2012_val_00024064.JPEG:n03314780 +ILSVRC2012_val_00024065.JPEG:n04069434 +ILSVRC2012_val_00024066.JPEG:n02791270 +ILSVRC2012_val_00024067.JPEG:n04125021 +ILSVRC2012_val_00024068.JPEG:n07880968 +ILSVRC2012_val_00024069.JPEG:n03314780 +ILSVRC2012_val_00024070.JPEG:n04346328 +ILSVRC2012_val_00024071.JPEG:n04335435 +ILSVRC2012_val_00024072.JPEG:n02093647 +ILSVRC2012_val_00024073.JPEG:n04532106 +ILSVRC2012_val_00024074.JPEG:n04465501 +ILSVRC2012_val_00024075.JPEG:n02102177 +ILSVRC2012_val_00024076.JPEG:n04344873 +ILSVRC2012_val_00024077.JPEG:n03788195 +ILSVRC2012_val_00024078.JPEG:n03803284 +ILSVRC2012_val_00024079.JPEG:n09835506 +ILSVRC2012_val_00024080.JPEG:n01872401 +ILSVRC2012_val_00024081.JPEG:n01688243 +ILSVRC2012_val_00024082.JPEG:n02233338 +ILSVRC2012_val_00024083.JPEG:n03633091 +ILSVRC2012_val_00024084.JPEG:n03888605 +ILSVRC2012_val_00024085.JPEG:n02095570 +ILSVRC2012_val_00024086.JPEG:n04579145 +ILSVRC2012_val_00024087.JPEG:n03598930 +ILSVRC2012_val_00024088.JPEG:n02980441 +ILSVRC2012_val_00024089.JPEG:n03095699 +ILSVRC2012_val_00024090.JPEG:n02088466 +ILSVRC2012_val_00024091.JPEG:n04296562 +ILSVRC2012_val_00024092.JPEG:n01739381 +ILSVRC2012_val_00024093.JPEG:n02033041 +ILSVRC2012_val_00024094.JPEG:n04346328 +ILSVRC2012_val_00024095.JPEG:n01695060 +ILSVRC2012_val_00024096.JPEG:n03733281 +ILSVRC2012_val_00024097.JPEG:n04265275 +ILSVRC2012_val_00024098.JPEG:n01796340 +ILSVRC2012_val_00024099.JPEG:n07880968 +ILSVRC2012_val_00024100.JPEG:n02894605 +ILSVRC2012_val_00024101.JPEG:n04465501 +ILSVRC2012_val_00024102.JPEG:n01644900 +ILSVRC2012_val_00024103.JPEG:n03100240 +ILSVRC2012_val_00024104.JPEG:n03447721 +ILSVRC2012_val_00024105.JPEG:n03792782 +ILSVRC2012_val_00024106.JPEG:n01828970 +ILSVRC2012_val_00024107.JPEG:n02486261 +ILSVRC2012_val_00024108.JPEG:n02690373 +ILSVRC2012_val_00024109.JPEG:n01774750 +ILSVRC2012_val_00024110.JPEG:n09229709 +ILSVRC2012_val_00024111.JPEG:n03045698 +ILSVRC2012_val_00024112.JPEG:n03874293 +ILSVRC2012_val_00024113.JPEG:n12267677 +ILSVRC2012_val_00024114.JPEG:n03637318 +ILSVRC2012_val_00024115.JPEG:n02398521 +ILSVRC2012_val_00024116.JPEG:n02782093 +ILSVRC2012_val_00024117.JPEG:n01728572 +ILSVRC2012_val_00024118.JPEG:n02457408 +ILSVRC2012_val_00024119.JPEG:n04005630 +ILSVRC2012_val_00024120.JPEG:n04525305 +ILSVRC2012_val_00024121.JPEG:n01820546 +ILSVRC2012_val_00024122.JPEG:n02138441 +ILSVRC2012_val_00024123.JPEG:n03532672 +ILSVRC2012_val_00024124.JPEG:n02808440 +ILSVRC2012_val_00024125.JPEG:n12985857 +ILSVRC2012_val_00024126.JPEG:n02085620 +ILSVRC2012_val_00024127.JPEG:n04584207 +ILSVRC2012_val_00024128.JPEG:n02125311 +ILSVRC2012_val_00024129.JPEG:n07742313 +ILSVRC2012_val_00024130.JPEG:n03355925 +ILSVRC2012_val_00024131.JPEG:n03868242 +ILSVRC2012_val_00024132.JPEG:n03871628 +ILSVRC2012_val_00024133.JPEG:n03840681 +ILSVRC2012_val_00024134.JPEG:n04310018 +ILSVRC2012_val_00024135.JPEG:n02793495 +ILSVRC2012_val_00024136.JPEG:n02489166 +ILSVRC2012_val_00024137.JPEG:n02727426 +ILSVRC2012_val_00024138.JPEG:n04592741 +ILSVRC2012_val_00024139.JPEG:n02841315 +ILSVRC2012_val_00024140.JPEG:n02490219 +ILSVRC2012_val_00024141.JPEG:n04273569 +ILSVRC2012_val_00024142.JPEG:n04228054 +ILSVRC2012_val_00024143.JPEG:n03991062 +ILSVRC2012_val_00024144.JPEG:n02093647 +ILSVRC2012_val_00024145.JPEG:n02113023 +ILSVRC2012_val_00024146.JPEG:n01698640 +ILSVRC2012_val_00024147.JPEG:n04591713 +ILSVRC2012_val_00024148.JPEG:n02111277 +ILSVRC2012_val_00024149.JPEG:n04596742 +ILSVRC2012_val_00024150.JPEG:n02110627 +ILSVRC2012_val_00024151.JPEG:n03720891 +ILSVRC2012_val_00024152.JPEG:n04251144 +ILSVRC2012_val_00024153.JPEG:n03179701 +ILSVRC2012_val_00024154.JPEG:n02091244 +ILSVRC2012_val_00024155.JPEG:n07745940 +ILSVRC2012_val_00024156.JPEG:n03000247 +ILSVRC2012_val_00024157.JPEG:n04243546 +ILSVRC2012_val_00024158.JPEG:n07697313 +ILSVRC2012_val_00024159.JPEG:n03127925 +ILSVRC2012_val_00024160.JPEG:n01985128 +ILSVRC2012_val_00024161.JPEG:n03942813 +ILSVRC2012_val_00024162.JPEG:n02013706 +ILSVRC2012_val_00024163.JPEG:n02483708 +ILSVRC2012_val_00024164.JPEG:n01632458 +ILSVRC2012_val_00024165.JPEG:n02279972 +ILSVRC2012_val_00024166.JPEG:n02009912 +ILSVRC2012_val_00024167.JPEG:n02256656 +ILSVRC2012_val_00024168.JPEG:n01768244 +ILSVRC2012_val_00024169.JPEG:n02091635 +ILSVRC2012_val_00024170.JPEG:n03770679 +ILSVRC2012_val_00024171.JPEG:n12144580 +ILSVRC2012_val_00024172.JPEG:n01806567 +ILSVRC2012_val_00024173.JPEG:n04536866 +ILSVRC2012_val_00024174.JPEG:n03991062 +ILSVRC2012_val_00024175.JPEG:n02391049 +ILSVRC2012_val_00024176.JPEG:n02326432 +ILSVRC2012_val_00024177.JPEG:n04443257 +ILSVRC2012_val_00024178.JPEG:n02097047 +ILSVRC2012_val_00024179.JPEG:n02101006 +ILSVRC2012_val_00024180.JPEG:n02051845 +ILSVRC2012_val_00024181.JPEG:n03933933 +ILSVRC2012_val_00024182.JPEG:n03595614 +ILSVRC2012_val_00024183.JPEG:n07695742 +ILSVRC2012_val_00024184.JPEG:n07579787 +ILSVRC2012_val_00024185.JPEG:n02120079 +ILSVRC2012_val_00024186.JPEG:n02110627 +ILSVRC2012_val_00024187.JPEG:n02095314 +ILSVRC2012_val_00024188.JPEG:n03201208 +ILSVRC2012_val_00024189.JPEG:n03803284 +ILSVRC2012_val_00024190.JPEG:n02444819 +ILSVRC2012_val_00024191.JPEG:n03899768 +ILSVRC2012_val_00024192.JPEG:n02233338 +ILSVRC2012_val_00024193.JPEG:n02747177 +ILSVRC2012_val_00024194.JPEG:n03483316 +ILSVRC2012_val_00024195.JPEG:n04136333 +ILSVRC2012_val_00024196.JPEG:n03220513 +ILSVRC2012_val_00024197.JPEG:n03623198 +ILSVRC2012_val_00024198.JPEG:n03134739 +ILSVRC2012_val_00024199.JPEG:n03630383 +ILSVRC2012_val_00024200.JPEG:n02808440 +ILSVRC2012_val_00024201.JPEG:n03769881 +ILSVRC2012_val_00024202.JPEG:n02799071 +ILSVRC2012_val_00024203.JPEG:n04019541 +ILSVRC2012_val_00024204.JPEG:n01498041 +ILSVRC2012_val_00024205.JPEG:n04428191 +ILSVRC2012_val_00024206.JPEG:n02094433 +ILSVRC2012_val_00024207.JPEG:n03450230 +ILSVRC2012_val_00024208.JPEG:n02092002 +ILSVRC2012_val_00024209.JPEG:n03929660 +ILSVRC2012_val_00024210.JPEG:n03000134 +ILSVRC2012_val_00024211.JPEG:n01914609 +ILSVRC2012_val_00024212.JPEG:n03721384 +ILSVRC2012_val_00024213.JPEG:n04389033 +ILSVRC2012_val_00024214.JPEG:n02128385 +ILSVRC2012_val_00024215.JPEG:n03000247 +ILSVRC2012_val_00024216.JPEG:n02091244 +ILSVRC2012_val_00024217.JPEG:n02108000 +ILSVRC2012_val_00024218.JPEG:n02110063 +ILSVRC2012_val_00024219.JPEG:n02128385 +ILSVRC2012_val_00024220.JPEG:n02641379 +ILSVRC2012_val_00024221.JPEG:n01664065 +ILSVRC2012_val_00024222.JPEG:n02109525 +ILSVRC2012_val_00024223.JPEG:n07802026 +ILSVRC2012_val_00024224.JPEG:n07714571 +ILSVRC2012_val_00024225.JPEG:n03691459 +ILSVRC2012_val_00024226.JPEG:n02109961 +ILSVRC2012_val_00024227.JPEG:n01688243 +ILSVRC2012_val_00024228.JPEG:n04515003 +ILSVRC2012_val_00024229.JPEG:n04252225 +ILSVRC2012_val_00024230.JPEG:n02877765 +ILSVRC2012_val_00024231.JPEG:n03476991 +ILSVRC2012_val_00024232.JPEG:n07717410 +ILSVRC2012_val_00024233.JPEG:n04389033 +ILSVRC2012_val_00024234.JPEG:n02129165 +ILSVRC2012_val_00024235.JPEG:n01440764 +ILSVRC2012_val_00024236.JPEG:n12985857 +ILSVRC2012_val_00024237.JPEG:n04371430 +ILSVRC2012_val_00024238.JPEG:n03447721 +ILSVRC2012_val_00024239.JPEG:n02441942 +ILSVRC2012_val_00024240.JPEG:n02110958 +ILSVRC2012_val_00024241.JPEG:n02094433 +ILSVRC2012_val_00024242.JPEG:n04146614 +ILSVRC2012_val_00024243.JPEG:n03857828 +ILSVRC2012_val_00024244.JPEG:n03788195 +ILSVRC2012_val_00024245.JPEG:n03804744 +ILSVRC2012_val_00024246.JPEG:n02102040 +ILSVRC2012_val_00024247.JPEG:n02317335 +ILSVRC2012_val_00024248.JPEG:n09246464 +ILSVRC2012_val_00024249.JPEG:n02110958 +ILSVRC2012_val_00024250.JPEG:n02256656 +ILSVRC2012_val_00024251.JPEG:n03781244 +ILSVRC2012_val_00024252.JPEG:n01689811 +ILSVRC2012_val_00024253.JPEG:n02487347 +ILSVRC2012_val_00024254.JPEG:n02092002 +ILSVRC2012_val_00024255.JPEG:n03733805 +ILSVRC2012_val_00024256.JPEG:n01531178 +ILSVRC2012_val_00024257.JPEG:n02454379 +ILSVRC2012_val_00024258.JPEG:n02088238 +ILSVRC2012_val_00024259.JPEG:n01729322 +ILSVRC2012_val_00024260.JPEG:n01945685 +ILSVRC2012_val_00024261.JPEG:n01774384 +ILSVRC2012_val_00024262.JPEG:n01632458 +ILSVRC2012_val_00024263.JPEG:n03776460 +ILSVRC2012_val_00024264.JPEG:n01877812 +ILSVRC2012_val_00024265.JPEG:n07615774 +ILSVRC2012_val_00024266.JPEG:n02423022 +ILSVRC2012_val_00024267.JPEG:n03384352 +ILSVRC2012_val_00024268.JPEG:n01518878 +ILSVRC2012_val_00024269.JPEG:n03000684 +ILSVRC2012_val_00024270.JPEG:n02018207 +ILSVRC2012_val_00024271.JPEG:n03876231 +ILSVRC2012_val_00024272.JPEG:n02113799 +ILSVRC2012_val_00024273.JPEG:n01855032 +ILSVRC2012_val_00024274.JPEG:n02910353 +ILSVRC2012_val_00024275.JPEG:n02109047 +ILSVRC2012_val_00024276.JPEG:n03967562 +ILSVRC2012_val_00024277.JPEG:n02112018 +ILSVRC2012_val_00024278.JPEG:n02708093 +ILSVRC2012_val_00024279.JPEG:n02417914 +ILSVRC2012_val_00024280.JPEG:n13040303 +ILSVRC2012_val_00024281.JPEG:n04005630 +ILSVRC2012_val_00024282.JPEG:n02794156 +ILSVRC2012_val_00024283.JPEG:n01689811 +ILSVRC2012_val_00024284.JPEG:n02113186 +ILSVRC2012_val_00024285.JPEG:n03476991 +ILSVRC2012_val_00024286.JPEG:n03773504 +ILSVRC2012_val_00024287.JPEG:n03868863 +ILSVRC2012_val_00024288.JPEG:n03788365 +ILSVRC2012_val_00024289.JPEG:n02133161 +ILSVRC2012_val_00024290.JPEG:n02708093 +ILSVRC2012_val_00024291.JPEG:n07718747 +ILSVRC2012_val_00024292.JPEG:n02106030 +ILSVRC2012_val_00024293.JPEG:n03916031 +ILSVRC2012_val_00024294.JPEG:n02493793 +ILSVRC2012_val_00024295.JPEG:n02277742 +ILSVRC2012_val_00024296.JPEG:n02701002 +ILSVRC2012_val_00024297.JPEG:n04238763 +ILSVRC2012_val_00024298.JPEG:n07742313 +ILSVRC2012_val_00024299.JPEG:n01755581 +ILSVRC2012_val_00024300.JPEG:n02321529 +ILSVRC2012_val_00024301.JPEG:n01728572 +ILSVRC2012_val_00024302.JPEG:n12057211 +ILSVRC2012_val_00024303.JPEG:n03016953 +ILSVRC2012_val_00024304.JPEG:n04009552 +ILSVRC2012_val_00024305.JPEG:n02107312 +ILSVRC2012_val_00024306.JPEG:n04486054 +ILSVRC2012_val_00024307.JPEG:n03837869 +ILSVRC2012_val_00024308.JPEG:n04127249 +ILSVRC2012_val_00024309.JPEG:n03837869 +ILSVRC2012_val_00024310.JPEG:n03895866 +ILSVRC2012_val_00024311.JPEG:n03032252 +ILSVRC2012_val_00024312.JPEG:n04380533 +ILSVRC2012_val_00024313.JPEG:n02777292 +ILSVRC2012_val_00024314.JPEG:n01729322 +ILSVRC2012_val_00024315.JPEG:n02607072 +ILSVRC2012_val_00024316.JPEG:n03792972 +ILSVRC2012_val_00024317.JPEG:n03930630 +ILSVRC2012_val_00024318.JPEG:n02814533 +ILSVRC2012_val_00024319.JPEG:n04005630 +ILSVRC2012_val_00024320.JPEG:n04099969 +ILSVRC2012_val_00024321.JPEG:n02110806 +ILSVRC2012_val_00024322.JPEG:n03594734 +ILSVRC2012_val_00024323.JPEG:n03697007 +ILSVRC2012_val_00024324.JPEG:n02071294 +ILSVRC2012_val_00024325.JPEG:n02346627 +ILSVRC2012_val_00024326.JPEG:n02096294 +ILSVRC2012_val_00024327.JPEG:n01440764 +ILSVRC2012_val_00024328.JPEG:n12267677 +ILSVRC2012_val_00024329.JPEG:n02097658 +ILSVRC2012_val_00024330.JPEG:n02111889 +ILSVRC2012_val_00024331.JPEG:n03825788 +ILSVRC2012_val_00024332.JPEG:n04153751 +ILSVRC2012_val_00024333.JPEG:n04259630 +ILSVRC2012_val_00024334.JPEG:n04254680 +ILSVRC2012_val_00024335.JPEG:n02092002 +ILSVRC2012_val_00024336.JPEG:n01833805 +ILSVRC2012_val_00024337.JPEG:n04200800 +ILSVRC2012_val_00024338.JPEG:n04435653 +ILSVRC2012_val_00024339.JPEG:n07753113 +ILSVRC2012_val_00024340.JPEG:n03888257 +ILSVRC2012_val_00024341.JPEG:n01744401 +ILSVRC2012_val_00024342.JPEG:n04192698 +ILSVRC2012_val_00024343.JPEG:n02415577 +ILSVRC2012_val_00024344.JPEG:n04550184 +ILSVRC2012_val_00024345.JPEG:n02097474 +ILSVRC2012_val_00024346.JPEG:n02793495 +ILSVRC2012_val_00024347.JPEG:n04252225 +ILSVRC2012_val_00024348.JPEG:n03388549 +ILSVRC2012_val_00024349.JPEG:n02422106 +ILSVRC2012_val_00024350.JPEG:n02807133 +ILSVRC2012_val_00024351.JPEG:n02090622 +ILSVRC2012_val_00024352.JPEG:n03598930 +ILSVRC2012_val_00024353.JPEG:n01592084 +ILSVRC2012_val_00024354.JPEG:n01924916 +ILSVRC2012_val_00024355.JPEG:n07584110 +ILSVRC2012_val_00024356.JPEG:n02114712 +ILSVRC2012_val_00024357.JPEG:n03874599 +ILSVRC2012_val_00024358.JPEG:n03590841 +ILSVRC2012_val_00024359.JPEG:n09246464 +ILSVRC2012_val_00024360.JPEG:n04589890 +ILSVRC2012_val_00024361.JPEG:n03794056 +ILSVRC2012_val_00024362.JPEG:n03180011 +ILSVRC2012_val_00024363.JPEG:n02104029 +ILSVRC2012_val_00024364.JPEG:n03272562 +ILSVRC2012_val_00024365.JPEG:n04263257 +ILSVRC2012_val_00024366.JPEG:n03874599 +ILSVRC2012_val_00024367.JPEG:n07714990 +ILSVRC2012_val_00024368.JPEG:n02791124 +ILSVRC2012_val_00024369.JPEG:n03690938 +ILSVRC2012_val_00024370.JPEG:n02837789 +ILSVRC2012_val_00024371.JPEG:n02138441 +ILSVRC2012_val_00024372.JPEG:n02859443 +ILSVRC2012_val_00024373.JPEG:n03026506 +ILSVRC2012_val_00024374.JPEG:n02442845 +ILSVRC2012_val_00024375.JPEG:n04004767 +ILSVRC2012_val_00024376.JPEG:n02397096 +ILSVRC2012_val_00024377.JPEG:n04120489 +ILSVRC2012_val_00024378.JPEG:n01882714 +ILSVRC2012_val_00024379.JPEG:n03124170 +ILSVRC2012_val_00024380.JPEG:n03992509 +ILSVRC2012_val_00024381.JPEG:n01818515 +ILSVRC2012_val_00024382.JPEG:n03124170 +ILSVRC2012_val_00024383.JPEG:n02002724 +ILSVRC2012_val_00024384.JPEG:n03680355 +ILSVRC2012_val_00024385.JPEG:n02096051 +ILSVRC2012_val_00024386.JPEG:n02492660 +ILSVRC2012_val_00024387.JPEG:n04033995 +ILSVRC2012_val_00024388.JPEG:n04019541 +ILSVRC2012_val_00024389.JPEG:n02108915 +ILSVRC2012_val_00024390.JPEG:n01872401 +ILSVRC2012_val_00024391.JPEG:n04366367 +ILSVRC2012_val_00024392.JPEG:n04501370 +ILSVRC2012_val_00024393.JPEG:n04355338 +ILSVRC2012_val_00024394.JPEG:n03661043 +ILSVRC2012_val_00024395.JPEG:n02536864 +ILSVRC2012_val_00024396.JPEG:n01796340 +ILSVRC2012_val_00024397.JPEG:n02326432 +ILSVRC2012_val_00024398.JPEG:n02493509 +ILSVRC2012_val_00024399.JPEG:n02099849 +ILSVRC2012_val_00024400.JPEG:n02096051 +ILSVRC2012_val_00024401.JPEG:n02974003 +ILSVRC2012_val_00024402.JPEG:n03481172 +ILSVRC2012_val_00024403.JPEG:n03089624 +ILSVRC2012_val_00024404.JPEG:n01773157 +ILSVRC2012_val_00024405.JPEG:n03445777 +ILSVRC2012_val_00024406.JPEG:n02138441 +ILSVRC2012_val_00024407.JPEG:n07565083 +ILSVRC2012_val_00024408.JPEG:n03916031 +ILSVRC2012_val_00024409.JPEG:n02363005 +ILSVRC2012_val_00024410.JPEG:n01944390 +ILSVRC2012_val_00024411.JPEG:n02093754 +ILSVRC2012_val_00024412.JPEG:n04560804 +ILSVRC2012_val_00024413.JPEG:n12267677 +ILSVRC2012_val_00024414.JPEG:n03967562 +ILSVRC2012_val_00024415.JPEG:n07932039 +ILSVRC2012_val_00024416.JPEG:n03666591 +ILSVRC2012_val_00024417.JPEG:n02256656 +ILSVRC2012_val_00024418.JPEG:n03770439 +ILSVRC2012_val_00024419.JPEG:n04509417 +ILSVRC2012_val_00024420.JPEG:n03720891 +ILSVRC2012_val_00024421.JPEG:n07565083 +ILSVRC2012_val_00024422.JPEG:n07875152 +ILSVRC2012_val_00024423.JPEG:n01843383 +ILSVRC2012_val_00024424.JPEG:n03481172 +ILSVRC2012_val_00024425.JPEG:n02708093 +ILSVRC2012_val_00024426.JPEG:n02165105 +ILSVRC2012_val_00024427.JPEG:n02123394 +ILSVRC2012_val_00024428.JPEG:n01644900 +ILSVRC2012_val_00024429.JPEG:n02109961 +ILSVRC2012_val_00024430.JPEG:n04335435 +ILSVRC2012_val_00024431.JPEG:n02096177 +ILSVRC2012_val_00024432.JPEG:n02110185 +ILSVRC2012_val_00024433.JPEG:n02687172 +ILSVRC2012_val_00024434.JPEG:n04116512 +ILSVRC2012_val_00024435.JPEG:n01693334 +ILSVRC2012_val_00024436.JPEG:n03133878 +ILSVRC2012_val_00024437.JPEG:n02493793 +ILSVRC2012_val_00024438.JPEG:n01806143 +ILSVRC2012_val_00024439.JPEG:n07892512 +ILSVRC2012_val_00024440.JPEG:n03670208 +ILSVRC2012_val_00024441.JPEG:n04264628 +ILSVRC2012_val_00024442.JPEG:n03014705 +ILSVRC2012_val_00024443.JPEG:n07615774 +ILSVRC2012_val_00024444.JPEG:n02992211 +ILSVRC2012_val_00024445.JPEG:n03063599 +ILSVRC2012_val_00024446.JPEG:n04209239 +ILSVRC2012_val_00024447.JPEG:n02489166 +ILSVRC2012_val_00024448.JPEG:n07920052 +ILSVRC2012_val_00024449.JPEG:n04081281 +ILSVRC2012_val_00024450.JPEG:n04486054 +ILSVRC2012_val_00024451.JPEG:n02783161 +ILSVRC2012_val_00024452.JPEG:n03594734 +ILSVRC2012_val_00024453.JPEG:n03016953 +ILSVRC2012_val_00024454.JPEG:n02834397 +ILSVRC2012_val_00024455.JPEG:n04409515 +ILSVRC2012_val_00024456.JPEG:n03544143 +ILSVRC2012_val_00024457.JPEG:n01924916 +ILSVRC2012_val_00024458.JPEG:n02174001 +ILSVRC2012_val_00024459.JPEG:n04599235 +ILSVRC2012_val_00024460.JPEG:n07754684 +ILSVRC2012_val_00024461.JPEG:n07753275 +ILSVRC2012_val_00024462.JPEG:n02112706 +ILSVRC2012_val_00024463.JPEG:n03197337 +ILSVRC2012_val_00024464.JPEG:n02095570 +ILSVRC2012_val_00024465.JPEG:n02120079 +ILSVRC2012_val_00024466.JPEG:n03804744 +ILSVRC2012_val_00024467.JPEG:n01820546 +ILSVRC2012_val_00024468.JPEG:n02099849 +ILSVRC2012_val_00024469.JPEG:n04004767 +ILSVRC2012_val_00024470.JPEG:n02092339 +ILSVRC2012_val_00024471.JPEG:n03983396 +ILSVRC2012_val_00024472.JPEG:n01749939 +ILSVRC2012_val_00024473.JPEG:n04162706 +ILSVRC2012_val_00024474.JPEG:n04264628 +ILSVRC2012_val_00024475.JPEG:n03598930 +ILSVRC2012_val_00024476.JPEG:n02098286 +ILSVRC2012_val_00024477.JPEG:n07892512 +ILSVRC2012_val_00024478.JPEG:n03929660 +ILSVRC2012_val_00024479.JPEG:n04209133 +ILSVRC2012_val_00024480.JPEG:n03000684 +ILSVRC2012_val_00024481.JPEG:n04589890 +ILSVRC2012_val_00024482.JPEG:n02963159 +ILSVRC2012_val_00024483.JPEG:n02206856 +ILSVRC2012_val_00024484.JPEG:n03970156 +ILSVRC2012_val_00024485.JPEG:n04418357 +ILSVRC2012_val_00024486.JPEG:n02090379 +ILSVRC2012_val_00024487.JPEG:n03785016 +ILSVRC2012_val_00024488.JPEG:n02488291 +ILSVRC2012_val_00024489.JPEG:n04501370 +ILSVRC2012_val_00024490.JPEG:n04118538 +ILSVRC2012_val_00024491.JPEG:n04311174 +ILSVRC2012_val_00024492.JPEG:n03838899 +ILSVRC2012_val_00024493.JPEG:n02906734 +ILSVRC2012_val_00024494.JPEG:n01665541 +ILSVRC2012_val_00024495.JPEG:n03188531 +ILSVRC2012_val_00024496.JPEG:n03642806 +ILSVRC2012_val_00024497.JPEG:n03220513 +ILSVRC2012_val_00024498.JPEG:n02105855 +ILSVRC2012_val_00024499.JPEG:n03642806 +ILSVRC2012_val_00024500.JPEG:n02123394 +ILSVRC2012_val_00024501.JPEG:n02457408 +ILSVRC2012_val_00024502.JPEG:n03208938 +ILSVRC2012_val_00024503.JPEG:n04536866 +ILSVRC2012_val_00024504.JPEG:n02056570 +ILSVRC2012_val_00024505.JPEG:n02088466 +ILSVRC2012_val_00024506.JPEG:n04019541 +ILSVRC2012_val_00024507.JPEG:n02165456 +ILSVRC2012_val_00024508.JPEG:n02097209 +ILSVRC2012_val_00024509.JPEG:n02108000 +ILSVRC2012_val_00024510.JPEG:n04536866 +ILSVRC2012_val_00024511.JPEG:n02777292 +ILSVRC2012_val_00024512.JPEG:n02939185 +ILSVRC2012_val_00024513.JPEG:n04366367 +ILSVRC2012_val_00024514.JPEG:n01616318 +ILSVRC2012_val_00024515.JPEG:n03337140 +ILSVRC2012_val_00024516.JPEG:n04229816 +ILSVRC2012_val_00024517.JPEG:n03792782 +ILSVRC2012_val_00024518.JPEG:n07831146 +ILSVRC2012_val_00024519.JPEG:n03903868 +ILSVRC2012_val_00024520.JPEG:n03041632 +ILSVRC2012_val_00024521.JPEG:n02089867 +ILSVRC2012_val_00024522.JPEG:n07695742 +ILSVRC2012_val_00024523.JPEG:n03534580 +ILSVRC2012_val_00024524.JPEG:n03271574 +ILSVRC2012_val_00024525.JPEG:n01843383 +ILSVRC2012_val_00024526.JPEG:n07836838 +ILSVRC2012_val_00024527.JPEG:n02279972 +ILSVRC2012_val_00024528.JPEG:n07584110 +ILSVRC2012_val_00024529.JPEG:n02119789 +ILSVRC2012_val_00024530.JPEG:n01843065 +ILSVRC2012_val_00024531.JPEG:n02206856 +ILSVRC2012_val_00024532.JPEG:n03042490 +ILSVRC2012_val_00024533.JPEG:n02104029 +ILSVRC2012_val_00024534.JPEG:n04447861 +ILSVRC2012_val_00024535.JPEG:n03814906 +ILSVRC2012_val_00024536.JPEG:n02280649 +ILSVRC2012_val_00024537.JPEG:n03494278 +ILSVRC2012_val_00024538.JPEG:n02256656 +ILSVRC2012_val_00024539.JPEG:n02909870 +ILSVRC2012_val_00024540.JPEG:n03602883 +ILSVRC2012_val_00024541.JPEG:n01748264 +ILSVRC2012_val_00024542.JPEG:n02093428 +ILSVRC2012_val_00024543.JPEG:n03841143 +ILSVRC2012_val_00024544.JPEG:n03710193 +ILSVRC2012_val_00024545.JPEG:n01675722 +ILSVRC2012_val_00024546.JPEG:n02395406 +ILSVRC2012_val_00024547.JPEG:n03250847 +ILSVRC2012_val_00024548.JPEG:n02397096 +ILSVRC2012_val_00024549.JPEG:n12267677 +ILSVRC2012_val_00024550.JPEG:n03770679 +ILSVRC2012_val_00024551.JPEG:n02007558 +ILSVRC2012_val_00024552.JPEG:n03642806 +ILSVRC2012_val_00024553.JPEG:n07871810 +ILSVRC2012_val_00024554.JPEG:n03742115 +ILSVRC2012_val_00024555.JPEG:n02190166 +ILSVRC2012_val_00024556.JPEG:n07716358 +ILSVRC2012_val_00024557.JPEG:n01978455 +ILSVRC2012_val_00024558.JPEG:n02169497 +ILSVRC2012_val_00024559.JPEG:n04204347 +ILSVRC2012_val_00024560.JPEG:n03417042 +ILSVRC2012_val_00024561.JPEG:n02793495 +ILSVRC2012_val_00024562.JPEG:n03530642 +ILSVRC2012_val_00024563.JPEG:n03188531 +ILSVRC2012_val_00024564.JPEG:n02105505 +ILSVRC2012_val_00024565.JPEG:n02804414 +ILSVRC2012_val_00024566.JPEG:n02093754 +ILSVRC2012_val_00024567.JPEG:n02092339 +ILSVRC2012_val_00024568.JPEG:n02860847 +ILSVRC2012_val_00024569.JPEG:n02085936 +ILSVRC2012_val_00024570.JPEG:n02786058 +ILSVRC2012_val_00024571.JPEG:n02056570 +ILSVRC2012_val_00024572.JPEG:n02165456 +ILSVRC2012_val_00024573.JPEG:n03710637 +ILSVRC2012_val_00024574.JPEG:n04200800 +ILSVRC2012_val_00024575.JPEG:n04592741 +ILSVRC2012_val_00024576.JPEG:n03935335 +ILSVRC2012_val_00024577.JPEG:n02102973 +ILSVRC2012_val_00024578.JPEG:n04296562 +ILSVRC2012_val_00024579.JPEG:n04328186 +ILSVRC2012_val_00024580.JPEG:n12267677 +ILSVRC2012_val_00024581.JPEG:n01824575 +ILSVRC2012_val_00024582.JPEG:n02494079 +ILSVRC2012_val_00024583.JPEG:n02730930 +ILSVRC2012_val_00024584.JPEG:n02356798 +ILSVRC2012_val_00024585.JPEG:n03937543 +ILSVRC2012_val_00024586.JPEG:n03290653 +ILSVRC2012_val_00024587.JPEG:n02109047 +ILSVRC2012_val_00024588.JPEG:n02112137 +ILSVRC2012_val_00024589.JPEG:n02104365 +ILSVRC2012_val_00024590.JPEG:n02085620 +ILSVRC2012_val_00024591.JPEG:n09246464 +ILSVRC2012_val_00024592.JPEG:n01817953 +ILSVRC2012_val_00024593.JPEG:n03345487 +ILSVRC2012_val_00024594.JPEG:n02410509 +ILSVRC2012_val_00024595.JPEG:n02281787 +ILSVRC2012_val_00024596.JPEG:n04487081 +ILSVRC2012_val_00024597.JPEG:n01770393 +ILSVRC2012_val_00024598.JPEG:n03814906 +ILSVRC2012_val_00024599.JPEG:n01728920 +ILSVRC2012_val_00024600.JPEG:n02481823 +ILSVRC2012_val_00024601.JPEG:n01768244 +ILSVRC2012_val_00024602.JPEG:n03891251 +ILSVRC2012_val_00024603.JPEG:n04111531 +ILSVRC2012_val_00024604.JPEG:n03347037 +ILSVRC2012_val_00024605.JPEG:n03929660 +ILSVRC2012_val_00024606.JPEG:n02951585 +ILSVRC2012_val_00024607.JPEG:n02840245 +ILSVRC2012_val_00024608.JPEG:n02489166 +ILSVRC2012_val_00024609.JPEG:n01756291 +ILSVRC2012_val_00024610.JPEG:n02669723 +ILSVRC2012_val_00024611.JPEG:n07583066 +ILSVRC2012_val_00024612.JPEG:n02268443 +ILSVRC2012_val_00024613.JPEG:n04552348 +ILSVRC2012_val_00024614.JPEG:n04263257 +ILSVRC2012_val_00024615.JPEG:n04371774 +ILSVRC2012_val_00024616.JPEG:n03379051 +ILSVRC2012_val_00024617.JPEG:n04355338 +ILSVRC2012_val_00024618.JPEG:n04355933 +ILSVRC2012_val_00024619.JPEG:n04118538 +ILSVRC2012_val_00024620.JPEG:n04099969 +ILSVRC2012_val_00024621.JPEG:n04507155 +ILSVRC2012_val_00024622.JPEG:n02480495 +ILSVRC2012_val_00024623.JPEG:n03814639 +ILSVRC2012_val_00024624.JPEG:n02105855 +ILSVRC2012_val_00024625.JPEG:n02487347 +ILSVRC2012_val_00024626.JPEG:n04553703 +ILSVRC2012_val_00024627.JPEG:n04310018 +ILSVRC2012_val_00024628.JPEG:n03895866 +ILSVRC2012_val_00024629.JPEG:n03000247 +ILSVRC2012_val_00024630.JPEG:n01796340 +ILSVRC2012_val_00024631.JPEG:n03903868 +ILSVRC2012_val_00024632.JPEG:n03903868 +ILSVRC2012_val_00024633.JPEG:n07583066 +ILSVRC2012_val_00024634.JPEG:n04192698 +ILSVRC2012_val_00024635.JPEG:n02018795 +ILSVRC2012_val_00024636.JPEG:n02096177 +ILSVRC2012_val_00024637.JPEG:n02098286 +ILSVRC2012_val_00024638.JPEG:n03970156 +ILSVRC2012_val_00024639.JPEG:n03733281 +ILSVRC2012_val_00024640.JPEG:n07614500 +ILSVRC2012_val_00024641.JPEG:n03388043 +ILSVRC2012_val_00024642.JPEG:n02110958 +ILSVRC2012_val_00024643.JPEG:n01601694 +ILSVRC2012_val_00024644.JPEG:n07715103 +ILSVRC2012_val_00024645.JPEG:n02127052 +ILSVRC2012_val_00024646.JPEG:n02325366 +ILSVRC2012_val_00024647.JPEG:n03673027 +ILSVRC2012_val_00024648.JPEG:n02950826 +ILSVRC2012_val_00024649.JPEG:n02091467 +ILSVRC2012_val_00024650.JPEG:n03110669 +ILSVRC2012_val_00024651.JPEG:n03840681 +ILSVRC2012_val_00024652.JPEG:n03680355 +ILSVRC2012_val_00024653.JPEG:n02441942 +ILSVRC2012_val_00024654.JPEG:n03485407 +ILSVRC2012_val_00024655.JPEG:n02097474 +ILSVRC2012_val_00024656.JPEG:n02398521 +ILSVRC2012_val_00024657.JPEG:n02776631 +ILSVRC2012_val_00024658.JPEG:n02701002 +ILSVRC2012_val_00024659.JPEG:n02325366 +ILSVRC2012_val_00024660.JPEG:n03388043 +ILSVRC2012_val_00024661.JPEG:n07873807 +ILSVRC2012_val_00024662.JPEG:n03763968 +ILSVRC2012_val_00024663.JPEG:n04515003 +ILSVRC2012_val_00024664.JPEG:n02094258 +ILSVRC2012_val_00024665.JPEG:n02422699 +ILSVRC2012_val_00024666.JPEG:n01667114 +ILSVRC2012_val_00024667.JPEG:n04263257 +ILSVRC2012_val_00024668.JPEG:n07590611 +ILSVRC2012_val_00024669.JPEG:n02110185 +ILSVRC2012_val_00024670.JPEG:n03899768 +ILSVRC2012_val_00024671.JPEG:n03877845 +ILSVRC2012_val_00024672.JPEG:n03197337 +ILSVRC2012_val_00024673.JPEG:n12144580 +ILSVRC2012_val_00024674.JPEG:n04152593 +ILSVRC2012_val_00024675.JPEG:n02108089 +ILSVRC2012_val_00024676.JPEG:n02493793 +ILSVRC2012_val_00024677.JPEG:n02105855 +ILSVRC2012_val_00024678.JPEG:n03481172 +ILSVRC2012_val_00024679.JPEG:n04228054 +ILSVRC2012_val_00024680.JPEG:n03899768 +ILSVRC2012_val_00024681.JPEG:n02093754 +ILSVRC2012_val_00024682.JPEG:n01737021 +ILSVRC2012_val_00024683.JPEG:n02415577 +ILSVRC2012_val_00024684.JPEG:n01685808 +ILSVRC2012_val_00024685.JPEG:n01773157 +ILSVRC2012_val_00024686.JPEG:n02101388 +ILSVRC2012_val_00024687.JPEG:n03710721 +ILSVRC2012_val_00024688.JPEG:n01873310 +ILSVRC2012_val_00024689.JPEG:n03627232 +ILSVRC2012_val_00024690.JPEG:n02708093 +ILSVRC2012_val_00024691.JPEG:n02102318 +ILSVRC2012_val_00024692.JPEG:n07747607 +ILSVRC2012_val_00024693.JPEG:n02791124 +ILSVRC2012_val_00024694.JPEG:n02870880 +ILSVRC2012_val_00024695.JPEG:n03388549 +ILSVRC2012_val_00024696.JPEG:n04372370 +ILSVRC2012_val_00024697.JPEG:n03775071 +ILSVRC2012_val_00024698.JPEG:n04347754 +ILSVRC2012_val_00024699.JPEG:n03026506 +ILSVRC2012_val_00024700.JPEG:n07720875 +ILSVRC2012_val_00024701.JPEG:n01883070 +ILSVRC2012_val_00024702.JPEG:n03690938 +ILSVRC2012_val_00024703.JPEG:n03776460 +ILSVRC2012_val_00024704.JPEG:n01558993 +ILSVRC2012_val_00024705.JPEG:n04552348 +ILSVRC2012_val_00024706.JPEG:n03457902 +ILSVRC2012_val_00024707.JPEG:n07768694 +ILSVRC2012_val_00024708.JPEG:n04356056 +ILSVRC2012_val_00024709.JPEG:n04485082 +ILSVRC2012_val_00024710.JPEG:n09288635 +ILSVRC2012_val_00024711.JPEG:n07760859 +ILSVRC2012_val_00024712.JPEG:n03991062 +ILSVRC2012_val_00024713.JPEG:n04136333 +ILSVRC2012_val_00024714.JPEG:n03938244 +ILSVRC2012_val_00024715.JPEG:n02102177 +ILSVRC2012_val_00024716.JPEG:n03991062 +ILSVRC2012_val_00024717.JPEG:n04550184 +ILSVRC2012_val_00024718.JPEG:n04127249 +ILSVRC2012_val_00024719.JPEG:n01498041 +ILSVRC2012_val_00024720.JPEG:n03691459 +ILSVRC2012_val_00024721.JPEG:n03255030 +ILSVRC2012_val_00024722.JPEG:n02417914 +ILSVRC2012_val_00024723.JPEG:n02099429 +ILSVRC2012_val_00024724.JPEG:n04254777 +ILSVRC2012_val_00024725.JPEG:n04277352 +ILSVRC2012_val_00024726.JPEG:n01855032 +ILSVRC2012_val_00024727.JPEG:n01983481 +ILSVRC2012_val_00024728.JPEG:n04604644 +ILSVRC2012_val_00024729.JPEG:n02102973 +ILSVRC2012_val_00024730.JPEG:n02790996 +ILSVRC2012_val_00024731.JPEG:n02094258 +ILSVRC2012_val_00024732.JPEG:n02489166 +ILSVRC2012_val_00024733.JPEG:n03887697 +ILSVRC2012_val_00024734.JPEG:n02443114 +ILSVRC2012_val_00024735.JPEG:n04228054 +ILSVRC2012_val_00024736.JPEG:n01667778 +ILSVRC2012_val_00024737.JPEG:n02172182 +ILSVRC2012_val_00024738.JPEG:n04133789 +ILSVRC2012_val_00024739.JPEG:n03196217 +ILSVRC2012_val_00024740.JPEG:n02018207 +ILSVRC2012_val_00024741.JPEG:n03124170 +ILSVRC2012_val_00024742.JPEG:n02841315 +ILSVRC2012_val_00024743.JPEG:n02174001 +ILSVRC2012_val_00024744.JPEG:n02138441 +ILSVRC2012_val_00024745.JPEG:n02364673 +ILSVRC2012_val_00024746.JPEG:n03874599 +ILSVRC2012_val_00024747.JPEG:n02690373 +ILSVRC2012_val_00024748.JPEG:n12267677 +ILSVRC2012_val_00024749.JPEG:n02071294 +ILSVRC2012_val_00024750.JPEG:n02396427 +ILSVRC2012_val_00024751.JPEG:n02100236 +ILSVRC2012_val_00024752.JPEG:n04125021 +ILSVRC2012_val_00024753.JPEG:n01704323 +ILSVRC2012_val_00024754.JPEG:n02281406 +ILSVRC2012_val_00024755.JPEG:n02226429 +ILSVRC2012_val_00024756.JPEG:n02097298 +ILSVRC2012_val_00024757.JPEG:n02787622 +ILSVRC2012_val_00024758.JPEG:n02086910 +ILSVRC2012_val_00024759.JPEG:n02415577 +ILSVRC2012_val_00024760.JPEG:n02123597 +ILSVRC2012_val_00024761.JPEG:n03977966 +ILSVRC2012_val_00024762.JPEG:n03743016 +ILSVRC2012_val_00024763.JPEG:n02951585 +ILSVRC2012_val_00024764.JPEG:n04548280 +ILSVRC2012_val_00024765.JPEG:n03216828 +ILSVRC2012_val_00024766.JPEG:n02096437 +ILSVRC2012_val_00024767.JPEG:n02233338 +ILSVRC2012_val_00024768.JPEG:n02536864 +ILSVRC2012_val_00024769.JPEG:n01773157 +ILSVRC2012_val_00024770.JPEG:n03657121 +ILSVRC2012_val_00024771.JPEG:n02883205 +ILSVRC2012_val_00024772.JPEG:n03777754 +ILSVRC2012_val_00024773.JPEG:n01843065 +ILSVRC2012_val_00024774.JPEG:n15075141 +ILSVRC2012_val_00024775.JPEG:n04462240 +ILSVRC2012_val_00024776.JPEG:n02086240 +ILSVRC2012_val_00024777.JPEG:n03832673 +ILSVRC2012_val_00024778.JPEG:n04026417 +ILSVRC2012_val_00024779.JPEG:n04346328 +ILSVRC2012_val_00024780.JPEG:n02808440 +ILSVRC2012_val_00024781.JPEG:n04152593 +ILSVRC2012_val_00024782.JPEG:n03017168 +ILSVRC2012_val_00024783.JPEG:n03710193 +ILSVRC2012_val_00024784.JPEG:n02110341 +ILSVRC2012_val_00024785.JPEG:n02111500 +ILSVRC2012_val_00024786.JPEG:n02117135 +ILSVRC2012_val_00024787.JPEG:n02018207 +ILSVRC2012_val_00024788.JPEG:n03769881 +ILSVRC2012_val_00024789.JPEG:n02087394 +ILSVRC2012_val_00024790.JPEG:n04286575 +ILSVRC2012_val_00024791.JPEG:n02105855 +ILSVRC2012_val_00024792.JPEG:n03218198 +ILSVRC2012_val_00024793.JPEG:n04509417 +ILSVRC2012_val_00024794.JPEG:n02749479 +ILSVRC2012_val_00024795.JPEG:n01756291 +ILSVRC2012_val_00024796.JPEG:n03584254 +ILSVRC2012_val_00024797.JPEG:n07613480 +ILSVRC2012_val_00024798.JPEG:n02437312 +ILSVRC2012_val_00024799.JPEG:n04458633 +ILSVRC2012_val_00024800.JPEG:n01518878 +ILSVRC2012_val_00024801.JPEG:n01677366 +ILSVRC2012_val_00024802.JPEG:n02797295 +ILSVRC2012_val_00024803.JPEG:n07717410 +ILSVRC2012_val_00024804.JPEG:n03775071 +ILSVRC2012_val_00024805.JPEG:n04209133 +ILSVRC2012_val_00024806.JPEG:n03425413 +ILSVRC2012_val_00024807.JPEG:n04347754 +ILSVRC2012_val_00024808.JPEG:n02028035 +ILSVRC2012_val_00024809.JPEG:n02085936 +ILSVRC2012_val_00024810.JPEG:n04317175 +ILSVRC2012_val_00024811.JPEG:n04310018 +ILSVRC2012_val_00024812.JPEG:n13044778 +ILSVRC2012_val_00024813.JPEG:n01693334 +ILSVRC2012_val_00024814.JPEG:n03047690 +ILSVRC2012_val_00024815.JPEG:n03983396 +ILSVRC2012_val_00024816.JPEG:n02268443 +ILSVRC2012_val_00024817.JPEG:n04442312 +ILSVRC2012_val_00024818.JPEG:n02109961 +ILSVRC2012_val_00024819.JPEG:n04019541 +ILSVRC2012_val_00024820.JPEG:n04335435 +ILSVRC2012_val_00024821.JPEG:n07932039 +ILSVRC2012_val_00024822.JPEG:n03743016 +ILSVRC2012_val_00024823.JPEG:n02268443 +ILSVRC2012_val_00024824.JPEG:n04523525 +ILSVRC2012_val_00024825.JPEG:n02134418 +ILSVRC2012_val_00024826.JPEG:n02860847 +ILSVRC2012_val_00024827.JPEG:n02096051 +ILSVRC2012_val_00024828.JPEG:n02817516 +ILSVRC2012_val_00024829.JPEG:n04238763 +ILSVRC2012_val_00024830.JPEG:n12620546 +ILSVRC2012_val_00024831.JPEG:n02092002 +ILSVRC2012_val_00024832.JPEG:n13037406 +ILSVRC2012_val_00024833.JPEG:n03000134 +ILSVRC2012_val_00024834.JPEG:n04228054 +ILSVRC2012_val_00024835.JPEG:n02002724 +ILSVRC2012_val_00024836.JPEG:n02086079 +ILSVRC2012_val_00024837.JPEG:n03394916 +ILSVRC2012_val_00024838.JPEG:n04265275 +ILSVRC2012_val_00024839.JPEG:n04136333 +ILSVRC2012_val_00024840.JPEG:n02481823 +ILSVRC2012_val_00024841.JPEG:n04041544 +ILSVRC2012_val_00024842.JPEG:n03272562 +ILSVRC2012_val_00024843.JPEG:n02999410 +ILSVRC2012_val_00024844.JPEG:n02488702 +ILSVRC2012_val_00024845.JPEG:n01824575 +ILSVRC2012_val_00024846.JPEG:n03967562 +ILSVRC2012_val_00024847.JPEG:n02730930 +ILSVRC2012_val_00024848.JPEG:n01843383 +ILSVRC2012_val_00024849.JPEG:n04604644 +ILSVRC2012_val_00024850.JPEG:n02177972 +ILSVRC2012_val_00024851.JPEG:n01744401 +ILSVRC2012_val_00024852.JPEG:n07860988 +ILSVRC2012_val_00024853.JPEG:n04153751 +ILSVRC2012_val_00024854.JPEG:n01491361 +ILSVRC2012_val_00024855.JPEG:n03297495 +ILSVRC2012_val_00024856.JPEG:n04346328 +ILSVRC2012_val_00024857.JPEG:n03956157 +ILSVRC2012_val_00024858.JPEG:n02325366 +ILSVRC2012_val_00024859.JPEG:n02974003 +ILSVRC2012_val_00024860.JPEG:n03733281 +ILSVRC2012_val_00024861.JPEG:n03899768 +ILSVRC2012_val_00024862.JPEG:n07717556 +ILSVRC2012_val_00024863.JPEG:n02114367 +ILSVRC2012_val_00024864.JPEG:n04366367 +ILSVRC2012_val_00024865.JPEG:n03400231 +ILSVRC2012_val_00024866.JPEG:n02808440 +ILSVRC2012_val_00024867.JPEG:n01968897 +ILSVRC2012_val_00024868.JPEG:n02259212 +ILSVRC2012_val_00024869.JPEG:n03642806 +ILSVRC2012_val_00024870.JPEG:n01955084 +ILSVRC2012_val_00024871.JPEG:n03776460 +ILSVRC2012_val_00024872.JPEG:n09835506 +ILSVRC2012_val_00024873.JPEG:n01775062 +ILSVRC2012_val_00024874.JPEG:n02979186 +ILSVRC2012_val_00024875.JPEG:n02093991 +ILSVRC2012_val_00024876.JPEG:n04263257 +ILSVRC2012_val_00024877.JPEG:n04485082 +ILSVRC2012_val_00024878.JPEG:n04482393 +ILSVRC2012_val_00024879.JPEG:n03179701 +ILSVRC2012_val_00024880.JPEG:n01739381 +ILSVRC2012_val_00024881.JPEG:n02088238 +ILSVRC2012_val_00024882.JPEG:n03991062 +ILSVRC2012_val_00024883.JPEG:n13040303 +ILSVRC2012_val_00024884.JPEG:n01534433 +ILSVRC2012_val_00024885.JPEG:n01978455 +ILSVRC2012_val_00024886.JPEG:n02480495 +ILSVRC2012_val_00024887.JPEG:n02086910 +ILSVRC2012_val_00024888.JPEG:n02097209 +ILSVRC2012_val_00024889.JPEG:n02096294 +ILSVRC2012_val_00024890.JPEG:n04209133 +ILSVRC2012_val_00024891.JPEG:n09428293 +ILSVRC2012_val_00024892.JPEG:n03018349 +ILSVRC2012_val_00024893.JPEG:n07871810 +ILSVRC2012_val_00024894.JPEG:n01986214 +ILSVRC2012_val_00024895.JPEG:n01491361 +ILSVRC2012_val_00024896.JPEG:n02106662 +ILSVRC2012_val_00024897.JPEG:n03028079 +ILSVRC2012_val_00024898.JPEG:n04179913 +ILSVRC2012_val_00024899.JPEG:n04264628 +ILSVRC2012_val_00024900.JPEG:n03450230 +ILSVRC2012_val_00024901.JPEG:n04376876 +ILSVRC2012_val_00024902.JPEG:n02129165 +ILSVRC2012_val_00024903.JPEG:n02127052 +ILSVRC2012_val_00024904.JPEG:n02111500 +ILSVRC2012_val_00024905.JPEG:n04254680 +ILSVRC2012_val_00024906.JPEG:n02951358 +ILSVRC2012_val_00024907.JPEG:n03854065 +ILSVRC2012_val_00024908.JPEG:n02488702 +ILSVRC2012_val_00024909.JPEG:n02834397 +ILSVRC2012_val_00024910.JPEG:n02128757 +ILSVRC2012_val_00024911.JPEG:n03075370 +ILSVRC2012_val_00024912.JPEG:n07583066 +ILSVRC2012_val_00024913.JPEG:n03047690 +ILSVRC2012_val_00024914.JPEG:n01829413 +ILSVRC2012_val_00024915.JPEG:n03124043 +ILSVRC2012_val_00024916.JPEG:n01843065 +ILSVRC2012_val_00024917.JPEG:n07697537 +ILSVRC2012_val_00024918.JPEG:n07734744 +ILSVRC2012_val_00024919.JPEG:n02834397 +ILSVRC2012_val_00024920.JPEG:n02814860 +ILSVRC2012_val_00024921.JPEG:n02481823 +ILSVRC2012_val_00024922.JPEG:n04356056 +ILSVRC2012_val_00024923.JPEG:n03124043 +ILSVRC2012_val_00024924.JPEG:n01990800 +ILSVRC2012_val_00024925.JPEG:n03291819 +ILSVRC2012_val_00024926.JPEG:n02487347 +ILSVRC2012_val_00024927.JPEG:n03658185 +ILSVRC2012_val_00024928.JPEG:n04404412 +ILSVRC2012_val_00024929.JPEG:n03791053 +ILSVRC2012_val_00024930.JPEG:n03866082 +ILSVRC2012_val_00024931.JPEG:n02930766 +ILSVRC2012_val_00024932.JPEG:n02074367 +ILSVRC2012_val_00024933.JPEG:n02777292 +ILSVRC2012_val_00024934.JPEG:n04458633 +ILSVRC2012_val_00024935.JPEG:n02098286 +ILSVRC2012_val_00024936.JPEG:n02843684 +ILSVRC2012_val_00024937.JPEG:n04592741 +ILSVRC2012_val_00024938.JPEG:n01641577 +ILSVRC2012_val_00024939.JPEG:n03529860 +ILSVRC2012_val_00024940.JPEG:n01484850 +ILSVRC2012_val_00024941.JPEG:n04141076 +ILSVRC2012_val_00024942.JPEG:n03485407 +ILSVRC2012_val_00024943.JPEG:n03590841 +ILSVRC2012_val_00024944.JPEG:n04037443 +ILSVRC2012_val_00024945.JPEG:n07613480 +ILSVRC2012_val_00024946.JPEG:n01688243 +ILSVRC2012_val_00024947.JPEG:n04074963 +ILSVRC2012_val_00024948.JPEG:n02701002 +ILSVRC2012_val_00024949.JPEG:n03535780 +ILSVRC2012_val_00024950.JPEG:n02090379 +ILSVRC2012_val_00024951.JPEG:n02111889 +ILSVRC2012_val_00024952.JPEG:n06874185 +ILSVRC2012_val_00024953.JPEG:n07693725 +ILSVRC2012_val_00024954.JPEG:n07802026 +ILSVRC2012_val_00024955.JPEG:n07754684 +ILSVRC2012_val_00024956.JPEG:n01774384 +ILSVRC2012_val_00024957.JPEG:n01514668 +ILSVRC2012_val_00024958.JPEG:n02028035 +ILSVRC2012_val_00024959.JPEG:n04423845 +ILSVRC2012_val_00024960.JPEG:n02096051 +ILSVRC2012_val_00024961.JPEG:n02115641 +ILSVRC2012_val_00024962.JPEG:n01774384 +ILSVRC2012_val_00024963.JPEG:n02894605 +ILSVRC2012_val_00024964.JPEG:n03026506 +ILSVRC2012_val_00024965.JPEG:n02666196 +ILSVRC2012_val_00024966.JPEG:n03690938 +ILSVRC2012_val_00024967.JPEG:n02112706 +ILSVRC2012_val_00024968.JPEG:n03787032 +ILSVRC2012_val_00024969.JPEG:n01748264 +ILSVRC2012_val_00024970.JPEG:n03733131 +ILSVRC2012_val_00024971.JPEG:n03920288 +ILSVRC2012_val_00024972.JPEG:n04141076 +ILSVRC2012_val_00024973.JPEG:n02101006 +ILSVRC2012_val_00024974.JPEG:n03944341 +ILSVRC2012_val_00024975.JPEG:n12267677 +ILSVRC2012_val_00024976.JPEG:n03782006 +ILSVRC2012_val_00024977.JPEG:n03924679 +ILSVRC2012_val_00024978.JPEG:n02437616 +ILSVRC2012_val_00024979.JPEG:n02992529 +ILSVRC2012_val_00024980.JPEG:n02871525 +ILSVRC2012_val_00024981.JPEG:n02104029 +ILSVRC2012_val_00024982.JPEG:n03376595 +ILSVRC2012_val_00024983.JPEG:n04243546 +ILSVRC2012_val_00024984.JPEG:n03854065 +ILSVRC2012_val_00024985.JPEG:n03983396 +ILSVRC2012_val_00024986.JPEG:n02104029 +ILSVRC2012_val_00024987.JPEG:n01883070 +ILSVRC2012_val_00024988.JPEG:n07716906 +ILSVRC2012_val_00024989.JPEG:n02092002 +ILSVRC2012_val_00024990.JPEG:n02114855 +ILSVRC2012_val_00024991.JPEG:n03255030 +ILSVRC2012_val_00024992.JPEG:n01873310 +ILSVRC2012_val_00024993.JPEG:n01704323 +ILSVRC2012_val_00024994.JPEG:n04192698 +ILSVRC2012_val_00024995.JPEG:n03485407 +ILSVRC2012_val_00024996.JPEG:n02916936 +ILSVRC2012_val_00024997.JPEG:n07590611 +ILSVRC2012_val_00024998.JPEG:n02869837 +ILSVRC2012_val_00024999.JPEG:n03527444 +ILSVRC2012_val_00025000.JPEG:n03595614 +ILSVRC2012_val_00025001.JPEG:n02105412 +ILSVRC2012_val_00025002.JPEG:n09835506 +ILSVRC2012_val_00025003.JPEG:n04033901 +ILSVRC2012_val_00025004.JPEG:n04285008 +ILSVRC2012_val_00025005.JPEG:n02326432 +ILSVRC2012_val_00025006.JPEG:n02104029 +ILSVRC2012_val_00025007.JPEG:n07716906 +ILSVRC2012_val_00025008.JPEG:n07760859 +ILSVRC2012_val_00025009.JPEG:n03832673 +ILSVRC2012_val_00025010.JPEG:n03492542 +ILSVRC2012_val_00025011.JPEG:n02408429 +ILSVRC2012_val_00025012.JPEG:n03781244 +ILSVRC2012_val_00025013.JPEG:n02099849 +ILSVRC2012_val_00025014.JPEG:n03840681 +ILSVRC2012_val_00025015.JPEG:n02092339 +ILSVRC2012_val_00025016.JPEG:n03590841 +ILSVRC2012_val_00025017.JPEG:n01685808 +ILSVRC2012_val_00025018.JPEG:n01694178 +ILSVRC2012_val_00025019.JPEG:n07753592 +ILSVRC2012_val_00025020.JPEG:n03535780 +ILSVRC2012_val_00025021.JPEG:n02730930 +ILSVRC2012_val_00025022.JPEG:n04270147 +ILSVRC2012_val_00025023.JPEG:n02011460 +ILSVRC2012_val_00025024.JPEG:n04483307 +ILSVRC2012_val_00025025.JPEG:n01688243 +ILSVRC2012_val_00025026.JPEG:n01737021 +ILSVRC2012_val_00025027.JPEG:n02033041 +ILSVRC2012_val_00025028.JPEG:n03100240 +ILSVRC2012_val_00025029.JPEG:n03447447 +ILSVRC2012_val_00025030.JPEG:n03584829 +ILSVRC2012_val_00025031.JPEG:n02483362 +ILSVRC2012_val_00025032.JPEG:n03998194 +ILSVRC2012_val_00025033.JPEG:n02483362 +ILSVRC2012_val_00025034.JPEG:n03481172 +ILSVRC2012_val_00025035.JPEG:n01558993 +ILSVRC2012_val_00025036.JPEG:n04606251 +ILSVRC2012_val_00025037.JPEG:n01537544 +ILSVRC2012_val_00025038.JPEG:n02808440 +ILSVRC2012_val_00025039.JPEG:n03825788 +ILSVRC2012_val_00025040.JPEG:n01773157 +ILSVRC2012_val_00025041.JPEG:n04507155 +ILSVRC2012_val_00025042.JPEG:n04141076 +ILSVRC2012_val_00025043.JPEG:n02504013 +ILSVRC2012_val_00025044.JPEG:n04562935 +ILSVRC2012_val_00025045.JPEG:n07590611 +ILSVRC2012_val_00025046.JPEG:n04357314 +ILSVRC2012_val_00025047.JPEG:n01608432 +ILSVRC2012_val_00025048.JPEG:n02097658 +ILSVRC2012_val_00025049.JPEG:n03950228 +ILSVRC2012_val_00025050.JPEG:n02814860 +ILSVRC2012_val_00025051.JPEG:n01498041 +ILSVRC2012_val_00025052.JPEG:n04553703 +ILSVRC2012_val_00025053.JPEG:n12768682 +ILSVRC2012_val_00025054.JPEG:n03032252 +ILSVRC2012_val_00025055.JPEG:n02097474 +ILSVRC2012_val_00025056.JPEG:n01955084 +ILSVRC2012_val_00025057.JPEG:n07695742 +ILSVRC2012_val_00025058.JPEG:n02483708 +ILSVRC2012_val_00025059.JPEG:n02106550 +ILSVRC2012_val_00025060.JPEG:n04515003 +ILSVRC2012_val_00025061.JPEG:n02226429 +ILSVRC2012_val_00025062.JPEG:n04370456 +ILSVRC2012_val_00025063.JPEG:n03000684 +ILSVRC2012_val_00025064.JPEG:n03837869 +ILSVRC2012_val_00025065.JPEG:n02113799 +ILSVRC2012_val_00025066.JPEG:n02102480 +ILSVRC2012_val_00025067.JPEG:n03459775 +ILSVRC2012_val_00025068.JPEG:n02120079 +ILSVRC2012_val_00025069.JPEG:n02071294 +ILSVRC2012_val_00025070.JPEG:n13054560 +ILSVRC2012_val_00025071.JPEG:n04192698 +ILSVRC2012_val_00025072.JPEG:n02504458 +ILSVRC2012_val_00025073.JPEG:n04372370 +ILSVRC2012_val_00025074.JPEG:n04251144 +ILSVRC2012_val_00025075.JPEG:n02006656 +ILSVRC2012_val_00025076.JPEG:n03908618 +ILSVRC2012_val_00025077.JPEG:n04311174 +ILSVRC2012_val_00025078.JPEG:n03018349 +ILSVRC2012_val_00025079.JPEG:n13133613 +ILSVRC2012_val_00025080.JPEG:n03796401 +ILSVRC2012_val_00025081.JPEG:n04409515 +ILSVRC2012_val_00025082.JPEG:n02102480 +ILSVRC2012_val_00025083.JPEG:n02843684 +ILSVRC2012_val_00025084.JPEG:n04040759 +ILSVRC2012_val_00025085.JPEG:n02086646 +ILSVRC2012_val_00025086.JPEG:n02948072 +ILSVRC2012_val_00025087.JPEG:n07836838 +ILSVRC2012_val_00025088.JPEG:n03476684 +ILSVRC2012_val_00025089.JPEG:n02236044 +ILSVRC2012_val_00025090.JPEG:n04296562 +ILSVRC2012_val_00025091.JPEG:n02017213 +ILSVRC2012_val_00025092.JPEG:n04612504 +ILSVRC2012_val_00025093.JPEG:n02769748 +ILSVRC2012_val_00025094.JPEG:n07717410 +ILSVRC2012_val_00025095.JPEG:n07717410 +ILSVRC2012_val_00025096.JPEG:n01751748 +ILSVRC2012_val_00025097.JPEG:n03773504 +ILSVRC2012_val_00025098.JPEG:n02085782 +ILSVRC2012_val_00025099.JPEG:n04562935 +ILSVRC2012_val_00025100.JPEG:n04239074 +ILSVRC2012_val_00025101.JPEG:n07760859 +ILSVRC2012_val_00025102.JPEG:n07768694 +ILSVRC2012_val_00025103.JPEG:n03160309 +ILSVRC2012_val_00025104.JPEG:n01692333 +ILSVRC2012_val_00025105.JPEG:n03045698 +ILSVRC2012_val_00025106.JPEG:n03272562 +ILSVRC2012_val_00025107.JPEG:n04417672 +ILSVRC2012_val_00025108.JPEG:n03954731 +ILSVRC2012_val_00025109.JPEG:n04505470 +ILSVRC2012_val_00025110.JPEG:n04154565 +ILSVRC2012_val_00025111.JPEG:n03691459 +ILSVRC2012_val_00025112.JPEG:n04209239 +ILSVRC2012_val_00025113.JPEG:n04409515 +ILSVRC2012_val_00025114.JPEG:n02363005 +ILSVRC2012_val_00025115.JPEG:n07734744 +ILSVRC2012_val_00025116.JPEG:n02422699 +ILSVRC2012_val_00025117.JPEG:n03529860 +ILSVRC2012_val_00025118.JPEG:n04235860 +ILSVRC2012_val_00025119.JPEG:n04536866 +ILSVRC2012_val_00025120.JPEG:n01981276 +ILSVRC2012_val_00025121.JPEG:n03888257 +ILSVRC2012_val_00025122.JPEG:n02276258 +ILSVRC2012_val_00025123.JPEG:n03388043 +ILSVRC2012_val_00025124.JPEG:n07718472 +ILSVRC2012_val_00025125.JPEG:n02869837 +ILSVRC2012_val_00025126.JPEG:n02006656 +ILSVRC2012_val_00025127.JPEG:n03595614 +ILSVRC2012_val_00025128.JPEG:n02917067 +ILSVRC2012_val_00025129.JPEG:n01440764 +ILSVRC2012_val_00025130.JPEG:n01855032 +ILSVRC2012_val_00025131.JPEG:n03930630 +ILSVRC2012_val_00025132.JPEG:n02105505 +ILSVRC2012_val_00025133.JPEG:n01491361 +ILSVRC2012_val_00025134.JPEG:n03345487 +ILSVRC2012_val_00025135.JPEG:n04372370 +ILSVRC2012_val_00025136.JPEG:n03187595 +ILSVRC2012_val_00025137.JPEG:n01491361 +ILSVRC2012_val_00025138.JPEG:n04264628 +ILSVRC2012_val_00025139.JPEG:n04557648 +ILSVRC2012_val_00025140.JPEG:n02119022 +ILSVRC2012_val_00025141.JPEG:n02607072 +ILSVRC2012_val_00025142.JPEG:n02396427 +ILSVRC2012_val_00025143.JPEG:n07615774 +ILSVRC2012_val_00025144.JPEG:n04553703 +ILSVRC2012_val_00025145.JPEG:n07718472 +ILSVRC2012_val_00025146.JPEG:n03530642 +ILSVRC2012_val_00025147.JPEG:n02100583 +ILSVRC2012_val_00025148.JPEG:n04557648 +ILSVRC2012_val_00025149.JPEG:n03485407 +ILSVRC2012_val_00025150.JPEG:n07745940 +ILSVRC2012_val_00025151.JPEG:n01531178 +ILSVRC2012_val_00025152.JPEG:n03954731 +ILSVRC2012_val_00025153.JPEG:n04465501 +ILSVRC2012_val_00025154.JPEG:n12768682 +ILSVRC2012_val_00025155.JPEG:n04486054 +ILSVRC2012_val_00025156.JPEG:n03595614 +ILSVRC2012_val_00025157.JPEG:n04548362 +ILSVRC2012_val_00025158.JPEG:n07753113 +ILSVRC2012_val_00025159.JPEG:n02701002 +ILSVRC2012_val_00025160.JPEG:n04525038 +ILSVRC2012_val_00025161.JPEG:n02317335 +ILSVRC2012_val_00025162.JPEG:n02443484 +ILSVRC2012_val_00025163.JPEG:n02939185 +ILSVRC2012_val_00025164.JPEG:n03314780 +ILSVRC2012_val_00025165.JPEG:n02089078 +ILSVRC2012_val_00025166.JPEG:n02859443 +ILSVRC2012_val_00025167.JPEG:n02091467 +ILSVRC2012_val_00025168.JPEG:n02124075 +ILSVRC2012_val_00025169.JPEG:n03690938 +ILSVRC2012_val_00025170.JPEG:n02091831 +ILSVRC2012_val_00025171.JPEG:n02454379 +ILSVRC2012_val_00025172.JPEG:n04065272 +ILSVRC2012_val_00025173.JPEG:n03196217 +ILSVRC2012_val_00025174.JPEG:n02655020 +ILSVRC2012_val_00025175.JPEG:n04487394 +ILSVRC2012_val_00025176.JPEG:n04286575 +ILSVRC2012_val_00025177.JPEG:n03125729 +ILSVRC2012_val_00025178.JPEG:n03854065 +ILSVRC2012_val_00025179.JPEG:n03670208 +ILSVRC2012_val_00025180.JPEG:n02108422 +ILSVRC2012_val_00025181.JPEG:n02102480 +ILSVRC2012_val_00025182.JPEG:n02988304 +ILSVRC2012_val_00025183.JPEG:n02009229 +ILSVRC2012_val_00025184.JPEG:n02099267 +ILSVRC2012_val_00025185.JPEG:n02097209 +ILSVRC2012_val_00025186.JPEG:n02948072 +ILSVRC2012_val_00025187.JPEG:n02110806 +ILSVRC2012_val_00025188.JPEG:n02177972 +ILSVRC2012_val_00025189.JPEG:n03494278 +ILSVRC2012_val_00025190.JPEG:n01737021 +ILSVRC2012_val_00025191.JPEG:n13133613 +ILSVRC2012_val_00025192.JPEG:n04447861 +ILSVRC2012_val_00025193.JPEG:n04591713 +ILSVRC2012_val_00025194.JPEG:n03495258 +ILSVRC2012_val_00025195.JPEG:n02859443 +ILSVRC2012_val_00025196.JPEG:n02860847 +ILSVRC2012_val_00025197.JPEG:n04554684 +ILSVRC2012_val_00025198.JPEG:n03637318 +ILSVRC2012_val_00025199.JPEG:n04258138 +ILSVRC2012_val_00025200.JPEG:n01797886 +ILSVRC2012_val_00025201.JPEG:n03095699 +ILSVRC2012_val_00025202.JPEG:n04041544 +ILSVRC2012_val_00025203.JPEG:n03602883 +ILSVRC2012_val_00025204.JPEG:n04525038 +ILSVRC2012_val_00025205.JPEG:n03706229 +ILSVRC2012_val_00025206.JPEG:n02093859 +ILSVRC2012_val_00025207.JPEG:n02119022 +ILSVRC2012_val_00025208.JPEG:n02454379 +ILSVRC2012_val_00025209.JPEG:n07614500 +ILSVRC2012_val_00025210.JPEG:n02276258 +ILSVRC2012_val_00025211.JPEG:n07714571 +ILSVRC2012_val_00025212.JPEG:n02177972 +ILSVRC2012_val_00025213.JPEG:n02129604 +ILSVRC2012_val_00025214.JPEG:n01601694 +ILSVRC2012_val_00025215.JPEG:n04355338 +ILSVRC2012_val_00025216.JPEG:n02999410 +ILSVRC2012_val_00025217.JPEG:n07760859 +ILSVRC2012_val_00025218.JPEG:n02165456 +ILSVRC2012_val_00025219.JPEG:n02111129 +ILSVRC2012_val_00025220.JPEG:n03220513 +ILSVRC2012_val_00025221.JPEG:n02437616 +ILSVRC2012_val_00025222.JPEG:n04465501 +ILSVRC2012_val_00025223.JPEG:n03272010 +ILSVRC2012_val_00025224.JPEG:n02167151 +ILSVRC2012_val_00025225.JPEG:n02174001 +ILSVRC2012_val_00025226.JPEG:n02607072 +ILSVRC2012_val_00025227.JPEG:n04254120 +ILSVRC2012_val_00025228.JPEG:n07584110 +ILSVRC2012_val_00025229.JPEG:n03388549 +ILSVRC2012_val_00025230.JPEG:n03063599 +ILSVRC2012_val_00025231.JPEG:n02795169 +ILSVRC2012_val_00025232.JPEG:n02727426 +ILSVRC2012_val_00025233.JPEG:n02799071 +ILSVRC2012_val_00025234.JPEG:n10565667 +ILSVRC2012_val_00025235.JPEG:n02454379 +ILSVRC2012_val_00025236.JPEG:n07717410 +ILSVRC2012_val_00025237.JPEG:n02504013 +ILSVRC2012_val_00025238.JPEG:n04266014 +ILSVRC2012_val_00025239.JPEG:n04493381 +ILSVRC2012_val_00025240.JPEG:n03832673 +ILSVRC2012_val_00025241.JPEG:n02033041 +ILSVRC2012_val_00025242.JPEG:n02447366 +ILSVRC2012_val_00025243.JPEG:n03314780 +ILSVRC2012_val_00025244.JPEG:n02930766 +ILSVRC2012_val_00025245.JPEG:n02110806 +ILSVRC2012_val_00025246.JPEG:n04033901 +ILSVRC2012_val_00025247.JPEG:n02870880 +ILSVRC2012_val_00025248.JPEG:n01872401 +ILSVRC2012_val_00025249.JPEG:n03063689 +ILSVRC2012_val_00025250.JPEG:n03814906 +ILSVRC2012_val_00025251.JPEG:n01798484 +ILSVRC2012_val_00025252.JPEG:n02219486 +ILSVRC2012_val_00025253.JPEG:n02111129 +ILSVRC2012_val_00025254.JPEG:n03124170 +ILSVRC2012_val_00025255.JPEG:n03443371 +ILSVRC2012_val_00025256.JPEG:n01855672 +ILSVRC2012_val_00025257.JPEG:n03089624 +ILSVRC2012_val_00025258.JPEG:n04239074 +ILSVRC2012_val_00025259.JPEG:n03814906 +ILSVRC2012_val_00025260.JPEG:n04285008 +ILSVRC2012_val_00025261.JPEG:n02097474 +ILSVRC2012_val_00025262.JPEG:n01819313 +ILSVRC2012_val_00025263.JPEG:n02364673 +ILSVRC2012_val_00025264.JPEG:n03773504 +ILSVRC2012_val_00025265.JPEG:n04310018 +ILSVRC2012_val_00025266.JPEG:n04398044 +ILSVRC2012_val_00025267.JPEG:n13054560 +ILSVRC2012_val_00025268.JPEG:n01665541 +ILSVRC2012_val_00025269.JPEG:n02025239 +ILSVRC2012_val_00025270.JPEG:n03976657 +ILSVRC2012_val_00025271.JPEG:n04553703 +ILSVRC2012_val_00025272.JPEG:n07715103 +ILSVRC2012_val_00025273.JPEG:n02018795 +ILSVRC2012_val_00025274.JPEG:n03794056 +ILSVRC2012_val_00025275.JPEG:n03595614 +ILSVRC2012_val_00025276.JPEG:n03026506 +ILSVRC2012_val_00025277.JPEG:n02128925 +ILSVRC2012_val_00025278.JPEG:n03717622 +ILSVRC2012_val_00025279.JPEG:n03041632 +ILSVRC2012_val_00025280.JPEG:n04417672 +ILSVRC2012_val_00025281.JPEG:n07753275 +ILSVRC2012_val_00025282.JPEG:n07718747 +ILSVRC2012_val_00025283.JPEG:n01728920 +ILSVRC2012_val_00025284.JPEG:n03447447 +ILSVRC2012_val_00025285.JPEG:n02114548 +ILSVRC2012_val_00025286.JPEG:n02769748 +ILSVRC2012_val_00025287.JPEG:n01784675 +ILSVRC2012_val_00025288.JPEG:n02100877 +ILSVRC2012_val_00025289.JPEG:n02097658 +ILSVRC2012_val_00025290.JPEG:n04523525 +ILSVRC2012_val_00025291.JPEG:n02002556 +ILSVRC2012_val_00025292.JPEG:n03404251 +ILSVRC2012_val_00025293.JPEG:n03786901 +ILSVRC2012_val_00025294.JPEG:n04162706 +ILSVRC2012_val_00025295.JPEG:n02776631 +ILSVRC2012_val_00025296.JPEG:n13133613 +ILSVRC2012_val_00025297.JPEG:n04254777 +ILSVRC2012_val_00025298.JPEG:n04355338 +ILSVRC2012_val_00025299.JPEG:n02104029 +ILSVRC2012_val_00025300.JPEG:n04201297 +ILSVRC2012_val_00025301.JPEG:n03775071 +ILSVRC2012_val_00025302.JPEG:n02093754 +ILSVRC2012_val_00025303.JPEG:n03992509 +ILSVRC2012_val_00025304.JPEG:n03134739 +ILSVRC2012_val_00025305.JPEG:n12057211 +ILSVRC2012_val_00025306.JPEG:n04116512 +ILSVRC2012_val_00025307.JPEG:n02281787 +ILSVRC2012_val_00025308.JPEG:n07920052 +ILSVRC2012_val_00025309.JPEG:n02105641 +ILSVRC2012_val_00025310.JPEG:n01943899 +ILSVRC2012_val_00025311.JPEG:n03841143 +ILSVRC2012_val_00025312.JPEG:n02487347 +ILSVRC2012_val_00025313.JPEG:n04486054 +ILSVRC2012_val_00025314.JPEG:n02281787 +ILSVRC2012_val_00025315.JPEG:n02342885 +ILSVRC2012_val_00025316.JPEG:n03775546 +ILSVRC2012_val_00025317.JPEG:n02011460 +ILSVRC2012_val_00025318.JPEG:n02089078 +ILSVRC2012_val_00025319.JPEG:n03776460 +ILSVRC2012_val_00025320.JPEG:n04423845 +ILSVRC2012_val_00025321.JPEG:n02865351 +ILSVRC2012_val_00025322.JPEG:n03089624 +ILSVRC2012_val_00025323.JPEG:n04371774 +ILSVRC2012_val_00025324.JPEG:n01514859 +ILSVRC2012_val_00025325.JPEG:n01734418 +ILSVRC2012_val_00025326.JPEG:n02328150 +ILSVRC2012_val_00025327.JPEG:n09468604 +ILSVRC2012_val_00025328.JPEG:n03063689 +ILSVRC2012_val_00025329.JPEG:n02951585 +ILSVRC2012_val_00025330.JPEG:n02095314 +ILSVRC2012_val_00025331.JPEG:n03792972 +ILSVRC2012_val_00025332.JPEG:n03776460 +ILSVRC2012_val_00025333.JPEG:n02346627 +ILSVRC2012_val_00025334.JPEG:n02894605 +ILSVRC2012_val_00025335.JPEG:n01775062 +ILSVRC2012_val_00025336.JPEG:n02130308 +ILSVRC2012_val_00025337.JPEG:n04192698 +ILSVRC2012_val_00025338.JPEG:n13044778 +ILSVRC2012_val_00025339.JPEG:n01751748 +ILSVRC2012_val_00025340.JPEG:n07697537 +ILSVRC2012_val_00025341.JPEG:n03868242 +ILSVRC2012_val_00025342.JPEG:n04525038 +ILSVRC2012_val_00025343.JPEG:n02259212 +ILSVRC2012_val_00025344.JPEG:n02391049 +ILSVRC2012_val_00025345.JPEG:n04399382 +ILSVRC2012_val_00025346.JPEG:n02667093 +ILSVRC2012_val_00025347.JPEG:n01530575 +ILSVRC2012_val_00025348.JPEG:n01632777 +ILSVRC2012_val_00025349.JPEG:n03259280 +ILSVRC2012_val_00025350.JPEG:n02840245 +ILSVRC2012_val_00025351.JPEG:n04019541 +ILSVRC2012_val_00025352.JPEG:n02422699 +ILSVRC2012_val_00025353.JPEG:n02113712 +ILSVRC2012_val_00025354.JPEG:n03930630 +ILSVRC2012_val_00025355.JPEG:n02643566 +ILSVRC2012_val_00025356.JPEG:n02231487 +ILSVRC2012_val_00025357.JPEG:n04487394 +ILSVRC2012_val_00025358.JPEG:n03937543 +ILSVRC2012_val_00025359.JPEG:n03355925 +ILSVRC2012_val_00025360.JPEG:n01828970 +ILSVRC2012_val_00025361.JPEG:n01580077 +ILSVRC2012_val_00025362.JPEG:n07932039 +ILSVRC2012_val_00025363.JPEG:n02877765 +ILSVRC2012_val_00025364.JPEG:n02167151 +ILSVRC2012_val_00025365.JPEG:n03476991 +ILSVRC2012_val_00025366.JPEG:n02825657 +ILSVRC2012_val_00025367.JPEG:n01751748 +ILSVRC2012_val_00025368.JPEG:n03207941 +ILSVRC2012_val_00025369.JPEG:n03840681 +ILSVRC2012_val_00025370.JPEG:n09288635 +ILSVRC2012_val_00025371.JPEG:n01843383 +ILSVRC2012_val_00025372.JPEG:n04536866 +ILSVRC2012_val_00025373.JPEG:n03814906 +ILSVRC2012_val_00025374.JPEG:n04429376 +ILSVRC2012_val_00025375.JPEG:n04428191 +ILSVRC2012_val_00025376.JPEG:n03814906 +ILSVRC2012_val_00025377.JPEG:n04344873 +ILSVRC2012_val_00025378.JPEG:n01693334 +ILSVRC2012_val_00025379.JPEG:n03417042 +ILSVRC2012_val_00025380.JPEG:n02747177 +ILSVRC2012_val_00025381.JPEG:n01986214 +ILSVRC2012_val_00025382.JPEG:n02277742 +ILSVRC2012_val_00025383.JPEG:n03127747 +ILSVRC2012_val_00025384.JPEG:n02422699 +ILSVRC2012_val_00025385.JPEG:n12985857 +ILSVRC2012_val_00025386.JPEG:n02672831 +ILSVRC2012_val_00025387.JPEG:n02823428 +ILSVRC2012_val_00025388.JPEG:n02112018 +ILSVRC2012_val_00025389.JPEG:n04037443 +ILSVRC2012_val_00025390.JPEG:n07695742 +ILSVRC2012_val_00025391.JPEG:n02536864 +ILSVRC2012_val_00025392.JPEG:n02788148 +ILSVRC2012_val_00025393.JPEG:n02088364 +ILSVRC2012_val_00025394.JPEG:n02105251 +ILSVRC2012_val_00025395.JPEG:n02105641 +ILSVRC2012_val_00025396.JPEG:n02123159 +ILSVRC2012_val_00025397.JPEG:n03729826 +ILSVRC2012_val_00025398.JPEG:n03125729 +ILSVRC2012_val_00025399.JPEG:n04179913 +ILSVRC2012_val_00025400.JPEG:n02097474 +ILSVRC2012_val_00025401.JPEG:n03297495 +ILSVRC2012_val_00025402.JPEG:n03042490 +ILSVRC2012_val_00025403.JPEG:n04252225 +ILSVRC2012_val_00025404.JPEG:n03141823 +ILSVRC2012_val_00025405.JPEG:n09193705 +ILSVRC2012_val_00025406.JPEG:n04149813 +ILSVRC2012_val_00025407.JPEG:n02655020 +ILSVRC2012_val_00025408.JPEG:n03788365 +ILSVRC2012_val_00025409.JPEG:n03085013 +ILSVRC2012_val_00025410.JPEG:n02037110 +ILSVRC2012_val_00025411.JPEG:n01944390 +ILSVRC2012_val_00025412.JPEG:n02120505 +ILSVRC2012_val_00025413.JPEG:n04536866 +ILSVRC2012_val_00025414.JPEG:n07695742 +ILSVRC2012_val_00025415.JPEG:n02951358 +ILSVRC2012_val_00025416.JPEG:n03417042 +ILSVRC2012_val_00025417.JPEG:n03733131 +ILSVRC2012_val_00025418.JPEG:n04325704 +ILSVRC2012_val_00025419.JPEG:n03843555 +ILSVRC2012_val_00025420.JPEG:n03179701 +ILSVRC2012_val_00025421.JPEG:n02009229 +ILSVRC2012_val_00025422.JPEG:n04523525 +ILSVRC2012_val_00025423.JPEG:n02098413 +ILSVRC2012_val_00025424.JPEG:n02096585 +ILSVRC2012_val_00025425.JPEG:n03424325 +ILSVRC2012_val_00025426.JPEG:n02105162 +ILSVRC2012_val_00025427.JPEG:n04590129 +ILSVRC2012_val_00025428.JPEG:n01537544 +ILSVRC2012_val_00025429.JPEG:n02093991 +ILSVRC2012_val_00025430.JPEG:n03394916 +ILSVRC2012_val_00025431.JPEG:n01514668 +ILSVRC2012_val_00025432.JPEG:n13133613 +ILSVRC2012_val_00025433.JPEG:n03445924 +ILSVRC2012_val_00025434.JPEG:n03873416 +ILSVRC2012_val_00025435.JPEG:n01632458 +ILSVRC2012_val_00025436.JPEG:n03706229 +ILSVRC2012_val_00025437.JPEG:n02085782 +ILSVRC2012_val_00025438.JPEG:n01632777 +ILSVRC2012_val_00025439.JPEG:n04371430 +ILSVRC2012_val_00025440.JPEG:n12144580 +ILSVRC2012_val_00025441.JPEG:n01665541 +ILSVRC2012_val_00025442.JPEG:n02102040 +ILSVRC2012_val_00025443.JPEG:n02701002 +ILSVRC2012_val_00025444.JPEG:n04131690 +ILSVRC2012_val_00025445.JPEG:n04347754 +ILSVRC2012_val_00025446.JPEG:n13040303 +ILSVRC2012_val_00025447.JPEG:n01775062 +ILSVRC2012_val_00025448.JPEG:n02114712 +ILSVRC2012_val_00025449.JPEG:n01833805 +ILSVRC2012_val_00025450.JPEG:n03759954 +ILSVRC2012_val_00025451.JPEG:n02860847 +ILSVRC2012_val_00025452.JPEG:n04330267 +ILSVRC2012_val_00025453.JPEG:n02859443 +ILSVRC2012_val_00025454.JPEG:n02138441 +ILSVRC2012_val_00025455.JPEG:n01774384 +ILSVRC2012_val_00025456.JPEG:n07717556 +ILSVRC2012_val_00025457.JPEG:n04311004 +ILSVRC2012_val_00025458.JPEG:n03908714 +ILSVRC2012_val_00025459.JPEG:n02361337 +ILSVRC2012_val_00025460.JPEG:n04065272 +ILSVRC2012_val_00025461.JPEG:n04146614 +ILSVRC2012_val_00025462.JPEG:n04179913 +ILSVRC2012_val_00025463.JPEG:n01697457 +ILSVRC2012_val_00025464.JPEG:n03857828 +ILSVRC2012_val_00025465.JPEG:n04285008 +ILSVRC2012_val_00025466.JPEG:n02089078 +ILSVRC2012_val_00025467.JPEG:n01755581 +ILSVRC2012_val_00025468.JPEG:n02056570 +ILSVRC2012_val_00025469.JPEG:n02701002 +ILSVRC2012_val_00025470.JPEG:n02483708 +ILSVRC2012_val_00025471.JPEG:n02101556 +ILSVRC2012_val_00025472.JPEG:n01737021 +ILSVRC2012_val_00025473.JPEG:n03874599 +ILSVRC2012_val_00025474.JPEG:n02107683 +ILSVRC2012_val_00025475.JPEG:n03657121 +ILSVRC2012_val_00025476.JPEG:n01592084 +ILSVRC2012_val_00025477.JPEG:n03995372 +ILSVRC2012_val_00025478.JPEG:n03788195 +ILSVRC2012_val_00025479.JPEG:n02100877 +ILSVRC2012_val_00025480.JPEG:n03447447 +ILSVRC2012_val_00025481.JPEG:n09399592 +ILSVRC2012_val_00025482.JPEG:n04350905 +ILSVRC2012_val_00025483.JPEG:n04266014 +ILSVRC2012_val_00025484.JPEG:n02979186 +ILSVRC2012_val_00025485.JPEG:n02988304 +ILSVRC2012_val_00025486.JPEG:n02879718 +ILSVRC2012_val_00025487.JPEG:n03032252 +ILSVRC2012_val_00025488.JPEG:n01530575 +ILSVRC2012_val_00025489.JPEG:n03291819 +ILSVRC2012_val_00025490.JPEG:n04131690 +ILSVRC2012_val_00025491.JPEG:n02037110 +ILSVRC2012_val_00025492.JPEG:n01632458 +ILSVRC2012_val_00025493.JPEG:n02102177 +ILSVRC2012_val_00025494.JPEG:n04367480 +ILSVRC2012_val_00025495.JPEG:n01807496 +ILSVRC2012_val_00025496.JPEG:n02107908 +ILSVRC2012_val_00025497.JPEG:n01740131 +ILSVRC2012_val_00025498.JPEG:n02096585 +ILSVRC2012_val_00025499.JPEG:n04235860 +ILSVRC2012_val_00025500.JPEG:n02363005 +ILSVRC2012_val_00025501.JPEG:n02110958 +ILSVRC2012_val_00025502.JPEG:n07711569 +ILSVRC2012_val_00025503.JPEG:n03384352 +ILSVRC2012_val_00025504.JPEG:n03530642 +ILSVRC2012_val_00025505.JPEG:n03761084 +ILSVRC2012_val_00025506.JPEG:n03602883 +ILSVRC2012_val_00025507.JPEG:n01531178 +ILSVRC2012_val_00025508.JPEG:n01774384 +ILSVRC2012_val_00025509.JPEG:n04456115 +ILSVRC2012_val_00025510.JPEG:n01985128 +ILSVRC2012_val_00025511.JPEG:n01694178 +ILSVRC2012_val_00025512.JPEG:n03065424 +ILSVRC2012_val_00025513.JPEG:n04589890 +ILSVRC2012_val_00025514.JPEG:n04049303 +ILSVRC2012_val_00025515.JPEG:n07248320 +ILSVRC2012_val_00025516.JPEG:n06874185 +ILSVRC2012_val_00025517.JPEG:n04604644 +ILSVRC2012_val_00025518.JPEG:n01775062 +ILSVRC2012_val_00025519.JPEG:n02123597 +ILSVRC2012_val_00025520.JPEG:n02095570 +ILSVRC2012_val_00025521.JPEG:n01985128 +ILSVRC2012_val_00025522.JPEG:n02115913 +ILSVRC2012_val_00025523.JPEG:n01622779 +ILSVRC2012_val_00025524.JPEG:n01601694 +ILSVRC2012_val_00025525.JPEG:n04589890 +ILSVRC2012_val_00025526.JPEG:n01560419 +ILSVRC2012_val_00025527.JPEG:n01440764 +ILSVRC2012_val_00025528.JPEG:n02051845 +ILSVRC2012_val_00025529.JPEG:n03218198 +ILSVRC2012_val_00025530.JPEG:n03047690 +ILSVRC2012_val_00025531.JPEG:n03854065 +ILSVRC2012_val_00025532.JPEG:n02442845 +ILSVRC2012_val_00025533.JPEG:n02361337 +ILSVRC2012_val_00025534.JPEG:n02835271 +ILSVRC2012_val_00025535.JPEG:n01531178 +ILSVRC2012_val_00025536.JPEG:n02108422 +ILSVRC2012_val_00025537.JPEG:n02115913 +ILSVRC2012_val_00025538.JPEG:n03141823 +ILSVRC2012_val_00025539.JPEG:n02088238 +ILSVRC2012_val_00025540.JPEG:n03690938 +ILSVRC2012_val_00025541.JPEG:n03207941 +ILSVRC2012_val_00025542.JPEG:n02510455 +ILSVRC2012_val_00025543.JPEG:n01806143 +ILSVRC2012_val_00025544.JPEG:n01740131 +ILSVRC2012_val_00025545.JPEG:n03854065 +ILSVRC2012_val_00025546.JPEG:n02488291 +ILSVRC2012_val_00025547.JPEG:n04428191 +ILSVRC2012_val_00025548.JPEG:n03063599 +ILSVRC2012_val_00025549.JPEG:n02101556 +ILSVRC2012_val_00025550.JPEG:n02087046 +ILSVRC2012_val_00025551.JPEG:n02101556 +ILSVRC2012_val_00025552.JPEG:n03792972 +ILSVRC2012_val_00025553.JPEG:n04296562 +ILSVRC2012_val_00025554.JPEG:n02101006 +ILSVRC2012_val_00025555.JPEG:n02776631 +ILSVRC2012_val_00025556.JPEG:n01773797 +ILSVRC2012_val_00025557.JPEG:n03709823 +ILSVRC2012_val_00025558.JPEG:n04458633 +ILSVRC2012_val_00025559.JPEG:n02281406 +ILSVRC2012_val_00025560.JPEG:n03691459 +ILSVRC2012_val_00025561.JPEG:n03692522 +ILSVRC2012_val_00025562.JPEG:n02089867 +ILSVRC2012_val_00025563.JPEG:n03868863 +ILSVRC2012_val_00025564.JPEG:n02012849 +ILSVRC2012_val_00025565.JPEG:n03763968 +ILSVRC2012_val_00025566.JPEG:n01944390 +ILSVRC2012_val_00025567.JPEG:n01667114 +ILSVRC2012_val_00025568.JPEG:n03950228 +ILSVRC2012_val_00025569.JPEG:n02128385 +ILSVRC2012_val_00025570.JPEG:n02319095 +ILSVRC2012_val_00025571.JPEG:n04553703 +ILSVRC2012_val_00025572.JPEG:n03452741 +ILSVRC2012_val_00025573.JPEG:n03345487 +ILSVRC2012_val_00025574.JPEG:n02672831 +ILSVRC2012_val_00025575.JPEG:n03935335 +ILSVRC2012_val_00025576.JPEG:n02104365 +ILSVRC2012_val_00025577.JPEG:n01592084 +ILSVRC2012_val_00025578.JPEG:n04149813 +ILSVRC2012_val_00025579.JPEG:n03594734 +ILSVRC2012_val_00025580.JPEG:n02233338 +ILSVRC2012_val_00025581.JPEG:n01688243 +ILSVRC2012_val_00025582.JPEG:n07718472 +ILSVRC2012_val_00025583.JPEG:n03394916 +ILSVRC2012_val_00025584.JPEG:n13040303 +ILSVRC2012_val_00025585.JPEG:n01986214 +ILSVRC2012_val_00025586.JPEG:n02510455 +ILSVRC2012_val_00025587.JPEG:n04285008 +ILSVRC2012_val_00025588.JPEG:n03956157 +ILSVRC2012_val_00025589.JPEG:n02264363 +ILSVRC2012_val_00025590.JPEG:n03127747 +ILSVRC2012_val_00025591.JPEG:n03445777 +ILSVRC2012_val_00025592.JPEG:n04467665 +ILSVRC2012_val_00025593.JPEG:n03240683 +ILSVRC2012_val_00025594.JPEG:n03065424 +ILSVRC2012_val_00025595.JPEG:n04517823 +ILSVRC2012_val_00025596.JPEG:n02165105 +ILSVRC2012_val_00025597.JPEG:n03602883 +ILSVRC2012_val_00025598.JPEG:n01753488 +ILSVRC2012_val_00025599.JPEG:n04399382 +ILSVRC2012_val_00025600.JPEG:n09256479 +ILSVRC2012_val_00025601.JPEG:n02086910 +ILSVRC2012_val_00025602.JPEG:n03956157 +ILSVRC2012_val_00025603.JPEG:n03485794 +ILSVRC2012_val_00025604.JPEG:n02484975 +ILSVRC2012_val_00025605.JPEG:n02666196 +ILSVRC2012_val_00025606.JPEG:n02097209 +ILSVRC2012_val_00025607.JPEG:n03535780 +ILSVRC2012_val_00025608.JPEG:n02112018 +ILSVRC2012_val_00025609.JPEG:n03109150 +ILSVRC2012_val_00025610.JPEG:n04590129 +ILSVRC2012_val_00025611.JPEG:n01667778 +ILSVRC2012_val_00025612.JPEG:n02787622 +ILSVRC2012_val_00025613.JPEG:n02088364 +ILSVRC2012_val_00025614.JPEG:n03388549 +ILSVRC2012_val_00025615.JPEG:n02494079 +ILSVRC2012_val_00025616.JPEG:n01843065 +ILSVRC2012_val_00025617.JPEG:n02108551 +ILSVRC2012_val_00025618.JPEG:n03929855 +ILSVRC2012_val_00025619.JPEG:n03498962 +ILSVRC2012_val_00025620.JPEG:n02109525 +ILSVRC2012_val_00025621.JPEG:n04328186 +ILSVRC2012_val_00025622.JPEG:n09256479 +ILSVRC2012_val_00025623.JPEG:n04540053 +ILSVRC2012_val_00025624.JPEG:n03459775 +ILSVRC2012_val_00025625.JPEG:n03982430 +ILSVRC2012_val_00025626.JPEG:n02444819 +ILSVRC2012_val_00025627.JPEG:n01494475 +ILSVRC2012_val_00025628.JPEG:n02086079 +ILSVRC2012_val_00025629.JPEG:n02125311 +ILSVRC2012_val_00025630.JPEG:n03529860 +ILSVRC2012_val_00025631.JPEG:n01843383 +ILSVRC2012_val_00025632.JPEG:n03992509 +ILSVRC2012_val_00025633.JPEG:n01641577 +ILSVRC2012_val_00025634.JPEG:n04099969 +ILSVRC2012_val_00025635.JPEG:n04254777 +ILSVRC2012_val_00025636.JPEG:n01608432 +ILSVRC2012_val_00025637.JPEG:n02346627 +ILSVRC2012_val_00025638.JPEG:n02397096 +ILSVRC2012_val_00025639.JPEG:n02676566 +ILSVRC2012_val_00025640.JPEG:n01491361 +ILSVRC2012_val_00025641.JPEG:n02074367 +ILSVRC2012_val_00025642.JPEG:n04252225 +ILSVRC2012_val_00025643.JPEG:n04485082 +ILSVRC2012_val_00025644.JPEG:n02092002 +ILSVRC2012_val_00025645.JPEG:n02098286 +ILSVRC2012_val_00025646.JPEG:n02727426 +ILSVRC2012_val_00025647.JPEG:n03100240 +ILSVRC2012_val_00025648.JPEG:n13054560 +ILSVRC2012_val_00025649.JPEG:n02097298 +ILSVRC2012_val_00025650.JPEG:n02123045 +ILSVRC2012_val_00025651.JPEG:n02002724 +ILSVRC2012_val_00025652.JPEG:n02109047 +ILSVRC2012_val_00025653.JPEG:n03131574 +ILSVRC2012_val_00025654.JPEG:n02692877 +ILSVRC2012_val_00025655.JPEG:n02088632 +ILSVRC2012_val_00025656.JPEG:n04465501 +ILSVRC2012_val_00025657.JPEG:n02930766 +ILSVRC2012_val_00025658.JPEG:n01843065 +ILSVRC2012_val_00025659.JPEG:n03697007 +ILSVRC2012_val_00025660.JPEG:n02102973 +ILSVRC2012_val_00025661.JPEG:n04147183 +ILSVRC2012_val_00025662.JPEG:n02117135 +ILSVRC2012_val_00025663.JPEG:n07754684 +ILSVRC2012_val_00025664.JPEG:n02787622 +ILSVRC2012_val_00025665.JPEG:n02114548 +ILSVRC2012_val_00025666.JPEG:n04515003 +ILSVRC2012_val_00025667.JPEG:n01855672 +ILSVRC2012_val_00025668.JPEG:n01682714 +ILSVRC2012_val_00025669.JPEG:n02110063 +ILSVRC2012_val_00025670.JPEG:n04127249 +ILSVRC2012_val_00025671.JPEG:n03127925 +ILSVRC2012_val_00025672.JPEG:n04429376 +ILSVRC2012_val_00025673.JPEG:n03710193 +ILSVRC2012_val_00025674.JPEG:n03796401 +ILSVRC2012_val_00025675.JPEG:n02786058 +ILSVRC2012_val_00025676.JPEG:n02794156 +ILSVRC2012_val_00025677.JPEG:n02112018 +ILSVRC2012_val_00025678.JPEG:n02423022 +ILSVRC2012_val_00025679.JPEG:n02094114 +ILSVRC2012_val_00025680.JPEG:n02092339 +ILSVRC2012_val_00025681.JPEG:n03344393 +ILSVRC2012_val_00025682.JPEG:n03888605 +ILSVRC2012_val_00025683.JPEG:n02437312 +ILSVRC2012_val_00025684.JPEG:n02107574 +ILSVRC2012_val_00025685.JPEG:n03710637 +ILSVRC2012_val_00025686.JPEG:n01491361 +ILSVRC2012_val_00025687.JPEG:n04074963 +ILSVRC2012_val_00025688.JPEG:n02128385 +ILSVRC2012_val_00025689.JPEG:n04044716 +ILSVRC2012_val_00025690.JPEG:n02093991 +ILSVRC2012_val_00025691.JPEG:n02113186 +ILSVRC2012_val_00025692.JPEG:n01592084 +ILSVRC2012_val_00025693.JPEG:n07714990 +ILSVRC2012_val_00025694.JPEG:n02174001 +ILSVRC2012_val_00025695.JPEG:n02777292 +ILSVRC2012_val_00025696.JPEG:n02090379 +ILSVRC2012_val_00025697.JPEG:n04509417 +ILSVRC2012_val_00025698.JPEG:n02486261 +ILSVRC2012_val_00025699.JPEG:n02841315 +ILSVRC2012_val_00025700.JPEG:n02096051 +ILSVRC2012_val_00025701.JPEG:n01768244 +ILSVRC2012_val_00025702.JPEG:n03895866 +ILSVRC2012_val_00025703.JPEG:n03891332 +ILSVRC2012_val_00025704.JPEG:n02102177 +ILSVRC2012_val_00025705.JPEG:n04525038 +ILSVRC2012_val_00025706.JPEG:n03777754 +ILSVRC2012_val_00025707.JPEG:n07716906 +ILSVRC2012_val_00025708.JPEG:n02091244 +ILSVRC2012_val_00025709.JPEG:n02966687 +ILSVRC2012_val_00025710.JPEG:n01981276 +ILSVRC2012_val_00025711.JPEG:n02092339 +ILSVRC2012_val_00025712.JPEG:n04612504 +ILSVRC2012_val_00025713.JPEG:n09229709 +ILSVRC2012_val_00025714.JPEG:n02099429 +ILSVRC2012_val_00025715.JPEG:n04540053 +ILSVRC2012_val_00025716.JPEG:n03935335 +ILSVRC2012_val_00025717.JPEG:n01644373 +ILSVRC2012_val_00025718.JPEG:n02088466 +ILSVRC2012_val_00025719.JPEG:n04380533 +ILSVRC2012_val_00025720.JPEG:n02105162 +ILSVRC2012_val_00025721.JPEG:n02916936 +ILSVRC2012_val_00025722.JPEG:n01944390 +ILSVRC2012_val_00025723.JPEG:n02123159 +ILSVRC2012_val_00025724.JPEG:n03459775 +ILSVRC2012_val_00025725.JPEG:n01944390 +ILSVRC2012_val_00025726.JPEG:n02100735 +ILSVRC2012_val_00025727.JPEG:n01740131 +ILSVRC2012_val_00025728.JPEG:n03599486 +ILSVRC2012_val_00025729.JPEG:n02169497 +ILSVRC2012_val_00025730.JPEG:n03888605 +ILSVRC2012_val_00025731.JPEG:n04296562 +ILSVRC2012_val_00025732.JPEG:n03794056 +ILSVRC2012_val_00025733.JPEG:n03110669 +ILSVRC2012_val_00025734.JPEG:n02356798 +ILSVRC2012_val_00025735.JPEG:n03032252 +ILSVRC2012_val_00025736.JPEG:n04482393 +ILSVRC2012_val_00025737.JPEG:n03888605 +ILSVRC2012_val_00025738.JPEG:n01748264 +ILSVRC2012_val_00025739.JPEG:n02098413 +ILSVRC2012_val_00025740.JPEG:n03967562 +ILSVRC2012_val_00025741.JPEG:n03706229 +ILSVRC2012_val_00025742.JPEG:n13052670 +ILSVRC2012_val_00025743.JPEG:n04252225 +ILSVRC2012_val_00025744.JPEG:n02009229 +ILSVRC2012_val_00025745.JPEG:n04252225 +ILSVRC2012_val_00025746.JPEG:n09421951 +ILSVRC2012_val_00025747.JPEG:n01930112 +ILSVRC2012_val_00025748.JPEG:n04461696 +ILSVRC2012_val_00025749.JPEG:n04208210 +ILSVRC2012_val_00025750.JPEG:n02443484 +ILSVRC2012_val_00025751.JPEG:n03045698 +ILSVRC2012_val_00025752.JPEG:n03967562 +ILSVRC2012_val_00025753.JPEG:n07880968 +ILSVRC2012_val_00025754.JPEG:n02177972 +ILSVRC2012_val_00025755.JPEG:n01698640 +ILSVRC2012_val_00025756.JPEG:n02704792 +ILSVRC2012_val_00025757.JPEG:n04328186 +ILSVRC2012_val_00025758.JPEG:n01828970 +ILSVRC2012_val_00025759.JPEG:n04482393 +ILSVRC2012_val_00025760.JPEG:n03400231 +ILSVRC2012_val_00025761.JPEG:n03394916 +ILSVRC2012_val_00025762.JPEG:n04467665 +ILSVRC2012_val_00025763.JPEG:n04259630 +ILSVRC2012_val_00025764.JPEG:n01860187 +ILSVRC2012_val_00025765.JPEG:n03868863 +ILSVRC2012_val_00025766.JPEG:n03000134 +ILSVRC2012_val_00025767.JPEG:n02783161 +ILSVRC2012_val_00025768.JPEG:n02509815 +ILSVRC2012_val_00025769.JPEG:n04465501 +ILSVRC2012_val_00025770.JPEG:n02417914 +ILSVRC2012_val_00025771.JPEG:n04482393 +ILSVRC2012_val_00025772.JPEG:n02787622 +ILSVRC2012_val_00025773.JPEG:n02089867 +ILSVRC2012_val_00025774.JPEG:n03240683 +ILSVRC2012_val_00025775.JPEG:n02403003 +ILSVRC2012_val_00025776.JPEG:n04296562 +ILSVRC2012_val_00025777.JPEG:n02782093 +ILSVRC2012_val_00025778.JPEG:n02892201 +ILSVRC2012_val_00025779.JPEG:n03777754 +ILSVRC2012_val_00025780.JPEG:n04612504 +ILSVRC2012_val_00025781.JPEG:n03372029 +ILSVRC2012_val_00025782.JPEG:n01756291 +ILSVRC2012_val_00025783.JPEG:n03902125 +ILSVRC2012_val_00025784.JPEG:n03355925 +ILSVRC2012_val_00025785.JPEG:n01843383 +ILSVRC2012_val_00025786.JPEG:n04579432 +ILSVRC2012_val_00025787.JPEG:n02091134 +ILSVRC2012_val_00025788.JPEG:n04579432 +ILSVRC2012_val_00025789.JPEG:n03481172 +ILSVRC2012_val_00025790.JPEG:n02841315 +ILSVRC2012_val_00025791.JPEG:n07831146 +ILSVRC2012_val_00025792.JPEG:n03075370 +ILSVRC2012_val_00025793.JPEG:n02009912 +ILSVRC2012_val_00025794.JPEG:n04201297 +ILSVRC2012_val_00025795.JPEG:n02396427 +ILSVRC2012_val_00025796.JPEG:n01753488 +ILSVRC2012_val_00025797.JPEG:n03249569 +ILSVRC2012_val_00025798.JPEG:n04090263 +ILSVRC2012_val_00025799.JPEG:n01704323 +ILSVRC2012_val_00025800.JPEG:n02526121 +ILSVRC2012_val_00025801.JPEG:n04204347 +ILSVRC2012_val_00025802.JPEG:n02777292 +ILSVRC2012_val_00025803.JPEG:n03126707 +ILSVRC2012_val_00025804.JPEG:n04254120 +ILSVRC2012_val_00025805.JPEG:n02111277 +ILSVRC2012_val_00025806.JPEG:n01582220 +ILSVRC2012_val_00025807.JPEG:n02206856 +ILSVRC2012_val_00025808.JPEG:n02939185 +ILSVRC2012_val_00025809.JPEG:n01693334 +ILSVRC2012_val_00025810.JPEG:n02641379 +ILSVRC2012_val_00025811.JPEG:n04263257 +ILSVRC2012_val_00025812.JPEG:n04347754 +ILSVRC2012_val_00025813.JPEG:n07734744 +ILSVRC2012_val_00025814.JPEG:n01990800 +ILSVRC2012_val_00025815.JPEG:n04399382 +ILSVRC2012_val_00025816.JPEG:n04270147 +ILSVRC2012_val_00025817.JPEG:n03944341 +ILSVRC2012_val_00025818.JPEG:n01773549 +ILSVRC2012_val_00025819.JPEG:n03259280 +ILSVRC2012_val_00025820.JPEG:n02089078 +ILSVRC2012_val_00025821.JPEG:n02094433 +ILSVRC2012_val_00025822.JPEG:n04525305 +ILSVRC2012_val_00025823.JPEG:n04493381 +ILSVRC2012_val_00025824.JPEG:n01669191 +ILSVRC2012_val_00025825.JPEG:n02066245 +ILSVRC2012_val_00025826.JPEG:n02841315 +ILSVRC2012_val_00025827.JPEG:n03796401 +ILSVRC2012_val_00025828.JPEG:n04371430 +ILSVRC2012_val_00025829.JPEG:n04548362 +ILSVRC2012_val_00025830.JPEG:n03944341 +ILSVRC2012_val_00025831.JPEG:n01773157 +ILSVRC2012_val_00025832.JPEG:n03223299 +ILSVRC2012_val_00025833.JPEG:n03692522 +ILSVRC2012_val_00025834.JPEG:n03594945 +ILSVRC2012_val_00025835.JPEG:n02100877 +ILSVRC2012_val_00025836.JPEG:n03000134 +ILSVRC2012_val_00025837.JPEG:n02783161 +ILSVRC2012_val_00025838.JPEG:n03345487 +ILSVRC2012_val_00025839.JPEG:n02802426 +ILSVRC2012_val_00025840.JPEG:n01944390 +ILSVRC2012_val_00025841.JPEG:n02817516 +ILSVRC2012_val_00025842.JPEG:n02102973 +ILSVRC2012_val_00025843.JPEG:n03956157 +ILSVRC2012_val_00025844.JPEG:n03627232 +ILSVRC2012_val_00025845.JPEG:n02114712 +ILSVRC2012_val_00025846.JPEG:n03837869 +ILSVRC2012_val_00025847.JPEG:n02797295 +ILSVRC2012_val_00025848.JPEG:n04458633 +ILSVRC2012_val_00025849.JPEG:n03196217 +ILSVRC2012_val_00025850.JPEG:n02963159 +ILSVRC2012_val_00025851.JPEG:n02110341 +ILSVRC2012_val_00025852.JPEG:n02108551 +ILSVRC2012_val_00025853.JPEG:n09468604 +ILSVRC2012_val_00025854.JPEG:n03452741 +ILSVRC2012_val_00025855.JPEG:n02174001 +ILSVRC2012_val_00025856.JPEG:n04380533 +ILSVRC2012_val_00025857.JPEG:n07716358 +ILSVRC2012_val_00025858.JPEG:n04037443 +ILSVRC2012_val_00025859.JPEG:n03803284 +ILSVRC2012_val_00025860.JPEG:n03958227 +ILSVRC2012_val_00025861.JPEG:n09288635 +ILSVRC2012_val_00025862.JPEG:n04442312 +ILSVRC2012_val_00025863.JPEG:n03272562 +ILSVRC2012_val_00025864.JPEG:n03891251 +ILSVRC2012_val_00025865.JPEG:n04118776 +ILSVRC2012_val_00025866.JPEG:n04532670 +ILSVRC2012_val_00025867.JPEG:n01742172 +ILSVRC2012_val_00025868.JPEG:n03733281 +ILSVRC2012_val_00025869.JPEG:n02102177 +ILSVRC2012_val_00025870.JPEG:n03026506 +ILSVRC2012_val_00025871.JPEG:n02606052 +ILSVRC2012_val_00025872.JPEG:n01818515 +ILSVRC2012_val_00025873.JPEG:n04589890 +ILSVRC2012_val_00025874.JPEG:n04428191 +ILSVRC2012_val_00025875.JPEG:n02279972 +ILSVRC2012_val_00025876.JPEG:n02123045 +ILSVRC2012_val_00025877.JPEG:n04254120 +ILSVRC2012_val_00025878.JPEG:n03000684 +ILSVRC2012_val_00025879.JPEG:n01983481 +ILSVRC2012_val_00025880.JPEG:n02704792 +ILSVRC2012_val_00025881.JPEG:n07590611 +ILSVRC2012_val_00025882.JPEG:n04162706 +ILSVRC2012_val_00025883.JPEG:n02088632 +ILSVRC2012_val_00025884.JPEG:n02112706 +ILSVRC2012_val_00025885.JPEG:n03938244 +ILSVRC2012_val_00025886.JPEG:n02112018 +ILSVRC2012_val_00025887.JPEG:n02123597 +ILSVRC2012_val_00025888.JPEG:n01531178 +ILSVRC2012_val_00025889.JPEG:n02325366 +ILSVRC2012_val_00025890.JPEG:n03000684 +ILSVRC2012_val_00025891.JPEG:n02066245 +ILSVRC2012_val_00025892.JPEG:n02859443 +ILSVRC2012_val_00025893.JPEG:n03063599 +ILSVRC2012_val_00025894.JPEG:n07753113 +ILSVRC2012_val_00025895.JPEG:n02999410 +ILSVRC2012_val_00025896.JPEG:n03777568 +ILSVRC2012_val_00025897.JPEG:n02108089 +ILSVRC2012_val_00025898.JPEG:n01872401 +ILSVRC2012_val_00025899.JPEG:n02025239 +ILSVRC2012_val_00025900.JPEG:n01484850 +ILSVRC2012_val_00025901.JPEG:n03899768 +ILSVRC2012_val_00025902.JPEG:n04162706 +ILSVRC2012_val_00025903.JPEG:n02110341 +ILSVRC2012_val_00025904.JPEG:n02091467 +ILSVRC2012_val_00025905.JPEG:n04417672 +ILSVRC2012_val_00025906.JPEG:n03000134 +ILSVRC2012_val_00025907.JPEG:n04356056 +ILSVRC2012_val_00025908.JPEG:n04417672 +ILSVRC2012_val_00025909.JPEG:n01689811 +ILSVRC2012_val_00025910.JPEG:n02412080 +ILSVRC2012_val_00025911.JPEG:n02086646 +ILSVRC2012_val_00025912.JPEG:n02096294 +ILSVRC2012_val_00025913.JPEG:n01622779 +ILSVRC2012_val_00025914.JPEG:n02089973 +ILSVRC2012_val_00025915.JPEG:n02835271 +ILSVRC2012_val_00025916.JPEG:n09193705 +ILSVRC2012_val_00025917.JPEG:n04111531 +ILSVRC2012_val_00025918.JPEG:n04456115 +ILSVRC2012_val_00025919.JPEG:n09193705 +ILSVRC2012_val_00025920.JPEG:n03633091 +ILSVRC2012_val_00025921.JPEG:n07749582 +ILSVRC2012_val_00025922.JPEG:n07697537 +ILSVRC2012_val_00025923.JPEG:n02860847 +ILSVRC2012_val_00025924.JPEG:n01855672 +ILSVRC2012_val_00025925.JPEG:n03743016 +ILSVRC2012_val_00025926.JPEG:n02077923 +ILSVRC2012_val_00025927.JPEG:n07754684 +ILSVRC2012_val_00025928.JPEG:n01833805 +ILSVRC2012_val_00025929.JPEG:n02013706 +ILSVRC2012_val_00025930.JPEG:n03976657 +ILSVRC2012_val_00025931.JPEG:n03134739 +ILSVRC2012_val_00025932.JPEG:n03720891 +ILSVRC2012_val_00025933.JPEG:n02837789 +ILSVRC2012_val_00025934.JPEG:n04355933 +ILSVRC2012_val_00025935.JPEG:n03584829 +ILSVRC2012_val_00025936.JPEG:n09472597 +ILSVRC2012_val_00025937.JPEG:n01843065 +ILSVRC2012_val_00025938.JPEG:n01749939 +ILSVRC2012_val_00025939.JPEG:n03717622 +ILSVRC2012_val_00025940.JPEG:n03982430 +ILSVRC2012_val_00025941.JPEG:n02504458 +ILSVRC2012_val_00025942.JPEG:n02127052 +ILSVRC2012_val_00025943.JPEG:n03127747 +ILSVRC2012_val_00025944.JPEG:n04026417 +ILSVRC2012_val_00025945.JPEG:n03866082 +ILSVRC2012_val_00025946.JPEG:n01872401 +ILSVRC2012_val_00025947.JPEG:n02094258 +ILSVRC2012_val_00025948.JPEG:n03291819 +ILSVRC2012_val_00025949.JPEG:n02110627 +ILSVRC2012_val_00025950.JPEG:n03982430 +ILSVRC2012_val_00025951.JPEG:n02093256 +ILSVRC2012_val_00025952.JPEG:n02277742 +ILSVRC2012_val_00025953.JPEG:n02965783 +ILSVRC2012_val_00025954.JPEG:n04428191 +ILSVRC2012_val_00025955.JPEG:n01740131 +ILSVRC2012_val_00025956.JPEG:n02795169 +ILSVRC2012_val_00025957.JPEG:n02119789 +ILSVRC2012_val_00025958.JPEG:n03535780 +ILSVRC2012_val_00025959.JPEG:n03461385 +ILSVRC2012_val_00025960.JPEG:n01980166 +ILSVRC2012_val_00025961.JPEG:n02486410 +ILSVRC2012_val_00025962.JPEG:n03720891 +ILSVRC2012_val_00025963.JPEG:n04597913 +ILSVRC2012_val_00025964.JPEG:n03666591 +ILSVRC2012_val_00025965.JPEG:n02843684 +ILSVRC2012_val_00025966.JPEG:n04252225 +ILSVRC2012_val_00025967.JPEG:n10565667 +ILSVRC2012_val_00025968.JPEG:n02268443 +ILSVRC2012_val_00025969.JPEG:n01491361 +ILSVRC2012_val_00025970.JPEG:n02098105 +ILSVRC2012_val_00025971.JPEG:n03775071 +ILSVRC2012_val_00025972.JPEG:n03187595 +ILSVRC2012_val_00025973.JPEG:n07760859 +ILSVRC2012_val_00025974.JPEG:n02259212 +ILSVRC2012_val_00025975.JPEG:n03042490 +ILSVRC2012_val_00025976.JPEG:n03942813 +ILSVRC2012_val_00025977.JPEG:n04069434 +ILSVRC2012_val_00025978.JPEG:n04120489 +ILSVRC2012_val_00025979.JPEG:n01820546 +ILSVRC2012_val_00025980.JPEG:n04548280 +ILSVRC2012_val_00025981.JPEG:n07718472 +ILSVRC2012_val_00025982.JPEG:n02417914 +ILSVRC2012_val_00025983.JPEG:n02095314 +ILSVRC2012_val_00025984.JPEG:n06874185 +ILSVRC2012_val_00025985.JPEG:n03447447 +ILSVRC2012_val_00025986.JPEG:n03983396 +ILSVRC2012_val_00025987.JPEG:n04592741 +ILSVRC2012_val_00025988.JPEG:n02102177 +ILSVRC2012_val_00025989.JPEG:n03649909 +ILSVRC2012_val_00025990.JPEG:n03594945 +ILSVRC2012_val_00025991.JPEG:n02099712 +ILSVRC2012_val_00025992.JPEG:n04370456 +ILSVRC2012_val_00025993.JPEG:n04517823 +ILSVRC2012_val_00025994.JPEG:n07875152 +ILSVRC2012_val_00025995.JPEG:n03207941 +ILSVRC2012_val_00025996.JPEG:n02398521 +ILSVRC2012_val_00025997.JPEG:n03954731 +ILSVRC2012_val_00025998.JPEG:n01796340 +ILSVRC2012_val_00025999.JPEG:n01798484 +ILSVRC2012_val_00026000.JPEG:n02113712 +ILSVRC2012_val_00026001.JPEG:n01491361 +ILSVRC2012_val_00026002.JPEG:n04423845 +ILSVRC2012_val_00026003.JPEG:n03483316 +ILSVRC2012_val_00026004.JPEG:n04461696 +ILSVRC2012_val_00026005.JPEG:n02106550 +ILSVRC2012_val_00026006.JPEG:n01773157 +ILSVRC2012_val_00026007.JPEG:n13052670 +ILSVRC2012_val_00026008.JPEG:n02091244 +ILSVRC2012_val_00026009.JPEG:n03706229 +ILSVRC2012_val_00026010.JPEG:n01560419 +ILSVRC2012_val_00026011.JPEG:n03832673 +ILSVRC2012_val_00026012.JPEG:n02492660 +ILSVRC2012_val_00026013.JPEG:n04099969 +ILSVRC2012_val_00026014.JPEG:n03982430 +ILSVRC2012_val_00026015.JPEG:n04532670 +ILSVRC2012_val_00026016.JPEG:n01631663 +ILSVRC2012_val_00026017.JPEG:n02085782 +ILSVRC2012_val_00026018.JPEG:n01728920 +ILSVRC2012_val_00026019.JPEG:n03240683 +ILSVRC2012_val_00026020.JPEG:n04584207 +ILSVRC2012_val_00026021.JPEG:n01806567 +ILSVRC2012_val_00026022.JPEG:n01729977 +ILSVRC2012_val_00026023.JPEG:n01601694 +ILSVRC2012_val_00026024.JPEG:n04350905 +ILSVRC2012_val_00026025.JPEG:n04179913 +ILSVRC2012_val_00026026.JPEG:n04592741 +ILSVRC2012_val_00026027.JPEG:n02108422 +ILSVRC2012_val_00026028.JPEG:n02110806 +ILSVRC2012_val_00026029.JPEG:n02814533 +ILSVRC2012_val_00026030.JPEG:n01773797 +ILSVRC2012_val_00026031.JPEG:n02704792 +ILSVRC2012_val_00026032.JPEG:n02782093 +ILSVRC2012_val_00026033.JPEG:n03916031 +ILSVRC2012_val_00026034.JPEG:n03467068 +ILSVRC2012_val_00026035.JPEG:n03710721 +ILSVRC2012_val_00026036.JPEG:n04554684 +ILSVRC2012_val_00026037.JPEG:n01955084 +ILSVRC2012_val_00026038.JPEG:n07717556 +ILSVRC2012_val_00026039.JPEG:n02009229 +ILSVRC2012_val_00026040.JPEG:n02256656 +ILSVRC2012_val_00026041.JPEG:n03095699 +ILSVRC2012_val_00026042.JPEG:n02094258 +ILSVRC2012_val_00026043.JPEG:n02486410 +ILSVRC2012_val_00026044.JPEG:n02027492 +ILSVRC2012_val_00026045.JPEG:n04200800 +ILSVRC2012_val_00026046.JPEG:n04371430 +ILSVRC2012_val_00026047.JPEG:n03662601 +ILSVRC2012_val_00026048.JPEG:n02444819 +ILSVRC2012_val_00026049.JPEG:n01665541 +ILSVRC2012_val_00026050.JPEG:n01614925 +ILSVRC2012_val_00026051.JPEG:n02112018 +ILSVRC2012_val_00026052.JPEG:n03773504 +ILSVRC2012_val_00026053.JPEG:n04505470 +ILSVRC2012_val_00026054.JPEG:n02951358 +ILSVRC2012_val_00026055.JPEG:n02948072 +ILSVRC2012_val_00026056.JPEG:n02101556 +ILSVRC2012_val_00026057.JPEG:n03868242 +ILSVRC2012_val_00026058.JPEG:n02093256 +ILSVRC2012_val_00026059.JPEG:n01641577 +ILSVRC2012_val_00026060.JPEG:n02128385 +ILSVRC2012_val_00026061.JPEG:n03000684 +ILSVRC2012_val_00026062.JPEG:n03874293 +ILSVRC2012_val_00026063.JPEG:n03134739 +ILSVRC2012_val_00026064.JPEG:n01440764 +ILSVRC2012_val_00026065.JPEG:n02268853 +ILSVRC2012_val_00026066.JPEG:n07584110 +ILSVRC2012_val_00026067.JPEG:n04399382 +ILSVRC2012_val_00026068.JPEG:n01843065 +ILSVRC2012_val_00026069.JPEG:n03188531 +ILSVRC2012_val_00026070.JPEG:n02086240 +ILSVRC2012_val_00026071.JPEG:n04540053 +ILSVRC2012_val_00026072.JPEG:n01829413 +ILSVRC2012_val_00026073.JPEG:n04462240 +ILSVRC2012_val_00026074.JPEG:n03018349 +ILSVRC2012_val_00026075.JPEG:n03782006 +ILSVRC2012_val_00026076.JPEG:n07730033 +ILSVRC2012_val_00026077.JPEG:n03676483 +ILSVRC2012_val_00026078.JPEG:n04275548 +ILSVRC2012_val_00026079.JPEG:n03930630 +ILSVRC2012_val_00026080.JPEG:n03764736 +ILSVRC2012_val_00026081.JPEG:n02226429 +ILSVRC2012_val_00026082.JPEG:n02007558 +ILSVRC2012_val_00026083.JPEG:n04149813 +ILSVRC2012_val_00026084.JPEG:n01820546 +ILSVRC2012_val_00026085.JPEG:n01829413 +ILSVRC2012_val_00026086.JPEG:n02110185 +ILSVRC2012_val_00026087.JPEG:n02107683 +ILSVRC2012_val_00026088.JPEG:n03840681 +ILSVRC2012_val_00026089.JPEG:n02018207 +ILSVRC2012_val_00026090.JPEG:n01833805 +ILSVRC2012_val_00026091.JPEG:n03902125 +ILSVRC2012_val_00026092.JPEG:n03868863 +ILSVRC2012_val_00026093.JPEG:n03443371 +ILSVRC2012_val_00026094.JPEG:n02113978 +ILSVRC2012_val_00026095.JPEG:n03793489 +ILSVRC2012_val_00026096.JPEG:n02859443 +ILSVRC2012_val_00026097.JPEG:n02097047 +ILSVRC2012_val_00026098.JPEG:n04192698 +ILSVRC2012_val_00026099.JPEG:n07590611 +ILSVRC2012_val_00026100.JPEG:n07880968 +ILSVRC2012_val_00026101.JPEG:n07697537 +ILSVRC2012_val_00026102.JPEG:n02342885 +ILSVRC2012_val_00026103.JPEG:n02398521 +ILSVRC2012_val_00026104.JPEG:n02002724 +ILSVRC2012_val_00026105.JPEG:n02910353 +ILSVRC2012_val_00026106.JPEG:n02442845 +ILSVRC2012_val_00026107.JPEG:n02906734 +ILSVRC2012_val_00026108.JPEG:n02494079 +ILSVRC2012_val_00026109.JPEG:n02091831 +ILSVRC2012_val_00026110.JPEG:n02823750 +ILSVRC2012_val_00026111.JPEG:n04447861 +ILSVRC2012_val_00026112.JPEG:n01796340 +ILSVRC2012_val_00026113.JPEG:n03089624 +ILSVRC2012_val_00026114.JPEG:n03924679 +ILSVRC2012_val_00026115.JPEG:n01980166 +ILSVRC2012_val_00026116.JPEG:n04435653 +ILSVRC2012_val_00026117.JPEG:n03649909 +ILSVRC2012_val_00026118.JPEG:n02107142 +ILSVRC2012_val_00026119.JPEG:n02110063 +ILSVRC2012_val_00026120.JPEG:n02403003 +ILSVRC2012_val_00026121.JPEG:n04081281 +ILSVRC2012_val_00026122.JPEG:n01735189 +ILSVRC2012_val_00026123.JPEG:n01532829 +ILSVRC2012_val_00026124.JPEG:n03891251 +ILSVRC2012_val_00026125.JPEG:n02077923 +ILSVRC2012_val_00026126.JPEG:n03977966 +ILSVRC2012_val_00026127.JPEG:n03452741 +ILSVRC2012_val_00026128.JPEG:n04465501 +ILSVRC2012_val_00026129.JPEG:n02777292 +ILSVRC2012_val_00026130.JPEG:n02113799 +ILSVRC2012_val_00026131.JPEG:n04367480 +ILSVRC2012_val_00026132.JPEG:n03787032 +ILSVRC2012_val_00026133.JPEG:n01744401 +ILSVRC2012_val_00026134.JPEG:n02667093 +ILSVRC2012_val_00026135.JPEG:n03933933 +ILSVRC2012_val_00026136.JPEG:n01580077 +ILSVRC2012_val_00026137.JPEG:n02794156 +ILSVRC2012_val_00026138.JPEG:n01796340 +ILSVRC2012_val_00026139.JPEG:n02002556 +ILSVRC2012_val_00026140.JPEG:n02837789 +ILSVRC2012_val_00026141.JPEG:n01818515 +ILSVRC2012_val_00026142.JPEG:n09835506 +ILSVRC2012_val_00026143.JPEG:n04604644 +ILSVRC2012_val_00026144.JPEG:n01917289 +ILSVRC2012_val_00026145.JPEG:n03180011 +ILSVRC2012_val_00026146.JPEG:n02102480 +ILSVRC2012_val_00026147.JPEG:n03873416 +ILSVRC2012_val_00026148.JPEG:n03995372 +ILSVRC2012_val_00026149.JPEG:n03884397 +ILSVRC2012_val_00026150.JPEG:n03657121 +ILSVRC2012_val_00026151.JPEG:n02093754 +ILSVRC2012_val_00026152.JPEG:n02102318 +ILSVRC2012_val_00026153.JPEG:n02097658 +ILSVRC2012_val_00026154.JPEG:n02108422 +ILSVRC2012_val_00026155.JPEG:n01855672 +ILSVRC2012_val_00026156.JPEG:n02489166 +ILSVRC2012_val_00026157.JPEG:n03208938 +ILSVRC2012_val_00026158.JPEG:n02116738 +ILSVRC2012_val_00026159.JPEG:n07802026 +ILSVRC2012_val_00026160.JPEG:n03584254 +ILSVRC2012_val_00026161.JPEG:n02108000 +ILSVRC2012_val_00026162.JPEG:n09256479 +ILSVRC2012_val_00026163.JPEG:n02892767 +ILSVRC2012_val_00026164.JPEG:n02105162 +ILSVRC2012_val_00026165.JPEG:n03388549 +ILSVRC2012_val_00026166.JPEG:n02870880 +ILSVRC2012_val_00026167.JPEG:n02116738 +ILSVRC2012_val_00026168.JPEG:n01807496 +ILSVRC2012_val_00026169.JPEG:n03045698 +ILSVRC2012_val_00026170.JPEG:n03717622 +ILSVRC2012_val_00026171.JPEG:n03109150 +ILSVRC2012_val_00026172.JPEG:n03388549 +ILSVRC2012_val_00026173.JPEG:n02437616 +ILSVRC2012_val_00026174.JPEG:n07930864 +ILSVRC2012_val_00026175.JPEG:n03991062 +ILSVRC2012_val_00026176.JPEG:n03709823 +ILSVRC2012_val_00026177.JPEG:n03680355 +ILSVRC2012_val_00026178.JPEG:n02033041 +ILSVRC2012_val_00026179.JPEG:n02843684 +ILSVRC2012_val_00026180.JPEG:n02795169 +ILSVRC2012_val_00026181.JPEG:n02236044 +ILSVRC2012_val_00026182.JPEG:n02509815 +ILSVRC2012_val_00026183.JPEG:n04442312 +ILSVRC2012_val_00026184.JPEG:n12998815 +ILSVRC2012_val_00026185.JPEG:n03255030 +ILSVRC2012_val_00026186.JPEG:n02111889 +ILSVRC2012_val_00026187.JPEG:n03595614 +ILSVRC2012_val_00026188.JPEG:n03788195 +ILSVRC2012_val_00026189.JPEG:n02690373 +ILSVRC2012_val_00026190.JPEG:n01756291 +ILSVRC2012_val_00026191.JPEG:n01698640 +ILSVRC2012_val_00026192.JPEG:n07565083 +ILSVRC2012_val_00026193.JPEG:n01983481 +ILSVRC2012_val_00026194.JPEG:n03445777 +ILSVRC2012_val_00026195.JPEG:n03998194 +ILSVRC2012_val_00026196.JPEG:n02879718 +ILSVRC2012_val_00026197.JPEG:n07930864 +ILSVRC2012_val_00026198.JPEG:n03255030 +ILSVRC2012_val_00026199.JPEG:n02086646 +ILSVRC2012_val_00026200.JPEG:n04120489 +ILSVRC2012_val_00026201.JPEG:n03733281 +ILSVRC2012_val_00026202.JPEG:n01667114 +ILSVRC2012_val_00026203.JPEG:n03532672 +ILSVRC2012_val_00026204.JPEG:n03179701 +ILSVRC2012_val_00026205.JPEG:n04229816 +ILSVRC2012_val_00026206.JPEG:n03733281 +ILSVRC2012_val_00026207.JPEG:n09256479 +ILSVRC2012_val_00026208.JPEG:n02105251 +ILSVRC2012_val_00026209.JPEG:n03146219 +ILSVRC2012_val_00026210.JPEG:n04330267 +ILSVRC2012_val_00026211.JPEG:n06874185 +ILSVRC2012_val_00026212.JPEG:n12620546 +ILSVRC2012_val_00026213.JPEG:n01641577 +ILSVRC2012_val_00026214.JPEG:n02106550 +ILSVRC2012_val_00026215.JPEG:n02445715 +ILSVRC2012_val_00026216.JPEG:n03146219 +ILSVRC2012_val_00026217.JPEG:n02493793 +ILSVRC2012_val_00026218.JPEG:n02509815 +ILSVRC2012_val_00026219.JPEG:n02804610 +ILSVRC2012_val_00026220.JPEG:n03590841 +ILSVRC2012_val_00026221.JPEG:n01871265 +ILSVRC2012_val_00026222.JPEG:n02483362 +ILSVRC2012_val_00026223.JPEG:n02437616 +ILSVRC2012_val_00026224.JPEG:n03895866 +ILSVRC2012_val_00026225.JPEG:n02071294 +ILSVRC2012_val_00026226.JPEG:n03291819 +ILSVRC2012_val_00026227.JPEG:n13044778 +ILSVRC2012_val_00026228.JPEG:n02114855 +ILSVRC2012_val_00026229.JPEG:n01984695 +ILSVRC2012_val_00026230.JPEG:n02500267 +ILSVRC2012_val_00026231.JPEG:n06359193 +ILSVRC2012_val_00026232.JPEG:n01843065 +ILSVRC2012_val_00026233.JPEG:n03763968 +ILSVRC2012_val_00026234.JPEG:n02643566 +ILSVRC2012_val_00026235.JPEG:n04258138 +ILSVRC2012_val_00026236.JPEG:n02667093 +ILSVRC2012_val_00026237.JPEG:n07734744 +ILSVRC2012_val_00026238.JPEG:n04153751 +ILSVRC2012_val_00026239.JPEG:n02138441 +ILSVRC2012_val_00026240.JPEG:n03188531 +ILSVRC2012_val_00026241.JPEG:n07802026 +ILSVRC2012_val_00026242.JPEG:n02100583 +ILSVRC2012_val_00026243.JPEG:n07860988 +ILSVRC2012_val_00026244.JPEG:n01817953 +ILSVRC2012_val_00026245.JPEG:n02106166 +ILSVRC2012_val_00026246.JPEG:n02483708 +ILSVRC2012_val_00026247.JPEG:n03782006 +ILSVRC2012_val_00026248.JPEG:n02007558 +ILSVRC2012_val_00026249.JPEG:n04476259 +ILSVRC2012_val_00026250.JPEG:n02835271 +ILSVRC2012_val_00026251.JPEG:n03124170 +ILSVRC2012_val_00026252.JPEG:n04550184 +ILSVRC2012_val_00026253.JPEG:n03661043 +ILSVRC2012_val_00026254.JPEG:n04204238 +ILSVRC2012_val_00026255.JPEG:n03776460 +ILSVRC2012_val_00026256.JPEG:n03837869 +ILSVRC2012_val_00026257.JPEG:n04443257 +ILSVRC2012_val_00026258.JPEG:n02486261 +ILSVRC2012_val_00026259.JPEG:n01537544 +ILSVRC2012_val_00026260.JPEG:n02317335 +ILSVRC2012_val_00026261.JPEG:n02134418 +ILSVRC2012_val_00026262.JPEG:n04557648 +ILSVRC2012_val_00026263.JPEG:n01872401 +ILSVRC2012_val_00026264.JPEG:n04209239 +ILSVRC2012_val_00026265.JPEG:n01677366 +ILSVRC2012_val_00026266.JPEG:n02100735 +ILSVRC2012_val_00026267.JPEG:n02096437 +ILSVRC2012_val_00026268.JPEG:n04479046 +ILSVRC2012_val_00026269.JPEG:n01693334 +ILSVRC2012_val_00026270.JPEG:n02965783 +ILSVRC2012_val_00026271.JPEG:n01514859 +ILSVRC2012_val_00026272.JPEG:n07613480 +ILSVRC2012_val_00026273.JPEG:n02108422 +ILSVRC2012_val_00026274.JPEG:n01914609 +ILSVRC2012_val_00026275.JPEG:n03482405 +ILSVRC2012_val_00026276.JPEG:n03710637 +ILSVRC2012_val_00026277.JPEG:n04009552 +ILSVRC2012_val_00026278.JPEG:n02106166 +ILSVRC2012_val_00026279.JPEG:n01531178 +ILSVRC2012_val_00026280.JPEG:n02704792 +ILSVRC2012_val_00026281.JPEG:n04487394 +ILSVRC2012_val_00026282.JPEG:n02834397 +ILSVRC2012_val_00026283.JPEG:n02108915 +ILSVRC2012_val_00026284.JPEG:n02484975 +ILSVRC2012_val_00026285.JPEG:n04310018 +ILSVRC2012_val_00026286.JPEG:n02095570 +ILSVRC2012_val_00026287.JPEG:n03447721 +ILSVRC2012_val_00026288.JPEG:n02119022 +ILSVRC2012_val_00026289.JPEG:n03017168 +ILSVRC2012_val_00026290.JPEG:n03697007 +ILSVRC2012_val_00026291.JPEG:n03249569 +ILSVRC2012_val_00026292.JPEG:n02835271 +ILSVRC2012_val_00026293.JPEG:n04591713 +ILSVRC2012_val_00026294.JPEG:n03347037 +ILSVRC2012_val_00026295.JPEG:n02791124 +ILSVRC2012_val_00026296.JPEG:n01692333 +ILSVRC2012_val_00026297.JPEG:n01882714 +ILSVRC2012_val_00026298.JPEG:n03196217 +ILSVRC2012_val_00026299.JPEG:n02422699 +ILSVRC2012_val_00026300.JPEG:n04041544 +ILSVRC2012_val_00026301.JPEG:n03796401 +ILSVRC2012_val_00026302.JPEG:n02028035 +ILSVRC2012_val_00026303.JPEG:n02966193 +ILSVRC2012_val_00026304.JPEG:n04235860 +ILSVRC2012_val_00026305.JPEG:n03642806 +ILSVRC2012_val_00026306.JPEG:n03838899 +ILSVRC2012_val_00026307.JPEG:n02510455 +ILSVRC2012_val_00026308.JPEG:n01930112 +ILSVRC2012_val_00026309.JPEG:n03781244 +ILSVRC2012_val_00026310.JPEG:n02091032 +ILSVRC2012_val_00026311.JPEG:n02025239 +ILSVRC2012_val_00026312.JPEG:n03196217 +ILSVRC2012_val_00026313.JPEG:n02094114 +ILSVRC2012_val_00026314.JPEG:n01978455 +ILSVRC2012_val_00026315.JPEG:n04254120 +ILSVRC2012_val_00026316.JPEG:n13040303 +ILSVRC2012_val_00026317.JPEG:n03459775 +ILSVRC2012_val_00026318.JPEG:n07716358 +ILSVRC2012_val_00026319.JPEG:n03016953 +ILSVRC2012_val_00026320.JPEG:n03876231 +ILSVRC2012_val_00026321.JPEG:n02892767 +ILSVRC2012_val_00026322.JPEG:n04069434 +ILSVRC2012_val_00026323.JPEG:n02256656 +ILSVRC2012_val_00026324.JPEG:n02168699 +ILSVRC2012_val_00026325.JPEG:n02128757 +ILSVRC2012_val_00026326.JPEG:n01986214 +ILSVRC2012_val_00026327.JPEG:n02009229 +ILSVRC2012_val_00026328.JPEG:n02790996 +ILSVRC2012_val_00026329.JPEG:n03630383 +ILSVRC2012_val_00026330.JPEG:n07718747 +ILSVRC2012_val_00026331.JPEG:n02361337 +ILSVRC2012_val_00026332.JPEG:n02951585 +ILSVRC2012_val_00026333.JPEG:n07873807 +ILSVRC2012_val_00026334.JPEG:n03223299 +ILSVRC2012_val_00026335.JPEG:n07836838 +ILSVRC2012_val_00026336.JPEG:n04266014 +ILSVRC2012_val_00026337.JPEG:n03956157 +ILSVRC2012_val_00026338.JPEG:n02002724 +ILSVRC2012_val_00026339.JPEG:n02077923 +ILSVRC2012_val_00026340.JPEG:n02002556 +ILSVRC2012_val_00026341.JPEG:n02951358 +ILSVRC2012_val_00026342.JPEG:n03259280 +ILSVRC2012_val_00026343.JPEG:n02113186 +ILSVRC2012_val_00026344.JPEG:n02843684 +ILSVRC2012_val_00026345.JPEG:n04332243 +ILSVRC2012_val_00026346.JPEG:n01775062 +ILSVRC2012_val_00026347.JPEG:n02777292 +ILSVRC2012_val_00026348.JPEG:n04118538 +ILSVRC2012_val_00026349.JPEG:n02226429 +ILSVRC2012_val_00026350.JPEG:n03908618 +ILSVRC2012_val_00026351.JPEG:n02782093 +ILSVRC2012_val_00026352.JPEG:n03777568 +ILSVRC2012_val_00026353.JPEG:n02101556 +ILSVRC2012_val_00026354.JPEG:n02701002 +ILSVRC2012_val_00026355.JPEG:n02018795 +ILSVRC2012_val_00026356.JPEG:n02102318 +ILSVRC2012_val_00026357.JPEG:n03045698 +ILSVRC2012_val_00026358.JPEG:n04254680 +ILSVRC2012_val_00026359.JPEG:n02692877 +ILSVRC2012_val_00026360.JPEG:n12620546 +ILSVRC2012_val_00026361.JPEG:n02325366 +ILSVRC2012_val_00026362.JPEG:n01560419 +ILSVRC2012_val_00026363.JPEG:n02977058 +ILSVRC2012_val_00026364.JPEG:n03127925 +ILSVRC2012_val_00026365.JPEG:n04325704 +ILSVRC2012_val_00026366.JPEG:n03483316 +ILSVRC2012_val_00026367.JPEG:n02101556 +ILSVRC2012_val_00026368.JPEG:n03450230 +ILSVRC2012_val_00026369.JPEG:n04264628 +ILSVRC2012_val_00026370.JPEG:n02101556 +ILSVRC2012_val_00026371.JPEG:n03482405 +ILSVRC2012_val_00026372.JPEG:n07715103 +ILSVRC2012_val_00026373.JPEG:n03544143 +ILSVRC2012_val_00026374.JPEG:n02395406 +ILSVRC2012_val_00026375.JPEG:n01797886 +ILSVRC2012_val_00026376.JPEG:n03207941 +ILSVRC2012_val_00026377.JPEG:n04389033 +ILSVRC2012_val_00026378.JPEG:n01978455 +ILSVRC2012_val_00026379.JPEG:n01755581 +ILSVRC2012_val_00026380.JPEG:n02708093 +ILSVRC2012_val_00026381.JPEG:n03461385 +ILSVRC2012_val_00026382.JPEG:n02342885 +ILSVRC2012_val_00026383.JPEG:n01930112 +ILSVRC2012_val_00026384.JPEG:n04009552 +ILSVRC2012_val_00026385.JPEG:n02804610 +ILSVRC2012_val_00026386.JPEG:n13037406 +ILSVRC2012_val_00026387.JPEG:n02092339 +ILSVRC2012_val_00026388.JPEG:n02106550 +ILSVRC2012_val_00026389.JPEG:n04033995 +ILSVRC2012_val_00026390.JPEG:n02395406 +ILSVRC2012_val_00026391.JPEG:n03733131 +ILSVRC2012_val_00026392.JPEG:n02859443 +ILSVRC2012_val_00026393.JPEG:n04008634 +ILSVRC2012_val_00026394.JPEG:n02841315 +ILSVRC2012_val_00026395.JPEG:n02412080 +ILSVRC2012_val_00026396.JPEG:n03785016 +ILSVRC2012_val_00026397.JPEG:n01440764 +ILSVRC2012_val_00026398.JPEG:n03100240 +ILSVRC2012_val_00026399.JPEG:n01665541 +ILSVRC2012_val_00026400.JPEG:n03710721 +ILSVRC2012_val_00026401.JPEG:n04599235 +ILSVRC2012_val_00026402.JPEG:n04370456 +ILSVRC2012_val_00026403.JPEG:n02124075 +ILSVRC2012_val_00026404.JPEG:n02138441 +ILSVRC2012_val_00026405.JPEG:n03085013 +ILSVRC2012_val_00026406.JPEG:n01744401 +ILSVRC2012_val_00026407.JPEG:n04296562 +ILSVRC2012_val_00026408.JPEG:n09835506 +ILSVRC2012_val_00026409.JPEG:n03785016 +ILSVRC2012_val_00026410.JPEG:n07754684 +ILSVRC2012_val_00026411.JPEG:n04311004 +ILSVRC2012_val_00026412.JPEG:n02124075 +ILSVRC2012_val_00026413.JPEG:n02802426 +ILSVRC2012_val_00026414.JPEG:n04239074 +ILSVRC2012_val_00026415.JPEG:n02971356 +ILSVRC2012_val_00026416.JPEG:n02009229 +ILSVRC2012_val_00026417.JPEG:n02096177 +ILSVRC2012_val_00026418.JPEG:n01695060 +ILSVRC2012_val_00026419.JPEG:n03954731 +ILSVRC2012_val_00026420.JPEG:n01828970 +ILSVRC2012_val_00026421.JPEG:n02086240 +ILSVRC2012_val_00026422.JPEG:n02447366 +ILSVRC2012_val_00026423.JPEG:n03095699 +ILSVRC2012_val_00026424.JPEG:n03590841 +ILSVRC2012_val_00026425.JPEG:n03482405 +ILSVRC2012_val_00026426.JPEG:n02107574 +ILSVRC2012_val_00026427.JPEG:n02096294 +ILSVRC2012_val_00026428.JPEG:n03085013 +ILSVRC2012_val_00026429.JPEG:n04456115 +ILSVRC2012_val_00026430.JPEG:n04486054 +ILSVRC2012_val_00026431.JPEG:n04599235 +ILSVRC2012_val_00026432.JPEG:n03141823 +ILSVRC2012_val_00026433.JPEG:n04263257 +ILSVRC2012_val_00026434.JPEG:n03877845 +ILSVRC2012_val_00026435.JPEG:n04428191 +ILSVRC2012_val_00026436.JPEG:n03976657 +ILSVRC2012_val_00026437.JPEG:n02797295 +ILSVRC2012_val_00026438.JPEG:n03637318 +ILSVRC2012_val_00026439.JPEG:n03041632 +ILSVRC2012_val_00026440.JPEG:n07579787 +ILSVRC2012_val_00026441.JPEG:n02687172 +ILSVRC2012_val_00026442.JPEG:n03201208 +ILSVRC2012_val_00026443.JPEG:n04579145 +ILSVRC2012_val_00026444.JPEG:n01608432 +ILSVRC2012_val_00026445.JPEG:n02099849 +ILSVRC2012_val_00026446.JPEG:n01667114 +ILSVRC2012_val_00026447.JPEG:n04372370 +ILSVRC2012_val_00026448.JPEG:n02106166 +ILSVRC2012_val_00026449.JPEG:n03075370 +ILSVRC2012_val_00026450.JPEG:n02138441 +ILSVRC2012_val_00026451.JPEG:n03028079 +ILSVRC2012_val_00026452.JPEG:n01930112 +ILSVRC2012_val_00026453.JPEG:n03388183 +ILSVRC2012_val_00026454.JPEG:n03825788 +ILSVRC2012_val_00026455.JPEG:n13044778 +ILSVRC2012_val_00026456.JPEG:n02687172 +ILSVRC2012_val_00026457.JPEG:n03692522 +ILSVRC2012_val_00026458.JPEG:n02391049 +ILSVRC2012_val_00026459.JPEG:n04254120 +ILSVRC2012_val_00026460.JPEG:n03146219 +ILSVRC2012_val_00026461.JPEG:n03126707 +ILSVRC2012_val_00026462.JPEG:n02025239 +ILSVRC2012_val_00026463.JPEG:n07714571 +ILSVRC2012_val_00026464.JPEG:n02869837 +ILSVRC2012_val_00026465.JPEG:n01580077 +ILSVRC2012_val_00026466.JPEG:n03594945 +ILSVRC2012_val_00026467.JPEG:n02109525 +ILSVRC2012_val_00026468.JPEG:n04099969 +ILSVRC2012_val_00026469.JPEG:n03792972 +ILSVRC2012_val_00026470.JPEG:n03623198 +ILSVRC2012_val_00026471.JPEG:n01872401 +ILSVRC2012_val_00026472.JPEG:n02441942 +ILSVRC2012_val_00026473.JPEG:n03032252 +ILSVRC2012_val_00026474.JPEG:n02687172 +ILSVRC2012_val_00026475.JPEG:n02096294 +ILSVRC2012_val_00026476.JPEG:n02037110 +ILSVRC2012_val_00026477.JPEG:n04310018 +ILSVRC2012_val_00026478.JPEG:n02280649 +ILSVRC2012_val_00026479.JPEG:n03992509 +ILSVRC2012_val_00026480.JPEG:n04037443 +ILSVRC2012_val_00026481.JPEG:n01806567 +ILSVRC2012_val_00026482.JPEG:n02325366 +ILSVRC2012_val_00026483.JPEG:n03372029 +ILSVRC2012_val_00026484.JPEG:n02259212 +ILSVRC2012_val_00026485.JPEG:n04371430 +ILSVRC2012_val_00026486.JPEG:n02391049 +ILSVRC2012_val_00026487.JPEG:n01755581 +ILSVRC2012_val_00026488.JPEG:n01820546 +ILSVRC2012_val_00026489.JPEG:n02264363 +ILSVRC2012_val_00026490.JPEG:n01494475 +ILSVRC2012_val_00026491.JPEG:n03201208 +ILSVRC2012_val_00026492.JPEG:n01774750 +ILSVRC2012_val_00026493.JPEG:n03259280 +ILSVRC2012_val_00026494.JPEG:n02687172 +ILSVRC2012_val_00026495.JPEG:n04090263 +ILSVRC2012_val_00026496.JPEG:n02483708 +ILSVRC2012_val_00026497.JPEG:n04487081 +ILSVRC2012_val_00026498.JPEG:n03218198 +ILSVRC2012_val_00026499.JPEG:n02480495 +ILSVRC2012_val_00026500.JPEG:n01692333 +ILSVRC2012_val_00026501.JPEG:n03017168 +ILSVRC2012_val_00026502.JPEG:n01843065 +ILSVRC2012_val_00026503.JPEG:n03930630 +ILSVRC2012_val_00026504.JPEG:n02056570 +ILSVRC2012_val_00026505.JPEG:n03041632 +ILSVRC2012_val_00026506.JPEG:n02799071 +ILSVRC2012_val_00026507.JPEG:n03344393 +ILSVRC2012_val_00026508.JPEG:n01514859 +ILSVRC2012_val_00026509.JPEG:n02113978 +ILSVRC2012_val_00026510.JPEG:n02027492 +ILSVRC2012_val_00026511.JPEG:n01981276 +ILSVRC2012_val_00026512.JPEG:n02397096 +ILSVRC2012_val_00026513.JPEG:n04192698 +ILSVRC2012_val_00026514.JPEG:n03134739 +ILSVRC2012_val_00026515.JPEG:n02666196 +ILSVRC2012_val_00026516.JPEG:n02117135 +ILSVRC2012_val_00026517.JPEG:n04461696 +ILSVRC2012_val_00026518.JPEG:n02231487 +ILSVRC2012_val_00026519.JPEG:n09246464 +ILSVRC2012_val_00026520.JPEG:n04149813 +ILSVRC2012_val_00026521.JPEG:n02102040 +ILSVRC2012_val_00026522.JPEG:n02086910 +ILSVRC2012_val_00026523.JPEG:n04355338 +ILSVRC2012_val_00026524.JPEG:n02457408 +ILSVRC2012_val_00026525.JPEG:n02093428 +ILSVRC2012_val_00026526.JPEG:n01689811 +ILSVRC2012_val_00026527.JPEG:n03481172 +ILSVRC2012_val_00026528.JPEG:n07836838 +ILSVRC2012_val_00026529.JPEG:n03803284 +ILSVRC2012_val_00026530.JPEG:n01910747 +ILSVRC2012_val_00026531.JPEG:n04553703 +ILSVRC2012_val_00026532.JPEG:n03478589 +ILSVRC2012_val_00026533.JPEG:n03584829 +ILSVRC2012_val_00026534.JPEG:n04254777 +ILSVRC2012_val_00026535.JPEG:n04254120 +ILSVRC2012_val_00026536.JPEG:n02105505 +ILSVRC2012_val_00026537.JPEG:n02361337 +ILSVRC2012_val_00026538.JPEG:n03992509 +ILSVRC2012_val_00026539.JPEG:n02804610 +ILSVRC2012_val_00026540.JPEG:n02102318 +ILSVRC2012_val_00026541.JPEG:n01560419 +ILSVRC2012_val_00026542.JPEG:n01773549 +ILSVRC2012_val_00026543.JPEG:n03902125 +ILSVRC2012_val_00026544.JPEG:n06359193 +ILSVRC2012_val_00026545.JPEG:n02129165 +ILSVRC2012_val_00026546.JPEG:n02120079 +ILSVRC2012_val_00026547.JPEG:n02113712 +ILSVRC2012_val_00026548.JPEG:n01728920 +ILSVRC2012_val_00026549.JPEG:n03160309 +ILSVRC2012_val_00026550.JPEG:n07871810 +ILSVRC2012_val_00026551.JPEG:n04258138 +ILSVRC2012_val_00026552.JPEG:n03045698 +ILSVRC2012_val_00026553.JPEG:n04552348 +ILSVRC2012_val_00026554.JPEG:n13044778 +ILSVRC2012_val_00026555.JPEG:n03717622 +ILSVRC2012_val_00026556.JPEG:n02025239 +ILSVRC2012_val_00026557.JPEG:n02268443 +ILSVRC2012_val_00026558.JPEG:n02108915 +ILSVRC2012_val_00026559.JPEG:n04542943 +ILSVRC2012_val_00026560.JPEG:n03240683 +ILSVRC2012_val_00026561.JPEG:n02966687 +ILSVRC2012_val_00026562.JPEG:n07754684 +ILSVRC2012_val_00026563.JPEG:n03991062 +ILSVRC2012_val_00026564.JPEG:n02769748 +ILSVRC2012_val_00026565.JPEG:n03187595 +ILSVRC2012_val_00026566.JPEG:n03271574 +ILSVRC2012_val_00026567.JPEG:n02256656 +ILSVRC2012_val_00026568.JPEG:n03637318 +ILSVRC2012_val_00026569.JPEG:n04357314 +ILSVRC2012_val_00026570.JPEG:n03207941 +ILSVRC2012_val_00026571.JPEG:n01728920 +ILSVRC2012_val_00026572.JPEG:n04074963 +ILSVRC2012_val_00026573.JPEG:n03000684 +ILSVRC2012_val_00026574.JPEG:n04118538 +ILSVRC2012_val_00026575.JPEG:n03888257 +ILSVRC2012_val_00026576.JPEG:n03000134 +ILSVRC2012_val_00026577.JPEG:n02930766 +ILSVRC2012_val_00026578.JPEG:n02437616 +ILSVRC2012_val_00026579.JPEG:n01622779 +ILSVRC2012_val_00026580.JPEG:n03954731 +ILSVRC2012_val_00026581.JPEG:n04266014 +ILSVRC2012_val_00026582.JPEG:n02108915 +ILSVRC2012_val_00026583.JPEG:n01729977 +ILSVRC2012_val_00026584.JPEG:n04553703 +ILSVRC2012_val_00026585.JPEG:n02328150 +ILSVRC2012_val_00026586.JPEG:n07715103 +ILSVRC2012_val_00026587.JPEG:n03617480 +ILSVRC2012_val_00026588.JPEG:n02441942 +ILSVRC2012_val_00026589.JPEG:n01734418 +ILSVRC2012_val_00026590.JPEG:n02229544 +ILSVRC2012_val_00026591.JPEG:n02259212 +ILSVRC2012_val_00026592.JPEG:n03017168 +ILSVRC2012_val_00026593.JPEG:n02077923 +ILSVRC2012_val_00026594.JPEG:n03871628 +ILSVRC2012_val_00026595.JPEG:n02025239 +ILSVRC2012_val_00026596.JPEG:n02992211 +ILSVRC2012_val_00026597.JPEG:n01978287 +ILSVRC2012_val_00026598.JPEG:n01755581 +ILSVRC2012_val_00026599.JPEG:n04008634 +ILSVRC2012_val_00026600.JPEG:n01773797 +ILSVRC2012_val_00026601.JPEG:n04209239 +ILSVRC2012_val_00026602.JPEG:n04584207 +ILSVRC2012_val_00026603.JPEG:n02493793 +ILSVRC2012_val_00026604.JPEG:n01616318 +ILSVRC2012_val_00026605.JPEG:n04127249 +ILSVRC2012_val_00026606.JPEG:n01877812 +ILSVRC2012_val_00026607.JPEG:n02814860 +ILSVRC2012_val_00026608.JPEG:n03535780 +ILSVRC2012_val_00026609.JPEG:n04040759 +ILSVRC2012_val_00026610.JPEG:n02879718 +ILSVRC2012_val_00026611.JPEG:n02514041 +ILSVRC2012_val_00026612.JPEG:n04592741 +ILSVRC2012_val_00026613.JPEG:n03854065 +ILSVRC2012_val_00026614.JPEG:n01614925 +ILSVRC2012_val_00026615.JPEG:n04026417 +ILSVRC2012_val_00026616.JPEG:n03837869 +ILSVRC2012_val_00026617.JPEG:n02865351 +ILSVRC2012_val_00026618.JPEG:n04239074 +ILSVRC2012_val_00026619.JPEG:n06794110 +ILSVRC2012_val_00026620.JPEG:n02190166 +ILSVRC2012_val_00026621.JPEG:n04208210 +ILSVRC2012_val_00026622.JPEG:n02088238 +ILSVRC2012_val_00026623.JPEG:n02497673 +ILSVRC2012_val_00026624.JPEG:n03179701 +ILSVRC2012_val_00026625.JPEG:n04613696 +ILSVRC2012_val_00026626.JPEG:n01693334 +ILSVRC2012_val_00026627.JPEG:n02672831 +ILSVRC2012_val_00026628.JPEG:n02817516 +ILSVRC2012_val_00026629.JPEG:n02106662 +ILSVRC2012_val_00026630.JPEG:n04392985 +ILSVRC2012_val_00026631.JPEG:n03777754 +ILSVRC2012_val_00026632.JPEG:n03649909 +ILSVRC2012_val_00026633.JPEG:n04311004 +ILSVRC2012_val_00026634.JPEG:n01664065 +ILSVRC2012_val_00026635.JPEG:n04389033 +ILSVRC2012_val_00026636.JPEG:n02807133 +ILSVRC2012_val_00026637.JPEG:n03476991 +ILSVRC2012_val_00026638.JPEG:n03141823 +ILSVRC2012_val_00026639.JPEG:n03793489 +ILSVRC2012_val_00026640.JPEG:n02988304 +ILSVRC2012_val_00026641.JPEG:n03325584 +ILSVRC2012_val_00026642.JPEG:n01871265 +ILSVRC2012_val_00026643.JPEG:n09288635 +ILSVRC2012_val_00026644.JPEG:n04326547 +ILSVRC2012_val_00026645.JPEG:n02110063 +ILSVRC2012_val_00026646.JPEG:n03220513 +ILSVRC2012_val_00026647.JPEG:n02093859 +ILSVRC2012_val_00026648.JPEG:n01693334 +ILSVRC2012_val_00026649.JPEG:n02815834 +ILSVRC2012_val_00026650.JPEG:n02107574 +ILSVRC2012_val_00026651.JPEG:n04487081 +ILSVRC2012_val_00026652.JPEG:n04347754 +ILSVRC2012_val_00026653.JPEG:n07695742 +ILSVRC2012_val_00026654.JPEG:n04086273 +ILSVRC2012_val_00026655.JPEG:n04493381 +ILSVRC2012_val_00026656.JPEG:n01580077 +ILSVRC2012_val_00026657.JPEG:n02910353 +ILSVRC2012_val_00026658.JPEG:n07754684 +ILSVRC2012_val_00026659.JPEG:n04067472 +ILSVRC2012_val_00026660.JPEG:n12768682 +ILSVRC2012_val_00026661.JPEG:n01675722 +ILSVRC2012_val_00026662.JPEG:n02437312 +ILSVRC2012_val_00026663.JPEG:n04417672 +ILSVRC2012_val_00026664.JPEG:n03868863 +ILSVRC2012_val_00026665.JPEG:n13054560 +ILSVRC2012_val_00026666.JPEG:n02100735 +ILSVRC2012_val_00026667.JPEG:n03888605 +ILSVRC2012_val_00026668.JPEG:n04009552 +ILSVRC2012_val_00026669.JPEG:n04238763 +ILSVRC2012_val_00026670.JPEG:n03876231 +ILSVRC2012_val_00026671.JPEG:n03706229 +ILSVRC2012_val_00026672.JPEG:n02859443 +ILSVRC2012_val_00026673.JPEG:n01530575 +ILSVRC2012_val_00026674.JPEG:n01824575 +ILSVRC2012_val_00026675.JPEG:n02096437 +ILSVRC2012_val_00026676.JPEG:n04486054 +ILSVRC2012_val_00026677.JPEG:n02704792 +ILSVRC2012_val_00026678.JPEG:n02110185 +ILSVRC2012_val_00026679.JPEG:n01824575 +ILSVRC2012_val_00026680.JPEG:n12620546 +ILSVRC2012_val_00026681.JPEG:n03814906 +ILSVRC2012_val_00026682.JPEG:n04154565 +ILSVRC2012_val_00026683.JPEG:n02058221 +ILSVRC2012_val_00026684.JPEG:n02111129 +ILSVRC2012_val_00026685.JPEG:n03690938 +ILSVRC2012_val_00026686.JPEG:n03857828 +ILSVRC2012_val_00026687.JPEG:n01534433 +ILSVRC2012_val_00026688.JPEG:n09229709 +ILSVRC2012_val_00026689.JPEG:n02086910 +ILSVRC2012_val_00026690.JPEG:n04507155 +ILSVRC2012_val_00026691.JPEG:n02098105 +ILSVRC2012_val_00026692.JPEG:n02089078 +ILSVRC2012_val_00026693.JPEG:n04355933 +ILSVRC2012_val_00026694.JPEG:n02930766 +ILSVRC2012_val_00026695.JPEG:n03384352 +ILSVRC2012_val_00026696.JPEG:n02892201 +ILSVRC2012_val_00026697.JPEG:n03992509 +ILSVRC2012_val_00026698.JPEG:n02109961 +ILSVRC2012_val_00026699.JPEG:n04479046 +ILSVRC2012_val_00026700.JPEG:n03000247 +ILSVRC2012_val_00026701.JPEG:n03047690 +ILSVRC2012_val_00026702.JPEG:n04258138 +ILSVRC2012_val_00026703.JPEG:n04005630 +ILSVRC2012_val_00026704.JPEG:n02281787 +ILSVRC2012_val_00026705.JPEG:n01693334 +ILSVRC2012_val_00026706.JPEG:n03379051 +ILSVRC2012_val_00026707.JPEG:n01614925 +ILSVRC2012_val_00026708.JPEG:n04479046 +ILSVRC2012_val_00026709.JPEG:n04591713 +ILSVRC2012_val_00026710.JPEG:n03920288 +ILSVRC2012_val_00026711.JPEG:n02051845 +ILSVRC2012_val_00026712.JPEG:n01756291 +ILSVRC2012_val_00026713.JPEG:n02107312 +ILSVRC2012_val_00026714.JPEG:n04435653 +ILSVRC2012_val_00026715.JPEG:n03325584 +ILSVRC2012_val_00026716.JPEG:n02058221 +ILSVRC2012_val_00026717.JPEG:n02107683 +ILSVRC2012_val_00026718.JPEG:n02111277 +ILSVRC2012_val_00026719.JPEG:n03786901 +ILSVRC2012_val_00026720.JPEG:n07768694 +ILSVRC2012_val_00026721.JPEG:n03891332 +ILSVRC2012_val_00026722.JPEG:n04204347 +ILSVRC2012_val_00026723.JPEG:n03400231 +ILSVRC2012_val_00026724.JPEG:n03961711 +ILSVRC2012_val_00026725.JPEG:n02490219 +ILSVRC2012_val_00026726.JPEG:n03347037 +ILSVRC2012_val_00026727.JPEG:n04597913 +ILSVRC2012_val_00026728.JPEG:n02090721 +ILSVRC2012_val_00026729.JPEG:n03450230 +ILSVRC2012_val_00026730.JPEG:n02112137 +ILSVRC2012_val_00026731.JPEG:n03250847 +ILSVRC2012_val_00026732.JPEG:n03868242 +ILSVRC2012_val_00026733.JPEG:n02058221 +ILSVRC2012_val_00026734.JPEG:n04141327 +ILSVRC2012_val_00026735.JPEG:n03761084 +ILSVRC2012_val_00026736.JPEG:n02090379 +ILSVRC2012_val_00026737.JPEG:n02486261 +ILSVRC2012_val_00026738.JPEG:n02095570 +ILSVRC2012_val_00026739.JPEG:n01749939 +ILSVRC2012_val_00026740.JPEG:n02804610 +ILSVRC2012_val_00026741.JPEG:n04273569 +ILSVRC2012_val_00026742.JPEG:n02777292 +ILSVRC2012_val_00026743.JPEG:n03930630 +ILSVRC2012_val_00026744.JPEG:n03775546 +ILSVRC2012_val_00026745.JPEG:n07716906 +ILSVRC2012_val_00026746.JPEG:n02916936 +ILSVRC2012_val_00026747.JPEG:n02930766 +ILSVRC2012_val_00026748.JPEG:n03709823 +ILSVRC2012_val_00026749.JPEG:n02056570 +ILSVRC2012_val_00026750.JPEG:n02412080 +ILSVRC2012_val_00026751.JPEG:n02666196 +ILSVRC2012_val_00026752.JPEG:n03196217 +ILSVRC2012_val_00026753.JPEG:n04479046 +ILSVRC2012_val_00026754.JPEG:n04509417 +ILSVRC2012_val_00026755.JPEG:n01532829 +ILSVRC2012_val_00026756.JPEG:n07697313 +ILSVRC2012_val_00026757.JPEG:n02493793 +ILSVRC2012_val_00026758.JPEG:n02058221 +ILSVRC2012_val_00026759.JPEG:n04252077 +ILSVRC2012_val_00026760.JPEG:n02002556 +ILSVRC2012_val_00026761.JPEG:n02085936 +ILSVRC2012_val_00026762.JPEG:n03063599 +ILSVRC2012_val_00026763.JPEG:n04273569 +ILSVRC2012_val_00026764.JPEG:n04550184 +ILSVRC2012_val_00026765.JPEG:n03710193 +ILSVRC2012_val_00026766.JPEG:n01742172 +ILSVRC2012_val_00026767.JPEG:n02443484 +ILSVRC2012_val_00026768.JPEG:n03720891 +ILSVRC2012_val_00026769.JPEG:n03706229 +ILSVRC2012_val_00026770.JPEG:n02643566 +ILSVRC2012_val_00026771.JPEG:n03218198 +ILSVRC2012_val_00026772.JPEG:n03877845 +ILSVRC2012_val_00026773.JPEG:n01630670 +ILSVRC2012_val_00026774.JPEG:n07714990 +ILSVRC2012_val_00026775.JPEG:n02264363 +ILSVRC2012_val_00026776.JPEG:n01532829 +ILSVRC2012_val_00026777.JPEG:n04540053 +ILSVRC2012_val_00026778.JPEG:n02113712 +ILSVRC2012_val_00026779.JPEG:n04259630 +ILSVRC2012_val_00026780.JPEG:n03661043 +ILSVRC2012_val_00026781.JPEG:n03220513 +ILSVRC2012_val_00026782.JPEG:n03445924 +ILSVRC2012_val_00026783.JPEG:n07831146 +ILSVRC2012_val_00026784.JPEG:n01530575 +ILSVRC2012_val_00026785.JPEG:n03691459 +ILSVRC2012_val_00026786.JPEG:n01773157 +ILSVRC2012_val_00026787.JPEG:n06785654 +ILSVRC2012_val_00026788.JPEG:n03290653 +ILSVRC2012_val_00026789.JPEG:n03995372 +ILSVRC2012_val_00026790.JPEG:n03866082 +ILSVRC2012_val_00026791.JPEG:n02276258 +ILSVRC2012_val_00026792.JPEG:n03777568 +ILSVRC2012_val_00026793.JPEG:n01675722 +ILSVRC2012_val_00026794.JPEG:n12985857 +ILSVRC2012_val_00026795.JPEG:n02835271 +ILSVRC2012_val_00026796.JPEG:n03444034 +ILSVRC2012_val_00026797.JPEG:n02101006 +ILSVRC2012_val_00026798.JPEG:n03637318 +ILSVRC2012_val_00026799.JPEG:n03787032 +ILSVRC2012_val_00026800.JPEG:n04258138 +ILSVRC2012_val_00026801.JPEG:n03535780 +ILSVRC2012_val_00026802.JPEG:n04065272 +ILSVRC2012_val_00026803.JPEG:n02099267 +ILSVRC2012_val_00026804.JPEG:n03347037 +ILSVRC2012_val_00026805.JPEG:n01755581 +ILSVRC2012_val_00026806.JPEG:n03908714 +ILSVRC2012_val_00026807.JPEG:n02056570 +ILSVRC2012_val_00026808.JPEG:n02093647 +ILSVRC2012_val_00026809.JPEG:n01729977 +ILSVRC2012_val_00026810.JPEG:n04344873 +ILSVRC2012_val_00026811.JPEG:n01847000 +ILSVRC2012_val_00026812.JPEG:n02112350 +ILSVRC2012_val_00026813.JPEG:n01632458 +ILSVRC2012_val_00026814.JPEG:n04562935 +ILSVRC2012_val_00026815.JPEG:n03325584 +ILSVRC2012_val_00026816.JPEG:n04127249 +ILSVRC2012_val_00026817.JPEG:n04141076 +ILSVRC2012_val_00026818.JPEG:n04554684 +ILSVRC2012_val_00026819.JPEG:n07714571 +ILSVRC2012_val_00026820.JPEG:n02027492 +ILSVRC2012_val_00026821.JPEG:n03532672 +ILSVRC2012_val_00026822.JPEG:n02992529 +ILSVRC2012_val_00026823.JPEG:n02321529 +ILSVRC2012_val_00026824.JPEG:n03538406 +ILSVRC2012_val_00026825.JPEG:n03721384 +ILSVRC2012_val_00026826.JPEG:n02013706 +ILSVRC2012_val_00026827.JPEG:n04599235 +ILSVRC2012_val_00026828.JPEG:n02093991 +ILSVRC2012_val_00026829.JPEG:n02777292 +ILSVRC2012_val_00026830.JPEG:n02123394 +ILSVRC2012_val_00026831.JPEG:n07747607 +ILSVRC2012_val_00026832.JPEG:n03424325 +ILSVRC2012_val_00026833.JPEG:n03976657 +ILSVRC2012_val_00026834.JPEG:n04209239 +ILSVRC2012_val_00026835.JPEG:n02951585 +ILSVRC2012_val_00026836.JPEG:n07753592 +ILSVRC2012_val_00026837.JPEG:n04443257 +ILSVRC2012_val_00026838.JPEG:n03388183 +ILSVRC2012_val_00026839.JPEG:n10148035 +ILSVRC2012_val_00026840.JPEG:n03344393 +ILSVRC2012_val_00026841.JPEG:n04336792 +ILSVRC2012_val_00026842.JPEG:n02120505 +ILSVRC2012_val_00026843.JPEG:n01981276 +ILSVRC2012_val_00026844.JPEG:n03933933 +ILSVRC2012_val_00026845.JPEG:n01829413 +ILSVRC2012_val_00026846.JPEG:n03916031 +ILSVRC2012_val_00026847.JPEG:n02776631 +ILSVRC2012_val_00026848.JPEG:n01775062 +ILSVRC2012_val_00026849.JPEG:n04286575 +ILSVRC2012_val_00026850.JPEG:n04209239 +ILSVRC2012_val_00026851.JPEG:n07730033 +ILSVRC2012_val_00026852.JPEG:n02099712 +ILSVRC2012_val_00026853.JPEG:n07613480 +ILSVRC2012_val_00026854.JPEG:n02100583 +ILSVRC2012_val_00026855.JPEG:n03733805 +ILSVRC2012_val_00026856.JPEG:n03873416 +ILSVRC2012_val_00026857.JPEG:n04476259 +ILSVRC2012_val_00026858.JPEG:n02113799 +ILSVRC2012_val_00026859.JPEG:n02690373 +ILSVRC2012_val_00026860.JPEG:n09468604 +ILSVRC2012_val_00026861.JPEG:n02009912 +ILSVRC2012_val_00026862.JPEG:n01980166 +ILSVRC2012_val_00026863.JPEG:n02096294 +ILSVRC2012_val_00026864.JPEG:n03764736 +ILSVRC2012_val_00026865.JPEG:n03417042 +ILSVRC2012_val_00026866.JPEG:n03000134 +ILSVRC2012_val_00026867.JPEG:n10565667 +ILSVRC2012_val_00026868.JPEG:n04120489 +ILSVRC2012_val_00026869.JPEG:n02114855 +ILSVRC2012_val_00026870.JPEG:n04039381 +ILSVRC2012_val_00026871.JPEG:n04376876 +ILSVRC2012_val_00026872.JPEG:n02843684 +ILSVRC2012_val_00026873.JPEG:n02643566 +ILSVRC2012_val_00026874.JPEG:n03924679 +ILSVRC2012_val_00026875.JPEG:n03958227 +ILSVRC2012_val_00026876.JPEG:n03773504 +ILSVRC2012_val_00026877.JPEG:n02276258 +ILSVRC2012_val_00026878.JPEG:n03776460 +ILSVRC2012_val_00026879.JPEG:n03000684 +ILSVRC2012_val_00026880.JPEG:n02129165 +ILSVRC2012_val_00026881.JPEG:n03445924 +ILSVRC2012_val_00026882.JPEG:n02108089 +ILSVRC2012_val_00026883.JPEG:n04310018 +ILSVRC2012_val_00026884.JPEG:n03873416 +ILSVRC2012_val_00026885.JPEG:n02236044 +ILSVRC2012_val_00026886.JPEG:n03483316 +ILSVRC2012_val_00026887.JPEG:n02099601 +ILSVRC2012_val_00026888.JPEG:n02115913 +ILSVRC2012_val_00026889.JPEG:n02441942 +ILSVRC2012_val_00026890.JPEG:n03967562 +ILSVRC2012_val_00026891.JPEG:n04479046 +ILSVRC2012_val_00026892.JPEG:n04344873 +ILSVRC2012_val_00026893.JPEG:n02123597 +ILSVRC2012_val_00026894.JPEG:n02229544 +ILSVRC2012_val_00026895.JPEG:n03179701 +ILSVRC2012_val_00026896.JPEG:n02791124 +ILSVRC2012_val_00026897.JPEG:n04525305 +ILSVRC2012_val_00026898.JPEG:n03976657 +ILSVRC2012_val_00026899.JPEG:n04147183 +ILSVRC2012_val_00026900.JPEG:n02835271 +ILSVRC2012_val_00026901.JPEG:n01685808 +ILSVRC2012_val_00026902.JPEG:n02280649 +ILSVRC2012_val_00026903.JPEG:n01768244 +ILSVRC2012_val_00026904.JPEG:n02489166 +ILSVRC2012_val_00026905.JPEG:n04355338 +ILSVRC2012_val_00026906.JPEG:n02279972 +ILSVRC2012_val_00026907.JPEG:n03770679 +ILSVRC2012_val_00026908.JPEG:n01498041 +ILSVRC2012_val_00026909.JPEG:n04041544 +ILSVRC2012_val_00026910.JPEG:n02085620 +ILSVRC2012_val_00026911.JPEG:n02086240 +ILSVRC2012_val_00026912.JPEG:n03532672 +ILSVRC2012_val_00026913.JPEG:n02268853 +ILSVRC2012_val_00026914.JPEG:n02978881 +ILSVRC2012_val_00026915.JPEG:n02363005 +ILSVRC2012_val_00026916.JPEG:n04442312 +ILSVRC2012_val_00026917.JPEG:n02280649 +ILSVRC2012_val_00026918.JPEG:n02108915 +ILSVRC2012_val_00026919.JPEG:n04380533 +ILSVRC2012_val_00026920.JPEG:n04462240 +ILSVRC2012_val_00026921.JPEG:n03271574 +ILSVRC2012_val_00026922.JPEG:n03930630 +ILSVRC2012_val_00026923.JPEG:n02892767 +ILSVRC2012_val_00026924.JPEG:n01797886 +ILSVRC2012_val_00026925.JPEG:n01978287 +ILSVRC2012_val_00026926.JPEG:n02437616 +ILSVRC2012_val_00026927.JPEG:n03920288 +ILSVRC2012_val_00026928.JPEG:n03160309 +ILSVRC2012_val_00026929.JPEG:n01560419 +ILSVRC2012_val_00026930.JPEG:n02666196 +ILSVRC2012_val_00026931.JPEG:n03424325 +ILSVRC2012_val_00026932.JPEG:n02514041 +ILSVRC2012_val_00026933.JPEG:n02790996 +ILSVRC2012_val_00026934.JPEG:n02397096 +ILSVRC2012_val_00026935.JPEG:n01775062 +ILSVRC2012_val_00026936.JPEG:n02071294 +ILSVRC2012_val_00026937.JPEG:n02100583 +ILSVRC2012_val_00026938.JPEG:n04380533 +ILSVRC2012_val_00026939.JPEG:n01990800 +ILSVRC2012_val_00026940.JPEG:n03903868 +ILSVRC2012_val_00026941.JPEG:n07583066 +ILSVRC2012_val_00026942.JPEG:n02013706 +ILSVRC2012_val_00026943.JPEG:n02130308 +ILSVRC2012_val_00026944.JPEG:n02113023 +ILSVRC2012_val_00026945.JPEG:n03884397 +ILSVRC2012_val_00026946.JPEG:n03000684 +ILSVRC2012_val_00026947.JPEG:n04037443 +ILSVRC2012_val_00026948.JPEG:n01687978 +ILSVRC2012_val_00026949.JPEG:n02058221 +ILSVRC2012_val_00026950.JPEG:n02704792 +ILSVRC2012_val_00026951.JPEG:n07693725 +ILSVRC2012_val_00026952.JPEG:n04039381 +ILSVRC2012_val_00026953.JPEG:n03461385 +ILSVRC2012_val_00026954.JPEG:n01950731 +ILSVRC2012_val_00026955.JPEG:n03773504 +ILSVRC2012_val_00026956.JPEG:n02104365 +ILSVRC2012_val_00026957.JPEG:n04536866 +ILSVRC2012_val_00026958.JPEG:n02328150 +ILSVRC2012_val_00026959.JPEG:n07871810 +ILSVRC2012_val_00026960.JPEG:n03372029 +ILSVRC2012_val_00026961.JPEG:n04462240 +ILSVRC2012_val_00026962.JPEG:n02133161 +ILSVRC2012_val_00026963.JPEG:n02808304 +ILSVRC2012_val_00026964.JPEG:n03443371 +ILSVRC2012_val_00026965.JPEG:n01843065 +ILSVRC2012_val_00026966.JPEG:n01914609 +ILSVRC2012_val_00026967.JPEG:n01855032 +ILSVRC2012_val_00026968.JPEG:n04380533 +ILSVRC2012_val_00026969.JPEG:n02086646 +ILSVRC2012_val_00026970.JPEG:n02363005 +ILSVRC2012_val_00026971.JPEG:n04296562 +ILSVRC2012_val_00026972.JPEG:n04033995 +ILSVRC2012_val_00026973.JPEG:n02871525 +ILSVRC2012_val_00026974.JPEG:n03742115 +ILSVRC2012_val_00026975.JPEG:n02704792 +ILSVRC2012_val_00026976.JPEG:n02108915 +ILSVRC2012_val_00026977.JPEG:n03670208 +ILSVRC2012_val_00026978.JPEG:n02093428 +ILSVRC2012_val_00026979.JPEG:n04428191 +ILSVRC2012_val_00026980.JPEG:n09421951 +ILSVRC2012_val_00026981.JPEG:n01984695 +ILSVRC2012_val_00026982.JPEG:n02128757 +ILSVRC2012_val_00026983.JPEG:n01917289 +ILSVRC2012_val_00026984.JPEG:n04033901 +ILSVRC2012_val_00026985.JPEG:n02092002 +ILSVRC2012_val_00026986.JPEG:n03840681 +ILSVRC2012_val_00026987.JPEG:n03476684 +ILSVRC2012_val_00026988.JPEG:n04286575 +ILSVRC2012_val_00026989.JPEG:n04423845 +ILSVRC2012_val_00026990.JPEG:n02951358 +ILSVRC2012_val_00026991.JPEG:n03877845 +ILSVRC2012_val_00026992.JPEG:n01728572 +ILSVRC2012_val_00026993.JPEG:n03481172 +ILSVRC2012_val_00026994.JPEG:n03208938 +ILSVRC2012_val_00026995.JPEG:n02487347 +ILSVRC2012_val_00026996.JPEG:n02107908 +ILSVRC2012_val_00026997.JPEG:n07565083 +ILSVRC2012_val_00026998.JPEG:n04479046 +ILSVRC2012_val_00026999.JPEG:n03832673 +ILSVRC2012_val_00027000.JPEG:n02948072 +ILSVRC2012_val_00027001.JPEG:n02950826 +ILSVRC2012_val_00027002.JPEG:n03929660 +ILSVRC2012_val_00027003.JPEG:n04370456 +ILSVRC2012_val_00027004.JPEG:n02978881 +ILSVRC2012_val_00027005.JPEG:n01498041 +ILSVRC2012_val_00027006.JPEG:n02783161 +ILSVRC2012_val_00027007.JPEG:n03697007 +ILSVRC2012_val_00027008.JPEG:n01820546 +ILSVRC2012_val_00027009.JPEG:n03026506 +ILSVRC2012_val_00027010.JPEG:n04584207 +ILSVRC2012_val_00027011.JPEG:n02091467 +ILSVRC2012_val_00027012.JPEG:n02422699 +ILSVRC2012_val_00027013.JPEG:n02123045 +ILSVRC2012_val_00027014.JPEG:n03793489 +ILSVRC2012_val_00027015.JPEG:n03958227 +ILSVRC2012_val_00027016.JPEG:n02443484 +ILSVRC2012_val_00027017.JPEG:n02098286 +ILSVRC2012_val_00027018.JPEG:n02788148 +ILSVRC2012_val_00027019.JPEG:n04392985 +ILSVRC2012_val_00027020.JPEG:n12768682 +ILSVRC2012_val_00027021.JPEG:n03843555 +ILSVRC2012_val_00027022.JPEG:n02894605 +ILSVRC2012_val_00027023.JPEG:n04372370 +ILSVRC2012_val_00027024.JPEG:n02077923 +ILSVRC2012_val_00027025.JPEG:n02111889 +ILSVRC2012_val_00027026.JPEG:n01770393 +ILSVRC2012_val_00027027.JPEG:n02840245 +ILSVRC2012_val_00027028.JPEG:n01631663 +ILSVRC2012_val_00027029.JPEG:n02786058 +ILSVRC2012_val_00027030.JPEG:n04462240 +ILSVRC2012_val_00027031.JPEG:n02264363 +ILSVRC2012_val_00027032.JPEG:n03942813 +ILSVRC2012_val_00027033.JPEG:n02457408 +ILSVRC2012_val_00027034.JPEG:n03476991 +ILSVRC2012_val_00027035.JPEG:n02107312 +ILSVRC2012_val_00027036.JPEG:n02917067 +ILSVRC2012_val_00027037.JPEG:n04612504 +ILSVRC2012_val_00027038.JPEG:n02100583 +ILSVRC2012_val_00027039.JPEG:n04239074 +ILSVRC2012_val_00027040.JPEG:n04476259 +ILSVRC2012_val_00027041.JPEG:n02105855 +ILSVRC2012_val_00027042.JPEG:n03929855 +ILSVRC2012_val_00027043.JPEG:n02389026 +ILSVRC2012_val_00027044.JPEG:n04389033 +ILSVRC2012_val_00027045.JPEG:n03876231 +ILSVRC2012_val_00027046.JPEG:n04041544 +ILSVRC2012_val_00027047.JPEG:n01806143 +ILSVRC2012_val_00027048.JPEG:n07584110 +ILSVRC2012_val_00027049.JPEG:n02814533 +ILSVRC2012_val_00027050.JPEG:n03868863 +ILSVRC2012_val_00027051.JPEG:n02104365 +ILSVRC2012_val_00027052.JPEG:n02128925 +ILSVRC2012_val_00027053.JPEG:n02105251 +ILSVRC2012_val_00027054.JPEG:n04447861 +ILSVRC2012_val_00027055.JPEG:n04517823 +ILSVRC2012_val_00027056.JPEG:n02395406 +ILSVRC2012_val_00027057.JPEG:n04208210 +ILSVRC2012_val_00027058.JPEG:n02091831 +ILSVRC2012_val_00027059.JPEG:n04330267 +ILSVRC2012_val_00027060.JPEG:n02444819 +ILSVRC2012_val_00027061.JPEG:n02815834 +ILSVRC2012_val_00027062.JPEG:n02264363 +ILSVRC2012_val_00027063.JPEG:n01484850 +ILSVRC2012_val_00027064.JPEG:n02105641 +ILSVRC2012_val_00027065.JPEG:n02808440 +ILSVRC2012_val_00027066.JPEG:n02116738 +ILSVRC2012_val_00027067.JPEG:n01873310 +ILSVRC2012_val_00027068.JPEG:n03792972 +ILSVRC2012_val_00027069.JPEG:n02125311 +ILSVRC2012_val_00027070.JPEG:n01855032 +ILSVRC2012_val_00027071.JPEG:n02704792 +ILSVRC2012_val_00027072.JPEG:n07717556 +ILSVRC2012_val_00027073.JPEG:n03814906 +ILSVRC2012_val_00027074.JPEG:n01667114 +ILSVRC2012_val_00027075.JPEG:n03857828 +ILSVRC2012_val_00027076.JPEG:n01784675 +ILSVRC2012_val_00027077.JPEG:n02091032 +ILSVRC2012_val_00027078.JPEG:n04409515 +ILSVRC2012_val_00027079.JPEG:n01614925 +ILSVRC2012_val_00027080.JPEG:n03769881 +ILSVRC2012_val_00027081.JPEG:n02814533 +ILSVRC2012_val_00027082.JPEG:n02093754 +ILSVRC2012_val_00027083.JPEG:n07747607 +ILSVRC2012_val_00027084.JPEG:n03857828 +ILSVRC2012_val_00027085.JPEG:n04277352 +ILSVRC2012_val_00027086.JPEG:n02104029 +ILSVRC2012_val_00027087.JPEG:n04131690 +ILSVRC2012_val_00027088.JPEG:n02951358 +ILSVRC2012_val_00027089.JPEG:n02134084 +ILSVRC2012_val_00027090.JPEG:n07749582 +ILSVRC2012_val_00027091.JPEG:n03126707 +ILSVRC2012_val_00027092.JPEG:n04325704 +ILSVRC2012_val_00027093.JPEG:n02497673 +ILSVRC2012_val_00027094.JPEG:n02105412 +ILSVRC2012_val_00027095.JPEG:n01685808 +ILSVRC2012_val_00027096.JPEG:n07871810 +ILSVRC2012_val_00027097.JPEG:n02927161 +ILSVRC2012_val_00027098.JPEG:n04380533 +ILSVRC2012_val_00027099.JPEG:n04152593 +ILSVRC2012_val_00027100.JPEG:n02106382 +ILSVRC2012_val_00027101.JPEG:n04350905 +ILSVRC2012_val_00027102.JPEG:n01795545 +ILSVRC2012_val_00027103.JPEG:n03871628 +ILSVRC2012_val_00027104.JPEG:n02965783 +ILSVRC2012_val_00027105.JPEG:n07614500 +ILSVRC2012_val_00027106.JPEG:n03884397 +ILSVRC2012_val_00027107.JPEG:n03980874 +ILSVRC2012_val_00027108.JPEG:n02492035 +ILSVRC2012_val_00027109.JPEG:n02113712 +ILSVRC2012_val_00027110.JPEG:n03417042 +ILSVRC2012_val_00027111.JPEG:n04259630 +ILSVRC2012_val_00027112.JPEG:n03483316 +ILSVRC2012_val_00027113.JPEG:n01494475 +ILSVRC2012_val_00027114.JPEG:n02088238 +ILSVRC2012_val_00027115.JPEG:n07565083 +ILSVRC2012_val_00027116.JPEG:n07753113 +ILSVRC2012_val_00027117.JPEG:n04366367 +ILSVRC2012_val_00027118.JPEG:n04120489 +ILSVRC2012_val_00027119.JPEG:n04429376 +ILSVRC2012_val_00027120.JPEG:n02091467 +ILSVRC2012_val_00027121.JPEG:n02112350 +ILSVRC2012_val_00027122.JPEG:n02699494 +ILSVRC2012_val_00027123.JPEG:n03995372 +ILSVRC2012_val_00027124.JPEG:n02113186 +ILSVRC2012_val_00027125.JPEG:n01685808 +ILSVRC2012_val_00027126.JPEG:n03347037 +ILSVRC2012_val_00027127.JPEG:n02843684 +ILSVRC2012_val_00027128.JPEG:n02108089 +ILSVRC2012_val_00027129.JPEG:n03825788 +ILSVRC2012_val_00027130.JPEG:n03773504 +ILSVRC2012_val_00027131.JPEG:n02787622 +ILSVRC2012_val_00027132.JPEG:n04325704 +ILSVRC2012_val_00027133.JPEG:n03796401 +ILSVRC2012_val_00027134.JPEG:n01698640 +ILSVRC2012_val_00027135.JPEG:n03045698 +ILSVRC2012_val_00027136.JPEG:n02422699 +ILSVRC2012_val_00027137.JPEG:n04417672 +ILSVRC2012_val_00027138.JPEG:n04141327 +ILSVRC2012_val_00027139.JPEG:n04118538 +ILSVRC2012_val_00027140.JPEG:n02113624 +ILSVRC2012_val_00027141.JPEG:n04550184 +ILSVRC2012_val_00027142.JPEG:n01728572 +ILSVRC2012_val_00027143.JPEG:n04380533 +ILSVRC2012_val_00027144.JPEG:n04209133 +ILSVRC2012_val_00027145.JPEG:n01537544 +ILSVRC2012_val_00027146.JPEG:n07920052 +ILSVRC2012_val_00027147.JPEG:n04317175 +ILSVRC2012_val_00027148.JPEG:n01742172 +ILSVRC2012_val_00027149.JPEG:n02786058 +ILSVRC2012_val_00027150.JPEG:n03417042 +ILSVRC2012_val_00027151.JPEG:n03770679 +ILSVRC2012_val_00027152.JPEG:n02804414 +ILSVRC2012_val_00027153.JPEG:n02236044 +ILSVRC2012_val_00027154.JPEG:n03085013 +ILSVRC2012_val_00027155.JPEG:n04019541 +ILSVRC2012_val_00027156.JPEG:n03661043 +ILSVRC2012_val_00027157.JPEG:n03769881 +ILSVRC2012_val_00027158.JPEG:n01773797 +ILSVRC2012_val_00027159.JPEG:n02835271 +ILSVRC2012_val_00027160.JPEG:n01494475 +ILSVRC2012_val_00027161.JPEG:n01773797 +ILSVRC2012_val_00027162.JPEG:n02097298 +ILSVRC2012_val_00027163.JPEG:n01667114 +ILSVRC2012_val_00027164.JPEG:n02106030 +ILSVRC2012_val_00027165.JPEG:n02106030 +ILSVRC2012_val_00027166.JPEG:n03146219 +ILSVRC2012_val_00027167.JPEG:n01930112 +ILSVRC2012_val_00027168.JPEG:n02102177 +ILSVRC2012_val_00027169.JPEG:n13040303 +ILSVRC2012_val_00027170.JPEG:n04357314 +ILSVRC2012_val_00027171.JPEG:n04264628 +ILSVRC2012_val_00027172.JPEG:n07875152 +ILSVRC2012_val_00027173.JPEG:n04371774 +ILSVRC2012_val_00027174.JPEG:n02099849 +ILSVRC2012_val_00027175.JPEG:n03127925 +ILSVRC2012_val_00027176.JPEG:n02869837 +ILSVRC2012_val_00027177.JPEG:n03710193 +ILSVRC2012_val_00027178.JPEG:n02097130 +ILSVRC2012_val_00027179.JPEG:n07730033 +ILSVRC2012_val_00027180.JPEG:n04311004 +ILSVRC2012_val_00027181.JPEG:n03085013 +ILSVRC2012_val_00027182.JPEG:n02102040 +ILSVRC2012_val_00027183.JPEG:n04486054 +ILSVRC2012_val_00027184.JPEG:n02111889 +ILSVRC2012_val_00027185.JPEG:n04204238 +ILSVRC2012_val_00027186.JPEG:n03792972 +ILSVRC2012_val_00027187.JPEG:n03450230 +ILSVRC2012_val_00027188.JPEG:n03617480 +ILSVRC2012_val_00027189.JPEG:n02124075 +ILSVRC2012_val_00027190.JPEG:n03495258 +ILSVRC2012_val_00027191.JPEG:n03769881 +ILSVRC2012_val_00027192.JPEG:n02916936 +ILSVRC2012_val_00027193.JPEG:n01704323 +ILSVRC2012_val_00027194.JPEG:n03063599 +ILSVRC2012_val_00027195.JPEG:n01883070 +ILSVRC2012_val_00027196.JPEG:n01614925 +ILSVRC2012_val_00027197.JPEG:n04311004 +ILSVRC2012_val_00027198.JPEG:n01692333 +ILSVRC2012_val_00027199.JPEG:n03125729 +ILSVRC2012_val_00027200.JPEG:n04192698 +ILSVRC2012_val_00027201.JPEG:n03874293 +ILSVRC2012_val_00027202.JPEG:n03496892 +ILSVRC2012_val_00027203.JPEG:n04118776 +ILSVRC2012_val_00027204.JPEG:n02454379 +ILSVRC2012_val_00027205.JPEG:n04116512 +ILSVRC2012_val_00027206.JPEG:n01677366 +ILSVRC2012_val_00027207.JPEG:n01514668 +ILSVRC2012_val_00027208.JPEG:n03476991 +ILSVRC2012_val_00027209.JPEG:n03733805 +ILSVRC2012_val_00027210.JPEG:n03942813 +ILSVRC2012_val_00027211.JPEG:n03095699 +ILSVRC2012_val_00027212.JPEG:n02883205 +ILSVRC2012_val_00027213.JPEG:n02091467 +ILSVRC2012_val_00027214.JPEG:n02817516 +ILSVRC2012_val_00027215.JPEG:n06794110 +ILSVRC2012_val_00027216.JPEG:n03131574 +ILSVRC2012_val_00027217.JPEG:n02101388 +ILSVRC2012_val_00027218.JPEG:n01978455 +ILSVRC2012_val_00027219.JPEG:n02106382 +ILSVRC2012_val_00027220.JPEG:n02108915 +ILSVRC2012_val_00027221.JPEG:n03216828 +ILSVRC2012_val_00027222.JPEG:n07615774 +ILSVRC2012_val_00027223.JPEG:n07730033 +ILSVRC2012_val_00027224.JPEG:n01770393 +ILSVRC2012_val_00027225.JPEG:n04371430 +ILSVRC2012_val_00027226.JPEG:n02123159 +ILSVRC2012_val_00027227.JPEG:n01984695 +ILSVRC2012_val_00027228.JPEG:n01737021 +ILSVRC2012_val_00027229.JPEG:n02825657 +ILSVRC2012_val_00027230.JPEG:n02099267 +ILSVRC2012_val_00027231.JPEG:n03658185 +ILSVRC2012_val_00027232.JPEG:n02815834 +ILSVRC2012_val_00027233.JPEG:n02120079 +ILSVRC2012_val_00027234.JPEG:n03908714 +ILSVRC2012_val_00027235.JPEG:n04554684 +ILSVRC2012_val_00027236.JPEG:n04604644 +ILSVRC2012_val_00027237.JPEG:n03109150 +ILSVRC2012_val_00027238.JPEG:n03866082 +ILSVRC2012_val_00027239.JPEG:n03908714 +ILSVRC2012_val_00027240.JPEG:n03617480 +ILSVRC2012_val_00027241.JPEG:n02093647 +ILSVRC2012_val_00027242.JPEG:n02510455 +ILSVRC2012_val_00027243.JPEG:n04074963 +ILSVRC2012_val_00027244.JPEG:n03089624 +ILSVRC2012_val_00027245.JPEG:n02095314 +ILSVRC2012_val_00027246.JPEG:n03218198 +ILSVRC2012_val_00027247.JPEG:n02817516 +ILSVRC2012_val_00027248.JPEG:n01943899 +ILSVRC2012_val_00027249.JPEG:n03854065 +ILSVRC2012_val_00027250.JPEG:n03891251 +ILSVRC2012_val_00027251.JPEG:n04423845 +ILSVRC2012_val_00027252.JPEG:n04131690 +ILSVRC2012_val_00027253.JPEG:n04442312 +ILSVRC2012_val_00027254.JPEG:n01537544 +ILSVRC2012_val_00027255.JPEG:n03325584 +ILSVRC2012_val_00027256.JPEG:n02095889 +ILSVRC2012_val_00027257.JPEG:n03291819 +ILSVRC2012_val_00027258.JPEG:n03042490 +ILSVRC2012_val_00027259.JPEG:n02504013 +ILSVRC2012_val_00027260.JPEG:n03146219 +ILSVRC2012_val_00027261.JPEG:n04252077 +ILSVRC2012_val_00027262.JPEG:n02328150 +ILSVRC2012_val_00027263.JPEG:n01697457 +ILSVRC2012_val_00027264.JPEG:n02655020 +ILSVRC2012_val_00027265.JPEG:n04606251 +ILSVRC2012_val_00027266.JPEG:n07720875 +ILSVRC2012_val_00027267.JPEG:n02091831 +ILSVRC2012_val_00027268.JPEG:n02097209 +ILSVRC2012_val_00027269.JPEG:n01630670 +ILSVRC2012_val_00027270.JPEG:n01950731 +ILSVRC2012_val_00027271.JPEG:n01910747 +ILSVRC2012_val_00027272.JPEG:n07695742 +ILSVRC2012_val_00027273.JPEG:n03063689 +ILSVRC2012_val_00027274.JPEG:n01871265 +ILSVRC2012_val_00027275.JPEG:n03478589 +ILSVRC2012_val_00027276.JPEG:n07583066 +ILSVRC2012_val_00027277.JPEG:n02109525 +ILSVRC2012_val_00027278.JPEG:n03982430 +ILSVRC2012_val_00027279.JPEG:n04270147 +ILSVRC2012_val_00027280.JPEG:n01871265 +ILSVRC2012_val_00027281.JPEG:n02033041 +ILSVRC2012_val_00027282.JPEG:n03476991 +ILSVRC2012_val_00027283.JPEG:n01494475 +ILSVRC2012_val_00027284.JPEG:n09229709 +ILSVRC2012_val_00027285.JPEG:n03967562 +ILSVRC2012_val_00027286.JPEG:n03902125 +ILSVRC2012_val_00027287.JPEG:n02837789 +ILSVRC2012_val_00027288.JPEG:n04311004 +ILSVRC2012_val_00027289.JPEG:n04228054 +ILSVRC2012_val_00027290.JPEG:n02087394 +ILSVRC2012_val_00027291.JPEG:n04147183 +ILSVRC2012_val_00027292.JPEG:n02133161 +ILSVRC2012_val_00027293.JPEG:n03100240 +ILSVRC2012_val_00027294.JPEG:n04204238 +ILSVRC2012_val_00027295.JPEG:n02445715 +ILSVRC2012_val_00027296.JPEG:n03481172 +ILSVRC2012_val_00027297.JPEG:n04487394 +ILSVRC2012_val_00027298.JPEG:n03796401 +ILSVRC2012_val_00027299.JPEG:n02978881 +ILSVRC2012_val_00027300.JPEG:n01877812 +ILSVRC2012_val_00027301.JPEG:n01496331 +ILSVRC2012_val_00027302.JPEG:n07717410 +ILSVRC2012_val_00027303.JPEG:n02871525 +ILSVRC2012_val_00027304.JPEG:n02442845 +ILSVRC2012_val_00027305.JPEG:n02112706 +ILSVRC2012_val_00027306.JPEG:n02879718 +ILSVRC2012_val_00027307.JPEG:n03085013 +ILSVRC2012_val_00027308.JPEG:n02799071 +ILSVRC2012_val_00027309.JPEG:n03902125 +ILSVRC2012_val_00027310.JPEG:n02965783 +ILSVRC2012_val_00027311.JPEG:n02281406 +ILSVRC2012_val_00027312.JPEG:n04404412 +ILSVRC2012_val_00027313.JPEG:n02123159 +ILSVRC2012_val_00027314.JPEG:n02747177 +ILSVRC2012_val_00027315.JPEG:n04548280 +ILSVRC2012_val_00027316.JPEG:n04591713 +ILSVRC2012_val_00027317.JPEG:n04044716 +ILSVRC2012_val_00027318.JPEG:n03742115 +ILSVRC2012_val_00027319.JPEG:n02992211 +ILSVRC2012_val_00027320.JPEG:n07717410 +ILSVRC2012_val_00027321.JPEG:n10148035 +ILSVRC2012_val_00027322.JPEG:n02099429 +ILSVRC2012_val_00027323.JPEG:n02486261 +ILSVRC2012_val_00027324.JPEG:n04447861 +ILSVRC2012_val_00027325.JPEG:n03843555 +ILSVRC2012_val_00027326.JPEG:n04263257 +ILSVRC2012_val_00027327.JPEG:n04330267 +ILSVRC2012_val_00027328.JPEG:n02787622 +ILSVRC2012_val_00027329.JPEG:n02823750 +ILSVRC2012_val_00027330.JPEG:n01740131 +ILSVRC2012_val_00027331.JPEG:n04235860 +ILSVRC2012_val_00027332.JPEG:n03498962 +ILSVRC2012_val_00027333.JPEG:n02492660 +ILSVRC2012_val_00027334.JPEG:n02437312 +ILSVRC2012_val_00027335.JPEG:n07718747 +ILSVRC2012_val_00027336.JPEG:n03803284 +ILSVRC2012_val_00027337.JPEG:n02364673 +ILSVRC2012_val_00027338.JPEG:n02906734 +ILSVRC2012_val_00027339.JPEG:n07684084 +ILSVRC2012_val_00027340.JPEG:n03970156 +ILSVRC2012_val_00027341.JPEG:n03825788 +ILSVRC2012_val_00027342.JPEG:n03814906 +ILSVRC2012_val_00027343.JPEG:n07715103 +ILSVRC2012_val_00027344.JPEG:n02749479 +ILSVRC2012_val_00027345.JPEG:n02815834 +ILSVRC2012_val_00027346.JPEG:n02877765 +ILSVRC2012_val_00027347.JPEG:n02088364 +ILSVRC2012_val_00027348.JPEG:n02088632 +ILSVRC2012_val_00027349.JPEG:n04270147 +ILSVRC2012_val_00027350.JPEG:n07248320 +ILSVRC2012_val_00027351.JPEG:n01514668 +ILSVRC2012_val_00027352.JPEG:n01883070 +ILSVRC2012_val_00027353.JPEG:n02276258 +ILSVRC2012_val_00027354.JPEG:n04554684 +ILSVRC2012_val_00027355.JPEG:n02009229 +ILSVRC2012_val_00027356.JPEG:n07248320 +ILSVRC2012_val_00027357.JPEG:n01924916 +ILSVRC2012_val_00027358.JPEG:n03376595 +ILSVRC2012_val_00027359.JPEG:n03983396 +ILSVRC2012_val_00027360.JPEG:n02112018 +ILSVRC2012_val_00027361.JPEG:n01770393 +ILSVRC2012_val_00027362.JPEG:n02403003 +ILSVRC2012_val_00027363.JPEG:n02051845 +ILSVRC2012_val_00027364.JPEG:n02870880 +ILSVRC2012_val_00027365.JPEG:n02484975 +ILSVRC2012_val_00027366.JPEG:n02113799 +ILSVRC2012_val_00027367.JPEG:n03717622 +ILSVRC2012_val_00027368.JPEG:n07930864 +ILSVRC2012_val_00027369.JPEG:n07717410 +ILSVRC2012_val_00027370.JPEG:n02730930 +ILSVRC2012_val_00027371.JPEG:n03874599 +ILSVRC2012_val_00027372.JPEG:n02105162 +ILSVRC2012_val_00027373.JPEG:n02099712 +ILSVRC2012_val_00027374.JPEG:n01530575 +ILSVRC2012_val_00027375.JPEG:n03891332 +ILSVRC2012_val_00027376.JPEG:n01773157 +ILSVRC2012_val_00027377.JPEG:n02808440 +ILSVRC2012_val_00027378.JPEG:n02177972 +ILSVRC2012_val_00027379.JPEG:n03759954 +ILSVRC2012_val_00027380.JPEG:n07579787 +ILSVRC2012_val_00027381.JPEG:n02877765 +ILSVRC2012_val_00027382.JPEG:n03958227 +ILSVRC2012_val_00027383.JPEG:n03977966 +ILSVRC2012_val_00027384.JPEG:n03825788 +ILSVRC2012_val_00027385.JPEG:n03028079 +ILSVRC2012_val_00027386.JPEG:n04501370 +ILSVRC2012_val_00027387.JPEG:n02259212 +ILSVRC2012_val_00027388.JPEG:n03961711 +ILSVRC2012_val_00027389.JPEG:n03496892 +ILSVRC2012_val_00027390.JPEG:n03706229 +ILSVRC2012_val_00027391.JPEG:n04409515 +ILSVRC2012_val_00027392.JPEG:n12144580 +ILSVRC2012_val_00027393.JPEG:n03769881 +ILSVRC2012_val_00027394.JPEG:n09193705 +ILSVRC2012_val_00027395.JPEG:n02782093 +ILSVRC2012_val_00027396.JPEG:n01734418 +ILSVRC2012_val_00027397.JPEG:n04285008 +ILSVRC2012_val_00027398.JPEG:n02120505 +ILSVRC2012_val_00027399.JPEG:n02111277 +ILSVRC2012_val_00027400.JPEG:n02640242 +ILSVRC2012_val_00027401.JPEG:n02790996 +ILSVRC2012_val_00027402.JPEG:n02099267 +ILSVRC2012_val_00027403.JPEG:n07871810 +ILSVRC2012_val_00027404.JPEG:n01986214 +ILSVRC2012_val_00027405.JPEG:n01984695 +ILSVRC2012_val_00027406.JPEG:n12985857 +ILSVRC2012_val_00027407.JPEG:n04542943 +ILSVRC2012_val_00027408.JPEG:n03888605 +ILSVRC2012_val_00027409.JPEG:n04074963 +ILSVRC2012_val_00027410.JPEG:n10565667 +ILSVRC2012_val_00027411.JPEG:n04483307 +ILSVRC2012_val_00027412.JPEG:n09835506 +ILSVRC2012_val_00027413.JPEG:n02129165 +ILSVRC2012_val_00027414.JPEG:n03538406 +ILSVRC2012_val_00027415.JPEG:n01498041 +ILSVRC2012_val_00027416.JPEG:n04461696 +ILSVRC2012_val_00027417.JPEG:n03944341 +ILSVRC2012_val_00027418.JPEG:n03259280 +ILSVRC2012_val_00027419.JPEG:n01484850 +ILSVRC2012_val_00027420.JPEG:n04486054 +ILSVRC2012_val_00027421.JPEG:n03788195 +ILSVRC2012_val_00027422.JPEG:n09193705 +ILSVRC2012_val_00027423.JPEG:n03530642 +ILSVRC2012_val_00027424.JPEG:n04557648 +ILSVRC2012_val_00027425.JPEG:n02892201 +ILSVRC2012_val_00027426.JPEG:n04509417 +ILSVRC2012_val_00027427.JPEG:n03041632 +ILSVRC2012_val_00027428.JPEG:n02093256 +ILSVRC2012_val_00027429.JPEG:n02391049 +ILSVRC2012_val_00027430.JPEG:n04479046 +ILSVRC2012_val_00027431.JPEG:n03961711 +ILSVRC2012_val_00027432.JPEG:n15075141 +ILSVRC2012_val_00027433.JPEG:n02108915 +ILSVRC2012_val_00027434.JPEG:n01847000 +ILSVRC2012_val_00027435.JPEG:n02325366 +ILSVRC2012_val_00027436.JPEG:n03770439 +ILSVRC2012_val_00027437.JPEG:n03676483 +ILSVRC2012_val_00027438.JPEG:n06794110 +ILSVRC2012_val_00027439.JPEG:n01770393 +ILSVRC2012_val_00027440.JPEG:n02788148 +ILSVRC2012_val_00027441.JPEG:n03127925 +ILSVRC2012_val_00027442.JPEG:n03710721 +ILSVRC2012_val_00027443.JPEG:n02484975 +ILSVRC2012_val_00027444.JPEG:n02536864 +ILSVRC2012_val_00027445.JPEG:n02105855 +ILSVRC2012_val_00027446.JPEG:n03733131 +ILSVRC2012_val_00027447.JPEG:n04435653 +ILSVRC2012_val_00027448.JPEG:n02124075 +ILSVRC2012_val_00027449.JPEG:n03792782 +ILSVRC2012_val_00027450.JPEG:n04465501 +ILSVRC2012_val_00027451.JPEG:n01644373 +ILSVRC2012_val_00027452.JPEG:n02085620 +ILSVRC2012_val_00027453.JPEG:n03720891 +ILSVRC2012_val_00027454.JPEG:n03814639 +ILSVRC2012_val_00027455.JPEG:n03133878 +ILSVRC2012_val_00027456.JPEG:n02892201 +ILSVRC2012_val_00027457.JPEG:n02077923 +ILSVRC2012_val_00027458.JPEG:n02992211 +ILSVRC2012_val_00027459.JPEG:n02114712 +ILSVRC2012_val_00027460.JPEG:n02410509 +ILSVRC2012_val_00027461.JPEG:n03733131 +ILSVRC2012_val_00027462.JPEG:n03843555 +ILSVRC2012_val_00027463.JPEG:n02917067 +ILSVRC2012_val_00027464.JPEG:n02128385 +ILSVRC2012_val_00027465.JPEG:n04009552 +ILSVRC2012_val_00027466.JPEG:n03888605 +ILSVRC2012_val_00027467.JPEG:n03388043 +ILSVRC2012_val_00027468.JPEG:n04596742 +ILSVRC2012_val_00027469.JPEG:n03935335 +ILSVRC2012_val_00027470.JPEG:n06785654 +ILSVRC2012_val_00027471.JPEG:n02356798 +ILSVRC2012_val_00027472.JPEG:n02398521 +ILSVRC2012_val_00027473.JPEG:n03445924 +ILSVRC2012_val_00027474.JPEG:n03041632 +ILSVRC2012_val_00027475.JPEG:n03535780 +ILSVRC2012_val_00027476.JPEG:n07753113 +ILSVRC2012_val_00027477.JPEG:n02834397 +ILSVRC2012_val_00027478.JPEG:n01824575 +ILSVRC2012_val_00027479.JPEG:n07697313 +ILSVRC2012_val_00027480.JPEG:n04487081 +ILSVRC2012_val_00027481.JPEG:n02509815 +ILSVRC2012_val_00027482.JPEG:n02106550 +ILSVRC2012_val_00027483.JPEG:n01704323 +ILSVRC2012_val_00027484.JPEG:n01742172 +ILSVRC2012_val_00027485.JPEG:n02094433 +ILSVRC2012_val_00027486.JPEG:n01817953 +ILSVRC2012_val_00027487.JPEG:n03032252 +ILSVRC2012_val_00027488.JPEG:n01742172 +ILSVRC2012_val_00027489.JPEG:n02483362 +ILSVRC2012_val_00027490.JPEG:n02096437 +ILSVRC2012_val_00027491.JPEG:n02487347 +ILSVRC2012_val_00027492.JPEG:n02096294 +ILSVRC2012_val_00027493.JPEG:n04465501 +ILSVRC2012_val_00027494.JPEG:n02948072 +ILSVRC2012_val_00027495.JPEG:n03424325 +ILSVRC2012_val_00027496.JPEG:n02111500 +ILSVRC2012_val_00027497.JPEG:n02114367 +ILSVRC2012_val_00027498.JPEG:n01537544 +ILSVRC2012_val_00027499.JPEG:n01945685 +ILSVRC2012_val_00027500.JPEG:n02607072 +ILSVRC2012_val_00027501.JPEG:n04005630 +ILSVRC2012_val_00027502.JPEG:n04127249 +ILSVRC2012_val_00027503.JPEG:n07714990 +ILSVRC2012_val_00027504.JPEG:n03662601 +ILSVRC2012_val_00027505.JPEG:n03179701 +ILSVRC2012_val_00027506.JPEG:n09468604 +ILSVRC2012_val_00027507.JPEG:n01530575 +ILSVRC2012_val_00027508.JPEG:n03100240 +ILSVRC2012_val_00027509.JPEG:n06359193 +ILSVRC2012_val_00027510.JPEG:n02510455 +ILSVRC2012_val_00027511.JPEG:n02120079 +ILSVRC2012_val_00027512.JPEG:n02096437 +ILSVRC2012_val_00027513.JPEG:n03141823 +ILSVRC2012_val_00027514.JPEG:n01484850 +ILSVRC2012_val_00027515.JPEG:n04579432 +ILSVRC2012_val_00027516.JPEG:n04118538 +ILSVRC2012_val_00027517.JPEG:n02094433 +ILSVRC2012_val_00027518.JPEG:n02086910 +ILSVRC2012_val_00027519.JPEG:n01622779 +ILSVRC2012_val_00027520.JPEG:n07747607 +ILSVRC2012_val_00027521.JPEG:n07718747 +ILSVRC2012_val_00027522.JPEG:n02106030 +ILSVRC2012_val_00027523.JPEG:n02363005 +ILSVRC2012_val_00027524.JPEG:n03599486 +ILSVRC2012_val_00027525.JPEG:n03637318 +ILSVRC2012_val_00027526.JPEG:n02101388 +ILSVRC2012_val_00027527.JPEG:n03662601 +ILSVRC2012_val_00027528.JPEG:n03188531 +ILSVRC2012_val_00027529.JPEG:n02104029 +ILSVRC2012_val_00027530.JPEG:n11939491 +ILSVRC2012_val_00027531.JPEG:n04238763 +ILSVRC2012_val_00027532.JPEG:n01945685 +ILSVRC2012_val_00027533.JPEG:n02834397 +ILSVRC2012_val_00027534.JPEG:n02099712 +ILSVRC2012_val_00027535.JPEG:n01558993 +ILSVRC2012_val_00027536.JPEG:n03450230 +ILSVRC2012_val_00027537.JPEG:n03838899 +ILSVRC2012_val_00027538.JPEG:n04243546 +ILSVRC2012_val_00027539.JPEG:n02123159 +ILSVRC2012_val_00027540.JPEG:n04536866 +ILSVRC2012_val_00027541.JPEG:n02808304 +ILSVRC2012_val_00027542.JPEG:n04120489 +ILSVRC2012_val_00027543.JPEG:n03127925 +ILSVRC2012_val_00027544.JPEG:n04505470 +ILSVRC2012_val_00027545.JPEG:n03782006 +ILSVRC2012_val_00027546.JPEG:n02281406 +ILSVRC2012_val_00027547.JPEG:n04252225 +ILSVRC2012_val_00027548.JPEG:n02776631 +ILSVRC2012_val_00027549.JPEG:n02444819 +ILSVRC2012_val_00027550.JPEG:n04005630 +ILSVRC2012_val_00027551.JPEG:n03717622 +ILSVRC2012_val_00027552.JPEG:n03961711 +ILSVRC2012_val_00027553.JPEG:n03444034 +ILSVRC2012_val_00027554.JPEG:n03970156 +ILSVRC2012_val_00027555.JPEG:n01824575 +ILSVRC2012_val_00027556.JPEG:n02396427 +ILSVRC2012_val_00027557.JPEG:n02165456 +ILSVRC2012_val_00027558.JPEG:n02226429 +ILSVRC2012_val_00027559.JPEG:n02056570 +ILSVRC2012_val_00027560.JPEG:n07693725 +ILSVRC2012_val_00027561.JPEG:n04599235 +ILSVRC2012_val_00027562.JPEG:n03944341 +ILSVRC2012_val_00027563.JPEG:n02134418 +ILSVRC2012_val_00027564.JPEG:n03788365 +ILSVRC2012_val_00027565.JPEG:n07717410 +ILSVRC2012_val_00027566.JPEG:n04264628 +ILSVRC2012_val_00027567.JPEG:n03967562 +ILSVRC2012_val_00027568.JPEG:n04265275 +ILSVRC2012_val_00027569.JPEG:n03584254 +ILSVRC2012_val_00027570.JPEG:n01614925 +ILSVRC2012_val_00027571.JPEG:n07720875 +ILSVRC2012_val_00027572.JPEG:n03814639 +ILSVRC2012_val_00027573.JPEG:n04370456 +ILSVRC2012_val_00027574.JPEG:n04037443 +ILSVRC2012_val_00027575.JPEG:n03297495 +ILSVRC2012_val_00027576.JPEG:n02129604 +ILSVRC2012_val_00027577.JPEG:n03131574 +ILSVRC2012_val_00027578.JPEG:n04243546 +ILSVRC2012_val_00027579.JPEG:n02105855 +ILSVRC2012_val_00027580.JPEG:n03895866 +ILSVRC2012_val_00027581.JPEG:n03216828 +ILSVRC2012_val_00027582.JPEG:n02317335 +ILSVRC2012_val_00027583.JPEG:n02106030 +ILSVRC2012_val_00027584.JPEG:n03661043 +ILSVRC2012_val_00027585.JPEG:n01924916 +ILSVRC2012_val_00027586.JPEG:n02165456 +ILSVRC2012_val_00027587.JPEG:n04536866 +ILSVRC2012_val_00027588.JPEG:n01616318 +ILSVRC2012_val_00027589.JPEG:n02799071 +ILSVRC2012_val_00027590.JPEG:n03788195 +ILSVRC2012_val_00027591.JPEG:n02363005 +ILSVRC2012_val_00027592.JPEG:n01924916 +ILSVRC2012_val_00027593.JPEG:n04461696 +ILSVRC2012_val_00027594.JPEG:n04270147 +ILSVRC2012_val_00027595.JPEG:n02843684 +ILSVRC2012_val_00027596.JPEG:n04258138 +ILSVRC2012_val_00027597.JPEG:n03944341 +ILSVRC2012_val_00027598.JPEG:n01737021 +ILSVRC2012_val_00027599.JPEG:n01882714 +ILSVRC2012_val_00027600.JPEG:n02817516 +ILSVRC2012_val_00027601.JPEG:n02097298 +ILSVRC2012_val_00027602.JPEG:n01843383 +ILSVRC2012_val_00027603.JPEG:n04019541 +ILSVRC2012_val_00027604.JPEG:n04118776 +ILSVRC2012_val_00027605.JPEG:n02799071 +ILSVRC2012_val_00027606.JPEG:n03967562 +ILSVRC2012_val_00027607.JPEG:n03494278 +ILSVRC2012_val_00027608.JPEG:n02229544 +ILSVRC2012_val_00027609.JPEG:n04325704 +ILSVRC2012_val_00027610.JPEG:n03967562 +ILSVRC2012_val_00027611.JPEG:n13044778 +ILSVRC2012_val_00027612.JPEG:n03344393 +ILSVRC2012_val_00027613.JPEG:n04557648 +ILSVRC2012_val_00027614.JPEG:n03447721 +ILSVRC2012_val_00027615.JPEG:n09472597 +ILSVRC2012_val_00027616.JPEG:n04118538 +ILSVRC2012_val_00027617.JPEG:n03424325 +ILSVRC2012_val_00027618.JPEG:n04599235 +ILSVRC2012_val_00027619.JPEG:n01530575 +ILSVRC2012_val_00027620.JPEG:n02835271 +ILSVRC2012_val_00027621.JPEG:n09472597 +ILSVRC2012_val_00027622.JPEG:n02092002 +ILSVRC2012_val_00027623.JPEG:n02730930 +ILSVRC2012_val_00027624.JPEG:n04599235 +ILSVRC2012_val_00027625.JPEG:n02422699 +ILSVRC2012_val_00027626.JPEG:n03657121 +ILSVRC2012_val_00027627.JPEG:n01622779 +ILSVRC2012_val_00027628.JPEG:n03903868 +ILSVRC2012_val_00027629.JPEG:n02090721 +ILSVRC2012_val_00027630.JPEG:n04443257 +ILSVRC2012_val_00027631.JPEG:n01734418 +ILSVRC2012_val_00027632.JPEG:n07714571 +ILSVRC2012_val_00027633.JPEG:n01496331 +ILSVRC2012_val_00027634.JPEG:n02264363 +ILSVRC2012_val_00027635.JPEG:n03483316 +ILSVRC2012_val_00027636.JPEG:n03742115 +ILSVRC2012_val_00027637.JPEG:n07714990 +ILSVRC2012_val_00027638.JPEG:n03590841 +ILSVRC2012_val_00027639.JPEG:n03871628 +ILSVRC2012_val_00027640.JPEG:n04311174 +ILSVRC2012_val_00027641.JPEG:n02114548 +ILSVRC2012_val_00027642.JPEG:n03255030 +ILSVRC2012_val_00027643.JPEG:n02105505 +ILSVRC2012_val_00027644.JPEG:n07579787 +ILSVRC2012_val_00027645.JPEG:n07697313 +ILSVRC2012_val_00027646.JPEG:n03400231 +ILSVRC2012_val_00027647.JPEG:n06874185 +ILSVRC2012_val_00027648.JPEG:n04591713 +ILSVRC2012_val_00027649.JPEG:n04509417 +ILSVRC2012_val_00027650.JPEG:n03255030 +ILSVRC2012_val_00027651.JPEG:n03404251 +ILSVRC2012_val_00027652.JPEG:n02268853 +ILSVRC2012_val_00027653.JPEG:n07613480 +ILSVRC2012_val_00027654.JPEG:n07768694 +ILSVRC2012_val_00027655.JPEG:n02321529 +ILSVRC2012_val_00027656.JPEG:n01818515 +ILSVRC2012_val_00027657.JPEG:n01877812 +ILSVRC2012_val_00027658.JPEG:n02895154 +ILSVRC2012_val_00027659.JPEG:n03485794 +ILSVRC2012_val_00027660.JPEG:n04553703 +ILSVRC2012_val_00027661.JPEG:n02364673 +ILSVRC2012_val_00027662.JPEG:n09229709 +ILSVRC2012_val_00027663.JPEG:n02916936 +ILSVRC2012_val_00027664.JPEG:n04235860 +ILSVRC2012_val_00027665.JPEG:n07932039 +ILSVRC2012_val_00027666.JPEG:n15075141 +ILSVRC2012_val_00027667.JPEG:n02006656 +ILSVRC2012_val_00027668.JPEG:n02487347 +ILSVRC2012_val_00027669.JPEG:n02087394 +ILSVRC2012_val_00027670.JPEG:n02480855 +ILSVRC2012_val_00027671.JPEG:n04372370 +ILSVRC2012_val_00027672.JPEG:n03733805 +ILSVRC2012_val_00027673.JPEG:n02979186 +ILSVRC2012_val_00027674.JPEG:n02033041 +ILSVRC2012_val_00027675.JPEG:n10565667 +ILSVRC2012_val_00027676.JPEG:n02006656 +ILSVRC2012_val_00027677.JPEG:n02099267 +ILSVRC2012_val_00027678.JPEG:n02108915 +ILSVRC2012_val_00027679.JPEG:n03930630 +ILSVRC2012_val_00027680.JPEG:n01728572 +ILSVRC2012_val_00027681.JPEG:n04552348 +ILSVRC2012_val_00027682.JPEG:n02090721 +ILSVRC2012_val_00027683.JPEG:n02870880 +ILSVRC2012_val_00027684.JPEG:n02951585 +ILSVRC2012_val_00027685.JPEG:n04259630 +ILSVRC2012_val_00027686.JPEG:n02328150 +ILSVRC2012_val_00027687.JPEG:n04435653 +ILSVRC2012_val_00027688.JPEG:n02843684 +ILSVRC2012_val_00027689.JPEG:n03788195 +ILSVRC2012_val_00027690.JPEG:n03887697 +ILSVRC2012_val_00027691.JPEG:n04335435 +ILSVRC2012_val_00027692.JPEG:n04228054 +ILSVRC2012_val_00027693.JPEG:n01608432 +ILSVRC2012_val_00027694.JPEG:n04355933 +ILSVRC2012_val_00027695.JPEG:n02123045 +ILSVRC2012_val_00027696.JPEG:n04589890 +ILSVRC2012_val_00027697.JPEG:n04086273 +ILSVRC2012_val_00027698.JPEG:n03832673 +ILSVRC2012_val_00027699.JPEG:n02111277 +ILSVRC2012_val_00027700.JPEG:n01704323 +ILSVRC2012_val_00027701.JPEG:n03599486 +ILSVRC2012_val_00027702.JPEG:n04254680 +ILSVRC2012_val_00027703.JPEG:n02086240 +ILSVRC2012_val_00027704.JPEG:n02817516 +ILSVRC2012_val_00027705.JPEG:n02487347 +ILSVRC2012_val_00027706.JPEG:n04592741 +ILSVRC2012_val_00027707.JPEG:n03272010 +ILSVRC2012_val_00027708.JPEG:n02018795 +ILSVRC2012_val_00027709.JPEG:n01930112 +ILSVRC2012_val_00027710.JPEG:n03223299 +ILSVRC2012_val_00027711.JPEG:n03388043 +ILSVRC2012_val_00027712.JPEG:n03888605 +ILSVRC2012_val_00027713.JPEG:n04040759 +ILSVRC2012_val_00027714.JPEG:n02169497 +ILSVRC2012_val_00027715.JPEG:n02793495 +ILSVRC2012_val_00027716.JPEG:n04376876 +ILSVRC2012_val_00027717.JPEG:n02177972 +ILSVRC2012_val_00027718.JPEG:n04485082 +ILSVRC2012_val_00027719.JPEG:n07717410 +ILSVRC2012_val_00027720.JPEG:n04081281 +ILSVRC2012_val_00027721.JPEG:n03109150 +ILSVRC2012_val_00027722.JPEG:n02090622 +ILSVRC2012_val_00027723.JPEG:n03482405 +ILSVRC2012_val_00027724.JPEG:n01664065 +ILSVRC2012_val_00027725.JPEG:n03032252 +ILSVRC2012_val_00027726.JPEG:n03355925 +ILSVRC2012_val_00027727.JPEG:n01910747 +ILSVRC2012_val_00027728.JPEG:n04536866 +ILSVRC2012_val_00027729.JPEG:n03000247 +ILSVRC2012_val_00027730.JPEG:n03527444 +ILSVRC2012_val_00027731.JPEG:n02025239 +ILSVRC2012_val_00027732.JPEG:n04254777 +ILSVRC2012_val_00027733.JPEG:n04141975 +ILSVRC2012_val_00027734.JPEG:n03793489 +ILSVRC2012_val_00027735.JPEG:n02979186 +ILSVRC2012_val_00027736.JPEG:n02127052 +ILSVRC2012_val_00027737.JPEG:n01847000 +ILSVRC2012_val_00027738.JPEG:n02328150 +ILSVRC2012_val_00027739.JPEG:n02909870 +ILSVRC2012_val_00027740.JPEG:n10565667 +ILSVRC2012_val_00027741.JPEG:n03709823 +ILSVRC2012_val_00027742.JPEG:n02992211 +ILSVRC2012_val_00027743.JPEG:n02093859 +ILSVRC2012_val_00027744.JPEG:n07747607 +ILSVRC2012_val_00027745.JPEG:n07717410 +ILSVRC2012_val_00027746.JPEG:n03249569 +ILSVRC2012_val_00027747.JPEG:n01734418 +ILSVRC2012_val_00027748.JPEG:n03944341 +ILSVRC2012_val_00027749.JPEG:n04344873 +ILSVRC2012_val_00027750.JPEG:n01677366 +ILSVRC2012_val_00027751.JPEG:n02108000 +ILSVRC2012_val_00027752.JPEG:n03876231 +ILSVRC2012_val_00027753.JPEG:n04461696 +ILSVRC2012_val_00027754.JPEG:n06596364 +ILSVRC2012_val_00027755.JPEG:n09428293 +ILSVRC2012_val_00027756.JPEG:n03482405 +ILSVRC2012_val_00027757.JPEG:n02088094 +ILSVRC2012_val_00027758.JPEG:n04136333 +ILSVRC2012_val_00027759.JPEG:n04204238 +ILSVRC2012_val_00027760.JPEG:n01697457 +ILSVRC2012_val_00027761.JPEG:n04074963 +ILSVRC2012_val_00027762.JPEG:n01514859 +ILSVRC2012_val_00027763.JPEG:n02106662 +ILSVRC2012_val_00027764.JPEG:n04252225 +ILSVRC2012_val_00027765.JPEG:n02117135 +ILSVRC2012_val_00027766.JPEG:n03476684 +ILSVRC2012_val_00027767.JPEG:n01770393 +ILSVRC2012_val_00027768.JPEG:n02795169 +ILSVRC2012_val_00027769.JPEG:n03733131 +ILSVRC2012_val_00027770.JPEG:n03676483 +ILSVRC2012_val_00027771.JPEG:n04133789 +ILSVRC2012_val_00027772.JPEG:n04435653 +ILSVRC2012_val_00027773.JPEG:n01728920 +ILSVRC2012_val_00027774.JPEG:n04033995 +ILSVRC2012_val_00027775.JPEG:n04355933 +ILSVRC2012_val_00027776.JPEG:n01675722 +ILSVRC2012_val_00027777.JPEG:n03717622 +ILSVRC2012_val_00027778.JPEG:n04428191 +ILSVRC2012_val_00027779.JPEG:n03535780 +ILSVRC2012_val_00027780.JPEG:n02105162 +ILSVRC2012_val_00027781.JPEG:n07753275 +ILSVRC2012_val_00027782.JPEG:n04483307 +ILSVRC2012_val_00027783.JPEG:n02917067 +ILSVRC2012_val_00027784.JPEG:n04118776 +ILSVRC2012_val_00027785.JPEG:n03000684 +ILSVRC2012_val_00027786.JPEG:n03000134 +ILSVRC2012_val_00027787.JPEG:n02281787 +ILSVRC2012_val_00027788.JPEG:n01770393 +ILSVRC2012_val_00027789.JPEG:n02326432 +ILSVRC2012_val_00027790.JPEG:n01753488 +ILSVRC2012_val_00027791.JPEG:n02167151 +ILSVRC2012_val_00027792.JPEG:n02808304 +ILSVRC2012_val_00027793.JPEG:n04392985 +ILSVRC2012_val_00027794.JPEG:n03197337 +ILSVRC2012_val_00027795.JPEG:n03100240 +ILSVRC2012_val_00027796.JPEG:n04286575 +ILSVRC2012_val_00027797.JPEG:n03127925 +ILSVRC2012_val_00027798.JPEG:n01945685 +ILSVRC2012_val_00027799.JPEG:n02536864 +ILSVRC2012_val_00027800.JPEG:n02799071 +ILSVRC2012_val_00027801.JPEG:n02783161 +ILSVRC2012_val_00027802.JPEG:n02346627 +ILSVRC2012_val_00027803.JPEG:n02264363 +ILSVRC2012_val_00027804.JPEG:n02088364 +ILSVRC2012_val_00027805.JPEG:n02093754 +ILSVRC2012_val_00027806.JPEG:n03617480 +ILSVRC2012_val_00027807.JPEG:n02105162 +ILSVRC2012_val_00027808.JPEG:n02966687 +ILSVRC2012_val_00027809.JPEG:n01795545 +ILSVRC2012_val_00027810.JPEG:n02091831 +ILSVRC2012_val_00027811.JPEG:n01537544 +ILSVRC2012_val_00027812.JPEG:n03041632 +ILSVRC2012_val_00027813.JPEG:n02834397 +ILSVRC2012_val_00027814.JPEG:n02699494 +ILSVRC2012_val_00027815.JPEG:n03404251 +ILSVRC2012_val_00027816.JPEG:n01860187 +ILSVRC2012_val_00027817.JPEG:n04550184 +ILSVRC2012_val_00027818.JPEG:n02992211 +ILSVRC2012_val_00027819.JPEG:n02437312 +ILSVRC2012_val_00027820.JPEG:n02098105 +ILSVRC2012_val_00027821.JPEG:n07590611 +ILSVRC2012_val_00027822.JPEG:n03527444 +ILSVRC2012_val_00027823.JPEG:n07583066 +ILSVRC2012_val_00027824.JPEG:n01748264 +ILSVRC2012_val_00027825.JPEG:n02966687 +ILSVRC2012_val_00027826.JPEG:n03803284 +ILSVRC2012_val_00027827.JPEG:n04366367 +ILSVRC2012_val_00027828.JPEG:n02119022 +ILSVRC2012_val_00027829.JPEG:n01740131 +ILSVRC2012_val_00027830.JPEG:n02099601 +ILSVRC2012_val_00027831.JPEG:n01534433 +ILSVRC2012_val_00027832.JPEG:n04606251 +ILSVRC2012_val_00027833.JPEG:n02099601 +ILSVRC2012_val_00027834.JPEG:n02488702 +ILSVRC2012_val_00027835.JPEG:n04336792 +ILSVRC2012_val_00027836.JPEG:n02391049 +ILSVRC2012_val_00027837.JPEG:n02086646 +ILSVRC2012_val_00027838.JPEG:n02086079 +ILSVRC2012_val_00027839.JPEG:n02110806 +ILSVRC2012_val_00027840.JPEG:n02110341 +ILSVRC2012_val_00027841.JPEG:n04447861 +ILSVRC2012_val_00027842.JPEG:n02119789 +ILSVRC2012_val_00027843.JPEG:n04162706 +ILSVRC2012_val_00027844.JPEG:n02259212 +ILSVRC2012_val_00027845.JPEG:n03124043 +ILSVRC2012_val_00027846.JPEG:n02101388 +ILSVRC2012_val_00027847.JPEG:n03630383 +ILSVRC2012_val_00027848.JPEG:n02980441 +ILSVRC2012_val_00027849.JPEG:n02494079 +ILSVRC2012_val_00027850.JPEG:n03602883 +ILSVRC2012_val_00027851.JPEG:n01695060 +ILSVRC2012_val_00027852.JPEG:n04141327 +ILSVRC2012_val_00027853.JPEG:n04266014 +ILSVRC2012_val_00027854.JPEG:n03047690 +ILSVRC2012_val_00027855.JPEG:n02097209 +ILSVRC2012_val_00027856.JPEG:n02113023 +ILSVRC2012_val_00027857.JPEG:n02174001 +ILSVRC2012_val_00027858.JPEG:n01669191 +ILSVRC2012_val_00027859.JPEG:n01667778 +ILSVRC2012_val_00027860.JPEG:n02096051 +ILSVRC2012_val_00027861.JPEG:n04251144 +ILSVRC2012_val_00027862.JPEG:n02112706 +ILSVRC2012_val_00027863.JPEG:n02988304 +ILSVRC2012_val_00027864.JPEG:n03461385 +ILSVRC2012_val_00027865.JPEG:n03447447 +ILSVRC2012_val_00027866.JPEG:n02077923 +ILSVRC2012_val_00027867.JPEG:n03887697 +ILSVRC2012_val_00027868.JPEG:n02342885 +ILSVRC2012_val_00027869.JPEG:n01641577 +ILSVRC2012_val_00027870.JPEG:n01616318 +ILSVRC2012_val_00027871.JPEG:n02007558 +ILSVRC2012_val_00027872.JPEG:n01698640 +ILSVRC2012_val_00027873.JPEG:n04033995 +ILSVRC2012_val_00027874.JPEG:n03804744 +ILSVRC2012_val_00027875.JPEG:n02110063 +ILSVRC2012_val_00027876.JPEG:n03355925 +ILSVRC2012_val_00027877.JPEG:n01667114 +ILSVRC2012_val_00027878.JPEG:n01914609 +ILSVRC2012_val_00027879.JPEG:n03804744 +ILSVRC2012_val_00027880.JPEG:n02669723 +ILSVRC2012_val_00027881.JPEG:n07836838 +ILSVRC2012_val_00027882.JPEG:n02412080 +ILSVRC2012_val_00027883.JPEG:n03743016 +ILSVRC2012_val_00027884.JPEG:n04336792 +ILSVRC2012_val_00027885.JPEG:n13052670 +ILSVRC2012_val_00027886.JPEG:n03791053 +ILSVRC2012_val_00027887.JPEG:n03776460 +ILSVRC2012_val_00027888.JPEG:n03017168 +ILSVRC2012_val_00027889.JPEG:n04404412 +ILSVRC2012_val_00027890.JPEG:n03777754 +ILSVRC2012_val_00027891.JPEG:n04037443 +ILSVRC2012_val_00027892.JPEG:n03796401 +ILSVRC2012_val_00027893.JPEG:n04404412 +ILSVRC2012_val_00027894.JPEG:n06596364 +ILSVRC2012_val_00027895.JPEG:n02105412 +ILSVRC2012_val_00027896.JPEG:n04023962 +ILSVRC2012_val_00027897.JPEG:n01734418 +ILSVRC2012_val_00027898.JPEG:n02328150 +ILSVRC2012_val_00027899.JPEG:n02101006 +ILSVRC2012_val_00027900.JPEG:n07684084 +ILSVRC2012_val_00027901.JPEG:n02002556 +ILSVRC2012_val_00027902.JPEG:n13133613 +ILSVRC2012_val_00027903.JPEG:n07248320 +ILSVRC2012_val_00027904.JPEG:n01753488 +ILSVRC2012_val_00027905.JPEG:n02107908 +ILSVRC2012_val_00027906.JPEG:n02123394 +ILSVRC2012_val_00027907.JPEG:n04154565 +ILSVRC2012_val_00027908.JPEG:n02504458 +ILSVRC2012_val_00027909.JPEG:n13052670 +ILSVRC2012_val_00027910.JPEG:n04008634 +ILSVRC2012_val_00027911.JPEG:n02916936 +ILSVRC2012_val_00027912.JPEG:n02107683 +ILSVRC2012_val_00027913.JPEG:n02134084 +ILSVRC2012_val_00027914.JPEG:n02443484 +ILSVRC2012_val_00027915.JPEG:n07720875 +ILSVRC2012_val_00027916.JPEG:n04493381 +ILSVRC2012_val_00027917.JPEG:n03761084 +ILSVRC2012_val_00027918.JPEG:n02102040 +ILSVRC2012_val_00027919.JPEG:n03089624 +ILSVRC2012_val_00027920.JPEG:n01985128 +ILSVRC2012_val_00027921.JPEG:n01753488 +ILSVRC2012_val_00027922.JPEG:n02137549 +ILSVRC2012_val_00027923.JPEG:n09835506 +ILSVRC2012_val_00027924.JPEG:n03443371 +ILSVRC2012_val_00027925.JPEG:n02346627 +ILSVRC2012_val_00027926.JPEG:n02002556 +ILSVRC2012_val_00027927.JPEG:n04589890 +ILSVRC2012_val_00027928.JPEG:n04562935 +ILSVRC2012_val_00027929.JPEG:n01632777 +ILSVRC2012_val_00027930.JPEG:n02317335 +ILSVRC2012_val_00027931.JPEG:n01632458 +ILSVRC2012_val_00027932.JPEG:n02493509 +ILSVRC2012_val_00027933.JPEG:n02398521 +ILSVRC2012_val_00027934.JPEG:n03970156 +ILSVRC2012_val_00027935.JPEG:n02667093 +ILSVRC2012_val_00027936.JPEG:n03825788 +ILSVRC2012_val_00027937.JPEG:n02086646 +ILSVRC2012_val_00027938.JPEG:n13044778 +ILSVRC2012_val_00027939.JPEG:n02088238 +ILSVRC2012_val_00027940.JPEG:n01776313 +ILSVRC2012_val_00027941.JPEG:n02481823 +ILSVRC2012_val_00027942.JPEG:n04423845 +ILSVRC2012_val_00027943.JPEG:n03047690 +ILSVRC2012_val_00027944.JPEG:n07749582 +ILSVRC2012_val_00027945.JPEG:n02977058 +ILSVRC2012_val_00027946.JPEG:n01796340 +ILSVRC2012_val_00027947.JPEG:n02110627 +ILSVRC2012_val_00027948.JPEG:n02910353 +ILSVRC2012_val_00027949.JPEG:n03201208 +ILSVRC2012_val_00027950.JPEG:n01728572 +ILSVRC2012_val_00027951.JPEG:n02114367 +ILSVRC2012_val_00027952.JPEG:n03980874 +ILSVRC2012_val_00027953.JPEG:n02776631 +ILSVRC2012_val_00027954.JPEG:n02165456 +ILSVRC2012_val_00027955.JPEG:n02437312 +ILSVRC2012_val_00027956.JPEG:n02364673 +ILSVRC2012_val_00027957.JPEG:n03764736 +ILSVRC2012_val_00027958.JPEG:n04041544 +ILSVRC2012_val_00027959.JPEG:n12998815 +ILSVRC2012_val_00027960.JPEG:n03388043 +ILSVRC2012_val_00027961.JPEG:n03803284 +ILSVRC2012_val_00027962.JPEG:n02113624 +ILSVRC2012_val_00027963.JPEG:n02102318 +ILSVRC2012_val_00027964.JPEG:n03424325 +ILSVRC2012_val_00027965.JPEG:n03250847 +ILSVRC2012_val_00027966.JPEG:n09288635 +ILSVRC2012_val_00027967.JPEG:n03924679 +ILSVRC2012_val_00027968.JPEG:n03956157 +ILSVRC2012_val_00027969.JPEG:n01910747 +ILSVRC2012_val_00027970.JPEG:n04560804 +ILSVRC2012_val_00027971.JPEG:n07714990 +ILSVRC2012_val_00027972.JPEG:n04542943 +ILSVRC2012_val_00027973.JPEG:n07716906 +ILSVRC2012_val_00027974.JPEG:n02128925 +ILSVRC2012_val_00027975.JPEG:n04487394 +ILSVRC2012_val_00027976.JPEG:n04399382 +ILSVRC2012_val_00027977.JPEG:n04044716 +ILSVRC2012_val_00027978.JPEG:n04465501 +ILSVRC2012_val_00027979.JPEG:n03854065 +ILSVRC2012_val_00027980.JPEG:n02398521 +ILSVRC2012_val_00027981.JPEG:n02823750 +ILSVRC2012_val_00027982.JPEG:n07583066 +ILSVRC2012_val_00027983.JPEG:n02107312 +ILSVRC2012_val_00027984.JPEG:n04584207 +ILSVRC2012_val_00027985.JPEG:n01829413 +ILSVRC2012_val_00027986.JPEG:n01833805 +ILSVRC2012_val_00027987.JPEG:n02417914 +ILSVRC2012_val_00027988.JPEG:n04081281 +ILSVRC2012_val_00027989.JPEG:n02088364 +ILSVRC2012_val_00027990.JPEG:n02113799 +ILSVRC2012_val_00027991.JPEG:n04376876 +ILSVRC2012_val_00027992.JPEG:n02093991 +ILSVRC2012_val_00027993.JPEG:n02730930 +ILSVRC2012_val_00027994.JPEG:n04133789 +ILSVRC2012_val_00027995.JPEG:n02442845 +ILSVRC2012_val_00027996.JPEG:n02018207 +ILSVRC2012_val_00027997.JPEG:n03930630 +ILSVRC2012_val_00027998.JPEG:n02910353 +ILSVRC2012_val_00027999.JPEG:n02730930 +ILSVRC2012_val_00028000.JPEG:n03776460 +ILSVRC2012_val_00028001.JPEG:n02088364 +ILSVRC2012_val_00028002.JPEG:n04264628 +ILSVRC2012_val_00028003.JPEG:n07714990 +ILSVRC2012_val_00028004.JPEG:n04461696 +ILSVRC2012_val_00028005.JPEG:n03372029 +ILSVRC2012_val_00028006.JPEG:n02090379 +ILSVRC2012_val_00028007.JPEG:n01819313 +ILSVRC2012_val_00028008.JPEG:n03657121 +ILSVRC2012_val_00028009.JPEG:n02106662 +ILSVRC2012_val_00028010.JPEG:n02109525 +ILSVRC2012_val_00028011.JPEG:n02500267 +ILSVRC2012_val_00028012.JPEG:n04376876 +ILSVRC2012_val_00028013.JPEG:n04483307 +ILSVRC2012_val_00028014.JPEG:n03843555 +ILSVRC2012_val_00028015.JPEG:n13037406 +ILSVRC2012_val_00028016.JPEG:n02097047 +ILSVRC2012_val_00028017.JPEG:n02403003 +ILSVRC2012_val_00028018.JPEG:n03290653 +ILSVRC2012_val_00028019.JPEG:n02690373 +ILSVRC2012_val_00028020.JPEG:n02536864 +ILSVRC2012_val_00028021.JPEG:n02091467 +ILSVRC2012_val_00028022.JPEG:n03843555 +ILSVRC2012_val_00028023.JPEG:n04044716 +ILSVRC2012_val_00028024.JPEG:n01537544 +ILSVRC2012_val_00028025.JPEG:n02037110 +ILSVRC2012_val_00028026.JPEG:n04146614 +ILSVRC2012_val_00028027.JPEG:n04612504 +ILSVRC2012_val_00028028.JPEG:n01484850 +ILSVRC2012_val_00028029.JPEG:n07684084 +ILSVRC2012_val_00028030.JPEG:n03220513 +ILSVRC2012_val_00028031.JPEG:n04326547 +ILSVRC2012_val_00028032.JPEG:n03127925 +ILSVRC2012_val_00028033.JPEG:n02971356 +ILSVRC2012_val_00028034.JPEG:n03476991 +ILSVRC2012_val_00028035.JPEG:n01774384 +ILSVRC2012_val_00028036.JPEG:n07565083 +ILSVRC2012_val_00028037.JPEG:n02672831 +ILSVRC2012_val_00028038.JPEG:n03967562 +ILSVRC2012_val_00028039.JPEG:n03998194 +ILSVRC2012_val_00028040.JPEG:n09229709 +ILSVRC2012_val_00028041.JPEG:n01641577 +ILSVRC2012_val_00028042.JPEG:n01682714 +ILSVRC2012_val_00028043.JPEG:n04204347 +ILSVRC2012_val_00028044.JPEG:n03160309 +ILSVRC2012_val_00028045.JPEG:n03478589 +ILSVRC2012_val_00028046.JPEG:n03792972 +ILSVRC2012_val_00028047.JPEG:n04458633 +ILSVRC2012_val_00028048.JPEG:n04392985 +ILSVRC2012_val_00028049.JPEG:n02480855 +ILSVRC2012_val_00028050.JPEG:n02099429 +ILSVRC2012_val_00028051.JPEG:n07714571 +ILSVRC2012_val_00028052.JPEG:n02098105 +ILSVRC2012_val_00028053.JPEG:n02963159 +ILSVRC2012_val_00028054.JPEG:n02777292 +ILSVRC2012_val_00028055.JPEG:n03529860 +ILSVRC2012_val_00028056.JPEG:n03706229 +ILSVRC2012_val_00028057.JPEG:n12057211 +ILSVRC2012_val_00028058.JPEG:n04612504 +ILSVRC2012_val_00028059.JPEG:n04554684 +ILSVRC2012_val_00028060.JPEG:n03590841 +ILSVRC2012_val_00028061.JPEG:n03661043 +ILSVRC2012_val_00028062.JPEG:n04065272 +ILSVRC2012_val_00028063.JPEG:n01531178 +ILSVRC2012_val_00028064.JPEG:n07614500 +ILSVRC2012_val_00028065.JPEG:n02017213 +ILSVRC2012_val_00028066.JPEG:n02859443 +ILSVRC2012_val_00028067.JPEG:n04235860 +ILSVRC2012_val_00028068.JPEG:n02256656 +ILSVRC2012_val_00028069.JPEG:n03481172 +ILSVRC2012_val_00028070.JPEG:n02110063 +ILSVRC2012_val_00028071.JPEG:n02281787 +ILSVRC2012_val_00028072.JPEG:n04579432 +ILSVRC2012_val_00028073.JPEG:n01985128 +ILSVRC2012_val_00028074.JPEG:n02363005 +ILSVRC2012_val_00028075.JPEG:n04317175 +ILSVRC2012_val_00028076.JPEG:n01737021 +ILSVRC2012_val_00028077.JPEG:n03216828 +ILSVRC2012_val_00028078.JPEG:n02095570 +ILSVRC2012_val_00028079.JPEG:n07714571 +ILSVRC2012_val_00028080.JPEG:n04525305 +ILSVRC2012_val_00028081.JPEG:n07565083 +ILSVRC2012_val_00028082.JPEG:n03494278 +ILSVRC2012_val_00028083.JPEG:n04525038 +ILSVRC2012_val_00028084.JPEG:n01494475 +ILSVRC2012_val_00028085.JPEG:n04404412 +ILSVRC2012_val_00028086.JPEG:n07718747 +ILSVRC2012_val_00028087.JPEG:n03903868 +ILSVRC2012_val_00028088.JPEG:n04376876 +ILSVRC2012_val_00028089.JPEG:n02088632 +ILSVRC2012_val_00028090.JPEG:n07720875 +ILSVRC2012_val_00028091.JPEG:n02111277 +ILSVRC2012_val_00028092.JPEG:n01728920 +ILSVRC2012_val_00028093.JPEG:n04311004 +ILSVRC2012_val_00028094.JPEG:n02877765 +ILSVRC2012_val_00028095.JPEG:n06785654 +ILSVRC2012_val_00028096.JPEG:n01978455 +ILSVRC2012_val_00028097.JPEG:n01729977 +ILSVRC2012_val_00028098.JPEG:n02906734 +ILSVRC2012_val_00028099.JPEG:n01601694 +ILSVRC2012_val_00028100.JPEG:n04429376 +ILSVRC2012_val_00028101.JPEG:n02676566 +ILSVRC2012_val_00028102.JPEG:n03733281 +ILSVRC2012_val_00028103.JPEG:n02106382 +ILSVRC2012_val_00028104.JPEG:n02817516 +ILSVRC2012_val_00028105.JPEG:n04039381 +ILSVRC2012_val_00028106.JPEG:n04356056 +ILSVRC2012_val_00028107.JPEG:n01514859 +ILSVRC2012_val_00028108.JPEG:n03791053 +ILSVRC2012_val_00028109.JPEG:n04376876 +ILSVRC2012_val_00028110.JPEG:n03630383 +ILSVRC2012_val_00028111.JPEG:n04252077 +ILSVRC2012_val_00028112.JPEG:n04417672 +ILSVRC2012_val_00028113.JPEG:n01641577 +ILSVRC2012_val_00028114.JPEG:n04141076 +ILSVRC2012_val_00028115.JPEG:n02025239 +ILSVRC2012_val_00028116.JPEG:n02992529 +ILSVRC2012_val_00028117.JPEG:n02672831 +ILSVRC2012_val_00028118.JPEG:n02088466 +ILSVRC2012_val_00028119.JPEG:n01797886 +ILSVRC2012_val_00028120.JPEG:n04501370 +ILSVRC2012_val_00028121.JPEG:n04149813 +ILSVRC2012_val_00028122.JPEG:n02172182 +ILSVRC2012_val_00028123.JPEG:n04336792 +ILSVRC2012_val_00028124.JPEG:n04417672 +ILSVRC2012_val_00028125.JPEG:n03944341 +ILSVRC2012_val_00028126.JPEG:n03961711 +ILSVRC2012_val_00028127.JPEG:n04493381 +ILSVRC2012_val_00028128.JPEG:n04258138 +ILSVRC2012_val_00028129.JPEG:n04523525 +ILSVRC2012_val_00028130.JPEG:n02423022 +ILSVRC2012_val_00028131.JPEG:n02102177 +ILSVRC2012_val_00028132.JPEG:n02865351 +ILSVRC2012_val_00028133.JPEG:n04507155 +ILSVRC2012_val_00028134.JPEG:n07930864 +ILSVRC2012_val_00028135.JPEG:n02097047 +ILSVRC2012_val_00028136.JPEG:n03916031 +ILSVRC2012_val_00028137.JPEG:n02892201 +ILSVRC2012_val_00028138.JPEG:n04254680 +ILSVRC2012_val_00028139.JPEG:n01608432 +ILSVRC2012_val_00028140.JPEG:n04461696 +ILSVRC2012_val_00028141.JPEG:n03483316 +ILSVRC2012_val_00028142.JPEG:n02500267 +ILSVRC2012_val_00028143.JPEG:n02916936 +ILSVRC2012_val_00028144.JPEG:n03452741 +ILSVRC2012_val_00028145.JPEG:n02892201 +ILSVRC2012_val_00028146.JPEG:n02113186 +ILSVRC2012_val_00028147.JPEG:n03775546 +ILSVRC2012_val_00028148.JPEG:n03478589 +ILSVRC2012_val_00028149.JPEG:n03633091 +ILSVRC2012_val_00028150.JPEG:n04599235 +ILSVRC2012_val_00028151.JPEG:n03065424 +ILSVRC2012_val_00028152.JPEG:n02097209 +ILSVRC2012_val_00028153.JPEG:n01873310 +ILSVRC2012_val_00028154.JPEG:n04604644 +ILSVRC2012_val_00028155.JPEG:n04418357 +ILSVRC2012_val_00028156.JPEG:n03794056 +ILSVRC2012_val_00028157.JPEG:n03179701 +ILSVRC2012_val_00028158.JPEG:n01440764 +ILSVRC2012_val_00028159.JPEG:n01806143 +ILSVRC2012_val_00028160.JPEG:n02093859 +ILSVRC2012_val_00028161.JPEG:n01496331 +ILSVRC2012_val_00028162.JPEG:n01669191 +ILSVRC2012_val_00028163.JPEG:n04367480 +ILSVRC2012_val_00028164.JPEG:n02971356 +ILSVRC2012_val_00028165.JPEG:n02114548 +ILSVRC2012_val_00028166.JPEG:n03249569 +ILSVRC2012_val_00028167.JPEG:n01796340 +ILSVRC2012_val_00028168.JPEG:n07613480 +ILSVRC2012_val_00028169.JPEG:n04505470 +ILSVRC2012_val_00028170.JPEG:n03804744 +ILSVRC2012_val_00028171.JPEG:n02950826 +ILSVRC2012_val_00028172.JPEG:n03743016 +ILSVRC2012_val_00028173.JPEG:n02777292 +ILSVRC2012_val_00028174.JPEG:n03089624 +ILSVRC2012_val_00028175.JPEG:n02110341 +ILSVRC2012_val_00028176.JPEG:n03485407 +ILSVRC2012_val_00028177.JPEG:n02480855 +ILSVRC2012_val_00028178.JPEG:n02356798 +ILSVRC2012_val_00028179.JPEG:n02910353 +ILSVRC2012_val_00028180.JPEG:n03662601 +ILSVRC2012_val_00028181.JPEG:n01601694 +ILSVRC2012_val_00028182.JPEG:n04141076 +ILSVRC2012_val_00028183.JPEG:n03384352 +ILSVRC2012_val_00028184.JPEG:n02492660 +ILSVRC2012_val_00028185.JPEG:n03376595 +ILSVRC2012_val_00028186.JPEG:n02776631 +ILSVRC2012_val_00028187.JPEG:n02025239 +ILSVRC2012_val_00028188.JPEG:n04065272 +ILSVRC2012_val_00028189.JPEG:n02033041 +ILSVRC2012_val_00028190.JPEG:n03417042 +ILSVRC2012_val_00028191.JPEG:n09332890 +ILSVRC2012_val_00028192.JPEG:n02097658 +ILSVRC2012_val_00028193.JPEG:n04552348 +ILSVRC2012_val_00028194.JPEG:n03447447 +ILSVRC2012_val_00028195.JPEG:n03781244 +ILSVRC2012_val_00028196.JPEG:n03000684 +ILSVRC2012_val_00028197.JPEG:n01749939 +ILSVRC2012_val_00028198.JPEG:n01677366 +ILSVRC2012_val_00028199.JPEG:n02094114 +ILSVRC2012_val_00028200.JPEG:n04465501 +ILSVRC2012_val_00028201.JPEG:n04372370 +ILSVRC2012_val_00028202.JPEG:n02281787 +ILSVRC2012_val_00028203.JPEG:n03196217 +ILSVRC2012_val_00028204.JPEG:n02277742 +ILSVRC2012_val_00028205.JPEG:n02701002 +ILSVRC2012_val_00028206.JPEG:n03290653 +ILSVRC2012_val_00028207.JPEG:n03452741 +ILSVRC2012_val_00028208.JPEG:n01806143 +ILSVRC2012_val_00028209.JPEG:n04037443 +ILSVRC2012_val_00028210.JPEG:n03825788 +ILSVRC2012_val_00028211.JPEG:n04266014 +ILSVRC2012_val_00028212.JPEG:n07716906 +ILSVRC2012_val_00028213.JPEG:n02123597 +ILSVRC2012_val_00028214.JPEG:n02110063 +ILSVRC2012_val_00028215.JPEG:n02981792 +ILSVRC2012_val_00028216.JPEG:n03804744 +ILSVRC2012_val_00028217.JPEG:n02134418 +ILSVRC2012_val_00028218.JPEG:n03970156 +ILSVRC2012_val_00028219.JPEG:n02483362 +ILSVRC2012_val_00028220.JPEG:n02486261 +ILSVRC2012_val_00028221.JPEG:n01514668 +ILSVRC2012_val_00028222.JPEG:n02134084 +ILSVRC2012_val_00028223.JPEG:n03970156 +ILSVRC2012_val_00028224.JPEG:n01558993 +ILSVRC2012_val_00028225.JPEG:n01644373 +ILSVRC2012_val_00028226.JPEG:n03692522 +ILSVRC2012_val_00028227.JPEG:n03804744 +ILSVRC2012_val_00028228.JPEG:n02804414 +ILSVRC2012_val_00028229.JPEG:n02108551 +ILSVRC2012_val_00028230.JPEG:n01560419 +ILSVRC2012_val_00028231.JPEG:n02490219 +ILSVRC2012_val_00028232.JPEG:n03710637 +ILSVRC2012_val_00028233.JPEG:n03673027 +ILSVRC2012_val_00028234.JPEG:n04552348 +ILSVRC2012_val_00028235.JPEG:n02094114 +ILSVRC2012_val_00028236.JPEG:n03967562 +ILSVRC2012_val_00028237.JPEG:n03776460 +ILSVRC2012_val_00028238.JPEG:n02447366 +ILSVRC2012_val_00028239.JPEG:n03733805 +ILSVRC2012_val_00028240.JPEG:n03127925 +ILSVRC2012_val_00028241.JPEG:n02279972 +ILSVRC2012_val_00028242.JPEG:n09428293 +ILSVRC2012_val_00028243.JPEG:n03089624 +ILSVRC2012_val_00028244.JPEG:n03938244 +ILSVRC2012_val_00028245.JPEG:n04041544 +ILSVRC2012_val_00028246.JPEG:n02113712 +ILSVRC2012_val_00028247.JPEG:n03594734 +ILSVRC2012_val_00028248.JPEG:n02206856 +ILSVRC2012_val_00028249.JPEG:n03485794 +ILSVRC2012_val_00028250.JPEG:n02256656 +ILSVRC2012_val_00028251.JPEG:n02981792 +ILSVRC2012_val_00028252.JPEG:n03347037 +ILSVRC2012_val_00028253.JPEG:n03026506 +ILSVRC2012_val_00028254.JPEG:n04356056 +ILSVRC2012_val_00028255.JPEG:n09332890 +ILSVRC2012_val_00028256.JPEG:n07565083 +ILSVRC2012_val_00028257.JPEG:n07760859 +ILSVRC2012_val_00028258.JPEG:n04286575 +ILSVRC2012_val_00028259.JPEG:n02790996 +ILSVRC2012_val_00028260.JPEG:n01873310 +ILSVRC2012_val_00028261.JPEG:n03337140 +ILSVRC2012_val_00028262.JPEG:n04483307 +ILSVRC2012_val_00028263.JPEG:n02281787 +ILSVRC2012_val_00028264.JPEG:n02114548 +ILSVRC2012_val_00028265.JPEG:n12057211 +ILSVRC2012_val_00028266.JPEG:n02971356 +ILSVRC2012_val_00028267.JPEG:n04591713 +ILSVRC2012_val_00028268.JPEG:n04371774 +ILSVRC2012_val_00028269.JPEG:n03841143 +ILSVRC2012_val_00028270.JPEG:n02229544 +ILSVRC2012_val_00028271.JPEG:n02794156 +ILSVRC2012_val_00028272.JPEG:n04270147 +ILSVRC2012_val_00028273.JPEG:n04090263 +ILSVRC2012_val_00028274.JPEG:n04592741 +ILSVRC2012_val_00028275.JPEG:n02120505 +ILSVRC2012_val_00028276.JPEG:n02120505 +ILSVRC2012_val_00028277.JPEG:n03532672 +ILSVRC2012_val_00028278.JPEG:n03062245 +ILSVRC2012_val_00028279.JPEG:n03089624 +ILSVRC2012_val_00028280.JPEG:n03710193 +ILSVRC2012_val_00028281.JPEG:n03792972 +ILSVRC2012_val_00028282.JPEG:n02085936 +ILSVRC2012_val_00028283.JPEG:n01924916 +ILSVRC2012_val_00028284.JPEG:n01692333 +ILSVRC2012_val_00028285.JPEG:n04428191 +ILSVRC2012_val_00028286.JPEG:n13044778 +ILSVRC2012_val_00028287.JPEG:n06359193 +ILSVRC2012_val_00028288.JPEG:n07693725 +ILSVRC2012_val_00028289.JPEG:n02916936 +ILSVRC2012_val_00028290.JPEG:n02488702 +ILSVRC2012_val_00028291.JPEG:n02489166 +ILSVRC2012_val_00028292.JPEG:n02102318 +ILSVRC2012_val_00028293.JPEG:n03980874 +ILSVRC2012_val_00028294.JPEG:n04265275 +ILSVRC2012_val_00028295.JPEG:n04429376 +ILSVRC2012_val_00028296.JPEG:n02480855 +ILSVRC2012_val_00028297.JPEG:n07873807 +ILSVRC2012_val_00028298.JPEG:n03478589 +ILSVRC2012_val_00028299.JPEG:n02071294 +ILSVRC2012_val_00028300.JPEG:n02097298 +ILSVRC2012_val_00028301.JPEG:n01734418 +ILSVRC2012_val_00028302.JPEG:n02123159 +ILSVRC2012_val_00028303.JPEG:n02951585 +ILSVRC2012_val_00028304.JPEG:n07714990 +ILSVRC2012_val_00028305.JPEG:n02859443 +ILSVRC2012_val_00028306.JPEG:n04447861 +ILSVRC2012_val_00028307.JPEG:n02096585 +ILSVRC2012_val_00028308.JPEG:n03902125 +ILSVRC2012_val_00028309.JPEG:n04525038 +ILSVRC2012_val_00028310.JPEG:n03028079 +ILSVRC2012_val_00028311.JPEG:n03866082 +ILSVRC2012_val_00028312.JPEG:n03891332 +ILSVRC2012_val_00028313.JPEG:n03220513 +ILSVRC2012_val_00028314.JPEG:n03207743 +ILSVRC2012_val_00028315.JPEG:n04589890 +ILSVRC2012_val_00028316.JPEG:n03871628 +ILSVRC2012_val_00028317.JPEG:n01774750 +ILSVRC2012_val_00028318.JPEG:n02125311 +ILSVRC2012_val_00028319.JPEG:n02747177 +ILSVRC2012_val_00028320.JPEG:n04153751 +ILSVRC2012_val_00028321.JPEG:n02101556 +ILSVRC2012_val_00028322.JPEG:n02095570 +ILSVRC2012_val_00028323.JPEG:n01629819 +ILSVRC2012_val_00028324.JPEG:n03042490 +ILSVRC2012_val_00028325.JPEG:n01872401 +ILSVRC2012_val_00028326.JPEG:n04311004 +ILSVRC2012_val_00028327.JPEG:n04228054 +ILSVRC2012_val_00028328.JPEG:n03983396 +ILSVRC2012_val_00028329.JPEG:n04456115 +ILSVRC2012_val_00028330.JPEG:n04070727 +ILSVRC2012_val_00028331.JPEG:n02490219 +ILSVRC2012_val_00028332.JPEG:n02093256 +ILSVRC2012_val_00028333.JPEG:n03710193 +ILSVRC2012_val_00028334.JPEG:n03742115 +ILSVRC2012_val_00028335.JPEG:n03841143 +ILSVRC2012_val_00028336.JPEG:n04285008 +ILSVRC2012_val_00028337.JPEG:n02074367 +ILSVRC2012_val_00028338.JPEG:n02526121 +ILSVRC2012_val_00028339.JPEG:n02116738 +ILSVRC2012_val_00028340.JPEG:n03666591 +ILSVRC2012_val_00028341.JPEG:n02363005 +ILSVRC2012_val_00028342.JPEG:n02910353 +ILSVRC2012_val_00028343.JPEG:n02219486 +ILSVRC2012_val_00028344.JPEG:n03063599 +ILSVRC2012_val_00028345.JPEG:n01955084 +ILSVRC2012_val_00028346.JPEG:n02104029 +ILSVRC2012_val_00028347.JPEG:n02114855 +ILSVRC2012_val_00028348.JPEG:n04023962 +ILSVRC2012_val_00028349.JPEG:n04376876 +ILSVRC2012_val_00028350.JPEG:n04275548 +ILSVRC2012_val_00028351.JPEG:n01682714 +ILSVRC2012_val_00028352.JPEG:n01641577 +ILSVRC2012_val_00028353.JPEG:n02676566 +ILSVRC2012_val_00028354.JPEG:n07892512 +ILSVRC2012_val_00028355.JPEG:n01775062 +ILSVRC2012_val_00028356.JPEG:n03457902 +ILSVRC2012_val_00028357.JPEG:n04486054 +ILSVRC2012_val_00028358.JPEG:n03457902 +ILSVRC2012_val_00028359.JPEG:n02843684 +ILSVRC2012_val_00028360.JPEG:n07768694 +ILSVRC2012_val_00028361.JPEG:n04026417 +ILSVRC2012_val_00028362.JPEG:n03355925 +ILSVRC2012_val_00028363.JPEG:n02025239 +ILSVRC2012_val_00028364.JPEG:n03781244 +ILSVRC2012_val_00028365.JPEG:n03947888 +ILSVRC2012_val_00028366.JPEG:n02280649 +ILSVRC2012_val_00028367.JPEG:n03450230 +ILSVRC2012_val_00028368.JPEG:n02098286 +ILSVRC2012_val_00028369.JPEG:n03776460 +ILSVRC2012_val_00028370.JPEG:n03594945 +ILSVRC2012_val_00028371.JPEG:n07734744 +ILSVRC2012_val_00028372.JPEG:n02276258 +ILSVRC2012_val_00028373.JPEG:n07720875 +ILSVRC2012_val_00028374.JPEG:n02988304 +ILSVRC2012_val_00028375.JPEG:n03595614 +ILSVRC2012_val_00028376.JPEG:n02951358 +ILSVRC2012_val_00028377.JPEG:n03764736 +ILSVRC2012_val_00028378.JPEG:n02939185 +ILSVRC2012_val_00028379.JPEG:n02091134 +ILSVRC2012_val_00028380.JPEG:n01978287 +ILSVRC2012_val_00028381.JPEG:n02268443 +ILSVRC2012_val_00028382.JPEG:n03127747 +ILSVRC2012_val_00028383.JPEG:n03814639 +ILSVRC2012_val_00028384.JPEG:n03874293 +ILSVRC2012_val_00028385.JPEG:n04081281 +ILSVRC2012_val_00028386.JPEG:n07768694 +ILSVRC2012_val_00028387.JPEG:n07715103 +ILSVRC2012_val_00028388.JPEG:n02790996 +ILSVRC2012_val_00028389.JPEG:n03160309 +ILSVRC2012_val_00028390.JPEG:n04525038 +ILSVRC2012_val_00028391.JPEG:n02013706 +ILSVRC2012_val_00028392.JPEG:n04540053 +ILSVRC2012_val_00028393.JPEG:n02105056 +ILSVRC2012_val_00028394.JPEG:n07715103 +ILSVRC2012_val_00028395.JPEG:n01860187 +ILSVRC2012_val_00028396.JPEG:n07920052 +ILSVRC2012_val_00028397.JPEG:n01687978 +ILSVRC2012_val_00028398.JPEG:n07590611 +ILSVRC2012_val_00028399.JPEG:n03394916 +ILSVRC2012_val_00028400.JPEG:n03947888 +ILSVRC2012_val_00028401.JPEG:n01945685 +ILSVRC2012_val_00028402.JPEG:n02110063 +ILSVRC2012_val_00028403.JPEG:n04074963 +ILSVRC2012_val_00028404.JPEG:n04606251 +ILSVRC2012_val_00028405.JPEG:n03594945 +ILSVRC2012_val_00028406.JPEG:n04254120 +ILSVRC2012_val_00028407.JPEG:n03187595 +ILSVRC2012_val_00028408.JPEG:n02110958 +ILSVRC2012_val_00028409.JPEG:n02977058 +ILSVRC2012_val_00028410.JPEG:n07930864 +ILSVRC2012_val_00028411.JPEG:n02099601 +ILSVRC2012_val_00028412.JPEG:n03590841 +ILSVRC2012_val_00028413.JPEG:n02441942 +ILSVRC2012_val_00028414.JPEG:n01806567 +ILSVRC2012_val_00028415.JPEG:n02643566 +ILSVRC2012_val_00028416.JPEG:n03874293 +ILSVRC2012_val_00028417.JPEG:n03255030 +ILSVRC2012_val_00028418.JPEG:n04487394 +ILSVRC2012_val_00028419.JPEG:n07760859 +ILSVRC2012_val_00028420.JPEG:n02112137 +ILSVRC2012_val_00028421.JPEG:n04486054 +ILSVRC2012_val_00028422.JPEG:n01496331 +ILSVRC2012_val_00028423.JPEG:n03337140 +ILSVRC2012_val_00028424.JPEG:n01882714 +ILSVRC2012_val_00028425.JPEG:n02113978 +ILSVRC2012_val_00028426.JPEG:n07615774 +ILSVRC2012_val_00028427.JPEG:n02168699 +ILSVRC2012_val_00028428.JPEG:n04465501 +ILSVRC2012_val_00028429.JPEG:n02086910 +ILSVRC2012_val_00028430.JPEG:n04136333 +ILSVRC2012_val_00028431.JPEG:n04254120 +ILSVRC2012_val_00028432.JPEG:n03530642 +ILSVRC2012_val_00028433.JPEG:n03187595 +ILSVRC2012_val_00028434.JPEG:n01770393 +ILSVRC2012_val_00028435.JPEG:n02422106 +ILSVRC2012_val_00028436.JPEG:n03709823 +ILSVRC2012_val_00028437.JPEG:n02910353 +ILSVRC2012_val_00028438.JPEG:n01855672 +ILSVRC2012_val_00028439.JPEG:n02361337 +ILSVRC2012_val_00028440.JPEG:n01580077 +ILSVRC2012_val_00028441.JPEG:n01694178 +ILSVRC2012_val_00028442.JPEG:n04120489 +ILSVRC2012_val_00028443.JPEG:n04517823 +ILSVRC2012_val_00028444.JPEG:n03775546 +ILSVRC2012_val_00028445.JPEG:n01773157 +ILSVRC2012_val_00028446.JPEG:n03775546 +ILSVRC2012_val_00028447.JPEG:n03777568 +ILSVRC2012_val_00028448.JPEG:n04355933 +ILSVRC2012_val_00028449.JPEG:n01784675 +ILSVRC2012_val_00028450.JPEG:n01498041 +ILSVRC2012_val_00028451.JPEG:n02422699 +ILSVRC2012_val_00028452.JPEG:n04447861 +ILSVRC2012_val_00028453.JPEG:n02177972 +ILSVRC2012_val_00028454.JPEG:n02319095 +ILSVRC2012_val_00028455.JPEG:n03935335 +ILSVRC2012_val_00028456.JPEG:n03980874 +ILSVRC2012_val_00028457.JPEG:n03976657 +ILSVRC2012_val_00028458.JPEG:n02442845 +ILSVRC2012_val_00028459.JPEG:n02085782 +ILSVRC2012_val_00028460.JPEG:n03976467 +ILSVRC2012_val_00028461.JPEG:n07583066 +ILSVRC2012_val_00028462.JPEG:n04461696 +ILSVRC2012_val_00028463.JPEG:n04467665 +ILSVRC2012_val_00028464.JPEG:n02105641 +ILSVRC2012_val_00028465.JPEG:n04501370 +ILSVRC2012_val_00028466.JPEG:n03777754 +ILSVRC2012_val_00028467.JPEG:n04065272 +ILSVRC2012_val_00028468.JPEG:n03447721 +ILSVRC2012_val_00028469.JPEG:n02206856 +ILSVRC2012_val_00028470.JPEG:n03459775 +ILSVRC2012_val_00028471.JPEG:n03947888 +ILSVRC2012_val_00028472.JPEG:n04111531 +ILSVRC2012_val_00028473.JPEG:n02807133 +ILSVRC2012_val_00028474.JPEG:n03481172 +ILSVRC2012_val_00028475.JPEG:n01983481 +ILSVRC2012_val_00028476.JPEG:n03733131 +ILSVRC2012_val_00028477.JPEG:n02105641 +ILSVRC2012_val_00028478.JPEG:n03841143 +ILSVRC2012_val_00028479.JPEG:n03976467 +ILSVRC2012_val_00028480.JPEG:n02391049 +ILSVRC2012_val_00028481.JPEG:n03196217 +ILSVRC2012_val_00028482.JPEG:n02422699 +ILSVRC2012_val_00028483.JPEG:n04462240 +ILSVRC2012_val_00028484.JPEG:n04328186 +ILSVRC2012_val_00028485.JPEG:n04310018 +ILSVRC2012_val_00028486.JPEG:n04417672 +ILSVRC2012_val_00028487.JPEG:n03018349 +ILSVRC2012_val_00028488.JPEG:n02965783 +ILSVRC2012_val_00028489.JPEG:n01629819 +ILSVRC2012_val_00028490.JPEG:n03207941 +ILSVRC2012_val_00028491.JPEG:n04311174 +ILSVRC2012_val_00028492.JPEG:n02226429 +ILSVRC2012_val_00028493.JPEG:n02363005 +ILSVRC2012_val_00028494.JPEG:n03041632 +ILSVRC2012_val_00028495.JPEG:n04033901 +ILSVRC2012_val_00028496.JPEG:n02410509 +ILSVRC2012_val_00028497.JPEG:n02112137 +ILSVRC2012_val_00028498.JPEG:n02747177 +ILSVRC2012_val_00028499.JPEG:n02825657 +ILSVRC2012_val_00028500.JPEG:n02097298 +ILSVRC2012_val_00028501.JPEG:n02992529 +ILSVRC2012_val_00028502.JPEG:n03032252 +ILSVRC2012_val_00028503.JPEG:n01734418 +ILSVRC2012_val_00028504.JPEG:n04090263 +ILSVRC2012_val_00028505.JPEG:n04201297 +ILSVRC2012_val_00028506.JPEG:n02094258 +ILSVRC2012_val_00028507.JPEG:n04111531 +ILSVRC2012_val_00028508.JPEG:n04265275 +ILSVRC2012_val_00028509.JPEG:n04065272 +ILSVRC2012_val_00028510.JPEG:n02676566 +ILSVRC2012_val_00028511.JPEG:n03388043 +ILSVRC2012_val_00028512.JPEG:n07930864 +ILSVRC2012_val_00028513.JPEG:n02423022 +ILSVRC2012_val_00028514.JPEG:n02108551 +ILSVRC2012_val_00028515.JPEG:n03424325 +ILSVRC2012_val_00028516.JPEG:n02815834 +ILSVRC2012_val_00028517.JPEG:n04228054 +ILSVRC2012_val_00028518.JPEG:n02097209 +ILSVRC2012_val_00028519.JPEG:n02137549 +ILSVRC2012_val_00028520.JPEG:n03314780 +ILSVRC2012_val_00028521.JPEG:n01608432 +ILSVRC2012_val_00028522.JPEG:n01820546 +ILSVRC2012_val_00028523.JPEG:n02109961 +ILSVRC2012_val_00028524.JPEG:n01580077 +ILSVRC2012_val_00028525.JPEG:n07579787 +ILSVRC2012_val_00028526.JPEG:n03788365 +ILSVRC2012_val_00028527.JPEG:n02749479 +ILSVRC2012_val_00028528.JPEG:n03930313 +ILSVRC2012_val_00028529.JPEG:n01806567 +ILSVRC2012_val_00028530.JPEG:n02927161 +ILSVRC2012_val_00028531.JPEG:n04447861 +ILSVRC2012_val_00028532.JPEG:n04548362 +ILSVRC2012_val_00028533.JPEG:n02259212 +ILSVRC2012_val_00028534.JPEG:n04252225 +ILSVRC2012_val_00028535.JPEG:n02105162 +ILSVRC2012_val_00028536.JPEG:n03345487 +ILSVRC2012_val_00028537.JPEG:n02727426 +ILSVRC2012_val_00028538.JPEG:n07584110 +ILSVRC2012_val_00028539.JPEG:n04005630 +ILSVRC2012_val_00028540.JPEG:n02096294 +ILSVRC2012_val_00028541.JPEG:n04273569 +ILSVRC2012_val_00028542.JPEG:n02422106 +ILSVRC2012_val_00028543.JPEG:n03534580 +ILSVRC2012_val_00028544.JPEG:n09288635 +ILSVRC2012_val_00028545.JPEG:n01795545 +ILSVRC2012_val_00028546.JPEG:n02397096 +ILSVRC2012_val_00028547.JPEG:n02730930 +ILSVRC2012_val_00028548.JPEG:n01806143 +ILSVRC2012_val_00028549.JPEG:n03661043 +ILSVRC2012_val_00028550.JPEG:n02807133 +ILSVRC2012_val_00028551.JPEG:n02277742 +ILSVRC2012_val_00028552.JPEG:n07613480 +ILSVRC2012_val_00028553.JPEG:n03297495 +ILSVRC2012_val_00028554.JPEG:n03761084 +ILSVRC2012_val_00028555.JPEG:n03109150 +ILSVRC2012_val_00028556.JPEG:n07716906 +ILSVRC2012_val_00028557.JPEG:n12267677 +ILSVRC2012_val_00028558.JPEG:n04204238 +ILSVRC2012_val_00028559.JPEG:n04204347 +ILSVRC2012_val_00028560.JPEG:n04596742 +ILSVRC2012_val_00028561.JPEG:n03710637 +ILSVRC2012_val_00028562.JPEG:n02481823 +ILSVRC2012_val_00028563.JPEG:n02669723 +ILSVRC2012_val_00028564.JPEG:n01491361 +ILSVRC2012_val_00028565.JPEG:n01629819 +ILSVRC2012_val_00028566.JPEG:n03982430 +ILSVRC2012_val_00028567.JPEG:n02869837 +ILSVRC2012_val_00028568.JPEG:n01843065 +ILSVRC2012_val_00028569.JPEG:n04311174 +ILSVRC2012_val_00028570.JPEG:n01820546 +ILSVRC2012_val_00028571.JPEG:n01677366 +ILSVRC2012_val_00028572.JPEG:n02108089 +ILSVRC2012_val_00028573.JPEG:n01807496 +ILSVRC2012_val_00028574.JPEG:n03710721 +ILSVRC2012_val_00028575.JPEG:n03063599 +ILSVRC2012_val_00028576.JPEG:n03498962 +ILSVRC2012_val_00028577.JPEG:n01729322 +ILSVRC2012_val_00028578.JPEG:n02769748 +ILSVRC2012_val_00028579.JPEG:n02268853 +ILSVRC2012_val_00028580.JPEG:n04081281 +ILSVRC2012_val_00028581.JPEG:n03983396 +ILSVRC2012_val_00028582.JPEG:n06359193 +ILSVRC2012_val_00028583.JPEG:n02127052 +ILSVRC2012_val_00028584.JPEG:n02107142 +ILSVRC2012_val_00028585.JPEG:n02488702 +ILSVRC2012_val_00028586.JPEG:n02006656 +ILSVRC2012_val_00028587.JPEG:n07831146 +ILSVRC2012_val_00028588.JPEG:n02676566 +ILSVRC2012_val_00028589.JPEG:n04277352 +ILSVRC2012_val_00028590.JPEG:n03527444 +ILSVRC2012_val_00028591.JPEG:n03372029 +ILSVRC2012_val_00028592.JPEG:n03314780 +ILSVRC2012_val_00028593.JPEG:n02114712 +ILSVRC2012_val_00028594.JPEG:n01978287 +ILSVRC2012_val_00028595.JPEG:n03337140 +ILSVRC2012_val_00028596.JPEG:n03538406 +ILSVRC2012_val_00028597.JPEG:n02917067 +ILSVRC2012_val_00028598.JPEG:n01756291 +ILSVRC2012_val_00028599.JPEG:n01667778 +ILSVRC2012_val_00028600.JPEG:n01795545 +ILSVRC2012_val_00028601.JPEG:n01631663 +ILSVRC2012_val_00028602.JPEG:n02088364 +ILSVRC2012_val_00028603.JPEG:n02808304 +ILSVRC2012_val_00028604.JPEG:n01797886 +ILSVRC2012_val_00028605.JPEG:n02104029 +ILSVRC2012_val_00028606.JPEG:n03201208 +ILSVRC2012_val_00028607.JPEG:n01558993 +ILSVRC2012_val_00028608.JPEG:n03967562 +ILSVRC2012_val_00028609.JPEG:n04428191 +ILSVRC2012_val_00028610.JPEG:n02494079 +ILSVRC2012_val_00028611.JPEG:n04162706 +ILSVRC2012_val_00028612.JPEG:n04515003 +ILSVRC2012_val_00028613.JPEG:n04040759 +ILSVRC2012_val_00028614.JPEG:n01774750 +ILSVRC2012_val_00028615.JPEG:n01943899 +ILSVRC2012_val_00028616.JPEG:n02098413 +ILSVRC2012_val_00028617.JPEG:n02099601 +ILSVRC2012_val_00028618.JPEG:n04270147 +ILSVRC2012_val_00028619.JPEG:n02417914 +ILSVRC2012_val_00028620.JPEG:n03065424 +ILSVRC2012_val_00028621.JPEG:n07734744 +ILSVRC2012_val_00028622.JPEG:n02007558 +ILSVRC2012_val_00028623.JPEG:n02119789 +ILSVRC2012_val_00028624.JPEG:n07695742 +ILSVRC2012_val_00028625.JPEG:n02364673 +ILSVRC2012_val_00028626.JPEG:n01689811 +ILSVRC2012_val_00028627.JPEG:n02672831 +ILSVRC2012_val_00028628.JPEG:n02124075 +ILSVRC2012_val_00028629.JPEG:n01644900 +ILSVRC2012_val_00028630.JPEG:n04335435 +ILSVRC2012_val_00028631.JPEG:n02086646 +ILSVRC2012_val_00028632.JPEG:n02095889 +ILSVRC2012_val_00028633.JPEG:n02105251 +ILSVRC2012_val_00028634.JPEG:n02391049 +ILSVRC2012_val_00028635.JPEG:n01955084 +ILSVRC2012_val_00028636.JPEG:n02480495 +ILSVRC2012_val_00028637.JPEG:n03032252 +ILSVRC2012_val_00028638.JPEG:n02808440 +ILSVRC2012_val_00028639.JPEG:n03637318 +ILSVRC2012_val_00028640.JPEG:n02877765 +ILSVRC2012_val_00028641.JPEG:n04597913 +ILSVRC2012_val_00028642.JPEG:n02112706 +ILSVRC2012_val_00028643.JPEG:n04590129 +ILSVRC2012_val_00028644.JPEG:n01910747 +ILSVRC2012_val_00028645.JPEG:n02895154 +ILSVRC2012_val_00028646.JPEG:n03062245 +ILSVRC2012_val_00028647.JPEG:n03775546 +ILSVRC2012_val_00028648.JPEG:n03372029 +ILSVRC2012_val_00028649.JPEG:n04228054 +ILSVRC2012_val_00028650.JPEG:n04258138 +ILSVRC2012_val_00028651.JPEG:n04074963 +ILSVRC2012_val_00028652.JPEG:n11879895 +ILSVRC2012_val_00028653.JPEG:n01986214 +ILSVRC2012_val_00028654.JPEG:n01943899 +ILSVRC2012_val_00028655.JPEG:n02138441 +ILSVRC2012_val_00028656.JPEG:n01806143 +ILSVRC2012_val_00028657.JPEG:n01983481 +ILSVRC2012_val_00028658.JPEG:n03478589 +ILSVRC2012_val_00028659.JPEG:n04389033 +ILSVRC2012_val_00028660.JPEG:n02951358 +ILSVRC2012_val_00028661.JPEG:n02102318 +ILSVRC2012_val_00028662.JPEG:n03763968 +ILSVRC2012_val_00028663.JPEG:n03594734 +ILSVRC2012_val_00028664.JPEG:n01689811 +ILSVRC2012_val_00028665.JPEG:n07753113 +ILSVRC2012_val_00028666.JPEG:n02074367 +ILSVRC2012_val_00028667.JPEG:n01819313 +ILSVRC2012_val_00028668.JPEG:n03467068 +ILSVRC2012_val_00028669.JPEG:n03393912 +ILSVRC2012_val_00028670.JPEG:n02056570 +ILSVRC2012_val_00028671.JPEG:n04008634 +ILSVRC2012_val_00028672.JPEG:n04254777 +ILSVRC2012_val_00028673.JPEG:n01644900 +ILSVRC2012_val_00028674.JPEG:n02106166 +ILSVRC2012_val_00028675.JPEG:n03891251 +ILSVRC2012_val_00028676.JPEG:n04435653 +ILSVRC2012_val_00028677.JPEG:n01773549 +ILSVRC2012_val_00028678.JPEG:n03729826 +ILSVRC2012_val_00028679.JPEG:n01770081 +ILSVRC2012_val_00028680.JPEG:n03529860 +ILSVRC2012_val_00028681.JPEG:n03110669 +ILSVRC2012_val_00028682.JPEG:n03841143 +ILSVRC2012_val_00028683.JPEG:n02091244 +ILSVRC2012_val_00028684.JPEG:n04067472 +ILSVRC2012_val_00028685.JPEG:n04371430 +ILSVRC2012_val_00028686.JPEG:n03796401 +ILSVRC2012_val_00028687.JPEG:n03782006 +ILSVRC2012_val_00028688.JPEG:n04238763 +ILSVRC2012_val_00028689.JPEG:n01784675 +ILSVRC2012_val_00028690.JPEG:n04019541 +ILSVRC2012_val_00028691.JPEG:n02097209 +ILSVRC2012_val_00028692.JPEG:n02259212 +ILSVRC2012_val_00028693.JPEG:n03956157 +ILSVRC2012_val_00028694.JPEG:n02112706 +ILSVRC2012_val_00028695.JPEG:n02111889 +ILSVRC2012_val_00028696.JPEG:n03527444 +ILSVRC2012_val_00028697.JPEG:n02167151 +ILSVRC2012_val_00028698.JPEG:n04442312 +ILSVRC2012_val_00028699.JPEG:n07695742 +ILSVRC2012_val_00028700.JPEG:n03710193 +ILSVRC2012_val_00028701.JPEG:n04074963 +ILSVRC2012_val_00028702.JPEG:n02099849 +ILSVRC2012_val_00028703.JPEG:n02134418 +ILSVRC2012_val_00028704.JPEG:n02825657 +ILSVRC2012_val_00028705.JPEG:n13037406 +ILSVRC2012_val_00028706.JPEG:n02085782 +ILSVRC2012_val_00028707.JPEG:n02417914 +ILSVRC2012_val_00028708.JPEG:n12620546 +ILSVRC2012_val_00028709.JPEG:n04275548 +ILSVRC2012_val_00028710.JPEG:n02804610 +ILSVRC2012_val_00028711.JPEG:n04146614 +ILSVRC2012_val_00028712.JPEG:n01514668 +ILSVRC2012_val_00028713.JPEG:n01443537 +ILSVRC2012_val_00028714.JPEG:n04509417 +ILSVRC2012_val_00028715.JPEG:n02892201 +ILSVRC2012_val_00028716.JPEG:n02088466 +ILSVRC2012_val_00028717.JPEG:n03065424 +ILSVRC2012_val_00028718.JPEG:n04254120 +ILSVRC2012_val_00028719.JPEG:n03792972 +ILSVRC2012_val_00028720.JPEG:n01924916 +ILSVRC2012_val_00028721.JPEG:n02037110 +ILSVRC2012_val_00028722.JPEG:n07697537 +ILSVRC2012_val_00028723.JPEG:n03394916 +ILSVRC2012_val_00028724.JPEG:n02101006 +ILSVRC2012_val_00028725.JPEG:n02110806 +ILSVRC2012_val_00028726.JPEG:n03146219 +ILSVRC2012_val_00028727.JPEG:n02814860 +ILSVRC2012_val_00028728.JPEG:n03649909 +ILSVRC2012_val_00028729.JPEG:n03127747 +ILSVRC2012_val_00028730.JPEG:n01980166 +ILSVRC2012_val_00028731.JPEG:n02092002 +ILSVRC2012_val_00028732.JPEG:n03787032 +ILSVRC2012_val_00028733.JPEG:n02133161 +ILSVRC2012_val_00028734.JPEG:n03874599 +ILSVRC2012_val_00028735.JPEG:n04201297 +ILSVRC2012_val_00028736.JPEG:n02106550 +ILSVRC2012_val_00028737.JPEG:n07615774 +ILSVRC2012_val_00028738.JPEG:n03710637 +ILSVRC2012_val_00028739.JPEG:n03527444 +ILSVRC2012_val_00028740.JPEG:n07714990 +ILSVRC2012_val_00028741.JPEG:n03017168 +ILSVRC2012_val_00028742.JPEG:n02111500 +ILSVRC2012_val_00028743.JPEG:n01744401 +ILSVRC2012_val_00028744.JPEG:n03950228 +ILSVRC2012_val_00028745.JPEG:n02410509 +ILSVRC2012_val_00028746.JPEG:n02483708 +ILSVRC2012_val_00028747.JPEG:n07583066 +ILSVRC2012_val_00028748.JPEG:n04589890 +ILSVRC2012_val_00028749.JPEG:n02655020 +ILSVRC2012_val_00028750.JPEG:n02259212 +ILSVRC2012_val_00028751.JPEG:n01990800 +ILSVRC2012_val_00028752.JPEG:n03457902 +ILSVRC2012_val_00028753.JPEG:n07920052 +ILSVRC2012_val_00028754.JPEG:n04505470 +ILSVRC2012_val_00028755.JPEG:n02111129 +ILSVRC2012_val_00028756.JPEG:n03216828 +ILSVRC2012_val_00028757.JPEG:n02892767 +ILSVRC2012_val_00028758.JPEG:n02095314 +ILSVRC2012_val_00028759.JPEG:n02092002 +ILSVRC2012_val_00028760.JPEG:n01664065 +ILSVRC2012_val_00028761.JPEG:n03944341 +ILSVRC2012_val_00028762.JPEG:n03495258 +ILSVRC2012_val_00028763.JPEG:n01737021 +ILSVRC2012_val_00028764.JPEG:n01677366 +ILSVRC2012_val_00028765.JPEG:n01806567 +ILSVRC2012_val_00028766.JPEG:n02097298 +ILSVRC2012_val_00028767.JPEG:n04532670 +ILSVRC2012_val_00028768.JPEG:n04522168 +ILSVRC2012_val_00028769.JPEG:n02708093 +ILSVRC2012_val_00028770.JPEG:n02066245 +ILSVRC2012_val_00028771.JPEG:n02971356 +ILSVRC2012_val_00028772.JPEG:n02906734 +ILSVRC2012_val_00028773.JPEG:n03492542 +ILSVRC2012_val_00028774.JPEG:n03930313 +ILSVRC2012_val_00028775.JPEG:n02396427 +ILSVRC2012_val_00028776.JPEG:n02037110 +ILSVRC2012_val_00028777.JPEG:n03297495 +ILSVRC2012_val_00028778.JPEG:n03017168 +ILSVRC2012_val_00028779.JPEG:n01773797 +ILSVRC2012_val_00028780.JPEG:n03786901 +ILSVRC2012_val_00028781.JPEG:n02910353 +ILSVRC2012_val_00028782.JPEG:n02102177 +ILSVRC2012_val_00028783.JPEG:n02730930 +ILSVRC2012_val_00028784.JPEG:n02480495 +ILSVRC2012_val_00028785.JPEG:n04562935 +ILSVRC2012_val_00028786.JPEG:n02109525 +ILSVRC2012_val_00028787.JPEG:n02988304 +ILSVRC2012_val_00028788.JPEG:n02091467 +ILSVRC2012_val_00028789.JPEG:n04204238 +ILSVRC2012_val_00028790.JPEG:n04476259 +ILSVRC2012_val_00028791.JPEG:n01532829 +ILSVRC2012_val_00028792.JPEG:n03208938 +ILSVRC2012_val_00028793.JPEG:n04532106 +ILSVRC2012_val_00028794.JPEG:n02165105 +ILSVRC2012_val_00028795.JPEG:n01677366 +ILSVRC2012_val_00028796.JPEG:n07715103 +ILSVRC2012_val_00028797.JPEG:n02795169 +ILSVRC2012_val_00028798.JPEG:n02127052 +ILSVRC2012_val_00028799.JPEG:n02098286 +ILSVRC2012_val_00028800.JPEG:n01728572 +ILSVRC2012_val_00028801.JPEG:n01833805 +ILSVRC2012_val_00028802.JPEG:n02445715 +ILSVRC2012_val_00028803.JPEG:n02259212 +ILSVRC2012_val_00028804.JPEG:n04209133 +ILSVRC2012_val_00028805.JPEG:n07711569 +ILSVRC2012_val_00028806.JPEG:n07860988 +ILSVRC2012_val_00028807.JPEG:n09421951 +ILSVRC2012_val_00028808.JPEG:n03125729 +ILSVRC2012_val_00028809.JPEG:n04141076 +ILSVRC2012_val_00028810.JPEG:n01742172 +ILSVRC2012_val_00028811.JPEG:n03063689 +ILSVRC2012_val_00028812.JPEG:n01704323 +ILSVRC2012_val_00028813.JPEG:n01748264 +ILSVRC2012_val_00028814.JPEG:n01770393 +ILSVRC2012_val_00028815.JPEG:n01955084 +ILSVRC2012_val_00028816.JPEG:n02894605 +ILSVRC2012_val_00028817.JPEG:n03792972 +ILSVRC2012_val_00028818.JPEG:n04141975 +ILSVRC2012_val_00028819.JPEG:n02672831 +ILSVRC2012_val_00028820.JPEG:n03018349 +ILSVRC2012_val_00028821.JPEG:n02971356 +ILSVRC2012_val_00028822.JPEG:n02859443 +ILSVRC2012_val_00028823.JPEG:n07749582 +ILSVRC2012_val_00028824.JPEG:n03792782 +ILSVRC2012_val_00028825.JPEG:n02398521 +ILSVRC2012_val_00028826.JPEG:n04254777 +ILSVRC2012_val_00028827.JPEG:n02326432 +ILSVRC2012_val_00028828.JPEG:n03877472 +ILSVRC2012_val_00028829.JPEG:n02123045 +ILSVRC2012_val_00028830.JPEG:n03623198 +ILSVRC2012_val_00028831.JPEG:n02342885 +ILSVRC2012_val_00028832.JPEG:n03187595 +ILSVRC2012_val_00028833.JPEG:n03884397 +ILSVRC2012_val_00028834.JPEG:n04330267 +ILSVRC2012_val_00028835.JPEG:n04266014 +ILSVRC2012_val_00028836.JPEG:n02138441 +ILSVRC2012_val_00028837.JPEG:n03538406 +ILSVRC2012_val_00028838.JPEG:n03000247 +ILSVRC2012_val_00028839.JPEG:n02363005 +ILSVRC2012_val_00028840.JPEG:n02883205 +ILSVRC2012_val_00028841.JPEG:n07753592 +ILSVRC2012_val_00028842.JPEG:n04371430 +ILSVRC2012_val_00028843.JPEG:n03871628 +ILSVRC2012_val_00028844.JPEG:n03633091 +ILSVRC2012_val_00028845.JPEG:n04023962 +ILSVRC2012_val_00028846.JPEG:n01740131 +ILSVRC2012_val_00028847.JPEG:n04251144 +ILSVRC2012_val_00028848.JPEG:n02870880 +ILSVRC2012_val_00028849.JPEG:n02009912 +ILSVRC2012_val_00028850.JPEG:n03461385 +ILSVRC2012_val_00028851.JPEG:n02328150 +ILSVRC2012_val_00028852.JPEG:n01945685 +ILSVRC2012_val_00028853.JPEG:n02280649 +ILSVRC2012_val_00028854.JPEG:n02012849 +ILSVRC2012_val_00028855.JPEG:n02112137 +ILSVRC2012_val_00028856.JPEG:n04326547 +ILSVRC2012_val_00028857.JPEG:n02117135 +ILSVRC2012_val_00028858.JPEG:n07930864 +ILSVRC2012_val_00028859.JPEG:n04136333 +ILSVRC2012_val_00028860.JPEG:n04370456 +ILSVRC2012_val_00028861.JPEG:n01737021 +ILSVRC2012_val_00028862.JPEG:n01817953 +ILSVRC2012_val_00028863.JPEG:n03888605 +ILSVRC2012_val_00028864.JPEG:n03452741 +ILSVRC2012_val_00028865.JPEG:n04330267 +ILSVRC2012_val_00028866.JPEG:n07932039 +ILSVRC2012_val_00028867.JPEG:n02398521 +ILSVRC2012_val_00028868.JPEG:n07930864 +ILSVRC2012_val_00028869.JPEG:n03787032 +ILSVRC2012_val_00028870.JPEG:n02112350 +ILSVRC2012_val_00028871.JPEG:n12267677 +ILSVRC2012_val_00028872.JPEG:n03494278 +ILSVRC2012_val_00028873.JPEG:n07693725 +ILSVRC2012_val_00028874.JPEG:n03857828 +ILSVRC2012_val_00028875.JPEG:n02815834 +ILSVRC2012_val_00028876.JPEG:n04376876 +ILSVRC2012_val_00028877.JPEG:n03874293 +ILSVRC2012_val_00028878.JPEG:n04371774 +ILSVRC2012_val_00028879.JPEG:n03929855 +ILSVRC2012_val_00028880.JPEG:n02841315 +ILSVRC2012_val_00028881.JPEG:n02090721 +ILSVRC2012_val_00028882.JPEG:n09468604 +ILSVRC2012_val_00028883.JPEG:n02488291 +ILSVRC2012_val_00028884.JPEG:n02106662 +ILSVRC2012_val_00028885.JPEG:n03461385 +ILSVRC2012_val_00028886.JPEG:n04485082 +ILSVRC2012_val_00028887.JPEG:n03995372 +ILSVRC2012_val_00028888.JPEG:n02493793 +ILSVRC2012_val_00028889.JPEG:n01914609 +ILSVRC2012_val_00028890.JPEG:n02002556 +ILSVRC2012_val_00028891.JPEG:n07711569 +ILSVRC2012_val_00028892.JPEG:n02098286 +ILSVRC2012_val_00028893.JPEG:n07693725 +ILSVRC2012_val_00028894.JPEG:n02422106 +ILSVRC2012_val_00028895.JPEG:n02110958 +ILSVRC2012_val_00028896.JPEG:n04613696 +ILSVRC2012_val_00028897.JPEG:n03692522 +ILSVRC2012_val_00028898.JPEG:n07920052 +ILSVRC2012_val_00028899.JPEG:n02799071 +ILSVRC2012_val_00028900.JPEG:n04037443 +ILSVRC2012_val_00028901.JPEG:n02113978 +ILSVRC2012_val_00028902.JPEG:n01530575 +ILSVRC2012_val_00028903.JPEG:n10565667 +ILSVRC2012_val_00028904.JPEG:n10148035 +ILSVRC2012_val_00028905.JPEG:n03773504 +ILSVRC2012_val_00028906.JPEG:n03347037 +ILSVRC2012_val_00028907.JPEG:n09193705 +ILSVRC2012_val_00028908.JPEG:n02113978 +ILSVRC2012_val_00028909.JPEG:n01882714 +ILSVRC2012_val_00028910.JPEG:n03527444 +ILSVRC2012_val_00028911.JPEG:n02979186 +ILSVRC2012_val_00028912.JPEG:n01877812 +ILSVRC2012_val_00028913.JPEG:n02111129 +ILSVRC2012_val_00028914.JPEG:n03417042 +ILSVRC2012_val_00028915.JPEG:n03461385 +ILSVRC2012_val_00028916.JPEG:n02114855 +ILSVRC2012_val_00028917.JPEG:n12768682 +ILSVRC2012_val_00028918.JPEG:n01950731 +ILSVRC2012_val_00028919.JPEG:n02667093 +ILSVRC2012_val_00028920.JPEG:n02011460 +ILSVRC2012_val_00028921.JPEG:n03290653 +ILSVRC2012_val_00028922.JPEG:n02108000 +ILSVRC2012_val_00028923.JPEG:n04229816 +ILSVRC2012_val_00028924.JPEG:n01930112 +ILSVRC2012_val_00028925.JPEG:n02486261 +ILSVRC2012_val_00028926.JPEG:n04542943 +ILSVRC2012_val_00028927.JPEG:n04235860 +ILSVRC2012_val_00028928.JPEG:n07768694 +ILSVRC2012_val_00028929.JPEG:n02403003 +ILSVRC2012_val_00028930.JPEG:n03786901 +ILSVRC2012_val_00028931.JPEG:n02396427 +ILSVRC2012_val_00028932.JPEG:n02109047 +ILSVRC2012_val_00028933.JPEG:n01968897 +ILSVRC2012_val_00028934.JPEG:n03388043 +ILSVRC2012_val_00028935.JPEG:n04258138 +ILSVRC2012_val_00028936.JPEG:n02112137 +ILSVRC2012_val_00028937.JPEG:n02607072 +ILSVRC2012_val_00028938.JPEG:n02134084 +ILSVRC2012_val_00028939.JPEG:n03837869 +ILSVRC2012_val_00028940.JPEG:n04200800 +ILSVRC2012_val_00028941.JPEG:n02071294 +ILSVRC2012_val_00028942.JPEG:n04141076 +ILSVRC2012_val_00028943.JPEG:n02085620 +ILSVRC2012_val_00028944.JPEG:n03218198 +ILSVRC2012_val_00028945.JPEG:n02098286 +ILSVRC2012_val_00028946.JPEG:n02099601 +ILSVRC2012_val_00028947.JPEG:n04099969 +ILSVRC2012_val_00028948.JPEG:n03216828 +ILSVRC2012_val_00028949.JPEG:n02892767 +ILSVRC2012_val_00028950.JPEG:n03482405 +ILSVRC2012_val_00028951.JPEG:n03838899 +ILSVRC2012_val_00028952.JPEG:n03018349 +ILSVRC2012_val_00028953.JPEG:n04487394 +ILSVRC2012_val_00028954.JPEG:n04141076 +ILSVRC2012_val_00028955.JPEG:n02106382 +ILSVRC2012_val_00028956.JPEG:n11939491 +ILSVRC2012_val_00028957.JPEG:n03100240 +ILSVRC2012_val_00028958.JPEG:n03908714 +ILSVRC2012_val_00028959.JPEG:n07831146 +ILSVRC2012_val_00028960.JPEG:n09256479 +ILSVRC2012_val_00028961.JPEG:n12267677 +ILSVRC2012_val_00028962.JPEG:n04152593 +ILSVRC2012_val_00028963.JPEG:n02093428 +ILSVRC2012_val_00028964.JPEG:n02791270 +ILSVRC2012_val_00028965.JPEG:n02099429 +ILSVRC2012_val_00028966.JPEG:n02105056 +ILSVRC2012_val_00028967.JPEG:n03223299 +ILSVRC2012_val_00028968.JPEG:n02643566 +ILSVRC2012_val_00028969.JPEG:n07720875 +ILSVRC2012_val_00028970.JPEG:n02124075 +ILSVRC2012_val_00028971.JPEG:n02699494 +ILSVRC2012_val_00028972.JPEG:n03888605 +ILSVRC2012_val_00028973.JPEG:n03249569 +ILSVRC2012_val_00028974.JPEG:n03584254 +ILSVRC2012_val_00028975.JPEG:n02981792 +ILSVRC2012_val_00028976.JPEG:n04133789 +ILSVRC2012_val_00028977.JPEG:n03534580 +ILSVRC2012_val_00028978.JPEG:n01518878 +ILSVRC2012_val_00028979.JPEG:n02704792 +ILSVRC2012_val_00028980.JPEG:n07747607 +ILSVRC2012_val_00028981.JPEG:n13037406 +ILSVRC2012_val_00028982.JPEG:n02488291 +ILSVRC2012_val_00028983.JPEG:n03538406 +ILSVRC2012_val_00028984.JPEG:n03627232 +ILSVRC2012_val_00028985.JPEG:n02099429 +ILSVRC2012_val_00028986.JPEG:n02704792 +ILSVRC2012_val_00028987.JPEG:n07684084 +ILSVRC2012_val_00028988.JPEG:n03733805 +ILSVRC2012_val_00028989.JPEG:n02397096 +ILSVRC2012_val_00028990.JPEG:n02114367 +ILSVRC2012_val_00028991.JPEG:n02319095 +ILSVRC2012_val_00028992.JPEG:n02086646 +ILSVRC2012_val_00028993.JPEG:n02094433 +ILSVRC2012_val_00028994.JPEG:n04133789 +ILSVRC2012_val_00028995.JPEG:n04483307 +ILSVRC2012_val_00028996.JPEG:n02504013 +ILSVRC2012_val_00028997.JPEG:n04525038 +ILSVRC2012_val_00028998.JPEG:n04265275 +ILSVRC2012_val_00028999.JPEG:n04209239 +ILSVRC2012_val_00029000.JPEG:n03967562 +ILSVRC2012_val_00029001.JPEG:n02129165 +ILSVRC2012_val_00029002.JPEG:n03777754 +ILSVRC2012_val_00029003.JPEG:n09835506 +ILSVRC2012_val_00029004.JPEG:n02727426 +ILSVRC2012_val_00029005.JPEG:n01693334 +ILSVRC2012_val_00029006.JPEG:n02457408 +ILSVRC2012_val_00029007.JPEG:n02128925 +ILSVRC2012_val_00029008.JPEG:n03903868 +ILSVRC2012_val_00029009.JPEG:n04409515 +ILSVRC2012_val_00029010.JPEG:n01950731 +ILSVRC2012_val_00029011.JPEG:n06359193 +ILSVRC2012_val_00029012.JPEG:n03187595 +ILSVRC2012_val_00029013.JPEG:n01950731 +ILSVRC2012_val_00029014.JPEG:n04041544 +ILSVRC2012_val_00029015.JPEG:n02892767 +ILSVRC2012_val_00029016.JPEG:n02363005 +ILSVRC2012_val_00029017.JPEG:n04355338 +ILSVRC2012_val_00029018.JPEG:n02277742 +ILSVRC2012_val_00029019.JPEG:n04090263 +ILSVRC2012_val_00029020.JPEG:n03314780 +ILSVRC2012_val_00029021.JPEG:n04285008 +ILSVRC2012_val_00029022.JPEG:n01847000 +ILSVRC2012_val_00029023.JPEG:n02094433 +ILSVRC2012_val_00029024.JPEG:n02098105 +ILSVRC2012_val_00029025.JPEG:n07892512 +ILSVRC2012_val_00029026.JPEG:n09229709 +ILSVRC2012_val_00029027.JPEG:n03527444 +ILSVRC2012_val_00029028.JPEG:n03530642 +ILSVRC2012_val_00029029.JPEG:n01774384 +ILSVRC2012_val_00029030.JPEG:n01773157 +ILSVRC2012_val_00029031.JPEG:n04366367 +ILSVRC2012_val_00029032.JPEG:n03676483 +ILSVRC2012_val_00029033.JPEG:n01930112 +ILSVRC2012_val_00029034.JPEG:n03933933 +ILSVRC2012_val_00029035.JPEG:n03877845 +ILSVRC2012_val_00029036.JPEG:n02104365 +ILSVRC2012_val_00029037.JPEG:n07697537 +ILSVRC2012_val_00029038.JPEG:n02444819 +ILSVRC2012_val_00029039.JPEG:n13037406 +ILSVRC2012_val_00029040.JPEG:n04296562 +ILSVRC2012_val_00029041.JPEG:n02457408 +ILSVRC2012_val_00029042.JPEG:n11879895 +ILSVRC2012_val_00029043.JPEG:n04120489 +ILSVRC2012_val_00029044.JPEG:n03958227 +ILSVRC2012_val_00029045.JPEG:n03187595 +ILSVRC2012_val_00029046.JPEG:n03930630 +ILSVRC2012_val_00029047.JPEG:n02277742 +ILSVRC2012_val_00029048.JPEG:n01774750 +ILSVRC2012_val_00029049.JPEG:n04550184 +ILSVRC2012_val_00029050.JPEG:n02837789 +ILSVRC2012_val_00029051.JPEG:n04479046 +ILSVRC2012_val_00029052.JPEG:n02500267 +ILSVRC2012_val_00029053.JPEG:n04317175 +ILSVRC2012_val_00029054.JPEG:n07875152 +ILSVRC2012_val_00029055.JPEG:n01687978 +ILSVRC2012_val_00029056.JPEG:n02088094 +ILSVRC2012_val_00029057.JPEG:n02814533 +ILSVRC2012_val_00029058.JPEG:n02109961 +ILSVRC2012_val_00029059.JPEG:n02117135 +ILSVRC2012_val_00029060.JPEG:n04579145 +ILSVRC2012_val_00029061.JPEG:n07880968 +ILSVRC2012_val_00029062.JPEG:n02190166 +ILSVRC2012_val_00029063.JPEG:n02396427 +ILSVRC2012_val_00029064.JPEG:n04542943 +ILSVRC2012_val_00029065.JPEG:n04357314 +ILSVRC2012_val_00029066.JPEG:n02114855 +ILSVRC2012_val_00029067.JPEG:n03920288 +ILSVRC2012_val_00029068.JPEG:n02120079 +ILSVRC2012_val_00029069.JPEG:n01776313 +ILSVRC2012_val_00029070.JPEG:n01847000 +ILSVRC2012_val_00029071.JPEG:n04447861 +ILSVRC2012_val_00029072.JPEG:n04019541 +ILSVRC2012_val_00029073.JPEG:n03201208 +ILSVRC2012_val_00029074.JPEG:n03857828 +ILSVRC2012_val_00029075.JPEG:n03404251 +ILSVRC2012_val_00029076.JPEG:n07754684 +ILSVRC2012_val_00029077.JPEG:n09256479 +ILSVRC2012_val_00029078.JPEG:n02442845 +ILSVRC2012_val_00029079.JPEG:n06794110 +ILSVRC2012_val_00029080.JPEG:n02917067 +ILSVRC2012_val_00029081.JPEG:n04592741 +ILSVRC2012_val_00029082.JPEG:n02389026 +ILSVRC2012_val_00029083.JPEG:n03444034 +ILSVRC2012_val_00029084.JPEG:n03724870 +ILSVRC2012_val_00029085.JPEG:n02895154 +ILSVRC2012_val_00029086.JPEG:n02165456 +ILSVRC2012_val_00029087.JPEG:n03804744 +ILSVRC2012_val_00029088.JPEG:n01742172 +ILSVRC2012_val_00029089.JPEG:n02037110 +ILSVRC2012_val_00029090.JPEG:n02087046 +ILSVRC2012_val_00029091.JPEG:n02865351 +ILSVRC2012_val_00029092.JPEG:n02025239 +ILSVRC2012_val_00029093.JPEG:n03887697 +ILSVRC2012_val_00029094.JPEG:n02814533 +ILSVRC2012_val_00029095.JPEG:n04133789 +ILSVRC2012_val_00029096.JPEG:n03891332 +ILSVRC2012_val_00029097.JPEG:n02483708 +ILSVRC2012_val_00029098.JPEG:n07714571 +ILSVRC2012_val_00029099.JPEG:n03982430 +ILSVRC2012_val_00029100.JPEG:n04579145 +ILSVRC2012_val_00029101.JPEG:n02127052 +ILSVRC2012_val_00029102.JPEG:n07932039 +ILSVRC2012_val_00029103.JPEG:n04238763 +ILSVRC2012_val_00029104.JPEG:n03710637 +ILSVRC2012_val_00029105.JPEG:n02825657 +ILSVRC2012_val_00029106.JPEG:n03977966 +ILSVRC2012_val_00029107.JPEG:n02321529 +ILSVRC2012_val_00029108.JPEG:n02493509 +ILSVRC2012_val_00029109.JPEG:n02219486 +ILSVRC2012_val_00029110.JPEG:n09193705 +ILSVRC2012_val_00029111.JPEG:n01950731 +ILSVRC2012_val_00029112.JPEG:n03457902 +ILSVRC2012_val_00029113.JPEG:n03908714 +ILSVRC2012_val_00029114.JPEG:n03980874 +ILSVRC2012_val_00029115.JPEG:n02113624 +ILSVRC2012_val_00029116.JPEG:n03393912 +ILSVRC2012_val_00029117.JPEG:n03379051 +ILSVRC2012_val_00029118.JPEG:n01688243 +ILSVRC2012_val_00029119.JPEG:n02971356 +ILSVRC2012_val_00029120.JPEG:n04243546 +ILSVRC2012_val_00029121.JPEG:n02510455 +ILSVRC2012_val_00029122.JPEG:n02092002 +ILSVRC2012_val_00029123.JPEG:n02116738 +ILSVRC2012_val_00029124.JPEG:n02391049 +ILSVRC2012_val_00029125.JPEG:n04111531 +ILSVRC2012_val_00029126.JPEG:n02128925 +ILSVRC2012_val_00029127.JPEG:n02097047 +ILSVRC2012_val_00029128.JPEG:n02071294 +ILSVRC2012_val_00029129.JPEG:n04462240 +ILSVRC2012_val_00029130.JPEG:n01748264 +ILSVRC2012_val_00029131.JPEG:n02086910 +ILSVRC2012_val_00029132.JPEG:n04326547 +ILSVRC2012_val_00029133.JPEG:n02107908 +ILSVRC2012_val_00029134.JPEG:n06874185 +ILSVRC2012_val_00029135.JPEG:n03773504 +ILSVRC2012_val_00029136.JPEG:n04039381 +ILSVRC2012_val_00029137.JPEG:n03874293 +ILSVRC2012_val_00029138.JPEG:n04482393 +ILSVRC2012_val_00029139.JPEG:n04371774 +ILSVRC2012_val_00029140.JPEG:n02088094 +ILSVRC2012_val_00029141.JPEG:n03887697 +ILSVRC2012_val_00029142.JPEG:n03452741 +ILSVRC2012_val_00029143.JPEG:n07802026 +ILSVRC2012_val_00029144.JPEG:n02509815 +ILSVRC2012_val_00029145.JPEG:n03347037 +ILSVRC2012_val_00029146.JPEG:n03983396 +ILSVRC2012_val_00029147.JPEG:n01774750 +ILSVRC2012_val_00029148.JPEG:n02879718 +ILSVRC2012_val_00029149.JPEG:n03888257 +ILSVRC2012_val_00029150.JPEG:n01796340 +ILSVRC2012_val_00029151.JPEG:n07717556 +ILSVRC2012_val_00029152.JPEG:n02112706 +ILSVRC2012_val_00029153.JPEG:n01742172 +ILSVRC2012_val_00029154.JPEG:n12998815 +ILSVRC2012_val_00029155.JPEG:n03271574 +ILSVRC2012_val_00029156.JPEG:n01775062 +ILSVRC2012_val_00029157.JPEG:n02112706 +ILSVRC2012_val_00029158.JPEG:n04153751 +ILSVRC2012_val_00029159.JPEG:n04350905 +ILSVRC2012_val_00029160.JPEG:n02481823 +ILSVRC2012_val_00029161.JPEG:n02487347 +ILSVRC2012_val_00029162.JPEG:n01950731 +ILSVRC2012_val_00029163.JPEG:n02667093 +ILSVRC2012_val_00029164.JPEG:n02089973 +ILSVRC2012_val_00029165.JPEG:n04592741 +ILSVRC2012_val_00029166.JPEG:n03393912 +ILSVRC2012_val_00029167.JPEG:n02840245 +ILSVRC2012_val_00029168.JPEG:n02006656 +ILSVRC2012_val_00029169.JPEG:n01498041 +ILSVRC2012_val_00029170.JPEG:n04548362 +ILSVRC2012_val_00029171.JPEG:n02782093 +ILSVRC2012_val_00029172.JPEG:n09193705 +ILSVRC2012_val_00029173.JPEG:n02443114 +ILSVRC2012_val_00029174.JPEG:n01773549 +ILSVRC2012_val_00029175.JPEG:n02093428 +ILSVRC2012_val_00029176.JPEG:n04116512 +ILSVRC2012_val_00029177.JPEG:n01770393 +ILSVRC2012_val_00029178.JPEG:n02128925 +ILSVRC2012_val_00029179.JPEG:n02939185 +ILSVRC2012_val_00029180.JPEG:n04133789 +ILSVRC2012_val_00029181.JPEG:n02777292 +ILSVRC2012_val_00029182.JPEG:n03976657 +ILSVRC2012_val_00029183.JPEG:n03876231 +ILSVRC2012_val_00029184.JPEG:n02443114 +ILSVRC2012_val_00029185.JPEG:n04590129 +ILSVRC2012_val_00029186.JPEG:n02114855 +ILSVRC2012_val_00029187.JPEG:n04335435 +ILSVRC2012_val_00029188.JPEG:n03372029 +ILSVRC2012_val_00029189.JPEG:n04418357 +ILSVRC2012_val_00029190.JPEG:n02109961 +ILSVRC2012_val_00029191.JPEG:n02088094 +ILSVRC2012_val_00029192.JPEG:n02279972 +ILSVRC2012_val_00029193.JPEG:n03657121 +ILSVRC2012_val_00029194.JPEG:n04482393 +ILSVRC2012_val_00029195.JPEG:n04229816 +ILSVRC2012_val_00029196.JPEG:n02264363 +ILSVRC2012_val_00029197.JPEG:n04136333 +ILSVRC2012_val_00029198.JPEG:n02027492 +ILSVRC2012_val_00029199.JPEG:n03617480 +ILSVRC2012_val_00029200.JPEG:n07753592 +ILSVRC2012_val_00029201.JPEG:n03459775 +ILSVRC2012_val_00029202.JPEG:n04154565 +ILSVRC2012_val_00029203.JPEG:n03425413 +ILSVRC2012_val_00029204.JPEG:n01955084 +ILSVRC2012_val_00029205.JPEG:n03127925 +ILSVRC2012_val_00029206.JPEG:n02017213 +ILSVRC2012_val_00029207.JPEG:n02437616 +ILSVRC2012_val_00029208.JPEG:n01774384 +ILSVRC2012_val_00029209.JPEG:n07760859 +ILSVRC2012_val_00029210.JPEG:n01818515 +ILSVRC2012_val_00029211.JPEG:n03000684 +ILSVRC2012_val_00029212.JPEG:n02128385 +ILSVRC2012_val_00029213.JPEG:n04487081 +ILSVRC2012_val_00029214.JPEG:n02105505 +ILSVRC2012_val_00029215.JPEG:n03376595 +ILSVRC2012_val_00029216.JPEG:n02130308 +ILSVRC2012_val_00029217.JPEG:n02108000 +ILSVRC2012_val_00029218.JPEG:n03042490 +ILSVRC2012_val_00029219.JPEG:n02992211 +ILSVRC2012_val_00029220.JPEG:n07718472 +ILSVRC2012_val_00029221.JPEG:n02417914 +ILSVRC2012_val_00029222.JPEG:n02701002 +ILSVRC2012_val_00029223.JPEG:n02058221 +ILSVRC2012_val_00029224.JPEG:n03888605 +ILSVRC2012_val_00029225.JPEG:n01694178 +ILSVRC2012_val_00029226.JPEG:n01855672 +ILSVRC2012_val_00029227.JPEG:n02168699 +ILSVRC2012_val_00029228.JPEG:n02676566 +ILSVRC2012_val_00029229.JPEG:n04507155 +ILSVRC2012_val_00029230.JPEG:n03777754 +ILSVRC2012_val_00029231.JPEG:n01704323 +ILSVRC2012_val_00029232.JPEG:n02088094 +ILSVRC2012_val_00029233.JPEG:n03444034 +ILSVRC2012_val_00029234.JPEG:n02883205 +ILSVRC2012_val_00029235.JPEG:n02909870 +ILSVRC2012_val_00029236.JPEG:n02787622 +ILSVRC2012_val_00029237.JPEG:n02102973 +ILSVRC2012_val_00029238.JPEG:n02514041 +ILSVRC2012_val_00029239.JPEG:n03085013 +ILSVRC2012_val_00029240.JPEG:n04328186 +ILSVRC2012_val_00029241.JPEG:n02494079 +ILSVRC2012_val_00029242.JPEG:n02093428 +ILSVRC2012_val_00029243.JPEG:n01986214 +ILSVRC2012_val_00029244.JPEG:n03594945 +ILSVRC2012_val_00029245.JPEG:n01847000 +ILSVRC2012_val_00029246.JPEG:n02110958 +ILSVRC2012_val_00029247.JPEG:n04252077 +ILSVRC2012_val_00029248.JPEG:n03041632 +ILSVRC2012_val_00029249.JPEG:n09421951 +ILSVRC2012_val_00029250.JPEG:n03776460 +ILSVRC2012_val_00029251.JPEG:n03676483 +ILSVRC2012_val_00029252.JPEG:n02804610 +ILSVRC2012_val_00029253.JPEG:n02112350 +ILSVRC2012_val_00029254.JPEG:n02096294 +ILSVRC2012_val_00029255.JPEG:n02108089 +ILSVRC2012_val_00029256.JPEG:n03690938 +ILSVRC2012_val_00029257.JPEG:n04372370 +ILSVRC2012_val_00029258.JPEG:n03877845 +ILSVRC2012_val_00029259.JPEG:n02111500 +ILSVRC2012_val_00029260.JPEG:n04476259 +ILSVRC2012_val_00029261.JPEG:n02104029 +ILSVRC2012_val_00029262.JPEG:n02085782 +ILSVRC2012_val_00029263.JPEG:n03424325 +ILSVRC2012_val_00029264.JPEG:n01943899 +ILSVRC2012_val_00029265.JPEG:n02443114 +ILSVRC2012_val_00029266.JPEG:n02865351 +ILSVRC2012_val_00029267.JPEG:n02129604 +ILSVRC2012_val_00029268.JPEG:n04487394 +ILSVRC2012_val_00029269.JPEG:n02493509 +ILSVRC2012_val_00029270.JPEG:n03026506 +ILSVRC2012_val_00029271.JPEG:n04136333 +ILSVRC2012_val_00029272.JPEG:n04507155 +ILSVRC2012_val_00029273.JPEG:n04356056 +ILSVRC2012_val_00029274.JPEG:n04039381 +ILSVRC2012_val_00029275.JPEG:n03944341 +ILSVRC2012_val_00029276.JPEG:n03947888 +ILSVRC2012_val_00029277.JPEG:n02098105 +ILSVRC2012_val_00029278.JPEG:n02133161 +ILSVRC2012_val_00029279.JPEG:n02841315 +ILSVRC2012_val_00029280.JPEG:n04251144 +ILSVRC2012_val_00029281.JPEG:n02094114 +ILSVRC2012_val_00029282.JPEG:n04505470 +ILSVRC2012_val_00029283.JPEG:n01829413 +ILSVRC2012_val_00029284.JPEG:n02493509 +ILSVRC2012_val_00029285.JPEG:n11879895 +ILSVRC2012_val_00029286.JPEG:n07875152 +ILSVRC2012_val_00029287.JPEG:n01983481 +ILSVRC2012_val_00029288.JPEG:n02500267 +ILSVRC2012_val_00029289.JPEG:n02085620 +ILSVRC2012_val_00029290.JPEG:n13040303 +ILSVRC2012_val_00029291.JPEG:n03902125 +ILSVRC2012_val_00029292.JPEG:n12620546 +ILSVRC2012_val_00029293.JPEG:n03599486 +ILSVRC2012_val_00029294.JPEG:n03891332 +ILSVRC2012_val_00029295.JPEG:n02102480 +ILSVRC2012_val_00029296.JPEG:n04118538 +ILSVRC2012_val_00029297.JPEG:n01807496 +ILSVRC2012_val_00029298.JPEG:n01860187 +ILSVRC2012_val_00029299.JPEG:n03444034 +ILSVRC2012_val_00029300.JPEG:n01491361 +ILSVRC2012_val_00029301.JPEG:n07831146 +ILSVRC2012_val_00029302.JPEG:n02666196 +ILSVRC2012_val_00029303.JPEG:n02892767 +ILSVRC2012_val_00029304.JPEG:n13040303 +ILSVRC2012_val_00029305.JPEG:n03032252 +ILSVRC2012_val_00029306.JPEG:n02125311 +ILSVRC2012_val_00029307.JPEG:n02168699 +ILSVRC2012_val_00029308.JPEG:n02117135 +ILSVRC2012_val_00029309.JPEG:n02395406 +ILSVRC2012_val_00029310.JPEG:n01537544 +ILSVRC2012_val_00029311.JPEG:n07753275 +ILSVRC2012_val_00029312.JPEG:n04428191 +ILSVRC2012_val_00029313.JPEG:n02109961 +ILSVRC2012_val_00029314.JPEG:n04235860 +ILSVRC2012_val_00029315.JPEG:n02417914 +ILSVRC2012_val_00029316.JPEG:n04584207 +ILSVRC2012_val_00029317.JPEG:n04070727 +ILSVRC2012_val_00029318.JPEG:n01873310 +ILSVRC2012_val_00029319.JPEG:n02749479 +ILSVRC2012_val_00029320.JPEG:n02769748 +ILSVRC2012_val_00029321.JPEG:n07714571 +ILSVRC2012_val_00029322.JPEG:n04367480 +ILSVRC2012_val_00029323.JPEG:n02012849 +ILSVRC2012_val_00029324.JPEG:n01665541 +ILSVRC2012_val_00029325.JPEG:n02167151 +ILSVRC2012_val_00029326.JPEG:n02088466 +ILSVRC2012_val_00029327.JPEG:n03527444 +ILSVRC2012_val_00029328.JPEG:n04409515 +ILSVRC2012_val_00029329.JPEG:n02013706 +ILSVRC2012_val_00029330.JPEG:n03325584 +ILSVRC2012_val_00029331.JPEG:n02441942 +ILSVRC2012_val_00029332.JPEG:n07613480 +ILSVRC2012_val_00029333.JPEG:n02101006 +ILSVRC2012_val_00029334.JPEG:n02088632 +ILSVRC2012_val_00029335.JPEG:n02129604 +ILSVRC2012_val_00029336.JPEG:n01685808 +ILSVRC2012_val_00029337.JPEG:n02966687 +ILSVRC2012_val_00029338.JPEG:n04367480 +ILSVRC2012_val_00029339.JPEG:n03908618 +ILSVRC2012_val_00029340.JPEG:n02977058 +ILSVRC2012_val_00029341.JPEG:n04111531 +ILSVRC2012_val_00029342.JPEG:n03042490 +ILSVRC2012_val_00029343.JPEG:n03717622 +ILSVRC2012_val_00029344.JPEG:n06785654 +ILSVRC2012_val_00029345.JPEG:n02980441 +ILSVRC2012_val_00029346.JPEG:n01968897 +ILSVRC2012_val_00029347.JPEG:n01843065 +ILSVRC2012_val_00029348.JPEG:n04554684 +ILSVRC2012_val_00029349.JPEG:n04523525 +ILSVRC2012_val_00029350.JPEG:n04417672 +ILSVRC2012_val_00029351.JPEG:n01855672 +ILSVRC2012_val_00029352.JPEG:n03873416 +ILSVRC2012_val_00029353.JPEG:n02100877 +ILSVRC2012_val_00029354.JPEG:n02105505 +ILSVRC2012_val_00029355.JPEG:n03492542 +ILSVRC2012_val_00029356.JPEG:n01833805 +ILSVRC2012_val_00029357.JPEG:n04116512 +ILSVRC2012_val_00029358.JPEG:n04487394 +ILSVRC2012_val_00029359.JPEG:n02105505 +ILSVRC2012_val_00029360.JPEG:n03297495 +ILSVRC2012_val_00029361.JPEG:n02119022 +ILSVRC2012_val_00029362.JPEG:n04392985 +ILSVRC2012_val_00029363.JPEG:n02108422 +ILSVRC2012_val_00029364.JPEG:n02098413 +ILSVRC2012_val_00029365.JPEG:n02012849 +ILSVRC2012_val_00029366.JPEG:n04487394 +ILSVRC2012_val_00029367.JPEG:n01990800 +ILSVRC2012_val_00029368.JPEG:n02817516 +ILSVRC2012_val_00029369.JPEG:n03216828 +ILSVRC2012_val_00029370.JPEG:n03187595 +ILSVRC2012_val_00029371.JPEG:n07871810 +ILSVRC2012_val_00029372.JPEG:n02669723 +ILSVRC2012_val_00029373.JPEG:n02229544 +ILSVRC2012_val_00029374.JPEG:n02966687 +ILSVRC2012_val_00029375.JPEG:n02113712 +ILSVRC2012_val_00029376.JPEG:n03930313 +ILSVRC2012_val_00029377.JPEG:n03417042 +ILSVRC2012_val_00029378.JPEG:n02389026 +ILSVRC2012_val_00029379.JPEG:n03249569 +ILSVRC2012_val_00029380.JPEG:n03633091 +ILSVRC2012_val_00029381.JPEG:n02096294 +ILSVRC2012_val_00029382.JPEG:n02110627 +ILSVRC2012_val_00029383.JPEG:n03916031 +ILSVRC2012_val_00029384.JPEG:n07920052 +ILSVRC2012_val_00029385.JPEG:n04146614 +ILSVRC2012_val_00029386.JPEG:n03207743 +ILSVRC2012_val_00029387.JPEG:n02325366 +ILSVRC2012_val_00029388.JPEG:n03954731 +ILSVRC2012_val_00029389.JPEG:n04133789 +ILSVRC2012_val_00029390.JPEG:n03788195 +ILSVRC2012_val_00029391.JPEG:n03982430 +ILSVRC2012_val_00029392.JPEG:n02112706 +ILSVRC2012_val_00029393.JPEG:n02017213 +ILSVRC2012_val_00029394.JPEG:n02492660 +ILSVRC2012_val_00029395.JPEG:n03976467 +ILSVRC2012_val_00029396.JPEG:n03792782 +ILSVRC2012_val_00029397.JPEG:n02123159 +ILSVRC2012_val_00029398.JPEG:n07754684 +ILSVRC2012_val_00029399.JPEG:n03444034 +ILSVRC2012_val_00029400.JPEG:n03063599 +ILSVRC2012_val_00029401.JPEG:n02326432 +ILSVRC2012_val_00029402.JPEG:n02009912 +ILSVRC2012_val_00029403.JPEG:n04154565 +ILSVRC2012_val_00029404.JPEG:n03492542 +ILSVRC2012_val_00029405.JPEG:n03649909 +ILSVRC2012_val_00029406.JPEG:n02101388 +ILSVRC2012_val_00029407.JPEG:n02091134 +ILSVRC2012_val_00029408.JPEG:n02892201 +ILSVRC2012_val_00029409.JPEG:n02077923 +ILSVRC2012_val_00029410.JPEG:n02168699 +ILSVRC2012_val_00029411.JPEG:n04239074 +ILSVRC2012_val_00029412.JPEG:n03899768 +ILSVRC2012_val_00029413.JPEG:n04461696 +ILSVRC2012_val_00029414.JPEG:n03124170 +ILSVRC2012_val_00029415.JPEG:n09428293 +ILSVRC2012_val_00029416.JPEG:n03000247 +ILSVRC2012_val_00029417.JPEG:n01558993 +ILSVRC2012_val_00029418.JPEG:n02104365 +ILSVRC2012_val_00029419.JPEG:n02093991 +ILSVRC2012_val_00029420.JPEG:n03837869 +ILSVRC2012_val_00029421.JPEG:n02169497 +ILSVRC2012_val_00029422.JPEG:n03492542 +ILSVRC2012_val_00029423.JPEG:n03706229 +ILSVRC2012_val_00029424.JPEG:n02129165 +ILSVRC2012_val_00029425.JPEG:n03216828 +ILSVRC2012_val_00029426.JPEG:n03662601 +ILSVRC2012_val_00029427.JPEG:n02444819 +ILSVRC2012_val_00029428.JPEG:n03930313 +ILSVRC2012_val_00029429.JPEG:n04039381 +ILSVRC2012_val_00029430.JPEG:n01601694 +ILSVRC2012_val_00029431.JPEG:n04228054 +ILSVRC2012_val_00029432.JPEG:n02788148 +ILSVRC2012_val_00029433.JPEG:n03133878 +ILSVRC2012_val_00029434.JPEG:n01983481 +ILSVRC2012_val_00029435.JPEG:n02093859 +ILSVRC2012_val_00029436.JPEG:n02106166 +ILSVRC2012_val_00029437.JPEG:n02102973 +ILSVRC2012_val_00029438.JPEG:n03982430 +ILSVRC2012_val_00029439.JPEG:n02667093 +ILSVRC2012_val_00029440.JPEG:n03891332 +ILSVRC2012_val_00029441.JPEG:n01592084 +ILSVRC2012_val_00029442.JPEG:n02172182 +ILSVRC2012_val_00029443.JPEG:n03404251 +ILSVRC2012_val_00029444.JPEG:n02259212 +ILSVRC2012_val_00029445.JPEG:n03250847 +ILSVRC2012_val_00029446.JPEG:n02817516 +ILSVRC2012_val_00029447.JPEG:n07747607 +ILSVRC2012_val_00029448.JPEG:n03063599 +ILSVRC2012_val_00029449.JPEG:n03935335 +ILSVRC2012_val_00029450.JPEG:n02085620 +ILSVRC2012_val_00029451.JPEG:n02092002 +ILSVRC2012_val_00029452.JPEG:n02999410 +ILSVRC2012_val_00029453.JPEG:n02504458 +ILSVRC2012_val_00029454.JPEG:n03100240 +ILSVRC2012_val_00029455.JPEG:n04392985 +ILSVRC2012_val_00029456.JPEG:n02105855 +ILSVRC2012_val_00029457.JPEG:n07718747 +ILSVRC2012_val_00029458.JPEG:n03721384 +ILSVRC2012_val_00029459.JPEG:n02483362 +ILSVRC2012_val_00029460.JPEG:n01629819 +ILSVRC2012_val_00029461.JPEG:n02107683 +ILSVRC2012_val_00029462.JPEG:n02951358 +ILSVRC2012_val_00029463.JPEG:n07920052 +ILSVRC2012_val_00029464.JPEG:n03733805 +ILSVRC2012_val_00029465.JPEG:n02483362 +ILSVRC2012_val_00029466.JPEG:n01798484 +ILSVRC2012_val_00029467.JPEG:n04418357 +ILSVRC2012_val_00029468.JPEG:n04251144 +ILSVRC2012_val_00029469.JPEG:n03197337 +ILSVRC2012_val_00029470.JPEG:n03908618 +ILSVRC2012_val_00029471.JPEG:n01978287 +ILSVRC2012_val_00029472.JPEG:n01817953 +ILSVRC2012_val_00029473.JPEG:n04486054 +ILSVRC2012_val_00029474.JPEG:n04127249 +ILSVRC2012_val_00029475.JPEG:n01945685 +ILSVRC2012_val_00029476.JPEG:n07711569 +ILSVRC2012_val_00029477.JPEG:n02088238 +ILSVRC2012_val_00029478.JPEG:n02105641 +ILSVRC2012_val_00029479.JPEG:n02910353 +ILSVRC2012_val_00029480.JPEG:n07892512 +ILSVRC2012_val_00029481.JPEG:n01484850 +ILSVRC2012_val_00029482.JPEG:n03657121 +ILSVRC2012_val_00029483.JPEG:n02859443 +ILSVRC2012_val_00029484.JPEG:n07860988 +ILSVRC2012_val_00029485.JPEG:n04141327 +ILSVRC2012_val_00029486.JPEG:n03868863 +ILSVRC2012_val_00029487.JPEG:n01768244 +ILSVRC2012_val_00029488.JPEG:n03657121 +ILSVRC2012_val_00029489.JPEG:n02102973 +ILSVRC2012_val_00029490.JPEG:n02111500 +ILSVRC2012_val_00029491.JPEG:n01632458 +ILSVRC2012_val_00029492.JPEG:n02319095 +ILSVRC2012_val_00029493.JPEG:n04328186 +ILSVRC2012_val_00029494.JPEG:n04311004 +ILSVRC2012_val_00029495.JPEG:n01558993 +ILSVRC2012_val_00029496.JPEG:n01773549 +ILSVRC2012_val_00029497.JPEG:n01622779 +ILSVRC2012_val_00029498.JPEG:n02442845 +ILSVRC2012_val_00029499.JPEG:n07768694 +ILSVRC2012_val_00029500.JPEG:n01632777 +ILSVRC2012_val_00029501.JPEG:n03733805 +ILSVRC2012_val_00029502.JPEG:n03133878 +ILSVRC2012_val_00029503.JPEG:n02012849 +ILSVRC2012_val_00029504.JPEG:n03496892 +ILSVRC2012_val_00029505.JPEG:n02066245 +ILSVRC2012_val_00029506.JPEG:n02094433 +ILSVRC2012_val_00029507.JPEG:n03271574 +ILSVRC2012_val_00029508.JPEG:n02128757 +ILSVRC2012_val_00029509.JPEG:n03792782 +ILSVRC2012_val_00029510.JPEG:n02018795 +ILSVRC2012_val_00029511.JPEG:n01630670 +ILSVRC2012_val_00029512.JPEG:n02101006 +ILSVRC2012_val_00029513.JPEG:n04067472 +ILSVRC2012_val_00029514.JPEG:n02100583 +ILSVRC2012_val_00029515.JPEG:n04317175 +ILSVRC2012_val_00029516.JPEG:n03602883 +ILSVRC2012_val_00029517.JPEG:n04141327 +ILSVRC2012_val_00029518.JPEG:n02102040 +ILSVRC2012_val_00029519.JPEG:n07875152 +ILSVRC2012_val_00029520.JPEG:n02892201 +ILSVRC2012_val_00029521.JPEG:n04127249 +ILSVRC2012_val_00029522.JPEG:n07753275 +ILSVRC2012_val_00029523.JPEG:n04355338 +ILSVRC2012_val_00029524.JPEG:n02236044 +ILSVRC2012_val_00029525.JPEG:n01749939 +ILSVRC2012_val_00029526.JPEG:n07717556 +ILSVRC2012_val_00029527.JPEG:n02317335 +ILSVRC2012_val_00029528.JPEG:n02606052 +ILSVRC2012_val_00029529.JPEG:n04483307 +ILSVRC2012_val_00029530.JPEG:n04435653 +ILSVRC2012_val_00029531.JPEG:n04264628 +ILSVRC2012_val_00029532.JPEG:n04347754 +ILSVRC2012_val_00029533.JPEG:n04179913 +ILSVRC2012_val_00029534.JPEG:n07583066 +ILSVRC2012_val_00029535.JPEG:n04146614 +ILSVRC2012_val_00029536.JPEG:n03478589 +ILSVRC2012_val_00029537.JPEG:n03599486 +ILSVRC2012_val_00029538.JPEG:n02676566 +ILSVRC2012_val_00029539.JPEG:n02264363 +ILSVRC2012_val_00029540.JPEG:n04371430 +ILSVRC2012_val_00029541.JPEG:n03782006 +ILSVRC2012_val_00029542.JPEG:n04604644 +ILSVRC2012_val_00029543.JPEG:n03180011 +ILSVRC2012_val_00029544.JPEG:n03045698 +ILSVRC2012_val_00029545.JPEG:n03887697 +ILSVRC2012_val_00029546.JPEG:n02085936 +ILSVRC2012_val_00029547.JPEG:n07614500 +ILSVRC2012_val_00029548.JPEG:n04296562 +ILSVRC2012_val_00029549.JPEG:n02074367 +ILSVRC2012_val_00029550.JPEG:n01729977 +ILSVRC2012_val_00029551.JPEG:n02018795 +ILSVRC2012_val_00029552.JPEG:n01735189 +ILSVRC2012_val_00029553.JPEG:n03777568 +ILSVRC2012_val_00029554.JPEG:n03775546 +ILSVRC2012_val_00029555.JPEG:n02091244 +ILSVRC2012_val_00029556.JPEG:n03838899 +ILSVRC2012_val_00029557.JPEG:n04357314 +ILSVRC2012_val_00029558.JPEG:n01945685 +ILSVRC2012_val_00029559.JPEG:n03788365 +ILSVRC2012_val_00029560.JPEG:n02441942 +ILSVRC2012_val_00029561.JPEG:n04429376 +ILSVRC2012_val_00029562.JPEG:n02119022 +ILSVRC2012_val_00029563.JPEG:n01945685 +ILSVRC2012_val_00029564.JPEG:n03627232 +ILSVRC2012_val_00029565.JPEG:n02056570 +ILSVRC2012_val_00029566.JPEG:n02437616 +ILSVRC2012_val_00029567.JPEG:n03590841 +ILSVRC2012_val_00029568.JPEG:n01491361 +ILSVRC2012_val_00029569.JPEG:n01871265 +ILSVRC2012_val_00029570.JPEG:n04442312 +ILSVRC2012_val_00029571.JPEG:n01833805 +ILSVRC2012_val_00029572.JPEG:n04596742 +ILSVRC2012_val_00029573.JPEG:n04553703 +ILSVRC2012_val_00029574.JPEG:n04487394 +ILSVRC2012_val_00029575.JPEG:n03763968 +ILSVRC2012_val_00029576.JPEG:n02514041 +ILSVRC2012_val_00029577.JPEG:n11879895 +ILSVRC2012_val_00029578.JPEG:n04525038 +ILSVRC2012_val_00029579.JPEG:n02510455 +ILSVRC2012_val_00029580.JPEG:n04275548 +ILSVRC2012_val_00029581.JPEG:n01531178 +ILSVRC2012_val_00029582.JPEG:n04162706 +ILSVRC2012_val_00029583.JPEG:n03240683 +ILSVRC2012_val_00029584.JPEG:n04589890 +ILSVRC2012_val_00029585.JPEG:n03871628 +ILSVRC2012_val_00029586.JPEG:n04443257 +ILSVRC2012_val_00029587.JPEG:n02655020 +ILSVRC2012_val_00029588.JPEG:n04264628 +ILSVRC2012_val_00029589.JPEG:n01843383 +ILSVRC2012_val_00029590.JPEG:n02138441 +ILSVRC2012_val_00029591.JPEG:n02091032 +ILSVRC2012_val_00029592.JPEG:n02281406 +ILSVRC2012_val_00029593.JPEG:n03272010 +ILSVRC2012_val_00029594.JPEG:n03775546 +ILSVRC2012_val_00029595.JPEG:n03345487 +ILSVRC2012_val_00029596.JPEG:n03532672 +ILSVRC2012_val_00029597.JPEG:n02814860 +ILSVRC2012_val_00029598.JPEG:n07714571 +ILSVRC2012_val_00029599.JPEG:n02423022 +ILSVRC2012_val_00029600.JPEG:n03187595 +ILSVRC2012_val_00029601.JPEG:n03992509 +ILSVRC2012_val_00029602.JPEG:n03933933 +ILSVRC2012_val_00029603.JPEG:n03956157 +ILSVRC2012_val_00029604.JPEG:n07920052 +ILSVRC2012_val_00029605.JPEG:n01981276 +ILSVRC2012_val_00029606.JPEG:n03710721 +ILSVRC2012_val_00029607.JPEG:n04201297 +ILSVRC2012_val_00029608.JPEG:n09472597 +ILSVRC2012_val_00029609.JPEG:n02097130 +ILSVRC2012_val_00029610.JPEG:n02111889 +ILSVRC2012_val_00029611.JPEG:n03929660 +ILSVRC2012_val_00029612.JPEG:n02804610 +ILSVRC2012_val_00029613.JPEG:n03961711 +ILSVRC2012_val_00029614.JPEG:n07613480 +ILSVRC2012_val_00029615.JPEG:n01755581 +ILSVRC2012_val_00029616.JPEG:n02277742 +ILSVRC2012_val_00029617.JPEG:n03452741 +ILSVRC2012_val_00029618.JPEG:n02396427 +ILSVRC2012_val_00029619.JPEG:n01514859 +ILSVRC2012_val_00029620.JPEG:n04590129 +ILSVRC2012_val_00029621.JPEG:n04116512 +ILSVRC2012_val_00029622.JPEG:n01631663 +ILSVRC2012_val_00029623.JPEG:n07711569 +ILSVRC2012_val_00029624.JPEG:n02134084 +ILSVRC2012_val_00029625.JPEG:n04332243 +ILSVRC2012_val_00029626.JPEG:n04517823 +ILSVRC2012_val_00029627.JPEG:n01558993 +ILSVRC2012_val_00029628.JPEG:n02817516 +ILSVRC2012_val_00029629.JPEG:n02088632 +ILSVRC2012_val_00029630.JPEG:n03457902 +ILSVRC2012_val_00029631.JPEG:n01775062 +ILSVRC2012_val_00029632.JPEG:n02328150 +ILSVRC2012_val_00029633.JPEG:n02804610 +ILSVRC2012_val_00029634.JPEG:n02077923 +ILSVRC2012_val_00029635.JPEG:n02129604 +ILSVRC2012_val_00029636.JPEG:n02095314 +ILSVRC2012_val_00029637.JPEG:n03388183 +ILSVRC2012_val_00029638.JPEG:n02536864 +ILSVRC2012_val_00029639.JPEG:n03134739 +ILSVRC2012_val_00029640.JPEG:n03014705 +ILSVRC2012_val_00029641.JPEG:n02423022 +ILSVRC2012_val_00029642.JPEG:n04254120 +ILSVRC2012_val_00029643.JPEG:n03776460 +ILSVRC2012_val_00029644.JPEG:n03788195 +ILSVRC2012_val_00029645.JPEG:n03637318 +ILSVRC2012_val_00029646.JPEG:n02112706 +ILSVRC2012_val_00029647.JPEG:n03777568 +ILSVRC2012_val_00029648.JPEG:n02089078 +ILSVRC2012_val_00029649.JPEG:n03838899 +ILSVRC2012_val_00029650.JPEG:n03661043 +ILSVRC2012_val_00029651.JPEG:n02687172 +ILSVRC2012_val_00029652.JPEG:n02097658 +ILSVRC2012_val_00029653.JPEG:n02395406 +ILSVRC2012_val_00029654.JPEG:n01820546 +ILSVRC2012_val_00029655.JPEG:n03788365 +ILSVRC2012_val_00029656.JPEG:n02963159 +ILSVRC2012_val_00029657.JPEG:n02097298 +ILSVRC2012_val_00029658.JPEG:n07717556 +ILSVRC2012_val_00029659.JPEG:n02114367 +ILSVRC2012_val_00029660.JPEG:n02219486 +ILSVRC2012_val_00029661.JPEG:n04442312 +ILSVRC2012_val_00029662.JPEG:n04536866 +ILSVRC2012_val_00029663.JPEG:n02979186 +ILSVRC2012_val_00029664.JPEG:n04458633 +ILSVRC2012_val_00029665.JPEG:n07584110 +ILSVRC2012_val_00029666.JPEG:n03633091 +ILSVRC2012_val_00029667.JPEG:n04501370 +ILSVRC2012_val_00029668.JPEG:n03000684 +ILSVRC2012_val_00029669.JPEG:n02417914 +ILSVRC2012_val_00029670.JPEG:n02093859 +ILSVRC2012_val_00029671.JPEG:n04228054 +ILSVRC2012_val_00029672.JPEG:n03478589 +ILSVRC2012_val_00029673.JPEG:n02112137 +ILSVRC2012_val_00029674.JPEG:n03642806 +ILSVRC2012_val_00029675.JPEG:n02113712 +ILSVRC2012_val_00029676.JPEG:n02817516 +ILSVRC2012_val_00029677.JPEG:n03980874 +ILSVRC2012_val_00029678.JPEG:n01644900 +ILSVRC2012_val_00029679.JPEG:n11879895 +ILSVRC2012_val_00029680.JPEG:n04347754 +ILSVRC2012_val_00029681.JPEG:n03788195 +ILSVRC2012_val_00029682.JPEG:n02825657 +ILSVRC2012_val_00029683.JPEG:n02119789 +ILSVRC2012_val_00029684.JPEG:n02128925 +ILSVRC2012_val_00029685.JPEG:n02129604 +ILSVRC2012_val_00029686.JPEG:n04523525 +ILSVRC2012_val_00029687.JPEG:n04162706 +ILSVRC2012_val_00029688.JPEG:n03000247 +ILSVRC2012_val_00029689.JPEG:n04347754 +ILSVRC2012_val_00029690.JPEG:n02447366 +ILSVRC2012_val_00029691.JPEG:n02096294 +ILSVRC2012_val_00029692.JPEG:n02002724 +ILSVRC2012_val_00029693.JPEG:n02098413 +ILSVRC2012_val_00029694.JPEG:n03467068 +ILSVRC2012_val_00029695.JPEG:n01582220 +ILSVRC2012_val_00029696.JPEG:n02002556 +ILSVRC2012_val_00029697.JPEG:n03063689 +ILSVRC2012_val_00029698.JPEG:n01855672 +ILSVRC2012_val_00029699.JPEG:n02971356 +ILSVRC2012_val_00029700.JPEG:n02086240 +ILSVRC2012_val_00029701.JPEG:n02817516 +ILSVRC2012_val_00029702.JPEG:n01930112 +ILSVRC2012_val_00029703.JPEG:n02490219 +ILSVRC2012_val_00029704.JPEG:n09428293 +ILSVRC2012_val_00029705.JPEG:n02091467 +ILSVRC2012_val_00029706.JPEG:n03710637 +ILSVRC2012_val_00029707.JPEG:n02917067 +ILSVRC2012_val_00029708.JPEG:n06596364 +ILSVRC2012_val_00029709.JPEG:n01532829 +ILSVRC2012_val_00029710.JPEG:n02056570 +ILSVRC2012_val_00029711.JPEG:n04560804 +ILSVRC2012_val_00029712.JPEG:n01735189 +ILSVRC2012_val_00029713.JPEG:n04557648 +ILSVRC2012_val_00029714.JPEG:n07711569 +ILSVRC2012_val_00029715.JPEG:n06785654 +ILSVRC2012_val_00029716.JPEG:n04118776 +ILSVRC2012_val_00029717.JPEG:n02860847 +ILSVRC2012_val_00029718.JPEG:n02007558 +ILSVRC2012_val_00029719.JPEG:n02356798 +ILSVRC2012_val_00029720.JPEG:n04070727 +ILSVRC2012_val_00029721.JPEG:n02489166 +ILSVRC2012_val_00029722.JPEG:n07714990 +ILSVRC2012_val_00029723.JPEG:n02104365 +ILSVRC2012_val_00029724.JPEG:n02007558 +ILSVRC2012_val_00029725.JPEG:n03649909 +ILSVRC2012_val_00029726.JPEG:n01667114 +ILSVRC2012_val_00029727.JPEG:n01641577 +ILSVRC2012_val_00029728.JPEG:n03028079 +ILSVRC2012_val_00029729.JPEG:n03494278 +ILSVRC2012_val_00029730.JPEG:n07880968 +ILSVRC2012_val_00029731.JPEG:n03775071 +ILSVRC2012_val_00029732.JPEG:n01632458 +ILSVRC2012_val_00029733.JPEG:n01990800 +ILSVRC2012_val_00029734.JPEG:n02442845 +ILSVRC2012_val_00029735.JPEG:n02119022 +ILSVRC2012_val_00029736.JPEG:n02006656 +ILSVRC2012_val_00029737.JPEG:n02701002 +ILSVRC2012_val_00029738.JPEG:n02483362 +ILSVRC2012_val_00029739.JPEG:n03124170 +ILSVRC2012_val_00029740.JPEG:n01531178 +ILSVRC2012_val_00029741.JPEG:n02704792 +ILSVRC2012_val_00029742.JPEG:n02099849 +ILSVRC2012_val_00029743.JPEG:n01873310 +ILSVRC2012_val_00029744.JPEG:n01735189 +ILSVRC2012_val_00029745.JPEG:n04462240 +ILSVRC2012_val_00029746.JPEG:n03065424 +ILSVRC2012_val_00029747.JPEG:n04398044 +ILSVRC2012_val_00029748.JPEG:n04120489 +ILSVRC2012_val_00029749.JPEG:n04330267 +ILSVRC2012_val_00029750.JPEG:n03967562 +ILSVRC2012_val_00029751.JPEG:n02099601 +ILSVRC2012_val_00029752.JPEG:n03388043 +ILSVRC2012_val_00029753.JPEG:n02100583 +ILSVRC2012_val_00029754.JPEG:n02093991 +ILSVRC2012_val_00029755.JPEG:n09399592 +ILSVRC2012_val_00029756.JPEG:n01773797 +ILSVRC2012_val_00029757.JPEG:n03761084 +ILSVRC2012_val_00029758.JPEG:n02342885 +ILSVRC2012_val_00029759.JPEG:n02206856 +ILSVRC2012_val_00029760.JPEG:n02098286 +ILSVRC2012_val_00029761.JPEG:n03207743 +ILSVRC2012_val_00029762.JPEG:n13040303 +ILSVRC2012_val_00029763.JPEG:n01629819 +ILSVRC2012_val_00029764.JPEG:n02927161 +ILSVRC2012_val_00029765.JPEG:n04125021 +ILSVRC2012_val_00029766.JPEG:n04554684 +ILSVRC2012_val_00029767.JPEG:n02328150 +ILSVRC2012_val_00029768.JPEG:n03476684 +ILSVRC2012_val_00029769.JPEG:n02114367 +ILSVRC2012_val_00029770.JPEG:n03793489 +ILSVRC2012_val_00029771.JPEG:n03633091 +ILSVRC2012_val_00029772.JPEG:n03930630 +ILSVRC2012_val_00029773.JPEG:n02871525 +ILSVRC2012_val_00029774.JPEG:n02097474 +ILSVRC2012_val_00029775.JPEG:n02113799 +ILSVRC2012_val_00029776.JPEG:n02408429 +ILSVRC2012_val_00029777.JPEG:n03899768 +ILSVRC2012_val_00029778.JPEG:n07831146 +ILSVRC2012_val_00029779.JPEG:n04525038 +ILSVRC2012_val_00029780.JPEG:n02808304 +ILSVRC2012_val_00029781.JPEG:n03724870 +ILSVRC2012_val_00029782.JPEG:n02033041 +ILSVRC2012_val_00029783.JPEG:n02110063 +ILSVRC2012_val_00029784.JPEG:n03063689 +ILSVRC2012_val_00029785.JPEG:n01855672 +ILSVRC2012_val_00029786.JPEG:n02395406 +ILSVRC2012_val_00029787.JPEG:n04254680 +ILSVRC2012_val_00029788.JPEG:n03063689 +ILSVRC2012_val_00029789.JPEG:n02487347 +ILSVRC2012_val_00029790.JPEG:n02640242 +ILSVRC2012_val_00029791.JPEG:n03457902 +ILSVRC2012_val_00029792.JPEG:n12267677 +ILSVRC2012_val_00029793.JPEG:n04482393 +ILSVRC2012_val_00029794.JPEG:n04009552 +ILSVRC2012_val_00029795.JPEG:n02174001 +ILSVRC2012_val_00029796.JPEG:n01990800 +ILSVRC2012_val_00029797.JPEG:n04209133 +ILSVRC2012_val_00029798.JPEG:n01950731 +ILSVRC2012_val_00029799.JPEG:n02113186 +ILSVRC2012_val_00029800.JPEG:n03095699 +ILSVRC2012_val_00029801.JPEG:n01770081 +ILSVRC2012_val_00029802.JPEG:n04127249 +ILSVRC2012_val_00029803.JPEG:n02971356 +ILSVRC2012_val_00029804.JPEG:n02490219 +ILSVRC2012_val_00029805.JPEG:n04044716 +ILSVRC2012_val_00029806.JPEG:n01667778 +ILSVRC2012_val_00029807.JPEG:n03710721 +ILSVRC2012_val_00029808.JPEG:n03141823 +ILSVRC2012_val_00029809.JPEG:n04099969 +ILSVRC2012_val_00029810.JPEG:n02325366 +ILSVRC2012_val_00029811.JPEG:n04599235 +ILSVRC2012_val_00029812.JPEG:n01978455 +ILSVRC2012_val_00029813.JPEG:n03599486 +ILSVRC2012_val_00029814.JPEG:n02090622 +ILSVRC2012_val_00029815.JPEG:n03630383 +ILSVRC2012_val_00029816.JPEG:n02117135 +ILSVRC2012_val_00029817.JPEG:n02037110 +ILSVRC2012_val_00029818.JPEG:n02219486 +ILSVRC2012_val_00029819.JPEG:n03297495 +ILSVRC2012_val_00029820.JPEG:n02105505 +ILSVRC2012_val_00029821.JPEG:n04263257 +ILSVRC2012_val_00029822.JPEG:n02442845 +ILSVRC2012_val_00029823.JPEG:n04266014 +ILSVRC2012_val_00029824.JPEG:n03393912 +ILSVRC2012_val_00029825.JPEG:n02115641 +ILSVRC2012_val_00029826.JPEG:n02883205 +ILSVRC2012_val_00029827.JPEG:n01729977 +ILSVRC2012_val_00029828.JPEG:n03047690 +ILSVRC2012_val_00029829.JPEG:n02361337 +ILSVRC2012_val_00029830.JPEG:n04560804 +ILSVRC2012_val_00029831.JPEG:n02106662 +ILSVRC2012_val_00029832.JPEG:n03876231 +ILSVRC2012_val_00029833.JPEG:n03041632 +ILSVRC2012_val_00029834.JPEG:n02098105 +ILSVRC2012_val_00029835.JPEG:n01560419 +ILSVRC2012_val_00029836.JPEG:n02089078 +ILSVRC2012_val_00029837.JPEG:n03218198 +ILSVRC2012_val_00029838.JPEG:n04153751 +ILSVRC2012_val_00029839.JPEG:n02123597 +ILSVRC2012_val_00029840.JPEG:n03584829 +ILSVRC2012_val_00029841.JPEG:n02930766 +ILSVRC2012_val_00029842.JPEG:n03781244 +ILSVRC2012_val_00029843.JPEG:n02264363 +ILSVRC2012_val_00029844.JPEG:n07711569 +ILSVRC2012_val_00029845.JPEG:n04418357 +ILSVRC2012_val_00029846.JPEG:n06596364 +ILSVRC2012_val_00029847.JPEG:n03345487 +ILSVRC2012_val_00029848.JPEG:n02835271 +ILSVRC2012_val_00029849.JPEG:n04467665 +ILSVRC2012_val_00029850.JPEG:n03450230 +ILSVRC2012_val_00029851.JPEG:n03692522 +ILSVRC2012_val_00029852.JPEG:n03929660 +ILSVRC2012_val_00029853.JPEG:n03935335 +ILSVRC2012_val_00029854.JPEG:n01630670 +ILSVRC2012_val_00029855.JPEG:n02120505 +ILSVRC2012_val_00029856.JPEG:n02172182 +ILSVRC2012_val_00029857.JPEG:n03777754 +ILSVRC2012_val_00029858.JPEG:n04209133 +ILSVRC2012_val_00029859.JPEG:n01687978 +ILSVRC2012_val_00029860.JPEG:n03481172 +ILSVRC2012_val_00029861.JPEG:n02088094 +ILSVRC2012_val_00029862.JPEG:n02112350 +ILSVRC2012_val_00029863.JPEG:n03982430 +ILSVRC2012_val_00029864.JPEG:n02124075 +ILSVRC2012_val_00029865.JPEG:n03854065 +ILSVRC2012_val_00029866.JPEG:n04141076 +ILSVRC2012_val_00029867.JPEG:n06785654 +ILSVRC2012_val_00029868.JPEG:n02981792 +ILSVRC2012_val_00029869.JPEG:n03207941 +ILSVRC2012_val_00029870.JPEG:n03028079 +ILSVRC2012_val_00029871.JPEG:n13133613 +ILSVRC2012_val_00029872.JPEG:n02423022 +ILSVRC2012_val_00029873.JPEG:n03777568 +ILSVRC2012_val_00029874.JPEG:n02328150 +ILSVRC2012_val_00029875.JPEG:n02037110 +ILSVRC2012_val_00029876.JPEG:n02092002 +ILSVRC2012_val_00029877.JPEG:n02655020 +ILSVRC2012_val_00029878.JPEG:n04443257 +ILSVRC2012_val_00029879.JPEG:n02963159 +ILSVRC2012_val_00029880.JPEG:n01687978 +ILSVRC2012_val_00029881.JPEG:n09193705 +ILSVRC2012_val_00029882.JPEG:n10148035 +ILSVRC2012_val_00029883.JPEG:n03065424 +ILSVRC2012_val_00029884.JPEG:n03792972 +ILSVRC2012_val_00029885.JPEG:n02013706 +ILSVRC2012_val_00029886.JPEG:n01494475 +ILSVRC2012_val_00029887.JPEG:n07860988 +ILSVRC2012_val_00029888.JPEG:n02099267 +ILSVRC2012_val_00029889.JPEG:n04355933 +ILSVRC2012_val_00029890.JPEG:n02457408 +ILSVRC2012_val_00029891.JPEG:n01943899 +ILSVRC2012_val_00029892.JPEG:n03733131 +ILSVRC2012_val_00029893.JPEG:n04252077 +ILSVRC2012_val_00029894.JPEG:n02978881 +ILSVRC2012_val_00029895.JPEG:n03868863 +ILSVRC2012_val_00029896.JPEG:n03544143 +ILSVRC2012_val_00029897.JPEG:n03692522 +ILSVRC2012_val_00029898.JPEG:n12768682 +ILSVRC2012_val_00029899.JPEG:n02088094 +ILSVRC2012_val_00029900.JPEG:n04023962 +ILSVRC2012_val_00029901.JPEG:n02793495 +ILSVRC2012_val_00029902.JPEG:n03840681 +ILSVRC2012_val_00029903.JPEG:n01773549 +ILSVRC2012_val_00029904.JPEG:n03843555 +ILSVRC2012_val_00029905.JPEG:n04482393 +ILSVRC2012_val_00029906.JPEG:n07753592 +ILSVRC2012_val_00029907.JPEG:n03673027 +ILSVRC2012_val_00029908.JPEG:n07930864 +ILSVRC2012_val_00029909.JPEG:n01685808 +ILSVRC2012_val_00029910.JPEG:n02037110 +ILSVRC2012_val_00029911.JPEG:n02787622 +ILSVRC2012_val_00029912.JPEG:n06596364 +ILSVRC2012_val_00029913.JPEG:n02033041 +ILSVRC2012_val_00029914.JPEG:n04204238 +ILSVRC2012_val_00029915.JPEG:n12267677 +ILSVRC2012_val_00029916.JPEG:n02321529 +ILSVRC2012_val_00029917.JPEG:n03404251 +ILSVRC2012_val_00029918.JPEG:n03000684 +ILSVRC2012_val_00029919.JPEG:n07753592 +ILSVRC2012_val_00029920.JPEG:n03804744 +ILSVRC2012_val_00029921.JPEG:n01514668 +ILSVRC2012_val_00029922.JPEG:n03594945 +ILSVRC2012_val_00029923.JPEG:n02110627 +ILSVRC2012_val_00029924.JPEG:n03793489 +ILSVRC2012_val_00029925.JPEG:n04243546 +ILSVRC2012_val_00029926.JPEG:n02490219 +ILSVRC2012_val_00029927.JPEG:n02817516 +ILSVRC2012_val_00029928.JPEG:n03291819 +ILSVRC2012_val_00029929.JPEG:n02100877 +ILSVRC2012_val_00029930.JPEG:n01440764 +ILSVRC2012_val_00029931.JPEG:n04209239 +ILSVRC2012_val_00029932.JPEG:n02088364 +ILSVRC2012_val_00029933.JPEG:n04590129 +ILSVRC2012_val_00029934.JPEG:n02110806 +ILSVRC2012_val_00029935.JPEG:n09229709 +ILSVRC2012_val_00029936.JPEG:n02447366 +ILSVRC2012_val_00029937.JPEG:n04606251 +ILSVRC2012_val_00029938.JPEG:n04562935 +ILSVRC2012_val_00029939.JPEG:n02128385 +ILSVRC2012_val_00029940.JPEG:n02837789 +ILSVRC2012_val_00029941.JPEG:n02363005 +ILSVRC2012_val_00029942.JPEG:n04133789 +ILSVRC2012_val_00029943.JPEG:n02165456 +ILSVRC2012_val_00029944.JPEG:n03649909 +ILSVRC2012_val_00029945.JPEG:n03661043 +ILSVRC2012_val_00029946.JPEG:n02107683 +ILSVRC2012_val_00029947.JPEG:n01688243 +ILSVRC2012_val_00029948.JPEG:n01843383 +ILSVRC2012_val_00029949.JPEG:n03891251 +ILSVRC2012_val_00029950.JPEG:n12620546 +ILSVRC2012_val_00029951.JPEG:n03832673 +ILSVRC2012_val_00029952.JPEG:n03452741 +ILSVRC2012_val_00029953.JPEG:n04074963 +ILSVRC2012_val_00029954.JPEG:n04228054 +ILSVRC2012_val_00029955.JPEG:n03982430 +ILSVRC2012_val_00029956.JPEG:n01795545 +ILSVRC2012_val_00029957.JPEG:n02877765 +ILSVRC2012_val_00029958.JPEG:n03196217 +ILSVRC2012_val_00029959.JPEG:n04435653 +ILSVRC2012_val_00029960.JPEG:n02105505 +ILSVRC2012_val_00029961.JPEG:n04467665 +ILSVRC2012_val_00029962.JPEG:n07695742 +ILSVRC2012_val_00029963.JPEG:n02672831 +ILSVRC2012_val_00029964.JPEG:n03690938 +ILSVRC2012_val_00029965.JPEG:n04456115 +ILSVRC2012_val_00029966.JPEG:n04125021 +ILSVRC2012_val_00029967.JPEG:n15075141 +ILSVRC2012_val_00029968.JPEG:n03761084 +ILSVRC2012_val_00029969.JPEG:n04487394 +ILSVRC2012_val_00029970.JPEG:n02108089 +ILSVRC2012_val_00029971.JPEG:n07932039 +ILSVRC2012_val_00029972.JPEG:n01806567 +ILSVRC2012_val_00029973.JPEG:n02089078 +ILSVRC2012_val_00029974.JPEG:n02028035 +ILSVRC2012_val_00029975.JPEG:n03623198 +ILSVRC2012_val_00029976.JPEG:n02108551 +ILSVRC2012_val_00029977.JPEG:n01632458 +ILSVRC2012_val_00029978.JPEG:n03445924 +ILSVRC2012_val_00029979.JPEG:n01739381 +ILSVRC2012_val_00029980.JPEG:n03887697 +ILSVRC2012_val_00029981.JPEG:n07836838 +ILSVRC2012_val_00029982.JPEG:n02364673 +ILSVRC2012_val_00029983.JPEG:n03355925 +ILSVRC2012_val_00029984.JPEG:n02113799 +ILSVRC2012_val_00029985.JPEG:n04476259 +ILSVRC2012_val_00029986.JPEG:n02437312 +ILSVRC2012_val_00029987.JPEG:n03534580 +ILSVRC2012_val_00029988.JPEG:n03841143 +ILSVRC2012_val_00029989.JPEG:n03131574 +ILSVRC2012_val_00029990.JPEG:n07697537 +ILSVRC2012_val_00029991.JPEG:n01818515 +ILSVRC2012_val_00029992.JPEG:n03929660 +ILSVRC2012_val_00029993.JPEG:n02093647 +ILSVRC2012_val_00029994.JPEG:n02892767 +ILSVRC2012_val_00029995.JPEG:n03916031 +ILSVRC2012_val_00029996.JPEG:n04081281 +ILSVRC2012_val_00029997.JPEG:n04443257 +ILSVRC2012_val_00029998.JPEG:n02441942 +ILSVRC2012_val_00029999.JPEG:n01534433 +ILSVRC2012_val_00030000.JPEG:n01843383 +ILSVRC2012_val_00030001.JPEG:n02951358 +ILSVRC2012_val_00030002.JPEG:n02089078 +ILSVRC2012_val_00030003.JPEG:n03874293 +ILSVRC2012_val_00030004.JPEG:n03127925 +ILSVRC2012_val_00030005.JPEG:n02094258 +ILSVRC2012_val_00030006.JPEG:n04366367 +ILSVRC2012_val_00030007.JPEG:n03485407 +ILSVRC2012_val_00030008.JPEG:n04597913 +ILSVRC2012_val_00030009.JPEG:n01755581 +ILSVRC2012_val_00030010.JPEG:n01795545 +ILSVRC2012_val_00030011.JPEG:n01601694 +ILSVRC2012_val_00030012.JPEG:n01944390 +ILSVRC2012_val_00030013.JPEG:n03124170 +ILSVRC2012_val_00030014.JPEG:n02395406 +ILSVRC2012_val_00030015.JPEG:n03594734 +ILSVRC2012_val_00030016.JPEG:n01685808 +ILSVRC2012_val_00030017.JPEG:n01582220 +ILSVRC2012_val_00030018.JPEG:n02110627 +ILSVRC2012_val_00030019.JPEG:n03991062 +ILSVRC2012_val_00030020.JPEG:n02699494 +ILSVRC2012_val_00030021.JPEG:n09472597 +ILSVRC2012_val_00030022.JPEG:n02500267 +ILSVRC2012_val_00030023.JPEG:n03476991 +ILSVRC2012_val_00030024.JPEG:n02963159 +ILSVRC2012_val_00030025.JPEG:n02089867 +ILSVRC2012_val_00030026.JPEG:n01697457 +ILSVRC2012_val_00030027.JPEG:n03347037 +ILSVRC2012_val_00030028.JPEG:n01806143 +ILSVRC2012_val_00030029.JPEG:n02074367 +ILSVRC2012_val_00030030.JPEG:n02699494 +ILSVRC2012_val_00030031.JPEG:n04090263 +ILSVRC2012_val_00030032.JPEG:n03763968 +ILSVRC2012_val_00030033.JPEG:n02422699 +ILSVRC2012_val_00030034.JPEG:n04070727 +ILSVRC2012_val_00030035.JPEG:n01694178 +ILSVRC2012_val_00030036.JPEG:n01797886 +ILSVRC2012_val_00030037.JPEG:n03459775 +ILSVRC2012_val_00030038.JPEG:n03977966 +ILSVRC2012_val_00030039.JPEG:n01751748 +ILSVRC2012_val_00030040.JPEG:n03803284 +ILSVRC2012_val_00030041.JPEG:n01950731 +ILSVRC2012_val_00030042.JPEG:n01532829 +ILSVRC2012_val_00030043.JPEG:n02454379 +ILSVRC2012_val_00030044.JPEG:n02051845 +ILSVRC2012_val_00030045.JPEG:n03976657 +ILSVRC2012_val_00030046.JPEG:n07248320 +ILSVRC2012_val_00030047.JPEG:n07753275 +ILSVRC2012_val_00030048.JPEG:n09332890 +ILSVRC2012_val_00030049.JPEG:n02002556 +ILSVRC2012_val_00030050.JPEG:n03602883 +ILSVRC2012_val_00030051.JPEG:n12057211 +ILSVRC2012_val_00030052.JPEG:n02123045 +ILSVRC2012_val_00030053.JPEG:n02950826 +ILSVRC2012_val_00030054.JPEG:n02219486 +ILSVRC2012_val_00030055.JPEG:n02115641 +ILSVRC2012_val_00030056.JPEG:n02085936 +ILSVRC2012_val_00030057.JPEG:n02951585 +ILSVRC2012_val_00030058.JPEG:n02111889 +ILSVRC2012_val_00030059.JPEG:n02102480 +ILSVRC2012_val_00030060.JPEG:n01443537 +ILSVRC2012_val_00030061.JPEG:n02105162 +ILSVRC2012_val_00030062.JPEG:n02794156 +ILSVRC2012_val_00030063.JPEG:n04479046 +ILSVRC2012_val_00030064.JPEG:n03047690 +ILSVRC2012_val_00030065.JPEG:n02105412 +ILSVRC2012_val_00030066.JPEG:n02692877 +ILSVRC2012_val_00030067.JPEG:n01739381 +ILSVRC2012_val_00030068.JPEG:n07930864 +ILSVRC2012_val_00030069.JPEG:n04552348 +ILSVRC2012_val_00030070.JPEG:n02835271 +ILSVRC2012_val_00030071.JPEG:n01531178 +ILSVRC2012_val_00030072.JPEG:n04120489 +ILSVRC2012_val_00030073.JPEG:n01582220 +ILSVRC2012_val_00030074.JPEG:n02840245 +ILSVRC2012_val_00030075.JPEG:n02422106 +ILSVRC2012_val_00030076.JPEG:n01697457 +ILSVRC2012_val_00030077.JPEG:n03075370 +ILSVRC2012_val_00030078.JPEG:n04136333 +ILSVRC2012_val_00030079.JPEG:n03874599 +ILSVRC2012_val_00030080.JPEG:n03492542 +ILSVRC2012_val_00030081.JPEG:n02389026 +ILSVRC2012_val_00030082.JPEG:n03207743 +ILSVRC2012_val_00030083.JPEG:n02089867 +ILSVRC2012_val_00030084.JPEG:n04136333 +ILSVRC2012_val_00030085.JPEG:n06359193 +ILSVRC2012_val_00030086.JPEG:n02106382 +ILSVRC2012_val_00030087.JPEG:n02101006 +ILSVRC2012_val_00030088.JPEG:n02091467 +ILSVRC2012_val_00030089.JPEG:n03325584 +ILSVRC2012_val_00030090.JPEG:n01616318 +ILSVRC2012_val_00030091.JPEG:n02804610 +ILSVRC2012_val_00030092.JPEG:n07717556 +ILSVRC2012_val_00030093.JPEG:n02111500 +ILSVRC2012_val_00030094.JPEG:n01608432 +ILSVRC2012_val_00030095.JPEG:n02007558 +ILSVRC2012_val_00030096.JPEG:n03887697 +ILSVRC2012_val_00030097.JPEG:n02107142 +ILSVRC2012_val_00030098.JPEG:n02641379 +ILSVRC2012_val_00030099.JPEG:n07734744 +ILSVRC2012_val_00030100.JPEG:n03710193 +ILSVRC2012_val_00030101.JPEG:n02231487 +ILSVRC2012_val_00030102.JPEG:n02028035 +ILSVRC2012_val_00030103.JPEG:n04296562 +ILSVRC2012_val_00030104.JPEG:n04009552 +ILSVRC2012_val_00030105.JPEG:n02977058 +ILSVRC2012_val_00030106.JPEG:n03710721 +ILSVRC2012_val_00030107.JPEG:n03884397 +ILSVRC2012_val_00030108.JPEG:n03775546 +ILSVRC2012_val_00030109.JPEG:n07892512 +ILSVRC2012_val_00030110.JPEG:n04254777 +ILSVRC2012_val_00030111.JPEG:n07697537 +ILSVRC2012_val_00030112.JPEG:n03792782 +ILSVRC2012_val_00030113.JPEG:n02102480 +ILSVRC2012_val_00030114.JPEG:n03000247 +ILSVRC2012_val_00030115.JPEG:n02117135 +ILSVRC2012_val_00030116.JPEG:n01796340 +ILSVRC2012_val_00030117.JPEG:n02892201 +ILSVRC2012_val_00030118.JPEG:n04254680 +ILSVRC2012_val_00030119.JPEG:n04040759 +ILSVRC2012_val_00030120.JPEG:n01773549 +ILSVRC2012_val_00030121.JPEG:n04040759 +ILSVRC2012_val_00030122.JPEG:n03124170 +ILSVRC2012_val_00030123.JPEG:n02790996 +ILSVRC2012_val_00030124.JPEG:n04037443 +ILSVRC2012_val_00030125.JPEG:n02033041 +ILSVRC2012_val_00030126.JPEG:n04509417 +ILSVRC2012_val_00030127.JPEG:n01484850 +ILSVRC2012_val_00030128.JPEG:n03697007 +ILSVRC2012_val_00030129.JPEG:n04208210 +ILSVRC2012_val_00030130.JPEG:n04209133 +ILSVRC2012_val_00030131.JPEG:n02497673 +ILSVRC2012_val_00030132.JPEG:n03840681 +ILSVRC2012_val_00030133.JPEG:n03785016 +ILSVRC2012_val_00030134.JPEG:n04086273 +ILSVRC2012_val_00030135.JPEG:n02085936 +ILSVRC2012_val_00030136.JPEG:n02134084 +ILSVRC2012_val_00030137.JPEG:n03404251 +ILSVRC2012_val_00030138.JPEG:n02098286 +ILSVRC2012_val_00030139.JPEG:n07734744 +ILSVRC2012_val_00030140.JPEG:n03998194 +ILSVRC2012_val_00030141.JPEG:n02086910 +ILSVRC2012_val_00030142.JPEG:n03250847 +ILSVRC2012_val_00030143.JPEG:n03983396 +ILSVRC2012_val_00030144.JPEG:n04336792 +ILSVRC2012_val_00030145.JPEG:n03457902 +ILSVRC2012_val_00030146.JPEG:n03026506 +ILSVRC2012_val_00030147.JPEG:n03980874 +ILSVRC2012_val_00030148.JPEG:n01818515 +ILSVRC2012_val_00030149.JPEG:n04507155 +ILSVRC2012_val_00030150.JPEG:n03933933 +ILSVRC2012_val_00030151.JPEG:n13037406 +ILSVRC2012_val_00030152.JPEG:n04235860 +ILSVRC2012_val_00030153.JPEG:n02504013 +ILSVRC2012_val_00030154.JPEG:n03297495 +ILSVRC2012_val_00030155.JPEG:n02802426 +ILSVRC2012_val_00030156.JPEG:n01491361 +ILSVRC2012_val_00030157.JPEG:n02916936 +ILSVRC2012_val_00030158.JPEG:n01755581 +ILSVRC2012_val_00030159.JPEG:n02727426 +ILSVRC2012_val_00030160.JPEG:n04228054 +ILSVRC2012_val_00030161.JPEG:n03584254 +ILSVRC2012_val_00030162.JPEG:n04317175 +ILSVRC2012_val_00030163.JPEG:n01667114 +ILSVRC2012_val_00030164.JPEG:n04486054 +ILSVRC2012_val_00030165.JPEG:n02110341 +ILSVRC2012_val_00030166.JPEG:n04465501 +ILSVRC2012_val_00030167.JPEG:n02974003 +ILSVRC2012_val_00030168.JPEG:n12768682 +ILSVRC2012_val_00030169.JPEG:n12998815 +ILSVRC2012_val_00030170.JPEG:n02111129 +ILSVRC2012_val_00030171.JPEG:n11879895 +ILSVRC2012_val_00030172.JPEG:n03775546 +ILSVRC2012_val_00030173.JPEG:n03496892 +ILSVRC2012_val_00030174.JPEG:n03791053 +ILSVRC2012_val_00030175.JPEG:n01768244 +ILSVRC2012_val_00030176.JPEG:n09421951 +ILSVRC2012_val_00030177.JPEG:n04192698 +ILSVRC2012_val_00030178.JPEG:n04517823 +ILSVRC2012_val_00030179.JPEG:n02514041 +ILSVRC2012_val_00030180.JPEG:n12985857 +ILSVRC2012_val_00030181.JPEG:n13054560 +ILSVRC2012_val_00030182.JPEG:n04330267 +ILSVRC2012_val_00030183.JPEG:n03388549 +ILSVRC2012_val_00030184.JPEG:n04254120 +ILSVRC2012_val_00030185.JPEG:n04423845 +ILSVRC2012_val_00030186.JPEG:n11879895 +ILSVRC2012_val_00030187.JPEG:n02776631 +ILSVRC2012_val_00030188.JPEG:n02137549 +ILSVRC2012_val_00030189.JPEG:n03495258 +ILSVRC2012_val_00030190.JPEG:n03355925 +ILSVRC2012_val_00030191.JPEG:n02486410 +ILSVRC2012_val_00030192.JPEG:n02749479 +ILSVRC2012_val_00030193.JPEG:n03187595 +ILSVRC2012_val_00030194.JPEG:n03388043 +ILSVRC2012_val_00030195.JPEG:n04005630 +ILSVRC2012_val_00030196.JPEG:n02100877 +ILSVRC2012_val_00030197.JPEG:n07714990 +ILSVRC2012_val_00030198.JPEG:n06359193 +ILSVRC2012_val_00030199.JPEG:n02096051 +ILSVRC2012_val_00030200.JPEG:n02105641 +ILSVRC2012_val_00030201.JPEG:n07579787 +ILSVRC2012_val_00030202.JPEG:n09472597 +ILSVRC2012_val_00030203.JPEG:n04355338 +ILSVRC2012_val_00030204.JPEG:n03680355 +ILSVRC2012_val_00030205.JPEG:n02730930 +ILSVRC2012_val_00030206.JPEG:n03874599 +ILSVRC2012_val_00030207.JPEG:n02730930 +ILSVRC2012_val_00030208.JPEG:n04552348 +ILSVRC2012_val_00030209.JPEG:n03535780 +ILSVRC2012_val_00030210.JPEG:n01753488 +ILSVRC2012_val_00030211.JPEG:n02012849 +ILSVRC2012_val_00030212.JPEG:n01704323 +ILSVRC2012_val_00030213.JPEG:n02097209 +ILSVRC2012_val_00030214.JPEG:n03908714 +ILSVRC2012_val_00030215.JPEG:n04589890 +ILSVRC2012_val_00030216.JPEG:n04372370 +ILSVRC2012_val_00030217.JPEG:n01443537 +ILSVRC2012_val_00030218.JPEG:n03457902 +ILSVRC2012_val_00030219.JPEG:n04238763 +ILSVRC2012_val_00030220.JPEG:n09246464 +ILSVRC2012_val_00030221.JPEG:n01739381 +ILSVRC2012_val_00030222.JPEG:n02488702 +ILSVRC2012_val_00030223.JPEG:n04026417 +ILSVRC2012_val_00030224.JPEG:n01530575 +ILSVRC2012_val_00030225.JPEG:n07749582 +ILSVRC2012_val_00030226.JPEG:n02102480 +ILSVRC2012_val_00030227.JPEG:n04557648 +ILSVRC2012_val_00030228.JPEG:n02096585 +ILSVRC2012_val_00030229.JPEG:n01740131 +ILSVRC2012_val_00030230.JPEG:n04389033 +ILSVRC2012_val_00030231.JPEG:n03314780 +ILSVRC2012_val_00030232.JPEG:n07875152 +ILSVRC2012_val_00030233.JPEG:n02492660 +ILSVRC2012_val_00030234.JPEG:n12057211 +ILSVRC2012_val_00030235.JPEG:n04371430 +ILSVRC2012_val_00030236.JPEG:n02099267 +ILSVRC2012_val_00030237.JPEG:n03495258 +ILSVRC2012_val_00030238.JPEG:n02096051 +ILSVRC2012_val_00030239.JPEG:n02105162 +ILSVRC2012_val_00030240.JPEG:n02105641 +ILSVRC2012_val_00030241.JPEG:n03016953 +ILSVRC2012_val_00030242.JPEG:n02808440 +ILSVRC2012_val_00030243.JPEG:n03598930 +ILSVRC2012_val_00030244.JPEG:n04542943 +ILSVRC2012_val_00030245.JPEG:n01855672 +ILSVRC2012_val_00030246.JPEG:n03733281 +ILSVRC2012_val_00030247.JPEG:n07717410 +ILSVRC2012_val_00030248.JPEG:n02504013 +ILSVRC2012_val_00030249.JPEG:n02091831 +ILSVRC2012_val_00030250.JPEG:n04133789 +ILSVRC2012_val_00030251.JPEG:n04356056 +ILSVRC2012_val_00030252.JPEG:n02879718 +ILSVRC2012_val_00030253.JPEG:n03891251 +ILSVRC2012_val_00030254.JPEG:n03379051 +ILSVRC2012_val_00030255.JPEG:n02113978 +ILSVRC2012_val_00030256.JPEG:n09288635 +ILSVRC2012_val_00030257.JPEG:n02444819 +ILSVRC2012_val_00030258.JPEG:n01945685 +ILSVRC2012_val_00030259.JPEG:n03980874 +ILSVRC2012_val_00030260.JPEG:n02526121 +ILSVRC2012_val_00030261.JPEG:n02101556 +ILSVRC2012_val_00030262.JPEG:n04040759 +ILSVRC2012_val_00030263.JPEG:n02009229 +ILSVRC2012_val_00030264.JPEG:n03837869 +ILSVRC2012_val_00030265.JPEG:n04311174 +ILSVRC2012_val_00030266.JPEG:n07583066 +ILSVRC2012_val_00030267.JPEG:n02777292 +ILSVRC2012_val_00030268.JPEG:n03950228 +ILSVRC2012_val_00030269.JPEG:n02129165 +ILSVRC2012_val_00030270.JPEG:n02114548 +ILSVRC2012_val_00030271.JPEG:n02100735 +ILSVRC2012_val_00030272.JPEG:n04590129 +ILSVRC2012_val_00030273.JPEG:n03400231 +ILSVRC2012_val_00030274.JPEG:n03868242 +ILSVRC2012_val_00030275.JPEG:n02074367 +ILSVRC2012_val_00030276.JPEG:n06874185 +ILSVRC2012_val_00030277.JPEG:n04141327 +ILSVRC2012_val_00030278.JPEG:n01833805 +ILSVRC2012_val_00030279.JPEG:n09288635 +ILSVRC2012_val_00030280.JPEG:n04070727 +ILSVRC2012_val_00030281.JPEG:n02795169 +ILSVRC2012_val_00030282.JPEG:n03944341 +ILSVRC2012_val_00030283.JPEG:n01560419 +ILSVRC2012_val_00030284.JPEG:n03187595 +ILSVRC2012_val_00030285.JPEG:n02092339 +ILSVRC2012_val_00030286.JPEG:n03388043 +ILSVRC2012_val_00030287.JPEG:n03255030 +ILSVRC2012_val_00030288.JPEG:n04532670 +ILSVRC2012_val_00030289.JPEG:n02120505 +ILSVRC2012_val_00030290.JPEG:n02894605 +ILSVRC2012_val_00030291.JPEG:n02101388 +ILSVRC2012_val_00030292.JPEG:n01608432 +ILSVRC2012_val_00030293.JPEG:n03995372 +ILSVRC2012_val_00030294.JPEG:n02259212 +ILSVRC2012_val_00030295.JPEG:n03908618 +ILSVRC2012_val_00030296.JPEG:n03223299 +ILSVRC2012_val_00030297.JPEG:n02107683 +ILSVRC2012_val_00030298.JPEG:n07932039 +ILSVRC2012_val_00030299.JPEG:n03063689 +ILSVRC2012_val_00030300.JPEG:n01629819 +ILSVRC2012_val_00030301.JPEG:n03982430 +ILSVRC2012_val_00030302.JPEG:n03188531 +ILSVRC2012_val_00030303.JPEG:n01748264 +ILSVRC2012_val_00030304.JPEG:n03877472 +ILSVRC2012_val_00030305.JPEG:n02115913 +ILSVRC2012_val_00030306.JPEG:n01748264 +ILSVRC2012_val_00030307.JPEG:n04350905 +ILSVRC2012_val_00030308.JPEG:n04070727 +ILSVRC2012_val_00030309.JPEG:n02643566 +ILSVRC2012_val_00030310.JPEG:n02966193 +ILSVRC2012_val_00030311.JPEG:n01770393 +ILSVRC2012_val_00030312.JPEG:n02672831 +ILSVRC2012_val_00030313.JPEG:n02494079 +ILSVRC2012_val_00030314.JPEG:n02930766 +ILSVRC2012_val_00030315.JPEG:n03259280 +ILSVRC2012_val_00030316.JPEG:n02442845 +ILSVRC2012_val_00030317.JPEG:n03903868 +ILSVRC2012_val_00030318.JPEG:n03710721 +ILSVRC2012_val_00030319.JPEG:n02690373 +ILSVRC2012_val_00030320.JPEG:n01531178 +ILSVRC2012_val_00030321.JPEG:n01496331 +ILSVRC2012_val_00030322.JPEG:n03710721 +ILSVRC2012_val_00030323.JPEG:n02088094 +ILSVRC2012_val_00030324.JPEG:n07717556 +ILSVRC2012_val_00030325.JPEG:n03920288 +ILSVRC2012_val_00030326.JPEG:n02089078 +ILSVRC2012_val_00030327.JPEG:n02109525 +ILSVRC2012_val_00030328.JPEG:n02808304 +ILSVRC2012_val_00030329.JPEG:n03447447 +ILSVRC2012_val_00030330.JPEG:n04548280 +ILSVRC2012_val_00030331.JPEG:n02906734 +ILSVRC2012_val_00030332.JPEG:n07716358 +ILSVRC2012_val_00030333.JPEG:n01774384 +ILSVRC2012_val_00030334.JPEG:n03637318 +ILSVRC2012_val_00030335.JPEG:n02909870 +ILSVRC2012_val_00030336.JPEG:n03788195 +ILSVRC2012_val_00030337.JPEG:n02699494 +ILSVRC2012_val_00030338.JPEG:n04355338 +ILSVRC2012_val_00030339.JPEG:n02095889 +ILSVRC2012_val_00030340.JPEG:n02606052 +ILSVRC2012_val_00030341.JPEG:n03623198 +ILSVRC2012_val_00030342.JPEG:n01641577 +ILSVRC2012_val_00030343.JPEG:n01669191 +ILSVRC2012_val_00030344.JPEG:n02457408 +ILSVRC2012_val_00030345.JPEG:n03627232 +ILSVRC2012_val_00030346.JPEG:n02769748 +ILSVRC2012_val_00030347.JPEG:n04311004 +ILSVRC2012_val_00030348.JPEG:n03584254 +ILSVRC2012_val_00030349.JPEG:n03220513 +ILSVRC2012_val_00030350.JPEG:n03530642 +ILSVRC2012_val_00030351.JPEG:n04285008 +ILSVRC2012_val_00030352.JPEG:n01644373 +ILSVRC2012_val_00030353.JPEG:n09421951 +ILSVRC2012_val_00030354.JPEG:n03733281 +ILSVRC2012_val_00030355.JPEG:n03047690 +ILSVRC2012_val_00030356.JPEG:n02808304 +ILSVRC2012_val_00030357.JPEG:n03720891 +ILSVRC2012_val_00030358.JPEG:n02437616 +ILSVRC2012_val_00030359.JPEG:n07684084 +ILSVRC2012_val_00030360.JPEG:n01749939 +ILSVRC2012_val_00030361.JPEG:n04409515 +ILSVRC2012_val_00030362.JPEG:n02494079 +ILSVRC2012_val_00030363.JPEG:n02948072 +ILSVRC2012_val_00030364.JPEG:n02110806 +ILSVRC2012_val_00030365.JPEG:n02077923 +ILSVRC2012_val_00030366.JPEG:n01924916 +ILSVRC2012_val_00030367.JPEG:n01496331 +ILSVRC2012_val_00030368.JPEG:n04604644 +ILSVRC2012_val_00030369.JPEG:n02667093 +ILSVRC2012_val_00030370.JPEG:n02107142 +ILSVRC2012_val_00030371.JPEG:n01692333 +ILSVRC2012_val_00030372.JPEG:n04277352 +ILSVRC2012_val_00030373.JPEG:n04254777 +ILSVRC2012_val_00030374.JPEG:n02676566 +ILSVRC2012_val_00030375.JPEG:n12144580 +ILSVRC2012_val_00030376.JPEG:n03630383 +ILSVRC2012_val_00030377.JPEG:n02095889 +ILSVRC2012_val_00030378.JPEG:n03666591 +ILSVRC2012_val_00030379.JPEG:n03937543 +ILSVRC2012_val_00030380.JPEG:n01498041 +ILSVRC2012_val_00030381.JPEG:n03272562 +ILSVRC2012_val_00030382.JPEG:n09472597 +ILSVRC2012_val_00030383.JPEG:n03223299 +ILSVRC2012_val_00030384.JPEG:n04456115 +ILSVRC2012_val_00030385.JPEG:n02099601 +ILSVRC2012_val_00030386.JPEG:n03000134 +ILSVRC2012_val_00030387.JPEG:n02951585 +ILSVRC2012_val_00030388.JPEG:n03717622 +ILSVRC2012_val_00030389.JPEG:n01910747 +ILSVRC2012_val_00030390.JPEG:n06596364 +ILSVRC2012_val_00030391.JPEG:n01820546 +ILSVRC2012_val_00030392.JPEG:n02018795 +ILSVRC2012_val_00030393.JPEG:n04264628 +ILSVRC2012_val_00030394.JPEG:n02096177 +ILSVRC2012_val_00030395.JPEG:n01944390 +ILSVRC2012_val_00030396.JPEG:n01978287 +ILSVRC2012_val_00030397.JPEG:n01818515 +ILSVRC2012_val_00030398.JPEG:n03125729 +ILSVRC2012_val_00030399.JPEG:n02093256 +ILSVRC2012_val_00030400.JPEG:n01855032 +ILSVRC2012_val_00030401.JPEG:n02009912 +ILSVRC2012_val_00030402.JPEG:n02097047 +ILSVRC2012_val_00030403.JPEG:n02113712 +ILSVRC2012_val_00030404.JPEG:n01883070 +ILSVRC2012_val_00030405.JPEG:n01774750 +ILSVRC2012_val_00030406.JPEG:n01665541 +ILSVRC2012_val_00030407.JPEG:n02093428 +ILSVRC2012_val_00030408.JPEG:n01980166 +ILSVRC2012_val_00030409.JPEG:n04392985 +ILSVRC2012_val_00030410.JPEG:n03947888 +ILSVRC2012_val_00030411.JPEG:n02690373 +ILSVRC2012_val_00030412.JPEG:n02090721 +ILSVRC2012_val_00030413.JPEG:n04023962 +ILSVRC2012_val_00030414.JPEG:n03476684 +ILSVRC2012_val_00030415.JPEG:n04389033 +ILSVRC2012_val_00030416.JPEG:n03729826 +ILSVRC2012_val_00030417.JPEG:n02910353 +ILSVRC2012_val_00030418.JPEG:n01632458 +ILSVRC2012_val_00030419.JPEG:n02167151 +ILSVRC2012_val_00030420.JPEG:n02676566 +ILSVRC2012_val_00030421.JPEG:n03045698 +ILSVRC2012_val_00030422.JPEG:n01770081 +ILSVRC2012_val_00030423.JPEG:n04238763 +ILSVRC2012_val_00030424.JPEG:n10148035 +ILSVRC2012_val_00030425.JPEG:n04344873 +ILSVRC2012_val_00030426.JPEG:n02481823 +ILSVRC2012_val_00030427.JPEG:n04467665 +ILSVRC2012_val_00030428.JPEG:n02013706 +ILSVRC2012_val_00030429.JPEG:n02088238 +ILSVRC2012_val_00030430.JPEG:n02877765 +ILSVRC2012_val_00030431.JPEG:n01833805 +ILSVRC2012_val_00030432.JPEG:n07718747 +ILSVRC2012_val_00030433.JPEG:n02091467 +ILSVRC2012_val_00030434.JPEG:n03627232 +ILSVRC2012_val_00030435.JPEG:n04141076 +ILSVRC2012_val_00030436.JPEG:n04209239 +ILSVRC2012_val_00030437.JPEG:n01950731 +ILSVRC2012_val_00030438.JPEG:n04467665 +ILSVRC2012_val_00030439.JPEG:n03976657 +ILSVRC2012_val_00030440.JPEG:n03729826 +ILSVRC2012_val_00030441.JPEG:n04398044 +ILSVRC2012_val_00030442.JPEG:n07754684 +ILSVRC2012_val_00030443.JPEG:n04465501 +ILSVRC2012_val_00030444.JPEG:n01776313 +ILSVRC2012_val_00030445.JPEG:n02111129 +ILSVRC2012_val_00030446.JPEG:n03207743 +ILSVRC2012_val_00030447.JPEG:n03201208 +ILSVRC2012_val_00030448.JPEG:n01847000 +ILSVRC2012_val_00030449.JPEG:n02085936 +ILSVRC2012_val_00030450.JPEG:n03710721 +ILSVRC2012_val_00030451.JPEG:n04599235 +ILSVRC2012_val_00030452.JPEG:n02817516 +ILSVRC2012_val_00030453.JPEG:n02807133 +ILSVRC2012_val_00030454.JPEG:n04389033 +ILSVRC2012_val_00030455.JPEG:n02840245 +ILSVRC2012_val_00030456.JPEG:n04423845 +ILSVRC2012_val_00030457.JPEG:n07718472 +ILSVRC2012_val_00030458.JPEG:n02356798 +ILSVRC2012_val_00030459.JPEG:n02167151 +ILSVRC2012_val_00030460.JPEG:n02966687 +ILSVRC2012_val_00030461.JPEG:n02790996 +ILSVRC2012_val_00030462.JPEG:n02840245 +ILSVRC2012_val_00030463.JPEG:n02342885 +ILSVRC2012_val_00030464.JPEG:n02437312 +ILSVRC2012_val_00030465.JPEG:n07716906 +ILSVRC2012_val_00030466.JPEG:n02233338 +ILSVRC2012_val_00030467.JPEG:n03379051 +ILSVRC2012_val_00030468.JPEG:n01990800 +ILSVRC2012_val_00030469.JPEG:n02443114 +ILSVRC2012_val_00030470.JPEG:n01498041 +ILSVRC2012_val_00030471.JPEG:n03337140 +ILSVRC2012_val_00030472.JPEG:n02165105 +ILSVRC2012_val_00030473.JPEG:n04525305 +ILSVRC2012_val_00030474.JPEG:n02226429 +ILSVRC2012_val_00030475.JPEG:n01558993 +ILSVRC2012_val_00030476.JPEG:n02110341 +ILSVRC2012_val_00030477.JPEG:n04069434 +ILSVRC2012_val_00030478.JPEG:n01644900 +ILSVRC2012_val_00030479.JPEG:n02096177 +ILSVRC2012_val_00030480.JPEG:n04347754 +ILSVRC2012_val_00030481.JPEG:n03127747 +ILSVRC2012_val_00030482.JPEG:n02106382 +ILSVRC2012_val_00030483.JPEG:n01608432 +ILSVRC2012_val_00030484.JPEG:n02412080 +ILSVRC2012_val_00030485.JPEG:n02134084 +ILSVRC2012_val_00030486.JPEG:n04486054 +ILSVRC2012_val_00030487.JPEG:n04026417 +ILSVRC2012_val_00030488.JPEG:n02437616 +ILSVRC2012_val_00030489.JPEG:n04081281 +ILSVRC2012_val_00030490.JPEG:n04417672 +ILSVRC2012_val_00030491.JPEG:n02018207 +ILSVRC2012_val_00030492.JPEG:n03018349 +ILSVRC2012_val_00030493.JPEG:n03595614 +ILSVRC2012_val_00030494.JPEG:n02120079 +ILSVRC2012_val_00030495.JPEG:n03388183 +ILSVRC2012_val_00030496.JPEG:n03902125 +ILSVRC2012_val_00030497.JPEG:n02403003 +ILSVRC2012_val_00030498.JPEG:n03933933 +ILSVRC2012_val_00030499.JPEG:n09193705 +ILSVRC2012_val_00030500.JPEG:n01872401 +ILSVRC2012_val_00030501.JPEG:n03534580 +ILSVRC2012_val_00030502.JPEG:n02129165 +ILSVRC2012_val_00030503.JPEG:n03710193 +ILSVRC2012_val_00030504.JPEG:n01981276 +ILSVRC2012_val_00030505.JPEG:n02259212 +ILSVRC2012_val_00030506.JPEG:n07873807 +ILSVRC2012_val_00030507.JPEG:n01843065 +ILSVRC2012_val_00030508.JPEG:n02457408 +ILSVRC2012_val_00030509.JPEG:n02837789 +ILSVRC2012_val_00030510.JPEG:n02177972 +ILSVRC2012_val_00030511.JPEG:n02951585 +ILSVRC2012_val_00030512.JPEG:n02101006 +ILSVRC2012_val_00030513.JPEG:n02965783 +ILSVRC2012_val_00030514.JPEG:n04482393 +ILSVRC2012_val_00030515.JPEG:n01616318 +ILSVRC2012_val_00030516.JPEG:n04465501 +ILSVRC2012_val_00030517.JPEG:n03485407 +ILSVRC2012_val_00030518.JPEG:n02086646 +ILSVRC2012_val_00030519.JPEG:n02085620 +ILSVRC2012_val_00030520.JPEG:n02361337 +ILSVRC2012_val_00030521.JPEG:n01753488 +ILSVRC2012_val_00030522.JPEG:n04579145 +ILSVRC2012_val_00030523.JPEG:n01682714 +ILSVRC2012_val_00030524.JPEG:n02105641 +ILSVRC2012_val_00030525.JPEG:n04065272 +ILSVRC2012_val_00030526.JPEG:n01968897 +ILSVRC2012_val_00030527.JPEG:n02102973 +ILSVRC2012_val_00030528.JPEG:n12144580 +ILSVRC2012_val_00030529.JPEG:n04372370 +ILSVRC2012_val_00030530.JPEG:n02127052 +ILSVRC2012_val_00030531.JPEG:n02690373 +ILSVRC2012_val_00030532.JPEG:n02895154 +ILSVRC2012_val_00030533.JPEG:n04049303 +ILSVRC2012_val_00030534.JPEG:n03676483 +ILSVRC2012_val_00030535.JPEG:n02268443 +ILSVRC2012_val_00030536.JPEG:n02869837 +ILSVRC2012_val_00030537.JPEG:n02206856 +ILSVRC2012_val_00030538.JPEG:n04201297 +ILSVRC2012_val_00030539.JPEG:n02091244 +ILSVRC2012_val_00030540.JPEG:n02101556 +ILSVRC2012_val_00030541.JPEG:n02843684 +ILSVRC2012_val_00030542.JPEG:n04380533 +ILSVRC2012_val_00030543.JPEG:n07753275 +ILSVRC2012_val_00030544.JPEG:n01534433 +ILSVRC2012_val_00030545.JPEG:n02027492 +ILSVRC2012_val_00030546.JPEG:n02971356 +ILSVRC2012_val_00030547.JPEG:n04118538 +ILSVRC2012_val_00030548.JPEG:n03384352 +ILSVRC2012_val_00030549.JPEG:n03444034 +ILSVRC2012_val_00030550.JPEG:n03676483 +ILSVRC2012_val_00030551.JPEG:n03495258 +ILSVRC2012_val_00030552.JPEG:n02666196 +ILSVRC2012_val_00030553.JPEG:n01756291 +ILSVRC2012_val_00030554.JPEG:n03482405 +ILSVRC2012_val_00030555.JPEG:n02098413 +ILSVRC2012_val_00030556.JPEG:n04355933 +ILSVRC2012_val_00030557.JPEG:n03841143 +ILSVRC2012_val_00030558.JPEG:n02120079 +ILSVRC2012_val_00030559.JPEG:n02417914 +ILSVRC2012_val_00030560.JPEG:n03857828 +ILSVRC2012_val_00030561.JPEG:n02114712 +ILSVRC2012_val_00030562.JPEG:n01729977 +ILSVRC2012_val_00030563.JPEG:n01770081 +ILSVRC2012_val_00030564.JPEG:n03733131 +ILSVRC2012_val_00030565.JPEG:n03793489 +ILSVRC2012_val_00030566.JPEG:n03590841 +ILSVRC2012_val_00030567.JPEG:n02088364 +ILSVRC2012_val_00030568.JPEG:n01847000 +ILSVRC2012_val_00030569.JPEG:n11939491 +ILSVRC2012_val_00030570.JPEG:n03724870 +ILSVRC2012_val_00030571.JPEG:n02025239 +ILSVRC2012_val_00030572.JPEG:n07717556 +ILSVRC2012_val_00030573.JPEG:n02119789 +ILSVRC2012_val_00030574.JPEG:n03016953 +ILSVRC2012_val_00030575.JPEG:n02129165 +ILSVRC2012_val_00030576.JPEG:n04033901 +ILSVRC2012_val_00030577.JPEG:n02790996 +ILSVRC2012_val_00030578.JPEG:n02012849 +ILSVRC2012_val_00030579.JPEG:n02099429 +ILSVRC2012_val_00030580.JPEG:n03691459 +ILSVRC2012_val_00030581.JPEG:n04330267 +ILSVRC2012_val_00030582.JPEG:n10148035 +ILSVRC2012_val_00030583.JPEG:n03888257 +ILSVRC2012_val_00030584.JPEG:n07584110 +ILSVRC2012_val_00030585.JPEG:n02096437 +ILSVRC2012_val_00030586.JPEG:n04515003 +ILSVRC2012_val_00030587.JPEG:n02804610 +ILSVRC2012_val_00030588.JPEG:n02096437 +ILSVRC2012_val_00030589.JPEG:n04418357 +ILSVRC2012_val_00030590.JPEG:n02033041 +ILSVRC2012_val_00030591.JPEG:n02092339 +ILSVRC2012_val_00030592.JPEG:n12620546 +ILSVRC2012_val_00030593.JPEG:n01669191 +ILSVRC2012_val_00030594.JPEG:n03160309 +ILSVRC2012_val_00030595.JPEG:n02112137 +ILSVRC2012_val_00030596.JPEG:n02172182 +ILSVRC2012_val_00030597.JPEG:n03110669 +ILSVRC2012_val_00030598.JPEG:n04380533 +ILSVRC2012_val_00030599.JPEG:n03673027 +ILSVRC2012_val_00030600.JPEG:n03347037 +ILSVRC2012_val_00030601.JPEG:n04201297 +ILSVRC2012_val_00030602.JPEG:n02492660 +ILSVRC2012_val_00030603.JPEG:n02110958 +ILSVRC2012_val_00030604.JPEG:n02783161 +ILSVRC2012_val_00030605.JPEG:n02483708 +ILSVRC2012_val_00030606.JPEG:n02110958 +ILSVRC2012_val_00030607.JPEG:n04120489 +ILSVRC2012_val_00030608.JPEG:n03908618 +ILSVRC2012_val_00030609.JPEG:n02423022 +ILSVRC2012_val_00030610.JPEG:n04350905 +ILSVRC2012_val_00030611.JPEG:n04153751 +ILSVRC2012_val_00030612.JPEG:n02444819 +ILSVRC2012_val_00030613.JPEG:n02114548 +ILSVRC2012_val_00030614.JPEG:n07747607 +ILSVRC2012_val_00030615.JPEG:n07614500 +ILSVRC2012_val_00030616.JPEG:n04070727 +ILSVRC2012_val_00030617.JPEG:n04074963 +ILSVRC2012_val_00030618.JPEG:n01616318 +ILSVRC2012_val_00030619.JPEG:n02112706 +ILSVRC2012_val_00030620.JPEG:n02096437 +ILSVRC2012_val_00030621.JPEG:n04228054 +ILSVRC2012_val_00030622.JPEG:n01644900 +ILSVRC2012_val_00030623.JPEG:n01756291 +ILSVRC2012_val_00030624.JPEG:n02442845 +ILSVRC2012_val_00030625.JPEG:n03980874 +ILSVRC2012_val_00030626.JPEG:n02441942 +ILSVRC2012_val_00030627.JPEG:n04149813 +ILSVRC2012_val_00030628.JPEG:n03950228 +ILSVRC2012_val_00030629.JPEG:n01843383 +ILSVRC2012_val_00030630.JPEG:n02910353 +ILSVRC2012_val_00030631.JPEG:n03207743 +ILSVRC2012_val_00030632.JPEG:n04263257 +ILSVRC2012_val_00030633.JPEG:n02099429 +ILSVRC2012_val_00030634.JPEG:n04486054 +ILSVRC2012_val_00030635.JPEG:n02606052 +ILSVRC2012_val_00030636.JPEG:n04238763 +ILSVRC2012_val_00030637.JPEG:n02099601 +ILSVRC2012_val_00030638.JPEG:n02177972 +ILSVRC2012_val_00030639.JPEG:n03584829 +ILSVRC2012_val_00030640.JPEG:n04356056 +ILSVRC2012_val_00030641.JPEG:n03673027 +ILSVRC2012_val_00030642.JPEG:n02086646 +ILSVRC2012_val_00030643.JPEG:n04485082 +ILSVRC2012_val_00030644.JPEG:n02692877 +ILSVRC2012_val_00030645.JPEG:n03761084 +ILSVRC2012_val_00030646.JPEG:n03249569 +ILSVRC2012_val_00030647.JPEG:n04252077 +ILSVRC2012_val_00030648.JPEG:n02092339 +ILSVRC2012_val_00030649.JPEG:n01770081 +ILSVRC2012_val_00030650.JPEG:n02877765 +ILSVRC2012_val_00030651.JPEG:n02129604 +ILSVRC2012_val_00030652.JPEG:n03032252 +ILSVRC2012_val_00030653.JPEG:n13044778 +ILSVRC2012_val_00030654.JPEG:n02607072 +ILSVRC2012_val_00030655.JPEG:n03498962 +ILSVRC2012_val_00030656.JPEG:n02120505 +ILSVRC2012_val_00030657.JPEG:n01534433 +ILSVRC2012_val_00030658.JPEG:n01491361 +ILSVRC2012_val_00030659.JPEG:n07730033 +ILSVRC2012_val_00030660.JPEG:n02098413 +ILSVRC2012_val_00030661.JPEG:n02793495 +ILSVRC2012_val_00030662.JPEG:n02017213 +ILSVRC2012_val_00030663.JPEG:n02100877 +ILSVRC2012_val_00030664.JPEG:n02948072 +ILSVRC2012_val_00030665.JPEG:n02398521 +ILSVRC2012_val_00030666.JPEG:n03498962 +ILSVRC2012_val_00030667.JPEG:n02494079 +ILSVRC2012_val_00030668.JPEG:n04026417 +ILSVRC2012_val_00030669.JPEG:n03259280 +ILSVRC2012_val_00030670.JPEG:n04209133 +ILSVRC2012_val_00030671.JPEG:n02094258 +ILSVRC2012_val_00030672.JPEG:n02028035 +ILSVRC2012_val_00030673.JPEG:n03627232 +ILSVRC2012_val_00030674.JPEG:n03529860 +ILSVRC2012_val_00030675.JPEG:n02077923 +ILSVRC2012_val_00030676.JPEG:n03843555 +ILSVRC2012_val_00030677.JPEG:n03873416 +ILSVRC2012_val_00030678.JPEG:n02116738 +ILSVRC2012_val_00030679.JPEG:n03995372 +ILSVRC2012_val_00030680.JPEG:n02104365 +ILSVRC2012_val_00030681.JPEG:n04347754 +ILSVRC2012_val_00030682.JPEG:n04590129 +ILSVRC2012_val_00030683.JPEG:n03657121 +ILSVRC2012_val_00030684.JPEG:n01774384 +ILSVRC2012_val_00030685.JPEG:n03937543 +ILSVRC2012_val_00030686.JPEG:n07836838 +ILSVRC2012_val_00030687.JPEG:n04127249 +ILSVRC2012_val_00030688.JPEG:n02391049 +ILSVRC2012_val_00030689.JPEG:n04296562 +ILSVRC2012_val_00030690.JPEG:n02492035 +ILSVRC2012_val_00030691.JPEG:n04254120 +ILSVRC2012_val_00030692.JPEG:n04201297 +ILSVRC2012_val_00030693.JPEG:n02115641 +ILSVRC2012_val_00030694.JPEG:n02094258 +ILSVRC2012_val_00030695.JPEG:n03729826 +ILSVRC2012_val_00030696.JPEG:n02090379 +ILSVRC2012_val_00030697.JPEG:n02165456 +ILSVRC2012_val_00030698.JPEG:n02107142 +ILSVRC2012_val_00030699.JPEG:n01518878 +ILSVRC2012_val_00030700.JPEG:n03649909 +ILSVRC2012_val_00030701.JPEG:n01558993 +ILSVRC2012_val_00030702.JPEG:n01843383 +ILSVRC2012_val_00030703.JPEG:n01695060 +ILSVRC2012_val_00030704.JPEG:n02134084 +ILSVRC2012_val_00030705.JPEG:n02101556 +ILSVRC2012_val_00030706.JPEG:n02123045 +ILSVRC2012_val_00030707.JPEG:n03929855 +ILSVRC2012_val_00030708.JPEG:n02110185 +ILSVRC2012_val_00030709.JPEG:n03291819 +ILSVRC2012_val_00030710.JPEG:n02099601 +ILSVRC2012_val_00030711.JPEG:n04443257 +ILSVRC2012_val_00030712.JPEG:n02487347 +ILSVRC2012_val_00030713.JPEG:n01795545 +ILSVRC2012_val_00030714.JPEG:n04458633 +ILSVRC2012_val_00030715.JPEG:n02229544 +ILSVRC2012_val_00030716.JPEG:n03325584 +ILSVRC2012_val_00030717.JPEG:n04086273 +ILSVRC2012_val_00030718.JPEG:n03017168 +ILSVRC2012_val_00030719.JPEG:n01729977 +ILSVRC2012_val_00030720.JPEG:n03388043 +ILSVRC2012_val_00030721.JPEG:n01675722 +ILSVRC2012_val_00030722.JPEG:n02009229 +ILSVRC2012_val_00030723.JPEG:n03126707 +ILSVRC2012_val_00030724.JPEG:n02117135 +ILSVRC2012_val_00030725.JPEG:n03873416 +ILSVRC2012_val_00030726.JPEG:n04332243 +ILSVRC2012_val_00030727.JPEG:n02486410 +ILSVRC2012_val_00030728.JPEG:n03394916 +ILSVRC2012_val_00030729.JPEG:n02480855 +ILSVRC2012_val_00030730.JPEG:n02837789 +ILSVRC2012_val_00030731.JPEG:n03018349 +ILSVRC2012_val_00030732.JPEG:n03998194 +ILSVRC2012_val_00030733.JPEG:n04317175 +ILSVRC2012_val_00030734.JPEG:n01819313 +ILSVRC2012_val_00030735.JPEG:n03291819 +ILSVRC2012_val_00030736.JPEG:n01664065 +ILSVRC2012_val_00030737.JPEG:n02128385 +ILSVRC2012_val_00030738.JPEG:n02417914 +ILSVRC2012_val_00030739.JPEG:n04040759 +ILSVRC2012_val_00030740.JPEG:n01440764 +ILSVRC2012_val_00030741.JPEG:n09468604 +ILSVRC2012_val_00030742.JPEG:n03240683 +ILSVRC2012_val_00030743.JPEG:n07248320 +ILSVRC2012_val_00030744.JPEG:n11939491 +ILSVRC2012_val_00030745.JPEG:n02971356 +ILSVRC2012_val_00030746.JPEG:n02096437 +ILSVRC2012_val_00030747.JPEG:n02101556 +ILSVRC2012_val_00030748.JPEG:n04467665 +ILSVRC2012_val_00030749.JPEG:n03983396 +ILSVRC2012_val_00030750.JPEG:n04146614 +ILSVRC2012_val_00030751.JPEG:n04252077 +ILSVRC2012_val_00030752.JPEG:n03476684 +ILSVRC2012_val_00030753.JPEG:n02777292 +ILSVRC2012_val_00030754.JPEG:n03617480 +ILSVRC2012_val_00030755.JPEG:n04004767 +ILSVRC2012_val_00030756.JPEG:n02102177 +ILSVRC2012_val_00030757.JPEG:n02088632 +ILSVRC2012_val_00030758.JPEG:n07749582 +ILSVRC2012_val_00030759.JPEG:n04264628 +ILSVRC2012_val_00030760.JPEG:n04487081 +ILSVRC2012_val_00030761.JPEG:n02808440 +ILSVRC2012_val_00030762.JPEG:n04399382 +ILSVRC2012_val_00030763.JPEG:n03961711 +ILSVRC2012_val_00030764.JPEG:n04229816 +ILSVRC2012_val_00030765.JPEG:n03977966 +ILSVRC2012_val_00030766.JPEG:n03133878 +ILSVRC2012_val_00030767.JPEG:n03877845 +ILSVRC2012_val_00030768.JPEG:n03995372 +ILSVRC2012_val_00030769.JPEG:n04131690 +ILSVRC2012_val_00030770.JPEG:n02093754 +ILSVRC2012_val_00030771.JPEG:n02110806 +ILSVRC2012_val_00030772.JPEG:n01872401 +ILSVRC2012_val_00030773.JPEG:n02106662 +ILSVRC2012_val_00030774.JPEG:n07836838 +ILSVRC2012_val_00030775.JPEG:n04553703 +ILSVRC2012_val_00030776.JPEG:n02095314 +ILSVRC2012_val_00030777.JPEG:n12620546 +ILSVRC2012_val_00030778.JPEG:n02231487 +ILSVRC2012_val_00030779.JPEG:n02277742 +ILSVRC2012_val_00030780.JPEG:n04456115 +ILSVRC2012_val_00030781.JPEG:n02643566 +ILSVRC2012_val_00030782.JPEG:n02317335 +ILSVRC2012_val_00030783.JPEG:n04008634 +ILSVRC2012_val_00030784.JPEG:n04476259 +ILSVRC2012_val_00030785.JPEG:n04550184 +ILSVRC2012_val_00030786.JPEG:n02107908 +ILSVRC2012_val_00030787.JPEG:n02125311 +ILSVRC2012_val_00030788.JPEG:n03355925 +ILSVRC2012_val_00030789.JPEG:n03769881 +ILSVRC2012_val_00030790.JPEG:n07615774 +ILSVRC2012_val_00030791.JPEG:n02443114 +ILSVRC2012_val_00030792.JPEG:n02167151 +ILSVRC2012_val_00030793.JPEG:n04590129 +ILSVRC2012_val_00030794.JPEG:n12620546 +ILSVRC2012_val_00030795.JPEG:n02177972 +ILSVRC2012_val_00030796.JPEG:n03866082 +ILSVRC2012_val_00030797.JPEG:n07718472 +ILSVRC2012_val_00030798.JPEG:n02102318 +ILSVRC2012_val_00030799.JPEG:n07697313 +ILSVRC2012_val_00030800.JPEG:n03384352 +ILSVRC2012_val_00030801.JPEG:n04330267 +ILSVRC2012_val_00030802.JPEG:n03874293 +ILSVRC2012_val_00030803.JPEG:n03895866 +ILSVRC2012_val_00030804.JPEG:n02444819 +ILSVRC2012_val_00030805.JPEG:n03908714 +ILSVRC2012_val_00030806.JPEG:n02395406 +ILSVRC2012_val_00030807.JPEG:n04355933 +ILSVRC2012_val_00030808.JPEG:n03220513 +ILSVRC2012_val_00030809.JPEG:n04147183 +ILSVRC2012_val_00030810.JPEG:n02099267 +ILSVRC2012_val_00030811.JPEG:n01983481 +ILSVRC2012_val_00030812.JPEG:n01770081 +ILSVRC2012_val_00030813.JPEG:n02095570 +ILSVRC2012_val_00030814.JPEG:n01695060 +ILSVRC2012_val_00030815.JPEG:n02115641 +ILSVRC2012_val_00030816.JPEG:n04355338 +ILSVRC2012_val_00030817.JPEG:n07584110 +ILSVRC2012_val_00030818.JPEG:n02843684 +ILSVRC2012_val_00030819.JPEG:n04023962 +ILSVRC2012_val_00030820.JPEG:n02102480 +ILSVRC2012_val_00030821.JPEG:n04116512 +ILSVRC2012_val_00030822.JPEG:n02094258 +ILSVRC2012_val_00030823.JPEG:n04326547 +ILSVRC2012_val_00030824.JPEG:n02951358 +ILSVRC2012_val_00030825.JPEG:n01784675 +ILSVRC2012_val_00030826.JPEG:n03494278 +ILSVRC2012_val_00030827.JPEG:n03935335 +ILSVRC2012_val_00030828.JPEG:n02106662 +ILSVRC2012_val_00030829.JPEG:n02256656 +ILSVRC2012_val_00030830.JPEG:n03944341 +ILSVRC2012_val_00030831.JPEG:n02105641 +ILSVRC2012_val_00030832.JPEG:n02666196 +ILSVRC2012_val_00030833.JPEG:n03982430 +ILSVRC2012_val_00030834.JPEG:n02814533 +ILSVRC2012_val_00030835.JPEG:n04204238 +ILSVRC2012_val_00030836.JPEG:n07730033 +ILSVRC2012_val_00030837.JPEG:n01807496 +ILSVRC2012_val_00030838.JPEG:n03042490 +ILSVRC2012_val_00030839.JPEG:n02963159 +ILSVRC2012_val_00030840.JPEG:n02504458 +ILSVRC2012_val_00030841.JPEG:n03535780 +ILSVRC2012_val_00030842.JPEG:n04355933 +ILSVRC2012_val_00030843.JPEG:n02009229 +ILSVRC2012_val_00030844.JPEG:n02423022 +ILSVRC2012_val_00030845.JPEG:n01582220 +ILSVRC2012_val_00030846.JPEG:n07614500 +ILSVRC2012_val_00030847.JPEG:n02321529 +ILSVRC2012_val_00030848.JPEG:n03272562 +ILSVRC2012_val_00030849.JPEG:n03642806 +ILSVRC2012_val_00030850.JPEG:n04251144 +ILSVRC2012_val_00030851.JPEG:n02115913 +ILSVRC2012_val_00030852.JPEG:n02107312 +ILSVRC2012_val_00030853.JPEG:n03924679 +ILSVRC2012_val_00030854.JPEG:n02699494 +ILSVRC2012_val_00030855.JPEG:n03908714 +ILSVRC2012_val_00030856.JPEG:n04522168 +ILSVRC2012_val_00030857.JPEG:n09246464 +ILSVRC2012_val_00030858.JPEG:n03617480 +ILSVRC2012_val_00030859.JPEG:n02231487 +ILSVRC2012_val_00030860.JPEG:n02127052 +ILSVRC2012_val_00030861.JPEG:n04335435 +ILSVRC2012_val_00030862.JPEG:n02804610 +ILSVRC2012_val_00030863.JPEG:n02437616 +ILSVRC2012_val_00030864.JPEG:n03249569 +ILSVRC2012_val_00030865.JPEG:n01682714 +ILSVRC2012_val_00030866.JPEG:n02790996 +ILSVRC2012_val_00030867.JPEG:n03742115 +ILSVRC2012_val_00030868.JPEG:n02112350 +ILSVRC2012_val_00030869.JPEG:n02837789 +ILSVRC2012_val_00030870.JPEG:n04371774 +ILSVRC2012_val_00030871.JPEG:n03443371 +ILSVRC2012_val_00030872.JPEG:n02992529 +ILSVRC2012_val_00030873.JPEG:n01688243 +ILSVRC2012_val_00030874.JPEG:n03733281 +ILSVRC2012_val_00030875.JPEG:n07875152 +ILSVRC2012_val_00030876.JPEG:n02105641 +ILSVRC2012_val_00030877.JPEG:n02110958 +ILSVRC2012_val_00030878.JPEG:n02018795 +ILSVRC2012_val_00030879.JPEG:n04482393 +ILSVRC2012_val_00030880.JPEG:n03063689 +ILSVRC2012_val_00030881.JPEG:n02328150 +ILSVRC2012_val_00030882.JPEG:n02109525 +ILSVRC2012_val_00030883.JPEG:n02071294 +ILSVRC2012_val_00030884.JPEG:n02808304 +ILSVRC2012_val_00030885.JPEG:n03530642 +ILSVRC2012_val_00030886.JPEG:n03970156 +ILSVRC2012_val_00030887.JPEG:n01860187 +ILSVRC2012_val_00030888.JPEG:n02102973 +ILSVRC2012_val_00030889.JPEG:n03220513 +ILSVRC2012_val_00030890.JPEG:n03032252 +ILSVRC2012_val_00030891.JPEG:n01797886 +ILSVRC2012_val_00030892.JPEG:n03792782 +ILSVRC2012_val_00030893.JPEG:n02085936 +ILSVRC2012_val_00030894.JPEG:n04487394 +ILSVRC2012_val_00030895.JPEG:n02790996 +ILSVRC2012_val_00030896.JPEG:n01773157 +ILSVRC2012_val_00030897.JPEG:n04367480 +ILSVRC2012_val_00030898.JPEG:n03290653 +ILSVRC2012_val_00030899.JPEG:n03478589 +ILSVRC2012_val_00030900.JPEG:n04542943 +ILSVRC2012_val_00030901.JPEG:n07579787 +ILSVRC2012_val_00030902.JPEG:n02190166 +ILSVRC2012_val_00030903.JPEG:n06785654 +ILSVRC2012_val_00030904.JPEG:n02002724 +ILSVRC2012_val_00030905.JPEG:n01740131 +ILSVRC2012_val_00030906.JPEG:n04033995 +ILSVRC2012_val_00030907.JPEG:n01978287 +ILSVRC2012_val_00030908.JPEG:n02011460 +ILSVRC2012_val_00030909.JPEG:n03937543 +ILSVRC2012_val_00030910.JPEG:n02096437 +ILSVRC2012_val_00030911.JPEG:n01534433 +ILSVRC2012_val_00030912.JPEG:n02978881 +ILSVRC2012_val_00030913.JPEG:n03445924 +ILSVRC2012_val_00030914.JPEG:n07716358 +ILSVRC2012_val_00030915.JPEG:n02093428 +ILSVRC2012_val_00030916.JPEG:n01776313 +ILSVRC2012_val_00030917.JPEG:n02704792 +ILSVRC2012_val_00030918.JPEG:n01687978 +ILSVRC2012_val_00030919.JPEG:n04550184 +ILSVRC2012_val_00030920.JPEG:n02102973 +ILSVRC2012_val_00030921.JPEG:n02165456 +ILSVRC2012_val_00030922.JPEG:n03347037 +ILSVRC2012_val_00030923.JPEG:n01755581 +ILSVRC2012_val_00030924.JPEG:n02111889 +ILSVRC2012_val_00030925.JPEG:n03967562 +ILSVRC2012_val_00030926.JPEG:n01491361 +ILSVRC2012_val_00030927.JPEG:n02437616 +ILSVRC2012_val_00030928.JPEG:n02089078 +ILSVRC2012_val_00030929.JPEG:n02123597 +ILSVRC2012_val_00030930.JPEG:n04507155 +ILSVRC2012_val_00030931.JPEG:n03110669 +ILSVRC2012_val_00030932.JPEG:n03868242 +ILSVRC2012_val_00030933.JPEG:n03874599 +ILSVRC2012_val_00030934.JPEG:n02120505 +ILSVRC2012_val_00030935.JPEG:n03930313 +ILSVRC2012_val_00030936.JPEG:n02165105 +ILSVRC2012_val_00030937.JPEG:n04604644 +ILSVRC2012_val_00030938.JPEG:n03445777 +ILSVRC2012_val_00030939.JPEG:n02099712 +ILSVRC2012_val_00030940.JPEG:n02009229 +ILSVRC2012_val_00030941.JPEG:n04389033 +ILSVRC2012_val_00030942.JPEG:n04371774 +ILSVRC2012_val_00030943.JPEG:n02437616 +ILSVRC2012_val_00030944.JPEG:n04243546 +ILSVRC2012_val_00030945.JPEG:n03794056 +ILSVRC2012_val_00030946.JPEG:n03775071 +ILSVRC2012_val_00030947.JPEG:n04479046 +ILSVRC2012_val_00030948.JPEG:n03796401 +ILSVRC2012_val_00030949.JPEG:n02892767 +ILSVRC2012_val_00030950.JPEG:n03929660 +ILSVRC2012_val_00030951.JPEG:n02133161 +ILSVRC2012_val_00030952.JPEG:n03944341 +ILSVRC2012_val_00030953.JPEG:n03884397 +ILSVRC2012_val_00030954.JPEG:n04589890 +ILSVRC2012_val_00030955.JPEG:n03590841 +ILSVRC2012_val_00030956.JPEG:n02071294 +ILSVRC2012_val_00030957.JPEG:n04263257 +ILSVRC2012_val_00030958.JPEG:n01768244 +ILSVRC2012_val_00030959.JPEG:n02410509 +ILSVRC2012_val_00030960.JPEG:n04465501 +ILSVRC2012_val_00030961.JPEG:n02098286 +ILSVRC2012_val_00030962.JPEG:n02747177 +ILSVRC2012_val_00030963.JPEG:n02105162 +ILSVRC2012_val_00030964.JPEG:n01667114 +ILSVRC2012_val_00030965.JPEG:n02999410 +ILSVRC2012_val_00030966.JPEG:n01560419 +ILSVRC2012_val_00030967.JPEG:n07749582 +ILSVRC2012_val_00030968.JPEG:n01968897 +ILSVRC2012_val_00030969.JPEG:n02130308 +ILSVRC2012_val_00030970.JPEG:n02110806 +ILSVRC2012_val_00030971.JPEG:n02106382 +ILSVRC2012_val_00030972.JPEG:n07590611 +ILSVRC2012_val_00030973.JPEG:n07697537 +ILSVRC2012_val_00030974.JPEG:n04591157 +ILSVRC2012_val_00030975.JPEG:n04462240 +ILSVRC2012_val_00030976.JPEG:n02988304 +ILSVRC2012_val_00030977.JPEG:n03126707 +ILSVRC2012_val_00030978.JPEG:n02727426 +ILSVRC2012_val_00030979.JPEG:n04127249 +ILSVRC2012_val_00030980.JPEG:n02843684 +ILSVRC2012_val_00030981.JPEG:n03179701 +ILSVRC2012_val_00030982.JPEG:n02443484 +ILSVRC2012_val_00030983.JPEG:n04344873 +ILSVRC2012_val_00030984.JPEG:n02280649 +ILSVRC2012_val_00030985.JPEG:n03216828 +ILSVRC2012_val_00030986.JPEG:n12985857 +ILSVRC2012_val_00030987.JPEG:n04548280 +ILSVRC2012_val_00030988.JPEG:n03602883 +ILSVRC2012_val_00030989.JPEG:n03447721 +ILSVRC2012_val_00030990.JPEG:n01694178 +ILSVRC2012_val_00030991.JPEG:n02415577 +ILSVRC2012_val_00030992.JPEG:n02699494 +ILSVRC2012_val_00030993.JPEG:n03085013 +ILSVRC2012_val_00030994.JPEG:n02895154 +ILSVRC2012_val_00030995.JPEG:n04371774 +ILSVRC2012_val_00030996.JPEG:n03495258 +ILSVRC2012_val_00030997.JPEG:n03791053 +ILSVRC2012_val_00030998.JPEG:n02641379 +ILSVRC2012_val_00030999.JPEG:n02980441 +ILSVRC2012_val_00031000.JPEG:n02950826 +ILSVRC2012_val_00031001.JPEG:n02110063 +ILSVRC2012_val_00031002.JPEG:n03788195 +ILSVRC2012_val_00031003.JPEG:n01693334 +ILSVRC2012_val_00031004.JPEG:n02606052 +ILSVRC2012_val_00031005.JPEG:n07742313 +ILSVRC2012_val_00031006.JPEG:n02113624 +ILSVRC2012_val_00031007.JPEG:n03874293 +ILSVRC2012_val_00031008.JPEG:n04209239 +ILSVRC2012_val_00031009.JPEG:n03388043 +ILSVRC2012_val_00031010.JPEG:n02927161 +ILSVRC2012_val_00031011.JPEG:n03944341 +ILSVRC2012_val_00031012.JPEG:n04579432 +ILSVRC2012_val_00031013.JPEG:n03759954 +ILSVRC2012_val_00031014.JPEG:n02101388 +ILSVRC2012_val_00031015.JPEG:n01978287 +ILSVRC2012_val_00031016.JPEG:n03443371 +ILSVRC2012_val_00031017.JPEG:n02129604 +ILSVRC2012_val_00031018.JPEG:n01693334 +ILSVRC2012_val_00031019.JPEG:n07742313 +ILSVRC2012_val_00031020.JPEG:n01770393 +ILSVRC2012_val_00031021.JPEG:n06785654 +ILSVRC2012_val_00031022.JPEG:n03126707 +ILSVRC2012_val_00031023.JPEG:n02058221 +ILSVRC2012_val_00031024.JPEG:n03721384 +ILSVRC2012_val_00031025.JPEG:n02093647 +ILSVRC2012_val_00031026.JPEG:n07684084 +ILSVRC2012_val_00031027.JPEG:n03775546 +ILSVRC2012_val_00031028.JPEG:n03494278 +ILSVRC2012_val_00031029.JPEG:n03131574 +ILSVRC2012_val_00031030.JPEG:n02823428 +ILSVRC2012_val_00031031.JPEG:n02111889 +ILSVRC2012_val_00031032.JPEG:n04208210 +ILSVRC2012_val_00031033.JPEG:n02190166 +ILSVRC2012_val_00031034.JPEG:n04228054 +ILSVRC2012_val_00031035.JPEG:n03888257 +ILSVRC2012_val_00031036.JPEG:n02169497 +ILSVRC2012_val_00031037.JPEG:n01770081 +ILSVRC2012_val_00031038.JPEG:n02974003 +ILSVRC2012_val_00031039.JPEG:n03637318 +ILSVRC2012_val_00031040.JPEG:n02089078 +ILSVRC2012_val_00031041.JPEG:n02117135 +ILSVRC2012_val_00031042.JPEG:n02457408 +ILSVRC2012_val_00031043.JPEG:n02606052 +ILSVRC2012_val_00031044.JPEG:n03877845 +ILSVRC2012_val_00031045.JPEG:n02776631 +ILSVRC2012_val_00031046.JPEG:n01882714 +ILSVRC2012_val_00031047.JPEG:n03325584 +ILSVRC2012_val_00031048.JPEG:n02095314 +ILSVRC2012_val_00031049.JPEG:n02102973 +ILSVRC2012_val_00031050.JPEG:n02236044 +ILSVRC2012_val_00031051.JPEG:n02090622 +ILSVRC2012_val_00031052.JPEG:n02797295 +ILSVRC2012_val_00031053.JPEG:n01775062 +ILSVRC2012_val_00031054.JPEG:n02098286 +ILSVRC2012_val_00031055.JPEG:n03498962 +ILSVRC2012_val_00031056.JPEG:n02128385 +ILSVRC2012_val_00031057.JPEG:n02783161 +ILSVRC2012_val_00031058.JPEG:n07768694 +ILSVRC2012_val_00031059.JPEG:n03337140 +ILSVRC2012_val_00031060.JPEG:n01751748 +ILSVRC2012_val_00031061.JPEG:n04447861 +ILSVRC2012_val_00031062.JPEG:n02172182 +ILSVRC2012_val_00031063.JPEG:n03743016 +ILSVRC2012_val_00031064.JPEG:n03599486 +ILSVRC2012_val_00031065.JPEG:n04380533 +ILSVRC2012_val_00031066.JPEG:n07892512 +ILSVRC2012_val_00031067.JPEG:n03598930 +ILSVRC2012_val_00031068.JPEG:n02085782 +ILSVRC2012_val_00031069.JPEG:n01685808 +ILSVRC2012_val_00031070.JPEG:n02879718 +ILSVRC2012_val_00031071.JPEG:n01491361 +ILSVRC2012_val_00031072.JPEG:n04273569 +ILSVRC2012_val_00031073.JPEG:n02441942 +ILSVRC2012_val_00031074.JPEG:n04553703 +ILSVRC2012_val_00031075.JPEG:n03649909 +ILSVRC2012_val_00031076.JPEG:n03141823 +ILSVRC2012_val_00031077.JPEG:n02115641 +ILSVRC2012_val_00031078.JPEG:n04372370 +ILSVRC2012_val_00031079.JPEG:n04265275 +ILSVRC2012_val_00031080.JPEG:n04493381 +ILSVRC2012_val_00031081.JPEG:n06596364 +ILSVRC2012_val_00031082.JPEG:n02825657 +ILSVRC2012_val_00031083.JPEG:n02480495 +ILSVRC2012_val_00031084.JPEG:n02097298 +ILSVRC2012_val_00031085.JPEG:n03532672 +ILSVRC2012_val_00031086.JPEG:n01531178 +ILSVRC2012_val_00031087.JPEG:n03843555 +ILSVRC2012_val_00031088.JPEG:n03770679 +ILSVRC2012_val_00031089.JPEG:n02346627 +ILSVRC2012_val_00031090.JPEG:n02127052 +ILSVRC2012_val_00031091.JPEG:n03297495 +ILSVRC2012_val_00031092.JPEG:n02869837 +ILSVRC2012_val_00031093.JPEG:n02106166 +ILSVRC2012_val_00031094.JPEG:n01440764 +ILSVRC2012_val_00031095.JPEG:n02510455 +ILSVRC2012_val_00031096.JPEG:n02095570 +ILSVRC2012_val_00031097.JPEG:n02177972 +ILSVRC2012_val_00031098.JPEG:n03347037 +ILSVRC2012_val_00031099.JPEG:n01978455 +ILSVRC2012_val_00031100.JPEG:n02488702 +ILSVRC2012_val_00031101.JPEG:n02791124 +ILSVRC2012_val_00031102.JPEG:n04229816 +ILSVRC2012_val_00031103.JPEG:n01675722 +ILSVRC2012_val_00031104.JPEG:n03630383 +ILSVRC2012_val_00031105.JPEG:n01930112 +ILSVRC2012_val_00031106.JPEG:n04005630 +ILSVRC2012_val_00031107.JPEG:n04039381 +ILSVRC2012_val_00031108.JPEG:n03950228 +ILSVRC2012_val_00031109.JPEG:n04592741 +ILSVRC2012_val_00031110.JPEG:n01914609 +ILSVRC2012_val_00031111.JPEG:n02129165 +ILSVRC2012_val_00031112.JPEG:n01871265 +ILSVRC2012_val_00031113.JPEG:n03902125 +ILSVRC2012_val_00031114.JPEG:n01689811 +ILSVRC2012_val_00031115.JPEG:n03534580 +ILSVRC2012_val_00031116.JPEG:n01945685 +ILSVRC2012_val_00031117.JPEG:n01773549 +ILSVRC2012_val_00031118.JPEG:n02089867 +ILSVRC2012_val_00031119.JPEG:n03788195 +ILSVRC2012_val_00031120.JPEG:n02788148 +ILSVRC2012_val_00031121.JPEG:n02113023 +ILSVRC2012_val_00031122.JPEG:n03534580 +ILSVRC2012_val_00031123.JPEG:n04592741 +ILSVRC2012_val_00031124.JPEG:n02797295 +ILSVRC2012_val_00031125.JPEG:n03017168 +ILSVRC2012_val_00031126.JPEG:n04355933 +ILSVRC2012_val_00031127.JPEG:n02097209 +ILSVRC2012_val_00031128.JPEG:n02167151 +ILSVRC2012_val_00031129.JPEG:n04026417 +ILSVRC2012_val_00031130.JPEG:n03271574 +ILSVRC2012_val_00031131.JPEG:n02105251 +ILSVRC2012_val_00031132.JPEG:n04004767 +ILSVRC2012_val_00031133.JPEG:n02108000 +ILSVRC2012_val_00031134.JPEG:n04350905 +ILSVRC2012_val_00031135.JPEG:n02106662 +ILSVRC2012_val_00031136.JPEG:n03201208 +ILSVRC2012_val_00031137.JPEG:n03126707 +ILSVRC2012_val_00031138.JPEG:n01443537 +ILSVRC2012_val_00031139.JPEG:n02837789 +ILSVRC2012_val_00031140.JPEG:n02165456 +ILSVRC2012_val_00031141.JPEG:n03796401 +ILSVRC2012_val_00031142.JPEG:n02870880 +ILSVRC2012_val_00031143.JPEG:n02641379 +ILSVRC2012_val_00031144.JPEG:n01622779 +ILSVRC2012_val_00031145.JPEG:n02113023 +ILSVRC2012_val_00031146.JPEG:n07880968 +ILSVRC2012_val_00031147.JPEG:n02165456 +ILSVRC2012_val_00031148.JPEG:n03840681 +ILSVRC2012_val_00031149.JPEG:n03372029 +ILSVRC2012_val_00031150.JPEG:n04044716 +ILSVRC2012_val_00031151.JPEG:n03840681 +ILSVRC2012_val_00031152.JPEG:n03692522 +ILSVRC2012_val_00031153.JPEG:n03992509 +ILSVRC2012_val_00031154.JPEG:n02085620 +ILSVRC2012_val_00031155.JPEG:n03530642 +ILSVRC2012_val_00031156.JPEG:n02113186 +ILSVRC2012_val_00031157.JPEG:n02086079 +ILSVRC2012_val_00031158.JPEG:n07614500 +ILSVRC2012_val_00031159.JPEG:n09468604 +ILSVRC2012_val_00031160.JPEG:n03602883 +ILSVRC2012_val_00031161.JPEG:n09468604 +ILSVRC2012_val_00031162.JPEG:n04270147 +ILSVRC2012_val_00031163.JPEG:n04146614 +ILSVRC2012_val_00031164.JPEG:n02892201 +ILSVRC2012_val_00031165.JPEG:n03958227 +ILSVRC2012_val_00031166.JPEG:n03832673 +ILSVRC2012_val_00031167.JPEG:n02268443 +ILSVRC2012_val_00031168.JPEG:n02236044 +ILSVRC2012_val_00031169.JPEG:n01494475 +ILSVRC2012_val_00031170.JPEG:n02009912 +ILSVRC2012_val_00031171.JPEG:n01532829 +ILSVRC2012_val_00031172.JPEG:n02093754 +ILSVRC2012_val_00031173.JPEG:n03404251 +ILSVRC2012_val_00031174.JPEG:n03770439 +ILSVRC2012_val_00031175.JPEG:n07734744 +ILSVRC2012_val_00031176.JPEG:n04252077 +ILSVRC2012_val_00031177.JPEG:n07714571 +ILSVRC2012_val_00031178.JPEG:n02120079 +ILSVRC2012_val_00031179.JPEG:n01665541 +ILSVRC2012_val_00031180.JPEG:n02123394 +ILSVRC2012_val_00031181.JPEG:n03240683 +ILSVRC2012_val_00031182.JPEG:n04264628 +ILSVRC2012_val_00031183.JPEG:n02457408 +ILSVRC2012_val_00031184.JPEG:n07614500 +ILSVRC2012_val_00031185.JPEG:n02124075 +ILSVRC2012_val_00031186.JPEG:n03425413 +ILSVRC2012_val_00031187.JPEG:n03133878 +ILSVRC2012_val_00031188.JPEG:n07930864 +ILSVRC2012_val_00031189.JPEG:n03160309 +ILSVRC2012_val_00031190.JPEG:n02484975 +ILSVRC2012_val_00031191.JPEG:n02086240 +ILSVRC2012_val_00031192.JPEG:n02978881 +ILSVRC2012_val_00031193.JPEG:n04404412 +ILSVRC2012_val_00031194.JPEG:n02643566 +ILSVRC2012_val_00031195.JPEG:n02494079 +ILSVRC2012_val_00031196.JPEG:n02749479 +ILSVRC2012_val_00031197.JPEG:n02114855 +ILSVRC2012_val_00031198.JPEG:n02106166 +ILSVRC2012_val_00031199.JPEG:n02114712 +ILSVRC2012_val_00031200.JPEG:n03662601 +ILSVRC2012_val_00031201.JPEG:n07583066 +ILSVRC2012_val_00031202.JPEG:n02396427 +ILSVRC2012_val_00031203.JPEG:n02108089 +ILSVRC2012_val_00031204.JPEG:n04335435 +ILSVRC2012_val_00031205.JPEG:n03017168 +ILSVRC2012_val_00031206.JPEG:n02113186 +ILSVRC2012_val_00031207.JPEG:n04493381 +ILSVRC2012_val_00031208.JPEG:n02909870 +ILSVRC2012_val_00031209.JPEG:n03075370 +ILSVRC2012_val_00031210.JPEG:n03627232 +ILSVRC2012_val_00031211.JPEG:n03794056 +ILSVRC2012_val_00031212.JPEG:n01734418 +ILSVRC2012_val_00031213.JPEG:n02951358 +ILSVRC2012_val_00031214.JPEG:n02457408 +ILSVRC2012_val_00031215.JPEG:n02883205 +ILSVRC2012_val_00031216.JPEG:n02917067 +ILSVRC2012_val_00031217.JPEG:n03250847 +ILSVRC2012_val_00031218.JPEG:n02804610 +ILSVRC2012_val_00031219.JPEG:n02110958 +ILSVRC2012_val_00031220.JPEG:n02088364 +ILSVRC2012_val_00031221.JPEG:n03891251 +ILSVRC2012_val_00031222.JPEG:n02641379 +ILSVRC2012_val_00031223.JPEG:n02098105 +ILSVRC2012_val_00031224.JPEG:n02113624 +ILSVRC2012_val_00031225.JPEG:n02027492 +ILSVRC2012_val_00031226.JPEG:n02066245 +ILSVRC2012_val_00031227.JPEG:n02168699 +ILSVRC2012_val_00031228.JPEG:n06359193 +ILSVRC2012_val_00031229.JPEG:n03627232 +ILSVRC2012_val_00031230.JPEG:n09229709 +ILSVRC2012_val_00031231.JPEG:n02749479 +ILSVRC2012_val_00031232.JPEG:n04355338 +ILSVRC2012_val_00031233.JPEG:n04252225 +ILSVRC2012_val_00031234.JPEG:n02939185 +ILSVRC2012_val_00031235.JPEG:n01632777 +ILSVRC2012_val_00031236.JPEG:n02395406 +ILSVRC2012_val_00031237.JPEG:n02219486 +ILSVRC2012_val_00031238.JPEG:n02988304 +ILSVRC2012_val_00031239.JPEG:n01518878 +ILSVRC2012_val_00031240.JPEG:n03891332 +ILSVRC2012_val_00031241.JPEG:n02114548 +ILSVRC2012_val_00031242.JPEG:n02892767 +ILSVRC2012_val_00031243.JPEG:n01491361 +ILSVRC2012_val_00031244.JPEG:n03933933 +ILSVRC2012_val_00031245.JPEG:n02795169 +ILSVRC2012_val_00031246.JPEG:n09472597 +ILSVRC2012_val_00031247.JPEG:n07579787 +ILSVRC2012_val_00031248.JPEG:n03032252 +ILSVRC2012_val_00031249.JPEG:n02093754 +ILSVRC2012_val_00031250.JPEG:n13054560 +ILSVRC2012_val_00031251.JPEG:n03891251 +ILSVRC2012_val_00031252.JPEG:n02105505 +ILSVRC2012_val_00031253.JPEG:n02132136 +ILSVRC2012_val_00031254.JPEG:n07873807 +ILSVRC2012_val_00031255.JPEG:n02640242 +ILSVRC2012_val_00031256.JPEG:n04461696 +ILSVRC2012_val_00031257.JPEG:n04613696 +ILSVRC2012_val_00031258.JPEG:n09468604 +ILSVRC2012_val_00031259.JPEG:n02113186 +ILSVRC2012_val_00031260.JPEG:n02493509 +ILSVRC2012_val_00031261.JPEG:n04553703 +ILSVRC2012_val_00031262.JPEG:n01968897 +ILSVRC2012_val_00031263.JPEG:n04296562 +ILSVRC2012_val_00031264.JPEG:n03467068 +ILSVRC2012_val_00031265.JPEG:n03763968 +ILSVRC2012_val_00031266.JPEG:n04209239 +ILSVRC2012_val_00031267.JPEG:n02219486 +ILSVRC2012_val_00031268.JPEG:n03888257 +ILSVRC2012_val_00031269.JPEG:n01871265 +ILSVRC2012_val_00031270.JPEG:n03325584 +ILSVRC2012_val_00031271.JPEG:n03272562 +ILSVRC2012_val_00031272.JPEG:n03854065 +ILSVRC2012_val_00031273.JPEG:n01558993 +ILSVRC2012_val_00031274.JPEG:n03670208 +ILSVRC2012_val_00031275.JPEG:n01665541 +ILSVRC2012_val_00031276.JPEG:n03325584 +ILSVRC2012_val_00031277.JPEG:n01695060 +ILSVRC2012_val_00031278.JPEG:n02457408 +ILSVRC2012_val_00031279.JPEG:n02797295 +ILSVRC2012_val_00031280.JPEG:n02950826 +ILSVRC2012_val_00031281.JPEG:n02099429 +ILSVRC2012_val_00031282.JPEG:n03291819 +ILSVRC2012_val_00031283.JPEG:n02939185 +ILSVRC2012_val_00031284.JPEG:n03976467 +ILSVRC2012_val_00031285.JPEG:n02120079 +ILSVRC2012_val_00031286.JPEG:n02879718 +ILSVRC2012_val_00031287.JPEG:n04579145 +ILSVRC2012_val_00031288.JPEG:n04120489 +ILSVRC2012_val_00031289.JPEG:n01632458 +ILSVRC2012_val_00031290.JPEG:n02009912 +ILSVRC2012_val_00031291.JPEG:n04328186 +ILSVRC2012_val_00031292.JPEG:n06874185 +ILSVRC2012_val_00031293.JPEG:n02398521 +ILSVRC2012_val_00031294.JPEG:n02488291 +ILSVRC2012_val_00031295.JPEG:n02107312 +ILSVRC2012_val_00031296.JPEG:n03026506 +ILSVRC2012_val_00031297.JPEG:n02119022 +ILSVRC2012_val_00031298.JPEG:n01843383 +ILSVRC2012_val_00031299.JPEG:n03657121 +ILSVRC2012_val_00031300.JPEG:n03062245 +ILSVRC2012_val_00031301.JPEG:n07584110 +ILSVRC2012_val_00031302.JPEG:n02091032 +ILSVRC2012_val_00031303.JPEG:n03476991 +ILSVRC2012_val_00031304.JPEG:n02013706 +ILSVRC2012_val_00031305.JPEG:n02607072 +ILSVRC2012_val_00031306.JPEG:n02113712 +ILSVRC2012_val_00031307.JPEG:n03788365 +ILSVRC2012_val_00031308.JPEG:n04355338 +ILSVRC2012_val_00031309.JPEG:n04428191 +ILSVRC2012_val_00031310.JPEG:n04442312 +ILSVRC2012_val_00031311.JPEG:n01753488 +ILSVRC2012_val_00031312.JPEG:n12620546 +ILSVRC2012_val_00031313.JPEG:n03417042 +ILSVRC2012_val_00031314.JPEG:n02108089 +ILSVRC2012_val_00031315.JPEG:n07871810 +ILSVRC2012_val_00031316.JPEG:n03930313 +ILSVRC2012_val_00031317.JPEG:n04019541 +ILSVRC2012_val_00031318.JPEG:n04074963 +ILSVRC2012_val_00031319.JPEG:n02408429 +ILSVRC2012_val_00031320.JPEG:n02817516 +ILSVRC2012_val_00031321.JPEG:n01955084 +ILSVRC2012_val_00031322.JPEG:n02747177 +ILSVRC2012_val_00031323.JPEG:n09472597 +ILSVRC2012_val_00031324.JPEG:n03866082 +ILSVRC2012_val_00031325.JPEG:n02099267 +ILSVRC2012_val_00031326.JPEG:n03782006 +ILSVRC2012_val_00031327.JPEG:n03998194 +ILSVRC2012_val_00031328.JPEG:n02823428 +ILSVRC2012_val_00031329.JPEG:n04487081 +ILSVRC2012_val_00031330.JPEG:n03956157 +ILSVRC2012_val_00031331.JPEG:n03854065 +ILSVRC2012_val_00031332.JPEG:n02002556 +ILSVRC2012_val_00031333.JPEG:n01440764 +ILSVRC2012_val_00031334.JPEG:n02093256 +ILSVRC2012_val_00031335.JPEG:n02229544 +ILSVRC2012_val_00031336.JPEG:n02109047 +ILSVRC2012_val_00031337.JPEG:n03160309 +ILSVRC2012_val_00031338.JPEG:n02825657 +ILSVRC2012_val_00031339.JPEG:n02423022 +ILSVRC2012_val_00031340.JPEG:n03016953 +ILSVRC2012_val_00031341.JPEG:n04179913 +ILSVRC2012_val_00031342.JPEG:n01860187 +ILSVRC2012_val_00031343.JPEG:n02107574 +ILSVRC2012_val_00031344.JPEG:n06359193 +ILSVRC2012_val_00031345.JPEG:n02088094 +ILSVRC2012_val_00031346.JPEG:n04065272 +ILSVRC2012_val_00031347.JPEG:n02088632 +ILSVRC2012_val_00031348.JPEG:n02130308 +ILSVRC2012_val_00031349.JPEG:n03769881 +ILSVRC2012_val_00031350.JPEG:n02966193 +ILSVRC2012_val_00031351.JPEG:n06794110 +ILSVRC2012_val_00031352.JPEG:n07590611 +ILSVRC2012_val_00031353.JPEG:n03924679 +ILSVRC2012_val_00031354.JPEG:n04153751 +ILSVRC2012_val_00031355.JPEG:n02112706 +ILSVRC2012_val_00031356.JPEG:n02509815 +ILSVRC2012_val_00031357.JPEG:n04335435 +ILSVRC2012_val_00031358.JPEG:n04579432 +ILSVRC2012_val_00031359.JPEG:n02815834 +ILSVRC2012_val_00031360.JPEG:n02361337 +ILSVRC2012_val_00031361.JPEG:n02123159 +ILSVRC2012_val_00031362.JPEG:n03133878 +ILSVRC2012_val_00031363.JPEG:n02457408 +ILSVRC2012_val_00031364.JPEG:n02092002 +ILSVRC2012_val_00031365.JPEG:n04347754 +ILSVRC2012_val_00031366.JPEG:n03775071 +ILSVRC2012_val_00031367.JPEG:n03498962 +ILSVRC2012_val_00031368.JPEG:n02101388 +ILSVRC2012_val_00031369.JPEG:n03447447 +ILSVRC2012_val_00031370.JPEG:n02443114 +ILSVRC2012_val_00031371.JPEG:n04039381 +ILSVRC2012_val_00031372.JPEG:n02791124 +ILSVRC2012_val_00031373.JPEG:n02104365 +ILSVRC2012_val_00031374.JPEG:n01776313 +ILSVRC2012_val_00031375.JPEG:n04442312 +ILSVRC2012_val_00031376.JPEG:n03584254 +ILSVRC2012_val_00031377.JPEG:n02094258 +ILSVRC2012_val_00031378.JPEG:n02086646 +ILSVRC2012_val_00031379.JPEG:n04370456 +ILSVRC2012_val_00031380.JPEG:n01797886 +ILSVRC2012_val_00031381.JPEG:n03724870 +ILSVRC2012_val_00031382.JPEG:n01775062 +ILSVRC2012_val_00031383.JPEG:n02687172 +ILSVRC2012_val_00031384.JPEG:n02091244 +ILSVRC2012_val_00031385.JPEG:n03124043 +ILSVRC2012_val_00031386.JPEG:n01632777 +ILSVRC2012_val_00031387.JPEG:n02787622 +ILSVRC2012_val_00031388.JPEG:n01930112 +ILSVRC2012_val_00031389.JPEG:n01664065 +ILSVRC2012_val_00031390.JPEG:n01734418 +ILSVRC2012_val_00031391.JPEG:n02110063 +ILSVRC2012_val_00031392.JPEG:n01818515 +ILSVRC2012_val_00031393.JPEG:n04336792 +ILSVRC2012_val_00031394.JPEG:n03793489 +ILSVRC2012_val_00031395.JPEG:n02097298 +ILSVRC2012_val_00031396.JPEG:n02017213 +ILSVRC2012_val_00031397.JPEG:n04273569 +ILSVRC2012_val_00031398.JPEG:n03485794 +ILSVRC2012_val_00031399.JPEG:n02002724 +ILSVRC2012_val_00031400.JPEG:n04507155 +ILSVRC2012_val_00031401.JPEG:n11879895 +ILSVRC2012_val_00031402.JPEG:n02087046 +ILSVRC2012_val_00031403.JPEG:n02486410 +ILSVRC2012_val_00031404.JPEG:n04033995 +ILSVRC2012_val_00031405.JPEG:n03345487 +ILSVRC2012_val_00031406.JPEG:n03692522 +ILSVRC2012_val_00031407.JPEG:n04347754 +ILSVRC2012_val_00031408.JPEG:n01986214 +ILSVRC2012_val_00031409.JPEG:n03873416 +ILSVRC2012_val_00031410.JPEG:n03483316 +ILSVRC2012_val_00031411.JPEG:n02101556 +ILSVRC2012_val_00031412.JPEG:n03425413 +ILSVRC2012_val_00031413.JPEG:n03000684 +ILSVRC2012_val_00031414.JPEG:n02114367 +ILSVRC2012_val_00031415.JPEG:n02113712 +ILSVRC2012_val_00031416.JPEG:n03535780 +ILSVRC2012_val_00031417.JPEG:n02454379 +ILSVRC2012_val_00031418.JPEG:n03788195 +ILSVRC2012_val_00031419.JPEG:n02086240 +ILSVRC2012_val_00031420.JPEG:n02095889 +ILSVRC2012_val_00031421.JPEG:n02422699 +ILSVRC2012_val_00031422.JPEG:n03400231 +ILSVRC2012_val_00031423.JPEG:n03690938 +ILSVRC2012_val_00031424.JPEG:n01494475 +ILSVRC2012_val_00031425.JPEG:n02099601 +ILSVRC2012_val_00031426.JPEG:n04612504 +ILSVRC2012_val_00031427.JPEG:n07753275 +ILSVRC2012_val_00031428.JPEG:n03814639 +ILSVRC2012_val_00031429.JPEG:n02165105 +ILSVRC2012_val_00031430.JPEG:n03314780 +ILSVRC2012_val_00031431.JPEG:n03478589 +ILSVRC2012_val_00031432.JPEG:n01796340 +ILSVRC2012_val_00031433.JPEG:n02105641 +ILSVRC2012_val_00031434.JPEG:n01847000 +ILSVRC2012_val_00031435.JPEG:n01877812 +ILSVRC2012_val_00031436.JPEG:n02447366 +ILSVRC2012_val_00031437.JPEG:n03929660 +ILSVRC2012_val_00031438.JPEG:n02992529 +ILSVRC2012_val_00031439.JPEG:n02088094 +ILSVRC2012_val_00031440.JPEG:n07745940 +ILSVRC2012_val_00031441.JPEG:n04522168 +ILSVRC2012_val_00031442.JPEG:n04069434 +ILSVRC2012_val_00031443.JPEG:n12620546 +ILSVRC2012_val_00031444.JPEG:n03673027 +ILSVRC2012_val_00031445.JPEG:n03998194 +ILSVRC2012_val_00031446.JPEG:n03028079 +ILSVRC2012_val_00031447.JPEG:n04252225 +ILSVRC2012_val_00031448.JPEG:n02033041 +ILSVRC2012_val_00031449.JPEG:n01843065 +ILSVRC2012_val_00031450.JPEG:n07720875 +ILSVRC2012_val_00031451.JPEG:n02099712 +ILSVRC2012_val_00031452.JPEG:n02939185 +ILSVRC2012_val_00031453.JPEG:n02098413 +ILSVRC2012_val_00031454.JPEG:n04296562 +ILSVRC2012_val_00031455.JPEG:n03796401 +ILSVRC2012_val_00031456.JPEG:n01729977 +ILSVRC2012_val_00031457.JPEG:n02859443 +ILSVRC2012_val_00031458.JPEG:n02105251 +ILSVRC2012_val_00031459.JPEG:n02860847 +ILSVRC2012_val_00031460.JPEG:n04209133 +ILSVRC2012_val_00031461.JPEG:n02108000 +ILSVRC2012_val_00031462.JPEG:n04235860 +ILSVRC2012_val_00031463.JPEG:n02782093 +ILSVRC2012_val_00031464.JPEG:n02814533 +ILSVRC2012_val_00031465.JPEG:n01614925 +ILSVRC2012_val_00031466.JPEG:n01484850 +ILSVRC2012_val_00031467.JPEG:n01669191 +ILSVRC2012_val_00031468.JPEG:n04525305 +ILSVRC2012_val_00031469.JPEG:n07716906 +ILSVRC2012_val_00031470.JPEG:n02119022 +ILSVRC2012_val_00031471.JPEG:n03721384 +ILSVRC2012_val_00031472.JPEG:n02259212 +ILSVRC2012_val_00031473.JPEG:n03976657 +ILSVRC2012_val_00031474.JPEG:n02415577 +ILSVRC2012_val_00031475.JPEG:n04392985 +ILSVRC2012_val_00031476.JPEG:n04023962 +ILSVRC2012_val_00031477.JPEG:n02793495 +ILSVRC2012_val_00031478.JPEG:n04592741 +ILSVRC2012_val_00031479.JPEG:n02233338 +ILSVRC2012_val_00031480.JPEG:n02777292 +ILSVRC2012_val_00031481.JPEG:n01514859 +ILSVRC2012_val_00031482.JPEG:n03127747 +ILSVRC2012_val_00031483.JPEG:n04548362 +ILSVRC2012_val_00031484.JPEG:n03947888 +ILSVRC2012_val_00031485.JPEG:n03792782 +ILSVRC2012_val_00031486.JPEG:n03445777 +ILSVRC2012_val_00031487.JPEG:n04592741 +ILSVRC2012_val_00031488.JPEG:n02165105 +ILSVRC2012_val_00031489.JPEG:n02105056 +ILSVRC2012_val_00031490.JPEG:n04525038 +ILSVRC2012_val_00031491.JPEG:n02395406 +ILSVRC2012_val_00031492.JPEG:n02129604 +ILSVRC2012_val_00031493.JPEG:n09399592 +ILSVRC2012_val_00031494.JPEG:n09229709 +ILSVRC2012_val_00031495.JPEG:n06785654 +ILSVRC2012_val_00031496.JPEG:n03045698 +ILSVRC2012_val_00031497.JPEG:n04380533 +ILSVRC2012_val_00031498.JPEG:n02835271 +ILSVRC2012_val_00031499.JPEG:n07715103 +ILSVRC2012_val_00031500.JPEG:n03692522 +ILSVRC2012_val_00031501.JPEG:n02950826 +ILSVRC2012_val_00031502.JPEG:n02259212 +ILSVRC2012_val_00031503.JPEG:n03773504 +ILSVRC2012_val_00031504.JPEG:n04560804 +ILSVRC2012_val_00031505.JPEG:n04355933 +ILSVRC2012_val_00031506.JPEG:n02167151 +ILSVRC2012_val_00031507.JPEG:n01695060 +ILSVRC2012_val_00031508.JPEG:n02091635 +ILSVRC2012_val_00031509.JPEG:n07745940 +ILSVRC2012_val_00031510.JPEG:n03958227 +ILSVRC2012_val_00031511.JPEG:n03642806 +ILSVRC2012_val_00031512.JPEG:n01537544 +ILSVRC2012_val_00031513.JPEG:n03733131 +ILSVRC2012_val_00031514.JPEG:n02028035 +ILSVRC2012_val_00031515.JPEG:n02667093 +ILSVRC2012_val_00031516.JPEG:n03617480 +ILSVRC2012_val_00031517.JPEG:n02443484 +ILSVRC2012_val_00031518.JPEG:n04532106 +ILSVRC2012_val_00031519.JPEG:n06874185 +ILSVRC2012_val_00031520.JPEG:n02730930 +ILSVRC2012_val_00031521.JPEG:n01632458 +ILSVRC2012_val_00031522.JPEG:n04067472 +ILSVRC2012_val_00031523.JPEG:n09246464 +ILSVRC2012_val_00031524.JPEG:n02264363 +ILSVRC2012_val_00031525.JPEG:n09229709 +ILSVRC2012_val_00031526.JPEG:n02708093 +ILSVRC2012_val_00031527.JPEG:n03804744 +ILSVRC2012_val_00031528.JPEG:n03042490 +ILSVRC2012_val_00031529.JPEG:n03347037 +ILSVRC2012_val_00031530.JPEG:n02120079 +ILSVRC2012_val_00031531.JPEG:n02098105 +ILSVRC2012_val_00031532.JPEG:n02092339 +ILSVRC2012_val_00031533.JPEG:n03017168 +ILSVRC2012_val_00031534.JPEG:n02099429 +ILSVRC2012_val_00031535.JPEG:n03160309 +ILSVRC2012_val_00031536.JPEG:n12267677 +ILSVRC2012_val_00031537.JPEG:n03642806 +ILSVRC2012_val_00031538.JPEG:n07579787 +ILSVRC2012_val_00031539.JPEG:n02817516 +ILSVRC2012_val_00031540.JPEG:n01770393 +ILSVRC2012_val_00031541.JPEG:n01667114 +ILSVRC2012_val_00031542.JPEG:n04417672 +ILSVRC2012_val_00031543.JPEG:n04515003 +ILSVRC2012_val_00031544.JPEG:n02091134 +ILSVRC2012_val_00031545.JPEG:n02090721 +ILSVRC2012_val_00031546.JPEG:n04428191 +ILSVRC2012_val_00031547.JPEG:n02086646 +ILSVRC2012_val_00031548.JPEG:n04536866 +ILSVRC2012_val_00031549.JPEG:n03000684 +ILSVRC2012_val_00031550.JPEG:n01692333 +ILSVRC2012_val_00031551.JPEG:n04591157 +ILSVRC2012_val_00031552.JPEG:n03967562 +ILSVRC2012_val_00031553.JPEG:n03743016 +ILSVRC2012_val_00031554.JPEG:n04579145 +ILSVRC2012_val_00031555.JPEG:n02110063 +ILSVRC2012_val_00031556.JPEG:n04040759 +ILSVRC2012_val_00031557.JPEG:n02074367 +ILSVRC2012_val_00031558.JPEG:n03100240 +ILSVRC2012_val_00031559.JPEG:n04552348 +ILSVRC2012_val_00031560.JPEG:n02916936 +ILSVRC2012_val_00031561.JPEG:n03485407 +ILSVRC2012_val_00031562.JPEG:n02489166 +ILSVRC2012_val_00031563.JPEG:n03271574 +ILSVRC2012_val_00031564.JPEG:n01677366 +ILSVRC2012_val_00031565.JPEG:n02457408 +ILSVRC2012_val_00031566.JPEG:n02966193 +ILSVRC2012_val_00031567.JPEG:n04152593 +ILSVRC2012_val_00031568.JPEG:n01491361 +ILSVRC2012_val_00031569.JPEG:n01748264 +ILSVRC2012_val_00031570.JPEG:n03530642 +ILSVRC2012_val_00031571.JPEG:n03840681 +ILSVRC2012_val_00031572.JPEG:n01768244 +ILSVRC2012_val_00031573.JPEG:n02226429 +ILSVRC2012_val_00031574.JPEG:n03642806 +ILSVRC2012_val_00031575.JPEG:n02002556 +ILSVRC2012_val_00031576.JPEG:n03598930 +ILSVRC2012_val_00031577.JPEG:n01631663 +ILSVRC2012_val_00031578.JPEG:n03787032 +ILSVRC2012_val_00031579.JPEG:n03954731 +ILSVRC2012_val_00031580.JPEG:n04462240 +ILSVRC2012_val_00031581.JPEG:n03680355 +ILSVRC2012_val_00031582.JPEG:n02013706 +ILSVRC2012_val_00031583.JPEG:n03271574 +ILSVRC2012_val_00031584.JPEG:n04357314 +ILSVRC2012_val_00031585.JPEG:n02397096 +ILSVRC2012_val_00031586.JPEG:n01697457 +ILSVRC2012_val_00031587.JPEG:n02441942 +ILSVRC2012_val_00031588.JPEG:n03661043 +ILSVRC2012_val_00031589.JPEG:n01985128 +ILSVRC2012_val_00031590.JPEG:n03658185 +ILSVRC2012_val_00031591.JPEG:n02099267 +ILSVRC2012_val_00031592.JPEG:n04522168 +ILSVRC2012_val_00031593.JPEG:n13037406 +ILSVRC2012_val_00031594.JPEG:n02108422 +ILSVRC2012_val_00031595.JPEG:n04111531 +ILSVRC2012_val_00031596.JPEG:n01728920 +ILSVRC2012_val_00031597.JPEG:n02085620 +ILSVRC2012_val_00031598.JPEG:n01644373 +ILSVRC2012_val_00031599.JPEG:n02101388 +ILSVRC2012_val_00031600.JPEG:n02795169 +ILSVRC2012_val_00031601.JPEG:n02100877 +ILSVRC2012_val_00031602.JPEG:n04509417 +ILSVRC2012_val_00031603.JPEG:n02088466 +ILSVRC2012_val_00031604.JPEG:n02769748 +ILSVRC2012_val_00031605.JPEG:n02965783 +ILSVRC2012_val_00031606.JPEG:n03649909 +ILSVRC2012_val_00031607.JPEG:n03179701 +ILSVRC2012_val_00031608.JPEG:n01742172 +ILSVRC2012_val_00031609.JPEG:n01877812 +ILSVRC2012_val_00031610.JPEG:n03769881 +ILSVRC2012_val_00031611.JPEG:n03000247 +ILSVRC2012_val_00031612.JPEG:n02106662 +ILSVRC2012_val_00031613.JPEG:n03888605 +ILSVRC2012_val_00031614.JPEG:n03937543 +ILSVRC2012_val_00031615.JPEG:n04346328 +ILSVRC2012_val_00031616.JPEG:n03976467 +ILSVRC2012_val_00031617.JPEG:n03187595 +ILSVRC2012_val_00031618.JPEG:n15075141 +ILSVRC2012_val_00031619.JPEG:n03062245 +ILSVRC2012_val_00031620.JPEG:n03710721 +ILSVRC2012_val_00031621.JPEG:n04009552 +ILSVRC2012_val_00031622.JPEG:n02447366 +ILSVRC2012_val_00031623.JPEG:n02107574 +ILSVRC2012_val_00031624.JPEG:n03970156 +ILSVRC2012_val_00031625.JPEG:n03991062 +ILSVRC2012_val_00031626.JPEG:n02098413 +ILSVRC2012_val_00031627.JPEG:n07892512 +ILSVRC2012_val_00031628.JPEG:n03529860 +ILSVRC2012_val_00031629.JPEG:n03935335 +ILSVRC2012_val_00031630.JPEG:n01531178 +ILSVRC2012_val_00031631.JPEG:n02835271 +ILSVRC2012_val_00031632.JPEG:n03787032 +ILSVRC2012_val_00031633.JPEG:n02101388 +ILSVRC2012_val_00031634.JPEG:n02085620 +ILSVRC2012_val_00031635.JPEG:n02701002 +ILSVRC2012_val_00031636.JPEG:n11939491 +ILSVRC2012_val_00031637.JPEG:n01698640 +ILSVRC2012_val_00031638.JPEG:n02233338 +ILSVRC2012_val_00031639.JPEG:n11879895 +ILSVRC2012_val_00031640.JPEG:n02101556 +ILSVRC2012_val_00031641.JPEG:n07753592 +ILSVRC2012_val_00031642.JPEG:n02441942 +ILSVRC2012_val_00031643.JPEG:n07871810 +ILSVRC2012_val_00031644.JPEG:n01914609 +ILSVRC2012_val_00031645.JPEG:n02132136 +ILSVRC2012_val_00031646.JPEG:n02097658 +ILSVRC2012_val_00031647.JPEG:n07720875 +ILSVRC2012_val_00031648.JPEG:n02259212 +ILSVRC2012_val_00031649.JPEG:n01560419 +ILSVRC2012_val_00031650.JPEG:n02510455 +ILSVRC2012_val_00031651.JPEG:n04200800 +ILSVRC2012_val_00031652.JPEG:n04254777 +ILSVRC2012_val_00031653.JPEG:n01616318 +ILSVRC2012_val_00031654.JPEG:n04522168 +ILSVRC2012_val_00031655.JPEG:n02100236 +ILSVRC2012_val_00031656.JPEG:n04356056 +ILSVRC2012_val_00031657.JPEG:n07615774 +ILSVRC2012_val_00031658.JPEG:n03160309 +ILSVRC2012_val_00031659.JPEG:n02666196 +ILSVRC2012_val_00031660.JPEG:n02169497 +ILSVRC2012_val_00031661.JPEG:n03207941 +ILSVRC2012_val_00031662.JPEG:n07831146 +ILSVRC2012_val_00031663.JPEG:n04131690 +ILSVRC2012_val_00031664.JPEG:n04136333 +ILSVRC2012_val_00031665.JPEG:n02895154 +ILSVRC2012_val_00031666.JPEG:n02002556 +ILSVRC2012_val_00031667.JPEG:n04311174 +ILSVRC2012_val_00031668.JPEG:n04243546 +ILSVRC2012_val_00031669.JPEG:n13052670 +ILSVRC2012_val_00031670.JPEG:n02895154 +ILSVRC2012_val_00031671.JPEG:n03527444 +ILSVRC2012_val_00031672.JPEG:n02090622 +ILSVRC2012_val_00031673.JPEG:n04429376 +ILSVRC2012_val_00031674.JPEG:n01667778 +ILSVRC2012_val_00031675.JPEG:n01871265 +ILSVRC2012_val_00031676.JPEG:n01608432 +ILSVRC2012_val_00031677.JPEG:n03424325 +ILSVRC2012_val_00031678.JPEG:n02111129 +ILSVRC2012_val_00031679.JPEG:n02094114 +ILSVRC2012_val_00031680.JPEG:n03706229 +ILSVRC2012_val_00031681.JPEG:n02883205 +ILSVRC2012_val_00031682.JPEG:n07590611 +ILSVRC2012_val_00031683.JPEG:n02948072 +ILSVRC2012_val_00031684.JPEG:n01770393 +ILSVRC2012_val_00031685.JPEG:n03290653 +ILSVRC2012_val_00031686.JPEG:n02128925 +ILSVRC2012_val_00031687.JPEG:n02110185 +ILSVRC2012_val_00031688.JPEG:n02110341 +ILSVRC2012_val_00031689.JPEG:n01796340 +ILSVRC2012_val_00031690.JPEG:n02342885 +ILSVRC2012_val_00031691.JPEG:n02487347 +ILSVRC2012_val_00031692.JPEG:n04310018 +ILSVRC2012_val_00031693.JPEG:n02091635 +ILSVRC2012_val_00031694.JPEG:n02708093 +ILSVRC2012_val_00031695.JPEG:n03016953 +ILSVRC2012_val_00031696.JPEG:n02264363 +ILSVRC2012_val_00031697.JPEG:n04372370 +ILSVRC2012_val_00031698.JPEG:n03272562 +ILSVRC2012_val_00031699.JPEG:n02089078 +ILSVRC2012_val_00031700.JPEG:n03764736 +ILSVRC2012_val_00031701.JPEG:n02963159 +ILSVRC2012_val_00031702.JPEG:n03874599 +ILSVRC2012_val_00031703.JPEG:n02641379 +ILSVRC2012_val_00031704.JPEG:n01984695 +ILSVRC2012_val_00031705.JPEG:n02802426 +ILSVRC2012_val_00031706.JPEG:n02346627 +ILSVRC2012_val_00031707.JPEG:n03773504 +ILSVRC2012_val_00031708.JPEG:n04273569 +ILSVRC2012_val_00031709.JPEG:n02111889 +ILSVRC2012_val_00031710.JPEG:n03498962 +ILSVRC2012_val_00031711.JPEG:n03141823 +ILSVRC2012_val_00031712.JPEG:n04350905 +ILSVRC2012_val_00031713.JPEG:n02095314 +ILSVRC2012_val_00031714.JPEG:n04335435 +ILSVRC2012_val_00031715.JPEG:n03388183 +ILSVRC2012_val_00031716.JPEG:n01537544 +ILSVRC2012_val_00031717.JPEG:n03947888 +ILSVRC2012_val_00031718.JPEG:n02106662 +ILSVRC2012_val_00031719.JPEG:n03854065 +ILSVRC2012_val_00031720.JPEG:n01484850 +ILSVRC2012_val_00031721.JPEG:n02086079 +ILSVRC2012_val_00031722.JPEG:n07714571 +ILSVRC2012_val_00031723.JPEG:n01768244 +ILSVRC2012_val_00031724.JPEG:n04070727 +ILSVRC2012_val_00031725.JPEG:n03494278 +ILSVRC2012_val_00031726.JPEG:n03584829 +ILSVRC2012_val_00031727.JPEG:n03837869 +ILSVRC2012_val_00031728.JPEG:n01945685 +ILSVRC2012_val_00031729.JPEG:n03733281 +ILSVRC2012_val_00031730.JPEG:n04429376 +ILSVRC2012_val_00031731.JPEG:n02099601 +ILSVRC2012_val_00031732.JPEG:n04554684 +ILSVRC2012_val_00031733.JPEG:n04509417 +ILSVRC2012_val_00031734.JPEG:n01943899 +ILSVRC2012_val_00031735.JPEG:n07565083 +ILSVRC2012_val_00031736.JPEG:n04515003 +ILSVRC2012_val_00031737.JPEG:n03777754 +ILSVRC2012_val_00031738.JPEG:n03594734 +ILSVRC2012_val_00031739.JPEG:n03777568 +ILSVRC2012_val_00031740.JPEG:n03840681 +ILSVRC2012_val_00031741.JPEG:n02536864 +ILSVRC2012_val_00031742.JPEG:n04442312 +ILSVRC2012_val_00031743.JPEG:n03127747 +ILSVRC2012_val_00031744.JPEG:n03445777 +ILSVRC2012_val_00031745.JPEG:n04579432 +ILSVRC2012_val_00031746.JPEG:n03063599 +ILSVRC2012_val_00031747.JPEG:n02113978 +ILSVRC2012_val_00031748.JPEG:n03787032 +ILSVRC2012_val_00031749.JPEG:n01742172 +ILSVRC2012_val_00031750.JPEG:n02487347 +ILSVRC2012_val_00031751.JPEG:n04486054 +ILSVRC2012_val_00031752.JPEG:n02093859 +ILSVRC2012_val_00031753.JPEG:n04162706 +ILSVRC2012_val_00031754.JPEG:n02328150 +ILSVRC2012_val_00031755.JPEG:n03482405 +ILSVRC2012_val_00031756.JPEG:n04517823 +ILSVRC2012_val_00031757.JPEG:n07615774 +ILSVRC2012_val_00031758.JPEG:n04192698 +ILSVRC2012_val_00031759.JPEG:n02808304 +ILSVRC2012_val_00031760.JPEG:n02037110 +ILSVRC2012_val_00031761.JPEG:n04254120 +ILSVRC2012_val_00031762.JPEG:n02490219 +ILSVRC2012_val_00031763.JPEG:n07684084 +ILSVRC2012_val_00031764.JPEG:n02094258 +ILSVRC2012_val_00031765.JPEG:n02814533 +ILSVRC2012_val_00031766.JPEG:n02174001 +ILSVRC2012_val_00031767.JPEG:n07753275 +ILSVRC2012_val_00031768.JPEG:n04033901 +ILSVRC2012_val_00031769.JPEG:n02481823 +ILSVRC2012_val_00031770.JPEG:n03770679 +ILSVRC2012_val_00031771.JPEG:n03134739 +ILSVRC2012_val_00031772.JPEG:n01560419 +ILSVRC2012_val_00031773.JPEG:n04275548 +ILSVRC2012_val_00031774.JPEG:n01667778 +ILSVRC2012_val_00031775.JPEG:n01737021 +ILSVRC2012_val_00031776.JPEG:n01806567 +ILSVRC2012_val_00031777.JPEG:n04456115 +ILSVRC2012_val_00031778.JPEG:n07613480 +ILSVRC2012_val_00031779.JPEG:n01737021 +ILSVRC2012_val_00031780.JPEG:n03761084 +ILSVRC2012_val_00031781.JPEG:n07753592 +ILSVRC2012_val_00031782.JPEG:n04461696 +ILSVRC2012_val_00031783.JPEG:n04336792 +ILSVRC2012_val_00031784.JPEG:n02137549 +ILSVRC2012_val_00031785.JPEG:n02100735 +ILSVRC2012_val_00031786.JPEG:n04005630 +ILSVRC2012_val_00031787.JPEG:n02112706 +ILSVRC2012_val_00031788.JPEG:n12144580 +ILSVRC2012_val_00031789.JPEG:n03785016 +ILSVRC2012_val_00031790.JPEG:n03372029 +ILSVRC2012_val_00031791.JPEG:n04486054 +ILSVRC2012_val_00031792.JPEG:n02117135 +ILSVRC2012_val_00031793.JPEG:n01667778 +ILSVRC2012_val_00031794.JPEG:n02927161 +ILSVRC2012_val_00031795.JPEG:n07760859 +ILSVRC2012_val_00031796.JPEG:n03924679 +ILSVRC2012_val_00031797.JPEG:n04040759 +ILSVRC2012_val_00031798.JPEG:n07742313 +ILSVRC2012_val_00031799.JPEG:n02106030 +ILSVRC2012_val_00031800.JPEG:n03388549 +ILSVRC2012_val_00031801.JPEG:n03950228 +ILSVRC2012_val_00031802.JPEG:n01768244 +ILSVRC2012_val_00031803.JPEG:n07734744 +ILSVRC2012_val_00031804.JPEG:n04479046 +ILSVRC2012_val_00031805.JPEG:n02791124 +ILSVRC2012_val_00031806.JPEG:n01807496 +ILSVRC2012_val_00031807.JPEG:n04357314 +ILSVRC2012_val_00031808.JPEG:n01484850 +ILSVRC2012_val_00031809.JPEG:n03888605 +ILSVRC2012_val_00031810.JPEG:n04277352 +ILSVRC2012_val_00031811.JPEG:n04326547 +ILSVRC2012_val_00031812.JPEG:n03876231 +ILSVRC2012_val_00031813.JPEG:n07584110 +ILSVRC2012_val_00031814.JPEG:n02092002 +ILSVRC2012_val_00031815.JPEG:n01667778 +ILSVRC2012_val_00031816.JPEG:n01682714 +ILSVRC2012_val_00031817.JPEG:n02091831 +ILSVRC2012_val_00031818.JPEG:n02108089 +ILSVRC2012_val_00031819.JPEG:n02951585 +ILSVRC2012_val_00031820.JPEG:n02219486 +ILSVRC2012_val_00031821.JPEG:n02090379 +ILSVRC2012_val_00031822.JPEG:n01950731 +ILSVRC2012_val_00031823.JPEG:n02089867 +ILSVRC2012_val_00031824.JPEG:n01828970 +ILSVRC2012_val_00031825.JPEG:n03837869 +ILSVRC2012_val_00031826.JPEG:n01978287 +ILSVRC2012_val_00031827.JPEG:n02092002 +ILSVRC2012_val_00031828.JPEG:n02814533 +ILSVRC2012_val_00031829.JPEG:n01664065 +ILSVRC2012_val_00031830.JPEG:n12768682 +ILSVRC2012_val_00031831.JPEG:n07930864 +ILSVRC2012_val_00031832.JPEG:n04357314 +ILSVRC2012_val_00031833.JPEG:n02802426 +ILSVRC2012_val_00031834.JPEG:n02089867 +ILSVRC2012_val_00031835.JPEG:n03063689 +ILSVRC2012_val_00031836.JPEG:n03535780 +ILSVRC2012_val_00031837.JPEG:n04591713 +ILSVRC2012_val_00031838.JPEG:n03796401 +ILSVRC2012_val_00031839.JPEG:n02877765 +ILSVRC2012_val_00031840.JPEG:n02823428 +ILSVRC2012_val_00031841.JPEG:n07717410 +ILSVRC2012_val_00031842.JPEG:n04612504 +ILSVRC2012_val_00031843.JPEG:n03642806 +ILSVRC2012_val_00031844.JPEG:n04033995 +ILSVRC2012_val_00031845.JPEG:n02095889 +ILSVRC2012_val_00031846.JPEG:n04074963 +ILSVRC2012_val_00031847.JPEG:n01855032 +ILSVRC2012_val_00031848.JPEG:n04270147 +ILSVRC2012_val_00031849.JPEG:n03110669 +ILSVRC2012_val_00031850.JPEG:n03255030 +ILSVRC2012_val_00031851.JPEG:n03530642 +ILSVRC2012_val_00031852.JPEG:n10148035 +ILSVRC2012_val_00031853.JPEG:n07745940 +ILSVRC2012_val_00031854.JPEG:n02490219 +ILSVRC2012_val_00031855.JPEG:n02074367 +ILSVRC2012_val_00031856.JPEG:n02097130 +ILSVRC2012_val_00031857.JPEG:n02106662 +ILSVRC2012_val_00031858.JPEG:n03891332 +ILSVRC2012_val_00031859.JPEG:n02089973 +ILSVRC2012_val_00031860.JPEG:n04209239 +ILSVRC2012_val_00031861.JPEG:n04548280 +ILSVRC2012_val_00031862.JPEG:n04154565 +ILSVRC2012_val_00031863.JPEG:n02037110 +ILSVRC2012_val_00031864.JPEG:n02113978 +ILSVRC2012_val_00031865.JPEG:n02115913 +ILSVRC2012_val_00031866.JPEG:n02018795 +ILSVRC2012_val_00031867.JPEG:n02823428 +ILSVRC2012_val_00031868.JPEG:n02091032 +ILSVRC2012_val_00031869.JPEG:n03874293 +ILSVRC2012_val_00031870.JPEG:n04146614 +ILSVRC2012_val_00031871.JPEG:n04560804 +ILSVRC2012_val_00031872.JPEG:n04522168 +ILSVRC2012_val_00031873.JPEG:n07717556 +ILSVRC2012_val_00031874.JPEG:n04311004 +ILSVRC2012_val_00031875.JPEG:n02105855 +ILSVRC2012_val_00031876.JPEG:n02109961 +ILSVRC2012_val_00031877.JPEG:n02134084 +ILSVRC2012_val_00031878.JPEG:n02930766 +ILSVRC2012_val_00031879.JPEG:n01855032 +ILSVRC2012_val_00031880.JPEG:n02480495 +ILSVRC2012_val_00031881.JPEG:n02509815 +ILSVRC2012_val_00031882.JPEG:n02100877 +ILSVRC2012_val_00031883.JPEG:n02795169 +ILSVRC2012_val_00031884.JPEG:n02125311 +ILSVRC2012_val_00031885.JPEG:n01734418 +ILSVRC2012_val_00031886.JPEG:n03124043 +ILSVRC2012_val_00031887.JPEG:n02165105 +ILSVRC2012_val_00031888.JPEG:n02840245 +ILSVRC2012_val_00031889.JPEG:n03759954 +ILSVRC2012_val_00031890.JPEG:n01622779 +ILSVRC2012_val_00031891.JPEG:n02442845 +ILSVRC2012_val_00031892.JPEG:n04328186 +ILSVRC2012_val_00031893.JPEG:n04152593 +ILSVRC2012_val_00031894.JPEG:n04554684 +ILSVRC2012_val_00031895.JPEG:n02965783 +ILSVRC2012_val_00031896.JPEG:n02510455 +ILSVRC2012_val_00031897.JPEG:n03445777 +ILSVRC2012_val_00031898.JPEG:n07615774 +ILSVRC2012_val_00031899.JPEG:n12998815 +ILSVRC2012_val_00031900.JPEG:n07717410 +ILSVRC2012_val_00031901.JPEG:n03742115 +ILSVRC2012_val_00031902.JPEG:n04264628 +ILSVRC2012_val_00031903.JPEG:n02165456 +ILSVRC2012_val_00031904.JPEG:n04074963 +ILSVRC2012_val_00031905.JPEG:n02098105 +ILSVRC2012_val_00031906.JPEG:n02132136 +ILSVRC2012_val_00031907.JPEG:n01872401 +ILSVRC2012_val_00031908.JPEG:n02441942 +ILSVRC2012_val_00031909.JPEG:n04560804 +ILSVRC2012_val_00031910.JPEG:n02422699 +ILSVRC2012_val_00031911.JPEG:n02802426 +ILSVRC2012_val_00031912.JPEG:n07768694 +ILSVRC2012_val_00031913.JPEG:n01518878 +ILSVRC2012_val_00031914.JPEG:n02096051 +ILSVRC2012_val_00031915.JPEG:n02786058 +ILSVRC2012_val_00031916.JPEG:n02483708 +ILSVRC2012_val_00031917.JPEG:n02099601 +ILSVRC2012_val_00031918.JPEG:n04435653 +ILSVRC2012_val_00031919.JPEG:n01630670 +ILSVRC2012_val_00031920.JPEG:n02177972 +ILSVRC2012_val_00031921.JPEG:n13052670 +ILSVRC2012_val_00031922.JPEG:n02028035 +ILSVRC2012_val_00031923.JPEG:n01978455 +ILSVRC2012_val_00031924.JPEG:n13054560 +ILSVRC2012_val_00031925.JPEG:n02165105 +ILSVRC2012_val_00031926.JPEG:n04317175 +ILSVRC2012_val_00031927.JPEG:n01739381 +ILSVRC2012_val_00031928.JPEG:n02168699 +ILSVRC2012_val_00031929.JPEG:n02483362 +ILSVRC2012_val_00031930.JPEG:n02342885 +ILSVRC2012_val_00031931.JPEG:n02007558 +ILSVRC2012_val_00031932.JPEG:n01798484 +ILSVRC2012_val_00031933.JPEG:n04579145 +ILSVRC2012_val_00031934.JPEG:n02361337 +ILSVRC2012_val_00031935.JPEG:n02643566 +ILSVRC2012_val_00031936.JPEG:n04147183 +ILSVRC2012_val_00031937.JPEG:n04208210 +ILSVRC2012_val_00031938.JPEG:n01798484 +ILSVRC2012_val_00031939.JPEG:n02488291 +ILSVRC2012_val_00031940.JPEG:n03773504 +ILSVRC2012_val_00031941.JPEG:n03662601 +ILSVRC2012_val_00031942.JPEG:n02483708 +ILSVRC2012_val_00031943.JPEG:n01986214 +ILSVRC2012_val_00031944.JPEG:n04005630 +ILSVRC2012_val_00031945.JPEG:n02165105 +ILSVRC2012_val_00031946.JPEG:n02009229 +ILSVRC2012_val_00031947.JPEG:n03814639 +ILSVRC2012_val_00031948.JPEG:n04462240 +ILSVRC2012_val_00031949.JPEG:n02090379 +ILSVRC2012_val_00031950.JPEG:n03786901 +ILSVRC2012_val_00031951.JPEG:n01734418 +ILSVRC2012_val_00031952.JPEG:n01770081 +ILSVRC2012_val_00031953.JPEG:n02814533 +ILSVRC2012_val_00031954.JPEG:n03445777 +ILSVRC2012_val_00031955.JPEG:n03196217 +ILSVRC2012_val_00031956.JPEG:n02747177 +ILSVRC2012_val_00031957.JPEG:n02493793 +ILSVRC2012_val_00031958.JPEG:n03970156 +ILSVRC2012_val_00031959.JPEG:n02165105 +ILSVRC2012_val_00031960.JPEG:n03930313 +ILSVRC2012_val_00031961.JPEG:n02169497 +ILSVRC2012_val_00031962.JPEG:n04204347 +ILSVRC2012_val_00031963.JPEG:n02113712 +ILSVRC2012_val_00031964.JPEG:n02979186 +ILSVRC2012_val_00031965.JPEG:n02085782 +ILSVRC2012_val_00031966.JPEG:n04265275 +ILSVRC2012_val_00031967.JPEG:n01694178 +ILSVRC2012_val_00031968.JPEG:n09229709 +ILSVRC2012_val_00031969.JPEG:n04317175 +ILSVRC2012_val_00031970.JPEG:n07760859 +ILSVRC2012_val_00031971.JPEG:n02865351 +ILSVRC2012_val_00031972.JPEG:n03841143 +ILSVRC2012_val_00031973.JPEG:n01601694 +ILSVRC2012_val_00031974.JPEG:n02128925 +ILSVRC2012_val_00031975.JPEG:n03908714 +ILSVRC2012_val_00031976.JPEG:n01775062 +ILSVRC2012_val_00031977.JPEG:n01770393 +ILSVRC2012_val_00031978.JPEG:n02877765 +ILSVRC2012_val_00031979.JPEG:n03902125 +ILSVRC2012_val_00031980.JPEG:n01744401 +ILSVRC2012_val_00031981.JPEG:n02094114 +ILSVRC2012_val_00031982.JPEG:n03271574 +ILSVRC2012_val_00031983.JPEG:n04372370 +ILSVRC2012_val_00031984.JPEG:n07697313 +ILSVRC2012_val_00031985.JPEG:n04229816 +ILSVRC2012_val_00031986.JPEG:n02692877 +ILSVRC2012_val_00031987.JPEG:n01537544 +ILSVRC2012_val_00031988.JPEG:n04153751 +ILSVRC2012_val_00031989.JPEG:n02490219 +ILSVRC2012_val_00031990.JPEG:n09193705 +ILSVRC2012_val_00031991.JPEG:n02951585 +ILSVRC2012_val_00031992.JPEG:n01986214 +ILSVRC2012_val_00031993.JPEG:n02865351 +ILSVRC2012_val_00031994.JPEG:n02105855 +ILSVRC2012_val_00031995.JPEG:n04392985 +ILSVRC2012_val_00031996.JPEG:n03825788 +ILSVRC2012_val_00031997.JPEG:n04265275 +ILSVRC2012_val_00031998.JPEG:n12267677 +ILSVRC2012_val_00031999.JPEG:n03787032 +ILSVRC2012_val_00032000.JPEG:n02088632 +ILSVRC2012_val_00032001.JPEG:n04507155 +ILSVRC2012_val_00032002.JPEG:n03481172 +ILSVRC2012_val_00032003.JPEG:n03868242 +ILSVRC2012_val_00032004.JPEG:n02797295 +ILSVRC2012_val_00032005.JPEG:n02500267 +ILSVRC2012_val_00032006.JPEG:n02480855 +ILSVRC2012_val_00032007.JPEG:n03956157 +ILSVRC2012_val_00032008.JPEG:n02948072 +ILSVRC2012_val_00032009.JPEG:n03792782 +ILSVRC2012_val_00032010.JPEG:n03478589 +ILSVRC2012_val_00032011.JPEG:n04590129 +ILSVRC2012_val_00032012.JPEG:n01729322 +ILSVRC2012_val_00032013.JPEG:n02105056 +ILSVRC2012_val_00032014.JPEG:n02837789 +ILSVRC2012_val_00032015.JPEG:n03393912 +ILSVRC2012_val_00032016.JPEG:n02319095 +ILSVRC2012_val_00032017.JPEG:n02100735 +ILSVRC2012_val_00032018.JPEG:n02093256 +ILSVRC2012_val_00032019.JPEG:n03782006 +ILSVRC2012_val_00032020.JPEG:n03388043 +ILSVRC2012_val_00032021.JPEG:n03891251 +ILSVRC2012_val_00032022.JPEG:n02391049 +ILSVRC2012_val_00032023.JPEG:n02167151 +ILSVRC2012_val_00032024.JPEG:n03045698 +ILSVRC2012_val_00032025.JPEG:n01534433 +ILSVRC2012_val_00032026.JPEG:n04067472 +ILSVRC2012_val_00032027.JPEG:n02105641 +ILSVRC2012_val_00032028.JPEG:n04423845 +ILSVRC2012_val_00032029.JPEG:n01983481 +ILSVRC2012_val_00032030.JPEG:n03160309 +ILSVRC2012_val_00032031.JPEG:n02802426 +ILSVRC2012_val_00032032.JPEG:n09428293 +ILSVRC2012_val_00032033.JPEG:n02106382 +ILSVRC2012_val_00032034.JPEG:n04325704 +ILSVRC2012_val_00032035.JPEG:n02444819 +ILSVRC2012_val_00032036.JPEG:n01755581 +ILSVRC2012_val_00032037.JPEG:n02895154 +ILSVRC2012_val_00032038.JPEG:n02129604 +ILSVRC2012_val_00032039.JPEG:n02910353 +ILSVRC2012_val_00032040.JPEG:n07873807 +ILSVRC2012_val_00032041.JPEG:n07716358 +ILSVRC2012_val_00032042.JPEG:n03325584 +ILSVRC2012_val_00032043.JPEG:n02104029 +ILSVRC2012_val_00032044.JPEG:n01883070 +ILSVRC2012_val_00032045.JPEG:n02408429 +ILSVRC2012_val_00032046.JPEG:n02992529 +ILSVRC2012_val_00032047.JPEG:n02111277 +ILSVRC2012_val_00032048.JPEG:n04141327 +ILSVRC2012_val_00032049.JPEG:n02098105 +ILSVRC2012_val_00032050.JPEG:n12998815 +ILSVRC2012_val_00032051.JPEG:n04133789 +ILSVRC2012_val_00032052.JPEG:n02837789 +ILSVRC2012_val_00032053.JPEG:n02321529 +ILSVRC2012_val_00032054.JPEG:n04041544 +ILSVRC2012_val_00032055.JPEG:n03131574 +ILSVRC2012_val_00032056.JPEG:n01968897 +ILSVRC2012_val_00032057.JPEG:n03721384 +ILSVRC2012_val_00032058.JPEG:n09428293 +ILSVRC2012_val_00032059.JPEG:n03637318 +ILSVRC2012_val_00032060.JPEG:n04536866 +ILSVRC2012_val_00032061.JPEG:n01641577 +ILSVRC2012_val_00032062.JPEG:n01828970 +ILSVRC2012_val_00032063.JPEG:n02794156 +ILSVRC2012_val_00032064.JPEG:n02105855 +ILSVRC2012_val_00032065.JPEG:n02825657 +ILSVRC2012_val_00032066.JPEG:n02100735 +ILSVRC2012_val_00032067.JPEG:n02487347 +ILSVRC2012_val_00032068.JPEG:n02281406 +ILSVRC2012_val_00032069.JPEG:n04550184 +ILSVRC2012_val_00032070.JPEG:n02804414 +ILSVRC2012_val_00032071.JPEG:n03594734 +ILSVRC2012_val_00032072.JPEG:n01806143 +ILSVRC2012_val_00032073.JPEG:n09256479 +ILSVRC2012_val_00032074.JPEG:n04204238 +ILSVRC2012_val_00032075.JPEG:n03544143 +ILSVRC2012_val_00032076.JPEG:n04350905 +ILSVRC2012_val_00032077.JPEG:n04380533 +ILSVRC2012_val_00032078.JPEG:n03459775 +ILSVRC2012_val_00032079.JPEG:n04509417 +ILSVRC2012_val_00032080.JPEG:n02480495 +ILSVRC2012_val_00032081.JPEG:n04204347 +ILSVRC2012_val_00032082.JPEG:n03967562 +ILSVRC2012_val_00032083.JPEG:n03666591 +ILSVRC2012_val_00032084.JPEG:n03481172 +ILSVRC2012_val_00032085.JPEG:n03179701 +ILSVRC2012_val_00032086.JPEG:n01728920 +ILSVRC2012_val_00032087.JPEG:n09835506 +ILSVRC2012_val_00032088.JPEG:n02509815 +ILSVRC2012_val_00032089.JPEG:n11939491 +ILSVRC2012_val_00032090.JPEG:n02125311 +ILSVRC2012_val_00032091.JPEG:n01774750 +ILSVRC2012_val_00032092.JPEG:n01924916 +ILSVRC2012_val_00032093.JPEG:n04380533 +ILSVRC2012_val_00032094.JPEG:n03496892 +ILSVRC2012_val_00032095.JPEG:n02510455 +ILSVRC2012_val_00032096.JPEG:n02808304 +ILSVRC2012_val_00032097.JPEG:n04328186 +ILSVRC2012_val_00032098.JPEG:n04009552 +ILSVRC2012_val_00032099.JPEG:n02105505 +ILSVRC2012_val_00032100.JPEG:n02454379 +ILSVRC2012_val_00032101.JPEG:n04507155 +ILSVRC2012_val_00032102.JPEG:n01592084 +ILSVRC2012_val_00032103.JPEG:n04118538 +ILSVRC2012_val_00032104.JPEG:n01644373 +ILSVRC2012_val_00032105.JPEG:n02965783 +ILSVRC2012_val_00032106.JPEG:n03742115 +ILSVRC2012_val_00032107.JPEG:n07715103 +ILSVRC2012_val_00032108.JPEG:n03733281 +ILSVRC2012_val_00032109.JPEG:n02268853 +ILSVRC2012_val_00032110.JPEG:n03967562 +ILSVRC2012_val_00032111.JPEG:n02107574 +ILSVRC2012_val_00032112.JPEG:n04597913 +ILSVRC2012_val_00032113.JPEG:n01798484 +ILSVRC2012_val_00032114.JPEG:n04562935 +ILSVRC2012_val_00032115.JPEG:n04584207 +ILSVRC2012_val_00032116.JPEG:n07717556 +ILSVRC2012_val_00032117.JPEG:n02110958 +ILSVRC2012_val_00032118.JPEG:n04597913 +ILSVRC2012_val_00032119.JPEG:n07693725 +ILSVRC2012_val_00032120.JPEG:n02086910 +ILSVRC2012_val_00032121.JPEG:n04136333 +ILSVRC2012_val_00032122.JPEG:n01843383 +ILSVRC2012_val_00032123.JPEG:n02794156 +ILSVRC2012_val_00032124.JPEG:n02101556 +ILSVRC2012_val_00032125.JPEG:n04192698 +ILSVRC2012_val_00032126.JPEG:n02389026 +ILSVRC2012_val_00032127.JPEG:n03250847 +ILSVRC2012_val_00032128.JPEG:n01817953 +ILSVRC2012_val_00032129.JPEG:n01682714 +ILSVRC2012_val_00032130.JPEG:n01491361 +ILSVRC2012_val_00032131.JPEG:n06874185 +ILSVRC2012_val_00032132.JPEG:n02093647 +ILSVRC2012_val_00032133.JPEG:n02483362 +ILSVRC2012_val_00032134.JPEG:n04435653 +ILSVRC2012_val_00032135.JPEG:n01667778 +ILSVRC2012_val_00032136.JPEG:n04548280 +ILSVRC2012_val_00032137.JPEG:n03133878 +ILSVRC2012_val_00032138.JPEG:n02840245 +ILSVRC2012_val_00032139.JPEG:n01950731 +ILSVRC2012_val_00032140.JPEG:n04229816 +ILSVRC2012_val_00032141.JPEG:n01817953 +ILSVRC2012_val_00032142.JPEG:n04346328 +ILSVRC2012_val_00032143.JPEG:n07871810 +ILSVRC2012_val_00032144.JPEG:n04493381 +ILSVRC2012_val_00032145.JPEG:n03476684 +ILSVRC2012_val_00032146.JPEG:n01882714 +ILSVRC2012_val_00032147.JPEG:n03100240 +ILSVRC2012_val_00032148.JPEG:n02105505 +ILSVRC2012_val_00032149.JPEG:n03623198 +ILSVRC2012_val_00032150.JPEG:n02128925 +ILSVRC2012_val_00032151.JPEG:n07749582 +ILSVRC2012_val_00032152.JPEG:n03124170 +ILSVRC2012_val_00032153.JPEG:n03042490 +ILSVRC2012_val_00032154.JPEG:n01531178 +ILSVRC2012_val_00032155.JPEG:n03180011 +ILSVRC2012_val_00032156.JPEG:n02276258 +ILSVRC2012_val_00032157.JPEG:n03538406 +ILSVRC2012_val_00032158.JPEG:n01843383 +ILSVRC2012_val_00032159.JPEG:n01833805 +ILSVRC2012_val_00032160.JPEG:n02109047 +ILSVRC2012_val_00032161.JPEG:n01735189 +ILSVRC2012_val_00032162.JPEG:n01514859 +ILSVRC2012_val_00032163.JPEG:n02396427 +ILSVRC2012_val_00032164.JPEG:n01537544 +ILSVRC2012_val_00032165.JPEG:n07920052 +ILSVRC2012_val_00032166.JPEG:n02077923 +ILSVRC2012_val_00032167.JPEG:n03661043 +ILSVRC2012_val_00032168.JPEG:n03445924 +ILSVRC2012_val_00032169.JPEG:n01514859 +ILSVRC2012_val_00032170.JPEG:n04418357 +ILSVRC2012_val_00032171.JPEG:n01630670 +ILSVRC2012_val_00032172.JPEG:n02256656 +ILSVRC2012_val_00032173.JPEG:n02980441 +ILSVRC2012_val_00032174.JPEG:n01985128 +ILSVRC2012_val_00032175.JPEG:n03787032 +ILSVRC2012_val_00032176.JPEG:n09399592 +ILSVRC2012_val_00032177.JPEG:n02096177 +ILSVRC2012_val_00032178.JPEG:n03095699 +ILSVRC2012_val_00032179.JPEG:n02791270 +ILSVRC2012_val_00032180.JPEG:n02002556 +ILSVRC2012_val_00032181.JPEG:n02099429 +ILSVRC2012_val_00032182.JPEG:n02687172 +ILSVRC2012_val_00032183.JPEG:n04487081 +ILSVRC2012_val_00032184.JPEG:n03775071 +ILSVRC2012_val_00032185.JPEG:n04120489 +ILSVRC2012_val_00032186.JPEG:n02100877 +ILSVRC2012_val_00032187.JPEG:n04131690 +ILSVRC2012_val_00032188.JPEG:n02111277 +ILSVRC2012_val_00032189.JPEG:n04008634 +ILSVRC2012_val_00032190.JPEG:n03796401 +ILSVRC2012_val_00032191.JPEG:n03690938 +ILSVRC2012_val_00032192.JPEG:n03496892 +ILSVRC2012_val_00032193.JPEG:n02487347 +ILSVRC2012_val_00032194.JPEG:n02098286 +ILSVRC2012_val_00032195.JPEG:n04398044 +ILSVRC2012_val_00032196.JPEG:n02281787 +ILSVRC2012_val_00032197.JPEG:n02641379 +ILSVRC2012_val_00032198.JPEG:n03179701 +ILSVRC2012_val_00032199.JPEG:n03110669 +ILSVRC2012_val_00032200.JPEG:n03314780 +ILSVRC2012_val_00032201.JPEG:n03388549 +ILSVRC2012_val_00032202.JPEG:n02441942 +ILSVRC2012_val_00032203.JPEG:n02091831 +ILSVRC2012_val_00032204.JPEG:n03933933 +ILSVRC2012_val_00032205.JPEG:n07584110 +ILSVRC2012_val_00032206.JPEG:n02510455 +ILSVRC2012_val_00032207.JPEG:n02437312 +ILSVRC2012_val_00032208.JPEG:n02417914 +ILSVRC2012_val_00032209.JPEG:n02110806 +ILSVRC2012_val_00032210.JPEG:n02667093 +ILSVRC2012_val_00032211.JPEG:n03384352 +ILSVRC2012_val_00032212.JPEG:n03529860 +ILSVRC2012_val_00032213.JPEG:n04209239 +ILSVRC2012_val_00032214.JPEG:n04254120 +ILSVRC2012_val_00032215.JPEG:n04310018 +ILSVRC2012_val_00032216.JPEG:n07615774 +ILSVRC2012_val_00032217.JPEG:n01984695 +ILSVRC2012_val_00032218.JPEG:n03188531 +ILSVRC2012_val_00032219.JPEG:n02701002 +ILSVRC2012_val_00032220.JPEG:n01749939 +ILSVRC2012_val_00032221.JPEG:n03494278 +ILSVRC2012_val_00032222.JPEG:n04317175 +ILSVRC2012_val_00032223.JPEG:n02480855 +ILSVRC2012_val_00032224.JPEG:n04553703 +ILSVRC2012_val_00032225.JPEG:n04591713 +ILSVRC2012_val_00032226.JPEG:n02093991 +ILSVRC2012_val_00032227.JPEG:n03496892 +ILSVRC2012_val_00032228.JPEG:n03498962 +ILSVRC2012_val_00032229.JPEG:n02870880 +ILSVRC2012_val_00032230.JPEG:n07734744 +ILSVRC2012_val_00032231.JPEG:n02090622 +ILSVRC2012_val_00032232.JPEG:n02095889 +ILSVRC2012_val_00032233.JPEG:n03089624 +ILSVRC2012_val_00032234.JPEG:n03814906 +ILSVRC2012_val_00032235.JPEG:n01443537 +ILSVRC2012_val_00032236.JPEG:n03775546 +ILSVRC2012_val_00032237.JPEG:n03895866 +ILSVRC2012_val_00032238.JPEG:n04254680 +ILSVRC2012_val_00032239.JPEG:n02093991 +ILSVRC2012_val_00032240.JPEG:n02094433 +ILSVRC2012_val_00032241.JPEG:n03709823 +ILSVRC2012_val_00032242.JPEG:n04133789 +ILSVRC2012_val_00032243.JPEG:n04356056 +ILSVRC2012_val_00032244.JPEG:n09421951 +ILSVRC2012_val_00032245.JPEG:n03781244 +ILSVRC2012_val_00032246.JPEG:n03970156 +ILSVRC2012_val_00032247.JPEG:n03709823 +ILSVRC2012_val_00032248.JPEG:n03873416 +ILSVRC2012_val_00032249.JPEG:n03950228 +ILSVRC2012_val_00032250.JPEG:n03425413 +ILSVRC2012_val_00032251.JPEG:n09229709 +ILSVRC2012_val_00032252.JPEG:n03141823 +ILSVRC2012_val_00032253.JPEG:n03290653 +ILSVRC2012_val_00032254.JPEG:n01675722 +ILSVRC2012_val_00032255.JPEG:n04259630 +ILSVRC2012_val_00032256.JPEG:n04613696 +ILSVRC2012_val_00032257.JPEG:n03838899 +ILSVRC2012_val_00032258.JPEG:n01443537 +ILSVRC2012_val_00032259.JPEG:n03617480 +ILSVRC2012_val_00032260.JPEG:n02112350 +ILSVRC2012_val_00032261.JPEG:n01774384 +ILSVRC2012_val_00032262.JPEG:n02108915 +ILSVRC2012_val_00032263.JPEG:n03876231 +ILSVRC2012_val_00032264.JPEG:n02099429 +ILSVRC2012_val_00032265.JPEG:n02226429 +ILSVRC2012_val_00032266.JPEG:n01770393 +ILSVRC2012_val_00032267.JPEG:n01694178 +ILSVRC2012_val_00032268.JPEG:n06794110 +ILSVRC2012_val_00032269.JPEG:n03220513 +ILSVRC2012_val_00032270.JPEG:n11879895 +ILSVRC2012_val_00032271.JPEG:n03124043 +ILSVRC2012_val_00032272.JPEG:n02105855 +ILSVRC2012_val_00032273.JPEG:n02486410 +ILSVRC2012_val_00032274.JPEG:n04004767 +ILSVRC2012_val_00032275.JPEG:n09835506 +ILSVRC2012_val_00032276.JPEG:n07745940 +ILSVRC2012_val_00032277.JPEG:n02097047 +ILSVRC2012_val_00032278.JPEG:n03721384 +ILSVRC2012_val_00032279.JPEG:n03133878 +ILSVRC2012_val_00032280.JPEG:n02093647 +ILSVRC2012_val_00032281.JPEG:n06794110 +ILSVRC2012_val_00032282.JPEG:n04317175 +ILSVRC2012_val_00032283.JPEG:n02134418 +ILSVRC2012_val_00032284.JPEG:n02692877 +ILSVRC2012_val_00032285.JPEG:n02128757 +ILSVRC2012_val_00032286.JPEG:n03794056 +ILSVRC2012_val_00032287.JPEG:n02727426 +ILSVRC2012_val_00032288.JPEG:n01484850 +ILSVRC2012_val_00032289.JPEG:n02514041 +ILSVRC2012_val_00032290.JPEG:n02106382 +ILSVRC2012_val_00032291.JPEG:n02097298 +ILSVRC2012_val_00032292.JPEG:n04613696 +ILSVRC2012_val_00032293.JPEG:n02701002 +ILSVRC2012_val_00032294.JPEG:n03770439 +ILSVRC2012_val_00032295.JPEG:n01855672 +ILSVRC2012_val_00032296.JPEG:n02328150 +ILSVRC2012_val_00032297.JPEG:n03944341 +ILSVRC2012_val_00032298.JPEG:n09468604 +ILSVRC2012_val_00032299.JPEG:n02281787 +ILSVRC2012_val_00032300.JPEG:n04554684 +ILSVRC2012_val_00032301.JPEG:n02098105 +ILSVRC2012_val_00032302.JPEG:n03179701 +ILSVRC2012_val_00032303.JPEG:n02174001 +ILSVRC2012_val_00032304.JPEG:n02109961 +ILSVRC2012_val_00032305.JPEG:n03742115 +ILSVRC2012_val_00032306.JPEG:n04562935 +ILSVRC2012_val_00032307.JPEG:n03729826 +ILSVRC2012_val_00032308.JPEG:n04133789 +ILSVRC2012_val_00032309.JPEG:n04086273 +ILSVRC2012_val_00032310.JPEG:n01514859 +ILSVRC2012_val_00032311.JPEG:n04597913 +ILSVRC2012_val_00032312.JPEG:n04476259 +ILSVRC2012_val_00032313.JPEG:n01914609 +ILSVRC2012_val_00032314.JPEG:n02095889 +ILSVRC2012_val_00032315.JPEG:n03125729 +ILSVRC2012_val_00032316.JPEG:n04366367 +ILSVRC2012_val_00032317.JPEG:n02443114 +ILSVRC2012_val_00032318.JPEG:n02098413 +ILSVRC2012_val_00032319.JPEG:n03599486 +ILSVRC2012_val_00032320.JPEG:n01614925 +ILSVRC2012_val_00032321.JPEG:n04483307 +ILSVRC2012_val_00032322.JPEG:n02105412 +ILSVRC2012_val_00032323.JPEG:n01631663 +ILSVRC2012_val_00032324.JPEG:n02500267 +ILSVRC2012_val_00032325.JPEG:n02095889 +ILSVRC2012_val_00032326.JPEG:n04264628 +ILSVRC2012_val_00032327.JPEG:n07753592 +ILSVRC2012_val_00032328.JPEG:n02123597 +ILSVRC2012_val_00032329.JPEG:n03884397 +ILSVRC2012_val_00032330.JPEG:n04579432 +ILSVRC2012_val_00032331.JPEG:n03938244 +ILSVRC2012_val_00032332.JPEG:n07831146 +ILSVRC2012_val_00032333.JPEG:n02101006 +ILSVRC2012_val_00032334.JPEG:n02092002 +ILSVRC2012_val_00032335.JPEG:n02006656 +ILSVRC2012_val_00032336.JPEG:n02106166 +ILSVRC2012_val_00032337.JPEG:n04596742 +ILSVRC2012_val_00032338.JPEG:n03770679 +ILSVRC2012_val_00032339.JPEG:n04149813 +ILSVRC2012_val_00032340.JPEG:n04599235 +ILSVRC2012_val_00032341.JPEG:n04332243 +ILSVRC2012_val_00032342.JPEG:n03379051 +ILSVRC2012_val_00032343.JPEG:n01776313 +ILSVRC2012_val_00032344.JPEG:n01806567 +ILSVRC2012_val_00032345.JPEG:n09468604 +ILSVRC2012_val_00032346.JPEG:n04554684 +ILSVRC2012_val_00032347.JPEG:n02747177 +ILSVRC2012_val_00032348.JPEG:n04243546 +ILSVRC2012_val_00032349.JPEG:n03838899 +ILSVRC2012_val_00032350.JPEG:n01855032 +ILSVRC2012_val_00032351.JPEG:n01917289 +ILSVRC2012_val_00032352.JPEG:n02226429 +ILSVRC2012_val_00032353.JPEG:n03706229 +ILSVRC2012_val_00032354.JPEG:n03843555 +ILSVRC2012_val_00032355.JPEG:n07615774 +ILSVRC2012_val_00032356.JPEG:n02268853 +ILSVRC2012_val_00032357.JPEG:n04141975 +ILSVRC2012_val_00032358.JPEG:n01728920 +ILSVRC2012_val_00032359.JPEG:n01531178 +ILSVRC2012_val_00032360.JPEG:n03838899 +ILSVRC2012_val_00032361.JPEG:n09472597 +ILSVRC2012_val_00032362.JPEG:n01847000 +ILSVRC2012_val_00032363.JPEG:n13133613 +ILSVRC2012_val_00032364.JPEG:n04522168 +ILSVRC2012_val_00032365.JPEG:n02088466 +ILSVRC2012_val_00032366.JPEG:n09193705 +ILSVRC2012_val_00032367.JPEG:n03445924 +ILSVRC2012_val_00032368.JPEG:n02092002 +ILSVRC2012_val_00032369.JPEG:n02640242 +ILSVRC2012_val_00032370.JPEG:n07742313 +ILSVRC2012_val_00032371.JPEG:n04612504 +ILSVRC2012_val_00032372.JPEG:n01986214 +ILSVRC2012_val_00032373.JPEG:n09229709 +ILSVRC2012_val_00032374.JPEG:n02488291 +ILSVRC2012_val_00032375.JPEG:n02643566 +ILSVRC2012_val_00032376.JPEG:n03891251 +ILSVRC2012_val_00032377.JPEG:n09468604 +ILSVRC2012_val_00032378.JPEG:n01983481 +ILSVRC2012_val_00032379.JPEG:n07920052 +ILSVRC2012_val_00032380.JPEG:n03770679 +ILSVRC2012_val_00032381.JPEG:n02097130 +ILSVRC2012_val_00032382.JPEG:n03769881 +ILSVRC2012_val_00032383.JPEG:n03498962 +ILSVRC2012_val_00032384.JPEG:n07697537 +ILSVRC2012_val_00032385.JPEG:n02422699 +ILSVRC2012_val_00032386.JPEG:n04254777 +ILSVRC2012_val_00032387.JPEG:n03452741 +ILSVRC2012_val_00032388.JPEG:n04152593 +ILSVRC2012_val_00032389.JPEG:n01616318 +ILSVRC2012_val_00032390.JPEG:n02259212 +ILSVRC2012_val_00032391.JPEG:n03690938 +ILSVRC2012_val_00032392.JPEG:n04501370 +ILSVRC2012_val_00032393.JPEG:n04355933 +ILSVRC2012_val_00032394.JPEG:n01498041 +ILSVRC2012_val_00032395.JPEG:n04023962 +ILSVRC2012_val_00032396.JPEG:n02488702 +ILSVRC2012_val_00032397.JPEG:n04443257 +ILSVRC2012_val_00032398.JPEG:n02091134 +ILSVRC2012_val_00032399.JPEG:n02978881 +ILSVRC2012_val_00032400.JPEG:n02091244 +ILSVRC2012_val_00032401.JPEG:n01756291 +ILSVRC2012_val_00032402.JPEG:n04120489 +ILSVRC2012_val_00032403.JPEG:n04141327 +ILSVRC2012_val_00032404.JPEG:n02504458 +ILSVRC2012_val_00032405.JPEG:n01667778 +ILSVRC2012_val_00032406.JPEG:n02108089 +ILSVRC2012_val_00032407.JPEG:n03843555 +ILSVRC2012_val_00032408.JPEG:n02951358 +ILSVRC2012_val_00032409.JPEG:n01807496 +ILSVRC2012_val_00032410.JPEG:n02102318 +ILSVRC2012_val_00032411.JPEG:n07745940 +ILSVRC2012_val_00032412.JPEG:n06794110 +ILSVRC2012_val_00032413.JPEG:n02363005 +ILSVRC2012_val_00032414.JPEG:n07753113 +ILSVRC2012_val_00032415.JPEG:n01644900 +ILSVRC2012_val_00032416.JPEG:n02363005 +ILSVRC2012_val_00032417.JPEG:n01484850 +ILSVRC2012_val_00032418.JPEG:n02105056 +ILSVRC2012_val_00032419.JPEG:n02107312 +ILSVRC2012_val_00032420.JPEG:n03482405 +ILSVRC2012_val_00032421.JPEG:n01945685 +ILSVRC2012_val_00032422.JPEG:n02823750 +ILSVRC2012_val_00032423.JPEG:n02090622 +ILSVRC2012_val_00032424.JPEG:n03710193 +ILSVRC2012_val_00032425.JPEG:n03379051 +ILSVRC2012_val_00032426.JPEG:n07873807 +ILSVRC2012_val_00032427.JPEG:n04263257 +ILSVRC2012_val_00032428.JPEG:n03062245 +ILSVRC2012_val_00032429.JPEG:n02088632 +ILSVRC2012_val_00032430.JPEG:n04208210 +ILSVRC2012_val_00032431.JPEG:n04141327 +ILSVRC2012_val_00032432.JPEG:n07932039 +ILSVRC2012_val_00032433.JPEG:n02951358 +ILSVRC2012_val_00032434.JPEG:n02790996 +ILSVRC2012_val_00032435.JPEG:n02777292 +ILSVRC2012_val_00032436.JPEG:n02804414 +ILSVRC2012_val_00032437.JPEG:n03970156 +ILSVRC2012_val_00032438.JPEG:n04501370 +ILSVRC2012_val_00032439.JPEG:n02641379 +ILSVRC2012_val_00032440.JPEG:n01774750 +ILSVRC2012_val_00032441.JPEG:n01498041 +ILSVRC2012_val_00032442.JPEG:n04116512 +ILSVRC2012_val_00032443.JPEG:n02233338 +ILSVRC2012_val_00032444.JPEG:n03706229 +ILSVRC2012_val_00032445.JPEG:n02097047 +ILSVRC2012_val_00032446.JPEG:n07697537 +ILSVRC2012_val_00032447.JPEG:n02444819 +ILSVRC2012_val_00032448.JPEG:n04153751 +ILSVRC2012_val_00032449.JPEG:n02398521 +ILSVRC2012_val_00032450.JPEG:n03908714 +ILSVRC2012_val_00032451.JPEG:n02088632 +ILSVRC2012_val_00032452.JPEG:n02113712 +ILSVRC2012_val_00032453.JPEG:n02132136 +ILSVRC2012_val_00032454.JPEG:n04258138 +ILSVRC2012_val_00032455.JPEG:n03425413 +ILSVRC2012_val_00032456.JPEG:n02397096 +ILSVRC2012_val_00032457.JPEG:n02443484 +ILSVRC2012_val_00032458.JPEG:n06785654 +ILSVRC2012_val_00032459.JPEG:n04367480 +ILSVRC2012_val_00032460.JPEG:n03717622 +ILSVRC2012_val_00032461.JPEG:n03721384 +ILSVRC2012_val_00032462.JPEG:n02981792 +ILSVRC2012_val_00032463.JPEG:n01955084 +ILSVRC2012_val_00032464.JPEG:n02090721 +ILSVRC2012_val_00032465.JPEG:n02879718 +ILSVRC2012_val_00032466.JPEG:n02113712 +ILSVRC2012_val_00032467.JPEG:n02417914 +ILSVRC2012_val_00032468.JPEG:n02093859 +ILSVRC2012_val_00032469.JPEG:n02009912 +ILSVRC2012_val_00032470.JPEG:n02006656 +ILSVRC2012_val_00032471.JPEG:n01770393 +ILSVRC2012_val_00032472.JPEG:n02701002 +ILSVRC2012_val_00032473.JPEG:n01818515 +ILSVRC2012_val_00032474.JPEG:n12998815 +ILSVRC2012_val_00032475.JPEG:n03532672 +ILSVRC2012_val_00032476.JPEG:n03666591 +ILSVRC2012_val_00032477.JPEG:n06794110 +ILSVRC2012_val_00032478.JPEG:n03110669 +ILSVRC2012_val_00032479.JPEG:n03220513 +ILSVRC2012_val_00032480.JPEG:n03976467 +ILSVRC2012_val_00032481.JPEG:n02396427 +ILSVRC2012_val_00032482.JPEG:n03888257 +ILSVRC2012_val_00032483.JPEG:n02514041 +ILSVRC2012_val_00032484.JPEG:n02837789 +ILSVRC2012_val_00032485.JPEG:n07711569 +ILSVRC2012_val_00032486.JPEG:n07613480 +ILSVRC2012_val_00032487.JPEG:n03075370 +ILSVRC2012_val_00032488.JPEG:n07684084 +ILSVRC2012_val_00032489.JPEG:n02708093 +ILSVRC2012_val_00032490.JPEG:n02099267 +ILSVRC2012_val_00032491.JPEG:n03131574 +ILSVRC2012_val_00032492.JPEG:n01843383 +ILSVRC2012_val_00032493.JPEG:n02091032 +ILSVRC2012_val_00032494.JPEG:n03796401 +ILSVRC2012_val_00032495.JPEG:n04243546 +ILSVRC2012_val_00032496.JPEG:n04389033 +ILSVRC2012_val_00032497.JPEG:n03014705 +ILSVRC2012_val_00032498.JPEG:n03868863 +ILSVRC2012_val_00032499.JPEG:n01883070 +ILSVRC2012_val_00032500.JPEG:n01744401 +ILSVRC2012_val_00032501.JPEG:n12267677 +ILSVRC2012_val_00032502.JPEG:n03876231 +ILSVRC2012_val_00032503.JPEG:n01847000 +ILSVRC2012_val_00032504.JPEG:n02219486 +ILSVRC2012_val_00032505.JPEG:n01955084 +ILSVRC2012_val_00032506.JPEG:n03089624 +ILSVRC2012_val_00032507.JPEG:n04350905 +ILSVRC2012_val_00032508.JPEG:n02119022 +ILSVRC2012_val_00032509.JPEG:n04004767 +ILSVRC2012_val_00032510.JPEG:n02793495 +ILSVRC2012_val_00032511.JPEG:n03404251 +ILSVRC2012_val_00032512.JPEG:n03014705 +ILSVRC2012_val_00032513.JPEG:n01677366 +ILSVRC2012_val_00032514.JPEG:n03690938 +ILSVRC2012_val_00032515.JPEG:n04162706 +ILSVRC2012_val_00032516.JPEG:n04552348 +ILSVRC2012_val_00032517.JPEG:n01985128 +ILSVRC2012_val_00032518.JPEG:n07873807 +ILSVRC2012_val_00032519.JPEG:n02526121 +ILSVRC2012_val_00032520.JPEG:n07932039 +ILSVRC2012_val_00032521.JPEG:n02102973 +ILSVRC2012_val_00032522.JPEG:n02108000 +ILSVRC2012_val_00032523.JPEG:n04493381 +ILSVRC2012_val_00032524.JPEG:n02097130 +ILSVRC2012_val_00032525.JPEG:n04086273 +ILSVRC2012_val_00032526.JPEG:n03832673 +ILSVRC2012_val_00032527.JPEG:n02088364 +ILSVRC2012_val_00032528.JPEG:n02119789 +ILSVRC2012_val_00032529.JPEG:n02113712 +ILSVRC2012_val_00032530.JPEG:n07716906 +ILSVRC2012_val_00032531.JPEG:n03792972 +ILSVRC2012_val_00032532.JPEG:n02097658 +ILSVRC2012_val_00032533.JPEG:n02226429 +ILSVRC2012_val_00032534.JPEG:n09428293 +ILSVRC2012_val_00032535.JPEG:n02116738 +ILSVRC2012_val_00032536.JPEG:n07753113 +ILSVRC2012_val_00032537.JPEG:n02777292 +ILSVRC2012_val_00032538.JPEG:n02017213 +ILSVRC2012_val_00032539.JPEG:n04209239 +ILSVRC2012_val_00032540.JPEG:n02077923 +ILSVRC2012_val_00032541.JPEG:n02509815 +ILSVRC2012_val_00032542.JPEG:n07716906 +ILSVRC2012_val_00032543.JPEG:n02843684 +ILSVRC2012_val_00032544.JPEG:n02417914 +ILSVRC2012_val_00032545.JPEG:n07920052 +ILSVRC2012_val_00032546.JPEG:n09288635 +ILSVRC2012_val_00032547.JPEG:n01980166 +ILSVRC2012_val_00032548.JPEG:n09193705 +ILSVRC2012_val_00032549.JPEG:n03124043 +ILSVRC2012_val_00032550.JPEG:n03944341 +ILSVRC2012_val_00032551.JPEG:n02219486 +ILSVRC2012_val_00032552.JPEG:n02127052 +ILSVRC2012_val_00032553.JPEG:n04147183 +ILSVRC2012_val_00032554.JPEG:n02106550 +ILSVRC2012_val_00032555.JPEG:n04550184 +ILSVRC2012_val_00032556.JPEG:n01728572 +ILSVRC2012_val_00032557.JPEG:n02102480 +ILSVRC2012_val_00032558.JPEG:n04371430 +ILSVRC2012_val_00032559.JPEG:n03983396 +ILSVRC2012_val_00032560.JPEG:n02815834 +ILSVRC2012_val_00032561.JPEG:n04264628 +ILSVRC2012_val_00032562.JPEG:n04356056 +ILSVRC2012_val_00032563.JPEG:n02096294 +ILSVRC2012_val_00032564.JPEG:n02106382 +ILSVRC2012_val_00032565.JPEG:n07579787 +ILSVRC2012_val_00032566.JPEG:n02536864 +ILSVRC2012_val_00032567.JPEG:n03630383 +ILSVRC2012_val_00032568.JPEG:n02114367 +ILSVRC2012_val_00032569.JPEG:n03781244 +ILSVRC2012_val_00032570.JPEG:n03271574 +ILSVRC2012_val_00032571.JPEG:n01739381 +ILSVRC2012_val_00032572.JPEG:n04008634 +ILSVRC2012_val_00032573.JPEG:n03594734 +ILSVRC2012_val_00032574.JPEG:n03201208 +ILSVRC2012_val_00032575.JPEG:n02058221 +ILSVRC2012_val_00032576.JPEG:n02134418 +ILSVRC2012_val_00032577.JPEG:n10148035 +ILSVRC2012_val_00032578.JPEG:n01631663 +ILSVRC2012_val_00032579.JPEG:n02526121 +ILSVRC2012_val_00032580.JPEG:n02002556 +ILSVRC2012_val_00032581.JPEG:n02095314 +ILSVRC2012_val_00032582.JPEG:n02098105 +ILSVRC2012_val_00032583.JPEG:n04509417 +ILSVRC2012_val_00032584.JPEG:n04612504 +ILSVRC2012_val_00032585.JPEG:n02497673 +ILSVRC2012_val_00032586.JPEG:n01580077 +ILSVRC2012_val_00032587.JPEG:n01697457 +ILSVRC2012_val_00032588.JPEG:n03109150 +ILSVRC2012_val_00032589.JPEG:n09468604 +ILSVRC2012_val_00032590.JPEG:n03874293 +ILSVRC2012_val_00032591.JPEG:n02109961 +ILSVRC2012_val_00032592.JPEG:n02110627 +ILSVRC2012_val_00032593.JPEG:n02892201 +ILSVRC2012_val_00032594.JPEG:n02088364 +ILSVRC2012_val_00032595.JPEG:n03100240 +ILSVRC2012_val_00032596.JPEG:n03532672 +ILSVRC2012_val_00032597.JPEG:n02892767 +ILSVRC2012_val_00032598.JPEG:n07860988 +ILSVRC2012_val_00032599.JPEG:n03337140 +ILSVRC2012_val_00032600.JPEG:n02951358 +ILSVRC2012_val_00032601.JPEG:n03691459 +ILSVRC2012_val_00032602.JPEG:n03134739 +ILSVRC2012_val_00032603.JPEG:n02422106 +ILSVRC2012_val_00032604.JPEG:n02788148 +ILSVRC2012_val_00032605.JPEG:n03814906 +ILSVRC2012_val_00032606.JPEG:n02444819 +ILSVRC2012_val_00032607.JPEG:n06785654 +ILSVRC2012_val_00032608.JPEG:n04612504 +ILSVRC2012_val_00032609.JPEG:n02123394 +ILSVRC2012_val_00032610.JPEG:n03042490 +ILSVRC2012_val_00032611.JPEG:n04116512 +ILSVRC2012_val_00032612.JPEG:n03527444 +ILSVRC2012_val_00032613.JPEG:n09288635 +ILSVRC2012_val_00032614.JPEG:n01983481 +ILSVRC2012_val_00032615.JPEG:n09332890 +ILSVRC2012_val_00032616.JPEG:n07715103 +ILSVRC2012_val_00032617.JPEG:n01828970 +ILSVRC2012_val_00032618.JPEG:n04037443 +ILSVRC2012_val_00032619.JPEG:n03089624 +ILSVRC2012_val_00032620.JPEG:n02504458 +ILSVRC2012_val_00032621.JPEG:n01917289 +ILSVRC2012_val_00032622.JPEG:n03223299 +ILSVRC2012_val_00032623.JPEG:n02119022 +ILSVRC2012_val_00032624.JPEG:n02206856 +ILSVRC2012_val_00032625.JPEG:n04252077 +ILSVRC2012_val_00032626.JPEG:n02012849 +ILSVRC2012_val_00032627.JPEG:n02037110 +ILSVRC2012_val_00032628.JPEG:n01751748 +ILSVRC2012_val_00032629.JPEG:n07930864 +ILSVRC2012_val_00032630.JPEG:n04131690 +ILSVRC2012_val_00032631.JPEG:n07697313 +ILSVRC2012_val_00032632.JPEG:n02841315 +ILSVRC2012_val_00032633.JPEG:n03950228 +ILSVRC2012_val_00032634.JPEG:n04254680 +ILSVRC2012_val_00032635.JPEG:n04141975 +ILSVRC2012_val_00032636.JPEG:n03983396 +ILSVRC2012_val_00032637.JPEG:n02124075 +ILSVRC2012_val_00032638.JPEG:n12998815 +ILSVRC2012_val_00032639.JPEG:n03709823 +ILSVRC2012_val_00032640.JPEG:n01689811 +ILSVRC2012_val_00032641.JPEG:n02966687 +ILSVRC2012_val_00032642.JPEG:n03590841 +ILSVRC2012_val_00032643.JPEG:n02002556 +ILSVRC2012_val_00032644.JPEG:n01770393 +ILSVRC2012_val_00032645.JPEG:n04532106 +ILSVRC2012_val_00032646.JPEG:n02109961 +ILSVRC2012_val_00032647.JPEG:n04286575 +ILSVRC2012_val_00032648.JPEG:n02910353 +ILSVRC2012_val_00032649.JPEG:n03785016 +ILSVRC2012_val_00032650.JPEG:n04125021 +ILSVRC2012_val_00032651.JPEG:n04370456 +ILSVRC2012_val_00032652.JPEG:n02115641 +ILSVRC2012_val_00032653.JPEG:n03874293 +ILSVRC2012_val_00032654.JPEG:n13054560 +ILSVRC2012_val_00032655.JPEG:n02480855 +ILSVRC2012_val_00032656.JPEG:n02105855 +ILSVRC2012_val_00032657.JPEG:n01773157 +ILSVRC2012_val_00032658.JPEG:n02108915 +ILSVRC2012_val_00032659.JPEG:n02108000 +ILSVRC2012_val_00032660.JPEG:n03764736 +ILSVRC2012_val_00032661.JPEG:n02231487 +ILSVRC2012_val_00032662.JPEG:n04507155 +ILSVRC2012_val_00032663.JPEG:n01744401 +ILSVRC2012_val_00032664.JPEG:n04325704 +ILSVRC2012_val_00032665.JPEG:n02526121 +ILSVRC2012_val_00032666.JPEG:n04371774 +ILSVRC2012_val_00032667.JPEG:n01582220 +ILSVRC2012_val_00032668.JPEG:n02088094 +ILSVRC2012_val_00032669.JPEG:n12267677 +ILSVRC2012_val_00032670.JPEG:n07880968 +ILSVRC2012_val_00032671.JPEG:n04266014 +ILSVRC2012_val_00032672.JPEG:n02417914 +ILSVRC2012_val_00032673.JPEG:n04270147 +ILSVRC2012_val_00032674.JPEG:n07684084 +ILSVRC2012_val_00032675.JPEG:n01443537 +ILSVRC2012_val_00032676.JPEG:n03866082 +ILSVRC2012_val_00032677.JPEG:n04179913 +ILSVRC2012_val_00032678.JPEG:n02422106 +ILSVRC2012_val_00032679.JPEG:n07697537 +ILSVRC2012_val_00032680.JPEG:n02687172 +ILSVRC2012_val_00032681.JPEG:n03803284 +ILSVRC2012_val_00032682.JPEG:n01692333 +ILSVRC2012_val_00032683.JPEG:n04192698 +ILSVRC2012_val_00032684.JPEG:n02481823 +ILSVRC2012_val_00032685.JPEG:n02115913 +ILSVRC2012_val_00032686.JPEG:n03404251 +ILSVRC2012_val_00032687.JPEG:n02138441 +ILSVRC2012_val_00032688.JPEG:n02999410 +ILSVRC2012_val_00032689.JPEG:n03388183 +ILSVRC2012_val_00032690.JPEG:n02317335 +ILSVRC2012_val_00032691.JPEG:n03759954 +ILSVRC2012_val_00032692.JPEG:n04335435 +ILSVRC2012_val_00032693.JPEG:n03814906 +ILSVRC2012_val_00032694.JPEG:n03692522 +ILSVRC2012_val_00032695.JPEG:n13052670 +ILSVRC2012_val_00032696.JPEG:n03729826 +ILSVRC2012_val_00032697.JPEG:n02790996 +ILSVRC2012_val_00032698.JPEG:n02012849 +ILSVRC2012_val_00032699.JPEG:n03935335 +ILSVRC2012_val_00032700.JPEG:n01667114 +ILSVRC2012_val_00032701.JPEG:n07836838 +ILSVRC2012_val_00032702.JPEG:n01580077 +ILSVRC2012_val_00032703.JPEG:n07615774 +ILSVRC2012_val_00032704.JPEG:n03535780 +ILSVRC2012_val_00032705.JPEG:n02226429 +ILSVRC2012_val_00032706.JPEG:n03903868 +ILSVRC2012_val_00032707.JPEG:n02999410 +ILSVRC2012_val_00032708.JPEG:n03532672 +ILSVRC2012_val_00032709.JPEG:n03498962 +ILSVRC2012_val_00032710.JPEG:n01531178 +ILSVRC2012_val_00032711.JPEG:n03868242 +ILSVRC2012_val_00032712.JPEG:n02128757 +ILSVRC2012_val_00032713.JPEG:n03793489 +ILSVRC2012_val_00032714.JPEG:n01755581 +ILSVRC2012_val_00032715.JPEG:n09332890 +ILSVRC2012_val_00032716.JPEG:n02087394 +ILSVRC2012_val_00032717.JPEG:n03920288 +ILSVRC2012_val_00032718.JPEG:n02128385 +ILSVRC2012_val_00032719.JPEG:n03495258 +ILSVRC2012_val_00032720.JPEG:n02114712 +ILSVRC2012_val_00032721.JPEG:n03976467 +ILSVRC2012_val_00032722.JPEG:n04259630 +ILSVRC2012_val_00032723.JPEG:n02794156 +ILSVRC2012_val_00032724.JPEG:n01774384 +ILSVRC2012_val_00032725.JPEG:n02091467 +ILSVRC2012_val_00032726.JPEG:n04467665 +ILSVRC2012_val_00032727.JPEG:n02091635 +ILSVRC2012_val_00032728.JPEG:n04579432 +ILSVRC2012_val_00032729.JPEG:n03599486 +ILSVRC2012_val_00032730.JPEG:n02328150 +ILSVRC2012_val_00032731.JPEG:n04147183 +ILSVRC2012_val_00032732.JPEG:n02486410 +ILSVRC2012_val_00032733.JPEG:n04252077 +ILSVRC2012_val_00032734.JPEG:n02395406 +ILSVRC2012_val_00032735.JPEG:n07584110 +ILSVRC2012_val_00032736.JPEG:n03075370 +ILSVRC2012_val_00032737.JPEG:n02138441 +ILSVRC2012_val_00032738.JPEG:n02105505 +ILSVRC2012_val_00032739.JPEG:n04311004 +ILSVRC2012_val_00032740.JPEG:n04086273 +ILSVRC2012_val_00032741.JPEG:n04435653 +ILSVRC2012_val_00032742.JPEG:n04467665 +ILSVRC2012_val_00032743.JPEG:n04201297 +ILSVRC2012_val_00032744.JPEG:n01689811 +ILSVRC2012_val_00032745.JPEG:n03345487 +ILSVRC2012_val_00032746.JPEG:n02090379 +ILSVRC2012_val_00032747.JPEG:n02776631 +ILSVRC2012_val_00032748.JPEG:n04023962 +ILSVRC2012_val_00032749.JPEG:n02114367 +ILSVRC2012_val_00032750.JPEG:n13044778 +ILSVRC2012_val_00032751.JPEG:n02917067 +ILSVRC2012_val_00032752.JPEG:n07711569 +ILSVRC2012_val_00032753.JPEG:n03452741 +ILSVRC2012_val_00032754.JPEG:n01734418 +ILSVRC2012_val_00032755.JPEG:n03272010 +ILSVRC2012_val_00032756.JPEG:n01744401 +ILSVRC2012_val_00032757.JPEG:n09399592 +ILSVRC2012_val_00032758.JPEG:n02114855 +ILSVRC2012_val_00032759.JPEG:n03594734 +ILSVRC2012_val_00032760.JPEG:n02860847 +ILSVRC2012_val_00032761.JPEG:n04141076 +ILSVRC2012_val_00032762.JPEG:n02133161 +ILSVRC2012_val_00032763.JPEG:n03804744 +ILSVRC2012_val_00032764.JPEG:n01924916 +ILSVRC2012_val_00032765.JPEG:n04532106 +ILSVRC2012_val_00032766.JPEG:n01770081 +ILSVRC2012_val_00032767.JPEG:n02096177 +ILSVRC2012_val_00032768.JPEG:n02797295 +ILSVRC2012_val_00032769.JPEG:n03188531 +ILSVRC2012_val_00032770.JPEG:n04204347 +ILSVRC2012_val_00032771.JPEG:n03063689 +ILSVRC2012_val_00032772.JPEG:n02841315 +ILSVRC2012_val_00032773.JPEG:n02276258 +ILSVRC2012_val_00032774.JPEG:n02086646 +ILSVRC2012_val_00032775.JPEG:n03775071 +ILSVRC2012_val_00032776.JPEG:n03947888 +ILSVRC2012_val_00032777.JPEG:n02137549 +ILSVRC2012_val_00032778.JPEG:n03063599 +ILSVRC2012_val_00032779.JPEG:n02074367 +ILSVRC2012_val_00032780.JPEG:n02051845 +ILSVRC2012_val_00032781.JPEG:n03832673 +ILSVRC2012_val_00032782.JPEG:n03982430 +ILSVRC2012_val_00032783.JPEG:n01776313 +ILSVRC2012_val_00032784.JPEG:n02102177 +ILSVRC2012_val_00032785.JPEG:n02106550 +ILSVRC2012_val_00032786.JPEG:n03929855 +ILSVRC2012_val_00032787.JPEG:n04201297 +ILSVRC2012_val_00032788.JPEG:n01592084 +ILSVRC2012_val_00032789.JPEG:n02906734 +ILSVRC2012_val_00032790.JPEG:n03124043 +ILSVRC2012_val_00032791.JPEG:n03598930 +ILSVRC2012_val_00032792.JPEG:n07590611 +ILSVRC2012_val_00032793.JPEG:n02091635 +ILSVRC2012_val_00032794.JPEG:n02128757 +ILSVRC2012_val_00032795.JPEG:n04204347 +ILSVRC2012_val_00032796.JPEG:n01698640 +ILSVRC2012_val_00032797.JPEG:n01955084 +ILSVRC2012_val_00032798.JPEG:n03891251 +ILSVRC2012_val_00032799.JPEG:n02823428 +ILSVRC2012_val_00032800.JPEG:n03417042 +ILSVRC2012_val_00032801.JPEG:n03666591 +ILSVRC2012_val_00032802.JPEG:n03958227 +ILSVRC2012_val_00032803.JPEG:n03895866 +ILSVRC2012_val_00032804.JPEG:n02690373 +ILSVRC2012_val_00032805.JPEG:n01667778 +ILSVRC2012_val_00032806.JPEG:n02692877 +ILSVRC2012_val_00032807.JPEG:n03532672 +ILSVRC2012_val_00032808.JPEG:n07920052 +ILSVRC2012_val_00032809.JPEG:n03924679 +ILSVRC2012_val_00032810.JPEG:n03085013 +ILSVRC2012_val_00032811.JPEG:n07697313 +ILSVRC2012_val_00032812.JPEG:n02444819 +ILSVRC2012_val_00032813.JPEG:n02992211 +ILSVRC2012_val_00032814.JPEG:n07248320 +ILSVRC2012_val_00032815.JPEG:n02950826 +ILSVRC2012_val_00032816.JPEG:n02077923 +ILSVRC2012_val_00032817.JPEG:n03786901 +ILSVRC2012_val_00032818.JPEG:n03016953 +ILSVRC2012_val_00032819.JPEG:n02111889 +ILSVRC2012_val_00032820.JPEG:n02892201 +ILSVRC2012_val_00032821.JPEG:n02786058 +ILSVRC2012_val_00032822.JPEG:n02106382 +ILSVRC2012_val_00032823.JPEG:n02877765 +ILSVRC2012_val_00032824.JPEG:n02687172 +ILSVRC2012_val_00032825.JPEG:n02747177 +ILSVRC2012_val_00032826.JPEG:n02105412 +ILSVRC2012_val_00032827.JPEG:n07753113 +ILSVRC2012_val_00032828.JPEG:n03207743 +ILSVRC2012_val_00032829.JPEG:n04418357 +ILSVRC2012_val_00032830.JPEG:n02009912 +ILSVRC2012_val_00032831.JPEG:n01580077 +ILSVRC2012_val_00032832.JPEG:n01616318 +ILSVRC2012_val_00032833.JPEG:n04273569 +ILSVRC2012_val_00032834.JPEG:n01945685 +ILSVRC2012_val_00032835.JPEG:n03706229 +ILSVRC2012_val_00032836.JPEG:n04326547 +ILSVRC2012_val_00032837.JPEG:n02105056 +ILSVRC2012_val_00032838.JPEG:n13037406 +ILSVRC2012_val_00032839.JPEG:n03459775 +ILSVRC2012_val_00032840.JPEG:n02526121 +ILSVRC2012_val_00032841.JPEG:n02837789 +ILSVRC2012_val_00032842.JPEG:n04346328 +ILSVRC2012_val_00032843.JPEG:n01819313 +ILSVRC2012_val_00032844.JPEG:n02321529 +ILSVRC2012_val_00032845.JPEG:n03916031 +ILSVRC2012_val_00032846.JPEG:n03026506 +ILSVRC2012_val_00032847.JPEG:n02105251 +ILSVRC2012_val_00032848.JPEG:n04599235 +ILSVRC2012_val_00032849.JPEG:n01518878 +ILSVRC2012_val_00032850.JPEG:n02110627 +ILSVRC2012_val_00032851.JPEG:n01984695 +ILSVRC2012_val_00032852.JPEG:n01943899 +ILSVRC2012_val_00032853.JPEG:n04069434 +ILSVRC2012_val_00032854.JPEG:n02113023 +ILSVRC2012_val_00032855.JPEG:n01531178 +ILSVRC2012_val_00032856.JPEG:n03947888 +ILSVRC2012_val_00032857.JPEG:n03733805 +ILSVRC2012_val_00032858.JPEG:n03873416 +ILSVRC2012_val_00032859.JPEG:n02087394 +ILSVRC2012_val_00032860.JPEG:n04273569 +ILSVRC2012_val_00032861.JPEG:n03690938 +ILSVRC2012_val_00032862.JPEG:n02281787 +ILSVRC2012_val_00032863.JPEG:n04515003 +ILSVRC2012_val_00032864.JPEG:n01630670 +ILSVRC2012_val_00032865.JPEG:n03445924 +ILSVRC2012_val_00032866.JPEG:n04317175 +ILSVRC2012_val_00032867.JPEG:n02395406 +ILSVRC2012_val_00032868.JPEG:n02018207 +ILSVRC2012_val_00032869.JPEG:n02128385 +ILSVRC2012_val_00032870.JPEG:n03255030 +ILSVRC2012_val_00032871.JPEG:n02169497 +ILSVRC2012_val_00032872.JPEG:n03717622 +ILSVRC2012_val_00032873.JPEG:n03602883 +ILSVRC2012_val_00032874.JPEG:n02488291 +ILSVRC2012_val_00032875.JPEG:n01622779 +ILSVRC2012_val_00032876.JPEG:n03992509 +ILSVRC2012_val_00032877.JPEG:n02877765 +ILSVRC2012_val_00032878.JPEG:n03873416 +ILSVRC2012_val_00032879.JPEG:n01855672 +ILSVRC2012_val_00032880.JPEG:n03478589 +ILSVRC2012_val_00032881.JPEG:n03404251 +ILSVRC2012_val_00032882.JPEG:n07584110 +ILSVRC2012_val_00032883.JPEG:n03980874 +ILSVRC2012_val_00032884.JPEG:n03476684 +ILSVRC2012_val_00032885.JPEG:n02138441 +ILSVRC2012_val_00032886.JPEG:n02977058 +ILSVRC2012_val_00032887.JPEG:n02105162 +ILSVRC2012_val_00032888.JPEG:n03485407 +ILSVRC2012_val_00032889.JPEG:n01616318 +ILSVRC2012_val_00032890.JPEG:n02051845 +ILSVRC2012_val_00032891.JPEG:n03793489 +ILSVRC2012_val_00032892.JPEG:n01768244 +ILSVRC2012_val_00032893.JPEG:n04209239 +ILSVRC2012_val_00032894.JPEG:n03930630 +ILSVRC2012_val_00032895.JPEG:n04532106 +ILSVRC2012_val_00032896.JPEG:n03259280 +ILSVRC2012_val_00032897.JPEG:n02841315 +ILSVRC2012_val_00032898.JPEG:n02966193 +ILSVRC2012_val_00032899.JPEG:n03980874 +ILSVRC2012_val_00032900.JPEG:n04532106 +ILSVRC2012_val_00032901.JPEG:n02981792 +ILSVRC2012_val_00032902.JPEG:n01776313 +ILSVRC2012_val_00032903.JPEG:n04355338 +ILSVRC2012_val_00032904.JPEG:n02110341 +ILSVRC2012_val_00032905.JPEG:n03697007 +ILSVRC2012_val_00032906.JPEG:n02454379 +ILSVRC2012_val_00032907.JPEG:n02655020 +ILSVRC2012_val_00032908.JPEG:n03841143 +ILSVRC2012_val_00032909.JPEG:n07584110 +ILSVRC2012_val_00032910.JPEG:n02123394 +ILSVRC2012_val_00032911.JPEG:n03255030 +ILSVRC2012_val_00032912.JPEG:n07711569 +ILSVRC2012_val_00032913.JPEG:n03724870 +ILSVRC2012_val_00032914.JPEG:n03110669 +ILSVRC2012_val_00032915.JPEG:n03133878 +ILSVRC2012_val_00032916.JPEG:n01641577 +ILSVRC2012_val_00032917.JPEG:n01644373 +ILSVRC2012_val_00032918.JPEG:n04049303 +ILSVRC2012_val_00032919.JPEG:n07768694 +ILSVRC2012_val_00032920.JPEG:n03075370 +ILSVRC2012_val_00032921.JPEG:n02823428 +ILSVRC2012_val_00032922.JPEG:n02640242 +ILSVRC2012_val_00032923.JPEG:n02104365 +ILSVRC2012_val_00032924.JPEG:n04009552 +ILSVRC2012_val_00032925.JPEG:n02129604 +ILSVRC2012_val_00032926.JPEG:n03733805 +ILSVRC2012_val_00032927.JPEG:n02281787 +ILSVRC2012_val_00032928.JPEG:n04208210 +ILSVRC2012_val_00032929.JPEG:n04067472 +ILSVRC2012_val_00032930.JPEG:n01514859 +ILSVRC2012_val_00032931.JPEG:n03384352 +ILSVRC2012_val_00032932.JPEG:n03544143 +ILSVRC2012_val_00032933.JPEG:n03355925 +ILSVRC2012_val_00032934.JPEG:n01694178 +ILSVRC2012_val_00032935.JPEG:n03950228 +ILSVRC2012_val_00032936.JPEG:n07717556 +ILSVRC2012_val_00032937.JPEG:n02317335 +ILSVRC2012_val_00032938.JPEG:n02113799 +ILSVRC2012_val_00032939.JPEG:n07583066 +ILSVRC2012_val_00032940.JPEG:n02999410 +ILSVRC2012_val_00032941.JPEG:n07760859 +ILSVRC2012_val_00032942.JPEG:n02410509 +ILSVRC2012_val_00032943.JPEG:n02013706 +ILSVRC2012_val_00032944.JPEG:n04285008 +ILSVRC2012_val_00032945.JPEG:n04296562 +ILSVRC2012_val_00032946.JPEG:n03196217 +ILSVRC2012_val_00032947.JPEG:n03000134 +ILSVRC2012_val_00032948.JPEG:n02110627 +ILSVRC2012_val_00032949.JPEG:n04442312 +ILSVRC2012_val_00032950.JPEG:n02787622 +ILSVRC2012_val_00032951.JPEG:n02443484 +ILSVRC2012_val_00032952.JPEG:n02137549 +ILSVRC2012_val_00032953.JPEG:n03337140 +ILSVRC2012_val_00032954.JPEG:n03594734 +ILSVRC2012_val_00032955.JPEG:n02879718 +ILSVRC2012_val_00032956.JPEG:n02415577 +ILSVRC2012_val_00032957.JPEG:n02092339 +ILSVRC2012_val_00032958.JPEG:n03450230 +ILSVRC2012_val_00032959.JPEG:n02102040 +ILSVRC2012_val_00032960.JPEG:n07747607 +ILSVRC2012_val_00032961.JPEG:n03085013 +ILSVRC2012_val_00032962.JPEG:n03026506 +ILSVRC2012_val_00032963.JPEG:n06874185 +ILSVRC2012_val_00032964.JPEG:n02493793 +ILSVRC2012_val_00032965.JPEG:n03532672 +ILSVRC2012_val_00032966.JPEG:n01644900 +ILSVRC2012_val_00032967.JPEG:n03792782 +ILSVRC2012_val_00032968.JPEG:n04004767 +ILSVRC2012_val_00032969.JPEG:n02966193 +ILSVRC2012_val_00032970.JPEG:n01784675 +ILSVRC2012_val_00032971.JPEG:n13037406 +ILSVRC2012_val_00032972.JPEG:n03481172 +ILSVRC2012_val_00032973.JPEG:n03775546 +ILSVRC2012_val_00032974.JPEG:n04033995 +ILSVRC2012_val_00032975.JPEG:n02101556 +ILSVRC2012_val_00032976.JPEG:n03666591 +ILSVRC2012_val_00032977.JPEG:n04317175 +ILSVRC2012_val_00032978.JPEG:n01882714 +ILSVRC2012_val_00032979.JPEG:n02640242 +ILSVRC2012_val_00032980.JPEG:n03063689 +ILSVRC2012_val_00032981.JPEG:n04560804 +ILSVRC2012_val_00032982.JPEG:n01860187 +ILSVRC2012_val_00032983.JPEG:n04376876 +ILSVRC2012_val_00032984.JPEG:n04523525 +ILSVRC2012_val_00032985.JPEG:n01833805 +ILSVRC2012_val_00032986.JPEG:n02169497 +ILSVRC2012_val_00032987.JPEG:n03314780 +ILSVRC2012_val_00032988.JPEG:n02988304 +ILSVRC2012_val_00032989.JPEG:n02168699 +ILSVRC2012_val_00032990.JPEG:n04044716 +ILSVRC2012_val_00032991.JPEG:n02109961 +ILSVRC2012_val_00032992.JPEG:n01770393 +ILSVRC2012_val_00032993.JPEG:n01531178 +ILSVRC2012_val_00032994.JPEG:n04152593 +ILSVRC2012_val_00032995.JPEG:n02106662 +ILSVRC2012_val_00032996.JPEG:n04389033 +ILSVRC2012_val_00032997.JPEG:n01735189 +ILSVRC2012_val_00032998.JPEG:n07871810 +ILSVRC2012_val_00032999.JPEG:n04277352 +ILSVRC2012_val_00033000.JPEG:n02077923 +ILSVRC2012_val_00033001.JPEG:n03347037 +ILSVRC2012_val_00033002.JPEG:n02111500 +ILSVRC2012_val_00033003.JPEG:n02088238 +ILSVRC2012_val_00033004.JPEG:n03534580 +ILSVRC2012_val_00033005.JPEG:n03314780 +ILSVRC2012_val_00033006.JPEG:n02791270 +ILSVRC2012_val_00033007.JPEG:n04548280 +ILSVRC2012_val_00033008.JPEG:n03109150 +ILSVRC2012_val_00033009.JPEG:n03944341 +ILSVRC2012_val_00033010.JPEG:n02137549 +ILSVRC2012_val_00033011.JPEG:n04523525 +ILSVRC2012_val_00033012.JPEG:n04592741 +ILSVRC2012_val_00033013.JPEG:n04266014 +ILSVRC2012_val_00033014.JPEG:n01978455 +ILSVRC2012_val_00033015.JPEG:n02091032 +ILSVRC2012_val_00033016.JPEG:n04398044 +ILSVRC2012_val_00033017.JPEG:n02113624 +ILSVRC2012_val_00033018.JPEG:n02408429 +ILSVRC2012_val_00033019.JPEG:n04417672 +ILSVRC2012_val_00033020.JPEG:n04009552 +ILSVRC2012_val_00033021.JPEG:n02231487 +ILSVRC2012_val_00033022.JPEG:n04599235 +ILSVRC2012_val_00033023.JPEG:n07248320 +ILSVRC2012_val_00033024.JPEG:n04086273 +ILSVRC2012_val_00033025.JPEG:n04606251 +ILSVRC2012_val_00033026.JPEG:n03532672 +ILSVRC2012_val_00033027.JPEG:n02112137 +ILSVRC2012_val_00033028.JPEG:n09256479 +ILSVRC2012_val_00033029.JPEG:n04523525 +ILSVRC2012_val_00033030.JPEG:n01697457 +ILSVRC2012_val_00033031.JPEG:n03662601 +ILSVRC2012_val_00033032.JPEG:n04070727 +ILSVRC2012_val_00033033.JPEG:n02098286 +ILSVRC2012_val_00033034.JPEG:n02017213 +ILSVRC2012_val_00033035.JPEG:n02177972 +ILSVRC2012_val_00033036.JPEG:n01689811 +ILSVRC2012_val_00033037.JPEG:n03697007 +ILSVRC2012_val_00033038.JPEG:n03874599 +ILSVRC2012_val_00033039.JPEG:n02110185 +ILSVRC2012_val_00033040.JPEG:n04417672 +ILSVRC2012_val_00033041.JPEG:n04310018 +ILSVRC2012_val_00033042.JPEG:n02130308 +ILSVRC2012_val_00033043.JPEG:n04252077 +ILSVRC2012_val_00033044.JPEG:n03534580 +ILSVRC2012_val_00033045.JPEG:n01860187 +ILSVRC2012_val_00033046.JPEG:n03814906 +ILSVRC2012_val_00033047.JPEG:n02442845 +ILSVRC2012_val_00033048.JPEG:n04487394 +ILSVRC2012_val_00033049.JPEG:n02090379 +ILSVRC2012_val_00033050.JPEG:n01930112 +ILSVRC2012_val_00033051.JPEG:n07860988 +ILSVRC2012_val_00033052.JPEG:n02869837 +ILSVRC2012_val_00033053.JPEG:n02231487 +ILSVRC2012_val_00033054.JPEG:n03956157 +ILSVRC2012_val_00033055.JPEG:n03482405 +ILSVRC2012_val_00033056.JPEG:n02489166 +ILSVRC2012_val_00033057.JPEG:n02107683 +ILSVRC2012_val_00033058.JPEG:n01677366 +ILSVRC2012_val_00033059.JPEG:n01806143 +ILSVRC2012_val_00033060.JPEG:n03775071 +ILSVRC2012_val_00033061.JPEG:n02825657 +ILSVRC2012_val_00033062.JPEG:n02783161 +ILSVRC2012_val_00033063.JPEG:n01622779 +ILSVRC2012_val_00033064.JPEG:n02268853 +ILSVRC2012_val_00033065.JPEG:n04044716 +ILSVRC2012_val_00033066.JPEG:n04540053 +ILSVRC2012_val_00033067.JPEG:n02107142 +ILSVRC2012_val_00033068.JPEG:n04487394 +ILSVRC2012_val_00033069.JPEG:n03376595 +ILSVRC2012_val_00033070.JPEG:n01496331 +ILSVRC2012_val_00033071.JPEG:n02815834 +ILSVRC2012_val_00033072.JPEG:n02099267 +ILSVRC2012_val_00033073.JPEG:n04229816 +ILSVRC2012_val_00033074.JPEG:n07615774 +ILSVRC2012_val_00033075.JPEG:n03272562 +ILSVRC2012_val_00033076.JPEG:n01855672 +ILSVRC2012_val_00033077.JPEG:n02804414 +ILSVRC2012_val_00033078.JPEG:n01818515 +ILSVRC2012_val_00033079.JPEG:n02704792 +ILSVRC2012_val_00033080.JPEG:n02483708 +ILSVRC2012_val_00033081.JPEG:n01629819 +ILSVRC2012_val_00033082.JPEG:n03393912 +ILSVRC2012_val_00033083.JPEG:n03794056 +ILSVRC2012_val_00033084.JPEG:n01644373 +ILSVRC2012_val_00033085.JPEG:n02951585 +ILSVRC2012_val_00033086.JPEG:n02497673 +ILSVRC2012_val_00033087.JPEG:n02415577 +ILSVRC2012_val_00033088.JPEG:n01871265 +ILSVRC2012_val_00033089.JPEG:n07718747 +ILSVRC2012_val_00033090.JPEG:n02966193 +ILSVRC2012_val_00033091.JPEG:n03017168 +ILSVRC2012_val_00033092.JPEG:n01530575 +ILSVRC2012_val_00033093.JPEG:n02319095 +ILSVRC2012_val_00033094.JPEG:n02090379 +ILSVRC2012_val_00033095.JPEG:n03297495 +ILSVRC2012_val_00033096.JPEG:n03388183 +ILSVRC2012_val_00033097.JPEG:n03825788 +ILSVRC2012_val_00033098.JPEG:n01798484 +ILSVRC2012_val_00033099.JPEG:n03814906 +ILSVRC2012_val_00033100.JPEG:n02027492 +ILSVRC2012_val_00033101.JPEG:n02111889 +ILSVRC2012_val_00033102.JPEG:n04118538 +ILSVRC2012_val_00033103.JPEG:n02356798 +ILSVRC2012_val_00033104.JPEG:n01983481 +ILSVRC2012_val_00033105.JPEG:n01986214 +ILSVRC2012_val_00033106.JPEG:n02808440 +ILSVRC2012_val_00033107.JPEG:n02486261 +ILSVRC2012_val_00033108.JPEG:n01751748 +ILSVRC2012_val_00033109.JPEG:n03777568 +ILSVRC2012_val_00033110.JPEG:n04335435 +ILSVRC2012_val_00033111.JPEG:n07720875 +ILSVRC2012_val_00033112.JPEG:n03633091 +ILSVRC2012_val_00033113.JPEG:n03534580 +ILSVRC2012_val_00033114.JPEG:n04141975 +ILSVRC2012_val_00033115.JPEG:n04162706 +ILSVRC2012_val_00033116.JPEG:n03998194 +ILSVRC2012_val_00033117.JPEG:n07579787 +ILSVRC2012_val_00033118.JPEG:n02676566 +ILSVRC2012_val_00033119.JPEG:n03483316 +ILSVRC2012_val_00033120.JPEG:n01693334 +ILSVRC2012_val_00033121.JPEG:n04238763 +ILSVRC2012_val_00033122.JPEG:n02071294 +ILSVRC2012_val_00033123.JPEG:n04493381 +ILSVRC2012_val_00033124.JPEG:n07875152 +ILSVRC2012_val_00033125.JPEG:n01753488 +ILSVRC2012_val_00033126.JPEG:n02091635 +ILSVRC2012_val_00033127.JPEG:n03314780 +ILSVRC2012_val_00033128.JPEG:n03291819 +ILSVRC2012_val_00033129.JPEG:n03924679 +ILSVRC2012_val_00033130.JPEG:n12768682 +ILSVRC2012_val_00033131.JPEG:n06794110 +ILSVRC2012_val_00033132.JPEG:n03291819 +ILSVRC2012_val_00033133.JPEG:n03544143 +ILSVRC2012_val_00033134.JPEG:n01698640 +ILSVRC2012_val_00033135.JPEG:n06785654 +ILSVRC2012_val_00033136.JPEG:n03782006 +ILSVRC2012_val_00033137.JPEG:n04154565 +ILSVRC2012_val_00033138.JPEG:n02012849 +ILSVRC2012_val_00033139.JPEG:n07930864 +ILSVRC2012_val_00033140.JPEG:n03017168 +ILSVRC2012_val_00033141.JPEG:n04133789 +ILSVRC2012_val_00033142.JPEG:n02138441 +ILSVRC2012_val_00033143.JPEG:n03769881 +ILSVRC2012_val_00033144.JPEG:n03773504 +ILSVRC2012_val_00033145.JPEG:n07930864 +ILSVRC2012_val_00033146.JPEG:n04589890 +ILSVRC2012_val_00033147.JPEG:n01806143 +ILSVRC2012_val_00033148.JPEG:n03207743 +ILSVRC2012_val_00033149.JPEG:n02097474 +ILSVRC2012_val_00033150.JPEG:n01582220 +ILSVRC2012_val_00033151.JPEG:n02939185 +ILSVRC2012_val_00033152.JPEG:n02640242 +ILSVRC2012_val_00033153.JPEG:n02981792 +ILSVRC2012_val_00033154.JPEG:n03657121 +ILSVRC2012_val_00033155.JPEG:n02106166 +ILSVRC2012_val_00033156.JPEG:n02666196 +ILSVRC2012_val_00033157.JPEG:n01751748 +ILSVRC2012_val_00033158.JPEG:n03188531 +ILSVRC2012_val_00033159.JPEG:n01768244 +ILSVRC2012_val_00033160.JPEG:n04429376 +ILSVRC2012_val_00033161.JPEG:n02690373 +ILSVRC2012_val_00033162.JPEG:n01806567 +ILSVRC2012_val_00033163.JPEG:n02319095 +ILSVRC2012_val_00033164.JPEG:n02107683 +ILSVRC2012_val_00033165.JPEG:n04550184 +ILSVRC2012_val_00033166.JPEG:n04350905 +ILSVRC2012_val_00033167.JPEG:n01797886 +ILSVRC2012_val_00033168.JPEG:n04447861 +ILSVRC2012_val_00033169.JPEG:n04485082 +ILSVRC2012_val_00033170.JPEG:n03443371 +ILSVRC2012_val_00033171.JPEG:n04229816 +ILSVRC2012_val_00033172.JPEG:n03443371 +ILSVRC2012_val_00033173.JPEG:n04579145 +ILSVRC2012_val_00033174.JPEG:n03125729 +ILSVRC2012_val_00033175.JPEG:n03942813 +ILSVRC2012_val_00033176.JPEG:n03649909 +ILSVRC2012_val_00033177.JPEG:n02119022 +ILSVRC2012_val_00033178.JPEG:n02105251 +ILSVRC2012_val_00033179.JPEG:n12144580 +ILSVRC2012_val_00033180.JPEG:n02992529 +ILSVRC2012_val_00033181.JPEG:n01518878 +ILSVRC2012_val_00033182.JPEG:n02977058 +ILSVRC2012_val_00033183.JPEG:n01968897 +ILSVRC2012_val_00033184.JPEG:n02233338 +ILSVRC2012_val_00033185.JPEG:n03642806 +ILSVRC2012_val_00033186.JPEG:n01833805 +ILSVRC2012_val_00033187.JPEG:n09421951 +ILSVRC2012_val_00033188.JPEG:n01985128 +ILSVRC2012_val_00033189.JPEG:n01824575 +ILSVRC2012_val_00033190.JPEG:n04286575 +ILSVRC2012_val_00033191.JPEG:n04330267 +ILSVRC2012_val_00033192.JPEG:n02106166 +ILSVRC2012_val_00033193.JPEG:n07875152 +ILSVRC2012_val_00033194.JPEG:n02094258 +ILSVRC2012_val_00033195.JPEG:n02123394 +ILSVRC2012_val_00033196.JPEG:n01537544 +ILSVRC2012_val_00033197.JPEG:n04493381 +ILSVRC2012_val_00033198.JPEG:n02102480 +ILSVRC2012_val_00033199.JPEG:n02086240 +ILSVRC2012_val_00033200.JPEG:n02085782 +ILSVRC2012_val_00033201.JPEG:n03786901 +ILSVRC2012_val_00033202.JPEG:n04254680 +ILSVRC2012_val_00033203.JPEG:n03721384 +ILSVRC2012_val_00033204.JPEG:n04311174 +ILSVRC2012_val_00033205.JPEG:n04487394 +ILSVRC2012_val_00033206.JPEG:n02099267 +ILSVRC2012_val_00033207.JPEG:n03207941 +ILSVRC2012_val_00033208.JPEG:n02883205 +ILSVRC2012_val_00033209.JPEG:n02672831 +ILSVRC2012_val_00033210.JPEG:n04008634 +ILSVRC2012_val_00033211.JPEG:n03868863 +ILSVRC2012_val_00033212.JPEG:n04251144 +ILSVRC2012_val_00033213.JPEG:n03529860 +ILSVRC2012_val_00033214.JPEG:n01608432 +ILSVRC2012_val_00033215.JPEG:n02093647 +ILSVRC2012_val_00033216.JPEG:n02028035 +ILSVRC2012_val_00033217.JPEG:n03982430 +ILSVRC2012_val_00033218.JPEG:n01687978 +ILSVRC2012_val_00033219.JPEG:n01632458 +ILSVRC2012_val_00033220.JPEG:n03125729 +ILSVRC2012_val_00033221.JPEG:n02389026 +ILSVRC2012_val_00033222.JPEG:n02085782 +ILSVRC2012_val_00033223.JPEG:n06359193 +ILSVRC2012_val_00033224.JPEG:n03459775 +ILSVRC2012_val_00033225.JPEG:n01773797 +ILSVRC2012_val_00033226.JPEG:n02093754 +ILSVRC2012_val_00033227.JPEG:n04275548 +ILSVRC2012_val_00033228.JPEG:n02120505 +ILSVRC2012_val_00033229.JPEG:n03450230 +ILSVRC2012_val_00033230.JPEG:n03854065 +ILSVRC2012_val_00033231.JPEG:n02096177 +ILSVRC2012_val_00033232.JPEG:n02112706 +ILSVRC2012_val_00033233.JPEG:n02089867 +ILSVRC2012_val_00033234.JPEG:n02138441 +ILSVRC2012_val_00033235.JPEG:n02504458 +ILSVRC2012_val_00033236.JPEG:n02865351 +ILSVRC2012_val_00033237.JPEG:n04479046 +ILSVRC2012_val_00033238.JPEG:n03180011 +ILSVRC2012_val_00033239.JPEG:n03223299 +ILSVRC2012_val_00033240.JPEG:n02804414 +ILSVRC2012_val_00033241.JPEG:n02134418 +ILSVRC2012_val_00033242.JPEG:n01751748 +ILSVRC2012_val_00033243.JPEG:n02483708 +ILSVRC2012_val_00033244.JPEG:n01692333 +ILSVRC2012_val_00033245.JPEG:n02992211 +ILSVRC2012_val_00033246.JPEG:n03404251 +ILSVRC2012_val_00033247.JPEG:n07716906 +ILSVRC2012_val_00033248.JPEG:n01924916 +ILSVRC2012_val_00033249.JPEG:n07695742 +ILSVRC2012_val_00033250.JPEG:n02112137 +ILSVRC2012_val_00033251.JPEG:n02692877 +ILSVRC2012_val_00033252.JPEG:n02423022 +ILSVRC2012_val_00033253.JPEG:n02860847 +ILSVRC2012_val_00033254.JPEG:n01877812 +ILSVRC2012_val_00033255.JPEG:n04326547 +ILSVRC2012_val_00033256.JPEG:n02051845 +ILSVRC2012_val_00033257.JPEG:n01855672 +ILSVRC2012_val_00033258.JPEG:n02667093 +ILSVRC2012_val_00033259.JPEG:n01829413 +ILSVRC2012_val_00033260.JPEG:n07760859 +ILSVRC2012_val_00033261.JPEG:n01630670 +ILSVRC2012_val_00033262.JPEG:n02869837 +ILSVRC2012_val_00033263.JPEG:n02086910 +ILSVRC2012_val_00033264.JPEG:n01740131 +ILSVRC2012_val_00033265.JPEG:n02398521 +ILSVRC2012_val_00033266.JPEG:n03016953 +ILSVRC2012_val_00033267.JPEG:n02091134 +ILSVRC2012_val_00033268.JPEG:n02096585 +ILSVRC2012_val_00033269.JPEG:n02093647 +ILSVRC2012_val_00033270.JPEG:n03220513 +ILSVRC2012_val_00033271.JPEG:n07716906 +ILSVRC2012_val_00033272.JPEG:n03188531 +ILSVRC2012_val_00033273.JPEG:n03627232 +ILSVRC2012_val_00033274.JPEG:n03690938 +ILSVRC2012_val_00033275.JPEG:n02788148 +ILSVRC2012_val_00033276.JPEG:n04254680 +ILSVRC2012_val_00033277.JPEG:n02493509 +ILSVRC2012_val_00033278.JPEG:n02098413 +ILSVRC2012_val_00033279.JPEG:n03532672 +ILSVRC2012_val_00033280.JPEG:n02111889 +ILSVRC2012_val_00033281.JPEG:n01843065 +ILSVRC2012_val_00033282.JPEG:n02666196 +ILSVRC2012_val_00033283.JPEG:n02457408 +ILSVRC2012_val_00033284.JPEG:n03785016 +ILSVRC2012_val_00033285.JPEG:n02097474 +ILSVRC2012_val_00033286.JPEG:n02704792 +ILSVRC2012_val_00033287.JPEG:n03868863 +ILSVRC2012_val_00033288.JPEG:n04540053 +ILSVRC2012_val_00033289.JPEG:n03529860 +ILSVRC2012_val_00033290.JPEG:n04238763 +ILSVRC2012_val_00033291.JPEG:n03658185 +ILSVRC2012_val_00033292.JPEG:n03970156 +ILSVRC2012_val_00033293.JPEG:n04285008 +ILSVRC2012_val_00033294.JPEG:n02526121 +ILSVRC2012_val_00033295.JPEG:n02096585 +ILSVRC2012_val_00033296.JPEG:n03814639 +ILSVRC2012_val_00033297.JPEG:n03180011 +ILSVRC2012_val_00033298.JPEG:n02480855 +ILSVRC2012_val_00033299.JPEG:n03594945 +ILSVRC2012_val_00033300.JPEG:n02101006 +ILSVRC2012_val_00033301.JPEG:n04517823 +ILSVRC2012_val_00033302.JPEG:n12985857 +ILSVRC2012_val_00033303.JPEG:n02104029 +ILSVRC2012_val_00033304.JPEG:n04111531 +ILSVRC2012_val_00033305.JPEG:n01729322 +ILSVRC2012_val_00033306.JPEG:n03773504 +ILSVRC2012_val_00033307.JPEG:n01580077 +ILSVRC2012_val_00033308.JPEG:n02098413 +ILSVRC2012_val_00033309.JPEG:n04065272 +ILSVRC2012_val_00033310.JPEG:n02085936 +ILSVRC2012_val_00033311.JPEG:n02093859 +ILSVRC2012_val_00033312.JPEG:n02104365 +ILSVRC2012_val_00033313.JPEG:n09472597 +ILSVRC2012_val_00033314.JPEG:n02865351 +ILSVRC2012_val_00033315.JPEG:n04254680 +ILSVRC2012_val_00033316.JPEG:n02951358 +ILSVRC2012_val_00033317.JPEG:n02281787 +ILSVRC2012_val_00033318.JPEG:n01496331 +ILSVRC2012_val_00033319.JPEG:n02093256 +ILSVRC2012_val_00033320.JPEG:n01910747 +ILSVRC2012_val_00033321.JPEG:n04509417 +ILSVRC2012_val_00033322.JPEG:n02417914 +ILSVRC2012_val_00033323.JPEG:n02389026 +ILSVRC2012_val_00033324.JPEG:n03666591 +ILSVRC2012_val_00033325.JPEG:n06794110 +ILSVRC2012_val_00033326.JPEG:n03786901 +ILSVRC2012_val_00033327.JPEG:n07695742 +ILSVRC2012_val_00033328.JPEG:n02133161 +ILSVRC2012_val_00033329.JPEG:n04540053 +ILSVRC2012_val_00033330.JPEG:n02782093 +ILSVRC2012_val_00033331.JPEG:n01871265 +ILSVRC2012_val_00033332.JPEG:n03690938 +ILSVRC2012_val_00033333.JPEG:n02028035 +ILSVRC2012_val_00033334.JPEG:n02106550 +ILSVRC2012_val_00033335.JPEG:n02494079 +ILSVRC2012_val_00033336.JPEG:n07831146 +ILSVRC2012_val_00033337.JPEG:n01498041 +ILSVRC2012_val_00033338.JPEG:n02130308 +ILSVRC2012_val_00033339.JPEG:n04483307 +ILSVRC2012_val_00033340.JPEG:n01820546 +ILSVRC2012_val_00033341.JPEG:n02105056 +ILSVRC2012_val_00033342.JPEG:n04487081 +ILSVRC2012_val_00033343.JPEG:n09332890 +ILSVRC2012_val_00033344.JPEG:n02437312 +ILSVRC2012_val_00033345.JPEG:n03692522 +ILSVRC2012_val_00033346.JPEG:n02871525 +ILSVRC2012_val_00033347.JPEG:n02326432 +ILSVRC2012_val_00033348.JPEG:n07749582 +ILSVRC2012_val_00033349.JPEG:n02992211 +ILSVRC2012_val_00033350.JPEG:n02497673 +ILSVRC2012_val_00033351.JPEG:n03544143 +ILSVRC2012_val_00033352.JPEG:n13052670 +ILSVRC2012_val_00033353.JPEG:n13133613 +ILSVRC2012_val_00033354.JPEG:n07714571 +ILSVRC2012_val_00033355.JPEG:n03868863 +ILSVRC2012_val_00033356.JPEG:n02606052 +ILSVRC2012_val_00033357.JPEG:n02111129 +ILSVRC2012_val_00033358.JPEG:n03874293 +ILSVRC2012_val_00033359.JPEG:n02190166 +ILSVRC2012_val_00033360.JPEG:n02226429 +ILSVRC2012_val_00033361.JPEG:n02363005 +ILSVRC2012_val_00033362.JPEG:n02443484 +ILSVRC2012_val_00033363.JPEG:n04579145 +ILSVRC2012_val_00033364.JPEG:n03425413 +ILSVRC2012_val_00033365.JPEG:n03018349 +ILSVRC2012_val_00033366.JPEG:n03452741 +ILSVRC2012_val_00033367.JPEG:n02791124 +ILSVRC2012_val_00033368.JPEG:n02346627 +ILSVRC2012_val_00033369.JPEG:n02128757 +ILSVRC2012_val_00033370.JPEG:n03998194 +ILSVRC2012_val_00033371.JPEG:n03530642 +ILSVRC2012_val_00033372.JPEG:n01592084 +ILSVRC2012_val_00033373.JPEG:n01917289 +ILSVRC2012_val_00033374.JPEG:n03764736 +ILSVRC2012_val_00033375.JPEG:n07615774 +ILSVRC2012_val_00033376.JPEG:n03977966 +ILSVRC2012_val_00033377.JPEG:n02877765 +ILSVRC2012_val_00033378.JPEG:n02089973 +ILSVRC2012_val_00033379.JPEG:n01986214 +ILSVRC2012_val_00033380.JPEG:n01872401 +ILSVRC2012_val_00033381.JPEG:n03942813 +ILSVRC2012_val_00033382.JPEG:n01689811 +ILSVRC2012_val_00033383.JPEG:n02834397 +ILSVRC2012_val_00033384.JPEG:n07714990 +ILSVRC2012_val_00033385.JPEG:n02486261 +ILSVRC2012_val_00033386.JPEG:n02397096 +ILSVRC2012_val_00033387.JPEG:n04467665 +ILSVRC2012_val_00033388.JPEG:n02909870 +ILSVRC2012_val_00033389.JPEG:n04517823 +ILSVRC2012_val_00033390.JPEG:n04131690 +ILSVRC2012_val_00033391.JPEG:n01728572 +ILSVRC2012_val_00033392.JPEG:n01729322 +ILSVRC2012_val_00033393.JPEG:n01797886 +ILSVRC2012_val_00033394.JPEG:n02108551 +ILSVRC2012_val_00033395.JPEG:n03866082 +ILSVRC2012_val_00033396.JPEG:n01677366 +ILSVRC2012_val_00033397.JPEG:n02979186 +ILSVRC2012_val_00033398.JPEG:n03710637 +ILSVRC2012_val_00033399.JPEG:n03933933 +ILSVRC2012_val_00033400.JPEG:n03930313 +ILSVRC2012_val_00033401.JPEG:n03899768 +ILSVRC2012_val_00033402.JPEG:n03763968 +ILSVRC2012_val_00033403.JPEG:n02326432 +ILSVRC2012_val_00033404.JPEG:n02107142 +ILSVRC2012_val_00033405.JPEG:n02066245 +ILSVRC2012_val_00033406.JPEG:n04099969 +ILSVRC2012_val_00033407.JPEG:n07860988 +ILSVRC2012_val_00033408.JPEG:n07695742 +ILSVRC2012_val_00033409.JPEG:n01924916 +ILSVRC2012_val_00033410.JPEG:n03895866 +ILSVRC2012_val_00033411.JPEG:n03788365 +ILSVRC2012_val_00033412.JPEG:n01632777 +ILSVRC2012_val_00033413.JPEG:n02787622 +ILSVRC2012_val_00033414.JPEG:n01768244 +ILSVRC2012_val_00033415.JPEG:n01768244 +ILSVRC2012_val_00033416.JPEG:n03146219 +ILSVRC2012_val_00033417.JPEG:n06785654 +ILSVRC2012_val_00033418.JPEG:n02110341 +ILSVRC2012_val_00033419.JPEG:n03400231 +ILSVRC2012_val_00033420.JPEG:n02123045 +ILSVRC2012_val_00033421.JPEG:n02025239 +ILSVRC2012_val_00033422.JPEG:n03670208 +ILSVRC2012_val_00033423.JPEG:n01784675 +ILSVRC2012_val_00033424.JPEG:n03982430 +ILSVRC2012_val_00033425.JPEG:n04485082 +ILSVRC2012_val_00033426.JPEG:n03208938 +ILSVRC2012_val_00033427.JPEG:n01990800 +ILSVRC2012_val_00033428.JPEG:n03930313 +ILSVRC2012_val_00033429.JPEG:n02708093 +ILSVRC2012_val_00033430.JPEG:n04597913 +ILSVRC2012_val_00033431.JPEG:n01796340 +ILSVRC2012_val_00033432.JPEG:n02100236 +ILSVRC2012_val_00033433.JPEG:n01608432 +ILSVRC2012_val_00033434.JPEG:n01828970 +ILSVRC2012_val_00033435.JPEG:n01614925 +ILSVRC2012_val_00033436.JPEG:n03400231 +ILSVRC2012_val_00033437.JPEG:n01631663 +ILSVRC2012_val_00033438.JPEG:n03759954 +ILSVRC2012_val_00033439.JPEG:n01872401 +ILSVRC2012_val_00033440.JPEG:n01917289 +ILSVRC2012_val_00033441.JPEG:n02690373 +ILSVRC2012_val_00033442.JPEG:n01664065 +ILSVRC2012_val_00033443.JPEG:n03016953 +ILSVRC2012_val_00033444.JPEG:n04376876 +ILSVRC2012_val_00033445.JPEG:n01664065 +ILSVRC2012_val_00033446.JPEG:n02950826 +ILSVRC2012_val_00033447.JPEG:n04557648 +ILSVRC2012_val_00033448.JPEG:n02793495 +ILSVRC2012_val_00033449.JPEG:n02111129 +ILSVRC2012_val_00033450.JPEG:n01968897 +ILSVRC2012_val_00033451.JPEG:n03781244 +ILSVRC2012_val_00033452.JPEG:n07871810 +ILSVRC2012_val_00033453.JPEG:n02641379 +ILSVRC2012_val_00033454.JPEG:n02097209 +ILSVRC2012_val_00033455.JPEG:n02109047 +ILSVRC2012_val_00033456.JPEG:n03065424 +ILSVRC2012_val_00033457.JPEG:n03838899 +ILSVRC2012_val_00033458.JPEG:n04501370 +ILSVRC2012_val_00033459.JPEG:n01753488 +ILSVRC2012_val_00033460.JPEG:n04049303 +ILSVRC2012_val_00033461.JPEG:n02097047 +ILSVRC2012_val_00033462.JPEG:n04311004 +ILSVRC2012_val_00033463.JPEG:n03538406 +ILSVRC2012_val_00033464.JPEG:n03666591 +ILSVRC2012_val_00033465.JPEG:n02017213 +ILSVRC2012_val_00033466.JPEG:n02093647 +ILSVRC2012_val_00033467.JPEG:n04409515 +ILSVRC2012_val_00033468.JPEG:n03207743 +ILSVRC2012_val_00033469.JPEG:n01843065 +ILSVRC2012_val_00033470.JPEG:n03697007 +ILSVRC2012_val_00033471.JPEG:n03291819 +ILSVRC2012_val_00033472.JPEG:n03197337 +ILSVRC2012_val_00033473.JPEG:n03000247 +ILSVRC2012_val_00033474.JPEG:n02443484 +ILSVRC2012_val_00033475.JPEG:n03891251 +ILSVRC2012_val_00033476.JPEG:n02085782 +ILSVRC2012_val_00033477.JPEG:n04033901 +ILSVRC2012_val_00033478.JPEG:n03658185 +ILSVRC2012_val_00033479.JPEG:n01819313 +ILSVRC2012_val_00033480.JPEG:n03388549 +ILSVRC2012_val_00033481.JPEG:n02606052 +ILSVRC2012_val_00033482.JPEG:n04612504 +ILSVRC2012_val_00033483.JPEG:n01582220 +ILSVRC2012_val_00033484.JPEG:n02883205 +ILSVRC2012_val_00033485.JPEG:n04467665 +ILSVRC2012_val_00033486.JPEG:n03535780 +ILSVRC2012_val_00033487.JPEG:n04326547 +ILSVRC2012_val_00033488.JPEG:n03895866 +ILSVRC2012_val_00033489.JPEG:n02095889 +ILSVRC2012_val_00033490.JPEG:n02123045 +ILSVRC2012_val_00033491.JPEG:n03777568 +ILSVRC2012_val_00033492.JPEG:n01631663 +ILSVRC2012_val_00033493.JPEG:n02999410 +ILSVRC2012_val_00033494.JPEG:n07717410 +ILSVRC2012_val_00033495.JPEG:n02837789 +ILSVRC2012_val_00033496.JPEG:n04461696 +ILSVRC2012_val_00033497.JPEG:n07720875 +ILSVRC2012_val_00033498.JPEG:n03141823 +ILSVRC2012_val_00033499.JPEG:n03216828 +ILSVRC2012_val_00033500.JPEG:n04589890 +ILSVRC2012_val_00033501.JPEG:n02105641 +ILSVRC2012_val_00033502.JPEG:n03196217 +ILSVRC2012_val_00033503.JPEG:n01797886 +ILSVRC2012_val_00033504.JPEG:n07742313 +ILSVRC2012_val_00033505.JPEG:n02396427 +ILSVRC2012_val_00033506.JPEG:n04532106 +ILSVRC2012_val_00033507.JPEG:n02655020 +ILSVRC2012_val_00033508.JPEG:n02437312 +ILSVRC2012_val_00033509.JPEG:n03028079 +ILSVRC2012_val_00033510.JPEG:n02037110 +ILSVRC2012_val_00033511.JPEG:n03788365 +ILSVRC2012_val_00033512.JPEG:n01978455 +ILSVRC2012_val_00033513.JPEG:n02483362 +ILSVRC2012_val_00033514.JPEG:n02444819 +ILSVRC2012_val_00033515.JPEG:n01580077 +ILSVRC2012_val_00033516.JPEG:n04347754 +ILSVRC2012_val_00033517.JPEG:n01728572 +ILSVRC2012_val_00033518.JPEG:n03063689 +ILSVRC2012_val_00033519.JPEG:n02106662 +ILSVRC2012_val_00033520.JPEG:n02672831 +ILSVRC2012_val_00033521.JPEG:n03895866 +ILSVRC2012_val_00033522.JPEG:n04560804 +ILSVRC2012_val_00033523.JPEG:n04540053 +ILSVRC2012_val_00033524.JPEG:n02233338 +ILSVRC2012_val_00033525.JPEG:n03777754 +ILSVRC2012_val_00033526.JPEG:n02788148 +ILSVRC2012_val_00033527.JPEG:n09472597 +ILSVRC2012_val_00033528.JPEG:n02484975 +ILSVRC2012_val_00033529.JPEG:n04404412 +ILSVRC2012_val_00033530.JPEG:n02087046 +ILSVRC2012_val_00033531.JPEG:n02089078 +ILSVRC2012_val_00033532.JPEG:n03255030 +ILSVRC2012_val_00033533.JPEG:n03095699 +ILSVRC2012_val_00033534.JPEG:n07714990 +ILSVRC2012_val_00033535.JPEG:n02641379 +ILSVRC2012_val_00033536.JPEG:n03218198 +ILSVRC2012_val_00033537.JPEG:n02481823 +ILSVRC2012_val_00033538.JPEG:n01514859 +ILSVRC2012_val_00033539.JPEG:n03337140 +ILSVRC2012_val_00033540.JPEG:n04399382 +ILSVRC2012_val_00033541.JPEG:n02641379 +ILSVRC2012_val_00033542.JPEG:n02129604 +ILSVRC2012_val_00033543.JPEG:n03982430 +ILSVRC2012_val_00033544.JPEG:n04127249 +ILSVRC2012_val_00033545.JPEG:n04125021 +ILSVRC2012_val_00033546.JPEG:n01774384 +ILSVRC2012_val_00033547.JPEG:n01740131 +ILSVRC2012_val_00033548.JPEG:n02325366 +ILSVRC2012_val_00033549.JPEG:n04041544 +ILSVRC2012_val_00033550.JPEG:n02667093 +ILSVRC2012_val_00033551.JPEG:n07836838 +ILSVRC2012_val_00033552.JPEG:n01739381 +ILSVRC2012_val_00033553.JPEG:n02108000 +ILSVRC2012_val_00033554.JPEG:n02277742 +ILSVRC2012_val_00033555.JPEG:n01950731 +ILSVRC2012_val_00033556.JPEG:n03777754 +ILSVRC2012_val_00033557.JPEG:n04310018 +ILSVRC2012_val_00033558.JPEG:n02917067 +ILSVRC2012_val_00033559.JPEG:n02835271 +ILSVRC2012_val_00033560.JPEG:n04515003 +ILSVRC2012_val_00033561.JPEG:n02119789 +ILSVRC2012_val_00033562.JPEG:n02966687 +ILSVRC2012_val_00033563.JPEG:n03085013 +ILSVRC2012_val_00033564.JPEG:n12144580 +ILSVRC2012_val_00033565.JPEG:n02071294 +ILSVRC2012_val_00033566.JPEG:n12998815 +ILSVRC2012_val_00033567.JPEG:n04162706 +ILSVRC2012_val_00033568.JPEG:n03028079 +ILSVRC2012_val_00033569.JPEG:n03218198 +ILSVRC2012_val_00033570.JPEG:n02895154 +ILSVRC2012_val_00033571.JPEG:n04562935 +ILSVRC2012_val_00033572.JPEG:n07613480 +ILSVRC2012_val_00033573.JPEG:n02128925 +ILSVRC2012_val_00033574.JPEG:n03649909 +ILSVRC2012_val_00033575.JPEG:n01629819 +ILSVRC2012_val_00033576.JPEG:n01883070 +ILSVRC2012_val_00033577.JPEG:n02098413 +ILSVRC2012_val_00033578.JPEG:n02002724 +ILSVRC2012_val_00033579.JPEG:n02106382 +ILSVRC2012_val_00033580.JPEG:n01530575 +ILSVRC2012_val_00033581.JPEG:n02113978 +ILSVRC2012_val_00033582.JPEG:n02124075 +ILSVRC2012_val_00033583.JPEG:n04332243 +ILSVRC2012_val_00033584.JPEG:n02655020 +ILSVRC2012_val_00033585.JPEG:n04239074 +ILSVRC2012_val_00033586.JPEG:n01910747 +ILSVRC2012_val_00033587.JPEG:n09399592 +ILSVRC2012_val_00033588.JPEG:n02096051 +ILSVRC2012_val_00033589.JPEG:n03930630 +ILSVRC2012_val_00033590.JPEG:n07693725 +ILSVRC2012_val_00033591.JPEG:n03933933 +ILSVRC2012_val_00033592.JPEG:n03187595 +ILSVRC2012_val_00033593.JPEG:n02281787 +ILSVRC2012_val_00033594.JPEG:n02892201 +ILSVRC2012_val_00033595.JPEG:n02108000 +ILSVRC2012_val_00033596.JPEG:n01687978 +ILSVRC2012_val_00033597.JPEG:n03803284 +ILSVRC2012_val_00033598.JPEG:n07892512 +ILSVRC2012_val_00033599.JPEG:n02074367 +ILSVRC2012_val_00033600.JPEG:n03891251 +ILSVRC2012_val_00033601.JPEG:n03384352 +ILSVRC2012_val_00033602.JPEG:n04409515 +ILSVRC2012_val_00033603.JPEG:n02107574 +ILSVRC2012_val_00033604.JPEG:n01860187 +ILSVRC2012_val_00033605.JPEG:n03529860 +ILSVRC2012_val_00033606.JPEG:n02280649 +ILSVRC2012_val_00033607.JPEG:n02860847 +ILSVRC2012_val_00033608.JPEG:n03325584 +ILSVRC2012_val_00033609.JPEG:n04409515 +ILSVRC2012_val_00033610.JPEG:n03692522 +ILSVRC2012_val_00033611.JPEG:n02089973 +ILSVRC2012_val_00033612.JPEG:n02782093 +ILSVRC2012_val_00033613.JPEG:n03208938 +ILSVRC2012_val_00033614.JPEG:n02980441 +ILSVRC2012_val_00033615.JPEG:n01693334 +ILSVRC2012_val_00033616.JPEG:n01773157 +ILSVRC2012_val_00033617.JPEG:n01729977 +ILSVRC2012_val_00033618.JPEG:n03063689 +ILSVRC2012_val_00033619.JPEG:n02865351 +ILSVRC2012_val_00033620.JPEG:n03459775 +ILSVRC2012_val_00033621.JPEG:n03637318 +ILSVRC2012_val_00033622.JPEG:n04263257 +ILSVRC2012_val_00033623.JPEG:n04604644 +ILSVRC2012_val_00033624.JPEG:n04311004 +ILSVRC2012_val_00033625.JPEG:n02120079 +ILSVRC2012_val_00033626.JPEG:n02112018 +ILSVRC2012_val_00033627.JPEG:n03196217 +ILSVRC2012_val_00033628.JPEG:n01871265 +ILSVRC2012_val_00033629.JPEG:n02804610 +ILSVRC2012_val_00033630.JPEG:n07892512 +ILSVRC2012_val_00033631.JPEG:n03124043 +ILSVRC2012_val_00033632.JPEG:n02219486 +ILSVRC2012_val_00033633.JPEG:n02089973 +ILSVRC2012_val_00033634.JPEG:n02109047 +ILSVRC2012_val_00033635.JPEG:n04040759 +ILSVRC2012_val_00033636.JPEG:n07711569 +ILSVRC2012_val_00033637.JPEG:n04458633 +ILSVRC2012_val_00033638.JPEG:n07720875 +ILSVRC2012_val_00033639.JPEG:n02277742 +ILSVRC2012_val_00033640.JPEG:n01675722 +ILSVRC2012_val_00033641.JPEG:n02119022 +ILSVRC2012_val_00033642.JPEG:n02106030 +ILSVRC2012_val_00033643.JPEG:n03763968 +ILSVRC2012_val_00033644.JPEG:n02105412 +ILSVRC2012_val_00033645.JPEG:n03017168 +ILSVRC2012_val_00033646.JPEG:n03857828 +ILSVRC2012_val_00033647.JPEG:n04346328 +ILSVRC2012_val_00033648.JPEG:n04005630 +ILSVRC2012_val_00033649.JPEG:n03492542 +ILSVRC2012_val_00033650.JPEG:n02480495 +ILSVRC2012_val_00033651.JPEG:n02090622 +ILSVRC2012_val_00033652.JPEG:n03814906 +ILSVRC2012_val_00033653.JPEG:n04004767 +ILSVRC2012_val_00033654.JPEG:n02992529 +ILSVRC2012_val_00033655.JPEG:n02692877 +ILSVRC2012_val_00033656.JPEG:n09332890 +ILSVRC2012_val_00033657.JPEG:n02979186 +ILSVRC2012_val_00033658.JPEG:n01770393 +ILSVRC2012_val_00033659.JPEG:n02129165 +ILSVRC2012_val_00033660.JPEG:n02391049 +ILSVRC2012_val_00033661.JPEG:n07871810 +ILSVRC2012_val_00033662.JPEG:n03355925 +ILSVRC2012_val_00033663.JPEG:n04398044 +ILSVRC2012_val_00033664.JPEG:n07860988 +ILSVRC2012_val_00033665.JPEG:n03961711 +ILSVRC2012_val_00033666.JPEG:n02089973 +ILSVRC2012_val_00033667.JPEG:n03404251 +ILSVRC2012_val_00033668.JPEG:n02395406 +ILSVRC2012_val_00033669.JPEG:n03063689 +ILSVRC2012_val_00033670.JPEG:n04070727 +ILSVRC2012_val_00033671.JPEG:n04552348 +ILSVRC2012_val_00033672.JPEG:n02112137 +ILSVRC2012_val_00033673.JPEG:n02110958 +ILSVRC2012_val_00033674.JPEG:n01753488 +ILSVRC2012_val_00033675.JPEG:n07697537 +ILSVRC2012_val_00033676.JPEG:n04389033 +ILSVRC2012_val_00033677.JPEG:n02783161 +ILSVRC2012_val_00033678.JPEG:n07693725 +ILSVRC2012_val_00033679.JPEG:n04286575 +ILSVRC2012_val_00033680.JPEG:n07753113 +ILSVRC2012_val_00033681.JPEG:n07716358 +ILSVRC2012_val_00033682.JPEG:n03394916 +ILSVRC2012_val_00033683.JPEG:n02093256 +ILSVRC2012_val_00033684.JPEG:n01737021 +ILSVRC2012_val_00033685.JPEG:n07836838 +ILSVRC2012_val_00033686.JPEG:n02268853 +ILSVRC2012_val_00033687.JPEG:n02130308 +ILSVRC2012_val_00033688.JPEG:n02906734 +ILSVRC2012_val_00033689.JPEG:n02134418 +ILSVRC2012_val_00033690.JPEG:n02108000 +ILSVRC2012_val_00033691.JPEG:n01560419 +ILSVRC2012_val_00033692.JPEG:n03131574 +ILSVRC2012_val_00033693.JPEG:n02133161 +ILSVRC2012_val_00033694.JPEG:n03000247 +ILSVRC2012_val_00033695.JPEG:n02279972 +ILSVRC2012_val_00033696.JPEG:n02951585 +ILSVRC2012_val_00033697.JPEG:n03733805 +ILSVRC2012_val_00033698.JPEG:n01677366 +ILSVRC2012_val_00033699.JPEG:n03976467 +ILSVRC2012_val_00033700.JPEG:n03535780 +ILSVRC2012_val_00033701.JPEG:n03938244 +ILSVRC2012_val_00033702.JPEG:n01644373 +ILSVRC2012_val_00033703.JPEG:n02109525 +ILSVRC2012_val_00033704.JPEG:n03649909 +ILSVRC2012_val_00033705.JPEG:n02190166 +ILSVRC2012_val_00033706.JPEG:n01692333 +ILSVRC2012_val_00033707.JPEG:n02910353 +ILSVRC2012_val_00033708.JPEG:n01807496 +ILSVRC2012_val_00033709.JPEG:n03982430 +ILSVRC2012_val_00033710.JPEG:n02974003 +ILSVRC2012_val_00033711.JPEG:n03950228 +ILSVRC2012_val_00033712.JPEG:n01978287 +ILSVRC2012_val_00033713.JPEG:n03720891 +ILSVRC2012_val_00033714.JPEG:n02892767 +ILSVRC2012_val_00033715.JPEG:n02504013 +ILSVRC2012_val_00033716.JPEG:n01855032 +ILSVRC2012_val_00033717.JPEG:n02483362 +ILSVRC2012_val_00033718.JPEG:n02025239 +ILSVRC2012_val_00033719.JPEG:n03868242 +ILSVRC2012_val_00033720.JPEG:n02094114 +ILSVRC2012_val_00033721.JPEG:n02109047 +ILSVRC2012_val_00033722.JPEG:n07749582 +ILSVRC2012_val_00033723.JPEG:n01669191 +ILSVRC2012_val_00033724.JPEG:n03785016 +ILSVRC2012_val_00033725.JPEG:n04041544 +ILSVRC2012_val_00033726.JPEG:n02087046 +ILSVRC2012_val_00033727.JPEG:n03272010 +ILSVRC2012_val_00033728.JPEG:n03447447 +ILSVRC2012_val_00033729.JPEG:n02783161 +ILSVRC2012_val_00033730.JPEG:n03976657 +ILSVRC2012_val_00033731.JPEG:n02087394 +ILSVRC2012_val_00033732.JPEG:n04548280 +ILSVRC2012_val_00033733.JPEG:n01860187 +ILSVRC2012_val_00033734.JPEG:n01689811 +ILSVRC2012_val_00033735.JPEG:n04584207 +ILSVRC2012_val_00033736.JPEG:n04251144 +ILSVRC2012_val_00033737.JPEG:n02113023 +ILSVRC2012_val_00033738.JPEG:n03977966 +ILSVRC2012_val_00033739.JPEG:n03792972 +ILSVRC2012_val_00033740.JPEG:n13054560 +ILSVRC2012_val_00033741.JPEG:n06785654 +ILSVRC2012_val_00033742.JPEG:n07734744 +ILSVRC2012_val_00033743.JPEG:n02115641 +ILSVRC2012_val_00033744.JPEG:n04606251 +ILSVRC2012_val_00033745.JPEG:n02277742 +ILSVRC2012_val_00033746.JPEG:n02794156 +ILSVRC2012_val_00033747.JPEG:n02137549 +ILSVRC2012_val_00033748.JPEG:n04479046 +ILSVRC2012_val_00033749.JPEG:n01753488 +ILSVRC2012_val_00033750.JPEG:n04485082 +ILSVRC2012_val_00033751.JPEG:n02100735 +ILSVRC2012_val_00033752.JPEG:n02869837 +ILSVRC2012_val_00033753.JPEG:n03534580 +ILSVRC2012_val_00033754.JPEG:n02879718 +ILSVRC2012_val_00033755.JPEG:n04525305 +ILSVRC2012_val_00033756.JPEG:n01829413 +ILSVRC2012_val_00033757.JPEG:n03792782 +ILSVRC2012_val_00033758.JPEG:n02109961 +ILSVRC2012_val_00033759.JPEG:n03443371 +ILSVRC2012_val_00033760.JPEG:n02009229 +ILSVRC2012_val_00033761.JPEG:n01744401 +ILSVRC2012_val_00033762.JPEG:n01728572 +ILSVRC2012_val_00033763.JPEG:n02098413 +ILSVRC2012_val_00033764.JPEG:n04311004 +ILSVRC2012_val_00033765.JPEG:n03272010 +ILSVRC2012_val_00033766.JPEG:n02095570 +ILSVRC2012_val_00033767.JPEG:n01632458 +ILSVRC2012_val_00033768.JPEG:n02783161 +ILSVRC2012_val_00033769.JPEG:n01644900 +ILSVRC2012_val_00033770.JPEG:n01601694 +ILSVRC2012_val_00033771.JPEG:n01608432 +ILSVRC2012_val_00033772.JPEG:n04335435 +ILSVRC2012_val_00033773.JPEG:n02086910 +ILSVRC2012_val_00033774.JPEG:n04418357 +ILSVRC2012_val_00033775.JPEG:n02097658 +ILSVRC2012_val_00033776.JPEG:n03124170 +ILSVRC2012_val_00033777.JPEG:n04228054 +ILSVRC2012_val_00033778.JPEG:n02494079 +ILSVRC2012_val_00033779.JPEG:n07754684 +ILSVRC2012_val_00033780.JPEG:n02493793 +ILSVRC2012_val_00033781.JPEG:n02165105 +ILSVRC2012_val_00033782.JPEG:n02133161 +ILSVRC2012_val_00033783.JPEG:n01847000 +ILSVRC2012_val_00033784.JPEG:n03394916 +ILSVRC2012_val_00033785.JPEG:n02105162 +ILSVRC2012_val_00033786.JPEG:n01950731 +ILSVRC2012_val_00033787.JPEG:n03970156 +ILSVRC2012_val_00033788.JPEG:n02233338 +ILSVRC2012_val_00033789.JPEG:n03045698 +ILSVRC2012_val_00033790.JPEG:n02099601 +ILSVRC2012_val_00033791.JPEG:n11939491 +ILSVRC2012_val_00033792.JPEG:n04467665 +ILSVRC2012_val_00033793.JPEG:n04346328 +ILSVRC2012_val_00033794.JPEG:n04347754 +ILSVRC2012_val_00033795.JPEG:n03063689 +ILSVRC2012_val_00033796.JPEG:n03100240 +ILSVRC2012_val_00033797.JPEG:n02127052 +ILSVRC2012_val_00033798.JPEG:n03887697 +ILSVRC2012_val_00033799.JPEG:n09428293 +ILSVRC2012_val_00033800.JPEG:n02361337 +ILSVRC2012_val_00033801.JPEG:n02606052 +ILSVRC2012_val_00033802.JPEG:n04590129 +ILSVRC2012_val_00033803.JPEG:n02692877 +ILSVRC2012_val_00033804.JPEG:n03796401 +ILSVRC2012_val_00033805.JPEG:n04532106 +ILSVRC2012_val_00033806.JPEG:n03538406 +ILSVRC2012_val_00033807.JPEG:n07747607 +ILSVRC2012_val_00033808.JPEG:n01978455 +ILSVRC2012_val_00033809.JPEG:n07717556 +ILSVRC2012_val_00033810.JPEG:n02894605 +ILSVRC2012_val_00033811.JPEG:n03134739 +ILSVRC2012_val_00033812.JPEG:n04243546 +ILSVRC2012_val_00033813.JPEG:n03903868 +ILSVRC2012_val_00033814.JPEG:n02879718 +ILSVRC2012_val_00033815.JPEG:n01824575 +ILSVRC2012_val_00033816.JPEG:n01877812 +ILSVRC2012_val_00033817.JPEG:n01770081 +ILSVRC2012_val_00033818.JPEG:n04525305 +ILSVRC2012_val_00033819.JPEG:n01773549 +ILSVRC2012_val_00033820.JPEG:n02099712 +ILSVRC2012_val_00033821.JPEG:n01774384 +ILSVRC2012_val_00033822.JPEG:n02823428 +ILSVRC2012_val_00033823.JPEG:n01860187 +ILSVRC2012_val_00033824.JPEG:n03461385 +ILSVRC2012_val_00033825.JPEG:n04366367 +ILSVRC2012_val_00033826.JPEG:n02167151 +ILSVRC2012_val_00033827.JPEG:n02454379 +ILSVRC2012_val_00033828.JPEG:n03777568 +ILSVRC2012_val_00033829.JPEG:n01833805 +ILSVRC2012_val_00033830.JPEG:n03761084 +ILSVRC2012_val_00033831.JPEG:n04542943 +ILSVRC2012_val_00033832.JPEG:n02504458 +ILSVRC2012_val_00033833.JPEG:n02033041 +ILSVRC2012_val_00033834.JPEG:n02095314 +ILSVRC2012_val_00033835.JPEG:n03527444 +ILSVRC2012_val_00033836.JPEG:n02280649 +ILSVRC2012_val_00033837.JPEG:n02123045 +ILSVRC2012_val_00033838.JPEG:n01644373 +ILSVRC2012_val_00033839.JPEG:n12998815 +ILSVRC2012_val_00033840.JPEG:n03792972 +ILSVRC2012_val_00033841.JPEG:n02480495 +ILSVRC2012_val_00033842.JPEG:n03417042 +ILSVRC2012_val_00033843.JPEG:n02091467 +ILSVRC2012_val_00033844.JPEG:n02415577 +ILSVRC2012_val_00033845.JPEG:n12985857 +ILSVRC2012_val_00033846.JPEG:n03544143 +ILSVRC2012_val_00033847.JPEG:n04370456 +ILSVRC2012_val_00033848.JPEG:n02110806 +ILSVRC2012_val_00033849.JPEG:n03676483 +ILSVRC2012_val_00033850.JPEG:n03602883 +ILSVRC2012_val_00033851.JPEG:n03538406 +ILSVRC2012_val_00033852.JPEG:n04201297 +ILSVRC2012_val_00033853.JPEG:n03929855 +ILSVRC2012_val_00033854.JPEG:n02504013 +ILSVRC2012_val_00033855.JPEG:n10565667 +ILSVRC2012_val_00033856.JPEG:n02097130 +ILSVRC2012_val_00033857.JPEG:n03950228 +ILSVRC2012_val_00033858.JPEG:n01675722 +ILSVRC2012_val_00033859.JPEG:n04523525 +ILSVRC2012_val_00033860.JPEG:n02966687 +ILSVRC2012_val_00033861.JPEG:n02504458 +ILSVRC2012_val_00033862.JPEG:n02089973 +ILSVRC2012_val_00033863.JPEG:n01641577 +ILSVRC2012_val_00033864.JPEG:n04330267 +ILSVRC2012_val_00033865.JPEG:n04146614 +ILSVRC2012_val_00033866.JPEG:n01631663 +ILSVRC2012_val_00033867.JPEG:n02978881 +ILSVRC2012_val_00033868.JPEG:n07802026 +ILSVRC2012_val_00033869.JPEG:n04039381 +ILSVRC2012_val_00033870.JPEG:n03485794 +ILSVRC2012_val_00033871.JPEG:n03825788 +ILSVRC2012_val_00033872.JPEG:n04265275 +ILSVRC2012_val_00033873.JPEG:n03141823 +ILSVRC2012_val_00033874.JPEG:n04033995 +ILSVRC2012_val_00033875.JPEG:n03179701 +ILSVRC2012_val_00033876.JPEG:n01986214 +ILSVRC2012_val_00033877.JPEG:n04604644 +ILSVRC2012_val_00033878.JPEG:n02730930 +ILSVRC2012_val_00033879.JPEG:n03920288 +ILSVRC2012_val_00033880.JPEG:n02799071 +ILSVRC2012_val_00033881.JPEG:n04399382 +ILSVRC2012_val_00033882.JPEG:n04023962 +ILSVRC2012_val_00033883.JPEG:n02951358 +ILSVRC2012_val_00033884.JPEG:n02114367 +ILSVRC2012_val_00033885.JPEG:n02074367 +ILSVRC2012_val_00033886.JPEG:n03992509 +ILSVRC2012_val_00033887.JPEG:n03000134 +ILSVRC2012_val_00033888.JPEG:n01824575 +ILSVRC2012_val_00033889.JPEG:n04525305 +ILSVRC2012_val_00033890.JPEG:n02119789 +ILSVRC2012_val_00033891.JPEG:n03899768 +ILSVRC2012_val_00033892.JPEG:n03617480 +ILSVRC2012_val_00033893.JPEG:n02012849 +ILSVRC2012_val_00033894.JPEG:n03814639 +ILSVRC2012_val_00033895.JPEG:n04347754 +ILSVRC2012_val_00033896.JPEG:n04597913 +ILSVRC2012_val_00033897.JPEG:n02113799 +ILSVRC2012_val_00033898.JPEG:n04562935 +ILSVRC2012_val_00033899.JPEG:n03777754 +ILSVRC2012_val_00033900.JPEG:n02687172 +ILSVRC2012_val_00033901.JPEG:n02066245 +ILSVRC2012_val_00033902.JPEG:n02704792 +ILSVRC2012_val_00033903.JPEG:n01751748 +ILSVRC2012_val_00033904.JPEG:n02090622 +ILSVRC2012_val_00033905.JPEG:n03857828 +ILSVRC2012_val_00033906.JPEG:n03777754 +ILSVRC2012_val_00033907.JPEG:n02130308 +ILSVRC2012_val_00033908.JPEG:n02606052 +ILSVRC2012_val_00033909.JPEG:n03483316 +ILSVRC2012_val_00033910.JPEG:n02808440 +ILSVRC2012_val_00033911.JPEG:n02114712 +ILSVRC2012_val_00033912.JPEG:n01774384 +ILSVRC2012_val_00033913.JPEG:n09468604 +ILSVRC2012_val_00033914.JPEG:n03045698 +ILSVRC2012_val_00033915.JPEG:n02107574 +ILSVRC2012_val_00033916.JPEG:n02112706 +ILSVRC2012_val_00033917.JPEG:n03777754 +ILSVRC2012_val_00033918.JPEG:n04209239 +ILSVRC2012_val_00033919.JPEG:n07745940 +ILSVRC2012_val_00033920.JPEG:n02690373 +ILSVRC2012_val_00033921.JPEG:n07584110 +ILSVRC2012_val_00033922.JPEG:n03388549 +ILSVRC2012_val_00033923.JPEG:n03977966 +ILSVRC2012_val_00033924.JPEG:n04584207 +ILSVRC2012_val_00033925.JPEG:n02279972 +ILSVRC2012_val_00033926.JPEG:n02443114 +ILSVRC2012_val_00033927.JPEG:n02493509 +ILSVRC2012_val_00033928.JPEG:n02494079 +ILSVRC2012_val_00033929.JPEG:n03063599 +ILSVRC2012_val_00033930.JPEG:n01774750 +ILSVRC2012_val_00033931.JPEG:n01968897 +ILSVRC2012_val_00033932.JPEG:n01695060 +ILSVRC2012_val_00033933.JPEG:n04380533 +ILSVRC2012_val_00033934.JPEG:n02128757 +ILSVRC2012_val_00033935.JPEG:n09256479 +ILSVRC2012_val_00033936.JPEG:n02909870 +ILSVRC2012_val_00033937.JPEG:n04501370 +ILSVRC2012_val_00033938.JPEG:n03935335 +ILSVRC2012_val_00033939.JPEG:n07693725 +ILSVRC2012_val_00033940.JPEG:n04591713 +ILSVRC2012_val_00033941.JPEG:n03787032 +ILSVRC2012_val_00033942.JPEG:n01498041 +ILSVRC2012_val_00033943.JPEG:n03042490 +ILSVRC2012_val_00033944.JPEG:n02086910 +ILSVRC2012_val_00033945.JPEG:n01855672 +ILSVRC2012_val_00033946.JPEG:n04596742 +ILSVRC2012_val_00033947.JPEG:n02445715 +ILSVRC2012_val_00033948.JPEG:n02859443 +ILSVRC2012_val_00033949.JPEG:n02804610 +ILSVRC2012_val_00033950.JPEG:n03709823 +ILSVRC2012_val_00033951.JPEG:n02488291 +ILSVRC2012_val_00033952.JPEG:n02410509 +ILSVRC2012_val_00033953.JPEG:n03393912 +ILSVRC2012_val_00033954.JPEG:n03498962 +ILSVRC2012_val_00033955.JPEG:n03131574 +ILSVRC2012_val_00033956.JPEG:n03791053 +ILSVRC2012_val_00033957.JPEG:n03763968 +ILSVRC2012_val_00033958.JPEG:n02097130 +ILSVRC2012_val_00033959.JPEG:n03042490 +ILSVRC2012_val_00033960.JPEG:n01641577 +ILSVRC2012_val_00033961.JPEG:n01677366 +ILSVRC2012_val_00033962.JPEG:n01828970 +ILSVRC2012_val_00033963.JPEG:n02096051 +ILSVRC2012_val_00033964.JPEG:n03888605 +ILSVRC2012_val_00033965.JPEG:n02094114 +ILSVRC2012_val_00033966.JPEG:n02892201 +ILSVRC2012_val_00033967.JPEG:n02486261 +ILSVRC2012_val_00033968.JPEG:n03983396 +ILSVRC2012_val_00033969.JPEG:n02133161 +ILSVRC2012_val_00033970.JPEG:n03602883 +ILSVRC2012_val_00033971.JPEG:n03065424 +ILSVRC2012_val_00033972.JPEG:n02749479 +ILSVRC2012_val_00033973.JPEG:n02791124 +ILSVRC2012_val_00033974.JPEG:n01968897 +ILSVRC2012_val_00033975.JPEG:n02797295 +ILSVRC2012_val_00033976.JPEG:n02877765 +ILSVRC2012_val_00033977.JPEG:n01843065 +ILSVRC2012_val_00033978.JPEG:n02892201 +ILSVRC2012_val_00033979.JPEG:n03786901 +ILSVRC2012_val_00033980.JPEG:n02174001 +ILSVRC2012_val_00033981.JPEG:n03133878 +ILSVRC2012_val_00033982.JPEG:n02107908 +ILSVRC2012_val_00033983.JPEG:n04136333 +ILSVRC2012_val_00033984.JPEG:n02437616 +ILSVRC2012_val_00033985.JPEG:n04592741 +ILSVRC2012_val_00033986.JPEG:n04044716 +ILSVRC2012_val_00033987.JPEG:n01773157 +ILSVRC2012_val_00033988.JPEG:n02130308 +ILSVRC2012_val_00033989.JPEG:n02325366 +ILSVRC2012_val_00033990.JPEG:n04591713 +ILSVRC2012_val_00033991.JPEG:n04090263 +ILSVRC2012_val_00033992.JPEG:n03902125 +ILSVRC2012_val_00033993.JPEG:n03670208 +ILSVRC2012_val_00033994.JPEG:n07753113 +ILSVRC2012_val_00033995.JPEG:n03866082 +ILSVRC2012_val_00033996.JPEG:n04201297 +ILSVRC2012_val_00033997.JPEG:n02093859 +ILSVRC2012_val_00033998.JPEG:n02410509 +ILSVRC2012_val_00033999.JPEG:n02823750 +ILSVRC2012_val_00034000.JPEG:n01740131 +ILSVRC2012_val_00034001.JPEG:n03417042 +ILSVRC2012_val_00034002.JPEG:n03874293 +ILSVRC2012_val_00034003.JPEG:n03710193 +ILSVRC2012_val_00034004.JPEG:n02871525 +ILSVRC2012_val_00034005.JPEG:n02091467 +ILSVRC2012_val_00034006.JPEG:n04254120 +ILSVRC2012_val_00034007.JPEG:n02109525 +ILSVRC2012_val_00034008.JPEG:n04404412 +ILSVRC2012_val_00034009.JPEG:n02094433 +ILSVRC2012_val_00034010.JPEG:n11939491 +ILSVRC2012_val_00034011.JPEG:n02107683 +ILSVRC2012_val_00034012.JPEG:n04356056 +ILSVRC2012_val_00034013.JPEG:n02002556 +ILSVRC2012_val_00034014.JPEG:n02168699 +ILSVRC2012_val_00034015.JPEG:n01945685 +ILSVRC2012_val_00034016.JPEG:n04376876 +ILSVRC2012_val_00034017.JPEG:n04033901 +ILSVRC2012_val_00034018.JPEG:n01530575 +ILSVRC2012_val_00034019.JPEG:n03838899 +ILSVRC2012_val_00034020.JPEG:n01776313 +ILSVRC2012_val_00034021.JPEG:n03028079 +ILSVRC2012_val_00034022.JPEG:n03658185 +ILSVRC2012_val_00034023.JPEG:n04310018 +ILSVRC2012_val_00034024.JPEG:n02090379 +ILSVRC2012_val_00034025.JPEG:n02109525 +ILSVRC2012_val_00034026.JPEG:n04376876 +ILSVRC2012_val_00034027.JPEG:n04418357 +ILSVRC2012_val_00034028.JPEG:n04409515 +ILSVRC2012_val_00034029.JPEG:n07583066 +ILSVRC2012_val_00034030.JPEG:n03841143 +ILSVRC2012_val_00034031.JPEG:n02837789 +ILSVRC2012_val_00034032.JPEG:n03494278 +ILSVRC2012_val_00034033.JPEG:n03457902 +ILSVRC2012_val_00034034.JPEG:n02497673 +ILSVRC2012_val_00034035.JPEG:n02504013 +ILSVRC2012_val_00034036.JPEG:n02110063 +ILSVRC2012_val_00034037.JPEG:n02835271 +ILSVRC2012_val_00034038.JPEG:n01491361 +ILSVRC2012_val_00034039.JPEG:n02807133 +ILSVRC2012_val_00034040.JPEG:n02085782 +ILSVRC2012_val_00034041.JPEG:n02088364 +ILSVRC2012_val_00034042.JPEG:n02607072 +ILSVRC2012_val_00034043.JPEG:n02120505 +ILSVRC2012_val_00034044.JPEG:n07718472 +ILSVRC2012_val_00034045.JPEG:n03781244 +ILSVRC2012_val_00034046.JPEG:n02389026 +ILSVRC2012_val_00034047.JPEG:n03026506 +ILSVRC2012_val_00034048.JPEG:n02769748 +ILSVRC2012_val_00034049.JPEG:n02096177 +ILSVRC2012_val_00034050.JPEG:n02840245 +ILSVRC2012_val_00034051.JPEG:n02606052 +ILSVRC2012_val_00034052.JPEG:n03857828 +ILSVRC2012_val_00034053.JPEG:n03837869 +ILSVRC2012_val_00034054.JPEG:n01735189 +ILSVRC2012_val_00034055.JPEG:n02093256 +ILSVRC2012_val_00034056.JPEG:n02112706 +ILSVRC2012_val_00034057.JPEG:n02749479 +ILSVRC2012_val_00034058.JPEG:n04525038 +ILSVRC2012_val_00034059.JPEG:n03982430 +ILSVRC2012_val_00034060.JPEG:n02510455 +ILSVRC2012_val_00034061.JPEG:n02410509 +ILSVRC2012_val_00034062.JPEG:n03680355 +ILSVRC2012_val_00034063.JPEG:n02105505 +ILSVRC2012_val_00034064.JPEG:n03017168 +ILSVRC2012_val_00034065.JPEG:n02120079 +ILSVRC2012_val_00034066.JPEG:n03532672 +ILSVRC2012_val_00034067.JPEG:n03992509 +ILSVRC2012_val_00034068.JPEG:n02009229 +ILSVRC2012_val_00034069.JPEG:n02106166 +ILSVRC2012_val_00034070.JPEG:n02105056 +ILSVRC2012_val_00034071.JPEG:n02422699 +ILSVRC2012_val_00034072.JPEG:n03770439 +ILSVRC2012_val_00034073.JPEG:n03794056 +ILSVRC2012_val_00034074.JPEG:n03777568 +ILSVRC2012_val_00034075.JPEG:n02110806 +ILSVRC2012_val_00034076.JPEG:n01950731 +ILSVRC2012_val_00034077.JPEG:n04371430 +ILSVRC2012_val_00034078.JPEG:n03417042 +ILSVRC2012_val_00034079.JPEG:n03743016 +ILSVRC2012_val_00034080.JPEG:n01729977 +ILSVRC2012_val_00034081.JPEG:n02669723 +ILSVRC2012_val_00034082.JPEG:n02094433 +ILSVRC2012_val_00034083.JPEG:n04251144 +ILSVRC2012_val_00034084.JPEG:n02119022 +ILSVRC2012_val_00034085.JPEG:n01697457 +ILSVRC2012_val_00034086.JPEG:n01682714 +ILSVRC2012_val_00034087.JPEG:n07614500 +ILSVRC2012_val_00034088.JPEG:n02127052 +ILSVRC2012_val_00034089.JPEG:n03042490 +ILSVRC2012_val_00034090.JPEG:n02113799 +ILSVRC2012_val_00034091.JPEG:n04399382 +ILSVRC2012_val_00034092.JPEG:n03794056 +ILSVRC2012_val_00034093.JPEG:n02963159 +ILSVRC2012_val_00034094.JPEG:n02730930 +ILSVRC2012_val_00034095.JPEG:n01592084 +ILSVRC2012_val_00034096.JPEG:n04067472 +ILSVRC2012_val_00034097.JPEG:n02815834 +ILSVRC2012_val_00034098.JPEG:n07753592 +ILSVRC2012_val_00034099.JPEG:n13052670 +ILSVRC2012_val_00034100.JPEG:n07875152 +ILSVRC2012_val_00034101.JPEG:n06785654 +ILSVRC2012_val_00034102.JPEG:n04509417 +ILSVRC2012_val_00034103.JPEG:n03977966 +ILSVRC2012_val_00034104.JPEG:n03345487 +ILSVRC2012_val_00034105.JPEG:n03223299 +ILSVRC2012_val_00034106.JPEG:n04277352 +ILSVRC2012_val_00034107.JPEG:n06794110 +ILSVRC2012_val_00034108.JPEG:n02389026 +ILSVRC2012_val_00034109.JPEG:n07920052 +ILSVRC2012_val_00034110.JPEG:n02100877 +ILSVRC2012_val_00034111.JPEG:n04435653 +ILSVRC2012_val_00034112.JPEG:n04239074 +ILSVRC2012_val_00034113.JPEG:n04069434 +ILSVRC2012_val_00034114.JPEG:n03617480 +ILSVRC2012_val_00034115.JPEG:n01494475 +ILSVRC2012_val_00034116.JPEG:n02672831 +ILSVRC2012_val_00034117.JPEG:n07831146 +ILSVRC2012_val_00034118.JPEG:n02097047 +ILSVRC2012_val_00034119.JPEG:n03814639 +ILSVRC2012_val_00034120.JPEG:n02514041 +ILSVRC2012_val_00034121.JPEG:n02091635 +ILSVRC2012_val_00034122.JPEG:n01687978 +ILSVRC2012_val_00034123.JPEG:n02116738 +ILSVRC2012_val_00034124.JPEG:n01630670 +ILSVRC2012_val_00034125.JPEG:n01695060 +ILSVRC2012_val_00034126.JPEG:n04204238 +ILSVRC2012_val_00034127.JPEG:n04090263 +ILSVRC2012_val_00034128.JPEG:n04081281 +ILSVRC2012_val_00034129.JPEG:n01819313 +ILSVRC2012_val_00034130.JPEG:n02132136 +ILSVRC2012_val_00034131.JPEG:n03787032 +ILSVRC2012_val_00034132.JPEG:n04044716 +ILSVRC2012_val_00034133.JPEG:n15075141 +ILSVRC2012_val_00034134.JPEG:n03954731 +ILSVRC2012_val_00034135.JPEG:n04389033 +ILSVRC2012_val_00034136.JPEG:n02002556 +ILSVRC2012_val_00034137.JPEG:n04591157 +ILSVRC2012_val_00034138.JPEG:n04133789 +ILSVRC2012_val_00034139.JPEG:n04277352 +ILSVRC2012_val_00034140.JPEG:n02641379 +ILSVRC2012_val_00034141.JPEG:n03733805 +ILSVRC2012_val_00034142.JPEG:n04417672 +ILSVRC2012_val_00034143.JPEG:n02403003 +ILSVRC2012_val_00034144.JPEG:n01580077 +ILSVRC2012_val_00034145.JPEG:n03920288 +ILSVRC2012_val_00034146.JPEG:n03673027 +ILSVRC2012_val_00034147.JPEG:n07697537 +ILSVRC2012_val_00034148.JPEG:n07836838 +ILSVRC2012_val_00034149.JPEG:n04243546 +ILSVRC2012_val_00034150.JPEG:n02977058 +ILSVRC2012_val_00034151.JPEG:n07684084 +ILSVRC2012_val_00034152.JPEG:n07697537 +ILSVRC2012_val_00034153.JPEG:n02132136 +ILSVRC2012_val_00034154.JPEG:n03131574 +ILSVRC2012_val_00034155.JPEG:n02093647 +ILSVRC2012_val_00034156.JPEG:n03443371 +ILSVRC2012_val_00034157.JPEG:n03134739 +ILSVRC2012_val_00034158.JPEG:n04550184 +ILSVRC2012_val_00034159.JPEG:n03891251 +ILSVRC2012_val_00034160.JPEG:n02087394 +ILSVRC2012_val_00034161.JPEG:n07697537 +ILSVRC2012_val_00034162.JPEG:n07583066 +ILSVRC2012_val_00034163.JPEG:n04522168 +ILSVRC2012_val_00034164.JPEG:n04493381 +ILSVRC2012_val_00034165.JPEG:n04065272 +ILSVRC2012_val_00034166.JPEG:n02097130 +ILSVRC2012_val_00034167.JPEG:n04467665 +ILSVRC2012_val_00034168.JPEG:n01614925 +ILSVRC2012_val_00034169.JPEG:n03961711 +ILSVRC2012_val_00034170.JPEG:n02802426 +ILSVRC2012_val_00034171.JPEG:n02089078 +ILSVRC2012_val_00034172.JPEG:n02018207 +ILSVRC2012_val_00034173.JPEG:n03947888 +ILSVRC2012_val_00034174.JPEG:n01748264 +ILSVRC2012_val_00034175.JPEG:n02280649 +ILSVRC2012_val_00034176.JPEG:n02002556 +ILSVRC2012_val_00034177.JPEG:n03709823 +ILSVRC2012_val_00034178.JPEG:n01494475 +ILSVRC2012_val_00034179.JPEG:n03485794 +ILSVRC2012_val_00034180.JPEG:n04479046 +ILSVRC2012_val_00034181.JPEG:n02108551 +ILSVRC2012_val_00034182.JPEG:n03325584 +ILSVRC2012_val_00034183.JPEG:n03188531 +ILSVRC2012_val_00034184.JPEG:n02091032 +ILSVRC2012_val_00034185.JPEG:n02259212 +ILSVRC2012_val_00034186.JPEG:n02033041 +ILSVRC2012_val_00034187.JPEG:n03290653 +ILSVRC2012_val_00034188.JPEG:n04033995 +ILSVRC2012_val_00034189.JPEG:n07614500 +ILSVRC2012_val_00034190.JPEG:n02169497 +ILSVRC2012_val_00034191.JPEG:n04553703 +ILSVRC2012_val_00034192.JPEG:n02268443 +ILSVRC2012_val_00034193.JPEG:n09288635 +ILSVRC2012_val_00034194.JPEG:n01843383 +ILSVRC2012_val_00034195.JPEG:n04428191 +ILSVRC2012_val_00034196.JPEG:n03717622 +ILSVRC2012_val_00034197.JPEG:n02268853 +ILSVRC2012_val_00034198.JPEG:n02012849 +ILSVRC2012_val_00034199.JPEG:n02894605 +ILSVRC2012_val_00034200.JPEG:n02134418 +ILSVRC2012_val_00034201.JPEG:n01751748 +ILSVRC2012_val_00034202.JPEG:n02823750 +ILSVRC2012_val_00034203.JPEG:n02177972 +ILSVRC2012_val_00034204.JPEG:n03424325 +ILSVRC2012_val_00034205.JPEG:n02397096 +ILSVRC2012_val_00034206.JPEG:n07753275 +ILSVRC2012_val_00034207.JPEG:n02417914 +ILSVRC2012_val_00034208.JPEG:n03379051 +ILSVRC2012_val_00034209.JPEG:n02096585 +ILSVRC2012_val_00034210.JPEG:n03814639 +ILSVRC2012_val_00034211.JPEG:n03355925 +ILSVRC2012_val_00034212.JPEG:n03127747 +ILSVRC2012_val_00034213.JPEG:n02264363 +ILSVRC2012_val_00034214.JPEG:n03733131 +ILSVRC2012_val_00034215.JPEG:n02481823 +ILSVRC2012_val_00034216.JPEG:n03447447 +ILSVRC2012_val_00034217.JPEG:n04409515 +ILSVRC2012_val_00034218.JPEG:n02066245 +ILSVRC2012_val_00034219.JPEG:n02102318 +ILSVRC2012_val_00034220.JPEG:n03028079 +ILSVRC2012_val_00034221.JPEG:n02107574 +ILSVRC2012_val_00034222.JPEG:n04026417 +ILSVRC2012_val_00034223.JPEG:n02058221 +ILSVRC2012_val_00034224.JPEG:n02106662 +ILSVRC2012_val_00034225.JPEG:n02607072 +ILSVRC2012_val_00034226.JPEG:n01641577 +ILSVRC2012_val_00034227.JPEG:n03376595 +ILSVRC2012_val_00034228.JPEG:n07892512 +ILSVRC2012_val_00034229.JPEG:n11939491 +ILSVRC2012_val_00034230.JPEG:n02488702 +ILSVRC2012_val_00034231.JPEG:n09421951 +ILSVRC2012_val_00034232.JPEG:n01910747 +ILSVRC2012_val_00034233.JPEG:n02364673 +ILSVRC2012_val_00034234.JPEG:n07248320 +ILSVRC2012_val_00034235.JPEG:n03908714 +ILSVRC2012_val_00034236.JPEG:n02939185 +ILSVRC2012_val_00034237.JPEG:n02099601 +ILSVRC2012_val_00034238.JPEG:n03680355 +ILSVRC2012_val_00034239.JPEG:n02095889 +ILSVRC2012_val_00034240.JPEG:n02917067 +ILSVRC2012_val_00034241.JPEG:n04380533 +ILSVRC2012_val_00034242.JPEG:n01592084 +ILSVRC2012_val_00034243.JPEG:n02109525 +ILSVRC2012_val_00034244.JPEG:n02123394 +ILSVRC2012_val_00034245.JPEG:n02236044 +ILSVRC2012_val_00034246.JPEG:n02346627 +ILSVRC2012_val_00034247.JPEG:n12057211 +ILSVRC2012_val_00034248.JPEG:n12620546 +ILSVRC2012_val_00034249.JPEG:n04346328 +ILSVRC2012_val_00034250.JPEG:n01531178 +ILSVRC2012_val_00034251.JPEG:n01735189 +ILSVRC2012_val_00034252.JPEG:n04152593 +ILSVRC2012_val_00034253.JPEG:n04487394 +ILSVRC2012_val_00034254.JPEG:n02123597 +ILSVRC2012_val_00034255.JPEG:n01768244 +ILSVRC2012_val_00034256.JPEG:n02129604 +ILSVRC2012_val_00034257.JPEG:n09193705 +ILSVRC2012_val_00034258.JPEG:n04131690 +ILSVRC2012_val_00034259.JPEG:n02085936 +ILSVRC2012_val_00034260.JPEG:n02088238 +ILSVRC2012_val_00034261.JPEG:n03538406 +ILSVRC2012_val_00034262.JPEG:n03131574 +ILSVRC2012_val_00034263.JPEG:n02110185 +ILSVRC2012_val_00034264.JPEG:n03124043 +ILSVRC2012_val_00034265.JPEG:n03000247 +ILSVRC2012_val_00034266.JPEG:n02107574 +ILSVRC2012_val_00034267.JPEG:n02110958 +ILSVRC2012_val_00034268.JPEG:n03018349 +ILSVRC2012_val_00034269.JPEG:n02930766 +ILSVRC2012_val_00034270.JPEG:n02229544 +ILSVRC2012_val_00034271.JPEG:n02483362 +ILSVRC2012_val_00034272.JPEG:n03887697 +ILSVRC2012_val_00034273.JPEG:n01773797 +ILSVRC2012_val_00034274.JPEG:n02264363 +ILSVRC2012_val_00034275.JPEG:n02088364 +ILSVRC2012_val_00034276.JPEG:n04127249 +ILSVRC2012_val_00034277.JPEG:n02113023 +ILSVRC2012_val_00034278.JPEG:n03146219 +ILSVRC2012_val_00034279.JPEG:n02114855 +ILSVRC2012_val_00034280.JPEG:n04536866 +ILSVRC2012_val_00034281.JPEG:n03770679 +ILSVRC2012_val_00034282.JPEG:n01796340 +ILSVRC2012_val_00034283.JPEG:n03866082 +ILSVRC2012_val_00034284.JPEG:n04380533 +ILSVRC2012_val_00034285.JPEG:n03764736 +ILSVRC2012_val_00034286.JPEG:n07749582 +ILSVRC2012_val_00034287.JPEG:n03658185 +ILSVRC2012_val_00034288.JPEG:n04579145 +ILSVRC2012_val_00034289.JPEG:n01784675 +ILSVRC2012_val_00034290.JPEG:n01644373 +ILSVRC2012_val_00034291.JPEG:n02110063 +ILSVRC2012_val_00034292.JPEG:n02971356 +ILSVRC2012_val_00034293.JPEG:n02494079 +ILSVRC2012_val_00034294.JPEG:n02361337 +ILSVRC2012_val_00034295.JPEG:n02490219 +ILSVRC2012_val_00034296.JPEG:n03803284 +ILSVRC2012_val_00034297.JPEG:n02113624 +ILSVRC2012_val_00034298.JPEG:n02106550 +ILSVRC2012_val_00034299.JPEG:n03814906 +ILSVRC2012_val_00034300.JPEG:n03180011 +ILSVRC2012_val_00034301.JPEG:n01872401 +ILSVRC2012_val_00034302.JPEG:n02730930 +ILSVRC2012_val_00034303.JPEG:n04548280 +ILSVRC2012_val_00034304.JPEG:n02814860 +ILSVRC2012_val_00034305.JPEG:n02105162 +ILSVRC2012_val_00034306.JPEG:n03676483 +ILSVRC2012_val_00034307.JPEG:n01871265 +ILSVRC2012_val_00034308.JPEG:n07716358 +ILSVRC2012_val_00034309.JPEG:n04476259 +ILSVRC2012_val_00034310.JPEG:n03887697 +ILSVRC2012_val_00034311.JPEG:n07697537 +ILSVRC2012_val_00034312.JPEG:n02514041 +ILSVRC2012_val_00034313.JPEG:n04004767 +ILSVRC2012_val_00034314.JPEG:n04371774 +ILSVRC2012_val_00034315.JPEG:n01855032 +ILSVRC2012_val_00034316.JPEG:n01518878 +ILSVRC2012_val_00034317.JPEG:n09835506 +ILSVRC2012_val_00034318.JPEG:n01943899 +ILSVRC2012_val_00034319.JPEG:n03908714 +ILSVRC2012_val_00034320.JPEG:n03400231 +ILSVRC2012_val_00034321.JPEG:n02129604 +ILSVRC2012_val_00034322.JPEG:n02492035 +ILSVRC2012_val_00034323.JPEG:n04252225 +ILSVRC2012_val_00034324.JPEG:n02107312 +ILSVRC2012_val_00034325.JPEG:n03443371 +ILSVRC2012_val_00034326.JPEG:n02950826 +ILSVRC2012_val_00034327.JPEG:n03814639 +ILSVRC2012_val_00034328.JPEG:n02951585 +ILSVRC2012_val_00034329.JPEG:n04265275 +ILSVRC2012_val_00034330.JPEG:n01806567 +ILSVRC2012_val_00034331.JPEG:n03482405 +ILSVRC2012_val_00034332.JPEG:n01882714 +ILSVRC2012_val_00034333.JPEG:n01580077 +ILSVRC2012_val_00034334.JPEG:n02091831 +ILSVRC2012_val_00034335.JPEG:n04266014 +ILSVRC2012_val_00034336.JPEG:n02895154 +ILSVRC2012_val_00034337.JPEG:n04532106 +ILSVRC2012_val_00034338.JPEG:n02999410 +ILSVRC2012_val_00034339.JPEG:n03729826 +ILSVRC2012_val_00034340.JPEG:n03345487 +ILSVRC2012_val_00034341.JPEG:n02105162 +ILSVRC2012_val_00034342.JPEG:n02690373 +ILSVRC2012_val_00034343.JPEG:n04597913 +ILSVRC2012_val_00034344.JPEG:n04325704 +ILSVRC2012_val_00034345.JPEG:n03461385 +ILSVRC2012_val_00034346.JPEG:n01695060 +ILSVRC2012_val_00034347.JPEG:n01818515 +ILSVRC2012_val_00034348.JPEG:n09472597 +ILSVRC2012_val_00034349.JPEG:n01806567 +ILSVRC2012_val_00034350.JPEG:n07754684 +ILSVRC2012_val_00034351.JPEG:n04326547 +ILSVRC2012_val_00034352.JPEG:n02093859 +ILSVRC2012_val_00034353.JPEG:n04049303 +ILSVRC2012_val_00034354.JPEG:n02641379 +ILSVRC2012_val_00034355.JPEG:n03196217 +ILSVRC2012_val_00034356.JPEG:n02088466 +ILSVRC2012_val_00034357.JPEG:n04376876 +ILSVRC2012_val_00034358.JPEG:n02009229 +ILSVRC2012_val_00034359.JPEG:n03929855 +ILSVRC2012_val_00034360.JPEG:n02025239 +ILSVRC2012_val_00034361.JPEG:n03814906 +ILSVRC2012_val_00034362.JPEG:n03291819 +ILSVRC2012_val_00034363.JPEG:n04612504 +ILSVRC2012_val_00034364.JPEG:n03000134 +ILSVRC2012_val_00034365.JPEG:n02837789 +ILSVRC2012_val_00034366.JPEG:n07718747 +ILSVRC2012_val_00034367.JPEG:n03459775 +ILSVRC2012_val_00034368.JPEG:n02281406 +ILSVRC2012_val_00034369.JPEG:n01693334 +ILSVRC2012_val_00034370.JPEG:n02219486 +ILSVRC2012_val_00034371.JPEG:n04266014 +ILSVRC2012_val_00034372.JPEG:n04399382 +ILSVRC2012_val_00034373.JPEG:n01774750 +ILSVRC2012_val_00034374.JPEG:n02980441 +ILSVRC2012_val_00034375.JPEG:n03062245 +ILSVRC2012_val_00034376.JPEG:n04418357 +ILSVRC2012_val_00034377.JPEG:n02841315 +ILSVRC2012_val_00034378.JPEG:n04239074 +ILSVRC2012_val_00034379.JPEG:n02117135 +ILSVRC2012_val_00034380.JPEG:n03908714 +ILSVRC2012_val_00034381.JPEG:n04429376 +ILSVRC2012_val_00034382.JPEG:n02089867 +ILSVRC2012_val_00034383.JPEG:n01641577 +ILSVRC2012_val_00034384.JPEG:n02444819 +ILSVRC2012_val_00034385.JPEG:n04277352 +ILSVRC2012_val_00034386.JPEG:n01443537 +ILSVRC2012_val_00034387.JPEG:n04522168 +ILSVRC2012_val_00034388.JPEG:n02137549 +ILSVRC2012_val_00034389.JPEG:n03770439 +ILSVRC2012_val_00034390.JPEG:n03697007 +ILSVRC2012_val_00034391.JPEG:n07248320 +ILSVRC2012_val_00034392.JPEG:n04523525 +ILSVRC2012_val_00034393.JPEG:n04141975 +ILSVRC2012_val_00034394.JPEG:n04442312 +ILSVRC2012_val_00034395.JPEG:n02979186 +ILSVRC2012_val_00034396.JPEG:n03929855 +ILSVRC2012_val_00034397.JPEG:n03160309 +ILSVRC2012_val_00034398.JPEG:n07613480 +ILSVRC2012_val_00034399.JPEG:n04154565 +ILSVRC2012_val_00034400.JPEG:n03452741 +ILSVRC2012_val_00034401.JPEG:n03063689 +ILSVRC2012_val_00034402.JPEG:n01983481 +ILSVRC2012_val_00034403.JPEG:n03884397 +ILSVRC2012_val_00034404.JPEG:n02687172 +ILSVRC2012_val_00034405.JPEG:n01622779 +ILSVRC2012_val_00034406.JPEG:n01774750 +ILSVRC2012_val_00034407.JPEG:n02096051 +ILSVRC2012_val_00034408.JPEG:n04074963 +ILSVRC2012_val_00034409.JPEG:n03207941 +ILSVRC2012_val_00034410.JPEG:n02107908 +ILSVRC2012_val_00034411.JPEG:n03180011 +ILSVRC2012_val_00034412.JPEG:n04557648 +ILSVRC2012_val_00034413.JPEG:n01491361 +ILSVRC2012_val_00034414.JPEG:n04209239 +ILSVRC2012_val_00034415.JPEG:n02091467 +ILSVRC2012_val_00034416.JPEG:n03930313 +ILSVRC2012_val_00034417.JPEG:n03417042 +ILSVRC2012_val_00034418.JPEG:n02395406 +ILSVRC2012_val_00034419.JPEG:n02112350 +ILSVRC2012_val_00034420.JPEG:n02108915 +ILSVRC2012_val_00034421.JPEG:n02123597 +ILSVRC2012_val_00034422.JPEG:n04125021 +ILSVRC2012_val_00034423.JPEG:n03777754 +ILSVRC2012_val_00034424.JPEG:n09288635 +ILSVRC2012_val_00034425.JPEG:n02066245 +ILSVRC2012_val_00034426.JPEG:n03196217 +ILSVRC2012_val_00034427.JPEG:n04118538 +ILSVRC2012_val_00034428.JPEG:n03733281 +ILSVRC2012_val_00034429.JPEG:n02106550 +ILSVRC2012_val_00034430.JPEG:n02111889 +ILSVRC2012_val_00034431.JPEG:n03720891 +ILSVRC2012_val_00034432.JPEG:n04604644 +ILSVRC2012_val_00034433.JPEG:n03016953 +ILSVRC2012_val_00034434.JPEG:n03249569 +ILSVRC2012_val_00034435.JPEG:n04039381 +ILSVRC2012_val_00034436.JPEG:n02100735 +ILSVRC2012_val_00034437.JPEG:n01582220 +ILSVRC2012_val_00034438.JPEG:n02423022 +ILSVRC2012_val_00034439.JPEG:n03764736 +ILSVRC2012_val_00034440.JPEG:n03109150 +ILSVRC2012_val_00034441.JPEG:n02028035 +ILSVRC2012_val_00034442.JPEG:n02510455 +ILSVRC2012_val_00034443.JPEG:n01735189 +ILSVRC2012_val_00034444.JPEG:n02666196 +ILSVRC2012_val_00034445.JPEG:n02992211 +ILSVRC2012_val_00034446.JPEG:n04356056 +ILSVRC2012_val_00034447.JPEG:n03240683 +ILSVRC2012_val_00034448.JPEG:n01978455 +ILSVRC2012_val_00034449.JPEG:n04579145 +ILSVRC2012_val_00034450.JPEG:n02963159 +ILSVRC2012_val_00034451.JPEG:n09288635 +ILSVRC2012_val_00034452.JPEG:n02442845 +ILSVRC2012_val_00034453.JPEG:n04606251 +ILSVRC2012_val_00034454.JPEG:n02087046 +ILSVRC2012_val_00034455.JPEG:n03344393 +ILSVRC2012_val_00034456.JPEG:n01883070 +ILSVRC2012_val_00034457.JPEG:n03697007 +ILSVRC2012_val_00034458.JPEG:n03891251 +ILSVRC2012_val_00034459.JPEG:n03662601 +ILSVRC2012_val_00034460.JPEG:n02138441 +ILSVRC2012_val_00034461.JPEG:n01753488 +ILSVRC2012_val_00034462.JPEG:n04613696 +ILSVRC2012_val_00034463.JPEG:n01950731 +ILSVRC2012_val_00034464.JPEG:n03485794 +ILSVRC2012_val_00034465.JPEG:n02110341 +ILSVRC2012_val_00034466.JPEG:n02892767 +ILSVRC2012_val_00034467.JPEG:n02492035 +ILSVRC2012_val_00034468.JPEG:n04273569 +ILSVRC2012_val_00034469.JPEG:n04008634 +ILSVRC2012_val_00034470.JPEG:n02095314 +ILSVRC2012_val_00034471.JPEG:n03794056 +ILSVRC2012_val_00034472.JPEG:n09472597 +ILSVRC2012_val_00034473.JPEG:n02802426 +ILSVRC2012_val_00034474.JPEG:n07716906 +ILSVRC2012_val_00034475.JPEG:n03792972 +ILSVRC2012_val_00034476.JPEG:n01872401 +ILSVRC2012_val_00034477.JPEG:n03673027 +ILSVRC2012_val_00034478.JPEG:n02279972 +ILSVRC2012_val_00034479.JPEG:n02910353 +ILSVRC2012_val_00034480.JPEG:n03933933 +ILSVRC2012_val_00034481.JPEG:n03938244 +ILSVRC2012_val_00034482.JPEG:n01558993 +ILSVRC2012_val_00034483.JPEG:n03908714 +ILSVRC2012_val_00034484.JPEG:n01914609 +ILSVRC2012_val_00034485.JPEG:n02101006 +ILSVRC2012_val_00034486.JPEG:n02672831 +ILSVRC2012_val_00034487.JPEG:n04067472 +ILSVRC2012_val_00034488.JPEG:n02526121 +ILSVRC2012_val_00034489.JPEG:n07836838 +ILSVRC2012_val_00034490.JPEG:n02817516 +ILSVRC2012_val_00034491.JPEG:n07742313 +ILSVRC2012_val_00034492.JPEG:n01828970 +ILSVRC2012_val_00034493.JPEG:n04286575 +ILSVRC2012_val_00034494.JPEG:n03649909 +ILSVRC2012_val_00034495.JPEG:n02107683 +ILSVRC2012_val_00034496.JPEG:n02988304 +ILSVRC2012_val_00034497.JPEG:n02165456 +ILSVRC2012_val_00034498.JPEG:n04560804 +ILSVRC2012_val_00034499.JPEG:n01629819 +ILSVRC2012_val_00034500.JPEG:n03814906 +ILSVRC2012_val_00034501.JPEG:n03782006 +ILSVRC2012_val_00034502.JPEG:n02264363 +ILSVRC2012_val_00034503.JPEG:n02909870 +ILSVRC2012_val_00034504.JPEG:n09246464 +ILSVRC2012_val_00034505.JPEG:n02328150 +ILSVRC2012_val_00034506.JPEG:n02730930 +ILSVRC2012_val_00034507.JPEG:n04596742 +ILSVRC2012_val_00034508.JPEG:n03095699 +ILSVRC2012_val_00034509.JPEG:n03146219 +ILSVRC2012_val_00034510.JPEG:n01824575 +ILSVRC2012_val_00034511.JPEG:n03977966 +ILSVRC2012_val_00034512.JPEG:n01807496 +ILSVRC2012_val_00034513.JPEG:n02500267 +ILSVRC2012_val_00034514.JPEG:n02098105 +ILSVRC2012_val_00034515.JPEG:n01796340 +ILSVRC2012_val_00034516.JPEG:n02113978 +ILSVRC2012_val_00034517.JPEG:n02948072 +ILSVRC2012_val_00034518.JPEG:n03089624 +ILSVRC2012_val_00034519.JPEG:n04550184 +ILSVRC2012_val_00034520.JPEG:n07565083 +ILSVRC2012_val_00034521.JPEG:n03529860 +ILSVRC2012_val_00034522.JPEG:n03544143 +ILSVRC2012_val_00034523.JPEG:n02791270 +ILSVRC2012_val_00034524.JPEG:n03775071 +ILSVRC2012_val_00034525.JPEG:n03710721 +ILSVRC2012_val_00034526.JPEG:n13044778 +ILSVRC2012_val_00034527.JPEG:n02504458 +ILSVRC2012_val_00034528.JPEG:n02514041 +ILSVRC2012_val_00034529.JPEG:n03743016 +ILSVRC2012_val_00034530.JPEG:n03483316 +ILSVRC2012_val_00034531.JPEG:n12985857 +ILSVRC2012_val_00034532.JPEG:n03709823 +ILSVRC2012_val_00034533.JPEG:n04465501 +ILSVRC2012_val_00034534.JPEG:n03028079 +ILSVRC2012_val_00034535.JPEG:n04209239 +ILSVRC2012_val_00034536.JPEG:n01807496 +ILSVRC2012_val_00034537.JPEG:n02859443 +ILSVRC2012_val_00034538.JPEG:n04398044 +ILSVRC2012_val_00034539.JPEG:n03337140 +ILSVRC2012_val_00034540.JPEG:n02783161 +ILSVRC2012_val_00034541.JPEG:n02500267 +ILSVRC2012_val_00034542.JPEG:n01644373 +ILSVRC2012_val_00034543.JPEG:n07711569 +ILSVRC2012_val_00034544.JPEG:n03888257 +ILSVRC2012_val_00034545.JPEG:n02655020 +ILSVRC2012_val_00034546.JPEG:n09399592 +ILSVRC2012_val_00034547.JPEG:n03197337 +ILSVRC2012_val_00034548.JPEG:n02007558 +ILSVRC2012_val_00034549.JPEG:n03961711 +ILSVRC2012_val_00034550.JPEG:n04542943 +ILSVRC2012_val_00034551.JPEG:n02116738 +ILSVRC2012_val_00034552.JPEG:n01580077 +ILSVRC2012_val_00034553.JPEG:n02088632 +ILSVRC2012_val_00034554.JPEG:n02096294 +ILSVRC2012_val_00034555.JPEG:n03388183 +ILSVRC2012_val_00034556.JPEG:n02099267 +ILSVRC2012_val_00034557.JPEG:n03445924 +ILSVRC2012_val_00034558.JPEG:n04133789 +ILSVRC2012_val_00034559.JPEG:n04332243 +ILSVRC2012_val_00034560.JPEG:n03201208 +ILSVRC2012_val_00034561.JPEG:n03032252 +ILSVRC2012_val_00034562.JPEG:n02504458 +ILSVRC2012_val_00034563.JPEG:n02979186 +ILSVRC2012_val_00034564.JPEG:n04584207 +ILSVRC2012_val_00034565.JPEG:n03535780 +ILSVRC2012_val_00034566.JPEG:n02229544 +ILSVRC2012_val_00034567.JPEG:n02111500 +ILSVRC2012_val_00034568.JPEG:n04525305 +ILSVRC2012_val_00034569.JPEG:n03197337 +ILSVRC2012_val_00034570.JPEG:n02398521 +ILSVRC2012_val_00034571.JPEG:n02088238 +ILSVRC2012_val_00034572.JPEG:n02364673 +ILSVRC2012_val_00034573.JPEG:n04146614 +ILSVRC2012_val_00034574.JPEG:n02113186 +ILSVRC2012_val_00034575.JPEG:n02391049 +ILSVRC2012_val_00034576.JPEG:n02098286 +ILSVRC2012_val_00034577.JPEG:n04548362 +ILSVRC2012_val_00034578.JPEG:n02009229 +ILSVRC2012_val_00034579.JPEG:n07802026 +ILSVRC2012_val_00034580.JPEG:n07716906 +ILSVRC2012_val_00034581.JPEG:n02111889 +ILSVRC2012_val_00034582.JPEG:n02730930 +ILSVRC2012_val_00034583.JPEG:n01632777 +ILSVRC2012_val_00034584.JPEG:n02099601 +ILSVRC2012_val_00034585.JPEG:n02981792 +ILSVRC2012_val_00034586.JPEG:n03637318 +ILSVRC2012_val_00034587.JPEG:n01735189 +ILSVRC2012_val_00034588.JPEG:n04049303 +ILSVRC2012_val_00034589.JPEG:n02129165 +ILSVRC2012_val_00034590.JPEG:n02443484 +ILSVRC2012_val_00034591.JPEG:n03770679 +ILSVRC2012_val_00034592.JPEG:n04149813 +ILSVRC2012_val_00034593.JPEG:n01622779 +ILSVRC2012_val_00034594.JPEG:n03110669 +ILSVRC2012_val_00034595.JPEG:n01945685 +ILSVRC2012_val_00034596.JPEG:n03937543 +ILSVRC2012_val_00034597.JPEG:n02977058 +ILSVRC2012_val_00034598.JPEG:n02457408 +ILSVRC2012_val_00034599.JPEG:n03041632 +ILSVRC2012_val_00034600.JPEG:n01694178 +ILSVRC2012_val_00034601.JPEG:n03095699 +ILSVRC2012_val_00034602.JPEG:n02085936 +ILSVRC2012_val_00034603.JPEG:n04252077 +ILSVRC2012_val_00034604.JPEG:n03529860 +ILSVRC2012_val_00034605.JPEG:n01978455 +ILSVRC2012_val_00034606.JPEG:n01768244 +ILSVRC2012_val_00034607.JPEG:n06359193 +ILSVRC2012_val_00034608.JPEG:n02107908 +ILSVRC2012_val_00034609.JPEG:n04162706 +ILSVRC2012_val_00034610.JPEG:n03494278 +ILSVRC2012_val_00034611.JPEG:n02009912 +ILSVRC2012_val_00034612.JPEG:n01740131 +ILSVRC2012_val_00034613.JPEG:n03717622 +ILSVRC2012_val_00034614.JPEG:n13054560 +ILSVRC2012_val_00034615.JPEG:n03014705 +ILSVRC2012_val_00034616.JPEG:n02087394 +ILSVRC2012_val_00034617.JPEG:n02093991 +ILSVRC2012_val_00034618.JPEG:n03063689 +ILSVRC2012_val_00034619.JPEG:n02113023 +ILSVRC2012_val_00034620.JPEG:n03733131 +ILSVRC2012_val_00034621.JPEG:n04493381 +ILSVRC2012_val_00034622.JPEG:n03825788 +ILSVRC2012_val_00034623.JPEG:n02643566 +ILSVRC2012_val_00034624.JPEG:n03495258 +ILSVRC2012_val_00034625.JPEG:n06794110 +ILSVRC2012_val_00034626.JPEG:n02280649 +ILSVRC2012_val_00034627.JPEG:n04065272 +ILSVRC2012_val_00034628.JPEG:n02110958 +ILSVRC2012_val_00034629.JPEG:n03452741 +ILSVRC2012_val_00034630.JPEG:n03314780 +ILSVRC2012_val_00034631.JPEG:n01828970 +ILSVRC2012_val_00034632.JPEG:n02871525 +ILSVRC2012_val_00034633.JPEG:n04447861 +ILSVRC2012_val_00034634.JPEG:n02815834 +ILSVRC2012_val_00034635.JPEG:n04417672 +ILSVRC2012_val_00034636.JPEG:n04328186 +ILSVRC2012_val_00034637.JPEG:n02134418 +ILSVRC2012_val_00034638.JPEG:n03788365 +ILSVRC2012_val_00034639.JPEG:n03877845 +ILSVRC2012_val_00034640.JPEG:n04487081 +ILSVRC2012_val_00034641.JPEG:n02500267 +ILSVRC2012_val_00034642.JPEG:n03372029 +ILSVRC2012_val_00034643.JPEG:n03837869 +ILSVRC2012_val_00034644.JPEG:n01968897 +ILSVRC2012_val_00034645.JPEG:n03443371 +ILSVRC2012_val_00034646.JPEG:n12768682 +ILSVRC2012_val_00034647.JPEG:n01685808 +ILSVRC2012_val_00034648.JPEG:n03584829 +ILSVRC2012_val_00034649.JPEG:n02814860 +ILSVRC2012_val_00034650.JPEG:n03485407 +ILSVRC2012_val_00034651.JPEG:n03670208 +ILSVRC2012_val_00034652.JPEG:n01817953 +ILSVRC2012_val_00034653.JPEG:n03026506 +ILSVRC2012_val_00034654.JPEG:n01440764 +ILSVRC2012_val_00034655.JPEG:n01685808 +ILSVRC2012_val_00034656.JPEG:n03691459 +ILSVRC2012_val_00034657.JPEG:n04141076 +ILSVRC2012_val_00034658.JPEG:n04179913 +ILSVRC2012_val_00034659.JPEG:n03670208 +ILSVRC2012_val_00034660.JPEG:n01755581 +ILSVRC2012_val_00034661.JPEG:n03958227 +ILSVRC2012_val_00034662.JPEG:n03388043 +ILSVRC2012_val_00034663.JPEG:n03223299 +ILSVRC2012_val_00034664.JPEG:n02504013 +ILSVRC2012_val_00034665.JPEG:n01773549 +ILSVRC2012_val_00034666.JPEG:n01694178 +ILSVRC2012_val_00034667.JPEG:n02112018 +ILSVRC2012_val_00034668.JPEG:n01739381 +ILSVRC2012_val_00034669.JPEG:n01695060 +ILSVRC2012_val_00034670.JPEG:n01980166 +ILSVRC2012_val_00034671.JPEG:n03788365 +ILSVRC2012_val_00034672.JPEG:n03187595 +ILSVRC2012_val_00034673.JPEG:n02277742 +ILSVRC2012_val_00034674.JPEG:n01669191 +ILSVRC2012_val_00034675.JPEG:n02892201 +ILSVRC2012_val_00034676.JPEG:n02123045 +ILSVRC2012_val_00034677.JPEG:n07747607 +ILSVRC2012_val_00034678.JPEG:n04604644 +ILSVRC2012_val_00034679.JPEG:n04149813 +ILSVRC2012_val_00034680.JPEG:n04074963 +ILSVRC2012_val_00034681.JPEG:n02111277 +ILSVRC2012_val_00034682.JPEG:n02101006 +ILSVRC2012_val_00034683.JPEG:n03961711 +ILSVRC2012_val_00034684.JPEG:n01978287 +ILSVRC2012_val_00034685.JPEG:n03127747 +ILSVRC2012_val_00034686.JPEG:n02129604 +ILSVRC2012_val_00034687.JPEG:n07717410 +ILSVRC2012_val_00034688.JPEG:n02264363 +ILSVRC2012_val_00034689.JPEG:n07802026 +ILSVRC2012_val_00034690.JPEG:n02089973 +ILSVRC2012_val_00034691.JPEG:n02096585 +ILSVRC2012_val_00034692.JPEG:n04243546 +ILSVRC2012_val_00034693.JPEG:n01688243 +ILSVRC2012_val_00034694.JPEG:n02817516 +ILSVRC2012_val_00034695.JPEG:n04596742 +ILSVRC2012_val_00034696.JPEG:n03673027 +ILSVRC2012_val_00034697.JPEG:n02797295 +ILSVRC2012_val_00034698.JPEG:n07753113 +ILSVRC2012_val_00034699.JPEG:n01685808 +ILSVRC2012_val_00034700.JPEG:n02871525 +ILSVRC2012_val_00034701.JPEG:n02093991 +ILSVRC2012_val_00034702.JPEG:n01984695 +ILSVRC2012_val_00034703.JPEG:n07760859 +ILSVRC2012_val_00034704.JPEG:n03032252 +ILSVRC2012_val_00034705.JPEG:n07711569 +ILSVRC2012_val_00034706.JPEG:n02280649 +ILSVRC2012_val_00034707.JPEG:n03761084 +ILSVRC2012_val_00034708.JPEG:n03160309 +ILSVRC2012_val_00034709.JPEG:n03891332 +ILSVRC2012_val_00034710.JPEG:n02883205 +ILSVRC2012_val_00034711.JPEG:n04372370 +ILSVRC2012_val_00034712.JPEG:n04041544 +ILSVRC2012_val_00034713.JPEG:n04552348 +ILSVRC2012_val_00034714.JPEG:n04264628 +ILSVRC2012_val_00034715.JPEG:n04041544 +ILSVRC2012_val_00034716.JPEG:n01910747 +ILSVRC2012_val_00034717.JPEG:n03950228 +ILSVRC2012_val_00034718.JPEG:n02666196 +ILSVRC2012_val_00034719.JPEG:n04204347 +ILSVRC2012_val_00034720.JPEG:n01560419 +ILSVRC2012_val_00034721.JPEG:n04204238 +ILSVRC2012_val_00034722.JPEG:n02236044 +ILSVRC2012_val_00034723.JPEG:n03131574 +ILSVRC2012_val_00034724.JPEG:n04487081 +ILSVRC2012_val_00034725.JPEG:n02018795 +ILSVRC2012_val_00034726.JPEG:n02843684 +ILSVRC2012_val_00034727.JPEG:n03000684 +ILSVRC2012_val_00034728.JPEG:n01667778 +ILSVRC2012_val_00034729.JPEG:n02115641 +ILSVRC2012_val_00034730.JPEG:n04548362 +ILSVRC2012_val_00034731.JPEG:n01943899 +ILSVRC2012_val_00034732.JPEG:n02100877 +ILSVRC2012_val_00034733.JPEG:n02093256 +ILSVRC2012_val_00034734.JPEG:n02018207 +ILSVRC2012_val_00034735.JPEG:n02112137 +ILSVRC2012_val_00034736.JPEG:n03141823 +ILSVRC2012_val_00034737.JPEG:n02093754 +ILSVRC2012_val_00034738.JPEG:n02174001 +ILSVRC2012_val_00034739.JPEG:n04476259 +ILSVRC2012_val_00034740.JPEG:n02480495 +ILSVRC2012_val_00034741.JPEG:n03887697 +ILSVRC2012_val_00034742.JPEG:n02769748 +ILSVRC2012_val_00034743.JPEG:n02002724 +ILSVRC2012_val_00034744.JPEG:n02113978 +ILSVRC2012_val_00034745.JPEG:n02110627 +ILSVRC2012_val_00034746.JPEG:n03874293 +ILSVRC2012_val_00034747.JPEG:n02107574 +ILSVRC2012_val_00034748.JPEG:n02109047 +ILSVRC2012_val_00034749.JPEG:n01855032 +ILSVRC2012_val_00034750.JPEG:n02794156 +ILSVRC2012_val_00034751.JPEG:n03134739 +ILSVRC2012_val_00034752.JPEG:n07742313 +ILSVRC2012_val_00034753.JPEG:n03124043 +ILSVRC2012_val_00034754.JPEG:n02486261 +ILSVRC2012_val_00034755.JPEG:n02992529 +ILSVRC2012_val_00034756.JPEG:n01734418 +ILSVRC2012_val_00034757.JPEG:n02321529 +ILSVRC2012_val_00034758.JPEG:n03047690 +ILSVRC2012_val_00034759.JPEG:n02879718 +ILSVRC2012_val_00034760.JPEG:n02025239 +ILSVRC2012_val_00034761.JPEG:n03131574 +ILSVRC2012_val_00034762.JPEG:n04347754 +ILSVRC2012_val_00034763.JPEG:n03216828 +ILSVRC2012_val_00034764.JPEG:n02264363 +ILSVRC2012_val_00034765.JPEG:n03041632 +ILSVRC2012_val_00034766.JPEG:n02071294 +ILSVRC2012_val_00034767.JPEG:n01914609 +ILSVRC2012_val_00034768.JPEG:n02497673 +ILSVRC2012_val_00034769.JPEG:n02172182 +ILSVRC2012_val_00034770.JPEG:n01667778 +ILSVRC2012_val_00034771.JPEG:n02106550 +ILSVRC2012_val_00034772.JPEG:n02814860 +ILSVRC2012_val_00034773.JPEG:n01773549 +ILSVRC2012_val_00034774.JPEG:n01986214 +ILSVRC2012_val_00034775.JPEG:n02236044 +ILSVRC2012_val_00034776.JPEG:n02009912 +ILSVRC2012_val_00034777.JPEG:n02487347 +ILSVRC2012_val_00034778.JPEG:n01755581 +ILSVRC2012_val_00034779.JPEG:n03623198 +ILSVRC2012_val_00034780.JPEG:n02445715 +ILSVRC2012_val_00034781.JPEG:n06794110 +ILSVRC2012_val_00034782.JPEG:n02085620 +ILSVRC2012_val_00034783.JPEG:n04482393 +ILSVRC2012_val_00034784.JPEG:n01820546 +ILSVRC2012_val_00034785.JPEG:n04579145 +ILSVRC2012_val_00034786.JPEG:n02326432 +ILSVRC2012_val_00034787.JPEG:n07754684 +ILSVRC2012_val_00034788.JPEG:n04111531 +ILSVRC2012_val_00034789.JPEG:n03724870 +ILSVRC2012_val_00034790.JPEG:n02093256 +ILSVRC2012_val_00034791.JPEG:n07711569 +ILSVRC2012_val_00034792.JPEG:n02017213 +ILSVRC2012_val_00034793.JPEG:n01688243 +ILSVRC2012_val_00034794.JPEG:n01669191 +ILSVRC2012_val_00034795.JPEG:n01664065 +ILSVRC2012_val_00034796.JPEG:n02092339 +ILSVRC2012_val_00034797.JPEG:n02108551 +ILSVRC2012_val_00034798.JPEG:n04525305 +ILSVRC2012_val_00034799.JPEG:n03950228 +ILSVRC2012_val_00034800.JPEG:n03929660 +ILSVRC2012_val_00034801.JPEG:n03956157 +ILSVRC2012_val_00034802.JPEG:n03891332 +ILSVRC2012_val_00034803.JPEG:n04493381 +ILSVRC2012_val_00034804.JPEG:n02102973 +ILSVRC2012_val_00034805.JPEG:n03255030 +ILSVRC2012_val_00034806.JPEG:n01990800 +ILSVRC2012_val_00034807.JPEG:n02500267 +ILSVRC2012_val_00034808.JPEG:n02281406 +ILSVRC2012_val_00034809.JPEG:n01824575 +ILSVRC2012_val_00034810.JPEG:n03032252 +ILSVRC2012_val_00034811.JPEG:n02129165 +ILSVRC2012_val_00034812.JPEG:n02356798 +ILSVRC2012_val_00034813.JPEG:n03538406 +ILSVRC2012_val_00034814.JPEG:n02009229 +ILSVRC2012_val_00034815.JPEG:n02097658 +ILSVRC2012_val_00034816.JPEG:n03095699 +ILSVRC2012_val_00034817.JPEG:n03786901 +ILSVRC2012_val_00034818.JPEG:n03743016 +ILSVRC2012_val_00034819.JPEG:n02980441 +ILSVRC2012_val_00034820.JPEG:n07742313 +ILSVRC2012_val_00034821.JPEG:n02106166 +ILSVRC2012_val_00034822.JPEG:n03314780 +ILSVRC2012_val_00034823.JPEG:n02097209 +ILSVRC2012_val_00034824.JPEG:n04037443 +ILSVRC2012_val_00034825.JPEG:n04086273 +ILSVRC2012_val_00034826.JPEG:n03394916 +ILSVRC2012_val_00034827.JPEG:n02037110 +ILSVRC2012_val_00034828.JPEG:n02112018 +ILSVRC2012_val_00034829.JPEG:n03379051 +ILSVRC2012_val_00034830.JPEG:n02951585 +ILSVRC2012_val_00034831.JPEG:n04501370 +ILSVRC2012_val_00034832.JPEG:n04355338 +ILSVRC2012_val_00034833.JPEG:n03874293 +ILSVRC2012_val_00034834.JPEG:n04153751 +ILSVRC2012_val_00034835.JPEG:n07930864 +ILSVRC2012_val_00034836.JPEG:n02930766 +ILSVRC2012_val_00034837.JPEG:n01496331 +ILSVRC2012_val_00034838.JPEG:n04265275 +ILSVRC2012_val_00034839.JPEG:n02256656 +ILSVRC2012_val_00034840.JPEG:n01667114 +ILSVRC2012_val_00034841.JPEG:n03630383 +ILSVRC2012_val_00034842.JPEG:n04591713 +ILSVRC2012_val_00034843.JPEG:n02704792 +ILSVRC2012_val_00034844.JPEG:n03207743 +ILSVRC2012_val_00034845.JPEG:n03854065 +ILSVRC2012_val_00034846.JPEG:n03720891 +ILSVRC2012_val_00034847.JPEG:n07873807 +ILSVRC2012_val_00034848.JPEG:n02120505 +ILSVRC2012_val_00034849.JPEG:n02099849 +ILSVRC2012_val_00034850.JPEG:n04152593 +ILSVRC2012_val_00034851.JPEG:n02100877 +ILSVRC2012_val_00034852.JPEG:n04560804 +ILSVRC2012_val_00034853.JPEG:n03792972 +ILSVRC2012_val_00034854.JPEG:n03733131 +ILSVRC2012_val_00034855.JPEG:n13133613 +ILSVRC2012_val_00034856.JPEG:n02114548 +ILSVRC2012_val_00034857.JPEG:n03000247 +ILSVRC2012_val_00034858.JPEG:n04146614 +ILSVRC2012_val_00034859.JPEG:n04398044 +ILSVRC2012_val_00034860.JPEG:n02325366 +ILSVRC2012_val_00034861.JPEG:n03633091 +ILSVRC2012_val_00034862.JPEG:n09256479 +ILSVRC2012_val_00034863.JPEG:n03617480 +ILSVRC2012_val_00034864.JPEG:n01530575 +ILSVRC2012_val_00034865.JPEG:n03633091 +ILSVRC2012_val_00034866.JPEG:n03018349 +ILSVRC2012_val_00034867.JPEG:n01768244 +ILSVRC2012_val_00034868.JPEG:n02871525 +ILSVRC2012_val_00034869.JPEG:n04040759 +ILSVRC2012_val_00034870.JPEG:n03658185 +ILSVRC2012_val_00034871.JPEG:n03272562 +ILSVRC2012_val_00034872.JPEG:n02447366 +ILSVRC2012_val_00034873.JPEG:n04392985 +ILSVRC2012_val_00034874.JPEG:n02797295 +ILSVRC2012_val_00034875.JPEG:n03903868 +ILSVRC2012_val_00034876.JPEG:n04548362 +ILSVRC2012_val_00034877.JPEG:n07714571 +ILSVRC2012_val_00034878.JPEG:n03884397 +ILSVRC2012_val_00034879.JPEG:n03888605 +ILSVRC2012_val_00034880.JPEG:n02105505 +ILSVRC2012_val_00034881.JPEG:n03666591 +ILSVRC2012_val_00034882.JPEG:n03063599 +ILSVRC2012_val_00034883.JPEG:n03530642 +ILSVRC2012_val_00034884.JPEG:n02097474 +ILSVRC2012_val_00034885.JPEG:n04483307 +ILSVRC2012_val_00034886.JPEG:n04554684 +ILSVRC2012_val_00034887.JPEG:n02978881 +ILSVRC2012_val_00034888.JPEG:n02492660 +ILSVRC2012_val_00034889.JPEG:n03692522 +ILSVRC2012_val_00034890.JPEG:n04589890 +ILSVRC2012_val_00034891.JPEG:n04579432 +ILSVRC2012_val_00034892.JPEG:n02127052 +ILSVRC2012_val_00034893.JPEG:n02112706 +ILSVRC2012_val_00034894.JPEG:n02804610 +ILSVRC2012_val_00034895.JPEG:n02190166 +ILSVRC2012_val_00034896.JPEG:n11939491 +ILSVRC2012_val_00034897.JPEG:n03000134 +ILSVRC2012_val_00034898.JPEG:n01697457 +ILSVRC2012_val_00034899.JPEG:n12620546 +ILSVRC2012_val_00034900.JPEG:n02256656 +ILSVRC2012_val_00034901.JPEG:n01968897 +ILSVRC2012_val_00034902.JPEG:n02950826 +ILSVRC2012_val_00034903.JPEG:n03127925 +ILSVRC2012_val_00034904.JPEG:n02939185 +ILSVRC2012_val_00034905.JPEG:n06596364 +ILSVRC2012_val_00034906.JPEG:n02091134 +ILSVRC2012_val_00034907.JPEG:n03877472 +ILSVRC2012_val_00034908.JPEG:n02113799 +ILSVRC2012_val_00034909.JPEG:n02102973 +ILSVRC2012_val_00034910.JPEG:n02027492 +ILSVRC2012_val_00034911.JPEG:n03498962 +ILSVRC2012_val_00034912.JPEG:n02834397 +ILSVRC2012_val_00034913.JPEG:n07248320 +ILSVRC2012_val_00034914.JPEG:n04286575 +ILSVRC2012_val_00034915.JPEG:n01735189 +ILSVRC2012_val_00034916.JPEG:n02417914 +ILSVRC2012_val_00034917.JPEG:n03690938 +ILSVRC2012_val_00034918.JPEG:n03404251 +ILSVRC2012_val_00034919.JPEG:n01739381 +ILSVRC2012_val_00034920.JPEG:n02099267 +ILSVRC2012_val_00034921.JPEG:n02219486 +ILSVRC2012_val_00034922.JPEG:n02108089 +ILSVRC2012_val_00034923.JPEG:n02206856 +ILSVRC2012_val_00034924.JPEG:n03208938 +ILSVRC2012_val_00034925.JPEG:n03127747 +ILSVRC2012_val_00034926.JPEG:n02279972 +ILSVRC2012_val_00034927.JPEG:n02281406 +ILSVRC2012_val_00034928.JPEG:n02113023 +ILSVRC2012_val_00034929.JPEG:n01601694 +ILSVRC2012_val_00034930.JPEG:n07715103 +ILSVRC2012_val_00034931.JPEG:n02107908 +ILSVRC2012_val_00034932.JPEG:n02120079 +ILSVRC2012_val_00034933.JPEG:n02102318 +ILSVRC2012_val_00034934.JPEG:n02096051 +ILSVRC2012_val_00034935.JPEG:n01990800 +ILSVRC2012_val_00034936.JPEG:n02917067 +ILSVRC2012_val_00034937.JPEG:n03372029 +ILSVRC2012_val_00034938.JPEG:n03538406 +ILSVRC2012_val_00034939.JPEG:n12267677 +ILSVRC2012_val_00034940.JPEG:n03314780 +ILSVRC2012_val_00034941.JPEG:n03903868 +ILSVRC2012_val_00034942.JPEG:n02009229 +ILSVRC2012_val_00034943.JPEG:n02100236 +ILSVRC2012_val_00034944.JPEG:n03759954 +ILSVRC2012_val_00034945.JPEG:n02277742 +ILSVRC2012_val_00034946.JPEG:n03804744 +ILSVRC2012_val_00034947.JPEG:n02966687 +ILSVRC2012_val_00034948.JPEG:n02102318 +ILSVRC2012_val_00034949.JPEG:n09835506 +ILSVRC2012_val_00034950.JPEG:n01484850 +ILSVRC2012_val_00034951.JPEG:n02097047 +ILSVRC2012_val_00034952.JPEG:n02795169 +ILSVRC2012_val_00034953.JPEG:n03673027 +ILSVRC2012_val_00034954.JPEG:n02169497 +ILSVRC2012_val_00034955.JPEG:n03532672 +ILSVRC2012_val_00034956.JPEG:n04067472 +ILSVRC2012_val_00034957.JPEG:n01944390 +ILSVRC2012_val_00034958.JPEG:n02786058 +ILSVRC2012_val_00034959.JPEG:n04019541 +ILSVRC2012_val_00034960.JPEG:n01665541 +ILSVRC2012_val_00034961.JPEG:n04162706 +ILSVRC2012_val_00034962.JPEG:n01695060 +ILSVRC2012_val_00034963.JPEG:n04116512 +ILSVRC2012_val_00034964.JPEG:n03680355 +ILSVRC2012_val_00034965.JPEG:n04548280 +ILSVRC2012_val_00034966.JPEG:n04517823 +ILSVRC2012_val_00034967.JPEG:n02883205 +ILSVRC2012_val_00034968.JPEG:n02869837 +ILSVRC2012_val_00034969.JPEG:n01871265 +ILSVRC2012_val_00034970.JPEG:n01737021 +ILSVRC2012_val_00034971.JPEG:n01496331 +ILSVRC2012_val_00034972.JPEG:n01773797 +ILSVRC2012_val_00034973.JPEG:n04562935 +ILSVRC2012_val_00034974.JPEG:n03617480 +ILSVRC2012_val_00034975.JPEG:n03930630 +ILSVRC2012_val_00034976.JPEG:n04033901 +ILSVRC2012_val_00034977.JPEG:n04270147 +ILSVRC2012_val_00034978.JPEG:n03388183 +ILSVRC2012_val_00034979.JPEG:n02823428 +ILSVRC2012_val_00034980.JPEG:n02090622 +ILSVRC2012_val_00034981.JPEG:n02504013 +ILSVRC2012_val_00034982.JPEG:n04356056 +ILSVRC2012_val_00034983.JPEG:n02510455 +ILSVRC2012_val_00034984.JPEG:n01860187 +ILSVRC2012_val_00034985.JPEG:n02492660 +ILSVRC2012_val_00034986.JPEG:n02879718 +ILSVRC2012_val_00034987.JPEG:n02669723 +ILSVRC2012_val_00034988.JPEG:n15075141 +ILSVRC2012_val_00034989.JPEG:n04263257 +ILSVRC2012_val_00034990.JPEG:n02422106 +ILSVRC2012_val_00034991.JPEG:n04350905 +ILSVRC2012_val_00034992.JPEG:n02105056 +ILSVRC2012_val_00034993.JPEG:n02102973 +ILSVRC2012_val_00034994.JPEG:n03776460 +ILSVRC2012_val_00034995.JPEG:n03857828 +ILSVRC2012_val_00034996.JPEG:n02120505 +ILSVRC2012_val_00034997.JPEG:n02105412 +ILSVRC2012_val_00034998.JPEG:n02643566 +ILSVRC2012_val_00034999.JPEG:n03291819 +ILSVRC2012_val_00035000.JPEG:n04447861 +ILSVRC2012_val_00035001.JPEG:n03938244 +ILSVRC2012_val_00035002.JPEG:n07717556 +ILSVRC2012_val_00035003.JPEG:n02423022 +ILSVRC2012_val_00035004.JPEG:n03450230 +ILSVRC2012_val_00035005.JPEG:n01770393 +ILSVRC2012_val_00035006.JPEG:n04254680 +ILSVRC2012_val_00035007.JPEG:n03530642 +ILSVRC2012_val_00035008.JPEG:n03476991 +ILSVRC2012_val_00035009.JPEG:n03710721 +ILSVRC2012_val_00035010.JPEG:n04116512 +ILSVRC2012_val_00035011.JPEG:n04398044 +ILSVRC2012_val_00035012.JPEG:n02930766 +ILSVRC2012_val_00035013.JPEG:n04370456 +ILSVRC2012_val_00035014.JPEG:n02231487 +ILSVRC2012_val_00035015.JPEG:n04019541 +ILSVRC2012_val_00035016.JPEG:n03476991 +ILSVRC2012_val_00035017.JPEG:n04366367 +ILSVRC2012_val_00035018.JPEG:n02930766 +ILSVRC2012_val_00035019.JPEG:n01728920 +ILSVRC2012_val_00035020.JPEG:n03908618 +ILSVRC2012_val_00035021.JPEG:n07615774 +ILSVRC2012_val_00035022.JPEG:n06794110 +ILSVRC2012_val_00035023.JPEG:n01744401 +ILSVRC2012_val_00035024.JPEG:n04153751 +ILSVRC2012_val_00035025.JPEG:n03187595 +ILSVRC2012_val_00035026.JPEG:n02009912 +ILSVRC2012_val_00035027.JPEG:n02096437 +ILSVRC2012_val_00035028.JPEG:n02018207 +ILSVRC2012_val_00035029.JPEG:n02363005 +ILSVRC2012_val_00035030.JPEG:n07717410 +ILSVRC2012_val_00035031.JPEG:n02939185 +ILSVRC2012_val_00035032.JPEG:n03495258 +ILSVRC2012_val_00035033.JPEG:n03787032 +ILSVRC2012_val_00035034.JPEG:n03920288 +ILSVRC2012_val_00035035.JPEG:n04392985 +ILSVRC2012_val_00035036.JPEG:n02109961 +ILSVRC2012_val_00035037.JPEG:n04325704 +ILSVRC2012_val_00035038.JPEG:n03240683 +ILSVRC2012_val_00035039.JPEG:n01773157 +ILSVRC2012_val_00035040.JPEG:n02317335 +ILSVRC2012_val_00035041.JPEG:n03929660 +ILSVRC2012_val_00035042.JPEG:n02493509 +ILSVRC2012_val_00035043.JPEG:n03920288 +ILSVRC2012_val_00035044.JPEG:n03447721 +ILSVRC2012_val_00035045.JPEG:n02486261 +ILSVRC2012_val_00035046.JPEG:n04562935 +ILSVRC2012_val_00035047.JPEG:n01829413 +ILSVRC2012_val_00035048.JPEG:n01930112 +ILSVRC2012_val_00035049.JPEG:n02104365 +ILSVRC2012_val_00035050.JPEG:n02992211 +ILSVRC2012_val_00035051.JPEG:n04033901 +ILSVRC2012_val_00035052.JPEG:n03710193 +ILSVRC2012_val_00035053.JPEG:n02797295 +ILSVRC2012_val_00035054.JPEG:n01847000 +ILSVRC2012_val_00035055.JPEG:n02100583 +ILSVRC2012_val_00035056.JPEG:n04483307 +ILSVRC2012_val_00035057.JPEG:n03874599 +ILSVRC2012_val_00035058.JPEG:n04275548 +ILSVRC2012_val_00035059.JPEG:n04540053 +ILSVRC2012_val_00035060.JPEG:n01558993 +ILSVRC2012_val_00035061.JPEG:n04560804 +ILSVRC2012_val_00035062.JPEG:n04542943 +ILSVRC2012_val_00035063.JPEG:n01773549 +ILSVRC2012_val_00035064.JPEG:n04317175 +ILSVRC2012_val_00035065.JPEG:n03935335 +ILSVRC2012_val_00035066.JPEG:n07717410 +ILSVRC2012_val_00035067.JPEG:n02165456 +ILSVRC2012_val_00035068.JPEG:n03832673 +ILSVRC2012_val_00035069.JPEG:n01692333 +ILSVRC2012_val_00035070.JPEG:n03788195 +ILSVRC2012_val_00035071.JPEG:n07831146 +ILSVRC2012_val_00035072.JPEG:n03590841 +ILSVRC2012_val_00035073.JPEG:n03840681 +ILSVRC2012_val_00035074.JPEG:n02277742 +ILSVRC2012_val_00035075.JPEG:n09472597 +ILSVRC2012_val_00035076.JPEG:n07614500 +ILSVRC2012_val_00035077.JPEG:n04548280 +ILSVRC2012_val_00035078.JPEG:n03443371 +ILSVRC2012_val_00035079.JPEG:n04532670 +ILSVRC2012_val_00035080.JPEG:n01774750 +ILSVRC2012_val_00035081.JPEG:n04486054 +ILSVRC2012_val_00035082.JPEG:n03127747 +ILSVRC2012_val_00035083.JPEG:n03676483 +ILSVRC2012_val_00035084.JPEG:n02669723 +ILSVRC2012_val_00035085.JPEG:n02017213 +ILSVRC2012_val_00035086.JPEG:n01945685 +ILSVRC2012_val_00035087.JPEG:n02219486 +ILSVRC2012_val_00035088.JPEG:n04599235 +ILSVRC2012_val_00035089.JPEG:n03530642 +ILSVRC2012_val_00035090.JPEG:n04254777 +ILSVRC2012_val_00035091.JPEG:n02111500 +ILSVRC2012_val_00035092.JPEG:n03125729 +ILSVRC2012_val_00035093.JPEG:n01631663 +ILSVRC2012_val_00035094.JPEG:n07880968 +ILSVRC2012_val_00035095.JPEG:n02111277 +ILSVRC2012_val_00035096.JPEG:n01817953 +ILSVRC2012_val_00035097.JPEG:n03776460 +ILSVRC2012_val_00035098.JPEG:n01622779 +ILSVRC2012_val_00035099.JPEG:n03240683 +ILSVRC2012_val_00035100.JPEG:n02906734 +ILSVRC2012_val_00035101.JPEG:n02391049 +ILSVRC2012_val_00035102.JPEG:n01695060 +ILSVRC2012_val_00035103.JPEG:n04023962 +ILSVRC2012_val_00035104.JPEG:n01514668 +ILSVRC2012_val_00035105.JPEG:n04133789 +ILSVRC2012_val_00035106.JPEG:n02871525 +ILSVRC2012_val_00035107.JPEG:n02277742 +ILSVRC2012_val_00035108.JPEG:n02090721 +ILSVRC2012_val_00035109.JPEG:n01693334 +ILSVRC2012_val_00035110.JPEG:n04074963 +ILSVRC2012_val_00035111.JPEG:n07693725 +ILSVRC2012_val_00035112.JPEG:n01873310 +ILSVRC2012_val_00035113.JPEG:n02279972 +ILSVRC2012_val_00035114.JPEG:n02971356 +ILSVRC2012_val_00035115.JPEG:n02071294 +ILSVRC2012_val_00035116.JPEG:n03991062 +ILSVRC2012_val_00035117.JPEG:n02088238 +ILSVRC2012_val_00035118.JPEG:n03538406 +ILSVRC2012_val_00035119.JPEG:n04552348 +ILSVRC2012_val_00035120.JPEG:n02112706 +ILSVRC2012_val_00035121.JPEG:n04229816 +ILSVRC2012_val_00035122.JPEG:n03126707 +ILSVRC2012_val_00035123.JPEG:n01518878 +ILSVRC2012_val_00035124.JPEG:n03903868 +ILSVRC2012_val_00035125.JPEG:n13054560 +ILSVRC2012_val_00035126.JPEG:n04149813 +ILSVRC2012_val_00035127.JPEG:n01828970 +ILSVRC2012_val_00035128.JPEG:n03197337 +ILSVRC2012_val_00035129.JPEG:n02443114 +ILSVRC2012_val_00035130.JPEG:n03255030 +ILSVRC2012_val_00035131.JPEG:n01558993 +ILSVRC2012_val_00035132.JPEG:n03529860 +ILSVRC2012_val_00035133.JPEG:n04069434 +ILSVRC2012_val_00035134.JPEG:n02396427 +ILSVRC2012_val_00035135.JPEG:n03197337 +ILSVRC2012_val_00035136.JPEG:n02356798 +ILSVRC2012_val_00035137.JPEG:n02504013 +ILSVRC2012_val_00035138.JPEG:n02641379 +ILSVRC2012_val_00035139.JPEG:n02017213 +ILSVRC2012_val_00035140.JPEG:n01882714 +ILSVRC2012_val_00035141.JPEG:n01514859 +ILSVRC2012_val_00035142.JPEG:n04429376 +ILSVRC2012_val_00035143.JPEG:n04366367 +ILSVRC2012_val_00035144.JPEG:n04443257 +ILSVRC2012_val_00035145.JPEG:n03075370 +ILSVRC2012_val_00035146.JPEG:n03782006 +ILSVRC2012_val_00035147.JPEG:n02927161 +ILSVRC2012_val_00035148.JPEG:n03899768 +ILSVRC2012_val_00035149.JPEG:n07715103 +ILSVRC2012_val_00035150.JPEG:n03980874 +ILSVRC2012_val_00035151.JPEG:n01514668 +ILSVRC2012_val_00035152.JPEG:n03761084 +ILSVRC2012_val_00035153.JPEG:n01773797 +ILSVRC2012_val_00035154.JPEG:n02120079 +ILSVRC2012_val_00035155.JPEG:n04131690 +ILSVRC2012_val_00035156.JPEG:n07248320 +ILSVRC2012_val_00035157.JPEG:n02133161 +ILSVRC2012_val_00035158.JPEG:n02096051 +ILSVRC2012_val_00035159.JPEG:n13052670 +ILSVRC2012_val_00035160.JPEG:n02979186 +ILSVRC2012_val_00035161.JPEG:n02113023 +ILSVRC2012_val_00035162.JPEG:n03594945 +ILSVRC2012_val_00035163.JPEG:n02123045 +ILSVRC2012_val_00035164.JPEG:n02120505 +ILSVRC2012_val_00035165.JPEG:n02119022 +ILSVRC2012_val_00035166.JPEG:n02493793 +ILSVRC2012_val_00035167.JPEG:n01728572 +ILSVRC2012_val_00035168.JPEG:n03482405 +ILSVRC2012_val_00035169.JPEG:n01980166 +ILSVRC2012_val_00035170.JPEG:n07745940 +ILSVRC2012_val_00035171.JPEG:n01773549 +ILSVRC2012_val_00035172.JPEG:n02123394 +ILSVRC2012_val_00035173.JPEG:n02093754 +ILSVRC2012_val_00035174.JPEG:n03534580 +ILSVRC2012_val_00035175.JPEG:n02174001 +ILSVRC2012_val_00035176.JPEG:n02641379 +ILSVRC2012_val_00035177.JPEG:n01693334 +ILSVRC2012_val_00035178.JPEG:n01983481 +ILSVRC2012_val_00035179.JPEG:n02793495 +ILSVRC2012_val_00035180.JPEG:n04456115 +ILSVRC2012_val_00035181.JPEG:n04141327 +ILSVRC2012_val_00035182.JPEG:n02096585 +ILSVRC2012_val_00035183.JPEG:n01855672 +ILSVRC2012_val_00035184.JPEG:n03223299 +ILSVRC2012_val_00035185.JPEG:n03544143 +ILSVRC2012_val_00035186.JPEG:n02321529 +ILSVRC2012_val_00035187.JPEG:n09193705 +ILSVRC2012_val_00035188.JPEG:n04409515 +ILSVRC2012_val_00035189.JPEG:n02105162 +ILSVRC2012_val_00035190.JPEG:n03775546 +ILSVRC2012_val_00035191.JPEG:n01990800 +ILSVRC2012_val_00035192.JPEG:n02128757 +ILSVRC2012_val_00035193.JPEG:n03769881 +ILSVRC2012_val_00035194.JPEG:n03314780 +ILSVRC2012_val_00035195.JPEG:n03598930 +ILSVRC2012_val_00035196.JPEG:n03452741 +ILSVRC2012_val_00035197.JPEG:n03388183 +ILSVRC2012_val_00035198.JPEG:n03958227 +ILSVRC2012_val_00035199.JPEG:n02236044 +ILSVRC2012_val_00035200.JPEG:n04208210 +ILSVRC2012_val_00035201.JPEG:n07693725 +ILSVRC2012_val_00035202.JPEG:n01945685 +ILSVRC2012_val_00035203.JPEG:n04579432 +ILSVRC2012_val_00035204.JPEG:n02486410 +ILSVRC2012_val_00035205.JPEG:n02791270 +ILSVRC2012_val_00035206.JPEG:n02099429 +ILSVRC2012_val_00035207.JPEG:n02074367 +ILSVRC2012_val_00035208.JPEG:n04208210 +ILSVRC2012_val_00035209.JPEG:n01981276 +ILSVRC2012_val_00035210.JPEG:n03240683 +ILSVRC2012_val_00035211.JPEG:n03425413 +ILSVRC2012_val_00035212.JPEG:n02115913 +ILSVRC2012_val_00035213.JPEG:n03124043 +ILSVRC2012_val_00035214.JPEG:n02002724 +ILSVRC2012_val_00035215.JPEG:n02667093 +ILSVRC2012_val_00035216.JPEG:n03724870 +ILSVRC2012_val_00035217.JPEG:n07730033 +ILSVRC2012_val_00035218.JPEG:n03733281 +ILSVRC2012_val_00035219.JPEG:n04522168 +ILSVRC2012_val_00035220.JPEG:n07717556 +ILSVRC2012_val_00035221.JPEG:n03977966 +ILSVRC2012_val_00035222.JPEG:n03788365 +ILSVRC2012_val_00035223.JPEG:n01484850 +ILSVRC2012_val_00035224.JPEG:n03482405 +ILSVRC2012_val_00035225.JPEG:n03623198 +ILSVRC2012_val_00035226.JPEG:n07892512 +ILSVRC2012_val_00035227.JPEG:n07711569 +ILSVRC2012_val_00035228.JPEG:n03710637 +ILSVRC2012_val_00035229.JPEG:n03376595 +ILSVRC2012_val_00035230.JPEG:n04141975 +ILSVRC2012_val_00035231.JPEG:n02981792 +ILSVRC2012_val_00035232.JPEG:n03804744 +ILSVRC2012_val_00035233.JPEG:n02107312 +ILSVRC2012_val_00035234.JPEG:n03733131 +ILSVRC2012_val_00035235.JPEG:n01739381 +ILSVRC2012_val_00035236.JPEG:n04252077 +ILSVRC2012_val_00035237.JPEG:n03445924 +ILSVRC2012_val_00035238.JPEG:n04599235 +ILSVRC2012_val_00035239.JPEG:n02422699 +ILSVRC2012_val_00035240.JPEG:n03637318 +ILSVRC2012_val_00035241.JPEG:n03673027 +ILSVRC2012_val_00035242.JPEG:n03425413 +ILSVRC2012_val_00035243.JPEG:n02442845 +ILSVRC2012_val_00035244.JPEG:n02325366 +ILSVRC2012_val_00035245.JPEG:n02410509 +ILSVRC2012_val_00035246.JPEG:n02641379 +ILSVRC2012_val_00035247.JPEG:n02165105 +ILSVRC2012_val_00035248.JPEG:n02769748 +ILSVRC2012_val_00035249.JPEG:n02859443 +ILSVRC2012_val_00035250.JPEG:n01806567 +ILSVRC2012_val_00035251.JPEG:n03527444 +ILSVRC2012_val_00035252.JPEG:n02099601 +ILSVRC2012_val_00035253.JPEG:n07715103 +ILSVRC2012_val_00035254.JPEG:n01531178 +ILSVRC2012_val_00035255.JPEG:n04599235 +ILSVRC2012_val_00035256.JPEG:n07697313 +ILSVRC2012_val_00035257.JPEG:n02091244 +ILSVRC2012_val_00035258.JPEG:n04317175 +ILSVRC2012_val_00035259.JPEG:n02823428 +ILSVRC2012_val_00035260.JPEG:n02096437 +ILSVRC2012_val_00035261.JPEG:n02236044 +ILSVRC2012_val_00035262.JPEG:n02190166 +ILSVRC2012_val_00035263.JPEG:n02948072 +ILSVRC2012_val_00035264.JPEG:n01728920 +ILSVRC2012_val_00035265.JPEG:n01728572 +ILSVRC2012_val_00035266.JPEG:n03000684 +ILSVRC2012_val_00035267.JPEG:n03133878 +ILSVRC2012_val_00035268.JPEG:n02017213 +ILSVRC2012_val_00035269.JPEG:n01978287 +ILSVRC2012_val_00035270.JPEG:n03775071 +ILSVRC2012_val_00035271.JPEG:n04479046 +ILSVRC2012_val_00035272.JPEG:n07720875 +ILSVRC2012_val_00035273.JPEG:n06785654 +ILSVRC2012_val_00035274.JPEG:n01843383 +ILSVRC2012_val_00035275.JPEG:n02108089 +ILSVRC2012_val_00035276.JPEG:n02606052 +ILSVRC2012_val_00035277.JPEG:n02794156 +ILSVRC2012_val_00035278.JPEG:n02100583 +ILSVRC2012_val_00035279.JPEG:n12620546 +ILSVRC2012_val_00035280.JPEG:n02412080 +ILSVRC2012_val_00035281.JPEG:n01677366 +ILSVRC2012_val_00035282.JPEG:n03710637 +ILSVRC2012_val_00035283.JPEG:n07753275 +ILSVRC2012_val_00035284.JPEG:n02417914 +ILSVRC2012_val_00035285.JPEG:n04019541 +ILSVRC2012_val_00035286.JPEG:n01697457 +ILSVRC2012_val_00035287.JPEG:n01806143 +ILSVRC2012_val_00035288.JPEG:n03759954 +ILSVRC2012_val_00035289.JPEG:n02115913 +ILSVRC2012_val_00035290.JPEG:n12985857 +ILSVRC2012_val_00035291.JPEG:n03530642 +ILSVRC2012_val_00035292.JPEG:n02133161 +ILSVRC2012_val_00035293.JPEG:n02086240 +ILSVRC2012_val_00035294.JPEG:n02782093 +ILSVRC2012_val_00035295.JPEG:n02259212 +ILSVRC2012_val_00035296.JPEG:n02110806 +ILSVRC2012_val_00035297.JPEG:n03733131 +ILSVRC2012_val_00035298.JPEG:n02096294 +ILSVRC2012_val_00035299.JPEG:n04229816 +ILSVRC2012_val_00035300.JPEG:n06794110 +ILSVRC2012_val_00035301.JPEG:n02699494 +ILSVRC2012_val_00035302.JPEG:n03761084 +ILSVRC2012_val_00035303.JPEG:n01592084 +ILSVRC2012_val_00035304.JPEG:n07695742 +ILSVRC2012_val_00035305.JPEG:n01631663 +ILSVRC2012_val_00035306.JPEG:n03017168 +ILSVRC2012_val_00035307.JPEG:n04350905 +ILSVRC2012_val_00035308.JPEG:n02256656 +ILSVRC2012_val_00035309.JPEG:n04285008 +ILSVRC2012_val_00035310.JPEG:n01984695 +ILSVRC2012_val_00035311.JPEG:n04275548 +ILSVRC2012_val_00035312.JPEG:n01883070 +ILSVRC2012_val_00035313.JPEG:n03047690 +ILSVRC2012_val_00035314.JPEG:n02445715 +ILSVRC2012_val_00035315.JPEG:n02088094 +ILSVRC2012_val_00035316.JPEG:n03223299 +ILSVRC2012_val_00035317.JPEG:n01729322 +ILSVRC2012_val_00035318.JPEG:n03837869 +ILSVRC2012_val_00035319.JPEG:n02102480 +ILSVRC2012_val_00035320.JPEG:n02088364 +ILSVRC2012_val_00035321.JPEG:n02102177 +ILSVRC2012_val_00035322.JPEG:n04265275 +ILSVRC2012_val_00035323.JPEG:n02319095 +ILSVRC2012_val_00035324.JPEG:n02229544 +ILSVRC2012_val_00035325.JPEG:n03759954 +ILSVRC2012_val_00035326.JPEG:n02869837 +ILSVRC2012_val_00035327.JPEG:n04209133 +ILSVRC2012_val_00035328.JPEG:n03291819 +ILSVRC2012_val_00035329.JPEG:n04371774 +ILSVRC2012_val_00035330.JPEG:n02138441 +ILSVRC2012_val_00035331.JPEG:n02417914 +ILSVRC2012_val_00035332.JPEG:n02128757 +ILSVRC2012_val_00035333.JPEG:n02098286 +ILSVRC2012_val_00035334.JPEG:n04591157 +ILSVRC2012_val_00035335.JPEG:n03443371 +ILSVRC2012_val_00035336.JPEG:n03902125 +ILSVRC2012_val_00035337.JPEG:n02422106 +ILSVRC2012_val_00035338.JPEG:n04423845 +ILSVRC2012_val_00035339.JPEG:n04465501 +ILSVRC2012_val_00035340.JPEG:n13052670 +ILSVRC2012_val_00035341.JPEG:n02087394 +ILSVRC2012_val_00035342.JPEG:n04367480 +ILSVRC2012_val_00035343.JPEG:n07742313 +ILSVRC2012_val_00035344.JPEG:n03538406 +ILSVRC2012_val_00035345.JPEG:n03492542 +ILSVRC2012_val_00035346.JPEG:n03868863 +ILSVRC2012_val_00035347.JPEG:n02088632 +ILSVRC2012_val_00035348.JPEG:n01582220 +ILSVRC2012_val_00035349.JPEG:n03876231 +ILSVRC2012_val_00035350.JPEG:n03770439 +ILSVRC2012_val_00035351.JPEG:n02977058 +ILSVRC2012_val_00035352.JPEG:n03457902 +ILSVRC2012_val_00035353.JPEG:n03874293 +ILSVRC2012_val_00035354.JPEG:n03902125 +ILSVRC2012_val_00035355.JPEG:n03929855 +ILSVRC2012_val_00035356.JPEG:n02391049 +ILSVRC2012_val_00035357.JPEG:n03180011 +ILSVRC2012_val_00035358.JPEG:n03956157 +ILSVRC2012_val_00035359.JPEG:n02790996 +ILSVRC2012_val_00035360.JPEG:n02099712 +ILSVRC2012_val_00035361.JPEG:n01980166 +ILSVRC2012_val_00035362.JPEG:n04041544 +ILSVRC2012_val_00035363.JPEG:n02033041 +ILSVRC2012_val_00035364.JPEG:n03976657 +ILSVRC2012_val_00035365.JPEG:n01751748 +ILSVRC2012_val_00035366.JPEG:n02127052 +ILSVRC2012_val_00035367.JPEG:n01494475 +ILSVRC2012_val_00035368.JPEG:n02128385 +ILSVRC2012_val_00035369.JPEG:n04204347 +ILSVRC2012_val_00035370.JPEG:n03690938 +ILSVRC2012_val_00035371.JPEG:n03759954 +ILSVRC2012_val_00035372.JPEG:n02412080 +ILSVRC2012_val_00035373.JPEG:n04204238 +ILSVRC2012_val_00035374.JPEG:n03662601 +ILSVRC2012_val_00035375.JPEG:n02114855 +ILSVRC2012_val_00035376.JPEG:n03788365 +ILSVRC2012_val_00035377.JPEG:n02104029 +ILSVRC2012_val_00035378.JPEG:n02101556 +ILSVRC2012_val_00035379.JPEG:n01737021 +ILSVRC2012_val_00035380.JPEG:n09288635 +ILSVRC2012_val_00035381.JPEG:n02096177 +ILSVRC2012_val_00035382.JPEG:n02492035 +ILSVRC2012_val_00035383.JPEG:n04238763 +ILSVRC2012_val_00035384.JPEG:n03393912 +ILSVRC2012_val_00035385.JPEG:n04149813 +ILSVRC2012_val_00035386.JPEG:n02398521 +ILSVRC2012_val_00035387.JPEG:n01742172 +ILSVRC2012_val_00035388.JPEG:n02130308 +ILSVRC2012_val_00035389.JPEG:n01534433 +ILSVRC2012_val_00035390.JPEG:n04404412 +ILSVRC2012_val_00035391.JPEG:n02107683 +ILSVRC2012_val_00035392.JPEG:n02708093 +ILSVRC2012_val_00035393.JPEG:n04209239 +ILSVRC2012_val_00035394.JPEG:n07715103 +ILSVRC2012_val_00035395.JPEG:n07718747 +ILSVRC2012_val_00035396.JPEG:n04462240 +ILSVRC2012_val_00035397.JPEG:n02510455 +ILSVRC2012_val_00035398.JPEG:n02098105 +ILSVRC2012_val_00035399.JPEG:n02277742 +ILSVRC2012_val_00035400.JPEG:n02096437 +ILSVRC2012_val_00035401.JPEG:n02802426 +ILSVRC2012_val_00035402.JPEG:n02486261 +ILSVRC2012_val_00035403.JPEG:n02091134 +ILSVRC2012_val_00035404.JPEG:n03272010 +ILSVRC2012_val_00035405.JPEG:n01491361 +ILSVRC2012_val_00035406.JPEG:n04604644 +ILSVRC2012_val_00035407.JPEG:n02640242 +ILSVRC2012_val_00035408.JPEG:n03692522 +ILSVRC2012_val_00035409.JPEG:n02229544 +ILSVRC2012_val_00035410.JPEG:n07720875 +ILSVRC2012_val_00035411.JPEG:n04606251 +ILSVRC2012_val_00035412.JPEG:n04201297 +ILSVRC2012_val_00035413.JPEG:n11939491 +ILSVRC2012_val_00035414.JPEG:n02088364 +ILSVRC2012_val_00035415.JPEG:n02655020 +ILSVRC2012_val_00035416.JPEG:n03657121 +ILSVRC2012_val_00035417.JPEG:n02112350 +ILSVRC2012_val_00035418.JPEG:n02326432 +ILSVRC2012_val_00035419.JPEG:n03445777 +ILSVRC2012_val_00035420.JPEG:n02028035 +ILSVRC2012_val_00035421.JPEG:n04326547 +ILSVRC2012_val_00035422.JPEG:n03400231 +ILSVRC2012_val_00035423.JPEG:n02091032 +ILSVRC2012_val_00035424.JPEG:n03710193 +ILSVRC2012_val_00035425.JPEG:n01742172 +ILSVRC2012_val_00035426.JPEG:n01806567 +ILSVRC2012_val_00035427.JPEG:n03485407 +ILSVRC2012_val_00035428.JPEG:n03450230 +ILSVRC2012_val_00035429.JPEG:n01735189 +ILSVRC2012_val_00035430.JPEG:n02319095 +ILSVRC2012_val_00035431.JPEG:n03467068 +ILSVRC2012_val_00035432.JPEG:n04458633 +ILSVRC2012_val_00035433.JPEG:n03394916 +ILSVRC2012_val_00035434.JPEG:n02500267 +ILSVRC2012_val_00035435.JPEG:n04525038 +ILSVRC2012_val_00035436.JPEG:n02112137 +ILSVRC2012_val_00035437.JPEG:n02107908 +ILSVRC2012_val_00035438.JPEG:n12768682 +ILSVRC2012_val_00035439.JPEG:n02119789 +ILSVRC2012_val_00035440.JPEG:n03662601 +ILSVRC2012_val_00035441.JPEG:n07860988 +ILSVRC2012_val_00035442.JPEG:n04584207 +ILSVRC2012_val_00035443.JPEG:n07932039 +ILSVRC2012_val_00035444.JPEG:n03062245 +ILSVRC2012_val_00035445.JPEG:n07745940 +ILSVRC2012_val_00035446.JPEG:n03085013 +ILSVRC2012_val_00035447.JPEG:n04465501 +ILSVRC2012_val_00035448.JPEG:n02483708 +ILSVRC2012_val_00035449.JPEG:n03379051 +ILSVRC2012_val_00035450.JPEG:n01631663 +ILSVRC2012_val_00035451.JPEG:n01773157 +ILSVRC2012_val_00035452.JPEG:n02364673 +ILSVRC2012_val_00035453.JPEG:n02917067 +ILSVRC2012_val_00035454.JPEG:n02488702 +ILSVRC2012_val_00035455.JPEG:n02105412 +ILSVRC2012_val_00035456.JPEG:n02423022 +ILSVRC2012_val_00035457.JPEG:n03868242 +ILSVRC2012_val_00035458.JPEG:n02018207 +ILSVRC2012_val_00035459.JPEG:n02113624 +ILSVRC2012_val_00035460.JPEG:n04041544 +ILSVRC2012_val_00035461.JPEG:n04548280 +ILSVRC2012_val_00035462.JPEG:n03483316 +ILSVRC2012_val_00035463.JPEG:n03444034 +ILSVRC2012_val_00035464.JPEG:n02125311 +ILSVRC2012_val_00035465.JPEG:n02281406 +ILSVRC2012_val_00035466.JPEG:n04041544 +ILSVRC2012_val_00035467.JPEG:n03223299 +ILSVRC2012_val_00035468.JPEG:n03602883 +ILSVRC2012_val_00035469.JPEG:n12144580 +ILSVRC2012_val_00035470.JPEG:n04192698 +ILSVRC2012_val_00035471.JPEG:n07831146 +ILSVRC2012_val_00035472.JPEG:n01748264 +ILSVRC2012_val_00035473.JPEG:n02096177 +ILSVRC2012_val_00035474.JPEG:n01798484 +ILSVRC2012_val_00035475.JPEG:n03075370 +ILSVRC2012_val_00035476.JPEG:n01807496 +ILSVRC2012_val_00035477.JPEG:n04479046 +ILSVRC2012_val_00035478.JPEG:n03457902 +ILSVRC2012_val_00035479.JPEG:n02504013 +ILSVRC2012_val_00035480.JPEG:n02097047 +ILSVRC2012_val_00035481.JPEG:n07583066 +ILSVRC2012_val_00035482.JPEG:n02979186 +ILSVRC2012_val_00035483.JPEG:n03595614 +ILSVRC2012_val_00035484.JPEG:n04286575 +ILSVRC2012_val_00035485.JPEG:n09246464 +ILSVRC2012_val_00035486.JPEG:n02981792 +ILSVRC2012_val_00035487.JPEG:n03220513 +ILSVRC2012_val_00035488.JPEG:n02090379 +ILSVRC2012_val_00035489.JPEG:n02037110 +ILSVRC2012_val_00035490.JPEG:n02009912 +ILSVRC2012_val_00035491.JPEG:n07860988 +ILSVRC2012_val_00035492.JPEG:n04435653 +ILSVRC2012_val_00035493.JPEG:n02486261 +ILSVRC2012_val_00035494.JPEG:n02129604 +ILSVRC2012_val_00035495.JPEG:n01491361 +ILSVRC2012_val_00035496.JPEG:n04579432 +ILSVRC2012_val_00035497.JPEG:n02165456 +ILSVRC2012_val_00035498.JPEG:n03259280 +ILSVRC2012_val_00035499.JPEG:n01860187 +ILSVRC2012_val_00035500.JPEG:n03796401 +ILSVRC2012_val_00035501.JPEG:n02356798 +ILSVRC2012_val_00035502.JPEG:n01828970 +ILSVRC2012_val_00035503.JPEG:n02206856 +ILSVRC2012_val_00035504.JPEG:n03983396 +ILSVRC2012_val_00035505.JPEG:n02783161 +ILSVRC2012_val_00035506.JPEG:n03134739 +ILSVRC2012_val_00035507.JPEG:n02823428 +ILSVRC2012_val_00035508.JPEG:n04371430 +ILSVRC2012_val_00035509.JPEG:n04118776 +ILSVRC2012_val_00035510.JPEG:n02106166 +ILSVRC2012_val_00035511.JPEG:n02988304 +ILSVRC2012_val_00035512.JPEG:n01770081 +ILSVRC2012_val_00035513.JPEG:n04465501 +ILSVRC2012_val_00035514.JPEG:n03447447 +ILSVRC2012_val_00035515.JPEG:n03976467 +ILSVRC2012_val_00035516.JPEG:n02977058 +ILSVRC2012_val_00035517.JPEG:n02058221 +ILSVRC2012_val_00035518.JPEG:n02280649 +ILSVRC2012_val_00035519.JPEG:n03445777 +ILSVRC2012_val_00035520.JPEG:n03884397 +ILSVRC2012_val_00035521.JPEG:n01797886 +ILSVRC2012_val_00035522.JPEG:n03240683 +ILSVRC2012_val_00035523.JPEG:n03485794 +ILSVRC2012_val_00035524.JPEG:n02974003 +ILSVRC2012_val_00035525.JPEG:n04548280 +ILSVRC2012_val_00035526.JPEG:n02168699 +ILSVRC2012_val_00035527.JPEG:n07716906 +ILSVRC2012_val_00035528.JPEG:n02002556 +ILSVRC2012_val_00035529.JPEG:n01632777 +ILSVRC2012_val_00035530.JPEG:n02111129 +ILSVRC2012_val_00035531.JPEG:n02492035 +ILSVRC2012_val_00035532.JPEG:n02123159 +ILSVRC2012_val_00035533.JPEG:n03424325 +ILSVRC2012_val_00035534.JPEG:n02231487 +ILSVRC2012_val_00035535.JPEG:n01641577 +ILSVRC2012_val_00035536.JPEG:n07873807 +ILSVRC2012_val_00035537.JPEG:n02363005 +ILSVRC2012_val_00035538.JPEG:n02100877 +ILSVRC2012_val_00035539.JPEG:n03777568 +ILSVRC2012_val_00035540.JPEG:n01530575 +ILSVRC2012_val_00035541.JPEG:n03998194 +ILSVRC2012_val_00035542.JPEG:n01829413 +ILSVRC2012_val_00035543.JPEG:n02480855 +ILSVRC2012_val_00035544.JPEG:n09288635 +ILSVRC2012_val_00035545.JPEG:n02321529 +ILSVRC2012_val_00035546.JPEG:n02509815 +ILSVRC2012_val_00035547.JPEG:n03482405 +ILSVRC2012_val_00035548.JPEG:n04493381 +ILSVRC2012_val_00035549.JPEG:n02319095 +ILSVRC2012_val_00035550.JPEG:n03223299 +ILSVRC2012_val_00035551.JPEG:n03388549 +ILSVRC2012_val_00035552.JPEG:n02113186 +ILSVRC2012_val_00035553.JPEG:n02093859 +ILSVRC2012_val_00035554.JPEG:n07718747 +ILSVRC2012_val_00035555.JPEG:n01855032 +ILSVRC2012_val_00035556.JPEG:n10148035 +ILSVRC2012_val_00035557.JPEG:n07753113 +ILSVRC2012_val_00035558.JPEG:n04154565 +ILSVRC2012_val_00035559.JPEG:n02423022 +ILSVRC2012_val_00035560.JPEG:n04179913 +ILSVRC2012_val_00035561.JPEG:n02486410 +ILSVRC2012_val_00035562.JPEG:n02106382 +ILSVRC2012_val_00035563.JPEG:n02033041 +ILSVRC2012_val_00035564.JPEG:n02483708 +ILSVRC2012_val_00035565.JPEG:n01537544 +ILSVRC2012_val_00035566.JPEG:n02123597 +ILSVRC2012_val_00035567.JPEG:n03240683 +ILSVRC2012_val_00035568.JPEG:n04026417 +ILSVRC2012_val_00035569.JPEG:n02108422 +ILSVRC2012_val_00035570.JPEG:n09399592 +ILSVRC2012_val_00035571.JPEG:n02104365 +ILSVRC2012_val_00035572.JPEG:n03794056 +ILSVRC2012_val_00035573.JPEG:n01776313 +ILSVRC2012_val_00035574.JPEG:n02787622 +ILSVRC2012_val_00035575.JPEG:n03854065 +ILSVRC2012_val_00035576.JPEG:n01729977 +ILSVRC2012_val_00035577.JPEG:n02127052 +ILSVRC2012_val_00035578.JPEG:n03942813 +ILSVRC2012_val_00035579.JPEG:n02109047 +ILSVRC2012_val_00035580.JPEG:n03133878 +ILSVRC2012_val_00035581.JPEG:n03775071 +ILSVRC2012_val_00035582.JPEG:n02268443 +ILSVRC2012_val_00035583.JPEG:n04118776 +ILSVRC2012_val_00035584.JPEG:n02009912 +ILSVRC2012_val_00035585.JPEG:n02111889 +ILSVRC2012_val_00035586.JPEG:n04542943 +ILSVRC2012_val_00035587.JPEG:n03759954 +ILSVRC2012_val_00035588.JPEG:n03633091 +ILSVRC2012_val_00035589.JPEG:n03124043 +ILSVRC2012_val_00035590.JPEG:n03016953 +ILSVRC2012_val_00035591.JPEG:n02133161 +ILSVRC2012_val_00035592.JPEG:n02106030 +ILSVRC2012_val_00035593.JPEG:n01773797 +ILSVRC2012_val_00035594.JPEG:n03887697 +ILSVRC2012_val_00035595.JPEG:n04501370 +ILSVRC2012_val_00035596.JPEG:n04120489 +ILSVRC2012_val_00035597.JPEG:n02096051 +ILSVRC2012_val_00035598.JPEG:n01682714 +ILSVRC2012_val_00035599.JPEG:n03133878 +ILSVRC2012_val_00035600.JPEG:n02992211 +ILSVRC2012_val_00035601.JPEG:n01795545 +ILSVRC2012_val_00035602.JPEG:n02033041 +ILSVRC2012_val_00035603.JPEG:n04285008 +ILSVRC2012_val_00035604.JPEG:n02113978 +ILSVRC2012_val_00035605.JPEG:n02006656 +ILSVRC2012_val_00035606.JPEG:n01768244 +ILSVRC2012_val_00035607.JPEG:n02837789 +ILSVRC2012_val_00035608.JPEG:n01622779 +ILSVRC2012_val_00035609.JPEG:n02091831 +ILSVRC2012_val_00035610.JPEG:n02992529 +ILSVRC2012_val_00035611.JPEG:n03929660 +ILSVRC2012_val_00035612.JPEG:n02493793 +ILSVRC2012_val_00035613.JPEG:n03447447 +ILSVRC2012_val_00035614.JPEG:n02013706 +ILSVRC2012_val_00035615.JPEG:n03478589 +ILSVRC2012_val_00035616.JPEG:n07615774 +ILSVRC2012_val_00035617.JPEG:n03530642 +ILSVRC2012_val_00035618.JPEG:n02410509 +ILSVRC2012_val_00035619.JPEG:n01968897 +ILSVRC2012_val_00035620.JPEG:n04252077 +ILSVRC2012_val_00035621.JPEG:n03976467 +ILSVRC2012_val_00035622.JPEG:n07871810 +ILSVRC2012_val_00035623.JPEG:n01697457 +ILSVRC2012_val_00035624.JPEG:n04200800 +ILSVRC2012_val_00035625.JPEG:n01806567 +ILSVRC2012_val_00035626.JPEG:n03998194 +ILSVRC2012_val_00035627.JPEG:n03721384 +ILSVRC2012_val_00035628.JPEG:n02107683 +ILSVRC2012_val_00035629.JPEG:n02950826 +ILSVRC2012_val_00035630.JPEG:n02834397 +ILSVRC2012_val_00035631.JPEG:n02978881 +ILSVRC2012_val_00035632.JPEG:n02106166 +ILSVRC2012_val_00035633.JPEG:n02098413 +ILSVRC2012_val_00035634.JPEG:n04204238 +ILSVRC2012_val_00035635.JPEG:n04328186 +ILSVRC2012_val_00035636.JPEG:n01943899 +ILSVRC2012_val_00035637.JPEG:n03494278 +ILSVRC2012_val_00035638.JPEG:n01798484 +ILSVRC2012_val_00035639.JPEG:n07714990 +ILSVRC2012_val_00035640.JPEG:n02105056 +ILSVRC2012_val_00035641.JPEG:n04033995 +ILSVRC2012_val_00035642.JPEG:n03207743 +ILSVRC2012_val_00035643.JPEG:n03459775 +ILSVRC2012_val_00035644.JPEG:n02704792 +ILSVRC2012_val_00035645.JPEG:n03379051 +ILSVRC2012_val_00035646.JPEG:n04372370 +ILSVRC2012_val_00035647.JPEG:n01855032 +ILSVRC2012_val_00035648.JPEG:n03124170 +ILSVRC2012_val_00035649.JPEG:n04039381 +ILSVRC2012_val_00035650.JPEG:n04355338 +ILSVRC2012_val_00035651.JPEG:n01774384 +ILSVRC2012_val_00035652.JPEG:n03016953 +ILSVRC2012_val_00035653.JPEG:n02486261 +ILSVRC2012_val_00035654.JPEG:n01632777 +ILSVRC2012_val_00035655.JPEG:n02319095 +ILSVRC2012_val_00035656.JPEG:n02106550 +ILSVRC2012_val_00035657.JPEG:n03476684 +ILSVRC2012_val_00035658.JPEG:n01644900 +ILSVRC2012_val_00035659.JPEG:n03729826 +ILSVRC2012_val_00035660.JPEG:n03047690 +ILSVRC2012_val_00035661.JPEG:n04179913 +ILSVRC2012_val_00035662.JPEG:n02437312 +ILSVRC2012_val_00035663.JPEG:n03769881 +ILSVRC2012_val_00035664.JPEG:n01664065 +ILSVRC2012_val_00035665.JPEG:n02107683 +ILSVRC2012_val_00035666.JPEG:n09835506 +ILSVRC2012_val_00035667.JPEG:n01784675 +ILSVRC2012_val_00035668.JPEG:n02483362 +ILSVRC2012_val_00035669.JPEG:n02089867 +ILSVRC2012_val_00035670.JPEG:n04356056 +ILSVRC2012_val_00035671.JPEG:n03666591 +ILSVRC2012_val_00035672.JPEG:n06359193 +ILSVRC2012_val_00035673.JPEG:n02277742 +ILSVRC2012_val_00035674.JPEG:n04456115 +ILSVRC2012_val_00035675.JPEG:n02099267 +ILSVRC2012_val_00035676.JPEG:n03657121 +ILSVRC2012_val_00035677.JPEG:n04149813 +ILSVRC2012_val_00035678.JPEG:n07579787 +ILSVRC2012_val_00035679.JPEG:n04372370 +ILSVRC2012_val_00035680.JPEG:n02095314 +ILSVRC2012_val_00035681.JPEG:n03496892 +ILSVRC2012_val_00035682.JPEG:n02483708 +ILSVRC2012_val_00035683.JPEG:n04417672 +ILSVRC2012_val_00035684.JPEG:n04447861 +ILSVRC2012_val_00035685.JPEG:n02804610 +ILSVRC2012_val_00035686.JPEG:n03126707 +ILSVRC2012_val_00035687.JPEG:n01704323 +ILSVRC2012_val_00035688.JPEG:n09332890 +ILSVRC2012_val_00035689.JPEG:n02090379 +ILSVRC2012_val_00035690.JPEG:n03837869 +ILSVRC2012_val_00035691.JPEG:n11939491 +ILSVRC2012_val_00035692.JPEG:n03866082 +ILSVRC2012_val_00035693.JPEG:n03733131 +ILSVRC2012_val_00035694.JPEG:n02165456 +ILSVRC2012_val_00035695.JPEG:n04443257 +ILSVRC2012_val_00035696.JPEG:n02281787 +ILSVRC2012_val_00035697.JPEG:n02398521 +ILSVRC2012_val_00035698.JPEG:n07718472 +ILSVRC2012_val_00035699.JPEG:n02106382 +ILSVRC2012_val_00035700.JPEG:n02066245 +ILSVRC2012_val_00035701.JPEG:n04428191 +ILSVRC2012_val_00035702.JPEG:n03527444 +ILSVRC2012_val_00035703.JPEG:n03085013 +ILSVRC2012_val_00035704.JPEG:n02112350 +ILSVRC2012_val_00035705.JPEG:n02094433 +ILSVRC2012_val_00035706.JPEG:n03942813 +ILSVRC2012_val_00035707.JPEG:n02398521 +ILSVRC2012_val_00035708.JPEG:n02865351 +ILSVRC2012_val_00035709.JPEG:n03908618 +ILSVRC2012_val_00035710.JPEG:n02229544 +ILSVRC2012_val_00035711.JPEG:n01981276 +ILSVRC2012_val_00035712.JPEG:n03208938 +ILSVRC2012_val_00035713.JPEG:n02236044 +ILSVRC2012_val_00035714.JPEG:n04542943 +ILSVRC2012_val_00035715.JPEG:n02804610 +ILSVRC2012_val_00035716.JPEG:n02843684 +ILSVRC2012_val_00035717.JPEG:n01687978 +ILSVRC2012_val_00035718.JPEG:n02447366 +ILSVRC2012_val_00035719.JPEG:n02099849 +ILSVRC2012_val_00035720.JPEG:n03017168 +ILSVRC2012_val_00035721.JPEG:n02999410 +ILSVRC2012_val_00035722.JPEG:n02013706 +ILSVRC2012_val_00035723.JPEG:n02102040 +ILSVRC2012_val_00035724.JPEG:n02825657 +ILSVRC2012_val_00035725.JPEG:n02091831 +ILSVRC2012_val_00035726.JPEG:n01833805 +ILSVRC2012_val_00035727.JPEG:n02117135 +ILSVRC2012_val_00035728.JPEG:n01910747 +ILSVRC2012_val_00035729.JPEG:n03724870 +ILSVRC2012_val_00035730.JPEG:n04209133 +ILSVRC2012_val_00035731.JPEG:n04328186 +ILSVRC2012_val_00035732.JPEG:n03761084 +ILSVRC2012_val_00035733.JPEG:n04509417 +ILSVRC2012_val_00035734.JPEG:n04612504 +ILSVRC2012_val_00035735.JPEG:n01537544 +ILSVRC2012_val_00035736.JPEG:n01748264 +ILSVRC2012_val_00035737.JPEG:n04542943 +ILSVRC2012_val_00035738.JPEG:n02892767 +ILSVRC2012_val_00035739.JPEG:n04332243 +ILSVRC2012_val_00035740.JPEG:n04591713 +ILSVRC2012_val_00035741.JPEG:n02116738 +ILSVRC2012_val_00035742.JPEG:n07714990 +ILSVRC2012_val_00035743.JPEG:n03782006 +ILSVRC2012_val_00035744.JPEG:n07697313 +ILSVRC2012_val_00035745.JPEG:n03692522 +ILSVRC2012_val_00035746.JPEG:n02776631 +ILSVRC2012_val_00035747.JPEG:n03197337 +ILSVRC2012_val_00035748.JPEG:n06874185 +ILSVRC2012_val_00035749.JPEG:n02089867 +ILSVRC2012_val_00035750.JPEG:n02790996 +ILSVRC2012_val_00035751.JPEG:n02979186 +ILSVRC2012_val_00035752.JPEG:n03938244 +ILSVRC2012_val_00035753.JPEG:n03028079 +ILSVRC2012_val_00035754.JPEG:n02823428 +ILSVRC2012_val_00035755.JPEG:n04133789 +ILSVRC2012_val_00035756.JPEG:n02794156 +ILSVRC2012_val_00035757.JPEG:n02815834 +ILSVRC2012_val_00035758.JPEG:n03063599 +ILSVRC2012_val_00035759.JPEG:n10148035 +ILSVRC2012_val_00035760.JPEG:n02486261 +ILSVRC2012_val_00035761.JPEG:n04435653 +ILSVRC2012_val_00035762.JPEG:n01943899 +ILSVRC2012_val_00035763.JPEG:n02391049 +ILSVRC2012_val_00035764.JPEG:n02090622 +ILSVRC2012_val_00035765.JPEG:n04542943 +ILSVRC2012_val_00035766.JPEG:n02058221 +ILSVRC2012_val_00035767.JPEG:n02089867 +ILSVRC2012_val_00035768.JPEG:n02115641 +ILSVRC2012_val_00035769.JPEG:n03930313 +ILSVRC2012_val_00035770.JPEG:n02105412 +ILSVRC2012_val_00035771.JPEG:n03691459 +ILSVRC2012_val_00035772.JPEG:n03781244 +ILSVRC2012_val_00035773.JPEG:n03721384 +ILSVRC2012_val_00035774.JPEG:n01484850 +ILSVRC2012_val_00035775.JPEG:n03201208 +ILSVRC2012_val_00035776.JPEG:n03710721 +ILSVRC2012_val_00035777.JPEG:n03384352 +ILSVRC2012_val_00035778.JPEG:n02410509 +ILSVRC2012_val_00035779.JPEG:n03787032 +ILSVRC2012_val_00035780.JPEG:n03970156 +ILSVRC2012_val_00035781.JPEG:n02105251 +ILSVRC2012_val_00035782.JPEG:n03958227 +ILSVRC2012_val_00035783.JPEG:n02690373 +ILSVRC2012_val_00035784.JPEG:n01729322 +ILSVRC2012_val_00035785.JPEG:n01518878 +ILSVRC2012_val_00035786.JPEG:n04254680 +ILSVRC2012_val_00035787.JPEG:n02988304 +ILSVRC2012_val_00035788.JPEG:n03670208 +ILSVRC2012_val_00035789.JPEG:n04033901 +ILSVRC2012_val_00035790.JPEG:n02018795 +ILSVRC2012_val_00035791.JPEG:n02749479 +ILSVRC2012_val_00035792.JPEG:n03447721 +ILSVRC2012_val_00035793.JPEG:n02093428 +ILSVRC2012_val_00035794.JPEG:n02099712 +ILSVRC2012_val_00035795.JPEG:n02094114 +ILSVRC2012_val_00035796.JPEG:n02814860 +ILSVRC2012_val_00035797.JPEG:n02167151 +ILSVRC2012_val_00035798.JPEG:n04525305 +ILSVRC2012_val_00035799.JPEG:n02483362 +ILSVRC2012_val_00035800.JPEG:n02105251 +ILSVRC2012_val_00035801.JPEG:n02817516 +ILSVRC2012_val_00035802.JPEG:n04125021 +ILSVRC2012_val_00035803.JPEG:n02979186 +ILSVRC2012_val_00035804.JPEG:n01829413 +ILSVRC2012_val_00035805.JPEG:n02097658 +ILSVRC2012_val_00035806.JPEG:n02909870 +ILSVRC2012_val_00035807.JPEG:n01558993 +ILSVRC2012_val_00035808.JPEG:n03216828 +ILSVRC2012_val_00035809.JPEG:n02280649 +ILSVRC2012_val_00035810.JPEG:n02051845 +ILSVRC2012_val_00035811.JPEG:n02115913 +ILSVRC2012_val_00035812.JPEG:n03938244 +ILSVRC2012_val_00035813.JPEG:n04522168 +ILSVRC2012_val_00035814.JPEG:n01632458 +ILSVRC2012_val_00035815.JPEG:n02106382 +ILSVRC2012_val_00035816.JPEG:n02939185 +ILSVRC2012_val_00035817.JPEG:n04111531 +ILSVRC2012_val_00035818.JPEG:n01693334 +ILSVRC2012_val_00035819.JPEG:n02268853 +ILSVRC2012_val_00035820.JPEG:n02109525 +ILSVRC2012_val_00035821.JPEG:n02125311 +ILSVRC2012_val_00035822.JPEG:n03617480 +ILSVRC2012_val_00035823.JPEG:n02437616 +ILSVRC2012_val_00035824.JPEG:n04146614 +ILSVRC2012_val_00035825.JPEG:n03832673 +ILSVRC2012_val_00035826.JPEG:n02870880 +ILSVRC2012_val_00035827.JPEG:n04554684 +ILSVRC2012_val_00035828.JPEG:n02071294 +ILSVRC2012_val_00035829.JPEG:n02971356 +ILSVRC2012_val_00035830.JPEG:n03775071 +ILSVRC2012_val_00035831.JPEG:n04326547 +ILSVRC2012_val_00035832.JPEG:n11879895 +ILSVRC2012_val_00035833.JPEG:n01531178 +ILSVRC2012_val_00035834.JPEG:n02667093 +ILSVRC2012_val_00035835.JPEG:n04317175 +ILSVRC2012_val_00035836.JPEG:n02027492 +ILSVRC2012_val_00035837.JPEG:n02002556 +ILSVRC2012_val_00035838.JPEG:n02206856 +ILSVRC2012_val_00035839.JPEG:n03527444 +ILSVRC2012_val_00035840.JPEG:n04557648 +ILSVRC2012_val_00035841.JPEG:n04467665 +ILSVRC2012_val_00035842.JPEG:n01742172 +ILSVRC2012_val_00035843.JPEG:n02100236 +ILSVRC2012_val_00035844.JPEG:n02096437 +ILSVRC2012_val_00035845.JPEG:n13054560 +ILSVRC2012_val_00035846.JPEG:n02389026 +ILSVRC2012_val_00035847.JPEG:n02098105 +ILSVRC2012_val_00035848.JPEG:n07871810 +ILSVRC2012_val_00035849.JPEG:n02488291 +ILSVRC2012_val_00035850.JPEG:n04251144 +ILSVRC2012_val_00035851.JPEG:n12057211 +ILSVRC2012_val_00035852.JPEG:n04483307 +ILSVRC2012_val_00035853.JPEG:n01917289 +ILSVRC2012_val_00035854.JPEG:n03637318 +ILSVRC2012_val_00035855.JPEG:n01950731 +ILSVRC2012_val_00035856.JPEG:n01955084 +ILSVRC2012_val_00035857.JPEG:n02869837 +ILSVRC2012_val_00035858.JPEG:n04037443 +ILSVRC2012_val_00035859.JPEG:n02099267 +ILSVRC2012_val_00035860.JPEG:n04254120 +ILSVRC2012_val_00035861.JPEG:n02493793 +ILSVRC2012_val_00035862.JPEG:n12144580 +ILSVRC2012_val_00035863.JPEG:n01968897 +ILSVRC2012_val_00035864.JPEG:n03770679 +ILSVRC2012_val_00035865.JPEG:n02910353 +ILSVRC2012_val_00035866.JPEG:n04146614 +ILSVRC2012_val_00035867.JPEG:n04154565 +ILSVRC2012_val_00035868.JPEG:n02128757 +ILSVRC2012_val_00035869.JPEG:n04380533 +ILSVRC2012_val_00035870.JPEG:n03530642 +ILSVRC2012_val_00035871.JPEG:n02640242 +ILSVRC2012_val_00035872.JPEG:n01530575 +ILSVRC2012_val_00035873.JPEG:n04325704 +ILSVRC2012_val_00035874.JPEG:n04562935 +ILSVRC2012_val_00035875.JPEG:n03838899 +ILSVRC2012_val_00035876.JPEG:n02692877 +ILSVRC2012_val_00035877.JPEG:n03692522 +ILSVRC2012_val_00035878.JPEG:n03916031 +ILSVRC2012_val_00035879.JPEG:n02486261 +ILSVRC2012_val_00035880.JPEG:n03724870 +ILSVRC2012_val_00035881.JPEG:n02099267 +ILSVRC2012_val_00035882.JPEG:n03207941 +ILSVRC2012_val_00035883.JPEG:n02128925 +ILSVRC2012_val_00035884.JPEG:n03461385 +ILSVRC2012_val_00035885.JPEG:n01950731 +ILSVRC2012_val_00035886.JPEG:n02492660 +ILSVRC2012_val_00035887.JPEG:n02102973 +ILSVRC2012_val_00035888.JPEG:n07749582 +ILSVRC2012_val_00035889.JPEG:n04310018 +ILSVRC2012_val_00035890.JPEG:n02110806 +ILSVRC2012_val_00035891.JPEG:n02105056 +ILSVRC2012_val_00035892.JPEG:n09428293 +ILSVRC2012_val_00035893.JPEG:n02087394 +ILSVRC2012_val_00035894.JPEG:n15075141 +ILSVRC2012_val_00035895.JPEG:n03141823 +ILSVRC2012_val_00035896.JPEG:n03709823 +ILSVRC2012_val_00035897.JPEG:n03930630 +ILSVRC2012_val_00035898.JPEG:n02280649 +ILSVRC2012_val_00035899.JPEG:n04069434 +ILSVRC2012_val_00035900.JPEG:n07718747 +ILSVRC2012_val_00035901.JPEG:n02480495 +ILSVRC2012_val_00035902.JPEG:n07754684 +ILSVRC2012_val_00035903.JPEG:n12985857 +ILSVRC2012_val_00035904.JPEG:n03602883 +ILSVRC2012_val_00035905.JPEG:n01665541 +ILSVRC2012_val_00035906.JPEG:n04465501 +ILSVRC2012_val_00035907.JPEG:n02788148 +ILSVRC2012_val_00035908.JPEG:n02114548 +ILSVRC2012_val_00035909.JPEG:n07753275 +ILSVRC2012_val_00035910.JPEG:n03788195 +ILSVRC2012_val_00035911.JPEG:n02814860 +ILSVRC2012_val_00035912.JPEG:n02090379 +ILSVRC2012_val_00035913.JPEG:n03425413 +ILSVRC2012_val_00035914.JPEG:n01751748 +ILSVRC2012_val_00035915.JPEG:n04311174 +ILSVRC2012_val_00035916.JPEG:n01796340 +ILSVRC2012_val_00035917.JPEG:n07613480 +ILSVRC2012_val_00035918.JPEG:n03445777 +ILSVRC2012_val_00035919.JPEG:n04404412 +ILSVRC2012_val_00035920.JPEG:n03124170 +ILSVRC2012_val_00035921.JPEG:n02364673 +ILSVRC2012_val_00035922.JPEG:n01829413 +ILSVRC2012_val_00035923.JPEG:n03134739 +ILSVRC2012_val_00035924.JPEG:n07730033 +ILSVRC2012_val_00035925.JPEG:n03379051 +ILSVRC2012_val_00035926.JPEG:n04485082 +ILSVRC2012_val_00035927.JPEG:n03250847 +ILSVRC2012_val_00035928.JPEG:n07730033 +ILSVRC2012_val_00035929.JPEG:n07714571 +ILSVRC2012_val_00035930.JPEG:n02790996 +ILSVRC2012_val_00035931.JPEG:n03160309 +ILSVRC2012_val_00035932.JPEG:n02268443 +ILSVRC2012_val_00035933.JPEG:n02093859 +ILSVRC2012_val_00035934.JPEG:n13052670 +ILSVRC2012_val_00035935.JPEG:n02086910 +ILSVRC2012_val_00035936.JPEG:n01632458 +ILSVRC2012_val_00035937.JPEG:n04259630 +ILSVRC2012_val_00035938.JPEG:n01806567 +ILSVRC2012_val_00035939.JPEG:n02094433 +ILSVRC2012_val_00035940.JPEG:n02093647 +ILSVRC2012_val_00035941.JPEG:n02111500 +ILSVRC2012_val_00035942.JPEG:n03876231 +ILSVRC2012_val_00035943.JPEG:n01883070 +ILSVRC2012_val_00035944.JPEG:n02098286 +ILSVRC2012_val_00035945.JPEG:n04483307 +ILSVRC2012_val_00035946.JPEG:n03344393 +ILSVRC2012_val_00035947.JPEG:n01592084 +ILSVRC2012_val_00035948.JPEG:n04579432 +ILSVRC2012_val_00035949.JPEG:n04152593 +ILSVRC2012_val_00035950.JPEG:n04579145 +ILSVRC2012_val_00035951.JPEG:n03998194 +ILSVRC2012_val_00035952.JPEG:n02093256 +ILSVRC2012_val_00035953.JPEG:n01616318 +ILSVRC2012_val_00035954.JPEG:n03085013 +ILSVRC2012_val_00035955.JPEG:n03527444 +ILSVRC2012_val_00035956.JPEG:n04116512 +ILSVRC2012_val_00035957.JPEG:n02514041 +ILSVRC2012_val_00035958.JPEG:n03627232 +ILSVRC2012_val_00035959.JPEG:n03376595 +ILSVRC2012_val_00035960.JPEG:n04443257 +ILSVRC2012_val_00035961.JPEG:n03095699 +ILSVRC2012_val_00035962.JPEG:n02403003 +ILSVRC2012_val_00035963.JPEG:n04589890 +ILSVRC2012_val_00035964.JPEG:n01910747 +ILSVRC2012_val_00035965.JPEG:n02978881 +ILSVRC2012_val_00035966.JPEG:n02727426 +ILSVRC2012_val_00035967.JPEG:n01985128 +ILSVRC2012_val_00035968.JPEG:n03482405 +ILSVRC2012_val_00035969.JPEG:n02132136 +ILSVRC2012_val_00035970.JPEG:n04277352 +ILSVRC2012_val_00035971.JPEG:n13133613 +ILSVRC2012_val_00035972.JPEG:n02033041 +ILSVRC2012_val_00035973.JPEG:n02100877 +ILSVRC2012_val_00035974.JPEG:n01806143 +ILSVRC2012_val_00035975.JPEG:n03733805 +ILSVRC2012_val_00035976.JPEG:n01748264 +ILSVRC2012_val_00035977.JPEG:n02483362 +ILSVRC2012_val_00035978.JPEG:n03776460 +ILSVRC2012_val_00035979.JPEG:n02105412 +ILSVRC2012_val_00035980.JPEG:n03887697 +ILSVRC2012_val_00035981.JPEG:n01773157 +ILSVRC2012_val_00035982.JPEG:n02056570 +ILSVRC2012_val_00035983.JPEG:n02808440 +ILSVRC2012_val_00035984.JPEG:n02007558 +ILSVRC2012_val_00035985.JPEG:n04146614 +ILSVRC2012_val_00035986.JPEG:n02097130 +ILSVRC2012_val_00035987.JPEG:n03888605 +ILSVRC2012_val_00035988.JPEG:n02412080 +ILSVRC2012_val_00035989.JPEG:n01806567 +ILSVRC2012_val_00035990.JPEG:n02457408 +ILSVRC2012_val_00035991.JPEG:n03935335 +ILSVRC2012_val_00035992.JPEG:n03775071 +ILSVRC2012_val_00035993.JPEG:n07697313 +ILSVRC2012_val_00035994.JPEG:n01774750 +ILSVRC2012_val_00035995.JPEG:n07873807 +ILSVRC2012_val_00035996.JPEG:n07749582 +ILSVRC2012_val_00035997.JPEG:n02091134 +ILSVRC2012_val_00035998.JPEG:n02871525 +ILSVRC2012_val_00035999.JPEG:n02117135 +ILSVRC2012_val_00036000.JPEG:n03657121 +ILSVRC2012_val_00036001.JPEG:n03661043 +ILSVRC2012_val_00036002.JPEG:n02088632 +ILSVRC2012_val_00036003.JPEG:n03776460 +ILSVRC2012_val_00036004.JPEG:n02120505 +ILSVRC2012_val_00036005.JPEG:n02165456 +ILSVRC2012_val_00036006.JPEG:n03089624 +ILSVRC2012_val_00036007.JPEG:n03485794 +ILSVRC2012_val_00036008.JPEG:n01534433 +ILSVRC2012_val_00036009.JPEG:n02835271 +ILSVRC2012_val_00036010.JPEG:n03240683 +ILSVRC2012_val_00036011.JPEG:n04251144 +ILSVRC2012_val_00036012.JPEG:n02086910 +ILSVRC2012_val_00036013.JPEG:n03447447 +ILSVRC2012_val_00036014.JPEG:n04200800 +ILSVRC2012_val_00036015.JPEG:n01582220 +ILSVRC2012_val_00036016.JPEG:n02655020 +ILSVRC2012_val_00036017.JPEG:n04458633 +ILSVRC2012_val_00036018.JPEG:n04371430 +ILSVRC2012_val_00036019.JPEG:n02097047 +ILSVRC2012_val_00036020.JPEG:n03970156 +ILSVRC2012_val_00036021.JPEG:n04418357 +ILSVRC2012_val_00036022.JPEG:n04243546 +ILSVRC2012_val_00036023.JPEG:n02098413 +ILSVRC2012_val_00036024.JPEG:n02992529 +ILSVRC2012_val_00036025.JPEG:n03384352 +ILSVRC2012_val_00036026.JPEG:n02640242 +ILSVRC2012_val_00036027.JPEG:n02894605 +ILSVRC2012_val_00036028.JPEG:n03920288 +ILSVRC2012_val_00036029.JPEG:n03250847 +ILSVRC2012_val_00036030.JPEG:n02607072 +ILSVRC2012_val_00036031.JPEG:n04326547 +ILSVRC2012_val_00036032.JPEG:n04485082 +ILSVRC2012_val_00036033.JPEG:n03868863 +ILSVRC2012_val_00036034.JPEG:n09472597 +ILSVRC2012_val_00036035.JPEG:n02027492 +ILSVRC2012_val_00036036.JPEG:n02692877 +ILSVRC2012_val_00036037.JPEG:n03388549 +ILSVRC2012_val_00036038.JPEG:n03874599 +ILSVRC2012_val_00036039.JPEG:n02096051 +ILSVRC2012_val_00036040.JPEG:n01847000 +ILSVRC2012_val_00036041.JPEG:n02328150 +ILSVRC2012_val_00036042.JPEG:n01534433 +ILSVRC2012_val_00036043.JPEG:n02910353 +ILSVRC2012_val_00036044.JPEG:n01829413 +ILSVRC2012_val_00036045.JPEG:n02107142 +ILSVRC2012_val_00036046.JPEG:n03977966 +ILSVRC2012_val_00036047.JPEG:n02090622 +ILSVRC2012_val_00036048.JPEG:n03444034 +ILSVRC2012_val_00036049.JPEG:n04418357 +ILSVRC2012_val_00036050.JPEG:n04254680 +ILSVRC2012_val_00036051.JPEG:n02692877 +ILSVRC2012_val_00036052.JPEG:n02002724 +ILSVRC2012_val_00036053.JPEG:n03535780 +ILSVRC2012_val_00036054.JPEG:n02108551 +ILSVRC2012_val_00036055.JPEG:n02112350 +ILSVRC2012_val_00036056.JPEG:n15075141 +ILSVRC2012_val_00036057.JPEG:n04141975 +ILSVRC2012_val_00036058.JPEG:n04507155 +ILSVRC2012_val_00036059.JPEG:n04509417 +ILSVRC2012_val_00036060.JPEG:n11939491 +ILSVRC2012_val_00036061.JPEG:n02112706 +ILSVRC2012_val_00036062.JPEG:n02110627 +ILSVRC2012_val_00036063.JPEG:n03125729 +ILSVRC2012_val_00036064.JPEG:n03680355 +ILSVRC2012_val_00036065.JPEG:n01644373 +ILSVRC2012_val_00036066.JPEG:n01644373 +ILSVRC2012_val_00036067.JPEG:n01756291 +ILSVRC2012_val_00036068.JPEG:n01753488 +ILSVRC2012_val_00036069.JPEG:n02098105 +ILSVRC2012_val_00036070.JPEG:n02342885 +ILSVRC2012_val_00036071.JPEG:n03759954 +ILSVRC2012_val_00036072.JPEG:n02110958 +ILSVRC2012_val_00036073.JPEG:n02797295 +ILSVRC2012_val_00036074.JPEG:n02006656 +ILSVRC2012_val_00036075.JPEG:n02111500 +ILSVRC2012_val_00036076.JPEG:n04033901 +ILSVRC2012_val_00036077.JPEG:n01784675 +ILSVRC2012_val_00036078.JPEG:n04277352 +ILSVRC2012_val_00036079.JPEG:n02489166 +ILSVRC2012_val_00036080.JPEG:n02481823 +ILSVRC2012_val_00036081.JPEG:n02398521 +ILSVRC2012_val_00036082.JPEG:n01739381 +ILSVRC2012_val_00036083.JPEG:n02823428 +ILSVRC2012_val_00036084.JPEG:n02939185 +ILSVRC2012_val_00036085.JPEG:n12985857 +ILSVRC2012_val_00036086.JPEG:n04275548 +ILSVRC2012_val_00036087.JPEG:n04127249 +ILSVRC2012_val_00036088.JPEG:n02087394 +ILSVRC2012_val_00036089.JPEG:n03920288 +ILSVRC2012_val_00036090.JPEG:n04482393 +ILSVRC2012_val_00036091.JPEG:n03100240 +ILSVRC2012_val_00036092.JPEG:n03000684 +ILSVRC2012_val_00036093.JPEG:n07248320 +ILSVRC2012_val_00036094.JPEG:n02454379 +ILSVRC2012_val_00036095.JPEG:n02361337 +ILSVRC2012_val_00036096.JPEG:n03218198 +ILSVRC2012_val_00036097.JPEG:n02106030 +ILSVRC2012_val_00036098.JPEG:n03544143 +ILSVRC2012_val_00036099.JPEG:n04456115 +ILSVRC2012_val_00036100.JPEG:n02165105 +ILSVRC2012_val_00036101.JPEG:n03188531 +ILSVRC2012_val_00036102.JPEG:n01641577 +ILSVRC2012_val_00036103.JPEG:n07742313 +ILSVRC2012_val_00036104.JPEG:n03761084 +ILSVRC2012_val_00036105.JPEG:n01518878 +ILSVRC2012_val_00036106.JPEG:n04376876 +ILSVRC2012_val_00036107.JPEG:n03782006 +ILSVRC2012_val_00036108.JPEG:n02422699 +ILSVRC2012_val_00036109.JPEG:n01773797 +ILSVRC2012_val_00036110.JPEG:n02106550 +ILSVRC2012_val_00036111.JPEG:n04590129 +ILSVRC2012_val_00036112.JPEG:n03902125 +ILSVRC2012_val_00036113.JPEG:n02823750 +ILSVRC2012_val_00036114.JPEG:n03393912 +ILSVRC2012_val_00036115.JPEG:n04090263 +ILSVRC2012_val_00036116.JPEG:n01737021 +ILSVRC2012_val_00036117.JPEG:n02129165 +ILSVRC2012_val_00036118.JPEG:n01498041 +ILSVRC2012_val_00036119.JPEG:n03792782 +ILSVRC2012_val_00036120.JPEG:n02966687 +ILSVRC2012_val_00036121.JPEG:n02504458 +ILSVRC2012_val_00036122.JPEG:n03838899 +ILSVRC2012_val_00036123.JPEG:n01689811 +ILSVRC2012_val_00036124.JPEG:n04347754 +ILSVRC2012_val_00036125.JPEG:n01608432 +ILSVRC2012_val_00036126.JPEG:n01817953 +ILSVRC2012_val_00036127.JPEG:n02536864 +ILSVRC2012_val_00036128.JPEG:n01729977 +ILSVRC2012_val_00036129.JPEG:n02096437 +ILSVRC2012_val_00036130.JPEG:n03924679 +ILSVRC2012_val_00036131.JPEG:n02096437 +ILSVRC2012_val_00036132.JPEG:n01798484 +ILSVRC2012_val_00036133.JPEG:n02869837 +ILSVRC2012_val_00036134.JPEG:n04336792 +ILSVRC2012_val_00036135.JPEG:n03485407 +ILSVRC2012_val_00036136.JPEG:n03868863 +ILSVRC2012_val_00036137.JPEG:n04376876 +ILSVRC2012_val_00036138.JPEG:n03602883 +ILSVRC2012_val_00036139.JPEG:n02128925 +ILSVRC2012_val_00036140.JPEG:n02102973 +ILSVRC2012_val_00036141.JPEG:n02447366 +ILSVRC2012_val_00036142.JPEG:n07716358 +ILSVRC2012_val_00036143.JPEG:n03857828 +ILSVRC2012_val_00036144.JPEG:n04517823 +ILSVRC2012_val_00036145.JPEG:n03837869 +ILSVRC2012_val_00036146.JPEG:n07749582 +ILSVRC2012_val_00036147.JPEG:n02105162 +ILSVRC2012_val_00036148.JPEG:n02281787 +ILSVRC2012_val_00036149.JPEG:n02769748 +ILSVRC2012_val_00036150.JPEG:n02085620 +ILSVRC2012_val_00036151.JPEG:n01751748 +ILSVRC2012_val_00036152.JPEG:n02093647 +ILSVRC2012_val_00036153.JPEG:n04423845 +ILSVRC2012_val_00036154.JPEG:n02488702 +ILSVRC2012_val_00036155.JPEG:n03485794 +ILSVRC2012_val_00036156.JPEG:n03908714 +ILSVRC2012_val_00036157.JPEG:n01498041 +ILSVRC2012_val_00036158.JPEG:n02231487 +ILSVRC2012_val_00036159.JPEG:n02108551 +ILSVRC2012_val_00036160.JPEG:n03179701 +ILSVRC2012_val_00036161.JPEG:n02786058 +ILSVRC2012_val_00036162.JPEG:n01855032 +ILSVRC2012_val_00036163.JPEG:n04147183 +ILSVRC2012_val_00036164.JPEG:n04254680 +ILSVRC2012_val_00036165.JPEG:n04557648 +ILSVRC2012_val_00036166.JPEG:n01728572 +ILSVRC2012_val_00036167.JPEG:n04325704 +ILSVRC2012_val_00036168.JPEG:n07860988 +ILSVRC2012_val_00036169.JPEG:n01847000 +ILSVRC2012_val_00036170.JPEG:n13044778 +ILSVRC2012_val_00036171.JPEG:n03445777 +ILSVRC2012_val_00036172.JPEG:n03447447 +ILSVRC2012_val_00036173.JPEG:n02169497 +ILSVRC2012_val_00036174.JPEG:n03290653 +ILSVRC2012_val_00036175.JPEG:n03376595 +ILSVRC2012_val_00036176.JPEG:n02094114 +ILSVRC2012_val_00036177.JPEG:n03854065 +ILSVRC2012_val_00036178.JPEG:n02422699 +ILSVRC2012_val_00036179.JPEG:n01796340 +ILSVRC2012_val_00036180.JPEG:n03459775 +ILSVRC2012_val_00036181.JPEG:n02091244 +ILSVRC2012_val_00036182.JPEG:n04399382 +ILSVRC2012_val_00036183.JPEG:n03476684 +ILSVRC2012_val_00036184.JPEG:n02951585 +ILSVRC2012_val_00036185.JPEG:n03207941 +ILSVRC2012_val_00036186.JPEG:n02174001 +ILSVRC2012_val_00036187.JPEG:n03445777 +ILSVRC2012_val_00036188.JPEG:n01950731 +ILSVRC2012_val_00036189.JPEG:n04562935 +ILSVRC2012_val_00036190.JPEG:n01728572 +ILSVRC2012_val_00036191.JPEG:n02089973 +ILSVRC2012_val_00036192.JPEG:n01945685 +ILSVRC2012_val_00036193.JPEG:n02791270 +ILSVRC2012_val_00036194.JPEG:n04090263 +ILSVRC2012_val_00036195.JPEG:n01665541 +ILSVRC2012_val_00036196.JPEG:n02264363 +ILSVRC2012_val_00036197.JPEG:n04228054 +ILSVRC2012_val_00036198.JPEG:n03345487 +ILSVRC2012_val_00036199.JPEG:n03947888 +ILSVRC2012_val_00036200.JPEG:n01944390 +ILSVRC2012_val_00036201.JPEG:n04153751 +ILSVRC2012_val_00036202.JPEG:n01664065 +ILSVRC2012_val_00036203.JPEG:n03223299 +ILSVRC2012_val_00036204.JPEG:n02930766 +ILSVRC2012_val_00036205.JPEG:n04404412 +ILSVRC2012_val_00036206.JPEG:n03992509 +ILSVRC2012_val_00036207.JPEG:n01877812 +ILSVRC2012_val_00036208.JPEG:n02977058 +ILSVRC2012_val_00036209.JPEG:n09835506 +ILSVRC2012_val_00036210.JPEG:n12267677 +ILSVRC2012_val_00036211.JPEG:n03127747 +ILSVRC2012_val_00036212.JPEG:n01980166 +ILSVRC2012_val_00036213.JPEG:n09835506 +ILSVRC2012_val_00036214.JPEG:n07753113 +ILSVRC2012_val_00036215.JPEG:n02860847 +ILSVRC2012_val_00036216.JPEG:n02840245 +ILSVRC2012_val_00036217.JPEG:n01748264 +ILSVRC2012_val_00036218.JPEG:n03891251 +ILSVRC2012_val_00036219.JPEG:n02484975 +ILSVRC2012_val_00036220.JPEG:n02095314 +ILSVRC2012_val_00036221.JPEG:n03063689 +ILSVRC2012_val_00036222.JPEG:n04372370 +ILSVRC2012_val_00036223.JPEG:n11879895 +ILSVRC2012_val_00036224.JPEG:n02447366 +ILSVRC2012_val_00036225.JPEG:n01795545 +ILSVRC2012_val_00036226.JPEG:n03201208 +ILSVRC2012_val_00036227.JPEG:n01797886 +ILSVRC2012_val_00036228.JPEG:n04548362 +ILSVRC2012_val_00036229.JPEG:n03028079 +ILSVRC2012_val_00036230.JPEG:n03201208 +ILSVRC2012_val_00036231.JPEG:n02109047 +ILSVRC2012_val_00036232.JPEG:n03804744 +ILSVRC2012_val_00036233.JPEG:n03417042 +ILSVRC2012_val_00036234.JPEG:n02111500 +ILSVRC2012_val_00036235.JPEG:n02109047 +ILSVRC2012_val_00036236.JPEG:n02415577 +ILSVRC2012_val_00036237.JPEG:n04456115 +ILSVRC2012_val_00036238.JPEG:n02486410 +ILSVRC2012_val_00036239.JPEG:n03976657 +ILSVRC2012_val_00036240.JPEG:n02109525 +ILSVRC2012_val_00036241.JPEG:n03602883 +ILSVRC2012_val_00036242.JPEG:n03937543 +ILSVRC2012_val_00036243.JPEG:n02492660 +ILSVRC2012_val_00036244.JPEG:n02127052 +ILSVRC2012_val_00036245.JPEG:n02641379 +ILSVRC2012_val_00036246.JPEG:n03146219 +ILSVRC2012_val_00036247.JPEG:n02091635 +ILSVRC2012_val_00036248.JPEG:n02110185 +ILSVRC2012_val_00036249.JPEG:n04389033 +ILSVRC2012_val_00036250.JPEG:n04330267 +ILSVRC2012_val_00036251.JPEG:n02165456 +ILSVRC2012_val_00036252.JPEG:n04152593 +ILSVRC2012_val_00036253.JPEG:n04548362 +ILSVRC2012_val_00036254.JPEG:n02094433 +ILSVRC2012_val_00036255.JPEG:n04372370 +ILSVRC2012_val_00036256.JPEG:n03208938 +ILSVRC2012_val_00036257.JPEG:n02356798 +ILSVRC2012_val_00036258.JPEG:n02666196 +ILSVRC2012_val_00036259.JPEG:n02279972 +ILSVRC2012_val_00036260.JPEG:n03661043 +ILSVRC2012_val_00036261.JPEG:n03187595 +ILSVRC2012_val_00036262.JPEG:n03131574 +ILSVRC2012_val_00036263.JPEG:n07742313 +ILSVRC2012_val_00036264.JPEG:n02104029 +ILSVRC2012_val_00036265.JPEG:n02172182 +ILSVRC2012_val_00036266.JPEG:n02090622 +ILSVRC2012_val_00036267.JPEG:n02085782 +ILSVRC2012_val_00036268.JPEG:n02123159 +ILSVRC2012_val_00036269.JPEG:n02105855 +ILSVRC2012_val_00036270.JPEG:n02422106 +ILSVRC2012_val_00036271.JPEG:n01667114 +ILSVRC2012_val_00036272.JPEG:n01943899 +ILSVRC2012_val_00036273.JPEG:n03692522 +ILSVRC2012_val_00036274.JPEG:n03788195 +ILSVRC2012_val_00036275.JPEG:n07718472 +ILSVRC2012_val_00036276.JPEG:n03146219 +ILSVRC2012_val_00036277.JPEG:n04553703 +ILSVRC2012_val_00036278.JPEG:n09472597 +ILSVRC2012_val_00036279.JPEG:n04447861 +ILSVRC2012_val_00036280.JPEG:n02790996 +ILSVRC2012_val_00036281.JPEG:n03673027 +ILSVRC2012_val_00036282.JPEG:n02102040 +ILSVRC2012_val_00036283.JPEG:n07565083 +ILSVRC2012_val_00036284.JPEG:n01532829 +ILSVRC2012_val_00036285.JPEG:n02276258 +ILSVRC2012_val_00036286.JPEG:n04141327 +ILSVRC2012_val_00036287.JPEG:n01817953 +ILSVRC2012_val_00036288.JPEG:n04118538 +ILSVRC2012_val_00036289.JPEG:n01990800 +ILSVRC2012_val_00036290.JPEG:n02123597 +ILSVRC2012_val_00036291.JPEG:n01751748 +ILSVRC2012_val_00036292.JPEG:n02025239 +ILSVRC2012_val_00036293.JPEG:n01644373 +ILSVRC2012_val_00036294.JPEG:n03355925 +ILSVRC2012_val_00036295.JPEG:n02177972 +ILSVRC2012_val_00036296.JPEG:n04286575 +ILSVRC2012_val_00036297.JPEG:n04009552 +ILSVRC2012_val_00036298.JPEG:n03899768 +ILSVRC2012_val_00036299.JPEG:n03857828 +ILSVRC2012_val_00036300.JPEG:n04613696 +ILSVRC2012_val_00036301.JPEG:n02120079 +ILSVRC2012_val_00036302.JPEG:n02007558 +ILSVRC2012_val_00036303.JPEG:n04311174 +ILSVRC2012_val_00036304.JPEG:n03594945 +ILSVRC2012_val_00036305.JPEG:n04355338 +ILSVRC2012_val_00036306.JPEG:n03325584 +ILSVRC2012_val_00036307.JPEG:n07590611 +ILSVRC2012_val_00036308.JPEG:n07831146 +ILSVRC2012_val_00036309.JPEG:n03899768 +ILSVRC2012_val_00036310.JPEG:n02165105 +ILSVRC2012_val_00036311.JPEG:n06359193 +ILSVRC2012_val_00036312.JPEG:n06874185 +ILSVRC2012_val_00036313.JPEG:n03657121 +ILSVRC2012_val_00036314.JPEG:n02056570 +ILSVRC2012_val_00036315.JPEG:n09428293 +ILSVRC2012_val_00036316.JPEG:n04597913 +ILSVRC2012_val_00036317.JPEG:n02114855 +ILSVRC2012_val_00036318.JPEG:n04548280 +ILSVRC2012_val_00036319.JPEG:n03065424 +ILSVRC2012_val_00036320.JPEG:n01986214 +ILSVRC2012_val_00036321.JPEG:n03623198 +ILSVRC2012_val_00036322.JPEG:n04485082 +ILSVRC2012_val_00036323.JPEG:n03888605 +ILSVRC2012_val_00036324.JPEG:n02114855 +ILSVRC2012_val_00036325.JPEG:n02917067 +ILSVRC2012_val_00036326.JPEG:n04067472 +ILSVRC2012_val_00036327.JPEG:n03457902 +ILSVRC2012_val_00036328.JPEG:n03775071 +ILSVRC2012_val_00036329.JPEG:n07579787 +ILSVRC2012_val_00036330.JPEG:n02509815 +ILSVRC2012_val_00036331.JPEG:n04458633 +ILSVRC2012_val_00036332.JPEG:n03347037 +ILSVRC2012_val_00036333.JPEG:n02098105 +ILSVRC2012_val_00036334.JPEG:n12985857 +ILSVRC2012_val_00036335.JPEG:n03691459 +ILSVRC2012_val_00036336.JPEG:n04525305 +ILSVRC2012_val_00036337.JPEG:n01817953 +ILSVRC2012_val_00036338.JPEG:n03393912 +ILSVRC2012_val_00036339.JPEG:n04251144 +ILSVRC2012_val_00036340.JPEG:n02088364 +ILSVRC2012_val_00036341.JPEG:n02526121 +ILSVRC2012_val_00036342.JPEG:n02444819 +ILSVRC2012_val_00036343.JPEG:n02088238 +ILSVRC2012_val_00036344.JPEG:n02051845 +ILSVRC2012_val_00036345.JPEG:n01667114 +ILSVRC2012_val_00036346.JPEG:n04487394 +ILSVRC2012_val_00036347.JPEG:n04125021 +ILSVRC2012_val_00036348.JPEG:n02883205 +ILSVRC2012_val_00036349.JPEG:n04162706 +ILSVRC2012_val_00036350.JPEG:n02085936 +ILSVRC2012_val_00036351.JPEG:n02807133 +ILSVRC2012_val_00036352.JPEG:n02978881 +ILSVRC2012_val_00036353.JPEG:n04350905 +ILSVRC2012_val_00036354.JPEG:n01843383 +ILSVRC2012_val_00036355.JPEG:n02906734 +ILSVRC2012_val_00036356.JPEG:n01608432 +ILSVRC2012_val_00036357.JPEG:n02950826 +ILSVRC2012_val_00036358.JPEG:n04131690 +ILSVRC2012_val_00036359.JPEG:n02823428 +ILSVRC2012_val_00036360.JPEG:n02106030 +ILSVRC2012_val_00036361.JPEG:n01818515 +ILSVRC2012_val_00036362.JPEG:n03840681 +ILSVRC2012_val_00036363.JPEG:n03443371 +ILSVRC2012_val_00036364.JPEG:n03447447 +ILSVRC2012_val_00036365.JPEG:n02492660 +ILSVRC2012_val_00036366.JPEG:n11879895 +ILSVRC2012_val_00036367.JPEG:n02981792 +ILSVRC2012_val_00036368.JPEG:n01514668 +ILSVRC2012_val_00036369.JPEG:n02701002 +ILSVRC2012_val_00036370.JPEG:n04192698 +ILSVRC2012_val_00036371.JPEG:n02106030 +ILSVRC2012_val_00036372.JPEG:n07717410 +ILSVRC2012_val_00036373.JPEG:n03492542 +ILSVRC2012_val_00036374.JPEG:n06794110 +ILSVRC2012_val_00036375.JPEG:n03977966 +ILSVRC2012_val_00036376.JPEG:n04008634 +ILSVRC2012_val_00036377.JPEG:n07768694 +ILSVRC2012_val_00036378.JPEG:n04515003 +ILSVRC2012_val_00036379.JPEG:n02111889 +ILSVRC2012_val_00036380.JPEG:n02363005 +ILSVRC2012_val_00036381.JPEG:n01930112 +ILSVRC2012_val_00036382.JPEG:n04447861 +ILSVRC2012_val_00036383.JPEG:n07684084 +ILSVRC2012_val_00036384.JPEG:n01883070 +ILSVRC2012_val_00036385.JPEG:n03250847 +ILSVRC2012_val_00036386.JPEG:n02825657 +ILSVRC2012_val_00036387.JPEG:n03793489 +ILSVRC2012_val_00036388.JPEG:n01616318 +ILSVRC2012_val_00036389.JPEG:n02110341 +ILSVRC2012_val_00036390.JPEG:n06596364 +ILSVRC2012_val_00036391.JPEG:n04456115 +ILSVRC2012_val_00036392.JPEG:n01749939 +ILSVRC2012_val_00036393.JPEG:n03180011 +ILSVRC2012_val_00036394.JPEG:n02690373 +ILSVRC2012_val_00036395.JPEG:n02088094 +ILSVRC2012_val_00036396.JPEG:n01984695 +ILSVRC2012_val_00036397.JPEG:n02493793 +ILSVRC2012_val_00036398.JPEG:n09428293 +ILSVRC2012_val_00036399.JPEG:n03888605 +ILSVRC2012_val_00036400.JPEG:n09229709 +ILSVRC2012_val_00036401.JPEG:n02128757 +ILSVRC2012_val_00036402.JPEG:n04239074 +ILSVRC2012_val_00036403.JPEG:n04040759 +ILSVRC2012_val_00036404.JPEG:n03062245 +ILSVRC2012_val_00036405.JPEG:n02168699 +ILSVRC2012_val_00036406.JPEG:n02977058 +ILSVRC2012_val_00036407.JPEG:n01773157 +ILSVRC2012_val_00036408.JPEG:n02101388 +ILSVRC2012_val_00036409.JPEG:n03459775 +ILSVRC2012_val_00036410.JPEG:n04532106 +ILSVRC2012_val_00036411.JPEG:n04026417 +ILSVRC2012_val_00036412.JPEG:n02870880 +ILSVRC2012_val_00036413.JPEG:n04179913 +ILSVRC2012_val_00036414.JPEG:n02115913 +ILSVRC2012_val_00036415.JPEG:n04525038 +ILSVRC2012_val_00036416.JPEG:n11939491 +ILSVRC2012_val_00036417.JPEG:n02165105 +ILSVRC2012_val_00036418.JPEG:n04258138 +ILSVRC2012_val_00036419.JPEG:n09472597 +ILSVRC2012_val_00036420.JPEG:n01491361 +ILSVRC2012_val_00036421.JPEG:n03706229 +ILSVRC2012_val_00036422.JPEG:n03937543 +ILSVRC2012_val_00036423.JPEG:n01855672 +ILSVRC2012_val_00036424.JPEG:n03673027 +ILSVRC2012_val_00036425.JPEG:n02443484 +ILSVRC2012_val_00036426.JPEG:n03706229 +ILSVRC2012_val_00036427.JPEG:n04149813 +ILSVRC2012_val_00036428.JPEG:n03599486 +ILSVRC2012_val_00036429.JPEG:n03272562 +ILSVRC2012_val_00036430.JPEG:n01704323 +ILSVRC2012_val_00036431.JPEG:n01537544 +ILSVRC2012_val_00036432.JPEG:n03424325 +ILSVRC2012_val_00036433.JPEG:n02085782 +ILSVRC2012_val_00036434.JPEG:n02190166 +ILSVRC2012_val_00036435.JPEG:n04592741 +ILSVRC2012_val_00036436.JPEG:n02504458 +ILSVRC2012_val_00036437.JPEG:n04086273 +ILSVRC2012_val_00036438.JPEG:n07754684 +ILSVRC2012_val_00036439.JPEG:n02443484 +ILSVRC2012_val_00036440.JPEG:n02086910 +ILSVRC2012_val_00036441.JPEG:n01756291 +ILSVRC2012_val_00036442.JPEG:n01873310 +ILSVRC2012_val_00036443.JPEG:n02096437 +ILSVRC2012_val_00036444.JPEG:n02870880 +ILSVRC2012_val_00036445.JPEG:n02106166 +ILSVRC2012_val_00036446.JPEG:n07613480 +ILSVRC2012_val_00036447.JPEG:n03018349 +ILSVRC2012_val_00036448.JPEG:n03447721 +ILSVRC2012_val_00036449.JPEG:n04335435 +ILSVRC2012_val_00036450.JPEG:n02114855 +ILSVRC2012_val_00036451.JPEG:n07760859 +ILSVRC2012_val_00036452.JPEG:n03825788 +ILSVRC2012_val_00036453.JPEG:n02107142 +ILSVRC2012_val_00036454.JPEG:n02095570 +ILSVRC2012_val_00036455.JPEG:n01697457 +ILSVRC2012_val_00036456.JPEG:n03837869 +ILSVRC2012_val_00036457.JPEG:n02018795 +ILSVRC2012_val_00036458.JPEG:n02113624 +ILSVRC2012_val_00036459.JPEG:n03781244 +ILSVRC2012_val_00036460.JPEG:n03942813 +ILSVRC2012_val_00036461.JPEG:n02445715 +ILSVRC2012_val_00036462.JPEG:n02111129 +ILSVRC2012_val_00036463.JPEG:n04372370 +ILSVRC2012_val_00036464.JPEG:n02115641 +ILSVRC2012_val_00036465.JPEG:n07802026 +ILSVRC2012_val_00036466.JPEG:n02137549 +ILSVRC2012_val_00036467.JPEG:n02099429 +ILSVRC2012_val_00036468.JPEG:n03998194 +ILSVRC2012_val_00036469.JPEG:n04162706 +ILSVRC2012_val_00036470.JPEG:n03208938 +ILSVRC2012_val_00036471.JPEG:n02486410 +ILSVRC2012_val_00036472.JPEG:n02536864 +ILSVRC2012_val_00036473.JPEG:n02437616 +ILSVRC2012_val_00036474.JPEG:n02128757 +ILSVRC2012_val_00036475.JPEG:n04604644 +ILSVRC2012_val_00036476.JPEG:n03016953 +ILSVRC2012_val_00036477.JPEG:n04404412 +ILSVRC2012_val_00036478.JPEG:n02096585 +ILSVRC2012_val_00036479.JPEG:n01494475 +ILSVRC2012_val_00036480.JPEG:n03657121 +ILSVRC2012_val_00036481.JPEG:n04259630 +ILSVRC2012_val_00036482.JPEG:n04423845 +ILSVRC2012_val_00036483.JPEG:n03388549 +ILSVRC2012_val_00036484.JPEG:n02640242 +ILSVRC2012_val_00036485.JPEG:n02988304 +ILSVRC2012_val_00036486.JPEG:n02165456 +ILSVRC2012_val_00036487.JPEG:n03924679 +ILSVRC2012_val_00036488.JPEG:n04086273 +ILSVRC2012_val_00036489.JPEG:n02492660 +ILSVRC2012_val_00036490.JPEG:n02113624 +ILSVRC2012_val_00036491.JPEG:n02093859 +ILSVRC2012_val_00036492.JPEG:n02089867 +ILSVRC2012_val_00036493.JPEG:n04192698 +ILSVRC2012_val_00036494.JPEG:n01944390 +ILSVRC2012_val_00036495.JPEG:n01632777 +ILSVRC2012_val_00036496.JPEG:n02966687 +ILSVRC2012_val_00036497.JPEG:n02107908 +ILSVRC2012_val_00036498.JPEG:n02098286 +ILSVRC2012_val_00036499.JPEG:n07831146 +ILSVRC2012_val_00036500.JPEG:n02007558 +ILSVRC2012_val_00036501.JPEG:n04536866 +ILSVRC2012_val_00036502.JPEG:n02808304 +ILSVRC2012_val_00036503.JPEG:n07718472 +ILSVRC2012_val_00036504.JPEG:n03930630 +ILSVRC2012_val_00036505.JPEG:n07754684 +ILSVRC2012_val_00036506.JPEG:n01774750 +ILSVRC2012_val_00036507.JPEG:n03980874 +ILSVRC2012_val_00036508.JPEG:n03384352 +ILSVRC2012_val_00036509.JPEG:n02104029 +ILSVRC2012_val_00036510.JPEG:n02769748 +ILSVRC2012_val_00036511.JPEG:n02058221 +ILSVRC2012_val_00036512.JPEG:n01695060 +ILSVRC2012_val_00036513.JPEG:n03929660 +ILSVRC2012_val_00036514.JPEG:n13040303 +ILSVRC2012_val_00036515.JPEG:n03089624 +ILSVRC2012_val_00036516.JPEG:n04443257 +ILSVRC2012_val_00036517.JPEG:n04428191 +ILSVRC2012_val_00036518.JPEG:n03775546 +ILSVRC2012_val_00036519.JPEG:n04517823 +ILSVRC2012_val_00036520.JPEG:n01945685 +ILSVRC2012_val_00036521.JPEG:n03216828 +ILSVRC2012_val_00036522.JPEG:n02965783 +ILSVRC2012_val_00036523.JPEG:n02088466 +ILSVRC2012_val_00036524.JPEG:n04133789 +ILSVRC2012_val_00036525.JPEG:n03838899 +ILSVRC2012_val_00036526.JPEG:n02123597 +ILSVRC2012_val_00036527.JPEG:n02128385 +ILSVRC2012_val_00036528.JPEG:n02486410 +ILSVRC2012_val_00036529.JPEG:n03124170 +ILSVRC2012_val_00036530.JPEG:n03530642 +ILSVRC2012_val_00036531.JPEG:n02500267 +ILSVRC2012_val_00036532.JPEG:n12768682 +ILSVRC2012_val_00036533.JPEG:n02128385 +ILSVRC2012_val_00036534.JPEG:n01592084 +ILSVRC2012_val_00036535.JPEG:n02526121 +ILSVRC2012_val_00036536.JPEG:n04356056 +ILSVRC2012_val_00036537.JPEG:n02137549 +ILSVRC2012_val_00036538.JPEG:n03854065 +ILSVRC2012_val_00036539.JPEG:n07684084 +ILSVRC2012_val_00036540.JPEG:n01855032 +ILSVRC2012_val_00036541.JPEG:n02992211 +ILSVRC2012_val_00036542.JPEG:n02484975 +ILSVRC2012_val_00036543.JPEG:n02106030 +ILSVRC2012_val_00036544.JPEG:n09421951 +ILSVRC2012_val_00036545.JPEG:n04367480 +ILSVRC2012_val_00036546.JPEG:n09256479 +ILSVRC2012_val_00036547.JPEG:n02119022 +ILSVRC2012_val_00036548.JPEG:n02493509 +ILSVRC2012_val_00036549.JPEG:n03803284 +ILSVRC2012_val_00036550.JPEG:n01685808 +ILSVRC2012_val_00036551.JPEG:n07697537 +ILSVRC2012_val_00036552.JPEG:n01807496 +ILSVRC2012_val_00036553.JPEG:n03733281 +ILSVRC2012_val_00036554.JPEG:n03417042 +ILSVRC2012_val_00036555.JPEG:n02219486 +ILSVRC2012_val_00036556.JPEG:n09229709 +ILSVRC2012_val_00036557.JPEG:n02526121 +ILSVRC2012_val_00036558.JPEG:n03908714 +ILSVRC2012_val_00036559.JPEG:n04204347 +ILSVRC2012_val_00036560.JPEG:n03527444 +ILSVRC2012_val_00036561.JPEG:n01740131 +ILSVRC2012_val_00036562.JPEG:n02492035 +ILSVRC2012_val_00036563.JPEG:n02094258 +ILSVRC2012_val_00036564.JPEG:n03769881 +ILSVRC2012_val_00036565.JPEG:n03026506 +ILSVRC2012_val_00036566.JPEG:n02804414 +ILSVRC2012_val_00036567.JPEG:n02489166 +ILSVRC2012_val_00036568.JPEG:n02883205 +ILSVRC2012_val_00036569.JPEG:n03482405 +ILSVRC2012_val_00036570.JPEG:n04366367 +ILSVRC2012_val_00036571.JPEG:n03868863 +ILSVRC2012_val_00036572.JPEG:n03891332 +ILSVRC2012_val_00036573.JPEG:n01797886 +ILSVRC2012_val_00036574.JPEG:n03447447 +ILSVRC2012_val_00036575.JPEG:n04399382 +ILSVRC2012_val_00036576.JPEG:n04146614 +ILSVRC2012_val_00036577.JPEG:n02423022 +ILSVRC2012_val_00036578.JPEG:n02268443 +ILSVRC2012_val_00036579.JPEG:n03250847 +ILSVRC2012_val_00036580.JPEG:n07753592 +ILSVRC2012_val_00036581.JPEG:n01984695 +ILSVRC2012_val_00036582.JPEG:n03709823 +ILSVRC2012_val_00036583.JPEG:n03884397 +ILSVRC2012_val_00036584.JPEG:n03630383 +ILSVRC2012_val_00036585.JPEG:n03814639 +ILSVRC2012_val_00036586.JPEG:n02834397 +ILSVRC2012_val_00036587.JPEG:n01737021 +ILSVRC2012_val_00036588.JPEG:n03786901 +ILSVRC2012_val_00036589.JPEG:n01775062 +ILSVRC2012_val_00036590.JPEG:n01883070 +ILSVRC2012_val_00036591.JPEG:n09428293 +ILSVRC2012_val_00036592.JPEG:n03977966 +ILSVRC2012_val_00036593.JPEG:n07754684 +ILSVRC2012_val_00036594.JPEG:n03384352 +ILSVRC2012_val_00036595.JPEG:n02794156 +ILSVRC2012_val_00036596.JPEG:n13054560 +ILSVRC2012_val_00036597.JPEG:n02132136 +ILSVRC2012_val_00036598.JPEG:n02769748 +ILSVRC2012_val_00036599.JPEG:n07718747 +ILSVRC2012_val_00036600.JPEG:n02950826 +ILSVRC2012_val_00036601.JPEG:n01930112 +ILSVRC2012_val_00036602.JPEG:n02086240 +ILSVRC2012_val_00036603.JPEG:n02125311 +ILSVRC2012_val_00036604.JPEG:n03947888 +ILSVRC2012_val_00036605.JPEG:n02840245 +ILSVRC2012_val_00036606.JPEG:n03220513 +ILSVRC2012_val_00036607.JPEG:n03720891 +ILSVRC2012_val_00036608.JPEG:n02791270 +ILSVRC2012_val_00036609.JPEG:n02802426 +ILSVRC2012_val_00036610.JPEG:n03866082 +ILSVRC2012_val_00036611.JPEG:n03825788 +ILSVRC2012_val_00036612.JPEG:n02487347 +ILSVRC2012_val_00036613.JPEG:n02169497 +ILSVRC2012_val_00036614.JPEG:n02860847 +ILSVRC2012_val_00036615.JPEG:n01728920 +ILSVRC2012_val_00036616.JPEG:n03535780 +ILSVRC2012_val_00036617.JPEG:n03710193 +ILSVRC2012_val_00036618.JPEG:n02091467 +ILSVRC2012_val_00036619.JPEG:n04243546 +ILSVRC2012_val_00036620.JPEG:n01616318 +ILSVRC2012_val_00036621.JPEG:n03942813 +ILSVRC2012_val_00036622.JPEG:n02128757 +ILSVRC2012_val_00036623.JPEG:n04049303 +ILSVRC2012_val_00036624.JPEG:n04417672 +ILSVRC2012_val_00036625.JPEG:n02127052 +ILSVRC2012_val_00036626.JPEG:n03838899 +ILSVRC2012_val_00036627.JPEG:n03729826 +ILSVRC2012_val_00036628.JPEG:n02909870 +ILSVRC2012_val_00036629.JPEG:n09421951 +ILSVRC2012_val_00036630.JPEG:n04515003 +ILSVRC2012_val_00036631.JPEG:n02165105 +ILSVRC2012_val_00036632.JPEG:n03146219 +ILSVRC2012_val_00036633.JPEG:n04423845 +ILSVRC2012_val_00036634.JPEG:n03602883 +ILSVRC2012_val_00036635.JPEG:n01930112 +ILSVRC2012_val_00036636.JPEG:n04208210 +ILSVRC2012_val_00036637.JPEG:n03887697 +ILSVRC2012_val_00036638.JPEG:n03761084 +ILSVRC2012_val_00036639.JPEG:n02268853 +ILSVRC2012_val_00036640.JPEG:n04392985 +ILSVRC2012_val_00036641.JPEG:n03649909 +ILSVRC2012_val_00036642.JPEG:n03447721 +ILSVRC2012_val_00036643.JPEG:n02692877 +ILSVRC2012_val_00036644.JPEG:n12267677 +ILSVRC2012_val_00036645.JPEG:n07715103 +ILSVRC2012_val_00036646.JPEG:n04392985 +ILSVRC2012_val_00036647.JPEG:n04509417 +ILSVRC2012_val_00036648.JPEG:n04041544 +ILSVRC2012_val_00036649.JPEG:n03538406 +ILSVRC2012_val_00036650.JPEG:n01664065 +ILSVRC2012_val_00036651.JPEG:n03179701 +ILSVRC2012_val_00036652.JPEG:n01820546 +ILSVRC2012_val_00036653.JPEG:n04204347 +ILSVRC2012_val_00036654.JPEG:n03929660 +ILSVRC2012_val_00036655.JPEG:n02102973 +ILSVRC2012_val_00036656.JPEG:n03903868 +ILSVRC2012_val_00036657.JPEG:n01742172 +ILSVRC2012_val_00036658.JPEG:n01770081 +ILSVRC2012_val_00036659.JPEG:n03109150 +ILSVRC2012_val_00036660.JPEG:n04273569 +ILSVRC2012_val_00036661.JPEG:n02123045 +ILSVRC2012_val_00036662.JPEG:n07590611 +ILSVRC2012_val_00036663.JPEG:n13037406 +ILSVRC2012_val_00036664.JPEG:n02102177 +ILSVRC2012_val_00036665.JPEG:n03000247 +ILSVRC2012_val_00036666.JPEG:n02410509 +ILSVRC2012_val_00036667.JPEG:n02088632 +ILSVRC2012_val_00036668.JPEG:n07768694 +ILSVRC2012_val_00036669.JPEG:n06785654 +ILSVRC2012_val_00036670.JPEG:n03393912 +ILSVRC2012_val_00036671.JPEG:n03496892 +ILSVRC2012_val_00036672.JPEG:n04275548 +ILSVRC2012_val_00036673.JPEG:n03854065 +ILSVRC2012_val_00036674.JPEG:n04355933 +ILSVRC2012_val_00036675.JPEG:n01807496 +ILSVRC2012_val_00036676.JPEG:n07720875 +ILSVRC2012_val_00036677.JPEG:n04584207 +ILSVRC2012_val_00036678.JPEG:n03792782 +ILSVRC2012_val_00036679.JPEG:n03208938 +ILSVRC2012_val_00036680.JPEG:n02666196 +ILSVRC2012_val_00036681.JPEG:n04149813 +ILSVRC2012_val_00036682.JPEG:n02107683 +ILSVRC2012_val_00036683.JPEG:n04049303 +ILSVRC2012_val_00036684.JPEG:n04118538 +ILSVRC2012_val_00036685.JPEG:n04418357 +ILSVRC2012_val_00036686.JPEG:n02877765 +ILSVRC2012_val_00036687.JPEG:n01883070 +ILSVRC2012_val_00036688.JPEG:n02509815 +ILSVRC2012_val_00036689.JPEG:n10565667 +ILSVRC2012_val_00036690.JPEG:n02497673 +ILSVRC2012_val_00036691.JPEG:n02115913 +ILSVRC2012_val_00036692.JPEG:n03837869 +ILSVRC2012_val_00036693.JPEG:n02190166 +ILSVRC2012_val_00036694.JPEG:n04592741 +ILSVRC2012_val_00036695.JPEG:n04285008 +ILSVRC2012_val_00036696.JPEG:n04606251 +ILSVRC2012_val_00036697.JPEG:n03075370 +ILSVRC2012_val_00036698.JPEG:n04125021 +ILSVRC2012_val_00036699.JPEG:n03796401 +ILSVRC2012_val_00036700.JPEG:n02091134 +ILSVRC2012_val_00036701.JPEG:n03792972 +ILSVRC2012_val_00036702.JPEG:n01824575 +ILSVRC2012_val_00036703.JPEG:n02086079 +ILSVRC2012_val_00036704.JPEG:n01855032 +ILSVRC2012_val_00036705.JPEG:n07742313 +ILSVRC2012_val_00036706.JPEG:n03393912 +ILSVRC2012_val_00036707.JPEG:n03958227 +ILSVRC2012_val_00036708.JPEG:n02137549 +ILSVRC2012_val_00036709.JPEG:n02113978 +ILSVRC2012_val_00036710.JPEG:n02356798 +ILSVRC2012_val_00036711.JPEG:n02808440 +ILSVRC2012_val_00036712.JPEG:n02105412 +ILSVRC2012_val_00036713.JPEG:n01797886 +ILSVRC2012_val_00036714.JPEG:n04204347 +ILSVRC2012_val_00036715.JPEG:n03837869 +ILSVRC2012_val_00036716.JPEG:n02111277 +ILSVRC2012_val_00036717.JPEG:n02777292 +ILSVRC2012_val_00036718.JPEG:n02129604 +ILSVRC2012_val_00036719.JPEG:n07930864 +ILSVRC2012_val_00036720.JPEG:n02489166 +ILSVRC2012_val_00036721.JPEG:n03459775 +ILSVRC2012_val_00036722.JPEG:n01644900 +ILSVRC2012_val_00036723.JPEG:n04149813 +ILSVRC2012_val_00036724.JPEG:n03854065 +ILSVRC2012_val_00036725.JPEG:n03125729 +ILSVRC2012_val_00036726.JPEG:n04141076 +ILSVRC2012_val_00036727.JPEG:n04505470 +ILSVRC2012_val_00036728.JPEG:n02089973 +ILSVRC2012_val_00036729.JPEG:n02172182 +ILSVRC2012_val_00036730.JPEG:n04266014 +ILSVRC2012_val_00036731.JPEG:n04606251 +ILSVRC2012_val_00036732.JPEG:n07768694 +ILSVRC2012_val_00036733.JPEG:n09472597 +ILSVRC2012_val_00036734.JPEG:n02134418 +ILSVRC2012_val_00036735.JPEG:n03623198 +ILSVRC2012_val_00036736.JPEG:n02793495 +ILSVRC2012_val_00036737.JPEG:n01484850 +ILSVRC2012_val_00036738.JPEG:n02276258 +ILSVRC2012_val_00036739.JPEG:n02095889 +ILSVRC2012_val_00036740.JPEG:n03733281 +ILSVRC2012_val_00036741.JPEG:n03535780 +ILSVRC2012_val_00036742.JPEG:n03983396 +ILSVRC2012_val_00036743.JPEG:n02640242 +ILSVRC2012_val_00036744.JPEG:n01818515 +ILSVRC2012_val_00036745.JPEG:n02051845 +ILSVRC2012_val_00036746.JPEG:n03544143 +ILSVRC2012_val_00036747.JPEG:n02092002 +ILSVRC2012_val_00036748.JPEG:n02906734 +ILSVRC2012_val_00036749.JPEG:n01518878 +ILSVRC2012_val_00036750.JPEG:n03769881 +ILSVRC2012_val_00036751.JPEG:n02087046 +ILSVRC2012_val_00036752.JPEG:n03891332 +ILSVRC2012_val_00036753.JPEG:n04392985 +ILSVRC2012_val_00036754.JPEG:n03485794 +ILSVRC2012_val_00036755.JPEG:n03445777 +ILSVRC2012_val_00036756.JPEG:n02115913 +ILSVRC2012_val_00036757.JPEG:n02321529 +ILSVRC2012_val_00036758.JPEG:n03633091 +ILSVRC2012_val_00036759.JPEG:n01984695 +ILSVRC2012_val_00036760.JPEG:n04590129 +ILSVRC2012_val_00036761.JPEG:n02268443 +ILSVRC2012_val_00036762.JPEG:n02676566 +ILSVRC2012_val_00036763.JPEG:n02134084 +ILSVRC2012_val_00036764.JPEG:n03658185 +ILSVRC2012_val_00036765.JPEG:n02091134 +ILSVRC2012_val_00036766.JPEG:n03733805 +ILSVRC2012_val_00036767.JPEG:n02488702 +ILSVRC2012_val_00036768.JPEG:n02869837 +ILSVRC2012_val_00036769.JPEG:n02640242 +ILSVRC2012_val_00036770.JPEG:n03160309 +ILSVRC2012_val_00036771.JPEG:n02443484 +ILSVRC2012_val_00036772.JPEG:n02441942 +ILSVRC2012_val_00036773.JPEG:n01775062 +ILSVRC2012_val_00036774.JPEG:n02825657 +ILSVRC2012_val_00036775.JPEG:n12144580 +ILSVRC2012_val_00036776.JPEG:n04591713 +ILSVRC2012_val_00036777.JPEG:n02783161 +ILSVRC2012_val_00036778.JPEG:n01882714 +ILSVRC2012_val_00036779.JPEG:n02815834 +ILSVRC2012_val_00036780.JPEG:n02814860 +ILSVRC2012_val_00036781.JPEG:n02102177 +ILSVRC2012_val_00036782.JPEG:n02988304 +ILSVRC2012_val_00036783.JPEG:n03376595 +ILSVRC2012_val_00036784.JPEG:n02165105 +ILSVRC2012_val_00036785.JPEG:n04081281 +ILSVRC2012_val_00036786.JPEG:n03495258 +ILSVRC2012_val_00036787.JPEG:n09193705 +ILSVRC2012_val_00036788.JPEG:n04493381 +ILSVRC2012_val_00036789.JPEG:n02815834 +ILSVRC2012_val_00036790.JPEG:n11939491 +ILSVRC2012_val_00036791.JPEG:n02883205 +ILSVRC2012_val_00036792.JPEG:n03063689 +ILSVRC2012_val_00036793.JPEG:n02095570 +ILSVRC2012_val_00036794.JPEG:n04033901 +ILSVRC2012_val_00036795.JPEG:n03937543 +ILSVRC2012_val_00036796.JPEG:n02107908 +ILSVRC2012_val_00036797.JPEG:n07742313 +ILSVRC2012_val_00036798.JPEG:n02114712 +ILSVRC2012_val_00036799.JPEG:n02971356 +ILSVRC2012_val_00036800.JPEG:n02906734 +ILSVRC2012_val_00036801.JPEG:n02814860 +ILSVRC2012_val_00036802.JPEG:n01692333 +ILSVRC2012_val_00036803.JPEG:n02808440 +ILSVRC2012_val_00036804.JPEG:n03706229 +ILSVRC2012_val_00036805.JPEG:n04335435 +ILSVRC2012_val_00036806.JPEG:n03791053 +ILSVRC2012_val_00036807.JPEG:n03742115 +ILSVRC2012_val_00036808.JPEG:n02099429 +ILSVRC2012_val_00036809.JPEG:n02877765 +ILSVRC2012_val_00036810.JPEG:n02321529 +ILSVRC2012_val_00036811.JPEG:n03814639 +ILSVRC2012_val_00036812.JPEG:n01592084 +ILSVRC2012_val_00036813.JPEG:n03272562 +ILSVRC2012_val_00036814.JPEG:n02786058 +ILSVRC2012_val_00036815.JPEG:n01667114 +ILSVRC2012_val_00036816.JPEG:n03947888 +ILSVRC2012_val_00036817.JPEG:n02100735 +ILSVRC2012_val_00036818.JPEG:n04409515 +ILSVRC2012_val_00036819.JPEG:n01601694 +ILSVRC2012_val_00036820.JPEG:n03777568 +ILSVRC2012_val_00036821.JPEG:n12620546 +ILSVRC2012_val_00036822.JPEG:n06794110 +ILSVRC2012_val_00036823.JPEG:n02483708 +ILSVRC2012_val_00036824.JPEG:n03666591 +ILSVRC2012_val_00036825.JPEG:n03759954 +ILSVRC2012_val_00036826.JPEG:n01871265 +ILSVRC2012_val_00036827.JPEG:n02790996 +ILSVRC2012_val_00036828.JPEG:n01955084 +ILSVRC2012_val_00036829.JPEG:n03868863 +ILSVRC2012_val_00036830.JPEG:n03026506 +ILSVRC2012_val_00036831.JPEG:n04070727 +ILSVRC2012_val_00036832.JPEG:n02233338 +ILSVRC2012_val_00036833.JPEG:n01983481 +ILSVRC2012_val_00036834.JPEG:n02640242 +ILSVRC2012_val_00036835.JPEG:n01819313 +ILSVRC2012_val_00036836.JPEG:n02794156 +ILSVRC2012_val_00036837.JPEG:n03017168 +ILSVRC2012_val_00036838.JPEG:n02486261 +ILSVRC2012_val_00036839.JPEG:n04118776 +ILSVRC2012_val_00036840.JPEG:n02769748 +ILSVRC2012_val_00036841.JPEG:n03250847 +ILSVRC2012_val_00036842.JPEG:n02113799 +ILSVRC2012_val_00036843.JPEG:n02105056 +ILSVRC2012_val_00036844.JPEG:n02108422 +ILSVRC2012_val_00036845.JPEG:n01806567 +ILSVRC2012_val_00036846.JPEG:n04229816 +ILSVRC2012_val_00036847.JPEG:n09256479 +ILSVRC2012_val_00036848.JPEG:n04141327 +ILSVRC2012_val_00036849.JPEG:n01692333 +ILSVRC2012_val_00036850.JPEG:n01644373 +ILSVRC2012_val_00036851.JPEG:n02493509 +ILSVRC2012_val_00036852.JPEG:n02892201 +ILSVRC2012_val_00036853.JPEG:n02346627 +ILSVRC2012_val_00036854.JPEG:n07747607 +ILSVRC2012_val_00036855.JPEG:n04120489 +ILSVRC2012_val_00036856.JPEG:n03032252 +ILSVRC2012_val_00036857.JPEG:n04081281 +ILSVRC2012_val_00036858.JPEG:n09468604 +ILSVRC2012_val_00036859.JPEG:n02108422 +ILSVRC2012_val_00036860.JPEG:n07753113 +ILSVRC2012_val_00036861.JPEG:n02441942 +ILSVRC2012_val_00036862.JPEG:n03775071 +ILSVRC2012_val_00036863.JPEG:n02319095 +ILSVRC2012_val_00036864.JPEG:n04579145 +ILSVRC2012_val_00036865.JPEG:n02097474 +ILSVRC2012_val_00036866.JPEG:n03697007 +ILSVRC2012_val_00036867.JPEG:n02769748 +ILSVRC2012_val_00036868.JPEG:n02129604 +ILSVRC2012_val_00036869.JPEG:n04141076 +ILSVRC2012_val_00036870.JPEG:n04476259 +ILSVRC2012_val_00036871.JPEG:n02442845 +ILSVRC2012_val_00036872.JPEG:n04442312 +ILSVRC2012_val_00036873.JPEG:n02012849 +ILSVRC2012_val_00036874.JPEG:n01806567 +ILSVRC2012_val_00036875.JPEG:n03337140 +ILSVRC2012_val_00036876.JPEG:n02097209 +ILSVRC2012_val_00036877.JPEG:n03207941 +ILSVRC2012_val_00036878.JPEG:n01632458 +ILSVRC2012_val_00036879.JPEG:n01818515 +ILSVRC2012_val_00036880.JPEG:n02233338 +ILSVRC2012_val_00036881.JPEG:n02088094 +ILSVRC2012_val_00036882.JPEG:n02727426 +ILSVRC2012_val_00036883.JPEG:n04239074 +ILSVRC2012_val_00036884.JPEG:n03095699 +ILSVRC2012_val_00036885.JPEG:n04606251 +ILSVRC2012_val_00036886.JPEG:n03902125 +ILSVRC2012_val_00036887.JPEG:n02099267 +ILSVRC2012_val_00036888.JPEG:n02086240 +ILSVRC2012_val_00036889.JPEG:n03337140 +ILSVRC2012_val_00036890.JPEG:n02085782 +ILSVRC2012_val_00036891.JPEG:n02412080 +ILSVRC2012_val_00036892.JPEG:n03637318 +ILSVRC2012_val_00036893.JPEG:n01734418 +ILSVRC2012_val_00036894.JPEG:n02113023 +ILSVRC2012_val_00036895.JPEG:n04251144 +ILSVRC2012_val_00036896.JPEG:n03764736 +ILSVRC2012_val_00036897.JPEG:n02114855 +ILSVRC2012_val_00036898.JPEG:n02799071 +ILSVRC2012_val_00036899.JPEG:n01675722 +ILSVRC2012_val_00036900.JPEG:n02843684 +ILSVRC2012_val_00036901.JPEG:n01756291 +ILSVRC2012_val_00036902.JPEG:n04417672 +ILSVRC2012_val_00036903.JPEG:n02835271 +ILSVRC2012_val_00036904.JPEG:n04141076 +ILSVRC2012_val_00036905.JPEG:n04389033 +ILSVRC2012_val_00036906.JPEG:n04482393 +ILSVRC2012_val_00036907.JPEG:n02087394 +ILSVRC2012_val_00036908.JPEG:n02115641 +ILSVRC2012_val_00036909.JPEG:n03017168 +ILSVRC2012_val_00036910.JPEG:n01753488 +ILSVRC2012_val_00036911.JPEG:n02514041 +ILSVRC2012_val_00036912.JPEG:n04509417 +ILSVRC2012_val_00036913.JPEG:n02089973 +ILSVRC2012_val_00036914.JPEG:n03075370 +ILSVRC2012_val_00036915.JPEG:n01644373 +ILSVRC2012_val_00036916.JPEG:n03791053 +ILSVRC2012_val_00036917.JPEG:n04265275 +ILSVRC2012_val_00036918.JPEG:n02111500 +ILSVRC2012_val_00036919.JPEG:n02097209 +ILSVRC2012_val_00036920.JPEG:n04458633 +ILSVRC2012_val_00036921.JPEG:n07802026 +ILSVRC2012_val_00036922.JPEG:n04141076 +ILSVRC2012_val_00036923.JPEG:n04597913 +ILSVRC2012_val_00036924.JPEG:n02281787 +ILSVRC2012_val_00036925.JPEG:n12057211 +ILSVRC2012_val_00036926.JPEG:n02277742 +ILSVRC2012_val_00036927.JPEG:n07716906 +ILSVRC2012_val_00036928.JPEG:n03920288 +ILSVRC2012_val_00036929.JPEG:n04326547 +ILSVRC2012_val_00036930.JPEG:n03127747 +ILSVRC2012_val_00036931.JPEG:n03404251 +ILSVRC2012_val_00036932.JPEG:n02108915 +ILSVRC2012_val_00036933.JPEG:n02127052 +ILSVRC2012_val_00036934.JPEG:n02391049 +ILSVRC2012_val_00036935.JPEG:n04229816 +ILSVRC2012_val_00036936.JPEG:n02837789 +ILSVRC2012_val_00036937.JPEG:n03314780 +ILSVRC2012_val_00036938.JPEG:n02089973 +ILSVRC2012_val_00036939.JPEG:n04296562 +ILSVRC2012_val_00036940.JPEG:n02791270 +ILSVRC2012_val_00036941.JPEG:n03000134 +ILSVRC2012_val_00036942.JPEG:n01644900 +ILSVRC2012_val_00036943.JPEG:n04209133 +ILSVRC2012_val_00036944.JPEG:n01669191 +ILSVRC2012_val_00036945.JPEG:n02107142 +ILSVRC2012_val_00036946.JPEG:n03908714 +ILSVRC2012_val_00036947.JPEG:n03045698 +ILSVRC2012_val_00036948.JPEG:n03485794 +ILSVRC2012_val_00036949.JPEG:n02108551 +ILSVRC2012_val_00036950.JPEG:n02807133 +ILSVRC2012_val_00036951.JPEG:n02892767 +ILSVRC2012_val_00036952.JPEG:n04525305 +ILSVRC2012_val_00036953.JPEG:n02493509 +ILSVRC2012_val_00036954.JPEG:n10148035 +ILSVRC2012_val_00036955.JPEG:n03201208 +ILSVRC2012_val_00036956.JPEG:n03690938 +ILSVRC2012_val_00036957.JPEG:n04505470 +ILSVRC2012_val_00036958.JPEG:n02206856 +ILSVRC2012_val_00036959.JPEG:n02098105 +ILSVRC2012_val_00036960.JPEG:n03478589 +ILSVRC2012_val_00036961.JPEG:n02123597 +ILSVRC2012_val_00036962.JPEG:n02783161 +ILSVRC2012_val_00036963.JPEG:n01667114 +ILSVRC2012_val_00036964.JPEG:n02106550 +ILSVRC2012_val_00036965.JPEG:n03733805 +ILSVRC2012_val_00036966.JPEG:n03424325 +ILSVRC2012_val_00036967.JPEG:n01882714 +ILSVRC2012_val_00036968.JPEG:n01855672 +ILSVRC2012_val_00036969.JPEG:n01855672 +ILSVRC2012_val_00036970.JPEG:n01983481 +ILSVRC2012_val_00036971.JPEG:n01695060 +ILSVRC2012_val_00036972.JPEG:n01847000 +ILSVRC2012_val_00036973.JPEG:n02799071 +ILSVRC2012_val_00036974.JPEG:n04428191 +ILSVRC2012_val_00036975.JPEG:n03223299 +ILSVRC2012_val_00036976.JPEG:n13052670 +ILSVRC2012_val_00036977.JPEG:n02101556 +ILSVRC2012_val_00036978.JPEG:n04265275 +ILSVRC2012_val_00036979.JPEG:n03016953 +ILSVRC2012_val_00036980.JPEG:n01775062 +ILSVRC2012_val_00036981.JPEG:n04033901 +ILSVRC2012_val_00036982.JPEG:n01753488 +ILSVRC2012_val_00036983.JPEG:n03146219 +ILSVRC2012_val_00036984.JPEG:n04235860 +ILSVRC2012_val_00036985.JPEG:n03759954 +ILSVRC2012_val_00036986.JPEG:n03788195 +ILSVRC2012_val_00036987.JPEG:n07749582 +ILSVRC2012_val_00036988.JPEG:n01829413 +ILSVRC2012_val_00036989.JPEG:n02093256 +ILSVRC2012_val_00036990.JPEG:n02231487 +ILSVRC2012_val_00036991.JPEG:n04536866 +ILSVRC2012_val_00036992.JPEG:n03146219 +ILSVRC2012_val_00036993.JPEG:n04004767 +ILSVRC2012_val_00036994.JPEG:n02493793 +ILSVRC2012_val_00036995.JPEG:n04371774 +ILSVRC2012_val_00036996.JPEG:n02395406 +ILSVRC2012_val_00036997.JPEG:n02114712 +ILSVRC2012_val_00036998.JPEG:n02747177 +ILSVRC2012_val_00036999.JPEG:n01560419 +ILSVRC2012_val_00037000.JPEG:n03814906 +ILSVRC2012_val_00037001.JPEG:n04141327 +ILSVRC2012_val_00037002.JPEG:n01833805 +ILSVRC2012_val_00037003.JPEG:n03825788 +ILSVRC2012_val_00037004.JPEG:n02128925 +ILSVRC2012_val_00037005.JPEG:n02120079 +ILSVRC2012_val_00037006.JPEG:n03658185 +ILSVRC2012_val_00037007.JPEG:n03935335 +ILSVRC2012_val_00037008.JPEG:n03530642 +ILSVRC2012_val_00037009.JPEG:n01968897 +ILSVRC2012_val_00037010.JPEG:n02114548 +ILSVRC2012_val_00037011.JPEG:n03873416 +ILSVRC2012_val_00037012.JPEG:n01985128 +ILSVRC2012_val_00037013.JPEG:n01514859 +ILSVRC2012_val_00037014.JPEG:n02669723 +ILSVRC2012_val_00037015.JPEG:n04311174 +ILSVRC2012_val_00037016.JPEG:n03141823 +ILSVRC2012_val_00037017.JPEG:n01872401 +ILSVRC2012_val_00037018.JPEG:n03920288 +ILSVRC2012_val_00037019.JPEG:n02927161 +ILSVRC2012_val_00037020.JPEG:n02397096 +ILSVRC2012_val_00037021.JPEG:n04357314 +ILSVRC2012_val_00037022.JPEG:n03535780 +ILSVRC2012_val_00037023.JPEG:n03127925 +ILSVRC2012_val_00037024.JPEG:n01807496 +ILSVRC2012_val_00037025.JPEG:n02895154 +ILSVRC2012_val_00037026.JPEG:n02794156 +ILSVRC2012_val_00037027.JPEG:n03666591 +ILSVRC2012_val_00037028.JPEG:n04004767 +ILSVRC2012_val_00037029.JPEG:n04039381 +ILSVRC2012_val_00037030.JPEG:n04179913 +ILSVRC2012_val_00037031.JPEG:n01828970 +ILSVRC2012_val_00037032.JPEG:n02128385 +ILSVRC2012_val_00037033.JPEG:n02095570 +ILSVRC2012_val_00037034.JPEG:n04592741 +ILSVRC2012_val_00037035.JPEG:n02793495 +ILSVRC2012_val_00037036.JPEG:n02096177 +ILSVRC2012_val_00037037.JPEG:n01631663 +ILSVRC2012_val_00037038.JPEG:n02111500 +ILSVRC2012_val_00037039.JPEG:n12057211 +ILSVRC2012_val_00037040.JPEG:n04356056 +ILSVRC2012_val_00037041.JPEG:n02894605 +ILSVRC2012_val_00037042.JPEG:n02226429 +ILSVRC2012_val_00037043.JPEG:n04482393 +ILSVRC2012_val_00037044.JPEG:n01950731 +ILSVRC2012_val_00037045.JPEG:n03452741 +ILSVRC2012_val_00037046.JPEG:n01632777 +ILSVRC2012_val_00037047.JPEG:n03197337 +ILSVRC2012_val_00037048.JPEG:n04505470 +ILSVRC2012_val_00037049.JPEG:n04599235 +ILSVRC2012_val_00037050.JPEG:n01484850 +ILSVRC2012_val_00037051.JPEG:n04501370 +ILSVRC2012_val_00037052.JPEG:n02095570 +ILSVRC2012_val_00037053.JPEG:n02276258 +ILSVRC2012_val_00037054.JPEG:n02410509 +ILSVRC2012_val_00037055.JPEG:n04037443 +ILSVRC2012_val_00037056.JPEG:n02276258 +ILSVRC2012_val_00037057.JPEG:n04418357 +ILSVRC2012_val_00037058.JPEG:n02892767 +ILSVRC2012_val_00037059.JPEG:n02099267 +ILSVRC2012_val_00037060.JPEG:n03791053 +ILSVRC2012_val_00037061.JPEG:n04599235 +ILSVRC2012_val_00037062.JPEG:n03642806 +ILSVRC2012_val_00037063.JPEG:n03530642 +ILSVRC2012_val_00037064.JPEG:n07718472 +ILSVRC2012_val_00037065.JPEG:n07693725 +ILSVRC2012_val_00037066.JPEG:n11939491 +ILSVRC2012_val_00037067.JPEG:n02793495 +ILSVRC2012_val_00037068.JPEG:n02988304 +ILSVRC2012_val_00037069.JPEG:n02096051 +ILSVRC2012_val_00037070.JPEG:n01514668 +ILSVRC2012_val_00037071.JPEG:n01616318 +ILSVRC2012_val_00037072.JPEG:n04243546 +ILSVRC2012_val_00037073.JPEG:n02808440 +ILSVRC2012_val_00037074.JPEG:n04270147 +ILSVRC2012_val_00037075.JPEG:n02106030 +ILSVRC2012_val_00037076.JPEG:n04344873 +ILSVRC2012_val_00037077.JPEG:n07930864 +ILSVRC2012_val_00037078.JPEG:n03444034 +ILSVRC2012_val_00037079.JPEG:n07860988 +ILSVRC2012_val_00037080.JPEG:n02119022 +ILSVRC2012_val_00037081.JPEG:n02108000 +ILSVRC2012_val_00037082.JPEG:n04562935 +ILSVRC2012_val_00037083.JPEG:n02105162 +ILSVRC2012_val_00037084.JPEG:n02492035 +ILSVRC2012_val_00037085.JPEG:n02823750 +ILSVRC2012_val_00037086.JPEG:n03481172 +ILSVRC2012_val_00037087.JPEG:n02108000 +ILSVRC2012_val_00037088.JPEG:n04310018 +ILSVRC2012_val_00037089.JPEG:n02107142 +ILSVRC2012_val_00037090.JPEG:n02226429 +ILSVRC2012_val_00037091.JPEG:n02074367 +ILSVRC2012_val_00037092.JPEG:n03785016 +ILSVRC2012_val_00037093.JPEG:n04553703 +ILSVRC2012_val_00037094.JPEG:n03495258 +ILSVRC2012_val_00037095.JPEG:n07579787 +ILSVRC2012_val_00037096.JPEG:n07745940 +ILSVRC2012_val_00037097.JPEG:n02111277 +ILSVRC2012_val_00037098.JPEG:n04476259 +ILSVRC2012_val_00037099.JPEG:n03476684 +ILSVRC2012_val_00037100.JPEG:n04487081 +ILSVRC2012_val_00037101.JPEG:n02091134 +ILSVRC2012_val_00037102.JPEG:n07714571 +ILSVRC2012_val_00037103.JPEG:n02105251 +ILSVRC2012_val_00037104.JPEG:n04404412 +ILSVRC2012_val_00037105.JPEG:n04398044 +ILSVRC2012_val_00037106.JPEG:n01924916 +ILSVRC2012_val_00037107.JPEG:n02487347 +ILSVRC2012_val_00037108.JPEG:n12620546 +ILSVRC2012_val_00037109.JPEG:n03255030 +ILSVRC2012_val_00037110.JPEG:n04325704 +ILSVRC2012_val_00037111.JPEG:n02093647 +ILSVRC2012_val_00037112.JPEG:n02814533 +ILSVRC2012_val_00037113.JPEG:n03125729 +ILSVRC2012_val_00037114.JPEG:n03000247 +ILSVRC2012_val_00037115.JPEG:n02492035 +ILSVRC2012_val_00037116.JPEG:n01530575 +ILSVRC2012_val_00037117.JPEG:n02108915 +ILSVRC2012_val_00037118.JPEG:n02114367 +ILSVRC2012_val_00037119.JPEG:n01796340 +ILSVRC2012_val_00037120.JPEG:n13044778 +ILSVRC2012_val_00037121.JPEG:n04522168 +ILSVRC2012_val_00037122.JPEG:n02443114 +ILSVRC2012_val_00037123.JPEG:n04589890 +ILSVRC2012_val_00037124.JPEG:n04201297 +ILSVRC2012_val_00037125.JPEG:n03733805 +ILSVRC2012_val_00037126.JPEG:n02168699 +ILSVRC2012_val_00037127.JPEG:n01616318 +ILSVRC2012_val_00037128.JPEG:n03594945 +ILSVRC2012_val_00037129.JPEG:n04479046 +ILSVRC2012_val_00037130.JPEG:n02391049 +ILSVRC2012_val_00037131.JPEG:n02892201 +ILSVRC2012_val_00037132.JPEG:n04447861 +ILSVRC2012_val_00037133.JPEG:n02134084 +ILSVRC2012_val_00037134.JPEG:n02096294 +ILSVRC2012_val_00037135.JPEG:n01484850 +ILSVRC2012_val_00037136.JPEG:n03930630 +ILSVRC2012_val_00037137.JPEG:n02090721 +ILSVRC2012_val_00037138.JPEG:n04118538 +ILSVRC2012_val_00037139.JPEG:n02445715 +ILSVRC2012_val_00037140.JPEG:n06596364 +ILSVRC2012_val_00037141.JPEG:n03599486 +ILSVRC2012_val_00037142.JPEG:n04579145 +ILSVRC2012_val_00037143.JPEG:n09468604 +ILSVRC2012_val_00037144.JPEG:n01986214 +ILSVRC2012_val_00037145.JPEG:n01820546 +ILSVRC2012_val_00037146.JPEG:n02526121 +ILSVRC2012_val_00037147.JPEG:n02408429 +ILSVRC2012_val_00037148.JPEG:n03854065 +ILSVRC2012_val_00037149.JPEG:n01855032 +ILSVRC2012_val_00037150.JPEG:n03272562 +ILSVRC2012_val_00037151.JPEG:n09288635 +ILSVRC2012_val_00037152.JPEG:n02106550 +ILSVRC2012_val_00037153.JPEG:n02095314 +ILSVRC2012_val_00037154.JPEG:n01667778 +ILSVRC2012_val_00037155.JPEG:n02137549 +ILSVRC2012_val_00037156.JPEG:n02483708 +ILSVRC2012_val_00037157.JPEG:n02804610 +ILSVRC2012_val_00037158.JPEG:n04125021 +ILSVRC2012_val_00037159.JPEG:n03769881 +ILSVRC2012_val_00037160.JPEG:n02814533 +ILSVRC2012_val_00037161.JPEG:n07718472 +ILSVRC2012_val_00037162.JPEG:n04263257 +ILSVRC2012_val_00037163.JPEG:n03877472 +ILSVRC2012_val_00037164.JPEG:n02107312 +ILSVRC2012_val_00037165.JPEG:n03042490 +ILSVRC2012_val_00037166.JPEG:n01697457 +ILSVRC2012_val_00037167.JPEG:n09468604 +ILSVRC2012_val_00037168.JPEG:n03146219 +ILSVRC2012_val_00037169.JPEG:n02799071 +ILSVRC2012_val_00037170.JPEG:n03764736 +ILSVRC2012_val_00037171.JPEG:n02493793 +ILSVRC2012_val_00037172.JPEG:n03787032 +ILSVRC2012_val_00037173.JPEG:n02808304 +ILSVRC2012_val_00037174.JPEG:n03485407 +ILSVRC2012_val_00037175.JPEG:n01740131 +ILSVRC2012_val_00037176.JPEG:n04589890 +ILSVRC2012_val_00037177.JPEG:n01914609 +ILSVRC2012_val_00037178.JPEG:n02883205 +ILSVRC2012_val_00037179.JPEG:n04254680 +ILSVRC2012_val_00037180.JPEG:n03777568 +ILSVRC2012_val_00037181.JPEG:n02280649 +ILSVRC2012_val_00037182.JPEG:n02102040 +ILSVRC2012_val_00037183.JPEG:n02823750 +ILSVRC2012_val_00037184.JPEG:n04147183 +ILSVRC2012_val_00037185.JPEG:n02091467 +ILSVRC2012_val_00037186.JPEG:n04069434 +ILSVRC2012_val_00037187.JPEG:n01729977 +ILSVRC2012_val_00037188.JPEG:n01818515 +ILSVRC2012_val_00037189.JPEG:n04023962 +ILSVRC2012_val_00037190.JPEG:n03584254 +ILSVRC2012_val_00037191.JPEG:n02095314 +ILSVRC2012_val_00037192.JPEG:n03983396 +ILSVRC2012_val_00037193.JPEG:n03956157 +ILSVRC2012_val_00037194.JPEG:n02097209 +ILSVRC2012_val_00037195.JPEG:n02095314 +ILSVRC2012_val_00037196.JPEG:n02825657 +ILSVRC2012_val_00037197.JPEG:n02107142 +ILSVRC2012_val_00037198.JPEG:n02219486 +ILSVRC2012_val_00037199.JPEG:n03796401 +ILSVRC2012_val_00037200.JPEG:n01687978 +ILSVRC2012_val_00037201.JPEG:n03944341 +ILSVRC2012_val_00037202.JPEG:n02097658 +ILSVRC2012_val_00037203.JPEG:n07718747 +ILSVRC2012_val_00037204.JPEG:n04552348 +ILSVRC2012_val_00037205.JPEG:n04263257 +ILSVRC2012_val_00037206.JPEG:n03942813 +ILSVRC2012_val_00037207.JPEG:n02037110 +ILSVRC2012_val_00037208.JPEG:n03787032 +ILSVRC2012_val_00037209.JPEG:n03642806 +ILSVRC2012_val_00037210.JPEG:n01689811 +ILSVRC2012_val_00037211.JPEG:n02102973 +ILSVRC2012_val_00037212.JPEG:n02480495 +ILSVRC2012_val_00037213.JPEG:n07684084 +ILSVRC2012_val_00037214.JPEG:n02408429 +ILSVRC2012_val_00037215.JPEG:n04356056 +ILSVRC2012_val_00037216.JPEG:n02117135 +ILSVRC2012_val_00037217.JPEG:n07584110 +ILSVRC2012_val_00037218.JPEG:n04265275 +ILSVRC2012_val_00037219.JPEG:n02493793 +ILSVRC2012_val_00037220.JPEG:n01682714 +ILSVRC2012_val_00037221.JPEG:n01981276 +ILSVRC2012_val_00037222.JPEG:n04592741 +ILSVRC2012_val_00037223.JPEG:n03976467 +ILSVRC2012_val_00037224.JPEG:n02948072 +ILSVRC2012_val_00037225.JPEG:n04086273 +ILSVRC2012_val_00037226.JPEG:n04277352 +ILSVRC2012_val_00037227.JPEG:n13054560 +ILSVRC2012_val_00037228.JPEG:n02480495 +ILSVRC2012_val_00037229.JPEG:n01983481 +ILSVRC2012_val_00037230.JPEG:n02085782 +ILSVRC2012_val_00037231.JPEG:n03598930 +ILSVRC2012_val_00037232.JPEG:n03345487 +ILSVRC2012_val_00037233.JPEG:n02017213 +ILSVRC2012_val_00037234.JPEG:n03179701 +ILSVRC2012_val_00037235.JPEG:n01984695 +ILSVRC2012_val_00037236.JPEG:n04296562 +ILSVRC2012_val_00037237.JPEG:n04507155 +ILSVRC2012_val_00037238.JPEG:n04328186 +ILSVRC2012_val_00037239.JPEG:n01534433 +ILSVRC2012_val_00037240.JPEG:n02494079 +ILSVRC2012_val_00037241.JPEG:n03916031 +ILSVRC2012_val_00037242.JPEG:n04376876 +ILSVRC2012_val_00037243.JPEG:n02093428 +ILSVRC2012_val_00037244.JPEG:n01843383 +ILSVRC2012_val_00037245.JPEG:n01924916 +ILSVRC2012_val_00037246.JPEG:n03207743 +ILSVRC2012_val_00037247.JPEG:n07747607 +ILSVRC2012_val_00037248.JPEG:n03785016 +ILSVRC2012_val_00037249.JPEG:n03388549 +ILSVRC2012_val_00037250.JPEG:n02113624 +ILSVRC2012_val_00037251.JPEG:n03961711 +ILSVRC2012_val_00037252.JPEG:n02086646 +ILSVRC2012_val_00037253.JPEG:n02134084 +ILSVRC2012_val_00037254.JPEG:n04606251 +ILSVRC2012_val_00037255.JPEG:n04493381 +ILSVRC2012_val_00037256.JPEG:n02096585 +ILSVRC2012_val_00037257.JPEG:n02992529 +ILSVRC2012_val_00037258.JPEG:n03891332 +ILSVRC2012_val_00037259.JPEG:n01616318 +ILSVRC2012_val_00037260.JPEG:n01496331 +ILSVRC2012_val_00037261.JPEG:n01694178 +ILSVRC2012_val_00037262.JPEG:n01695060 +ILSVRC2012_val_00037263.JPEG:n04026417 +ILSVRC2012_val_00037264.JPEG:n01695060 +ILSVRC2012_val_00037265.JPEG:n02117135 +ILSVRC2012_val_00037266.JPEG:n03584254 +ILSVRC2012_val_00037267.JPEG:n04336792 +ILSVRC2012_val_00037268.JPEG:n01698640 +ILSVRC2012_val_00037269.JPEG:n02177972 +ILSVRC2012_val_00037270.JPEG:n04532670 +ILSVRC2012_val_00037271.JPEG:n02859443 +ILSVRC2012_val_00037272.JPEG:n02095889 +ILSVRC2012_val_00037273.JPEG:n01682714 +ILSVRC2012_val_00037274.JPEG:n11879895 +ILSVRC2012_val_00037275.JPEG:n02114855 +ILSVRC2012_val_00037276.JPEG:n02484975 +ILSVRC2012_val_00037277.JPEG:n02097047 +ILSVRC2012_val_00037278.JPEG:n04204238 +ILSVRC2012_val_00037279.JPEG:n04604644 +ILSVRC2012_val_00037280.JPEG:n01775062 +ILSVRC2012_val_00037281.JPEG:n03775071 +ILSVRC2012_val_00037282.JPEG:n01773549 +ILSVRC2012_val_00037283.JPEG:n03956157 +ILSVRC2012_val_00037284.JPEG:n03792972 +ILSVRC2012_val_00037285.JPEG:n04404412 +ILSVRC2012_val_00037286.JPEG:n09835506 +ILSVRC2012_val_00037287.JPEG:n07717556 +ILSVRC2012_val_00037288.JPEG:n02037110 +ILSVRC2012_val_00037289.JPEG:n02361337 +ILSVRC2012_val_00037290.JPEG:n02105412 +ILSVRC2012_val_00037291.JPEG:n04447861 +ILSVRC2012_val_00037292.JPEG:n02835271 +ILSVRC2012_val_00037293.JPEG:n03240683 +ILSVRC2012_val_00037294.JPEG:n07613480 +ILSVRC2012_val_00037295.JPEG:n02422699 +ILSVRC2012_val_00037296.JPEG:n02488702 +ILSVRC2012_val_00037297.JPEG:n01776313 +ILSVRC2012_val_00037298.JPEG:n04579432 +ILSVRC2012_val_00037299.JPEG:n04116512 +ILSVRC2012_val_00037300.JPEG:n03857828 +ILSVRC2012_val_00037301.JPEG:n02676566 +ILSVRC2012_val_00037302.JPEG:n03063599 +ILSVRC2012_val_00037303.JPEG:n02397096 +ILSVRC2012_val_00037304.JPEG:n02977058 +ILSVRC2012_val_00037305.JPEG:n02089867 +ILSVRC2012_val_00037306.JPEG:n04429376 +ILSVRC2012_val_00037307.JPEG:n03018349 +ILSVRC2012_val_00037308.JPEG:n13037406 +ILSVRC2012_val_00037309.JPEG:n03998194 +ILSVRC2012_val_00037310.JPEG:n01693334 +ILSVRC2012_val_00037311.JPEG:n01770081 +ILSVRC2012_val_00037312.JPEG:n03991062 +ILSVRC2012_val_00037313.JPEG:n03141823 +ILSVRC2012_val_00037314.JPEG:n03691459 +ILSVRC2012_val_00037315.JPEG:n04039381 +ILSVRC2012_val_00037316.JPEG:n02894605 +ILSVRC2012_val_00037317.JPEG:n02096177 +ILSVRC2012_val_00037318.JPEG:n02093256 +ILSVRC2012_val_00037319.JPEG:n02917067 +ILSVRC2012_val_00037320.JPEG:n03791053 +ILSVRC2012_val_00037321.JPEG:n03976467 +ILSVRC2012_val_00037322.JPEG:n02795169 +ILSVRC2012_val_00037323.JPEG:n02112706 +ILSVRC2012_val_00037324.JPEG:n01692333 +ILSVRC2012_val_00037325.JPEG:n02111129 +ILSVRC2012_val_00037326.JPEG:n03110669 +ILSVRC2012_val_00037327.JPEG:n03803284 +ILSVRC2012_val_00037328.JPEG:n01592084 +ILSVRC2012_val_00037329.JPEG:n02514041 +ILSVRC2012_val_00037330.JPEG:n02104365 +ILSVRC2012_val_00037331.JPEG:n02089867 +ILSVRC2012_val_00037332.JPEG:n07860988 +ILSVRC2012_val_00037333.JPEG:n02093256 +ILSVRC2012_val_00037334.JPEG:n02403003 +ILSVRC2012_val_00037335.JPEG:n04522168 +ILSVRC2012_val_00037336.JPEG:n02837789 +ILSVRC2012_val_00037337.JPEG:n01855032 +ILSVRC2012_val_00037338.JPEG:n02793495 +ILSVRC2012_val_00037339.JPEG:n02093991 +ILSVRC2012_val_00037340.JPEG:n02437312 +ILSVRC2012_val_00037341.JPEG:n02980441 +ILSVRC2012_val_00037342.JPEG:n04116512 +ILSVRC2012_val_00037343.JPEG:n02120079 +ILSVRC2012_val_00037344.JPEG:n04371774 +ILSVRC2012_val_00037345.JPEG:n02104365 +ILSVRC2012_val_00037346.JPEG:n04153751 +ILSVRC2012_val_00037347.JPEG:n02091635 +ILSVRC2012_val_00037348.JPEG:n01775062 +ILSVRC2012_val_00037349.JPEG:n04310018 +ILSVRC2012_val_00037350.JPEG:n03529860 +ILSVRC2012_val_00037351.JPEG:n02105162 +ILSVRC2012_val_00037352.JPEG:n02814860 +ILSVRC2012_val_00037353.JPEG:n02088364 +ILSVRC2012_val_00037354.JPEG:n02116738 +ILSVRC2012_val_00037355.JPEG:n03630383 +ILSVRC2012_val_00037356.JPEG:n02229544 +ILSVRC2012_val_00037357.JPEG:n04111531 +ILSVRC2012_val_00037358.JPEG:n01882714 +ILSVRC2012_val_00037359.JPEG:n01917289 +ILSVRC2012_val_00037360.JPEG:n03877472 +ILSVRC2012_val_00037361.JPEG:n02346627 +ILSVRC2012_val_00037362.JPEG:n03476991 +ILSVRC2012_val_00037363.JPEG:n02115641 +ILSVRC2012_val_00037364.JPEG:n03110669 +ILSVRC2012_val_00037365.JPEG:n02799071 +ILSVRC2012_val_00037366.JPEG:n03272562 +ILSVRC2012_val_00037367.JPEG:n01729322 +ILSVRC2012_val_00037368.JPEG:n03599486 +ILSVRC2012_val_00037369.JPEG:n03445777 +ILSVRC2012_val_00037370.JPEG:n04099969 +ILSVRC2012_val_00037371.JPEG:n02536864 +ILSVRC2012_val_00037372.JPEG:n03026506 +ILSVRC2012_val_00037373.JPEG:n03899768 +ILSVRC2012_val_00037374.JPEG:n04485082 +ILSVRC2012_val_00037375.JPEG:n01440764 +ILSVRC2012_val_00037376.JPEG:n04370456 +ILSVRC2012_val_00037377.JPEG:n04125021 +ILSVRC2012_val_00037378.JPEG:n07565083 +ILSVRC2012_val_00037379.JPEG:n02012849 +ILSVRC2012_val_00037380.JPEG:n02437616 +ILSVRC2012_val_00037381.JPEG:n02281406 +ILSVRC2012_val_00037382.JPEG:n03141823 +ILSVRC2012_val_00037383.JPEG:n01440764 +ILSVRC2012_val_00037384.JPEG:n04548362 +ILSVRC2012_val_00037385.JPEG:n03584254 +ILSVRC2012_val_00037386.JPEG:n04366367 +ILSVRC2012_val_00037387.JPEG:n04069434 +ILSVRC2012_val_00037388.JPEG:n02108551 +ILSVRC2012_val_00037389.JPEG:n07697313 +ILSVRC2012_val_00037390.JPEG:n02916936 +ILSVRC2012_val_00037391.JPEG:n03124043 +ILSVRC2012_val_00037392.JPEG:n01697457 +ILSVRC2012_val_00037393.JPEG:n02095570 +ILSVRC2012_val_00037394.JPEG:n03016953 +ILSVRC2012_val_00037395.JPEG:n02441942 +ILSVRC2012_val_00037396.JPEG:n02106382 +ILSVRC2012_val_00037397.JPEG:n01833805 +ILSVRC2012_val_00037398.JPEG:n03045698 +ILSVRC2012_val_00037399.JPEG:n04404412 +ILSVRC2012_val_00037400.JPEG:n03888605 +ILSVRC2012_val_00037401.JPEG:n04259630 +ILSVRC2012_val_00037402.JPEG:n03075370 +ILSVRC2012_val_00037403.JPEG:n03124170 +ILSVRC2012_val_00037404.JPEG:n03534580 +ILSVRC2012_val_00037405.JPEG:n04277352 +ILSVRC2012_val_00037406.JPEG:n03717622 +ILSVRC2012_val_00037407.JPEG:n02526121 +ILSVRC2012_val_00037408.JPEG:n01797886 +ILSVRC2012_val_00037409.JPEG:n04133789 +ILSVRC2012_val_00037410.JPEG:n02105855 +ILSVRC2012_val_00037411.JPEG:n03530642 +ILSVRC2012_val_00037412.JPEG:n02130308 +ILSVRC2012_val_00037413.JPEG:n01980166 +ILSVRC2012_val_00037414.JPEG:n04192698 +ILSVRC2012_val_00037415.JPEG:n04336792 +ILSVRC2012_val_00037416.JPEG:n07742313 +ILSVRC2012_val_00037417.JPEG:n01692333 +ILSVRC2012_val_00037418.JPEG:n02279972 +ILSVRC2012_val_00037419.JPEG:n04371430 +ILSVRC2012_val_00037420.JPEG:n01592084 +ILSVRC2012_val_00037421.JPEG:n09332890 +ILSVRC2012_val_00037422.JPEG:n04332243 +ILSVRC2012_val_00037423.JPEG:n04392985 +ILSVRC2012_val_00037424.JPEG:n07720875 +ILSVRC2012_val_00037425.JPEG:n03478589 +ILSVRC2012_val_00037426.JPEG:n03291819 +ILSVRC2012_val_00037427.JPEG:n04560804 +ILSVRC2012_val_00037428.JPEG:n02106030 +ILSVRC2012_val_00037429.JPEG:n04049303 +ILSVRC2012_val_00037430.JPEG:n02927161 +ILSVRC2012_val_00037431.JPEG:n07753113 +ILSVRC2012_val_00037432.JPEG:n04065272 +ILSVRC2012_val_00037433.JPEG:n02835271 +ILSVRC2012_val_00037434.JPEG:n03047690 +ILSVRC2012_val_00037435.JPEG:n03538406 +ILSVRC2012_val_00037436.JPEG:n01582220 +ILSVRC2012_val_00037437.JPEG:n02113624 +ILSVRC2012_val_00037438.JPEG:n03792782 +ILSVRC2012_val_00037439.JPEG:n04116512 +ILSVRC2012_val_00037440.JPEG:n02093859 +ILSVRC2012_val_00037441.JPEG:n03961711 +ILSVRC2012_val_00037442.JPEG:n02109047 +ILSVRC2012_val_00037443.JPEG:n07831146 +ILSVRC2012_val_00037444.JPEG:n02825657 +ILSVRC2012_val_00037445.JPEG:n13054560 +ILSVRC2012_val_00037446.JPEG:n02951585 +ILSVRC2012_val_00037447.JPEG:n02442845 +ILSVRC2012_val_00037448.JPEG:n02817516 +ILSVRC2012_val_00037449.JPEG:n03874599 +ILSVRC2012_val_00037450.JPEG:n02093859 +ILSVRC2012_val_00037451.JPEG:n01755581 +ILSVRC2012_val_00037452.JPEG:n02860847 +ILSVRC2012_val_00037453.JPEG:n02167151 +ILSVRC2012_val_00037454.JPEG:n01537544 +ILSVRC2012_val_00037455.JPEG:n02099601 +ILSVRC2012_val_00037456.JPEG:n02111500 +ILSVRC2012_val_00037457.JPEG:n03670208 +ILSVRC2012_val_00037458.JPEG:n03179701 +ILSVRC2012_val_00037459.JPEG:n02093647 +ILSVRC2012_val_00037460.JPEG:n03444034 +ILSVRC2012_val_00037461.JPEG:n03131574 +ILSVRC2012_val_00037462.JPEG:n02111500 +ILSVRC2012_val_00037463.JPEG:n04069434 +ILSVRC2012_val_00037464.JPEG:n01744401 +ILSVRC2012_val_00037465.JPEG:n03220513 +ILSVRC2012_val_00037466.JPEG:n03393912 +ILSVRC2012_val_00037467.JPEG:n02486261 +ILSVRC2012_val_00037468.JPEG:n03372029 +ILSVRC2012_val_00037469.JPEG:n01728572 +ILSVRC2012_val_00037470.JPEG:n02422106 +ILSVRC2012_val_00037471.JPEG:n01833805 +ILSVRC2012_val_00037472.JPEG:n03594734 +ILSVRC2012_val_00037473.JPEG:n13044778 +ILSVRC2012_val_00037474.JPEG:n02074367 +ILSVRC2012_val_00037475.JPEG:n02391049 +ILSVRC2012_val_00037476.JPEG:n07873807 +ILSVRC2012_val_00037477.JPEG:n09468604 +ILSVRC2012_val_00037478.JPEG:n02799071 +ILSVRC2012_val_00037479.JPEG:n03832673 +ILSVRC2012_val_00037480.JPEG:n02361337 +ILSVRC2012_val_00037481.JPEG:n02111277 +ILSVRC2012_val_00037482.JPEG:n04204238 +ILSVRC2012_val_00037483.JPEG:n02172182 +ILSVRC2012_val_00037484.JPEG:n04562935 +ILSVRC2012_val_00037485.JPEG:n02100735 +ILSVRC2012_val_00037486.JPEG:n02007558 +ILSVRC2012_val_00037487.JPEG:n03630383 +ILSVRC2012_val_00037488.JPEG:n01484850 +ILSVRC2012_val_00037489.JPEG:n02484975 +ILSVRC2012_val_00037490.JPEG:n02096051 +ILSVRC2012_val_00037491.JPEG:n02206856 +ILSVRC2012_val_00037492.JPEG:n03770679 +ILSVRC2012_val_00037493.JPEG:n04265275 +ILSVRC2012_val_00037494.JPEG:n09246464 +ILSVRC2012_val_00037495.JPEG:n09835506 +ILSVRC2012_val_00037496.JPEG:n07614500 +ILSVRC2012_val_00037497.JPEG:n09472597 +ILSVRC2012_val_00037498.JPEG:n03379051 +ILSVRC2012_val_00037499.JPEG:n03457902 +ILSVRC2012_val_00037500.JPEG:n01855032 +ILSVRC2012_val_00037501.JPEG:n04201297 +ILSVRC2012_val_00037502.JPEG:n02951585 +ILSVRC2012_val_00037503.JPEG:n13133613 +ILSVRC2012_val_00037504.JPEG:n03770439 +ILSVRC2012_val_00037505.JPEG:n02172182 +ILSVRC2012_val_00037506.JPEG:n03992509 +ILSVRC2012_val_00037507.JPEG:n03617480 +ILSVRC2012_val_00037508.JPEG:n02802426 +ILSVRC2012_val_00037509.JPEG:n02676566 +ILSVRC2012_val_00037510.JPEG:n01687978 +ILSVRC2012_val_00037511.JPEG:n07711569 +ILSVRC2012_val_00037512.JPEG:n03690938 +ILSVRC2012_val_00037513.JPEG:n02869837 +ILSVRC2012_val_00037514.JPEG:n03942813 +ILSVRC2012_val_00037515.JPEG:n04332243 +ILSVRC2012_val_00037516.JPEG:n01491361 +ILSVRC2012_val_00037517.JPEG:n12768682 +ILSVRC2012_val_00037518.JPEG:n01910747 +ILSVRC2012_val_00037519.JPEG:n04179913 +ILSVRC2012_val_00037520.JPEG:n03627232 +ILSVRC2012_val_00037521.JPEG:n13037406 +ILSVRC2012_val_00037522.JPEG:n07745940 +ILSVRC2012_val_00037523.JPEG:n04152593 +ILSVRC2012_val_00037524.JPEG:n01806143 +ILSVRC2012_val_00037525.JPEG:n07565083 +ILSVRC2012_val_00037526.JPEG:n03627232 +ILSVRC2012_val_00037527.JPEG:n12267677 +ILSVRC2012_val_00037528.JPEG:n03837869 +ILSVRC2012_val_00037529.JPEG:n02094433 +ILSVRC2012_val_00037530.JPEG:n04238763 +ILSVRC2012_val_00037531.JPEG:n03496892 +ILSVRC2012_val_00037532.JPEG:n04612504 +ILSVRC2012_val_00037533.JPEG:n02807133 +ILSVRC2012_val_00037534.JPEG:n02106166 +ILSVRC2012_val_00037535.JPEG:n02484975 +ILSVRC2012_val_00037536.JPEG:n03208938 +ILSVRC2012_val_00037537.JPEG:n04065272 +ILSVRC2012_val_00037538.JPEG:n02107574 +ILSVRC2012_val_00037539.JPEG:n07715103 +ILSVRC2012_val_00037540.JPEG:n04517823 +ILSVRC2012_val_00037541.JPEG:n10565667 +ILSVRC2012_val_00037542.JPEG:n02807133 +ILSVRC2012_val_00037543.JPEG:n03717622 +ILSVRC2012_val_00037544.JPEG:n04557648 +ILSVRC2012_val_00037545.JPEG:n04591157 +ILSVRC2012_val_00037546.JPEG:n02326432 +ILSVRC2012_val_00037547.JPEG:n06874185 +ILSVRC2012_val_00037548.JPEG:n04442312 +ILSVRC2012_val_00037549.JPEG:n03042490 +ILSVRC2012_val_00037550.JPEG:n03188531 +ILSVRC2012_val_00037551.JPEG:n04487394 +ILSVRC2012_val_00037552.JPEG:n02006656 +ILSVRC2012_val_00037553.JPEG:n01729322 +ILSVRC2012_val_00037554.JPEG:n03929660 +ILSVRC2012_val_00037555.JPEG:n03425413 +ILSVRC2012_val_00037556.JPEG:n03216828 +ILSVRC2012_val_00037557.JPEG:n02346627 +ILSVRC2012_val_00037558.JPEG:n02526121 +ILSVRC2012_val_00037559.JPEG:n02089078 +ILSVRC2012_val_00037560.JPEG:n01669191 +ILSVRC2012_val_00037561.JPEG:n10565667 +ILSVRC2012_val_00037562.JPEG:n04376876 +ILSVRC2012_val_00037563.JPEG:n04258138 +ILSVRC2012_val_00037564.JPEG:n02489166 +ILSVRC2012_val_00037565.JPEG:n02493793 +ILSVRC2012_val_00037566.JPEG:n03584829 +ILSVRC2012_val_00037567.JPEG:n03379051 +ILSVRC2012_val_00037568.JPEG:n02094114 +ILSVRC2012_val_00037569.JPEG:n01514668 +ILSVRC2012_val_00037570.JPEG:n03770439 +ILSVRC2012_val_00037571.JPEG:n02231487 +ILSVRC2012_val_00037572.JPEG:n01855032 +ILSVRC2012_val_00037573.JPEG:n03180011 +ILSVRC2012_val_00037574.JPEG:n04606251 +ILSVRC2012_val_00037575.JPEG:n03916031 +ILSVRC2012_val_00037576.JPEG:n01774750 +ILSVRC2012_val_00037577.JPEG:n02087394 +ILSVRC2012_val_00037578.JPEG:n03297495 +ILSVRC2012_val_00037579.JPEG:n01968897 +ILSVRC2012_val_00037580.JPEG:n02105056 +ILSVRC2012_val_00037581.JPEG:n01491361 +ILSVRC2012_val_00037582.JPEG:n02114712 +ILSVRC2012_val_00037583.JPEG:n02097130 +ILSVRC2012_val_00037584.JPEG:n02692877 +ILSVRC2012_val_00037585.JPEG:n04125021 +ILSVRC2012_val_00037586.JPEG:n03476684 +ILSVRC2012_val_00037587.JPEG:n03658185 +ILSVRC2012_val_00037588.JPEG:n02966687 +ILSVRC2012_val_00037589.JPEG:n02259212 +ILSVRC2012_val_00037590.JPEG:n03355925 +ILSVRC2012_val_00037591.JPEG:n13133613 +ILSVRC2012_val_00037592.JPEG:n03394916 +ILSVRC2012_val_00037593.JPEG:n02107312 +ILSVRC2012_val_00037594.JPEG:n02788148 +ILSVRC2012_val_00037595.JPEG:n02109961 +ILSVRC2012_val_00037596.JPEG:n01440764 +ILSVRC2012_val_00037597.JPEG:n03124043 +ILSVRC2012_val_00037598.JPEG:n06359193 +ILSVRC2012_val_00037599.JPEG:n04133789 +ILSVRC2012_val_00037600.JPEG:n02500267 +ILSVRC2012_val_00037601.JPEG:n04209133 +ILSVRC2012_val_00037602.JPEG:n03344393 +ILSVRC2012_val_00037603.JPEG:n03494278 +ILSVRC2012_val_00037604.JPEG:n02977058 +ILSVRC2012_val_00037605.JPEG:n03710637 +ILSVRC2012_val_00037606.JPEG:n01622779 +ILSVRC2012_val_00037607.JPEG:n09421951 +ILSVRC2012_val_00037608.JPEG:n02790996 +ILSVRC2012_val_00037609.JPEG:n02089078 +ILSVRC2012_val_00037610.JPEG:n02256656 +ILSVRC2012_val_00037611.JPEG:n01531178 +ILSVRC2012_val_00037612.JPEG:n04479046 +ILSVRC2012_val_00037613.JPEG:n04141327 +ILSVRC2012_val_00037614.JPEG:n03000134 +ILSVRC2012_val_00037615.JPEG:n02504013 +ILSVRC2012_val_00037616.JPEG:n03627232 +ILSVRC2012_val_00037617.JPEG:n02114712 +ILSVRC2012_val_00037618.JPEG:n03325584 +ILSVRC2012_val_00037619.JPEG:n03773504 +ILSVRC2012_val_00037620.JPEG:n04004767 +ILSVRC2012_val_00037621.JPEG:n04266014 +ILSVRC2012_val_00037622.JPEG:n02977058 +ILSVRC2012_val_00037623.JPEG:n02125311 +ILSVRC2012_val_00037624.JPEG:n02281406 +ILSVRC2012_val_00037625.JPEG:n03291819 +ILSVRC2012_val_00037626.JPEG:n01675722 +ILSVRC2012_val_00037627.JPEG:n02138441 +ILSVRC2012_val_00037628.JPEG:n03804744 +ILSVRC2012_val_00037629.JPEG:n03000684 +ILSVRC2012_val_00037630.JPEG:n02114367 +ILSVRC2012_val_00037631.JPEG:n03187595 +ILSVRC2012_val_00037632.JPEG:n01943899 +ILSVRC2012_val_00037633.JPEG:n02125311 +ILSVRC2012_val_00037634.JPEG:n02113624 +ILSVRC2012_val_00037635.JPEG:n02823428 +ILSVRC2012_val_00037636.JPEG:n02233338 +ILSVRC2012_val_00037637.JPEG:n03110669 +ILSVRC2012_val_00037638.JPEG:n02500267 +ILSVRC2012_val_00037639.JPEG:n03594734 +ILSVRC2012_val_00037640.JPEG:n03347037 +ILSVRC2012_val_00037641.JPEG:n01990800 +ILSVRC2012_val_00037642.JPEG:n02074367 +ILSVRC2012_val_00037643.JPEG:n02396427 +ILSVRC2012_val_00037644.JPEG:n03954731 +ILSVRC2012_val_00037645.JPEG:n02687172 +ILSVRC2012_val_00037646.JPEG:n02883205 +ILSVRC2012_val_00037647.JPEG:n03127925 +ILSVRC2012_val_00037648.JPEG:n02111500 +ILSVRC2012_val_00037649.JPEG:n07718747 +ILSVRC2012_val_00037650.JPEG:n02447366 +ILSVRC2012_val_00037651.JPEG:n04286575 +ILSVRC2012_val_00037652.JPEG:n02930766 +ILSVRC2012_val_00037653.JPEG:n01664065 +ILSVRC2012_val_00037654.JPEG:n04153751 +ILSVRC2012_val_00037655.JPEG:n01687978 +ILSVRC2012_val_00037656.JPEG:n02422699 +ILSVRC2012_val_00037657.JPEG:n02791270 +ILSVRC2012_val_00037658.JPEG:n02835271 +ILSVRC2012_val_00037659.JPEG:n02504458 +ILSVRC2012_val_00037660.JPEG:n01917289 +ILSVRC2012_val_00037661.JPEG:n04252077 +ILSVRC2012_val_00037662.JPEG:n04548280 +ILSVRC2012_val_00037663.JPEG:n03089624 +ILSVRC2012_val_00037664.JPEG:n07590611 +ILSVRC2012_val_00037665.JPEG:n07754684 +ILSVRC2012_val_00037666.JPEG:n01739381 +ILSVRC2012_val_00037667.JPEG:n04483307 +ILSVRC2012_val_00037668.JPEG:n01914609 +ILSVRC2012_val_00037669.JPEG:n02087046 +ILSVRC2012_val_00037670.JPEG:n03697007 +ILSVRC2012_val_00037671.JPEG:n04039381 +ILSVRC2012_val_00037672.JPEG:n01820546 +ILSVRC2012_val_00037673.JPEG:n04355338 +ILSVRC2012_val_00037674.JPEG:n02100735 +ILSVRC2012_val_00037675.JPEG:n03032252 +ILSVRC2012_val_00037676.JPEG:n02091467 +ILSVRC2012_val_00037677.JPEG:n01728572 +ILSVRC2012_val_00037678.JPEG:n02002556 +ILSVRC2012_val_00037679.JPEG:n03874599 +ILSVRC2012_val_00037680.JPEG:n02859443 +ILSVRC2012_val_00037681.JPEG:n04146614 +ILSVRC2012_val_00037682.JPEG:n03534580 +ILSVRC2012_val_00037683.JPEG:n04532106 +ILSVRC2012_val_00037684.JPEG:n01981276 +ILSVRC2012_val_00037685.JPEG:n03814639 +ILSVRC2012_val_00037686.JPEG:n01689811 +ILSVRC2012_val_00037687.JPEG:n06359193 +ILSVRC2012_val_00037688.JPEG:n01675722 +ILSVRC2012_val_00037689.JPEG:n03888605 +ILSVRC2012_val_00037690.JPEG:n07714990 +ILSVRC2012_val_00037691.JPEG:n04476259 +ILSVRC2012_val_00037692.JPEG:n02536864 +ILSVRC2012_val_00037693.JPEG:n02492035 +ILSVRC2012_val_00037694.JPEG:n04265275 +ILSVRC2012_val_00037695.JPEG:n02948072 +ILSVRC2012_val_00037696.JPEG:n03804744 +ILSVRC2012_val_00037697.JPEG:n04380533 +ILSVRC2012_val_00037698.JPEG:n01518878 +ILSVRC2012_val_00037699.JPEG:n04005630 +ILSVRC2012_val_00037700.JPEG:n07590611 +ILSVRC2012_val_00037701.JPEG:n04417672 +ILSVRC2012_val_00037702.JPEG:n03709823 +ILSVRC2012_val_00037703.JPEG:n02105412 +ILSVRC2012_val_00037704.JPEG:n02363005 +ILSVRC2012_val_00037705.JPEG:n01494475 +ILSVRC2012_val_00037706.JPEG:n03680355 +ILSVRC2012_val_00037707.JPEG:n02951358 +ILSVRC2012_val_00037708.JPEG:n04597913 +ILSVRC2012_val_00037709.JPEG:n03998194 +ILSVRC2012_val_00037710.JPEG:n01855032 +ILSVRC2012_val_00037711.JPEG:n02018795 +ILSVRC2012_val_00037712.JPEG:n03271574 +ILSVRC2012_val_00037713.JPEG:n02167151 +ILSVRC2012_val_00037714.JPEG:n02009912 +ILSVRC2012_val_00037715.JPEG:n03825788 +ILSVRC2012_val_00037716.JPEG:n04482393 +ILSVRC2012_val_00037717.JPEG:n01774750 +ILSVRC2012_val_00037718.JPEG:n02500267 +ILSVRC2012_val_00037719.JPEG:n01514859 +ILSVRC2012_val_00037720.JPEG:n03908618 +ILSVRC2012_val_00037721.JPEG:n03761084 +ILSVRC2012_val_00037722.JPEG:n03633091 +ILSVRC2012_val_00037723.JPEG:n02096177 +ILSVRC2012_val_00037724.JPEG:n03729826 +ILSVRC2012_val_00037725.JPEG:n07717556 +ILSVRC2012_val_00037726.JPEG:n03670208 +ILSVRC2012_val_00037727.JPEG:n01773797 +ILSVRC2012_val_00037728.JPEG:n04554684 +ILSVRC2012_val_00037729.JPEG:n01697457 +ILSVRC2012_val_00037730.JPEG:n03691459 +ILSVRC2012_val_00037731.JPEG:n02138441 +ILSVRC2012_val_00037732.JPEG:n03764736 +ILSVRC2012_val_00037733.JPEG:n02123394 +ILSVRC2012_val_00037734.JPEG:n04192698 +ILSVRC2012_val_00037735.JPEG:n04120489 +ILSVRC2012_val_00037736.JPEG:n07615774 +ILSVRC2012_val_00037737.JPEG:n03929855 +ILSVRC2012_val_00037738.JPEG:n02494079 +ILSVRC2012_val_00037739.JPEG:n01669191 +ILSVRC2012_val_00037740.JPEG:n01498041 +ILSVRC2012_val_00037741.JPEG:n03250847 +ILSVRC2012_val_00037742.JPEG:n03924679 +ILSVRC2012_val_00037743.JPEG:n02356798 +ILSVRC2012_val_00037744.JPEG:n02823750 +ILSVRC2012_val_00037745.JPEG:n03447721 +ILSVRC2012_val_00037746.JPEG:n02058221 +ILSVRC2012_val_00037747.JPEG:n07930864 +ILSVRC2012_val_00037748.JPEG:n01530575 +ILSVRC2012_val_00037749.JPEG:n04428191 +ILSVRC2012_val_00037750.JPEG:n04372370 +ILSVRC2012_val_00037751.JPEG:n03840681 +ILSVRC2012_val_00037752.JPEG:n02027492 +ILSVRC2012_val_00037753.JPEG:n01498041 +ILSVRC2012_val_00037754.JPEG:n07718472 +ILSVRC2012_val_00037755.JPEG:n03954731 +ILSVRC2012_val_00037756.JPEG:n04099969 +ILSVRC2012_val_00037757.JPEG:n03954731 +ILSVRC2012_val_00037758.JPEG:n01770081 +ILSVRC2012_val_00037759.JPEG:n03445924 +ILSVRC2012_val_00037760.JPEG:n03045698 +ILSVRC2012_val_00037761.JPEG:n03527444 +ILSVRC2012_val_00037762.JPEG:n02840245 +ILSVRC2012_val_00037763.JPEG:n04201297 +ILSVRC2012_val_00037764.JPEG:n01735189 +ILSVRC2012_val_00037765.JPEG:n01986214 +ILSVRC2012_val_00037766.JPEG:n02002724 +ILSVRC2012_val_00037767.JPEG:n02113978 +ILSVRC2012_val_00037768.JPEG:n02177972 +ILSVRC2012_val_00037769.JPEG:n03908714 +ILSVRC2012_val_00037770.JPEG:n03888257 +ILSVRC2012_val_00037771.JPEG:n02100236 +ILSVRC2012_val_00037772.JPEG:n02437312 +ILSVRC2012_val_00037773.JPEG:n02236044 +ILSVRC2012_val_00037774.JPEG:n07871810 +ILSVRC2012_val_00037775.JPEG:n03775071 +ILSVRC2012_val_00037776.JPEG:n03947888 +ILSVRC2012_val_00037777.JPEG:n03933933 +ILSVRC2012_val_00037778.JPEG:n02066245 +ILSVRC2012_val_00037779.JPEG:n02128385 +ILSVRC2012_val_00037780.JPEG:n01491361 +ILSVRC2012_val_00037781.JPEG:n02493509 +ILSVRC2012_val_00037782.JPEG:n07717556 +ILSVRC2012_val_00037783.JPEG:n02865351 +ILSVRC2012_val_00037784.JPEG:n03187595 +ILSVRC2012_val_00037785.JPEG:n02666196 +ILSVRC2012_val_00037786.JPEG:n01917289 +ILSVRC2012_val_00037787.JPEG:n01770081 +ILSVRC2012_val_00037788.JPEG:n02788148 +ILSVRC2012_val_00037789.JPEG:n03661043 +ILSVRC2012_val_00037790.JPEG:n02481823 +ILSVRC2012_val_00037791.JPEG:n02085620 +ILSVRC2012_val_00037792.JPEG:n02799071 +ILSVRC2012_val_00037793.JPEG:n03590841 +ILSVRC2012_val_00037794.JPEG:n01749939 +ILSVRC2012_val_00037795.JPEG:n01614925 +ILSVRC2012_val_00037796.JPEG:n02950826 +ILSVRC2012_val_00037797.JPEG:n02088632 +ILSVRC2012_val_00037798.JPEG:n01498041 +ILSVRC2012_val_00037799.JPEG:n02105162 +ILSVRC2012_val_00037800.JPEG:n01737021 +ILSVRC2012_val_00037801.JPEG:n02690373 +ILSVRC2012_val_00037802.JPEG:n03584254 +ILSVRC2012_val_00037803.JPEG:n02791124 +ILSVRC2012_val_00037804.JPEG:n02088238 +ILSVRC2012_val_00037805.JPEG:n04328186 +ILSVRC2012_val_00037806.JPEG:n01582220 +ILSVRC2012_val_00037807.JPEG:n02231487 +ILSVRC2012_val_00037808.JPEG:n03717622 +ILSVRC2012_val_00037809.JPEG:n01751748 +ILSVRC2012_val_00037810.JPEG:n03721384 +ILSVRC2012_val_00037811.JPEG:n02108422 +ILSVRC2012_val_00037812.JPEG:n01669191 +ILSVRC2012_val_00037813.JPEG:n02980441 +ILSVRC2012_val_00037814.JPEG:n04243546 +ILSVRC2012_val_00037815.JPEG:n03982430 +ILSVRC2012_val_00037816.JPEG:n02422106 +ILSVRC2012_val_00037817.JPEG:n03014705 +ILSVRC2012_val_00037818.JPEG:n04371774 +ILSVRC2012_val_00037819.JPEG:n04125021 +ILSVRC2012_val_00037820.JPEG:n02090622 +ILSVRC2012_val_00037821.JPEG:n01930112 +ILSVRC2012_val_00037822.JPEG:n04552348 +ILSVRC2012_val_00037823.JPEG:n03764736 +ILSVRC2012_val_00037824.JPEG:n01582220 +ILSVRC2012_val_00037825.JPEG:n02056570 +ILSVRC2012_val_00037826.JPEG:n02089973 +ILSVRC2012_val_00037827.JPEG:n09399592 +ILSVRC2012_val_00037828.JPEG:n03450230 +ILSVRC2012_val_00037829.JPEG:n03770679 +ILSVRC2012_val_00037830.JPEG:n03445924 +ILSVRC2012_val_00037831.JPEG:n02007558 +ILSVRC2012_val_00037832.JPEG:n02268443 +ILSVRC2012_val_00037833.JPEG:n02396427 +ILSVRC2012_val_00037834.JPEG:n01440764 +ILSVRC2012_val_00037835.JPEG:n03062245 +ILSVRC2012_val_00037836.JPEG:n02134418 +ILSVRC2012_val_00037837.JPEG:n03594734 +ILSVRC2012_val_00037838.JPEG:n02094433 +ILSVRC2012_val_00037839.JPEG:n04264628 +ILSVRC2012_val_00037840.JPEG:n02992211 +ILSVRC2012_val_00037841.JPEG:n02093428 +ILSVRC2012_val_00037842.JPEG:n02100735 +ILSVRC2012_val_00037843.JPEG:n04367480 +ILSVRC2012_val_00037844.JPEG:n03764736 +ILSVRC2012_val_00037845.JPEG:n03041632 +ILSVRC2012_val_00037846.JPEG:n01443537 +ILSVRC2012_val_00037847.JPEG:n03476684 +ILSVRC2012_val_00037848.JPEG:n09229709 +ILSVRC2012_val_00037849.JPEG:n04355338 +ILSVRC2012_val_00037850.JPEG:n02128385 +ILSVRC2012_val_00037851.JPEG:n04550184 +ILSVRC2012_val_00037852.JPEG:n01806567 +ILSVRC2012_val_00037853.JPEG:n02098413 +ILSVRC2012_val_00037854.JPEG:n04086273 +ILSVRC2012_val_00037855.JPEG:n02090379 +ILSVRC2012_val_00037856.JPEG:n03958227 +ILSVRC2012_val_00037857.JPEG:n02091467 +ILSVRC2012_val_00037858.JPEG:n02108000 +ILSVRC2012_val_00037859.JPEG:n03658185 +ILSVRC2012_val_00037860.JPEG:n02843684 +ILSVRC2012_val_00037861.JPEG:n01440764 +ILSVRC2012_val_00037862.JPEG:n02981792 +ILSVRC2012_val_00037863.JPEG:n07892512 +ILSVRC2012_val_00037864.JPEG:n03297495 +ILSVRC2012_val_00037865.JPEG:n03692522 +ILSVRC2012_val_00037866.JPEG:n03937543 +ILSVRC2012_val_00037867.JPEG:n03691459 +ILSVRC2012_val_00037868.JPEG:n03240683 +ILSVRC2012_val_00037869.JPEG:n02977058 +ILSVRC2012_val_00037870.JPEG:n07730033 +ILSVRC2012_val_00037871.JPEG:n04591713 +ILSVRC2012_val_00037872.JPEG:n11939491 +ILSVRC2012_val_00037873.JPEG:n03902125 +ILSVRC2012_val_00037874.JPEG:n02783161 +ILSVRC2012_val_00037875.JPEG:n04355338 +ILSVRC2012_val_00037876.JPEG:n02281406 +ILSVRC2012_val_00037877.JPEG:n03538406 +ILSVRC2012_val_00037878.JPEG:n01608432 +ILSVRC2012_val_00037879.JPEG:n03935335 +ILSVRC2012_val_00037880.JPEG:n01983481 +ILSVRC2012_val_00037881.JPEG:n02730930 +ILSVRC2012_val_00037882.JPEG:n01968897 +ILSVRC2012_val_00037883.JPEG:n03769881 +ILSVRC2012_val_00037884.JPEG:n04493381 +ILSVRC2012_val_00037885.JPEG:n02112018 +ILSVRC2012_val_00037886.JPEG:n02391049 +ILSVRC2012_val_00037887.JPEG:n04389033 +ILSVRC2012_val_00037888.JPEG:n03775546 +ILSVRC2012_val_00037889.JPEG:n02172182 +ILSVRC2012_val_00037890.JPEG:n09399592 +ILSVRC2012_val_00037891.JPEG:n02093991 +ILSVRC2012_val_00037892.JPEG:n01806143 +ILSVRC2012_val_00037893.JPEG:n02226429 +ILSVRC2012_val_00037894.JPEG:n01669191 +ILSVRC2012_val_00037895.JPEG:n04125021 +ILSVRC2012_val_00037896.JPEG:n02113712 +ILSVRC2012_val_00037897.JPEG:n02860847 +ILSVRC2012_val_00037898.JPEG:n02074367 +ILSVRC2012_val_00037899.JPEG:n02447366 +ILSVRC2012_val_00037900.JPEG:n02783161 +ILSVRC2012_val_00037901.JPEG:n02454379 +ILSVRC2012_val_00037902.JPEG:n01984695 +ILSVRC2012_val_00037903.JPEG:n03721384 +ILSVRC2012_val_00037904.JPEG:n03633091 +ILSVRC2012_val_00037905.JPEG:n03376595 +ILSVRC2012_val_00037906.JPEG:n02120505 +ILSVRC2012_val_00037907.JPEG:n02105505 +ILSVRC2012_val_00037908.JPEG:n04517823 +ILSVRC2012_val_00037909.JPEG:n03372029 +ILSVRC2012_val_00037910.JPEG:n03527444 +ILSVRC2012_val_00037911.JPEG:n03786901 +ILSVRC2012_val_00037912.JPEG:n03478589 +ILSVRC2012_val_00037913.JPEG:n02066245 +ILSVRC2012_val_00037914.JPEG:n07892512 +ILSVRC2012_val_00037915.JPEG:n01491361 +ILSVRC2012_val_00037916.JPEG:n02108089 +ILSVRC2012_val_00037917.JPEG:n03325584 +ILSVRC2012_val_00037918.JPEG:n03717622 +ILSVRC2012_val_00037919.JPEG:n03773504 +ILSVRC2012_val_00037920.JPEG:n01582220 +ILSVRC2012_val_00037921.JPEG:n03676483 +ILSVRC2012_val_00037922.JPEG:n04540053 +ILSVRC2012_val_00037923.JPEG:n07248320 +ILSVRC2012_val_00037924.JPEG:n04118538 +ILSVRC2012_val_00037925.JPEG:n02095314 +ILSVRC2012_val_00037926.JPEG:n12267677 +ILSVRC2012_val_00037927.JPEG:n03602883 +ILSVRC2012_val_00037928.JPEG:n02815834 +ILSVRC2012_val_00037929.JPEG:n03379051 +ILSVRC2012_val_00037930.JPEG:n02172182 +ILSVRC2012_val_00037931.JPEG:n02107142 +ILSVRC2012_val_00037932.JPEG:n06874185 +ILSVRC2012_val_00037933.JPEG:n01776313 +ILSVRC2012_val_00037934.JPEG:n07714571 +ILSVRC2012_val_00037935.JPEG:n01775062 +ILSVRC2012_val_00037936.JPEG:n03452741 +ILSVRC2012_val_00037937.JPEG:n03916031 +ILSVRC2012_val_00037938.JPEG:n04118538 +ILSVRC2012_val_00037939.JPEG:n01580077 +ILSVRC2012_val_00037940.JPEG:n02497673 +ILSVRC2012_val_00037941.JPEG:n01518878 +ILSVRC2012_val_00037942.JPEG:n03673027 +ILSVRC2012_val_00037943.JPEG:n02101388 +ILSVRC2012_val_00037944.JPEG:n03187595 +ILSVRC2012_val_00037945.JPEG:n04350905 +ILSVRC2012_val_00037946.JPEG:n02408429 +ILSVRC2012_val_00037947.JPEG:n03417042 +ILSVRC2012_val_00037948.JPEG:n02514041 +ILSVRC2012_val_00037949.JPEG:n02116738 +ILSVRC2012_val_00037950.JPEG:n03476684 +ILSVRC2012_val_00037951.JPEG:n02497673 +ILSVRC2012_val_00037952.JPEG:n04285008 +ILSVRC2012_val_00037953.JPEG:n03126707 +ILSVRC2012_val_00037954.JPEG:n03544143 +ILSVRC2012_val_00037955.JPEG:n04147183 +ILSVRC2012_val_00037956.JPEG:n03481172 +ILSVRC2012_val_00037957.JPEG:n04041544 +ILSVRC2012_val_00037958.JPEG:n02268443 +ILSVRC2012_val_00037959.JPEG:n09472597 +ILSVRC2012_val_00037960.JPEG:n02085782 +ILSVRC2012_val_00037961.JPEG:n03400231 +ILSVRC2012_val_00037962.JPEG:n03954731 +ILSVRC2012_val_00037963.JPEG:n04074963 +ILSVRC2012_val_00037964.JPEG:n03782006 +ILSVRC2012_val_00037965.JPEG:n02281787 +ILSVRC2012_val_00037966.JPEG:n04023962 +ILSVRC2012_val_00037967.JPEG:n04008634 +ILSVRC2012_val_00037968.JPEG:n07875152 +ILSVRC2012_val_00037969.JPEG:n07716906 +ILSVRC2012_val_00037970.JPEG:n02109525 +ILSVRC2012_val_00037971.JPEG:n03995372 +ILSVRC2012_val_00037972.JPEG:n02096177 +ILSVRC2012_val_00037973.JPEG:n01981276 +ILSVRC2012_val_00037974.JPEG:n03884397 +ILSVRC2012_val_00037975.JPEG:n02509815 +ILSVRC2012_val_00037976.JPEG:n03529860 +ILSVRC2012_val_00037977.JPEG:n03584829 +ILSVRC2012_val_00037978.JPEG:n02268853 +ILSVRC2012_val_00037979.JPEG:n04141975 +ILSVRC2012_val_00037980.JPEG:n04599235 +ILSVRC2012_val_00037981.JPEG:n03759954 +ILSVRC2012_val_00037982.JPEG:n02894605 +ILSVRC2012_val_00037983.JPEG:n02454379 +ILSVRC2012_val_00037984.JPEG:n03014705 +ILSVRC2012_val_00037985.JPEG:n02786058 +ILSVRC2012_val_00037986.JPEG:n04505470 +ILSVRC2012_val_00037987.JPEG:n02172182 +ILSVRC2012_val_00037988.JPEG:n02979186 +ILSVRC2012_val_00037989.JPEG:n02091635 +ILSVRC2012_val_00037990.JPEG:n02007558 +ILSVRC2012_val_00037991.JPEG:n02797295 +ILSVRC2012_val_00037992.JPEG:n02817516 +ILSVRC2012_val_00037993.JPEG:n02233338 +ILSVRC2012_val_00037994.JPEG:n04099969 +ILSVRC2012_val_00037995.JPEG:n03250847 +ILSVRC2012_val_00037996.JPEG:n02950826 +ILSVRC2012_val_00037997.JPEG:n02124075 +ILSVRC2012_val_00037998.JPEG:n01484850 +ILSVRC2012_val_00037999.JPEG:n02096294 +ILSVRC2012_val_00038000.JPEG:n02965783 +ILSVRC2012_val_00038001.JPEG:n01943899 +ILSVRC2012_val_00038002.JPEG:n02028035 +ILSVRC2012_val_00038003.JPEG:n04486054 +ILSVRC2012_val_00038004.JPEG:n02417914 +ILSVRC2012_val_00038005.JPEG:n03445777 +ILSVRC2012_val_00038006.JPEG:n04009552 +ILSVRC2012_val_00038007.JPEG:n02125311 +ILSVRC2012_val_00038008.JPEG:n03770439 +ILSVRC2012_val_00038009.JPEG:n02018207 +ILSVRC2012_val_00038010.JPEG:n02219486 +ILSVRC2012_val_00038011.JPEG:n04111531 +ILSVRC2012_val_00038012.JPEG:n09288635 +ILSVRC2012_val_00038013.JPEG:n03825788 +ILSVRC2012_val_00038014.JPEG:n03223299 +ILSVRC2012_val_00038015.JPEG:n04606251 +ILSVRC2012_val_00038016.JPEG:n02396427 +ILSVRC2012_val_00038017.JPEG:n07717410 +ILSVRC2012_val_00038018.JPEG:n02111277 +ILSVRC2012_val_00038019.JPEG:n04515003 +ILSVRC2012_val_00038020.JPEG:n02643566 +ILSVRC2012_val_00038021.JPEG:n03733131 +ILSVRC2012_val_00038022.JPEG:n02093428 +ILSVRC2012_val_00038023.JPEG:n01807496 +ILSVRC2012_val_00038024.JPEG:n02480855 +ILSVRC2012_val_00038025.JPEG:n03527444 +ILSVRC2012_val_00038026.JPEG:n02099849 +ILSVRC2012_val_00038027.JPEG:n04482393 +ILSVRC2012_val_00038028.JPEG:n02361337 +ILSVRC2012_val_00038029.JPEG:n02107574 +ILSVRC2012_val_00038030.JPEG:n04201297 +ILSVRC2012_val_00038031.JPEG:n03633091 +ILSVRC2012_val_00038032.JPEG:n04033995 +ILSVRC2012_val_00038033.JPEG:n02641379 +ILSVRC2012_val_00038034.JPEG:n02790996 +ILSVRC2012_val_00038035.JPEG:n02190166 +ILSVRC2012_val_00038036.JPEG:n03127747 +ILSVRC2012_val_00038037.JPEG:n02483362 +ILSVRC2012_val_00038038.JPEG:n03126707 +ILSVRC2012_val_00038039.JPEG:n03590841 +ILSVRC2012_val_00038040.JPEG:n07717410 +ILSVRC2012_val_00038041.JPEG:n04033901 +ILSVRC2012_val_00038042.JPEG:n02676566 +ILSVRC2012_val_00038043.JPEG:n07875152 +ILSVRC2012_val_00038044.JPEG:n02100236 +ILSVRC2012_val_00038045.JPEG:n04584207 +ILSVRC2012_val_00038046.JPEG:n01737021 +ILSVRC2012_val_00038047.JPEG:n02493509 +ILSVRC2012_val_00038048.JPEG:n02105251 +ILSVRC2012_val_00038049.JPEG:n03930630 +ILSVRC2012_val_00038050.JPEG:n03873416 +ILSVRC2012_val_00038051.JPEG:n02396427 +ILSVRC2012_val_00038052.JPEG:n02493793 +ILSVRC2012_val_00038053.JPEG:n03250847 +ILSVRC2012_val_00038054.JPEG:n02088466 +ILSVRC2012_val_00038055.JPEG:n02814533 +ILSVRC2012_val_00038056.JPEG:n02108000 +ILSVRC2012_val_00038057.JPEG:n01443537 +ILSVRC2012_val_00038058.JPEG:n02988304 +ILSVRC2012_val_00038059.JPEG:n01944390 +ILSVRC2012_val_00038060.JPEG:n04285008 +ILSVRC2012_val_00038061.JPEG:n04356056 +ILSVRC2012_val_00038062.JPEG:n01930112 +ILSVRC2012_val_00038063.JPEG:n03630383 +ILSVRC2012_val_00038064.JPEG:n02281406 +ILSVRC2012_val_00038065.JPEG:n02346627 +ILSVRC2012_val_00038066.JPEG:n04493381 +ILSVRC2012_val_00038067.JPEG:n03709823 +ILSVRC2012_val_00038068.JPEG:n01755581 +ILSVRC2012_val_00038069.JPEG:n02018795 +ILSVRC2012_val_00038070.JPEG:n07802026 +ILSVRC2012_val_00038071.JPEG:n11939491 +ILSVRC2012_val_00038072.JPEG:n07836838 +ILSVRC2012_val_00038073.JPEG:n04429376 +ILSVRC2012_val_00038074.JPEG:n03967562 +ILSVRC2012_val_00038075.JPEG:n02113023 +ILSVRC2012_val_00038076.JPEG:n03724870 +ILSVRC2012_val_00038077.JPEG:n03792972 +ILSVRC2012_val_00038078.JPEG:n01753488 +ILSVRC2012_val_00038079.JPEG:n07875152 +ILSVRC2012_val_00038080.JPEG:n07753592 +ILSVRC2012_val_00038081.JPEG:n04357314 +ILSVRC2012_val_00038082.JPEG:n03642806 +ILSVRC2012_val_00038083.JPEG:n04131690 +ILSVRC2012_val_00038084.JPEG:n04258138 +ILSVRC2012_val_00038085.JPEG:n01667114 +ILSVRC2012_val_00038086.JPEG:n02782093 +ILSVRC2012_val_00038087.JPEG:n02493509 +ILSVRC2012_val_00038088.JPEG:n04465501 +ILSVRC2012_val_00038089.JPEG:n07583066 +ILSVRC2012_val_00038090.JPEG:n02256656 +ILSVRC2012_val_00038091.JPEG:n01532829 +ILSVRC2012_val_00038092.JPEG:n01872401 +ILSVRC2012_val_00038093.JPEG:n07684084 +ILSVRC2012_val_00038094.JPEG:n03763968 +ILSVRC2012_val_00038095.JPEG:n04579145 +ILSVRC2012_val_00038096.JPEG:n03492542 +ILSVRC2012_val_00038097.JPEG:n04417672 +ILSVRC2012_val_00038098.JPEG:n04350905 +ILSVRC2012_val_00038099.JPEG:n04069434 +ILSVRC2012_val_00038100.JPEG:n03866082 +ILSVRC2012_val_00038101.JPEG:n04311174 +ILSVRC2012_val_00038102.JPEG:n01756291 +ILSVRC2012_val_00038103.JPEG:n02797295 +ILSVRC2012_val_00038104.JPEG:n03642806 +ILSVRC2012_val_00038105.JPEG:n03676483 +ILSVRC2012_val_00038106.JPEG:n03697007 +ILSVRC2012_val_00038107.JPEG:n02087046 +ILSVRC2012_val_00038108.JPEG:n03207941 +ILSVRC2012_val_00038109.JPEG:n04201297 +ILSVRC2012_val_00038110.JPEG:n02074367 +ILSVRC2012_val_00038111.JPEG:n01608432 +ILSVRC2012_val_00038112.JPEG:n02111500 +ILSVRC2012_val_00038113.JPEG:n03633091 +ILSVRC2012_val_00038114.JPEG:n02804610 +ILSVRC2012_val_00038115.JPEG:n04562935 +ILSVRC2012_val_00038116.JPEG:n02093859 +ILSVRC2012_val_00038117.JPEG:n03935335 +ILSVRC2012_val_00038118.JPEG:n02051845 +ILSVRC2012_val_00038119.JPEG:n01990800 +ILSVRC2012_val_00038120.JPEG:n02799071 +ILSVRC2012_val_00038121.JPEG:n04228054 +ILSVRC2012_val_00038122.JPEG:n02100877 +ILSVRC2012_val_00038123.JPEG:n01755581 +ILSVRC2012_val_00038124.JPEG:n02129604 +ILSVRC2012_val_00038125.JPEG:n02727426 +ILSVRC2012_val_00038126.JPEG:n01860187 +ILSVRC2012_val_00038127.JPEG:n04326547 +ILSVRC2012_val_00038128.JPEG:n03776460 +ILSVRC2012_val_00038129.JPEG:n02206856 +ILSVRC2012_val_00038130.JPEG:n02093256 +ILSVRC2012_val_00038131.JPEG:n01968897 +ILSVRC2012_val_00038132.JPEG:n02326432 +ILSVRC2012_val_00038133.JPEG:n03770679 +ILSVRC2012_val_00038134.JPEG:n02509815 +ILSVRC2012_val_00038135.JPEG:n02978881 +ILSVRC2012_val_00038136.JPEG:n03018349 +ILSVRC2012_val_00038137.JPEG:n03394916 +ILSVRC2012_val_00038138.JPEG:n02977058 +ILSVRC2012_val_00038139.JPEG:n03891332 +ILSVRC2012_val_00038140.JPEG:n01665541 +ILSVRC2012_val_00038141.JPEG:n04141327 +ILSVRC2012_val_00038142.JPEG:n02233338 +ILSVRC2012_val_00038143.JPEG:n02092339 +ILSVRC2012_val_00038144.JPEG:n03388549 +ILSVRC2012_val_00038145.JPEG:n04548362 +ILSVRC2012_val_00038146.JPEG:n04296562 +ILSVRC2012_val_00038147.JPEG:n04067472 +ILSVRC2012_val_00038148.JPEG:n03014705 +ILSVRC2012_val_00038149.JPEG:n02747177 +ILSVRC2012_val_00038150.JPEG:n02441942 +ILSVRC2012_val_00038151.JPEG:n04081281 +ILSVRC2012_val_00038152.JPEG:n03290653 +ILSVRC2012_val_00038153.JPEG:n02066245 +ILSVRC2012_val_00038154.JPEG:n01983481 +ILSVRC2012_val_00038155.JPEG:n02085936 +ILSVRC2012_val_00038156.JPEG:n01518878 +ILSVRC2012_val_00038157.JPEG:n02085620 +ILSVRC2012_val_00038158.JPEG:n04346328 +ILSVRC2012_val_00038159.JPEG:n01601694 +ILSVRC2012_val_00038160.JPEG:n01532829 +ILSVRC2012_val_00038161.JPEG:n03992509 +ILSVRC2012_val_00038162.JPEG:n01694178 +ILSVRC2012_val_00038163.JPEG:n02437616 +ILSVRC2012_val_00038164.JPEG:n04612504 +ILSVRC2012_val_00038165.JPEG:n02666196 +ILSVRC2012_val_00038166.JPEG:n03950228 +ILSVRC2012_val_00038167.JPEG:n02093754 +ILSVRC2012_val_00038168.JPEG:n02123597 +ILSVRC2012_val_00038169.JPEG:n01817953 +ILSVRC2012_val_00038170.JPEG:n02190166 +ILSVRC2012_val_00038171.JPEG:n04067472 +ILSVRC2012_val_00038172.JPEG:n03933933 +ILSVRC2012_val_00038173.JPEG:n02398521 +ILSVRC2012_val_00038174.JPEG:n02097130 +ILSVRC2012_val_00038175.JPEG:n03444034 +ILSVRC2012_val_00038176.JPEG:n03792972 +ILSVRC2012_val_00038177.JPEG:n04418357 +ILSVRC2012_val_00038178.JPEG:n01871265 +ILSVRC2012_val_00038179.JPEG:n03208938 +ILSVRC2012_val_00038180.JPEG:n01768244 +ILSVRC2012_val_00038181.JPEG:n02174001 +ILSVRC2012_val_00038182.JPEG:n02219486 +ILSVRC2012_val_00038183.JPEG:n01774384 +ILSVRC2012_val_00038184.JPEG:n07742313 +ILSVRC2012_val_00038185.JPEG:n04355933 +ILSVRC2012_val_00038186.JPEG:n02129165 +ILSVRC2012_val_00038187.JPEG:n07742313 +ILSVRC2012_val_00038188.JPEG:n01697457 +ILSVRC2012_val_00038189.JPEG:n04310018 +ILSVRC2012_val_00038190.JPEG:n02669723 +ILSVRC2012_val_00038191.JPEG:n04367480 +ILSVRC2012_val_00038192.JPEG:n01592084 +ILSVRC2012_val_00038193.JPEG:n02105251 +ILSVRC2012_val_00038194.JPEG:n02113799 +ILSVRC2012_val_00038195.JPEG:n07565083 +ILSVRC2012_val_00038196.JPEG:n02091032 +ILSVRC2012_val_00038197.JPEG:n02011460 +ILSVRC2012_val_00038198.JPEG:n03773504 +ILSVRC2012_val_00038199.JPEG:n02445715 +ILSVRC2012_val_00038200.JPEG:n04275548 +ILSVRC2012_val_00038201.JPEG:n02112018 +ILSVRC2012_val_00038202.JPEG:n01632458 +ILSVRC2012_val_00038203.JPEG:n02486261 +ILSVRC2012_val_00038204.JPEG:n07714990 +ILSVRC2012_val_00038205.JPEG:n02106550 +ILSVRC2012_val_00038206.JPEG:n03478589 +ILSVRC2012_val_00038207.JPEG:n02963159 +ILSVRC2012_val_00038208.JPEG:n03743016 +ILSVRC2012_val_00038209.JPEG:n04146614 +ILSVRC2012_val_00038210.JPEG:n03970156 +ILSVRC2012_val_00038211.JPEG:n03874293 +ILSVRC2012_val_00038212.JPEG:n07749582 +ILSVRC2012_val_00038213.JPEG:n06874185 +ILSVRC2012_val_00038214.JPEG:n01950731 +ILSVRC2012_val_00038215.JPEG:n01498041 +ILSVRC2012_val_00038216.JPEG:n04090263 +ILSVRC2012_val_00038217.JPEG:n02077923 +ILSVRC2012_val_00038218.JPEG:n02106662 +ILSVRC2012_val_00038219.JPEG:n02786058 +ILSVRC2012_val_00038220.JPEG:n04591157 +ILSVRC2012_val_00038221.JPEG:n03481172 +ILSVRC2012_val_00038222.JPEG:n03924679 +ILSVRC2012_val_00038223.JPEG:n02500267 +ILSVRC2012_val_00038224.JPEG:n04258138 +ILSVRC2012_val_00038225.JPEG:n04540053 +ILSVRC2012_val_00038226.JPEG:n03160309 +ILSVRC2012_val_00038227.JPEG:n02087394 +ILSVRC2012_val_00038228.JPEG:n03494278 +ILSVRC2012_val_00038229.JPEG:n04325704 +ILSVRC2012_val_00038230.JPEG:n01669191 +ILSVRC2012_val_00038231.JPEG:n02108551 +ILSVRC2012_val_00038232.JPEG:n01980166 +ILSVRC2012_val_00038233.JPEG:n03314780 +ILSVRC2012_val_00038234.JPEG:n02808440 +ILSVRC2012_val_00038235.JPEG:n04447861 +ILSVRC2012_val_00038236.JPEG:n02281787 +ILSVRC2012_val_00038237.JPEG:n02095889 +ILSVRC2012_val_00038238.JPEG:n02489166 +ILSVRC2012_val_00038239.JPEG:n02114367 +ILSVRC2012_val_00038240.JPEG:n04344873 +ILSVRC2012_val_00038241.JPEG:n02058221 +ILSVRC2012_val_00038242.JPEG:n02444819 +ILSVRC2012_val_00038243.JPEG:n02988304 +ILSVRC2012_val_00038244.JPEG:n03495258 +ILSVRC2012_val_00038245.JPEG:n02002556 +ILSVRC2012_val_00038246.JPEG:n03874293 +ILSVRC2012_val_00038247.JPEG:n02085782 +ILSVRC2012_val_00038248.JPEG:n01695060 +ILSVRC2012_val_00038249.JPEG:n02870880 +ILSVRC2012_val_00038250.JPEG:n01608432 +ILSVRC2012_val_00038251.JPEG:n02948072 +ILSVRC2012_val_00038252.JPEG:n04067472 +ILSVRC2012_val_00038253.JPEG:n02098286 +ILSVRC2012_val_00038254.JPEG:n02093428 +ILSVRC2012_val_00038255.JPEG:n04009552 +ILSVRC2012_val_00038256.JPEG:n12267677 +ILSVRC2012_val_00038257.JPEG:n02085782 +ILSVRC2012_val_00038258.JPEG:n03376595 +ILSVRC2012_val_00038259.JPEG:n04335435 +ILSVRC2012_val_00038260.JPEG:n03891332 +ILSVRC2012_val_00038261.JPEG:n03733281 +ILSVRC2012_val_00038262.JPEG:n02264363 +ILSVRC2012_val_00038263.JPEG:n02132136 +ILSVRC2012_val_00038264.JPEG:n04263257 +ILSVRC2012_val_00038265.JPEG:n01698640 +ILSVRC2012_val_00038266.JPEG:n01753488 +ILSVRC2012_val_00038267.JPEG:n07714990 +ILSVRC2012_val_00038268.JPEG:n03417042 +ILSVRC2012_val_00038269.JPEG:n03259280 +ILSVRC2012_val_00038270.JPEG:n01737021 +ILSVRC2012_val_00038271.JPEG:n04118538 +ILSVRC2012_val_00038272.JPEG:n01773797 +ILSVRC2012_val_00038273.JPEG:n03124170 +ILSVRC2012_val_00038274.JPEG:n03874293 +ILSVRC2012_val_00038275.JPEG:n09421951 +ILSVRC2012_val_00038276.JPEG:n02747177 +ILSVRC2012_val_00038277.JPEG:n09288635 +ILSVRC2012_val_00038278.JPEG:n04136333 +ILSVRC2012_val_00038279.JPEG:n03956157 +ILSVRC2012_val_00038280.JPEG:n02093256 +ILSVRC2012_val_00038281.JPEG:n03729826 +ILSVRC2012_val_00038282.JPEG:n03538406 +ILSVRC2012_val_00038283.JPEG:n01774384 +ILSVRC2012_val_00038284.JPEG:n04355338 +ILSVRC2012_val_00038285.JPEG:n02105251 +ILSVRC2012_val_00038286.JPEG:n02403003 +ILSVRC2012_val_00038287.JPEG:n01697457 +ILSVRC2012_val_00038288.JPEG:n01828970 +ILSVRC2012_val_00038289.JPEG:n02892767 +ILSVRC2012_val_00038290.JPEG:n02018207 +ILSVRC2012_val_00038291.JPEG:n02134084 +ILSVRC2012_val_00038292.JPEG:n03733805 +ILSVRC2012_val_00038293.JPEG:n07930864 +ILSVRC2012_val_00038294.JPEG:n02097474 +ILSVRC2012_val_00038295.JPEG:n04507155 +ILSVRC2012_val_00038296.JPEG:n04344873 +ILSVRC2012_val_00038297.JPEG:n02950826 +ILSVRC2012_val_00038298.JPEG:n03721384 +ILSVRC2012_val_00038299.JPEG:n01943899 +ILSVRC2012_val_00038300.JPEG:n07920052 +ILSVRC2012_val_00038301.JPEG:n02319095 +ILSVRC2012_val_00038302.JPEG:n04149813 +ILSVRC2012_val_00038303.JPEG:n02364673 +ILSVRC2012_val_00038304.JPEG:n01742172 +ILSVRC2012_val_00038305.JPEG:n04428191 +ILSVRC2012_val_00038306.JPEG:n03450230 +ILSVRC2012_val_00038307.JPEG:n09399592 +ILSVRC2012_val_00038308.JPEG:n01689811 +ILSVRC2012_val_00038309.JPEG:n01978287 +ILSVRC2012_val_00038310.JPEG:n07716358 +ILSVRC2012_val_00038311.JPEG:n02074367 +ILSVRC2012_val_00038312.JPEG:n04557648 +ILSVRC2012_val_00038313.JPEG:n03062245 +ILSVRC2012_val_00038314.JPEG:n02105251 +ILSVRC2012_val_00038315.JPEG:n07716906 +ILSVRC2012_val_00038316.JPEG:n03623198 +ILSVRC2012_val_00038317.JPEG:n03125729 +ILSVRC2012_val_00038318.JPEG:n03876231 +ILSVRC2012_val_00038319.JPEG:n04509417 +ILSVRC2012_val_00038320.JPEG:n03041632 +ILSVRC2012_val_00038321.JPEG:n04347754 +ILSVRC2012_val_00038322.JPEG:n06359193 +ILSVRC2012_val_00038323.JPEG:n04118538 +ILSVRC2012_val_00038324.JPEG:n01806143 +ILSVRC2012_val_00038325.JPEG:n07749582 +ILSVRC2012_val_00038326.JPEG:n02105855 +ILSVRC2012_val_00038327.JPEG:n13052670 +ILSVRC2012_val_00038328.JPEG:n02094114 +ILSVRC2012_val_00038329.JPEG:n03775071 +ILSVRC2012_val_00038330.JPEG:n01873310 +ILSVRC2012_val_00038331.JPEG:n03788195 +ILSVRC2012_val_00038332.JPEG:n04311004 +ILSVRC2012_val_00038333.JPEG:n03018349 +ILSVRC2012_val_00038334.JPEG:n03089624 +ILSVRC2012_val_00038335.JPEG:n02087046 +ILSVRC2012_val_00038336.JPEG:n03379051 +ILSVRC2012_val_00038337.JPEG:n04493381 +ILSVRC2012_val_00038338.JPEG:n07714990 +ILSVRC2012_val_00038339.JPEG:n03895866 +ILSVRC2012_val_00038340.JPEG:n15075141 +ILSVRC2012_val_00038341.JPEG:n07684084 +ILSVRC2012_val_00038342.JPEG:n01755581 +ILSVRC2012_val_00038343.JPEG:n07715103 +ILSVRC2012_val_00038344.JPEG:n04285008 +ILSVRC2012_val_00038345.JPEG:n03476991 +ILSVRC2012_val_00038346.JPEG:n04049303 +ILSVRC2012_val_00038347.JPEG:n03496892 +ILSVRC2012_val_00038348.JPEG:n03041632 +ILSVRC2012_val_00038349.JPEG:n02403003 +ILSVRC2012_val_00038350.JPEG:n03832673 +ILSVRC2012_val_00038351.JPEG:n04131690 +ILSVRC2012_val_00038352.JPEG:n04479046 +ILSVRC2012_val_00038353.JPEG:n04479046 +ILSVRC2012_val_00038354.JPEG:n02259212 +ILSVRC2012_val_00038355.JPEG:n01734418 +ILSVRC2012_val_00038356.JPEG:n02002556 +ILSVRC2012_val_00038357.JPEG:n03179701 +ILSVRC2012_val_00038358.JPEG:n03992509 +ILSVRC2012_val_00038359.JPEG:n07932039 +ILSVRC2012_val_00038360.JPEG:n04467665 +ILSVRC2012_val_00038361.JPEG:n02099712 +ILSVRC2012_val_00038362.JPEG:n04456115 +ILSVRC2012_val_00038363.JPEG:n03690938 +ILSVRC2012_val_00038364.JPEG:n04367480 +ILSVRC2012_val_00038365.JPEG:n01729322 +ILSVRC2012_val_00038366.JPEG:n03961711 +ILSVRC2012_val_00038367.JPEG:n03841143 +ILSVRC2012_val_00038368.JPEG:n02963159 +ILSVRC2012_val_00038369.JPEG:n03476991 +ILSVRC2012_val_00038370.JPEG:n04074963 +ILSVRC2012_val_00038371.JPEG:n02077923 +ILSVRC2012_val_00038372.JPEG:n01532829 +ILSVRC2012_val_00038373.JPEG:n02865351 +ILSVRC2012_val_00038374.JPEG:n02966687 +ILSVRC2012_val_00038375.JPEG:n01694178 +ILSVRC2012_val_00038376.JPEG:n03017168 +ILSVRC2012_val_00038377.JPEG:n04429376 +ILSVRC2012_val_00038378.JPEG:n03935335 +ILSVRC2012_val_00038379.JPEG:n09246464 +ILSVRC2012_val_00038380.JPEG:n04004767 +ILSVRC2012_val_00038381.JPEG:n03208938 +ILSVRC2012_val_00038382.JPEG:n04111531 +ILSVRC2012_val_00038383.JPEG:n04389033 +ILSVRC2012_val_00038384.JPEG:n07760859 +ILSVRC2012_val_00038385.JPEG:n04326547 +ILSVRC2012_val_00038386.JPEG:n04209239 +ILSVRC2012_val_00038387.JPEG:n07697537 +ILSVRC2012_val_00038388.JPEG:n03785016 +ILSVRC2012_val_00038389.JPEG:n04367480 +ILSVRC2012_val_00038390.JPEG:n04037443 +ILSVRC2012_val_00038391.JPEG:n04311174 +ILSVRC2012_val_00038392.JPEG:n02814533 +ILSVRC2012_val_00038393.JPEG:n02113799 +ILSVRC2012_val_00038394.JPEG:n02825657 +ILSVRC2012_val_00038395.JPEG:n02672831 +ILSVRC2012_val_00038396.JPEG:n02114855 +ILSVRC2012_val_00038397.JPEG:n02090622 +ILSVRC2012_val_00038398.JPEG:n09399592 +ILSVRC2012_val_00038399.JPEG:n04482393 +ILSVRC2012_val_00038400.JPEG:n01910747 +ILSVRC2012_val_00038401.JPEG:n04417672 +ILSVRC2012_val_00038402.JPEG:n04162706 +ILSVRC2012_val_00038403.JPEG:n02098413 +ILSVRC2012_val_00038404.JPEG:n07717556 +ILSVRC2012_val_00038405.JPEG:n01580077 +ILSVRC2012_val_00038406.JPEG:n02092002 +ILSVRC2012_val_00038407.JPEG:n03014705 +ILSVRC2012_val_00038408.JPEG:n04370456 +ILSVRC2012_val_00038409.JPEG:n02835271 +ILSVRC2012_val_00038410.JPEG:n03047690 +ILSVRC2012_val_00038411.JPEG:n03944341 +ILSVRC2012_val_00038412.JPEG:n07613480 +ILSVRC2012_val_00038413.JPEG:n02361337 +ILSVRC2012_val_00038414.JPEG:n02356798 +ILSVRC2012_val_00038415.JPEG:n02835271 +ILSVRC2012_val_00038416.JPEG:n02011460 +ILSVRC2012_val_00038417.JPEG:n02096051 +ILSVRC2012_val_00038418.JPEG:n01843065 +ILSVRC2012_val_00038419.JPEG:n03498962 +ILSVRC2012_val_00038420.JPEG:n07583066 +ILSVRC2012_val_00038421.JPEG:n07734744 +ILSVRC2012_val_00038422.JPEG:n04277352 +ILSVRC2012_val_00038423.JPEG:n02088632 +ILSVRC2012_val_00038424.JPEG:n09835506 +ILSVRC2012_val_00038425.JPEG:n04141327 +ILSVRC2012_val_00038426.JPEG:n01820546 +ILSVRC2012_val_00038427.JPEG:n03218198 +ILSVRC2012_val_00038428.JPEG:n03825788 +ILSVRC2012_val_00038429.JPEG:n04310018 +ILSVRC2012_val_00038430.JPEG:n02099849 +ILSVRC2012_val_00038431.JPEG:n02025239 +ILSVRC2012_val_00038432.JPEG:n07753275 +ILSVRC2012_val_00038433.JPEG:n03876231 +ILSVRC2012_val_00038434.JPEG:n02099267 +ILSVRC2012_val_00038435.JPEG:n03794056 +ILSVRC2012_val_00038436.JPEG:n07590611 +ILSVRC2012_val_00038437.JPEG:n01740131 +ILSVRC2012_val_00038438.JPEG:n02091032 +ILSVRC2012_val_00038439.JPEG:n04200800 +ILSVRC2012_val_00038440.JPEG:n01770081 +ILSVRC2012_val_00038441.JPEG:n02869837 +ILSVRC2012_val_00038442.JPEG:n03379051 +ILSVRC2012_val_00038443.JPEG:n01833805 +ILSVRC2012_val_00038444.JPEG:n03929855 +ILSVRC2012_val_00038445.JPEG:n02749479 +ILSVRC2012_val_00038446.JPEG:n01644900 +ILSVRC2012_val_00038447.JPEG:n03445777 +ILSVRC2012_val_00038448.JPEG:n02110627 +ILSVRC2012_val_00038449.JPEG:n01630670 +ILSVRC2012_val_00038450.JPEG:n04273569 +ILSVRC2012_val_00038451.JPEG:n04483307 +ILSVRC2012_val_00038452.JPEG:n02138441 +ILSVRC2012_val_00038453.JPEG:n07892512 +ILSVRC2012_val_00038454.JPEG:n01983481 +ILSVRC2012_val_00038455.JPEG:n02108422 +ILSVRC2012_val_00038456.JPEG:n02948072 +ILSVRC2012_val_00038457.JPEG:n02094258 +ILSVRC2012_val_00038458.JPEG:n03141823 +ILSVRC2012_val_00038459.JPEG:n01632458 +ILSVRC2012_val_00038460.JPEG:n04517823 +ILSVRC2012_val_00038461.JPEG:n04380533 +ILSVRC2012_val_00038462.JPEG:n09472597 +ILSVRC2012_val_00038463.JPEG:n02165456 +ILSVRC2012_val_00038464.JPEG:n01930112 +ILSVRC2012_val_00038465.JPEG:n03018349 +ILSVRC2012_val_00038466.JPEG:n02268853 +ILSVRC2012_val_00038467.JPEG:n01770081 +ILSVRC2012_val_00038468.JPEG:n04141975 +ILSVRC2012_val_00038469.JPEG:n03998194 +ILSVRC2012_val_00038470.JPEG:n03384352 +ILSVRC2012_val_00038471.JPEG:n04147183 +ILSVRC2012_val_00038472.JPEG:n03045698 +ILSVRC2012_val_00038473.JPEG:n03791053 +ILSVRC2012_val_00038474.JPEG:n03944341 +ILSVRC2012_val_00038475.JPEG:n02536864 +ILSVRC2012_val_00038476.JPEG:n01829413 +ILSVRC2012_val_00038477.JPEG:n02088466 +ILSVRC2012_val_00038478.JPEG:n01694178 +ILSVRC2012_val_00038479.JPEG:n02106382 +ILSVRC2012_val_00038480.JPEG:n01748264 +ILSVRC2012_val_00038481.JPEG:n03759954 +ILSVRC2012_val_00038482.JPEG:n12985857 +ILSVRC2012_val_00038483.JPEG:n04254680 +ILSVRC2012_val_00038484.JPEG:n04465501 +ILSVRC2012_val_00038485.JPEG:n02795169 +ILSVRC2012_val_00038486.JPEG:n02096177 +ILSVRC2012_val_00038487.JPEG:n02444819 +ILSVRC2012_val_00038488.JPEG:n01558993 +ILSVRC2012_val_00038489.JPEG:n02115641 +ILSVRC2012_val_00038490.JPEG:n03445924 +ILSVRC2012_val_00038491.JPEG:n02701002 +ILSVRC2012_val_00038492.JPEG:n06359193 +ILSVRC2012_val_00038493.JPEG:n01773549 +ILSVRC2012_val_00038494.JPEG:n03637318 +ILSVRC2012_val_00038495.JPEG:n02437312 +ILSVRC2012_val_00038496.JPEG:n04332243 +ILSVRC2012_val_00038497.JPEG:n02865351 +ILSVRC2012_val_00038498.JPEG:n02088632 +ILSVRC2012_val_00038499.JPEG:n04067472 +ILSVRC2012_val_00038500.JPEG:n02092002 +ILSVRC2012_val_00038501.JPEG:n03956157 +ILSVRC2012_val_00038502.JPEG:n04326547 +ILSVRC2012_val_00038503.JPEG:n02786058 +ILSVRC2012_val_00038504.JPEG:n01784675 +ILSVRC2012_val_00038505.JPEG:n01847000 +ILSVRC2012_val_00038506.JPEG:n04146614 +ILSVRC2012_val_00038507.JPEG:n03666591 +ILSVRC2012_val_00038508.JPEG:n04310018 +ILSVRC2012_val_00038509.JPEG:n01914609 +ILSVRC2012_val_00038510.JPEG:n07695742 +ILSVRC2012_val_00038511.JPEG:n03404251 +ILSVRC2012_val_00038512.JPEG:n03891251 +ILSVRC2012_val_00038513.JPEG:n06874185 +ILSVRC2012_val_00038514.JPEG:n03062245 +ILSVRC2012_val_00038515.JPEG:n03355925 +ILSVRC2012_val_00038516.JPEG:n12267677 +ILSVRC2012_val_00038517.JPEG:n04254120 +ILSVRC2012_val_00038518.JPEG:n07714990 +ILSVRC2012_val_00038519.JPEG:n02233338 +ILSVRC2012_val_00038520.JPEG:n02804414 +ILSVRC2012_val_00038521.JPEG:n03062245 +ILSVRC2012_val_00038522.JPEG:n02018795 +ILSVRC2012_val_00038523.JPEG:n07720875 +ILSVRC2012_val_00038524.JPEG:n03075370 +ILSVRC2012_val_00038525.JPEG:n03530642 +ILSVRC2012_val_00038526.JPEG:n01980166 +ILSVRC2012_val_00038527.JPEG:n01667114 +ILSVRC2012_val_00038528.JPEG:n04553703 +ILSVRC2012_val_00038529.JPEG:n09468604 +ILSVRC2012_val_00038530.JPEG:n06794110 +ILSVRC2012_val_00038531.JPEG:n04367480 +ILSVRC2012_val_00038532.JPEG:n02963159 +ILSVRC2012_val_00038533.JPEG:n03710193 +ILSVRC2012_val_00038534.JPEG:n01980166 +ILSVRC2012_val_00038535.JPEG:n03000134 +ILSVRC2012_val_00038536.JPEG:n03938244 +ILSVRC2012_val_00038537.JPEG:n02231487 +ILSVRC2012_val_00038538.JPEG:n02493509 +ILSVRC2012_val_00038539.JPEG:n03447721 +ILSVRC2012_val_00038540.JPEG:n07583066 +ILSVRC2012_val_00038541.JPEG:n09472597 +ILSVRC2012_val_00038542.JPEG:n03877845 +ILSVRC2012_val_00038543.JPEG:n04147183 +ILSVRC2012_val_00038544.JPEG:n04229816 +ILSVRC2012_val_00038545.JPEG:n12998815 +ILSVRC2012_val_00038546.JPEG:n03877472 +ILSVRC2012_val_00038547.JPEG:n07718472 +ILSVRC2012_val_00038548.JPEG:n03063599 +ILSVRC2012_val_00038549.JPEG:n01665541 +ILSVRC2012_val_00038550.JPEG:n02111889 +ILSVRC2012_val_00038551.JPEG:n06596364 +ILSVRC2012_val_00038552.JPEG:n02094433 +ILSVRC2012_val_00038553.JPEG:n01817953 +ILSVRC2012_val_00038554.JPEG:n02091635 +ILSVRC2012_val_00038555.JPEG:n01755581 +ILSVRC2012_val_00038556.JPEG:n01740131 +ILSVRC2012_val_00038557.JPEG:n01592084 +ILSVRC2012_val_00038558.JPEG:n03673027 +ILSVRC2012_val_00038559.JPEG:n03467068 +ILSVRC2012_val_00038560.JPEG:n03924679 +ILSVRC2012_val_00038561.JPEG:n04467665 +ILSVRC2012_val_00038562.JPEG:n03733805 +ILSVRC2012_val_00038563.JPEG:n01833805 +ILSVRC2012_val_00038564.JPEG:n03089624 +ILSVRC2012_val_00038565.JPEG:n02091635 +ILSVRC2012_val_00038566.JPEG:n02489166 +ILSVRC2012_val_00038567.JPEG:n02112350 +ILSVRC2012_val_00038568.JPEG:n04192698 +ILSVRC2012_val_00038569.JPEG:n02102040 +ILSVRC2012_val_00038570.JPEG:n02823428 +ILSVRC2012_val_00038571.JPEG:n04074963 +ILSVRC2012_val_00038572.JPEG:n01872401 +ILSVRC2012_val_00038573.JPEG:n04579145 +ILSVRC2012_val_00038574.JPEG:n03788365 +ILSVRC2012_val_00038575.JPEG:n04086273 +ILSVRC2012_val_00038576.JPEG:n02009229 +ILSVRC2012_val_00038577.JPEG:n07753113 +ILSVRC2012_val_00038578.JPEG:n02504458 +ILSVRC2012_val_00038579.JPEG:n02002724 +ILSVRC2012_val_00038580.JPEG:n02097474 +ILSVRC2012_val_00038581.JPEG:n07754684 +ILSVRC2012_val_00038582.JPEG:n03134739 +ILSVRC2012_val_00038583.JPEG:n02113978 +ILSVRC2012_val_00038584.JPEG:n02403003 +ILSVRC2012_val_00038585.JPEG:n03998194 +ILSVRC2012_val_00038586.JPEG:n01688243 +ILSVRC2012_val_00038587.JPEG:n03891332 +ILSVRC2012_val_00038588.JPEG:n04133789 +ILSVRC2012_val_00038589.JPEG:n02111500 +ILSVRC2012_val_00038590.JPEG:n02916936 +ILSVRC2012_val_00038591.JPEG:n07248320 +ILSVRC2012_val_00038592.JPEG:n04404412 +ILSVRC2012_val_00038593.JPEG:n04209239 +ILSVRC2012_val_00038594.JPEG:n07590611 +ILSVRC2012_val_00038595.JPEG:n03673027 +ILSVRC2012_val_00038596.JPEG:n04008634 +ILSVRC2012_val_00038597.JPEG:n03272010 +ILSVRC2012_val_00038598.JPEG:n13040303 +ILSVRC2012_val_00038599.JPEG:n09399592 +ILSVRC2012_val_00038600.JPEG:n02007558 +ILSVRC2012_val_00038601.JPEG:n02488291 +ILSVRC2012_val_00038602.JPEG:n07716906 +ILSVRC2012_val_00038603.JPEG:n04009552 +ILSVRC2012_val_00038604.JPEG:n02111889 +ILSVRC2012_val_00038605.JPEG:n03658185 +ILSVRC2012_val_00038606.JPEG:n01980166 +ILSVRC2012_val_00038607.JPEG:n04367480 +ILSVRC2012_val_00038608.JPEG:n02892201 +ILSVRC2012_val_00038609.JPEG:n04423845 +ILSVRC2012_val_00038610.JPEG:n03131574 +ILSVRC2012_val_00038611.JPEG:n04041544 +ILSVRC2012_val_00038612.JPEG:n04266014 +ILSVRC2012_val_00038613.JPEG:n03825788 +ILSVRC2012_val_00038614.JPEG:n02033041 +ILSVRC2012_val_00038615.JPEG:n02002724 +ILSVRC2012_val_00038616.JPEG:n01871265 +ILSVRC2012_val_00038617.JPEG:n04099969 +ILSVRC2012_val_00038618.JPEG:n02321529 +ILSVRC2012_val_00038619.JPEG:n02666196 +ILSVRC2012_val_00038620.JPEG:n01698640 +ILSVRC2012_val_00038621.JPEG:n03709823 +ILSVRC2012_val_00038622.JPEG:n02356798 +ILSVRC2012_val_00038623.JPEG:n03089624 +ILSVRC2012_val_00038624.JPEG:n03873416 +ILSVRC2012_val_00038625.JPEG:n02097130 +ILSVRC2012_val_00038626.JPEG:n02108089 +ILSVRC2012_val_00038627.JPEG:n04258138 +ILSVRC2012_val_00038628.JPEG:n01667778 +ILSVRC2012_val_00038629.JPEG:n04456115 +ILSVRC2012_val_00038630.JPEG:n03492542 +ILSVRC2012_val_00038631.JPEG:n02363005 +ILSVRC2012_val_00038632.JPEG:n01871265 +ILSVRC2012_val_00038633.JPEG:n01950731 +ILSVRC2012_val_00038634.JPEG:n04153751 +ILSVRC2012_val_00038635.JPEG:n01984695 +ILSVRC2012_val_00038636.JPEG:n01614925 +ILSVRC2012_val_00038637.JPEG:n02110958 +ILSVRC2012_val_00038638.JPEG:n01824575 +ILSVRC2012_val_00038639.JPEG:n01981276 +ILSVRC2012_val_00038640.JPEG:n15075141 +ILSVRC2012_val_00038641.JPEG:n03814906 +ILSVRC2012_val_00038642.JPEG:n03874599 +ILSVRC2012_val_00038643.JPEG:n04118776 +ILSVRC2012_val_00038644.JPEG:n01675722 +ILSVRC2012_val_00038645.JPEG:n02939185 +ILSVRC2012_val_00038646.JPEG:n03742115 +ILSVRC2012_val_00038647.JPEG:n01697457 +ILSVRC2012_val_00038648.JPEG:n02326432 +ILSVRC2012_val_00038649.JPEG:n02090622 +ILSVRC2012_val_00038650.JPEG:n04532106 +ILSVRC2012_val_00038651.JPEG:n03983396 +ILSVRC2012_val_00038652.JPEG:n02415577 +ILSVRC2012_val_00038653.JPEG:n02412080 +ILSVRC2012_val_00038654.JPEG:n02102480 +ILSVRC2012_val_00038655.JPEG:n03459775 +ILSVRC2012_val_00038656.JPEG:n04380533 +ILSVRC2012_val_00038657.JPEG:n04254777 +ILSVRC2012_val_00038658.JPEG:n01631663 +ILSVRC2012_val_00038659.JPEG:n03404251 +ILSVRC2012_val_00038660.JPEG:n07871810 +ILSVRC2012_val_00038661.JPEG:n02123045 +ILSVRC2012_val_00038662.JPEG:n02226429 +ILSVRC2012_val_00038663.JPEG:n01871265 +ILSVRC2012_val_00038664.JPEG:n01820546 +ILSVRC2012_val_00038665.JPEG:n01688243 +ILSVRC2012_val_00038666.JPEG:n02825657 +ILSVRC2012_val_00038667.JPEG:n01689811 +ILSVRC2012_val_00038668.JPEG:n02095570 +ILSVRC2012_val_00038669.JPEG:n04019541 +ILSVRC2012_val_00038670.JPEG:n03777754 +ILSVRC2012_val_00038671.JPEG:n01748264 +ILSVRC2012_val_00038672.JPEG:n02123045 +ILSVRC2012_val_00038673.JPEG:n02129604 +ILSVRC2012_val_00038674.JPEG:n02105056 +ILSVRC2012_val_00038675.JPEG:n02125311 +ILSVRC2012_val_00038676.JPEG:n02089973 +ILSVRC2012_val_00038677.JPEG:n03649909 +ILSVRC2012_val_00038678.JPEG:n04540053 +ILSVRC2012_val_00038679.JPEG:n03670208 +ILSVRC2012_val_00038680.JPEG:n02097209 +ILSVRC2012_val_00038681.JPEG:n01819313 +ILSVRC2012_val_00038682.JPEG:n03110669 +ILSVRC2012_val_00038683.JPEG:n02124075 +ILSVRC2012_val_00038684.JPEG:n02437616 +ILSVRC2012_val_00038685.JPEG:n01843383 +ILSVRC2012_val_00038686.JPEG:n03935335 +ILSVRC2012_val_00038687.JPEG:n02782093 +ILSVRC2012_val_00038688.JPEG:n07753113 +ILSVRC2012_val_00038689.JPEG:n03791053 +ILSVRC2012_val_00038690.JPEG:n02111129 +ILSVRC2012_val_00038691.JPEG:n07614500 +ILSVRC2012_val_00038692.JPEG:n03761084 +ILSVRC2012_val_00038693.JPEG:n03676483 +ILSVRC2012_val_00038694.JPEG:n01978455 +ILSVRC2012_val_00038695.JPEG:n03857828 +ILSVRC2012_val_00038696.JPEG:n02488702 +ILSVRC2012_val_00038697.JPEG:n02165456 +ILSVRC2012_val_00038698.JPEG:n07734744 +ILSVRC2012_val_00038699.JPEG:n03991062 +ILSVRC2012_val_00038700.JPEG:n02860847 +ILSVRC2012_val_00038701.JPEG:n03954731 +ILSVRC2012_val_00038702.JPEG:n03045698 +ILSVRC2012_val_00038703.JPEG:n03944341 +ILSVRC2012_val_00038704.JPEG:n02111129 +ILSVRC2012_val_00038705.JPEG:n02092002 +ILSVRC2012_val_00038706.JPEG:n03891251 +ILSVRC2012_val_00038707.JPEG:n02130308 +ILSVRC2012_val_00038708.JPEG:n01945685 +ILSVRC2012_val_00038709.JPEG:n03188531 +ILSVRC2012_val_00038710.JPEG:n02457408 +ILSVRC2012_val_00038711.JPEG:n03085013 +ILSVRC2012_val_00038712.JPEG:n03796401 +ILSVRC2012_val_00038713.JPEG:n13052670 +ILSVRC2012_val_00038714.JPEG:n02398521 +ILSVRC2012_val_00038715.JPEG:n03743016 +ILSVRC2012_val_00038716.JPEG:n02229544 +ILSVRC2012_val_00038717.JPEG:n03160309 +ILSVRC2012_val_00038718.JPEG:n02276258 +ILSVRC2012_val_00038719.JPEG:n02276258 +ILSVRC2012_val_00038720.JPEG:n02504013 +ILSVRC2012_val_00038721.JPEG:n02281406 +ILSVRC2012_val_00038722.JPEG:n02877765 +ILSVRC2012_val_00038723.JPEG:n03649909 +ILSVRC2012_val_00038724.JPEG:n07697313 +ILSVRC2012_val_00038725.JPEG:n02058221 +ILSVRC2012_val_00038726.JPEG:n02077923 +ILSVRC2012_val_00038727.JPEG:n03394916 +ILSVRC2012_val_00038728.JPEG:n02256656 +ILSVRC2012_val_00038729.JPEG:n04328186 +ILSVRC2012_val_00038730.JPEG:n02009229 +ILSVRC2012_val_00038731.JPEG:n03476684 +ILSVRC2012_val_00038732.JPEG:n03388549 +ILSVRC2012_val_00038733.JPEG:n07714571 +ILSVRC2012_val_00038734.JPEG:n09193705 +ILSVRC2012_val_00038735.JPEG:n02396427 +ILSVRC2012_val_00038736.JPEG:n01806567 +ILSVRC2012_val_00038737.JPEG:n02090379 +ILSVRC2012_val_00038738.JPEG:n02100583 +ILSVRC2012_val_00038739.JPEG:n04483307 +ILSVRC2012_val_00038740.JPEG:n02120079 +ILSVRC2012_val_00038741.JPEG:n01914609 +ILSVRC2012_val_00038742.JPEG:n01630670 +ILSVRC2012_val_00038743.JPEG:n04259630 +ILSVRC2012_val_00038744.JPEG:n07695742 +ILSVRC2012_val_00038745.JPEG:n02106030 +ILSVRC2012_val_00038746.JPEG:n02883205 +ILSVRC2012_val_00038747.JPEG:n02398521 +ILSVRC2012_val_00038748.JPEG:n03995372 +ILSVRC2012_val_00038749.JPEG:n07590611 +ILSVRC2012_val_00038750.JPEG:n04099969 +ILSVRC2012_val_00038751.JPEG:n02110063 +ILSVRC2012_val_00038752.JPEG:n03785016 +ILSVRC2012_val_00038753.JPEG:n02669723 +ILSVRC2012_val_00038754.JPEG:n03125729 +ILSVRC2012_val_00038755.JPEG:n04442312 +ILSVRC2012_val_00038756.JPEG:n07920052 +ILSVRC2012_val_00038757.JPEG:n02497673 +ILSVRC2012_val_00038758.JPEG:n02454379 +ILSVRC2012_val_00038759.JPEG:n02091831 +ILSVRC2012_val_00038760.JPEG:n02454379 +ILSVRC2012_val_00038761.JPEG:n02088632 +ILSVRC2012_val_00038762.JPEG:n02115641 +ILSVRC2012_val_00038763.JPEG:n03761084 +ILSVRC2012_val_00038764.JPEG:n02606052 +ILSVRC2012_val_00038765.JPEG:n02264363 +ILSVRC2012_val_00038766.JPEG:n01843065 +ILSVRC2012_val_00038767.JPEG:n03623198 +ILSVRC2012_val_00038768.JPEG:n03445777 +ILSVRC2012_val_00038769.JPEG:n02481823 +ILSVRC2012_val_00038770.JPEG:n01773157 +ILSVRC2012_val_00038771.JPEG:n03109150 +ILSVRC2012_val_00038772.JPEG:n04458633 +ILSVRC2012_val_00038773.JPEG:n02165456 +ILSVRC2012_val_00038774.JPEG:n02190166 +ILSVRC2012_val_00038775.JPEG:n04111531 +ILSVRC2012_val_00038776.JPEG:n03197337 +ILSVRC2012_val_00038777.JPEG:n04542943 +ILSVRC2012_val_00038778.JPEG:n04507155 +ILSVRC2012_val_00038779.JPEG:n02089867 +ILSVRC2012_val_00038780.JPEG:n02342885 +ILSVRC2012_val_00038781.JPEG:n02099601 +ILSVRC2012_val_00038782.JPEG:n03787032 +ILSVRC2012_val_00038783.JPEG:n03483316 +ILSVRC2012_val_00038784.JPEG:n02454379 +ILSVRC2012_val_00038785.JPEG:n04041544 +ILSVRC2012_val_00038786.JPEG:n02086079 +ILSVRC2012_val_00038787.JPEG:n04485082 +ILSVRC2012_val_00038788.JPEG:n07831146 +ILSVRC2012_val_00038789.JPEG:n02106030 +ILSVRC2012_val_00038790.JPEG:n03445777 +ILSVRC2012_val_00038791.JPEG:n02398521 +ILSVRC2012_val_00038792.JPEG:n02666196 +ILSVRC2012_val_00038793.JPEG:n02009912 +ILSVRC2012_val_00038794.JPEG:n01534433 +ILSVRC2012_val_00038795.JPEG:n03126707 +ILSVRC2012_val_00038796.JPEG:n12057211 +ILSVRC2012_val_00038797.JPEG:n04355933 +ILSVRC2012_val_00038798.JPEG:n02025239 +ILSVRC2012_val_00038799.JPEG:n04336792 +ILSVRC2012_val_00038800.JPEG:n02906734 +ILSVRC2012_val_00038801.JPEG:n02002556 +ILSVRC2012_val_00038802.JPEG:n04487394 +ILSVRC2012_val_00038803.JPEG:n03291819 +ILSVRC2012_val_00038804.JPEG:n01614925 +ILSVRC2012_val_00038805.JPEG:n04235860 +ILSVRC2012_val_00038806.JPEG:n04270147 +ILSVRC2012_val_00038807.JPEG:n03291819 +ILSVRC2012_val_00038808.JPEG:n03837869 +ILSVRC2012_val_00038809.JPEG:n04192698 +ILSVRC2012_val_00038810.JPEG:n04120489 +ILSVRC2012_val_00038811.JPEG:n02930766 +ILSVRC2012_val_00038812.JPEG:n02128385 +ILSVRC2012_val_00038813.JPEG:n02837789 +ILSVRC2012_val_00038814.JPEG:n02105505 +ILSVRC2012_val_00038815.JPEG:n01704323 +ILSVRC2012_val_00038816.JPEG:n02481823 +ILSVRC2012_val_00038817.JPEG:n03384352 +ILSVRC2012_val_00038818.JPEG:n02167151 +ILSVRC2012_val_00038819.JPEG:n07753592 +ILSVRC2012_val_00038820.JPEG:n07614500 +ILSVRC2012_val_00038821.JPEG:n02134084 +ILSVRC2012_val_00038822.JPEG:n04515003 +ILSVRC2012_val_00038823.JPEG:n01729322 +ILSVRC2012_val_00038824.JPEG:n04033901 +ILSVRC2012_val_00038825.JPEG:n02134418 +ILSVRC2012_val_00038826.JPEG:n01514668 +ILSVRC2012_val_00038827.JPEG:n03942813 +ILSVRC2012_val_00038828.JPEG:n02101556 +ILSVRC2012_val_00038829.JPEG:n03642806 +ILSVRC2012_val_00038830.JPEG:n03733131 +ILSVRC2012_val_00038831.JPEG:n03290653 +ILSVRC2012_val_00038832.JPEG:n02174001 +ILSVRC2012_val_00038833.JPEG:n01784675 +ILSVRC2012_val_00038834.JPEG:n03777754 +ILSVRC2012_val_00038835.JPEG:n03942813 +ILSVRC2012_val_00038836.JPEG:n02802426 +ILSVRC2012_val_00038837.JPEG:n04049303 +ILSVRC2012_val_00038838.JPEG:n03535780 +ILSVRC2012_val_00038839.JPEG:n02492035 +ILSVRC2012_val_00038840.JPEG:n04070727 +ILSVRC2012_val_00038841.JPEG:n03075370 +ILSVRC2012_val_00038842.JPEG:n04372370 +ILSVRC2012_val_00038843.JPEG:n07860988 +ILSVRC2012_val_00038844.JPEG:n04367480 +ILSVRC2012_val_00038845.JPEG:n03786901 +ILSVRC2012_val_00038846.JPEG:n04562935 +ILSVRC2012_val_00038847.JPEG:n07590611 +ILSVRC2012_val_00038848.JPEG:n02102973 +ILSVRC2012_val_00038849.JPEG:n07248320 +ILSVRC2012_val_00038850.JPEG:n03095699 +ILSVRC2012_val_00038851.JPEG:n04009552 +ILSVRC2012_val_00038852.JPEG:n07614500 +ILSVRC2012_val_00038853.JPEG:n09288635 +ILSVRC2012_val_00038854.JPEG:n03724870 +ILSVRC2012_val_00038855.JPEG:n04258138 +ILSVRC2012_val_00038856.JPEG:n01698640 +ILSVRC2012_val_00038857.JPEG:n07753113 +ILSVRC2012_val_00038858.JPEG:n04263257 +ILSVRC2012_val_00038859.JPEG:n01755581 +ILSVRC2012_val_00038860.JPEG:n04447861 +ILSVRC2012_val_00038861.JPEG:n02666196 +ILSVRC2012_val_00038862.JPEG:n03733281 +ILSVRC2012_val_00038863.JPEG:n02051845 +ILSVRC2012_val_00038864.JPEG:n02058221 +ILSVRC2012_val_00038865.JPEG:n03958227 +ILSVRC2012_val_00038866.JPEG:n02403003 +ILSVRC2012_val_00038867.JPEG:n02097474 +ILSVRC2012_val_00038868.JPEG:n02099429 +ILSVRC2012_val_00038869.JPEG:n02484975 +ILSVRC2012_val_00038870.JPEG:n07836838 +ILSVRC2012_val_00038871.JPEG:n10565667 +ILSVRC2012_val_00038872.JPEG:n07720875 +ILSVRC2012_val_00038873.JPEG:n02486261 +ILSVRC2012_val_00038874.JPEG:n02321529 +ILSVRC2012_val_00038875.JPEG:n01755581 +ILSVRC2012_val_00038876.JPEG:n03100240 +ILSVRC2012_val_00038877.JPEG:n03063599 +ILSVRC2012_val_00038878.JPEG:n01664065 +ILSVRC2012_val_00038879.JPEG:n02783161 +ILSVRC2012_val_00038880.JPEG:n03803284 +ILSVRC2012_val_00038881.JPEG:n03110669 +ILSVRC2012_val_00038882.JPEG:n02086240 +ILSVRC2012_val_00038883.JPEG:n02487347 +ILSVRC2012_val_00038884.JPEG:n02097209 +ILSVRC2012_val_00038885.JPEG:n04310018 +ILSVRC2012_val_00038886.JPEG:n02012849 +ILSVRC2012_val_00038887.JPEG:n04120489 +ILSVRC2012_val_00038888.JPEG:n03482405 +ILSVRC2012_val_00038889.JPEG:n02447366 +ILSVRC2012_val_00038890.JPEG:n01749939 +ILSVRC2012_val_00038891.JPEG:n03478589 +ILSVRC2012_val_00038892.JPEG:n02963159 +ILSVRC2012_val_00038893.JPEG:n04428191 +ILSVRC2012_val_00038894.JPEG:n04285008 +ILSVRC2012_val_00038895.JPEG:n01530575 +ILSVRC2012_val_00038896.JPEG:n02111129 +ILSVRC2012_val_00038897.JPEG:n03109150 +ILSVRC2012_val_00038898.JPEG:n07697313 +ILSVRC2012_val_00038899.JPEG:n02802426 +ILSVRC2012_val_00038900.JPEG:n03690938 +ILSVRC2012_val_00038901.JPEG:n01914609 +ILSVRC2012_val_00038902.JPEG:n02481823 +ILSVRC2012_val_00038903.JPEG:n02259212 +ILSVRC2012_val_00038904.JPEG:n03538406 +ILSVRC2012_val_00038905.JPEG:n15075141 +ILSVRC2012_val_00038906.JPEG:n03649909 +ILSVRC2012_val_00038907.JPEG:n04483307 +ILSVRC2012_val_00038908.JPEG:n04613696 +ILSVRC2012_val_00038909.JPEG:n10565667 +ILSVRC2012_val_00038910.JPEG:n02488702 +ILSVRC2012_val_00038911.JPEG:n02094258 +ILSVRC2012_val_00038912.JPEG:n02096585 +ILSVRC2012_val_00038913.JPEG:n02127052 +ILSVRC2012_val_00038914.JPEG:n02391049 +ILSVRC2012_val_00038915.JPEG:n01734418 +ILSVRC2012_val_00038916.JPEG:n09332890 +ILSVRC2012_val_00038917.JPEG:n03379051 +ILSVRC2012_val_00038918.JPEG:n02133161 +ILSVRC2012_val_00038919.JPEG:n12144580 +ILSVRC2012_val_00038920.JPEG:n02099429 +ILSVRC2012_val_00038921.JPEG:n04447861 +ILSVRC2012_val_00038922.JPEG:n04120489 +ILSVRC2012_val_00038923.JPEG:n07860988 +ILSVRC2012_val_00038924.JPEG:n02129604 +ILSVRC2012_val_00038925.JPEG:n03065424 +ILSVRC2012_val_00038926.JPEG:n02095314 +ILSVRC2012_val_00038927.JPEG:n04154565 +ILSVRC2012_val_00038928.JPEG:n02655020 +ILSVRC2012_val_00038929.JPEG:n02165105 +ILSVRC2012_val_00038930.JPEG:n04275548 +ILSVRC2012_val_00038931.JPEG:n02415577 +ILSVRC2012_val_00038932.JPEG:n02786058 +ILSVRC2012_val_00038933.JPEG:n02091467 +ILSVRC2012_val_00038934.JPEG:n03444034 +ILSVRC2012_val_00038935.JPEG:n01498041 +ILSVRC2012_val_00038936.JPEG:n07590611 +ILSVRC2012_val_00038937.JPEG:n04554684 +ILSVRC2012_val_00038938.JPEG:n02109047 +ILSVRC2012_val_00038939.JPEG:n04552348 +ILSVRC2012_val_00038940.JPEG:n03814639 +ILSVRC2012_val_00038941.JPEG:n03125729 +ILSVRC2012_val_00038942.JPEG:n03888257 +ILSVRC2012_val_00038943.JPEG:n03950228 +ILSVRC2012_val_00038944.JPEG:n02089973 +ILSVRC2012_val_00038945.JPEG:n03967562 +ILSVRC2012_val_00038946.JPEG:n02749479 +ILSVRC2012_val_00038947.JPEG:n03729826 +ILSVRC2012_val_00038948.JPEG:n02018207 +ILSVRC2012_val_00038949.JPEG:n04487081 +ILSVRC2012_val_00038950.JPEG:n03017168 +ILSVRC2012_val_00038951.JPEG:n03976657 +ILSVRC2012_val_00038952.JPEG:n03938244 +ILSVRC2012_val_00038953.JPEG:n02769748 +ILSVRC2012_val_00038954.JPEG:n07836838 +ILSVRC2012_val_00038955.JPEG:n02002724 +ILSVRC2012_val_00038956.JPEG:n03100240 +ILSVRC2012_val_00038957.JPEG:n03598930 +ILSVRC2012_val_00038958.JPEG:n04479046 +ILSVRC2012_val_00038959.JPEG:n01644373 +ILSVRC2012_val_00038960.JPEG:n02708093 +ILSVRC2012_val_00038961.JPEG:n02134418 +ILSVRC2012_val_00038962.JPEG:n13054560 +ILSVRC2012_val_00038963.JPEG:n09332890 +ILSVRC2012_val_00038964.JPEG:n03133878 +ILSVRC2012_val_00038965.JPEG:n04554684 +ILSVRC2012_val_00038966.JPEG:n03041632 +ILSVRC2012_val_00038967.JPEG:n02869837 +ILSVRC2012_val_00038968.JPEG:n03014705 +ILSVRC2012_val_00038969.JPEG:n02510455 +ILSVRC2012_val_00038970.JPEG:n03954731 +ILSVRC2012_val_00038971.JPEG:n02788148 +ILSVRC2012_val_00038972.JPEG:n02859443 +ILSVRC2012_val_00038973.JPEG:n02640242 +ILSVRC2012_val_00038974.JPEG:n02087046 +ILSVRC2012_val_00038975.JPEG:n03891332 +ILSVRC2012_val_00038976.JPEG:n02124075 +ILSVRC2012_val_00038977.JPEG:n03476684 +ILSVRC2012_val_00038978.JPEG:n04270147 +ILSVRC2012_val_00038979.JPEG:n04542943 +ILSVRC2012_val_00038980.JPEG:n03916031 +ILSVRC2012_val_00038981.JPEG:n02051845 +ILSVRC2012_val_00038982.JPEG:n02104029 +ILSVRC2012_val_00038983.JPEG:n04270147 +ILSVRC2012_val_00038984.JPEG:n02422106 +ILSVRC2012_val_00038985.JPEG:n03692522 +ILSVRC2012_val_00038986.JPEG:n02115641 +ILSVRC2012_val_00038987.JPEG:n02447366 +ILSVRC2012_val_00038988.JPEG:n03710721 +ILSVRC2012_val_00038989.JPEG:n02112018 +ILSVRC2012_val_00038990.JPEG:n03000134 +ILSVRC2012_val_00038991.JPEG:n02105162 +ILSVRC2012_val_00038992.JPEG:n02097047 +ILSVRC2012_val_00038993.JPEG:n02356798 +ILSVRC2012_val_00038994.JPEG:n04037443 +ILSVRC2012_val_00038995.JPEG:n02071294 +ILSVRC2012_val_00038996.JPEG:n07892512 +ILSVRC2012_val_00038997.JPEG:n03924679 +ILSVRC2012_val_00038998.JPEG:n01687978 +ILSVRC2012_val_00038999.JPEG:n02098286 +ILSVRC2012_val_00039000.JPEG:n03345487 +ILSVRC2012_val_00039001.JPEG:n04254777 +ILSVRC2012_val_00039002.JPEG:n03680355 +ILSVRC2012_val_00039003.JPEG:n02963159 +ILSVRC2012_val_00039004.JPEG:n01582220 +ILSVRC2012_val_00039005.JPEG:n04090263 +ILSVRC2012_val_00039006.JPEG:n03761084 +ILSVRC2012_val_00039007.JPEG:n04604644 +ILSVRC2012_val_00039008.JPEG:n02097209 +ILSVRC2012_val_00039009.JPEG:n03109150 +ILSVRC2012_val_00039010.JPEG:n02088632 +ILSVRC2012_val_00039011.JPEG:n03937543 +ILSVRC2012_val_00039012.JPEG:n01943899 +ILSVRC2012_val_00039013.JPEG:n02093647 +ILSVRC2012_val_00039014.JPEG:n02093428 +ILSVRC2012_val_00039015.JPEG:n03461385 +ILSVRC2012_val_00039016.JPEG:n04270147 +ILSVRC2012_val_00039017.JPEG:n04389033 +ILSVRC2012_val_00039018.JPEG:n03534580 +ILSVRC2012_val_00039019.JPEG:n09468604 +ILSVRC2012_val_00039020.JPEG:n02107312 +ILSVRC2012_val_00039021.JPEG:n01797886 +ILSVRC2012_val_00039022.JPEG:n02090379 +ILSVRC2012_val_00039023.JPEG:n02871525 +ILSVRC2012_val_00039024.JPEG:n01667778 +ILSVRC2012_val_00039025.JPEG:n01773549 +ILSVRC2012_val_00039026.JPEG:n01755581 +ILSVRC2012_val_00039027.JPEG:n02093991 +ILSVRC2012_val_00039028.JPEG:n04350905 +ILSVRC2012_val_00039029.JPEG:n03995372 +ILSVRC2012_val_00039030.JPEG:n02280649 +ILSVRC2012_val_00039031.JPEG:n03933933 +ILSVRC2012_val_00039032.JPEG:n02226429 +ILSVRC2012_val_00039033.JPEG:n03207941 +ILSVRC2012_val_00039034.JPEG:n09399592 +ILSVRC2012_val_00039035.JPEG:n02106030 +ILSVRC2012_val_00039036.JPEG:n03590841 +ILSVRC2012_val_00039037.JPEG:n02966193 +ILSVRC2012_val_00039038.JPEG:n03787032 +ILSVRC2012_val_00039039.JPEG:n02115913 +ILSVRC2012_val_00039040.JPEG:n04099969 +ILSVRC2012_val_00039041.JPEG:n04273569 +ILSVRC2012_val_00039042.JPEG:n02037110 +ILSVRC2012_val_00039043.JPEG:n01917289 +ILSVRC2012_val_00039044.JPEG:n04254777 +ILSVRC2012_val_00039045.JPEG:n03888257 +ILSVRC2012_val_00039046.JPEG:n02807133 +ILSVRC2012_val_00039047.JPEG:n04589890 +ILSVRC2012_val_00039048.JPEG:n02091032 +ILSVRC2012_val_00039049.JPEG:n01685808 +ILSVRC2012_val_00039050.JPEG:n07714571 +ILSVRC2012_val_00039051.JPEG:n03777568 +ILSVRC2012_val_00039052.JPEG:n03379051 +ILSVRC2012_val_00039053.JPEG:n03028079 +ILSVRC2012_val_00039054.JPEG:n04275548 +ILSVRC2012_val_00039055.JPEG:n02395406 +ILSVRC2012_val_00039056.JPEG:n04040759 +ILSVRC2012_val_00039057.JPEG:n02109961 +ILSVRC2012_val_00039058.JPEG:n01872401 +ILSVRC2012_val_00039059.JPEG:n03825788 +ILSVRC2012_val_00039060.JPEG:n02112706 +ILSVRC2012_val_00039061.JPEG:n03692522 +ILSVRC2012_val_00039062.JPEG:n02086910 +ILSVRC2012_val_00039063.JPEG:n02321529 +ILSVRC2012_val_00039064.JPEG:n03131574 +ILSVRC2012_val_00039065.JPEG:n04311004 +ILSVRC2012_val_00039066.JPEG:n03929855 +ILSVRC2012_val_00039067.JPEG:n01514859 +ILSVRC2012_val_00039068.JPEG:n03804744 +ILSVRC2012_val_00039069.JPEG:n03417042 +ILSVRC2012_val_00039070.JPEG:n02794156 +ILSVRC2012_val_00039071.JPEG:n07730033 +ILSVRC2012_val_00039072.JPEG:n04120489 +ILSVRC2012_val_00039073.JPEG:n02342885 +ILSVRC2012_val_00039074.JPEG:n04041544 +ILSVRC2012_val_00039075.JPEG:n04366367 +ILSVRC2012_val_00039076.JPEG:n02116738 +ILSVRC2012_val_00039077.JPEG:n02992211 +ILSVRC2012_val_00039078.JPEG:n02276258 +ILSVRC2012_val_00039079.JPEG:n02895154 +ILSVRC2012_val_00039080.JPEG:n01984695 +ILSVRC2012_val_00039081.JPEG:n03661043 +ILSVRC2012_val_00039082.JPEG:n03207941 +ILSVRC2012_val_00039083.JPEG:n02025239 +ILSVRC2012_val_00039084.JPEG:n02123045 +ILSVRC2012_val_00039085.JPEG:n02117135 +ILSVRC2012_val_00039086.JPEG:n02107908 +ILSVRC2012_val_00039087.JPEG:n02815834 +ILSVRC2012_val_00039088.JPEG:n04355933 +ILSVRC2012_val_00039089.JPEG:n03598930 +ILSVRC2012_val_00039090.JPEG:n07742313 +ILSVRC2012_val_00039091.JPEG:n03876231 +ILSVRC2012_val_00039092.JPEG:n02259212 +ILSVRC2012_val_00039093.JPEG:n01775062 +ILSVRC2012_val_00039094.JPEG:n03617480 +ILSVRC2012_val_00039095.JPEG:n03840681 +ILSVRC2012_val_00039096.JPEG:n03902125 +ILSVRC2012_val_00039097.JPEG:n02930766 +ILSVRC2012_val_00039098.JPEG:n03633091 +ILSVRC2012_val_00039099.JPEG:n04404412 +ILSVRC2012_val_00039100.JPEG:n03825788 +ILSVRC2012_val_00039101.JPEG:n03337140 +ILSVRC2012_val_00039102.JPEG:n02018795 +ILSVRC2012_val_00039103.JPEG:n02447366 +ILSVRC2012_val_00039104.JPEG:n07613480 +ILSVRC2012_val_00039105.JPEG:n02493793 +ILSVRC2012_val_00039106.JPEG:n01694178 +ILSVRC2012_val_00039107.JPEG:n12620546 +ILSVRC2012_val_00039108.JPEG:n06874185 +ILSVRC2012_val_00039109.JPEG:n02443484 +ILSVRC2012_val_00039110.JPEG:n04209133 +ILSVRC2012_val_00039111.JPEG:n04515003 +ILSVRC2012_val_00039112.JPEG:n04540053 +ILSVRC2012_val_00039113.JPEG:n01796340 +ILSVRC2012_val_00039114.JPEG:n03623198 +ILSVRC2012_val_00039115.JPEG:n02108551 +ILSVRC2012_val_00039116.JPEG:n03763968 +ILSVRC2012_val_00039117.JPEG:n02410509 +ILSVRC2012_val_00039118.JPEG:n11879895 +ILSVRC2012_val_00039119.JPEG:n03832673 +ILSVRC2012_val_00039120.JPEG:n03930630 +ILSVRC2012_val_00039121.JPEG:n02490219 +ILSVRC2012_val_00039122.JPEG:n03937543 +ILSVRC2012_val_00039123.JPEG:n02111889 +ILSVRC2012_val_00039124.JPEG:n02096437 +ILSVRC2012_val_00039125.JPEG:n04154565 +ILSVRC2012_val_00039126.JPEG:n02971356 +ILSVRC2012_val_00039127.JPEG:n02865351 +ILSVRC2012_val_00039128.JPEG:n03776460 +ILSVRC2012_val_00039129.JPEG:n02777292 +ILSVRC2012_val_00039130.JPEG:n02190166 +ILSVRC2012_val_00039131.JPEG:n04612504 +ILSVRC2012_val_00039132.JPEG:n04081281 +ILSVRC2012_val_00039133.JPEG:n02747177 +ILSVRC2012_val_00039134.JPEG:n03777754 +ILSVRC2012_val_00039135.JPEG:n02445715 +ILSVRC2012_val_00039136.JPEG:n03857828 +ILSVRC2012_val_00039137.JPEG:n11939491 +ILSVRC2012_val_00039138.JPEG:n01981276 +ILSVRC2012_val_00039139.JPEG:n04041544 +ILSVRC2012_val_00039140.JPEG:n04458633 +ILSVRC2012_val_00039141.JPEG:n03447721 +ILSVRC2012_val_00039142.JPEG:n02106030 +ILSVRC2012_val_00039143.JPEG:n02834397 +ILSVRC2012_val_00039144.JPEG:n02097474 +ILSVRC2012_val_00039145.JPEG:n01877812 +ILSVRC2012_val_00039146.JPEG:n02085936 +ILSVRC2012_val_00039147.JPEG:n02096051 +ILSVRC2012_val_00039148.JPEG:n03272562 +ILSVRC2012_val_00039149.JPEG:n03793489 +ILSVRC2012_val_00039150.JPEG:n02099849 +ILSVRC2012_val_00039151.JPEG:n03649909 +ILSVRC2012_val_00039152.JPEG:n01882714 +ILSVRC2012_val_00039153.JPEG:n02860847 +ILSVRC2012_val_00039154.JPEG:n04039381 +ILSVRC2012_val_00039155.JPEG:n04264628 +ILSVRC2012_val_00039156.JPEG:n02484975 +ILSVRC2012_val_00039157.JPEG:n02167151 +ILSVRC2012_val_00039158.JPEG:n02074367 +ILSVRC2012_val_00039159.JPEG:n01773549 +ILSVRC2012_val_00039160.JPEG:n04367480 +ILSVRC2012_val_00039161.JPEG:n07718747 +ILSVRC2012_val_00039162.JPEG:n02841315 +ILSVRC2012_val_00039163.JPEG:n02910353 +ILSVRC2012_val_00039164.JPEG:n02106550 +ILSVRC2012_val_00039165.JPEG:n03602883 +ILSVRC2012_val_00039166.JPEG:n04153751 +ILSVRC2012_val_00039167.JPEG:n03992509 +ILSVRC2012_val_00039168.JPEG:n09468604 +ILSVRC2012_val_00039169.JPEG:n02129604 +ILSVRC2012_val_00039170.JPEG:n09229709 +ILSVRC2012_val_00039171.JPEG:n02056570 +ILSVRC2012_val_00039172.JPEG:n03594734 +ILSVRC2012_val_00039173.JPEG:n02111277 +ILSVRC2012_val_00039174.JPEG:n07590611 +ILSVRC2012_val_00039175.JPEG:n02704792 +ILSVRC2012_val_00039176.JPEG:n03868863 +ILSVRC2012_val_00039177.JPEG:n02115641 +ILSVRC2012_val_00039178.JPEG:n02444819 +ILSVRC2012_val_00039179.JPEG:n02808304 +ILSVRC2012_val_00039180.JPEG:n04355338 +ILSVRC2012_val_00039181.JPEG:n02281787 +ILSVRC2012_val_00039182.JPEG:n02138441 +ILSVRC2012_val_00039183.JPEG:n03814906 +ILSVRC2012_val_00039184.JPEG:n04409515 +ILSVRC2012_val_00039185.JPEG:n01739381 +ILSVRC2012_val_00039186.JPEG:n03495258 +ILSVRC2012_val_00039187.JPEG:n03627232 +ILSVRC2012_val_00039188.JPEG:n02085620 +ILSVRC2012_val_00039189.JPEG:n02190166 +ILSVRC2012_val_00039190.JPEG:n03355925 +ILSVRC2012_val_00039191.JPEG:n03188531 +ILSVRC2012_val_00039192.JPEG:n02100735 +ILSVRC2012_val_00039193.JPEG:n03961711 +ILSVRC2012_val_00039194.JPEG:n02823428 +ILSVRC2012_val_00039195.JPEG:n07860988 +ILSVRC2012_val_00039196.JPEG:n01740131 +ILSVRC2012_val_00039197.JPEG:n09229709 +ILSVRC2012_val_00039198.JPEG:n03777568 +ILSVRC2012_val_00039199.JPEG:n03908618 +ILSVRC2012_val_00039200.JPEG:n02108551 +ILSVRC2012_val_00039201.JPEG:n02177972 +ILSVRC2012_val_00039202.JPEG:n09288635 +ILSVRC2012_val_00039203.JPEG:n01693334 +ILSVRC2012_val_00039204.JPEG:n02106382 +ILSVRC2012_val_00039205.JPEG:n04026417 +ILSVRC2012_val_00039206.JPEG:n03388183 +ILSVRC2012_val_00039207.JPEG:n02002724 +ILSVRC2012_val_00039208.JPEG:n03208938 +ILSVRC2012_val_00039209.JPEG:n04517823 +ILSVRC2012_val_00039210.JPEG:n04336792 +ILSVRC2012_val_00039211.JPEG:n03658185 +ILSVRC2012_val_00039212.JPEG:n02097474 +ILSVRC2012_val_00039213.JPEG:n02690373 +ILSVRC2012_val_00039214.JPEG:n13044778 +ILSVRC2012_val_00039215.JPEG:n02281787 +ILSVRC2012_val_00039216.JPEG:n02641379 +ILSVRC2012_val_00039217.JPEG:n02130308 +ILSVRC2012_val_00039218.JPEG:n02704792 +ILSVRC2012_val_00039219.JPEG:n01582220 +ILSVRC2012_val_00039220.JPEG:n02027492 +ILSVRC2012_val_00039221.JPEG:n04525305 +ILSVRC2012_val_00039222.JPEG:n02119789 +ILSVRC2012_val_00039223.JPEG:n13054560 +ILSVRC2012_val_00039224.JPEG:n03724870 +ILSVRC2012_val_00039225.JPEG:n02488291 +ILSVRC2012_val_00039226.JPEG:n07697313 +ILSVRC2012_val_00039227.JPEG:n02132136 +ILSVRC2012_val_00039228.JPEG:n04336792 +ILSVRC2012_val_00039229.JPEG:n03983396 +ILSVRC2012_val_00039230.JPEG:n03944341 +ILSVRC2012_val_00039231.JPEG:n01774384 +ILSVRC2012_val_00039232.JPEG:n02027492 +ILSVRC2012_val_00039233.JPEG:n02091134 +ILSVRC2012_val_00039234.JPEG:n07860988 +ILSVRC2012_val_00039235.JPEG:n02106550 +ILSVRC2012_val_00039236.JPEG:n04357314 +ILSVRC2012_val_00039237.JPEG:n03662601 +ILSVRC2012_val_00039238.JPEG:n03868242 +ILSVRC2012_val_00039239.JPEG:n03804744 +ILSVRC2012_val_00039240.JPEG:n02112350 +ILSVRC2012_val_00039241.JPEG:n01774750 +ILSVRC2012_val_00039242.JPEG:n02088238 +ILSVRC2012_val_00039243.JPEG:n07718472 +ILSVRC2012_val_00039244.JPEG:n01742172 +ILSVRC2012_val_00039245.JPEG:n02992529 +ILSVRC2012_val_00039246.JPEG:n04404412 +ILSVRC2012_val_00039247.JPEG:n02089867 +ILSVRC2012_val_00039248.JPEG:n03345487 +ILSVRC2012_val_00039249.JPEG:n02437312 +ILSVRC2012_val_00039250.JPEG:n02930766 +ILSVRC2012_val_00039251.JPEG:n13133613 +ILSVRC2012_val_00039252.JPEG:n02206856 +ILSVRC2012_val_00039253.JPEG:n02486410 +ILSVRC2012_val_00039254.JPEG:n03843555 +ILSVRC2012_val_00039255.JPEG:n04476259 +ILSVRC2012_val_00039256.JPEG:n02094433 +ILSVRC2012_val_00039257.JPEG:n01843065 +ILSVRC2012_val_00039258.JPEG:n07714571 +ILSVRC2012_val_00039259.JPEG:n02389026 +ILSVRC2012_val_00039260.JPEG:n04099969 +ILSVRC2012_val_00039261.JPEG:n01843065 +ILSVRC2012_val_00039262.JPEG:n03180011 +ILSVRC2012_val_00039263.JPEG:n09472597 +ILSVRC2012_val_00039264.JPEG:n03670208 +ILSVRC2012_val_00039265.JPEG:n01751748 +ILSVRC2012_val_00039266.JPEG:n01807496 +ILSVRC2012_val_00039267.JPEG:n02229544 +ILSVRC2012_val_00039268.JPEG:n02101006 +ILSVRC2012_val_00039269.JPEG:n03188531 +ILSVRC2012_val_00039270.JPEG:n03290653 +ILSVRC2012_val_00039271.JPEG:n02403003 +ILSVRC2012_val_00039272.JPEG:n02699494 +ILSVRC2012_val_00039273.JPEG:n04266014 +ILSVRC2012_val_00039274.JPEG:n02708093 +ILSVRC2012_val_00039275.JPEG:n04399382 +ILSVRC2012_val_00039276.JPEG:n02804414 +ILSVRC2012_val_00039277.JPEG:n07747607 +ILSVRC2012_val_00039278.JPEG:n02749479 +ILSVRC2012_val_00039279.JPEG:n03424325 +ILSVRC2012_val_00039280.JPEG:n04522168 +ILSVRC2012_val_00039281.JPEG:n01843065 +ILSVRC2012_val_00039282.JPEG:n01682714 +ILSVRC2012_val_00039283.JPEG:n02138441 +ILSVRC2012_val_00039284.JPEG:n11879895 +ILSVRC2012_val_00039285.JPEG:n04355338 +ILSVRC2012_val_00039286.JPEG:n03662601 +ILSVRC2012_val_00039287.JPEG:n03658185 +ILSVRC2012_val_00039288.JPEG:n03483316 +ILSVRC2012_val_00039289.JPEG:n07718747 +ILSVRC2012_val_00039290.JPEG:n03476684 +ILSVRC2012_val_00039291.JPEG:n02110958 +ILSVRC2012_val_00039292.JPEG:n04040759 +ILSVRC2012_val_00039293.JPEG:n03814906 +ILSVRC2012_val_00039294.JPEG:n04461696 +ILSVRC2012_val_00039295.JPEG:n02492660 +ILSVRC2012_val_00039296.JPEG:n04044716 +ILSVRC2012_val_00039297.JPEG:n04596742 +ILSVRC2012_val_00039298.JPEG:n01770081 +ILSVRC2012_val_00039299.JPEG:n01806143 +ILSVRC2012_val_00039300.JPEG:n04589890 +ILSVRC2012_val_00039301.JPEG:n03016953 +ILSVRC2012_val_00039302.JPEG:n02493793 +ILSVRC2012_val_00039303.JPEG:n01983481 +ILSVRC2012_val_00039304.JPEG:n01484850 +ILSVRC2012_val_00039305.JPEG:n02981792 +ILSVRC2012_val_00039306.JPEG:n03710637 +ILSVRC2012_val_00039307.JPEG:n02104029 +ILSVRC2012_val_00039308.JPEG:n01498041 +ILSVRC2012_val_00039309.JPEG:n03976657 +ILSVRC2012_val_00039310.JPEG:n04009552 +ILSVRC2012_val_00039311.JPEG:n02790996 +ILSVRC2012_val_00039312.JPEG:n04235860 +ILSVRC2012_val_00039313.JPEG:n04447861 +ILSVRC2012_val_00039314.JPEG:n01910747 +ILSVRC2012_val_00039315.JPEG:n03481172 +ILSVRC2012_val_00039316.JPEG:n04090263 +ILSVRC2012_val_00039317.JPEG:n03929660 +ILSVRC2012_val_00039318.JPEG:n07248320 +ILSVRC2012_val_00039319.JPEG:n03271574 +ILSVRC2012_val_00039320.JPEG:n03661043 +ILSVRC2012_val_00039321.JPEG:n03954731 +ILSVRC2012_val_00039322.JPEG:n03016953 +ILSVRC2012_val_00039323.JPEG:n07614500 +ILSVRC2012_val_00039324.JPEG:n03920288 +ILSVRC2012_val_00039325.JPEG:n02091244 +ILSVRC2012_val_00039326.JPEG:n02676566 +ILSVRC2012_val_00039327.JPEG:n13044778 +ILSVRC2012_val_00039328.JPEG:n03843555 +ILSVRC2012_val_00039329.JPEG:n07871810 +ILSVRC2012_val_00039330.JPEG:n03832673 +ILSVRC2012_val_00039331.JPEG:n04252225 +ILSVRC2012_val_00039332.JPEG:n02174001 +ILSVRC2012_val_00039333.JPEG:n03832673 +ILSVRC2012_val_00039334.JPEG:n10148035 +ILSVRC2012_val_00039335.JPEG:n02280649 +ILSVRC2012_val_00039336.JPEG:n09229709 +ILSVRC2012_val_00039337.JPEG:n06874185 +ILSVRC2012_val_00039338.JPEG:n02823428 +ILSVRC2012_val_00039339.JPEG:n02692877 +ILSVRC2012_val_00039340.JPEG:n02823428 +ILSVRC2012_val_00039341.JPEG:n07753592 +ILSVRC2012_val_00039342.JPEG:n02782093 +ILSVRC2012_val_00039343.JPEG:n03459775 +ILSVRC2012_val_00039344.JPEG:n09288635 +ILSVRC2012_val_00039345.JPEG:n04204347 +ILSVRC2012_val_00039346.JPEG:n02483708 +ILSVRC2012_val_00039347.JPEG:n04461696 +ILSVRC2012_val_00039348.JPEG:n02791124 +ILSVRC2012_val_00039349.JPEG:n03710193 +ILSVRC2012_val_00039350.JPEG:n12768682 +ILSVRC2012_val_00039351.JPEG:n04435653 +ILSVRC2012_val_00039352.JPEG:n04204347 +ILSVRC2012_val_00039353.JPEG:n02669723 +ILSVRC2012_val_00039354.JPEG:n03657121 +ILSVRC2012_val_00039355.JPEG:n01518878 +ILSVRC2012_val_00039356.JPEG:n04026417 +ILSVRC2012_val_00039357.JPEG:n02319095 +ILSVRC2012_val_00039358.JPEG:n03791053 +ILSVRC2012_val_00039359.JPEG:n02110063 +ILSVRC2012_val_00039360.JPEG:n02281787 +ILSVRC2012_val_00039361.JPEG:n03197337 +ILSVRC2012_val_00039362.JPEG:n04152593 +ILSVRC2012_val_00039363.JPEG:n02025239 +ILSVRC2012_val_00039364.JPEG:n03633091 +ILSVRC2012_val_00039365.JPEG:n02259212 +ILSVRC2012_val_00039366.JPEG:n02423022 +ILSVRC2012_val_00039367.JPEG:n03891332 +ILSVRC2012_val_00039368.JPEG:n03874293 +ILSVRC2012_val_00039369.JPEG:n02071294 +ILSVRC2012_val_00039370.JPEG:n01773797 +ILSVRC2012_val_00039371.JPEG:n07711569 +ILSVRC2012_val_00039372.JPEG:n02007558 +ILSVRC2012_val_00039373.JPEG:n13133613 +ILSVRC2012_val_00039374.JPEG:n02017213 +ILSVRC2012_val_00039375.JPEG:n04270147 +ILSVRC2012_val_00039376.JPEG:n02113624 +ILSVRC2012_val_00039377.JPEG:n02916936 +ILSVRC2012_val_00039378.JPEG:n01675722 +ILSVRC2012_val_00039379.JPEG:n07614500 +ILSVRC2012_val_00039380.JPEG:n03673027 +ILSVRC2012_val_00039381.JPEG:n02109961 +ILSVRC2012_val_00039382.JPEG:n02950826 +ILSVRC2012_val_00039383.JPEG:n02966193 +ILSVRC2012_val_00039384.JPEG:n01685808 +ILSVRC2012_val_00039385.JPEG:n02804610 +ILSVRC2012_val_00039386.JPEG:n02095314 +ILSVRC2012_val_00039387.JPEG:n03929855 +ILSVRC2012_val_00039388.JPEG:n10565667 +ILSVRC2012_val_00039389.JPEG:n02013706 +ILSVRC2012_val_00039390.JPEG:n02123394 +ILSVRC2012_val_00039391.JPEG:n03590841 +ILSVRC2012_val_00039392.JPEG:n07711569 +ILSVRC2012_val_00039393.JPEG:n02113799 +ILSVRC2012_val_00039394.JPEG:n07860988 +ILSVRC2012_val_00039395.JPEG:n04367480 +ILSVRC2012_val_00039396.JPEG:n07873807 +ILSVRC2012_val_00039397.JPEG:n02096585 +ILSVRC2012_val_00039398.JPEG:n02002724 +ILSVRC2012_val_00039399.JPEG:n02134418 +ILSVRC2012_val_00039400.JPEG:n02398521 +ILSVRC2012_val_00039401.JPEG:n04033901 +ILSVRC2012_val_00039402.JPEG:n02110063 +ILSVRC2012_val_00039403.JPEG:n09468604 +ILSVRC2012_val_00039404.JPEG:n01990800 +ILSVRC2012_val_00039405.JPEG:n04423845 +ILSVRC2012_val_00039406.JPEG:n02177972 +ILSVRC2012_val_00039407.JPEG:n04447861 +ILSVRC2012_val_00039408.JPEG:n02096585 +ILSVRC2012_val_00039409.JPEG:n02442845 +ILSVRC2012_val_00039410.JPEG:n04265275 +ILSVRC2012_val_00039411.JPEG:n04317175 +ILSVRC2012_val_00039412.JPEG:n01807496 +ILSVRC2012_val_00039413.JPEG:n04366367 +ILSVRC2012_val_00039414.JPEG:n03814906 +ILSVRC2012_val_00039415.JPEG:n12998815 +ILSVRC2012_val_00039416.JPEG:n03482405 +ILSVRC2012_val_00039417.JPEG:n03884397 +ILSVRC2012_val_00039418.JPEG:n03673027 +ILSVRC2012_val_00039419.JPEG:n03673027 +ILSVRC2012_val_00039420.JPEG:n03793489 +ILSVRC2012_val_00039421.JPEG:n02443114 +ILSVRC2012_val_00039422.JPEG:n02988304 +ILSVRC2012_val_00039423.JPEG:n02422106 +ILSVRC2012_val_00039424.JPEG:n04326547 +ILSVRC2012_val_00039425.JPEG:n02992529 +ILSVRC2012_val_00039426.JPEG:n01860187 +ILSVRC2012_val_00039427.JPEG:n03895866 +ILSVRC2012_val_00039428.JPEG:n03180011 +ILSVRC2012_val_00039429.JPEG:n04118776 +ILSVRC2012_val_00039430.JPEG:n03461385 +ILSVRC2012_val_00039431.JPEG:n04275548 +ILSVRC2012_val_00039432.JPEG:n15075141 +ILSVRC2012_val_00039433.JPEG:n03761084 +ILSVRC2012_val_00039434.JPEG:n01944390 +ILSVRC2012_val_00039435.JPEG:n04317175 +ILSVRC2012_val_00039436.JPEG:n04152593 +ILSVRC2012_val_00039437.JPEG:n02927161 +ILSVRC2012_val_00039438.JPEG:n03956157 +ILSVRC2012_val_00039439.JPEG:n02085620 +ILSVRC2012_val_00039440.JPEG:n02727426 +ILSVRC2012_val_00039441.JPEG:n01667114 +ILSVRC2012_val_00039442.JPEG:n04493381 +ILSVRC2012_val_00039443.JPEG:n01729322 +ILSVRC2012_val_00039444.JPEG:n04081281 +ILSVRC2012_val_00039445.JPEG:n01484850 +ILSVRC2012_val_00039446.JPEG:n03124043 +ILSVRC2012_val_00039447.JPEG:n02841315 +ILSVRC2012_val_00039448.JPEG:n02108089 +ILSVRC2012_val_00039449.JPEG:n03345487 +ILSVRC2012_val_00039450.JPEG:n02892201 +ILSVRC2012_val_00039451.JPEG:n07875152 +ILSVRC2012_val_00039452.JPEG:n02093991 +ILSVRC2012_val_00039453.JPEG:n03697007 +ILSVRC2012_val_00039454.JPEG:n02119789 +ILSVRC2012_val_00039455.JPEG:n01739381 +ILSVRC2012_val_00039456.JPEG:n02319095 +ILSVRC2012_val_00039457.JPEG:n02361337 +ILSVRC2012_val_00039458.JPEG:n01883070 +ILSVRC2012_val_00039459.JPEG:n02492035 +ILSVRC2012_val_00039460.JPEG:n02107312 +ILSVRC2012_val_00039461.JPEG:n07715103 +ILSVRC2012_val_00039462.JPEG:n04264628 +ILSVRC2012_val_00039463.JPEG:n01843065 +ILSVRC2012_val_00039464.JPEG:n07860988 +ILSVRC2012_val_00039465.JPEG:n01795545 +ILSVRC2012_val_00039466.JPEG:n01592084 +ILSVRC2012_val_00039467.JPEG:n03676483 +ILSVRC2012_val_00039468.JPEG:n04254120 +ILSVRC2012_val_00039469.JPEG:n03223299 +ILSVRC2012_val_00039470.JPEG:n03220513 +ILSVRC2012_val_00039471.JPEG:n02108915 +ILSVRC2012_val_00039472.JPEG:n03873416 +ILSVRC2012_val_00039473.JPEG:n02128925 +ILSVRC2012_val_00039474.JPEG:n02389026 +ILSVRC2012_val_00039475.JPEG:n01698640 +ILSVRC2012_val_00039476.JPEG:n15075141 +ILSVRC2012_val_00039477.JPEG:n03028079 +ILSVRC2012_val_00039478.JPEG:n01644900 +ILSVRC2012_val_00039479.JPEG:n01694178 +ILSVRC2012_val_00039480.JPEG:n03761084 +ILSVRC2012_val_00039481.JPEG:n03873416 +ILSVRC2012_val_00039482.JPEG:n03710637 +ILSVRC2012_val_00039483.JPEG:n03924679 +ILSVRC2012_val_00039484.JPEG:n03627232 +ILSVRC2012_val_00039485.JPEG:n04542943 +ILSVRC2012_val_00039486.JPEG:n03095699 +ILSVRC2012_val_00039487.JPEG:n02100236 +ILSVRC2012_val_00039488.JPEG:n01784675 +ILSVRC2012_val_00039489.JPEG:n01744401 +ILSVRC2012_val_00039490.JPEG:n04153751 +ILSVRC2012_val_00039491.JPEG:n03770439 +ILSVRC2012_val_00039492.JPEG:n02107142 +ILSVRC2012_val_00039493.JPEG:n03297495 +ILSVRC2012_val_00039494.JPEG:n07753275 +ILSVRC2012_val_00039495.JPEG:n04008634 +ILSVRC2012_val_00039496.JPEG:n07615774 +ILSVRC2012_val_00039497.JPEG:n04550184 +ILSVRC2012_val_00039498.JPEG:n02110806 +ILSVRC2012_val_00039499.JPEG:n04404412 +ILSVRC2012_val_00039500.JPEG:n03976467 +ILSVRC2012_val_00039501.JPEG:n07715103 +ILSVRC2012_val_00039502.JPEG:n04525038 +ILSVRC2012_val_00039503.JPEG:n02776631 +ILSVRC2012_val_00039504.JPEG:n02099267 +ILSVRC2012_val_00039505.JPEG:n02095314 +ILSVRC2012_val_00039506.JPEG:n03028079 +ILSVRC2012_val_00039507.JPEG:n02100236 +ILSVRC2012_val_00039508.JPEG:n03930630 +ILSVRC2012_val_00039509.JPEG:n03188531 +ILSVRC2012_val_00039510.JPEG:n02094258 +ILSVRC2012_val_00039511.JPEG:n04554684 +ILSVRC2012_val_00039512.JPEG:n03887697 +ILSVRC2012_val_00039513.JPEG:n02116738 +ILSVRC2012_val_00039514.JPEG:n02007558 +ILSVRC2012_val_00039515.JPEG:n02102973 +ILSVRC2012_val_00039516.JPEG:n02130308 +ILSVRC2012_val_00039517.JPEG:n04328186 +ILSVRC2012_val_00039518.JPEG:n04141076 +ILSVRC2012_val_00039519.JPEG:n03220513 +ILSVRC2012_val_00039520.JPEG:n02444819 +ILSVRC2012_val_00039521.JPEG:n04458633 +ILSVRC2012_val_00039522.JPEG:n01735189 +ILSVRC2012_val_00039523.JPEG:n02701002 +ILSVRC2012_val_00039524.JPEG:n02071294 +ILSVRC2012_val_00039525.JPEG:n01498041 +ILSVRC2012_val_00039526.JPEG:n04070727 +ILSVRC2012_val_00039527.JPEG:n04423845 +ILSVRC2012_val_00039528.JPEG:n02089973 +ILSVRC2012_val_00039529.JPEG:n04141975 +ILSVRC2012_val_00039530.JPEG:n01729322 +ILSVRC2012_val_00039531.JPEG:n01824575 +ILSVRC2012_val_00039532.JPEG:n04251144 +ILSVRC2012_val_00039533.JPEG:n01692333 +ILSVRC2012_val_00039534.JPEG:n01484850 +ILSVRC2012_val_00039535.JPEG:n04208210 +ILSVRC2012_val_00039536.JPEG:n01667114 +ILSVRC2012_val_00039537.JPEG:n04458633 +ILSVRC2012_val_00039538.JPEG:n04141076 +ILSVRC2012_val_00039539.JPEG:n02058221 +ILSVRC2012_val_00039540.JPEG:n02088466 +ILSVRC2012_val_00039541.JPEG:n07760859 +ILSVRC2012_val_00039542.JPEG:n04560804 +ILSVRC2012_val_00039543.JPEG:n02099267 +ILSVRC2012_val_00039544.JPEG:n03000134 +ILSVRC2012_val_00039545.JPEG:n02481823 +ILSVRC2012_val_00039546.JPEG:n02788148 +ILSVRC2012_val_00039547.JPEG:n02097047 +ILSVRC2012_val_00039548.JPEG:n04487081 +ILSVRC2012_val_00039549.JPEG:n04286575 +ILSVRC2012_val_00039550.JPEG:n02233338 +ILSVRC2012_val_00039551.JPEG:n04344873 +ILSVRC2012_val_00039552.JPEG:n02490219 +ILSVRC2012_val_00039553.JPEG:n02123159 +ILSVRC2012_val_00039554.JPEG:n02120079 +ILSVRC2012_val_00039555.JPEG:n02114855 +ILSVRC2012_val_00039556.JPEG:n02088238 +ILSVRC2012_val_00039557.JPEG:n01775062 +ILSVRC2012_val_00039558.JPEG:n04136333 +ILSVRC2012_val_00039559.JPEG:n03344393 +ILSVRC2012_val_00039560.JPEG:n03535780 +ILSVRC2012_val_00039561.JPEG:n02074367 +ILSVRC2012_val_00039562.JPEG:n03782006 +ILSVRC2012_val_00039563.JPEG:n02487347 +ILSVRC2012_val_00039564.JPEG:n02134418 +ILSVRC2012_val_00039565.JPEG:n02500267 +ILSVRC2012_val_00039566.JPEG:n03208938 +ILSVRC2012_val_00039567.JPEG:n04162706 +ILSVRC2012_val_00039568.JPEG:n02410509 +ILSVRC2012_val_00039569.JPEG:n02091635 +ILSVRC2012_val_00039570.JPEG:n04417672 +ILSVRC2012_val_00039571.JPEG:n01537544 +ILSVRC2012_val_00039572.JPEG:n02951358 +ILSVRC2012_val_00039573.JPEG:n02116738 +ILSVRC2012_val_00039574.JPEG:n03594734 +ILSVRC2012_val_00039575.JPEG:n03775071 +ILSVRC2012_val_00039576.JPEG:n03594945 +ILSVRC2012_val_00039577.JPEG:n04532670 +ILSVRC2012_val_00039578.JPEG:n01695060 +ILSVRC2012_val_00039579.JPEG:n02277742 +ILSVRC2012_val_00039580.JPEG:n02123597 +ILSVRC2012_val_00039581.JPEG:n02883205 +ILSVRC2012_val_00039582.JPEG:n07932039 +ILSVRC2012_val_00039583.JPEG:n02497673 +ILSVRC2012_val_00039584.JPEG:n07754684 +ILSVRC2012_val_00039585.JPEG:n02112018 +ILSVRC2012_val_00039586.JPEG:n03538406 +ILSVRC2012_val_00039587.JPEG:n03895866 +ILSVRC2012_val_00039588.JPEG:n01494475 +ILSVRC2012_val_00039589.JPEG:n02177972 +ILSVRC2012_val_00039590.JPEG:n03197337 +ILSVRC2012_val_00039591.JPEG:n02105641 +ILSVRC2012_val_00039592.JPEG:n02992529 +ILSVRC2012_val_00039593.JPEG:n04070727 +ILSVRC2012_val_00039594.JPEG:n02109525 +ILSVRC2012_val_00039595.JPEG:n02125311 +ILSVRC2012_val_00039596.JPEG:n04456115 +ILSVRC2012_val_00039597.JPEG:n02980441 +ILSVRC2012_val_00039598.JPEG:n03841143 +ILSVRC2012_val_00039599.JPEG:n03938244 +ILSVRC2012_val_00039600.JPEG:n03661043 +ILSVRC2012_val_00039601.JPEG:n01756291 +ILSVRC2012_val_00039602.JPEG:n03794056 +ILSVRC2012_val_00039603.JPEG:n02018207 +ILSVRC2012_val_00039604.JPEG:n03126707 +ILSVRC2012_val_00039605.JPEG:n01614925 +ILSVRC2012_val_00039606.JPEG:n03992509 +ILSVRC2012_val_00039607.JPEG:n03127925 +ILSVRC2012_val_00039608.JPEG:n02115913 +ILSVRC2012_val_00039609.JPEG:n03773504 +ILSVRC2012_val_00039610.JPEG:n02776631 +ILSVRC2012_val_00039611.JPEG:n09472597 +ILSVRC2012_val_00039612.JPEG:n02177972 +ILSVRC2012_val_00039613.JPEG:n03532672 +ILSVRC2012_val_00039614.JPEG:n04476259 +ILSVRC2012_val_00039615.JPEG:n04517823 +ILSVRC2012_val_00039616.JPEG:n13052670 +ILSVRC2012_val_00039617.JPEG:n07753275 +ILSVRC2012_val_00039618.JPEG:n01685808 +ILSVRC2012_val_00039619.JPEG:n04120489 +ILSVRC2012_val_00039620.JPEG:n02120079 +ILSVRC2012_val_00039621.JPEG:n02123159 +ILSVRC2012_val_00039622.JPEG:n02087046 +ILSVRC2012_val_00039623.JPEG:n03598930 +ILSVRC2012_val_00039624.JPEG:n02487347 +ILSVRC2012_val_00039625.JPEG:n03065424 +ILSVRC2012_val_00039626.JPEG:n04517823 +ILSVRC2012_val_00039627.JPEG:n02797295 +ILSVRC2012_val_00039628.JPEG:n02804414 +ILSVRC2012_val_00039629.JPEG:n02843684 +ILSVRC2012_val_00039630.JPEG:n02018795 +ILSVRC2012_val_00039631.JPEG:n03976657 +ILSVRC2012_val_00039632.JPEG:n04005630 +ILSVRC2012_val_00039633.JPEG:n02699494 +ILSVRC2012_val_00039634.JPEG:n03814906 +ILSVRC2012_val_00039635.JPEG:n09332890 +ILSVRC2012_val_00039636.JPEG:n02493793 +ILSVRC2012_val_00039637.JPEG:n04442312 +ILSVRC2012_val_00039638.JPEG:n02100877 +ILSVRC2012_val_00039639.JPEG:n04532670 +ILSVRC2012_val_00039640.JPEG:n03047690 +ILSVRC2012_val_00039641.JPEG:n02077923 +ILSVRC2012_val_00039642.JPEG:n03733281 +ILSVRC2012_val_00039643.JPEG:n04266014 +ILSVRC2012_val_00039644.JPEG:n09835506 +ILSVRC2012_val_00039645.JPEG:n02492660 +ILSVRC2012_val_00039646.JPEG:n04330267 +ILSVRC2012_val_00039647.JPEG:n07716358 +ILSVRC2012_val_00039648.JPEG:n01601694 +ILSVRC2012_val_00039649.JPEG:n04579432 +ILSVRC2012_val_00039650.JPEG:n04380533 +ILSVRC2012_val_00039651.JPEG:n01749939 +ILSVRC2012_val_00039652.JPEG:n03444034 +ILSVRC2012_val_00039653.JPEG:n03400231 +ILSVRC2012_val_00039654.JPEG:n03584254 +ILSVRC2012_val_00039655.JPEG:n03710721 +ILSVRC2012_val_00039656.JPEG:n03895866 +ILSVRC2012_val_00039657.JPEG:n04591713 +ILSVRC2012_val_00039658.JPEG:n03903868 +ILSVRC2012_val_00039659.JPEG:n02088364 +ILSVRC2012_val_00039660.JPEG:n04141975 +ILSVRC2012_val_00039661.JPEG:n01774384 +ILSVRC2012_val_00039662.JPEG:n02112018 +ILSVRC2012_val_00039663.JPEG:n04485082 +ILSVRC2012_val_00039664.JPEG:n04259630 +ILSVRC2012_val_00039665.JPEG:n03041632 +ILSVRC2012_val_00039666.JPEG:n02097130 +ILSVRC2012_val_00039667.JPEG:n03775546 +ILSVRC2012_val_00039668.JPEG:n02093991 +ILSVRC2012_val_00039669.JPEG:n01742172 +ILSVRC2012_val_00039670.JPEG:n09193705 +ILSVRC2012_val_00039671.JPEG:n01984695 +ILSVRC2012_val_00039672.JPEG:n01924916 +ILSVRC2012_val_00039673.JPEG:n02190166 +ILSVRC2012_val_00039674.JPEG:n03706229 +ILSVRC2012_val_00039675.JPEG:n13037406 +ILSVRC2012_val_00039676.JPEG:n04604644 +ILSVRC2012_val_00039677.JPEG:n03602883 +ILSVRC2012_val_00039678.JPEG:n02504458 +ILSVRC2012_val_00039679.JPEG:n03467068 +ILSVRC2012_val_00039680.JPEG:n04536866 +ILSVRC2012_val_00039681.JPEG:n04398044 +ILSVRC2012_val_00039682.JPEG:n01986214 +ILSVRC2012_val_00039683.JPEG:n03777754 +ILSVRC2012_val_00039684.JPEG:n02066245 +ILSVRC2012_val_00039685.JPEG:n02346627 +ILSVRC2012_val_00039686.JPEG:n04370456 +ILSVRC2012_val_00039687.JPEG:n02108551 +ILSVRC2012_val_00039688.JPEG:n04204238 +ILSVRC2012_val_00039689.JPEG:n04371430 +ILSVRC2012_val_00039690.JPEG:n03792972 +ILSVRC2012_val_00039691.JPEG:n02441942 +ILSVRC2012_val_00039692.JPEG:n02096294 +ILSVRC2012_val_00039693.JPEG:n02699494 +ILSVRC2012_val_00039694.JPEG:n04589890 +ILSVRC2012_val_00039695.JPEG:n02085936 +ILSVRC2012_val_00039696.JPEG:n02105056 +ILSVRC2012_val_00039697.JPEG:n02415577 +ILSVRC2012_val_00039698.JPEG:n07734744 +ILSVRC2012_val_00039699.JPEG:n02098286 +ILSVRC2012_val_00039700.JPEG:n02113186 +ILSVRC2012_val_00039701.JPEG:n02096294 +ILSVRC2012_val_00039702.JPEG:n02871525 +ILSVRC2012_val_00039703.JPEG:n03873416 +ILSVRC2012_val_00039704.JPEG:n01784675 +ILSVRC2012_val_00039705.JPEG:n02788148 +ILSVRC2012_val_00039706.JPEG:n02051845 +ILSVRC2012_val_00039707.JPEG:n07930864 +ILSVRC2012_val_00039708.JPEG:n01692333 +ILSVRC2012_val_00039709.JPEG:n02111889 +ILSVRC2012_val_00039710.JPEG:n03662601 +ILSVRC2012_val_00039711.JPEG:n02097474 +ILSVRC2012_val_00039712.JPEG:n02165456 +ILSVRC2012_val_00039713.JPEG:n03595614 +ILSVRC2012_val_00039714.JPEG:n03452741 +ILSVRC2012_val_00039715.JPEG:n04606251 +ILSVRC2012_val_00039716.JPEG:n03796401 +ILSVRC2012_val_00039717.JPEG:n03452741 +ILSVRC2012_val_00039718.JPEG:n07693725 +ILSVRC2012_val_00039719.JPEG:n02112018 +ILSVRC2012_val_00039720.JPEG:n03388549 +ILSVRC2012_val_00039721.JPEG:n04562935 +ILSVRC2012_val_00039722.JPEG:n13133613 +ILSVRC2012_val_00039723.JPEG:n04461696 +ILSVRC2012_val_00039724.JPEG:n01796340 +ILSVRC2012_val_00039725.JPEG:n04270147 +ILSVRC2012_val_00039726.JPEG:n03187595 +ILSVRC2012_val_00039727.JPEG:n03666591 +ILSVRC2012_val_00039728.JPEG:n04120489 +ILSVRC2012_val_00039729.JPEG:n04522168 +ILSVRC2012_val_00039730.JPEG:n02111500 +ILSVRC2012_val_00039731.JPEG:n03976467 +ILSVRC2012_val_00039732.JPEG:n01729322 +ILSVRC2012_val_00039733.JPEG:n02364673 +ILSVRC2012_val_00039734.JPEG:n04356056 +ILSVRC2012_val_00039735.JPEG:n02797295 +ILSVRC2012_val_00039736.JPEG:n02114855 +ILSVRC2012_val_00039737.JPEG:n02749479 +ILSVRC2012_val_00039738.JPEG:n04357314 +ILSVRC2012_val_00039739.JPEG:n07565083 +ILSVRC2012_val_00039740.JPEG:n02676566 +ILSVRC2012_val_00039741.JPEG:n02088466 +ILSVRC2012_val_00039742.JPEG:n02823750 +ILSVRC2012_val_00039743.JPEG:n02093256 +ILSVRC2012_val_00039744.JPEG:n02256656 +ILSVRC2012_val_00039745.JPEG:n02119022 +ILSVRC2012_val_00039746.JPEG:n02883205 +ILSVRC2012_val_00039747.JPEG:n03584254 +ILSVRC2012_val_00039748.JPEG:n03775071 +ILSVRC2012_val_00039749.JPEG:n01682714 +ILSVRC2012_val_00039750.JPEG:n03124170 +ILSVRC2012_val_00039751.JPEG:n04201297 +ILSVRC2012_val_00039752.JPEG:n04044716 +ILSVRC2012_val_00039753.JPEG:n01629819 +ILSVRC2012_val_00039754.JPEG:n12998815 +ILSVRC2012_val_00039755.JPEG:n07584110 +ILSVRC2012_val_00039756.JPEG:n04532106 +ILSVRC2012_val_00039757.JPEG:n03825788 +ILSVRC2012_val_00039758.JPEG:n04501370 +ILSVRC2012_val_00039759.JPEG:n01560419 +ILSVRC2012_val_00039760.JPEG:n03065424 +ILSVRC2012_val_00039761.JPEG:n02106030 +ILSVRC2012_val_00039762.JPEG:n04229816 +ILSVRC2012_val_00039763.JPEG:n03623198 +ILSVRC2012_val_00039764.JPEG:n02280649 +ILSVRC2012_val_00039765.JPEG:n06785654 +ILSVRC2012_val_00039766.JPEG:n02342885 +ILSVRC2012_val_00039767.JPEG:n02488291 +ILSVRC2012_val_00039768.JPEG:n02606052 +ILSVRC2012_val_00039769.JPEG:n03271574 +ILSVRC2012_val_00039770.JPEG:n04070727 +ILSVRC2012_val_00039771.JPEG:n03717622 +ILSVRC2012_val_00039772.JPEG:n02447366 +ILSVRC2012_val_00039773.JPEG:n03065424 +ILSVRC2012_val_00039774.JPEG:n03527444 +ILSVRC2012_val_00039775.JPEG:n01943899 +ILSVRC2012_val_00039776.JPEG:n02095889 +ILSVRC2012_val_00039777.JPEG:n02132136 +ILSVRC2012_val_00039778.JPEG:n04204347 +ILSVRC2012_val_00039779.JPEG:n03026506 +ILSVRC2012_val_00039780.JPEG:n01749939 +ILSVRC2012_val_00039781.JPEG:n03742115 +ILSVRC2012_val_00039782.JPEG:n02105162 +ILSVRC2012_val_00039783.JPEG:n03733281 +ILSVRC2012_val_00039784.JPEG:n02006656 +ILSVRC2012_val_00039785.JPEG:n04552348 +ILSVRC2012_val_00039786.JPEG:n02493793 +ILSVRC2012_val_00039787.JPEG:n02992211 +ILSVRC2012_val_00039788.JPEG:n02089867 +ILSVRC2012_val_00039789.JPEG:n04111531 +ILSVRC2012_val_00039790.JPEG:n04590129 +ILSVRC2012_val_00039791.JPEG:n03982430 +ILSVRC2012_val_00039792.JPEG:n03495258 +ILSVRC2012_val_00039793.JPEG:n02640242 +ILSVRC2012_val_00039794.JPEG:n02099429 +ILSVRC2012_val_00039795.JPEG:n02132136 +ILSVRC2012_val_00039796.JPEG:n02444819 +ILSVRC2012_val_00039797.JPEG:n02056570 +ILSVRC2012_val_00039798.JPEG:n03494278 +ILSVRC2012_val_00039799.JPEG:n01773157 +ILSVRC2012_val_00039800.JPEG:n02137549 +ILSVRC2012_val_00039801.JPEG:n01534433 +ILSVRC2012_val_00039802.JPEG:n02018795 +ILSVRC2012_val_00039803.JPEG:n03630383 +ILSVRC2012_val_00039804.JPEG:n02281787 +ILSVRC2012_val_00039805.JPEG:n04120489 +ILSVRC2012_val_00039806.JPEG:n02104029 +ILSVRC2012_val_00039807.JPEG:n02098413 +ILSVRC2012_val_00039808.JPEG:n02488702 +ILSVRC2012_val_00039809.JPEG:n03379051 +ILSVRC2012_val_00039810.JPEG:n02807133 +ILSVRC2012_val_00039811.JPEG:n04591713 +ILSVRC2012_val_00039812.JPEG:n02110185 +ILSVRC2012_val_00039813.JPEG:n04209239 +ILSVRC2012_val_00039814.JPEG:n01558993 +ILSVRC2012_val_00039815.JPEG:n04325704 +ILSVRC2012_val_00039816.JPEG:n04264628 +ILSVRC2012_val_00039817.JPEG:n03291819 +ILSVRC2012_val_00039818.JPEG:n02793495 +ILSVRC2012_val_00039819.JPEG:n02133161 +ILSVRC2012_val_00039820.JPEG:n03908714 +ILSVRC2012_val_00039821.JPEG:n03584254 +ILSVRC2012_val_00039822.JPEG:n02091831 +ILSVRC2012_val_00039823.JPEG:n02099429 +ILSVRC2012_val_00039824.JPEG:n09835506 +ILSVRC2012_val_00039825.JPEG:n01798484 +ILSVRC2012_val_00039826.JPEG:n03041632 +ILSVRC2012_val_00039827.JPEG:n02808304 +ILSVRC2012_val_00039828.JPEG:n04136333 +ILSVRC2012_val_00039829.JPEG:n09428293 +ILSVRC2012_val_00039830.JPEG:n04465501 +ILSVRC2012_val_00039831.JPEG:n01688243 +ILSVRC2012_val_00039832.JPEG:n02093428 +ILSVRC2012_val_00039833.JPEG:n02129165 +ILSVRC2012_val_00039834.JPEG:n07749582 +ILSVRC2012_val_00039835.JPEG:n03197337 +ILSVRC2012_val_00039836.JPEG:n04392985 +ILSVRC2012_val_00039837.JPEG:n04367480 +ILSVRC2012_val_00039838.JPEG:n02484975 +ILSVRC2012_val_00039839.JPEG:n02607072 +ILSVRC2012_val_00039840.JPEG:n03089624 +ILSVRC2012_val_00039841.JPEG:n04116512 +ILSVRC2012_val_00039842.JPEG:n04286575 +ILSVRC2012_val_00039843.JPEG:n02233338 +ILSVRC2012_val_00039844.JPEG:n04118538 +ILSVRC2012_val_00039845.JPEG:n04254777 +ILSVRC2012_val_00039846.JPEG:n02410509 +ILSVRC2012_val_00039847.JPEG:n02091244 +ILSVRC2012_val_00039848.JPEG:n03016953 +ILSVRC2012_val_00039849.JPEG:n03026506 +ILSVRC2012_val_00039850.JPEG:n02113978 +ILSVRC2012_val_00039851.JPEG:n02091032 +ILSVRC2012_val_00039852.JPEG:n02096585 +ILSVRC2012_val_00039853.JPEG:n04179913 +ILSVRC2012_val_00039854.JPEG:n01775062 +ILSVRC2012_val_00039855.JPEG:n03903868 +ILSVRC2012_val_00039856.JPEG:n04277352 +ILSVRC2012_val_00039857.JPEG:n02841315 +ILSVRC2012_val_00039858.JPEG:n04597913 +ILSVRC2012_val_00039859.JPEG:n01614925 +ILSVRC2012_val_00039860.JPEG:n04067472 +ILSVRC2012_val_00039861.JPEG:n03876231 +ILSVRC2012_val_00039862.JPEG:n02095889 +ILSVRC2012_val_00039863.JPEG:n02100877 +ILSVRC2012_val_00039864.JPEG:n03444034 +ILSVRC2012_val_00039865.JPEG:n01484850 +ILSVRC2012_val_00039866.JPEG:n02490219 +ILSVRC2012_val_00039867.JPEG:n03272010 +ILSVRC2012_val_00039868.JPEG:n12057211 +ILSVRC2012_val_00039869.JPEG:n03980874 +ILSVRC2012_val_00039870.JPEG:n02097474 +ILSVRC2012_val_00039871.JPEG:n04270147 +ILSVRC2012_val_00039872.JPEG:n04429376 +ILSVRC2012_val_00039873.JPEG:n04111531 +ILSVRC2012_val_00039874.JPEG:n09399592 +ILSVRC2012_val_00039875.JPEG:n04005630 +ILSVRC2012_val_00039876.JPEG:n03595614 +ILSVRC2012_val_00039877.JPEG:n02123045 +ILSVRC2012_val_00039878.JPEG:n03657121 +ILSVRC2012_val_00039879.JPEG:n07892512 +ILSVRC2012_val_00039880.JPEG:n03840681 +ILSVRC2012_val_00039881.JPEG:n04296562 +ILSVRC2012_val_00039882.JPEG:n02807133 +ILSVRC2012_val_00039883.JPEG:n01806567 +ILSVRC2012_val_00039884.JPEG:n04258138 +ILSVRC2012_val_00039885.JPEG:n02114367 +ILSVRC2012_val_00039886.JPEG:n01675722 +ILSVRC2012_val_00039887.JPEG:n02794156 +ILSVRC2012_val_00039888.JPEG:n01698640 +ILSVRC2012_val_00039889.JPEG:n04296562 +ILSVRC2012_val_00039890.JPEG:n07717556 +ILSVRC2012_val_00039891.JPEG:n03476991 +ILSVRC2012_val_00039892.JPEG:n04005630 +ILSVRC2012_val_00039893.JPEG:n02099712 +ILSVRC2012_val_00039894.JPEG:n02099429 +ILSVRC2012_val_00039895.JPEG:n03721384 +ILSVRC2012_val_00039896.JPEG:n04277352 +ILSVRC2012_val_00039897.JPEG:n03127925 +ILSVRC2012_val_00039898.JPEG:n02256656 +ILSVRC2012_val_00039899.JPEG:n03201208 +ILSVRC2012_val_00039900.JPEG:n02088466 +ILSVRC2012_val_00039901.JPEG:n02086079 +ILSVRC2012_val_00039902.JPEG:n01632458 +ILSVRC2012_val_00039903.JPEG:n04376876 +ILSVRC2012_val_00039904.JPEG:n03998194 +ILSVRC2012_val_00039905.JPEG:n01440764 +ILSVRC2012_val_00039906.JPEG:n02704792 +ILSVRC2012_val_00039907.JPEG:n01855032 +ILSVRC2012_val_00039908.JPEG:n03095699 +ILSVRC2012_val_00039909.JPEG:n04355933 +ILSVRC2012_val_00039910.JPEG:n04465501 +ILSVRC2012_val_00039911.JPEG:n03841143 +ILSVRC2012_val_00039912.JPEG:n04501370 +ILSVRC2012_val_00039913.JPEG:n01558993 +ILSVRC2012_val_00039914.JPEG:n03042490 +ILSVRC2012_val_00039915.JPEG:n01950731 +ILSVRC2012_val_00039916.JPEG:n03935335 +ILSVRC2012_val_00039917.JPEG:n04584207 +ILSVRC2012_val_00039918.JPEG:n01984695 +ILSVRC2012_val_00039919.JPEG:n02747177 +ILSVRC2012_val_00039920.JPEG:n03775546 +ILSVRC2012_val_00039921.JPEG:n04525038 +ILSVRC2012_val_00039922.JPEG:n01632777 +ILSVRC2012_val_00039923.JPEG:n04485082 +ILSVRC2012_val_00039924.JPEG:n04116512 +ILSVRC2012_val_00039925.JPEG:n02486410 +ILSVRC2012_val_00039926.JPEG:n02096585 +ILSVRC2012_val_00039927.JPEG:n02096051 +ILSVRC2012_val_00039928.JPEG:n02110627 +ILSVRC2012_val_00039929.JPEG:n03272010 +ILSVRC2012_val_00039930.JPEG:n03775546 +ILSVRC2012_val_00039931.JPEG:n02123597 +ILSVRC2012_val_00039932.JPEG:n02992529 +ILSVRC2012_val_00039933.JPEG:n01632458 +ILSVRC2012_val_00039934.JPEG:n02089078 +ILSVRC2012_val_00039935.JPEG:n03954731 +ILSVRC2012_val_00039936.JPEG:n02437616 +ILSVRC2012_val_00039937.JPEG:n02120505 +ILSVRC2012_val_00039938.JPEG:n04507155 +ILSVRC2012_val_00039939.JPEG:n02114712 +ILSVRC2012_val_00039940.JPEG:n03532672 +ILSVRC2012_val_00039941.JPEG:n03983396 +ILSVRC2012_val_00039942.JPEG:n02108000 +ILSVRC2012_val_00039943.JPEG:n01514859 +ILSVRC2012_val_00039944.JPEG:n07802026 +ILSVRC2012_val_00039945.JPEG:n02951358 +ILSVRC2012_val_00039946.JPEG:n01882714 +ILSVRC2012_val_00039947.JPEG:n04505470 +ILSVRC2012_val_00039948.JPEG:n02231487 +ILSVRC2012_val_00039949.JPEG:n03388043 +ILSVRC2012_val_00039950.JPEG:n04482393 +ILSVRC2012_val_00039951.JPEG:n02112018 +ILSVRC2012_val_00039952.JPEG:n04008634 +ILSVRC2012_val_00039953.JPEG:n02606052 +ILSVRC2012_val_00039954.JPEG:n04273569 +ILSVRC2012_val_00039955.JPEG:n03594734 +ILSVRC2012_val_00039956.JPEG:n04532670 +ILSVRC2012_val_00039957.JPEG:n01855032 +ILSVRC2012_val_00039958.JPEG:n02342885 +ILSVRC2012_val_00039959.JPEG:n03950228 +ILSVRC2012_val_00039960.JPEG:n02093859 +ILSVRC2012_val_00039961.JPEG:n02841315 +ILSVRC2012_val_00039962.JPEG:n02025239 +ILSVRC2012_val_00039963.JPEG:n03930630 +ILSVRC2012_val_00039964.JPEG:n01797886 +ILSVRC2012_val_00039965.JPEG:n03240683 +ILSVRC2012_val_00039966.JPEG:n01775062 +ILSVRC2012_val_00039967.JPEG:n02321529 +ILSVRC2012_val_00039968.JPEG:n02342885 +ILSVRC2012_val_00039969.JPEG:n02108551 +ILSVRC2012_val_00039970.JPEG:n03216828 +ILSVRC2012_val_00039971.JPEG:n02281406 +ILSVRC2012_val_00039972.JPEG:n03710721 +ILSVRC2012_val_00039973.JPEG:n04201297 +ILSVRC2012_val_00039974.JPEG:n01950731 +ILSVRC2012_val_00039975.JPEG:n03216828 +ILSVRC2012_val_00039976.JPEG:n07880968 +ILSVRC2012_val_00039977.JPEG:n04208210 +ILSVRC2012_val_00039978.JPEG:n02514041 +ILSVRC2012_val_00039979.JPEG:n02123597 +ILSVRC2012_val_00039980.JPEG:n04517823 +ILSVRC2012_val_00039981.JPEG:n04553703 +ILSVRC2012_val_00039982.JPEG:n03482405 +ILSVRC2012_val_00039983.JPEG:n07697313 +ILSVRC2012_val_00039984.JPEG:n03690938 +ILSVRC2012_val_00039985.JPEG:n02444819 +ILSVRC2012_val_00039986.JPEG:n04049303 +ILSVRC2012_val_00039987.JPEG:n03085013 +ILSVRC2012_val_00039988.JPEG:n01843065 +ILSVRC2012_val_00039989.JPEG:n03709823 +ILSVRC2012_val_00039990.JPEG:n02117135 +ILSVRC2012_val_00039991.JPEG:n02787622 +ILSVRC2012_val_00039992.JPEG:n07579787 +ILSVRC2012_val_00039993.JPEG:n02099601 +ILSVRC2012_val_00039994.JPEG:n04229816 +ILSVRC2012_val_00039995.JPEG:n03776460 +ILSVRC2012_val_00039996.JPEG:n01644900 +ILSVRC2012_val_00039997.JPEG:n07579787 +ILSVRC2012_val_00039998.JPEG:n03733281 +ILSVRC2012_val_00039999.JPEG:n09472597 +ILSVRC2012_val_00040000.JPEG:n01797886 +ILSVRC2012_val_00040001.JPEG:n07802026 +ILSVRC2012_val_00040002.JPEG:n01806567 +ILSVRC2012_val_00040003.JPEG:n02108551 +ILSVRC2012_val_00040004.JPEG:n02093754 +ILSVRC2012_val_00040005.JPEG:n02132136 +ILSVRC2012_val_00040006.JPEG:n04254120 +ILSVRC2012_val_00040007.JPEG:n03877472 +ILSVRC2012_val_00040008.JPEG:n02480855 +ILSVRC2012_val_00040009.JPEG:n04285008 +ILSVRC2012_val_00040010.JPEG:n15075141 +ILSVRC2012_val_00040011.JPEG:n04325704 +ILSVRC2012_val_00040012.JPEG:n09332890 +ILSVRC2012_val_00040013.JPEG:n03947888 +ILSVRC2012_val_00040014.JPEG:n01828970 +ILSVRC2012_val_00040015.JPEG:n02106030 +ILSVRC2012_val_00040016.JPEG:n04501370 +ILSVRC2012_val_00040017.JPEG:n07730033 +ILSVRC2012_val_00040018.JPEG:n02113186 +ILSVRC2012_val_00040019.JPEG:n03026506 +ILSVRC2012_val_00040020.JPEG:n04266014 +ILSVRC2012_val_00040021.JPEG:n11939491 +ILSVRC2012_val_00040022.JPEG:n04270147 +ILSVRC2012_val_00040023.JPEG:n03777754 +ILSVRC2012_val_00040024.JPEG:n04522168 +ILSVRC2012_val_00040025.JPEG:n01860187 +ILSVRC2012_val_00040026.JPEG:n02443484 +ILSVRC2012_val_00040027.JPEG:n02835271 +ILSVRC2012_val_00040028.JPEG:n04125021 +ILSVRC2012_val_00040029.JPEG:n02794156 +ILSVRC2012_val_00040030.JPEG:n06596364 +ILSVRC2012_val_00040031.JPEG:n04265275 +ILSVRC2012_val_00040032.JPEG:n04136333 +ILSVRC2012_val_00040033.JPEG:n10565667 +ILSVRC2012_val_00040034.JPEG:n04483307 +ILSVRC2012_val_00040035.JPEG:n02277742 +ILSVRC2012_val_00040036.JPEG:n02094433 +ILSVRC2012_val_00040037.JPEG:n07716906 +ILSVRC2012_val_00040038.JPEG:n01514859 +ILSVRC2012_val_00040039.JPEG:n02397096 +ILSVRC2012_val_00040040.JPEG:n02102318 +ILSVRC2012_val_00040041.JPEG:n04442312 +ILSVRC2012_val_00040042.JPEG:n03680355 +ILSVRC2012_val_00040043.JPEG:n02086240 +ILSVRC2012_val_00040044.JPEG:n02174001 +ILSVRC2012_val_00040045.JPEG:n02277742 +ILSVRC2012_val_00040046.JPEG:n03832673 +ILSVRC2012_val_00040047.JPEG:n01768244 +ILSVRC2012_val_00040048.JPEG:n01739381 +ILSVRC2012_val_00040049.JPEG:n02361337 +ILSVRC2012_val_00040050.JPEG:n02607072 +ILSVRC2012_val_00040051.JPEG:n01843383 +ILSVRC2012_val_00040052.JPEG:n02091467 +ILSVRC2012_val_00040053.JPEG:n02090721 +ILSVRC2012_val_00040054.JPEG:n01756291 +ILSVRC2012_val_00040055.JPEG:n02099429 +ILSVRC2012_val_00040056.JPEG:n01806567 +ILSVRC2012_val_00040057.JPEG:n02966687 +ILSVRC2012_val_00040058.JPEG:n02094258 +ILSVRC2012_val_00040059.JPEG:n01986214 +ILSVRC2012_val_00040060.JPEG:n07697537 +ILSVRC2012_val_00040061.JPEG:n02909870 +ILSVRC2012_val_00040062.JPEG:n03967562 +ILSVRC2012_val_00040063.JPEG:n04296562 +ILSVRC2012_val_00040064.JPEG:n03388043 +ILSVRC2012_val_00040065.JPEG:n04482393 +ILSVRC2012_val_00040066.JPEG:n09421951 +ILSVRC2012_val_00040067.JPEG:n07614500 +ILSVRC2012_val_00040068.JPEG:n02865351 +ILSVRC2012_val_00040069.JPEG:n02089973 +ILSVRC2012_val_00040070.JPEG:n04557648 +ILSVRC2012_val_00040071.JPEG:n01537544 +ILSVRC2012_val_00040072.JPEG:n01819313 +ILSVRC2012_val_00040073.JPEG:n03929855 +ILSVRC2012_val_00040074.JPEG:n04136333 +ILSVRC2012_val_00040075.JPEG:n03977966 +ILSVRC2012_val_00040076.JPEG:n04099969 +ILSVRC2012_val_00040077.JPEG:n01675722 +ILSVRC2012_val_00040078.JPEG:n03832673 +ILSVRC2012_val_00040079.JPEG:n02643566 +ILSVRC2012_val_00040080.JPEG:n07749582 +ILSVRC2012_val_00040081.JPEG:n04275548 +ILSVRC2012_val_00040082.JPEG:n04005630 +ILSVRC2012_val_00040083.JPEG:n02074367 +ILSVRC2012_val_00040084.JPEG:n03623198 +ILSVRC2012_val_00040085.JPEG:n03495258 +ILSVRC2012_val_00040086.JPEG:n04296562 +ILSVRC2012_val_00040087.JPEG:n02437312 +ILSVRC2012_val_00040088.JPEG:n02113799 +ILSVRC2012_val_00040089.JPEG:n03874599 +ILSVRC2012_val_00040090.JPEG:n02454379 +ILSVRC2012_val_00040091.JPEG:n02877765 +ILSVRC2012_val_00040092.JPEG:n02109525 +ILSVRC2012_val_00040093.JPEG:n04270147 +ILSVRC2012_val_00040094.JPEG:n01729977 +ILSVRC2012_val_00040095.JPEG:n02950826 +ILSVRC2012_val_00040096.JPEG:n02110063 +ILSVRC2012_val_00040097.JPEG:n03216828 +ILSVRC2012_val_00040098.JPEG:n01484850 +ILSVRC2012_val_00040099.JPEG:n03062245 +ILSVRC2012_val_00040100.JPEG:n02128385 +ILSVRC2012_val_00040101.JPEG:n04228054 +ILSVRC2012_val_00040102.JPEG:n03179701 +ILSVRC2012_val_00040103.JPEG:n01796340 +ILSVRC2012_val_00040104.JPEG:n01694178 +ILSVRC2012_val_00040105.JPEG:n02088094 +ILSVRC2012_val_00040106.JPEG:n03942813 +ILSVRC2012_val_00040107.JPEG:n02869837 +ILSVRC2012_val_00040108.JPEG:n03770439 +ILSVRC2012_val_00040109.JPEG:n02097658 +ILSVRC2012_val_00040110.JPEG:n03047690 +ILSVRC2012_val_00040111.JPEG:n03742115 +ILSVRC2012_val_00040112.JPEG:n03724870 +ILSVRC2012_val_00040113.JPEG:n02966687 +ILSVRC2012_val_00040114.JPEG:n02098286 +ILSVRC2012_val_00040115.JPEG:n01687978 +ILSVRC2012_val_00040116.JPEG:n02100236 +ILSVRC2012_val_00040117.JPEG:n01616318 +ILSVRC2012_val_00040118.JPEG:n04442312 +ILSVRC2012_val_00040119.JPEG:n02396427 +ILSVRC2012_val_00040120.JPEG:n03998194 +ILSVRC2012_val_00040121.JPEG:n01773549 +ILSVRC2012_val_00040122.JPEG:n07747607 +ILSVRC2012_val_00040123.JPEG:n01944390 +ILSVRC2012_val_00040124.JPEG:n03891332 +ILSVRC2012_val_00040125.JPEG:n03045698 +ILSVRC2012_val_00040126.JPEG:n03877472 +ILSVRC2012_val_00040127.JPEG:n03207941 +ILSVRC2012_val_00040128.JPEG:n02494079 +ILSVRC2012_val_00040129.JPEG:n01819313 +ILSVRC2012_val_00040130.JPEG:n02093754 +ILSVRC2012_val_00040131.JPEG:n02088238 +ILSVRC2012_val_00040132.JPEG:n02168699 +ILSVRC2012_val_00040133.JPEG:n04515003 +ILSVRC2012_val_00040134.JPEG:n01675722 +ILSVRC2012_val_00040135.JPEG:n02018207 +ILSVRC2012_val_00040136.JPEG:n02690373 +ILSVRC2012_val_00040137.JPEG:n03777568 +ILSVRC2012_val_00040138.JPEG:n03026506 +ILSVRC2012_val_00040139.JPEG:n02342885 +ILSVRC2012_val_00040140.JPEG:n02102040 +ILSVRC2012_val_00040141.JPEG:n07583066 +ILSVRC2012_val_00040142.JPEG:n03961711 +ILSVRC2012_val_00040143.JPEG:n02916936 +ILSVRC2012_val_00040144.JPEG:n03958227 +ILSVRC2012_val_00040145.JPEG:n01698640 +ILSVRC2012_val_00040146.JPEG:n07714990 +ILSVRC2012_val_00040147.JPEG:n02483708 +ILSVRC2012_val_00040148.JPEG:n03680355 +ILSVRC2012_val_00040149.JPEG:n04141975 +ILSVRC2012_val_00040150.JPEG:n02085936 +ILSVRC2012_val_00040151.JPEG:n07930864 +ILSVRC2012_val_00040152.JPEG:n03691459 +ILSVRC2012_val_00040153.JPEG:n02892767 +ILSVRC2012_val_00040154.JPEG:n03770679 +ILSVRC2012_val_00040155.JPEG:n03450230 +ILSVRC2012_val_00040156.JPEG:n02165456 +ILSVRC2012_val_00040157.JPEG:n04560804 +ILSVRC2012_val_00040158.JPEG:n01614925 +ILSVRC2012_val_00040159.JPEG:n04458633 +ILSVRC2012_val_00040160.JPEG:n02500267 +ILSVRC2012_val_00040161.JPEG:n02190166 +ILSVRC2012_val_00040162.JPEG:n04380533 +ILSVRC2012_val_00040163.JPEG:n02950826 +ILSVRC2012_val_00040164.JPEG:n07860988 +ILSVRC2012_val_00040165.JPEG:n02346627 +ILSVRC2012_val_00040166.JPEG:n03814906 +ILSVRC2012_val_00040167.JPEG:n02494079 +ILSVRC2012_val_00040168.JPEG:n01817953 +ILSVRC2012_val_00040169.JPEG:n09421951 +ILSVRC2012_val_00040170.JPEG:n03041632 +ILSVRC2012_val_00040171.JPEG:n04371430 +ILSVRC2012_val_00040172.JPEG:n04371430 +ILSVRC2012_val_00040173.JPEG:n03743016 +ILSVRC2012_val_00040174.JPEG:n01630670 +ILSVRC2012_val_00040175.JPEG:n04074963 +ILSVRC2012_val_00040176.JPEG:n04326547 +ILSVRC2012_val_00040177.JPEG:n02894605 +ILSVRC2012_val_00040178.JPEG:n02086910 +ILSVRC2012_val_00040179.JPEG:n03935335 +ILSVRC2012_val_00040180.JPEG:n04461696 +ILSVRC2012_val_00040181.JPEG:n03476991 +ILSVRC2012_val_00040182.JPEG:n03697007 +ILSVRC2012_val_00040183.JPEG:n01818515 +ILSVRC2012_val_00040184.JPEG:n04263257 +ILSVRC2012_val_00040185.JPEG:n02088238 +ILSVRC2012_val_00040186.JPEG:n07697313 +ILSVRC2012_val_00040187.JPEG:n02110806 +ILSVRC2012_val_00040188.JPEG:n07747607 +ILSVRC2012_val_00040189.JPEG:n02108422 +ILSVRC2012_val_00040190.JPEG:n02641379 +ILSVRC2012_val_00040191.JPEG:n04507155 +ILSVRC2012_val_00040192.JPEG:n02124075 +ILSVRC2012_val_00040193.JPEG:n12985857 +ILSVRC2012_val_00040194.JPEG:n02342885 +ILSVRC2012_val_00040195.JPEG:n07697537 +ILSVRC2012_val_00040196.JPEG:n03742115 +ILSVRC2012_val_00040197.JPEG:n12998815 +ILSVRC2012_val_00040198.JPEG:n04591713 +ILSVRC2012_val_00040199.JPEG:n03450230 +ILSVRC2012_val_00040200.JPEG:n02110185 +ILSVRC2012_val_00040201.JPEG:n02091831 +ILSVRC2012_val_00040202.JPEG:n03424325 +ILSVRC2012_val_00040203.JPEG:n01795545 +ILSVRC2012_val_00040204.JPEG:n04507155 +ILSVRC2012_val_00040205.JPEG:n01616318 +ILSVRC2012_val_00040206.JPEG:n01704323 +ILSVRC2012_val_00040207.JPEG:n03887697 +ILSVRC2012_val_00040208.JPEG:n02128925 +ILSVRC2012_val_00040209.JPEG:n01824575 +ILSVRC2012_val_00040210.JPEG:n02099712 +ILSVRC2012_val_00040211.JPEG:n03498962 +ILSVRC2012_val_00040212.JPEG:n04273569 +ILSVRC2012_val_00040213.JPEG:n04090263 +ILSVRC2012_val_00040214.JPEG:n01775062 +ILSVRC2012_val_00040215.JPEG:n03970156 +ILSVRC2012_val_00040216.JPEG:n02480855 +ILSVRC2012_val_00040217.JPEG:n02730930 +ILSVRC2012_val_00040218.JPEG:n02326432 +ILSVRC2012_val_00040219.JPEG:n04355933 +ILSVRC2012_val_00040220.JPEG:n03355925 +ILSVRC2012_val_00040221.JPEG:n01734418 +ILSVRC2012_val_00040222.JPEG:n02107908 +ILSVRC2012_val_00040223.JPEG:n01978287 +ILSVRC2012_val_00040224.JPEG:n03874599 +ILSVRC2012_val_00040225.JPEG:n03478589 +ILSVRC2012_val_00040226.JPEG:n03788365 +ILSVRC2012_val_00040227.JPEG:n02325366 +ILSVRC2012_val_00040228.JPEG:n02445715 +ILSVRC2012_val_00040229.JPEG:n03180011 +ILSVRC2012_val_00040230.JPEG:n03792782 +ILSVRC2012_val_00040231.JPEG:n01667778 +ILSVRC2012_val_00040232.JPEG:n02490219 +ILSVRC2012_val_00040233.JPEG:n01882714 +ILSVRC2012_val_00040234.JPEG:n04005630 +ILSVRC2012_val_00040235.JPEG:n04118538 +ILSVRC2012_val_00040236.JPEG:n03775071 +ILSVRC2012_val_00040237.JPEG:n03792782 +ILSVRC2012_val_00040238.JPEG:n02123045 +ILSVRC2012_val_00040239.JPEG:n02264363 +ILSVRC2012_val_00040240.JPEG:n02776631 +ILSVRC2012_val_00040241.JPEG:n01773157 +ILSVRC2012_val_00040242.JPEG:n01614925 +ILSVRC2012_val_00040243.JPEG:n04548362 +ILSVRC2012_val_00040244.JPEG:n02009912 +ILSVRC2012_val_00040245.JPEG:n02487347 +ILSVRC2012_val_00040246.JPEG:n03272562 +ILSVRC2012_val_00040247.JPEG:n01685808 +ILSVRC2012_val_00040248.JPEG:n02835271 +ILSVRC2012_val_00040249.JPEG:n02110063 +ILSVRC2012_val_00040250.JPEG:n04153751 +ILSVRC2012_val_00040251.JPEG:n02123045 +ILSVRC2012_val_00040252.JPEG:n02417914 +ILSVRC2012_val_00040253.JPEG:n04208210 +ILSVRC2012_val_00040254.JPEG:n03476684 +ILSVRC2012_val_00040255.JPEG:n01768244 +ILSVRC2012_val_00040256.JPEG:n07697313 +ILSVRC2012_val_00040257.JPEG:n02100583 +ILSVRC2012_val_00040258.JPEG:n02504013 +ILSVRC2012_val_00040259.JPEG:n04040759 +ILSVRC2012_val_00040260.JPEG:n04067472 +ILSVRC2012_val_00040261.JPEG:n01798484 +ILSVRC2012_val_00040262.JPEG:n07248320 +ILSVRC2012_val_00040263.JPEG:n02094258 +ILSVRC2012_val_00040264.JPEG:n02483708 +ILSVRC2012_val_00040265.JPEG:n04557648 +ILSVRC2012_val_00040266.JPEG:n01828970 +ILSVRC2012_val_00040267.JPEG:n02172182 +ILSVRC2012_val_00040268.JPEG:n03658185 +ILSVRC2012_val_00040269.JPEG:n02493509 +ILSVRC2012_val_00040270.JPEG:n03991062 +ILSVRC2012_val_00040271.JPEG:n03494278 +ILSVRC2012_val_00040272.JPEG:n03291819 +ILSVRC2012_val_00040273.JPEG:n02410509 +ILSVRC2012_val_00040274.JPEG:n03733805 +ILSVRC2012_val_00040275.JPEG:n04579432 +ILSVRC2012_val_00040276.JPEG:n03124043 +ILSVRC2012_val_00040277.JPEG:n02966193 +ILSVRC2012_val_00040278.JPEG:n02190166 +ILSVRC2012_val_00040279.JPEG:n02526121 +ILSVRC2012_val_00040280.JPEG:n07753592 +ILSVRC2012_val_00040281.JPEG:n07753592 +ILSVRC2012_val_00040282.JPEG:n07768694 +ILSVRC2012_val_00040283.JPEG:n09246464 +ILSVRC2012_val_00040284.JPEG:n07711569 +ILSVRC2012_val_00040285.JPEG:n02018795 +ILSVRC2012_val_00040286.JPEG:n02105056 +ILSVRC2012_val_00040287.JPEG:n01669191 +ILSVRC2012_val_00040288.JPEG:n02268853 +ILSVRC2012_val_00040289.JPEG:n02488291 +ILSVRC2012_val_00040290.JPEG:n02793495 +ILSVRC2012_val_00040291.JPEG:n02101556 +ILSVRC2012_val_00040292.JPEG:n04476259 +ILSVRC2012_val_00040293.JPEG:n07584110 +ILSVRC2012_val_00040294.JPEG:n04542943 +ILSVRC2012_val_00040295.JPEG:n03670208 +ILSVRC2012_val_00040296.JPEG:n03929855 +ILSVRC2012_val_00040297.JPEG:n04204347 +ILSVRC2012_val_00040298.JPEG:n02094433 +ILSVRC2012_val_00040299.JPEG:n09472597 +ILSVRC2012_val_00040300.JPEG:n04479046 +ILSVRC2012_val_00040301.JPEG:n01667778 +ILSVRC2012_val_00040302.JPEG:n03459775 +ILSVRC2012_val_00040303.JPEG:n02056570 +ILSVRC2012_val_00040304.JPEG:n12620546 +ILSVRC2012_val_00040305.JPEG:n04286575 +ILSVRC2012_val_00040306.JPEG:n02795169 +ILSVRC2012_val_00040307.JPEG:n04209239 +ILSVRC2012_val_00040308.JPEG:n02101556 +ILSVRC2012_val_00040309.JPEG:n04532670 +ILSVRC2012_val_00040310.JPEG:n02009229 +ILSVRC2012_val_00040311.JPEG:n04584207 +ILSVRC2012_val_00040312.JPEG:n02795169 +ILSVRC2012_val_00040313.JPEG:n02112350 +ILSVRC2012_val_00040314.JPEG:n01667778 +ILSVRC2012_val_00040315.JPEG:n02939185 +ILSVRC2012_val_00040316.JPEG:n03908618 +ILSVRC2012_val_00040317.JPEG:n01753488 +ILSVRC2012_val_00040318.JPEG:n02841315 +ILSVRC2012_val_00040319.JPEG:n03388183 +ILSVRC2012_val_00040320.JPEG:n03218198 +ILSVRC2012_val_00040321.JPEG:n02776631 +ILSVRC2012_val_00040322.JPEG:n02363005 +ILSVRC2012_val_00040323.JPEG:n02130308 +ILSVRC2012_val_00040324.JPEG:n06596364 +ILSVRC2012_val_00040325.JPEG:n02814860 +ILSVRC2012_val_00040326.JPEG:n02110063 +ILSVRC2012_val_00040327.JPEG:n02117135 +ILSVRC2012_val_00040328.JPEG:n07684084 +ILSVRC2012_val_00040329.JPEG:n04254680 +ILSVRC2012_val_00040330.JPEG:n03109150 +ILSVRC2012_val_00040331.JPEG:n02408429 +ILSVRC2012_val_00040332.JPEG:n04389033 +ILSVRC2012_val_00040333.JPEG:n04483307 +ILSVRC2012_val_00040334.JPEG:n01797886 +ILSVRC2012_val_00040335.JPEG:n02095889 +ILSVRC2012_val_00040336.JPEG:n03958227 +ILSVRC2012_val_00040337.JPEG:n04548280 +ILSVRC2012_val_00040338.JPEG:n02410509 +ILSVRC2012_val_00040339.JPEG:n03837869 +ILSVRC2012_val_00040340.JPEG:n03720891 +ILSVRC2012_val_00040341.JPEG:n04435653 +ILSVRC2012_val_00040342.JPEG:n01498041 +ILSVRC2012_val_00040343.JPEG:n02749479 +ILSVRC2012_val_00040344.JPEG:n07718747 +ILSVRC2012_val_00040345.JPEG:n04461696 +ILSVRC2012_val_00040346.JPEG:n03388043 +ILSVRC2012_val_00040347.JPEG:n02133161 +ILSVRC2012_val_00040348.JPEG:n02165105 +ILSVRC2012_val_00040349.JPEG:n02817516 +ILSVRC2012_val_00040350.JPEG:n04532670 +ILSVRC2012_val_00040351.JPEG:n02013706 +ILSVRC2012_val_00040352.JPEG:n01682714 +ILSVRC2012_val_00040353.JPEG:n02102177 +ILSVRC2012_val_00040354.JPEG:n03290653 +ILSVRC2012_val_00040355.JPEG:n04086273 +ILSVRC2012_val_00040356.JPEG:n02090379 +ILSVRC2012_val_00040357.JPEG:n01797886 +ILSVRC2012_val_00040358.JPEG:n01440764 +ILSVRC2012_val_00040359.JPEG:n01818515 +ILSVRC2012_val_00040360.JPEG:n04562935 +ILSVRC2012_val_00040361.JPEG:n02782093 +ILSVRC2012_val_00040362.JPEG:n03793489 +ILSVRC2012_val_00040363.JPEG:n11879895 +ILSVRC2012_val_00040364.JPEG:n02814860 +ILSVRC2012_val_00040365.JPEG:n02669723 +ILSVRC2012_val_00040366.JPEG:n02974003 +ILSVRC2012_val_00040367.JPEG:n07693725 +ILSVRC2012_val_00040368.JPEG:n02104029 +ILSVRC2012_val_00040369.JPEG:n03372029 +ILSVRC2012_val_00040370.JPEG:n03045698 +ILSVRC2012_val_00040371.JPEG:n03100240 +ILSVRC2012_val_00040372.JPEG:n02127052 +ILSVRC2012_val_00040373.JPEG:n07579787 +ILSVRC2012_val_00040374.JPEG:n03874599 +ILSVRC2012_val_00040375.JPEG:n02504458 +ILSVRC2012_val_00040376.JPEG:n02132136 +ILSVRC2012_val_00040377.JPEG:n03692522 +ILSVRC2012_val_00040378.JPEG:n04517823 +ILSVRC2012_val_00040379.JPEG:n03223299 +ILSVRC2012_val_00040380.JPEG:n04418357 +ILSVRC2012_val_00040381.JPEG:n02110806 +ILSVRC2012_val_00040382.JPEG:n01728572 +ILSVRC2012_val_00040383.JPEG:n04259630 +ILSVRC2012_val_00040384.JPEG:n03930313 +ILSVRC2012_val_00040385.JPEG:n02321529 +ILSVRC2012_val_00040386.JPEG:n02105251 +ILSVRC2012_val_00040387.JPEG:n04317175 +ILSVRC2012_val_00040388.JPEG:n01491361 +ILSVRC2012_val_00040389.JPEG:n07753275 +ILSVRC2012_val_00040390.JPEG:n02028035 +ILSVRC2012_val_00040391.JPEG:n04476259 +ILSVRC2012_val_00040392.JPEG:n03742115 +ILSVRC2012_val_00040393.JPEG:n03032252 +ILSVRC2012_val_00040394.JPEG:n02328150 +ILSVRC2012_val_00040395.JPEG:n04591713 +ILSVRC2012_val_00040396.JPEG:n02088094 +ILSVRC2012_val_00040397.JPEG:n02190166 +ILSVRC2012_val_00040398.JPEG:n04067472 +ILSVRC2012_val_00040399.JPEG:n03134739 +ILSVRC2012_val_00040400.JPEG:n02102318 +ILSVRC2012_val_00040401.JPEG:n03026506 +ILSVRC2012_val_00040402.JPEG:n04371430 +ILSVRC2012_val_00040403.JPEG:n03535780 +ILSVRC2012_val_00040404.JPEG:n01614925 +ILSVRC2012_val_00040405.JPEG:n02111889 +ILSVRC2012_val_00040406.JPEG:n03977966 +ILSVRC2012_val_00040407.JPEG:n03131574 +ILSVRC2012_val_00040408.JPEG:n02071294 +ILSVRC2012_val_00040409.JPEG:n02110627 +ILSVRC2012_val_00040410.JPEG:n02109961 +ILSVRC2012_val_00040411.JPEG:n02412080 +ILSVRC2012_val_00040412.JPEG:n01580077 +ILSVRC2012_val_00040413.JPEG:n06359193 +ILSVRC2012_val_00040414.JPEG:n04209133 +ILSVRC2012_val_00040415.JPEG:n03775546 +ILSVRC2012_val_00040416.JPEG:n03630383 +ILSVRC2012_val_00040417.JPEG:n01753488 +ILSVRC2012_val_00040418.JPEG:n02672831 +ILSVRC2012_val_00040419.JPEG:n02092339 +ILSVRC2012_val_00040420.JPEG:n01644900 +ILSVRC2012_val_00040421.JPEG:n07730033 +ILSVRC2012_val_00040422.JPEG:n03124043 +ILSVRC2012_val_00040423.JPEG:n04065272 +ILSVRC2012_val_00040424.JPEG:n03697007 +ILSVRC2012_val_00040425.JPEG:n01616318 +ILSVRC2012_val_00040426.JPEG:n01558993 +ILSVRC2012_val_00040427.JPEG:n02107683 +ILSVRC2012_val_00040428.JPEG:n04044716 +ILSVRC2012_val_00040429.JPEG:n03877472 +ILSVRC2012_val_00040430.JPEG:n02786058 +ILSVRC2012_val_00040431.JPEG:n02087046 +ILSVRC2012_val_00040432.JPEG:n07717410 +ILSVRC2012_val_00040433.JPEG:n04019541 +ILSVRC2012_val_00040434.JPEG:n01622779 +ILSVRC2012_val_00040435.JPEG:n03337140 +ILSVRC2012_val_00040436.JPEG:n02978881 +ILSVRC2012_val_00040437.JPEG:n04131690 +ILSVRC2012_val_00040438.JPEG:n03887697 +ILSVRC2012_val_00040439.JPEG:n01582220 +ILSVRC2012_val_00040440.JPEG:n02536864 +ILSVRC2012_val_00040441.JPEG:n04065272 +ILSVRC2012_val_00040442.JPEG:n02977058 +ILSVRC2012_val_00040443.JPEG:n03825788 +ILSVRC2012_val_00040444.JPEG:n01687978 +ILSVRC2012_val_00040445.JPEG:n01756291 +ILSVRC2012_val_00040446.JPEG:n04486054 +ILSVRC2012_val_00040447.JPEG:n01737021 +ILSVRC2012_val_00040448.JPEG:n01968897 +ILSVRC2012_val_00040449.JPEG:n03047690 +ILSVRC2012_val_00040450.JPEG:n02106166 +ILSVRC2012_val_00040451.JPEG:n02259212 +ILSVRC2012_val_00040452.JPEG:n02326432 +ILSVRC2012_val_00040453.JPEG:n04476259 +ILSVRC2012_val_00040454.JPEG:n02115913 +ILSVRC2012_val_00040455.JPEG:n02006656 +ILSVRC2012_val_00040456.JPEG:n04254120 +ILSVRC2012_val_00040457.JPEG:n02871525 +ILSVRC2012_val_00040458.JPEG:n03220513 +ILSVRC2012_val_00040459.JPEG:n03769881 +ILSVRC2012_val_00040460.JPEG:n03692522 +ILSVRC2012_val_00040461.JPEG:n02730930 +ILSVRC2012_val_00040462.JPEG:n04235860 +ILSVRC2012_val_00040463.JPEG:n02112018 +ILSVRC2012_val_00040464.JPEG:n02107142 +ILSVRC2012_val_00040465.JPEG:n02834397 +ILSVRC2012_val_00040466.JPEG:n04008634 +ILSVRC2012_val_00040467.JPEG:n02100583 +ILSVRC2012_val_00040468.JPEG:n01729977 +ILSVRC2012_val_00040469.JPEG:n07714571 +ILSVRC2012_val_00040470.JPEG:n01629819 +ILSVRC2012_val_00040471.JPEG:n02028035 +ILSVRC2012_val_00040472.JPEG:n03724870 +ILSVRC2012_val_00040473.JPEG:n04355933 +ILSVRC2012_val_00040474.JPEG:n01614925 +ILSVRC2012_val_00040475.JPEG:n07714571 +ILSVRC2012_val_00040476.JPEG:n07584110 +ILSVRC2012_val_00040477.JPEG:n02870880 +ILSVRC2012_val_00040478.JPEG:n13054560 +ILSVRC2012_val_00040479.JPEG:n02727426 +ILSVRC2012_val_00040480.JPEG:n03877472 +ILSVRC2012_val_00040481.JPEG:n04263257 +ILSVRC2012_val_00040482.JPEG:n04127249 +ILSVRC2012_val_00040483.JPEG:n03630383 +ILSVRC2012_val_00040484.JPEG:n01978287 +ILSVRC2012_val_00040485.JPEG:n13044778 +ILSVRC2012_val_00040486.JPEG:n02509815 +ILSVRC2012_val_00040487.JPEG:n04251144 +ILSVRC2012_val_00040488.JPEG:n04141327 +ILSVRC2012_val_00040489.JPEG:n12620546 +ILSVRC2012_val_00040490.JPEG:n03388043 +ILSVRC2012_val_00040491.JPEG:n02951358 +ILSVRC2012_val_00040492.JPEG:n02412080 +ILSVRC2012_val_00040493.JPEG:n03110669 +ILSVRC2012_val_00040494.JPEG:n03937543 +ILSVRC2012_val_00040495.JPEG:n04044716 +ILSVRC2012_val_00040496.JPEG:n02101388 +ILSVRC2012_val_00040497.JPEG:n07716358 +ILSVRC2012_val_00040498.JPEG:n04462240 +ILSVRC2012_val_00040499.JPEG:n03933933 +ILSVRC2012_val_00040500.JPEG:n02840245 +ILSVRC2012_val_00040501.JPEG:n03485407 +ILSVRC2012_val_00040502.JPEG:n03461385 +ILSVRC2012_val_00040503.JPEG:n02119789 +ILSVRC2012_val_00040504.JPEG:n01944390 +ILSVRC2012_val_00040505.JPEG:n01924916 +ILSVRC2012_val_00040506.JPEG:n04127249 +ILSVRC2012_val_00040507.JPEG:n04209239 +ILSVRC2012_val_00040508.JPEG:n03908618 +ILSVRC2012_val_00040509.JPEG:n03133878 +ILSVRC2012_val_00040510.JPEG:n03992509 +ILSVRC2012_val_00040511.JPEG:n02410509 +ILSVRC2012_val_00040512.JPEG:n03796401 +ILSVRC2012_val_00040513.JPEG:n01798484 +ILSVRC2012_val_00040514.JPEG:n04557648 +ILSVRC2012_val_00040515.JPEG:n02088632 +ILSVRC2012_val_00040516.JPEG:n03000247 +ILSVRC2012_val_00040517.JPEG:n02971356 +ILSVRC2012_val_00040518.JPEG:n03840681 +ILSVRC2012_val_00040519.JPEG:n01776313 +ILSVRC2012_val_00040520.JPEG:n01773157 +ILSVRC2012_val_00040521.JPEG:n04366367 +ILSVRC2012_val_00040522.JPEG:n03325584 +ILSVRC2012_val_00040523.JPEG:n03873416 +ILSVRC2012_val_00040524.JPEG:n01807496 +ILSVRC2012_val_00040525.JPEG:n02790996 +ILSVRC2012_val_00040526.JPEG:n09421951 +ILSVRC2012_val_00040527.JPEG:n07734744 +ILSVRC2012_val_00040528.JPEG:n03000247 +ILSVRC2012_val_00040529.JPEG:n04597913 +ILSVRC2012_val_00040530.JPEG:n04332243 +ILSVRC2012_val_00040531.JPEG:n02408429 +ILSVRC2012_val_00040532.JPEG:n01677366 +ILSVRC2012_val_00040533.JPEG:n02229544 +ILSVRC2012_val_00040534.JPEG:n03891251 +ILSVRC2012_val_00040535.JPEG:n02110063 +ILSVRC2012_val_00040536.JPEG:n03532672 +ILSVRC2012_val_00040537.JPEG:n03937543 +ILSVRC2012_val_00040538.JPEG:n01558993 +ILSVRC2012_val_00040539.JPEG:n04540053 +ILSVRC2012_val_00040540.JPEG:n12057211 +ILSVRC2012_val_00040541.JPEG:n03388183 +ILSVRC2012_val_00040542.JPEG:n02841315 +ILSVRC2012_val_00040543.JPEG:n09399592 +ILSVRC2012_val_00040544.JPEG:n03933933 +ILSVRC2012_val_00040545.JPEG:n02823428 +ILSVRC2012_val_00040546.JPEG:n02102040 +ILSVRC2012_val_00040547.JPEG:n02690373 +ILSVRC2012_val_00040548.JPEG:n02895154 +ILSVRC2012_val_00040549.JPEG:n02085936 +ILSVRC2012_val_00040550.JPEG:n04458633 +ILSVRC2012_val_00040551.JPEG:n02415577 +ILSVRC2012_val_00040552.JPEG:n04579432 +ILSVRC2012_val_00040553.JPEG:n04557648 +ILSVRC2012_val_00040554.JPEG:n03630383 +ILSVRC2012_val_00040555.JPEG:n02009912 +ILSVRC2012_val_00040556.JPEG:n02113978 +ILSVRC2012_val_00040557.JPEG:n03000247 +ILSVRC2012_val_00040558.JPEG:n09246464 +ILSVRC2012_val_00040559.JPEG:n03498962 +ILSVRC2012_val_00040560.JPEG:n02992211 +ILSVRC2012_val_00040561.JPEG:n03249569 +ILSVRC2012_val_00040562.JPEG:n03930313 +ILSVRC2012_val_00040563.JPEG:n01632458 +ILSVRC2012_val_00040564.JPEG:n02086910 +ILSVRC2012_val_00040565.JPEG:n02097209 +ILSVRC2012_val_00040566.JPEG:n03032252 +ILSVRC2012_val_00040567.JPEG:n01496331 +ILSVRC2012_val_00040568.JPEG:n04118538 +ILSVRC2012_val_00040569.JPEG:n03272010 +ILSVRC2012_val_00040570.JPEG:n02095314 +ILSVRC2012_val_00040571.JPEG:n02930766 +ILSVRC2012_val_00040572.JPEG:n02112137 +ILSVRC2012_val_00040573.JPEG:n03697007 +ILSVRC2012_val_00040574.JPEG:n04127249 +ILSVRC2012_val_00040575.JPEG:n04141076 +ILSVRC2012_val_00040576.JPEG:n03376595 +ILSVRC2012_val_00040577.JPEG:n07613480 +ILSVRC2012_val_00040578.JPEG:n04023962 +ILSVRC2012_val_00040579.JPEG:n03958227 +ILSVRC2012_val_00040580.JPEG:n04515003 +ILSVRC2012_val_00040581.JPEG:n04596742 +ILSVRC2012_val_00040582.JPEG:n02108000 +ILSVRC2012_val_00040583.JPEG:n03874599 +ILSVRC2012_val_00040584.JPEG:n01776313 +ILSVRC2012_val_00040585.JPEG:n02088238 +ILSVRC2012_val_00040586.JPEG:n01950731 +ILSVRC2012_val_00040587.JPEG:n02086910 +ILSVRC2012_val_00040588.JPEG:n03384352 +ILSVRC2012_val_00040589.JPEG:n02093859 +ILSVRC2012_val_00040590.JPEG:n02088632 +ILSVRC2012_val_00040591.JPEG:n02749479 +ILSVRC2012_val_00040592.JPEG:n01631663 +ILSVRC2012_val_00040593.JPEG:n01955084 +ILSVRC2012_val_00040594.JPEG:n04275548 +ILSVRC2012_val_00040595.JPEG:n02493793 +ILSVRC2012_val_00040596.JPEG:n03690938 +ILSVRC2012_val_00040597.JPEG:n02802426 +ILSVRC2012_val_00040598.JPEG:n02110341 +ILSVRC2012_val_00040599.JPEG:n02906734 +ILSVRC2012_val_00040600.JPEG:n02124075 +ILSVRC2012_val_00040601.JPEG:n03991062 +ILSVRC2012_val_00040602.JPEG:n03584254 +ILSVRC2012_val_00040603.JPEG:n03444034 +ILSVRC2012_val_00040604.JPEG:n02979186 +ILSVRC2012_val_00040605.JPEG:n03888605 +ILSVRC2012_val_00040606.JPEG:n01534433 +ILSVRC2012_val_00040607.JPEG:n02129165 +ILSVRC2012_val_00040608.JPEG:n01614925 +ILSVRC2012_val_00040609.JPEG:n02397096 +ILSVRC2012_val_00040610.JPEG:n12985857 +ILSVRC2012_val_00040611.JPEG:n02123159 +ILSVRC2012_val_00040612.JPEG:n01984695 +ILSVRC2012_val_00040613.JPEG:n02097047 +ILSVRC2012_val_00040614.JPEG:n01616318 +ILSVRC2012_val_00040615.JPEG:n02117135 +ILSVRC2012_val_00040616.JPEG:n01682714 +ILSVRC2012_val_00040617.JPEG:n03814906 +ILSVRC2012_val_00040618.JPEG:n02105251 +ILSVRC2012_val_00040619.JPEG:n01877812 +ILSVRC2012_val_00040620.JPEG:n04367480 +ILSVRC2012_val_00040621.JPEG:n01770081 +ILSVRC2012_val_00040622.JPEG:n02099849 +ILSVRC2012_val_00040623.JPEG:n02328150 +ILSVRC2012_val_00040624.JPEG:n07590611 +ILSVRC2012_val_00040625.JPEG:n07734744 +ILSVRC2012_val_00040626.JPEG:n03673027 +ILSVRC2012_val_00040627.JPEG:n02129165 +ILSVRC2012_val_00040628.JPEG:n02111500 +ILSVRC2012_val_00040629.JPEG:n04090263 +ILSVRC2012_val_00040630.JPEG:n02129604 +ILSVRC2012_val_00040631.JPEG:n02894605 +ILSVRC2012_val_00040632.JPEG:n02128757 +ILSVRC2012_val_00040633.JPEG:n04238763 +ILSVRC2012_val_00040634.JPEG:n03720891 +ILSVRC2012_val_00040635.JPEG:n03793489 +ILSVRC2012_val_00040636.JPEG:n03424325 +ILSVRC2012_val_00040637.JPEG:n07716358 +ILSVRC2012_val_00040638.JPEG:n02493509 +ILSVRC2012_val_00040639.JPEG:n02099849 +ILSVRC2012_val_00040640.JPEG:n02091244 +ILSVRC2012_val_00040641.JPEG:n02097658 +ILSVRC2012_val_00040642.JPEG:n02138441 +ILSVRC2012_val_00040643.JPEG:n03047690 +ILSVRC2012_val_00040644.JPEG:n02093647 +ILSVRC2012_val_00040645.JPEG:n02108915 +ILSVRC2012_val_00040646.JPEG:n04263257 +ILSVRC2012_val_00040647.JPEG:n02129165 +ILSVRC2012_val_00040648.JPEG:n04335435 +ILSVRC2012_val_00040649.JPEG:n07760859 +ILSVRC2012_val_00040650.JPEG:n02091831 +ILSVRC2012_val_00040651.JPEG:n03445924 +ILSVRC2012_val_00040652.JPEG:n02280649 +ILSVRC2012_val_00040653.JPEG:n02640242 +ILSVRC2012_val_00040654.JPEG:n04613696 +ILSVRC2012_val_00040655.JPEG:n03527444 +ILSVRC2012_val_00040656.JPEG:n01798484 +ILSVRC2012_val_00040657.JPEG:n03995372 +ILSVRC2012_val_00040658.JPEG:n01728572 +ILSVRC2012_val_00040659.JPEG:n04004767 +ILSVRC2012_val_00040660.JPEG:n02099267 +ILSVRC2012_val_00040661.JPEG:n07920052 +ILSVRC2012_val_00040662.JPEG:n03709823 +ILSVRC2012_val_00040663.JPEG:n02095570 +ILSVRC2012_val_00040664.JPEG:n02018795 +ILSVRC2012_val_00040665.JPEG:n03642806 +ILSVRC2012_val_00040666.JPEG:n04074963 +ILSVRC2012_val_00040667.JPEG:n04141327 +ILSVRC2012_val_00040668.JPEG:n01917289 +ILSVRC2012_val_00040669.JPEG:n04131690 +ILSVRC2012_val_00040670.JPEG:n03250847 +ILSVRC2012_val_00040671.JPEG:n02104365 +ILSVRC2012_val_00040672.JPEG:n03602883 +ILSVRC2012_val_00040673.JPEG:n02093428 +ILSVRC2012_val_00040674.JPEG:n03109150 +ILSVRC2012_val_00040675.JPEG:n03240683 +ILSVRC2012_val_00040676.JPEG:n02086079 +ILSVRC2012_val_00040677.JPEG:n02114712 +ILSVRC2012_val_00040678.JPEG:n02093256 +ILSVRC2012_val_00040679.JPEG:n02102040 +ILSVRC2012_val_00040680.JPEG:n03495258 +ILSVRC2012_val_00040681.JPEG:n04584207 +ILSVRC2012_val_00040682.JPEG:n02870880 +ILSVRC2012_val_00040683.JPEG:n02916936 +ILSVRC2012_val_00040684.JPEG:n07875152 +ILSVRC2012_val_00040685.JPEG:n07583066 +ILSVRC2012_val_00040686.JPEG:n02730930 +ILSVRC2012_val_00040687.JPEG:n04019541 +ILSVRC2012_val_00040688.JPEG:n04254120 +ILSVRC2012_val_00040689.JPEG:n02666196 +ILSVRC2012_val_00040690.JPEG:n03141823 +ILSVRC2012_val_00040691.JPEG:n03063689 +ILSVRC2012_val_00040692.JPEG:n06596364 +ILSVRC2012_val_00040693.JPEG:n02906734 +ILSVRC2012_val_00040694.JPEG:n03445777 +ILSVRC2012_val_00040695.JPEG:n02971356 +ILSVRC2012_val_00040696.JPEG:n03891332 +ILSVRC2012_val_00040697.JPEG:n07892512 +ILSVRC2012_val_00040698.JPEG:n02442845 +ILSVRC2012_val_00040699.JPEG:n03527444 +ILSVRC2012_val_00040700.JPEG:n02667093 +ILSVRC2012_val_00040701.JPEG:n01806143 +ILSVRC2012_val_00040702.JPEG:n03902125 +ILSVRC2012_val_00040703.JPEG:n02457408 +ILSVRC2012_val_00040704.JPEG:n01693334 +ILSVRC2012_val_00040705.JPEG:n02799071 +ILSVRC2012_val_00040706.JPEG:n02814533 +ILSVRC2012_val_00040707.JPEG:n06874185 +ILSVRC2012_val_00040708.JPEG:n02088466 +ILSVRC2012_val_00040709.JPEG:n03825788 +ILSVRC2012_val_00040710.JPEG:n01484850 +ILSVRC2012_val_00040711.JPEG:n03355925 +ILSVRC2012_val_00040712.JPEG:n02095889 +ILSVRC2012_val_00040713.JPEG:n02086646 +ILSVRC2012_val_00040714.JPEG:n03942813 +ILSVRC2012_val_00040715.JPEG:n03425413 +ILSVRC2012_val_00040716.JPEG:n04550184 +ILSVRC2012_val_00040717.JPEG:n02817516 +ILSVRC2012_val_00040718.JPEG:n04049303 +ILSVRC2012_val_00040719.JPEG:n04483307 +ILSVRC2012_val_00040720.JPEG:n02097209 +ILSVRC2012_val_00040721.JPEG:n03388549 +ILSVRC2012_val_00040722.JPEG:n02815834 +ILSVRC2012_val_00040723.JPEG:n02487347 +ILSVRC2012_val_00040724.JPEG:n02074367 +ILSVRC2012_val_00040725.JPEG:n02113186 +ILSVRC2012_val_00040726.JPEG:n02536864 +ILSVRC2012_val_00040727.JPEG:n02114855 +ILSVRC2012_val_00040728.JPEG:n07697313 +ILSVRC2012_val_00040729.JPEG:n03938244 +ILSVRC2012_val_00040730.JPEG:n02492035 +ILSVRC2012_val_00040731.JPEG:n02085620 +ILSVRC2012_val_00040732.JPEG:n02085620 +ILSVRC2012_val_00040733.JPEG:n03223299 +ILSVRC2012_val_00040734.JPEG:n04273569 +ILSVRC2012_val_00040735.JPEG:n03496892 +ILSVRC2012_val_00040736.JPEG:n03866082 +ILSVRC2012_val_00040737.JPEG:n03065424 +ILSVRC2012_val_00040738.JPEG:n03877845 +ILSVRC2012_val_00040739.JPEG:n02871525 +ILSVRC2012_val_00040740.JPEG:n03404251 +ILSVRC2012_val_00040741.JPEG:n04462240 +ILSVRC2012_val_00040742.JPEG:n02113799 +ILSVRC2012_val_00040743.JPEG:n02093859 +ILSVRC2012_val_00040744.JPEG:n03742115 +ILSVRC2012_val_00040745.JPEG:n02123045 +ILSVRC2012_val_00040746.JPEG:n04487081 +ILSVRC2012_val_00040747.JPEG:n02107312 +ILSVRC2012_val_00040748.JPEG:n03938244 +ILSVRC2012_val_00040749.JPEG:n02966687 +ILSVRC2012_val_00040750.JPEG:n02342885 +ILSVRC2012_val_00040751.JPEG:n03781244 +ILSVRC2012_val_00040752.JPEG:n02493509 +ILSVRC2012_val_00040753.JPEG:n02134084 +ILSVRC2012_val_00040754.JPEG:n02749479 +ILSVRC2012_val_00040755.JPEG:n07749582 +ILSVRC2012_val_00040756.JPEG:n12144580 +ILSVRC2012_val_00040757.JPEG:n02114548 +ILSVRC2012_val_00040758.JPEG:n13052670 +ILSVRC2012_val_00040759.JPEG:n07753113 +ILSVRC2012_val_00040760.JPEG:n03777754 +ILSVRC2012_val_00040761.JPEG:n07615774 +ILSVRC2012_val_00040762.JPEG:n02483708 +ILSVRC2012_val_00040763.JPEG:n01784675 +ILSVRC2012_val_00040764.JPEG:n01978287 +ILSVRC2012_val_00040765.JPEG:n02536864 +ILSVRC2012_val_00040766.JPEG:n02443484 +ILSVRC2012_val_00040767.JPEG:n03877472 +ILSVRC2012_val_00040768.JPEG:n04074963 +ILSVRC2012_val_00040769.JPEG:n01632777 +ILSVRC2012_val_00040770.JPEG:n02815834 +ILSVRC2012_val_00040771.JPEG:n01669191 +ILSVRC2012_val_00040772.JPEG:n02104029 +ILSVRC2012_val_00040773.JPEG:n02093859 +ILSVRC2012_val_00040774.JPEG:n01883070 +ILSVRC2012_val_00040775.JPEG:n01774750 +ILSVRC2012_val_00040776.JPEG:n01667778 +ILSVRC2012_val_00040777.JPEG:n01728920 +ILSVRC2012_val_00040778.JPEG:n02219486 +ILSVRC2012_val_00040779.JPEG:n03124170 +ILSVRC2012_val_00040780.JPEG:n02123394 +ILSVRC2012_val_00040781.JPEG:n01740131 +ILSVRC2012_val_00040782.JPEG:n04228054 +ILSVRC2012_val_00040783.JPEG:n01592084 +ILSVRC2012_val_00040784.JPEG:n02128925 +ILSVRC2012_val_00040785.JPEG:n02281787 +ILSVRC2012_val_00040786.JPEG:n02093647 +ILSVRC2012_val_00040787.JPEG:n01667778 +ILSVRC2012_val_00040788.JPEG:n02128925 +ILSVRC2012_val_00040789.JPEG:n01978287 +ILSVRC2012_val_00040790.JPEG:n02130308 +ILSVRC2012_val_00040791.JPEG:n03065424 +ILSVRC2012_val_00040792.JPEG:n12620546 +ILSVRC2012_val_00040793.JPEG:n13052670 +ILSVRC2012_val_00040794.JPEG:n02480855 +ILSVRC2012_val_00040795.JPEG:n03376595 +ILSVRC2012_val_00040796.JPEG:n07734744 +ILSVRC2012_val_00040797.JPEG:n04019541 +ILSVRC2012_val_00040798.JPEG:n02536864 +ILSVRC2012_val_00040799.JPEG:n04350905 +ILSVRC2012_val_00040800.JPEG:n01773549 +ILSVRC2012_val_00040801.JPEG:n03782006 +ILSVRC2012_val_00040802.JPEG:n02111129 +ILSVRC2012_val_00040803.JPEG:n01806567 +ILSVRC2012_val_00040804.JPEG:n07753275 +ILSVRC2012_val_00040805.JPEG:n02256656 +ILSVRC2012_val_00040806.JPEG:n01984695 +ILSVRC2012_val_00040807.JPEG:n04443257 +ILSVRC2012_val_00040808.JPEG:n02410509 +ILSVRC2012_val_00040809.JPEG:n02092339 +ILSVRC2012_val_00040810.JPEG:n02115913 +ILSVRC2012_val_00040811.JPEG:n01806143 +ILSVRC2012_val_00040812.JPEG:n02815834 +ILSVRC2012_val_00040813.JPEG:n03908618 +ILSVRC2012_val_00040814.JPEG:n02279972 +ILSVRC2012_val_00040815.JPEG:n03691459 +ILSVRC2012_val_00040816.JPEG:n03216828 +ILSVRC2012_val_00040817.JPEG:n04370456 +ILSVRC2012_val_00040818.JPEG:n02676566 +ILSVRC2012_val_00040819.JPEG:n03710721 +ILSVRC2012_val_00040820.JPEG:n01629819 +ILSVRC2012_val_00040821.JPEG:n03967562 +ILSVRC2012_val_00040822.JPEG:n03482405 +ILSVRC2012_val_00040823.JPEG:n04487081 +ILSVRC2012_val_00040824.JPEG:n01744401 +ILSVRC2012_val_00040825.JPEG:n02454379 +ILSVRC2012_val_00040826.JPEG:n02007558 +ILSVRC2012_val_00040827.JPEG:n03201208 +ILSVRC2012_val_00040828.JPEG:n03793489 +ILSVRC2012_val_00040829.JPEG:n03902125 +ILSVRC2012_val_00040830.JPEG:n02672831 +ILSVRC2012_val_00040831.JPEG:n03447447 +ILSVRC2012_val_00040832.JPEG:n02749479 +ILSVRC2012_val_00040833.JPEG:n01440764 +ILSVRC2012_val_00040834.JPEG:n03538406 +ILSVRC2012_val_00040835.JPEG:n03794056 +ILSVRC2012_val_00040836.JPEG:n02097130 +ILSVRC2012_val_00040837.JPEG:n04332243 +ILSVRC2012_val_00040838.JPEG:n02814860 +ILSVRC2012_val_00040839.JPEG:n02488291 +ILSVRC2012_val_00040840.JPEG:n03032252 +ILSVRC2012_val_00040841.JPEG:n02137549 +ILSVRC2012_val_00040842.JPEG:n02281406 +ILSVRC2012_val_00040843.JPEG:n01494475 +ILSVRC2012_val_00040844.JPEG:n02749479 +ILSVRC2012_val_00040845.JPEG:n04458633 +ILSVRC2012_val_00040846.JPEG:n01847000 +ILSVRC2012_val_00040847.JPEG:n03825788 +ILSVRC2012_val_00040848.JPEG:n01819313 +ILSVRC2012_val_00040849.JPEG:n01847000 +ILSVRC2012_val_00040850.JPEG:n03908618 +ILSVRC2012_val_00040851.JPEG:n03444034 +ILSVRC2012_val_00040852.JPEG:n02483362 +ILSVRC2012_val_00040853.JPEG:n04254680 +ILSVRC2012_val_00040854.JPEG:n02123597 +ILSVRC2012_val_00040855.JPEG:n03838899 +ILSVRC2012_val_00040856.JPEG:n02104029 +ILSVRC2012_val_00040857.JPEG:n03633091 +ILSVRC2012_val_00040858.JPEG:n03775546 +ILSVRC2012_val_00040859.JPEG:n01807496 +ILSVRC2012_val_00040860.JPEG:n03692522 +ILSVRC2012_val_00040861.JPEG:n03721384 +ILSVRC2012_val_00040862.JPEG:n04208210 +ILSVRC2012_val_00040863.JPEG:n02892767 +ILSVRC2012_val_00040864.JPEG:n02086240 +ILSVRC2012_val_00040865.JPEG:n02492660 +ILSVRC2012_val_00040866.JPEG:n04049303 +ILSVRC2012_val_00040867.JPEG:n04238763 +ILSVRC2012_val_00040868.JPEG:n03793489 +ILSVRC2012_val_00040869.JPEG:n02107574 +ILSVRC2012_val_00040870.JPEG:n02364673 +ILSVRC2012_val_00040871.JPEG:n02134084 +ILSVRC2012_val_00040872.JPEG:n02092339 +ILSVRC2012_val_00040873.JPEG:n02906734 +ILSVRC2012_val_00040874.JPEG:n04371774 +ILSVRC2012_val_00040875.JPEG:n02097658 +ILSVRC2012_val_00040876.JPEG:n02102040 +ILSVRC2012_val_00040877.JPEG:n01968897 +ILSVRC2012_val_00040878.JPEG:n02090622 +ILSVRC2012_val_00040879.JPEG:n03916031 +ILSVRC2012_val_00040880.JPEG:n03658185 +ILSVRC2012_val_00040881.JPEG:n02536864 +ILSVRC2012_val_00040882.JPEG:n03697007 +ILSVRC2012_val_00040883.JPEG:n03924679 +ILSVRC2012_val_00040884.JPEG:n02325366 +ILSVRC2012_val_00040885.JPEG:n03337140 +ILSVRC2012_val_00040886.JPEG:n02999410 +ILSVRC2012_val_00040887.JPEG:n01983481 +ILSVRC2012_val_00040888.JPEG:n03141823 +ILSVRC2012_val_00040889.JPEG:n03662601 +ILSVRC2012_val_00040890.JPEG:n01729322 +ILSVRC2012_val_00040891.JPEG:n02676566 +ILSVRC2012_val_00040892.JPEG:n02992211 +ILSVRC2012_val_00040893.JPEG:n03089624 +ILSVRC2012_val_00040894.JPEG:n01632777 +ILSVRC2012_val_00040895.JPEG:n02443484 +ILSVRC2012_val_00040896.JPEG:n03534580 +ILSVRC2012_val_00040897.JPEG:n01847000 +ILSVRC2012_val_00040898.JPEG:n02102318 +ILSVRC2012_val_00040899.JPEG:n01855032 +ILSVRC2012_val_00040900.JPEG:n03961711 +ILSVRC2012_val_00040901.JPEG:n03895866 +ILSVRC2012_val_00040902.JPEG:n02892767 +ILSVRC2012_val_00040903.JPEG:n01601694 +ILSVRC2012_val_00040904.JPEG:n02443484 +ILSVRC2012_val_00040905.JPEG:n03930313 +ILSVRC2012_val_00040906.JPEG:n03062245 +ILSVRC2012_val_00040907.JPEG:n02988304 +ILSVRC2012_val_00040908.JPEG:n02090622 +ILSVRC2012_val_00040909.JPEG:n02107908 +ILSVRC2012_val_00040910.JPEG:n03290653 +ILSVRC2012_val_00040911.JPEG:n04542943 +ILSVRC2012_val_00040912.JPEG:n04296562 +ILSVRC2012_val_00040913.JPEG:n01986214 +ILSVRC2012_val_00040914.JPEG:n02233338 +ILSVRC2012_val_00040915.JPEG:n02093991 +ILSVRC2012_val_00040916.JPEG:n03482405 +ILSVRC2012_val_00040917.JPEG:n02966193 +ILSVRC2012_val_00040918.JPEG:n03786901 +ILSVRC2012_val_00040919.JPEG:n02027492 +ILSVRC2012_val_00040920.JPEG:n04392985 +ILSVRC2012_val_00040921.JPEG:n03376595 +ILSVRC2012_val_00040922.JPEG:n07714990 +ILSVRC2012_val_00040923.JPEG:n02504013 +ILSVRC2012_val_00040924.JPEG:n04606251 +ILSVRC2012_val_00040925.JPEG:n03724870 +ILSVRC2012_val_00040926.JPEG:n02093991 +ILSVRC2012_val_00040927.JPEG:n03933933 +ILSVRC2012_val_00040928.JPEG:n02804414 +ILSVRC2012_val_00040929.JPEG:n03063599 +ILSVRC2012_val_00040930.JPEG:n01698640 +ILSVRC2012_val_00040931.JPEG:n03498962 +ILSVRC2012_val_00040932.JPEG:n04252225 +ILSVRC2012_val_00040933.JPEG:n02013706 +ILSVRC2012_val_00040934.JPEG:n03026506 +ILSVRC2012_val_00040935.JPEG:n03787032 +ILSVRC2012_val_00040936.JPEG:n04536866 +ILSVRC2012_val_00040937.JPEG:n02100583 +ILSVRC2012_val_00040938.JPEG:n01582220 +ILSVRC2012_val_00040939.JPEG:n02500267 +ILSVRC2012_val_00040940.JPEG:n03388183 +ILSVRC2012_val_00040941.JPEG:n07693725 +ILSVRC2012_val_00040942.JPEG:n02033041 +ILSVRC2012_val_00040943.JPEG:n03908714 +ILSVRC2012_val_00040944.JPEG:n02219486 +ILSVRC2012_val_00040945.JPEG:n02730930 +ILSVRC2012_val_00040946.JPEG:n03710193 +ILSVRC2012_val_00040947.JPEG:n02108915 +ILSVRC2012_val_00040948.JPEG:n01749939 +ILSVRC2012_val_00040949.JPEG:n02817516 +ILSVRC2012_val_00040950.JPEG:n01729977 +ILSVRC2012_val_00040951.JPEG:n02086910 +ILSVRC2012_val_00040952.JPEG:n02107908 +ILSVRC2012_val_00040953.JPEG:n03450230 +ILSVRC2012_val_00040954.JPEG:n07565083 +ILSVRC2012_val_00040955.JPEG:n02128385 +ILSVRC2012_val_00040956.JPEG:n03141823 +ILSVRC2012_val_00040957.JPEG:n04259630 +ILSVRC2012_val_00040958.JPEG:n01914609 +ILSVRC2012_val_00040959.JPEG:n07697537 +ILSVRC2012_val_00040960.JPEG:n04447861 +ILSVRC2012_val_00040961.JPEG:n02099849 +ILSVRC2012_val_00040962.JPEG:n03126707 +ILSVRC2012_val_00040963.JPEG:n01943899 +ILSVRC2012_val_00040964.JPEG:n04118776 +ILSVRC2012_val_00040965.JPEG:n02791124 +ILSVRC2012_val_00040966.JPEG:n03763968 +ILSVRC2012_val_00040967.JPEG:n03492542 +ILSVRC2012_val_00040968.JPEG:n02094433 +ILSVRC2012_val_00040969.JPEG:n04366367 +ILSVRC2012_val_00040970.JPEG:n01614925 +ILSVRC2012_val_00040971.JPEG:n02007558 +ILSVRC2012_val_00040972.JPEG:n02128757 +ILSVRC2012_val_00040973.JPEG:n04019541 +ILSVRC2012_val_00040974.JPEG:n04612504 +ILSVRC2012_val_00040975.JPEG:n02841315 +ILSVRC2012_val_00040976.JPEG:n13044778 +ILSVRC2012_val_00040977.JPEG:n04147183 +ILSVRC2012_val_00040978.JPEG:n03933933 +ILSVRC2012_val_00040979.JPEG:n02110627 +ILSVRC2012_val_00040980.JPEG:n02226429 +ILSVRC2012_val_00040981.JPEG:n01631663 +ILSVRC2012_val_00040982.JPEG:n03676483 +ILSVRC2012_val_00040983.JPEG:n02487347 +ILSVRC2012_val_00040984.JPEG:n04507155 +ILSVRC2012_val_00040985.JPEG:n03216828 +ILSVRC2012_val_00040986.JPEG:n07718472 +ILSVRC2012_val_00040987.JPEG:n02058221 +ILSVRC2012_val_00040988.JPEG:n03127747 +ILSVRC2012_val_00040989.JPEG:n07745940 +ILSVRC2012_val_00040990.JPEG:n02102177 +ILSVRC2012_val_00040991.JPEG:n02113712 +ILSVRC2012_val_00040992.JPEG:n02965783 +ILSVRC2012_val_00040993.JPEG:n03840681 +ILSVRC2012_val_00040994.JPEG:n04310018 +ILSVRC2012_val_00040995.JPEG:n01774384 +ILSVRC2012_val_00040996.JPEG:n02177972 +ILSVRC2012_val_00040997.JPEG:n03063599 +ILSVRC2012_val_00040998.JPEG:n01697457 +ILSVRC2012_val_00040999.JPEG:n03759954 +ILSVRC2012_val_00041000.JPEG:n02085620 +ILSVRC2012_val_00041001.JPEG:n07753113 +ILSVRC2012_val_00041002.JPEG:n03393912 +ILSVRC2012_val_00041003.JPEG:n02692877 +ILSVRC2012_val_00041004.JPEG:n03868242 +ILSVRC2012_val_00041005.JPEG:n02403003 +ILSVRC2012_val_00041006.JPEG:n03249569 +ILSVRC2012_val_00041007.JPEG:n03884397 +ILSVRC2012_val_00041008.JPEG:n02396427 +ILSVRC2012_val_00041009.JPEG:n03457902 +ILSVRC2012_val_00041010.JPEG:n07718747 +ILSVRC2012_val_00041011.JPEG:n02167151 +ILSVRC2012_val_00041012.JPEG:n04154565 +ILSVRC2012_val_00041013.JPEG:n04147183 +ILSVRC2012_val_00041014.JPEG:n04118538 +ILSVRC2012_val_00041015.JPEG:n03124043 +ILSVRC2012_val_00041016.JPEG:n04372370 +ILSVRC2012_val_00041017.JPEG:n01667114 +ILSVRC2012_val_00041018.JPEG:n03998194 +ILSVRC2012_val_00041019.JPEG:n03995372 +ILSVRC2012_val_00041020.JPEG:n10565667 +ILSVRC2012_val_00041021.JPEG:n01798484 +ILSVRC2012_val_00041022.JPEG:n04591157 +ILSVRC2012_val_00041023.JPEG:n03127747 +ILSVRC2012_val_00041024.JPEG:n02105641 +ILSVRC2012_val_00041025.JPEG:n03485407 +ILSVRC2012_val_00041026.JPEG:n02102177 +ILSVRC2012_val_00041027.JPEG:n04461696 +ILSVRC2012_val_00041028.JPEG:n01824575 +ILSVRC2012_val_00041029.JPEG:n02066245 +ILSVRC2012_val_00041030.JPEG:n04317175 +ILSVRC2012_val_00041031.JPEG:n02107312 +ILSVRC2012_val_00041032.JPEG:n06874185 +ILSVRC2012_val_00041033.JPEG:n04465501 +ILSVRC2012_val_00041034.JPEG:n02939185 +ILSVRC2012_val_00041035.JPEG:n04019541 +ILSVRC2012_val_00041036.JPEG:n03459775 +ILSVRC2012_val_00041037.JPEG:n04548280 +ILSVRC2012_val_00041038.JPEG:n03047690 +ILSVRC2012_val_00041039.JPEG:n04325704 +ILSVRC2012_val_00041040.JPEG:n07871810 +ILSVRC2012_val_00041041.JPEG:n01819313 +ILSVRC2012_val_00041042.JPEG:n03782006 +ILSVRC2012_val_00041043.JPEG:n02086079 +ILSVRC2012_val_00041044.JPEG:n03584254 +ILSVRC2012_val_00041045.JPEG:n03929660 +ILSVRC2012_val_00041046.JPEG:n02492035 +ILSVRC2012_val_00041047.JPEG:n03670208 +ILSVRC2012_val_00041048.JPEG:n02412080 +ILSVRC2012_val_00041049.JPEG:n02109525 +ILSVRC2012_val_00041050.JPEG:n02397096 +ILSVRC2012_val_00041051.JPEG:n01582220 +ILSVRC2012_val_00041052.JPEG:n03188531 +ILSVRC2012_val_00041053.JPEG:n02105641 +ILSVRC2012_val_00041054.JPEG:n02033041 +ILSVRC2012_val_00041055.JPEG:n03992509 +ILSVRC2012_val_00041056.JPEG:n02328150 +ILSVRC2012_val_00041057.JPEG:n03000684 +ILSVRC2012_val_00041058.JPEG:n03126707 +ILSVRC2012_val_00041059.JPEG:n07590611 +ILSVRC2012_val_00041060.JPEG:n02102480 +ILSVRC2012_val_00041061.JPEG:n07684084 +ILSVRC2012_val_00041062.JPEG:n07590611 +ILSVRC2012_val_00041063.JPEG:n09421951 +ILSVRC2012_val_00041064.JPEG:n04285008 +ILSVRC2012_val_00041065.JPEG:n02930766 +ILSVRC2012_val_00041066.JPEG:n04604644 +ILSVRC2012_val_00041067.JPEG:n03584829 +ILSVRC2012_val_00041068.JPEG:n03447721 +ILSVRC2012_val_00041069.JPEG:n01693334 +ILSVRC2012_val_00041070.JPEG:n02910353 +ILSVRC2012_val_00041071.JPEG:n03532672 +ILSVRC2012_val_00041072.JPEG:n04127249 +ILSVRC2012_val_00041073.JPEG:n04154565 +ILSVRC2012_val_00041074.JPEG:n03014705 +ILSVRC2012_val_00041075.JPEG:n13052670 +ILSVRC2012_val_00041076.JPEG:n03483316 +ILSVRC2012_val_00041077.JPEG:n02817516 +ILSVRC2012_val_00041078.JPEG:n03759954 +ILSVRC2012_val_00041079.JPEG:n03733805 +ILSVRC2012_val_00041080.JPEG:n04204238 +ILSVRC2012_val_00041081.JPEG:n02110341 +ILSVRC2012_val_00041082.JPEG:n04147183 +ILSVRC2012_val_00041083.JPEG:n02007558 +ILSVRC2012_val_00041084.JPEG:n02268443 +ILSVRC2012_val_00041085.JPEG:n03133878 +ILSVRC2012_val_00041086.JPEG:n03255030 +ILSVRC2012_val_00041087.JPEG:n02442845 +ILSVRC2012_val_00041088.JPEG:n02018207 +ILSVRC2012_val_00041089.JPEG:n04069434 +ILSVRC2012_val_00041090.JPEG:n02667093 +ILSVRC2012_val_00041091.JPEG:n03866082 +ILSVRC2012_val_00041092.JPEG:n02113978 +ILSVRC2012_val_00041093.JPEG:n02108000 +ILSVRC2012_val_00041094.JPEG:n03832673 +ILSVRC2012_val_00041095.JPEG:n04039381 +ILSVRC2012_val_00041096.JPEG:n01677366 +ILSVRC2012_val_00041097.JPEG:n01955084 +ILSVRC2012_val_00041098.JPEG:n02113023 +ILSVRC2012_val_00041099.JPEG:n04371430 +ILSVRC2012_val_00041100.JPEG:n03134739 +ILSVRC2012_val_00041101.JPEG:n03840681 +ILSVRC2012_val_00041102.JPEG:n07714571 +ILSVRC2012_val_00041103.JPEG:n01955084 +ILSVRC2012_val_00041104.JPEG:n03785016 +ILSVRC2012_val_00041105.JPEG:n03924679 +ILSVRC2012_val_00041106.JPEG:n04443257 +ILSVRC2012_val_00041107.JPEG:n03709823 +ILSVRC2012_val_00041108.JPEG:n04204347 +ILSVRC2012_val_00041109.JPEG:n02086079 +ILSVRC2012_val_00041110.JPEG:n02361337 +ILSVRC2012_val_00041111.JPEG:n04317175 +ILSVRC2012_val_00041112.JPEG:n09229709 +ILSVRC2012_val_00041113.JPEG:n04270147 +ILSVRC2012_val_00041114.JPEG:n01518878 +ILSVRC2012_val_00041115.JPEG:n02105412 +ILSVRC2012_val_00041116.JPEG:n07720875 +ILSVRC2012_val_00041117.JPEG:n02177972 +ILSVRC2012_val_00041118.JPEG:n02098105 +ILSVRC2012_val_00041119.JPEG:n03534580 +ILSVRC2012_val_00041120.JPEG:n02492660 +ILSVRC2012_val_00041121.JPEG:n03954731 +ILSVRC2012_val_00041122.JPEG:n03874599 +ILSVRC2012_val_00041123.JPEG:n04243546 +ILSVRC2012_val_00041124.JPEG:n04344873 +ILSVRC2012_val_00041125.JPEG:n04252077 +ILSVRC2012_val_00041126.JPEG:n02009229 +ILSVRC2012_val_00041127.JPEG:n01774384 +ILSVRC2012_val_00041128.JPEG:n03843555 +ILSVRC2012_val_00041129.JPEG:n02988304 +ILSVRC2012_val_00041130.JPEG:n02422699 +ILSVRC2012_val_00041131.JPEG:n03045698 +ILSVRC2012_val_00041132.JPEG:n03775071 +ILSVRC2012_val_00041133.JPEG:n02098105 +ILSVRC2012_val_00041134.JPEG:n04099969 +ILSVRC2012_val_00041135.JPEG:n01582220 +ILSVRC2012_val_00041136.JPEG:n03026506 +ILSVRC2012_val_00041137.JPEG:n02099849 +ILSVRC2012_val_00041138.JPEG:n02814860 +ILSVRC2012_val_00041139.JPEG:n02980441 +ILSVRC2012_val_00041140.JPEG:n07875152 +ILSVRC2012_val_00041141.JPEG:n01873310 +ILSVRC2012_val_00041142.JPEG:n02117135 +ILSVRC2012_val_00041143.JPEG:n02510455 +ILSVRC2012_val_00041144.JPEG:n02108422 +ILSVRC2012_val_00041145.JPEG:n04599235 +ILSVRC2012_val_00041146.JPEG:n03450230 +ILSVRC2012_val_00041147.JPEG:n02105505 +ILSVRC2012_val_00041148.JPEG:n04239074 +ILSVRC2012_val_00041149.JPEG:n04131690 +ILSVRC2012_val_00041150.JPEG:n04033995 +ILSVRC2012_val_00041151.JPEG:n03445924 +ILSVRC2012_val_00041152.JPEG:n01558993 +ILSVRC2012_val_00041153.JPEG:n02791270 +ILSVRC2012_val_00041154.JPEG:n03770679 +ILSVRC2012_val_00041155.JPEG:n02480855 +ILSVRC2012_val_00041156.JPEG:n02134084 +ILSVRC2012_val_00041157.JPEG:n02098286 +ILSVRC2012_val_00041158.JPEG:n03478589 +ILSVRC2012_val_00041159.JPEG:n01744401 +ILSVRC2012_val_00041160.JPEG:n04532670 +ILSVRC2012_val_00041161.JPEG:n02105412 +ILSVRC2012_val_00041162.JPEG:n03874599 +ILSVRC2012_val_00041163.JPEG:n04125021 +ILSVRC2012_val_00041164.JPEG:n01682714 +ILSVRC2012_val_00041165.JPEG:n02747177 +ILSVRC2012_val_00041166.JPEG:n02992211 +ILSVRC2012_val_00041167.JPEG:n03710193 +ILSVRC2012_val_00041168.JPEG:n01514859 +ILSVRC2012_val_00041169.JPEG:n01687978 +ILSVRC2012_val_00041170.JPEG:n04418357 +ILSVRC2012_val_00041171.JPEG:n02017213 +ILSVRC2012_val_00041172.JPEG:n01677366 +ILSVRC2012_val_00041173.JPEG:n02281406 +ILSVRC2012_val_00041174.JPEG:n02138441 +ILSVRC2012_val_00041175.JPEG:n03594945 +ILSVRC2012_val_00041176.JPEG:n02106030 +ILSVRC2012_val_00041177.JPEG:n03017168 +ILSVRC2012_val_00041178.JPEG:n02105251 +ILSVRC2012_val_00041179.JPEG:n04273569 +ILSVRC2012_val_00041180.JPEG:n02488291 +ILSVRC2012_val_00041181.JPEG:n09332890 +ILSVRC2012_val_00041182.JPEG:n03873416 +ILSVRC2012_val_00041183.JPEG:n02895154 +ILSVRC2012_val_00041184.JPEG:n02494079 +ILSVRC2012_val_00041185.JPEG:n02437616 +ILSVRC2012_val_00041186.JPEG:n01692333 +ILSVRC2012_val_00041187.JPEG:n04311004 +ILSVRC2012_val_00041188.JPEG:n03218198 +ILSVRC2012_val_00041189.JPEG:n02110185 +ILSVRC2012_val_00041190.JPEG:n02256656 +ILSVRC2012_val_00041191.JPEG:n07880968 +ILSVRC2012_val_00041192.JPEG:n02666196 +ILSVRC2012_val_00041193.JPEG:n03337140 +ILSVRC2012_val_00041194.JPEG:n04399382 +ILSVRC2012_val_00041195.JPEG:n04265275 +ILSVRC2012_val_00041196.JPEG:n04254120 +ILSVRC2012_val_00041197.JPEG:n01798484 +ILSVRC2012_val_00041198.JPEG:n03602883 +ILSVRC2012_val_00041199.JPEG:n03825788 +ILSVRC2012_val_00041200.JPEG:n01833805 +ILSVRC2012_val_00041201.JPEG:n02704792 +ILSVRC2012_val_00041202.JPEG:n01734418 +ILSVRC2012_val_00041203.JPEG:n03594734 +ILSVRC2012_val_00041204.JPEG:n02701002 +ILSVRC2012_val_00041205.JPEG:n02085620 +ILSVRC2012_val_00041206.JPEG:n01582220 +ILSVRC2012_val_00041207.JPEG:n03623198 +ILSVRC2012_val_00041208.JPEG:n03000134 +ILSVRC2012_val_00041209.JPEG:n02992211 +ILSVRC2012_val_00041210.JPEG:n03691459 +ILSVRC2012_val_00041211.JPEG:n02526121 +ILSVRC2012_val_00041212.JPEG:n03998194 +ILSVRC2012_val_00041213.JPEG:n01990800 +ILSVRC2012_val_00041214.JPEG:n03933933 +ILSVRC2012_val_00041215.JPEG:n02950826 +ILSVRC2012_val_00041216.JPEG:n01748264 +ILSVRC2012_val_00041217.JPEG:n15075141 +ILSVRC2012_val_00041218.JPEG:n10565667 +ILSVRC2012_val_00041219.JPEG:n15075141 +ILSVRC2012_val_00041220.JPEG:n02116738 +ILSVRC2012_val_00041221.JPEG:n02643566 +ILSVRC2012_val_00041222.JPEG:n02837789 +ILSVRC2012_val_00041223.JPEG:n04005630 +ILSVRC2012_val_00041224.JPEG:n02091134 +ILSVRC2012_val_00041225.JPEG:n02071294 +ILSVRC2012_val_00041226.JPEG:n10148035 +ILSVRC2012_val_00041227.JPEG:n02951358 +ILSVRC2012_val_00041228.JPEG:n04127249 +ILSVRC2012_val_00041229.JPEG:n03866082 +ILSVRC2012_val_00041230.JPEG:n04579145 +ILSVRC2012_val_00041231.JPEG:n04239074 +ILSVRC2012_val_00041232.JPEG:n02492035 +ILSVRC2012_val_00041233.JPEG:n02107683 +ILSVRC2012_val_00041234.JPEG:n04239074 +ILSVRC2012_val_00041235.JPEG:n04004767 +ILSVRC2012_val_00041236.JPEG:n04550184 +ILSVRC2012_val_00041237.JPEG:n03961711 +ILSVRC2012_val_00041238.JPEG:n03201208 +ILSVRC2012_val_00041239.JPEG:n03207941 +ILSVRC2012_val_00041240.JPEG:n03134739 +ILSVRC2012_val_00041241.JPEG:n02892767 +ILSVRC2012_val_00041242.JPEG:n03394916 +ILSVRC2012_val_00041243.JPEG:n02398521 +ILSVRC2012_val_00041244.JPEG:n03868863 +ILSVRC2012_val_00041245.JPEG:n02486410 +ILSVRC2012_val_00041246.JPEG:n04487394 +ILSVRC2012_val_00041247.JPEG:n03394916 +ILSVRC2012_val_00041248.JPEG:n01496331 +ILSVRC2012_val_00041249.JPEG:n04418357 +ILSVRC2012_val_00041250.JPEG:n02168699 +ILSVRC2012_val_00041251.JPEG:n02097209 +ILSVRC2012_val_00041252.JPEG:n01537544 +ILSVRC2012_val_00041253.JPEG:n01687978 +ILSVRC2012_val_00041254.JPEG:n02799071 +ILSVRC2012_val_00041255.JPEG:n04009552 +ILSVRC2012_val_00041256.JPEG:n03345487 +ILSVRC2012_val_00041257.JPEG:n04346328 +ILSVRC2012_val_00041258.JPEG:n12057211 +ILSVRC2012_val_00041259.JPEG:n03485794 +ILSVRC2012_val_00041260.JPEG:n02443484 +ILSVRC2012_val_00041261.JPEG:n02229544 +ILSVRC2012_val_00041262.JPEG:n02840245 +ILSVRC2012_val_00041263.JPEG:n02415577 +ILSVRC2012_val_00041264.JPEG:n02104029 +ILSVRC2012_val_00041265.JPEG:n03792782 +ILSVRC2012_val_00041266.JPEG:n03888605 +ILSVRC2012_val_00041267.JPEG:n02128925 +ILSVRC2012_val_00041268.JPEG:n03045698 +ILSVRC2012_val_00041269.JPEG:n03837869 +ILSVRC2012_val_00041270.JPEG:n02749479 +ILSVRC2012_val_00041271.JPEG:n04033995 +ILSVRC2012_val_00041272.JPEG:n02422106 +ILSVRC2012_val_00041273.JPEG:n03404251 +ILSVRC2012_val_00041274.JPEG:n04208210 +ILSVRC2012_val_00041275.JPEG:n02113712 +ILSVRC2012_val_00041276.JPEG:n03459775 +ILSVRC2012_val_00041277.JPEG:n02514041 +ILSVRC2012_val_00041278.JPEG:n04371430 +ILSVRC2012_val_00041279.JPEG:n01644373 +ILSVRC2012_val_00041280.JPEG:n03447721 +ILSVRC2012_val_00041281.JPEG:n13052670 +ILSVRC2012_val_00041282.JPEG:n03492542 +ILSVRC2012_val_00041283.JPEG:n04366367 +ILSVRC2012_val_00041284.JPEG:n01968897 +ILSVRC2012_val_00041285.JPEG:n02033041 +ILSVRC2012_val_00041286.JPEG:n02114712 +ILSVRC2012_val_00041287.JPEG:n02804414 +ILSVRC2012_val_00041288.JPEG:n01796340 +ILSVRC2012_val_00041289.JPEG:n04009552 +ILSVRC2012_val_00041290.JPEG:n04597913 +ILSVRC2012_val_00041291.JPEG:n03141823 +ILSVRC2012_val_00041292.JPEG:n04612504 +ILSVRC2012_val_00041293.JPEG:n01729322 +ILSVRC2012_val_00041294.JPEG:n02492660 +ILSVRC2012_val_00041295.JPEG:n03792972 +ILSVRC2012_val_00041296.JPEG:n02130308 +ILSVRC2012_val_00041297.JPEG:n03400231 +ILSVRC2012_val_00041298.JPEG:n01632777 +ILSVRC2012_val_00041299.JPEG:n03085013 +ILSVRC2012_val_00041300.JPEG:n01729322 +ILSVRC2012_val_00041301.JPEG:n02095570 +ILSVRC2012_val_00041302.JPEG:n03970156 +ILSVRC2012_val_00041303.JPEG:n04009552 +ILSVRC2012_val_00041304.JPEG:n03950228 +ILSVRC2012_val_00041305.JPEG:n02086646 +ILSVRC2012_val_00041306.JPEG:n02108000 +ILSVRC2012_val_00041307.JPEG:n03196217 +ILSVRC2012_val_00041308.JPEG:n01580077 +ILSVRC2012_val_00041309.JPEG:n04275548 +ILSVRC2012_val_00041310.JPEG:n04599235 +ILSVRC2012_val_00041311.JPEG:n01774750 +ILSVRC2012_val_00041312.JPEG:n03498962 +ILSVRC2012_val_00041313.JPEG:n03457902 +ILSVRC2012_val_00041314.JPEG:n03930630 +ILSVRC2012_val_00041315.JPEG:n04590129 +ILSVRC2012_val_00041316.JPEG:n01968897 +ILSVRC2012_val_00041317.JPEG:n04462240 +ILSVRC2012_val_00041318.JPEG:n04554684 +ILSVRC2012_val_00041319.JPEG:n02840245 +ILSVRC2012_val_00041320.JPEG:n02804414 +ILSVRC2012_val_00041321.JPEG:n07614500 +ILSVRC2012_val_00041322.JPEG:n03482405 +ILSVRC2012_val_00041323.JPEG:n02871525 +ILSVRC2012_val_00041324.JPEG:n04192698 +ILSVRC2012_val_00041325.JPEG:n02699494 +ILSVRC2012_val_00041326.JPEG:n03388183 +ILSVRC2012_val_00041327.JPEG:n04153751 +ILSVRC2012_val_00041328.JPEG:n03733281 +ILSVRC2012_val_00041329.JPEG:n01797886 +ILSVRC2012_val_00041330.JPEG:n01689811 +ILSVRC2012_val_00041331.JPEG:n02777292 +ILSVRC2012_val_00041332.JPEG:n02389026 +ILSVRC2012_val_00041333.JPEG:n03788365 +ILSVRC2012_val_00041334.JPEG:n01514859 +ILSVRC2012_val_00041335.JPEG:n02102480 +ILSVRC2012_val_00041336.JPEG:n03942813 +ILSVRC2012_val_00041337.JPEG:n02111129 +ILSVRC2012_val_00041338.JPEG:n03017168 +ILSVRC2012_val_00041339.JPEG:n02105855 +ILSVRC2012_val_00041340.JPEG:n04328186 +ILSVRC2012_val_00041341.JPEG:n02115641 +ILSVRC2012_val_00041342.JPEG:n02093647 +ILSVRC2012_val_00041343.JPEG:n02415577 +ILSVRC2012_val_00041344.JPEG:n02536864 +ILSVRC2012_val_00041345.JPEG:n13044778 +ILSVRC2012_val_00041346.JPEG:n02113712 +ILSVRC2012_val_00041347.JPEG:n02123394 +ILSVRC2012_val_00041348.JPEG:n01735189 +ILSVRC2012_val_00041349.JPEG:n03085013 +ILSVRC2012_val_00041350.JPEG:n03127747 +ILSVRC2012_val_00041351.JPEG:n02105641 +ILSVRC2012_val_00041352.JPEG:n04606251 +ILSVRC2012_val_00041353.JPEG:n02814533 +ILSVRC2012_val_00041354.JPEG:n02980441 +ILSVRC2012_val_00041355.JPEG:n02910353 +ILSVRC2012_val_00041356.JPEG:n02098105 +ILSVRC2012_val_00041357.JPEG:n04380533 +ILSVRC2012_val_00041358.JPEG:n02098286 +ILSVRC2012_val_00041359.JPEG:n02018795 +ILSVRC2012_val_00041360.JPEG:n02788148 +ILSVRC2012_val_00041361.JPEG:n01807496 +ILSVRC2012_val_00041362.JPEG:n03908714 +ILSVRC2012_val_00041363.JPEG:n03388549 +ILSVRC2012_val_00041364.JPEG:n02100877 +ILSVRC2012_val_00041365.JPEG:n03982430 +ILSVRC2012_val_00041366.JPEG:n01986214 +ILSVRC2012_val_00041367.JPEG:n04201297 +ILSVRC2012_val_00041368.JPEG:n03347037 +ILSVRC2012_val_00041369.JPEG:n04008634 +ILSVRC2012_val_00041370.JPEG:n04557648 +ILSVRC2012_val_00041371.JPEG:n03445924 +ILSVRC2012_val_00041372.JPEG:n02980441 +ILSVRC2012_val_00041373.JPEG:n03131574 +ILSVRC2012_val_00041374.JPEG:n02948072 +ILSVRC2012_val_00041375.JPEG:n01797886 +ILSVRC2012_val_00041376.JPEG:n04005630 +ILSVRC2012_val_00041377.JPEG:n02111889 +ILSVRC2012_val_00041378.JPEG:n02325366 +ILSVRC2012_val_00041379.JPEG:n01728920 +ILSVRC2012_val_00041380.JPEG:n02129165 +ILSVRC2012_val_00041381.JPEG:n02168699 +ILSVRC2012_val_00041382.JPEG:n04465501 +ILSVRC2012_val_00041383.JPEG:n01728572 +ILSVRC2012_val_00041384.JPEG:n02105641 +ILSVRC2012_val_00041385.JPEG:n01774384 +ILSVRC2012_val_00041386.JPEG:n04418357 +ILSVRC2012_val_00041387.JPEG:n02325366 +ILSVRC2012_val_00041388.JPEG:n03888605 +ILSVRC2012_val_00041389.JPEG:n04149813 +ILSVRC2012_val_00041390.JPEG:n02281406 +ILSVRC2012_val_00041391.JPEG:n03599486 +ILSVRC2012_val_00041392.JPEG:n03124170 +ILSVRC2012_val_00041393.JPEG:n02100583 +ILSVRC2012_val_00041394.JPEG:n03956157 +ILSVRC2012_val_00041395.JPEG:n03788195 +ILSVRC2012_val_00041396.JPEG:n04286575 +ILSVRC2012_val_00041397.JPEG:n04136333 +ILSVRC2012_val_00041398.JPEG:n04344873 +ILSVRC2012_val_00041399.JPEG:n03743016 +ILSVRC2012_val_00041400.JPEG:n01494475 +ILSVRC2012_val_00041401.JPEG:n01910747 +ILSVRC2012_val_00041402.JPEG:n02787622 +ILSVRC2012_val_00041403.JPEG:n04562935 +ILSVRC2012_val_00041404.JPEG:n02909870 +ILSVRC2012_val_00041405.JPEG:n02974003 +ILSVRC2012_val_00041406.JPEG:n02111500 +ILSVRC2012_val_00041407.JPEG:n03388549 +ILSVRC2012_val_00041408.JPEG:n04550184 +ILSVRC2012_val_00041409.JPEG:n07745940 +ILSVRC2012_val_00041410.JPEG:n03673027 +ILSVRC2012_val_00041411.JPEG:n02727426 +ILSVRC2012_val_00041412.JPEG:n03207743 +ILSVRC2012_val_00041413.JPEG:n04487081 +ILSVRC2012_val_00041414.JPEG:n04009552 +ILSVRC2012_val_00041415.JPEG:n02130308 +ILSVRC2012_val_00041416.JPEG:n02105412 +ILSVRC2012_val_00041417.JPEG:n03476991 +ILSVRC2012_val_00041418.JPEG:n01632458 +ILSVRC2012_val_00041419.JPEG:n02790996 +ILSVRC2012_val_00041420.JPEG:n04505470 +ILSVRC2012_val_00041421.JPEG:n04380533 +ILSVRC2012_val_00041422.JPEG:n02108422 +ILSVRC2012_val_00041423.JPEG:n07920052 +ILSVRC2012_val_00041424.JPEG:n03467068 +ILSVRC2012_val_00041425.JPEG:n03249569 +ILSVRC2012_val_00041426.JPEG:n03633091 +ILSVRC2012_val_00041427.JPEG:n02124075 +ILSVRC2012_val_00041428.JPEG:n03763968 +ILSVRC2012_val_00041429.JPEG:n03710637 +ILSVRC2012_val_00041430.JPEG:n03100240 +ILSVRC2012_val_00041431.JPEG:n02256656 +ILSVRC2012_val_00041432.JPEG:n03461385 +ILSVRC2012_val_00041433.JPEG:n02869837 +ILSVRC2012_val_00041434.JPEG:n02948072 +ILSVRC2012_val_00041435.JPEG:n03991062 +ILSVRC2012_val_00041436.JPEG:n02091244 +ILSVRC2012_val_00041437.JPEG:n04476259 +ILSVRC2012_val_00041438.JPEG:n02099429 +ILSVRC2012_val_00041439.JPEG:n02346627 +ILSVRC2012_val_00041440.JPEG:n02782093 +ILSVRC2012_val_00041441.JPEG:n02457408 +ILSVRC2012_val_00041442.JPEG:n02009229 +ILSVRC2012_val_00041443.JPEG:n02910353 +ILSVRC2012_val_00041444.JPEG:n02087046 +ILSVRC2012_val_00041445.JPEG:n01877812 +ILSVRC2012_val_00041446.JPEG:n03787032 +ILSVRC2012_val_00041447.JPEG:n02281406 +ILSVRC2012_val_00041448.JPEG:n04461696 +ILSVRC2012_val_00041449.JPEG:n03782006 +ILSVRC2012_val_00041450.JPEG:n01924916 +ILSVRC2012_val_00041451.JPEG:n03223299 +ILSVRC2012_val_00041452.JPEG:n01768244 +ILSVRC2012_val_00041453.JPEG:n04023962 +ILSVRC2012_val_00041454.JPEG:n07717410 +ILSVRC2012_val_00041455.JPEG:n03062245 +ILSVRC2012_val_00041456.JPEG:n07875152 +ILSVRC2012_val_00041457.JPEG:n03393912 +ILSVRC2012_val_00041458.JPEG:n02364673 +ILSVRC2012_val_00041459.JPEG:n03937543 +ILSVRC2012_val_00041460.JPEG:n02101388 +ILSVRC2012_val_00041461.JPEG:n04548280 +ILSVRC2012_val_00041462.JPEG:n12620546 +ILSVRC2012_val_00041463.JPEG:n03584829 +ILSVRC2012_val_00041464.JPEG:n04606251 +ILSVRC2012_val_00041465.JPEG:n02776631 +ILSVRC2012_val_00041466.JPEG:n04443257 +ILSVRC2012_val_00041467.JPEG:n02788148 +ILSVRC2012_val_00041468.JPEG:n03838899 +ILSVRC2012_val_00041469.JPEG:n02051845 +ILSVRC2012_val_00041470.JPEG:n07768694 +ILSVRC2012_val_00041471.JPEG:n03498962 +ILSVRC2012_val_00041472.JPEG:n02100583 +ILSVRC2012_val_00041473.JPEG:n02102177 +ILSVRC2012_val_00041474.JPEG:n07716358 +ILSVRC2012_val_00041475.JPEG:n04589890 +ILSVRC2012_val_00041476.JPEG:n02128757 +ILSVRC2012_val_00041477.JPEG:n02489166 +ILSVRC2012_val_00041478.JPEG:n03417042 +ILSVRC2012_val_00041479.JPEG:n03355925 +ILSVRC2012_val_00041480.JPEG:n02111889 +ILSVRC2012_val_00041481.JPEG:n03297495 +ILSVRC2012_val_00041482.JPEG:n03180011 +ILSVRC2012_val_00041483.JPEG:n03196217 +ILSVRC2012_val_00041484.JPEG:n02859443 +ILSVRC2012_val_00041485.JPEG:n02321529 +ILSVRC2012_val_00041486.JPEG:n04443257 +ILSVRC2012_val_00041487.JPEG:n03089624 +ILSVRC2012_val_00041488.JPEG:n07730033 +ILSVRC2012_val_00041489.JPEG:n03874293 +ILSVRC2012_val_00041490.JPEG:n03594945 +ILSVRC2012_val_00041491.JPEG:n02423022 +ILSVRC2012_val_00041492.JPEG:n11879895 +ILSVRC2012_val_00041493.JPEG:n02104029 +ILSVRC2012_val_00041494.JPEG:n02916936 +ILSVRC2012_val_00041495.JPEG:n02403003 +ILSVRC2012_val_00041496.JPEG:n03709823 +ILSVRC2012_val_00041497.JPEG:n04467665 +ILSVRC2012_val_00041498.JPEG:n01833805 +ILSVRC2012_val_00041499.JPEG:n02119022 +ILSVRC2012_val_00041500.JPEG:n02687172 +ILSVRC2012_val_00041501.JPEG:n02492660 +ILSVRC2012_val_00041502.JPEG:n02877765 +ILSVRC2012_val_00041503.JPEG:n02099429 +ILSVRC2012_val_00041504.JPEG:n03942813 +ILSVRC2012_val_00041505.JPEG:n02105855 +ILSVRC2012_val_00041506.JPEG:n02168699 +ILSVRC2012_val_00041507.JPEG:n07565083 +ILSVRC2012_val_00041508.JPEG:n03895866 +ILSVRC2012_val_00041509.JPEG:n03126707 +ILSVRC2012_val_00041510.JPEG:n02346627 +ILSVRC2012_val_00041511.JPEG:n02606052 +ILSVRC2012_val_00041512.JPEG:n03670208 +ILSVRC2012_val_00041513.JPEG:n02114548 +ILSVRC2012_val_00041514.JPEG:n02109047 +ILSVRC2012_val_00041515.JPEG:n03916031 +ILSVRC2012_val_00041516.JPEG:n01871265 +ILSVRC2012_val_00041517.JPEG:n04523525 +ILSVRC2012_val_00041518.JPEG:n02690373 +ILSVRC2012_val_00041519.JPEG:n03014705 +ILSVRC2012_val_00041520.JPEG:n02356798 +ILSVRC2012_val_00041521.JPEG:n02128385 +ILSVRC2012_val_00041522.JPEG:n02133161 +ILSVRC2012_val_00041523.JPEG:n03884397 +ILSVRC2012_val_00041524.JPEG:n02108915 +ILSVRC2012_val_00041525.JPEG:n03759954 +ILSVRC2012_val_00041526.JPEG:n03630383 +ILSVRC2012_val_00041527.JPEG:n02106382 +ILSVRC2012_val_00041528.JPEG:n02256656 +ILSVRC2012_val_00041529.JPEG:n02085936 +ILSVRC2012_val_00041530.JPEG:n03197337 +ILSVRC2012_val_00041531.JPEG:n03661043 +ILSVRC2012_val_00041532.JPEG:n04590129 +ILSVRC2012_val_00041533.JPEG:n03958227 +ILSVRC2012_val_00041534.JPEG:n04525038 +ILSVRC2012_val_00041535.JPEG:n02037110 +ILSVRC2012_val_00041536.JPEG:n03956157 +ILSVRC2012_val_00041537.JPEG:n03717622 +ILSVRC2012_val_00041538.JPEG:n02326432 +ILSVRC2012_val_00041539.JPEG:n03249569 +ILSVRC2012_val_00041540.JPEG:n01631663 +ILSVRC2012_val_00041541.JPEG:n01687978 +ILSVRC2012_val_00041542.JPEG:n12144580 +ILSVRC2012_val_00041543.JPEG:n02277742 +ILSVRC2012_val_00041544.JPEG:n03692522 +ILSVRC2012_val_00041545.JPEG:n04507155 +ILSVRC2012_val_00041546.JPEG:n04389033 +ILSVRC2012_val_00041547.JPEG:n04548280 +ILSVRC2012_val_00041548.JPEG:n01914609 +ILSVRC2012_val_00041549.JPEG:n01776313 +ILSVRC2012_val_00041550.JPEG:n03125729 +ILSVRC2012_val_00041551.JPEG:n02096051 +ILSVRC2012_val_00041552.JPEG:n02769748 +ILSVRC2012_val_00041553.JPEG:n04131690 +ILSVRC2012_val_00041554.JPEG:n02669723 +ILSVRC2012_val_00041555.JPEG:n04376876 +ILSVRC2012_val_00041556.JPEG:n01818515 +ILSVRC2012_val_00041557.JPEG:n02091244 +ILSVRC2012_val_00041558.JPEG:n03207743 +ILSVRC2012_val_00041559.JPEG:n03134739 +ILSVRC2012_val_00041560.JPEG:n03838899 +ILSVRC2012_val_00041561.JPEG:n02641379 +ILSVRC2012_val_00041562.JPEG:n02666196 +ILSVRC2012_val_00041563.JPEG:n02397096 +ILSVRC2012_val_00041564.JPEG:n02009229 +ILSVRC2012_val_00041565.JPEG:n02410509 +ILSVRC2012_val_00041566.JPEG:n02276258 +ILSVRC2012_val_00041567.JPEG:n03062245 +ILSVRC2012_val_00041568.JPEG:n02097130 +ILSVRC2012_val_00041569.JPEG:n02093754 +ILSVRC2012_val_00041570.JPEG:n02123045 +ILSVRC2012_val_00041571.JPEG:n04357314 +ILSVRC2012_val_00041572.JPEG:n03089624 +ILSVRC2012_val_00041573.JPEG:n02091244 +ILSVRC2012_val_00041574.JPEG:n01685808 +ILSVRC2012_val_00041575.JPEG:n02412080 +ILSVRC2012_val_00041576.JPEG:n03841143 +ILSVRC2012_val_00041577.JPEG:n01807496 +ILSVRC2012_val_00041578.JPEG:n02098286 +ILSVRC2012_val_00041579.JPEG:n02124075 +ILSVRC2012_val_00041580.JPEG:n02086646 +ILSVRC2012_val_00041581.JPEG:n03627232 +ILSVRC2012_val_00041582.JPEG:n09468604 +ILSVRC2012_val_00041583.JPEG:n01768244 +ILSVRC2012_val_00041584.JPEG:n07920052 +ILSVRC2012_val_00041585.JPEG:n03976467 +ILSVRC2012_val_00041586.JPEG:n03534580 +ILSVRC2012_val_00041587.JPEG:n03617480 +ILSVRC2012_val_00041588.JPEG:n04467665 +ILSVRC2012_val_00041589.JPEG:n07584110 +ILSVRC2012_val_00041590.JPEG:n04040759 +ILSVRC2012_val_00041591.JPEG:n02090379 +ILSVRC2012_val_00041592.JPEG:n03393912 +ILSVRC2012_val_00041593.JPEG:n01945685 +ILSVRC2012_val_00041594.JPEG:n04482393 +ILSVRC2012_val_00041595.JPEG:n01537544 +ILSVRC2012_val_00041596.JPEG:n02231487 +ILSVRC2012_val_00041597.JPEG:n02137549 +ILSVRC2012_val_00041598.JPEG:n03045698 +ILSVRC2012_val_00041599.JPEG:n04346328 +ILSVRC2012_val_00041600.JPEG:n04597913 +ILSVRC2012_val_00041601.JPEG:n02114367 +ILSVRC2012_val_00041602.JPEG:n07613480 +ILSVRC2012_val_00041603.JPEG:n02892767 +ILSVRC2012_val_00041604.JPEG:n04209133 +ILSVRC2012_val_00041605.JPEG:n02097047 +ILSVRC2012_val_00041606.JPEG:n02100877 +ILSVRC2012_val_00041607.JPEG:n02480855 +ILSVRC2012_val_00041608.JPEG:n03259280 +ILSVRC2012_val_00041609.JPEG:n03272010 +ILSVRC2012_val_00041610.JPEG:n07684084 +ILSVRC2012_val_00041611.JPEG:n03743016 +ILSVRC2012_val_00041612.JPEG:n01773549 +ILSVRC2012_val_00041613.JPEG:n02708093 +ILSVRC2012_val_00041614.JPEG:n02939185 +ILSVRC2012_val_00041615.JPEG:n03617480 +ILSVRC2012_val_00041616.JPEG:n01753488 +ILSVRC2012_val_00041617.JPEG:n07880968 +ILSVRC2012_val_00041618.JPEG:n03218198 +ILSVRC2012_val_00041619.JPEG:n02871525 +ILSVRC2012_val_00041620.JPEG:n02093256 +ILSVRC2012_val_00041621.JPEG:n01798484 +ILSVRC2012_val_00041622.JPEG:n02417914 +ILSVRC2012_val_00041623.JPEG:n02108915 +ILSVRC2012_val_00041624.JPEG:n04125021 +ILSVRC2012_val_00041625.JPEG:n03126707 +ILSVRC2012_val_00041626.JPEG:n04285008 +ILSVRC2012_val_00041627.JPEG:n02526121 +ILSVRC2012_val_00041628.JPEG:n04111531 +ILSVRC2012_val_00041629.JPEG:n02089078 +ILSVRC2012_val_00041630.JPEG:n02927161 +ILSVRC2012_val_00041631.JPEG:n02971356 +ILSVRC2012_val_00041632.JPEG:n04553703 +ILSVRC2012_val_00041633.JPEG:n02442845 +ILSVRC2012_val_00041634.JPEG:n01945685 +ILSVRC2012_val_00041635.JPEG:n01491361 +ILSVRC2012_val_00041636.JPEG:n04347754 +ILSVRC2012_val_00041637.JPEG:n04371774 +ILSVRC2012_val_00041638.JPEG:n09428293 +ILSVRC2012_val_00041639.JPEG:n04370456 +ILSVRC2012_val_00041640.JPEG:n01682714 +ILSVRC2012_val_00041641.JPEG:n01664065 +ILSVRC2012_val_00041642.JPEG:n02085620 +ILSVRC2012_val_00041643.JPEG:n02114855 +ILSVRC2012_val_00041644.JPEG:n03255030 +ILSVRC2012_val_00041645.JPEG:n02130308 +ILSVRC2012_val_00041646.JPEG:n04200800 +ILSVRC2012_val_00041647.JPEG:n02447366 +ILSVRC2012_val_00041648.JPEG:n04127249 +ILSVRC2012_val_00041649.JPEG:n02110185 +ILSVRC2012_val_00041650.JPEG:n02793495 +ILSVRC2012_val_00041651.JPEG:n03944341 +ILSVRC2012_val_00041652.JPEG:n03196217 +ILSVRC2012_val_00041653.JPEG:n02096294 +ILSVRC2012_val_00041654.JPEG:n04133789 +ILSVRC2012_val_00041655.JPEG:n07754684 +ILSVRC2012_val_00041656.JPEG:n03384352 +ILSVRC2012_val_00041657.JPEG:n03459775 +ILSVRC2012_val_00041658.JPEG:n04579145 +ILSVRC2012_val_00041659.JPEG:n01682714 +ILSVRC2012_val_00041660.JPEG:n03041632 +ILSVRC2012_val_00041661.JPEG:n07860988 +ILSVRC2012_val_00041662.JPEG:n06596364 +ILSVRC2012_val_00041663.JPEG:n04296562 +ILSVRC2012_val_00041664.JPEG:n04152593 +ILSVRC2012_val_00041665.JPEG:n01698640 +ILSVRC2012_val_00041666.JPEG:n03792972 +ILSVRC2012_val_00041667.JPEG:n04067472 +ILSVRC2012_val_00041668.JPEG:n03394916 +ILSVRC2012_val_00041669.JPEG:n01728920 +ILSVRC2012_val_00041670.JPEG:n04597913 +ILSVRC2012_val_00041671.JPEG:n04090263 +ILSVRC2012_val_00041672.JPEG:n03445777 +ILSVRC2012_val_00041673.JPEG:n13040303 +ILSVRC2012_val_00041674.JPEG:n07717556 +ILSVRC2012_val_00041675.JPEG:n01914609 +ILSVRC2012_val_00041676.JPEG:n07730033 +ILSVRC2012_val_00041677.JPEG:n02108089 +ILSVRC2012_val_00041678.JPEG:n04597913 +ILSVRC2012_val_00041679.JPEG:n02786058 +ILSVRC2012_val_00041680.JPEG:n06785654 +ILSVRC2012_val_00041681.JPEG:n03956157 +ILSVRC2012_val_00041682.JPEG:n04584207 +ILSVRC2012_val_00041683.JPEG:n03697007 +ILSVRC2012_val_00041684.JPEG:n02114712 +ILSVRC2012_val_00041685.JPEG:n02749479 +ILSVRC2012_val_00041686.JPEG:n07248320 +ILSVRC2012_val_00041687.JPEG:n03673027 +ILSVRC2012_val_00041688.JPEG:n02090379 +ILSVRC2012_val_00041689.JPEG:n04501370 +ILSVRC2012_val_00041690.JPEG:n01917289 +ILSVRC2012_val_00041691.JPEG:n04265275 +ILSVRC2012_val_00041692.JPEG:n04515003 +ILSVRC2012_val_00041693.JPEG:n03710721 +ILSVRC2012_val_00041694.JPEG:n03495258 +ILSVRC2012_val_00041695.JPEG:n04532670 +ILSVRC2012_val_00041696.JPEG:n04040759 +ILSVRC2012_val_00041697.JPEG:n01829413 +ILSVRC2012_val_00041698.JPEG:n02840245 +ILSVRC2012_val_00041699.JPEG:n02699494 +ILSVRC2012_val_00041700.JPEG:n02106550 +ILSVRC2012_val_00041701.JPEG:n03089624 +ILSVRC2012_val_00041702.JPEG:n02105056 +ILSVRC2012_val_00041703.JPEG:n02860847 +ILSVRC2012_val_00041704.JPEG:n02487347 +ILSVRC2012_val_00041705.JPEG:n02085782 +ILSVRC2012_val_00041706.JPEG:n03888257 +ILSVRC2012_val_00041707.JPEG:n03691459 +ILSVRC2012_val_00041708.JPEG:n02398521 +ILSVRC2012_val_00041709.JPEG:n04398044 +ILSVRC2012_val_00041710.JPEG:n01687978 +ILSVRC2012_val_00041711.JPEG:n04371774 +ILSVRC2012_val_00041712.JPEG:n02777292 +ILSVRC2012_val_00041713.JPEG:n01664065 +ILSVRC2012_val_00041714.JPEG:n04476259 +ILSVRC2012_val_00041715.JPEG:n04548280 +ILSVRC2012_val_00041716.JPEG:n12144580 +ILSVRC2012_val_00041717.JPEG:n02669723 +ILSVRC2012_val_00041718.JPEG:n02095314 +ILSVRC2012_val_00041719.JPEG:n02877765 +ILSVRC2012_val_00041720.JPEG:n04429376 +ILSVRC2012_val_00041721.JPEG:n03400231 +ILSVRC2012_val_00041722.JPEG:n03729826 +ILSVRC2012_val_00041723.JPEG:n02825657 +ILSVRC2012_val_00041724.JPEG:n02802426 +ILSVRC2012_val_00041725.JPEG:n03733281 +ILSVRC2012_val_00041726.JPEG:n03124043 +ILSVRC2012_val_00041727.JPEG:n07871810 +ILSVRC2012_val_00041728.JPEG:n02169497 +ILSVRC2012_val_00041729.JPEG:n04263257 +ILSVRC2012_val_00041730.JPEG:n01689811 +ILSVRC2012_val_00041731.JPEG:n04485082 +ILSVRC2012_val_00041732.JPEG:n04099969 +ILSVRC2012_val_00041733.JPEG:n03902125 +ILSVRC2012_val_00041734.JPEG:n04371430 +ILSVRC2012_val_00041735.JPEG:n02091635 +ILSVRC2012_val_00041736.JPEG:n03344393 +ILSVRC2012_val_00041737.JPEG:n02815834 +ILSVRC2012_val_00041738.JPEG:n13044778 +ILSVRC2012_val_00041739.JPEG:n02100877 +ILSVRC2012_val_00041740.JPEG:n02130308 +ILSVRC2012_val_00041741.JPEG:n09246464 +ILSVRC2012_val_00041742.JPEG:n02843684 +ILSVRC2012_val_00041743.JPEG:n01735189 +ILSVRC2012_val_00041744.JPEG:n06874185 +ILSVRC2012_val_00041745.JPEG:n02100583 +ILSVRC2012_val_00041746.JPEG:n02100877 +ILSVRC2012_val_00041747.JPEG:n15075141 +ILSVRC2012_val_00041748.JPEG:n02109525 +ILSVRC2012_val_00041749.JPEG:n02486410 +ILSVRC2012_val_00041750.JPEG:n02950826 +ILSVRC2012_val_00041751.JPEG:n01871265 +ILSVRC2012_val_00041752.JPEG:n02823750 +ILSVRC2012_val_00041753.JPEG:n07583066 +ILSVRC2012_val_00041754.JPEG:n02051845 +ILSVRC2012_val_00041755.JPEG:n01751748 +ILSVRC2012_val_00041756.JPEG:n02483362 +ILSVRC2012_val_00041757.JPEG:n03908618 +ILSVRC2012_val_00041758.JPEG:n02977058 +ILSVRC2012_val_00041759.JPEG:n02111889 +ILSVRC2012_val_00041760.JPEG:n04447861 +ILSVRC2012_val_00041761.JPEG:n02114855 +ILSVRC2012_val_00041762.JPEG:n02095314 +ILSVRC2012_val_00041763.JPEG:n02804414 +ILSVRC2012_val_00041764.JPEG:n02489166 +ILSVRC2012_val_00041765.JPEG:n04277352 +ILSVRC2012_val_00041766.JPEG:n02236044 +ILSVRC2012_val_00041767.JPEG:n02408429 +ILSVRC2012_val_00041768.JPEG:n02655020 +ILSVRC2012_val_00041769.JPEG:n01693334 +ILSVRC2012_val_00041770.JPEG:n03447721 +ILSVRC2012_val_00041771.JPEG:n02093647 +ILSVRC2012_val_00041772.JPEG:n02791124 +ILSVRC2012_val_00041773.JPEG:n02077923 +ILSVRC2012_val_00041774.JPEG:n04536866 +ILSVRC2012_val_00041775.JPEG:n03291819 +ILSVRC2012_val_00041776.JPEG:n02093859 +ILSVRC2012_val_00041777.JPEG:n02115641 +ILSVRC2012_val_00041778.JPEG:n04254680 +ILSVRC2012_val_00041779.JPEG:n04501370 +ILSVRC2012_val_00041780.JPEG:n04019541 +ILSVRC2012_val_00041781.JPEG:n02795169 +ILSVRC2012_val_00041782.JPEG:n03459775 +ILSVRC2012_val_00041783.JPEG:n04209133 +ILSVRC2012_val_00041784.JPEG:n07860988 +ILSVRC2012_val_00041785.JPEG:n04553703 +ILSVRC2012_val_00041786.JPEG:n02484975 +ILSVRC2012_val_00041787.JPEG:n03530642 +ILSVRC2012_val_00041788.JPEG:n02906734 +ILSVRC2012_val_00041789.JPEG:n04325704 +ILSVRC2012_val_00041790.JPEG:n04008634 +ILSVRC2012_val_00041791.JPEG:n12057211 +ILSVRC2012_val_00041792.JPEG:n02342885 +ILSVRC2012_val_00041793.JPEG:n04344873 +ILSVRC2012_val_00041794.JPEG:n03794056 +ILSVRC2012_val_00041795.JPEG:n02107142 +ILSVRC2012_val_00041796.JPEG:n04090263 +ILSVRC2012_val_00041797.JPEG:n02009229 +ILSVRC2012_val_00041798.JPEG:n02971356 +ILSVRC2012_val_00041799.JPEG:n02504458 +ILSVRC2012_val_00041800.JPEG:n04273569 +ILSVRC2012_val_00041801.JPEG:n09399592 +ILSVRC2012_val_00041802.JPEG:n03272562 +ILSVRC2012_val_00041803.JPEG:n02277742 +ILSVRC2012_val_00041804.JPEG:n02279972 +ILSVRC2012_val_00041805.JPEG:n07930864 +ILSVRC2012_val_00041806.JPEG:n02917067 +ILSVRC2012_val_00041807.JPEG:n04004767 +ILSVRC2012_val_00041808.JPEG:n04392985 +ILSVRC2012_val_00041809.JPEG:n07718747 +ILSVRC2012_val_00041810.JPEG:n02089078 +ILSVRC2012_val_00041811.JPEG:n03903868 +ILSVRC2012_val_00041812.JPEG:n03208938 +ILSVRC2012_val_00041813.JPEG:n02133161 +ILSVRC2012_val_00041814.JPEG:n03376595 +ILSVRC2012_val_00041815.JPEG:n02978881 +ILSVRC2012_val_00041816.JPEG:n03201208 +ILSVRC2012_val_00041817.JPEG:n02834397 +ILSVRC2012_val_00041818.JPEG:n02443484 +ILSVRC2012_val_00041819.JPEG:n02085620 +ILSVRC2012_val_00041820.JPEG:n02111889 +ILSVRC2012_val_00041821.JPEG:n03532672 +ILSVRC2012_val_00041822.JPEG:n04263257 +ILSVRC2012_val_00041823.JPEG:n03661043 +ILSVRC2012_val_00041824.JPEG:n15075141 +ILSVRC2012_val_00041825.JPEG:n04200800 +ILSVRC2012_val_00041826.JPEG:n03786901 +ILSVRC2012_val_00041827.JPEG:n01873310 +ILSVRC2012_val_00041828.JPEG:n04423845 +ILSVRC2012_val_00041829.JPEG:n01737021 +ILSVRC2012_val_00041830.JPEG:n02951358 +ILSVRC2012_val_00041831.JPEG:n02116738 +ILSVRC2012_val_00041832.JPEG:n01798484 +ILSVRC2012_val_00041833.JPEG:n03980874 +ILSVRC2012_val_00041834.JPEG:n02834397 +ILSVRC2012_val_00041835.JPEG:n02398521 +ILSVRC2012_val_00041836.JPEG:n01531178 +ILSVRC2012_val_00041837.JPEG:n07734744 +ILSVRC2012_val_00041838.JPEG:n01847000 +ILSVRC2012_val_00041839.JPEG:n03841143 +ILSVRC2012_val_00041840.JPEG:n02110185 +ILSVRC2012_val_00041841.JPEG:n13044778 +ILSVRC2012_val_00041842.JPEG:n02727426 +ILSVRC2012_val_00041843.JPEG:n02799071 +ILSVRC2012_val_00041844.JPEG:n02107908 +ILSVRC2012_val_00041845.JPEG:n01806143 +ILSVRC2012_val_00041846.JPEG:n03770679 +ILSVRC2012_val_00041847.JPEG:n03967562 +ILSVRC2012_val_00041848.JPEG:n02086646 +ILSVRC2012_val_00041849.JPEG:n02892767 +ILSVRC2012_val_00041850.JPEG:n01855032 +ILSVRC2012_val_00041851.JPEG:n02165105 +ILSVRC2012_val_00041852.JPEG:n01514859 +ILSVRC2012_val_00041853.JPEG:n04037443 +ILSVRC2012_val_00041854.JPEG:n03877472 +ILSVRC2012_val_00041855.JPEG:n03729826 +ILSVRC2012_val_00041856.JPEG:n01728920 +ILSVRC2012_val_00041857.JPEG:n02676566 +ILSVRC2012_val_00041858.JPEG:n03627232 +ILSVRC2012_val_00041859.JPEG:n04069434 +ILSVRC2012_val_00041860.JPEG:n04192698 +ILSVRC2012_val_00041861.JPEG:n02486261 +ILSVRC2012_val_00041862.JPEG:n02795169 +ILSVRC2012_val_00041863.JPEG:n04033901 +ILSVRC2012_val_00041864.JPEG:n01824575 +ILSVRC2012_val_00041865.JPEG:n02105641 +ILSVRC2012_val_00041866.JPEG:n02444819 +ILSVRC2012_val_00041867.JPEG:n01824575 +ILSVRC2012_val_00041868.JPEG:n03908714 +ILSVRC2012_val_00041869.JPEG:n04239074 +ILSVRC2012_val_00041870.JPEG:n02102480 +ILSVRC2012_val_00041871.JPEG:n02264363 +ILSVRC2012_val_00041872.JPEG:n01498041 +ILSVRC2012_val_00041873.JPEG:n02930766 +ILSVRC2012_val_00041874.JPEG:n04355933 +ILSVRC2012_val_00041875.JPEG:n04125021 +ILSVRC2012_val_00041876.JPEG:n03481172 +ILSVRC2012_val_00041877.JPEG:n02123159 +ILSVRC2012_val_00041878.JPEG:n02099712 +ILSVRC2012_val_00041879.JPEG:n04209239 +ILSVRC2012_val_00041880.JPEG:n02111889 +ILSVRC2012_val_00041881.JPEG:n02002556 +ILSVRC2012_val_00041882.JPEG:n03690938 +ILSVRC2012_val_00041883.JPEG:n04429376 +ILSVRC2012_val_00041884.JPEG:n03814906 +ILSVRC2012_val_00041885.JPEG:n04525305 +ILSVRC2012_val_00041886.JPEG:n02107908 +ILSVRC2012_val_00041887.JPEG:n01692333 +ILSVRC2012_val_00041888.JPEG:n04127249 +ILSVRC2012_val_00041889.JPEG:n01914609 +ILSVRC2012_val_00041890.JPEG:n04201297 +ILSVRC2012_val_00041891.JPEG:n02807133 +ILSVRC2012_val_00041892.JPEG:n01985128 +ILSVRC2012_val_00041893.JPEG:n02979186 +ILSVRC2012_val_00041894.JPEG:n02088238 +ILSVRC2012_val_00041895.JPEG:n03594945 +ILSVRC2012_val_00041896.JPEG:n03388043 +ILSVRC2012_val_00041897.JPEG:n09468604 +ILSVRC2012_val_00041898.JPEG:n03729826 +ILSVRC2012_val_00041899.JPEG:n02704792 +ILSVRC2012_val_00041900.JPEG:n07930864 +ILSVRC2012_val_00041901.JPEG:n03355925 +ILSVRC2012_val_00041902.JPEG:n04554684 +ILSVRC2012_val_00041903.JPEG:n04131690 +ILSVRC2012_val_00041904.JPEG:n04026417 +ILSVRC2012_val_00041905.JPEG:n02437616 +ILSVRC2012_val_00041906.JPEG:n03769881 +ILSVRC2012_val_00041907.JPEG:n04330267 +ILSVRC2012_val_00041908.JPEG:n02091831 +ILSVRC2012_val_00041909.JPEG:n01797886 +ILSVRC2012_val_00041910.JPEG:n02687172 +ILSVRC2012_val_00041911.JPEG:n02906734 +ILSVRC2012_val_00041912.JPEG:n02091635 +ILSVRC2012_val_00041913.JPEG:n02814533 +ILSVRC2012_val_00041914.JPEG:n02114712 +ILSVRC2012_val_00041915.JPEG:n03770439 +ILSVRC2012_val_00041916.JPEG:n04099969 +ILSVRC2012_val_00041917.JPEG:n04033995 +ILSVRC2012_val_00041918.JPEG:n02085936 +ILSVRC2012_val_00041919.JPEG:n01644900 +ILSVRC2012_val_00041920.JPEG:n02930766 +ILSVRC2012_val_00041921.JPEG:n01917289 +ILSVRC2012_val_00041922.JPEG:n01704323 +ILSVRC2012_val_00041923.JPEG:n04515003 +ILSVRC2012_val_00041924.JPEG:n01950731 +ILSVRC2012_val_00041925.JPEG:n03888257 +ILSVRC2012_val_00041926.JPEG:n07836838 +ILSVRC2012_val_00041927.JPEG:n02687172 +ILSVRC2012_val_00041928.JPEG:n02102318 +ILSVRC2012_val_00041929.JPEG:n02106030 +ILSVRC2012_val_00041930.JPEG:n02676566 +ILSVRC2012_val_00041931.JPEG:n01749939 +ILSVRC2012_val_00041932.JPEG:n03314780 +ILSVRC2012_val_00041933.JPEG:n03690938 +ILSVRC2012_val_00041934.JPEG:n02823750 +ILSVRC2012_val_00041935.JPEG:n03344393 +ILSVRC2012_val_00041936.JPEG:n03666591 +ILSVRC2012_val_00041937.JPEG:n04458633 +ILSVRC2012_val_00041938.JPEG:n04398044 +ILSVRC2012_val_00041939.JPEG:n01440764 +ILSVRC2012_val_00041940.JPEG:n04482393 +ILSVRC2012_val_00041941.JPEG:n03075370 +ILSVRC2012_val_00041942.JPEG:n02701002 +ILSVRC2012_val_00041943.JPEG:n04023962 +ILSVRC2012_val_00041944.JPEG:n01558993 +ILSVRC2012_val_00041945.JPEG:n07716358 +ILSVRC2012_val_00041946.JPEG:n02325366 +ILSVRC2012_val_00041947.JPEG:n02106382 +ILSVRC2012_val_00041948.JPEG:n04590129 +ILSVRC2012_val_00041949.JPEG:n10148035 +ILSVRC2012_val_00041950.JPEG:n02236044 +ILSVRC2012_val_00041951.JPEG:n04252077 +ILSVRC2012_val_00041952.JPEG:n12144580 +ILSVRC2012_val_00041953.JPEG:n02110627 +ILSVRC2012_val_00041954.JPEG:n03000134 +ILSVRC2012_val_00041955.JPEG:n02086079 +ILSVRC2012_val_00041956.JPEG:n03032252 +ILSVRC2012_val_00041957.JPEG:n02408429 +ILSVRC2012_val_00041958.JPEG:n03394916 +ILSVRC2012_val_00041959.JPEG:n02871525 +ILSVRC2012_val_00041960.JPEG:n01806567 +ILSVRC2012_val_00041961.JPEG:n02127052 +ILSVRC2012_val_00041962.JPEG:n02879718 +ILSVRC2012_val_00041963.JPEG:n03032252 +ILSVRC2012_val_00041964.JPEG:n03935335 +ILSVRC2012_val_00041965.JPEG:n04482393 +ILSVRC2012_val_00041966.JPEG:n03710721 +ILSVRC2012_val_00041967.JPEG:n04522168 +ILSVRC2012_val_00041968.JPEG:n04371430 +ILSVRC2012_val_00041969.JPEG:n04579145 +ILSVRC2012_val_00041970.JPEG:n03967562 +ILSVRC2012_val_00041971.JPEG:n03201208 +ILSVRC2012_val_00041972.JPEG:n04355338 +ILSVRC2012_val_00041973.JPEG:n04328186 +ILSVRC2012_val_00041974.JPEG:n04111531 +ILSVRC2012_val_00041975.JPEG:n01968897 +ILSVRC2012_val_00041976.JPEG:n02115913 +ILSVRC2012_val_00041977.JPEG:n01518878 +ILSVRC2012_val_00041978.JPEG:n04344873 +ILSVRC2012_val_00041979.JPEG:n02814533 +ILSVRC2012_val_00041980.JPEG:n01697457 +ILSVRC2012_val_00041981.JPEG:n04371430 +ILSVRC2012_val_00041982.JPEG:n01855032 +ILSVRC2012_val_00041983.JPEG:n01806143 +ILSVRC2012_val_00041984.JPEG:n03598930 +ILSVRC2012_val_00041985.JPEG:n02971356 +ILSVRC2012_val_00041986.JPEG:n03372029 +ILSVRC2012_val_00041987.JPEG:n02101388 +ILSVRC2012_val_00041988.JPEG:n02963159 +ILSVRC2012_val_00041989.JPEG:n02391049 +ILSVRC2012_val_00041990.JPEG:n01560419 +ILSVRC2012_val_00041991.JPEG:n02114367 +ILSVRC2012_val_00041992.JPEG:n03933933 +ILSVRC2012_val_00041993.JPEG:n03259280 +ILSVRC2012_val_00041994.JPEG:n01756291 +ILSVRC2012_val_00041995.JPEG:n04479046 +ILSVRC2012_val_00041996.JPEG:n07583066 +ILSVRC2012_val_00041997.JPEG:n03792972 +ILSVRC2012_val_00041998.JPEG:n02100877 +ILSVRC2012_val_00041999.JPEG:n07768694 +ILSVRC2012_val_00042000.JPEG:n02007558 +ILSVRC2012_val_00042001.JPEG:n03937543 +ILSVRC2012_val_00042002.JPEG:n03666591 +ILSVRC2012_val_00042003.JPEG:n02104029 +ILSVRC2012_val_00042004.JPEG:n01910747 +ILSVRC2012_val_00042005.JPEG:n02095889 +ILSVRC2012_val_00042006.JPEG:n04417672 +ILSVRC2012_val_00042007.JPEG:n03769881 +ILSVRC2012_val_00042008.JPEG:n03929855 +ILSVRC2012_val_00042009.JPEG:n02641379 +ILSVRC2012_val_00042010.JPEG:n02229544 +ILSVRC2012_val_00042011.JPEG:n07614500 +ILSVRC2012_val_00042012.JPEG:n04311174 +ILSVRC2012_val_00042013.JPEG:n02361337 +ILSVRC2012_val_00042014.JPEG:n07753592 +ILSVRC2012_val_00042015.JPEG:n02206856 +ILSVRC2012_val_00042016.JPEG:n04090263 +ILSVRC2012_val_00042017.JPEG:n03444034 +ILSVRC2012_val_00042018.JPEG:n04525305 +ILSVRC2012_val_00042019.JPEG:n02281406 +ILSVRC2012_val_00042020.JPEG:n02526121 +ILSVRC2012_val_00042021.JPEG:n01807496 +ILSVRC2012_val_00042022.JPEG:n02096294 +ILSVRC2012_val_00042023.JPEG:n01667778 +ILSVRC2012_val_00042024.JPEG:n02480855 +ILSVRC2012_val_00042025.JPEG:n07711569 +ILSVRC2012_val_00042026.JPEG:n02009229 +ILSVRC2012_val_00042027.JPEG:n01697457 +ILSVRC2012_val_00042028.JPEG:n03271574 +ILSVRC2012_val_00042029.JPEG:n01687978 +ILSVRC2012_val_00042030.JPEG:n02100236 +ILSVRC2012_val_00042031.JPEG:n03908714 +ILSVRC2012_val_00042032.JPEG:n01531178 +ILSVRC2012_val_00042033.JPEG:n02364673 +ILSVRC2012_val_00042034.JPEG:n03773504 +ILSVRC2012_val_00042035.JPEG:n03000684 +ILSVRC2012_val_00042036.JPEG:n02981792 +ILSVRC2012_val_00042037.JPEG:n04485082 +ILSVRC2012_val_00042038.JPEG:n01797886 +ILSVRC2012_val_00042039.JPEG:n03498962 +ILSVRC2012_val_00042040.JPEG:n03538406 +ILSVRC2012_val_00042041.JPEG:n03530642 +ILSVRC2012_val_00042042.JPEG:n01872401 +ILSVRC2012_val_00042043.JPEG:n02342885 +ILSVRC2012_val_00042044.JPEG:n02457408 +ILSVRC2012_val_00042045.JPEG:n02480495 +ILSVRC2012_val_00042046.JPEG:n02480855 +ILSVRC2012_val_00042047.JPEG:n01770393 +ILSVRC2012_val_00042048.JPEG:n01560419 +ILSVRC2012_val_00042049.JPEG:n01665541 +ILSVRC2012_val_00042050.JPEG:n04540053 +ILSVRC2012_val_00042051.JPEG:n04346328 +ILSVRC2012_val_00042052.JPEG:n04485082 +ILSVRC2012_val_00042053.JPEG:n02091635 +ILSVRC2012_val_00042054.JPEG:n03733805 +ILSVRC2012_val_00042055.JPEG:n02120505 +ILSVRC2012_val_00042056.JPEG:n02988304 +ILSVRC2012_val_00042057.JPEG:n04049303 +ILSVRC2012_val_00042058.JPEG:n02607072 +ILSVRC2012_val_00042059.JPEG:n02488702 +ILSVRC2012_val_00042060.JPEG:n03026506 +ILSVRC2012_val_00042061.JPEG:n07718472 +ILSVRC2012_val_00042062.JPEG:n03627232 +ILSVRC2012_val_00042063.JPEG:n03388043 +ILSVRC2012_val_00042064.JPEG:n02403003 +ILSVRC2012_val_00042065.JPEG:n03627232 +ILSVRC2012_val_00042066.JPEG:n03877845 +ILSVRC2012_val_00042067.JPEG:n03388043 +ILSVRC2012_val_00042068.JPEG:n02487347 +ILSVRC2012_val_00042069.JPEG:n04005630 +ILSVRC2012_val_00042070.JPEG:n01682714 +ILSVRC2012_val_00042071.JPEG:n01818515 +ILSVRC2012_val_00042072.JPEG:n04311174 +ILSVRC2012_val_00042073.JPEG:n01664065 +ILSVRC2012_val_00042074.JPEG:n04509417 +ILSVRC2012_val_00042075.JPEG:n02086910 +ILSVRC2012_val_00042076.JPEG:n02219486 +ILSVRC2012_val_00042077.JPEG:n04392985 +ILSVRC2012_val_00042078.JPEG:n04344873 +ILSVRC2012_val_00042079.JPEG:n01685808 +ILSVRC2012_val_00042080.JPEG:n07717410 +ILSVRC2012_val_00042081.JPEG:n03384352 +ILSVRC2012_val_00042082.JPEG:n01728920 +ILSVRC2012_val_00042083.JPEG:n02027492 +ILSVRC2012_val_00042084.JPEG:n02012849 +ILSVRC2012_val_00042085.JPEG:n04336792 +ILSVRC2012_val_00042086.JPEG:n02481823 +ILSVRC2012_val_00042087.JPEG:n07565083 +ILSVRC2012_val_00042088.JPEG:n03868863 +ILSVRC2012_val_00042089.JPEG:n03179701 +ILSVRC2012_val_00042090.JPEG:n02109525 +ILSVRC2012_val_00042091.JPEG:n04330267 +ILSVRC2012_val_00042092.JPEG:n03982430 +ILSVRC2012_val_00042093.JPEG:n03272010 +ILSVRC2012_val_00042094.JPEG:n04005630 +ILSVRC2012_val_00042095.JPEG:n02112137 +ILSVRC2012_val_00042096.JPEG:n03770439 +ILSVRC2012_val_00042097.JPEG:n02088094 +ILSVRC2012_val_00042098.JPEG:n02114548 +ILSVRC2012_val_00042099.JPEG:n02091032 +ILSVRC2012_val_00042100.JPEG:n01728572 +ILSVRC2012_val_00042101.JPEG:n03240683 +ILSVRC2012_val_00042102.JPEG:n02808440 +ILSVRC2012_val_00042103.JPEG:n02486410 +ILSVRC2012_val_00042104.JPEG:n02930766 +ILSVRC2012_val_00042105.JPEG:n01737021 +ILSVRC2012_val_00042106.JPEG:n03733805 +ILSVRC2012_val_00042107.JPEG:n03110669 +ILSVRC2012_val_00042108.JPEG:n03016953 +ILSVRC2012_val_00042109.JPEG:n01748264 +ILSVRC2012_val_00042110.JPEG:n02325366 +ILSVRC2012_val_00042111.JPEG:n01748264 +ILSVRC2012_val_00042112.JPEG:n02364673 +ILSVRC2012_val_00042113.JPEG:n02017213 +ILSVRC2012_val_00042114.JPEG:n04252077 +ILSVRC2012_val_00042115.JPEG:n02860847 +ILSVRC2012_val_00042116.JPEG:n03124043 +ILSVRC2012_val_00042117.JPEG:n03461385 +ILSVRC2012_val_00042118.JPEG:n02090721 +ILSVRC2012_val_00042119.JPEG:n03998194 +ILSVRC2012_val_00042120.JPEG:n02095570 +ILSVRC2012_val_00042121.JPEG:n07753113 +ILSVRC2012_val_00042122.JPEG:n04423845 +ILSVRC2012_val_00042123.JPEG:n04044716 +ILSVRC2012_val_00042124.JPEG:n01695060 +ILSVRC2012_val_00042125.JPEG:n01632458 +ILSVRC2012_val_00042126.JPEG:n02643566 +ILSVRC2012_val_00042127.JPEG:n02167151 +ILSVRC2012_val_00042128.JPEG:n01860187 +ILSVRC2012_val_00042129.JPEG:n02403003 +ILSVRC2012_val_00042130.JPEG:n02840245 +ILSVRC2012_val_00042131.JPEG:n03658185 +ILSVRC2012_val_00042132.JPEG:n04116512 +ILSVRC2012_val_00042133.JPEG:n02096294 +ILSVRC2012_val_00042134.JPEG:n01735189 +ILSVRC2012_val_00042135.JPEG:n01514859 +ILSVRC2012_val_00042136.JPEG:n04131690 +ILSVRC2012_val_00042137.JPEG:n02978881 +ILSVRC2012_val_00042138.JPEG:n03461385 +ILSVRC2012_val_00042139.JPEG:n03944341 +ILSVRC2012_val_00042140.JPEG:n02441942 +ILSVRC2012_val_00042141.JPEG:n07753113 +ILSVRC2012_val_00042142.JPEG:n01693334 +ILSVRC2012_val_00042143.JPEG:n09399592 +ILSVRC2012_val_00042144.JPEG:n02105412 +ILSVRC2012_val_00042145.JPEG:n03400231 +ILSVRC2012_val_00042146.JPEG:n04550184 +ILSVRC2012_val_00042147.JPEG:n02823428 +ILSVRC2012_val_00042148.JPEG:n02112137 +ILSVRC2012_val_00042149.JPEG:n03920288 +ILSVRC2012_val_00042150.JPEG:n04509417 +ILSVRC2012_val_00042151.JPEG:n03785016 +ILSVRC2012_val_00042152.JPEG:n03534580 +ILSVRC2012_val_00042153.JPEG:n02066245 +ILSVRC2012_val_00042154.JPEG:n02807133 +ILSVRC2012_val_00042155.JPEG:n01924916 +ILSVRC2012_val_00042156.JPEG:n02017213 +ILSVRC2012_val_00042157.JPEG:n03796401 +ILSVRC2012_val_00042158.JPEG:n02090721 +ILSVRC2012_val_00042159.JPEG:n01981276 +ILSVRC2012_val_00042160.JPEG:n02497673 +ILSVRC2012_val_00042161.JPEG:n09399592 +ILSVRC2012_val_00042162.JPEG:n01749939 +ILSVRC2012_val_00042163.JPEG:n03344393 +ILSVRC2012_val_00042164.JPEG:n03344393 +ILSVRC2012_val_00042165.JPEG:n02490219 +ILSVRC2012_val_00042166.JPEG:n04335435 +ILSVRC2012_val_00042167.JPEG:n04065272 +ILSVRC2012_val_00042168.JPEG:n07873807 +ILSVRC2012_val_00042169.JPEG:n03314780 +ILSVRC2012_val_00042170.JPEG:n03530642 +ILSVRC2012_val_00042171.JPEG:n02783161 +ILSVRC2012_val_00042172.JPEG:n02114548 +ILSVRC2012_val_00042173.JPEG:n02319095 +ILSVRC2012_val_00042174.JPEG:n03018349 +ILSVRC2012_val_00042175.JPEG:n01498041 +ILSVRC2012_val_00042176.JPEG:n02859443 +ILSVRC2012_val_00042177.JPEG:n02096051 +ILSVRC2012_val_00042178.JPEG:n04251144 +ILSVRC2012_val_00042179.JPEG:n03042490 +ILSVRC2012_val_00042180.JPEG:n02167151 +ILSVRC2012_val_00042181.JPEG:n02096294 +ILSVRC2012_val_00042182.JPEG:n09246464 +ILSVRC2012_val_00042183.JPEG:n12985857 +ILSVRC2012_val_00042184.JPEG:n02100583 +ILSVRC2012_val_00042185.JPEG:n03240683 +ILSVRC2012_val_00042186.JPEG:n02236044 +ILSVRC2012_val_00042187.JPEG:n02356798 +ILSVRC2012_val_00042188.JPEG:n02317335 +ILSVRC2012_val_00042189.JPEG:n02859443 +ILSVRC2012_val_00042190.JPEG:n02510455 +ILSVRC2012_val_00042191.JPEG:n01945685 +ILSVRC2012_val_00042192.JPEG:n03792972 +ILSVRC2012_val_00042193.JPEG:n02011460 +ILSVRC2012_val_00042194.JPEG:n03220513 +ILSVRC2012_val_00042195.JPEG:n04141076 +ILSVRC2012_val_00042196.JPEG:n03662601 +ILSVRC2012_val_00042197.JPEG:n07745940 +ILSVRC2012_val_00042198.JPEG:n02747177 +ILSVRC2012_val_00042199.JPEG:n12998815 +ILSVRC2012_val_00042200.JPEG:n04209133 +ILSVRC2012_val_00042201.JPEG:n02097130 +ILSVRC2012_val_00042202.JPEG:n01685808 +ILSVRC2012_val_00042203.JPEG:n04273569 +ILSVRC2012_val_00042204.JPEG:n04515003 +ILSVRC2012_val_00042205.JPEG:n02094258 +ILSVRC2012_val_00042206.JPEG:n02109047 +ILSVRC2012_val_00042207.JPEG:n03028079 +ILSVRC2012_val_00042208.JPEG:n02408429 +ILSVRC2012_val_00042209.JPEG:n03777754 +ILSVRC2012_val_00042210.JPEG:n02113186 +ILSVRC2012_val_00042211.JPEG:n02500267 +ILSVRC2012_val_00042212.JPEG:n03891251 +ILSVRC2012_val_00042213.JPEG:n02112018 +ILSVRC2012_val_00042214.JPEG:n04487081 +ILSVRC2012_val_00042215.JPEG:n02927161 +ILSVRC2012_val_00042216.JPEG:n01664065 +ILSVRC2012_val_00042217.JPEG:n03534580 +ILSVRC2012_val_00042218.JPEG:n03729826 +ILSVRC2012_val_00042219.JPEG:n03187595 +ILSVRC2012_val_00042220.JPEG:n02105505 +ILSVRC2012_val_00042221.JPEG:n07718747 +ILSVRC2012_val_00042222.JPEG:n02802426 +ILSVRC2012_val_00042223.JPEG:n02226429 +ILSVRC2012_val_00042224.JPEG:n04116512 +ILSVRC2012_val_00042225.JPEG:n01756291 +ILSVRC2012_val_00042226.JPEG:n01817953 +ILSVRC2012_val_00042227.JPEG:n07714990 +ILSVRC2012_val_00042228.JPEG:n02457408 +ILSVRC2012_val_00042229.JPEG:n03109150 +ILSVRC2012_val_00042230.JPEG:n04026417 +ILSVRC2012_val_00042231.JPEG:n02437312 +ILSVRC2012_val_00042232.JPEG:n02124075 +ILSVRC2012_val_00042233.JPEG:n02113978 +ILSVRC2012_val_00042234.JPEG:n03109150 +ILSVRC2012_val_00042235.JPEG:n02389026 +ILSVRC2012_val_00042236.JPEG:n06785654 +ILSVRC2012_val_00042237.JPEG:n03089624 +ILSVRC2012_val_00042238.JPEG:n03444034 +ILSVRC2012_val_00042239.JPEG:n04149813 +ILSVRC2012_val_00042240.JPEG:n02091032 +ILSVRC2012_val_00042241.JPEG:n04376876 +ILSVRC2012_val_00042242.JPEG:n02606052 +ILSVRC2012_val_00042243.JPEG:n03492542 +ILSVRC2012_val_00042244.JPEG:n04579145 +ILSVRC2012_val_00042245.JPEG:n01496331 +ILSVRC2012_val_00042246.JPEG:n01592084 +ILSVRC2012_val_00042247.JPEG:n04141975 +ILSVRC2012_val_00042248.JPEG:n01580077 +ILSVRC2012_val_00042249.JPEG:n02112706 +ILSVRC2012_val_00042250.JPEG:n03388043 +ILSVRC2012_val_00042251.JPEG:n02256656 +ILSVRC2012_val_00042252.JPEG:n02087394 +ILSVRC2012_val_00042253.JPEG:n04179913 +ILSVRC2012_val_00042254.JPEG:n07930864 +ILSVRC2012_val_00042255.JPEG:n04355338 +ILSVRC2012_val_00042256.JPEG:n03874293 +ILSVRC2012_val_00042257.JPEG:n04033995 +ILSVRC2012_val_00042258.JPEG:n02088364 +ILSVRC2012_val_00042259.JPEG:n03535780 +ILSVRC2012_val_00042260.JPEG:n03476991 +ILSVRC2012_val_00042261.JPEG:n04336792 +ILSVRC2012_val_00042262.JPEG:n03888257 +ILSVRC2012_val_00042263.JPEG:n07836838 +ILSVRC2012_val_00042264.JPEG:n03028079 +ILSVRC2012_val_00042265.JPEG:n03877845 +ILSVRC2012_val_00042266.JPEG:n03982430 +ILSVRC2012_val_00042267.JPEG:n02116738 +ILSVRC2012_val_00042268.JPEG:n04596742 +ILSVRC2012_val_00042269.JPEG:n03843555 +ILSVRC2012_val_00042270.JPEG:n15075141 +ILSVRC2012_val_00042271.JPEG:n04325704 +ILSVRC2012_val_00042272.JPEG:n04398044 +ILSVRC2012_val_00042273.JPEG:n02134084 +ILSVRC2012_val_00042274.JPEG:n02132136 +ILSVRC2012_val_00042275.JPEG:n03602883 +ILSVRC2012_val_00042276.JPEG:n01955084 +ILSVRC2012_val_00042277.JPEG:n02268853 +ILSVRC2012_val_00042278.JPEG:n02490219 +ILSVRC2012_val_00042279.JPEG:n04044716 +ILSVRC2012_val_00042280.JPEG:n02492660 +ILSVRC2012_val_00042281.JPEG:n01770393 +ILSVRC2012_val_00042282.JPEG:n03447447 +ILSVRC2012_val_00042283.JPEG:n07871810 +ILSVRC2012_val_00042284.JPEG:n01739381 +ILSVRC2012_val_00042285.JPEG:n03933933 +ILSVRC2012_val_00042286.JPEG:n02110958 +ILSVRC2012_val_00042287.JPEG:n04517823 +ILSVRC2012_val_00042288.JPEG:n10565667 +ILSVRC2012_val_00042289.JPEG:n02087046 +ILSVRC2012_val_00042290.JPEG:n02909870 +ILSVRC2012_val_00042291.JPEG:n07747607 +ILSVRC2012_val_00042292.JPEG:n13037406 +ILSVRC2012_val_00042293.JPEG:n03743016 +ILSVRC2012_val_00042294.JPEG:n02113023 +ILSVRC2012_val_00042295.JPEG:n07716358 +ILSVRC2012_val_00042296.JPEG:n01828970 +ILSVRC2012_val_00042297.JPEG:n04579145 +ILSVRC2012_val_00042298.JPEG:n04482393 +ILSVRC2012_val_00042299.JPEG:n02169497 +ILSVRC2012_val_00042300.JPEG:n04371430 +ILSVRC2012_val_00042301.JPEG:n01751748 +ILSVRC2012_val_00042302.JPEG:n01632777 +ILSVRC2012_val_00042303.JPEG:n02106382 +ILSVRC2012_val_00042304.JPEG:n01697457 +ILSVRC2012_val_00042305.JPEG:n04074963 +ILSVRC2012_val_00042306.JPEG:n03062245 +ILSVRC2012_val_00042307.JPEG:n02607072 +ILSVRC2012_val_00042308.JPEG:n03868863 +ILSVRC2012_val_00042309.JPEG:n04409515 +ILSVRC2012_val_00042310.JPEG:n01829413 +ILSVRC2012_val_00042311.JPEG:n04254680 +ILSVRC2012_val_00042312.JPEG:n01728920 +ILSVRC2012_val_00042313.JPEG:n02802426 +ILSVRC2012_val_00042314.JPEG:n03666591 +ILSVRC2012_val_00042315.JPEG:n01984695 +ILSVRC2012_val_00042316.JPEG:n02708093 +ILSVRC2012_val_00042317.JPEG:n02090721 +ILSVRC2012_val_00042318.JPEG:n02089973 +ILSVRC2012_val_00042319.JPEG:n02099849 +ILSVRC2012_val_00042320.JPEG:n02134084 +ILSVRC2012_val_00042321.JPEG:n13133613 +ILSVRC2012_val_00042322.JPEG:n03733281 +ILSVRC2012_val_00042323.JPEG:n02268853 +ILSVRC2012_val_00042324.JPEG:n04347754 +ILSVRC2012_val_00042325.JPEG:n02115641 +ILSVRC2012_val_00042326.JPEG:n04346328 +ILSVRC2012_val_00042327.JPEG:n02769748 +ILSVRC2012_val_00042328.JPEG:n01665541 +ILSVRC2012_val_00042329.JPEG:n03961711 +ILSVRC2012_val_00042330.JPEG:n02391049 +ILSVRC2012_val_00042331.JPEG:n01675722 +ILSVRC2012_val_00042332.JPEG:n02017213 +ILSVRC2012_val_00042333.JPEG:n03045698 +ILSVRC2012_val_00042334.JPEG:n02356798 +ILSVRC2012_val_00042335.JPEG:n02977058 +ILSVRC2012_val_00042336.JPEG:n01873310 +ILSVRC2012_val_00042337.JPEG:n02276258 +ILSVRC2012_val_00042338.JPEG:n03692522 +ILSVRC2012_val_00042339.JPEG:n02107908 +ILSVRC2012_val_00042340.JPEG:n03954731 +ILSVRC2012_val_00042341.JPEG:n04389033 +ILSVRC2012_val_00042342.JPEG:n02226429 +ILSVRC2012_val_00042343.JPEG:n03676483 +ILSVRC2012_val_00042344.JPEG:n02107908 +ILSVRC2012_val_00042345.JPEG:n01484850 +ILSVRC2012_val_00042346.JPEG:n01774750 +ILSVRC2012_val_00042347.JPEG:n02979186 +ILSVRC2012_val_00042348.JPEG:n03761084 +ILSVRC2012_val_00042349.JPEG:n03623198 +ILSVRC2012_val_00042350.JPEG:n03445777 +ILSVRC2012_val_00042351.JPEG:n03770679 +ILSVRC2012_val_00042352.JPEG:n01728572 +ILSVRC2012_val_00042353.JPEG:n03495258 +ILSVRC2012_val_00042354.JPEG:n04613696 +ILSVRC2012_val_00042355.JPEG:n02441942 +ILSVRC2012_val_00042356.JPEG:n03594734 +ILSVRC2012_val_00042357.JPEG:n02114855 +ILSVRC2012_val_00042358.JPEG:n02883205 +ILSVRC2012_val_00042359.JPEG:n04311174 +ILSVRC2012_val_00042360.JPEG:n04532670 +ILSVRC2012_val_00042361.JPEG:n02134418 +ILSVRC2012_val_00042362.JPEG:n03717622 +ILSVRC2012_val_00042363.JPEG:n02859443 +ILSVRC2012_val_00042364.JPEG:n03930313 +ILSVRC2012_val_00042365.JPEG:n03126707 +ILSVRC2012_val_00042366.JPEG:n03977966 +ILSVRC2012_val_00042367.JPEG:n03983396 +ILSVRC2012_val_00042368.JPEG:n04456115 +ILSVRC2012_val_00042369.JPEG:n07760859 +ILSVRC2012_val_00042370.JPEG:n01532829 +ILSVRC2012_val_00042371.JPEG:n04208210 +ILSVRC2012_val_00042372.JPEG:n03991062 +ILSVRC2012_val_00042373.JPEG:n04131690 +ILSVRC2012_val_00042374.JPEG:n03649909 +ILSVRC2012_val_00042375.JPEG:n03425413 +ILSVRC2012_val_00042376.JPEG:n02017213 +ILSVRC2012_val_00042377.JPEG:n02974003 +ILSVRC2012_val_00042378.JPEG:n03958227 +ILSVRC2012_val_00042379.JPEG:n02408429 +ILSVRC2012_val_00042380.JPEG:n01614925 +ILSVRC2012_val_00042381.JPEG:n03884397 +ILSVRC2012_val_00042382.JPEG:n04429376 +ILSVRC2012_val_00042383.JPEG:n01749939 +ILSVRC2012_val_00042384.JPEG:n01756291 +ILSVRC2012_val_00042385.JPEG:n01498041 +ILSVRC2012_val_00042386.JPEG:n03992509 +ILSVRC2012_val_00042387.JPEG:n03532672 +ILSVRC2012_val_00042388.JPEG:n04286575 +ILSVRC2012_val_00042389.JPEG:n03376595 +ILSVRC2012_val_00042390.JPEG:n02108000 +ILSVRC2012_val_00042391.JPEG:n02108551 +ILSVRC2012_val_00042392.JPEG:n07565083 +ILSVRC2012_val_00042393.JPEG:n03792782 +ILSVRC2012_val_00042394.JPEG:n02089867 +ILSVRC2012_val_00042395.JPEG:n07684084 +ILSVRC2012_val_00042396.JPEG:n03404251 +ILSVRC2012_val_00042397.JPEG:n03871628 +ILSVRC2012_val_00042398.JPEG:n04311004 +ILSVRC2012_val_00042399.JPEG:n13040303 +ILSVRC2012_val_00042400.JPEG:n02111129 +ILSVRC2012_val_00042401.JPEG:n02422699 +ILSVRC2012_val_00042402.JPEG:n03733281 +ILSVRC2012_val_00042403.JPEG:n04153751 +ILSVRC2012_val_00042404.JPEG:n04179913 +ILSVRC2012_val_00042405.JPEG:n02268443 +ILSVRC2012_val_00042406.JPEG:n02443114 +ILSVRC2012_val_00042407.JPEG:n03485794 +ILSVRC2012_val_00042408.JPEG:n07579787 +ILSVRC2012_val_00042409.JPEG:n02110063 +ILSVRC2012_val_00042410.JPEG:n01616318 +ILSVRC2012_val_00042411.JPEG:n03871628 +ILSVRC2012_val_00042412.JPEG:n07697537 +ILSVRC2012_val_00042413.JPEG:n02114367 +ILSVRC2012_val_00042414.JPEG:n02091134 +ILSVRC2012_val_00042415.JPEG:n02883205 +ILSVRC2012_val_00042416.JPEG:n02814533 +ILSVRC2012_val_00042417.JPEG:n03871628 +ILSVRC2012_val_00042418.JPEG:n02105056 +ILSVRC2012_val_00042419.JPEG:n02865351 +ILSVRC2012_val_00042420.JPEG:n03991062 +ILSVRC2012_val_00042421.JPEG:n02104365 +ILSVRC2012_val_00042422.JPEG:n04275548 +ILSVRC2012_val_00042423.JPEG:n03929660 +ILSVRC2012_val_00042424.JPEG:n03814639 +ILSVRC2012_val_00042425.JPEG:n02834397 +ILSVRC2012_val_00042426.JPEG:n03792782 +ILSVRC2012_val_00042427.JPEG:n07730033 +ILSVRC2012_val_00042428.JPEG:n02445715 +ILSVRC2012_val_00042429.JPEG:n02804610 +ILSVRC2012_val_00042430.JPEG:n02119789 +ILSVRC2012_val_00042431.JPEG:n04040759 +ILSVRC2012_val_00042432.JPEG:n02415577 +ILSVRC2012_val_00042433.JPEG:n02206856 +ILSVRC2012_val_00042434.JPEG:n02114367 +ILSVRC2012_val_00042435.JPEG:n04493381 +ILSVRC2012_val_00042436.JPEG:n02276258 +ILSVRC2012_val_00042437.JPEG:n03991062 +ILSVRC2012_val_00042438.JPEG:n02236044 +ILSVRC2012_val_00042439.JPEG:n04332243 +ILSVRC2012_val_00042440.JPEG:n07760859 +ILSVRC2012_val_00042441.JPEG:n02504013 +ILSVRC2012_val_00042442.JPEG:n02090379 +ILSVRC2012_val_00042443.JPEG:n02445715 +ILSVRC2012_val_00042444.JPEG:n10565667 +ILSVRC2012_val_00042445.JPEG:n04487081 +ILSVRC2012_val_00042446.JPEG:n09472597 +ILSVRC2012_val_00042447.JPEG:n04398044 +ILSVRC2012_val_00042448.JPEG:n01873310 +ILSVRC2012_val_00042449.JPEG:n02087046 +ILSVRC2012_val_00042450.JPEG:n03788365 +ILSVRC2012_val_00042451.JPEG:n02097658 +ILSVRC2012_val_00042452.JPEG:n03467068 +ILSVRC2012_val_00042453.JPEG:n07717410 +ILSVRC2012_val_00042454.JPEG:n03642806 +ILSVRC2012_val_00042455.JPEG:n03063689 +ILSVRC2012_val_00042456.JPEG:n01914609 +ILSVRC2012_val_00042457.JPEG:n03792782 +ILSVRC2012_val_00042458.JPEG:n12267677 +ILSVRC2012_val_00042459.JPEG:n03220513 +ILSVRC2012_val_00042460.JPEG:n02119789 +ILSVRC2012_val_00042461.JPEG:n02950826 +ILSVRC2012_val_00042462.JPEG:n02113712 +ILSVRC2012_val_00042463.JPEG:n03697007 +ILSVRC2012_val_00042464.JPEG:n04009552 +ILSVRC2012_val_00042465.JPEG:n03876231 +ILSVRC2012_val_00042466.JPEG:n10148035 +ILSVRC2012_val_00042467.JPEG:n03590841 +ILSVRC2012_val_00042468.JPEG:n03461385 +ILSVRC2012_val_00042469.JPEG:n02814860 +ILSVRC2012_val_00042470.JPEG:n03729826 +ILSVRC2012_val_00042471.JPEG:n03255030 +ILSVRC2012_val_00042472.JPEG:n09288635 +ILSVRC2012_val_00042473.JPEG:n02094114 +ILSVRC2012_val_00042474.JPEG:n04550184 +ILSVRC2012_val_00042475.JPEG:n02115913 +ILSVRC2012_val_00042476.JPEG:n01990800 +ILSVRC2012_val_00042477.JPEG:n02112350 +ILSVRC2012_val_00042478.JPEG:n12998815 +ILSVRC2012_val_00042479.JPEG:n02672831 +ILSVRC2012_val_00042480.JPEG:n01860187 +ILSVRC2012_val_00042481.JPEG:n04493381 +ILSVRC2012_val_00042482.JPEG:n02979186 +ILSVRC2012_val_00042483.JPEG:n02441942 +ILSVRC2012_val_00042484.JPEG:n02128757 +ILSVRC2012_val_00042485.JPEG:n01883070 +ILSVRC2012_val_00042486.JPEG:n03803284 +ILSVRC2012_val_00042487.JPEG:n03417042 +ILSVRC2012_val_00042488.JPEG:n02992211 +ILSVRC2012_val_00042489.JPEG:n04462240 +ILSVRC2012_val_00042490.JPEG:n03759954 +ILSVRC2012_val_00042491.JPEG:n01984695 +ILSVRC2012_val_00042492.JPEG:n07584110 +ILSVRC2012_val_00042493.JPEG:n04118538 +ILSVRC2012_val_00042494.JPEG:n02105412 +ILSVRC2012_val_00042495.JPEG:n03218198 +ILSVRC2012_val_00042496.JPEG:n02835271 +ILSVRC2012_val_00042497.JPEG:n03314780 +ILSVRC2012_val_00042498.JPEG:n04070727 +ILSVRC2012_val_00042499.JPEG:n03325584 +ILSVRC2012_val_00042500.JPEG:n01742172 +ILSVRC2012_val_00042501.JPEG:n04266014 +ILSVRC2012_val_00042502.JPEG:n03447447 +ILSVRC2012_val_00042503.JPEG:n02701002 +ILSVRC2012_val_00042504.JPEG:n01877812 +ILSVRC2012_val_00042505.JPEG:n03062245 +ILSVRC2012_val_00042506.JPEG:n01592084 +ILSVRC2012_val_00042507.JPEG:n01924916 +ILSVRC2012_val_00042508.JPEG:n03781244 +ILSVRC2012_val_00042509.JPEG:n01798484 +ILSVRC2012_val_00042510.JPEG:n02730930 +ILSVRC2012_val_00042511.JPEG:n02417914 +ILSVRC2012_val_00042512.JPEG:n02791124 +ILSVRC2012_val_00042513.JPEG:n02412080 +ILSVRC2012_val_00042514.JPEG:n09256479 +ILSVRC2012_val_00042515.JPEG:n04008634 +ILSVRC2012_val_00042516.JPEG:n02493793 +ILSVRC2012_val_00042517.JPEG:n07753275 +ILSVRC2012_val_00042518.JPEG:n03980874 +ILSVRC2012_val_00042519.JPEG:n02280649 +ILSVRC2012_val_00042520.JPEG:n03400231 +ILSVRC2012_val_00042521.JPEG:n03476991 +ILSVRC2012_val_00042522.JPEG:n02787622 +ILSVRC2012_val_00042523.JPEG:n02086240 +ILSVRC2012_val_00042524.JPEG:n04041544 +ILSVRC2012_val_00042525.JPEG:n04370456 +ILSVRC2012_val_00042526.JPEG:n04591713 +ILSVRC2012_val_00042527.JPEG:n03062245 +ILSVRC2012_val_00042528.JPEG:n04254120 +ILSVRC2012_val_00042529.JPEG:n02125311 +ILSVRC2012_val_00042530.JPEG:n03920288 +ILSVRC2012_val_00042531.JPEG:n02088364 +ILSVRC2012_val_00042532.JPEG:n02002724 +ILSVRC2012_val_00042533.JPEG:n02107683 +ILSVRC2012_val_00042534.JPEG:n01498041 +ILSVRC2012_val_00042535.JPEG:n04550184 +ILSVRC2012_val_00042536.JPEG:n01984695 +ILSVRC2012_val_00042537.JPEG:n04584207 +ILSVRC2012_val_00042538.JPEG:n02971356 +ILSVRC2012_val_00042539.JPEG:n03961711 +ILSVRC2012_val_00042540.JPEG:n02447366 +ILSVRC2012_val_00042541.JPEG:n01855672 +ILSVRC2012_val_00042542.JPEG:n03126707 +ILSVRC2012_val_00042543.JPEG:n03481172 +ILSVRC2012_val_00042544.JPEG:n02640242 +ILSVRC2012_val_00042545.JPEG:n03376595 +ILSVRC2012_val_00042546.JPEG:n02814860 +ILSVRC2012_val_00042547.JPEG:n01498041 +ILSVRC2012_val_00042548.JPEG:n04442312 +ILSVRC2012_val_00042549.JPEG:n03776460 +ILSVRC2012_val_00042550.JPEG:n01882714 +ILSVRC2012_val_00042551.JPEG:n04485082 +ILSVRC2012_val_00042552.JPEG:n03201208 +ILSVRC2012_val_00042553.JPEG:n01978455 +ILSVRC2012_val_00042554.JPEG:n04456115 +ILSVRC2012_val_00042555.JPEG:n03467068 +ILSVRC2012_val_00042556.JPEG:n02086240 +ILSVRC2012_val_00042557.JPEG:n02256656 +ILSVRC2012_val_00042558.JPEG:n04517823 +ILSVRC2012_val_00042559.JPEG:n03291819 +ILSVRC2012_val_00042560.JPEG:n04263257 +ILSVRC2012_val_00042561.JPEG:n02106662 +ILSVRC2012_val_00042562.JPEG:n02823750 +ILSVRC2012_val_00042563.JPEG:n03527444 +ILSVRC2012_val_00042564.JPEG:n01807496 +ILSVRC2012_val_00042565.JPEG:n02112018 +ILSVRC2012_val_00042566.JPEG:n02860847 +ILSVRC2012_val_00042567.JPEG:n01980166 +ILSVRC2012_val_00042568.JPEG:n01514859 +ILSVRC2012_val_00042569.JPEG:n02879718 +ILSVRC2012_val_00042570.JPEG:n02128925 +ILSVRC2012_val_00042571.JPEG:n03944341 +ILSVRC2012_val_00042572.JPEG:n07831146 +ILSVRC2012_val_00042573.JPEG:n04049303 +ILSVRC2012_val_00042574.JPEG:n04004767 +ILSVRC2012_val_00042575.JPEG:n04254120 +ILSVRC2012_val_00042576.JPEG:n02108422 +ILSVRC2012_val_00042577.JPEG:n07871810 +ILSVRC2012_val_00042578.JPEG:n01775062 +ILSVRC2012_val_00042579.JPEG:n02808304 +ILSVRC2012_val_00042580.JPEG:n03929660 +ILSVRC2012_val_00042581.JPEG:n02667093 +ILSVRC2012_val_00042582.JPEG:n07716906 +ILSVRC2012_val_00042583.JPEG:n03697007 +ILSVRC2012_val_00042584.JPEG:n12057211 +ILSVRC2012_val_00042585.JPEG:n03196217 +ILSVRC2012_val_00042586.JPEG:n01855032 +ILSVRC2012_val_00042587.JPEG:n02097047 +ILSVRC2012_val_00042588.JPEG:n02444819 +ILSVRC2012_val_00042589.JPEG:n07711569 +ILSVRC2012_val_00042590.JPEG:n02071294 +ILSVRC2012_val_00042591.JPEG:n06596364 +ILSVRC2012_val_00042592.JPEG:n03584829 +ILSVRC2012_val_00042593.JPEG:n02025239 +ILSVRC2012_val_00042594.JPEG:n09256479 +ILSVRC2012_val_00042595.JPEG:n02484975 +ILSVRC2012_val_00042596.JPEG:n02840245 +ILSVRC2012_val_00042597.JPEG:n02814533 +ILSVRC2012_val_00042598.JPEG:n03188531 +ILSVRC2012_val_00042599.JPEG:n03891332 +ILSVRC2012_val_00042600.JPEG:n01560419 +ILSVRC2012_val_00042601.JPEG:n02110185 +ILSVRC2012_val_00042602.JPEG:n01685808 +ILSVRC2012_val_00042603.JPEG:n03207941 +ILSVRC2012_val_00042604.JPEG:n02096294 +ILSVRC2012_val_00042605.JPEG:n02672831 +ILSVRC2012_val_00042606.JPEG:n04311004 +ILSVRC2012_val_00042607.JPEG:n04265275 +ILSVRC2012_val_00042608.JPEG:n07730033 +ILSVRC2012_val_00042609.JPEG:n04296562 +ILSVRC2012_val_00042610.JPEG:n02167151 +ILSVRC2012_val_00042611.JPEG:n02110341 +ILSVRC2012_val_00042612.JPEG:n03832673 +ILSVRC2012_val_00042613.JPEG:n03709823 +ILSVRC2012_val_00042614.JPEG:n02115641 +ILSVRC2012_val_00042615.JPEG:n02510455 +ILSVRC2012_val_00042616.JPEG:n04325704 +ILSVRC2012_val_00042617.JPEG:n02129604 +ILSVRC2012_val_00042618.JPEG:n04296562 +ILSVRC2012_val_00042619.JPEG:n13037406 +ILSVRC2012_val_00042620.JPEG:n04554684 +ILSVRC2012_val_00042621.JPEG:n03706229 +ILSVRC2012_val_00042622.JPEG:n02500267 +ILSVRC2012_val_00042623.JPEG:n02101388 +ILSVRC2012_val_00042624.JPEG:n02206856 +ILSVRC2012_val_00042625.JPEG:n02111889 +ILSVRC2012_val_00042626.JPEG:n04442312 +ILSVRC2012_val_00042627.JPEG:n02102973 +ILSVRC2012_val_00042628.JPEG:n02098105 +ILSVRC2012_val_00042629.JPEG:n02906734 +ILSVRC2012_val_00042630.JPEG:n01770081 +ILSVRC2012_val_00042631.JPEG:n13054560 +ILSVRC2012_val_00042632.JPEG:n04325704 +ILSVRC2012_val_00042633.JPEG:n02909870 +ILSVRC2012_val_00042634.JPEG:n02927161 +ILSVRC2012_val_00042635.JPEG:n03976467 +ILSVRC2012_val_00042636.JPEG:n03014705 +ILSVRC2012_val_00042637.JPEG:n02483362 +ILSVRC2012_val_00042638.JPEG:n02012849 +ILSVRC2012_val_00042639.JPEG:n02321529 +ILSVRC2012_val_00042640.JPEG:n03841143 +ILSVRC2012_val_00042641.JPEG:n04389033 +ILSVRC2012_val_00042642.JPEG:n02094258 +ILSVRC2012_val_00042643.JPEG:n15075141 +ILSVRC2012_val_00042644.JPEG:n03733805 +ILSVRC2012_val_00042645.JPEG:n03958227 +ILSVRC2012_val_00042646.JPEG:n03792972 +ILSVRC2012_val_00042647.JPEG:n04542943 +ILSVRC2012_val_00042648.JPEG:n02979186 +ILSVRC2012_val_00042649.JPEG:n07614500 +ILSVRC2012_val_00042650.JPEG:n03666591 +ILSVRC2012_val_00042651.JPEG:n03929855 +ILSVRC2012_val_00042652.JPEG:n07802026 +ILSVRC2012_val_00042653.JPEG:n02974003 +ILSVRC2012_val_00042654.JPEG:n02319095 +ILSVRC2012_val_00042655.JPEG:n02804414 +ILSVRC2012_val_00042656.JPEG:n04325704 +ILSVRC2012_val_00042657.JPEG:n02109525 +ILSVRC2012_val_00042658.JPEG:n02999410 +ILSVRC2012_val_00042659.JPEG:n02120079 +ILSVRC2012_val_00042660.JPEG:n04404412 +ILSVRC2012_val_00042661.JPEG:n01871265 +ILSVRC2012_val_00042662.JPEG:n03871628 +ILSVRC2012_val_00042663.JPEG:n03337140 +ILSVRC2012_val_00042664.JPEG:n01667778 +ILSVRC2012_val_00042665.JPEG:n01819313 +ILSVRC2012_val_00042666.JPEG:n04532670 +ILSVRC2012_val_00042667.JPEG:n02319095 +ILSVRC2012_val_00042668.JPEG:n03457902 +ILSVRC2012_val_00042669.JPEG:n02978881 +ILSVRC2012_val_00042670.JPEG:n02119789 +ILSVRC2012_val_00042671.JPEG:n04026417 +ILSVRC2012_val_00042672.JPEG:n01693334 +ILSVRC2012_val_00042673.JPEG:n01744401 +ILSVRC2012_val_00042674.JPEG:n03825788 +ILSVRC2012_val_00042675.JPEG:n04273569 +ILSVRC2012_val_00042676.JPEG:n03942813 +ILSVRC2012_val_00042677.JPEG:n01984695 +ILSVRC2012_val_00042678.JPEG:n02727426 +ILSVRC2012_val_00042679.JPEG:n01820546 +ILSVRC2012_val_00042680.JPEG:n04487081 +ILSVRC2012_val_00042681.JPEG:n03956157 +ILSVRC2012_val_00042682.JPEG:n04465501 +ILSVRC2012_val_00042683.JPEG:n04579145 +ILSVRC2012_val_00042684.JPEG:n02117135 +ILSVRC2012_val_00042685.JPEG:n04447861 +ILSVRC2012_val_00042686.JPEG:n03085013 +ILSVRC2012_val_00042687.JPEG:n02134084 +ILSVRC2012_val_00042688.JPEG:n03769881 +ILSVRC2012_val_00042689.JPEG:n03717622 +ILSVRC2012_val_00042690.JPEG:n02105251 +ILSVRC2012_val_00042691.JPEG:n03761084 +ILSVRC2012_val_00042692.JPEG:n02088466 +ILSVRC2012_val_00042693.JPEG:n01872401 +ILSVRC2012_val_00042694.JPEG:n02807133 +ILSVRC2012_val_00042695.JPEG:n03775546 +ILSVRC2012_val_00042696.JPEG:n03590841 +ILSVRC2012_val_00042697.JPEG:n03617480 +ILSVRC2012_val_00042698.JPEG:n01677366 +ILSVRC2012_val_00042699.JPEG:n02119789 +ILSVRC2012_val_00042700.JPEG:n02226429 +ILSVRC2012_val_00042701.JPEG:n04409515 +ILSVRC2012_val_00042702.JPEG:n03995372 +ILSVRC2012_val_00042703.JPEG:n02013706 +ILSVRC2012_val_00042704.JPEG:n07697537 +ILSVRC2012_val_00042705.JPEG:n02025239 +ILSVRC2012_val_00042706.JPEG:n02114712 +ILSVRC2012_val_00042707.JPEG:n03394916 +ILSVRC2012_val_00042708.JPEG:n02494079 +ILSVRC2012_val_00042709.JPEG:n01968897 +ILSVRC2012_val_00042710.JPEG:n03977966 +ILSVRC2012_val_00042711.JPEG:n11879895 +ILSVRC2012_val_00042712.JPEG:n03492542 +ILSVRC2012_val_00042713.JPEG:n03843555 +ILSVRC2012_val_00042714.JPEG:n03742115 +ILSVRC2012_val_00042715.JPEG:n04208210 +ILSVRC2012_val_00042716.JPEG:n02423022 +ILSVRC2012_val_00042717.JPEG:n04515003 +ILSVRC2012_val_00042718.JPEG:n13054560 +ILSVRC2012_val_00042719.JPEG:n02483708 +ILSVRC2012_val_00042720.JPEG:n04507155 +ILSVRC2012_val_00042721.JPEG:n07717410 +ILSVRC2012_val_00042722.JPEG:n03255030 +ILSVRC2012_val_00042723.JPEG:n03133878 +ILSVRC2012_val_00042724.JPEG:n03877845 +ILSVRC2012_val_00042725.JPEG:n04344873 +ILSVRC2012_val_00042726.JPEG:n04540053 +ILSVRC2012_val_00042727.JPEG:n09399592 +ILSVRC2012_val_00042728.JPEG:n04517823 +ILSVRC2012_val_00042729.JPEG:n04086273 +ILSVRC2012_val_00042730.JPEG:n02978881 +ILSVRC2012_val_00042731.JPEG:n02115641 +ILSVRC2012_val_00042732.JPEG:n04461696 +ILSVRC2012_val_00042733.JPEG:n02102973 +ILSVRC2012_val_00042734.JPEG:n02277742 +ILSVRC2012_val_00042735.JPEG:n04399382 +ILSVRC2012_val_00042736.JPEG:n04330267 +ILSVRC2012_val_00042737.JPEG:n03661043 +ILSVRC2012_val_00042738.JPEG:n13037406 +ILSVRC2012_val_00042739.JPEG:n04604644 +ILSVRC2012_val_00042740.JPEG:n03958227 +ILSVRC2012_val_00042741.JPEG:n02397096 +ILSVRC2012_val_00042742.JPEG:n04125021 +ILSVRC2012_val_00042743.JPEG:n03445924 +ILSVRC2012_val_00042744.JPEG:n03492542 +ILSVRC2012_val_00042745.JPEG:n02092339 +ILSVRC2012_val_00042746.JPEG:n03787032 +ILSVRC2012_val_00042747.JPEG:n03791053 +ILSVRC2012_val_00042748.JPEG:n02804414 +ILSVRC2012_val_00042749.JPEG:n01753488 +ILSVRC2012_val_00042750.JPEG:n07754684 +ILSVRC2012_val_00042751.JPEG:n01496331 +ILSVRC2012_val_00042752.JPEG:n01990800 +ILSVRC2012_val_00042753.JPEG:n04356056 +ILSVRC2012_val_00042754.JPEG:n04065272 +ILSVRC2012_val_00042755.JPEG:n01756291 +ILSVRC2012_val_00042756.JPEG:n04136333 +ILSVRC2012_val_00042757.JPEG:n03662601 +ILSVRC2012_val_00042758.JPEG:n02006656 +ILSVRC2012_val_00042759.JPEG:n02326432 +ILSVRC2012_val_00042760.JPEG:n02018795 +ILSVRC2012_val_00042761.JPEG:n03777568 +ILSVRC2012_val_00042762.JPEG:n07932039 +ILSVRC2012_val_00042763.JPEG:n04265275 +ILSVRC2012_val_00042764.JPEG:n02268853 +ILSVRC2012_val_00042765.JPEG:n03649909 +ILSVRC2012_val_00042766.JPEG:n04548362 +ILSVRC2012_val_00042767.JPEG:n03538406 +ILSVRC2012_val_00042768.JPEG:n02104365 +ILSVRC2012_val_00042769.JPEG:n03062245 +ILSVRC2012_val_00042770.JPEG:n04131690 +ILSVRC2012_val_00042771.JPEG:n01955084 +ILSVRC2012_val_00042772.JPEG:n04606251 +ILSVRC2012_val_00042773.JPEG:n04037443 +ILSVRC2012_val_00042774.JPEG:n01990800 +ILSVRC2012_val_00042775.JPEG:n02892767 +ILSVRC2012_val_00042776.JPEG:n02113023 +ILSVRC2012_val_00042777.JPEG:n03873416 +ILSVRC2012_val_00042778.JPEG:n04254680 +ILSVRC2012_val_00042779.JPEG:n02444819 +ILSVRC2012_val_00042780.JPEG:n04606251 +ILSVRC2012_val_00042781.JPEG:n02091032 +ILSVRC2012_val_00042782.JPEG:n03623198 +ILSVRC2012_val_00042783.JPEG:n01693334 +ILSVRC2012_val_00042784.JPEG:n04162706 +ILSVRC2012_val_00042785.JPEG:n04476259 +ILSVRC2012_val_00042786.JPEG:n01773157 +ILSVRC2012_val_00042787.JPEG:n02510455 +ILSVRC2012_val_00042788.JPEG:n01616318 +ILSVRC2012_val_00042789.JPEG:n02782093 +ILSVRC2012_val_00042790.JPEG:n04209133 +ILSVRC2012_val_00042791.JPEG:n03777568 +ILSVRC2012_val_00042792.JPEG:n12998815 +ILSVRC2012_val_00042793.JPEG:n04417672 +ILSVRC2012_val_00042794.JPEG:n12620546 +ILSVRC2012_val_00042795.JPEG:n04517823 +ILSVRC2012_val_00042796.JPEG:n02259212 +ILSVRC2012_val_00042797.JPEG:n02727426 +ILSVRC2012_val_00042798.JPEG:n02797295 +ILSVRC2012_val_00042799.JPEG:n03062245 +ILSVRC2012_val_00042800.JPEG:n02794156 +ILSVRC2012_val_00042801.JPEG:n04347754 +ILSVRC2012_val_00042802.JPEG:n03417042 +ILSVRC2012_val_00042803.JPEG:n02123159 +ILSVRC2012_val_00042804.JPEG:n03530642 +ILSVRC2012_val_00042805.JPEG:n07715103 +ILSVRC2012_val_00042806.JPEG:n07716906 +ILSVRC2012_val_00042807.JPEG:n03874599 +ILSVRC2012_val_00042808.JPEG:n04179913 +ILSVRC2012_val_00042809.JPEG:n01877812 +ILSVRC2012_val_00042810.JPEG:n02101388 +ILSVRC2012_val_00042811.JPEG:n02233338 +ILSVRC2012_val_00042812.JPEG:n04141327 +ILSVRC2012_val_00042813.JPEG:n02666196 +ILSVRC2012_val_00042814.JPEG:n04131690 +ILSVRC2012_val_00042815.JPEG:n03032252 +ILSVRC2012_val_00042816.JPEG:n02114367 +ILSVRC2012_val_00042817.JPEG:n03045698 +ILSVRC2012_val_00042818.JPEG:n02090721 +ILSVRC2012_val_00042819.JPEG:n02815834 +ILSVRC2012_val_00042820.JPEG:n07873807 +ILSVRC2012_val_00042821.JPEG:n02965783 +ILSVRC2012_val_00042822.JPEG:n04429376 +ILSVRC2012_val_00042823.JPEG:n04604644 +ILSVRC2012_val_00042824.JPEG:n01855032 +ILSVRC2012_val_00042825.JPEG:n02018795 +ILSVRC2012_val_00042826.JPEG:n03729826 +ILSVRC2012_val_00042827.JPEG:n04404412 +ILSVRC2012_val_00042828.JPEG:n07615774 +ILSVRC2012_val_00042829.JPEG:n02013706 +ILSVRC2012_val_00042830.JPEG:n01955084 +ILSVRC2012_val_00042831.JPEG:n01774750 +ILSVRC2012_val_00042832.JPEG:n01644373 +ILSVRC2012_val_00042833.JPEG:n02096177 +ILSVRC2012_val_00042834.JPEG:n02114712 +ILSVRC2012_val_00042835.JPEG:n03891332 +ILSVRC2012_val_00042836.JPEG:n03482405 +ILSVRC2012_val_00042837.JPEG:n03916031 +ILSVRC2012_val_00042838.JPEG:n02099849 +ILSVRC2012_val_00042839.JPEG:n02480855 +ILSVRC2012_val_00042840.JPEG:n13044778 +ILSVRC2012_val_00042841.JPEG:n02226429 +ILSVRC2012_val_00042842.JPEG:n03670208 +ILSVRC2012_val_00042843.JPEG:n13133613 +ILSVRC2012_val_00042844.JPEG:n03670208 +ILSVRC2012_val_00042845.JPEG:n04125021 +ILSVRC2012_val_00042846.JPEG:n02276258 +ILSVRC2012_val_00042847.JPEG:n03131574 +ILSVRC2012_val_00042848.JPEG:n03929855 +ILSVRC2012_val_00042849.JPEG:n02687172 +ILSVRC2012_val_00042850.JPEG:n02443484 +ILSVRC2012_val_00042851.JPEG:n02101006 +ILSVRC2012_val_00042852.JPEG:n04367480 +ILSVRC2012_val_00042853.JPEG:n02109525 +ILSVRC2012_val_00042854.JPEG:n04049303 +ILSVRC2012_val_00042855.JPEG:n02096051 +ILSVRC2012_val_00042856.JPEG:n03929660 +ILSVRC2012_val_00042857.JPEG:n02776631 +ILSVRC2012_val_00042858.JPEG:n02027492 +ILSVRC2012_val_00042859.JPEG:n01795545 +ILSVRC2012_val_00042860.JPEG:n02109525 +ILSVRC2012_val_00042861.JPEG:n03584829 +ILSVRC2012_val_00042862.JPEG:n03595614 +ILSVRC2012_val_00042863.JPEG:n02992211 +ILSVRC2012_val_00042864.JPEG:n04243546 +ILSVRC2012_val_00042865.JPEG:n03404251 +ILSVRC2012_val_00042866.JPEG:n04023962 +ILSVRC2012_val_00042867.JPEG:n03085013 +ILSVRC2012_val_00042868.JPEG:n02128385 +ILSVRC2012_val_00042869.JPEG:n02111129 +ILSVRC2012_val_00042870.JPEG:n04613696 +ILSVRC2012_val_00042871.JPEG:n04152593 +ILSVRC2012_val_00042872.JPEG:n02978881 +ILSVRC2012_val_00042873.JPEG:n02909870 +ILSVRC2012_val_00042874.JPEG:n10565667 +ILSVRC2012_val_00042875.JPEG:n03467068 +ILSVRC2012_val_00042876.JPEG:n02280649 +ILSVRC2012_val_00042877.JPEG:n03763968 +ILSVRC2012_val_00042878.JPEG:n02056570 +ILSVRC2012_val_00042879.JPEG:n02504458 +ILSVRC2012_val_00042880.JPEG:n03958227 +ILSVRC2012_val_00042881.JPEG:n03874599 +ILSVRC2012_val_00042882.JPEG:n02133161 +ILSVRC2012_val_00042883.JPEG:n03871628 +ILSVRC2012_val_00042884.JPEG:n02099849 +ILSVRC2012_val_00042885.JPEG:n03179701 +ILSVRC2012_val_00042886.JPEG:n01985128 +ILSVRC2012_val_00042887.JPEG:n02112137 +ILSVRC2012_val_00042888.JPEG:n02098413 +ILSVRC2012_val_00042889.JPEG:n01945685 +ILSVRC2012_val_00042890.JPEG:n02105505 +ILSVRC2012_val_00042891.JPEG:n03796401 +ILSVRC2012_val_00042892.JPEG:n04152593 +ILSVRC2012_val_00042893.JPEG:n02410509 +ILSVRC2012_val_00042894.JPEG:n01665541 +ILSVRC2012_val_00042895.JPEG:n04147183 +ILSVRC2012_val_00042896.JPEG:n02655020 +ILSVRC2012_val_00042897.JPEG:n02233338 +ILSVRC2012_val_00042898.JPEG:n03297495 +ILSVRC2012_val_00042899.JPEG:n01776313 +ILSVRC2012_val_00042900.JPEG:n01945685 +ILSVRC2012_val_00042901.JPEG:n03710193 +ILSVRC2012_val_00042902.JPEG:n04462240 +ILSVRC2012_val_00042903.JPEG:n03956157 +ILSVRC2012_val_00042904.JPEG:n02229544 +ILSVRC2012_val_00042905.JPEG:n02782093 +ILSVRC2012_val_00042906.JPEG:n04355338 +ILSVRC2012_val_00042907.JPEG:n03000684 +ILSVRC2012_val_00042908.JPEG:n04542943 +ILSVRC2012_val_00042909.JPEG:n02111277 +ILSVRC2012_val_00042910.JPEG:n04505470 +ILSVRC2012_val_00042911.JPEG:n03196217 +ILSVRC2012_val_00042912.JPEG:n02112706 +ILSVRC2012_val_00042913.JPEG:n03590841 +ILSVRC2012_val_00042914.JPEG:n03197337 +ILSVRC2012_val_00042915.JPEG:n02526121 +ILSVRC2012_val_00042916.JPEG:n04522168 +ILSVRC2012_val_00042917.JPEG:n01877812 +ILSVRC2012_val_00042918.JPEG:n03617480 +ILSVRC2012_val_00042919.JPEG:n02870880 +ILSVRC2012_val_00042920.JPEG:n04591713 +ILSVRC2012_val_00042921.JPEG:n06359193 +ILSVRC2012_val_00042922.JPEG:n02110958 +ILSVRC2012_val_00042923.JPEG:n07892512 +ILSVRC2012_val_00042924.JPEG:n03796401 +ILSVRC2012_val_00042925.JPEG:n03047690 +ILSVRC2012_val_00042926.JPEG:n01518878 +ILSVRC2012_val_00042927.JPEG:n04263257 +ILSVRC2012_val_00042928.JPEG:n01910747 +ILSVRC2012_val_00042929.JPEG:n07753275 +ILSVRC2012_val_00042930.JPEG:n01882714 +ILSVRC2012_val_00042931.JPEG:n04033901 +ILSVRC2012_val_00042932.JPEG:n01784675 +ILSVRC2012_val_00042933.JPEG:n02489166 +ILSVRC2012_val_00042934.JPEG:n03534580 +ILSVRC2012_val_00042935.JPEG:n04447861 +ILSVRC2012_val_00042936.JPEG:n02403003 +ILSVRC2012_val_00042937.JPEG:n07717556 +ILSVRC2012_val_00042938.JPEG:n02027492 +ILSVRC2012_val_00042939.JPEG:n03710721 +ILSVRC2012_val_00042940.JPEG:n02281787 +ILSVRC2012_val_00042941.JPEG:n02807133 +ILSVRC2012_val_00042942.JPEG:n03124170 +ILSVRC2012_val_00042943.JPEG:n02396427 +ILSVRC2012_val_00042944.JPEG:n02981792 +ILSVRC2012_val_00042945.JPEG:n04613696 +ILSVRC2012_val_00042946.JPEG:n02481823 +ILSVRC2012_val_00042947.JPEG:n04522168 +ILSVRC2012_val_00042948.JPEG:n03930313 +ILSVRC2012_val_00042949.JPEG:n10565667 +ILSVRC2012_val_00042950.JPEG:n03776460 +ILSVRC2012_val_00042951.JPEG:n03180011 +ILSVRC2012_val_00042952.JPEG:n04235860 +ILSVRC2012_val_00042953.JPEG:n02397096 +ILSVRC2012_val_00042954.JPEG:n03016953 +ILSVRC2012_val_00042955.JPEG:n03838899 +ILSVRC2012_val_00042956.JPEG:n09193705 +ILSVRC2012_val_00042957.JPEG:n04404412 +ILSVRC2012_val_00042958.JPEG:n04336792 +ILSVRC2012_val_00042959.JPEG:n02978881 +ILSVRC2012_val_00042960.JPEG:n07720875 +ILSVRC2012_val_00042961.JPEG:n04286575 +ILSVRC2012_val_00042962.JPEG:n12985857 +ILSVRC2012_val_00042963.JPEG:n07613480 +ILSVRC2012_val_00042964.JPEG:n03063689 +ILSVRC2012_val_00042965.JPEG:n02206856 +ILSVRC2012_val_00042966.JPEG:n02011460 +ILSVRC2012_val_00042967.JPEG:n02769748 +ILSVRC2012_val_00042968.JPEG:n02317335 +ILSVRC2012_val_00042969.JPEG:n02749479 +ILSVRC2012_val_00042970.JPEG:n01770081 +ILSVRC2012_val_00042971.JPEG:n02422699 +ILSVRC2012_val_00042972.JPEG:n02088094 +ILSVRC2012_val_00042973.JPEG:n02906734 +ILSVRC2012_val_00042974.JPEG:n06785654 +ILSVRC2012_val_00042975.JPEG:n04152593 +ILSVRC2012_val_00042976.JPEG:n03916031 +ILSVRC2012_val_00042977.JPEG:n02113186 +ILSVRC2012_val_00042978.JPEG:n02115913 +ILSVRC2012_val_00042979.JPEG:n02791124 +ILSVRC2012_val_00042980.JPEG:n03764736 +ILSVRC2012_val_00042981.JPEG:n02356798 +ILSVRC2012_val_00042982.JPEG:n02979186 +ILSVRC2012_val_00042983.JPEG:n02749479 +ILSVRC2012_val_00042984.JPEG:n03630383 +ILSVRC2012_val_00042985.JPEG:n03259280 +ILSVRC2012_val_00042986.JPEG:n04023962 +ILSVRC2012_val_00042987.JPEG:n04026417 +ILSVRC2012_val_00042988.JPEG:n02909870 +ILSVRC2012_val_00042989.JPEG:n03404251 +ILSVRC2012_val_00042990.JPEG:n03868863 +ILSVRC2012_val_00042991.JPEG:n03495258 +ILSVRC2012_val_00042992.JPEG:n03899768 +ILSVRC2012_val_00042993.JPEG:n03733805 +ILSVRC2012_val_00042994.JPEG:n02823750 +ILSVRC2012_val_00042995.JPEG:n02086079 +ILSVRC2012_val_00042996.JPEG:n04356056 +ILSVRC2012_val_00042997.JPEG:n03196217 +ILSVRC2012_val_00042998.JPEG:n01806143 +ILSVRC2012_val_00042999.JPEG:n07718472 +ILSVRC2012_val_00043000.JPEG:n04335435 +ILSVRC2012_val_00043001.JPEG:n03937543 +ILSVRC2012_val_00043002.JPEG:n04070727 +ILSVRC2012_val_00043003.JPEG:n01631663 +ILSVRC2012_val_00043004.JPEG:n02643566 +ILSVRC2012_val_00043005.JPEG:n11879895 +ILSVRC2012_val_00043006.JPEG:n03690938 +ILSVRC2012_val_00043007.JPEG:n02093428 +ILSVRC2012_val_00043008.JPEG:n02105641 +ILSVRC2012_val_00043009.JPEG:n02091134 +ILSVRC2012_val_00043010.JPEG:n03131574 +ILSVRC2012_val_00043011.JPEG:n03485407 +ILSVRC2012_val_00043012.JPEG:n01677366 +ILSVRC2012_val_00043013.JPEG:n02099601 +ILSVRC2012_val_00043014.JPEG:n02123045 +ILSVRC2012_val_00043015.JPEG:n02443114 +ILSVRC2012_val_00043016.JPEG:n02134418 +ILSVRC2012_val_00043017.JPEG:n04370456 +ILSVRC2012_val_00043018.JPEG:n01883070 +ILSVRC2012_val_00043019.JPEG:n04141076 +ILSVRC2012_val_00043020.JPEG:n03467068 +ILSVRC2012_val_00043021.JPEG:n02105162 +ILSVRC2012_val_00043022.JPEG:n02226429 +ILSVRC2012_val_00043023.JPEG:n02397096 +ILSVRC2012_val_00043024.JPEG:n02692877 +ILSVRC2012_val_00043025.JPEG:n02447366 +ILSVRC2012_val_00043026.JPEG:n13037406 +ILSVRC2012_val_00043027.JPEG:n09332890 +ILSVRC2012_val_00043028.JPEG:n04482393 +ILSVRC2012_val_00043029.JPEG:n03877845 +ILSVRC2012_val_00043030.JPEG:n02102480 +ILSVRC2012_val_00043031.JPEG:n10565667 +ILSVRC2012_val_00043032.JPEG:n02791270 +ILSVRC2012_val_00043033.JPEG:n02669723 +ILSVRC2012_val_00043034.JPEG:n02808304 +ILSVRC2012_val_00043035.JPEG:n04548362 +ILSVRC2012_val_00043036.JPEG:n03658185 +ILSVRC2012_val_00043037.JPEG:n02489166 +ILSVRC2012_val_00043038.JPEG:n02098286 +ILSVRC2012_val_00043039.JPEG:n07615774 +ILSVRC2012_val_00043040.JPEG:n04532106 +ILSVRC2012_val_00043041.JPEG:n01807496 +ILSVRC2012_val_00043042.JPEG:n02992529 +ILSVRC2012_val_00043043.JPEG:n01694178 +ILSVRC2012_val_00043044.JPEG:n04428191 +ILSVRC2012_val_00043045.JPEG:n03445924 +ILSVRC2012_val_00043046.JPEG:n07742313 +ILSVRC2012_val_00043047.JPEG:n04037443 +ILSVRC2012_val_00043048.JPEG:n03887697 +ILSVRC2012_val_00043049.JPEG:n01630670 +ILSVRC2012_val_00043050.JPEG:n02099267 +ILSVRC2012_val_00043051.JPEG:n02123597 +ILSVRC2012_val_00043052.JPEG:n01981276 +ILSVRC2012_val_00043053.JPEG:n02825657 +ILSVRC2012_val_00043054.JPEG:n02106662 +ILSVRC2012_val_00043055.JPEG:n03657121 +ILSVRC2012_val_00043056.JPEG:n03249569 +ILSVRC2012_val_00043057.JPEG:n03218198 +ILSVRC2012_val_00043058.JPEG:n04152593 +ILSVRC2012_val_00043059.JPEG:n12985857 +ILSVRC2012_val_00043060.JPEG:n03160309 +ILSVRC2012_val_00043061.JPEG:n02939185 +ILSVRC2012_val_00043062.JPEG:n01817953 +ILSVRC2012_val_00043063.JPEG:n01773157 +ILSVRC2012_val_00043064.JPEG:n02999410 +ILSVRC2012_val_00043065.JPEG:n03482405 +ILSVRC2012_val_00043066.JPEG:n04200800 +ILSVRC2012_val_00043067.JPEG:n02488702 +ILSVRC2012_val_00043068.JPEG:n03272562 +ILSVRC2012_val_00043069.JPEG:n03992509 +ILSVRC2012_val_00043070.JPEG:n03544143 +ILSVRC2012_val_00043071.JPEG:n04141327 +ILSVRC2012_val_00043072.JPEG:n02099712 +ILSVRC2012_val_00043073.JPEG:n03016953 +ILSVRC2012_val_00043074.JPEG:n02107142 +ILSVRC2012_val_00043075.JPEG:n01751748 +ILSVRC2012_val_00043076.JPEG:n02009912 +ILSVRC2012_val_00043077.JPEG:n02087394 +ILSVRC2012_val_00043078.JPEG:n04355933 +ILSVRC2012_val_00043079.JPEG:n02117135 +ILSVRC2012_val_00043080.JPEG:n13054560 +ILSVRC2012_val_00043081.JPEG:n02006656 +ILSVRC2012_val_00043082.JPEG:n03733805 +ILSVRC2012_val_00043083.JPEG:n03710193 +ILSVRC2012_val_00043084.JPEG:n04141076 +ILSVRC2012_val_00043085.JPEG:n01608432 +ILSVRC2012_val_00043086.JPEG:n09835506 +ILSVRC2012_val_00043087.JPEG:n04398044 +ILSVRC2012_val_00043088.JPEG:n07579787 +ILSVRC2012_val_00043089.JPEG:n02099712 +ILSVRC2012_val_00043090.JPEG:n02123597 +ILSVRC2012_val_00043091.JPEG:n07836838 +ILSVRC2012_val_00043092.JPEG:n04131690 +ILSVRC2012_val_00043093.JPEG:n04090263 +ILSVRC2012_val_00043094.JPEG:n02981792 +ILSVRC2012_val_00043095.JPEG:n02018795 +ILSVRC2012_val_00043096.JPEG:n03602883 +ILSVRC2012_val_00043097.JPEG:n02074367 +ILSVRC2012_val_00043098.JPEG:n02443484 +ILSVRC2012_val_00043099.JPEG:n02871525 +ILSVRC2012_val_00043100.JPEG:n02457408 +ILSVRC2012_val_00043101.JPEG:n02799071 +ILSVRC2012_val_00043102.JPEG:n03764736 +ILSVRC2012_val_00043103.JPEG:n03804744 +ILSVRC2012_val_00043104.JPEG:n02190166 +ILSVRC2012_val_00043105.JPEG:n03769881 +ILSVRC2012_val_00043106.JPEG:n04399382 +ILSVRC2012_val_00043107.JPEG:n04553703 +ILSVRC2012_val_00043108.JPEG:n02058221 +ILSVRC2012_val_00043109.JPEG:n02981792 +ILSVRC2012_val_00043110.JPEG:n01692333 +ILSVRC2012_val_00043111.JPEG:n01631663 +ILSVRC2012_val_00043112.JPEG:n03868242 +ILSVRC2012_val_00043113.JPEG:n06785654 +ILSVRC2012_val_00043114.JPEG:n03977966 +ILSVRC2012_val_00043115.JPEG:n04423845 +ILSVRC2012_val_00043116.JPEG:n02791124 +ILSVRC2012_val_00043117.JPEG:n02128385 +ILSVRC2012_val_00043118.JPEG:n01664065 +ILSVRC2012_val_00043119.JPEG:n01756291 +ILSVRC2012_val_00043120.JPEG:n07802026 +ILSVRC2012_val_00043121.JPEG:n02979186 +ILSVRC2012_val_00043122.JPEG:n02814533 +ILSVRC2012_val_00043123.JPEG:n12768682 +ILSVRC2012_val_00043124.JPEG:n04201297 +ILSVRC2012_val_00043125.JPEG:n07742313 +ILSVRC2012_val_00043126.JPEG:n02489166 +ILSVRC2012_val_00043127.JPEG:n02120079 +ILSVRC2012_val_00043128.JPEG:n03743016 +ILSVRC2012_val_00043129.JPEG:n03482405 +ILSVRC2012_val_00043130.JPEG:n01795545 +ILSVRC2012_val_00043131.JPEG:n02108551 +ILSVRC2012_val_00043132.JPEG:n02096051 +ILSVRC2012_val_00043133.JPEG:n02951358 +ILSVRC2012_val_00043134.JPEG:n02169497 +ILSVRC2012_val_00043135.JPEG:n04532106 +ILSVRC2012_val_00043136.JPEG:n02268443 +ILSVRC2012_val_00043137.JPEG:n03676483 +ILSVRC2012_val_00043138.JPEG:n01798484 +ILSVRC2012_val_00043139.JPEG:n02113712 +ILSVRC2012_val_00043140.JPEG:n07697313 +ILSVRC2012_val_00043141.JPEG:n02112018 +ILSVRC2012_val_00043142.JPEG:n04525038 +ILSVRC2012_val_00043143.JPEG:n03982430 +ILSVRC2012_val_00043144.JPEG:n04239074 +ILSVRC2012_val_00043145.JPEG:n02123597 +ILSVRC2012_val_00043146.JPEG:n03063689 +ILSVRC2012_val_00043147.JPEG:n02091134 +ILSVRC2012_val_00043148.JPEG:n02138441 +ILSVRC2012_val_00043149.JPEG:n03255030 +ILSVRC2012_val_00043150.JPEG:n02012849 +ILSVRC2012_val_00043151.JPEG:n02879718 +ILSVRC2012_val_00043152.JPEG:n02111277 +ILSVRC2012_val_00043153.JPEG:n02088466 +ILSVRC2012_val_00043154.JPEG:n02105056 +ILSVRC2012_val_00043155.JPEG:n01776313 +ILSVRC2012_val_00043156.JPEG:n04584207 +ILSVRC2012_val_00043157.JPEG:n02095314 +ILSVRC2012_val_00043158.JPEG:n01806567 +ILSVRC2012_val_00043159.JPEG:n01770393 +ILSVRC2012_val_00043160.JPEG:n03271574 +ILSVRC2012_val_00043161.JPEG:n03599486 +ILSVRC2012_val_00043162.JPEG:n10148035 +ILSVRC2012_val_00043163.JPEG:n03627232 +ILSVRC2012_val_00043164.JPEG:n04275548 +ILSVRC2012_val_00043165.JPEG:n03063689 +ILSVRC2012_val_00043166.JPEG:n03016953 +ILSVRC2012_val_00043167.JPEG:n01990800 +ILSVRC2012_val_00043168.JPEG:n04141076 +ILSVRC2012_val_00043169.JPEG:n03131574 +ILSVRC2012_val_00043170.JPEG:n01968897 +ILSVRC2012_val_00043171.JPEG:n02093256 +ILSVRC2012_val_00043172.JPEG:n01774750 +ILSVRC2012_val_00043173.JPEG:n01855672 +ILSVRC2012_val_00043174.JPEG:n04435653 +ILSVRC2012_val_00043175.JPEG:n03127747 +ILSVRC2012_val_00043176.JPEG:n03657121 +ILSVRC2012_val_00043177.JPEG:n03529860 +ILSVRC2012_val_00043178.JPEG:n07730033 +ILSVRC2012_val_00043179.JPEG:n02837789 +ILSVRC2012_val_00043180.JPEG:n01828970 +ILSVRC2012_val_00043181.JPEG:n02002556 +ILSVRC2012_val_00043182.JPEG:n02132136 +ILSVRC2012_val_00043183.JPEG:n03873416 +ILSVRC2012_val_00043184.JPEG:n03424325 +ILSVRC2012_val_00043185.JPEG:n04259630 +ILSVRC2012_val_00043186.JPEG:n02097130 +ILSVRC2012_val_00043187.JPEG:n03272562 +ILSVRC2012_val_00043188.JPEG:n03496892 +ILSVRC2012_val_00043189.JPEG:n04525305 +ILSVRC2012_val_00043190.JPEG:n03916031 +ILSVRC2012_val_00043191.JPEG:n01644373 +ILSVRC2012_val_00043192.JPEG:n04591713 +ILSVRC2012_val_00043193.JPEG:n02504013 +ILSVRC2012_val_00043194.JPEG:n02091831 +ILSVRC2012_val_00043195.JPEG:n01847000 +ILSVRC2012_val_00043196.JPEG:n03000684 +ILSVRC2012_val_00043197.JPEG:n01770393 +ILSVRC2012_val_00043198.JPEG:n03763968 +ILSVRC2012_val_00043199.JPEG:n02093754 +ILSVRC2012_val_00043200.JPEG:n03063689 +ILSVRC2012_val_00043201.JPEG:n02085782 +ILSVRC2012_val_00043202.JPEG:n03290653 +ILSVRC2012_val_00043203.JPEG:n03777568 +ILSVRC2012_val_00043204.JPEG:n07718472 +ILSVRC2012_val_00043205.JPEG:n02090721 +ILSVRC2012_val_00043206.JPEG:n02089078 +ILSVRC2012_val_00043207.JPEG:n03792782 +ILSVRC2012_val_00043208.JPEG:n13037406 +ILSVRC2012_val_00043209.JPEG:n02111889 +ILSVRC2012_val_00043210.JPEG:n04550184 +ILSVRC2012_val_00043211.JPEG:n03063599 +ILSVRC2012_val_00043212.JPEG:n04229816 +ILSVRC2012_val_00043213.JPEG:n04238763 +ILSVRC2012_val_00043214.JPEG:n01693334 +ILSVRC2012_val_00043215.JPEG:n03743016 +ILSVRC2012_val_00043216.JPEG:n02108551 +ILSVRC2012_val_00043217.JPEG:n04604644 +ILSVRC2012_val_00043218.JPEG:n02281787 +ILSVRC2012_val_00043219.JPEG:n02119789 +ILSVRC2012_val_00043220.JPEG:n02808304 +ILSVRC2012_val_00043221.JPEG:n09332890 +ILSVRC2012_val_00043222.JPEG:n02106550 +ILSVRC2012_val_00043223.JPEG:n07802026 +ILSVRC2012_val_00043224.JPEG:n03249569 +ILSVRC2012_val_00043225.JPEG:n07836838 +ILSVRC2012_val_00043226.JPEG:n03775546 +ILSVRC2012_val_00043227.JPEG:n04204347 +ILSVRC2012_val_00043228.JPEG:n04592741 +ILSVRC2012_val_00043229.JPEG:n01498041 +ILSVRC2012_val_00043230.JPEG:n03929660 +ILSVRC2012_val_00043231.JPEG:n02077923 +ILSVRC2012_val_00043232.JPEG:n02108089 +ILSVRC2012_val_00043233.JPEG:n02094433 +ILSVRC2012_val_00043234.JPEG:n02107574 +ILSVRC2012_val_00043235.JPEG:n13133613 +ILSVRC2012_val_00043236.JPEG:n02749479 +ILSVRC2012_val_00043237.JPEG:n03249569 +ILSVRC2012_val_00043238.JPEG:n02641379 +ILSVRC2012_val_00043239.JPEG:n03804744 +ILSVRC2012_val_00043240.JPEG:n02321529 +ILSVRC2012_val_00043241.JPEG:n01797886 +ILSVRC2012_val_00043242.JPEG:n02690373 +ILSVRC2012_val_00043243.JPEG:n13054560 +ILSVRC2012_val_00043244.JPEG:n02950826 +ILSVRC2012_val_00043245.JPEG:n01737021 +ILSVRC2012_val_00043246.JPEG:n01689811 +ILSVRC2012_val_00043247.JPEG:n01664065 +ILSVRC2012_val_00043248.JPEG:n07693725 +ILSVRC2012_val_00043249.JPEG:n02342885 +ILSVRC2012_val_00043250.JPEG:n02169497 +ILSVRC2012_val_00043251.JPEG:n09288635 +ILSVRC2012_val_00043252.JPEG:n02087394 +ILSVRC2012_val_00043253.JPEG:n03376595 +ILSVRC2012_val_00043254.JPEG:n02120505 +ILSVRC2012_val_00043255.JPEG:n03938244 +ILSVRC2012_val_00043256.JPEG:n03345487 +ILSVRC2012_val_00043257.JPEG:n02500267 +ILSVRC2012_val_00043258.JPEG:n01797886 +ILSVRC2012_val_00043259.JPEG:n04443257 +ILSVRC2012_val_00043260.JPEG:n03492542 +ILSVRC2012_val_00043261.JPEG:n02094258 +ILSVRC2012_val_00043262.JPEG:n03721384 +ILSVRC2012_val_00043263.JPEG:n13044778 +ILSVRC2012_val_00043264.JPEG:n03868863 +ILSVRC2012_val_00043265.JPEG:n07711569 +ILSVRC2012_val_00043266.JPEG:n02236044 +ILSVRC2012_val_00043267.JPEG:n04081281 +ILSVRC2012_val_00043268.JPEG:n03838899 +ILSVRC2012_val_00043269.JPEG:n04596742 +ILSVRC2012_val_00043270.JPEG:n02111500 +ILSVRC2012_val_00043271.JPEG:n04251144 +ILSVRC2012_val_00043272.JPEG:n02100583 +ILSVRC2012_val_00043273.JPEG:n07714571 +ILSVRC2012_val_00043274.JPEG:n04238763 +ILSVRC2012_val_00043275.JPEG:n02105412 +ILSVRC2012_val_00043276.JPEG:n02443484 +ILSVRC2012_val_00043277.JPEG:n04019541 +ILSVRC2012_val_00043278.JPEG:n03394916 +ILSVRC2012_val_00043279.JPEG:n03776460 +ILSVRC2012_val_00043280.JPEG:n03000134 +ILSVRC2012_val_00043281.JPEG:n02109525 +ILSVRC2012_val_00043282.JPEG:n02109525 +ILSVRC2012_val_00043283.JPEG:n02870880 +ILSVRC2012_val_00043284.JPEG:n03393912 +ILSVRC2012_val_00043285.JPEG:n03197337 +ILSVRC2012_val_00043286.JPEG:n04081281 +ILSVRC2012_val_00043287.JPEG:n03763968 +ILSVRC2012_val_00043288.JPEG:n01688243 +ILSVRC2012_val_00043289.JPEG:n02110806 +ILSVRC2012_val_00043290.JPEG:n02834397 +ILSVRC2012_val_00043291.JPEG:n02939185 +ILSVRC2012_val_00043292.JPEG:n02279972 +ILSVRC2012_val_00043293.JPEG:n03888605 +ILSVRC2012_val_00043294.JPEG:n02268443 +ILSVRC2012_val_00043295.JPEG:n02988304 +ILSVRC2012_val_00043296.JPEG:n04310018 +ILSVRC2012_val_00043297.JPEG:n04285008 +ILSVRC2012_val_00043298.JPEG:n09246464 +ILSVRC2012_val_00043299.JPEG:n02389026 +ILSVRC2012_val_00043300.JPEG:n01558993 +ILSVRC2012_val_00043301.JPEG:n01955084 +ILSVRC2012_val_00043302.JPEG:n01930112 +ILSVRC2012_val_00043303.JPEG:n01644373 +ILSVRC2012_val_00043304.JPEG:n12620546 +ILSVRC2012_val_00043305.JPEG:n02093256 +ILSVRC2012_val_00043306.JPEG:n09256479 +ILSVRC2012_val_00043307.JPEG:n02002724 +ILSVRC2012_val_00043308.JPEG:n03160309 +ILSVRC2012_val_00043309.JPEG:n04204238 +ILSVRC2012_val_00043310.JPEG:n01753488 +ILSVRC2012_val_00043311.JPEG:n03393912 +ILSVRC2012_val_00043312.JPEG:n01641577 +ILSVRC2012_val_00043313.JPEG:n02100735 +ILSVRC2012_val_00043314.JPEG:n04584207 +ILSVRC2012_val_00043315.JPEG:n02100236 +ILSVRC2012_val_00043316.JPEG:n02879718 +ILSVRC2012_val_00043317.JPEG:n02988304 +ILSVRC2012_val_00043318.JPEG:n02105162 +ILSVRC2012_val_00043319.JPEG:n02110806 +ILSVRC2012_val_00043320.JPEG:n04258138 +ILSVRC2012_val_00043321.JPEG:n03590841 +ILSVRC2012_val_00043322.JPEG:n02927161 +ILSVRC2012_val_00043323.JPEG:n01498041 +ILSVRC2012_val_00043324.JPEG:n03720891 +ILSVRC2012_val_00043325.JPEG:n04515003 +ILSVRC2012_val_00043326.JPEG:n02134418 +ILSVRC2012_val_00043327.JPEG:n03014705 +ILSVRC2012_val_00043328.JPEG:n03344393 +ILSVRC2012_val_00043329.JPEG:n02783161 +ILSVRC2012_val_00043330.JPEG:n04443257 +ILSVRC2012_val_00043331.JPEG:n02492660 +ILSVRC2012_val_00043332.JPEG:n03218198 +ILSVRC2012_val_00043333.JPEG:n01755581 +ILSVRC2012_val_00043334.JPEG:n02090622 +ILSVRC2012_val_00043335.JPEG:n03179701 +ILSVRC2012_val_00043336.JPEG:n04252225 +ILSVRC2012_val_00043337.JPEG:n04417672 +ILSVRC2012_val_00043338.JPEG:n04037443 +ILSVRC2012_val_00043339.JPEG:n04065272 +ILSVRC2012_val_00043340.JPEG:n03721384 +ILSVRC2012_val_00043341.JPEG:n02089973 +ILSVRC2012_val_00043342.JPEG:n02091635 +ILSVRC2012_val_00043343.JPEG:n03804744 +ILSVRC2012_val_00043344.JPEG:n09288635 +ILSVRC2012_val_00043345.JPEG:n04613696 +ILSVRC2012_val_00043346.JPEG:n03796401 +ILSVRC2012_val_00043347.JPEG:n07714990 +ILSVRC2012_val_00043348.JPEG:n01770393 +ILSVRC2012_val_00043349.JPEG:n01742172 +ILSVRC2012_val_00043350.JPEG:n02128385 +ILSVRC2012_val_00043351.JPEG:n03492542 +ILSVRC2012_val_00043352.JPEG:n03916031 +ILSVRC2012_val_00043353.JPEG:n01883070 +ILSVRC2012_val_00043354.JPEG:n01739381 +ILSVRC2012_val_00043355.JPEG:n02980441 +ILSVRC2012_val_00043356.JPEG:n02966687 +ILSVRC2012_val_00043357.JPEG:n04486054 +ILSVRC2012_val_00043358.JPEG:n04443257 +ILSVRC2012_val_00043359.JPEG:n01984695 +ILSVRC2012_val_00043360.JPEG:n03026506 +ILSVRC2012_val_00043361.JPEG:n02808440 +ILSVRC2012_val_00043362.JPEG:n02977058 +ILSVRC2012_val_00043363.JPEG:n02114367 +ILSVRC2012_val_00043364.JPEG:n02094114 +ILSVRC2012_val_00043365.JPEG:n02326432 +ILSVRC2012_val_00043366.JPEG:n03016953 +ILSVRC2012_val_00043367.JPEG:n02106166 +ILSVRC2012_val_00043368.JPEG:n03710193 +ILSVRC2012_val_00043369.JPEG:n01644373 +ILSVRC2012_val_00043370.JPEG:n02091134 +ILSVRC2012_val_00043371.JPEG:n03259280 +ILSVRC2012_val_00043372.JPEG:n03018349 +ILSVRC2012_val_00043373.JPEG:n03791053 +ILSVRC2012_val_00043374.JPEG:n04008634 +ILSVRC2012_val_00043375.JPEG:n02095570 +ILSVRC2012_val_00043376.JPEG:n07718747 +ILSVRC2012_val_00043377.JPEG:n03376595 +ILSVRC2012_val_00043378.JPEG:n07717410 +ILSVRC2012_val_00043379.JPEG:n02894605 +ILSVRC2012_val_00043380.JPEG:n07583066 +ILSVRC2012_val_00043381.JPEG:n02281787 +ILSVRC2012_val_00043382.JPEG:n03483316 +ILSVRC2012_val_00043383.JPEG:n02105505 +ILSVRC2012_val_00043384.JPEG:n03837869 +ILSVRC2012_val_00043385.JPEG:n04591713 +ILSVRC2012_val_00043386.JPEG:n02749479 +ILSVRC2012_val_00043387.JPEG:n01514668 +ILSVRC2012_val_00043388.JPEG:n02090379 +ILSVRC2012_val_00043389.JPEG:n03424325 +ILSVRC2012_val_00043390.JPEG:n03642806 +ILSVRC2012_val_00043391.JPEG:n02089973 +ILSVRC2012_val_00043392.JPEG:n01532829 +ILSVRC2012_val_00043393.JPEG:n02105641 +ILSVRC2012_val_00043394.JPEG:n04591713 +ILSVRC2012_val_00043395.JPEG:n01819313 +ILSVRC2012_val_00043396.JPEG:n02127052 +ILSVRC2012_val_00043397.JPEG:n03124043 +ILSVRC2012_val_00043398.JPEG:n03649909 +ILSVRC2012_val_00043399.JPEG:n02113186 +ILSVRC2012_val_00043400.JPEG:n04067472 +ILSVRC2012_val_00043401.JPEG:n02114548 +ILSVRC2012_val_00043402.JPEG:n03791053 +ILSVRC2012_val_00043403.JPEG:n03792782 +ILSVRC2012_val_00043404.JPEG:n02093991 +ILSVRC2012_val_00043405.JPEG:n03530642 +ILSVRC2012_val_00043406.JPEG:n02397096 +ILSVRC2012_val_00043407.JPEG:n02281787 +ILSVRC2012_val_00043408.JPEG:n03661043 +ILSVRC2012_val_00043409.JPEG:n03495258 +ILSVRC2012_val_00043410.JPEG:n02174001 +ILSVRC2012_val_00043411.JPEG:n07880968 +ILSVRC2012_val_00043412.JPEG:n03459775 +ILSVRC2012_val_00043413.JPEG:n02100236 +ILSVRC2012_val_00043414.JPEG:n02727426 +ILSVRC2012_val_00043415.JPEG:n01820546 +ILSVRC2012_val_00043416.JPEG:n02988304 +ILSVRC2012_val_00043417.JPEG:n02112350 +ILSVRC2012_val_00043418.JPEG:n03476684 +ILSVRC2012_val_00043419.JPEG:n04238763 +ILSVRC2012_val_00043420.JPEG:n02028035 +ILSVRC2012_val_00043421.JPEG:n02120505 +ILSVRC2012_val_00043422.JPEG:n01704323 +ILSVRC2012_val_00043423.JPEG:n03047690 +ILSVRC2012_val_00043424.JPEG:n02268443 +ILSVRC2012_val_00043425.JPEG:n02443114 +ILSVRC2012_val_00043426.JPEG:n02112137 +ILSVRC2012_val_00043427.JPEG:n02879718 +ILSVRC2012_val_00043428.JPEG:n01697457 +ILSVRC2012_val_00043429.JPEG:n04264628 +ILSVRC2012_val_00043430.JPEG:n03314780 +ILSVRC2012_val_00043431.JPEG:n03649909 +ILSVRC2012_val_00043432.JPEG:n02133161 +ILSVRC2012_val_00043433.JPEG:n07730033 +ILSVRC2012_val_00043434.JPEG:n03670208 +ILSVRC2012_val_00043435.JPEG:n02835271 +ILSVRC2012_val_00043436.JPEG:n03584829 +ILSVRC2012_val_00043437.JPEG:n02326432 +ILSVRC2012_val_00043438.JPEG:n03916031 +ILSVRC2012_val_00043439.JPEG:n03485794 +ILSVRC2012_val_00043440.JPEG:n03314780 +ILSVRC2012_val_00043441.JPEG:n02342885 +ILSVRC2012_val_00043442.JPEG:n02105412 +ILSVRC2012_val_00043443.JPEG:n02321529 +ILSVRC2012_val_00043444.JPEG:n01669191 +ILSVRC2012_val_00043445.JPEG:n07742313 +ILSVRC2012_val_00043446.JPEG:n03045698 +ILSVRC2012_val_00043447.JPEG:n02510455 +ILSVRC2012_val_00043448.JPEG:n04201297 +ILSVRC2012_val_00043449.JPEG:n03710721 +ILSVRC2012_val_00043450.JPEG:n02966687 +ILSVRC2012_val_00043451.JPEG:n02094258 +ILSVRC2012_val_00043452.JPEG:n02109047 +ILSVRC2012_val_00043453.JPEG:n03376595 +ILSVRC2012_val_00043454.JPEG:n03017168 +ILSVRC2012_val_00043455.JPEG:n01924916 +ILSVRC2012_val_00043456.JPEG:n02017213 +ILSVRC2012_val_00043457.JPEG:n02086079 +ILSVRC2012_val_00043458.JPEG:n03666591 +ILSVRC2012_val_00043459.JPEG:n04465501 +ILSVRC2012_val_00043460.JPEG:n02981792 +ILSVRC2012_val_00043461.JPEG:n03832673 +ILSVRC2012_val_00043462.JPEG:n01806567 +ILSVRC2012_val_00043463.JPEG:n02793495 +ILSVRC2012_val_00043464.JPEG:n02110806 +ILSVRC2012_val_00043465.JPEG:n01833805 +ILSVRC2012_val_00043466.JPEG:n01622779 +ILSVRC2012_val_00043467.JPEG:n02493509 +ILSVRC2012_val_00043468.JPEG:n03495258 +ILSVRC2012_val_00043469.JPEG:n03485407 +ILSVRC2012_val_00043470.JPEG:n02051845 +ILSVRC2012_val_00043471.JPEG:n04141975 +ILSVRC2012_val_00043472.JPEG:n02909870 +ILSVRC2012_val_00043473.JPEG:n01698640 +ILSVRC2012_val_00043474.JPEG:n02096294 +ILSVRC2012_val_00043475.JPEG:n02009912 +ILSVRC2012_val_00043476.JPEG:n02097658 +ILSVRC2012_val_00043477.JPEG:n02018207 +ILSVRC2012_val_00043478.JPEG:n02804414 +ILSVRC2012_val_00043479.JPEG:n03095699 +ILSVRC2012_val_00043480.JPEG:n01665541 +ILSVRC2012_val_00043481.JPEG:n03532672 +ILSVRC2012_val_00043482.JPEG:n02102177 +ILSVRC2012_val_00043483.JPEG:n01806143 +ILSVRC2012_val_00043484.JPEG:n01847000 +ILSVRC2012_val_00043485.JPEG:n07693725 +ILSVRC2012_val_00043486.JPEG:n02268853 +ILSVRC2012_val_00043487.JPEG:n03530642 +ILSVRC2012_val_00043488.JPEG:n03908618 +ILSVRC2012_val_00043489.JPEG:n03781244 +ILSVRC2012_val_00043490.JPEG:n04286575 +ILSVRC2012_val_00043491.JPEG:n02111129 +ILSVRC2012_val_00043492.JPEG:n04273569 +ILSVRC2012_val_00043493.JPEG:n04590129 +ILSVRC2012_val_00043494.JPEG:n02100583 +ILSVRC2012_val_00043495.JPEG:n03916031 +ILSVRC2012_val_00043496.JPEG:n04404412 +ILSVRC2012_val_00043497.JPEG:n02708093 +ILSVRC2012_val_00043498.JPEG:n03160309 +ILSVRC2012_val_00043499.JPEG:n07579787 +ILSVRC2012_val_00043500.JPEG:n03476991 +ILSVRC2012_val_00043501.JPEG:n04204238 +ILSVRC2012_val_00043502.JPEG:n03344393 +ILSVRC2012_val_00043503.JPEG:n09193705 +ILSVRC2012_val_00043504.JPEG:n01665541 +ILSVRC2012_val_00043505.JPEG:n01968897 +ILSVRC2012_val_00043506.JPEG:n03180011 +ILSVRC2012_val_00043507.JPEG:n02948072 +ILSVRC2012_val_00043508.JPEG:n01871265 +ILSVRC2012_val_00043509.JPEG:n01843383 +ILSVRC2012_val_00043510.JPEG:n02494079 +ILSVRC2012_val_00043511.JPEG:n02105505 +ILSVRC2012_val_00043512.JPEG:n02356798 +ILSVRC2012_val_00043513.JPEG:n02769748 +ILSVRC2012_val_00043514.JPEG:n01955084 +ILSVRC2012_val_00043515.JPEG:n01990800 +ILSVRC2012_val_00043516.JPEG:n02113712 +ILSVRC2012_val_00043517.JPEG:n03976657 +ILSVRC2012_val_00043518.JPEG:n03633091 +ILSVRC2012_val_00043519.JPEG:n03937543 +ILSVRC2012_val_00043520.JPEG:n04252225 +ILSVRC2012_val_00043521.JPEG:n02442845 +ILSVRC2012_val_00043522.JPEG:n03461385 +ILSVRC2012_val_00043523.JPEG:n03014705 +ILSVRC2012_val_00043524.JPEG:n01644900 +ILSVRC2012_val_00043525.JPEG:n03924679 +ILSVRC2012_val_00043526.JPEG:n04152593 +ILSVRC2012_val_00043527.JPEG:n02974003 +ILSVRC2012_val_00043528.JPEG:n02804414 +ILSVRC2012_val_00043529.JPEG:n03290653 +ILSVRC2012_val_00043530.JPEG:n04344873 +ILSVRC2012_val_00043531.JPEG:n02326432 +ILSVRC2012_val_00043532.JPEG:n04371430 +ILSVRC2012_val_00043533.JPEG:n03485794 +ILSVRC2012_val_00043534.JPEG:n02107142 +ILSVRC2012_val_00043535.JPEG:n03483316 +ILSVRC2012_val_00043536.JPEG:n04330267 +ILSVRC2012_val_00043537.JPEG:n01883070 +ILSVRC2012_val_00043538.JPEG:n02105505 +ILSVRC2012_val_00043539.JPEG:n03062245 +ILSVRC2012_val_00043540.JPEG:n03924679 +ILSVRC2012_val_00043541.JPEG:n02326432 +ILSVRC2012_val_00043542.JPEG:n03761084 +ILSVRC2012_val_00043543.JPEG:n02104029 +ILSVRC2012_val_00043544.JPEG:n02074367 +ILSVRC2012_val_00043545.JPEG:n04023962 +ILSVRC2012_val_00043546.JPEG:n02123597 +ILSVRC2012_val_00043547.JPEG:n04264628 +ILSVRC2012_val_00043548.JPEG:n03902125 +ILSVRC2012_val_00043549.JPEG:n02077923 +ILSVRC2012_val_00043550.JPEG:n02927161 +ILSVRC2012_val_00043551.JPEG:n03272562 +ILSVRC2012_val_00043552.JPEG:n04399382 +ILSVRC2012_val_00043553.JPEG:n07875152 +ILSVRC2012_val_00043554.JPEG:n03478589 +ILSVRC2012_val_00043555.JPEG:n03680355 +ILSVRC2012_val_00043556.JPEG:n02093428 +ILSVRC2012_val_00043557.JPEG:n03903868 +ILSVRC2012_val_00043558.JPEG:n02396427 +ILSVRC2012_val_00043559.JPEG:n01753488 +ILSVRC2012_val_00043560.JPEG:n01914609 +ILSVRC2012_val_00043561.JPEG:n04487081 +ILSVRC2012_val_00043562.JPEG:n03372029 +ILSVRC2012_val_00043563.JPEG:n01753488 +ILSVRC2012_val_00043564.JPEG:n02096585 +ILSVRC2012_val_00043565.JPEG:n07747607 +ILSVRC2012_val_00043566.JPEG:n01601694 +ILSVRC2012_val_00043567.JPEG:n03146219 +ILSVRC2012_val_00043568.JPEG:n03733131 +ILSVRC2012_val_00043569.JPEG:n03124043 +ILSVRC2012_val_00043570.JPEG:n02090622 +ILSVRC2012_val_00043571.JPEG:n03063599 +ILSVRC2012_val_00043572.JPEG:n03599486 +ILSVRC2012_val_00043573.JPEG:n03976657 +ILSVRC2012_val_00043574.JPEG:n07880968 +ILSVRC2012_val_00043575.JPEG:n02086910 +ILSVRC2012_val_00043576.JPEG:n02494079 +ILSVRC2012_val_00043577.JPEG:n02100735 +ILSVRC2012_val_00043578.JPEG:n01693334 +ILSVRC2012_val_00043579.JPEG:n02966193 +ILSVRC2012_val_00043580.JPEG:n02089973 +ILSVRC2012_val_00043581.JPEG:n03866082 +ILSVRC2012_val_00043582.JPEG:n02640242 +ILSVRC2012_val_00043583.JPEG:n02094433 +ILSVRC2012_val_00043584.JPEG:n03947888 +ILSVRC2012_val_00043585.JPEG:n01592084 +ILSVRC2012_val_00043586.JPEG:n04039381 +ILSVRC2012_val_00043587.JPEG:n04263257 +ILSVRC2012_val_00043588.JPEG:n04326547 +ILSVRC2012_val_00043589.JPEG:n02841315 +ILSVRC2012_val_00043590.JPEG:n04009552 +ILSVRC2012_val_00043591.JPEG:n02099712 +ILSVRC2012_val_00043592.JPEG:n03271574 +ILSVRC2012_val_00043593.JPEG:n02701002 +ILSVRC2012_val_00043594.JPEG:n03791053 +ILSVRC2012_val_00043595.JPEG:n04252077 +ILSVRC2012_val_00043596.JPEG:n07717410 +ILSVRC2012_val_00043597.JPEG:n02027492 +ILSVRC2012_val_00043598.JPEG:n02097474 +ILSVRC2012_val_00043599.JPEG:n02113799 +ILSVRC2012_val_00043600.JPEG:n01773797 +ILSVRC2012_val_00043601.JPEG:n11939491 +ILSVRC2012_val_00043602.JPEG:n03494278 +ILSVRC2012_val_00043603.JPEG:n02971356 +ILSVRC2012_val_00043604.JPEG:n02509815 +ILSVRC2012_val_00043605.JPEG:n02107683 +ILSVRC2012_val_00043606.JPEG:n04328186 +ILSVRC2012_val_00043607.JPEG:n03998194 +ILSVRC2012_val_00043608.JPEG:n03938244 +ILSVRC2012_val_00043609.JPEG:n03721384 +ILSVRC2012_val_00043610.JPEG:n02089973 +ILSVRC2012_val_00043611.JPEG:n07684084 +ILSVRC2012_val_00043612.JPEG:n04613696 +ILSVRC2012_val_00043613.JPEG:n03476991 +ILSVRC2012_val_00043614.JPEG:n03444034 +ILSVRC2012_val_00043615.JPEG:n03272010 +ILSVRC2012_val_00043616.JPEG:n02219486 +ILSVRC2012_val_00043617.JPEG:n07613480 +ILSVRC2012_val_00043618.JPEG:n03899768 +ILSVRC2012_val_00043619.JPEG:n01770393 +ILSVRC2012_val_00043620.JPEG:n04532106 +ILSVRC2012_val_00043621.JPEG:n04264628 +ILSVRC2012_val_00043622.JPEG:n03314780 +ILSVRC2012_val_00043623.JPEG:n02422106 +ILSVRC2012_val_00043624.JPEG:n01689811 +ILSVRC2012_val_00043625.JPEG:n04154565 +ILSVRC2012_val_00043626.JPEG:n03991062 +ILSVRC2012_val_00043627.JPEG:n02088094 +ILSVRC2012_val_00043628.JPEG:n03384352 +ILSVRC2012_val_00043629.JPEG:n02088632 +ILSVRC2012_val_00043630.JPEG:n03146219 +ILSVRC2012_val_00043631.JPEG:n02017213 +ILSVRC2012_val_00043632.JPEG:n02123597 +ILSVRC2012_val_00043633.JPEG:n01806567 +ILSVRC2012_val_00043634.JPEG:n01740131 +ILSVRC2012_val_00043635.JPEG:n01829413 +ILSVRC2012_val_00043636.JPEG:n04004767 +ILSVRC2012_val_00043637.JPEG:n04355338 +ILSVRC2012_val_00043638.JPEG:n04044716 +ILSVRC2012_val_00043639.JPEG:n01735189 +ILSVRC2012_val_00043640.JPEG:n03218198 +ILSVRC2012_val_00043641.JPEG:n02108422 +ILSVRC2012_val_00043642.JPEG:n07831146 +ILSVRC2012_val_00043643.JPEG:n02110185 +ILSVRC2012_val_00043644.JPEG:n07932039 +ILSVRC2012_val_00043645.JPEG:n03658185 +ILSVRC2012_val_00043646.JPEG:n01773797 +ILSVRC2012_val_00043647.JPEG:n09288635 +ILSVRC2012_val_00043648.JPEG:n02133161 +ILSVRC2012_val_00043649.JPEG:n01820546 +ILSVRC2012_val_00043650.JPEG:n09332890 +ILSVRC2012_val_00043651.JPEG:n09468604 +ILSVRC2012_val_00043652.JPEG:n03935335 +ILSVRC2012_val_00043653.JPEG:n04562935 +ILSVRC2012_val_00043654.JPEG:n03908714 +ILSVRC2012_val_00043655.JPEG:n02167151 +ILSVRC2012_val_00043656.JPEG:n03216828 +ILSVRC2012_val_00043657.JPEG:n02497673 +ILSVRC2012_val_00043658.JPEG:n04493381 +ILSVRC2012_val_00043659.JPEG:n03452741 +ILSVRC2012_val_00043660.JPEG:n02117135 +ILSVRC2012_val_00043661.JPEG:n04131690 +ILSVRC2012_val_00043662.JPEG:n02120505 +ILSVRC2012_val_00043663.JPEG:n03743016 +ILSVRC2012_val_00043664.JPEG:n02364673 +ILSVRC2012_val_00043665.JPEG:n03980874 +ILSVRC2012_val_00043666.JPEG:n04462240 +ILSVRC2012_val_00043667.JPEG:n02804414 +ILSVRC2012_val_00043668.JPEG:n02051845 +ILSVRC2012_val_00043669.JPEG:n02808440 +ILSVRC2012_val_00043670.JPEG:n02172182 +ILSVRC2012_val_00043671.JPEG:n09428293 +ILSVRC2012_val_00043672.JPEG:n02093428 +ILSVRC2012_val_00043673.JPEG:n03220513 +ILSVRC2012_val_00043674.JPEG:n02699494 +ILSVRC2012_val_00043675.JPEG:n03803284 +ILSVRC2012_val_00043676.JPEG:n03804744 +ILSVRC2012_val_00043677.JPEG:n02514041 +ILSVRC2012_val_00043678.JPEG:n04099969 +ILSVRC2012_val_00043679.JPEG:n04296562 +ILSVRC2012_val_00043680.JPEG:n03388549 +ILSVRC2012_val_00043681.JPEG:n12998815 +ILSVRC2012_val_00043682.JPEG:n03933933 +ILSVRC2012_val_00043683.JPEG:n04208210 +ILSVRC2012_val_00043684.JPEG:n02410509 +ILSVRC2012_val_00043685.JPEG:n04482393 +ILSVRC2012_val_00043686.JPEG:n04487081 +ILSVRC2012_val_00043687.JPEG:n02486261 +ILSVRC2012_val_00043688.JPEG:n02113799 +ILSVRC2012_val_00043689.JPEG:n04228054 +ILSVRC2012_val_00043690.JPEG:n09835506 +ILSVRC2012_val_00043691.JPEG:n04067472 +ILSVRC2012_val_00043692.JPEG:n01664065 +ILSVRC2012_val_00043693.JPEG:n04428191 +ILSVRC2012_val_00043694.JPEG:n01740131 +ILSVRC2012_val_00043695.JPEG:n02493509 +ILSVRC2012_val_00043696.JPEG:n11939491 +ILSVRC2012_val_00043697.JPEG:n03042490 +ILSVRC2012_val_00043698.JPEG:n03584254 +ILSVRC2012_val_00043699.JPEG:n09468604 +ILSVRC2012_val_00043700.JPEG:n04120489 +ILSVRC2012_val_00043701.JPEG:n02483708 +ILSVRC2012_val_00043702.JPEG:n01498041 +ILSVRC2012_val_00043703.JPEG:n03786901 +ILSVRC2012_val_00043704.JPEG:n04523525 +ILSVRC2012_val_00043705.JPEG:n02165105 +ILSVRC2012_val_00043706.JPEG:n03888605 +ILSVRC2012_val_00043707.JPEG:n02115913 +ILSVRC2012_val_00043708.JPEG:n04201297 +ILSVRC2012_val_00043709.JPEG:n04501370 +ILSVRC2012_val_00043710.JPEG:n04037443 +ILSVRC2012_val_00043711.JPEG:n02172182 +ILSVRC2012_val_00043712.JPEG:n03793489 +ILSVRC2012_val_00043713.JPEG:n03724870 +ILSVRC2012_val_00043714.JPEG:n02391049 +ILSVRC2012_val_00043715.JPEG:n04069434 +ILSVRC2012_val_00043716.JPEG:n02807133 +ILSVRC2012_val_00043717.JPEG:n02056570 +ILSVRC2012_val_00043718.JPEG:n07584110 +ILSVRC2012_val_00043719.JPEG:n04398044 +ILSVRC2012_val_00043720.JPEG:n04398044 +ILSVRC2012_val_00043721.JPEG:n03854065 +ILSVRC2012_val_00043722.JPEG:n02655020 +ILSVRC2012_val_00043723.JPEG:n02107312 +ILSVRC2012_val_00043724.JPEG:n04366367 +ILSVRC2012_val_00043725.JPEG:n04086273 +ILSVRC2012_val_00043726.JPEG:n03485407 +ILSVRC2012_val_00043727.JPEG:n02104029 +ILSVRC2012_val_00043728.JPEG:n04251144 +ILSVRC2012_val_00043729.JPEG:n03627232 +ILSVRC2012_val_00043730.JPEG:n02132136 +ILSVRC2012_val_00043731.JPEG:n02979186 +ILSVRC2012_val_00043732.JPEG:n02317335 +ILSVRC2012_val_00043733.JPEG:n03201208 +ILSVRC2012_val_00043734.JPEG:n04479046 +ILSVRC2012_val_00043735.JPEG:n03452741 +ILSVRC2012_val_00043736.JPEG:n04258138 +ILSVRC2012_val_00043737.JPEG:n07590611 +ILSVRC2012_val_00043738.JPEG:n04149813 +ILSVRC2012_val_00043739.JPEG:n04355933 +ILSVRC2012_val_00043740.JPEG:n03207941 +ILSVRC2012_val_00043741.JPEG:n04479046 +ILSVRC2012_val_00043742.JPEG:n02441942 +ILSVRC2012_val_00043743.JPEG:n03866082 +ILSVRC2012_val_00043744.JPEG:n07583066 +ILSVRC2012_val_00043745.JPEG:n03445777 +ILSVRC2012_val_00043746.JPEG:n03017168 +ILSVRC2012_val_00043747.JPEG:n02672831 +ILSVRC2012_val_00043748.JPEG:n04204238 +ILSVRC2012_val_00043749.JPEG:n04326547 +ILSVRC2012_val_00043750.JPEG:n02113712 +ILSVRC2012_val_00043751.JPEG:n01514668 +ILSVRC2012_val_00043752.JPEG:n02415577 +ILSVRC2012_val_00043753.JPEG:n03706229 +ILSVRC2012_val_00043754.JPEG:n02981792 +ILSVRC2012_val_00043755.JPEG:n02840245 +ILSVRC2012_val_00043756.JPEG:n04389033 +ILSVRC2012_val_00043757.JPEG:n03992509 +ILSVRC2012_val_00043758.JPEG:n02403003 +ILSVRC2012_val_00043759.JPEG:n04005630 +ILSVRC2012_val_00043760.JPEG:n03637318 +ILSVRC2012_val_00043761.JPEG:n04371430 +ILSVRC2012_val_00043762.JPEG:n04347754 +ILSVRC2012_val_00043763.JPEG:n02100583 +ILSVRC2012_val_00043764.JPEG:n01518878 +ILSVRC2012_val_00043765.JPEG:n02319095 +ILSVRC2012_val_00043766.JPEG:n02492035 +ILSVRC2012_val_00043767.JPEG:n04597913 +ILSVRC2012_val_00043768.JPEG:n02206856 +ILSVRC2012_val_00043769.JPEG:n02025239 +ILSVRC2012_val_00043770.JPEG:n04591157 +ILSVRC2012_val_00043771.JPEG:n01773549 +ILSVRC2012_val_00043772.JPEG:n04081281 +ILSVRC2012_val_00043773.JPEG:n07697537 +ILSVRC2012_val_00043774.JPEG:n01682714 +ILSVRC2012_val_00043775.JPEG:n04069434 +ILSVRC2012_val_00043776.JPEG:n02085782 +ILSVRC2012_val_00043777.JPEG:n02655020 +ILSVRC2012_val_00043778.JPEG:n07714571 +ILSVRC2012_val_00043779.JPEG:n01614925 +ILSVRC2012_val_00043780.JPEG:n04008634 +ILSVRC2012_val_00043781.JPEG:n07873807 +ILSVRC2012_val_00043782.JPEG:n04131690 +ILSVRC2012_val_00043783.JPEG:n03680355 +ILSVRC2012_val_00043784.JPEG:n02422699 +ILSVRC2012_val_00043785.JPEG:n07753592 +ILSVRC2012_val_00043786.JPEG:n03840681 +ILSVRC2012_val_00043787.JPEG:n06785654 +ILSVRC2012_val_00043788.JPEG:n01530575 +ILSVRC2012_val_00043789.JPEG:n02096051 +ILSVRC2012_val_00043790.JPEG:n03764736 +ILSVRC2012_val_00043791.JPEG:n02108089 +ILSVRC2012_val_00043792.JPEG:n04044716 +ILSVRC2012_val_00043793.JPEG:n03384352 +ILSVRC2012_val_00043794.JPEG:n01818515 +ILSVRC2012_val_00043795.JPEG:n02056570 +ILSVRC2012_val_00043796.JPEG:n02097130 +ILSVRC2012_val_00043797.JPEG:n01665541 +ILSVRC2012_val_00043798.JPEG:n01688243 +ILSVRC2012_val_00043799.JPEG:n04131690 +ILSVRC2012_val_00043800.JPEG:n04606251 +ILSVRC2012_val_00043801.JPEG:n01616318 +ILSVRC2012_val_00043802.JPEG:n01688243 +ILSVRC2012_val_00043803.JPEG:n02113186 +ILSVRC2012_val_00043804.JPEG:n04613696 +ILSVRC2012_val_00043805.JPEG:n01737021 +ILSVRC2012_val_00043806.JPEG:n02776631 +ILSVRC2012_val_00043807.JPEG:n03995372 +ILSVRC2012_val_00043808.JPEG:n01806143 +ILSVRC2012_val_00043809.JPEG:n01753488 +ILSVRC2012_val_00043810.JPEG:n04037443 +ILSVRC2012_val_00043811.JPEG:n02879718 +ILSVRC2012_val_00043812.JPEG:n04009552 +ILSVRC2012_val_00043813.JPEG:n02110806 +ILSVRC2012_val_00043814.JPEG:n04332243 +ILSVRC2012_val_00043815.JPEG:n04560804 +ILSVRC2012_val_00043816.JPEG:n03884397 +ILSVRC2012_val_00043817.JPEG:n02110958 +ILSVRC2012_val_00043818.JPEG:n03888605 +ILSVRC2012_val_00043819.JPEG:n01685808 +ILSVRC2012_val_00043820.JPEG:n07565083 +ILSVRC2012_val_00043821.JPEG:n02883205 +ILSVRC2012_val_00043822.JPEG:n02492660 +ILSVRC2012_val_00043823.JPEG:n01798484 +ILSVRC2012_val_00043824.JPEG:n03100240 +ILSVRC2012_val_00043825.JPEG:n02088094 +ILSVRC2012_val_00043826.JPEG:n04229816 +ILSVRC2012_val_00043827.JPEG:n02098286 +ILSVRC2012_val_00043828.JPEG:n02841315 +ILSVRC2012_val_00043829.JPEG:n03017168 +ILSVRC2012_val_00043830.JPEG:n04120489 +ILSVRC2012_val_00043831.JPEG:n07718747 +ILSVRC2012_val_00043832.JPEG:n03933933 +ILSVRC2012_val_00043833.JPEG:n04355933 +ILSVRC2012_val_00043834.JPEG:n04483307 +ILSVRC2012_val_00043835.JPEG:n02107142 +ILSVRC2012_val_00043836.JPEG:n01744401 +ILSVRC2012_val_00043837.JPEG:n02093991 +ILSVRC2012_val_00043838.JPEG:n02112137 +ILSVRC2012_val_00043839.JPEG:n02085936 +ILSVRC2012_val_00043840.JPEG:n03929855 +ILSVRC2012_val_00043841.JPEG:n02051845 +ILSVRC2012_val_00043842.JPEG:n02091831 +ILSVRC2012_val_00043843.JPEG:n01740131 +ILSVRC2012_val_00043844.JPEG:n02948072 +ILSVRC2012_val_00043845.JPEG:n02112706 +ILSVRC2012_val_00043846.JPEG:n04584207 +ILSVRC2012_val_00043847.JPEG:n04070727 +ILSVRC2012_val_00043848.JPEG:n03584254 +ILSVRC2012_val_00043849.JPEG:n04235860 +ILSVRC2012_val_00043850.JPEG:n01749939 +ILSVRC2012_val_00043851.JPEG:n02086079 +ILSVRC2012_val_00043852.JPEG:n03424325 +ILSVRC2012_val_00043853.JPEG:n04485082 +ILSVRC2012_val_00043854.JPEG:n02165456 +ILSVRC2012_val_00043855.JPEG:n03259280 +ILSVRC2012_val_00043856.JPEG:n02132136 +ILSVRC2012_val_00043857.JPEG:n03445924 +ILSVRC2012_val_00043858.JPEG:n12768682 +ILSVRC2012_val_00043859.JPEG:n03325584 +ILSVRC2012_val_00043860.JPEG:n01644373 +ILSVRC2012_val_00043861.JPEG:n02361337 +ILSVRC2012_val_00043862.JPEG:n04523525 +ILSVRC2012_val_00043863.JPEG:n07753592 +ILSVRC2012_val_00043864.JPEG:n04067472 +ILSVRC2012_val_00043865.JPEG:n04579145 +ILSVRC2012_val_00043866.JPEG:n07880968 +ILSVRC2012_val_00043867.JPEG:n02231487 +ILSVRC2012_val_00043868.JPEG:n04486054 +ILSVRC2012_val_00043869.JPEG:n03658185 +ILSVRC2012_val_00043870.JPEG:n04429376 +ILSVRC2012_val_00043871.JPEG:n03126707 +ILSVRC2012_val_00043872.JPEG:n02085620 +ILSVRC2012_val_00043873.JPEG:n02104365 +ILSVRC2012_val_00043874.JPEG:n02692877 +ILSVRC2012_val_00043875.JPEG:n04557648 +ILSVRC2012_val_00043876.JPEG:n04606251 +ILSVRC2012_val_00043877.JPEG:n03888605 +ILSVRC2012_val_00043878.JPEG:n02105412 +ILSVRC2012_val_00043879.JPEG:n06785654 +ILSVRC2012_val_00043880.JPEG:n02101388 +ILSVRC2012_val_00043881.JPEG:n03393912 +ILSVRC2012_val_00043882.JPEG:n04370456 +ILSVRC2012_val_00043883.JPEG:n12985857 +ILSVRC2012_val_00043884.JPEG:n07871810 +ILSVRC2012_val_00043885.JPEG:n03742115 +ILSVRC2012_val_00043886.JPEG:n04238763 +ILSVRC2012_val_00043887.JPEG:n02101006 +ILSVRC2012_val_00043888.JPEG:n02090379 +ILSVRC2012_val_00043889.JPEG:n09399592 +ILSVRC2012_val_00043890.JPEG:n07930864 +ILSVRC2012_val_00043891.JPEG:n02123597 +ILSVRC2012_val_00043892.JPEG:n03494278 +ILSVRC2012_val_00043893.JPEG:n02363005 +ILSVRC2012_val_00043894.JPEG:n07892512 +ILSVRC2012_val_00043895.JPEG:n02776631 +ILSVRC2012_val_00043896.JPEG:n03785016 +ILSVRC2012_val_00043897.JPEG:n07930864 +ILSVRC2012_val_00043898.JPEG:n02123394 +ILSVRC2012_val_00043899.JPEG:n01855032 +ILSVRC2012_val_00043900.JPEG:n02883205 +ILSVRC2012_val_00043901.JPEG:n02091831 +ILSVRC2012_val_00043902.JPEG:n03868242 +ILSVRC2012_val_00043903.JPEG:n02930766 +ILSVRC2012_val_00043904.JPEG:n01945685 +ILSVRC2012_val_00043905.JPEG:n03594734 +ILSVRC2012_val_00043906.JPEG:n02493793 +ILSVRC2012_val_00043907.JPEG:n02398521 +ILSVRC2012_val_00043908.JPEG:n04501370 +ILSVRC2012_val_00043909.JPEG:n03417042 +ILSVRC2012_val_00043910.JPEG:n02815834 +ILSVRC2012_val_00043911.JPEG:n03710637 +ILSVRC2012_val_00043912.JPEG:n02100583 +ILSVRC2012_val_00043913.JPEG:n02497673 +ILSVRC2012_val_00043914.JPEG:n02894605 +ILSVRC2012_val_00043915.JPEG:n03895866 +ILSVRC2012_val_00043916.JPEG:n01756291 +ILSVRC2012_val_00043917.JPEG:n02091032 +ILSVRC2012_val_00043918.JPEG:n02120505 +ILSVRC2012_val_00043919.JPEG:n03980874 +ILSVRC2012_val_00043920.JPEG:n07745940 +ILSVRC2012_val_00043921.JPEG:n02769748 +ILSVRC2012_val_00043922.JPEG:n04208210 +ILSVRC2012_val_00043923.JPEG:n01990800 +ILSVRC2012_val_00043924.JPEG:n02397096 +ILSVRC2012_val_00043925.JPEG:n01692333 +ILSVRC2012_val_00043926.JPEG:n03814639 +ILSVRC2012_val_00043927.JPEG:n01855672 +ILSVRC2012_val_00043928.JPEG:n04154565 +ILSVRC2012_val_00043929.JPEG:n02317335 +ILSVRC2012_val_00043930.JPEG:n02815834 +ILSVRC2012_val_00043931.JPEG:n07693725 +ILSVRC2012_val_00043932.JPEG:n03720891 +ILSVRC2012_val_00043933.JPEG:n02110627 +ILSVRC2012_val_00043934.JPEG:n13037406 +ILSVRC2012_val_00043935.JPEG:n02391049 +ILSVRC2012_val_00043936.JPEG:n04131690 +ILSVRC2012_val_00043937.JPEG:n01930112 +ILSVRC2012_val_00043938.JPEG:n07760859 +ILSVRC2012_val_00043939.JPEG:n03770679 +ILSVRC2012_val_00043940.JPEG:n02111500 +ILSVRC2012_val_00043941.JPEG:n04252225 +ILSVRC2012_val_00043942.JPEG:n01877812 +ILSVRC2012_val_00043943.JPEG:n03180011 +ILSVRC2012_val_00043944.JPEG:n13044778 +ILSVRC2012_val_00043945.JPEG:n02492660 +ILSVRC2012_val_00043946.JPEG:n04273569 +ILSVRC2012_val_00043947.JPEG:n04004767 +ILSVRC2012_val_00043948.JPEG:n04238763 +ILSVRC2012_val_00043949.JPEG:n03706229 +ILSVRC2012_val_00043950.JPEG:n04357314 +ILSVRC2012_val_00043951.JPEG:n01641577 +ILSVRC2012_val_00043952.JPEG:n04311174 +ILSVRC2012_val_00043953.JPEG:n03109150 +ILSVRC2012_val_00043954.JPEG:n03866082 +ILSVRC2012_val_00043955.JPEG:n03933933 +ILSVRC2012_val_00043956.JPEG:n02412080 +ILSVRC2012_val_00043957.JPEG:n03207743 +ILSVRC2012_val_00043958.JPEG:n03218198 +ILSVRC2012_val_00043959.JPEG:n07716906 +ILSVRC2012_val_00043960.JPEG:n03218198 +ILSVRC2012_val_00043961.JPEG:n02667093 +ILSVRC2012_val_00043962.JPEG:n02799071 +ILSVRC2012_val_00043963.JPEG:n02346627 +ILSVRC2012_val_00043964.JPEG:n03874293 +ILSVRC2012_val_00043965.JPEG:n01537544 +ILSVRC2012_val_00043966.JPEG:n01728572 +ILSVRC2012_val_00043967.JPEG:n03804744 +ILSVRC2012_val_00043968.JPEG:n01855672 +ILSVRC2012_val_00043969.JPEG:n01744401 +ILSVRC2012_val_00043970.JPEG:n02747177 +ILSVRC2012_val_00043971.JPEG:n02939185 +ILSVRC2012_val_00043972.JPEG:n02676566 +ILSVRC2012_val_00043973.JPEG:n02950826 +ILSVRC2012_val_00043974.JPEG:n02097298 +ILSVRC2012_val_00043975.JPEG:n01819313 +ILSVRC2012_val_00043976.JPEG:n02276258 +ILSVRC2012_val_00043977.JPEG:n09428293 +ILSVRC2012_val_00043978.JPEG:n01682714 +ILSVRC2012_val_00043979.JPEG:n03710637 +ILSVRC2012_val_00043980.JPEG:n03920288 +ILSVRC2012_val_00043981.JPEG:n02672831 +ILSVRC2012_val_00043982.JPEG:n02447366 +ILSVRC2012_val_00043983.JPEG:n02860847 +ILSVRC2012_val_00043984.JPEG:n02412080 +ILSVRC2012_val_00043985.JPEG:n04254680 +ILSVRC2012_val_00043986.JPEG:n01692333 +ILSVRC2012_val_00043987.JPEG:n02807133 +ILSVRC2012_val_00043988.JPEG:n03394916 +ILSVRC2012_val_00043989.JPEG:n13133613 +ILSVRC2012_val_00043990.JPEG:n01806567 +ILSVRC2012_val_00043991.JPEG:n07720875 +ILSVRC2012_val_00043992.JPEG:n07836838 +ILSVRC2012_val_00043993.JPEG:n02088094 +ILSVRC2012_val_00043994.JPEG:n02102040 +ILSVRC2012_val_00043995.JPEG:n01580077 +ILSVRC2012_val_00043996.JPEG:n03775546 +ILSVRC2012_val_00043997.JPEG:n04238763 +ILSVRC2012_val_00043998.JPEG:n04118776 +ILSVRC2012_val_00043999.JPEG:n04540053 +ILSVRC2012_val_00044000.JPEG:n02096294 +ILSVRC2012_val_00044001.JPEG:n02441942 +ILSVRC2012_val_00044002.JPEG:n03781244 +ILSVRC2012_val_00044003.JPEG:n02093256 +ILSVRC2012_val_00044004.JPEG:n02988304 +ILSVRC2012_val_00044005.JPEG:n02423022 +ILSVRC2012_val_00044006.JPEG:n07871810 +ILSVRC2012_val_00044007.JPEG:n01704323 +ILSVRC2012_val_00044008.JPEG:n02132136 +ILSVRC2012_val_00044009.JPEG:n01560419 +ILSVRC2012_val_00044010.JPEG:n02206856 +ILSVRC2012_val_00044011.JPEG:n01833805 +ILSVRC2012_val_00044012.JPEG:n02980441 +ILSVRC2012_val_00044013.JPEG:n11879895 +ILSVRC2012_val_00044014.JPEG:n07875152 +ILSVRC2012_val_00044015.JPEG:n03930313 +ILSVRC2012_val_00044016.JPEG:n03042490 +ILSVRC2012_val_00044017.JPEG:n03954731 +ILSVRC2012_val_00044018.JPEG:n03933933 +ILSVRC2012_val_00044019.JPEG:n03126707 +ILSVRC2012_val_00044020.JPEG:n03461385 +ILSVRC2012_val_00044021.JPEG:n02114855 +ILSVRC2012_val_00044022.JPEG:n03929660 +ILSVRC2012_val_00044023.JPEG:n04550184 +ILSVRC2012_val_00044024.JPEG:n02783161 +ILSVRC2012_val_00044025.JPEG:n03944341 +ILSVRC2012_val_00044026.JPEG:n07693725 +ILSVRC2012_val_00044027.JPEG:n02123045 +ILSVRC2012_val_00044028.JPEG:n09288635 +ILSVRC2012_val_00044029.JPEG:n03196217 +ILSVRC2012_val_00044030.JPEG:n03297495 +ILSVRC2012_val_00044031.JPEG:n02091831 +ILSVRC2012_val_00044032.JPEG:n03670208 +ILSVRC2012_val_00044033.JPEG:n04487394 +ILSVRC2012_val_00044034.JPEG:n02105251 +ILSVRC2012_val_00044035.JPEG:n02454379 +ILSVRC2012_val_00044036.JPEG:n02099849 +ILSVRC2012_val_00044037.JPEG:n04409515 +ILSVRC2012_val_00044038.JPEG:n01592084 +ILSVRC2012_val_00044039.JPEG:n02092002 +ILSVRC2012_val_00044040.JPEG:n07590611 +ILSVRC2012_val_00044041.JPEG:n03992509 +ILSVRC2012_val_00044042.JPEG:n02412080 +ILSVRC2012_val_00044043.JPEG:n03075370 +ILSVRC2012_val_00044044.JPEG:n02447366 +ILSVRC2012_val_00044045.JPEG:n02669723 +ILSVRC2012_val_00044046.JPEG:n12985857 +ILSVRC2012_val_00044047.JPEG:n03584254 +ILSVRC2012_val_00044048.JPEG:n01753488 +ILSVRC2012_val_00044049.JPEG:n02708093 +ILSVRC2012_val_00044050.JPEG:n02497673 +ILSVRC2012_val_00044051.JPEG:n04069434 +ILSVRC2012_val_00044052.JPEG:n01484850 +ILSVRC2012_val_00044053.JPEG:n07873807 +ILSVRC2012_val_00044054.JPEG:n03492542 +ILSVRC2012_val_00044055.JPEG:n03457902 +ILSVRC2012_val_00044056.JPEG:n03670208 +ILSVRC2012_val_00044057.JPEG:n04376876 +ILSVRC2012_val_00044058.JPEG:n01697457 +ILSVRC2012_val_00044059.JPEG:n02101556 +ILSVRC2012_val_00044060.JPEG:n11879895 +ILSVRC2012_val_00044061.JPEG:n02071294 +ILSVRC2012_val_00044062.JPEG:n03710193 +ILSVRC2012_val_00044063.JPEG:n03961711 +ILSVRC2012_val_00044064.JPEG:n03930313 +ILSVRC2012_val_00044065.JPEG:n02793495 +ILSVRC2012_val_00044066.JPEG:n12768682 +ILSVRC2012_val_00044067.JPEG:n03657121 +ILSVRC2012_val_00044068.JPEG:n04596742 +ILSVRC2012_val_00044069.JPEG:n04204238 +ILSVRC2012_val_00044070.JPEG:n02093754 +ILSVRC2012_val_00044071.JPEG:n03961711 +ILSVRC2012_val_00044072.JPEG:n09472597 +ILSVRC2012_val_00044073.JPEG:n03379051 +ILSVRC2012_val_00044074.JPEG:n02417914 +ILSVRC2012_val_00044075.JPEG:n02107312 +ILSVRC2012_val_00044076.JPEG:n02489166 +ILSVRC2012_val_00044077.JPEG:n01828970 +ILSVRC2012_val_00044078.JPEG:n03884397 +ILSVRC2012_val_00044079.JPEG:n04251144 +ILSVRC2012_val_00044080.JPEG:n03792782 +ILSVRC2012_val_00044081.JPEG:n02782093 +ILSVRC2012_val_00044082.JPEG:n01820546 +ILSVRC2012_val_00044083.JPEG:n02981792 +ILSVRC2012_val_00044084.JPEG:n06359193 +ILSVRC2012_val_00044085.JPEG:n03443371 +ILSVRC2012_val_00044086.JPEG:n01735189 +ILSVRC2012_val_00044087.JPEG:n04501370 +ILSVRC2012_val_00044088.JPEG:n03673027 +ILSVRC2012_val_00044089.JPEG:n03770679 +ILSVRC2012_val_00044090.JPEG:n03085013 +ILSVRC2012_val_00044091.JPEG:n02112706 +ILSVRC2012_val_00044092.JPEG:n01978287 +ILSVRC2012_val_00044093.JPEG:n02794156 +ILSVRC2012_val_00044094.JPEG:n02087394 +ILSVRC2012_val_00044095.JPEG:n01443537 +ILSVRC2012_val_00044096.JPEG:n04286575 +ILSVRC2012_val_00044097.JPEG:n02123394 +ILSVRC2012_val_00044098.JPEG:n04264628 +ILSVRC2012_val_00044099.JPEG:n03337140 +ILSVRC2012_val_00044100.JPEG:n03710721 +ILSVRC2012_val_00044101.JPEG:n03947888 +ILSVRC2012_val_00044102.JPEG:n02514041 +ILSVRC2012_val_00044103.JPEG:n02328150 +ILSVRC2012_val_00044104.JPEG:n02110185 +ILSVRC2012_val_00044105.JPEG:n03992509 +ILSVRC2012_val_00044106.JPEG:n02965783 +ILSVRC2012_val_00044107.JPEG:n02096177 +ILSVRC2012_val_00044108.JPEG:n01824575 +ILSVRC2012_val_00044109.JPEG:n03929855 +ILSVRC2012_val_00044110.JPEG:n02815834 +ILSVRC2012_val_00044111.JPEG:n02643566 +ILSVRC2012_val_00044112.JPEG:n01744401 +ILSVRC2012_val_00044113.JPEG:n02672831 +ILSVRC2012_val_00044114.JPEG:n02447366 +ILSVRC2012_val_00044115.JPEG:n06874185 +ILSVRC2012_val_00044116.JPEG:n04325704 +ILSVRC2012_val_00044117.JPEG:n02317335 +ILSVRC2012_val_00044118.JPEG:n03126707 +ILSVRC2012_val_00044119.JPEG:n02056570 +ILSVRC2012_val_00044120.JPEG:n02457408 +ILSVRC2012_val_00044121.JPEG:n03443371 +ILSVRC2012_val_00044122.JPEG:n04125021 +ILSVRC2012_val_00044123.JPEG:n03866082 +ILSVRC2012_val_00044124.JPEG:n03127747 +ILSVRC2012_val_00044125.JPEG:n04311004 +ILSVRC2012_val_00044126.JPEG:n02134084 +ILSVRC2012_val_00044127.JPEG:n01910747 +ILSVRC2012_val_00044128.JPEG:n07716358 +ILSVRC2012_val_00044129.JPEG:n02134418 +ILSVRC2012_val_00044130.JPEG:n02071294 +ILSVRC2012_val_00044131.JPEG:n04335435 +ILSVRC2012_val_00044132.JPEG:n03594734 +ILSVRC2012_val_00044133.JPEG:n06359193 +ILSVRC2012_val_00044134.JPEG:n04336792 +ILSVRC2012_val_00044135.JPEG:n02097474 +ILSVRC2012_val_00044136.JPEG:n07717410 +ILSVRC2012_val_00044137.JPEG:n02092339 +ILSVRC2012_val_00044138.JPEG:n04376876 +ILSVRC2012_val_00044139.JPEG:n03785016 +ILSVRC2012_val_00044140.JPEG:n02087394 +ILSVRC2012_val_00044141.JPEG:n02825657 +ILSVRC2012_val_00044142.JPEG:n03208938 +ILSVRC2012_val_00044143.JPEG:n03720891 +ILSVRC2012_val_00044144.JPEG:n04366367 +ILSVRC2012_val_00044145.JPEG:n02480855 +ILSVRC2012_val_00044146.JPEG:n03124043 +ILSVRC2012_val_00044147.JPEG:n04067472 +ILSVRC2012_val_00044148.JPEG:n03180011 +ILSVRC2012_val_00044149.JPEG:n04049303 +ILSVRC2012_val_00044150.JPEG:n04243546 +ILSVRC2012_val_00044151.JPEG:n04423845 +ILSVRC2012_val_00044152.JPEG:n03127747 +ILSVRC2012_val_00044153.JPEG:n02259212 +ILSVRC2012_val_00044154.JPEG:n03697007 +ILSVRC2012_val_00044155.JPEG:n04136333 +ILSVRC2012_val_00044156.JPEG:n04590129 +ILSVRC2012_val_00044157.JPEG:n03942813 +ILSVRC2012_val_00044158.JPEG:n02268443 +ILSVRC2012_val_00044159.JPEG:n04008634 +ILSVRC2012_val_00044160.JPEG:n04254680 +ILSVRC2012_val_00044161.JPEG:n04125021 +ILSVRC2012_val_00044162.JPEG:n04040759 +ILSVRC2012_val_00044163.JPEG:n03924679 +ILSVRC2012_val_00044164.JPEG:n04485082 +ILSVRC2012_val_00044165.JPEG:n02410509 +ILSVRC2012_val_00044166.JPEG:n04259630 +ILSVRC2012_val_00044167.JPEG:n03584829 +ILSVRC2012_val_00044168.JPEG:n03196217 +ILSVRC2012_val_00044169.JPEG:n03776460 +ILSVRC2012_val_00044170.JPEG:n01774750 +ILSVRC2012_val_00044171.JPEG:n09421951 +ILSVRC2012_val_00044172.JPEG:n07802026 +ILSVRC2012_val_00044173.JPEG:n04399382 +ILSVRC2012_val_00044174.JPEG:n04536866 +ILSVRC2012_val_00044175.JPEG:n04525038 +ILSVRC2012_val_00044176.JPEG:n02091467 +ILSVRC2012_val_00044177.JPEG:n03902125 +ILSVRC2012_val_00044178.JPEG:n03544143 +ILSVRC2012_val_00044179.JPEG:n02791270 +ILSVRC2012_val_00044180.JPEG:n03888605 +ILSVRC2012_val_00044181.JPEG:n03376595 +ILSVRC2012_val_00044182.JPEG:n02397096 +ILSVRC2012_val_00044183.JPEG:n03777754 +ILSVRC2012_val_00044184.JPEG:n04592741 +ILSVRC2012_val_00044185.JPEG:n03047690 +ILSVRC2012_val_00044186.JPEG:n07693725 +ILSVRC2012_val_00044187.JPEG:n02113978 +ILSVRC2012_val_00044188.JPEG:n04398044 +ILSVRC2012_val_00044189.JPEG:n02783161 +ILSVRC2012_val_00044190.JPEG:n04596742 +ILSVRC2012_val_00044191.JPEG:n03785016 +ILSVRC2012_val_00044192.JPEG:n01582220 +ILSVRC2012_val_00044193.JPEG:n02791270 +ILSVRC2012_val_00044194.JPEG:n02791124 +ILSVRC2012_val_00044195.JPEG:n02129165 +ILSVRC2012_val_00044196.JPEG:n03404251 +ILSVRC2012_val_00044197.JPEG:n03670208 +ILSVRC2012_val_00044198.JPEG:n03903868 +ILSVRC2012_val_00044199.JPEG:n02978881 +ILSVRC2012_val_00044200.JPEG:n02094433 +ILSVRC2012_val_00044201.JPEG:n04252225 +ILSVRC2012_val_00044202.JPEG:n02096177 +ILSVRC2012_val_00044203.JPEG:n03496892 +ILSVRC2012_val_00044204.JPEG:n03000684 +ILSVRC2012_val_00044205.JPEG:n03983396 +ILSVRC2012_val_00044206.JPEG:n02111277 +ILSVRC2012_val_00044207.JPEG:n03720891 +ILSVRC2012_val_00044208.JPEG:n03782006 +ILSVRC2012_val_00044209.JPEG:n01829413 +ILSVRC2012_val_00044210.JPEG:n04153751 +ILSVRC2012_val_00044211.JPEG:n03271574 +ILSVRC2012_val_00044212.JPEG:n03538406 +ILSVRC2012_val_00044213.JPEG:n03970156 +ILSVRC2012_val_00044214.JPEG:n03924679 +ILSVRC2012_val_00044215.JPEG:n02088094 +ILSVRC2012_val_00044216.JPEG:n01806143 +ILSVRC2012_val_00044217.JPEG:n02113978 +ILSVRC2012_val_00044218.JPEG:n03207941 +ILSVRC2012_val_00044219.JPEG:n03347037 +ILSVRC2012_val_00044220.JPEG:n03633091 +ILSVRC2012_val_00044221.JPEG:n03404251 +ILSVRC2012_val_00044222.JPEG:n04579145 +ILSVRC2012_val_00044223.JPEG:n02276258 +ILSVRC2012_val_00044224.JPEG:n02086240 +ILSVRC2012_val_00044225.JPEG:n02799071 +ILSVRC2012_val_00044226.JPEG:n03871628 +ILSVRC2012_val_00044227.JPEG:n02087394 +ILSVRC2012_val_00044228.JPEG:n02264363 +ILSVRC2012_val_00044229.JPEG:n03478589 +ILSVRC2012_val_00044230.JPEG:n03788365 +ILSVRC2012_val_00044231.JPEG:n02097658 +ILSVRC2012_val_00044232.JPEG:n02093647 +ILSVRC2012_val_00044233.JPEG:n07920052 +ILSVRC2012_val_00044234.JPEG:n03788195 +ILSVRC2012_val_00044235.JPEG:n03720891 +ILSVRC2012_val_00044236.JPEG:n07717556 +ILSVRC2012_val_00044237.JPEG:n02113023 +ILSVRC2012_val_00044238.JPEG:n01855032 +ILSVRC2012_val_00044239.JPEG:n07802026 +ILSVRC2012_val_00044240.JPEG:n02037110 +ILSVRC2012_val_00044241.JPEG:n03832673 +ILSVRC2012_val_00044242.JPEG:n04350905 +ILSVRC2012_val_00044243.JPEG:n07613480 +ILSVRC2012_val_00044244.JPEG:n02814860 +ILSVRC2012_val_00044245.JPEG:n03777754 +ILSVRC2012_val_00044246.JPEG:n03218198 +ILSVRC2012_val_00044247.JPEG:n02441942 +ILSVRC2012_val_00044248.JPEG:n02115913 +ILSVRC2012_val_00044249.JPEG:n02109961 +ILSVRC2012_val_00044250.JPEG:n04347754 +ILSVRC2012_val_00044251.JPEG:n03841143 +ILSVRC2012_val_00044252.JPEG:n02786058 +ILSVRC2012_val_00044253.JPEG:n02690373 +ILSVRC2012_val_00044254.JPEG:n07697313 +ILSVRC2012_val_00044255.JPEG:n07613480 +ILSVRC2012_val_00044256.JPEG:n01873310 +ILSVRC2012_val_00044257.JPEG:n03874599 +ILSVRC2012_val_00044258.JPEG:n02113624 +ILSVRC2012_val_00044259.JPEG:n02992211 +ILSVRC2012_val_00044260.JPEG:n07871810 +ILSVRC2012_val_00044261.JPEG:n03388183 +ILSVRC2012_val_00044262.JPEG:n01644900 +ILSVRC2012_val_00044263.JPEG:n04067472 +ILSVRC2012_val_00044264.JPEG:n04039381 +ILSVRC2012_val_00044265.JPEG:n02361337 +ILSVRC2012_val_00044266.JPEG:n04039381 +ILSVRC2012_val_00044267.JPEG:n04370456 +ILSVRC2012_val_00044268.JPEG:n01843065 +ILSVRC2012_val_00044269.JPEG:n01877812 +ILSVRC2012_val_00044270.JPEG:n02488291 +ILSVRC2012_val_00044271.JPEG:n03692522 +ILSVRC2012_val_00044272.JPEG:n02669723 +ILSVRC2012_val_00044273.JPEG:n03018349 +ILSVRC2012_val_00044274.JPEG:n03207743 +ILSVRC2012_val_00044275.JPEG:n02096177 +ILSVRC2012_val_00044276.JPEG:n01514859 +ILSVRC2012_val_00044277.JPEG:n02105056 +ILSVRC2012_val_00044278.JPEG:n03495258 +ILSVRC2012_val_00044279.JPEG:n03207743 +ILSVRC2012_val_00044280.JPEG:n04523525 +ILSVRC2012_val_00044281.JPEG:n03259280 +ILSVRC2012_val_00044282.JPEG:n03127747 +ILSVRC2012_val_00044283.JPEG:n02988304 +ILSVRC2012_val_00044284.JPEG:n02096437 +ILSVRC2012_val_00044285.JPEG:n02087394 +ILSVRC2012_val_00044286.JPEG:n04370456 +ILSVRC2012_val_00044287.JPEG:n01882714 +ILSVRC2012_val_00044288.JPEG:n01644900 +ILSVRC2012_val_00044289.JPEG:n11879895 +ILSVRC2012_val_00044290.JPEG:n03814639 +ILSVRC2012_val_00044291.JPEG:n03763968 +ILSVRC2012_val_00044292.JPEG:n03788365 +ILSVRC2012_val_00044293.JPEG:n04579145 +ILSVRC2012_val_00044294.JPEG:n03837869 +ILSVRC2012_val_00044295.JPEG:n04429376 +ILSVRC2012_val_00044296.JPEG:n02219486 +ILSVRC2012_val_00044297.JPEG:n03983396 +ILSVRC2012_val_00044298.JPEG:n04591157 +ILSVRC2012_val_00044299.JPEG:n07693725 +ILSVRC2012_val_00044300.JPEG:n02281787 +ILSVRC2012_val_00044301.JPEG:n01829413 +ILSVRC2012_val_00044302.JPEG:n04606251 +ILSVRC2012_val_00044303.JPEG:n02795169 +ILSVRC2012_val_00044304.JPEG:n03467068 +ILSVRC2012_val_00044305.JPEG:n02486410 +ILSVRC2012_val_00044306.JPEG:n04505470 +ILSVRC2012_val_00044307.JPEG:n02488702 +ILSVRC2012_val_00044308.JPEG:n02108089 +ILSVRC2012_val_00044309.JPEG:n02783161 +ILSVRC2012_val_00044310.JPEG:n06596364 +ILSVRC2012_val_00044311.JPEG:n01558993 +ILSVRC2012_val_00044312.JPEG:n07871810 +ILSVRC2012_val_00044313.JPEG:n02655020 +ILSVRC2012_val_00044314.JPEG:n02256656 +ILSVRC2012_val_00044315.JPEG:n03290653 +ILSVRC2012_val_00044316.JPEG:n03131574 +ILSVRC2012_val_00044317.JPEG:n01829413 +ILSVRC2012_val_00044318.JPEG:n02930766 +ILSVRC2012_val_00044319.JPEG:n03529860 +ILSVRC2012_val_00044320.JPEG:n01871265 +ILSVRC2012_val_00044321.JPEG:n01675722 +ILSVRC2012_val_00044322.JPEG:n02840245 +ILSVRC2012_val_00044323.JPEG:n04392985 +ILSVRC2012_val_00044324.JPEG:n04286575 +ILSVRC2012_val_00044325.JPEG:n03404251 +ILSVRC2012_val_00044326.JPEG:n02823428 +ILSVRC2012_val_00044327.JPEG:n02951585 +ILSVRC2012_val_00044328.JPEG:n02077923 +ILSVRC2012_val_00044329.JPEG:n03000247 +ILSVRC2012_val_00044330.JPEG:n01843065 +ILSVRC2012_val_00044331.JPEG:n02804414 +ILSVRC2012_val_00044332.JPEG:n04525038 +ILSVRC2012_val_00044333.JPEG:n01749939 +ILSVRC2012_val_00044334.JPEG:n03095699 +ILSVRC2012_val_00044335.JPEG:n04552348 +ILSVRC2012_val_00044336.JPEG:n03532672 +ILSVRC2012_val_00044337.JPEG:n03527444 +ILSVRC2012_val_00044338.JPEG:n03947888 +ILSVRC2012_val_00044339.JPEG:n02667093 +ILSVRC2012_val_00044340.JPEG:n02346627 +ILSVRC2012_val_00044341.JPEG:n01667114 +ILSVRC2012_val_00044342.JPEG:n07749582 +ILSVRC2012_val_00044343.JPEG:n02128385 +ILSVRC2012_val_00044344.JPEG:n02093754 +ILSVRC2012_val_00044345.JPEG:n02092002 +ILSVRC2012_val_00044346.JPEG:n02782093 +ILSVRC2012_val_00044347.JPEG:n04310018 +ILSVRC2012_val_00044348.JPEG:n02104365 +ILSVRC2012_val_00044349.JPEG:n02134418 +ILSVRC2012_val_00044350.JPEG:n03769881 +ILSVRC2012_val_00044351.JPEG:n02776631 +ILSVRC2012_val_00044352.JPEG:n01984695 +ILSVRC2012_val_00044353.JPEG:n02097658 +ILSVRC2012_val_00044354.JPEG:n02095570 +ILSVRC2012_val_00044355.JPEG:n02321529 +ILSVRC2012_val_00044356.JPEG:n02108000 +ILSVRC2012_val_00044357.JPEG:n02098413 +ILSVRC2012_val_00044358.JPEG:n03623198 +ILSVRC2012_val_00044359.JPEG:n03100240 +ILSVRC2012_val_00044360.JPEG:n03109150 +ILSVRC2012_val_00044361.JPEG:n02168699 +ILSVRC2012_val_00044362.JPEG:n03017168 +ILSVRC2012_val_00044363.JPEG:n01819313 +ILSVRC2012_val_00044364.JPEG:n02117135 +ILSVRC2012_val_00044365.JPEG:n03871628 +ILSVRC2012_val_00044366.JPEG:n03924679 +ILSVRC2012_val_00044367.JPEG:n04399382 +ILSVRC2012_val_00044368.JPEG:n15075141 +ILSVRC2012_val_00044369.JPEG:n03884397 +ILSVRC2012_val_00044370.JPEG:n03425413 +ILSVRC2012_val_00044371.JPEG:n03584829 +ILSVRC2012_val_00044372.JPEG:n03976467 +ILSVRC2012_val_00044373.JPEG:n02979186 +ILSVRC2012_val_00044374.JPEG:n02124075 +ILSVRC2012_val_00044375.JPEG:n02869837 +ILSVRC2012_val_00044376.JPEG:n03998194 +ILSVRC2012_val_00044377.JPEG:n02025239 +ILSVRC2012_val_00044378.JPEG:n01558993 +ILSVRC2012_val_00044379.JPEG:n04044716 +ILSVRC2012_val_00044380.JPEG:n02107908 +ILSVRC2012_val_00044381.JPEG:n04404412 +ILSVRC2012_val_00044382.JPEG:n04266014 +ILSVRC2012_val_00044383.JPEG:n03944341 +ILSVRC2012_val_00044384.JPEG:n01751748 +ILSVRC2012_val_00044385.JPEG:n02025239 +ILSVRC2012_val_00044386.JPEG:n04040759 +ILSVRC2012_val_00044387.JPEG:n02102973 +ILSVRC2012_val_00044388.JPEG:n03930630 +ILSVRC2012_val_00044389.JPEG:n09246464 +ILSVRC2012_val_00044390.JPEG:n02174001 +ILSVRC2012_val_00044391.JPEG:n02389026 +ILSVRC2012_val_00044392.JPEG:n03764736 +ILSVRC2012_val_00044393.JPEG:n01795545 +ILSVRC2012_val_00044394.JPEG:n02790996 +ILSVRC2012_val_00044395.JPEG:n02526121 +ILSVRC2012_val_00044396.JPEG:n03133878 +ILSVRC2012_val_00044397.JPEG:n03124043 +ILSVRC2012_val_00044398.JPEG:n02979186 +ILSVRC2012_val_00044399.JPEG:n02093754 +ILSVRC2012_val_00044400.JPEG:n03598930 +ILSVRC2012_val_00044401.JPEG:n03250847 +ILSVRC2012_val_00044402.JPEG:n02134084 +ILSVRC2012_val_00044403.JPEG:n03733281 +ILSVRC2012_val_00044404.JPEG:n02226429 +ILSVRC2012_val_00044405.JPEG:n04019541 +ILSVRC2012_val_00044406.JPEG:n02105855 +ILSVRC2012_val_00044407.JPEG:n02256656 +ILSVRC2012_val_00044408.JPEG:n02787622 +ILSVRC2012_val_00044409.JPEG:n04435653 +ILSVRC2012_val_00044410.JPEG:n03599486 +ILSVRC2012_val_00044411.JPEG:n03733131 +ILSVRC2012_val_00044412.JPEG:n02325366 +ILSVRC2012_val_00044413.JPEG:n03259280 +ILSVRC2012_val_00044414.JPEG:n03028079 +ILSVRC2012_val_00044415.JPEG:n03476684 +ILSVRC2012_val_00044416.JPEG:n03133878 +ILSVRC2012_val_00044417.JPEG:n03590841 +ILSVRC2012_val_00044418.JPEG:n03197337 +ILSVRC2012_val_00044419.JPEG:n04525038 +ILSVRC2012_val_00044420.JPEG:n03494278 +ILSVRC2012_val_00044421.JPEG:n04270147 +ILSVRC2012_val_00044422.JPEG:n01860187 +ILSVRC2012_val_00044423.JPEG:n02086910 +ILSVRC2012_val_00044424.JPEG:n02457408 +ILSVRC2012_val_00044425.JPEG:n03627232 +ILSVRC2012_val_00044426.JPEG:n03133878 +ILSVRC2012_val_00044427.JPEG:n03947888 +ILSVRC2012_val_00044428.JPEG:n02823428 +ILSVRC2012_val_00044429.JPEG:n02097298 +ILSVRC2012_val_00044430.JPEG:n02108000 +ILSVRC2012_val_00044431.JPEG:n04540053 +ILSVRC2012_val_00044432.JPEG:n03141823 +ILSVRC2012_val_00044433.JPEG:n03201208 +ILSVRC2012_val_00044434.JPEG:n03476991 +ILSVRC2012_val_00044435.JPEG:n02113023 +ILSVRC2012_val_00044436.JPEG:n03777754 +ILSVRC2012_val_00044437.JPEG:n03854065 +ILSVRC2012_val_00044438.JPEG:n02415577 +ILSVRC2012_val_00044439.JPEG:n02974003 +ILSVRC2012_val_00044440.JPEG:n01820546 +ILSVRC2012_val_00044441.JPEG:n02087046 +ILSVRC2012_val_00044442.JPEG:n04149813 +ILSVRC2012_val_00044443.JPEG:n04332243 +ILSVRC2012_val_00044444.JPEG:n02090379 +ILSVRC2012_val_00044445.JPEG:n04509417 +ILSVRC2012_val_00044446.JPEG:n07760859 +ILSVRC2012_val_00044447.JPEG:n03637318 +ILSVRC2012_val_00044448.JPEG:n02672831 +ILSVRC2012_val_00044449.JPEG:n03141823 +ILSVRC2012_val_00044450.JPEG:n03538406 +ILSVRC2012_val_00044451.JPEG:n03201208 +ILSVRC2012_val_00044452.JPEG:n04286575 +ILSVRC2012_val_00044453.JPEG:n02097658 +ILSVRC2012_val_00044454.JPEG:n03873416 +ILSVRC2012_val_00044455.JPEG:n04515003 +ILSVRC2012_val_00044456.JPEG:n09193705 +ILSVRC2012_val_00044457.JPEG:n02939185 +ILSVRC2012_val_00044458.JPEG:n03933933 +ILSVRC2012_val_00044459.JPEG:n01749939 +ILSVRC2012_val_00044460.JPEG:n03483316 +ILSVRC2012_val_00044461.JPEG:n02098105 +ILSVRC2012_val_00044462.JPEG:n02107908 +ILSVRC2012_val_00044463.JPEG:n02130308 +ILSVRC2012_val_00044464.JPEG:n02105641 +ILSVRC2012_val_00044465.JPEG:n04458633 +ILSVRC2012_val_00044466.JPEG:n03692522 +ILSVRC2012_val_00044467.JPEG:n02777292 +ILSVRC2012_val_00044468.JPEG:n07565083 +ILSVRC2012_val_00044469.JPEG:n02708093 +ILSVRC2012_val_00044470.JPEG:n02783161 +ILSVRC2012_val_00044471.JPEG:n04037443 +ILSVRC2012_val_00044472.JPEG:n04259630 +ILSVRC2012_val_00044473.JPEG:n02112706 +ILSVRC2012_val_00044474.JPEG:n07802026 +ILSVRC2012_val_00044475.JPEG:n01729977 +ILSVRC2012_val_00044476.JPEG:n02168699 +ILSVRC2012_val_00044477.JPEG:n04192698 +ILSVRC2012_val_00044478.JPEG:n04209133 +ILSVRC2012_val_00044479.JPEG:n07590611 +ILSVRC2012_val_00044480.JPEG:n01729322 +ILSVRC2012_val_00044481.JPEG:n02028035 +ILSVRC2012_val_00044482.JPEG:n04579432 +ILSVRC2012_val_00044483.JPEG:n01518878 +ILSVRC2012_val_00044484.JPEG:n02443484 +ILSVRC2012_val_00044485.JPEG:n07742313 +ILSVRC2012_val_00044486.JPEG:n04376876 +ILSVRC2012_val_00044487.JPEG:n04019541 +ILSVRC2012_val_00044488.JPEG:n02791270 +ILSVRC2012_val_00044489.JPEG:n02906734 +ILSVRC2012_val_00044490.JPEG:n02264363 +ILSVRC2012_val_00044491.JPEG:n02233338 +ILSVRC2012_val_00044492.JPEG:n06874185 +ILSVRC2012_val_00044493.JPEG:n04069434 +ILSVRC2012_val_00044494.JPEG:n13044778 +ILSVRC2012_val_00044495.JPEG:n02981792 +ILSVRC2012_val_00044496.JPEG:n02117135 +ILSVRC2012_val_00044497.JPEG:n03775071 +ILSVRC2012_val_00044498.JPEG:n03249569 +ILSVRC2012_val_00044499.JPEG:n04239074 +ILSVRC2012_val_00044500.JPEG:n03868242 +ILSVRC2012_val_00044501.JPEG:n02099267 +ILSVRC2012_val_00044502.JPEG:n03467068 +ILSVRC2012_val_00044503.JPEG:n02791270 +ILSVRC2012_val_00044504.JPEG:n01632777 +ILSVRC2012_val_00044505.JPEG:n01817953 +ILSVRC2012_val_00044506.JPEG:n04325704 +ILSVRC2012_val_00044507.JPEG:n01582220 +ILSVRC2012_val_00044508.JPEG:n04081281 +ILSVRC2012_val_00044509.JPEG:n03838899 +ILSVRC2012_val_00044510.JPEG:n02865351 +ILSVRC2012_val_00044511.JPEG:n02445715 +ILSVRC2012_val_00044512.JPEG:n04009552 +ILSVRC2012_val_00044513.JPEG:n02089867 +ILSVRC2012_val_00044514.JPEG:n02256656 +ILSVRC2012_val_00044515.JPEG:n01860187 +ILSVRC2012_val_00044516.JPEG:n02815834 +ILSVRC2012_val_00044517.JPEG:n04447861 +ILSVRC2012_val_00044518.JPEG:n03786901 +ILSVRC2012_val_00044519.JPEG:n04120489 +ILSVRC2012_val_00044520.JPEG:n03584254 +ILSVRC2012_val_00044521.JPEG:n03255030 +ILSVRC2012_val_00044522.JPEG:n02006656 +ILSVRC2012_val_00044523.JPEG:n03187595 +ILSVRC2012_val_00044524.JPEG:n04152593 +ILSVRC2012_val_00044525.JPEG:n03467068 +ILSVRC2012_val_00044526.JPEG:n03942813 +ILSVRC2012_val_00044527.JPEG:n03947888 +ILSVRC2012_val_00044528.JPEG:n07831146 +ILSVRC2012_val_00044529.JPEG:n02090721 +ILSVRC2012_val_00044530.JPEG:n04532670 +ILSVRC2012_val_00044531.JPEG:n03018349 +ILSVRC2012_val_00044532.JPEG:n02093991 +ILSVRC2012_val_00044533.JPEG:n01917289 +ILSVRC2012_val_00044534.JPEG:n01729322 +ILSVRC2012_val_00044535.JPEG:n02108422 +ILSVRC2012_val_00044536.JPEG:n03197337 +ILSVRC2012_val_00044537.JPEG:n02951585 +ILSVRC2012_val_00044538.JPEG:n04263257 +ILSVRC2012_val_00044539.JPEG:n07932039 +ILSVRC2012_val_00044540.JPEG:n01537544 +ILSVRC2012_val_00044541.JPEG:n03495258 +ILSVRC2012_val_00044542.JPEG:n01755581 +ILSVRC2012_val_00044543.JPEG:n02096051 +ILSVRC2012_val_00044544.JPEG:n01737021 +ILSVRC2012_val_00044545.JPEG:n04120489 +ILSVRC2012_val_00044546.JPEG:n02111500 +ILSVRC2012_val_00044547.JPEG:n03895866 +ILSVRC2012_val_00044548.JPEG:n02106166 +ILSVRC2012_val_00044549.JPEG:n04350905 +ILSVRC2012_val_00044550.JPEG:n04081281 +ILSVRC2012_val_00044551.JPEG:n02791124 +ILSVRC2012_val_00044552.JPEG:n04501370 +ILSVRC2012_val_00044553.JPEG:n02115913 +ILSVRC2012_val_00044554.JPEG:n02088466 +ILSVRC2012_val_00044555.JPEG:n07614500 +ILSVRC2012_val_00044556.JPEG:n02410509 +ILSVRC2012_val_00044557.JPEG:n01740131 +ILSVRC2012_val_00044558.JPEG:n03483316 +ILSVRC2012_val_00044559.JPEG:n02701002 +ILSVRC2012_val_00044560.JPEG:n03792782 +ILSVRC2012_val_00044561.JPEG:n03995372 +ILSVRC2012_val_00044562.JPEG:n03016953 +ILSVRC2012_val_00044563.JPEG:n02536864 +ILSVRC2012_val_00044564.JPEG:n12144580 +ILSVRC2012_val_00044565.JPEG:n02011460 +ILSVRC2012_val_00044566.JPEG:n04355933 +ILSVRC2012_val_00044567.JPEG:n02423022 +ILSVRC2012_val_00044568.JPEG:n03658185 +ILSVRC2012_val_00044569.JPEG:n03344393 +ILSVRC2012_val_00044570.JPEG:n02096177 +ILSVRC2012_val_00044571.JPEG:n03692522 +ILSVRC2012_val_00044572.JPEG:n04423845 +ILSVRC2012_val_00044573.JPEG:n02110185 +ILSVRC2012_val_00044574.JPEG:n02177972 +ILSVRC2012_val_00044575.JPEG:n03197337 +ILSVRC2012_val_00044576.JPEG:n03924679 +ILSVRC2012_val_00044577.JPEG:n01749939 +ILSVRC2012_val_00044578.JPEG:n02229544 +ILSVRC2012_val_00044579.JPEG:n03000247 +ILSVRC2012_val_00044580.JPEG:n01744401 +ILSVRC2012_val_00044581.JPEG:n02321529 +ILSVRC2012_val_00044582.JPEG:n03874293 +ILSVRC2012_val_00044583.JPEG:n03481172 +ILSVRC2012_val_00044584.JPEG:n01872401 +ILSVRC2012_val_00044585.JPEG:n02112018 +ILSVRC2012_val_00044586.JPEG:n02492035 +ILSVRC2012_val_00044587.JPEG:n03670208 +ILSVRC2012_val_00044588.JPEG:n04372370 +ILSVRC2012_val_00044589.JPEG:n01697457 +ILSVRC2012_val_00044590.JPEG:n02788148 +ILSVRC2012_val_00044591.JPEG:n01796340 +ILSVRC2012_val_00044592.JPEG:n03272562 +ILSVRC2012_val_00044593.JPEG:n02098286 +ILSVRC2012_val_00044594.JPEG:n03781244 +ILSVRC2012_val_00044595.JPEG:n03666591 +ILSVRC2012_val_00044596.JPEG:n13037406 +ILSVRC2012_val_00044597.JPEG:n04532670 +ILSVRC2012_val_00044598.JPEG:n03394916 +ILSVRC2012_val_00044599.JPEG:n01744401 +ILSVRC2012_val_00044600.JPEG:n02114855 +ILSVRC2012_val_00044601.JPEG:n04542943 +ILSVRC2012_val_00044602.JPEG:n02860847 +ILSVRC2012_val_00044603.JPEG:n02268443 +ILSVRC2012_val_00044604.JPEG:n04254120 +ILSVRC2012_val_00044605.JPEG:n02088466 +ILSVRC2012_val_00044606.JPEG:n11939491 +ILSVRC2012_val_00044607.JPEG:n03788195 +ILSVRC2012_val_00044608.JPEG:n07860988 +ILSVRC2012_val_00044609.JPEG:n03832673 +ILSVRC2012_val_00044610.JPEG:n02134084 +ILSVRC2012_val_00044611.JPEG:n02092339 +ILSVRC2012_val_00044612.JPEG:n02797295 +ILSVRC2012_val_00044613.JPEG:n04252077 +ILSVRC2012_val_00044614.JPEG:n04591713 +ILSVRC2012_val_00044615.JPEG:n02096177 +ILSVRC2012_val_00044616.JPEG:n03134739 +ILSVRC2012_val_00044617.JPEG:n03982430 +ILSVRC2012_val_00044618.JPEG:n02107574 +ILSVRC2012_val_00044619.JPEG:n02233338 +ILSVRC2012_val_00044620.JPEG:n07697313 +ILSVRC2012_val_00044621.JPEG:n03891332 +ILSVRC2012_val_00044622.JPEG:n03325584 +ILSVRC2012_val_00044623.JPEG:n03208938 +ILSVRC2012_val_00044624.JPEG:n01518878 +ILSVRC2012_val_00044625.JPEG:n02509815 +ILSVRC2012_val_00044626.JPEG:n03710721 +ILSVRC2012_val_00044627.JPEG:n04487394 +ILSVRC2012_val_00044628.JPEG:n03014705 +ILSVRC2012_val_00044629.JPEG:n02099429 +ILSVRC2012_val_00044630.JPEG:n02834397 +ILSVRC2012_val_00044631.JPEG:n04141975 +ILSVRC2012_val_00044632.JPEG:n01978455 +ILSVRC2012_val_00044633.JPEG:n03891332 +ILSVRC2012_val_00044634.JPEG:n02870880 +ILSVRC2012_val_00044635.JPEG:n04265275 +ILSVRC2012_val_00044636.JPEG:n02497673 +ILSVRC2012_val_00044637.JPEG:n01955084 +ILSVRC2012_val_00044638.JPEG:n02963159 +ILSVRC2012_val_00044639.JPEG:n02099712 +ILSVRC2012_val_00044640.JPEG:n02793495 +ILSVRC2012_val_00044641.JPEG:n03691459 +ILSVRC2012_val_00044642.JPEG:n02085782 +ILSVRC2012_val_00044643.JPEG:n03991062 +ILSVRC2012_val_00044644.JPEG:n02088094 +ILSVRC2012_val_00044645.JPEG:n07711569 +ILSVRC2012_val_00044646.JPEG:n02346627 +ILSVRC2012_val_00044647.JPEG:n07695742 +ILSVRC2012_val_00044648.JPEG:n03218198 +ILSVRC2012_val_00044649.JPEG:n01784675 +ILSVRC2012_val_00044650.JPEG:n02799071 +ILSVRC2012_val_00044651.JPEG:n03944341 +ILSVRC2012_val_00044652.JPEG:n03179701 +ILSVRC2012_val_00044653.JPEG:n02415577 +ILSVRC2012_val_00044654.JPEG:n04370456 +ILSVRC2012_val_00044655.JPEG:n04443257 +ILSVRC2012_val_00044656.JPEG:n04254777 +ILSVRC2012_val_00044657.JPEG:n01496331 +ILSVRC2012_val_00044658.JPEG:n02699494 +ILSVRC2012_val_00044659.JPEG:n01677366 +ILSVRC2012_val_00044660.JPEG:n02514041 +ILSVRC2012_val_00044661.JPEG:n02086240 +ILSVRC2012_val_00044662.JPEG:n02107908 +ILSVRC2012_val_00044663.JPEG:n11879895 +ILSVRC2012_val_00044664.JPEG:n03770679 +ILSVRC2012_val_00044665.JPEG:n02749479 +ILSVRC2012_val_00044666.JPEG:n03803284 +ILSVRC2012_val_00044667.JPEG:n04485082 +ILSVRC2012_val_00044668.JPEG:n03201208 +ILSVRC2012_val_00044669.JPEG:n03045698 +ILSVRC2012_val_00044670.JPEG:n03944341 +ILSVRC2012_val_00044671.JPEG:n01930112 +ILSVRC2012_val_00044672.JPEG:n02113186 +ILSVRC2012_val_00044673.JPEG:n04286575 +ILSVRC2012_val_00044674.JPEG:n03706229 +ILSVRC2012_val_00044675.JPEG:n02871525 +ILSVRC2012_val_00044676.JPEG:n01774384 +ILSVRC2012_val_00044677.JPEG:n01855032 +ILSVRC2012_val_00044678.JPEG:n02109047 +ILSVRC2012_val_00044679.JPEG:n02114548 +ILSVRC2012_val_00044680.JPEG:n12998815 +ILSVRC2012_val_00044681.JPEG:n03218198 +ILSVRC2012_val_00044682.JPEG:n03216828 +ILSVRC2012_val_00044683.JPEG:n04371774 +ILSVRC2012_val_00044684.JPEG:n02114712 +ILSVRC2012_val_00044685.JPEG:n04548280 +ILSVRC2012_val_00044686.JPEG:n02276258 +ILSVRC2012_val_00044687.JPEG:n04033995 +ILSVRC2012_val_00044688.JPEG:n03393912 +ILSVRC2012_val_00044689.JPEG:n03980874 +ILSVRC2012_val_00044690.JPEG:n04389033 +ILSVRC2012_val_00044691.JPEG:n07583066 +ILSVRC2012_val_00044692.JPEG:n01704323 +ILSVRC2012_val_00044693.JPEG:n03445924 +ILSVRC2012_val_00044694.JPEG:n02018795 +ILSVRC2012_val_00044695.JPEG:n03445777 +ILSVRC2012_val_00044696.JPEG:n02098286 +ILSVRC2012_val_00044697.JPEG:n03838899 +ILSVRC2012_val_00044698.JPEG:n01689811 +ILSVRC2012_val_00044699.JPEG:n03666591 +ILSVRC2012_val_00044700.JPEG:n03000247 +ILSVRC2012_val_00044701.JPEG:n02099712 +ILSVRC2012_val_00044702.JPEG:n03483316 +ILSVRC2012_val_00044703.JPEG:n04505470 +ILSVRC2012_val_00044704.JPEG:n02490219 +ILSVRC2012_val_00044705.JPEG:n04239074 +ILSVRC2012_val_00044706.JPEG:n01531178 +ILSVRC2012_val_00044707.JPEG:n02116738 +ILSVRC2012_val_00044708.JPEG:n01950731 +ILSVRC2012_val_00044709.JPEG:n02113624 +ILSVRC2012_val_00044710.JPEG:n04204238 +ILSVRC2012_val_00044711.JPEG:n02276258 +ILSVRC2012_val_00044712.JPEG:n07715103 +ILSVRC2012_val_00044713.JPEG:n03026506 +ILSVRC2012_val_00044714.JPEG:n02108551 +ILSVRC2012_val_00044715.JPEG:n02127052 +ILSVRC2012_val_00044716.JPEG:n02088466 +ILSVRC2012_val_00044717.JPEG:n02093256 +ILSVRC2012_val_00044718.JPEG:n02102040 +ILSVRC2012_val_00044719.JPEG:n03976657 +ILSVRC2012_val_00044720.JPEG:n04532670 +ILSVRC2012_val_00044721.JPEG:n03776460 +ILSVRC2012_val_00044722.JPEG:n03220513 +ILSVRC2012_val_00044723.JPEG:n03903868 +ILSVRC2012_val_00044724.JPEG:n03792972 +ILSVRC2012_val_00044725.JPEG:n03529860 +ILSVRC2012_val_00044726.JPEG:n02009229 +ILSVRC2012_val_00044727.JPEG:n02113624 +ILSVRC2012_val_00044728.JPEG:n02447366 +ILSVRC2012_val_00044729.JPEG:n03461385 +ILSVRC2012_val_00044730.JPEG:n02102318 +ILSVRC2012_val_00044731.JPEG:n04263257 +ILSVRC2012_val_00044732.JPEG:n02114855 +ILSVRC2012_val_00044733.JPEG:n02676566 +ILSVRC2012_val_00044734.JPEG:n03425413 +ILSVRC2012_val_00044735.JPEG:n03538406 +ILSVRC2012_val_00044736.JPEG:n03666591 +ILSVRC2012_val_00044737.JPEG:n03272010 +ILSVRC2012_val_00044738.JPEG:n07768694 +ILSVRC2012_val_00044739.JPEG:n04392985 +ILSVRC2012_val_00044740.JPEG:n04330267 +ILSVRC2012_val_00044741.JPEG:n03026506 +ILSVRC2012_val_00044742.JPEG:n07730033 +ILSVRC2012_val_00044743.JPEG:n02094258 +ILSVRC2012_val_00044744.JPEG:n04515003 +ILSVRC2012_val_00044745.JPEG:n04265275 +ILSVRC2012_val_00044746.JPEG:n13044778 +ILSVRC2012_val_00044747.JPEG:n02965783 +ILSVRC2012_val_00044748.JPEG:n02120505 +ILSVRC2012_val_00044749.JPEG:n02058221 +ILSVRC2012_val_00044750.JPEG:n03314780 +ILSVRC2012_val_00044751.JPEG:n02793495 +ILSVRC2012_val_00044752.JPEG:n02708093 +ILSVRC2012_val_00044753.JPEG:n03633091 +ILSVRC2012_val_00044754.JPEG:n03014705 +ILSVRC2012_val_00044755.JPEG:n01665541 +ILSVRC2012_val_00044756.JPEG:n02526121 +ILSVRC2012_val_00044757.JPEG:n04067472 +ILSVRC2012_val_00044758.JPEG:n04428191 +ILSVRC2012_val_00044759.JPEG:n07836838 +ILSVRC2012_val_00044760.JPEG:n02177972 +ILSVRC2012_val_00044761.JPEG:n01817953 +ILSVRC2012_val_00044762.JPEG:n04296562 +ILSVRC2012_val_00044763.JPEG:n04099969 +ILSVRC2012_val_00044764.JPEG:n03956157 +ILSVRC2012_val_00044765.JPEG:n02114367 +ILSVRC2012_val_00044766.JPEG:n02091635 +ILSVRC2012_val_00044767.JPEG:n02113978 +ILSVRC2012_val_00044768.JPEG:n03838899 +ILSVRC2012_val_00044769.JPEG:n02437616 +ILSVRC2012_val_00044770.JPEG:n04370456 +ILSVRC2012_val_00044771.JPEG:n02423022 +ILSVRC2012_val_00044772.JPEG:n02112706 +ILSVRC2012_val_00044773.JPEG:n02096585 +ILSVRC2012_val_00044774.JPEG:n02497673 +ILSVRC2012_val_00044775.JPEG:n04505470 +ILSVRC2012_val_00044776.JPEG:n02098286 +ILSVRC2012_val_00044777.JPEG:n02319095 +ILSVRC2012_val_00044778.JPEG:n04560804 +ILSVRC2012_val_00044779.JPEG:n03976657 +ILSVRC2012_val_00044780.JPEG:n04330267 +ILSVRC2012_val_00044781.JPEG:n02481823 +ILSVRC2012_val_00044782.JPEG:n04532670 +ILSVRC2012_val_00044783.JPEG:n12057211 +ILSVRC2012_val_00044784.JPEG:n03584254 +ILSVRC2012_val_00044785.JPEG:n04065272 +ILSVRC2012_val_00044786.JPEG:n04596742 +ILSVRC2012_val_00044787.JPEG:n02823428 +ILSVRC2012_val_00044788.JPEG:n01494475 +ILSVRC2012_val_00044789.JPEG:n03133878 +ILSVRC2012_val_00044790.JPEG:n07579787 +ILSVRC2012_val_00044791.JPEG:n04141975 +ILSVRC2012_val_00044792.JPEG:n03794056 +ILSVRC2012_val_00044793.JPEG:n03000684 +ILSVRC2012_val_00044794.JPEG:n04067472 +ILSVRC2012_val_00044795.JPEG:n02108422 +ILSVRC2012_val_00044796.JPEG:n04254777 +ILSVRC2012_val_00044797.JPEG:n01616318 +ILSVRC2012_val_00044798.JPEG:n03814906 +ILSVRC2012_val_00044799.JPEG:n03444034 +ILSVRC2012_val_00044800.JPEG:n04277352 +ILSVRC2012_val_00044801.JPEG:n04612504 +ILSVRC2012_val_00044802.JPEG:n02917067 +ILSVRC2012_val_00044803.JPEG:n03729826 +ILSVRC2012_val_00044804.JPEG:n02095314 +ILSVRC2012_val_00044805.JPEG:n03796401 +ILSVRC2012_val_00044806.JPEG:n04486054 +ILSVRC2012_val_00044807.JPEG:n03637318 +ILSVRC2012_val_00044808.JPEG:n02786058 +ILSVRC2012_val_00044809.JPEG:n03661043 +ILSVRC2012_val_00044810.JPEG:n03400231 +ILSVRC2012_val_00044811.JPEG:n02112350 +ILSVRC2012_val_00044812.JPEG:n03980874 +ILSVRC2012_val_00044813.JPEG:n04251144 +ILSVRC2012_val_00044814.JPEG:n01978287 +ILSVRC2012_val_00044815.JPEG:n03483316 +ILSVRC2012_val_00044816.JPEG:n03633091 +ILSVRC2012_val_00044817.JPEG:n04597913 +ILSVRC2012_val_00044818.JPEG:n02093647 +ILSVRC2012_val_00044819.JPEG:n02097474 +ILSVRC2012_val_00044820.JPEG:n02097130 +ILSVRC2012_val_00044821.JPEG:n03998194 +ILSVRC2012_val_00044822.JPEG:n01689811 +ILSVRC2012_val_00044823.JPEG:n04482393 +ILSVRC2012_val_00044824.JPEG:n02231487 +ILSVRC2012_val_00044825.JPEG:n04328186 +ILSVRC2012_val_00044826.JPEG:n03188531 +ILSVRC2012_val_00044827.JPEG:n02490219 +ILSVRC2012_val_00044828.JPEG:n04579432 +ILSVRC2012_val_00044829.JPEG:n09256479 +ILSVRC2012_val_00044830.JPEG:n03770439 +ILSVRC2012_val_00044831.JPEG:n07697537 +ILSVRC2012_val_00044832.JPEG:n02389026 +ILSVRC2012_val_00044833.JPEG:n04252225 +ILSVRC2012_val_00044834.JPEG:n03594945 +ILSVRC2012_val_00044835.JPEG:n04310018 +ILSVRC2012_val_00044836.JPEG:n01978455 +ILSVRC2012_val_00044837.JPEG:n03803284 +ILSVRC2012_val_00044838.JPEG:n03063689 +ILSVRC2012_val_00044839.JPEG:n01924916 +ILSVRC2012_val_00044840.JPEG:n03240683 +ILSVRC2012_val_00044841.JPEG:n03837869 +ILSVRC2012_val_00044842.JPEG:n02114712 +ILSVRC2012_val_00044843.JPEG:n02999410 +ILSVRC2012_val_00044844.JPEG:n04371774 +ILSVRC2012_val_00044845.JPEG:n03676483 +ILSVRC2012_val_00044846.JPEG:n02091467 +ILSVRC2012_val_00044847.JPEG:n03196217 +ILSVRC2012_val_00044848.JPEG:n03347037 +ILSVRC2012_val_00044849.JPEG:n04487081 +ILSVRC2012_val_00044850.JPEG:n03888257 +ILSVRC2012_val_00044851.JPEG:n03787032 +ILSVRC2012_val_00044852.JPEG:n01631663 +ILSVRC2012_val_00044853.JPEG:n03447721 +ILSVRC2012_val_00044854.JPEG:n02086079 +ILSVRC2012_val_00044855.JPEG:n01644373 +ILSVRC2012_val_00044856.JPEG:n09468604 +ILSVRC2012_val_00044857.JPEG:n07613480 +ILSVRC2012_val_00044858.JPEG:n04356056 +ILSVRC2012_val_00044859.JPEG:n04493381 +ILSVRC2012_val_00044860.JPEG:n06785654 +ILSVRC2012_val_00044861.JPEG:n03179701 +ILSVRC2012_val_00044862.JPEG:n01675722 +ILSVRC2012_val_00044863.JPEG:n04429376 +ILSVRC2012_val_00044864.JPEG:n02966193 +ILSVRC2012_val_00044865.JPEG:n03584254 +ILSVRC2012_val_00044866.JPEG:n03673027 +ILSVRC2012_val_00044867.JPEG:n03223299 +ILSVRC2012_val_00044868.JPEG:n03443371 +ILSVRC2012_val_00044869.JPEG:n02106382 +ILSVRC2012_val_00044870.JPEG:n04125021 +ILSVRC2012_val_00044871.JPEG:n03786901 +ILSVRC2012_val_00044872.JPEG:n04467665 +ILSVRC2012_val_00044873.JPEG:n03498962 +ILSVRC2012_val_00044874.JPEG:n03662601 +ILSVRC2012_val_00044875.JPEG:n02088632 +ILSVRC2012_val_00044876.JPEG:n02510455 +ILSVRC2012_val_00044877.JPEG:n12998815 +ILSVRC2012_val_00044878.JPEG:n02747177 +ILSVRC2012_val_00044879.JPEG:n04252077 +ILSVRC2012_val_00044880.JPEG:n12267677 +ILSVRC2012_val_00044881.JPEG:n04501370 +ILSVRC2012_val_00044882.JPEG:n02113978 +ILSVRC2012_val_00044883.JPEG:n03141823 +ILSVRC2012_val_00044884.JPEG:n01817953 +ILSVRC2012_val_00044885.JPEG:n03126707 +ILSVRC2012_val_00044886.JPEG:n03110669 +ILSVRC2012_val_00044887.JPEG:n02910353 +ILSVRC2012_val_00044888.JPEG:n03417042 +ILSVRC2012_val_00044889.JPEG:n09193705 +ILSVRC2012_val_00044890.JPEG:n02102318 +ILSVRC2012_val_00044891.JPEG:n01807496 +ILSVRC2012_val_00044892.JPEG:n02268443 +ILSVRC2012_val_00044893.JPEG:n01632777 +ILSVRC2012_val_00044894.JPEG:n02814533 +ILSVRC2012_val_00044895.JPEG:n07875152 +ILSVRC2012_val_00044896.JPEG:n01484850 +ILSVRC2012_val_00044897.JPEG:n02092339 +ILSVRC2012_val_00044898.JPEG:n02791124 +ILSVRC2012_val_00044899.JPEG:n04417672 +ILSVRC2012_val_00044900.JPEG:n03160309 +ILSVRC2012_val_00044901.JPEG:n02134418 +ILSVRC2012_val_00044902.JPEG:n03483316 +ILSVRC2012_val_00044903.JPEG:n01829413 +ILSVRC2012_val_00044904.JPEG:n02095889 +ILSVRC2012_val_00044905.JPEG:n07693725 +ILSVRC2012_val_00044906.JPEG:n04579145 +ILSVRC2012_val_00044907.JPEG:n03942813 +ILSVRC2012_val_00044908.JPEG:n02091134 +ILSVRC2012_val_00044909.JPEG:n04209239 +ILSVRC2012_val_00044910.JPEG:n07584110 +ILSVRC2012_val_00044911.JPEG:n04590129 +ILSVRC2012_val_00044912.JPEG:n03873416 +ILSVRC2012_val_00044913.JPEG:n02105056 +ILSVRC2012_val_00044914.JPEG:n02488291 +ILSVRC2012_val_00044915.JPEG:n04136333 +ILSVRC2012_val_00044916.JPEG:n01855032 +ILSVRC2012_val_00044917.JPEG:n04525305 +ILSVRC2012_val_00044918.JPEG:n04039381 +ILSVRC2012_val_00044919.JPEG:n02025239 +ILSVRC2012_val_00044920.JPEG:n03476991 +ILSVRC2012_val_00044921.JPEG:n01614925 +ILSVRC2012_val_00044922.JPEG:n01735189 +ILSVRC2012_val_00044923.JPEG:n02894605 +ILSVRC2012_val_00044924.JPEG:n04505470 +ILSVRC2012_val_00044925.JPEG:n02127052 +ILSVRC2012_val_00044926.JPEG:n12267677 +ILSVRC2012_val_00044927.JPEG:n02865351 +ILSVRC2012_val_00044928.JPEG:n03481172 +ILSVRC2012_val_00044929.JPEG:n02445715 +ILSVRC2012_val_00044930.JPEG:n02892767 +ILSVRC2012_val_00044931.JPEG:n02974003 +ILSVRC2012_val_00044932.JPEG:n03249569 +ILSVRC2012_val_00044933.JPEG:n01860187 +ILSVRC2012_val_00044934.JPEG:n01687978 +ILSVRC2012_val_00044935.JPEG:n03733805 +ILSVRC2012_val_00044936.JPEG:n03445777 +ILSVRC2012_val_00044937.JPEG:n02676566 +ILSVRC2012_val_00044938.JPEG:n07734744 +ILSVRC2012_val_00044939.JPEG:n03544143 +ILSVRC2012_val_00044940.JPEG:n03676483 +ILSVRC2012_val_00044941.JPEG:n03877845 +ILSVRC2012_val_00044942.JPEG:n03372029 +ILSVRC2012_val_00044943.JPEG:n03977966 +ILSVRC2012_val_00044944.JPEG:n02090721 +ILSVRC2012_val_00044945.JPEG:n03676483 +ILSVRC2012_val_00044946.JPEG:n02655020 +ILSVRC2012_val_00044947.JPEG:n02134418 +ILSVRC2012_val_00044948.JPEG:n02364673 +ILSVRC2012_val_00044949.JPEG:n02110627 +ILSVRC2012_val_00044950.JPEG:n03527444 +ILSVRC2012_val_00044951.JPEG:n04317175 +ILSVRC2012_val_00044952.JPEG:n02280649 +ILSVRC2012_val_00044953.JPEG:n02788148 +ILSVRC2012_val_00044954.JPEG:n02119789 +ILSVRC2012_val_00044955.JPEG:n02804610 +ILSVRC2012_val_00044956.JPEG:n04435653 +ILSVRC2012_val_00044957.JPEG:n02120505 +ILSVRC2012_val_00044958.JPEG:n02802426 +ILSVRC2012_val_00044959.JPEG:n02606052 +ILSVRC2012_val_00044960.JPEG:n07717410 +ILSVRC2012_val_00044961.JPEG:n03290653 +ILSVRC2012_val_00044962.JPEG:n03017168 +ILSVRC2012_val_00044963.JPEG:n02087046 +ILSVRC2012_val_00044964.JPEG:n02093647 +ILSVRC2012_val_00044965.JPEG:n04259630 +ILSVRC2012_val_00044966.JPEG:n01819313 +ILSVRC2012_val_00044967.JPEG:n03467068 +ILSVRC2012_val_00044968.JPEG:n02113712 +ILSVRC2012_val_00044969.JPEG:n03935335 +ILSVRC2012_val_00044970.JPEG:n02927161 +ILSVRC2012_val_00044971.JPEG:n02113186 +ILSVRC2012_val_00044972.JPEG:n03673027 +ILSVRC2012_val_00044973.JPEG:n04200800 +ILSVRC2012_val_00044974.JPEG:n04192698 +ILSVRC2012_val_00044975.JPEG:n01518878 +ILSVRC2012_val_00044976.JPEG:n03417042 +ILSVRC2012_val_00044977.JPEG:n02093754 +ILSVRC2012_val_00044978.JPEG:n02088364 +ILSVRC2012_val_00044979.JPEG:n02749479 +ILSVRC2012_val_00044980.JPEG:n01688243 +ILSVRC2012_val_00044981.JPEG:n04070727 +ILSVRC2012_val_00044982.JPEG:n04604644 +ILSVRC2012_val_00044983.JPEG:n02457408 +ILSVRC2012_val_00044984.JPEG:n06874185 +ILSVRC2012_val_00044985.JPEG:n04483307 +ILSVRC2012_val_00044986.JPEG:n02422106 +ILSVRC2012_val_00044987.JPEG:n01692333 +ILSVRC2012_val_00044988.JPEG:n02834397 +ILSVRC2012_val_00044989.JPEG:n03485794 +ILSVRC2012_val_00044990.JPEG:n02219486 +ILSVRC2012_val_00044991.JPEG:n01950731 +ILSVRC2012_val_00044992.JPEG:n02028035 +ILSVRC2012_val_00044993.JPEG:n01644900 +ILSVRC2012_val_00044994.JPEG:n03125729 +ILSVRC2012_val_00044995.JPEG:n12144580 +ILSVRC2012_val_00044996.JPEG:n01682714 +ILSVRC2012_val_00044997.JPEG:n03843555 +ILSVRC2012_val_00044998.JPEG:n03602883 +ILSVRC2012_val_00044999.JPEG:n02018795 +ILSVRC2012_val_00045000.JPEG:n03447447 +ILSVRC2012_val_00045001.JPEG:n02865351 +ILSVRC2012_val_00045002.JPEG:n03223299 +ILSVRC2012_val_00045003.JPEG:n03355925 +ILSVRC2012_val_00045004.JPEG:n04592741 +ILSVRC2012_val_00045005.JPEG:n02106662 +ILSVRC2012_val_00045006.JPEG:n02033041 +ILSVRC2012_val_00045007.JPEG:n01820546 +ILSVRC2012_val_00045008.JPEG:n03761084 +ILSVRC2012_val_00045009.JPEG:n02165105 +ILSVRC2012_val_00045010.JPEG:n02397096 +ILSVRC2012_val_00045011.JPEG:n02101556 +ILSVRC2012_val_00045012.JPEG:n04328186 +ILSVRC2012_val_00045013.JPEG:n03933933 +ILSVRC2012_val_00045014.JPEG:n03355925 +ILSVRC2012_val_00045015.JPEG:n04328186 +ILSVRC2012_val_00045016.JPEG:n03950228 +ILSVRC2012_val_00045017.JPEG:n03134739 +ILSVRC2012_val_00045018.JPEG:n03535780 +ILSVRC2012_val_00045019.JPEG:n01748264 +ILSVRC2012_val_00045020.JPEG:n04330267 +ILSVRC2012_val_00045021.JPEG:n02699494 +ILSVRC2012_val_00045022.JPEG:n01985128 +ILSVRC2012_val_00045023.JPEG:n02978881 +ILSVRC2012_val_00045024.JPEG:n04141327 +ILSVRC2012_val_00045025.JPEG:n02403003 +ILSVRC2012_val_00045026.JPEG:n02120079 +ILSVRC2012_val_00045027.JPEG:n07579787 +ILSVRC2012_val_00045028.JPEG:n02317335 +ILSVRC2012_val_00045029.JPEG:n02509815 +ILSVRC2012_val_00045030.JPEG:n04146614 +ILSVRC2012_val_00045031.JPEG:n01944390 +ILSVRC2012_val_00045032.JPEG:n04467665 +ILSVRC2012_val_00045033.JPEG:n02927161 +ILSVRC2012_val_00045034.JPEG:n12620546 +ILSVRC2012_val_00045035.JPEG:n02098286 +ILSVRC2012_val_00045036.JPEG:n01914609 +ILSVRC2012_val_00045037.JPEG:n02486410 +ILSVRC2012_val_00045038.JPEG:n02963159 +ILSVRC2012_val_00045039.JPEG:n03085013 +ILSVRC2012_val_00045040.JPEG:n04525305 +ILSVRC2012_val_00045041.JPEG:n04141076 +ILSVRC2012_val_00045042.JPEG:n01742172 +ILSVRC2012_val_00045043.JPEG:n01798484 +ILSVRC2012_val_00045044.JPEG:n02102480 +ILSVRC2012_val_00045045.JPEG:n01729322 +ILSVRC2012_val_00045046.JPEG:n03938244 +ILSVRC2012_val_00045047.JPEG:n02096585 +ILSVRC2012_val_00045048.JPEG:n04099969 +ILSVRC2012_val_00045049.JPEG:n02437616 +ILSVRC2012_val_00045050.JPEG:n03729826 +ILSVRC2012_val_00045051.JPEG:n01829413 +ILSVRC2012_val_00045052.JPEG:n03527444 +ILSVRC2012_val_00045053.JPEG:n04086273 +ILSVRC2012_val_00045054.JPEG:n02013706 +ILSVRC2012_val_00045055.JPEG:n03594734 +ILSVRC2012_val_00045056.JPEG:n02105855 +ILSVRC2012_val_00045057.JPEG:n04536866 +ILSVRC2012_val_00045058.JPEG:n02489166 +ILSVRC2012_val_00045059.JPEG:n02093991 +ILSVRC2012_val_00045060.JPEG:n02109525 +ILSVRC2012_val_00045061.JPEG:n01930112 +ILSVRC2012_val_00045062.JPEG:n01580077 +ILSVRC2012_val_00045063.JPEG:n02457408 +ILSVRC2012_val_00045064.JPEG:n04328186 +ILSVRC2012_val_00045065.JPEG:n01751748 +ILSVRC2012_val_00045066.JPEG:n03026506 +ILSVRC2012_val_00045067.JPEG:n04235860 +ILSVRC2012_val_00045068.JPEG:n02113023 +ILSVRC2012_val_00045069.JPEG:n03063689 +ILSVRC2012_val_00045070.JPEG:n01882714 +ILSVRC2012_val_00045071.JPEG:n03930630 +ILSVRC2012_val_00045072.JPEG:n03710721 +ILSVRC2012_val_00045073.JPEG:n04264628 +ILSVRC2012_val_00045074.JPEG:n04081281 +ILSVRC2012_val_00045075.JPEG:n04116512 +ILSVRC2012_val_00045076.JPEG:n04044716 +ILSVRC2012_val_00045077.JPEG:n01697457 +ILSVRC2012_val_00045078.JPEG:n04330267 +ILSVRC2012_val_00045079.JPEG:n02860847 +ILSVRC2012_val_00045080.JPEG:n02107908 +ILSVRC2012_val_00045081.JPEG:n04399382 +ILSVRC2012_val_00045082.JPEG:n03873416 +ILSVRC2012_val_00045083.JPEG:n04509417 +ILSVRC2012_val_00045084.JPEG:n03792972 +ILSVRC2012_val_00045085.JPEG:n02102318 +ILSVRC2012_val_00045086.JPEG:n01883070 +ILSVRC2012_val_00045087.JPEG:n07742313 +ILSVRC2012_val_00045088.JPEG:n02033041 +ILSVRC2012_val_00045089.JPEG:n12620546 +ILSVRC2012_val_00045090.JPEG:n03995372 +ILSVRC2012_val_00045091.JPEG:n02086646 +ILSVRC2012_val_00045092.JPEG:n03485794 +ILSVRC2012_val_00045093.JPEG:n07747607 +ILSVRC2012_val_00045094.JPEG:n02098413 +ILSVRC2012_val_00045095.JPEG:n03877472 +ILSVRC2012_val_00045096.JPEG:n02106550 +ILSVRC2012_val_00045097.JPEG:n04263257 +ILSVRC2012_val_00045098.JPEG:n02134418 +ILSVRC2012_val_00045099.JPEG:n04263257 +ILSVRC2012_val_00045100.JPEG:n04606251 +ILSVRC2012_val_00045101.JPEG:n01630670 +ILSVRC2012_val_00045102.JPEG:n02280649 +ILSVRC2012_val_00045103.JPEG:n02504013 +ILSVRC2012_val_00045104.JPEG:n02871525 +ILSVRC2012_val_00045105.JPEG:n04081281 +ILSVRC2012_val_00045106.JPEG:n03782006 +ILSVRC2012_val_00045107.JPEG:n01514668 +ILSVRC2012_val_00045108.JPEG:n02396427 +ILSVRC2012_val_00045109.JPEG:n02093428 +ILSVRC2012_val_00045110.JPEG:n02979186 +ILSVRC2012_val_00045111.JPEG:n04254777 +ILSVRC2012_val_00045112.JPEG:n04009552 +ILSVRC2012_val_00045113.JPEG:n03602883 +ILSVRC2012_val_00045114.JPEG:n07747607 +ILSVRC2012_val_00045115.JPEG:n04562935 +ILSVRC2012_val_00045116.JPEG:n02033041 +ILSVRC2012_val_00045117.JPEG:n04505470 +ILSVRC2012_val_00045118.JPEG:n02906734 +ILSVRC2012_val_00045119.JPEG:n03045698 +ILSVRC2012_val_00045120.JPEG:n01629819 +ILSVRC2012_val_00045121.JPEG:n04613696 +ILSVRC2012_val_00045122.JPEG:n07717556 +ILSVRC2012_val_00045123.JPEG:n02487347 +ILSVRC2012_val_00045124.JPEG:n01917289 +ILSVRC2012_val_00045125.JPEG:n01817953 +ILSVRC2012_val_00045126.JPEG:n07753275 +ILSVRC2012_val_00045127.JPEG:n02457408 +ILSVRC2012_val_00045128.JPEG:n02992529 +ILSVRC2012_val_00045129.JPEG:n01742172 +ILSVRC2012_val_00045130.JPEG:n03950228 +ILSVRC2012_val_00045131.JPEG:n03584254 +ILSVRC2012_val_00045132.JPEG:n02526121 +ILSVRC2012_val_00045133.JPEG:n01494475 +ILSVRC2012_val_00045134.JPEG:n02085936 +ILSVRC2012_val_00045135.JPEG:n02391049 +ILSVRC2012_val_00045136.JPEG:n04355933 +ILSVRC2012_val_00045137.JPEG:n03950228 +ILSVRC2012_val_00045138.JPEG:n03584829 +ILSVRC2012_val_00045139.JPEG:n02128385 +ILSVRC2012_val_00045140.JPEG:n01872401 +ILSVRC2012_val_00045141.JPEG:n02091467 +ILSVRC2012_val_00045142.JPEG:n03481172 +ILSVRC2012_val_00045143.JPEG:n04204347 +ILSVRC2012_val_00045144.JPEG:n03899768 +ILSVRC2012_val_00045145.JPEG:n02107312 +ILSVRC2012_val_00045146.JPEG:n02692877 +ILSVRC2012_val_00045147.JPEG:n04606251 +ILSVRC2012_val_00045148.JPEG:n03770679 +ILSVRC2012_val_00045149.JPEG:n07749582 +ILSVRC2012_val_00045150.JPEG:n01558993 +ILSVRC2012_val_00045151.JPEG:n02099712 +ILSVRC2012_val_00045152.JPEG:n03792782 +ILSVRC2012_val_00045153.JPEG:n03791053 +ILSVRC2012_val_00045154.JPEG:n04317175 +ILSVRC2012_val_00045155.JPEG:n02086079 +ILSVRC2012_val_00045156.JPEG:n02480855 +ILSVRC2012_val_00045157.JPEG:n01682714 +ILSVRC2012_val_00045158.JPEG:n04509417 +ILSVRC2012_val_00045159.JPEG:n03792972 +ILSVRC2012_val_00045160.JPEG:n02108551 +ILSVRC2012_val_00045161.JPEG:n02606052 +ILSVRC2012_val_00045162.JPEG:n03995372 +ILSVRC2012_val_00045163.JPEG:n04336792 +ILSVRC2012_val_00045164.JPEG:n02490219 +ILSVRC2012_val_00045165.JPEG:n07695742 +ILSVRC2012_val_00045166.JPEG:n12998815 +ILSVRC2012_val_00045167.JPEG:n03759954 +ILSVRC2012_val_00045168.JPEG:n04265275 +ILSVRC2012_val_00045169.JPEG:n02971356 +ILSVRC2012_val_00045170.JPEG:n03661043 +ILSVRC2012_val_00045171.JPEG:n02120505 +ILSVRC2012_val_00045172.JPEG:n01530575 +ILSVRC2012_val_00045173.JPEG:n03690938 +ILSVRC2012_val_00045174.JPEG:n02422106 +ILSVRC2012_val_00045175.JPEG:n02120079 +ILSVRC2012_val_00045176.JPEG:n07873807 +ILSVRC2012_val_00045177.JPEG:n04579432 +ILSVRC2012_val_00045178.JPEG:n03930313 +ILSVRC2012_val_00045179.JPEG:n09288635 +ILSVRC2012_val_00045180.JPEG:n02509815 +ILSVRC2012_val_00045181.JPEG:n03998194 +ILSVRC2012_val_00045182.JPEG:n03791053 +ILSVRC2012_val_00045183.JPEG:n01930112 +ILSVRC2012_val_00045184.JPEG:n03991062 +ILSVRC2012_val_00045185.JPEG:n02125311 +ILSVRC2012_val_00045186.JPEG:n02909870 +ILSVRC2012_val_00045187.JPEG:n07718747 +ILSVRC2012_val_00045188.JPEG:n01729322 +ILSVRC2012_val_00045189.JPEG:n02133161 +ILSVRC2012_val_00045190.JPEG:n03763968 +ILSVRC2012_val_00045191.JPEG:n03944341 +ILSVRC2012_val_00045192.JPEG:n01943899 +ILSVRC2012_val_00045193.JPEG:n02445715 +ILSVRC2012_val_00045194.JPEG:n04443257 +ILSVRC2012_val_00045195.JPEG:n02109047 +ILSVRC2012_val_00045196.JPEG:n04141327 +ILSVRC2012_val_00045197.JPEG:n03041632 +ILSVRC2012_val_00045198.JPEG:n01592084 +ILSVRC2012_val_00045199.JPEG:n02906734 +ILSVRC2012_val_00045200.JPEG:n01828970 +ILSVRC2012_val_00045201.JPEG:n03388549 +ILSVRC2012_val_00045202.JPEG:n01917289 +ILSVRC2012_val_00045203.JPEG:n02859443 +ILSVRC2012_val_00045204.JPEG:n02110958 +ILSVRC2012_val_00045205.JPEG:n03956157 +ILSVRC2012_val_00045206.JPEG:n02797295 +ILSVRC2012_val_00045207.JPEG:n02100583 +ILSVRC2012_val_00045208.JPEG:n02776631 +ILSVRC2012_val_00045209.JPEG:n03485407 +ILSVRC2012_val_00045210.JPEG:n04285008 +ILSVRC2012_val_00045211.JPEG:n03623198 +ILSVRC2012_val_00045212.JPEG:n01753488 +ILSVRC2012_val_00045213.JPEG:n03146219 +ILSVRC2012_val_00045214.JPEG:n03535780 +ILSVRC2012_val_00045215.JPEG:n12768682 +ILSVRC2012_val_00045216.JPEG:n12768682 +ILSVRC2012_val_00045217.JPEG:n02100583 +ILSVRC2012_val_00045218.JPEG:n03976657 +ILSVRC2012_val_00045219.JPEG:n04251144 +ILSVRC2012_val_00045220.JPEG:n03444034 +ILSVRC2012_val_00045221.JPEG:n03980874 +ILSVRC2012_val_00045222.JPEG:n02066245 +ILSVRC2012_val_00045223.JPEG:n01692333 +ILSVRC2012_val_00045224.JPEG:n03223299 +ILSVRC2012_val_00045225.JPEG:n04461696 +ILSVRC2012_val_00045226.JPEG:n09835506 +ILSVRC2012_val_00045227.JPEG:n02206856 +ILSVRC2012_val_00045228.JPEG:n13040303 +ILSVRC2012_val_00045229.JPEG:n02088094 +ILSVRC2012_val_00045230.JPEG:n02487347 +ILSVRC2012_val_00045231.JPEG:n03781244 +ILSVRC2012_val_00045232.JPEG:n03832673 +ILSVRC2012_val_00045233.JPEG:n02917067 +ILSVRC2012_val_00045234.JPEG:n01806567 +ILSVRC2012_val_00045235.JPEG:n03776460 +ILSVRC2012_val_00045236.JPEG:n04208210 +ILSVRC2012_val_00045237.JPEG:n04462240 +ILSVRC2012_val_00045238.JPEG:n02093428 +ILSVRC2012_val_00045239.JPEG:n02123045 +ILSVRC2012_val_00045240.JPEG:n03047690 +ILSVRC2012_val_00045241.JPEG:n04201297 +ILSVRC2012_val_00045242.JPEG:n02895154 +ILSVRC2012_val_00045243.JPEG:n04252225 +ILSVRC2012_val_00045244.JPEG:n03837869 +ILSVRC2012_val_00045245.JPEG:n01877812 +ILSVRC2012_val_00045246.JPEG:n03961711 +ILSVRC2012_val_00045247.JPEG:n01753488 +ILSVRC2012_val_00045248.JPEG:n02105505 +ILSVRC2012_val_00045249.JPEG:n02112018 +ILSVRC2012_val_00045250.JPEG:n02110627 +ILSVRC2012_val_00045251.JPEG:n02389026 +ILSVRC2012_val_00045252.JPEG:n02782093 +ILSVRC2012_val_00045253.JPEG:n02099712 +ILSVRC2012_val_00045254.JPEG:n03742115 +ILSVRC2012_val_00045255.JPEG:n04141076 +ILSVRC2012_val_00045256.JPEG:n01735189 +ILSVRC2012_val_00045257.JPEG:n02879718 +ILSVRC2012_val_00045258.JPEG:n03594734 +ILSVRC2012_val_00045259.JPEG:n04462240 +ILSVRC2012_val_00045260.JPEG:n02788148 +ILSVRC2012_val_00045261.JPEG:n02106166 +ILSVRC2012_val_00045262.JPEG:n03991062 +ILSVRC2012_val_00045263.JPEG:n01820546 +ILSVRC2012_val_00045264.JPEG:n04259630 +ILSVRC2012_val_00045265.JPEG:n04310018 +ILSVRC2012_val_00045266.JPEG:n15075141 +ILSVRC2012_val_00045267.JPEG:n03717622 +ILSVRC2012_val_00045268.JPEG:n03595614 +ILSVRC2012_val_00045269.JPEG:n03598930 +ILSVRC2012_val_00045270.JPEG:n02132136 +ILSVRC2012_val_00045271.JPEG:n03630383 +ILSVRC2012_val_00045272.JPEG:n03692522 +ILSVRC2012_val_00045273.JPEG:n04591157 +ILSVRC2012_val_00045274.JPEG:n04154565 +ILSVRC2012_val_00045275.JPEG:n02346627 +ILSVRC2012_val_00045276.JPEG:n02687172 +ILSVRC2012_val_00045277.JPEG:n07693725 +ILSVRC2012_val_00045278.JPEG:n02514041 +ILSVRC2012_val_00045279.JPEG:n02128757 +ILSVRC2012_val_00045280.JPEG:n02095314 +ILSVRC2012_val_00045281.JPEG:n01855032 +ILSVRC2012_val_00045282.JPEG:n03942813 +ILSVRC2012_val_00045283.JPEG:n03485407 +ILSVRC2012_val_00045284.JPEG:n13133613 +ILSVRC2012_val_00045285.JPEG:n03062245 +ILSVRC2012_val_00045286.JPEG:n03447447 +ILSVRC2012_val_00045287.JPEG:n02895154 +ILSVRC2012_val_00045288.JPEG:n04380533 +ILSVRC2012_val_00045289.JPEG:n02364673 +ILSVRC2012_val_00045290.JPEG:n03146219 +ILSVRC2012_val_00045291.JPEG:n02109961 +ILSVRC2012_val_00045292.JPEG:n02113799 +ILSVRC2012_val_00045293.JPEG:n02859443 +ILSVRC2012_val_00045294.JPEG:n01558993 +ILSVRC2012_val_00045295.JPEG:n02119789 +ILSVRC2012_val_00045296.JPEG:n01930112 +ILSVRC2012_val_00045297.JPEG:n04275548 +ILSVRC2012_val_00045298.JPEG:n03602883 +ILSVRC2012_val_00045299.JPEG:n02497673 +ILSVRC2012_val_00045300.JPEG:n02037110 +ILSVRC2012_val_00045301.JPEG:n03026506 +ILSVRC2012_val_00045302.JPEG:n07930864 +ILSVRC2012_val_00045303.JPEG:n04330267 +ILSVRC2012_val_00045304.JPEG:n02480495 +ILSVRC2012_val_00045305.JPEG:n02107683 +ILSVRC2012_val_00045306.JPEG:n03786901 +ILSVRC2012_val_00045307.JPEG:n01917289 +ILSVRC2012_val_00045308.JPEG:n03133878 +ILSVRC2012_val_00045309.JPEG:n04532670 +ILSVRC2012_val_00045310.JPEG:n01775062 +ILSVRC2012_val_00045311.JPEG:n03633091 +ILSVRC2012_val_00045312.JPEG:n03777568 +ILSVRC2012_val_00045313.JPEG:n01945685 +ILSVRC2012_val_00045314.JPEG:n03109150 +ILSVRC2012_val_00045315.JPEG:n03792972 +ILSVRC2012_val_00045316.JPEG:n02895154 +ILSVRC2012_val_00045317.JPEG:n04548362 +ILSVRC2012_val_00045318.JPEG:n02114855 +ILSVRC2012_val_00045319.JPEG:n03775071 +ILSVRC2012_val_00045320.JPEG:n07717556 +ILSVRC2012_val_00045321.JPEG:n02483362 +ILSVRC2012_val_00045322.JPEG:n02909870 +ILSVRC2012_val_00045323.JPEG:n02027492 +ILSVRC2012_val_00045324.JPEG:n07584110 +ILSVRC2012_val_00045325.JPEG:n03594734 +ILSVRC2012_val_00045326.JPEG:n03642806 +ILSVRC2012_val_00045327.JPEG:n03877845 +ILSVRC2012_val_00045328.JPEG:n03379051 +ILSVRC2012_val_00045329.JPEG:n02927161 +ILSVRC2012_val_00045330.JPEG:n04417672 +ILSVRC2012_val_00045331.JPEG:n04009552 +ILSVRC2012_val_00045332.JPEG:n04004767 +ILSVRC2012_val_00045333.JPEG:n02799071 +ILSVRC2012_val_00045334.JPEG:n03874599 +ILSVRC2012_val_00045335.JPEG:n01883070 +ILSVRC2012_val_00045336.JPEG:n03933933 +ILSVRC2012_val_00045337.JPEG:n03450230 +ILSVRC2012_val_00045338.JPEG:n01698640 +ILSVRC2012_val_00045339.JPEG:n03146219 +ILSVRC2012_val_00045340.JPEG:n02113023 +ILSVRC2012_val_00045341.JPEG:n03379051 +ILSVRC2012_val_00045342.JPEG:n03160309 +ILSVRC2012_val_00045343.JPEG:n01968897 +ILSVRC2012_val_00045344.JPEG:n03976467 +ILSVRC2012_val_00045345.JPEG:n04328186 +ILSVRC2012_val_00045346.JPEG:n02018207 +ILSVRC2012_val_00045347.JPEG:n02123597 +ILSVRC2012_val_00045348.JPEG:n02791124 +ILSVRC2012_val_00045349.JPEG:n01729977 +ILSVRC2012_val_00045350.JPEG:n04228054 +ILSVRC2012_val_00045351.JPEG:n02966687 +ILSVRC2012_val_00045352.JPEG:n02094258 +ILSVRC2012_val_00045353.JPEG:n03425413 +ILSVRC2012_val_00045354.JPEG:n01819313 +ILSVRC2012_val_00045355.JPEG:n02100236 +ILSVRC2012_val_00045356.JPEG:n02389026 +ILSVRC2012_val_00045357.JPEG:n02108551 +ILSVRC2012_val_00045358.JPEG:n02085620 +ILSVRC2012_val_00045359.JPEG:n03791053 +ILSVRC2012_val_00045360.JPEG:n03916031 +ILSVRC2012_val_00045361.JPEG:n01871265 +ILSVRC2012_val_00045362.JPEG:n01698640 +ILSVRC2012_val_00045363.JPEG:n02100877 +ILSVRC2012_val_00045364.JPEG:n03146219 +ILSVRC2012_val_00045365.JPEG:n03903868 +ILSVRC2012_val_00045366.JPEG:n03803284 +ILSVRC2012_val_00045367.JPEG:n04204238 +ILSVRC2012_val_00045368.JPEG:n04037443 +ILSVRC2012_val_00045369.JPEG:n02128925 +ILSVRC2012_val_00045370.JPEG:n03131574 +ILSVRC2012_val_00045371.JPEG:n02823428 +ILSVRC2012_val_00045372.JPEG:n09421951 +ILSVRC2012_val_00045373.JPEG:n03884397 +ILSVRC2012_val_00045374.JPEG:n07742313 +ILSVRC2012_val_00045375.JPEG:n03871628 +ILSVRC2012_val_00045376.JPEG:n01770081 +ILSVRC2012_val_00045377.JPEG:n04540053 +ILSVRC2012_val_00045378.JPEG:n03000134 +ILSVRC2012_val_00045379.JPEG:n02443114 +ILSVRC2012_val_00045380.JPEG:n04476259 +ILSVRC2012_val_00045381.JPEG:n04317175 +ILSVRC2012_val_00045382.JPEG:n02091032 +ILSVRC2012_val_00045383.JPEG:n07248320 +ILSVRC2012_val_00045384.JPEG:n04146614 +ILSVRC2012_val_00045385.JPEG:n04532106 +ILSVRC2012_val_00045386.JPEG:n07920052 +ILSVRC2012_val_00045387.JPEG:n02484975 +ILSVRC2012_val_00045388.JPEG:n04612504 +ILSVRC2012_val_00045389.JPEG:n01530575 +ILSVRC2012_val_00045390.JPEG:n03929660 +ILSVRC2012_val_00045391.JPEG:n04540053 +ILSVRC2012_val_00045392.JPEG:n01796340 +ILSVRC2012_val_00045393.JPEG:n01828970 +ILSVRC2012_val_00045394.JPEG:n04162706 +ILSVRC2012_val_00045395.JPEG:n03481172 +ILSVRC2012_val_00045396.JPEG:n03983396 +ILSVRC2012_val_00045397.JPEG:n02777292 +ILSVRC2012_val_00045398.JPEG:n02018795 +ILSVRC2012_val_00045399.JPEG:n02869837 +ILSVRC2012_val_00045400.JPEG:n02835271 +ILSVRC2012_val_00045401.JPEG:n03201208 +ILSVRC2012_val_00045402.JPEG:n01518878 +ILSVRC2012_val_00045403.JPEG:n12057211 +ILSVRC2012_val_00045404.JPEG:n03787032 +ILSVRC2012_val_00045405.JPEG:n02641379 +ILSVRC2012_val_00045406.JPEG:n04554684 +ILSVRC2012_val_00045407.JPEG:n02791124 +ILSVRC2012_val_00045408.JPEG:n01819313 +ILSVRC2012_val_00045409.JPEG:n02389026 +ILSVRC2012_val_00045410.JPEG:n04090263 +ILSVRC2012_val_00045411.JPEG:n03908618 +ILSVRC2012_val_00045412.JPEG:n03792972 +ILSVRC2012_val_00045413.JPEG:n02484975 +ILSVRC2012_val_00045414.JPEG:n07590611 +ILSVRC2012_val_00045415.JPEG:n01530575 +ILSVRC2012_val_00045416.JPEG:n12985857 +ILSVRC2012_val_00045417.JPEG:n09229709 +ILSVRC2012_val_00045418.JPEG:n01755581 +ILSVRC2012_val_00045419.JPEG:n03627232 +ILSVRC2012_val_00045420.JPEG:n02123159 +ILSVRC2012_val_00045421.JPEG:n03775546 +ILSVRC2012_val_00045422.JPEG:n04596742 +ILSVRC2012_val_00045423.JPEG:n04346328 +ILSVRC2012_val_00045424.JPEG:n02669723 +ILSVRC2012_val_00045425.JPEG:n07753592 +ILSVRC2012_val_00045426.JPEG:n07613480 +ILSVRC2012_val_00045427.JPEG:n03884397 +ILSVRC2012_val_00045428.JPEG:n02892201 +ILSVRC2012_val_00045429.JPEG:n01924916 +ILSVRC2012_val_00045430.JPEG:n04467665 +ILSVRC2012_val_00045431.JPEG:n02488291 +ILSVRC2012_val_00045432.JPEG:n03868242 +ILSVRC2012_val_00045433.JPEG:n02356798 +ILSVRC2012_val_00045434.JPEG:n04265275 +ILSVRC2012_val_00045435.JPEG:n02077923 +ILSVRC2012_val_00045436.JPEG:n02102973 +ILSVRC2012_val_00045437.JPEG:n03457902 +ILSVRC2012_val_00045438.JPEG:n02190166 +ILSVRC2012_val_00045439.JPEG:n03259280 +ILSVRC2012_val_00045440.JPEG:n02105162 +ILSVRC2012_val_00045441.JPEG:n02091831 +ILSVRC2012_val_00045442.JPEG:n02256656 +ILSVRC2012_val_00045443.JPEG:n01872401 +ILSVRC2012_val_00045444.JPEG:n02493793 +ILSVRC2012_val_00045445.JPEG:n02408429 +ILSVRC2012_val_00045446.JPEG:n02106550 +ILSVRC2012_val_00045447.JPEG:n03929660 +ILSVRC2012_val_00045448.JPEG:n03325584 +ILSVRC2012_val_00045449.JPEG:n04332243 +ILSVRC2012_val_00045450.JPEG:n04270147 +ILSVRC2012_val_00045451.JPEG:n01630670 +ILSVRC2012_val_00045452.JPEG:n03250847 +ILSVRC2012_val_00045453.JPEG:n02114367 +ILSVRC2012_val_00045454.JPEG:n02106166 +ILSVRC2012_val_00045455.JPEG:n03134739 +ILSVRC2012_val_00045456.JPEG:n02814860 +ILSVRC2012_val_00045457.JPEG:n02110063 +ILSVRC2012_val_00045458.JPEG:n03903868 +ILSVRC2012_val_00045459.JPEG:n02395406 +ILSVRC2012_val_00045460.JPEG:n04311174 +ILSVRC2012_val_00045461.JPEG:n03532672 +ILSVRC2012_val_00045462.JPEG:n02840245 +ILSVRC2012_val_00045463.JPEG:n01986214 +ILSVRC2012_val_00045464.JPEG:n04429376 +ILSVRC2012_val_00045465.JPEG:n02119022 +ILSVRC2012_val_00045466.JPEG:n03218198 +ILSVRC2012_val_00045467.JPEG:n02783161 +ILSVRC2012_val_00045468.JPEG:n03770439 +ILSVRC2012_val_00045469.JPEG:n02089867 +ILSVRC2012_val_00045470.JPEG:n02966687 +ILSVRC2012_val_00045471.JPEG:n03658185 +ILSVRC2012_val_00045472.JPEG:n09193705 +ILSVRC2012_val_00045473.JPEG:n03085013 +ILSVRC2012_val_00045474.JPEG:n02971356 +ILSVRC2012_val_00045475.JPEG:n04049303 +ILSVRC2012_val_00045476.JPEG:n11939491 +ILSVRC2012_val_00045477.JPEG:n02105641 +ILSVRC2012_val_00045478.JPEG:n03494278 +ILSVRC2012_val_00045479.JPEG:n02364673 +ILSVRC2012_val_00045480.JPEG:n01534433 +ILSVRC2012_val_00045481.JPEG:n01735189 +ILSVRC2012_val_00045482.JPEG:n02105855 +ILSVRC2012_val_00045483.JPEG:n03743016 +ILSVRC2012_val_00045484.JPEG:n07718472 +ILSVRC2012_val_00045485.JPEG:n02113799 +ILSVRC2012_val_00045486.JPEG:n04443257 +ILSVRC2012_val_00045487.JPEG:n02096294 +ILSVRC2012_val_00045488.JPEG:n02128925 +ILSVRC2012_val_00045489.JPEG:n02264363 +ILSVRC2012_val_00045490.JPEG:n03796401 +ILSVRC2012_val_00045491.JPEG:n02444819 +ILSVRC2012_val_00045492.JPEG:n03770679 +ILSVRC2012_val_00045493.JPEG:n02093647 +ILSVRC2012_val_00045494.JPEG:n03483316 +ILSVRC2012_val_00045495.JPEG:n02107574 +ILSVRC2012_val_00045496.JPEG:n04127249 +ILSVRC2012_val_00045497.JPEG:n02978881 +ILSVRC2012_val_00045498.JPEG:n13054560 +ILSVRC2012_val_00045499.JPEG:n02823750 +ILSVRC2012_val_00045500.JPEG:n03794056 +ILSVRC2012_val_00045501.JPEG:n03000684 +ILSVRC2012_val_00045502.JPEG:n01496331 +ILSVRC2012_val_00045503.JPEG:n01807496 +ILSVRC2012_val_00045504.JPEG:n02791270 +ILSVRC2012_val_00045505.JPEG:n01860187 +ILSVRC2012_val_00045506.JPEG:n03218198 +ILSVRC2012_val_00045507.JPEG:n02364673 +ILSVRC2012_val_00045508.JPEG:n03498962 +ILSVRC2012_val_00045509.JPEG:n04153751 +ILSVRC2012_val_00045510.JPEG:n01688243 +ILSVRC2012_val_00045511.JPEG:n03388183 +ILSVRC2012_val_00045512.JPEG:n01968897 +ILSVRC2012_val_00045513.JPEG:n02172182 +ILSVRC2012_val_00045514.JPEG:n02112018 +ILSVRC2012_val_00045515.JPEG:n02883205 +ILSVRC2012_val_00045516.JPEG:n03854065 +ILSVRC2012_val_00045517.JPEG:n12267677 +ILSVRC2012_val_00045518.JPEG:n02094258 +ILSVRC2012_val_00045519.JPEG:n04254120 +ILSVRC2012_val_00045520.JPEG:n01855672 +ILSVRC2012_val_00045521.JPEG:n02100877 +ILSVRC2012_val_00045522.JPEG:n03344393 +ILSVRC2012_val_00045523.JPEG:n07693725 +ILSVRC2012_val_00045524.JPEG:n02669723 +ILSVRC2012_val_00045525.JPEG:n02264363 +ILSVRC2012_val_00045526.JPEG:n03763968 +ILSVRC2012_val_00045527.JPEG:n03637318 +ILSVRC2012_val_00045528.JPEG:n04447861 +ILSVRC2012_val_00045529.JPEG:n01984695 +ILSVRC2012_val_00045530.JPEG:n12267677 +ILSVRC2012_val_00045531.JPEG:n04335435 +ILSVRC2012_val_00045532.JPEG:n02120505 +ILSVRC2012_val_00045533.JPEG:n02104365 +ILSVRC2012_val_00045534.JPEG:n03450230 +ILSVRC2012_val_00045535.JPEG:n04286575 +ILSVRC2012_val_00045536.JPEG:n03207941 +ILSVRC2012_val_00045537.JPEG:n02106166 +ILSVRC2012_val_00045538.JPEG:n03325584 +ILSVRC2012_val_00045539.JPEG:n03793489 +ILSVRC2012_val_00045540.JPEG:n03788365 +ILSVRC2012_val_00045541.JPEG:n03877845 +ILSVRC2012_val_00045542.JPEG:n02190166 +ILSVRC2012_val_00045543.JPEG:n02051845 +ILSVRC2012_val_00045544.JPEG:n02100583 +ILSVRC2012_val_00045545.JPEG:n02104029 +ILSVRC2012_val_00045546.JPEG:n06359193 +ILSVRC2012_val_00045547.JPEG:n01514859 +ILSVRC2012_val_00045548.JPEG:n02106550 +ILSVRC2012_val_00045549.JPEG:n02165456 +ILSVRC2012_val_00045550.JPEG:n02276258 +ILSVRC2012_val_00045551.JPEG:n01514859 +ILSVRC2012_val_00045552.JPEG:n03485407 +ILSVRC2012_val_00045553.JPEG:n01632777 +ILSVRC2012_val_00045554.JPEG:n02408429 +ILSVRC2012_val_00045555.JPEG:n03124043 +ILSVRC2012_val_00045556.JPEG:n03717622 +ILSVRC2012_val_00045557.JPEG:n04252225 +ILSVRC2012_val_00045558.JPEG:n04517823 +ILSVRC2012_val_00045559.JPEG:n03425413 +ILSVRC2012_val_00045560.JPEG:n04310018 +ILSVRC2012_val_00045561.JPEG:n03017168 +ILSVRC2012_val_00045562.JPEG:n03832673 +ILSVRC2012_val_00045563.JPEG:n01770081 +ILSVRC2012_val_00045564.JPEG:n03127925 +ILSVRC2012_val_00045565.JPEG:n02089867 +ILSVRC2012_val_00045566.JPEG:n03461385 +ILSVRC2012_val_00045567.JPEG:n03485407 +ILSVRC2012_val_00045568.JPEG:n01592084 +ILSVRC2012_val_00045569.JPEG:n02256656 +ILSVRC2012_val_00045570.JPEG:n03146219 +ILSVRC2012_val_00045571.JPEG:n01795545 +ILSVRC2012_val_00045572.JPEG:n03947888 +ILSVRC2012_val_00045573.JPEG:n07693725 +ILSVRC2012_val_00045574.JPEG:n04483307 +ILSVRC2012_val_00045575.JPEG:n02002556 +ILSVRC2012_val_00045576.JPEG:n04532670 +ILSVRC2012_val_00045577.JPEG:n04049303 +ILSVRC2012_val_00045578.JPEG:n02892201 +ILSVRC2012_val_00045579.JPEG:n03857828 +ILSVRC2012_val_00045580.JPEG:n01494475 +ILSVRC2012_val_00045581.JPEG:n01601694 +ILSVRC2012_val_00045582.JPEG:n04131690 +ILSVRC2012_val_00045583.JPEG:n02666196 +ILSVRC2012_val_00045584.JPEG:n02098286 +ILSVRC2012_val_00045585.JPEG:n02641379 +ILSVRC2012_val_00045586.JPEG:n04228054 +ILSVRC2012_val_00045587.JPEG:n03980874 +ILSVRC2012_val_00045588.JPEG:n04590129 +ILSVRC2012_val_00045589.JPEG:n01616318 +ILSVRC2012_val_00045590.JPEG:n03690938 +ILSVRC2012_val_00045591.JPEG:n04127249 +ILSVRC2012_val_00045592.JPEG:n03345487 +ILSVRC2012_val_00045593.JPEG:n02113023 +ILSVRC2012_val_00045594.JPEG:n01749939 +ILSVRC2012_val_00045595.JPEG:n04229816 +ILSVRC2012_val_00045596.JPEG:n02927161 +ILSVRC2012_val_00045597.JPEG:n03956157 +ILSVRC2012_val_00045598.JPEG:n02111500 +ILSVRC2012_val_00045599.JPEG:n01756291 +ILSVRC2012_val_00045600.JPEG:n02492035 +ILSVRC2012_val_00045601.JPEG:n02119022 +ILSVRC2012_val_00045602.JPEG:n02443114 +ILSVRC2012_val_00045603.JPEG:n02950826 +ILSVRC2012_val_00045604.JPEG:n02319095 +ILSVRC2012_val_00045605.JPEG:n04346328 +ILSVRC2012_val_00045606.JPEG:n02128757 +ILSVRC2012_val_00045607.JPEG:n03998194 +ILSVRC2012_val_00045608.JPEG:n02667093 +ILSVRC2012_val_00045609.JPEG:n01943899 +ILSVRC2012_val_00045610.JPEG:n04467665 +ILSVRC2012_val_00045611.JPEG:n01530575 +ILSVRC2012_val_00045612.JPEG:n01614925 +ILSVRC2012_val_00045613.JPEG:n04346328 +ILSVRC2012_val_00045614.JPEG:n02093754 +ILSVRC2012_val_00045615.JPEG:n03733805 +ILSVRC2012_val_00045616.JPEG:n03742115 +ILSVRC2012_val_00045617.JPEG:n03197337 +ILSVRC2012_val_00045618.JPEG:n02107908 +ILSVRC2012_val_00045619.JPEG:n01737021 +ILSVRC2012_val_00045620.JPEG:n02281787 +ILSVRC2012_val_00045621.JPEG:n03141823 +ILSVRC2012_val_00045622.JPEG:n04254120 +ILSVRC2012_val_00045623.JPEG:n01532829 +ILSVRC2012_val_00045624.JPEG:n02526121 +ILSVRC2012_val_00045625.JPEG:n02966687 +ILSVRC2012_val_00045626.JPEG:n02484975 +ILSVRC2012_val_00045627.JPEG:n03832673 +ILSVRC2012_val_00045628.JPEG:n02113799 +ILSVRC2012_val_00045629.JPEG:n03958227 +ILSVRC2012_val_00045630.JPEG:n04350905 +ILSVRC2012_val_00045631.JPEG:n03623198 +ILSVRC2012_val_00045632.JPEG:n06874185 +ILSVRC2012_val_00045633.JPEG:n03337140 +ILSVRC2012_val_00045634.JPEG:n02097658 +ILSVRC2012_val_00045635.JPEG:n04311174 +ILSVRC2012_val_00045636.JPEG:n04201297 +ILSVRC2012_val_00045637.JPEG:n03908714 +ILSVRC2012_val_00045638.JPEG:n01740131 +ILSVRC2012_val_00045639.JPEG:n03929855 +ILSVRC2012_val_00045640.JPEG:n02509815 +ILSVRC2012_val_00045641.JPEG:n03903868 +ILSVRC2012_val_00045642.JPEG:n03658185 +ILSVRC2012_val_00045643.JPEG:n01843065 +ILSVRC2012_val_00045644.JPEG:n04557648 +ILSVRC2012_val_00045645.JPEG:n04392985 +ILSVRC2012_val_00045646.JPEG:n02454379 +ILSVRC2012_val_00045647.JPEG:n02493793 +ILSVRC2012_val_00045648.JPEG:n04275548 +ILSVRC2012_val_00045649.JPEG:n03220513 +ILSVRC2012_val_00045650.JPEG:n02606052 +ILSVRC2012_val_00045651.JPEG:n04118776 +ILSVRC2012_val_00045652.JPEG:n02514041 +ILSVRC2012_val_00045653.JPEG:n07684084 +ILSVRC2012_val_00045654.JPEG:n03388183 +ILSVRC2012_val_00045655.JPEG:n02794156 +ILSVRC2012_val_00045656.JPEG:n01632777 +ILSVRC2012_val_00045657.JPEG:n04238763 +ILSVRC2012_val_00045658.JPEG:n04372370 +ILSVRC2012_val_00045659.JPEG:n03876231 +ILSVRC2012_val_00045660.JPEG:n02948072 +ILSVRC2012_val_00045661.JPEG:n02096437 +ILSVRC2012_val_00045662.JPEG:n02497673 +ILSVRC2012_val_00045663.JPEG:n03843555 +ILSVRC2012_val_00045664.JPEG:n07565083 +ILSVRC2012_val_00045665.JPEG:n02097130 +ILSVRC2012_val_00045666.JPEG:n04509417 +ILSVRC2012_val_00045667.JPEG:n03255030 +ILSVRC2012_val_00045668.JPEG:n02129165 +ILSVRC2012_val_00045669.JPEG:n01682714 +ILSVRC2012_val_00045670.JPEG:n07753275 +ILSVRC2012_val_00045671.JPEG:n09472597 +ILSVRC2012_val_00045672.JPEG:n02134418 +ILSVRC2012_val_00045673.JPEG:n02219486 +ILSVRC2012_val_00045674.JPEG:n02097047 +ILSVRC2012_val_00045675.JPEG:n03063689 +ILSVRC2012_val_00045676.JPEG:n02091467 +ILSVRC2012_val_00045677.JPEG:n03781244 +ILSVRC2012_val_00045678.JPEG:n02807133 +ILSVRC2012_val_00045679.JPEG:n03814906 +ILSVRC2012_val_00045680.JPEG:n04355338 +ILSVRC2012_val_00045681.JPEG:n04579145 +ILSVRC2012_val_00045682.JPEG:n03272010 +ILSVRC2012_val_00045683.JPEG:n02086646 +ILSVRC2012_val_00045684.JPEG:n02106662 +ILSVRC2012_val_00045685.JPEG:n03956157 +ILSVRC2012_val_00045686.JPEG:n02783161 +ILSVRC2012_val_00045687.JPEG:n02112137 +ILSVRC2012_val_00045688.JPEG:n03188531 +ILSVRC2012_val_00045689.JPEG:n03126707 +ILSVRC2012_val_00045690.JPEG:n01608432 +ILSVRC2012_val_00045691.JPEG:n03337140 +ILSVRC2012_val_00045692.JPEG:n01847000 +ILSVRC2012_val_00045693.JPEG:n04125021 +ILSVRC2012_val_00045694.JPEG:n04147183 +ILSVRC2012_val_00045695.JPEG:n07720875 +ILSVRC2012_val_00045696.JPEG:n02319095 +ILSVRC2012_val_00045697.JPEG:n02510455 +ILSVRC2012_val_00045698.JPEG:n04311174 +ILSVRC2012_val_00045699.JPEG:n03584254 +ILSVRC2012_val_00045700.JPEG:n04542943 +ILSVRC2012_val_00045701.JPEG:n02102480 +ILSVRC2012_val_00045702.JPEG:n02114712 +ILSVRC2012_val_00045703.JPEG:n02268443 +ILSVRC2012_val_00045704.JPEG:n07718472 +ILSVRC2012_val_00045705.JPEG:n03792972 +ILSVRC2012_val_00045706.JPEG:n03724870 +ILSVRC2012_val_00045707.JPEG:n04239074 +ILSVRC2012_val_00045708.JPEG:n02091134 +ILSVRC2012_val_00045709.JPEG:n02129604 +ILSVRC2012_val_00045710.JPEG:n03127925 +ILSVRC2012_val_00045711.JPEG:n02086646 +ILSVRC2012_val_00045712.JPEG:n03207941 +ILSVRC2012_val_00045713.JPEG:n01819313 +ILSVRC2012_val_00045714.JPEG:n04522168 +ILSVRC2012_val_00045715.JPEG:n03271574 +ILSVRC2012_val_00045716.JPEG:n04487394 +ILSVRC2012_val_00045717.JPEG:n03710193 +ILSVRC2012_val_00045718.JPEG:n02105855 +ILSVRC2012_val_00045719.JPEG:n03131574 +ILSVRC2012_val_00045720.JPEG:n02105251 +ILSVRC2012_val_00045721.JPEG:n02095889 +ILSVRC2012_val_00045722.JPEG:n03384352 +ILSVRC2012_val_00045723.JPEG:n07880968 +ILSVRC2012_val_00045724.JPEG:n02259212 +ILSVRC2012_val_00045725.JPEG:n04069434 +ILSVRC2012_val_00045726.JPEG:n01669191 +ILSVRC2012_val_00045727.JPEG:n03710193 +ILSVRC2012_val_00045728.JPEG:n01855672 +ILSVRC2012_val_00045729.JPEG:n13037406 +ILSVRC2012_val_00045730.JPEG:n01484850 +ILSVRC2012_val_00045731.JPEG:n04476259 +ILSVRC2012_val_00045732.JPEG:n03871628 +ILSVRC2012_val_00045733.JPEG:n01774750 +ILSVRC2012_val_00045734.JPEG:n02108551 +ILSVRC2012_val_00045735.JPEG:n02090622 +ILSVRC2012_val_00045736.JPEG:n03733281 +ILSVRC2012_val_00045737.JPEG:n03724870 +ILSVRC2012_val_00045738.JPEG:n03976657 +ILSVRC2012_val_00045739.JPEG:n02099267 +ILSVRC2012_val_00045740.JPEG:n04127249 +ILSVRC2012_val_00045741.JPEG:n02097474 +ILSVRC2012_val_00045742.JPEG:n02056570 +ILSVRC2012_val_00045743.JPEG:n01795545 +ILSVRC2012_val_00045744.JPEG:n07714571 +ILSVRC2012_val_00045745.JPEG:n02107142 +ILSVRC2012_val_00045746.JPEG:n01608432 +ILSVRC2012_val_00045747.JPEG:n02113023 +ILSVRC2012_val_00045748.JPEG:n04486054 +ILSVRC2012_val_00045749.JPEG:n03876231 +ILSVRC2012_val_00045750.JPEG:n04270147 +ILSVRC2012_val_00045751.JPEG:n03461385 +ILSVRC2012_val_00045752.JPEG:n13040303 +ILSVRC2012_val_00045753.JPEG:n02102318 +ILSVRC2012_val_00045754.JPEG:n02910353 +ILSVRC2012_val_00045755.JPEG:n02094114 +ILSVRC2012_val_00045756.JPEG:n02786058 +ILSVRC2012_val_00045757.JPEG:n02992211 +ILSVRC2012_val_00045758.JPEG:n02396427 +ILSVRC2012_val_00045759.JPEG:n04344873 +ILSVRC2012_val_00045760.JPEG:n02097130 +ILSVRC2012_val_00045761.JPEG:n01443537 +ILSVRC2012_val_00045762.JPEG:n04325704 +ILSVRC2012_val_00045763.JPEG:n02093428 +ILSVRC2012_val_00045764.JPEG:n04258138 +ILSVRC2012_val_00045765.JPEG:n07584110 +ILSVRC2012_val_00045766.JPEG:n03443371 +ILSVRC2012_val_00045767.JPEG:n03481172 +ILSVRC2012_val_00045768.JPEG:n02110341 +ILSVRC2012_val_00045769.JPEG:n04141975 +ILSVRC2012_val_00045770.JPEG:n02226429 +ILSVRC2012_val_00045771.JPEG:n02281406 +ILSVRC2012_val_00045772.JPEG:n04141327 +ILSVRC2012_val_00045773.JPEG:n04118538 +ILSVRC2012_val_00045774.JPEG:n02037110 +ILSVRC2012_val_00045775.JPEG:n02226429 +ILSVRC2012_val_00045776.JPEG:n01692333 +ILSVRC2012_val_00045777.JPEG:n03916031 +ILSVRC2012_val_00045778.JPEG:n02787622 +ILSVRC2012_val_00045779.JPEG:n03594945 +ILSVRC2012_val_00045780.JPEG:n07860988 +ILSVRC2012_val_00045781.JPEG:n03729826 +ILSVRC2012_val_00045782.JPEG:n04515003 +ILSVRC2012_val_00045783.JPEG:n04612504 +ILSVRC2012_val_00045784.JPEG:n02007558 +ILSVRC2012_val_00045785.JPEG:n01560419 +ILSVRC2012_val_00045786.JPEG:n02951358 +ILSVRC2012_val_00045787.JPEG:n02837789 +ILSVRC2012_val_00045788.JPEG:n04456115 +ILSVRC2012_val_00045789.JPEG:n04239074 +ILSVRC2012_val_00045790.JPEG:n02094433 +ILSVRC2012_val_00045791.JPEG:n04553703 +ILSVRC2012_val_00045792.JPEG:n03045698 +ILSVRC2012_val_00045793.JPEG:n03874599 +ILSVRC2012_val_00045794.JPEG:n03595614 +ILSVRC2012_val_00045795.JPEG:n02514041 +ILSVRC2012_val_00045796.JPEG:n03876231 +ILSVRC2012_val_00045797.JPEG:n04467665 +ILSVRC2012_val_00045798.JPEG:n04146614 +ILSVRC2012_val_00045799.JPEG:n02089973 +ILSVRC2012_val_00045800.JPEG:n04005630 +ILSVRC2012_val_00045801.JPEG:n04266014 +ILSVRC2012_val_00045802.JPEG:n04074963 +ILSVRC2012_val_00045803.JPEG:n03527444 +ILSVRC2012_val_00045804.JPEG:n04355338 +ILSVRC2012_val_00045805.JPEG:n09246464 +ILSVRC2012_val_00045806.JPEG:n03980874 +ILSVRC2012_val_00045807.JPEG:n01990800 +ILSVRC2012_val_00045808.JPEG:n03697007 +ILSVRC2012_val_00045809.JPEG:n13133613 +ILSVRC2012_val_00045810.JPEG:n07613480 +ILSVRC2012_val_00045811.JPEG:n02655020 +ILSVRC2012_val_00045812.JPEG:n03240683 +ILSVRC2012_val_00045813.JPEG:n04111531 +ILSVRC2012_val_00045814.JPEG:n01871265 +ILSVRC2012_val_00045815.JPEG:n01695060 +ILSVRC2012_val_00045816.JPEG:n03478589 +ILSVRC2012_val_00045817.JPEG:n04265275 +ILSVRC2012_val_00045818.JPEG:n02094433 +ILSVRC2012_val_00045819.JPEG:n02009229 +ILSVRC2012_val_00045820.JPEG:n02708093 +ILSVRC2012_val_00045821.JPEG:n03447447 +ILSVRC2012_val_00045822.JPEG:n03216828 +ILSVRC2012_val_00045823.JPEG:n04371430 +ILSVRC2012_val_00045824.JPEG:n03991062 +ILSVRC2012_val_00045825.JPEG:n02607072 +ILSVRC2012_val_00045826.JPEG:n02481823 +ILSVRC2012_val_00045827.JPEG:n02102318 +ILSVRC2012_val_00045828.JPEG:n09256479 +ILSVRC2012_val_00045829.JPEG:n02123597 +ILSVRC2012_val_00045830.JPEG:n02927161 +ILSVRC2012_val_00045831.JPEG:n01737021 +ILSVRC2012_val_00045832.JPEG:n01675722 +ILSVRC2012_val_00045833.JPEG:n11939491 +ILSVRC2012_val_00045834.JPEG:n03937543 +ILSVRC2012_val_00045835.JPEG:n03729826 +ILSVRC2012_val_00045836.JPEG:n01820546 +ILSVRC2012_val_00045837.JPEG:n01847000 +ILSVRC2012_val_00045838.JPEG:n02112137 +ILSVRC2012_val_00045839.JPEG:n01675722 +ILSVRC2012_val_00045840.JPEG:n04613696 +ILSVRC2012_val_00045841.JPEG:n02974003 +ILSVRC2012_val_00045842.JPEG:n03384352 +ILSVRC2012_val_00045843.JPEG:n03627232 +ILSVRC2012_val_00045844.JPEG:n04429376 +ILSVRC2012_val_00045845.JPEG:n01756291 +ILSVRC2012_val_00045846.JPEG:n03496892 +ILSVRC2012_val_00045847.JPEG:n02398521 +ILSVRC2012_val_00045848.JPEG:n02168699 +ILSVRC2012_val_00045849.JPEG:n03000247 +ILSVRC2012_val_00045850.JPEG:n01739381 +ILSVRC2012_val_00045851.JPEG:n04371430 +ILSVRC2012_val_00045852.JPEG:n04335435 +ILSVRC2012_val_00045853.JPEG:n03532672 +ILSVRC2012_val_00045854.JPEG:n02441942 +ILSVRC2012_val_00045855.JPEG:n03400231 +ILSVRC2012_val_00045856.JPEG:n03793489 +ILSVRC2012_val_00045857.JPEG:n01795545 +ILSVRC2012_val_00045858.JPEG:n01740131 +ILSVRC2012_val_00045859.JPEG:n02110806 +ILSVRC2012_val_00045860.JPEG:n03063599 +ILSVRC2012_val_00045861.JPEG:n02095314 +ILSVRC2012_val_00045862.JPEG:n04579432 +ILSVRC2012_val_00045863.JPEG:n04591157 +ILSVRC2012_val_00045864.JPEG:n02321529 +ILSVRC2012_val_00045865.JPEG:n03661043 +ILSVRC2012_val_00045866.JPEG:n01440764 +ILSVRC2012_val_00045867.JPEG:n04228054 +ILSVRC2012_val_00045868.JPEG:n04462240 +ILSVRC2012_val_00045869.JPEG:n03877472 +ILSVRC2012_val_00045870.JPEG:n03720891 +ILSVRC2012_val_00045871.JPEG:n02514041 +ILSVRC2012_val_00045872.JPEG:n03272562 +ILSVRC2012_val_00045873.JPEG:n01601694 +ILSVRC2012_val_00045874.JPEG:n02091467 +ILSVRC2012_val_00045875.JPEG:n04041544 +ILSVRC2012_val_00045876.JPEG:n03796401 +ILSVRC2012_val_00045877.JPEG:n03594734 +ILSVRC2012_val_00045878.JPEG:n02089078 +ILSVRC2012_val_00045879.JPEG:n02493793 +ILSVRC2012_val_00045880.JPEG:n01440764 +ILSVRC2012_val_00045881.JPEG:n09399592 +ILSVRC2012_val_00045882.JPEG:n03775071 +ILSVRC2012_val_00045883.JPEG:n04296562 +ILSVRC2012_val_00045884.JPEG:n02099849 +ILSVRC2012_val_00045885.JPEG:n02804610 +ILSVRC2012_val_00045886.JPEG:n03384352 +ILSVRC2012_val_00045887.JPEG:n02088632 +ILSVRC2012_val_00045888.JPEG:n04026417 +ILSVRC2012_val_00045889.JPEG:n02794156 +ILSVRC2012_val_00045890.JPEG:n01968897 +ILSVRC2012_val_00045891.JPEG:n02133161 +ILSVRC2012_val_00045892.JPEG:n03777754 +ILSVRC2012_val_00045893.JPEG:n02494079 +ILSVRC2012_val_00045894.JPEG:n02107142 +ILSVRC2012_val_00045895.JPEG:n03710193 +ILSVRC2012_val_00045896.JPEG:n02640242 +ILSVRC2012_val_00045897.JPEG:n04209133 +ILSVRC2012_val_00045898.JPEG:n02443114 +ILSVRC2012_val_00045899.JPEG:n03259280 +ILSVRC2012_val_00045900.JPEG:n02172182 +ILSVRC2012_val_00045901.JPEG:n02089078 +ILSVRC2012_val_00045902.JPEG:n04049303 +ILSVRC2012_val_00045903.JPEG:n02093647 +ILSVRC2012_val_00045904.JPEG:n06785654 +ILSVRC2012_val_00045905.JPEG:n03733131 +ILSVRC2012_val_00045906.JPEG:n03476991 +ILSVRC2012_val_00045907.JPEG:n04259630 +ILSVRC2012_val_00045908.JPEG:n01768244 +ILSVRC2012_val_00045909.JPEG:n13037406 +ILSVRC2012_val_00045910.JPEG:n02168699 +ILSVRC2012_val_00045911.JPEG:n02013706 +ILSVRC2012_val_00045912.JPEG:n02089078 +ILSVRC2012_val_00045913.JPEG:n01817953 +ILSVRC2012_val_00045914.JPEG:n02280649 +ILSVRC2012_val_00045915.JPEG:n02877765 +ILSVRC2012_val_00045916.JPEG:n04273569 +ILSVRC2012_val_00045917.JPEG:n02097209 +ILSVRC2012_val_00045918.JPEG:n06785654 +ILSVRC2012_val_00045919.JPEG:n02104365 +ILSVRC2012_val_00045920.JPEG:n02107908 +ILSVRC2012_val_00045921.JPEG:n02484975 +ILSVRC2012_val_00045922.JPEG:n02906734 +ILSVRC2012_val_00045923.JPEG:n09468604 +ILSVRC2012_val_00045924.JPEG:n01632777 +ILSVRC2012_val_00045925.JPEG:n01494475 +ILSVRC2012_val_00045926.JPEG:n01983481 +ILSVRC2012_val_00045927.JPEG:n04372370 +ILSVRC2012_val_00045928.JPEG:n02364673 +ILSVRC2012_val_00045929.JPEG:n02730930 +ILSVRC2012_val_00045930.JPEG:n02100583 +ILSVRC2012_val_00045931.JPEG:n04127249 +ILSVRC2012_val_00045932.JPEG:n03355925 +ILSVRC2012_val_00045933.JPEG:n02108089 +ILSVRC2012_val_00045934.JPEG:n03197337 +ILSVRC2012_val_00045935.JPEG:n03857828 +ILSVRC2012_val_00045936.JPEG:n01496331 +ILSVRC2012_val_00045937.JPEG:n02110341 +ILSVRC2012_val_00045938.JPEG:n04074963 +ILSVRC2012_val_00045939.JPEG:n02087046 +ILSVRC2012_val_00045940.JPEG:n03000684 +ILSVRC2012_val_00045941.JPEG:n03485794 +ILSVRC2012_val_00045942.JPEG:n02500267 +ILSVRC2012_val_00045943.JPEG:n02105162 +ILSVRC2012_val_00045944.JPEG:n03425413 +ILSVRC2012_val_00045945.JPEG:n01944390 +ILSVRC2012_val_00045946.JPEG:n02112018 +ILSVRC2012_val_00045947.JPEG:n04005630 +ILSVRC2012_val_00045948.JPEG:n01582220 +ILSVRC2012_val_00045949.JPEG:n04275548 +ILSVRC2012_val_00045950.JPEG:n07754684 +ILSVRC2012_val_00045951.JPEG:n02011460 +ILSVRC2012_val_00045952.JPEG:n02132136 +ILSVRC2012_val_00045953.JPEG:n01748264 +ILSVRC2012_val_00045954.JPEG:n04228054 +ILSVRC2012_val_00045955.JPEG:n02980441 +ILSVRC2012_val_00045956.JPEG:n02113624 +ILSVRC2012_val_00045957.JPEG:n04597913 +ILSVRC2012_val_00045958.JPEG:n02123159 +ILSVRC2012_val_00045959.JPEG:n02027492 +ILSVRC2012_val_00045960.JPEG:n04590129 +ILSVRC2012_val_00045961.JPEG:n02114548 +ILSVRC2012_val_00045962.JPEG:n03208938 +ILSVRC2012_val_00045963.JPEG:n02099267 +ILSVRC2012_val_00045964.JPEG:n03538406 +ILSVRC2012_val_00045965.JPEG:n03218198 +ILSVRC2012_val_00045966.JPEG:n04254120 +ILSVRC2012_val_00045967.JPEG:n03337140 +ILSVRC2012_val_00045968.JPEG:n02089078 +ILSVRC2012_val_00045969.JPEG:n02701002 +ILSVRC2012_val_00045970.JPEG:n02086240 +ILSVRC2012_val_00045971.JPEG:n02088632 +ILSVRC2012_val_00045972.JPEG:n01943899 +ILSVRC2012_val_00045973.JPEG:n13052670 +ILSVRC2012_val_00045974.JPEG:n04606251 +ILSVRC2012_val_00045975.JPEG:n09229709 +ILSVRC2012_val_00045976.JPEG:n01687978 +ILSVRC2012_val_00045977.JPEG:n03929660 +ILSVRC2012_val_00045978.JPEG:n02093754 +ILSVRC2012_val_00045979.JPEG:n01729322 +ILSVRC2012_val_00045980.JPEG:n02107908 +ILSVRC2012_val_00045981.JPEG:n07715103 +ILSVRC2012_val_00045982.JPEG:n03773504 +ILSVRC2012_val_00045983.JPEG:n04592741 +ILSVRC2012_val_00045984.JPEG:n02107908 +ILSVRC2012_val_00045985.JPEG:n02264363 +ILSVRC2012_val_00045986.JPEG:n04154565 +ILSVRC2012_val_00045987.JPEG:n02098105 +ILSVRC2012_val_00045988.JPEG:n03485794 +ILSVRC2012_val_00045989.JPEG:n02791270 +ILSVRC2012_val_00045990.JPEG:n06874185 +ILSVRC2012_val_00045991.JPEG:n02488702 +ILSVRC2012_val_00045992.JPEG:n03014705 +ILSVRC2012_val_00045993.JPEG:n03657121 +ILSVRC2012_val_00045994.JPEG:n03854065 +ILSVRC2012_val_00045995.JPEG:n02107574 +ILSVRC2012_val_00045996.JPEG:n02669723 +ILSVRC2012_val_00045997.JPEG:n03950228 +ILSVRC2012_val_00045998.JPEG:n02317335 +ILSVRC2012_val_00045999.JPEG:n04133789 +ILSVRC2012_val_00046000.JPEG:n01685808 +ILSVRC2012_val_00046001.JPEG:n03933933 +ILSVRC2012_val_00046002.JPEG:n02097047 +ILSVRC2012_val_00046003.JPEG:n02011460 +ILSVRC2012_val_00046004.JPEG:n01819313 +ILSVRC2012_val_00046005.JPEG:n03982430 +ILSVRC2012_val_00046006.JPEG:n01784675 +ILSVRC2012_val_00046007.JPEG:n03670208 +ILSVRC2012_val_00046008.JPEG:n03220513 +ILSVRC2012_val_00046009.JPEG:n04118538 +ILSVRC2012_val_00046010.JPEG:n02782093 +ILSVRC2012_val_00046011.JPEG:n02783161 +ILSVRC2012_val_00046012.JPEG:n03496892 +ILSVRC2012_val_00046013.JPEG:n02107574 +ILSVRC2012_val_00046014.JPEG:n04040759 +ILSVRC2012_val_00046015.JPEG:n02013706 +ILSVRC2012_val_00046016.JPEG:n02777292 +ILSVRC2012_val_00046017.JPEG:n01775062 +ILSVRC2012_val_00046018.JPEG:n01748264 +ILSVRC2012_val_00046019.JPEG:n03018349 +ILSVRC2012_val_00046020.JPEG:n04111531 +ILSVRC2012_val_00046021.JPEG:n02089867 +ILSVRC2012_val_00046022.JPEG:n09246464 +ILSVRC2012_val_00046023.JPEG:n04548280 +ILSVRC2012_val_00046024.JPEG:n07734744 +ILSVRC2012_val_00046025.JPEG:n03291819 +ILSVRC2012_val_00046026.JPEG:n04552348 +ILSVRC2012_val_00046027.JPEG:n03871628 +ILSVRC2012_val_00046028.JPEG:n07753113 +ILSVRC2012_val_00046029.JPEG:n01729322 +ILSVRC2012_val_00046030.JPEG:n07715103 +ILSVRC2012_val_00046031.JPEG:n04596742 +ILSVRC2012_val_00046032.JPEG:n02128385 +ILSVRC2012_val_00046033.JPEG:n03976467 +ILSVRC2012_val_00046034.JPEG:n04548280 +ILSVRC2012_val_00046035.JPEG:n02497673 +ILSVRC2012_val_00046036.JPEG:n02134418 +ILSVRC2012_val_00046037.JPEG:n02105251 +ILSVRC2012_val_00046038.JPEG:n03970156 +ILSVRC2012_val_00046039.JPEG:n01749939 +ILSVRC2012_val_00046040.JPEG:n01795545 +ILSVRC2012_val_00046041.JPEG:n01855032 +ILSVRC2012_val_00046042.JPEG:n02395406 +ILSVRC2012_val_00046043.JPEG:n02098413 +ILSVRC2012_val_00046044.JPEG:n02111500 +ILSVRC2012_val_00046045.JPEG:n02895154 +ILSVRC2012_val_00046046.JPEG:n07565083 +ILSVRC2012_val_00046047.JPEG:n03742115 +ILSVRC2012_val_00046048.JPEG:n02108089 +ILSVRC2012_val_00046049.JPEG:n02321529 +ILSVRC2012_val_00046050.JPEG:n02971356 +ILSVRC2012_val_00046051.JPEG:n02437616 +ILSVRC2012_val_00046052.JPEG:n03208938 +ILSVRC2012_val_00046053.JPEG:n01667114 +ILSVRC2012_val_00046054.JPEG:n02226429 +ILSVRC2012_val_00046055.JPEG:n03877845 +ILSVRC2012_val_00046056.JPEG:n02910353 +ILSVRC2012_val_00046057.JPEG:n04070727 +ILSVRC2012_val_00046058.JPEG:n04152593 +ILSVRC2012_val_00046059.JPEG:n01883070 +ILSVRC2012_val_00046060.JPEG:n02870880 +ILSVRC2012_val_00046061.JPEG:n02504458 +ILSVRC2012_val_00046062.JPEG:n04243546 +ILSVRC2012_val_00046063.JPEG:n02096051 +ILSVRC2012_val_00046064.JPEG:n03899768 +ILSVRC2012_val_00046065.JPEG:n02321529 +ILSVRC2012_val_00046066.JPEG:n03877845 +ILSVRC2012_val_00046067.JPEG:n03450230 +ILSVRC2012_val_00046068.JPEG:n03290653 +ILSVRC2012_val_00046069.JPEG:n01664065 +ILSVRC2012_val_00046070.JPEG:n03908714 +ILSVRC2012_val_00046071.JPEG:n01537544 +ILSVRC2012_val_00046072.JPEG:n02088238 +ILSVRC2012_val_00046073.JPEG:n01882714 +ILSVRC2012_val_00046074.JPEG:n01773549 +ILSVRC2012_val_00046075.JPEG:n04418357 +ILSVRC2012_val_00046076.JPEG:n02727426 +ILSVRC2012_val_00046077.JPEG:n01872401 +ILSVRC2012_val_00046078.JPEG:n02106382 +ILSVRC2012_val_00046079.JPEG:n03991062 +ILSVRC2012_val_00046080.JPEG:n02017213 +ILSVRC2012_val_00046081.JPEG:n02018207 +ILSVRC2012_val_00046082.JPEG:n04370456 +ILSVRC2012_val_00046083.JPEG:n02219486 +ILSVRC2012_val_00046084.JPEG:n02669723 +ILSVRC2012_val_00046085.JPEG:n01694178 +ILSVRC2012_val_00046086.JPEG:n01784675 +ILSVRC2012_val_00046087.JPEG:n03443371 +ILSVRC2012_val_00046088.JPEG:n02114548 +ILSVRC2012_val_00046089.JPEG:n01806567 +ILSVRC2012_val_00046090.JPEG:n04090263 +ILSVRC2012_val_00046091.JPEG:n07932039 +ILSVRC2012_val_00046092.JPEG:n01608432 +ILSVRC2012_val_00046093.JPEG:n02281406 +ILSVRC2012_val_00046094.JPEG:n04238763 +ILSVRC2012_val_00046095.JPEG:n01664065 +ILSVRC2012_val_00046096.JPEG:n02028035 +ILSVRC2012_val_00046097.JPEG:n01917289 +ILSVRC2012_val_00046098.JPEG:n03793489 +ILSVRC2012_val_00046099.JPEG:n04209239 +ILSVRC2012_val_00046100.JPEG:n03042490 +ILSVRC2012_val_00046101.JPEG:n03400231 +ILSVRC2012_val_00046102.JPEG:n02356798 +ILSVRC2012_val_00046103.JPEG:n03065424 +ILSVRC2012_val_00046104.JPEG:n04335435 +ILSVRC2012_val_00046105.JPEG:n01664065 +ILSVRC2012_val_00046106.JPEG:n01692333 +ILSVRC2012_val_00046107.JPEG:n07880968 +ILSVRC2012_val_00046108.JPEG:n03297495 +ILSVRC2012_val_00046109.JPEG:n02841315 +ILSVRC2012_val_00046110.JPEG:n03095699 +ILSVRC2012_val_00046111.JPEG:n07697313 +ILSVRC2012_val_00046112.JPEG:n09399592 +ILSVRC2012_val_00046113.JPEG:n01917289 +ILSVRC2012_val_00046114.JPEG:n03724870 +ILSVRC2012_val_00046115.JPEG:n13133613 +ILSVRC2012_val_00046116.JPEG:n03787032 +ILSVRC2012_val_00046117.JPEG:n02493793 +ILSVRC2012_val_00046118.JPEG:n03843555 +ILSVRC2012_val_00046119.JPEG:n01629819 +ILSVRC2012_val_00046120.JPEG:n03843555 +ILSVRC2012_val_00046121.JPEG:n04461696 +ILSVRC2012_val_00046122.JPEG:n01669191 +ILSVRC2012_val_00046123.JPEG:n03976657 +ILSVRC2012_val_00046124.JPEG:n02097047 +ILSVRC2012_val_00046125.JPEG:n03773504 +ILSVRC2012_val_00046126.JPEG:n02951585 +ILSVRC2012_val_00046127.JPEG:n04398044 +ILSVRC2012_val_00046128.JPEG:n03599486 +ILSVRC2012_val_00046129.JPEG:n03250847 +ILSVRC2012_val_00046130.JPEG:n03796401 +ILSVRC2012_val_00046131.JPEG:n01737021 +ILSVRC2012_val_00046132.JPEG:n02776631 +ILSVRC2012_val_00046133.JPEG:n03599486 +ILSVRC2012_val_00046134.JPEG:n02110806 +ILSVRC2012_val_00046135.JPEG:n04254680 +ILSVRC2012_val_00046136.JPEG:n02138441 +ILSVRC2012_val_00046137.JPEG:n02483362 +ILSVRC2012_val_00046138.JPEG:n02747177 +ILSVRC2012_val_00046139.JPEG:n03733805 +ILSVRC2012_val_00046140.JPEG:n04118538 +ILSVRC2012_val_00046141.JPEG:n01829413 +ILSVRC2012_val_00046142.JPEG:n02112137 +ILSVRC2012_val_00046143.JPEG:n02102318 +ILSVRC2012_val_00046144.JPEG:n02097474 +ILSVRC2012_val_00046145.JPEG:n02119789 +ILSVRC2012_val_00046146.JPEG:n04136333 +ILSVRC2012_val_00046147.JPEG:n04579432 +ILSVRC2012_val_00046148.JPEG:n02493509 +ILSVRC2012_val_00046149.JPEG:n01667778 +ILSVRC2012_val_00046150.JPEG:n02442845 +ILSVRC2012_val_00046151.JPEG:n02097209 +ILSVRC2012_val_00046152.JPEG:n03404251 +ILSVRC2012_val_00046153.JPEG:n02488291 +ILSVRC2012_val_00046154.JPEG:n02091032 +ILSVRC2012_val_00046155.JPEG:n01882714 +ILSVRC2012_val_00046156.JPEG:n04081281 +ILSVRC2012_val_00046157.JPEG:n02963159 +ILSVRC2012_val_00046158.JPEG:n02088632 +ILSVRC2012_val_00046159.JPEG:n01491361 +ILSVRC2012_val_00046160.JPEG:n04380533 +ILSVRC2012_val_00046161.JPEG:n04423845 +ILSVRC2012_val_00046162.JPEG:n01629819 +ILSVRC2012_val_00046163.JPEG:n03956157 +ILSVRC2012_val_00046164.JPEG:n04548362 +ILSVRC2012_val_00046165.JPEG:n02804610 +ILSVRC2012_val_00046166.JPEG:n04310018 +ILSVRC2012_val_00046167.JPEG:n04251144 +ILSVRC2012_val_00046168.JPEG:n07860988 +ILSVRC2012_val_00046169.JPEG:n02692877 +ILSVRC2012_val_00046170.JPEG:n03938244 +ILSVRC2012_val_00046171.JPEG:n01484850 +ILSVRC2012_val_00046172.JPEG:n04325704 +ILSVRC2012_val_00046173.JPEG:n01560419 +ILSVRC2012_val_00046174.JPEG:n02916936 +ILSVRC2012_val_00046175.JPEG:n02442845 +ILSVRC2012_val_00046176.JPEG:n03998194 +ILSVRC2012_val_00046177.JPEG:n04330267 +ILSVRC2012_val_00046178.JPEG:n03425413 +ILSVRC2012_val_00046179.JPEG:n07932039 +ILSVRC2012_val_00046180.JPEG:n01984695 +ILSVRC2012_val_00046181.JPEG:n03345487 +ILSVRC2012_val_00046182.JPEG:n03259280 +ILSVRC2012_val_00046183.JPEG:n07768694 +ILSVRC2012_val_00046184.JPEG:n02444819 +ILSVRC2012_val_00046185.JPEG:n01675722 +ILSVRC2012_val_00046186.JPEG:n02328150 +ILSVRC2012_val_00046187.JPEG:n04070727 +ILSVRC2012_val_00046188.JPEG:n04423845 +ILSVRC2012_val_00046189.JPEG:n03729826 +ILSVRC2012_val_00046190.JPEG:n07684084 +ILSVRC2012_val_00046191.JPEG:n03485794 +ILSVRC2012_val_00046192.JPEG:n03498962 +ILSVRC2012_val_00046193.JPEG:n01753488 +ILSVRC2012_val_00046194.JPEG:n03958227 +ILSVRC2012_val_00046195.JPEG:n02895154 +ILSVRC2012_val_00046196.JPEG:n03100240 +ILSVRC2012_val_00046197.JPEG:n02110806 +ILSVRC2012_val_00046198.JPEG:n04118776 +ILSVRC2012_val_00046199.JPEG:n02105056 +ILSVRC2012_val_00046200.JPEG:n03874293 +ILSVRC2012_val_00046201.JPEG:n04037443 +ILSVRC2012_val_00046202.JPEG:n03496892 +ILSVRC2012_val_00046203.JPEG:n07745940 +ILSVRC2012_val_00046204.JPEG:n03871628 +ILSVRC2012_val_00046205.JPEG:n03372029 +ILSVRC2012_val_00046206.JPEG:n02100735 +ILSVRC2012_val_00046207.JPEG:n02132136 +ILSVRC2012_val_00046208.JPEG:n03623198 +ILSVRC2012_val_00046209.JPEG:n03666591 +ILSVRC2012_val_00046210.JPEG:n02823750 +ILSVRC2012_val_00046211.JPEG:n01735189 +ILSVRC2012_val_00046212.JPEG:n02106382 +ILSVRC2012_val_00046213.JPEG:n07697537 +ILSVRC2012_val_00046214.JPEG:n02454379 +ILSVRC2012_val_00046215.JPEG:n04311004 +ILSVRC2012_val_00046216.JPEG:n03110669 +ILSVRC2012_val_00046217.JPEG:n04009552 +ILSVRC2012_val_00046218.JPEG:n02074367 +ILSVRC2012_val_00046219.JPEG:n02442845 +ILSVRC2012_val_00046220.JPEG:n02099601 +ILSVRC2012_val_00046221.JPEG:n09246464 +ILSVRC2012_val_00046222.JPEG:n03814906 +ILSVRC2012_val_00046223.JPEG:n04049303 +ILSVRC2012_val_00046224.JPEG:n01749939 +ILSVRC2012_val_00046225.JPEG:n03803284 +ILSVRC2012_val_00046226.JPEG:n02667093 +ILSVRC2012_val_00046227.JPEG:n03908714 +ILSVRC2012_val_00046228.JPEG:n04409515 +ILSVRC2012_val_00046229.JPEG:n03290653 +ILSVRC2012_val_00046230.JPEG:n07730033 +ILSVRC2012_val_00046231.JPEG:n02268443 +ILSVRC2012_val_00046232.JPEG:n03028079 +ILSVRC2012_val_00046233.JPEG:n02514041 +ILSVRC2012_val_00046234.JPEG:n04592741 +ILSVRC2012_val_00046235.JPEG:n07720875 +ILSVRC2012_val_00046236.JPEG:n02988304 +ILSVRC2012_val_00046237.JPEG:n02606052 +ILSVRC2012_val_00046238.JPEG:n03877472 +ILSVRC2012_val_00046239.JPEG:n01798484 +ILSVRC2012_val_00046240.JPEG:n03742115 +ILSVRC2012_val_00046241.JPEG:n04461696 +ILSVRC2012_val_00046242.JPEG:n02917067 +ILSVRC2012_val_00046243.JPEG:n01629819 +ILSVRC2012_val_00046244.JPEG:n04486054 +ILSVRC2012_val_00046245.JPEG:n04548362 +ILSVRC2012_val_00046246.JPEG:n02860847 +ILSVRC2012_val_00046247.JPEG:n02107683 +ILSVRC2012_val_00046248.JPEG:n01944390 +ILSVRC2012_val_00046249.JPEG:n03786901 +ILSVRC2012_val_00046250.JPEG:n04044716 +ILSVRC2012_val_00046251.JPEG:n01824575 +ILSVRC2012_val_00046252.JPEG:n01440764 +ILSVRC2012_val_00046253.JPEG:n02279972 +ILSVRC2012_val_00046254.JPEG:n01914609 +ILSVRC2012_val_00046255.JPEG:n03272562 +ILSVRC2012_val_00046256.JPEG:n07590611 +ILSVRC2012_val_00046257.JPEG:n01728572 +ILSVRC2012_val_00046258.JPEG:n01687978 +ILSVRC2012_val_00046259.JPEG:n03791053 +ILSVRC2012_val_00046260.JPEG:n01518878 +ILSVRC2012_val_00046261.JPEG:n02950826 +ILSVRC2012_val_00046262.JPEG:n03982430 +ILSVRC2012_val_00046263.JPEG:n02966193 +ILSVRC2012_val_00046264.JPEG:n03841143 +ILSVRC2012_val_00046265.JPEG:n02672831 +ILSVRC2012_val_00046266.JPEG:n02787622 +ILSVRC2012_val_00046267.JPEG:n02165105 +ILSVRC2012_val_00046268.JPEG:n04525038 +ILSVRC2012_val_00046269.JPEG:n03662601 +ILSVRC2012_val_00046270.JPEG:n12057211 +ILSVRC2012_val_00046271.JPEG:n04522168 +ILSVRC2012_val_00046272.JPEG:n04613696 +ILSVRC2012_val_00046273.JPEG:n02088632 +ILSVRC2012_val_00046274.JPEG:n01985128 +ILSVRC2012_val_00046275.JPEG:n09472597 +ILSVRC2012_val_00046276.JPEG:n03271574 +ILSVRC2012_val_00046277.JPEG:n01687978 +ILSVRC2012_val_00046278.JPEG:n04147183 +ILSVRC2012_val_00046279.JPEG:n07875152 +ILSVRC2012_val_00046280.JPEG:n01580077 +ILSVRC2012_val_00046281.JPEG:n03393912 +ILSVRC2012_val_00046282.JPEG:n03903868 +ILSVRC2012_val_00046283.JPEG:n04074963 +ILSVRC2012_val_00046284.JPEG:n03788365 +ILSVRC2012_val_00046285.JPEG:n01843065 +ILSVRC2012_val_00046286.JPEG:n03690938 +ILSVRC2012_val_00046287.JPEG:n02105056 +ILSVRC2012_val_00046288.JPEG:n04525305 +ILSVRC2012_val_00046289.JPEG:n01631663 +ILSVRC2012_val_00046290.JPEG:n02097047 +ILSVRC2012_val_00046291.JPEG:n02486410 +ILSVRC2012_val_00046292.JPEG:n04152593 +ILSVRC2012_val_00046293.JPEG:n02879718 +ILSVRC2012_val_00046294.JPEG:n04443257 +ILSVRC2012_val_00046295.JPEG:n02102040 +ILSVRC2012_val_00046296.JPEG:n02093859 +ILSVRC2012_val_00046297.JPEG:n02127052 +ILSVRC2012_val_00046298.JPEG:n09332890 +ILSVRC2012_val_00046299.JPEG:n01770393 +ILSVRC2012_val_00046300.JPEG:n03527444 +ILSVRC2012_val_00046301.JPEG:n03697007 +ILSVRC2012_val_00046302.JPEG:n04515003 +ILSVRC2012_val_00046303.JPEG:n07873807 +ILSVRC2012_val_00046304.JPEG:n04429376 +ILSVRC2012_val_00046305.JPEG:n03991062 +ILSVRC2012_val_00046306.JPEG:n03085013 +ILSVRC2012_val_00046307.JPEG:n01828970 +ILSVRC2012_val_00046308.JPEG:n01608432 +ILSVRC2012_val_00046309.JPEG:n03930313 +ILSVRC2012_val_00046310.JPEG:n02105641 +ILSVRC2012_val_00046311.JPEG:n01756291 +ILSVRC2012_val_00046312.JPEG:n02500267 +ILSVRC2012_val_00046313.JPEG:n04039381 +ILSVRC2012_val_00046314.JPEG:n02168699 +ILSVRC2012_val_00046315.JPEG:n03259280 +ILSVRC2012_val_00046316.JPEG:n01855032 +ILSVRC2012_val_00046317.JPEG:n10565667 +ILSVRC2012_val_00046318.JPEG:n02115641 +ILSVRC2012_val_00046319.JPEG:n04515003 +ILSVRC2012_val_00046320.JPEG:n02669723 +ILSVRC2012_val_00046321.JPEG:n02988304 +ILSVRC2012_val_00046322.JPEG:n03825788 +ILSVRC2012_val_00046323.JPEG:n02025239 +ILSVRC2012_val_00046324.JPEG:n03706229 +ILSVRC2012_val_00046325.JPEG:n01914609 +ILSVRC2012_val_00046326.JPEG:n03344393 +ILSVRC2012_val_00046327.JPEG:n04049303 +ILSVRC2012_val_00046328.JPEG:n03259280 +ILSVRC2012_val_00046329.JPEG:n02091244 +ILSVRC2012_val_00046330.JPEG:n02514041 +ILSVRC2012_val_00046331.JPEG:n03065424 +ILSVRC2012_val_00046332.JPEG:n12057211 +ILSVRC2012_val_00046333.JPEG:n02027492 +ILSVRC2012_val_00046334.JPEG:n04118538 +ILSVRC2012_val_00046335.JPEG:n04141076 +ILSVRC2012_val_00046336.JPEG:n03899768 +ILSVRC2012_val_00046337.JPEG:n04462240 +ILSVRC2012_val_00046338.JPEG:n02096051 +ILSVRC2012_val_00046339.JPEG:n02978881 +ILSVRC2012_val_00046340.JPEG:n02114855 +ILSVRC2012_val_00046341.JPEG:n04509417 +ILSVRC2012_val_00046342.JPEG:n04505470 +ILSVRC2012_val_00046343.JPEG:n03201208 +ILSVRC2012_val_00046344.JPEG:n01986214 +ILSVRC2012_val_00046345.JPEG:n02417914 +ILSVRC2012_val_00046346.JPEG:n01677366 +ILSVRC2012_val_00046347.JPEG:n07747607 +ILSVRC2012_val_00046348.JPEG:n04409515 +ILSVRC2012_val_00046349.JPEG:n01685808 +ILSVRC2012_val_00046350.JPEG:n04599235 +ILSVRC2012_val_00046351.JPEG:n03187595 +ILSVRC2012_val_00046352.JPEG:n03657121 +ILSVRC2012_val_00046353.JPEG:n15075141 +ILSVRC2012_val_00046354.JPEG:n04372370 +ILSVRC2012_val_00046355.JPEG:n02966687 +ILSVRC2012_val_00046356.JPEG:n01820546 +ILSVRC2012_val_00046357.JPEG:n03344393 +ILSVRC2012_val_00046358.JPEG:n03476991 +ILSVRC2012_val_00046359.JPEG:n03763968 +ILSVRC2012_val_00046360.JPEG:n04070727 +ILSVRC2012_val_00046361.JPEG:n03041632 +ILSVRC2012_val_00046362.JPEG:n01877812 +ILSVRC2012_val_00046363.JPEG:n07248320 +ILSVRC2012_val_00046364.JPEG:n07875152 +ILSVRC2012_val_00046365.JPEG:n02892767 +ILSVRC2012_val_00046366.JPEG:n03355925 +ILSVRC2012_val_00046367.JPEG:n01685808 +ILSVRC2012_val_00046368.JPEG:n04228054 +ILSVRC2012_val_00046369.JPEG:n03843555 +ILSVRC2012_val_00046370.JPEG:n01755581 +ILSVRC2012_val_00046371.JPEG:n04347754 +ILSVRC2012_val_00046372.JPEG:n02277742 +ILSVRC2012_val_00046373.JPEG:n03000247 +ILSVRC2012_val_00046374.JPEG:n07742313 +ILSVRC2012_val_00046375.JPEG:n07875152 +ILSVRC2012_val_00046376.JPEG:n03075370 +ILSVRC2012_val_00046377.JPEG:n02799071 +ILSVRC2012_val_00046378.JPEG:n03133878 +ILSVRC2012_val_00046379.JPEG:n06596364 +ILSVRC2012_val_00046380.JPEG:n01806143 +ILSVRC2012_val_00046381.JPEG:n03930313 +ILSVRC2012_val_00046382.JPEG:n03930313 +ILSVRC2012_val_00046383.JPEG:n02730930 +ILSVRC2012_val_00046384.JPEG:n01773797 +ILSVRC2012_val_00046385.JPEG:n03902125 +ILSVRC2012_val_00046386.JPEG:n03721384 +ILSVRC2012_val_00046387.JPEG:n02951358 +ILSVRC2012_val_00046388.JPEG:n02119022 +ILSVRC2012_val_00046389.JPEG:n01744401 +ILSVRC2012_val_00046390.JPEG:n02112706 +ILSVRC2012_val_00046391.JPEG:n02396427 +ILSVRC2012_val_00046392.JPEG:n03633091 +ILSVRC2012_val_00046393.JPEG:n01514668 +ILSVRC2012_val_00046394.JPEG:n03791053 +ILSVRC2012_val_00046395.JPEG:n02395406 +ILSVRC2012_val_00046396.JPEG:n04370456 +ILSVRC2012_val_00046397.JPEG:n03657121 +ILSVRC2012_val_00046398.JPEG:n02096585 +ILSVRC2012_val_00046399.JPEG:n02107312 +ILSVRC2012_val_00046400.JPEG:n03970156 +ILSVRC2012_val_00046401.JPEG:n03126707 +ILSVRC2012_val_00046402.JPEG:n02105251 +ILSVRC2012_val_00046403.JPEG:n02442845 +ILSVRC2012_val_00046404.JPEG:n04461696 +ILSVRC2012_val_00046405.JPEG:n07715103 +ILSVRC2012_val_00046406.JPEG:n03873416 +ILSVRC2012_val_00046407.JPEG:n01677366 +ILSVRC2012_val_00046408.JPEG:n02012849 +ILSVRC2012_val_00046409.JPEG:n03527444 +ILSVRC2012_val_00046410.JPEG:n01798484 +ILSVRC2012_val_00046411.JPEG:n04562935 +ILSVRC2012_val_00046412.JPEG:n02279972 +ILSVRC2012_val_00046413.JPEG:n02423022 +ILSVRC2012_val_00046414.JPEG:n03992509 +ILSVRC2012_val_00046415.JPEG:n01592084 +ILSVRC2012_val_00046416.JPEG:n03788195 +ILSVRC2012_val_00046417.JPEG:n02259212 +ILSVRC2012_val_00046418.JPEG:n04462240 +ILSVRC2012_val_00046419.JPEG:n03929660 +ILSVRC2012_val_00046420.JPEG:n02090622 +ILSVRC2012_val_00046421.JPEG:n04254120 +ILSVRC2012_val_00046422.JPEG:n01592084 +ILSVRC2012_val_00046423.JPEG:n02109961 +ILSVRC2012_val_00046424.JPEG:n03769881 +ILSVRC2012_val_00046425.JPEG:n02268443 +ILSVRC2012_val_00046426.JPEG:n02909870 +ILSVRC2012_val_00046427.JPEG:n01641577 +ILSVRC2012_val_00046428.JPEG:n04550184 +ILSVRC2012_val_00046429.JPEG:n04507155 +ILSVRC2012_val_00046430.JPEG:n01630670 +ILSVRC2012_val_00046431.JPEG:n04152593 +ILSVRC2012_val_00046432.JPEG:n02090379 +ILSVRC2012_val_00046433.JPEG:n01983481 +ILSVRC2012_val_00046434.JPEG:n09421951 +ILSVRC2012_val_00046435.JPEG:n04517823 +ILSVRC2012_val_00046436.JPEG:n01744401 +ILSVRC2012_val_00046437.JPEG:n07745940 +ILSVRC2012_val_00046438.JPEG:n01843383 +ILSVRC2012_val_00046439.JPEG:n03476684 +ILSVRC2012_val_00046440.JPEG:n01735189 +ILSVRC2012_val_00046441.JPEG:n03930313 +ILSVRC2012_val_00046442.JPEG:n03916031 +ILSVRC2012_val_00046443.JPEG:n02093991 +ILSVRC2012_val_00046444.JPEG:n03207743 +ILSVRC2012_val_00046445.JPEG:n02787622 +ILSVRC2012_val_00046446.JPEG:n02106166 +ILSVRC2012_val_00046447.JPEG:n04398044 +ILSVRC2012_val_00046448.JPEG:n04428191 +ILSVRC2012_val_00046449.JPEG:n04209133 +ILSVRC2012_val_00046450.JPEG:n02085620 +ILSVRC2012_val_00046451.JPEG:n09835506 +ILSVRC2012_val_00046452.JPEG:n01871265 +ILSVRC2012_val_00046453.JPEG:n03459775 +ILSVRC2012_val_00046454.JPEG:n02089973 +ILSVRC2012_val_00046455.JPEG:n02643566 +ILSVRC2012_val_00046456.JPEG:n02481823 +ILSVRC2012_val_00046457.JPEG:n02123159 +ILSVRC2012_val_00046458.JPEG:n07875152 +ILSVRC2012_val_00046459.JPEG:n04557648 +ILSVRC2012_val_00046460.JPEG:n03196217 +ILSVRC2012_val_00046461.JPEG:n04033995 +ILSVRC2012_val_00046462.JPEG:n02037110 +ILSVRC2012_val_00046463.JPEG:n01955084 +ILSVRC2012_val_00046464.JPEG:n03089624 +ILSVRC2012_val_00046465.JPEG:n01751748 +ILSVRC2012_val_00046466.JPEG:n02099429 +ILSVRC2012_val_00046467.JPEG:n03325584 +ILSVRC2012_val_00046468.JPEG:n03445777 +ILSVRC2012_val_00046469.JPEG:n03902125 +ILSVRC2012_val_00046470.JPEG:n02116738 +ILSVRC2012_val_00046471.JPEG:n02799071 +ILSVRC2012_val_00046472.JPEG:n02843684 +ILSVRC2012_val_00046473.JPEG:n03109150 +ILSVRC2012_val_00046474.JPEG:n02869837 +ILSVRC2012_val_00046475.JPEG:n06794110 +ILSVRC2012_val_00046476.JPEG:n03908618 +ILSVRC2012_val_00046477.JPEG:n02105251 +ILSVRC2012_val_00046478.JPEG:n02790996 +ILSVRC2012_val_00046479.JPEG:n02966687 +ILSVRC2012_val_00046480.JPEG:n09256479 +ILSVRC2012_val_00046481.JPEG:n02939185 +ILSVRC2012_val_00046482.JPEG:n04417672 +ILSVRC2012_val_00046483.JPEG:n02113624 +ILSVRC2012_val_00046484.JPEG:n04266014 +ILSVRC2012_val_00046485.JPEG:n02174001 +ILSVRC2012_val_00046486.JPEG:n02483362 +ILSVRC2012_val_00046487.JPEG:n03127925 +ILSVRC2012_val_00046488.JPEG:n03717622 +ILSVRC2012_val_00046489.JPEG:n01744401 +ILSVRC2012_val_00046490.JPEG:n01739381 +ILSVRC2012_val_00046491.JPEG:n02606052 +ILSVRC2012_val_00046492.JPEG:n03290653 +ILSVRC2012_val_00046493.JPEG:n04330267 +ILSVRC2012_val_00046494.JPEG:n02486410 +ILSVRC2012_val_00046495.JPEG:n02457408 +ILSVRC2012_val_00046496.JPEG:n04355338 +ILSVRC2012_val_00046497.JPEG:n01498041 +ILSVRC2012_val_00046498.JPEG:n02134418 +ILSVRC2012_val_00046499.JPEG:n01440764 +ILSVRC2012_val_00046500.JPEG:n04552348 +ILSVRC2012_val_00046501.JPEG:n02319095 +ILSVRC2012_val_00046502.JPEG:n03781244 +ILSVRC2012_val_00046503.JPEG:n07730033 +ILSVRC2012_val_00046504.JPEG:n04525038 +ILSVRC2012_val_00046505.JPEG:n02018795 +ILSVRC2012_val_00046506.JPEG:n03494278 +ILSVRC2012_val_00046507.JPEG:n04589890 +ILSVRC2012_val_00046508.JPEG:n01829413 +ILSVRC2012_val_00046509.JPEG:n04456115 +ILSVRC2012_val_00046510.JPEG:n04118776 +ILSVRC2012_val_00046511.JPEG:n02687172 +ILSVRC2012_val_00046512.JPEG:n02992529 +ILSVRC2012_val_00046513.JPEG:n07932039 +ILSVRC2012_val_00046514.JPEG:n03075370 +ILSVRC2012_val_00046515.JPEG:n04557648 +ILSVRC2012_val_00046516.JPEG:n01728920 +ILSVRC2012_val_00046517.JPEG:n01688243 +ILSVRC2012_val_00046518.JPEG:n02443484 +ILSVRC2012_val_00046519.JPEG:n03843555 +ILSVRC2012_val_00046520.JPEG:n03786901 +ILSVRC2012_val_00046521.JPEG:n03016953 +ILSVRC2012_val_00046522.JPEG:n02536864 +ILSVRC2012_val_00046523.JPEG:n04125021 +ILSVRC2012_val_00046524.JPEG:n01514668 +ILSVRC2012_val_00046525.JPEG:n04461696 +ILSVRC2012_val_00046526.JPEG:n01983481 +ILSVRC2012_val_00046527.JPEG:n02493509 +ILSVRC2012_val_00046528.JPEG:n07614500 +ILSVRC2012_val_00046529.JPEG:n01776313 +ILSVRC2012_val_00046530.JPEG:n02091467 +ILSVRC2012_val_00046531.JPEG:n02106030 +ILSVRC2012_val_00046532.JPEG:n02814860 +ILSVRC2012_val_00046533.JPEG:n02002556 +ILSVRC2012_val_00046534.JPEG:n01818515 +ILSVRC2012_val_00046535.JPEG:n03160309 +ILSVRC2012_val_00046536.JPEG:n02092339 +ILSVRC2012_val_00046537.JPEG:n02013706 +ILSVRC2012_val_00046538.JPEG:n01753488 +ILSVRC2012_val_00046539.JPEG:n01739381 +ILSVRC2012_val_00046540.JPEG:n02981792 +ILSVRC2012_val_00046541.JPEG:n01753488 +ILSVRC2012_val_00046542.JPEG:n02704792 +ILSVRC2012_val_00046543.JPEG:n09332890 +ILSVRC2012_val_00046544.JPEG:n02317335 +ILSVRC2012_val_00046545.JPEG:n03255030 +ILSVRC2012_val_00046546.JPEG:n04201297 +ILSVRC2012_val_00046547.JPEG:n02093256 +ILSVRC2012_val_00046548.JPEG:n01688243 +ILSVRC2012_val_00046549.JPEG:n03792782 +ILSVRC2012_val_00046550.JPEG:n03028079 +ILSVRC2012_val_00046551.JPEG:n01944390 +ILSVRC2012_val_00046552.JPEG:n02107908 +ILSVRC2012_val_00046553.JPEG:n03803284 +ILSVRC2012_val_00046554.JPEG:n03775546 +ILSVRC2012_val_00046555.JPEG:n02128757 +ILSVRC2012_val_00046556.JPEG:n04542943 +ILSVRC2012_val_00046557.JPEG:n04560804 +ILSVRC2012_val_00046558.JPEG:n02514041 +ILSVRC2012_val_00046559.JPEG:n04204347 +ILSVRC2012_val_00046560.JPEG:n02916936 +ILSVRC2012_val_00046561.JPEG:n03344393 +ILSVRC2012_val_00046562.JPEG:n02364673 +ILSVRC2012_val_00046563.JPEG:n03942813 +ILSVRC2012_val_00046564.JPEG:n01614925 +ILSVRC2012_val_00046565.JPEG:n02494079 +ILSVRC2012_val_00046566.JPEG:n04542943 +ILSVRC2012_val_00046567.JPEG:n07742313 +ILSVRC2012_val_00046568.JPEG:n02490219 +ILSVRC2012_val_00046569.JPEG:n03843555 +ILSVRC2012_val_00046570.JPEG:n02281406 +ILSVRC2012_val_00046571.JPEG:n02493793 +ILSVRC2012_val_00046572.JPEG:n02123597 +ILSVRC2012_val_00046573.JPEG:n04613696 +ILSVRC2012_val_00046574.JPEG:n01796340 +ILSVRC2012_val_00046575.JPEG:n07753592 +ILSVRC2012_val_00046576.JPEG:n03384352 +ILSVRC2012_val_00046577.JPEG:n03916031 +ILSVRC2012_val_00046578.JPEG:n03908714 +ILSVRC2012_val_00046579.JPEG:n03992509 +ILSVRC2012_val_00046580.JPEG:n04201297 +ILSVRC2012_val_00046581.JPEG:n03637318 +ILSVRC2012_val_00046582.JPEG:n02977058 +ILSVRC2012_val_00046583.JPEG:n02091032 +ILSVRC2012_val_00046584.JPEG:n02494079 +ILSVRC2012_val_00046585.JPEG:n03673027 +ILSVRC2012_val_00046586.JPEG:n04548362 +ILSVRC2012_val_00046587.JPEG:n01950731 +ILSVRC2012_val_00046588.JPEG:n03721384 +ILSVRC2012_val_00046589.JPEG:n02999410 +ILSVRC2012_val_00046590.JPEG:n02483362 +ILSVRC2012_val_00046591.JPEG:n02111277 +ILSVRC2012_val_00046592.JPEG:n03709823 +ILSVRC2012_val_00046593.JPEG:n02087046 +ILSVRC2012_val_00046594.JPEG:n03929660 +ILSVRC2012_val_00046595.JPEG:n07930864 +ILSVRC2012_val_00046596.JPEG:n03954731 +ILSVRC2012_val_00046597.JPEG:n03063599 +ILSVRC2012_val_00046598.JPEG:n03692522 +ILSVRC2012_val_00046599.JPEG:n02018207 +ILSVRC2012_val_00046600.JPEG:n03788195 +ILSVRC2012_val_00046601.JPEG:n04040759 +ILSVRC2012_val_00046602.JPEG:n02011460 +ILSVRC2012_val_00046603.JPEG:n07871810 +ILSVRC2012_val_00046604.JPEG:n03690938 +ILSVRC2012_val_00046605.JPEG:n04486054 +ILSVRC2012_val_00046606.JPEG:n01986214 +ILSVRC2012_val_00046607.JPEG:n04591713 +ILSVRC2012_val_00046608.JPEG:n04127249 +ILSVRC2012_val_00046609.JPEG:n01807496 +ILSVRC2012_val_00046610.JPEG:n02095570 +ILSVRC2012_val_00046611.JPEG:n01981276 +ILSVRC2012_val_00046612.JPEG:n02128925 +ILSVRC2012_val_00046613.JPEG:n02992529 +ILSVRC2012_val_00046614.JPEG:n02815834 +ILSVRC2012_val_00046615.JPEG:n01698640 +ILSVRC2012_val_00046616.JPEG:n01632458 +ILSVRC2012_val_00046617.JPEG:n02492660 +ILSVRC2012_val_00046618.JPEG:n02319095 +ILSVRC2012_val_00046619.JPEG:n03938244 +ILSVRC2012_val_00046620.JPEG:n03876231 +ILSVRC2012_val_00046621.JPEG:n01798484 +ILSVRC2012_val_00046622.JPEG:n03666591 +ILSVRC2012_val_00046623.JPEG:n02110806 +ILSVRC2012_val_00046624.JPEG:n03782006 +ILSVRC2012_val_00046625.JPEG:n01943899 +ILSVRC2012_val_00046626.JPEG:n02643566 +ILSVRC2012_val_00046627.JPEG:n04120489 +ILSVRC2012_val_00046628.JPEG:n04399382 +ILSVRC2012_val_00046629.JPEG:n02085782 +ILSVRC2012_val_00046630.JPEG:n04389033 +ILSVRC2012_val_00046631.JPEG:n07714571 +ILSVRC2012_val_00046632.JPEG:n01614925 +ILSVRC2012_val_00046633.JPEG:n03494278 +ILSVRC2012_val_00046634.JPEG:n04141076 +ILSVRC2012_val_00046635.JPEG:n03388043 +ILSVRC2012_val_00046636.JPEG:n04118776 +ILSVRC2012_val_00046637.JPEG:n03291819 +ILSVRC2012_val_00046638.JPEG:n02389026 +ILSVRC2012_val_00046639.JPEG:n04209133 +ILSVRC2012_val_00046640.JPEG:n01685808 +ILSVRC2012_val_00046641.JPEG:n03769881 +ILSVRC2012_val_00046642.JPEG:n04074963 +ILSVRC2012_val_00046643.JPEG:n04458633 +ILSVRC2012_val_00046644.JPEG:n04532670 +ILSVRC2012_val_00046645.JPEG:n02484975 +ILSVRC2012_val_00046646.JPEG:n07579787 +ILSVRC2012_val_00046647.JPEG:n02058221 +ILSVRC2012_val_00046648.JPEG:n03000134 +ILSVRC2012_val_00046649.JPEG:n01704323 +ILSVRC2012_val_00046650.JPEG:n04044716 +ILSVRC2012_val_00046651.JPEG:n03000684 +ILSVRC2012_val_00046652.JPEG:n03179701 +ILSVRC2012_val_00046653.JPEG:n07716906 +ILSVRC2012_val_00046654.JPEG:n01518878 +ILSVRC2012_val_00046655.JPEG:n02497673 +ILSVRC2012_val_00046656.JPEG:n03445924 +ILSVRC2012_val_00046657.JPEG:n02093647 +ILSVRC2012_val_00046658.JPEG:n02410509 +ILSVRC2012_val_00046659.JPEG:n03026506 +ILSVRC2012_val_00046660.JPEG:n04153751 +ILSVRC2012_val_00046661.JPEG:n04141076 +ILSVRC2012_val_00046662.JPEG:n03532672 +ILSVRC2012_val_00046663.JPEG:n04201297 +ILSVRC2012_val_00046664.JPEG:n07836838 +ILSVRC2012_val_00046665.JPEG:n03188531 +ILSVRC2012_val_00046666.JPEG:n02486410 +ILSVRC2012_val_00046667.JPEG:n04275548 +ILSVRC2012_val_00046668.JPEG:n02133161 +ILSVRC2012_val_00046669.JPEG:n03394916 +ILSVRC2012_val_00046670.JPEG:n02098105 +ILSVRC2012_val_00046671.JPEG:n04376876 +ILSVRC2012_val_00046672.JPEG:n02106382 +ILSVRC2012_val_00046673.JPEG:n03483316 +ILSVRC2012_val_00046674.JPEG:n02490219 +ILSVRC2012_val_00046675.JPEG:n03032252 +ILSVRC2012_val_00046676.JPEG:n03770439 +ILSVRC2012_val_00046677.JPEG:n02025239 +ILSVRC2012_val_00046678.JPEG:n03840681 +ILSVRC2012_val_00046679.JPEG:n03496892 +ILSVRC2012_val_00046680.JPEG:n03633091 +ILSVRC2012_val_00046681.JPEG:n02837789 +ILSVRC2012_val_00046682.JPEG:n03126707 +ILSVRC2012_val_00046683.JPEG:n02104365 +ILSVRC2012_val_00046684.JPEG:n04584207 +ILSVRC2012_val_00046685.JPEG:n04347754 +ILSVRC2012_val_00046686.JPEG:n04243546 +ILSVRC2012_val_00046687.JPEG:n02110185 +ILSVRC2012_val_00046688.JPEG:n02865351 +ILSVRC2012_val_00046689.JPEG:n02167151 +ILSVRC2012_val_00046690.JPEG:n02871525 +ILSVRC2012_val_00046691.JPEG:n02088466 +ILSVRC2012_val_00046692.JPEG:n02138441 +ILSVRC2012_val_00046693.JPEG:n02804610 +ILSVRC2012_val_00046694.JPEG:n03935335 +ILSVRC2012_val_00046695.JPEG:n02782093 +ILSVRC2012_val_00046696.JPEG:n01744401 +ILSVRC2012_val_00046697.JPEG:n09472597 +ILSVRC2012_val_00046698.JPEG:n03445924 +ILSVRC2012_val_00046699.JPEG:n01737021 +ILSVRC2012_val_00046700.JPEG:n02102480 +ILSVRC2012_val_00046701.JPEG:n02086646 +ILSVRC2012_val_00046702.JPEG:n02137549 +ILSVRC2012_val_00046703.JPEG:n02481823 +ILSVRC2012_val_00046704.JPEG:n02107574 +ILSVRC2012_val_00046705.JPEG:n02096437 +ILSVRC2012_val_00046706.JPEG:n02701002 +ILSVRC2012_val_00046707.JPEG:n03272562 +ILSVRC2012_val_00046708.JPEG:n02978881 +ILSVRC2012_val_00046709.JPEG:n01737021 +ILSVRC2012_val_00046710.JPEG:n01824575 +ILSVRC2012_val_00046711.JPEG:n03887697 +ILSVRC2012_val_00046712.JPEG:n02097298 +ILSVRC2012_val_00046713.JPEG:n03692522 +ILSVRC2012_val_00046714.JPEG:n02437312 +ILSVRC2012_val_00046715.JPEG:n03814639 +ILSVRC2012_val_00046716.JPEG:n02236044 +ILSVRC2012_val_00046717.JPEG:n02094433 +ILSVRC2012_val_00046718.JPEG:n07742313 +ILSVRC2012_val_00046719.JPEG:n04398044 +ILSVRC2012_val_00046720.JPEG:n03255030 +ILSVRC2012_val_00046721.JPEG:n04258138 +ILSVRC2012_val_00046722.JPEG:n02422106 +ILSVRC2012_val_00046723.JPEG:n06785654 +ILSVRC2012_val_00046724.JPEG:n02319095 +ILSVRC2012_val_00046725.JPEG:n03692522 +ILSVRC2012_val_00046726.JPEG:n04350905 +ILSVRC2012_val_00046727.JPEG:n04252077 +ILSVRC2012_val_00046728.JPEG:n03804744 +ILSVRC2012_val_00046729.JPEG:n03131574 +ILSVRC2012_val_00046730.JPEG:n02107312 +ILSVRC2012_val_00046731.JPEG:n07583066 +ILSVRC2012_val_00046732.JPEG:n02006656 +ILSVRC2012_val_00046733.JPEG:n01608432 +ILSVRC2012_val_00046734.JPEG:n04428191 +ILSVRC2012_val_00046735.JPEG:n04346328 +ILSVRC2012_val_00046736.JPEG:n02493793 +ILSVRC2012_val_00046737.JPEG:n04040759 +ILSVRC2012_val_00046738.JPEG:n03733281 +ILSVRC2012_val_00046739.JPEG:n02093754 +ILSVRC2012_val_00046740.JPEG:n01677366 +ILSVRC2012_val_00046741.JPEG:n02481823 +ILSVRC2012_val_00046742.JPEG:n11939491 +ILSVRC2012_val_00046743.JPEG:n13044778 +ILSVRC2012_val_00046744.JPEG:n04070727 +ILSVRC2012_val_00046745.JPEG:n02500267 +ILSVRC2012_val_00046746.JPEG:n03347037 +ILSVRC2012_val_00046747.JPEG:n03942813 +ILSVRC2012_val_00046748.JPEG:n03218198 +ILSVRC2012_val_00046749.JPEG:n02747177 +ILSVRC2012_val_00046750.JPEG:n04286575 +ILSVRC2012_val_00046751.JPEG:n01530575 +ILSVRC2012_val_00046752.JPEG:n02437312 +ILSVRC2012_val_00046753.JPEG:n02090379 +ILSVRC2012_val_00046754.JPEG:n04447861 +ILSVRC2012_val_00046755.JPEG:n01843383 +ILSVRC2012_val_00046756.JPEG:n01629819 +ILSVRC2012_val_00046757.JPEG:n01871265 +ILSVRC2012_val_00046758.JPEG:n02077923 +ILSVRC2012_val_00046759.JPEG:n02105162 +ILSVRC2012_val_00046760.JPEG:n03873416 +ILSVRC2012_val_00046761.JPEG:n02106662 +ILSVRC2012_val_00046762.JPEG:n02096437 +ILSVRC2012_val_00046763.JPEG:n02132136 +ILSVRC2012_val_00046764.JPEG:n03000684 +ILSVRC2012_val_00046765.JPEG:n01917289 +ILSVRC2012_val_00046766.JPEG:n02777292 +ILSVRC2012_val_00046767.JPEG:n02077923 +ILSVRC2012_val_00046768.JPEG:n02110063 +ILSVRC2012_val_00046769.JPEG:n02027492 +ILSVRC2012_val_00046770.JPEG:n02124075 +ILSVRC2012_val_00046771.JPEG:n04467665 +ILSVRC2012_val_00046772.JPEG:n04192698 +ILSVRC2012_val_00046773.JPEG:n04525305 +ILSVRC2012_val_00046774.JPEG:n12057211 +ILSVRC2012_val_00046775.JPEG:n02894605 +ILSVRC2012_val_00046776.JPEG:n02108551 +ILSVRC2012_val_00046777.JPEG:n04392985 +ILSVRC2012_val_00046778.JPEG:n01742172 +ILSVRC2012_val_00046779.JPEG:n02825657 +ILSVRC2012_val_00046780.JPEG:n04336792 +ILSVRC2012_val_00046781.JPEG:n04265275 +ILSVRC2012_val_00046782.JPEG:n02172182 +ILSVRC2012_val_00046783.JPEG:n02483362 +ILSVRC2012_val_00046784.JPEG:n02168699 +ILSVRC2012_val_00046785.JPEG:n02088094 +ILSVRC2012_val_00046786.JPEG:n02128925 +ILSVRC2012_val_00046787.JPEG:n03764736 +ILSVRC2012_val_00046788.JPEG:n02113712 +ILSVRC2012_val_00046789.JPEG:n03197337 +ILSVRC2012_val_00046790.JPEG:n03393912 +ILSVRC2012_val_00046791.JPEG:n03804744 +ILSVRC2012_val_00046792.JPEG:n07697313 +ILSVRC2012_val_00046793.JPEG:n03770679 +ILSVRC2012_val_00046794.JPEG:n02795169 +ILSVRC2012_val_00046795.JPEG:n02104365 +ILSVRC2012_val_00046796.JPEG:n10148035 +ILSVRC2012_val_00046797.JPEG:n01534433 +ILSVRC2012_val_00046798.JPEG:n03089624 +ILSVRC2012_val_00046799.JPEG:n10565667 +ILSVRC2012_val_00046800.JPEG:n04536866 +ILSVRC2012_val_00046801.JPEG:n02259212 +ILSVRC2012_val_00046802.JPEG:n01828970 +ILSVRC2012_val_00046803.JPEG:n01667114 +ILSVRC2012_val_00046804.JPEG:n02110958 +ILSVRC2012_val_00046805.JPEG:n03841143 +ILSVRC2012_val_00046806.JPEG:n03325584 +ILSVRC2012_val_00046807.JPEG:n03450230 +ILSVRC2012_val_00046808.JPEG:n04423845 +ILSVRC2012_val_00046809.JPEG:n04149813 +ILSVRC2012_val_00046810.JPEG:n02802426 +ILSVRC2012_val_00046811.JPEG:n03876231 +ILSVRC2012_val_00046812.JPEG:n03868242 +ILSVRC2012_val_00046813.JPEG:n07614500 +ILSVRC2012_val_00046814.JPEG:n04356056 +ILSVRC2012_val_00046815.JPEG:n02128925 +ILSVRC2012_val_00046816.JPEG:n03379051 +ILSVRC2012_val_00046817.JPEG:n02099712 +ILSVRC2012_val_00046818.JPEG:n02870880 +ILSVRC2012_val_00046819.JPEG:n02085936 +ILSVRC2012_val_00046820.JPEG:n13044778 +ILSVRC2012_val_00046821.JPEG:n03388043 +ILSVRC2012_val_00046822.JPEG:n02113712 +ILSVRC2012_val_00046823.JPEG:n02113624 +ILSVRC2012_val_00046824.JPEG:n03141823 +ILSVRC2012_val_00046825.JPEG:n02110627 +ILSVRC2012_val_00046826.JPEG:n03394916 +ILSVRC2012_val_00046827.JPEG:n04548362 +ILSVRC2012_val_00046828.JPEG:n02927161 +ILSVRC2012_val_00046829.JPEG:n01914609 +ILSVRC2012_val_00046830.JPEG:n04275548 +ILSVRC2012_val_00046831.JPEG:n03271574 +ILSVRC2012_val_00046832.JPEG:n03527444 +ILSVRC2012_val_00046833.JPEG:n01530575 +ILSVRC2012_val_00046834.JPEG:n03775546 +ILSVRC2012_val_00046835.JPEG:n02965783 +ILSVRC2012_val_00046836.JPEG:n02105505 +ILSVRC2012_val_00046837.JPEG:n03982430 +ILSVRC2012_val_00046838.JPEG:n04258138 +ILSVRC2012_val_00046839.JPEG:n03201208 +ILSVRC2012_val_00046840.JPEG:n07684084 +ILSVRC2012_val_00046841.JPEG:n02437616 +ILSVRC2012_val_00046842.JPEG:n03388043 +ILSVRC2012_val_00046843.JPEG:n04389033 +ILSVRC2012_val_00046844.JPEG:n02841315 +ILSVRC2012_val_00046845.JPEG:n03250847 +ILSVRC2012_val_00046846.JPEG:n02480495 +ILSVRC2012_val_00046847.JPEG:n01749939 +ILSVRC2012_val_00046848.JPEG:n12998815 +ILSVRC2012_val_00046849.JPEG:n02114712 +ILSVRC2012_val_00046850.JPEG:n02056570 +ILSVRC2012_val_00046851.JPEG:n03602883 +ILSVRC2012_val_00046852.JPEG:n02281406 +ILSVRC2012_val_00046853.JPEG:n02086079 +ILSVRC2012_val_00046854.JPEG:n03769881 +ILSVRC2012_val_00046855.JPEG:n03791053 +ILSVRC2012_val_00046856.JPEG:n02165456 +ILSVRC2012_val_00046857.JPEG:n02747177 +ILSVRC2012_val_00046858.JPEG:n13040303 +ILSVRC2012_val_00046859.JPEG:n04023962 +ILSVRC2012_val_00046860.JPEG:n02948072 +ILSVRC2012_val_00046861.JPEG:n04243546 +ILSVRC2012_val_00046862.JPEG:n02690373 +ILSVRC2012_val_00046863.JPEG:n04442312 +ILSVRC2012_val_00046864.JPEG:n03837869 +ILSVRC2012_val_00046865.JPEG:n04417672 +ILSVRC2012_val_00046866.JPEG:n13054560 +ILSVRC2012_val_00046867.JPEG:n02106166 +ILSVRC2012_val_00046868.JPEG:n01776313 +ILSVRC2012_val_00046869.JPEG:n02667093 +ILSVRC2012_val_00046870.JPEG:n07565083 +ILSVRC2012_val_00046871.JPEG:n13133613 +ILSVRC2012_val_00046872.JPEG:n07730033 +ILSVRC2012_val_00046873.JPEG:n02488291 +ILSVRC2012_val_00046874.JPEG:n04423845 +ILSVRC2012_val_00046875.JPEG:n03623198 +ILSVRC2012_val_00046876.JPEG:n03977966 +ILSVRC2012_val_00046877.JPEG:n03866082 +ILSVRC2012_val_00046878.JPEG:n02100735 +ILSVRC2012_val_00046879.JPEG:n02834397 +ILSVRC2012_val_00046880.JPEG:n04461696 +ILSVRC2012_val_00046881.JPEG:n02089078 +ILSVRC2012_val_00046882.JPEG:n01694178 +ILSVRC2012_val_00046883.JPEG:n01944390 +ILSVRC2012_val_00046884.JPEG:n03706229 +ILSVRC2012_val_00046885.JPEG:n03223299 +ILSVRC2012_val_00046886.JPEG:n03980874 +ILSVRC2012_val_00046887.JPEG:n03991062 +ILSVRC2012_val_00046888.JPEG:n04004767 +ILSVRC2012_val_00046889.JPEG:n04201297 +ILSVRC2012_val_00046890.JPEG:n03761084 +ILSVRC2012_val_00046891.JPEG:n03443371 +ILSVRC2012_val_00046892.JPEG:n02033041 +ILSVRC2012_val_00046893.JPEG:n02138441 +ILSVRC2012_val_00046894.JPEG:n01924916 +ILSVRC2012_val_00046895.JPEG:n04133789 +ILSVRC2012_val_00046896.JPEG:n06359193 +ILSVRC2012_val_00046897.JPEG:n02091032 +ILSVRC2012_val_00046898.JPEG:n02981792 +ILSVRC2012_val_00046899.JPEG:n03180011 +ILSVRC2012_val_00046900.JPEG:n04522168 +ILSVRC2012_val_00046901.JPEG:n04317175 +ILSVRC2012_val_00046902.JPEG:n02106662 +ILSVRC2012_val_00046903.JPEG:n01847000 +ILSVRC2012_val_00046904.JPEG:n12768682 +ILSVRC2012_val_00046905.JPEG:n03496892 +ILSVRC2012_val_00046906.JPEG:n02892767 +ILSVRC2012_val_00046907.JPEG:n07684084 +ILSVRC2012_val_00046908.JPEG:n01877812 +ILSVRC2012_val_00046909.JPEG:n03345487 +ILSVRC2012_val_00046910.JPEG:n03495258 +ILSVRC2012_val_00046911.JPEG:n03661043 +ILSVRC2012_val_00046912.JPEG:n01990800 +ILSVRC2012_val_00046913.JPEG:n03417042 +ILSVRC2012_val_00046914.JPEG:n04330267 +ILSVRC2012_val_00046915.JPEG:n01443537 +ILSVRC2012_val_00046916.JPEG:n02397096 +ILSVRC2012_val_00046917.JPEG:n01582220 +ILSVRC2012_val_00046918.JPEG:n01910747 +ILSVRC2012_val_00046919.JPEG:n02025239 +ILSVRC2012_val_00046920.JPEG:n03724870 +ILSVRC2012_val_00046921.JPEG:n02787622 +ILSVRC2012_val_00046922.JPEG:n02892201 +ILSVRC2012_val_00046923.JPEG:n02086079 +ILSVRC2012_val_00046924.JPEG:n04417672 +ILSVRC2012_val_00046925.JPEG:n04550184 +ILSVRC2012_val_00046926.JPEG:n04525305 +ILSVRC2012_val_00046927.JPEG:n03877845 +ILSVRC2012_val_00046928.JPEG:n07718472 +ILSVRC2012_val_00046929.JPEG:n04266014 +ILSVRC2012_val_00046930.JPEG:n02396427 +ILSVRC2012_val_00046931.JPEG:n01773797 +ILSVRC2012_val_00046932.JPEG:n02009912 +ILSVRC2012_val_00046933.JPEG:n01795545 +ILSVRC2012_val_00046934.JPEG:n02120079 +ILSVRC2012_val_00046935.JPEG:n02105505 +ILSVRC2012_val_00046936.JPEG:n04252077 +ILSVRC2012_val_00046937.JPEG:n07734744 +ILSVRC2012_val_00046938.JPEG:n02793495 +ILSVRC2012_val_00046939.JPEG:n04372370 +ILSVRC2012_val_00046940.JPEG:n02667093 +ILSVRC2012_val_00046941.JPEG:n01629819 +ILSVRC2012_val_00046942.JPEG:n02493793 +ILSVRC2012_val_00046943.JPEG:n02640242 +ILSVRC2012_val_00046944.JPEG:n01748264 +ILSVRC2012_val_00046945.JPEG:n02134418 +ILSVRC2012_val_00046946.JPEG:n04335435 +ILSVRC2012_val_00046947.JPEG:n02966687 +ILSVRC2012_val_00046948.JPEG:n01608432 +ILSVRC2012_val_00046949.JPEG:n03325584 +ILSVRC2012_val_00046950.JPEG:n02013706 +ILSVRC2012_val_00046951.JPEG:n02364673 +ILSVRC2012_val_00046952.JPEG:n02791124 +ILSVRC2012_val_00046953.JPEG:n02979186 +ILSVRC2012_val_00046954.JPEG:n04493381 +ILSVRC2012_val_00046955.JPEG:n03045698 +ILSVRC2012_val_00046956.JPEG:n03032252 +ILSVRC2012_val_00046957.JPEG:n02092339 +ILSVRC2012_val_00046958.JPEG:n01806143 +ILSVRC2012_val_00046959.JPEG:n03535780 +ILSVRC2012_val_00046960.JPEG:n02319095 +ILSVRC2012_val_00046961.JPEG:n04562935 +ILSVRC2012_val_00046962.JPEG:n01873310 +ILSVRC2012_val_00046963.JPEG:n02279972 +ILSVRC2012_val_00046964.JPEG:n02124075 +ILSVRC2012_val_00046965.JPEG:n03482405 +ILSVRC2012_val_00046966.JPEG:n02056570 +ILSVRC2012_val_00046967.JPEG:n02823750 +ILSVRC2012_val_00046968.JPEG:n02823428 +ILSVRC2012_val_00046969.JPEG:n01443537 +ILSVRC2012_val_00046970.JPEG:n02860847 +ILSVRC2012_val_00046971.JPEG:n02690373 +ILSVRC2012_val_00046972.JPEG:n03825788 +ILSVRC2012_val_00046973.JPEG:n04461696 +ILSVRC2012_val_00046974.JPEG:n02106030 +ILSVRC2012_val_00046975.JPEG:n01983481 +ILSVRC2012_val_00046976.JPEG:n01632777 +ILSVRC2012_val_00046977.JPEG:n04562935 +ILSVRC2012_val_00046978.JPEG:n01847000 +ILSVRC2012_val_00046979.JPEG:n03661043 +ILSVRC2012_val_00046980.JPEG:n03272010 +ILSVRC2012_val_00046981.JPEG:n02113978 +ILSVRC2012_val_00046982.JPEG:n04550184 +ILSVRC2012_val_00046983.JPEG:n02699494 +ILSVRC2012_val_00046984.JPEG:n04505470 +ILSVRC2012_val_00046985.JPEG:n01629819 +ILSVRC2012_val_00046986.JPEG:n03944341 +ILSVRC2012_val_00046987.JPEG:n03792782 +ILSVRC2012_val_00046988.JPEG:n02071294 +ILSVRC2012_val_00046989.JPEG:n02114367 +ILSVRC2012_val_00046990.JPEG:n04536866 +ILSVRC2012_val_00046991.JPEG:n02910353 +ILSVRC2012_val_00046992.JPEG:n03355925 +ILSVRC2012_val_00046993.JPEG:n03908618 +ILSVRC2012_val_00046994.JPEG:n02786058 +ILSVRC2012_val_00046995.JPEG:n02097047 +ILSVRC2012_val_00046996.JPEG:n02088094 +ILSVRC2012_val_00046997.JPEG:n02089867 +ILSVRC2012_val_00046998.JPEG:n04356056 +ILSVRC2012_val_00046999.JPEG:n02095570 +ILSVRC2012_val_00047000.JPEG:n01756291 +ILSVRC2012_val_00047001.JPEG:n02441942 +ILSVRC2012_val_00047002.JPEG:n04208210 +ILSVRC2012_val_00047003.JPEG:n07693725 +ILSVRC2012_val_00047004.JPEG:n02088094 +ILSVRC2012_val_00047005.JPEG:n06596364 +ILSVRC2012_val_00047006.JPEG:n02992529 +ILSVRC2012_val_00047007.JPEG:n04081281 +ILSVRC2012_val_00047008.JPEG:n03467068 +ILSVRC2012_val_00047009.JPEG:n01847000 +ILSVRC2012_val_00047010.JPEG:n01693334 +ILSVRC2012_val_00047011.JPEG:n03680355 +ILSVRC2012_val_00047012.JPEG:n04501370 +ILSVRC2012_val_00047013.JPEG:n03763968 +ILSVRC2012_val_00047014.JPEG:n01917289 +ILSVRC2012_val_00047015.JPEG:n02669723 +ILSVRC2012_val_00047016.JPEG:n01924916 +ILSVRC2012_val_00047017.JPEG:n02110958 +ILSVRC2012_val_00047018.JPEG:n04041544 +ILSVRC2012_val_00047019.JPEG:n02110806 +ILSVRC2012_val_00047020.JPEG:n02134084 +ILSVRC2012_val_00047021.JPEG:n02130308 +ILSVRC2012_val_00047022.JPEG:n02443484 +ILSVRC2012_val_00047023.JPEG:n02843684 +ILSVRC2012_val_00047024.JPEG:n01968897 +ILSVRC2012_val_00047025.JPEG:n01855672 +ILSVRC2012_val_00047026.JPEG:n02113799 +ILSVRC2012_val_00047027.JPEG:n03584829 +ILSVRC2012_val_00047028.JPEG:n12768682 +ILSVRC2012_val_00047029.JPEG:n01531178 +ILSVRC2012_val_00047030.JPEG:n03197337 +ILSVRC2012_val_00047031.JPEG:n01784675 +ILSVRC2012_val_00047032.JPEG:n03075370 +ILSVRC2012_val_00047033.JPEG:n04252077 +ILSVRC2012_val_00047034.JPEG:n03935335 +ILSVRC2012_val_00047035.JPEG:n02999410 +ILSVRC2012_val_00047036.JPEG:n07716358 +ILSVRC2012_val_00047037.JPEG:n04238763 +ILSVRC2012_val_00047038.JPEG:n07753275 +ILSVRC2012_val_00047039.JPEG:n02279972 +ILSVRC2012_val_00047040.JPEG:n02666196 +ILSVRC2012_val_00047041.JPEG:n02007558 +ILSVRC2012_val_00047042.JPEG:n02105251 +ILSVRC2012_val_00047043.JPEG:n02226429 +ILSVRC2012_val_00047044.JPEG:n01751748 +ILSVRC2012_val_00047045.JPEG:n02127052 +ILSVRC2012_val_00047046.JPEG:n04579145 +ILSVRC2012_val_00047047.JPEG:n02051845 +ILSVRC2012_val_00047048.JPEG:n02445715 +ILSVRC2012_val_00047049.JPEG:n02102177 +ILSVRC2012_val_00047050.JPEG:n03759954 +ILSVRC2012_val_00047051.JPEG:n03179701 +ILSVRC2012_val_00047052.JPEG:n02007558 +ILSVRC2012_val_00047053.JPEG:n03649909 +ILSVRC2012_val_00047054.JPEG:n03992509 +ILSVRC2012_val_00047055.JPEG:n03447721 +ILSVRC2012_val_00047056.JPEG:n02916936 +ILSVRC2012_val_00047057.JPEG:n03196217 +ILSVRC2012_val_00047058.JPEG:n01883070 +ILSVRC2012_val_00047059.JPEG:n01983481 +ILSVRC2012_val_00047060.JPEG:n03000684 +ILSVRC2012_val_00047061.JPEG:n01756291 +ILSVRC2012_val_00047062.JPEG:n02111277 +ILSVRC2012_val_00047063.JPEG:n03857828 +ILSVRC2012_val_00047064.JPEG:n04479046 +ILSVRC2012_val_00047065.JPEG:n02177972 +ILSVRC2012_val_00047066.JPEG:n04067472 +ILSVRC2012_val_00047067.JPEG:n03444034 +ILSVRC2012_val_00047068.JPEG:n03854065 +ILSVRC2012_val_00047069.JPEG:n03720891 +ILSVRC2012_val_00047070.JPEG:n04208210 +ILSVRC2012_val_00047071.JPEG:n01740131 +ILSVRC2012_val_00047072.JPEG:n04423845 +ILSVRC2012_val_00047073.JPEG:n01855672 +ILSVRC2012_val_00047074.JPEG:n03388549 +ILSVRC2012_val_00047075.JPEG:n02206856 +ILSVRC2012_val_00047076.JPEG:n04606251 +ILSVRC2012_val_00047077.JPEG:n03887697 +ILSVRC2012_val_00047078.JPEG:n02865351 +ILSVRC2012_val_00047079.JPEG:n04579145 +ILSVRC2012_val_00047080.JPEG:n01496331 +ILSVRC2012_val_00047081.JPEG:n02804414 +ILSVRC2012_val_00047082.JPEG:n02787622 +ILSVRC2012_val_00047083.JPEG:n04004767 +ILSVRC2012_val_00047084.JPEG:n02097047 +ILSVRC2012_val_00047085.JPEG:n02490219 +ILSVRC2012_val_00047086.JPEG:n03529860 +ILSVRC2012_val_00047087.JPEG:n03680355 +ILSVRC2012_val_00047088.JPEG:n03942813 +ILSVRC2012_val_00047089.JPEG:n01632458 +ILSVRC2012_val_00047090.JPEG:n03733281 +ILSVRC2012_val_00047091.JPEG:n03584829 +ILSVRC2012_val_00047092.JPEG:n02797295 +ILSVRC2012_val_00047093.JPEG:n02966687 +ILSVRC2012_val_00047094.JPEG:n01824575 +ILSVRC2012_val_00047095.JPEG:n07831146 +ILSVRC2012_val_00047096.JPEG:n04366367 +ILSVRC2012_val_00047097.JPEG:n03666591 +ILSVRC2012_val_00047098.JPEG:n03788195 +ILSVRC2012_val_00047099.JPEG:n02966193 +ILSVRC2012_val_00047100.JPEG:n03042490 +ILSVRC2012_val_00047101.JPEG:n06874185 +ILSVRC2012_val_00047102.JPEG:n03345487 +ILSVRC2012_val_00047103.JPEG:n02123597 +ILSVRC2012_val_00047104.JPEG:n02895154 +ILSVRC2012_val_00047105.JPEG:n01664065 +ILSVRC2012_val_00047106.JPEG:n01819313 +ILSVRC2012_val_00047107.JPEG:n12985857 +ILSVRC2012_val_00047108.JPEG:n01855672 +ILSVRC2012_val_00047109.JPEG:n02095314 +ILSVRC2012_val_00047110.JPEG:n02102973 +ILSVRC2012_val_00047111.JPEG:n02966193 +ILSVRC2012_val_00047112.JPEG:n02115913 +ILSVRC2012_val_00047113.JPEG:n03590841 +ILSVRC2012_val_00047114.JPEG:n02093991 +ILSVRC2012_val_00047115.JPEG:n02169497 +ILSVRC2012_val_00047116.JPEG:n02814860 +ILSVRC2012_val_00047117.JPEG:n02089078 +ILSVRC2012_val_00047118.JPEG:n02138441 +ILSVRC2012_val_00047119.JPEG:n02113712 +ILSVRC2012_val_00047120.JPEG:n02883205 +ILSVRC2012_val_00047121.JPEG:n01601694 +ILSVRC2012_val_00047122.JPEG:n01774384 +ILSVRC2012_val_00047123.JPEG:n04111531 +ILSVRC2012_val_00047124.JPEG:n03000134 +ILSVRC2012_val_00047125.JPEG:n02088364 +ILSVRC2012_val_00047126.JPEG:n02489166 +ILSVRC2012_val_00047127.JPEG:n01914609 +ILSVRC2012_val_00047128.JPEG:n04009552 +ILSVRC2012_val_00047129.JPEG:n03680355 +ILSVRC2012_val_00047130.JPEG:n03843555 +ILSVRC2012_val_00047131.JPEG:n03950228 +ILSVRC2012_val_00047132.JPEG:n03680355 +ILSVRC2012_val_00047133.JPEG:n04597913 +ILSVRC2012_val_00047134.JPEG:n04347754 +ILSVRC2012_val_00047135.JPEG:n04116512 +ILSVRC2012_val_00047136.JPEG:n02747177 +ILSVRC2012_val_00047137.JPEG:n01514668 +ILSVRC2012_val_00047138.JPEG:n02840245 +ILSVRC2012_val_00047139.JPEG:n03483316 +ILSVRC2012_val_00047140.JPEG:n07715103 +ILSVRC2012_val_00047141.JPEG:n04153751 +ILSVRC2012_val_00047142.JPEG:n02500267 +ILSVRC2012_val_00047143.JPEG:n03998194 +ILSVRC2012_val_00047144.JPEG:n15075141 +ILSVRC2012_val_00047145.JPEG:n03930313 +ILSVRC2012_val_00047146.JPEG:n02112706 +ILSVRC2012_val_00047147.JPEG:n03888257 +ILSVRC2012_val_00047148.JPEG:n02110063 +ILSVRC2012_val_00047149.JPEG:n02108000 +ILSVRC2012_val_00047150.JPEG:n02102973 +ILSVRC2012_val_00047151.JPEG:n02483708 +ILSVRC2012_val_00047152.JPEG:n02097474 +ILSVRC2012_val_00047153.JPEG:n02011460 +ILSVRC2012_val_00047154.JPEG:n02492035 +ILSVRC2012_val_00047155.JPEG:n02814860 +ILSVRC2012_val_00047156.JPEG:n02009229 +ILSVRC2012_val_00047157.JPEG:n03877845 +ILSVRC2012_val_00047158.JPEG:n06596364 +ILSVRC2012_val_00047159.JPEG:n07248320 +ILSVRC2012_val_00047160.JPEG:n04344873 +ILSVRC2012_val_00047161.JPEG:n04536866 +ILSVRC2012_val_00047162.JPEG:n02823750 +ILSVRC2012_val_00047163.JPEG:n03291819 +ILSVRC2012_val_00047164.JPEG:n01770081 +ILSVRC2012_val_00047165.JPEG:n02892767 +ILSVRC2012_val_00047166.JPEG:n03481172 +ILSVRC2012_val_00047167.JPEG:n02066245 +ILSVRC2012_val_00047168.JPEG:n04370456 +ILSVRC2012_val_00047169.JPEG:n02264363 +ILSVRC2012_val_00047170.JPEG:n03670208 +ILSVRC2012_val_00047171.JPEG:n02397096 +ILSVRC2012_val_00047172.JPEG:n03075370 +ILSVRC2012_val_00047173.JPEG:n02087394 +ILSVRC2012_val_00047174.JPEG:n02536864 +ILSVRC2012_val_00047175.JPEG:n04599235 +ILSVRC2012_val_00047176.JPEG:n03982430 +ILSVRC2012_val_00047177.JPEG:n04523525 +ILSVRC2012_val_00047178.JPEG:n04522168 +ILSVRC2012_val_00047179.JPEG:n13052670 +ILSVRC2012_val_00047180.JPEG:n03633091 +ILSVRC2012_val_00047181.JPEG:n04067472 +ILSVRC2012_val_00047182.JPEG:n02988304 +ILSVRC2012_val_00047183.JPEG:n04486054 +ILSVRC2012_val_00047184.JPEG:n01677366 +ILSVRC2012_val_00047185.JPEG:n02492660 +ILSVRC2012_val_00047186.JPEG:n03127747 +ILSVRC2012_val_00047187.JPEG:n02112350 +ILSVRC2012_val_00047188.JPEG:n04336792 +ILSVRC2012_val_00047189.JPEG:n03417042 +ILSVRC2012_val_00047190.JPEG:n13133613 +ILSVRC2012_val_00047191.JPEG:n01608432 +ILSVRC2012_val_00047192.JPEG:n02865351 +ILSVRC2012_val_00047193.JPEG:n02129165 +ILSVRC2012_val_00047194.JPEG:n01773157 +ILSVRC2012_val_00047195.JPEG:n04258138 +ILSVRC2012_val_00047196.JPEG:n04041544 +ILSVRC2012_val_00047197.JPEG:n04252077 +ILSVRC2012_val_00047198.JPEG:n03197337 +ILSVRC2012_val_00047199.JPEG:n03794056 +ILSVRC2012_val_00047200.JPEG:n03877845 +ILSVRC2012_val_00047201.JPEG:n04346328 +ILSVRC2012_val_00047202.JPEG:n02086910 +ILSVRC2012_val_00047203.JPEG:n01694178 +ILSVRC2012_val_00047204.JPEG:n03445924 +ILSVRC2012_val_00047205.JPEG:n04532670 +ILSVRC2012_val_00047206.JPEG:n03781244 +ILSVRC2012_val_00047207.JPEG:n04141975 +ILSVRC2012_val_00047208.JPEG:n03124170 +ILSVRC2012_val_00047209.JPEG:n03874293 +ILSVRC2012_val_00047210.JPEG:n03498962 +ILSVRC2012_val_00047211.JPEG:n01739381 +ILSVRC2012_val_00047212.JPEG:n02791270 +ILSVRC2012_val_00047213.JPEG:n07892512 +ILSVRC2012_val_00047214.JPEG:n03444034 +ILSVRC2012_val_00047215.JPEG:n02105162 +ILSVRC2012_val_00047216.JPEG:n01734418 +ILSVRC2012_val_00047217.JPEG:n04070727 +ILSVRC2012_val_00047218.JPEG:n02916936 +ILSVRC2012_val_00047219.JPEG:n03840681 +ILSVRC2012_val_00047220.JPEG:n04399382 +ILSVRC2012_val_00047221.JPEG:n07749582 +ILSVRC2012_val_00047222.JPEG:n02480495 +ILSVRC2012_val_00047223.JPEG:n04515003 +ILSVRC2012_val_00047224.JPEG:n01688243 +ILSVRC2012_val_00047225.JPEG:n02107142 +ILSVRC2012_val_00047226.JPEG:n01914609 +ILSVRC2012_val_00047227.JPEG:n01742172 +ILSVRC2012_val_00047228.JPEG:n07753113 +ILSVRC2012_val_00047229.JPEG:n01828970 +ILSVRC2012_val_00047230.JPEG:n01797886 +ILSVRC2012_val_00047231.JPEG:n04606251 +ILSVRC2012_val_00047232.JPEG:n03062245 +ILSVRC2012_val_00047233.JPEG:n03400231 +ILSVRC2012_val_00047234.JPEG:n03483316 +ILSVRC2012_val_00047235.JPEG:n02978881 +ILSVRC2012_val_00047236.JPEG:n02109047 +ILSVRC2012_val_00047237.JPEG:n02795169 +ILSVRC2012_val_00047238.JPEG:n01728920 +ILSVRC2012_val_00047239.JPEG:n03530642 +ILSVRC2012_val_00047240.JPEG:n04209133 +ILSVRC2012_val_00047241.JPEG:n02105641 +ILSVRC2012_val_00047242.JPEG:n02111277 +ILSVRC2012_val_00047243.JPEG:n01737021 +ILSVRC2012_val_00047244.JPEG:n02092339 +ILSVRC2012_val_00047245.JPEG:n04589890 +ILSVRC2012_val_00047246.JPEG:n02454379 +ILSVRC2012_val_00047247.JPEG:n12267677 +ILSVRC2012_val_00047248.JPEG:n03627232 +ILSVRC2012_val_00047249.JPEG:n01990800 +ILSVRC2012_val_00047250.JPEG:n02109047 +ILSVRC2012_val_00047251.JPEG:n03314780 +ILSVRC2012_val_00047252.JPEG:n01798484 +ILSVRC2012_val_00047253.JPEG:n03691459 +ILSVRC2012_val_00047254.JPEG:n02669723 +ILSVRC2012_val_00047255.JPEG:n03781244 +ILSVRC2012_val_00047256.JPEG:n03467068 +ILSVRC2012_val_00047257.JPEG:n01770081 +ILSVRC2012_val_00047258.JPEG:n01796340 +ILSVRC2012_val_00047259.JPEG:n03930313 +ILSVRC2012_val_00047260.JPEG:n02226429 +ILSVRC2012_val_00047261.JPEG:n02514041 +ILSVRC2012_val_00047262.JPEG:n02356798 +ILSVRC2012_val_00047263.JPEG:n07880968 +ILSVRC2012_val_00047264.JPEG:n04131690 +ILSVRC2012_val_00047265.JPEG:n02807133 +ILSVRC2012_val_00047266.JPEG:n03841143 +ILSVRC2012_val_00047267.JPEG:n02346627 +ILSVRC2012_val_00047268.JPEG:n02397096 +ILSVRC2012_val_00047269.JPEG:n02963159 +ILSVRC2012_val_00047270.JPEG:n02641379 +ILSVRC2012_val_00047271.JPEG:n02093428 +ILSVRC2012_val_00047272.JPEG:n01537544 +ILSVRC2012_val_00047273.JPEG:n02814860 +ILSVRC2012_val_00047274.JPEG:n04074963 +ILSVRC2012_val_00047275.JPEG:n02109525 +ILSVRC2012_val_00047276.JPEG:n02085782 +ILSVRC2012_val_00047277.JPEG:n02102973 +ILSVRC2012_val_00047278.JPEG:n02319095 +ILSVRC2012_val_00047279.JPEG:n02437616 +ILSVRC2012_val_00047280.JPEG:n02395406 +ILSVRC2012_val_00047281.JPEG:n02488291 +ILSVRC2012_val_00047282.JPEG:n03777568 +ILSVRC2012_val_00047283.JPEG:n03710193 +ILSVRC2012_val_00047284.JPEG:n09421951 +ILSVRC2012_val_00047285.JPEG:n03838899 +ILSVRC2012_val_00047286.JPEG:n04004767 +ILSVRC2012_val_00047287.JPEG:n02011460 +ILSVRC2012_val_00047288.JPEG:n02526121 +ILSVRC2012_val_00047289.JPEG:n02112018 +ILSVRC2012_val_00047290.JPEG:n02687172 +ILSVRC2012_val_00047291.JPEG:n02825657 +ILSVRC2012_val_00047292.JPEG:n01882714 +ILSVRC2012_val_00047293.JPEG:n01968897 +ILSVRC2012_val_00047294.JPEG:n03196217 +ILSVRC2012_val_00047295.JPEG:n02101556 +ILSVRC2012_val_00047296.JPEG:n04389033 +ILSVRC2012_val_00047297.JPEG:n04127249 +ILSVRC2012_val_00047298.JPEG:n04254680 +ILSVRC2012_val_00047299.JPEG:n03063689 +ILSVRC2012_val_00047300.JPEG:n04125021 +ILSVRC2012_val_00047301.JPEG:n01689811 +ILSVRC2012_val_00047302.JPEG:n04325704 +ILSVRC2012_val_00047303.JPEG:n02137549 +ILSVRC2012_val_00047304.JPEG:n10565667 +ILSVRC2012_val_00047305.JPEG:n02391049 +ILSVRC2012_val_00047306.JPEG:n07836838 +ILSVRC2012_val_00047307.JPEG:n04584207 +ILSVRC2012_val_00047308.JPEG:n02423022 +ILSVRC2012_val_00047309.JPEG:n02088364 +ILSVRC2012_val_00047310.JPEG:n03961711 +ILSVRC2012_val_00047311.JPEG:n02457408 +ILSVRC2012_val_00047312.JPEG:n03535780 +ILSVRC2012_val_00047313.JPEG:n02412080 +ILSVRC2012_val_00047314.JPEG:n03017168 +ILSVRC2012_val_00047315.JPEG:n02979186 +ILSVRC2012_val_00047316.JPEG:n02676566 +ILSVRC2012_val_00047317.JPEG:n01860187 +ILSVRC2012_val_00047318.JPEG:n02423022 +ILSVRC2012_val_00047319.JPEG:n03891332 +ILSVRC2012_val_00047320.JPEG:n01494475 +ILSVRC2012_val_00047321.JPEG:n01704323 +ILSVRC2012_val_00047322.JPEG:n04423845 +ILSVRC2012_val_00047323.JPEG:n03976467 +ILSVRC2012_val_00047324.JPEG:n02091831 +ILSVRC2012_val_00047325.JPEG:n02101006 +ILSVRC2012_val_00047326.JPEG:n01491361 +ILSVRC2012_val_00047327.JPEG:n03063689 +ILSVRC2012_val_00047328.JPEG:n01910747 +ILSVRC2012_val_00047329.JPEG:n01784675 +ILSVRC2012_val_00047330.JPEG:n03967562 +ILSVRC2012_val_00047331.JPEG:n02094114 +ILSVRC2012_val_00047332.JPEG:n04065272 +ILSVRC2012_val_00047333.JPEG:n01534433 +ILSVRC2012_val_00047334.JPEG:n04372370 +ILSVRC2012_val_00047335.JPEG:n02879718 +ILSVRC2012_val_00047336.JPEG:n02871525 +ILSVRC2012_val_00047337.JPEG:n02168699 +ILSVRC2012_val_00047338.JPEG:n01784675 +ILSVRC2012_val_00047339.JPEG:n03492542 +ILSVRC2012_val_00047340.JPEG:n02101388 +ILSVRC2012_val_00047341.JPEG:n07718472 +ILSVRC2012_val_00047342.JPEG:n02110185 +ILSVRC2012_val_00047343.JPEG:n12998815 +ILSVRC2012_val_00047344.JPEG:n03127925 +ILSVRC2012_val_00047345.JPEG:n03207743 +ILSVRC2012_val_00047346.JPEG:n12057211 +ILSVRC2012_val_00047347.JPEG:n07565083 +ILSVRC2012_val_00047348.JPEG:n04525038 +ILSVRC2012_val_00047349.JPEG:n04118776 +ILSVRC2012_val_00047350.JPEG:n01616318 +ILSVRC2012_val_00047351.JPEG:n02965783 +ILSVRC2012_val_00047352.JPEG:n02206856 +ILSVRC2012_val_00047353.JPEG:n03899768 +ILSVRC2012_val_00047354.JPEG:n01687978 +ILSVRC2012_val_00047355.JPEG:n03379051 +ILSVRC2012_val_00047356.JPEG:n02104029 +ILSVRC2012_val_00047357.JPEG:n04229816 +ILSVRC2012_val_00047358.JPEG:n03124170 +ILSVRC2012_val_00047359.JPEG:n02281406 +ILSVRC2012_val_00047360.JPEG:n03032252 +ILSVRC2012_val_00047361.JPEG:n02101556 +ILSVRC2012_val_00047362.JPEG:n02980441 +ILSVRC2012_val_00047363.JPEG:n03485794 +ILSVRC2012_val_00047364.JPEG:n04366367 +ILSVRC2012_val_00047365.JPEG:n02492035 +ILSVRC2012_val_00047366.JPEG:n03599486 +ILSVRC2012_val_00047367.JPEG:n04548362 +ILSVRC2012_val_00047368.JPEG:n03764736 +ILSVRC2012_val_00047369.JPEG:n07760859 +ILSVRC2012_val_00047370.JPEG:n01978287 +ILSVRC2012_val_00047371.JPEG:n04505470 +ILSVRC2012_val_00047372.JPEG:n02488291 +ILSVRC2012_val_00047373.JPEG:n02782093 +ILSVRC2012_val_00047374.JPEG:n03417042 +ILSVRC2012_val_00047375.JPEG:n02486261 +ILSVRC2012_val_00047376.JPEG:n03843555 +ILSVRC2012_val_00047377.JPEG:n02319095 +ILSVRC2012_val_00047378.JPEG:n02493509 +ILSVRC2012_val_00047379.JPEG:n01798484 +ILSVRC2012_val_00047380.JPEG:n03857828 +ILSVRC2012_val_00047381.JPEG:n03950228 +ILSVRC2012_val_00047382.JPEG:n02791124 +ILSVRC2012_val_00047383.JPEG:n03207941 +ILSVRC2012_val_00047384.JPEG:n01751748 +ILSVRC2012_val_00047385.JPEG:n03916031 +ILSVRC2012_val_00047386.JPEG:n04074963 +ILSVRC2012_val_00047387.JPEG:n03724870 +ILSVRC2012_val_00047388.JPEG:n13133613 +ILSVRC2012_val_00047389.JPEG:n03937543 +ILSVRC2012_val_00047390.JPEG:n03255030 +ILSVRC2012_val_00047391.JPEG:n04372370 +ILSVRC2012_val_00047392.JPEG:n02168699 +ILSVRC2012_val_00047393.JPEG:n03920288 +ILSVRC2012_val_00047394.JPEG:n02514041 +ILSVRC2012_val_00047395.JPEG:n02112350 +ILSVRC2012_val_00047396.JPEG:n01443537 +ILSVRC2012_val_00047397.JPEG:n01807496 +ILSVRC2012_val_00047398.JPEG:n04070727 +ILSVRC2012_val_00047399.JPEG:n01675722 +ILSVRC2012_val_00047400.JPEG:n01518878 +ILSVRC2012_val_00047401.JPEG:n03599486 +ILSVRC2012_val_00047402.JPEG:n04162706 +ILSVRC2012_val_00047403.JPEG:n04147183 +ILSVRC2012_val_00047404.JPEG:n01795545 +ILSVRC2012_val_00047405.JPEG:n01698640 +ILSVRC2012_val_00047406.JPEG:n01873310 +ILSVRC2012_val_00047407.JPEG:n07718472 +ILSVRC2012_val_00047408.JPEG:n04033995 +ILSVRC2012_val_00047409.JPEG:n04418357 +ILSVRC2012_val_00047410.JPEG:n04429376 +ILSVRC2012_val_00047411.JPEG:n02110806 +ILSVRC2012_val_00047412.JPEG:n01944390 +ILSVRC2012_val_00047413.JPEG:n09835506 +ILSVRC2012_val_00047414.JPEG:n02092339 +ILSVRC2012_val_00047415.JPEG:n02948072 +ILSVRC2012_val_00047416.JPEG:n01978455 +ILSVRC2012_val_00047417.JPEG:n02100236 +ILSVRC2012_val_00047418.JPEG:n03710193 +ILSVRC2012_val_00047419.JPEG:n04517823 +ILSVRC2012_val_00047420.JPEG:n04154565 +ILSVRC2012_val_00047421.JPEG:n03761084 +ILSVRC2012_val_00047422.JPEG:n02346627 +ILSVRC2012_val_00047423.JPEG:n02672831 +ILSVRC2012_val_00047424.JPEG:n02422106 +ILSVRC2012_val_00047425.JPEG:n01664065 +ILSVRC2012_val_00047426.JPEG:n04125021 +ILSVRC2012_val_00047427.JPEG:n03450230 +ILSVRC2012_val_00047428.JPEG:n03980874 +ILSVRC2012_val_00047429.JPEG:n03642806 +ILSVRC2012_val_00047430.JPEG:n03866082 +ILSVRC2012_val_00047431.JPEG:n01494475 +ILSVRC2012_val_00047432.JPEG:n01910747 +ILSVRC2012_val_00047433.JPEG:n02229544 +ILSVRC2012_val_00047434.JPEG:n01770393 +ILSVRC2012_val_00047435.JPEG:n02114367 +ILSVRC2012_val_00047436.JPEG:n07920052 +ILSVRC2012_val_00047437.JPEG:n01872401 +ILSVRC2012_val_00047438.JPEG:n02109047 +ILSVRC2012_val_00047439.JPEG:n03884397 +ILSVRC2012_val_00047440.JPEG:n02704792 +ILSVRC2012_val_00047441.JPEG:n07716906 +ILSVRC2012_val_00047442.JPEG:n03843555 +ILSVRC2012_val_00047443.JPEG:n03095699 +ILSVRC2012_val_00047444.JPEG:n04532106 +ILSVRC2012_val_00047445.JPEG:n02093754 +ILSVRC2012_val_00047446.JPEG:n02879718 +ILSVRC2012_val_00047447.JPEG:n04515003 +ILSVRC2012_val_00047448.JPEG:n07718747 +ILSVRC2012_val_00047449.JPEG:n02094258 +ILSVRC2012_val_00047450.JPEG:n03838899 +ILSVRC2012_val_00047451.JPEG:n03126707 +ILSVRC2012_val_00047452.JPEG:n07730033 +ILSVRC2012_val_00047453.JPEG:n03085013 +ILSVRC2012_val_00047454.JPEG:n03680355 +ILSVRC2012_val_00047455.JPEG:n02123045 +ILSVRC2012_val_00047456.JPEG:n02279972 +ILSVRC2012_val_00047457.JPEG:n02086240 +ILSVRC2012_val_00047458.JPEG:n02134418 +ILSVRC2012_val_00047459.JPEG:n03388549 +ILSVRC2012_val_00047460.JPEG:n03637318 +ILSVRC2012_val_00047461.JPEG:n03345487 +ILSVRC2012_val_00047462.JPEG:n04517823 +ILSVRC2012_val_00047463.JPEG:n03476991 +ILSVRC2012_val_00047464.JPEG:n07734744 +ILSVRC2012_val_00047465.JPEG:n03602883 +ILSVRC2012_val_00047466.JPEG:n04371774 +ILSVRC2012_val_00047467.JPEG:n04229816 +ILSVRC2012_val_00047468.JPEG:n03249569 +ILSVRC2012_val_00047469.JPEG:n02676566 +ILSVRC2012_val_00047470.JPEG:n02011460 +ILSVRC2012_val_00047471.JPEG:n02916936 +ILSVRC2012_val_00047472.JPEG:n01806567 +ILSVRC2012_val_00047473.JPEG:n02814533 +ILSVRC2012_val_00047474.JPEG:n01560419 +ILSVRC2012_val_00047475.JPEG:n03970156 +ILSVRC2012_val_00047476.JPEG:n01978455 +ILSVRC2012_val_00047477.JPEG:n02823750 +ILSVRC2012_val_00047478.JPEG:n02883205 +ILSVRC2012_val_00047479.JPEG:n02110627 +ILSVRC2012_val_00047480.JPEG:n03787032 +ILSVRC2012_val_00047481.JPEG:n10148035 +ILSVRC2012_val_00047482.JPEG:n04596742 +ILSVRC2012_val_00047483.JPEG:n04033995 +ILSVRC2012_val_00047484.JPEG:n02444819 +ILSVRC2012_val_00047485.JPEG:n03954731 +ILSVRC2012_val_00047486.JPEG:n04311174 +ILSVRC2012_val_00047487.JPEG:n02095889 +ILSVRC2012_val_00047488.JPEG:n01914609 +ILSVRC2012_val_00047489.JPEG:n03710193 +ILSVRC2012_val_00047490.JPEG:n02782093 +ILSVRC2012_val_00047491.JPEG:n01820546 +ILSVRC2012_val_00047492.JPEG:n02091134 +ILSVRC2012_val_00047493.JPEG:n04355933 +ILSVRC2012_val_00047494.JPEG:n02389026 +ILSVRC2012_val_00047495.JPEG:n04090263 +ILSVRC2012_val_00047496.JPEG:n04254120 +ILSVRC2012_val_00047497.JPEG:n01820546 +ILSVRC2012_val_00047498.JPEG:n01641577 +ILSVRC2012_val_00047499.JPEG:n02106550 +ILSVRC2012_val_00047500.JPEG:n02326432 +ILSVRC2012_val_00047501.JPEG:n03532672 +ILSVRC2012_val_00047502.JPEG:n03065424 +ILSVRC2012_val_00047503.JPEG:n07836838 +ILSVRC2012_val_00047504.JPEG:n02786058 +ILSVRC2012_val_00047505.JPEG:n04235860 +ILSVRC2012_val_00047506.JPEG:n04264628 +ILSVRC2012_val_00047507.JPEG:n02091244 +ILSVRC2012_val_00047508.JPEG:n03773504 +ILSVRC2012_val_00047509.JPEG:n02013706 +ILSVRC2012_val_00047510.JPEG:n04458633 +ILSVRC2012_val_00047511.JPEG:n04270147 +ILSVRC2012_val_00047512.JPEG:n07711569 +ILSVRC2012_val_00047513.JPEG:n04325704 +ILSVRC2012_val_00047514.JPEG:n03017168 +ILSVRC2012_val_00047515.JPEG:n02112350 +ILSVRC2012_val_00047516.JPEG:n04192698 +ILSVRC2012_val_00047517.JPEG:n02769748 +ILSVRC2012_val_00047518.JPEG:n02096051 +ILSVRC2012_val_00047519.JPEG:n04149813 +ILSVRC2012_val_00047520.JPEG:n02483708 +ILSVRC2012_val_00047521.JPEG:n04040759 +ILSVRC2012_val_00047522.JPEG:n04265275 +ILSVRC2012_val_00047523.JPEG:n02071294 +ILSVRC2012_val_00047524.JPEG:n07873807 +ILSVRC2012_val_00047525.JPEG:n02488702 +ILSVRC2012_val_00047526.JPEG:n04200800 +ILSVRC2012_val_00047527.JPEG:n02134084 +ILSVRC2012_val_00047528.JPEG:n04418357 +ILSVRC2012_val_00047529.JPEG:n04552348 +ILSVRC2012_val_00047530.JPEG:n02999410 +ILSVRC2012_val_00047531.JPEG:n02817516 +ILSVRC2012_val_00047532.JPEG:n01981276 +ILSVRC2012_val_00047533.JPEG:n02233338 +ILSVRC2012_val_00047534.JPEG:n02504458 +ILSVRC2012_val_00047535.JPEG:n02116738 +ILSVRC2012_val_00047536.JPEG:n03633091 +ILSVRC2012_val_00047537.JPEG:n03372029 +ILSVRC2012_val_00047538.JPEG:n07714990 +ILSVRC2012_val_00047539.JPEG:n04552348 +ILSVRC2012_val_00047540.JPEG:n02504458 +ILSVRC2012_val_00047541.JPEG:n02172182 +ILSVRC2012_val_00047542.JPEG:n03691459 +ILSVRC2012_val_00047543.JPEG:n02089078 +ILSVRC2012_val_00047544.JPEG:n03594734 +ILSVRC2012_val_00047545.JPEG:n02643566 +ILSVRC2012_val_00047546.JPEG:n01665541 +ILSVRC2012_val_00047547.JPEG:n01818515 +ILSVRC2012_val_00047548.JPEG:n02802426 +ILSVRC2012_val_00047549.JPEG:n03662601 +ILSVRC2012_val_00047550.JPEG:n03495258 +ILSVRC2012_val_00047551.JPEG:n01773797 +ILSVRC2012_val_00047552.JPEG:n02206856 +ILSVRC2012_val_00047553.JPEG:n03710721 +ILSVRC2012_val_00047554.JPEG:n04442312 +ILSVRC2012_val_00047555.JPEG:n02137549 +ILSVRC2012_val_00047556.JPEG:n03657121 +ILSVRC2012_val_00047557.JPEG:n04311004 +ILSVRC2012_val_00047558.JPEG:n03775071 +ILSVRC2012_val_00047559.JPEG:n03630383 +ILSVRC2012_val_00047560.JPEG:n02412080 +ILSVRC2012_val_00047561.JPEG:n01443537 +ILSVRC2012_val_00047562.JPEG:n03874293 +ILSVRC2012_val_00047563.JPEG:n03874599 +ILSVRC2012_val_00047564.JPEG:n07590611 +ILSVRC2012_val_00047565.JPEG:n04162706 +ILSVRC2012_val_00047566.JPEG:n02108551 +ILSVRC2012_val_00047567.JPEG:n07749582 +ILSVRC2012_val_00047568.JPEG:n02804414 +ILSVRC2012_val_00047569.JPEG:n03777754 +ILSVRC2012_val_00047570.JPEG:n03584829 +ILSVRC2012_val_00047571.JPEG:n02699494 +ILSVRC2012_val_00047572.JPEG:n02097298 +ILSVRC2012_val_00047573.JPEG:n03661043 +ILSVRC2012_val_00047574.JPEG:n01774750 +ILSVRC2012_val_00047575.JPEG:n03594945 +ILSVRC2012_val_00047576.JPEG:n04005630 +ILSVRC2012_val_00047577.JPEG:n07697313 +ILSVRC2012_val_00047578.JPEG:n02009229 +ILSVRC2012_val_00047579.JPEG:n03529860 +ILSVRC2012_val_00047580.JPEG:n04355933 +ILSVRC2012_val_00047581.JPEG:n03899768 +ILSVRC2012_val_00047582.JPEG:n03337140 +ILSVRC2012_val_00047583.JPEG:n02110958 +ILSVRC2012_val_00047584.JPEG:n02092339 +ILSVRC2012_val_00047585.JPEG:n02097130 +ILSVRC2012_val_00047586.JPEG:n03337140 +ILSVRC2012_val_00047587.JPEG:n01818515 +ILSVRC2012_val_00047588.JPEG:n03345487 +ILSVRC2012_val_00047589.JPEG:n01496331 +ILSVRC2012_val_00047590.JPEG:n03124043 +ILSVRC2012_val_00047591.JPEG:n02095570 +ILSVRC2012_val_00047592.JPEG:n01558993 +ILSVRC2012_val_00047593.JPEG:n03814906 +ILSVRC2012_val_00047594.JPEG:n03216828 +ILSVRC2012_val_00047595.JPEG:n03930630 +ILSVRC2012_val_00047596.JPEG:n06874185 +ILSVRC2012_val_00047597.JPEG:n02113799 +ILSVRC2012_val_00047598.JPEG:n07720875 +ILSVRC2012_val_00047599.JPEG:n03887697 +ILSVRC2012_val_00047600.JPEG:n03697007 +ILSVRC2012_val_00047601.JPEG:n02231487 +ILSVRC2012_val_00047602.JPEG:n02669723 +ILSVRC2012_val_00047603.JPEG:n02480855 +ILSVRC2012_val_00047604.JPEG:n04366367 +ILSVRC2012_val_00047605.JPEG:n03706229 +ILSVRC2012_val_00047606.JPEG:n03529860 +ILSVRC2012_val_00047607.JPEG:n03924679 +ILSVRC2012_val_00047608.JPEG:n03527444 +ILSVRC2012_val_00047609.JPEG:n01770393 +ILSVRC2012_val_00047610.JPEG:n04493381 +ILSVRC2012_val_00047611.JPEG:n04532670 +ILSVRC2012_val_00047612.JPEG:n02883205 +ILSVRC2012_val_00047613.JPEG:n04192698 +ILSVRC2012_val_00047614.JPEG:n02129604 +ILSVRC2012_val_00047615.JPEG:n02669723 +ILSVRC2012_val_00047616.JPEG:n04259630 +ILSVRC2012_val_00047617.JPEG:n02091831 +ILSVRC2012_val_00047618.JPEG:n09332890 +ILSVRC2012_val_00047619.JPEG:n01883070 +ILSVRC2012_val_00047620.JPEG:n04026417 +ILSVRC2012_val_00047621.JPEG:n03485407 +ILSVRC2012_val_00047622.JPEG:n01877812 +ILSVRC2012_val_00047623.JPEG:n01644900 +ILSVRC2012_val_00047624.JPEG:n09256479 +ILSVRC2012_val_00047625.JPEG:n04286575 +ILSVRC2012_val_00047626.JPEG:n01601694 +ILSVRC2012_val_00047627.JPEG:n04428191 +ILSVRC2012_val_00047628.JPEG:n03065424 +ILSVRC2012_val_00047629.JPEG:n03770439 +ILSVRC2012_val_00047630.JPEG:n02174001 +ILSVRC2012_val_00047631.JPEG:n02110341 +ILSVRC2012_val_00047632.JPEG:n02916936 +ILSVRC2012_val_00047633.JPEG:n04086273 +ILSVRC2012_val_00047634.JPEG:n03393912 +ILSVRC2012_val_00047635.JPEG:n02701002 +ILSVRC2012_val_00047636.JPEG:n03991062 +ILSVRC2012_val_00047637.JPEG:n01608432 +ILSVRC2012_val_00047638.JPEG:n04273569 +ILSVRC2012_val_00047639.JPEG:n04522168 +ILSVRC2012_val_00047640.JPEG:n07760859 +ILSVRC2012_val_00047641.JPEG:n02493793 +ILSVRC2012_val_00047642.JPEG:n02804414 +ILSVRC2012_val_00047643.JPEG:n02229544 +ILSVRC2012_val_00047644.JPEG:n04009552 +ILSVRC2012_val_00047645.JPEG:n03874599 +ILSVRC2012_val_00047646.JPEG:n03649909 +ILSVRC2012_val_00047647.JPEG:n07614500 +ILSVRC2012_val_00047648.JPEG:n02094433 +ILSVRC2012_val_00047649.JPEG:n02097298 +ILSVRC2012_val_00047650.JPEG:n03662601 +ILSVRC2012_val_00047651.JPEG:n03450230 +ILSVRC2012_val_00047652.JPEG:n02093256 +ILSVRC2012_val_00047653.JPEG:n04033995 +ILSVRC2012_val_00047654.JPEG:n02113023 +ILSVRC2012_val_00047655.JPEG:n09246464 +ILSVRC2012_val_00047656.JPEG:n01704323 +ILSVRC2012_val_00047657.JPEG:n02488702 +ILSVRC2012_val_00047658.JPEG:n02096294 +ILSVRC2012_val_00047659.JPEG:n04536866 +ILSVRC2012_val_00047660.JPEG:n07873807 +ILSVRC2012_val_00047661.JPEG:n03770439 +ILSVRC2012_val_00047662.JPEG:n04409515 +ILSVRC2012_val_00047663.JPEG:n04532106 +ILSVRC2012_val_00047664.JPEG:n04542943 +ILSVRC2012_val_00047665.JPEG:n07584110 +ILSVRC2012_val_00047666.JPEG:n02808304 +ILSVRC2012_val_00047667.JPEG:n03903868 +ILSVRC2012_val_00047668.JPEG:n03888605 +ILSVRC2012_val_00047669.JPEG:n02051845 +ILSVRC2012_val_00047670.JPEG:n02115641 +ILSVRC2012_val_00047671.JPEG:n02099267 +ILSVRC2012_val_00047672.JPEG:n03452741 +ILSVRC2012_val_00047673.JPEG:n03498962 +ILSVRC2012_val_00047674.JPEG:n01945685 +ILSVRC2012_val_00047675.JPEG:n01692333 +ILSVRC2012_val_00047676.JPEG:n03930630 +ILSVRC2012_val_00047677.JPEG:n02794156 +ILSVRC2012_val_00047678.JPEG:n04311004 +ILSVRC2012_val_00047679.JPEG:n03482405 +ILSVRC2012_val_00047680.JPEG:n04540053 +ILSVRC2012_val_00047681.JPEG:n09256479 +ILSVRC2012_val_00047682.JPEG:n02607072 +ILSVRC2012_val_00047683.JPEG:n02281406 +ILSVRC2012_val_00047684.JPEG:n03991062 +ILSVRC2012_val_00047685.JPEG:n02056570 +ILSVRC2012_val_00047686.JPEG:n04243546 +ILSVRC2012_val_00047687.JPEG:n03100240 +ILSVRC2012_val_00047688.JPEG:n01532829 +ILSVRC2012_val_00047689.JPEG:n03127747 +ILSVRC2012_val_00047690.JPEG:n02119022 +ILSVRC2012_val_00047691.JPEG:n02666196 +ILSVRC2012_val_00047692.JPEG:n03379051 +ILSVRC2012_val_00047693.JPEG:n04417672 +ILSVRC2012_val_00047694.JPEG:n07920052 +ILSVRC2012_val_00047695.JPEG:n03617480 +ILSVRC2012_val_00047696.JPEG:n01818515 +ILSVRC2012_val_00047697.JPEG:n03998194 +ILSVRC2012_val_00047698.JPEG:n03388183 +ILSVRC2012_val_00047699.JPEG:n02113799 +ILSVRC2012_val_00047700.JPEG:n04344873 +ILSVRC2012_val_00047701.JPEG:n03590841 +ILSVRC2012_val_00047702.JPEG:n04228054 +ILSVRC2012_val_00047703.JPEG:n04228054 +ILSVRC2012_val_00047704.JPEG:n02231487 +ILSVRC2012_val_00047705.JPEG:n03888257 +ILSVRC2012_val_00047706.JPEG:n04086273 +ILSVRC2012_val_00047707.JPEG:n02090622 +ILSVRC2012_val_00047708.JPEG:n03933933 +ILSVRC2012_val_00047709.JPEG:n02422106 +ILSVRC2012_val_00047710.JPEG:n03720891 +ILSVRC2012_val_00047711.JPEG:n02093991 +ILSVRC2012_val_00047712.JPEG:n04347754 +ILSVRC2012_val_00047713.JPEG:n01630670 +ILSVRC2012_val_00047714.JPEG:n03843555 +ILSVRC2012_val_00047715.JPEG:n03729826 +ILSVRC2012_val_00047716.JPEG:n01644900 +ILSVRC2012_val_00047717.JPEG:n02264363 +ILSVRC2012_val_00047718.JPEG:n03126707 +ILSVRC2012_val_00047719.JPEG:n12057211 +ILSVRC2012_val_00047720.JPEG:n04461696 +ILSVRC2012_val_00047721.JPEG:n02098286 +ILSVRC2012_val_00047722.JPEG:n02276258 +ILSVRC2012_val_00047723.JPEG:n04552348 +ILSVRC2012_val_00047724.JPEG:n01514668 +ILSVRC2012_val_00047725.JPEG:n04243546 +ILSVRC2012_val_00047726.JPEG:n02871525 +ILSVRC2012_val_00047727.JPEG:n02106382 +ILSVRC2012_val_00047728.JPEG:n02100583 +ILSVRC2012_val_00047729.JPEG:n02085936 +ILSVRC2012_val_00047730.JPEG:n04487081 +ILSVRC2012_val_00047731.JPEG:n03995372 +ILSVRC2012_val_00047732.JPEG:n01601694 +ILSVRC2012_val_00047733.JPEG:n02279972 +ILSVRC2012_val_00047734.JPEG:n03444034 +ILSVRC2012_val_00047735.JPEG:n07730033 +ILSVRC2012_val_00047736.JPEG:n02011460 +ILSVRC2012_val_00047737.JPEG:n02099601 +ILSVRC2012_val_00047738.JPEG:n04536866 +ILSVRC2012_val_00047739.JPEG:n03014705 +ILSVRC2012_val_00047740.JPEG:n02486261 +ILSVRC2012_val_00047741.JPEG:n04590129 +ILSVRC2012_val_00047742.JPEG:n04265275 +ILSVRC2012_val_00047743.JPEG:n03447447 +ILSVRC2012_val_00047744.JPEG:n02102177 +ILSVRC2012_val_00047745.JPEG:n03388043 +ILSVRC2012_val_00047746.JPEG:n01665541 +ILSVRC2012_val_00047747.JPEG:n03924679 +ILSVRC2012_val_00047748.JPEG:n06874185 +ILSVRC2012_val_00047749.JPEG:n03018349 +ILSVRC2012_val_00047750.JPEG:n02403003 +ILSVRC2012_val_00047751.JPEG:n03196217 +ILSVRC2012_val_00047752.JPEG:n02132136 +ILSVRC2012_val_00047753.JPEG:n01514859 +ILSVRC2012_val_00047754.JPEG:n02397096 +ILSVRC2012_val_00047755.JPEG:n02113186 +ILSVRC2012_val_00047756.JPEG:n03924679 +ILSVRC2012_val_00047757.JPEG:n02096437 +ILSVRC2012_val_00047758.JPEG:n07831146 +ILSVRC2012_val_00047759.JPEG:n04584207 +ILSVRC2012_val_00047760.JPEG:n03777568 +ILSVRC2012_val_00047761.JPEG:n02276258 +ILSVRC2012_val_00047762.JPEG:n02108915 +ILSVRC2012_val_00047763.JPEG:n04540053 +ILSVRC2012_val_00047764.JPEG:n03874293 +ILSVRC2012_val_00047765.JPEG:n02033041 +ILSVRC2012_val_00047766.JPEG:n04270147 +ILSVRC2012_val_00047767.JPEG:n02114367 +ILSVRC2012_val_00047768.JPEG:n07730033 +ILSVRC2012_val_00047769.JPEG:n02342885 +ILSVRC2012_val_00047770.JPEG:n03929660 +ILSVRC2012_val_00047771.JPEG:n03032252 +ILSVRC2012_val_00047772.JPEG:n02992211 +ILSVRC2012_val_00047773.JPEG:n03658185 +ILSVRC2012_val_00047774.JPEG:n02777292 +ILSVRC2012_val_00047775.JPEG:n02879718 +ILSVRC2012_val_00047776.JPEG:n02319095 +ILSVRC2012_val_00047777.JPEG:n07760859 +ILSVRC2012_val_00047778.JPEG:n03888257 +ILSVRC2012_val_00047779.JPEG:n02910353 +ILSVRC2012_val_00047780.JPEG:n03868863 +ILSVRC2012_val_00047781.JPEG:n04133789 +ILSVRC2012_val_00047782.JPEG:n04136333 +ILSVRC2012_val_00047783.JPEG:n04356056 +ILSVRC2012_val_00047784.JPEG:n02028035 +ILSVRC2012_val_00047785.JPEG:n03000134 +ILSVRC2012_val_00047786.JPEG:n03355925 +ILSVRC2012_val_00047787.JPEG:n04326547 +ILSVRC2012_val_00047788.JPEG:n02494079 +ILSVRC2012_val_00047789.JPEG:n04099969 +ILSVRC2012_val_00047790.JPEG:n02966193 +ILSVRC2012_val_00047791.JPEG:n04147183 +ILSVRC2012_val_00047792.JPEG:n02966193 +ILSVRC2012_val_00047793.JPEG:n07697313 +ILSVRC2012_val_00047794.JPEG:n03877472 +ILSVRC2012_val_00047795.JPEG:n02486261 +ILSVRC2012_val_00047796.JPEG:n02510455 +ILSVRC2012_val_00047797.JPEG:n07720875 +ILSVRC2012_val_00047798.JPEG:n03764736 +ILSVRC2012_val_00047799.JPEG:n04239074 +ILSVRC2012_val_00047800.JPEG:n02443484 +ILSVRC2012_val_00047801.JPEG:n07720875 +ILSVRC2012_val_00047802.JPEG:n02840245 +ILSVRC2012_val_00047803.JPEG:n03782006 +ILSVRC2012_val_00047804.JPEG:n02119789 +ILSVRC2012_val_00047805.JPEG:n04328186 +ILSVRC2012_val_00047806.JPEG:n02417914 +ILSVRC2012_val_00047807.JPEG:n03216828 +ILSVRC2012_val_00047808.JPEG:n02108551 +ILSVRC2012_val_00047809.JPEG:n02013706 +ILSVRC2012_val_00047810.JPEG:n01734418 +ILSVRC2012_val_00047811.JPEG:n03729826 +ILSVRC2012_val_00047812.JPEG:n01689811 +ILSVRC2012_val_00047813.JPEG:n04522168 +ILSVRC2012_val_00047814.JPEG:n02422106 +ILSVRC2012_val_00047815.JPEG:n04004767 +ILSVRC2012_val_00047816.JPEG:n12620546 +ILSVRC2012_val_00047817.JPEG:n04041544 +ILSVRC2012_val_00047818.JPEG:n04116512 +ILSVRC2012_val_00047819.JPEG:n03478589 +ILSVRC2012_val_00047820.JPEG:n02174001 +ILSVRC2012_val_00047821.JPEG:n04486054 +ILSVRC2012_val_00047822.JPEG:n02107142 +ILSVRC2012_val_00047823.JPEG:n02422699 +ILSVRC2012_val_00047824.JPEG:n03400231 +ILSVRC2012_val_00047825.JPEG:n07930864 +ILSVRC2012_val_00047826.JPEG:n04200800 +ILSVRC2012_val_00047827.JPEG:n01582220 +ILSVRC2012_val_00047828.JPEG:n07753592 +ILSVRC2012_val_00047829.JPEG:n02690373 +ILSVRC2012_val_00047830.JPEG:n07880968 +ILSVRC2012_val_00047831.JPEG:n03958227 +ILSVRC2012_val_00047832.JPEG:n01665541 +ILSVRC2012_val_00047833.JPEG:n01847000 +ILSVRC2012_val_00047834.JPEG:n12768682 +ILSVRC2012_val_00047835.JPEG:n03478589 +ILSVRC2012_val_00047836.JPEG:n02091467 +ILSVRC2012_val_00047837.JPEG:n02787622 +ILSVRC2012_val_00047838.JPEG:n02776631 +ILSVRC2012_val_00047839.JPEG:n03000247 +ILSVRC2012_val_00047840.JPEG:n04074963 +ILSVRC2012_val_00047841.JPEG:n03743016 +ILSVRC2012_val_00047842.JPEG:n03325584 +ILSVRC2012_val_00047843.JPEG:n09246464 +ILSVRC2012_val_00047844.JPEG:n03871628 +ILSVRC2012_val_00047845.JPEG:n01740131 +ILSVRC2012_val_00047846.JPEG:n09288635 +ILSVRC2012_val_00047847.JPEG:n02730930 +ILSVRC2012_val_00047848.JPEG:n03884397 +ILSVRC2012_val_00047849.JPEG:n03775546 +ILSVRC2012_val_00047850.JPEG:n02114712 +ILSVRC2012_val_00047851.JPEG:n07718472 +ILSVRC2012_val_00047852.JPEG:n01728920 +ILSVRC2012_val_00047853.JPEG:n02494079 +ILSVRC2012_val_00047854.JPEG:n01774750 +ILSVRC2012_val_00047855.JPEG:n03967562 +ILSVRC2012_val_00047856.JPEG:n07718747 +ILSVRC2012_val_00047857.JPEG:n02906734 +ILSVRC2012_val_00047858.JPEG:n03444034 +ILSVRC2012_val_00047859.JPEG:n02408429 +ILSVRC2012_val_00047860.JPEG:n02319095 +ILSVRC2012_val_00047861.JPEG:n04330267 +ILSVRC2012_val_00047862.JPEG:n02113624 +ILSVRC2012_val_00047863.JPEG:n02231487 +ILSVRC2012_val_00047864.JPEG:n04141076 +ILSVRC2012_val_00047865.JPEG:n04552348 +ILSVRC2012_val_00047866.JPEG:n03759954 +ILSVRC2012_val_00047867.JPEG:n04120489 +ILSVRC2012_val_00047868.JPEG:n02869837 +ILSVRC2012_val_00047869.JPEG:n03838899 +ILSVRC2012_val_00047870.JPEG:n02268443 +ILSVRC2012_val_00047871.JPEG:n02321529 +ILSVRC2012_val_00047872.JPEG:n04023962 +ILSVRC2012_val_00047873.JPEG:n03843555 +ILSVRC2012_val_00047874.JPEG:n04525038 +ILSVRC2012_val_00047875.JPEG:n02361337 +ILSVRC2012_val_00047876.JPEG:n03924679 +ILSVRC2012_val_00047877.JPEG:n02236044 +ILSVRC2012_val_00047878.JPEG:n01530575 +ILSVRC2012_val_00047879.JPEG:n02877765 +ILSVRC2012_val_00047880.JPEG:n01980166 +ILSVRC2012_val_00047881.JPEG:n03777568 +ILSVRC2012_val_00047882.JPEG:n04008634 +ILSVRC2012_val_00047883.JPEG:n04579145 +ILSVRC2012_val_00047884.JPEG:n07873807 +ILSVRC2012_val_00047885.JPEG:n03207743 +ILSVRC2012_val_00047886.JPEG:n03970156 +ILSVRC2012_val_00047887.JPEG:n04254680 +ILSVRC2012_val_00047888.JPEG:n03345487 +ILSVRC2012_val_00047889.JPEG:n02454379 +ILSVRC2012_val_00047890.JPEG:n03110669 +ILSVRC2012_val_00047891.JPEG:n01980166 +ILSVRC2012_val_00047892.JPEG:n02536864 +ILSVRC2012_val_00047893.JPEG:n04285008 +ILSVRC2012_val_00047894.JPEG:n07684084 +ILSVRC2012_val_00047895.JPEG:n01924916 +ILSVRC2012_val_00047896.JPEG:n02108915 +ILSVRC2012_val_00047897.JPEG:n04074963 +ILSVRC2012_val_00047898.JPEG:n03837869 +ILSVRC2012_val_00047899.JPEG:n01882714 +ILSVRC2012_val_00047900.JPEG:n03873416 +ILSVRC2012_val_00047901.JPEG:n02169497 +ILSVRC2012_val_00047902.JPEG:n02687172 +ILSVRC2012_val_00047903.JPEG:n02268853 +ILSVRC2012_val_00047904.JPEG:n02906734 +ILSVRC2012_val_00047905.JPEG:n03018349 +ILSVRC2012_val_00047906.JPEG:n04310018 +ILSVRC2012_val_00047907.JPEG:n02978881 +ILSVRC2012_val_00047908.JPEG:n01693334 +ILSVRC2012_val_00047909.JPEG:n04542943 +ILSVRC2012_val_00047910.JPEG:n03770679 +ILSVRC2012_val_00047911.JPEG:n02123045 +ILSVRC2012_val_00047912.JPEG:n02974003 +ILSVRC2012_val_00047913.JPEG:n02086646 +ILSVRC2012_val_00047914.JPEG:n01530575 +ILSVRC2012_val_00047915.JPEG:n03786901 +ILSVRC2012_val_00047916.JPEG:n03710193 +ILSVRC2012_val_00047917.JPEG:n03388183 +ILSVRC2012_val_00047918.JPEG:n02112350 +ILSVRC2012_val_00047919.JPEG:n02113186 +ILSVRC2012_val_00047920.JPEG:n01883070 +ILSVRC2012_val_00047921.JPEG:n04552348 +ILSVRC2012_val_00047922.JPEG:n04344873 +ILSVRC2012_val_00047923.JPEG:n01773157 +ILSVRC2012_val_00047924.JPEG:n02109961 +ILSVRC2012_val_00047925.JPEG:n02123159 +ILSVRC2012_val_00047926.JPEG:n04404412 +ILSVRC2012_val_00047927.JPEG:n01917289 +ILSVRC2012_val_00047928.JPEG:n02169497 +ILSVRC2012_val_00047929.JPEG:n03899768 +ILSVRC2012_val_00047930.JPEG:n03697007 +ILSVRC2012_val_00047931.JPEG:n03874599 +ILSVRC2012_val_00047932.JPEG:n02669723 +ILSVRC2012_val_00047933.JPEG:n07717556 +ILSVRC2012_val_00047934.JPEG:n04147183 +ILSVRC2012_val_00047935.JPEG:n03424325 +ILSVRC2012_val_00047936.JPEG:n03498962 +ILSVRC2012_val_00047937.JPEG:n07715103 +ILSVRC2012_val_00047938.JPEG:n01632777 +ILSVRC2012_val_00047939.JPEG:n02264363 +ILSVRC2012_val_00047940.JPEG:n03018349 +ILSVRC2012_val_00047941.JPEG:n01669191 +ILSVRC2012_val_00047942.JPEG:n04204238 +ILSVRC2012_val_00047943.JPEG:n01829413 +ILSVRC2012_val_00047944.JPEG:n03785016 +ILSVRC2012_val_00047945.JPEG:n01871265 +ILSVRC2012_val_00047946.JPEG:n02992529 +ILSVRC2012_val_00047947.JPEG:n04127249 +ILSVRC2012_val_00047948.JPEG:n01774384 +ILSVRC2012_val_00047949.JPEG:n13040303 +ILSVRC2012_val_00047950.JPEG:n02090721 +ILSVRC2012_val_00047951.JPEG:n07615774 +ILSVRC2012_val_00047952.JPEG:n02231487 +ILSVRC2012_val_00047953.JPEG:n03126707 +ILSVRC2012_val_00047954.JPEG:n04399382 +ILSVRC2012_val_00047955.JPEG:n02127052 +ILSVRC2012_val_00047956.JPEG:n02480495 +ILSVRC2012_val_00047957.JPEG:n04357314 +ILSVRC2012_val_00047958.JPEG:n04597913 +ILSVRC2012_val_00047959.JPEG:n04311174 +ILSVRC2012_val_00047960.JPEG:n04376876 +ILSVRC2012_val_00047961.JPEG:n03344393 +ILSVRC2012_val_00047962.JPEG:n04146614 +ILSVRC2012_val_00047963.JPEG:n01622779 +ILSVRC2012_val_00047964.JPEG:n04325704 +ILSVRC2012_val_00047965.JPEG:n03527444 +ILSVRC2012_val_00047966.JPEG:n07753275 +ILSVRC2012_val_00047967.JPEG:n02422699 +ILSVRC2012_val_00047968.JPEG:n03759954 +ILSVRC2012_val_00047969.JPEG:n01824575 +ILSVRC2012_val_00047970.JPEG:n01704323 +ILSVRC2012_val_00047971.JPEG:n04067472 +ILSVRC2012_val_00047972.JPEG:n01872401 +ILSVRC2012_val_00047973.JPEG:n02114712 +ILSVRC2012_val_00047974.JPEG:n02979186 +ILSVRC2012_val_00047975.JPEG:n07615774 +ILSVRC2012_val_00047976.JPEG:n02094433 +ILSVRC2012_val_00047977.JPEG:n02106550 +ILSVRC2012_val_00047978.JPEG:n01930112 +ILSVRC2012_val_00047979.JPEG:n02086079 +ILSVRC2012_val_00047980.JPEG:n07754684 +ILSVRC2012_val_00047981.JPEG:n02088238 +ILSVRC2012_val_00047982.JPEG:n03764736 +ILSVRC2012_val_00047983.JPEG:n02077923 +ILSVRC2012_val_00047984.JPEG:n01770081 +ILSVRC2012_val_00047985.JPEG:n03763968 +ILSVRC2012_val_00047986.JPEG:n03544143 +ILSVRC2012_val_00047987.JPEG:n03777568 +ILSVRC2012_val_00047988.JPEG:n03706229 +ILSVRC2012_val_00047989.JPEG:n07871810 +ILSVRC2012_val_00047990.JPEG:n02100583 +ILSVRC2012_val_00047991.JPEG:n02096585 +ILSVRC2012_val_00047992.JPEG:n03538406 +ILSVRC2012_val_00047993.JPEG:n02794156 +ILSVRC2012_val_00047994.JPEG:n04325704 +ILSVRC2012_val_00047995.JPEG:n04127249 +ILSVRC2012_val_00047996.JPEG:n02277742 +ILSVRC2012_val_00047997.JPEG:n03314780 +ILSVRC2012_val_00047998.JPEG:n13037406 +ILSVRC2012_val_00047999.JPEG:n02607072 +ILSVRC2012_val_00048000.JPEG:n07720875 +ILSVRC2012_val_00048001.JPEG:n02277742 +ILSVRC2012_val_00048002.JPEG:n02412080 +ILSVRC2012_val_00048003.JPEG:n13054560 +ILSVRC2012_val_00048004.JPEG:n02865351 +ILSVRC2012_val_00048005.JPEG:n03467068 +ILSVRC2012_val_00048006.JPEG:n03891251 +ILSVRC2012_val_00048007.JPEG:n02089973 +ILSVRC2012_val_00048008.JPEG:n02002724 +ILSVRC2012_val_00048009.JPEG:n02017213 +ILSVRC2012_val_00048010.JPEG:n02917067 +ILSVRC2012_val_00048011.JPEG:n01665541 +ILSVRC2012_val_00048012.JPEG:n07714990 +ILSVRC2012_val_00048013.JPEG:n03372029 +ILSVRC2012_val_00048014.JPEG:n03584254 +ILSVRC2012_val_00048015.JPEG:n03662601 +ILSVRC2012_val_00048016.JPEG:n03337140 +ILSVRC2012_val_00048017.JPEG:n02692877 +ILSVRC2012_val_00048018.JPEG:n02110627 +ILSVRC2012_val_00048019.JPEG:n04201297 +ILSVRC2012_val_00048020.JPEG:n04154565 +ILSVRC2012_val_00048021.JPEG:n03637318 +ILSVRC2012_val_00048022.JPEG:n03255030 +ILSVRC2012_val_00048023.JPEG:n07745940 +ILSVRC2012_val_00048024.JPEG:n02056570 +ILSVRC2012_val_00048025.JPEG:n03895866 +ILSVRC2012_val_00048026.JPEG:n02169497 +ILSVRC2012_val_00048027.JPEG:n01818515 +ILSVRC2012_val_00048028.JPEG:n04493381 +ILSVRC2012_val_00048029.JPEG:n03041632 +ILSVRC2012_val_00048030.JPEG:n02110627 +ILSVRC2012_val_00048031.JPEG:n04553703 +ILSVRC2012_val_00048032.JPEG:n02099429 +ILSVRC2012_val_00048033.JPEG:n09428293 +ILSVRC2012_val_00048034.JPEG:n03495258 +ILSVRC2012_val_00048035.JPEG:n02483708 +ILSVRC2012_val_00048036.JPEG:n04336792 +ILSVRC2012_val_00048037.JPEG:n02825657 +ILSVRC2012_val_00048038.JPEG:n03891251 +ILSVRC2012_val_00048039.JPEG:n01860187 +ILSVRC2012_val_00048040.JPEG:n09472597 +ILSVRC2012_val_00048041.JPEG:n01753488 +ILSVRC2012_val_00048042.JPEG:n04540053 +ILSVRC2012_val_00048043.JPEG:n02895154 +ILSVRC2012_val_00048044.JPEG:n02321529 +ILSVRC2012_val_00048045.JPEG:n03259280 +ILSVRC2012_val_00048046.JPEG:n01630670 +ILSVRC2012_val_00048047.JPEG:n03000134 +ILSVRC2012_val_00048048.JPEG:n03866082 +ILSVRC2012_val_00048049.JPEG:n01514859 +ILSVRC2012_val_00048050.JPEG:n07873807 +ILSVRC2012_val_00048051.JPEG:n02105056 +ILSVRC2012_val_00048052.JPEG:n01978455 +ILSVRC2012_val_00048053.JPEG:n02009912 +ILSVRC2012_val_00048054.JPEG:n03794056 +ILSVRC2012_val_00048055.JPEG:n03720891 +ILSVRC2012_val_00048056.JPEG:n03995372 +ILSVRC2012_val_00048057.JPEG:n02869837 +ILSVRC2012_val_00048058.JPEG:n02169497 +ILSVRC2012_val_00048059.JPEG:n03425413 +ILSVRC2012_val_00048060.JPEG:n04355338 +ILSVRC2012_val_00048061.JPEG:n02977058 +ILSVRC2012_val_00048062.JPEG:n02916936 +ILSVRC2012_val_00048063.JPEG:n03840681 +ILSVRC2012_val_00048064.JPEG:n04560804 +ILSVRC2012_val_00048065.JPEG:n03042490 +ILSVRC2012_val_00048066.JPEG:n07734744 +ILSVRC2012_val_00048067.JPEG:n03706229 +ILSVRC2012_val_00048068.JPEG:n01774384 +ILSVRC2012_val_00048069.JPEG:n03530642 +ILSVRC2012_val_00048070.JPEG:n02346627 +ILSVRC2012_val_00048071.JPEG:n02105251 +ILSVRC2012_val_00048072.JPEG:n02229544 +ILSVRC2012_val_00048073.JPEG:n04522168 +ILSVRC2012_val_00048074.JPEG:n03535780 +ILSVRC2012_val_00048075.JPEG:n02105505 +ILSVRC2012_val_00048076.JPEG:n02168699 +ILSVRC2012_val_00048077.JPEG:n02138441 +ILSVRC2012_val_00048078.JPEG:n04131690 +ILSVRC2012_val_00048079.JPEG:n02172182 +ILSVRC2012_val_00048080.JPEG:n02111129 +ILSVRC2012_val_00048081.JPEG:n02776631 +ILSVRC2012_val_00048082.JPEG:n03785016 +ILSVRC2012_val_00048083.JPEG:n03895866 +ILSVRC2012_val_00048084.JPEG:n02457408 +ILSVRC2012_val_00048085.JPEG:n03146219 +ILSVRC2012_val_00048086.JPEG:n02134084 +ILSVRC2012_val_00048087.JPEG:n02097130 +ILSVRC2012_val_00048088.JPEG:n02361337 +ILSVRC2012_val_00048089.JPEG:n07720875 +ILSVRC2012_val_00048090.JPEG:n01871265 +ILSVRC2012_val_00048091.JPEG:n02231487 +ILSVRC2012_val_00048092.JPEG:n07717556 +ILSVRC2012_val_00048093.JPEG:n04328186 +ILSVRC2012_val_00048094.JPEG:n04317175 +ILSVRC2012_val_00048095.JPEG:n03065424 +ILSVRC2012_val_00048096.JPEG:n02442845 +ILSVRC2012_val_00048097.JPEG:n03729826 +ILSVRC2012_val_00048098.JPEG:n02892201 +ILSVRC2012_val_00048099.JPEG:n02489166 +ILSVRC2012_val_00048100.JPEG:n03721384 +ILSVRC2012_val_00048101.JPEG:n02096437 +ILSVRC2012_val_00048102.JPEG:n02093647 +ILSVRC2012_val_00048103.JPEG:n03376595 +ILSVRC2012_val_00048104.JPEG:n01692333 +ILSVRC2012_val_00048105.JPEG:n02134084 +ILSVRC2012_val_00048106.JPEG:n01978287 +ILSVRC2012_val_00048107.JPEG:n01592084 +ILSVRC2012_val_00048108.JPEG:n02504458 +ILSVRC2012_val_00048109.JPEG:n03544143 +ILSVRC2012_val_00048110.JPEG:n04039381 +ILSVRC2012_val_00048111.JPEG:n02690373 +ILSVRC2012_val_00048112.JPEG:n01756291 +ILSVRC2012_val_00048113.JPEG:n03814639 +ILSVRC2012_val_00048114.JPEG:n03443371 +ILSVRC2012_val_00048115.JPEG:n03633091 +ILSVRC2012_val_00048116.JPEG:n02066245 +ILSVRC2012_val_00048117.JPEG:n03868242 +ILSVRC2012_val_00048118.JPEG:n02133161 +ILSVRC2012_val_00048119.JPEG:n01496331 +ILSVRC2012_val_00048120.JPEG:n02108915 +ILSVRC2012_val_00048121.JPEG:n03325584 +ILSVRC2012_val_00048122.JPEG:n03372029 +ILSVRC2012_val_00048123.JPEG:n02085782 +ILSVRC2012_val_00048124.JPEG:n04026417 +ILSVRC2012_val_00048125.JPEG:n02111500 +ILSVRC2012_val_00048126.JPEG:n03482405 +ILSVRC2012_val_00048127.JPEG:n04149813 +ILSVRC2012_val_00048128.JPEG:n02108551 +ILSVRC2012_val_00048129.JPEG:n03337140 +ILSVRC2012_val_00048130.JPEG:n03970156 +ILSVRC2012_val_00048131.JPEG:n02443484 +ILSVRC2012_val_00048132.JPEG:n03657121 +ILSVRC2012_val_00048133.JPEG:n03633091 +ILSVRC2012_val_00048134.JPEG:n01675722 +ILSVRC2012_val_00048135.JPEG:n02965783 +ILSVRC2012_val_00048136.JPEG:n03908714 +ILSVRC2012_val_00048137.JPEG:n03777754 +ILSVRC2012_val_00048138.JPEG:n03394916 +ILSVRC2012_val_00048139.JPEG:n06794110 +ILSVRC2012_val_00048140.JPEG:n02492660 +ILSVRC2012_val_00048141.JPEG:n02099429 +ILSVRC2012_val_00048142.JPEG:n01828970 +ILSVRC2012_val_00048143.JPEG:n04404412 +ILSVRC2012_val_00048144.JPEG:n01532829 +ILSVRC2012_val_00048145.JPEG:n02109047 +ILSVRC2012_val_00048146.JPEG:n07768694 +ILSVRC2012_val_00048147.JPEG:n02104365 +ILSVRC2012_val_00048148.JPEG:n01632777 +ILSVRC2012_val_00048149.JPEG:n02794156 +ILSVRC2012_val_00048150.JPEG:n02807133 +ILSVRC2012_val_00048151.JPEG:n07615774 +ILSVRC2012_val_00048152.JPEG:n01532829 +ILSVRC2012_val_00048153.JPEG:n13040303 +ILSVRC2012_val_00048154.JPEG:n04149813 +ILSVRC2012_val_00048155.JPEG:n01828970 +ILSVRC2012_val_00048156.JPEG:n03345487 +ILSVRC2012_val_00048157.JPEG:n02096585 +ILSVRC2012_val_00048158.JPEG:n03291819 +ILSVRC2012_val_00048159.JPEG:n07754684 +ILSVRC2012_val_00048160.JPEG:n02123597 +ILSVRC2012_val_00048161.JPEG:n04266014 +ILSVRC2012_val_00048162.JPEG:n02114855 +ILSVRC2012_val_00048163.JPEG:n02018207 +ILSVRC2012_val_00048164.JPEG:n04532106 +ILSVRC2012_val_00048165.JPEG:n04579432 +ILSVRC2012_val_00048166.JPEG:n09246464 +ILSVRC2012_val_00048167.JPEG:n02088364 +ILSVRC2012_val_00048168.JPEG:n07615774 +ILSVRC2012_val_00048169.JPEG:n04487394 +ILSVRC2012_val_00048170.JPEG:n04612504 +ILSVRC2012_val_00048171.JPEG:n07613480 +ILSVRC2012_val_00048172.JPEG:n02058221 +ILSVRC2012_val_00048173.JPEG:n03980874 +ILSVRC2012_val_00048174.JPEG:n02134418 +ILSVRC2012_val_00048175.JPEG:n01622779 +ILSVRC2012_val_00048176.JPEG:n04209239 +ILSVRC2012_val_00048177.JPEG:n02692877 +ILSVRC2012_val_00048178.JPEG:n01560419 +ILSVRC2012_val_00048179.JPEG:n02870880 +ILSVRC2012_val_00048180.JPEG:n03445924 +ILSVRC2012_val_00048181.JPEG:n02117135 +ILSVRC2012_val_00048182.JPEG:n04356056 +ILSVRC2012_val_00048183.JPEG:n02097047 +ILSVRC2012_val_00048184.JPEG:n02281406 +ILSVRC2012_val_00048185.JPEG:n04243546 +ILSVRC2012_val_00048186.JPEG:n02129604 +ILSVRC2012_val_00048187.JPEG:n02395406 +ILSVRC2012_val_00048188.JPEG:n02089973 +ILSVRC2012_val_00048189.JPEG:n09332890 +ILSVRC2012_val_00048190.JPEG:n07747607 +ILSVRC2012_val_00048191.JPEG:n09246464 +ILSVRC2012_val_00048192.JPEG:n04417672 +ILSVRC2012_val_00048193.JPEG:n02859443 +ILSVRC2012_val_00048194.JPEG:n02105251 +ILSVRC2012_val_00048195.JPEG:n02012849 +ILSVRC2012_val_00048196.JPEG:n03724870 +ILSVRC2012_val_00048197.JPEG:n04562935 +ILSVRC2012_val_00048198.JPEG:n02790996 +ILSVRC2012_val_00048199.JPEG:n02825657 +ILSVRC2012_val_00048200.JPEG:n02510455 +ILSVRC2012_val_00048201.JPEG:n03884397 +ILSVRC2012_val_00048202.JPEG:n04069434 +ILSVRC2012_val_00048203.JPEG:n01843383 +ILSVRC2012_val_00048204.JPEG:n01440764 +ILSVRC2012_val_00048205.JPEG:n02909870 +ILSVRC2012_val_00048206.JPEG:n04344873 +ILSVRC2012_val_00048207.JPEG:n13054560 +ILSVRC2012_val_00048208.JPEG:n03976657 +ILSVRC2012_val_00048209.JPEG:n04270147 +ILSVRC2012_val_00048210.JPEG:n02804610 +ILSVRC2012_val_00048211.JPEG:n03792972 +ILSVRC2012_val_00048212.JPEG:n01704323 +ILSVRC2012_val_00048213.JPEG:n01689811 +ILSVRC2012_val_00048214.JPEG:n03908714 +ILSVRC2012_val_00048215.JPEG:n03062245 +ILSVRC2012_val_00048216.JPEG:n03376595 +ILSVRC2012_val_00048217.JPEG:n02442845 +ILSVRC2012_val_00048218.JPEG:n04589890 +ILSVRC2012_val_00048219.JPEG:n02114855 +ILSVRC2012_val_00048220.JPEG:n04465501 +ILSVRC2012_val_00048221.JPEG:n01664065 +ILSVRC2012_val_00048222.JPEG:n07711569 +ILSVRC2012_val_00048223.JPEG:n02457408 +ILSVRC2012_val_00048224.JPEG:n02165105 +ILSVRC2012_val_00048225.JPEG:n02389026 +ILSVRC2012_val_00048226.JPEG:n03207743 +ILSVRC2012_val_00048227.JPEG:n04081281 +ILSVRC2012_val_00048228.JPEG:n04458633 +ILSVRC2012_val_00048229.JPEG:n01843065 +ILSVRC2012_val_00048230.JPEG:n04335435 +ILSVRC2012_val_00048231.JPEG:n03444034 +ILSVRC2012_val_00048232.JPEG:n04311174 +ILSVRC2012_val_00048233.JPEG:n02128385 +ILSVRC2012_val_00048234.JPEG:n01819313 +ILSVRC2012_val_00048235.JPEG:n02098413 +ILSVRC2012_val_00048236.JPEG:n02110341 +ILSVRC2012_val_00048237.JPEG:n06874185 +ILSVRC2012_val_00048238.JPEG:n02098413 +ILSVRC2012_val_00048239.JPEG:n02007558 +ILSVRC2012_val_00048240.JPEG:n02077923 +ILSVRC2012_val_00048241.JPEG:n04461696 +ILSVRC2012_val_00048242.JPEG:n01514859 +ILSVRC2012_val_00048243.JPEG:n03388549 +ILSVRC2012_val_00048244.JPEG:n03447721 +ILSVRC2012_val_00048245.JPEG:n03207743 +ILSVRC2012_val_00048246.JPEG:n02443114 +ILSVRC2012_val_00048247.JPEG:n01664065 +ILSVRC2012_val_00048248.JPEG:n03825788 +ILSVRC2012_val_00048249.JPEG:n02799071 +ILSVRC2012_val_00048250.JPEG:n01753488 +ILSVRC2012_val_00048251.JPEG:n03642806 +ILSVRC2012_val_00048252.JPEG:n01847000 +ILSVRC2012_val_00048253.JPEG:n09421951 +ILSVRC2012_val_00048254.JPEG:n02086910 +ILSVRC2012_val_00048255.JPEG:n02441942 +ILSVRC2012_val_00048256.JPEG:n03141823 +ILSVRC2012_val_00048257.JPEG:n01664065 +ILSVRC2012_val_00048258.JPEG:n03642806 +ILSVRC2012_val_00048259.JPEG:n02364673 +ILSVRC2012_val_00048260.JPEG:n03884397 +ILSVRC2012_val_00048261.JPEG:n02033041 +ILSVRC2012_val_00048262.JPEG:n04019541 +ILSVRC2012_val_00048263.JPEG:n04266014 +ILSVRC2012_val_00048264.JPEG:n07749582 +ILSVRC2012_val_00048265.JPEG:n01818515 +ILSVRC2012_val_00048266.JPEG:n02415577 +ILSVRC2012_val_00048267.JPEG:n02804414 +ILSVRC2012_val_00048268.JPEG:n04599235 +ILSVRC2012_val_00048269.JPEG:n01910747 +ILSVRC2012_val_00048270.JPEG:n02965783 +ILSVRC2012_val_00048271.JPEG:n04111531 +ILSVRC2012_val_00048272.JPEG:n03794056 +ILSVRC2012_val_00048273.JPEG:n02088364 +ILSVRC2012_val_00048274.JPEG:n03733805 +ILSVRC2012_val_00048275.JPEG:n02497673 +ILSVRC2012_val_00048276.JPEG:n04296562 +ILSVRC2012_val_00048277.JPEG:n01983481 +ILSVRC2012_val_00048278.JPEG:n04041544 +ILSVRC2012_val_00048279.JPEG:n07892512 +ILSVRC2012_val_00048280.JPEG:n02085936 +ILSVRC2012_val_00048281.JPEG:n03929855 +ILSVRC2012_val_00048282.JPEG:n02396427 +ILSVRC2012_val_00048283.JPEG:n03854065 +ILSVRC2012_val_00048284.JPEG:n02802426 +ILSVRC2012_val_00048285.JPEG:n01751748 +ILSVRC2012_val_00048286.JPEG:n01632458 +ILSVRC2012_val_00048287.JPEG:n03207941 +ILSVRC2012_val_00048288.JPEG:n02110627 +ILSVRC2012_val_00048289.JPEG:n04554684 +ILSVRC2012_val_00048290.JPEG:n03729826 +ILSVRC2012_val_00048291.JPEG:n02480495 +ILSVRC2012_val_00048292.JPEG:n01914609 +ILSVRC2012_val_00048293.JPEG:n04200800 +ILSVRC2012_val_00048294.JPEG:n02480495 +ILSVRC2012_val_00048295.JPEG:n01630670 +ILSVRC2012_val_00048296.JPEG:n03825788 +ILSVRC2012_val_00048297.JPEG:n04458633 +ILSVRC2012_val_00048298.JPEG:n07754684 +ILSVRC2012_val_00048299.JPEG:n01756291 +ILSVRC2012_val_00048300.JPEG:n02807133 +ILSVRC2012_val_00048301.JPEG:n02099712 +ILSVRC2012_val_00048302.JPEG:n03223299 +ILSVRC2012_val_00048303.JPEG:n03394916 +ILSVRC2012_val_00048304.JPEG:n02100735 +ILSVRC2012_val_00048305.JPEG:n04548362 +ILSVRC2012_val_00048306.JPEG:n01774750 +ILSVRC2012_val_00048307.JPEG:n03085013 +ILSVRC2012_val_00048308.JPEG:n02974003 +ILSVRC2012_val_00048309.JPEG:n04004767 +ILSVRC2012_val_00048310.JPEG:n02111129 +ILSVRC2012_val_00048311.JPEG:n02113799 +ILSVRC2012_val_00048312.JPEG:n02963159 +ILSVRC2012_val_00048313.JPEG:n04275548 +ILSVRC2012_val_00048314.JPEG:n06874185 +ILSVRC2012_val_00048315.JPEG:n02105855 +ILSVRC2012_val_00048316.JPEG:n03710193 +ILSVRC2012_val_00048317.JPEG:n02916936 +ILSVRC2012_val_00048318.JPEG:n03125729 +ILSVRC2012_val_00048319.JPEG:n04209239 +ILSVRC2012_val_00048320.JPEG:n04033995 +ILSVRC2012_val_00048321.JPEG:n07930864 +ILSVRC2012_val_00048322.JPEG:n03443371 +ILSVRC2012_val_00048323.JPEG:n04604644 +ILSVRC2012_val_00048324.JPEG:n03788195 +ILSVRC2012_val_00048325.JPEG:n04238763 +ILSVRC2012_val_00048326.JPEG:n02174001 +ILSVRC2012_val_00048327.JPEG:n03637318 +ILSVRC2012_val_00048328.JPEG:n07615774 +ILSVRC2012_val_00048329.JPEG:n04200800 +ILSVRC2012_val_00048330.JPEG:n02107142 +ILSVRC2012_val_00048331.JPEG:n03709823 +ILSVRC2012_val_00048332.JPEG:n03786901 +ILSVRC2012_val_00048333.JPEG:n02086079 +ILSVRC2012_val_00048334.JPEG:n03201208 +ILSVRC2012_val_00048335.JPEG:n03000684 +ILSVRC2012_val_00048336.JPEG:n04099969 +ILSVRC2012_val_00048337.JPEG:n02102480 +ILSVRC2012_val_00048338.JPEG:n01950731 +ILSVRC2012_val_00048339.JPEG:n07753113 +ILSVRC2012_val_00048340.JPEG:n02013706 +ILSVRC2012_val_00048341.JPEG:n04536866 +ILSVRC2012_val_00048342.JPEG:n02423022 +ILSVRC2012_val_00048343.JPEG:n02687172 +ILSVRC2012_val_00048344.JPEG:n04208210 +ILSVRC2012_val_00048345.JPEG:n04596742 +ILSVRC2012_val_00048346.JPEG:n02051845 +ILSVRC2012_val_00048347.JPEG:n01833805 +ILSVRC2012_val_00048348.JPEG:n02058221 +ILSVRC2012_val_00048349.JPEG:n03344393 +ILSVRC2012_val_00048350.JPEG:n03857828 +ILSVRC2012_val_00048351.JPEG:n01978287 +ILSVRC2012_val_00048352.JPEG:n04118538 +ILSVRC2012_val_00048353.JPEG:n03976657 +ILSVRC2012_val_00048354.JPEG:n03717622 +ILSVRC2012_val_00048355.JPEG:n02097130 +ILSVRC2012_val_00048356.JPEG:n09399592 +ILSVRC2012_val_00048357.JPEG:n01768244 +ILSVRC2012_val_00048358.JPEG:n02317335 +ILSVRC2012_val_00048359.JPEG:n04204238 +ILSVRC2012_val_00048360.JPEG:n01580077 +ILSVRC2012_val_00048361.JPEG:n02097298 +ILSVRC2012_val_00048362.JPEG:n03673027 +ILSVRC2012_val_00048363.JPEG:n02013706 +ILSVRC2012_val_00048364.JPEG:n02105251 +ILSVRC2012_val_00048365.JPEG:n07697313 +ILSVRC2012_val_00048366.JPEG:n03980874 +ILSVRC2012_val_00048367.JPEG:n02804610 +ILSVRC2012_val_00048368.JPEG:n02125311 +ILSVRC2012_val_00048369.JPEG:n03781244 +ILSVRC2012_val_00048370.JPEG:n02095570 +ILSVRC2012_val_00048371.JPEG:n03344393 +ILSVRC2012_val_00048372.JPEG:n02408429 +ILSVRC2012_val_00048373.JPEG:n02110627 +ILSVRC2012_val_00048374.JPEG:n02807133 +ILSVRC2012_val_00048375.JPEG:n02129604 +ILSVRC2012_val_00048376.JPEG:n04332243 +ILSVRC2012_val_00048377.JPEG:n04398044 +ILSVRC2012_val_00048378.JPEG:n13044778 +ILSVRC2012_val_00048379.JPEG:n02098413 +ILSVRC2012_val_00048380.JPEG:n02129604 +ILSVRC2012_val_00048381.JPEG:n03763968 +ILSVRC2012_val_00048382.JPEG:n03028079 +ILSVRC2012_val_00048383.JPEG:n02108000 +ILSVRC2012_val_00048384.JPEG:n03825788 +ILSVRC2012_val_00048385.JPEG:n02116738 +ILSVRC2012_val_00048386.JPEG:n04344873 +ILSVRC2012_val_00048387.JPEG:n03924679 +ILSVRC2012_val_00048388.JPEG:n02486261 +ILSVRC2012_val_00048389.JPEG:n02667093 +ILSVRC2012_val_00048390.JPEG:n03584254 +ILSVRC2012_val_00048391.JPEG:n04554684 +ILSVRC2012_val_00048392.JPEG:n07932039 +ILSVRC2012_val_00048393.JPEG:n01872401 +ILSVRC2012_val_00048394.JPEG:n02128757 +ILSVRC2012_val_00048395.JPEG:n02966687 +ILSVRC2012_val_00048396.JPEG:n02101556 +ILSVRC2012_val_00048397.JPEG:n03207941 +ILSVRC2012_val_00048398.JPEG:n04476259 +ILSVRC2012_val_00048399.JPEG:n07684084 +ILSVRC2012_val_00048400.JPEG:n02109525 +ILSVRC2012_val_00048401.JPEG:n02268443 +ILSVRC2012_val_00048402.JPEG:n03793489 +ILSVRC2012_val_00048403.JPEG:n02106662 +ILSVRC2012_val_00048404.JPEG:n04335435 +ILSVRC2012_val_00048405.JPEG:n03146219 +ILSVRC2012_val_00048406.JPEG:n01774384 +ILSVRC2012_val_00048407.JPEG:n03980874 +ILSVRC2012_val_00048408.JPEG:n01930112 +ILSVRC2012_val_00048409.JPEG:n03485794 +ILSVRC2012_val_00048410.JPEG:n03710193 +ILSVRC2012_val_00048411.JPEG:n04525305 +ILSVRC2012_val_00048412.JPEG:n03916031 +ILSVRC2012_val_00048413.JPEG:n07565083 +ILSVRC2012_val_00048414.JPEG:n02264363 +ILSVRC2012_val_00048415.JPEG:n03676483 +ILSVRC2012_val_00048416.JPEG:n04235860 +ILSVRC2012_val_00048417.JPEG:n02808304 +ILSVRC2012_val_00048418.JPEG:n03796401 +ILSVRC2012_val_00048419.JPEG:n12620546 +ILSVRC2012_val_00048420.JPEG:n02098286 +ILSVRC2012_val_00048421.JPEG:n02091831 +ILSVRC2012_val_00048422.JPEG:n02319095 +ILSVRC2012_val_00048423.JPEG:n02264363 +ILSVRC2012_val_00048424.JPEG:n04317175 +ILSVRC2012_val_00048425.JPEG:n04120489 +ILSVRC2012_val_00048426.JPEG:n02788148 +ILSVRC2012_val_00048427.JPEG:n02110341 +ILSVRC2012_val_00048428.JPEG:n04252077 +ILSVRC2012_val_00048429.JPEG:n07715103 +ILSVRC2012_val_00048430.JPEG:n04540053 +ILSVRC2012_val_00048431.JPEG:n03016953 +ILSVRC2012_val_00048432.JPEG:n02091244 +ILSVRC2012_val_00048433.JPEG:n02640242 +ILSVRC2012_val_00048434.JPEG:n04612504 +ILSVRC2012_val_00048435.JPEG:n03000134 +ILSVRC2012_val_00048436.JPEG:n02112706 +ILSVRC2012_val_00048437.JPEG:n01532829 +ILSVRC2012_val_00048438.JPEG:n02115913 +ILSVRC2012_val_00048439.JPEG:n02101556 +ILSVRC2012_val_00048440.JPEG:n02119789 +ILSVRC2012_val_00048441.JPEG:n04252225 +ILSVRC2012_val_00048442.JPEG:n03492542 +ILSVRC2012_val_00048443.JPEG:n03272010 +ILSVRC2012_val_00048444.JPEG:n03770679 +ILSVRC2012_val_00048445.JPEG:n01629819 +ILSVRC2012_val_00048446.JPEG:n04517823 +ILSVRC2012_val_00048447.JPEG:n04366367 +ILSVRC2012_val_00048448.JPEG:n02410509 +ILSVRC2012_val_00048449.JPEG:n03623198 +ILSVRC2012_val_00048450.JPEG:n03777754 +ILSVRC2012_val_00048451.JPEG:n03899768 +ILSVRC2012_val_00048452.JPEG:n04367480 +ILSVRC2012_val_00048453.JPEG:n04525305 +ILSVRC2012_val_00048454.JPEG:n03208938 +ILSVRC2012_val_00048455.JPEG:n02951358 +ILSVRC2012_val_00048456.JPEG:n03110669 +ILSVRC2012_val_00048457.JPEG:n04483307 +ILSVRC2012_val_00048458.JPEG:n04517823 +ILSVRC2012_val_00048459.JPEG:n02422699 +ILSVRC2012_val_00048460.JPEG:n04509417 +ILSVRC2012_val_00048461.JPEG:n03590841 +ILSVRC2012_val_00048462.JPEG:n09332890 +ILSVRC2012_val_00048463.JPEG:n01629819 +ILSVRC2012_val_00048464.JPEG:n04557648 +ILSVRC2012_val_00048465.JPEG:n09421951 +ILSVRC2012_val_00048466.JPEG:n13052670 +ILSVRC2012_val_00048467.JPEG:n01677366 +ILSVRC2012_val_00048468.JPEG:n02058221 +ILSVRC2012_val_00048469.JPEG:n02102318 +ILSVRC2012_val_00048470.JPEG:n03126707 +ILSVRC2012_val_00048471.JPEG:n04548280 +ILSVRC2012_val_00048472.JPEG:n03187595 +ILSVRC2012_val_00048473.JPEG:n02966687 +ILSVRC2012_val_00048474.JPEG:n03938244 +ILSVRC2012_val_00048475.JPEG:n02486261 +ILSVRC2012_val_00048476.JPEG:n02096177 +ILSVRC2012_val_00048477.JPEG:n02165105 +ILSVRC2012_val_00048478.JPEG:n02979186 +ILSVRC2012_val_00048479.JPEG:n04310018 +ILSVRC2012_val_00048480.JPEG:n01669191 +ILSVRC2012_val_00048481.JPEG:n04356056 +ILSVRC2012_val_00048482.JPEG:n01644373 +ILSVRC2012_val_00048483.JPEG:n03676483 +ILSVRC2012_val_00048484.JPEG:n04311174 +ILSVRC2012_val_00048485.JPEG:n03617480 +ILSVRC2012_val_00048486.JPEG:n02107908 +ILSVRC2012_val_00048487.JPEG:n04310018 +ILSVRC2012_val_00048488.JPEG:n02100236 +ILSVRC2012_val_00048489.JPEG:n03623198 +ILSVRC2012_val_00048490.JPEG:n03841143 +ILSVRC2012_val_00048491.JPEG:n02488702 +ILSVRC2012_val_00048492.JPEG:n04507155 +ILSVRC2012_val_00048493.JPEG:n02097130 +ILSVRC2012_val_00048494.JPEG:n02769748 +ILSVRC2012_val_00048495.JPEG:n03781244 +ILSVRC2012_val_00048496.JPEG:n02441942 +ILSVRC2012_val_00048497.JPEG:n03240683 +ILSVRC2012_val_00048498.JPEG:n02115641 +ILSVRC2012_val_00048499.JPEG:n02117135 +ILSVRC2012_val_00048500.JPEG:n02137549 +ILSVRC2012_val_00048501.JPEG:n02113023 +ILSVRC2012_val_00048502.JPEG:n02129165 +ILSVRC2012_val_00048503.JPEG:n04532106 +ILSVRC2012_val_00048504.JPEG:n04118538 +ILSVRC2012_val_00048505.JPEG:n01774750 +ILSVRC2012_val_00048506.JPEG:n02917067 +ILSVRC2012_val_00048507.JPEG:n03394916 +ILSVRC2012_val_00048508.JPEG:n04458633 +ILSVRC2012_val_00048509.JPEG:n01704323 +ILSVRC2012_val_00048510.JPEG:n04399382 +ILSVRC2012_val_00048511.JPEG:n02410509 +ILSVRC2012_val_00048512.JPEG:n02111277 +ILSVRC2012_val_00048513.JPEG:n02102177 +ILSVRC2012_val_00048514.JPEG:n03000247 +ILSVRC2012_val_00048515.JPEG:n02107683 +ILSVRC2012_val_00048516.JPEG:n04037443 +ILSVRC2012_val_00048517.JPEG:n03445777 +ILSVRC2012_val_00048518.JPEG:n04296562 +ILSVRC2012_val_00048519.JPEG:n02971356 +ILSVRC2012_val_00048520.JPEG:n04418357 +ILSVRC2012_val_00048521.JPEG:n02730930 +ILSVRC2012_val_00048522.JPEG:n03841143 +ILSVRC2012_val_00048523.JPEG:n01774384 +ILSVRC2012_val_00048524.JPEG:n03271574 +ILSVRC2012_val_00048525.JPEG:n02443114 +ILSVRC2012_val_00048526.JPEG:n12144580 +ILSVRC2012_val_00048527.JPEG:n02097298 +ILSVRC2012_val_00048528.JPEG:n02948072 +ILSVRC2012_val_00048529.JPEG:n04179913 +ILSVRC2012_val_00048530.JPEG:n02105251 +ILSVRC2012_val_00048531.JPEG:n03888605 +ILSVRC2012_val_00048532.JPEG:n03208938 +ILSVRC2012_val_00048533.JPEG:n04265275 +ILSVRC2012_val_00048534.JPEG:n09421951 +ILSVRC2012_val_00048535.JPEG:n02408429 +ILSVRC2012_val_00048536.JPEG:n02101388 +ILSVRC2012_val_00048537.JPEG:n02105056 +ILSVRC2012_val_00048538.JPEG:n07836838 +ILSVRC2012_val_00048539.JPEG:n04591713 +ILSVRC2012_val_00048540.JPEG:n02011460 +ILSVRC2012_val_00048541.JPEG:n04532106 +ILSVRC2012_val_00048542.JPEG:n01698640 +ILSVRC2012_val_00048543.JPEG:n04330267 +ILSVRC2012_val_00048544.JPEG:n04039381 +ILSVRC2012_val_00048545.JPEG:n04542943 +ILSVRC2012_val_00048546.JPEG:n02317335 +ILSVRC2012_val_00048547.JPEG:n02504013 +ILSVRC2012_val_00048548.JPEG:n01704323 +ILSVRC2012_val_00048549.JPEG:n01829413 +ILSVRC2012_val_00048550.JPEG:n04357314 +ILSVRC2012_val_00048551.JPEG:n04252077 +ILSVRC2012_val_00048552.JPEG:n01601694 +ILSVRC2012_val_00048553.JPEG:n02006656 +ILSVRC2012_val_00048554.JPEG:n03124043 +ILSVRC2012_val_00048555.JPEG:n02965783 +ILSVRC2012_val_00048556.JPEG:n02814533 +ILSVRC2012_val_00048557.JPEG:n03347037 +ILSVRC2012_val_00048558.JPEG:n03920288 +ILSVRC2012_val_00048559.JPEG:n03874599 +ILSVRC2012_val_00048560.JPEG:n02364673 +ILSVRC2012_val_00048561.JPEG:n03496892 +ILSVRC2012_val_00048562.JPEG:n01978455 +ILSVRC2012_val_00048563.JPEG:n03544143 +ILSVRC2012_val_00048564.JPEG:n04252077 +ILSVRC2012_val_00048565.JPEG:n03630383 +ILSVRC2012_val_00048566.JPEG:n03717622 +ILSVRC2012_val_00048567.JPEG:n03141823 +ILSVRC2012_val_00048568.JPEG:n04259630 +ILSVRC2012_val_00048569.JPEG:n03785016 +ILSVRC2012_val_00048570.JPEG:n02174001 +ILSVRC2012_val_00048571.JPEG:n02869837 +ILSVRC2012_val_00048572.JPEG:n04335435 +ILSVRC2012_val_00048573.JPEG:n02687172 +ILSVRC2012_val_00048574.JPEG:n01729977 +ILSVRC2012_val_00048575.JPEG:n02018795 +ILSVRC2012_val_00048576.JPEG:n01494475 +ILSVRC2012_val_00048577.JPEG:n03529860 +ILSVRC2012_val_00048578.JPEG:n02106166 +ILSVRC2012_val_00048579.JPEG:n04553703 +ILSVRC2012_val_00048580.JPEG:n04523525 +ILSVRC2012_val_00048581.JPEG:n02445715 +ILSVRC2012_val_00048582.JPEG:n03891332 +ILSVRC2012_val_00048583.JPEG:n02747177 +ILSVRC2012_val_00048584.JPEG:n03676483 +ILSVRC2012_val_00048585.JPEG:n02667093 +ILSVRC2012_val_00048586.JPEG:n07920052 +ILSVRC2012_val_00048587.JPEG:n02910353 +ILSVRC2012_val_00048588.JPEG:n02097209 +ILSVRC2012_val_00048589.JPEG:n03991062 +ILSVRC2012_val_00048590.JPEG:n04204238 +ILSVRC2012_val_00048591.JPEG:n02110341 +ILSVRC2012_val_00048592.JPEG:n02089867 +ILSVRC2012_val_00048593.JPEG:n01776313 +ILSVRC2012_val_00048594.JPEG:n02328150 +ILSVRC2012_val_00048595.JPEG:n03180011 +ILSVRC2012_val_00048596.JPEG:n07717410 +ILSVRC2012_val_00048597.JPEG:n03047690 +ILSVRC2012_val_00048598.JPEG:n04505470 +ILSVRC2012_val_00048599.JPEG:n03014705 +ILSVRC2012_val_00048600.JPEG:n01518878 +ILSVRC2012_val_00048601.JPEG:n01807496 +ILSVRC2012_val_00048602.JPEG:n04591713 +ILSVRC2012_val_00048603.JPEG:n02999410 +ILSVRC2012_val_00048604.JPEG:n04254777 +ILSVRC2012_val_00048605.JPEG:n02870880 +ILSVRC2012_val_00048606.JPEG:n02002556 +ILSVRC2012_val_00048607.JPEG:n02095889 +ILSVRC2012_val_00048608.JPEG:n02487347 +ILSVRC2012_val_00048609.JPEG:n03944341 +ILSVRC2012_val_00048610.JPEG:n03770679 +ILSVRC2012_val_00048611.JPEG:n03794056 +ILSVRC2012_val_00048612.JPEG:n03759954 +ILSVRC2012_val_00048613.JPEG:n02093991 +ILSVRC2012_val_00048614.JPEG:n01968897 +ILSVRC2012_val_00048615.JPEG:n03743016 +ILSVRC2012_val_00048616.JPEG:n03388183 +ILSVRC2012_val_00048617.JPEG:n03775546 +ILSVRC2012_val_00048618.JPEG:n02437312 +ILSVRC2012_val_00048619.JPEG:n04120489 +ILSVRC2012_val_00048620.JPEG:n03642806 +ILSVRC2012_val_00048621.JPEG:n02808440 +ILSVRC2012_val_00048622.JPEG:n04099969 +ILSVRC2012_val_00048623.JPEG:n03891332 +ILSVRC2012_val_00048624.JPEG:n03958227 +ILSVRC2012_val_00048625.JPEG:n02113799 +ILSVRC2012_val_00048626.JPEG:n03998194 +ILSVRC2012_val_00048627.JPEG:n02104029 +ILSVRC2012_val_00048628.JPEG:n03250847 +ILSVRC2012_val_00048629.JPEG:n02100877 +ILSVRC2012_val_00048630.JPEG:n07714990 +ILSVRC2012_val_00048631.JPEG:n03110669 +ILSVRC2012_val_00048632.JPEG:n02676566 +ILSVRC2012_val_00048633.JPEG:n03347037 +ILSVRC2012_val_00048634.JPEG:n03530642 +ILSVRC2012_val_00048635.JPEG:n10565667 +ILSVRC2012_val_00048636.JPEG:n02108000 +ILSVRC2012_val_00048637.JPEG:n03110669 +ILSVRC2012_val_00048638.JPEG:n03690938 +ILSVRC2012_val_00048639.JPEG:n02095314 +ILSVRC2012_val_00048640.JPEG:n02012849 +ILSVRC2012_val_00048641.JPEG:n02277742 +ILSVRC2012_val_00048642.JPEG:n01532829 +ILSVRC2012_val_00048643.JPEG:n04553703 +ILSVRC2012_val_00048644.JPEG:n02051845 +ILSVRC2012_val_00048645.JPEG:n04456115 +ILSVRC2012_val_00048646.JPEG:n03998194 +ILSVRC2012_val_00048647.JPEG:n02417914 +ILSVRC2012_val_00048648.JPEG:n03594734 +ILSVRC2012_val_00048649.JPEG:n01775062 +ILSVRC2012_val_00048650.JPEG:n02105855 +ILSVRC2012_val_00048651.JPEG:n03903868 +ILSVRC2012_val_00048652.JPEG:n02096294 +ILSVRC2012_val_00048653.JPEG:n04371774 +ILSVRC2012_val_00048654.JPEG:n02927161 +ILSVRC2012_val_00048655.JPEG:n03657121 +ILSVRC2012_val_00048656.JPEG:n03937543 +ILSVRC2012_val_00048657.JPEG:n04532106 +ILSVRC2012_val_00048658.JPEG:n01883070 +ILSVRC2012_val_00048659.JPEG:n01537544 +ILSVRC2012_val_00048660.JPEG:n02667093 +ILSVRC2012_val_00048661.JPEG:n02104029 +ILSVRC2012_val_00048662.JPEG:n02487347 +ILSVRC2012_val_00048663.JPEG:n02104365 +ILSVRC2012_val_00048664.JPEG:n02051845 +ILSVRC2012_val_00048665.JPEG:n04243546 +ILSVRC2012_val_00048666.JPEG:n02006656 +ILSVRC2012_val_00048667.JPEG:n02808304 +ILSVRC2012_val_00048668.JPEG:n04251144 +ILSVRC2012_val_00048669.JPEG:n02356798 +ILSVRC2012_val_00048670.JPEG:n02391049 +ILSVRC2012_val_00048671.JPEG:n07753275 +ILSVRC2012_val_00048672.JPEG:n02974003 +ILSVRC2012_val_00048673.JPEG:n03482405 +ILSVRC2012_val_00048674.JPEG:n09193705 +ILSVRC2012_val_00048675.JPEG:n01694178 +ILSVRC2012_val_00048676.JPEG:n02168699 +ILSVRC2012_val_00048677.JPEG:n12768682 +ILSVRC2012_val_00048678.JPEG:n03272562 +ILSVRC2012_val_00048679.JPEG:n03710193 +ILSVRC2012_val_00048680.JPEG:n03843555 +ILSVRC2012_val_00048681.JPEG:n03126707 +ILSVRC2012_val_00048682.JPEG:n03196217 +ILSVRC2012_val_00048683.JPEG:n06785654 +ILSVRC2012_val_00048684.JPEG:n04350905 +ILSVRC2012_val_00048685.JPEG:n07873807 +ILSVRC2012_val_00048686.JPEG:n04310018 +ILSVRC2012_val_00048687.JPEG:n02264363 +ILSVRC2012_val_00048688.JPEG:n02492660 +ILSVRC2012_val_00048689.JPEG:n10565667 +ILSVRC2012_val_00048690.JPEG:n04275548 +ILSVRC2012_val_00048691.JPEG:n04147183 +ILSVRC2012_val_00048692.JPEG:n04366367 +ILSVRC2012_val_00048693.JPEG:n02114855 +ILSVRC2012_val_00048694.JPEG:n02100236 +ILSVRC2012_val_00048695.JPEG:n04154565 +ILSVRC2012_val_00048696.JPEG:n02276258 +ILSVRC2012_val_00048697.JPEG:n03424325 +ILSVRC2012_val_00048698.JPEG:n03777568 +ILSVRC2012_val_00048699.JPEG:n03494278 +ILSVRC2012_val_00048700.JPEG:n01806143 +ILSVRC2012_val_00048701.JPEG:n03459775 +ILSVRC2012_val_00048702.JPEG:n03598930 +ILSVRC2012_val_00048703.JPEG:n03967562 +ILSVRC2012_val_00048704.JPEG:n03775546 +ILSVRC2012_val_00048705.JPEG:n04418357 +ILSVRC2012_val_00048706.JPEG:n02412080 +ILSVRC2012_val_00048707.JPEG:n04591157 +ILSVRC2012_val_00048708.JPEG:n01770081 +ILSVRC2012_val_00048709.JPEG:n03877472 +ILSVRC2012_val_00048710.JPEG:n01531178 +ILSVRC2012_val_00048711.JPEG:n03794056 +ILSVRC2012_val_00048712.JPEG:n04485082 +ILSVRC2012_val_00048713.JPEG:n03786901 +ILSVRC2012_val_00048714.JPEG:n01773797 +ILSVRC2012_val_00048715.JPEG:n04254680 +ILSVRC2012_val_00048716.JPEG:n02128925 +ILSVRC2012_val_00048717.JPEG:n02128757 +ILSVRC2012_val_00048718.JPEG:n02442845 +ILSVRC2012_val_00048719.JPEG:n02606052 +ILSVRC2012_val_00048720.JPEG:n02099429 +ILSVRC2012_val_00048721.JPEG:n04442312 +ILSVRC2012_val_00048722.JPEG:n01807496 +ILSVRC2012_val_00048723.JPEG:n02107312 +ILSVRC2012_val_00048724.JPEG:n03710637 +ILSVRC2012_val_00048725.JPEG:n02027492 +ILSVRC2012_val_00048726.JPEG:n03016953 +ILSVRC2012_val_00048727.JPEG:n02017213 +ILSVRC2012_val_00048728.JPEG:n12768682 +ILSVRC2012_val_00048729.JPEG:n04192698 +ILSVRC2012_val_00048730.JPEG:n02747177 +ILSVRC2012_val_00048731.JPEG:n04532106 +ILSVRC2012_val_00048732.JPEG:n01537544 +ILSVRC2012_val_00048733.JPEG:n04254777 +ILSVRC2012_val_00048734.JPEG:n03259280 +ILSVRC2012_val_00048735.JPEG:n02025239 +ILSVRC2012_val_00048736.JPEG:n09835506 +ILSVRC2012_val_00048737.JPEG:n02096437 +ILSVRC2012_val_00048738.JPEG:n04372370 +ILSVRC2012_val_00048739.JPEG:n02797295 +ILSVRC2012_val_00048740.JPEG:n03871628 +ILSVRC2012_val_00048741.JPEG:n02481823 +ILSVRC2012_val_00048742.JPEG:n03837869 +ILSVRC2012_val_00048743.JPEG:n02268443 +ILSVRC2012_val_00048744.JPEG:n04522168 +ILSVRC2012_val_00048745.JPEG:n03690938 +ILSVRC2012_val_00048746.JPEG:n04550184 +ILSVRC2012_val_00048747.JPEG:n03657121 +ILSVRC2012_val_00048748.JPEG:n02105251 +ILSVRC2012_val_00048749.JPEG:n01833805 +ILSVRC2012_val_00048750.JPEG:n01755581 +ILSVRC2012_val_00048751.JPEG:n07734744 +ILSVRC2012_val_00048752.JPEG:n01873310 +ILSVRC2012_val_00048753.JPEG:n03538406 +ILSVRC2012_val_00048754.JPEG:n01688243 +ILSVRC2012_val_00048755.JPEG:n03452741 +ILSVRC2012_val_00048756.JPEG:n02120505 +ILSVRC2012_val_00048757.JPEG:n02412080 +ILSVRC2012_val_00048758.JPEG:n04254120 +ILSVRC2012_val_00048759.JPEG:n04019541 +ILSVRC2012_val_00048760.JPEG:n02112706 +ILSVRC2012_val_00048761.JPEG:n02100735 +ILSVRC2012_val_00048762.JPEG:n03201208 +ILSVRC2012_val_00048763.JPEG:n03134739 +ILSVRC2012_val_00048764.JPEG:n02514041 +ILSVRC2012_val_00048765.JPEG:n04065272 +ILSVRC2012_val_00048766.JPEG:n02165105 +ILSVRC2012_val_00048767.JPEG:n04443257 +ILSVRC2012_val_00048768.JPEG:n04149813 +ILSVRC2012_val_00048769.JPEG:n03871628 +ILSVRC2012_val_00048770.JPEG:n02100236 +ILSVRC2012_val_00048771.JPEG:n02412080 +ILSVRC2012_val_00048772.JPEG:n02992211 +ILSVRC2012_val_00048773.JPEG:n02951358 +ILSVRC2012_val_00048774.JPEG:n03776460 +ILSVRC2012_val_00048775.JPEG:n02666196 +ILSVRC2012_val_00048776.JPEG:n03000134 +ILSVRC2012_val_00048777.JPEG:n12144580 +ILSVRC2012_val_00048778.JPEG:n03141823 +ILSVRC2012_val_00048779.JPEG:n02110341 +ILSVRC2012_val_00048780.JPEG:n02094114 +ILSVRC2012_val_00048781.JPEG:n02504458 +ILSVRC2012_val_00048782.JPEG:n04389033 +ILSVRC2012_val_00048783.JPEG:n02085936 +ILSVRC2012_val_00048784.JPEG:n04553703 +ILSVRC2012_val_00048785.JPEG:n03594734 +ILSVRC2012_val_00048786.JPEG:n09468604 +ILSVRC2012_val_00048787.JPEG:n03980874 +ILSVRC2012_val_00048788.JPEG:n07831146 +ILSVRC2012_val_00048789.JPEG:n03141823 +ILSVRC2012_val_00048790.JPEG:n13054560 +ILSVRC2012_val_00048791.JPEG:n01704323 +ILSVRC2012_val_00048792.JPEG:n02356798 +ILSVRC2012_val_00048793.JPEG:n03970156 +ILSVRC2012_val_00048794.JPEG:n02071294 +ILSVRC2012_val_00048795.JPEG:n06794110 +ILSVRC2012_val_00048796.JPEG:n02860847 +ILSVRC2012_val_00048797.JPEG:n03970156 +ILSVRC2012_val_00048798.JPEG:n11879895 +ILSVRC2012_val_00048799.JPEG:n04389033 +ILSVRC2012_val_00048800.JPEG:n01770393 +ILSVRC2012_val_00048801.JPEG:n02104365 +ILSVRC2012_val_00048802.JPEG:n02033041 +ILSVRC2012_val_00048803.JPEG:n07754684 +ILSVRC2012_val_00048804.JPEG:n02666196 +ILSVRC2012_val_00048805.JPEG:n03658185 +ILSVRC2012_val_00048806.JPEG:n03447447 +ILSVRC2012_val_00048807.JPEG:n03840681 +ILSVRC2012_val_00048808.JPEG:n01990800 +ILSVRC2012_val_00048809.JPEG:n03992509 +ILSVRC2012_val_00048810.JPEG:n02319095 +ILSVRC2012_val_00048811.JPEG:n04540053 +ILSVRC2012_val_00048812.JPEG:n04141975 +ILSVRC2012_val_00048813.JPEG:n03026506 +ILSVRC2012_val_00048814.JPEG:n02009229 +ILSVRC2012_val_00048815.JPEG:n07880968 +ILSVRC2012_val_00048816.JPEG:n03459775 +ILSVRC2012_val_00048817.JPEG:n02488291 +ILSVRC2012_val_00048818.JPEG:n02108551 +ILSVRC2012_val_00048819.JPEG:n03793489 +ILSVRC2012_val_00048820.JPEG:n03041632 +ILSVRC2012_val_00048821.JPEG:n03887697 +ILSVRC2012_val_00048822.JPEG:n12057211 +ILSVRC2012_val_00048823.JPEG:n07875152 +ILSVRC2012_val_00048824.JPEG:n01828970 +ILSVRC2012_val_00048825.JPEG:n01796340 +ILSVRC2012_val_00048826.JPEG:n03494278 +ILSVRC2012_val_00048827.JPEG:n02281787 +ILSVRC2012_val_00048828.JPEG:n01698640 +ILSVRC2012_val_00048829.JPEG:n01537544 +ILSVRC2012_val_00048830.JPEG:n02110185 +ILSVRC2012_val_00048831.JPEG:n04209133 +ILSVRC2012_val_00048832.JPEG:n02536864 +ILSVRC2012_val_00048833.JPEG:n07714990 +ILSVRC2012_val_00048834.JPEG:n02100236 +ILSVRC2012_val_00048835.JPEG:n04317175 +ILSVRC2012_val_00048836.JPEG:n04265275 +ILSVRC2012_val_00048837.JPEG:n01983481 +ILSVRC2012_val_00048838.JPEG:n01833805 +ILSVRC2012_val_00048839.JPEG:n02808440 +ILSVRC2012_val_00048840.JPEG:n01443537 +ILSVRC2012_val_00048841.JPEG:n07697313 +ILSVRC2012_val_00048842.JPEG:n02109525 +ILSVRC2012_val_00048843.JPEG:n03935335 +ILSVRC2012_val_00048844.JPEG:n03903868 +ILSVRC2012_val_00048845.JPEG:n04074963 +ILSVRC2012_val_00048846.JPEG:n01807496 +ILSVRC2012_val_00048847.JPEG:n03729826 +ILSVRC2012_val_00048848.JPEG:n04111531 +ILSVRC2012_val_00048849.JPEG:n07860988 +ILSVRC2012_val_00048850.JPEG:n04133789 +ILSVRC2012_val_00048851.JPEG:n03873416 +ILSVRC2012_val_00048852.JPEG:n03991062 +ILSVRC2012_val_00048853.JPEG:n03028079 +ILSVRC2012_val_00048854.JPEG:n03207743 +ILSVRC2012_val_00048855.JPEG:n02487347 +ILSVRC2012_val_00048856.JPEG:n03207941 +ILSVRC2012_val_00048857.JPEG:n03920288 +ILSVRC2012_val_00048858.JPEG:n02100735 +ILSVRC2012_val_00048859.JPEG:n02105855 +ILSVRC2012_val_00048860.JPEG:n03544143 +ILSVRC2012_val_00048861.JPEG:n02071294 +ILSVRC2012_val_00048862.JPEG:n03496892 +ILSVRC2012_val_00048863.JPEG:n03461385 +ILSVRC2012_val_00048864.JPEG:n01443537 +ILSVRC2012_val_00048865.JPEG:n04239074 +ILSVRC2012_val_00048866.JPEG:n03956157 +ILSVRC2012_val_00048867.JPEG:n04553703 +ILSVRC2012_val_00048868.JPEG:n04371430 +ILSVRC2012_val_00048869.JPEG:n12057211 +ILSVRC2012_val_00048870.JPEG:n04118776 +ILSVRC2012_val_00048871.JPEG:n02793495 +ILSVRC2012_val_00048872.JPEG:n02808304 +ILSVRC2012_val_00048873.JPEG:n03709823 +ILSVRC2012_val_00048874.JPEG:n02099267 +ILSVRC2012_val_00048875.JPEG:n03063599 +ILSVRC2012_val_00048876.JPEG:n03018349 +ILSVRC2012_val_00048877.JPEG:n02009912 +ILSVRC2012_val_00048878.JPEG:n03467068 +ILSVRC2012_val_00048879.JPEG:n03637318 +ILSVRC2012_val_00048880.JPEG:n12998815 +ILSVRC2012_val_00048881.JPEG:n04153751 +ILSVRC2012_val_00048882.JPEG:n03063599 +ILSVRC2012_val_00048883.JPEG:n02132136 +ILSVRC2012_val_00048884.JPEG:n02879718 +ILSVRC2012_val_00048885.JPEG:n02835271 +ILSVRC2012_val_00048886.JPEG:n03089624 +ILSVRC2012_val_00048887.JPEG:n01734418 +ILSVRC2012_val_00048888.JPEG:n02027492 +ILSVRC2012_val_00048889.JPEG:n04133789 +ILSVRC2012_val_00048890.JPEG:n01491361 +ILSVRC2012_val_00048891.JPEG:n03041632 +ILSVRC2012_val_00048892.JPEG:n02361337 +ILSVRC2012_val_00048893.JPEG:n03710637 +ILSVRC2012_val_00048894.JPEG:n02169497 +ILSVRC2012_val_00048895.JPEG:n02268443 +ILSVRC2012_val_00048896.JPEG:n03291819 +ILSVRC2012_val_00048897.JPEG:n02492660 +ILSVRC2012_val_00048898.JPEG:n04069434 +ILSVRC2012_val_00048899.JPEG:n03457902 +ILSVRC2012_val_00048900.JPEG:n04200800 +ILSVRC2012_val_00048901.JPEG:n04429376 +ILSVRC2012_val_00048902.JPEG:n01945685 +ILSVRC2012_val_00048903.JPEG:n02910353 +ILSVRC2012_val_00048904.JPEG:n02096177 +ILSVRC2012_val_00048905.JPEG:n04204347 +ILSVRC2012_val_00048906.JPEG:n03347037 +ILSVRC2012_val_00048907.JPEG:n01806567 +ILSVRC2012_val_00048908.JPEG:n02002724 +ILSVRC2012_val_00048909.JPEG:n01675722 +ILSVRC2012_val_00048910.JPEG:n04404412 +ILSVRC2012_val_00048911.JPEG:n03476684 +ILSVRC2012_val_00048912.JPEG:n03868242 +ILSVRC2012_val_00048913.JPEG:n01773157 +ILSVRC2012_val_00048914.JPEG:n02102040 +ILSVRC2012_val_00048915.JPEG:n02088094 +ILSVRC2012_val_00048916.JPEG:n02797295 +ILSVRC2012_val_00048917.JPEG:n07831146 +ILSVRC2012_val_00048918.JPEG:n03764736 +ILSVRC2012_val_00048919.JPEG:n03000684 +ILSVRC2012_val_00048920.JPEG:n02536864 +ILSVRC2012_val_00048921.JPEG:n01983481 +ILSVRC2012_val_00048922.JPEG:n02106550 +ILSVRC2012_val_00048923.JPEG:n04065272 +ILSVRC2012_val_00048924.JPEG:n01685808 +ILSVRC2012_val_00048925.JPEG:n02090622 +ILSVRC2012_val_00048926.JPEG:n04579432 +ILSVRC2012_val_00048927.JPEG:n04204238 +ILSVRC2012_val_00048928.JPEG:n13054560 +ILSVRC2012_val_00048929.JPEG:n03016953 +ILSVRC2012_val_00048930.JPEG:n03937543 +ILSVRC2012_val_00048931.JPEG:n04229816 +ILSVRC2012_val_00048932.JPEG:n02492660 +ILSVRC2012_val_00048933.JPEG:n03445924 +ILSVRC2012_val_00048934.JPEG:n11939491 +ILSVRC2012_val_00048935.JPEG:n03544143 +ILSVRC2012_val_00048936.JPEG:n02894605 +ILSVRC2012_val_00048937.JPEG:n07697537 +ILSVRC2012_val_00048938.JPEG:n04153751 +ILSVRC2012_val_00048939.JPEG:n02483362 +ILSVRC2012_val_00048940.JPEG:n02134084 +ILSVRC2012_val_00048941.JPEG:n04208210 +ILSVRC2012_val_00048942.JPEG:n03197337 +ILSVRC2012_val_00048943.JPEG:n01753488 +ILSVRC2012_val_00048944.JPEG:n03680355 +ILSVRC2012_val_00048945.JPEG:n03938244 +ILSVRC2012_val_00048946.JPEG:n03857828 +ILSVRC2012_val_00048947.JPEG:n03761084 +ILSVRC2012_val_00048948.JPEG:n02105162 +ILSVRC2012_val_00048949.JPEG:n03742115 +ILSVRC2012_val_00048950.JPEG:n02536864 +ILSVRC2012_val_00048951.JPEG:n02930766 +ILSVRC2012_val_00048952.JPEG:n01514668 +ILSVRC2012_val_00048953.JPEG:n03876231 +ILSVRC2012_val_00048954.JPEG:n02493509 +ILSVRC2012_val_00048955.JPEG:n02095314 +ILSVRC2012_val_00048956.JPEG:n04517823 +ILSVRC2012_val_00048957.JPEG:n01729977 +ILSVRC2012_val_00048958.JPEG:n04442312 +ILSVRC2012_val_00048959.JPEG:n11939491 +ILSVRC2012_val_00048960.JPEG:n01614925 +ILSVRC2012_val_00048961.JPEG:n03496892 +ILSVRC2012_val_00048962.JPEG:n02281787 +ILSVRC2012_val_00048963.JPEG:n02095570 +ILSVRC2012_val_00048964.JPEG:n02105505 +ILSVRC2012_val_00048965.JPEG:n04127249 +ILSVRC2012_val_00048966.JPEG:n04579432 +ILSVRC2012_val_00048967.JPEG:n03804744 +ILSVRC2012_val_00048968.JPEG:n04613696 +ILSVRC2012_val_00048969.JPEG:n01440764 +ILSVRC2012_val_00048970.JPEG:n04133789 +ILSVRC2012_val_00048971.JPEG:n02115641 +ILSVRC2012_val_00048972.JPEG:n02099849 +ILSVRC2012_val_00048973.JPEG:n04493381 +ILSVRC2012_val_00048974.JPEG:n02102480 +ILSVRC2012_val_00048975.JPEG:n11939491 +ILSVRC2012_val_00048976.JPEG:n07565083 +ILSVRC2012_val_00048977.JPEG:n03425413 +ILSVRC2012_val_00048978.JPEG:n01756291 +ILSVRC2012_val_00048979.JPEG:n02132136 +ILSVRC2012_val_00048980.JPEG:n02109525 +ILSVRC2012_val_00048981.JPEG:n03995372 +ILSVRC2012_val_00048982.JPEG:n12057211 +ILSVRC2012_val_00048983.JPEG:n07697537 +ILSVRC2012_val_00048984.JPEG:n04023962 +ILSVRC2012_val_00048985.JPEG:n03690938 +ILSVRC2012_val_00048986.JPEG:n03676483 +ILSVRC2012_val_00048987.JPEG:n03868863 +ILSVRC2012_val_00048988.JPEG:n04147183 +ILSVRC2012_val_00048989.JPEG:n02895154 +ILSVRC2012_val_00048990.JPEG:n01773549 +ILSVRC2012_val_00048991.JPEG:n01667114 +ILSVRC2012_val_00048992.JPEG:n12267677 +ILSVRC2012_val_00048993.JPEG:n04507155 +ILSVRC2012_val_00048994.JPEG:n03658185 +ILSVRC2012_val_00048995.JPEG:n01644373 +ILSVRC2012_val_00048996.JPEG:n06785654 +ILSVRC2012_val_00048997.JPEG:n02114548 +ILSVRC2012_val_00048998.JPEG:n04065272 +ILSVRC2012_val_00048999.JPEG:n04118538 +ILSVRC2012_val_00049000.JPEG:n01491361 +ILSVRC2012_val_00049001.JPEG:n03792782 +ILSVRC2012_val_00049002.JPEG:n03773504 +ILSVRC2012_val_00049003.JPEG:n07831146 +ILSVRC2012_val_00049004.JPEG:n02092002 +ILSVRC2012_val_00049005.JPEG:n02808304 +ILSVRC2012_val_00049006.JPEG:n04330267 +ILSVRC2012_val_00049007.JPEG:n02437312 +ILSVRC2012_val_00049008.JPEG:n03481172 +ILSVRC2012_val_00049009.JPEG:n03706229 +ILSVRC2012_val_00049010.JPEG:n02100583 +ILSVRC2012_val_00049011.JPEG:n04347754 +ILSVRC2012_val_00049012.JPEG:n02666196 +ILSVRC2012_val_00049013.JPEG:n04074963 +ILSVRC2012_val_00049014.JPEG:n03976467 +ILSVRC2012_val_00049015.JPEG:n02090721 +ILSVRC2012_val_00049016.JPEG:n02002556 +ILSVRC2012_val_00049017.JPEG:n01728572 +ILSVRC2012_val_00049018.JPEG:n02129165 +ILSVRC2012_val_00049019.JPEG:n02483362 +ILSVRC2012_val_00049020.JPEG:n01910747 +ILSVRC2012_val_00049021.JPEG:n03887697 +ILSVRC2012_val_00049022.JPEG:n02422106 +ILSVRC2012_val_00049023.JPEG:n04039381 +ILSVRC2012_val_00049024.JPEG:n02356798 +ILSVRC2012_val_00049025.JPEG:n04350905 +ILSVRC2012_val_00049026.JPEG:n02871525 +ILSVRC2012_val_00049027.JPEG:n02086079 +ILSVRC2012_val_00049028.JPEG:n04485082 +ILSVRC2012_val_00049029.JPEG:n04116512 +ILSVRC2012_val_00049030.JPEG:n02346627 +ILSVRC2012_val_00049031.JPEG:n02840245 +ILSVRC2012_val_00049032.JPEG:n03345487 +ILSVRC2012_val_00049033.JPEG:n04336792 +ILSVRC2012_val_00049034.JPEG:n03777568 +ILSVRC2012_val_00049035.JPEG:n02797295 +ILSVRC2012_val_00049036.JPEG:n02093428 +ILSVRC2012_val_00049037.JPEG:n04037443 +ILSVRC2012_val_00049038.JPEG:n03188531 +ILSVRC2012_val_00049039.JPEG:n03538406 +ILSVRC2012_val_00049040.JPEG:n02108089 +ILSVRC2012_val_00049041.JPEG:n02268853 +ILSVRC2012_val_00049042.JPEG:n02219486 +ILSVRC2012_val_00049043.JPEG:n02415577 +ILSVRC2012_val_00049044.JPEG:n02113978 +ILSVRC2012_val_00049045.JPEG:n04367480 +ILSVRC2012_val_00049046.JPEG:n02111277 +ILSVRC2012_val_00049047.JPEG:n07754684 +ILSVRC2012_val_00049048.JPEG:n03207941 +ILSVRC2012_val_00049049.JPEG:n02708093 +ILSVRC2012_val_00049050.JPEG:n02791124 +ILSVRC2012_val_00049051.JPEG:n04239074 +ILSVRC2012_val_00049052.JPEG:n01872401 +ILSVRC2012_val_00049053.JPEG:n03124043 +ILSVRC2012_val_00049054.JPEG:n02788148 +ILSVRC2012_val_00049055.JPEG:n03933933 +ILSVRC2012_val_00049056.JPEG:n01798484 +ILSVRC2012_val_00049057.JPEG:n03065424 +ILSVRC2012_val_00049058.JPEG:n03658185 +ILSVRC2012_val_00049059.JPEG:n09421951 +ILSVRC2012_val_00049060.JPEG:n03000247 +ILSVRC2012_val_00049061.JPEG:n02669723 +ILSVRC2012_val_00049062.JPEG:n04592741 +ILSVRC2012_val_00049063.JPEG:n02097130 +ILSVRC2012_val_00049064.JPEG:n02105641 +ILSVRC2012_val_00049065.JPEG:n01629819 +ILSVRC2012_val_00049066.JPEG:n02793495 +ILSVRC2012_val_00049067.JPEG:n03954731 +ILSVRC2012_val_00049068.JPEG:n04141327 +ILSVRC2012_val_00049069.JPEG:n02966687 +ILSVRC2012_val_00049070.JPEG:n02769748 +ILSVRC2012_val_00049071.JPEG:n02281787 +ILSVRC2012_val_00049072.JPEG:n01687978 +ILSVRC2012_val_00049073.JPEG:n04229816 +ILSVRC2012_val_00049074.JPEG:n04009552 +ILSVRC2012_val_00049075.JPEG:n04418357 +ILSVRC2012_val_00049076.JPEG:n04461696 +ILSVRC2012_val_00049077.JPEG:n02006656 +ILSVRC2012_val_00049078.JPEG:n03770439 +ILSVRC2012_val_00049079.JPEG:n02017213 +ILSVRC2012_val_00049080.JPEG:n07716358 +ILSVRC2012_val_00049081.JPEG:n02445715 +ILSVRC2012_val_00049082.JPEG:n02389026 +ILSVRC2012_val_00049083.JPEG:n02948072 +ILSVRC2012_val_00049084.JPEG:n06785654 +ILSVRC2012_val_00049085.JPEG:n02268443 +ILSVRC2012_val_00049086.JPEG:n03457902 +ILSVRC2012_val_00049087.JPEG:n04118776 +ILSVRC2012_val_00049088.JPEG:n12768682 +ILSVRC2012_val_00049089.JPEG:n02095314 +ILSVRC2012_val_00049090.JPEG:n01518878 +ILSVRC2012_val_00049091.JPEG:n04275548 +ILSVRC2012_val_00049092.JPEG:n02894605 +ILSVRC2012_val_00049093.JPEG:n01843383 +ILSVRC2012_val_00049094.JPEG:n02840245 +ILSVRC2012_val_00049095.JPEG:n07697313 +ILSVRC2012_val_00049096.JPEG:n07930864 +ILSVRC2012_val_00049097.JPEG:n02690373 +ILSVRC2012_val_00049098.JPEG:n02788148 +ILSVRC2012_val_00049099.JPEG:n04081281 +ILSVRC2012_val_00049100.JPEG:n03127925 +ILSVRC2012_val_00049101.JPEG:n03706229 +ILSVRC2012_val_00049102.JPEG:n03721384 +ILSVRC2012_val_00049103.JPEG:n01632458 +ILSVRC2012_val_00049104.JPEG:n04265275 +ILSVRC2012_val_00049105.JPEG:n01924916 +ILSVRC2012_val_00049106.JPEG:n02979186 +ILSVRC2012_val_00049107.JPEG:n01872401 +ILSVRC2012_val_00049108.JPEG:n04235860 +ILSVRC2012_val_00049109.JPEG:n04476259 +ILSVRC2012_val_00049110.JPEG:n07697537 +ILSVRC2012_val_00049111.JPEG:n02488702 +ILSVRC2012_val_00049112.JPEG:n03920288 +ILSVRC2012_val_00049113.JPEG:n03670208 +ILSVRC2012_val_00049114.JPEG:n04493381 +ILSVRC2012_val_00049115.JPEG:n02113712 +ILSVRC2012_val_00049116.JPEG:n01682714 +ILSVRC2012_val_00049117.JPEG:n03271574 +ILSVRC2012_val_00049118.JPEG:n03018349 +ILSVRC2012_val_00049119.JPEG:n01641577 +ILSVRC2012_val_00049120.JPEG:n02422699 +ILSVRC2012_val_00049121.JPEG:n02807133 +ILSVRC2012_val_00049122.JPEG:n02749479 +ILSVRC2012_val_00049123.JPEG:n02749479 +ILSVRC2012_val_00049124.JPEG:n02480495 +ILSVRC2012_val_00049125.JPEG:n02120505 +ILSVRC2012_val_00049126.JPEG:n02277742 +ILSVRC2012_val_00049127.JPEG:n03935335 +ILSVRC2012_val_00049128.JPEG:n03759954 +ILSVRC2012_val_00049129.JPEG:n02113186 +ILSVRC2012_val_00049130.JPEG:n02100236 +ILSVRC2012_val_00049131.JPEG:n03126707 +ILSVRC2012_val_00049132.JPEG:n04458633 +ILSVRC2012_val_00049133.JPEG:n02281406 +ILSVRC2012_val_00049134.JPEG:n01775062 +ILSVRC2012_val_00049135.JPEG:n04204347 +ILSVRC2012_val_00049136.JPEG:n02116738 +ILSVRC2012_val_00049137.JPEG:n03388043 +ILSVRC2012_val_00049138.JPEG:n04418357 +ILSVRC2012_val_00049139.JPEG:n02100583 +ILSVRC2012_val_00049140.JPEG:n03584829 +ILSVRC2012_val_00049141.JPEG:n01592084 +ILSVRC2012_val_00049142.JPEG:n04456115 +ILSVRC2012_val_00049143.JPEG:n01728920 +ILSVRC2012_val_00049144.JPEG:n02091635 +ILSVRC2012_val_00049145.JPEG:n03637318 +ILSVRC2012_val_00049146.JPEG:n02105056 +ILSVRC2012_val_00049147.JPEG:n02110627 +ILSVRC2012_val_00049148.JPEG:n02776631 +ILSVRC2012_val_00049149.JPEG:n03788365 +ILSVRC2012_val_00049150.JPEG:n03179701 +ILSVRC2012_val_00049151.JPEG:n02009912 +ILSVRC2012_val_00049152.JPEG:n02219486 +ILSVRC2012_val_00049153.JPEG:n04179913 +ILSVRC2012_val_00049154.JPEG:n07590611 +ILSVRC2012_val_00049155.JPEG:n03903868 +ILSVRC2012_val_00049156.JPEG:n04560804 +ILSVRC2012_val_00049157.JPEG:n01917289 +ILSVRC2012_val_00049158.JPEG:n04133789 +ILSVRC2012_val_00049159.JPEG:n02085620 +ILSVRC2012_val_00049160.JPEG:n03259280 +ILSVRC2012_val_00049161.JPEG:n02484975 +ILSVRC2012_val_00049162.JPEG:n01744401 +ILSVRC2012_val_00049163.JPEG:n07836838 +ILSVRC2012_val_00049164.JPEG:n07753592 +ILSVRC2012_val_00049165.JPEG:n03673027 +ILSVRC2012_val_00049166.JPEG:n01494475 +ILSVRC2012_val_00049167.JPEG:n01728572 +ILSVRC2012_val_00049168.JPEG:n02174001 +ILSVRC2012_val_00049169.JPEG:n07873807 +ILSVRC2012_val_00049170.JPEG:n02058221 +ILSVRC2012_val_00049171.JPEG:n04252225 +ILSVRC2012_val_00049172.JPEG:n03782006 +ILSVRC2012_val_00049173.JPEG:n04133789 +ILSVRC2012_val_00049174.JPEG:n15075141 +ILSVRC2012_val_00049175.JPEG:n02106662 +ILSVRC2012_val_00049176.JPEG:n02346627 +ILSVRC2012_val_00049177.JPEG:n03769881 +ILSVRC2012_val_00049178.JPEG:n03630383 +ILSVRC2012_val_00049179.JPEG:n03871628 +ILSVRC2012_val_00049180.JPEG:n01984695 +ILSVRC2012_val_00049181.JPEG:n01514668 +ILSVRC2012_val_00049182.JPEG:n01749939 +ILSVRC2012_val_00049183.JPEG:n03457902 +ILSVRC2012_val_00049184.JPEG:n04347754 +ILSVRC2012_val_00049185.JPEG:n04370456 +ILSVRC2012_val_00049186.JPEG:n02892201 +ILSVRC2012_val_00049187.JPEG:n01693334 +ILSVRC2012_val_00049188.JPEG:n03109150 +ILSVRC2012_val_00049189.JPEG:n02102973 +ILSVRC2012_val_00049190.JPEG:n02098413 +ILSVRC2012_val_00049191.JPEG:n01930112 +ILSVRC2012_val_00049192.JPEG:n02834397 +ILSVRC2012_val_00049193.JPEG:n02091032 +ILSVRC2012_val_00049194.JPEG:n02489166 +ILSVRC2012_val_00049195.JPEG:n12985857 +ILSVRC2012_val_00049196.JPEG:n02092339 +ILSVRC2012_val_00049197.JPEG:n03995372 +ILSVRC2012_val_00049198.JPEG:n02089078 +ILSVRC2012_val_00049199.JPEG:n03709823 +ILSVRC2012_val_00049200.JPEG:n02111500 +ILSVRC2012_val_00049201.JPEG:n02268443 +ILSVRC2012_val_00049202.JPEG:n02410509 +ILSVRC2012_val_00049203.JPEG:n01798484 +ILSVRC2012_val_00049204.JPEG:n03720891 +ILSVRC2012_val_00049205.JPEG:n03868863 +ILSVRC2012_val_00049206.JPEG:n02092002 +ILSVRC2012_val_00049207.JPEG:n03018349 +ILSVRC2012_val_00049208.JPEG:n04487394 +ILSVRC2012_val_00049209.JPEG:n03240683 +ILSVRC2012_val_00049210.JPEG:n03803284 +ILSVRC2012_val_00049211.JPEG:n07579787 +ILSVRC2012_val_00049212.JPEG:n02804414 +ILSVRC2012_val_00049213.JPEG:n03887697 +ILSVRC2012_val_00049214.JPEG:n04542943 +ILSVRC2012_val_00049215.JPEG:n02113023 +ILSVRC2012_val_00049216.JPEG:n02607072 +ILSVRC2012_val_00049217.JPEG:n01882714 +ILSVRC2012_val_00049218.JPEG:n02102040 +ILSVRC2012_val_00049219.JPEG:n07697537 +ILSVRC2012_val_00049220.JPEG:n02443114 +ILSVRC2012_val_00049221.JPEG:n01986214 +ILSVRC2012_val_00049222.JPEG:n02777292 +ILSVRC2012_val_00049223.JPEG:n02939185 +ILSVRC2012_val_00049224.JPEG:n02009229 +ILSVRC2012_val_00049225.JPEG:n03769881 +ILSVRC2012_val_00049226.JPEG:n04554684 +ILSVRC2012_val_00049227.JPEG:n02037110 +ILSVRC2012_val_00049228.JPEG:n02817516 +ILSVRC2012_val_00049229.JPEG:n02089078 +ILSVRC2012_val_00049230.JPEG:n03691459 +ILSVRC2012_val_00049231.JPEG:n03680355 +ILSVRC2012_val_00049232.JPEG:n04591713 +ILSVRC2012_val_00049233.JPEG:n03804744 +ILSVRC2012_val_00049234.JPEG:n03617480 +ILSVRC2012_val_00049235.JPEG:n01795545 +ILSVRC2012_val_00049236.JPEG:n02865351 +ILSVRC2012_val_00049237.JPEG:n02840245 +ILSVRC2012_val_00049238.JPEG:n02909870 +ILSVRC2012_val_00049239.JPEG:n02101006 +ILSVRC2012_val_00049240.JPEG:n04208210 +ILSVRC2012_val_00049241.JPEG:n04487081 +ILSVRC2012_val_00049242.JPEG:n02111889 +ILSVRC2012_val_00049243.JPEG:n04264628 +ILSVRC2012_val_00049244.JPEG:n01629819 +ILSVRC2012_val_00049245.JPEG:n02111129 +ILSVRC2012_val_00049246.JPEG:n12768682 +ILSVRC2012_val_00049247.JPEG:n03134739 +ILSVRC2012_val_00049248.JPEG:n03075370 +ILSVRC2012_val_00049249.JPEG:n13037406 +ILSVRC2012_val_00049250.JPEG:n02100735 +ILSVRC2012_val_00049251.JPEG:n04330267 +ILSVRC2012_val_00049252.JPEG:n04540053 +ILSVRC2012_val_00049253.JPEG:n01498041 +ILSVRC2012_val_00049254.JPEG:n03874599 +ILSVRC2012_val_00049255.JPEG:n03874599 +ILSVRC2012_val_00049256.JPEG:n04485082 +ILSVRC2012_val_00049257.JPEG:n03095699 +ILSVRC2012_val_00049258.JPEG:n04252225 +ILSVRC2012_val_00049259.JPEG:n02172182 +ILSVRC2012_val_00049260.JPEG:n01667114 +ILSVRC2012_val_00049261.JPEG:n04557648 +ILSVRC2012_val_00049262.JPEG:n02119022 +ILSVRC2012_val_00049263.JPEG:n02091467 +ILSVRC2012_val_00049264.JPEG:n04350905 +ILSVRC2012_val_00049265.JPEG:n01817953 +ILSVRC2012_val_00049266.JPEG:n01985128 +ILSVRC2012_val_00049267.JPEG:n04067472 +ILSVRC2012_val_00049268.JPEG:n02504013 +ILSVRC2012_val_00049269.JPEG:n04476259 +ILSVRC2012_val_00049270.JPEG:n09229709 +ILSVRC2012_val_00049271.JPEG:n02865351 +ILSVRC2012_val_00049272.JPEG:n02105251 +ILSVRC2012_val_00049273.JPEG:n03255030 +ILSVRC2012_val_00049274.JPEG:n02325366 +ILSVRC2012_val_00049275.JPEG:n04200800 +ILSVRC2012_val_00049276.JPEG:n03065424 +ILSVRC2012_val_00049277.JPEG:n04330267 +ILSVRC2012_val_00049278.JPEG:n02403003 +ILSVRC2012_val_00049279.JPEG:n02123159 +ILSVRC2012_val_00049280.JPEG:n02326432 +ILSVRC2012_val_00049281.JPEG:n02097130 +ILSVRC2012_val_00049282.JPEG:n02966687 +ILSVRC2012_val_00049283.JPEG:n04591157 +ILSVRC2012_val_00049284.JPEG:n03538406 +ILSVRC2012_val_00049285.JPEG:n02107908 +ILSVRC2012_val_00049286.JPEG:n02009912 +ILSVRC2012_val_00049287.JPEG:n01644900 +ILSVRC2012_val_00049288.JPEG:n02356798 +ILSVRC2012_val_00049289.JPEG:n04201297 +ILSVRC2012_val_00049290.JPEG:n04235860 +ILSVRC2012_val_00049291.JPEG:n02110185 +ILSVRC2012_val_00049292.JPEG:n03544143 +ILSVRC2012_val_00049293.JPEG:n02787622 +ILSVRC2012_val_00049294.JPEG:n04296562 +ILSVRC2012_val_00049295.JPEG:n02804414 +ILSVRC2012_val_00049296.JPEG:n02114367 +ILSVRC2012_val_00049297.JPEG:n02894605 +ILSVRC2012_val_00049298.JPEG:n02119022 +ILSVRC2012_val_00049299.JPEG:n02965783 +ILSVRC2012_val_00049300.JPEG:n03837869 +ILSVRC2012_val_00049301.JPEG:n01955084 +ILSVRC2012_val_00049302.JPEG:n02701002 +ILSVRC2012_val_00049303.JPEG:n02137549 +ILSVRC2012_val_00049304.JPEG:n03794056 +ILSVRC2012_val_00049305.JPEG:n03759954 +ILSVRC2012_val_00049306.JPEG:n03956157 +ILSVRC2012_val_00049307.JPEG:n03461385 +ILSVRC2012_val_00049308.JPEG:n02939185 +ILSVRC2012_val_00049309.JPEG:n07892512 +ILSVRC2012_val_00049310.JPEG:n07715103 +ILSVRC2012_val_00049311.JPEG:n01742172 +ILSVRC2012_val_00049312.JPEG:n04350905 +ILSVRC2012_val_00049313.JPEG:n01817953 +ILSVRC2012_val_00049314.JPEG:n02865351 +ILSVRC2012_val_00049315.JPEG:n02002556 +ILSVRC2012_val_00049316.JPEG:n01644900 +ILSVRC2012_val_00049317.JPEG:n02795169 +ILSVRC2012_val_00049318.JPEG:n03617480 +ILSVRC2012_val_00049319.JPEG:n03207743 +ILSVRC2012_val_00049320.JPEG:n02403003 +ILSVRC2012_val_00049321.JPEG:n03109150 +ILSVRC2012_val_00049322.JPEG:n03590841 +ILSVRC2012_val_00049323.JPEG:n02480855 +ILSVRC2012_val_00049324.JPEG:n02091032 +ILSVRC2012_val_00049325.JPEG:n07584110 +ILSVRC2012_val_00049326.JPEG:n02102318 +ILSVRC2012_val_00049327.JPEG:n02111277 +ILSVRC2012_val_00049328.JPEG:n02692877 +ILSVRC2012_val_00049329.JPEG:n04604644 +ILSVRC2012_val_00049330.JPEG:n03793489 +ILSVRC2012_val_00049331.JPEG:n01877812 +ILSVRC2012_val_00049332.JPEG:n02412080 +ILSVRC2012_val_00049333.JPEG:n01698640 +ILSVRC2012_val_00049334.JPEG:n02110806 +ILSVRC2012_val_00049335.JPEG:n04019541 +ILSVRC2012_val_00049336.JPEG:n04476259 +ILSVRC2012_val_00049337.JPEG:n04584207 +ILSVRC2012_val_00049338.JPEG:n02012849 +ILSVRC2012_val_00049339.JPEG:n03720891 +ILSVRC2012_val_00049340.JPEG:n04311174 +ILSVRC2012_val_00049341.JPEG:n03459775 +ILSVRC2012_val_00049342.JPEG:n03781244 +ILSVRC2012_val_00049343.JPEG:n09428293 +ILSVRC2012_val_00049344.JPEG:n02106550 +ILSVRC2012_val_00049345.JPEG:n02132136 +ILSVRC2012_val_00049346.JPEG:n03630383 +ILSVRC2012_val_00049347.JPEG:n02128925 +ILSVRC2012_val_00049348.JPEG:n03903868 +ILSVRC2012_val_00049349.JPEG:n03814639 +ILSVRC2012_val_00049350.JPEG:n01630670 +ILSVRC2012_val_00049351.JPEG:n02106550 +ILSVRC2012_val_00049352.JPEG:n01855672 +ILSVRC2012_val_00049353.JPEG:n01807496 +ILSVRC2012_val_00049354.JPEG:n02088364 +ILSVRC2012_val_00049355.JPEG:n03290653 +ILSVRC2012_val_00049356.JPEG:n02109525 +ILSVRC2012_val_00049357.JPEG:n03902125 +ILSVRC2012_val_00049358.JPEG:n07583066 +ILSVRC2012_val_00049359.JPEG:n04542943 +ILSVRC2012_val_00049360.JPEG:n03937543 +ILSVRC2012_val_00049361.JPEG:n07583066 +ILSVRC2012_val_00049362.JPEG:n04008634 +ILSVRC2012_val_00049363.JPEG:n04532670 +ILSVRC2012_val_00049364.JPEG:n02095314 +ILSVRC2012_val_00049365.JPEG:n04118538 +ILSVRC2012_val_00049366.JPEG:n07584110 +ILSVRC2012_val_00049367.JPEG:n02747177 +ILSVRC2012_val_00049368.JPEG:n03929855 +ILSVRC2012_val_00049369.JPEG:n01950731 +ILSVRC2012_val_00049370.JPEG:n07742313 +ILSVRC2012_val_00049371.JPEG:n03649909 +ILSVRC2012_val_00049372.JPEG:n02319095 +ILSVRC2012_val_00049373.JPEG:n01697457 +ILSVRC2012_val_00049374.JPEG:n02092339 +ILSVRC2012_val_00049375.JPEG:n09332890 +ILSVRC2012_val_00049376.JPEG:n04347754 +ILSVRC2012_val_00049377.JPEG:n02480495 +ILSVRC2012_val_00049378.JPEG:n03478589 +ILSVRC2012_val_00049379.JPEG:n07880968 +ILSVRC2012_val_00049380.JPEG:n03935335 +ILSVRC2012_val_00049381.JPEG:n03976657 +ILSVRC2012_val_00049382.JPEG:n02835271 +ILSVRC2012_val_00049383.JPEG:n04367480 +ILSVRC2012_val_00049384.JPEG:n02177972 +ILSVRC2012_val_00049385.JPEG:n04070727 +ILSVRC2012_val_00049386.JPEG:n04277352 +ILSVRC2012_val_00049387.JPEG:n04125021 +ILSVRC2012_val_00049388.JPEG:n03134739 +ILSVRC2012_val_00049389.JPEG:n02128757 +ILSVRC2012_val_00049390.JPEG:n02504013 +ILSVRC2012_val_00049391.JPEG:n04111531 +ILSVRC2012_val_00049392.JPEG:n04152593 +ILSVRC2012_val_00049393.JPEG:n04591713 +ILSVRC2012_val_00049394.JPEG:n03400231 +ILSVRC2012_val_00049395.JPEG:n01704323 +ILSVRC2012_val_00049396.JPEG:n12768682 +ILSVRC2012_val_00049397.JPEG:n02110806 +ILSVRC2012_val_00049398.JPEG:n04418357 +ILSVRC2012_val_00049399.JPEG:n02536864 +ILSVRC2012_val_00049400.JPEG:n04409515 +ILSVRC2012_val_00049401.JPEG:n04542943 +ILSVRC2012_val_00049402.JPEG:n03763968 +ILSVRC2012_val_00049403.JPEG:n03662601 +ILSVRC2012_val_00049404.JPEG:n02490219 +ILSVRC2012_val_00049405.JPEG:n02086240 +ILSVRC2012_val_00049406.JPEG:n04404412 +ILSVRC2012_val_00049407.JPEG:n07718747 +ILSVRC2012_val_00049408.JPEG:n02096051 +ILSVRC2012_val_00049409.JPEG:n04599235 +ILSVRC2012_val_00049410.JPEG:n01944390 +ILSVRC2012_val_00049411.JPEG:n01990800 +ILSVRC2012_val_00049412.JPEG:n04152593 +ILSVRC2012_val_00049413.JPEG:n02807133 +ILSVRC2012_val_00049414.JPEG:n02086910 +ILSVRC2012_val_00049415.JPEG:n03347037 +ILSVRC2012_val_00049416.JPEG:n01847000 +ILSVRC2012_val_00049417.JPEG:n02107683 +ILSVRC2012_val_00049418.JPEG:n02279972 +ILSVRC2012_val_00049419.JPEG:n04019541 +ILSVRC2012_val_00049420.JPEG:n01695060 +ILSVRC2012_val_00049421.JPEG:n02087046 +ILSVRC2012_val_00049422.JPEG:n03891251 +ILSVRC2012_val_00049423.JPEG:n04154565 +ILSVRC2012_val_00049424.JPEG:n04398044 +ILSVRC2012_val_00049425.JPEG:n02504013 +ILSVRC2012_val_00049426.JPEG:n02138441 +ILSVRC2012_val_00049427.JPEG:n04285008 +ILSVRC2012_val_00049428.JPEG:n03942813 +ILSVRC2012_val_00049429.JPEG:n04239074 +ILSVRC2012_val_00049430.JPEG:n02704792 +ILSVRC2012_val_00049431.JPEG:n03794056 +ILSVRC2012_val_00049432.JPEG:n04476259 +ILSVRC2012_val_00049433.JPEG:n04483307 +ILSVRC2012_val_00049434.JPEG:n03982430 +ILSVRC2012_val_00049435.JPEG:n02109047 +ILSVRC2012_val_00049436.JPEG:n11939491 +ILSVRC2012_val_00049437.JPEG:n04335435 +ILSVRC2012_val_00049438.JPEG:n02727426 +ILSVRC2012_val_00049439.JPEG:n03781244 +ILSVRC2012_val_00049440.JPEG:n01978455 +ILSVRC2012_val_00049441.JPEG:n03887697 +ILSVRC2012_val_00049442.JPEG:n02268853 +ILSVRC2012_val_00049443.JPEG:n02607072 +ILSVRC2012_val_00049444.JPEG:n02009229 +ILSVRC2012_val_00049445.JPEG:n04371774 +ILSVRC2012_val_00049446.JPEG:n07892512 +ILSVRC2012_val_00049447.JPEG:n04523525 +ILSVRC2012_val_00049448.JPEG:n01748264 +ILSVRC2012_val_00049449.JPEG:n03924679 +ILSVRC2012_val_00049450.JPEG:n04200800 +ILSVRC2012_val_00049451.JPEG:n04026417 +ILSVRC2012_val_00049452.JPEG:n04208210 +ILSVRC2012_val_00049453.JPEG:n04548362 +ILSVRC2012_val_00049454.JPEG:n04389033 +ILSVRC2012_val_00049455.JPEG:n04152593 +ILSVRC2012_val_00049456.JPEG:n02910353 +ILSVRC2012_val_00049457.JPEG:n07697313 +ILSVRC2012_val_00049458.JPEG:n03196217 +ILSVRC2012_val_00049459.JPEG:n04200800 +ILSVRC2012_val_00049460.JPEG:n02279972 +ILSVRC2012_val_00049461.JPEG:n01917289 +ILSVRC2012_val_00049462.JPEG:n02488291 +ILSVRC2012_val_00049463.JPEG:n02808304 +ILSVRC2012_val_00049464.JPEG:n03992509 +ILSVRC2012_val_00049465.JPEG:n02804414 +ILSVRC2012_val_00049466.JPEG:n01774750 +ILSVRC2012_val_00049467.JPEG:n04442312 +ILSVRC2012_val_00049468.JPEG:n03535780 +ILSVRC2012_val_00049469.JPEG:n02802426 +ILSVRC2012_val_00049470.JPEG:n04044716 +ILSVRC2012_val_00049471.JPEG:n02128385 +ILSVRC2012_val_00049472.JPEG:n07697313 +ILSVRC2012_val_00049473.JPEG:n04179913 +ILSVRC2012_val_00049474.JPEG:n03400231 +ILSVRC2012_val_00049475.JPEG:n03095699 +ILSVRC2012_val_00049476.JPEG:n03871628 +ILSVRC2012_val_00049477.JPEG:n02129165 +ILSVRC2012_val_00049478.JPEG:n01773797 +ILSVRC2012_val_00049479.JPEG:n03691459 +ILSVRC2012_val_00049480.JPEG:n02018795 +ILSVRC2012_val_00049481.JPEG:n04116512 +ILSVRC2012_val_00049482.JPEG:n03089624 +ILSVRC2012_val_00049483.JPEG:n02127052 +ILSVRC2012_val_00049484.JPEG:n02111129 +ILSVRC2012_val_00049485.JPEG:n02093256 +ILSVRC2012_val_00049486.JPEG:n03742115 +ILSVRC2012_val_00049487.JPEG:n04429376 +ILSVRC2012_val_00049488.JPEG:n02009229 +ILSVRC2012_val_00049489.JPEG:n02815834 +ILSVRC2012_val_00049490.JPEG:n07747607 +ILSVRC2012_val_00049491.JPEG:n03481172 +ILSVRC2012_val_00049492.JPEG:n03220513 +ILSVRC2012_val_00049493.JPEG:n03495258 +ILSVRC2012_val_00049494.JPEG:n02974003 +ILSVRC2012_val_00049495.JPEG:n01704323 +ILSVRC2012_val_00049496.JPEG:n04277352 +ILSVRC2012_val_00049497.JPEG:n07684084 +ILSVRC2012_val_00049498.JPEG:n02107574 +ILSVRC2012_val_00049499.JPEG:n02276258 +ILSVRC2012_val_00049500.JPEG:n12998815 +ILSVRC2012_val_00049501.JPEG:n03617480 +ILSVRC2012_val_00049502.JPEG:n03721384 +ILSVRC2012_val_00049503.JPEG:n02992529 +ILSVRC2012_val_00049504.JPEG:n02321529 +ILSVRC2012_val_00049505.JPEG:n03933933 +ILSVRC2012_val_00049506.JPEG:n03764736 +ILSVRC2012_val_00049507.JPEG:n03764736 +ILSVRC2012_val_00049508.JPEG:n02317335 +ILSVRC2012_val_00049509.JPEG:n04235860 +ILSVRC2012_val_00049510.JPEG:n02808440 +ILSVRC2012_val_00049511.JPEG:n02110341 +ILSVRC2012_val_00049512.JPEG:n04542943 +ILSVRC2012_val_00049513.JPEG:n02442845 +ILSVRC2012_val_00049514.JPEG:n02869837 +ILSVRC2012_val_00049515.JPEG:n01742172 +ILSVRC2012_val_00049516.JPEG:n02088632 +ILSVRC2012_val_00049517.JPEG:n02120079 +ILSVRC2012_val_00049518.JPEG:n04259630 +ILSVRC2012_val_00049519.JPEG:n03447447 +ILSVRC2012_val_00049520.JPEG:n03876231 +ILSVRC2012_val_00049521.JPEG:n02037110 +ILSVRC2012_val_00049522.JPEG:n01914609 +ILSVRC2012_val_00049523.JPEG:n02102040 +ILSVRC2012_val_00049524.JPEG:n13054560 +ILSVRC2012_val_00049525.JPEG:n03930630 +ILSVRC2012_val_00049526.JPEG:n03759954 +ILSVRC2012_val_00049527.JPEG:n07584110 +ILSVRC2012_val_00049528.JPEG:n04259630 +ILSVRC2012_val_00049529.JPEG:n03291819 +ILSVRC2012_val_00049530.JPEG:n07697537 +ILSVRC2012_val_00049531.JPEG:n01614925 +ILSVRC2012_val_00049532.JPEG:n03814906 +ILSVRC2012_val_00049533.JPEG:n04540053 +ILSVRC2012_val_00049534.JPEG:n02116738 +ILSVRC2012_val_00049535.JPEG:n01776313 +ILSVRC2012_val_00049536.JPEG:n03954731 +ILSVRC2012_val_00049537.JPEG:n04479046 +ILSVRC2012_val_00049538.JPEG:n03658185 +ILSVRC2012_val_00049539.JPEG:n04357314 +ILSVRC2012_val_00049540.JPEG:n03763968 +ILSVRC2012_val_00049541.JPEG:n01755581 +ILSVRC2012_val_00049542.JPEG:n01749939 +ILSVRC2012_val_00049543.JPEG:n02981792 +ILSVRC2012_val_00049544.JPEG:n03485407 +ILSVRC2012_val_00049545.JPEG:n02442845 +ILSVRC2012_val_00049546.JPEG:n04548280 +ILSVRC2012_val_00049547.JPEG:n07880968 +ILSVRC2012_val_00049548.JPEG:n02825657 +ILSVRC2012_val_00049549.JPEG:n09332890 +ILSVRC2012_val_00049550.JPEG:n04596742 +ILSVRC2012_val_00049551.JPEG:n04596742 +ILSVRC2012_val_00049552.JPEG:n02930766 +ILSVRC2012_val_00049553.JPEG:n01843383 +ILSVRC2012_val_00049554.JPEG:n03532672 +ILSVRC2012_val_00049555.JPEG:n13133613 +ILSVRC2012_val_00049556.JPEG:n02963159 +ILSVRC2012_val_00049557.JPEG:n03759954 +ILSVRC2012_val_00049558.JPEG:n02098413 +ILSVRC2012_val_00049559.JPEG:n04367480 +ILSVRC2012_val_00049560.JPEG:n02643566 +ILSVRC2012_val_00049561.JPEG:n04254777 +ILSVRC2012_val_00049562.JPEG:n02415577 +ILSVRC2012_val_00049563.JPEG:n04560804 +ILSVRC2012_val_00049564.JPEG:n04485082 +ILSVRC2012_val_00049565.JPEG:n03781244 +ILSVRC2012_val_00049566.JPEG:n04597913 +ILSVRC2012_val_00049567.JPEG:n04482393 +ILSVRC2012_val_00049568.JPEG:n01530575 +ILSVRC2012_val_00049569.JPEG:n03250847 +ILSVRC2012_val_00049570.JPEG:n02108089 +ILSVRC2012_val_00049571.JPEG:n04404412 +ILSVRC2012_val_00049572.JPEG:n02687172 +ILSVRC2012_val_00049573.JPEG:n03786901 +ILSVRC2012_val_00049574.JPEG:n02108000 +ILSVRC2012_val_00049575.JPEG:n02687172 +ILSVRC2012_val_00049576.JPEG:n02317335 +ILSVRC2012_val_00049577.JPEG:n02606052 +ILSVRC2012_val_00049578.JPEG:n02165105 +ILSVRC2012_val_00049579.JPEG:n03045698 +ILSVRC2012_val_00049580.JPEG:n03218198 +ILSVRC2012_val_00049581.JPEG:n02415577 +ILSVRC2012_val_00049582.JPEG:n04069434 +ILSVRC2012_val_00049583.JPEG:n04482393 +ILSVRC2012_val_00049584.JPEG:n01806143 +ILSVRC2012_val_00049585.JPEG:n01443537 +ILSVRC2012_val_00049586.JPEG:n02100735 +ILSVRC2012_val_00049587.JPEG:n04153751 +ILSVRC2012_val_00049588.JPEG:n04254777 +ILSVRC2012_val_00049589.JPEG:n02091467 +ILSVRC2012_val_00049590.JPEG:n03482405 +ILSVRC2012_val_00049591.JPEG:n02794156 +ILSVRC2012_val_00049592.JPEG:n07754684 +ILSVRC2012_val_00049593.JPEG:n03495258 +ILSVRC2012_val_00049594.JPEG:n04542943 +ILSVRC2012_val_00049595.JPEG:n01797886 +ILSVRC2012_val_00049596.JPEG:n03085013 +ILSVRC2012_val_00049597.JPEG:n03792972 +ILSVRC2012_val_00049598.JPEG:n01980166 +ILSVRC2012_val_00049599.JPEG:n02782093 +ILSVRC2012_val_00049600.JPEG:n03920288 +ILSVRC2012_val_00049601.JPEG:n03666591 +ILSVRC2012_val_00049602.JPEG:n01695060 +ILSVRC2012_val_00049603.JPEG:n02486410 +ILSVRC2012_val_00049604.JPEG:n02088364 +ILSVRC2012_val_00049605.JPEG:n02389026 +ILSVRC2012_val_00049606.JPEG:n07753592 +ILSVRC2012_val_00049607.JPEG:n07248320 +ILSVRC2012_val_00049608.JPEG:n03355925 +ILSVRC2012_val_00049609.JPEG:n01737021 +ILSVRC2012_val_00049610.JPEG:n04266014 +ILSVRC2012_val_00049611.JPEG:n02167151 +ILSVRC2012_val_00049612.JPEG:n03930630 +ILSVRC2012_val_00049613.JPEG:n02133161 +ILSVRC2012_val_00049614.JPEG:n02107142 +ILSVRC2012_val_00049615.JPEG:n03180011 +ILSVRC2012_val_00049616.JPEG:n04023962 +ILSVRC2012_val_00049617.JPEG:n01443537 +ILSVRC2012_val_00049618.JPEG:n02443114 +ILSVRC2012_val_00049619.JPEG:n02892201 +ILSVRC2012_val_00049620.JPEG:n03109150 +ILSVRC2012_val_00049621.JPEG:n01872401 +ILSVRC2012_val_00049622.JPEG:n07565083 +ILSVRC2012_val_00049623.JPEG:n02815834 +ILSVRC2012_val_00049624.JPEG:n02206856 +ILSVRC2012_val_00049625.JPEG:n03729826 +ILSVRC2012_val_00049626.JPEG:n10565667 +ILSVRC2012_val_00049627.JPEG:n02111129 +ILSVRC2012_val_00049628.JPEG:n02704792 +ILSVRC2012_val_00049629.JPEG:n02117135 +ILSVRC2012_val_00049630.JPEG:n03000247 +ILSVRC2012_val_00049631.JPEG:n02129604 +ILSVRC2012_val_00049632.JPEG:n04550184 +ILSVRC2012_val_00049633.JPEG:n03089624 +ILSVRC2012_val_00049634.JPEG:n03785016 +ILSVRC2012_val_00049635.JPEG:n01689811 +ILSVRC2012_val_00049636.JPEG:n02441942 +ILSVRC2012_val_00049637.JPEG:n01641577 +ILSVRC2012_val_00049638.JPEG:n02229544 +ILSVRC2012_val_00049639.JPEG:n01622779 +ILSVRC2012_val_00049640.JPEG:n02089973 +ILSVRC2012_val_00049641.JPEG:n02791270 +ILSVRC2012_val_00049642.JPEG:n02102177 +ILSVRC2012_val_00049643.JPEG:n02114855 +ILSVRC2012_val_00049644.JPEG:n13040303 +ILSVRC2012_val_00049645.JPEG:n03944341 +ILSVRC2012_val_00049646.JPEG:n01667114 +ILSVRC2012_val_00049647.JPEG:n04149813 +ILSVRC2012_val_00049648.JPEG:n03792972 +ILSVRC2012_val_00049649.JPEG:n02869837 +ILSVRC2012_val_00049650.JPEG:n02112706 +ILSVRC2012_val_00049651.JPEG:n13044778 +ILSVRC2012_val_00049652.JPEG:n01688243 +ILSVRC2012_val_00049653.JPEG:n02097658 +ILSVRC2012_val_00049654.JPEG:n02109961 +ILSVRC2012_val_00049655.JPEG:n03791053 +ILSVRC2012_val_00049656.JPEG:n04286575 +ILSVRC2012_val_00049657.JPEG:n01985128 +ILSVRC2012_val_00049658.JPEG:n03014705 +ILSVRC2012_val_00049659.JPEG:n04265275 +ILSVRC2012_val_00049660.JPEG:n04467665 +ILSVRC2012_val_00049661.JPEG:n01985128 +ILSVRC2012_val_00049662.JPEG:n04344873 +ILSVRC2012_val_00049663.JPEG:n04335435 +ILSVRC2012_val_00049664.JPEG:n02676566 +ILSVRC2012_val_00049665.JPEG:n01806143 +ILSVRC2012_val_00049666.JPEG:n04599235 +ILSVRC2012_val_00049667.JPEG:n02093859 +ILSVRC2012_val_00049668.JPEG:n04486054 +ILSVRC2012_val_00049669.JPEG:n01601694 +ILSVRC2012_val_00049670.JPEG:n02966193 +ILSVRC2012_val_00049671.JPEG:n02965783 +ILSVRC2012_val_00049672.JPEG:n02099712 +ILSVRC2012_val_00049673.JPEG:n02808440 +ILSVRC2012_val_00049674.JPEG:n03785016 +ILSVRC2012_val_00049675.JPEG:n04285008 +ILSVRC2012_val_00049676.JPEG:n04141076 +ILSVRC2012_val_00049677.JPEG:n07760859 +ILSVRC2012_val_00049678.JPEG:n03717622 +ILSVRC2012_val_00049679.JPEG:n01917289 +ILSVRC2012_val_00049680.JPEG:n03942813 +ILSVRC2012_val_00049681.JPEG:n04409515 +ILSVRC2012_val_00049682.JPEG:n01819313 +ILSVRC2012_val_00049683.JPEG:n03255030 +ILSVRC2012_val_00049684.JPEG:n02328150 +ILSVRC2012_val_00049685.JPEG:n07590611 +ILSVRC2012_val_00049686.JPEG:n01985128 +ILSVRC2012_val_00049687.JPEG:n03998194 +ILSVRC2012_val_00049688.JPEG:n12985857 +ILSVRC2012_val_00049689.JPEG:n03014705 +ILSVRC2012_val_00049690.JPEG:n02823428 +ILSVRC2012_val_00049691.JPEG:n03127747 +ILSVRC2012_val_00049692.JPEG:n02825657 +ILSVRC2012_val_00049693.JPEG:n03935335 +ILSVRC2012_val_00049694.JPEG:n02793495 +ILSVRC2012_val_00049695.JPEG:n04509417 +ILSVRC2012_val_00049696.JPEG:n02655020 +ILSVRC2012_val_00049697.JPEG:n07873807 +ILSVRC2012_val_00049698.JPEG:n02906734 +ILSVRC2012_val_00049699.JPEG:n03720891 +ILSVRC2012_val_00049700.JPEG:n04037443 +ILSVRC2012_val_00049701.JPEG:n04254120 +ILSVRC2012_val_00049702.JPEG:n07614500 +ILSVRC2012_val_00049703.JPEG:n01667114 +ILSVRC2012_val_00049704.JPEG:n02415577 +ILSVRC2012_val_00049705.JPEG:n03710637 +ILSVRC2012_val_00049706.JPEG:n02361337 +ILSVRC2012_val_00049707.JPEG:n04081281 +ILSVRC2012_val_00049708.JPEG:n04070727 +ILSVRC2012_val_00049709.JPEG:n03649909 +ILSVRC2012_val_00049710.JPEG:n07720875 +ILSVRC2012_val_00049711.JPEG:n02011460 +ILSVRC2012_val_00049712.JPEG:n01443537 +ILSVRC2012_val_00049713.JPEG:n04525305 +ILSVRC2012_val_00049714.JPEG:n02894605 +ILSVRC2012_val_00049715.JPEG:n02113712 +ILSVRC2012_val_00049716.JPEG:n09229709 +ILSVRC2012_val_00049717.JPEG:n04367480 +ILSVRC2012_val_00049718.JPEG:n04266014 +ILSVRC2012_val_00049719.JPEG:n02105056 +ILSVRC2012_val_00049720.JPEG:n09421951 +ILSVRC2012_val_00049721.JPEG:n02814860 +ILSVRC2012_val_00049722.JPEG:n02167151 +ILSVRC2012_val_00049723.JPEG:n01744401 +ILSVRC2012_val_00049724.JPEG:n02808304 +ILSVRC2012_val_00049725.JPEG:n02106030 +ILSVRC2012_val_00049726.JPEG:n02074367 +ILSVRC2012_val_00049727.JPEG:n02536864 +ILSVRC2012_val_00049728.JPEG:n04485082 +ILSVRC2012_val_00049729.JPEG:n03538406 +ILSVRC2012_val_00049730.JPEG:n02108915 +ILSVRC2012_val_00049731.JPEG:n02114548 +ILSVRC2012_val_00049732.JPEG:n01698640 +ILSVRC2012_val_00049733.JPEG:n04286575 +ILSVRC2012_val_00049734.JPEG:n02797295 +ILSVRC2012_val_00049735.JPEG:n02124075 +ILSVRC2012_val_00049736.JPEG:n02927161 +ILSVRC2012_val_00049737.JPEG:n02747177 +ILSVRC2012_val_00049738.JPEG:n02641379 +ILSVRC2012_val_00049739.JPEG:n02325366 +ILSVRC2012_val_00049740.JPEG:n02536864 +ILSVRC2012_val_00049741.JPEG:n03697007 +ILSVRC2012_val_00049742.JPEG:n02281406 +ILSVRC2012_val_00049743.JPEG:n03017168 +ILSVRC2012_val_00049744.JPEG:n02090721 +ILSVRC2012_val_00049745.JPEG:n03776460 +ILSVRC2012_val_00049746.JPEG:n02037110 +ILSVRC2012_val_00049747.JPEG:n03100240 +ILSVRC2012_val_00049748.JPEG:n04398044 +ILSVRC2012_val_00049749.JPEG:n02871525 +ILSVRC2012_val_00049750.JPEG:n03792782 +ILSVRC2012_val_00049751.JPEG:n02787622 +ILSVRC2012_val_00049752.JPEG:n03180011 +ILSVRC2012_val_00049753.JPEG:n04522168 +ILSVRC2012_val_00049754.JPEG:n04266014 +ILSVRC2012_val_00049755.JPEG:n03218198 +ILSVRC2012_val_00049756.JPEG:n02088094 +ILSVRC2012_val_00049757.JPEG:n02097298 +ILSVRC2012_val_00049758.JPEG:n04548362 +ILSVRC2012_val_00049759.JPEG:n03196217 +ILSVRC2012_val_00049760.JPEG:n02095889 +ILSVRC2012_val_00049761.JPEG:n01873310 +ILSVRC2012_val_00049762.JPEG:n02088466 +ILSVRC2012_val_00049763.JPEG:n01968897 +ILSVRC2012_val_00049764.JPEG:n04548280 +ILSVRC2012_val_00049765.JPEG:n04604644 +ILSVRC2012_val_00049766.JPEG:n02090379 +ILSVRC2012_val_00049767.JPEG:n03787032 +ILSVRC2012_val_00049768.JPEG:n04229816 +ILSVRC2012_val_00049769.JPEG:n03891251 +ILSVRC2012_val_00049770.JPEG:n02356798 +ILSVRC2012_val_00049771.JPEG:n04350905 +ILSVRC2012_val_00049772.JPEG:n03782006 +ILSVRC2012_val_00049773.JPEG:n01664065 +ILSVRC2012_val_00049774.JPEG:n03950228 +ILSVRC2012_val_00049775.JPEG:n01601694 +ILSVRC2012_val_00049776.JPEG:n01558993 +ILSVRC2012_val_00049777.JPEG:n02777292 +ILSVRC2012_val_00049778.JPEG:n02091134 +ILSVRC2012_val_00049779.JPEG:n02088632 +ILSVRC2012_val_00049780.JPEG:n02442845 +ILSVRC2012_val_00049781.JPEG:n02137549 +ILSVRC2012_val_00049782.JPEG:n01669191 +ILSVRC2012_val_00049783.JPEG:n02007558 +ILSVRC2012_val_00049784.JPEG:n03782006 +ILSVRC2012_val_00049785.JPEG:n03692522 +ILSVRC2012_val_00049786.JPEG:n02916936 +ILSVRC2012_val_00049787.JPEG:n04357314 +ILSVRC2012_val_00049788.JPEG:n02132136 +ILSVRC2012_val_00049789.JPEG:n03930630 +ILSVRC2012_val_00049790.JPEG:n04019541 +ILSVRC2012_val_00049791.JPEG:n04005630 +ILSVRC2012_val_00049792.JPEG:n02102480 +ILSVRC2012_val_00049793.JPEG:n03443371 +ILSVRC2012_val_00049794.JPEG:n04523525 +ILSVRC2012_val_00049795.JPEG:n03814906 +ILSVRC2012_val_00049796.JPEG:n07693725 +ILSVRC2012_val_00049797.JPEG:n04371774 +ILSVRC2012_val_00049798.JPEG:n04209239 +ILSVRC2012_val_00049799.JPEG:n03720891 +ILSVRC2012_val_00049800.JPEG:n02086079 +ILSVRC2012_val_00049801.JPEG:n02071294 +ILSVRC2012_val_00049802.JPEG:n01774384 +ILSVRC2012_val_00049803.JPEG:n01560419 +ILSVRC2012_val_00049804.JPEG:n04204238 +ILSVRC2012_val_00049805.JPEG:n02101556 +ILSVRC2012_val_00049806.JPEG:n03998194 +ILSVRC2012_val_00049807.JPEG:n04486054 +ILSVRC2012_val_00049808.JPEG:n04505470 +ILSVRC2012_val_00049809.JPEG:n02089867 +ILSVRC2012_val_00049810.JPEG:n04179913 +ILSVRC2012_val_00049811.JPEG:n02112018 +ILSVRC2012_val_00049812.JPEG:n04201297 +ILSVRC2012_val_00049813.JPEG:n03673027 +ILSVRC2012_val_00049814.JPEG:n03908714 +ILSVRC2012_val_00049815.JPEG:n02105056 +ILSVRC2012_val_00049816.JPEG:n02791270 +ILSVRC2012_val_00049817.JPEG:n03775071 +ILSVRC2012_val_00049818.JPEG:n03785016 +ILSVRC2012_val_00049819.JPEG:n02088238 +ILSVRC2012_val_00049820.JPEG:n04376876 +ILSVRC2012_val_00049821.JPEG:n03272562 +ILSVRC2012_val_00049822.JPEG:n02132136 +ILSVRC2012_val_00049823.JPEG:n01748264 +ILSVRC2012_val_00049824.JPEG:n02939185 +ILSVRC2012_val_00049825.JPEG:n03485794 +ILSVRC2012_val_00049826.JPEG:n02105412 +ILSVRC2012_val_00049827.JPEG:n02814860 +ILSVRC2012_val_00049828.JPEG:n03527444 +ILSVRC2012_val_00049829.JPEG:n03803284 +ILSVRC2012_val_00049830.JPEG:n02396427 +ILSVRC2012_val_00049831.JPEG:n03877845 +ILSVRC2012_val_00049832.JPEG:n07614500 +ILSVRC2012_val_00049833.JPEG:n01514859 +ILSVRC2012_val_00049834.JPEG:n02105056 +ILSVRC2012_val_00049835.JPEG:n03047690 +ILSVRC2012_val_00049836.JPEG:n04254120 +ILSVRC2012_val_00049837.JPEG:n03218198 +ILSVRC2012_val_00049838.JPEG:n02910353 +ILSVRC2012_val_00049839.JPEG:n04328186 +ILSVRC2012_val_00049840.JPEG:n03776460 +ILSVRC2012_val_00049841.JPEG:n02109961 +ILSVRC2012_val_00049842.JPEG:n03467068 +ILSVRC2012_val_00049843.JPEG:n02704792 +ILSVRC2012_val_00049844.JPEG:n04136333 +ILSVRC2012_val_00049845.JPEG:n02169497 +ILSVRC2012_val_00049846.JPEG:n02094114 +ILSVRC2012_val_00049847.JPEG:n03837869 +ILSVRC2012_val_00049848.JPEG:n03131574 +ILSVRC2012_val_00049849.JPEG:n02090622 +ILSVRC2012_val_00049850.JPEG:n04238763 +ILSVRC2012_val_00049851.JPEG:n01682714 +ILSVRC2012_val_00049852.JPEG:n03388043 +ILSVRC2012_val_00049853.JPEG:n04493381 +ILSVRC2012_val_00049854.JPEG:n04040759 +ILSVRC2012_val_00049855.JPEG:n02099601 +ILSVRC2012_val_00049856.JPEG:n03803284 +ILSVRC2012_val_00049857.JPEG:n02101388 +ILSVRC2012_val_00049858.JPEG:n13044778 +ILSVRC2012_val_00049859.JPEG:n04483307 +ILSVRC2012_val_00049860.JPEG:n03404251 +ILSVRC2012_val_00049861.JPEG:n02090622 +ILSVRC2012_val_00049862.JPEG:n12768682 +ILSVRC2012_val_00049863.JPEG:n04367480 +ILSVRC2012_val_00049864.JPEG:n03134739 +ILSVRC2012_val_00049865.JPEG:n02356798 +ILSVRC2012_val_00049866.JPEG:n02408429 +ILSVRC2012_val_00049867.JPEG:n02974003 +ILSVRC2012_val_00049868.JPEG:n02101388 +ILSVRC2012_val_00049869.JPEG:n03124170 +ILSVRC2012_val_00049870.JPEG:n04435653 +ILSVRC2012_val_00049871.JPEG:n02105855 +ILSVRC2012_val_00049872.JPEG:n07920052 +ILSVRC2012_val_00049873.JPEG:n03272010 +ILSVRC2012_val_00049874.JPEG:n03180011 +ILSVRC2012_val_00049875.JPEG:n07717556 +ILSVRC2012_val_00049876.JPEG:n04235860 +ILSVRC2012_val_00049877.JPEG:n07716358 +ILSVRC2012_val_00049878.JPEG:n02088094 +ILSVRC2012_val_00049879.JPEG:n07873807 +ILSVRC2012_val_00049880.JPEG:n03775071 +ILSVRC2012_val_00049881.JPEG:n02110341 +ILSVRC2012_val_00049882.JPEG:n02817516 +ILSVRC2012_val_00049883.JPEG:n03146219 +ILSVRC2012_val_00049884.JPEG:n02113186 +ILSVRC2012_val_00049885.JPEG:n09246464 +ILSVRC2012_val_00049886.JPEG:n02119022 +ILSVRC2012_val_00049887.JPEG:n03240683 +ILSVRC2012_val_00049888.JPEG:n03706229 +ILSVRC2012_val_00049889.JPEG:n02701002 +ILSVRC2012_val_00049890.JPEG:n04154565 +ILSVRC2012_val_00049891.JPEG:n03467068 +ILSVRC2012_val_00049892.JPEG:n03843555 +ILSVRC2012_val_00049893.JPEG:n02107683 +ILSVRC2012_val_00049894.JPEG:n02088094 +ILSVRC2012_val_00049895.JPEG:n02108915 +ILSVRC2012_val_00049896.JPEG:n02786058 +ILSVRC2012_val_00049897.JPEG:n02326432 +ILSVRC2012_val_00049898.JPEG:n01629819 +ILSVRC2012_val_00049899.JPEG:n01614925 +ILSVRC2012_val_00049900.JPEG:n12267677 +ILSVRC2012_val_00049901.JPEG:n02108422 +ILSVRC2012_val_00049902.JPEG:n02481823 +ILSVRC2012_val_00049903.JPEG:n02892201 +ILSVRC2012_val_00049904.JPEG:n02877765 +ILSVRC2012_val_00049905.JPEG:n01955084 +ILSVRC2012_val_00049906.JPEG:n12057211 +ILSVRC2012_val_00049907.JPEG:n03063689 +ILSVRC2012_val_00049908.JPEG:n02113978 +ILSVRC2012_val_00049909.JPEG:n02777292 +ILSVRC2012_val_00049910.JPEG:n03717622 +ILSVRC2012_val_00049911.JPEG:n02787622 +ILSVRC2012_val_00049912.JPEG:n02437312 +ILSVRC2012_val_00049913.JPEG:n03992509 +ILSVRC2012_val_00049914.JPEG:n01930112 +ILSVRC2012_val_00049915.JPEG:n02500267 +ILSVRC2012_val_00049916.JPEG:n03627232 +ILSVRC2012_val_00049917.JPEG:n04505470 +ILSVRC2012_val_00049918.JPEG:n03250847 +ILSVRC2012_val_00049919.JPEG:n03400231 +ILSVRC2012_val_00049920.JPEG:n02977058 +ILSVRC2012_val_00049921.JPEG:n04554684 +ILSVRC2012_val_00049922.JPEG:n04456115 +ILSVRC2012_val_00049923.JPEG:n04147183 +ILSVRC2012_val_00049924.JPEG:n03676483 +ILSVRC2012_val_00049925.JPEG:n04465501 +ILSVRC2012_val_00049926.JPEG:n02094114 +ILSVRC2012_val_00049927.JPEG:n04532106 +ILSVRC2012_val_00049928.JPEG:n07892512 +ILSVRC2012_val_00049929.JPEG:n04557648 +ILSVRC2012_val_00049930.JPEG:n03482405 +ILSVRC2012_val_00049931.JPEG:n02088238 +ILSVRC2012_val_00049932.JPEG:n03991062 +ILSVRC2012_val_00049933.JPEG:n01751748 +ILSVRC2012_val_00049934.JPEG:n02104029 +ILSVRC2012_val_00049935.JPEG:n03733281 +ILSVRC2012_val_00049936.JPEG:n02536864 +ILSVRC2012_val_00049937.JPEG:n01860187 +ILSVRC2012_val_00049938.JPEG:n03133878 +ILSVRC2012_val_00049939.JPEG:n02110627 +ILSVRC2012_val_00049940.JPEG:n03208938 +ILSVRC2012_val_00049941.JPEG:n04192698 +ILSVRC2012_val_00049942.JPEG:n02106166 +ILSVRC2012_val_00049943.JPEG:n03028079 +ILSVRC2012_val_00049944.JPEG:n04515003 +ILSVRC2012_val_00049945.JPEG:n03787032 +ILSVRC2012_val_00049946.JPEG:n04317175 +ILSVRC2012_val_00049947.JPEG:n03447721 +ILSVRC2012_val_00049948.JPEG:n02326432 +ILSVRC2012_val_00049949.JPEG:n03535780 +ILSVRC2012_val_00049950.JPEG:n03998194 +ILSVRC2012_val_00049951.JPEG:n04560804 +ILSVRC2012_val_00049952.JPEG:n04507155 +ILSVRC2012_val_00049953.JPEG:n03134739 +ILSVRC2012_val_00049954.JPEG:n01697457 +ILSVRC2012_val_00049955.JPEG:n04270147 +ILSVRC2012_val_00049956.JPEG:n02107683 +ILSVRC2012_val_00049957.JPEG:n04525305 +ILSVRC2012_val_00049958.JPEG:n02410509 +ILSVRC2012_val_00049959.JPEG:n02099712 +ILSVRC2012_val_00049960.JPEG:n02132136 +ILSVRC2012_val_00049961.JPEG:n02268853 +ILSVRC2012_val_00049962.JPEG:n01817953 +ILSVRC2012_val_00049963.JPEG:n03929855 +ILSVRC2012_val_00049964.JPEG:n07615774 +ILSVRC2012_val_00049965.JPEG:n02100735 +ILSVRC2012_val_00049966.JPEG:n01833805 +ILSVRC2012_val_00049967.JPEG:n03207743 +ILSVRC2012_val_00049968.JPEG:n04584207 +ILSVRC2012_val_00049969.JPEG:n04266014 +ILSVRC2012_val_00049970.JPEG:n07248320 +ILSVRC2012_val_00049971.JPEG:n03467068 +ILSVRC2012_val_00049972.JPEG:n03908618 +ILSVRC2012_val_00049973.JPEG:n02133161 +ILSVRC2012_val_00049974.JPEG:n02486410 +ILSVRC2012_val_00049975.JPEG:n01755581 +ILSVRC2012_val_00049976.JPEG:n02445715 +ILSVRC2012_val_00049977.JPEG:n01914609 +ILSVRC2012_val_00049978.JPEG:n02841315 +ILSVRC2012_val_00049979.JPEG:n02877765 +ILSVRC2012_val_00049980.JPEG:n01697457 +ILSVRC2012_val_00049981.JPEG:n01981276 +ILSVRC2012_val_00049982.JPEG:n06794110 +ILSVRC2012_val_00049983.JPEG:n04485082 +ILSVRC2012_val_00049984.JPEG:n02119022 +ILSVRC2012_val_00049985.JPEG:n02481823 +ILSVRC2012_val_00049986.JPEG:n02802426 +ILSVRC2012_val_00049987.JPEG:n01689811 +ILSVRC2012_val_00049988.JPEG:n01796340 +ILSVRC2012_val_00049989.JPEG:n02667093 +ILSVRC2012_val_00049990.JPEG:n01622779 +ILSVRC2012_val_00049991.JPEG:n01980166 +ILSVRC2012_val_00049992.JPEG:n02442845 +ILSVRC2012_val_00049993.JPEG:n04328186 +ILSVRC2012_val_00049994.JPEG:n01871265 +ILSVRC2012_val_00049995.JPEG:n03729826 +ILSVRC2012_val_00049996.JPEG:n02123394 +ILSVRC2012_val_00049997.JPEG:n01630670 +ILSVRC2012_val_00049998.JPEG:n02106166 +ILSVRC2012_val_00049999.JPEG:n10148035 +ILSVRC2012_val_00050000.JPEG:n02437616 \ No newline at end of file diff --git a/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py b/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py new file mode 100644 index 00000000000..160e2b8bf44 --- /dev/null +++ b/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Process imagenet validate dataset. +""" +import os + +from mindspore import log as logger + + +def preprocess_imagenet_validation_dataset(train_dataset_path, validation_dataset_path, image_label_mapping_file): + """ + Call this function before read imagenet validation dataset. + + Args: + train_dataset_path (str): train dataset path + validation_dataset_path (str): validation dataset path + image_label_mapping_file (str): imagenet_validate_dataset_2012_image_dir_map.txt file path + """ + train_dataset_path = os.path.realpath(train_dataset_path) + sub_dir = [dir.name for dir in os.scandir(train_dataset_path) if dir.is_dir()] + for sub_dir_name in sub_dir: + validate_sub_dir = os.path.join(validation_dataset_path, sub_dir_name) + validate_sub_dir = os.path.realpath(validate_sub_dir) + if not os.path.exists(validate_sub_dir): + os.makedirs(validate_sub_dir) + + mappings = [mapping.strip() for mapping in open(image_label_mapping_file).readlines()] + for mapping in mappings: + image_dir = mapping.split(':') + old_image_path = os.path.join(validation_dataset_path, image_dir[0]) + old_image_path = os.path.realpath(old_image_path) + if not os.path.exists(old_image_path): + logger.warning('Image is not existed %s', old_image_path) + new_image_sub_dir = os.path.join(validation_dataset_path, image_dir[1]) + new_image_sub_dir = os.path.realpath(new_image_sub_dir) + new_image_path = os.path.join(new_image_sub_dir, image_dir[0]) + new_image_path = os.path.realpath(new_image_path) + if not os.path.exists(new_image_sub_dir): + logger.warning('Image sub dir is not existed %s', new_image_sub_dir) + os.rename(old_image_path, new_image_path) diff --git a/mindspore/dataset/engine/__init__.py b/mindspore/dataset/engine/__init__.py new file mode 100644 index 00000000000..720b56b96d5 --- /dev/null +++ b/mindspore/dataset/engine/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +""" +Introduction to dataset/engine: + +dataset/engine supports various formats of datasets, including ImageNet, TFData, +MNIST, Cifar10/100, Manifest, MindRecord, etc. This module could load data in +high performance and parse data precisely. It also provides the following +operations for users to preprocess data: shuffle, batch, repeat, map, and zip. +""" + +from .datasets import * +from .iterators import * +from .serializer_deserializer import serialize, deserialize, show, compare +from .samplers import * +from ..core.configuration import config, ConfigurationManager + + +__all__ = ["config", "ConfigurationManager", "zip", "StorageDataset", + "ImageFolderDatasetV2", "MnistDataset", + "MindDataset", "GeneratorDataset", "TFRecordDataset", + "ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", + "VOCDataset", "Schema", "DistributedSampler", "PKSampler", "RandomSampler", + "SequentialSampler", "SubsetRandomSampler", "WeightedRandomSampler"] diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py new file mode 100644 index 00000000000..8aaa6212b6b --- /dev/null +++ b/mindspore/dataset/engine/datasets.py @@ -0,0 +1,2617 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +datasets.py supports various formats of datasets, including ImageNet, TFData, +MNIST, Cifar10/100, Manifest, MindRecord, etc. This module could load data in +high performance and parse data precisely. It also provides the following +operations for users to preprocess data: shuffle, batch, repeat, map, and zip. +""" +import glob +import json +import math +import os +import random +import uuid +from enum import Enum +from importlib import import_module + +import numpy as np +from mindspore._c_dataengine import DataType, TFReaderOp, ImageFolderOp, CifarOp, MnistOp, ManifestOp, \ + MindRecordOp, CBatchInfo +from mindspore._c_expression import typing + +from mindspore import log as logger +from . import samplers +from .iterators import DictIterator, TupleIterator +from .validators import check, check_batch, check_shuffle, check_map, check_repeat, check_zip, check_rename, \ + check_project, check_imagefolderdatasetv2, check_mnist_cifar_dataset, check_manifestdataset, \ + check_tfrecorddataset, check_vocdataset, check_celebadataset, check_minddataset, check_generatordataset, \ + check_zip_dataset +from ..core.datatypes import mstype_to_detype, mstypelist_to_detypelist + +try: + context = import_module("mindspore.context") +except ModuleNotFoundError: + context = None + + +class Shuffle(str, Enum): + GLOBAL: str = "global" + FILES: str = "file" + + +@check_zip +def zip(datasets): + """ + Zips the datasets in the input tuple of datasets. + + Args: + datasets (tuple of class Dataset): A tuple of datasets to be zipped together. + The number of datasets should be more than 1. + + Returns: + DatasetOp, ZipDataset. + + Raises: + ValueError: If the number of datasets is 1. + TypeError: If datasets is not a tuple. + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir1 = "path/to/imagefolder_directory1" + >>> dataset_dir2 = "path/to/imagefolder_directory2" + >>> ds1 = ds.ImageFolderDatasetV2(dataset_dir1, num_parallel_workers=8) + >>> ds2 = ds.ImageFolderDatasetV2(dataset_dir2, num_parallel_workers=8) + >>> + >>> # creates a dataset which is the combination of ds1 and ds2 + >>> data = ds.zip((ds1, ds2)) + """ + if len(datasets) <= 1: + raise ValueError( + "Can't zip empty or just one dataset!") + if not isinstance(datasets, tuple): + raise TypeError("The zip function %s type error!" % (datasets)) + return ZipDataset(datasets) + + +def get_num_rows(num_rows, num_shards): + """ + Get the number rows of the dataset according to the shards. + + Args: + num_rows (int): The number rows of the dataset should be more than 0. + The number rows of the dataset should be more than 0. + num_shards (int or None): Number of shards that the dataset should be divided into. + The number of shards should be None or more than 1. + + Returns: + Int, number of rows. + + Raises: + ValueError: If num_rows is invalid (< 0). + ValueError: If num_shards is invalid (<= 0). + """ + if num_rows < 0: + raise ValueError("num_rows is invalid (< 0)") + + if num_shards is not None: + if num_shards <= 0: + raise ValueError("num_shards is invalid (<= 0)") + if num_rows % num_shards == 0: + num_rows = num_rows // num_shards + else: + num_rows = num_rows // num_shards + 1 + return num_rows + + +class Dataset: + """ + Abstract class to represent a dataset in DataEngine's data pipeline. + + This class is the base class of SourceDataset and DatasetOp, and represents + a node in the data flow graph. + + Args: + num_parallel_workers (int, optional): Number of workers to process the Dataset in parallel + (default=None). + """ + + def __init__(self, num_parallel_workers=None): + self.input = [] + self.output = [] + self.num_parallel_workers = num_parallel_workers + self._device_iter = 0 + self._input_indexs = () + self._output_types = None + self._output_shapes = None + self._dataset_size = None + self._batch_size = None + self._num_classes = None + self._repeat_count = None + + def get_args(self): + """ + Returns attributes (member variables) related to the current class. + + Must include all arguments passed to the __init__() of the current class, excluding 'input_dataset'. + + Args: + + Returns: + Python dictionary. + """ + args = dict() + args["num_parallel_workers"] = self.num_parallel_workers + return args + + @check_batch + def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None, + input_columns=None): + """ + Combines batch_size number of consecutive rows into batches. + + For any child node, a batch is treated as a single row. + For any column, all the elements within that column must have the same shape. + If a per_batch_map callable is provided, it will be applied to the batches of tensors. + + Note: + The order of using repeat and batch reflects the number of batches. Recommend that + repeat operation should be used after batch operation. + + Args: + batch_size (int or function): The number of rows each batch is created with. An + int or callable which takes exactly 1 parameter, BatchInfo. + drop_remainder (bool, optional): Determines whether or not to drop the last + possibly incomplete batch (default=False). If True, and if there are less + than batch_size rows available to make the last batch, then those rows will + be dropped and not propogated to the child node. + num_parallel_workers (int, optional): Number of workers to process the Dataset in parallel (default=None). + per_batch_map (callable, optional): Per batch map callable. A callable which takes + (list[Tensor], list[Tensor], ..., BatchInfo) as input parameters. Each list[Tensor] represent a batch of + Tensors on a given column. The number of lists should match with number of entries in input_columns. The + last parameter of the callable should always be a BatchInfo object. + input_columns (list of string, optional): List of names of the input columns. The size of the list should + match with signature of per_batch_map callable. + + Returns: + BatchDataset, dataset batched. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object. + >>> # creates a dataset where every 100 rows is combined into a batch + >>> # and drops the last incomplete batch if there is one. + >>> data = data.batch(100, True) + """ + return BatchDataset(self, batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns) + + @check_shuffle + def shuffle(self, buffer_size): + """ + Randomly shuffles the rows of this dataset using the following algorithm: + + 1. Make a shuffle buffer that contains the first buffer_size rows. + 2. Randomly select an element from the shuffle buffer to be the next row + propogated to the child node. + 3. Get the next row (if any) from the parent node and put it in the shuffle buffer. + 4. Repeat steps 2 and 3 until there are no more rows left in the shuffle buffer. + + A seed can be provided to be used on the first epoch. In every subsequent + epoch, the seed is changed to a new one, randomly generated value. + + Args: + buffer_size (int): The size of the buffer (must be larger than 1) for + shuffling. Setting buffer_size equal to the number of rows in the entire + dataset will result in a global shuffle. + + Returns: + ShuffleDataset, dataset shuffled. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object + >>> # optionally set the seed for the first epoch + >>> ds.config.set_seed(58) + >>> + >>> # creates a shuffled dataset using a shuffle buffer of size 4 + >>> data = data.shuffle(4) + """ + return ShuffleDataset(self, buffer_size) + + @check_map + def map(self, input_columns=None, operations=None, output_columns=None, columns_order=None, + num_parallel_workers=None): + """ + Applies each operation in operations to this dataset. + + The order of operations is determined by the position of each operation in operations. + operations[0] will be applied first, then operations[1], then operations[2], etc. + + Each operation will be passed one or more columns from the dataset as input, and zero or + more columns will be outputted. The first operation will be passed the columns specified + in input_columns as input. If there is more than one operator in operations, the outputted + columns of the previous operation are used as the input columns for the next operation. + The columns outputted by the very last operation will be assigned names specified by + output_columns. + + Only the columns specified in columns_order will be propagated to the child node. These + columns will be in the same order as specified in columns_order. + + Args: + input_columns (list[str]): List of the names of the columns that will be passed to + the first operation as input. The size of this list must match the number of + input columns expected by the first operator. (default=None, the first + operation will be passed however many columns that is required, starting from + the first column). + operations (list[TensorOp] or Python list[functions]): List of operations to be + applied on the dataset. Operations are applied in the order they appear in this list. + output_columns (list[str], optional): List of names assigned to the columns outputted by + the last operation. This parameter is mandatory if len(input_columns) != + len(output_columns). The size of this list must match the number of output + columns of the last operation. (default=None, output columns will have the same + name as the input columns, i.e., the columns will be replaced). + columns_order (list[str], optional): list of all the desired columns to propagate to the + child node. This list must be a subset of all the columns in the dataset after + all operations are applied. The order of the columns in each row propagated to the + child node follow the order they appear in this list. The parameter is mandatory + if the len(input_columns) != len(output_columns). (default=None, all columns + will be propagated to the child node, the order of the columns will remain the + same). + num_parallel_workers (int, optional): Number of threads used to process the dataset in + parallel (default=None, the value from the config will be used). + + Returns: + MapDataset, dataset after mapping operation. + + Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.dataset.transforms.vision.c_transforms as c_transforms + >>> + >>> # data is an instance of Dataset which has 2 columns, "image" and "label". + >>> # ds_pyfunc is an instance of Dataset which has 3 columns, "col0", "col1", and "col2". Each column is + >>> # a 2d array of integers. + >>> + >>> # This config is a global setting, meaning that all future operations which + >>> # uses this config value will use 2 worker threads, unless if specified + >>> # otherwise in their constructor. set_num_parallel_workers can be called + >>> # again later if a different number of worker threads are needed. + >>> ds.config.set_num_parallel_workers(2) + >>> + >>> # Two operations, which takes 1 column for input and outputs 1 column. + >>> decode_op = c_transforms.Decode(rgb_format=True) + >>> random_jitter_op = c_transforms.RandomColorAdjust((0.8, 0.8), (1, 1), (1, 1), (0, 0)) + >>> + >>> # 1) Simple map example + >>> + >>> operations = [decode_op] + >>> input_columns = ["image"] + >>> + >>> # Applies decode_op on column "image". This column will be replaced by the outputed + >>> # column of decode_op. Since columns_order is not provided, both columns "image" + >>> # and "label" will be propagated to the child node in their original order. + >>> ds_decoded = data.map(input_columns, operations) + >>> + >>> # Rename column "image" to "decoded_image" + >>> output_columns = ["decoded_image"] + >>> ds_decoded = data.map(input_columns, operations, output_columns) + >>> + >>> # Specify the order of the columns. + >>> columns_order ["label", "image"] + >>> ds_decoded = data.map(input_columns, operations, None, columns_order) + >>> + >>> # Rename column "image" to "decoded_image" and also specify the order of the columns. + >>> columns_order ["label", "decoded_image"] + >>> output_columns = ["decoded_image"] + >>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order) + >>> + >>> # Rename column "image" to "decoded_image" and keep only this column. + >>> columns_order ["decoded_image"] + >>> output_columns = ["decoded_image"] + >>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order) + >>> + >>> # Simple example using pyfunc. Renaming columns and specifying column order + >>> # work in the same way as the previous examples. + >>> input_columns = ["col0"] + >>> operations = [(lambda x: x + 1)] + >>> ds_mapped = ds_pyfunc.map(input_columns, operations) + >>> + >>> # 2) Map example with more than one operation + >>> + >>> # If this list of operations is used with map, decode_op will be applied + >>> # first, then random_jitter_op will be applied. + >>> operations = [decode_op, random_jitter_op] + >>> + >>> input_columns = ["image"] + >>> + >>> # Creates a dataset where the images are decoded, then randomly color jittered. + >>> # decode_op takes column "image" as input and outputs one column. The column + >>> # outputted by decode_op is passed as input to random_jitter_op. + >>> # random_jitter_op will output one column. Column "image" will be replaced by + >>> # the column outputted by random_jitter_op (the very last operation). All other + >>> # columns are unchanged. Since columns_order is not specified, the order of the + >>> # columns will remain the same. + >>> ds_mapped = data.map(input_columns, operations) + >>> + >>> # Creates a dataset that is identical to ds_mapped, except the column "image" + >>> # that is outputted by random_jitter_op is renamed to "image_transformed". + >>> # Specifying column order works in the same way as examples in 1). + >>> output_columns = ["image_transformed"] + >>> ds_mapped_and_renamed = data.map(input_columns, operation, output_columns) + >>> + >>> # Multiple operations using pyfunc. Renaming columns and specifying column order + >>> # work in the same way as examples in 1). + >>> input_columns = ["col0"] + >>> operations = [(lambda x: x + x), (lambda x: x - 1)] + >>> output_columns = ["col0_mapped"] + >>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns) + >>> + >>> # 3) Example where number of input columns is not equal to number of output columns + >>> + >>> # operations[0] is a lambda that takes 2 columns as input and outputs 3 columns. + >>> # operations[1] is a lambda that takes 3 columns as input and outputs 1 column. + >>> # operations[1] is a lambda that takes 1 column as input and outputs 4 columns. + >>> # + >>> # Note: the number of output columns of operation[i] must equal the number of + >>> # input columns of operation[i+1]. Otherwise, this map call will also result + >>> # in an error. + >>> operations = [(lambda x y: (x, x + y, x + y + 1)), + >>> (lambda x y z: x * y * z), + >>> (lambda x: (x % 2, x % 3, x % 5, x % 7))] + >>> + >>> # Note: because the number of input columns is not the same as the number of + >>> # output columns, the output_columns and columns_order parameter must be + >>> # specified. Otherwise, this map call will also result in an error. + >>> input_columns = ["col2", "col0"] + >>> output_columns = ["mod2", "mod3", "mod5", "mod7"] + >>> + >>> # Propagate all columns to the child node in this order: + >>> columns_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"] + >>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order) + >>> + >>> # Propagate some columns to the child node in this order: + >>> columns_order = ["mod7", "mod3", "col1"] + >>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order) + """ + return MapDataset(self, input_columns, operations, output_columns, columns_order, num_parallel_workers) + + @check_repeat + def repeat(self, count=None): + """ + Repeats this dataset count times. Repeat indefinitely if the count is None or -1. + + Note: + The order of using repeat and batch reflects the number of batches. Recommend that + repeat operation should be used after batch operation. + If dataset_sink_mode is False (feed mode), here repeat operation is invalid. + + Args: + count (int): Number of times the dataset should be repeated (default=None). + + Returns: + RepeatDataset, dataset repeated. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object. + >>> # creates a dataset where the dataset is repeated for 50 epochs + >>> repeated = data.repeat(50) + >>> + >>> # creates a dataset where each epoch is shuffled individually + >>> shuffled_and_repeated = data.shuffle(10) + >>> shuffled_and_repeated = shuffled_and_repeated.repeat(50) + >>> + >>> # creates a dataset where the dataset is first repeated for + >>> # 50 epochs before shuffling. the shuffle operator will treat + >>> # the entire 50 epochs as one big dataset. + >>> repeat_and_shuffle = data.repeat(50) + >>> repeat_and_shuffle = repeat_and_shuffle.shuffle(10) + """ + return RepeatDataset(self, count) + + @check_zip_dataset + def zip(self, datasets): + """ + Zips the datasets in the input tuple of datasets. Columns in the input datasets must not have the same name. + + Args: + datasets (tuple or class Dataset): A tuple of datasets or a single class Dataset + to be zipped together with this dataset. + + Returns: + ZipDataset, dataset zipped. + + Examples: + >>> import mindspore.dataset as ds + >>> # ds1 and ds2 are instances of Dataset object + >>> # creates a dataset which is the combination of ds1 and ds2 + >>> data = ds1.zip(ds2) + """ + if isinstance(datasets, tuple): + datasets = (self, *datasets) + elif isinstance(datasets, Dataset): + datasets = (self, datasets) + else: + raise TypeError("The zip function %s type error!" % (datasets)) + return ZipDataset(datasets) + + @check_rename + def rename(self, input_columns, output_columns): + """ + Renames the columns in input datasets. + + Args: + input_columns (list[str]): list of names of the input columns. + output_columns (list[str]): list of names of the output columns. + + Returns: + RenameDataset, dataset renamed. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object. + >>> input_columns = ["input_col1", "input_col2", "input_col3"] + >>> output_columns = ["output_col1", "output_col2", "output_col3"] + >>> + >>> # creates a dataset where input_col1 is renamed to output_col1, and + >>> # input_col2 is renamed to output_col2, and input_col3 is renamed + >>> # to output_col3. + >>> data = data.rename(input_columns=input_columns, output_columns=output_columns) + """ + + return RenameDataset(self, input_columns, output_columns) + + @check_project + def project(self, columns): + """ + Projects certain columns in input datasets. + + The specified columns will be selected from the dataset and passed down + the pipeline in the order specified. The other columns are discarded. + + Args: + columns(list[str]): list of names of the columns to project. + + Returns: + ProjectDataset, dataset projected. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object + >>> columns_to_project = ["column3", "column1", "column2"] + >>> + >>> # creates a dataset that consist of column3, column1, column2 + >>> # in that order, regardless of the original order of columns. + >>> data = data.project(columns=columns_to_project) + """ + + return ProjectDataset(self, columns) + + def device_que(self, prefetch_size=None): + """ + Returns a transferredDataset that transfer data through tdt. + + Args: + prefetch_size (int, optional): prefetch number of records ahead of the + user's request (default=None). + + Return: + TransferDataset, dataset for transferring. + """ + return self.to_device() + + def to_device(self, num_batch=None): + """ + Transfers data through CPU, GPU or Ascend devices. + + Args: + num_batch (int, optional): limit the number of batch to be sent to device (default=None). + + Returns: + TransferDataset, dataset for transferring. + + Raises: + TypeError: If device_type is empty. + ValueError: If device_type is not 'Ascend', 'GPU' or 'CPU'. + ValueError: If num_batch is None or 0 or larger than int_max. + RuntimeError: If dataset is unknown. + RuntimeError: If distribution file path is given but failed to read. + """ + if num_batch is None: + num_batch = self.get_dataset_size() + repeat_count = self.get_repeat_count() + num_batch = num_batch * repeat_count + + queue_name = str(uuid.uuid1()) + + if context: + device_type = context.get_context("device_target") + else: + device_type = "CPU" + + if device_type == "": + raise TypeError("Please set device_type in context") + + if device_type not in ('Ascend', 'GPU', 'CPU'): + raise ValueError("only support CPU, Ascend, GPU") + + if num_batch is None or num_batch == 0: + raise ValueError("num_batch is None or 0.") + + def get_distribution(output_dataset): + dev_id = 0 + if isinstance(output_dataset, (StorageDataset, GeneratorDataset, MindDataset)): + return output_dataset.distribution, dev_id + if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, ImageFolderDatasetV2, + ManifestDataset, MnistDataset, VOCDataset, CelebADataset)): + sampler = output_dataset.sampler + if isinstance(sampler, samplers.DistributedSampler): + dev_id = sampler.shard_id + return "", dev_id + if isinstance(output_dataset, TFRecordDataset): + if output_dataset.shard_id is not None: + dev_id = output_dataset.shard_id + return "", dev_id + + if not output_dataset.input: + raise RuntimeError("Unknown output_dataset: {}".format(type(output_dataset))) + input_dataset = output_dataset.input[0] + return get_distribution(input_dataset) + + distribution_path, device_id = get_distribution(self) + if distribution_path == "": + return TransferDataset(self, queue_name, device_id, device_type, num_batch) + try: + with open(distribution_path, 'r') as distribution_f: + dist = json.load(distribution_f) + device_id = dist["deviceId"] + except json.decoder.JSONDecodeError: + raise RuntimeError("Json decode error when load distribution file") + except Exception: + raise RuntimeError("Distribution file failed to read") + + return TransferDataset(self, queue_name, device_id, device_type, num_batch) + + def create_tuple_iterator(self, columns=None): + """ + Create an Iterator over the dataset. The data retrieved will be a list of ndarray of data. + + To specify which columns to list and the order needed, use columns_list. If columns_list + is not provided, the order of the columns will not be changed. + + Args: + columns (list[str], optional): List of columns to be used to specify the order of columns + (defaults=None, means all columns). + + Returns: + Iterator, list of ndarray. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object + >>> # creates an iterator. The columns in the data obtained by the + >>> # iterator will not be changed. + >>> iterator = data.create_tuple_iterator() + >>> for item in iterator: + >>> # convert the returned tuple to a list and print + >>> print(list(item)) + """ + return TupleIterator(self, columns) + + def create_dict_iterator(self): + """ + Create an Iterator over the dataset. + + The data retrieved will be a dictionary. The order + of the columns in the dictionary may not be the same as the original order. + + Returns: + Iterator, dictionary of column_name-ndarray pair. + + Examples: + >>> import mindspore.dataset as ds + >>> # data is an instance of Dataset object + >>> # creates an iterator. The columns in the data obtained by the + >>> # iterator might be changed. + >>> iterator = data.create_dict_iterator() + >>> for item in iterator: + >>> # print the data in column1 + >>> print(item["column1"]) + + """ + return DictIterator(self) + + def __iter__(self): + """Create an Iterator over the dataset.""" + return self.create_tuple_iterator() + + @staticmethod + def read_dir(dir_path, schema, columns_list=None, num_parallel_workers=None, + deterministic_output=True, prefetch_size=None, shuffle=False, seed=None, distribution=""): + """ + Append the path of all files in the dir_path to StorageDataset. + + Args: + dir_path (str): Path to the directory that contains the dataset. + schema (str): Path to the json schema file. + columns_list (list[str], optional): List of columns to be read (default=None). + If not provided, read all columns. + num_parallel_workers (int, optional): Number of workers to process the Dataset in parallel + (default=None). + deterministic_output (bool, optional): Whether the result of this dataset can be reproduced + or not (default=True). If True, performance might be affected. + prefetch_size (int, optional): Prefetch number of records ahead of the + user's request (default=None). + shuffle (bool, optional): Shuffle the list of files in the directory (default=False). + seed (int, optional): Create a random generator with a fixed seed. If set to None, + create a random seed (default=None). + distribution (str, optional): The path of distribution config file (default=""). + + Returns: + StorageDataset. + + Raises: + ValueError: If dataset folder does not exist. + ValueError: If dataset folder permission denied. + """ + logger.warning("WARN_DEPRECATED: The usage of read_dir is deprecated, please use TFRecordDataset with GLOB.") + + list_files = [] + + if not os.path.isdir(dir_path): + raise ValueError("The dataset folder does not exist!") + if not os.access(dir_path, os.R_OK): + raise ValueError("The dataset folder permission denied!") + + for root, _, files in os.walk(dir_path): + for file in files: + list_files.append(os.path.join(root, file)) + + list_files.sort() + + if shuffle: + rand = random.Random(seed) + rand.shuffle(list_files) + + return StorageDataset(list_files, schema, distribution, columns_list, num_parallel_workers, + deterministic_output, prefetch_size) + + @property + def input_indexs(self): + return self._input_indexs + + @input_indexs.setter + def input_indexs(self, value): + self._input_indexs = value + + def _get_pipeline_info(self): + device_iter = TupleIterator(self) + self._output_shapes = device_iter.get_output_shapes() + self._output_types = device_iter.get_output_types() + if self._dataset_size is None: + self._dataset_size = device_iter.get_dataset_size() + self._batch_size = device_iter.get_batch_size() + self._num_classes = device_iter.num_classes() + self._repeat_count = device_iter.get_repeat_count() + device_iter.release() + + def output_shapes(self): + """ + Get the shapes of output data. + + Return: + List, list of shape of each column. + """ + if self._output_shapes is None: + self._get_pipeline_info() + return self._output_shapes + + def output_types(self): + """ + Get the types of output data. + + Return: + List of data type. + """ + if self._output_types is None: + self._get_pipeline_info() + return self._output_types + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self.input: + return self.input[0].get_dataset_size() + return None + + def num_classes(self): + """ + Get the number of classes in a dataset. + + Return: + Number, number of classes. + """ + if self.input: + return self.input[0].num_classes() + return None + + def get_batch_size(self): + """ + Get the size of a batch. + + Return: + Number, the number of data in a batch. + """ + if self.input: + return self.input[0].get_batch_size() + return 1 + + def get_repeat_count(self): + """ + Get the replication times in RepeatDataset else 1 + + Return: + Number, the count of repeat. + """ + if self.input: + return self.input[0].get_repeat_count() + return 1 + + def get_class_indexing(self): + """ + Get the class index. + + Return: + Dict, A str-to-int mapping from label name to index. + """ + if self.input: + return self.input[0].get_class_indexing() + return None + + def reset(self): + """Reset the dataset for next epoch""" + + +class SourceDataset(Dataset): + """ + Abstract class to represent a source dataset which produces content to the data pipeline. + """ + + # No need for __init__ since it is the same as the super's init + + +class DatasetOp(Dataset): + """ + Abstract class to represent a operations on dataset. + """ + + # No need for __init__ since it is the same as the super's init + + +class BatchDataset(DatasetOp): + """ + The result of applying Batch operator to the input dataset. + + Args: + input_dataset (Dataset): Input Dataset to be batched. + batch_size (int): The size of the batch. + drop_remainder (bool, optional): Whether drop the remainder batch of data (drop_remainder=False). + If True, the last incomplete batch will be dropped. + """ + + def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, + per_batch_map=None, input_columns=None): + super().__init__(num_parallel_workers) + + if BatchDataset._is_ancestor_of_repeat(input_dataset): + logger.warning("Repeat is located before batch, data from two epochs can be batched together.") + + self.batch_size = batch_size + self.drop_remainder = drop_remainder + self.per_batch_map = per_batch_map + self.input_columns = input_columns + self.input.append(input_dataset) + input_dataset.output.append(self) + self._input_indexs = input_dataset.input_indexs + + def get_args(self): + args = super().get_args() + args["batch_size"] = self.batch_size + args["drop_remainder"] = self.drop_remainder + args["per_batch_map"] = self.per_batch_map + args["input_columns"] = self.input_columns + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + child_size = self.input[0].get_dataset_size() + if child_size is not None: + if self.drop_remainder: + return math.floor(child_size / self.batch_size) + return math.ceil(child_size / self.batch_size) + return None + + def get_batch_size(self): + """ + Get the size of a batch. + + Return: + Number, the number of data in a batch. + """ + return self.batch_size + + @staticmethod + def _is_ancestor_of_repeat(dataset): + """ + Utility function to find the case where repeat is used before batch. + + Args: + dataset (Dataset): dataset to be checked + Return: + True or False + """ + if isinstance(dataset, RepeatDataset): + return True + flag = False + for input_dataset in dataset.input: + flag = flag | BatchDataset._is_ancestor_of_repeat(input_dataset) + return flag + + +class BatchInfo(CBatchInfo): + """ + The information object associates with the current batch of tensors. + """ + + def get_batch_num(self): + """ + Return the batch number of the current batch. + + Return: + Number, number of the current batch. + """ + return + + def get_epoch_num(self): + """ + Return the epoch number of the current batch. + + Return: + Number, number of the current epoch. + """ + return + + +class ShuffleDataset(DatasetOp): + """ + The result of applying Shuffle operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be shuffled. + buffer_size (int): The size of the buffer. + """ + + def __init__(self, input_dataset, buffer_size): + super().__init__() + self.buffer_size = buffer_size + self.input.append(input_dataset) + input_dataset.output.append(self) + self._input_indexs = input_dataset.input_indexs + + def get_args(self): + args = super().get_args() + args["buffer_size"] = self.buffer_size + return args + + +class MapDataset(DatasetOp): + """ + The result of applying Map operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be mapped. + input_columns (list[str]): List of names of the input columns + (default=None, the operations will be applied on the first columns in the dataset). + The size of the list should match the number of inputs of the first operator. + operations (TensorOp): A function mapping a nested structure of tensors + to another nested structure of tensor (default=None). + output_columns (list[str], optional): list of names of the output columns. + The size of the list should match the number of outputs of the last operator + (default=None, output columns will be the input columns, i.e., the columns will + be replaced). + columns_order (list[str], optional): list of all the desired columns of the dataset (default=None). + The argument is mandatory if len(input_columns) != len(output_columns). + num_parallel_workers (int, optional): Number of workers to process the Dataset + in parallel (default=None). + + Raises: + ValueError: If len(input_columns) != len(output_columns) and columns_order is not specified. + """ + + def __init__(self, input_dataset, input_columns=None, operations=None, output_columns=None, columns_order=None, + num_parallel_workers=None): + super().__init__(num_parallel_workers) + self.input.append(input_dataset) + if input_columns is not None and not isinstance(input_columns, list): + input_columns = [input_columns] + self.input_columns = input_columns + if operations is not None and not isinstance(operations, list): + operations = [operations] + self.operations = operations + if output_columns is not None and not isinstance(output_columns, list): + output_columns = [output_columns] + self.output_columns = output_columns + self.columns_order = columns_order + + if self.input_columns and self.output_columns \ + and len(self.input_columns) != len(self.output_columns) \ + and self.columns_order is None: + raise ValueError("When (len(input_columns) != len(output_columns)), columns_order must be specified.") + + input_dataset.output.append(self) + self._input_indexs = input_dataset.input_indexs + + def get_args(self): + args = super().get_args() + args["input_columns"] = self.input_columns + args["operations"] = self.operations + args["output_columns"] = self.output_columns + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + return self.input[0].get_dataset_size() + + +class RepeatDataset(DatasetOp): + """ + The result of applying Repeat operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be repeated. + count (int): Number of times the dataset should be repeated. + """ + + def __init__(self, input_dataset, count): + super().__init__() + if count is None: + self.count = -1 + else: + self.count = count + self.input.append(input_dataset) + input_dataset.output.append(self) + self._input_indexs = input_dataset.input_indexs + + def get_args(self): + args = super().get_args() + args["count"] = self.count + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + child_size = self.input[0].get_dataset_size() + if child_size is not None: + return child_size + return None + + def get_repeat_count(self): + """ + Get the replication times in RepeatDataset. + + Return: + Number, the count of repeat. + """ + return self.count + + +class ZipDataset(DatasetOp): + """ + The result of applying Zip operator to the input Dataset. + + Args: + datasets (tuple): A tuple of datasets to be zipped together. + + Raises: + TypeError: If dataset is not an instance of Dataset. + """ + + def __init__(self, datasets): + super().__init__() + for dataset in datasets: + if not isinstance(dataset, Dataset): + raise TypeError("The parameter %s of zip has type error!" % (dataset)) + self.datasets = datasets + for data in datasets: + self.input.append(data) + data.output.append(self) + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + children_sizes = [c.get_dataset_size() for c in self.input] + if all(c is not None for c in children_sizes): + return min(children_sizes) + return None + + def num_classes(self): + """ + Get the number of classes in a dataset. + + Return: + Number, number of classes. + """ + return None + + def get_args(self): + args = super().get_args() + return args + + +class RenameDataset(DatasetOp): + """ + The result of applying Rename operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be Renamed. + input_column_names (list[str]): list of names of the input columns. + output_column_names (list[str]): list of names of the output columns. + """ + + def __init__(self, input_dataset, input_columns, output_columns): + super().__init__() + if not isinstance(input_columns, list): + input_columns = [input_columns] + if not isinstance(output_columns, list): + output_columns = [output_columns] + self.input_column_names = input_columns + self.output_column_names = output_columns + self.input.append(input_dataset) + input_dataset.output.append(self) + self._input_indexs = input_dataset.input_indexs + + def get_args(self): + args = super().get_args() + args["input_columns"] = self.input_column_names + args["output_columns"] = self.output_column_names + return args + + +class ProjectDataset(DatasetOp): + """ + The result of applying Project operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be Project. + columns (list[str]): List of names of the columns to project. + prefetch_size (int, optional): Prefetch number of records ahead of the + user's request (default=None). + """ + + def __init__(self, input_dataset, columns, prefetch_size=None): + super().__init__() + if not isinstance(columns, list): + columns = [columns] + self.columns = columns + self.input.append(input_dataset) + self.prefetch_size = prefetch_size + + input_dataset.output.append(self) + self._input_indexs = input_dataset.input_indexs + + def get_args(self): + args = super().get_args() + args["columns"] = self.columns + args["prefetch_size"] = self.prefetch_size + return args + + +class TransferDataset(DatasetOp): + """ + The result of applying TDT operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be transferred. + queue_name (str): Name of device queue. + device_id (int): Id of device. + device_type (str): Type of device, including "CPU", "GPU", and "Ascend". + num_batch (int): limit the number of batch to be sent to device (default=None). + """ + + def __init__(self, input_dataset, queue_name, device_id, device_type, num_batch=None): + super().__init__() + self.input.append(input_dataset) + input_dataset.output.append(self) + self.queue_name = queue_name + self._input_indexs = input_dataset.input_indexs + self._device_type = device_type + self._device_id = device_id + self.__num_batch = num_batch + self.iterator = None + + def get_args(self): + args = super().get_args() + args["queue_name"] = self.queue_name + args["device_type"] = self._device_type + args["device_id"] = self._device_id + args["num_batch"] = self.__num_batch + return args + + def create_dict_iterator(self): + raise RuntimeError("TransferDataset is not iterable") + + def create_tuple_iterator(self, columns=None): + raise RuntimeError("TransferDataset is not iterable") + + def __iter__(self): + raise RuntimeError("TransferDataset is not iterable") + + def output_shapes(self): + raise RuntimeError("TransferDataset does not support output_shapes") + + def output_types(self): + raise RuntimeError("TransferDataset does not support output_types") + + def send(self): + # need to keep iterator alive so the executionTree is not destroyed + self.iterator = TupleIterator(self) + + +class StorageDataset(SourceDataset): + """ + A source dataset that reads and parses datasets stored on disk in various formats, including TFData format. + + Args: + dataset_files (list[str]): List of files to be read. + schema (str): Path to the json schema file. + distribution (str, optional): Path of distribution config file (default=""). + columns_list (list[str], optional): List of columns to be read (default=None, read all columns). + num_parallel_workers (int, optional): Number of parallel working threads (default=None). + deterministic_output (bool, optional): Whether the result of this dataset can be reproduced + or not (default=True). If True, performance might be affected. + prefetch_size (int, optional): Prefetch number of records ahead of the user's request (default=None). + + Raises: + RuntimeError: If schema file failed to read. + RuntimeError: If distribution file path is given but failed to read. + """ + + @check + def __init__(self, dataset_files, schema, distribution="", columns_list=None, num_parallel_workers=None, + deterministic_output=None, prefetch_size=None): + super().__init__(num_parallel_workers) + logger.warning("WARN_DEPRECATED: The usage of StorageDataset is deprecated, please use TFRecordDataset.") + self.dataset_files = dataset_files + try: + with open(schema, 'r') as load_f: + json.load(load_f) + except json.decoder.JSONDecodeError: + raise RuntimeError("Json decode error when load schema file") + except Exception: + raise RuntimeError("Schema file failed to load") + + if distribution != "": + try: + with open(distribution, 'r') as load_d: + json.load(load_d) + except json.decoder.JSONDecodeError: + raise RuntimeError("Json decode error when load distribution file") + except Exception: + raise RuntimeError("Distribution file failed to load") + if self.dataset_files is None: + schema = None + distribution = None + self.schema = schema + self.distribution = distribution + self.columns_list = columns_list + self.deterministic_output = deterministic_output + self.prefetch_size = prefetch_size + + def get_args(self): + args = super().get_args() + args["dataset_files"] = self.dataset_files + args["schema"] = self.schema + args["distribution"] = self.distribution + args["columns_list"] = self.columns_list + args["deterministic_output"] = self.deterministic_output + args["prefetch_size"] = self.prefetch_size + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self._dataset_size is None: + self._get_pipeline_info() + return self._dataset_size + + # manually set dataset_size as a temporary solution. + def set_dataset_size(self, value): + logger.warning("WARN_DEPRECATED: This method is deprecated. Please use get_dataset_size directly.") + if value >= 0: + self._dataset_size = value + else: + raise ValueError('set dataset_size with negative value {}'.format(value)) + + def num_classes(self): + """ + Get the number of classes in dataset. + + Return: + Number, number of classes. + + Raises: + ValueError: If dataset type is invalid. + ValueError: If dataset is not Imagenet dataset or manifest dataset. + RuntimeError: If schema file is given but failed to load. + """ + cur_dataset = self + while cur_dataset.input: + cur_dataset = cur_dataset.input[0] + if not hasattr(cur_dataset, "schema"): + raise ValueError("Dataset type is invalid") + # Only IMAGENET/MANIFEST support numclass + try: + with open(cur_dataset.schema, 'r') as load_f: + load_dict = json.load(load_f) + except json.decoder.JSONDecodeError: + raise RuntimeError("Json decode error when load schema file") + except Exception: + raise RuntimeError("Schema file failed to load") + if load_dict["datasetType"] != "IMAGENET" and load_dict["datasetType"] != "MANIFEST": + raise ValueError("%s dataset does not support num_classes!" % (load_dict["datasetType"])) + + if self._num_classes is None: + self._get_pipeline_info() + return self._num_classes + + +class RangeDataset(SourceDataset): + """ + A source dataset that reads and parses datasets stored on disk in a range. + + Args: + start (int): starting index. + stop (int): ending index. + step (int): step size in a range. + """ + + def __init__(self, start, stop, step): + super().__init__() + self.start = start + self.stop = stop + self.step = step + + def get_args(self): + args = super().get_args() + args["start"] = self.start + args["stop"] = self.stop + args["step"] = self.step + return args + + +def _select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id): + """ + Create sampler based on user input. + + Args: + num_samples (int): Number of samples + input_sampler (Iterable / Sampler): Sampler from user + shuffle (bool): Shuffle + num_shards (int): Number of shard for sharding + shard_id (int): Shard ID + """ + if shuffle is None: + if input_sampler is not None: + # If shuffle is not specified, user provided sampler, use user's sampler + return input_sampler + if num_shards is not None: + # If shuffle is not specified, sharding enabled, use distributed random sampler + shuffle = True + return samplers.DistributedSampler(num_shards, shard_id, shuffle=shuffle) + # If shuffle is not specified, sharding disabled, use random sampler + if num_samples is not None: + return samplers.RandomSampler(replacement=True, num_samples=num_samples) + return samplers.RandomSampler() + if shuffle is True: + if num_shards is not None: + # If shuffle enabled, sharding enabled, use distributed random sampler + return samplers.DistributedSampler(num_shards, shard_id, shuffle=shuffle) + # If shuffle enabled, sharding disabled, use random sampler + if num_samples is not None: + return samplers.RandomSampler(replacement=True, num_samples=num_samples) + return samplers.RandomSampler() + if num_shards is not None: + # If shuffle disabled, sharding enabled, use distributed sequential sampler + return samplers.DistributedSampler(num_shards, shard_id, shuffle=shuffle) + # If shuffle disabled, sharding disabled, use sequential sampler + return samplers.SequentialSampler() + + + +class ImageFolderDatasetV2(SourceDataset): + """ + A source dataset that reads images from a tree of directories. + + All images within one folder have the same label. + The generated dataset has two columns ['image', 'label']. + The shape of the image column is [image_size] if decode flag is False, or [H,W,C] + otherwise. + The type of the image tensor is uint8. The label is just a scalar uint64 + tensor. + This dataset can take in a sampler. sampler and shuffle are mutually exclusive. Table + below shows what input args are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_samples (int, optional): The number of images to be included in the dataset + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, set in the config). + shuffle (bool, optional): Whether or not to perform shuffle on the dataset + (default=None, expected order behavior shown in the table). + sampler (Sampler, optional): Object used to choose samples from the + dataset (default=None, expected order behavior shown in the table). + extensions (list[str], optional): List of file extensions to be + included in the dataset (default=None). + class_indexing (dict, optional): A str-to-int mapping from folder name to index + (default=None, the folder names will be sorted + alphabetically and each class will be given a + unique index starting from 0). + decode (bool, optional): decode the images after reading (default=False). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + + Raises: + RuntimeError: If sampler and shuffle are specified at the same time. + RuntimeError: If sampler and sharding are specified at the same time. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + RuntimeError: If class_indexing is not a dictionary. + ValueError: If shard_id is invalid (< 0 or >= num_shards). + + Examples: + >>> import mindspore.dataset as ds + >>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images + >>> dataset_dir = "/path/to/imagefolder_directory" + >>> # 1) read all samples (image files) in dataset_dir with 8 threads + >>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8) + >>> # 2) read all samples (image files) from folder cat and folder dog with label 0 and 1 + >>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir,class_indexing={"cat":0,"dog":1}) + >>> # 3) read all samples (image files) in dataset_dir with extensions .JPEG and .png (case sensitive) + >>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, extensions={".JPEG",".png"}) + """ + + @check_imagefolderdatasetv2 + def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, + shuffle=None, sampler=None, extensions=None, class_indexing=None, + decode=False, num_shards=None, shard_id=None): + super().__init__(num_parallel_workers) + + self.dataset_dir = dataset_dir + self.sampler = _select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + self.num_samples = num_samples + self.shuffle_level = shuffle + self.extensions = extensions + self.class_indexing = class_indexing + self.decode = decode + self.num_shards = num_shards + self.shard_id = shard_id + + def get_args(self): + args = super().get_args() + args["dataset_dir"] = self.dataset_dir + args["num_samples"] = self.num_samples + args["sampler"] = self.sampler + args["shuffle"] = self.shuffle_level + args["extensions"] = self.extensions + args["class_indexing"] = self.class_indexing + args["decode"] = self.decode + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + num_rows = ImageFolderOp.get_num_rows_and_classes(self.dataset_dir, num_samples)[0] + + return get_num_rows(num_rows, self.num_shards) + + def num_classes(self): + """ + Get the number of classes in dataset. + + Return: + Number, number of classes. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + return ImageFolderOp.get_num_rows_and_classes(self.dataset_dir, num_samples)[1] + + +class MnistDataset(SourceDataset): + """ + A source dataset for reading and parsing the Mnist dataset. + + The generated dataset has two columns ['image', 'label']. + The type of the image tensor is uint8. The label is just a scalar uint32 tensor. + This dataset can take in a sampler. sampler and shuffle are mutually exclusive. Table + below shows what input args are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_samples (int, optional): The number of images to be included in the dataset + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=value, set in the config). + shuffle (bool, optional): Whether or not to perform shuffle on the dataset + (default=None, expected order behavior shown in the table). + sampler (Sampler, optional): Object used to choose samples from the + dataset (default=None, expected order behavior shown in the table). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + + Raises: + RuntimeError: If sampler and shuffle are specified at the same time. + RuntimeError: If sampler and sharding are specified at the same time. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + ValueError: If shard_id is invalid (< 0 or >= num_shards). + + Examples: + >>> import mindspore.dataset as ds + >>> dataset_dir = "/path/to/mnist_folder" + >>> # 1) read 3 samples from mnist_dataset + >>> mnist_dataset = ds.MnistDataset(dataset_dir=dataset_dir, num_samples=3) + >>> # in mnist_dataset dataset, each dictionary has keys "image" and "label" + """ + + @check_mnist_cifar_dataset + def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, + shuffle=None, sampler=None, num_shards=None, shard_id=None): + super().__init__(num_parallel_workers) + + self.dataset_dir = dataset_dir + self.sampler = _select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + self.num_samples = num_samples + self.shuffle_level = shuffle + self.num_shards = num_shards + self.shard_id = shard_id + + def get_args(self): + args = super().get_args() + args["dataset_dir"] = self.dataset_dir + args["num_samples"] = self.num_samples + args["shuffle"] = self.shuffle_level + args["sampler"] = self.sampler + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + + num_rows = MnistOp.get_num_rows(self.dataset_dir, num_samples) + + return get_num_rows(num_rows, self.num_shards) + + +class MindDataset(SourceDataset): + """ + A source dataset that reads from shard files and database. + + Args: + dataset_file (str): one of file names in dataset. + columns_list (list[str], optional): List of columns to be read (default=None). + num_parallel_workers (int, optional): The number of readers (default=None). + shuffle (bool, optional): Whether or not to perform shuffle on the dataset + (default=None, performs shuffle). + num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + block_reader (bool, optional): Whether read data by block mode (default=False). + + Raises: + ValueError: If num_shards is specified but shard_id is None. + ValueError: If shard_id is specified but num_shards is None. + ValueError: If block reader is true but partition is specified. + """ + + @check_minddataset + def __init__(self, dataset_file, columns_list=None, num_parallel_workers=None, + shuffle=None, num_shards=None, shard_id=None, block_reader=False): + super().__init__(num_parallel_workers) + self.dataset_file = dataset_file + self.columns_list = columns_list + self.global_shuffle = not bool(shuffle is False) + self.distribution = "" + + if num_shards is None: + self.partitions = None + else: + self.partitions = [num_shards, shard_id] + + if block_reader is True and self.partitions is not None: + raise ValueError("block reader not allowed true when use partitions") + + if block_reader is True: + logger.warning("WARN: global shuffle is not used.") + + self.num_shards = num_shards + self.shard_id = shard_id + self.block_reader = block_reader + + def get_args(self): + args = super().get_args() + args["dataset_file"] = self.dataset_file + args["columns_list"] = self.columns_list + args["global_shuffle"] = self.global_shuffle + args["partitions"] = self.partitions + args["block_reader"] = self.block_reader + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + + num_rows = MindRecordOp.get_num_rows(self.dataset_file) + if self.partitions is not None and self.partitions[0] > 0: + if num_rows % self.partitions[0] == 0: + num_rows = num_rows // self.partitions[0] + else: + num_rows = num_rows // self.partitions[0] + 1 + return num_rows + + +def ds_fn(dataset): + for val in dataset: + # convert output tensors to ndarrays + yield tuple([np.array(x) for x in val]) + + +def sampler_fn(sampler, dataset): + for i in sampler: + val = dataset[i] + # convert output tensors to ndarrays + yield tuple([np.array(x) for x in val]) + + +class GeneratorDataset(SourceDataset): + """ + A source dataset that generate data from calling generator function each epoch. + + Args: + generator_function (callable): + A callable object that returns an Generator object that supports the iter() protocol. + Generator object is required to return a tuple of numpy array as a row of the dataset on next(). + column_names (list[str]): List of column names of the dataset. + column_types (list[mindspore.dtype], optional): List of column data types of the dataset (default=None). + If provided, sanity check will be performed on generator output. + prefetch_size (int, optional): Prefetch number of records ahead of the user's request (default=None). + sampler (Sampler, optional): Object used to choose samples from the dataset (default=None). + + Examples: + >>> import mindspore.dataset as ds + >>> # 1) generator function that generates multi-dimensional data + >>> def generator_md(): + >>> for i in range(64): + >>> yield (np.array([[i, i + 1], [i + 2, i + 3]]),) + >>> # create multi_dimension_generator_dataset with GeneratorMD() and column name "multi_dimensional_data" + >>> multi_dimension_generator_dataset = ds.GeneratorDataset(generator_md, ["multi_dimensional_data"]) + >>> # 2) generator function that generates multi-columns data + >>> def generator_mc(maxid = 64): + >>> for i in range(maxid): + >>> yield (np.array([i]), np.array([[i, i + 1], [i + 2, i + 3]])) + >>> # create multi_column_generator_dataset with GeneratorMC() and column names "col1" and "col2" + >>> multi_column_generator_dataset = ds.GeneratorDataset(generator_mc, ["col1, col2"]) + """ + + @check_generatordataset + def __init__(self, generator_function, column_names, column_types=None, prefetch_size=None, sampler=None): + super().__init__(1) + if sampler is not None: + self.generator_function = (lambda: sampler_fn(sampler, generator_function)) + else: + try: + # test to see if generator_function is iterable + iter(generator_function) + except TypeError: + # generator_function was not iterable, assume it is a function + self.generator_function = generator_function + else: + # generator_function was iterable, build a function around it + self.generator_function = (lambda: ds_fn(generator_function)) + + self.column_names = column_names + + if column_types is not None: + self.column_types = mstypelist_to_detypelist(column_types) + else: + self.column_types = column_types + self.distribution = "" + self.prefetch_size = prefetch_size + self.sampler = sampler + + def get_args(self): + args = super().get_args() + args["generator_function"] = self.generator_function + args["column_names"] = self.column_names + args["column_types"] = self.column_types + args["prefetch_size"] = self.prefetch_size + args["sampler"] = self.sampler + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + return self._dataset_size + + # manually set dataset_size as a temporary solution. + def set_dataset_size(self, value): + if value >= 0: + self._dataset_size = value + else: + raise ValueError('set dataset_size with negative value {}'.format(value)) + + +class TFRecordDataset(SourceDataset): + """ + A source dataset that reads and parses datasets stored on disk in TFData format. + + Args: + dataset_files (str or list[str]): String or list of files to be read or glob strings to search for a pattern of + files. The list will be sorted in a lexicographical order. + schema (str or Schema, optional): Path to the json schema file or schema object (default=None). + If the schema is not provided, the meta data from the TFData file is considered the schema. + columns_list (list[str], optional): List of columns to be read (default=None, read all columns) + num_samples (int, optional): number of samples(rows) to read (default=None, reads the full dataset). + num_parallel_workers (int, optional): number of workers to read the data + (default=None, number set in the config). + shuffle (bool, Shuffle level, optional): perform reshuffling of the data every epoch (default=Shuffle.GLOBAL). + If shuffle is False, no shuffling will be performed; + If shuffle is True, the behavior is the same as setting shuffle to be Shuffle.GLOBAL + Otherwise, there are two levels of shuffling: + + - Shuffle.GLOBAL: Shuffle both the files and samples. + + - Shuffle.FILES: Shuffle files only. + + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + shard_equal_rows (bool): Get equal rows for all shards(default=False). If shard_equal_rows is false, number + of rows of each shard may be not equal. + Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.common.dtype as mstype + >>> dataset_files = ["/path/to/1", "/path/to/2"] # contains 1 or multiple tf data files + >>> # 1) get all rows from dataset_files with no explicit schema: + >>> # The meta-data in the first row will be used as a schema. + >>> tfdataset = ds.TFRecordDataset(dataset_files=dataset_files) + >>> # 2) get all rows from dataset_files with user-defined schema: + >>> schema = ds.Schema() + >>> schema.add_column('col_1d', de_type=mstype.int64, shape=[2]) + >>> tfdataset = ds.TFRecordDataset(dataset_files=dataset_files, schema=schema) + >>> # 3) get all rows from dataset_files with schema file "./schema.json": + >>> tfdataset = ds.TFRecordDataset(dataset_files=dataset_files, schema="./schema.json") + """ + + @staticmethod + def _find_files(patterns): + """ + Utility function to search for files with the given glob patterns. + + Args: + patterns (str or list[str]): string or list of patterns to be searched. + + Returns: + List, files. + """ + + def flat(lists): + return list(np.array(lists).flatten()) + + if not isinstance(patterns, list): + patterns = [patterns] + + file_list = flat([glob.glob(file, recursive=True) for file in patterns]) + if file_list: # not empty + return file_list + raise ValueError("The list of path names matching the patterns is empty.") + + @check_tfrecorddataset + def __init__(self, dataset_files, schema=None, columns_list=None, num_samples=None, num_parallel_workers=None, + shuffle=Shuffle.GLOBAL, num_shards=None, shard_id=None, shard_equal_rows=False): + super().__init__(num_parallel_workers) + self.dataset_files = self._find_files(dataset_files) + self.dataset_files.sort() + self.num_shards = num_shards + self.shard_id = shard_id + schema_obj = None + if (schema is not None) and (not isinstance(schema, Schema)): + schema_obj = Schema(schema) # read the schema file and convert to schema object to validate it + self.schema = schema + self.columns_list = columns_list + self.num_samples = num_samples + if schema_obj is not None and num_samples is None: + self.num_samples = schema_obj.num_rows + + if not isinstance(shuffle, (bool, Shuffle)): + raise TypeError("shuffle should be of boolean or enum 'Shuffle'.") + if not isinstance(shuffle, Shuffle): + if shuffle: + self.shuffle_level = Shuffle.GLOBAL + self.shuffle_files = True + else: + self.shuffle_level = None + self.shuffle_files = False + else: + self.shuffle_level = shuffle + self.shuffle_files = True + self.shard_equal_rows = shard_equal_rows + + def get_args(self): + args = super().get_args() + args["dataset_files"] = self.dataset_files + if self.schema is not None: + if isinstance(self.schema, Schema): + self.schema.datasetType = 'TF' + if self.num_samples is not None: + self.schema.num_rows = self.num_samples + args["schema_json_string"] = self.schema.to_json() + else: + args["schema_file_path"] = self.schema + args["schema"] = self.schema + args["columns_list"] = self.columns_list + args["num_samples"] = self.num_samples + if self.shuffle_files is not None: + args["shuffle_files"] = self.shuffle_files + args["shuffle"] = self.shuffle_level + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + args["shard_equal_rows"] = self.shard_equal_rows + return args + + def get_dataset_size(self, estimate=False): + """ + Get the number of batches in an epoch. + + Args: + estimate (bool, optional): Fast estimation of the dataset size instead of a full scan. + + Return: + Number, number of batches. + """ + num_rows = TFReaderOp.get_num_rows(self.dataset_files, 8, estimate) + num_rows = get_num_rows(num_rows, self.num_shards) + if self.num_samples is None: + return num_rows + return min(self.num_samples, num_rows) + + +class ManifestDataset(SourceDataset): + """ + A source dataset that reads images from a manifest file. + + The generated dataset has two columns ['image', 'label']. + The shape of the image column is [image_size] if decode flag is False, or [H,W,C] + otherwise. + The type of the image tensor is uint8. The label is just a scalar uint64 + tensor. + This dataset can take in a sampler. sampler and shuffle are mutually exclusive. Table + below shows what input args are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Args: + dataset_file (str): File to be read. + usage (str, optional): Need train, eval or inference data (default="train"). + num_samples (int, optional): The number of images to be included in the dataset. + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, number set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected + order behavior shown in the table). + sampler (Sampler, optional): Object used to choose samples from the + dataset (default=None, expected order behavior shown in the table). + class_indexing (dict, optional): A str-to-int mapping from label name to index + (default=None, the folder names will be sorted alphabetically and each + class will be given a unique index starting from 0). + decode (bool, optional): decode the images after reading (defaults=False). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + + Raises: + RuntimeError: If sampler and shuffle are specified at the same time. + RuntimeError: If sampler and sharding are specified at the same time. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + RuntimeError: If class_indexing is not a dictionary. + ValueError: If shard_id is invalid (< 0 or >= num_shards). + + Examples: + >>> import mindspore.dataset as ds + >>> dataset_file = "/path/to/manifest_file.manifest" + >>> # 1) read all samples specified in manifest_file dataset with 8 threads for training: + >>> manifest_dataset = ds.ManifestDataset(dataset_file, usage="train", num_parallel_workers=8) + >>> # 2) reads samples (specified in manifest_file.manifest) for shard 0 in a 2-way distributed training setup: + >>> manifest_dataset = ds.ManifestDataset(dataset_file, num_shards=2, shard_id=0) + + """ + + @check_manifestdataset + def __init__(self, dataset_file, usage="train", num_samples=None, num_parallel_workers=None, + shuffle=None, sampler=None, class_indexing=None, decode=False, num_shards=None, shard_id=None): + super().__init__(num_parallel_workers) + + self.dataset_file = dataset_file + self.sampler = _select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + + if class_indexing is not None and not isinstance(class_indexing, dict): + raise RuntimeError("class_indexing should be a dictionary.") + + self.num_samples = num_samples + self.class_indexing = class_indexing + self.decode = decode + self.usage = usage + self.shuffle_level = shuffle + self.num_shards = num_shards + self.shard_id = shard_id + + def get_args(self): + args = super().get_args() + args["dataset_file"] = self.dataset_file + args["usage"] = self.usage + args["num_samples"] = self.num_samples + args["shuffle"] = self.shuffle_level + args["sampler"] = self.sampler + args["class_indexing"] = self.class_indexing + args["decode"] = self.decode + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + + if self.class_indexing is None: + class_indexing = dict() + else: + class_indexing = self.class_indexing + + num_rows = ManifestOp.get_num_rows_and_classes(self.dataset_file, num_samples, class_indexing, self.usage)[0] + + return get_num_rows(num_rows, self.num_shards) + + def num_classes(self): + """ + Get the number of classes in a dataset. + + Return: + Number, number of classes. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + + if self.class_indexing is None: + class_indexing = dict() + else: + class_indexing = self.class_indexing + + return ManifestOp.get_num_rows_and_classes(self.dataset_file, num_samples, class_indexing, self.usage)[1] + + def get_class_indexing(self): + """ + Get the class index + + Return: + Dict, A str-to-int mapping from label name to index. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + + if self.class_indexing is None: + class_indexing = dict() + else: + class_indexing = self.class_indexing + + return ManifestOp.get_class_indexing(self.dataset_file, num_samples, class_indexing, self.usage) + + +class Cifar10Dataset(SourceDataset): + """ + A source dataset that reads cifar10 data. + + The generated dataset has two columns ['image', 'label']. + The type of the image tensor is uint8. The label is just a scalar uint32 + tensor. + This dataset can take in a sampler. sampler and shuffle are mutually exclusive. Table + below shows what input args are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_samples (int, optional): The number of images to be included in the dataset. + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, number set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected + order behavior shown in the table). + sampler (Sampler, optional): Object used to choose samples from the + dataset (default=None, expected order behavior shown in the table). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + + Raises: + RuntimeError: If sampler and shuffle are specified at the same time. + RuntimeError: If sampler and sharding are specified at the same time. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + ValueError: If shard_id is invalid (< 0 or >= num_shards). + + Examples: + >>> import mindspore.dataset as ds + >>> dataset_dir = "/path/to/cifar10_dataset_directory" + >>> # 1) get all samples from CIFAR10 dataset in sequence: + >>> dataset = ds.Cifar10Dataset(dataset_dir=dataset_dir,shuffle=False) + >>> # 2) randomly select 350 samples from CIFAR10 dataset: + >>> dataset = ds.Cifar10Dataset(dataset_dir=dataset_dir,num_samples=350, shuffle=True) + >>> # 3) get samples from CIFAR10 dataset for shard 0 in a 2 way distributed training: + >>> dataset = ds.Cifar10Dataset(dataset_dir=dataset_dir,num_shards=2,shard_id=0) + >>> # in CIFAR10 dataset, each dictionary has keys "image" and "label" + """ + + @check_mnist_cifar_dataset + def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, + shuffle=None, sampler=None, num_shards=None, shard_id=None): + super().__init__(num_parallel_workers) + + self.dataset_dir = dataset_dir + self.sampler = _select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + self.num_samples = num_samples + self.num_shards = num_shards + self.shard_id = shard_id + self.shuffle_level = shuffle + + def get_args(self): + args = super().get_args() + args["dataset_dir"] = self.dataset_dir + args["num_samples"] = self.num_samples + args["sampler"] = self.sampler + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + args["shuffle"] = self.shuffle_level + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + + num_rows = CifarOp.get_num_rows(self.dataset_dir, num_samples, True) + + return get_num_rows(num_rows, self.num_shards) + + +class Cifar100Dataset(SourceDataset): + """ + A source dataset that reads cifar100 data. + + The generated dataset has three columns ['image', 'coarse_label', 'fine_label']. + The type of the image tensor is uint8. The coarse and fine are just a scalar uint32 + tensor. + This dataset can take in a sampler. sampler and shuffle are mutually exclusive. Table + below shows what input args are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_samples (int, optional): The number of images to be included in the dataset. + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, number set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected + order behavior shown in the table). + sampler (Sampler, optional): Object used to choose samples from the + dataset (default=None, expected order behavior shown in the table). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + + Raises: + RuntimeError: If sampler and shuffle are specified at the same time. + RuntimeError: If sampler and sharding are specified at the same time. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + ValueError: If shard_id is invalid (< 0 or >= num_shards). + + Examples: + >>> import mindspore.dataset as ds + >>> dataset_dir = "/path/to/cifar100_dataset_directory" + >>> # 1) get all samples from CIFAR100 dataset in sequence: + >>> cifar100_dataset = ds.Cifar100Dataset(dataset_dir=dataset_dir,shuffle=False) + >>> # 2) randomly select 350 samples from CIFAR100 dataset: + >>> cifar100_dataset = ds.Cifar100Dataset(dataset_dir=dataset_dir,num_samples=350, shuffle=True) + >>> # in CIFAR100 dataset, each dictionary has 3 keys: "image", "fine_label" and "coarse_label" + """ + + @check_mnist_cifar_dataset + def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, + shuffle=None, sampler=None, num_shards=None, shard_id=None): + super().__init__(num_parallel_workers) + + self.dataset_dir = dataset_dir + self.sampler = _select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + self.num_samples = num_samples + self.num_shards = num_shards + self.shard_id = shard_id + self.shuffle_level = shuffle + + def get_args(self): + args = super().get_args() + args["dataset_dir"] = self.dataset_dir + args["num_samples"] = self.num_samples + args["sampler"] = self.sampler + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + args["shuffle"] = self.shuffle_level + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + if self.num_samples is None: + num_samples = 0 + else: + num_samples = self.num_samples + + num_rows = CifarOp.get_num_rows(self.dataset_dir, num_samples, False) + + return get_num_rows(num_rows, self.num_shards) + + +class Schema: + """ + Class to represent a schema of dataset. + + Args: + schema_file(str): Path of schema file (default=None). + + Return: + Schema object, schema info about dataset. + + Raises: + RuntimeError: If schema file failed to load. + + Example: + >>> import mindspore.dataset as ds + >>> import mindspore.common.dtype as mstype + >>> # create schema, specify column name, mindspore.dtype and shape of the column + >>> schema = ds.Schema() + >>> schema.add_column('col1', de_type=mstype.int64, shape=[2]) + """ + + def __init__(self, schema_file=None): + if schema_file is None: + self.columns = [] + self.dataset_type = '' + self.num_rows = 0 + else: + try: + with open(schema_file, 'r') as load_f: + json_obj = json.load(load_f) + self.from_json(json_obj) + except json.decoder.JSONDecodeError: + raise RuntimeError("Schema file failed to load") + + def add_column(self, name, de_type, shape=None): + """ + Add new column to the schema. + + Args: + name (str): name of the column. + de_type (str): data type of the column. + shape (list[int], optional): shape of the column + (default=None, [-1] which is an unknown shape of rank 1). + + Raises: + ValueError: If column type is unknown. + """ + new_column = dict() + new_column["name"] = name + if isinstance(de_type, typing.Type): + de_type = mstype_to_detype(de_type) + new_column["type"] = str(de_type) + elif isinstance(de_type, str): + new_column["type"] = str(DataType(de_type)) + else: + raise ValueError("Unknown column type") + + if shape is not None: + new_column["shape"] = shape + new_column["rank"] = len(shape) + else: + new_column["rank"] = 1 + self.columns.append(new_column) + + def to_json(self): + """ + Get a JSON string of the schema. + + Returns: + Str, JSON string of the schema. + """ + json_file = dict() + json_file["columns"] = self.columns + if self.dataset_type: + json_file["datasetType"] = self.dataset_type + if self.num_rows: + json_file["numRows"] = self.num_rows + return json.dumps(json_file, indent=2) + + def parse_columns(self, columns): + """ + Parse the columns and add it to self. + + Args: + columns (list[str]): names of columns. + + Raises: + RuntimeError: If failed to parse schema file. + RuntimeError: If unknown items in schema file. + RuntimeError: If column's name field is missing. + RuntimeError: If column's type field is missing. + """ + self.columns = [] + for col in columns: + name = None + shape = None + data_type = None + col_details = None + if isinstance(columns, list): + col_details = col + if "name" in col: + name = col["name"] + elif isinstance(columns, dict): + col_details = columns[col] + name = col + else: + raise RuntimeError("Error parsing the schema file") + + for k, v in col_details.items(): + if k == "shape": + shape = v + elif k == "type": + data_type = v + elif k in ("t_impl", "rank"): + pass + else: + raise RuntimeError("Unknown field %s" % k) + + if name is None: + raise RuntimeError("Column's name field is missing.") + if data_type is None: + raise RuntimeError("Column's type field is missing.") + self.add_column(name, data_type, shape) + + def from_json(self, json_obj): + """ + Get schema file from json file. + + Args: + json_obj(dictionary): object of json parsed. + + Raises: + RuntimeError: if there is unknown item in the object. + RuntimeError: if dataset type is missing in the object. + RuntimeError: if columns are missing in the object. + """ + for k, v in json_obj.items(): + if k == "datasetType": + self.dataset_type = v + elif k == "numRows": + self.num_rows = v + elif k == "columns": + self.parse_columns(v) + else: + raise RuntimeError("Unknown field %s" % k) + + if self.dataset_type is None: + raise RuntimeError("DatasetType field is missing.") + if self.columns is None: + raise RuntimeError("Columns are missing.") + + def __str__(self): + return self.to_json() + + +class VOCDataset(SourceDataset): + """ + A source dataset for reading and parsing VOC dataset. + + The generated dataset has two columns ['image', 'target']. + The shape of both column is [image_size] if decode flag is False, or [H, W, C] + otherwise. + The type of both tensor is uint8. + This dataset can take in a sampler. sampler and shuffle are mutually exclusive. Table + below shows what input args are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_samples (int, optional): The number of images to be included in the dataset + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, number set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected + order behavior shown in the table). + decode (bool, optional): Decode the images after reading (default=False). + sampler (Sampler, optional): Object used to choose samples from the dataset + (default=None, expected order behavior shown in the table). + distribution (str, optional): Path to the json distribution file to configure + dataset sharding (default=None). This argument should be specified + only when no 'sampler' is used. + + Raises: + RuntimeError: If distribution and sampler are specified at the same time. + RuntimeError: If distribution is failed to read. + RuntimeError: If shuffle and sampler are specified at the same time. + + Examples: + >>> import mindspore.dataset as ds + >>> dataset_dir = "/path/to/voc_dataset_directory" + >>> # 1) read all VOC dataset samples in dataset_dir with 8 threads in random order: + >>> voc_dataset = ds.VOCDataset(dataset_dir, num_parallel_workers=8) + >>> # 2) read then decode all VOC dataset samples in dataset_dir in sequence: + >>> voc_dataset = ds.VOCDataset(dataset_dir, decode=True, shuffle=False) + >>> # in VOC dataset, each dictionary has keys "image" and "target" + """ + + @check_vocdataset + def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, + shuffle=None, decode=False, sampler=None, distribution=None): + super().__init__(num_parallel_workers) + self.dataset_dir = dataset_dir + self.sampler = sampler + if distribution is not None: + if sampler is not None: + raise RuntimeError("Cannot specify distribution and sampler at the same time.") + try: + with open(distribution, 'r') as load_d: + json.load(load_d) + except json.decoder.JSONDecodeError: + raise RuntimeError("Json decode error when load distribution file") + except Exception: + raise RuntimeError("Distribution file has failed to load.") + elif shuffle is not None: + if sampler is not None: + raise RuntimeError("Cannot specify shuffle and sampler at the same time.") + self.num_samples = num_samples + self.decode = decode + self.distribution = distribution + self.shuffle_level = shuffle + + def get_args(self): + args = super().get_args() + args["dataset_dir"] = self.dataset_dir + args["num_samples"] = self.num_samples + args["sampler"] = self.sampler + args["decode"] = self.decode + args["shuffle"] = self.shuffle_level + args["distribution"] = self.distribution + return args + + def get_dataset_size(self): + """ + Get the number of batches in an epoch. + + Return: + Number, number of batches. + """ + return self.num_samples + + +class CelebADataset(SourceDataset): + """ + A source dataset for reading and parsing CelebA dataset.Only support list_attr_celeba.txt currently + + Note: + The generated dataset has two columns ['image', 'attr']. + The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_parallel_workers (int, optional): Number of workers to read the data (default=value set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None). + dataset_type (string): one of 'all', 'train', 'valid' or 'test'. + sampler (Sampler, optional): Object used to choose samples from the dataset (default=None). + decode (bool, optional): decode the images after reading (default=False). + extensions (list[str], optional): List of file extensions to be + included in the dataset (default=None). + num_samples (int, optional): The number of images to be included in the dataset. + (default=None, all images). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. + """ + + @check_celebadataset + def __init__(self, dataset_dir, num_parallel_workers=None, shuffle=None, dataset_type='all', + sampler=None, decode=False, extensions=None, num_samples=None, num_shards=None, shard_id=None): + super().__init__(num_parallel_workers) + self.dataset_dir = dataset_dir + self.sampler = _select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + self.num_parallel_workers = num_parallel_workers + self.decode = decode + self.extensions = extensions + self.num_samples = num_samples + self.dataset_type = dataset_type + self.num_shards = num_shards + self.shard_id = shard_id + self.shuffle_level = shuffle + + def get_args(self): + args = super().get_args() + args["dataset_dir"] = self.dataset_dir + args["sampler"] = self.sampler + args["shuffle"] = self.shuffle_level + args["decode"] = self.decode + args["extensions"] = self.extensions + args["num_samples"] = self.num_samples + args["dataset_type"] = self.dataset_type + args["num_shards"] = self.num_shards + args["shard_id"] = self.shard_id + return args diff --git a/mindspore/dataset/engine/iterators.py b/mindspore/dataset/engine/iterators.py new file mode 100644 index 00000000000..806df64fdc4 --- /dev/null +++ b/mindspore/dataset/engine/iterators.py @@ -0,0 +1,269 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Built-in iterators. +""" +from abc import abstractmethod + +from mindspore._c_dataengine import DEPipeline +from mindspore._c_dataengine import OpName + +from mindspore import log as logger +from . import datasets as de + +ITERATORS_LIST = list() + + +def _cleanup(): + for itr in ITERATORS_LIST: + itr.release() + + +def alter_tree(node): + """Traversing the python Dataset tree/graph to perform some alteration to some specific nodes.""" + if not node.input: + return _alter_node(node) + + converted_children = [] + for input_op in node.input: + converted_children.append(alter_tree(input_op)) + node.input = converted_children + return _alter_node(node) + + +def _alter_node(node): + """Performing some alteration to a dataset node. A common alteration is to insert a node.""" + if isinstance(node, de.TFRecordDataset) and node.shuffle_level == de.Shuffle.GLOBAL: + # Remove the connection between the parent's node to the current node because we are inserting a node. + if node.output: + node.output.pop() + # Perform a fast scan for average rows per file + avg_rows_per_file = node.get_dataset_size(True) // len(node.dataset_files) + # Shuffle between 4 files with a minimum size of 10000 rows + new_shuffle = node.shuffle(max(avg_rows_per_file * 4, 10000)) + return new_shuffle + + if isinstance(node, de.MapDataset): + if node.columns_order is not None: + # Remove the connection between the parent's node to the current node because we are inserting a node. + if node.output: + node.output.pop() + + return node.project(node.columns_order) + return node + + +class Iterator: + """ + General Iterator over a dataset. + + Attributes: + dataset: Dataset to be iterated over + + """ + + def __init__(self, dataset): + ITERATORS_LIST.append(self) + self.dataset = alter_tree(dataset) + if not self.__is_tree(): + raise ValueError("The data pipeline is not a tree (i.e., one node has 2 consumers)") + self.depipeline = DEPipeline() + + # for manifest temporary use + self.__batch_node(self.dataset, 0) + + root = self.__convert_node_postorder(self.dataset) + self.depipeline.AssignRootNode(root) + self.depipeline.LaunchTreeExec() + + def __is_tree_node(self, node): + """Check if a node is tree node.""" + if not node.input: + if len(node.output) > 1: + return False + + if len(node.output) > 1: + return False + + for input_node in node.input: + cls = self.__is_tree_node(input_node) + if not cls: + return False + return True + + def __is_tree(self): + return self.__is_tree_node(self.dataset) + + @staticmethod + def __get_dataset_type(dataset): + """Get the dataset type.""" + op_type = None + if isinstance(dataset, de.ShuffleDataset): + op_type = OpName.SHUFFLE + elif isinstance(dataset, de.MindDataset): + op_type = OpName.MINDRECORD + elif isinstance(dataset, de.BatchDataset): + op_type = OpName.BATCH + elif isinstance(dataset, de.ZipDataset): + op_type = OpName.ZIP + elif isinstance(dataset, de.MapDataset): + op_type = OpName.MAP + elif isinstance(dataset, de.RepeatDataset): + op_type = OpName.REPEAT + elif isinstance(dataset, de.StorageDataset): + op_type = OpName.STORAGE + elif isinstance(dataset, de.ImageFolderDatasetV2): + op_type = OpName.IMAGEFOLDER + elif isinstance(dataset, de.GeneratorDataset): + op_type = OpName.GENERATOR + elif isinstance(dataset, de.TransferDataset): + op_type = OpName.DEVICEQUEUE + elif isinstance(dataset, de.RenameDataset): + op_type = OpName.RENAME + elif isinstance(dataset, de.TFRecordDataset): + op_type = OpName.TFREADER + elif isinstance(dataset, de.ProjectDataset): + op_type = OpName.PROJECT + elif isinstance(dataset, de.MnistDataset): + op_type = OpName.MNIST + elif isinstance(dataset, de.ManifestDataset): + op_type = OpName.MANIFEST + elif isinstance(dataset, de.VOCDataset): + op_type = OpName.VOC + elif isinstance(dataset, de.Cifar10Dataset): + op_type = OpName.CIFAR10 + elif isinstance(dataset, de.Cifar100Dataset): + op_type = OpName.CIFAR100 + elif isinstance(dataset, de.CelebADataset): + op_type = OpName.CELEBA + else: + raise ValueError("Unsupported DatasetOp") + + return op_type + + # Convert python node into C node and add to C layer execution tree in postorder traversal. + def __convert_node_postorder(self, node): + op_type = self.__get_dataset_type(node) + c_node = self.depipeline.AddNodeToTree(op_type, node.get_args()) + + for py_child in node.input: + c_child = self.__convert_node_postorder(py_child) + self.depipeline.AddChildToParentNode(c_child, c_node) + + return c_node + + def __batch_node(self, dataset, level): + """Recursively get batch node in the dataset tree.""" + if isinstance(dataset, de.BatchDataset): + return + for input_op in dataset.input: + self.__batch_node(input_op, level + 1) + + @staticmethod + def __print_local(dataset, level): + """Recursively print the name and address of nodes in the dataset tree.""" + name = dataset.__class__.__name__ + ptr = hex(id(dataset)) + for _ in range(level): + logger.info("\t", end='') + if not dataset.input: + logger.info("-%s (%s)", name, ptr) + else: + logger.info("+%s (%s)", name, ptr) + for input_op in dataset.input: + Iterator.__print_local(input_op, level + 1) + + def print(self): + """ + Print the dataset tree + + """ + self.__print_local(self.dataset, 0) + + def release(self): + if hasattr(self, 'depipeline') and self.depipeline: + del self.depipeline + + @abstractmethod + def get_next(self): + pass + + def __next__(self): + data = self.get_next() + if not data: + raise StopIteration + return data + + def get_output_shapes(self): + return [t for t in self.depipeline.GetOutputShapes()] + + def get_output_types(self): + return [t for t in self.depipeline.GetOutputTypes()] + + def get_dataset_size(self): + return self.depipeline.GetDatasetSize() + + def get_batch_size(self): + return self.depipeline.GetBatchSize() + + def get_repeat_count(self): + return self.depipeline.GetRepeatCount() + + def num_classes(self): + return self.depipeline.GetNumClasses() + + +class DictIterator(Iterator): + """ + The derived class of Iterator with dict type. + """ + + def __iter__(self): + return self + + def get_next(self): + """ + Returns the next record in the dataset as dictionary + + Returns: + Dict, the next record in the dataset. + """ + + return {k: v.as_array() for k, v in self.depipeline.GetNextAsMap().items()} + + +class TupleIterator(Iterator): + """ + The derived class of Iterator with list type. + """ + + def __init__(self, dataset, columns=None): + if columns is not None: + if not isinstance(columns, list): + columns = [columns] + dataset = dataset.project(columns) + super().__init__(dataset) + + def __iter__(self): + return self + + def get_next(self): + """ + Returns the next record in the dataset as a list + + Returns: + List, the next record in the dataset. + """ + + return [t.as_array() for t in self.depipeline.GetNextAsList()] diff --git a/mindspore/dataset/engine/samplers.py b/mindspore/dataset/engine/samplers.py new file mode 100644 index 00000000000..ed36e72b651 --- /dev/null +++ b/mindspore/dataset/engine/samplers.py @@ -0,0 +1,241 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Sampler module provides several samplers to generate sampling data from dataset. +There are following samplers: DistributedSampler, PKSampler, RandomSampler, +SequentialSampler, SubsetRandomSampler, WeightedRandomSampler. +""" + +import mindspore._c_dataengine as cde + + +class DistributedSampler(): + """ + Sampler that access a shard of the dataset. + + Args: + num_shards (int): Number of shards to divide the dataset into. + shard_id (int): Shard ID of the current shard within num_shards. + shuffle (bool, optional): If true, the indices are shuffled (default=True). + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> + >>> # creates a distributed sampler with 10 shards total. This shard is shard 5 + >>> sampler = ds.DistributedSampler(10, 5) + >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + + Raises: + ValueError: If num_shards is not positive. + ValueError: If shard_id is smaller than 0 or equal to num_shards or larger than num_shards. + ValueError: If shuffle is not a boolean value. + """ + + def __init__(self, num_shards, shard_id, shuffle=True): + if num_shards <= 0: + raise ValueError("num_shards should be a positive integer value, but got num_shards={}".format(num_shards)) + + if shard_id < 0 or shard_id >= num_shards: + raise ValueError("shard_id is invalid, shard_id={}".format(shard_id)) + + if not isinstance(shuffle, bool): + raise ValueError("shuffle should be a boolean value, but got shuffle={}".format(shuffle)) + + self.num_shards = num_shards + self.shard_id = shard_id + self.shuffle = shuffle + self.seed = 0 + + def create(self): + # each time user calls create_dict_iterator() (to do repeat) sampler would get a different seed to shuffle + self.seed += 1 + return cde.DistributedSampler(self.num_shards, self.shard_id, self.shuffle, self.seed) + + +class PKSampler(): + """ + Samples K elements for each P class in the dataset. + + Args: + num_val (int): Number of elements to sample for each class. + num_class (int, optional): Number of classes to sample (default=None, all classes). + shuffle (bool, optional): If true, the class IDs are shuffled (default=False). + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> + >>> # creates a PKSampler that will get 3 samples from every class. + >>> sampler = ds.PKSampler(3) + >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + + Raises: + ValueError: If num_val is not positive. + NotImplementedError: If num_class is not None. + ValueError: If shuffle is not boolean. + """ + + def __init__(self, num_val, num_class=None, shuffle=False): + if num_val <= 0: + raise ValueError("num_val should be a positive integer value, but got num_val={}".format(num_val)) + + if num_class is not None: + raise NotImplementedError + + if not isinstance(shuffle, bool): + raise ValueError("shuffle should be a boolean value, but got shuffle={}".format(shuffle)) + + self.num_val = num_val + self.shuffle = shuffle + + def create(self): + return cde.PKSampler(self.num_val, self.shuffle) + + +class RandomSampler(): + """ + Samples the elements randomly. + + Args: + replacement (bool, optional): If True, put the sample ID back for the next draw (default=False). + num_samples (int, optional): Number of elements to sample (default=None, all elements). This + argument should be specified only when 'replacement' is "True". + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> + >>> # creates a RandomSampler + >>> sampler = ds.RandomSampler() + >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + + Raises: + ValueError: If replacement is not boolean. + ValueError: If num_samples is not None and replacement is false. + ValueError: If num_samples is not positive. + """ + + def __init__(self, replacement=False, num_samples=None): + if not isinstance(replacement, bool): + raise ValueError("replacement should be a boolean value, but got replacement={}".format(replacement)) + + if num_samples is not None: + if num_samples <= 0: + raise ValueError("num_samples should be a positive integer " + "value, but got num_samples={}".format(num_samples)) + + self.replacement = replacement + self.num_samples = num_samples + + def create(self): + # If num_samples is not specified, then call constructor #2 + if self.num_samples is None: + return cde.RandomSampler(self.replacement) + return cde.RandomSampler(self.replacement, self.num_samples) + + +class SequentialSampler(): + """ + Samples the dataset elements sequentially, same as not having a sampler. + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> + >>> # creates a SequentialSampler + >>> sampler = ds.SequentialSampler() + >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + """ + + def create(self): + return cde.SequentialSampler() + + +class SubsetRandomSampler(): + """ + Samples the elements randomly from a sequence of indices. + + Args: + indices (list[int]): A sequence of indices. + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> + >>> indices = [0, 1, 2, 3, 7, 88, 119] + >>> + >>> # creates a SubsetRandomSampler, will sample from the provided indices + >>> sampler = ds.SubsetRandomSampler() + >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + """ + + def __init__(self, indices): + if not isinstance(indices, list): + indices = [indices] + + self.indices = indices + + def create(self): + return cde.SubsetRandomSampler(self.indices) + + +class WeightedRandomSampler(): + """ + Samples the elements from [0, len(weights) - 1] randomly with the given weights (probabilities). + + Args: + weights (list[float]): A sequence of weights, not necessarily summing up to 1. + num_samples (int): Number of elements to sample. + replacement (bool, optional): If True, put the sample ID back for the next draw (default=True). + + Examples: + >>> import mindspore.dataset as ds + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> + >>> weights = [0.9, 0.01, 0.4, 0.8, 0.1, 0.1, 0.3] + >>> + >>> # creates a WeightedRandomSampler that will sample 4 elements without replacement + >>> sampler = ds.WeightedRandomSampler(weights, 4) + >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + + Raises: + ValueError: If num_samples is not positive. + ValueError: If replacement is not boolean. + """ + + def __init__(self, weights, num_samples, replacement=True): + if not isinstance(weights, list): + weights = [weights] + + if num_samples <= 0: + raise ValueError("num_samples should be a positive integer " + "value, but got num_samples={}".format(num_samples)) + + if not isinstance(replacement, bool): + raise ValueError("replacement should be a boolean value, but got replacement={}".format(replacement)) + + self.weights = weights + self.num_samples = num_samples + self.replacement = replacement + + def create(self): + return cde.WeightedRandomSampler(self.weights, self.num_samples, self.replacement) diff --git a/mindspore/dataset/engine/serializer_deserializer.py b/mindspore/dataset/engine/serializer_deserializer.py new file mode 100644 index 00000000000..d1ed5c47cd2 --- /dev/null +++ b/mindspore/dataset/engine/serializer_deserializer.py @@ -0,0 +1,436 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Functions to support dataset serialize and deserialize. +""" +import json +import os +import sys + +from mindspore import log as logger +from . import datasets as de +from ..transforms.vision.utils import Inter, Border + + +def serialize(dataset, json_filepath=None): + """ + Serialize dataset pipeline into a json file. + + Args: + dataset (Dataset): the starting node. + json_filepath (string): a filepath where a serialized json file will be generated. + + Returns: + dict containing the serialized dataset graph. + + Raises: + OSError cannot open a file + + Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.dataset.transforms.c_transforms as C + >>> DATA_DIR = "../../data/testMnistData" + >>> data = ds.MnistDataset(DATA_DIR, 100) + >>> one_hot_encode = C.OneHot(10) # num_classes is input argument + >>> data = data.map(input_column_names="label", operation=one_hot_encode) + >>> data = data.batch(batch_size=10, drop_remainder=True) + >>> + >>> ds.engine.serialize(data, json_filepath="mnist_dataset_pipeline.json") # serialize it to json file + >>> serialized_data = ds.engine.serialize(data) # serialize it to python dict + """ + serialized_pipeline = traverse(dataset) + if json_filepath: + with open(json_filepath, 'w') as json_file: + json.dump(serialized_pipeline, json_file, indent=2) + return serialized_pipeline + + +def deserialize(input_dict=None, json_filepath=None): + """ + Construct a de pipeline from a json file produced by de.serialize(). + + Args: + input_dict (dict): a python dictionary containing a serialized dataset graph + json_filepath (string): a path to the json file. + + Returns: + de.Dataset or None if error occurs. + + Raises: + OSError cannot open a file. + + Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.dataset.transforms.c_transforms as C + >>> DATA_DIR = "../../data/testMnistData" + >>> data = ds.MnistDataset(DATA_DIR, 100) + >>> one_hot_encode = C.OneHot(10) # num_classes is input argument + >>> data = data.map(input_column_names="label", operation=one_hot_encode) + >>> data = data.batch(batch_size=10, drop_remainder=True) + >>> + >>> # Use case 1: to/from json file + >>> ds.engine.serialize(data, json_filepath="mnist_dataset_pipeline.json") + >>> data = ds.engine.deserialize(json_filepath="mnist_dataset_pipeline.json") + >>> # Use case 2: to/from python dictionary + >>> serialized_data = ds.engine.serialize(data) + >>> data = ds.engine.deserialize(input_dict=serialized_data) + + """ + data = None + if input_dict: + data = construct_pipeline(input_dict) + + if json_filepath: + dict_pipeline = dict() + with open(json_filepath, 'r') as json_file: + dict_pipeline = json.load(json_file) + data = construct_pipeline(dict_pipeline) + + return data + + +def expand_path(node_repr, key, val): + """Convert relative to absolute path.""" + if isinstance(val, list): + node_repr[key] = [os.path.abspath(file) for file in val] + else: + node_repr[key] = os.path.abspath(val) + + +def serialize_operations(node_repr, key, val): + """Serialize tensor op (python object) to dictionary.""" + if isinstance(val, list): + node_repr[key] = [] + for op in val: + node_repr[key].append(op.__dict__) + # Extracting module and name information from a python object + # Example: tensor_op_module is 'minddata.transforms.c_transforms' and tensor_op_name is 'Decode' + node_repr[key][-1]['tensor_op_name'] = type(op).__name__ + node_repr[key][-1]['tensor_op_module'] = type(op).__module__ + else: + node_repr[key] = val.__dict__ + node_repr[key]['tensor_op_name'] = type(val).__name__ + node_repr[key]['tensor_op_module'] = type(val).__module__ + + +def serialize_sampler(node_repr, val): + """Serialize sampler object to dictionary.""" + node_repr['sampler'] = val.__dict__ + node_repr['sampler']['sampler_module'] = type(val).__module__ + node_repr['sampler']['sampler_name'] = type(val).__name__ + + +def traverse(node): + """Pre-order traverse the pipeline and capture the information as we go.""" + # Node representation (node_repr) is a python dictionary that capture and store the + # dataset pipeline information before dumping it to JSON or other format. + node_repr = dict() + node_repr['op_type'] = type(node).__name__ + node_repr['op_module'] = type(node).__module__ + + # Start with an empty list of children, will be added later as we traverse this node. + node_repr["children"] = [] + + # Retrieve the information about the current node. It should include arguments + # passed to the node during object construction. + node_args = node.get_args() + for k, v in node_args.items(): + # Store the information about this node into node_repr. + # Further serialize the object in the arguments if needed. + if k == 'operations': + serialize_operations(node_repr, k, v) + elif k == 'sampler': + serialize_sampler(node_repr, v) + elif k in set(['schema', 'dataset_files', 'dataset_dir', 'schema_file_path']): + expand_path(node_repr, k, v) + else: + node_repr[k] = v + + # Leaf node doesn't have input attribute. + if not node.input: + return node_repr + + # Recursively traverse the child and assign it to the current node_repr['children']. + for child in node.input: + node_repr["children"].append(traverse(child)) + + return node_repr + + +def show(dataset, indentation=2): + """ + Write the dataset pipeline graph onto logger.info. + + Args: + dataset (Dataset): the starting node. + indentation (int, optional): indentation used by the json print. Pass None to not indent. + """ + + pipeline = traverse(dataset) + logger.info(json.dumps(pipeline, indent=indentation)) + + +def compare(pipeline1, pipeline2): + """ + Compare if two dataset pipelines are the same. + + Args: + pipeline1 (Dataset): a dataset pipeline. + pipeline2 (Dataset): a dataset pipeline. + """ + + return traverse(pipeline1) == traverse(pipeline2) + + +def construct_pipeline(node): + """Construct the python Dataset objects by following the dictionary deserialized from json file.""" + op_type = node.get('op_type') + if not op_type: + raise ValueError("op_type field in the json file can't be None.") + + # Instantiate python Dataset object based on the current dictionary element + dataset = create_node(node) + # Initially it is not connected to any other object. + dataset.input = [] + + # Construct the children too and add edge between the children and parent. + for child in node['children']: + dataset.input.append(construct_pipeline(child)) + + return dataset + + +def create_node(node): + """Parse the key, value in the node dictionary and instantiate the python Dataset object""" + logger.info('creating node: %s', node['op_type']) + dataset_op = node['op_type'] + op_module = node['op_module'] + + # Get the python class to be instantiated. + # Example: + # "op_type": "MapDataset", + # "op_module": "mindspore.dataset.datasets", + pyclass = getattr(sys.modules[op_module], dataset_op) + + pyobj = None + # Find a matching Dataset class and call the constructor with the corresponding args. + # When a new Dataset class is introduced, another if clause and parsing code needs to be added. + if dataset_op == 'StorageDataset': + pyobj = pyclass(node['dataset_files'], node['schema'], node.get('distribution'), + node.get('columns_list'), node.get('num_parallel_workers')) + + elif dataset_op == 'ImageFolderDatasetV2': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'), + node.get('shuffle'), sampler, node.get('extensions'), + node.get('class_indexing'), node.get('decode'), node.get('num_shards'), + node.get('shard_id')) + + elif dataset_op == 'RangeDataset': + pyobj = pyclass(node['start'], node['stop'], node['step']) + + elif dataset_op == 'ImageFolderDataset': + pyobj = pyclass(node['dataset_dir'], node['schema'], node.get('distribution'), + node.get('column_list'), node.get('num_parallel_workers'), + node.get('deterministic_output'), node.get('prefetch_size'), + node.get('labels_filename'), node.get('dataset_usage')) + + elif dataset_op == 'MnistDataset': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'), + node.get('shuffle'), sampler, node.get('num_shards'), node.get('shard_id')) + + elif dataset_op == 'MindDataset': + pyobj = pyclass(node['dataset_file'], node.get('column_list'), + node.get('num_parallel_workers'), node.get('seed'), node.get('num_shards'), + node.get('shard_id'), node.get('block_reader')) + + elif dataset_op == 'TFRecordDataset': + pyobj = pyclass(node['dataset_files'], node.get('schema'), node.get('column_list'), + node.get('num_samples'), node.get('num_parallel_workers'), + de.Shuffle(node.get('shuffle')), node.get('num_shards'), node.get('shard_id')) + + elif dataset_op == 'ManifestDataset': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_file'], node['usage'], node.get('num_samples'), + node.get('num_parallel_workers'), node.get('shuffle'), sampler, + node.get('class_indexing'), node.get('decode'), node.get('num_shards'), + node.get('shard_id')) + + elif dataset_op == 'Cifar10Dataset': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'), + node.get('shuffle'), sampler, node.get('num_shards'), node.get('shard_id')) + + elif dataset_op == 'Cifar100Dataset': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'), + node.get('shuffle'), sampler, node.get('num_shards'), node.get('shard_id')) + + elif dataset_op == 'VOCDataset': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'), + node.get('shuffle'), node.get('decode'), sampler, node.get('distribution')) + + elif dataset_op == 'CelebADataset': + sampler = construct_sampler(node.get('sampler')) + pyobj = pyclass(node['dataset_dir'], node.get('num_parallel_workers'), node.get('shuffle'), + node.get('dataset_type'), sampler, node.get('decode'), node.get('extensions'), + node.get('num_samples'), sampler, node.get('num_shards'), node.get('shard_id')) + + elif dataset_op == 'GeneratorDataset': + # Serializing py function can be done using marshal library + raise RuntimeError(dataset_op + " is not yet supported") + + elif dataset_op == 'RepeatDataset': + pyobj = de.Dataset().repeat(node.get('count')) + + elif dataset_op == 'MapDataset': + tensor_ops = construct_tensor_ops(node.get('operations')) + pyobj = de.Dataset().map(node.get('input_columns'), tensor_ops, node.get('output_columns'), + node.get('columns_order'), node.get('num_parallel_workers')) + + elif dataset_op == 'ShuffleDataset': + pyobj = de.Dataset().shuffle(node.get('buffer_size')) + + elif dataset_op == 'BatchDataset': + pyobj = de.Dataset().batch(node['batch_size'], node.get('drop_remainder')) + + elif dataset_op == 'CacheDataset': + # Member function cache() is not defined in class Dataset yet. + raise RuntimeError(dataset_op + " is not yet supported") + + elif dataset_op == 'FilterDataset': + # Member function filter() is not defined in class Dataset yet. + raise RuntimeError(dataset_op + " is not yet supported") + + elif dataset_op == 'TakeDataset': + # Member function take() is not defined in class Dataset yet. + raise RuntimeError(dataset_op + " is not yet supported") + + elif dataset_op == 'ZipDataset': + # Create ZipDataset instance, giving dummy input dataset that will be overrided in the caller. + pyobj = de.ZipDataset((de.Dataset(), de.Dataset())) + + elif dataset_op == 'RenameDataset': + pyobj = de.Dataset().rename(node['input_columns'], node['output_columns']) + + elif dataset_op == 'ProjectDataset': + pyobj = de.Dataset().project(node['columns']) + + elif dataset_op == 'TransferDataset': + pyobj = de.Dataset().to_device() + + else: + raise RuntimeError(dataset_op + " is not yet supported by ds.engine.deserialize()") + + return pyobj + + +def construct_sampler(in_sampler): + """Instantiate Sampler object based on the information from dictionary['sampler']""" + sampler_name = in_sampler['sampler_name'] + sampler_module = in_sampler['sampler_module'] + sampler_class = getattr(sys.modules[sampler_module], sampler_name) + sampler = None + if sampler_name == 'DistributedSampler': + sampler = sampler_class(in_sampler['num_shards'], in_sampler['shard_id'], in_sampler.get('shuffle')) + elif sampler_name == 'PKSampler': + sampler = sampler_class(in_sampler['num_val'], in_sampler.get('num_class'), in_sampler('shuffle')) + elif sampler_name == 'RandomSampler': + sampler = sampler_class(in_sampler.get('replacement'), in_sampler.get('num_samples')) + elif sampler_name == 'SequentialSampler': + sampler = sampler_class() + elif sampler_name == 'SubsetRandomSampler': + sampler = sampler_class(in_sampler['indices']) + elif sampler_name == 'WeightedRandomSampler': + sampler = sampler_class(in_sampler['weights'], in_sampler['num_samples'], in_sampler.get('replacement')) + else: + raise ValueError("Sampler type is unknown: " + sampler_name) + + return sampler + + +def construct_tensor_ops(operations): + """Instantiate tensor op object(s) based on the information from dictionary['operations']""" + result = [] + for op in operations: + op_module = op['tensor_op_module'] + op_name = op['tensor_op_name'] + op_class = getattr(sys.modules[op_module], op_name) + + if op_name == 'Decode': + result.append(op_class(op.get('rgb'))) + + elif op_name == 'Normalize': + result.append(op_class(op['mean'], op['std'])) + + elif op_name == 'RandomCrop': + result.append(op_class(op['size'], op.get('padding'), op.get('pad_if_needed'), + op.get('fill_value'), Border(op.get('padding_mode')))) + + elif op_name == 'RandomHorizontalFlip': + result.append(op_class(op.get('prob'))) + + elif op_name == 'RandomVerticalFlip': + result.append(op_class(op.get('prob'))) + + elif op_name == 'Resize': + result.append(op_class(op['size'], Inter(op.get('interpolation')))) + + elif op_name == 'RandomResizedCrop': + result.append(op_class(op['size'], op.get('scale'), op.get('ratio'), + Inter(op.get('interpolation')), op.get('max_attempts'))) + + elif op_name == 'CenterCrop': + result.append(op_class(op['size'])) + + elif op_name == 'RandomColorAdjust': + result.append(op_class(op.get('brightness'), op.get('contrast'), op.get('saturation'), + op.get('hue'))) + + elif op_name == 'RandomRotation': + result.append(op_class(op['degree'], op.get('resample'), op.get('expand'), + op.get('center'), op.get('fill_value'))) + + elif op_name == 'Rescale': + result.append(op_class(op['rescale'], op['shift'])) + + elif op_name == 'RandomResize': + result.append(op_class(op['size'])) + + elif op_name == 'TypeCast': + result.append(op_class(op['data_type'])) + + elif op_name == 'HWC2CHW': + result.append(op_class()) + + elif op_name == 'CHW2HWC': + raise ValueError("Tensor op is not supported: " + op_name) + + elif op_name == 'OneHot': + result.append(op_class(op['num_classes'])) + + elif op_name == 'RandomCropDecodeResize': + result.append(op_class(op['size'], op.get('scale'), op.get('ratio'), + Inter(op.get('interpolation')), op.get('max_attempts'))) + + elif op_name == 'Pad': + result.append(op_class(op['padding'], op['fill_value'], Border(op['padding_mode']))) + + else: + raise ValueError("Tensor op name is unknown: " + op_name) + + return result diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py new file mode 100644 index 00000000000..4dc1867808d --- /dev/null +++ b/mindspore/dataset/engine/validators.py @@ -0,0 +1,732 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Built-in validators. +""" +import inspect as ins +import os +from functools import wraps +from multiprocessing import cpu_count +from . import samplers +from . import datasets + +INT32_MAX = 2147483647 + + +def check(method): + """Check the function parameters and return the function .""" + func_name = method.__name__ + # Required parameter + req_param_int = [] + req_param_bool = [] + # Non-required parameter + nreq_param_int = [] + nreq_param_bool = [] + + if func_name in 'repeat': + nreq_param_int = ['count', 'prefetch_size'] + + if func_name in 'take': + req_param_int = ['count'] + nreq_param_int = ['prefetch_size'] + + elif func_name in 'shuffle': + req_param_int = ['buffer_size'] + nreq_param_bool = ['reshuffle_each_iteration'] + nreq_param_int = ['prefetch_size', 'seed'] + + elif func_name in 'batch': + req_param_int = ['batch_size'] + nreq_param_int = ['num_parallel_workers', 'prefetch_size'] + nreq_param_bool = ['drop_remainder'] + + elif func_name in ('zip', 'filter', 'cache', 'rename', 'project'): + nreq_param_int = ['prefetch_size'] + + elif func_name in ('map', '__init__'): + nreq_param_int = ['num_parallel_workers', 'prefetch_size', 'seed'] + nreq_param_bool = ['block_reader'] + + @wraps(method) + def wrapper(*args, **kwargs): + + def _make_key(): + sig = ins.signature(method) + params = sig.parameters + keys = list(params.keys()) + param_dic = dict() + for name, value in enumerate(args): + param_dic[keys[name]] = value + param_dic.update(zip(params.keys(), args)) + param_dic.update(kwargs) + + for name, value in params.items(): + if name not in param_dic: + param_dic[name] = value.default + return param_dic + + # check type + def _check_param_type(arg, param_name, param_type=None): + if param_type is not None and not isinstance(arg, param_type): + raise ValueError( + "The %s function %s type error!" % (func_name, param_name)) + + # check range + def _check_param_range(arg, param_name): + if isinstance(arg, int) and param_name == "seed" and ( + arg < 0 or arg > 2147483647): + raise ValueError( + "The %s function %s exceeds the boundary!" % ( + func_name, param_name)) + if isinstance(arg, int) and param_name == "count" and ((arg <= 0 and arg != -1) or arg > 2147483647): + raise ValueError( + "The %s function %s exceeds the boundary!" % ( + func_name, param_name)) + if isinstance(arg, int) and param_name == "prefetch_size" and ( + arg <= 0 or arg > 1024): + raise ValueError( + "The %s function %s exceeds the boundary!" % ( + func_name, param_name)) + if isinstance(arg, int) and param_name == "num_parallel_workers" and ( + arg <= 0 or arg > cpu_count()): + raise ValueError( + "The %s function %s exceeds the boundary(%s)!" % ( + func_name, param_name, cpu_count())) + if isinstance(arg, int) and param_name != "seed" \ + and param_name != "count" and param_name != "prefetch_size" \ + and param_name != "num_parallel_workers" and (arg <= 0 or arg > 2147483647): + raise ValueError( + "The %s function %s exceeds the boundary!" % ( + func_name, param_name)) + + key = _make_key() + # check integer + for karg in req_param_int: + _check_param_type(key[karg], karg, int) + _check_param_range(key[karg], karg) + for karg in nreq_param_int: + if karg in key: + if key[karg] is not None: + _check_param_type(key[karg], karg, int) + _check_param_range(key[karg], karg) + # check bool + for karg in req_param_bool: + _check_param_type(key[karg], karg, bool) + for karg in nreq_param_bool: + if karg in key: + if key[karg] is not None: + _check_param_type(key[karg], karg, bool) + + if func_name in '__init__': + if 'columns_list' in key.keys(): + columns_list = key['columns_list'] + if columns_list is not None: + _check_param_type(columns_list, 'columns_list', list) + + if 'columns' in key.keys(): + columns = key['columns'] + if columns is not None: + _check_param_type(columns, 'columns', list) + + if 'partitions' in key.keys(): + partitions = key['partitions'] + if partitions is not None: + _check_param_type(partitions, 'partitions', list) + + if 'schema' in key.keys(): + schema = key['schema'] + if schema is not None: + check_filename(schema) + if not os.path.isfile(schema) or not os.access(schema, os.R_OK): + raise ValueError( + "The file %s does not exist or permission denied!" % schema) + + if 'dataset_dir' in key.keys(): + dataset_dir = key['dataset_dir'] + if dataset_dir is not None: + if not os.path.isdir(dataset_dir) or not os.access(dataset_dir, os.R_OK): + raise ValueError( + "The folder %s does not exist or permission denied!" % dataset_dir) + + if 'dataset_files' in key.keys(): + dataset_files = key['dataset_files'] + if not dataset_files: + raise ValueError( + "The dataset file does not exists!") + if dataset_files is not None: + _check_param_type(dataset_files, 'dataset_files', list) + for file in dataset_files: + if not os.path.isfile(file) or not os.access(file, os.R_OK): + raise ValueError( + "The file %s does not exist or permission denied!" % file) + + if 'dataset_file' in key.keys(): + dataset_file = key['dataset_file'] + if not dataset_file: + raise ValueError( + "The dataset file does not exists!") + check_filename(dataset_file) + if dataset_file is not None: + if not os.path.isfile(dataset_file) or not os.access(dataset_file, os.R_OK): + raise ValueError( + "The file %s does not exist or permission denied!" % dataset_file) + + return method(*args, **kwargs) + + return wrapper + + +def check_filename(path): + """ + check the filename in the path + + Args: + path (str): the path + + Returns: + Exception: when error + """ + if not isinstance(path, str): + raise ValueError("path: {} is not string".format(path)) + filename = os.path.basename(path) + + # '#', ':', '|', ' ', '}', '"', '+', '!', ']', '[', '\\', '`', + # '&', '.', '/', '@', "'", '^', ',', '_', '<', ';', '~', '>', + # '*', '(', '%', ')', '-', '=', '{', '?', '$' + forbidden_symbols = set(r'\/:*?"<>|`&\';') + + if set(filename) & forbidden_symbols: + raise ValueError(r"filename should not contains \/:*?\"<>|`&;\'") + + if filename.startswith(' ') or filename.endswith(' '): + raise ValueError("filename should not start/end with space") + + return True + + +def make_param_dict(method, args, kwargs): + """Return a dictionary of the method's args and kwargs.""" + sig = ins.signature(method) + params = sig.parameters + keys = list(params.keys()) + param_dict = dict() + for name, value in enumerate(args): + param_dict[keys[name]] = value + param_dict.update(zip(params.keys(), args)) + param_dict.update(kwargs) + + for name, value in params.items(): + if name not in param_dict: + param_dict[name] = value.default + return param_dict + + +def check_type(param, param_name, valid_type): + if (not isinstance(param, valid_type)) or (valid_type == int and isinstance(param, bool)): + raise TypeError("Wrong input type for {0}, should be {1}, got {2}".format(param_name, valid_type, type(param))) + + +def check_param_type(param_list, param_dict, param_type): + for param_name in param_list: + if param_dict.get(param_name) is not None: + if param_name == 'num_parallel_workers': + check_num_parallel_workers(param_dict.get(param_name)) + else: + check_type(param_dict.get(param_name), param_name, param_type) + + +def check_positive_int32(param, param_name): + check_interval_closed(param, param_name, [1, INT32_MAX]) + + +def check_interval_closed(param, param_name, valid_range): + if param < valid_range[0] or param > valid_range[1]: + raise ValueError("The value of {0} exceeds the closed interval range {1}.".format(param_name, valid_range)) + + +def check_num_parallel_workers(value): + check_type(value, 'num_parallel_workers', int) + if value <= 0 or value > cpu_count(): + raise ValueError("num_parallel_workers exceeds the boundary between 0 and {}!".format(cpu_count())) + + +def check_dataset_dir(dataset_dir): + if not os.path.isdir(dataset_dir) or not os.access(dataset_dir, os.R_OK): + raise ValueError("The folder {} does not exist or permission denied!".format(dataset_dir)) + + +def check_dataset_file(dataset_file): + check_filename(dataset_file) + if not os.path.isfile(dataset_file) or not os.access(dataset_file, os.R_OK): + raise ValueError("The file {} does not exist or permission denied!".format(dataset_file)) + + +def check_sampler_shuffle_shard_options(param_dict): + """check for valid shuffle, sampler, num_shards, and shard_id inputs.""" + shuffle, sampler = param_dict.get('shuffle'), param_dict.get('sampler') + num_shards, shard_id = param_dict.get('num_shards'), param_dict.get('shard_id') + + if sampler is not None and not isinstance(sampler, ( + samplers.DistributedSampler, samplers.PKSampler, samplers.RandomSampler, samplers.SequentialSampler, + samplers.SubsetRandomSampler, samplers.WeightedRandomSampler)): + raise ValueError("sampler is not a valid Sampler type.") + + if sampler is not None: + if shuffle is not None: + raise RuntimeError("sampler and shuffle cannot be specified at the same time.") + + if num_shards is not None: + raise RuntimeError("sampler and sharding cannot be specified at the same time.") + + if num_shards is not None: + if shard_id is None: + raise RuntimeError("num_shards is specified and currently requires shard_id as well.") + if shard_id < 0 or shard_id >= num_shards: + raise ValueError("shard_id is invalid, shard_id={}".format(shard_id)) + + if num_shards is None and shard_id is not None: + raise RuntimeError("shard_id is specified but num_shards is not.") + + +def check_imagefolderdatasetv2(method): + """A wrapper that wrap a parameter checker to the original Dataset(ImageFolderDatasetV2).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_bool = ['shuffle', 'decode'] + nreq_param_list = ['extensions'] + nreq_param_dict = ['class_indexing'] + + # check dataset_dir; required argument + dataset_dir = param_dict.get('dataset_dir') + if dataset_dir is None: + raise ValueError("dataset_dir is not provided.") + check_dataset_dir(dataset_dir) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_bool, param_dict, bool) + + check_param_type(nreq_param_list, param_dict, list) + + check_param_type(nreq_param_dict, param_dict, dict) + + check_sampler_shuffle_shard_options(param_dict) + + return method(*args, **kwargs) + + return new_method + + +def check_mnist_cifar_dataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(ManifestDataset, Cifar10/100Dataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_bool = ['shuffle'] + + # check dataset_dir; required argument + dataset_dir = param_dict.get('dataset_dir') + if dataset_dir is None: + raise ValueError("dataset_dir is not provided.") + check_dataset_dir(dataset_dir) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_bool, param_dict, bool) + + check_sampler_shuffle_shard_options(param_dict) + + return method(*args, **kwargs) + + return new_method + + +def check_manifestdataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(ManifestDataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_bool = ['shuffle', 'decode'] + nreq_param_str = ['usage'] + nreq_param_dict = ['class_indexing'] + + # check dataset_file; required argument + dataset_file = param_dict.get('dataset_file') + if dataset_file is None: + raise ValueError("dataset_file is not provided.") + check_dataset_file(dataset_file) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_bool, param_dict, bool) + + check_param_type(nreq_param_str, param_dict, str) + + check_param_type(nreq_param_dict, param_dict, dict) + + check_sampler_shuffle_shard_options(param_dict) + + return method(*args, **kwargs) + + return new_method + + +def check_tfrecorddataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(TFRecordDataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_list = ['columns_list'] + + # check dataset_files; required argument + dataset_files = param_dict.get('dataset_files') + if dataset_files is None: + raise ValueError("dataset_files is not provided.") + if not isinstance(dataset_files, (str, list)): + raise TypeError("dataset_files should be of type str or a list of strings.") + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_list, param_dict, list) + + return method(*args, **kwargs) + + return new_method + + +def check_vocdataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(VOCDataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers'] + nreq_param_bool = ['shuffle', 'decode'] + nreq_param_str = ['distribution'] + + # check dataset_dir; required argument + dataset_dir = param_dict.get('dataset_dir') + if dataset_dir is None: + raise ValueError("dataset_dir is not provided.") + check_dataset_dir(dataset_dir) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_bool, param_dict, bool) + + check_param_type(nreq_param_str, param_dict, str) + + return method(*args, **kwargs) + + return new_method + + +def check_celebadataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(CelebADataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_bool = ['shuffle', 'decode'] + nreq_param_list = ['extensions'] + nreq_param_str = ['dataset_type'] + + # check dataset_dir; required argument + dataset_dir = param_dict.get('dataset_dir') + if dataset_dir is None: + raise ValueError("dataset_dir is not provided.") + check_dataset_dir(dataset_dir) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_bool, param_dict, bool) + + check_param_type(nreq_param_list, param_dict, list) + + check_param_type(nreq_param_str, param_dict, str) + + dataset_type = param_dict.get('dataset_type') + if dataset_type is not None and dataset_type not in ('all', 'train', 'valid', 'test'): + raise ValueError("dataset_type should be one of 'all', 'train', 'valid' or 'test'.") + + check_sampler_shuffle_shard_options(param_dict) + + sampler = param_dict.get('sampler') + if sampler is not None and isinstance(sampler, samplers.PKSampler): + raise ValueError("CelebADataset does not support PKSampler.") + + return method(*args, **kwargs) + + return new_method + + +def check_minddataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(MindDataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'seed', 'num_shards', 'shard_id'] + nreq_param_list = ['columns_list'] + nreq_param_bool = ['block_reader'] + + # check dataset_file; required argument + dataset_file = param_dict.get('dataset_file') + if dataset_file is None: + raise ValueError("dataset_file is not provided.") + check_dataset_file(dataset_file) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_list, param_dict, list) + + check_param_type(nreq_param_bool, param_dict, bool) + + num_shards, shard_id = param_dict.get('num_shards'), param_dict.get('shard_id') + if (num_shards is not None and shard_id is None) or (num_shards is None and shard_id is not None): + raise ValueError("num_shards and shard_id need to be set or not set at the same time") + + return method(*args, **kwargs) + + return new_method + + +def check_generatordataset(method): + """A wrapper that wrap a parameter checker to the original Dataset(GeneratorDataset).""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['prefetch_size'] + nreq_param_list = ['column_names', 'column_types'] + + # check generator_function; required argument + generator_function = param_dict.get('generator_function') + if generator_function is None: + raise ValueError("generator_function is not provided.") + + # check column_names; required argument + column_names = param_dict.get('column_names') + if column_names is None: + raise ValueError("column_names is not provided.") + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_list, param_dict, list) + + return method(*args, **kwargs) + + return new_method + + +def check_batch_size(batch_size): + if not (isinstance(batch_size, int) or (callable(batch_size))): + raise ValueError("batch_size should either be an int or a callable.") + if callable(batch_size): + sig = ins.signature(batch_size) + if len(sig.parameters) != 1: + raise ValueError("batch_size callable should take one parameter (BatchInfo).") + + +def check_count(count): + check_type(count, 'count', int) + if (count <= 0 and count != -1) or count > INT32_MAX: + raise ValueError("repeat count should be either -1 or positive integer.") + + +def check_columns(columns, name): + if isinstance(columns, list): + for column in columns: + if not isinstance(column, str): + raise TypeError("Each column in {0} should be of type str. Got {1}.".format(name, type(column))) + elif not isinstance(columns, str): + raise TypeError("{} should be either a list of strings or a single string.".format(name)) + + +def check_batch(method): + """check the input arguments of batch.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_int = ['num_parallel_workers'] + nreq_param_bool = ['drop_remainder'] + nreq_param_columns = ['input_columns'] + + # check batch_size; required argument + batch_size = param_dict.get("batch_size") + if batch_size is None: + raise ValueError("batch_size is not provided.") + check_batch_size(batch_size) + + check_param_type(nreq_param_int, param_dict, int) + + check_param_type(nreq_param_bool, param_dict, bool) + + for param_name in nreq_param_columns: + param = param_dict.get(param_name) + if param is not None: + check_columns(param, param_name) + + per_batch_map, input_columns = param_dict.get('per_batch_map'), param_dict.get('input_columns') + if (per_batch_map is None) != (input_columns is None): + # These two parameters appear together. + raise ValueError("per_batch_map and input_columns need to be passed in together.") + + if input_columns is not None: + if not input_columns: # Check whether input_columns is empty. + raise ValueError("input_columns can not be empty") + if len(input_columns) != (len(ins.signature(per_batch_map).parameters) - 1): + raise ValueError("the signature of per_batch_map should match with input columns") + + return method(*args, **kwargs) + + return new_method + + +def check_shuffle(method): + """check the input arguments of shuffle.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + # check buffer_size; required argument + buffer_size = param_dict.get("buffer_size") + if buffer_size is None: + raise ValueError("buffer_size is not provided.") + check_type(buffer_size, 'buffer_size', int) + check_interval_closed(buffer_size, 'buffer_size', [2, INT32_MAX]) + + return method(*args, **kwargs) + + return new_method + + +def check_map(method): + """check the input arguments of map.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + nreq_param_list = ['columns_order'] + nreq_param_int = ['num_parallel_workers'] + nreq_param_columns = ['input_columns', 'output_columns'] + + check_param_type(nreq_param_list, param_dict, list) + check_param_type(nreq_param_int, param_dict, int) + for param_name in nreq_param_columns: + param = param_dict.get(param_name) + if param is not None: + check_columns(param, param_name) + + return method(*args, **kwargs) + + return new_method + + +def check_repeat(method): + """check the input arguments of repeat.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + count = param_dict.get('count') + if count is not None: + check_count(count) + + return method(*args, **kwargs) + + return new_method + + +def check_zip(method): + """check the input arguments of zip.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + # check datasets; required argument + ds = param_dict.get("datasets") + if ds is None: + raise ValueError("datasets is not provided.") + check_type(ds, 'datasets', tuple) + + return method(*args, **kwargs) + + return new_method + + +def check_zip_dataset(method): + """check the input arguments of zip method in `Dataset`.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + # check datasets; required argument + ds = param_dict.get("datasets") + if ds is None: + raise ValueError("datasets is not provided.") + + if not isinstance(ds, (tuple, datasets.Dataset)): + raise ValueError("datasets is not tuple or of type Dataset.") + + return method(*args, **kwargs) + + return new_method + +def check_rename(method): + """check the input arguments of rename.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + req_param_columns = ['input_columns', 'output_columns'] + # check req_param_list; required arguments + for param_name in req_param_columns: + param = param_dict.get(param_name) + if param is None: + raise ValueError("{} is not provided.".format(param_name)) + check_columns(param, param_name) + + return method(*args, **kwargs) + + return new_method + + +def check_project(method): + """check the input arguments of project.""" + @wraps(method) + def new_method(*args, **kwargs): + param_dict = make_param_dict(method, args, kwargs) + + # check columns; required argument + columns = param_dict.get("columns") + if columns is None: + raise ValueError("columns is not provided.") + check_columns(columns, 'columns') + + return method(*args, **kwargs) + + return new_method diff --git a/mindspore/dataset/transforms/__init__.py b/mindspore/dataset/transforms/__init__.py new file mode 100644 index 00000000000..ffca5fb1785 --- /dev/null +++ b/mindspore/dataset/transforms/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module is to support common augmentations. C_transforms is a high performance +image augmentation module which is developed with c++ opencv. Py_transforms +provide more kinds of image augmentations which is developed with python PIL. +""" +from . import vision +from . import c_transforms +from . import py_transforms diff --git a/mindspore/dataset/transforms/c_transforms.py b/mindspore/dataset/transforms/c_transforms.py new file mode 100644 index 00000000000..91fb4865317 --- /dev/null +++ b/mindspore/dataset/transforms/c_transforms.py @@ -0,0 +1,50 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module c_transforms provides common operations, including OneHotOp and TypeCast. +""" +import mindspore._c_dataengine as cde + +from .validators import check_num_classes, check_de_type +from ..core.datatypes import mstype_to_detype + + +class OneHot(cde.OneHotOp): + """ + Tensor operation to apply one hot encoding. + + Args: + num_classes (int): Number of classes of the label. + """ + + @check_num_classes + def __init__(self, num_classes): + self.num_classes = num_classes + super().__init__(num_classes) + + +class TypeCast(cde.TypeCastOp): + """ + Tensor operation to cast to a given MindSpore data type. + + Args: + data_type (mindspore.dtype): mindspore.dtype to be casted to. + """ + + @check_de_type + def __init__(self, data_type): + data_type = mstype_to_detype(data_type) + self.data_type = str(data_type) + super().__init__(data_type) diff --git a/mindspore/dataset/transforms/py_transforms.py b/mindspore/dataset/transforms/py_transforms.py new file mode 100644 index 00000000000..9b5c7eb3bb4 --- /dev/null +++ b/mindspore/dataset/transforms/py_transforms.py @@ -0,0 +1,49 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +""" +This module py_transforms is implemented basing on python. It provides common +operations including OneHotOp. +""" + +from .validators import check_one_hot_op +from .vision import py_transforms_util as util + + +class OneHotOp: + """ + Apply one hot encoding transformation to the input label, make label be more smoothing and continuous. + + Args: + num_classes (int): Num class of object in dataset, type is int and value over 0. + smoothing_rate (float): The adjustable Hyper parameter decides the label smoothing level , 0.0 means not do it. + """ + + @check_one_hot_op + def __init__(self, num_classes, smoothing_rate=0.0): + self.num_classes = num_classes + self.smoothing_rate = smoothing_rate + + def __call__(self, label): + """ + Call method. + + Args: + label (numpy.ndarray): label to be applied label smoothing. + + Returns: + label (numpy.ndarray), label after being Smoothed. + """ + return util.one_hot_encoding(label, self.num_classes, self.smoothing_rate) diff --git a/mindspore/dataset/transforms/validators.py b/mindspore/dataset/transforms/validators.py new file mode 100644 index 00000000000..5572e5285e3 --- /dev/null +++ b/mindspore/dataset/transforms/validators.py @@ -0,0 +1,179 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Validators for TensorOps. +""" +from functools import wraps +from mindspore._c_expression import typing + + +# POS_INT_MIN is used to limit values from starting from 0 +POS_INT_MIN = 1 +UINT8_MAX = 255 +UINT8_MIN = 0 +UINT32_MAX = 4294967295 +UINT32_MIN = 0 +UINT64_MAX = 18446744073709551615 +UINT64_MIN = 0 +INT32_MAX = 2147483647 +INT32_MIN = -2147483648 +INT64_MAX = 9223372036854775807 +INT64_MIN = -9223372036854775808 +FLOAT_MAX_INTEGER = 16777216 +FLOAT_MIN_INTEGER = -16777216 +DOUBLE_MAX_INTEGER = 9007199254740992 +DOUBLE_MIN_INTEGER = -9007199254740992 + + +def check_type(value, valid_type): + if not isinstance(value, valid_type): + raise ValueError("Wrong input type") + + +def check_value(value, valid_range): + if value < valid_range[0] or value > valid_range[1]: + raise ValueError("Input is not within the required range") + + +def check_range(values, valid_range): + if not valid_range[0] <= values[0] <= values[1] <= valid_range[1]: + raise ValueError("Input range is not valid") + + +def check_positive(value): + if value <= 0: + raise ValueError("Input must greater than 0") + + +def check_positive_float(value, valid_max=None): + if value <= 0 or not isinstance(value, float) or (valid_max is not None and value > valid_max): + raise ValueError("Input need to be a valid positive float.") + + +def check_bool(value): + if not isinstance(value, bool): + raise ValueError("Value needs to be a boolean.") + + +def check_2tuple(value): + if not (isinstance(value, tuple) and len(value) == 2): + raise ValueError("Value needs to be a 2-tuple.") + + +def check_list(value): + if not isinstance(value, list): + raise ValueError("The input needs to be a list.") + + +def check_uint8(value): + if not isinstance(value, int): + raise ValueError("The input needs to be a integer") + check_value(value, [UINT8_MIN, UINT8_MAX]) + + +def check_uint32(value): + if not isinstance(value, int): + raise ValueError("The input needs to be a integer") + check_value(value, [UINT32_MIN, UINT32_MAX]) + + +def check_pos_int32(value): + """Checks for int values starting from 1""" + if not isinstance(value, int): + raise ValueError("The input needs to be a integer") + check_value(value, [POS_INT_MIN, INT32_MAX]) + + +def check_uint64(value): + if not isinstance(value, int): + raise ValueError("The input needs to be a integer") + check_value(value, [UINT64_MIN, UINT64_MAX]) + + +def check_pos_int64(value): + if not isinstance(value, int): + raise ValueError("The input needs to be a integer") + check_value(value, [UINT64_MIN, INT64_MAX]) + + +def check_pos_float32(value): + check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER]) + + +def check_pos_float64(value): + check_value(value, [UINT64_MIN, DOUBLE_MAX_INTEGER]) + + +def check_one_hot_op(method): + """Wrapper method to check the parameters of one hot op.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + num_classes, smoothing_rate = args + if "num_classes" in kwargs: + num_classes = kwargs.get("num_classes") + if "smoothing_rate" in kwargs: + smoothing_rate = kwargs.get("smoothing_rate") + + if num_classes is None: + raise ValueError("num_classes") + check_pos_int32(num_classes) + kwargs["num_classes"] = num_classes + if smoothing_rate is not None: + check_value(smoothing_rate, [0., 1.]) + kwargs["smoothing_rate"] = smoothing_rate + + return method(self, **kwargs) + + return new_method + + +def check_num_classes(method): + """Wrapper method to check the parameters of number of classes.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + num_classes = (list(args) + [None])[0] + if "num_classes" in kwargs: + num_classes = kwargs.get("num_classes") + if num_classes is None: + raise ValueError("num_classes is not provided.") + + check_pos_int32(num_classes) + kwargs["num_classes"] = num_classes + + return method(self, **kwargs) + + return new_method + + +def check_de_type(method): + """Wrapper method to check the parameters of data type.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + data_type = (list(args) + [None])[0] + if "data_type" in kwargs: + data_type = kwargs.get("data_type") + + if data_type is None: + raise ValueError("data_type is not provided.") + if not isinstance(data_type, typing.Type): + raise TypeError("data_type is not a MindSpore data type.") + kwargs["data_type"] = data_type + + return method(self, **kwargs) + + return new_method diff --git a/mindspore/dataset/transforms/vision/__init__.py b/mindspore/dataset/transforms/vision/__init__.py new file mode 100644 index 00000000000..14b5e00faec --- /dev/null +++ b/mindspore/dataset/transforms/vision/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module is to support vision augmentations. It includes two parts: +c_transforms and py_transforms. C_transforms is a high performance +image augmentation module which is developed with c++ opencv. Py_transforms +provide more kinds of image augmentations which is developed with python PIL. +""" +from . import c_transforms +from . import py_transforms +from .utils import Inter, Border diff --git a/mindspore/dataset/transforms/vision/c_transforms.py b/mindspore/dataset/transforms/vision/c_transforms.py new file mode 100644 index 00000000000..171eb846a80 --- /dev/null +++ b/mindspore/dataset/transforms/vision/c_transforms.py @@ -0,0 +1,449 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The module vision.c_transforms is inheritted from _c_dataengine +which is implemented basing on opencv in C++. It's a high performance module to +process image augmentations. Users can apply suitable augmentations on image data +to improve their training models. + +.. Note:: + Constructor's arguments for every class in this module must be saved into the + class attributes (self.xxx) to support save() and load(). + +Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.dataset.transforms.c_transforms as c_transforms + >>> import mindspore.dataset.transforms.vision.c_transforms as vision + >>> dataset_dir = "path/to/imagefolder_directory" + >>> # create a dataset that reads all files in dataset_dir with 8 threads + >>> dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8) + >>> # create a list of transformations to be applied to the image data + >>> transforms_list = [vision.Decode(), + >>> vision.Resize((256, 256)), + >>> vision.RandomRotation((0, 15)), + >>> vision.Normalize((100, 115.0, 121.0), (71.0, 68.0, 70.0)), + >>> vision.HWC2CHW()] + >>> onehot_op = c_transforms.OneHot(num_classes) + >>> # apply the transform to the dataset through dataset.map() + >>> dataset = dataset.map(input_columns="image", operations=transforms_list) + >>> dataset = dataset.map(input_columns="label", operations=onehot_op) +""" +import mindspore._c_dataengine as cde + +from .utils import Inter, Border +from .validators import check_prob, check_crop, check_resize_interpolation, check_random_resize_crop, \ + check_normalize_c, check_random_crop, check_random_color_adjust, check_random_rotation, \ + check_resize, check_rescale, check_pad, check_cutout + +DE_C_INTER_MODE = {Inter.NEAREST: cde.InterpolationMode.DE_INTER_NEAREST_NEIGHBOUR, + Inter.LINEAR: cde.InterpolationMode.DE_INTER_LINEAR, + Inter.CUBIC: cde.InterpolationMode.DE_INTER_CUBIC} + +DE_C_BORDER_TYPE = {Border.CONSTANT: cde.BorderType.DE_BORDER_CONSTANT, + Border.EDGE: cde.BorderType.DE_BORDER_EDGE, + Border.REFLECT: cde.BorderType.DE_BORDER_REFLECT, + Border.SYMMETRIC: cde.BorderType.DE_BORDER_SYMMETRIC} + + +class Decode(cde.DecodeOp): + """ + Decode the input image in RGB mode. + """ + + def __init__(self, rgb=True): + self.rgb = rgb + super().__init__(self.rgb) + + +class CutOut(cde.CutOutOp): + """ + Randomly cut (mask) out a given number of square patches from the input Numpy image array. + + Args: + length (int): The side length of each square patch. + num_patches (int, optional): Number of patches to be cut out of an image (default=1). + """ + + @check_cutout + def __init__(self, length, num_patches=1): + self.length = length + self.num_patches = num_patches + fill_value = (0, 0, 0) + super().__init__(length, length, num_patches, False, *fill_value) + + +class Normalize(cde.NormalizeOp): + """ + Normalize the input image with respect to mean and standard deviation. + + Args: + mean (list): List of mean values for each channel, w.r.t channel order. + std (list): List of standard deviations for each channel, w.r.t. channel order. + """ + + @check_normalize_c + def __init__(self, mean, std): + self.mean = mean + self.std = std + super().__init__(*mean, *std) + + +class RandomCrop(cde.RandomCropOp): + """ + Crop the input image at a random location. + + Args: + size (int or sequence): The output size of the cropped image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + padding (int or sequence, optional): The number of pixels to pad the image (default=None). + If a single number is provided, it pads all borders with this value. + If a tuple or list of 2 values are provided, it pads the (left and top) + with the first value and (right and bottom) with the second value. + If 4 values are provided as a list or tuple, + it pads the left, top, right and bottom respectively. + pad_if_needed (bool, optional): Pad the image if either side is smaller than + the given output size (default=False). + fill_value (int or tuple, optional): The pixel intensity of the borders if + the padding_mode is Border.CONSTANT (default=0). If it is a 3-tuple, it is used to + fill R, G, B channels respectively. + padding_mode (Border mode, optional): The method of padding (default=Border.CONSTANT). Can be any of + [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. + + - Border.CONSTANT, means it fills the border with constant values. + + - Border.EDGE, means it pads with the last value on the edge. + + - Border.REFLECT, means it reflects the values on the edge omitting the last + value of edge. + + - Border.SYMMETRIC, means it reflects the values on the edge repeating the last + value of edge. + """ + + @check_random_crop + def __init__(self, size, padding=None, pad_if_needed=False, fill_value=0, padding_mode=Border.CONSTANT): + self.size = size + self.padding = padding + self.pad_if_needed = pad_if_needed + self.fill_value = fill_value + self.padding_mode = padding_mode.value + if padding is None: + padding = (0, 0, 0, 0) + if isinstance(fill_value, int): # temporary fix + fill_value = tuple([fill_value] * 3) + border_type = DE_C_BORDER_TYPE[padding_mode] + super().__init__(*size, *padding, border_type, pad_if_needed, *fill_value) + + +class RandomHorizontalFlip(cde.RandomHorizontalFlipOp): + """ + Flip the input image horizontally, randomly with a given probability. + + Args: + prob (float): Probability of the image being flipped (default=0.5). + """ + + @check_prob + def __init__(self, prob=0.5): + self.prob = prob + super().__init__(prob) + + +class RandomVerticalFlip(cde.RandomVerticalFlipOp): + """ + Flip the input image vertically, randomly with a given probability. + + Args: + prob (float): Probability of the image being flipped (default=0.5). + """ + + @check_prob + def __init__(self, prob=0.5): + self.prob = prob + super().__init__(prob) + + +class Resize(cde.ResizeOp): + """ + Resize the input image to the given size. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.LINEAR). + It can be any of [Inter.LINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.LINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + """ + + @check_resize_interpolation + def __init__(self, size, interpolation=Inter.LINEAR): + self.size = size + self.interpolation = interpolation + interpoltn = DE_C_INTER_MODE[interpolation] + if isinstance(size, int): + super().__init__(size, interpolation=interpoltn) + else: + super().__init__(*size, interpoltn) + + +class RandomResizedCrop(cde.RandomCropAndResizeOp): + """ + Crop the input image to a random size and aspect ratio. + + Args: + size (int or sequence): The size of the output image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + scale (tuple, optional): Range (min, max) of respective size of the original + size to be cropped (default=(0.08, 1.0)). + ratio (tuple, optional): Range (min, max) of aspect ratio to be cropped + (default=(3. / 4., 4. / 3.)). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + + max_attempts (int, optional): The maximum number of attempts to propose a valid + crop_area (default=10). If exceeded, fall back to use center_crop instead. + """ + + @check_random_resize_crop + def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), + interpolation=Inter.BILINEAR, max_attempts=10): + self.size = size + self.scale = scale + self.ratio = ratio + self.interpolation = interpolation + self.max_attempts = max_attempts + interpoltn = DE_C_INTER_MODE[interpolation] + super().__init__(*size, *scale, *ratio, interpoltn, max_attempts) + + +class CenterCrop(cde.CenterCropOp): + """ + Crops the input image at the center to the given size. + + Args: + size (int or sequence): The output size of the cropped image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + """ + + @check_crop + def __init__(self, size): + self.size = size + super().__init__(*size) + + +class RandomColorAdjust(cde.RandomColorAdjustOp): + """ + Randomly adjust the brightness, contrast, saturation, and hue of the input image. + + Args: + brightness (float or tuple, optional): Brightness adjustment factor (default=(1, 1)). Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-brightness), 1+brightness]. + If it is a sequence, it should be [min, max] for the range. + contrast (float or tuple, optional): Contrast adjustment factor (default=(1, 1)). Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-contrast), 1+contrast]. + If it is a sequence, it should be [min, max] for the range. + saturation (float or tuple, optional): Saturation adjustment factor (default=(1, 1)). Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-saturation), 1+saturation]. + If it is a sequence, it should be [min, max] for the range. + hue (float or tuple, optional): Hue adjustment factor (default=(0, 0)). + If it is a float, the range will be [-hue, hue]. Value should be 0 <= hue <= 0.5. + If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5. + """ + + @check_random_color_adjust + def __init__(self, brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0)): + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + super().__init__(*brightness, *contrast, *saturation, *hue) + + +class RandomRotation(cde.RandomRotationOp): + """ + Rotate the input image by a random angle. + + Args: + degrees (int or float or sequence): Range of random rotation degrees. + If degrees is a number, the range will be converted to (-degrees, degrees). + If degrees is a sequence, it should be (min, max). + resample (Inter mode, optional): An optional resampling filter (default=Inter.NEAREST). + If omitted, or if the image has mode "1" or "P", it is set to be Inter.NEAREST. + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means resample method is bilinear interpolation. + + - Inter.NEAREST, means resample method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means resample method is bicubic interpolation. + + expand (bool, optional): Optional expansion flag (default=False). If set to True, expand the output + image to make it large enough to hold the entire rotated image. + If set to False or omitted, make the output image the same size as the input. + Note that the expand flag assumes rotation around the center and no translation. + center (tuple, optional): Optional center of rotation (a 2-tuple) (default=None). + Origin is the top left corner. None sets to the center of the image. + fill_value (int or tuple, optional): Optional fill color for the area outside the rotated image (default=0). + If it is a 3-tuple, it is used for R, G, B channels respectively. + If it is an int, it is used for all RGB channels. + """ + + @check_random_rotation + def __init__(self, degrees, resample=Inter.NEAREST, expand=False, center=None, fill_value=0): + self.degrees = degrees + self.resample = resample + self.expand = expand + self.center = center + self.fill_value = fill_value + if center is None: + center = (-1, -1) + if isinstance(fill_value, int): # temporary fix + fill_value = tuple([fill_value] * 3) + interpolation = DE_C_INTER_MODE[resample] + super().__init__(*degrees, *center, interpolation, expand, *fill_value) + + +class Rescale(cde.RescaleOp): + """ + Tensor operation to rescale the input image. + + Args: + rescale (float): Rescale factor. + shift (float): Shift factor. + """ + + @check_rescale + def __init__(self, rescale, shift): + self.rescale = rescale + self.shift = shift + super().__init__(rescale, shift) + + +class RandomResize(cde.RandomResizeOp): + """ + Tensor operation to resize the input image using a randomly selected interpolation mode. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + """ + + @check_resize + def __init__(self, size): + self.size = size + if isinstance(size, int): + super().__init__(size) + else: + super().__init__(*size) + + +class HWC2CHW(cde.ChannelSwapOp): + """ + Transpose the input image; shape (H, W, C) to shape (C, H, W). + """ + + +class RandomCropDecodeResize(cde.RandomCropDecodeResizeOp): + """ + Equivalent to RandomResizedCrop, but crops before decodes. + + Args: + size (int or sequence, optional): The size of the output image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + scale (tuple, optional): Range (min, max) of respective size of the + original size to be cropped (default=(0.08, 1.0)). + ratio (tuple, optional): Range (min, max) of aspect ratio to be + cropped (default=(3. / 4., 4. / 3.)). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + + max_attempts (int, optional): The maximum number of attempts to propose a valid crop_area (default=10). + If exceeded, fall back to use center_crop instead. + """ + + @check_random_resize_crop + def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), + interpolation=Inter.BILINEAR, max_attempts=10): + self.size = size + self.scale = scale + self.ratio = ratio + self.interpolation = interpolation + self.max_attempts = max_attempts + interpoltn = DE_C_INTER_MODE[interpolation] + super().__init__(*size, *scale, *ratio, interpoltn, max_attempts) + + +class Pad(cde.PadOp): + """ + Pads the image according to padding parameters. + + Args: + padding (int or sequence): The number of pixels to pad the image. + If a single number is provided, it pads all borders with this value. + If a tuple or list of 2 values are provided, it pads the (left and top) + with the first value and (right and bottom) with the second value. + If 4 values are provided as a list or tuple, + it pads the left, top, right and bottom respectively. + fill_value (int or tuple, optional): The pixel intensity of the borders if + the padding_mode is Border.CONSTANT (default=0). If it is a 3-tuple, it is used to + fill R, G, B channels respectively. + padding_mode (Border mode): The method of padding (default=Border.CONSTANT). Can be any of + [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. + + - Border.CONSTANT, means it fills the border with constant values. + + - Border.EDGE, means it pads with the last value on the edge. + + - Border.REFLECT, means it reflects the values on the edge omitting the last + value of edge. + + - Border.SYMMETRIC, means it reflects the values on the edge repeating the last + value of edge. + """ + + @check_pad + def __init__(self, padding, fill_value=0, padding_mode=Border.CONSTANT): + self.padding = padding + self.fill_value = fill_value + self.padding_mode = padding_mode + if isinstance(fill_value, int): # temporary fix + fill_value = tuple([fill_value] * 3) + padding_mode = DE_C_BORDER_TYPE[padding_mode] + super().__init__(*padding, padding_mode, *fill_value) diff --git a/mindspore/dataset/transforms/vision/py_transforms.py b/mindspore/dataset/transforms/vision/py_transforms.py new file mode 100644 index 00000000000..f5ab5d873ba --- /dev/null +++ b/mindspore/dataset/transforms/vision/py_transforms.py @@ -0,0 +1,1314 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +""" +The module vision.py_transforms is implemented basing on python +PIL. This module provides many kinds of image augmentations. It also provides +transferring methods between PIL Image and numpy array. For users who prefer +python PIL in image learning task, this module is a good tool to process image +augmentations. Users could also self-define their own augmentations with python +PIL. +""" +import numbers +import random + +import numpy as np +from PIL import Image + +from . import py_transforms_util as util +from .validators import check_prob, check_crop, check_resize_interpolation, check_random_resize_crop, \ + check_normalize_py, check_random_crop, check_random_color_adjust, check_random_rotation, \ + check_transforms_list, check_random_apply, check_ten_crop, check_num_channels, check_pad, \ + check_random_perspective, check_random_erasing, check_cutout, check_linear_transform, check_random_affine, \ + check_mix_up +from .utils import Inter, Border + +DE_PY_INTER_MODE = {Inter.NEAREST: Image.NEAREST, + Inter.LINEAR: Image.LINEAR, + Inter.CUBIC: Image.CUBIC} + +DE_PY_BORDER_TYPE = {Border.CONSTANT: 'constant', + Border.EDGE: 'edge', + Border.REFLECT: 'reflect', + Border.SYMMETRIC: 'symmetric'} + + +class ComposeOp: + """ + Compose a list of transforms. + + .. Note:: + ComposeOp takes a list of transformations either provided in py_transforms or from user-defined implementation; + each can be an initialized transformation class or a lambda function, as long as the output from the last + transformation is a single tensor of type numpy.ndarray. See below for an example of how to use ComposeOp + with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda + functions. + + Args: + transforms (list): List of transformations to be applied. + + Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.dataset.transforms.vision.py_transforms as py_transforms + >>> dataset_dir = "path/to/imagefolder_directory" + >>> # create a dataset that reads all files in dataset_dir with 8 threads + >>> dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8) + >>> # create a list of transformations to be applied to the image data + >>> transform = py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor(), + >>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)), + >>> py_transforms.RandomErasing()]) + >>> # apply the transform to the dataset through dataset.map() + >>> dataset = dataset.map(input_columns="image", operations=transform()) + """ + + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self): + """ + Call method. + + Returns: + lambda function, Lambda function that takes in an img to apply transformations on. + """ + return lambda img: util.compose(img, self.transforms) + + +class ToTensor: + """ + Convert the input Numpy image array or PIL image of shape (H,W,C) to a Numpy ndarray of shape (C,H,W). + + Note: + The ranges of values in height and width dimension changes from [0, 255] to [0.0, 1.0]. Type cast to output_type + (default Numpy float 32). + The range of channel dimension remains the same. + + Args: + output_type (numpy datatype, optional): The datatype of the numpy output (default=np.float32). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) + """ + + def __init__(self, output_type=np.float32): + self.output_type = output_type + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): PIL Image to be converted to numpy.ndarray. + + Returns: + img (numpy.ndarray), Converted image. + """ + return util.to_tensor(img, self.output_type) + + +class ToType: + """ + Convert the input Numpy image array to desired numpy dtype. + + Args: + output_type (numpy datatype): The datatype of the numpy output. e.g. np.float32. + + Examples: + >>> import numpy as np + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor(), + >>> py_transforms.ToType(np.float32)]) + """ + + def __init__(self, output_type): + self.output_type = output_type + + def __call__(self, img): + """ + Call method. + + Args: + numpy object : numpy object to be type swapped. + + Returns: + img (numpy.ndarray), Converted image. + """ + return util.to_type(img, self.output_type) + + +class HWC2CHW: + """ + Transpose a Numpy image array; shape (H, W, C) to shape (C, H, W). + """ + + def __call__(self, img): + """ + Call method. + + Args: + img (numpy.ndarray): Image array, of shape (H, W, C), to have channels swapped. + + Returns: + img (numpy.ndarray), Image array, of shape (C, H, W), with channels swapped. + """ + return util.hwc_to_chw(img) + + +class ToPIL: + """ + Convert the input decoded Numpy image array of RGB mode to a PIL Image of RGB mode. + + Examples: + >>> # data is already decoded, but not in PIL Image format + >>> py_transforms.ComposeOp([py_transforms.ToPIL(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) + """ + + def __call__(self, img): + """ + Call method. + + Args: + img (numpy.ndarray): Decoded image array, of RGB mode, to be converted to PIL Image. + + Returns: + img (PIL Image), Image converted to PIL Image of RGB mode. + """ + return util.to_pil(img) + + +class Decode: + """ + Decode the input image to PIL Image format in RGB mode. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) + """ + + def __call__(self, img): + """ + Call method. + + Args: + img (Bytes-like Objects):Image to be decoded. + + Returns: + img (PIL Image), Decoded image in RGB mode. + """ + return util.decode(img) + + +class Normalize: + """ + Normalize the input Numpy image array of shape (H, W, C) with the given mean and standard deviation. + + The values of the array need to be in range [0.0, 1.0]. + + Args: + mean (list): List of mean values for each channel, w.r.t channel order. + std (list): List of standard deviations for each channel, w.r.t. channel order. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor(), + >>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))]) + """ + + @check_normalize_py + def __init__(self, mean, std): + self.mean = mean + self.std = std + + def __call__(self, img): + """ + Call method. + + Args: + img (numpy.ndarray): Image array to be normalized. + + Returns: + img (numpy.ndarray), Normalized Image array. + """ + return util.normalize(img, self.mean, self.std) + + +class RandomCrop: + """ + Crop the input PIL Image at a random location. + + Args: + size (int or sequence): The output size of the cropped image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + padding (int or sequence, optional): The number of pixels to pad the image (default=None). + If a single number is provided, it pads all borders with this value. + If a tuple or list of 2 values are provided, it pads the (left and top) + with the first value and (right and bottom) with the second value. + If 4 values are provided as a list or tuple, + it pads the left, top, right and bottom respectively. + pad_if_needed (bool, optional): Pad the image if either side is smaller than + the given output size (default=False). + fill_value (int or tuple, optional): filling value (default=0). + The pixel intensity of the borders if the padding_mode is Border.CONSTANT. + If it is a 3-tuple, it is used to fill R, G, B channels respectively. + padding_mode (str, optional): The method of padding (default=Border.CONSTANT). Can be any of + [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. + + - Border.CONSTANT, means it fills the border with constant values. + + - Border.EDGE, means it pads with the last value on the edge. + + - Border.REFLECT, means it reflects the values on the edge omitting the last + value of edge. + + - Border.SYMMETRIC, means it reflects the values on the edge repeating the last + value of edge. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomCrop(224), + >>> py_transforms.ToTensor()]) + """ + + @check_random_crop + def __init__(self, size, padding=None, pad_if_needed=False, fill_value=0, padding_mode=Border.CONSTANT): + self.size = size + self.padding = padding + self.pad_if_needed = pad_if_needed + self.fill_value = fill_value + self.padding_mode = DE_PY_BORDER_TYPE[padding_mode] + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be randomly cropped. + + Returns: + PIL Image, Cropped image. + """ + return util.random_crop(img, self.size, self.padding, self.pad_if_needed, + self.fill_value, self.padding_mode) + + +class RandomHorizontalFlip: + """ + Randomly flip the input image horizontally with a given probability. + + Args: + prob (float, optional): Probability of the image being flipped (default=0.5). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) + """ + + @check_prob + def __init__(self, prob=0.5): + self.prob = prob + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be flipped horizontally. + + Returns: + img (PIL Image), Randomly flipped image. + """ + return util.random_horizontal_flip(img, self.prob) + + +class RandomVerticalFlip: + """ + Randomly flip the input image vertically with a given probability. + + Args: + prob (float, optional): Probability of the image being flipped (default=0.5). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomVerticalFlip(0.5), + >>> py_transforms.ToTensor()]) + """ + + @check_prob + def __init__(self, prob=0.5): + self.prob = prob + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be flipped vertically. + + Returns: + img (PIL Image), Randomly flipped image. + """ + return util.random_vertical_flip(img, self.prob) + + +class Resize: + """ + Resize the input PIL Image to the given size. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.Resize(256), + >>> py_transforms.ToTensor()]) + """ + + @check_resize_interpolation + def __init__(self, size, interpolation=Inter.BILINEAR): + self.size = size + self.interpolation = DE_PY_INTER_MODE[interpolation] + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be resized. + + Returns: + img (PIL Image), Resize image. + """ + return util.resize(img, self.size, self.interpolation) + + +class RandomResizedCrop: + """ + Extract crop from the input image and resize it to a random size and aspect ratio. + + Args: + size (int or sequence): The size of the output image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + scale (tuple, optional): Range (min, max) of respective size of the original size + to be cropped (default=(0.08, 1.0)). + ratio (tuple, optional): Range (min, max) of aspect ratio to be cropped (default=(3. / 4., 4. / 3.)). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BILINEAR). + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + + max_attempts (int, optional): The maximum number of attempts to propose a valid + crop_area (default=10). If exceeded, fall back to use center_crop instead. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomResizedCrop(224), + >>> py_transforms.ToTensor()]) + """ + + @check_random_resize_crop + def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), + interpolation=Inter.BILINEAR, max_attempts=10): + self.size = size + self.scale = scale + self.ratio = ratio + self.interpolation = DE_PY_INTER_MODE[interpolation] + self.max_attempts = max_attempts + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be randomly cropped and resized. + + Returns: + img (PIL Image), Randomly cropped and resized image. + """ + return util.random_resize_crop(img, self.size, self.scale, self.ratio, + self.interpolation, self.max_attempts) + + +class CenterCrop: + """ + Crop the central reigion of the input PIL Image to the given size. + + Args: + size (int or sequence): The output size of the cropped image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.CenterCrop(64), + >>> py_transforms.ToTensor()]) + """ + + @check_crop + def __init__(self, size): + self.size = size + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be center cropped. + + Returns: + img (PIL Image), Cropped image. + """ + return util.center_crop(img, self.size) + + +class RandomColorAdjust: + """ + Perform a random brightness, contrast, saturation, and hue adjustment on the input PIL image. + + Args: + brightness (float or tuple, optional): Brightness adjustment factor (default=(1, 1)). Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-brightness), 1+brightness]. + If it is a sequence, it should be [min, max] for the range. + contrast (float or tuple, optional): Contrast adjustment factor (default=(1, 1)). Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-contrast), 1+contrast]. + If it is a sequence, it should be [min, max] for the range. + saturation (float or tuple, optional): Saturation adjustment factor (default=(1, 1)). Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-saturation), 1+saturation]. + If it is a sequence, it should be [min, max] for the range. + hue (float or tuple, optional): Hue adjustment factor (default=(0, 0)). + If it is a float, the range will be [-hue, hue]. Value should be 0 <= hue <= 0.5. + If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), + >>> py_transforms.ToTensor()]) + """ + + @check_random_color_adjust + def __init__(self, brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0)): + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to have its color adjusted randomly. + + Returns: + img (PIL Image), Image after random adjustment of its color. + """ + return util.random_color_adjust(img, self.brightness, self.contrast, self.saturation, self.hue) + + +class RandomRotation: + """ + Rotate the input PIL image by a random angle. + + Note: + See https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.rotate. + + Args: + degrees (int or float or sequence): Range of random rotation degrees. + If degrees is a number, the range will be converted to (-degrees, degrees). + If degrees is a sequence, it should be (min, max). + resample (Inter mode, optional): An optional resampling filter (default=Inter.NEAREST). + If omitted, or if the image has mode "1" or "P", it is set to be Inter.NEAREST. + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means resampling method is bilinear interpolation. + + - Inter.NEAREST, means resampling method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means resampling method is bicubic interpolation. + + expand (bool, optional): Optional expansion flag (default=False). If set to True, expand the output + image to make it large enough to hold the entire rotated image. + If set to False or omitted, make the output image the same size as the input. + Note that the expand flag assumes rotation around the center and no translation. + center (tuple, optional): Optional center of rotation (a 2-tuple) (default=None). + Origin is the top left corner. Default None sets to the center of the image. + fill_value (int or tuple, optional): Optional fill color for the area outside the rotated + image (default=0). + If it is a 3-tuple, it is used for R, G, B channels respectively. + If it is an int, it is used for all RGB channels. Default is 0. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomRotation(30), + >>> py_transforms.ToTensor()]) + """ + + @check_random_rotation + def __init__(self, degrees, resample=Inter.NEAREST, expand=False, center=None, fill_value=0): + self.degrees = degrees + self.resample = DE_PY_INTER_MODE[resample] + self.expand = expand + self.center = center + self.fill_value = fill_value + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be rotated. + + Returns: + img (PIL Image), Rotated image. + """ + return util.random_rotation(img, self.degrees, self.resample, self.expand, self.center, self.fill_value) + + +class RandomOrder: + """ + Perform a series of transforms to the input PIL image in a random oreder. + + Args: + transforms (list): List of the transformations to be applied. + + Examples: + >>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)] + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomOrder(transforms_list), + >>> py_transforms.ToTensor()]) + """ + + @check_transforms_list + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be applied transformations in a random order. + + Returns: + img (PIL Image), Transformed image. + """ + return util.random_order(img, self.transforms) + + +class RandomApply: + """ + Randomly perform a series of transforms with a given probability. + + Args: + transforms (list): List of transformations to be applied. + prob (float, optional): The probability to apply the transformation list (default=0.5). + + Examples: + >>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)] + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomApply(transforms_list, prob=0.6), + >>> py_transforms.ToTensor()]) + """ + + @check_random_apply + def __init__(self, transforms, prob=0.5): + self.prob = prob + self.transforms = transforms + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be randomly applied a list transformations. + + Returns: + img (PIL Image), Transformed image. + """ + return util.random_apply(img, self.transforms, self.prob) + + +class RandomChoice: + """ + Randomly select one transform from a series of transforms and applies that on the image. + + Args: + transforms (list): List of transformations to be chosen from to apply. + + Examples: + >>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)] + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomChoice(transforms_list), + >>> py_transforms.ToTensor()]) + """ + + @check_transforms_list + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be applied transformation. + + Returns: + img (PIL Image), Transformed image. + """ + return util.random_choice(img, self.transforms) + + +class FiveCrop: + """ + Generate 5 cropped images (one central and four corners). + + Args: + size (int or sequence): The output size of the crop. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.FiveCrop(size), + >>> # 4D stack of 5 images + >>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])]) + """ + + @check_crop + def __init__(self, size): + self.size = size + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): PIL Image to be cropped. + + Returns: + img_tuple (tuple), a tuple of 5 PIL images + (top_left, top_right, bottom_left, bottom_right, center). + """ + return util.five_crop(img, self.size) + + +class TenCrop: + """ + Generate 10 cropped images (first 5 from FiveCrop, second 5 from their flipped version). + + Args: + size (int or sequence): The output size of the crop. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + use_vertical_flip (bool, optional): Flip the image vertically instead of horizontally + if set to True (default=False). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.TenCrop(size), + >>> # 4D stack of 10 images + >>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])]) + """ + + @check_ten_crop + def __init__(self, size, use_vertical_flip=False): + self.size = size + self.use_vertical_flip = use_vertical_flip + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): PIL Image to be cropped. + + Returns: + img_tuple (tuple), a tuple of 10 PIL images + (top_left, top_right, bottom_left, bottom_right, center) of original image + + (top_left, top_right, bottom_left, bottom_right, center) of flipped image. + """ + return util.ten_crop(img, self.size, self.use_vertical_flip) + + +class Grayscale: + """ + Convert the input PIL image to grayscale image. + + Args: + num_output_channels (int): Number of channels of the output grayscale image (1 or 3). + Default is 1. If set to 3, the returned image has 3 identical RGB channels. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.Grayscale(3), + >>> py_transforms.ToTensor()]) + """ + + @check_num_channels + def __init__(self, num_output_channels=1): + self.num_output_channels = num_output_channels + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): PIL image to be converted to grayscale. + + Returns: + img (PIL Image), grayscaled image. + """ + return util.grayscale(img, num_output_channels=self.num_output_channels) + + +class RandomGrayscale: + """ + Randomly convert the input image into grayscale image with a given probability. + + Args: + prob (float, optional): Probability of the image being converted to grayscale (default=0.1). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomGrayscale(0.3), + >>> py_transforms.ToTensor()]) + """ + + @check_prob + def __init__(self, prob=0.1): + self.prob = prob + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): PIL image to be converted to grayscale randomly. + + Returns: + img (PIL Image), Randomly grayscale image, same number of channels as input image. + If input image has 1 channel, the output grayscale image is 1 channel. + If input image has 3 channels, the output image has 3 identical grayscale channels. + """ + if img.mode == 'L': + num_output_channels = 1 + else: + num_output_channels = 3 + + if self.prob > random.random(): + return util.grayscale(img, num_output_channels=num_output_channels) + return img + + +class Pad: + """ + Pad the input PIL image according to padding parameters. + + Args: + padding (int or sequence): The number of pixels to pad the image. + If a single number is provided, it pads all borders with this value. + If a tuple or list of 2 values are provided, it pads the (left and top) + with the first value and (right and bottom) with the second value. + If 4 values are provided as a list or tuple, + it pads the left, top, right and bottom respectively. + fill_value (int or tuple, optional): Filling value (default=0). The pixel intensity + of the borders if the padding_mode is Border.CONSTANT. + If it is a 3-tuple, it is used to fill R, G, B channels respectively. + padding_mode (Border mode, optional): The method of padding (default=Border.CONSTANT). + Can be any of [Border.CONSTANT, Border.EDGE, Border.REFLECT, Border.SYMMETRIC]. + + - Border.CONSTANT, means it fills the border with constant values. + + - Border.EDGE, means it pads with the last value on the edge. + + - Border.REFLECT, means it reflects the values on the edge omitting the last + value of edge. + + - Border.SYMMETRIC, means it reflects the values on the edge repeating the last + value of edge. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> # adds 10 pixels (default black) to each side of the border of the image + >>> py_transforms.Pad(padding=10), + >>> py_transforms.ToTensor()]) + """ + + @check_pad + def __init__(self, padding, fill_value=0, padding_mode=Border.CONSTANT): + self.padding = padding + self.fill_value = fill_value + self.padding_mode = DE_PY_BORDER_TYPE[padding_mode] + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be padded. + + Returns: + img (PIL Image), Padded image. + """ + return util.pad(img, self.padding, self.fill_value, self.padding_mode) + + +class RandomPerspective: + """ + Randomly apply perspective transformation to the input PIL Image with a given probability. + + Args: + distortion_scale (float, optional): The scale of distortion, float between 0 and 1 (default=0.5). + prob (float, optional): Probability of the image being applied perspective transformation (default=0.5). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.BICUBIC). + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomPerspective(prob=0.1), + >>> py_transforms.ToTensor()]) + """ + + @check_random_perspective + def __init__(self, distortion_scale=0.5, prob=0.5, interpolation=Inter.BICUBIC): + self.distortion_scale = distortion_scale + self.prob = prob + self.interpolation = DE_PY_INTER_MODE[interpolation] + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): PIL Image to be applied perspective transformation randomly. + + Returns: + img (PIL Image), Image after being perspectively transformed randomly. + """ + if self.prob > random.random(): + start_points, end_points = util.get_perspective_params(img, self.distortion_scale) + return util.perspective(img, start_points, end_points, self.interpolation) + return img + + +class RandomErasing: + """ + Erase the pixels, within a selected rectangle region, to the given value. + + Randomly applied on the input Numpy image array with a given probability. + + Zhun Zhong et al. 'Random Erasing Data Augmentation' 2017 See https://arxiv.org/pdf/1708.04896.pdf + + Args: + prob (float, optional): Probability of applying RandomErasing (default=0.5). + scale (sequence of floats, optional): Range of the relative erase area to the + original image (default=(0.02, 0.33)). + ratio (sequence of floats, optional): Range of the aspect ratio of the erase + area (default=(0.3, 3.3)). + value (int or sequence): Erasing value (default=0). + If value is a single int, it is applied to all pixels to be erases. + If value is a sequence of length 3, it is applied to R, G, B channels respectively. + If value is a str 'random', the erase value will be obtained from a standard normal distribution. + inplace (bool, optional): Apply this transform inplace (default=False). + max_attempts (int, optional): The maximum number of attempts to propose a valid + erase_area (default=10). If exceeded, return the original image. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.ToTensor(), + >>> py_transforms.RandomErasing(value='random')]) + """ + + @check_random_erasing + def __init__(self, prob=0.5, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False, max_attempts=10): + self.prob = prob + self.scale = scale + self.ratio = ratio + self.value = value + self.inplace = inplace + self.max_attempts = max_attempts + + def __call__(self, np_img): + """ + Call method. + + Args: + np_img (numpy.ndarray): Numpy image array of shape (C, H, W) to be randomly erased. + + Returns: + np_img (numpy.ndarray), Erased Numpy image array. + """ + bounded = True + if self.prob > random.random(): + i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, self.scale, self.ratio, + self.value, bounded, self.max_attempts) + return util.erase(np_img, i, j, erase_h, erase_w, erase_value, self.inplace) + return np_img + + +class Cutout: + """ + Randomly cut (mask) out a given number of square patches from the input Numpy image array. + + Terrance DeVries and Graham W. Taylor 'Improved Regularization of Convolutional Neural Networks with Cutout' 2017 + See https://arxiv.org/pdf/1708.04552.pdf + + Args: + length (int): The side length of each square patch. + num_patches (int, optional): Number of patches to be cut out of an image (default=1). + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.ToTensor(), + >>> py_transforms.Cutout(80)]) + """ + + @check_cutout + def __init__(self, length, num_patches=1): + self.length = length + self.num_patches = num_patches + + def __call__(self, np_img): + """ + Call method. + + Args: + np_img (numpy.ndarray): Numpy image array of shape (C, H, W) to be cut out. + + Returns: + np_img (numpy.ndarray), Numpy image array with square patches cut out. + """ + if not isinstance(np_img, np.ndarray): + raise TypeError('img should be Numpy array. Got {}'.format(type(np_img))) + _, image_h, image_w = np_img.shape + scale = (self.length * self.length) / (image_h * image_w) + bounded = False + + for _ in range(self.num_patches): + i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded, 1) + np_img = util.erase(np_img, i, j, erase_h, erase_w, erase_value) + return np_img + + +class LinearTransformation: + """ + Apply linear transformation to the input Numpy image array, given a square transformation matrix and + a mean_vector. + + The transformation first flattens the input array and subtract mean_vector from it, then computes the + dot product with the transformation matrix, and reshapes it back to its original shape. + + Args: + transformation_matrix (numpy.ndarray): a square transformation matrix of shape (D, D), D = C x H x W. + mean_vector (numpy.ndarray): a numpy ndarray of shape (D,) where D = C x H x W. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.Resize(256), + >>> py_transforms.ToTensor(), + >>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)]) + """ + + @check_linear_transform + def __init__(self, transformation_matrix, mean_vector): + self.transformation_matrix = transformation_matrix + self.mean_vector = mean_vector + + def __call__(self, np_img): + """ + Call method. + + Args: + np_img (numpy.ndarray): Numpy image array of shape (C, H, W) to be linear transformed. + + Returns: + np_img (numpy.ndarray), Linear transformed image. + """ + return util.linear_transform(np_img, self.transformation_matrix, self.mean_vector) + + +class RandomAffine: + """ + Apply Random affine transformation to the input PIL image. + + Args: + degrees (int or float or sequence): Range of the rotation degrees. + If degrees is a number, the range will be (-degrees, degrees). + If degrees is a sequence, it should be (min, max). + translate (sequence, optional): Sequence (tx, ty) of maximum translation in + x(horizontal) and y(vertical) directions (default=None). + The horizontal and vertical shift is selected randomly from the range: + (-tx*width, tx*width) and (-ty*height, ty*height), respectively. + If None, no translations gets applied. + scale (sequence, optional): Scaling factor interval (default=None, riginal scale is used). + shear (int or float or sequence, optional): Range of shear factor (default=None). + If a number 'shear', then a shear parallel to the x axis in the range of (-shear, +shear) is applied. + If a tuple or list of size 2, then a shear parallel to the x axis in the range of (shear[0], shear[1]) + is applied. + If a tuple of list of size 4, then a shear parallel to x axis in the range of (shear[0], shear[1]) + and a shear parallel to y axis in the range of (shear[2], shear[3]) is applied. + If None, no shear is applied. + resample (Inter mode, optional): An optional resampling filter (default=Inter.NEAREST). + If omitted, or if the image has mode "1" or "P", it is set to be Inter.NEAREST. + It can be any of [Inter.BILINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.BILINEAR, means resample method is bilinear interpolation. + + - Inter.NEAREST, means resample method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means resample method is bicubic interpolation. + + fill_value (tuple or int, optional): Optional fill_value to fill the area outside the transform + in the output image. Used only in Pillow versions > 5.0.0 (default=0, filling is performed). + + Raises: + ValueError: If degrees is negative. + ValueError: If translation value is not between 0 and 1. + ValueError: If scale is not positive. + ValueError: If shear is a number but is not positive. + TypeError: If degrees is not a number or a list or a tuple. + If degrees is a list or tuple, its length is not 2. + TypeError: If translate is specified but is not list or a tuple of length 2. + TypeError: If scale is not a list or tuple of length 2. + TypeError: If shear is not a list or tuple of length 2 or 4. + + Examples: + >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)), + >>> py_transforms.ToTensor()]) + """ + + @check_random_affine + def __init__(self, degrees, translate=None, scale=None, shear=None, resample=Inter.NEAREST, fill_value=0): + # Parameter checking + # rotation + if isinstance(degrees, numbers.Number): + if degrees < 0: + raise ValueError("If degrees is a single number, it must be positive.") + self.degrees = (-degrees, degrees) + elif isinstance(degrees, (tuple, list)) and len(degrees) == 2: + self.degrees = degrees + else: + raise TypeError("If degrees is a list or tuple, it must be of length 2.") + + # translation + if translate is not None: + if isinstance(translate, (tuple, list)) and len(translate) == 2: + for t in translate: + if t < 0.0 or t > 1.0: + raise ValueError("translation values should be between 0 and 1") + else: + raise TypeError("translate should be a list or tuple of length 2.") + self.translate = translate + + # scale + if scale is not None: + if isinstance(scale, (tuple, list)) and len(scale) == 2: + for s in scale: + if s <= 0: + raise ValueError("scale values should be positive") + else: + raise TypeError("scale should be a list or tuple of length 2.") + self.scale_ranges = scale + + # shear + if shear is not None: + if isinstance(shear, numbers.Number): + if shear < 0: + raise ValueError("If shear is a single number, it must be positive.") + self.shear = (-1 * shear, shear) + elif isinstance(shear, (tuple, list)) and (len(shear) == 2 or len(shear) == 4): + # X-Axis shear with [min, max] + if len(shear) == 2: + self.shear = [shear[0], shear[1], 0., 0.] + elif len(shear) == 4: + self.shear = [s for s in shear] + else: + raise TypeError("shear should be a list or tuple and it must be of length 2 or 4.") + else: + self.shear = shear + + # resample + self.resample = DE_PY_INTER_MODE[resample] + + # fill_value + self.fill_value = fill_value + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be applied affine transformation. + + Returns: + img (PIL Image), Randomly affine transformed image. + """ + # rotation + angle = random.uniform(self.degrees[0], self.degrees[1]) + + # translation + if self.translate is not None: + max_dx = self.translate[0] * img.size[0] + max_dy = self.translate[1] * img.size[1] + translations = (np.round(random.uniform(-max_dx, max_dx)), + np.round(random.uniform(-max_dy, max_dy))) + else: + translations = (0, 0) + + # scale + if self.scale_ranges is not None: + scale = random.uniform(self.scale_ranges[0], self.scale_ranges[1]) + else: + scale = 1.0 + + # shear + if self.shear is not None: + if len(self.shear) == 2: + shear = [random.uniform(self.shear[0], self.shear[1]), 0.] + elif len(self.shear) == 4: + shear = [random.uniform(self.shear[0], self.shear[1]), + random.uniform(self.shear[2], self.shear[3])] + else: + shear = 0.0 + + return util.random_affine(img, + angle, + translations, + scale, + shear, + self.resample, + self.fill_value) + + +class MixUp: + """ + Apply mix up transformation to the input image and label, make one input data combined with others. + + Args: + batch_size (int): the batch size of dataset. + alpha (float): the mix up rate. + is_single (bool): for deciding using single batch or muti batch mix up transformation. + """ + + @check_mix_up + def __init__(self, batch_size, alpha, is_single=True): + self.image = 0 + self.label = 0 + self.is_first = True + self.batch_size = batch_size + self.alpha = alpha + self.is_single = is_single + + def __call__(self, image, label): + """ + Call method. + + Args: + image (numpy.ndarray): numpy Image to be applied mix up transformation. + label(numpy.ndarray): numpy label to be applied mix up transformation. + + Returns: + image (numpy.ndarray): numpy Image after being applied mix up transformation. + label(numpy.ndarray): numpy label after being applied mix up transformation. + """ + if self.is_single: + return util.mix_up_single(self.batch_size, image, label, self.alpha) + return util.mix_up_muti(self, self.batch_size, image, label, self.alpha) + + +class RgbToHsv: + """ + Convert a Numpy RGB image or one batch Numpy RGB images to HSV images. + + Args: + is_hwc (bool): The flag of image shape, (H, W, C) or (N, H, W, C) if True + and (C, H, W) or (N, C, H, W) if False (default=False). + """ + + def __init__(self, is_hwc=False): + self.is_hwc = is_hwc + + def __call__(self, rgb_imgs): + """ + Call method. + + Args: + rgb_imgs (numpy.ndarray): Numpy RGB images array of shape (H, W, C) or (N, H, W, C), + or (C, H, W) or (N, C, H, W) to be converted. + + Returns: + np_hsv_img (numpy.ndarray), Numpy HSV images with same shape of rgb_imgs. + """ + return util.rgb_to_hsvs(rgb_imgs, self.is_hwc) + + +class HsvToRgb: + """ + Convert a Numpy HSV image or one batch Numpy HSV images to RGB images. + + Args: + is_hwc (bool): The flag of image shape, (H, W, C) or (N, H, W, C) if True + and (C, H, W) or (N, C, H, W) if False (default=False). + """ + + def __init__(self, is_hwc=False): + self.is_hwc = is_hwc + + def __call__(self, hsv_imgs): + """ + Call method. + + Args: + hsv_imgs (numpy.ndarray): Numpy HSV images array of shape (H, W, C) or (N, H, W, C), + or (C, H, W) or (N, C, H, W) to be converted. + + Returns: + rgb_imgs (numpy.ndarray), Numpy RGB image with same shape of hsv_imgs. + """ + return util.hsv_to_rgbs(hsv_imgs, self.is_hwc) diff --git a/mindspore/dataset/transforms/vision/py_transforms_util.py b/mindspore/dataset/transforms/vision/py_transforms_util.py new file mode 100644 index 00000000000..10c71bbe385 --- /dev/null +++ b/mindspore/dataset/transforms/vision/py_transforms_util.py @@ -0,0 +1,1410 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Built-in py_transforms_utils functions. +""" +import io +import math +import numbers +import random +import colorsys + +import numpy as np +from PIL import Image, ImageOps, ImageEnhance, __version__ + +from .utils import Inter + +augment_error_message = 'img should be PIL Image. Got {}. Use Decode() for encoded data or ToPIL() for decoded data.' + + +def is_pil(img): + """ + Check if the input image is PIL format. + + Args: + img: Image to be checked. + + Returns: + Bool, True if input is PIL image. + """ + return isinstance(img, Image.Image) + + +def is_numpy(img): + """ + Check if the input image is Numpy format. + + Args: + img: Image to be checked. + + Returns: + Bool, True if input is Numpy image. + """ + return isinstance(img, np.ndarray) + + +def compose(img, transforms): + """ + Compose a list of transforms and apply on the image. + + Args: + img (numpy.ndarray): An image in Numpy ndarray. + transforms (list): A list of transform Class objects to be composed. + + Returns: + img (numpy.ndarray), An augmented image in Numpy ndarray. + """ + if is_numpy(img): + for transform in transforms: + img = transform(img) + if is_numpy(img): + return img + raise TypeError('img should be Numpy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img))) + raise TypeError('img should be Numpy ndarray. Got {}.'.format(type(img))) + + +def normalize(img, mean, std): + """ + Normalize the image between [0, 1] with respect to mean and standard deviation. + + Args: + img (numpy.ndarray): Image array of shape CHW to be normalized. + mean (list): List of mean values for each channel, w.r.t channel order. + std (list): List of standard deviations for each channel, w.r.t. channel order. + + Returns: + img (numpy.ndarray), Normalized image. + """ + if not is_numpy(img): + raise TypeError('img should be Numpy Image. Got {}'.format(type(img))) + + num_channels = img.shape[0] # shape is (C, H, W) + + if len(mean) != len(std): + raise ValueError("Length of mean and std must be equal") + # if length equal to 1, adjust the mean and std arrays to have the correct + # number of channels (replicate the values) + if len(mean) == 1: + mean = [mean[0]] * num_channels + std = [std[0]] * num_channels + elif len(mean) != num_channels: + raise ValueError("Length of mean and std must both be 1 or equal to the number of channels({0})" + .format(num_channels)) + + mean = np.array(mean, dtype=img.dtype) + std = np.array(std, dtype=img.dtype) + return (img - mean[:, None, None]) / std[:, None, None] + + +def decode(img): + """ + Decode the input image to PIL Image format in RGB mode. + + Args: + img: Image to be decoded. + + Returns: + img (PIL Image), Decoded image in RGB mode. + """ + + try: + data = io.BytesIO(img) + img = Image.open(data) + return img.convert('RGB') + except IOError as e: + raise ValueError("{0}\nWARNING: Failed to decode given image.".format(e)) + except AttributeError as e: + raise ValueError("{0}\nWARNING: Failed to decode, Image might already be decoded.".format(e)) + + +def hwc_to_chw(img): + """ + Transpose the input image; shape (H, W, C) to shape (C, H, W). + + Args: + img (numpy.ndarray): Image to be converted. + + Returns: + img (numpy.ndarray), Converted image. + """ + if is_numpy(img): + return img.transpose(2, 0, 1).copy() + raise TypeError('img should be Numpy array. Got {}'.format(type(img))) + + +def to_tensor(img, output_type): + """ + Change the input image (PIL Image or Numpy image array) to numpy format. + + Args: + img (PIL Image or numpy.ndarray): Image to be converted. + output_type: The datatype of the numpy output. e.g. np.float32 + + Returns: + img (numpy.ndarray), Converted image. + """ + if not (is_pil(img) or is_numpy(img)): + raise TypeError('img should be PIL Image or Numpy array. Got {}'.format(type(img))) + + img = np.asarray(img) + if img.ndim not in (2, 3): + raise ValueError('img dimension should be 2 or 3. Got {}'.format(img.ndim)) + + if img.ndim == 2: + img = img[:, :, None] + + img = hwc_to_chw(img) + + img = img / 255. + return to_type(img, output_type) + + +def to_pil(img): + """ + Convert the input image to PIL format. + + Args: + img: Image to be converted. + + Returns: + img (PIL Image), Converted image. + """ + if not is_pil(img): + return Image.fromarray(img) + return img + + +def horizontal_flip(img): + """ + Flip the input image horizontally. + + Args: + img (PIL Image): Image to be flipped horizontally. + + Returns: + img (PIL Image), Horizontally flipped image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + return img.transpose(Image.FLIP_LEFT_RIGHT) + + +def vertical_flip(img): + """ + Flip the input image vertically. + + Args: + img (PIL Image): Image to be flipped vertically. + + Returns: + img (PIL Image), Vertically flipped image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + return img.transpose(Image.FLIP_TOP_BOTTOM) + + +def random_horizontal_flip(img, prob): + """ + Randomly flip the input image horizontally. + + Args: + img (PIL Image): Image to be flipped. + If the given probability is above the random probability, then the image is flipped. + prob (float): Probability of the image being flipped. + + Returns: + img (PIL Image), Converted image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if prob > random.random(): + img = horizontal_flip(img) + return img + + +def random_vertical_flip(img, prob): + """ + Randomly flip the input image vertically. + + Args: + img (PIL Image): Image to be flipped. + If the given probability is above the random probability, then the image is flipped. + prob (float): Probability of the image being flipped. + + Returns: + img (PIL Image), Converted image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if prob > random.random(): + img = vertical_flip(img) + return img + + +def crop(img, top, left, height, width): + """ + Crop the input PIL Image. + + Args: + img (PIL Image): Image to be cropped. (0,0) denotes the top left corner of the image, + in the directions of (width, height). + top (int): Vertical component of the top left corner of the crop box. + left (int): Horizontal component of the top left corner of the crop box. + height (int): Height of the crop box. + width (int): Width of the crop box. + + Returns: + img (PIL Image), Cropped image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + return img.crop((left, top, left + width, top + height)) + + +def resize(img, size, interpolation=Inter.BILINEAR): + """ + Resize the input PIL Image to desired size. + + Args: + img (PIL Image): Image to be resized. + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of (height, width), this will be the desired output size. + interpolation (interpolation mode): Image interpolation mode. Default is Inter.BILINEAR = 2. + + Returns: + img (PIL Image), Resized image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + if not (isinstance(size, int) or (isinstance(size, (list, tuple)) and len(size) == 2)): + raise TypeError('Size should be a single number or a list/tuple (h, w) of length 2.' + 'Got {}'.format(size)) + + if isinstance(size, int): + img_width, img_height = img.size + aspect_ratio = img_width / img_height # maintain the aspect ratio + if (img_width <= img_height and img_width == size) or \ + (img_height <= img_width and img_height == size): + return img + if img_width < img_height: + out_width = size + out_height = int(size / aspect_ratio) + return img.resize((out_width, out_height), interpolation) + out_height = size + out_width = int(size * aspect_ratio) + return img.resize((out_width, out_height), interpolation) + return img.resize(size[::-1], interpolation) + + +def center_crop(img, size): + """ + Crop the input PIL Image at the center to the given size. + + Args: + img (PIL Image): Image to be cropped. + size (int or tuple): The size of the crop box. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + + Returns: + img (PIL Image), Cropped image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if isinstance(size, int): + size = (size, size) + img_width, img_height = img.size + crop_height, crop_width = size + crop_top = int(round((img_height - crop_height) / 2.)) + crop_left = int(round((img_width - crop_width) / 2.)) + return crop(img, crop_top, crop_left, crop_height, crop_width) + + +def random_resize_crop(img, size, scale, ratio, interpolation=Inter.BILINEAR, max_attempts=10): + """ + Crop the input PIL Image to a random size and aspect ratio. + + Args: + img (PIL Image): Image to be randomly cropped and resized. + size (int or sequence): The size of the output image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + scale (tuple): Range (min, max) of respective size of the original size to be cropped. + ratio (tuple): Range (min, max) of aspect ratio to be cropped. + interpolation (interpolation mode): Image interpolation mode. Default is Inter.BILINEAR = 2. + max_attempts (int): The maximum number of attempts to propose a valid crop_area. Default 10. + If exceeded, fall back to use center_crop instead. + + Returns: + img (PIL Image), Randomly cropped and resized image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + if isinstance(size, int): + size = (size, size) + elif isinstance(size, (tuple, list)) and len(size) == 2: + size = size + else: + raise TypeError("Size should be a single integer or a list/tuple (h, w) of length 2.") + + if scale[0] > scale[1] or ratio[0] > ratio[1]: + raise ValueError("Range should be in the order of (min, max).") + + def _input_to_factor(img, scale, ratio): + img_width, img_height = img.size + img_area = img_width * img_height + + for _ in range(max_attempts): + crop_area = random.uniform(scale[0], scale[1]) * img_area + # in case of non-symmetrical aspect ratios, + # use uniform distribution on a logarithmic scale. + log_ratio = (math.log(ratio[0]), math.log(ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + + width = int(round(math.sqrt(crop_area * aspect_ratio))) + height = int(round(width / aspect_ratio)) + + if 0 < width <= img_width and 0 < height <= img_height: + top = random.randint(0, img_height - height) + left = random.randint(0, img_width - width) + return top, left, height, width + + # exceeding max_attempts, use center crop + img_ratio = img_width / img_height + if img_ratio < ratio[0]: + width = img_width + height = int(round(width / ratio[0])) + elif img_ratio > ratio[1]: + height = img_height + width = int(round(height * ratio[1])) + else: + width = img_width + height = img_height + top = int(round((img_height - height) / 2.)) + left = int(round((img_width - width) / 2.)) + return top, left, height, width + + top, left, height, width = _input_to_factor(img, scale, ratio) + img = crop(img, top, left, height, width) + img = resize(img, size, interpolation) + return img + + +def random_crop(img, size, padding, pad_if_needed, fill_value, padding_mode): + """ + Crop the input PIL Image at a random location. + + Args: + img (PIL Image): Image to be randomly cropped. + size (int or sequence): The output size of the cropped image. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + padding (int or sequence, optional): The number of pixels to pad the image. + If a single number is provided, it pads all borders with this value. + If a tuple or list of 2 values are provided, it pads the (left and top) + with the first value and (right and bottom) with the second value. + If 4 values are provided as a list or tuple, + it pads the left, top, right and bottom respectively. + Default is None. + pad_if_needed (bool): Pad the image if either side is smaller than + the given output size. Default is False. + fill_value (int or tuple): The pixel intensity of the borders if + the padding_mode is 'constant'. If it is a 3-tuple, it is used to + fill R, G, B channels respectively. + padding_mode (str): The method of padding. Can be any of + ['constant', 'edge', 'reflect', 'symmetric']. + - 'constant', means it fills the border with constant values + - 'edge', means it pads with the last value on the edge + - 'reflect', means it reflects the values on the edge omitting the last + value of edge + - 'symmetric', means it reflects the values on the edge repeating the last + value of edge + + Returns: + PIL Image, Cropped image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + if isinstance(size, int): + size = (size, size) + elif isinstance(size, (tuple, list)) and len(size) == 2: + size = size + else: + raise TypeError("Size should be a single integer or a list/tuple (h, w) of length 2.") + + def _input_to_factor(img, size): + img_width, img_height = img.size + height, width = size + if width == img_width and height == img_height: + return 0, 0, img_height, img_width + + top = random.randint(0, img_height - height) + left = random.randint(0, img_width - width) + return top, left, height, width + + if padding is not None: + img = pad(img, padding, fill_value, padding_mode) + # pad width when needed, img.size (width, height), crop size (height, width) + if pad_if_needed and img.size[0] < size[1]: + img = pad(img, (size[1] - img.size[0], 0), fill_value, padding_mode) + # pad height when needed + if pad_if_needed and img.size[1] < size[0]: + img = pad(img, (0, size[0] - img.size[1]), fill_value, padding_mode) + + top, left, height, width = _input_to_factor(img, size) + return crop(img, top, left, height, width) + + +def adjust_brightness(img, brightness_factor): + """ + Adjust brightness of an image. + + Args: + img (PIL Image): Image to be adjusted. + brightness_factor (float): A non negative number indicated the factor by which + the brightness is adjusted. 0 gives a black image, 1 gives the original. + + Returns: + img (PIL Image), Brightness adjusted image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + enhancer = ImageEnhance.Brightness(img) + img = enhancer.enhance(brightness_factor) + return img + + +def adjust_contrast(img, contrast_factor): + """ + Adjust contrast of an image. + + Args: + img (PIL Image): PIL Image to be adjusted. + contrast_factor (float): A non negative number indicated the factor by which + the contrast is adjusted. 0 gives a solid gray image, 1 gives the original. + + Returns: + img (PIL Image), Contrast adjusted image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + enhancer = ImageEnhance.Contrast(img) + img = enhancer.enhance(contrast_factor) + return img + + +def adjust_saturation(img, saturation_factor): + """ + Adjust saturation of an image. + + Args: + img (PIL Image): PIL Image to be adjusted. + saturation_factor (float): A non negative number indicated the factor by which + the saturation is adjusted. 0 will give a black and white image, 1 will + give the original. + + Returns: + img (PIL Image), Saturation adjusted image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + enhancer = ImageEnhance.Color(img) + img = enhancer.enhance(saturation_factor) + return img + + +def adjust_hue(img, hue_factor): + """ + Adjust hue of an image. The Hue is changed by changing the HSV values after image is converted to HSV. + + Args: + img (PIL Image): PIL Image to be adjusted. + hue_factor (float): Amount to shift the Hue channel. Value should be in + [-0.5, 0.5]. 0.5 and -0.5 give complete reversal of hue channel. This + is because Hue wraps around when rotated 360 degrees. + 0 means no shift that gives the original image while both -0.5 and 0.5 + will give an image with complementary colors . + + Returns: + img (PIL Image), Hue adjusted image. + """ + if not -0.5 <= hue_factor <= 0.5: + raise ValueError('hue_factor {} is not in [-0.5, 0.5].'.format(hue_factor)) + + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + input_mode = img.mode + if input_mode in {'L', '1', 'I', 'F'}: + return img + + h, s, v = img.convert('HSV').split() + + np_h = np.array(h, dtype=np.uint8) + + with np.errstate(over='ignore'): + np_h += np.uint8(hue_factor * 255) + h = Image.fromarray(np_h, 'L') + + img = Image.merge('HSV', (h, s, v)).convert(input_mode) + return img + + +def to_type(img, output_type): + """ + Convert the Numpy image array to desired numpy dtype. + + Args: + img (numpy): Numpy image to cast to desired numpy dtype. + output_type (numpy datatype): Numpy dtype to cast to. + + Returns: + img (numpy.ndarray), Converted image. + """ + if not is_numpy(img): + raise TypeError('img should be Numpy Image. Got {}'.format(type(img))) + + return img.astype(output_type) + + +def rotate(img, angle, resample, expand, center, fill_value): + """ + Rotate the input PIL image by angle. + + Args: + img (PIL Image): Image to be rotated. + angle (int or float): Rotation angle in degrees, counter-clockwise. + resample (Inter.NEAREST, or Inter.BILINEAR, Inter.BICUBIC, optional): An optional resampling filter. + If omitted, or if the image has mode "1" or "P", it is set to be Inter.NEAREST. + expand (bool, optional): Optional expansion flag. If set to True, expand the output + image to make it large enough to hold the entire rotated image. + If set to False or omitted, make the output image the same size as the input. + Note that the expand flag assumes rotation around the center and no translation. + center (tuple, optional): Optional center of rotation (a 2-tuple). + Origin is the top left corner. + fill_value (int or tuple): Optional fill color for the area outside the rotated image. + If it is a 3-tuple, it is used for R, G, B channels respectively. + If it is an int, it is used for all RGB channels. + + Returns: + img (PIL Image), Rotated image. + + https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.rotate + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if isinstance(fill_value, int): + fill_value = tuple([fill_value] * 3) + + return img.rotate(angle, resample, expand, center, fillcolor=fill_value) + + +def random_color_adjust(img, brightness, contrast, saturation, hue): + """ + Randomly adjust the brightness, contrast, saturation, and hue of an image. + + Args: + img (PIL Image): Image to have its color adjusted randomly. + brightness (float or tuple): Brightness adjustment factor. Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-brightness), 1+brightness]. + If it is a sequence, it should be [min, max] for the range. + contrast (float or tuple): Contrast adjustment factor. Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-contrast), 1+contrast]. + If it is a sequence, it should be [min, max] for the range. + saturation (float or tuple): Saturation adjustment factor. Cannot be negative. + If it is a float, the factor is uniformly chosen from the range [max(0, 1-saturation), 1+saturation]. + If it is a sequence, it should be [min, max] for the range. + hue (float or tuple): Hue adjustment factor. + If it is a float, the range will be [-hue, hue]. Value should be 0 <= hue <= 0.5. + If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5. + + Returns: + img (PIL Image), Image after random adjustment of its color. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + def _input_to_factor(value, input_name, center=1, bound=(0, float('inf')), non_negative=True): + if isinstance(value, numbers.Number): + if value < 0: + raise ValueError("The input value of {} cannot be negative.".format(input_name)) + # convert value into a range + value = [center - value, center + value] + if non_negative: + value[0] = max(0, value[0]) + elif isinstance(value, (list, tuple)) and len(value) == 2: + if not bound[0] <= value[0] <= value[1] <= bound[1]: + raise ValueError("Please check your value range of {} is valid and " + "within the bound {}".format(input_name, bound)) + else: + raise TypeError("Input of {} should be either a single value, or a list/tuple of " + "length 2.".format(input_name)) + factor = random.uniform(value[0], value[1]) + return factor + + brightness_factor = _input_to_factor(brightness, 'brightness') + contrast_factor = _input_to_factor(contrast, 'contrast') + saturation_factor = _input_to_factor(saturation, 'saturation') + hue_factor = _input_to_factor(hue, 'hue', center=0, bound=(-0.5, 0.5), non_negative=False) + + transforms = [] + transforms.append(lambda img: adjust_brightness(img, brightness_factor)) + transforms.append(lambda img: adjust_contrast(img, contrast_factor)) + transforms.append(lambda img: adjust_saturation(img, saturation_factor)) + transforms.append(lambda img: adjust_hue(img, hue_factor)) + + # apply color adjustments in a random order + random.shuffle(transforms) + for transform in transforms: + img = transform(img) + + return img + + +def random_rotation(img, degrees, resample, expand, center, fill_value): + """ + Rotate the input PIL image by a random angle. + + Args: + img (PIL Image): Image to be rotated. + degrees (int or float or sequence): Range of random rotation degrees. + If degrees is a number, the range will be converted to (-degrees, degrees). + If degrees is a sequence, it should be (min, max). + resample (Inter.NEAREST, or Inter.BILINEAR, Inter.BICUBIC, optional): An optional resampling filter. + If omitted, or if the image has mode "1" or "P", it is set to be Inter.NEAREST. + expand (bool, optional): Optional expansion flag. If set to True, expand the output + image to make it large enough to hold the entire rotated image. + If set to False or omitted, make the output image the same size as the input. + Note that the expand flag assumes rotation around the center and no translation. + center (tuple, optional): Optional center of rotation (a 2-tuple). + Origin is the top left corner. + fill_value (int or tuple): Optional fill color for the area outside the rotated image. + If it is a 3-tuple, it is used for R, G, B channels respectively. + If it is an int, it is used for all RGB channels. + + Returns: + img (PIL Image), Rotated image. + + https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.rotate + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if isinstance(degrees, numbers.Number): + if degrees < 0: + raise ValueError("If degrees is a single number, it cannot be negative.") + degrees = (-degrees, degrees) + elif isinstance(degrees, (list, tuple)): + if len(degrees) != 2: + raise ValueError("If degrees is a sequence, the length must be 2.") + else: + raise TypeError("Degrees must be a single non-negative number or a sequence") + + angle = random.uniform(degrees[0], degrees[1]) + return rotate(img, angle, resample, expand, center, fill_value) + + +def random_order(img, transforms): + """ + Applies a list of transforms in a random order. + + Args: + img: Image to be applied transformations in a random order. + transforms (list): List of the transformations to be applied. + + Returns: + img, Transformed image. + """ + random.shuffle(transforms) + for transform in transforms: + img = transform(img) + return img + + +def random_apply(img, transforms, prob): + """ + Apply a list of transformation, randomly with a given probability. + + Args: + img: Image to be randomly applied a list transformations. + transforms (list): List of transformations to be applied. + prob (float): The probability to apply the transformation list. + + Returns: + img, Transformed image. + """ + if prob < random.random(): + return img + for transform in transforms: + img = transform(img) + return img + + +def random_choice(img, transforms): + """ + Random selects one transform from a list of transforms and applies that on the image. + + Args: + img: Image to be applied transformation. + transforms (list): List of transformations to be chosen from to apply. + + Returns: + img, Transformed image. + """ + return random.choice(transforms)(img) + + +def five_crop(img, size): + """ + Generate 5 cropped images (one central and four corners). + + Args: + img (PIL Image): PIL Image to be cropped. + size (int or sequence): The output size of the crop. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + + Returns: + img_tuple (tuple), a tuple of 5 PIL images + (top_left, top_right, bottom_left, bottom_right, center). + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if isinstance(size, int): + size = (size, size) + elif isinstance(size, (tuple, list)) and len(size) == 2: + size = size + else: + raise TypeError("Size should be a single number or a list/tuple (h, w) of length 2.") + + # PIL Image.size returns in (width, height) order + img_width, img_height = img.size + crop_height, crop_width = size + if crop_height > img_height or crop_width > img_width: + raise ValueError("Crop size {} is larger than input image size {}".format(size, (img_height, img_width))) + center = center_crop(img, (crop_height, crop_width)) + top_left = img.crop((0, 0, crop_width, crop_height)) + top_right = img.crop((img_width - crop_width, 0, img_width, crop_height)) + bottom_left = img.crop((0, img_height - crop_height, crop_width, img_height)) + bottom_right = img.crop((img_width - crop_width, img_height - crop_height, img_width, img_height)) + + return top_left, top_right, bottom_left, bottom_right, center + + +def ten_crop(img, size, use_vertical_flip=False): + """ + Generate 10 cropped images (first 5 from FiveCrop, second 5 from their flipped version). + + The default is horizontal flipping, use_vertical_flip=False. + + Args: + img (PIL Image): PIL Image to be cropped. + size (int or sequence): The output size of the crop. + If size is an int, a square crop of size (size, size) is returned. + If size is a sequence of length 2, it should be (height, width). + use_vertical_flip (bool): Flip the image vertically instead of horizontally if set to True. + + Returns: + img_tuple (tuple), a tuple of 10 PIL images + (top_left, top_right, bottom_left, bottom_right, center) of original image + + (top_left, top_right, bottom_left, bottom_right, center) of flipped image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if isinstance(size, int): + size = (size, size) + elif isinstance(size, (tuple, list)) and len(size) == 2: + size = size + else: + raise TypeError("Size should be a single number or a list/tuple (h, w) of length 2.") + + first_five_crop = five_crop(img, size) + + if use_vertical_flip: + img = vertical_flip(img) + else: + img = horizontal_flip(img) + + second_five_crop = five_crop(img, size) + + return first_five_crop + second_five_crop + + +def grayscale(img, num_output_channels): + """ + Convert the input PIL image to grayscale image. + + Args: + img (PIL Image): PIL image to be converted to grayscale. + num_output_channels (int): Number of channels of the output grayscale image (1 or 3). + + Returns: + img (PIL Image), grayscaled image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if num_output_channels == 1: + img = img.convert('L') + elif num_output_channels == 3: + # each channel is the same grayscale layer + img = img.convert('L') + np_gray = np.array(img, dtype=np.uint8) + np_img = np.dstack([np_gray, np_gray, np_gray]) + img = Image.fromarray(np_img, 'RGB') + else: + raise ValueError('num_output_channels should be either 1 or 3. Got {}'.format(num_output_channels)) + + return img + + +def pad(img, padding, fill_value, padding_mode): + """ + Pads the image according to padding parameters. + + Args: + img (PIL Image): Image to be padded. + padding (int or sequence, optional): The number of pixels to pad the image. + If a single number is provided, it pads all borders with this value. + If a tuple or list of 2 values are provided, it pads the (left and top) + with the first value and (right and bottom) with the second value. + If 4 values are provided as a list or tuple, + it pads the left, top, right and bottom respectively. + Default is None. + fill_value (int or tuple): The pixel intensity of the borders if + the padding_mode is "constant". If it is a 3-tuple, it is used to + fill R, G, B channels respectively. + padding_mode (str): The method of padding. Can be any of + ['constant', 'edge', 'reflect', 'symmetric']. + - 'constant', means it fills the border with constant values + - 'edge', means it pads with the last value on the edge + - 'reflect', means it reflects the values on the edge omitting the last + value of edge + - 'symmetric', means it reflects the values on the edge repeating the last + value of edge + + Returns: + img (PIL Image), Padded image. + """ + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + if isinstance(padding, numbers.Number): + top = bottom = left = right = padding + + elif isinstance(padding, (tuple, list)): + if len(padding) == 2: + left = right = padding[0] + top = bottom = padding[1] + elif len(padding) == 4: + left = padding[0] + top = padding[1] + right = padding[2] + bottom = padding[3] + else: + raise ValueError("The size of the padding list or tuple should be 2 or 4.") + else: + raise TypeError("Padding can be any of: a number, a tuple or list of size 2 or 4.") + + if not isinstance(fill_value, (numbers.Number, str, tuple)): + raise TypeError("fill_value can be any of: an integer, a string or a tuple.") + + if padding_mode not in ['constant', 'edge', 'reflect', 'symmetric']: + raise ValueError("Padding mode can be any of ['constant', 'edge', 'reflect', 'symmetric'].") + + if padding_mode == 'constant': + if img.mode == 'P': + palette = img.getpalette() + image = ImageOps.expand(img, border=padding, fill=fill_value) + image.putpalette(palette) + return image + return ImageOps.expand(img, border=padding, fill=fill_value) + + if img.mode == 'P': + palette = img.getpalette() + img = np.asarray(img) + img = np.pad(img, ((top, bottom), (left, right)), padding_mode) + img = Image.fromarray(img) + img.putpalette(palette) + return img + + img = np.asarray(img) + if len(img.shape) == 3: + img = np.pad(img, ((top, bottom), (left, right), (0, 0)), padding_mode) + if len(img.shape) == 2: + img = np.pad(img, ((top, bottom), (left, right)), padding_mode) + + return Image.fromarray(img) + + +def get_perspective_params(img, distortion_scale): + """Helper function to get parameters for RandomPerspective. + """ + img_width, img_height = img.size + distorted_half_width = int(img_width / 2 * distortion_scale) + distorted_half_height = int(img_height / 2 * distortion_scale) + top_left = (random.randint(0, distorted_half_width), + random.randint(0, distorted_half_height)) + top_right = (random.randint(img_width - distorted_half_width - 1, img_width - 1), + random.randint(0, distorted_half_height)) + bottom_right = (random.randint(img_width - distorted_half_width - 1, img_width - 1), + random.randint(img_height - distorted_half_height - 1, img_height - 1)) + bottom_left = (random.randint(0, distorted_half_width), + random.randint(img_height - distorted_half_height - 1, img_height - 1)) + + start_points = [(0, 0), (img_width - 1, 0), (img_width - 1, img_height - 1), (0, img_height - 1)] + end_points = [top_left, top_right, bottom_right, bottom_left] + return start_points, end_points + + +def perspective(img, start_points, end_points, interpolation=Inter.BICUBIC): + """ + Apply perspective transformation to the input PIL Image. + + Args: + img (PIL Image): PIL Image to be applied perspective transformation. + start_points (list): List of [top_left, top_right, bottom_right, bottom_left] of the original image. + end_points: List of [top_left, top_right, bottom_right, bottom_left] of the transformed image. + interpolation (interpolation mode): Image interpolation mode, Default is Inter.BICUBIC = 3. + + Returns: + img (PIL Image), Image after being perspectively transformed. + """ + + def _input_to_coeffs(original_points, transformed_points): + # Get the coefficients (a, b, c, d, e, f, g, h) for the perspective transforms. + # According to "Using Projective Geometry to Correct a Camera" from AMS. + # http://www.ams.org/publicoutreach/feature-column/fc-2013-03 + # https://github.com/python-pillow/Pillow/blob/master/src/libImaging/Geometry.c#L377 + + matrix = [] + for pt1, pt2 in zip(transformed_points, original_points): + matrix.append([pt1[0], pt1[1], 1, 0, 0, 0, -pt2[0] * pt1[0], -pt2[0] * pt1[1]]) + matrix.append([0, 0, 0, pt1[0], pt1[1], 1, -pt2[1] * pt1[0], -pt2[1] * pt1[1]]) + matrix_a = np.array(matrix, dtype=np.float) + matrix_b = np.array(original_points, dtype=np.float).reshape(8) + res = np.linalg.lstsq(matrix_a, matrix_b, rcond=None)[0] + return res.tolist() + + if not is_pil(img): + raise TypeError(augment_error_message.format(type(img))) + + coeffs = _input_to_coeffs(start_points, end_points) + return img.transform(img.size, Image.PERSPECTIVE, coeffs, interpolation) + + +def get_erase_params(np_img, scale, ratio, value, bounded, max_attempts): + """Helper function to get parameters for RandomErasing/ Cutout. + """ + if not is_numpy(np_img): + raise TypeError('img should be Numpy array. Got {}'.format(type(np_img))) + + image_c, image_h, image_w = np_img.shape + area = image_h * image_w + + for _ in range(max_attempts): + erase_area = random.uniform(scale[0], scale[1]) * area + aspect_ratio = random.uniform(ratio[0], ratio[1]) + erase_w = int(round(math.sqrt(erase_area * aspect_ratio))) + erase_h = int(round(erase_w / aspect_ratio)) + erase_shape = (image_c, erase_h, erase_w) + + if erase_h < image_h and erase_w < image_w: + if bounded: + i = random.randint(0, image_h - erase_h) + j = random.randint(0, image_w - erase_w) + else: + def clip(x, lower, upper): + return max(lower, min(x, upper)) + + x = random.randint(0, image_w) + y = random.randint(0, image_h) + x1 = clip(x - erase_w // 2, 0, image_w) + x2 = clip(x + erase_w // 2, 0, image_w) + y1 = clip(y - erase_h // 2, 0, image_h) + y2 = clip(y + erase_h // 2, 0, image_h) + + i, j, erase_h, erase_w = y1, x1, y2 - y1, x2 - x1 + + if isinstance(value, numbers.Number): + erase_value = value + elif isinstance(value, (str, bytes)): + erase_value = np.random.normal(loc=0.0, scale=1.0, size=erase_shape) + elif isinstance(value, (tuple, list)) and len(value) == 3: + value = np.array(value) + erase_value = np.multiply(np.ones(erase_shape), value[:, None, None]) + else: + raise ValueError("The value for erasing should be either a single value, or a string " + "'random', or a sequence of 3 elements for RGB respectively.") + + return i, j, erase_h, erase_w, erase_value + + # exceeding max_attempts, return original image + return 0, 0, image_h, image_w, np_img + + +def erase(np_img, i, j, height, width, erase_value, inplace=False): + """ + Erase the pixels, within a selected rectangle region, to the given value. Applied on the input Numpy image array. + + Args: + np_img (numpy.ndarray): Numpy image array of shape (C, H, W) to be erased. + i (int): The height component of the top left corner (height, width). + j (int): The width component of the top left corner (height, width). + height (int): Height of the erased region. + width (int): Width of the erased region. + erase_value: Erase value return from helper function get_erase_params(). + inplace (bool, optional): Apply this transform inplace. Default is False. + + Returns: + np_img (numpy.ndarray), Erased Numpy image array. + """ + if not is_numpy(np_img): + raise TypeError('img should be Numpy array. Got {}'.format(type(np_img))) + + if not inplace: + np_img = np_img.copy() + # (i, j) here are the coordinates of axes (height, width) as in CHW + np_img[:, i:i + height, j:j + width] = erase_value + return np_img + + +def linear_transform(np_img, transformation_matrix, mean_vector): + """ + Apply linear transformation to the input Numpy image array, given a square transformation matrix and a mean_vector. + + The transformation first flattens the input array and subtract mean_vector from it, then computes the + dot product with the transformation matrix, and reshapes it back to its original shape. + + Args: + np_img (numpy.ndarray): Numpy image array of shape (C, H, W) to be linear transformed. + transformation_matrix (numpy.ndarray): a square transformation matrix of shape (D, D), D = C x H x W. + mean_vector (numpy.ndarray): a numpy ndarray of shape (D,) where D = C x H x W. + + Returns: + np_img (numpy.ndarray), Linear transformed image. + """ + if not is_numpy(np_img): + raise TypeError('img should be Numpy array. Got {}'.format(type(np_img))) + if transformation_matrix.shape[0] != transformation_matrix.shape[1]: + raise ValueError("transformation_matrix should be a square matrix. " + "Got shape {} instead".format(transformation_matrix.shape)) + if np.prod(np_img.shape) != transformation_matrix.shape[0]: + raise ValueError("transformation_matrix shape {0} not compatible with " + "Numpy Image shape {1}.".format(transformation_matrix.shape, np_img.shape)) + if mean_vector.shape[0] != transformation_matrix.shape[0]: + raise ValueError("mean_vector length {0} should match either one dimension of the square " + "transformation_matrix {1}.".format(mean_vector.shape[0], transformation_matrix.shape)) + zero_centered_img = np_img.reshape(1, -1) - mean_vector + transformed_img = np.dot(zero_centered_img, transformation_matrix).reshape(np_img.shape) + return transformed_img + + +def random_affine(img, angle, translations, scale, shear, resample, fill_value=0): + """ + Applies a random Affine transformation on the input PIL image. + + Args: + img (PIL Image): Image to be applied affine transformation. + angle (int or float): Rotation angle in degrees, clockwise. + translations (sequence): Translations in horizontal and vertical axis. + scale (float): Scale parameter, a single number. + shear (float or sequence): Shear amount parallel to x and y axis. + resample (Inter.NEAREST, or Inter.BILINEAR, Inter.BICUBIC, optional): An optional resampling filter. + fill_value (tuple or int, optional): Optional fill_value to fill the area outside the transform + in the output image. Used only in Pillow versions > 5.0.0. + If None, no filling is performed. + + Returns: + img (PIL Image), Randomly affine transformed image. + + """ + if not is_pil(img): + raise ValueError("Input image should be a Pillow image.") + + output_size = img.size + center = (img.size[0] * 0.5 + 0.5, img.size[1] * 0.5 + 0.5) + + angle = math.radians(angle) + if isinstance(shear, (tuple, list)) and len(shear) == 2: + shear = [math.radians(s) for s in shear] + elif isinstance(shear, numbers.Number): + shear = math.radians(shear) + shear = [shear, 0] + else: + raise ValueError( + "Shear should be a single value or a tuple/list containing " + + "two values. Got {}".format(shear)) + + scale = 1.0 / scale + + # Inverted rotation matrix with scale and shear + d = math.cos(angle + shear[0]) * math.cos(angle + shear[1]) + \ + math.sin(angle + shear[0]) * math.sin(angle + shear[1]) + matrix = [ + math.cos(angle + shear[0]), math.sin(angle + shear[0]), 0, + -math.sin(angle + shear[1]), math.cos(angle + shear[1]), 0 + ] + matrix = [scale / d * m for m in matrix] + + # Apply inverse of translation and of center translation: RSS^-1 * C^-1 * T^-1 + matrix[2] += matrix[0] * (-center[0] - translations[0]) + matrix[1] * (-center[1] - translations[1]) + matrix[5] += matrix[3] * (-center[0] - translations[0]) + matrix[4] * (-center[1] - translations[1]) + + # Apply center translation: C * RSS^-1 * C^-1 * T^-1 + matrix[2] += center[0] + matrix[5] += center[1] + + if __version__ >= '5': + kwargs = {"fillcolor": fill_value} + else: + kwargs = {} + return img.transform(output_size, Image.AFFINE, matrix, resample, **kwargs) + + +def one_hot_encoding(label, num_classes, epsilon): + """ + Apply label smoothing transformation to the input label, and make label be more smoothing and continuous. + + Args: + label (numpy.ndarray): label to be applied label smoothing. + num_classes (int): Num class of object in dataset, value should over 0. + epsilon (float): The adjustable Hyper parameter. Default is 0.0. + + Returns: + img (numpy.ndarray), label after being one hot encoded and done label smoothed. + """ + if label > num_classes: + raise ValueError('the num_classes is smaller than the category number.') + + num_elements = label.size + one_hot_label = np.zeros((num_elements, num_classes), dtype=int) + + if isinstance(label, list) is False: + label = [label] + for index in range(num_elements): + one_hot_label[index, label[index]] = 1 + + return (1 - epsilon) * one_hot_label + epsilon / num_classes + + +def mix_up_single(batch_size, img, label, alpha=0.2): + """ + Apply mix up transformation to image and label in single batch internal, One hot encoding should done before this. + + Args: + batch_size (int): the batch size of dataset. + img (numpy.ndarray): numpy Image to be applied mix up transformation. + label (numpy.ndarray): numpy label to be applied mix up transformation. + alpha (float): the mix up rate. + + Returns: + mix_img (numpy.ndarray): numpy Image after being applied mix up transformation. + mix_label (numpy.ndarray): numpy label after being applied mix up transformation. + """ + + def cir_shift(data): + index = list(range(1, batch_size)) + [0] + data = data[index, ...] + return data + + lam = np.random.beta(alpha, alpha, batch_size) + lam_img = lam.reshape((batch_size, 1, 1, 1)) + mix_img = lam_img * img + (1 - lam_img) * cir_shift(img) + + lam_label = lam.reshape((batch_size, 1)) + mix_label = lam_label * label + (1 - lam_label) * cir_shift(label) + + return mix_img, mix_label + + +def mix_up_muti(tmp, batch_size, img, label, alpha=0.2): + """ + Apply mix up transformation to image and label in continuous batch, one hot encoding should done before this. + + Args: + tmp (class object): mainly for saving the tmp parameter. + batch_size (int): the batch size of dataset. + img (numpy.ndarray): numpy Image to be applied mix up transformation. + label (numpy.ndarray): numpy label to be applied mix up transformation. + alpha (float): refer to the mix up rate. + + Returns: + mix_img (numpy.ndarray): numpy Image after being applied mix up transformation. + mix_label (numpy.ndarray): numpy label after being applied mix up transformation. + """ + lam = np.random.beta(alpha, alpha, batch_size) + if tmp.is_first: + lam = np.ones(batch_size) + tmp.is_first = False + + lam_img = lam.reshape((batch_size, 1, 1, 1)) + mix_img = lam_img * img + (1 - lam_img) * tmp.image + + lam_label = lam.reshape(batch_size, 1) + mix_label = lam_label * label + (1 - lam_label) * tmp.label + tmp.image = mix_img + tmp.label = mix_label + + return mix_img, mix_label + + +def rgb_to_hsv(np_rgb_img, is_hwc): + """ + Convert RGB img to HSV img. + + Args: + np_rgb_img (numpy.ndarray): Numpy RGB image array of shape (H, W, C) or (C, H, W) to be converted. + is_hwc (Bool): If True, the shape of np_hsv_img is (H, W, C), otherwise must be (C, H, W). + + Returns: + np_hsv_img (numpy.ndarray), Numpy HSV image with same type of np_rgb_img. + """ + if is_hwc: + r, g, b = np_rgb_img[:, :, 0], np_rgb_img[:, :, 1], np_rgb_img[:, :, 2] + else: + r, g, b = np_rgb_img[0, :, :], np_rgb_img[1, :, :], np_rgb_img[2, :, :] + to_hsv = np.vectorize(colorsys.rgb_to_hsv) + h, s, v = to_hsv(r, g, b) + if is_hwc: + axis = 2 + else: + axis = 0 + np_hsv_img = np.stack((h, s, v), axis=axis) + return np_hsv_img + + +def rgb_to_hsvs(np_rgb_imgs, is_hwc): + """ + Convert RGB imgs to HSV imgs. + + Args: + np_rgb_imgs (numpy.ndarray): Numpy RGB images array of shape (H, W, C) or (N, H, W, C), + or (C, H, W) or (N, C, H, W) to be converted. + is_hwc (Bool): If True, the shape of np_rgb_imgs is (H, W, C) or (N, H, W, C); + If False, the shape of np_rgb_imgs is (C, H, W) or (N, C, H, W). + + Returns: + np_hsv_imgs (numpy.ndarray), Numpy HSV images with same type of np_rgb_imgs. + """ + if not is_numpy(np_rgb_imgs): + raise TypeError('img should be Numpy Image. Got {}'.format(type(np_rgb_imgs))) + + shape_size = len(np_rgb_imgs.shape) + + if not shape_size in (3, 4): + raise TypeError('img shape should be (H, W, C)/(N, H, W, C)/(C,H,W)/(N,C,H,W). \ + Got {}'.format(np_rgb_imgs.shape)) + + if shape_size == 3: + batch_size = 0 + if is_hwc: + num_channels = np_rgb_imgs.shape[2] + else: + num_channels = np_rgb_imgs.shape[0] + else: + batch_size = np_rgb_imgs.shape[0] + if is_hwc: + num_channels = np_rgb_imgs.shape[3] + else: + num_channels = np_rgb_imgs.shape[1] + + if num_channels != 3: + raise TypeError('img should be 3 channels RGB img. Got {} channels'.format(num_channels)) + if batch_size == 0: + return rgb_to_hsv(np_rgb_imgs, is_hwc) + return np.array([rgb_to_hsv(img, is_hwc) for img in np_rgb_imgs]) + + +def hsv_to_rgb(np_hsv_img, is_hwc): + """ + Convert HSV img to RGB img. + + Args: + np_hsv_img (numpy.ndarray): Numpy HSV image array of shape (H, W, C) or (C, H, W) to be converted. + is_hwc (Bool): If True, the shape of np_hsv_img is (H, W, C), otherwise must be (C, H, W). + + Returns: + np_rgb_img (numpy.ndarray), Numpy HSV image with same shape of np_hsv_img. + """ + if is_hwc: + h, s, v = np_hsv_img[:, :, 0], np_hsv_img[:, :, 1], np_hsv_img[:, :, 2] + else: + h, s, v = np_hsv_img[0, :, :], np_hsv_img[1, :, :], np_hsv_img[2, :, :] + to_rgb = np.vectorize(colorsys.hsv_to_rgb) + r, g, b = to_rgb(h, s, v) + + if is_hwc: + axis = 2 + else: + axis = 0 + np_rgb_img = np.stack((r, g, b), axis=axis) + return np_rgb_img + + +def hsv_to_rgbs(np_hsv_imgs, is_hwc): + """ + Convert HSV imgs to RGB imgs. + + Args: + np_hsv_imgs (numpy.ndarray): Numpy HSV images array of shape (H, W, C) or (N, H, W, C), + or (C, H, W) or (N, C, H, W) to be converted. + is_hwc (Bool): If True, the shape of np_hsv_imgs is (H, W, C) or (N, H, W, C); + If False, the shape of np_hsv_imgs is (C, H, W) or (N, C, H, W). + + Returns: + np_rgb_imgs (numpy.ndarray), Numpy RGB images with same type of np_hsv_imgs. + """ + if not is_numpy(np_hsv_imgs): + raise TypeError('img should be Numpy Image. Got {}'.format(type(np_hsv_imgs))) + + shape_size = len(np_hsv_imgs.shape) + + if not shape_size in (3, 4): + raise TypeError('img shape should be (H, W, C)/(N, H, W, C)/(C,H,W)/(N,C,H,W). \ + Got {}'.format(np_hsv_imgs.shape)) + + if shape_size == 3: + batch_size = 0 + if is_hwc: + num_channels = np_hsv_imgs.shape[2] + else: + num_channels = np_hsv_imgs.shape[0] + else: + batch_size = np_hsv_imgs.shape[0] + if is_hwc: + num_channels = np_hsv_imgs.shape[3] + else: + num_channels = np_hsv_imgs.shape[1] + + if num_channels != 3: + raise TypeError('img should be 3 channels RGB img. Got {} channels'.format(num_channels)) + if batch_size == 0: + return hsv_to_rgb(np_hsv_imgs, is_hwc) + return np.array([hsv_to_rgb(img, is_hwc) for img in np_hsv_imgs]) diff --git a/mindspore/dataset/transforms/vision/utils.py b/mindspore/dataset/transforms/vision/utils.py new file mode 100644 index 00000000000..1abc66a4677 --- /dev/null +++ b/mindspore/dataset/transforms/vision/utils.py @@ -0,0 +1,32 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Interpolation Mode, Resampling Filters +""" +from enum import Enum, IntEnum + + +class Inter(IntEnum): + NEAREST = 0 + BILINEAR = LINEAR = 1 + BICUBIC = CUBIC = 2 + + +# Padding Mode, Border Type +# Note: This class derived from class str to support json serializable. +class Border(str, Enum): + CONSTANT: str = "constant" + EDGE: str = "edge" + REFLECT: str = "reflect" + SYMMETRIC: str = "symmetric" diff --git a/mindspore/dataset/transforms/vision/validators.py b/mindspore/dataset/transforms/vision/validators.py new file mode 100644 index 00000000000..17f50e068d4 --- /dev/null +++ b/mindspore/dataset/transforms/vision/validators.py @@ -0,0 +1,809 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Validators for TensorOps. +""" +import numbers +from functools import wraps + +from .utils import Inter, Border +from ...transforms.validators import check_pos_int32, check_pos_float32, check_value, check_uint8, FLOAT_MAX_INTEGER, \ + check_bool, check_2tuple, check_range, check_list, check_type, check_positive, INT32_MAX + + +def check_inter_mode(mode): + if not isinstance(mode, Inter): + raise ValueError("Invalid interpolation mode.") + + +def check_border_type(mode): + if not isinstance(mode, Border): + raise ValueError("Invalid padding mode.") + + +def check_crop_size(size): + """Wrapper method to check the parameters of crop size.""" + if isinstance(size, int): + size = (size, size) + elif isinstance(size, (tuple, list)) and len(size) == 2: + size = size + else: + raise TypeError("Size should be a single integer or a list/tuple (h, w) of length 2.") + for value in size: + check_value(value, (1, INT32_MAX)) + return size + + +def check_resize_size(size): + """Wrapper method to check the parameters of resize.""" + if isinstance(size, int): + check_pos_int32(size) + elif isinstance(size, (tuple, list)) and len(size) == 2: + for value in size: + check_value(value, (1, INT32_MAX)) + else: + raise TypeError("Size should be a single integer or a list/tuple (h, w) of length 2.") + return size + + +def check_normalize_c_param(mean, std): + if len(mean) != len(std): + raise ValueError("Length of mean and std must be equal") + for mean_value in mean: + check_pos_float32(mean_value) + for std_value in std: + check_pos_float32(std_value) + + +def check_normalize_py_param(mean, std): + if len(mean) != len(std): + raise ValueError("Length of mean and std must be equal") + for mean_value in mean: + check_value(mean_value, [0., 1.]) + for std_value in std: + check_value(std_value, [0., 1.]) + + +def check_fill_value(fill_value): + if isinstance(fill_value, int): + check_uint8(fill_value) + elif isinstance(fill_value, tuple) and len(fill_value) == 3: + for value in fill_value: + check_uint8(value) + else: + raise TypeError("fill_value should be a single integer or a 3-tuple.") + return fill_value + + +def check_padding(padding): + """Parsing the padding arguments and check if it is legal.""" + if isinstance(padding, numbers.Number): + top = bottom = left = right = padding + + elif isinstance(padding, (tuple, list)): + if len(padding) == 2: + left = right = padding[0] + top = bottom = padding[1] + elif len(padding) == 4: + left = padding[0] + top = padding[1] + right = padding[2] + bottom = padding[3] + else: + raise ValueError("The size of the padding list or tuple should be 2 or 4.") + else: + raise TypeError("Padding can be any of: a number, a tuple or list of size 2 or 4.") + return left, top, right, bottom + + +def check_degrees(degrees): + """Check if the degrees is legal.""" + if isinstance(degrees, numbers.Number): + if degrees < 0: + raise ValueError("If degrees is a single number, it cannot be negative.") + degrees = (-degrees, degrees) + elif isinstance(degrees, (list, tuple)): + if len(degrees) != 2: + raise ValueError("If degrees is a sequence, the length must be 2.") + else: + raise TypeError("Degrees must be a single non-negative number or a sequence") + return degrees + + +def check_random_color_adjust_param(value, input_name, center=1, bound=(0, FLOAT_MAX_INTEGER), non_negative=True): + """Check the parameters in random color adjust operation.""" + if isinstance(value, numbers.Number): + if value < 0: + raise ValueError("The input value of {} cannot be negative.".format(input_name)) + # convert value into a range + value = [center - value, center + value] + if non_negative: + value[0] = max(0, value[0]) + elif isinstance(value, (list, tuple)) and len(value) == 2: + if not bound[0] <= value[0] <= value[1] <= bound[1]: + raise ValueError("Please check your value range of {} is valid and " + "within the bound {}".format(input_name, bound)) + else: + raise TypeError("Input of {} should be either a single value, or a list/tuple of " + "length 2.".format(input_name)) + factor = (value[0], value[1]) + return factor + + +def check_erasing_value(value): + if not (isinstance(value, (numbers.Number, str, bytes)) or + (isinstance(value, (tuple, list)) and len(value) == 3)): + raise ValueError("The value for erasing should be either a single value, " + "or a string 'random', or a sequence of 3 elements for RGB respectively.") + + +def check_crop(method): + """A wrapper that wrap a parameter checker to the original function(crop operation).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + size = (list(args) + [None])[0] + if "size" in kwargs: + size = kwargs.get("size") + if size is None: + raise ValueError("size is not provided.") + size = check_crop_size(size) + kwargs["size"] = size + + return method(self, **kwargs) + + return new_method + + +def check_resize_interpolation(method): + """A wrapper that wrap a parameter checker to the original function(resize interpolation operation).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + size, interpolation = args + if "size" in kwargs: + size = kwargs.get("size") + if "interpolation" in kwargs: + interpolation = kwargs.get("interpolation") + + if size is None: + raise ValueError("size is not provided.") + size = check_resize_size(size) + kwargs["size"] = size + + if interpolation is not None: + check_inter_mode(interpolation) + kwargs["interpolation"] = interpolation + + return method(self, **kwargs) + + return new_method + + +def check_resize(method): + """A wrapper that wrap a parameter checker to the original function(resize operation).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + size = (list(args) + [None])[0] + if "size" in kwargs: + size = kwargs.get("size") + + if size is None: + raise ValueError("size is not provided.") + size = check_resize_size(size) + kwargs["size"] = size + + return method(self, **kwargs) + + return new_method + + +def check_random_resize_crop(method): + """A wrapper that wrap a parameter checker to the original function(random resize crop operation).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 5 * [None])[:5] + size, scale, ratio, interpolation, max_attempts = args + if "size" in kwargs: + size = kwargs.get("size") + if "scale" in kwargs: + scale = kwargs.get("scale") + if "ratio" in kwargs: + ratio = kwargs.get("ratio") + if "interpolation" in kwargs: + interpolation = kwargs.get("interpolation") + if "max_attempts" in kwargs: + max_attempts = kwargs.get("max_attempts") + + if size is None: + raise ValueError("size is not provided.") + size = check_crop_size(size) + kwargs["size"] = size + + if scale is not None: + check_range(scale, [0, FLOAT_MAX_INTEGER]) + kwargs["scale"] = scale + if ratio is not None: + check_range(ratio, [0, FLOAT_MAX_INTEGER]) + kwargs["ratio"] = ratio + if interpolation is not None: + check_inter_mode(interpolation) + kwargs["interpolation"] = interpolation + if max_attempts is not None: + check_pos_int32(max_attempts) + kwargs["max_attempts"] = max_attempts + + return method(self, **kwargs) + + return new_method + + +def check_prob(method): + """A wrapper that wrap a parameter checker(check the probability) to the original function.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + prob = (list(args) + [None])[0] + if "prob" in kwargs: + prob = kwargs.get("prob") + if prob is not None: + check_value(prob, [0., 1.]) + kwargs["prob"] = prob + + return method(self, **kwargs) + + return new_method + + +def check_normalize_c(method): + """A wrapper that wrap a parameter checker to the original function(normalize operation written in C++).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + mean, std = args + if "mean" in kwargs: + mean = kwargs.get("mean") + if "std" in kwargs: + std = kwargs.get("std") + + if mean is None: + raise ValueError("mean is not provided.") + if std is None: + raise ValueError("std is not provided.") + check_normalize_c_param(mean, std) + kwargs["mean"] = mean + kwargs["std"] = std + + return method(self, **kwargs) + + return new_method + + +def check_normalize_py(method): + """A wrapper that wrap a parameter checker to the original function(normalize operation written in Python).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + mean, std = args + if "mean" in kwargs: + mean = kwargs.get("mean") + if "std" in kwargs: + std = kwargs.get("std") + + if mean is None: + raise ValueError("mean is not provided.") + if std is None: + raise ValueError("std is not provided.") + check_normalize_py_param(mean, std) + kwargs["mean"] = mean + kwargs["std"] = std + + return method(self, **kwargs) + + return new_method + + +def check_random_crop(method): + """Wrapper method to check the parameters of random crop.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 4 * [None])[:5] + size, padding, pad_if_needed, fill_value, padding_mode = args + + if "size" in kwargs: + size = kwargs.get("size") + if "padding" in kwargs: + padding = kwargs.get("padding") + if "fill_value" in kwargs: + fill_value = kwargs.get("fill_value") + if "padding_mode" in kwargs: + padding_mode = kwargs.get("padding_mode") + if "pad_if_needed" in kwargs: + pad_if_needed = kwargs.get("pad_if_needed") + + if size is None: + raise ValueError("size is not provided.") + size = check_crop_size(size) + kwargs["size"] = size + + if padding is not None: + padding = check_padding(padding) + kwargs["padding"] = padding + if fill_value is not None: + fill_value = check_fill_value(fill_value) + kwargs["fill_value"] = fill_value + if padding_mode is not None: + check_border_type(padding_mode) + kwargs["padding_mode"] = padding_mode + if pad_if_needed is not None: + kwargs["pad_if_needed"] = pad_if_needed + + return method(self, **kwargs) + + return new_method + + +def check_random_color_adjust(method): + """Wrapper method to check the parameters of random color adjust.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 4 * [None])[:4] + brightness, contrast, saturation, hue = args + if "brightness" in kwargs: + brightness = kwargs.get("brightness") + if "contrast" in kwargs: + contrast = kwargs.get("contrast") + if "saturation" in kwargs: + saturation = kwargs.get("saturation") + if "hue" in kwargs: + hue = kwargs.get("hue") + + if brightness is not None: + kwargs["brightness"] = check_random_color_adjust_param(brightness, "brightness") + if contrast is not None: + kwargs["contrast"] = check_random_color_adjust_param(contrast, "contrast") + if saturation is not None: + kwargs["saturation"] = check_random_color_adjust_param(saturation, "saturation") + if hue is not None: + kwargs["hue"] = check_random_color_adjust_param(hue, 'hue', center=0, bound=(-0.5, 0.5), non_negative=False) + + return method(self, **kwargs) + + return new_method + + +def check_random_rotation(method): + """Wrapper method to check the parameters of random rotation.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 5 * [None])[:5] + degrees, resample, expand, center, fill_value = args + if "degrees" in kwargs: + degrees = kwargs.get("degrees") + if "resample" in kwargs: + resample = kwargs.get("resample") + if "expand" in kwargs: + expand = kwargs.get("expand") + if "center" in kwargs: + center = kwargs.get("center") + if "fill_value" in kwargs: + fill_value = kwargs.get("fill_value") + + if degrees is None: + raise ValueError("degrees is not provided.") + degrees = check_degrees(degrees) + kwargs["degrees"] = degrees + + if resample is not None: + check_inter_mode(resample) + kwargs["resample"] = resample + if expand is not None: + check_bool(expand) + kwargs["expand"] = expand + if center is not None: + check_2tuple(center) + kwargs["center"] = center + if fill_value is not None: + fill_value = check_fill_value(fill_value) + kwargs["fill_value"] = fill_value + + return method(self, **kwargs) + + return new_method + + +def check_transforms_list(method): + """Wrapper method to check the parameters of transform list.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + transforms = (list(args) + [None])[0] + if "transforms" in kwargs: + transforms = kwargs.get("transforms") + if transforms is None: + raise ValueError("transforms is not provided.") + + check_list(transforms) + kwargs["transforms"] = transforms + + return method(self, **kwargs) + + return new_method + + +def check_random_apply(method): + """Wrapper method to check the parameters of random apply.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + transforms, prob = (list(args) + 2 * [None])[:2] + if "transforms" in kwargs: + transforms = kwargs.get("transforms") + if transforms is None: + raise ValueError("transforms is not provided.") + check_list(transforms) + kwargs["transforms"] = transforms + + if "prob" in kwargs: + prob = kwargs.get("prob") + if prob is not None: + check_value(prob, [0., 1.]) + kwargs["prob"] = prob + + return method(self, **kwargs) + + return new_method + + +def check_ten_crop(method): + """Wrapper method to check the parameters of crop.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + size, use_vertical_flip = args + if "size" in kwargs: + size = kwargs.get("size") + if "use_vertical_flip" in kwargs: + use_vertical_flip = kwargs.get("use_vertical_flip") + + if size is None: + raise ValueError("size is not provided.") + size = check_crop_size(size) + kwargs["size"] = size + + if use_vertical_flip is not None: + check_bool(use_vertical_flip) + kwargs["use_vertical_flip"] = use_vertical_flip + + return method(self, **kwargs) + + return new_method + + +def check_num_channels(method): + """Wrapper method to check the parameters of number of channels.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + num_output_channels = (list(args) + [None])[0] + if "num_output_channels" in kwargs: + num_output_channels = kwargs.get("num_output_channels") + if num_output_channels is not None: + if num_output_channels not in (1, 3): + raise ValueError("Number of channels of the output grayscale image" + "should be either 1 or 3. Got {0}".format(num_output_channels)) + kwargs["num_output_channels"] = num_output_channels + + return method(self, **kwargs) + + return new_method + + +def check_pad(method): + """Wrapper method to check the parameters of random pad.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 3 * [None])[:3] + padding, fill_value, padding_mode = args + if "padding" in kwargs: + padding = kwargs.get("padding") + if "fill_value" in kwargs: + fill_value = kwargs.get("fill_value") + if "padding_mode" in kwargs: + padding_mode = kwargs.get("padding_mode") + + if padding is None: + raise ValueError("padding is not provided.") + padding = check_padding(padding) + kwargs["padding"] = padding + + if fill_value is not None: + fill_value = check_fill_value(fill_value) + kwargs["fill_value"] = fill_value + if padding_mode is not None: + check_border_type(padding_mode) + kwargs["padding_mode"] = padding_mode + + return method(self, **kwargs) + + return new_method + + +def check_random_perspective(method): + """Wrapper method to check the parameters of random perspective.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 3 * [None])[:3] + distortion_scale, prob, interpolation = args + if "distortion_scale" in kwargs: + distortion_scale = kwargs.get("distortion_scale") + if "prob" in kwargs: + prob = kwargs.get("prob") + if "interpolation" in kwargs: + interpolation = kwargs.get("interpolation") + + if distortion_scale is not None: + check_value(distortion_scale, [0., 1.]) + kwargs["distortion_scale"] = distortion_scale + if prob is not None: + check_value(prob, [0., 1.]) + kwargs["prob"] = prob + if interpolation is not None: + check_inter_mode(interpolation) + kwargs["interpolation"] = interpolation + + return method(self, **kwargs) + + return new_method + + +def check_mix_up(method): + """Wrapper method to check the parameters of mix up.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 3 * [None])[:3] + batch_size, alpha, is_single = args + if "batch_size" in kwargs: + batch_size = kwargs.get("batch_size") + if "alpha" in kwargs: + alpha = kwargs.get("alpha") + if "is_single" in kwargs: + is_single = kwargs.get("is_single") + + if batch_size is None: + raise ValueError("batch_size") + check_pos_int32(batch_size) + kwargs["batch_size"] = batch_size + if alpha is None: + raise ValueError("alpha") + check_positive(alpha) + kwargs["alpha"] = alpha + if is_single is not None: + check_type(is_single, bool) + kwargs["is_single"] = is_single + + return method(self, **kwargs) + + return new_method + + +def check_random_erasing(method): + """Wrapper method to check the parameters of random erasing.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 6 * [None])[:6] + prob, scale, ratio, value, inplace, max_attempts = args + if "prob" in kwargs: + prob = kwargs.get("prob") + if "scale" in kwargs: + scale = kwargs.get("scale") + if "ratio" in kwargs: + ratio = kwargs.get("ratio") + if "value" in kwargs: + value = kwargs.get("value") + if "inplace" in kwargs: + inplace = kwargs.get("inplace") + if "max_attempts" in kwargs: + max_attempts = kwargs.get("max_attempts") + + if prob is not None: + check_value(prob, [0., 1.]) + kwargs["prob"] = prob + if scale is not None: + check_range(scale, [0, FLOAT_MAX_INTEGER]) + kwargs["scale"] = scale + if ratio is not None: + check_range(ratio, [0, FLOAT_MAX_INTEGER]) + kwargs["ratio"] = ratio + if value is not None: + check_erasing_value(value) + kwargs["value"] = value + if inplace is not None: + check_bool(inplace) + kwargs["inplace"] = inplace + if max_attempts is not None: + check_pos_int32(max_attempts) + kwargs["max_attempts"] = max_attempts + + return method(self, **kwargs) + + return new_method + + +def check_cutout(method): + """Wrapper method to check the parameters of cutout operation.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + length, num_patches = args + if "length" in kwargs: + length = kwargs.get("length") + if "num_patches" in kwargs: + num_patches = kwargs.get("num_patches") + + if length is None: + raise ValueError("length") + check_pos_int32(length) + kwargs["length"] = length + + if num_patches is not None: + check_pos_int32(num_patches) + kwargs["num_patches"] = num_patches + + return method(self, **kwargs) + + return new_method + + +def check_linear_transform(method): + """Wrapper method to check the parameters of linear transform.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 2 * [None])[:2] + transformation_matrix, mean_vector = args + if "transformation_matrix" in kwargs: + transformation_matrix = kwargs.get("transformation_matrix") + if "mean_vector" in kwargs: + mean_vector = kwargs.get("mean_vector") + + if transformation_matrix is None: + raise ValueError("transformation_matrix is not provided.") + if mean_vector is None: + raise ValueError("mean_vector is not provided.") + + if transformation_matrix.shape[0] != transformation_matrix.shape[1]: + raise ValueError("transformation_matrix should be a square matrix. " + "Got shape {} instead".format(transformation_matrix.shape)) + if mean_vector.shape[0] != transformation_matrix.shape[0]: + raise ValueError("mean_vector length {0} should match either one dimension of the square" + "transformation_matrix {1}.".format(mean_vector.shape[0], transformation_matrix.shape)) + + kwargs["transformation_matrix"] = transformation_matrix + kwargs["mean_vector"] = mean_vector + + return method(self, **kwargs) + + return new_method + + +def check_random_affine(method): + """Wrapper method to check the parameters of random affine.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + args = (list(args) + 6 * [None])[:6] + degrees, translate, scale, shear, resample, fill_value = args + if "degrees" in kwargs: + degrees = kwargs.get("degrees") + if "translate" in kwargs: + translate = kwargs.get("translate") + if "scale" in kwargs: + scale = kwargs.get("scale") + if "shear" in kwargs: + shear = kwargs.get("shear") + if "resample" in kwargs: + resample = kwargs.get("resample") + if "fill_value" in kwargs: + fill_value = kwargs.get("fill_value") + + if degrees is None: + raise ValueError("degrees is not provided.") + degrees = check_degrees(degrees) + kwargs["degrees"] = degrees + + if translate is not None: + if isinstance(translate, (tuple, list)) and len(translate) == 2: + for t in translate: + if t < 0.0 or t > 1.0: + raise ValueError("translation values should be between 0 and 1") + else: + raise TypeError("translate should be a list or tuple of length 2.") + kwargs["translate"] = translate + + if scale is not None: + if isinstance(scale, (tuple, list)) and len(scale) == 2: + for s in scale: + if s <= 0: + raise ValueError("scale values should be positive") + else: + raise TypeError("scale should be a list or tuple of length 2.") + kwargs["scale"] = scale + + if shear is not None: + if isinstance(shear, numbers.Number): + if shear < 0: + raise ValueError("If shear is a single number, it must be positive.") + shear = (-1 * shear, shear) + elif isinstance(shear, (tuple, list)) and (len(shear) == 2 or len(shear) == 4): + # X-Axis shear with [min, max] + if len(shear) == 2: + shear = [shear[0], shear[1], 0., 0.] + elif len(shear) == 4: + shear = [s for s in shear] + else: + raise TypeError("shear should be a list or tuple and it must be of length 2 or 4.") + kwargs["shear"] = shear + + if resample is not None: + check_inter_mode(resample) + kwargs["resample"] = resample + if fill_value is not None: + fill_value = check_fill_value(fill_value) + kwargs["fill_value"] = fill_value + + return method(self, **kwargs) + + return new_method + + +def check_rescale(method): + """Wrapper method to check the parameters of rescale.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + rescale, shift = (list(args) + 2 * [None])[:2] + if "rescale" in kwargs: + rescale = kwargs.get("rescale") + if "shift" in kwargs: + shift = kwargs.get("shift") + + if rescale is None: + raise ValueError("rescale is not provided.") + check_pos_float32(rescale) + kwargs["rescale"] = rescale + + if shift is None: + raise ValueError("shift is not provided.") + if not isinstance(shift, numbers.Number): + raise TypeError("shift is not a number.") + kwargs["shift"] = shift + + return method(self, **kwargs) + + return new_method diff --git a/mindspore/log.py b/mindspore/log.py new file mode 100644 index 00000000000..38455e2e18c --- /dev/null +++ b/mindspore/log.py @@ -0,0 +1,517 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +log module +""" +import sys +import os +import stat +import time +import fcntl +import logging +from logging.handlers import RotatingFileHandler +import traceback +import threading + +__all__ = ['get_level', 'get_log_config'] + +# The lock for setting up the logger +_setup_logger_lock = threading.Lock() + +# When getting the logger, Used to check whether +# the logger already exists +_global_logger = None + +# The flag for enable console output +_std_on = '1' +# The flag for disable console output +_std_off = '0' +# Rotating max bytes, default is 50M +_logger_def_max_bytes = '52428800' +# Rotating backup count, default is 30 +_logger_def_backup_count = '30' +# The default log level +_logger_def_level = '2' + +# Log level name and level mapping +_name_to_level = { + 'ERROR': 40, + 'WARNING': 30, + 'INFO': 20, + 'DEBUG': 10, +} + +# GLog level and level name +_gloglevel_to_name = { + '3': 'ERROR', + '2': 'WARNING', + '1': 'INFO', + '0': 'DEBUG', +} + +# The mapping of logger configurations to glog configurations +_confmap_dict = {'level': 'GLOG_v', 'console': 'GLOG_logtostderr', 'filepath': 'GLOG_log_dir', + 'maxBytes': 'logger_maxBytes', 'backupCount': 'logger_backupCount'} + + +class _MultiCompatibleRotatingFileHandler(RotatingFileHandler): + """Inherit RotatingFileHandler for multiprocess compatibility.""" + + def rolling_rename(self): + """Rolling rename log files and set permission of Log file""" + for i in range(self.backupCount - 1, 0, -1): + sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i)) + dfn = self.rotation_filename("%s.%d" % (self.baseFilename, i + 1)) + if os.path.exists(sfn): + if os.path.exists(dfn): + os.remove(dfn) + # Modify the permission of Log file + os.chmod(sfn, stat.S_IREAD) + os.rename(sfn, dfn) + + def doRollover(self): + """Override doRollover for multiprocess compatibility + and setting permission of Log file""" + if self.stream: + self.stream.close() + self.stream = None + + # Attain an exclusive lock with bloking mode by `fcntl` module. + with open(self.baseFilename, 'a') as file_pointer: + fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) + + if self.backupCount > 0: + self.rolling_rename() + + dfn = self.rotation_filename(self.baseFilename + ".1") + if os.path.exists(dfn): + os.remove(dfn) + # Modify the permission of Log file + os.chmod(self.baseFilename, stat.S_IREAD) + self.rotate(self.baseFilename, dfn) + + with open(self.baseFilename, 'a'): + # Modify the permission of Log file + os.chmod(self.baseFilename, stat.S_IREAD | stat.S_IWRITE) + + if not self.delay: + self.stream = self._open() + + +class _DataFormatter(logging.Formatter): + """Log formatter""" + + def __init__(self, sub_module, fmt=None, **kwargs): + """ + Initialization of logFormatter. + + Args: + sub_module (str): The submodule name. + fmt (str): Specified format pattern. Default: None. + """ + super(_DataFormatter, self).__init__(fmt=fmt, **kwargs) + self.sub_module = sub_module.upper() + + def formatTime(self, record, datefmt=None): + """ + Override formatTime for uniform format %Y-%m-%d-%H:%M:%S.SSS.SSS + + Args: + record (str): Log record. + datefmt (str): Date format. + + Returns: + str, formatted timestamp. + """ + created_time = self.converter(record.created) + if datefmt: + return time.strftime(datefmt, created_time) + + timestamp = time.strftime('%Y-%m-%d-%H:%M:%S', created_time) + msecs = str(round(record.msecs * 1000)) + # Format the time stamp + return f'{timestamp}.{msecs[:3]}.{msecs[3:]}' + + def format(self, record): + """ + Apply log format with specified pattern. + + Args: + record (str): Format pattern. + + Returns: + str, formatted log content according to format pattern. + """ + # NOTICE: when the Installation directory of mindspore changed, + # ms_home_path must be changed + ms_install_home_path = 'mindspore' + idx = record.pathname.rfind(ms_install_home_path) + if idx >= 0: + # Get the relative path of the file + record.filepath = record.pathname[idx:] + else: + record.filepath = record.pathname + record.sub_module = self.sub_module + return super().format(record) + + +def _get_logger(): + """ + Get logger instance. + + Returns: + Logger, a logger. + """ + if _global_logger: + return _global_logger + + kwargs = _get_env_config() + _verify_config(kwargs) + logger = _setup_logger(_adapt_cfg(kwargs)) + return logger + + +def _adapt_cfg(kwargs): + """ + Glog configurations converted to logger configurations. + + Args: + kwargs (dict): The dictionary of log configurations. + + - console (str): Whether to output log to stdout. + - level (str): Log level. + - filepath (str): The path for saving logs, if console is false, a file path must be assigned. + - maxBytes (str): The Maximum value of a log file for rotating, only valid if console is false. + - backupCount (str): The count of rotating backup log files, only valid if console is false. + + Returns: + Dict, the input parameter dictionary. + """ + kwargs['level'] = _gloglevel_to_name.get(kwargs.get('level', _logger_def_level)) + kwargs['console'] = not kwargs.get('console') == _std_off + kwargs['maxBytes'] = int(kwargs.get('maxBytes', _logger_def_max_bytes)) + kwargs['backupCount'] = int(kwargs.get('backupCount', _logger_def_backup_count)) + return kwargs + + +def info(msg, *args, **kwargs): + """ + Log a message with severity 'INFO' on the MindSpore logger. + + Examples: + >>> from mindspore import log as logger + >>> logger.info("The arg(%s) is: %r", name, arg) + """ + _get_logger().info(msg, *args, **kwargs) + + +def debug(msg, *args, **kwargs): + """ + Log a message with severity 'DEBUG' on the MindSpore logger. + + Examples: + >>> from mindspore import log as logger + >>> logger.debug("The arg(%s) is: %r", name, arg) + """ + _get_logger().debug(msg, *args, **kwargs) + + +def error(msg, *args, **kwargs): + """Log a message with severity 'ERROR' on the MindSpore logger.""" + _get_logger().error(msg, *args, **kwargs) + + +def warning(msg, *args, **kwargs): + """Log a message with severity 'WARNING' on the MindSpore logger.""" + _get_logger().warning(msg, *args, **kwargs) + + +def get_level(): + """ + Get the logger level. + + Returns: + str, the Log level includes 3(ERROR), 2(WARNING), 1(INFO), 0(DEBUG). + + Examples: + >>> import os + >>> os.environ['GLOG_v'] = '0' + >>> from mindspore import log as logger + >>> logger.get_level() + """ + # level and glog level mapping dictionary + level_to_glog_level = dict(zip(_name_to_level.values(), _gloglevel_to_name.keys())) + + return level_to_glog_level.get(_get_logger().getEffectiveLevel()) + + +def _get_formatter(): + """ + Get the string of log formatter. + + Returns: + str, the string of log formatter. + """ + formatter = '[%(levelname)s] %(sub_module)s(%(process)d:' \ + '%(thread)d,%(processName)s):%(asctime)s ' \ + '[%(filepath)s:%(lineno)d] %(message)s' + return formatter + + +def _get_env_config(): + """ + Get configurations from environment variables. + + Returns: + Dict, the dictionary of configurations. + """ + config_dict = {} + for key, env_value in _confmap_dict.items(): + value = os.environ.get(env_value) + if value: + config_dict[key] = value.strip() + return config_dict + + +def _verify_config(kwargs): + """ + Verify log configurations. + + Args: + kwargs (dict): The dictionary of log configurations. + + - console (str): Whether to output log to stdout. + - level (str): Log level. + - filepath (str): The path for saving logs, if console is false, a file path must be assigned. + - maxBytes (str): The Maximum value of a log file for rotating, only valid if console is false. + - backupCount (str): The count of rotating backup log files, only valid if console is false. + """ + # Check the input value of level + level = kwargs.get('level', None) + if level is not None: + _verify_level(level) + + # Check the input value of console + console = kwargs.get('console', None) + file_path = kwargs.get('filepath', None) + + if console is not None: + if not console.isdigit() or console not in (_std_off, _std_on): + raise ValueError(f'Incorrect value, The value of {_confmap_dict["console"]} must be 0 or 1,' + f' Output log to console, configure to 1.') + + if console == _std_off and not file_path: + raise ValueError(f'When {_confmap_dict["console"]} is set to 0, The directory of ' + f'saving log must be set, {_confmap_dict["filepath"]} cannot be empty.') + + # Check the input value of filepath + if console == _std_off and file_path is not None: + file_real_path = os.path.realpath(file_path) + if not os.path.exists(file_real_path): + raise ValueError(f'The file path does not exist. ' + f'{_confmap_dict["filepath"]}:{file_path}') + + # Check the input value of maxBytes + max_bytes = kwargs.get('maxBytes', None) + if console == _std_off and max_bytes is not None: + if not max_bytes.isdigit(): + raise ValueError(f'Incorrect value, The value of {_confmap_dict["maxBytes"]} must be positive integer. ' + f'{_confmap_dict["maxBytes"]}:{max_bytes}') + + # Check the input value of backupCount + backup_count = kwargs.get('backupCount', None) + if console == _std_off and backup_count is not None: + if not backup_count.isdigit(): + raise ValueError(f'Incorrect value, The value of {_confmap_dict["backupCount"]} must be positive ' + f'integer. {_confmap_dict["backupCount"]}:{backup_count}') + + +def _verify_level(level): + """ + Verify log level. + + Args: + level (str): The log level. + """ + level_name = _gloglevel_to_name.get(level, None) + + # Check the value of input level + if level_name not in _name_to_level: + raise ValueError(f'Incorrect log level:{level}, Please check the log level configuration, ' + f'desired log level :{_gloglevel_to_name}') + + +def get_log_config(): + """ + Get logger configurations. + + Returns: + Dict, the dictionary of logger configurations. + + Examples: + >>> import os + >>> os.environ['GLOG_v'] = '1' + >>> os.environ['GLOG_logtostderr'] = '0' + >>> os.environ['GLOG_log_dir'] = '/var/log/mindspore' + >>> os.environ['logger_maxBytes'] = '5242880' + >>> os.environ['logger_backupCount'] = '10' + >>> from mindspore import log as logger + >>> logger.get_log_config() + """ + logger = _get_logger() + handler = logger.handlers[0] + config_dict = {} + config_dict['GLOG_v'] = get_level() + config_dict['GLOG_logtostderr'] = _std_on + + if handler.name == 'FileHandler': + config_dict['GLOG_logtostderr'] = _std_off + # Separating file path and name + file_path_and_name = os.path.split(handler.baseFilename) + config_dict['GLOG_log_dir'] = file_path_and_name[0] + config_dict['logger_maxBytes'] = handler.maxBytes + config_dict['logger_backupCount'] = handler.backupCount + return config_dict + + +def _clear_handler(logger): + """Clear the handlers that has been set, avoid repeated loading""" + for handler in logger.handlers: + logger.removeHandler(handler) + + +def _find_caller(stack_info=False): + """ + Find the stack frame of the caller. + + Override findCaller on the logger, Support for getting log record. + Find the stack frame of the caller so that we can note the source + file name, function name and line number. + + Args: + stack_info (bool): If the value is true, print stack information to the log. Default: False. + + Returns: + tuple, the tuple of the frame data. + """ + f = sys._getframe(3) + sinfo = None + # log_file is used to check caller stack frame + log_file = os.path.normcase(f.f_code.co_filename) + f = f.f_back + rv = "(unknown file)", 0, "(unknown function)", None + while f: + co = f.f_code + filename = os.path.normcase(co.co_filename) + if log_file == filename: + f = f.f_back + continue + if stack_info: + sinfo = _get_stack_info(f) + rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) + break + return rv + + +def _get_stack_info(frame): + """ + Get the stack informations. + + Args: + frame(frame): the frame requiring informations. + + Returns: + str, the string of the stack informations. + """ + sinfo = None + stack_prefix = 'Stack (most recent call last):\n' + sinfo = stack_prefix + "".join(traceback.format_stack(frame)) + return sinfo + + +def _setup_logger(kwargs): + """ + Set up the logger. + + Args: + kwargs (dict): The dictionary of log configurations. + + - console (bool): Whether to output log to stdout. Default: True. + - level (str): Log level. Default: WARNING. + - filepath (str): The path for saving logs, if console is false, a file path must be assigned. + - maxBytes (int): The Maximum value of a log file for rotating, only valid if console is false. + Default: 52428800. + - backupCount (int): The count of rotating backup log files, only valid if console is false. Default: 30. + + Returns: + Logger, well-configured logger. + """ + + # The name of Submodule + sub_module = 'ME' + # The name of Base log file + log_name = 'mindspore.log' + + global _global_logger + + _setup_logger_lock.acquire() + try: + if _global_logger: + return _global_logger + + logger = logging.getLogger(name=f'{sub_module}.{log_name}') + # Override findCaller on the logger, Support for getting log record + logger.findCaller = _find_caller + console = kwargs.get('console', True) + # Set log level + logger.setLevel(kwargs.get('level', logging.WARNING)) + # Set "propagate" attribute to False, stop searching up the hierarchy, + # avoid to load the handler of the root logger + logger.propagate = False + # Get the formatter for handler + formatter = _get_formatter() + + # Clean up handle to avoid repeated loading + _clear_handler(logger) + + # Set streamhandler for the console appender + if console: + console_handler = logging.StreamHandler(sys.stderr) + console_handler.name = 'StreamHandler' + console_handler.formatter = _DataFormatter(sub_module, formatter) + logger.addHandler(console_handler) + + # Set rotatingFileHandler for the file appender + else: + # filepath cannot be null, checked in function _verify_config () + logfile_dir = os.path.realpath(kwargs.get('filepath')) + file_name = f'{logfile_dir}/{log_name}' + logfile_handler = _MultiCompatibleRotatingFileHandler( + filename=file_name, + # Rotating max bytes, default is 50M + maxBytes=kwargs.get('maxBytes', _logger_def_max_bytes), + # Rotating backup count, default is 30 + backupCount=kwargs.get('backupCount', _logger_def_backup_count), + encoding='utf8' + ) + logfile_handler.name = 'FileHandler' + logfile_handler.formatter = _DataFormatter(sub_module, formatter) + logger.addHandler(logfile_handler) + + _global_logger = logger + + finally: + _setup_logger_lock.release() + return _global_logger diff --git a/mindspore/mindrecord/__init__.inter b/mindspore/mindrecord/__init__.inter new file mode 100644 index 00000000000..ca1d50153b1 --- /dev/null +++ b/mindspore/mindrecord/__init__.inter @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Introduction to mindrecord: + +Mindrecord is a module to implement reading, writing, search and +converting for MindSpore format dataset. Users could load(modify) +mindrecord data through FileReader(FileWriter). Users could also +convert other format dataset to mindrecord data through +corresponding sub-module. +""" + +# common +from .common.exceptions import * + +# export +from .filewriter import FileWriter +from .filereader import FileReader +from .mindpage import MindPage +from .shardutils import SUCCESS, FAILED +from .tools.cifar10_to_mr import Cifar10ToMR +from .tools.cifar100_to_mr import Cifar100ToMR +from .tools.imagenet_to_mr import ImageNetToMR +from .tools.mnist_to_mr import MnistToMR + +# internal +from .shardheader import ShardHeader +from .shardreader import ShardReader +from .shardsegment import ShardSegment +from .shardindexgenerator import ShardIndexGenerator +from .shardwriter import ShardWriter +from .shardutils import MAX_CONSUMER_COUNT # function + diff --git a/mindspore/mindrecord/__init__.py b/mindspore/mindrecord/__init__.py new file mode 100644 index 00000000000..31fb801c46b --- /dev/null +++ b/mindspore/mindrecord/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Introduction to mindrecord: + +Mindrecord is a module to implement reading, writing, search and +converting for MindSpore format dataset. Users could load(modify) +mindrecord data through FileReader(FileWriter). Users could also +convert other format dataset to mindrecord data through +corresponding sub-module. +""" + +from .filewriter import FileWriter +from .filereader import FileReader +from .mindpage import MindPage +from .common.exceptions import * +from .shardutils import SUCCESS, FAILED +from .tools.cifar10_to_mr import Cifar10ToMR +from .tools.cifar100_to_mr import Cifar100ToMR +from .tools.imagenet_to_mr import ImageNetToMR +from .tools.mnist_to_mr import MnistToMR + +__all__ = ['FileWriter', 'FileReader', 'MindPage', + 'Cifar10ToMR', 'Cifar100ToMR', 'ImageNetToMR', 'MnistToMR', + 'SUCCESS', 'FAILED'] diff --git a/mindspore/mindrecord/common/__init__.py b/mindspore/mindrecord/common/__init__.py new file mode 100644 index 00000000000..685b1f0f1ab --- /dev/null +++ b/mindspore/mindrecord/common/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +init common +""" diff --git a/mindspore/mindrecord/common/constant.py b/mindspore/mindrecord/common/constant.py new file mode 100644 index 00000000000..2def8d9fe21 --- /dev/null +++ b/mindspore/mindrecord/common/constant.py @@ -0,0 +1,20 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +constants in MindRecord. +""" + +# SYS ID will be used to create error code. +SYS_ID = 0b00101010 diff --git a/mindspore/mindrecord/common/enums.py b/mindspore/mindrecord/common/enums.py new file mode 100644 index 00000000000..a76b2068f50 --- /dev/null +++ b/mindspore/mindrecord/common/enums.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +enums for exceptions +""" + +class BaseEnum: + """ + Enum base class. + """ + +class LogRuntime(BaseEnum): + """Log runtime enum.""" + RT_HOST = 0b01 + RT_DEVICE = 0b10 + +class ErrorCodeType(BaseEnum): + """Error code type enum.""" + ERROR_CODE = 0b01 + EXCEPTION_CODE = 0b10 + +class ErrorLevel(BaseEnum): + """Error level.""" + COMMON_LEVEL = 0b000 + SUGGESTION_LEVEL = 0b001 + MINOR_LEVEL = 0b010 + MAJOR_LEVEL = 0b011 + CRITICAL_LEVEL = 0b100 diff --git a/mindspore/mindrecord/common/exceptions.py b/mindspore/mindrecord/common/exceptions.py new file mode 100644 index 00000000000..02af36c7d96 --- /dev/null +++ b/mindspore/mindrecord/common/exceptions.py @@ -0,0 +1,285 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Define custom exception + +Error rule: + EXCEPTIONS key: The error code key should be as same as the realized class name. + exception No: common module error No is 1-99, + unknown error No is 0, + shard* api error No is 100-199 + file* api error No is 200-299 + error message: It is the base error message. + You can recover error message in realized class __init__ function. +""" +from .enums import LogRuntime, ErrorCodeType, ErrorLevel +from .constant import SYS_ID +EXCEPTIONS = dict( + # the format of list is [exception No, base error message] + UnknownError=[0, 'Unknown Error.'], + ParamTypeError=[1, 'Param type is error.'], + ParamValueError=[2, 'Param value is error.'], + ParamMissError=[3, 'Param missing.'], + PathNotExistsError=[4, 'Path does not exist.'], + DbConnectionError=[5, 'Db connection is error.'], + + # MindRecord error 100-199 for shard* + MRMOpenError=[100, 'MindRecord could not open.'], + MRMOpenForAppendError=[101, 'MindRecord could not open for append.'], + MRMInvalidPageSizeError=[102, 'Failed to set page size.'], + MRMInvalidHeaderSizeError=[103, 'Failed to set header size.'], + MRMSetHeaderError=[104, 'Failed to set header.'], + MRMWriteDatasetError=[105, 'Failed to write dataset.'], + MRMCommitError=[107, 'Failed to commit.'], + + MRMLaunchError=[108, 'Failed to launch.'], + MRMFinishError=[109, 'Failed to finish.'], + MRMCloseError=[110, 'Failed to close.'], + + MRMAddSchemaError=[111, 'Failed to add schema.'], + MRMAddIndexError=[112, 'Failed to add index field.'], + MRMBuildSchemaError=[113, 'Failed to build schema.'], + MRMGetMetaError=[114, 'Failed to get meta info.'], + + MRMIndexGeneratorError=[115, 'Failed to create index generator.'], + MRMGenerateIndexError=[116, 'Failed to generate index.'], + + MRMInitSegmentError=[117, 'Failed to initialize segment.'], + MRMFetchCandidateFieldsError=[118, 'Failed to fetch candidate category fields.'], + MRMReadCategoryInfoError=[119, 'Failed to read category information.'], + MRMFetchDataError=[120, 'Failed to fetch data by category.'], + + + # MindRecord error 200-299 for File* and MindPage + MRMInvalidSchemaError=[200, 'Schema is error.'], + MRMValidateDataError=[201, 'Raw data is valid.'], + MRMDefineIndexError=[202, 'Index field is error.'], + MRMDefineBlobError=[203, 'Blob field is error.'], + MRMUnsupportedSchemaError=[204, 'Schema is not supported.'], + MRMDefineCategoryError=[205, 'Category field is error.'], + +) + +class MindRecordException(Exception): + """MindRecord base error class.""" + + def __init__(self): + """Initialize an error which may occurs in mindrecord.""" + super(MindRecordException, self).__init__() + class_name = self.__class__.__name__ + error_item = EXCEPTIONS[class_name] if class_name in EXCEPTIONS \ + else EXCEPTIONS['UnknownError'] + self._error_msg = error_item[1] + self._error_code = MindRecordException.transform_error_code(error_item[0]) + + @property + def error_msg(self): + """return the description of this error.""" + return self._error_msg + + @error_msg.setter + def error_msg(self, msg): + self._error_msg = msg + + @property + def error_code(self): + """return the unique error number of this error.""" + return self._error_code + + def __str__(self): + return "[{}]: error_code: {}, error_msg: {}".format( + self.__class__.__name__, self._error_code, self._error_msg) + + @staticmethod + def transform_error_code(exception_no): + """ + Transform mindrecord exception no to GE error code. + + error_code = ((0xFF & runtime) << 30) \ + | ((0xFF & error_code_type) << 28) \ + | ((0xFF & error_level) << 25) \ + | ((0xFF & sys_id) << 17) \ + | ((0xFF & mod_id) << 12) \ + | (0x0FFF & exception_no) + Args: + exception_no: Integer. Exception number. + + Returns: + Integer, error code. + + """ + runtime = LogRuntime.RT_HOST + error_code_type = ErrorCodeType.ERROR_CODE + error_level = ErrorLevel.COMMON_LEVEL + exception_no_range_per_module = 100 + mod_id = int(exception_no / exception_no_range_per_module) + 1 + error_code = (((0xFF & runtime) << 30) + | ((0xFF & error_code_type) << 28) + | ((0xFF & error_level) << 25) + | ((0xFF & SYS_ID) << 17) + | ((0xFF & mod_id) << 12) + | (0x0FFF & exception_no)) + return error_code + + +class UnknownError(MindRecordException): + """Raise an unknown error when an unknown error occurs.""" + + +class ParamValueError(MindRecordException): + """ + Request param value error. + """ + + def __init__(self, error_detail): + super(ParamValueError, self).__init__() + self.error_msg = 'Invalid parameter value. {}'.format(error_detail) + + +class ParamTypeError(MindRecordException): + """ + Request param type error. + """ + + def __init__(self, param_name, expected_type): + super(ParamTypeError, self).__init__() + self.error_msg = "Invalid parameter type. '{}' expect {} type." \ + "".format(param_name, expected_type) + + +class ParamMissError(MindRecordException): + """ + missing param error. + """ + + def __init__(self, param_name): + super(ParamMissError, self).__init__() + self.error_msg = "Param missing. '{}' is required.".format(param_name) + +class PathNotExistsError(MindRecordException): + """ + invalid path. + """ + def __init__(self, error_path): + super(PathNotExistsError, self).__init__() + self.error_msg = 'Invalid path. {}'.format(error_path) + +class DbConnectionError(MindRecordException): + """ + Database connection error. + """ + def __init__(self, error_detail): + super(DbConnectionError, self).__init__() + self.error_msg = 'Db connection is error. Detail: {}'.format(error_detail) +## +class MRMOpenError(MindRecordException): + """ + Raised when could not open mind record file successfully. + """ + def __init__(self): + super(MRMOpenError, self).__init__() + self.error_msg = 'MindRecord File could not open successfully.' + +class MRMOpenForAppendError(MindRecordException): + """ + Raised when could not open mind record file successfully for append. + """ + def __init__(self): + super(MRMOpenForAppendError, self).__init__() + self.error_msg = 'MindRecord File could not open successfully for append.' + +class MRMInvalidPageSizeError(MindRecordException): + pass + +class MRMInvalidHeaderSizeError(MindRecordException): + pass + +class MRMSetHeaderError(MindRecordException): + pass + +class MRMWriteDatasetError(MindRecordException): + pass + +class MRMCommitError(MindRecordException): + pass + +class MRMLaunchError(MindRecordException): + pass + +class MRMFinishError(MindRecordException): + pass + +class MRMCloseError(MindRecordException): + pass + +class MRMAddSchemaError(MindRecordException): + pass + +class MRMAddIndexError(MindRecordException): + pass + +class MRMBuildSchemaError(MindRecordException): + pass + +class MRMGetMetaError(MindRecordException): + pass + +class MRMIndexGeneratorError(MindRecordException): + pass + +class MRMGenerateIndexError(MindRecordException): + pass + +class MRMInitSegmentError(MindRecordException): + pass + +class MRMFetchCandidateFieldsError(MindRecordException): + pass + +class MRMReadCategoryInfoError(MindRecordException): + pass + +class MRMFetchDataError(MindRecordException): + pass + +class MRMInvalidSchemaError(MindRecordException): + def __init__(self, error_detail): + super(MRMInvalidSchemaError, self).__init__() + self.error_msg = 'Schema format is error. Detail: {}'.format(error_detail) + +class MRMValidateDataError(MindRecordException): + def __init__(self, error_detail): + super(MRMValidateDataError, self).__init__() + self.error_msg = 'Raw data do not match the schema. Detail: {}'.format(error_detail) + +class MRMDefineIndexError(MindRecordException): + def __init__(self, error_detail): + super(MRMDefineIndexError, self).__init__() + self.error_msg = 'Failed to define index field. Detail: {}'.format(error_detail) + +class MRMDefineBlobError(MindRecordException): + def __init__(self, error_detail): + super(MRMDefineBlobError, self).__init__() + self.error_msg = 'Failed to define blob field. Detail: {}'.format(error_detail) + +class MRMUnsupportedSchemaError(MindRecordException): + def __init__(self, error_detail): + super(MRMUnsupportedSchemaError, self).__init__() + self.error_msg = 'Schema is not supported. Detail: {}'.format(error_detail) + +class MRMDefineCategoryError(MindRecordException): + def __init__(self, error_detail): + super(MRMDefineCategoryError, self).__init__() + self.error_msg = 'Failed to define category field. Detail: {}'.format(error_detail) diff --git a/mindspore/mindrecord/filereader.py b/mindspore/mindrecord/filereader.py new file mode 100644 index 00000000000..80d6ffc7866 --- /dev/null +++ b/mindspore/mindrecord/filereader.py @@ -0,0 +1,93 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to read data from mindrecord. +""" +from .shardreader import ShardReader +from .shardheader import ShardHeader +from .shardutils import populate_data +from .shardutils import MIN_CONSUMER_COUNT, MAX_CONSUMER_COUNT, check_filename +from .common.exceptions import ParamValueError, ParamTypeError + +__all__ = ['FileReader'] + +class FileReader: + """ + Class to read MindRecord File series. + + Args: + file_name (str): File name of MindRecord File. + num_consumer(int, optional): Number of consumer threads which load data to memory (default=4). + It should not be smaller than 1 or larger than the number of CPU. + columns (list[str], optional): List of fields which correspond data would be read (default=None). + operator(int, optional): Reserved parameter for operators (default=None). + + Raises: + ParamValueError: If file_name, num_consumer or columns is invalid. + """ + def __init__(self, file_name, num_consumer=4, columns=None, operator=None): + check_filename(file_name) + self._file_name = file_name + + if num_consumer is not None: + if isinstance(num_consumer, int): + if num_consumer < MIN_CONSUMER_COUNT or num_consumer > MAX_CONSUMER_COUNT(): + raise ParamValueError("Consumer number should between {} and {}." + .format(MIN_CONSUMER_COUNT, MAX_CONSUMER_COUNT())) + else: + raise ParamValueError("Consumer number is illegal.") + else: + raise ParamValueError("Consumer number is illegal.") + if columns: + if isinstance(columns, list): + self._columns = columns + else: + raise ParamTypeError('columns', 'list') + else: + self._columns = None + self._reader = ShardReader() + self._reader.open(file_name, num_consumer, columns, operator) + self._header = ShardHeader(self._reader.get_header()) + self._reader.launch() + + + def get_next(self): + """ + Yield a batch of data according to columns at a time. + + Yields: + dict: keys is the same as columns. + + Raises: + MRMUnsupportedSchemaError: If schema is invalid. + """ + iterator = self._reader.get_next() + while iterator: + for blob, raw in iterator: + yield populate_data(raw, blob, self._columns, self._header.blob_fields, self._header.schema) + iterator = self._reader.get_next() + + def finish(self): + """ + Stop reader worker. + + Raises: + MRMFinishError: If failed to finish worker threads. + """ + return self._reader.finish() + + def close(self): + """Stop reader worker and close File.""" + return self._reader.close() diff --git a/mindspore/mindrecord/filewriter.py b/mindspore/mindrecord/filewriter.py new file mode 100644 index 00000000000..0e44b618579 --- /dev/null +++ b/mindspore/mindrecord/filewriter.py @@ -0,0 +1,406 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to write data into mindrecord. +""" +import os +import re +import stat +import numpy as np +from mindspore import log as logger +from .shardwriter import ShardWriter +from .shardreader import ShardReader +from .shardheader import ShardHeader +from .shardindexgenerator import ShardIndexGenerator +from .shardutils import MIN_SHARD_COUNT, MAX_SHARD_COUNT, VALID_ATTRIBUTES, VALID_ARRAY_ATTRIBUTES, \ + check_filename, VALUE_TYPE_MAP +from .common.exceptions import ParamValueError, ParamTypeError, MRMInvalidSchemaError, MRMDefineIndexError, \ + MRMValidateDataError + +__all__ = ['FileWriter'] + +class FileWriter: + """ + Class to write user defined raw data into MindRecord File series. + + Args: + file_name (str): File name of MindRecord File. + shard_num (int, optional): Number of MindRecord File (default=1). + It should be between [1, 1000]. + + Raises: + ParamValueError: If file_name or shard_num is invalid. + """ + def __init__(self, file_name, shard_num=1): + check_filename(file_name) + self._file_name = file_name + + if shard_num is not None: + if isinstance(shard_num, int): + if shard_num < MIN_SHARD_COUNT or shard_num > MAX_SHARD_COUNT: + raise ParamValueError("Shard number should between {} and {}." + .format(MIN_SHARD_COUNT, MAX_SHARD_COUNT)) + else: + raise ParamValueError("Shard num is illegal.") + else: + raise ParamValueError("Shard num is illegal.") + + self._shard_num = shard_num + self._index_generator = True + suffix_shard_size = len(str(self._shard_num - 1)) + + if self._shard_num == 1: + self._paths = [self._file_name] + else: + self._paths = ["{}{}".format(self._file_name, + str(x).rjust(suffix_shard_size, '0')) + for x in range(self._shard_num)] + + self._append = False + self._header = ShardHeader() + self._writer = ShardWriter() + self._generator = None + + @classmethod + def open_for_append(cls, file_name): + """ + Open MindRecord file and get ready to append data. + + Args: + file_name (str): String of MindRecord file name. + + Returns: + Instance of FileWriter. + + Raises: + ParamValueError: If file_name is invalid. + FileNameError: If path contains invalid character. + MRMOpenError: If failed to open MindRecord File. + MRMOpenForAppendError: If failed to open file for appending data. + """ + check_filename(file_name) + # construct ShardHeader + reader = ShardReader() + reader.open(file_name) + header = ShardHeader(reader.get_header()) + reader.close() + + instance = cls("append") + instance.init_append(file_name, header) + return instance + + def init_append(self, file_name, header): + self._append = True + self._file_name = file_name + self._header = header + self._writer.open_for_append(file_name) + + def add_schema(self, content, desc=None): + """ + Returns a schema id if added schema successfully, or raise exception. + + Args: + content (dict): Dict of user defined schema. + desc (str, optional): String of schema description (default=None). + + Returns: + int, schema id. + + Raises: + MRMInvalidSchemaError: If schema is invalid. + MRMBuildSchemaError: If failed to build schema. + MRMAddSchemaError: If failed to add schema. + """ + ret, error_msg = self._validate_schema(content) + if ret is False: + raise MRMInvalidSchemaError(error_msg) + schema = self._header.build_schema(content, desc) + return self._header.add_schema(schema) + + def add_index(self, index_fields): + """ + Select index fields from schema to accelerate reading. + + Args: + index_fields (list[str]): Fields would be set as index which should be primitive type. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + ParamTypeError: If index field is invalid. + MRMDefineIndexError: If index field is not primitive type. + MRMAddIndexError: If failed to add index field. + """ + if not index_fields or not isinstance(index_fields, list): + raise ParamTypeError('index_fields', 'list') + + for field in index_fields: + if field in self._header.blob_fields: + raise MRMDefineIndexError("Failed to set field {} since it's not primitive type.".format(field)) + if not isinstance(field, str): + raise ParamTypeError('index field', 'str') + return self._header.add_index_fields(index_fields) + + def _verify_based_on_schema(self, raw_data): + """ + Verify data according to schema and remove invalid data if validation failed. + + 1) allowed data type contains: "int32", "int64", "float32", "float64", "string", "bytes". + + Args: + raw_data (list[dict]): List of raw data. + """ + error_data_dic = {} + schema_content = self._header.schema + for field in schema_content: + for i, v in enumerate(raw_data): + if i in error_data_dic: + continue + + if field not in v: + error_data_dic[i] = "for schema, {} th data is wrong, " \ + "there is not '{}' object in the raw data.".format(i, field) + continue + field_type = type(v[field]).__name__ + if field_type not in VALUE_TYPE_MAP: + error_data_dic[i] = "for schema, {} th data is wrong, " \ + "data type for '{}' is not matched.".format(i, field) + continue + + if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]: + error_data_dic[i] = "for schema, {} th data is wrong, " \ + "data type for '{}' is not matched.".format(i, field) + continue + + if field_type == 'ndarray': + if 'shape' not in schema_content[field]: + error_data_dic[i] = "for schema, {} th data is wrong, " \ + "data type for '{}' is not matched.".format(i, field) + else: + try: + np.reshape(v[field], schema_content[field]['shape']) + except ValueError: + error_data_dic[i] = "for schema, {} th data is wrong, " \ + "data type for '{}' is not matched.".format(i, field) + error_data_dic = sorted(error_data_dic.items(), reverse=True) + for i, v in error_data_dic: + raw_data.pop(i) + logger.warning(v) + + def _verify_based_on_blob_fields(self, raw_data): + """ + Verify data according to blob fields which is sub set of schema's fields. + + Raise exception if validation failed. + 1) allowed data type contains: "int32", "int64", "float32", "float64", "string", "bytes". + + Args: + raw_data (list[dict]): List of raw data. + + Raises: + MRMValidateDataError: If data does not match blob fields. + """ + schema_content = self._header.schema + for field in schema_content: + for i, v in enumerate(raw_data): + if field not in v: + raise MRMValidateDataError("for schema, {} th data is wrong: "\ + "there is not '{}' object in the raw data.".format(i, field)) + if field in self._header.blob_fields: + field_type = type(v[field]).__name__ + if field_type not in VALUE_TYPE_MAP: + raise MRMValidateDataError("for schema, {} th data is wrong: "\ + "data type for '{}' is not matched.".format(i, field)) + if schema_content[field]["type"] not in VALUE_TYPE_MAP[field_type]: + raise MRMValidateDataError("for schema, {} th data is wrong: "\ + "data type for '{}' is not matched.".format(i, field)) + if field_type == 'ndarray': + if 'shape' not in schema_content[field]: + raise MRMValidateDataError("for schema, {} th data is wrong: " \ + "data type for '{}' is not matched.".format(i, field)) + try: + # tuple or list + np.reshape(v[field], schema_content[field]['shape']) + except ValueError: + raise MRMValidateDataError("for schema, {} th data is wrong: " \ + "data type for '{}' is not matched.".format(i, field)) + + def write_raw_data(self, raw_data, validate=True): + """ + Write raw data and generate sequential pair of MindRecord File. + + Args: + raw_data (list[dict]): List of raw data. + validate (bool, optional): Validate data according schema if it equals to True, + or validate data according to blob fields (default=True). + + Raises: + ParamTypeError: If index field is invalid. + MRMOpenError: If failed to open MindRecord File. + MRMValidateDataError: If data does not match blob fields. + MRMSetHeaderError: If failed to set header. + MRMWriteDatasetError: If failed to write dataset. + """ + if not self._writer.is_open: + self._writer.open(self._paths) + if not self._writer.get_shard_header(): + self._writer.set_shard_header(self._header) + if not isinstance(raw_data, list): + raise ParamTypeError('raw_data', 'list') + for each_raw in raw_data: + if not isinstance(each_raw, dict): + raise ParamTypeError('raw_data item', 'dict') + if validate is True: + self._verify_based_on_schema(raw_data) + elif validate is False: + self._verify_based_on_blob_fields(raw_data) + return self._writer.write_raw_data(raw_data, validate) + + def set_header_size(self, header_size): + """ + Set the size of header. + + Args: + header_size (int): Size of header, between 16KB and 128MB. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMInvalidHeaderSizeError: If failed to set header size. + + """ + return self._writer.set_header_size(header_size) + + def set_page_size(self, page_size): + """ + Set the size of Page. + + Args: + page_size (int): Size of page, between 32KB and 256MB. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMInvalidPageSizeError: If failed to set page size. + """ + return self._writer.set_page_size(page_size) + + def commit(self): + """ + Flush data to disk and generate the correspond db files. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMOpenError: If failed to open MindRecord File. + MRMSetHeaderError: If failed to set header. + MRMIndexGeneratorError: If failed to create index generator. + MRMGenerateIndexError: If failed to write to database. + MRMCommitError: If failed to flush data to disk. + """ + if not self._writer.is_open: + self._writer.open(self._paths) + # permit commit without data + if not self._writer.get_shard_header(): + self._writer.set_shard_header(self._header) + ret = self._writer.commit() + if self._index_generator is True: + if self._append: + self._generator = ShardIndexGenerator(self._file_name, self._append) + elif len(self._paths) >= 1: + self._generator = ShardIndexGenerator(os.path.realpath(self._paths[0]), self._append) + self._generator.build() + self._generator.write_to_db() + + # change the file mode to 600 + for item in self._paths: + if os.path.exists(item): + os.chmod(item, stat.S_IRUSR | stat.S_IWUSR) + index_file = item + ".db" + if os.path.exists(index_file): + os.chmod(index_file, stat.S_IRUSR | stat.S_IWUSR) + + return ret + + def _validate_array(self, k, v): + """ + Validate array item in schema + + Args: + k (str): Key in dict. + v (dict): Sub dict in schema + + Returns: + bool, True or False. + str, error message. + """ + if v['type'] not in VALID_ARRAY_ATTRIBUTES: + error = "Field '{}' contain illegal " \ + "attribute '{}'.".format(k, v['type']) + return False, error + if 'shape' in v: + if isinstance(v['shape'], list) is False: + error = "Field '{}' contain illegal " \ + "attribute '{}'.".format(k, v['shape']) + return False, error + else: + error = "Field '{}' contains illegal attributes.".format(v) + return False, error + return True, '' + + def _validate_schema(self, content): + """ + Validate schema and return validation result and error message. + + Args: + content (dict): Dict of raw schema. + + Returns: + bool, True or False. + str, error message. + """ + error = '' + if not content: + error = 'Schema content is empty.' + return False, error + if isinstance(content, dict) is False: + error = 'Schema content should be dict.' + return False, error + for k, v in content.items(): + if not re.match(r'^[0-9a-zA-Z\_]+$', k): + error = "Field '{}' should be composed of " \ + "'0-9' or 'a-z' or 'A-Z' or '_'.".format(k) + return False, error + if v and isinstance(v, dict): + if len(v) == 1 and 'type' in v: + if v['type'] not in VALID_ATTRIBUTES: + error = "Field '{}' contain illegal " \ + "attribute '{}'.".format(k, v['type']) + return False, error + elif len(v) == 2 and 'type' in v: + res_1, res_2 = self._validate_array(k, v) + if res_1 is not True: + return res_1, res_2 + else: + error = "Field '{}' contains illegal attributes.".format(v) + return False, error + else: + error = "Field '{}' should be dict.".format(k) + return False, error + return True, error diff --git a/mindspore/mindrecord/mindpage.py b/mindspore/mindrecord/mindpage.py new file mode 100644 index 00000000000..2d19006af48 --- /dev/null +++ b/mindspore/mindrecord/mindpage.py @@ -0,0 +1,164 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to support reading page from mindrecord. +""" + +from mindspore import log as logger +from .shardsegment import ShardSegment +from .shardutils import MIN_CONSUMER_COUNT, MAX_CONSUMER_COUNT, check_filename +from .common.exceptions import ParamValueError, ParamTypeError, MRMDefineCategoryError + +__all__ = ['MindPage'] + +class MindPage: + """ + Class to read MindRecord File series in pagination. + + Args: + file_name (str): File name of MindRecord File. + num_consumer(int, optional): Number of consumer threads which load data to memory (default=4). + It should not be smaller than 1 or larger than the number of CPU. + + Raises: + ParamValueError: If file_name, num_consumer or columns is invalid. + MRMInitSegmentError: If failed to initialize ShardSegment. + """ + def __init__(self, file_name, num_consumer=4): + check_filename(file_name) + self._file_name = file_name + + if num_consumer is not None: + if isinstance(num_consumer, int): + if num_consumer < MIN_CONSUMER_COUNT or num_consumer > MAX_CONSUMER_COUNT(): + raise ParamValueError("Consumer number should between {} and {}." + .format(MIN_CONSUMER_COUNT, MAX_CONSUMER_COUNT())) + else: + raise ParamValueError("Consumer number is illegal.") + else: + raise ParamValueError("Consumer number is illegal.") + + self._segment = ShardSegment() + self._segment.open(file_name, num_consumer) + self._category_field = None + self._candidate_fields = [field[:field.rfind('_')] for field in self._segment.get_category_fields()] + + @property + def candidate_fields(self): + """ + Return candidate category fields. + + Returns: + list[str], by which data could be grouped. + """ + return self._candidate_fields + + def get_category_fields(self): + """Return candidate category fields.""" + logger.warning("WARN_DEPRECATED: The usage of get_category_fields is deprecated." + " Please use candidate_fields") + return self.candidate_fields + + def set_category_field(self, category_field): + """ + Set category field for reading. + + Note: + Should be a candidate category field. + + Args: + category_field (str): String of category field name. + + Returns: + MSRStatus, SUCCESS or FAILED. + """ + logger.warning("WARN_DEPRECATED: The usage of set_category_field is deprecated." + " Please use category_field") + if not category_field or not isinstance(category_field, str): + raise ParamTypeError('category_fields', 'str') + if category_field not in self._candidate_fields: + raise MRMDefineCategoryError("Field '{}' is not a candidate category field.".format(category_field)) + return self._segment.set_category_field(category_field) + + @property + def category_field(self): + """Getter function for category field""" + return self._category_field + + @category_field.setter + def category_field(self, category_field): + """Setter function for category field""" + if not category_field or not isinstance(category_field, str): + raise ParamTypeError('category_fields', 'str') + if category_field not in self._candidate_fields: + raise MRMDefineCategoryError("Field '{}' is not a candidate category field.".format(category_field)) + self._category_field = category_field + return self._segment.set_category_field(self._category_field) + + def read_category_info(self): + """ + Return category information when data is grouped by indicated category field. + + Returns: + str, description of group information. + + Raises: + MRMReadCategoryInfoError: If failed to read category information. + """ + return self._segment.read_category_info() + + def read_at_page_by_id(self, category_id, page, num_row): + """ + Query by category id in pagination. + + Args: + category_id (int): Category id, referred to the return of read_category_info. + page (int): Index of page. + num_row (int): Number of rows in a page. + + Returns: + List, list[dict]. + + Raises: + ParamValueError: If any parameter is invalid. + MRMFetchDataError: If failed to read by category id. + MRMUnsupportedSchemaError: If schema is invalid. + """ + if category_id < 0: + raise ParamValueError("Category id should be greater than 0.") + if page < 0: + raise ParamValueError("Page should be greater than 0.") + if num_row < 0: + raise ParamValueError("num_row should be greater than 0.") + return self._segment.read_at_page_by_id(category_id, page, num_row) + + def read_at_page_by_name(self, category_name, page, num_row): + """ + Query by category name in pagination. + + Args: + category_name (str): String of category field's value, + referred to the return of read_category_info. + page (int): Index of page. + num_row (int): Number of row in a page. + + Returns: + str, read at page. + """ + if page < 0: + raise ParamValueError("Page should be greater than 0.") + if num_row < 0: + raise ParamValueError("num_row should be greater than 0.") + return self._segment.read_at_page_by_name(category_name, page, num_row) diff --git a/mindspore/mindrecord/shardheader.py b/mindspore/mindrecord/shardheader.py new file mode 100644 index 00000000000..a92abb90ae8 --- /dev/null +++ b/mindspore/mindrecord/shardheader.py @@ -0,0 +1,140 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to write data into mindrecord. +""" +import mindspore._c_mindrecord as ms +from mindspore import log as logger +from .common.exceptions import MRMAddSchemaError, MRMAddIndexError, MRMBuildSchemaError, MRMGetMetaError + +__all__ = ['ShardHeader'] + +class ShardHeader: + """ + Wrapper class which is represent ShardHeader class in c++ module. + + The class would store meta data of MindRecord File. + """ + def __init__(self, header=None): + if header: + self._header = header + else: + self._header = ms.ShardHeader() + + def add_schema(self, schema): + """ + Add object of ShardSchema. + + Args: + schema (ShardSchema): Object of ShardSchema. + + Returns: + int, schema id. + + Raises: + MRMAddSchemaError: If failed to add schema. + """ + schema_id = self._header.add_schema(schema) + if schema_id == -1: + logger.error("Failed to add schema.") + raise MRMAddSchemaError + return schema_id + + def add_index_fields(self, index_fields): + """ + Add object of ShardSchema. + + Args: + index_fields (list[str]): + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMAddSchemaError: If failed to add index field. + """ + ret = self._header.add_index_fields(index_fields) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to add index field.") + raise MRMAddIndexError + return ret + + def build_schema(self, content, desc=None): + """ + Build raw schema to generate schema object. + + Args: + content (dict): Dict of user defined schema. + desc (str,optional): String of schema description. + + Returns: + Class ShardSchema. + + Raises: + MRMBuildSchemaError: If failed to build schema. + """ + desc = desc if desc else "" + schema = ms.Schema.build(desc, content) + if not schema: + logger.error("Failed to add build schema.") + raise MRMBuildSchemaError + return schema + + @property + def header(self): + """Getter of header""" + return self._header + + def _get_schema(self): + """ + Get schema info. + + Returns: + List of dict. + """ + return self._get_meta()['schema'] + + def _get_blob_fields(self): + """ + Get blob fields info. + + Returns: + List of dict. + """ + return self._get_meta()['blob_fields'] + + def _get_meta(self): + """ + Get metadata including schema, blob fields .etc. + + Returns: + List of dict. + """ + ret = self._header.get_meta() + if ret and len(ret) == 1: + return ret[0].get_schema_content() + + logger.error("Failed to get meta info.") + raise MRMGetMetaError + + @property + def blob_fields(self): + """Getter of blob fields""" + return self._get_blob_fields() + + @property + def schema(self): + """Getter of schema""" + return self._get_schema() diff --git a/mindspore/mindrecord/shardindexgenerator.py b/mindspore/mindrecord/shardindexgenerator.py new file mode 100644 index 00000000000..825d16afe37 --- /dev/null +++ b/mindspore/mindrecord/shardindexgenerator.py @@ -0,0 +1,73 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to write data into mindrecord. +""" +import mindspore._c_mindrecord as ms +from mindspore import log as logger +from .common.exceptions import MRMIndexGeneratorError, MRMGenerateIndexError + +__all__ = ['ShardIndexGenerator'] + +class ShardIndexGenerator: + """ + Wrapper class which is represent ShardIndexGenerator class in c++ module. + + The class would generate db files for accelerating reading. + + Args: + path (str): Absolute path of MindRecord File. + append (bool): If True, open existed MindRecord Files for appending, or create new MindRecord Files. + + Raises: + MRMIndexGeneratorError: If failed to create index generator. + """ + def __init__(self, path, append=False): + self._generator = ms.ShardIndexGenerator(path, append) + if not self._generator: + logger.error("Failed to create index generator.") + raise MRMIndexGeneratorError + + def build(self): + """ + Build index generator. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMGenerateIndexError: If failed to build index generator. + """ + ret = self._generator.build() + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to build index generator.") + raise MRMGenerateIndexError + return ret + + def write_to_db(self): + """ + Create index field in table for reading data. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMGenerateIndexError: If failed to write to database. + """ + ret = self._generator.write_to_db() + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to write to database.") + raise MRMGenerateIndexError + return ret diff --git a/mindspore/mindrecord/shardreader.py b/mindspore/mindrecord/shardreader.py new file mode 100644 index 00000000000..de4b82f95a3 --- /dev/null +++ b/mindspore/mindrecord/shardreader.py @@ -0,0 +1,118 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to read data from mindrecord. +""" +import mindspore._c_mindrecord as ms +from mindspore import log as logger +from .common.exceptions import MRMOpenError, MRMLaunchError, MRMFinishError + +__all__ = ['ShardReader'] + +class ShardReader: + """ + Wrapper class which is represent ShardReader class in c++ module. + + The class would read a batch of data from MindRecord File series. + """ + def __init__(self): + self._reader = ms.ShardReader() + + def open(self, file_name, num_consumer=4, columns=None, operator=None): + """ + Open file and prepare to read MindRecord File. + + Args: + file_name (str): File name of MindRecord File. + num_consumer (int): Number of worker threads which load data in parallel. Default: 4. + columns (list[str]): List of fields which correspond data would be read. + operator(int): Reserved parameter for operators. Default: None. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMOpenError: If failed to open MindRecord File. + """ + columns = columns if columns else [] + operator = operator if operator else [] + ret = self._reader.open(file_name, num_consumer, columns, operator) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to open {}.".format(file_name)) + raise MRMOpenError + return ret + + def launch(self): + """ + Launch the worker threads to load data. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMLaunchError: If failed to launch worker threads. + """ + ret = self._reader.launch(False) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to launch worker threads.") + raise MRMLaunchError + return ret + + def get_next(self): + """ + Return a batch of data including blob data and raw data. + + Returns: + list of dict. + """ + return self._reader.get_next() + + def get_blob_fields(self): + """ + Return blob fields of MindRecord. + + Returns: + list of str. + """ + return self._reader.get_blob_fields() + + def get_header(self): + """ + Return header of MindRecord. + + Returns: + pointer object refer to header. + """ + return self._reader.get_header() + + def finish(self): + """ + stop the worker threads. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMFinishError: If failed to finish worker threads. + """ + ret = self._reader.finish() + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to finish worker threads.") + raise MRMFinishError + return ret + + def close(self): + """close MindRecord File.""" + self._reader.close() diff --git a/mindspore/mindrecord/shardsegment.py b/mindspore/mindrecord/shardsegment.py new file mode 100644 index 00000000000..963fe25fb9c --- /dev/null +++ b/mindspore/mindrecord/shardsegment.py @@ -0,0 +1,144 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to read page from mindrecord. +""" +import mindspore._c_mindrecord as ms +from mindspore import log as logger +from .shardutils import populate_data, SUCCESS +from .shardheader import ShardHeader +from .common.exceptions import MRMOpenError, MRMFetchCandidateFieldsError, MRMReadCategoryInfoError, MRMFetchDataError + +__all__ = ['ShardSegment'] + +class ShardSegment: + """ + Wrapper class which is represent ShardSegment class in c++ module. + + The class would query data from MindRecord File in pagination. + + """ + def __init__(self): + self._segment = ms.ShardSegment() + self._header = None + self._columns = None + + def open(self, file_name, num_consumer=4, columns=None, operator=None): + """ + Initialize the ShardSegment. + + Args: + file_name (str): File name of MindRecord File. + num_consumer (int): Number of worker threads which load data in parallel. Default: 4. + columns (list[str]): List of fields which correspond data would be read. + operator(int): Reserved parameter for operators. Default: None. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMOpenError: If failed to open MindRecord File. + """ + self._columns = columns if columns else [] + operator = operator if operator else [] + ret = self._segment.open(file_name, num_consumer, self._columns, operator) + if ret != SUCCESS: + logger.error("Failed to open {}.".format(file_name)) + raise MRMOpenError + self._header = ShardHeader(self._segment.get_header()) + return ret + + def get_category_fields(self): + """ + Get candidate category fields. + + Returns: + list[str], by which data could be grouped. + + Raises: + MRMFetchCandidateFieldsError: If failed to get candidate category fields. + """ + ret, fields = self._segment.get_category_fields() + if ret != SUCCESS: + logger.error("Failed to get candidate category fields.") + raise MRMFetchCandidateFieldsError + return fields + + + def set_category_field(self, category_field): + """Select one category field to use.""" + return self._segment.set_category_field(category_field) + + def read_category_info(self): + """ + Get the group info by the current category field. + + Returns: + str, description fo group information. + + Raises: + MRMReadCategoryInfoError: If failed to read category information. + """ + ret, category_info = self._segment.read_category_info() + if ret != SUCCESS: + logger.error("Failed to read category information.") + raise MRMReadCategoryInfoError + return category_info + + def read_at_page_by_id(self, category_id, page, num_row): + """ + Get the data of some page by category id. + + Args: + category_id (int): Category id, referred to the return of read_category_info. + page (int): Index of page. + num_row (int): Number of rows in a page. + + Returns: + list[dict] + + Raises: + MRMFetchDataError: If failed to read by category id. + MRMUnsupportedSchemaError: If schema is invalid. + """ + ret, data = self._segment.read_at_page_by_id(category_id, page, num_row) + if ret != SUCCESS: + logger.error("Failed to read by category id.") + raise MRMFetchDataError + return [populate_data(raw, blob, self._columns, self._header.blob_fields, + self._header.schema) for blob, raw in data] + + def read_at_page_by_name(self, category_name, page, num_row): + """ + Get the data of some page by category name. + + Args: + category_name (str): Category name, referred to the return of read_category_info. + page (int): Index of page. + num_row (int): Number of rows in a page. + + Returns: + list[dict] + + Raises: + MRMFetchDataError: If failed to read by category name. + MRMUnsupportedSchemaError: If schema is invalid. + """ + ret, data = self._segment.read_at_page_by_name(category_name, page, num_row) + if ret != SUCCESS: + logger.error("Failed to read by category name.") + raise MRMFetchDataError + return [populate_data(raw, blob, self._columns, self._header.blob_fields, + self._header.schema) for blob, raw in data] diff --git a/mindspore/mindrecord/shardutils.py b/mindspore/mindrecord/shardutils.py new file mode 100644 index 00000000000..2f57505800a --- /dev/null +++ b/mindspore/mindrecord/shardutils.py @@ -0,0 +1,130 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to write data into mindrecord. +""" +import os +import numpy as np +import mindspore._c_mindrecord as ms +from .common.exceptions import ParamValueError, MRMUnsupportedSchemaError + +SUCCESS = ms.MSRStatus.SUCCESS +FAILED = ms.MSRStatus.FAILED +DATASET_NLP = ms.ShardType.NLP +DATASET_CV = ms.ShardType.CV + +MIN_HEADER_SIZE = ms.MIN_HEADER_SIZE +MAX_HEADER_SIZE = ms.MAX_HEADER_SIZE +MIN_PAGE_SIZE = ms.MIN_PAGE_SIZE +MAX_PAGE_SIZE = ms.MAX_PAGE_SIZE +MIN_SHARD_COUNT = ms.MIN_SHARD_COUNT +MAX_SHARD_COUNT = ms.MAX_SHARD_COUNT +MIN_CONSUMER_COUNT = ms.MIN_CONSUMER_COUNT +MAX_CONSUMER_COUNT = ms.get_max_thread_num + +VALUE_TYPE_MAP = {"int": ["int32", "int64"], "float": ["float32", "float64"], "str": "string", "bytes": "bytes", + "int32": "int32", "int64": "int64", "float32": "float32", "float64": "float64", + "ndarray": ["int32", "int64", "float32", "float64"]} + +VALID_ATTRIBUTES = ["int32", "int64", "float32", "float64", "string", "bytes"] +VALID_ARRAY_ATTRIBUTES = ["int32", "int64", "float32", "float64"] + + +def check_filename(path): + """ + check the filename in the path. + + Args: + path (str): the path. + + Raises: + ParamValueError: If path is not string. + FileNameError: If path contains invalid character. + + Returns: + Bool, whether filename is valid. + """ + if not path: + raise ParamValueError('File path is not allowed None or empty!') + if not isinstance(path, str): + raise ParamValueError("File path: {} is not string.".format(path)) + file_name = os.path.basename(path) + + # '#', ':', '|', ' ', '}', '"', '+', '!', ']', '[', '\\', '`', + # '&', '.', '/', '@', "'", '^', ',', '_', '<', ';', '~', '>', + # '*', '(', '%', ')', '-', '=', '{', '?', '$' + forbidden_symbols = set(r'\/:*?"<>|`&\';') + + if set(file_name) & forbidden_symbols: + raise ParamValueError(r"File name should not contains \/:*?\"<>|`&;\'") + + if file_name.startswith(' ') or file_name.endswith(' '): + raise ParamValueError("File name should not start/end with space.") + + return True + +def populate_data(raw, blob, columns, blob_fields, schema): + """ + Reconstruct data form raw and blob data. + + Args: + raw (Dict): Data contain primitive data like "int32", "int64", "float32", "float64", "string", "bytes". + blob (Bytes): Data contain bytes and ndarray data. + columns(List): List of column name which will be populated. + blob_fields (List): Refer to the field which data stored in blob. + schema(Dict): Dict of Schema + + Raises: + MRMUnsupportedSchemaError: If schema is invalid. + """ + if raw: + # remove dummy fileds + raw = {k: v for k, v in raw.items() if k in schema} + if not blob_fields: + return raw + blob_bytes = bytes(blob) + + def _render_raw(field, blob_data): + data_type = schema[field]['type'] + data_shape = schema[field]['shape'] if 'shape' in schema[field] else [] + if columns and field not in columns: + return + if data_shape: + try: + raw[field] = np.reshape(np.frombuffer(blob_data, dtype=data_type), data_shape) + except ValueError: + raise MRMUnsupportedSchemaError('Shape in schema is illegal.') + else: + raw[field] = blob_data + + if len(blob_fields) == 1: + _render_raw(blob_fields[0], blob_bytes) + return raw + + def _int_from_bytes(xbytes: bytes) -> int: + return int.from_bytes(xbytes, 'big') + + def _blob_at_position(pos): + start = 0 + for _ in range(pos): + n_bytes = _int_from_bytes(blob_bytes[start : start + 8]) + start += 8 + n_bytes + n_bytes = _int_from_bytes(blob_bytes[start : start + 8]) + start += 8 + return blob_bytes[start : start + n_bytes] + + for i, blob_field in enumerate(blob_fields): + _render_raw(blob_field, _blob_at_position(i)) + return raw diff --git a/mindspore/mindrecord/shardwriter.py b/mindspore/mindrecord/shardwriter.py new file mode 100644 index 00000000000..0ef23d4ce66 --- /dev/null +++ b/mindspore/mindrecord/shardwriter.py @@ -0,0 +1,218 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module is to write data into mindrecord. +""" +import numpy as np +import mindspore._c_mindrecord as ms +from mindspore import log as logger +from .common.exceptions import MRMOpenError, MRMOpenForAppendError, MRMInvalidHeaderSizeError, \ + MRMInvalidPageSizeError, MRMSetHeaderError, MRMWriteDatasetError, MRMCommitError + +__all__ = ['ShardWriter'] + +class ShardWriter: + """ + Wrapper class which is represent shardWrite class in c++ module. + + The class would write MindRecord File series. + """ + def __init__(self): + self._writer = ms.ShardWriter() + self._header = None + self._is_open = False + + def open(self, paths): + """ + Open a new MindRecord File and prepare to write raw data. + + Args: + paths (list[str]): List of file path. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMOpenError: If failed to open MindRecord File. + """ + ret = self._writer.open(paths, False) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to open paths") + raise MRMOpenError + self._is_open = True + return ret + + def open_for_append(self, path): + """ + Open a existed MindRecord File and prepare to append raw data. + + Args: + path (str): String of file path. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMOpenForAppendError: If failed to append MindRecord File. + """ + ret = self._writer.open_for_append(path) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to open path to append.") + raise MRMOpenForAppendError + self._is_open = True + return ret + + def set_header_size(self, header_size): + """ + Set the size of header. + + Args: + header_size (int): Size of header, between 16KB and 128MB. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMInvalidHeaderSizeError: If failed to set header size. + """ + ret = self._writer.set_header_size(header_size) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to set header size.") + raise MRMInvalidHeaderSizeError + return ret + + def set_page_size(self, page_size): + """ + Set the size of page. + + Args: + page_size (int): Size of page, between 16KB and 128MB. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMInvalidPageSizeError: If failed to set page size. + """ + ret = self._writer.set_page_size(page_size) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to set page size.") + raise MRMInvalidPageSizeError + return ret + + def set_shard_header(self, shard_header): + """ + Set header which contains schema and index before write raw data. + + Args: + shard_header (ShardHeader): Object of ShardHeader. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMSetHeaderError: If failed to set header. + """ + self._header = shard_header + ret = self._writer.set_shard_header(shard_header.header) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to set header.") + raise MRMSetHeaderError + return ret + + def get_shard_header(self): + return self._header + + def write_raw_data(self, data, validate=True): + """ + Write raw data of cv dataset. + + Filter data according to schema and separate blob data from raw data. + Support data verify according to schema and remove the invalid data. + + Args: + data (list[dict]): List of raw data. + validate (bool, optional): verify data according schema if it equals to True. + + Returns: + MSRStatus, SUCCESS or FAILED. + + Raises: + MRMWriteCVError: If failed to write cv type dataset. + """ + blob_data = [] + raw_data = [] + # slice data to blob data and raw data + for item in data: + row_blob = self._merge_blob({field: item[field] for field in self._header.blob_fields}) + if row_blob: + blob_data.append(list(row_blob)) + # filter raw data according to schema + row_raw = {field: item[field] + for field in self._header.schema.keys() - self._header.blob_fields if field in item} + if row_raw: + raw_data.append(row_raw) + raw_data = {0: raw_data} if raw_data else {} + ret = self._writer.write_raw_data(raw_data, blob_data, validate) + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to write dataset.") + raise MRMWriteDatasetError + return ret + + def _merge_blob(self, blob_data): + """ + Merge multiple blob data whose type is bytes or ndarray + + Args: + blob_data (dict): Dict of blob data + + Returns: + bytes, merged blob data + """ + if len(blob_data) == 1: + values = [v for v in blob_data.values()] + return bytes(values[0]) + # convert int to bytes + def int_to_bytes(x: int) -> bytes: + return x.to_bytes(8, 'big') + merged = bytes() + for _, v in blob_data.items(): + # convert ndarray to bytes + if isinstance(v, np.ndarray): + v = v.tobytes() + merged += int_to_bytes(len(v)) + merged += v + return merged + + def commit(self): + """ + Flush data to disk. + + Returns: + Class MSRStatus, SUCCESS or FAILED. + + Raises: + MRMCommitError: If failed to flush data to disk. + """ + ret = self._writer.commit() + if ret != ms.MSRStatus.SUCCESS: + logger.error("Failed to commit.") + raise MRMCommitError + return ret + + @property + def is_open(self): + """getter function""" + return self._is_open diff --git a/mindspore/mindrecord/tools/__init__.py b/mindspore/mindrecord/tools/__init__.py new file mode 100644 index 00000000000..f0cbe66a55e --- /dev/null +++ b/mindspore/mindrecord/tools/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +init tools +""" diff --git a/mindspore/mindrecord/tools/cifar10.py b/mindspore/mindrecord/tools/cifar10.py new file mode 100644 index 00000000000..0ede2cc0caa --- /dev/null +++ b/mindspore/mindrecord/tools/cifar10.py @@ -0,0 +1,127 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Cifar10 reader class. +""" +import builtins +import io +import pickle +import re +import os +import numpy as np + +from ..shardutils import check_filename + +__all__ = ['Cifar10'] + +safe_builtins = { + 'range', + 'complex', + 'set', + 'frozenset', + 'slice', +} + +class RestrictedUnpickler(pickle.Unpickler): + """ + Unpickle allowing only few safe classes from the builtins module or numpy + + Raises: + pickle.UnpicklingError: If there is a problem unpickling an object + """ + def find_class(self, module, name): + # Only allow safe classes from builtins and numpy + if module == "builtins" and name in safe_builtins: + return getattr(builtins, name) + if module == "numpy.core.multiarray" and name == "_reconstruct": + return getattr(np.core.multiarray, name) + if module == "numpy": + return getattr(np, name) + # Forbid everything else. + raise pickle.UnpicklingError("global '%s.%s' is forbidden" %(module, name)) + +def restricted_loads(s): + """Helper function analogous to pickle.loads().""" + if isinstance(s, str): + raise TypeError("can not load pickle from unicode string") + f = io.BytesIO(s) + return RestrictedUnpickler(f, encoding='bytes').load() + +class Cifar10: + """ + Class to convert cifar10 to MindRecord. + + Args: + path (str): cifar10 directory which contain data_batch_* and test_batch. + one_hot (bool): one_hot flag. + """ + class Test: + pass + + def __init__(self, path, one_hot=True): + check_filename(path) + self.path = path + if not isinstance(one_hot, bool): + raise ValueError("The parameter one_hot must be bool") + self.one_hot = one_hot + self.images = None + self.labels = None + + def load_data(self): + """ + Returns a list which contain data & label, test & label. + + Returns: + list, train images, train labels and test images, test labels + """ + dic = {} + images = np.zeros([10000, 3, 32, 32]) + labels = [] + files = os.listdir(self.path) + for file in files: + if re.match("data_batch_*", file): + with open(os.path.join(self.path, file), 'rb') as f: #load train data + dic = restricted_loads(f.read()) + images = np.r_[images, dic[b"data"].reshape([-1, 3, 32, 32])] + labels.append(dic[b"labels"]) + elif re.match("test_batch", file): #load test data + with open(os.path.join(self.path, file), 'rb') as f: + dic = restricted_loads(f.read()) + test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32])) + test_labels = np.array(dic[b"labels"]) + dic["train_images"] = images[10000:].transpose(0, 2, 3, 1) + dic["train_labels"] = np.array(labels).reshape([-1, 1]) + dic["test_images"] = test_images.transpose(0, 2, 3, 1) + dic["test_labels"] = test_labels.reshape([-1, 1]) + if self.one_hot: + dic["train_labels"] = self._one_hot(dic["train_labels"], 10) + dic["test_labels"] = self._one_hot(dic["test_labels"], 10) + + self.images, self.labels = dic["train_images"], dic["train_labels"] + self.Test.images, self.Test.labels = dic["test_images"], dic["test_labels"] + return [dic["train_images"], dic["train_labels"], dic["test_images"], dic["test_labels"]] + + def _one_hot(self, labels, num): + """ + Returns a numpy. + + Returns: + Object, numpy array. + """ + size = labels.shape[0] + label_one_hot = np.zeros([size, num]) + for i in range(size): + label_one_hot[i, np.squeeze(labels[i])] = 1 + return label_one_hot diff --git a/mindspore/mindrecord/tools/cifar100.py b/mindspore/mindrecord/tools/cifar100.py new file mode 100644 index 00000000000..339025bb0ae --- /dev/null +++ b/mindspore/mindrecord/tools/cifar100.py @@ -0,0 +1,140 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Cifar100 reader class. +""" +import builtins +import io +import pickle +import os +import numpy as np + +from ..shardutils import check_filename + +__all__ = ['Cifar100'] + +safe_builtins = { + 'range', + 'complex', + 'set', + 'frozenset', + 'slice', +} + +class RestrictedUnpickler(pickle.Unpickler): + """ + Unpickle allowing only few safe classes from the builtins module or numpy + + Raises: + pickle.UnpicklingError: If there is a problem unpickling an object + """ + def find_class(self, module, name): + # Only allow safe classes from builtins and numpy + if module == "builtins" and name in safe_builtins: + return getattr(builtins, name) + if module == "numpy.core.multiarray" and name == "_reconstruct": + return getattr(np.core.multiarray, name) + if module == "numpy": + return getattr(np, name) + # Forbid everything else. + raise pickle.UnpicklingError("global '%s.%s' is forbidden" %(module, name)) + +def restricted_loads(s): + """Helper function analogous to pickle.loads().""" + if isinstance(s, str): + raise TypeError("can not load pickle from unicode string") + f = io.BytesIO(s) + return RestrictedUnpickler(f, encoding='bytes').load() + +class Cifar100: + """ + Class to convert cifar100 to MindRecord. + + Cifar100 contains train & test data. There are 500 training images and + 100 testing images per class. The 100 classes in the CIFAR-100 are grouped + into 20 superclasses. + + Args: + path (str): cifar100 directory which contain train and test binary data. + one_hot (bool): one_hot flag. + """ + class Test: + pass + + def __init__(self, path, one_hot=True): + check_filename(path) + self.path = path + if not isinstance(one_hot, bool): + raise ValueError("The parameter one_hot must be bool") + self.one_hot = one_hot + self.images = None + self.fine_labels = None + self.coarse_labels = None + + def load_data(self): + """ + Returns a list which contain train data & two labels, test data & two labels. + + Returns: + list, train and test images, fine labels, coarse labels. + """ + dic = {} + fine_labels = [] + coarse_labels = [] + files = os.listdir(self.path) + for file in files: + if file == "train": + with open(os.path.join(self.path, file), 'rb') as f: #load train data + dic = restricted_loads(f.read()) + images = np.array(dic[b"data"].reshape([-1, 3, 32, 32])) + fine_labels.append(dic[b"fine_labels"]) + coarse_labels.append(dic[b"coarse_labels"]) + elif file == "test": #load test data + with open(os.path.join(self.path, file), 'rb') as f: + dic = restricted_loads(f.read()) + test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32])) + test_fine_labels = np.array(dic[b"fine_labels"]) + test_coarse_labels = np.array(dic[b"coarse_labels"]) + dic["train_images"] = images.transpose(0, 2, 3, 1) + dic["train_fine_labels"] = np.array(fine_labels).reshape([-1, 1]) + dic["train_coarse_labels"] = np.array(coarse_labels).reshape([-1, 1]) + dic["test_images"] = test_images.transpose(0, 2, 3, 1) + dic["test_fine_labels"] = test_fine_labels.reshape([-1, 1]) + dic["test_coarse_labels"] = test_coarse_labels.reshape([-1, 1]) + if self.one_hot: + dic["train_fine_labels"] = self._one_hot(dic["train_fine_labels"], 100) + dic["train_coarse_labels"] = self._one_hot(dic["train_coarse_labels"], 20) + dic["test_fine_labels"] = self._one_hot(dic["test_fine_labels"], 100) + dic["test_coarse_labels"] = self._one_hot(dic["test_coarse_labels"], 20) + + self.images, self.fine_labels, self.coarse_labels = \ + dic["train_images"], dic["train_fine_labels"], dic["train_coarse_labels"] + self.Test.images, self.Test.fine_labels, self.Test.coarse_labels = \ + dic["test_images"], dic["test_fine_labels"], dic["test_coarse_labels"] + return [dic["train_images"], dic["train_fine_labels"], dic["train_coarse_labels"], + dic["test_images"], dic["test_fine_labels"], dic["test_coarse_labels"]] + + def _one_hot(self, labels, num): + """ + Returns a numpy. + + Returns: + Object, numpy array. + """ + size = labels.shape[0] + label_one_hot = np.zeros([size, num]) + for i in range(size): + label_one_hot[i, np.squeeze(labels[i])] = 1 + return label_one_hot diff --git a/mindspore/mindrecord/tools/cifar100_to_mr.py b/mindspore/mindrecord/tools/cifar100_to_mr.py new file mode 100644 index 00000000000..a359de853d7 --- /dev/null +++ b/mindspore/mindrecord/tools/cifar100_to_mr.py @@ -0,0 +1,155 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Cifar100 convert tool for MindRecord. +""" + +from importlib import import_module +import os +import numpy as np + +from mindspore import log as logger +from .cifar100 import Cifar100 +from ..common.exceptions import PathNotExistsError +from ..filewriter import FileWriter +from ..shardutils import check_filename +try: + cv2 = import_module("cv2") +except ModuleNotFoundError: + cv2 = None + +__all__ = ['Cifar100ToMR'] + +class Cifar100ToMR: + """ + Class is for transformation from cifar100 to MindRecord. + + Args: + source (str): the cifar100 directory to be transformed. + destination (str): the MindRecord file path to transform into. + + Raises: + ValueError: If source or destination is invalid. + """ + def __init__(self, source, destination): + check_filename(source) + self.source = source + + files = os.listdir(self.source) + train_data_flag = False + test_data_flag = False + for file in files: + if file == "train": + train_data_flag = True + if file == "test": + test_data_flag = True + if not train_data_flag: + raise PathNotExistsError("train") + + if not test_data_flag: + raise PathNotExistsError("test") + + check_filename(destination) + self.destination = destination + self.writer = None + + def transform(self, fields=None): + """ + Executes transformation from cifar100 to MindRecord. + + Args: + fields (list[str]): list of index field, ie. ["fine_label", "coarse_label"]. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + if fields and not isinstance(fields, list): + raise ValueError("The parameter fields should be None or list") + + cifar100_data = Cifar100(self.source, False) + cifar100_data.load_data() + + images = cifar100_data.images + logger.info("train images: {}".format(images.shape)) + fine_labels = cifar100_data.fine_labels + logger.info("train images fine label: {}".format(fine_labels.shape)) + coarse_labels = cifar100_data.coarse_labels + logger.info("train images coarse label: {}".format(coarse_labels.shape)) + + test_images = cifar100_data.Test.images + logger.info("test images: {}".format(test_images.shape)) + test_fine_labels = cifar100_data.Test.fine_labels + logger.info("test images fine label: {}".format(fine_labels.shape)) + test_coarse_labels = cifar100_data.Test.coarse_labels + logger.info("test images coarse label: {}".format(coarse_labels.shape)) + + data_list = _construct_raw_data(images, fine_labels, coarse_labels) + test_data_list = _construct_raw_data(test_images, test_fine_labels, test_coarse_labels) + + _generate_mindrecord(self.destination, data_list, fields, "img_train") + _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") + +def _construct_raw_data(images, fine_labels, coarse_labels): + """ + Construct raw data from cifar100 data. + + Args: + images (list): image list from cifar100. + fine_labels (list): fine label list from cifar100. + coarse_labels (list): coarse label list from cifar100. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + if not cv2: + raise ModuleNotFoundError("opencv-python module not found, please use pip install it.") + + raw_data = [] + for i, img in enumerate(images): + fine_label = np.int(fine_labels[i][0]) + coarse_label = np.int(coarse_labels[i][0]) + _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]]) + row_data = {"id": int(i), + "data": img.tobytes(), + "fine_label": int(fine_label), + "coarse_label": int(coarse_label)} + raw_data.append(row_data) + return raw_data + +def _generate_mindrecord(file_name, raw_data, fields, schema_desc): + """ + Generate MindRecord file from raw data. + + Args: + file_name (str): File name of MindRecord File. + fields (list[str]): Fields would be set as index which + could not belong to blob fields and type could not be 'array' or 'bytes'. + raw_data (dict): Dict of raw data. + schema_desc (str): String of schema description. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + schema = {"id": {"type": "int64"}, "fine_label": {"type": "int64"}, + "coarse_label": {"type": "int64"}, "data": {"type": "bytes"}} + + logger.info("transformed MindRecord schema is: {}".format(schema)) + + writer = FileWriter(file_name, 1) + writer.add_schema(schema, schema_desc) + if fields and isinstance(fields, list): + writer.add_index(fields) + writer.write_raw_data(raw_data) + return writer.commit() diff --git a/mindspore/mindrecord/tools/cifar10_to_mr.py b/mindspore/mindrecord/tools/cifar10_to_mr.py new file mode 100644 index 00000000000..53f2d7ddea3 --- /dev/null +++ b/mindspore/mindrecord/tools/cifar10_to_mr.py @@ -0,0 +1,151 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Cifar10 convert tool for MindRecord. +""" + +from importlib import import_module +import os +import numpy as np + +from mindspore import log as logger +from .cifar10 import Cifar10 +from ..common.exceptions import PathNotExistsError +from ..filewriter import FileWriter +from ..shardutils import check_filename, SUCCESS, FAILED +try: + cv2 = import_module("cv2") +except ModuleNotFoundError: + cv2 = None + +__all__ = ['Cifar10ToMR'] + +class Cifar10ToMR: + """ + Class is for transformation from cifar10 to MindRecord. + + Args: + source (str): the cifar10 directory to be transformed. + destination (str): the MindRecord file path to transform into. + + Raises: + ValueError: If source or destination is invalid. + """ + def __init__(self, source, destination): + check_filename(source) + self.source = source + + files = os.listdir(self.source) + train_data_flag = False + test_data_flag = False + for file in files: + if file.startswith("data_batch_"): + train_data_flag = True + if file.startswith("test_batch"): + test_data_flag = True + if not train_data_flag: + raise PathNotExistsError("data_batch_*") + + if not test_data_flag: + raise PathNotExistsError("test_batch") + + check_filename(destination) + self.destination = destination + self.writer = None + + def transform(self, fields=None): + """ + Executes transformation from cifar10 to MindRecord. + + Args: + fields (list[str], optional): list of index fields, ie. ["label"] (default=None). + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + if fields and not isinstance(fields, list): + raise ValueError("The parameter fields should be None or list") + + cifar10_data = Cifar10(self.source, False) + cifar10_data.load_data() + + images = cifar10_data.images + logger.info("train images: {}".format(images.shape)) + labels = cifar10_data.labels + logger.info("train images label: {}".format(labels.shape)) + + test_images = cifar10_data.Test.images + logger.info("test images: {}".format(test_images.shape)) + test_labels = cifar10_data.Test.labels + logger.info("test images label: {}".format(test_labels.shape)) + + data_list = _construct_raw_data(images, labels) + test_data_list = _construct_raw_data(test_images, test_labels) + + if _generate_mindrecord(self.destination, data_list, fields, "img_train") != SUCCESS: + return FAILED + if _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") != SUCCESS: + return FAILED + return SUCCESS + +def _construct_raw_data(images, labels): + """ + Construct raw data from cifar10 data. + + Args: + images (list): image list from cifar10. + labels (list): label list from cifar10. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + if not cv2: + raise ModuleNotFoundError("opencv-python module not found, please use pip install it.") + + raw_data = [] + for i, img in enumerate(images): + label = np.int(labels[i][0]) + _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]]) + row_data = {"id": int(i), + "data": img.tobytes(), + "label": int(label)} + raw_data.append(row_data) + return raw_data + +def _generate_mindrecord(file_name, raw_data, fields, schema_desc): + """ + Generate MindRecord file from raw data. + + Args: + file_name (str): File name of MindRecord File. + fields (list[str]): Fields would be set as index which + could not belong to blob fields and type could not be 'array' or 'bytes'. + raw_data (dict): dict of raw data. + schema_desc (str): String of schema description. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + schema = {"id": {"type": "int64"}, "label": {"type": "int64"}, + "data": {"type": "bytes"}} + + logger.info("transformed MindRecord schema is: {}".format(schema)) + + writer = FileWriter(file_name, 1) + writer.add_schema(schema, schema_desc) + if fields and isinstance(fields, list): + writer.add_index(fields) + writer.write_raw_data(raw_data) + return writer.commit() diff --git a/mindspore/mindrecord/tools/imagenet_to_mr.py b/mindspore/mindrecord/tools/imagenet_to_mr.py new file mode 100644 index 00000000000..8c8de689c1f --- /dev/null +++ b/mindspore/mindrecord/tools/imagenet_to_mr.py @@ -0,0 +1,168 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Imagenet convert tool for MindRecord. +""" +import os +import time + +from mindspore import log as logger +from ..common.exceptions import PathNotExistsError +from ..filewriter import FileWriter +from ..shardutils import check_filename + +__all__ = ['ImageNetToMR'] + +class ImageNetToMR: + """ + Class is for transformation from imagenet to MindRecord. + + Args: + map_file (str): the map file which indicate label. + the map file content should like this: + + .. code-block:: + + n02119789 1 pen + n02100735 2 notebook + n02110185 3 mouse + n02096294 4 orange + + image_dir (str): image directory contains n02119789, n02100735, n02110185, n02096294 dir. + destination (str): the MindRecord file path to transform into. + partition_number (int, optional): partition size (default=1). + + Raises: + ValueError: If map_file, image_dir or destination is invalid. + """ + def __init__(self, map_file, image_dir, destination, partition_number=1): + check_filename(map_file) + self.map_file = map_file + + check_filename(image_dir) + self.image_dir = image_dir + + check_filename(destination) + self.destination = destination + + if partition_number is not None: + if not isinstance(partition_number, int): + raise ValueError("The parameter partition_number must be int") + self.partition_number = partition_number + else: + raise ValueError("The parameter partition_number must be int") + + self.writer = FileWriter(self.destination, self.partition_number) + + def _get_imagenet_as_dict(self): + """ + Get data from imagenet as dict. + + Yields: + data (dict of list): imagenet data list which contains dict. + """ + if not os.path.exists(self.map_file): + raise IOError("map file {} not exists".format(self.map_file)) + + label_dict = {} + with open(self.map_file) as fp: + line = fp.readline() + while line: + labels = line.split(" ") + label_dict[labels[1]] = labels[0] + line = fp.readline() + + # get all the dir which are n02087046, n02094114, n02109525 + dir_paths = {} + for item in label_dict: + real_path = os.path.join(self.image_dir, label_dict[item]) + if not os.path.isdir(real_path): + logger.warning("{} dir is not exist".format(real_path)) + continue + dir_paths[item] = real_path + + if not dir_paths: + raise PathNotExistsError("not valid image dir in {}".format(self.image_dir)) + + # get the filename, label and image binary as a dict + for label in dir_paths: + for item in os.listdir(dir_paths[label]): + file_name = os.path.join(dir_paths[label], item) + if not item.endswith("JPEG") and not item.endswith("jpg"): + logger.warning("{} file is not suffix with JPEG/jpg, skip it.".format(file_name)) + continue + data = {} + data["file_name"] = str(file_name) + data["label"] = int(label) + + # get the image data + image_file = open(file_name, "rb") + image_bytes = image_file.read() + image_file.close() + data["data"] = image_bytes + yield data + + def transform(self): + """ + Executes transformation from imagenet to MindRecord. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + t0_total = time.time() + + imagenet_schema_json = {"label": {"type": "int64"}, + "data": {"type": "bytes"}, + "file_name": {"type": "string"}} + + logger.info("transformed MindRecord schema is: {}".format(imagenet_schema_json)) + + # set the header size + self.writer.set_header_size(1<<24) + + # set the page size + self.writer.set_page_size(1<<26) + + # create the schema + self.writer.add_schema(imagenet_schema_json, "imagenet_schema") + + # add the index + self.writer.add_index(["label", "file_name"]) + + imagenet_iter = self._get_imagenet_as_dict() + batch_size = 256 + transform_count = 0 + while True: + data_list = [] + try: + for _ in range(batch_size): + data_list.append(imagenet_iter.__next__()) + transform_count += 1 + self.writer.write_raw_data(data_list) + logger.info("transformed {} record...".format(transform_count)) + except StopIteration: + if data_list: + self.writer.write_raw_data(data_list) + logger.info("transformed {} record...".format(transform_count)) + break + + ret = self.writer.commit() + + t1_total = time.time() + logger.info("--------------------------------------------") + logger.info("END. Total time: {}".format(t1_total - t0_total)) + logger.info("--------------------------------------------") + + return ret diff --git a/mindspore/mindrecord/tools/mnist_to_mr.py b/mindspore/mindrecord/tools/mnist_to_mr.py new file mode 100644 index 00000000000..462ab7fb53b --- /dev/null +++ b/mindspore/mindrecord/tools/mnist_to_mr.py @@ -0,0 +1,235 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Mnist convert tool for MindRecord +""" + +from importlib import import_module +import os +import time +import gzip +import numpy as np + +from mindspore import log as logger +from ..filewriter import FileWriter +from ..shardutils import check_filename, SUCCESS, FAILED + +try: + cv2 = import_module("cv2") +except ModuleNotFoundError: + cv2 = None + +__all__ = ['MnistToMR'] + +class MnistToMR: + """ + Class is for transformation from Mnist to MindRecord. + + Args: + source (str): directory which contain t10k-images-idx3-ubyte.gz, + train-images-idx3-ubyte.gz, t10k-labels-idx1-ubyte.gz, + train-labels-idx1-ubyte.gz. + destination (str): the MindRecord file directory to transform into. + partition_number (int, optional): partition size (default=1). + + Raises: + ValueError: If source/destination/partition_number is invalid. + """ + + def __init__(self, source, destination, partition_number=1): + self.image_size = 28 + self.num_channels = 1 + + check_filename(source) + + self.source = source + self.train_data_filename_ = os.path.join(self.source, 'train-images-idx3-ubyte.gz') + self.train_labels_filename_ = os.path.join(self.source, 'train-labels-idx1-ubyte.gz') + self.test_data_filename_ = os.path.join(self.source, 't10k-images-idx3-ubyte.gz') + self.test_labels_filename_ = os.path.join(self.source, 't10k-labels-idx1-ubyte.gz') + + check_filename(self.train_data_filename_) + check_filename(self.train_labels_filename_) + check_filename(self.test_data_filename_) + check_filename(self.test_labels_filename_) + check_filename(destination) + + if partition_number is not None: + if not isinstance(partition_number, int): + raise ValueError("The parameter partition_number must be int") + self.partition_number = partition_number + else: + raise ValueError("The parameter partition_number must be int") + + self.writer_train = FileWriter("{}_train.mindrecord".format(destination), self.partition_number) + self.writer_test = FileWriter("{}_test.mindrecord".format(destination), self.partition_number) + + self.mnist_schema_json = {"label": {"type": "int64"}, "data": {"type": "bytes"}} + + def _extract_images(self, filename, num_images): + """Extract the images into a 4D tensor [image index, y, x, channels].""" + with gzip.open(filename) as bytestream: + bytestream.read(16) + buf = bytestream.read(self.image_size * self.image_size * num_images * self.num_channels) + data = np.frombuffer(buf, dtype=np.uint8) + data = data.reshape(num_images, self.image_size, self.image_size, self.num_channels) + return data + + def _extract_labels(self, filename, num_images): + """Extract the labels into a vector of int64 label IDs.""" + with gzip.open(filename) as bytestream: + bytestream.read(8) + buf = bytestream.read(1 * num_images) + labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64) + return labels + + def _mnist_train_iterator(self): + """ + get data from mnist train data and label file. + + Yields: + data (dict of list): mnist data list which contains dict. + """ + train_data = self._extract_images(self.train_data_filename_, 60000) + train_labels = self._extract_labels(self.train_labels_filename_, 60000) + for data, label in zip(train_data, train_labels): + _, img = cv2.imencode(".jpeg", data) + yield {"label": int(label), "data": img.tobytes()} + + def _mnist_test_iterator(self): + """ + get data from mnist test data and label file. + + Yields: + data (dict of list): mnist data list which contains dict. + """ + test_data = self._extract_images(self.test_data_filename_, 10000) + test_labels = self._extract_labels(self.test_labels_filename_, 10000) + for data, label in zip(test_data, test_labels): + _, img = cv2.imencode(".jpeg", data) + yield {"label": int(label), "data": img.tobytes()} + + def _transform_train(self): + """ + Executes transformation from Mnist train part to MindRecord. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + t0_total = time.time() + + logger.info("transformed MindRecord schema is: {}".format(self.mnist_schema_json)) + + # set the header size + self.writer_train.set_header_size(1 << 24) + + # set the page size + self.writer_train.set_page_size(1 << 26) + + # create the schema + self.writer_train.add_schema(self.mnist_schema_json, "mnist_schema") + # add the index + self.writer_train.add_index(["label"]) + + train_iter = self._mnist_train_iterator() + batch_size = 256 + transform_count = 0 + while True: + data_list = [] + try: + for _ in range(batch_size): + data_list.append(train_iter.__next__()) + transform_count += 1 + self.writer_train.write_raw_data(data_list) + logger.info("transformed {} record...".format(transform_count)) + except StopIteration: + if data_list: + self.writer_train.write_raw_data(data_list) + logger.info("transformed {} record...".format(transform_count)) + break + + ret = self.writer_train.commit() + + t1_total = time.time() + logger.info("--------------------------------------------") + logger.info("Total time [train]: {}".format(t1_total - t0_total)) + logger.info("--------------------------------------------") + + return ret + + def _transform_test(self): + """ + Executes transformation from Mnist test part to MindRecord. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + t0_total = time.time() + + logger.info("transformed MindRecord schema is: {}".format(self.mnist_schema_json)) + + # set the header size + self.writer_test.set_header_size(1 << 24) + + # set the page size + self.writer_test.set_page_size(1 << 26) + + # create the schema + self.writer_test.add_schema(self.mnist_schema_json, "mnist_schema") + + # add the index + self.writer_test.add_index(["label"]) + + train_iter = self._mnist_test_iterator() + batch_size = 256 + transform_count = 0 + while True: + data_list = [] + try: + for _ in range(batch_size): + data_list.append(train_iter.__next__()) + transform_count += 1 + self.writer_test.write_raw_data(data_list) + logger.info("transformed {} record...".format(transform_count)) + except StopIteration: + if data_list: + self.writer_test.write_raw_data(data_list) + logger.info("transformed {} record...".format(transform_count)) + break + + ret = self.writer_test.commit() + + t1_total = time.time() + logger.info("--------------------------------------------") + logger.info("Total time [test]: {}".format(t1_total - t0_total)) + logger.info("--------------------------------------------") + + return ret + + def transform(self): + """ + Executes transformation from Mnist to MindRecord. + + Returns: + SUCCESS/FAILED, whether successfully written into MindRecord. + """ + if not cv2: + raise ModuleNotFoundError("opencv-python module not found, please use pip install it.") + + if self._transform_train() == FAILED: + return FAILED + if self._transform_test() == FAILED: + return FAILED + + return SUCCESS diff --git a/mindspore/model_zoo/Bert_NEZHA/__init__.py b/mindspore/model_zoo/Bert_NEZHA/__init__.py new file mode 100644 index 00000000000..4f4584a4b48 --- /dev/null +++ b/mindspore/model_zoo/Bert_NEZHA/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Bert Init.""" +from .bert_for_pre_training import BertNetworkWithLoss, BertPreTraining, \ + BertPretrainingLoss, GetMaskedLMOutput, GetNextSentenceOutput, \ + BertTrainOneStepCell, BertTrainOneStepWithLossScaleCell +from .bert_model import BertAttention, BertConfig, BertEncoderCell, BertModel, \ + BertOutput, BertSelfAttention, BertTransformer, EmbeddingLookup, \ + EmbeddingPostprocessor, RelaPosEmbeddingsGenerator, RelaPosMatrixGenerator, \ + SaturateCast, CreateAttentionMaskFromInputMask + +__all__ = [ + "BertNetworkWithLoss", "BertPreTraining", "BertPretrainingLoss", + "GetMaskedLMOutput", "GetNextSentenceOutput", "BertTrainOneStepCell", "BertTrainOneStepWithLossScaleCell", + "BertAttention", "BertConfig", "BertEncoderCell", "BertModel", "BertOutput", + "BertSelfAttention", "BertTransformer", "EmbeddingLookup", + "EmbeddingPostprocessor", "RelaPosEmbeddingsGenerator", + "RelaPosMatrixGenerator", "SaturateCast", "CreateAttentionMaskFromInputMask" +] diff --git a/mindspore/model_zoo/Bert_NEZHA/bert_for_pre_training.py b/mindspore/model_zoo/Bert_NEZHA/bert_for_pre_training.py new file mode 100644 index 00000000000..22a212a489c --- /dev/null +++ b/mindspore/model_zoo/Bert_NEZHA/bert_for_pre_training.py @@ -0,0 +1,450 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Bert for pretraining.""" +import numpy as np + +import mindspore.nn as nn +from mindspore.common.initializer import initializer, TruncatedNormal +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore.common import dtype as mstype +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +from mindspore.train.parallel_utils import ParallelMode +from mindspore.communication.management import get_group_size +from mindspore import context +from .bert_model import BertModel + +GRADIENT_CLIP_TYPE = 1 +GRADIENT_CLIP_VALUE = 1.0 + + +class ClipGradients(nn.Cell): + """ + Clip gradients. + + Inputs: + grads (tuple[Tensor]): Gradients. + clip_type (int): The way to clip, 0 for 'value', 1 for 'norm'. + clip_value (float): Specifies how much to clip. + + Outputs: + tuple[Tensor], clipped gradients. + """ + def __init__(self): + super(ClipGradients, self).__init__() + self.clip_by_norm = nn.ClipByNorm() + self.cast = P.Cast() + self.dtype = P.DType() + + def construct(self, + grads, + clip_type, + clip_value): + if clip_type != 0 and clip_type != 1: + return grads + + new_grads = () + for grad in grads: + dt = self.dtype(grad) + if clip_type == 0: + t = C.clip_by_value(grad, self.cast(F.tuple_to_array((-clip_value,)), dt), + self.cast(F.tuple_to_array((clip_value,)), dt)) + else: + t = self.clip_by_norm(grad, self.cast(F.tuple_to_array((clip_value,)), dt)) + new_grads = new_grads + (t,) + + return new_grads + + +class GetMaskedLMOutput(nn.Cell): + """ + Get masked lm output. + + Args: + config (BertConfig): The config of BertModel. + + Returns: + Tensor, masked lm output. + """ + def __init__(self, config): + super(GetMaskedLMOutput, self).__init__() + self.width = config.hidden_size + self.reshape = P.Reshape() + self.gather = P.GatherV2() + + weight_init = TruncatedNormal(config.initializer_range) + self.dense = nn.Dense(self.width, + config.hidden_size, + weight_init=weight_init, + activation=config.hidden_act).to_float(config.compute_type) + self.layernorm = nn.LayerNorm(config.hidden_size).to_float(config.compute_type) + self.output_bias = Parameter( + initializer( + 'zero', + config.vocab_size), + name='output_bias') + self.matmul = P.MatMul(transpose_b=True) + self.log_softmax = nn.LogSoftmax(axis=-1) + self.shape_flat_offsets = (-1, 1) + self.rng = Tensor(np.array(range(0, config.batch_size)).astype(np.int32)) + self.last_idx = (-1,) + self.shape_flat_sequence_tensor = (config.batch_size * config.seq_length, self.width) + self.seq_length_tensor = Tensor(np.array((config.seq_length,)).astype(np.int32)) + self.cast = P.Cast() + self.compute_type = config.compute_type + self.dtype = config.dtype + + def construct(self, + input_tensor, + output_weights, + positions): + flat_offsets = self.reshape( + self.rng * self.seq_length_tensor, self.shape_flat_offsets) + flat_position = self.reshape(positions + flat_offsets, self.last_idx) + flat_sequence_tensor = self.reshape(input_tensor, self.shape_flat_sequence_tensor) + input_tensor = self.gather(flat_sequence_tensor, flat_position, 0) + input_tensor = self.cast(input_tensor, self.compute_type) + output_weights = self.cast(output_weights, self.compute_type) + input_tensor = self.dense(input_tensor) + input_tensor = self.layernorm(input_tensor) + logits = self.matmul(input_tensor, output_weights) + logits = self.cast(logits, self.dtype) + logits = logits + self.output_bias + log_probs = self.log_softmax(logits) + return log_probs + + +class GetNextSentenceOutput(nn.Cell): + """ + Get next sentence output. + + Args: + config (BertConfig): The config of Bert. + + Returns: + Tensor, next sentence output. + """ + def __init__(self, config): + super(GetNextSentenceOutput, self).__init__() + self.log_softmax = P.LogSoftmax() + self.weight_init = TruncatedNormal(config.initializer_range) + self.dense = nn.Dense(config.hidden_size, 2, + weight_init=self.weight_init, has_bias=True).to_float(config.compute_type) + self.dtype = config.dtype + self.cast = P.Cast() + + def construct(self, input_tensor): + logits = self.dense(input_tensor) + logits = self.cast(logits, self.dtype) + log_prob = self.log_softmax(logits) + return log_prob + + +class BertPreTraining(nn.Cell): + """ + Bert pretraining network. + + Args: + config (BertConfig): The config of BertModel. + is_training (bool): Specifies whether to use the training mode. + use_one_hot_embeddings (bool): Specifies whether to use one-hot for embeddings. + + Returns: + Tensor, prediction_scores, seq_relationship_score. + """ + def __init__(self, config, is_training, use_one_hot_embeddings): + super(BertPreTraining, self).__init__() + self.bert = BertModel(config, is_training, use_one_hot_embeddings) + self.cls1 = GetMaskedLMOutput(config) + self.cls2 = GetNextSentenceOutput(config) + + def construct(self, input_ids, input_mask, token_type_id, + masked_lm_positions): + sequence_output, pooled_output, embedding_table = \ + self.bert(input_ids, token_type_id, input_mask) + prediction_scores = self.cls1(sequence_output, + embedding_table, + masked_lm_positions) + seq_relationship_score = self.cls2(pooled_output) + return prediction_scores, seq_relationship_score + + +class BertPretrainingLoss(nn.Cell): + """ + Provide bert pre-training loss. + + Args: + config (BertConfig): The config of BertModel. + + Returns: + Tensor, total loss. + """ + def __init__(self, config): + super(BertPretrainingLoss, self).__init__() + self.vocab_size = config.vocab_size + self.onehot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.reduce_sum = P.ReduceSum() + self.reduce_mean = P.ReduceMean() + self.reshape = P.Reshape() + self.last_idx = (-1,) + self.neg = P.Neg() + self.cast = P.Cast() + + def construct(self, prediction_scores, seq_relationship_score, masked_lm_ids, + masked_lm_weights, next_sentence_labels): + """Defines the computation performed.""" + label_ids = self.reshape(masked_lm_ids, self.last_idx) + label_weights = self.cast(self.reshape(masked_lm_weights, self.last_idx), mstype.float32) + one_hot_labels = self.onehot(label_ids, self.vocab_size, self.on_value, self.off_value) + + per_example_loss = self.neg(self.reduce_sum(prediction_scores * one_hot_labels, self.last_idx)) + numerator = self.reduce_sum(label_weights * per_example_loss, ()) + denominator = self.reduce_sum(label_weights, ()) + self.cast(F.tuple_to_array((1e-5,)), mstype.float32) + masked_lm_loss = numerator / denominator + + # next_sentence_loss + labels = self.reshape(next_sentence_labels, self.last_idx) + one_hot_labels = self.onehot(labels, 2, self.on_value, self.off_value) + per_example_loss = self.neg(self.reduce_sum( + one_hot_labels * seq_relationship_score, self.last_idx)) + next_sentence_loss = self.reduce_mean(per_example_loss, self.last_idx) + + # total_loss + total_loss = masked_lm_loss + next_sentence_loss + + return total_loss + + +class BertNetworkWithLoss(nn.Cell): + """ + Provide bert pre-training loss through network. + + Args: + config (BertConfig): The config of BertModel. + is_training (bool): Specifies whether to use the training mode. + use_one_hot_embeddings (bool): Specifies whether to use one-hot for embeddings. Default: False. + + Returns: + Tensor, the loss of the network. + """ + def __init__(self, config, is_training, use_one_hot_embeddings=False): + super(BertNetworkWithLoss, self).__init__() + self.bert = BertPreTraining(config, is_training, use_one_hot_embeddings) + self.loss = BertPretrainingLoss(config) + self.cast = P.Cast() + + def construct(self, + input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights): + prediction_scores, seq_relationship_score = \ + self.bert(input_ids, input_mask, token_type_id, masked_lm_positions) + total_loss = self.loss(prediction_scores, seq_relationship_score, + masked_lm_ids, masked_lm_weights, next_sentence_labels) + return self.cast(total_loss, mstype.float32) + + +class BertTrainOneStepCell(nn.Cell): + """ + Encapsulation class of bert network training. + + Append an optimizer to the training network after that the construct + function can be called to create the backward graph. + + Args: + network (Cell): The training network. Note that loss function should have been added. + optimizer (Optimizer): Optimizer for updating the weights. + sens (Number): The adjust parameter. Default: 1.0. + """ + def __init__(self, network, optimizer, sens=1.0): + super(BertTrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]: + self.reducer_flag = True + self.grad_reducer = None + if self.reducer_flag: + mean = context.get_auto_parallel_context("mirror_mean") + degree = get_group_size() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + + self.clip_gradients = ClipGradients() + self.cast = P.Cast() + + def set_sens(self, value): + self.sens = value + + def construct(self, + input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights): + """Defines the computation performed.""" + weights = self.weights + + loss = self.network(input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights) + grads = self.grad(self.network, weights)(input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights, + self.cast(F.tuple_to_array((self.sens,)), + mstype.float32)) + grads = self.clip_gradients(grads, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + + succ = self.optimizer(grads) + return F.depend(loss, succ) + + +grad_scale = C.MultitypeFuncGraph("grad_scale") +reciprocal = P.Reciprocal() + + +@grad_scale.register("Tensor", "Tensor") +def tensor_grad_scale(scale, grad): + return grad * reciprocal(scale) + + +class BertTrainOneStepWithLossScaleCell(nn.Cell): + """ + Encapsulation class of bert network training. + + Append an optimizer to the training network after that the construct + function can be called to create the backward graph. + + Args: + network (Cell): The training network. Note that loss function should have been added. + optimizer (Optimizer): Optimizer for updating the weights. + scale_update_cell (Cell): Cell to do the loss scale. Default: None. + """ + def __init__(self, network, optimizer, scale_update_cell=None): + super(BertTrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + self.reducer_flag = False + self.allreduce = P.AllReduce() + self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]: + self.reducer_flag = True + self.grad_reducer = None + if self.reducer_flag: + mean = context.get_auto_parallel_context("mirror_mean") + degree = get_group_size() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) + self.clip_gradients = ClipGradients() + self.cast = P.Cast() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_before_grad = P.NPUClearFloatStatus() + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.depend_parameter_use = P.ControlDepend(depend_mode=1) + self.base = Tensor(1, mstype.float32) + self.less_equal = P.LessEqual() + self.hyper_map = C.HyperMap() + self.loss_scale = None + self.loss_scaling_manager = scale_update_cell + if scale_update_cell: + self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), + name="loss_scale") + self.add_flags(has_effect=True) + def construct(self, + input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights, + sens=None): + """Defines the computation performed.""" + weights = self.weights + # alloc status + init = self.alloc_status() + self.clear_before_grad(init) + loss = self.network(input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights) + if sens is None: + scaling_sens = self.loss_scale + else: + scaling_sens = sens + grads = self.grad(self.network, weights)(input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights, + self.cast(scaling_sens, + mstype.float32)) + grads = self.hyper_map(F.partial(grad_scale, scaling_sens), grads) + grads = self.clip_gradients(grads, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + self.get_status(init) + flag_sum = self.reduce_sum(init, (0,)) + if self.is_distributed: + # sum overflow flag over devices + flag_reduce = self.allreduce(flag_sum) + cond = self.less_equal(self.base, flag_reduce) + else: + cond = self.less_equal(self.base, flag_sum) + overflow = cond + if sens is None: + overflow = self.loss_scaling_manager(self.loss_scale, cond) + if overflow: + succ = False + else: + succ = self.optimizer(grads) + ret = (loss, cond) + return F.depend(ret, succ) diff --git a/mindspore/model_zoo/Bert_NEZHA/bert_model.py b/mindspore/model_zoo/Bert_NEZHA/bert_model.py new file mode 100644 index 00000000000..f20c57dd751 --- /dev/null +++ b/mindspore/model_zoo/Bert_NEZHA/bert_model.py @@ -0,0 +1,917 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Bert model.""" + +import math +import copy +import numpy as np +import mindspore.common.dtype as mstype +import mindspore.nn as nn +import mindspore.ops.functional as F +from mindspore.common.initializer import TruncatedNormal, initializer +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter + + +class BertConfig: + """ + Configuration for `BertModel`. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input sequence. Default: 128. + vocab_size (int): The shape of each embedding vector. Default: 32000. + hidden_size (int): Size of the bert encoder layers. Default: 768. + num_hidden_layers (int): Number of hidden layers in the BertTransformer encoder + cell. Default: 12. + num_attention_heads (int): Number of attention heads in the BertTransformer + encoder cell. Default: 12. + intermediate_size (int): Size of intermediate layer in the BertTransformer + encoder cell. Default: 3072. + hidden_act (str): Activation function used in the BertTransformer encoder + cell. Default: "gelu". + hidden_dropout_prob (float): The dropout probability for BertOutput. Default: 0.1. + attention_probs_dropout_prob (float): The dropout probability for + BertAttention. Default: 0.1. + max_position_embeddings (int): Maximum length of sequences used in this + model. Default: 512. + type_vocab_size (int): Size of token type vocab. Default: 16. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + use_relative_positions (bool): Specifies whether to use relative positions. Default: False. + input_mask_from_dataset (bool): Specifies whether to use the input mask that loaded from + dataset. Default: True. + token_type_ids_from_dataset (bool): Specifies whether to use the token type ids that loaded + from dataset. Default: True. + dtype (:class:`mindspore.dtype`): Data type of the input. Default: mstype.float32. + compute_type (:class:`mindspore.dtype`): Compute type in BertTransformer. Default: mstype.float32. + """ + def __init__(self, + batch_size, + seq_length=128, + vocab_size=32000, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + use_relative_positions=False, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32): + self.batch_size = batch_size + self.seq_length = seq_length + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.input_mask_from_dataset = input_mask_from_dataset + self.token_type_ids_from_dataset = token_type_ids_from_dataset + self.use_relative_positions = use_relative_positions + self.dtype = dtype + self.compute_type = compute_type + + +class EmbeddingLookup(nn.Cell): + """ + A embeddings lookup table with a fixed dictionary and size. + + Args: + vocab_size (int): Size of the dictionary of embeddings. + embedding_size (int): The size of each embedding vector. + embedding_shape (list): [batch_size, seq_length, embedding_size], the shape of + each embedding vector. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + """ + def __init__(self, + vocab_size, + embedding_size, + embedding_shape, + use_one_hot_embeddings=False, + initializer_range=0.02): + super(EmbeddingLookup, self).__init__() + self.vocab_size = vocab_size + self.use_one_hot_embeddings = use_one_hot_embeddings + self.embedding_table = Parameter(initializer + (TruncatedNormal(initializer_range), + [vocab_size, embedding_size]), + name='embedding_table') + self.expand = P.ExpandDims() + self.shape_flat = (-1,) + self.gather = P.GatherV2() + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.array_mul = P.MatMul() + self.reshape = P.Reshape() + self.shape = tuple(embedding_shape) + + def construct(self, input_ids): + extended_ids = self.expand(input_ids, -1) + flat_ids = self.reshape(extended_ids, self.shape_flat) + if self.use_one_hot_embeddings: + one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value) + output_for_reshape = self.array_mul( + one_hot_ids, self.embedding_table) + else: + output_for_reshape = self.gather(self.embedding_table, flat_ids, 0) + output = self.reshape(output_for_reshape, self.shape) + return output, self.embedding_table + + +class EmbeddingPostprocessor(nn.Cell): + """ + Postprocessors apply positional and token type embeddings to word embeddings. + + Args: + embedding_size (int): The size of each embedding vector. + embedding_shape (list): [batch_size, seq_length, embedding_size], the shape of + each embedding vector. + use_token_type (bool): Specifies whether to use token type embeddings. Default: False. + token_type_vocab_size (int): Size of token type vocab. Default: 16. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + max_position_embeddings (int): Maximum length of sequences used in this + model. Default: 512. + dropout_prob (float): The dropout probability. Default: 0.1. + """ + def __init__(self, + embedding_size, + embedding_shape, + use_token_type=False, + token_type_vocab_size=16, + use_one_hot_embeddings=False, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1): + super(EmbeddingPostprocessor, self).__init__() + self.use_token_type = use_token_type + self.token_type_vocab_size = token_type_vocab_size + self.use_one_hot_embeddings = use_one_hot_embeddings + self.max_position_embeddings = max_position_embeddings + self.embedding_table = Parameter(initializer + (TruncatedNormal(initializer_range), + [token_type_vocab_size, + embedding_size]), + name='embedding_table') + + self.shape_flat = (-1,) + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.1, mstype.float32) + self.array_mul = P.MatMul() + self.reshape = P.Reshape() + self.shape = tuple(embedding_shape) + self.layernorm = nn.LayerNorm(embedding_size) + self.dropout = nn.Dropout(1 - dropout_prob) + self.gather = P.GatherV2() + + def construct(self, token_type_ids, word_embeddings): + output = word_embeddings + if self.use_token_type: + flat_ids = self.reshape(token_type_ids, self.shape_flat) + if self.use_one_hot_embeddings: + one_hot_ids = self.one_hot(flat_ids, + self.token_type_vocab_size, self.on_value, self.off_value) + token_type_embeddings = self.array_mul(one_hot_ids, + self.embedding_table) + else: + token_type_embeddings = self.gather(self.embedding_table, flat_ids, 0) + token_type_embeddings = self.reshape(token_type_embeddings, self.shape) + output += token_type_embeddings + output = self.layernorm(output) + output = self.dropout(output) + return output + + +class BertOutput(nn.Cell): + """ + Apply a linear computation to hidden status and a residual computation to input. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + dropout_prob (float): The dropout probability. Default: 0.1. + compute_type (:class:`mindspore.dtype`): Compute type in BertTransformer. Default: mstype.float32. + """ + def __init__(self, + in_channels, + out_channels, + initializer_range=0.02, + dropout_prob=0.1, + compute_type=mstype.float32): + super(BertOutput, self).__init__() + self.dense = nn.Dense(in_channels, out_channels, + weight_init=TruncatedNormal(initializer_range)).to_float(compute_type) + self.dropout = nn.Dropout(1 - dropout_prob) + self.add = P.TensorAdd() + self.layernorm = nn.LayerNorm(out_channels).to_float(compute_type) + self.cast = P.Cast() + + def construct(self, hidden_status, input_tensor): + output = self.dense(hidden_status) + output = self.dropout(output) + output = self.add(output, input_tensor) + output = self.layernorm(output) + return output + + +class RelaPosMatrixGenerator(nn.Cell): + """ + Generates matrix of relative positions between inputs. + + Args: + length (int): Length of one dim for the matrix to be generated. + max_relative_position (int): Max value of relative position. + """ + def __init__(self, length, max_relative_position): + super(RelaPosMatrixGenerator, self).__init__() + self._length = length + self._max_relative_position = Tensor(max_relative_position, dtype=mstype.int32) + self._min_relative_position = Tensor(-max_relative_position, dtype=mstype.int32) + self.range_length = -length + 1 + + self.tile = P.Tile() + self.range_mat = P.Reshape() + self.sub = P.Sub() + self.expanddims = P.ExpandDims() + self.cast = P.Cast() + + def construct(self): + range_vec_row_out = self.cast(F.tuple_to_array(F.make_range(self._length)), mstype.int32) + range_vec_col_out = self.range_mat(range_vec_row_out, (self._length, -1)) + tile_row_out = self.tile(range_vec_row_out, (self._length,)) + tile_col_out = self.tile(range_vec_col_out, (1, self._length)) + range_mat_out = self.range_mat(tile_row_out, (self._length, self._length)) + transpose_out = self.range_mat(tile_col_out, (self._length, self._length)) + distance_mat = self.sub(range_mat_out, transpose_out) + + distance_mat_clipped = C.clip_by_value(distance_mat, + self._min_relative_position, + self._max_relative_position) + + # Shift values to be >=0. Each integer still uniquely identifies a + # relative position difference. + final_mat = distance_mat_clipped + self._max_relative_position + return final_mat + + +class RelaPosEmbeddingsGenerator(nn.Cell): + """ + Generates tensor of size [length, length, depth]. + + Args: + length (int): Length of one dim for the matrix to be generated. + depth (int): Size of each attention head. + max_relative_position (int): Maxmum value of relative position. + initializer_range (float): Initialization value of TruncatedNormal. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + """ + def __init__(self, + length, + depth, + max_relative_position, + initializer_range, + use_one_hot_embeddings=False): + super(RelaPosEmbeddingsGenerator, self).__init__() + self.depth = depth + self.vocab_size = max_relative_position * 2 + 1 + self.use_one_hot_embeddings = use_one_hot_embeddings + + self.embeddings_table = Parameter( + initializer(TruncatedNormal(initializer_range), + [self.vocab_size, self.depth]), + name='embeddings_for_position') + + self.relative_positions_matrix = RelaPosMatrixGenerator(length=length, + max_relative_position=max_relative_position) + self.reshape = P.Reshape() + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.shape = P.Shape() + self.gather = P.GatherV2() # index_select + self.matmul = P.BatchMatMul() + + def construct(self): + relative_positions_matrix_out = self.relative_positions_matrix() + + # Generate embedding for each relative position of dimension depth. + if self.use_one_hot_embeddings: + flat_relative_positions_matrix = self.reshape(relative_positions_matrix_out, (-1,)) + one_hot_relative_positions_matrix = self.one_hot( + flat_relative_positions_matrix, self.vocab_size, self.on_value, self.off_value) + embeddings = self.matmul(one_hot_relative_positions_matrix, self.embeddings_table) + my_shape = self.shape(relative_positions_matrix_out) + (self.depth,) + embeddings = self.reshape(embeddings, my_shape) + else: + embeddings = self.gather(self.embeddings_table, + relative_positions_matrix_out, 0) + return embeddings + + +class SaturateCast(nn.Cell): + """ + Performs a safe saturating cast. This operation applies proper clamping before casting to prevent + the danger that the value will overflow or underflow. + + Args: + src_type (:class:`mindspore.dtype`): The type of the elements of the input tensor. Default: mstype.float32. + dst_type (:class:`mindspore.dtype`): The type of the elements of the output tensor. Default: mstype.float32. + """ + def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): + super(SaturateCast, self).__init__() + np_type = mstype.dtype_to_nptype(dst_type) + min_type = np.finfo(np_type).min + max_type = np.finfo(np_type).max + + self.tensor_min_type = Tensor([min_type], dtype=src_type) + self.tensor_max_type = Tensor([max_type], dtype=src_type) + + self.min_op = P.Minimum() + self.max_op = P.Maximum() + self.cast = P.Cast() + self.dst_type = dst_type + + def construct(self, x): + out = self.max_op(x, self.tensor_min_type) + out = self.min_op(out, self.tensor_max_type) + return self.cast(out, self.dst_type) + + +class BertAttention(nn.Cell): + """ + Apply multi-headed attention from "from_tensor" to "to_tensor". + + Args: + batch_size (int): Batch size of input datasets. + from_tensor_width (int): Size of last dim of from_tensor. + to_tensor_width (int): Size of last dim of to_tensor. + from_seq_length (int): Length of from_tensor sequence. + to_seq_length (int): Length of to_tensor sequence. + num_attention_heads (int): Number of attention heads. Default: 1. + size_per_head (int): Size of each attention head. Default: 512. + query_act (str): Activation function for the query transform. Default: None. + key_act (str): Activation function for the key transform. Default: None. + value_act (str): Activation function for the value transform. Default: None. + has_attention_mask (bool): Specifies whether to use attention mask. Default: False. + attention_probs_dropout_prob (float): The dropout probability for + BertAttention. Default: 0.0. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + do_return_2d_tensor (bool): True for return 2d tensor. False for return 3d + tensor. Default: False. + use_relative_positions (bool): Specifies whether to use relative positions. Default: False. + compute_type (:class:`mindspore.dtype`): Compute type in BertAttention. Default: mstype.float32. + """ + def __init__(self, + batch_size, + from_tensor_width, + to_tensor_width, + from_seq_length, + to_seq_length, + num_attention_heads=1, + size_per_head=512, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=False, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=False, + use_relative_positions=False, + compute_type=mstype.float32): + + super(BertAttention, self).__init__() + self.batch_size = batch_size + self.from_seq_length = from_seq_length + self.to_seq_length = to_seq_length + self.num_attention_heads = num_attention_heads + self.size_per_head = size_per_head + self.has_attention_mask = has_attention_mask + self.use_relative_positions = use_relative_positions + + self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))], dtype=compute_type) + self.reshape = P.Reshape() + self.shape_from_2d = (-1, from_tensor_width) + self.shape_to_2d = (-1, to_tensor_width) + weight = TruncatedNormal(initializer_range) + units = num_attention_heads * size_per_head + self.query_layer = nn.Dense(from_tensor_width, + units, + activation=query_act, + weight_init=weight).to_float(compute_type) + self.key_layer = nn.Dense(to_tensor_width, + units, + activation=key_act, + weight_init=weight).to_float(compute_type) + self.value_layer = nn.Dense(to_tensor_width, + units, + activation=value_act, + weight_init=weight).to_float(compute_type) + + self.shape_from = (batch_size, from_seq_length, num_attention_heads, size_per_head) + self.shape_to = ( + batch_size, to_seq_length, num_attention_heads, size_per_head) + + self.matmul_trans_b = P.BatchMatMul(transpose_b=True) + self.multiply = P.Mul() + self.transpose = P.Transpose() + self.trans_shape = (0, 2, 1, 3) + self.trans_shape_relative = (2, 0, 1, 3) + self.trans_shape_position = (1, 2, 0, 3) + self.multiply_data = Tensor([-10000.0,], dtype=compute_type) + self.batch_num = batch_size * num_attention_heads + self.matmul = P.BatchMatMul() + + self.softmax = nn.Softmax() + self.dropout = nn.Dropout(1 - attention_probs_dropout_prob) + + if self.has_attention_mask: + self.expand_dims = P.ExpandDims() + self.sub = P.Sub() + self.add = P.TensorAdd() + self.cast = P.Cast() + self.get_dtype = P.DType() + if do_return_2d_tensor: + self.shape_return = (batch_size * from_seq_length, num_attention_heads * size_per_head) + else: + self.shape_return = (batch_size, from_seq_length, num_attention_heads * size_per_head) + + self.cast_compute_type = SaturateCast(dst_type=compute_type) + self._generate_relative_positions_embeddings = \ + RelaPosEmbeddingsGenerator(length=to_seq_length, + depth=size_per_head, + max_relative_position=16, + initializer_range=initializer_range, + use_one_hot_embeddings=use_one_hot_embeddings) + + def construct(self, from_tensor, to_tensor, attention_mask): + # reshape 2d/3d input tensors to 2d + from_tensor_2d = self.reshape(from_tensor, self.shape_from_2d) + to_tensor_2d = self.reshape(to_tensor, self.shape_to_2d) + query_out = self.query_layer(from_tensor_2d) + key_out = self.key_layer(to_tensor_2d) + value_out = self.value_layer(to_tensor_2d) + + query_layer = self.reshape(query_out, self.shape_from) + query_layer = self.transpose(query_layer, self.trans_shape) + key_layer = self.reshape(key_out, self.shape_to) + key_layer = self.transpose(key_layer, self.trans_shape) + + attention_scores = self.matmul_trans_b(query_layer, key_layer) + + # use_relative_position, supplementary logic + if self.use_relative_positions: + # 'relations_keys' = [F|T, F|T, H] + relations_keys = self._generate_relative_positions_embeddings() + relations_keys = self.cast_compute_type(relations_keys) + # query_layer_t is [F, B, N, H] + query_layer_t = self.transpose(query_layer, self.trans_shape_relative) + # query_layer_r is [F, B * N, H] + query_layer_r = self.reshape(query_layer_t, + (self.from_seq_length, + self.batch_num, + self.size_per_head)) + # key_position_scores is [F, B * N, F|T] + key_position_scores = self.matmul_trans_b(query_layer_r, + relations_keys) + # key_position_scores_r is [F, B, N, F|T] + key_position_scores_r = self.reshape(key_position_scores, + (self.from_seq_length, + self.batch_size, + self.num_attention_heads, + self.from_seq_length)) + # key_position_scores_r_t is [B, N, F, F|T] + key_position_scores_r_t = self.transpose(key_position_scores_r, + self.trans_shape_position) + attention_scores = attention_scores + key_position_scores_r_t + + attention_scores = self.multiply(attention_scores, self.scores_mul) + + if self.has_attention_mask: + attention_mask = self.expand_dims(attention_mask, 1) + multiply_out = self.sub(self.cast(F.tuple_to_array((1.0,)), self.get_dtype(attention_scores)), + self.cast(attention_mask, self.get_dtype(attention_scores))) + + adder = self.multiply(multiply_out, self.multiply_data) + attention_scores = self.add(adder, attention_scores) + + attention_probs = self.softmax(attention_scores) + attention_probs = self.dropout(attention_probs) + + value_layer = self.reshape(value_out, self.shape_to) + value_layer = self.transpose(value_layer, self.trans_shape) + context_layer = self.matmul(attention_probs, value_layer) + + # use_relative_position, supplementary logic + if self.use_relative_positions: + # 'relations_values' = [F|T, F|T, H] + relations_values = self._generate_relative_positions_embeddings() + relations_values = self.cast_compute_type(relations_values) + # attention_probs_t is [F, B, N, T] + attention_probs_t = self.transpose(attention_probs, self.trans_shape_relative) + # attention_probs_r is [F, B * N, T] + attention_probs_r = self.reshape( + attention_probs_t, + (self.from_seq_length, + self.batch_num, + self.to_seq_length)) + # value_position_scores is [F, B * N, H] + value_position_scores = self.matmul(attention_probs_r, + relations_values) + # value_position_scores_r is [F, B, N, H] + value_position_scores_r = self.reshape(value_position_scores, + (self.from_seq_length, + self.batch_size, + self.num_attention_heads, + self.size_per_head)) + # value_position_scores_r_t is [B, N, F, H] + value_position_scores_r_t = self.transpose(value_position_scores_r, + self.trans_shape_position) + context_layer = context_layer + value_position_scores_r_t + + context_layer = self.transpose(context_layer, self.trans_shape) + context_layer = self.reshape(context_layer, self.shape_return) + + return context_layer + + +class BertSelfAttention(nn.Cell): + """ + Apply self-attention. + + Args: + batch_size (int): Batch size of input dataset. + seq_length (int): Length of input sequence. + hidden_size (int): Size of the bert encoder layers. + num_attention_heads (int): Number of attention heads. Default: 12. + attention_probs_dropout_prob (float): The dropout probability for + BertAttention. Default: 0.1. + use_one_hot_embeddings (bool): Specifies whether to use one_hot encoding form. Default: False. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + hidden_dropout_prob (float): The dropout probability for BertOutput. Default: 0.1. + use_relative_positions (bool): Specifies whether to use relative positions. Default: False. + compute_type (:class:`mindspore.dtype`): Compute type in BertSelfAttention. Default: mstype.float32. + """ + def __init__(self, + batch_size, + seq_length, + hidden_size, + num_attention_heads=12, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.1, + use_relative_positions=False, + compute_type=mstype.float32): + super(BertSelfAttention, self).__init__() + if hidden_size % num_attention_heads != 0: + raise ValueError("The hidden size (%d) is not a multiple of the number " + "of attention heads (%d)" % (hidden_size, num_attention_heads)) + + self.size_per_head = int(hidden_size / num_attention_heads) + + self.attention = BertAttention( + batch_size=batch_size, + from_tensor_width=hidden_size, + to_tensor_width=hidden_size, + from_seq_length=seq_length, + to_seq_length=seq_length, + num_attention_heads=num_attention_heads, + size_per_head=self.size_per_head, + attention_probs_dropout_prob=attention_probs_dropout_prob, + use_one_hot_embeddings=use_one_hot_embeddings, + initializer_range=initializer_range, + use_relative_positions=use_relative_positions, + has_attention_mask=True, + do_return_2d_tensor=True, + compute_type=compute_type) + + self.output = BertOutput(in_channels=hidden_size, + out_channels=hidden_size, + initializer_range=initializer_range, + dropout_prob=hidden_dropout_prob, + compute_type=compute_type) + self.reshape = P.Reshape() + self.shape = (-1, hidden_size) + + def construct(self, input_tensor, attention_mask): + input_tensor = self.reshape(input_tensor, self.shape) + attention_output = self.attention(input_tensor, input_tensor, attention_mask) + output = self.output(attention_output, input_tensor) + return output + + +class BertEncoderCell(nn.Cell): + """ + Encoder cells used in BertTransformer. + + Args: + batch_size (int): Batch size of input dataset. + hidden_size (int): Size of the bert encoder layers. Default: 768. + seq_length (int): Length of input sequence. Default: 512. + num_attention_heads (int): Number of attention heads. Default: 12. + intermediate_size (int): Size of intermediate layer. Default: 3072. + attention_probs_dropout_prob (float): The dropout probability for + BertAttention. Default: 0.02. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + hidden_dropout_prob (float): The dropout probability for BertOutput. Default: 0.1. + use_relative_positions (bool): Specifies whether to use relative positions. Default: False. + hidden_act (str): Activation function. Default: "gelu". + compute_type (:class:`mindspore.dtype`): Compute type in attention. Default: mstype.float32. + """ + def __init__(self, + batch_size, + hidden_size=768, + seq_length=512, + num_attention_heads=12, + intermediate_size=3072, + attention_probs_dropout_prob=0.02, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.1, + use_relative_positions=False, + hidden_act="gelu", + compute_type=mstype.float32): + super(BertEncoderCell, self).__init__() + self.attention = BertSelfAttention( + batch_size=batch_size, + hidden_size=hidden_size, + seq_length=seq_length, + num_attention_heads=num_attention_heads, + attention_probs_dropout_prob=attention_probs_dropout_prob, + use_one_hot_embeddings=use_one_hot_embeddings, + initializer_range=initializer_range, + hidden_dropout_prob=hidden_dropout_prob, + use_relative_positions=use_relative_positions, + compute_type=compute_type) + self.intermediate = nn.Dense(in_channels=hidden_size, + out_channels=intermediate_size, + activation=hidden_act, + weight_init=TruncatedNormal(initializer_range)).to_float(compute_type) + self.output = BertOutput(in_channels=intermediate_size, + out_channels=hidden_size, + initializer_range=initializer_range, + dropout_prob=hidden_dropout_prob, + compute_type=compute_type) + + def construct(self, hidden_states, attention_mask): + # self-attention + attention_output = self.attention(hidden_states, attention_mask) + # feed construct + intermediate_output = self.intermediate(attention_output) + # add and normalize + output = self.output(intermediate_output, attention_output) + return output + + +class BertTransformer(nn.Cell): + """ + Multi-layer bert transformer. + + Args: + batch_size (int): Batch size of input dataset. + hidden_size (int): Size of the encoder layers. + seq_length (int): Length of input sequence. + num_hidden_layers (int): Number of hidden layers in encoder cells. + num_attention_heads (int): Number of attention heads in encoder cells. Default: 12. + intermediate_size (int): Size of intermediate layer in encoder cells. Default: 3072. + attention_probs_dropout_prob (float): The dropout probability for + BertAttention. Default: 0.1. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + initializer_range (float): Initialization value of TruncatedNormal. Default: 0.02. + hidden_dropout_prob (float): The dropout probability for BertOutput. Default: 0.1. + use_relative_positions (bool): Specifies whether to use relative positions. Default: False. + hidden_act (str): Activation function used in the encoder cells. Default: "gelu". + compute_type (:class:`mindspore.dtype`): Compute type in BertTransformer. Default: mstype.float32. + return_all_encoders (bool): Specifies whether to return all encoders. Default: False. + """ + def __init__(self, + batch_size, + hidden_size, + seq_length, + num_hidden_layers, + num_attention_heads=12, + intermediate_size=3072, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.1, + use_relative_positions=False, + hidden_act="gelu", + compute_type=mstype.float32, + return_all_encoders=False): + super(BertTransformer, self).__init__() + self.return_all_encoders = return_all_encoders + + layers = [] + for _ in range(num_hidden_layers): + layer = BertEncoderCell(batch_size=batch_size, + hidden_size=hidden_size, + seq_length=seq_length, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + use_one_hot_embeddings=use_one_hot_embeddings, + initializer_range=initializer_range, + hidden_dropout_prob=hidden_dropout_prob, + use_relative_positions=use_relative_positions, + hidden_act=hidden_act, + compute_type=compute_type) + layers.append(layer) + + self.layers = nn.CellList(layers) + + self.reshape = P.Reshape() + self.shape = (-1, hidden_size) + self.out_shape = (batch_size, seq_length, hidden_size) + + def construct(self, input_tensor, attention_mask): + prev_output = self.reshape(input_tensor, self.shape) + + all_encoder_layers = () + for layer_module in self.layers: + layer_output = layer_module(prev_output, attention_mask) + prev_output = layer_output + + if self.return_all_encoders: + layer_output = self.reshape(layer_output, self.out_shape) + all_encoder_layers = all_encoder_layers + (layer_output,) + + if not self.return_all_encoders: + prev_output = self.reshape(prev_output, self.out_shape) + all_encoder_layers = all_encoder_layers + (prev_output,) + return all_encoder_layers + + +class CreateAttentionMaskFromInputMask(nn.Cell): + """ + Create attention mask according to input mask. + + Args: + config (Class): Configuration for BertModel. + """ + def __init__(self, config): + super(CreateAttentionMaskFromInputMask, self).__init__() + self.input_mask_from_dataset = config.input_mask_from_dataset + self.input_mask = None + + if not self.input_mask_from_dataset: + self.input_mask = initializer( + "ones", [config.batch_size, config.seq_length], mstype.int32) + + self.cast = P.Cast() + self.reshape = P.Reshape() + self.shape = (config.batch_size, 1, config.seq_length) + self.broadcast_ones = initializer( + "ones", [config.batch_size, config.seq_length, 1], mstype.float32) + self.batch_matmul = P.BatchMatMul() + + def construct(self, input_mask): + if not self.input_mask_from_dataset: + input_mask = self.input_mask + + input_mask = self.cast(self.reshape(input_mask, self.shape), mstype.float32) + attention_mask = self.batch_matmul(self.broadcast_ones, input_mask) + return attention_mask + + +class BertModel(nn.Cell): + """ + Bidirectional Encoder Representations from Transformers. + + Args: + config (Class): Configuration for BertModel. + is_training (bool): True for training mode. False for eval mode. + use_one_hot_embeddings (bool): Specifies whether to use one hot encoding form. Default: False. + """ + def __init__(self, + config, + is_training, + use_one_hot_embeddings=False): + super(BertModel, self).__init__() + config = copy.deepcopy(config) + if not is_training: + config.hidden_dropout_prob = 0.0 + config.attention_probs_dropout_prob = 0.0 + + self.input_mask_from_dataset = config.input_mask_from_dataset + self.token_type_ids_from_dataset = config.token_type_ids_from_dataset + self.batch_size = config.batch_size + self.seq_length = config.seq_length + self.hidden_size = config.hidden_size + self.num_hidden_layers = config.num_hidden_layers + self.embedding_size = config.hidden_size + self.token_type_ids = None + + self.last_idx = self.num_hidden_layers - 1 + output_embedding_shape = [self.batch_size, self.seq_length, + self.embedding_size] + + if not self.token_type_ids_from_dataset: + self.token_type_ids = initializer( + "zeros", [self.batch_size, self.seq_length], mstype.int32) + + self.bert_embedding_lookup = EmbeddingLookup( + vocab_size=config.vocab_size, + embedding_size=self.embedding_size, + embedding_shape=output_embedding_shape, + use_one_hot_embeddings=use_one_hot_embeddings, + initializer_range=config.initializer_range) + + self.bert_embedding_postprocessor = EmbeddingPostprocessor( + embedding_size=self.embedding_size, + embedding_shape=output_embedding_shape, + use_token_type=True, + token_type_vocab_size=config.type_vocab_size, + use_one_hot_embeddings=use_one_hot_embeddings, + initializer_range=0.02, + max_position_embeddings=config.max_position_embeddings, + dropout_prob=config.hidden_dropout_prob) + + self.bert_encoder = BertTransformer( + batch_size=self.batch_size, + hidden_size=self.hidden_size, + seq_length=self.seq_length, + num_attention_heads=config.num_attention_heads, + num_hidden_layers=self.num_hidden_layers, + intermediate_size=config.intermediate_size, + attention_probs_dropout_prob=config.attention_probs_dropout_prob, + use_one_hot_embeddings=use_one_hot_embeddings, + initializer_range=config.initializer_range, + hidden_dropout_prob=config.hidden_dropout_prob, + use_relative_positions=config.use_relative_positions, + hidden_act=config.hidden_act, + compute_type=config.compute_type, + return_all_encoders=True) + + self.cast = P.Cast() + self.dtype = config.dtype + self.cast_compute_type = SaturateCast(dst_type=config.compute_type) + self.slice = P.StridedSlice() + + self.squeeze_1 = P.Squeeze(axis=1) + self.dense = nn.Dense(self.hidden_size, self.hidden_size, + activation="tanh", + weight_init=TruncatedNormal(config.initializer_range)).to_float(config.compute_type) + self._create_attention_mask_from_input_mask = CreateAttentionMaskFromInputMask(config) + + def construct(self, input_ids, token_type_ids, input_mask): + + # embedding + if not self.token_type_ids_from_dataset: + token_type_ids = self.token_type_ids + word_embeddings, embedding_tables = self.bert_embedding_lookup(input_ids) + embedding_output = self.bert_embedding_postprocessor(token_type_ids, + word_embeddings) + + # attention mask [batch_size, seq_length, seq_length] + attention_mask = self._create_attention_mask_from_input_mask(input_mask) + + # bert encoder + encoder_output = self.bert_encoder(self.cast_compute_type(embedding_output), + attention_mask) + + sequence_output = self.cast(encoder_output[self.last_idx], self.dtype) + + # pooler + sequence_slice = self.slice(sequence_output, + (0, 0, 0), + (self.batch_size, 1, self.hidden_size), + (1, 1, 1)) + first_token = self.squeeze_1(sequence_slice) + pooled_output = self.dense(first_token) + pooled_output = self.cast(pooled_output, self.dtype) + + return sequence_output, pooled_output, embedding_tables diff --git a/mindspore/model_zoo/__init__.py b/mindspore/model_zoo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mindspore/model_zoo/lenet.py b/mindspore/model_zoo/lenet.py new file mode 100644 index 00000000000..a22eef1a968 --- /dev/null +++ b/mindspore/model_zoo/lenet.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""LeNet.""" +import mindspore.ops.operations as P +import mindspore.nn as nn +from mindspore.common.initializer import TruncatedNormal + + +def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): + """weight initial for conv layer""" + weight = weight_variable() + return nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, padding=padding, + weight_init=weight, has_bias=False, pad_mode="valid") + + +def fc_with_initialize(input_channels, out_channels): + """weight initial for fc layer""" + weight = weight_variable() + bias = weight_variable() + return nn.Dense(input_channels, out_channels, weight, bias) + + +def weight_variable(): + """weight initial""" + return TruncatedNormal(0.02) + + +class LeNet5(nn.Cell): + """ + Lenet network + + Args: + num_class (int): Num classes. Default: 10. + + Returns: + Tensor, output tensor + Examples: + >>> LeNet(num_class=10) + + """ + def __init__(self, num_class=10): + super(LeNet5, self).__init__() + self.num_class = num_class + self.batch_size = 32 + self.conv1 = conv(1, 6, 5) + self.conv2 = conv(6, 16, 5) + self.fc1 = fc_with_initialize(16 * 5 * 5, 120) + self.fc2 = fc_with_initialize(120, 84) + self.fc3 = fc_with_initialize(84, self.num_class) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.reshape = P.Reshape() + + def construct(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.reshape(x, (self.batch_size, -1)) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.relu(x) + x = self.fc3(x) + return x diff --git a/mindspore/model_zoo/resnet.py b/mindspore/model_zoo/resnet.py new file mode 100755 index 00000000000..403f66e4150 --- /dev/null +++ b/mindspore/model_zoo/resnet.py @@ -0,0 +1,262 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor + + +def _weight_variable(shape, factor=0.01): + init_value = np.random.randn(*shape).astype(np.float32) * factor + return Tensor(init_value) + + +def _conv3x3(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 3, 3) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv1x1(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 1, 1) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv7x7(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 7, 7) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=0, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _fc(in_channel, out_channel): + weight_shape = (out_channel, in_channel) + weight = _weight_variable(weight_shape) + return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0) + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = P.MaxPoolWithArgmax(pad_mode='same', window=3, stride=2) + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1, argmax = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) diff --git a/mindspore/model_zoo/vgg.py b/mindspore/model_zoo/vgg.py new file mode 100644 index 00000000000..76f265c326c --- /dev/null +++ b/mindspore/model_zoo/vgg.py @@ -0,0 +1,98 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""VGG.""" +import mindspore.nn as nn +from mindspore.ops import operations as P + + +def _make_layer(base, batch_norm): + """Make stage network of VGG.""" + layers = [] + in_channels = 3 + for v in base: + if v == 'M': + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + conv2d = nn.Conv2d(in_channels=in_channels, + out_channels=v, + kernel_size=3, + padding=1, + pad_mode='pad') + if batch_norm: + layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU()] + else: + layers += [conv2d, nn.ReLU()] + in_channels = v + return nn.SequentialCell(layers) + + +class Vgg(nn.Cell): + """ + VGG network definition. + + Args: + base (list): Configuration for different layers, mainly the channel number of Conv layer. + num_classes (int): Class numbers. Default: 1000. + batch_norm (bool): Whether to do the batchnorm. Default: False. + batch_size (int): Batch size. Default: 1. + + Returns: + Tensor, infer output tensor. + + Examples: + >>> VGG([64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], + >>> num_classes=1000, batch_norm=False, batch_size=1) + """ + def __init__(self, base, num_classes=1000, batch_norm=False, batch_size=1): + super(Vgg, self).__init__() + self.layers = _make_layer(base, batch_norm=batch_norm) + self.avgpool = nn.AvgPool2d(7) + self.reshape = P.Reshape() + self.shp = (batch_size, -1) + self.classifier = nn.SequentialCell([ + nn.Dense(512 * 7 * 7, 4096), + nn.ReLU(), + nn.Dense(4096, 4096), + nn.ReLU(), + nn.Dense(4096, num_classes)]) + + def construct(self, x): + x = self.layers(x) + x = self.avgpool(x) + x = self.reshape(x, self.shp) + x = self.classifier(x) + return x + + +cfg = { + '11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + '13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + '16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], + '19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], +} + + +def vgg16(): + """ + Get VGG16 neural network. + + Returns: + Cell, cell instance of VGG16 neural network. + + Examples: + >>> vgg16() + """ + net = Vgg(cfg['16'], num_classes=1000, batch_norm=False, batch_size=1) + return net diff --git a/mindspore/model_zoo/yolov3.py b/mindspore/model_zoo/yolov3.py new file mode 100644 index 00000000000..5ac3b67086b --- /dev/null +++ b/mindspore/model_zoo/yolov3.py @@ -0,0 +1,677 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""YOLOv3 based on ResNet18.""" + +import mindspore as ms +import mindspore.nn as nn +from mindspore import context, Tensor +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.communication.management import get_group_size +from mindspore.common.initializer import TruncatedNormal +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C + + +def weight_variable(): + """Weight variable.""" + return TruncatedNormal(0.02) + + +class _conv_with_pad(nn.Cell): + """Create Conv2D with padding.""" + def __init__(self, in_channels, out_channels, kernel_size, stride=1): + super(_conv_with_pad, self).__init__() + total_pad = kernel_size - 1 + pad_begin = total_pad // 2 + pad_end = total_pad - pad_begin + self.pad = P.Pad(((0, 0), (0, 0), (pad_begin, pad_end), (pad_begin, pad_end))) + self.conv = nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, padding=0, pad_mode='valid', + weight_init=weight_variable()) + def construct(self, x): + x = self.pad(x) + x = self.conv(x) + return x + + +def _fused_bn(channels, momentum=0.99): + """Get a fused batchnorm.""" + return nn.BatchNorm2d(channels, momentum=momentum) + + +def _conv_bn_relu(in_channel, + out_channel, + ksize, + stride=1, + padding=0, + dilation=1, + alpha=0.1, + momentum=0.99, + pad_mode="same"): + """Get a conv2d batchnorm and relu layer.""" + return nn.SequentialCell( + [nn.Conv2d(in_channel, + out_channel, + kernel_size=ksize, + stride=stride, + padding=padding, + dilation=dilation, + pad_mode=pad_mode), + nn.BatchNorm2d(out_channel, momentum=momentum), + nn.LeakyReLU(alpha)] + ) + + +class BasicBlock(nn.Cell): + """ + ResNet basic block. + + Args: + in_channels (int): Input channel. + out_channels (int): Output channel. + stride (int): Stride size for the initial convolutional layer. Default:1. + momentum (float): Momentum for batchnorm layer. Default:0.1. + + Returns: + Tensor, output tensor. + + Examples: + BasicBlock(3,256,stride=2,down_sample=True). + """ + expansion = 1 + + def __init__(self, + in_channels, + out_channels, + stride=1, + momentum=0.99): + super(BasicBlock, self).__init__() + + self.conv1 = _conv_with_pad(in_channels, out_channels, 3, stride=stride) + self.bn1 = _fused_bn(out_channels, momentum=momentum) + self.conv2 = _conv_with_pad(out_channels, out_channels, 3) + self.bn2 = _fused_bn(out_channels, momentum=momentum) + self.relu = P.ReLU() + self.down_sample_layer = None + self.downsample = (in_channels != out_channels) + if self.downsample: + self.down_sample_layer = _conv_with_pad(in_channels, out_channels, 1, stride=stride) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = self.conv2(x) + x = self.bn2(x) + + if self.downsample: + identity = self.down_sample_layer(identity) + + out = self.add(x, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet network. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of different layers. + in_channels (int): Input channel. + out_channels (int): Output channel. + num_classes (int): Class number. Default:100. + + Returns: + Tensor, output tensor. + + Examples: + ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + 100). + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides=None, + num_classes=80): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of " + "layer_num, inchannel, outchannel list must be 4!") + + self.conv1 = _conv_with_pad(3, 64, 7, stride=2) + self.bn1 = _fused_bn(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same') + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.num_classes = num_classes + if num_classes: + self.reduce_mean = P.ReduceMean(keep_dims=True) + self.end_point = nn.Dense(out_channels[3], num_classes, has_bias=True, + weight_init=weight_variable(), + bias_init=weight_variable()) + self.squeeze = P.Squeeze(axis=(2, 3)) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + """ + Make Layer for ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the initial convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + _make_layer(BasicBlock, 3, 128, 256, 2). + """ + layers = [] + + resblk = block(in_channel, out_channel, stride=stride) + layers.append(resblk) + + for _ in range(1, layer_num - 1): + resblk = block(out_channel, out_channel, stride=1) + layers.append(resblk) + + resblk = block(out_channel, out_channel, stride=1) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = c5 + if self.num_classes: + out = self.reduce_mean(c5, (2, 3)) + out = self.squeeze(out) + out = self.end_point(out) + + return c3, c4, out + + +def resnet18(class_num=10): + """ + Get ResNet18 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet18 neural network. + + Examples: + resnet18(100). + """ + return ResNet(BasicBlock, + [2, 2, 2, 2], + [64, 64, 128, 256], + [64, 128, 256, 512], + [1, 2, 2, 2], + num_classes=class_num) + + +class YoloBlock(nn.Cell): + """ + YoloBlock for YOLOv3. + + Args: + in_channels (int): Input channel. + out_chls (int): Middle channel. + out_channels (int): Output channel. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3). + + Examples: + YoloBlock(1024, 512, 255). + + """ + def __init__(self, in_channels, out_chls, out_channels): + super(YoloBlock, self).__init__() + out_chls_2 = out_chls * 2 + + self.conv0 = _conv_bn_relu(in_channels, out_chls, ksize=1) + self.conv1 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv2 = _conv_bn_relu(out_chls_2, out_chls, ksize=1) + self.conv3 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv4 = _conv_bn_relu(out_chls_2, out_chls, ksize=1) + self.conv5 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv6 = nn.Conv2d(out_chls_2, out_channels, kernel_size=1, stride=1, has_bias=True) + + def construct(self, x): + c1 = self.conv0(x) + c2 = self.conv1(c1) + + c3 = self.conv2(c2) + c4 = self.conv3(c3) + + c5 = self.conv4(c4) + c6 = self.conv5(c5) + + out = self.conv6(c6) + return c5, out + + +class YOLOv3(nn.Cell): + """ + YOLOv3 Network. + + Note: + backbone = resnet18. + + Args: + feature_shape (list): Input image shape, [N,C,H,W]. + backbone_shape (list): resnet18 output channels shape. + backbone (Cell): Backbone Network. + out_channel (int): Output channel. + + Returns: + Tensor, output tensor. + + Examples: + YOLOv3(feature_shape=[1,3,416,416], + backbone_shape=[64, 128, 256, 512, 1024] + backbone=darknet53(), + out_channel=255). + """ + def __init__(self, feature_shape, backbone_shape, backbone, out_channel): + super(YOLOv3, self).__init__() + self.out_channel = out_channel + self.net = backbone + self.backblock0 = YoloBlock(backbone_shape[-1], out_chls=backbone_shape[-2], out_channels=out_channel) + + self.conv1 = _conv_bn_relu(in_channel=backbone_shape[-2], out_channel=backbone_shape[-2]//2, ksize=1) + self.upsample1 = P.ResizeNearestNeighbor((feature_shape[2]//16, feature_shape[3]//16)) + self.backblock1 = YoloBlock(in_channels=backbone_shape[-2]+backbone_shape[-3], + out_chls=backbone_shape[-3], + out_channels=out_channel) + + self.conv2 = _conv_bn_relu(in_channel=backbone_shape[-3], out_channel=backbone_shape[-3]//2, ksize=1) + self.upsample2 = P.ResizeNearestNeighbor((feature_shape[2]//8, feature_shape[3]//8)) + self.backblock2 = YoloBlock(in_channels=backbone_shape[-3]+backbone_shape[-4], + out_chls=backbone_shape[-4], + out_channels=out_channel) + self.concat = P.Concat(axis=1) + + def construct(self, x): + # input_shape of x is (batch_size, 3, h, w) + # feature_map1 is (batch_size, backbone_shape[2], h/8, w/8) + # feature_map2 is (batch_size, backbone_shape[3], h/16, w/16) + # feature_map3 is (batch_size, backbone_shape[4], h/32, w/32) + feature_map1, feature_map2, feature_map3 = self.net(x) + con1, big_object_output = self.backblock0(feature_map3) + + con1 = self.conv1(con1) + ups1 = self.upsample1(con1) + con1 = self.concat((ups1, feature_map2)) + con2, medium_object_output = self.backblock1(con1) + + con2 = self.conv2(con2) + ups2 = self.upsample2(con2) + con3 = self.concat((ups2, feature_map1)) + _, small_object_output = self.backblock2(con3) + + return big_object_output, medium_object_output, small_object_output + + +class DetectionBlock(nn.Cell): + """ + YOLOv3 detection Network. It will finally output the detection result. + + Args: + scale (str): Character, scale. + config (Class): YOLOv3 config. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3). + + Examples: + DetectionBlock(scale='l',stride=32). + """ + + def __init__(self, scale, config): + super(DetectionBlock, self).__init__() + + self.config = config + if scale == 's': + idx = (0, 1, 2) + elif scale == 'm': + idx = (3, 4, 5) + elif scale == 'l': + idx = (6, 7, 8) + else: + raise KeyError("Invalid scale value for DetectionBlock") + self.anchors = Tensor([self.config.anchor_scales[i] for i in idx], ms.float32) + self.num_anchors_per_scale = 3 + self.num_attrib = 4 + 1 + self.config.num_classes + self.ignore_threshold = 0.5 + self.lambda_coord = 1 + + self.sigmoid = nn.Sigmoid() + self.reshape = P.Reshape() + self.tile = P.Tile() + self.concat = P.Concat(axis=-1) + self.input_shape = Tensor(tuple(config.img_shape[::-1]), ms.float32) + + def construct(self, x): + num_batch = P.Shape()(x)[0] + grid_size = P.Shape()(x)[2:4] + + # Reshape and transpose the feature to [n, 3, grid_size[0], grid_size[1], num_attrib] + prediction = P.Reshape()(x, (num_batch, + self.num_anchors_per_scale, + self.num_attrib, + grid_size[0], + grid_size[1])) + prediction = P.Transpose()(prediction, (0, 3, 4, 1, 2)) + + range_x = range(grid_size[1]) + range_y = range(grid_size[0]) + grid_x = P.Cast()(F.tuple_to_array(range_x), ms.float32) + grid_y = P.Cast()(F.tuple_to_array(range_y), ms.float32) + # Tensor of shape [grid_size[0], grid_size[1], 1, 1] representing the coordinate of x/y axis for each grid + grid_x = self.tile(self.reshape(grid_x, (1, 1, -1, 1, 1)), (1, grid_size[0], 1, 1, 1)) + grid_y = self.tile(self.reshape(grid_y, (1, -1, 1, 1, 1)), (1, 1, grid_size[1], 1, 1)) + # Shape is [grid_size[0], grid_size[1], 1, 2] + grid = self.concat((grid_x, grid_y)) + + box_xy = prediction[:, :, :, :, :2] + box_wh = prediction[:, :, :, :, 2:4] + box_confidence = prediction[:, :, :, :, 4:5] + box_probs = prediction[:, :, :, :, 5:] + + box_xy = (self.sigmoid(box_xy) + grid) / P.Cast()(F.tuple_to_array((grid_size[1], grid_size[0])), ms.float32) + box_wh = P.Exp()(box_wh) * self.anchors / self.input_shape + box_confidence = self.sigmoid(box_confidence) + box_probs = self.sigmoid(box_probs) + + if self.training: + return grid, prediction, box_xy, box_wh + return self.concat((box_xy, box_wh, box_confidence, box_probs)) + + +class Iou(nn.Cell): + """Calculate the iou of boxes.""" + def __init__(self): + super(Iou, self).__init__() + self.min = P.Minimum() + self.max = P.Maximum() + + def construct(self, box1, box2): + box1_xy = box1[:, :, :, :, :, :2] + box1_wh = box1[:, :, :, :, :, 2:4] + box1_mins = box1_xy - box1_wh / F.scalar_to_array(2.0) + box1_maxs = box1_xy + box1_wh / F.scalar_to_array(2.0) + + box2_xy = box2[:, :, :, :, :, :2] + box2_wh = box2[:, :, :, :, :, 2:4] + box2_mins = box2_xy - box2_wh / F.scalar_to_array(2.0) + box2_maxs = box2_xy + box2_wh / F.scalar_to_array(2.0) + + intersect_mins = self.max(box1_mins, box2_mins) + intersect_maxs = self.min(box1_maxs, box2_maxs) + intersect_wh = self.max(intersect_maxs - intersect_mins, F.scalar_to_array(0.0)) + + intersect_area = P.Squeeze(-1)(intersect_wh[:, :, :, :, :, 0:1]) * \ + P.Squeeze(-1)(intersect_wh[:, :, :, :, :, 1:2]) + box1_area = P.Squeeze(-1)(box1_wh[:, :, :, :, :, 0:1]) * P.Squeeze(-1)(box1_wh[:, :, :, :, :, 1:2]) + box2_area = P.Squeeze(-1)(box2_wh[:, :, :, :, :, 0:1]) * P.Squeeze(-1)(box2_wh[:, :, :, :, :, 1:2]) + + iou = intersect_area / (box1_area + box2_area - intersect_area) + return iou + + +class YoloLossBlock(nn.Cell): + """ + YOLOv3 Loss block cell. It will finally output loss of the scale. + + Args: + scale (str): Three scale here, 's', 'm' and 'l'. + config (Class): The default config of YOLOv3. + + Returns: + Tensor, loss of the scale. + + Examples: + YoloLossBlock('l', ConfigYOLOV3ResNet18()). + """ + + def __init__(self, scale, config): + super(YoloLossBlock, self).__init__() + self.config = config + if scale == 's': + idx = (0, 1, 2) + elif scale == 'm': + idx = (3, 4, 5) + elif scale == 'l': + idx = (6, 7, 8) + else: + raise KeyError("Invalid scale value for DetectionBlock") + self.anchors = Tensor([self.config.anchor_scales[i] for i in idx], ms.float32) + self.ignore_threshold = Tensor(self.config.ignore_threshold, ms.float32) + self.concat = P.Concat(axis=-1) + self.iou = Iou() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + self.reduce_max = P.ReduceMax(keep_dims=False) + self.input_shape = Tensor(tuple(config.img_shape[::-1]), ms.float32) + + def construct(self, grid, prediction, pred_xy, pred_wh, y_true, gt_box): + + object_mask = y_true[:, :, :, :, 4:5] + class_probs = y_true[:, :, :, :, 5:] + + grid_shape = P.Shape()(prediction)[1:3] + grid_shape = P.Cast()(F.tuple_to_array(grid_shape[::-1]), ms.float32) + + pred_boxes = self.concat((pred_xy, pred_wh)) + true_xy = y_true[:, :, :, :, :2] * grid_shape - grid + true_wh = y_true[:, :, :, :, 2:4] + true_wh = P.Select()(P.Equal()(true_wh, 0.0), + P.Fill()(P.DType()(true_wh), P.Shape()(true_wh), 1.0), + true_wh) + true_wh = P.Log()(true_wh / self.anchors * self.input_shape) + box_loss_scale = 2 - y_true[:, :, :, :, 2:3] * y_true[:, :, :, :, 3:4] + + gt_shape = P.Shape()(gt_box) + gt_box = P.Reshape()(gt_box, (gt_shape[0], 1, 1, 1, gt_shape[1], gt_shape[2])) + + iou = self.iou(P.ExpandDims()(pred_boxes, -2), gt_box) # [batch, grid[0], grid[1], num_anchor, num_gt] + best_iou = self.reduce_max(iou, -1) # [batch, grid[0], grid[1], num_anchor] + ignore_mask = best_iou < self.ignore_threshold + ignore_mask = P.Cast()(ignore_mask, ms.float32) + ignore_mask = P.ExpandDims()(ignore_mask, -1) + ignore_mask = F.stop_gradient(ignore_mask) + + xy_loss = object_mask * box_loss_scale * self.cross_entropy(prediction[:, :, :, :, :2], true_xy) + wh_loss = object_mask * box_loss_scale * 0.5 * P.Square()(true_wh - prediction[:, :, :, :, 2:4]) + confidence_loss = self.cross_entropy(prediction[:, :, :, :, 4:5], object_mask) + confidence_loss = object_mask * confidence_loss + (1 - object_mask) * confidence_loss * ignore_mask + class_loss = object_mask * self.cross_entropy(prediction[:, :, :, :, 5:], class_probs) + + # Get smooth loss + xy_loss = self.reduce_sum(xy_loss, ()) + wh_loss = self.reduce_sum(wh_loss, ()) + confidence_loss = self.reduce_sum(confidence_loss, ()) + class_loss = self.reduce_sum(class_loss, ()) + + loss = xy_loss + wh_loss + confidence_loss + class_loss + return loss / P.Shape()(prediction)[0] + + +class yolov3_resnet18(nn.Cell): + """ + ResNet based YOLOv3 network. + + Args: + config (Class): YOLOv3 config. + + Returns: + Cell, cell instance of ResNet based YOLOv3 neural network. + + Examples: + yolov3_resnet18(80, [1,3,416,416]). + """ + + def __init__(self, config): + super(yolov3_resnet18, self).__init__() + self.config = config + + # YOLOv3 network + self.feature_map = YOLOv3(feature_shape=self.config.feature_shape, + backbone=ResNet(BasicBlock, + self.config.backbone_layers, + self.config.backbone_input_shape, + self.config.backbone_shape, + self.config.backbone_stride, + num_classes=None), + backbone_shape=self.config.backbone_shape, + out_channel=self.config.out_channel) + + # prediction on the default anchor boxes + self.detect_1 = DetectionBlock('l', self.config) + self.detect_2 = DetectionBlock('m', self.config) + self.detect_3 = DetectionBlock('s', self.config) + + def construct(self, x): + big_object_output, medium_object_output, small_object_output = self.feature_map(x) + output_big = self.detect_1(big_object_output) + output_me = self.detect_2(medium_object_output) + output_small = self.detect_3(small_object_output) + + return output_big, output_me, output_small + + +class YoloWithLossCell(nn.Cell): + """" + Provide YOLOv3 training loss through network. + + Args: + network (Cell): The training network. + config (Class): YOLOv3 config. + + Returns: + Tensor, the loss of the network. + """ + def __init__(self, network, config): + super(YoloWithLossCell, self).__init__() + self.yolo_network = network + self.config = config + self.loss_big = YoloLossBlock('l', self.config) + self.loss_me = YoloLossBlock('m', self.config) + self.loss_small = YoloLossBlock('s', self.config) + + def construct(self, x, y_true_0, y_true_1, y_true_2, gt_0, gt_1, gt_2): + yolo_out = self.yolo_network(x) + loss_l = self.loss_big(yolo_out[0][0], yolo_out[0][1], yolo_out[0][2], yolo_out[0][3], y_true_0, gt_0) + loss_m = self.loss_me(yolo_out[1][0], yolo_out[1][1], yolo_out[1][2], yolo_out[1][3], y_true_1, gt_1) + loss_s = self.loss_small(yolo_out[2][0], yolo_out[2][1], yolo_out[2][2], yolo_out[2][3], y_true_2, gt_2) + return loss_l + loss_m + loss_s + + +class TrainingWrapper(nn.Cell): + """ + Encapsulation class of YOLOv3 network training. + + Append an optimizer to the training network after that the construct + function can be called to create the backward graph. + + Args: + network (Cell): The training network. Note that loss function should have been added. + optimizer (Optimizer): Optimizer for updating the weights. + sens (Number): The adjust parameter. Default: 1.0. + """ + def __init__(self, network, optimizer, sens=1.0): + super(TrainingWrapper, self).__init__(auto_prefix=False) + self.network = network + self.weights = ms.ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.grad_reducer = None + self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + if self.parallel_mode in [ms.ParallelMode.DATA_PARALLEL, ms.ParallelMode.HYBRID_PARALLEL]: + self.reducer_flag = True + if self.reducer_flag: + mean = context.get_auto_parallel_context("mirror_mean") + if auto_parallel_context().get_device_num_is_set(): + degree = context.get_auto_parallel_context("device_num") + else: + degree = get_group_size() + self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) + + def construct(self, *args): + weights = self.weights + loss = self.network(*args) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(*args, sens) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + return F.depend(loss, self.optimizer(grads)) diff --git a/mindspore/nn/__init__.py b/mindspore/nn/__init__.py new file mode 100644 index 00000000000..f3f59edcbf5 --- /dev/null +++ b/mindspore/nn/__init__.py @@ -0,0 +1,35 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Neural Networks Cells. + +Pre-defined building blocks or computing units to construct Neural Networks. +""" +from . import layer, loss, optim, metrics, wrap +from .cell import Cell +from .layer import * +from .loss import * +from .optim import * +from .metrics import * +from .wrap import * + +__all__ = ["Cell"] +__all__.extend(layer.__all__) +__all__.extend(loss.__all__) +__all__.extend(optim.__all__) +__all__.extend(metrics.__all__) +__all__.extend(wrap.__all__) + +__all__.sort() diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py new file mode 100755 index 00000000000..088f3f3e57a --- /dev/null +++ b/mindspore/nn/cell.py @@ -0,0 +1,653 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""cell""" +import time +import gc +from collections import OrderedDict +from mindspore import log as logger +from .. import context +from ..common import dtype as mstype +from ..common.api import _executor +from .._checkparam import _check_str_by_regular +from ..common.parameter import Parameter, ParameterTuple +from .._c_expression import init_ge +from ..ops.primitive import Primitive +from ..parallel._tensor import _load_tensor_by_layout +from ..parallel._utils import _get_parallel_mode +from ..common.tensor import Tensor + + +class Cell: + """ + Base class for all neural network. + + A 'Cell' could be a single neural network cell, such as conv2d, relu, batch_norm, etc. or a composition of + cells to constructing a network. + + Note: + In general, the autograd algorithm will automatically generate the implementation of the gradient function, + but if bprop method is implemented, the gradient function + will be replaced by the bprop. The bprop implementation will receive a Tensor `dout` containing the gradient + of the loss w.r.t. the output, and a Tensor `out` containing the forward result. The bprop need to compute the + gradient of the loss w.r.t. the inputs, gradient of the loss w.r.t. Parameter variables is not supported + currently. + + Args: + auto_prefix (bool): Recursively generate namespaces. Default: True. + + Examples: + >>> class MyCell(Cell): + >>> def __init__(self): + >>> super(MyCell, self).__init__() + >>> self.relu = P.ReLU() + >>> + >>> def construct(self, x): + >>> return self.relu(x) + """ + def __init__(self, auto_prefix=True): + self._params = OrderedDict() + self._cells = OrderedDict() + self.training = False + self.pynative = False + self._auto_prefix = auto_prefix + self._scope = None + self._phase = 'train' + self._parameter_layout_dict = {} + self._create_time = int(time.time() * 1e9) + init_ge() + # call gc to release GE session resources used by non-used cell objects + gc.collect() + self._construct_inputs_num = 0 + self._construct_inputs_names = [] + if _get_parallel_mode() in ["auto_parallel", "semi_auto_parallel"]: + self._get_construct_inputs_number_and_name() + self._parallel_inputs_run = None + + @property + def create_time(self): + return self._create_time + + @property + def cell_init_args(self): + return self._cell_init_args + + @cell_init_args.setter + def cell_init_args(self, value): + if not isinstance(value, str): + raise TypeError("'cell_init_args' must be string type.") + self._cell_init_args = value + + @property + def phase(self): + return self._phase + + @phase.setter + def phase(self, value): + if not isinstance(value, str): + raise TypeError("'phase' must be string type.") + self._phase = value + + @property + def parameter_layout_dict(self): + return self._parameter_layout_dict + + @parameter_layout_dict.setter + def parameter_layout_dict(self, value): + if not isinstance(value, dict): + raise TypeError("'parameter_layout_dict' must be dict type.") + self._parameter_layout_dict = value + + def get_func_graph_proto(self): + """Return graph binary proto.""" + return _executor._get_func_graph_proto(self.phase + "." + str(self.create_time), "anf_ir", True) + + def __getattr__(self, name): + if '_params' in self.__dict__: + params = self.__dict__['_params'] + if name in params: + return params[name] + if '_cells' in self.__dict__: + cells = self.__dict__['_cells'] + if name in cells: + return cells[name] + raise AttributeError("'{}' object has no attribute '{}'.".format(type(self).__name__, name)) + + def __del__(self): + if hasattr(self, "_create_time"): + _executor.del_net_res(str(self._create_time)) + + def __delattr__(self, name): + if name in self._params: + del self._params[name] + elif name in self._cells: + del self._cells[name] + else: + object.__delattr__(self, name) + + def __call__(self, *inputs): + if context.get_context("mode") == context.GRAPH_MODE: + out = self.compile_and_run(*inputs) + return out + return self.construct(*inputs) + + def __setattr__(self, name, value): + cells = self.__dict__.get('_cells') + params = self.__dict__.get('_params') + if isinstance(value, Parameter): + if params is None: + raise AttributeError("Can not assign params before Cell.__init__() call.") + if name in self.__dict__: + if self.__dict__[name] is not None: + raise TypeError("Expected type is not in (Parameter, Cell), but got Parameter.") + del self.__dict__[name] + if cells and name in cells: + raise TypeError("Expected type is Cell, but got Parameter.") + self.insert_param_to_cell(name, value) + elif isinstance(value, ParameterTuple): + if params is None: + raise AttributeError("Can not assign params before Cell.__init__() call.") + for item in value: + self.insert_param_to_cell(item.name, item, check_name=False) + object.__setattr__(self, name, value) + elif isinstance(value, Cell): + if cells is None: + raise AttributeError("Can not assign cells before Cell.__init__() call.") + if name in self.__dict__: + del self.__dict__[name] + if params and name in params: + raise TypeError("Expected type is Parameter, but got Cell.") + if self._auto_prefix: + value.update_parameters_name(name + '.') + cells[name] = value + elif params and name in params: + if value is not None: + raise TypeError("Expected type in (Parameter, ParameterTuple), but got {}.".format(type(value))) + self.insert_param_to_cell(name, None) + elif cells and name in cells: + if value is not None: + raise TypeError("Expected type is cell, but got {}.".format(type(value))) + self._cells[name] = None + else: + if isinstance(value, Primitive): + value.set_prim_instance_name(name) + object.__setattr__(self, name, value) + + def extend_repr(self): + """ + Sets the extended representation of the Cell. + + To print customized extended information, re-implement this method in your own cells. + """ + return '' + + def __repr__(self): + extra_str = self.extend_repr() + info_str = self.__class__.__name__ + '<' + if self._cells: + sub_str = '\n' + if extra_str: + sub_str += '{}\n'.format(self.extend_repr()) + for key, value in self._cells.items(): + sub_str += '({}): {}\n'.format(key, repr(value)) + sub_str = sub_str.replace('\n', '\n ') + '>' + info_str += sub_str + else: + info_str += extra_str + '>' + return info_str + + def load_parameter_slice(self, params): + """ + Replace parameters with sliced tensors by parallel strategies. + + Please refer to the usage in source code of `mindspore.common._Executor.compile`. + + Args: + params (dict): The parameters dictionary used for init data graph. + """ + + if params is None: + for key in self.parameters_dict(): + tensor = self.parameters_dict()[key].data + if key not in self.parameter_layout_dict: + logger.info("layout dict does not contain the key %s", key) + continue + layout = self.parameter_layout_dict[key] + new_tensor = _load_tensor_by_layout(tensor, layout) + self.parameters_dict()[key].set_parameter_data(new_tensor) + elif isinstance(params, OrderedDict): + for key in params: + tensor = params[key].data + if key not in self.parameter_layout_dict: + logger.info("layout dict does not contain the key %s", key) + continue + layout = self.parameter_layout_dict[key] + new_tensor = _load_tensor_by_layout(tensor, layout) + params[key].set_parameter_data(new_tensor) + else: + raise TypeError('Parameters need OrderedDict type, but got {}'. + format(type(params))) + + def _load_inputs(self, *inputs): + """ + Slice inputs tensors by parallel strategies. + + Args: + inputs (Function or Cell): inputs of construct method. + """ + + parallel_inputs_run = [] + if len(inputs) > self._construct_inputs_num: + raise ValueError('Len of inputs: {} is bigger than self._construct_inputs_num: {}.'. + format(len(inputs), self._construct_inputs_num)) + for i, tensor in enumerate(inputs): + key = self._construct_inputs_names[i] + # if input is not used, self.parameter_layout_dict may not contain the key + if key not in self.parameter_layout_dict: + logger.warning("layout dict does not contain the key %s", key) + parallel_inputs_run.append(tensor) + else: + layout = self.parameter_layout_dict[key] + new_tensor = _load_tensor_by_layout(tensor, layout) + parallel_inputs_run.append(new_tensor) + return tuple(parallel_inputs_run) + + def _get_construct_inputs_number_and_name(self): + """Compute self._construct_inputs_names and self._construct_inputs_num""" + import inspect + from mindspore._extends.parse.parser import get_parse_method_of_class + + fn = get_parse_method_of_class(self) + inspect.getfullargspec(fn) + self._construct_inputs_num = fn.__code__.co_argcount + self._construct_inputs_names = fn.__code__.co_varnames + + assert self._construct_inputs_num > 0 + assert self._construct_inputs_names[0] == 'self' + assert self._construct_inputs_num - 1 <= len(self._construct_inputs_names) + self._construct_inputs_names = self._construct_inputs_names[1:self._construct_inputs_num] + self._construct_inputs_num = self._construct_inputs_num - 1 + + def compile_and_run(self, *inputs): + """ + Compiles and runs cell. + + Args: + inputs (tuple): Input parameters. + + Returns: + Object, the result of executing. + """ + _, compile_flag = _executor.compile(self, *inputs, phase=self.phase) + + if _get_parallel_mode() in ["auto_parallel", "semi_auto_parallel"]: + if inputs and isinstance(inputs[0], Tensor) and inputs[0].virtual_flag and (not compile_flag): + parallel_inputs_run = self._parallel_inputs_run + else: + self._parallel_inputs_run = self._load_inputs(*inputs) + parallel_inputs_run = self._parallel_inputs_run + return _executor(self, *parallel_inputs_run, phase=self.phase) + return _executor(self, *inputs, phase=self.phase) + + def exec_checkpoint_graph(self): + """Executes saving checkpoint graph operation.""" + _executor(self, phase='save') + + def insert_param_to_cell(self, param_name, param, check_name=True): + """ + Adds a parameter to the current cell. + + Inserts a parameter with given name to the cell. Please refer to the usage in + source code of `mindspore.nn.Cell.__setattr__`. + + Args: + param_name (str): Name of the parameter. + param (Parameter): Parameter to be inserted to the cell. + check_name (bool): Determines whether the name input is compatible. Default: True. + + Raises: + KeyError: If the name of parameter is null or contains dot. + AttributeError: If user did not call init() first. + TypeError: If the type of parameter is not Parameter. + """ + if not param_name: + raise KeyError("The name of parameter should not be null.") + if check_name and '.' in param_name: + raise KeyError("The name of parameter should not contain \".\"") + if '_params' not in self.__dict__: + raise AttributeError("You need call init() first.") + if hasattr(self, param_name) and param_name not in self._params: + raise KeyError("Duplicated parameter name '{}'.".format(param_name)) + if not isinstance(param, Parameter) and param is not None: + raise TypeError("The type of parameter should be 'Parameter' if not None.") + self._params[param_name] = param + + def insert_child_to_cell(self, child_name, child): + """ + Adds a child cell to the current cell. + + Inserts a subcell with given name to current cell. + + Args: + child_name (str): Name of the child cell. + child (Cell): The child cell to be inserted. + + Raises: + KeyError: Child Cell's name is incorrect or duplicated with the other child name. + TypeError: Child Cell's type is incorrect. + """ + if not child_name or '.' in child_name: + raise KeyError("Child cell name is incorrect.") + if hasattr(self, child_name) and child_name not in self._cells: + raise KeyError("Duplicate child name '{}'.".format(child_name)) + if not isinstance(child, Cell) and child is not None: + raise TypeError("Child cell type is incorrect.") + self._cells[child_name] = child + + def construct(self, *inputs): + """ + Defines the computation to be performed. + + This method should be overridden by all subclasses. + + Note: + The inputs of the top cell only allow Tensor. + Other types (tuple, list, int etc.) are forbidden. + + Returns: + Tensor, returns the computed result. + """ + raise NotImplementedError + + def parameters_dict(self, recurse=True): + """ + Gets parameters dictionary. + + Gets the parameters dictionary of this cell. + + Args: + recurse (bool): Whether contains the parameters of subcells. Default: True. + + Returns: + OrderedDict, return parameters dictionary. + """ + param_dict = OrderedDict() + for param in self.get_parameters(expand=recurse): + param_dict[param.name] = param + return param_dict + + def parameters_broadcast_dict(self, recurse=True): + param_dict = OrderedDict() + for param in self.get_parameters(expand=recurse): + if param.layerwise_parallel is False: + param_dict[param.name] = param + if not param_dict: + return None + return param_dict + + def update_parameters_name(self, prefix='', recurse=True): + """ + Updates the names of parameters with given prefix string. + + Adds the given prefix to the names of parameters. + + Args: + prefix (str): The prefix string. + recurse (bool): Whether contains the parameters of subcells. Default: True. + """ + + _check_str_by_regular(prefix) + for name, param in self.parameters_and_names(expand=recurse): + if prefix != '': + param.is_init = False + param.name = prefix + name + + def trainable_params(self, recurse=True): + """ + Returns all trainable parameters. + + Returns a list of all trainable parmeters. + + Args: + recurse (bool): Whether contains the trainable parameters of subcells. Default: True. + + Returns: + List, the list of trainable parameters. + """ + return list(filter(lambda x: x.requires_grad, self.get_parameters(expand=recurse))) + + def untrainable_params(self, recurse=True): + """ + Returns all untrainable parameters. + + Returns a list of all untrainable parmeters. + + Args: + recurse (bool): Whether contains the untrainable parameters of subcells. Default: True. + + Returns: + List, the list of untrainable parameters. + """ + return list(filter(lambda x: not x.requires_grad, self.get_parameters(expand=recurse))) + + def get_parameters(self, expand=True): + """ + Returns an iterator over cell parameters. + + Yields parameters of this cell. If `expand` is True, yield parameters of this cell and all subcells. + + Args: + expand (bool): If True, yields parameters of this cell and all subcells. Otherwise, yields only parameters + that are direct members of this cell. Default: True. + + Examples: + >>> net = Net() + >>> for item in net.get_parameters(): + >>> print(item) + """ + for _, param in self.parameters_and_names(expand=expand): + yield param + + def check_names(self): + names = set("") + for value, param in self.parameters_and_names(): + if param.name in names: + raise ValueError("The value of {} is {}, its name '{}' already exists.". + format(value, param, param.name)) + names.add(param.name) + + def parameters_and_names(self, name_prefix='', expand=True): + """ + Returns an iterator over cell parameters. + + Includes the parameter's name and itself. + + Args: + name_prefix (str): Namespace. Default: ''. + expand (bool): If True, yields parameters of this cell and all subcells. Otherwise, yields only parameters + that are direct members of this cell. Default: True. + + Examples: + >>> n = Net() + >>> names = [] + >>> for m in n.parameters_and_names(): + >>> if m[0]: + >>> names.append(m[0]) + """ + cells = [] + if expand: + cells = self.cells_and_names(name_prefix=name_prefix) + else: + cells.append((name_prefix, self)) + + params_set = set() + for cell_name, cell in cells: + params = cell._params.items() + for par_name, par in params: + if par and par not in params_set: + params_set.add(par) + + par_new_name = par_name + if cell_name: + par_new_name = cell_name + '.' + par_new_name + + yield par_new_name, par + + def cells_and_names(self, cells=None, name_prefix=''): + """ + Returns an iterator over all cells in the network. + + Includes the cell's name and itself. + + Args: + cells (str): Cells to iterate over. Default: None. + name_prefix (str): Namespace. Default: ''. + + Examples: + >>> n = Net() + >>> names = [] + >>> for m in n.cells_and_names(): + >>> if m[0]: + >>> names.append(m[0]) + """ + t_cells = cells if cells else set() + if self in t_cells: + return + + t_cells.add(self) + yield name_prefix, self + + for name, cell in self._cells.items(): + if cell: + cells_name_prefix = name + if name_prefix: + cells_name_prefix = name_prefix + '.' + cells_name_prefix + for ele in cell.cells_and_names(t_cells, cells_name_prefix): + yield ele + + def cells(self): + """Returns an iterator over immediate cells.""" + return self.name_cells().values() + + def _set_scope(self, name): + """Sets the name on the first time.""" + if self._scope is None: + self._scope = name + + def _children_scope_recursive(self, parent_prefix='Default'): + """Generates the scope of each layer of the network recursively.""" + reserve_class_name_in_scope = context.get_context("reserve_class_name_in_scope") + + for name, cell in self.name_cells().items(): + yield parent_prefix + "/" + name + (("-" + cell.__class__.__name__) + if reserve_class_name_in_scope else ""), cell + + for name, cell in self.name_cells().items(): + for key, value in cell._children_scope_recursive(parent_prefix + "/" + name + + (("-" + cell.__class__.__name__) + if reserve_class_name_in_scope else "")): + yield key, value + + def get_scope(self): + """Returns the scope of a cell object in one network.""" + return self._scope + + def generate_scope(self): + """Generate the scope for every cell object in the network.""" + for name, cell in self._children_scope_recursive(): + cell._set_scope(name) + + def name_cells(self): + """ + Returns an iterator over all cells in the network. + + Include name of the cell and cell itself. + """ + value_set = set() + cells = OrderedDict() + for name, cell in self._cells.items(): + if cell is not None and cell not in value_set: + value_set.add(cell) + cells[name] = cell + return cells + + def add_flags(self, **flags): + for x in flags: + if not isinstance(flags[x], bool): + raise TypeError(f"Flags (f{x}) must be bool but {type(flags[x])}.") + if not hasattr(self, "_mindspore_flags"): + self._mindspore_flags = {} + self._mindspore_flags.update({**flags}) + self.__dict__.update({**flags}) + return self + + def add_flags_recursive(self, **flags): + self.add_flags(**flags) + if hasattr(self, '_cell_init_args'): + self._cell_init_args += str({**flags}) + for cell in self.cells(): + cell.add_flags_recursive(**flags) + return self + + def to_float(self, dst_type): + """ + Add cast on all inputs of cell and child cells to run with certain float type. + + If `dst_type is mindspore.dtype.float16`, all the inputs of Cell including input, Parameter, Tensor + as const will be cast to float16. Please refer to the usage in source code of + `mindspore.train.amp.build_train_network`. + + Note: + Call multiple times will overwrite the previous. + + Args: + dst_type (:class:`mindspore.dtype`): Transfer Cell to Run with dst_type. + dst_type can be `mindspore.dtype.float16` or `mindspore.dtype.float32`. + + Raises: + ValueError: If dst_type is not float32 or float16. + """ + if dst_type not in (mstype.float16, mstype.float32): + raise ValueError("dst_type should inside float32 or float16.") + flags = {'fp16': dst_type == mstype.float16, 'fp32': dst_type == mstype.float32} + self.add_flags_recursive(**flags) + return self + + def set_train(self, mode=True): + """ + Sets the cell to training mode. + + The cell itself and all children cells will be set to training mode. + + Args: + mode (bool): Specifies whether the model is training. Default: True. + """ + if mode is False: + self._phase = 'predict' + else: + self._phase = 'train' + self.add_flags_recursive(training=mode) + return self + + def set_broadcast_flag(self, mode=True): + """ + Set the cell to data_parallel mode. + + The cell can be accessed as an attribute using the given name. + + Args: + mode (bool): Specifies whether the model is data_parallel. Default: True. + """ + self.add_flags_recursive(broadcast_flag=mode) + return self diff --git a/mindspore/nn/layer/__init__.py b/mindspore/nn/layer/__init__.py new file mode 100644 index 00000000000..bb29935602e --- /dev/null +++ b/mindspore/nn/layer/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Layer. + +The high-level components(Cells) used to construct the neural network. +""" +from .activation import Softmax, LogSoftmax, ReLU, ReLU6, Tanh, GELU, ELU, Sigmoid, PReLU, get_activation, LeakyReLU +from .normalization import BatchNorm1d, BatchNorm2d, LayerNorm +from .container import SequentialCell, CellList +from .conv import Conv2d, Conv2dTranspose +from .lstm import LSTM +from .basic import Dropout, Flatten, Dense, ClipByNorm, Norm, OneHot +from .embedding import Embedding +from .pooling import AvgPool2d, MaxPool2d + +__all__ = ['Softmax', 'LogSoftmax', 'ReLU', 'ReLU6', 'Tanh', 'GELU', 'Sigmoid', 'PReLU', 'get_activation', 'LeakyReLU', + 'BatchNorm1d', 'BatchNorm2d', 'LayerNorm', 'ELU', + 'SequentialCell', 'CellList', + 'Conv2d', 'Conv2dTranspose', + 'LSTM', + 'Dropout', 'Flatten', 'Dense', 'ClipByNorm', 'Norm', 'OneHot', + 'Embedding', + 'AvgPool2d', 'MaxPool2d', + ] diff --git a/mindspore/nn/layer/activation.py b/mindspore/nn/layer/activation.py new file mode 100644 index 00000000000..00f2afe7034 --- /dev/null +++ b/mindspore/nn/layer/activation.py @@ -0,0 +1,365 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""activation""" +import numpy as np +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from mindspore.common.tensor import Tensor +from mindspore._extends import cell_attr_register +from ..cell import Cell + + +class Softmax(Cell): + r""" + Softmax activation function. + + Applies the Softmax function to an n-dimensional input Tensor. + + The input is a Tensor of logits transformed with exponential function and then + normalized to lie in range [0, 1] and sum up to 1. + + Softmax is defined as: + + .. math:: + \text{softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_{j=0}^{n-1}\exp(x_j)}, + + where :math:`x_{i}` is the :math:`i`-th slice along the given dim of the input Tensor. + + Args: + axis (Union[int, tuple[int]]): The axis to apply Softmax operation. Default: -1, means the last dimension. + + Inputs: + - **x** (Tensor) - The input of Softmax. + + Outputs: + Tensor, which has the same type and shape as `x` with values in the range[0,1]. + + """ + def __init__(self, axis=-1): + super(Softmax, self).__init__() + self.softmax = P.Softmax(axis) + + def construct(self, x): + return self.softmax(x) + + +class LogSoftmax(Cell): + r""" + LogSoftmax activation function. + + Applies the LogSoftmax function to n-dimensional input tensor. + + The input is transformed with Softmax function and then with log function to lie in range[-inf,0). + + Logsoftmax is defined as: + :math:`\text{logsoftmax}(x_i) = \log \left(\frac{\exp(x_i)}{\sum_{j=0}^{n-1} \exp(x_j)}\right)`, + where :math:`x_{i}` is the :math:`i`-th slice along the given dim of the input Tensor. + + Args: + axis (int): The axis to apply LogSoftmax operation. Default: -1, means the last dimension. + + Inputs: + - **x** (Tensor) - The input of LogSoftmax. + + Outputs: + Tensor, which has the same type and shape as the input as `x` with values in the range[-inf,0). + + """ + + def __init__(self, axis=-1): + super(LogSoftmax, self).__init__() + self.log_softmax = P.LogSoftmax(axis) + + def construct(self, x): + return self.log_softmax(x) + + +class ELU(Cell): + r""" + Exponential Linear Uint activation function. + + Applies the exponential linear unit function element-wise. + The activation function defined as: + + .. math:: + E_{i} = + \begin{cases} + x, &\text{if } x \geq 0; \cr + \text{alpha} * (\exp(x_i) - 1), &\text{otherwise.} + \end{cases} + + Args: + alpha (float): The coefficient of negative factor whose type is float. Default: 1.0. + + Inputs: + - **input_data** (Tensor) - The input of ELU. + + Outputs: + Tensor, with the same type and shape as the `input_data`. + + """ + def __init__(self, alpha=1.0): + super(ELU, self).__init__() + self.elu = P.Elu(alpha) + + def construct(self, x): + return self.elu(x) + + +class ReLU(Cell): + r""" + Rectified Linear Unit activation function. + + Applies the rectified linear unit function element-wise. It returns + element-wise :math:`\max(0, x)`, specially, the neurons with the negative output + will suppressed and the active neurons will stay the same. + + Inputs: + - **input_data** (Tensor) - The input of ReLU. + + Outputs: + Tensor, with the same type and shape as the `input_data`. + + """ + def __init__(self): + super(ReLU, self).__init__() + self.relu = P.ReLU() + + def construct(self, x): + return self.relu(x) + + +class ReLU6(Cell): + r""" + Compute ReLU6 activation function. + + ReLU6 is similar to ReLU with a upper limit of 6, which if the inputs are greater than 6, the outputs + will be suppressed to 6. + It computes element-wise as :math:`\min(\max(0, x), 6)`. The input is a Tensor of any valid shape. + + Inputs: + - **input_data** (Tensor) - The input of ReLU6. + + Outputs: + Tensor, which has the same type with `input_data`. + + """ + def __init__(self): + super(ReLU6, self).__init__() + self.relu6 = P.ReLU6() + + def construct(self, x): + return self.relu6(x) + + +class LeakyReLU(Cell): + r""" + Leaky ReLU activation function. + + LeakyReLU is similar to ReLU, but LeakyReLU has a slope that makes it not equal to 0 at x < 0. + The activation function is defined as: + + .. math:: + \text{leaky_relu}(x) = \begin{cases}x, &\text{if } x \geq 0; \cr + \text{alpha} * x, &\text{otherwise.}\end{cases} + + See https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf + + Args: + alpha (float): Slope of the activation function at x < 0. Default: 0.2. + + Inputs: + - **input_x** (Tensor) - The input of LeakyReLU. + + Outputs: + Tensor, has the same type and shape with the `input_x`. + + """ + def __init__(self, alpha=0.2): + super(LeakyReLU, self).__init__() + self.greater_equal = P.GreaterEqual() + self.mul = P.Mul() + self.alpha = alpha + + def construct(self, x): + alpha = P.Cast()(F.scalar_to_array(self.alpha), P.DType()(x)) + if self.alpha <= 1: + out = P.Maximum()(alpha * x, x) + else: + out = P.Minimum()(alpha * x, x) + return out + + +class Tanh(Cell): + r""" + Tanh activation function. + + Applies the Tanh function element-wise, returns a new tensor with the hyperbolic tangent of the elements of input, + The input is a Tensor with any valid shape. + + Tanh function is defined as: + + .. math:: + tanh(x_i) = \frac{\exp(x_i) - \exp(-x_i)}{\exp(x_i) + \exp(-x_i)} = \frac{\exp(2x_i) - 1}{\exp(2x_i) + 1}, + + where :math:`x_i` is an element of the input Tensor. + + Inputs: + - **input_data** (Tensor) - The input of Tanh. + + Outputs: + Tensor, with the same type and shape as the `input_data`. + + """ + def __init__(self): + super(Tanh, self).__init__() + self.tanh = P.Tanh() + + def construct(self, x): + return self.tanh(x) + + +class GELU(Cell): + """ + Gaussian error linear unit activation function. + + Applies GELU function to each element of the input. The input is a Tensor with any valid shape. + + GELU is defined as: + :math:`GELU(x_i) = x_i*P(X < x_i)`, where :math:`P` is the cumulative distribution function + of standard Gaussian distribution and :math:`x_i` is the element of the input. + + Inputs: + - **input_data** (Tensor) - The input of Tanh. + + Outputs: + Tensor, with the same type and shape as the `input_data`. + + """ + def __init__(self): + super(GELU, self).__init__() + self.gelu = P.Gelu() + + def construct(self, x): + return self.gelu(x) + + +class Sigmoid(Cell): + r""" + Sigmoid activation function. + + Applies sigmoid-type activation element-wise. + + Sigmoid function is defined as: + :math:`\text{sigmoid}(x_i) = \frac{1}{1 + \exp(-x_i)}`, where :math:`x_i` is the element of the input. + + Inputs: + - **input_data** (Tensor) - The input of Tanh. + + Outputs: + Tensor, with the same type and shape as the `input_data`. + + """ + def __init__(self): + super(Sigmoid, self).__init__() + self.sigmoid = P.Sigmoid() + + def construct(self, x): + return self.sigmoid(x) + + +class PReLU(Cell): + r""" + PReLU activation function. + + Applies the PReLU function element-wise. + + PReLU is defined as: :math:`prelu(x_i)= \max(0, x_i) + w * \min(0, x_i)`, where :math:`x_i` + is an element of an channel of the input. + + Here :math:`w` is an learnable parameter with default initial value 0.25. + Parameter :math:`w` has dimensionality of the argument channel. If called without argument + channel, a single parameter :math:`w` will be shared across all channels. + + Args: + channel (int): The dimension of input. Default: 1. + w (float): The initial value of w. Default: 0.25. + + Inputs: + - **input_data** (Tensor) - The input of Tanh. + + Outputs: + Tensor, with the same type and shape as the `input_data`. + + """ + @cell_attr_register(attrs="") + def __init__(self, channel=1, w=0.25): + super(PReLU, self).__init__() + if isinstance(w, (np.float32, float)): + tmp = np.empty((channel,), dtype=np.float32) + tmp.fill(w) + w = Tensor(tmp) + elif isinstance(w, list): + w = Tensor(w) + + if not isinstance(w, Tensor): + raise TypeError("w only support np.float32, float or Tensor type.") + + self.w = Parameter(initializer(w, [channel]), name='a') + self.prelu = P.PReLU() + self.relu = P.ReLU() + self.assign = P.Assign() + + def construct(self, x): + u = self.relu(self.w) + v = self.prelu(x, u) + if self.training: + self.assign(self.w, u) + return v + + +_activation = { + 'softmax': Softmax, + 'logsoftmax': LogSoftmax, + 'relu': ReLU, + 'tanh': Tanh, + 'gelu': GELU, + 'sigmoid': Sigmoid, + 'prelu': PReLU, + 'leakyrelu': LeakyReLU +} + + +def get_activation(name): + """ + Gets the activation function. + + Args: + name (str): The name of the activation function. + + Returns: + Function, the activation function. + + Examples: + >>> sigmoid = nn.get_activation('sigmoid') + """ + if not name: + return None + + if name not in _activation: + raise KeyError("Unknown activation type") + return _activation[name]() diff --git a/mindspore/nn/layer/basic.py b/mindspore/nn/layer/basic.py new file mode 100644 index 00000000000..e5ed147b857 --- /dev/null +++ b/mindspore/nn/layer/basic.py @@ -0,0 +1,377 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""basic""" +import numpy as np +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.common.initializer import initializer +from mindspore._checkparam import check_int_positive, check_bool +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops.functional import identity +from mindspore.common.parameter import Parameter +from mindspore._extends import cell_attr_register +from ..cell import Cell +from .activation import get_activation +from ..._checkparam import ParamValidator as validator + + +class Dropout(Cell): + r""" + Dropout layer for the input. + + Randomly set some elements of the input tensor to zero with probability :math:`1 - keep_prob` during training + using samples from a Bernoulli distribution. + + Note: + Each channel will be zeroed out independently on every construct call. + + The outputs are scaled by a factor of :math:`\frac{1}{keep_prob}` during training so + that the output layer remains at a similar scale. During inference, this + layer returns the same tensor as the input. + + This technique is proposed in paper `Dropout: A Simple Way to Prevent Neural Networks from Overfitting + `_ and proved to be effective to reduce + over-fitting and prevents neurons from co-adaptation. See more details in `Improving neural networks by + preventing co-adaptation of feature detectors + `_. + + Args: + keep_prob (float): The keep rate, greater than 0 and less equal than 1. E.g. rate=0.9, + dropping out 10% of input units. Default: 0.5. + seed0 (int): The first random seed. Default: 0. + seed1 (int): The second random seed. Default: 0. + dtype (:class:`mindspore.dtype`): Data type of input. Default: mindspore.float32. + + Raises: + ValueError: If keep_prob is not in range (0, 1). + + Inputs: + - **input** (Tensor) - An N-D Tensor. + + Outputs: + Tensor, output tensor with the same shape as the input. + + Examples: + >>> x = mindspore.Tensor(np.ones([20, 16, 50]), mindspore.float32) + >>> net = nn.Dropout(keep_prob=0.8) + >>> net(x) + """ + def __init__(self, keep_prob=0.5, seed0=0, seed1=0, dtype=mstype.float32): + super(Dropout, self).__init__() + if keep_prob <= 0 or keep_prob > 1: + raise ValueError("dropout probability should be a number in range (0, 1], but got {}".format(keep_prob)) + validator.check_subclass("dtype", dtype, mstype.number_type) + self.keep_prob = Tensor(keep_prob) + self.seed0 = seed0 + self.seed1 = seed1 + self.dtype = dtype + self.get_shape = P.Shape() + self.dropout_gen_mask = P.DropoutGenMask(Seed0=seed0, Seed1=seed1) + self.dropout_do_mask = P.DropoutDoMask() + self.cast = P.Cast() + + def construct(self, x): + shape = self.get_shape(x) + dtype = P.DType()(x) + keep_prob = self.cast(self.keep_prob, dtype) + output = self.dropout_gen_mask(shape, keep_prob) + return self.dropout_do_mask(x, output, keep_prob) + + def extend_repr(self): + str_info = 'keep_prob={}, Seed0={}, Seed1={}, dtype={}' \ + .format(self.keep_prob, self.seed0, self.seed1, self.dtype) + return str_info + + +class Flatten(Cell): + r""" + Flatten layer for the input. + + Flattens a tensor without changing dimension of batch size on the 0-th axis. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, \ldots)` to be flattened. + + Outputs: + Tensor, the shape of the output tensor is :math:`(N, X)`, where :math:`X` is + the product of the remaining dimensions. + + Examples: + >>> net = nn.Flatten() + >>> input = mindspore.Tensor(np.array([[[1.2, 1.2], [2.1, 2.1]], [[2.2, 2.2], [3.2, 3.2]]]), mindspore.float32) + >>> input.shape() + (2, 2, 2) + >>> net(input) + [[1.2 1.2 2.1 2.1] + [2.2 2.2 3.2 3.2]] + """ + def __init__(self): + super(Flatten, self).__init__() + + def construct(self, x): + return F.reshape(x, (F.shape(x)[0], -1)) + + +class Dense(Cell): + r""" + The fully connected layer. + + Applies dense-connected layer for the input. This layer implements the operation as: + + .. math:: + \text{outputs} = \text{activation}(\text{inputs} * \text{kernel} + \text{bias}), + + where :math:`\text{activation}` is the activation function passed as the activation + argument (if passed in), :math:`\text{activation}` is a weight matrix with the same + data type as the inputs created by the layer, and :math:`\text{bias}` is a bias vector + with the same data type as the inputs created by the layer (only if has_bias is True). + + Args: + in_channels (int): The number of channels in the input space. + out_channels (int): The number of channels in the output space. + weight_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable weight_init parameter. The dtype + is same as input x. The values of str refer to the function `initializer`. Default: 'normal'. + bias_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable bias_init parameter. The dtype is + same as input x. The values of str refer to the function `initializer`. Default: 'zeros'. + has_bias (bool): Specifies whether the layer uses a bias vector. Default: True. + activation (str): Regularizer function applied to the output of the layer, eg. 'relu'. Default: None. + + Returns: + Tensor, output tensor. + + Raises: + ValueError: If weight_init or bias_init shape is incorrect. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, in_channels)`. + + Outputs: + Tensor of shape :math:`(N, out_channels)`. + + Examples: + >>> net = nn.Dense(3, 4) + >>> input = mindspore.Tensor(np.random.randint(0, 255, [2, 3]), mindspore.float32) + >>> net(input) + [[ 2.5246444 2.2738023 0.5711005 -3.9399147 ] + [ 1.0739875 4.0155234 0.94188046 -5.459526 ]] + """ + @cell_attr_register(attrs=['has_bias', 'activation']) + def __init__(self, + in_channels, + out_channels, + weight_init='normal', + bias_init='zeros', + has_bias=True, + activation=None): + super(Dense, self).__init__() + self.in_channels = check_int_positive(in_channels) + self.out_channels = check_int_positive(out_channels) + self.has_bias = check_bool(has_bias) + + if isinstance(weight_init, Tensor): + if weight_init.dim() != 2 or weight_init.shape()[0] != out_channels or \ + weight_init.shape()[1] != in_channels: + raise ValueError("weight_init shape error") + + self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") + + if self.has_bias: + if isinstance(bias_init, Tensor): + if bias_init.dim() != 1 or bias_init.shape()[0] != out_channels: + raise ValueError("bias_init shape error") + + self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") + + self.matmul = P.MatMul(transpose_b=True) + self.bias_add = P.BiasAdd() + + self.activation = get_activation(activation) + self.activation_flag = self.activation is not None + + def construct(self, x): + output = self.matmul(x, self.weight) + if self.has_bias: + output = self.bias_add(output, self.bias) + if self.activation_flag: + return self.activation(output) + return output + + def extend_repr(self): + str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ + .format(self.in_channels, self.out_channels, self.weight, self.has_bias) + if self.has_bias: + str_info = str_info + ', bias={}'.format(self.bias) + + if self.activation_flag: + str_info = str_info + ', activation={}'.format(self.activation) + + return str_info + + +class ClipByNorm(Cell): + r""" + Clips tensor values to a maximum :math:`L_2`-norm. + + The output of this layer remains the same if the :math:`L_2`-norm of the input tensor + is not greater than the argument clip_norm. Otherwise the tensor will be normalized as: + + .. math:: + \text{output}(X) = \frac{\text{clip_norm} * X}{L_2(X)}, + + where :math:`L_2(X)` is the :math:`L_2`-norm of :math:`X`. + + Inputs: + - **input** (Tensor) - Tensor of shape N-D. + - **clip_norm** (Tensor) - A scalar Tensor of shape :math:`()` or :math:`(1)` and of + the same type as the input Tensor. + + Outputs: + Tensor, clipped tensor with the same shape as the input. + + Examples: + >>> net = nn.ClipByNorm() + >>> input = mindspore.Tensor(np.random.randint(0, 10, [4, 16]), mindspore.float32) + >>> clip_norm = mindspore.Tensor(np.array([100]).astype(np.float32)) + >>> net(input, clip_norm) + + """ + def __init__(self): + super(ClipByNorm, self).__init__() + self.reduce_sum = P.ReduceSum(keep_dims=True) + self.select_ = P.Select() + self.greater_ = P.Greater() + self.axis = () + self.cast = P.Cast() + self.zero = Tensor(np.array([0.0]).astype(np.float32)) + self.sqrt = P.Sqrt() + self.max_op = P.Maximum() + self.shape = P.Shape() + self.reshape = P.Reshape() + self.fill = P.Fill() + self.expand_dims = P.ExpandDims() + self.dtype = P.DType() + + def construct(self, x, clip_norm): + mul_x = F.square(x) + l2sum = self.cast(self.reduce_sum(mul_x, self.axis), mstype.float32) + cond = self.greater_(l2sum, self.zero) + ones_ = self.fill(self.dtype(cond), self.shape(cond), 1.0) + + l2sum_safe = self.select_(cond, l2sum, self.cast(ones_, self.dtype(l2sum))) + l2norm = self.select_(cond, self.sqrt(l2sum_safe), l2sum) + + intermediate = x * clip_norm + max_norm = self.max_op(l2norm, clip_norm) + values_clip = self.cast(intermediate, mstype.float32) / self.expand_dims(max_norm, -1) + values_clip = self.reshape(values_clip, self.shape(x)) + values_clip = identity(values_clip) + return values_clip + + +class Norm(Cell): + """ + Computes the norm of vectors, currently including Euclidean norm, i.e., :math:`L_2`-norm. + + Args: + axis (tuple): The axis over which to compute vector norms. Default: (). + keep_dims (bool): If True, the axis indicated in `axis` are kept with size 1. Otherwise, + the dimensions in `axis` are removed from the output shape. Default: False. + + Returns: + Tensor, a Tensor of the same type as input, containing the vector or matrix norms. + + Inputs: + - **input** (Tensor) - Tensor which is not empty. + + Outputs: + Tensor, output tensor with dimensions in 'axis' reduced to 1 will be returned if 'keep_dims' is True; + otherwise a Tensor with dimensions in 'axis' removed is returned. + + Examples: + >>> net = nn.Norm(axis=0) + >>> input = mindspore.Tensor(np.random.randint(0, 10, [4, 16]), mindspore.float32) + >>> net(input) + """ + def __init__(self, axis=(), keep_dims=False): + super(Norm, self).__init__() + self.axis = axis + self.keep_dims = keep_dims + self.reduce_sum = P.ReduceSum(True) + self.sqrt = P.Sqrt() + self.squeeze = P.Squeeze(self.axis) + + def construct(self, x): + x = self.sqrt(self.reduce_sum(F.square(x), self.axis)) + + if not self.keep_dims: + x = self.squeeze(x) + return x + + def extend_repr(self): + str_info = 'axis={}, keep_dims={}'.format(self.axis, self.keep_dims) + return str_info + + +class OneHot(Cell): + """ + Returns a one-hot tensor. + + The locations represented by indices in argument 'indices' take value on_value, + while all other locations take value off_value. + + Note: + If the input indices is rank :math:`N`, the output will have rank :math:`N+1`. The new + axis is created at dimension `axis`. + + Args: + axis (int): Features x depth if axis == -1, depth x features + if axis == 0. Default: -1. + depth (int): A scalar defining the depth of the one hot dimension. Default: 1. + on_value (float): A scalar defining the value to fill in output[i][j] + when indices[j] = i. Default: 1.0. + off_value (float): A scalar defining the value to fill in output[i][j] + when indices[j] != i. Default: 0.0. + dtype (:class:`mindspore.dtype`): Default: mindspore.float32. + + Inputs: + - **indices** (Tensor) - A tensor of indices of data type mindspore.int32 and arbitrary shape. + + Outputs: + Tensor, the one-hot tensor of data type 'dtype' with dimension at 'axis' expanded to 'depth' and filled with + on_value and off_value. + + Examples: + >>> net = nn.OneHot(depth=4, axis=1) + >>> indices = mindspore.Tensor([[1, 3], [0, 2]], dtype=mindspore.int32) + >>> net(indices) + [[[0. 0.] + [1. 0.] + [0. 0.] + [0. 1.]] + [[1. 0.] + [0. 0.] + [0. 1.] + [0. 0.]]] + """ + def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, dtype=mstype.float32): + super(OneHot, self).__init__() + self.onehot = P.OneHot(axis) + self.depth = depth + self.on_value = Tensor(on_value, dtype) + self.off_value = Tensor(off_value, dtype) + + def construct(self, indices): + return self.onehot(indices, self.depth, self.on_value, self.off_value) diff --git a/mindspore/nn/layer/container.py b/mindspore/nn/layer/container.py new file mode 100644 index 00000000000..76c72ce421d --- /dev/null +++ b/mindspore/nn/layer/container.py @@ -0,0 +1,247 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""container""" +from collections import OrderedDict +from abc import abstractmethod, ABCMeta + +from ..cell import Cell + + +def _valid_index(cell_num, index): + if not isinstance(index, int): + raise TypeError("Index {} is not int type") + if not -cell_num <= index < cell_num: + raise IndexError("Index should be a number in range [{}, {}), but got {}" + .format(-cell_num, cell_num, index)) + return index % cell_num + + +def _valid_cell(cell): + if issubclass(cell.__class__, Cell): + return True + raise TypeError('Cell {} is not subclass of Cell'.format(cell)) + + +class _CellListBase(metaclass=ABCMeta): + """ + An interface for base the cell as list. + + The sequential cell may be iterated using the construct method using for-in statement. + But there are some scenarios that the construct method built-in does not fit. + For convenience, we provide an interface that indicates the sequential + cell may be interpretated as list of cells, so it can be accessed using + iterator or subscript when a sequential cell instantiate is accessed + by iterator or subscript , it will be interpretated as a list of cells. + """ + def __init__(self): + super(_CellListBase, self).__init__() + self.__cell_as_list__ = True + + @abstractmethod + def __len__(self): + pass + + @abstractmethod + def __getitem__(self, index): + pass + + def construct(self): + raise NotImplementedError + + +class SequentialCell(Cell): + """ + Sequential cell container. + + A list of Cells will be added to it in the order they are passed in the constructor. + Alternatively, an ordered dict of cells can also be passed in. + + Args: + args (list, optional): List of subclass of Cell. + + Raises: + TypeError: If arg is not of type list or OrderedDict. + + Inputs: + - **input** (Tensor) - Tensor with shape according to the first Cell in the sequence. + + Outputs: + Tensor, the output Tensor with shape depending on the input and defined sequence of Cells. + + Examples: + >>> conv = nn.Conv2d(3, 2, 3, pad_mode='valid') + >>> bn = nn.BatchNorm2d(2) + >>> relu = nn.ReLU() + >>> seq = nn.SequentialCell([conv, bn, relu]) + >>> + >>> x = mindspore.Tensor(np.random.random((1, 3, 4, 4)), dtype=mindspore.float32) + >>> seq(x) + [[[[0.02531557 0. ] + [0.04933941 0.04880078]] + [[0. 0. ] + [0. 0. ]]]] + """ + def __init__(self, *args): + super(SequentialCell, self).__init__() + if len(args) == 1: + cells = args[0] + if isinstance(cells, list): + for index, cell in enumerate(cells): + self.insert_child_to_cell(str(index), cell) + elif isinstance(cells, OrderedDict): + for name, cell in cells.items(): + self.insert_child_to_cell(name, cell) + else: + raise TypeError('Cells must be list or orderedDict') + self.cell_list = list(self._cells.values()) + + def __getitem__(self, index): + if isinstance(index, slice): + return self.__class__( + OrderedDict(list(self._cells.items())[index])) + index = _valid_index(len(self), index) + return list(self._cells.values())[index] + + def __setitem__(self, index, cell): + if _valid_cell(cell): + index = _valid_index(len(self), index) + key = list(self._cells.keys())[index] + self._cells[key] = cell + self.cell_list = list(self._cells.values()) + + def __delitem__(self, index): + if isinstance(index, int): + index = _valid_index(len(self), index) + key = list(self._cells.keys())[index] + del self._cells[key] + elif isinstance(index, slice): + keys = list(self._cells.keys())[index] + for key in keys: + del self._cells[key] + else: + raise TypeError('Index {} is not int type or slice type'.format(index)) + self.cell_list = list(self._cells.values()) + + def __len__(self): + return len(self._cells) + + def construct(self, input_data): + """Processes the input with the defined sequence of Cells.""" + for cell in self.cell_list: + input_data = cell(input_data) + return input_data + + +class CellList(_CellListBase, Cell): + """ + Holds Cells in a list. + + CellList can be indexed like a regular Python list, but cells it + contains are properly registered, and will be visible by all Cell methods. + + Args: + args (list, optional): List of subclass of Cell. + + Examples: + >>> conv = nn.Conv2d(100, 20, 3) + >>> bn = nn.BatchNorm2d(20) + >>> relu = nn.ReLU() + >>> cell_ls = nn.CellList([bn]) + >>> cell_ls.insert(0, conv) + >>> cell_ls.append(relu) + >>> x = mindspore.Tensor(np.random.random((1, 3, 4, 4)), dtype=mindspore.float32) + >>> # not same as nn.SequentialCell, `cell_ls(x)` is not correct + >>> cell_ls + CellList< (0): Conv2d + (1): BatchNorm2d + (2): ReLU<> > + """ + def __init__(self, *args): + super(CellList, self).__init__() + if len(args) == 1: + self.extend(args[0]) + + def __getitem__(self, index): + if isinstance(index, slice): + return self.__class__(list(self._cells.values())[index]) + if isinstance(index, int): + index = _valid_index(len(self), index) + return self._cells[str(index)] + raise TypeError('Index {} is not int type or slice type'.format(index)) + + def __setitem__(self, index, cell): + if not isinstance(index, int) and _valid_cell(cell): + raise TypeError('Index {} is not int type'.format(index)) + index = _valid_index(len(self), index) + self._cells[str(index)] = cell + + def __delitem__(self, index): + if isinstance(index, int): + index = _valid_index(len(self), index) + del self._cells[str(index)] + elif isinstance(index, slice): + keys = list(self._cells.keys())[index] + for key in keys: + del self._cells[key] + else: + raise TypeError('Index {} is not int type or slice type'.format(index)) + # adjust orderedDict + temp_dict = OrderedDict() + for idx, cell in enumerate(self._cells.values()): + temp_dict[str(idx)] = cell + self._cells = temp_dict + + def __len__(self): + return len(self._cells) + + def __iter__(self): + return iter(self._cells.values()) + + def __iadd__(self, cells): + self.extend(cells) + return self + + def insert(self, index, cell): + """Inserts a given cell before a given index in the list.""" + idx = _valid_index(len(self), index) + _valid_cell(cell) + length = len(self) + while length > idx: + self._cells[str(length)] = self._cells[str(length - 1)] + length -= 1 + self._cells[str(idx)] = cell + + def extend(self, cells): + """ + Appends cells from a Python iterable to the end of the list. + + Raises: + TypeError: If the cells is not a list of subcells. + """ + if not isinstance(cells, list): + raise TypeError('Cells {} should be list of subcells'.format(cells)) + for cell in cells: + if _valid_cell(cell): + self._cells[str(len(self))] = cell + return self + + def append(self, cell): + """Appends a given cell to the end of the list.""" + if _valid_cell(cell): + self._cells[str(len(self))] = cell + return self + + def construct(self, *inputs): + raise NotImplementedError diff --git a/mindspore/nn/layer/conv.py b/mindspore/nn/layer/conv.py new file mode 100644 index 00000000000..6c78e1a715d --- /dev/null +++ b/mindspore/nn/layer/conv.py @@ -0,0 +1,379 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""conv""" +from mindspore import log as logger +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from mindspore._checkparam import check_bool, twice, check_int_positive, check_int_non_negative, check_int +from mindspore._extends import cell_attr_register +from ..cell import Cell + + +class _Conv(Cell): + """ + Applies a N-D convolution over an input signal composed of several input planes. + """ + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init): + super(_Conv, self).__init__() + self.in_channels = check_int_positive(in_channels) + self.out_channels = check_int_positive(out_channels) + self.kernel_size = kernel_size + self.stride = check_int_positive(stride) + self.pad_mode = pad_mode + self.padding = check_int_non_negative(padding) + self.dilation = check_int(dilation) + self.group = check_int_positive(group) + self.has_bias = has_bias + if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ + (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ + kernel_size[0] < 1 or kernel_size[1] < 1: + raise ValueError("Attr 'kernel_size' of 'Conv2D' Op passed " + + str(self.kernel_size) + ", should be a int or tuple and equal to or greater than 1.") + if in_channels % group != 0: + raise ValueError("Attr 'in_channels' of 'Conv2D' Op must be divisible by " + "attr 'group' of 'Conv2D' Op.") + if out_channels % group != 0: + raise ValueError("Attr 'out_channels' of 'Conv2D' Op must be divisible by " + "attr 'group' of 'Conv2D' Op.") + + self.weight = Parameter(initializer(weight_init, [out_channels, in_channels // group, *kernel_size]), + name='weight') + + if check_bool(has_bias): + self.bias = Parameter(initializer(bias_init, [out_channels]), name='bias') + else: + if bias_init != 'zeros': + logger.warning("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") + self.bias = None + + def construct(self, *inputs): + """Must be overridden by all subclasses.""" + raise NotImplementedError + + +class Conv2d(_Conv): + r""" + 2D convolution layer. + + Applies a 2D convolution over an input tensor which is typically of shape :math:`(N, C_{in}, H_{in}, W_{in})`, + where :math:`N` is batch size and :math:`C_{in}` is channel number. For each batch of shape + :math:`(C_{in}, H_{in}, W_{in})`, the formula is defined as: + + .. math:: + + out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{ij}, X_i) + b_j, + + where :math:`ccor` is cross correlation operator, :math:`C_{in}` is the input channel number, :math:`j` ranges + from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to :math:`i`-th channel of the :math:`j`-th + filter and :math:`out_{j}` corresponds to the :math:`j`-th channel of the output. :math:`W_{ij}` is a slice + of kernel and it has shape :math:`(\text{ks_h}, \text{ks_w})`, where :math:`\text{ks_h}` and + :math:`\text{ks_w}` are height and width of the convolution kernel. The full kernel has shape + :math:`(C_{out}, C_{in} // \text{group}, \text{ks_h}, \text{ks_w})`, where group is the group number + to split the input in the channel dimension. + + If the 'pad_mode' is set to be "valid", the output height and width will be + :math:`\left \lfloor{1 + \frac{H_{in} + 2 \times \text{padding} - \text{ks_h} - + (\text{ks_h} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` and + :math:`\left \lfloor{1 + \frac{W_{in} + 2 \times \text{padding} - \text{ks_w} - + (\text{ks_w} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` respectively. + + The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition + `_. + + Args: + in_channels (int): The number of input channel :math:`C_{in}`. + out_channels (int): The number of output channel :math:`C_{out}`. + kernel_size (Union[int, tuple]): The data type is int or tuple with 2 integers. Specifies the height + and width of the 2D convolution window. Single int means the value if for both height and width of + the kernel. A tuple of 2 ints means the first value is for the height and the other is for the + width of the kernel. + stride (int): Specifies stride for all spatial dimensions with the same value. Value of stride should be + greater or equal to 1 but bounded by the height and width of the input. Default: 1. + pad_mode (str): Specifies padding mode. The optional values are + "same", "valid", "pad". Default: "same". + + - same: Adopts the way of completion. Output height and width will be the same as the input. + Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. Otherwise, the + last extra padding will be done from the bottom and the right side. If this mode is set, `padding` + must be 0. + + - valid: Adopts the way of discarding. The possibly largest height and width of output will be return + without padding. Extra pixels will be discarded. If this mode is set, `padding` + must be 0. + + - pad: Implicit paddings on both sides of the input. The number of `padding` will be padded to the input + Tensor borders. `padding` should be greater than or equal to 0. + + padding (int): Implicit paddings on both sides of the input. Default: 0. + dilation (int): Specifying the dilation rate to use for dilated convolution. If set to be :math:`k > 1`, + there will be :math:`k - 1` pixels skipped for each sampling location. Its value should be greater + or equal to 1 and bounded by the height and width of the input. Default: 1. + group (int): Split filter into groups, `in_ channels` and `out_channels` should be + divisible by the number of groups. Default: 1. + has_bias (bool): Specifies whether the layer uses a bias vector. Default: False. + weight_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the convolution kernel. + It can be a Tensor, a string, an Initializer or a numbers.Number. When a string is specified, + values from 'TruncatedNormal', 'Normal', 'Uniform', 'HeUniform' and 'XavierUniform' distributions as well + as constant 'One' and 'Zero' distributions are possible. Alias 'xavier_uniform', 'he_uniform', 'ones' + and 'zeros' are acceptable. Uppercase and lowercase are both acceptable. Refer to the values of + Initializer for more details. Default: 'normal'. + bias_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the bias vector. Possible + Initializer and string are the same as 'weight_init'. Refer to the values of + Initializer for more details. Default: 'zeros'. + + Returns: + Tensor, output tensor. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> net = nn.Conv2d(120, 240, 4, has_bias=False, weight_init='normal') + >>> input = mindspore.Tensor(np.ones([1, 120, 1024, 640]), mindspore.float32) + >>> net(input).shape() + (1, 240, 1024, 637) + """ + @cell_attr_register + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + kernel_size = twice(kernel_size) + super(Conv2d, self).__init__( + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + self.conv2d = P.Conv2D(out_channel=self.out_channels, + kernel_size=self.kernel_size, + mode=1, + pad_mode=self.pad_mode, + pad=self.padding, + stride=self.stride, + dilation=self.dilation, + group=self.group) + self.bias_add = P.BiasAdd() + if pad_mode not in ('valid', 'same', 'pad'): + raise ValueError('Attr \'pad_mode\' of \'Conv2d\' Op passed ' + + str(pad_mode) + ', should be one of values in \'valid\', \'same\', \'pad\'.') + + def construct(self, x): + output = self.conv2d(x, self.weight) + if self.has_bias: + output = self.bias_add(output, self.bias) + return output + + def extend_repr(self): + s = 'input_channels={}, output_channels={}, kernel_size={},' \ + 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ + 'group={}, has_bias={},' \ + 'weight_init={}, bias_init={}'.format( + self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.has_bias, + self.weight, + self.bias) + + if self.has_bias: + s += ', bias={}'.format(self.bias) + return s + + +class Conv2dTranspose(_Conv): + r""" + 2D transposed convolution layer. + + Compute a 2D transposed convolution, which is also know as a deconvolution + (although it is not actual deconvolution). + + Input is typically of shape :math:`(N, C, H, W)`, where :math:`N` is batch size and :math:`C` is channel number. + + Args: + in_channels (int): The number of channels in the input space. + out_channels (int): The number of channels in the output space. + kernel_size (Union[int, tuple]): int or tuple with 2 integers, which specifies the height + and width of the 2D convolution window.Single int means the value if for both height and width of + the kernel. A tuple of 2 ints means the first value is for the height and the other is for the + width of the kernel. + stride (int): Specifies the same value for all spatial dimensions. Default: 1. + pad_mode (str): Select the mode of the pad. The optional values are + "pad", "same", "valid". Default: "same". + + - pad: Implicit paddings on both sides of the input. + + - same: Adopted the way of completion. + + - valid: Adopted the way of discarding. + padding (int): Implicit paddings on both sides of the input. Default: 0. + dilation (int): Specifies the dilation rate to use for dilated + convolution. Default: 1. + group (int): Split filter into groups, `in_channels` and `out_channels` should be + divisible by the number of groups. Default: 1. + has_bias (bool): Specifies whether the layer uses a bias vector. Default: False. + weight_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the convolution kernel. + It can be a Tensor, a string, an Initializer or a numbers.Number. When a string is specified, + values from 'TruncatedNormal', 'Normal', 'Uniform', 'HeUniform' and 'XavierUniform' distributions as well + as constant 'One' and 'Zero' distributions are possible. Alias 'xavier_uniform', 'he_uniform', 'ones' + and 'zeros' are acceptable. Uppercase and lowercase are both acceptable. Refer to the values of + Initializer for more details. Default: 'normal'. + bias_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the bias vector. Possible + Initializer and string are the same as 'weight_init'. Refer to the values of + Initializer for more details. Default: 'zeros'. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> net = nn.Conv2dTranspose(3, 64, 4, has_bias=False, weight_init='normal') + >>> input = Tensor(np.ones([1, 3, 16, 50]), mstype.float32) + >>> net(input) + """ + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + kernel_size = twice(kernel_size) + # out_channels and in_channels swap. + # cause Conv2DBackpropInput's out_channel refers to Conv2D's out_channel, + # then Conv2dTranspose's out_channel refers to Conv2DBackpropInput's in_channel. + super(Conv2dTranspose, self).__init__( + out_channels, + in_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + + self.out_channels = out_channels + self.in_channels = in_channels + self.shape = P.Shape() + if pad_mode not in ('valid', 'same', 'pad'): + raise ValueError('Attr \'pad_mode\' of \'Conv2dTranspose\' Op passed ' + + str(pad_mode) + ', should be one of values in \'valid\', \'same\', \'pad\'.') + self.is_valid = self.pad_mode == 'valid' + self.is_same = self.pad_mode == 'same' + self.is_pad = self.pad_mode == 'pad' + if check_bool(has_bias): + self.bias = Parameter(initializer(bias_init, [out_channels]), name='bias') + + # cause Conv2DBackpropInput's out_channel refers to Conv2D's out_channel. + self.conv2d_transpose = P.Conv2DBackpropInput(out_channel=in_channels, + kernel_size=kernel_size, + mode=1, + pad_mode=pad_mode, + pad=padding, + stride=stride, + dilation=dilation, + group=group) + self.bias_add = P.BiasAdd() + + def set_strategy(self, strategy): + self.conv2d_transpose.set_strategy(strategy) + return self + + def _deconv_output_length(self, input_length, filter_size): + """Calculate the width and height of output.""" + length = 0 + if self.is_valid: + if filter_size - self.stride > 0: + length = input_length * self.stride + filter_size - self.stride + else: + length = input_length * self.stride + elif self.is_same: + length = input_length * self.stride + elif self.is_pad: + length = input_length * self.stride - 2 * self.padding + filter_size + \ + (filter_size - 1) * (self.dilation - 1) - self.stride + + return length + + def construct(self, x): + n, _, h, w = self.shape(x) + h_out = self._deconv_output_length(h, self.kernel_size[0]) + w_out = self._deconv_output_length(w, self.kernel_size[1]) + if self.has_bias: + return self.bias_add(self.conv2d_transpose(x, self.weight, (n, self.out_channels, h_out, w_out)), + self.bias) + return self.conv2d_transpose(x, self.weight, (n, self.out_channels, h_out, w_out)) + + def extend_repr(self): + s = 'input_channels={}, output_channels={}, kernel_size={},' \ + 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ + 'group={}, has_bias={},' \ + 'weight_init={}, bias_init={}'.format(self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.has_bias, + self.weight, + self.bias) + return s diff --git a/mindspore/nn/layer/embedding.py b/mindspore/nn/layer/embedding.py new file mode 100755 index 00000000000..9579f352269 --- /dev/null +++ b/mindspore/nn/layer/embedding.py @@ -0,0 +1,103 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""embedding""" +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from ..cell import Cell +from ..._checkparam import ParamValidator as validator + + +class Embedding(Cell): + r""" + A simple lookup table that stores embeddings of a fixed dictionary and size. + + This module is often used to store word embeddings and retrieve them using + indices. The input to the module is a list of indices, and the output is + the corresponding word embeddings. + + Note: + When 'use_one_hot' is set to True, the input should be of type mindspore.int32. + + Args: + vocab_size (int): Size of the dictionary of embeddings. + embedding_size (int): The size of each embedding vector. + use_one_hot (bool): Specifies whether to apply one_hot encoding form. Default: False. + embedding_table (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the embedding_table. + Refer to class `initializer` for the values of string when a string + is specified. Default: 'normal'. + dtype (:class:`mindspore.dtype`): Data type of input. Default: mindspore.float32. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(\text{vocab_size})`. + + Outputs: + Tensor of shape :math:`(\text{vocab_size}, \text{embedding_size})`. + + Examples: + >>> net = nn.Embedding(20000, 768, True) + >>> input_data = mindspore.Tensor(np.ones([8, 128]), mindspore.int32) + >>> + >>> # Maps the input word IDs to word embedding. + >>> output = net(input_data) + >>> output.shape() + (8, 128, 768) + """ + def __init__(self, vocab_size, embedding_size, use_one_hot=False, embedding_table='normal', dtype=mstype.float32): + super(Embedding, self).__init__() + validator.check_subclass("dtype", dtype, mstype.number_type) + self.vocab_size = vocab_size + self.embedding_size = embedding_size + self.use_one_hot = use_one_hot + self.embedding_table = Parameter(initializer(embedding_table, [vocab_size, embedding_size]), + name='embedding_table') + self.dtype = dtype + self.expand = P.ExpandDims() + self.reshape_flat = P.Reshape() + self.shp_flat = (-1,) + self.gather = P.GatherV2() + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, self.dtype) + self.off_value = Tensor(0.0, self.dtype) + self.array_mul = P.MatMul() + self.reshape = P.Reshape() + self.get_shp = P.Shape() + + def construct(self, ids): + extended_ids = self.expand(ids, -1) + out_shape = self.get_shp(ids) + (self.embedding_size,) + flat_ids = self.reshape_flat(extended_ids, self.shp_flat) + + if self.use_one_hot: + one_hot_ids = self.one_hot(flat_ids, self.vocab_size, self.on_value, self.off_value) + output_for_reshape = self.array_mul(one_hot_ids, self.embedding_table) + else: + output_for_reshape = self.gather(self.embedding_table, flat_ids, 0) + + output = self.reshape(output_for_reshape, out_shape) + return output + + def extend_repr(self): + s = 'vocab_size={}, embedding_size={},' \ + 'use_one_hot={}, ' \ + 'embedding_table={}, dtype={}'.format( + self.vocab_size, + self.embedding_size, + self.use_one_hot, + self.embedding_table, + self.dtype) + return s diff --git a/mindspore/nn/layer/lstm.py b/mindspore/nn/layer/lstm.py new file mode 100755 index 00000000000..317f754f678 --- /dev/null +++ b/mindspore/nn/layer/lstm.py @@ -0,0 +1,155 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""lstm""" +from mindspore.ops import operations as P +from mindspore.nn.cell import Cell +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from mindspore._checkparam import ParamValidator as validator + + +class LSTM(Cell): + r""" + LSTM (Long Short-Term Memory) layer. + + Applies a LSTM to the input. + + There are two pipelines connecting two consecutive cells in a LSTM model; one is cell state pipeline + and another is hidden state pipeline. Denote two consecutive time nodes as :math:`t-1` and :math:`t`. + Given an input :math:`x_t` at time :math:`t`, an hidden state :math:`h_{t-1}` and an cell + state :math:`c_{t-1}` of the layer at time :math:`{t-1}`, the cell state and hidden state at + time :math:`t` is computed using an gating mechanism. Input gate :math:`i_t` is designed to protect the cell + from perturbation by irrelevant inputs. Forget gate :math:`f_t` affords protection of the cell by forgetting + some information in the past, which is stored in :math:`h_{t-1}`. Output gate :math:`o_t` protects other + units from perturbation by currently irrelevant memory contents. Candidate cell state :math:`\tilde{c}_t` is + calculated with the current input, on which the input gate will be applied. Finally, current cell state + :math:`c_{t}` and hidden state :math:`h_{t}` are computed with the calculated gates and cell states. The complete + formulation is as follows. + + .. math:: + \begin{array}{ll} \\ + i_t = \sigma(W_{ix} x_t + b_{ix} + W_{ih} h_{(t-1)} + b_{ih}) \\ + f_t = \sigma(W_{fx} x_t + b_{fx} + W_{fh} h_{(t-1)} + b_{fh}) \\ + \tilde{c}_t = \tanh(W_{cx} x_t + b_{cx} + W_{ch} h_{(t-1)} + b_{ch}) \\ + o_t = \sigma(W_{ox} x_t + b_{ox} + W_{oh} h_{(t-1)} + b_{oh}) \\ + c_t = f_t * c_{(t-1)} + i_t * \tilde{c}_t \\ + h_t = o_t * \tanh(c_t) \\ + \end{array} + + Here :math:`\sigma` is the sigmoid function, and :math:`*` is the Hadamard product. :math:`W, b` + are learnable weights between the output and the input in the formula. For instance, + :math:`W_{ix}, b_{ix}` are the weight and bias used to transform from input :math:`x` to :math:`i`. + Details can be found in paper `LONG SHORT-TERM MEMORY + `_ and + `Long Short-Term Memory Recurrent Neural Network Architectures for Large Scale Acoustic Modeling + `_. + + Args: + input_size (int): Number of features of input. + hidden_size (int): Number of features of hidden layer. + num_layers (int): Number of layers of stacked LSTM . Default: 1. + has_bias (bool): Specifies whether has bias `b_ih` and `b_hh`. Default: True. + batch_first (bool): Specifies whether the first dimension of input is batch_size. Default: False. + dropout (float): If not 0, append `Dropout` layer on the outputs of each + LSTM layer except the last layer. Default 0. The range of dropout is [0.0, 1.0]. + bidirectional (bool): Specifies whether this is a bidirectional LSTM. If set True, + number of directions will be 2 otherwise number of directions is 1. Default: False. + + Inputs: + - **input** (Tensor) - Tensor of shape (seq_len, batch_size, `input_size`). + - **hx** (tuple) - A tuple of two Tensors (h_0, c_0) both of data type mindspore.float32 or + mindspore.float16 and shape (num_directions * `num_layers`, batch_size, `hidden_size`). + Data type of `hx` should be the same of `input`. + + Outputs: + Tuple, a tuple constains (`output`, (`h_n`, `c_n`)). + + - **output** (Tensor) - Tensor of shape (seq_len, batch_size, num_directions * `hidden_size`). + - **hx_n** (tuple) - A tuple of two Tensor (h_n, c_n) both of shape + (num_directions * `num_layers`, batch_size, `hidden_size`). + + Examples: + >>> class LstmNet(nn.Cell): + >>> def __init__(self, input_size, hidden_size, num_layers, has_bias, batch_first, bidirectional): + >>> super(LstmNet, self).__init__() + >>> self.lstm = nn.LSTM(input_size=input_size, + >>> hidden_size=hidden_size, + >>> num_layers=num_layers, + >>> has_bias=has_bias, + >>> batch_first=batch_first, + >>> bidirectional=bidirectional, + >>> dropout=0.0) + >>> + >>> def construct(self, inp, h0, c0): + >>> return self.lstm(inp, (h0, c0)) + >>> + >>> net = LstmNet(10, 12, 2, has_bias=True, batch_first=True, bidirectional=False) + >>> input = mindspore.Tensor(np.ones([3, 5, 10]).astype(np.float32)) + >>> h0 = mindspore.Tensor(np.ones([1 * 2, 3, 12]).astype(np.float32)) + >>> c0 = mindspore.Tensor(np.ones([1 * 2, 3, 12]).astype(np.float32)) + >>> output, (hn, cn) = net(input, h0, c0) + """ + def __init__(self, + input_size, + hidden_size, + num_layers=1, + has_bias=True, + batch_first=False, + dropout=0, + bidirectional=False): + super(LSTM, self).__init__() + self.input_size = input_size + self.hidden_size = hidden_size + self.num_layers = num_layers + self.has_bias = has_bias + self.batch_first = validator.check_type("batch_first", batch_first, [bool]) + self.dropout = float(dropout) + self.bidirectional = bidirectional + + if self.batch_first: + self.transpose1 = P.Transpose() + self.transpose2 = P.Transpose() + self.lstm = P.LSTM(input_size=self.input_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + has_bias=self.has_bias, + bidirectional=self.bidirectional, + dropout=self.dropout) + + num_directions = 2 if self.bidirectional else 1 + + weight_size = 0 + gate_size = 4 * self.hidden_size + for layer in range(self.num_layers): + input_layer_size = self.input_size if layer == 0 else self.hidden_size * num_directions + increment_size = gate_size * input_layer_size + increment_size += gate_size * self.hidden_size + if self.has_bias: + increment_size += 2 * gate_size + weight_size += increment_size * num_directions + + self.weight = Parameter(initializer(0.0, [weight_size, 1, 1]), name='weight') + + self.fill = P.Fill() + self.shape = P.Shape() + + def construct(self, x, hx): + if self.batch_first: + x = self.transpose1(x, (1, 0, 2)) + h0, c0 = hx + output, hn, cn, _ = self.lstm(x, h0, c0, self.weight) + if self.batch_first: + output = self.transpose2(output, (1, 0, 2)) + return (output, (hn, cn)) diff --git a/mindspore/nn/layer/normalization.py b/mindspore/nn/layer/normalization.py new file mode 100644 index 00000000000..fdfa25e1838 --- /dev/null +++ b/mindspore/nn/layer/normalization.py @@ -0,0 +1,289 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""normalization""" +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as DT +import mindspore.context as context +from mindspore._extends import cell_attr_register +from ..cell import Cell + + +class _BatchNorm(Cell): + """Batch Normalization base class.""" + @cell_attr_register + def __init__(self, + num_features, + eps=1e-5, + momentum=0.9, + affine=True, + gamma_init='ones', + beta_init='zeros', + moving_mean_init='zeros', + moving_var_init='ones', + use_batch_statistics=True): + super(_BatchNorm, self).__init__() + if num_features < 1: + raise ValueError("num_features must be at least 1") + + if momentum < 0 or momentum > 1: + raise ValueError("momentum should be a number in range [0, 1], but got {}".format(momentum)) + + self.use_batch_statistics = use_batch_statistics + self.num_features = num_features + self.eps = eps + self.moving_mean = Parameter(initializer( + moving_mean_init, num_features), name="mean", requires_grad=False) + self.moving_variance = Parameter(initializer( + moving_var_init, num_features), name="variance", requires_grad=False) + self.gamma = Parameter(initializer( + gamma_init, num_features), name="gamma", requires_grad=affine) + self.beta = Parameter(initializer( + beta_init, num_features), name="beta", requires_grad=affine) + + if context.get_context("enable_ge"): + self.is_ge_backend = True + self.momentum = Tensor(1.0 - momentum, DT.float32) + self.bn_train = P.BatchNorm(is_training=True, + epsilon=self.eps) + else: + self.is_ge_backend = False + self.momentum = 1.0 - momentum + self.bn_train = P.FusedBatchNorm(mode=1, + epsilon=self.eps, + momentum=self.momentum) + self.bn_infer = P.BatchNorm(is_training=False, epsilon=self.eps) + + data_parallel_strategy = ((1,), (1,)) + data_parallel_strategy_one = ((1,), ()) + self.sub_mean = P.Sub().set_strategy(data_parallel_strategy) + self.sub_var = P.Sub().set_strategy(data_parallel_strategy) + self.mul_mean = P.Mul().set_strategy(data_parallel_strategy_one) + self.mul_var = P.Mul().set_strategy(data_parallel_strategy_one) + self.assign_sub_mean = P.AssignSub().set_strategy(data_parallel_strategy) + self.assign_sub_var = P.AssignSub().set_strategy(data_parallel_strategy) + + def _check_data_dim(self, x): + raise NotImplementedError + + def construct(self, x): + if self.training and self.use_batch_statistics: + if self.is_ge_backend: + y, batch_mean, batch_var, _, _ = \ + self.bn_train(x, + self.gamma, + self.beta, + None, + None) + + mean_sub = self.sub_mean(self.moving_mean, batch_mean) + temp_mean = self.mul_mean(mean_sub, self.momentum) + mean_sub2 = self.sub_var(self.moving_variance, batch_var) + temp_variance = self.mul_var(mean_sub2, self.momentum) + y = F.depend(y, self.assign_sub_mean(self.moving_mean, temp_mean)) + y = F.depend(y, self.assign_sub_var(self.moving_variance, temp_variance)) + else: + y = self.bn_train(x, + self.gamma, + self.beta, + self.moving_mean, + self.moving_variance)[0] + else: + y = self.bn_infer(x, + self.gamma, + self.beta, + self.moving_mean, + self.moving_variance)[0] + return y + + def extend_repr(self): + return 'num_features={}, eps={}, momentum={}, gamma={}, beta={}, moving_mean={}, moving_variance={}'.format( + self.num_features, self.eps, self.momentum, self.gamma, self.beta, self.moving_mean, self.moving_variance) + + +class BatchNorm1d(_BatchNorm): + r""" + Batch normalization layer over a 2D input. + + Batch Normalization is widely used in convolutional networks. This layer + applies Batch Normalization over a 2D input (a mini-batch of 1D inputs) to + reduce internal covariate shift as described in the paper + `Batch Normalization: Accelerating Deep Network Training by + Reducing Internal Covariate Shift `_. It + rescales and recenters the feature using a mini-batch of data and + the learned parameters which can be described in the following formula. + + .. math:: + y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta + + Args: + num_features (int): `C` from an expected input of size (N, C). + eps (float): A value added to the denominator for numerical stability. Default: 1e-5. + momentum (float): A floating hyperparameter of the momentum for the + running_mean and running_var computation. Default: 0.9. + gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'ones'. + beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'zeros'. + moving_mean_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving mean. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'zeros'. + moving_var_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving variance. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'ones'. + use_batch_statistics (bool): If true, use the mean value and variance value of current batch data, else use + the mean value and variance value of specified value. Default: True. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> net = nn.BatchNorm1d(num_features=16) + >>> input = mindspore.Tensor(np.random.randint(0, 255, [3, 16]), mindspore.float32) + >>> net(input) + """ + def _check_data_dim(self, x): + if x.dim() != 2: + pass + + +class BatchNorm2d(_BatchNorm): + r""" + Batch normalization layer over a 4D input. + + Batch Normalization is widely used in convolutional networks. This layer + applies Batch Normalization over a 4D input (a mini-batch of 2D inputs with + additional channel dimension) to avoid internal covariate shift as described + in the paper `Batch Normalization: Accelerating Deep Network Training by + Reducing Internal Covariate Shift `_. It + rescales and recenters the feature using a mini-batch of data and + the learned parameters which can be described in the following formula. + + .. math:: + y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta + + Args: + num_features (int): `C` from an expected input of size (N, C, H, W). + eps (float): A value added to the denominator for numerical stability. Default: 1e-5. + momentum (float): A floating hyperparameter of the momentum for the + running_mean and running_var computation. Default: 0.9. + gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'ones'. + beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'zeros'. + moving_mean_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving mean. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'zeros'. + moving_var_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving variance. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'ones'. + use_batch_statistics (bool): If true, use the mean value and variance value of current batch data, else use + the mean value and variance value of specified value. Default: True. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> net = nn.BatchNorm2d(num_features=3) + >>> input = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) + >>> net(input) + """ + def _check_data_dim(self, x): + if x.dim() != 4: + pass + + +class LayerNorm(Cell): + r""" + Applies Layer Normalization over a mini-batch of inputs. + + Layer normalization is widely used in recurrent neural networks. It applies + normalization over a mini-batch of inputs for each single training case as described + in the paper `Layer Normalization `_. Unlike batch + normalization, layer normalization performs exactly the same computation at training and + testing times. It can be described using the following formula. It is applied across all channels + and pixel but only one batch size. + + .. math:: + y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta + + Args: + normalized_shape (Union(tuple[int], list[int]): The normalization is performed over axes + `begin_norm_axis ... R - 1` and centering and scaling parameters are calculated over + `begin_params_axis ... R - 1`. + begin_norm_axis (int): It first normalization dimension: normalization will be performed along dimensions + `begin_norm_axis: rank(inputs)`, the value should be in [-1, rank(input)). Default: -1. + begin_params_axis (int): The first parameter(beta, gamma)dimension: scale and centering parameters + will have dimensions `begin_params_axis: rank(inputs)` and will be broadcast with + the normalized inputs accordingly, the value should be in [-1, rank(input)). Default: -1. + gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'ones'. + beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight. + The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform', + 'he_uniform', etc. Default: 'zeros'. + + Inputs: + - **input_x** (Tensor) - The shape of 'input_x' is input_shape = `(x_1, x_2, ..., x_R)`, + and `input_shape[begin_norm_axis:]` is equal to `normalized_shape`. + + Outputs: + Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `input_x`. + + Examples: + >>> x = Tensor(np.ones([20, 5, 10, 10], np.float32)) + >>> shape1 = x.shape()[1:] + >>> m = LayerNorm(shape1, begin_norm_axis=1, begin_params_axis=1) + >>> m(x) + """ + def __init__(self, + normalized_shape, + begin_norm_axis=-1, + begin_params_axis=-1, + gamma_init='ones', + beta_init='zeros', + ): + super(LayerNorm, self).__init__() + self.normalized_shape = normalized_shape + self.begin_norm_axis = begin_norm_axis + self.begin_params_axis = begin_params_axis + self.gamma = Parameter(initializer( + gamma_init, normalized_shape), name="gamma") + self.beta = Parameter(initializer( + beta_init, normalized_shape), name="beta") + self.layer_norm = P.LayerNorm(begin_norm_axis=self.begin_norm_axis, begin_params_axis=self.begin_params_axis) + + def construct(self, input_x): + y, _, _ = self.layer_norm(input_x, self.gamma, self.beta) + return y + + def extend_repr(self): + """Display instance object as string.""" + s = 'normalized_shape={}, begin_norm_axis={}, begin_params_axis={}, gamma{}, beta={}'.format( + self.normalized_shape, self.begin_norm_axis, self.begin_params_axis, self.gamma, self.beta) + return s diff --git a/mindspore/nn/layer/pooling.py b/mindspore/nn/layer/pooling.py new file mode 100644 index 00000000000..0a4bd1662bf --- /dev/null +++ b/mindspore/nn/layer/pooling.py @@ -0,0 +1,200 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""pooling""" + +from mindspore.ops import operations as P +from mindspore._checkparam import ParamValidator as validator +from mindspore._checkparam import Rel +from ..cell import Cell + + +class _PoolNd(Cell): + """N-D AvgPool""" + + def __init__(self, + kernel_size, + stride, + pad_mode, + padding=0, + pool=None): + super(_PoolNd, self).__init__() + self.kernel_size = kernel_size + self.stride = stride + self.pad_mode = pad_mode + self.padding = validator.check_integer('padding', padding, 0, Rel.GE) + self.pool = pool + if self.pool is None: + raise NotImplementedError + + def construct(self, x): + return self.pool(x) + + def extend_repr(self): + return 'kernel_size={kernel_size}, stride={stride}, pad_mode={pad_mode}'.format(**self.__dict__) + + +class MaxPool2d(_PoolNd): + r""" + Max pooling operation for temporal data. + + Applies a 2D max pooling over an input Tensor which can be regarded as a composition of 2D planes. + + Typically the input is of shape :math:`(N_{in}, C_{in}, H_{in}, W_{in})`, MaxPool2d outputs + regional maximum in the :math:`(H_{in}, W_{in})`-dimension. Given kernel size + :math:`ks = (h_{ker}, w_{ker})` and stride :math:`s = (s_0, s_1)`, the operation is as follows. + + .. math:: + \text{output}(N_i, C_j, h, w) = \max_{m=0, \ldots, h_{ker}-1} \max_{n=0, \ldots, w_{ker}-1} + \text{input}(N_i, C_j, s_0 \times h + m, s_1 \times w + n) + + Note: + pad_mode for training only supports "same" and "valid". + + Args: + kernel_size (int): Size of the window to take a max over. + stride (int): Stride size of the window. Default: None. + pad_mode (str): Select the mode of the pad. The optional values are + "same" and "valid". Default: "valid". + + - same: Adopts the way of completion. Output height and width will be the same as + the input. Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. Otherwise, the + last extra padding will be done from the bottom and the right side. + + - valid: Adopts the way of discarding. The possibly largest height and width of output will be return + without padding. Extra pixels will be discarded. + padding (int): Now is not supported, mplicit zero padding to be added on both sides. Default: 0. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> pool = MaxPool2d(kernel_size=3, stride=1) + >>> x = mindspore.Tensor(np.random.randint(0, 10, [1, 2, 4, 4]), mindspore.float32) + [[[[1. 5. 5. 1.] + [0. 3. 4. 8.] + [4. 2. 7. 6.] + [4. 9. 0. 1.]] + [[3. 6. 2. 6.] + [4. 4. 7. 8.] + [0. 0. 4. 0.] + [1. 8. 7. 0.]]]] + >>> output = pool(x) + >>> output.shape() + (1, 2, 2, 2) + >>> output + [[[[7. 8.] + [9. 9.]] + [[7. 8.] + [8. 8.]]]] + """ + def __init__(self, + kernel_size=1, + stride=1, + pad_mode="VALID", + padding=0): + max_pool = P.MaxPool(ksize=kernel_size, + strides=stride, + padding=pad_mode) + self.is_autodiff_backend = False + if self.is_autodiff_backend: + + # At present, pad mode of max pool is not unified, so it is a temporarily avoided + pad_mode = validator.check_string('pad_mode', pad_mode.lower(), ['valid', 'same']) + + max_pool = P.MaxPoolWithArgmax(window=kernel_size, + stride=stride, + pad_mode=pad_mode, + pad=padding) + super(MaxPool2d, self).__init__(kernel_size, stride, pad_mode, padding, max_pool) + + def construct(self, x): + if self.is_autodiff_backend: + out = self.pool(x)[0] + else: + out = self.pool(x) + return out + + +class AvgPool2d(_PoolNd): + r""" + Average pooling for temporal data. + + Applies a 2D average pooling over an input Tensor which can be regarded as a composition of 2D input planes. + + Typically the input is of shape :math:`(N_{in}, C_{in}, H_{in}, W_{in})`, AvgPool2d outputs + regional average in the :math:`(H_{in}, W_{in})`-dimension. Given kernel size + :math:`ks = (h_{ker}, w_{ker})` and stride :math:`s = (s_0, s_1)`, the operation is as follows. + + .. math:: + \text{output}(N_i, C_j, h, w) = \frac{1}{h_{ker} * w_{ker}} \sum_{m=0}^{h_{ker}-1} \sum_{n=0}^{w_{ker}-1} + \text{input}(N_i, C_j, s_0 \times h + m, s_1 \times w + n) + + Note: + pad_mode for training only supports "same" and "valid". + + Args: + kernel_size (int): Size of the window to take a max over. + stride (int): Stride size of the window. Default: None. + pad_mode (str): Select the mode of the pad. The optional values are + "same", "valid". Default: "valid". + + - same: Adopts the way of completion. Output height and width will be the same as + the input. Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. Otherwise, the + last extra padding will be done from the bottom and the right side. + + - valid: Adopts the way of discarding. The possibly largest height and width of output will be return + without padding. Extra pixels will be discarded. + padding (int): Now is not supported, implicit zero padding to be added on both sides. Default: 0. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> pool = AvgPool2d(kernel_size=3, stride=1) + >>> x = mindspore.Tensor(np.random.randint(0, 10, [1, 2, 4, 4]), mindspore.float32) + [[[[5. 5. 9. 9.] + [8. 4. 3. 0.] + [2. 7. 1. 2.] + [1. 8. 3. 3.]] + [[6. 8. 2. 4.] + [3. 0. 2. 1.] + [0. 8. 9. 7.] + [2. 1. 4. 9.]]]] + >>> output = pool(x) + >>> output.shape() + (1, 2, 2, 2) + >>> output + [[[[4.888889 4.4444447] + [4.111111 3.4444444]] + [[4.2222223 4.5555553] + [3.2222223 4.5555553]]]] + """ + def __init__(self, + kernel_size=1, + stride=1, + pad_mode="VALID", + padding=0): + avg_pool = P.AvgPool(ksize=kernel_size, + strides=stride, + padding=pad_mode) + super(AvgPool2d, self).__init__(kernel_size, stride, pad_mode, padding, avg_pool) diff --git a/mindspore/nn/loss/__init__.py b/mindspore/nn/loss/__init__.py new file mode 100644 index 00000000000..f08f5aa721c --- /dev/null +++ b/mindspore/nn/loss/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Loss. + +Cells of loss function. Loss function in machine learning is the target of the model. +It shows how well the model works on a dataset and the optimization target which the optimizer is searching. +""" + +from .loss import L1Loss, MSELoss, SmoothL1Loss, \ + SoftmaxCrossEntropyWithLogits, SoftmaxCrossEntropyExpand + +__all__ = ['L1Loss', 'MSELoss', 'SmoothL1Loss', + 'SoftmaxCrossEntropyWithLogits', + 'SoftmaxCrossEntropyExpand'] diff --git a/mindspore/nn/loss/loss.py b/mindspore/nn/loss/loss.py new file mode 100644 index 00000000000..340cbe73d84 --- /dev/null +++ b/mindspore/nn/loss/loss.py @@ -0,0 +1,324 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""loss""" +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.nn.cell import Cell +from ... import context + + +class _Loss(Cell): + """ + Base class for other losses. + """ + def __init__(self, reduction='mean'): + super(_Loss, self).__init__() + if reduction is None: + reduction = 'none' + + if reduction not in ('mean', 'sum', 'none'): + raise ValueError(f"reduction method for {reduction.lower()} is not supported") + + self.average = True + self.reduce = True + if reduction == 'sum': + self.average = False + if reduction == 'none': + self.reduce = False + + self.reduce_mean = P.ReduceMean() + self.reduce_sum = P.ReduceSum() + + def get_axis(self, x): + shape = F.shape(x) + length = F.tuple_len(shape) + perm = F.make_range(0, length) + return perm + + def get_loss(self, x): + if self.reduce and self.average: + x = self.reduce_mean(x, self.get_axis(x)) + if self.reduce and not self.average: + x = self.reduce_sum(x, self.get_axis(x)) + return x + + def construct(self, base, target): + raise NotImplementedError + + +class L1Loss(_Loss): + r""" + L1Loss creates a criterion to measure the mean absolute error (MAE) between :math:`x` and :math:`y` by element, + where :math:`x` is the input Tensor and :math:`y` is the target Tensor. + + For simplicity, let :math:`x` and :math:`y` be 1-dimensional Tensor with length :math:`N`, + the unreduced loss (i.e. with argument reduction set to 'none') of :math:`x` and :math:`y` is given as: + + .. math:: + L(x, y) = \{l_1,\dots,l_N\}, \quad \text{with } l_n = \left| x_n - y_n \right| + + When argument reduction is 'mean', the mean value of :math:`L(x, y)` will be returned. + When argument reduction is 'sum', the sum of :math:`L(x, y)` will be returned. :math:`N` is the batch size. + + Args: + reduction (str): Type of reduction to apply to loss. The optional values are "mean", "sum", "none". + Default: "mean". + + Inputs: + - **input_data** (Tensor) - Tensor of shape :math:`(x_1, x_2, ..., x_R)`. + - **target_data** (Tensor) - Tensor of shape :math:`(y_1, y_2, ..., y_S)`. + + Outputs: + Tensor, loss float tensor. + + Examples: + >>> loss = L1Loss() + >>> input_data = Tensor(np.array([1, 2, 3]), mstype.float32) + >>> target_data = Tensor(np.array([1, 2, 2]), mstype.float32) + >>> loss(input_data, target_data) + """ + def __init__(self, reduction='mean'): + super(L1Loss, self).__init__(reduction) + self.abs = P.Abs() + + def construct(self, base, target): + x = self.abs(base - target) + return self.get_loss(x) + + +class MSELoss(_Loss): + r""" + MSELoss create a criterion to measures the mean squared error (squared L2-norm) between :math:`x` and :math:`y` + by element, where :math:`x` is the input and :math:`y` is the target. + + For simplicity, let :math:`x` and :math:`y` be 1-dimensional Tensor with length :math:`N`, + the unreduced loss (i.e. with argument reduction set to 'none') of :math:`x` and :math:`y` is given as: + + .. math:: + L(x, y) = \{l_1,\dots,l_N\}, \quad \text{with} \quad l_n = (x_n - y_n)^2. + + When argument reduction is 'mean', the mean value of :math:`L(x, y)` will be returned. + When argument reduction is 'sum', the sum of :math:`L(x, y)` will be returned. :math:`N` is the batch size. + + Args: + reduction (str): Type of reduction to apply to loss. The optional values are "mean", "sum", "none". + Default: "mean". + + Inputs: + - **input_data** (Tensor) - Tensor of shape :math:`(x_1, x_2, ..., x_R)`. + - **target_data** (Tensor) - Tensor of shape :math:`(y_1, y_2, ..., y_S)`. + + Outputs: + Tensor, weighted loss float tensor. + + Examples: + >>> loss = MSELoss() + >>> input_data = Tensor(np.array([1, 2, 3]), mstype.float32) + >>> target_data = Tensor(np.array([1, 2, 2]), mstype.float32) + >>> loss(input_data, target_data) + """ + def construct(self, base, target): + x = F.square(base - target) + return self.get_loss(x) + + +class SmoothL1Loss(_Loss): + r""" + A loss class for learning region proposals. + + SmoothL1Loss can be regarded as modified version of L1Loss or a combination of L1Loss and L2Loss. + L1Loss computes the element-wise absolute difference between two input Tensor while L2Loss computes the + squared difference between two input Tensor. L2Loss often leads to faster convergence but it is less + robust to outliers. + + Given two input :math:`x,\ y` of length :math:`N`, the unreduced SmoothL1Loss can be described + as follows: + + .. math:: + L_{i} = + \begin{cases} + 0.5 (x_i - y_i)^2, & \text{if } |x_i - y_i| < \text{sigma}; \\ + |x_i - y_i| - 0.5, & \text{otherwise. } + \end{cases} + + Here :math:`\text{sigma}` controls the point where the loss function changes from quadratic to linear. + Its default value is 1.0. :math:`N` is the batch size. This function returns an + unreduced loss Tensor. + + Args: + sigma (float): A parameter used to control the point where the function will change from + quadratic to linear. Default: 1.0. + + Inputs: + - **input_data** (Tensor) - Tensor of shape :math:`(x_1, x_2, ..., x_R)`. + - **target_data** (Tensor) - Tensor of shape :math:`(y_1, y_2, ..., y_S)`. + + Outputs: + Tensor, loss float tensor. + + Examples: + >>> loss = SmoothL1Loss() + >>> input_data = Tensor(np.array([1, 2, 3]), mstype.float32) + >>> target_data = Tensor(np.array([1, 2, 2]), mstype.float32) + >>> loss(input_data, target_data) + """ + def __init__(self, sigma=1.0): + super(SmoothL1Loss, self).__init__() + self.sigma = sigma + self.smooth_l1_loss = P.SmoothL1Loss(self.sigma) + + def construct(self, base, target): + return self.smooth_l1_loss(base, target) + + +class SoftmaxCrossEntropyWithLogits(_Loss): + r""" + Computes softmax cross entropy between logits and labels. + + Measures the distribution error between the probabilities of the input (computed with softmax function) and the + target where the classes are mutually exclusive (only one class is positive) using cross entropy loss. + + Typical input into this function is unnormalized scores and target of each class. + Scores Tensor :math:`x` is of shape :math:`(N, C)` and target Tensor :math:`t` is a + Tensor of shape :math:`(N, C)` which contains one-hot labels of length :math:`C`. + + For each batch :math:`N_i`, the loss is given as: + + .. math:: + \ell(x_i, t_i) = -w_{t_i} \log\left(\frac{\exp(x_{t_i})}{\sum_j \exp(x_j)}\right) + = w_{t_i} \left(-x_{t_i} + \log\left(\sum_j \exp(x_i)\right)\right), + where :math:`x_i` is a 1D score Tensor, :math:`t_i` is the target class and + :math:`w` is a weight Tensor to generate weighted loss for each class. When not specified, + weight Tensor is set to be None and weight is the same (:math:`1`) for all class. + + Note: + While the target classes are mutually exclusive, i.e., only one class is positive in the target, the predicted + probabilities need not be exclusive. All that is required is that the predicted probability distribution + of entry is a valid one. + + Args: + is_grad (bool): Specifies whether calculate grad only. Default: True. + sparse (bool): Specifies whether labels use sparse format or not. Default: False. + reduction (Union[str, None]): Type of reduction to apply to loss. Support 'sum' or 'mean' If None, + do not reduction. Default: None. + + Inputs: + - **logits** (Tensor) - Tensor of shape :math:`(x_1, x_2, ..., x_R)`. + - **labels** (Tensor) - Tensor of shape :math:`(y_1, y_2, ..., y_S)`. If `sparse` is True, The type of + `labels` is mstype.int32. If `sparse` is False, the type of `labels` is same as the type of `logits`. + + Outputs: + Tensor, a tensor of the same shape as logits with the component-wise + logistic losses. + + Examples: + >>> loss = SoftmaxCrossEntropyWithLogits(sparse=True) + >>> logits = Tensor(np.random.randint(0, 9, [1, 10]), mstype.float32) + >>> labels_np = np.zeros([1, 10]).astype(np.int32) + >>> labels_np[0][0] = 1 + >>> labels = Tensor(labels_np) + >>> loss(logits, labels) + """ + def __init__(self, + is_grad=True, + sparse=False, + reduction=None): + super(SoftmaxCrossEntropyWithLogits, self).__init__(reduction) + self.is_grad = is_grad + self.sparse = sparse + self.softmax_cross_entropy = P.SoftmaxCrossEntropyWithLogits() + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.is_cpugpu = context.get_context('device_target') in ["CPU", "GPU"] + + if self.is_cpugpu: + self.sparse_softmax_cross_entropy = P.SparseSoftmaxCrossEntropyWithLogits(is_grad=self.is_grad) + + def construct(self, logits, labels): + if self.is_cpugpu and self.sparse: + x = self.sparse_softmax_cross_entropy(logits, labels) + return x + + if self.sparse: + labels = self.one_hot(labels, F.shape(logits)[-1], self.on_value, self.off_value) + x = self.softmax_cross_entropy(logits, labels)[0] + return self.get_loss(x) + + +class SoftmaxCrossEntropyExpand(Cell): + r""" + Computes softmax cross entropy between logits and labels. Implemented by expanded formula. + + This is a wrapper of several functions. + + .. math:: + \ell(x_i, t_i) = -log\left(\frac{\exp(x_{t_i})}{\sum_j \exp(x_j)}\right), + where :math:`x_i` is a 1D score Tensor, :math:`t_i` is the target class. + + Note: + When argument sparse is set to True, the format of label is the index + range from :math:`0` to :math:`C - 1` instead of one-hot vectors. + + Args: + sparse(bool): Specifies whether labels use sparse format or not. Default: False. + + Inputs: + - **input_data** (Tensor) - Tensor of shape :math:`(x_1, x_2, ..., x_R)`. + - **label** (Tensor) - Tensor of shape :math:`(y_1, y_2, ..., y_S)`. + + Outputs: + Tensor, a scalar tensor including the mean loss. + + Examples: + >>> loss = SoftmaxCrossEntropyExpand(sparse=True) + >>> input_data = Tensor(np.ones([64, 512]), dtype=mstype.float32) + >>> label = Tensor(np.ones([64]), dtype=mstype.int32) + >>> loss(input_data, label) + """ + def __init__(self, sparse=False): + super(SoftmaxCrossEntropyExpand, self).__init__() + self.exp = P.Exp() + self.reduce_sum = P.ReduceSum(keep_dims=True) + self.onehot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.div = P.Div() + self.log = P.Log() + self.sum_cross_entropy = P.ReduceSum(keep_dims=False) + self.mul = P.Mul() + self.mul2 = P.Mul() + self.cast = P.Cast() + self.reduce_mean = P.ReduceMean(keep_dims=False) + self.sparse = sparse + self.reduce_max = P.ReduceMax(keep_dims=True) + self.sub = P.Sub() + + def construct(self, logit, label): + logit_max = self.reduce_max(logit, -1) + exp = self.exp(self.sub(logit, logit_max)) + exp_sum = self.reduce_sum(exp, -1) + softmax_result = self.div(exp, exp_sum) + if self.sparse: + label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + + softmax_result_log = self.log(softmax_result) + loss = self.sum_cross_entropy((self.mul(softmax_result_log, label)), -1) + loss = self.mul2(F.scalar_to_array(-1.0), loss) + loss = self.reduce_mean(loss, -1) + + return loss diff --git a/mindspore/nn/metrics/__init__.py b/mindspore/nn/metrics/__init__.py new file mode 100755 index 00000000000..490e1620fab --- /dev/null +++ b/mindspore/nn/metrics/__init__.py @@ -0,0 +1,119 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Metrics. + +Functions to measure the performance of the machine learning models +on the evaluation dataset. It's used to choose the best model. +""" +from .accuracy import Accuracy +from .error import MAE, MSE +from .evaluation import EvaluationBase +from .metric import Metric +from .precision import Precision +from .recall import Recall +from .fbeta import Fbeta, F1 +from .topk import TopKCategoricalAccuracy, Top1CategoricalAccuracy, Top5CategoricalAccuracy +from .loss import Loss + +__all__ = [ + "names", "get_metric_fn", + "Accuracy", + "MAE", "MSE", + "EvaluationBase", + "Metric", + "Precision", + "Recall", + "Fbeta", + "F1", + "TopKCategoricalAccuracy", + "Top1CategoricalAccuracy", + "Top5CategoricalAccuracy", + "Loss", +] + +__factory__ = { + 'accuracy': Accuracy, + 'acc': Accuracy, + 'precision': Precision, + 'recall': Recall, + 'F1': F1, + 'topk': TopKCategoricalAccuracy, + 'top_1_accuracy': Top1CategoricalAccuracy, + 'top_5_accuracy': Top5CategoricalAccuracy, + 'mae': MAE, + 'mse': MSE, + 'loss': Loss, +} + + +def names(): + """ + Get the names of the metric methods. + + Returns: + List, the name list of metric methods. + """ + return sorted(__factory__.keys()) + + +def get_metric_fn(name, *args, **kwargs): + """ + Gets the metric method base on the input name. + + Args: + name (str): The name of metric method. Refer to the '__factory__' + object for the currently supported metrics. + args: Arguments for the metric function. + kwargs: Keyword arguments for the metric function. + + Returns: + Metric object, class instance of the metric method. + + Examples: + >>> metric = get_metric_fn('precision', eval_type='classification') + """ + if name not in __factory__: + raise KeyError("Unknown Metric:", name) + return __factory__[name](*args, **kwargs) + + +def get_metrics(metrics): + """ + Get metrics used in evaluation. + + Args: + metrics (Union[dict, set]): Dict or set of metrics to be evaluated by the model during training and + testing. eg: {'accuracy', 'recall'}. + + Returns: + dict, the key is metric name, the value is class instance of metric method. + """ + if metrics is None: + return metrics + + if isinstance(metrics, dict): + for name, metric in metrics.items(): + if not isinstance(name, str) or not isinstance(metric, Metric): + raise TypeError("Metrics format error. key in metrics should be str \ + and value in metrics should be subclass of Metric") + return metrics + if isinstance(metrics, set): + out_metrics = {} + for name in metrics: + out_metrics[name] = get_metric_fn(name) + return out_metrics + + raise TypeError("Metrics should be None, dict or set, but got {}".format(metrics)) diff --git a/mindspore/nn/metrics/accuracy.py b/mindspore/nn/metrics/accuracy.py new file mode 100644 index 00000000000..5a11fa9d08b --- /dev/null +++ b/mindspore/nn/metrics/accuracy.py @@ -0,0 +1,111 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Accuracy.""" +import numpy as np +from .evaluation import EvaluationBase + + +class Accuracy(EvaluationBase): + r""" + Calculates the accuracy for classification and multilabel data. + + The accuracy class creates two local variables, correct number and total number that are used to compute the + frequency with which predictions matches labels. This frequency is ultimately returned as the accuracy: an + idempotent operation that simply divides correct number by total number. + + .. math:: + \text{accuracy} =\frac{\text{true_positive} + \text{true_negative}} + {\text{true_positive} + \text{true_negative} + \text{false_positive} + \text{false_negative}} + + Args: + eval_type (str): Metric to calculate the accuracy over a dataset, for + classification (single-label), and multilabel (multilabel classification). + Default: 'classification'. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]]), mindspore.float32) + >>> y = mindspore.Tensor(np.array([1, 0, 1]), mindspore.float32) + >>> metric = nn.Accuracy('classification') + >>> metric.clear() + >>> metric.update(x, y) + >>> accuracy = metric.eval() + """ + def __init__(self, eval_type='classification'): + super(Accuracy, self).__init__(eval_type) + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._correct_num = 0 + self._total_num = 0 + self._class_num = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result :math:`y_{pred}` and :math:`y`. + + Args: + inputs: Input `y_pred` and `y`. `y_pred` and `y` are a `Tensor`, a list or an array. + `y_pred` is in most cases (not strictly) a list of floating numbers in range :math:`[0, 1]` + and the shape is :math:`(N, C)`, where :math:`N` is the number of cases and :math:`C` + is the number of categories. For 'multilabel' evaluation type, `y_pred` can only be one-hot + encoding with values 0 or 1. Indices with 1 indicate positive category. `y` contains values + of integers. The shape is :math:`(N, C)` if one-hot encoding is used. One-hot encoding + should be used when 'eval_type' is 'multilabel'. Shape can also be :math:`(N, 1)` if category + index is used in 'classification' evaluation type. + + Raises: + ValueError: If the number of the input is not 2. + """ + if len(inputs) != 2: + raise ValueError('Accuracy need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + if self._type == 'classification' and y_pred.ndim == y.ndim and self._check_onehot_data(y): + y = y.argmax(axis=1) + self._check_shape(y_pred, y) + self._check_value(y_pred, y) + + if self._class_num == 0: + self._class_num = y_pred.shape[1] + elif y_pred.shape[1] != self._class_num: + raise ValueError('Class number not match, last input data contain {} classes, but current data contain {} ' + 'classes'.format(self._class_num, y_pred.shape[1])) + + if self._type == 'classification': + indices = y_pred.argmax(axis=1) + result = (np.equal(indices, y) * 1).reshape(-1) + elif self._type == 'multilabel': + dimension_index = y_pred.ndim - 1 + y_pred = y_pred.swapaxes(1, dimension_index).reshape(-1, self._class_num) + y = y.swapaxes(1, dimension_index).reshape(-1, self._class_num) + result = np.equal(y_pred, y).all(axis=1) * 1 + + self._correct_num += result.sum() + self._total_num += result.shape[0] + + def eval(self): + """ + Computes the accuracy. + + Returns: + Float, the computed result. + + Raises: + RuntimeError: If the sample size is 0. + """ + if self._total_num == 0: + raise RuntimeError('Accuary can not be calculated, because the number of samples is 0.') + return self._correct_num / self._total_num diff --git a/mindspore/nn/metrics/error.py b/mindspore/nn/metrics/error.py new file mode 100644 index 00000000000..5dbd83645bc --- /dev/null +++ b/mindspore/nn/metrics/error.py @@ -0,0 +1,146 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Error.""" +import numpy as np +from .metric import Metric + + +class MAE(Metric): + r""" + Calculates the mean absolute error. + + Creates a criterion that measures the mean absolute error (MAE) + between each element in the input: :math:`x` and the target: :math:`y`. + + .. math:: + \text{MAE} = \frac{\sum_{i=1}^n \|y_i - x_i\|}{n} + + Here :math:`y_i` is the prediction and :math:`x_i` is the true value. + + Note: + The method `update` must be called with the form `update(y_pred, y)`. + + Examples: + >>> x = mindspore.Tensor(np.array([0.1, 0.2, 0.6, 0.9]), mindspore.float32) + >>> y = mindspore.Tensor(np.array([0.1, 0.25, 0.7, 0.9]), mindspore.float32) + >>> error = nn.MAE() + >>> error.clear() + >>> error.update(x, y) + >>> result = error.eval() + """ + def __init__(self): + super(MAE, self).__init__() + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._abs_error_sum = 0 + self._samples_num = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result :math:`y_{pred}` and :math:`y`. + + Args: + inputs: Input `y_pred` and `y` for calculating mean absolute error where the shape of + `y_pred` and `y` are both N-D and the shape are the same. + + Raises: + ValueError: If the number of the input is not 2. + """ + if len(inputs) != 2: + raise ValueError('Mean absolute error need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + abs_error_sum = np.abs(y.reshape(y_pred.shape) - y_pred) + self._abs_error_sum += abs_error_sum.sum() + self._samples_num += y.shape[0] + + def eval(self): + """ + Computes the mean absolute error. + + Returns: + Float, the computed result. + + Raises: + RuntimeError: If the number of the total samples is 0. + """ + if self._samples_num == 0: + raise RuntimeError('Total samples num must not be 0.') + return self._abs_error_sum / self._samples_num + + +class MSE(Metric): + r""" + Measures the mean squared error. + + Creates a criterion that measures the mean squared error (squared L2 + norm) between each element in the input: :math:`x` and the target: :math:`y`. + + .. math:: + \text{MSE}(x,\ y) = \frac{\sum_{i=1}^n(y_i - x_i)^2}{n}, + where :math:`n` is batch size. + + Examples: + >>> x = mindspore.Tensor(np.array([0.1, 0.2, 0.6, 0.9]), mindspore.float32) + >>> y = mindspore.Tensor(np.array([0.1, 0.25, 0.5, 0.9]), mindspore.float32) + >>> error = MSE() + >>> error.clear() + >>> error.update(x, y) + >>> result = error.eval() + """ + def __init__(self): + super(MSE, self).__init__() + self.clear() + + def clear(self): + """Clear the internal evaluation result.""" + self._squared_error_sum = 0 + self._samples_num = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result :math:`y_{pred}` and :math:`y`. + + Args: + inputs: Input `y_pred` and `y` for calculating mean square error where the shape of + `y_pred` and `y` are both N-D and the shape are the same. + + Raises: + ValueError: If the number of input is not 2. + """ + if len(inputs) != 2: + raise ValueError('Mean squared error need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + squared_error_sum = np.power(y.reshape(y_pred.shape) - y_pred, 2) + self._squared_error_sum += squared_error_sum.sum() + self._samples_num += y.shape[0] + + def eval(self): + """ + Compute the mean squared error. + + Returns: + Float, the computed result. + + Raises: + RuntimeError: If the number of samples is 0. + """ + if self._samples_num == 0: + raise RuntimeError('The number of input samples must not be 0.') + return self._squared_error_sum / self._samples_num diff --git a/mindspore/nn/metrics/evaluation.py b/mindspore/nn/metrics/evaluation.py new file mode 100644 index 00000000000..dc250007513 --- /dev/null +++ b/mindspore/nn/metrics/evaluation.py @@ -0,0 +1,103 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Evaluation.""" +import numpy as np +from .metric import Metric + +_eval_types = {'classification', 'multilabel'} + + +class EvaluationBase(Metric): + """ + Base class of evaluation. + + Note: + Please refer to the definition of class `Accuracy`. + + Args: + eval_type (str): Type of evaluation must be in {'classification', 'multilabel'}. + + Raises: + TypeError: If the input type is not classification or multilabel. + """ + def __init__(self, eval_type): + super(EvaluationBase, self).__init__() + if eval_type not in _eval_types: + raise TypeError('Type must be in {}, but got {}'.format(_eval_types, eval_type)) + self._type = eval_type + + def _check_shape(self, y_pred, y): + """ + Checks the shapes of y_pred and y. + + Args: + y_pred (Tensor): Predict array. + y (Tensor): Target array. + """ + if self._type == 'classification': + if y_pred.ndim != y.ndim + 1: + raise ValueError('Classification case, dims of y_pred equal dims of y add 1, ' + 'but got y_pred: {} dims and y: {} dims'.format(y_pred.ndim, y.ndim)) + if y.shape != (y_pred.shape[0],) + y_pred.shape[2:]: + raise ValueError('Classification case, y_pred shape and y shape can not match. ' + 'got y_pred shape is {} and y shape is {}'.format(y_pred.shape, y.shape)) + else: + if y_pred.ndim != y.ndim: + raise ValueError('{} case, dims of y_pred need equal with dims of y, but got y_pred: {} ' + 'dims and y: {} dims.'.format(self._type, y_pred.ndim, y.ndim)) + if y_pred.shape != y.shape: + raise ValueError('{} case, y_pred shape need equal with y shape, but got y_pred: {} and y: {}'. + format(self._type, y_pred.shape, y.shape)) + + def _check_value(self, y_pred, y): + """ + Checks the values of y_pred and y. + + Args: + y_pred (Tensor): Predict array. + y (Tensor): Target array. + """ + if self._type != 'classification' and not (np.equal(y_pred ** 2, y_pred).all() and np.equal(y ** 2, y).all()): + raise ValueError('For multilabel case, input value must be 1 or 0.') + + def clear(self): + """ + A interface describes the behavior of clearing the internal evaluation result. + + Note: + All subclasses should override this interface. + """ + raise NotImplementedError + + def update(self, *inputs): + """ + A interface describes the behavior of updating the internal evaluation result. + + Note: + All subclasses should override this interface. + + Args: + inputs: The first item is predicted array and the second item is target array. + """ + raise NotImplementedError + + def eval(self): + """ + A interface describes the behavior of computing the evaluation result. + + Note: + All subclasses should override this interface. + """ + raise NotImplementedError diff --git a/mindspore/nn/metrics/fbeta.py b/mindspore/nn/metrics/fbeta.py new file mode 100755 index 00000000000..f38febf3b14 --- /dev/null +++ b/mindspore/nn/metrics/fbeta.py @@ -0,0 +1,136 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Fbeta.""" +import sys +import numpy as np +from mindspore._checkparam import ParamValidator as validator +from .metric import Metric + + +class Fbeta(Metric): + r""" + Calculates the fbeta score. + + Fbeta score is a weighted mean of precison and recall. + + .. math:: + F_\beta=\frac{(1+\beta^2) \cdot true positive} + {(1+\beta^2) \cdot true positive +\beta^2 \cdot false negative + false positive} + + Args: + beta (float): The weight of precision. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + >>> y = mindspore.Tensor(np.array([1, 0, 1])) + >>> metric = nn.Fbeta(1) + >>> metric.update(x, y) + >>> fbeta = metric.eval() + [0.66666667 0.66666667] + """ + def __init__(self, beta): + super(Fbeta, self).__init__() + self.eps = sys.float_info.min + if not beta > 0: + raise ValueError('`beta` must greater than zero, but got {}'.format(beta)) + self.beta = beta + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._true_positives = 0 + self._actual_positives = 0 + self._positives = 0 + self._class_num = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result `y_pred` and `y`. + + Args: + inputs: Input `y_pred` and `y`. `y_pred` and `y` are Tensor, list or numpy.ndarray. + `y_pred` is in most cases (not strictly) a list of floating numbers in range :math:`[0, 1]` + and the shape is :math:`(N, C)`, where :math:`N` is the number of cases and :math:`C` + is the number of categories. y contains values of integers. The shape is :math:`(N, C)` + if one-hot encoding is used. Shape can also be :math:`(N, 1)` if category index is used. + """ + if len(inputs) != 2: + raise ValueError('Fbeta need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + if y_pred.ndim == y.ndim and self._check_onehot_data(y): + y = y.argmax(axis=1) + + if self._class_num == 0: + self._class_num = y_pred.shape[1] + elif y_pred.shape[1] != self._class_num: + raise ValueError('Class number not match, last input data contain {} classes, but current data contain {} ' + 'classes'.format(self._class_num, y_pred.shape[1])) + class_num = self._class_num + + if y.max() + 1 > class_num: + raise ValueError('y_pred contains {} classes less than y contains {} classes.'. + format(class_num, y.max() + 1)) + y = np.eye(class_num)[y.reshape(-1)] + indices = y_pred.argmax(axis=1).reshape(-1) + y_pred = np.eye(class_num)[indices] + + positives = y_pred.sum(axis=0) + actual_positives = y.sum(axis=0) + true_positives = (y * y_pred).sum(axis=0) + + self._true_positives += true_positives + self._positives += positives + self._actual_positives += actual_positives + + def eval(self, average=False): + """ + Computes the fbeta. + + Args: + average (bool): Whether to calculate the average fbeta. Default value is False. + + Returns: + Float, computed result. + """ + validator.check_type("average", average, [bool]) + if self._class_num == 0: + raise RuntimeError('Input number of samples can not be 0.') + + fbeta = (1.0 + self.beta ** 2) * self._true_positives / \ + (self.beta ** 2 * self._actual_positives + self._positives + self.eps) + + if average: + return fbeta.mean() + return fbeta + + +class F1(Fbeta): + r""" + Calculates the F1 score. F1 is a special case of Fbeta when beta is 1. + Refer to class `Fbeta` for more details. + + .. math:: + F_\beta=\frac{2\cdot true positive}{2\cdot true positive + false negative + false positive} + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + >>> y = mindspore.Tensor(np.array([1, 0, 1])) + >>> metric = nn.F1() + >>> metric.update(x, y) + >>> fbeta = metric.eval() + """ + def __init__(self): + super(F1, self).__init__(1.0) diff --git a/mindspore/nn/metrics/loss.py b/mindspore/nn/metrics/loss.py new file mode 100644 index 00000000000..bc4c58ef2f0 --- /dev/null +++ b/mindspore/nn/metrics/loss.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Loss for evaluation""" +from .metric import Metric + + +class Loss(Metric): + r""" + Calculates the average of the loss. If method 'update' is called every :math:`n` iterations, the result of + evaluation will be: + + .. math:: + loss = \frac{\sum_{k=1}^{n}loss_k}{n} + + Examples: + >>> x = mindspore.Tensor(np.array(0.2), mindspore.float32) + >>> loss = nn.Loss() + >>> loss.clear() + >>> loss.update(x) + >>> result = loss.eval() + 0.20000000298023224 + """ + def __init__(self): + super(Loss, self).__init__() + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._sum_loss = 0 + self._total_num = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result. + + Args: + inputs: Inputs contain only one element, the element is loss. The dimension of + loss should be 0 or 1. + + Raises: + ValueError: If the length of inputs is not 1. + ValueError: If the dimensions of loss is not 1. + """ + if len(inputs) != 1: + raise ValueError('Length of inputs must be 1, but got {}'.format(len(inputs))) + + loss = self._convert_data(inputs[0]) + + if loss.ndim == 0: + loss = loss.reshape(1) + + if loss.ndim != 1: + raise ValueError("Dimensions of loss must be 1, but got {}".format(loss.ndim)) + + loss = loss.mean(-1) + self._sum_loss += loss + self._total_num += 1 + + def eval(self): + """ + Calculates the average of the loss. + + Returns: + Float, the average of the loss. + + Raises: + RuntimeError: If the total number is 0. + """ + if self._total_num == 0: + raise RuntimeError('Total number can not be 0.') + return self._sum_loss / self._total_num diff --git a/mindspore/nn/metrics/metric.py b/mindspore/nn/metrics/metric.py new file mode 100644 index 00000000000..d8e38796533 --- /dev/null +++ b/mindspore/nn/metrics/metric.py @@ -0,0 +1,113 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Metric base class.""" +from abc import ABCMeta, abstractmethod +import numpy as np +from mindspore.common.tensor import Tensor + + +class Metric(metaclass=ABCMeta): + """ + Base class of metric. + + + Note: + For examples of subclasses, please refer to the definition of class `MAE`, 'Recall' etc. + """ + def __init__(self): + pass + + def _convert_data(self, data): + """ + Convert data type to numpy array. + + Args: + data (Object): Input data. + + Returns: + Ndarray, data with `np.ndarray` type. + """ + if isinstance(data, Tensor): + data = data.asnumpy() + elif isinstance(data, list): + data = np.array(data) + elif isinstance(data, np.ndarray): + pass + else: + raise TypeError('Input data type must be tensor, list or numpy.ndarray') + return data + + def _check_onehot_data(self, data): + """ + Whether input data are one-hot encoding. + + Args: + data (numpy.array): Input data. + + Returns: + bool, return trun, if input data are one-hot encoding. + """ + if data.ndim > 1 and np.equal(data ** 2, data).all(): + shp = (data.shape[0],) + data.shape[2:] + if np.equal(np.ones(shp), data.sum(axis=1)).all(): + return True + return False + + def __call__(self, *inputs): + """ + Evaluate input data once. + + Args: + inputs (tuple): The first item is predict array, the second item is target array. + + Returns: + Float, compute result. + """ + self.clear() + self.update(*inputs) + return self.eval() + + @abstractmethod + def clear(self): + """ + A interface describes the behavior of clearing the internal evaluation result. + + Note: + All subclasses should override this interface. + """ + raise NotImplementedError('Must define clear function to use this base class') + + @abstractmethod + def eval(self): + """ + A interface describes the behavior of computing the evaluation result. + + Note: + All subclasses should override this interface. + """ + raise NotImplementedError('Must define eval function to use this base class') + + @abstractmethod + def update(self, *inputs): + """ + A interface describes the behavior of updating the internal evaluation result. + + Note: + All subclasses should override this interface. + + Args: + inputs: A variable-length input argument list. + """ + raise NotImplementedError('Must define update function to use this base class') diff --git a/mindspore/nn/metrics/precision.py b/mindspore/nn/metrics/precision.py new file mode 100644 index 00000000000..a2c8502002a --- /dev/null +++ b/mindspore/nn/metrics/precision.py @@ -0,0 +1,146 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Precision.""" +import sys + +import numpy as np + +from mindspore._checkparam import ParamValidator as validator +from .evaluation import EvaluationBase + + +class Precision(EvaluationBase): + r""" + Calculates precision for classification and multilabel data. + + The precision function creates two local variables, :math:`\text{true_positive}` and + :math:`\text{false_positive}`, that are used to compute the precision. This value is + ultimately returned as the precision, an idempotent operation that simply divides + :math:`\text{true_positive}` by the sum of :math:`\text{true_positive}` and :math:`\text{false_positive}`. + + .. math:: + \text{precision} = \frac{\text{true_positive}}{\text{true_positive} + \text{false_positive}} + + Note: + In the multi-label cases, the elements of :math:`y` and :math:`y_{pred}` should be 0 or 1. + + Args: + eval_type (str): Metric to calculate accuracy over a dataset, for classification or + multilabel. Default: 'classification'. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + >>> y = mindspore.Tensor(np.array([1, 0, 1])) + >>> metric = nn.Precision('classification') + >>> metric.clear() + >>> metric.update(x, y) + >>> precision = metric.eval() + [0.5 1. ] + """ + def __init__(self, eval_type='classification'): + super(Precision, self).__init__(eval_type) + self.eps = sys.float_info.min + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._class_num = 0 + if self._type == "multilabel": + self._true_positives = np.empty(0) + self._positives = np.empty(0) + self._true_positives_average = 0 + self._positives_average = 0 + else: + self._true_positives = 0 + self._positives = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result with `y_pred` and `y`. + + Args: + inputs: Input `y_pred` and `y`. `y_pred` and `y` are Tensor, list or numpy.ndarray. + `y_pred` is in most cases (not strictly) a list of floating numbers in range :math:`[0, 1]` + and the shape is :math:`(N, C)`, where :math:`N` is the number of cases and :math:`C` + is the number of categories. For 'multilabel' evaluation type, `y_pred` can only be one-hot + encoding with values 0 or 1. Indices with 1 indicate positive category. `y` contains values + of integers. The shape is :math:`(N, C)` if one-hot encoding is used. One-hot encoding + should be used when 'eval_type' is 'multilabel'. Shape can also be :math:`(N, 1)` if category + index is used in 'classification' evaluation type. + + Raises: + ValueError: If the number of input is not 2. + """ + if len(inputs) != 2: + raise ValueError('Precision need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + if self._type == 'classification' and y_pred.ndim == y.ndim and self._check_onehot_data(y): + y = y.argmax(axis=1) + self._check_shape(y_pred, y) + self._check_value(y_pred, y) + + if self._class_num == 0: + self._class_num = y_pred.shape[1] + elif y_pred.shape[1] != self._class_num: + raise ValueError('Class number not match, last input data contain {} classes, but current data contain {} ' + 'classes'.format(self._class_num, y_pred.shape[1])) + + class_num = self._class_num + if self._type == "classification": + if y.max() + 1 > class_num: + raise ValueError('y_pred contains {} classes less than y contains {} classes.'. + format(class_num, y.max() + 1)) + y = np.eye(class_num)[y.reshape(-1)] + indices = y_pred.argmax(axis=1).reshape(-1) + y_pred = np.eye(class_num)[indices] + elif self._type == "multilabel": + y_pred = y_pred.swapaxes(1, 0).reshape(class_num, -1) + y = y.swapaxes(1, 0).reshape(class_num, -1) + + positives = y_pred.sum(axis=0) + true_positives = (y * y_pred).sum(axis=0) + + if self._type == "multilabel": + self._true_positives_average += np.sum(true_positives / (positives + self.eps)) + self._positives_average += len(positives) + self._true_positives = np.concatenate((self._true_positives, true_positives), axis=0) + self._positives = np.concatenate((self._positives, positives), axis=0) + + else: + self._true_positives += true_positives + self._positives += positives + + def eval(self, average=False): + """ + Computes the precision. + + Args: + average (bool): Specify whether calculate the average precision. Default value is False. + + Returns: + Float, the computed result. + """ + if self._class_num == 0: + raise RuntimeError('Input number of samples can not be 0.') + + validator.check_type("average", average, [bool]) + result = self._true_positives / (self._positives + self.eps) + + if average: + if self._type == "multilabel": + result = self._true_positives_average / (self._positives_average + self.eps) + return result.mean() + return result diff --git a/mindspore/nn/metrics/recall.py b/mindspore/nn/metrics/recall.py new file mode 100644 index 00000000000..2ea284ec41c --- /dev/null +++ b/mindspore/nn/metrics/recall.py @@ -0,0 +1,146 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Recall.""" +import sys + +import numpy as np + +from mindspore._checkparam import ParamValidator as validator +from .evaluation import EvaluationBase + + +class Recall(EvaluationBase): + r""" + Calculate recall for classification and multilabel data. + + The recall class creates two local variables, :math:`\text{true_positive}` and :math:`\text{false_negative}`, + that are used to compute the recall. This value is ultimately returned as the recall, an idempotent operation + that simply divides :math:`\text{true_positive}` by the sum of :math:`\text{true_positive}` and + :math:`\text{false_negative}`. + + .. math:: + \text{recall} = \frac{\text{true_positive}}{\text{true_positive} + \text{false_negative}} + + Note: + In the multi-label cases, the elements of :math:`y` and :math:`y_{pred}` should be 0 or 1. + + Args: + eval_type (str): Metric to calculate the recall over a dataset, for classification or + multilabel. Default: 'classification'. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + >>> y = mindspore.Tensor(np.array([1, 0, 1])) + >>> metric = nn.Recall('classification') + >>> metric.clear() + >>> metric.update(x, y) + >>> recall = metric.eval() + [1. 0.5] + """ + def __init__(self, eval_type='classification'): + super(Recall, self).__init__(eval_type) + self.eps = sys.float_info.min + self.clear() + + def clear(self): + """Clears the internal evaluation result.""" + self._class_num = 0 + if self._type == "multilabel": + self._true_positives = np.empty(0) + self._actual_positives = np.empty(0) + self._true_positives_average = 0 + self._actual_positives_average = 0 + else: + self._true_positives = 0 + self._actual_positives = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result with `y_pred` and `y`. + + Args: + inputs: Input `y_pred` and `y`. `y_pred` and `y` are a `Tensor`, a list or an array. + `y_pred` is in most cases (not strictly) a list of floating numbers in range :math:`[0, 1]` + and the shape is :math:`(N, C)`, where :math:`N` is the number of cases and :math:`C` + is the number of categories. For 'multilabel' evaluation type, `y_pred` can only be one-hot + encoding with values 0 or 1. Indices with 1 indicate positive category. `y` contains values + of integers. The shape is :math:`(N, C)` if one-hot encoding is used. One-hot encoding + should be used when 'eval_type' is 'multilabel'. Shape can also be :math:`(N, 1)` if category + index is used in 'classification' evaluation type. + + + Raises: + ValueError: If the number of input is not 2. + """ + if len(inputs) != 2: + raise ValueError('Recall need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + if self._type == 'classification' and y_pred.ndim == y.ndim and self._check_onehot_data(y): + y = y.argmax(axis=1) + self._check_shape(y_pred, y) + self._check_value(y_pred, y) + + if self._class_num == 0: + self._class_num = y_pred.shape[1] + elif y_pred.shape[1] != self._class_num: + raise ValueError('Class number not match, last input data contain {} classes, but current data contain {} ' + 'classes'.format(self._class_num, y_pred.shape[1])) + + class_num = self._class_num + if self._type == "classification": + if y.max() + 1 > class_num: + raise ValueError('y_pred contains {} classes less than y contains {} classes.'. + format(class_num, y.max() + 1)) + y = np.eye(class_num)[y.reshape(-1)] + indices = y_pred.argmax(axis=1).reshape(-1) + y_pred = np.eye(class_num)[indices] + elif self._type == "multilabel": + y_pred = y_pred.swapaxes(1, 0).reshape(class_num, -1) + y = y.swapaxes(1, 0).reshape(class_num, -1) + + actual_positives = y.sum(axis=0) + true_positives = (y * y_pred).sum(axis=0) + + if self._type == "multilabel": + self._true_positives_average += np.sum(true_positives / (actual_positives + self.eps)) + self._actual_positives_average += len(actual_positives) + self._true_positives = np.concatenate((self._true_positives, true_positives), axis=0) + self._actual_positives = np.concatenate((self._actual_positives, actual_positives), axis=0) + else: + self._true_positives += true_positives + self._actual_positives += actual_positives + + def eval(self, average=False): + """ + Computes the recall. + + Args: + average (bool): Specify whether calculate the average recall. Default value is False. + + Returns: + Float, the computed result. + """ + if self._class_num == 0: + raise RuntimeError('Input number of samples can not be 0.') + + validator.check_type("average", average, [bool]) + result = self._true_positives / (self._actual_positives + self.eps) + + if average: + if self._type == "multilabel": + result = self._true_positives_average / (self._actual_positives_average + self.eps) + return result.mean() + return result diff --git a/mindspore/nn/metrics/topk.py b/mindspore/nn/metrics/topk.py new file mode 100644 index 00000000000..6afa6319403 --- /dev/null +++ b/mindspore/nn/metrics/topk.py @@ -0,0 +1,128 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Topk.""" +import numpy as np +from .metric import Metric + + +class TopKCategoricalAccuracy(Metric): + """ + Calculates the top-k categorical accuracy. + + Note: + The method `update` must receive input of the form :math:`(y_{pred}, y)`. If some samples have + the same accuracy, the first sample will be chosen. + + Args: + k (int): Specifies the top-k categorical accuracy to compute. + + Raises: + TypeError: If `k` is not int. + ValueError: If `k` is less than 1. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5, 0.3, 0.6, 0.2], [0.1, 0.35, 0.5, 0.2, 0.], + >>> [0.9, 0.6, 0.2, 0.01, 0.3]]), mindspore.float32) + >>> y = mindspore.Tensor(np.array([2, 0, 1]), mindspore.float32) + >>> topk = nn.TopKCategoricalAccuracy(3) + >>> topk.clear() + >>> topk.update(x, y) + >>> result = topk.eval() + 0.6666666666666666 + """ + def __init__(self, k): + super(TopKCategoricalAccuracy, self).__init__() + if not isinstance(k, int): + raise TypeError('k should be integer type, but got {}'.format(type(k))) + if k < 1: + raise ValueError('k must be at least 1, but got {}'.format(k)) + self.k = k + self.clear() + + def clear(self): + """Clear the internal evaluation result.""" + self._correct_num = 0 + self._samples_num = 0 + + def update(self, *inputs): + """ + Updates the internal evaluation result y_pred and y. + + Args: + inputs: Input y_pred and y. y_pred and y are Tensor, list or numpy.ndarray. + y_pred is in most cases (not strictly) a list of floating numbers in range :math:`[0, 1]` + and the shape is :math:`(N, C)`, where :math:`N` is the number of cases and :math:`C` + is the number of categories. y contains values of integers. The shape is :math:`(N, C)` + if one-hot encoding is used. Shape can also be :math:`(N, 1)` if category index is used. + """ + if len(inputs) != 2: + raise ValueError('Topk need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + if y_pred.ndim == y.ndim and self._check_onehot_data(y): + y = y.argmax(axis=1) + indices = np.argsort(-y_pred, axis=1)[:, :self.k] + repeated_y = y.reshape(-1, 1).repeat(self.k, axis=1) + correct = np.equal(indices, repeated_y).sum(axis=1) + self._correct_num += correct.sum() + self._samples_num += repeated_y.shape[0] + + def eval(self): + """ + Computes the top-k categorical accuracy. + + Returns: + Float, computed result. + """ + if self._samples_num == 0: + raise RuntimeError('Total samples num must not be 0.') + return self._correct_num / self._samples_num + + +class Top1CategoricalAccuracy(TopKCategoricalAccuracy): + """ + Calculates the top-1 categorical accuracy. This class is a specialized class for TopKCategoricalAccuracy. + Refer to class 'TopKCategoricalAccuracy' for more details. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5, 0.3, 0.6, 0.2], [0.1, 0.35, 0.5, 0.2, 0.], + >>> [0.9, 0.6, 0.2, 0.01, 0.3]]), mindspore.float32) + >>> y = mindspore.Tensor(np.array([2, 0, 1]), mindspore.float32) + >>> topk = nn.Top1CategoricalAccuracy() + >>> topk.clear() + >>> topk.update(x, y) + >>> result = topk.eval() + """ + def __init__(self): + super(Top1CategoricalAccuracy, self).__init__(1) + + +class Top5CategoricalAccuracy(TopKCategoricalAccuracy): + """ + Calculates the top-5 categorical accuracy. This class is a specialized class for TopKCategoricalAccuracy. + Refer to class 'TopKCategoricalAccuracy' for more details. + + Examples: + >>> x = mindspore.Tensor(np.array([[0.2, 0.5, 0.3, 0.6, 0.2], [0.1, 0.35, 0.5, 0.2, 0.], + >>> [0.9, 0.6, 0.2, 0.01, 0.3]]), mindspore.float32) + >>> y = mindspore.Tensor(np.array([2, 0, 1]), mindspore.float32) + >>> topk = nn.Top5CategoricalAccuracy() + >>> topk.clear() + >>> topk.update(x, y) + >>> result = topk.eval() + """ + def __init__(self): + super(Top5CategoricalAccuracy, self).__init__(5) diff --git a/mindspore/nn/optim/__init__.py b/mindspore/nn/optim/__init__.py new file mode 100644 index 00000000000..6f7f6fbd46e --- /dev/null +++ b/mindspore/nn/optim/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Optimizer. + +Provide common optimizers for training, such as SGD, ADAM, Momentum. +The optimizer is used to calculate and update the gradients. +""" +from .optimizer import Optimizer +from .momentum import Momentum +from .adam import Adam, AdamWeightDecay, AdamWeightDecayDynamicLR +from .lamb import Lamb +from .sgd import SGD +from .lars import LARS +from .ftrl import FTRL + +__all__ = ['Optimizer', 'Momentum', 'LARS', 'Adam', 'AdamWeightDecay', + 'AdamWeightDecayDynamicLR', 'Lamb', 'SGD', 'FTRL'] diff --git a/mindspore/nn/optim/adam.py b/mindspore/nn/optim/adam.py new file mode 100755 index 00000000000..2c901ae0819 --- /dev/null +++ b/mindspore/nn/optim/adam.py @@ -0,0 +1,356 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""adam""" +from typing import Iterable +import numpy as np + +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.common.parameter import Parameter +from mindspore.common.tensor import Tensor +from mindspore._checkparam import ParamValidator as validator +from mindspore._checkparam import Rel +from .optimizer import Optimizer, apply_decay, grad_scale + +_learning_rate_update_func = ['linear', 'cos', 'sin'] + +adam_opt = C.MultitypeFuncGraph("adam_opt") + + +@adam_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") +def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, gradient): + """ + Update parameters. + + Args: + beta1 (Tensor): The exponential decay rate for the 1st moment estimates. Should be in range (0.0, 1.0). + beta2 (Tensor): The exponential decay rate for the 2nd moment estimates. Should be in range (0.0, 1.0). + eps (Tensor): Term added to the denominator to improve numerical stability. Should be greater than 0. + lr (Tensor): Learning rate. + weight_decay_tensor (Tensor): Weight decay. Should be equal to or greater than 0. + param (Tensor): Parameters. + m (Tensor): m value of parameters. + v (Tensor): v value of parameters. + gradient (Tensor): Gradient of parameters. + + Returns: + Tensor, the new value of v after updating. + """ + op_mul = P.Mul() + op_square = P.Square() + op_sqrt = P.Sqrt() + op_cast = P.Cast() + op_reshape = P.Reshape() + op_shape = P.Shape() + + param = op_cast(param, mstype.float32) + m = op_cast(m, mstype.float32) + v = op_cast(v, mstype.float32) + gradient = op_cast(gradient, mstype.float32) + + next_m = op_mul(beta1, m) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) - beta1, gradient) + + next_v = op_mul(beta2, v) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) - beta2, op_square(gradient)) + + update = next_m / (op_sqrt(next_v) + eps) + update = update + op_mul(weight_decay_tensor, param) + + update_with_lr = op_mul(lr, update) + next_param = param - op_reshape(update_with_lr, op_shape(param)) + + next_v = F.depend(next_v, F.assign(param, next_param)) + next_v = F.depend(next_v, F.assign(m, next_m)) + next_v = F.depend(next_v, F.assign(v, next_v)) + return next_v + + +def _check_param_value(beta1, beta2, eps, weight_decay): + """Check the type of inputs.""" + validator.check_type("beta1", beta1, [float]) + validator.check_type("beta2", beta2, [float]) + validator.check_type("eps", eps, [float]) + validator.check_type("weight_dacay", weight_decay, [float]) + validator.check_number_range("beta1", beta1, 0.0, 1.0, Rel.INC_NEITHER) + validator.check_number_range("beta2", beta2, 0.0, 1.0, Rel.INC_NEITHER) + validator.check_number_range("eps", eps, 0.0, float("inf"), Rel.INC_NEITHER) + validator.check_number_range("weight_decay", weight_decay, 0.0, float("inf"), Rel.INC_LEFT) + + +@adam_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Number", "Tensor", "Tensor", "Tensor", + "Tensor") +def _run_opt_with_one_number(opt, lr, beta1_power, beta2_power, beta1, beta2, eps, gradient, params, moment1, + moment2): + """Apply adam optimizer to the weight parameter using Tensor.""" + success = True + success = F.depend(success, opt(params, moment1, moment2, beta1_power, beta2_power, lr, beta1, beta2, + eps, gradient)) + return success + + +@adam_opt.register("Function", "Number", "Tensor", "Tensor", "Tensor", "Tensor", "Number", "Tensor", "Tensor", "Tensor", + "Tensor") +def _run_opt_with_two_number(opt, lr, beta1_power, beta2_power, beta1, beta2, eps, gradient, params, moment1, + moment2): + """Apply adam optimizer to the weight parameter using Tensor.""" + success = True + success = F.depend(success, opt(params, moment1, moment2, beta1_power, beta2_power, lr, beta1, beta2, + eps, gradient)) + return success + + +class Adam(Optimizer): + r""" + Updates gradients by Adaptive Moment Estimation (Adam) algorithm. + + The Adam algorithm is proposed in `Adam: A Method for Stochastic Optimization `_. + + The updating formulas are as follows, + + .. math:: + \begin{array}{ll} \\ + m = \beta_1 * m + (1 - \beta_1) * g \\ + v = \beta_2 * v + (1 - \beta_2) * g * g \\ + l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\ + w = w - l * \frac{m}{\sqrt{v} + \epsilon} + \end{array} + + :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`, + :math:`g` represents `gradients`, :math:`l` represents scaling factor `lr`, :math:`\beta_1, \beta_2` represent + `beta1` and `beta2`, :math:`t` represents updating step while :math:`beta_1^t` and :math:`beta_2^t` represent + `beta1_power` and `beta2_power`, :math:`\alpha` represents `learning_rate`, :math:`w` represents `params`, + :math:`\epsilon` represents `eps`. + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `params` + should be class mindspore.Parameter. + learning_rate (float): The Learning rate. + beta1 (float): The exponential decay rate for the 1st moment estimates. Should be in range (0.0, 1.0). + beta2 (float): The exponential decay rate for the 2nd moment estimates. Should be in range (0.0, 1.0). + eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. + use_locking (bool): Whether to enable a lock to protect updating variable tensors. + If True, updating of the var, m, and v tensors will be protected by a lock. + If False, the result is unpredictable. Default: False. + use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients. + If True, updates the gradients using NAG. + If False, updates the gradients without using NAG. Default: False. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + loss_scale (float): A floating point value for the loss scale. Default: 1.0. + Should be equal to or greater than 1. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. + + Outputs: + Tensor[bool], the value is True. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = Adam(params=net.trainable_params()) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + """ + + def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, + use_nesterov=False, weight_decay=0.0, loss_scale=1.0): + super(Adam, self).__init__(learning_rate, params) + _check_param_value(beta1, beta2, eps, weight_decay) + validator.check_type("use_locking", use_locking, [bool]) + validator.check_type("use_nesterov", use_nesterov, [bool]) + validator.check_type("loss_scale", loss_scale, [float]) + validator.check_number_range("loss_scale", loss_scale, 1.0, float("inf"), Rel.INC_LEFT) + + self.dynamic_lr = False + if isinstance(learning_rate, Iterable) or \ + (isinstance(learning_rate, Tensor) and learning_rate.dim() == 1): + self.dynamic_lr = True + self.gather = P.GatherV2() + self.assignadd = P.AssignAdd() + self.global_step = Parameter(initializer(0, [1], mstype.int32), name="global_step") + self.axis = 0 + + self.beta1 = Tensor(beta1, mstype.float32) + self.beta2 = Tensor(beta2, mstype.float32) + self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power") + self.beta2_power = Parameter(initializer(1, [1], mstype.float32), name="beta2_power") + self.eps = eps + + self.moment1 = self.parameters.clone(prefix="moment1", init='zeros') + self.moment2 = self.parameters.clone(prefix="moment2", init='zeros') + + self.hyper_map = C.HyperMap() + self.opt = P.Adam(use_locking, use_nesterov) + self.weight_decay = weight_decay * loss_scale + self.reciprocal_scale = 1.0 / loss_scale + + self.pow = P.Pow() + self.sqrt = P.Sqrt() + self.one = Tensor(np.array([1.0]).astype(np.float32)) + self.realdiv = P.RealDiv() + + def construct(self, gradients): + params = self.parameters + moment1 = self.moment1 + moment2 = self.moment2 + if self.weight_decay > 0: + gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_tf, params, gradients) + if self.reciprocal_scale != 1.0: + gradients = self.hyper_map(F.partial(grad_scale, self.reciprocal_scale), gradients) + + lr = self.learning_rate + if self.dynamic_lr: + lr = self.gather(self.learning_rate, self.global_step, self.axis) + F.control_depend(lr, self.assignadd(self.global_step, self.one)) + + beta1_power = self.beta1_power * self.beta1 + self.beta1_power = beta1_power + beta2_power = self.beta2_power * self.beta2 + self.beta2_power = beta2_power + success = self.hyper_map(F.partial(adam_opt, self.opt, lr, beta1_power, beta2_power, self.beta1, + self.beta2, self.eps), + gradients, params, moment1, moment2) + + return success + + +class AdamWeightDecay(Optimizer): + """ + Implements Adam algorithm weight decay fix. + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `params` + should be class mindspore.Parameter. + learning_rate (float): A floating point value for the learning rate. Default: 1e-3. + beta1 (float): The exponential decay rate for the 1st moment estimates. Default: 0.9. + Should be in range (0.0, 1.0). + beta2 (float): The exponential decay rate for the 2nd moment estimates. Default: 0.999. + Should be in range (0.0, 1.0). + eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6. + Should be greater than 0. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. + + Outputs: + tuple[Parameter], the updated velocity value, the shape is the same as `params`. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = AdamWeightDecay(params=net.trainable_params()) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + """ + def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-6, weight_decay=0.0): + super(AdamWeightDecay, self).__init__(learning_rate, params) + _check_param_value(beta1, beta2, eps, weight_decay) + self.lr = Tensor(np.array([learning_rate]).astype(np.float32)) + self.beta1 = Tensor(np.array([beta1]).astype(np.float32)) + self.beta2 = Tensor(np.array([beta2]).astype(np.float32)) + self.eps = Tensor(np.array([eps]).astype(np.float32)) + self.weight_decay_tensor = Tensor(np.array([weight_decay]).astype(np.float32)) + + self.params = self.parameters + self.moments1 = self.params.clone(prefix="adam_m", init='zeros') + self.moments2 = self.params.clone(prefix="adam_v", init='zeros') + + self.hyper_map = C.HyperMap() + + def construct(self, gradients): + updated_velocity = self.hyper_map(F.partial(adam_opt, self.beta1, self.beta2, self.eps, self.lr, + self.weight_decay_tensor), + self.params, self.moments1, self.moments2, gradients) + + return updated_velocity + + +class AdamWeightDecayDynamicLR(Optimizer): + """ + Adam Weight Decay Dynamic Learning Rate (LR). + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `params` + should be class mindspore.Parameter. + decay_steps (int): The steps of the decay. + learning_rate (float): A floating point value for the learning rate. Default: 0.001. + end_learning_rate (float): A floating point value for the end learning rate. Default: 0.0001. + power (float): Power. Default: 10.0. + beta1 (float): The exponential decay rate for the 1st moment estimates. Default: 0.9. + Should be in range (0.0, 1.0). + beta2 (float): The exponential decay rate for the 2nd moment estimates. Default: 0.999. + Should be in range (0.0, 1.0). + eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6. + Should be greater than 0. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. + + Outputs: + tuple[Parameter], the updated velocity value, the shape is the same as `params`. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = AdamWeightDecayDynamicLR(params=net.trainable_params(), decay_steps=10) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + """ + def __init__(self, + params, + decay_steps, + learning_rate=0.001, + end_learning_rate=0.0001, + power=10.0, + beta1=0.9, + beta2=0.999, + eps=1e-6, + weight_decay=0.0): + super(AdamWeightDecayDynamicLR, self).__init__(learning_rate, params) + _check_param_value(beta1, beta2, eps, weight_decay) + + # turn them to scalar when me support scalar/tensor mix operations + self.global_step = Parameter(initializer(0, [1]), name="global_step") + self.decay_steps = Tensor(np.array([decay_steps]).astype(np.float32)) + self.end_learning_rate = Tensor(np.array([end_learning_rate]).astype(np.float32)) + self.diff_learning_rate = Tensor(np.array([learning_rate - end_learning_rate]).astype(np.float32)) + self.power = power + self.beta1 = Tensor(np.array([beta1]).astype(np.float32)) + self.beta2 = Tensor(np.array([beta2]).astype(np.float32)) + self.eps = Tensor(np.array([eps]).astype(np.float32)) + self.weight_decay_tensor = Tensor(np.array([weight_decay]).astype(np.float32)) + self.params = self.parameters + self.moments1 = self.params.clone(prefix="adam_m", init='zeros') + self.moments2 = self.params.clone(prefix="adam_v", init='zeros') + + self.hyper_map = C.HyperMap() + self.min = P.Minimum() + self.pow = P.Pow() + self.one = Tensor(np.array([1.0]).astype(np.float32)) + + def construct(self, gradients): + step = self.min(self.global_step, self.decay_steps) + p = step / self.decay_steps + lr = self.diff_learning_rate * self.pow(self.one - p, self.power) + self.end_learning_rate + updated_velocity = self.hyper_map(F.partial(adam_opt, self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor), + self.params, self.moments1, self.moments2, gradients) + + added_global_step = self.global_step + self.one + F.control_depend(lr, added_global_step) + self.global_step = added_global_step + + return updated_velocity diff --git a/mindspore/nn/optim/ftrl.py b/mindspore/nn/optim/ftrl.py new file mode 100644 index 00000000000..3f4da483ea2 --- /dev/null +++ b/mindspore/nn/optim/ftrl.py @@ -0,0 +1,120 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""FTRL""" +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.common import Tensor +import mindspore.common.dtype as mstype +from mindspore._checkparam import ParamValidator as validator +from mindspore._checkparam import Rel +from .optimizer import Optimizer, apply_decay, grad_scale + +ftrl_opt = C.MultitypeFuncGraph("ftrl_opt") +@ftrl_opt.register("Function", "Number", "Number", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt(opt, learning_rate, l1, l2, lr_power, linear, gradient, weight, moment): + """Apply ftrl optimizer to the weight parameter.""" + success = True + success = F.depend(success, opt(weight, moment, linear, gradient, learning_rate, l1, l2, lr_power)) + return success + +def _check_param(initial_accum, learning_rate, lr_power, l1, l2, use_locking, loss_scale=1.0, weight_decay=0.0): + validator.check_type("initial_accum", initial_accum, [float]) + validator.check("initial_accum", initial_accum, "", 0.0, Rel.GE) + + validator.check_type("learning_rate", learning_rate, [float]) + validator.check("learning_rate", learning_rate, "", 0.0, Rel.GT) + + validator.check_type("lr_power", lr_power, [float]) + validator.check("lr_power", lr_power, "", 0.0, Rel.LE) + + validator.check_type("l1", l1, [float]) + validator.check("l1", l1, "", 0.0, Rel.GE) + + validator.check_type("l2", l2, [float]) + validator.check("l2", l2, "", 0.0, Rel.GE) + + validator.check_type("use_locking", use_locking, [bool]) + + validator.check_type("loss_scale", loss_scale, [float]) + validator.check("loss_scale", loss_scale, "", 1.0, Rel.GE) + + validator.check_type("weight_decay", weight_decay, [float]) + validator.check("weight_decay", weight_decay, "", 0.0, Rel.GE) + + +class FTRL(Optimizer): + """ + Implement the FTRL algorithm with ApplyFtrl Operator. + + FTRL is an online convex optimization algorithm that adaptively chooses its regularization function + based on the loss functions. Refer to paper `Adaptive Bound Optimization for Online Convex Optimization + `_. Refer to paper `Ad Click Prediction: a View from the Trenches + `_ for engineering document. + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `params` + should be Parameter. + initial_accum (float): The starting value for accumulators, must be zero or positive values. Default: 0.1. + learning_rate (float): The learning rate value, should be positive. Default: 0.001. + lr_power (float): Learning rate power controls how the learning rate decreases during training, must be less + than or equal to zero. Use fixed learning rate if lr_power is zero. Default: -0.5. + l1 (float): l1 regularization strength, must be greater than or equal to zero. Default: 0.0. + l2 (float): l2 regularization strength, must be greater than or equal to zero. Default: 0.0. + use_locking (bool): If True use locks for update operation. Default: False. + loss_scale (float): Value for the loss scale. It should be equal to or greater than 1.0. Default: 1.0. + wegith_decay (float): Weight decay value to multiply weight, must be zero or positive value. Default: 0.0. + + Inputs: + - **grads** (tuple[Tensor]) - The gradients of `params` in optimizer, the shape is as same as the `params` + in optimizer. + + Outputs: + tuple[Parameter], the updated parameters, the shape is the same as `params`. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> opt = FTRL(net.trainable_params()) + >>> model = Model(net, loss_fn=loss, optimizer=opt, metrics=None) + """ + def __init__(self, params, initial_accum=0.1, learning_rate=0.001, lr_power=-0.5, l1=0.0, l2=0.0, + use_locking=False, loss_scale=1.0, weight_decay=0.0): + super(FTRL, self).__init__(learning_rate, params) + + _check_param(initial_accum, learning_rate, lr_power, l1, l2, use_locking, loss_scale, weight_decay) + self.moments = self.parameters.clone(prefix="moments", init=initial_accum) + self.linear = self.parameters.clone(prefix="linear", init='zeros') + self.l1 = l1 + self.l2 = l2 + self.lr_power = lr_power + self.reciprocal_scale = 1.0 / loss_scale + self.weight_decay = weight_decay + self.decay_tf = tuple((lambda:True)() for x in self.parameters) + self.hyper_map = C.HyperMap() + self.opt = P.ApplyFtrl(use_locking=use_locking) + self.one = Tensor(1, mstype.int32) + + def construct(self, grads): + params = self.parameters + moments = self.moments + linear = self.linear + if self.weight_decay > 0.0: + grads = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_tf, params, grads) + if self.reciprocal_scale != 1.0: + grads = self.hyper_map(F.partial(grad_scale, self.reciprocal_scale), grads) + lr = self.learning_rate + success = self.hyper_map(F.partial(ftrl_opt, self.opt, lr, self.l1, self.l2, self.lr_power), linear, grads, params, moments) + return success diff --git a/mindspore/nn/optim/lamb.py b/mindspore/nn/optim/lamb.py new file mode 100755 index 00000000000..e4fd3bf1d7e --- /dev/null +++ b/mindspore/nn/optim/lamb.py @@ -0,0 +1,232 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""lamb""" +import numpy as np +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.common.parameter import Parameter +from mindspore.common.tensor import Tensor +from mindspore._checkparam import ParamValidator as validator +from mindspore._checkparam import Rel +from .optimizer import Optimizer +from .. import layer + +num_one = Tensor(np.ones([1]), mstype.float32) + +lamb_opt = C.MultitypeFuncGraph("lamb_opt") + +@lamb_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", + "Tensor", "Bool") +def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, global_step, param, m, v, + gradient, decay_flag): + """ + Update parameters. + + Args: + beta1 (Tensor): The exponential decay rate for the 1st moment estimates. Should be in range (0.0, 1.0). + beta2 (Tensor): The exponential decay rate for the 2nd moment estimates. Should be in range (0.0, 1.0). + eps (Tensor): Term added to the denominator to improve numerical stability. Should be greater than 0. + lr (Tensor): Learning rate. + weight_decay_tensor (Tensor): Weight decay. Should be equal to or greater than 0. + global_step (Tensor): Global step. + param (Tensor): Parameters. + m (Tensor): m value of parameters. + v (Tensor): v value of parameters. + gradient (Tensor): Gradient of parameters. + decay_flag (bool): Specifies whether param update with weight decay. + + Returns: + Tensor, the new value of v after updating. + """ + op_mul = P.Mul() + op_sqrt = P.Sqrt() + op_rsqrt = P.Rsqrt() + op_square = P.Square() + op_cast = P.Cast() + op_reshape = P.Reshape() + op_shape = P.Shape() + op_pow = P.Pow() + op_norm = layer.Norm() + op_select = P.Select() + op_greater = P.Greater() + op_fill = P.Fill() + op_dtype = P.DType() + + param = op_cast(param, mstype.float32) + m = op_cast(m, mstype.float32) + v = op_cast(v, mstype.float32) + gradient = op_cast(gradient, mstype.float32) + + next_m = op_mul(beta1, m) + op_mul(op_cast(num_one, mstype.float32) - beta1, gradient) + + next_v = op_mul(beta2, v) + op_mul(op_cast(num_one, mstype.float32) - beta2, op_square(gradient)) + + next_mm = next_m / (op_cast(num_one, mstype.float32) + - op_pow(beta1, op_cast(global_step + num_one, mstype.float32))) + next_vv = next_v / (op_cast(num_one, mstype.float32) - + op_pow(beta2, op_cast(global_step + num_one, mstype.float32))) + w_norm = op_norm(param) + g_norm = op_norm(gradient) + + g_norm_hat = op_norm(op_mul(next_mm, op_rsqrt(next_vv + eps)) + weight_decay_tensor * param) + zeros = F.zeros_like_tensor(w_norm) + ones = op_fill(op_dtype(w_norm), op_shape(w_norm), 1.0) + trust_ratio = op_select( + op_greater(w_norm, zeros), + op_select(op_greater(g_norm, zeros), w_norm / g_norm_hat, ones), + ones) + tens = op_fill(op_dtype(trust_ratio), op_shape(trust_ratio), 10.0) + trust_ratio = C.clip_by_value(trust_ratio, zeros, tens) + update = next_mm / (op_sqrt(next_vv) + eps) + + if decay_flag: + update = update + op_mul(weight_decay_tensor, param) + + update_with_lr = op_mul(op_mul(trust_ratio, lr), update) + + next_param = param - op_reshape(update_with_lr, op_shape(param)) + + next_v = F.depend(next_v, F.assign(param, next_param)) + next_v = F.depend(next_v, F.assign(m, next_m)) + next_v = F.depend(next_v, F.assign(v, next_v)) + + return next_v + + +def _check_param_value(decay_steps, warmup_steps, start_learning_rate, + end_learning_rate, power, beta1, beta2, eps, weight_decay): + + """Check the type of inputs.""" + validator.check_type("decay_steps", decay_steps, [int]) + validator.check_type("warmup_steps", warmup_steps, [int]) + validator.check_type("start_learning_rate", start_learning_rate, [float]) + validator.check_type("end_learning_rate", end_learning_rate, [float]) + validator.check_type("power", power, [float]) + validator.check_type("beta1", beta1, [float]) + validator.check_type("beta2", beta2, [float]) + validator.check_type("eps", eps, [float]) + validator.check_type("weight_dacay", weight_decay, [float]) + validator.check_number_range("decay_steps", decay_steps, 1, float("inf"), Rel.INC_LEFT) + validator.check_number_range("beta1", beta1, 0.0, 1.0, Rel.INC_NEITHER) + validator.check_number_range("beta2", beta2, 0.0, 1.0, Rel.INC_NEITHER) + validator.check_number_range("eps", eps, 0.0, float("inf"), Rel.INC_NEITHER) + validator.check_number_range("weight_decay", weight_decay, 0.0, float("inf"), Rel.INC_LEFT) + + +class Lamb(Optimizer): + """ + Lamb Dynamic LR. + + LAMB is an optimization algorithm employing a layerwise adaptive large batch + optimization technique. Refer to the paper `LARGE BATCH OPTIMIZATION FOR DEEP LEARNING: TRAINING BERT IN 76 + MINUTES `_. + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `params` + should be class mindspore.Parameter. + decay_steps (int): The steps of the lr decay. Should be equal to or greater than 1. + warmup_steps (int): The steps of lr warm up. Default: 0. + start_learning_rate (float): A floating point value for the learning rate. Default: 0.1. + end_learning_rate (float): A floating point value for the end learning rate. Default: 0.0001. + power (float): The power of the polynomial. Default: 1.0. + beta1 (float): The exponential decay rate for the 1st moment estimates. Default: 0.9. + Should be in range (0.0, 1.0). + beta2 (float): The exponential decay rate for the 2nd moment estimates. Default: 0.999. + Should be in range (0.0, 1.0). + eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6. + Should be greater than 0. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. Should be equal to or greater than 0. + decay_filter (Function): A function to determine whether to apply weight decay on parameters. Default: + lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. + + Outputs: + tuple[Parameter], the updated velocity value, the shape is the same as `params`. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = Lamb(params=net.trainable_params(), decay_steps=10) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + """ + + def __init__(self, + params, + decay_steps, + warmup_steps=0, + start_learning_rate=0.1, + end_learning_rate=0.0001, + power=1.0, + beta1=0.9, + beta2=0.999, + eps=1e-6, + weight_decay=0.0, + decay_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name): + + super(Lamb, self).__init__(start_learning_rate, params) + _check_param_value(decay_steps, warmup_steps, start_learning_rate, end_learning_rate, + power, beta1, beta2, eps, weight_decay) + + # turn them to scalar when me support scalar/tensor mix operations + self.global_step = Parameter(initializer(0, [1]), name="global_step") + + self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32)) + self.warmup_flag = False + if warmup_steps > 0: + self.warmup_flag = True + self.decay_steps = Tensor(np.array([decay_steps]).astype(np.float32)) + self.start_learning_rate = Tensor(np.array([start_learning_rate]).astype(np.float32)) + self.end_learning_rate = Tensor(np.array([end_learning_rate]).astype(np.float32)) + self.diff_learning_rate = Tensor(np.array([start_learning_rate - end_learning_rate]).astype(np.float32)) + self.power = power + self.beta1 = Tensor(np.array([beta1]).astype(np.float32)) + self.beta2 = Tensor(np.array([beta2]).astype(np.float32)) + self.eps = Tensor(np.array([eps]).astype(np.float32)) + self.weight_decay_tensor = Tensor(np.array([weight_decay]).astype(np.float32)) + self.params = self.parameters + self.moments1 = self.params.clone(prefix="lamb_m", init='zeros') + self.moments2 = self.params.clone(prefix="lamb_v", init='zeros') + self.decay_flag = tuple(decay_filter(x) for x in self.params) + + self.hyper_map = C.HyperMap() + self.min = P.Minimum() + self.pow = P.Pow() + self.greater = P.Greater() + self.one = Tensor(np.array([1.0]).astype(np.float32)) + self.cast = P.Cast() + + def construct(self, gradients): + step = self.min(self.global_step, self.decay_steps) + p = step / self.decay_steps + lr = self.diff_learning_rate * self.pow(self.one - p, self.power) + self.end_learning_rate + if self.warmup_flag: + warmup_percent = self.global_step / self.warmup_steps + warmup_lr = self.start_learning_rate * warmup_percent + is_warmup = self.cast(self.greater(self.warmup_steps, self.global_step), mstype.float32) + lr = (self.one - is_warmup) * lr + is_warmup * warmup_lr + updated_velocity = self.hyper_map(F.partial(lamb_opt, self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor, self.global_step), + self.params, self.moments1, self.moments2, gradients, self.decay_flag) + + added_global_step = self.global_step + self.one + F.control_depend(lr, added_global_step) + self.global_step = added_global_step + + return updated_velocity diff --git a/mindspore/nn/optim/lars.py b/mindspore/nn/optim/lars.py new file mode 100755 index 00000000000..cdfe45de629 --- /dev/null +++ b/mindspore/nn/optim/lars.py @@ -0,0 +1,135 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""lars optimizer""" +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.common.parameter import Parameter +from mindspore.nn.cell import Cell +from .optimizer import grad_scale + +lars_opt = C.MultitypeFuncGraph("lars_opt") + + +@lars_opt.register("Function", "Number", "Tensor", "Tensor", "Tensor", "Bool", "Bool") +def _tensor_run_opt(lars, weight_decay, learning_rate, gradient, weight, decay_flag, lars_flag): + """Apply lars optimizer to the weight parameter.""" + if lars_flag: + op_reduce = P.ReduceSum() + w_square_sum = op_reduce(F.square(weight)) + grad_square_sum = op_reduce(F.square(gradient)) + if decay_flag: + grad_t = lars(weight, gradient, w_square_sum, grad_square_sum, weight_decay, learning_rate) + else: + num_zero = 0.0 + grad_t = lars(weight, gradient, w_square_sum, grad_square_sum, num_zero, learning_rate) + return grad_t + + return gradient + + +@lars_opt.register("Function", "Number", "Number", "Tensor", "Tensor", "Bool", "Bool") +def _tensor_run_opt_v2(lars, weight_decay, learning_rate, gradient, weight, decay_flag, lars_flag): + """Apply lars optimizer to the weight parameter.""" + if lars_flag: + op_reduce = P.ReduceSum() + w_square_sum = op_reduce(F.square(weight)) + grad_square_sum = op_reduce(F.square(gradient)) + if decay_flag: + grad_t = lars(weight, gradient, w_square_sum, grad_square_sum, weight_decay, learning_rate) + else: + num_zero = 0.0 + grad_t = lars(weight, gradient, w_square_sum, grad_square_sum, num_zero, learning_rate) + return grad_t + + return gradient + + +class LARS(Cell): + """ + Implements the LARS algorithm with LARSUpdate Operator. + + LARS is an optimization algorithm employing a large batch optimization technique. Refer to paper `LARGE BATCH + TRAINING OF CONVOLUTIONAL NETWORKS `_. + + Args: + optimizer (Optimizer): MindSpore optimizer for which to wrap and modify gradients. + epsilon (float): Term added to the denominator to improve numerical stability. Default: 1e-05. + hyperpara (float): Trust coefficient for calculating the local learning rate. Default: 0.001. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: False. + decay_filter (Function): A function to determine whether apply weight decay on parameters. Default: + lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. + lars_filter (Function): A function to determine whether apply lars algorithm. Default: + lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. + loss_scale (float): A floating point value for the loss scale. Default: 1.0. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params` in optimizer, the shape is + as same as the `params` in optimizer. + + Outputs: + Union[Tensor[bool], tuple[Parameter]], it depends on the output of `optimizer`. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> opt = Momentum(net.trainable_params(), 0.1, 0.9) + >>> opt_lars = LARS(opt, epsilon=1e-08, hyperpara=0.02) + >>> model = Model(net, loss_fn=loss, optimizer=opt_lars, metrics=None) + """ + + def __init__(self, optimizer, epsilon=1e-05, hyperpara=0.001, weight_decay=0.0, use_clip=False, + decay_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name, + lars_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name, loss_scale=1.0): + super(LARS, self).__init__(auto_prefix=False) + self.opt = optimizer + self.parameters = optimizer.parameters + self.learning_rate = optimizer.learning_rate + self.lars = P.LARSUpdate(epsilon, hyperpara, use_clip) + self.reciprocal_scale = 1.0 / loss_scale + self.weight_decay = weight_decay * loss_scale + self.cast = P.Cast() + self.decay_flag = tuple(decay_filter(x) for x in self.parameters) + self.lars_flag = tuple(lars_filter(x) for x in self.parameters) + self.hyper_map = C.HyperMap() + self.dynamic_lr = False + self.gather = None + self.global_step = None + self.axis = None + if not isinstance(self.learning_rate, float): + self.dynamic_lr = True + self.assignadd = P.AssignAdd() + self.gather = P.GatherV2() + self.global_step = Parameter(initializer(0, [1], mstype.int32), name="lars_global_step") + self.axis = 0 + + def construct(self, gradients): + params = self.parameters + if self.dynamic_lr: + lr = self.gather(self.learning_rate, self.global_step, self.axis) + F.control_depend(lr, self.assignadd(self.global_step, 1)) + else: + lr = F.scalar_to_array(self.learning_rate) + if self.reciprocal_scale != 1.0: + gradients = self.hyper_map(F.partial(grad_scale, self.reciprocal_scale), gradients) + + grad_t = self.hyper_map(F.partial(lars_opt, self.lars, self.weight_decay, lr), + gradients, params, self.decay_flag, self.lars_flag) + success = self.opt(grad_t) + + return success diff --git a/mindspore/nn/optim/momentum.py b/mindspore/nn/optim/momentum.py new file mode 100755 index 00000000000..2cc6d76a86e --- /dev/null +++ b/mindspore/nn/optim/momentum.py @@ -0,0 +1,130 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""momentum""" +from typing import Iterable + +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import mindspore.common.dtype as mstype +from mindspore.common import Tensor +from .optimizer import Optimizer, apply_decay, grad_scale + +momentum_opt = C.MultitypeFuncGraph("momentum_opt") + + +@momentum_opt.register("Function", "Number", "Number", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt(opt, learning_rate, momentum, gradient, weight, moment): + """Apply momentum optimizer to the weight parameter.""" + success = True + success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) + return success + + +@momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, moment): + """Apply momentum optimizer to the weight parameter using Tensor.""" + success = True + success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) + return success + + +@momentum_opt.register("Function", "Tensor", "Number", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt_dyn(opt, learning_rate, momentum, gradient, weight, moment): + """Apply momentum optimizer to the weight parameter using dynamic learning rate.""" + success = True + success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) + return success + + +class Momentum(Optimizer): + """ + Implements the Momentum algorithm. + + Refer to the paper on the importance of initialization and momentum in deep learning for more details. + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `parameters` + should be class mindspore.Parameter. + learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is + Iterable or a Tensor and the dims of the Tensor is 1, + use dynamic learning rate, then the i-th step will + take the i-th value as the learning rate. + When the learning_rate is float or learning_rate is a Tensor + but the dims of the Tensor is 0, use fixed learning rate. + Other cases are not supported. + momentum (float): Hyperparameter of type float, means momentum for the moving average. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + loss_scale (float): A floating point value for the loss scale. Default: 1.0. + decay_filter (Function): A function to determine whether to apply weight decay on parameters. Default: + lambda x: 'beta' not in x.name and 'gamma' not in x.name. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. + + Outputs: + Tensor[bool], the value is True. + + Raises: + ValueError: If the momentum is less than 0.0. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + """ + def __init__(self, params, learning_rate, momentum, weight_decay=0.0, loss_scale=1.0, + decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): + super(Momentum, self).__init__(learning_rate, params) + if isinstance(momentum, float) and momentum < 0.0: + raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum)) + if isinstance(learning_rate, Iterable) or \ + (isinstance(learning_rate, Tensor) and learning_rate.dim() == 1): + self.dynamic_lr = True + self.gather = P.GatherV2() + self.assignadd = P.AssignAdd() + self.global_step = Parameter(initializer(0, [1], mstype.int32), name="global_step") + self.axis = 0 + else: + self.dynamic_lr = False + self.gather = None + self.assignadd = None + self.global_step = None + self.axis = None + self.momentum = Parameter(momentum, name="momentum") + self.params = self.parameters + self.moments = self.params.clone(prefix="moments", init='zeros') + self.decay_tf = tuple(decay_filter(x) for x in self.parameters) + self.hyper_map = C.HyperMap() + self.opt = P.ApplyMomentum() + self.weight_decay = weight_decay * loss_scale + self.reciprocal_scale = 1.0 / loss_scale + self.one = Tensor(1, mstype.int32) + + def construct(self, gradients): + params = self.params + moments = self.moments + if self.weight_decay > 0: + gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_tf, params, gradients) + if self.reciprocal_scale != 1.0: + gradients = self.hyper_map(F.partial(grad_scale, self.reciprocal_scale), gradients) + if self.dynamic_lr: + lr = self.gather(self.learning_rate, self.global_step, self.axis) + F.control_depend(lr, self.assignadd(self.global_step, self.one)) + else: + lr = self.learning_rate + success = self.hyper_map(F.partial(momentum_opt, self.opt, lr, self.momentum), gradients, params, moments) + return success diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py new file mode 100755 index 00000000000..e2b0cddb711 --- /dev/null +++ b/mindspore/nn/optim/optimizer.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""optimizer""" +from typing import Iterable +import logging + +import numpy as np + +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.nn.cell import Cell +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore._checkparam import ParamValidator as validator +from mindspore._checkparam import Rel +from mindspore.common.tensor import Tensor + +logger = logging.getLogger('Optimizer') + +__all__ = ['Optimizer'] + + +class Optimizer(Cell): + """ + Base class for all optimizers. + + This class defines the API to add Ops to train a model. + + Note: + This class defines the API to add Ops to train a model. Never use + this class directly, but instead instantiate one of its subclasses. + + Args: + learning_rate (float): A floating point value for the learning rate. Should be greater than 0. + parameters (list): A list of parameter, which will be updated. The element in `parameters` + should be class mindspore.Parameter. + + Raises: + ValueError: If the learning_rate is a Tensor, but the dims of tensor is greater than 1. + TypeError: If the learning_rate is not any of the three types: float, Tensor, Iterable. + """ + + def __init__(self, learning_rate, parameters): + super(Optimizer, self).__init__() + if isinstance(learning_rate, float): + validator.check_number_range("learning rate", learning_rate, 0.0, float("inf"), Rel.INC_LEFT) + elif isinstance(learning_rate, Iterable): + learning_rate = Tensor(np.array(list(learning_rate)).astype(np.float32)) + elif isinstance(learning_rate, Tensor): + if learning_rate.dim() > 1: + raise ValueError("Learning rate should be a 0 or 1 dim `Tensor`," + f"but got {learning_rate.dim()}.") + else: + raise TypeError("Learning rate should be float, Tensor or Iterable.") + + if isinstance(learning_rate, Tensor) and learning_rate.dim() == 1 and learning_rate.size() < 2: + logger.warning("If want to use the dynamic learning rate, please make sure that " + "the number of elements in the list, tuple or tensor passed is greater than 1.") + self.learning_rate = Parameter(learning_rate, name="learning_rate") + self.parameters = ParameterTuple(parameters) + if not self.parameters: + raise ValueError("optimizer got an empty parameter list.") + + def construct(self, *hyper_params): + raise NotImplementedError + + +op_add = P.AddN() + +apply_decay = C.MultitypeFuncGraph("apply_decay") + + +@apply_decay.register("Number", "Bool", "Tensor", "Tensor") +def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): + """Get grad with weight_decay.""" + if if_apply: + return op_add((gradient, weight * F.scalar_to_array(weight_decay))) + return gradient + + +grad_scale = C.MultitypeFuncGraph("grad_scale") + + +@grad_scale.register("Number", "Tensor") +def tensor_grad_scale(scale, grad): + """Get grad with scale.""" + if scale == 1.0: + return grad + cast_op = P.Cast() + type_op = P.DType() + return grad * cast_op(F.scalar_to_array(scale), type_op(grad)) diff --git a/mindspore/nn/optim/sgd.py b/mindspore/nn/optim/sgd.py new file mode 100755 index 00000000000..92e9a11070c --- /dev/null +++ b/mindspore/nn/optim/sgd.py @@ -0,0 +1,135 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""sgd""" +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore._checkparam import ParamValidator as validator +import mindspore.common.dtype as mstype +from .optimizer import Optimizer, grad_scale + +sgd_opt = C.MultitypeFuncGraph("sgd_opt") + + +@sgd_opt.register("Function", "Number", "Number", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt(opt, learning_rate, momentum, gradient, weight, accum, stat): + """Apply sgd optimizer to the weight parameter.""" + success = True + success = F.depend(success, opt(weight, gradient, learning_rate, accum, momentum, stat)) + return success + + +@sgd_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, accum, stat): + """Apply sgd optimizer to the weight parameter using Tensor.""" + success = True + success = F.depend(success, opt(weight, gradient, learning_rate, accum, momentum, stat)) + return success + + +@sgd_opt.register("Function", "Tensor", "Number", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt_dyn(opt, learning_rate, momentum, gradient, weight, accum, stat): + """Apply sgd optimizer to the weight parameter using dynamic learning rate.""" + success = True + success = F.depend(success, opt(weight, gradient, learning_rate, accum, momentum, stat)) + return success + + +class SGD(Optimizer): + """ + Implements stochastic gradient descent (optionally with momentum). + + Introduction to SGD can be found at https://en.wikipedia.org/wiki/Stochastic_gradient_descent. + Nesterov momentum is based on the formula from paper `On the importance of initialization and + momentum in deep learning `_. + + Args: + params (list[Parameter]): A list of parameter, which will be updated. The element in `params` + should be class mindspore.Parameter. + learning_rate (float): A floating point value for the learning rate. Default: 0.1. + momentum (float): A floating point value the momentum. Default: 0. + dampening (float): A floating point value of dampening for momentum. Default: 0. + weight_decay (float): Weight decay (L2 penalty). Default: 0. + nesterov (bool): Enables the Nesterov momentum. Default: False. + loss_scale (float): A floating point value for the loss scale. Default: 1.0. + + Inputs: + - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. + + Outputs: + Tensor[bool], the value is True. + + Raises: + ValueError: If the momentum, dampening or weight_decay value is less than 0.0. + + Examples: + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = SGD(params=net.trainable_params()) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + """ + def __init__(self, params, learning_rate=0.1, momentum=0.0, dampening=0.0, weight_decay=0.0, nesterov=False, + loss_scale=1.0): + + super(SGD, self).__init__(learning_rate, params) + + if isinstance(momentum, float) and momentum < 0.0: + raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum)) + + if dampening < 0.0: + raise ValueError("dampening should be at least 0.0, but got dampening {}".format(dampening)) + self.dampening = dampening + + if weight_decay < 0.0: + raise ValueError("weight_decay should be at least 0.0, but got weight_decay {}".format(weight_decay)) + self.weight_decay = weight_decay + + validator.check_type("nesterov", nesterov, [bool]) + self.nesterov = nesterov + + self.opt = P.SGD(dampening, weight_decay, nesterov) + + self.dynamic_lr = False + self.gather = None + self.global_step = None + self.axis = None + if not isinstance(learning_rate, float): + self.dynamic_lr = True + self.gather = P.GatherV2() + self.assignadd = P.AssignAdd() + self.global_step = Parameter(initializer(0, [1], mstype.int32), name="global_step") + self.axis = 0 + self.momentum = Parameter(momentum, name="momentum") + self.params = self.parameters + self.accum = self.params.clone(prefix="accum", init='zeros') + self.stat = self.params.clone(prefix="stat", init='ones') + self.hyper_map = C.HyperMap() + + self.weight_decay = weight_decay * loss_scale + self.reciprocal_scale = 1.0 / loss_scale + + def construct(self, gradients): + params = self.params + accum = self.accum + stat = self.stat + if self.reciprocal_scale != 1.0: + gradients = self.hyper_map(F.partial(grad_scale, self.reciprocal_scale), gradients) + if self.dynamic_lr: + lr = self.gather(self.learning_rate, self.global_step, self.axis) + F.control_depend(lr, self.assignadd(self.global_step, 1)) + else: + lr = self.learning_rate + success = self.hyper_map(F.partial(sgd_opt, self.opt, lr, self.momentum), gradients, params, accum, stat) + return success diff --git a/mindspore/nn/wrap/__init__.py b/mindspore/nn/wrap/__init__.py new file mode 100644 index 00000000000..a07fc51a1f1 --- /dev/null +++ b/mindspore/nn/wrap/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Wrap cells for networks. + +Use the Wrapper to combine the loss or build the training steps. +""" +from .cell_wrapper import TrainOneStepCell, WithLossCell, WithGradCell, WithEvalCell, DataWrapper, \ + ParameterUpdate, GetNextSingleOp +from .loss_scale import TrainOneStepWithLossScaleCell, DynamicLossScaleUpdateCell, FixedLossScaleUpdateCell +from .grad_reducer import DistributedGradReducer + +__all__ = [ + "TrainOneStepCell", + "WithLossCell", + "WithGradCell", + "WithEvalCell", + "DataWrapper", + "GetNextSingleOp", + "TrainOneStepWithLossScaleCell", + "DistributedGradReducer", + "ParameterUpdate", + "DynamicLossScaleUpdateCell", + "FixedLossScaleUpdateCell" + ] diff --git a/mindspore/nn/wrap/cell_wrapper.py b/mindspore/nn/wrap/cell_wrapper.py new file mode 100644 index 00000000000..efdfc9367e3 --- /dev/null +++ b/mindspore/nn/wrap/cell_wrapper.py @@ -0,0 +1,387 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Cell_wrapper.""" +import copy +import numpy as np +from mindspore.train.parallel_utils import ParallelMode +from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_mirror_mean +from ...ops import composite as C, functional as F, operations as P +from ...common import Tensor, dtype as mstype +from ..cell import Cell +from ...common.initializer import initializer +from ...common.parameter import Parameter, ParameterTuple +from ...ops.operations.comm_ops import _VirtualDataset +from .grad_reducer import DistributedGradReducer + + +class WithLossCell(Cell): + r""" + Cell with loss function. + + Wraps the network with loss function. This Cell accepts data and label as inputs and + the computed loss will be returned. + + Args: + backbone (Cell): The target network to wrap. + loss_fn (Cell): The loss function used to compute loss. + + Inputs: + - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + + Outputs: + Tensor, a scalar tensor with shape :math:`()`. + + Examples: + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + >>> net_with_criterion = nn.WithLossCell(net, loss_fn) + >>> + >>> batch_size = 2 + >>> data = mindspore.Tensor(np.ones([batch_size, 3, 64, 64]).astype(np.float32) * 0.01) + >>> label = mindspore.Tensor(np.ones([batch_size, 1, 1, 1]).astype(np.int32)) + >>> + >>> net_with_criterion(data, label) + """ + + def __init__(self, backbone, loss_fn): + super(WithLossCell, self).__init__(auto_prefix=False) + self._backbone = backbone + self._loss_fn = loss_fn + + def construct(self, data, label): + """ + Computes loss based on the wrapped loss cell. + + Args: + data (Tensor): Tensor data to train. + label (Tensor): Tensor label data. + + Returns: + Tensor, compute result. + """ + out = self._backbone(data) + return self._loss_fn(out, label) + + @property + def backbone_network(self): + """ + Returns the backbone network. + + Returns: + Cell, the backbone network. + """ + return self._backbone + + +class WithGradCell(Cell): + r""" + Cell that returns the gradients. + + Wraps the network with backward cell to compute gradients. A network with a loss function is necessary + as argument. If loss function in None, the network must be a wrapper of network and loss function. This + Cell accepts data and label as inputs and returns gradients for each trainable parameter. + + Note: + Run in PyNative mode. + + Args: + network (Cell): The target network to wrap. + loss_fn (Cell): Primitive loss function used to compute gradients. Default: None. + sens (Union[None, Tensor, Scalar, Tuple ...]): The sensitive for backpropagation, the type and shape + should be same as the `network` output. If None, we will fill one to a same type shape of + output value. Default: None. + + Inputs: + - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + + Outputs: + list, a list of Tensors with identical shapes as trainable weights. + + Examples: + >>> # For a defined network Net without loss function + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> grad_net = nn.WithGradCell(net, loss_fn) + >>> + >>> # For a network wrapped with loss function + >>> net = Net() + >>> net_with_criterion = nn.WithLossCell(net, loss_fn) + >>> grad_net = nn.WithGradCell(net_with_criterion) + """ + + def __init__(self, network, loss_fn=None, sens=None): + super(WithGradCell, self).__init__(auto_prefix=False) + self.network = network + self.loss_fn = loss_fn + self.weights = ParameterTuple(network.trainable_params()) + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=(sens is not None)) + self.sens = sens + if loss_fn is None: + self.network_with_loss = network + else: + self.network_with_loss = WithLossCell(self.network, self.loss_fn) + self.network_with_loss.set_train() + + def construct(self, data, label): + """ + Computes gradients based on the wrapped gradients cell. + + Note: + Run in PyNative mode. + + Args: + data (Tensor): Tensor data to train. + label (Tensor): Tensor label data. + + Returns: + Tensor, return compute gradients. + """ + weights = self.weights + if self.sens is None: + grads = self.grad(self.network_with_loss, weights)(data, label) + else: + grads = self.grad(self.network_with_loss, weights)(data, label, self.sens) + return grads + + +class TrainOneStepCell(Cell): + r""" + Network training package class. + + Wraps the network with an optimizer. The resulting Cell be trained with input data and label. + Backward graph will be created in the construct function to do parameter updating. Different + parallel modes are available to run the training. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. + + Inputs: + - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + + Outputs: + Tensor, a scalar Tensor with shape :math:`()`. + + Examples: + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> loss_net = nn.WithLossCell(net, loss_fn) + >>> train_net = nn.TrainOneStepCell(loss_net, optim) + """ + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.grad_reducer = None + parallel_mode = _get_parallel_mode() + if parallel_mode in (ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL): + self.reducer_flag = True + if self.reducer_flag: + mean = _get_mirror_mean() + degree = _get_device_num() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + + def construct(self, data, label): + weights = self.weights + loss = self.network(data, label) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(data, label, sens) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + return F.depend(loss, self.optimizer(grads)) + + +class DataWrapper(Cell): + """ + Network training package class for dataset. + + DataWrapper wraps the input network with a dataset which automatically fetches data with 'GetNext' + function from the dataset channel 'queue_name' and does forward computation in the construct function. + + Args: + network (Cell): The training network for dataset. + dataset_types (list): The type of dataset. The list contains describes the types of the inputs. + dataset_shapes (list): The shapes of dataset. The list contains multiple sublists that describes + the shape of the inputs. + queue_name (str): The identification of dataset channel which specifies the dataset channel to supply + data for the network. + + Outputs: + Tensor, network output whose shape depends on the network. + + Examples: + >>> # call create_dataset function to create a regular dataset, refer to mindspore.dataset + >>> train_dataset = create_dataset() + >>> dataset_helper = mindspore.DatasetHelper(train_dataset) + >>> net = Net() + >>> net = DataWrapper(net, *(dataset_helper.types_shapes()), train_dataset.queue_name) + """ + + def __init__(self, network, dataset_types, dataset_shapes, queue_name): + super(DataWrapper, self).__init__(auto_prefix=False) + + self.get_next = P.GetNext(dataset_types, dataset_shapes, len(dataset_types), queue_name) + self.network = network + + def construct(self): + outputs = self.get_next() + return self.network(*outputs) + + +class GetNextSingleOp(Cell): + """ + Cell to run get next operation. + + Args: + dataset_types (list[:class:`mindspore.dtype`]): The types of dataset. + dataset_shapes (list[tuple[int]]): The shapes of dataset. + queue_name (str): Queue name to fetch the data. + + Detailed information, please refer to `ops.operations.GetNext`. + """ + + def __init__(self, dataset_types, dataset_shapes, queue_name): + super(GetNextSingleOp, self).__init__() + self.get_next = P.GetNext(dataset_types, dataset_shapes, len(dataset_types), queue_name) + + def construct(self): + return self.get_next() + + +class _VirtualDatasetCell(Cell): + """ + Wrap the network with virtual dataset to convert data parallel layout to model parallel layout. + + _VirtualDataset is a virtual Primitive, it does not exist in the final executing graph. Inputs and outpus + of _VirtualDataset are distributed in data parallel pattern, tensor redistribution Primitives is inserted + dynamically during the graph compile process. + + Note: + Only used in semi auto parallel and auto parallel mode. + + Args: + backbone (Cell): The target network to wrap. + + Examples: + >>> net = Net() + >>> net = _VirtualDatasetCell(net) + """ + + def __init__(self, backbone): + super(_VirtualDatasetCell, self).__init__(auto_prefix=False) + self._backbone = backbone + self._virtual_dataset = _VirtualDataset() + + def construct(self, data, label): + data_, label_ = self._virtual_dataset(data, label) + return self._backbone(data_, label_) + + +class WithEvalCell(Cell): + r""" + Cell that returns loss, output and label for evaluation. + + This Cell accepts a network and loss function as arguments and computes loss for model. + It returns loss, output and label to calculate the metrics. + + Args: + network (Cell): The network Cell. + loss_fn (Cell): The loss Cell. + + Inputs: + - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + + Outputs: + Tuple, containing a scalar loss Tensor, a network output Tensor of shape :math:`(N, \ldots)` + and a label Tensor of shape :math:`(N, \ldots)`. + + Examples: + >>> # For a defined network Net without loss function + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> eval_net = nn.WithEvalCell(net, loss_fn) + """ + + def __init__(self, network, loss_fn): + super(WithEvalCell, self).__init__(auto_prefix=False) + self._network = network + self._loss_fn = loss_fn + + def construct(self, data, label): + outputs = self._network(data) + loss = self._loss_fn(outputs, label) + + return loss, outputs, label + + +class ParameterUpdate(Cell): + """ + Cell that updates parameters. + + With this Cell, one can manually update `param` with the input `Tensor`. + + Args: + param (Parameter): The parameter to be updated manually. + + Raises: + KeyError: If parameter with the specified name do not exist. + + Examples: + >>> network = Net() + >>> param = network.parameters_dict()['learning_rate'] + >>> update = nn.ParameterUpdate(param) + >>> update.phase = "update_param" + >>> lr = mindspore.Tensor(0.001, mindspore.float32) + >>> update(lr) + """ + + def __init__(self, param): + super(ParameterUpdate, self).__init__(auto_prefix=False) + if not isinstance(param, Parameter): + raise TypeError("`param` must be `Parameter`, but got {}".format(param)) + + default_input = param.default_input + if isinstance(default_input, Tensor): + shape = default_input.shape() + zero_dtype = default_input.dtype() + elif isinstance(default_input, float): + shape = [1] + zero_dtype = mstype.float32 + elif isinstance(default_input, int): + shape = [1] + zero_dtype = mstype.int32 + else: + raise TypeError("`default_input` in `param` must be Tensor, float or int, but got {}".format(default_input)) + + self._param = Parameter(initializer(copy.deepcopy(default_input), shape), param.name) + self._param.is_init = True + self._zero = Tensor(np.zeros(shape), zero_dtype) + + def construct(self, x): + zero = self._param + self._zero + F.control_depend(zero, F.assign(self._param, x)) + return zero diff --git a/mindspore/nn/wrap/grad_reducer.py b/mindspore/nn/wrap/grad_reducer.py new file mode 100644 index 00000000000..8b34abc47b7 --- /dev/null +++ b/mindspore/nn/wrap/grad_reducer.py @@ -0,0 +1,196 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""grad reducer cell for distributed training""" +from mindspore.nn.cell import Cell +from mindspore.communication.management import GlobalComm, get_group_size +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp +import mindspore.common.dtype as mstype + +reduce_opt = C.MultitypeFuncGraph("reduce_opt") + +_all_reduce = AllReduce() + + +def _init_optimizer_allreduce(): + global _all_reduce + _all_reduce = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) + _all_reduce.add_prim_attr('fusion', 1) + + +@reduce_opt.register("Function", "Number", "Bool", "Tensor") +def _tensors_allreduce_mean(mul, degree, allreduce_filter, grad): + """ + Apply mean and allreduce on gradient. Allreduce is a communication operation used for distributed deep learning. + + Args: + mul (Primitive): Div operation. + degree (int): The mean coefficient. + allreduce_filter (bool): When it is true, allreduce would apply. + grad (Tensor): The gradient tensor before operation. + + Returns: + Tensor, the gradient tensor after operation. + """ + if allreduce_filter: + degree = F.scalar_cast(degree, F.dtype(grad)) + grad = _all_reduce(grad) + cast_op = P.Cast() + return mul(grad, cast_op(F.scalar_to_array(1.0/degree), F.dtype(grad))) + return grad + + +@reduce_opt.register("Bool", "Tensor") +def _tensors_allreduce(allreduce_filter, grad): + """ + Apply allreduce on gradient. + + Args: + allreduce_filter (bool): When it is true, allreduce would apply. + grad (Tensor): The gradient tensor before operation. + + Returns: + Tensor, the gradient tensor after operation. + """ + if allreduce_filter: + return _all_reduce(grad) + return grad + + +_get_datatype = C.MultitypeFuncGraph("_get_datatype") + + +@_get_datatype.register("Tensor") +def _tensors_get_datatype(grad): + """ + Acquire gradient datatype. + + Args: + grad (Tensor): The gradient tensor before operation. + + Returns: + mstype, the datatype of gradient. + """ + return F.dtype(grad) + + +_cast_datatype = C.MultitypeFuncGraph("_cast_datatype") + + +@_cast_datatype.register("TypeType", "Tensor") +def _tensors_cast_datatype(datatype, grad): + """ + Cast gradient to datatype. + + Args: + datatype (mstype): the destination datatype of gradient. + grad (Tensor): The gradient tensor before operation. + + Returns: + Tensor, the gradient tensor after operation. + """ + return F.cast(grad, datatype) + + +class DistributedGradReducer(Cell): + """ + A distributed optimizer. + + Constructs a gradient reducer Cell, which applies communication and average operations on + single-process gradient values. + + Args: + parameters (list): the parameters to be updated. + mean (bool): When mean is true, the mean coefficient (degree) would apply on gradients. Default: False. + degree (int): The mean coefficient. Usually it equals to device number. Default: None. + + Raises: + ValueError: If degree is not a int or less than 0. + + Examples: + >>> from mindspore.communication import get_group_size + >>> from mindspore.ops import composite as C + >>> from mindspore.ops import operations as P + >>> from mindspore.ops import functional as F + >>> from mindspore import context + >>> + >>> class TrainingWrapper(nn.Cell): + >>> def __init__(self, network, optimizer, sens=1.0): + >>> super(TrainingWrapper, self).__init__(auto_prefix=False) + >>> self.network = network + >>> self.weights = mindspore.ParameterTuple(network.trainable_params()) + >>> self.optimizer = optimizer + >>> self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + >>> self.sens = sens + >>> self.reducer_flag = False + >>> self.grad_reducer = None + >>> self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + >>> if self.parallel_mode in [mindspore.ParallelMode.DATA_PARALLEL, + >>> mindspore.ParallelMode.HYBRID_PARALLEL]: + >>> self.reducer_flag = True + >>> if self.reducer_flag: + >>> mean = context.get_auto_parallel_context("mirror_mean") + >>> if mean.get_device_num_is_set(): + >>> degree = context.get_auto_parallel_context("device_num") + >>> else: + >>> degree = get_group_size() + >>> self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) + >>> + >>> def construct(self, *args): + >>> weights = self.weights + >>> loss = self.network(*args) + >>> sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + >>> grads = self.grad(self.network, weights)(*args, sens) + >>> if self.reducer_flag: + >>> # apply grad reducer on grads + >>> grads = self.grad_reducer(grads) + >>> return F.depend(loss, self.optimizer(grads)) + >>> + >>> network = Net() + >>> optimizer = nn.Momentum(network.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> train_cell = TrainingWrapper(network, optimizer) + >>> inputs = mindspore.Tensor(np.ones([16, 16]).astype(np.float32)) + >>> label = mindspore.Tensor(np.zeros([16, 16]).astype(np.float32)) + >>> grads = train_cell(inputs, label) + """ + + def __init__(self, parameters, mean=True, degree=None): + super(DistributedGradReducer, self).__init__(auto_prefix=False) + self.hyper_map = C.HyperMap() + self.mul = P.Mul() + if degree is None: + self.degree = get_group_size() + else: + if not isinstance(degree, int) or degree <= 0: + raise ValueError("Parameter 'degree' in DistributedGradReducer should large than 0 and be int") + self.degree = degree + self.mean = mean + self.allreduce_filter = tuple(x.layerwise_parallel is False for x in parameters) + _init_optimizer_allreduce() + + def construct(self, grads): + # In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the + # result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, + # and cast back after the operation. + datatypes = self.hyper_map(F.partial(_get_datatype), grads) + grads = self.hyper_map(F.partial(_cast_datatype, mstype.float32), grads) + + if self.mean: + new_grad = self.hyper_map(F.partial(reduce_opt, self.mul, self.degree), self.allreduce_filter, grads) + else: + new_grad = self.hyper_map(F.partial(reduce_opt), self.allreduce_filter, grads) + + new_grad = self.hyper_map(F.partial(_cast_datatype), datatypes, new_grad) + return new_grad diff --git a/mindspore/nn/wrap/loss_scale.py b/mindspore/nn/wrap/loss_scale.py new file mode 100644 index 00000000000..f7c686f5350 --- /dev/null +++ b/mindspore/nn/wrap/loss_scale.py @@ -0,0 +1,257 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Loss scale cell for loss scale training.""" +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +from mindspore.train.parallel_utils import ParallelMode +from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_mirror_mean +from ..cell import Cell +from ...common import Tensor, ParameterTuple +from ...common.parameter import Parameter +from ...ops import functional as F +from ...ops import composite as C +from ...ops import operations as P +from ...ops.operations import NPUGetFloatStatus, NPUAllocFloatStatus, NPUClearFloatStatus, ReduceSum, LessEqual, \ + ControlDepend +from ...common import dtype as mstype + +_grad_scale = C.MultitypeFuncGraph("grad_scale") +reciprocal = P.Reciprocal() + + +@_grad_scale.register("Tensor", "Tensor") +def tensor_grad_scale(scale, grad): + return grad * reciprocal(scale) + + +class DynamicLossScaleUpdateCell(Cell): + r""" + Dynamic Loss scale update cell. + + For loss scaling training, the initial loss scaling value will be set to be `loss_scale_value`. + In every training step, the loss scaling value will be updated by loss scaling value/`scale_factor` + when there is overflow. And it will be increased by loss scaling value * `scale_factor` if there is no + overflow for a continuous `scale_window` steps. This cell is used for Graph mode training in which all + logic will be executed on device side(Another training mode is feed mode in which some logic will be + executed on host). + + Args: + loss_scale_value (float): Init loss scale. + scale_factor (int): Coefficient of increase and decrease. + scale_window (int): Maximum continuous training steps that do not have overflow. + + Inputs: + - **inputs** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + + Outputs: + Tensor, a scalar Tensor with shape :math:`()`. + + Examples: + >>> net_with_loss = Net() + >>> optimizer = nn.Momentum(net_with_loss.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> manager = nn.DynamicLossScaleUpdateCell(loss_scale_value=2**12, scale_factor=2, scale_window=1000) + >>> train_network = nn.TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell=manager) + >>> train_network.set_train() + >>> + >>> inputs = mindspore.Tensor(np.ones([16, 16]).astype(np.float32)) + >>> label = mindspore.Tensor(np.zeros([16, 16]).astype(np.float32)) + >>> output = train_network(inputs, label) + """ + + def __init__(self, + loss_scale_value, + scale_factor, + scale_window): + super(DynamicLossScaleUpdateCell, self).__init__() + + self.scale_window = Tensor(scale_window, dtype=mstype.int32) + self.scale_factor = Tensor(scale_factor, dtype=mstype.float32) + self.loss_scale_value = loss_scale_value + + self.cur_iter = Parameter(Tensor(1, dtype=mstype.int32), name="current_iterator_step") + self.last_overflow_iter = Parameter(Tensor(0, dtype=mstype.int32), name="last_overflow_iterator_step") + self.select = P.Select() + self.max = P.Maximum() + self.minimum_loss_scale = Tensor(1.0, dtype=mstype.float32) + self.reciprocal = P.Reciprocal() + self.less_equal = P.LessEqual() + self.logic_and = P.LogicalAnd() + self.logic_not = P.LogicalNot() + self.logic_or = P.LogicalOr() + self.const_true = Tensor(True, dtype=mstype.bool_) + + def get_loss_scale(self): + return self.loss_scale_value + + def construct(self, loss_scale, overflow): + overflow_cond = overflow + loss_scale_on_overflow = self.select(overflow_cond, self.max(loss_scale * self.reciprocal(self.scale_factor), + self.minimum_loss_scale), loss_scale) + should_inc = self.less_equal(self.scale_window, self.cur_iter - self.last_overflow_iter) + last_iter_cond = self.logic_or(overflow_cond, should_inc) + last_overflow_iter = self.select(last_iter_cond, self.cur_iter, self.last_overflow_iter) + assign_last_iter = F.assign(self.last_overflow_iter, last_overflow_iter) + update_scale_cond = self.logic_and(should_inc, self.logic_not(overflow_cond)) + scale_mul_res = loss_scale_on_overflow * self.scale_factor + scaled_loss_scale = self.select(update_scale_cond, scale_mul_res, loss_scale_on_overflow) + assign_scaled_loss_scale = F.assign(loss_scale, scaled_loss_scale) + inc_cur_iter = self.cur_iter + 1 + assing_cur_iter = F.assign(self.cur_iter, inc_cur_iter) + t = (assign_last_iter, assign_scaled_loss_scale, assing_cur_iter) + F.control_depend(assign_last_iter, assing_cur_iter) + return F.depend(overflow, t) + + +class FixedLossScaleUpdateCell(Cell): + """ + Static scale update cell, the loss scaling value will not be updated. + + For usage please refer to `DynamicLossScaleUpdateCell`. + + Args: + loss_scale_value (float): Init loss scale. + + Examples: + >>> net_with_loss = Net() + >>> optimizer = nn.Momentum(net_with_loss.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> manager = nn.FixedLossScaleUpdateCell(loss_scale_value=2**12, scale_factor=2, scale_window=1000) + >>> train_network = nn.TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell=manager) + >>> train_network.set_train() + >>> + >>> inputs = mindspore.Tensor(np.ones([16, 16]).astype(np.float32)) + >>> label = mindspore.Tensor(np.zeros([16, 16]).astype(np.float32)) + >>> output = train_network(inputs, label) + """ + + def __init__(self, loss_scale_value): + super(FixedLossScaleUpdateCell, self).__init__() + self.loss_scale_value = loss_scale_value + + def get_loss_scale(self): + return self.loss_scale_value + + def construct(self, _, overflow): + return overflow + + +class TrainOneStepWithLossScaleCell(Cell): + r""" + Network training with loss scaling. + + This is a training step with loss scaling. It takes a network, an optimizer and possibly a scale update + Cell as args. The loss scale value can be updated in both host side or device side. The + TrainOneStepWithLossScaleCell will be compiled to be graph which takes `data`, `label`, `sens` as input + data. The `sens` is acting as loss scaling value. If you want to update it on host side, the value should + be provided. If `sens` is not given, the loss scale update logic should be provied by `scale_update_cell`. + If `scale_update_cell` is not None and `sens` is provided, the `scale_update_cell` will be ignored. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + scale_update_cell(Cell): The loss scaling update logic cell. Default: None. + + Inputs: + - **inputs** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + - **scaling_sens** (Tensor) - Tensor of shape :math:`()`. + + Outputs: + Tuple of 3 Tensor, the loss, overflow flag and current loss scaling value. + + - **loss** (Tensor) - Tensor with shape :math:`()`. + - **overflow** (Tensor) - Tensor with shape :math:`()`, type is bool. + - **loss_scale** (Tensor) - Tensor with shape :math:`()`. + + Examples: + >>> net_with_loss = Net() + >>> optimizer = nn.Momentum(net_with_loss.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> manager = nn.DynamicLossScaleUpdateCell(init_loss_scale=2**12, scale_factor=2, scale_window=1000) + >>> train_network = nn.TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell=manager) + >>> train_network.set_train() + >>> + >>> inputs = mindspore.Tensor(np.ones([16, 16]).astype(np.float32)) + >>> label = mindspore.Tensor(np.zeros([16, 16]).astype(np.float32)) + >>> scaling_sens = mindspore.Tensor(np.full((1), np.finfo(np.float32).max), dtype=mindspore.float32) + >>> output = train_network(inputs, label, scaling_sens) + """ + + def __init__(self, network, optimizer, scale_update_cell=None): + super(TrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.hyper_map = C.HyperMap() + self.alloc_status = NPUAllocFloatStatus() + self.get_status = NPUGetFloatStatus() + self.clear_status = NPUClearFloatStatus() + self.reduce_sum = ReduceSum(keep_dims=False) + self.base = Tensor(1, mstype.float32) + self.less_equal = LessEqual() + self.depend_parameter_use = ControlDepend(depend_mode=1) + self.allreduce = P.AllReduce() + self.parallel_mode = _get_parallel_mode() + self.grad_reducer = None + self.reducer_flag = self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL] + if self.reducer_flag: + mean = _get_mirror_mean() + degree = _get_device_num() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + self.is_distributed = self.parallel_mode != ParallelMode.STAND_ALONE + + self.loss_scale = None + self.loss_scaling_manager = scale_update_cell + if scale_update_cell: + self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), + name="loss_scale") + self.add_flags(has_effect=True) + + def construct(self, data, label, sens=None): + weights = self.weights + loss = self.network(data, label) + # init overflow buffer + init = self.alloc_status() + # clear overflow buffer + self.clear_status(init) + if sens is None: + scaling_sens = self.loss_scale + else: + scaling_sens = sens + grads = self.grad(self.network, weights)(data, label, F.cast(scaling_sens, F.dtype(loss))) + grads = self.hyper_map(F.partial(_grad_scale, scaling_sens), grads) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + # get the overflow buffer + self.get_status(init) + # sum overflow buffer elements, 0:not overflow , >0:overflow + flag_sum = self.reduce_sum(init, (0,)) + if self.is_distributed: + # sum overflow flag over devices + flag_reduce = self.allreduce(flag_sum) + cond = self.less_equal(self.base, flag_reduce) + else: + cond = self.less_equal(self.base, flag_sum) + overflow = cond + if sens is None: + overflow = self.loss_scaling_manager(self.loss_scale, cond) + # if there is no overflow, do optimize + if overflow: + opt = False + else: + opt = self.optimizer(grads) + ret = (loss, cond, scaling_sens) + return F.depend(ret, opt) diff --git a/mindspore/ops/__init__.py b/mindspore/ops/__init__.py new file mode 100644 index 00000000000..23109b386a8 --- /dev/null +++ b/mindspore/ops/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Operators can be used in the construct function of Cell. + +Examples: + + >>> from mindspore.ops import operations as P + >>> from mindspore.ops import composite as C + >>> from mindspore.ops import functional as F + +Note: + - The Primitive operators in operations need to be used after instantiation. + - The composite operators are pre-defined combination of operator. + - The functional operators are the pre-instantiated Primitive operators, which can be used directly like a function. +""" + +from .primitive import Primitive, PrimitiveWithInfer, prim_attr_register +from .vm_impl_registry import get_vm_impl_fn, vm_impl_registry +from .op_info_register import op_info_register +from .primitive import constexpr +from .._c_expression import signature_rw, signature_kind + +__primitive__ = [ + "prim_attr_register", "Primitive", "PrimitiveWithInfer", + "signature_rw", "signature_kind" +] + +__all__ = ["get_vm_impl_fn", "vm_impl_registry", + "op_info_register", + "constexpr"] +__all__.extend(__primitive__) diff --git a/mindspore/ops/_grad/__init__.py b/mindspore/ops/_grad/__init__.py new file mode 100644 index 00000000000..9cf4104e5a1 --- /dev/null +++ b/mindspore/ops/_grad/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""grad impl.""" +from . import grad_array_ops, grad_comm_ops, grad_debug_ops, grad_implementations, \ + grad_math_ops, grad_nn_ops, grad_other_ops +from .grad_base import get_bprop_fn + +__all__ = ['get_bprop_fn'] diff --git a/mindspore/ops/_grad/grad_array_ops.py b/mindspore/ops/_grad/grad_array_ops.py new file mode 100644 index 00000000000..cf6247023e6 --- /dev/null +++ b/mindspore/ops/_grad/grad_array_ops.py @@ -0,0 +1,410 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""array_ops""" + +from .. import operations as P +from ..operations import _grad_ops as G +from ..composite.multitype_ops.zeros_like_impl import zeros_like +from .. import functional as F +from .grad_base import bprop_getters +from ..primitive import constexpr +from ... import context + +reduce_sum = P.ReduceSum() +unsorted_segment_sum = P.UnsortedSegmentSum() +transpose = P.Transpose() +shape_op = P.Shape() +reshape = P.Reshape() +invert_permutation = P.InvertPermutation() + + +@bprop_getters.register(P.Fill) +def get_bprop_fill(self): + """Generate bprop for Fill""" + + def bprop(dtype, dims, x, out, dout): + return zeros_like(dims), zeros_like(x) + + return bprop + + +@bprop_getters.register(P.DType) +def get_bprop_dtype(self): + """Generate bprop for DType""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.Cast) +def get_bprop_cast(self): + """Generate bprop for Cast""" + cast = P.Cast() + get_dtype = P.DType() + + def bprop(x, t, out, dout): + dx = cast(dout, get_dtype(x)) + return dx, zeros_like(t) + return bprop + + +@bprop_getters.register(P.Shape) +def get_bprop_shape(self): + """Generate bprop for Shape""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.Split) +def get_bprop_split(self): + """Generate bprop for Split""" + axis = self.axis + + def bprop(x, out, dout): + concat_op = P.Concat(axis) + dx = concat_op(dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.Rank) +def get_bprop_rank(self): + """Generate bprop for Rank""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.Reshape) +def get_bprop_reshape(self): + """Generate bprop for Reshape""" + + def bprop(x, shp, out, dout): + shapex = shape_op(x) + return reshape(dout, shapex), zeros_like(shp) + return bprop + + +@bprop_getters.register(P.ExpandDims) +def get_bprop_expand_dims(self): + """Generate bprop for ExpandDims""" + + def bprop(x, axis, out, dout): + shapex = shape_op(x) + return reshape(dout, shapex), zeros_like(axis) + return bprop + + +@bprop_getters.register(P.Squeeze) +def get_bprop_squeeze(self): + """Generate bprop for Squeeze""" + + def bprop(x, out, dout): + shapex = shape_op(x) + return (reshape(dout, shapex),) + return bprop + + +@bprop_getters.register(P.Flatten) +def get_bprop_flatten(self): + """Generate bprop for Flatten""" + flatten_grad = G.FlattenGrad() + + def bprop(x, out, dout): + dx = flatten_grad(dout, shape_op(x)) + return (dx,) + return bprop + + +@constexpr +def _tile_shape(multiples, shapex): + """Calculate [1,2], [3, 4] -> [1,3,2,4].""" + len_muli = len(multiples) + rank = len(shapex) + len_cmp = len_muli - rank + max_len = max(len_muli, rank) + i = 0 + j = 0 + ret = [] + while (i < max_len) and (j < max_len): + if len_cmp == 0: + ret.append(multiples[i]) + ret.append(shapex[j]) + i += 1 + j += 1 + elif len_cmp > 0: + ret.append(multiples[i]) + ret.append(1) + i += 1 + len_cmp -= 1 + else: + ret.append(1) + ret.append(shapex[j]) + len_cmp += 1 + return tuple(ret) + + +@bprop_getters.register(P.Tile) +def get_bprop_tile(self): + """Generate bprop for Tile""" + def bprop(x, multiples, out, dout): + shapex = shape_op(x) + r_shape = _tile_shape(multiples, shapex) + # 0 represents the start index, and 2 represents the step + axis = F.make_range(0, len(r_shape), 2) + dx = reduce_sum(reshape(dout, r_shape), axis) + dx = reshape(dx, shapex) + return dx, zeros_like(multiples) + return bprop + + +@bprop_getters.register(P.Transpose) +def get_bprop_transpose(self): + """Generate bprop for Transpose""" + + def bprop(x, perm, out, dout): + return transpose(dout, invert_permutation(perm)), zeros_like(perm) + return bprop + + +@bprop_getters.register(P.Concat) +def get_bprop_concat(self): + """Generate bprop for Concat""" + axis = self.axis + + def bprop(x, out, dout): + dx = () + out_offset = P.ConcatOffset(F.tuple_len(x), axis)(x) + for i in range(F.tuple_len(x)): + slice_out = P.Slice()(dout, out_offset[i], shape_op(x[i])) + dx = dx + (slice_out,) + return (dx,) + return bprop + + +@constexpr +def _slice_grad_pad(begins, sizes, shapes): + pads = tuple((begin, shape - begin - size) for begin, size, shape in zip(begins, sizes, shapes)) + return pads + + +@bprop_getters.register(P.Slice) +def get_bprop_slice(self): + """Generate bprop for Slice""" + + def bprop(x, begin, size, out, dout): + dx = P.Pad(_slice_grad_pad(begin, size, shape_op(x)))(dout) + return (dx,) + + def bprop_gpu(x, begin, size, out, dout): + dx = dx = G.SliceGrad()(dout, x, begin, size) + return (dx,) + + if context.get_context('device_target') == "GPU": + return bprop_gpu + return bprop + + +@constexpr +def _generate_shape_index(out_shape, indices_shape, axis): + out_rank = len(out_shape) + ind_rank = len(indices_shape) + if axis < 0: + axis += out_rank - ind_rank + 1 + perm_part1 = tuple(range(axis, axis + ind_rank)) + index = tuple(range(out_rank)) + perm = perm_part1 + index[:axis] + index[axis + ind_rank:] + return perm + + +@constexpr +def _generate_inverse_index(x_shape, axis): + x_rank = len(x_shape) + index = tuple(range(x_rank)) + if axis < 0: + axis += x_rank + perm = index[1:1 + axis] + (0,) + index[1 + axis:] + return perm + + +@bprop_getters.register(P.GatherV2) +def get_bprop_gather_v2(self): + """Generate bprop for GatherV2""" + def bprop(x, indices, axis, out, dout): + if F.rank(dout) == 0: + dout = P.ExpandDims()(dout, -1) + if F.rank(indices) == 0: + indices = P.ExpandDims()(indices, -1) + x_shp = shape_op(x) + out_shp = shape_op(dout) + ind_shp = shape_op(indices) + # Example: out_shape:(3,2,3) axis 1 -> (1,0,2) + perm_1 = _generate_shape_index(out_shp, ind_shp, axis) + values_transpose = transpose(dout, perm_1) + params_grad = unsorted_segment_sum(values_transpose, indices, shape_op(x)[axis]) + # Example: out_shape:(3,2,3) axis 2 -> (1,2,0) + perm_2 = _generate_inverse_index(x_shp, axis) + params_grad = transpose(params_grad, perm_2) + return params_grad, zeros_like(indices) + return bprop + + +@bprop_getters.register(P.StridedSlice) +def get_bprop_strided_slice(self): + """Generate bprop for StridedSlice""" + input_grad = G.StridedSliceGrad(self.begin_mask, + self.end_mask, + self.ellipsis_mask, + self.new_axis_mask, + self.shrink_axis_mask) + + def bprop(x, begin, end, strides, out, dout): + dx = input_grad(dout, shape_op(x), begin, end, strides) + return dx, zeros_like(begin), zeros_like(end), zeros_like(strides) + return bprop + + +@bprop_getters.register(P.Eye) +def get_bprop_eye(self): + """Generate bprop for Eye""" + + def bprop(n, m, t, out, dout): + return zeros_like(n), zeros_like(m), zeros_like(t) + return bprop + + +@bprop_getters.register(P.Select) +def get_bprop_select(self): + """Generate bprop for Select""" + select = P.Select() + + def bprop(cond, x, y, out, dout): + return zeros_like(cond), select(cond, dout, zeros_like(x)), select(cond, zeros_like(y), dout) + return bprop + + +@bprop_getters.register(P.OnesLike) +def get_bprop_oneslike(self): + """Generate bprop for OnesLike""" + + def bprop(x, out, dout): + return (zeros_like(x),) + + return bprop + + +@bprop_getters.register(P.ZerosLike) +def get_bprop_zeroslike(self): + """Generate bprop for OnesLike""" + + def bprop(x, out, dout): + return (zeros_like(x),) + + return bprop + + +@bprop_getters.register(P.ResizeNearestNeighbor) +def get_bprop_resize_nearest_neighbor(self): + """Generate bprop for ResizeNearestNeighbor""" + op = G.ResizeNearestNeighborGrad(self.align_corners) + + def bprop(inputs, out, dout): + shp = shape_op(inputs) + # 2 and 3 represent the height and width + shp = (shp[2], shp[3]) + return (op(dout, shp),) + + return bprop + + +@bprop_getters.register(P.GatherNd) +def get_bprop_gather_nd(self): + """Generate bprop for GatherNd""" + op = P.ScatterNd() + + def bprop(x, indices, out, dout): + shp = shape_op(x) + return op(indices, dout, shp), zeros_like(indices) + + return bprop + + +@bprop_getters.register(P.ScatterNd) +def get_bprop_scatter_nd(self): + """Generate bprop for ScatterNd""" + op = P.GatherNd() + + def bprop(indices, x, shape, out, dout): + return zeros_like(indices), op(dout, indices), zeros_like(shape) + + return bprop + + +@bprop_getters.register(P.ScatterNdUpdate) +def get_bprop_scatter_nd_update(self): + """Generate bprop for ScatterNdUpdate""" + op = P.GatherNd() + + def bprop(x, indices, update, out, dout): + return dout, zeros_like(indices), op(dout, indices) + + return bprop + + +@bprop_getters.register(P.Argmax) +def get_bprop_argmax(self): + """Generate bprop for Argmax""" + + def bprop(x, out, dout): + return (zeros_like(x),) + + return bprop + + +@bprop_getters.register(P.Argmin) +def get_bprop_argmin(self): + """Generate bprop for Argmin""" + + def bprop(x, out, dout): + return (zeros_like(x),) + + return bprop + + +@bprop_getters.register(P.SpaceToDepth) +def get_bprop_space_to_depth(self): + """Generate bprop for SpaceToDepth""" + op = P.DepthToSpace(self.block_size) + + def bprop(x, out, dout): + return (op(dout),) + + return bprop + + +@bprop_getters.register(P.DepthToSpace) +def get_bprop_depth_to_space(self): + """Generate bprop for DepthToSpace""" + op = P.SpaceToDepth(self.block_size) + + def bprop(x, out, dout): + return (op(dout),) + + return bprop diff --git a/mindspore/ops/_grad/grad_base.py b/mindspore/ops/_grad/grad_base.py new file mode 100644 index 00000000000..c75701d8d2b --- /dev/null +++ b/mindspore/ops/_grad/grad_base.py @@ -0,0 +1,30 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""grad base functions""" + +from .._register_for_op import Registry + + +bprop_getters = Registry() +bprops = Registry() + + +def get_bprop_fn(prim): + """get bprop function by primitive obj or prim name for c++""" + out = bprop_getters.get(prim, None) + if out: + return out(prim) + return bprops.get(prim, None) diff --git a/mindspore/ops/_grad/grad_comm_ops.py b/mindspore/ops/_grad/grad_comm_ops.py new file mode 100644 index 00000000000..3a31c8aeec2 --- /dev/null +++ b/mindspore/ops/_grad/grad_comm_ops.py @@ -0,0 +1,155 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Generate bprop for comm ops""" +import mindspore.common.dtype as mstype +from mindspore.ops import functional as F +from .. import operations as P +from ..composite.multitype_ops.zeros_like_impl import zeros_like +from ..operations.comm_ops import (AllGather, AllReduce, _AlltoAll, Broadcast, + _GetTensorSlice, _MirrorOperator, ReduceOp, + ReduceScatter, _VirtualDiv) +from .grad_base import bprop_getters + + +@bprop_getters.register(AllReduce) +def get_bprop_all_reduce(self): + """Generate bprop for AllReduce.""" + + all_reduce_grad = AllReduce(ReduceOp.SUM, self.group) + if self.instance_name: + instance_name = "grad" + self.instance_name + all_reduce_grad.set_prim_instance_name(instance_name) + equal = P.Equal() + cast = P.Cast() + mul = P.Mul() + dtype = P.DType() + + if self.op == ReduceOp.PROD: + raise RuntimeError("The bprop of ReduceOp.PROD is not supported yet.") + if self.op == ReduceOp.SUM: + + def bprop(x, out, dout): + dx = all_reduce_grad(dout) + return (dx,) + else: + + def bprop(x, out, dout): + dx = all_reduce_grad(dout) + z = equal(x, out) + z = cast(z, dtype(dx)) + dx = mul(dx, z) + return (dx,) + return bprop + + +@bprop_getters.register(Broadcast) +def get_bprop_broad_cast(self): + """Generate bprop for Broadcast.""" + + def bprop(x, out, dout): + return (dout,) + return bprop + + +@bprop_getters.register(AllGather) +def get_bprop_all_gather(self): + """Generate bprop for AllGather""" + reduce_scatter_grad = ReduceScatter(ReduceOp.SUM, self.group) + if self.instance_name: + instance_name = "grad" + self.instance_name + reduce_scatter_grad.set_prim_instance_name(instance_name) + + def bprop(x, out, dout): + dx = reduce_scatter_grad(dout) + return (dx,) + + return bprop + + +@bprop_getters.register(_AlltoAll) +def get_bprop_all_to_all(self): + """Generate bprop for AlltoAll.""" + all_to_all_grad = _AlltoAll(self.split_count, self.concat_dim, self.split_dim, self.group) + if self.instance_name: + instance_name = "grad" + self.instance_name + all_to_all_grad.set_prim_instance_name(instance_name) + + def bprop(x, out, dout): + dx = all_to_all_grad(dout) + return (dx,) + + return bprop + + +@bprop_getters.register(_MirrorOperator) +def get_bprop_mirror_operator(self): + """Backpropagator for _MirrorOperator, do allreduce for the devices in group(only for one group).""" + group = self.group + dev_num = self.dev_num + mean_flag = self.mean_flag + + all_reduce = AllReduce(group=group) + mul = P.Mul() + cast = P.Cast() + + fusion = 1 + if hasattr(self, 'fusion'): + fusion = self.fusion + all_reduce.add_prim_attr("fusion", fusion) + if hasattr(self, 'parameter'): + parameter = self.parameter + all_reduce.add_prim_attr("parameter", parameter) + + if self.instance_name: + instance_name = "grad_mirror" + self.instance_name + all_reduce.set_prim_instance_name(instance_name) + + def bprop(x, out, dout): + if mean_flag: + dx = all_reduce(dout) + float_one = F.scalar_cast(1.0, F.dtype(dx)) + num = F.scalar_cast(dev_num, F.dtype(dx)) + dx = mul(dx, cast(F.scalar_to_array(float_one/num), F.dtype(dx))) + else: + dx = all_reduce(dout) + + return (dx,) + return bprop + + +@bprop_getters.register(_VirtualDiv) +def get_bprop_virtual_div_operator(self): + """Backpropagator for _VirtualDiv, do Div for the divisor.""" + divisor = self.divisor + op = P.RealDiv() + cast = P.Cast() + dtype = P.DType() + + def bprop(x, out, dout): + if F.issubclass_(F.dtype(dout), mstype.bool_): + return (dout,) + dx = op(dout, cast(F.scalar_to_array(divisor), dtype(dout))) + return (dx,) + return bprop + + +@bprop_getters.register(_GetTensorSlice) +def get_bprop_get_tensor_slice_operator(self): + """Backpropagator for _GetTensorSlice""" + + def bprop(x, dev_mat, tensor_map, out, dout): + return (zeros_like(x),) + return bprop diff --git a/mindspore/ops/_grad/grad_debug_ops.py b/mindspore/ops/_grad/grad_debug_ops.py new file mode 100644 index 00000000000..431d82192f3 --- /dev/null +++ b/mindspore/ops/_grad/grad_debug_ops.py @@ -0,0 +1,59 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Generate bprop for debug ops""" + +from .. import operations as P +from ..composite.multitype_ops.zeros_like_impl import zeros_like +from .grad_base import bprop_getters + +# Unused parameters are placeholders. + + +@bprop_getters.register(P.ScalarSummary) +def get_bprop_scalar_summary(self): + """Generate bprop for ScalarSummary""" + + def bprop(tag, x, out, dout): + return tag, zeros_like(x) + return bprop + + +@bprop_getters.register(P.TensorSummary) +def get_bprop_tensor_summary(self): + """Generate bprop for TensorSummary""" + + def bprop(tag, x, out, dout): + return tag, zeros_like(x) + return bprop + + +@bprop_getters.register(P.ImageSummary) +def get_bprop_image_summary(self): + """Generate bprop for ImageSummary""" + + def bprop(tag, x, out, dout): + return tag, zeros_like(x) + return bprop + + +@bprop_getters.register(P.InsertGradientOf) +def get_bprop_insert_gradient_of(self): + """Generate bprop for InsertGradientOf""" + f = self.f + + def bprop(x, out, dout): + return (f(dout),) + return bprop diff --git a/mindspore/ops/_grad/grad_implementations.py b/mindspore/ops/_grad/grad_implementations.py new file mode 100644 index 00000000000..ffdd5e6a5ac --- /dev/null +++ b/mindspore/ops/_grad/grad_implementations.py @@ -0,0 +1,244 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""bprop primitives""" +from .. import functional as F +from ..composite import multitype_ops as C +from .grad_base import bprops + +# Unused parameters are placeholders. + + +@bprops.register("scalar_add") +def bprop_scalar_add(x, y, out, dout): + """Backpropagator for primitive `scalar_add`.""" + return dout, dout + + +@bprops.register("scalar_mul") +def bprop_scalar_mul(x, y, out, dout): + """Backpropagator for primitive `scalar_mul`.""" + return dout*y, dout*x + + +@bprops.register("scalar_sub") +def bprop_scalar_sub(x, y, out, dout): + """Backpropagator for primitive `scalar_sub`.""" + return dout, -dout + + +@bprops.register("scalar_div") +def bprop_scalar_div(x, y, out, dout): + """Backpropagator for primitive `scalar_div`.""" + return dout/y, (-dout) * (out/y) + + +@bprops.register("scalar_pow") +def bprop_scalar_pow(x, y, out, dout): + """Backpropagator for primitive `scalar_pow`.""" + return dout * (y * (x ** (y-1))), dout * (F.scalar_log(x) * out) + + +@bprops.register("scalar_exp") +def bprop_scalar_exp(x, out, dout): + """Backpropagator for primitive `scalar_exp`.""" + return (dout * out,) + + +@bprops.register("scalar_uadd") +def bprop_scalar_uadd(x, out, dout): + """Backpropagator for primitive `scalar_uadd`.""" + return (dout,) + + +@bprops.register("scalar_usub") +def bprop_scalar_usub(x, out, dout): + """Backpropagator for primitive `scalar_usub`.""" + return (-dout,) + + +@bprops.register("scalar_gt") +def bprop_scalar_gt(x, y, out, dout): + """Backpropagator for primitive `scalar_gt`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("scalar_lt") +def bprop_scalar_lt(x, y, out, dout): + """Backpropagator for primitive `scalar_lt`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("scalar_ge") +def bprop_scalar_ge(x, y, out, dout): + """Backpropagator for primitive `scalar_ge`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("scalar_le") +def bprop_scalar_le(x, y, out, dout): + """Backpropagator for primitive `scalar_le`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("scalar_eq") +def bprop_scalar_eq(x, y, out, dout): + """Backpropagator for primitive `scalar_eq`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("scalar_ne") +def bprop_scalar_ne(x, y, out, dout): + """Backpropagator for primitive `scalar_eq`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("scalar_cast") +def bprop_scalar_cast(x, t, out, dout): + """Backpropagator for primitive `scalar_cast`.""" + return F.scalar_cast(dout, F.typeof(x)), t + + +@bprops.register("tuple_getitem") +def bprop_tuple_getitem(data, idx, out, dout): + """Backpropagator for primitive `tuple_getitem`.""" + return F.tuple_setitem(C.zeros_like(data), idx, dout), C.zeros_like(idx) + + +@bprops.register("identity") +def bprop_identity(x, out, dout): + """Backpropagator for primitive `identity`.""" + return (dout,) + + +@bprops.register("make_ref") +def bprop_make_ref(key, x, y, out, dout): + """Backpropagator for primitive `make_ref`.""" + return (C.zeros_like(key), dout, C.zeros_like(y)) + + +@bprops.register("get_ref_value") +def bprop_get_ref_value(x, out, dout): + """Backpropagator for primitive `get_ref_value`.""" + return (dout,) + + +@bprops.register("get_ref_key") +def bprop_get_ref_key(x, out, dout): + """Backpropagator for primitive `get_ref_key`.""" + return (C.zeros_like(x),) + + +@bprops.register("scalar_to_array") +def bprop_scalar_to_array(x, out, dout): + """Backpropagator for primitive `scalar_to_array`.""" + return (F.array_to_scalar(dout),) + + +@bprops.register("array_to_scalar") +def bprop_array_to_scalar(x, out, dout): + """Backpropagator for primitive `array_to_scalar`.""" + return (F.scalar_to_array(dout),) + + +@bprops.register("dot") +def bprop_dot(x, y, out, dout): + """Backpropagator for primitive `dot`.""" + return F.dot(dout, F.transpose(y, (1, 0))), F.dot(F.transpose(x, (1, 0)), dout) + + +@bprops.register("reshape") +def bprop_reshape(xs, shp, out, dout): + """Backpropagator for primitive `reshape`.""" + return F.reshape(dout, F.shape(xs)), C.zeros_like(shp) + + +@bprops.register("distribute") +def bprop_distribute(arr, shp, out, dout): + """Backpropagator for primitive `distribute`.""" + return F.array_reduce(F.scalar_add, dout, F.shape(arr)), C.zeros_like(shp) + + +@bprops.register("shape") +def bprop_shape(arr, out, dout): + """Backpropagator for primitive `shape`.""" + return (C.zeros_like(arr),) + + +@bprops.register("broadcast_shape") +def bprop_broadcast_shape(shp1, shp2, out, dout): + """Backpropagator for primitive `broadcast_shape`.""" + return C.zeros_like(shp1), C.zeros_like(shp2) + + +@bprops.register("J") +def bprop_j(x, out, dout): + """Backpropagator for primitive `J`.""" + return (F.jinv(dout),) + + +@bprops.register("array_reduce") +def bprop_array_reduce(fn, x, shp, out, dout): + """Backpropagator for primitive `array_reduce`.""" + return F.distribute(dout, F.shape(x)), C.zeros_like(shp) + + +@bprops.register("depend") +def bprop_depend(x, y, out, dout): + """Backpropagator for primitive `depend`.""" + return dout, C.zeros_like(y) + + +@bprops.register("embed") +def bprop_embed(x, out, dout): + """Backpropagator for primitive `embed`.""" + return (C.zeros_like(x),) + + +@bprops.register("bool_not") +def bprop_bool_not(x, out, dout): + """Backpropagator for primitive `bool_not`.""" + return (C.zeros_like(x),) + + +@bprops.register("bool_or") +def bprop_bool_or(x, y, out, dout): + """Backpropagator for primitive `bool_or`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("stop_gradient") +def bprop_stop_gradient(x, out, dout): + """Backpropagator for primitive `stop_gradient`.""" + return (C.zeros_like(x),) + + +@bprops.register("bool_and") +def bprop_bool_and(x, y, out, dout): + """Backpropagator for primitive `bool_and`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("ControlDepend") +def bprop_control_depend(x, y, out, dout): + """Backpropagator for primitive `Control_depend`.""" + return C.zeros_like(x), C.zeros_like(y) + + +@bprops.register("switch") +def bprop_switch(cond, tb, fb, out, dout): + """Backpropagator for primitive `switch`.""" + return C.zeros_like(cond), F.switch(cond, dout, C.zeros_like(tb)), \ + F.switch(cond, C.zeros_like(fb), dout) diff --git a/mindspore/ops/_grad/grad_math_ops.py b/mindspore/ops/_grad/grad_math_ops.py new file mode 100755 index 00000000000..eb6ee9401df --- /dev/null +++ b/mindspore/ops/_grad/grad_math_ops.py @@ -0,0 +1,740 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Define the grad rules of math related operations.""" + + +from functools import reduce +from .. import functional as F +from .. import operations as P +from ..operations import _grad_ops as G +from ..composite.multitype_ops.zeros_like_impl import zeros_like +from ..functional import broadcast_gradient_args, reduced_shape, tuple_div +from .grad_base import bprop_getters +from ..primitive import constexpr + + +shape_op = P.Shape() +reduce_sum = P.ReduceSum() +reshape = P.Reshape() +tile = P.Tile() + + +def binop_grad_common(x, y, dx, dy): + """ + Common grad definition for binary operations. + + The function is usually used in backprop op to reduce additional dimensions created by broadcasting. + """ + shape_of_x = shape_op(x) + shape_of_y = shape_op(y) + rx = broadcast_gradient_args(shape_of_x, shape_of_y) + # if input shape is the same as dout shape, do not need to reduce + reduce_dx = dx + reduce_dy = dy + if rx[0]: + # if dx is scalar whose shape is (), do not need reduce + if shape_op(dx): + dx = reduce_sum(dx, rx[0]) + reduce_dx = reshape(dx, shape_of_x) + if rx[1]: + # if dy is scalar whose shape is (), do not need reduce + if shape_op(dy): + dy = reduce_sum(dy, rx[1]) + reduce_dy = reshape(dy, shape_of_y) + return reduce_dx, reduce_dy + + +def _sum_grad(x, axis, dout): + """Grad definition for `Sum` operation.""" + # input_shape = [2, 3] axis = [1] + input_shape = shape_op(x) + # output_shape_kept_dims = [2, 1] + output_shape_kept_dims = reduced_shape(input_shape, axis) + # tile_scaling = [1, 3] + tile_scaling = tuple_div(input_shape, output_shape_kept_dims) + grad = reshape(dout, output_shape_kept_dims) + return tile(grad, tile_scaling) + + +def _min_or_max_grad(x, axis, out, dout): + """Grad definition for `Min` and `Max` operations.""" + # input_shape = [2, 3] axis = [1] + input_shape = shape_op(x) + # output_shape_kept_dims = [2, 1] + output_shape_kept_dims = reduced_shape(input_shape, axis) + y = reshape(out, output_shape_kept_dims) + grad = reshape(dout, output_shape_kept_dims) + indicators = F.cast(F.equal(y, x), F.dtype(grad)) + min_num = F.cast(F.scalar_to_array(1e-24), F.dtype(grad)) + num_selected = reshape(reduce_sum(indicators, axis), output_shape_kept_dims) + min_num + return indicators / num_selected * grad + + +def _argmin_or_argmax_grad(x, axis, keep_dims, op, out, dout): + """ArgMinWiwhValue and ArgMaxWithValue grad.""" + expand = P.ExpandDims() + x_shape = F.shape(x) + x_dim = len(x_shape) + x_axis = axis + if x_axis < 0: + x_axis = axis + x_dim + onehot_axis = x_axis + depth = x_shape[x_axis] + if keep_dims: + dout_expand = dout[1] + out = op(x) + else: + dout_expand = expand(dout[1], onehot_axis) + if onehot_axis >= len(shape_op(out[0])): + onehot_axis = -1 + onehot = P.OneHot(onehot_axis) + type_x = F.dtype(x) + on_value = F.cast(F.scalar_to_array(1.0), type_x) + off_value = F.cast(F.scalar_to_array(0.0), type_x) + dx = dout_expand * onehot(out[0], depth, on_value, off_value) + return dx + + +@bprop_getters.register(P.MatMul) +def bprop_matmul(self): + """Grad definition for `MatMul` operation.""" + ta = self.transpose_a + tb = self.transpose_b + mul1 = P.MatMul(transpose_a=(ta and tb), + transpose_b=(ta or (not tb))) + mul2 = P.MatMul(transpose_a=((not ta) or tb), + transpose_b=(ta and tb)) + + def bprop(x, w, out, dout): + if ta: + dx = mul1(w, dout) + else: + dx = mul1(dout, w) + if tb: + dw = mul2(dout, x) + else: + dw = mul2(x, dout) + return dx, dw + return bprop + + +@bprop_getters.register(P.BatchMatMul) +def bprop_batchmatmul(self): + """Grad definition for `BatchMatMul` operation.""" + ta = self.transpose_a + tb = self.transpose_b + mul1 = P.BatchMatMul(transpose_a=(ta and tb), + transpose_b=(ta or (not tb))) + mul2 = P.BatchMatMul(transpose_a=((not ta) or tb), + transpose_b=(ta and tb)) + + def bprop(x, w, out, dout): + if ta: + dx = mul1(w, dout) + else: + dx = mul1(dout, w) + if tb: + dw = mul2(dout, x) + else: + dw = mul2(x, dout) + return dx, dw + return bprop + + +@bprop_getters.register(P.TensorAdd) +def get_bprop_tensor_add(self): + """Grad definition for `TensorAdd` operation.""" + + def bprop(x, y, out, dout): + return binop_grad_common(x, y, dout, dout) + return bprop + + +@bprop_getters.register(P.Neg) +def get_bprop_neg(self): + """Grad definition for `Neg` operation.""" + neg_grad = P.Neg() + + def bprop(x, out, dout): + dx = neg_grad(dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.Sub) +def get_bprop_sub(self): + """Grad definition for `Sub` operation.""" + neg_func = P.Neg() + + def bprop(x, y, out, dout): + return binop_grad_common(x, y, dout, neg_func(dout)) + return bprop + + +@bprop_getters.register(P.Mul) +def get_bprop_mul(self): + """Grad definition for `Mul` operation.""" + mul_func = P.Mul() + + def bprop(x, y, out, dout): + bc_dx = mul_func(dout, y) + bc_dy = mul_func(dout, x) + return binop_grad_common(x, y, bc_dx, bc_dy) + return bprop + + +@bprop_getters.register(P.RealDiv) +def get_bprop_real_div(self): + """Grad definition for `RealDiv` operation.""" + div_op = P.RealDiv() + neg = P.Neg() + mul_op = P.Mul() + + def bprop(x, y, out, dout): + bc_x = div_op(dout, y) + bc_y = neg(mul_op(bc_x, out)) + return binop_grad_common(x, y, bc_x, bc_y) + return bprop + + +@bprop_getters.register(P.Div) +def get_bprop_div(self): + """Grad definition for `Div` operation.""" + div_op = P.Div() + neg = P.Neg() + mul_op = P.Mul() + + def bprop(x, y, out, dout): + bc_x = div_op(dout, y) + bc_y = neg(mul_op(bc_x, out)) + return binop_grad_common(x, y, bc_x, bc_y) + return bprop + + +@bprop_getters.register(P.Floor) +def get_bprop_floor(self): + """Grad definition for `floor` operation.""" + fill_ = P.Fill() + shape_ = P.Shape() + dtype_ = P.DType() + + def bprop(x, out, dout): + bc_x = fill_(dtype_(x), shape_(x), 0.) + return (bc_x,) + return bprop + + +@bprop_getters.register(P.FloorDiv) +def get_bprop_floordiv(self): + """Grad definition for `FloorDiv` operation.""" + div_op = P.FloorDiv() + neg = P.Neg() + mul_op = P.Mul() + + def bprop(x, y, out, dout): + bc_x = div_op(dout, y) + bc_y = neg(mul_op(bc_x, out)) + return binop_grad_common(x, y, bc_x, bc_y) + return bprop + + +@bprop_getters.register(P.Square) +def get_bprop_square(self): + """Grad definition for `Square` operation.""" + mul_func = P.Mul() + fill_func = P.Fill() + dtype = P.DType() + + def bprop(x, out, dout): + temp = mul_func(dout, x) + dx = mul_func(fill_func(dtype(temp), shape_op(x), 2.0), temp) + return (dx,) + return bprop + + +@bprop_getters.register(P.Sqrt) +def get_bprop_sqrt(self): + """Grad definition for `Sqrt` operation.""" + mul_func = P.Mul() + fill_func = P.Fill() + div_op = P.RealDiv() + sqrt = P.Sqrt() + dtype = P.DType() + + def bprop(x, out, dout): + temp = div_op(fill_func(dtype(x), shape_op(x), 0.5), sqrt(x)) + dx = mul_func(dout, temp) + return (dx,) + return bprop + + +@bprop_getters.register(P.Rsqrt) +def get_bprop_rsqrt(self): + """Grad definition for `Rsqrt` operation.""" + + def bprop(x, out, dout): + grad = F.fill(F.dtype(x), F.shape(x), -0.5) / (F.sqrt(x)*x) + dx = dout * grad + return (dx,) + return bprop + + +@bprop_getters.register(P.Reciprocal) +def get_bprop_reciprocal(self): + """Grad definition for `Reciprocal` operation.""" + neg = P.Neg() + mul = P.Mul() + square = P.Square() + reciprocal = P.Reciprocal() + + def bprop(x, out, dout): + g = neg(reciprocal(square(x))) + dx = mul(dout, g) + return (dx,) + return bprop + + +@bprop_getters.register(P.Log) +def get_bprop_log(self): + """Grad definition for `Log` operation.""" + reciprocal = P.Reciprocal() + + def bprop(x, out, dout): + g = reciprocal(x) + dx = g * dout + return dx, 0 + return bprop + + +@bprop_getters.register(P.Pow) +def get_bprop_pow(self): + """Grad definition for `Pow` operation.""" + pow_ = P.Pow() + cast = P.Cast() + dtype = P.DType() + + def bprop(x, power, out, dout): + g = cast(F.tuple_to_array((power,)), dtype(x)) * pow_(x, power-1.0) + dx = g * dout + return dx, 0 + return bprop + + +@bprop_getters.register(P.Exp) +def get_bprop_exp(self): + """Grad definition for `Exp` operation.""" + exp_ = P.Exp() + + def bprop(x, out, dout): + g = exp_(x) + dx = g * dout + return (dx,) + return bprop + + +@bprop_getters.register(P.Minimum) +def get_bprop_minimum(self): + """Grad definition for `Minimum` operation.""" + input_grad = G.MinimumGrad() + + def bprop(x, y, out, dout): + dx, dy = input_grad(x, y, dout) + return dx, dy + return bprop + + +@bprop_getters.register(P.Maximum) +def get_bprop_maximum(self): + """Grad definition for `Maximum` operation.""" + input_grad = G.MaximumGrad() + + def bprop(x, y, out, dout): + dx, dy = input_grad(x, y, dout) + return dx, dy + return bprop + + +@bprop_getters.register(P.ReduceSum) +def get_bprop_reducesum(self): + """Grad definition for `ReduceSum` operation.""" + + def bprop(x, axis, out, dout): + dx = _sum_grad(x, axis, dout) + return dx, zeros_like(axis) + return bprop + + +@bprop_getters.register(P.CumSum) +def get_bprop_cumsum(self): + """Grad definition for `CumSum` operation.""" + cumsum = P.CumSum(exclusive=self.exclusive, reverse=not self.reverse) + def bprop(x, axis, out, dout): + return cumsum(dout, axis), zeros_like(axis) + return bprop + + +@constexpr +def _split_shape_index(input_shape, axis): + """Calculate reduce_prod grad transpose indices and perm shape.""" + rank = len(input_shape) + if isinstance(axis, int): + axis = tuple([axis]) + reduction_indices = tuple([(i + rank) % rank for i in axis]) + other_indices = tuple(set(range(rank)) - set(reduction_indices)) + reduced_num = reduce(lambda x, y: x * y, [input_shape[i] for i in reduction_indices]) + other_num = reduce(lambda x, y: x * y, [input_shape[i] for i in other_indices]) + perm = reduction_indices + other_indices + return tuple([reduced_num, other_num]), perm + + +@constexpr +def _invert_permutation(perm): + """Calculate invert permutation.""" + out = [0] * len(perm) + for i, value in enumerate(perm): + out[value] = i + return tuple(out) + + +@bprop_getters.register(P.ReduceProd) +def get_bprop_reduceprod(self): + """Grad definition for `ReduceProd` operation.""" + transpose = P.Transpose() + left_cumprod = P.CumProd(exclusive=True) + right_cumprod = P.CumProd(exclusive=True, reverse=True) + + def bprop(x, axis, out, dout): + """Grad definition for `Product` operation.""" + # Expand dout to full input shape + input_shape = shape_op(x) + output_shape_kept_dims = reduced_shape(input_shape, axis) + dout = reshape(dout, output_shape_kept_dims) + tile_scaling = tuple_div(input_shape, output_shape_kept_dims) + grad = tile(dout, tile_scaling) + + # Pack all reduced dimensions into a single one, so we can perform the cumprod ops. + pack_shape, perm = _split_shape_index(input_shape, axis) + permuted = transpose(x, perm) + permuted_shape = shape_op(permuted) + reshaped = reshape(permuted, pack_shape) + + # Calculate product, leaving out the current entry + left = left_cumprod(reshaped, 0) + right = right_cumprod(reshaped, 0) + y = reshape(left * right, permuted_shape) + + # Invert the transpose and reshape operations. + # Make sure to set the statically known shape information through a reshape. + out = transpose(y, _invert_permutation(perm)) * grad + dx = reshape(out, input_shape) + return dx, zeros_like(axis) + return bprop + + +@bprop_getters.register(P.CumProd) +def get_bprop_cumprod(self): + """Grad definition for `CumProd` operation.""" + cumprod = P.CumProd(exclusive=self.exclusive, reverse=self.reverse) + cumsum = P.CumSum(exclusive=self.exclusive, reverse=not self.reverse) + + def bprop(x, axis, out, dout): + """Grad definition for `Product` operation.""" + # This will fails when x contains 0 + prod = cumprod(x, axis) + out = cumsum(prod * dout, axis) + return out / x, zeros_like(axis) + return bprop + + +@bprop_getters.register(P.ReduceAll) +def get_bprop_reduceall(self): + """Grad definition for `ReduceAll` operation.""" + + def bprop(x, axis, out, dout): + return zeros_like(x), zeros_like(axis) + return bprop + + +@bprop_getters.register(P.ReduceMax) +def get_bprop_reducemax(self): + """Grad definition for `Max` operation.""" + + def bprop(x, axis, out, dout): + dx = _min_or_max_grad(x, axis, out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.ArgMaxWithValue) +def get_bprop_argmaxwithvalue(self): + """Grad definition for `ArgMaxWithValue` operation.""" + axis = self.axis + keep_dims = self.keep_dims + op = P.ArgMaxWithValue(axis) + + def bprop(x, out, dout): + dx = _argmin_or_argmax_grad(x, axis, keep_dims, op, out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.ReduceMin) +def get_bprop_reducemin(self): + """Grad definition for `ReduceMin` operation.""" + + def bprop(x, axis, out, dout): + dx = _min_or_max_grad(x, axis, out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.ArgMinWithValue) +def get_bprop_argminwithvalue(self): + """Generate bprop for ArgMinWithValue""" + axis = self.axis + keep_dims = self.keep_dims + op = P.ArgMinWithValue(axis) + + def bprop(x, out, dout): + dx = _argmin_or_argmax_grad(x, axis, keep_dims, op, out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.ReduceMean) +def get_bprop_reduce_mean(self): + """Grad definition for `ReduceMean` operation.""" + div_op = P.RealDiv() + cast = P.Cast() + dtype = P.DType() + + def bprop(x, axis, out, dout): + grad = _sum_grad(x, axis, dout) + div_shape = F.shape_mul(shape_op(x)) / F.shape_mul(shape_op(out)) + dx = div_op(grad, cast(F.scalar_to_array(div_shape), dtype(grad))) + return dx, zeros_like(axis) + return bprop + + +@bprop_getters.register(P.Equal) +def get_bprop_equal(self): + """Grad definition for `Equal` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + + return bprop + + +@bprop_getters.register(P.NotEqual) +def get_bprop_not_equal(self): + """Grad definition for `NotEqual` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.Greater) +def get_bprop_greater(self): + """Grad definition for `Greater` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.GreaterEqual) +def get_bprop_greater_equal(self): + """Grad definition for `GreaterEqual` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.Less) +def get_bprop_less(self): + """Grad definition for `Less` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.LessEqual) +def get_bprop_less_equal(self): + """Grad definition for `LessEqual` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.LogicalNot) +def get_bprop_logical_not(self): + """Grad definition for `LogicalNot` operation.""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.LogicalAnd) +def get_bprop_logical_and(self): + """Grad definition for `LogicalAnd` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.LogicalOr) +def get_bprop_logical_or(self): + """Grad definition for `LogicalOr` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.NPUAllocFloatStatus) +def get_bprop_npu_alloc_float_status(self): + """Grad definition for `NPUAllocFloatStatus` operation.""" + + def bprop(out, dout): + return () + return bprop + + +@bprop_getters.register(P.NPUGetFloatStatus) +def get_bprop_npu_get_float_status(self): + """Grad definition for `NPUGetFloatStatus` operation.""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.NPUClearFloatStatus) +def get_bprop_npu_clear_float_status(self): + """Grad definition for `NPUClearFloatStatus` operation.""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.AssignAdd) +def get_bprop_assign_add(self): + """Grad definition for `AssignAdd` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.AssignSub) +def get_bprop_assign_sub(self): + """Grad definition for `AssignSub` operation.""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop + + +@bprop_getters.register(P.Sin) +def get_bprop_sin(self): + """Grad definition for `Sin` operation.""" + cos = P.Cos() + + def bprop(x, out, dout): + dx = dout*cos(x) + return (dx,) + return bprop + + +@bprop_getters.register(P.Cos) +def get_bprop_cos(self): + """Grad definition for `Cos` operation.""" + sin = P.Sin() + neg = P.Neg() + + def bprop(x, out, dout): + dx = dout*neg(sin(x)) + return (dx,) + return bprop + + +@bprop_getters.register(P.ACos) +def get_bprop_acos(self): + """Grad definition for `ACos` operation.""" + input_grad = G.ACosGrad() + + def bprop(x, out, dout): + dx = input_grad(x, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.Abs) +def get_bprop_abs(self): + """Grad definition for `Abs` operation.""" + abs_grad = G.AbsGrad() + + def bprop(x, out, dout): + dx = abs_grad(x, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.ScalarCast) +def get_bprop_scalar_cast(self): + """Generate bprop for ScalarCast""" + + def bprop(x, t, out, dout): + return F.scalar_cast(dout, F.typeof(x)), zeros_like(t) + return bprop + + +@bprop_getters.register(P.AddN) +def get_bprop_scalar_addn(self): + """Generate bprop for AddN""" + + def bprop(x, out, dout): + dx = () + for _ in range(len(x)): + dx = dx + (dout,) + return dx + return bprop + + +@bprop_getters.register(P.Sign) +def get_bprop_sign(self): + """Generate bprop for Sign""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.Round) +def get_bprop_round(self): + """Generate bprop for Round""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop diff --git a/mindspore/ops/_grad/grad_nn_ops.py b/mindspore/ops/_grad/grad_nn_ops.py new file mode 100755 index 00000000000..bad99351a50 --- /dev/null +++ b/mindspore/ops/_grad/grad_nn_ops.py @@ -0,0 +1,508 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Define the grad rules of neural network related operations.""" + +from .. import functional as F +from .. import operations as P +from ..operations import _grad_ops as G +from ..composite.multitype_ops.zeros_like_impl import zeros_like +from .grad_base import bprop_getters + + +@bprop_getters.register(P.BiasAdd) +def get_bprop_bias_add(self): + """Grad definition for `BiasAdd` operation.""" + bias_grad = G.BiasAddGrad() + + def bprop(x, w, out, dout): + return dout, bias_grad(dout) + return bprop + + +@bprop_getters.register(P.Conv2D) +def get_bprop_conv2d(self): + """Grad definition for `Conv2D` operation.""" + input_grad = P.Conv2DBackpropInput( + self.out_channel, self.kernel_size, self.pad_mode, self.pad, self.pad_list, mode=self.mode, + dilation=self.dilation, stride=self.stride, group=self.group + ) + filter_grad = G.Conv2DBackpropFilter( + self.out_channel, self.kernel_size, self.pad_mode, self.pad, self.pad_list, mode=self.mode, + dilation=self.dilation, stride=self.stride, group=self.group + ) + get_shape = P.Shape() + + def bprop(x, w, out, dout): + dx = input_grad(dout, w, get_shape(x)) + dw = filter_grad(dout, x, get_shape(w)) + return dx, dw + return bprop + + +@bprop_getters.register(P.DepthwiseConv2dNative) +def get_bprop_depthwise_conv2d_native(self): + """Grad definition for `DepthwiseConv2dNative` operation.""" + input_grad = G.DepthwiseConv2dNativeBackpropInput( + self.channel_multiplier, self.kernel_size, self.pad_mode, self.pad, self.pads, self.mode, self.stride, + self.dilation, self.group + ) + filter_grad = G.DepthwiseConv2dNativeBackpropFilter( + self.channel_multiplier, self.kernel_size, self.pad_mode, self.pad, self.pads, self.mode, self.stride, + self.dilation, self.group + ) + get_shape = P.Shape() + + def bprop(x, w, out, dout): + dx = input_grad(get_shape(x), w, dout) + dw = filter_grad(x, get_shape(w), dout) + return dx, dw + return bprop + + +@bprop_getters.register(P.MaxPoolWithArgmax) +def get_bprop_max_pool_with_argmax(self): + """Grad definition for `MaxPoolWithArgmax` operation.""" + maxpool_grad = G.MaxPoolGradWithArgmax( + pad_mode=self.pad_mode, + window=self.window, + pad=self.pad, + stride=self.stride, + data_mode=self.data_mode, + ceil_mode=self.ceil_mode, + alpha=self.alpha, + beta=self.beta) + + def bprop(x, out, dout): + dx = maxpool_grad(x, dout[0], out[1]) + return (dx,) + return bprop + + +@bprop_getters.register(P.MaxPool) +def get_bprop_max_pool_grad(self): + """Grad definition for `MaxPool` operation.""" + maxpool_grad = G.MaxPoolGrad( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + + def bprop(x, out, dout): + dx = maxpool_grad(x, out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.AvgPool) +def get_bprop_avg_pool_grad(self): + """Grad definition for `AvgPool` operation.""" + avgpool_grad = G.AvgPoolGrad( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + shape_op = P.Shape() + + avgpool_grad_gpu = G.AvgPoolGradGpu( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + + def bprop(x, out, dout): + dx = avgpool_grad(shape_op(x), dout) + return (dx,) + + def bprop_gpu(x, out, dout): + dx = avgpool_grad_gpu(x, out, dout) + return (dx,) + + # the parameter of AvgPoolGrad in GPU and TBE/CPU is not same + if self.target == "GPU": + bprop_fn = bprop_gpu + else: + bprop_fn = bprop + + return bprop_fn + + +@bprop_getters.register(P.DropoutGenMask) +def get_bprop_dropout_gen_mask(self): + """Grad definition for `DropoutGenMask` operation.""" + + def bprop(shape, keep_prob, out, dout): + return (zeros_like(shape), zeros_like(keep_prob)) + return bprop + + +@bprop_getters.register(P.DropoutDoMask) +def get_bprop_dropout_do_mask(self): + """Grad definition for `DropoutDoMask` operation.""" + do_mask = P.DropoutDoMask() + + def bprop(x, y, keep_prob, out, dout): + return (do_mask(dout, y, keep_prob), zeros_like(y), zeros_like(keep_prob)) + return bprop + + +@bprop_getters.register(P.ReLU) +def get_bprop_relu(self): + """Grad definition for `ReLU` operation.""" + input_grad = G.ReluGrad() + + def bprop(x, out, dout): + dx = input_grad(dout, out) + return (dx,) + return bprop + + +@bprop_getters.register(P.ReLU6) +def get_bprop_relu6(self): + """Grad definition for `ReLU6` operation.""" + input_grad = G.ReLU6Grad() + + def bprop(x, out, dout): + dx = input_grad(dout, x) + return (dx,) + return bprop + + +@bprop_getters.register(P.Elu) +def get_bprop_elu(self): + """Grad definition for `Elu` operation.""" + input_grad = G.EluGrad() + + def bprop(x, out, dout): + dx = input_grad(dout, x) + return (dx,) + return bprop + + +@bprop_getters.register(P.Sigmoid) +def get_bprop_sigmoid(self): + """Grad definition for `Sigmoid` operation.""" + input_grad = G.SigmoidGrad() + + def bprop(x, out, dout): + dx = input_grad(out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.Softmax) +def get_bprop_softmax(self): + """Grad definition for `Softmax` operation.""" + sum_func = P.ReduceSum(keep_dims=True) + sub = P.Sub() + mul = P.Mul() + axis = self.axis + + def bprop(x, out, dout): + dx = mul(sub(dout, sum_func(mul(dout, out), axis)), out) + return (dx,) + return bprop + + +@bprop_getters.register(P.LogSoftmax) +def get_bprop_log_softmax(self): + """Grad definition for `LogSoftmax` operation.""" + logsoftmax_grad = G.LogSoftmaxGrad(self.axis) + + def bprop(x, out, dout): + dx = logsoftmax_grad(out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.Tanh) +def get_bprop_tanh(self): + """Grad definition for `Tanh` operation.""" + logsoftmax_grad = G.TanhGrad() + + def bprop(x, out, dout): + dx = logsoftmax_grad(out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.Gelu) +def get_bprop_gelu(self): + """Grad definition for `Gelu` operation.""" + input_grad = G.GeluGrad() + + def bprop(x, out, dout): + dx = input_grad(dout, x, out) + return (dx,) + return bprop + + +@bprop_getters.register(P.FusedBatchNorm) +def get_bprop_fused_batch_norm(self): + """Grad definition for `FusedBatchNorm` operation.""" + input_grad = G.FusedBatchNormGrad(self.epsilon, self.momentum) + + def bprop(x, scale, b, mean, variance, out, dout): + saved_mean = out[3] + saved_variance = out[4] + out = input_grad(dout[0], x, scale, saved_mean, saved_variance) + dx = out[0] + dscale = out[1] + dbias = out[2] + return dx, dscale, dbias, zeros_like(mean), zeros_like(variance) + return bprop + + +@bprop_getters.register(P.BatchNorm) +def get_bprop_batch_norm(self): + """Grad definition for `BatchNorm` operation.""" + is_training = self.is_training + input_grad = G.BatchNormGrad(is_training, self.epsilon) + + def bprop(x, scale, b, mean, variance, out, dout): + if is_training: + saved_reserve_1 = out[3] + saved_reserve_2 = out[4] + saved_reserve_3 = out[5] + else: + saved_reserve_1 = mean + saved_reserve_2 = variance + saved_reserve_3 = variance + out = input_grad(dout[0], x, scale, saved_reserve_1, saved_reserve_2, saved_reserve_3) + dx = out[0] + dscale = out[1] + dbias = out[2] + return dx, dscale, dbias, zeros_like(mean), zeros_like(variance) + return bprop + + +@bprop_getters.register(P.LayerNorm) +def get_bprop_layer_norm(self): + """Grad definition for `LayerNorm` operation.""" + layer_norm_grad = G.LayerNormGrad(self.begin_norm_axis, self.begin_params_axis) + + def bprop(x, gamma, beta, out, dout): + dx, d_gamma, d_beta = layer_norm_grad(x, dout[0], out[2], out[1], gamma) + return dx, d_gamma, d_beta + return bprop + + +@bprop_getters.register(P.L2Normalize) +def get_bprop_l2normalize(self): + """Grad definition for `L2Normalize` operation.""" + input_grad = G.L2NormalizeGrad(self.axis, self.epsilon) + + def bprop(x, out, dout): + dx = input_grad(x, out, dout) + return (dx,) + return bprop + + +@bprop_getters.register(P.SoftmaxCrossEntropyWithLogits) +def get_bprop_softmax_cross_entropy_with_logits(self): + """Grad definition for `SoftmaxCrossEntropyWithLogits` operation.""" + expand = P.ExpandDims() + + def bprop(logits, labels, out, dout): + grad = out[1] + grad = grad * expand(dout[0], -1) + return grad, zeros_like(labels) + return bprop + + +@bprop_getters.register(P.SparseSoftmaxCrossEntropyWithLogits) +def get_bprop_sparse_softmax_cross_entropy_with_logits(self): + """Grad definition for `SparseSoftmaxCrossEntropyWithLogits` operation.""" + is_grad = self.is_grad + grad_op = P.SparseSoftmaxCrossEntropyWithLogits(is_grad=True) + + def bprop(logits, labels, out, dout): + grad = out[0] + if not is_grad: + # if construct use loss + grad = grad_op(logits, labels) + grad = F.depend(grad, out) + grad = grad * dout + return grad, zeros_like(labels) + return bprop + + +@bprop_getters.register(P.ResizeBilinear) +def get_bprop_resize_bilinear(self): + """Grad definition for `ResizeBilinear` operation.""" + resize_grad = G.ResizeBilinearGrad(self.align_corners) + + def bprop(x, out, dout): + dx = resize_grad(dout, x) + return (dx,) + return bprop + + +@bprop_getters.register(P.OneHot) +def get_bprop_onehot(self): + """Grad definition for `OneHot` operation.""" + + def bprop(indices, depth, on_value, off_value, out, dout): + return zeros_like(indices), zeros_like(depth) + return bprop + + +@bprop_getters.register(P.TopK) +def get_bprop_top_kv2(self): + """Grad definition for `TopK` operation.""" + scatter = P.ScatterNd() + expand_dims = P.ExpandDims() + shape_op = P.Shape() + + def bprop(input_x, k, out, dout): + indices = out[1] + indices = expand_dims(indices, -1) + updates = dout[0] + shapes = shape_op(input_x) + return scatter(indices, updates, shapes), zeros_like(k) + return bprop + + +@bprop_getters.register(P.SmoothL1Loss) +def get_bprop_smooth_l1_loss(self): + """Grad definition for `SmoothL1Loss` operation.""" + grad = G.SmoothL1LossGrad(self.sigma) + + def bprop(prediction, target, out, dout): + dx = grad(prediction, target, dout) + return dx, zeros_like(target) + + return bprop + + +@bprop_getters.register(P.PReLU) +def get_bprop_prelu(self): + """Grad definition for `PReLU` operation.""" + grad = G.PReLUGrad() + + def bprop(x, w, out, dout): + dx, dw = grad(dout, x, w) + return dx, dw + + return bprop + + +@bprop_getters.register(P.LSTM) +def get_bprop_lstm(self): + """Grad definition for `LSTM` operation.""" + lstm_grad_data = G.LSTMGradData( + input_size=self.input_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + has_bias=self.has_bias, + bidirectional=self.bidirectional, + dropout=self.dropout + ) + + lstm_grad_weight = G.LSTMGradWeight( + input_size=self.input_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + has_bias=self.has_bias, + bidirectional=self.bidirectional, + dropout=self.dropout + ) + + def bprop(x, hx, cx, w, out, dout): + y, _, _, reserve, state = out + dy, dhy, dcy, _, _ = dout + dx, dhx, dcx = lstm_grad_data(y, dy, dhy, dcy, w, hx, cx, reserve, state) + dw = lstm_grad_weight(F.depend(x, dx), hx, y, reserve, state) + return dx, dhx, dcx, dw + return bprop + + +@bprop_getters.register(P.SigmoidCrossEntropyWithLogits) +def get_bprop_sigmoid_crossentropy_with_logits(self): + """Grad definition for `SigmoidCrossEntropyWithLogits` operation.""" + op = G.SigmoidCrossEntropyWithLogitsGrad() + + def bprop(x, y, out, dout): + dx = op(x, y, dout) + return (dx, zeros_like(y)) + return bprop + + +@bprop_getters.register(P.Pad) +def get_bprop_pad(self): + """Grad definition for `Pad` operation.""" + shape_op = P.Shape() + paddings = self.paddings + + def bprop(x, out, dout): + begin = () + for item in paddings: + begin += (item[0],) + shp = shape_op(x) + dx = P.Slice()(dout, begin, shp) + return (dx,) + return bprop + + +@bprop_getters.register(P.ROIAlign) +def get_bprop_roi_align(self): + """Grad definition for `ROIAlign` operation.""" + shape_op = P.Shape() + pooled_height = self.pooled_height + pooled_width = self.pooled_width + spatial_scale = self.spatial_scale + sample_num = self.sample_num + + def bprop(inputs, rois, out, dout): + rois_shape = shape_op(rois) + inputs_shape = shape_op(inputs) + dx = G.ROIAlignGrad(inputs_shape, + pooled_height, + pooled_width, + spatial_scale, + sample_num, + )(dout, rois) + return dx, zeros_like(rois_shape) + + return bprop + + +@bprop_getters.register(P.Conv2DBackpropInput) +def get_bprop_conv2d_backprop_input(self): + """Grad definition for `Conv2DBackpropInput` operation.""" + filter_grad = G.Conv2DBackpropFilter( + self.out_channel, self.kernel_size, self.pad_mode, self.pad, self.pad_list, mode=self.mode, + dilation=self.dilation, stride=self.stride, group=self.group + ) + input_grad = P.Conv2D( + self.out_channel, self.kernel_size, pad_mode=self.pad_mode.lower(), pad=self.pad, + dilation=self.dilation, stride=self.stride, group=self.group + ) + + def bprop(x, w, f_sizes, out, dout): + dx = input_grad(dout, w) + dw = filter_grad(x, dout, F.shape(w)) + return dx, dw + + return bprop + + +@bprop_getters.register(P.BinaryCrossEntropy) +def get_bprop_binary_cross_entropy(self): + """Grad definition for `BinaryCrossEntropy` operation.""" + grad = G.BinaryCrossEntropyGrad(self.reduction) + + def bprop(x, y, weight, out, dout): + dx = grad(x, y, dout, weight) + return dx, zeros_like(y), zeros_like(weight) + + return bprop diff --git a/mindspore/ops/_grad/grad_other_ops.py b/mindspore/ops/_grad/grad_other_ops.py new file mode 100644 index 00000000000..799b722ad8d --- /dev/null +++ b/mindspore/ops/_grad/grad_other_ops.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Generate bprop for other ops""" + +from .. import operations as P +from ..composite.multitype_ops.zeros_like_impl import zeros_like +from .grad_base import bprop_getters + +# Unused parameters are placeholders. + + +@bprop_getters.register(P.Assign) +def get_bprop_assign(self): + """Generate bprop for Assign""" + def bprop(x, y, out, dout): + return (x, zeros_like(y)) + return bprop + + +@bprop_getters.register(P.InvertPermutation) +def get_bprop_invert_permutation(self): + """Generate bprop for InvertPermutation""" + + def bprop(x, out, dout): + return (zeros_like(x),) + return bprop + + +@bprop_getters.register(P.IOU) +def get_bprop_iou(self): + """Generate bprop for IOU""" + + def bprop(x, y, out, dout): + return zeros_like(x), zeros_like(y) + return bprop diff --git a/mindspore/ops/_op_impl/__init__.py b/mindspore/ops/_op_impl/__init__.py new file mode 100644 index 00000000000..b8370cc64ef --- /dev/null +++ b/mindspore/ops/_op_impl/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Operators info register.""" + +from .akg.gpu import * +from .tbe import * + +__all__ = [] diff --git a/mindspore/ops/_op_impl/akg/__init__.py b/mindspore/ops/_op_impl/akg/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mindspore/ops/_op_impl/akg/gpu/__init__.py b/mindspore/ops/_op_impl/akg/gpu/__init__.py new file mode 100644 index 00000000000..2135794b5f9 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""__init__""" +from .cast import _cast_akg +from .equal import _equal_akg +from .mean import _simple_mean_akg +from .mean_grad import _simple_mean_grad_akg +from .mul import _mul_akg +from .relu6 import _relu6_akg +from .relu6_grad import _relu6_grad_akg +from .squeeze import _squeeze_akg +from .squeeze_grad import _squeeze_grad_akg +from .tile import _tile_akg diff --git a/mindspore/ops/_op_impl/akg/gpu/cast.py b/mindspore/ops/_op_impl/akg/gpu/cast.py new file mode 100644 index 00000000000..fb4b221be64 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/cast.py @@ -0,0 +1,57 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cast op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "Cast", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + { + "name": "dst_type", + "param_type": "required", + "type": "str" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _cast_akg(): + """Cast AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/equal.py b/mindspore/ops/_op_impl/akg/gpu/equal.py new file mode 100644 index 00000000000..c6cffdad24f --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/equal.py @@ -0,0 +1,62 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Equal op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "Equal", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + }, + { + "index": 1, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _equal_akg(): + """Equal AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/mean.py b/mindspore/ops/_op_impl/akg/gpu/mean.py new file mode 100644 index 00000000000..244af290bb4 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/mean.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SimpleMean op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "SimpleMean", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _simple_mean_akg(): + """SimpleMean AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/mean_grad.py b/mindspore/ops/_op_impl/akg/gpu/mean_grad.py new file mode 100644 index 00000000000..27c06746321 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/mean_grad.py @@ -0,0 +1,57 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SimpleMeanGrad op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "SimpleMeanGrad", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + { + "name": "input_shape", + "param_type": "required", + "type": "listInt" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "HEAD" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _simple_mean_grad_akg(): + """SimpleMeanGrad AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/mul.py b/mindspore/ops/_op_impl/akg/gpu/mul.py new file mode 100644 index 00000000000..d9e1a0b5d63 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/mul.py @@ -0,0 +1,62 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Mul op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "Mul", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + }, + { + "index": 1, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _mul_akg(): + """Mul AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/relu6.py b/mindspore/ops/_op_impl/akg/gpu/relu6.py new file mode 100644 index 00000000000..0de0a7e400d --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/relu6.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ReLU6 op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "ReLU6", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _relu6_akg(): + """ReLU6 AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/relu6_grad.py b/mindspore/ops/_op_impl/akg/gpu/relu6_grad.py new file mode 100644 index 00000000000..4d3c5e9a00b --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/relu6_grad.py @@ -0,0 +1,62 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ReLU6Grad op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "ReLU6Grad", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y_grad" + }, + { + "index": 1, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _relu6_grad_akg(): + """ReLU6Grad AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/squeeze.py b/mindspore/ops/_op_impl/akg/gpu/squeeze.py new file mode 100644 index 00000000000..9e766cdfd7e --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/squeeze.py @@ -0,0 +1,57 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Squeeze op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "Squeeze", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _squeeze_akg(): + """Squeeze AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/squeeze_grad.py b/mindspore/ops/_op_impl/akg/gpu/squeeze_grad.py new file mode 100644 index 00000000000..7584bd05f92 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/squeeze_grad.py @@ -0,0 +1,62 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SqueezeGrad op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "SqueezeGrad", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + { + "name": "x_shape", + "param_type": "required", + "type": "listInt" + }, + { + "name": "axis", + "param_type": "optional", + "type": "listInt" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y_grad" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _squeeze_grad_akg(): + """SqueezeGrad AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/akg/gpu/tile.py b/mindspore/ops/_op_impl/akg/gpu/tile.py new file mode 100644 index 00000000000..f110e9314e0 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/gpu/tile.py @@ -0,0 +1,57 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tile op""" +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "Tile", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "processor": "cuda", + "attr": [ + { + "name": "multiples", + "param_type": "required", + "type": "listInt" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32", "float16" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output" + } + ] +}""") +def _tile_akg(): + """Tile AutoDiff register""" + return diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py new file mode 100644 index 00000000000..b9b9af43967 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -0,0 +1,131 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""tbe ops""" +from .adam_apply_one_with_decay import _adam_apply_one_with_decay_tbe +from .add import _add_tbe +from .add_n import _add_n_tbe +from .apply_momentum import _apply_momentum_tbe +from .apply_adam import _apply_adam_tbe +from .assign import _assign_tbe +from .assign_add import _assign_add_tbe +from .assign_sub import _assign_sub_tbe +from .batch_matmul import _batch_matmul_tbe +from .batchnorm import _batch_norm_tbe +from .batchnorm_grad import _batch_norm_grad_tbe +from .bias_add import _bias_add_tbe +from .bias_add_grad import _bias_add_grad_tbe +from .cast import _cast_tbe +from .conv2d import _conv2d_tbe +from .conv2d_backprop_filter import _conv2d_backprop_filter_tbe +from .conv2d_backprop_input import _conv2d_backprop_input_tbe +from .dropout_do_mask import _dropout_do_mask_tbe +from .gelu import _gelu_tbe +from .gelu_grad import _gelu_grad_tbe +from .max_pool import _max_pool_tbe +from .max_pool_grad import _max_pool_grad_tbe +from .max_pool_grad_with_argmax import _max_pool_grad_with_argmax_tbe +from .max_pool_with_argmax import _max_pool_with_argmax_tbe +from .mul import _mul_tbe +from .real_div import _real_div_tbe +from .relu import _relu_tbe +from .relu_grad import _relu_grad_tbe +from .softmax_cross_entropy_with_logits import _softmax_cross_entropy_with_logits_tbe +from .sigmoid_cross_entropy_with_logits import _sigmoid_cross_entropy_with_logits_tbe +from .sigmoid_cross_entropy_with_logits_grad import _sigmoid_cross_entropy_with_logits_grad_tbe +from .tensor_add import _tensor_add_tbe +from .trans_data import _trans_data_tbe +from .topkv2 import _topk_v2_tbe +from .matmul import _matmul_tbe +from .sub import _sub_tbe +from .reduce_mean_d import _reduce_mean_d_tbe +from .scatter_nd import _scatter_nd_tbe +from .scatter_nd_d import _scatter_nd_d_tbe +from .reduce_mean import _reduce_mean_tbe +from .reshape import _reshape_tbe +from .expand_dims import _expand_dims_tbe +from .squeeze import _squeeze_tbe +from .tile import _tile_tbe +from .atomic_addr_clean import _atomic_addr_clean_tbe +from .gather_v2 import _gather_v2_tbe +from .bn_training_reduce import _bn_training_reduce_tbe +from .bn_training_reduce_grad import _bn_training_reduce_grad_tbe +from .bn_training_update import _bn_training_update_tbe +from .bn_training_update_grad import _bn_training_update_grad_tbe +from .reciprocal import _reciprocal_tbe +from .strideslice_d import _strided_slice_d_tbe +from .strideslicegrad_d import _strided_slice_grad_d_tbe +from .split_d import _split_d_tbe +from .exp import _exp_tbe +from .div import _div_tbe +from .log import _log_tbe +from .floor_div import _floor_div_tbe +from .zeros_like import _zeros_like_tbe +from .neg import _neg_tbe +from .npu_clear_float_status import _npu_clear_float_status_tbe +from .npu_get_float_status import _npu_get_float_status_tbe +from .npu_alloc_float_status import _npu_alloc_float_status_tbe +from .one_hot import _one_hot_tbe +from .equal import _equal_tbe +from .less import _less_tbe +from .less_equal import _less_equal_tbe +from .logical_and import _logical_and_tbe +from .logical_not import _logical_not_tbe +from .logical_or import _logical_or_tbe +from .reduce_max import _reduce_max_tbe +from .reduce_sum import _reduce_sum_tbe +from .tanh import _tanh_tbe +from .tanh_grad import _tanh_grad_tbe +from .softmax import _softmax_tbe +from .square import _square_tbe +from .sqrt import _sqrt_tbe +from .transpose_d import _transpose_d_tbe +from .unsorted_segment_sum import _unsorted_segment_sum_tbe +from .logsoftmax_grad import _logsoftmax_grad_tbe +from .logsoftmax import _logsoftmax_tbe +from .select import _select_tbe +from .pow import _pow_tbe +from .maximum import _maximum_tbe +from .minimum import _minimum_tbe +from .minimum_grad import _minimum_grad_tbe +from .maximum_grad import _maximum_grad_tbe +from .concat import _concat_tbe +from .slice import _slice_tbe +from .greater import _greater_tbe +from .clip_by_norm_no_div_sum import _clip_by_norm_no_div_sum_tbe +from .clip_by_value import _clip_by_value_tbe +from .layer_norm_beta_gamma_backprop import _layer_norm_beta_gamma_backprop_tbe +from .layer_norm import _layer_norm_tbe +from .layer_norm_grad import _layer_norm_grad_tbe +from .layer_norm_x_backprop import _layer_norm_x_backprop_tbe +from .square_sum_v1 import _square_sum_v1_tbe +from .square_sum_v2 import _square_sum_v2_tbe +from .confusion_transpose_d import _confusion_transpose_d_tbe +from .confusion_softmax_grad import _confusion_softmax_grad_tbe +from .lamb_update_with_lr_v2 import _lamb_update_with_lr_v2_tbe +from .lamb_next_mv import _lamb_next_mv_tbe +from .lamb_next_mv_with_decay_v1 import _lamb_next_mv_with_decay_v1_tbe +from .lamb_update_with_lr import _lamb_update_with_lr_tbe +from .rsqrt import _rsqrt_tbe +from .sigmoid import _sigmoid_tbe +from .sigmoid_grad import _sigmoid_grad_tbe +from .resize_nearest_neighbor_d import _resize_nearest_neighbor_d_tbe +from .resize_nearest_neighbor_grad_d import _resize_nearest_neighbor_grad_d_tbe +from .pad_d import _pad_d_tbe +from .arg_max_with_value import _arg_max_with_value_tbe +from .arg_min_with_value import _arg_min_with_value_tbe +from .fused_mul_add import _fused_mul_add_tbe +from .fused_mul_add_n import _fused_mul_add_n_tbe +from .fused_mul_apply_momentum import _fused_mul_apply_momentum_tbe diff --git a/mindspore/ops/_op_impl/tbe/adam_apply_one_with_decay.py b/mindspore/ops/_op_impl/tbe/adam_apply_one_with_decay.py new file mode 100644 index 00000000000..d1c43ca957f --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/adam_apply_one_with_decay.py @@ -0,0 +1,221 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""AdamApplyOneWithDecay op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "AdamApplyOneWithDecay", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "adam_apply_one_with_decay.so", + "compute_cost": 10, + "kernel_name": "adam_apply_one_with_decay", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input0", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input3", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input4", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 5, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "mul0_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 6, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "mul1_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 7, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "mul2_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 8, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "mul3_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 9, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "mul4_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 10, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "add2_y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output0", + "need_compile": true, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output1", + "need_compile": true, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output2", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _adam_apply_one_with_decay_tbe(): + """AdamApplyOneWithDecay TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/add.py b/mindspore/ops/_op_impl/tbe/add.py new file mode 100644 index 00000000000..95c31d89744 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/add.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Add op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Add", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "add.so", + "compute_cost": 10, + "kernel_name": "add", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", + "float", "int32", "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _add_tbe(): + """Add TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/add_n.py b/mindspore/ops/_op_impl/tbe/add_n.py new file mode 100644 index 00000000000..9177ed14c7e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/add_n.py @@ -0,0 +1,74 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""AddN op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "AddN", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "add_n.so", + "compute_cost": 10, + "kernel_name": "add_n", + "partial_flag": true, + "attr": [ + { + "name": "n", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float","int32","int32","int32" + ], + "format": [ + "DefaultFormat","NC1HWC0","FracZ","FRACTAL_NZ", + "DefaultFormat","NC1HWC0","FracZ","FRACTAL_NZ","DefaultFormat","NC1HWC0","FracZ" + ], + "name": "x", + "need_compile": false, + "param_type": "dynamic", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float","int32","int32","int32" + ], + "format": [ + "DefaultFormat","NC1HWC0","FracZ","FRACTAL_NZ", + "DefaultFormat","NC1HWC0","FracZ","FRACTAL_NZ","DefaultFormat","NC1HWC0","FracZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _add_n_tbe(): + """AddN TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_adam.py b/mindspore/ops/_op_impl/tbe/apply_adam.py new file mode 100644 index 00000000000..ae6b7d782ed --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_adam.py @@ -0,0 +1,195 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyAdam op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Adam", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "apply_adam.so", + "compute_cost": 10, + "kernel_name": "apply_adam", + "partial_flag": true, + "attr": [ + { + "name": "use_locking", + "param_type": "optional", + "type": "bool", + "value": "true,false", + "default_value":"false" + }, + { + "name": "use_nesterov", + "param_type": "optional", + "type": "bool", + "value": "true,false", + "default_value":"false" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "m", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "v", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "beta1_power", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "beta2_power", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 5, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "lr", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 6, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "beta1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 7, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "beta2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 8, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "epsilon", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 8, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "grad", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _apply_adam_tbe(): + """ApplyAdam TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_momentum.py b/mindspore/ops/_op_impl/tbe/apply_momentum.py new file mode 100644 index 00000000000..f2c6f5b15ef --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_momentum.py @@ -0,0 +1,125 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyMomentum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ApplyMomentum", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "apply_momentum.so", + "compute_cost": 10, + "kernel_name": "apply_momentum", + "partial_flag": true, + "attr": [ + { + "name": "use_nesterov", + "param_type": "optional", + "type": "bool", + "value": "true,false", + "default_value":"false" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "DefaultFormat", "FracZ", "C1HWNCoC0" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "DefaultFormat", "FracZ", "C1HWNCoC0" + ], + "name": "accum", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat" + ], + "name": "lr", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "DefaultFormat", "FracZ", "C1HWNCoC0" + ], + "name": "grad", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float16","float16","float16","float16","float","float","float", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat" + ], + "name": "momentum", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "NC1HWC0", "C1HWNCoC0", "DefaultFormat", "FracZ", "NC1HWC0", "DefaultFormat", "FracZ", "C1HWNCoC0" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _apply_momentum_tbe(): + """ApplyMomentum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/arg_max_with_value.py b/mindspore/ops/_op_impl/tbe/arg_max_with_value.py new file mode 100644 index 00000000000..e5ffe3d36f8 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/arg_max_with_value.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ArgMaxWithValue op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ArgMaxWithValue", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "arg_max_with_value.so", + "compute_cost": 10, + "kernel_name": "arg_max_with_value", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "indice", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "values", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _arg_max_with_value_tbe(): + """ArgMaxWithValue TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/arg_min_with_value.py b/mindspore/ops/_op_impl/tbe/arg_min_with_value.py new file mode 100644 index 00000000000..3d66b4534d7 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/arg_min_with_value.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ArgMinWithValue op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ArgMinWithValue", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "arg_min_with_value.so", + "compute_cost": 10, + "kernel_name": "arg_min_with_value", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "indice", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "values", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _arg_min_with_value_tbe(): + """ArgMinWithValue TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/assign.py b/mindspore/ops/_op_impl/tbe/assign.py new file mode 100644 index 00000000000..610a221a1ca --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/assign.py @@ -0,0 +1,106 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Assign op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Assign", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "assign.so", + "compute_cost": 10, + "kernel_name": "assign", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32", "uint32", "uint32", "uint32", "uint32", "int8", + "int8", "int8", "int8", "uint8", "uint8", "uint8", "uint8", "int16", "int16", "int16", + "int16", "uint16", "uint16", "uint16", "uint16", "int64", "int64", "int64", "int64", + "uint64", "uint64", "uint64", "uint64", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "FRACTAL_NZ" + ], + "name": "resource", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", "int32", + "int32", "int32", "uint32", "uint32", "uint32", "uint32", "int8", "int8", "int8", "int8", "uint8", + "uint8", "uint8", "uint8", "int16", "int16", "int16", "int16", "uint16", "uint16", "uint16", + "uint16", "int64", "int64", "int64", "int64", "uint64", "uint64", "uint64", "uint64", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "FRACTAL_NZ" + ], + "name": "value", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32", "uint32", "uint32", "uint32", "uint32", "int8", "int8", "int8", + "int8", "uint8", "uint8", "uint8", "uint8", "int16", "int16", "int16", "int16", "uint16", + "uint16", "uint16", "uint16", "int64", "int64", "int64", "int64", + "uint64", "uint64", "uint64", "uint64", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "FRACTAL_NZ" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _assign_tbe(): + """Assign TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/assign_add.py b/mindspore/ops/_op_impl/tbe/assign_add.py new file mode 100644 index 00000000000..94e0e781f53 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/assign_add.py @@ -0,0 +1,93 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""AssignAdd op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "AssignAdd", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "assignadd.so", + "compute_cost": 10, + "kernel_name": "assignadd", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", "int32", + "int32", "int32", "int8", "int8", "int8", "int8", "uint8", "uint8", "uint8", "uint8", "int64", + "int64", "int64", "int64" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "ref", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", "int32", + "int32", "int32", "int8", "int8", "int8", "int8", "uint8", "uint8", "uint8", "uint8", "int64", + "int64", "int64", "int64" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "value", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", "int32", + "int32", "int32", "int8", "int8", "int8", "int8", "uint8", "uint8", "uint8", "uint8", "int64", + "int64", "int64", "int64" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "output_ref", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _assign_add_tbe(): + """AssignAdd TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/assign_sub.py b/mindspore/ops/_op_impl/tbe/assign_sub.py new file mode 100644 index 00000000000..85104f6eb32 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/assign_sub.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""AssignSub op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "AssignSub", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "assign_sub.so", + "compute_cost": 10, + "kernel_name": "assign_sub", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int32", "int8", "uint8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float", "int32", "int8", "uint8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "value", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int32", "int8", "uint8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "out_ref", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _assign_sub_tbe(): + """AssignSub TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/atomic_addr_clean.py b/mindspore/ops/_op_impl/tbe/atomic_addr_clean.py new file mode 100644 index 00000000000..90186e6526c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/atomic_addr_clean.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""AtomicAddrClean op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "AtomicAddrClean", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "atomic_addr_clean.so", + "compute_cost": 10, + "kernel_name": "atomic_addr_clean", + "partial_flag": true, + "attr": [ + { + "name": "automic_add_mem_size", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + ], + "outputs": [ + ] +}""") +def _atomic_addr_clean_tbe(): + """AtomicAddrClean TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/batch_matmul.py b/mindspore/ops/_op_impl/tbe/batch_matmul.py new file mode 100644 index 00000000000..668791b6599 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/batch_matmul.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchMatMul op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BatchMatMul", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "batch_matmul.so", + "compute_cost": 10, + "kernel_name": "batch_matmul", + "partial_flag": true, + "attr": [ + { + "name": "transpose_x1", + "param_type": "required", + "type": "bool", + "value": "all" + }, + { + "name": "transpose_x2", + "param_type": "required", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","int32","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","FRACTAL_NZ","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","int32","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","FRACTAL_NZ","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","int32","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "bias", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","int32","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","FRACTAL_NZ","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _batch_matmul_tbe(): + """BatchMatMul TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/batchnorm.py b/mindspore/ops/_op_impl/tbe/batchnorm.py new file mode 100644 index 00000000000..263fcfb0f2f --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/batchnorm.py @@ -0,0 +1,187 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchNorm op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BatchNorm", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "batch_norm.so", + "compute_cost": 10, + "kernel_name": "batch_norm", + "partial_flag": true, + "attr": [ + { + "name": "epsilon", + "param_type": "required", + "type": "float", + "value": "all" + }, + { + "name": "data_format", + "param_type": "required", + "type": "str", + "value": "all" + }, + { + "name": "is_training", + "param_type": "required", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0", "DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float","float","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat", "NC1HWC0" + ], + "name": "scale", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float","float","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "offset", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float","float","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "mean", + "need_compile": false, + "param_type": "optional", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float","float","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "variance", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat","NC1HWC0", "DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required" + }, + { + "index": 1, + "dtype": [ + "float","float","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "batch_mean", + "param_type": "required" + }, + { + "index": 2, + "dtype": [ + "float", "float", "float", "float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "batch_variance", + "param_type": "required" + }, + { + "index": 3, + "dtype": [ + "float", "float", "float", "float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_1", + "param_type": "optional" + }, + { + "index": 4, + "dtype": [ + "float", "float", "float", "float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_2", + "param_type": "optional" + }, + { + "index": 5, + "dtype": [ + "float", "float", "float", "float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_3", + "param_type": "optional" + } + ] +}""") +def _batch_norm_tbe(): + """BatchNorm TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/batchnorm_grad.py b/mindspore/ops/_op_impl/tbe/batchnorm_grad.py new file mode 100644 index 00000000000..cc560c52833 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/batchnorm_grad.py @@ -0,0 +1,194 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchNormGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BatchNormGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "batchnormgrad.so", + "compute_cost": 10, + "kernel_name": "batchnormgrad", + "partial_flag": true, + "attr": [ + { + "name": "epsilon", + "param_type": "optional", + "type": "float", + "value": "all" + }, + { + "name": "data_format", + "param_type": "optional", + "type": "str", + "value": "all" + }, + { + "name": "is_training", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "y_backprop", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "scale", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 5, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_3", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "x_backprop", + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "scale_backprop", + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "offset_backprop", + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_4", + "param_type": "optional", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "reserve_space_5", + "param_type": "optional", + "shape": "all" + } + ] +}""") +def _batch_norm_grad_tbe(): + """BatchNormGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/bias_add.py b/mindspore/ops/_op_impl/tbe/bias_add.py new file mode 100644 index 00000000000..9081ed2c130 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/bias_add.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BiasAdd op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BiasAdd", + "imply_type": "TBE", + "fusion_type": "COMMREDUCE", + "async_flag": false, + "binfile_name": "bias_add.so", + "compute_cost": 10, + "kernel_name": "bias_add", + "partial_flag": true, + "attr": [ + { + "name": "data_format", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "int32", "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "int32", "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "bias", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "int32", "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _bias_add_tbe(): + """BiasAdd TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/bias_add_grad.py b/mindspore/ops/_op_impl/tbe/bias_add_grad.py new file mode 100644 index 00000000000..4a24e361bbb --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/bias_add_grad.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BiasAddGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BiasAddGrad", + "imply_type": "TBE", + "fusion_type": "COMMREDUCE", + "async_flag": false, + "binfile_name": "biasaddgrad.so", + "compute_cost": 10, + "kernel_name": "biasaddgrad", + "partial_flag": true, + "attr": [ + { + "name": "data_format", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","FRACTAL_NZ","DefaultFormat" + ], + "name": "out_backprop", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "output", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _bias_add_grad_tbe(): + """BiasAddGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/bn_training_reduce.py b/mindspore/ops/_op_impl/tbe/bn_training_reduce.py new file mode 100644 index 00000000000..3228ccd7917 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/bn_training_reduce.py @@ -0,0 +1,73 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchNorm op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BNTrainingReduce", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "bn_training_reduce.so", + "compute_cost": 10, + "kernel_name": "bn_training_reduce", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float","float" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "sum", + "param_type": "required" + }, + { + "index": 1, + "dtype": [ + "float","float" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "square_sum", + "param_type": "required" + } + ] +}""") +def _bn_training_reduce_tbe(): + """BNTrainingReduce TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py b/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py new file mode 100644 index 00000000000..3bb43fbb94a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py @@ -0,0 +1,147 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchNormGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BNTrainingReduceGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "bn_training_reduce_grad.so", + "compute_cost": 10, + "kernel_name": "bn_training_reduce_grad", + "partial_flag": true, + "attr": [ + { + "name": "epsilon", + "param_type": "optional", + "type": "float", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "grads", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "x_norm", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "diff_scale", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "diff_offset", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "scale", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 5, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "batch_mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 6, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "batch_variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _bn_training_reduce_grad_tbe(): + """BNTrainingReduceGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/bn_training_update.py b/mindspore/ops/_op_impl/tbe/bn_training_update.py new file mode 100644 index 00000000000..9d6838f0e43 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/bn_training_update.py @@ -0,0 +1,213 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchNormGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BNTrainingUpdate", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "bn_training_update.so", + "compute_cost": 10, + "kernel_name": "bn_training_update", + "partial_flag": true, + "attr": [ + { + "name": "factor", + "param_type": "optional", + "type": "float", + "value": "all" + }, + { + "name": "epsilon", + "param_type": "optional", + "type": "float", + "value": "all" + }, + { + "name": "isRef", + "param_type": "optional", + "type": "bool", + "default_value":"true", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "sum", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "square_sum", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "scale", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "offset", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 5, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 6, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "batch_mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "batch_variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _bn_training_update_tbe(): + """BNTrainingUpdate TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py b/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py new file mode 100644 index 00000000000..802ef6e91fa --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py @@ -0,0 +1,122 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchNormGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BNTrainingUpdateGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "bn_training_update_grad.so", + "compute_cost": 10, + "kernel_name": "bn_training_update_grad", + "partial_flag": true, + "attr": [ + { + "name": "epsilon", + "param_type": "optional", + "type": "float", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "grads", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "batch_mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "batch_variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "diff_scale", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float", "float" + ], + "format": [ + "NC1HWC0","NC1HWC0" + ], + "name": "diff_offset", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _bn_training_update_grad_tbe(): + """BNTrainingUpdateGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/cast.py b/mindspore/ops/_op_impl/tbe/cast.py new file mode 100644 index 00000000000..e443d776aec --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/cast.py @@ -0,0 +1,82 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Cast op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Cast", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "cast.so", + "compute_cost": 10, + "kernel_name": "cast", + "partial_flag": true, + "attr": [ + { + "name": "dst_type", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", + "int32", "int32", "int32", "int32", "int32", + "int8", "int8", "int8", "uint8", "uint8", "uint8", + "bool", "bool", "bool", "bool", "float16" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float", "int32", "float16", "int32", + "float16", "float", "int8", "uint8", "bool", + "float16", "float", "int32", "float16", "float", "int32", + "float16", "float", "int32", "uint8", "uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _cast_tbe(): + """Cast TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/clip_by_norm_no_div_sum.py b/mindspore/ops/_op_impl/tbe/clip_by_norm_no_div_sum.py new file mode 100644 index 00000000000..859315fb7b1 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/clip_by_norm_no_div_sum.py @@ -0,0 +1,103 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ClipByNormNoDivSum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ClipByNormNoDivSum", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "clip_by_norm_no_div_sum.so", + "compute_cost": 10, + "kernel_name": "clip_by_norm_no_div_sum", + "partial_flag": true, + "attr":[ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input3", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output_y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _clip_by_norm_no_div_sum_tbe(): + """ClipByNormNoDivSum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/clip_by_value.py b/mindspore/ops/_op_impl/tbe/clip_by_value.py new file mode 100644 index 00000000000..02ec0158a9f --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/clip_by_value.py @@ -0,0 +1,98 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ClipByValue op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ClipByValue", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "clip_by_value.so", + "compute_cost": 10, + "kernel_name": "clip_by_value", + "partial_flag": true, + "attr":[ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "clip_value_min", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "clip_value_max", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _clip_by_value_tbe(): + """ClipByValue TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/concat.py b/mindspore/ops/_op_impl/tbe/concat.py new file mode 100644 index 00000000000..3e5c577476d --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/concat.py @@ -0,0 +1,154 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Concat op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Concat", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "concat_d.so", + "compute_cost": 10, + "kernel_name": "concat_d", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", + "float16", + "float", + "float", + "int32", + "int32", + "int8", + "int8", + "int16", + "int16", + "int64", + "int64", + "uint8", + "uint8", + "uint16", + "uint16", + "uint32", + "uint32", + "uint64", + "uint64", + "bool", + "bool" + ], + "format": [ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0" + ], + "name": "input_values", + "need_compile": false, + "param_type": "dynamic", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", + "float16", + "float", + "float", + "int32", + "int32", + "int8", + "int8", + "int16", + "int16", + "int64", + "int64", + "uint8", + "uint8", + "uint16", + "uint16", + "uint32", + "uint32", + "uint64", + "uint64", + "bool", + "bool" + ], + "format": [ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "NC1HWC0" + ], + "name": "output_data", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _concat_tbe(): + """Concat TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/confusion_softmax_grad.py b/mindspore/ops/_op_impl/tbe/confusion_softmax_grad.py new file mode 100644 index 00000000000..cbd518d5414 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/confusion_softmax_grad.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ConfusionSoftmaxGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ConfusionSoftmaxGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "confusion_softmax_grad.so", + "compute_cost": 10, + "kernel_name": "confusion_softmax_grad", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "NC1HWC0", "FRACTAL_NZ", "DefaultFormat", "NC1HWC0" + ], + "name": "grad", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "NC1HWC0", "FRACTAL_NZ", "DefaultFormat", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "NC1HWC0", "FRACTAL_NZ", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _confusion_softmax_grad_tbe(): + """ConfusionSoftmaxGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py b/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py new file mode 100644 index 00000000000..db35107b5dd --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py @@ -0,0 +1,92 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ConfusionTransposeD op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ConfusionTransposeD", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "confusion_transpose_d.so", + "compute_cost": 10, + "kernel_name": "confusion_transpose_d", + "partial_flag": true, + "attr":[ + { + "name":"perm", + "param_type":"required", + "type":"listInt", + "value":"all" + }, + { + "name":"shape", + "param_type":"required", + "type":"listInt", + "value":"all" + }, + { + "name":"transpose_first", + "param_type":"required", + "type":"bool", + "value":"all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", + "uint64", "float16", "float", "int8", "int16", "int32", "int64", "uint8", "uint16", + "uint32", "uint64" + ], + "format": [ + "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", + "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", + "uint64", "float16", "float", "int8", "int16", "int32", "int64", "uint8", "uint16", + "uint32", "uint64" + ], + "format": [ + "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", + "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "FRACTAL_NZ", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _confusion_transpose_d_tbe(): + """ConfusionTransposeD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/conv2d.py b/mindspore/ops/_op_impl/tbe/conv2d.py new file mode 100644 index 00000000000..52a9eac1fad --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/conv2d.py @@ -0,0 +1,127 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Conv2D op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Conv2D", + "imply_type": "TBE", + "fusion_type": "CONVLUTION", + "async_flag": false, + "binfile_name": "conv2d.so", + "compute_cost": 10, + "kernel_name": "conv2d", + "partial_flag": true, + "attr": [ + { + "name": "stride", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "pad", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "dilation", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "offset_a", + "param_type": "optional", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FracZ" + ], + "name": "filter", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "bias", + "need_compile": false, + "param_type": "optional", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "int8" + ], + "format": [ + "DefaultFormat" + ], + "name": "offset_w", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _conv2d_tbe(): + """Conv2D TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py b/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py new file mode 100644 index 00000000000..2c1397c0b1b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py @@ -0,0 +1,102 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Conv2DBackpropFilter op""" +from mindspore.ops.op_info_register import op_info_register + + +# map to tbe kernel name conv2d_backprop_filter_d +@op_info_register("""{ + "op_name": "Conv2DBackpropFilter", + "imply_type": "TBE", + "fusion_type": "CONVLUTION", + "async_flag": false, + "binfile_name": "conv2d_backprop_filter_d.so", + "compute_cost": 10, + "kernel_name": "conv2d_backprop_filter_d", + "partial_flag": true, + "attr": [ + { + "name": "filter_sizes", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "stride", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "pad_mode", + "param_type": "required", + "type": "str", + "value": "all" + }, + { + "name": "dilation", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "out_backprop", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "FracZ" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _conv2d_backprop_filter_tbe(): + """Conv2DBackpropFilter TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/conv2d_backprop_input.py b/mindspore/ops/_op_impl/tbe/conv2d_backprop_input.py new file mode 100644 index 00000000000..d61989e4722 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/conv2d_backprop_input.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Conv2DBackpropInput op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Conv2DBackpropInput", + "imply_type": "TBE", + "fusion_type": "CONVLUTION", + "async_flag": false, + "binfile_name": "conv2d_backprop_input_d.so", + "compute_cost": 10, + "kernel_name": "conv2d_backprop_input_d", + "partial_flag": true, + "attr": [ + { + "name": "input_sizes", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "stride", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "pad_mode", + "param_type": "required", + "type": "str", + "value": "all" + }, + { + "name": "dilation", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "out_backprop", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FracZ" + ], + "name": "filter", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _conv2d_backprop_input_tbe(): + """Conv2DBackpropInput TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/div.py b/mindspore/ops/_op_impl/tbe/div.py new file mode 100644 index 00000000000..c71d6f38c4a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/div.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Div op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Div", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "div.so", + "compute_cost": 10, + "kernel_name": "div", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _div_tbe(): + """Div TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/dropout_do_mask.py b/mindspore/ops/_op_impl/tbe/dropout_do_mask.py new file mode 100644 index 00000000000..5f4557a4f5d --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/dropout_do_mask.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""DropoutdoMask op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "DropoutDoMask", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "drop_out_do_mask.so", + "compute_cost": 10, + "kernel_name": "drop_out_do_mask", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "uint8","uint8","uint8","uint8","uint8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "mask", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "keep_prob", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _dropout_do_mask_tbe(): + """DropoutdoMask TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/equal.py b/mindspore/ops/_op_impl/tbe/equal.py new file mode 100644 index 00000000000..db2e152c27a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/equal.py @@ -0,0 +1,79 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Equal op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Equal", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "equal.so", + "compute_cost": 10, + "kernel_name": "equal", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool","bool","bool","bool","bool","bool","bool","bool","bool","bool" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required" + } + ] +}""") +def _equal_tbe(): + """Equal TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/exp.py b/mindspore/ops/_op_impl/tbe/exp.py new file mode 100644 index 00000000000..e5f34e67b08 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/exp.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Exp op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Exp", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "exp.so", + "compute_cost": 10, + "kernel_name": "exp", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _exp_tbe(): + """Exp TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/expand_dims.py b/mindspore/ops/_op_impl/tbe/expand_dims.py new file mode 100644 index 00000000000..462a676e4f2 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/expand_dims.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ExpandDims op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ExpandDims", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "expand_dims.so", + "compute_cost": 10, + "kernel_name": "expand_dims", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _expand_dims_tbe(): + """ExpandDims TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/floor_div.py b/mindspore/ops/_op_impl/tbe/floor_div.py new file mode 100644 index 00000000000..fdc49e38054 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/floor_div.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FloorDiv op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "FloorDiv", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "floordiv.so", + "compute_cost": 10, + "kernel_name": "floordiv", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _floor_div_tbe(): + """FloorDiv TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/fused_mul_add.py b/mindspore/ops/_op_impl/tbe/fused_mul_add.py new file mode 100644 index 00000000000..96e18a89c74 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/fused_mul_add.py @@ -0,0 +1,106 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedMulAdd op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "FusedMulAdd", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "fused_mul_add.so", + "compute_cost": 10, + "kernel_name": "fused_mul_add", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x3", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _fused_mul_add_tbe(): + """FusedMulAdd TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/fused_mul_add_n.py b/mindspore/ops/_op_impl/tbe/fused_mul_add_n.py new file mode 100644 index 00000000000..a4046a253bf --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/fused_mul_add_n.py @@ -0,0 +1,99 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedMulAddN op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "FusedMulAddN", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "fused_mul_add_n.so", + "compute_cost": 10, + "kernel_name": "fused_mul_add_n", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x3", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _fused_mul_add_n_tbe(): + """FusedMulAddN TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/fused_mul_apply_momentum.py b/mindspore/ops/_op_impl/tbe/fused_mul_apply_momentum.py new file mode 100644 index 00000000000..e303ee042f3 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/fused_mul_apply_momentum.py @@ -0,0 +1,150 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""FusedMulApplyMomentum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "FusedMulApplyMomentum", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "fused_mul_apply_momentum.so", + "compute_cost": 10, + "kernel_name": "fused_mul_apply_momentum", + "partial_flag": true, + "attr": [ + { + "name": "use_nesterov", + "param_type": "optional", + "type": "bool", + "value": "true,false", + "default_value":"false" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "accum", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "lr", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "momentum", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 5, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16", + "float","float","float","float" + ], + "format": [ + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ", + "NC1HWC0","C1HWNCoC0","DefaultFormat","FracZ" + ], + "name": "var", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _fused_mul_apply_momentum_tbe(): + """FusedMulApplyMomentum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/gather_v2.py b/mindspore/ops/_op_impl/tbe/gather_v2.py new file mode 100644 index 00000000000..b0e14e99c01 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/gather_v2.py @@ -0,0 +1,107 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""AddN op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "GatherV2", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "gather_v2_d.so", + "compute_cost": 10, + "kernel_name": "gather_v2_d", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16","float16","float16", + "float","float","float","float","float","float", + "int32","int32","int32", "int32","int32","int32", + "uint8","uint8","uint8","uint8","uint8","uint8", + "int8","int8", "int8","int8","int8", "int8" + ], + "format": [ + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "int32","int32","int32","int64","int64","int64", + "int32","int32","int32","int64","int64","int64", + "int32","int32","int32","int64","int64","int64", + "int32","int32","int32","int64","int64","int64", + "int32","int32","int32","int64","int64","int64" + ], + "format": [ + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ" + ], + "name": "indices", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float16","float16","float16", + "float","float","float","float","float","float", + "int32","int32","int32", "int32","int32","int32", + "uint8","uint8","uint8","uint8","uint8","uint8", + "int8","int8", "int8","int8","int8", "int8" + ], + "format": [ + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ", + "DefaultFormat","NC1HWC0","FracZ","DefaultFormat","NC1HWC0","FracZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _gather_v2_tbe(): + """GatherV2 TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/gelu.py b/mindspore/ops/_op_impl/tbe/gelu.py new file mode 100644 index 00000000000..80933125470 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/gelu.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Gelu op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Gelu", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "gelu.so", + "compute_cost": 10, + "kernel_name": "gelu", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "FracZ","FracZ","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float16","float16","float16","float16","float","float","float","float" + ], + "format": [ + "FracZ","FracZ","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _gelu_tbe(): + """Gelu TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/gelu_grad.py b/mindspore/ops/_op_impl/tbe/gelu_grad.py new file mode 100644 index 00000000000..9b358262e08 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/gelu_grad.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""GeluGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "GeluGrad", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "gelu_grad.so", + "compute_cost": 10, + "kernel_name": "gelu_grad", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "dy", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "z", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _gelu_grad_tbe(): + """GeluGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/greater.py b/mindspore/ops/_op_impl/tbe/greater.py new file mode 100644 index 00000000000..09ee0e31af6 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/greater.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Greater op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Greater", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "greater.so", + "compute_cost": 10, + "kernel_name": "greater", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool","bool","bool","bool","bool","bool","bool","bool","bool","bool" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _greater_tbe(): + """Greater TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/lamb_next_mv.py b/mindspore/ops/_op_impl/tbe/lamb_next_mv.py new file mode 100644 index 00000000000..b432b47c3d9 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lamb_next_mv.py @@ -0,0 +1,292 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LambNextMV op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"LambNextMV", + "imply_type":"TBE", + "fusion_type":"ELEMWISE", + "async_flag":false, + "binfile_name":"lamb_next_m_v.so", + "compute_cost":10, + "kernel_name":"lamb_next_m_v", + "partial_flag":true, + "attr":[], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input3", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":3, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input4", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":4, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input5", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":5, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input6", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":6, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input7", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":7, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input8", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":8, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input9", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":9, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx0", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":10, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":11, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":12, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx3", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output3", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":3, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output4", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _lamb_next_mv_tbe(): + """LambNextMV TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/lamb_next_mv_with_decay_v1.py b/mindspore/ops/_op_impl/tbe/lamb_next_mv_with_decay_v1.py new file mode 100644 index 00000000000..bf5b6cd0e78 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lamb_next_mv_with_decay_v1.py @@ -0,0 +1,292 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LambNextMVWithDecayV1 op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"LambNextMVWithDecayV1", + "imply_type":"TBE", + "fusion_type":"OPAQUE", + "async_flag":false, + "binfile_name":"lamb_next_m_v_with_decay_v1.so", + "compute_cost":10, + "kernel_name":"lamb_next_m_v_with_decay_v1", + "partial_flag":true, + "attr":[], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input3", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":3, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input4", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":4, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input5", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":5, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input6", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":6, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input7", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":7, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input8", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":8, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input9", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":9, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx0", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":10, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":11, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":12, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"inputx3", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output3", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":3, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output4", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _lamb_next_mv_with_decay_v1_tbe(): + """LambNextMVWithDecayV1 TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/lamb_update_with_lr.py b/mindspore/ops/_op_impl/tbe/lamb_update_with_lr.py new file mode 100644 index 00000000000..a5062e74e2c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lamb_update_with_lr.py @@ -0,0 +1,187 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LambUpdateWithLr op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"LambUpdateWithLR", + "imply_type":"TBE", + "fusion_type":"ELEMWISE", + "async_flag":false, + "binfile_name":"lamb_update_with_lr.so", + "compute_cost":10, + "kernel_name":"lamb_update_with_lr", + "partial_flag":true, + "attr":[], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input3", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":3, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input4", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":4, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input5", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":5, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input6", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":6, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input7", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":7, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input8", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":8, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"input9", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"output_y", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _lamb_update_with_lr_tbe(): + """LambUpdateWithLr TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/lamb_update_with_lr_v2.py b/mindspore/ops/_op_impl/tbe/lamb_update_with_lr_v2.py new file mode 100644 index 00000000000..0900775b07e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lamb_update_with_lr_v2.py @@ -0,0 +1,157 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LambUpdateWithLrV2 op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"LambUpdateWithLrV2", + "imply_type":"TBE", + "fusion_type":"ELEMWISE", + "async_flag":false, + "binfile_name":"lamb_update_with_lr_v2.so", + "compute_cost":10, + "kernel_name":"lamb_update_with_lr_v2", + "partial_flag":true, + "attr":[], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"x1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"x2", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"x3", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":3, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"x4", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":4, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"x5", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":5, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"greater_y", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":6, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"select_e", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", + "float32" + ], + "format":[ + "DefaultFormat", + "DefaultFormat" + ], + "name":"y", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _lamb_update_with_lr_v2_tbe(): + """LambUpdateWithLrV2 TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/layer_norm.py b/mindspore/ops/_op_impl/tbe/layer_norm.py new file mode 100644 index 00000000000..5fd4a6387b2 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/layer_norm.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LayerNorm op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LayerNorm", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "layer_norm.so", + "compute_cost": 10, + "kernel_name": "layer_norm", + "partial_flag": true, + "attr": [ + { + "name": "begin_norm_axis", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "begin_params_axis", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "gamma", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "beta", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + + ], + "name": "mean", + "param_type": "required" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + + ], + "name": "variance", + "param_type": "required" + } + ] +}""") +def _layer_norm_tbe(): + """LayerNorm TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py b/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py new file mode 100644 index 00000000000..cdf0dad7445 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py @@ -0,0 +1,118 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LayerNormBetaGammaBackprop op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LayerNormBetaGammaBackprop", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "layer_norm_beta_gamma_backprop.so", + "compute_cost": 10, + "kernel_name": "layer_norm_beta_gamma_backprop", + "partial_flag": true, + "attr": [ + { + "name": "shape_gamma", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "dy", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "pd_gamma", + "param_type": "required" + }, + { + "index": 1, + "dtype": [ + "float","float","float","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "pd_beta", + "param_type": "required" + } + ] +}""") +def _layer_norm_beta_gamma_backprop_tbe(): + """LayerNormBetaGammaBackprop TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_grad.py b/mindspore/ops/_op_impl/tbe/layer_norm_grad.py new file mode 100644 index 00000000000..6ba46566150 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/layer_norm_grad.py @@ -0,0 +1,137 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LayerNormGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LayerNormGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "layer_norm_grad.so", + "compute_cost": 10, + "kernel_name": "layer_norm_grad", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "dy", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "gamma", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "pd_x", + "param_type": "required" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "pd_gamma", + "param_type": "required" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float","float" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "pd_beta", + "param_type": "required" + } + ] +}""") +def _layer_norm_grad_tbe(): + """LayerNormGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py b/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py new file mode 100644 index 00000000000..0557fdebc24 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py @@ -0,0 +1,115 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LayerNormXBackprop op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LayerNormXBackprop", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "layer_norm_x_backprop.so", + "compute_cost": 10, + "kernel_name": "layer_norm_x_backprop", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "dy", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "variance", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "mean", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 4, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "DefaultFormat","DefaultFormat","NC1HWC0","DefaultFormat","DefaultFormat","NC1HWC0" + ], + "name": "gamma", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float16","float","float","float" + ], + "format": [ + "FRACTAL_NZ","DefaultFormat","NC1HWC0","FRACTAL_NZ","DefaultFormat","NC1HWC0" + ], + "name": "pd_x", + "param_type": "required" + } + ] +}""") +def _layer_norm_x_backprop_tbe(): + """LayerNormXBackprop TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/less.py b/mindspore/ops/_op_impl/tbe/less.py new file mode 100644 index 00000000000..6e48d60341e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/less.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Less op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Less", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "less.so", + "compute_cost": 10, + "kernel_name": "less", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool","bool","bool","bool","bool","bool","bool","bool","bool","bool" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _less_tbe(): + """Less TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/less_equal.py b/mindspore/ops/_op_impl/tbe/less_equal.py new file mode 100644 index 00000000000..556389fa0e1 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/less_equal.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LessEqual op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LessEqual", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "less_equal.so", + "compute_cost": 10, + "kernel_name": "less_equal", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool","bool","bool","bool","bool","bool","bool","bool","bool","bool" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat", + "NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _less_equal_tbe(): + """LessEqual TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/log.py b/mindspore/ops/_op_impl/tbe/log.py new file mode 100644 index 00000000000..72a55757a28 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/log.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Log op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Log", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "log.so", + "compute_cost": 10, + "kernel_name": "log", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _log_tbe(): + """Log TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/logical_and.py b/mindspore/ops/_op_impl/tbe/logical_and.py new file mode 100644 index 00000000000..da4862450f0 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/logical_and.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LogicalAnd op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LogicalAnd", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "logical_and.so", + "compute_cost": 10, + "kernel_name": "logical_and", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _logical_and_tbe(): + """LogicalAnd TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/logical_not.py b/mindspore/ops/_op_impl/tbe/logical_not.py new file mode 100644 index 00000000000..4fe8094ffb8 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/logical_not.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LogicalNot op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LogicalNot", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "logical_not.so", + "compute_cost": 10, + "kernel_name": "logical_not", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _logical_not_tbe(): + """LogicalNot TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/logical_or.py b/mindspore/ops/_op_impl/tbe/logical_or.py new file mode 100644 index 00000000000..0f21bb61b0b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/logical_or.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LogicalOr op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LogicalOr", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "logical_or.so", + "compute_cost": 10, + "kernel_name": "logical_or", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _logical_or_tbe(): + """LogicalOr TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/logsoftmax.py b/mindspore/ops/_op_impl/tbe/logsoftmax.py new file mode 100644 index 00000000000..03e86579196 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/logsoftmax.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LogSoftmax op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LogSoftmax", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "log_softmax.so", + "compute_cost": 10, + "kernel_name": "log_softmax", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "logits", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "logsoftmax", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _logsoftmax_tbe(): + """LogSoftMaxGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/logsoftmax_grad.py b/mindspore/ops/_op_impl/tbe/logsoftmax_grad.py new file mode 100644 index 00000000000..f6858e9530c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/logsoftmax_grad.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LogSoftmaxGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "LogSoftmaxGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "log_softmax_grad.so", + "compute_cost": 10, + "kernel_name": "log_softmax_grad", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "grad", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _logsoftmax_grad_tbe(): + """LogSoftMaxGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/matmul.py b/mindspore/ops/_op_impl/tbe/matmul.py new file mode 100644 index 00000000000..d18d2cde48e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/matmul.py @@ -0,0 +1,102 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MatMul op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "MatMul", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "matmul.so", + "compute_cost": 10, + "kernel_name": "matmul", + "partial_flag": true, + "attr": [ + { + "name": "transpose_a", + "param_type": "required", + "type": "bool", + "value": "all" + }, + { + "name": "transpose_b", + "param_type": "required", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","int32" + ], + "format": [ + "FRACTAL_NZ","FRACTAL_NZ","DefaultFormat","DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float16","float","int32" + ], + "format": [ + "FRACTAL_NZ","FRACTAL_NZ","DefaultFormat","DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float","float","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x3", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float","int32" + ], + "format": [ + "FRACTAL_NZ","FRACTAL_NZ","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _matmul_tbe(): + """Mul TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool.py b/mindspore/ops/_op_impl/tbe/max_pool.py new file mode 100644 index 00000000000..6b10bc8d9ba --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool.py @@ -0,0 +1,87 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPool op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "MaxPool", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "max_pool.so", + "compute_cost": 10, + "kernel_name": "max_pool", + "partial_flag": true, + "attr": [ + { + "name": "ksize", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "strides", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "padding", + "param_type": "required", + "type": "str", + "value": "all" + }, + { + "name": "data_format", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "input_data", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "output_data", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _max_pool_tbe(): + """MaxPool TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool_grad.py b/mindspore/ops/_op_impl/tbe/max_pool_grad.py new file mode 100644 index 00000000000..7c942d01e05 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_grad.py @@ -0,0 +1,106 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "MaxPoolGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "max_pool_grad.so", + "compute_cost": 10, + "kernel_name": "max_pool_grad", + "partial_flag": true, + "attr": [ + { + "name": "ksize", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "strides", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "padding", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "grad", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "y", + "param_type": "required" + } + ] +}""") +def _max_pool_grad_tbe(): + """MaxPoolGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool_grad_with_argmax.py b/mindspore/ops/_op_impl/tbe/max_pool_grad_with_argmax.py new file mode 100644 index 00000000000..a167ef85f8c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_grad_with_argmax.py @@ -0,0 +1,108 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolGradWithArgmax op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "MaxPoolGradWithArgmax", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "max_pool_grad_with_argmax.so", + "compute_cost": 10, + "kernel_name": "max_pool_grad_with_argmax", + "partial_flag": true, + "attr": [ + { + "name": "window", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "stride", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "pad_mode", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "grad", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "uint16", "int64" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "argmax", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16" + ], + "format": [ + "NC1HWC0", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _max_pool_grad_with_argmax_tbe(): + """MaxPoolGradWithArgmax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool_with_argmax.py b/mindspore/ops/_op_impl/tbe/max_pool_with_argmax.py new file mode 100644 index 00000000000..04d0eeb92c1 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_with_argmax.py @@ -0,0 +1,95 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolWithArgmax op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "MaxPoolWithArgmax", + "imply_type": "TBE", + "fusion_type": "CONVLUTION", + "async_flag": false, + "binfile_name": "max_pool_with_argmax.so", + "compute_cost": 10, + "kernel_name": "max_pool_with_argmax", + "partial_flag": true, + "attr": [ + { + "name": "window", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "stride", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "pad_mode", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "uint16" + ], + "format": [ + "NC1HWC0" + ], + "name": "argmax", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _max_pool_with_argmax_tbe(): + """MaxPoolWithArgmax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/maximum.py b/mindspore/ops/_op_impl/tbe/maximum.py new file mode 100644 index 00000000000..2a4051f28d2 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/maximum.py @@ -0,0 +1,82 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Maximum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"Maximum", + "imply_type":"TBE", + "fusion_type":"ELEMWISE", + "async_flag":false, + "binfile_name":"maximum.so", + "compute_cost":10, + "kernel_name":"maximum", + "partial_flag":true, + "attr":[], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"x1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"x2", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"y", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _maximum_tbe(): + """Maximum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/maximum_grad.py b/mindspore/ops/_op_impl/tbe/maximum_grad.py new file mode 100644 index 00000000000..f602616da9e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/maximum_grad.py @@ -0,0 +1,125 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaximumGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"MaximumGrad", + "imply_type":"TBE", + "fusion_type":"OPAQUE", + "async_flag":false, + "binfile_name":"maximum_grad.so", + "compute_cost":10, + "kernel_name":"maximum_grad", + "partial_flag":true, + "attr":[ + { + "name":"grad_x", + "param_type":"optional", + "type":"bool", + "value":"all" + }, + { + "name":"grad_y", + "param_type":"optional", + "type":"bool", + "value":"all" + } + ], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"grads", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"x1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"x2", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"y1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"y2", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _maximum_grad_tbe(): + """MaximumGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/minimum.py b/mindspore/ops/_op_impl/tbe/minimum.py new file mode 100644 index 00000000000..2f7e5b80d71 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/minimum.py @@ -0,0 +1,88 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + + +"""Minimum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Minimum", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "minimum.so", + "compute_cost": 10, + "kernel_name": "minimum", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _minimum_tbe(): + """Minimum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/minimum_grad.py b/mindspore/ops/_op_impl/tbe/minimum_grad.py new file mode 100644 index 00000000000..d49b2aa184c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/minimum_grad.py @@ -0,0 +1,125 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MinimumGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"MinimumGrad", + "imply_type":"TBE", + "fusion_type":"OPAQUE", + "async_flag":false, + "binfile_name":"minimum_grad.so", + "compute_cost":10, + "kernel_name":"minimum_grad", + "partial_flag":true, + "attr":[ + { + "name":"grad_x", + "param_type":"optional", + "type":"bool", + "value":"all" + }, + { + "name":"grad_y", + "param_type":"optional", + "type":"bool", + "value":"all" + } + ], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"grads", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"x1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":2, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"x2", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"y1", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32" + ], + "format":[ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name":"y2", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _minimum_grad_tbe(): + """MinimumGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/mul.py b/mindspore/ops/_op_impl/tbe/mul.py new file mode 100644 index 00000000000..912d5e372f8 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/mul.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Mul op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Mul", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "mul.so", + "compute_cost": 10, + "kernel_name": "mul", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32", "int32", "int32", "int32", + "float16", "float16", "float16", "float16", "float16", + "float", "float", "float", "float","float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0", + "FRACTAL_NZ", "DefaultFormat", "FracZ", "C1HWNCoC0", "NC1HWC0" + ], + "name": "output", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _mul_tbe(): + """Mul TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/neg.py b/mindspore/ops/_op_impl/tbe/neg.py new file mode 100644 index 00000000000..bbfcd824ec3 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/neg.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Neg op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Neg", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "neg.so", + "compute_cost": 10, + "kernel_name": "neg", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float","float","float16","float16","int32","int32","int8","int8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float","float","float16","float16","int32","int32","int8","int8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _neg_tbe(): + """Neg TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/npu_alloc_float_status.py b/mindspore/ops/_op_impl/tbe/npu_alloc_float_status.py new file mode 100644 index 00000000000..6f65b9064ac --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/npu_alloc_float_status.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""NPUAllocFloatStatus op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "NPUAllocFloatStatus", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "n_p_u_alloc_float_status.so", + "compute_cost": 10, + "kernel_name": "n_p_u_alloc_float_status", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "DefaultFormat" + ], + "name": "data", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _npu_alloc_float_status_tbe(): + """NPUAllocFloatStatus TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/npu_clear_float_status.py b/mindspore/ops/_op_impl/tbe/npu_clear_float_status.py new file mode 100644 index 00000000000..d7e69673f20 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/npu_clear_float_status.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""NPUClearFloatStatus op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "NPUClearFloatStatus", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "n_p_u_clear_float_status.so", + "compute_cost": 10, + "kernel_name": "n_p_u_clear_float_status", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "DefaultFormat" + ], + "name": "addr", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "DefaultFormat" + ], + "name": "data", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _npu_clear_float_status_tbe(): + """NPUClearFloatStatus TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/npu_get_float_status.py b/mindspore/ops/_op_impl/tbe/npu_get_float_status.py new file mode 100644 index 00000000000..441fe3b271a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/npu_get_float_status.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""NPUGetFloatStatus op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "NPUGetFloatStatus", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "n_p_u_get_float_status.so", + "compute_cost": 10, + "kernel_name": "n_p_u_get_float_status", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "DefaultFormat" + ], + "name": "addr", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "DefaultFormat" + ], + "name": "data", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _npu_get_float_status_tbe(): + """NPUGetFloatStatus TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/one_hot.py b/mindspore/ops/_op_impl/tbe/one_hot.py new file mode 100644 index 00000000000..0af406dfc67 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/one_hot.py @@ -0,0 +1,109 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""OneHot op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "OneHot", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "one_hot.so", + "compute_cost": 10, + "kernel_name": "one_hot", + "partial_flag": true, + "attr": [ + { + "name": "depth", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "axis", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "int32","int32","int32","int32","int32", + "uint8","uint8","uint8","uint8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float32","int32","int8","uint8", + "float16","float32","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "on_value", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16","float32","int32","int8","uint8", + "float16","float32","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "off_value", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float32","int32","int8","uint8", + "float16","float32","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _one_hot_tbe(): + """OneHot TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/pad_d.py b/mindspore/ops/_op_impl/tbe/pad_d.py new file mode 100644 index 00000000000..25cb19816d0 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/pad_d.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Pad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "pad_d.so", + "compute_cost": 10, + "kernel_name": "pad_d", + "partial_flag": true, + "attr": [ + { + "name": "paddings", + "param_type": "optional", + "type": "listListInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int8","uint8","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int8","uint8","int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _pad_d_tbe(): + """Pad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/pow.py b/mindspore/ops/_op_impl/tbe/pow.py new file mode 100644 index 00000000000..aa67a8c9427 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/pow.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pow op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Pow", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "pow.so", + "compute_cost": 10, + "kernel_name": "pow", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int32", "int8", "uint8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float", "int32", "int8", "uint8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int32", "int8", "uint8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _pow_tbe(): + """Pow TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/real_div.py b/mindspore/ops/_op_impl/tbe/real_div.py new file mode 100644 index 00000000000..01f870b7cb4 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/real_div.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""RealDiv op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "RealDiv", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "realdiv.so", + "compute_cost": 10, + "kernel_name": "realdiv", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "z", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _real_div_tbe(): + """RealDiv TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/reciprocal.py b/mindspore/ops/_op_impl/tbe/reciprocal.py new file mode 100644 index 00000000000..ba039be4c53 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/reciprocal.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Add op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Reciprocal", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "reciprocal.so", + "compute_cost": 10, + "kernel_name": "reciprocal", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float32", "float32", "float32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "NHWC", "DefaultFormat", "NC1HWC0", "NHWC" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float32", "float32", "float32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "NHWC", "DefaultFormat", "NC1HWC0", "NHWC" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _reciprocal_tbe(): + """Add TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/reduce_max.py b/mindspore/ops/_op_impl/tbe/reduce_max.py new file mode 100644 index 00000000000..9c4981babc2 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/reduce_max.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ReduceMax op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ReduceMax", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "reduce_max_d.so", + "compute_cost": 10, + "kernel_name": "reduce_max_d", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "keep_dims", + "param_type": "required", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int8", "uint8", "bool", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int8", "uint8", "bool", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _reduce_max_tbe(): + """ReduceMax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/reduce_mean.py b/mindspore/ops/_op_impl/tbe/reduce_mean.py new file mode 100644 index 00000000000..c8776fa8b1c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/reduce_mean.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ReduceMean op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ReduceMean", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "reduce_mean.so", + "compute_cost": 10, + "kernel_name": "reduce_mean", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "keep_dims", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float16","int8","uint8" + ], + "format": [ + "NC1HWC0","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float16","int8","uint8" + ], + "format": [ + "NC1HWC0","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _reduce_mean_tbe(): + """ReduceMean TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/reduce_mean_d.py b/mindspore/ops/_op_impl/tbe/reduce_mean_d.py new file mode 100644 index 00000000000..59cfeb240bb --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/reduce_mean_d.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ReduceMeanD op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ReduceMeanD", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "reduce_mean_d.so", + "compute_cost": 10, + "kernel_name": "reduce_mean_d", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "keep_dims", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float","float16","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float","float16","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _reduce_mean_d_tbe(): + """Conv2D TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/reduce_sum.py b/mindspore/ops/_op_impl/tbe/reduce_sum.py new file mode 100644 index 00000000000..b15a4decccd --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/reduce_sum.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ReduceSum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ReduceSum", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "reduce_sum_d.so", + "compute_cost": 10, + "kernel_name": "reduce_sum_d", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "keep_dims", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _reduce_sum_tbe(): + """ReduceSum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/relu.py b/mindspore/ops/_op_impl/tbe/relu.py new file mode 100644 index 00000000000..7350f2ae35a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/relu.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ReLU op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ReLU", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "relu.so", + "compute_cost": 10, + "kernel_name": "relu", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float","int32", "int32", "int8", "int8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _relu_tbe(): + """Relu TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/relu_grad.py b/mindspore/ops/_op_impl/tbe/relu_grad.py new file mode 100644 index 00000000000..28b4574e04c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/relu_grad.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ReluGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ReluGrad", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "relugrad.so", + "compute_cost": 10, + "kernel_name": "relugrad", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0","DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "gradients", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "features", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32", "int8", "int8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "backprops", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _relu_grad_tbe(): + """ReluGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/reshape.py b/mindspore/ops/_op_impl/tbe/reshape.py new file mode 100644 index 00000000000..8386b7a6f0e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/reshape.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Reshape op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Reshape", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "reshape.so", + "compute_cost": 10, + "kernel_name": "reshape", + "partial_flag": true, + "attr": [ + { + "name": "shape", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _reshape_tbe(): + """Reshape TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor.py b/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor.py new file mode 100644 index 00000000000..63ecf0ede2c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ResizeNearestNeighbor op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ResizeNearestNeighbor", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "resize_nearest_neighbor_d.so", + "compute_cost": 10, + "kernel_name": "resize_nearest_neighbor_d", + "partial_flag": true, + "attr": [ + { + "name": "size", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "align_corners", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8", + "float16","float","int32","int8","uint8" + ], + "format": [ + "NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "images", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8", + "float16","float","int32","int8","uint8" + ], + "format": [ + "NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0", + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _resize_nearest_neighbor_d_tbe(): + """ResizeNearestNeighbor TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor_d.py b/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor_d.py new file mode 100644 index 00000000000..9595041401f --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor_d.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ResizeNearestNeighbor op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ResizeNearestNeighbor", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "resize_nearest_neighbor_d.so", + "compute_cost": 10, + "kernel_name": "resize_nearest_neighbor_d", + "partial_flag": true, + "attr": [ + { + "name": "size", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "align_corners", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0" + ], + "name": "images", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0","NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _resize_nearest_neighbor_d_tbe(): + """ResizeNearestNeighbor TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor_grad_d.py b/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor_grad_d.py new file mode 100644 index 00000000000..51cfaf51760 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/resize_nearest_neighbor_grad_d.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ResizeNearestNeighborgrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ResizeNearestNeighborGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "resize_nearest_neighbor_grad_d.so", + "compute_cost": 10, + "kernel_name": "resize_nearest_neighbor_grad_d", + "partial_flag": true, + "attr": [ + { + "name": "size", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "align_corners", + "param_type": "optional", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "NC1HWC0" + ], + "name": "grads", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float" + ], + "format": [ + "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _resize_nearest_neighbor_grad_d_tbe(): + """ResizeNearestNeighborGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/rsqrt.py b/mindspore/ops/_op_impl/tbe/rsqrt.py new file mode 100644 index 00000000000..e23d5053975 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/rsqrt.py @@ -0,0 +1,107 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Rsqrt op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"Rsqrt", + "imply_type":"TBE", + "fusion_type":"OPAQUE", + "async_flag":false, + "binfile_name":"rsqrt.so", + "compute_cost":10, + "kernel_name":"rsqrt", + "partial_flag":true, + "attr":[], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", + "float16", + "float16", + "float16", + "float16", + "float16", + "float", + "float", + "float", + "float", + "float", + "float" + ], + "format":[ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "FracZ", + "C1HWNCoC0", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "FracZ", + "C1HWNCoC0" + ], + "name":"x", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", + "float16", + "float16", + "float16", + "float16", + "float16", + "float", + "float", + "float", + "float", + "float", + "float" + ], + "format":[ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "FracZ", + "C1HWNCoC0", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "FracZ", + "C1HWNCoC0" + ], + "name":"y", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _rsqrt_tbe(): + """Rsqrt TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_nd.py b/mindspore/ops/_op_impl/tbe/scatter_nd.py new file mode 100644 index 00000000000..947fc57920a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_nd.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterNd op""" +from mindspore.ops.op_info_register import op_info_register + + +# map to tbe kernel name scatter_nd_d +@op_info_register("""{ + "op_name": "ScatterNd", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "scatter_nd_d.so", + "compute_cost": 10, + "kernel_name": "scatter_nd_d", + "partial_flag": true, + "attr": [ + { + "name": "shape", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32", "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "indices", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _scatter_nd_tbe(): + """Conv2D TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_nd_d.py b/mindspore/ops/_op_impl/tbe/scatter_nd_d.py new file mode 100644 index 00000000000..ad776fde497 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_nd_d.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterNdD op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ScatterNdD", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "scatter_nd_d.so", + "compute_cost": 10, + "kernel_name": "scatter_nd_d", + "partial_flag": true, + "attr": [ + { + "name": "shape", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "int32", "int32", "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "indices", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","int32","int8","uint8" + ], + "format": [ + "DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _scatter_nd_d_tbe(): + """ScatterNdD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/select.py b/mindspore/ops/_op_impl/tbe/select.py new file mode 100644 index 00000000000..c205e0de1de --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/select.py @@ -0,0 +1,107 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Select op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Select", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "select.so", + "compute_cost": 10, + "kernel_name": "select", + "partial_flag": true, + "attr":[ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "bool", "bool", "bool", "bool", "bool", "bool", "bool", "bool", "bool", "bool", + "bool", "bool", "bool", "bool", "bool", "bool", "bool", "bool", "bool", "bool" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "condition", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", + "int32", "int32", "int32", "int32", "int8", "int8", "int8", "int8", "uint8", + "uint8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32", "int8", "int8", "int8", "int8", "uint8", "uint8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32", "int8", "int8", "int8", "int8", "uint8", "uint8", "uint8", "uint8" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _select_tbe(): + """Select TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sigmoid.py b/mindspore/ops/_op_impl/tbe/sigmoid.py new file mode 100644 index 00000000000..cba9561d272 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sigmoid.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Sigmoid op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Sigmoid", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "Sigmoid.so", + "compute_cost": 10, + "kernel_name": "sigmoid", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float", + "float16","float", + "float16","float", + "float16","float", + "float16","float" + ], + "format": [ + "FracZ","FracZ", + "FRACTAL_NZ","FRACTAL_NZ", + "C1HWNCoC0","C1HWNCoC0", + "NC1HWC0","NC1HWC0", + "DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float", + "float16","float", + "float16","float", + "float16","float", + "float16","float" + ], + "format": [ + "FracZ","FracZ", + "FRACTAL_NZ","FRACTAL_NZ", + "C1HWNCoC0","C1HWNCoC0", + "NC1HWC0","NC1HWC0", + "DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _sigmoid_tbe(): + """Sigmoid TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sigmoid_cross_entropy_with_logits.py b/mindspore/ops/_op_impl/tbe/sigmoid_cross_entropy_with_logits.py new file mode 100644 index 00000000000..b20438d5fef --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sigmoid_cross_entropy_with_logits.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed unde:q!r the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SigmoidCrossEntropyWithLogits op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "SigmoidCrossEntropyWithLogits", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "sigmoid_cross_entropy_with_logits.so", + "compute_cost": 10, + "kernel_name": "sigmoid_cross_entropy_with_logits", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "predict", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "target", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "loss", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _sigmoid_cross_entropy_with_logits_tbe(): + """SigmoidCrossEntropyWithLogits TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sigmoid_cross_entropy_with_logits_grad.py b/mindspore/ops/_op_impl/tbe/sigmoid_cross_entropy_with_logits_grad.py new file mode 100644 index 00000000000..6e5df24cfe4 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sigmoid_cross_entropy_with_logits_grad.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed unde:q!r the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SigmoidCrossEntropyWithLogitsGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "SigmoidCrossEntropyWithLogitsGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "sigmoid_cross_entropy_with_logits_grad.so", + "compute_cost": 10, + "kernel_name": "sigmoid_cross_entropy_with_logits_grad", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "predict", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "target", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "dout", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat" + ], + "name": "gradient", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _sigmoid_cross_entropy_with_logits_grad_tbe(): + """SigmoidCrossEntropyWithLogitsGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sigmoid_grad.py b/mindspore/ops/_op_impl/tbe/sigmoid_grad.py new file mode 100644 index 00000000000..f0833e8a807 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sigmoid_grad.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SigmoidGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "SigmoidGrad", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "sigmoid_grad.so", + "compute_cost": 10, + "kernel_name": "sigmoid_grad", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float16","float" + ], + "format": [ + "NC1HWC0","NC1HWC0","DefaultFormat","DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16","float","float16","float" + ], + "format": [ + "NC1HWC0","NC1HWC0","DefaultFormat","DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float","float16","float" + ], + "format": [ + "NC1HWC0","NC1HWC0","DefaultFormat","DefaultFormat" + ], + "name": "z", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _sigmoid_grad_tbe(): + """SigmoidGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/slice.py b/mindspore/ops/_op_impl/tbe/slice.py new file mode 100644 index 00000000000..779f19cbc5d --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/slice.py @@ -0,0 +1,112 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Slice op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"Slice", + "imply_type":"TBE", + "fusion_type":"OPAQUE", + "async_flag":false, + "binfile_name":"slice_d.so", + "compute_cost":10, + "kernel_name":"slice_d", + "partial_flag":true, + "attr":[ + { + "name":"begin", + "param_type":"required", + "type":"listInt", + "value":"all" + }, + { + "name":"size", + "param_type":"required", + "type":"listInt", + "value":"all" + } + ], + "inputs":[ + { + "index":0, + "dtype":[ + "float", + "float16", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64" + ], + "format":[ + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat" + ], + "name":"x", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float", + "float16", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64" + ], + "format":[ + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat" + ], + "name":"y", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _slice_tbe(): + """Slice TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/softmax.py b/mindspore/ops/_op_impl/tbe/softmax.py new file mode 100644 index 00000000000..5a70d2605a5 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/softmax.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Softmax op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Softmax", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "softmax.so", + "compute_cost": 10, + "kernel_name": "softmax", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "NC1HWC0", "FRACTAL_NZ", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float" + ], + "format": [ + "FRACTAL_NZ", "DefaultFormat", "NC1HWC0", "FRACTAL_NZ", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _softmax_tbe(): + """Softmax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/softmax_cross_entropy_with_logits.py b/mindspore/ops/_op_impl/tbe/softmax_cross_entropy_with_logits.py new file mode 100644 index 00000000000..cc9f06948a7 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/softmax_cross_entropy_with_logits.py @@ -0,0 +1,91 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SoftmaxCrossEntropyWithLogits op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "SoftmaxCrossEntropyWithLogits", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "softmax_cross_entropy_with_logits.so", + "compute_cost": 10, + "kernel_name": "softmax_cross_entropy_with_logits", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input_features", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input_labels", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output_loss", + "need_compile": true, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output_backprop", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _softmax_cross_entropy_with_logits_tbe(): + """SoftmaxCrossEntropyWithLogits TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/split_d.py b/mindspore/ops/_op_impl/tbe/split_d.py new file mode 100644 index 00000000000..41311ffb90f --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/split_d.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Add op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Split", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "split_d.so", + "compute_cost": 10, + "kernel_name": "split_d", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "output_num", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16","float32", "float32", "int32", "int32", "int8", "int8", + "int16", "int16", "int64", "int64", "uint8", "uint8", "uint16", "uint16", + "uint32", "uint32", "uint64", "uint64", "bool", "bool" + ], + "format": [ + "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC" + , "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC" + , "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC" + ], + "name": "value", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16","float32", "float32", "int32", "int32", "int8", "int8", + "int16", "int16", "int64", "int64", "uint8", "uint8", "uint16", "uint16", + "uint32", "uint32", "uint64", "uint64", "bool", "bool" + ], + "format": [ + "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC" + , "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC" + , "DefaultFormat", "NHWC", "DefaultFormat", "NHWC", "DefaultFormat", "NHWC" + ], + "name": "output", + "need_compile": false, + "param_type": "dynamic", + "shape": "all" + } + ] +}""") +def _split_d_tbe(): + """Add TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sqrt.py b/mindspore/ops/_op_impl/tbe/sqrt.py new file mode 100644 index 00000000000..c73092886af --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sqrt.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Sqrt op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Sqrt", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "sqrt.so", + "compute_cost": 10, + "kernel_name": "sqrt", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "NHWC", "DefaultFormat", "NC1HWC0", "NHWC" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "NHWC", "DefaultFormat", "NC1HWC0", "NHWC" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _sqrt_tbe(): + """Sqrt TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/square.py b/mindspore/ops/_op_impl/tbe/square.py new file mode 100644 index 00000000000..03a81236ccc --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/square.py @@ -0,0 +1,69 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Square op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Square", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "square.so", + "compute_cost": 10, + "kernel_name": "sqrt", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "NHWC", "DefaultFormat", "NC1HWC0", "NHWC", + "DefaultFormat", "NC1HWC0", "NHWC" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float", "float", "float", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "NHWC", "DefaultFormat", "NC1HWC0", "NHWC", + "DefaultFormat", "NC1HWC0", "NHWC" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _square_tbe(): + """Square TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/square_sum_v1.py b/mindspore/ops/_op_impl/tbe/square_sum_v1.py new file mode 100644 index 00000000000..39b54002982 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/square_sum_v1.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SquareSumV1 op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "SquareSumV1", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "square_sum_v1.so", + "compute_cost": 10, + "kernel_name": "square_sum_v1", + "partial_flag": true, + "attr":[ + { + "name":"axis", + "param_type":"optional", + "type":"listInt", + "value":"all" + }, + { + "name":"keep_dims", + "param_type":"optional", + "type":"bool", + "value":"all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _square_sum_v1_tbe(): + """SquareSumV1 TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/square_sum_v2.py b/mindspore/ops/_op_impl/tbe/square_sum_v2.py new file mode 100644 index 00000000000..2f5ca49cc2e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/square_sum_v2.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SquareSumV2 op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "SquareSumV2", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "square_sum_v2.so", + "compute_cost": 10, + "kernel_name": "square_sum_v2", + "partial_flag": true, + "attr":[ + { + "name":"axis", + "param_type":"optional", + "type":"listInt", + "value":"all" + }, + { + "name":"keep_dims", + "param_type":"optional", + "type":"bool", + "value":"all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "input_x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "output2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _square_sum_v2_tbe(): + """SquareSumV2 TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/squeeze.py b/mindspore/ops/_op_impl/tbe/squeeze.py new file mode 100644 index 00000000000..765ffae2c13 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/squeeze.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Squeeze op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Squeeze", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "squeeze.so", + "compute_cost": 10, + "kernel_name": "squeeze", + "partial_flag": true, + "attr": [ + { + "name": "axis", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32" + ], + "format": [ + "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _squeeze_tbe(): + """Squeeze TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/strideslice_d.py b/mindspore/ops/_op_impl/tbe/strideslice_d.py new file mode 100644 index 00000000000..8f398b4215b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/strideslice_d.py @@ -0,0 +1,112 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""StridedSlice op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "StridedSlice", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "strided_slice_d.so", + "compute_cost": 10, + "kernel_name": "strided_slice_d", + "partial_flag": true, + "attr": [ + { + "name": "begin", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "end", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "strides", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "begin_mask", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "end_mask", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "ellipsis_mask", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "new_axis_mask", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "shrink_axis_mask", + "param_type": "required", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int32", "uint8", "bool", "int8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int32", "uint8", "bool", "int8" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _strided_slice_d_tbe(): + """StridedSlice TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/strideslicegrad_d.py b/mindspore/ops/_op_impl/tbe/strideslicegrad_d.py new file mode 100644 index 00000000000..adeeab9a811 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/strideslicegrad_d.py @@ -0,0 +1,120 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""StridedSliceGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "StridedSliceGrad", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "strided_slice_grad_d.so", + "compute_cost": 10, + "kernel_name": "strided_slice_grad_d", + "partial_flag": true, + "attr": [ + { + "name": "shapex", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "begin", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "end", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "strides", + "param_type": "optional", + "type": "listInt", + "value": "all" + }, + { + "name": "begin_mask", + "param_type": "optional", + "type": "int", + "value": "all" + }, + { + "name": "end_mask", + "param_type": "optional", + "type": "int", + "value": "all" + }, + { + "name": "ellipsis_mask", + "param_type": "optional", + "type": "int", + "value": "all" + }, + { + "name": "new_axis_mask", + "param_type": "optional", + "type": "int", + "value": "all" + }, + { + "name": "shrink_axis_mask", + "param_type": "optional", + "type": "int", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","uint8","uint8","int8","int8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0", + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "dy", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","uint8","uint8","int8","int8" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0", + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "output", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _strided_slice_grad_d_tbe(): + """StridedSliceGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sub.py b/mindspore/ops/_op_impl/tbe/sub.py new file mode 100644 index 00000000000..8d6ea4aa0e4 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sub.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Sub op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Sub", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "sub.so", + "compute_cost": 10, + "kernel_name": "sub", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _sub_tbe(): + """Add TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/tanh.py b/mindspore/ops/_op_impl/tbe/tanh.py new file mode 100644 index 00000000000..dd2737f2cee --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/tanh.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Tanh op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Tanh", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "tanh.so", + "compute_cost": 10, + "kernel_name": "tanh", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _tanh_tbe(): + """Tanh TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/tanh_grad.py b/mindspore/ops/_op_impl/tbe/tanh_grad.py new file mode 100644 index 00000000000..c50b5a3a5ae --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/tanh_grad.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""TanhGrad op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "TanhGrad", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "tanh_grad.so", + "compute_cost": 10, + "kernel_name": "tanh_grad", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "dy", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float", "float" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "NC1HWC0" + ], + "name": "z", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _tanh_grad_tbe(): + """TanhGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/tensor_add.py b/mindspore/ops/_op_impl/tbe/tensor_add.py new file mode 100644 index 00000000000..26a25c34b2d --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/tensor_add.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""TensorAdd op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "TensorAdd", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "add.so", + "compute_cost": 10, + "kernel_name": "add", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", "int32", + "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", "int32", + "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float16", "float16", "float16", "float", "float", "float", "float", "int32", + "int32", "int32", "int32" + ], + "format": [ + "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "NC1HWC0", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _tensor_add_tbe(): + """Add TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/tile.py b/mindspore/ops/_op_impl/tbe/tile.py new file mode 100644 index 00000000000..8299c500db6 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/tile.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Tile op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Tile", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "tile_d.so", + "compute_cost": 10, + "kernel_name": "tile_d", + "partial_flag": true, + "attr": [ + { + "name": "multiples", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32", "int32", "float16", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float32", "int32", "float16", "int32" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _tile_tbe(): + """Tile TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/topkv2.py b/mindspore/ops/_op_impl/tbe/topkv2.py new file mode 100644 index 00000000000..916b246a38e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/topkv2.py @@ -0,0 +1,102 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""TopKV2 op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "TopKV2", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "top_k_v2.so", + "compute_cost": 10, + "kernel_name": "top_k_v2", + "partial_flag": true, + "attr": [ + { + "name": "k", + "param_type": "required", + "type": "int", + "value": "all" + }, + { + "name": "sorted", + "param_type": "required", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "input_indices", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "values", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "int32" + ], + "format": [ + "DefaultFormat" + ], + "name": "indices", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _topk_v2_tbe(): + """TopKV2 TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/trans_data.py b/mindspore/ops/_op_impl/tbe/trans_data.py new file mode 100644 index 00000000000..1b7c8fa25df --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/trans_data.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""TransData op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "TransData", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "trans_data.so", + "compute_cost": 10, + "kernel_name": "trans_data", + "partial_flag": true, + "attr": [ + { + "name": "src_format", + "param_type": "required", + "type": "str", + "value": "DefaultFormat,NC1HWC0,FracZ,FRACTAL_NZ,HWCN,C1HWNCoC0" + }, + { + "name": "dst_format", + "param_type": "required", + "type": "str", + "value": "DefaultFormat,NC1HWC0,FracZ,FRACTAL_NZ,HWCN,C1HWNCoC0" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "bool", + "float","float","float","float","float","float","float","float","float","float", + "float16","float16","float16","float16","float16","float16","float16","float16","float16","float16" + ], + "format": [ + "DefaultFormat", + "DefaultFormat","DefaultFormat","DefaultFormat","FracZ","FRACTAL_NZ","NC1HWC0","HWCN","HWCN","C1HWNCoC0","FracZ", + "DefaultFormat","DefaultFormat","DefaultFormat","FracZ","FRACTAL_NZ","NC1HWC0","HWCN","HWCN","C1HWNCoC0","FracZ" + ], + "name": "src", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "bool", + "float","float","float","float","float","float","float","float","float","float", + "float16","float16","float16","float16","float16","float16","float16","float16","float16","float16" + ], + "format": [ + "NC1HWC0", + "NC1HWC0","FRACTAL_NZ","FracZ","DefaultFormat","DefaultFormat","DefaultFormat","FracZ","C1HWNCoC0","HWCN","HWCN", + "NC1HWC0","FRACTAL_NZ","FracZ","DefaultFormat","DefaultFormat","DefaultFormat","FracZ","C1HWNCoC0","HWCN","HWCN" + ], + "name": "dst", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _trans_data_tbe(): + """TransData TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/transpose_d.py b/mindspore/ops/_op_impl/tbe/transpose_d.py new file mode 100644 index 00000000000..e79a16adeb9 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/transpose_d.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""TransposeD op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "Transpose", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "transpose_d.so", + "compute_cost": 10, + "kernel_name": "transpose_d", + "partial_flag": true, + "attr": [ + { + "name": "perm", + "param_type": "optional", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16", "float", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64" + ], + "format": [ + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat", + "DefaultFormat", "DefaultFormat", "DefaultFormat", "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def _transpose_d_tbe(): + """TransposeD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py b/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py new file mode 100644 index 00000000000..2bc36b9e3d5 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py @@ -0,0 +1,197 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""UnsortedSegmentSum op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name":"UnsortedSegmentSum", + "imply_type":"TBE", + "fusion_type":"OPAQUE", + "async_flag":false, + "binfile_name":"unsorted_segment_sum_d.so", + "compute_cost":10, + "kernel_name":"unsorted_segment_sum_d", + "partial_flag":true, + "attr":[ + { + "name":"num_segments", + "param_type":"required", + "type":"int", + "value":"all" + } + ], + "inputs":[ + { + "index":0, + "dtype":[ + "float16", + "float16", + "float16", + "float16", + "float", + "float", + "float", + "float", + "int8", + "int8", + "int8", + "int8", + "uint8", + "uint8", + "uint8", + "uint8", + "int32", + "int32", + "int32", + "int32" + ], + "format":[ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat" + ], + "name":"x", + "need_compile":false, + "param_type":"required", + "shape":"all" + }, + { + "index":1, + "dtype":[ + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32", + "int32" + ], + "format":[ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat" + ], + "name":"segment_ids", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ], + "outputs":[ + { + "index":0, + "dtype":[ + "float16", + "float16", + "float16", + "float16", + "float", + "float", + "float", + "float", + "int8", + "int8", + "int8", + "int8", + "uint8", + "uint8", + "uint8", + "uint8", + "int32", + "int32", + "int32", + "int32" + ], + "format":[ + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat", + "DefaultFormat", + "NC1HWC0", + "DefaultFormat", + "DefaultFormat" + ], + "name":"y", + "need_compile":false, + "param_type":"required", + "shape":"all" + } + ] +}""") +def _unsorted_segment_sum_tbe(): + """UnsortedSegmentSum TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/zeros_like.py b/mindspore/ops/_op_impl/tbe/zeros_like.py new file mode 100644 index 00000000000..25f48b80a55 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/zeros_like.py @@ -0,0 +1,66 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ZerosLike op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "ZerosLike", + "imply_type": "TBE", + "fusion_type": "ELEMWISE", + "async_flag": false, + "binfile_name": "zeros_like.so", + "compute_cost": 10, + "kernel_name": "zeros_like", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8","bool","bool" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0", + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16","float16","float","float","int32","int32","int8","int8","uint8","uint8","bool","bool" + ], + "format": [ + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0", + "DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0","DefaultFormat","NC1HWC0" + ], + "name": "y", + "param_type": "required", + "shape": "all" + } + ] +}""") +def _zeros_like_tbe(): + """ZerosLike TBE register""" + return diff --git a/mindspore/ops/_register_for_op.py b/mindspore/ops/_register_for_op.py new file mode 100644 index 00000000000..5994ca751b5 --- /dev/null +++ b/mindspore/ops/_register_for_op.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Registry the relation.""" + +from collections import UserDict +from .primitive import Primitive + + +class Registry(UserDict): + """Registry class for registry functions for grad and vm_impl on Primitive.""" + + def register(self, prim): + def deco(fn): + """Decorate the function.""" + if isinstance(prim, str): + self[prim] = fn + elif issubclass(prim, Primitive): + self[id(prim)] = fn + return fn + return deco + + def get(self, prim_obj, default): + """Get the value by primitive.""" + fn = default + if isinstance(prim_obj, str) and prim_obj in self: + fn = self[prim_obj] + elif isinstance(prim_obj, Primitive): + key = id(prim_obj.__class__) + if key in self: + fn = self[key] + else: + key = prim_obj.name + if key in self: + fn = self[prim_obj.name] + return fn diff --git a/mindspore/ops/_utils/__init__.py b/mindspore/ops/_utils/__init__.py new file mode 100644 index 00000000000..00ce07453a7 --- /dev/null +++ b/mindspore/ops/_utils/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ops utils.""" +from .broadcast import _get_broadcast_shape + +__all__ = ['_get_broadcast_shape'] diff --git a/mindspore/ops/_utils/broadcast.py b/mindspore/ops/_utils/broadcast.py new file mode 100644 index 00000000000..2c9eb8a54ba --- /dev/null +++ b/mindspore/ops/_utils/broadcast.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""broadcast""" + + +def _get_broadcast_shape(x_shape, y_shape): + """ + Doing broadcast between tensor x and tensor y. + + Args: + x_shape (list): The shape of tensor x. + y_shape (list): The shape of tensor y. + + Returns: + List, the shape that broadcast between tensor x and tensor y. + + Raises: + ValueError: If tensor x and tensor y are not equal and could't broadcast. + + Examples: + >>> x_shape = [1, 2, 3] + >>> y_shape = [1, 2] + >>> broadcast_shape = _get_broadcast_shape(x_shape, y_shape) + """ + if x_shape == y_shape: + return x_shape + x_len = len(x_shape) + y_len = len(y_shape) + length = x_len if x_len < y_len else y_len + broadcast_shape_back = [] + + for i in range(-length, 0): + if x_shape[i] == 1: + broadcast_shape_back.append(y_shape[i]) + elif y_shape[i] == 1: + broadcast_shape_back.append(x_shape[i]) + elif x_shape[i] == y_shape[i]: + broadcast_shape_back.append(x_shape[i]) + else: + raise ValueError("The x_shape {} and y_shape {} can not broadcast.".format(x_shape, y_shape)) + + broadcast_shape_front = y_shape[0: y_len - length] if length == x_len else x_shape[0: x_len - length] + broadcast_shape = broadcast_shape_front + broadcast_shape_back + return broadcast_shape diff --git a/mindspore/ops/composite/__init__.py b/mindspore/ops/composite/__init__.py new file mode 100644 index 00000000000..e4c6e35d3a3 --- /dev/null +++ b/mindspore/ops/composite/__init__.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Composite operators. + +Pre-defined combination of operators. +""" + + +from .base import GradOperation, HyperMap, MultitypeFuncGraph, add_flags, \ + grad, grad_all, grad_all_with_sens, grad_by_list, grad_by_list_with_sens, grad_with_sens, \ + core, env_get, tail, zip_operation +from .clip_ops import clip_by_value +from .multitype_ops.add_impl import hyper_add +from .multitype_ops.ones_like_impl import ones_like +from .multitype_ops.zeros_like_impl import zeros_like + + +__all__ = [ + 'grad', + 'grad_by_list_with_sens', + 'grad_all', + 'grad_by_list', + 'grad_all_with_sens', + 'grad_with_sens', + 'env_get', + 'core', + 'add_flags', + 'tail', + 'MultitypeFuncGraph', + 'GradOperation', + 'HyperMap', + 'hyper_add', + 'zeros_like', + 'ones_like', + 'zip_operation', + 'clip_by_value'] diff --git a/mindspore/ops/composite/base.py b/mindspore/ops/composite/base.py new file mode 100644 index 00000000000..8670f4aa7ce --- /dev/null +++ b/mindspore/ops/composite/base.py @@ -0,0 +1,309 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Basic composite operations.""" + +from ..._c_expression import EnvInstance_, GradOperation_, HyperMap_, MultitypeFuncGraph_, Tail_, TensorSlice_, \ + TupleAdd_, TupleSlice_, UnpackCall_, ZipOperation_, ListAppend_ +from ...common import dtype as mstype +from ...common.api import ms_function +from .. import functional as F +from .. import operations as P + +__all__ = [EnvInstance_, TensorSlice_, TupleAdd_, TupleSlice_, UnpackCall_] + + +def add_flags(fn, **flags): + """ + An interface to add flag for a function. + + Note: + Only supports bool value. + + Args: + fn (Function): Function or cell to add flag. + flags (bool): Flags use kwargs. + + Returns: + Function, the fn added flags. + + Examples: + >>> add_flags(net, predit=True) + """ + # need set the attr and access on c++ + if not hasattr(fn, "_mindspore_flags"): + fn._mindspore_flags = {} + fn._mindspore_flags.update({**flags}) + return fn + + +def core(fn=None, **flags): + """ + A decorator to add flag to a function. + + By default, the function is marked core=True using this decorator to + set flag to a graph. + + Args: + fn (Function): Function to add flag. Default: None. + flags (dict): The following flags can be set core, which indicates that this is a core function or + other flag. Default: None. + """ + # need set the attr and access on c++ + + def deco(fn): + fn._mindspore_flags = { + 'core': True, + **flags, + } + return fn + + if fn is not None: + ret = deco(fn) + else: + ret = deco + return ret + + +class GradOperation(GradOperation_): + """ + An metafuncgraph object which is used to get the gradient of output of a network(function). + + The GradOperation will convert the network(function) into a back propagation graph. + + Args: + get_all (bool): If True, get all the gradients w.r.t inputs. Default: False. + get_by_list (bool): If True, get all the gradients w.r.t Parameter variables. + If get_all and get_by_list are both False, get the gradient w.r.t first input. + If get_all and get_by_list are both True, get the gradients w.r.t inputs and Parameter variables + at the same time in the form of ((grads w.r.t inputs), (grads w.r.t parameters)). Default: False. + sens_param (bool): Whether append sensitivity as input. If sens_param is False, + a 'ones_like(outputs)' sensitivity will be attached automatically. Default: False. + """ + + def __init__(self, name, + get_all=False, get_by_list=False, sens_param=False): + self.get_all = get_all + self.get_by_list = get_by_list + self.sens_param = sens_param + GradOperation_.__init__(self, name, get_all, get_by_list, sens_param) + self.grad_fn = None + self.fn = None + + def __call__(self, fn, weights=None): + grad_ = GradOperation('grad', self.get_all, self.get_by_list, self.sens_param) + if self.grad_fn is None or self.fn != fn: + if self.get_by_list: + @ms_function(obj=fn) + def after_grad(*args): + return grad_(fn, weights)(*args) + else: + @ms_function(obj=fn) + def after_grad(*args): + return grad_(fn)(*args) + self.grad_fn = after_grad + self.fn = fn + return self.grad_fn + + +grad = GradOperation('grad') +grad_all = GradOperation('get_all', get_all=True) +grad_by_list = GradOperation('get_by_list', get_by_list=True) +grad_with_sens = GradOperation('grad_with_sens', sens_param=True) +grad_all_with_sens = GradOperation('grad_all_with_sens', get_all=True, sens_param=True) +grad_by_list_with_sens = GradOperation('grad_by_list_with_sens', get_by_list=True, sens_param=True) + + +class MultitypeFuncGraph(MultitypeFuncGraph_): + """ + Generate multiply graph. + + MultitypeFuncGraph is a class used to generate graphs for function with different type as input. + + Args: + name (str): Operator name. + + Raises: + ValueError: Cannot find matching fn for the given args. + + Examples: + >>> # `add` is a metagraph object which will add two objects according to + >>> # input type using ".register" decorator. + >>> add = MultitypeFuncGraph('add') + + """ + + def __init__(self, name): + MultitypeFuncGraph_.__init__(self, name) + self.entries = list() + + def __call__(self, *args): + for sig, fn in self.entries: + if len(sig) != len(args): + continue + output = fn(*args) + return output + raise ValueError("Cannot find fn match given args.") + + def register(self, *type_names): + """Register a function for the given type string.""" + def deco(fn): + self.register_fn(type_names, fn) + self.entries.append((type_names, fn)) + return fn + return deco + + +class HyperMap(HyperMap_): + """ + Hypermap will apply the set operation on input sequences. + + Which will apply the operations of every elements of the sequence. + + Args: + ops (Union[MultitypeFuncGraph, None]): `ops` is the operation to apply. If `ops` is `None`, + the operations should be putted in the first input of the instance. + + Inputs: + - **args** (Tuple[sequence]) - If `ops` is not `None`, all the inputs should be the same length sequences, + and each row of the sequences. e.g. If args length is 2, and for `i` in length of each sequence + `(args[0][i], args[1][i])` will be the input of the operation. + + If `ops` is not `None`, the first input is the operation, and the other is inputs. + + Outputs: + sequence, the output will be same type and same length of sequence from input and the value of each element + is the result of operation apply each row of element. e.g. `operation(args[0][i], args[1][i])`. + """ + + def __init__(self, ops=None): + self.ops = ops + if ops: + HyperMap_.__init__(self, ops) + else: + HyperMap_.__init__(self) + + def __call__(self, *args): + func = args[0] + count = 0 + count_max = 1 + args_list = args[1:] + if self.ops is not None: + func = self.ops + args_list = args + for item in args_list: + if isinstance(item, (tuple, list)): + count_max = len(item) + break + + def get_item(x): + nonlocal count + if isinstance(x, (tuple, list)): + return x[count] + return x + + for i in range(count_max): + true_args = tuple(map(get_item, args_list)) + func(*true_args) + count = i + 1 + return True + + def register(self, *type_names): + """Register a function for the given type string.""" + + def deco(fn): + self.register_fn(type_names, fn) + return fn + return deco + + +class _ListAppend(ListAppend_): + """ + A metafuncgraph class that append one element to list. + + Args: + name (str): The name of the metafuncgraph object. + """ + def __init__(self, name): + ListAppend_.__init__(self, name) + + def __call__(self, *args): + pass + + +_append = _ListAppend("append") + + +class _Tail(Tail_): + """ + A metafuncgraph class that generates tail elements of the tuple. + + Args: + name (str): The name of the metafuncgraph object. + """ + + def __init__(self, name): + Tail_.__init__(self, name) + + def __call__(self, *args): + pass + + +tail = _Tail('tail') + + +class _ZipOperation(ZipOperation_): + """Generates a tuple of zip iterations for inputs.""" + + def __init__(self, name): + ZipOperation_.__init__(self, name) + + def __call__(self, *args): + pass + + +zip_operation = _ZipOperation('zip_operation') +"""`zip_operation` will generate a tuple of zip iterations of inputs.""" + + +env_get = MultitypeFuncGraph("env_get") + + +@env_get.register("EnvType", "Tensor") +def _tensor_env_get(env, parameter): + """Used to get env.""" + return F.env_getitem(env, F.ref_to_embed(parameter), F.zeros_like_tensor(parameter)) + + +_mp_cast_helper = MultitypeFuncGraph('mixed_precision_cast_helper') + + +@_mp_cast_helper.register("TypeType", "Number") +@core +def _mixed_precision_cast_helper_1(type_, x): + """if x is float cast to type.""" + # type_ is place holder + return x + + +@_mp_cast_helper.register("TypeType", "Tensor") +@core +def _mixed_precision_cast_helper_2(type_, x): + """if x is float cast to type.""" + if F.issubclass_(F.dtype(x), mstype.float_): + return P.Cast()(x, type_) + return x diff --git a/mindspore/ops/composite/clip_ops.py b/mindspore/ops/composite/clip_ops.py new file mode 100644 index 00000000000..a87b1c1678d --- /dev/null +++ b/mindspore/ops/composite/clip_ops.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Operations for clipping tensors to min/max values.""" + +from .. import operations as P + + +def clip_by_value(x, clip_value_min, clip_value_max): + """ + Clips tensor values to a specified min and max. + + Limits the value of :math:`x` to a range, whose lower limit is 'clip_value_min' + and upper limit is 'clip_value_max'. + + Note: + 'clip_value_min' needs to be less than or equal to 'clip_value_max'. + + Args: + x (Tensor): Input data. + clip_value_min (Tensor): The minimum value. + clip_value_max (Tensor): The maximum value. + + Returns: + Tensor, a clipped Tensor. + """ + min_op = P.Minimum() + max_op = P.Maximum() + x_min = min_op(x, clip_value_max) + x_max = max_op(x_min, clip_value_min) + return x_max diff --git a/mindspore/ops/composite/multitype_ops/__init__.py b/mindspore/ops/composite/multitype_ops/__init__.py new file mode 100644 index 00000000000..0ab8527ab45 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Multitype ops""" + +from .add_impl import add +from .sub_impl import sub +from .mul_impl import mul +from .div_impl import div +from .getitem_impl import getitem +from .zeros_like_impl import zeros_like +from .ones_like_impl import ones_like +from .equal_impl import equal +from .less_impl import less +from .less_equal_impl import less_equal +from .negative_impl import negative +from .logical_and_impl import logical_and +from .logical_or_impl import logical_or +__all__ = [ + 'add', + 'sub', + 'mul', + 'div', + 'zeros_like', + 'ones_like', + 'equal', + 'less', + 'less_equal', + 'negative', + 'getitem', + 'logical_and', + 'logical_or' +] diff --git a/mindspore/ops/composite/multitype_ops/add_impl.py b/mindspore/ops/composite/multitype_ops/add_impl.py new file mode 100644 index 00000000000..2b1f83679e3 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/add_impl.py @@ -0,0 +1,164 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""add_impl""" + +from ...composite import base +from ... import functional as F + + +add = base.MultitypeFuncGraph('add') +"""`add` is a metafuncgraph object which will add two objects according to input type using ".register" decorator.""" + + +_add_backward = base.MultitypeFuncGraph('add_backward') +""" +`_add_backward` is an metafuncgraph object which will add_backward two objects according to input type +using ".register" decorator. +""" + + +class _TupleAdd(base.TupleAdd_): + """ + Adding two tuples. + + Args: + x (tuple): x + y (tuple): y + + Returns: + Tuple, consists of elements of x and elements of y. + """ + + def __init__(self, name): + base.TupleAdd_.__init__(self, name) + + def __call__(self, *args): + pass + + +_tuple_add = _TupleAdd('tuple_add') +"""`_tuple_add` is an metafuncgraph object which will concatenate two tuples to form a tuple.""" + + +@add.register("Number", "Number") +@_add_backward.register("Number", "Number") +def _scalar_add_scalar(x, y): + """ + Returns the sum of two numbers. + + Args: + x (Number): x + y (Number): y + + Returns: + Number, equal to x + y, has the same type as x. + """ + return F.scalar_add(x, y) + + +@add.register("Number", "Tensor") +def _scalar_add_tensor(x, y): + """ + Number is added to tensor. + + Args: + x (Number): x + y (Tensor): The dtype is same as x. + + Returns: + Tensor, has the same dtype as x. + """ + z = F.scalar_to_tensor(x, F.dtype(y)) + return F.tensor_add(z, y) + + +@add.register("Tensor", "Number") +def _tensor_add_scalar(x, y): + """ + Tensor is added to number. + + Args: + x (Tensor): x + y (Number): The dtype is same as x. + + Returns: + Tensor, has the same dtype as x. + """ + z = F.scalar_to_tensor(y, F.dtype(x)) + return F.tensor_add(x, z) + + +@add.register("Tensor", "Tensor") +def _tensor_add_tensor(x, y): + """ + Returns x + y element-wise. + + Args: + x (Tensor): x + y (Tensor): The dtype is same as x. + + Returns: + Tensor, has the same dtype as x. + """ + return F.tensor_add(x, y) + + +@_add_backward.register("EnvType", "EnvType") +def _add_env(x, y): + """ + Adds two EnvType variables. + + Args: + x (EnvType): x + y (EnvType): y + + Returns: + EnvType, equal to x + y. + """ + return F.env_add(x, y) + + +@add.register("Tuple", "Tuple") +def _add_tuple(x, y): + """ + Adds two tuples. + + Args: + x (tuple): x + y (tuple): y + + Returns: + Tuple, consists of elements of x and elements of y. + """ + return _tuple_add(x, y) + + +@_add_backward.register("Tensor", "Tensor") +def _add_addn(x, y): + """ + Adds two tensors by element. + + Args: + x (Tensor): x + y (Tensor): The dtype is same as x. + + Returns: + Tensor, the dtype is same as x. + """ + return F.addn((x, y)) + + +hyper_add = base.HyperMap(_add_backward) diff --git a/mindspore/ops/composite/multitype_ops/div_impl.py b/mindspore/ops/composite/multitype_ops/div_impl.py new file mode 100644 index 00000000000..3edf3c8d9f9 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/div_impl.py @@ -0,0 +1,88 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""div_impl""" + +from ...composite import base +from ... import functional as F + + +div = base.MultitypeFuncGraph("div") +""" +div is a metafuncgraph object which will div two objects according to input type +using ".register" decorator +""" + + +@div.register("Number", "Number") +def _div_scalar(x, y): + """ + Two numbers divide. + + Args: + x (Number): x + y (NUmber): y + + Returns: + Number, equal to x / y, the type is same as x. + """ + return F.scalar_div(x, y) + + +@div.register("Tensor", "Tensor") +def _div_tensor(x, y): + """ + Two tensors divide by element. + + Args: + x (Tensor): x + y (Tensor): The dtype is same as x. + + Returns: + Tensor, has the same dtype as x. + """ + return F.tensor_div(x, y) + + +@div.register("Number", "Tensor") +def _scalar_div_tensor(x, y): + """ + Number divided by tensor. + + Args: + x (Number): x + y (Tensor): The dtype is same as x. + + Returns: + Tensor, has the same dtype as x. + """ + z = F.scalar_to_tensor(x, F.dtype(y)) + return F.tensor_div(z, y) + + +@div.register("Tensor", "Number") +def _tensor_div_scalar(x, y): + """ + Tensor divided by number. + + Args: + x (Tensor): x + y (Number): The dtype is same as x. + + Returns: + Tensor, has the same dtype as x. + """ + z = F.scalar_to_tensor(y, F.dtype(x)) + return F.tensor_div(x, z) diff --git a/mindspore/ops/composite/multitype_ops/equal_impl.py b/mindspore/ops/composite/multitype_ops/equal_impl.py new file mode 100644 index 00000000000..9ff7e6671e5 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/equal_impl.py @@ -0,0 +1,236 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""equal_impl""" + +from ...composite import base +from ... import functional as F + + +equal = base.MultitypeFuncGraph("equal") +""" +equal is a metafuncgraph object which will determine if two objects are equal according to input type +using ".register" decorator +""" + + +@equal.register("Number", "Number") +def _equal_scalar(x, y): + """ + Determine if two numbers are equal. + + Args: + x (Number): x + y (NUmber): y + + Returns: + bool, if x == y return true, x != y return false. + """ + return F.scalar_eq(x, y) + + +@equal.register("String", "String") +def _equal_string(x, y): + """ + Determine if two strings are equal. + + Args: + x: str + y: str + + Returns: + bool, if x == y return true, x != y return false. + """ + return F.string_eq(x, y) + + +@equal.register("String", "None") +def _string_equal_none(x, y): + """ + Determine if string equals none. + + Args: + x: str. + y: None. + + Returns: + bool, return false. + """ + return False + + +@equal.register("None", "String") +def _none_equal_string(x, y): + """ + Determine if string equals none. + + Args: + x: None. + y: str. + + Returns: + bool, return false. + """ + return False + + +@equal.register("None", "None") +def _none_equal_none(x, y): + """ + Determine if none equals none. + + Args: + x: None. + y: None. + + Returns: + bool, return true. + """ + return True + + +@equal.register("Number", "None") +def _scalar_equal_none(x, y): + """ + Determine if number equals none. + + Args: + x: Number. + y: None. + + Returns: + bool, return false. + """ + return False + + +@equal.register("None", "Number") +def _none_equal_scalar(x, y): + """ + Determine if number equals none. + + Args: + x: None. + y: NUmber. + + Returns: + bool, return false. + """ + return False + + +@equal.register("Tuple", "Tuple") +def _euqal_tuple(x, y): + """ + Determine if two tuples are equal by element. + + Args: + x (tuple): x + y (tuple): y + + Returns: + bool, if x and y are equal by element return true, else return false. + """ + return F.tuple_equal(x, y) + + +@equal.register("List", "List") +def _euqal_list(x, y): + """ + Determine if two lists are equal by element. + + Args: + x (list): x + y (list): y + + Returns: + bool, if x and y are equal by element return true, else return false. + """ + return F.list_equal(x, y) + + +@equal.register("Tuple", "None") +def _tuple_euqal_none(x, y): + """ + Determine if tuple element equals none element. + + Args: + x: Tuple. + y: None. + + Returns: + bool, return false. + """ + return False + + +@equal.register("None", "Tuple") +def _none_equal_tuple(x, y): + """ + Determine if tuple element equals none element. + + Args: + x: None. + y: Tuple. + + Returns: + bool, return false. + """ + return False + + +@equal.register("Tensor", "Tensor") +def _tensor_equal_tensor(x, y): + """ + Determine if two tensors are equal. + + Args: + x : Tensor. + y : Tensor. + + Returns: + bool, if x == y return true, x != y return false. + """ + return F.equal(x, y) + + +@equal.register("Tensor", "None") +def _tensor_equal_none(x, y): + """ + Determine if tensor equal none. + + Args: + x : Tensor. + y : None. + + Returns: + bool, return false. + """ + return False + + +@equal.register("None", "Tensor") +def _none_equal_tensor(x, y): + """ + Determine if tensor equal none. + + Args: + x : None. + y : Tensor. + + Returns: + bool, return false. + """ + return False diff --git a/mindspore/ops/composite/multitype_ops/getitem_impl.py b/mindspore/ops/composite/multitype_ops/getitem_impl.py new file mode 100644 index 00000000000..b2b46ebbb10 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/getitem_impl.py @@ -0,0 +1,177 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation for getitem.""" + +from ...composite import base +from ... import functional as F + + +getitem = base.MultitypeFuncGraph('getitem') +""" +getitem is a metafuncgraph object which will get item from an object according to input type +using ".register" decorator. +""" + + +class _TupleSlice(base.TupleSlice_): + """ + Slices a tuple. + + Inputs: + data (tuple): A tuple to be sliced. + s (slice): The index to slice tuple data. + + Outputs: + Tuple, consists of some elements of data. + """ + + def __init__(self, name): + base.TupleSlice_.__init__(self, name) + + def __call__(self, *args): + pass + + +_tuple_slice = _TupleSlice('tuple_slice') +"""_tuple_slice is an metafuncgraph object which will slice a tuple.""" + + +class _TensorSlice(base.TensorSlice_): + """ + Slices a tensor. + + Inputs: + data (Tensor): A tensor to be sliced. + s (slice): The index to slice tuple data. + + Outputs: + Tensor, consists of some elements of data. + """ + + def __init__(self, name): + base.TensorSlice_.__init__(self, name) + + def __call__(self, *args): + pass + + +_tensor_slice = _TensorSlice('tensor_slice') +"""_tensor_slice is an metafuncgraph object which will slice a tensor.""" + + +@getitem.register("Tuple", "Number") +def _tuple_getitem_by_number(data, number_index): + """ + Getting item of tuple by number index. + + Inputs: + data (tuple): A tuple to be sliced. + number_index (Number): Index in scalar. + + Outputs: + Type, is same as the element type of data. + """ + return F.tuple_getitem(data, number_index) + + +@getitem.register("Tuple", "Slice") +def _tuple_getitem_by_slice(data, slice_index): + """ + Getting item of tuple by slice index. + + Inputs: + data (tuple): data + slice_index (Slice): Index in slice. + + Outputs: + Tuple, element type is same as the element type of data. + """ + return _tuple_slice(data, slice_index) + + +@getitem.register("List", "Number") +def _list_getitem_by_number(data, number_index): + """ + Getting item of list by number index. + + Inputs: + data (list): data in list. + number_index (Number): Index in scalar. + + Outputs: + Type is same as the element type of data. + """ + return F.list_getitem(data, number_index) + + +@getitem.register("Dictionary", "String") +def _dict_getitem_by_key(data, key): + """ + Getting item of dictionary by key which is a string. + + Inputs: + data (Dictionary): data + key (str): Key of the data. + + Outputs: + Type, is as same as the element type of data. + """ + return F.dict_getitem(data, key) + + +@getitem.register("Tensor", "Number") +def _tensor_getitem_by_number(data, number_index): + """ + Getting item of tensor by number index. + + Inputs: + data (Tensor): A tensor. + number_index (Number): Index in scalar. + + Outputs: + Tensor, element type is as same as the element type of data. + """ + return _tensor_slice(data, number_index) + + +@getitem.register("Tensor", "Slice") +def _tensor_getitem_by_slice(data, slice_index): + """ + Getting item of tensor by slice index. + + Inputs: + data (Tensor): A tensor. + slice_index (Slice): Index in slice. + + Outputs: + Tensor, element type is same as the element type of data. + """ + return _tensor_slice(data, slice_index) + + +@getitem.register("Tensor", "Tuple") +def _tensor_getitem_by_slice_tuple(data, slice_tuple_index): + """ + Getting item of tensor by slice tuple index. + + Inputs: + data (Tensor): A tensor. + slice_tuple_index (tuple): Index in tuple. + + Outputs: + Tensor, element type is same as the element type of data. + """ + return _tensor_slice(data, slice_tuple_index) diff --git a/mindspore/ops/composite/multitype_ops/less_equal_impl.py b/mindspore/ops/composite/multitype_ops/less_equal_impl.py new file mode 100644 index 00000000000..f02ab61da12 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/less_equal_impl.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""less_equal_impl""" +from mindspore.ops.composite import base +from mindspore.ops import functional as F + +# less_equal is a metagraph object which will determine if two objects are less_equal according to input type +# using ".register" decorator +less_equal = base.MultitypeFuncGraph("less_equal") + + +@less_equal.register("Number", "Number") +def _less_equal_scala(x, y): + """ + Determine whether x is less equal than y + + Args: + x(Number): Number. + y(Number): Number. + + Returns: + bool, if x <= y return true, x > y return false. + """ + return F.scalar_le(x, y) + + +@less_equal.register("Tensor", "Tensor") +def _less_equal_tensor(x, y): + """ + Determine whether tensor x is less equal than tensor y elementwise + + Args: + x(Tensor): Tensor. + y(Tensor): Tensor. + + Returns: + Tensor, return value by operator P.LessEqual. + """ + return F.tensor_le(x, y) diff --git a/mindspore/ops/composite/multitype_ops/less_impl.py b/mindspore/ops/composite/multitype_ops/less_impl.py new file mode 100644 index 00000000000..c9c20657e58 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/less_impl.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""equal_impl""" +from mindspore.ops.composite import base +from mindspore.ops import functional as F + +# less is a metafuncgraph object which will determine if two objects are less according to input type +# using ".register" decorator +less = base.MultitypeFuncGraph("less") + + +@less.register("Number", "Number") +def _less_scala(x, y): + """ + Determine whether two numbers are less. + + Args: + x(Number): Number. + y(Number): Number. + + Returns: + bool, if x < y return true, x >= y return false. + """ + return F.scalar_lt(x, y) + + +@less.register("Tensor", "Tensor") +def _less_tensor(x, y): + """ + Determine whether two tensor are less by element. + + Args: + x(Tensor): Tensor. + y(Tensor): Tensor. + + Returns: + bool, if x and y are less elements by element return true, else return false. + """ + return F.tensor_lt(x, y) diff --git a/mindspore/ops/composite/multitype_ops/logical_and_impl.py b/mindspore/ops/composite/multitype_ops/logical_and_impl.py new file mode 100644 index 00000000000..324ce3a78d1 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/logical_and_impl.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""logical_and_impl""" +from mindspore.ops.composite import base +from mindspore.ops import functional as F + +# logical_and is a metagraph object which will generate function according to input type +# using ".register" decorator +logical_and = base.MultitypeFuncGraph("logical_and") + + +@logical_and.register("Number", "Number") +def _logical_and_scala(x, y): + """ + Return logical and operation result of x and y + + Args: + x(Number): Number. + y(Number): Number. + + Returns: + bool, Return logical and operation result of x and y + """ + return F.bool_and(x.__bool__(), y.__bool__()) + + +@logical_and.register("Tensor", "Tensor") +def _logical_and_tensor(x, y): + """ + Return logical and operation result of x and y + + Args: + x(Tensor): Tensor. + y(Tensor): Tensor. + + Returns: + Tensor, Return logical and operation result of x and y + """ + return F.logical_and(x, y) diff --git a/mindspore/ops/composite/multitype_ops/logical_or_impl.py b/mindspore/ops/composite/multitype_ops/logical_or_impl.py new file mode 100644 index 00000000000..fd106f76858 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/logical_or_impl.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""logical_or_impl""" +from mindspore.ops.composite import base +from mindspore.ops import functional as F + +# logical_or is a metagraph object which will generate function according to input type +# using ".register" decorator +logical_or = base.MultitypeFuncGraph("logical_or") + + +@logical_or.register("Number", "Number") +def _logical_or_scala(x, y): + """ + Return logical or operation result of x and y + + Args: + x(Number): Number. + y(Number): Number. + + Returns: + bool, Return logical or operation result of x and y + """ + return F.bool_or(x.__bool__(), y.__bool__()) + + +@logical_or.register("Tensor", "Tensor") +def _logical_or_tensor(x, y): + """ + Return logical operation or result of x and y + + Args: + x(Tensor): Tensor. + y(Tensor): Tensor. + + Returns: + Tensor, Return logical operation or result of x and y + """ + return F.logical_or(x, y) diff --git a/mindspore/ops/composite/multitype_ops/mul_impl.py b/mindspore/ops/composite/multitype_ops/mul_impl.py new file mode 100644 index 00000000000..1d4733a46b5 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/mul_impl.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation for internal polymorphism `mul` operations.""" + +from ...composite import base +from ... import functional as F + + +mul = base.MultitypeFuncGraph("mul") +""" +`mul` is a metafuncgraph object which will multiply two objects according to input type +using ".register" decorator. +""" + + +@mul.register("Number", "Number") +def _mul_scalar(x, y): + """ + Returns x * y where x and y are all scalars. + + Outputs: + Number, equal to x * y, the type is same as x. + """ + return F.scalar_mul(x, y) + + +@mul.register("Tensor", "Tensor") +def _mul_tensor(x, y): + """ + Returns x * y by element-wise where x and y are all tensors and have same dtype. + + Outputs: + Tensor, has the same dtype as x. + """ + return F.tensor_mul(x, y) + + +@mul.register("Number", "Tensor") +def _scalar_mul_tensor(x, y): + """ + Returns x * y where x is a scalar and y is a tensor. x and y have same dtype. + + Outputs: + Tensor, has the same dtype as x. + """ + z = F.scalar_to_tensor(x, F.dtype(y)) + return F.tensor_mul(z, y) + + +@mul.register("Tensor", "Number") +def _tensor_mul_scalar(x, y): + """ + Returns x * y where x is a tensor and y is a scalar. x and y hava same dtype. + + Outputs: + Tensor, has the same dtype as x. + """ + z = F.scalar_to_tensor(y, F.dtype(x)) + return F.tensor_mul(x, z) diff --git a/mindspore/ops/composite/multitype_ops/negative_impl.py b/mindspore/ops/composite/multitype_ops/negative_impl.py new file mode 100644 index 00000000000..81fc52e3321 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/negative_impl.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation for internal polymorphism `negative` operations.""" + +from ...composite import base +from ... import functional as F + + +negative = base.MultitypeFuncGraph("negative") +""" +`negative` is a metafuncgraph object which will give the negative of an object according to its input type +using ".register" decorator. +""" + + +@negative.register("Number") +def _neg_scalar(x): + """ + Returns the negative value of scalar x. + + Outputs: + Number, negative value of x. + """ + return F.scalar_usub(x) + + +@negative.register("Tensor") +def _negative_tensor(x): + """ + Returns the negative value of tensor x by element-wise. + + Returns: + Tensor, negative value of x by element-wise. + """ + return F.neg_tensor(x) diff --git a/mindspore/ops/composite/multitype_ops/ones_like_impl.py b/mindspore/ops/composite/multitype_ops/ones_like_impl.py new file mode 100644 index 00000000000..d88193f845d --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/ones_like_impl.py @@ -0,0 +1,54 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation for internal polymorphism `ones_like_leaf` operations.""" + +from ...composite import base +from ... import functional as F +from ... import operations as P + + +ones_like_leaf = base.MultitypeFuncGraph('ones_like_leaf') +""" +`ones_like_leaf` is a metafuncgraph object which will generate a tensor filled with one according to its input type +using ".register" decorator. +""" + + +@ones_like_leaf.register("Number") +def _ones_like_scalar(x): + """Returns 1 which has the same dtype as x where x is a scalar.""" + t = F.typeof(x) + return F.scalar_cast(1.0, t) + + +@ones_like_leaf.register("Tensor") +def _ones_like_tensor(x): + """Returns a tensor with the same shape and dtype as x and all elements ars 1.""" + return P.Fill()(P.DType()(x), P.Shape()(x), 1.0) + + +ones_like = base.HyperMap(ones_like_leaf) +""" +`ones_like` is a function which can generate a graph of `ones_like` operation according to input tensor dtype. + +Example: + >>> input = Tensor([2, 3], mindspore.int32) + >>> ones = ones_like(input) # The dtype of ones is mindspore.int32 + >>> input = Tensor([2, 3], mindspore.float16) + >>> ones = ones_like(input) # The dtype of ones is mindspore.float16 +""" diff --git a/mindspore/ops/composite/multitype_ops/sub_impl.py b/mindspore/ops/composite/multitype_ops/sub_impl.py new file mode 100644 index 00000000000..4a3224a859f --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/sub_impl.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation for internal polymorphism `sub` operations.""" + +from ...composite import base +from ... import functional as F + + +sub = base.MultitypeFuncGraph("sub") +""" +`sub` is a metafuncgraph object which will compute the subtraction of two objects +using ".register" decorator. +""" + + +@sub.register("Number", "Number") +def _sub_scalar(x, y): + """Returns x - y where x and y are all scalars.""" + return F.scalar_sub(x, y) + + +@sub.register("Tensor", "Tensor") +def _sub_tensor(x, y): + """Returns x - y where x and y are all tensors and have save dtype.""" + return F.tensor_sub(x, y) + + +@sub.register("Number", "Tensor") +def _scalar_sub_tensor(x, y): + """Returns x - y where x is a scalar and y is a tensor. x and y should have same dtype.""" + z = F.scalar_to_tensor(x, F.dtype(y)) + return F.tensor_sub(z, y) + + +@sub.register("Tensor", "Number") +def _tensor_sub_scalar(x, y): + """Returns x - y where x is a tensor and y is a scalar. x and y should have same dtype.""" + z = F.scalar_to_tensor(y, F.dtype(x)) + return F.tensor_sub(x, z) diff --git a/mindspore/ops/composite/multitype_ops/zeros_like_impl.py b/mindspore/ops/composite/multitype_ops/zeros_like_impl.py new file mode 100644 index 00000000000..1c1a4f1d129 --- /dev/null +++ b/mindspore/ops/composite/multitype_ops/zeros_like_impl.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation for internal polymorphism `zeros_like_leaf` operations.""" + +from ...composite import base +from ... import functional as F + + +zeros_like_leaf = base.MultitypeFuncGraph('zeros_like_leaf') +""" +`zeros_like_leaf` is a metafuncgraph object which will generate a tensor filled with one according to its input type +using ".register" decorator. +""" + + +@zeros_like_leaf.register("Number") +def _zeros_like_scala(x): + """Returns 0 which has the same dtype as x where x is a scalar.""" + return 0 + + +newenv = base.EnvInstance_() + + +@zeros_like_leaf.register("Function") +def _zeros_like_func(x): + """ + Derivation of a function. + + Args: + x (Function): x + + Returns: + EnvInstance_, value is newenv. + """ + # Unused parameters are placeholders. + return newenv + + +@zeros_like_leaf.register("Tensor") +def _zeros_like_tensor(x): + """Returns a tensor with the same shape and dtype as x and all elements ars 1.""" + return F.zeros_like_tensor(x) + + +@zeros_like_leaf.register("TypeType") +def _zeros_like_type_type(x): + """Returns x because x is a type. This is usually used in backprop progress.""" + return x + + +@zeros_like_leaf.register("None") +def _zeros_like_type_none(x): + """Returns None where x is and should be None. This is usually used in backprop progress.""" + return x + + +@zeros_like_leaf.register("RefKeyType") +def _zeros_like_refkey_type(x): + """ + Derivation of a type. + + Args: + x (RefKeyType): x + + Returns: + RefKeyType. + """ + return x + + +@zeros_like_leaf.register("Problem") +def _zeros_like_abstract_error(x): + """ + Derivation of a AbstractError. + + Args: + x (AbstractError): return x + + Returns: + x. + """ + return x + + +# zeros_like is an object that will generate graph of zero_like operation for different type +zeros_like = base.HyperMap(zeros_like_leaf) +"""`zeros_like` is an object that will generate graph of `zero_like` operation for different type.""" diff --git a/mindspore/ops/functional.py b/mindspore/ops/functional.py new file mode 100644 index 00000000000..5adb6fac573 --- /dev/null +++ b/mindspore/ops/functional.py @@ -0,0 +1,129 @@ +# This is the Python adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). +# +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""The names of functional part are summarized here.""" + +from mindspore.common._register_for_tensor import tensor_operator_registry +from .primitive import Primitive +from . import operations as P +from .operations import _grad_ops + +typeof = Primitive('typeof') +hastype = Primitive('hastype') +cast = P.Cast() +dtype = P.DType() + + +issubclass_ = P.IsSubClass() +isinstance_ = P.IsInstance() +fill = P.Fill() +shape = P.Shape() +rank = P.Rank() +reshape = P.Reshape() +# control_depend: represent dependency between two operators +control_depend = P.ControlDepend() +merge = P.Merge() +geswitch = P.GeSwitch() +addn = P.AddN() +tensor_add = P.TensorAdd() +neg_tensor = P.Neg() +tensor_lt = P.Less() +tensor_le = P.LessEqual() +tensor_sub = P.Sub() +tensor_mul = P.Mul() +tensor_div = P.RealDiv() +strided_slice = P.StridedSlice() +same_type_shape = P.SameTypeShape() +equal = P.Equal() +assign_sub = P.AssignSub() +assign = P.Assign() +square = P.Square() +sqrt = P.Sqrt() +scalar_to_array = P.ScalarToArray() +scalar_to_tensor = P.ScalarToTensor() +tuple_to_array = P.TupleToArray() +scalar_cast = P.ScalarCast() + + +tuple_setitem = Primitive('tuple_setitem') +tuple_getitem = Primitive('tuple_getitem') +list_getitem = Primitive('list_getitem') +dict_getitem = Primitive('dict_getitem') +tuple_div = Primitive("tuple_div") +tuple_len = Primitive("tuple_len") +tuple_reversed = Primitive("tuple_reversed") +make_range = Primitive("make_range") +make_tuple = Primitive('make_tuple') +make_dict = Primitive('make_dict') +make_list = Primitive('make_list') +make_slice = Primitive('make_slice') +tuple_equal = Primitive("tuple_equal") +list_equal = Primitive("list_equal") +make_ref = Primitive("make_ref") + + +scalar_add = Primitive('scalar_add') +scalar_mul = Primitive('scalar_mul') +scalar_sub = Primitive('scalar_sub') +scalar_div = Primitive('scalar_div') +scalar_log = Primitive('scalar_log') +scalar_pow = Primitive('scalar_pow') +scalar_gt = Primitive('scalar_gt') +scalar_ge = Primitive('scalar_ge') +scalar_le = Primitive('scalar_le') +scalar_lt = Primitive('scalar_lt') +scalar_eq = Primitive('scalar_eq') +scalar_ne = Primitive('scalar_ne') +scalar_uadd = Primitive('scalar_uadd') +scalar_usub = Primitive('scalar_usub') +scalar_mod = Primitive('scalar_mod') +string_eq = Primitive('string_equal') +bool_not = Primitive("bool_not") +bool_or = Primitive("bool_or") +bool_and = Primitive("bool_and") +logical_and = P.LogicalAnd() +logical_or = P.LogicalOr() +array_to_scalar = Primitive('array_to_scalar') +is_ = Primitive("is_") +is_not = Primitive("is_not") + +broadcast_gradient_args = Primitive('BroadcastGradientArgs') +dot = Primitive('dot') +array_reduce = Primitive('array_reduce') +partial = Primitive('partial') +zeros_like_tensor = Primitive('zeros_like_tensor') +identity = Primitive('identity') +distribute = Primitive('distribute') +# depend: mount a node to another node +depend = Primitive('depend') +embed = Primitive('embed') +ref_to_embed = _grad_ops.RefToEmbed() +env_setitem = Primitive('env_setitem') +env_getitem = Primitive('env_getitem') +env_add = Primitive('env_add') +J = Primitive('J') +switch = Primitive('switch') +# for sum bprop +reduced_shape = Primitive("reduced_shape") +# shape_mul:input mush be shape multiply elemts in tuple(shape) +shape_mul = Primitive("shape_mul") +# a primitive to compare between tuple. +stop_gradient = Primitive("stop_gradient") + +tensor_operator_registry.register('__add__', tensor_add) + +tensor_operator_registry.register('__mul__', tensor_mul) diff --git a/mindspore/ops/op_info_register.py b/mindspore/ops/op_info_register.py new file mode 100644 index 00000000000..80f40ff1d69 --- /dev/null +++ b/mindspore/ops/op_info_register.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Operators info register.""" + +import os +import inspect +from mindspore._c_expression import Oplib +from mindspore._checkparam import ParamValidator as validator + +# path of built-in op info register. +BUILT_IN_OPS_REGISTER_PATH = "mindspore/ops/_op_impl" + + +def op_info_register(op_info): + """ + A decorator used as register of operator implementation. + + Note: + 'op_info' must be a str of json format represent the op info, the op info will be added into oplib. + + Args: + op_info (str): op info of json format. + + Returns: + Function, returns a decorator for op info register. + """ + def register_decorator(func): + validator.check_type("op_info", op_info, [str]) + op_lib = Oplib() + file_path = os.path.realpath(inspect.getfile(func)) + # keep the path custom ops implementation. + imply_path = "" if BUILT_IN_OPS_REGISTER_PATH in file_path else file_path + if not op_lib.reg_op(op_info, imply_path): + raise ValueError('Invalid op info {}:\n{}\n'.format(file_path, op_info)) + + def wrapped_function(*args, **kwargs): + return func(*args, **kwargs) + return wrapped_function + return register_decorator diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py new file mode 100644 index 00000000000..a75b078df80 --- /dev/null +++ b/mindspore/ops/operations/__init__.py @@ -0,0 +1,229 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Primitive operator classes. + +A collection of operators to build nerual networks or computing functions. +""" + +from .array_ops import (Argmax, Argmin, Cast, ConcatOffset, Concat, + Diag, DType, ExpandDims, Eye, + Fill, GatherNd, GatherV2, InvertPermutation, + IsInstance, IsSubClass, ArgMaxWithValue, OnesLike, ZerosLike, + Rank, Reshape, ResizeNearestNeighbor, ArgMinWithValue, + SameTypeShape, + ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, + Shape, Size, Slice, Split, + Squeeze, StridedSlice, Tile, + Transpose, TruncatedNormal, TupleToArray, + UnsortedSegmentSum, SpaceToDepth, DepthToSpace) +from .comm_ops import (AllGather, AllReduce, _AlltoAll, ReduceScatter, Broadcast, + _MirrorOperator, ReduceOp, _VirtualDataset, + _VirtualDiv, _GetTensorSlice) +from .debug_ops import (ImageSummary, InsertGradientOf, ScalarSummary, + TensorSummary, Print) +from .control_ops import ControlDepend, GeSwitch, Merge +from .inner_ops import ScalarCast +from .math_ops import (Abs, ACos, AddN, AssignAdd, AssignSub, BatchMatMul, + ReduceMax, ReduceMin, ReduceMean, ReduceSum, ReduceAll, ReduceProd, CumProd, + Cos, Div, Equal, EqualCount, Exp, Floor, FloorDiv, + Greater, GreaterEqual, Less, LessEqual, Log, LogicalAnd, + LogicalNot, LogicalOr, MatMul, Maximum, + Minimum, Mul, Neg, NMSWithMask, NotEqual, + NPUAllocFloatStatus, NPUClearFloatStatus, + NPUGetFloatStatus, Pow, RealDiv, + Reciprocal, CumSum, + Sin, Sqrt, Rsqrt, + Square, Sub, TensorAdd, Sign, Round) +from .random_ops import (RandomChoiceWithMask) +from .nn_ops import (LSTM, SGD, Adam, ApplyMomentum, BatchNorm, + BiasAdd, Conv2D, + DepthwiseConv2dNative, + DropoutDoMask, + DropoutGenMask, Flatten, FusedBatchNorm, + Gelu, Elu, + GetNext, L2Normalize, LayerNorm, + LogSoftmax, + MaxPool, + AvgPool, Conv2DBackpropInput, + MaxPoolWithArgmax, OneHot, Pad, PReLU, ReLU, ReLU6, + ResizeBilinear, Sigmoid, + SigmoidCrossEntropyWithLogits, + SmoothL1Loss, Softmax, + SoftmaxCrossEntropyWithLogits, ROIAlign, + SparseSoftmaxCrossEntropyWithLogits, Tanh, + TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl) +from .other_ops import Assign, IOU, BoundingBoxDecode, BoundingBoxEncode, CheckValid, MakeRefKey + + +__all__ = [ + 'TensorAdd', + 'Argmax', + 'Argmin', + 'ArgMaxWithValue', + 'ArgMinWithValue', + 'AddN', + 'Sub', + 'CumSum', + 'MatMul', + 'BatchMatMul', + 'Mul', + 'Pow', + 'Exp', + 'Rsqrt', + 'Sqrt', + 'Square', + 'Conv2D', + 'Flatten', + 'MaxPoolWithArgmax', + 'FusedBatchNorm', + 'BatchNorm', + 'MaxPool', + 'TopK', + 'Adam', + 'Softmax', + 'LogSoftmax', + 'SoftmaxCrossEntropyWithLogits', + 'ROIAlign', + 'SparseSoftmaxCrossEntropyWithLogits', + 'SGD', + 'ApplyMomentum', + 'ExpandDims', + 'Cast', + 'IsSubClass', + 'IsInstance', + 'Reshape', + 'Squeeze', + 'Transpose', + 'OneHot', + 'GatherV2', + 'Concat', + 'Tile', + 'BiasAdd', + 'Gelu', + 'Minimum', + 'Maximum', + 'StridedSlice', + 'ReduceSum', + 'ReduceMean', + 'LayerNorm', + 'Rank', + 'Less', + 'LessEqual', + 'RealDiv', + 'Div', + 'TruncatedNormal', + 'Fill', + 'OnesLike', + 'ZerosLike', + 'Select', + 'Split', + 'ReLU', + 'ReLU6', + 'Elu', + 'Sigmoid', + 'Tanh', + 'RandomChoiceWithMask', + 'ResizeBilinear', + 'ScalarSummary', + 'ImageSummary', + 'TensorSummary', + "Print", + 'InsertGradientOf', + 'InvertPermutation', + 'Shape', + 'DropoutDoMask', + 'DropoutGenMask', + 'Neg', + 'Slice', + 'DType', + 'NPUAllocFloatStatus', + 'NPUGetFloatStatus', + 'NPUClearFloatStatus', + 'Reciprocal', + 'SmoothL1Loss', + 'ReduceAll', + 'ScalarToArray', + 'ScalarToTensor', + 'TupleToArray', + 'ControlDepend', + 'GeSwitch', + 'Merge', + 'SameTypeShape', + 'CheckValid', + 'BoundingBoxEncode', + 'BoundingBoxDecode', + 'L2Normalize', + 'ScatterNd', + 'ResizeNearestNeighbor', + 'Pad', + 'GatherNd', + 'ScatterNdUpdate', + 'Floor', + 'NMSWithMask', + 'IOU', + 'MakeRefKey', + 'AvgPool', + # Back Primitive + 'Equal', + 'EqualCount', + 'NotEqual', + 'Greater', + 'GreaterEqual', + 'LogicalNot', + 'LogicalAnd', + 'LogicalOr', + 'Size', + 'DepthwiseConv2dNative', + 'ConcatOffset', + 'UnsortedSegmentSum', + "AllGather", + "AllReduce", + "ReduceScatter", + "Broadcast", + "ReduceOp", + 'ScalarCast', + 'GetNext', + 'ReduceMax', + 'ReduceMin', + 'ReduceProd', + 'CumProd', + 'Log', + 'SigmoidCrossEntropyWithLogits', + 'FloorDiv', + "PReLU", + "Cos", + "ACos", + "Diag", + 'Eye', + 'Assign', + 'AssignAdd', + 'AssignSub', + "Sin", + "LSTM", + "Abs", + "BinaryCrossEntropy", + "SparseApplyAdagrad", + "SpaceToDepth", + "DepthToSpace", + "Conv2DBackpropInput", + "Sign", + "LARSUpdate", + "Round", + "ApplyFtrl", +] + +__all__.sort() diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py new file mode 100644 index 00000000000..a699c23adc0 --- /dev/null +++ b/mindspore/ops/operations/_grad_ops.py @@ -0,0 +1,961 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Operators for gradients.""" + +import math +from ..._c_expression import signature_rw as sig_rw +from ..._c_expression import signature_kind as sig_kind +from ..primitive import Primitive, PrimitiveWithInfer, prim_attr_register +from ..._checkparam import ParamValidator as validator +from ..._checkparam import Rel, check_int_positive, check_bool +from ...common import dtype as mstype + + +class AbsGrad(PrimitiveWithInfer): + """Computes gradients for abs operation.""" + + @prim_attr_register + def __init__(self): + """init AbsGrad""" + + def infer_shape(self, y, dy): + return y + + def infer_dtype(self, y, dy): + return y + + +class ACosGrad(PrimitiveWithInfer): + """ + Computes ACosGrad of input element-wise. + + Returns: + Tensor, has the same type as input. + """ + + @prim_attr_register + def __init__(self): + """init ACosGrad""" + + def infer_shape(self, x, dout): + validator.check_param_equal("x", x, "dout", dout) + return x + + def infer_dtype(self, x, dout): + args = {"x": x, "dout": dout} + validator.check_type_same(args, mstype.number_type) + return x + + +class BatchNormGrad(PrimitiveWithInfer): + """Performs grad of BatchNorm operation.""" + + @prim_attr_register + def __init__(self, is_training=False, epsilon=1e-5): + self.is_training = validator.check_type('is_training', is_training, (bool,)) + self.epsilon = validator.check_number_range('epsilon', epsilon, 0, 1, Rel.INC_RIGHT) + self.add_prim_attr('data_format', "NCHW") + + def infer_shape(self, y_backprop_shape, x_shape, scale_shape, reserve_1_shape, reserve_2_shape, reserve_3_shape): + validator.check("BatchNorm y_backprop_shape", y_backprop_shape, "BatchNorm x_shape", x_shape) + return (x_shape, scale_shape, scale_shape, reserve_1_shape, reserve_2_shape) + + def infer_dtype(self, y_backprop_type, x_type, scale_type, reserve_1_type, reserve_2_type, reserve_3_type): + return (x_type, scale_type, scale_type, reserve_1_type, reserve_2_type) + + +class BiasAddGrad(Primitive): + """Computes gradients of BiasAdd.""" + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['dout'], outputs=['output']) + self.add_prim_attr('data_format', 'NCHW') + + def __call__(self, d_output): + raise NotImplementedError + + +class BinaryCrossEntropyGrad(PrimitiveWithInfer): + """Computes gradients for `BinaryCrossEntropy` operation.""" + @prim_attr_register + def __init__(self, reduction='mean'): + self.reduction = validator.check_string('reduction', reduction, ['none', 'mean', 'sum']) + + def infer_shape(self, x_shape, y_shape, doutput_shape, weight_shape): + validator.check_param_equal('x_shape', x_shape, 'y_shape', y_shape) + if weight_shape: + validator.check_param_equal('y_shape', y_shape, 'weight_shape', weight_shape) + return x_shape + + def infer_dtype(self, x_type, y_type, doutput_type, weight_type): + args = {'x_type': x_type, 'y_type': y_type, 'doutput_type': doutput_type} + validator.check_type_same(args, (mstype.float16, mstype.float32)) + if weight_type: + validator.check_two_types_same('x_type', x_type, 'weight_type', weight_type) + return x_type + + +class Conv2DBackpropFilter(PrimitiveWithInfer): + """ + Computes the gradients of convolution with respect to the filter. + + Args: + out_channel (int): The dimensionality of the output space. + kernel_size (Union[int, tuple[int]]): The size of the convolution window. + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + pad (int): The pad value to fill. Default: 0. + mode (int): 0 Math convolutiuon, 1 cross-correlation convolution , + 2 deconvolution, 3 depthwise convolution. Default: 1. + stride (int): The stride to apply conv filter. Default: 1. + dilation (int): Specifies the dilation rate to use for dilated convolution. Default: 1. + group (int): Splits input into groups. Default: 1. + + Returns: + Tensor, the gradients of convolution. + """ + + @prim_attr_register + def __init__(self, + out_channel, + kernel_size, + pad_mode="valid", + pad=0, + pad_list=(0, 0, 0, 0), + mode=1, + stride=1, + dilation=1, + group=1): + """init Convolution""" + self.init_prim_io_names(inputs=['out_backprop', 'input', 'filter_sizes'], outputs=['output']) + self.out_channel = out_channel + self.kernel_size = kernel_size + self.mode = mode + pad_mode = pad_mode.upper() + self.add_prim_attr('pad_mode', pad_mode) + self.pad = pad + self.stride = stride + self.dilation = dilation + self.group = group + self.add_prim_attr('data_format', "NCHW") + + def __infer__(self, doutput, x, w_size): + w_size_v = w_size['value'] + validator.check_type('w_size', w_size_v, [tuple]) + for i, dim_len in enumerate(w_size_v): + validator.check_type("w_size[%d]" % i, dim_len, [int]) + validator.check_typename('x_dtype', x['dtype'], [mstype.int8, mstype.int32, mstype.float16, mstype.float32]) + validator.check_two_types_same('doutput_dtype', doutput['dtype'], 'x_dtype', x['dtype']) + out = { + 'value': None, + 'shape': w_size_v, + 'dtype': doutput['dtype'], + } + return out + + +class DepthwiseConv2dNativeBackpropFilter(PrimitiveWithInfer): + """ + Returns the gradient of filter for DepthwiseConv2dNative. + + Applies depthwise conv2d for the input, which will generate more channels with channel_multiplier. + + Refer to class DepthwiseConv2dNative for more details. + + Args: + channel_multiplier (int): The multipiler for the original output conv. + kernel_size (int or tuple): The size of the conv kernel. + mode (int): 0 Math convolutiuon, 1 cross-correlation convolution, + 2 deconvolution,3 depthwise convolution. Defaul: 3. + pad_mode (str): The mode to fill padding which can be: "valid", "same" or "pad". Default: "valid". + pad (int): The pad value to fill. Default: 0. + pads (tuple): The pad list like (top, bottom, left, right). Default: (0, 0, 0, 0). + stride (int): The stride to apply conv filter. Default: 1. + dilation (int): Specifies the space to use between kernel elements. Default: 1. + group (int): Splits input into groups. Default: 1. + + Returns: + Tensor, the value is the gradient of filter for DepthwiseConv2dNative. + """ + + @prim_attr_register + def __init__(self, + channel_multiplier, + kernel_size, + pad_mode="valid", + pad=0, + pads=(0, 0, 0, 0), + mode=3, + stride=1, + dilation=1, + group=1): + """init Convolution""" + self.init_prim_io_names(inputs=['input', 'filter_size', 'dout'], outputs=['output']) + self.channel_multiplier = channel_multiplier + self.kernel_size = kernel_size + self.mode = mode + self.pad_mode = pad_mode + self.pad = pad + self.pads = pads + self.stride = stride + self.dilation = dilation + self.group = group + self.add_prim_attr('data_format', "NCHW") + + def __call__(self, x, w_size, dout): + raise NotImplementedError + + def __infer__(self, x, w_size, dout): + w_size_v = w_size['value'] + args = {'x_dtype': x['dtype'], 'dout_type': dout['dtype']} + validator.check_type_same(args, mstype.number_type) + out = { + 'value': None, + 'shape': w_size_v, + 'dtype': dout['dtype'], + } + return out + + +class DepthwiseConv2dNativeBackpropInput(PrimitiveWithInfer): + """ + Returns the gradient of input for DepthwiseConv2dNative. + + Applies depthwise conv2d for the input, which will generate more channels with channel_multiplier. + + Args: + channel_multiplier (int): The multipiler for the original output conv. + kernel_size (int or tuple): The size of the conv kernel. + mode (int): 0 Math convolutiuon, 1 cross-correlation convolution , + 2 deconvolution,3 depthwise convolution. Default: 3. + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + pad (int): the pad value to fill. Default: 0. + pads (tuple): The pad list like (top, bottom, left, right). Default: (0, 0, 0, 0). + stride (int): the stride to apply conv filter. Default: 1. + dilation (int): Specifies the space to use between kernel elements. Default: 1. + group (int): Splits input into groups. Default: 1. + + Returns: + Tensor, the value is the gradient of input for DepthwiseConv2dNative. + """ + + @prim_attr_register + def __init__(self, + channel_multiplier, + kernel_size, + pad_mode="valid", + pad=0, + pads=(0, 0, 0, 0), + mode=3, + stride=1, + dilation=1, + group=1): + """init Convolution""" + self.init_prim_io_names(inputs=['input_size', 'filter', 'dout'], outputs=['output']) + self.channel_multiplier = channel_multiplier + self.kernel_size = kernel_size + self.mode = mode + self.pad_mode = pad_mode + self.pad = pad + self.pads = pads + self.stride = stride + self.dilation = dilation + self.group = group + self.add_prim_attr('data_format', "NCHW") + + def __call__(self, x_size, w, dout): + raise NotImplementedError + + def __infer__(self, x_size, w, dout): + args = {'w_dtype': w['dtype'], 'dout_type': dout['dtype']} + validator.check_type_same(args, mstype.number_type) + x_size_v = x_size['value'] + out = { + 'value': None, + 'shape': x_size_v, + 'dtype': dout['dtype'], + } + return out + + +class FlattenGrad(PrimitiveWithInfer): + """Performs gradients of Flatten.""" + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['x', 'shape'], outputs=['output']) + + def __infer__(self, *args): + out = { + 'value': None, + 'shape': args[1]['value'], + 'dtype': args[0]['dtype'], + } + return out + + +class FusedBatchNormGrad(Primitive): + """Gradients of FusedBatchNorm operation.""" + + @prim_attr_register + def __init__(self, epsilon=0.0, momentum=0.1): + self.init_prim_io_names(inputs=['dy', 'x', 'scale', 'save_mean', 'save_inv_variance'], + outputs=['dx', 'bn_scale', 'bn_bias']) + + def __call__(self, dy, x, scale, save_mean, save_inv_variance): + raise NotImplementedError + + +class GeluGrad(PrimitiveWithInfer): + """Gradients of Gelu operation.""" + + @prim_attr_register + def __init__(self): + """init GeluGrad""" + + def infer_shape(self, y_backprop_shape, x_shape, y_shape): + return x_shape + + def infer_dtype(self, y_backprop_dtype, x_dtype, y_dtype): + validator.check_typename("y_backprop_dtype", y_backprop_dtype, (mstype.float16, mstype.float32)) + validator.check_typename("x_dtype", x_dtype, (mstype.float16, mstype.float32)) + validator.check_typename("y_dtype", y_dtype, (mstype.float16, mstype.float32)) + return x_dtype + + +class _PoolGrad(PrimitiveWithInfer): + """Gradients of the max/avg pool operation.""" + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + self.init_prim_io_names(inputs=['x_origin', 'out_origin', 'grad'], outputs=['output']) + self.ksize = ksize + self.strides = strides + self.padding = padding + + self.ksize = validator.check_type('ksize', self.ksize, [int, tuple]) + self.strides = validator.check_type('strides', self.strides, [int, tuple]) + + validator.check_type('padding', self.padding, [str]) + self.padding = validator.check_string('padding', self.padding, ['VALID', 'SAME']) + self.add_prim_attr("padding", self.padding) + self.add_prim_attr('data_format', "NCHW") + + if isinstance(self.ksize, int): + self.pool_h = validator.check_integer("ksize", self.ksize, 1, Rel.GE) + self.pool_w = self.pool_h + self.add_prim_attr("ksize", (1, 1, self.ksize, self.ksize)) + elif isinstance(self.ksize, tuple): + if (len(self.ksize) != 2 and len(self.ksize) != 4): + raise ValueError('Attr \'ksize\' of \'Pool\' Op passed ' + + str(self.ksize)+', should be a int or a tuple of length 2 or 4.') + for ksize_val in self.ksize: + if (not isinstance(ksize_val, int)) or (ksize_val <= 0): + raise ValueError('Each value of attr \'ksize\' of \'MaxPool\' Op passed ' + + str(self.ksize)+', should be int and greater than 0.') + self.pool_h = self.ksize[-2] + self.pool_w = self.ksize[-1] + self.add_prim_attr("ksize", (1, 1, self.ksize[-2], self.ksize[-1])) + + if isinstance(self.strides, int): + self.stride_h = validator.check_integer("strides", self.strides, 1, Rel.GE) + self.stride_w = self.stride_h + self.add_prim_attr("strides", (1, 1, self.strides, self.strides)) + elif isinstance(self.strides, tuple): + if (len(self.strides) != 2 and len(self.strides) != 4): + raise ValueError('Attr \'strides\' of \'MaxPool\' Op passed ' + + str(self.strides)+', should be a int or a tuple of length 2 or 4.') + for stride_val in self.strides: + if (not isinstance(stride_val, int)) or (stride_val <= 0): + raise ValueError('Each value of attr \'strides\' of \'MaxPool\' Op passed ' + + str(self.strides)+', should be int and greater than 0.') + self.stride_h = self.strides[-2] + self.stride_w = self.strides[-1] + self.add_prim_attr("strides", (1, 1, self.strides[-2], self.strides[-1])) + + if self.padding == "VALID": + self.pad = 0 + elif self.padding == "SAME": + self.pad = math.floor((self.pool_h - 1) / 2) + else: + raise ValueError('The padding should be str and must be SAME or VALID,' + ' but got {}.'.format(self.padding)) + + +class AvgPoolGrad(_PoolGrad): + """Gradients of the avg pool operation.""" + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + super(AvgPoolGrad, self).__init__(ksize, strides, padding) + + def __infer__(self, origin_input, dout): + out = { + 'value': None, + 'shape': tuple(origin_input['value']), + 'dtype': dout['dtype'], + } + + return out + + +class AvgPoolGradGpu(_PoolGrad): + """Gradients of the avg pool operation for gpu.""" + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + super(AvgPoolGradGpu, self).__init__(ksize, strides, padding) + + def infer_shape(self, x1_shape, x2_shape, grad_shape): + return x1_shape + + def infer_dtype(self, x1_dtype, x2_dtype, grad_dtype): + return x1_dtype + + +class MaxPoolGrad(_PoolGrad): + """Performs gradients of the max pool operation.""" + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + super(MaxPoolGrad, self).__init__(ksize, strides, padding) + + def infer_shape(self, x1_shape, x2_shape, grad_shape): + return x1_shape + + def infer_dtype(self, x1_dtype, x2_dtype, grad_dtype): + return x1_dtype + + +class MaximumGrad(Primitive): + """Grad for maximum.""" + + @prim_attr_register + def __init__(self, grad_x=True, grad_y=True): + """Init MaximumGrad""" + + def __call__(self, x, y, dout): + raise NotImplementedError + + +class MaxPoolGradWithArgmax(PrimitiveWithInfer): + """Computes the gradients of MaxPoolWithArgmax.""" + + @prim_attr_register + def __init__(self, + pad_mode="valid", + window=0, + pad=0, + stride=1, + data_mode=1, + ceil_mode=0, + alpha=1.0, + beta=0.0): + self.init_prim_io_names(inputs=['x', 'grad', 'argmax'], outputs=['output']) + + self.window = window + self.pool_h = self.pool_w = window + self.pad = pad + self.pad_mode = pad_mode + self.stride = stride + self.data_mode = data_mode + self.ceil_mode = ceil_mode + + def infer_shape(self, x_shape, grad_shape, argmax_shape): + if not grad_shape: + raise TypeError("The dout of MaxPoolGradWithArgmax should be a Tensor.") + return x_shape + + def infer_dtype(self, x_dtype, grad_dtype, argmax_dtype): + return grad_dtype + + +class MinimumGrad(Primitive): + """Grad for minimum.""" + + @prim_attr_register + def __init__(self, grad_x=True, grad_y=True): + """Init MinimumGrad""" + + def __call__(self, x, y, dout): + raise NotImplementedError + + +class L2NormalizeGrad(PrimitiveWithInfer): + r""" + Gradients of L2 normalize. + + Args: + axis (int): The begin axis for the input to apply L2 normalize. Default: 0. + epsilon (float): A small value added for numerical stability. Default: 1e-4. + + Inputs: + - **input_x** (Tensor) - Should be the input `weight` of forward operator L2Normalize. + - **out** (Tensor) - Should be the output of forward operator L2Normalize. + - **dout** (Tensor) - The backprop of the next layer. + + Outputs: + Tensor, gradients of L2Normalize `input_x`. + """ + + @prim_attr_register + def __init__(self, axis=0, epsilon=1e-4): + validator.check_type('axis', axis, [int]) + validator.check_type('epsilon', epsilon, [int, float]) + + def infer_shape(self, input_x, out, dout): + validator.check_param_equal('input_x', input_x, 'out', out) + validator.check_param_equal('input_x', input_x, 'dout', dout) + return input_x + + def infer_dtype(self, input_x, out, dout): + args = {'input_x': input_x, 'out': out, 'dout': dout} + validator.check_type_same(args, mstype.number_type) + return input_x + + +class LayerNormGrad(Primitive): + """ + Applies the layer normalization to the input array. + + This operator will calculate the input gradients of layernorm. + + Args: + begin_norm_axis (int): The begin axis for the input to apply layernorm. Default: 1. + begin_params_axis (int): The begin axis for the parameter input to apply layernorm. Default: 1. + + Returns: + tuple[int], tuple of 3 values (the gradients of layernorm input, gamma, beta). + """ + + @prim_attr_register + def __init__(self, begin_norm_axis=1, begin_params_axis=1): + """init""" + self.begin_norm_axis = validator.check_type('begin_norm_axis', begin_norm_axis, [int]) + self.begin_params_axis = validator.check_type('begin_params_axis', begin_params_axis, [int]) + + def __call__(self, x, dy, variance, mean, gamma): + raise NotImplementedError + + +class LogSoftmaxGrad(PrimitiveWithInfer): + """Computes gradient for the Log Softmax activation.""" + + @prim_attr_register + def __init__(self, axis=-1): + """init LogSoftmaxGrad""" + validator.check_type("axis", axis, [int]) + + def infer_shape(self, dout, logits): + rank = len(logits) + validator.check_int_range('axis', self.axis, -rank - 1, rank, Rel.INC_BOTH) + return logits + + def infer_dtype(self, dout, logits): + validator.check_subclass("logits", logits, mstype.tensor) + return logits + + +class LSTMGradData(PrimitiveWithInfer): + """Computes the data gradients of LSTM.""" + + @prim_attr_register + def __init__(self, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + self.input_size = check_int_positive(input_size) + self.hidden_size = check_int_positive(hidden_size) + self.num_layers = check_int_positive(num_layers) + self.has_bias = check_bool(has_bias) + self.bidirectional = check_bool(bidirectional) + self.dropout = validator.check_type("dropout", dropout, [float]) + self.dropout = validator.check_number_range('dropout', dropout, 0, 1, Rel.INC_BOTH) + + if bidirectional: + self.num_directions = 2 + else: + self.num_directions = 1 + + def infer_shape(self, y_shape, dy_shape, dhy_shape, dcy_shape, w_shape, + hx_shape, cx_shape, reserve_shape, state_shape): + # dhy and dcy should be same shape + validator.check_integer("h_shape", len(dhy_shape), 3, Rel.EQ) + validator.check_integer("h_shape", len(dhy_shape), len(dcy_shape), Rel.EQ) + validator.check_integer("h_shape[0]", dhy_shape[0], dcy_shape[0], Rel.EQ) + validator.check_integer("h_shape[1]", dhy_shape[1], dcy_shape[1], Rel.EQ) + validator.check_integer("h_shape[2]", dhy_shape[2], dcy_shape[2], Rel.EQ) + + validator.check_integer("h_shape[0]", dhy_shape[0], self.num_layers * self.num_directions, Rel.EQ) + validator.check_integer("h_shape[2]", dhy_shape[2], self.hidden_size, Rel.EQ) + + # dy: (seq_len, batch_size, hidden_size * num_directions) + validator.check_integer("dy_shape", len(dy_shape), 3, Rel.EQ) + validator.check_integer("dy[1]", dy_shape[1], dhy_shape[1], Rel.EQ) + validator.check_integer("dy[2]", dy_shape[2], self.hidden_size * self.num_directions, Rel.EQ) + + # (seq_len, batch_size, input_size) + dx_shape = (y_shape[0], y_shape[1], self.input_size) + dhx_shape = dhy_shape + dcx_shape = dcy_shape + + return (dx_shape, dhx_shape, dcx_shape) + + def infer_dtype(self, y_dtype, dy_dtype, dhy_dtype, dcy_dtype, w_dtype, + hx_dtype, cx_dtype, reserve_dtype, state_dtype): + validator.check_typename("dy_dtype", dy_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("dhy_dtype", dhy_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("dcy_dtype", dcy_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("datatype", dy_dtype, (dhy_dtype.element_type(),)) + validator.check_typename("datatype", dy_dtype, (dcy_dtype.element_type(),)) + return (dy_dtype, dy_dtype, dy_dtype) + + +class LSTMGradWeight(PrimitiveWithInfer): + """Computes the weight gradients of LSTM.""" + + @prim_attr_register + def __init__(self, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + self.input_size = check_int_positive(input_size) + self.hidden_size = check_int_positive(hidden_size) + self.num_layers = check_int_positive(num_layers) + self.has_bias = check_bool(has_bias) + self.bidirectional = check_bool(bidirectional) + self.dropout = validator.check_type("dropout", dropout, [float]) + self.dropout = validator.check_number_range('dropout', dropout, 0, 1, Rel.INC_BOTH) + + if bidirectional: + self.num_directions = 2 + else: + self.num_directions = 1 + + def infer_shape(self, x_shape, hx_shape, y_shape, reserve_shape, state_shape): + weight_size = 0 + gate_size = 4 * self.hidden_size + for layer in range(self.num_layers): + for _ in range(self.num_directions): + input_layer_size = self.input_size if layer == 0 else self.hidden_size * self.num_directions + weight_size += gate_size * input_layer_size + weight_size += gate_size * self.hidden_size + if self.has_bias: + weight_size += 2 * gate_size + + return (weight_size, 1, 1) + + def infer_dtype(self, x_dtype, hx_dtype, y_dtype, reserve_dtype, state_dtype): + return hx_dtype + + +class PReLUGrad(PrimitiveWithInfer): + r""" + Gradients of PReLU operation. + + Inputs: + - **y_backprop** (Tensor) - Representing the backprop of the next layer. + - **input_x** (Tensor) - Should be the input `input_x` of forward operator PRelu. + - **weight** (Tensor) - Float Tensor, w > 0, should be the input `weight` of forward operator PRelu. + + Outputs: + Tensor, with the same type as `input_x`. + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, y_backprop_shape, A_shape, w_shape): + return y_backprop_shape, w_shape + + def infer_dtype(self, y_backprop_dtype, A_dtype, w_dtype): + validator.check_typename("y_backprop_dtype", y_backprop_dtype, (mstype.float16, mstype.float32)) + validator.check_typename("A_dtype", A_dtype, (mstype.float16, mstype.float32)) + validator.check_typename("w_dtype", w_dtype, (mstype.float16, mstype.float32)) + return y_backprop_dtype, w_dtype + + +class ReluGrad(Primitive): + """Performs grad of Relu operation.""" + + @prim_attr_register + def __init__(self): + """init ReluGrad""" + self.init_prim_io_names(inputs=['y_backprop', 'x'], outputs=['output']) + + def __call__(self, y_backprop, x): + raise NotImplementedError + + +class ReLU6Grad(PrimitiveWithInfer): + """Performs grad of ReLU6 operation.""" + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['y_grad', 'x'], outputs=['output']) + + def __call__(self, y_grad, x): + raise NotImplementedError + + def infer_shape(self, y_grad_shape, x_shape): + return x_shape + + def infer_dtype(self, y_grad_dtype, x_dtype): + validator.check_typename("y_grad_dtype", y_grad_dtype, (mstype.float16, mstype.float32)) + validator.check_typename("x_dtype", x_dtype, (mstype.float16, mstype.float32)) + return x_dtype + + +class EluGrad(PrimitiveWithInfer): + """Performs grad of Elu operation.""" + + @prim_attr_register + def __init__(self): + """Init EluGrad""" + + def infer_shape(self, y_grad_shape, x_shape): + return x_shape + + def infer_dtype(self, y_grad_dtype, x_dtype): + args_type = {'y_grad': y_grad_dtype, 'x': x_dtype} + validator.check_args_tensor(args_type) + args_dtype = {'y_grad_dtype': y_grad_dtype, 'x_dtype': x_dtype} + validator.check_type_same(args_dtype, mstype.float_type) + return x_dtype + + +class ResizeBilinearGrad(PrimitiveWithInfer): + """Performs grad of ResizeBilinear operation.""" + + @prim_attr_register + def __init__(self, align_corners=False): + """init""" + + def infer_shape(self, dout_shape, orig_shape): + return orig_shape + + def infer_dtype(self, dout_dtype, orig_type): + return dout_dtype + + +class ResizeNearestNeighborGrad(PrimitiveWithInfer): + """ + Compute gradient of `ResizeNearestNeighbor` operator. + + Note: + The shape of input parameter `size` must be (height, width). + + Args: + align_corners (bool): Whether the centers of the 4 corner pixels of the input + and output tensors are aligned. Default: False. + """ + + @prim_attr_register + def __init__(self, align_corners=False): + """Init ResizeNearestNeighborGrad""" + self.init_prim_io_names(inputs=['grads', 'size'], outputs=['y']) + + def __infer__(self, grads, size): + shp = (grads['shape'][0],) + (grads['shape'][1],) + size['value'] + return {'shape': shp, + 'dtype': grads['dtype'], + 'value': None} + + +class ROIAlignGrad(PrimitiveWithInfer): + """ + ROIAlignGrad operator. + + Args: + pooled_height (int): The output feature height. + pooled_width (int): The output feature width. + spatial_scale (float): The feature stride. + sample_num (int): Number of sampling points. Default: 2. + """ + + @prim_attr_register + def __init__(self, xdiff_shape, pooled_height, pooled_width, spatial_scale, sample_num=2): + """init ROIAlignGrad""" + validator.check_type("pooled_height", pooled_height, [int]) + validator.check_type("pooled_width", pooled_width, [int]) + validator.check_type("spatial_scale", spatial_scale, [float]) + validator.check_type("sample_num", sample_num, [int]) + validator.check_type("xdiff_shape", xdiff_shape, [tuple]) + self.xdiff_shape = xdiff_shape + self.pooled_height = pooled_height + self.pooled_width = pooled_width + self.spatial_scale = spatial_scale + self.sample_num = sample_num + + def infer_shape(self, ydiff_shape, rois_shape): + return self.xdiff_shape + + def infer_dtype(self, ydiff_type, rois_type): + return ydiff_type + + +class SigmoidGrad(PrimitiveWithInfer): + """Gets the gradient of Sigmoid operation.""" + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, out, dout): + return out + + def infer_dtype(self, out, dout): + validator.check_typename("dout dtype", dout, (mstype.float16, mstype.float32)) + validator.check_typename("out dtype", out, (mstype.float16, mstype.float32)) + args = {"out type": out, "dout type": dout} + validator.check_type_same(args, mstype.number_type) + return out + + +class SigmoidCrossEntropyWithLogitsGrad(PrimitiveWithInfer): + """Computes the gradients of `SigmoidCrossEntropyWithLogits`.""" + + @prim_attr_register + def __init__(self): + """Init SigmoidCrossEntropyWithLogitsGrad""" + self.init_prim_io_names(inputs=['x', 'y', 'dout'], outputs=['x_grad']) + + def infer_shape(self, x_shape, y_shape, dout_shape): + validator.check_param_equal("x_shape", x_shape, "y_shape", y_shape) + validator.check_param_equal("x_shape", x_shape, "dout_shape", dout_shape) + return x_shape + + def infer_dtype(self, x_dtype, y_dtype, dout_dtype): + args = {"x_dtype": x_dtype, "y_dtype": y_dtype, 'dout_dtype': dout_dtype} + validator.check_type_same(args, mstype.number_type) + return dout_dtype + + +class SliceGrad(PrimitiveWithInfer): + """Reverse of slice.""" + + @prim_attr_register + def __init__(self): + """init SliceGrad""" + self.init_prim_io_names(inputs=['dy', 'x', 'begin', 'size'], outputs=['dx']) + + def __infer__(self, dy, x, begin, size): + dy_shape, x_shape, size_value = dy['shape'], x['shape'], size['value'] + dy_shape_len = len(dy_shape) + for i in range(dy_shape_len): + validator.check(f'dy_shape[{i}]', dy_shape[i], f'x_shape[{i}]', x_shape[i], Rel.LE) + validator.check(f'dy_shape[{i}]', dy_shape[i], f'size_shape[{i}]', size_value[i], Rel.EQ) + return {'shape': x_shape, + 'dtype': x['dtype'], + 'value': None} + + +class SmoothL1LossGrad(PrimitiveWithInfer): + """Computes gradient for prediction on SmoothL1Loss.""" + + @prim_attr_register + def __init__(self, sigma=1.0): + pass + + def infer_shape(self, prediction, target, dloss): + validator.check_param_equal('prediction', prediction, 'target', target) + validator.check_param_equal('prediction', prediction, 'dloss', dloss) + return prediction + + def infer_dtype(self, prediction, target, dloss): + args = {"prediction": prediction, "target": target, 'dloss': dloss} + validator.check_type_same(args, mstype.number_type) + return dloss + + +class StridedSliceGrad(PrimitiveWithInfer): + """ + Performs grad of StridedSlice operation. + + Args: + begin_mask (int): Start indexing the slice. Default: 0. + end_mask (int): End indexing the slice. Default: 0. + ellipsis_mask (int): An int32 mask. Default: 0. + new_axis_mask (int): An int32 mask. Default: 0. + shrink_axis_mask (int): An int32 mask. Default: 0. + + Returns: + Tensor, has the same shape of input. + """ + + @prim_attr_register + def __init__(self, + begin_mask=0, + end_mask=0, + ellipsis_mask=0, + new_axis_mask=0, + shrink_axis_mask=0): + """init StrideSliceGrad""" + validator.check_type('begin_mask', begin_mask, [int]) + validator.check_type('end_mask', end_mask, [int]) + validator.check_type('ellipsis_mask', ellipsis_mask, [int]) + validator.check_type('new_axis_mask', new_axis_mask, [int]) + validator.check_type('shrink_axis_mask', shrink_axis_mask, [int]) + self.init_prim_io_names(inputs=['dy', 'shapex', 'begin', 'end', 'strides'], outputs=['output']) + + def __infer__(self, dy, shapex, begin, end, strides): + return {'shape': shapex['value'], + 'dtype': dy['dtype'], + 'value': None} + + +class TanhGrad(PrimitiveWithInfer): + """Computes gradient of hyperbolic tangent of input element-wise.""" + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, out, dout): + return out + + def infer_dtype(self, out, dout): + validator.check_subclass("out", out, mstype.tensor) + validator.check_subclass("dout", dout, mstype.tensor) + args = {"out type": out, "dout type": dout} + validator.check_type_same(args, mstype.number_type) + return out + + +class RefToEmbed(Primitive): + r""" + Make a key from Ref. + + The Key is a symbolic_key, is a embedding on Parameter, which is used as a key of the variable in env_type, + and get items by operation `env_get_item` with the symbolic_key instance. The `Parameter` is a ref. + + Inputs: + - **input** (Ref) - Target ref, ref is short for reference. The value of a Parameter is a ref. + + Outputs: + symbolic_key, made from the Ref. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.weight = mindspore.Parameter(1.0, name='weight') + >>> + >>> def construct(self): + >>> key = RefToEmbed()(self.weight) + >>> return key, self.weight + """ + __mindspore_signature__ = ( + ('variable', sig_rw.RW_REF, sig_kind.KIND_POSITIONAL_KEYWORD), + ) + @prim_attr_register + def __init__(self): + pass diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py new file mode 100644 index 00000000000..d0d3d5006c3 --- /dev/null +++ b/mindspore/ops/operations/array_ops.py @@ -0,0 +1,1963 @@ +# coding: utf-8 + +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +# limitations under the License. +# ============================================================================ + +"""Operators for array.""" + +import copy +import functools +import numbers +import numpy as np + +from ..._checkparam import ParamValidator as validator +from ..._checkparam import Rel +from ...common import dtype as mstype +from ...common.tensor import Tensor +from ..operations.math_ops import _check_infer_attr_reduce, _infer_shape_reduce +from ..primitive import Primitive, PrimitiveWithInfer, prim_attr_register + + +class ExpandDims(PrimitiveWithInfer): + """ + Adds an additional dimension at the given axis. + + Note: + If the specified axis is a negative number, the index is counted + backward from the end and starts at 1. + + Raises: + ValueError: If axis is not an integer or not in the valid range. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + - **axis** (int) - Specifies the dimension index at which to expand + the shape of `input_x`. The value of axis must be in the range + `[-input_x.dim()-1, input_x.dim()]`. Only constant value is allowed. + + Outputs: + Tensor, the shape of tensor is :math:`(1, x_1, x_2, ..., x_R)` if the + value of `axis` is 0. + + Examples: + >>> input_tensor = Tensor(np.array([[2, 2], [2, 2]]), mindspore.float32) + >>> expand_dims = ExpandDims() + >>> output = expand_dims(input_tensor, 0) + """ + + @prim_attr_register + def __init__(self): + """init ExpandDims""" + self.__setattr_flag__ = True + self.init_prim_io_names(inputs=['x', 'axis'], outputs=['output']) + + def __infer__(self, x, axis): + validator.check_subclass("input_x", x['dtype'], mstype.tensor) + x_shape = list(x['shape']) + axis_v = axis['value'] + rank = len(x_shape) + validator.check_const_input('axis', axis_v) + validator.check_type("axis", axis_v, [int]) + validator.check_int_range('axis', axis_v, -rank - 1, rank, Rel.INC_BOTH) + if axis_v < 0: + axis_v = rank + 1 + axis_v + x_shape.insert(axis_v, 1) + out = {'shape': x_shape, + 'dtype': x['dtype'], + 'value': None} + return out + + +class DType(PrimitiveWithInfer): + """ + Returns the data type of input tensor as mindspore.dtype. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + mindspore.dtype, the data type of a tensor. + + Examples: + >>> input_tensor = Tensor(np.array([[2, 2], [2, 2]]), mindspore.float32) + >>> type = DType()(input_tensor) + """ + + @prim_attr_register + def __init__(self): + """init DType""" + + def __infer__(self, x): + validator.check_subclass("input_x", x['dtype'], mstype.tensor) + out = {'shape': (), + 'dtype': mstype.type_type, + 'value': x['dtype'].element_type()} + return out + + +class SameTypeShape(PrimitiveWithInfer): + """ + Checks whether data type and shape of two tensors are the same. + + Raises: + ValueError: If not the same. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + - **input_y** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_S)`. + + Outputs: + Tensor, the shape of tensor is :math:`(x_1, x_2, ..., x_R)`, + if data type and shape of `input_x` and `input_y` are the same. + + Examples: + >>> input_x = Tensor(np.array([[2, 2], [2, 2]]), mindspore.float32) + >>> input_y = Tensor(np.array([[2, 2], [2, 2]]), mindspore.float32) + >>> out = SameTypeShape()(input_x, input_y) + """ + + @prim_attr_register + def __init__(self): + """init Same""" + + def __call__(self, x, y): + """run in PyNative mode""" + if x.dtype() != y.dtype(): + raise TypeError(f"The {x} and {y} should be same dtype.") + if x.shape() != y.shape(): + raise TypeError(f"The {x} and {y} should have same shape.") + return x + + def __infer__(self, x, y): + if x['dtype'] != y['dtype']: + raise TypeError(f"The {x} and {y} should be same dtype," + f" but got {x['dtype']} {y['dtype']}.") + if x['shape'] != y['shape']: + raise ValueError(f"The {x} and {y} should be same shape," + f" but got {x['shape']} {y['shape']}.") + return x + + +class Cast(PrimitiveWithInfer): + """ + Returns a tensor with the new specified data type. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + The tensor to be casted. + - **type** (dtype.Number) - The valid data type of the output tensor. Only constant value is allowed. + + Outputs: + Tensor, the shape of tensor is :math:`(x_1, x_2, ..., x_R)`, same as `input_x`. + + Examples: + >>> input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + >>> input_x = Tensor(input_np) + >>> type_dst = mindspore.int32 + >>> cast = Cast() + >>> result = cast(input_x, type_dst) + >>> expect = input_np.astype(type_dst) + """ + + @prim_attr_register + def __init__(self): + # if primitive need setattr in __infer__ need add this flag + """init Cast""" + self.init_prim_io_names(inputs=['x', 'dst_type'], outputs=['output']) + self.__setattr_flag__ = True + + def __infer__(self, x, t): + src_type = x['dtype'] + dst_type = t['value'] + + validator.check_subclass("input_x", src_type, [mstype.tensor, mstype.number]) + validator.check_subclass("type", dst_type, mstype.number, with_type_of=False) + + if isinstance(src_type, type(mstype.tensor)): + src_type = x['dtype'].element_type() + if isinstance(dst_type, type(mstype.tensor)): + dst_type = dst_type.element_type() + self.add_prim_attr('DstT', dst_type) + self.add_prim_attr('SrcT', src_type) + + value = None + if x['value'] is not None: + np_dst_type = mstype.dtype_to_nptype(dst_type) + if isinstance(x['value'], (int, float)): + value = Tensor(np.array(x['value']).astype(np_dst_type)) + else: + value = Tensor(x['value'].asnumpy().astype(np_dst_type)) + + out = {'shape': x['shape'], + 'dtype': mstype.tensor_type(t['value']), + 'value': value} + return out + + +class IsSubClass(PrimitiveWithInfer): + """ + Check whether one type is sub class of another type. + + Inputs: + - **sub_type** (mindspore.dtype) - The type to be check. Only constant value is allowed. + - **type_** (mindspore.dtype) - The target type. Only constant value is allowed. + + Outputs: + bool, the check result. + + Examples: + >>> result = IsSubClass()(mindspore.int32, mindspore.intc) + """ + + @prim_attr_register + def __init__(self): + pass + + def __infer__(self, sub_type, type_): + sub_type_t = sub_type['value'] + type_v = type_['value'] + + validator.check_type("sub_type", sub_type_t, [mstype.Type]) + validator.check_type("type_", type_v, [mstype.Type]) + + value = mstype.issubclass_(sub_type_t, type_v) + + out = {'shape': (), + 'dtype': mstype.type_type, + 'value': value} + return out + + +class IsInstance(PrimitiveWithInfer): + """ + Check whether an object is an instance of a target type. + + Inputs: + - **inst** (Any Object) - The instance to be check. Only constant value is allowed. + - **type_** (mindspore.dtype) - The target type. Only constant value is allowed. + + Outputs: + bool, the check result. + + Examples: + >>> a = 1 + >>> result = IsInstance()(a, mindspore.int32) + """ + + @prim_attr_register + def __init__(self): + pass + + def __infer__(self, inst, type_): + sub_type_t = inst['dtype'] + type_v = type_['value'] + + validator.check_const_input("inst", inst['value']) + validator.check_type("type_", type_v, [mstype.Type]) + + value = mstype.issubclass_(sub_type_t, type_v) + + out = {'shape': (), + 'dtype': mstype.type_type, + 'value': value} + return out + + +class Reshape(PrimitiveWithInfer): + """ + Reshapes input tensor with the same values based on a given shape tuple. + + Raises: + ValueError: Given a shape tuple, if it has more than one -1; or if the product + of its elements is less than or equal to 0 or cannot be divided by the product + of the input tensor shape; or if it does not match the input's array size. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + - **input_shape** (tuple[int]) - The input tuple is constructed by multiple + integers, i.e., :math:`(y_1, y_2, ..., y_S)`. Only constant value is allowed. + + Outputs: + Tensor, the shape of tensor is :math:`(y_1, y_2, ..., y_S)`. + + Examples: + >>> input_tensor = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]), mindspore.float32) + >>> reshape = Reshape() + >>> output = reshape(input_tensor, (3, 2)) + """ + + @prim_attr_register + def __init__(self): + """init Reshape""" + self.init_prim_io_names(inputs=['tensor', 'shape'], outputs=['output']) + self.__setattr_flag__ = True + + def __infer__(self, x, shape): + shape_v = shape['value'] + x_shp = x['shape'] + validator.check_subclass("x", x['dtype'], mstype.tensor) + validator.check_const_input("shape", shape_v) + validator.check_type("shape", shape_v, [tuple]) + shape_v = list(shape_v) + neg_index = -1 + dim_prod = 1 + for i, shp_i in enumerate(shape_v): + validator.check_type("shape[%d]" % i, shp_i, [int]) + if shp_i == -1: + if neg_index != -1: + raise ValueError(f'The shape can only has one -1 at most, but {shape_v}.') + neg_index = i + else: + dim_prod *= shp_i + arr_prod = np.prod(x_shp) + if dim_prod <= 0 or arr_prod % dim_prod != 0: + raise ValueError(f'The product of shape should > 0 and' + f' can be divided by prod of input {arr_prod},' + f' but shape {shape}, product of shape {dim_prod}.') + + if neg_index != -1: + shape_v[neg_index] = int(arr_prod / dim_prod) + dim_prod *= shape_v[neg_index] + if dim_prod != arr_prod: + raise ValueError(f'The shape arg for reshape must match array''s size' + f' input shape {arr_prod}, shape {dim_prod}.') + + value = None + if x['value'] is not None: + value = Tensor(x['value'].asnumpy().reshape(shape_v)) + + out = {'shape': tuple(shape_v), + 'dtype': x['dtype'], + 'value': value} + return out + + +class Shape(Primitive): + """ + Returns the shape of input tensor. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + tuple[int], the output tuple is constructed by multiple integers, + :math:`(x_1, x_2, ..., x_R)`. + + Examples: + >>> input_tensor = Tensor(np.ones(shape=[3, 2, 1]), mindspore.float32) + >>> shape = Shape() + >>> output = shape(input_tensor) + """ + + @prim_attr_register + def __init__(self): + """init Shape""" + + +class Squeeze(PrimitiveWithInfer): + """ + Returns a tensor with the same type but dimensions of 1 being removed based on axis. + + Note: + The dimension index starts at 0 and must be in the range `[-input.dim(), input.dim())`. + + Raises: + ValueError: If the corresponding dimension of the specified axis does not equal to 1. + + Args: + axis (int): Specifies the dimension indexes of shape to be removed, which will remove + all the dimensions that are equal to 1. If specified, it must be int32 or int64. + Default: (), an empty tuple. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, the shape of tensor is :math:`(x_1, x_2, ..., x_S)`. + + Examples: + >>> input_tensor = Tensor(np.ones(shape=[3, 2, 1]), mindspore.float32) + >>> squeeze = Squeeze(2) + >>> output = squeeze(input_tensor) + """ + + @prim_attr_register + def __init__(self, axis=()): + """init Squeeze""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + validator.check_type('axis', axis, [int, tuple]) + if isinstance(axis, tuple): + for item in axis: + validator.check_type("item", item, [int]) + else: + self.axis = (axis,) + self.add_prim_attr("axis", (axis,)) + + def infer_shape(self, x_shape): + axis = self.axis + x_shape = list(x_shape) + ndim = len(x_shape) + if not axis: + ret = [d for d in x_shape if d != 1] + else: + for a in axis: + validator.check_int_range('axis or its elements', a, -ndim, ndim - 1, Rel.INC_BOTH) + if x_shape[a] != 1: + raise ValueError('Cannot select an axis to squeeze out which has size not equal to one.') + ret = [x_shape[i] for i in range(ndim) if not (i in axis or (i - ndim) in axis)] + return ret + + def infer_dtype(self, x_dtype): + validator.check_subclass("x", x_dtype, mstype.tensor) + return x_dtype + + +class Transpose(PrimitiveWithInfer): + """ + Permutes the dimensions of input tensor according to input perm. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + - **input_perm** (tuple[int]) - The permutation to be converted. The input tuple is constructed by multiple + indexes. The length of `input_perm` and the shape of `input_x` should be the same. Only constant value is + allowed. + + Outputs: + Tensor, the type of output tensor is same as `input_x` and the shape of output tensor is decided by the + shape of `input_x` and the value of `input_perm`. + + Examples: + >>> input_tensor = Tensor(np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]), mindspore.float32) + >>> perm = (0, 2, 1) + >>> expect = np.array([[[1, 4], [2, 5], [3, 6]], [[7, 10], [8, 11], [9, 12]]]) + >>> transpose = Transpose() + >>> output = transpose(input_tensor, perm) + """ + + @prim_attr_register + def __init__(self): + """init Transpose""" + self.__setattr_flag__ = True + self.init_prim_io_names(inputs=['x', 'perm'], outputs=['output']) + + def __infer__(self, x, perm): + x_shape = x['shape'] + p_value = perm['value'] + x_type = x['dtype'] + if len(x_shape) != len(p_value): + raise ValueError('The dimension of x and perm must be equal.') + + validator.check_const_input("perm", p_value) + validator.check_type("p_value", p_value, [tuple]) + validator.check_subclass("x_type", x_type, mstype.tensor) + + tmp = list(p_value) + for i, dim in enumerate(p_value): + validator.check_integer("perm[%d]" % i, dim, 0, Rel.GE) + validator.check_integer("perm[%d]" % i, dim, len(p_value), Rel.LT) + tmp.remove(dim) + if dim in tmp: + raise ValueError('The value of perm is wrong.') + + out_shapes = [] + for i in p_value: + out_shapes.append(x_shape[i]) + out = {'shape': tuple(out_shapes), + 'dtype': x['dtype'], + 'value': None} + return out + + +class GatherV2(PrimitiveWithInfer): + """ + Returns a slice of input tensor based on the specified indices and axis. + + Inputs: + - **input_params** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + The original Tensor. + - **input_indices** (Tensor) - The shape of tensor is :math:`(y_1, y_2, ..., y_S)`. + Specifies the indices of elements of the original Tensor. Must be in the range + `[0, input_param.shape()[axis])`. + - **axis** (int) - Specifies the dimension index to gather indices. + + Outputs: + Tensor, the shape of tensor is :math:`(z_1, z_2, ..., z_N)`. + + Examples: + >>> params = Tensor(np.array([[1, 2, 7, 42], [3, 4, 54, 22], [2, 2, 55, 3]]), mindspore.float32) + >>> indices = Tensor(np.array([1, 2]), mindspore.int32) + >>> axis = 1 + >>> out = GatherV2()(params, indices, axis) + """ + + @prim_attr_register + def __init__(self): + """init index_select""" + self.__setattr_flag__ = True + self.init_prim_io_names(inputs=['params', 'indices', 'axis'], outputs=['output']) + + def __infer__(self, params, indices, axis): + validator.check_subclass("params", params['dtype'], mstype.tensor) + validator.check_subclass("indices", indices['dtype'], mstype.tensor) + validator.check_subclass("axis", axis['dtype'], mstype.int_) + validator.check_typename("element of indices", indices['dtype'], mstype.int_type) + validator.check_const_input("axis", axis['value']) + axis_v = axis['value'] + params_shp = params['shape'] + rank = len(params_shp) + validator.check_int_range("axis", axis_v, -rank, rank, Rel.INC_LEFT) + if axis_v < 0: + axis_v += rank + out_shape = params_shp[:axis_v] + indices['shape'] + params_shp[axis_v + 1:] + out = {'shape': out_shape, + 'dtype': params['dtype'], + 'value': None} + return out + + +class Split(PrimitiveWithInfer): + """ + Splits input tensor into output_num of tensors along the given axis and output numbers. + + Args: + axis (int): Index of the split position. Default: 0. + output_num (int): The number of output tensors. Default: 1. + + Raises: + ValueError: If axis is out of the range [-len(input_x.shape()), len(input_x.shape())), + or if the output_num is less than or equal to 0, or if the + dimension which to split cannot be evenly divided by output_num. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + tuple[Tensor], the shape of each output tensor is same, which is + :math:`(y_1, y_2, ..., y_S)`. + + Examples: + >>> split = Split(1, 2) + >>> x = Tensor(np.array([[1, 1, 1, 1], [2, 2, 2, 2]])) + >>> output = split(x) + """ + + @prim_attr_register + def __init__(self, axis=0, output_num=1): + """init Split""" + validator.check_type("axis", axis, [int]) + validator.check_type("output_num", output_num, [int]) + self.axis = axis + self.output_num = output_num + + def __infer__(self, x): + validator.check_subclass("x", x['dtype'], mstype.tensor) + x_shape = list(x['shape']) + dim = len(x_shape) + validator.check_int_range('axis value', self.axis, -dim, dim, Rel.INC_LEFT) + validator.check_integer("output_num", self.output_num, 0, Rel.GT) + output_valid_check = x_shape[self.axis] % self.output_num + validator.check_integer("the dimension which to split divides output_num", output_valid_check, 0, Rel.EQ) + x_shape[self.axis] = int(x_shape[self.axis] / self.output_num) + out_shapes = [] + out_dtypes = [] + for _ in range(self.output_num): + out_shapes.append(tuple(x_shape)) + out_dtypes.append(x['dtype']) + out_shapes = tuple(out_shapes) + out_dtypes = tuple(out_dtypes) + out = {'shape': out_shapes, + 'dtype': out_dtypes, + 'value': None} + return out + + +class Rank(PrimitiveWithInfer): + """ + Returns the rank of a tensor. + + Returns a 0-D int32 Tensor representing the rank of input; the rank of a tensor + is the number of indices required to uniquely select each element of the tensor. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor. 0-D int32 Tensor representing the rank of input, i.e., :math:`R`. + + Examples: + >>> input_tensor = Tensor(np.array([[2, 2], [2, 2]]), mindspore.float32) + >>> rank = Rank() + >>> rank(input_tensor) + """ + + @prim_attr_register + def __init__(self): + """init Rank""" + + def __infer__(self, x): + validator.check_subclass("x", x['dtype'], mstype.tensor) + out = {'shape': None, + 'dtype': None, + 'value': len(x['shape'])} + return out + + +class TruncatedNormal(PrimitiveWithInfer): + """ + Returns a tensor of the specified shape filled with truncated normal values. + + The generated values follow a normal distribution. + + Args: + seed (int): A int number used to create random seed. Default: 0. + dtype (:class:`mindspore.dtype`): Data type. Default: mindspore.float32. + + Inputs: + - **shape** (Tensor) - Shape of output tensor. The shape is a 1-D tensor, and type is int. + + Outputs: + Tensor, type of output tensor is same as attribute `dtype`. + + Examples: + >>> input_shape = Tensor(np.array([1, 2, 3])) + >>> truncated_normal = TruncatedNormal() + >>> output = truncated_normal(input_shape) + """ + + @prim_attr_register + def __init__(self, seed=0, dtype=mstype.float32): + """init TruncatedNormal""" + validator.check_type('seed', seed, [int]) + validator.check_typename('dtype', dtype, mstype.number_type) + + def __infer__(self, shape): + shape_t = shape['value'] + validator.check_subclass("shape", shape['dtype'], mstype.tensor) + shape_n = shape_t.asnumpy() + if shape_n.ndim != 1: + raise ValueError('The rank of input shape must be 1.') + if shape_n.dtype not in (np.int32, np.int64): + raise TypeError('The type of input shape must be int32 or int64.') + for i, item in enumerate(shape_n): + validator.check_integer(f"shape[{i}]", item.item(), 0, Rel.GT) + out = {'shape': tuple(shape_n), + 'dtype': mstype.tensor_type(self.dtype), + 'value': None} + return out + + +class Size(PrimitiveWithInfer): + r""" + Returns the elements count size of a tensor. + + Returns an int scalar representing the elements size of input, the total number of elements in the tensor. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + int, a scalar representing the elements size of `input_x`, tensor is the number of elements + in a tensor, :math:`size=x_1*x_2*...x_R`. + + Examples: + >>> input_tensor = Tensor(np.array([[2, 2], [2, 2]]), mindspore.float32) + >>> size = Size() + >>> output = size(input_tensor) + """ + + @prim_attr_register + def __init__(self): + """init Size""" + + def __infer__(self, x): + size = 1 + validator.check_subclass("x", x['dtype'], mstype.tensor) + shp = x['shape'] + if not shp: + size = 0 + else: + size = functools.reduce(lambda x, y: x * y, x['shape']) + out = {'shape': None, + 'dtype': mstype.int32, + 'value': size} + return out + + +class Fill(PrimitiveWithInfer): + """ + Creates a tensor filled with a scalar value. + + Creates a tensor with shape described by the first argument and fills it with values in the second argument. + + Inputs: + - **type** (mindspore.dtype) - The specified type of output tensor. Only constant value is allowed. + - **shape** (tuple) - The specified shape of output tensor. Only constant value is allowed. + - **value** (scalar) - Value to fill the returned tensor. Only constant value is allowed. + + Outputs: + Tensor, has the same type and shape as input value. + + Examples: + >>> fill = P.Fill() + >>> fill(P.DType()(x), (2, 2), 1) + """ + + @prim_attr_register + def __init__(self): + """init Fill""" + + def __infer__(self, dtype, dims, x): + validator.check_const_input("type", dtype['value']) + validator.check_const_input("shape", dims['value']) + validator.check_const_input("value", x['value']) + validator.check_type("shape", dims['value'], [tuple]) + validator.check_type("value", x['value'], [numbers.Number, bool]) + for item in dims['value']: + validator.check_type("item", item, [int]) + validator.check_integer("item", item, 0, Rel.GT) + x_dtype = dtype['value'] + valid_types = [mstype.bool_, mstype.int8, mstype.int32, mstype.int64, + mstype.uint8, mstype.uint32, mstype.uint64, + mstype.float16, mstype.float32, mstype.float64] + validator.check_typename("value", x_dtype, valid_types) + x_nptype = mstype.dtype_to_nptype(x_dtype) + ret = np.full(dims['value'], x['value'], x_nptype) + out = { + 'value': Tensor(ret), + 'shape': dims['value'], + 'dtype': x_nptype, + } + return out + + +class OnesLike(PrimitiveWithInfer): + """ + Creates a new tensor. All elements' value are 1. + + Returns a tensor of ones with the same shape and type as the input. + + Inputs: + - **input_x** (Tensor) - Input tensor. + + Outputs: + Tensor, has the same shape and type as `input_x` but filled with ones. + + Examples: + >>> oneslike = P.OnesLike() + >>> x = Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32)) + >>> output = oneslike(x) + """ + + @prim_attr_register + def __init__(self): + """Init OnesLike""" + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("x", x_dtype, mstype.tensor) + validator.check_typename('x_dtype', x_dtype, mstype.number_type + (mstype.bool_,)) + return x_dtype + + +class ZerosLike(PrimitiveWithInfer): + """ + Creates a new tensor. All elements value are 0. + + Returns a tensor of zeros with the same shape and type as the input tensor. + + Inputs: + - **input_x** (Tensor) - Input tensor. + + Outputs: + Tensor, has the same shape and type as `input_x` but filled with zeros. + + Examples: + >>> zeroslike = P.ZerosLike() + >>> x = Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32)) + >>> output = zeroslike(x) + """ + + @prim_attr_register + def __init__(self): + """Init ZerosLike""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("x", x_dtype, mstype.tensor) + validator.check_typename('x_dtype', x_dtype, mstype.number_type + (mstype.bool_,)) + return x_dtype + + +class TupleToArray(PrimitiveWithInfer): + """ + Converts a tuple to tensor. + + If the first number type of tuple is int, the output tensor type is int. Else, the output tensor type is float. + + Inputs: + - **input_x** (tuple) - A tuple of numbers. These numbers have the same type. Only constant value is allowed. + + Outputs: + Tensor, if the input tuple contain `N` numbers, then the output tensor shape is (N,). + + Examples: + >>> type = TupleToArray()((1,2,3)) + """ + + @prim_attr_register + def __init__(self): + """init TupleToArray""" + + def infer_value(self, x): + validator.check_const_input("x", x) + validator.check_type("x", x, [tuple]) + validator.check("size of x", len(x), '', 0, Rel.GT) + dtype = type(x[0]) + for i, item in enumerate(x): + validator.check_type(f"x[{i}]", item, [numbers.Number]) + if not all(isinstance(item, dtype) for item in x): + raise TypeError("All elements of input x must be have same type.") + if isinstance(x[0], int): + ret = np.array(x, np.int32) + else: + ret = np.array(x, np.float32) + + return Tensor(ret) + + +class ScalarToArray(PrimitiveWithInfer): + """ + Converts scalar to `Tensor`. + + Inputs: + - **input_x** (Union[int, float]) - The input is a scalar. Only constant value is allowed. + + Outputs: + Tensor. 0-D Tensor and the content is the input. + + Examples: + >>> op = ScalarToArray() + >>> data = 1.0 + >>> output = op(data) + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_value(self, x): + validator.check_const_input("x", x) + validator.check_type("x", x, [int, float]) + if isinstance(x, int): + ret = np.array(x, np.int32) + else: + ret = np.array(x, np.float32) + return Tensor(ret) + + +class ScalarToTensor(PrimitiveWithInfer): + """ + Converts scalar to `Tensor`, and convert data type to specified type. + + Inputs: + - **input_x** (Union[int, float]) - The input is a scalar. Only constant value is allowed. + - **dtype** (mindspore.dtype) - The target data type. Default: mindspore.float32. Only + constant value is allowed. + + Outputs: + Tensor. 0-D Tensor and the content is the input. + + Examples: + >>> op = ScalarToTensor() + >>> data = 1 + >>> output = op(data, mindspore.float32) + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_value(self, x, dtype=mstype.float32): + validator.check_const_input("x", x) + validator.check_type("x", x, [int, float]) + validator.check_subclass("dtype", dtype, mstype.number, with_type_of=False) + data_type = mstype.dtype_to_nptype(dtype) + return Tensor(np.array(x, data_type)) + + +class InvertPermutation(PrimitiveWithInfer): + r""" + Computes the inverse of an index permutation. + + Given a tuple input, this operation inserts a dimension of 1 at the dimension + This operation calculates the inverse of the index replacement. It requires a + 1-dimensional tuple x, which represents the array starting at zero, + and swaps each value with its index position. In other words, for the output + tuple y and the input tuple x, this operation calculates the following: + :math:`y[x[i]] = i, \quad i \in [0, 1, \ldots, \text{len}(x)-1]`. + + Note: + These values must include 0. There must be no duplicate values and the + values can not be negative. + + Inputs: + - **input_x** (tuple[int]) - The input tuple is constructed by multiple + integers, i.e., :math:`(y_1, y_2, ..., y_S)` representing the indices. + The values must include 0. There can be no duplicate values or negative values. + + Outputs: + tuple[int]. the lenth is same as input. + + Examples: + >>> invert = InvertPermutation() + >>> input_data = (3, 4, 0, 2, 1) + >>> output = invert(input_data) + >>> output == (2, 4, 3, 0, 1) + """ + + @prim_attr_register + def __init__(self): + """init InvertPermutation""" + + def __infer__(self, x): + x_shp = x['shape'] + x_value = x['value'] + validator.check_const_input("shape", x_shp) + validator.check_type("shape", x_shp, [tuple]) + z = [x_value[i] for i in range(len(x_value))] + z.sort() + + y = [None]*len(x_value) + for i, value in enumerate(x_value): + validator.check_type("input[%d]" % i, value, [int]) + validator.check(f'value', z[i], f'index', i) + y[value] = i + z.append(value) + return {'shape': x_shp, + 'dtype': x['dtype'], + 'value': tuple(y)} + + +class Argmax(PrimitiveWithInfer): + """ + Returns the indices of the max value of a tensor across the axis. + + If the shape of input tensor is :math:`(x_1, ..., x_N)`, the output tensor shape is + :math:`(x_1, ..., x_{axis-1}, x_{axis+1}, ..., x_N)`. + + Args: + axis (int): Axis on which Argmax operation applies. Default: -1. + output_type (:class:`mindspore.dtype`): An optional data type of `mindspore.dtype.int32` and + `mindspore.dtype.int64`. Default: `mindspore.dtype.int64`. + + Inputs: + - **input_x** (Tensor) - Input tensor. + + Outputs: + Tensor, indices of the max value of input tensor across the axis. + + Examples: + >>> input = Tensor(np.array([2.0, 3.1, 1.2])) + >>> index = Argmax()(input) + >>> assert index == Tensor(1, mindspore.int64) + """ + + @prim_attr_register + def __init__(self, axis=-1, output_type=mstype.int64): + """init Argmax""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + validator.check_type("axis", axis, [int]) + validator.check_typename('output_type', output_type, [mstype.int32, mstype.int64]) + self.axis = axis + self.add_prim_attr('output_type', output_type) + + def infer_shape(self, x_shape): + axis = self.axis + if axis is None: + axis = 0 + x_rank = len(x_shape) + validator.check_int_range("axis", axis, -x_rank, x_rank, Rel.INC_LEFT) + axis = axis + x_rank if axis < 0 else axis + ouput_shape = [x_shape[i] for i in range(x_rank) if i != axis] + return ouput_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("input_x", x_dtype, mstype.tensor) + return mstype.tensor_type(self.output_type) + + +class Argmin(PrimitiveWithInfer): + """ + Returns the indices of the min value of a tensor across the axis. + + If the shape of input tensor is :math:`(x_1, ..., x_N)`, the output tensor shape is + :math:`(x_1, ..., x_{axis-1}, x_{axis+1}, ..., x_N)`. + + Args: + axis (int): Axis on which Argmin operation applies. Default: -1. + output_type (:class:`mindspore.dtype`): An optional data type from: `mindspore.dtype.int32`, + `mindspore.dtype.int64`. Default: `mindspore.dtype.int64`. + + Inputs: + - **input_x** (Tensor) - Input tensor. + + Outputs: + Tensor, indices of the min value of input tensor across the axis. + + Examples: + >>> input = Tensor(np.array([2.0, 3.1, 1.2])) + >>> index = Argmin()(input) + >>> assert index == Tensor(2, mindspore.int64) + """ + + @prim_attr_register + def __init__(self, axis=-1, output_type=mstype.int64): + """init Argmin""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + validator.check_type("axis", axis, [int]) + self.axis = axis + self.add_prim_attr('output_type', output_type) + + def infer_shape(self, x_shape): + axis = self.axis + if axis is None: + axis = 0 + x_rank = len(x_shape) + validator.check_int_range("axis", axis, -x_rank, x_rank, Rel.INC_LEFT) + axis = axis + x_rank if axis < 0 else axis + ouput_shape = [x_shape[i] for i in range(x_rank) if i != axis] + return ouput_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("input_x", x_dtype, mstype.tensor) + return mstype.tensor_type(self.output_type) + + +class ArgMaxWithValue(PrimitiveWithInfer): + """ + Calculates maximum value with corresponding index. + + Calculates maximum value along with given axis for the input tensor. Returns the maximum values and indices. + + Note: + In auto_parallel and semi_auto_parallel mode, the first output index can not be used. + + Args: + axis (int): The dimension to reduce. Default: 0. + keep_dims (bool): Whether to reduce dimension, if true the output will keep same dimension with the input, + the output will reduce dimension if false. Default: False. + + Inputs: + - **input_x** (Tensor) - The input tensor, can be any dimension. Set the shape of input tensor as + :math:`(x_1, x_2, ..., x_N)`. + + Outputs: + Tensor, corresponding index and maximum value of input tensor. If `keep_dims` is true, the output tensors shape + is :math:`(x_1, x_2, ..., x_{axis-1}, 1, x_{axis+1}, ..., x_N)`. Else, the shape is + :math:`(x_1, x_2, ..., x_{axis-1}, x_{axis+1}, ..., x_N)`. + + Examples: + >>> input = Tensor(np.random.rand(5)) + >>> index, output = ArgMaxWithValue()(input) + """ + + @prim_attr_register + def __init__(self, axis=0, keep_dims=False): + """init ArgMaxWithValue""" + self.axis = axis + self.keep_dims = keep_dims + _check_infer_attr_reduce(axis, keep_dims) + + def infer_shape(self, x_shape): + axis = self.axis + x_rank = len(x_shape) + validator.check_int_range("axis", axis, -x_rank, x_rank, Rel.INC_LEFT) + ouput_shape = _infer_shape_reduce(x_shape, self.axis, self.keep_dims) + return ouput_shape, ouput_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("input_x", x_dtype, mstype.tensor) + return mstype.tensor_type(mstype.int32), x_dtype + + +class ArgMinWithValue(PrimitiveWithInfer): + """ + Calculates minimum value with corresponding index, return indices and values. + + Calculates minimum value along with given axis for the input tensor. Returns the minimum values and indices. + + Note: + In auto_parallel and semi_auto_parallel mode, the first output index can not be used. + + Args: + axis (int): The dimension to reduce. Default: 0. + keep_dims (bool): Whether to reduce dimension, if true the output will keep same dimension as the input, + the output will reduce dimension if false. Default: False. + + Inputs: + - **input_x** (Tensor) - The input tensor, can be any dimension. Set the shape of input tensor as + :math:`(x_1, x_2, ..., x_N)`. + + Outputs: + Tensor, corresponding index and minimum value of input tensor. If `keep_dims` is true, the output tensors shape + is :math:`(x_1, x_2, ..., x_{axis-1}, 1, x_{axis+1}, ..., x_N)`. Else, the shape is + :math:`(x_1, x_2, ..., x_{axis-1}, x_{axis+1}, ..., x_N)`. + + Examples: + >>> input = Tensor(np.random.rand(5)) + >>> index, output = ArgMinWithValue()(input) + """ + @prim_attr_register + def __init__(self, axis=0, keep_dims=False): + """init ArgMinWithValue""" + self.axis = axis + self.keep_dims = keep_dims + _check_infer_attr_reduce(axis, keep_dims) + + def infer_shape(self, x_shape): + axis = self.axis + x_rank = len(x_shape) + validator.check_int_range("axis", axis, -x_rank, x_rank, Rel.INC_LEFT) + ouput_shape = _infer_shape_reduce(x_shape, self.axis, self.keep_dims) + return ouput_shape, ouput_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("input_x", x_dtype, mstype.tensor) + return mstype.tensor_type(mstype.int32), x_dtype + + +class Tile(PrimitiveWithInfer): + r""" + Replicates a tensor with given multiples times. + + Creates a new tensor by replicating input multiples times. The dimension of + output tensor is the larger of the dimension length of input and the length of multiples. + + Inputs: + - **input_x** (Tensor) - 1-D or higher Tensor. Set the shape of input tensor as + :math:`(x_1, x_2, ..., x_S)`. + + - **multiples** (tuple[int]) - The input tuple is constructed by multiple + integers, i.e., :math:`(y_1, y_2, ..., y_S)`. The length of `multiples` + can't be smaller than the length of shape in `input_x`. + + Outputs: + Tensor, has the same type as the `input_x`. + + - If the length of `multiples` is the same as the length of shape in `input_x`, + then the shape of their corresponding positions can be multiplied, and + the shape of Outputs is :math:`(x_1*y_1, x_2*y_2, ..., x_S*y_R)`. + - If the length of `multiples` is larger than the length of shape in `input_x`, + fill in multiple 1 in front of the shape in `input_x` until their lengths are consistent. + Such as set the shape of `input_x` as :math:`(1, ..., x_1, x_2, ..., x_S)`, + then the shape of their corresponding positions can be multiplied, and + the shape of Outputs is :math:`(1*y_1, ..., x_S*y_R)`. + """ + + @prim_attr_register + def __init__(self): + """init Tile""" + self.init_prim_io_names(inputs=['x', 'multiples'], outputs=['output']) + + def __infer__(self, x, multiples): + multiples_v = multiples['value'] + x_shp = x['shape'] + validator.check_const_input("shape", multiples_v) + validator.check_type("shape", multiples_v, [tuple]) + for i, multiple in enumerate(multiples_v): + validator.check_type("multiples[%d]" % i, multiple, [int]) + validator.check_typename('x', x['dtype'], + [mstype.int16, mstype.int32, mstype.bool_, + mstype.float16, mstype.float32]) + len_sub = len(multiples_v) - len(x_shp) + multiples_w = None + if len_sub == 0: + multiples_w = multiples_v + if len_sub > 0: + for i in range(0, len_sub): + x_shp.insert(0, 1) + multiples_w = multiples_v + elif len_sub < 0: + raise ValueError("The length of multiples can not be smaller than the length of dimension in input_x.") + for i, a in enumerate(multiples_w): + x_shp[i] *= a + value = None + if x['value'] is not None: + value = Tensor(np.tile(x['value'].asnumpy(), multiples_w)) + return {'shape': x_shp, + 'dtype': x['dtype'], + 'value': value} + + +class UnsortedSegmentSum(PrimitiveWithInfer): + r""" + Computes the sum along segments of a tensor. + + Calculates a tensor such that :math:`\text{output}[i] = \sum_{segment\_ids[j] == i} \text{data}[j, \ldots]`, where + :math:`j` is a tuple describing the index of element in data. `segment_ids` selects which elements in data to sum + up. Segment_ids does not need to be sorted, and it does not need to cover all values in the entire valid value + range. + + If the sum of the given segment_ids :math:`i` is empty, then :math:`\text{output}[i] = 0`. If the given segment_ids + is negative, the value will be ignored. 'num_segments' should be equal to the number of different segment_ids. + + Inputs: + - **input_x** (Tensor) - The shape is :math:`(x_1, x_2, ..., x_R)`. + - **segment_ids** (Tensor) - Set the shape as :math:`(x_1, x_2, ..., x_N)`, where 0 < N <= R. Type must be int. + - **num_segments** (int) - Set :math:`z` as num_segments. + + Outputs: + Tensor, the shape is :math:`(z, x_{N+1}, ..., x_R)`. + + Examples: + >>> input_x = [1, 2, 3, 4] + >>> segment_ids = [0, 0, 1, 2] + >>> num_segments = 4 + >>> type = P.UnsortedSegmentSum()(input_x, segment_ids, num_segments) + """ + + @prim_attr_register + def __init__(self): + """init UnsortedSegmentSum""" + self.init_prim_io_names(inputs=['x', 'segment_ids', 'num_segments'], outputs=['y']) + + def __infer__(self, x, segment_ids, num_segments): + x_type = x['dtype'] + x_shp = x['shape'] + validator.check_subclass("input_x", x_type, mstype.tensor) + validator.check_type("x_shape", x_shp, [list]) + x_shp_len = len(x_shp) + validator.check_integer("rank of input_x", x_shp_len, 0, Rel.GT) + segment_ids_shp = segment_ids['shape'] + segment_ids_type = segment_ids['dtype'] + validator.check_subclass("segment_ids", segment_ids_type, mstype.tensor) + validator.check_type("segment_ids", segment_ids_shp, [list]) + segment_ids_shp_len = len(segment_ids_shp) + validator.check_integer("rank of segment_ids", segment_ids_shp_len, 0, Rel.GT) + validator.check(f'rank of input_x', len(x_shp), + 'rank of segments_id', len(segment_ids_shp), Rel.GE) + for i, value in enumerate(segment_ids_shp): + validator.check("ids[%d]" % i, value, 'input[%d]' % i, x_shp[i]) + num_segments_v = num_segments['value'] + validator.check_type('num_segments', num_segments_v, [int]) + validator.check_integer("num_segments", num_segments_v, 0, Rel.GT) + shp = [num_segments_v] + shp += x_shp[segment_ids_shp_len:] + out = {'shape': shp, + 'dtype': mstype.tensor_type(x_type.element_type()), + 'value': None} + return out + + +def _get_concat_offset(x_shp, x_type, axis): + """for concat and concatoffset check args and compute offset""" + validator.check_type("shape", x_shp, [tuple]) + validator.check_integer("len of input_x shape", len(x_shp), 0, Rel.GT) + validator.check_subclass("shape0", x_type[0], mstype.tensor) + validator.check_integer("len of input_x0 shape", len(x_shp[0]), 0, Rel.GT) + rank_base = len(x_shp[0]) + validator.check_int_range('axis', axis, -rank_base - 1, rank_base, Rel.INC_BOTH) + if axis < 0: + axis = axis + rank_base + all_shp = x_shp[0][axis] + offset = [0,] + for i in range(1, len(x_shp)): + v = x_shp[i] + validator.check('len of x_shp[%d]' % i, len(v), 'len of base', len(x_shp[0])) + validator.check('x_type[%d]' % i, x_type[i], 'base', x_type[0]) + for j in range(rank_base): + if j != axis and v[j] != x_shp[0][j]: + raise ValueError("Concat evaluator element %d shape in input can not concat with first element" % i) + offset.append(all_shp) + all_shp += v[axis] + return offset, all_shp, axis + + +class Concat(PrimitiveWithInfer): + r""" + Concat tensor in specified axis. + + Concat input tensors along with the given axis. + + Note: + The input data is a tuple of tensors. These tensors have the same rank `R`. Set the given axis as `m`, and + :math:`0 \le m < N`. Set the number of input tensors as `N`. For the :math:`i`-th tensor :math:`t_i` has + the shape :math:`(x_1, x_2, ..., x_{mi}, ..., x_R)`. :math:`x_{mi}` is the :math:`m`-th dimension of the + :math:`i`-th tensor. Then, the output tensor shape is + + .. math:: + (x_1, x_2, ..., \sum_{i=1}^Nx_{mi}, ..., x_R) + + Args: + axis (int): The specified axis. Default: 0. + + Inputs: + - **input_x** (tuple, list) - Tuple or list of input tensors. + + Outputs: + Tensor, the shape is :math:`(x_1, x_2, ..., \sum_{i=1}^Nx_{mi}, ..., x_R)`. + + Examples: + >>> data1 = Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32)) + >>> data2 = Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32)) + >>> op = Concat() + >>> output = op((data1, data2)) + """ + + @prim_attr_register + def __init__(self, axis=0): + """init Tile""" + self.__setattr_flag__ = True + validator.check_type("axis", axis, [int]) + + def __infer__(self, input_x): + axis = self.axis + x_shp = input_x['shape'] + x_type = input_x['dtype'] + _, all_shp, _ = _get_concat_offset(x_shp, x_type, axis) + self.add_prim_attr('T', x_type[0].element_type()) + self.add_prim_attr('inputNums', len(x_shp)) + ret_shp = x_shp[0].copy() + ret_shp[axis] = all_shp + out = {'shape': ret_shp, + 'dtype': x_type[0], + 'value': None} + return out + + +class Slice(PrimitiveWithInfer): + """ + Slice a tensor in specified shape. + + Args: + x (Tensor): The target tensor. + begin (tuple): The beginning of the slice. Only constant value is allowed. + size (tuple): The size of the slice. Only constant value is allowed. + + Returns: + Tensor. + + Examples: + >>> data = Tensor(np.array([3,2,3]).astype(np.int32)) + >>> type = P.Slice()(data, (1, 0, 0), (1, 1, 3)) + """ + + @prim_attr_register + def __init__(self): + """init slice""" + self.init_prim_io_names(inputs=['x', 'begin', 'size'], outputs=['output']) + + def __infer__(self, x, begin, size): + x_shape = x['shape'] + x_shp_len = len(x_shape) + validator.check_const_input('begin', begin['value']) + validator.check_const_input('size', size['value']) + begin_v, size_v = begin['value'], size['value'] + if begin_v is None or size_v is None: + return {'shape': None, + 'dtype': x['dtype'], + 'value': None} + for key, value in zip(('begin', 'size'), (begin_v, size_v)): + validator.check(f'len of {key}', len(value), + 'len x\'s dim', x_shp_len) + for i in range(x_shp_len): + if x_shape[i] < begin_v[i] + size_v[i]: + y = begin_v[i] + size_v[i] + raise ValueError("Slice shape can not bigger than orign shape %d, %d." % (x_shape[i], y)) + return {'shape': size_v, + 'dtype': x['dtype'], + 'value': None} + + +class ConcatOffset(PrimitiveWithInfer): + """primitive for computing Concat's gradient.""" + + @prim_attr_register + def __init__(self, N=2, axis=0): + """init ConcatOffset""" + + def __infer__(self, input_x): + axis = self.axis + x_shp = input_x['shape'] + x_type = input_x['dtype'] + offset, _, axis = _get_concat_offset(x_shp, x_type, axis) + self.add_prim_attr('T', x_type[0].element_type()) + offset_values = [] + for i in range(len(x_shp)): + values = [] + for j in range(len(x_shp[0])): + value = 0 + if j == axis: + value = offset[i] + values.append(value) + offset_values.append(tuple(values)) + out = {'shape': None, + 'dtype': None, + 'value': tuple(offset_values)} + return out + + +class Select(PrimitiveWithInfer): + r""" + + Return the selected elements, either from input :math:`x` or input :math:`y`, depending on the `condition`. + + Given a tensor as input, this operation inserts a dimension of 1 at the dimension, + if both :math:`x` and :math:`y` are none, the operation returns the coordinates of the true + element in the condition, the coordinates are returned as a two-dimensional + tensor, where the first dimension (row) represents the number of true elements + and the second dimension (columns) represents the coordinates of the true + elements. Keep in mind that the shape of the output tensor can vary depending + on how much of the true value is in the input. Indexes are output in row-first + order. + + If neither is None, :math:`x` and :math:`y` must have the same shape. If :math:`x` and :math:`y` are + scalars, the conditional tensor must be a scalar. If :math:`x` and :math:`y` are + higher-demensional vectors, the condition must be a vector whose size matches the + first dimension of :math:`x`, or must have the same shape as :math:`y`. + + The conditional tensor acts as an optional compensation (mask), which + determines whether the corresponding element / row in the output should be + selected from :math:`x` (if true) or :math:`y` (if false) based on the value of each + element. + + If condition is a vector, then :math:`x` and :math:`y` are higher-demensional matrices, then it + chooses to copy that row (external dimensions) from :math:`x` and :math:`y`. If condition has + the same shape as :math:`x` and :math:`y`, you can choose to copy these elements from :math:`x` + and :math:`y`. + + Inputs: + - **input_x** (Tensor[bool]) - The shape is :math:`(x_1, x_2, ..., x_N)`. + The condition tensor, decides whose element is chosen. + - **input_y** (Tensor) - The shape is :math:`(x_1, x_2, ..., x_N, ..., x_R)`. + The first input tensor. + - **input_z** (Tensor) - The shape is :math:`(x_1, x_2, ..., x_N, ..., x_R)`. + The second input tensor. + + Outputs: + Tensor, has the same shape as input_y. The shape is :math:`(x_1, x_2, ..., x_N, ..., x_R)`. + + Examples: + >>> select = Select() + >>> select([True, False],[2,3],[1,2]) + """ + + @prim_attr_register + def __init__(self): + """init""" + self.__setattr_flag__ = True + + def infer_shape(self, cond_shape, x_shape, y_shape): + if cond_shape != x_shape or x_shape != y_shape: + raise ValueError('The x_shape and y_shape must be the same as cond_shape.') + return x_shape + + def infer_dtype(self, cond_type, x_type, y_type): + self.add_prim_attr('T', x_type) + validator.check_subclass("x_type", x_type, mstype.tensor) + validator.check_subclass("y_type", y_type, mstype.tensor) + validator.check_typename("cond_type", cond_type, [mstype.bool_]) + if x_type != y_type: + raise TypeError('The x_type %s must be the same as y_type %s.' % (x_type, y_type)) + return x_type + + +class StridedSlice(PrimitiveWithInfer): + r""" + + Extracts a strided slice of a tensor. + + Given an input tensor, this operation inserts a dimension of length 1 at the dimension. + This operation extracts a fragment of size (end-begin)/stride from the given + 'input_tensor'. Starting from the position specified by the begin, the fragment + continues adding stride to the index until all dimensions are not less than end. + + Note: + The stride may be negative value, which causes reverse slicing. + The shape of `begin`, `end` and `strides` should be the same. + + Args: + begin_mask (int): Starting index of the slice. Default: 0. + end_mask (int): Ending index of the slice. Default: 0. + ellipsis_mask (int): An int mask. Default: 0. + new_axis_mask (int): An int mask. Default: 0. + shrink_axis_mask (int): An int mask. Default: 0. + + Inputs: + - **input_x** (Tensor) - The input Tensor. + - **begin** (tuple[int]) - A tuple which represents the location where to start. Only + constant value is allowed. + - **end** (tuple[int]) - A tuple or which represents the maximum location where to stop. + Only constant value is allowed. + - **strides** (tuple[int]) - A tuple which represents the stride continuously added + before reach the maximum location. Only constant value is allowed. + + Outputs: + Tensor. + Explain with the following example. + - In the 0th dim, begin is 1, end is 2, and strides is 1, + because :math:`1+1=2\geq2`, the interval is :math:`[1,2)`. + Thus, return the element with :math:`index = 1` in 0th dim, i.e., [[3, 3, 3], [4, 4, 4]]. + - In the 1st dim, similarly, the interval is :math:`[0,1)`. + Based on the return value of the 0th dim, return the element with :math:`index = 0`, + i.e., [3, 3, 3]. + - In the 2nd dim, similarly, the interval is :math:`[0,3)`. + Based on the return value of the 1st dim, return the element with :math:`index = 0,1,2`, + i.e., [3, 3, 3]. + - Finally, the output is [3, 3, 3]. + + Examples + >>> input_x = Tensor([[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]], [[5, 5, 5], [6, 6, 6]]]) + >>> slice = StridedSlice() + >>> output = slice(input_x, (1, 0, 0), (2, 1, 3), (1, 1, 1)) + >>> output.shape() + (1, 1, 3) + >>> output + [[[3, 3, 3]]] + """ + + @prim_attr_register + def __init__(self, + begin_mask=0, + end_mask=0, + ellipsis_mask=0, + new_axis_mask=0, + shrink_axis_mask=0): + """init StrideSlice""" + self.init_prim_io_names(inputs=['x', 'begin', 'end', 'strides'], outputs=['output']) + validator.check_type('begin_mask', begin_mask, [int]) + validator.check_type('end_mask', end_mask, [int]) + validator.check_type('ellipsis_mask', ellipsis_mask, [int]) + validator.check_type('new_axis_mask', new_axis_mask, [int]) + validator.check_type('shrink_axis_mask', shrink_axis_mask, [int]) + + def __infer__(self, x, begin, end, strides): + begin_shape, end_shape, strides_shape = begin['shape'], end['shape'], strides['shape'] + if begin_shape != strides_shape or end_shape != strides_shape: + raise ValueError("The shape of begin, end and strides in 'StridedSlice' must be equal.") + + validator.check_const_input("begin", begin['value']) + validator.check_const_input("end", end['value']) + validator.check_const_input("strides", strides['value']) + validator.check_type("begin", begin['value'], [tuple]) + validator.check_type("end", end['value'], [tuple]) + validator.check_type("strides", strides['value'], [tuple]) + + x_shape = x['shape'] + x_shp_len = len(x_shape) + begin_v, end_v, strides_v = begin['value'], end['value'], strides['value'] + ret_shape = [] + append_dimensions = [] + shrink_pos = bin(self.shrink_axis_mask)[::-1] + new_pos = bin(self.new_axis_mask)[::-1] + for i in range(x_shp_len): + # After the integer is converted to binary, it is a str and the first two chars are the flag char '0b' + if i < (len(new_pos) - 2) and new_pos[i] == '1': + ret_shape.append(1) + append_dimensions.append(x_shape[x_shp_len - 1 - len(append_dimensions)]) + continue + if i < (len(shrink_pos) - 2) and shrink_pos[i] == '1': + validator.check_integer(f'begin[{i}]', begin_v[i], -x_shape[i], Rel.GE) + validator.check_integer(f'begin[{i}]', begin_v[i], x_shape[i], Rel.LT) + continue + + begin_idx = begin_v[i] + end_idx = end_v[i] + strides_idx = strides_v[i] + if self.begin_mask: + begin_idx = 0 + if self.end_mask: + end_idx = x_shape[i] + validator.check_integer(f'begin[{i}]', begin_idx, x_shape[i], Rel.LE) + validator.check_integer(f'end[{i}]', end_idx, x_shape[i], Rel.LE) + validator.check_integer(f'strides[{i}]', strides_idx, 0, Rel.NE) + if strides_idx > 0: + # If sliced forward , end_idx >= begin_idx + validator.check(f'begin[{i}]', begin_idx, f'end[{i}]', end_idx, Rel.LE) + if begin_idx < 0 < end_idx: + # Turn negative begin_idx into positive values + begin_idx = x_shape[i] + begin_idx + num_elems = (end_idx - begin_idx + strides_idx - 1) // strides_idx + else: + # If sliced backwards, end_idx <= begin_idx + validator.check(f'begin[{i}]', begin_idx, f'end[{i}]', end_idx, Rel.GE) + if end_idx < 0 < begin_idx: + # Turn negative end_idx into positive values + end_idx = x_shape[i] + end_idx + num_elems = (end_idx - begin_idx + strides_idx + 1) // strides_idx + + ret_shape.append(num_elems) + if append_dimensions: + ret_shape += append_dimensions[::-1] + return {'shape': ret_shape, + 'dtype': x['dtype'], + 'value': None} + + +class Diag(PrimitiveWithInfer): + r""" + + Extract or construct a diagonal array. + + If input is a 2-D tensor, returns the diagonal of the input with the given offset. If + input is a 1-D tensor, returns the array of diagonals. If you use this function + to extract the diagonal and want to write to the result array, see the more + detailed documentation for "numpy.diagonal", whether you return a copy or a + view depends on the version of numpy you are using. + + Inputs: + - **input_x** (Tensor) - 1-D tensor or 2-D tensor. + + Outputs: + Tensor. + + Examples: + >>> diag = P.Diag() + >>> diag(x) + """ + + @prim_attr_register + def __init__(self): + """init Diag""" + + def infer_type(self, x): + args = {"x_dtype": x} + validator.check_subclass('input_x', x, mstype.tensor) + validator.check_type_same(args, mstype.number_type) + return x + + def infer_value(self, x): + validator.check("shape_length", len(x.shape()), "length", [1, 2], Rel.IN) + ret = np.diag(x.asnumpy()) + return Tensor(ret) + + +class Eye(PrimitiveWithInfer): + """ + + Creates a tensor with ones on the diagonal and zeros elsewhere. + + Inputs: + - **n** (int) - Number of rows of returned tensor + - **m** (int) - Number of columns of returned tensor + - **t** (mindspore.dtype) - Mindspore's dtype, The data type of the returned tensor. + + Outputs: + Tensor, a tensor with ones on the diagonal and zeros elsewhere. + + Examples: + >>> eye = P.Eye() + >>> out_tensor = eye(2, 2, mindspore.int32) + """ + + @prim_attr_register + def __init__(self): + """init Eye""" + + def infer_value(self, n, m, t): + validator.check_type("n", n, [int]) + validator.check_integer("n", n, 0, Rel.GT) + validator.check_type("m", m, [int]) + validator.check_integer("m", m, 0, Rel.GT) + args = {"dtype": t} + validator.check_type_same(args, mstype.number_type + (mstype.bool_,)) + np_type = mstype.dtype_to_nptype(t) + ret = np.eye(n, m, dtype=np_type) + return Tensor(ret) + + +class ScatterNd(PrimitiveWithInfer): + """ + Scatters a tensor into a new tensor depending on the specified indices. + + Creates an empty tensor, and set values by scattering the update tensor depending on indices. + + Inputs: + - **indices** (Tensor) - The index of scattering in the new tensor. + - **update** (Tensor) - The source Tensor to be scattered. + - **shape** (tuple[int]) - Define the shape of the output tensor. Has the same type as indices. + + Outputs: + Tensor, the new tensor, has the same type as `update` and the same shape as `shape`. + + Examples: + >>> op = ScatterNd() + >>> update = Tensor(np.array([3.2, 1.1]), mindspore.float32) + >>> indices = Tensor(np.array([[0, 1], [1, 1]]), mindspore.int32) + >>> shape = (3, 3) + >>> output = op(indices, update, shape) + """ + + @prim_attr_register + def __init__(self): + """Init ScatterNd""" + self.init_prim_io_names(inputs=['indices', 'update', 'shape'], outputs=['output']) + + def __infer__(self, indices, update, shape): + shp = shape['value'] + validator.check_subclass("indices_dtype", indices['dtype'], mstype.tensor) + validator.check_subclass("update_dtype", update['dtype'], mstype.tensor) + validator.check_typename("indices_dtype", indices['dtype'], mstype.int_type) + validator.check_type("shape", shp, [tuple]) + for i, x in enumerate(shp): + validator.check_integer("shape[%d]" % i, x, 0, Rel.GT) + + indices_shape, update_shape = indices["shape"], update["shape"] + if indices_shape[0] != update_shape[0]: + raise ValueError('The indices_shape[0] and update_shape[0] must be equal.') + + return {'shape': shp, + 'dtype': update['dtype'], + 'value': None} + + +class ResizeNearestNeighbor(PrimitiveWithInfer): + """ + Resize the input tensor by using nearest neighbor algorithm. + + Resize input tensor to given size by using nearest neighbor algorithm. The nearest + neighbor algorithm selects the value of the nearest point and does not consider the + values of neighboring points at all, yielding a piecewise-constant interpolant. + + Args: + size (Union[tuple, list]): The target size. The dimension of size must be 2. + align_corners (bool): Whether the centers of the 4 corner pixels of the input + and output tensors are aligned. Default: False. + + Inputs: + - **input_x** (Tensor) - The input tensor. The shape of the tensor is :math:`(N, C, H, W)`. + + Outputs: + Tensor, the shape of the output tensor is :math:`(N, NEW_C, NEW_H, W)`. + + Examples: + >>> input_tensor = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]), mindspore.float32) + >>> resize = ResizeNearestNeighbor((2, 2)) + >>> output = resize(input_tensor) + """ + + @prim_attr_register + def __init__(self, size, align_corners=False): + """Init ResizeNearestNeighbor""" + self.init_prim_io_names(inputs=['image_in'], outputs=['image_out']) + + def infer_shape(self, x): + validator.check('the dimension of input_x', len(x), '', 2, Rel.GE) + return tuple(x)[:-2] + tuple(self.size) + + def infer_dtype(self, x): + return x + + +class GatherNd(PrimitiveWithInfer): + """ + Gathers slices from a tensor by indices. + + Using given indices to gather slices from a tensor with a specified shape. + + Inputs: + - **input_x** (Tensor) - The target tensor to gather values. + - **indices** (Tensor) - The index tensor. + + Outputs: + Tensor, has the same type as `input_x` and the shape is indices_shape[:-1] + x_shape[indices_shape[-1]:]. + + Examples: + >>> input_x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]), mindspore.float32) + >>> indices = Tensor(np.array([[0, 0], [1, 1]]), mindspore.int32) + >>> op = GatherNd() + >>> output = op(input_x, indices) + """ + + @prim_attr_register + def __init__(self): + """Init GatherNd""" + self.init_prim_io_names(inputs=['input_x', 'indices'], outputs=['y']) + + def infer_shape(self, x_shape, indices_shape): + validator.check('the dimension of x', len(x_shape), + 'the dimension of indices', indices_shape[-1], Rel.GE) + return indices_shape[:-1] + x_shape[indices_shape[-1]:] + + def infer_dtype(self, x_dtype, indices_dtype): + validator.check_subclass("x_dtype", x_dtype, mstype.tensor) + validator.check_subclass("indices_dtype", indices_dtype, mstype.tensor) + validator.check_typename("indices_dtype", indices_dtype, mstype.int_type) + return x_dtype + + +class ScatterNdUpdate(PrimitiveWithInfer): + """ + Update tensor value by using input indices and value. + + Using given values to update tensor value, along with the input indices. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Defaule: True. + + Inputs: + - **input_x** (Tensor) - The target tensor. + - **indices** (Tensor) - The index of input tensor. + - **update** (Tensor) - The tensor to add to the input tensor, has the same type as input. + + Outputs: + Tensor, has the same shape and type as `input_x`. + + Examples: + >>> input_x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]), mindspore.float32) + >>> indices = Tensor(np.array([[0, 0], [1, 1]]), mindspore.int32) + >>> update = Tensor(np.array([1.0, 2.2]), mindspore.float32) + >>> op = ScatterNdUpdate() + >>> output = op(input_x, indices, update) + """ + + @prim_attr_register + def __init__(self, use_locking=True): + """Init ScatterNdUpdate""" + self.init_prim_io_names(inputs=['x', 'indices', 'value'], outputs=['y']) + + def infer_shape(self, x_shape, indices_shape, value_shape): + validator.check('the dimension of x', len(x_shape), + 'the dimension of indices', indices_shape[-1], Rel.GE) + if indices_shape[:-1] + x_shape[indices_shape[-1]:] != value_shape: + raise ValueError('Input value are not match with input indices.') + return x_shape + + def infer_dtype(self, x_dtype, indices_dtype, value_dtype): + validator.check_subclass("x_dtype", x_dtype, mstype.tensor) + validator.check_subclass("indices_dtype", indices_dtype, mstype.tensor) + validator.check_subclass("value_dtype", value_dtype, mstype.tensor) + validator.check_typename('indices_dtype', indices_dtype, mstype.int_type) + args = {"x_dtype": x_dtype, "value_dtype": value_dtype} + validator.check_type_same(args, (mstype.bool_,) + mstype.number_type) + return x_dtype + + +class SpaceToDepth(PrimitiveWithInfer): + r""" + Rearrange blocks of spatial data into depth. + + The output tensor's `height` dimension is :math:`height / block\_size`. + + The output tensor's `weight` dimension is :math:`weight / block\_size`. + + The depth of output tensor is :math:`block\_size * block\_size * input\_depth`. + + The input tensor's height and width must be divisible by `block_size`. + The data format is "NCHW". + + Args: + block_size (int): The block size used to divide spatial data. It must be >= 2. + + Inputs: + - **x** (Tensor) - The target tensor. + + Outputs: + Tensor, the same type as `x`. + + Examples: + >>> x = Tensor(np.random.rand(1,3,2,2), mindspore.float32) + >>> block_size = 2 + >>> op = SpaceToDepth(block_size) + >>> output = op(x) + >>> output.asnumpy().shape == (1,12,1,1) + """ + + @prim_attr_register + def __init__(self, block_size): + """Init SpaceToDepth""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + validator.check_type('block_size', block_size, [int]) + validator.check('block_size', block_size, '', 2, Rel.GE) + self.block_size = block_size + self.add_prim_attr("data_format", "NCHW") + + def infer_shape(self, x_shape): + validator.check('x dimension', len(x_shape), '', 4, Rel.EQ) + out_shape = copy.deepcopy(x_shape) + for i in range(2): + if out_shape[i+2] % self.block_size != 0: + raise ValueError(f'SpaceToDepth input shape[{i+2}] {out_shape[i+2]} should be ' + f'fully divided by block_size {self.block_size}') + out_shape[i+2] //= self.block_size + + out_shape[1] *= self.block_size * self.block_size + return out_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("x_dtype", x_dtype, mstype.tensor) + return x_dtype + + +class DepthToSpace(PrimitiveWithInfer): + r""" + Rearrange blocks of depth data into spatial dimensions. + + This is the reverse operation of SpaceToDepth. + + The output tensor's `height` dimension is :math:`height * block\_size`. + + The output tensor's `weight` dimension is :math:`weight * block\_size`. + + The depth of output tensor is :math:`input\_depth / (block\_size * block\_size)`. + + The input tensor's depth must be divisible by `block_size * block_size`. + The data format is "NCHW". + + Args: + block_size (int): The block size used to divide depth data. It must be >= 2. + + Inputs: + - **x** (Tensor) - The target tensor. + + Outputs: + Tensor, the same type as `x`. + + Examples: + >>> x = Tensor(np.random.rand(1,12,1,1), mindspore.float32) + >>> block_size = 2 + >>> op = DepthToSpace(block_size) + >>> output = op(x) + >>> output.asnumpy().shape == (1,3,2,2) + """ + + @prim_attr_register + def __init__(self, block_size): + """Init DepthToSpace""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + validator.check_type('block_size', block_size, [int]) + validator.check('block_size', block_size, '', 2, Rel.GE) + self.block_size = block_size + self.add_prim_attr("data_format", "NCHW") + + def infer_shape(self, x_shape): + validator.check('x dimension', len(x_shape), '', 4, Rel.EQ) + out_shape = copy.deepcopy(x_shape) + for i in range(2): + out_shape[i+2] *= self.block_size + + validator.check('x_shape[1] % (block_size*block_size)', x_shape[1] % (self.block_size*self.block_size), '', 0) + out_shape[1] //= self.block_size * self.block_size + return out_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("x_dtype", x_dtype, mstype.tensor) + return x_dtype diff --git a/mindspore/ops/operations/comm_ops.py b/mindspore/ops/operations/comm_ops.py new file mode 100644 index 00000000000..53a36863678 --- /dev/null +++ b/mindspore/ops/operations/comm_ops.py @@ -0,0 +1,413 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""comm_ops""" + +from ..._checkparam import ParamValidator as validator +from ...communication.management import get_rank, get_group_size, GlobalComm, get_group +from ...common import dtype as mstype +from ..primitive import PrimitiveWithInfer, prim_attr_register + + +class ReduceOp: + """ + Operation options for reduce tensors. + + There are four kinds of operation options, "SUM","MAX","MIN","PROD". + + - SUM: Take the sum. + - MAX: Take the maximum. + - MIN: Take the minimum. + - PROD: Take the product. + """ + SUM = "sum" + MAX = "max" + MIN = "min" + PROD = "prod" + + +class AllReduce(PrimitiveWithInfer): + """ + Reduces the tensor data across all devices in such a way that all devices will get the same final result. + + Note: + The operation of AllReduce does not support "prod" currently. + The input of AllReduce does not support dtype "Bool". + Tensor must have same shape and format in all processes participating in the collective. + + Args: + op (str): Specifies an operation used for element-wise reductions, + like sum, max, min. Default: ReduceOp.SUM. + group (str): The communication group to work on. Default: "hccl_world_group". + + Raises: + TypeError: If any of op and group is not a string + or fusion is not a integer or the input's dtype is bool. + ValueError: If op is "prod" + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, has the same shape of the input, i.e., :math:`(x_1, x_2, ..., x_R)`. + The contents depend on the specified operation. + + Examples: + >>> from mindspore.communication.management import init + >>> init('nccl') + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.allreduce_sum = AllReduce(ReduceOp.SUM, group="nccl_world_group") + >>> + >>> def construct(self, x): + >>> return self.allreduce_sum(x) + >>> + >>> input_ = Tensor(np.ones([2, 8]).astype(np.float32)) + >>> net = Net() + >>> output = net(input_) + """ + + @prim_attr_register + def __init__(self, op=ReduceOp.SUM, group=GlobalComm.WORLD_COMM_GROUP): + if not isinstance(op, type(ReduceOp.SUM)): + raise TypeError("The operation of AllReduce should be str.") + if op == ReduceOp.PROD: + raise RuntimeError("The operation of AllReduce 'prod' is not supported yet.") + if not isinstance(get_group(group), str): + raise TypeError("The group of AllReduce should be str.") + self.op = op + self.add_prim_attr('group', get_group(group)) + self.add_prim_attr('fusion', 0) + + def vm_impl(self, x): + """Implement by vm mode.""" + x = x.asnumpy() + return Tensor(x) + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + if x_dtype == mstype.bool_: + raise TypeError("AllReduce does not support 'Bool' as the dtype of input!") + return x_dtype + + +class AllGather(PrimitiveWithInfer): + """ + Gathers tensors from the specified communication group. + + Note: + Tensor must have the same shape and format in all processes participating in the collective. + + Args: + group (str): The communication group to work on. Default: "hccl_world_group". + + Raises: + TypeError: If group is not a string. + ValueError: If the local rank id of the calling process in the group + is larger than the group's rank size. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor. If the number of devices in the group is N, + then the shape of output is :math:`(N, x_1, x_2, ..., x_R)`. + + Examples: + >>> from mindspore.communication.management import init + >>> init('nccl') + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.allgather = AllGather(group="nccl_world_group") + >>> + >>> def construct(self, x): + >>> return self.allgather(x) + >>> + >>> input_ = Tensor(np.ones([2, 8]).astype(np.float32)) + >>> net = Net() + >>> output = net(input_) + """ + + @prim_attr_register + def __init__(self, group=GlobalComm.WORLD_COMM_GROUP): + if not isinstance(get_group(group), str): + raise TypeError("The group of AllGather should be str.") + self.rank = get_rank(get_group(group)) + self.rank_size = get_group_size(get_group(group)) + if self.rank >= self.rank_size: + raise ValueError("The rank of AllGather should be less than the rank_size.") + self.add_prim_attr('rank_size', self.rank_size) + self.add_prim_attr('group', get_group(group)) + + def infer_shape(self, x_shape): + x_shape[0] = x_shape[0] * self.rank_size + return x_shape + + def infer_dtype(self, x_dtype): + return x_dtype + + def __call__(self, tensor): + raise NotImplementedError + + +class ReduceScatter(PrimitiveWithInfer): + """ + Reduces and scatters tensors from the specified communication group. + + Note: + The back propagation of the op is not surported yet. Stay tuned for more. + Tensor must have the same shape and format in all processes participating in the collective. + Args: + op (str): Specifies an operation used for element-wise reductions, + like sum, max, avg. Default: ReduceOp.SUM. + group (str): The communication group to work on. Default: "hccl_world_group". + + Raises: + TypeError: If any of op and group is not a string + ValueError: If the first dimension of input can not be divided by rank size. + + Examples: + >>> from mindspore.communication.management import init + >>> init('nccl') + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.reducescatter = ReduceScatter(ReduceOp.SUM, group="nccl_world_group") + >>> + >>> def construct(self, x): + >>> return self.reducescatter(x) + >>> + >>> input_ = Tensor(np.ones([2, 8]).astype(np.float32)) + >>> net = Net() + >>> output = net(input_) + """ + + @prim_attr_register + def __init__(self, op=ReduceOp.SUM, group=GlobalComm.WORLD_COMM_GROUP): + if not isinstance(op, type(ReduceOp.SUM)): + raise TypeError("The operation of ReduceScatter should be {}.".format(type(ReduceOp.SUM))) + if not isinstance(get_group(group), str): + raise TypeError("The group of ReduceScatter should be str.") + self.op = op + self.rank_size = get_group_size(get_group(group)) + self.add_prim_attr('rank_size', self.rank_size) + self.add_prim_attr('group', get_group(group)) + + def infer_shape(self, x_shape): + if x_shape[0] % self.rank_size != 0: + raise ValueError("The first dimension of x should be divided by rank_size.") + x_shape[0] = int(x_shape[0]/self.rank_size) + return x_shape + + def infer_dtype(self, x_dtype): + return x_dtype + + def __call__(self, tensor): + raise NotImplementedError + + +class Broadcast(PrimitiveWithInfer): + """ + Broadcasts the tensor to the whole group. + + Note: + Tensor must have the same shape and format in all processes participating in the collective. + + Args: + root_rank (int): Source rank. Required in all processes except the one + that is sending the data. + group (str): The communication group to work on. Default: "hccl_world_group". + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, has the same shape of the input, i.e., :math:`(x_1, x_2, ..., x_R)`. + The contents depend on the data of the `root_rank` device. + + Raises: + TypeError: If root_rank is not a integer or group is not a string. + + Examples: + >>> from mindspore.communication.management import init + >>> init('nccl') + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.broadcast = Broadcast(1) + >>> + >>> def construct(self, x): + >>> return self.broadcast((x,)) + >>> + >>> input_ = Tensor(np.ones([2, 8]).astype(np.float32)) + >>> net = Net() + >>> output = net(input_) + """ + + @prim_attr_register + def __init__(self, root_rank, group=GlobalComm.WORLD_COMM_GROUP): + if not isinstance(root_rank, int): + raise TypeError("The root_rank of Broadcast should be int.") + if not isinstance(get_group(group), str): + raise TypeError("The group of Broadcast should be str.") + self.add_prim_attr('group', get_group(group)) + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + return x_dtype + + +class _AlltoAll(PrimitiveWithInfer): + """ + AlltoAll is a collective operation. + + AlltoAll sends data from the all processes to the all processes in the specified group. It has two phases: + + - The scatter phase: On each process, the operand is split into split_count number of blocks along the + split_dimensions, and the blocks are scattered to all processes, e.g., the ith block is send to the ith process. + - The gather phase: Each process concatenates the received blocks along the concat_dimension. + + Note: + Tensor must have the same shape and format in all processes participating in the collective. + + Args: + split_count (int): On each process, divide blocks into split_count number. + split_dim (int): On each process, split blocks along the split_dim. + concat_dim (int): On each process, gather the received blocks along the concat_dimension. + group (str): The communication group to work on. Default: "hccl_world_group". + + Raises: + TypeError: If group is not a string. + """ + + @prim_attr_register + def __init__(self, split_count, split_dim, concat_dim, group=GlobalComm.WORLD_COMM_GROUP): + """init AlltoAll""" + if not isinstance(get_group(group), str): + raise TypeError("The group of AllGather should be str.") + self.split_count = split_count + self.split_dim = split_dim + self.concat_dim = concat_dim + self.add_prim_attr('group', get_group(group)) + + def infer_shape(self, x_shape): + x_shape[self.concat_dim] = x_shape[self.concat_dim] * self.split_count + x_shape[self.split_dim] = int(x_shape[self.split_dim] / self.split_count) + return x_shape + + def infer_dtype(self, x_dtype): + return x_dtype + + def __call__(self, tensor): + return + + +class _MirrorOperator(PrimitiveWithInfer): + """ + Auto parallel virtual operator. Do nothing in forward, do all reduce and mean in backward. It is only for + internal use of parallel modules and cannot be called by users. + + Args: + group (str): The communication group to work on. Default: None. + dev_num (int): The device number of the group. Default: None. + mean_flag (bool): Whether use mean in backward. Default: None. + """ + + @prim_attr_register + def __init__(self, group=None, dev_num=None, mean_flag=None): + self.group = group + self.dev_num = dev_num + self.mean_flag = mean_flag + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + return x_dtype + + +mirror = _MirrorOperator() + + +class _VirtualDiv(PrimitiveWithInfer): + """ + Auto parallel virtual operator. Do nothing in forward, do Div in backward. + + Args: + divisor: float32 + """ + @prim_attr_register + def __init__(self, divisor=None): + self.divisor = divisor + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + return x_dtype + + +virtual_div = _VirtualDiv() + + +class _VirtualDataset(PrimitiveWithInfer): + """ + Auto parallel virtual dataset operator. + + It would insert Broadcast operator in forward computation and be deleted before backward computation. + """ + + @prim_attr_register + def __init__(self): + """init""" + + def infer_shape(self, *args): + if len(args) == 1: + return args[0] + return args + + def infer_dtype(self, *args): + if len(args) == 1: + return args[0] + return args + + +virtual_dataset = _VirtualDataset() + + +class _GetTensorSlice(PrimitiveWithInfer): + """ + Gets tensor slice by device matrix and tensor map. + + Args: + dev_mat (tuple): The device matrix of the slice tensor. + tensor_map (tuple): The tensor map of the slice tensor. + """ + + @prim_attr_register + def __init__(self): + """init ChunkTensor""" + + def infer_value(self, x, dev_mat, tensor_map): + from mindspore.parallel._tensor import _load_tensor + validator.check_type("dev_mat", dev_mat, [tuple]) + validator.check_type("tensor_map", tensor_map, [tuple]) + return _load_tensor(x, dev_mat, tensor_map) diff --git a/mindspore/ops/operations/control_ops.py b/mindspore/ops/operations/control_ops.py new file mode 100644 index 00000000000..242a3b155d3 --- /dev/null +++ b/mindspore/ops/operations/control_ops.py @@ -0,0 +1,157 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""control_ops""" + +from ...common import dtype as mstype +from ..._checkparam import ParamValidator as validator +from ..primitive import Primitive, PrimitiveWithInfer, prim_attr_register + + +class ControlDepend(Primitive): + """ + Adds control dependency relation between source and destination operation. + + In many cases, we need to control the execution order of operations. ControlDepend is designed for this. + ControlDepend will indicate the execution engine to run the operations in specific order. ControlDepend + tells the engine that the destination operations should depend on the source operation which means the source + operations should be executed before the destination. + + Args: + depend_mode (int): Use 0 for normal depend, 1 for depend on operations that used the parameter. Default: 0. + + Inputs: + - **src** (Any) - The source input. It can be a tuple of operations output or a single operation output. We do + not concern about the input data, but concern about the operation that generates the input data. + If `depend_mode = 1` is specified and the source input is parameter, we will try to find the operations that + used the parameter as input. + - **dst** (Any) - The destination input. It can be a tuple of operations output or a single operation output. + We do not concern about the input data, but concern about the operation that generates the input data. + If `depend_mode = 1` is specified and the source input is parameter, we will try to find the operations that + used the parameter as input. + + Outputs: + Bool. This operation has no actual data output, it will be used to setup the order of relative operations. + + Examples: + >>> # In the following example, the data calculation uses original global_step. After the calculation the global + >>> # step should be increased, so the add operation should depend on the data calculation operation. + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.global_step = Parameter(initializer(0, [1]), name="global_step") + >>> self.rate = 0.2 + >>> self.control_depend = ControlDepend() + >>> + >>> def construct(self, x): + >>> data = self.rate * self.global_step + x + >>> added_global_step = self.global_step + 1 + >>> self.global_step = added_global_step + >>> self.control_depend(data, added_global_step) + >>> return data + """ + + @prim_attr_register + def __init__(self, depend_mode=0): + """init""" + + def __call__(self, src, dst): + return src + + +class GeSwitch(PrimitiveWithInfer): + """ + Adds control switch to data. + + Switch data to flow into false or true branch depend on the condition. If the condition is true, + the true branch will be activated, or vise verse. + + Inputs: + - **data** (Tensor) - The data to be used for switch control. + - **pred** (Tensor) - It should be a scalar whose type is bool and shape is `()`, It is used as condition for + switch control. + Outputs: + tuple. Output is tuple(false_output, true_output). The Elements in the tuple has the same shape of input data. + The false_output connects with the false_branch and the true_output connects with the true_branch. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.square = P.Square() + >>> self.add = P.TensorAdd() + >>> self.value = Tensor(np.full((1), 3, dtype=np.float32)) + >>> self.switch = P.GeSwitch() + >>> self.merge = P.Merge() + >>> self.less = P.Less() + >>> + >>> def construct(self, x, y): + >>> cond = self.less(x, y) + >>> st1, sf1 = self.switch(x, cond) + >>> st2, sf2 = self.switch(y, cond) + >>> add_ret = self.add(st1, st2) + >>> st3, sf3 = self.switch(self.value, cond) + >>> sq_ret = self.square(sf3) + >>> ret = self.merge((add_ret, sq_ret)) + >>> return ret[0] + >>> + >>> x = Tensor(x_init, dtype=mindspore.float32) + >>> y = Tensor(y_init, dtype=mindspore.float32) + >>> net = Net() + >>> output = net(x, y) + """ + + @prim_attr_register + def __init__(self): + """init""" + + def __call__(self, data, pred): + raise NotImplementedError + + def infer_shape(self, data, pred): + validator.check_scalar_shape_input("pred", pred) + return (data, data) + + def infer_dtype(self, data_type, pred_type): + validator.check_type("pred", pred_type, [type(mstype.bool_)]) + return (data_type, data_type) + + +class Merge(PrimitiveWithInfer): + """ + Merges all input data to one. + + One and only one of the inputs should be selected as the output + + Inputs: + - **inputs** (Tuple) - The data to be merged. All tuple elements should have same shape. + + Outputs: + tuple. Output is tuple(`data`, `output_index`). The `data` has the same shape of `inputs` element. + """ + + @prim_attr_register + def __init__(self): + """init""" + + def __call__(self, *args): + raise NotImplementedError + + def infer_shape(self, inputs): + """merge select one input as its output""" + return (inputs[0], [1]) + + def infer_dtype(self, inputs): + return (inputs[0], mstype.int32) diff --git a/mindspore/ops/operations/debug_ops.py b/mindspore/ops/operations/debug_ops.py new file mode 100644 index 00000000000..6640ef87ca5 --- /dev/null +++ b/mindspore/ops/operations/debug_ops.py @@ -0,0 +1,184 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""debug_ops""" +from ...common import dtype as mstype +from ..primitive import Primitive, prim_attr_register, PrimitiveWithInfer + + +class ScalarSummary(Primitive): + """ + Output scalar to protocol buffer through scalar summary operator. + + Inputs: + - **name** (str) - The name of the input variable. + - **value** (Tensor) - The value of scalar. + + Examples: + >>> class SummaryDemo(nn.Cell): + >>> def __init__(self,): + >>> super(SummaryDemo, self).__init__() + >>> self.summary = P.ScalarSummary() + >>> self.add = P.TensorAdd() + >>> + >>> def construct(self, x, y): + >>> name = "x" + >>> self.summary(name, x) + >>> x = self.add(x, y) + >>> return x + """ + + @prim_attr_register + def __init__(self): + """init""" + + +class ImageSummary(Primitive): + """ + Output image tensor to protocol buffer through image summary operator. + + Inputs: + - **name** (str) - The name of the input variable. + - **value** (Tensor) - The value of image. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.summary = P.ImageSummary() + >>> + >>> def construct(self, x): + >>> name = "image" + >>> out = self.summary(name, x) + >>> return out + """ + + @prim_attr_register + def __init__(self): + """init""" + + +class TensorSummary(Primitive): + """ + Output tensor to protocol buffer through tensor summary operator. + + Inputs: + - **name** (str) - The name of the input variable. + - **value** (Tensor) - The value of tensor. + + Examples: + >>> class SummaryDemo(nn.Cell): + >>> def __init__(self,): + >>> super(SummaryDemo, self).__init__() + >>> self.summary = P.TensorSummary() + >>> self.add = P.TensorAdd() + >>> + >>> def construct(self, x, y): + >>> x = self.add(x, y) + >>> name = "x" + >>> self.summary(name, x) + >>> return x + """ + + @prim_attr_register + def __init__(self): + """init""" + + +class InsertGradientOf(PrimitiveWithInfer): + """ + Attach callback to graph node that will be invoked on the node's gradient. + + Args: + f (Function): MindSpore's Function. Callback function. + + Inputs: + - **input_x** (Tensor) - The graph node to attach to. + + Outputs: + Tensor, returns `input_x` directly. `InsertGradientOf` does not affect the forward result. + + Examples: + >>> def clip_gradient(dx): + >>> ret = dx + >>> if ret > 1.0: + >>> ret = 1.0 + >>> + >>> if ret < 0.2: + >>> ret = 0.2 + >>> + >>> return ret + >>> + >>> clip = P.InsertGradientOf(clip_gradient) + >>> def InsertGradientOfClipDemo(): + >>> def clip_test(x, y): + >>> x = clip(x) + >>> y = clip(y) + >>> c = x * y + >>> return c + >>> + >>> @ms_function + >>> def f(x, y): + >>> return clip_test(x, y) + >>> + >>> def fd(x, y): + >>> return C.grad_all(clip_test)(x, y) + >>> + >>> print("forward: ", f(1.1, 0.1)) + >>> print("clip_gradient:", fd(1.1, 0.1)) + """ + + @prim_attr_register + def __init__(self, f): + self.f = f + + def __call__(self, x): + """run in PyNative mode.""" + return x + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + return x_type + + +class Print(PrimitiveWithInfer): + """ + Output tensor to stdout. + + Inputs: + - **input_x** (Tensor) - The graph node to attach to. + + Examples: + >>> class PrintDemo(nn.Cell): + >>> def __init__(self,): + >>> super(PrintDemo, self).__init__() + >>> self.print = P.Print() + >>> + >>> def construct(self, x): + >>> self.print(x) + >>> return x + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, *inputs): + return [1] + + def infer_dtype(self, *inputs): + return mstype.int32 diff --git a/mindspore/ops/operations/inner_ops.py b/mindspore/ops/operations/inner_ops.py new file mode 100644 index 00000000000..ae7af9cd01a --- /dev/null +++ b/mindspore/ops/operations/inner_ops.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""inner_ops""" + +from ...common.dtype import tensor, dtype_to_pytype +from ..primitive import prim_attr_register, PrimitiveWithInfer + + +class ScalarCast(PrimitiveWithInfer): + """ + Cast the input scalar to another type. + + Inputs: + - **input_x** (scalar) - The input scalar. Only constant value is allowed. + - **input_y** (mindspore.dtype) - The type should cast to be. Only constant value is allowed. + + Outputs: + Scalar. The type is same as the python type corresponding to `input_y`. + + Examples: + >>> scalar_cast = P.ScalarCast() + >>> output = scalar_cast(255.0, mindspore.int32) + """ + + @prim_attr_register + def __init__(self): + pass + + def __infer__(self, x, t): + value, to = x['value'], t['value'] + if value is not None: + if isinstance(to, type(tensor)): + to = to.element_type() + np_type = dtype_to_pytype(to) + value = np_type(value) + out = {'shape': x['shape'], + 'dtype': t['value'], + 'value': value} + return out diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py new file mode 100644 index 00000000000..213e04e351b --- /dev/null +++ b/mindspore/ops/operations/math_ops.py @@ -0,0 +1,1857 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Operators for math.""" + +import numpy as np +from ..._c_expression import signature_rw as sig_rw +from ..._c_expression import signature_kind as sig_kind +from ..._c_expression import signature_dtype as sig_dtype +from ..._checkparam import ParamValidator as validator +from ..._checkparam import Rel +from ...common import dtype as mstype +from ...common.tensor import Tensor +from .._utils import _get_broadcast_shape +from ..primitive import PrimitiveWithInfer, prim_attr_register + + +def _infer_shape_reduce(x, axis, keep_dims): + """Common infer for reduce operator""" + + def reduce_one_axis(one_axis): + validator.check_int_range('axis', one_axis, -dim, dim, Rel.INC_LEFT) + if one_axis < 0: + one_axis += dim + axis_reduce.add(one_axis) + + validator.check_type('axis', axis, [int, tuple, list]) + dim = len(x) + axis_reduce = set() + + if isinstance(axis, int): + reduce_one_axis(axis) + else: + if not axis: + if keep_dims: + return [1] * dim + return [] + for index, one_axis in enumerate(axis): + validator.check_type('axis[%d]' % index, one_axis, [int]) + reduce_one_axis(one_axis) + + out_shape = [] + for i in range(dim): + if i in axis_reduce: + if keep_dims: + out_shape.append(1) + else: + out_shape.append(x[i]) + return out_shape + + +def _check_infer_attr_reduce(axis, keep_dims): + validator.check_type('keep_dims', keep_dims, [bool]) + validator.check_type('axis', axis, [int, tuple]) + if isinstance(axis, tuple): + for index, value in enumerate(axis): + validator.check_type('axis[%d]' % index, value, [int]) + + +class _BinaryOp(PrimitiveWithInfer): + """ + Define binary operators. + """ + + __mindspore_signature__ = (sig_dtype.T, sig_dtype.T) + + @prim_attr_register + def __init__(self): + """init _MathBinaryOp""" + self.init_prim_io_names(inputs=['x', 'y'], outputs=['output']) + + def infer_shape(self, x_shape, y_shape): + return _get_broadcast_shape(x_shape, y_shape) + + +class _MathBinaryOp(_BinaryOp): + """ + Define math binary operators. + """ + + @staticmethod + def do_infer_dtype(x_dtype, y_dtype, valid_dtype=mstype.number_type): + args_type = {"x": x_dtype, "y": y_dtype} + validator.check_args_tensor(args_type) + args_dtype = {"x_dtype": x_dtype, "y_dtype": y_dtype} + validator.check_type_same(args_dtype, valid_dtype) + return x_dtype + + def infer_dtype(self, x_dtype, y_dtype): + return _MathBinaryOp.do_infer_dtype(x_dtype, y_dtype) + + +class TensorAdd(_MathBinaryOp): + """ + Adds two input tensors element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> add = P.TensorAdd() + >>> x = Tensor(np.array([1,2,3]).astype(np.float32)) + >>> y = Tensor(np.array([4,5,6]).astype(np.float32)) + >>> add(x, y) + [5,7,9] + """ + + +class AssignAdd(PrimitiveWithInfer): + """ + Updates a `Parameter` by adding a value to it. + + Inputs: + - **input_x** (Parameter) - The `Parameter`. + - **input_y** (Union[scalar, Tensor]) - Has the same shape as `input_x`. + + Examples: + >>> class Net(Cell): + >>> def __init__(self): + >>> self.AssignAdd = P.AssignAdd() + >>> self.inputdata = Parameter(initializer(1, [1], mindspore.int64), name="global_step") + >>> + >>> def construct(self, x): + >>> self.AssignAdd(self.inputdata, x) + >>> return self.inputdata + >>> + >>> net = Net() + >>> x = Tensor(np.ones([1]).astype(np.int64)*100) + >>> net(x) + """ + __mindspore_signature__ = ( + ('variable', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD), + ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD) + ) + + @prim_attr_register + def __init__(self): + """init AssignAdd""" + self.init_prim_io_names(inputs=['ref', 'value'], outputs=['output']) + + def infer_shape(self, variable, value): + return value + + def infer_dtype(self, variable, value): + args = {"value": value} + validator.check_type_same(args, mstype.number_type) + return value + + +class AssignSub(PrimitiveWithInfer): + """ + Updates a `Parameter` by subtracting a value from it. + + Inputs: + - **input_x** (Parameter) - The `Parameter`. + - **input_y** (Union[scalar, Tensor]) - Has the same shape as `input_x`. + + Examples: + >>> class Net(Cell): + >>> def __init__(self): + >>> self.AssignSub = P.AssignSub() + >>> self.inputdata = Parameter(initializer(1, [1], mindspore.int64), name="global_step") + >>> + >>> def construct(self, x): + >>> self.AssignSub(self.inputdata, x) + >>> return self.inputdata + >>> + >>> net = Net() + >>> x = Tensor(np.ones([1]).astype(np.int64)*100) + >>> net(x) + """ + + __mindspore_signature__ = ( + ('variable', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD), + ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD) + ) + + @prim_attr_register + def __init__(self): + """init AssignSub""" + + def infer_shape(self, variable, value): + return value + + def infer_dtype(self, variable, value): + args = {"value": value} + validator.check_type_same(args, mstype.number_type) + return value + + +class _Reduce(PrimitiveWithInfer): + """ + Definition of base class of reduction class operators. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. + """ + + __mindspore_signature__ = ( + ('input_x', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD), + ('axis', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, ()), + ) + + @prim_attr_register + def __init__(self, keep_dims=False): + """init Reduce""" + validator.check_type('keep_dims', keep_dims, [bool]) + self.init_prim_io_names(inputs=['input_x', 'axis'], outputs=['y']) + + def do_infer(self, input_x, axis, valid_dtype=mstype.number_type): + axis_v = axis['value'] + input_shp = input_x['shape'] + validator.check_subclass('input_x', input_x['dtype'], mstype.tensor) + validator.check_typename('input_x', input_x['dtype'], valid_dtype) + input_shp = _infer_shape_reduce(input_shp, axis_v, self.keep_dims) + return {'shape': input_shp, + 'dtype': input_x['dtype'], + 'value': None} + + def __infer__(self, input_x, axis): + return self.do_infer(input_x, axis) + + +class ReduceMean(_Reduce): + """ + Reduce a dimension of a tensor by averaging all elements in the dimension. + + The dtype of the tensor to be reduced is number. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. Default : False. + + Inputs: + - **input_x** (Tensor[Number]) - The input tensor. + - **axis** (Union[int, tuple(int), list(int)]) - The dimensions to reduce. Default: (), reduce all dimensions. + Only constant value is allowed. + + Outputs: + Tensor, has the same dtype as the 'input_x'. + + - If axis is (), and keep_dims is false, + the output is a 0-D tensor representing the sum of all elements in the input tensor. + - If axis is int, set as 2, and keep_dims is false, + the shape of output is :math:`(x_1, x_3, ..., x_R)`. + - If axis is tuple(int), set as (2, 3), and keep_dims is false, + the shape of output is :math:`(x_1, x_4, ..., x_R)`. + + Examples: + >>> data = Tensor(np.random.randn(3, 4, 5, 6).astype(np.float32)) + >>> op = ReduceMean(keep_dims=True) + >>> output = op(data, 1) + """ + + +class ReduceSum(_Reduce): + """ + Reduce a dimension of a tensor by summing all elements in the dimension. + + The dtype of the tensor to be reduced is number. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. Default : False. + + Inputs: + - **input_x** (Tensor[Number]) - The input tensor. + - **axis** (Union[int, tuple(int), list(int)]) - The dimensions to reduce. Default: (), reduce all dimensions. + Only constant value is allowed. + + Outputs: + Tensor, has the same dtype as the 'input_x'. + + - If axis is (), and keep_dims is false, + the output is a 0-D tensor representing the sum of all elements in the input tensor. + - If axis is int, set as 2, and keep_dims is false, + the shape of output is :math:`(x_1, x_3, ..., x_R)`. + - If axis is tuple(int), set as (2, 3), and keep_dims is false, + the shape of output is :math:`(x_1, x_4, ..., x_R)`. + + Examples: + >>> data = Tensor(np.random.randn(3, 4, 5, 6).astype(np.float32)) + >>> op = ReduceSum(keep_dims=True) + >>> output = op(data, 1) + """ + + +class ReduceAll(_Reduce): + """ + Reduce a dimension of a tensor by the "logical and" of all elements in the dimension. + + The dtype of the tensor to be reduced is bool. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. + Default : False, don't keep these reduced dimensions. + + Inputs: + - **input_x** (Tensor[bool]) - The input tensor. + - **axis** (Union[int, tuple(int), list(int)]) - The dimensions to reduce. Default: (), reduce all dimensions. + Only constant value is allowed. + + Outputs: + Tensor, the dtype is bool. + + - If axis is (), and keep_dims is false, + the output is a 0-D tensor representing the "logical and" of of all elements in the input tensor. + - If axis is int, set as 2, and keep_dims is false, + and keep_dims is false, the shape of output is :math:`(x_1, x_3, ..., x_R)`. + - If axis is tuple(int), set as (2, 3), and keep_dims is false, + the shape of output is :math:`(x_1, x_4, ..., x_R)`. + + Examples: + >>> data = Tensor(np.array([[True, False], [True, True]])) + >>> op = ReduceAll(keep_dims=True) + >>> output = op(data, 1) + """ + + def __infer__(self, input_x, axis): + return self.do_infer(input_x, axis, (mstype.bool_,)) + + +class ReduceMax(_Reduce): + """ + Reduce a dimension of a tensor by the maximum value in this dimension. + + The dtype of the tensor to be reduced is number. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. + Default : False, don't keep these reduced dimensions. + + Inputs: + - **input_x** (Tensor[Number]) - The input tensor. + - **axis** (Union[int, tuple(int), list(int)]) - The dimensions to reduce. Default: (), reduce all dimensions. + Only constant value is allowed. + + Outputs: + Tensor, has the same dtype as the 'input_x'. + + - If axis is (), and keep_dims is false, + the output is a 0-D tensor representing the maximum of all elements in the input tensor. + - If axis is int, set as 2, and keep_dims is false, + the shape of output is :math:`(x_1, x_3, ..., x_R)`. + - If axis is tuple(int), set as (2, 3), and keep_dims is false, + the shape of output is :math:`(x_1, x_4, ..., x_R)`. + + Examples: + >>> data = Tensor(np.random.randn(3, 4, 5, 6).astype(np.float32)) + >>> op = ReduceMax(keep_dims=True) + >>> output = op(data, 1) + """ + + +class ReduceMin(_Reduce): + """ + Reduce a dimension of a tensor by the minimum value in the dimension. + + The dtype of the tensor to be reduced is number. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. + Default : False, don't keep these reduced dimensions. + + Inputs: + - **input_x** (Tensor[Number]) - The input tensor. + - **axis** (Union[int, tuple(int), list(int)]) - The dimensions to reduce. Default: (), reduce all dimensions. + Only constant value is allowed. + + Outputs: + Tensor, has the same dtype as the 'input_x'. + + - If axis is (), and keep_dims is false, + the output is a 0-D tensor representing the minimum of all elements in the input tensor. + - If axis is int, set as 2, and keep_dims is false, + the shape of output is :math:`(x_1, x_3, ..., x_R)`. + - If axis is tuple(int), set as (2, 3), and keep_dims is false, + the shape of output is :math:`(x_1, x_4, ..., x_R)`. + + Examples: + >>> data = Tensor(np.random.randn(3, 4, 5, 6).astype(np.float32)) + >>> op = ReduceMin(keep_dims=True) + >>> output = op(data, 1) + """ + + +class ReduceProd(_Reduce): + """ + Reduce a dimension of a tensor by multiplying all elements in the dimension. + + The dtype of the tensor to be reduced is number. + + Args: + keep_dims (bool): If True, keep these reduced dimensions and the length is 1. + If False, don't keep these dimensions. + Default : False, don't keep these reduced dimensions. + + Inputs: + - **input_x** (Tensor[Number]) - The input tensor. + - **axis** (Union[int, tuple(int), list(int)]) - The dimensions to reduce. Default: (), reduce all dimensions. + + Outputs: + Tensor, has the same dtype as the 'input_x'. + + - If axis is (), and keep_dims is false, + the output is a 0-D tensor representing the product of all elements in the input tensor. + - If axis is int, set as 2, and keep_dims is false, + the shape of output is :math:`(x_1, x_3, ..., x_R)`. + - If axis is tuple(int), set as (2, 3), and keep_dims is false, + the shape of output is :math:`(x_1, x_4, ..., x_R)`. + + Examples: + >>> data = Tensor(np.random.randn(3, 4, 5, 6).astype(np.float32)) + >>> op = ReduceProd(keep_dims=True) + >>> output = op(data, 1) + """ + + +class CumProd(PrimitiveWithInfer): + """ + Compute the cumulative product of the tensor x along axis. + + Args: + exclusive (bool): If True, perform exclusive cumulative product. Default: False. + reverse (bool): If True, reverse the result along axis. Default: False + + Inputs: + - **input_x** (Tensor[Number]) - The input tensor. + - **axis** (int) - The dimensions to compute the cumulative product. + + Outputs: + Tensor, has the same shape and dtype as the 'input_x'. + + Examples: + >>> data = Tensor(np.array([a, b, c]).astype(np.float32)) + >>> op0 = CumProd() + >>> output = op0(data, 0) # output=[a, a * b, a * b * c] + >>> op1 = CumProd(exclusive=True) + >>> output = op1(data, 0) # output=[1, a, a * b] + >>> op2 = CumProd(reverse=True) + >>> output = op2(data, 0) # output=[a * b * c, b * c, c] + >>> op3 = CumProd(exclusive=True, reverse=True) + >>> output = op3(data, 0) # output=[b * c, c, 1] + """ + @prim_attr_register + def __init__(self, exclusive=False, reverse=False): + self.exclusive = validator.check_type("exclusive", exclusive, [bool]) + self.reverse = validator.check_type("reverse", reverse, [bool]) + + def infer_shape(self, x_shape, axis_shape): + return x_shape + + def infer_dtype(self, x_type, axis_type): + validator.check_subclass('x_type', x_type, mstype.tensor) + validator.check_typename('x_type', x_type, mstype.number_type) + validator.check_subclass("axis_type", axis_type, mstype.int_) + return x_type + + +class MatMul(PrimitiveWithInfer): + """ + Multiplies matrix `a` by matrix `b`. + + The rank of input tensors must be `2`. + + Args: + transpose_a (bool): If True, `a` is transposed before multiplication. Default: False. + transpose_b (bool): If True, `b` is transposed before multiplication. Default: False. + + Inputs: + - **input_x** (Tensor) - The first tensor to be multiplied. The shape of the tensor is :math:`(N, C)`. If + `transpose_a` is True, its shape should be :math:`(N, C)` after transposing. + - **input_y** (Tensor) - The second tensor to be multiplied. The shape of the tensor is :math:`(C, M)`. If + `transpose_b` is True, its shape should be :math:`(C, M)` after transpose. + + Outputs: + Tensor, the shape of the output tensor is :math:`(N, M)`. + + Examples: + >>> input_x = Tensor(np.ones(shape=[1, 3]), mindspore.float32) + >>> input_y = Tensor(np.ones(shape=[3, 4]), mindspore.float32) + >>> matmul = MatMul() + >>> output = matmul(input_x, input_y) + """ + + @prim_attr_register + def __init__(self, transpose_a=False, transpose_b=False): + self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['output']) + self.__setattr_flag__ = True + validator.check_type("transpose_a", transpose_a, [bool]) + validator.check_type("transpose_b", transpose_b, [bool]) + + def check_shape_size(self, x, y): + if len(x) != 2 or len(y) != 2: + raise ValueError('MatMul input x, y should be the same dimension size and should be ' + + f'equal to 2, while x size = {len(x)}, y size= {len(y)}') + + def infer_shape(self, x, y): + self.check_shape_size(x, y) + cls_name = self.__class__.__name__ + # expected dimension of x, y, x:[...,a,b] y:[..., c,d], the dim size should be the same except the last two + for i in range(len(x) - 2): + if x[i] != y[i]: + raise ValueError(f'{cls_name} shape in dim[{i}] not the same, while x is {x[i]}, y is {y[i]}') + + # validate whether last two dims satifing matrix multiply + x_last = x[-2:] + y_last = y[-2:] + + x_col = x_last[not self.transpose_a] # x_col = x_last[1] if (not transpose_a) else x_last[0] + y_row = y_last[self.transpose_b] # y_row = y_last[0] if (not transpose_b) else y_last[1] + if x_col != y_row: + raise ValueError(f'{cls_name} evaluator shapes of inputs can not do this operator, got {x_col} and {y_row}' + + f' for {cls_name}, with x shape {x}(transpose_a={self.transpose_a})' + + f', y shape {y}(transpose_b={self.transpose_b}).') + # set attribute + self.add_prim_attr('transpose_x1', self.transpose_a) + self.add_prim_attr('transpose_x2', self.transpose_b) + + ret_dims = x[: -2] + [x_last[self.transpose_a], y_last[not self.transpose_b]] + return ret_dims + + def infer_dtype(self, x, y): + validator.check_subclass("x", x, mstype.tensor) + validator.check_subclass("y", y, mstype.tensor) + args = {"x dtype": x, "y dtype": y} + validator.check_type_same(args, mstype.float_type + mstype.int_type) + return x + + +class BatchMatMul(MatMul): + """ + Computes matrix multiplication between two tensors by batch + + `result[..., :, :] = tensor(a[..., :, :]) * tensor(b[..., :, :])`. + + The two input tensors must have same rank and the rank must be `3` at least. + + Args: + transpose_a (bool): If True, `a` is transposed on the last two dimensions before multiplication. + Default: False. + transpose_b (bool): If True, `b` is transposed on the last two dimensions before multiplication. + Default: False. + + Inputs: + - **input_x** (Tensor) - The first tensor to be multiplied. The shape of the tensor is :math:`(*B, N, C)`, + where :math:`*B` represents the batch size which can be multidimensional, :math:`N` and :math:`C` are the + size of the last two dimensions. If `transpose_a` is True, its shape should be :math:`(*B, C, N)`. + - **input_y** (Tensor) - The second tensor to be multiplied. The shape of the tensor is :math:`(*B, C, M)`. If + `transpose_b` is True, its shape should be :math:`(*B, M, C)`. + + Outputs: + Tensor, the shape of the output tensor is :math:`(*B, N, M)`. + + Examples: + >>> input_x = Tensor(np.ones(shape=[2, 4, 1, 3]), mindspore.float32) + >>> input_y = Tensor(np.ones(shape=[2, 4, 3, 4]), mindspore.float32) + >>> batmatmul = BatchMatMul() + >>> output = batmatmul(input_x, input_y) + >>> + >>> input_x = Tensor(np.ones(shape=[2, 4, 3, 1]), mindspore.float32) + >>> input_y = Tensor(np.ones(shape=[2, 4, 3, 4]), mindspore.float32) + >>> batmatmul = BatchMatMul(transpose_a=True) + >>> output = batmatmul(input_x, input_y) + """ + + @prim_attr_register + def __init__(self, transpose_a=False, transpose_b=False): + self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['output']) + self.__setattr_flag__ = True + validator.check_type("transpose_a", transpose_a, [bool]) + validator.check_type("transpose_b", transpose_b, [bool]) + + def check_shape_size(self, x, y): + if len(x) != len(y) or len(x) < 3: + raise ValueError('BatchMatMul input x, y should be the same dimension size and should be ' + 'greater or equal to 3,' + f' while x size = {len(x)}, y size= {len(y)}') + + +class CumSum(PrimitiveWithInfer): + """ + Computes the cumulative sum of input tensor along axis. + + Args: + exclusive (bool): If True, perform exclusive mode. Default: False. + reverse (bool): If True, perform inverse cumulative sum. Default: False. + + Inputs: + - **input** (Tensor) - The input tensor to accumulate. + - **axis** (int) - The axis to accumulate the tensor's value. + + Outputs: + Tensor, the shape of the output tensor is consistent with the input tensor's. + + Examples: + >>> input = Tensor(np.array([[3, 4, 6, 10],[1, 6, 7, 9],[4, 3, 8, 7],[1, 3, 7, 9]]).astype(np.float32)) + >>> cumsum = CumSum() + >>> output = cumsum(input, 1) + [[ 3. 7. 13. 23.] + [ 1. 7. 14. 23.] + [ 4. 7. 15. 22.] + [ 1. 4. 11. 20.]] + """ + + @prim_attr_register + def __init__(self, exclusive=False, reverse=False): + """init cumsum""" + self.exclusive = validator.check_type('exclusive', exclusive, [bool]) + self.add_prim_attr("exclusive", self.exclusive) + self.reverse = validator.check_type('reverse', reverse, [bool]) + self.add_prim_attr("reverse", self.reverse) + self.init_prim_io_names(inputs=['x', 'axis'], outputs=['y']) + + def __infer__(self, x, axis): + x_shp = x['shape'] + validator.check_type('axis', axis['value'], [int]) + validator.check_subclass('x', x['dtype'], mstype.tensor) + validator.check_typename('x', x['dtype'], [mstype.uint8, mstype.int8, + mstype.int32, mstype.float16, mstype.float32]) + return {'shape': x_shp, + 'dtype': x['dtype'], + 'value': None} + + +class AddN(PrimitiveWithInfer): + """ + Computes addition of all input tensors element-wise. + + All input tensors should have the same shape. + + Inputs: + - **input_x** (Union(tuple[Tensor], list[Tensor])) - The input tuple or list + is made up of multiple tensors whose dtype is number or bool to be added together. + + Outputs: + Tensor, has the same shape and dtype as each entry of the `input_x`. + + Examples: + >>> class NetAddN(nn.Cell): + >>> def __init__(self): + >>> super(NetAddN, self).__init__() + >>> self.addN = AddN() + >>> + >>> def construct(self, *z): + >>> return self.addN(z) + >>> + >>> net = NetAddN() + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([4, 5, 6]), mindspore.int32) + >>> net(input_x, input_y, input_x, input_y) + Tensor([10, 14, 18], shape=(3,), dtype=mindspore.int32) + """ + + @prim_attr_register + def __init__(self): + self.__setattr_flag__ = True + self.init_prim_io_names(inputs=["inputs"], outputs=["sum"]) + + def infer_shape(self, inputs): + validator.check_integer("inputs", len(inputs), 1, Rel.GE) + self.add_prim_attr('n', len(inputs)) + shp0 = inputs[0] + for i, shp in enumerate(inputs): + validator.check(f"shape of inputs[{i}]", shp, 'shape of inputs[0]', shp0) + return shp0 + + def infer_dtype(self, inputs): + validator.check_type("inputs", inputs, [tuple, list]) + validator.check_integer("inputs", len(inputs), 1, Rel.GE) + args = {} + for i, dtype in enumerate(inputs): + validator.check_subclass(f"inputs[{i}]", dtype, mstype.tensor) + args[f"inputs[{i}]"] = dtype + validator.check_type_same(args, mstype.number_type + (mstype.bool_,)) + return inputs[0] + + +class Neg(PrimitiveWithInfer): + """ + Returns a tensor with negative values of the input tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor whose dtype is number. + + Outputs: + Tensor, has the same shape and dtype as input. + """ + + @prim_attr_register + def __init__(self): + """init Neg""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + validator.check_typename("input_x", input_x, mstype.number_type) + return input_x + + +class Sub(_MathBinaryOp): + """ + Subtracts the second input tensor from the first input tensor element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([4, 5, 6]), mindspore.int32) + >>> sub = Sub() + >>> sub(input_x, input_y) + [-3, -3, -3] + """ + + +class Mul(_MathBinaryOp): + """ + Multiplies two tensors element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([4, 5, 6]), mindspore.int32) + >>> mul = Mul() + >>> mul(input_x, input_y) + [4, 10, 18] + """ + + +class Square(PrimitiveWithInfer): + """ + Returns square of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor whose dtype is number. + + Outputs: + Tensor, has the same shape and dtype as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.0, 2.0, 3.0]), mindspore.float32) + >>> square = Square() + >>> square(input_x) + [1.0, 4.0, 9.0] + """ + + @prim_attr_register + def __init__(self): + """init Square""" + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + validator.check_subclass("x", x_type, mstype.tensor) + validator.check_typename("x_dtype", x_type, mstype.number_type) + return x_type + + +class Rsqrt(PrimitiveWithInfer): + """ + Computes reciprocal of square root of input tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input of Rsqrt. Each element should be a non-negative number. + + Outputs: + Tensor, has the same type and shape as `input_x`. + + Examples: + >>> input_tensor = Tensor([[4, 4], [9, 9]], mindspore.float32) + >>> rsqrt = Rsqrt() + >>> rsqrt(input_tensor) + [[0.5, 0.5], [0.333333, 0.333333]] + """ + + @prim_attr_register + def __init__(self): + """init Rsqrt""" + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + validator.check_subclass("x", x_type, mstype.tensor) + validator.check_typename("x_dtype", x_type, mstype.number_type) + return x_type + + +class Sqrt(PrimitiveWithInfer): + """ + Returns square root of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor whose dtype is number. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.0, 4.0, 9.0]), mindspore.float32) + >>> sqrt = Sqrt() + >>> sqrt(input_x) + [1.0, 2.0, 3.0] + """ + + @prim_attr_register + def __init__(self): + """init Sqrt""" + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + validator.check_subclass("x", x_type, mstype.tensor) + validator.check_typename("x_dtype", x_type, mstype.number_type) + return x_type + + +class Reciprocal(PrimitiveWithInfer): + """ + Returns reciprocal of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.0, 2.0, 4.0]), mindspore.float32) + >>> reciprocal = Reciprocal() + >>> reciprocal(input_x) + [1.0, 0.5, 0.25] + """ + + @prim_attr_register + def __init__(self): + """init Reciprocal""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, x): + return x + + def infer_dtype(self, x): + validator.check_subclass("x", x, mstype.tensor) + return x + + +class Pow(PrimitiveWithInfer): + """ + Computes a tensor to the power of the second input. + + Inputs: + - **input_x** (Tensor) - The input tensor. + - **input_y** (Union[Tensor, Number]) - The exponent part. If exponent is a tensor, its shape must be able to + broadcast to the shape of the `input_x`. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.0, 2.0, 4.0]), mindspore.float32) + >>> input_y = 3.0 + >>> pow = Pow() + >>> pow(input_x, input_y) + [1.0, 8.0, 64.0] + >>> + >>> input_x = Tensor(np.array([1.0, 2.0, 4.0]), mindspore.float32) + >>> input_y = Tensor(np.array([2.0, 4.0, 3.0]), mindspore.float32) + >>> pow = Pow() + >>> pow(input_x, input_y) + [1.0, 16.0, 64.0] + """ + + @prim_attr_register + def __init__(self): + """init Multiply""" + + def infer_shape(self, x, power): + return x + + def infer_dtype(self, x, power): + validator.check_subclass("x", x, mstype.tensor) + validator.check_typename("power", power, mstype.number_type) + return x + + +class Exp(PrimitiveWithInfer): + """ + Returns exponential of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.0, 2.0, 4.0]), mindspore.float32) + >>> exp = Exp() + >>> exp(input_x) + [ 2.71828183, 7.3890561 , 54.59815003] + """ + + @prim_attr_register + def __init__(self): + """init Exp""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + validator.check_subclass("x", x_type, mstype.tensor) + return x_type + + +class Log(PrimitiveWithInfer): + """ + Returns the natural logarithm of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.0, 2.0, 4.0]), mindspore.float32) + >>> log = Log() + >>> log(input_x) + [0.0, 0.69314718, 1.38629436] + """ + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, x): + return x + + def infer_dtype(self, x): + validator.check_subclass("x", x, mstype.tensor) + return x + + +class Minimum(_MathBinaryOp): + """ + Computes the element-wise minimum of input tensors. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([1.0, 5.0, 3.0]), mindspore.float32) + >>> input_y = Tensor(np.array([4.0, 2.0, 6.0]), mindspore.float32) + >>> minimum = Minimum() + >>> minimum(input_x, input_y) + [1.0, 2.0, 3.0] + """ + + +class Maximum(_MathBinaryOp): + """ + Computes the element-wise maximum of input tensors. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([1.0, 5.0, 3.0]), mindspore.float32) + >>> input_y = Tensor(np.array([4.0, 2.0, 6.0]), mindspore.float32) + >>> maximum = Maximum() + >>> maximum(input_x, input_y) + [4.0, 5.0, 6.0] + """ + + +class RealDiv(_MathBinaryOp): + """ + Divide the first input tensor by the second input tensor in floating-point type element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([1.0, 2.0, 3.0]), mindspore.float32) + >>> input_y = Tensor(np.array([4.0, 5.0, 6.0]), mindspore.float32) + >>> realdiv = RealDiv() + >>> realdiv(input_x, input_y) + [0.25, 0.4, 0.5] + """ + + def infer_value(self, x, y): + if x is not None and y is not None: + x = x.asnumpy() + y = y.asnumpy() + out = x / y + out = np.array(out, x.dtype) + return Tensor(out) + return None + + +class Div(_MathBinaryOp): + """ + Computes the quotient of dividing the first input tensor by the second input tensor element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Raises: + ValueError: When `input_x` and `input_y` are not the same dtype. + + Examples: + >>> input_x = Tensor(np.array([-4.0, 5.0, 6.0]), mindspore.float32) + >>> input_y = Tensor(np.array([3.0, 2.0, 3.0]), mindspore.float32) + >>> div = Div() + >>> div(input_x, input_y) + [-2.0, 2.0, 2.0] + """ + + def infer_value(self, x, y): + if x is not None and y is not None: + x = x.asnumpy() + y = y.asnumpy() + return Tensor(x / y) + return None + + +class FloorDiv(_MathBinaryOp): + """ + Divide the first input tensor by the second input tensor element-wise and rounds down to the closest integer. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([2, 4, -1]), mindspore.int32) + >>> input_y = Tensor(np.array([3, 3, 3]), mindspore.int32) + >>> floor_div = FloorDiv() + >>> floor_div(input_x, input_y) + [0, 1, -1] + """ + + +class Floor(PrimitiveWithInfer): + """ + Round a tensor down to the closest integer element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. Its element data type must be float. + + Outputs: + Tensor, has the same shape as `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1.1, 2.5, -1.5]), mindspore.float32) + >>> floor = Floor() + >>> floor(input_x) + [1.0, 2.0, -2.0] + """ + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("x", x_dtype, mstype.tensor) + validator.check_typename("x_dtype", x_dtype, mstype.float_type) + return x_dtype + + +class _LogicBinaryOp(_BinaryOp): + """ + Define logic binary operators. + """ + + @staticmethod + def do_infer_dtype(x_dtype, y_dtype, valid_type=mstype.number_type): + args_type = {"x": x_dtype, "y": y_dtype} + validator.check_args_tensor(args_type) + args_dtype = {"x_dtype": x_dtype, "y_dtype": y_dtype} + validator.check_type_same(args_dtype, valid_type) + return mstype.tensor_type(mstype.bool_) + + def infer_dtype(self, x_dtype, y_dtype): + return _LogicBinaryOp.do_infer_dtype(x_dtype, y_dtype) + + +class Equal(_LogicBinaryOp): + """ + Computes the equivalence between two tensors element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number, bool]) - The first input is a tensor whose data type is number or bool, or + a number or a bool object. + - **input_y** (Union[Tensor, Number, bool]) - The second input tensor whose data type is same as 'input_x' or + a number or a bool object. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.float32) + >>> equal = Equal() + >>> equal(input_x, 2.0) + [False, True, False] + >>> + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 2, 4]), mindspore.int32) + >>> equal = Equal() + >>> equal(input_x, input_y) + [True, True, False] + """ + + def infer_dtype(self, x_dtype, y_dtype): + return _LogicBinaryOp.do_infer_dtype(x_dtype, y_dtype, mstype.number_type + (mstype.bool_,)) + + +class EqualCount(PrimitiveWithInfer): + """ + Computes the number of the same elements of two tensors. + + The two input tensors should have same shape. + + Inputs: + - **input_x** (Tensor) - The first input tensor. + - **input_y** (Tensor) - The second input tensor. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 2, 4]), mindspore.int32) + >>> equal_count = EqualCount() + >>> equal_count(input_x, input_y) + [2] + """ + + @prim_attr_register + def __init__(self): + """init EqualCount""" + self.init_prim_io_names(inputs=['x', 'y'], outputs=['output']) + + def infer_shape(self, x_shape, w_shape): + """Infer shape.""" + output_shape = (1,) + return output_shape + + def infer_dtype(self, x_dtype, w_dtype): + return x_dtype + + +class NotEqual(_LogicBinaryOp): + """ + Computes the non-equivalence of two tensors element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number, bool]) - The first input is a tensor whose data type is number or bool, or + a number or a bool object. + - **input_y** (Union[Tensor, Number, bool]) - The second input tensor whose data type is same as 'input_x' or + a number or a bool object. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.float32) + >>> not_equal = NotEqual() + >>> not_equal(input_x, 2.0) + [True, False, True] + >>> + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 2, 4]), mindspore.int32) + >>> not_equal = NotEqual() + >>> not_equal(input_x, input_y) + [False, False, True] + """ + + def infer_dtype(self, x_dtype, y_dtype): + return _LogicBinaryOp.do_infer_dtype(x_dtype, y_dtype, mstype.number_type + (mstype.bool_,)) + + +class Greater(_LogicBinaryOp): + """ + Computes the boolean value of :math:`x > y` element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is same as 'input_x'. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 1, 4]), mindspore.int32) + >>> greater = Greater() + >>> greater(input_x, input_y) + [False, True, False] + """ + + +class GreaterEqual(_LogicBinaryOp): + """ + Computes the boolean value of :math:`x >= y` element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool'. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 1, 4]), mindspore.int32) + >>> greater_equal = GreaterEqual() + >>> greater_equal(input_x, input_y) + [True, True, False] + """ + + +class Less(_LogicBinaryOp): + """ + Computes the boolean value of :math:`x < y` element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 1, 4]), mindspore.int32) + >>> less = Less() + >>> less(input_x, input_y) + [False, False, True] + """ + + +class LessEqual(_LogicBinaryOp): + """ + Computes the boolean value of :math:`x <= y` element-wise. + + The inputs must be two tensors or one tensor and one scalar. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be same. + When the inputs are one tensor and one scalar, the scalar cannot be a parameter, only can be a constant, + and the type of the scalar is the same as the data type of the tensor. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a tensor whose data type is number or a number. + - **input_y** (Union[Tensor, Number]) - The second input is a tensor whose data type is same as 'input_x' or + a number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool. + + Examples: + >>> input_x = Tensor(np.array([1, 2, 3]), mindspore.int32) + >>> input_y = Tensor(np.array([1, 1, 4]), mindspore.int32) + >>> less_equal = LessEqual() + >>> less_equal(input_x, input_y) + [True, False, True] + """ + + +class LogicalNot(PrimitiveWithInfer): + """ + Computes the "logical NOT" of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor whose dtype is bool + + Outputs: + Tensor, the shape is same as the `input_x`, and the dtype is bool. + + Examples: + >>> input_x = Tensor(np.array([True, False, True]), mindspore.bool_) + >>> logical_not = LogicalNot() + >>> logical_not(input_x) + [False, True, False] + """ + + @prim_attr_register + def __init__(self): + """init LogicalNot""" + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("x", x_dtype, mstype.tensor) + validator.check_typename("x_dtype", x_dtype, [mstype.bool_]) + return mstype.tensor_type(mstype.bool_) + + +class LogicalAnd(_LogicBinaryOp): + """ + Computes the "logical AND" of two tensors element-wise. + + The inputs must be two tensors or one tensor and one bool object. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be bool. + When the inputs are one tensor and one bool object, the bool object cannot be a parameter, only can be a constant, + and the data type of the tensor should be bool. + + Inputs: + - **input_x** (Union[Tensor, bool]) - The first input is a tensor whose data type is bool or a bool object. + - **input_y** (Union[Tensor, bool]) - The second input is a tensor whose data type is bool or a bool object. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool. + + Examples: + >>> input_x = Tensor(np.array([True, False, True]), mindspore.bool_) + >>> input_y = Tensor(np.array([True, True, False]), mindspore.bool_) + >>> logical_and = LogicalAnd() + >>> logical_and(input_x, input_y) + [True, False, False] + """ + + def infer_dtype(self, x_dtype, y_dtype): + return _LogicBinaryOp.do_infer_dtype(x_dtype, y_dtype, (mstype.bool_,)) + + +class LogicalOr(_LogicBinaryOp): + """ + Computes the "logical OR" of two tensors element-wise. + + The inputs must be two tensors or one tensor and one bool object. + When the inputs are two tensors, the shapes of them could be broadcast, + and the data types of them should be bool. + When the inputs are one tensor and one bool object, the bool object cannot be a parameter, only can be a constant, + and the data type of the tensor should be bool. + + Inputs: + - **input_x** (Union[Tensor, bool]) - The first input is a tensor whose data type is bool or a bool object. + - **input_y** (Union[Tensor, bool]) - The second input is a tensor whose data type is bool or a bool object. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, and the data type is bool. + + Examples: + >>> input_x = Tensor(np.array([True, False, True]), mindspore.bool_) + >>> input_y = Tensor(np.array([True, True, False]), mindspore.bool_) + >>> logical_or = LogicalOr() + >>> logical_or(input_x, input_y) + [True, True, True] + """ + + def infer_dtype(self, x_dtype, y_dtype): + return _LogicBinaryOp.do_infer_dtype(x_dtype, y_dtype, (mstype.bool_,)) + + +class NPUAllocFloatStatus(PrimitiveWithInfer): + """ + Allocates a flag to store the overflow status. + + The flag is a tensor whose shape is `(8,)` and data type is `mindspore.dtype.float32`. + + Note: + Examples: see `NPUGetFloatStatus`. + + Outputs: + Tensor, has the shape of `(8,)`. + + Examples: + >>> alloc_status = NPUAllocFloatStatus() + >>> init = alloc_status() + Tensor([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], shape=(8,), dtype=mindspore.float32) + """ + + @prim_attr_register + def __init__(self): + """init NPUAllocFloatStatus""" + self.add_prim_attr("_side_effect_flag", True) + + def infer_shape(self): + return [8] + + def infer_dtype(self): + return mstype.float32 + + +class NPUGetFloatStatus(PrimitiveWithInfer): + """ + Updates the flag which is the output tensor of `NPUAllocFloatStatus` with latest overflow status. + + The flag is a tensor whose shape is `(8,)` and data type is `mindspore.dtype.float32`. + If the sum of the flag equals 0, there is no overflow happened. If the sum of the flag is bigger than 0, there + is overflow happened. + + Inputs: + - **input_x** (Tensor) - The output tensor of `NPUAllocFloatStatus`. + + Outputs: + Tensor, has the same shape as `input_x`. All the elements in the tensor will be zero. + + Examples: + >>> alloc_status = NPUAllocFloatStatus() + >>> get_status = NPUGetFloatStatus() + >>> init = alloc_status() + >>> flag = get_status(init) + Tensor([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], shape=(8,), dtype=mindspore.float32) + """ + + @prim_attr_register + def __init__(self): + """init NPUGetFloatStatus""" + self.add_prim_attr("_side_effect_flag", True) + + def infer_shape(self, x_shape): + validator.check_integer("len(x_shape)", len(x_shape), 1, Rel.EQ) + validator.check_integer("x_shape[0]", x_shape[0], 8, Rel.EQ) + return [8] + + def infer_dtype(self, x_dtype): + args = {"x_dtype": x_dtype} + validator.check_type_same(args, [mstype.float32]) + return mstype.float32 + + +class NPUClearFloatStatus(PrimitiveWithInfer): + """ + Clear the flag which stores the overflow status. + + Note: + The flag is in the register on the `Ascend` device. It will be reset and can not be reused again after the + `NPUClearFloatStatus` is called. + + Examples: see `NPUGetFloatStatus`. + + Inputs: + - **input_x** (Tensor) - The output tensor of `NPUAllocFloatStatus`. + + Outputs: + Tensor, has the same shape as `input_x`. All the elements in the tensor will be zero. + + Examples: + >>> alloc_status = NPUAllocFloatStatus() + >>> get_status = NPUGetFloatStatus() + >>> clear_status = NPUClearFloatStatus() + >>> init = alloc_status() + >>> flag = get_status(init) + >>> clear = clear_status(init) + Tensor([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], shape=(8,), dtype=mindspore.float32) + """ + + @prim_attr_register + def __init__(self): + """init NPUClearFloatStatus""" + self.add_prim_attr("_side_effect_flag", True) + + def infer_shape(self, x_shape): + validator.check_integer("len(x_shape)", len(x_shape), 1, Rel.EQ) + validator.check_integer("x_shape[0]", x_shape[0], 8, Rel.EQ) + return [8] + + def infer_dtype(self, x_dtype): + args = {"x_dtype": x_dtype} + validator.check_type_same(args, [mstype.float32]) + return mstype.float32 + + +class Cos(PrimitiveWithInfer): + """ + Computes cosine of input element-wise. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, has the same shape as `input_x`. + + Examples: + >>> cos = Cos() + >>> X = Tensor(np.array([0.24, 0.83, 0.31, 0.09]), ms.float32) + >>> output = cos(X) + """ + + @prim_attr_register + def __init__(self): + """init Cos""" + + def infer_shape(self, x): + return x + + def infer_dtype(self, x): + validator.check_subclass("x_dtype", x, mstype.tensor) + validator.check_typename('x_dtype', x, mstype.number_type) + return x + + +class ACos(PrimitiveWithInfer): + """ + Computes arccosine of input element-wise. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, has the same shape as `input_x`. + + Examples: + >>> acos = ACos() + >>> X = Tensor(np.array([0.74, 0.04, 0.30, 0.56]), ms.float32) + >>> output = acos(X) + """ + + @prim_attr_register + def __init__(self): + """init ACos""" + + def infer_shape(self, x): + return x + + def infer_dtype(self, x): + validator.check_subclass("x_dtype", x, mstype.tensor) + validator.check_typename('x_dtype', x, mstype.number_type) + return x + + +class Sin(PrimitiveWithInfer): + """ + Computes sine of input element-wise. + + Inputs: + - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, has the same shape as `input_x`. + + Examples: + >>> sin = Sin() + >>> X = Tensor(np.array([0.62, 0.28, 0.43, 0.62]), ms.float32) + >>> output = sin(X) + """ + + @prim_attr_register + def __init__(self): + """Init Sin.""" + + def infer_shape(self, x): + return x + + def infer_dtype(self, x): + validator.check_subclass("x_dtype", x, mstype.tensor) + validator.check_typename('x_dtype', x, mstype.number_type) + return x + + +class NMSWithMask(PrimitiveWithInfer): + """ + Select some bounding boxes in descending order of score. + + Args: + iou_threshold (float): Specifies the threshold of overlap boxes with respect to + IOU. Default: 0.5. + + Raises: + ValueError: If the iou_threshold is not a float number, or if the first dimension + of input Tensor is less than or equal to 0, or if the data type of the input + Tensor is not float16 or float32. + + Inputs: + - **bboxes** (Tensor) - The shape of tensor is :math:`(N, 5)`. Input bounding boxes. + `N` is the number of input bounding boxes. Every bounding box + contains 5 values, the first 4 values are the coordinates of bounding + box, and the last value is the score of this bounding box. + + Outputs: + tuple[Tensor], tuple of three tensors, they are selected_boxes, selected_idx and selected_mask. + + - **selected_boxes** (Tensor) - The shape of tensor is :math:`(N, 5)`. Bounding boxes + list after non-max suppression calculation. + - **selected_idx** (Tensor) - The shape of tensor is :math:`(N,)`. The indexes list of + valid input bounding boxes. + - **selected_mask** (Tensor) - The shape of tensor is :math:`(N,)`. A mask list of + valid output bounding boxes. + + Examples: + >>> bbox = np.random.rand(128, 5) + >>> bbox[:, 2] += bbox[:, 0] + >>> bbox[:, 3] += bbox[:, 1] + >>> inputs = Tensor(bbox) + >>> nms = NMSWithMask(0.5) + >>> output_boxes, indices, mask = nms(inputs) + """ + + @prim_attr_register + def __init__(self, iou_threshold=0.5): + """Init NMSWithMask""" + validator.check_type("iou_threshold", iou_threshold, [float]) + self.init_prim_io_names(inputs=['bboxes'], outputs=['selected_boxes', 'selected_idx', 'selected_mask']) + + def infer_shape(self, bboxes_shape): + num = bboxes_shape[0] + validator.check_integer("bboxes_shape[0]", num, 0, Rel.GT) + return (bboxes_shape, (num,), (num,)) + + def infer_dtype(self, bboxes_dtype): + validator.check_typename("bboxes_dtype", bboxes_dtype, [mstype.float16, mstype.float32]) + return (bboxes_dtype, mstype.int32, mstype.bool_) + + +class Abs(PrimitiveWithInfer): + """ + Returns absolute value of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. + + Outputs: + Tensor, has the same shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([-1.0, 1.0, 0.0]), mindspore.float32) + >>> abs = Abs() + >>> abs(input_x) + [1.0, 1.0, 0.0] + """ + + @prim_attr_register + def __init__(self): + """init Abs""" + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + validator.check_subclass("x_dtype", x_type, mstype.tensor) + validator.check_typename('x_dtype', x_type, mstype.number_type) + return x_type + + def infer_value(self, x): + if x is not None: + x = x.asnumpy() + out = np.abs(x, dtype=x.dtype) + return Tensor(out) + return None + + +class Sign(PrimitiveWithInfer): + r""" + Perform :math:`sign` on tensor element-wise. + + Note: + .. math:: + sign(x) = \begin{cases} -1, &if\ x < 0 \cr + 0, &if\ x == 0 \cr + 1, &if\ x > 0\end{cases} + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, has the same shape and type as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([[2.0, 0.0, -1.0]]), mindspore.float32) + >>> sign = Sign() + >>> output = sign(input_x) + [[1.0, 0.0, -1.0]] + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass('x', x_dtype, mstype.tensor) + validator.check_typename('x_dtype', x_dtype, mstype.number_type) + return x_dtype + + +class Round(PrimitiveWithInfer): + """ + Returns half to even of a tensor element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, has the same shape and type as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([0.8, 1.5, 2.3, 2.5, -4.5]), mindspore.float32) + >>> round = Round() + >>> round(input_x) + [1.0, 2.0, 2.0, 2.0, -4.0] + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, x_shape): + return x_shape + + def infer_dtype(self, x_type): + validator.check_subclass("x_dtype", x_type, mstype.tensor) + validator.check_typename('x_dtype', x_type, mstype.number_type) + return x_type diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py new file mode 100644 index 00000000000..afa4c7dfe38 --- /dev/null +++ b/mindspore/ops/operations/nn_ops.py @@ -0,0 +1,2247 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Operators for nn.""" + +import math +import operator +from functools import reduce + +import numpy as np + +from ... import context +from ..._checkparam import ParamValidator as validator +from ..._checkparam import Rel, check_bool, check_int_positive +from ...common import dtype as mstype +from ..primitive import Primitive, PrimitiveWithInfer, prim_attr_register + + +class Flatten(PrimitiveWithInfer): + r""" + Flattens a tensor without changing its batch size on the 0-th axis. + + Inputs: + - **input_x** (Tensor) - Tensor of shape :math:`(N, \ldots)` to be flattened. + + Outputs: + Tensor, the shape of the output tensor is :math:`(N, X)`, where :math:`X` is + the product of the remaining dimension. + + Examples: + >>> input_tensor = Tensor(np.ones(shape=[1, 2, 3, 4]), mindspore.float32) + >>> flatten = Flatten() + >>> output = flatten(input_tensor) + >>> assert output.shape() == (1, 24) + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, input_x): + validator.check('input_x rank', len(input_x), '', 1, Rel.GE) + prod = 1 if len(input_x) == 1 else reduce(operator.mul, input_x[1:]) + return input_x[0], prod + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + return input_x + + +class Softmax(PrimitiveWithInfer): + r""" + Softmax operation. + + Applies the Softmax operation to the input tensor on the specified axis. + Suppose a slice along the given aixs :math:`x` then for each element :math:`x_i` + the Softmax function is shown as follows: + + .. math:: + \text{output}(x_i) = \frac{exp(x_i)}{\sum_{j = 0}^{N-1}\exp(x_j)}, + + where :math:`N` is the length of the tensor. + + Args: + axis (Union[int, tuple]): The axis to do the Softmax operation. Default: -1. + + Inputs: + - **logits** (Tensor) - The input of Softmax. + + Outputs: + Tensor, with the same type and shape as the logits. + """ + + @prim_attr_register + def __init__(self, axis=-1): + self.init_prim_io_names(inputs=['x'], outputs=['output']) + validator.check_type("axis", axis, [int, tuple]) + if isinstance(axis, int): + self.add_prim_attr('axis', (axis,)) + for item in self.axis: + validator.check_type("item of axis", item, [int]) + + def infer_shape(self, logits): + validator.check_shape_length("axis shape", len(self.axis), 1, Rel.GE) + rank = len(logits) + for axis_v in self.axis: + validator.check_int_range("axis", axis_v, -rank, rank, Rel.INC_LEFT) + return logits + + def infer_dtype(self, logits): + validator.check_subclass("logits", logits, mstype.tensor) + return logits + + +class LogSoftmax(PrimitiveWithInfer): + r""" + Log Softmax activation function. + + Applies the Log Softmax function to the input tensor on the specified axis. + Suppose a slice along the given aixs :math:`x` then for each element :math:`x_i` + the Log Softmax function is shown as follows: + + .. math:: + \text{output}(x_i) = \log \left(\frac{exp(x_i)} {\sum_{j = 0}^{N-1}\exp(x_j)}\right), + + where :math:`N` is the length of the Tensor. + + Args: + axis (int): The axis to do the Log softmax operation. Default: -1. + + Inputs: + - **logits** (Tensor) - The input of Log Softmax. + + Outputs: + Tensor, with the same type and shape as the logits. + """ + + @prim_attr_register + def __init__(self, axis=-1): + validator.check_type("axis", axis, [int]) + + def infer_shape(self, logits): + rank = len(logits) + validator.check_int_range('axis', self.axis, -rank - 1, rank, Rel.INC_BOTH) + return logits + + def infer_dtype(self, logits): + validator.check_subclass("logits", logits, mstype.tensor) + return logits + + +class ReLU(PrimitiveWithInfer): + r""" + Computes ReLU(Rectified Linear Unit) of input tensor element-wise. + + It returns :math:`\max(x,\ 0)` element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, with the same type and shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]], np.float32)) + >>> relu = ReLU() + >>> result = relu(input_x) + [[0, 4.0, 0.0], [2.0, 0.0, 9.0]] + """ + + @prim_attr_register + def __init__(self): + """init ReLU""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + validator.check_typename("input_x", input_x, mstype.number_type) + return input_x + + +class ReLU6(PrimitiveWithInfer): + r""" + Computes ReLU(Rectified Linear Unit) upper bounded by 6 of input tensor element-wise. + + It returns :math:`\min(\max(0,x), 6)` element-wise. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, with the same type and shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]], np.float32)) + >>> relu6 = ReLU6() + >>> result = relu6(input_x) + >>> assert result.asnumpy() == Tensor(np.array([[0, 4.0, 0.0], [2.0, 0.0, 6.0]], np.float32)).asnumpy() + """ + + @prim_attr_register + def __init__(self): + """init ReLU6""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + validator.check_typename("input_x", input_x, (mstype.float16, mstype.float32)) + return input_x + + +class Elu(PrimitiveWithInfer): + """ + Computes exponential linear: `alpha * (exp(x) - 1)` if x < 0, `x` otherwise. + The data type of input tensor should be float. + + Args: + alpha (float): The coefficient of negative factor whose type is float. Default: 1.0. + + Inputs: + - **input_x** (Tensor) - The input tensor whose data type should be float. + + Outputs: + Tensor, has the same shape and data type as `input_x`. + + Examples: + >>> input_x = Tensor(np.array([[-1.0, 4.0, -8.0], [2.0, -5.0, 9.0]], np.float32)) + >>> elu = Elu() + >>> result = elu(input_x) + Tensor([[-0.632 4.0 -0.999] + [2.0 -0.993 9.0 ]], shape=(2, 3), dtype=ms.float32) + """ + + @prim_attr_register + def __init__(self, alpha=1.0): + """Init Elu""" + validator.check_type("alpha", alpha, [float]) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + validator.check_typename("input_x_dtype", input_x, mstype.float_type) + return input_x + + +class Sigmoid(PrimitiveWithInfer): + r""" + Sigmoid activation function. + + Computes Sigmoid of input element-wise. The Sigmoid function is defined as: + + .. math:: + \text{sigmoid}(x_i) = \frac{1}{1 + exp(-x_i)}, + + where :math:`x_i` is the element of the input. + + Inputs: + - **input_x** (Tensor) - The input of Sigmoid. + + Outputs: + Tensor, with the same type and shape as the input_x. + """ + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['x'], outputs=['output']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + validator.check_typename("input_x", input_x, (mstype.float16, mstype.float32)) + return input_x + + +class Tanh(PrimitiveWithInfer): + r""" + Tanh activation function. + + Computes hyperbolic tangent of input element-wise. The Tanh function is defined as: + + .. math:: + tanh(x_i) = \frac{\exp(x_i) - \exp(-x_i)}{\exp(x_i) + \exp(-x_i)} = \frac{\exp(2x_i) - 1}{\exp(2x_i) + 1}, + + where :math:`x_i` is an element of the input Tensor. + + Inputs: + - **input_x** (Tensor) - The input of Tanh. + + Outputs: + Tensor, with the same type and shape as the input_x. + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + return input_x + + +class FusedBatchNorm(Primitive): + r""" + FusedBatchNorm is a BatchNorm that moving mean and moving variance will be computed instead of being loaded. + + Batch Normalization is widely used in convolutional networks. This operation applies + Batch Normalization over input to avoid internal covariate shift as described in the + paper `Batch Normalization: Accelerating Deep Network Training by Reducing Internal + Covariate Shift `_. It rescales and recenters the + feature using a mini-batch of data and the learned parameters which can be described + in the following formula. + + .. math:: + y = \frac{x - mean}{\sqrt{variance + \epsilon}} * \gamma + \beta + + where :math:`\gamma` is scale, :math:`\beta` is bias, :math:`\epsilon` is epsilon. + + Args: + mode (int): Mode of batch normalization, value is 0 or 1. Default: 0. + epsilon (float): A small value added for numerical stability. Default: 1e-5. + momentum (float): The hyper parameter to compute moving average for running_mean and running_var + (e.g. :math:`new\_running\_mean = momentum * running\_mean + (1 - momentum) * current\_mean`). + Momentum value should be [0, 1]. Default: 0.9. + + Inputs: + - **input_x** (Tensor) - Tensor of shape :math:`(N, C)`. + - **scale** (Tensor) - Tensor of shape :math:`(C,)`. + - **bias** (Tensor) - Tensor of shape :math:`(C,)`. + - **mean** (Tensor) - Tensor of shape :math:`(C,)`. + - **variance** (Tensor) - Tensor of shape :math:`(C,)`. + + Outputs: + Tuple of 5 Tensor, the normalized input and the updated parameters. + + - **output_x** (Tensor) - The same type and shape as the `input_x`. + - **updated_scale** (Tensor) - Tensor of shape :math:`(C,)`. + - **updated_bias** (Tensor) - Tensor of shape :math:`(C,)`. + - **updated_moving_mean** (Tensor) - Tensor of shape :math:`(C,)`. + - **updated_moving_variance** (Tensor) - Tensor of shape :math:`(C,)`. + """ + + @prim_attr_register + def __init__(self, mode=0, epsilon=1e-5, momentum=0.1): + self.init_prim_io_names(inputs=['x', 'scale', 'b', 'mean', 'variance'], + outputs=['y', 'running_mean', 'running_variance', 'save_mean', 'save_inv_variance']) + self.mode = validator.check_integer('mode', mode, [0, 1], Rel.IN) + self.epsilon = validator.check_number_range('epsilon', epsilon, 0, 1, Rel.INC_RIGHT) + self.momentum = validator.check_number_range('momentum', momentum, 0, 1, Rel.INC_BOTH) + + +class BatchNorm(PrimitiveWithInfer): + r""" + Batch Normalization for input data and updated parameters. + + Batch Normalization is widely used in convolutional neural networks. This operation + applies Batch Normalization over input to avoid internal covariate shift as described + in the paper `Batch Normalization: Accelerating Deep Network Training by Reducing Internal + Covariate Shift `_. It rescales and recenters the + features using a mini-batch of data and the learned parameters which can be described + in the following formula, + + .. math:: + y = \frac{x - mean}{\sqrt{variance + \epsilon}} * \gamma + \beta + + where :math:`\gamma` is scale, :math:`\beta` is bias, :math:`\epsilon` is epsilon. + + Args: + is_training (bool): If `is_training` is True, `mean` and `variance` are computed during training. + If `is_training` is False, they're loaded from checkpoint during inference. Default: False. + epsilon (float): A small value added for numerical stability. Default: 1e-5. + + Inputs: + - **input_x** (Tensor) - Tensor of shape :math:`(N, C)`. + - **scale** (Tensor) - Tensor of shape :math:`(C,)`. + - **bias** (Tensor) - Tensor of shape :math:`(C,)`. + - **mean** (Tensor) - Tensor of shape :math:`(C,)`. + - **variance** (Tensor) - Tensor of shape :math:`(C,)`. + + Outputs: + Tuple of 5 Tensor, the normalized inputs and the updated parameters. + + - **output_x** (Tensor) - The same type and shape as the input_x. The shape is :math:`(N, C)`. + - **updated_scale** (Tensor) - Tensor of shape :math:`(C,)`. + - **updated_bias** (Tensor) - Tensor of shape :math:`(C,)`. + - **reserve_space_1** (Tensor) - Tensor of shape :math:`(C,)`. + - **reserve_space_2** (Tensor) - Tensor of shape :math:`(C,)`. + - **reserve_space_3** (Tensor) - Tensor of shape :math:`(C,)`. + """ + + @prim_attr_register + def __init__(self, is_training=False, epsilon=1e-5): + self.is_training = validator.check_type('is_training', is_training, (bool,)) + self.epsilon = validator.check_number_range('epsilon', epsilon, 0, 1, Rel.INC_RIGHT) + self.add_prim_attr('data_format', "NCHW") + self.init_prim_io_names(inputs=['x', 'scale', 'offset', 'mean', 'variance'], + outputs=['y', 'batch_mean', 'batch_variance', 'reserve_space_1', 'reserve_space_2', + 'reserve_space_3']) + + def infer_shape(self, input_x, scale, bias, mean, variance): + validator.check("BatchNorm scale shape length", len(scale), "1", 1, Rel.EQ) + validator.check("BatchNorm scale shape", scale, "BatchNorm bias shape", bias) + validator.check("BatchNorm scale shape", scale[0], "BatchNorm input_x shape[1]", input_x[1]) + if not self.is_training: + validator.check("BatchNorm mean shape length", len(mean), "1", 1, Rel.EQ) + validator.check("BatchNorm mean shape", mean, "BatchNorm variance shape", variance) + validator.check("BatchNorm mean shape", mean, "BatchNorm scale shape", scale) + return (input_x, scale, scale, scale, scale, scale) + + def infer_dtype(self, input_x, scale, bias, mean, variance): + args = {"BatchNorm scale type": scale, "BatchNorm bias type": bias} + args_moving = {"BatchNorm mean type": mean, "BatchNorm variance type": variance} + validator.check_typename("input_x", input_x, [mstype.float32, mstype.float16]) + validator.check_type_same(args, [mstype.float32, mstype.float16]) + if self.is_training: + validator.check_type_same(args_moving, [mstype.float32, mstype.float16, None]) + else: + validator.check_type_same(args_moving, [mstype.float32, mstype.float16]) + return (input_x, scale, bias, input_x, input_x, input_x) + + +class Conv2D(PrimitiveWithInfer): + r""" + 2D convolution layer. + + Applies a 2D convolution over an input tensor which is typically of shape :math:`(N, C_{in}, H_{in}, W_{in})`, + where :math:`N` is batch size and :math:`C_{in}` is channel number. For each batch of shape + :math:`(C_{in}, H_{in}, W_{in})`, the formula is defined as: + + .. math:: + + out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{ij}, X_i) + b_j, + + where :math:`ccor` is cross correlation operator, :math:`C_{in}` is the input channel number, :math:`j` ranges + from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to :math:`i`-th channel of the :math:`j`-th + filter and :math:`out_{j}` corresponds to the :math:`j`-th channel of the output. :math:`W_{ij}` is a slice + of kernel and it has shape :math:`(\text{ks_h}, \text{ks_w})`, where :math:`\text{ks_h}` and + :math:`\text{ks_w}` are height and width of the convolution kernel. The full kernel has shape + :math:`(C_{out}, C_{in} // \text{group}, \text{ks_h}, \text{ks_w})`, where group is the group number + to split the input in the channel dimension. + + If the 'pad_mode' is set to be "valid", the output height and width will be + :math:`\left \lfloor{1 + \frac{H_{in} + 2 \times \text{padding} - \text{ks_h} - + (\text{ks_h} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` and + :math:`\left \lfloor{1 + \frac{W_{in} + 2 \times \text{padding} - \text{ks_w} - + (\text{ks_w} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` respectively. + + + The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition + `_. More detailed introduction can be found here: + http://cs231n.github.io/convolutional-networks/. + + Args: + out_channel (int): The dimension of the output. + kernel_size (Union[int, tuple[int]]): The kernel size of the 2D convolution. + mode (int): 0 Math convolutiuon, 1 cross-correlation convolution , + 2 deconvolution, 3 depthwise convolution. Default: 1. + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + pad (int): The pad value to fill. Default: 0. + stride (int): The stride to apply conv filter. Default: 1. + dilation (int): Specify the space to use between kernel elements. Default: 1. + group (int): Split input into groups. Default: 1. + + Returns: + Tensor, the value that applied 2D convolution. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + - **weight** (Tensor) - Set size of kernel is :math:`(K_1, K_2)`, then the shape is + :math:`(C_{out}, C_{in}, K_1, K_2)`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + """ + + @prim_attr_register + def __init__(self, + out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1): + """init Conv2D""" + self.init_prim_io_names(inputs=['x', 'w'], outputs=['output']) + self.kernel_size = kernel_size + self.kernel_size = validator.check_type('kernel_size', kernel_size, (int, tuple)) + if isinstance(self.kernel_size, int): + self.kernel_size = (self.kernel_size, self.kernel_size) + validator.check_integer('length of kernel_size', len(self.kernel_size), 2, Rel.GE) + validator.equal('type of pad', type(pad), 'not bool', not isinstance(pad, bool)) + validator.equal('type of pad', type(pad), 'int', isinstance(pad, int)) + self.pad_mode = validator.check_string('pad_mode', pad_mode, ['valid', 'same', 'pad']) + self.pad = validator.check_pad_value_by_mode(self.__class__.__name__, pad_mode, pad) + if self.pad_mode == 'pad': + validator.check_integer('pad', self.pad, 0, Rel.GE) + + self.mode = validator.check_integer('mode', mode, 1, Rel.EQ) + self.add_prim_attr('data_format', "NCHW") + self.out_channel = validator.check_integer('out_channel', out_channel, 0, Rel.GT) + self.group = validator.check_integer('group', group, 0, Rel.GT) + self.dilation = validator.check_integer('dilation', dilation, 1, Rel.GE) + validator.check_type('kernel_size', kernel_size, [int, tuple]) + if isinstance(kernel_size, int) and kernel_size < 1: + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') + if isinstance(kernel_size, tuple) and (len(kernel_size) != 2 or + (not isinstance(kernel_size[0], int)) or + (not isinstance(kernel_size[1], int)) or + kernel_size[0] < 1 or kernel_size[1] < 1): + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') + self.stride = validator.check_integer('stride', stride, 1, Rel.GE) + + def infer_shape(self, x_shape, w_shape): + validator.check_integer("weight_shape", len(w_shape), 4, Rel.EQ) + validator.check_integer("x_shape", len(x_shape), 4, Rel.EQ) + validator.check_param_equal("x_shape[1]", x_shape[1] // self.group, "w_shape[1]", w_shape[1]) + validator.check_param_equal('out_channel', self.out_channel, 'w_shape[0]', w_shape[0]) + validator.check_param_equal('kernel_size', self.kernel_size, 'w_shape[2:4]', tuple(w_shape[2:4])) + + kernel_size_h = w_shape[2] + kernel_size_w = w_shape[3] + + if self.pad_mode == "valid": + h_out = math.ceil((x_shape[2] - self.dilation * (kernel_size_h - 1)) / self.stride) + w_out = math.ceil((x_shape[3] - self.dilation * (kernel_size_w - 1)) / self.stride) + pad_top, pad_bottom, pad_left, pad_right = 0, 0, 0, 0 + elif self.pad_mode == "same": + h_out = math.ceil(x_shape[2] / self.stride) + w_out = math.ceil(x_shape[3] / self.stride) + + pad_needed_h = max(0, (h_out - 1) * self.stride + self.dilation * (kernel_size_h - 1) + 1 - x_shape[2]) + pad_top = math.floor(pad_needed_h / 2) + pad_bottom = pad_needed_h - pad_top + + pad_needed_w = max(0, (w_out - 1) * self.stride + self.dilation * (kernel_size_w - 1) + 1 - x_shape[3]) + pad_left = math.floor(pad_needed_w / 2) + pad_right = pad_needed_w - pad_left + elif self.pad_mode == 'pad': + pad_top, pad_bottom, pad_left, pad_right = self.pad, self.pad, self.pad, self.pad + + h_out = 1 + (x_shape[2] + 2 * self.pad - kernel_size_h - (kernel_size_h - 1) * (self.dilation - 1)) \ + / self.stride + w_out = 1 + (x_shape[3] + 2 * self.pad - kernel_size_w - (kernel_size_w - 1) * (self.dilation - 1)) \ + / self.stride + h_out = math.floor(h_out) + w_out = math.floor(w_out) + + self.pad_list = [pad_top, pad_bottom, pad_left, pad_right] + self.add_prim_attr('pad_list', (pad_top, pad_bottom, pad_left, pad_right)) + + out_channel = self.out_channel + out_shape = [x_shape[0], out_channel, h_out, w_out] + return out_shape + + def infer_dtype(self, x_dtype, w_dtype): + args = {'x_dtype': x_dtype, 'w_dtype': w_dtype} + validator.check_subclass('input', x_dtype, mstype.tensor) + validator.check_subclass('weight', w_dtype, mstype.tensor) + validator.check_type_same(args, [mstype.int8, mstype.int32, mstype.float16, mstype.float32]) + return x_dtype + + +class DepthwiseConv2dNative(PrimitiveWithInfer): + r""" + Returns the depth-wise convolution value for the input. + + Applies depthwise conv2d for the input, which will generate more channels with channel_multiplier. + Given an input tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})` where :math:`N` is the batch size and a + filter tensor with kernel size :math:`(ks_{h}, ks_{w})`, containing :math:`C_{in} * \text{channel_multiplier}` + convolutional filters of depth 1; it applies different filters to each input channel (channel_multiplier channels + for each with default value 1), then concatenates the results together. The output has + :math:`\text{in_channels} * \text{channel_multiplier}` channels. + + Args: + channel_multiplier (int): The multipiler for the original output conv. + kernel_size (int or tuple): The size of the conv kernel. + mode (int): 0 Math convolution, 1 cross-correlation convolution , + 2 deconvolution, 3 depthwise convolution. Default: 3. + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + pad (int): The pad value to fill. Default: 0. + stride (int): The stride to apply conv filter. Default: 1. + dilation (int): Specifies the dilation rate to use for dilated convolution. Default: 1. + group (int): Splits input into groups. Default: 1. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + - **weight** (Tensor) - Set size of kernel is :math:`(K_1, K_2)`, then the shape is + :math:`(C_{out}, C_{in}, K_1, K_2)`. + + Outputs: + Tensor of shape :math:`(N, C_{in} * \text{channel_multiplier}, H_{out}, W_{out})`. + """ + + @prim_attr_register + def __init__(self, + channel_multiplier, + kernel_size, + mode=3, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1): + """init DepthwiseConv2dNative""" + validator.check_pad_value_by_mode(self.__class__.__name__, pad_mode, pad) + validator.check_type("kernel_size", kernel_size, (int, tuple)) + if isinstance(kernel_size, int): + kernel_size = (kernel_size, kernel_size) + if isinstance(kernel_size, tuple) and (len(kernel_size) != 2 or + (not isinstance(kernel_size[0], int)) or + (not isinstance(kernel_size[1], int)) or + kernel_size[0] < 1 or kernel_size[1] < 1): + raise ValueError(f"Attr kernel_size of DepthwiseConv2dNative Op not passed " + f"{kernel_size}, should be a int or tuple and equal to or greater than 1.") + if pad_mode not in ("same", "valid", "pad"): + raise ValueError(f"Attr pad_mode of DepthwiseConv2dNative Op not passed" + f"{pad_mode} not in valid, same, pad.") + self.pad_mode = pad_mode + self.mode = validator.check_integer("mode", mode, 3, Rel.EQ) + self.add_prim_attr('data_format', "NCHW") + self.channel_multiplier = validator.check_integer("channel_multiplier", channel_multiplier, 0, Rel.GT) + self.group = validator.check_integer("group", group, 0, Rel.GT) + self.dilation = validator.check_integer("dilation", dilation, 1, Rel.GE) + self.kernel_size = validator.check_value_on_integer("kernel_size", kernel_size, 1, Rel.GE) + self.stride = validator.check_integer("stride", stride, 1, Rel.GE) + self.pad = pad + + def infer_shape(self, x_shape, w_shape): + validator.check_integer("weight_shape", len(w_shape), 4, Rel.EQ) + validator.check_integer("x_shape", len(x_shape), 4, Rel.EQ) + validator.check_param_equal("x_shape[1]", x_shape[1], "w_shape[1]", w_shape[1]) + validator.check_param_equal('kernel_size', self.kernel_size, 'w_shape[2:4]', tuple(w_shape[2:4])) + + kernel_size_h = w_shape[2] + kernel_size_w = w_shape[3] + + if self.pad_mode == "valid": + h_out = math.ceil((x_shape[2] - self.dilation * (kernel_size_h - 1)) / self.stride) + w_out = math.ceil((x_shape[3] - self.dilation * (kernel_size_w - 1)) / self.stride) + pad_top, pad_bottom, pad_left, pad_right = 0, 0, 0, 0 + elif self.pad_mode == "same": + h_out = math.ceil(x_shape[2] / self.stride) + w_out = math.ceil(x_shape[3] / self.stride) + + pad_needed_h = max(0, (h_out - 1) * self.stride + self.dilation * (kernel_size_h - 1) + 1 - x_shape[2]) + pad_top = math.floor(pad_needed_h / 2) + pad_bottom = pad_needed_h - pad_top + + pad_needed_w = max(0, (w_out - 1) * self.stride + self.dilation * (kernel_size_w - 1) + 1 - x_shape[3]) + pad_left = math.floor(pad_needed_w / 2) + pad_right = pad_needed_w - pad_left + elif self.pad_mode == 'pad': + pad_top, pad_bottom, pad_left, pad_right = self.pad, self.pad, self.pad, self.pad + + h_out = 1 + (x_shape[2] + 2 * self.pad - kernel_size_h - (kernel_size_h - 1) * (self.dilation - 1)) \ + / self.stride + w_out = 1 + (x_shape[3] + 2 * self.pad - kernel_size_w - (kernel_size_w - 1) * (self.dilation - 1)) \ + / self.stride + h_out = math.floor(h_out) + w_out = math.floor(w_out) + else: + raise ValueError(f"Attr pad_mode of DepthwiseConv2dNative Op not passed" + "{pad_mode} not in valid, same, pad.") + + self.pad_list = (pad_top, pad_bottom, pad_left, pad_right) + self.add_prim_attr('pads', self.pad_list) + + out_channel = self.channel_multiplier * x_shape[1] + out_shape = [x_shape[0], out_channel, h_out, w_out] + return out_shape + + def infer_dtype(self, x_dtype, w_dtype): + args = {'x_dtype': x_dtype, 'w_dtype': w_dtype} + validator.check_type_same(args, mstype.number_type) + return x_dtype + + +class MaxPoolWithArgmax(PrimitiveWithInfer): + r""" + Performs max pooling on the input Tensor and return both max values and indices. + + Typically the input is of shape :math:`(N_{in}, C_{in}, H_{in}, W_{in})`, MaxPool outputs + regional maximum in the :math:`(H_{in}, W_{in})`-dimension. Given kernel size + :math:`ks = (h_{ker}, w_{ker})` and stride :math:`s = (s_0, s_1)`, the operation is as follows. + + .. math:: + \text{output}(N_i, C_j, h, w) = \max_{m=0, \ldots, h_{ker}-1} \max_{n=0, \ldots, w_{ker}-1} + \text{input}(N_i, C_j, s_0 \times h + m, s_1 \times w + n) + + Args: + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + window (Union[int, tuple[int]]): The size of window, which is the kernel size, two `int` for width + and height. Default: 1. + pad (Union[int, tuple[int]]): If `pad_mode` is `pad`, the pad value to fill, two `int` for width + and height. Default: 0. + stride (Union[int, tuple[int]]): The stride of the window, that should be a tuple of two `int` for + width and height. Default: 1. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tuple of 2 Tensor, the maxpool result and where max values from. + + - **output** (Tensor) - Maxpooling result, with shape :math:`(N, C_{out}, H_{out}, W_{out})`. + - **mask** (Tensor) - Max values' index represented by the mask. + """ + + @prim_attr_register + def __init__(self, + pad_mode="valid", + window=1, + pad=0, + stride=1, + data_mode=1, + ceil_mode=0, + alpha=1.0, + beta=0.0): + self.init_prim_io_names(inputs=['x'], outputs=['output', 'argmax']) + self.window = validator.check_type('window', window, [int, tuple]) + if isinstance(window, int) and window <= 0: + raise ValueError('Attr \'window\' of \'MaxPoolWithArgmax\' Op passed ' + + str(self.window)+', should be a int or tuple and greater than 0.') + if isinstance(window, tuple) and (len(window) != 2 or + (not isinstance(window[0], int)) or + (not isinstance(window[1], int)) or + window[0] <= 0 or window[1] <= 0): + raise ValueError('Attr \'window\' of \'MaxPoolWithArgmax\' Op passed ' + + str(self.window)+', should be a int or tuple and greater than 0.') + self.pool_h = self.pool_w = window + self.pad_mode = validator.check_string('pad_mode', pad_mode, ['valid', 'same', 'pad']) + if self.pad_mode == "valid": + self.pad = 0 + elif self.pad_mode == "same": + self.pad = math.floor((self.window - 1) / 2) + elif self.pad_mode == "pad": + self.pad = validator.check_integer('pad', pad, 0, Rel.GE) + + self.data_mode = validator.check_integer('data_mode', data_mode, 1, Rel.EQ) + self.ceil_mode = validator.check_integer('ceil_mode', ceil_mode, 0, Rel.EQ) + self.stride = validator.check_integer('stride', stride, 1, Rel.GE) + self.alpha = validator.check_type('alpha', alpha, [int, float]) + self.beta = validator.check_type('beta', beta, [int, float]) + self.is_tbe = not context.get_context("enable_ge") and context.get_context("device_target") == "Ascend" + + def infer_shape(self, x_shape): + validator.check_integer("x_shape", len(x_shape), 4, Rel.EQ) + pad = self.pad + h_input = x_shape[2] + w_input = x_shape[3] + h_out = (h_input + 2 * pad - (self.window - 1) - 1) / self.stride + 1 + h_out = math.floor(h_out) + w_out = (w_input + 2 * pad - (self.window - 1) - 1) / self.stride + 1 + w_out = math.floor(w_out) + out_shape = [x_shape[0], x_shape[1], h_out, w_out] + for shape_value in out_shape: + if shape_value <= 0: + raise ValueError("The kernel size is not valid please check it if is larger than data's shape size.") + k_size_vec = [1, self.window, self.window, 1] + argmax_shape = [] + if self.is_tbe: + for i in range(4): + if i == 2: + dim = k_size_vec[i - 1] * k_size_vec[i] + argmax_shape.append(dim) + elif i == 3: + dim = math.ceil(out_shape[i - 1] * out_shape[i] / 16) + 1 + argmax_shape.append(dim) + else: + argmax_shape.append(x_shape[i]) + else: + argmax_shape = out_shape + return out_shape, argmax_shape + + def infer_dtype(self, x_dtype): + out_dtype = x_dtype + validator.check_typename("x_type", x_dtype, (mstype.float16, mstype.float32)) + argmax_dtype = mstype.int32 + return out_dtype, argmax_dtype + + +class _Pool(PrimitiveWithInfer): + r""" + Performs max/avg pooling operation. + + Args: + ksize (Union[int, tuple[int]]): The size of the window to take a max over, that should be a tuple + of two `int` for width and height. Default: 1. + stride (Union[int, tuple[int]]): The stride of the window, that should be a tuple of two `int` for + width and height. Default: 1. + padding (str): The optional values for pad mode "SAME", "VALID". Default: "VALID". + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + self.init_prim_io_names(inputs=['x'], outputs=['output']) + validator.check_type('padding', padding, [str]) + self.ksize = ksize + self.strides = strides + self.padding = padding.upper() + self.ksize = validator.check_type('ksize', self.ksize, [int, tuple]) + self.strides = validator.check_type('strides', self.strides, [int, tuple]) + self.padding = validator.check_string('padding', self.padding, ['VALID', 'SAME']) + self.init_prim_io_names(inputs=['x'], outputs=['output']) + self.add_prim_attr("padding", self.padding) + self.add_prim_attr('data_format', "NCHW") + + if isinstance(self.ksize, int): + self.pool_h = validator.check_integer("ksize", self.ksize, 1, Rel.GE) + self.pool_w = self.pool_h + self.add_prim_attr("ksize", (1, 1, self.ksize, self.ksize)) + elif isinstance(self.ksize, tuple): + if (len(self.ksize) != 2 or (not isinstance(self.ksize[0], int)) or (not isinstance(self.ksize[1], int)) + or self.ksize[0] <= 0 or self.ksize[1] <= 0): + raise ValueError('Each value of attr \'ksize\' of \'MaxPool\' Op passed ' + + str(self.ksize) + ', should be a int or a tuple of length 2 and greater than 0.') + self.pool_h = self.ksize[0] + self.pool_w = self.ksize[1] + self.add_prim_attr("ksize", (1, 1, self.ksize[0], self.ksize[1])) + + if isinstance(self.strides, int): + self.stride_h = validator.check_integer("strides", self.strides, 1, Rel.GE) + self.stride_w = self.stride_h + self.add_prim_attr("strides", (1, 1, self.strides, self.strides)) + elif isinstance(self.strides, tuple): + if (len(self.strides) != 2 or (not isinstance(self.strides[0], int)) or + (not isinstance(self.strides[1], int)) or self.strides[0] <= 0 or self.strides[1] <= 0): + raise ValueError('Each value of attr \'strides\' of \'MaxPool\' Op passed ' + + str(self.strides) + ', should be a int or a tuple of length 2 and greater than 0.') + self.stride_h = self.strides[0] + self.stride_w = self.strides[1] + self.add_prim_attr("strides", (1, 1, self.strides[0], self.strides[1])) + + if self.padding == "VALID": + self.pad = 0 + elif self.padding == "SAME": + self.pad = math.floor((self.pool_h - 1) / 2) + else: + raise ValueError('The padding should be str and must be SAME or VALID,' + ' but got {}.'.format(self.padding)) + self.add_prim_attr('pad', self.pad) + + def infer_shape(self, x_shape): + validator.check_integer("x_shape", len(x_shape), 4, Rel.EQ) + h_input = x_shape[2] + w_input = x_shape[3] + if self.padding == "VALID": + h_out = math.ceil((h_input - (self.pool_h - 1)) / self.stride_h) + w_out = math.ceil((w_input - (self.pool_w - 1)) / self.stride_w) + elif self.padding == "SAME": + h_out = math.ceil(h_input / self.stride_h) + w_out = math.ceil(w_input / self.stride_w) + else: + raise ValueError('The padding should be str and must be SAME or VALID,' + ' but got {}.'.format(self.padding)) + + out_shape = [x_shape[0], x_shape[1], h_out, w_out] + for shape_value in out_shape: + if shape_value <= 0: + raise ValueError("The kernel size is not valid please check it if is larger than data's shape size.") + return out_shape + + def infer_dtype(self, x_dtype): + validator.check_subclass("input", x_dtype, mstype.tensor) + return x_dtype + + +class MaxPool(_Pool): + r""" + Max pooling operation. + + Applies a 2D max pooling over an input Tensor which can be regarded as a composition of 2D planes. + + Typically the input is of shape :math:`(N_{in}, C_{in}, H_{in}, W_{in})`, MaxPool outputs + regional maximum in the :math:`(H_{in}, W_{in})`-dimension. Given kernel size + :math:`ks = (h_{ker}, w_{ker})` and stride :math:`s = (s_0, s_1)`, the operation is as follows. + + .. math:: + \text{output}(N_i, C_j, h, w) = \max_{m=0, \ldots, h_{ker}-1} \max_{n=0, \ldots, w_{ker}-1} + \text{input}(N_i, C_j, s_0 \times h + m, s_1 \times w + n) + + Args: + ksize (Union[int, tuple[int]]): The size of the window to take a max over, that should be a tuple + of two `int` for width and height. Default: 1. + stride (Union[int, tuple[int]]): The stride of the window, that should be a tuple of two `int` for + width and height. Default: 1. + padding (str): The optional values for pad mode "SAME", "VALID". Default: "VALID". + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor, with shape :math:`(N, C_{out}, H_{out}, W_{out})`. + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + super(MaxPool, self).__init__(ksize, strides, padding) + + +class AvgPool(_Pool): + r""" + Average pooling operation. + + Applies a 2D average pooling over an input Tensor which can be regarded as a composition of 2D input planes. + Typically the input is of shape :math:`(N_{in}, C_{in}, H_{in}, W_{in})`, AvgPool2d outputs + regional average in the :math:`(H_{in}, W_{in})`-dimension. Given kernel size + :math:`ks = (h_{ker}, w_{ker})` and stride :math:`s = (s_0, s_1)`, the operation is as follows. + + .. math:: + \text{output}(N_i, C_j, h, w) = \frac{1}{h_{ker} * w_{ker}} \sum_{m=0}^{h_{ker}-1} \sum_{n=0}^{w_{ker}-1} + \text{input}(N_i, C_j, s_0 \times h + m, s_1 \times w + n) + + Args: + ksize (Union[int, tuple[int]]): The size of the window to take a average over, that should be a tuple + of two `int` for width and height. Default: 1. + stride (Union[int, tuple[int]]): The stride of the window, that should be a tuple of two `int` for + width and height. Default: 1. + padding (str): The optional values for pad mode "SAME", "VALID". Default: "VALID". + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor, with shape :math:`(N, C_{out}, H_{out}, W_{out})`. + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + if context.get_context("device_target") == "GPU": + self.target = "GPU" + else: + self.target = "OTHER" + super(AvgPool, self).__init__(ksize, strides, padding) + + +class Conv2DBackpropInput(PrimitiveWithInfer): + """ + Computes the gradients of convolution with respect to the input. + + Args: + out_channel (int): The dimensionality of the output space. + kernel_size (Union[int, tuple[int]]): The size of the convolution window. + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + pad (int): The pad value to fill. Default: 0. + mode (int): 0 Math convolutiuon, 1 cross-correlation convolution , + 2 deconvolution, 3 depthwise convolution. Default: 1. + stride (int): The stride to apply conv filter. Default: 1. + dilation (int): Specifies the dilation rate to use for dilated convolution. Default: 1. + group (int): Splits input into groups. Default: 1. + + Returns: + Tensor, the gradients of convolution. + """ + + @prim_attr_register + def __init__(self, + out_channel, + kernel_size, + pad_mode="valid", + pad=0, + pad_list=None, + mode=1, + stride=1, + dilation=1, + group=1): + """init Conv2DBackpropInput""" + self.init_prim_io_names(inputs=['out_backprop', 'filter', 'input_sizes'], outputs=['output']) + self.out_channel = validator.check_integer('out_channel', out_channel, 0, Rel.GT) + self.kernel_size = validator.check_type('kernel_size', kernel_size, (int, tuple)) + if isinstance(self.kernel_size, int): + if kernel_size < 1: + raise ValueError('Attr \'kernel_size\' of \'Conv2DBackpropInput\' Op passed ' + + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') + self.kernel_size = (self.kernel_size, self.kernel_size) + elif isinstance(kernel_size, tuple) and (len(kernel_size) != 2 or + (not isinstance(kernel_size[0], int)) or + (not isinstance(kernel_size[1], int)) or + kernel_size[0] < 1 or kernel_size[1] < 1): + raise ValueError('Attr \'kernel_size\' of \'Conv2DBackpropInput\' Op passed ' + + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') + validator.equal('type of pad', type(pad), 'not bool', not isinstance(pad, bool)) + validator.equal('type of pad', type(pad), 'int', isinstance(pad, int)) + self.pad_mode = validator.check_string('pad_mode', pad_mode, ['valid', 'same', 'pad']) + self.pad = validator.check_pad_value_by_mode(self.__class__.__name__, pad_mode, pad) + self.mode = validator.check_integer('mode', mode, 1, Rel.EQ) + self.group = validator.check_integer('group', group, 0, Rel.GT) + self.dilation = validator.check_integer('dilation', dilation, 1, Rel.GE) + self.stride = validator.check_integer('stride', stride, 1, Rel.GE) + pad_mode = pad_mode.upper() + self.add_prim_attr('pad_mode', pad_mode) + self.add_prim_attr('data_format', "NCHW") + if pad_list: + self.pad_lsit = (validator.check_integer('pad_list', x, 0, Rel.GE) for x in pad_list) + + def __infer__(self, doutput, w, x_size): + x_size_v = x_size['value'] + validator.check_type('x_size', x_size_v, [tuple]) + for i, dim_len in enumerate(x_size_v): + validator.check_type("x_size[%d]" % i, dim_len, [int]) + validator.check_typename('w_dtype', w['dtype'], [mstype.int8, mstype.int32, mstype.float16, mstype.float32]) + validator.check_two_types_same('doutput_dtype', doutput['dtype'], 'w_dtype', w['dtype']) + + # infer shape + dout_shape = doutput['shape'] + kernel_h = self.kernel_size[0] + kernel_w = self.kernel_size[1] + # default pad mode is valid + pad_list = (0, 0, 0, 0) + if self.pad_list: + pad_list = tuple(self.pad_list) + elif self.pad_mode == "SAME": + pad_needed_h = max(0, (dout_shape[2] - 1) * self.stride + kernel_h - x_size_v[2]) + pad_top = math.floor(pad_needed_h / 2) + pad_bottom = pad_needed_h - pad_top + + pad_needed_w = max(0, (dout_shape[3] - 1) * self.stride + kernel_w - x_size_v[3]) + pad_left = math.floor(pad_needed_w / 2) + pad_right = pad_needed_w - pad_left + pad_list = (pad_top, pad_bottom, pad_left, pad_right) + elif self.pad_mode == 'PAD': + pad_list = (self.pad,) * 4 + self.add_prim_attr('pad_list', pad_list) + out = { + 'value': None, + 'shape': x_size_v, + 'dtype': doutput['dtype'], + } + return out + + +class BiasAdd(PrimitiveWithInfer): + r""" + Returns sum of input and bias tensor. + + Adds the 1-D bias tensor to the input tensor, and boardcasts the shape on all axis + except for the channel axis. + + Inputs: + - **input_x** (Tensor) - Input value, with shape :math:`(N, C)` or :math:`(N, C, H, W)`. + - **bias** (Tensor) - Bias value, with shape :math:`(C)`. + + Outputs: + Tensor, with the same shape and type as `input_x`. + """ + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['x', 'b'], outputs=['output']) + self.add_prim_attr('data_format', 'NCHW') + + def infer_shape(self, x_shape, b_shape): + if len(b_shape) != 1 or len(x_shape) < 2 or b_shape[0] != x_shape[1]: + raise ValueError("Input_x and bias shapes do not match", + "(require: rank of input_x must be at least 2, rank of bias must be 1, " + "input_x.dim[1] must equal bias.dim[0])," + " but got input_x shape {}, bias shape {}.".format(x_shape, b_shape)) + return x_shape + + def infer_dtype(self, x_type, b_type): + args = {"input_x type": x_type, "bias type": b_type} + validator.check_type_same(args, (mstype.float16, mstype.float32, mstype.int8, mstype.int32)) + return x_type + + +class TopK(PrimitiveWithInfer): + """ + Finds values and indices of the `k` largest entries along the last dimension. + + Args: + sorted (bool): If true, the resulting elements will + be sorted by the values in descending order. Default: False. + + Inputs: + - **input_x** (Tensor) - Input to be computed. + - **k** (int) - Number of top elements to be computed along the last dimension, constant input is needed. + + Outputs: + Tuple of 2 Tensor, the values and the indices. + + - **values** (Tensor) - The `k` largest elements along each last dimensional slice. + - **indices** (Tensor) - The indices of values within the last dimension of input. + + Examples: + >>> topk = TopK(sorted=True) + >>> x = Tensor(np.array([1, 2, 3, 4, 5]).astype(np.float16)) + >>> values, indices = topk(x) + >>> assert values == Tensor(np.array([5, 4, 3])) + >>> assert indices == Tensor(np.array([4, 3, 2])) + """ + + @prim_attr_register + def __init__(self, sorted=False): + validator.check_type("sorted", sorted, [bool]) + self.init_prim_io_names(inputs=['input', 'k'], + outputs=['values', 'indices']) + + def __infer__(self, input_x, k): + x_shape = list(input_x['shape']) + ndim = len(x_shape) - 1 + k_v = k['value'] + x_shape[ndim] = k_v + input_dtype = input_x['dtype'] + validator.check_typename("TopK input_dtype", + input_dtype, (mstype.float16, mstype.float32, mstype.int32)) + if not isinstance(k_v, int): + raise ValueError('The k must int.', k) + return {'shape': (x_shape, x_shape), + 'dtype': (input_dtype, mstype.int32), + 'value': None} + + +class SoftmaxCrossEntropyWithLogits(PrimitiveWithInfer): + r""" + Gets the softmax cross-entropy value between logits and labels which shoule be one-hot encoding. + + Note: + Sets input logits as `X`, input label as `Y`, output as `loss`. Then, + + .. math:: + p_{ij} = softmax(X_{ij}) = \frac{exp(x_i)}{\sum_{j = 0}^{N-1}\exp(x_j)} + + .. math:: + loss_{ij} = -\sum_j{Y_{ij} * ln(p_{ij})} + + Inputs: + - **logits** (Tensor) - Input logits, with shape :math:`(N, C)`. + - **labels** (Tensor) - Ground truth labels, with shape :math:`(N, C)`. + + Outputs: + Tuple of 2 Tensor, the loss shape is `(N,)`, and the dlogits with the same shape as `logits`. + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, logits_shape, labels_shape): + validator.check_param_equal("SoftmaxCrossEntropyWithLogits logits_shape", logits_shape, + "SoftmaxCrossEntropyWithLogits labels_shape", labels_shape) + loss_shape = [logits_shape[0]] + dlogits_shape = logits_shape + return (loss_shape, dlogits_shape) + + def infer_dtype(self, logits_type, labels_type): + args = {"SoftmaxCrossEntropyWithLogits logits_type": logits_type, + "SoftmaxCrossEntropyWithLogits labels_type": labels_type} + validator.check_type_same(args, (mstype.float16, mstype.float32)) + return (logits_type, logits_type) + + +class SparseSoftmaxCrossEntropyWithLogits(PrimitiveWithInfer): + r""" + Computes the softmax cross-entropy value between logits and sparse encoding labels. + + Note: + Sets input logits as `X`, input label as `Y`, output as `loss`. Then, + + .. math:: + p_{ij} = softmax(X_{ij}) = \frac{exp(x_i)}{\sum_{j = 0}^{N-1}\exp(x_j)} + + .. math:: + loss_{ij} = \begin{cases} -ln(p_{ij}), &j = y_i \cr -ln(1 - p_{ij}), & j \neq y_i \end{cases} + + .. math:: + loss = \sum_{ij} loss_{ij} + + Args: + is_grad (bool): If it's true, this operation returns the computed gradient. Default: False. + + Inputs: + - **logits** (Tensor) - Input logits, with shape :math:`(N, C)`. + - **labels** (Tensor) - Ground truth labels, with shape :math:`(N)`. + + Outputs: + Tensor, if `is_grad` is False, the output tensor is the value of loss which is a scalar tensor; + if `is_grad` is True, the output tensor is the gradient of input with the same shape as `logits`. + """ + + @prim_attr_register + def __init__(self, is_grad=False): + self.init_prim_io_names(inputs=['features', 'labels'], outputs=['output']) + self.is_grad = is_grad + self.add_prim_attr('sens', 1.0) + + def infer_shape(self, logits_shape, labels_shape): + validator.check_param_equal("SparseSoftmaxCrossEntropyWithLogits logits_shape", logits_shape[0], + "SparseSoftmaxCrossEntropyWithLogits labels_shape", labels_shape[0]) + loss_shape = [] + if self.is_grad: + return logits_shape + return loss_shape + + def infer_dtype(self, logits_type, labels_type): + validator.check_typename("SparseSoftmaxCrossEntropyWithLogits logits_type", + logits_type, (mstype.float16, mstype.float32)) + validator.check_typename("SparseSoftmaxCrossEntropyWithLogits labels_type", + labels_type, (mstype.int32, mstype.int64)) + return logits_type + + +class ApplyMomentum(PrimitiveWithInfer): + """ + Optimizer that implements the Momentum algorithm. + + Refer to the paper `On the importance of initialization and momentum in deep + learning `_ for more details. + + Args: + use_locking (bool): Enable a lock to protect the update of variable and accumlation tensors. Default: False. + use_nesterov (bool): Enable Nesterov momentum. Default: False. + gradient_scale (float): The scale of the gradient. Default: 1.0. + + Inputs: + - **variable** (Tensor) - Weights to be update. + - **accumulation** (Tensor) - Accumulated gradient value by moment weight. + - **learning_rate** (float) - Learning rate. + - **gradient** (Tensor) - Gradients. + - **momentum** (float) - Momentum. + + Outputs: + Tensor, parameters to be update. + + Examples: + >>> net = ResNet50() + >>> loss = SoftmaxCrossEntropyWithLogits() + >>> opt = ApplyMomentum(Tensor(np.array([0.001])), Tensor(np.array([0.9])), + filter(lambda x: x.requires_grad, net.get_parameters())) + >>> model = Model(net, loss, opt) + """ + + @prim_attr_register + def __init__(self, use_nesterov=False, use_locking=False, gradient_scale=1.0): + self.init_prim_io_names(inputs=['variable', 'accumulation', 'learning_rate', 'gradient', 'momentum'], + outputs=['output']) + + def infer_shape(self, v_shape, a_shape, l_shape, g_shape, m_shape): + validator.check(f'variable shape {v_shape}', len(v_shape), '', 0, Rel.GT) + validator.check(f'accumulation shape {a_shape}', len(a_shape), '', 0, Rel.GT) + validator.check(f'learning rate shape {l_shape}', len(l_shape), '', 0, Rel.GE) + validator.check(f'gradient shape {g_shape}', len(g_shape), '', 0, Rel.GE) + validator.check(f'momentum shape {m_shape}', len(m_shape), '', 0, Rel.GE) + return v_shape + + def infer_dtype(self, v_dtype, a_dtype, l_dtype, g_dtype, m_dtype): + validator.check_subclass("v_dtype", v_dtype, mstype.tensor) + validator.check_subclass("a_dtype", a_dtype, mstype.tensor) + v_type = validator.check_typename("v_dtype", v_dtype, [mstype.float16, mstype.float32, mstype.float64]) + validator.check_typename("a_dtype", a_dtype, [mstype.float16, mstype.float32, mstype.float64]) + validator.check_typename("l_dtype", l_dtype, [mstype.float16, mstype.float32, mstype.float64]) + validator.check_typename("g_dtype", g_dtype, [mstype.float16, mstype.float32, mstype.float64]) + validator.check_typename("m_dtype", m_dtype, [mstype.float16, mstype.float32, mstype.float64]) + return v_type + + +class SmoothL1Loss(PrimitiveWithInfer): + r""" + Computes smooth L1 loss, a robust L1 loss. + + SmoothL1Loss is a Loss similar to MSELoss but less sensitive to outliers as described in the + `Fast R-CNN `_ by Ross Girshick. + + Note: + Sets input prediction as `X`, input target as `Y`, output as `loss`. Then, + + .. math:: + \text{SmoothL1Loss} = \begin{cases}0.5x^{2}, &if \left |x \right |\leq \text{sigma} \cr + \left |x \right|-0.5, &\text{otherwise}\end{cases} + + Args: + sigma (float): A parameter used to control the point where the function will change from + quadratic to linear. Default: 1.0. + + Inputs: + - **prediction** (Tensor) - Predict data. + - **target** (Tensor) - Ground truth data, with the same type and shape as `prediction`. + + Outputs: + Tensor, with the same type and shape as `prediction`. + """ + + @prim_attr_register + def __init__(self, sigma=1.0): + validator.check_type('sigma', sigma, [float]) + validator.check('sigma', sigma, '', 0, Rel.GT) + self.init_prim_io_names(inputs=['prediction', 'target'], outputs=['output']) + + def infer_shape(self, prediction, target): + validator.check_param_equal('prediction shape', prediction, 'target shape', target) + return prediction + + def infer_dtype(self, prediction, target): + args = {"prediction": prediction, "target": target} + validator.check_type_same(args, (mstype.float16, mstype.float32)) + return prediction + + +class SGD(PrimitiveWithInfer): + """ + Computes stochastic gradient descent (optionally with momentum). + + Nesterov momentum is based on the formula from On the importance of + initialization and momentum in deep learning. + + Args: + dampening (float): The dampening for momentum. Default: 0.0. + weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + nesterov (bool): Enable Nesterov momentum. Default: False. + + Inputs: + - **parameters** (Tensor) - Parameters to be update. + - **gradient** (Tensor) - Gradients. + - **learning_rate** (Tensor) - Learning rate. e.g. Tensor(0.1, mindspore.float32). + - **accum** (Tensor) - Accum(velocity) to be update. + - **momentum** (Tensor) - Momentum. e.g. Tensor(0.1, mindspore.float32). + - **stat** (Tensor) - States to be updated with the same shape as gradient. Default: 1.0. + + Outputs: + Tensor, parameters to be update. + + Examples: + >>> net = ResNet50() + >>> loss = SoftmaxCrossEntropyWithLogits() + >>> opt = SGD(params=net.trainable_params(), learning_rate=lr, momentum=0.9) + >>> model = Model(net, loss, opt) + """ + + @prim_attr_register + def __init__(self, dampening=0.0, weight_decay=0.0, nesterov=False): + self.init_prim_io_names(inputs=['parameters', 'gradient', 'learning_rate', 'accum', 'momentum', 'stat'], + outputs=['output']) + + def infer_shape(self, parameters_shape, gradient_shape, learning_rate_shape, + accum_shape, momentum_shape, stat_shape): + validator.check(f'parameters shape {parameters_shape}', len(parameters_shape), '', 0, Rel.GT) + validator.check(f'gradient shape {gradient_shape}', len(gradient_shape), '', 0, Rel.GE) + validator.check(f'learning rate shape {learning_rate_shape}', len(learning_rate_shape), '', 0, Rel.GE) + validator.check(f'accumulation shape {accum_shape}', len(accum_shape), '', 0, Rel.GT) + validator.check(f'momentum shape {momentum_shape}', len(momentum_shape), '', 0, Rel.GE) + validator.check(f'stat shape {stat_shape}', len(stat_shape), '', 0, Rel.GE) + validator.check("gradient shape", gradient_shape, "stat shape", stat_shape) + return parameters_shape + + def infer_dtype(self, parameters_dtype, gradient_dtype, learning_rate_dtype, + accum_dtype, momentum_dtype, stat_dtype): + validator.check_typename("parameters_dtype", parameters_dtype, [mstype.float16, mstype.float32]) + validator.check_typename("gradient_dtype", gradient_dtype, [mstype.float16, mstype.float32]) + validator.check_typename("learning_rate_dtype", learning_rate_dtype, [mstype.float16, mstype.float32]) + validator.check_typename("accum_dtype", accum_dtype, [mstype.float16, mstype.float32]) + validator.check_typename("momentum_dtype", momentum_dtype, [mstype.float16, mstype.float32]) + validator.check_typename("stat_dtype", stat_dtype, [mstype.float16, mstype.float32]) + return parameters_dtype + + +class LayerNorm(Primitive): + r""" + Applies the Layer Normalization to the input tensor. + + This operator will normalize the input tensor on given axis. LayerNorm is described in the paper + `Layer Normalization `_. + + .. math:: + y = \frac{x - mean]}{\sqrt{variance + \epsilon}} * \gamma + \beta + + where :math:`\gamma` is scale, :math:`\beta` is bias, :math:`\epsilon` is epsilon. + + Args: + begin_norm_axis (int): The begin axis of the `input_x` to apply LayerNorm, + the value should be in [-1, rank(input)). Default: 1. + begin_params_axis (int): The begin axis of the parameter input (`gamma`, `beta`) to + apply LayerNorm, the value should be in [-1, rank(input)). Default: 1. + + Inputs: + - **input_x** (Tensor) - Tensor of shape :math:`(N, \ldots)`. + The input of LayerNorm. + - **gamma** (Tensor) - Tensor of shape :math:`(P_0, \ldots, P_\text{begin_params_axis})`. + The learnable parameter `gamma` as the scale on norm. + - **beta** (Tensor) - Tensor of shape :math:`(P_0, \ldots, P_\text{begin_params_axis})`. + The learnable parameter `beta` as the scale on norm. + + Outputs: + tuple[Tensor], tuple of 3 tensors, the normalized input and the updated parameters. + + - **output_x** (Tensor) - The normalized input, has the same type and shape as the `input_x`. + The shape is :math:`(N, C)`. + - **updated_gamma** (Tensor) - Tensor of shape :math:`(C,)`. + - **updated_beta** (Tensor) - Tensor of shape :math:`(C,)`. + """ + + @prim_attr_register + def __init__(self, begin_norm_axis=1, begin_params_axis=1): + validator.check_type('begin_norm_axis', begin_norm_axis, [int]) + validator.check_type('begin_params_axis', begin_params_axis, [int]) + + +class L2Normalize(PrimitiveWithInfer): + r""" + L2 normalization Operator. + + This operator will normalizes the input using the given axis. The function is shown as follows: + + .. math:: + \text{output} = \frac{x}{\sqrt{\text{max}(\text{sum} (\text{input_x}^2), \epsilon)}}, + + where :math:`\epsilon` is epsilon. + + Args: + axis (int): The begin axis for the input to apply L2 normalize. Default: 0. + epsilon (float): A small value added for numerical stability. Default: 1e-4. + + Inputs: + - **input_x** (Tensor) - Input to compute the normalization. + + Outputs: + Tensor, with the same type and shape as the input. + """ + + @prim_attr_register + def __init__(self, axis=0, epsilon=1e-4): + validator.check_type('axis', axis, [int]) + validator.check_type('epsilon', epsilon, [int, float]) + + def infer_shape(self, input_x): + dim = len(input_x) + validator.check_int_range('axis value', self.axis, -dim, dim, Rel.INC_LEFT) + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("x", input_x, mstype.tensor) + return input_x + + +class DropoutGenMask(Primitive): + """ + Generates the mask value for the input shape. + + Args: + Seed0 (int): Seed0 value for random generating. Default: 0. + Seed1 (int): Seed1 value for random generating. Default: 0. + + Inputs: + - **shape** (tuple[int]) - The shape of target mask. + - **keep_prob** (Tensor) - The keep rate, between 0 and 1, e.g. keep_prob = 0.9, + means dropping out 10% of input units. + + Outputs: + Tensor, the value of generated mask for input shape. + + Examples: + >>> dropout_gen_mask = DropoutGenMask() + >>> shape = (20, 16, 50) + >>> keep_prob = Tensor(0.5, mindspore.float32) + >>> mask = dropout_gen_mask(shape, keep_prob) + """ + + @prim_attr_register + def __init__(self, Seed0=0, Seed1=0): + self.init_prim_io_names(inputs=['shape', 'keep_prob'], outputs=['output']) + validator.check_type("Seed0", Seed0, [int]) + validator.check_type("Seed1", Seed1, [int]) + + +class DropoutDoMask(PrimitiveWithInfer): + """ + Applies dropout mask on the input tensor. + + Take the mask output of DropoutGenMask as input, and apply dropout on the input. + + Inputs: + - **input_x** (Tensor) - The input tensor. + - **mask** (Tensor) - The mask to be applied on `input_x`, which is the output of `DropoutGenMask`. And the + shape of `input_x` must be same as the value of `DropoutGenMask`'s input `shape`. If input wrong `mask`, + the output of `DropoutDoMask` are unpredictable. + - **keep_prob** (Tensor) - The keep rate, between 0 and 1, e.g. keep_prob = 0.9, + means dropping out 10% of input units. The value of `keep_prob` is same as the input `keep_prob` of + `DropoutGenMask`. + + Outputs: + Tensor, the value that applied dropout on. + + Examples: + >>> x = Tensor(np.ones([20, 16, 50]), mindspore.float32) + >>> shape = (20, 16, 50) + >>> keep_prob = Tensor(0.5, mindspore.float32) + >>> dropout_gen_mask = DropoutGenMask() + >>> dropout_do_mask = DropoutDoMask() + >>> mask = dropout_gen_mask(shape, keep_prob) + >>> output = dropout_do_mask(x, mask, keep_prob) + >>> assert output.shape() == (20, 16, 50) + """ + + @prim_attr_register + def __init__(self): + pass + + def __infer__(self, input_x, mask, keep_prob): + input_x_shape = input_x['shape'] + mask_shape = mask['shape'] + keep_prob_shape = keep_prob['shape'] + validator.check("keep_prob's dim", len(keep_prob_shape), '0(scalar)', 0) + size_x = reduce(lambda x, y: x * y, input_x_shape) + if len(mask_shape) != 1: + raise ValueError("DropoutDoMask mask shape should be 1-dimension.") + size_y = mask_shape[0] * 8 + if size_x > size_y: + raise ValueError(f"DropoutDoMask y mask do not math input input_x shape:" + "{input_x_shape}, mask shape: {mask_shape}.") + + validator.check_typename("input_x type", input_x['dtype'], [mstype.float32, mstype.float16, mstype.int32]) + validator.check_typename("input_mask type", mask['dtype'], [mstype.uint8]) + + keep_prob_v = keep_prob['value'] + if keep_prob_v is not None: + validator.check_const_input('keep_prob', keep_prob_v) + validator.check_number_range('keep_prob', keep_prob_v.asnumpy(), 0, 1, Rel.INC_BOTH) + + out = {'shape': input_x_shape, + 'dtype': input_x['dtype'], + 'value': None} + return out + + +class ResizeBilinear(PrimitiveWithInfer): + r""" + Resizes the image to certain size using bilinear interpolation. + + The resizing only affects the lower two dimensions which represent the height and width. The input images + can be represented by different data types, but the data types of output images are always float32. + + Args: + size (tuple[int]): A tuple of 2 int elements `(new_height, new_width)`, the new size for the images. + align_corners (bool): If it's true, rescale input by `(new_height - 1) / (height - 1)`, + which exactly aligns the 4 corners of images and resized images. If it's false, + rescale by `new_height / height`. Default: False. + + Inputs: + - **input** (Tensor) - Image to be resized. Tensor of shape `(N_i, ..., N_n, height, width)`. + + Outputs: + Tensor, resized image. Tensor of shape `(N_i, ..., N_n, new_height, new_width)` in `float32`. + + Examples: + >>> tensor = Tensor([[[[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]]]], mindspore.int32) + >>> resize_bilinear = P.ResizeBilinear((5, 5)) + >>> result = resize_bilinear(tensor) + >>> assert result.shape() == (5, 5) + """ + + @prim_attr_register + def __init__(self, size, align_corners=False): + pass + + def infer_shape(self, input_shape): + input_shape = list(input_shape) + batch, channel, _, _ = input_shape + out_shape = [batch, channel] + for i in self.size: + out_shape.append(int(i)) + return out_shape + + def infer_dtype(self, input_dtype): + return mstype.tensor_type(mstype.float32) + + +class OneHot(PrimitiveWithInfer): + r""" + Computes a one-hot tensor. + + Makes a new tensor, whose locations represented by indices in `indices` take value `on_value`, while all + other locations take value `off_value`. + + Note: + If the input indices is rank `N`, the output will have rank `N+1`. The new axis is created at dimension `axis`. + + Args: + axis (int): Position to insert the value. e.g. If `indices` shape is [n, c], and `axis` is `-1` the output shape + will be [n, c, depth], If `axis` is `0` the output shape will be [depth, n, c]. Default: -1. + + Inputs: + - **indices** (Tensor) - A tensor of indices. Tensor of shape :math:`(X_0, \ldots, X_n)`. + - **depth** (int) - A scalar defining the depth of the one hot dimension. + - **on_value** (Tensor) - A value to fill in output when `indices[j] = i`. + - **off_value** (Tensor) - A value to fill in output when `indices[j] != i`. + + Outputs: + Tensor, one_hot tensor. Tensor of shape :math:`(X_0, \ldots, X_{axis}, \text{depth} ,X_{axis+1}, \ldots, X_n)`. + + Examples: + >>> indices = Tensor(np.array([0, 1, 2]), mindspore.int32) + >>> depth, on_value, off_value = 3, Tensor(1.0, mindspore.float32), Tensor(0.0, mindspore.float32) + >>> onehot = OneHot() + >>> result = onehot(indices, depth, on_value, off_value) + [[1, 0, 0], [0, 1, 0], [0, 0, 1]] + """ + + @prim_attr_register + def __init__(self, axis=-1): + self.init_prim_io_names(inputs=['indices', 'depth', 'on_value', 'off_value'], outputs=['output']) + validator.check_type("axis", axis, [int]) + + def __infer__(self, indices, depth, on_value, off_value): + # check type + validator.check_subclass("indices", indices['dtype'], mstype.tensor) + validator.check_typename("indices", indices['dtype'], (mstype.int32,)) + validator.check_typename("depth", depth['dtype'], mstype.int_type) + validator.check_subclass("on_value", on_value['dtype'], mstype.tensor) + validator.check_subclass("off_value", off_value['dtype'], mstype.tensor) + args = {"on_value dtype": on_value['dtype'], "off_value dtype": off_value['dtype']} + validator.check_type_same(args, (mstype.float16, mstype.float32)) + + # check shape + indices_shp = indices['shape'] + validator.check_int_range("axis", self.axis, -1, len(indices_shp), Rel.INC_BOTH) + depth_val = depth['value'] + validator.check_integer("depth", depth_val, 0, Rel.GE) + # create new dimension at end if self.axis is -1 + indices_shp.insert(self.axis, depth_val) if self.axis >= 0 else indices_shp.append(depth_val) + + return {'shape': indices_shp, + 'dtype': on_value['dtype'], + 'value': None} + + +class Gelu(PrimitiveWithInfer): + r""" + Gaussian Error Linear Units activation function. + + GeLU is described in the paper `Gaussian Error Linear Units (GELUs) `_. + And also please refer to `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. + `_. + + Defined as follows: + + .. math:: + \text{output} = 0.5 * x * (1 + erf(x / \sqrt{2})), + + where :math:`erf` is the "Gauss error function" . + + Inputs: + - **input_x** (Tensor) - Input to compute the Gelu. + + Outputs: + Tensor, with the same type and shape as input. + + Examples: + >>> tensor = Tensor(np.array([1.0, 2.0, 3.0]), mindspore.float32) + >>> gelu = Gelu() + >>> result = gelu(tensor) + """ + + @prim_attr_register + def __init__(self): + """init GeLU""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_subclass("input_x", input_x, mstype.tensor) + validator.check_typename("input_x", input_x, (mstype.float16, mstype.float32)) + return input_x + + +class GetNext(PrimitiveWithInfer): + """ + Returns the next element in the dataset queue. + + Note: + GetNext op needs to be associated with network and also depends on the init_dataset interface, + it can't be used directly as a single op. + For details, please refer to `nn.cell_wrapper.DataWrapper` source code. + + Args: + types (list[:class:`mindspore.dtype`]): The type of the outputs. + shapes (list[tuple[int]]): The dimensionality of the outputs. + output_num (int): The output number, length of `types` and `shapes`. + shared_name (str): The queue name of `init_dataset` interface. + + Inputs: + No inputs. + + Outputs: + tuple[Tensor], the output of Dataset. The shape is described in `shapes` + and the type is described is `types`. + + Examples: + >>> get_next = GetNext([mindspore.float32, mindspore.int32], [[32, 1, 28, 28], [10]], 'shared_name') + >>> feature, label = get_next() + """ + + @prim_attr_register + def __init__(self, types, shapes, output_num, shared_name): + validator.check_type("types", types, [list, tuple]) + validator.check_type("shapes", shapes, [list, tuple]) + validator.check("types length", len(types), "shapes length", len(shapes)) + validator.check_type("output_num", output_num, [int]) + + def infer_shape(self): + return tuple(self.shapes) + + def infer_dtype(self): + return tuple(self.types) + + +class PReLU(PrimitiveWithInfer): + r""" + Parametric Rectified Linear Unit activation function. + + PReLU is described in the paper `Delving Deep into Rectifiers: Surpassing Human-Level Performance on + ImageNet Classification `_. Defined as follows: + + .. math:: + prelu(x_i)= \max(0, x_i) + \min(0, w * x_i), + + where :math:`x_i` is an element of an channel of the input. + + Inputs: + - **input_x** (Tensor) - Float tensor, representing the output of the preview layer. + - **weight** (Tensor) - Float Tensor, w > 0, there is only two shapes are legitimate, + 1 or the number of channels at input. + + Outputs: + Tensor, with the same type as `input_x`. + + Detailed information, please refer to `nn.PReLU`. + """ + + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, input_x_shape, weight_shape): + input_x_dim = len(input_x_shape) + weight_dim = len(weight_shape) + + if weight_dim != 1: + raise ValueError(f'weight_dim must be 1, while weight_dim is {weight_dim}.') + + if input_x_dim == 1 and weight_shape[0] != 1: + raise ValueError(f'when input_x_dim is 1, weight_shape[0] must be 1, ' + f'while weight_shape[0] is {weight_shape[0]}.') + + if input_x_dim != 1 and weight_shape[0] != input_x_shape[1] and weight_shape[0] != 1: + raise ValueError(f'channel of input_x and weight must be matched,' + f' while channel of input_x is {input_x_shape[1]},' + f' weight_shape[0] is {weight_shape[0]}.') + + return input_x_shape + + def infer_dtype(self, input_x_dtype, weight_dtype): + validator.check_subclass("input_x_dtype", input_x_dtype, mstype.tensor) + validator.check_subclass("weight_dtype", weight_dtype, mstype.tensor) + validator.check_typename("input_x_dtype", input_x_dtype, (mstype.float16, mstype.float32)) + validator.check_typename("weight_dtype", weight_dtype, (mstype.float16, mstype.float32)) + return input_x_dtype + + +class LSTM(PrimitiveWithInfer): + """ + Performs the long short term memory(LSTM) on the input. + + Detailed information, please refer to `nn.layer.LSTM`. + """ + + @prim_attr_register + def __init__(self, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + self.input_size = check_int_positive(input_size) + self.hidden_size = check_int_positive(hidden_size) + self.num_layers = check_int_positive(num_layers) + self.has_bias = check_bool(has_bias) + self.bidirectional = check_bool(bidirectional) + self.dropout = validator.check_type("dropout", dropout, [float]) + self.dropout = validator.check_number_range('dropout', dropout, 0, 1, Rel.INC_BOTH) + + if bidirectional: + self.num_directions = 2 + else: + self.num_directions = 1 + + def infer_shape(self, x_shape, h_shape, c_shape, w_shape): + # (batch, seq, feature) + validator.check_integer("x_shape", len(x_shape), 3, Rel.EQ) + + # h and c should be same shape + validator.check_integer("h_shape", len(h_shape), 3, Rel.EQ) + validator.check_integer("h_shape", len(h_shape), len(c_shape), Rel.EQ) + validator.check_integer("h_shape", h_shape[0], c_shape[0], Rel.EQ) + validator.check_integer("h_shape", h_shape[1], c_shape[1], Rel.EQ) + validator.check_integer("h_shape", h_shape[2], c_shape[2], Rel.EQ) + + # (num_layers * num_directions, batch, hidden_size) + validator.check_integer("h[0]", h_shape[0], self.num_layers * self.num_directions, Rel.EQ) + validator.check_integer("h[1]", h_shape[1], x_shape[1], Rel.EQ) + validator.check_integer("h[2]", h_shape[2], self.hidden_size, Rel.EQ) + + y_shape = (x_shape[0], x_shape[1], self.hidden_size * self.num_directions) + + # set arbitrary shape for reserved space + reserved_shape = (1, 1) + state_shape = (1, 1) + return (y_shape, h_shape, c_shape, reserved_shape, state_shape) + + def infer_dtype(self, x_dtype, h_dtype, c_dtype, w_dtype): + validator.check_typename("x_dtype", x_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("h_dtype", h_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("c_dtype", c_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("w_dtype", w_dtype, (mstype.float32, mstype.float16)) + validator.check_typename("datatype", x_dtype, (h_dtype.element_type(),)) + validator.check_typename("datatype", x_dtype, (c_dtype.element_type(),)) + validator.check_typename("datatype", x_dtype, (w_dtype.element_type(),)) + return (x_dtype, x_dtype, x_dtype, x_dtype, x_dtype) + + +class SigmoidCrossEntropyWithLogits(PrimitiveWithInfer): + r""" + Uses the given logits to compute sigmoid cross entropy. + + Note: + Sets input logits as `X`, input label as `Y`, output as `loss`. Then, + + .. math:: + p_{ij} = sigmoid(X_{ij}) = \frac{1}{1 + e^{-X_{ij}}} + + .. math:: + loss_{ij} = -[Y_{ij} * ln(p_{ij}) + (1 - Y_{ij})ln(1 - p_{ij})] + + Inputs: + - **logits** (Tensor) - Input logits. + - **label** (Tensor) - Ground truth label. + + Outputs: + Tensor, with the same shape and type as input `logits`. + """ + + @prim_attr_register + def __init__(self): + """Init SigmoidCrossEntropyWithLogits""" + self.init_prim_io_names(inputs=['predict', 'target'], outputs=['loss']) + + def infer_shape(self, x_shape, y_shape): + validator.check_param_equal("x_shape", x_shape, "y_shape", y_shape) + return x_shape + + def infer_dtype(self, x_dtype, y_dtype): + args = {"x_dtype": x_dtype, "y_dtype": y_dtype} + validator.check_type_same(args, mstype.number_type) + return x_dtype + + +class Pad(PrimitiveWithInfer): + """ + Pads input tensor according to the paddings. + + Args: + paddings (tuple): The shape of parameter `paddings` is (N, 2). N is the rank of input data. All elements of + paddings are int type. For `D` th dimension of input, paddings[D, 0] indicates how many sizes to be + extended ahead of the `D` th dimension of the input tensor, and paddings[D, 1] indicates how many sizes to + be extended behind of the `D` th dimension of the input tensor. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tensor, the tensor after padding. + + Examples: + >>> input_tensor = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]), mindspore.float32) + >>> pad_op = Pad(((1, 2), (2, 1))) + >>> output_tensor = pad_op(input_tensor) + >>> assert output_tensor == Tensor(np.array([[ 0. , 0. , 0. , 0. , 0. , 0. ], + >>> [ 0. , 0. , -0.1, 0.3, 3.6, 0. ], + >>> [ 0. , 0. , 0.4, 0.5, -3.2, 0. ], + >>> [ 0. , 0. , 0. , 0. , 0. , 0. ], + >>> [ 0. , 0. , 0. , 0. , 0. , 0. ]]), mindspore.float32) + """ + + @prim_attr_register + def __init__(self, paddings): + """Init Pad""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + if not isinstance(paddings, tuple): + raise TypeError('Paddings must be tuple type.') + for item in paddings: + if len(item) != 2: + raise ValueError('The shape of paddings must be (n, 2).') + + def infer_shape(self, x): + paddings = np.array(self.paddings) + validator.check_integer('paddings.shape', paddings.size, len(x) * 2, Rel.EQ) + if not np.all(paddings >= 0): + raise ValueError('All elements of paddings must be >= 0.') + y_shape = () + for i in range(int(paddings.size / 2)): + y_shape += ((x[i] + paddings[i, 0] + paddings[i, 1]),) + return y_shape + + def infer_dtype(self, x): + return x + + +class ROIAlign(PrimitiveWithInfer): + """ + Computes Region of Interest (RoI) Align operator. + + The operator computes the value of each sampling point by bilinear interpolation from the nearby grid points on the + feature map. No quantization is performed on any coordinates involved in the RoI, its bins, or the sampling + points. The details of (RoI) Align operator are described in `Mask R-CNN `_. + + Args: + pooled_height (int): The output features' height. + pooled_width (int): The output features' width. + spatial_scale (float): A scaling factor that maps the raw image coordinates to the input + feature map coordinates. Suppose the height of a RoI is `ori_h` in the raw image and `fea_h` in the + input feature map, the `spatial_scale` should be `fea_h / ori_h`. + sample_num (int): Number of sampling points. Default: 2. + + Inputs: + - **features** (Tensor) - The input features, whose shape should be `(N, C, H, W)`. + - **rois** (Tensor) - The shape is `(rois_n, 5)`. `rois_n` represents the number of RoI. The size of + the second dimension should be `5` and the `5` colunms are + `(image_index, top_left_x, top_left_y, bottom_right_x, bottom_right_y)`. `image_index` represents the + index of image. `top_left_x` and `top_left_y` represent the `x, y` coordinates of the top left corner + of corresponding RoI, respectively. `bottom_right_x` and `bottom_right_y` represent the `x, y` + coordinates of the bottom right corner of corresponding RoI, respectively. + + Outputs: + Tensor, the shape is `(rois_n, C, pooled_height, pooled_width)`. + + Examples: + >>> input_tensor = Tensor(np.array([[[[1., 2.], [3., 4.]]]]), mindspore.float32) + >>> rois = Tensor(np.array([[0, 0.2, 0.3, 0.2, 0.3]]), mindspore.float32) + >>> roi_align = P.ROIAlign(1, 1, 0.5, 2) + >>> output_tensor = roi_align(input_tensor, rois) + >>> assert output_tensor == Tensor(np.array([[[[2.15]]]]), mindspore.float32) + """ + + @prim_attr_register + def __init__(self, pooled_height, pooled_width, spatial_scale, sample_num=2): + """init ROIAlign""" + validator.check_type("pooled_height", pooled_height, [int]) + validator.check_type("pooled_width", pooled_width, [int]) + validator.check_type("spatial_scale", spatial_scale, [float]) + validator.check_type("sample_num", sample_num, [int]) + self.pooled_height = pooled_height + self.pooled_width = pooled_width + self.spatial_scale = spatial_scale + self.sample_num = sample_num + + def infer_shape(self, inputs_shape, rois_shape): + return [rois_shape[0], inputs_shape[1], self.pooled_height, self.pooled_width] + + def infer_dtype(self, inputs_type, rois_type): + return inputs_type + + +class Adam(PrimitiveWithInfer): + r""" + Updates gradients by Adaptive Moment Estimation (Adam) algorithm. + + The Adam algorithm is proposed in `Adam: A Method for Stochastic Optimization `_. + + The updating formulas are as follows, + + .. math:: + \begin{array}{ll} \\ + m = \beta_1 * m + (1 - \beta_1) * g \\ + v = \beta_2 * v + (1 - \beta_2) * g * g \\ + l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\ + w = w - l * \frac{m}{\sqrt{v} + \epsilon} + \end{array} + + :math:`m` represents the 1st moment vector, :math:`v` represents the 2nd moment vector, :math:`g` represents + `gradient`, :math:`l` represents scaling factor `lr`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`, + :math:`t` represents updating step while :math:`beta_1^t` and :math:`beta_2^t` represent `beta1_power` and + `beta2_power`, :math:`\alpha` represents `learning_rate`, :math:`w` represents `var`, :math:`\epsilon` represents + `epsilon`. + + Args: + use_locking (bool): Whether to enable a lock to protect updating variable tensors. + If True, updating of the var, m, and v tensors will be protected by a lock. + If False, the result is unpredictable. Default: False. + use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients. + If True, updates the gradients using NAG. + If False, updates the gradients without using NAG. Default: False. + + Inputs: + - **var** (Tensor) - Weights to be update. + - **m** (Tensor) - The 1st moment vector in the updating formula. + - **v** (Tensor) - the 2nd moment vector in the updating formula. + - **beta1_power** (float) - :math:`beta_1^t` in the updating formula. + - **beta2_power** (float) - :math:`beta_2^t` in the updating formula. + - **lr** (float) - :math:`l` in the updating formula. + - **beta1** (float) - The exponential decay rate for the 1st moment estimates. + - **beta2** (float) - The exponential decay rate for the 2nd moment estimates. + - **epsilon** (float) - Term added to the denominator to improve numerical stability. + - **gradient** (Tensor) - Gradients. + + Outputs: + Tensor, has the same shape and data type as `var`. + """ + + @prim_attr_register + def __init__(self, use_locking=False, use_nesterov=False): + validator.check_type("use_locking", use_locking, [bool]) + validator.check_type("use_nesterov", use_nesterov, [bool]) + + def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, + beta1_shape, beta2_shape, epsilon_shape, grad_shape): + validator.check_param_equal("var_shape", var_shape, "m_shape", m_shape) + validator.check_param_equal("var_shape", var_shape, "v_shape", v_shape) + validator.check_param_equal("var_shape", var_shape, "grad_shape", grad_shape) + return var_shape + + def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, beta2_power_dtype, lr_dtype, + beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype): + args = {"var_dtype": var_dtype, "m_dtype": m_dtype, "v_dtype": v_dtype, "grad_dtype": grad_dtype} + validator.check_type_same(args, mstype.number_type) + + args = {"beta1_power_dtype": beta1_power_dtype, "beta2_power_dtype": beta2_power_dtype, 'lr_dtype': lr_dtype, + "beta1_dtype": beta1_dtype, "beta2_dtype": beta2_dtype, "epsilon_dtype": epsilon_dtype} + validator.check_type_same(args, [mstype.float16, mstype.float32]) + return var_dtype + + +class BinaryCrossEntropy(PrimitiveWithInfer): + r""" + Computes the Binary Cross Entropy between the target and the output. + + Note: + Sets input as :math:`x`, input label as :math:`y`, output as :math:`\ell(x, y)`. + Let, + + .. math:: + L = \{l_1,\dots,l_N\}^\top, \quad + l_n = - w_n \left[ y_n \cdot \log x_n + (1 - y_n) \cdot \log (1 - x_n) \right] + + Then, + + .. math:: + \ell(x, y) = \begin{cases} + L, & \text{if reduction} = \text{'none';}\\ + \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\ + \operatorname{sum}(L), & \text{if reduction} = \text{'sum'.} + \end{cases} + + Args: + reduction (str): Specifies the reduction to apply to the output. + Its value should be one of 'none', 'mean', 'sum'. Default: 'mean'. + + Inputs: + - **input_x** (Tensor) - The input Tensor. + - **input_y** (Tensor) - The label Tensor which has same shape as `input_x`. + - **weight** (Tensor, optional) - A rescaling weight applied to the loss of each batch element. + And it should have same shape as `input_x`. Default: None. + + Outputs: + Tensor or Scalar, if `reduction` is 'none', then output is a tensor and same shape as `input_x`. + Otherwise it is a scalar. + """ + + @prim_attr_register + def __init__(self, reduction='mean'): + self.reduction = validator.check_string('reduction', reduction, ['none', 'mean', 'sum']) + + def infer_shape(self, x_shape, y_shape, weight_shape): + validator.check_param_equal('x_shape', x_shape, 'y_shape', y_shape) + if weight_shape: + validator.check_param_equal('y_shape', y_shape, 'weight_shape', weight_shape) + if self.reduction in ('mean', 'sum'): + shape = [] + else: + shape = x_shape + return shape + + def infer_dtype(self, x_type, y_type, weight_type): + args = {'x_type': x_type, 'y_type': y_type} + validator.check_type_same(args, (mstype.float16, mstype.float32)) + if weight_type: + validator.check_two_types_same('x_type', x_type, 'weight_type', weight_type) + return x_type + + +class SparseApplyAdagrad(PrimitiveWithInfer): + r""" + Update relevant entries according to the adagrad scheme. + + .. math:: + accum += grad * grad + .. math:: + var -= lr * grad * (1 / sqrt(accum)) + + Args: + lr (float): Learning rate. + use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. + + Inputs: + - **var** (Tensor) - Variable to be update. The type must be float32. + - **accum** (Tensor) - Accum to be update. The shape must be the same as `var`'s shape, + the type must be float32. + - **grad** (Tensor) - Gradient. The shape must be the same as `var`'s shape + except first dimension, the type must be float32. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. + The shape of `indices` must be the same as `grad` in first dimension, the type must be int32. + + Outputs: + Tensor, has the same shape and type as `var`. + """ + + @prim_attr_register + def __init__(self, lr, use_locking=False): + self.lr = validator.check_type("lr", lr, [float]) + self.use_locking = validator.check_type("use_locking", use_locking, [bool]) + + def infer_shape(self, var_shape, accum_shape, grad_shape, indices_shape): + validator.check_param_equal('var shape', var_shape, 'accum shape', accum_shape) + validator.check_param_equal('len of var shape', len(var_shape), 'len of grad shape', len(grad_shape)) + if len(var_shape) > 1: + validator.check_param_equal('var_shape', var_shape[1:], 'grad_shape', grad_shape[1:]) + validator.check_integer("len of indices shape", len(indices_shape), 1, Rel.EQ) + validator.check('the first dimension of grad', grad_shape[0], + 'the shape of indices', indices_shape[0], Rel.EQ) + return var_shape + + def infer_dtype(self, var_type, accum_type, grad_type, indices_type): + validator.check_subclass("var_type", var_type, mstype.tensor) + validator.check_subclass("accum_type", accum_type, mstype.tensor) + validator.check_subclass("grad_type", grad_type, mstype.tensor) + validator.check_subclass("indices_type", indices_type, mstype.tensor) + args = {'var_type': var_type, 'accum_type': accum_type, 'grad_type': grad_type} + validator.check_type_same(args, (mstype.float32,)) + validator.check_typename('indices_type', indices_type, [mstype.int32]) + return var_type + + +class LARSUpdate(PrimitiveWithInfer): + """ + Conduct lars (layer-wise adaptive rate scaling) update on the square sum of gradient. + + Args: + epsilon (float): Term added to the denominator to improve numerical stability. Default: 1e-05. + hyperpara (float): Trust coefficient for calculating the local learning rate. Default: 0.001. + use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: False. + + Inputs: + - **weight** (Tensor) - The weight to be update. + - **gradient** (Tensor) - The gradient of weight, which has the same shape and dtype with weight. + - **norm_weight** (Tensor) - A scalar tensor, representing the square sum of weight. + - **norm_gradient** (Tensor) - A scalar tensor, representing the square sum of gradient. + - **weight_decay** (Union[Number, Tensor]) - Weight decay. It should be a scalar tensor or number. + - **learning_rate** (Union[Number, Tensor]) - Learning rate. It should be a scalar tensor or number. + + Outputs: + Tensor, representing the new gradient. + """ + + @prim_attr_register + def __init__(self, epsilon=1e-05, hyperpara=0.001, use_clip=False): + """init""" + validator.check_type("epsilon", epsilon, [float]) + validator.check_type("hyperpara", hyperpara, [float]) + validator.check_type("use_clip", use_clip, [bool]) + + def infer_shape(self, weight_shape, gradient_shape, norm_weight_shape, norm_gradient_shape, weight_decay_shape, + learning_rate_shape): + validator.check_param_equal("Weight shape", weight_shape, "gradient shape", gradient_shape) + validator.check_param_equal("Norm weight shape", norm_weight_shape, "norm gradient shape", norm_gradient_shape) + shp_len = len(weight_decay_shape) + validator.check_shape_length("Weight decay's shape", shp_len, 1, Rel.LE) + if shp_len == 1: + validator.check_integer("Weight decay's shape", weight_decay_shape[0], 1, Rel.EQ) + shp_len = len(learning_rate_shape) + validator.check_shape_length("Learning rate's shape", shp_len, 1, Rel.LE) + if shp_len == 1: + validator.check_integer("Learning rate's shape", learning_rate_shape[0], 1, Rel.EQ) + return weight_shape + + def infer_dtype(self, weight_dtype, gradient_dtype, norm_weight_dtype, norm_gradient_dtype, + weight_decay_dtype, learning_rate_dtype): + args = {"Weight dtype": weight_dtype, "gradient dtype": gradient_dtype, "norm weight dtype": norm_weight_dtype, + "norm gradient dtype": norm_gradient_dtype} + validator.check_type_same(args, [mstype.float16, mstype.float32, mstype.int16, mstype.int32]) + validator.check_args_tensor(args) + validator.check_typename("weight_decay_dtype", weight_decay_dtype, + [mstype.float16, mstype.float32, mstype.float64]) + validator.check_typename("learning_rate_dtype", learning_rate_dtype, + [mstype.float16, mstype.float32, mstype.float64]) + return weight_dtype + + +class ApplyFtrl(PrimitiveWithInfer): + """ + Update relevant entries according to the FTRL scheme. + + Args: + use_locking (bool): Use locks for update operation if True . Default: False. + + Inputs: + - **var** (Tensor): The variable to be updated. + - **accum** (Tensor): The accum to be updated, must be same type and shape as `var`. + - **linear** (Tensor): The linear to be updated, must be same type and shape as `var`. + - **grad** (Tensor): Gradient. + - **lr** (Union[Number, Tensor]): The learning rate value, must be positive. Default: 0.001. + - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. + Default: 0.0. + - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. + Default: 0.0. + - **lr_power** (Union[Number, Tensor]): Learning rate power controls how the learning rate decreases + during training, must be less than or equal to zero. Use fixed learning rate if lr_power is zero. + Default: -0.5. + + Outputs: + Tensor, representing the updated var. + """ + @prim_attr_register + def __init__(self, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'linear', 'grad', 'lr', 'l1', 'l2', 'lr_power'], + outputs=['output']) + self.use_locking = validator.check_type("use_locking", use_locking, [bool]) + + def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, lr_shape, l1_shape, l2_shape, + lr_power_shape): + validator.check_param_equal('var shape', var_shape, 'accum shape', accum_shape) + validator.check_param_equal('var shape', var_shape, 'linear shape', linear_shape) + return var_shape + + def infer_dtype(self, var_type, accum_type, linear_type, grad_type, lr_type, l1_type, l2_type, lr_power_type): + validator.check_subclass("var_type", var_type, mstype.tensor) + validator.check_subclass("accum_type", accum_type, mstype.tensor) + validator.check_subclass("linear_type", linear_type, mstype.tensor) + validator.check_subclass("grad_type", grad_type, mstype.tensor) + args = {'var_type': var_type, 'accum_type': accum_type, 'linear_type': linear_type, 'grad_type': grad_type} + validator.check_type_same(args, (mstype.float32, mstype.float16)) + + validator.check_typename("lr", lr_type,[mstype.float16, mstype.float32]) + validator.check_typename("l1", l1_type,[mstype.float16, mstype.float32]) + validator.check_typename("l2", l2_type,[mstype.float16, mstype.float32]) + validator.check_typename("lr_power", lr_power_type,[mstype.float16, mstype.float32]) + return var_type \ No newline at end of file diff --git a/mindspore/ops/operations/other_ops.py b/mindspore/ops/operations/other_ops.py new file mode 100644 index 00000000000..e4d526ad014 --- /dev/null +++ b/mindspore/ops/operations/other_ops.py @@ -0,0 +1,273 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Other operators.""" +from ..._c_expression import signature_rw as sig_rw +from ..._c_expression import signature_kind as sig_kind +from ..._checkparam import ParamValidator as validator, Rel +from ...common import dtype as mstype +from ..primitive import Primitive, PrimitiveWithInfer, prim_attr_register + + +class Assign(PrimitiveWithInfer): + """ + Assign `Parameter` with a value. + + Inputs: + - **variable** (Parameter) - The `Parameter`. + - **value** (Tensor) - The value to assign. + + Outputs: + Tensor, has the same type as original `variable`. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.y = mindspore.Parameter(Tensor([1.0], mindspore.float32), name="y") + >>> + >>> def construct(self, x): + >>> Assign()(self.y, x) + >>> return x + >>> x = Tensor([2.0], mindspore.float32) + >>> net = Net() + >>> net(x) + """ + __mindspore_signature__ = ( + ('variable', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD), + ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD) + ) + @prim_attr_register + def __init__(self): + pass + + def infer_shape(self, variable, value): + return variable + + def infer_dtype(self, variable, value): + return variable + + +class BoundingBoxEncode(PrimitiveWithInfer): + """ + Encode bounding boxes locations. + + Args: + means (tuple): Means for encoding bounding boxes calculation. Default: (0.0, 0.0, 0.0, 0.0). + stds (tuple): Stds for encoding bounding boxes calculation. Default: (1.0, 1.0, 1.0, 1.0). + + Inputs: + - **anchor_box** (Tensor) - Anchor boxes. + - **groundtruth_box** (Tensor) - Ground truth boxes. + + Outputs: + Tensor, encoded bounding boxes. + + Examples: + >>> boundingbox_encode = BoundingBoxEncode(means=(0.0, 0.0, 0.0, 0.0), stds=(1.0, 1.0, 1.0, 1.0)) + >>> delta_box = boundingbox_encode(anchor_box, groundtruth_box) + """ + + @prim_attr_register + def __init__(self, means=(0.0, 0.0, 0.0, 0.0), stds=(1.0, 1.0, 1.0, 1.0)): + validator.check_type('means', means, [tuple]) + validator.check_type('stds', stds, [tuple]) + validator.check("means len", len(means), '', 4) + validator.check("stds len", len(stds), '', 4) + + def infer_shape(self, anchor_box, groundtruth_box): + validator.check('anchor_box shape[0]', anchor_box[0], 'groundtruth_box shape[0]', groundtruth_box[0]) + validator.check('anchor_box shape[1]', anchor_box[1], '', 4) + validator.check('groundtruth_box shape[1]', groundtruth_box[1], '', 4) + return anchor_box + + def infer_dtype(self, anchor_box, groundtruth_box): + args = {"anchor_box": anchor_box, + "groundtruth_box": groundtruth_box + } + validator.check_type_same(args, mstype.number_type) + return anchor_box + + +class BoundingBoxDecode(PrimitiveWithInfer): + """ + Decode bounding boxes locations. + + Args: + means (tuple): The means of deltas calculation. Default: (0.0, 0.0, 0.0, 0.0). + stds (tuple): The standard deviations of deltas calculation. Default: (1.0, 1.0, 1.0, 1.0). + max_shape (tuple): The max size limit for decoding box calculation. + wh_ratio_clip (float): The limit of width and height ratio for decoding box calculation. Default: 0.016. + + Inputs: + - **anchor_box** (Tensor) - Anchor boxes. + - **deltas** (Tensor) - Delta of boxes. + + Outputs: + Tensor, decoded boxes. + + Examples: + >>> boundingbox_decode = BoundingBoxDecode(means=(0.0, 0.0, 0.0, 0.0), stds=(1.0, 1.0, 1.0, 1.0), + max_shape=(768, 1280), wh_ratio_clip=0.016) + >>> bbox = boundingbox_decode(anchor_box, deltas) + """ + + @prim_attr_register + def __init__(self, max_shape, means=(0.0, 0.0, 0.0, 0.0), stds=(1.0, 1.0, 1.0, 1.0), wh_ratio_clip=0.016): + validator.check_type('means', means, [tuple]) + validator.check_type('stds', stds, [tuple]) + validator.check_type('wh_ratio_clip', wh_ratio_clip, [float]) + validator.check("means", len(means), '', 4) + validator.check("stds", len(stds), '', 4) + if max_shape is not None: + validator.check_type('max_shape', max_shape, [tuple]) + validator.check("max_shape", len(max_shape), '', 2) + + def infer_shape(self, anchor_box, deltas): + validator.check('anchor_box shape[0]', anchor_box[0], 'deltas shape[0]', deltas[0]) + validator.check('anchor_box shape[1]', anchor_box[1], '', 4) + validator.check('deltas shape[1]', deltas[1], '', 4) + return anchor_box + + def infer_dtype(self, anchor_box, deltas): + args = {"anchor_box": anchor_box, + "deltas": deltas + } + validator.check_type_same(args, mstype.number_type) + return anchor_box + + +class CheckValid(PrimitiveWithInfer): + """ + Check bounding box. + + Check whether the bounding box cross data and data border. + + Inputs: + - **bboxes** (Tensor) - Bounding boxes tensor with shape (N, 4). + - **img_metas** (Tensor) - Raw image size information, format (height, width, ratio). + + Outputs: + Tensor, the valided tensor. + """ + + @prim_attr_register + def __init__(self): + self.init_prim_io_names(inputs=['bboxes', 'img_metas'], outputs=['output']) + + def infer_shape(self, bboxes_shape, metas_shape): + validator.check_shape_length("bboxes shape length", len(bboxes_shape), 2, Rel.EQ) + validator.check("bboxes_shape[-1]", bboxes_shape[-1], "", 4, Rel.EQ) + validator.check_shape_length("img_metas shape length", len(metas_shape), 1, Rel.EQ) + validator.check("img_metas shape[0]", metas_shape[0], "", 3, Rel.EQ) + return bboxes_shape[:-1] + + def infer_dtype(self, bboxes_type, metas_type): + return mstype.bool_ + + +class IOU(PrimitiveWithInfer): + r""" + Calculate intersection over union for boxes. + + Compute the intersection over union (IOU) or the intersection over foreground (IOF) based on the ground-truth and + predicted regions. + + .. math:: + \text{IOU} = \frac{\text{Area of Overlap}}{\text{Area of Union}} + + \text{IOF} = \frac{\text{Area of Overlap}}{\text{Area of Ground Truth}} + + Args: + mode (string): The mode is used to specify the calculation method, + now support 'iou' (intersection over union) or 'iof' + (intersection over foreground) mode. Default: 'iou'. + + Inputs: + - **anchor_boxes** (Tensor) - Anchor boxes, tensor of shape (N, 4). "N" indicates the number of anchor boxes, + and the value "4" refers to "x0", "x1", "y0", and "y1". + - **gt_boxes** (Tensor) - Ground truth boxes, tensor of shape (M, 4). "M" indicates the number of ground + truth boxes, and the value "4" refers to "x0", "x1", "y0", and "y1". + + Outputs: + Tensor, the 'iou' values, tensor of shape (M, N). + + Raises: + KeyError: When `mode` is not 'iou' or 'iof'. + + Examples: + >>> iou = IOU() + >>> anchor_boxes = Tensor(np.random.randint(1,5, [10, 4])) + >>> gt_boxes = Tensor(np.random.randint(1,5, [3, 4])) + >>> iou(anchor_boxes, gt_boxes) + """ + + @prim_attr_register + def __init__(self, mode='iou'): + if mode not in {'iou', 'iof'}: + raise KeyError("Mode only support 'iou' or 'iof'.") + self.init_prim_io_names(inputs=['anchor_boxes', 'gt_boxes'], outputs=['overlap']) + + def infer_shape(self, anchor_boxes, gt_boxes): + validator.check('gt_boxes shape[1]', gt_boxes[1], '', 4) + validator.check('anchor_boxes shape[1]', anchor_boxes[1], '', 4) + validator.check('anchor_boxes rank', len(anchor_boxes), '', 2) + validator.check('gt_boxes rank', len(gt_boxes), '', 2) + iou = [gt_boxes[0], anchor_boxes[0]] + return iou + + def infer_dtype(self, anchor_boxes, gt_boxes): + validator.check_subclass("anchor_boxes", anchor_boxes, mstype.tensor) + validator.check_subclass("gt_boxes", gt_boxes, mstype.tensor) + args = {"anchor_boxes": anchor_boxes, "gt_boxes": gt_boxes} + validator.check_type_same(args, (mstype.float16,)) + return anchor_boxes + + +class MakeRefKey(Primitive): + """ + Make a RefKey instance by string. RefKey stores the name of Parameter, can be passed through the functions, + and used for Assign target. + + Args: + tag (str): Parameter name to make the RefKey. + + Inputs: + No input. + + Outputs: + RefKeyType, made from the Parameter name. + + Examples: + >>> from mindspore.ops import functional as F + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.y = Parameter(Tensor(np.ones([6, 8, 10], np.int32)), name="y") + >>> self.make_ref_key = MakeRefKey("y") + >>> + >>> def construct(self, x): + >>> key = self.make_ref_key() + >>> ref = F.make_ref(key, x, self.y) + >>> return ref * x + >>> + >>> x = Tensor(np.ones([3, 4, 5], np.int32)) + >>> net = Net() + >>> net(x) + """ + + @prim_attr_register + def __init__(self, tag): + validator.check_type('tag', tag, (str,)) diff --git a/mindspore/ops/operations/random_ops.py b/mindspore/ops/operations/random_ops.py new file mode 100644 index 00000000000..c8f59e898d2 --- /dev/null +++ b/mindspore/ops/operations/random_ops.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Operators for random.""" + +from ..._checkparam import ParamValidator as validator +from ..._checkparam import Rel +from ...common import dtype as mstype +from ..primitive import PrimitiveWithInfer, prim_attr_register + + +class RandomChoiceWithMask(PrimitiveWithInfer): + """ + Generates a random samply as index tensor with a mask tensor from a given tensor. + + The input must be a tensor of rank >= 2, the first dimension specify the number of sample. + The index tensor and the mask tensor have the same and fixed shape. The index tensor denotes the index + of the nonzero sample, while the mask tensor denotes which element in the index tensor are valid. + + Args: + count (int): Number of items expected to get. Default: 256. + seed (int): Random seed. + seed2 (int): Random seed2. + + Inputs: + - **input_x** (Tensor) - The input tensor. + + Outputs: + Tuple, two tensors, the first one is the index tensor and the other one is the mask tensor. + + Examples: + >>> rnd_choice_mask = RandomChoiceWithMask() + >>> input_x = Tensor(np.ones(shape=[240000, 4]), ms.bool_) + >>> output_y, output_mask = rnd_choice_mask(input_x) + """ + + @prim_attr_register + def __init__(self, count=256, seed=0, seed2=0): + """Init RandomChoiceWithMask""" + validator.check_type("count", count, [int]) + validator.check_integer("count", count, 0, Rel.GT) + validator.check_type('seed', seed, [int]) + validator.check_type('seed2', seed2, [int]) + + def infer_shape(self, x_shape): + validator.check_shape_length("input_x shape", len(x_shape), 1, Rel.GE) + return ([self.count, len(x_shape)], [self.count]) + + def infer_dtype(self, x_dtype): + validator.check_subclass('x_dtype', x_dtype, mstype.tensor) + validator.check_typename('x_dtype', x_dtype, [mstype.bool_]) + return (mstype.int32, mstype.bool_) diff --git a/mindspore/ops/primitive.py b/mindspore/ops/primitive.py new file mode 100644 index 00000000000..d281b4f76ca --- /dev/null +++ b/mindspore/ops/primitive.py @@ -0,0 +1,337 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""primitive""" + +import inspect +import copy +from mindspore.common.api import _wrap_func +from .._c_expression import Primitive_, real_run_op, prim_type +from .._c_expression import signature_rw as sig_rw +from .._c_expression import signature_kind as sig_kind +from .._c_expression import signature_dtype as sig_dtype + + +class Primitive(Primitive_): + """ + Primitive is base class for primitives in python. + + Args: + name (str): Name for current Primitive. + + Examples: + >>> add = Primitive('add') + >>> + >>> # or work with prim_attr_register: + >>> # init a Primitive class with attr1 and attr2 + >>> class Add(Primitive): + >>> @prim_attr_register + >>> def __init__(self, attr1, attr2): + >>> # check attr1 and attr2 or do some initializations + >>> # init a Primitive obj with attr1=1 and attr2=2 + >>> add = Add(attr1=1, attr2=2) + """ + + def __init__(self, name): + self.name = name + self.attrs = {} + self.init_attrs = {} + Primitive_.__init__(self, name, self) + if hasattr(self.__class__, '__mindspore_signature__'): + sig = self._fill_signature(self.__class__.__mindspore_signature__) + self.set_signatures(sig) + + def _fill_signature(self, signatures): + """fills signature.""" + signatures_new = [] + for signature in signatures: + if isinstance(signature, sig_dtype): + signatures_new.append(("argument", sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, + sig_kind.KIND_EMPTY_DEFAULT_VALUE, signature)) + else: + if len(signature) < 3: + raise ValueError(f"[Internal Error]Signature for one parameter len must > 3, but {signature}") + if len(signature) == 3: + signature += (sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T_EMPTY_DEFAULT_VALUE) + if len(signature) == 4: + signature += (sig_dtype.T_EMPTY_DEFAULT_VALUE,) + signatures_new.append(signature) + return tuple(signatures_new) + + def _clone(self): + """ + Deeply clones the primitive object. + + Calls the __init__() method with the same arguments. This method is called in parser if the + flag self.__setattr_flag__ is True. + """ + cloned = copy.deepcopy(self) + init_params = inspect.getfullargspec(cloned.__init__.decorated_func).args[1:] + init_args = {} + for name in init_params: + value = self.attrs[name] + init_args[name] = value + # __init__ should be called to construct cpp object. + cloned.__init__(**init_args) + for name in self.attrs: + value = self.attrs[name] + cloned.add_prim_attr(name, value) + return cloned + + def add_prim_attr(self, name, value): + """ + Adds primitive attribute. + + Args: + name (str): Attribute Name. + value (Any): Attribute value. + """ + self.__dict__[name] = value + self.attrs[name] = value + self.add_attr(name, value) + return self + + def set_strategy(self, strategy): + """ + Adds strategy to primitive attribute. + + Note: + Valid only in semi auto parallel or auto parallel mode. + + Args: + strategy (tuple): Strategy describes the distributed parallel mode of the current primitive. + """ + self.add_prim_attr("strategy", strategy) + return self + + def set_prim_instance_name(self, instance_name): + """ + Sets instance name to primitive operator. + + Note: + Will be called by default when user defines primitive operator. + + Args: + instance_name (str): Instance name of primitive operator set by user. + """ + self.set_instance_name(instance_name) + self.instance_name = instance_name + return self + + def __getattr__(self, item): + if item in super().get_attr_dict(): + return super().get_attr_dict()[item] + if item in self.attrs: + return self.attrs[item] + raise AttributeError(item) + + def __call__(self, *args): + output = _run_op(self, self.name, args) + return output + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, d): + self.__dict__.update(d) + + def init_prim_io_names(self, inputs, outputs): + """ + Initializes inputs and outpus name of Tensor or attributes. + + Args: + inputs (list[str]): list of inputs names. + outputs (list[str]): list of outputs names. + """ + # for checking para names with kernel implementation + self.add_prim_attr("input_names", inputs) + # for checking output number with kernel implementation + self.add_prim_attr("output_names", outputs) + + +class PrimitiveWithInfer(Primitive): + """ + PrimitiveWithInfer is base class for primitives in python and defines functions for infer of tracks in python. + + There are four method can be overide to define the infer logic of the primitive: __infer__(), infer_shape(), + infer_dtype(), and infer_value(). If __infer__() is defined in primitive, the __infer__() has highest priority + to be called. If __infer__() is not defined, infer_shape() and infer_dtype() can be defined to describle shape + and type infer logic. The infer_value() is used for constant propogation. + + Args: + name (str): Name for current Primitive. + + Examples: + >>> # init a Primitive class with infer + >>> class Add(PrimitiveWithInfer): + >>> @prim_attr_register + >>> def __init__(self): + >>> pass + >>> + >>> def infer_shape(self, x, y): + >>> return x # output shape same as first input 'x' + >>> + >>> def infer_dtype(self, x, y): + >>> return x # output type same as first input 'x' + >>> + >>> # init a Primitive obj + >>> add = Add() + """ + + def __init__(self, name): + Primitive.__init__(self, name) + self.set_prim_type(prim_type.py_infer_shape) + + def _clone(self): + """ + Deeply clones the primitive object. + + Calls the __init__() method with the same arguments. This method is called in parser if the + flag self.__setattr_flag__ is True. + """ + cloned_prim = Primitive._clone(self) + return cloned_prim + + def infer_shape(self, *args): + """ + Infer output shape based on input shape. + + Args: + inputs (tuple(int)): dimensions of input tensors. + outputs (tuple(int)): dimensions of output tensors. + + Note: + The shape of scalar is an empty tuple. + """ + return None + + def infer_dtype(self, *args): + """ + Infer output dtype based on input dtype. + + Args: + inputs (mstype): data type of inputs. + outputs (mstype): data type of outputs. + """ + return None + + def infer_value(self, *args): + """ + Infer output value based on input value at compile time. + + Args: + inputs (any): value of inputs. + outputs (any): value of outputs. + """ + return None + + def __infer__(self, *args): + """Infer shape, type, and value at the same time by using dictionary as arguments.""" + tracks = ['dtype', 'shape', 'value'] + out = {} + for track in tracks: + fn = getattr(self, 'infer_' + track) + # fn may return None + out[track] = fn(*(x[track] for x in args)) + return out + + +def prim_attr_register(fn): + """ + Primitive attributes register. + + Registering the decorator of the built-in operator primitive __init__ + function will add all the parameters of __init__ as operator attributes. + + Args: + fn (function): __init__ function of primitive. + + Returns: + function, original function. + """ + def deco(self, *args, **kwargs): + if isinstance(self, PrimitiveWithInfer): + PrimitiveWithInfer.__init__(self, self.__class__.__name__) + else: + Primitive.__init__(self, self.__class__.__name__) + bound_args = inspect.signature(fn).bind(self, *args, **kwargs) + bound_args.apply_defaults() + arguments = bound_args.arguments + del arguments['self'] + for name in arguments: + value = arguments[name] + self.add_prim_attr(name, value) + self.init_attrs[name] = value + fn(self, *args, **kwargs) + deco.decorated_func = fn + return deco + + +def constexpr(fn=None, get_instance=True, name=None): + """ + Makes a PrimitiveWithInfer operator, which infer the value while compiling. + + Args: + fn (function): A `fn` use as the infer_value of the output operator. + get_instance (bool): If true, returns the instance of operator, else returns the operator class. + name (str): Defines the operator name. If `name` is None, use the function name as op name. + + Examples: + >>> a = (1, 2) + >>> # make a operator to calculate tuple len + >>> @constexpr + >>> def tuple_len(x): + >>> return len(x) + >>> assert tuple_len(a) == 2 + >>> + >>> # make a operator class to calculate tuple len + >>> @constexpr(get_instance=False, name="TupleLen") + >>> def tuple_len_class(x): + >>> return len(x) + >>> assert tuple_len_class()(a) == 2 + """ + def deco(fn): + class CompileOp(PrimitiveWithInfer): + def __init__(self): + op_name = name if name else fn.__name__ + PrimitiveWithInfer.__init__(self, op_name) + + def infer_value(self, *args): + return fn(*args) + if get_instance: + return CompileOp() + return CompileOp + if fn is not None: + return deco(fn) + return deco + + +@_wrap_func +def _run_op(obj, op_name, args): + """Single op execution function supported by ge in PyNative mode.""" + op_mask = [0] * len(args) + op_inputs = [] + for i, arg in enumerate(args): + if hasattr(arg, '__parameter__'): + op_inputs.append(arg.default_input) + op_mask[i] = 1 + else: + op_inputs.append(arg) + output = real_run_op(obj, op_name, tuple(op_inputs), tuple(op_mask)) + if not output: + raise RuntimeError("Pynative run op %s failed!" % op_name) + if len(output) == 1: + output = output[0] + return output diff --git a/mindspore/ops/vm_impl_registry.py b/mindspore/ops/vm_impl_registry.py new file mode 100644 index 00000000000..62176164835 --- /dev/null +++ b/mindspore/ops/vm_impl_registry.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +User debug operators registry. + +User can define a python implementation of primitive operator and +register it with the registry, this mechanism applied for debugging currently. +""" + +from ._register_for_op import Registry + +vm_impl_registry = Registry() +""" +Register the python primitive debug implementation function of a primitive operator. + +Examples: + >>> @vm_impl_registry.register(P.Type) + >>> def vm_impl_dtype(self): + >>> def vm_impl(x): + >>> return type(x) + >>> return vm_impl +""" + + +def get_vm_impl_fn(prim): + """ + Gets vm function by primitive obj or primitive name for c++ + + Args: + prim (Union[Primitive, str]): primitive obj or primitive name for operator register by name. + + Returns: + function, vm function + """ + out = vm_impl_registry.get(prim, None) + if out: + return out(prim) + return None diff --git a/mindspore/parallel/__init__.py b/mindspore/parallel/__init__.py new file mode 100644 index 00000000000..c79704f1104 --- /dev/null +++ b/mindspore/parallel/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This interface is ONLY used in Auto-parallel procedure. +""" +from .dp_allreduce_fusion import set_fusion_strategy_by_idx, set_fusion_strategy_by_size +from .algo_parameter_config import get_algo_parameters, reset_algo_parameters, \ + set_algo_parameters + +__all__ = ["set_fusion_strategy_by_idx", "set_fusion_strategy_by_size", "get_algo_parameters", + "reset_algo_parameters", "set_algo_parameters"] diff --git a/mindspore/parallel/_auto_parallel_context.py b/mindspore/parallel/_auto_parallel_context.py new file mode 100644 index 00000000000..3564ad4395f --- /dev/null +++ b/mindspore/parallel/_auto_parallel_context.py @@ -0,0 +1,381 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Context of auto parallel""" +import threading +from mindspore._c_expression import AutoParallelContext +from mindspore._extends.pynative_helper import args_type_check + + +class _AutoParallelContext: + """ + _AutoParallelContext is the environment in which operations are executed + + Note: + Create a context through instantiating Context object is not recommended. + Should use auto_parallel_context() to get the context since Context is singleton. + """ + _instance = None + _instance_lock = threading.Lock() + + def __init__(self): + self._context_handle = AutoParallelContext.get_instance() + + def __new__(cls): + if cls._instance is None: + cls._instance_lock.acquire() + cls._instance = object.__new__(cls) + cls._instance_lock.release() + return cls._instance + + def check_context_handle(self): + """ + Check context handle. + + Raises: + ValueError: If the context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + + def set_device_num(self, device_num): + """ + Set device num for auto parallel. + + Args: + device_num (int): The device number. + + Raises: + ValueError: If the device num is not in [1, 4096]. + """ + self.check_context_handle() + if device_num < 1 or device_num > 4096: + raise ValueError("Device num must be in [1, 4096], but got {}".format(device_num)) + self._context_handle.set_device_num(device_num) + + def get_device_num(self): + """Get device num.""" + self.check_context_handle() + return self._context_handle.get_device_num() + + def set_global_rank(self, global_rank): + """ + Set global rank for auto parallel. + + Args: + global_rank (int): The rank id of current rank. + + Raises: + ValueError: If the global rank is not in [1, 4096]. + """ + self.check_context_handle() + if global_rank < 0 or global_rank > 4095: + raise ValueError("Global rank must be in [0, 4095], but got {}".format(global_rank)) + self._context_handle.set_global_rank(global_rank) + + def get_global_rank(self): + """Get current rank id.""" + self.check_context_handle() + return self._context_handle.get_global_rank() + + def set_mirror_mean(self, mirror_mean): + """ + Set mirror_mean flag. + + Note: + If mirror_mean is true, it will insert a div operator after parameter gradients allreduce. + + Args: + mirror_mean (bool): The mirror_mean flag. + """ + self.check_context_handle() + self._context_handle.set_mirror_mean(mirror_mean) + + def get_mirror_mean(self): + """Get mirror_mean flag.""" + self.check_context_handle() + return self._context_handle.get_mirror_mean() + + def set_cast_before_mirror(self, cast_before_mirror): + """ + Set cast_before_mirror. + + Note: + If cast_before_mirror is true, + it will convert tensor type from fp16 to fp32 before parameter gradients allreduce. + + Args: + cast_before_mirror (bool): The cast_before_mirror flag. + """ + self.check_context_handle() + self._context_handle.set_cast_before_mirror(cast_before_mirror) + + def get_cast_before_mirror(self): + """Get cast_before_mirror flag.""" + self.check_context_handle() + return self._context_handle.get_cast_before_mirror() + + def set_loss_repeated_mean(self, loss_repeated_mean): + """ + Set loss_repeated_mean flag. + + Note: + If loss_repeated_mean is true, + Distributed automatic differentiation will perform a mean operator + in backward in the case of repeated calculations. + + Args: + loss_repeated_mean (bool): The loss_repeated_mean flag. + """ + self.check_context_handle() + self._context_handle.set_loss_repeated_mean(loss_repeated_mean) + + def get_loss_repeated_mean(self): + """Get loss_repeated_mean flag.""" + self.check_context_handle() + return self._context_handle.get_loss_repeated_mean() + + def set_communication_backend(self, communication_backend): + """ + Set communication backend. + + Args: + communication_backend (str): The communication backend. + """ + self.check_context_handle() + self._context_handle.set_communication_backend(communication_backend) + + def get_communication_backend(self): + """Get communication backend.""" + self.check_context_handle() + return self._context_handle.get_communication_backend() + + def set_parallel_mode(self, parallel_mode): + """ + Set parallel mode for auto parallel. + + Args: + parallel_mode (str): The parallel mode of auto parallel. + + Raises: + ValueError: If parallel mode is not supported. + """ + self.check_context_handle() + ret = self._context_handle.set_parallel_mode(parallel_mode) + if ret is False: + raise ValueError("Parallel mode does not support {}".format(parallel_mode)) + + def get_parallel_mode(self): + """Get parallel mode.""" + self.check_context_handle() + return self._context_handle.get_parallel_mode() + + def set_strategy_search_mode(self, strategy_search_mode): + self.check_context_handle() + ret = self._context_handle.set_strategy_search_mode(strategy_search_mode) + if ret is False: + raise ValueError("Strategy search mode does not support {}".format(strategy_search_mode)) + + def get_strategy_search_mode(self): + self.check_context_handle() + return self._context_handle.get_strategy_search_mode() + + def set_parameter_broadcast(self, parameter_broadcast): + """ + Set parameter broadcast. + + Args: + parameter_broadcast (bool): Parameter broadcast or not. + """ + self.check_context_handle() + self._context_handle.set_parameter_broadcast(parameter_broadcast) + + def get_parameter_broadcast(self): + """Get parameter broadcast flag.""" + self.check_context_handle() + return self._context_handle.get_parameter_broadcast() + + def get_parameter_broadcast_is_set(self): + """Get parameter broadcast is set or not.""" + self.check_context_handle() + return self._context_handle.get_parameter_broadcast_is_set() + + def set_all_reduce_fusion_split_indices(self, indices): + """ + Set allreduce fusion strategy by parameters indices. + + Args: + indices (list): Indices list. + + Raises: + ValueError: If type of indices item is not int. + """ + self.check_context_handle() + for index in indices: + if not isinstance(index, int): + raise TypeError('indices has invalid value') + return self._context_handle.set_all_reduce_fusion_split_indices(indices) + + def get_all_reduce_fusion_split_indices(self): + """Get allreduce fusion split indices.""" + self.check_context_handle() + return self._context_handle.get_all_reduce_fusion_split_indices() + + def set_all_reduce_fusion_split_sizes(self, sizes): + """ + Set allreduce fusion strategy by parameters data sizes. + + Args: + sizes (list): Sizes list. + + Raises: + ValueError: If type of sizes item is not int. + """ + self.check_context_handle() + for size in sizes: + if not isinstance(size, int): + raise TypeError('sizes has invalid value') + return self._context_handle.set_all_reduce_fusion_split_sizes(sizes) + + def get_all_reduce_fusion_split_sizes(self): + """Get allreduce fusion split sizes.""" + self.check_context_handle() + return self._context_handle.get_all_reduce_fusion_split_sizes() + + def get_device_num_is_set(self): + """Get device number is set or not.""" + self.check_context_handle() + return self._context_handle.get_device_num_is_set() + + def get_global_rank_is_set(self): + """Get global rank is set or not.""" + self.check_context_handle() + return self._context_handle.get_global_rank_is_set() + + def reset(self): + """Reset all settings.""" + self.check_context_handle() + self._context_handle.reset() + + +_auto_parallel_context = None + + +def auto_parallel_context(): + """ + Get the global _auto_parallel_context, if it is not created, create a new one. + + Returns: + _AutoParallelContext, the global auto parallel context. + """ + global _auto_parallel_context + if _auto_parallel_context is None: + _auto_parallel_context = _AutoParallelContext() + return _auto_parallel_context + + +_set_auto_parallel_context_func_map = { + "device_num": auto_parallel_context().set_device_num, + "global_rank": auto_parallel_context().set_global_rank, + "mirror_mean": auto_parallel_context().set_mirror_mean, + "cast_before_mirror": auto_parallel_context().set_cast_before_mirror, + "loss_repeated_mean": auto_parallel_context().set_loss_repeated_mean, + "parallel_mode": auto_parallel_context().set_parallel_mode, + "parameter_broadcast": auto_parallel_context().set_parameter_broadcast} + + +_get_auto_parallel_context_func_map = { + "device_num": auto_parallel_context().get_device_num, + "global_rank": auto_parallel_context().get_global_rank, + "mirror_mean": auto_parallel_context().get_mirror_mean, + "cast_before_mirror": auto_parallel_context().get_cast_before_mirror, + "loss_repeated_mean": auto_parallel_context().get_loss_repeated_mean, + "parallel_mode": auto_parallel_context().get_parallel_mode, + "parameter_broadcast": auto_parallel_context().get_parameter_broadcast} + + +@args_type_check(device_num=int, global_rank=int, mirror_mean=bool, cast_before_mirror=bool, + loss_repeated_mean=bool, parallel_mode=str, parameter_broadcast=bool) +def _set_auto_parallel_context(**kwargs): + """ + Set auto parallel context. + + Note: + Attribute name is required for setting attributes. + + Args: + device_num (int): Available device number, the value must be in [1, 4096]. Default: 1. + global_rank (int): Global rank id, the value must be in [0, 4095]. Default: 0. + mirror_mean (bool): Whether to perform mean operator after all-reduce of mirror. Default: False. + loss_repeated_mean (bool): Whether to perform mean operator in backward in the case of repeated + calculations. Default: True. + cast_before_mirror (bool): Insert Mirror Op after the cast if this flag is True. Default: True. + parallel_mode (str): There are five kinds of parallel modes, "stand_alone", "data_parallel", + "hybrid_parallel", "semi_auto_parallel" and "auto_parallel". Default: "stand_alone". + + - stand_alone: Only one processor working. + + - data_parallel: Distributing the data across different processors. + + - hybrid_parallel: Achieving data parallelism and model parallelism manually. + + - semi_auto_parallel: Achieving data parallelism and model parallelism by + setting parallel strategies. + + - auto_parallel: Achieving parallelism automatically. + parameter_broadcast (bool): Indicating whether to broadcast parameters before training. + "stand_alone", "semi_auto_parallel" and "auto_parallel" do not support parameter + broadcast. Default: False. + + Raises: + ValueError: If input key is not attribute in auto parallel context. + """ + for key, value in kwargs.items(): + if key not in _set_auto_parallel_context_func_map: + raise ValueError("Set context keyword %s is not recognized!" % key) + set_func = _set_auto_parallel_context_func_map[key] + set_func(value) + + +def _get_auto_parallel_context(attr_key): + """ + Get auto parallel context attribute value according to the key. + + Args: + attr_key (str): The key of the attribute. + + Returns: + Return attribute value according to the key. + + Raises: + ValueError: If input key is not attribute in auto parallel context. + """ + if attr_key not in _get_auto_parallel_context_func_map: + raise ValueError("Get context keyword %s is not recognized!" % attr_key) + get_func = _get_auto_parallel_context_func_map[attr_key] + return get_func() + + +def _reset_auto_parallel_context(): + """ + Reset auto parallel context attributes to the default values: + + - device_num: 1. + - global_rank: 0. + - mirror_mean: False. + - cast_before_mirror: True. + - parallel_mode: "stand_alone". + - parameter_broadcast: False. + """ + auto_parallel_context().reset() diff --git a/mindspore/parallel/_cell_wrapper.py b/mindspore/parallel/_cell_wrapper.py new file mode 100644 index 00000000000..6533e8aee4f --- /dev/null +++ b/mindspore/parallel/_cell_wrapper.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Cell of auto parallel""" + +from mindspore.nn.cell import Cell +from mindspore.ops.operations.comm_ops import AllGather + + +_allgather_cell = None + + +class AllGatherCell(Cell): + """ + Allgather cell, used in model parallel scenario. + To allgather the selected parameter slice from each device. + """ + def __init__(self): + super(AllGatherCell, self).__init__(auto_prefix=False) + + self.allgather = AllGather() + + def construct(self, x): + x = self.allgather(x) + + return x + + +def get_allgather_cell(): + """Get AllGatherCell object.""" + global _allgather_cell + if not _allgather_cell: + _allgather_cell = AllGatherCell() + + return _allgather_cell + + +def destroy_allgather_cell(): + """Destroy AllGatherCell object.""" + global _allgather_cell + if _allgather_cell: + _allgather_cell = None diff --git a/mindspore/parallel/_cost_model_context.py b/mindspore/parallel/_cost_model_context.py new file mode 100644 index 00000000000..0920d66f419 --- /dev/null +++ b/mindspore/parallel/_cost_model_context.py @@ -0,0 +1,534 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Context of cost_model in auto_parallel""" +import threading +from mindspore._c_expression import CostModelContext +from mindspore._extends.pynative_helper import args_type_check + + +class _CostModelContext: + """ + _CostModelContext is the environment in which operations are executed + + Note: + Creating a context through instantiating Context object is not recommended. + Use cost_model_context() to get the context since Context is singleton. + """ + _instance = None + _instance_lock = threading.Lock() + + def __init__(self): + self._context_handle = CostModelContext.get_instance() + + def __new__(cls): + if cls._instance is None: + cls._instance_lock.acquire() + cls._instance = object.__new__(cls) + cls._instance_lock.release() + return cls._instance + + def set_device_memory_capacity(self, dev_mem_cap): + """ + Set device memory capacity. + + Args: + dev_mem_cap (float): The memory capacity for each device. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_device_memory_capacity(dev_mem_cap) + + def get_device_memory_capacity(self): + """ + Get device memory capacity. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_device_memory_capacity() + + def set_costmodel_alpha(self, alpha): + """ + Set costmodel alpha. + + Args: + alpha (float): The parameter costmodel_alpha used in strategy-searching algorithm. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_alpha(alpha) + + def get_costmodel_alpha(self): + """ + Get costmodel alpha. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_alpha() + + def set_costmodel_beta(self, beta): + """ + Set costmodel beta. + + Args: + beta (float): The parameter costmodel_beta used in strategy-searching algorithm. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_beta(beta) + + def get_costmodel_beta(self): + """ + Get costmodel beta. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_beta() + + def set_costmodel_gamma(self, gamma): + """ + Set costmodel gamma. + + Args: + gamma (float): The parameter costmodel_gamma used in strategy-searching algorithm. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_gamma(gamma) + + def get_costmodel_gamma(self): + """ + Get costmodel gamma. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_gamma() + + def set_costmodel_communi_threshold(self, threshold): + """ + Set costmodel communication threshold. + + Args: + threshold (float): A parameter used in adjusting communication calculation for practice. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_communi_threshold(threshold) + + def get_costmodel_communi_threshold(self): + """ + Get costmodel communication threshold. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_communi_threshold() + + def set_costmodel_communi_const(self, communi_const): + """ + Set costmodel communication const. + + Args: + const (float): A parameter used in adjusting communication calculation for practice. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_communi_const(communi_const) + + def get_costmodel_communi_const(self): + """ + Get costmodel communication const. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_communi_const() + + def set_costmodel_communi_bias(self, communi_bias): + """ + Set costmodel communication bias. + + Args: + bias (float): A parameter used in adjusting communication calculation for practice. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_communi_bias(communi_bias) + + def get_costmodel_communi_bias(self): + """ + Get costmodel communication bias. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_communi_bias() + + def set_costmodel_allreduce_fusion_algorithm(self, algorithm): + """ + Set costmodel allreduce fusion algorithm. + + Args: + algorithm (int): The AllReduce fusion algorithm of parameter gradients. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_algorithm(algorithm) + + def get_costmodel_allreduce_fusion_algorithm(self): + """ + Get costmodel allreduce fusion algorithm. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_algorithm() + + def set_costmodel_allreduce_fusion_times(self, allreduce_fusion_times): + """ + Set costmodel allreduce fusion times. + + Args: + allreduce_fusion_times (int): The AllReduce fusion times of parameter gradients. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_times(allreduce_fusion_times) + + def get_costmodel_allreduce_fusion_times(self): + """ + Get costmodel allreduce fusion times. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_times() + + def set_costmodel_allreduce_fusion_tail_percent(self, tail_percent): + """ + Set costmodel allreduce fusion tail percent. + + Args: + tail_percent (int): The percentage of backward computing time corresponding to the last parameter gradients + AllReduce in the whole backward computing time. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_tail_percent(tail_percent) + + def get_costmodel_allreduce_fusion_tail_percent(self): + """ + Get costmodel allreduce fusion tail percent. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_tail_percent() + + def set_costmodel_allreduce_fusion_tail_time(self, tail_time): + """ + Set costmodel allreduce fusion tail time. + + Args: + tail_time (int): The tail time of the last parameter gradients AllReduce after the end of backward + computation. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_tail_time(tail_time) + + def get_costmodel_allreduce_fusion_tail_time(self): + """ + Get costmodel allreduce fusion tail time. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_tail_time() + + def set_costmodel_allreduce_fusion_allreduce_inherent_time(self, allreduce_inherent_time): + """ + Set costmodel allreduce fusion allreduce inherent time. + + Args: + allreduce_inherent_time (int): The inherent cost time of AllReduce. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_allreduce_inherent_time(allreduce_inherent_time) + + def get_costmodel_allreduce_fusion_allreduce_inherent_time(self): + """ + Get costmodel allreduce fusion allreduce inherent time. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_allreduce_inherent_time() + + def set_costmodel_allreduce_fusion_allreduce_bandwidth(self, allreduce_bandwidth): + """ + Set costmodel allreduce fusion allreduce bandwidth. + + Args: + allreduce_bandwidth (int): The bancwidth of AllReduce. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_allreduce_bandwidth(allreduce_bandwidth) + + def get_costmodel_allreduce_fusion_allreduce_bandwidth(self): + """ + Get costmodel allreduce fusion allreduce bandwidth. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_allreduce_bandwidth() + + def set_costmodel_allreduce_fusion_computation_time_parameter(self, computation_time_parameter): + """ + Set costmodel allreduce fusion computation time parameter. + + Args: + computation_time_parameter (int): The parameter used to compute backward computation time. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.set_costmodel_allreduce_fusion_computation_time_parameter(computation_time_parameter) + + def get_costmodel_allreduce_fusion_computation_time_parameter(self): + """ + Get costmodel allreduce fusion computation time parameter. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + return self._context_handle.get_costmodel_allreduce_fusion_computation_time_parameter() + + def reset_cost_model(self): + """ + Reset cost model settings. + + Raises: + ValueError: If context handle is none. + """ + if self._context_handle is None: + raise ValueError("Context handle is none in context!!!") + self._context_handle.reset_cost_model() + + +_cost_model_context = None + + +def cost_model_context(): + """ + Get the global _cost_model_context. If it is not created, create a new one. + + Returns: + The global cost_model context. + """ + global _cost_model_context + if _cost_model_context is None: + _cost_model_context = _CostModelContext() + return _cost_model_context + + +set_cost_model_context_func_map = { + "device_memory_capacity": cost_model_context().set_device_memory_capacity, + "costmodel_alpha": cost_model_context().set_costmodel_alpha, + "costmodel_beta": cost_model_context().set_costmodel_beta, + "costmodel_gamma": cost_model_context().set_costmodel_gamma, + "costmodel_communi_threshold": cost_model_context().set_costmodel_communi_threshold, + "costmodel_communi_const": cost_model_context().set_costmodel_communi_const, + "costmodel_communi_bias": cost_model_context().set_costmodel_communi_bias, + "costmodel_allreduce_fusion_algorithm": cost_model_context().set_costmodel_allreduce_fusion_algorithm, + "costmodel_allreduce_fusion_times": cost_model_context().set_costmodel_allreduce_fusion_times, + "costmodel_allreduce_fusion_tail_percent": cost_model_context().set_costmodel_allreduce_fusion_tail_percent, + "costmodel_allreduce_fusion_tail_time": cost_model_context().set_costmodel_allreduce_fusion_tail_time, + "costmodel_allreduce_fusion_allreduce_inherent_time": + cost_model_context().set_costmodel_allreduce_fusion_allreduce_inherent_time, + "costmodel_allreduce_fusion_allreduce_bandwidth": + cost_model_context().set_costmodel_allreduce_fusion_allreduce_bandwidth, + "costmodel_allreduce_fusion_computation_time_parameter": + cost_model_context().set_costmodel_allreduce_fusion_computation_time_parameter} + + +get_cost_model_context_func_map = { + "device_memory_capacity": cost_model_context().get_device_memory_capacity, + "costmodel_alpha": cost_model_context().get_costmodel_alpha, + "costmodel_beta": cost_model_context().get_costmodel_beta, + "costmodel_gamma": cost_model_context().get_costmodel_gamma, + "costmodel_communi_threshold": cost_model_context().get_costmodel_communi_threshold, + "costmodel_communi_const": cost_model_context().get_costmodel_communi_const, + "costmodel_communi_bias": cost_model_context().get_costmodel_communi_bias, + "costmodel_allreduce_fusion_algorithm": cost_model_context().get_costmodel_allreduce_fusion_algorithm, + "costmodel_allreduce_fusion_times": cost_model_context().get_costmodel_allreduce_fusion_times, + "costmodel_allreduce_fusion_tail_percent": cost_model_context().get_costmodel_allreduce_fusion_tail_percent, + "costmodel_allreduce_fusion_tail_time": cost_model_context().get_costmodel_allreduce_fusion_tail_time, + "costmodel_allreduce_fusion_allreduce_inherent_time": + cost_model_context().get_costmodel_allreduce_fusion_allreduce_inherent_time, + "costmodel_allreduce_fusion_allreduce_bandwidth": + cost_model_context().get_costmodel_allreduce_fusion_allreduce_bandwidth, + "costmodel_allreduce_fusion_computation_time_parameter": + cost_model_context().get_costmodel_allreduce_fusion_computation_time_parameter} + + +@args_type_check(device_memory_capacity=float, costmodel_alpha=float, costmodel_beta=float, costmodel_gamma=float, + costmodel_communi_threshold=float, costmodel_communi_const=float, costmodel_communi_bias=float, + costmodel_allreduce_fusion_algorithm=int, costmodel_allreduce_fusion_times=int, + costmodel_allreduce_fusion_tail_percent=float, costmodel_allreduce_fusion_tail_time=float, + costmodel_allreduce_fusion_allreduce_inherent_time=float, + costmodel_allreduce_fusion_allreduce_bandwidth=float, + costmodel_allreduce_fusion_computation_time_parameter=float) +def set_cost_model_context(**kwargs): + """ + Set cost model context. + + Note: + Attribute name is needed. + + Args: + device_memory_capacity (float): The memory capacity for each device. + costmodel_alpha (float): The parameter costmodel_alpha used in strategy-searching algorithm. + costmodel_beta (float): The parameter costmodel_beta used in strategy-searching algorithm. + costmodel_gamma (float): The parameter costmodel_gamma used in strategy-searching algorithm. + costmodel_communi_threshold (float): A parameter used in adjusting communication calculation for practice. + costmodel_communi_const (float): A parameter used in adjusting communication calculation for practice. + costmodel_communi_bias (float): A parameter used in adjusting communication calculation for practice. + costmodel_allreduce_fusion_algorithm (int): The allreduce fusion algorithm. + 0: bypass allreduce fusion; + 1: only use backward computation time to group allreduce; + 2: use backward computation time and parameter gradient allreduce time to group allreduce. + costmodel_allreduce_fusion_times (int): The AllReduce fusion times of parameter gradients. + costmodel_allreduce_fusion_tail_percent (float): A parameter used in allreduce fusion algorithm. The percentage + of backward computing time corresponding to the last parameter gradients AllReduce in the whole backward + computing time. + costmodel_allreduce_fusion_tail_time (float): A parameter used in allreduce fusion algorithm. The tail time of + the last parameter gradients AllReduce after the end of backward computation. + costmodel_allreduce_fusion_allreduce_inherent_time (float): A parameter used in allreduce fusion algorithm. The + inherent cost time of AllReduce. + costmodel_allreduce_fusion_allreduce_bandwidth (float): A parameter used in allreduce fusion algorithm. The + bandwidth of AllReduce. + costmodel_allreduce_fusion_computation_time_parameter (float): A parameter used in allreduce fusion algorithm. + The parameter used to compute backward computation time. + + + + Raises: + ValueError: If context keyword is not recognized. + """ + for key, value in kwargs.items(): + if key not in set_cost_model_context_func_map: + raise ValueError("Set context keyword %s is not recognized!" % key) + set_func = set_cost_model_context_func_map[key] + set_func(value) + + +def get_cost_model_context(attr_key): + """ + Get cost model context attributes. + + Note: + Return value according to the attribute value. + + Args: + attr_key (str): The key of the attribute. + + Raises: + ValueError: If context keyword is not recognized. + """ + if attr_key not in get_cost_model_context_func_map: + raise ValueError("Get context keyword %s is not recognized!" % attr_key) + get_func = get_cost_model_context_func_map[attr_key] + return get_func() + + +def reset_cost_model_context(): + """Reset cost model context attributes.""" + cost_model_context().reset_cost_model() diff --git a/mindspore/parallel/_tensor.py b/mindspore/parallel/_tensor.py new file mode 100644 index 00000000000..3d314cf724d --- /dev/null +++ b/mindspore/parallel/_tensor.py @@ -0,0 +1,277 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""load tensor and combine tensor""" +import numpy as np + +from mindspore.common.tensor import Tensor +from ..communication.management import get_rank + + +def _get_tensor_strategy(dev_mat, tensor_map): + """ + Get split strategy by device arrangement and tensor map. + + Args: + dev_mat (list): The device matrix. + tensor_map (list): The map relation between tensor and devices. + + Returns: + List, the split strategy with the same size of np_tensor. + """ + tensor_strategy = [] + for dim in tensor_map: + if dim == -1: + tensor_strategy.append(1) + else: + tensor_strategy.append(dev_mat[-dim-1]) + return tensor_strategy + + +def _get_tensor_slice_index(device_arrangement, tensor_strategy, tensor_map, rank_index): + """ + Get the tensor slice index for the local device. + + Args: + device_arrangement (list): The device matrix. + tensor_strategy (list): The split strategy with the same size of np_tensor. + tensor_map (list): The map relation between tensor and devices. + rank_index (int): The rank of local device. + + Returns: + Integer, the index of the local device for tensor slices. + """ + device_coordinate = _rank_to_coordinate(rank_index, device_arrangement) + device_coordinate_new = _convert_to_new_device_coordinate(device_coordinate, tensor_map) + tensor_slice_index = _coordinate_to_rank(device_coordinate_new, tensor_strategy) + return tensor_slice_index + + +def _rank_to_coordinate(rank_index, device_arrangement): + """ + Convert rank index to device coordinate. + + Args: + rank_index (int): The index of the local device. + device_arrangement (list): The device matrix. + + Returns: + List, the coordinate for local device in the device matrix + """ + dim_len = len(device_arrangement) + device_coordinate = np.zeros(dim_len) + for i in range(dim_len): + size = device_arrangement[dim_len - 1 - i] + device_coordinate[dim_len - 1 - i] = rank_index % size + rank_index = int(rank_index / size) + return device_coordinate + + +def _coordinate_to_rank(device_coordinate, device_arrangement): + """ + Convert device coordinate to rank index. + + Args: + device_coordinate (list): The coordinate for local device in the device matrix. + device_arrangement (list): The device matrix. + + Returns: + Integer, the index of the local device for tensor slices. + """ + rank_index = 0 + size = 1 + for i in range(len(device_coordinate)): + rank_index += size * device_coordinate[len(device_coordinate) - 1 - i] + size *= device_arrangement[len(device_coordinate) - 1 - i] + return rank_index + + +def _convert_to_new_device_coordinate(device_coordinate, tensor_map): + """ + Convert device_coordinate according to the tensor map. + + Args: + device_coordinate (list): The coordinate for local device in the device matrix. + tensor_map (list): The map relation between tensor and devices. + + Returns: + List, the converted coordinate. + """ + device_coordinate_new = [] + for i in range(len(tensor_map)): + if tensor_map[len(tensor_map) - 1 - i] != -1: + device_coordinate_new.insert(0, device_coordinate[len(device_coordinate) - 1 - + tensor_map[len(tensor_map) - 1 - i]]) + else: + device_coordinate_new.insert(0, 0) + return device_coordinate_new + + +def _chunk_tensor(np_tensor, strategy, depth): + """ + Recursive function to chunk tensor. + + Args: + np_tensor (NDarray): The matrix to be split. + strategy (list): The split strategy with the same size of np_tensor. + depth (int): Recursion depth. + + Returns: + NDarray, the splited matrix. + + Raises: + ValueError: If np_tensor can not be split by strategy. + """ + output = [] + axis = len(np_tensor.shape) - depth + if np_tensor.shape[axis] % strategy[0] != 0: + raise ValueError("np_tensor can not be split by strategy!") + ret = list(np.split(np_tensor, strategy[0], axis)) + if depth == 1: + return ret + for ret_ in ret: + output.extend( + _chunk_tensor(ret_, strategy[len(strategy) - depth + 1:len(strategy)], depth - 1)) + + return output + + +def _chunk_tensor_by_strategy(np_tensor, strategy): + """ + Split the input by strategy. + + Args: + np_tensor (NDarray): The matrix to be split. + strategy (list): The split strategy with the same size of np_tensor. + + Returns: + NDarray, the splited matrix. + + Raises: + TypeError: If np_tensor is not ndarray + ValueError: If the length of np_tensor does not match the length of strategy. + """ + if not isinstance(np_tensor, np.ndarray): + raise TypeError("np_tensor should be ndarray!") + if len(strategy) != len(np_tensor.shape): + raise ValueError("The length of np_tensor does not match the length of strategy!") + return _chunk_tensor(np_tensor, strategy, len(strategy)) + + +def _load_tensor(tensor, dev_mat, tensor_map): + """ + Get the tensor slice of the local device by the device matrix and the tensor map + + Args: + tensor (Tensor): The tensor to be split. + dev_mat (list): The device matrix of devices. + tensor_map (list): The split strategy of tensor. + + Returns: + Tensor, the sliced tensor. + + Examples: + >>> tensor = Tensor(np.ones([32, 32])) + >>> dev_mat = [2, 4] + >>> tensor_map = [1, -1] + >>> tensor_slice = _load_tensor(tensor, dev_mat, tensor_map) + """ + rank = get_rank() + tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map) + tensor_slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, rank) + np_tensor = tensor.asnumpy() + np_tensor_list = _chunk_tensor_by_strategy(np_tensor, tensor_strategy) + np_tensor_slice = np_tensor_list[int(tensor_slice_index)] + tensor_slice = Tensor(np_tensor_slice) + return tensor_slice + + +def _load_tensor_by_layout(tensor, layout): + """ + Load tensor by layout. + + Args: + tensor (Tensor): The input tensor. + layout (tuple): The tensor layout in auto parallel. + + Returns: + Tensor, the sliced tensor.. + + Raises: + TypeError: If layout is not tuple. + ValueError: If the length of layout is not 2. + """ + if not isinstance(layout, tuple): + raise TypeError("layout should be tuple! layout is {}".format(layout)) + if len(layout) != 2: + raise ValueError("The length of layout must be 2! layout is {}".format(layout)) + dev_mat = layout[0] + tensor_map = layout[1] + if tensor.size() == 1: + return tensor + return _load_tensor(tensor, dev_mat, tensor_map) + + +def _reshape_param_data(param_data, dev_mat, tensor_map): + """ + Combine param slice by the device matrix and the tensor map, used in model parallel scenario. + + Args: + param_data (Tensor): The tensor to be reshaped, generated from all the device from AllGatherParamNet. + dev_mat (list): The device matrix of devices. + tensor_map (list): The split strategy of tensor. + + Returns: + Tensor, the combined tensor which with the whole data value. + + Examples: + >>> param_data = _allgather_param_net(param_data) + >>> dev_mat = [2, 2] + >>> tensor_map = [1, 0] + >>> tensor = _reshape_param_data(tensor_slices, dev_mat, tensor_map) + """ + + device_count = 1 + for dim in dev_mat: + device_count *= dim + + tensor_slices = np.split(param_data.asnumpy(), device_count, axis=0) + tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map) + + # get the actual number of slices,as: different devices may load the same slice + slice_count = 1 + for dim in tensor_strategy: + slice_count *= dim + + # reorder slices and remove duplicates based on device matrix and tensor_map + tensor_slices_new = list(range(slice_count)) + for i in range(device_count): + slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, i) + tensor_slices_new[int(slice_index)] = np.array(tensor_slices[i]) + + # combine slices to generate complete parameter + dim_len = len(tensor_strategy) + for i in range(dim_len): + ele_count = int(len(tensor_slices_new) / tensor_strategy[dim_len - 1 - i]) + tensor_slices_new_inner = [] + for j in range(ele_count): + new_tensor = tensor_slices_new[j * tensor_strategy[dim_len - 1 - i]] + for l in range(j * tensor_strategy[dim_len - 1 - i] + 1, + (j + 1) * tensor_strategy[dim_len - 1 - i]): + new_tensor = np.concatenate((new_tensor, tensor_slices_new[l]), axis=dim_len - 1 - i) + + tensor_slices_new_inner.insert(len(tensor_slices_new_inner), np.array(new_tensor)) + tensor_slices_new = tensor_slices_new_inner + + return Tensor(tensor_slices_new[0]) diff --git a/mindspore/parallel/_utils.py b/mindspore/parallel/_utils.py new file mode 100644 index 00000000000..3ce5463edf7 --- /dev/null +++ b/mindspore/parallel/_utils.py @@ -0,0 +1,253 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Utils of auto parallel""" + +from mindspore._c_expression import reset_op_id +from mindspore.communication.management import get_group_size, get_rank +from mindspore.parallel._auto_parallel_context import auto_parallel_context, _set_auto_parallel_context,\ + _reset_auto_parallel_context + + +def _get_parallel_mode(): + return auto_parallel_context().get_parallel_mode() + + +def _get_mirror_mean(): + return auto_parallel_context().get_mirror_mean() + + +def _get_device_num(): + """Get the device num.""" + parallel_mode = auto_parallel_context().get_parallel_mode() + if parallel_mode == "stand_alone": + device_num = 1 + return device_num + + if auto_parallel_context().get_device_num_is_set() is False: + device_num = get_group_size() + else: + device_num = auto_parallel_context().get_device_num() + return device_num + + +def _get_global_rank(): + """Get the global rank.""" + parallel_mode = auto_parallel_context().get_parallel_mode() + if parallel_mode == "stand_alone": + global_rank = 0 + return global_rank + + if auto_parallel_context().get_global_rank_is_set() is False: + global_rank = get_rank() + else: + global_rank = auto_parallel_context().get_global_rank() + return global_rank + + +def _get_parameter_broadcast(): + """Get the parameter broadcast.""" + parallel_mode = auto_parallel_context().get_parallel_mode() + if parallel_mode == "stand_alone": + parameter_broadcast = False + return parameter_broadcast + + if auto_parallel_context().get_parameter_broadcast_is_set() is True: + parameter_broadcast = auto_parallel_context().get_parameter_broadcast() + elif parallel_mode in ("data_parallel", "hybrid_parallel"): + parameter_broadcast = True + else: + parameter_broadcast = False + + return parameter_broadcast + + +def _device_number_check(parallel_mode, device_number): + """ + Check device num. + + Args: + parallel_mode (str): The parallel mode. + device_number (int): The device number. + """ + if parallel_mode == "stand_alone" and device_number != 1: + raise ValueError("If parallel_mode is stand_alone, device_number must be 1, " + "device_number: {0}, parallel_mode:{1}".format(device_number, parallel_mode)) + + +def _parameter_broadcast_check(parallel_mode, parameter_broadcast): + """ + Check parameter broadcast. + + Note: + If parallel mode is semi_auto_parallel or auto_parallel, parameter broadcast is not supported. Using the same + random seed to make sure parameters on multiple devices are the same. + + Args: + parallel_mode (str): The parallel mode. + parameter_broadcast (bool): The parameter broadcast. + + Raises: + ValueError: If parameter is broadcasted + but the parallel mode is "stand_alone" or "semi_auto_parallel" or "auto_parallel"). + """ + if parameter_broadcast is True and parallel_mode in ("stand_alone", "semi_auto_parallel", "auto_parallel"): + raise ValueError("stand_alone, semi_auto_parallel and auto_parallel " + "do not support parameter broadcast, parallel_mode: {0}, parameter_broadcast:{1}" + .format(parallel_mode, parameter_broadcast)) + + +_parallel_mode = None +_device_num = None +_global_rank = None +_parameter_broadcast = None +_mirror_mean = None +_cast_before_mirror = None +_loss_repeated_mean = None +_communication_backend = None +_has_checkpointed = False + + +def _checkpoint_auto_parallel_context(): + """checkpoint auto parallel context""" + global _has_checkpointed + if _has_checkpointed is True: + return + + global _parallel_mode + global _device_num + global _global_rank + global _parameter_broadcast + global _mirror_mean + global _cast_before_mirror + global _loss_repeated_mean + global _communication_backend + _parallel_mode = auto_parallel_context().get_parallel_mode() + _device_num = _get_device_num() + _global_rank = _get_global_rank() + _parameter_broadcast = auto_parallel_context().get_parameter_broadcast() + _mirror_mean = auto_parallel_context().get_mirror_mean() + _cast_before_mirror = auto_parallel_context().get_cast_before_mirror() + _loss_repeated_mean = auto_parallel_context().get_loss_repeated_mean() + _communication_backend = auto_parallel_context().get_communication_backend() + _has_checkpointed = True + + +def _restore_auto_parallel_context(): + """restore auto parallel context""" + global _parallel_mode + global _device_num + global _global_rank + global _parameter_broadcast + global _mirror_mean + global _cast_before_mirror + global _loss_repeated_mean + global _communication_backend + _set_auto_parallel_context(parallel_mode=_parallel_mode, device_num=_device_num, global_rank=_global_rank, + parameter_broadcast=_parameter_broadcast, mirror_mean=_mirror_mean, + cast_before_mirror=_cast_before_mirror, loss_repeated_mean=_loss_repeated_mean) + auto_parallel_context().set_communication_backend(_communication_backend) + + +def _reset_checkpoint_auto_parallel_context(): + """reset the _has_checkpointed""" + global _has_checkpointed + _has_checkpointed = False + + +def _callback_wrapper(list_callback, run_context, callback_type): + """ + reset the context for callback of model train + + Raises: + ValueError: If the type keyword is not recognized + """ + _callback_func_map = { + "begin": list_callback.begin, + "epoch_begin": list_callback.epoch_begin, + "step_begin": list_callback.step_begin, + "step_end": list_callback.step_end, + "epoch_end": list_callback.epoch_end, + "end": list_callback.end} + + if callback_type not in _callback_func_map: + raise ValueError("Get type keyword %s is not recognized!" % callback_type) + func = _callback_func_map[callback_type] + + if callback_type == "begin": + _reset_checkpoint_auto_parallel_context() + + _checkpoint_auto_parallel_context() + global _parallel_mode + if _parallel_mode == "stand_alone": + func(run_context) + return + + _reset_auto_parallel_context() + func(run_context) + _restore_auto_parallel_context() + + +PARAMETER_CLONED_INDEX = 0 + + +class _CloneInfo(): + """ + The clone info of parameter. + + Attributes: + be_cloned (bool): Whether the parameter is cloned. + cloned (bool): Whether the parameter clone from other parameter. + be_cloned_index (tuple): If the parameter is cloned, generate one index per clone. + cloned_index (int): If the parameter clone from other parameter, it has a unique index. + """ + def __init__(self): + self.be_cloned = False + self.cloned = False + self.be_cloned_index = [] + self.cloned_index = None + + +def _set_clone_info(clone_from, clone_to): + """ + Set the clone info. + + Args: + clone_from (_CloneInfo): The clone info of be_cloned parameter. + clone_to (_CloneInfo): The clone info of cloned parameter. + """ + global PARAMETER_CLONED_INDEX + clone_to.be_cloned = False + clone_to.cloned = True + clone_to.be_cloned_index = [] + clone_to.cloned_index = PARAMETER_CLONED_INDEX + + clone_from.be_cloned = True + clone_from.be_cloned_index.append(PARAMETER_CLONED_INDEX) + + PARAMETER_CLONED_INDEX = PARAMETER_CLONED_INDEX + 1 + + +def _get_python_op(op_name, op_path, instance_name, arglist): + """Get python operator.""" + module = __import__(op_path, fromlist=["None"]) + cls = getattr(module, op_name) + op = cls(*arglist) + op.set_prim_instance_name(instance_name) + return op + + +def _reset_op_id(): + """Reset op id.""" + reset_op_id() diff --git a/mindspore/parallel/algo_parameter_config.py b/mindspore/parallel/algo_parameter_config.py new file mode 100644 index 00000000000..aafc02367f1 --- /dev/null +++ b/mindspore/parallel/algo_parameter_config.py @@ -0,0 +1,184 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Configuration of parameters for strategy-searching algorithm in auto_parallel""" + +import threading +from mindspore._c_expression import CostModelContext +from mindspore._extends.pynative_helper import args_type_check + +__all__ = ["get_algo_parameters", "reset_algo_parameters", "set_algo_parameters"] + + +class _AlgoParameterConfig(): + """ + _AlgoParameterConfig is the configuration of setting parameters used in th algorithm. + + Note: + Creating a config through instantiating _AlgoParameterConfig object is not recommended. + Use algo_parameter_config() to get the configuration since _AlgoParameterConfig is singleton. + """ + _instance = None + _instance_lock = threading.Lock() + + def __init__(self): + self._config_handle = CostModelContext.get_instance() + + def check_config_handle(self): + """ + Check config handle. + + Raises: + ValueError: If the config handle is none. + """ + if self._config_handle is None: + raise ValueError("Config handle is none!!!") + + def set_simplify_cal(self, simplify_cal): + self.check_config_handle() + self._config_handle.set_simplify_cal(simplify_cal) + + def get_simplify_cal(self): + self.check_config_handle() + return self._config_handle.get_simplify_cal() + + def set_not_fully_use_devices(self, not_fully): + self.check_config_handle() + self._config_handle.set_not_fully_use_devices(not_fully) + + def get_not_fully_use_devices(self): + self.check_config_handle() + return self._config_handle.get_not_fully_use_devices() + + def set_elementwise_op_strategy_follow(self, element_strategy_follow): + self.check_config_handle() + self._config_handle.set_elementwise_op_strategy_follow(element_strategy_follow) + + def get_elementwise_op_strategy_follow(self): + self.check_config_handle() + return self._config_handle.get_elementwise_op_strategy_follow() + + def set_tensor_slice_align_enable(self, align_enable): + self.check_config_handle() + self._config_handle.set_tensor_slice_align_enable(align_enable) + + def get_tensor_slice_align_enable(self): + self.check_config_handle() + return self._config_handle.get_tensor_slice_align_enable() + + def set_tensor_slice_align_size(self, align_size): + """ + Set tensor slice align size. + + Args: + align_size (int): The minimum tensor slice shape. + + Raises: + ValueError: If align_size is not in [1, 1024]. + """ + self.check_config_handle() + if align_size < 1 or align_size > 1024: + raise ValueError('Align_size must be in [1, 1024], but got {}'.format(align_size)) + self._config_handle.set_tensor_slice_align_size(align_size) + + def get_tensor_slice_align_size(self): + self.check_config_handle() + return self._config_handle.get_tensor_slice_align_size() + + def reset_algo_parameters(self): + self.check_config_handle() + self._config_handle.reset_algo_parameters() + + +_g_algo_parameter_config = None + + +def _algo_parameter_config(): + """ + Get the global _g_algo_parameter_config. If it is not created, create a new one. + + Returns: + The global _g_algo_parameter_config. + """ + global _g_algo_parameter_config + if _g_algo_parameter_config is None: + _g_algo_parameter_config = _AlgoParameterConfig() + return _g_algo_parameter_config + + +set_algo_parameters_config_func_map = { + "simplify_cal": _algo_parameter_config().set_simplify_cal, + "not_fully_use_devices": _algo_parameter_config().set_not_fully_use_devices, + "elementwise_op_strategy_follow": _algo_parameter_config().set_elementwise_op_strategy_follow, + "tensor_slice_align_enable": _algo_parameter_config().set_tensor_slice_align_enable, + "tensor_slice_align_size": _algo_parameter_config().set_tensor_slice_align_size} + + +get_algo_parameters_config_func_map = { + "simplify_cal": _algo_parameter_config().get_simplify_cal, + "not_fully_use_devices": _algo_parameter_config().get_not_fully_use_devices, + "elementwise_op_strategy_follow": _algo_parameter_config().get_elementwise_op_strategy_follow, + "tensor_slice_align_enable": _algo_parameter_config().get_tensor_slice_align_enable, + "tensor_slice_align_size": _algo_parameter_config().get_tensor_slice_align_size} + + +@args_type_check(simplify_cal=bool, tensor_slice_align_enable=bool, tensor_slice_align_size=int, + not_fully_use_devices=bool, elementwise_op_strategy_follow=bool) +def set_algo_parameters(**kwargs): + """ + Set algo parameter config. + + Note: + Attribute name is needed. + + Args: + simplify_cal (bool): Whether simplifying calculations in strategy-searching algorithm. Default: True + tensor_slice_align_enable (bool): Whether checking tensor slice shape. Default: False + tensor_slice_align_size (int): The minimum tensor slice shape, the value must be in [1, 1024]. Default: 16 + not_fully_use_devices (bool): Whether generating strategies that not fully use devices. Default: False + elementwise_op_strategy_follow (bool): Whether the elementwise operator have the same strategies as its + subsequent operators. Default: False + + Raises: + ValueError: If context keyword is not recognized. + """ + for key, value in kwargs.items(): + if key not in set_algo_parameters_config_func_map: + raise ValueError("Set context keyword %s is not recognized!" % key) + set_func = set_algo_parameters_config_func_map[key] + set_func(value) + + +def get_algo_parameters(attr_key): + """ + Get algo parameter config attributes. + + Note: + Return value according to the attribute value. + + Args: + attr_key (str): The key of the attribute. + + Raises: + ValueError: If context keyword is not recognized. + """ + if attr_key not in get_algo_parameters_config_func_map: + raise ValueError("Get context keyword %s is not recognized!" % attr_key) + get_func = get_algo_parameters_config_func_map[attr_key] + return get_func() + + +def reset_algo_parameters(): + """Reset algo parameter attributes.""" + _algo_parameter_config().reset_algo_parameters() diff --git a/mindspore/parallel/dp_allreduce_fusion.py b/mindspore/parallel/dp_allreduce_fusion.py new file mode 100644 index 00000000000..979823bd806 --- /dev/null +++ b/mindspore/parallel/dp_allreduce_fusion.py @@ -0,0 +1,151 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Data paralell allreduce fusion""" + +import ctypes + +from mindspore import log as logger + +_MAX_GROUP_NAME_LEN = 127 +_HCCL_LIB = 'libhccl.so' + + +def _load_lib(): + try: + hccl_lib = ctypes.CDLL(_HCCL_LIB) + except RuntimeError: + logger.error('Get hccl lib error') + + return hccl_lib + + +def _c_str(string): + """Convert a python string to C string.""" + if not isinstance(string, str): + string = string.decode('ascii') + return ctypes.c_char_p(string.encode('utf-8')) + + +def _c_array(ctype, values): + """Create ctypes array from a python array.""" + return (ctype * len(values))(*values) + + +def set_fusion_strategy_by_idx(idxList, group="hccl_world_group"): + """ + A function set gradient segment strategy according to the index list. + + Note: + In the back propagation, + the fusion of the allreduce operators with a fusion attribute equals 1, + will be performed according to the idxList, + to achieve the effect of parallel between calculation and communication. + + Args: + idxList (list): The index list of the gradient. + group (str): The hccl communication group. + + Raises: + TypeError: If group is not a python str. + TypeError: If IdxList is not a python list. + TypeError: If type of idxList item is not int. + ValueError: If group name length is out of range. + ValueError: If idxList length is 0. + ValueError: If idxList item is less than 0. + RuntimeError: If allreduce split failed. + """ + try: + lib_ctype = _load_lib() + except RuntimeError: + logger.error('Load HCCL lib failed') + + if isinstance(group, (str)): + group_len = len(group) + if (group_len > _MAX_GROUP_NAME_LEN or group_len == 0): + raise ValueError('Group name len is out of range {_MAX_GROUP_NAME_LEN}') + else: + raise TypeError('Group must be a python str') + + if isinstance(idxList, (list)): + idx_len = len(idxList) + if idx_len == 0: + raise ValueError('IdxList length is 0') + else: + raise TypeError('IdxList must be a python list') + + for idx in idxList: + if isinstance(idx, (int)): + if idx < 0: + raise ValueError('Idx < 0') + else: + raise TypeError('Idx in idxList is invalid') + + c_array_idxList = _c_array(ctypes.c_uint, idxList) + c_idx_num = ctypes.c_uint(len(idxList)) + c_group = _c_str(group) + ret = lib_ctype.hcom_set_split_strategy_by_index(c_group, c_idx_num, c_array_idxList) + if ret != 0: + raise RuntimeError('Allreduce split error') + + +def set_fusion_strategy_by_size(dataSizeList, group="hccl_world_group"): + """ + A function set gradient segment strategy according to the data size percentage list. + + Note: + In the back propagation, + the fusion of the allreduce operators with a fusion attribute equals 1, + will be performed according to dataSizeList, + to achieve the effect of parallel between calculation and communication. + + Args: + dataSizeList (list): The data size percentage list of the gradient. + group (str): The hccl communication group. + + Raises: + TypeError: If group is not a python str. + TypeError: If dataSizeList is not a python list. + TypeError: If type of dataSizeList item is not int or float. + ValueError: If group name length is out of range. + ValueError: If dataSizeList length is 0. + ValueError: If dataSizeList item is less than 0. + RuntimeError: If allreduce split failed. + """ + try: + lib_ctype = _load_lib() + except RuntimeError: + logger.error('Load HCCL lib failed') + if isinstance(group, (str)): + group_len = len(group) + if group_len > _MAX_GROUP_NAME_LEN or group_len == 0: + raise ValueError('Group name is out of range {_MAX_GROUP_NAME_LEN}') + else: + raise TypeError('Group must be a python str') + if isinstance(dataSizeList, (list)): + len_data_size = len(dataSizeList) + if len_data_size == 0: + raise ValueError('DataSizeList length is 0') + else: + raise TypeError('DataSizeList must be a python list') + for dataSize in dataSizeList: + if not isinstance(dataSize, (int, float)): + raise TypeError('DataSize in dataSizeList is invalid') + + c_array_sizeList = _c_array(ctypes.c_float, dataSizeList) + c_size_num = ctypes.c_uint(len(dataSizeList)) + c_group = _c_str(group) + ret = lib_ctype.hcom_set_split_strategy_by_size(c_group, c_size_num, c_array_sizeList) + if ret != 0: + raise RuntimeError('Allreduce split error') diff --git a/mindspore/train/__init__.py b/mindspore/train/__init__.py new file mode 100644 index 00000000000..1895743e20c --- /dev/null +++ b/mindspore/train/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +High-Level training interfaces. + +Helper functions in train piplines. +""" +from .model import Model +from .parallel_utils import ParallelMode +from .dataset_helper import DatasetHelper +from . import amp + +__all__ = ["Model", "ParallelMode", "DatasetHelper", "amp"] diff --git a/mindspore/train/_utils.py b/mindspore/train/_utils.py new file mode 100644 index 00000000000..85b76290020 --- /dev/null +++ b/mindspore/train/_utils.py @@ -0,0 +1,198 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train utility.""" +import os +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.common.dtype import dtype_to_nptype +from mindspore.common import dtype as mstype +from mindspore import log as logger +from mindspore.common.api import _executor +from mindspore.common.dtype import pytype_to_dtype + + +def _convert_type(types): + """ + Convert from numpy type to tensor type. + + Args: + types (list): Numpy type list of element in dataset. + + Returns: + list, list of element in dataset. + """ + ms_types = [] + for np_type in types: + ms_type = pytype_to_dtype(np_type) + ms_types.append(ms_type) + return ms_types + + +def _get_types_and_shapes(dataset): + """Get dataset types and shapes.""" + dataset_types = _convert_type(dataset.output_types()) + dataset_shapes = dataset.output_shapes() + return dataset_types, dataset_shapes + + +def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): + """Initialize and execute the dataset graph.""" + batch_size = exec_dataset.get_batch_size() + input_indexs = exec_dataset.input_indexs + + # transform data format + dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) + exec_dataset = exec_dataset.device_que() + + _executor.init_dataset(exec_dataset.queue_name, + dataset_size, + batch_size, + dataset_types, + dataset_shapes, + input_indexs, + phase=phase) + + # engine dataset to write data to tdt queue + exec_dataset.send() + return exec_dataset + + +def _make_directory(path: str): + """Make directory.""" + real_path = None + if path is None or not isinstance(path, str) or path.strip() == "": + logger.error("The path(%r) is invalid type.", path) + raise TypeError("Input path is invaild type") + + # convert the relative paths + path = os.path.realpath(path) + logger.debug("The abs path is %r", path) + + # check the path is exist and write permissions? + if os.path.exists(path): + real_path = path + else: + # All exceptions need to be caught because create directory maybe have some limit(permissions) + logger.debug("The directory(%s) doesn't exist, will create it", path) + try: + os.makedirs(path) + real_path = path + except PermissionError as e: + logger.error("No write permission on the directory(%r), error = %r", path, e) + raise TypeError("No write permission on the directory.") + return real_path + + +def _construct_tensor_list(types, shapes, batch_expand_num=1): + """ + Construct list of tensors with types and shapes, used to initialize the network. + + Args: + types: List or Tuple. The output types of element in dataset. + shapes: List or Tuple. The output shapes of element in dataset. + batch_expand_num (int): Batch expand number. + + Returns: + List, list of Tensors. + """ + if len(types) != len(shapes): + raise ValueError("The length of dataset types must equal to dataset shapes, " + "but got dataset types={} and dataset shapes={}".format(types, shapes)) + tensor_list = [] + for type_, shape in zip(types, shapes): + new_shape = () + for i, item in enumerate(shape): + if i == 0: + new_shape += (item * batch_expand_num,) + else: + new_shape += (item,) + tensor = Tensor(np.zeros(new_shape, dtype_to_nptype(type_))) + tensor.virtual_flag = True + tensor_list.append(tensor) + return tensor_list + + +def _to_tensor(elem, scaling_sens=None): + """Conver numpy to tensor, adapt to minddata feed solution.""" + lst = [] + if not isinstance(elem, (tuple, list)): + elem = [elem] + for data in elem: + if not isinstance(data, np.ndarray): + if scaling_sens: + elem_tuple = tuple(elem) + (Tensor(scaling_sens, mstype.float32),) + else: + elem_tuple = tuple(elem) + return elem_tuple + lst.append(Tensor(data)) + if scaling_sens: + lst.append(Tensor(scaling_sens, mstype.float32)) + + return lst[0] if len(lst) == 1 else tuple(lst) + + +def _to_full_tensor(elem, device_num, global_rank, scaling_sens=None): + """Conver numpy to tensor, expanding batch dimension according to device_num, adapt to minddata feed solution.""" + lst = [] + if not isinstance(elem, (tuple, list)): + elem = [elem] + if global_rank >= device_num: + raise ValueError("The global rank must be smaller than device number, the global rank is {}, " + "the device num is {}".format(global_rank, device_num)) + + for data in elem: + if isinstance(data, np.ndarray): + data = Tensor(data) + if not isinstance(data, Tensor): + raise ValueError("elements in tensors must be Tensor") + shape_ = data.shape() + type_ = data.dtype() + new_shape = () + batchsize_per_device = 1 + for i, item in enumerate(shape_): + if i == 0: + new_shape += (item * device_num,) + batchsize_per_device = item + else: + new_shape += (item,) + new_tensor_numpy = np.zeros(new_shape, dtype_to_nptype(type_)) + start = global_rank * batchsize_per_device + new_tensor_numpy[start: start + batchsize_per_device] = data.asnumpy() + new_tensor = Tensor(new_tensor_numpy) + lst.append(new_tensor) + if scaling_sens: + lst.append(Tensor(scaling_sens, mstype.float32)) + return tuple(lst) + + +def _construct_input_tensors(dataset_types, dataset_shapes, device_number=1): + """Construct tensor list to initialize the network which implemented in dataset sink.""" + tensor_list_run = _construct_tensor_list(dataset_types, dataset_shapes, batch_expand_num=1) + tensor_list_compile = _construct_tensor_list(dataset_types, dataset_shapes, batch_expand_num=device_number) + return tensor_list_run, tensor_list_compile + + +def _to_full_shapes(shapes, device_num): + """Expanding batch dimension according to device_num, adapt to mindspore minddata graph solution.""" + new_shapes = [] + for shape in shapes: + new_shape = () + for i, item in enumerate(shape): + if i == 0: + new_shape += (item * device_num,) + else: + new_shape += (item,) + new_shapes.append(new_shape) + return new_shapes diff --git a/mindspore/train/amp.py b/mindspore/train/amp.py new file mode 100644 index 00000000000..5a70a86fddf --- /dev/null +++ b/mindspore/train/amp.py @@ -0,0 +1,158 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Auto mixed precision.""" +from easydict import EasyDict as edict + +from .. import nn +from .._checkparam import ParamValidator as validator +from .._checkparam import Rel +from ..common import dtype as mstype +from ..nn.wrap.cell_wrapper import _VirtualDatasetCell +from ..ops import functional as F +from ..ops.composite.base import _mp_cast_helper +from ..parallel._utils import _get_parallel_mode +from .loss_scale_manager import DynamicLossScaleManager, LossScaleManager +from .parallel_utils import ParallelMode +from .. import context + +__all__ = ["build_train_network"] + + +class OutputTo16(nn.Cell): + "Wrap cell for amp. Cast network output back to float16" + def __init__(self, op): + super(OutputTo16, self).__init__(auto_prefix=False) + self._op = op + + def construct(self, x): + return F.cast(self._op(x), mstype.float16) + + +def _do_keep_batchnorm_fp32(network): + cells = network.name_cells() + change = False + for name in cells: + subcell = cells[name] + if subcell == network: + continue + elif isinstance(subcell, (nn.BatchNorm2d, nn.BatchNorm1d)): + network._cells[name] = OutputTo16(subcell.to_float(mstype.float32)) + change = True + else: + _do_keep_batchnorm_fp32(subcell) + if isinstance(network, nn.SequentialCell) and change: + network.cell_list = list(network.cells()) + + +_config_level = { + "O0": { + "keep_batchnorm_fp32": False, + "cast_model_type": mstype.float32, + "loss_scale_manager": None}, + "O2": { + "keep_batchnorm_fp32": True, + "cast_model_type": mstype.float16, + "loss_scale_manager": DynamicLossScaleManager()}} + + +def _check_kwargs(key_words): + for arg in key_words: + if arg not in ['cast_model_type', 'keep_batchnorm_fp32', 'loss_scale_manager']: + raise ValueError(f"Unsupported arg '{arg}'") + + if 'cast_model_type' in key_words: + validator.check('cast_model_type', key_words['cast_model_type'], + [mstype.float16, mstype.float32], Rel.IN) + if 'keep_batchnorm_fp32' in key_words: + validator.check_isinstance('keep_batchnorm_fp32', key_words['keep_batchnorm_fp32'], bool) + if 'loss_scale_manager' in key_words: + loss_scale_manager = key_words['loss_scale_manager'] + if loss_scale_manager: + validator.check_isinstance('loss_scale_manager', loss_scale_manager, LossScaleManager) + +def build_train_network(network, optimizer, loss_fn=None, level='O0', **kwargs): + """ + Build the mixed precision training cell automatically. + + Args: + network (Cell): Definition of the network. + loss_fn (Union[None, Cell]): Definition of the loss_fn. If None, the `network` should have the loss inside. + Default: None. + optimizer (Optimizer): Optimizer to update the Parameter. + level (str): Supports [O0, O2]. + + - O0: Do not change. + - O2: Cast network to float16, keep batchnorm and `loss_fn` (if set) run in float32, + using dynamic loss scale. + + Default: "O0" + cast_model_type (:class:`mindspore.dtype`): Supports `mstype.float16` or `mstype.float32`. + If set to `mstype.float16`, use `float16` mode to train. If set, overwrite the level setting. + keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set, overwrite the level setting. + loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else + scale the loss by LossScaleManager. If set, overwrite the level setting. + """ + validator.check_isinstance('network', network, nn.Cell) + validator.check_isinstance('optimizer', optimizer, nn.Optimizer) + validator.check('level', level, "", ['O0', 'O2'], Rel.IN) + _check_kwargs(kwargs) + config = dict(_config_level[level], **kwargs) + config = edict(config) + + if config.cast_model_type == mstype.float16: + network.to_float(mstype.float16) + + if config.keep_batchnorm_fp32: + _do_keep_batchnorm_fp32(network) + + if loss_fn: + class WithLossCell(nn.Cell): + "Wrap loss for amp. Cast network output back to float32" + + def __init__(self, backbone, loss_fn): + super(WithLossCell, self).__init__(auto_prefix=False) + self._backbone = backbone + self._loss_fn = loss_fn + + def construct(self, data, label): + out = self._backbone(data) + label = _mp_cast_helper(mstype.float32, label) + return self._loss_fn(F.cast(out, mstype.float32), label) + + validator.check_isinstance('loss_fn', loss_fn, nn.Cell) + if config.cast_model_type == mstype.float16: + network = WithLossCell(network, loss_fn) + else: + network = nn.WithLossCell(network, loss_fn) + + if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + network = _VirtualDatasetCell(network) + + loss_scale = 1.0 + if config.loss_scale_manager is not None: + loss_scale_manager = config.loss_scale_manager + loss_scale = loss_scale_manager.get_loss_scale() + update_cell = loss_scale_manager.get_update_cell() + if update_cell is not None: + if not context.get_context("enable_ge"): + raise ValueError("Only `loss_scale_manager=None` and " + "`loss_scale_manager=FixedLossScaleManager(drop_overflow_update=False)`" + "are supported in current version. If you use `O2` option, please" + "use `loss_scale_manager=None` or `FixedLossScaleManager`") + network = nn.TrainOneStepWithLossScaleCell(network, optimizer, + scale_update_cell=update_cell).set_train() + return network + network = nn.TrainOneStepCell(network, optimizer, loss_scale).set_train() + return network diff --git a/mindspore/train/callback.py b/mindspore/train/callback.py new file mode 100644 index 00000000000..62f847089da --- /dev/null +++ b/mindspore/train/callback.py @@ -0,0 +1,700 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Callback related classes and functions.""" + +import os +import stat +import shutil +import time +import numpy as np + +import mindspore.context as context +from mindspore.train.serialization import _exec_save_checkpoint, _fill_param_into_net, _save_graph +from mindspore.train._utils import _make_directory +from mindspore import log as logger +from mindspore._checkparam import check_int_non_negative +from mindspore.common.tensor import Tensor +from .summary.summary_record import _cache_summary_tensor_data + + +__all__ = ["Callback", "LossMonitor", "TimeMonitor", "ModelCheckpoint", "SummaryStep", "CheckpointConfig", "RunContext"] + + +_cur_dir = os.getcwd() +_cur_net = None +_save_dir = _cur_dir + + +class _CheckpointManager: + """Manage checkpoint files according to train_config of checkpoint.""" + def __init__(self): + self._ckpoint_filelist = [] + + @property + def ckpoint_filelist(self): + """Get all the related checkpoint files managed here.""" + return self._ckpoint_filelist + + @property + def ckpoint_num(self): + """Get the number of the related checkpoint files managed here.""" + return len(self._ckpoint_filelist) + + def update_ckpoint_filelist(self, directory, prefix): + """Update the checkpoint file list.""" + self._ckpoint_filelist = [] + files = os.listdir(directory) + for filename in files: + if os.path.splitext(filename)[-1] == ".ckpt" and filename.startswith(prefix): + mid_name = filename[len(prefix):-5] + flag = True + for char in mid_name: + if char.isalpha(): + flag = False + if flag: + self._ckpoint_filelist.append(directory + '/' + filename) + + def remove_ckpoint_file(self, file_name): + """Remove the specified checkpoint file from this checkpoint manager and also from the directory.""" + try: + os.chmod(file_name, stat.S_IWRITE) + os.remove(file_name) + self._ckpoint_filelist.remove(file_name) + except OSError: + logger.warning("OSError, failed to remove the older ckpt file %s.", file_name) + except ValueError: + logger.warning("ValueError, failed to remove the older ckpt file %s.", file_name) + + def remove_oldest_ckpoint_file(self): + """Remove the oldest checkpoint file from this checkpoint manager and also from the directory.""" + ckpoint_files = sorted(self._ckpoint_filelist, key=os.path.getmtime) + self.remove_ckpoint_file(ckpoint_files[0]) + + def keep_one_ckpoint_per_minutes(self, minutes, cur_time): + """Only keep the latest one ckpt file per minutes, remove other files generated in [last_time, cur_time].""" + movs = [] + oldest_file = '' + oldest_time = cur_time + for ck_file in self._ckpoint_filelist: + modify_time = os.path.getmtime(ck_file) + if cur_time - modify_time < 60 * minutes: + movs.append(ck_file) + + if modify_time < oldest_time: + oldest_time = modify_time + oldest_file = ck_file + + for mv_file in movs: + if mv_file == oldest_file: + continue + self.remove_ckpoint_file(mv_file) + + +def _check_file_name_prefix(file_name_prefix): + """ + Check file name valid or not. + + File name can't include '/'. This file name naming convention only apply to Linux. + """ + if not isinstance(file_name_prefix, str) or file_name_prefix.find('/') >= 0: + return False + return True + + +def _chg_ckpt_file_name_if_same_exist(directory, prefix): + """Check if there is a file with the same name.""" + files = os.listdir(directory) + suffix_num = 0 + pre_len = len(prefix) + for filename in files: + name_ext = os.path.splitext(filename) + if name_ext[-1] != ".ckpt": + continue + # find same prefix file + if filename.find(prefix) == 0 and not filename[pre_len].isalpha(): + # add the max suffix + 1 + index = filename[pre_len:].find("-") + if index == 0: + suffix_num = max(suffix_num, 1) + elif index != -1: + num = filename[pre_len+1:pre_len+index] + if num.isdigit(): + suffix_num = max(suffix_num, int(num)+1) + + if suffix_num != 0: + prefix = prefix + "_" + str(suffix_num) + + return prefix + + +class CheckpointConfig: + """ + The config for model checkpoint. + + Args: + save_checkpoint_steps (int): Steps to save checkpoint. Default: 1. + save_checkpoint_seconds (int): Seconds to save checkpoint. Default: 0. + Can't be used with save_checkpoint_steps at the same time. + keep_checkpoint_max (int): Maximum step to save checkpoint. Default: 5. + keep_checkpoint_per_n_minutes (int): Keep one checkpoint every n minutes. Default: 0. + Can't be used with keep_checkpoint_max at the same time. + + Raises: + ValueError: If the input_param is None or 0. + + Examples: + >>> config = CheckpointConfig() + >>> ckpoint_cb = ModelCheckpoint(prefix="ck_prefix", directory='./', config=config) + >>> model.train(10, dataset, callbacks=ckpoint_cb) + """ + def __init__(self, + save_checkpoint_steps=1, + save_checkpoint_seconds=0, + keep_checkpoint_max=5, + keep_checkpoint_per_n_minutes=0): + + if not save_checkpoint_steps and not save_checkpoint_seconds and \ + not keep_checkpoint_max and not keep_checkpoint_per_n_minutes: + raise ValueError("The input_param can't be all None or 0") + + if save_checkpoint_steps: + save_checkpoint_steps = check_int_non_negative(save_checkpoint_steps) + if save_checkpoint_seconds: + save_checkpoint_seconds = check_int_non_negative(save_checkpoint_seconds) + if keep_checkpoint_max: + keep_checkpoint_max = check_int_non_negative(keep_checkpoint_max) + if keep_checkpoint_per_n_minutes: + keep_checkpoint_per_n_minutes = check_int_non_negative(keep_checkpoint_per_n_minutes) + + self._save_checkpoint_steps = save_checkpoint_steps + self._save_checkpoint_seconds = save_checkpoint_seconds + if self._save_checkpoint_steps and self._save_checkpoint_steps > 0: + self._save_checkpoint_seconds = None + + self._keep_checkpoint_max = keep_checkpoint_max + self._keep_checkpoint_per_n_minutes = keep_checkpoint_per_n_minutes + if self._keep_checkpoint_max and self._keep_checkpoint_max > 0: + self._keep_checkpoint_per_n_minutes = None + else: + if not self._keep_checkpoint_per_n_minutes or self._keep_checkpoint_per_n_minutes == 0: + self._keep_checkpoint_max = 1 + + @property + def save_checkpoint_steps(self): + """Get the value of _save_checkpoint_steps.""" + return self._save_checkpoint_steps + + @property + def save_checkpoint_seconds(self): + """Get the value of _save_checkpoint_seconds.""" + return self._save_checkpoint_seconds + + @property + def keep_checkpoint_max(self): + """Get the value of _keep_checkpoint_max.""" + return self._keep_checkpoint_max + + @property + def keep_checkpoint_per_n_minutes(self): + """Get the value of _keep_checkpoint_per_n_minutes.""" + return self._keep_checkpoint_per_n_minutes + + def get_checkpoint_policy(self): + """Get the policy of checkpoint.""" + checkpoint_policy = {'save_checkpoint_steps': self._save_checkpoint_steps, + 'save_checkpoint_seconds': self._save_checkpoint_seconds, + 'keep_checkpoint_max': self._keep_checkpoint_max, + 'keep_checkpoint_per_n_minutes': self._keep_checkpoint_per_n_minutes} + + return checkpoint_policy + + +def _set_cur_net(net): + """ + Set current net for which we are using to save checkpoint. + + Args: + net (Cell): train network + """ + global _cur_net + _cur_net = net + + +def _checkpoint_cb_for_save_op(parameter_list): + """ + The checkpoint callback function for MindSpore. + + Will be executed by checkpoint save op. + + Args: + parameter_list (list): Format is like [{"name",name},{"data",value}] and value type is Tensor. + + Returns: + bool, true: means save checkpoint success. + """ + if _cur_net is None: + logger.warning("_cur_net is None. parameters are not updated.") + return False + + logger.info("update parameters in the net.") + _fill_param_into_net(_cur_net, parameter_list) + _set_cur_net(None) + return True + + +def _summary_cb_for_save_op(summary_list): + """ + The summary callback function for MindSpore. + + Will be executed by summary op. + + Args: + summary_list (list): Format is like [{"name": tag_name, "data": tensor},...] and value is Scalar/Tensor. + + Returns: + bool, true: means save summary success. + """ + ret = _cache_summary_tensor_data(summary_list) + return ret + + +def _build_callbacks(callbacks): + """ + Contain a list of callback. + + Args: + callbacks (list): Callback functions list, Support None, a single Callback object, or a list. + + Returns: + List, a list of callback functions. + """ + if callbacks: + if isinstance(callbacks, tuple): + raise TypeError("Callbacks cannot be a tuple. Please check it.") + if not isinstance(callbacks, list): + callbacks = [callbacks] + else: + callbacks = [] + + excute_callbacks = [] + for cb in callbacks: + if cb is None or not isinstance(cb, Callback): + raise TypeError("Callback must inheriting base class Callback. Some callback is Wrong. Please check it.") + excute_callbacks.append(cb) + + return _ListCallback(excute_callbacks) + + +class _ListCallback: + """ + Sequential execution of callback functions. + + Execute Callback functions at certain points. + + Args: + callbacks (list): Callback functions list. + """ + def __init__(self, callbacks): + super(_ListCallback, self).__init__() + self._callbacks = callbacks + + def begin(self, run_context): + """Called once before network training.""" + for cb in self._callbacks: + cb.begin(run_context) + + def epoch_begin(self, run_context): + """Called before each epoch begin.""" + for cb in self._callbacks: + cb.epoch_begin(run_context) + + def epoch_end(self, run_context): + """Called after each epoch finished.""" + for cb in self._callbacks: + cb.epoch_end(run_context) + + def step_begin(self, run_context): + """Called before each epoch begin.""" + for cb in self._callbacks: + cb.step_begin(run_context) + + def step_end(self, run_context): + """Called after each step finished.""" + for cb in self._callbacks: + cb.step_end(run_context) + + def end(self, run_context): + """Called once after network training.""" + for cb in self._callbacks: + cb.end(run_context) + + +class Callback: + """ + Abstract base class used to build a callback function. + + Callback function will execution some operating to the current step or epoch. + + Examples: + >>> class Print_info(Callback): + >>> def step_end(self, run_context): + >>> cb_params = run_context.original_args() + >>> print(cb_params.cur_epoch_num) + >>> print(cb_params.cur_step_num) + >>> + >>> print_cb = Print_info() + >>> model.train(epoch, dataset, callback=print_cb) + """ + def __init__(self): + pass + + def begin(self, run_context): + """ + Called once before the network executing. + + Args: + run_context (RunContext): Include some information of the model. + """ + + def epoch_begin(self, run_context): + """ + Called before each epoch beginning. + + Args: + run_context (RunContext): Include some information of the model. + """ + + def epoch_end(self, run_context): + """ + Called after each epoch finished. + + Args: + run_context (RunContext): Include some information of the model. + """ + + def step_begin(self, run_context): + """ + Called before each epoch beginning. + + Args: + run_context (RunContext): Include some information of the model. + """ + + def step_end(self, run_context): + """ + Called after each step finished. + + Args: + run_context (RunContext): Include some information of the model. + """ + + def end(self, run_context): + """ + Called once after network training. + + Args: + run_context (RunContext): Include some information of the model. + """ + + +class SummaryStep(Callback): + """ + The summary callback class. + + Args: + summary (Object): Summary recode object. + flush_step (int): Number of interval steps to execute. Default: 10. + """ + def __init__(self, summary, flush_step=10): + super(SummaryStep, self).__init__() + if not isinstance(flush_step, int) or isinstance(flush_step, bool) or flush_step <= 0: + raise ValueError("`flush_step` should be int and greater than 0") + self._summary = summary + self._flush_step = flush_step + + def step_end(self, run_context): + """ + Save summary. + + Args: + run_context (RunContext): Context of the train running. + """ + cb_params = run_context.original_args() + if cb_params.cur_step_num % self._flush_step == 0: + self._summary.record(cb_params.cur_step_num, cb_params.train_network) + + @property + def summary_file_name(self): + return self._summary.full_file_name + + +class _InternalCallbackParam(dict): + """Internal callback object's parameters.""" + + def __getattr__(self, key): + return self[key] + + def __setattr__(self, key, value): + self[key] = value + + +class RunContext: + """ + Provides information about the model. + + Run call being made. Provides information about original request to model function. + callback objects can stop the loop by calling request_stop() of run_context. + + Args: + original_args (dict): Holding the related information of model etc. + """ + def __init__(self, original_args): + if not isinstance(original_args, dict): + raise TypeError("The arg of RunContext should be dict type.") + self._original_args = original_args + self._stop_requested = False + + def original_args(self): + """ + Get the _original_args object. + + Returns: + Dict, a object holding the original arguments of model. + """ + return self._original_args + + def request_stop(self): + """ + Sets stop requested during training. + + Callbacks can use this function to request stop of iterations. + model.train() checks whether this is called or not. + """ + self._stop_requested = True + + def get_stop_requested(self): + """ + Returns whether a stop is requested or not. + + Returns: + bool, if true, model.train() stops iterations. + """ + return self._stop_requested + + +class ModelCheckpoint(Callback): + """ + The checkpoint callback class. + + It is called to combine with train process and save the model and network parameters after traning. + + Args: + prefix (str): Checkpoint files names prefix. Default: "CKP". + directory (str): Lolder path into which checkpoint files will be saved. Default: None. + config (CheckpointConfig): Checkpoint strategy config. Default: None. + + Raises: + ValueError: If the prefix is invalid. + TypeError: If the config is not CheckpointConfig type. + """ + def __init__(self, prefix='CKP', directory=None, config=None): + super(ModelCheckpoint, self).__init__() + self._latest_ckpt_file_name = "" + self._init_time = time.time() + self._last_time = time.time() + self._last_time_for_keep = time.time() + self._last_triggered_step = 0 + + if _check_file_name_prefix(prefix): + self._prefix = prefix + else: + raise ValueError("Prefix {} for checkpoint file name invalid, " + "please check and correct it and then continue.".format(prefix)) + + if directory: + self._directory = _make_directory(directory) + else: + self._directory = _cur_dir + + if config is None: + self._config = CheckpointConfig() + else: + if not isinstance(config, CheckpointConfig): + raise TypeError("config should be CheckpointConfig type.") + self._config = config + + # get existing checkpoint files + self._manager = _CheckpointManager() + self._prefix = _chg_ckpt_file_name_if_same_exist(self._directory, self._prefix) + self._graph_saved = False + + def step_end(self, run_context): + """ + Save the checkpoint at the end of step. + + Args: + run_context (RunContext): Context of the train running. + """ + cb_params = run_context.original_args() + # save graph (only once) + if not self._graph_saved: + graph_file_name = os.path.join(self._directory, self._prefix + '-graph.meta') + _save_graph(cb_params.train_network, graph_file_name) + self._graph_saved = True + self._save_ckpt(cb_params) + + def end(self, run_context): + """ + Save the last checkpoint after training finished. + + Args: + run_context (RunContext): Context of the train running. + """ + cb_params = run_context.original_args() + _to_save_last_ckpt = True + self._save_ckpt(cb_params, _to_save_last_ckpt) + + from mindspore.parallel._cell_wrapper import destroy_allgather_cell + destroy_allgather_cell() + + def _check_save_ckpt(self, cb_params, force_to_save): + """Check whether save checkpoint files or not.""" + if self._config.save_checkpoint_steps and self._config.save_checkpoint_steps > 0: + if cb_params.cur_step_num >= self._last_triggered_step + self._config.save_checkpoint_steps \ + or force_to_save is True: + return True + elif self._config.save_checkpoint_seconds and self._config.save_checkpoint_seconds > 0: + self._cur_time = time.time() + if (self._cur_time - self._last_time) > self._config.save_checkpoint_seconds or force_to_save is True: + self._last_time = self._cur_time + return True + + return False + + def _save_ckpt(self, cb_params, force_to_save=False): + """Save checkpoint files.""" + if cb_params.cur_step_num == self._last_triggered_step: + return + + save_ckpt = self._check_save_ckpt(cb_params, force_to_save) + step_num_in_epoch = (cb_params.cur_step_num - 1) % cb_params.batch_num + 1 + + if save_ckpt: + cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \ + + str(step_num_in_epoch) + ".ckpt" + # update checkpoint file list. + self._manager.update_ckpoint_filelist(self._directory, self._prefix) + # keep checkpoint files number equal max number. + if self._config.keep_checkpoint_max and 0 < self._config.keep_checkpoint_max <= self._manager.ckpoint_num: + self._manager.remove_oldest_ckpoint_file() + elif self._config.keep_checkpoint_per_n_minutes and self._config.keep_checkpoint_per_n_minutes > 0: + self._cur_time_for_keep = time.time() + if (self._cur_time_for_keep - self._last_time_for_keep) \ + < self._config.keep_checkpoint_per_n_minutes * 60: + self._manager.keep_one_ckpoint_per_minutes(self._config.keep_checkpoint_per_n_minutes, + self._cur_time_for_keep) + + # generate the new checkpoint file and rename it. + global _save_dir + _save_dir = self._directory + cur_file = os.path.join(self._directory, cur_ckpoint_file) + tmp_ckpt_file_name_for_cur_process = str(os.getpid()) + "-" + 'parameters.ckpt' + gen_file = os.path.join(_save_dir, tmp_ckpt_file_name_for_cur_process) + self._last_time_for_keep = time.time() + self._last_triggered_step = cb_params.cur_step_num + + if context.get_context("enable_ge"): + _set_cur_net(cb_params.train_network) + cb_params.train_network.exec_checkpoint_graph() + + _exec_save_checkpoint(cb_params.train_network, gen_file) + + if os.path.exists(gen_file): + shutil.move(gen_file, cur_file) + self._latest_ckpt_file_name = cur_file + + @property + def latest_ckpt_file_name(self): + """Return the latest checkpoint path and file name.""" + return self._latest_ckpt_file_name + + +class LossMonitor(Callback): + """ + Monitor the loss in training. + + If the loss is NAN or INF, it will terminate training. + + Note: + If per_print_times is 0 do not print loss. + + Args: + per_print_times (int): Print loss every times. Default: 1. + + Raises: + ValueError: If print_step is not int or less than zero. + """ + def __init__(self, per_print_times=1): + super(LossMonitor, self).__init__() + if not isinstance(per_print_times, int) or per_print_times < 0: + raise ValueError("print_step must be int and >= 0.") + self._per_print_times = per_print_times + + def step_end(self, run_context): + cb_params = run_context.original_args() + loss = cb_params.net_outputs + + if isinstance(loss, (tuple, list)): + if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray): + loss = loss[0] + + if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray): + loss = np.mean(loss.asnumpy()) + + cur_step_in_epoch = (cb_params.cur_step_num - 1) % cb_params.batch_num + 1 + + if isinstance(loss, float) and (np.isnan(loss) or np.isinf(loss)): + raise ValueError("epoch: {} step: {}. Invalid loss, terminating training." + .format(cb_params.cur_epoch_num, cur_step_in_epoch)) + if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0: + print("epoch: %s step: %s, loss is %s" % (cb_params.cur_epoch_num, cur_step_in_epoch, loss), flush=True) + + +class TimeMonitor(Callback): + def __init__(self, data_size): + super(TimeMonitor, self).__init__() + self.data_size = data_size + self.step_time_cost = [] + self.epoch_time_cost = [] + self.per_step_time = [] + + def epoch_begin(self, run_context): + self.epoch_time = time.time() + + def epoch_end(self, run_context): + epoch_mseconds = (time.time() - self.epoch_time) * 1000 + per_step_mseconds = epoch_mseconds / self.data_size + self.epoch_time_cost.append(epoch_mseconds) + self.per_step_time.append(per_step_mseconds) + print("epoch time: {0}, per step time: {1}".format(epoch_mseconds, per_step_mseconds), flush=True) + + def step_begin(self, run_context): + self.step_time = time.time() + + def step_end(self, run_context): + step_mseconds = (time.time() - self.step_time) * 1000 + self.step_time_cost.append(step_mseconds) + print('step time', step_mseconds, flush=True) + diff --git a/mindspore/train/dataset_helper.py b/mindspore/train/dataset_helper.py new file mode 100644 index 00000000000..de26d72108f --- /dev/null +++ b/mindspore/train/dataset_helper.py @@ -0,0 +1,184 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dataset help for minddata dataset""" +from mindspore._checkparam import check_bool +from .. import context +from .parallel_utils import ParallelMode +from ._utils import _exec_datagraph, _get_types_and_shapes, _to_tensor, \ + _construct_tensor_list, _to_full_shapes, _to_full_tensor +from ..nn.wrap import GetNextSingleOp +from ..parallel._utils import _get_device_num, _get_global_rank, _get_parallel_mode + + +class DatasetHelper: + """ + Help function to use the Minddata dataset. + + According to different context, change the iter of dataset, to use the same for loop in different context. + + Note: + The iter of DatasetHelper will give one epoch data. + + Args: + dataset (DataSet): The dataset. + dataset_sink_mode (bool): If true use GetNext to fetch the data, or else feed the data from host. + Default: True. + + Examples: + >>> dataset_helper = DatasetHelper(dataset) + >>> for inputs in dataset_helper: + >>> outputs = network(*inputs) + """ + def __init__(self, dataset, dataset_sink_mode=True): + check_bool(dataset_sink_mode) + + iterclass = _DatasetIterGE + if not dataset_sink_mode: + iterclass = _DatasetIterFeed + elif not context.get_context("enable_ge"): + if context.get_context("enable_loop_sink"): + iterclass = _DatasetIterMSLoopSink + else: + iterclass = _DatasetIterMS + + self.iter = iterclass(dataset) + + def __iter__(self): + return self.iter.__iter__() + + # A temp solution for loop sink. Delete later + def types_shapes(self): + """Get the types and shapes from dataset on current config.""" + return self.iter.types_shapes() + + def loop_size(self): + """Get loop_size for every iteration.""" + return self.iter.loop_size + + +class _DatasetIter: + """Base iter for dataset help""" + def __init__(self, dataset): + self.loop_size = 1 + if not hasattr(dataset, '__ME_INITED__'): + if not hasattr(dataset, '__loop_size__'): + self.loop_size = dataset.get_dataset_size() + else: + self.loop_size = dataset.__loop_size__ + dataset.__ME_INITED__ = _exec_datagraph(dataset, self.loop_size).queue_name + + self.ind = 0 + self.dataset = dataset + dataset_types, dataset_shapes = _get_types_and_shapes(dataset) + self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes + # for self._parallel_mode equal to semi_auto_parallel or auto_parallel, use a complete tensor to + # compile, and slice tensor to run. The batch dimension of tensors for compile is device_number + # times the batch dimension of tensors for run + if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + device_num = _get_device_num() + dataset_shapes = _to_full_shapes(dataset_shapes, device_num) + + def __iter__(self): + self.ind = 0 + return self + + def __next__(self): + if self.ind >= self.loop_count: + raise StopIteration() + self.ind += 1 + return self.op() + + def types_shapes(self): + return self.dataset_types, self.dataset_shapes + + def get_loop_count(self, dataset): + loop_count = 1 + if hasattr(dataset, '__loop_size__'): + loop_size = dataset.__loop_size__ + if dataset.get_dataset_size() % loop_size != 0: + raise ValueError(f'Dataset size {dataset.get_dataset_size()} and ' + f'loop_size {loop_size} are not matched.') + loop_count = int(dataset.get_dataset_size()/loop_size) + return loop_count + + +class _DatasetIterMSLoopSink(_DatasetIter): + """Iter for context (enable_loop_sink=True)""" + def __init__(self, dataset): + super(_DatasetIterMSLoopSink, self).__init__(dataset) + self.loop_count = self.get_loop_count(dataset) + + def op(): + return tuple() + self.op = op + + +class _DatasetIterMS(_DatasetIter): + """Iter for context (enable_loop_sink=False)""" + def __init__(self, dataset): + super(_DatasetIterMS, self).__init__(dataset) + self.loop_count = dataset.get_dataset_size() + self.loop_size = 1 + queue_name = dataset.__ME_INITED__ + self.op = GetNextSingleOp(self.dataset_types, self.dataset_shapes, queue_name) + + +class _DatasetIterGE(_DatasetIter): + """Iter for ge""" + def __init__(self, dataset): + super(_DatasetIterGE, self).__init__(dataset) + self.loop_count = self.get_loop_count(dataset) + parallel_mode = _get_parallel_mode() + self.need_to_full = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) + batch_expand_num = 1 + if self.need_to_full: + batch_expand_num = _get_device_num() + tensor_list_run = _construct_tensor_list(self.dataset_types, self.dataset_shapes, batch_expand_num) + + def op(): + return tensor_list_run + self.op = op + + +class _DatasetIterFeed: + """Iter for feed data""" + def __init__(self, dataset): + self.dataset = dataset + self.device_num = _get_device_num() + self.global_rank = _get_global_rank() + self.repeat_count = dataset.get_repeat_count() + self.repeat_ind = 0 + self.loop_count = dataset.get_dataset_size() + self.ind = 0 + + parallel_mode = context.get_auto_parallel_context("parallel_mode") + self.need_to_full = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) + + def __iter__(self): + if self.repeat_ind % self.repeat_count == 0: + self.iter = self.dataset.__iter__() + + self.repeat_ind += 1 + self.ind = 0 + return self + + def __next__(self): + if self.ind >= self.loop_count: + raise StopIteration() + self.ind += 1 + data = self.iter.__next__() + if self.need_to_full: + return _to_full_tensor(data, self.device_num, self.global_rank) + return _to_tensor(data) diff --git a/mindspore/train/loss_scale_manager.py b/mindspore/train/loss_scale_manager.py new file mode 100644 index 00000000000..5650c58f620 --- /dev/null +++ b/mindspore/train/loss_scale_manager.py @@ -0,0 +1,143 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Loss scale manager abstract class.""" +from .._checkparam import ParamValidator as validator +from .._checkparam import Rel +from .. import nn + +__all__ = ["LossScaleManager", "FixedLossScaleManager", "DynamicLossScaleManager"] + + +class LossScaleManager: + """Loss scale manager abstract class.""" + def get_loss_scale(self): + """Get loss scale value.""" + + def update_loss_scale(self, overflow): + """ + Update loss scale value. + + Args: + overflow (bool): Whether it overflows. + """ + def get_update_cell(self): + """Get the loss scaling update logic cell.""" + +class FixedLossScaleManager(LossScaleManager): + """ + Fixed loss-scale manager. + + Args: + loss_scale (float): Loss scale. Default: 128.0. + drop_overflow_update (bool): whether to do optimizer if there is overflow. Default: True. + + Examples: + >>> loss_scale_manager = FixedLossScaleManager() + >>> model = Model(net, loss_scale_manager=loss_scale_manager) + """ + def __init__(self, loss_scale=128.0, drop_overflow_update=True): + if loss_scale < 1: + raise ValueError("loss_scale must be at least 1, " + "but got loss_scale {}".format(loss_scale)) + self._loss_scale = loss_scale + self._drop_overflow_update = drop_overflow_update + + def get_loss_scale(self): + """Get loss scale value.""" + return self._loss_scale + + def get_drop_overflow_update(self): + """Get the flag whether to drop optimizer update when there is overflow happened""" + return self._drop_overflow_update + + def update_loss_scale(self, overflow): + """ + Update loss scale value. + + Args: + overflow (bool): Whether it overflows. + """ + + def get_update_cell(self): + "Returns the cell for `TrainOneStepWithLossScaleCell`" + if not self._drop_overflow_update: + return None + return nn.FixedLossScaleUpdateCell(self._loss_scale) + + +class DynamicLossScaleManager(LossScaleManager): + """ + Dynamic loss-scale manager. + + Args: + init_loss_scale (float): Init loss scale. Default: 2**24. + scale_factor (int): Coefficient of increase and decrease. Default: 2. + scale_window (int): Maximum continuous normal steps when there is no overflow. Default: 2000. + + Examples: + >>> loss_scale_manager = DynamicLossScaleManager() + >>> model = Model(net, loss_scale_manager=loss_scale_manager) + """ + def __init__(self, + init_loss_scale=2 ** 24, + scale_factor=2, + scale_window=2000): + if init_loss_scale < 1.0: + raise ValueError("Loss scale value should be > 1") + self.loss_scale = init_loss_scale + validator.check_integer("scale_window", scale_window, 0, Rel.GT) + self.scale_window = scale_window + if scale_factor <= 0: + raise ValueError("Scale factor should be > 1") + self.scale_factor = scale_factor + self.increase_ratio = scale_factor + self.decrease_ratio = 1 / scale_factor + self.cur_iter = 1 + self.last_overflow_iter = 0 + self.bad_step_max = 1000 + self.bad_step = 0 + + def get_loss_scale(self): + """Get loss scale value.""" + return self.loss_scale + + def update_loss_scale(self, overflow): + """ + Update loss scale value. + + Args: + overflow: Boolean. Whether it overflows. + """ + if overflow: + self.loss_scale = max(self.loss_scale * self.decrease_ratio, 1) + self.last_overflow_iter = self.cur_iter + self.bad_step += 1 + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.loss_scale *= self.increase_ratio + self.bad_step = 0 + + if self.bad_step > self.bad_step_max: + raise RuntimeError("Dynamic loss scale Continuous overflow ", self.bad_step, " times") + + self.cur_iter += 1 + + def get_drop_overflow_update(self): + """Get the flag whether to drop optimizer update when there is overflow happened""" + return True + + def get_update_cell(self): + "Returns the cell for `TrainOneStepWithLossScaleCell`" + return nn.DynamicLossScaleUpdateCell(self.loss_scale, self.scale_factor, self.scale_window) diff --git a/mindspore/train/model.py b/mindspore/train/model.py new file mode 100755 index 00000000000..fe655433faa --- /dev/null +++ b/mindspore/train/model.py @@ -0,0 +1,543 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model.""" +import numpy as np + +from mindspore import log as logger +from ..common.tensor import Tensor +from ..nn.metrics import get_metrics +from .._checkparam import check_input_data, check_output_data, check_int_positive, check_bool +from .callback import _InternalCallbackParam, RunContext, _build_callbacks +from .. import context +from ..parallel._utils import _get_parallel_mode, _get_device_num, _get_global_rank, \ + _get_parameter_broadcast, _device_number_check, _parameter_broadcast_check, _callback_wrapper +from ..nn.metrics import Loss +from ..nn.wrap import WithLossCell, WithEvalCell, \ + DataWrapper +from ..nn.wrap.cell_wrapper import _VirtualDatasetCell +from .parallel_utils import ParallelMode +from ..common import dtype as mstype +from .dataset_helper import DatasetHelper +from . import amp + + +class Model: + """ + High-Level API for Training or Testing. + + `Model` groups layers into an object with training and inference features. + + Args: + network (Cell): The training or testing network. + loss_fn (Cell): Objective function, if loss_fn is None, the + network should contain the logic of loss and grads calculation, and the logic + of parallel if needed. Default: None. + optimizer (Cell): Optimizer for updating the weights. Default: None. + metrics (Union[dict, set]): Dict or set of metrics to be evaluated by the model during + training and testing. eg: {'accuracy', 'recall'}. Default: None. + eval_network (Cell): Network for evaluation. If not defined, `network` and `loss_fn` would be wrapped as + `eval_network`. Default: None. + eval_indexes (list): In case of defining the `eval_network`, if `eval_indexes` is None, all outputs of + `eval_network` would be passed to metrics, otherwise `eval_indexes` must contain three + elements, representing the positions of loss value, predict value and label, the loss + value would be passed to `Loss` metric, predict value and label would be passed to other + metric. Default: None. + amp_level (str): Option for argument `level` in `mindspore.amp.build_train_network`, level for mixed + precision training. Supports [O0, O2]. Default: "O0". + + - O0: Do not change. + - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. + + loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else + scale the loss by LossScaleManager. If it is set, overwrite the level setting. It's a eyword argument. + e.g. Use `loss_scale_manager=None` to set the value. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + >>> self.bn = nn.BatchNorm2d(64) + >>> self.relu = nn.ReLU() + >>> self.flatten = nn.Flatten() + >>> self.fc = nn.Dense(64*222*222, 3) # padding=0 + >>> + >>> def construct(self, x): + >>> x = self.conv(x) + >>> x = self.bn(x) + >>> x = self.relu(x) + >>> x = self.flatten(x) + >>> out = self.fc(x) + >>> return out + >>> + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + >>> dataset = get_dataset() + >>> model.train(2, dataset) + """ + + def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, + eval_indexes=None, amp_level="O0", **kwargs): + self._network = network + self._loss_fn = loss_fn + self._optimizer = optimizer + self._loss_scale_manager = None + self._loss_scale_manager_set = False + self._check_kwargs(kwargs) + if 'loss_scale_manager' in kwargs: + self._loss_scale_manager = kwargs['loss_scale_manager'] + self._loss_scale_manager_set = True + self._amp_level = amp_level + self._parallel_mode = _get_parallel_mode() + self._device_number = _get_device_num() + self._global_rank = _get_global_rank() + self._parameter_broadcast = _get_parameter_broadcast() + + self._train_network = self._build_train_network() + self._build_eval_network(metrics, eval_network, eval_indexes) + + def _check_kwargs(self, kwargs): + for arg in kwargs: + if arg not in ['loss_scale_manager']: + raise ValueError(f"Unsupport arg '{arg}'") + + def _build_train_network(self): + """Build train network""" + network = self._network + if self._optimizer: + if self._loss_scale_manager_set: + network = amp.build_train_network(network, + self._optimizer, + self._loss_fn, + level=self._amp_level, + loss_scale_manager=self._loss_scale_manager) + else: + network = amp.build_train_network(network, + self._optimizer, + self._loss_fn, + level=self._amp_level) + elif self._loss_fn: + network = WithLossCell(network, self._loss_fn) + # If need to check if loss_fn is not None, but optimizer is None + return network + + def _build_eval_network(self, metrics, eval_network, eval_indexes): + """Build the network for evaluation.""" + self._metric_fns = get_metrics(metrics) + if not self._metric_fns: + return + + if eval_network is not None: + if eval_indexes is not None and not (isinstance(eval_indexes, list) and len(eval_indexes) == 3): + raise ValueError("Eval_indexes must be a list or None. If eval_indexes is a list, length of it \ + must be three. But got {}".format(eval_indexes)) + + self._eval_network = eval_network + self._eval_indexes = eval_indexes + else: + if self._loss_fn is None: + raise ValueError("loss_fn can not be None.") + self._eval_network = WithEvalCell(self._network, self._loss_fn) + self._eval_indexes = [0, 1, 2] + + def _clear_metrics(self): + """Clear metrics local values.""" + for metric in self._metric_fns.values(): + metric.clear() + + def _update_metrics(self, outputs): + """Update metrics local values.""" + if self._eval_indexes is not None and len(outputs) < 3: + raise ValueError("The length of `outputs` must be greater than or equal to 3, \ + but got {}".format(len(outputs))) + + for metric in self._metric_fns.values(): + if self._eval_indexes is None: + metric.update(*outputs) + else: + if isinstance(metric, Loss): + metric.update(outputs[self._eval_indexes[0]]) + else: + metric.update(outputs[self._eval_indexes[1]], outputs[self._eval_indexes[2]]) + + def _get_metrics(self): + """Get metrics local values.""" + metrics = dict() + for key, value in self._metric_fns.items(): + metrics[key] = value.eval() + return metrics + + def _get_scaling_sens(self): + """get the scaling sens""" + scaling_sens = 1 + if self._loss_scale_manager is not None: + scaling_sens = self._loss_scale_manager.get_loss_scale() + if self._parallel_mode == ParallelMode.DATA_PARALLEL: + scaling_sens /= self._device_number + return scaling_sens + + def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): + """ + Training. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) will be + returned and passed to the network. Otherwise, a tuple (data, label) will + be returned, and the data and label are passed to the network and loss + function respectively. + callbacks (list): List of callback object. Callbacks which should be executed while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + """ + epoch = check_int_positive(epoch) + self._train_network.set_train() + + if self._parameter_broadcast: + self._train_network.set_broadcast_flag() + + # build callback list + list_callback = _build_callbacks(callbacks) + cb_params = _InternalCallbackParam() + cb_params.train_network = self._train_network + cb_params.epoch_num = epoch + cb_params.batch_num = train_dataset.get_dataset_size() + cb_params.mode = "train" + cb_params.loss_fn = self._loss_fn + cb_params.optimizer = self._optimizer + cb_params.parallel_mode = self._parallel_mode + cb_params.device_number = self._device_number + cb_params.train_dataset = train_dataset + cb_params.list_callback = list_callback + + if dataset_sink_mode and context.get_context("mode") == context.GRAPH_MODE: + self._train_dataset_sink_process(epoch, train_dataset, list_callback, cb_params) + else: + self._train_process(epoch, train_dataset, list_callback, cb_params) + + def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): + """ + Training process. The data would be passed to network through dataset channel. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + list_callback (_ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + """ + # remove later to deal with loop sink + need_wrap = False + if not hasattr(train_dataset, '__ME_INITED__') and context.get_context("enable_loop_sink"): + need_wrap = True + + dataset_helper = DatasetHelper(train_dataset) + # remove later to deal with loop sink + if need_wrap: + self._train_network = DataWrapper(self._train_network, *(dataset_helper.types_shapes()), + train_dataset.__ME_INITED__) + cb_params.train_network = self._train_network + self._train_network.set_train() + + cb_params.cur_step_num = 0 + loop_size = dataset_helper.loop_size() + run_context = RunContext(cb_params) + _callback_wrapper(list_callback, run_context, "begin") + + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + _callback_wrapper(list_callback, run_context, "epoch_begin") + + # for data sink dataset_helper only iter once, other wise iter epoch_size times. + for inputs in dataset_helper: + cb_params.cur_step_num += loop_size + _callback_wrapper(list_callback, run_context, "step_begin") + outputs = self._train_network(*inputs) + cb_params.net_outputs = outputs + _callback_wrapper(list_callback, run_context, "step_end") + + _callback_wrapper(list_callback, run_context, "epoch_end") + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + _callback_wrapper(list_callback, run_context, "end") + + def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): + """ + Training process. The data would be passed to network directly. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + list_callback (_ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + """ + dataset_helper = DatasetHelper(train_dataset, dataset_sink_mode=False) + cb_params.cur_step_num = 0 + run_context = RunContext(cb_params) + _callback_wrapper(list_callback, run_context, "begin") + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + + _callback_wrapper(list_callback, run_context, "epoch_begin") + + for next_element in dataset_helper: + len_element = len(next_element) + if self._loss_fn and len_element != 2: + raise ValueError("when loss_fn is not None, train_dataset should" + "return two elements, but got {}".format(len_element)) + cb_params.cur_step_num += 1 + _callback_wrapper(list_callback, run_context, "step_begin") + + overflow = False + if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): + scaling_sens = self._get_scaling_sens() + next_element = tuple(next_element) + (Tensor(scaling_sens, mstype.float32),) + + outputs = self._train_network(*next_element) + cb_params.net_outputs = outputs + if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): + _, overflow, _ = outputs + overflow = np.all(overflow.asnumpy()) + self._loss_scale_manager.update_loss_scale(overflow) + + _callback_wrapper(list_callback, run_context, "step_end") + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + train_dataset.reset() + + _callback_wrapper(list_callback, run_context, "epoch_end") + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + _callback_wrapper(list_callback, run_context, "end") + + def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): + """ + Training API where the iteration is controlled by python front-end. + + Configure to pynative mode, the training will be performed with dataset non-sink mode. + + Note: + CPU is not supported when dataset_sink_mode is true. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + callbacks (list): List of callback object. Callbacks which should be excuted while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + + + Examples: + >>> dataset = get_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> loss_scale_manager = FixedLossScaleManager() + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None, loss_scale_manager=loss_scale_manager) + >>> model.train(2, dataset) + """ + repeat_count = train_dataset.get_repeat_count() + if epoch != repeat_count: + logger.warning(f"The epoch_size {epoch} is not the same with dataset repeat_count {repeat_count}") + check_bool(dataset_sink_mode) + _device_number_check(self._parallel_mode, self._device_number) + _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) + + if context.get_context("device_target") in ["CPU", "GPU"] and context.get_context("enable_loop_sink"): + raise ValueError("CPU and GPU can't support loop sink, please set enable_loop_sink=False.") + + self._train(epoch, + train_dataset, + callbacks=callbacks, + dataset_sink_mode=dataset_sink_mode) + + def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): + """ + Evaluation. The data would be passed to network through dataset channel. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + list_callback (ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + """ + _device_number_check(self._parallel_mode, self._device_number) + + run_context = RunContext(cb_params) + + # remove later to deal with loop sink + need_wrap = False + if not hasattr(valid_dataset, '__ME_INITED__') and context.get_context("enable_loop_sink"): + need_wrap = True + + valid_dataset.__loop_size__ = 1 + dataset_helper = DatasetHelper(valid_dataset) + + # remove later to deal with loop sink + if need_wrap: + self._eval_network = DataWrapper(self._eval_network, *(dataset_helper.types_shapes()), + valid_dataset.__ME_INITED__) + self._eval_network.set_train(mode=False) + self._eval_network.phase = 'eval' + + list_callback.begin(run_context) + + for inputs in dataset_helper: + cb_params.cur_step_num += 1 + list_callback.step_begin(run_context) + + outputs = self._eval_network(*inputs) + + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + self._update_metrics(outputs) + + metrics = self._get_metrics() + cb_params.metrics = metrics + list_callback.end(run_context) + + return metrics + + def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): + """ + Evaluation. The data would be passed to network directly. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + list_callback (ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + """ + run_context = RunContext(cb_params) + list_callback.begin(run_context) + + dataset_helper = DatasetHelper(valid_dataset, dataset_sink_mode=False) + for next_element in dataset_helper: + list_callback.step_begin(run_context) + outputs = self._eval_network(*next_element) + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + self._update_metrics(outputs) + + metrics = self._get_metrics() + cb_params.metrics = metrics + list_callback.end(run_context) + return metrics + + def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): + """ + Evaluation API where the iteration is controlled by python front-end. + + Configure to pynative mode, the evaluation will be performed with dataset non-sink mode. + + Note: + CPU is not supported when dataset_sink_mode is true. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + callbacks (list): List of callback object. Callbacks which should be excuted + while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + + Examples: + >>> dataset = get_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> model = Model(net, loss_fn=loss, optimizer=None, metrics={'acc'}) + >>> model.eval(dataset) + """ + check_bool(dataset_sink_mode) + if not self._metric_fns: + raise ValueError("metric fn can not be None or empty.") + + list_callback = _build_callbacks(callbacks) + cb_params = _InternalCallbackParam() + cb_params.eval_network = self._eval_network + cb_params.valid_dataset = valid_dataset + cb_params.batch_num = valid_dataset.get_dataset_size() + cb_params.mode = "eval" + cb_params.cur_step_num = 0 + + self._eval_network.set_train(mode=False) + self._eval_network.phase = 'eval' + + self._clear_metrics() + + if dataset_sink_mode and context.get_context("mode") == context.GRAPH_MODE: + return self._eval_dataset_sink_process(valid_dataset, list_callback, cb_params) + return self._eval_process(valid_dataset, list_callback, cb_params) + + def predict(self, *predict_data): + """ + Generates output predictions for the input samples. + + Data could be single tensor, or list of tensor, tuple of tensor. + + Note: + Batch data should be put together in one tensor. + + Args: + predict_data (Tensor): Tensor of predict data. can be array, list or tuple. + + Returns: + Tensor, array(s) of predictions. + + Examples: + >>> input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mstype.float32) + >>> model = Model(Net()) + >>> model.predict(input_data) + """ + if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + self._network = _VirtualDatasetCell(self._network) + + self._network.set_train(False) + check_input_data(*predict_data, data_class=Tensor) + result = self._network(*predict_data) + + check_output_data(result) + return result + + +__all__ = ["Model"] diff --git a/mindspore/train/parallel_utils.py b/mindspore/train/parallel_utils.py new file mode 100644 index 00000000000..4f460a5d987 --- /dev/null +++ b/mindspore/train/parallel_utils.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Parallel utils""" + +__all__ = ["ParallelMode"] + + +class ParallelMode: + """ + Parallel mode options. + + There are five kinds of parallel modes, "STAND_ALONE", "DATA_PARALLEL", + "HYBRID_PARALLEL", "SEMI_AUTO_PARALLEL" and "AUTO_PARALLEL". Default: "STAND_ALONE". + + - STAND_ALONE: Only one processor working. + - DATA_PARALLEL: Distributing the data across different processors. + - HYBRID_PARALLEL: Achieving data parallelism and model parallelism manually. + - SEMI_AUTO_PARALLEL: Achieving data parallelism and model parallelism by setting parallel strategies. + - AUTO_PARALLEL: Achieving parallelism automatically. + + MODE_LIST: The list for all supported parallel modes. + """ + + STAND_ALONE = "stand_alone" + DATA_PARALLEL = "data_parallel" + HYBRID_PARALLEL = "hybrid_parallel" + SEMI_AUTO_PARALLEL = "semi_auto_parallel" + AUTO_PARALLEL = "auto_parallel" + MODE_LIST = [STAND_ALONE, DATA_PARALLEL, HYBRID_PARALLEL, SEMI_AUTO_PARALLEL, AUTO_PARALLEL] diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py new file mode 100644 index 00000000000..0478bbc0718 --- /dev/null +++ b/mindspore/train/serialization.py @@ -0,0 +1,438 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model and parameters serialization.""" +import os +import stat +import numpy as np + +import mindspore.nn as nn +import mindspore.context as context +from mindspore import log as logger +from mindspore.train.checkpoint_pb2 import Checkpoint +from mindspore.common.tensor import Tensor +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.common.api import _executor +from mindspore.common import dtype as mstype +from mindspore._checkparam import check_input_data + +__all__ = ["save_checkpoint", "load_checkpoint", "load_param_into_net", "export"] + +tensor_to_ms_type = {"Int8": mstype.int8, "Int16": mstype.int16, "Int32": mstype.int32, "Int64": mstype.int64, + "Float16": mstype.float16, "Float32": mstype.float32, "Float64": mstype.float64} + +tensor_to_np_type = {"Int8": np.int8, "Int16": np.int16, "Int32": np.int32, "Int64": np.int64, + "Float16": np.float16, "Float32": np.float32, "Float64": np.float64} + + +def _special_process_par(par, new_par): + """ + Processes the special condition. + + Like (12,2048,1,1)->(12,2048), this case is caused by GE 4 dimensions tensor. + """ + par_shape_len = len(par.data.shape()) + new_par_shape_len = len(new_par.data.shape()) + delta_len = new_par_shape_len - par_shape_len + delta_i = 0 + for delta_i in range(delta_len): + if new_par.data.shape()[par_shape_len + delta_i] != 1: + break + if delta_i == delta_len - 1: + new_val = new_par.data.asnumpy() + new_val = new_val.reshape(par.data.shape()) + par.set_parameter_data(Tensor(new_val, par.data.dtype())) + return True + return False + + +def _update_param(param, new_param): + """Updates param's data from new_param's data.""" + + if isinstance(param.data, Tensor) and isinstance(new_param.data, Tensor): + if param.data.dtype() != new_param.data.dtype(): + logger.error("Failed to combine the net and the parameters for param %s.", param.name) + msg = ("Net parameters {} type({}) different from parameter_dict's({})" + .format(param.name, param.data.dtype(), new_param.data.dtype())) + raise RuntimeError(msg) + + if param.data.shape() != new_param.data.shape(): + if not _special_process_par(param, new_param): + logger.error("Failed to combine the net and the parameters for param %s.", param.name) + msg = ("Net parameters {} shape({}) different from parameter_dict's({})" + .format(param.name, param.data.shape(), new_param.data.shape())) + raise RuntimeError(msg) + return + + param.set_parameter_data(new_param.data) + return + + if isinstance(param.data, Tensor) and not isinstance(new_param.data, Tensor): + if param.data.shape() != (1,) and param.data.shape() != (): + logger.error("Failed to combine the net and the parameters for param %s.", param.name) + msg = ("Net parameters {} shape({}) is not (1,), inconsitent with parameter_dict's(scalar)." + .format(param.name, param.data.shape())) + raise RuntimeError(msg) + param.set_parameter_data(initializer(new_param.data, param.data.shape(), param.data.dtype())) + + elif isinstance(new_param.data, Tensor) and not isinstance(param.data, Tensor): + logger.error("Failed to combine the net and the parameters for param %s.", param.name) + msg = ("Net parameters {} type({}) different from parameter_dict's({})" + .format(param.name, type(param.data), type(new_param.data))) + raise RuntimeError(msg) + + else: + param.set_parameter_data(type(param.data)(new_param.data)) + + +def save_checkpoint(parameter_list, ckpoint_file_name): + """ + Saves checkpoint info to a specified file. + + Args: + parameter_list (list): Parameters list, each element is a dict + like {"name":xx, "type":xx, "shape":xx, "data":xx}. + ckpoint_file_name (str): Checkpoint file name. + + Raises: + RuntimeError: Failed to save the Checkpoint file. + """ + logger.info("Execute save checkpoint process.") + checkpoint_list = Checkpoint() + + try: + for param in parameter_list: + param_value = checkpoint_list.value.add() + param_value.tag = param["name"] + param_tensor = param_value.tensor + param_data = param["data"].asnumpy().reshape(-1) + param_tensor.tensor_content = param_data.tostring() + param_tensor.tensor_type = str(param["data"].dtype()) + + if param['data'].shape() == (): + param_tensor.dims.append(0) + else: + for dim in param['data'].shape(): + param_tensor.dims.append(dim) + + with open(ckpoint_file_name, "wb") as f: + f.write(checkpoint_list.SerializeToString()) + os.chmod(ckpoint_file_name, stat.S_IRUSR) + + except BaseException as e: + logger.error("Failed to save the checkpoint file %s.", ckpoint_file_name) + raise RuntimeError(e.__str__()) + logger.info("Save checkpoint process finish.") + + +def load_checkpoint(ckpoint_file_name, net=None): + """ + Loads checkpoint info from a specified file. + + Args: + ckpoint_file_name (str): Checkpoint file name. + net (Cell): Cell network. Default: None + + Returns: + Dict, key is parameter name, value is a Parameter. + + Raises: + ValueError: Checkpoint file is incorrect. + """ + if not isinstance(ckpoint_file_name, str): + raise ValueError("The ckpoint_file_name must be String.") + + if not os.path.exists(ckpoint_file_name) or ckpoint_file_name[-5:] != ".ckpt": + raise ValueError("Please input the correct checkpoint file name.") + + if os.path.getsize(ckpoint_file_name) == 0: + raise ValueError("The checkpoint file may be empty, please make sure enter the correct file name.") + + logger.info("Execute load checkpoint process.") + checkpoint_list = Checkpoint() + + try: + with open(ckpoint_file_name, "rb") as f: + pb_content = f.read() + checkpoint_list.ParseFromString(pb_content) + except BaseException as e: + logger.error("Failed to read the checkpoint file %s, please check the correct of the file.", ckpoint_file_name) + raise ValueError(e.__str__()) + + parameter_dict = {} + + try: + for element in checkpoint_list.value: + data = element.tensor.tensor_content + data_type = element.tensor.tensor_type + np_type = tensor_to_np_type[data_type] + ms_type = tensor_to_ms_type[data_type] + param_data = np.fromstring(data, np_type) + dims = element.tensor.dims + + if dims in [[0], [1]]: + parameter_dict[element.tag] = Parameter(param_data[0], name=element.tag) + else: + param_dim = [] + for dim in dims: + param_dim.append(dim) + param_value = param_data.reshape(param_dim) + parameter_dict[element.tag] = Parameter(Tensor(param_value, ms_type), name=element.tag) + + logger.info("Load checkpoint process finish.") + + except BaseException as e: + logger.error("Failed to load the checkpoint file %s.", ckpoint_file_name) + raise RuntimeError(e.__str__()) + + if net: + load_param_into_net(net, parameter_dict) + + return parameter_dict + + +def load_param_into_net(net, parameter_dict): + """ + Loads parameters into network. + + Args: + net (Cell): Cell network. + parameter_dict (dict): Parameter dict. + + Raises: + TypeError: Argument is not a Cell, or parameter_dict is not a Parameter dict. + """ + if not isinstance(net, nn.Cell): + logger.error("Failed to combine the net and the parameters.") + msg = ("Argument net should be a Cell, but got {}.".format(type(net))) + raise TypeError(msg) + + if not isinstance(parameter_dict, dict): + logger.error("Failed to combine the net and the parameters.") + msg = ("Argument parameter_dict should be a dict, but got {}.".format(type(parameter_dict))) + raise TypeError(msg) + + logger.info("Execute parameter into net process.") + param_name_net_not_have = [] + for name in parameter_dict: + b_par_dict_have_par_of_net = False + for _, param in net.parameters_and_names(): + if name == param.name: + b_par_dict_have_par_of_net = True + # layerwise parallel parameter data loaded from checkpoint file, + # was a complete(merged) data, need to be splited + if param.layerwise_parallel: + new_param = parameter_dict[param.name] + _load_tensor_for_layerwise(new_param, param) + break + if not b_par_dict_have_par_of_net: + param_name_net_not_have.append(name) + + param_name_param_dict_not_have = [] + for _, param in net.parameters_and_names(): + if param.name in parameter_dict: + new_param = parameter_dict[param.name] + + if not isinstance(new_param, Parameter): + logger.error("Failed to combine the net and the parameters.") + msg = ("Argument parameter_dict element should be a Parameter, but got {}.".format(type(new_param))) + raise TypeError(msg) + _update_param(param, new_param) + else: + param_name_param_dict_not_have.append(param.name) + + logger.debug("Params not matched(in net but not in parameter_dict):") + for paramname in param_name_param_dict_not_have: + logger.debug("%s", paramname) + logger.debug("Params not matched(in parameter_dict but not in net):") + for paramname in param_name_net_not_have: + logger.debug("%s", paramname) + logger.info("Load parameter into net process finish.") + + +def _save_graph(network, file_name): + """ + Saves the graph of network to a file. + + Args: + network (Cell): Obtain a pipeline through network for saving graph. + file_name (str): Graph file name into which the graph will be saved. + """ + logger.info("Execute save the graph process.") + + graph_proto = network.get_func_graph_proto() + if graph_proto: + with open(file_name, "wb") as f: + f.write(graph_proto) + os.chmod(file_name, stat.S_IWUSR | stat.S_IRUSR) + + +def _exec_save_checkpoint(train_network, ckpoint_file_name): + """ + Saves checkpoint for 'ms' backend. + + Args: + train_network (Network): The train network for training. + ckpoint_file_name (str): The name of checkpoint file. + """ + + param_dict = {} + for _, param in train_network.parameters_and_names(): + param_dict[param.name] = param + + param_list = [] + for (key, value) in param_dict.items(): + each_param = {"name": key} + if isinstance(value.data, Tensor): + param_data = value.data + else: + param_data = Tensor(value.data) + + # in model parallel scenario, some parameters were spliteds to all the devices, + # which should be combined before saving + if key in train_network.parameter_layout_dict: + param_data = _get_merged_param_data(train_network, key, param_data) + + each_param["data"] = param_data + param_list.append(each_param) + + save_checkpoint(param_list, ckpoint_file_name) + + +def _get_merged_param_data(net, param_name, param_data): + """ + Gets the merged data(tensor) from tensor slice, by device arrangement and tensor map. + + Args: + net (Cell): MindSpore network. + param_name(str): The parameter name, which to be combined. + param_data(Tensor):The parameter data on the local device, + It was a slice of the whole parameter data. + Returns: + Tensor, the combined tensor which with the whole data value. + """ + layout = [] + layout = net.parameter_layout_dict[param_name] + if len(layout) < 2: + logger.info("layout dict does not contain the key %s", param_name) + return param_data + + dev_mat = layout[0] + tensor_map = layout[1] + + from mindspore.parallel._cell_wrapper import get_allgather_cell + from mindspore.parallel._tensor import _reshape_param_data + # while any dim is not equal to -1, means param is splited and needs to be merged + for dim in tensor_map: + if dim != -1: + allgather_net = get_allgather_cell() + param_data = allgather_net(param_data) + return _reshape_param_data(param_data, dev_mat, tensor_map) + + return param_data + + +def _load_tensor_for_layerwise(new_param, old_param): + """ + Replaces parameters with sliced tensors by layerwise parallel strategies. + + Args: + new_param (Parameter): The new layerwise parallel parameter, will be loaded into net. + old_param(Parameter): The current parameter in the net. + """ + if not isinstance(new_param.data, Tensor) or not isinstance(old_param.data, Tensor): + logger.error("Failed to combine the net and the parameters.") + msg = ("layerwise parallel parameter should be a Tensor, but got {}.".format(type(new_param.data))) + raise TypeError(msg) + + if old_param.data.shape() == new_param.data.shape(): + return + + from mindspore.parallel._tensor import _load_tensor + from mindspore.communication.management import get_group_size + dev_mat = [get_group_size()] + shape = new_param.data.shape() + for x in range(len(shape)): # dim 0 set 0, others set -1 + if x: + tensor_map.append(-1) + + new_tensor = _load_tensor(new_param.data, dev_mat, tensor_map) + new_param.set_parameter_data(new_tensor) + + +def _fill_param_into_net(net, parameter_list): + """ + Fills parameter_list into net. + + Args: + net (Cell): train network. + parameter_list (list): parameters list from ge callback. + """ + parameter_dict = {} + for each_param in parameter_list: + param_name = each_param["name"] + np_val = each_param["data"].asnumpy() + if np_val.shape == (1,): # to scalar + parameter_dict[param_name] = Parameter(np_val[0], name=param_name) + elif np_val.shape == (): + parameter_dict[param_name] = Parameter(np_val.tolist(), name=param_name) + else: + parameter_dict[param_name] = Parameter(Tensor(np_val), name=param_name) + + load_param_into_net(net, parameter_dict) + + +def export(net, *inputs, file_name, file_format='GEIR'): + """ + Exports MindSpore predict model to file in specified format. + + Args: + net (Cell): MindSpore network. + inputs (Tensor): Inputs of the `net`. + file_name (str): File name of model to export. + file_format (str): MindSpore currently supports 'GEIR', 'ONNX' and 'LITE' format for exported model. + + - GEIR: Graph Engine Intermidiate Representation. An intermidiate representation format of + Ascend model. + - ONNX: Open Neural Network eXchange. An open format built to represent machine learning models. + - LITE: Huawei model format for mobile. + """ + logger.info("exporting model file:%s format:%s.", file_name, file_format) + check_input_data(*inputs, data_class=Tensor) + + supported_formats = ['GEIR', 'ONNX', 'LITE'] + if file_format not in supported_formats: + raise ValueError(f'Illegal file format {file_format}, it must be one of {supported_formats}') + # switch network mode to infer when it is training + is_training = net.training + if is_training: + net.set_train(mode=False) + # export model + if file_format == 'GEIR': + _executor.compile(net, *inputs, phase='export') + _executor.export(net, file_name, file_format) + elif file_format == 'ONNX': # file_format is 'ONNX' + phase_name = 'export_onnx' + graph_id, _ = _executor.compile(net, *inputs, phase=phase_name) + onnx_stream = _executor._get_func_graph_proto(graph_id) + with open(file_name, 'wb') as f: + os.chmod(file_name, stat.S_IWUSR | stat.S_IRUSR) + f.write(onnx_stream) + elif file_format == 'LITE': # file_format is 'LITE' + context.set_context(save_ms_model=True, save_ms_model_path=file_name) + net(*inputs) + # restore network training mode + if is_training: + net.set_train(mode=True) diff --git a/mindspore/train/summary/__init__.py b/mindspore/train/summary/__init__.py new file mode 100644 index 00000000000..169317c0836 --- /dev/null +++ b/mindspore/train/summary/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +SummaryRecord. + +User can use SummaryRecord to dump the summary data, the summary is a series of operations +to collect data for analysis and visualization. +""" + +from .summary_record import SummaryRecord + +__all__ = ["SummaryRecord"] diff --git a/mindspore/train/summary/_event_writer.py b/mindspore/train/summary/_event_writer.py new file mode 100644 index 00000000000..c04308dcbc4 --- /dev/null +++ b/mindspore/train/summary/_event_writer.py @@ -0,0 +1,104 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Writes events to disk in a logdir.""" +import os +import time +import stat +from mindspore import log as logger +from ..._c_expression import EventWriter_ +from ._summary_adapter import package_init_event + + +class _WrapEventWriter(EventWriter_): + """ + Wrap the c++ EventWriter object. + + Args: + full_file_name (str): Include directory and file name. + """ + def __init__(self, full_file_name): + if full_file_name is not None: + EventWriter_.__init__(self, full_file_name) + + +class EventRecord: + """ + Creates a `EventFileWriter` and write event to file. + + Args: + full_file_name (str): Summary event file path and file name. + flush_time (int): The flush seconds to flush the pending events to disk. Default: 120. + """ + def __init__(self, full_file_name: str, flush_time: int = 120): + self.full_file_name = full_file_name + + # The first event will be flushed immediately. + self.flush_time = flush_time + self.next_flush_time = 0 + + # create event write object + self.event_writer = self._create_event_file() + self._init_event_file() + + # count the events + self.event_count = 0 + + def _create_event_file(self): + """Create the event write file.""" + with open(self.full_file_name, 'w'): + os.chmod(self.full_file_name, stat.S_IWUSR | stat.S_IRUSR) + + # create c++ event write object + event_writer = _WrapEventWriter(self.full_file_name) + return event_writer + + def _init_event_file(self): + """Send the init event to file.""" + self.event_writer.Write((package_init_event()).SerializeToString()) + self.flush() + return True + + def write_event_to_file(self, event_str): + """Write the event to file.""" + self.event_writer.Write(event_str) + + def get_data_count(self): + """Return the event count.""" + return self.event_count + + def flush_cycle(self): + """Flush file by timer.""" + self.event_count = self.event_count + 1 + # Flush the event writer every so often. + now = int(time.time()) + if now > self.next_flush_time: + self.flush() + # update the flush time + self.next_flush_time = now + self.flush_time + + def count_event(self): + """Count event.""" + logger.debug("Write the event count is %r", self.event_count) + self.event_count = self.event_count + 1 + return self.event_count + + def flush(self): + """Flush the event file to disk.""" + self.event_writer.Flush() + + def close(self): + """Flush the event file to disk and close the file.""" + self.flush() + self.event_writer.Shut() diff --git a/mindspore/train/summary/_summary_adapter.py b/mindspore/train/summary/_summary_adapter.py new file mode 100644 index 00000000000..29a57742717 --- /dev/null +++ b/mindspore/train/summary/_summary_adapter.py @@ -0,0 +1,426 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Generate the summary event which conform to proto format.""" +import time +import socket +from enum import Enum, unique +import numpy as np +from PIL import Image + +from mindspore import log as logger +from ..summary_pb2 import Event +from ..anf_ir_pb2 import ModelProto, DataType +from ..._checkparam import _check_str_by_regular + +# define the MindSpore image format +MS_IMAGE_TENSOR_FORMAT = 'NCHW' +# Set the Event mark +EVENT_FILE_NAME_MARK = ".out.events.summary." +# Set the init event of version and mark +EVENT_FILE_INIT_VERSION_MARK = "Mindspore.Event:" +EVENT_FILE_INIT_VERSION = 1 +# cache the summary data dict +# {id: SummaryData} +# |---[{"name": tag_name, "data": numpy}, {"name": tag_name, "data": numpy},...] +g_summary_data_dict = {} + +def save_summary_data(data_id, data): + """Save the global summary cache.""" + global g_summary_data_dict + g_summary_data_dict[data_id] = data + + +def del_summary_data(data_id): + """Save the global summary cache.""" + global g_summary_data_dict + if data_id in g_summary_data_dict: + del g_summary_data_dict[data_id] + else: + logger.warning("Can't del the data because data_id(%r) " + "does not have data in g_summary_data_dict", data_id) + +def get_summary_data(data_id): + """Save the global summary cache.""" + ret = None + global g_summary_data_dict + if data_id in g_summary_data_dict: + ret = g_summary_data_dict.get(data_id) + else: + logger.warning("The data_id(%r) does not have data in g_summary_data_dict", data_id) + return ret + +@unique +class SummaryType(Enum): + """ + Summary type. + + Args: + SCALAR (Number): Summary Scalar enum. + TENSOR (Number): Summary TENSOR enum. + IMAGE (Number): Summary image enum. + GRAPH (Number): Summary graph enum. + INVALID (Number): Unknow type. + """ + SCALAR = 1 # Scalar summary + TENSOR = 2 # Tensor summary + IMAGE = 3 # Image summary + GRAPH = 4 # graph + INVALID = 0xFF # unknow type + + +def get_event_file_name(prefix, suffix): + """ + Create file name: file_prefix + EVENT_FILE_NAME_MARK + time(seconds) + "." + Hostname + file_suffix. + + Args: + prefix (str): The prefix of file name. + suffix (str): The suffix of file name. + + Returns: + String, the name of event log file. + """ + _check_str_by_regular(prefix) + _check_str_by_regular(suffix) + file_name = "" + time_second = str(int(time.time())) + hostname = socket.gethostname() + + if prefix is not None: + file_name = file_name + prefix + + file_name = file_name + EVENT_FILE_NAME_MARK + time_second + "." + hostname + + if suffix is not None: + file_name = file_name + suffix + + return file_name + + +def package_init_event(): + """Package the summary init event.""" + init_event = Event() + init_event.wall_time = time.time() + version = EVENT_FILE_INIT_VERSION_MARK + str(EVENT_FILE_INIT_VERSION) + init_event.version = version + return init_event + + +def package_graph_event(data): + """ + Package the summary graph event. + + Args: + data (Bytes): Graph bytes string. + + Retruns: + Event, event log object. + """ + graph_event = Event() + graph_event.wall_time = time.time() + modelp = ModelProto() + modelp.ParseFromString(data) + graph_event.graph_def.CopyFrom(modelp.graph) + return graph_event + + +def package_summary_event(data_id, step): + """ + Package the summary to event protobuffer. + + Args: + data_id (Number): Summary data id. + step (Number): The recode step index. + + Returns: + Summary, the summary event. + """ + data_list = get_summary_data(data_id) + if data_list is None: + logger.error("The step(%r) does not have record data.", self.step) + del_summary_data(data_id) + # create the event of summary + summary_event = Event() + summary = summary_event.summary + + for value in data_list: + tag = value["name"] + data = value["data"] + summary_type = value["type"] + + # get the summary type and parse the tag + if summary_type is SummaryType.SCALAR: + logger.debug("Now process Scalar summary, tag = %r", tag) + summary_value = summary.value.add() + summary_value.tag = tag + summary_value.scalar_value = _get_scalar_summary(tag, data) + elif summary_type is SummaryType.TENSOR: + logger.debug("Now process Tensor summary, tag = %r", tag) + summary_value = summary.value.add() + summary_value.tag = tag + summary_tensor = summary_value.tensor + _get_tensor_summary(tag, data, summary_tensor) + elif summary_type is SummaryType.IMAGE: + logger.debug("Now process Image summary, tag = %r", tag) + summary_value = summary.value.add() + summary_value.tag = tag + summary_image = summary_value.image + _get_image_summary(tag, data, summary_image, MS_IMAGE_TENSOR_FORMAT) + else: + # The data is invalid ,jump the data + logger.error("Summary type is error, tag = %r", tag) + continue + + summary_event.wall_time = time.time() + summary_event.step = int(step) + return summary_event + + +def _nptype_to_prototype(np_value): + """ + Transform the np type to proto type. + + Args: + np_value (Type): Numpy data type. + + Returns: + Type, proto data type. + """ + np2pt_tbl = { + np.bool_: 'DT_BOOL', + np.int8: 'DT_INT8', + np.int16: 'DT_INT16', + np.int32: 'DT_INT32', + np.int64: 'DT_INT64', + np.uint8: 'DT_UINT8', + np.uint16: 'DT_UINT16', + np.uint32: 'DT_UINT32', + np.uint64: 'DT_UINT64', + np.float16: 'DT_FLOAT16', + np.float: 'DT_FLOAT64', + np.float32: 'DT_FLOAT32', + np.float64: 'DT_FLOAT64', + None: 'DT_UNDEFINED' + } + np_type = None + if np_value is None: + logger.error("The numpy value is none") + else: + np_type = np_value.dtype.type + + proto = np2pt_tbl.get(np_type, None) + if proto is None: + raise TypeError("No match for proto data type.") + + return proto + + +def _get_scalar_summary(tag: str, np_value): + """ + Package the scalar summary. + + Args: + tag (str): Summary tag describe. + np_value (Object): Scalary object. + + Returns: + Summary, return scalar summary content. + """ + logger.debug("Set(%r) the scalar summary value", tag) + if np_value.ndim == 0: + # is scalar + scalar_value = np_value.item() + elif np_value.ndim == 1: + # Because now GE can't providesumm the real shape info to convert the Tensor + # So consider the dim = 1, shape = (1,) tensor is scalar + scalar_value = np_value[0] + if np_value.shape != (1,): + logger.error("The tensor is not Scalar, tag = %r, Value = %r", tag, np_value) + else: + np_list = np_value.reshape(-1).tolist() + scalar_value = np_list[0] + logger.error("The value is not Scalar, tag = %r, Value = %r", tag, np_value) + + logger.debug("The tag(%r) value is: %r", tag, scalar_value) + return scalar_value + + +def _get_tensor_summary(tag: str, np_value, summary_tensor): + """ + Package the tensor summary. + + Args: + tag (str): Summary tag describe. + np_value (Type): Summary data type. + summary_tensor (Tensor): The tensor of summary. + + Retruns: + Summary, return tensor summary content. + """ + logger.debug("Set(%r) the tensor summary value", tag) + # get tensor dtype + tensor_dtype = _nptype_to_prototype(np_value) + summary_tensor.data_type = DataType.Value(tensor_dtype) + + # get the value list + tensor_value_list = np_value.reshape(-1).tolist() + summary_tensor.float_data.extend(tensor_value_list) + + # get the tensor dim + for v in np_value.shape: + summary_tensor.dims.append(v) + + return summary_tensor + + +def _get_image_summary(tag: str, np_value, summary_image, input_format='NCHW'): + """ + Package the image summary. + + Args: + tag (str): Summary tag describe. + np_value (Type): Summary data type. + summary_image (Tensor): The tensor of summary. + input_format (str): Data sort order index. Default: 'NCHW'. + + Returns: + Summary, return image summary content. + """ + logger.debug("Set(%r) the image summary value", tag) + if np_value.ndim != 4: + logger.error("The value is not Image, tag = %r, Value = %r", tag, np_value) + + # convert the tensor format + tensor = _convert_image_format(np_value, input_format) + + # convert the tensor dtype + # Do not assume that user passes in values in [0, 255], use data type to detect + scale_factor = 1 + if tensor.dtype == np.uint8: + scale_factor = 1 + elif np.max(tensor) <= 1 and np.min(tensor) >= 0: + scale_factor = 255 + tensor = tensor.astype(np.float32) + tensor = (tensor * scale_factor).astype(np.uint8) + + # create the image summary + height, width, channel, image_string = _make_image(tensor) + summary_image.height = height + summary_image.width = width + summary_image.colorspace = channel + summary_image.encoded_image = image_string + return summary_image + + +def _make_image(tensor, rescale=1): + """ + Convert a numpy representation of an image to Image protobuf. + + Args: + tensor (Tensor): The image data. + rescale (Number): The rescale value. Default: 1. + + Returns: + (Number, Number, Number, Bytes), return the height, width, channel, image string . + """ + height, width, channel = tensor.shape + scaled_height = int(height * rescale) + scaled_width = int(width * rescale) + image = Image.fromarray(tensor) + image = image.resize((scaled_width, scaled_height), Image.ANTIALIAS) + import io + output = io.BytesIO() + image.save(output, format='PNG') + image_string = output.getvalue() + output.close() + return height, width, channel, image_string + + +def _convert_image_format(np_tensor, input_format, out_format='HWC'): + """ + Convert the image format. + + Args: + np_tensor (Tensor): The image data. + input_format (str): Input data format. + out_format (str): The output data format. Default: 'HWC'. + + Returns: + Tensor, return format image. + """ + out_tensor = None + if np_tensor.ndim != len(input_format): + logger.error("The tensor(%r) can't convert the format(%r) because dim not same", + np_tensor, input_format) + return out_tensor + + input_format = input_format.upper() + + if len(input_format) == 4: + # convert the NCHW + if input_format != 'NCHW': + index = [input_format.find(c) for c in 'NCHW'] + tensor_nchw = np_tensor.transpose(index) + else: + tensor_nchw = np_tensor + + # make grid to expand N + tensor_chw = _make_canvas_for_imgs(tensor_nchw) + + # convert to out format + out_index = ['CHW'.find(c) for c in out_format] + out_tensor = tensor_chw.transpose(out_index) + else: + logger.error("Don't support the format(%r) convert", input_format) + return out_tensor + + +def _make_canvas_for_imgs(tensor, col_imgs=8): + """ + Expand the N, show imgs on a canvs. + + Args: + tensor (Tensor): The canvas value. + col_imgs (Number): The image colume number. Default: 8. + + Returns: + Tensor, retrun canvas of image. + """ + # expand the N1HW to N3HW + out_canvas = None + if tensor.shape[1] == 1: + tensor = np.concatenate([tensor, tensor, tensor], 1) + + # check the tensor format + if tensor.ndim != 4 or tensor.shape[1] != 3: + logger.error("The image tensor(%r) is not 'NCHW' format", tensor) + return out_canvas + + # expand the N + n = tensor.shape[0] + h = tensor.shape[2] + w = tensor.shape[3] + cols = min(n, col_imgs) + rows = int(np.ceil(float(n) / cols)) + + # creat the canvas: expand the n + out_canvas = np.zeros((3, h * rows, w * cols)) + i = 0 + for y in range(rows): + for x in range(cols): + if i >= n: + break + out_canvas[:, y * h:(y + 1) * h, x * w:(x + 1) * w] = tensor[i] + i = i + 1 + return out_canvas diff --git a/mindspore/train/summary/_summary_scheduler.py b/mindspore/train/summary/_summary_scheduler.py new file mode 100644 index 00000000000..fa5a228e6ad --- /dev/null +++ b/mindspore/train/summary/_summary_scheduler.py @@ -0,0 +1,305 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Schedule the event writer process.""" +import multiprocessing as mp +from enum import Enum, unique +from mindspore import log as logger +from ..._c_expression import Tensor +from ._summary_adapter import SummaryType, package_summary_event, save_summary_data + +# define the type of summary +FORMAT_SCALAR_STR = "Scalar" +FORMAT_TENSOR_STR = "Tensor" +FORMAT_IMAGE_STR = "Image" +FORMAT_BEGIN_SLICE = "[:" +FORMAT_END_SLICE = "]" + +# cache the summary data dict +# {id: SummaryData} +# |---[{"name": tag_name, "data": numpy}, {"name": tag_name, "data": numpy},...] +g_summary_data_id = 0 +g_summary_data_dict = {} +# cache the summary data file +g_summary_writer_id = 0 +g_summary_file = {} + + +@unique +class ScheduleMethod(Enum): + """Schedule method type.""" + FORMAL_WORKER = 0 # use the formal worker that receive small size data by queue + TEMP_WORKER = 1 # use the Temp worker that receive big size data by the global value(avoid copy) + CACHE_DATA = 2 # Cache data util have idle worker to process it + + +@unique +class WorkerStatus(Enum): + """Worker status.""" + WORKER_INIT = 0 # data is exist but not process + WORKER_PROCESSING = 1 # data is processing + WORKER_PROCESSED = 2 # data already processed + + +def _parse_tag_format(tag: str): + """ + Parse the tag. + + Args: + tag (str): Format: xxx[:Scalar] xxx[:Image] xxx[:Tensor]. + + Returns: + Tuple, (SummaryType, summary_tag). + """ + + summary_type = SummaryType.INVALID + summary_tag = tag + if tag is None: + logger.error("The tag is None") + return summary_type, summary_tag + + # search the slice + slice_begin = FORMAT_BEGIN_SLICE + slice_end = FORMAT_END_SLICE + index = tag.rfind(slice_begin) + if index is -1: + logger.error("The tag(%s) have not the key slice.", tag) + return summary_type, summary_tag + + # slice the tag + summary_tag = tag[:index] + + # check the slice end + if tag[-1:] != slice_end: + logger.error("The tag(%s) end format is error", tag) + return summary_type, summary_tag + + # check the type + type_str = tag[index + 2: -1] + logger.debug("The summary_tag is = %r", summary_tag) + logger.debug("The type_str value is = %r", type_str) + if type_str == FORMAT_SCALAR_STR: + summary_type = SummaryType.SCALAR + elif type_str == FORMAT_TENSOR_STR: + summary_type = SummaryType.TENSOR + elif type_str == FORMAT_IMAGE_STR: + summary_type = SummaryType.IMAGE + else: + logger.error("The tag(%s) type is invalid.", tag) + summary_type = SummaryType.INVALID + + return summary_type, summary_tag + + +class SummaryDataManager: + """Manage the summary global data cache.""" + def __init__(self): + global g_summary_data_dict + self.size = len(g_summary_data_dict) + + @classmethod + def summary_data_save(cls, data): + """Save the global summary cache.""" + global g_summary_data_id + data_id = g_summary_data_id + save_summary_data(data_id, data) + g_summary_data_id += 1 + return data_id + + @classmethod + def summary_file_set(cls, event_writer): + """Support the many event_writer.""" + global g_summary_file, g_summary_writer_id + g_summary_writer_id += 1 + g_summary_file[g_summary_writer_id] = event_writer + return g_summary_writer_id + + @classmethod + def summary_file_get(cls, writer_id=1): + ret = None + global g_summary_file + if writer_id in g_summary_file: + ret = g_summary_file.get(writer_id) + return ret + + +class WorkerScheduler: + """ + Create worker and schedule data to worker. + + Args: + writer_id (int): The index of writer. + """ + def __init__(self, writer_id): + # Create the process of write event file + self.write_lock = mp.Lock() + # Schedule info for all worker + # Format: {worker: (step, WorkerStatus)} + self.schedule_table = {} + # write id + self.writer_id = writer_id + self.has_graph = False + + def dispatch(self, step, data): + """ + Select schedule strategy and dispatch data. + + Args: + step (Number): The number of step index. + data (Object): The data of recode for summary. + + Retruns: + bool, run successfully or not. + """ + # save the data to global cache , convert the tensor to numpy + result, size, data = self._data_convert(data) + if result is False: + logger.error("The step(%r) summary data(%r) is invalid.", step, size) + return False + + data_id = SummaryDataManager.summary_data_save(data) + self._start_worker(step, data_id) + return True + + def _start_worker(self, step, data_id): + """ + Start worker. + + Args: + step (Number): The index of recode. + data_id (str): The id of work. + + Return: + bool, run successfully or not. + """ + # assign the worker + policy = self._make_policy() + if policy == ScheduleMethod.TEMP_WORKER: + worker = SummaryDataProcess(step, data_id, self.write_lock, self.writer_id) + # update the schedule table + self.schedule_table[worker] = (step, data_id, WorkerStatus.WORKER_INIT) + # start the worker + worker.start() + else: + logger.error("Do not support the other scheduler policy now.") + + # update the scheduler infor + self._update_scheduler() + return True + + def _data_convert(self, data_list): + """Convert the data.""" + if data_list is None: + logger.warning("The step does not have record data.") + return False, 0, None + + # convert the summary to numpy + size = 0 + for v_dict in data_list: + tag = v_dict["name"] + data = v_dict["data"] + # confirm the data is valid + summary_type, summary_tag = _parse_tag_format(tag) + if summary_type == SummaryType.INVALID: + logger.error("The data type is invalid, tag = %r, tensor = %r", tag, data) + return False, 0, None + if isinstance(data, Tensor): + # get the summary type and parse the tag + v_dict["name"] = summary_tag + v_dict["type"] = summary_type + v_dict["data"] = data.asnumpy() + size += v_dict["data"].size + else: + logger.error("The data type is invalid, tag = %r, tensor = %r", tag, data) + return False, 0, None + + return True, size, data_list + + def _update_scheduler(self): + """Check the worker status and update schedule table.""" + workers = list(self.schedule_table.keys()) + for worker in workers: + if not worker.is_alive(): + # update the table + worker.join() + del self.schedule_table[worker] + + def close(self): + """Confirm all worker is end.""" + workers = self.schedule_table.keys() + for worker in workers: + if worker.is_alive(): + worker.join() + + def _make_policy(self): + """Select the schedule strategy by data.""" + # now only support the temp worker + return ScheduleMethod.TEMP_WORKER + + +class SummaryDataProcess(mp.Process): + """ + Process that consume the summarydata. + + Args: + step (int): The index of step. + data_id (int): The index of summary data. + write_lock (Lock): The process lock for writer same file. + writer_id (int): The index of writer. + """ + def __init__(self, step, data_id, write_lock, writer_id): + super(SummaryDataProcess, self).__init__() + self.daemon = True + self.writer_id = writer_id + self.writer = SummaryDataManager.summary_file_get(self.writer_id) + if self.writer is None: + logger.error("The writer_id(%r) does not have writer", writer_id) + self.step = step + self.data_id = data_id + self.write_lock = write_lock + self.name = "SummaryDataConsumer_" + str(self.step) + + def run(self): + """The consumer is process the step data and exit.""" + # convert the data to event + # All exceptions need to be caught and end the queue + try: + logger.debug("process(%r) process a data(%r)", self.name, self.step) + # package the summary event + summary_event = package_summary_event(self.data_id, self.step) + # send the event to file + self._write_summary(summary_event) + except Exception as e: + logger.error("Summary data mq consumer exception occurred, value = %r", e) + + def _write_summary(self, summary_event): + """ + Write the summary to event file. + + Note: + The write record format: + 1 uint64 : data length. + 2 uint32 : mask crc value of data length. + 3 bytes : data. + 4 uint32 : mask crc value of data. + + Args: + summary_event (Event): The summary event of proto. + + """ + event_str = summary_event.SerializeToString() + self.write_lock.acquire() + self.writer.write_event_to_file(event_str) + self.writer.flush() + self.write_lock.release() diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py new file mode 100644 index 00000000000..d96ac4773ae --- /dev/null +++ b/mindspore/train/summary/summary_record.py @@ -0,0 +1,236 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Record the summary event.""" +import os +import threading +from mindspore import log as logger +from ._summary_scheduler import WorkerScheduler, SummaryDataManager +from ._summary_adapter import get_event_file_name, package_graph_event +from ._event_writer import EventRecord +from .._utils import _make_directory +from ..._checkparam import _check_str_by_regular + +# cache the summary data +_summary_tensor_cache = {} +_summary_lock = threading.Lock() + + +def _cache_summary_tensor_data(summary): + """ + Get the time of ms. + + Args: + summary (list): [{"name": tag_name, "data": tensor}, {"name": tag_name, "data": tensor},...]. + """ + _summary_lock.acquire() + if "SummaryRecord" in _summary_tensor_cache: + for record in summary: + _summary_tensor_cache["SummaryRecord"].append(record) + else: + _summary_tensor_cache["SummaryRecord"] = summary + _summary_lock.release() + return True + + +class SummaryRecord: + """ + Summary log record. + + SummaryRecord is used to record the summary value. + The API will create an event file in a given directory and add summaries and events to it. + + Args: + log_dir (str): The log_dir is a directory location to save the summary. + queue_max_size (int): The capacity of event queue.(reserved). Default: 0. + flush_time (int): Frequency to flush the summaries to disk, the unit is second. Default: 120. + file_prefix (str): The prefix of file. Default: "events". + file_suffix (str): The suffix of file. Default: "_MS". + network (Cell): Obtain a pipeline through network for saving graph summary. Default: None. + + Raises: + TypeError: If `queue_max_size` and `flush_time` is not int, or `file_prefix` and `file_suffix` is not str. + RuntimeError: If the log_dir can not be resolved to a canonicalized absolute pathname. + + Examples: + >>> summary_record = SummaryRecord(log_dir="/opt/log", queue_max_size=50, flush_time=6, + >>> file_prefix="xxx_", file_suffix="_yyy") + """ + def __init__(self, + log_dir, + queue_max_size=0, + flush_time=120, + file_prefix="events", + file_suffix="_MS", + network=None): + + _check_str_by_regular(file_prefix) + _check_str_by_regular(file_suffix) + self.log_path = _make_directory(log_dir) + + if not isinstance(queue_max_size, int) or not isinstance(flush_time, int): + raise TypeError("`queue_max_size` and `flush_time` should be int") + if not isinstance(file_prefix, str) or not isinstance(file_suffix, str): + raise TypeError("`file_prefix` and `file_suffix` should be str.") + + self.queue_max_size = queue_max_size + if queue_max_size < 0: + # 0 is not limit + logger.warning("The queue_max_size(%r) set error, will use the default value: 0", queue_max_size) + self.queue_max_size = 0 + + self.flush_time = flush_time + if flush_time <= 0: + logger.warning("The flush_time(%r) set error, will use the default value: 120", flush_time) + self.flush_time = 120 + + self.prefix = file_prefix + self.suffix = file_suffix + + # create the summary writer file + self.event_file_name = get_event_file_name(self.prefix, self.suffix) + if self.log_path[-1:] == '/': + self.full_file_name = self.log_path + self.event_file_name + else: + self.full_file_name = self.log_path + '/' + self.event_file_name + + try: + self.full_file_name = os.path.realpath(self.full_file_name) + except Exception as ex: + raise RuntimeError(ex) + self.event_writer = EventRecord(self.full_file_name, self.flush_time) + self.writer_id = SummaryDataManager.summary_file_set(self.event_writer) + self.worker_scheduler = WorkerScheduler(self.writer_id) + + self.step = 0 + self._closed = False + self.network = network + self.has_graph = False + + def record(self, step, train_network=None): + """ + Record the summary. + + Args: + step (int): Represents training step number. + train_network (Cell): The network that called the callback. + + Examples: + >>> summary_record = SummaryRecord(log_dir="/opt/log", queue_max_size=50, flush_time=6, + >>> file_prefix="xxx_", file_suffix="_yyy") + >>> summary_record.record(step=2) + + Returns: + bool, whether the record process is successful or not. + """ + logger.info("SummaryRecord step is %r.", step) + if self._closed: + logger.error("The record writer is closed.") + return False + if not isinstance(step, int) or isinstance(step, bool): + raise ValueError("`step` should be int") + # Set the current summary of train step + self.step = step + + if self.network is not None and self.has_graph is False: + graph_proto = self.network.get_func_graph_proto() + if graph_proto is None and train_network is not None: + graph_proto = train_network.get_func_graph_proto() + if graph_proto is None: + logger.error("Failed to get proto for graph") + else: + self.event_writer.write_event_to_file( + package_graph_event(graph_proto).SerializeToString()) + self.event_writer.flush() + self.has_graph = True + + data = _summary_tensor_cache.get("SummaryRecord") + if data is None: + logger.error("The step(%r) does not have record data.", self.step) + return False + if self.queue_max_size > 0 and len(data) > self.queue_max_size: + logger.error("The size of data record is %r, which is greater than queue_max_size %r.", len(data), + self.queue_max_size) + + # clean the data of cache + del _summary_tensor_cache["SummaryRecord"] + + # process the data + self.worker_scheduler.dispatch(self.step, data) + + # count & flush + self.event_writer.count_event() + self.event_writer.flush_cycle() + + logger.debug("Send the summary data to scheduler for saving, step = %d", self.step) + return True + + @property + def log_dir(self): + """ + Get the full path of the log file. + + Examples: + >>> summary_record = SummaryRecord(log_dir="/opt/log", queue_max_size=50, flush_time=6, + >>> file_prefix="xxx_", file_suffix="_yyy") + >>> print(summary_record.log_dir) + + Returns: + String, the full path of log file. + """ + return self.event_writer.full_file_name + + def flush(self): + """ + Flush the event file to disk. + + Call it to make sure that all pending events have been written to disk. + + Examples: + >>> summary_record = SummaryRecord(log_dir="/opt/log", queue_max_size=50, flush_time=6, + >>> file_prefix="xxx_", file_suffix="_yyy") + >>> summary_record.flush() + """ + if self._closed: + logger.error("The record writer is closed and can not flush.") + else: + self.event_writer.flush() + + def close(self): + """ + Flush all events and close summary records. + + Examples: + >>> summary_record = SummaryRecord(log_dir="/opt/log", queue_max_size=50, flush_time=6, + >>> file_prefix="xxx_", file_suffix="_yyy") + >>> summary_record.close() + """ + if not self._closed: + self._check_data_before_close() + self.worker_scheduler.close() + # event writer flush and close + self.event_writer.close() + self._closed = True + + def __del__(self): + """Process exit is called.""" + if hasattr(self, "worker_scheduler"): + if self.worker_scheduler: + self.close() + + def _check_data_before_close(self): + "Check whether there is any data in the cache, and if so, call record" + data = _summary_tensor_cache.get("SummaryRecord") + if data is not None: + self.record(self.step) diff --git a/package.sh b/package.sh new file mode 100755 index 00000000000..0d4147c9f66 --- /dev/null +++ b/package.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +BASEPATH=$(cd "$(dirname $0)"; pwd) +echo "${BASEPATH}" +cd "${BASEPATH}" +BUILD_PATH="${BASEPATH}/build" +PACKAGE_PATH="${BUILD_PATH}/package" +OUTPUT_PATH="${BASEPATH}/output" + +mk_new_dir() { + local create_dir="$1" # the target to make + + if [[ -d "${create_dir}" ]];then + rm -rf "${create_dir}" + fi + + mkdir -pv "${create_dir}" +} + +to_lower () { + echo "$1" | tr '[:upper:]' '[:lower:]' +} + +COMMIT_ID=$(git log --format='[sha1]:%h,[branch]:%d' -1 | sed 's/ //g') +export COMMIT_ID + +PYTHON=$(which python3) +PYTHON_VERSION=$("${PYTHON}" -V 2>&1 | awk '{print $2}' | cut -d. -f-2) +if [[ $(uname) == "Linux" ]]; then + if [[ "${PYTHON_VERSION}" == "3.7" ]]; then + PY_TAGS="cp37-cp37m" + elif [[ "${PYTHON_VERSION}" == "3.6" ]]; then + PY_TAGS="cp36-cp36m" + else + echo "Could not find 'Python 3.6' or 'Python 3.7'" + exit 1 + fi + PLATFORM_TAG=$(to_lower "$(uname)_$(uname -m)") +elif [[ $(uname) == "Darwin" ]]; then + if [[ "${PYTHON_VERSION}" == "3.7" || "${PYTHON_VERSION}" == "3.6" ]]; then + PY_TAGS="py3-none" + else + echo "Could not find 'Python 3.6' or 'Python 3.7'" + exit 1 + fi + PLATFORM_TAG="any" +fi +echo "=========${BASEPATH}===================" +mk_new_dir "${OUTPUT_PATH}" + +#copy necessary file to pack_path +cp ${BASEPATH}/mindspore/*.py "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/nn" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/_extends" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/parallel" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/mindrecord" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/train" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/model_zoo" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/common" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/ops" "${PACKAGE_PATH}/mindspore" +cp -rf "${BUILD_PATH}/../mindspore/communication" "${PACKAGE_PATH}/mindspore" + +if [[ "X$2" = "Xgpu" ]]; then + echo "package akg when gpu enable." + cp -rf "${BASEPATH}/mindspore/akg" "${PACKAGE_PATH}" + if [[ -d "${BUILD_PATH}/mindspore/incubator-tvm" ]]; then + cp -rf "${BUILD_PATH}/mindspore/incubator-tvm/topi/python/topi" "${PACKAGE_PATH}/akg" + cp -rf "${BUILD_PATH}/mindspore/incubator-tvm/python/tvm" "${PACKAGE_PATH}/akg" + fi +fi + +# move dataset +if [[ -d "${BASEPATH}/mindspore/dataset" ]]; then + cp -rf "${BASEPATH}/mindspore/dataset" "${PACKAGE_PATH}/mindspore" +fi + +cd "${PACKAGE_PATH}" +if [ -n "$1" ];then + export BACKEND_POLICY=$1 +else + export BACKEND_POLICY="ms" +fi +${PYTHON} "${BASEPATH}/setup_package.py" bdist_wheel + +chmod -R 700 ${PACKAGE_PATH}/mindspore/ +chmod -R 700 ${PACKAGE_PATH}/mindspore.egg-info/ + +# rename package +PACKAGE_FULL_NAME=$(find "${PACKAGE_PATH}" -iname "*.whl") +PACKAGE_BASE_NAME=$(echo ${PACKAGE_FULL_NAME} | awk -F / '{print $NF}' | awk -F - '{print $1"-"$2}') + +PACKAGE_NEW_NAME="${PACKAGE_BASE_NAME}-${PY_TAGS}-${PLATFORM_TAG}.whl" +cp -rf "${PACKAGE_PATH}/dist"/*.whl "${PACKAGE_PATH}/${PACKAGE_NEW_NAME}" +cp -f "${PACKAGE_PATH}/${PACKAGE_NEW_NAME}" "${OUTPUT_PATH}" + +cd "${BASEPATH}" + +echo "------Successfully created mindspore package------" diff --git a/predict/.gitignore b/predict/.gitignore new file mode 100644 index 00000000000..caf7aec495e --- /dev/null +++ b/predict/.gitignore @@ -0,0 +1,14 @@ +# git ignore file for predict + +#flatbuf generated file +schema/*_generated.h +schema/inner/*_generated.h +module/tvm_module/lite/include/*_generated.h + +#tvm fbs files +module/tvm_module/lite/tune/convert/*.fbs + +#doTest dir +test/doTest/ + + diff --git a/predict/CMakeLists.txt b/predict/CMakeLists.txt new file mode 100755 index 00000000000..26419327697 --- /dev/null +++ b/predict/CMakeLists.txt @@ -0,0 +1,78 @@ +cmake_minimum_required(VERSION 3.12.1) +project (mindspore-predict) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") +set(CMAKE_BUILD_TYPE "Release") + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") + +option(ENABLE_ASAN "Enable Google Sanitizer to find memory bugs" OFF) +option(ENABLE_PREDICT_ARM64 "predict arm64" OFF) +option(ENABLE_PREDICT_ARM32 "predict arm32" OFF) + +set(PREDICT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(PREDICT_BUILD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build) +set(3RD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../third_party) +set(DOTEST_DIR ${PREDICT_BUILD_DIR}/test/doTest) + +include_directories(${3RD_DIR}) +include_directories(${3RD_DIR}/flatbuffers/include/) +include_directories(${3RD_DIR}/protobuf/build/include/) +include_directories(${3RD_DIR}/googletest/googletest/include/) +include_directories(${3RD_DIR}/googletest/googlemock/include/) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/module/tvm_kernel/lite/include/) +include_directories(${PREDICT_DIR}/module/tvm_kernel/incubator-tvm/3rdparty/dlpack/include) +include_directories(common) + +if(ENABLE_PREDICT_ARM64 OR ENABLE_PREDICT_ARM32) + message("*********************predict compile arm*********************") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMS_USE_ARM=1") + set(ANDROID_NDK $ENV{ANDROID_NDK}) + if(ANDROID_NDK) + add_subdirectory(${3RD_DIR}/googletest ${CMAKE_BINARY_DIR}/googletest) + link_directories(${PREDICT_BUILD_DIR}/googletest/googlemock/gtest) + + add_subdirectory(${3RD_DIR}/securec ${CMAKE_BINARY_DIR}/securec) + link_directories(${PREDICT_BUILD_DIR}/securec/src) + else() + message(FATAL_ERROR "please set ANDROID_NDK in environment variable for example: export ANDROID_NDK=/root/usr/android-ndk-r16b/") + endif() + + include_directories(${ANDROID_SYSROOT}/usr/include/) + if(${ANDROID_ABI} STREQUAL "armeabi-v7a") + include_directories(${ANDROID_SYSROOT}/usr/include/arm-linux-androideabi) + elseif(${ANDROID_ABI} STREQUAL "arm64-v8a") + include_directories(${ANDROID_SYSROOT}/usr/include/aarch64-linux-android) + else() + include_directories(${ANDROID_SYSROOT}/usr/include/arm-linux-androideabi) + endif() + +else() + # include libsecurec.a x86 + message("*********************predict compile x86*********************") + if(EXISTS "${PREDICT_DIR}/../build/mindspore/securec/src/libsecurec.a") + link_directories(${PREDICT_DIR}/../build/mindspore/securec/src) + else() + include(${PREDICT_DIR}/../cmake/dependency_securec.cmake) + link_directories(${PREDICT_BUILD_DIR}/securec/src) + endif() + + # include libgtest.so x86 + if(EXISTS "${PREDICT_DIR}/../build/googletest/googlemock/gtest/libgtest.so") + link_directories(${PREDICT_DIR}/../build/googletest/googlemock/gtest) + else() + include(${PREDICT_DIR}/../cmake/dependency_gtest.cmake) + link_directories(${PREDICT_BUILD_DIR}/googletest/googlemock/gtest) + endif() +endif() + +if (CODE_COVERAGE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage -O0") +endif() + +add_subdirectory(common) +add_subdirectory(src) +add_subdirectory(benchmark) +add_subdirectory(test) +add_subdirectory(module) diff --git a/predict/benchmark/CMakeLists.txt b/predict/benchmark/CMakeLists.txt new file mode 100755 index 00000000000..22f87d8a97e --- /dev/null +++ b/predict/benchmark/CMakeLists.txt @@ -0,0 +1,38 @@ + +cmake_minimum_required(VERSION 3.12) +project(benchmark) + +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_BUILD_TYPE "Debug") + +#include 3rd +include_directories(${3RD_DIR}/protobuf/build/include) +include_directories(${3RD_DIR}/securec/include) +include_directories(${3RD_DIR}/flatbuffers/include) +include_directories(${3RD_DIR}/googletest/googletest/include) +include_directories(${3RD_DIR}/googletest/googlemock/include) +include_directories(${PREDICT_DIR}/module/tvm_kernel/incubator-tvm/3rdparty/dlpack/include) +include_directories(${3RD_DIR}/flatbuffers/include) +include_directories(${3RD_DIR}/securec/include) + +#include ms +include_directories(.) +include_directories(${PREDICT_DIR}) + +set(COMMON_SRC ${PREDICT_DIR}/common/flag_parser.cc + ${PREDICT_DIR}/common/file_utils.cc + ${PREDICT_DIR}/common/func_utils.cc + ${PREDICT_DIR}/common/mslog.cc + ${PREDICT_DIR}/common/utils.cc) + +link_directories(${CMAKE_CURRENT_SOURCE_DIR}/../output/lib/) + +add_executable(benchmark main.cc benchmark.cc ${COMMON_SRC}) + +target_link_libraries(benchmark mspredict libsecurec.a) +add_dependencies(benchmark tvm_kernel) +add_dependencies(benchmark securec) + +add_custom_command(TARGET benchmark POST_BUILD + COMMAND mkdir -pv ${DOTEST_DIR} + COMMAND cp ${PREDICT_BUILD_DIR}/benchmark/benchmark ${DOTEST_DIR}) diff --git a/predict/benchmark/README.md b/predict/benchmark/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/predict/benchmark/benchmark.cc b/predict/benchmark/benchmark.cc new file mode 100644 index 00000000000..c55d03e450e --- /dev/null +++ b/predict/benchmark/benchmark.cc @@ -0,0 +1,451 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark/benchmark.h" +#include +#include +#include +#include +#include +#include "include/session.h" + +namespace mindspore { +namespace predict { +STATUS Benchmark::GenerateRandomData(size_t size, void *data) { + MS_ASSERT(data != nullptr); + char *castedData = static_cast(data); + for (size_t i = 0; i < size; i++) { + castedData[i] = static_cast(i); + } + return RET_OK; +} + +STATUS Benchmark::GenerateInputData() { + for (Tensor *tensor : msInputs) { + MS_ASSERT(tensor != nullptr); + auto ret = tensor->MallocData(); + if (ret != RET_OK) { + MS_LOGE("MallocData for inTensor failed %d", ret); + return ret; + } + MS_ASSERT(tensor->GetData() != nullptr); + auto tensorByteSize = tensor->GetDataSize(); + auto status = GenerateRandomData(tensorByteSize, tensor->GetData()); + if (status != RET_OK) { + MS_LOGE("GenerateRandomData for inTensor failed %d", status); + return status; + } + } + return RET_OK; +} + +STATUS Benchmark::LoadInput() { + size_t size = 0; + char *graphBuf = ReadFile(_flags->modelPath.c_str(), &size); + if (graphBuf == nullptr) { + MS_LOGE("Load graph failed, path %s", _flags->modelPath.c_str()); + return RET_ERROR; + } + + this->msInputs = session->GetInput(); + + if (_flags->inDataPath.empty()) { + auto status = GenerateInputData(); + if (status != RET_OK) { + delete graphBuf; + MS_LOGE("Generate input data error %d", status); + return status; + } + } else { + auto status = ReadInputFile(); + if (status != RET_OK) { + delete graphBuf; + MS_LOGE("ReadInputFile error, %d", status); + return status; + } + } + delete graphBuf; + return RET_OK; +} + +STATUS Benchmark::ReadInputFile() { + MS_ASSERT(msInputs.size() <= 1); + if (msInputs.empty()) { + return RET_OK; + } + Tensor *inTensor = msInputs.at(0); + MS_ASSERT(inTensor != nullptr); + + size_t size; + char *binBuf = ReadFile(_flags->inDataPath.c_str(), &size); + if (binBuf == nullptr) { + return RET_ERROR; + } + auto tensorDataSize = inTensor->GetDataSize(); + if (size != tensorDataSize) { + MS_LOGE("Input binary file size error, required: %zu, in fact: %zu", tensorDataSize, size); + delete binBuf; + return RET_ERROR; + } + inTensor->SetData(binBuf); + binBuf = nullptr; + + return RET_OK; +} + +// calibData is FP32 +STATUS Benchmark::ReadCalibData() { + const char *calibDataPath = _flags->calibDataPath.c_str(); + // read calib data + std::ifstream inFile(calibDataPath); + if (!inFile.good()) { + MS_LOGE("file: %s is not exist", calibDataPath); + return RET_PARAM_INVALID; + } + + if (!inFile.is_open()) { + MS_LOGE("file: %s open failed", calibDataPath); + inFile.close(); + return RET_PARAM_INVALID; + } + + std::string line; + MS_LOGI("Start reading calibData file"); + std::string tensorName; + while (!inFile.eof()) { + getline(inFile, line); + std::stringstream stringLine1(line); + size_t dim = 0; + stringLine1 >> tensorName >> dim; + std::vector dims; + size_t shapeSize = 1; + for (size_t i = 0; i < dim; i++) { + size_t tmpDim; + stringLine1 >> tmpDim; + dims.push_back(tmpDim); + shapeSize *= tmpDim; + } + + getline(inFile, line); + std::stringstream stringLine2(line); + std::vector tensorData; + for (size_t i = 0; i < shapeSize; i++) { + float tmpData; + stringLine2 >> tmpData; + tensorData.push_back(tmpData); + } + + std::unique_ptr checkTensor(new CheckTensor(dims, tensorData)); + this->calibData.insert(std::make_pair(tensorName, checkTensor.release())); + } + inFile.close(); + MS_LOGI("Finish reading calibData file"); + return RET_OK; +} + +// tensorData need to be converter first +float Benchmark::CompareData(const std::string &nodeName, std::vector msShape, float *msTensorData) { + auto iter = this->calibData.find(nodeName); + if (iter != this->calibData.end()) { + std::vector castedMSShape; + size_t shapeSize = 1; + for (int64_t dim : msShape) { + castedMSShape.push_back(size_t(dim)); + shapeSize *= dim; + } + + CheckTensor *calibTensor = iter->second; + if (calibTensor->shape != castedMSShape) { + std::ostringstream oss; + oss << "Shape of mslite output("; + for (auto dim : castedMSShape) { + oss << dim << ","; + } + oss << ") and shape source model output("; + for (auto dim : calibTensor->shape) { + oss << dim << ","; + } + oss << ") are different"; + MS_LOGE("%s", oss.str().c_str()); + return -1; + } + + float meanBias = 0; + std::ostringstream outputData; + outputData << "Data of node " << nodeName << " : "; + for (size_t j = 0; j < shapeSize; j++) { + if (j < printNum) { + outputData << msTensorData[j] << " "; + } + if (fabs(calibTensor->data.at(j)) > minFloatThr) { + double bias = fabs(msTensorData[j] - calibTensor->data.at(j)) / fabs(calibTensor->data.at(j)); + meanBias += bias; + } + } + meanBias /= shapeSize; + MS_LOGI("%s", outputData.str().c_str()); + + if (meanBias <= minFloatThr) { + MS_LOGI("Mean bias of node %s : 0%%", nodeName.c_str()); + } else { + MS_LOGI("Mean bias of node %s : %f%%", nodeName.c_str(), meanBias * percentage); + } + return meanBias; + } else { + MS_LOGI("%s is not in Source Model output", nodeName.c_str()); + return -1; + } +} + +STATUS Benchmark::CompareOutput(const std::map> &msOutputs) { + float totalBias = 0; + int totalSize = 0; + bool hasError = false; + for (const auto &msOutput : msOutputs) { + std::string nodeName = msOutput.first; + auto tensors = msOutput.second; + for (auto tensor : tensors) { + MS_ASSERT(tensor->GetData() != nullptr); + float bias = CompareData(nodeName, tensor->GetDims(), static_cast(tensor->GetData())); + if (bias >= 0) { + totalBias += bias; + totalSize++; + } else { + hasError = true; + break; + } + } + } + + if (!hasError) { + float meanBias; + if (totalSize != 0) { + meanBias = totalBias / totalSize * percentage; + } else { + meanBias = 0; + } + + MS_LOGI("Mean bias all node : %f%%", meanBias); + + if (meanBias > 1) { + MS_LOGE("Mean bias of all nodes is too big: %f%%", meanBias); + return RET_ERROR; + } else { + return RET_OK; + } + } else { + MS_LOGE("Error in CompareData"); + return RET_ERROR; + } +} + +STATUS Benchmark::MarkPerformance() { + MS_LOGI("Running warm up loops..."); + for (int i = 0; i < _flags->warmUpLoopCount; i++) { + auto status = session->Run(msInputs); + if (status != RET_OK) { + MS_LOGE("Inference error %d", status); + return status; + } + } + + MS_LOGI("Running benchmark loops..."); + uint64_t timeMin = maxTimeThr; + uint64_t timeMax = 0; + uint64_t timeAvg = 0; + for (int i = 0; i < _flags->loopCount; i++) { + uint64_t start = GetTimeUs(); + auto status = session->Run(msInputs); + if (status != RET_OK) { + MS_LOGE("Inference error %d", status); + return status; + } + + uint64_t end = GetTimeUs(); + uint64_t time = end - start; + timeMin = std::min(timeMin, time); + timeMax = std::max(timeMax, time); + timeAvg += time; + + msOutputs = session->GetAllOutput(); + if (cleanData) { + for (auto &msOutput : msOutputs) { + for (auto &outputTensor : msOutput.second) { + delete outputTensor; + } + } + msOutputs.clear(); + } + } + if (_flags->loopCount > 0) { + timeAvg /= _flags->loopCount; + MS_LOGI("MinRunTime = %f ms, MaxRuntime = %f ms, AvgRunTime = %f ms", timeMin / US2MS, timeMax / US2MS, + timeAvg / US2MS); + } + return RET_OK; +} + +STATUS Benchmark::MarkAccuracy() { + MS_LOGI("MarkAccuracy"); + + auto status = session->Run(msInputs); + if (status != RET_OK) { + MS_LOGE("Inference error %d", status); + return status; + } + msOutputs = session->GetAllOutput(); + + ReadCalibData(); + status = CompareOutput(msOutputs); + if (cleanData) { + for (auto &msOutput : msOutputs) { + for (auto &outputTensor : msOutput.second) { + delete outputTensor; + } + } + msOutputs.clear(); + } + return status; +} + +STATUS Benchmark::CleanData() { + if (cleanData) { + for (auto &msInput : msInputs) { + delete msInput; + } + msInputs.clear(); + for (auto &data : calibData) { + data.second->shape.clear(); + data.second->data.clear(); + delete data.second; + } + calibData.clear(); + } + return RET_OK; +} + +STATUS Benchmark::RunBenchmark() { + // Load graph + std::string comment = modelName; + + MS_LOGI("start reading model file"); + size_t size = 0; + char *graphBuf = ReadFile(_flags->modelPath.c_str(), &size); + if (graphBuf == nullptr) { + MS_LOGE("Load graph failed while running %s", comment.c_str()); + return RET_ERROR; + } + + uint64_t startPrepareTime = GetTimeUs(); + session = CreateSession(graphBuf, size, ctx); + if (session == nullptr) { + delete graphBuf; + MS_LOGE("new session failed while running %s", comment.c_str()); + return RET_ERROR; + } + uint64_t endPrepareTime = GetTimeUs(); + MS_LOGI("PrepareTime = %f ms, ", (endPrepareTime - startPrepareTime) / US2MS); + + // Load input + MS_LOGI("start generate input data"); + auto status = LoadInput(); + if (status != RET_OK) { + delete graphBuf; + MS_LOGE("Generate input data error"); + return status; + } + + if (!_flags->calibDataPath.empty()) { + status = MarkAccuracy(); + if (status != RET_OK) { + delete graphBuf; + MS_LOGE("Run MarkAccuracy error: %d", status); + return status; + } + } else { + status = MarkPerformance(); + if (status != RET_OK) { + delete graphBuf; + MS_LOGE("Run MarkPerformance error: %d", status); + return status; + } + } + + CleanData(); + delete graphBuf; + return RET_OK; +} + +STATUS Benchmark::Init() { + if (this->_flags == nullptr) { + return RET_ERROR; + } + MS_LOGI("ModelPath = %s", this->_flags->modelPath.c_str()); + MS_LOGI("InDataPath = %s", this->_flags->inDataPath.c_str()); + MS_LOGI("TensorDataType = %s", this->_flags->tensorDataTypeIn.c_str()); + MS_LOGI("LoopCount = %d", this->_flags->loopCount); + MS_LOGI("WarmUpLoopCount = %d", this->_flags->warmUpLoopCount); + MS_LOGI("NumThreads = %d", this->_flags->numThreads); + MS_LOGI("calibDataPath = %s", this->_flags->calibDataPath.c_str()); + + this->_flags->inDataType = this->_flags->inDataTypeIn == "img" ? kImage : kBinary; + if (this->_flags->tensorDataTypeIn == "float") { + this->_flags->tensorDataType = DataType_DT_FLOAT; + } + + if (_flags->modelPath.empty()) { + MS_LOGE("modelPath is required"); + return RET_ERROR; + } + + modelName = _flags->modelPath.substr(_flags->modelPath.find_last_of("/") + 1); + + return RET_OK; +} + +int RunBenchmark(int argc, const char **argv) { + BenchmarkFlags flags; + Option err = flags.ParseFlags(argc, argv); + + if (err.IsSome()) { + std::cerr << err.Get() << std::endl; + std::cerr << flags.Usage() << std::endl; + return -1; + } + + if (flags.help) { + std::cerr << flags.Usage() << std::endl; + return 0; + } + + Benchmark mBenchmark(&flags); + auto status = mBenchmark.Init(); + if (status != RET_OK) { + MS_LOGE("Benchmark init Error : %d", status); + return 1; + } + + status = mBenchmark.RunBenchmark(); + if (status != RET_OK) { + MS_LOGE("Run Benchmark Error : %d", status); + return 1; + } + + MS_LOGI("end of benchmark"); + return 0; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/benchmark/benchmark.h b/predict/benchmark/benchmark.h new file mode 100644 index 00000000000..03cd117df0f --- /dev/null +++ b/predict/benchmark/benchmark.h @@ -0,0 +1,142 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_BENCHMARK_BENCHMARK_H_ +#define PREDICT_BENCHMARK_BENCHMARK_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/flag_parser.h" +#include "common/file_utils.h" +#include "common/func_utils.h" +#include "common/mslog.h" +#include "common/utils.h" +#include "include/errorcode.h" +#include "include/session.h" +#include "include/tensor.h" +#include "schema/inner/ms_generated.h" +#include "src/graph.h" +#include "src/graph_execution.h" +#include "src/op.h" + +namespace mindspore { +namespace predict { +enum InDataType { kImage = 0, kBinary = 1 }; + +struct CheckTensor { + CheckTensor(const std::vector &shape, const std::vector &data) { + this->shape = shape; + this->data = data; + } + std::vector shape; + std::vector data; +}; + +class BenchmarkFlags : public virtual FlagParser { + public: + BenchmarkFlags() { + // common + AddFlag(&BenchmarkFlags::modelPath, "modelPath", "Input model path", ""); + AddFlag(&BenchmarkFlags::tensorDataTypeIn, "tensorDataType", "Data type of input Tensor. float", "float"); + AddFlag(&BenchmarkFlags::inDataPath, "inDataPath", "Input data path, if not set, use random input", ""); + // MarkPerformance + AddFlag(&BenchmarkFlags::loopCount, "loopCount", "Run loop count", 10); + AddFlag(&BenchmarkFlags::numThreads, "numThreads", "Run threads number", 2); + AddFlag(&BenchmarkFlags::warmUpLoopCount, "warmUpLoopCount", "Run warm up loop", 3); + // MarkAccuracy + AddFlag(&BenchmarkFlags::calibDataPath, "calibDataPath", "Calibration data file path", ""); + } + + ~BenchmarkFlags() override = default; + + public: + // common + std::string modelPath; + std::string inDataPath; + InDataType inDataType; + std::string inDataTypeIn; + DataType tensorDataType; + std::string tensorDataTypeIn; + // MarkPerformance + int loopCount; + int numThreads; + int warmUpLoopCount; + // MarkAccuracy + std::string calibDataPath; +}; + +class Benchmark { + public: + explicit Benchmark(BenchmarkFlags *flags) : _flags(flags) {} + + virtual ~Benchmark() = default; + + STATUS Init(); + STATUS RunBenchmark(); + + private: + // call GenerateInputData or ReadInputFile to init inputTensors + STATUS LoadInput(); + + // call GenerateRandomData to fill inputTensors + STATUS GenerateInputData(); + + STATUS GenerateRandomData(size_t size, void *data); + + STATUS ReadInputFile(); + + STATUS ReadCalibData(); + + STATUS CleanData(); + + STATUS CompareOutput(const std::map> &msOutputs); + + float CompareData(const std::string &nodeName, std::vector msShape, float *msTensorData); + + STATUS MarkPerformance(); + + STATUS MarkAccuracy(); + + private: + BenchmarkFlags *_flags; + std::shared_ptr session; + Context ctx; + std::vector msInputs; + std::map> msOutputs; + std::unordered_map calibData; + std::string modelName = ""; + bool cleanData = true; + + const float US2MS = 1000.0f; + const float percentage = 100.0f; + const int printNum = 50; + const float minFloatThr = 0.0000001f; + + const uint64_t maxTimeThr = 1000000; +}; + +int RunBenchmark(int argc, const char **argv); +} // namespace predict +} // namespace mindspore +#endif // PREDICT_BENCHMARK_BENCHMARK_H_ diff --git a/predict/benchmark/main.cc b/predict/benchmark/main.cc new file mode 100644 index 00000000000..66e473a42a0 --- /dev/null +++ b/predict/benchmark/main.cc @@ -0,0 +1,24 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "benchmark/benchmark.h" + +int main(int argc, const char **argv) { + signal(SIGSEGV, mindspore::predict::CoreDumpTraceFunc); + return mindspore::predict::RunBenchmark(argc, argv); +} diff --git a/predict/common/CMakeLists.txt b/predict/common/CMakeLists.txt new file mode 100755 index 00000000000..3734c26bc0a --- /dev/null +++ b/predict/common/CMakeLists.txt @@ -0,0 +1,17 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../third_party) + +add_compile_options(-fPIC) + +add_library(common_mid OBJECT + ${CMAKE_CURRENT_SOURCE_DIR}/common.h + ${CMAKE_CURRENT_SOURCE_DIR}/graph_util.cc + ${CMAKE_CURRENT_SOURCE_DIR}/file_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/flag_parser.cc + ${CMAKE_CURRENT_SOURCE_DIR}/func_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/module_registry.cc + ${CMAKE_CURRENT_SOURCE_DIR}/mslog.cc + ${CMAKE_CURRENT_SOURCE_DIR}/storage.cc + ${CMAKE_CURRENT_SOURCE_DIR}/utils.cc) diff --git a/predict/common/common.h b/predict/common/common.h new file mode 100644 index 00000000000..d93139abae9 --- /dev/null +++ b/predict/common/common.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_COMMON_H_ +#define PREDICT_COMMON_COMMON_H_ + +#include +#include "schema/inner/ms_generated.h" + +namespace mindspore { +namespace predict { +enum NCHW_SHAPE { NCHW_N = 0, NCHW_C = 1, NCHW_H = 2, NCHW_W = 3 }; +enum NHWC_SHAPE { NHWC_N = 0, NHWC_H = 1, NHWC_W = 2, NHWC_C = 3 }; +enum HWCK_SHAPE { HWCK_H = 0, HWCK_W = 1, HWCK_C = 2, HWCK_K = 3 }; +enum KCHW_SHAPE { KCHW_K = 0, KCHW_C = 1, KCHW_H = 2, KCHW_W = 3 }; +enum CHW_SHAPE { CHW_C = 0, CHW_H = 1, CHW_W = 2 }; +enum HWC_SHAPE { HWC_H = 0, HWC_W = 1, HWC_C = 2 }; + +static constexpr int TENSOR_MAX_REFCOUNT = 999; + +static const char *DELIM_COLON = ":"; +static const char *DELIM_COMMA = ","; +static const char *DELIM_SLASH = "/"; +static const char *DELIM_DOUBLE_BACKSLASH = "\\"; + +// quantization relative +static const char QUANTIZED_UINT8[] = "QUANTIZED_UINT8"; +static const char QUANTIZED_INT8[] = "QUANTIZED_INT8"; +static const char QUANTIZED_INT16[] = "QUANTIZED_INT16"; +static const char QUANTIZED_UINT16[] = "QUANTIZED_UINT16"; +static const char QUANTIZED_FLOAT16[] = "FLOAT16"; +static const char QUANTIZED_FLOAT32[] = "FLOAT32"; +static const char QUANTIZATION_TYPE_DYNAMIC[] = "DYNAMIC"; +static const char QUANTIZATION_TYPE_STATIC[] = "STATIC"; +static const char CALIB_NORM[] = "NORM"; + +// dims +static const int32_t DIM_DEFAULT_SIZE = 4; + +static const Format DEFAULT_FORMAT = Format_NCHW; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_COMMON_H_ diff --git a/predict/common/file_utils.cc b/predict/common/file_utils.cc new file mode 100644 index 00000000000..94adf0f7ac4 --- /dev/null +++ b/predict/common/file_utils.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/file_utils.h" +#include + +namespace mindspore { +namespace predict { +char *ReadFile(const char *file, size_t *size) { + if (file == nullptr) { + MS_LOGE("file is nullptr"); + return nullptr; + } + MS_ASSERT(size != nullptr); + std::ifstream ifs(RealPath(file)); + if (!ifs.good()) { + MS_LOGE("file: %s is not exist", file); + return nullptr; + } + + if (!ifs.is_open()) { + MS_LOGE("file: %s open failed", file); + return nullptr; + } + + ifs.seekg(0, std::ios::end); + *size = ifs.tellg(); + std::unique_ptr buf(new (std::nothrow) char[*size]); + if (buf == nullptr) { + MS_LOGE("malloc buf failed, file:%s", file); + ifs.close(); + return nullptr; + } + + ifs.seekg(0, std::ios::beg); + ifs.read(buf.get(), *size); + ifs.close(); + + return buf.release(); +} + +std::string RealPath(const char *path) { + if (path == nullptr) { + MS_LOGE("path is nullptr"); + return ""; + } + if ((strlen(path)) >= PATH_MAX) { + MS_LOGE("path is too long"); + return ""; + } + + std::shared_ptr resolvedPath(new (std::nothrow) char[PATH_MAX]{0}); + if (resolvedPath == nullptr) { + MS_LOGE("new resolvedPath failed"); + return ""; + } + + auto ret = realpath(path, resolvedPath.get()); + if (ret == nullptr) { + MS_LOGE("realpath failed"); + return ""; + } + return resolvedPath.get(); +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/file_utils.h b/predict/common/file_utils.h new file mode 100644 index 00000000000..e67c1cf9f11 --- /dev/null +++ b/predict/common/file_utils.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_FILE_UTILS_H_ +#define PREDICT_COMMON_FILE_UTILS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "common/mslog.h" +#include "include/tensor.h" + +namespace mindspore { +namespace predict { +char *ReadFile(const char *file, size_t *size); + +std::string RealPath(const char *path); +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_FILE_UTILS_H_ diff --git a/predict/common/flag_parser.cc b/predict/common/flag_parser.cc new file mode 100644 index 00000000000..37482dc4095 --- /dev/null +++ b/predict/common/flag_parser.cc @@ -0,0 +1,179 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/flag_parser.h" + +namespace mindspore { +namespace predict { +// parse flags read from command line +Option FlagParser::ParseFlags(int argc, const char *const *argv, bool supportUnknown, + bool supportDuplicate) { + MS_ASSERT(argv != nullptr); + const int FLAG_PREFIX_LEN = 2; + // Get binary name + binName = GetFileName(argv[0]); + + std::multimap> keyValues; + for (int i = 1; i < argc; i++) { + std::string tmp = argv[i]; + Trim(&tmp); + const std::string flagItem(tmp); + + if (flagItem == "--") { + break; + } + + if (flagItem.find("--") == std::string::npos) { + continue; + } + + std::string key; + Option value = Option(None()); + + size_t pos = flagItem.find_first_of("="); + if (pos == std::string::npos && flagItem.find("--no-") != std::string::npos) { + key = flagItem.substr(FLAG_PREFIX_LEN); + } else if (pos == std::string::npos) { + key = flagItem.substr(FLAG_PREFIX_LEN); + } else { + key = flagItem.substr(FLAG_PREFIX_LEN, pos - FLAG_PREFIX_LEN); + value = Option(flagItem.substr(pos + 1)); + } + + keyValues.insert(std::pair>(key, value)); + } + + Option ret = Option(InnerParseFlags(&keyValues)); + if (ret.IsSome()) { + return Option(ret.Get()); + } + + return Option(None()); +} + +bool FlagParser::GetRealFlagName(const std::string &oriFlagName, std::string *flagName) { + MS_ASSERT(flagName != nullptr); + const int BOOL_TYPE_FLAG_PREFIX_LEN = 3; + bool opaque = false; + if (StartsWithPrefix(oriFlagName, "no-")) { + *flagName = oriFlagName.substr(BOOL_TYPE_FLAG_PREFIX_LEN); + opaque = true; + } else { + *flagName = oriFlagName; + } + return opaque; +} + +// Inner parse function +Option FlagParser::InnerParseFlags(std::multimap> *keyValues) { + MS_ASSERT(keyValues != nullptr); + for (auto it = keyValues->begin(); it != keyValues->end(); ++it) { + std::string flagName; + bool opaque = GetRealFlagName((*it).first, &flagName); + Option flagValue = (*it).second; + + auto item = flags.find(flagName); + if (item == flags.end()) { + return Option(std::string(flagName + " is not a valid flag")); + } + FlagInfo *flag = &(item->second); + if (flag == nullptr) { + return Option("Failed: flag is nullptr"); + } + if (flag->isParsed) { + return Option("Failed: already parsed flag: " + flagName); + } + std::string tmpValue; + if (!flag->isBoolean) { + if (opaque) { + return Option(flagName + " is not a boolean type"); + } + if (flagValue.IsNone()) { + return Option("No value provided for non-boolean type: " + flagName); + } + tmpValue = flagValue.Get(); + } else { + if (flagValue.IsNone() || flagValue.Get().empty()) { + tmpValue = !opaque ? "true" : "false"; + } else if (!opaque) { + tmpValue = flagValue.Get(); + } else { + return Option(std::string("Boolean flag can not have non-empty value")); + } + } + // begin to parse value + Option ret = flag->parse(this, tmpValue); + if (ret.IsNone()) { + return Option("Failed to parse value for: " + flag->flagName); + } + flag->isParsed = true; + } + + // to check flags not given in command line but added as in constructor + for (auto &flag : flags) { + if (flag.second.isRequired && !flag.second.isParsed) { + return Option("Error, value of '" + flag.first + "' not provided"); + } + } + + return Option(None()); +} + +void Replaceall(std::string *str, const std::string &oldValue, const std::string &newValue) { + if (str == nullptr) { + MS_LOGE("Input str is nullptr"); + return; + } + while (true) { + std::string::size_type pos(0); + if ((pos = str->find(oldValue)) != std::string::npos) { + str->replace(pos, oldValue.length(), newValue); + } else { + break; + } + } +} + +std::string FlagParser::Usage(const Option &usgMsg) const { + // first line, brief of the usage + std::string usageString = usgMsg.IsSome() ? usgMsg.Get() + "\n" : ""; + // usage of bin name + usageString += usageMsg.IsNone() ? "usage: " + binName + " [options]\n" : usageMsg.Get() + "\n"; + // help line of help message, usageLine:message of parametors + std::string helpLine = ""; + std::string usageLine = ""; + uint32_t i = 0; + for (auto flag = flags.begin(); flag != flags.end(); flag++) { + std::string flagName = flag->second.flagName; + std::string helpInfo = flag->second.helpInfo; + // parameter line + std::string thisLine = flag->second.isBoolean ? " --[no-]" + flagName : " --" + flagName + "=VALUE"; + if (++i < flags.size()) { + // add paramter help message of each line + thisLine += " " + helpInfo; + Replaceall(&helpInfo, "\n\r", "\n"); + usageLine += thisLine + "\n"; + } else { + // brief help message + helpLine = thisLine + " " + helpInfo + "\n"; + } + } + // total usage is brief of usage+ brief of bin + help message + brief of + // paramters + return usageString + helpLine + usageLine; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/flag_parser.h b/predict/common/flag_parser.h new file mode 100644 index 00000000000..f01b8df71e1 --- /dev/null +++ b/predict/common/flag_parser.h @@ -0,0 +1,291 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_FLAG_PARSER_H_ +#define PREDICT_COMMON_FLAG_PARSER_H_ + +#include +#include +#include +#include + +#include "common/utils.h" +#include "common/option.h" + +namespace mindspore { +namespace predict { +struct FlagInfo; + +struct Nothing {}; + +class FlagParser { + public: + FlagParser() { AddFlag(&FlagParser::help, "help", "print usage message", false); } + + virtual ~FlagParser() = default; + + // only support read flags from command line + virtual Option ParseFlags(int argc, const char *const *argv, bool supportUnknown = false, + bool supportDuplicate = false); + std::string Usage(const Option &usgMsg = Option(None())) const; + + template + void AddFlag(T1 *t1, const std::string &flagName, const std::string &helpInfo, const T2 &t2); + + template + void AddFlag(T1 Flags::*t1, const std::string &flagName, const std::string &helpInfo, const T2 &t2); + + template + void AddFlag(T Flags::*t, const std::string &flagName, const std::string &helpInfo); + + // Option-type fields + template + void AddFlag(Option Flags::*t, const std::string &flagName, const std::string &helpInfo); + bool help; + + protected: + std::string binName; + Option usageMsg; + + private: + struct FlagInfo { + std::string flagName; + bool isRequired; + bool isBoolean; + std::string helpInfo; + bool isParsed; + std::function(FlagParser *, const std::string &)> parse; + }; + + inline void AddFlag(const FlagInfo &flag); + + // construct a temporary flag + template + void ConstructFlag(Option Flags::*t, const std::string &flagName, const std::string &helpInfo, FlagInfo *flag); + + // construct a temporary flag + template + void ConstructFlag(T1 Flags::*t1, const std::string &flagName, const std::string &helpInfo, FlagInfo *flag); + + Option InnerParseFlags(std::multimap> *values); + + bool GetRealFlagName(const std::string &oriFlagName, std::string *flagName); + + std::map flags; +}; + +// convert to std::string +template +Option ConvertToString(T Flags::*t, const FlagParser &baseFlag) { + const Flags *flag = dynamic_cast(&baseFlag); + if (flag != nullptr) { + return std::to_string(flag->*t); + } + + return Option(None()); +} + +// construct for a Option-type flag +template +void FlagParser::ConstructFlag(Option Flags::*t1, const std::string &flagName, const std::string &helpInfo, + FlagInfo *flag) { + if (flag == nullptr) { + MS_LOGE("FlagInfo is nullptr"); + return; + } + flag->flagName = flagName; + flag->helpInfo = helpInfo; + flag->isBoolean = typeid(T) == typeid(bool); + flag->isParsed = false; +} + +// construct a temporary flag +template +void FlagParser::ConstructFlag(T Flags::*t1, const std::string &flagName, const std::string &helpInfo, FlagInfo *flag) { + if (flag == nullptr) { + MS_LOGE("FlagInfo is nullptr"); + return; + } + if (t1 == nullptr) { + MS_LOGE("t1 is nullptr"); + return; + } + flag->flagName = flagName; + flag->helpInfo = helpInfo; + flag->isBoolean = typeid(T) == typeid(bool); + flag->isParsed = false; +} + +inline void FlagParser::AddFlag(const FlagInfo &flagItem) { flags[flagItem.flagName] = flagItem; } + +template +void FlagParser::AddFlag(T Flags::*t, const std::string &flagName, const std::string &helpInfo) { + if (t == nullptr) { + MS_LOGE("t1 is nullptr"); + return; + } + + Flags *flag = dynamic_cast(this); + if (flag == nullptr) { + MS_LOGI("dynamic_cast failed"); + return; + } + + FlagInfo flagItem; + + // flagItem is as a output parameter + ConstructFlag(t, flagName, helpInfo, &flagItem); + flagItem.parse = [t](FlagParser *base, const std::string &value) -> Option { + Flags *flag = dynamic_cast(base); + if (base != nullptr) { + Option ret = Option(GenericParseValue(value)); + if (ret.IsNone()) { + return Option(None()); + } else { + flag->*t = ret.Get(); + } + } + + return Option(Nothing()); + }; + + flagItem.isRequired = true; + flagItem.helpInfo += + !helpInfo.empty() && helpInfo.find_last_of("\n\r") != helpInfo.size() - 1 ? " (default: " : "(default: "; + flagItem.helpInfo += ")"; + + // add this flag to a std::map + AddFlag(flagItem); +} + +template +void FlagParser::AddFlag(T1 *t1, const std::string &flagName, const std::string &helpInfo, const T2 &t2) { + if (t1 == nullptr) { + MS_LOGE("t1 is nullptr"); + return; + } + + FlagInfo flagItem; + + // flagItem is as a output parameter + ConstructFlag(t1, flagName, helpInfo, flagItem); + flagItem.parse = [t1](FlagParser *base, const std::string &value) -> Option { + if (base != nullptr) { + Option ret = Option(GenericParseValue(value)); + if (ret.IsNone()) { + return Option(None()); + } else { + *t1 = ret.Get(); + } + } + + return Option(Nothing()); + }; + + flagItem.isRequired = false; + *t1 = t2; + + flagItem.helpInfo += + !helpInfo.empty() && helpInfo.find_last_of("\n\r") != helpInfo.size() - 1 ? " (default: " : "(default: "; + flagItem.helpInfo += ToString(t2).Get(); + flagItem.helpInfo += ")"; + + // add this flag to a std::map + AddFlag(flagItem); +} + +template +void FlagParser::AddFlag(T1 Flags::*t1, const std::string &flagName, const std::string &helpInfo, const T2 &t2) { + if (t1 == nullptr) { + MS_LOGE("t1 is nullptr"); + return; + } + + Flags *flag = dynamic_cast(this); + if (flag == nullptr) { + MS_LOGI("dynamic_cast failed"); + return; + } + + FlagInfo flagItem; + + // flagItem is as a output parameter + ConstructFlag(t1, flagName, helpInfo, &flagItem); + flagItem.parse = [t1](FlagParser *base, const std::string &value) -> Option { + Flags *flag = dynamic_cast(base); + if (base != nullptr) { + Option ret = Option(GenericParseValue(value)); + if (ret.IsNone()) { + return Option(None()); + } else { + flag->*t1 = ret.Get(); + } + } + + return Option(Nothing()); + }; + + flagItem.isRequired = false; + flag->*t1 = t2; + + flagItem.helpInfo += + !helpInfo.empty() && helpInfo.find_last_of("\n\r") != helpInfo.size() - 1 ? " (default: " : "(default: "; + flagItem.helpInfo += ToString(t2).Get(); + flagItem.helpInfo += ")"; + + // add this flag to a std::map + AddFlag(flagItem); +} + +// option-type add flag +template +void FlagParser::AddFlag(Option Flags::*t, const std::string &flagName, const std::string &helpInfo) { + if (t == nullptr) { + MS_LOGE("t is nullptr"); + return; + } + + Flags *flag = dynamic_cast(this); + if (flag == nullptr) { + MS_LOGE("dynamic_cast failed"); + return; + } + + FlagInfo flagItem; + // flagItem is as a output parameter + ConstructFlag(t, flagName, helpInfo, &flagItem); + flagItem.isRequired = false; + flagItem.parse = [t](FlagParser *base, const std::string &value) -> Option { + Flags *flag = dynamic_cast(base); + if (base != nullptr) { + Option ret = Option(GenericParseValue(value)); + if (ret.IsNone()) { + return Option(None()); + } else { + flag->*t = Option(Some(ret.Get())); + } + } + + return Option(Nothing()); + }; + + // add this flag to a std::map + AddFlag(flagItem); +} +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_FLAG_PARSER_H_ diff --git a/predict/common/func_utils.cc b/predict/common/func_utils.cc new file mode 100644 index 00000000000..d2aeb8d9418 --- /dev/null +++ b/predict/common/func_utils.cc @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/func_utils.h" + +namespace mindspore { +namespace predict { +#if MS_USE_ARM +_Unwind_Reason_Code PrintTraceArm(_Unwind_Context *ctx, void *d) { + MS_ASSERT(ctx != nullptr); + MS_ASSERT(d != nullptr); + Dl_info info; + int *depth = static_cast(d); + auto ipAddr = static_cast(_Unwind_GetIP(ctx)); + if (dladdr(reinterpret_cast(ipAddr), &info)) { + const char *symbol = ""; + const char *dlfile = ""; + if (info.dli_sname) { + symbol = info.dli_sname; + } + if (info.dli_fname) { + dlfile = info.dli_fname; + } + MS_PRINT_ERROR("#%d: (%08lx) %s %s ", *depth, ipAddr, dlfile, symbol); + } + + (*depth)++; + return _URC_NO_REASON; +} +#endif + +void CoreDumpTraceFunc(int iSignum) { + MS_PRINT_ERROR("----- start get backtrace info -----"); +#if MS_USE_ARM + int depth = 0; + _Unwind_Backtrace(&PrintTraceArm, &depth); +#else + const auto maxDeep = 32; + const auto maxStringLen = 100; + void *apBuffer[maxStringLen]; + char **ppStrings; + + auto iStackDepth = backtrace(apBuffer, maxDeep); + if (0 > iStackDepth) { + KillProcess("Get backtrace depth failed"); + return; + } + MS_PRINT_ERROR("Current stack depth is %d", iStackDepth); + ppStrings = backtrace_symbols(apBuffer, iStackDepth); + if (nullptr == ppStrings) { + KillProcess("Get backtrace_symbols failed"); + return; + } + + for (int iLoop = 0; iLoop < iStackDepth; iLoop++) { + MS_PRINT_ERROR("%s \n", ppStrings[iLoop]); + } +#endif + MS_PRINT_ERROR("----- finish get backtrace info -----"); + KillProcess("Exit after core dump"); + return; // try exit 1 +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/func_utils.h b/predict/common/func_utils.h new file mode 100644 index 00000000000..da0389a584f --- /dev/null +++ b/predict/common/func_utils.h @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_FUNC_UTILS_H_ +#define PREDICT_COMMON_FUNC_UTILS_H_ + +#if MS_USE_ARM +#include +#include +#else +#include +#endif +#include "include/errorcode.h" +#include "common/mslog.h" + +namespace mindspore { +namespace predict { +void CoreDumpTraceFunc(int iSignum); +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_FUNC_UTILS_H_ diff --git a/predict/common/graph_util.cc b/predict/common/graph_util.cc new file mode 100644 index 00000000000..6394731bc6d --- /dev/null +++ b/predict/common/graph_util.cc @@ -0,0 +1,167 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/graph_util.h" +#include +#include +#include "common/mslog.h" +#include "include/errorcode.h" + +namespace mindspore { +namespace predict { +OpGraph *OpGraph::Build(const SubGraphDef &subGraphDef) { + auto graph = std::unique_ptr(new OpGraph()); + if (graph == nullptr) { + MS_LOGE("malloc opgraph failed"); + return nullptr; + } + + auto nodeDefs = subGraphDef.nodes(); + if (nodeDefs == nullptr) { + MS_LOGE("nodeDefs from subGraphDef is nullptr"); + return nullptr; + } + + uint32_t opCount = nodeDefs->size(); + for (uint32_t i = 0; i < opCount; i++) { + auto nodeDef = nodeDefs->GetAs(i); + MS_ASSERT(nodeDef != nullptr); + auto ret = graph->AddEdge(*nodeDef, *nodeDefs); + if (ret != RET_OK) { + MS_LOGE("%s add edge failed. ret:%d", nodeDef->opDef()->name()->c_str(), ret); + return nullptr; + } + } + + return graph.release(); +} + +int OpGraph::AddEdge(const NodeDef &srcNodeDef, const flatbuffers::Vector> &nodeDefs) { + MS_ASSERT(srcNodeDef.opDef() != nullptr); + MS_ASSERT(srcNodeDef.opDef()->name() != nullptr); + NODE_ID srcId = std::string(srcNodeDef.opDef()->name()->c_str()); + uint32_t opCount = nodeDefs.size(); + + MS_ASSERT(srcNodeDef.opDef()->outputIndex() != nullptr); + for (auto index : *(srcNodeDef.opDef()->outputIndex())) { + for (uint32_t i = 0; i < opCount; i++) { + auto dstNodeDef = nodeDefs.GetAs(i); + bool find = false; + MS_ASSERT(dstNodeDef != nullptr); + MS_ASSERT(dstNodeDef->opDef() != nullptr); + auto inputIndex = dstNodeDef->opDef()->inputIndex(); + MS_ASSERT(inputIndex != nullptr); + if (std::any_of(inputIndex->begin(), inputIndex->end(), [&index](int i) { return i == index; })) { + find = true; + } + + if (!find) { + continue; + } + MS_ASSERT(dstNodeDef->opDef()->name() != nullptr); + NODE_ID dstId = std::string(dstNodeDef->opDef()->name()->c_str()); + auto ret = AddEdge(srcId, dstId); + if (ret != RET_OK) { + return ret; + } + } + } + + return RET_OK; +} + +int OpGraph::AddEdge(const NODE_ID &srcId, const NODE_ID &dstId) { + auto srcNode = AddNode(srcId); + if (srcNode == nullptr) { + MS_LOGE("add srcNode failed"); + return RET_ERROR; + } + srcNode->AddOutEdge(dstId); + auto dstNode = AddNode(dstId); + if (dstNode == nullptr) { + MS_LOGE("add dstNode failed"); + return RET_ERROR; + } + dstNode->AddInEdge(srcId); + return RET_OK; +} + +OpNode *OpGraph::GetNode(const NODE_ID &nodeId) { + auto node = nodes.find(nodeId); + if (node == nodes.end()) { + return nullptr; + } + return node->second; +} + +OpNode *OpGraph::AddNode(const NODE_ID &nodeId) { + auto node = GetNode(nodeId); + if (node != nullptr) { + return node; + } + node = new (std::nothrow) OpNode(nodeId); + if (node == nullptr) { + MS_LOGE("new node failed"); + return nullptr; + } + nodes[nodeId] = node; + return node; +} + +std::unordered_set OpGraph::GetInputNode() { + std::unordered_set inputNodes; + for (const auto &iter : nodes) { + auto node = iter.second; + MS_ASSERT(node != nullptr); + if (node->GetAllInEdge().empty()) { + inputNodes.insert(node->ID()); + } + } + return inputNodes; +} + +std::unordered_set OpGraph::GetOutputNode() { + std::unordered_set outputNodes; + for (const auto &iter : nodes) { + auto node = iter.second; + MS_ASSERT(node != nullptr); + if (node->GetAllOutEdge().empty()) { + outputNodes.insert(node->ID()); + } + } + return outputNodes; +} + +OpGraph::~OpGraph() { + for (auto iter : nodes) { + if (iter.second != nullptr) { + delete iter.second; + } + } + nodes.clear(); +} + +NODE_ID OpNode::ID() { return id; } + +void OpNode::AddInEdge(const NODE_ID &nodeId) { inEdges.insert(nodeId); } + +void OpNode::AddOutEdge(const NODE_ID &nodeId) { outEdges.insert(nodeId); } + +std::unordered_set OpNode::GetAllInEdge() { return inEdges; } + +std::unordered_set OpNode::GetAllOutEdge() { return outEdges; } +} // namespace predict +} // namespace mindspore diff --git a/predict/common/graph_util.h b/predict/common/graph_util.h new file mode 100644 index 00000000000..9797edadf66 --- /dev/null +++ b/predict/common/graph_util.h @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_GRAPH_UTIL_H_ +#define PREDICT_COMMON_GRAPH_UTIL_H_ + +#include +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "schema/inner/ms_generated.h" + +namespace mindspore { +namespace predict { +using NODE_ID = std::string; + +class OpNode { + public: + explicit OpNode(NODE_ID nodeId) : id(std::move(nodeId)) {} + NODE_ID ID(); + void AddInEdge(const NODE_ID &nodeId); + void AddOutEdge(const NODE_ID &nodeId); + std::unordered_set GetAllInEdge(); + std::unordered_set GetAllOutEdge(); + + protected: + NODE_ID id; + std::unordered_set inEdges; + std::unordered_set outEdges; +}; + +class OpGraph { + public: + OpGraph() = default; + + ~OpGraph(); + + static OpGraph *Build(const SubGraphDef &subGraphDef); + + OpNode *GetNode(const NODE_ID &nodeId); + OpNode *AddNode(const NODE_ID &nodeId); + std::unordered_set GetInputNode(); + std::unordered_set GetOutputNode(); + + private: + int AddEdge(const NODE_ID &srcId, const NODE_ID &dstId); + int AddEdge(const NodeDef &srcNodeDef, const flatbuffers::Vector> &nodeDefs); + + protected: + std::unordered_map nodes; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_GRAPH_UTIL_H_ diff --git a/predict/common/module_registry.cc b/predict/common/module_registry.cc new file mode 100644 index 00000000000..da8992bb66f --- /dev/null +++ b/predict/common/module_registry.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/module_registry.h" + +namespace mindspore { +namespace predict { +ModuleRegistry *GetRegistryInstance() { + static ModuleRegistry registry; + return ®istry; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/module_registry.h b/predict/common/module_registry.h new file mode 100644 index 00000000000..9d7587e74a7 --- /dev/null +++ b/predict/common/module_registry.h @@ -0,0 +1,97 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_MODULE_REGISTRY_H_ +#define PREDICT_COMMON_MODULE_REGISTRY_H_ +#include +#include +#include +#include "common/mslog.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +class ModuleBase { + public: + virtual ~ModuleBase() = default; +}; + +template +class Module; + +class ModuleRegistry { + public: + ModuleRegistry() = default; + + virtual ~ModuleRegistry() = default; + + template + bool Register(const std::string &name, const T &t) { + modules[name] = &t; + return true; + } + + template + std::shared_ptr Create(const std::string &name) { + auto it = modules.find(name); + if (it == modules.end()) { + return nullptr; + } + auto *module = (Module *)it->second; + if (module == nullptr) { + return nullptr; + } else { + return module->Create(); + } + } + + template + T *GetInstance(const std::string &name) { + auto it = modules.find(name); + if (it == modules.end()) { + return nullptr; + } + auto *module = (Module *)it->second; + if (module == nullptr) { + return nullptr; + } else { + return module->GetInstance(); + } + } + + protected: + std::unordered_map modules; +}; + +ModuleRegistry *GetRegistryInstance() MSPREDICT_API; + +template +class ModuleRegistrar { + public: + ModuleRegistrar(const std::string &name, const T &module) { + auto registryInstance = GetRegistryInstance(); + if (registryInstance == nullptr) { + MS_LOGW("registryInstance is nullptr."); + } else { + registryInstance->Register(name, module); + } + } +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_MODULE_REGISTRY_H_ diff --git a/predict/common/mslog.cc b/predict/common/mslog.cc new file mode 100644 index 00000000000..a1b61bbc3d6 --- /dev/null +++ b/predict/common/mslog.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/mslog.h" +#include +#include +#include +#include +#include "include/errorcode.h" + +namespace mindspore { +namespace predict { +std::string GetEnv(const std::string &envvar) { + const char *value = std::getenv(envvar.c_str()); + if (value == nullptr) { + return std::string(); + } + return std::string(value); +} + +bool IsPrint(int level) { + auto envString = GetEnv("MSLOG"); + static int env = static_cast(std::strtol(!envString.empty() ? envString.c_str() : "3", nullptr, 0)); + if (env == INT_MIN || env == INT_MAX) { + env = WARN; + // enable the SP for binscope checking + std::string errorStr = "env exceeded the value that type int is able to represent"; + MS_LOGE("%s", errorStr.c_str()); + } + + return level >= env; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/mslog.h b/predict/common/mslog.h new file mode 100644 index 00000000000..a48d87f1fa7 --- /dev/null +++ b/predict/common/mslog.h @@ -0,0 +1,230 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_MSLOG_H_ +#define PREDICT_COMMON_MSLOG_H_ + +#include +#include +#include +#include +#include +#include + +#if defined(__ANDROID__) || defined(ANDROID) +#include +#endif +namespace mindspore { +namespace predict { +constexpr const char *TAG = "MS_PREDICT"; + +constexpr int DEBUG = 1; +constexpr int INFO = 2; +constexpr int WARN = 3; +constexpr int ERROR = 4; + +#define MSPREDICT_API __attribute__((visibility("default"))) + +bool MSPREDICT_API IsPrint(int level); + +#if !defined(__ANDROID__) && !defined(ANDROID) + +#if LOG_TO_FILE +#define MS_LOGD(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::DEBUG)) { \ + syslog(LOG_DEBUG, "%s|%d|%s[%d]|: " #fmt, mindspore::predict::TAG, \getpid(), __func__, __LINE__, ##args); \ + } \ + } +#define MS_LOGI(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::INFO)) { \ + syslog(LOG_INFO, "%s|%d|%s[%d]|: " #fmt, mindspore::predict::TAG, \getpid(), __func__, __LINE__, ##args); \ + } \ + } +#define MS_LOGW(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::WARN)) { \ + syslog(LOG_WARNING, "%s|%d|%s[%d]|: " #fmt, mindspore::predict::TAG, \getpid(), __func__, __LINE__, ##args); \ + } \ + } +#define MS_LOGE(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::ERROR)) { \ + syslog(LOG_ERR, "%s|%d|%s[%d]|: " #fmt, mindspore::predict::TAG, getpid(), __func__, __LINE__, ##args); \ + } \ + } +#else + +#define MS_LOGD(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::DEBUG)) { \ + printf("[DEBUG] %s|%d|%s|%s[%d]|: " #fmt "\r\n", mindspore::predict::TAG, getpid(), __FILE__, __func__, \ + __LINE__, ##args); \ + } \ + } +#define MS_LOGI(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::INFO)) { \ + printf("[INFO] %s|%d|%s|%s[%d]|: " #fmt "\r\n", mindspore::predict::TAG, getpid(), __FILE__, __func__, \ + __LINE__, ##args); \ + } \ + } +#define MS_LOGW(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::WARN)) { \ + printf("[WARN] %s|%d|%s|%s[%d]|: " #fmt "\r\n", mindspore::predict::TAG, getpid(), __FILE__, __func__, \ + __LINE__, ##args); \ + } \ + } +#define MS_LOGE(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::ERROR)) { \ + printf("[ERROR] %s|%d|%s|%s[%d]|: " #fmt "\r\n", mindspore::predict::TAG, getpid(), __FILE__, __func__, \ + __LINE__, ##args); \ + } \ + } +#endif + +#else + +#define MS_LOGD(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::DEBUG)) \ + __android_log_print(ANDROID_LOG_DEBUG, mindspore::predict::TAG, "|%d|%s[%d]|: " fmt, getpid(), __func__, \ + __LINE__, ##args); \ + } + +#define MS_LOGI(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::INFO)) \ + __android_log_print(ANDROID_LOG_INFO, mindspore::predict::TAG, "|%d|%s[%d]|: " fmt, getpid(), __func__, \ + __LINE__, ##args); \ + } + +#define MS_LOGW(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::WARN)) \ + __android_log_print(ANDROID_LOG_WARN, mindspore::predict::TAG, "|%d|%s[%d]|: " fmt, getpid(), __func__, \ + __LINE__, ##args); \ + } + +#define MS_LOGE(fmt, args...) \ + { \ + if (mindspore::predict::IsPrint(mindspore::predict::ERROR)) \ + __android_log_print(ANDROID_LOG_ERROR, mindspore::predict::TAG, "|%d|%s[%d]|: " fmt, getpid(), __func__, \ + __LINE__, ##args); \ + } + +#endif + +#define MS_LOG(severity) std::cout << std::endl +#define MS_DLOG(verboselevel) std::cout << std::endl +// Kill the process for safe exiting. +inline void KillProcess(const std::string &ret) { + MS_LOG(ERROR) << "mindspore Exit Tip:" << ret; + if (raise(SIGKILL) != 0) { + MS_LOGE("Send SIGKILL to kill process failed"); + } +} +} // namespace predict +} // namespace mindspore + +#define MS_ASSERT(expression) \ + do { \ + if (!(expression)) { \ + std::stringstream ss; \ + ss << "Assertion failed: " << #expression << ", file: " << __FILE__ << ", line: " << __LINE__; \ + mindspore::predict::KillProcess(ss.str()); \ + } \ + } while (0) + +#define MS_EXIT(ret) \ + do { \ + std::stringstream ss; \ + ss << (ret) << " ( file: " << __FILE__ << ", line: " << __LINE__ << " )."; \ + mindspore::predict::KillProcess(ss.str()); \ + } while (0) + +#define MS_PRINT_ERROR(fmt, args...) \ + printf(#fmt "\n", ##args); \ + MS_LOGE(fmt, ##args); + +#define MS_PRINT_INFO(fmt, args...) \ + printf(fmt "\n", ##args); \ + MS_LOGI(fmt, ##args); + +constexpr int LOG_CHECK_EVERY_FIRSTNUM = 10; +constexpr int LOG_CHECK_EVERY_NUM1 = 10; +constexpr int LOG_CHECK_EVERY_NUM2 = 100; +constexpr int LOG_CHECK_EVERY_NUM3 = 1000; +constexpr int LOG_CHECK_EVERY_NUM4 = 10000; + +#define LOG_CHECK_ID_CONCAT(word1, word2) word1##word2 + +#define LOG_CHECK_ID LOG_CHECK_ID_CONCAT(__FUNCTION__, __LINE__) + +#define LOG_CHECK_FIRST_N \ + [](uint32_t firstNum) { \ + static uint32_t LOG_CHECK_ID = 0; \ + ++LOG_CHECK_ID; \ + return (LOG_CHECK_ID <= firstNum); \ + } + +#define LOG_CHECK_EVERY_N1 \ + [](uint32_t firstNum, uint32_t num) { \ + static uint32_t LOG_CHECK_ID = 0; \ + ++LOG_CHECK_ID; \ + return ((LOG_CHECK_ID <= firstNum) || (LOG_CHECK_ID % num == 0)); \ + } + +#define LOG_CHECK_EVERY_N2 \ + [](uint32_t firstNum, uint32_t num1, uint32_t num2) { \ + static uint32_t LOG_CHECK_ID = 0; \ + ++LOG_CHECK_ID; \ + return ((LOG_CHECK_ID <= firstNum) || (LOG_CHECK_ID < num2 && LOG_CHECK_ID % num1 == 0) || \ + (LOG_CHECK_ID % num2 == 0)); \ + } + +#define LOG_CHECK_EVERY_N3 \ + [](uint32_t firstNum, uint32_t num1, uint32_t num2, uint32_t num3) { \ + static uint32_t LOG_CHECK_ID = 0; \ + ++LOG_CHECK_ID; \ + return ((LOG_CHECK_ID <= firstNum) || (LOG_CHECK_ID < num2 && LOG_CHECK_ID % num1 == 0) || \ + (LOG_CHECK_ID < num3 && LOG_CHECK_ID % num2 == 0) || (LOG_CHECK_ID % num3 == 0)); \ + } + +#define LOG_CHECK_EVERY_N4 \ + [](uint32_t firstNum, uint32_t num1, uint32_t num2, uint32_t num3, uint32_t num4) { \ + static uint32_t LOG_CHECK_ID = 0; \ + ++LOG_CHECK_ID; \ + return ((LOG_CHECK_ID <= firstNum) || (LOG_CHECK_ID < num2 && LOG_CHECK_ID % num1 == 0) || \ + (LOG_CHECK_ID < num3 && LOG_CHECK_ID % num2 == 0) || (LOG_CHECK_ID < num4 && LOG_CHECK_ID % num3 == 0) || \ + (LOG_CHECK_ID % num4 == 0)); \ + } + +#define LOG_CHECK_EVERY_N \ + []() { \ + static uint32_t LOG_CHECK_ID = 0; \ + ++LOG_CHECK_ID; \ + return ((LOG_CHECK_ID <= LOG_CHECK_EVERY_FIRSTNUM) || \ + (LOG_CHECK_ID < LOG_CHECK_EVERY_NUM2 && LOG_CHECK_ID % LOG_CHECK_EVERY_NUM1 == 0) || \ + (LOG_CHECK_ID < LOG_CHECK_EVERY_NUM3 && LOG_CHECK_ID % LOG_CHECK_EVERY_NUM2 == 0) || \ + (LOG_CHECK_ID < LOG_CHECK_EVERY_NUM4 && LOG_CHECK_ID % LOG_CHECK_EVERY_NUM3 == 0) || \ + (LOG_CHECK_ID % LOG_CHECK_EVERY_NUM4 == 0)); \ + } + +#endif // PREDICT_COMMON_MSLOG_H_ diff --git a/predict/common/op_utils.h b/predict/common/op_utils.h new file mode 100644 index 00000000000..35f01edce39 --- /dev/null +++ b/predict/common/op_utils.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_OP_UTILS_H_ +#define PREDICT_COMMON_OP_UTILS_H_ + +#include +#include +#include "schema/inner/ms_generated.h" + +namespace mindspore { +namespace predict { +inline OpT GetOpType(const OpDef &opDef) { return opDef.attr_type(); } + +inline OpT GetOpType(const NodeDef &nodeDef) { return GetOpType(*(nodeDef.opDef())); } + +inline std::string GetOpTypeName(const NodeDef &nodeDef) { return EnumNameOpT(GetOpType(nodeDef)); } + +inline std::string GetOpTypeName(const OpDef &opDef) { return EnumNameOpT(GetOpType(opDef)); } + +inline OpT GetOpType(const OpDefT &opDefT) { return opDefT.attr.type; } + +inline OpT GetOpType(const NodeDefT &nodeDefT) { return GetOpType(*(nodeDefT.opDef.get())); } + +inline std::string GetOpTypeName(const NodeDefT &nodeDefT) { return EnumNameOpT(GetOpType(nodeDefT)); } + +inline std::string GetOpTypeName(const OpDefT &opDefT) { return EnumNameOpT(GetOpType(opDefT)); } +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_OP_UTILS_H_ diff --git a/predict/common/option.h b/predict/common/option.h new file mode 100644 index 00000000000..ca72dde29b2 --- /dev/null +++ b/predict/common/option.h @@ -0,0 +1,119 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_OPTION_H_ +#define PREDICT_COMMON_OPTION_H_ + +#include +#include +#include "common/mslog.h" + +namespace mindspore { +namespace predict { +template +struct InnerSome { + explicit InnerSome(const T &t) : _t(std::move(t)) {} + + T _t; +}; + +template +InnerSome::type> Some(T &&t) { + return InnerSome::type>(std::forward(t)); +} + +struct None {}; + +template +class Option { + public: + Option() : state(NONE) {} + + explicit Option(const T &t) : data(t), state(SOME) {} + + explicit Option(T &&t) : data(std::move(t)), state(SOME) {} + + explicit Option(const InnerSome &some) : data(some._t), state(SOME) {} + + explicit Option(const None &none) : state(NONE) {} + + Option(const Option &that) : state(that.state) { + if (that.IsSome()) { + new (&data) T(that.data); + } + } + + virtual ~Option() = default; + + bool IsNone() const { return state == NONE; } + + bool IsSome() const { return state == SOME; } + + const T &Get() const & { + MS_ASSERT(IsSome()); + return data; + } + + T &Get() & { + MS_ASSERT(IsSome()); + return data; + } + + T &&Get() && { + MS_ASSERT(IsSome()); + return std::move(data); + } + + const T &&Get() const && { + MS_ASSERT(IsSome()); + return std::move(data); + } + + // oprerator override + Option &operator=(const Option &that) { + if (&that != this) { + if (IsSome()) { + data.~T(); + } + state = that.state; + if (that.IsSome()) { + new (&data) T(that.data); + } + } + + return *this; + } + + bool operator==(const Option &that) const { + return (IsNone() && that.IsNone()) || (IsSome() && that.IsSome() && data == that.data); + } + + bool operator!=(const Option &that) const { return !(*this == that); } + + bool operator==(const T &that) const { return IsSome() && data == that; } + + bool operator!=(const T &that) const { return !(*this == that); } + + private: + enum State { NONE = 0, SOME = 1 }; + + T data; + State state; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_OPTION_H_ diff --git a/predict/common/storage.cc b/predict/common/storage.cc new file mode 100644 index 00000000000..ade5861c746 --- /dev/null +++ b/predict/common/storage.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/storage.h" +#include "flatbuffers/flatbuffers.h" +#include "common/mslog.h" +#include "common/file_utils.h" + +namespace mindspore { +namespace predict { +int Storage::Save(const GraphDefT &graph, const std::string &outputPath) { + flatbuffers::FlatBufferBuilder builder(flatSize); + auto offset = GraphDef::Pack(builder, &graph); + builder.Finish(offset); + int size = builder.GetSize(); + auto content = builder.GetBufferPointer(); + if (content == nullptr) { + MS_LOGE("GetBufferPointer nullptr"); + return RET_ERROR; + } + std::string realPath = RealPath(outputPath.c_str()); + if (realPath.empty()) { + MS_LOGE("Output file path '%s' is not valid", outputPath.c_str()); + return RET_ERROR; + } + + std::ofstream output(realPath, std::ofstream::binary); + if (!output.is_open()) { + MS_LOGE("ofstream open failed"); + return RET_ERROR; + } + output.write((const char *)content, size); + output.close(); + return RET_OK; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/storage.h b/predict/common/storage.h new file mode 100644 index 00000000000..fc612ffb6b7 --- /dev/null +++ b/predict/common/storage.h @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_STORAGE_H_ +#define PREDICT_COMMON_STORAGE_H_ + +#include +#include +#include "include/errorcode.h" +#include "flatbuffers/flatbuffers.h" +#include "schema/inner/ms_generated.h" + +namespace mindspore { +namespace predict { +class Storage { + public: + int Save(const GraphDefT &graph, const std::string &outputPath); + const int flatSize = 1024; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_STORAGE_H_ diff --git a/predict/common/utils.cc b/predict/common/utils.cc new file mode 100644 index 00000000000..b186a4b8d84 --- /dev/null +++ b/predict/common/utils.cc @@ -0,0 +1,228 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/utils.h" + +namespace mindspore { +namespace predict { +uint64_t GetTimeUs() { + struct timespec ts = {0, 0}; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + // USECS_IN_SEC *NSECS_IN_USEC; + auto retval = static_cast((ts.tv_sec * USEC) + (ts.tv_nsec / MSEC)); + return retval; +} + +static const unsigned int FP32_BIT_SIZE = 32; +static const unsigned int FP32_EXPONENT_BIAS = 127; +static const unsigned int FP32_SIGNIFICAND = 23; + +static const unsigned int FP32_EXPONENT_MAX = 255; + +static const unsigned int FP16_BIT_SIZE = 16; +static const unsigned int FP16_EXPONENT_BIAS = 15; +static const unsigned int FP16_SIGNIFICAND = 10; + +static const int FP16_EXPONENT_MAX = 30; +static const int FP16_EXPONENT_MIN = -10; + +float ShortToFloat32(int16_t srcValue) { + uint16_t expHalf16 = srcValue & 0x7C00; + int exp1 = static_cast(expHalf16); + uint16_t mantissa16 = srcValue & 0x03FF; + int mantissa1 = static_cast(mantissa16); + int sign = static_cast(srcValue & 0x8000); + sign = sign << FP16_BIT_SIZE; + + // nan or inf + if (expHalf16 == 0x7C00) { + // nan + if (mantissa16 > 0) { + int res = (0x7FC00000 | sign); + int *iRes = &res; + MS_ASSERT(iRes != nullptr); + auto fres = static_cast(*iRes); + return fres; + } + // inf + int res = (0x7F800000 | sign); + int *iRes = &res; + MS_ASSERT(iRes != nullptr); + auto fres = static_cast(*iRes); + return fres; + } + if (expHalf16 != 0) { + exp1 += ((FP32_EXPONENT_BIAS - FP16_EXPONENT_BIAS) << FP16_SIGNIFICAND); // exponents converted to float32 bias + int res = (exp1 | mantissa1); + res = res << (FP32_SIGNIFICAND - FP16_SIGNIFICAND); + res = (res | sign); + int *iRes = &res; + + auto fres = static_cast(*iRes); + return fres; + } + + int xmm1 = exp1 > (1 << FP16_SIGNIFICAND) ? exp1 : (1 << FP16_SIGNIFICAND); + xmm1 = (xmm1 << (FP32_SIGNIFICAND - FP16_SIGNIFICAND)); + xmm1 += ((FP32_EXPONENT_BIAS - FP16_EXPONENT_BIAS - FP16_SIGNIFICAND) + << FP32_SIGNIFICAND); // add the bias difference to xmm1 + xmm1 = xmm1 | sign; // Combine with the sign mask + + auto res = static_cast(mantissa1); // Convert mantissa to float + res *= static_cast(xmm1); + + return res; +} + +int16_t Float32ToShort(float srcValue) { + auto srcValueBit = static_cast(srcValue); + int sign = srcValueBit >> (FP32_BIT_SIZE - 1); + int mantissa = srcValueBit & 0x007FFFFF; + // exponent + int exp = ((srcValueBit & 0x7F800000) >> FP32_SIGNIFICAND) + FP16_EXPONENT_BIAS - FP32_EXPONENT_BIAS; + int16_t res; + if (exp > 0 && exp < FP16_EXPONENT_MAX) { + // use rte rounding mode, round the significand, combine sign, exponent and significand into a short. + res = (sign << (FP16_BIT_SIZE - 1)) | (exp << FP16_SIGNIFICAND) | + ((mantissa + 0x00001000) >> (FP32_SIGNIFICAND - FP16_SIGNIFICAND)); + } else if (srcValueBit == 0) { + res = 0; + } else { + if (exp <= 0) { + if (exp < FP16_EXPONENT_MIN) { + // value is less than min half float point + res = 0; + } else { + // normalized single, magnitude is less than min normal half float point. + mantissa = (mantissa | 0x00800000) >> (1 - exp); + // round to nearest + if ((mantissa & 0x00001000) > 0) { + mantissa = mantissa + 0x00002000; + } + // combine sign & mantissa (exp is zero to get denormalized number) + res = (sign << FP16_EXPONENT_BIAS) | (mantissa >> (FP32_SIGNIFICAND - FP16_SIGNIFICAND)); + } + } else if (exp == (FP32_EXPONENT_MAX - FP32_EXPONENT_BIAS + FP16_EXPONENT_BIAS)) { + if (mantissa == 0) { + // input float is infinity, return infinity half + res = (sign << FP16_EXPONENT_BIAS) | 0x7C00; + } else { + // input float is NaN, return half NaN + res = (sign << FP16_EXPONENT_BIAS) | 0x7C00 | (mantissa >> (FP32_SIGNIFICAND - FP16_SIGNIFICAND)); + } + } else { + // exp > 0, normalized single, round to nearest + if ((mantissa & 0x00001000) > 0) { + mantissa = mantissa + 0x00002000; + if ((mantissa & 0x00800000) > 0) { + mantissa = 0; + exp = exp + 1; + } + } + if (exp > FP16_EXPONENT_MAX) { + // exponent overflow - return infinity half + res = (sign << FP16_EXPONENT_BIAS) | 0x7C00; + } else { + // combine sign, exp and mantissa into normalized half + res = (sign << FP16_EXPONENT_BIAS) | (exp << FP16_SIGNIFICAND) | + (mantissa >> (FP32_SIGNIFICAND - FP16_SIGNIFICAND)); + } + } + } + return res; +} +std::string Remove(const std::string &from, const std::string &subStr, Mode mode) { + std::string result = from; + if (mode == PREFIX) { + if (from.substr(0, subStr.length()) == subStr) { + result = from.substr(subStr.size()); + } + } else if (mode == SUFFIX) { + if (from.rfind(subStr) == from.size() - subStr.size()) { + result = from.substr(0, from.size() - subStr.size()); + } + } else { + size_t index; + while ((index = result.find(subStr)) != std::string::npos) { + result = result.erase(index, subStr.size()); + } + } + + return result; +} + +std::vector StrSplit(const std::string &str, const std::string &pattern) { + std::string::size_type pos; + std::vector result; + std::string tmpStr(str + pattern); + std::string::size_type size = tmpStr.size(); + + for (std::string::size_type i = 0; i < size; i++) { + pos = tmpStr.find(pattern, i); + if (pos < size) { + std::string s = tmpStr.substr(i, pos - i); + result.push_back(s); + i = pos + pattern.size() - 1; + } + } + return result; +} + +std::vector Tokenize(const std::string &src, const std::string &delimiters, + const Option &maxTokenNum) { + if (maxTokenNum.IsSome() && maxTokenNum.Get() == 0) { + return {}; + } + + std::vector tokens; + size_t offset = 0; + + while (true) { + size_t nonDelimiter = src.find_first_not_of(delimiters, offset); + if (nonDelimiter == std::string::npos) { + break; + } + size_t delimiter = src.find_first_of(delimiters, nonDelimiter); + if (delimiter == std::string::npos || (maxTokenNum.IsSome() && tokens.size() == maxTokenNum.Get() - 1)) { + tokens.push_back(src.substr(nonDelimiter)); + break; + } + + tokens.push_back(src.substr(nonDelimiter, delimiter - nonDelimiter)); + offset = delimiter; + } + return tokens; +} + +void ShortToFloat32(const int16_t *srcdata, float *dstdata, size_t elementSize) { + MS_ASSERT(srcdata != nullptr); + MS_ASSERT(dstdata != nullptr); + for (size_t i = 0; i < elementSize; i++) { + dstdata[i] = ShortToFloat32(srcdata[i]); + } +} + +void Float32ToShort(const float *srcdata, int16_t *dstdata, size_t elementSize) { + MS_ASSERT(srcdata != nullptr); + MS_ASSERT(dstdata != nullptr); + for (size_t i = 0; i < elementSize; i++) { + dstdata[i] = Float32ToShort(srcdata[i]); + } +} +} // namespace predict +} // namespace mindspore diff --git a/predict/common/utils.h b/predict/common/utils.h new file mode 100644 index 00000000000..e7d44fe9820 --- /dev/null +++ b/predict/common/utils.h @@ -0,0 +1,154 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_COMMON_UTILS_H_ +#define PREDICT_COMMON_UTILS_H_ + +#include +#include +#include +#include +#include +#include "common/mslog.h" +#include "common/option.h" +#include "include/errorcode.h" + +namespace mindspore { +namespace predict { +const int USEC = 1000000; +const int MSEC = 1000; + +uint64_t GetTimeUs(); + +int16_t Float32ToShort(float srcValue); + +float ShortToFloat32(int16_t srcValue); + +void ShortToFloat32(const int16_t *srcData, float *dstData, size_t elementSize); + +void Float32ToShort(const float *srcData, int16_t *dstData, size_t elementSize); + +template +bool IsContain(const std::vector &vec, T element) { + for (auto iter = vec.begin(); iter != vec.end(); iter++) { + if (*iter == element) { + return true; + } + } + return false; +} + +const char WHITESPACE[] = "\t\n\v\f\r "; +const char STR_TRUE[] = "true"; +const char STR_FALSE[] = "false"; + +template +Option ToString(T t) { + std::ostringstream out; + out << t; + if (!out.good()) { + return Option(None()); + } + + return Option(out.str()); +} + +template <> +inline Option ToString(bool value) { + return value ? Option(STR_TRUE) : Option(STR_FALSE); +} + +// get the file name from a given path +// for example: "/usr/bin", we will get "bin" +inline std::string GetFileName(const std::string &path) { + char delim = '/'; + + size_t i = path.rfind(delim, path.length()); + if (i != std::string::npos) { + return (path.substr(i + 1, path.length() - i)); + } + + return ""; +} + +// trim the white space character in a string +// see also: macro WHITESPACE defined above +inline void Trim(std::string *input) { + if (input == nullptr) { + return; + } + if (input->empty()) { + return; + } + + input->erase(0, input->find_first_not_of(WHITESPACE)); + input->erase(input->find_last_not_of(WHITESPACE) + 1); +} + +// to judge whether a string is starting with prefix +// for example: "hello world" is starting with "hello" +inline bool StartsWithPrefix(const std::string &source, const std::string &prefix) { + if (source.length() < prefix.length()) { + return false; + } + + return (source.compare(0, prefix.length(), prefix) == 0); +} + +// split string +std::vector StrSplit(const std::string &str, const std::string &pattern); + +// tokenize string +std::vector Tokenize(const std::string &src, const std::string &delimiters, + const Option &maxTokenNum = Option(None())); + +enum Mode { PREFIX, SUFFIX, ANY }; + +// remove redundant character +std::string Remove(const std::string &from, const std::string &subStr, Mode mode = ANY); + +template +inline Option GenericParseValue(const std::string &value) { + T ret; + std::istringstream input(value); + input >> ret; + + if (input && input.eof()) { + return Option(ret); + } + + return Option(None()); +} + +template <> +inline Option GenericParseValue(const std::string &value) { + return Option(value); +} + +template <> +inline Option GenericParseValue(const std::string &value) { + if (value == "true") { + return Option(true); + } else if (value == "false") { + return Option(false); + } + + return Option(None()); +} +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_COMMON_UTILS_H_ diff --git a/predict/include/context.h b/predict/include/context.h new file mode 100644 index 00000000000..cb30f1f8c29 --- /dev/null +++ b/predict/include/context.h @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_INCLUDE_CONTEXT_H_ +#define PREDICT_INCLUDE_CONTEXT_H_ + +#include +#include "dlpack/dlpack.h" +#include "include/tensor.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +///\brief Resource management definition of MindSpore predict. +class MSPREDICT_API Context { + public: + ///\brief Constructor of MindSpore predict context using default value for parameters. + /// + ///\return Instance of MindSpore predict context. + Context(); + + ///\brief Custum constructor of MindSpore predict context using input value for parameters. + /// + ///\param[in] threadNum The number of thread during the runtime. + ///\param[in] allocator The memory management during the runtime + ///\param[in] deviceCtx The device information during the runtime. + /// + ///\return Instance of MindSpore predict context. + Context(int threadNum, std::shared_ptr allocator, DLContext deviceCtx); + + ///\brief Destructor of MindSpore predict context. + virtual ~Context(); + + public: + DLContext deviceCtx; + int threadNum = 1; + std::shared_ptr allocator; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_INCLUDE_CONTEXT_H_ diff --git a/predict/include/errorcode.h b/predict/include/errorcode.h new file mode 100755 index 00000000000..5487673f162 --- /dev/null +++ b/predict/include/errorcode.h @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_INCLUDE_ERRORCODE_H_ +#define PREDICT_INCLUDE_ERRORCODE_H_ + +namespace mindspore { +namespace predict { +using STATUS = int; + +/* Success */ +constexpr int RET_OK = 0; /**< No error occurs. */ + +/* Common error code, range: [-1, -100]*/ +constexpr int RET_ERROR = -1; /**< Common error code. */ +constexpr int RET_NULL_PTR = -2; /**< NULL pointer returned.*/ +constexpr int RET_PARAM_INVALID = -3; /**< Invalid parameter.*/ +constexpr int RET_NO_CHANGE = -4; /**< No change. */ + +/* Executor error code, range: [-101,-200] */ +constexpr int RET_OUT_OF_TENSOR_RANGE = -101; /**< Failed to checking range. */ +constexpr int RET_INPUT_TENSOR_ERROR = -102; /**< Failed to checking input tensor. */ +constexpr int RET_REENTRANT_ERROR = -103; /**< Exist executor running. */ + +/* Graph error code, range: [-201,-300] */ +constexpr int RET_GRAPH_FILE_ERR = -201; /**< Failed to verify graph file. */ + +/* Node error code, range: [-301,-400] */ +constexpr int RET_NOT_FIND_OP = -301; /**< Failed to find OP. */ +constexpr int RET_INVALID_OP_NAME = -302; /**< Invalid OP name. */ +constexpr int RET_INVALID_OP_ATTR = -303; /**< Invalid OP attr. */ +constexpr int RET_OP_EXECUTE_FAILURE = -304; /**< Failed to execution OP. */ + +/* Tensor error code, range: [-401,-500] */ +constexpr int RET_FORMAT_ERR = -401; /**< Failed to checking tensor format. */ +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_INCLUDE_ERRORCODE_H_ diff --git a/predict/include/session.h b/predict/include/session.h new file mode 100644 index 00000000000..76fa2c4d6e2 --- /dev/null +++ b/predict/include/session.h @@ -0,0 +1,139 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_INCLUDE_SESSION_H_ +#define PREDICT_INCLUDE_SESSION_H_ + +#include +#include +#include +#include +#include +#include "include/context.h" +#include "include/tensor.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +using NODE_ID = std::string; + +///\brief Graph defined by MindSpore predict. +/// +///\note +/// The caller does not need to care about detailed implementation of this class, so just list the class name here. +class Graph; + +///\brief GraphExecution defined by MindSpore predict. +/// +///\note +/// The caller does not need to care about detailed implementation of this class, so just list the class name here. +class GraphExecution; + +///\brief MindSpore predict session. +/// +/// This class represents session of MindSpore predict. +/// +///\note +/// The caller needs to allocate and free memory of inputs and outputs. +/// New Session is not suggested, please use CreateSession function to create new session class. +class MSPREDICT_API Session { + public: + ///\brief Constructor of MindSpore predict session. + /// + ///\param[in] ctx The context of the session. + /// + ///\return Instance of MindSpore predict session. + explicit Session(const Context &ctx); + + ///\brief Destructor of MindSpore predict session. + ~Session(); + + ///\brief Init the session. + /// + ///\param[in] ctx The context of the session. + ///\param[in] size The size of the session. + ///\param[in] graphBuf The buffer of the graph, used for build session. + /// + ///\return Return RET_OK if the initialization is success, otherwhise return RET_ERROR. + int Init(const char *graphBuf, size_t size); + + ///\brief Get the input of session. + /// + ///\return Input node's input tensors if found, empty vector otherwise. + /// + ///\note + /// The caller needs to allocate and free memory of inputs. + std::vector GetInput(); + + ///\brief Run the session. + /// + ///\param[in] inputs The input of the session. + /// + ///\return Return RET_OK if run success, otherwhise return RET_ERROR. + ///\note + /// Currently input tensors' data format only support FORMAT_NCHW. + /// Currently input tensors' data type only support FLOAT. + int Run(const std::vector &inputs); + + ///\brief Get the output of session. + /// + ///\param[in] nodeName Given output node name. + /// + ///\return Output node's output tensors if found, empty vector otherwise. + /// + ///\note + /// The caller needs to free memory of outputs. + std::vector GetOutput(const std::string &nodeName); + + ///\brief Get the all output of session. + /// + ///\return Every output node's output tensors. + /// + ///\note + /// The caller needs to free memory of outputs. + std::map> GetAllOutput(); + + protected: + ///\brief Init the executor. + /// + ///\return Return RET_OK if the initialization is success, otherwhise return RET_ERROR. + int InitExecutor(); + + const Context &_ctx; + Graph *_graph = nullptr; + GraphExecution *_executor = nullptr; + bool reinitExecutor = true; +}; + +///\brief MindSpore predict neural network session create function +/// +/// This function used to create MindSpore predict neural network session, which will be used to run the neural network. +/// +///\param[in] sessionName The name of the session. +///\param[in] graphBuf The buffer of the graph, used for build session. +///\param[in] size The size of the session. +///\param[in] ctx The context of the session. +/// +///\return Instance of MindSpore predict session. +/// +///\note +/// The caller needs to allocate and free memory of graph buffer. +std::shared_ptr MSPREDICT_API CreateSession(const char *graphBuf, size_t size, const Context &ctx); +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_INCLUDE_SESSION_H_ diff --git a/predict/include/tensor.h b/predict/include/tensor.h new file mode 100644 index 00000000000..8a608b486d1 --- /dev/null +++ b/predict/include/tensor.h @@ -0,0 +1,259 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_INCLUDE_TENSOR_H_ +#define PREDICT_INCLUDE_TENSOR_H_ + +#include +#include +#include "dlpack/dlpack.h" +#include "schema/inner/ms_generated.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +///\brief Allocator definition of MindSpore predict. +class Allocator; + +///\brief Tensor definition of MindSpore predict. +class MSPREDICT_API Tensor { + public: + ///\brief Constructor of MindSpore predict tensor. + /// + ///\param[in] tensor Define the parameters of the tensor. + ///\param[in] copyData Malloc data for the tensor, and copy origin data from + /// input tensor. + /// + ///\return Instance of MindSpore predict tensor. + Tensor(const Tensor &tensor, bool copyData = false); + + ///\brief Constructor of MindSpore predict tensor. + /// + ///\param[in] dt Data Type of the tensor, see introduction to 'enum DataType' + /// for supported type. + ///\param[in] dims Dimension Values such as height and width, which defined + /// the shape of the tensor. + ///\param[in] format Tensor format, see introduction to 'enum Format' for + /// supported format. + ///\param[in] data Data of the tensor. + /// + ///\return Instance of MindSpore predict tensor. + /// + ///\note + /// Length of data should align with dt, format and dims, otherwise the + /// application might run into unexpected error, + /// such as segment fault. + /// For example, dt is DT_FLOAT, format is FORMAT_NCHW, dims is [1,3,300,300], + /// then minimum length of data should + /// be 1 * 3 * 300 * 300 * sizeof(float). + Tensor(DataType dt, const std::vector &dims, Format format, void *data); + + ///\brief Destructor of MindSpore predict tensor. + ~Tensor(); + + ///\brief Get MindSpore predict tensor. + /// + ///\param[in] Definition of the tensor. + /// + ///\return Address of MindSpore predict tensor. + static Tensor *CopyFromTensorDef(const TensorDef &tensordef); + + ///\brief Get dtype of MindSpore predict tensor. + /// + ///\return Dtype of MindSpore predict tensor. + DLDataType GetTensorDtype() const; + + ///\brief Get data of MindSpore predict tensor. + /// + ///\return Address of MindSpore predict tensor data. + void *GetData() const; + + ///\brief Set data of MindSpore predict tensor. + /// + ///\param[in] data Address for data of the MindSpore predict tensor instance. + /// + ///\note + /// Length of data should align with dt, format and dims, otherwise the + /// application might run into unexpected error, + /// such as segment fault. + /// For example, dt is DT_FLOAT, format is FORMAT_NCHW, dims is [1,3,300,300], + /// then minimum length of data should + /// be 1 * 3 * 300 * 300 * sizeof(float). + void SetData(void *data); + + ///\brief Get data type of MindSpore predict tensor. + /// + ///\return Data Type of the tensor. + DataType GetDataType() const; + + ///\brief Set data type of MindSpore predict tensor. + /// + ///\param[in] dt Data Type of the tensor, see introduction to 'enum DataType' + /// for supported type. + void SetDataType(DataType dt); + + ///\brief Get number of dimension of MindSpore predict tensor. + /// + ///\return Number of dimension of the MindSpore predict tensor. + int GetNDim() const; + + ///\brief Get dimension of MindSpore predict tensor. + /// + ///\return Dimension of the MindSpore predict tensor. + std::vector GetDims() const; + + ///\brief Set dimension of MindSpore predict tensor. + /// + ///\param[in] dims Vector that has values of dimension. + void SetDims(const std::vector &dims); + + ///\brief Get format of MindSpore predict tensor. + /// + ///\return Format of the MindSpore predict tensor. + Format GetFormat() const { return format; } + + ///\brief Set format of MindSpore predict tensor. + /// + ///\param[in] format Format of the tensor. + void SetFormat(Format format) { this->format = format; } + + ///\brief Get reference count of MindSpore predict tensor. + /// + ///\return Reference count of the MindSpore predict tensor. + int RefCount() { return refCount; } + + ///\brief Increase reference count of MindSpore predict tensor. + /// + ///\param[in] ref The increase of the reference count. + void AddRef(int ref) { refCount += ref; } + + ///\brief Decrease reference count of MindSpore predict tensor. + /// + ///\param[in] ref The decrease of the reference count. + void DefRef(int ref) { refCount -= ref; } + + ///\brief Get element size of MindSpore predict tensor. + /// + ///\return Element size of MindSpore predict tensor. + size_t GetElementSize() const; + + ///\brief Get data size of MindSpore predict tensor. + /// + ///\return Data size of MindSpore predict tensor. + size_t GetDataSize() const; + + ///\brief Get element size of MindSpore predict tensor in NC4HW4 format. + /// + ///\param[in] isNhwc Whether the current format is NHWC. + /// + ///\return Element size of MindSpore predict tensor in NC4HW4 format. + size_t GetNC4HW4ElementSize(bool isNhwc); + + ///\brief Get data size of MindSpore predict tensor in NC4HW4 format. + /// + ///\param[in] isNhwc Whether the current format is NHWC. + /// + ///\return Data size of MindSpore predict tensor in NC4HW4 format. + size_t GetNC4HW4DataSize(bool isNhwc); + + ///\brief Malloc data for the MindSpore predict tensor. + /// + ///\param[in] allocator The malloc source for data. + ///\param[in] refCount The reference count of the data. + /// + ///\return Return RET_OK if the data is successfully allocated, otherwhise return RET_ERROR. + int MallocData(std::shared_ptr allocator = nullptr, int refCount = 0); + + ///\brief Free the MindSpore predict tensor. + void FreeTensor(); + + ///\brief Free the data of MindSpore predict tensor. + void ForceFreeData(); + + ///\brief Free the data of MindSpore predict tensor. + void FreeData(); + + ///\brief Compare data size of MindSpore predict tensor in NC4HW4 format. + /// + ///\param[in] dst The compare tensor. + /// + ///\return The result of fuction. + bool CompareShape(const Tensor &dst); + + ///\brief Compare shape of MindSpore predict tensor with another shape. + /// + ///\param[in] other The compare shape information. + /// + ///\return The result of function. + bool CompareShape(const std::vector &other); + + ///\brief Get instance of MindSpore predict tensor. + /// + ///\return Instance of MindSpore predict dlTensor. + DLTensor *GetDLTensor() { return &dlTensor; } + + ///\brief Get height of MindSpore predict tensor. + /// + ///\return Height of MindSpore predict tensor. + int64_t Height() const; + + ///\brief Get width of MindSpore predict tensor. + /// + ///\return Width of MindSpore predict tensor. + int64_t Width() const; + + ///\brief Get channel of MindSpore predict tensor. + /// + ///\return Channel of MindSpore predict tensor. + int64_t Channel() const; + + ///\brief Get batch of MindSpore predict tensor. + /// + ///\return Batch of MindSpore predict tensor. + int64_t Batch() const; + + ///\brief Get stride of MindSpore predict tensor. + /// + ///\param[in] index the index of stride. + /// + ///\return Stride of MindSpore predict tensor. + int64_t Stride(int index) const; + + ///\brief Set stride of MindSpore predict tensor by input. + /// + ///\param[in] index Index of stride + ///\param[in] stride The stride to set + void SetStride(int index, int64_t stride); + + ///\brief Set stride of MindSpore predict tensor by dims. + void SetStride(); + void SetScale(bool isScale = true); + + private: + bool isScale = false; + int refCount = 0; + int isConst; + Format format; + DLTensor dlTensor; + std::shared_ptr allocator = nullptr; + std::vector scale; + std::vector zeroPoint; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_INCLUDE_TENSOR_H_ diff --git a/predict/module/CMakeLists.txt b/predict/module/CMakeLists.txt new file mode 100644 index 00000000000..9b978f1172a --- /dev/null +++ b/predict/module/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(tvm_kernel) diff --git a/predict/module/tvm_kernel/.gitignore b/predict/module/tvm_kernel/.gitignore new file mode 100644 index 00000000000..3b552d75dab --- /dev/null +++ b/predict/module/tvm_kernel/.gitignore @@ -0,0 +1,27 @@ +# Created by .ignore support plugin +# + +# filter python +*.pyc + +# filter build +*.so +*.o + +# filter coverage +coverage/ + +# filter report +*.xml + +# filter tvm +3rdparty/ + +# filter build +build/ +cmake-build-debug/ +.idea/ +TFLite_Detection_PostProcess_CI +app_run +output +tvm diff --git a/predict/module/tvm_kernel/.gitmodules b/predict/module/tvm_kernel/.gitmodules new file mode 100644 index 00000000000..8a987193a4e --- /dev/null +++ b/predict/module/tvm_kernel/.gitmodules @@ -0,0 +1,4 @@ +[submodule "3rdparty/incubator-tvm"] + path = 3rdparty/incubator-tvm + url = https://github.com/dmlc/tvm.git + branch = v0.5 diff --git a/predict/module/tvm_kernel/CMakeLists.txt b/predict/module/tvm_kernel/CMakeLists.txt new file mode 100755 index 00000000000..b99e257c3eb --- /dev/null +++ b/predict/module/tvm_kernel/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.12.1) +project(autotensor LANGUAGES CXX) +set (MINDSPORE "${PROJECT_SOURCE_DIR}/../../..") +set (TVM_KERNEL_LITE "${PROJECT_SOURCE_DIR}/lite") +set (THIRDPARTY "${MINDSPORE}/third_party") +set (TVM_CLEAN_SOURCE "${THIRDPARTY}/incubator-tvm") +set (TVM_BUILD_SOURCE "${PROJECT_SOURCE_DIR}/incubator-tvm") +set (BUILD_DIR "${PROJECT_SOURCE_DIR}") +set (TVM_KERNEL_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) +set (TVM_OUTPUT_DIR ${TVM_KERNEL_OUTPUT_DIR}/incubator-tvm) + +set (LLVM_CONFIG $ENV{LLVM_PATH}) +if (NOT LLVM_CONFIG) + message(FATAL_ERROR "please set LLVM_PATH in env") +endif() +set (CMAKE_BUILD_TYPE "Release") + +include(${TVM_BUILD_SOURCE}/cmake/util/Util.cmake) +include(${TVM_BUILD_SOURCE}/cmake/util/FindLLVM.cmake) +if(EXISTS ${TVM_BUILD_SOURCE}/cmake/config.cmake) + include(${TVM_BUILD_SOURCE}/cmake/config.cmake) +endif() +add_subdirectory(${TVM_KERNEL_LITE}) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + diff --git a/predict/module/tvm_kernel/lite/CMakeLists.txt b/predict/module/tvm_kernel/lite/CMakeLists.txt new file mode 100755 index 00000000000..50e2bf5c9b9 --- /dev/null +++ b/predict/module/tvm_kernel/lite/CMakeLists.txt @@ -0,0 +1,140 @@ +cmake_minimum_required(VERSION 3.12) +set(CMAKE_CXX_STANDARD 14) + +if(ENABLE_PREDICT_ARM64) + set(TARGS "arm64") +elseif(ENABLE_PREDICT_ARM32) + set(TARGS "arm32") +else() + set(TARGS "x86") +endif() +message("TARGET is set to ${TARGS}") + +set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_SKIP_RPATH TRUE) + +if(MSVC) + message("not support MSVC") +else(MSVC) + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-std=c++11" SUPPORT_CXX11) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + message("Build in Debug mode") + set(CMAKE_C_FLAGS "-O0 -g -Wall -Werror -fPIC [${CMAKE_C_FLAGS} -rdynamic") + set(CMAKE_CXX_FLAGS "-O0 -g -Wall -Werror -fPIC -std=c++11 ${CMAKE_CXX_FLAGS} -rdynamic") + else() + set(CMAKE_C_FLAGS "-D_FORTIFY_SOURCE=2 -O2 -fno-rtti -fvisibility=hidden -Wall -Werror -fPIC -fstack-protector-strong ${CMAKE_C_FLAGS}") + set(CMAKE_CXX_FLAGS "-D_FORTIFY_SOURCE=2 -O2 -fno-rtti -fvisibility=hidden -Wall -Werror -fPIC -fstack-protector-strong -std=c++11 ${CMAKE_CXX_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "-Wl,-z,relro,-z,now -Wl,-z,noexecstack") + endif () + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND + CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) + set(CMAKE_CXX_FLAGS "-Wall -Werror -faligned-new ${CMAKE_CXX_FLAGS}") + endif() + if (CODE_COVERAGE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -fprofile-arcs -ftest-coverage -O0") + endif() +endif(MSVC) + + +if("${TARGS}" STREQUAL "x86") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__x86_64__ -fno-strict-aliasing") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__x86_64__ -fno-strict-aliasing") +endif() + + +set(PRJ_SRC_DIR "${PROJECT_SOURCE_DIR}") +set(PRJ_KLIB_DIR "${PROJECT_SOURCE_DIR}") +set(PRJ_LITE_DIR "${PROJECT_SOURCE_DIR}/lite") + +# include directories +message("current PRJ DIR: ${PROJECT_SOURCE_DIR}") +message("current SUB_PRJ DIR: ${PRJ_SRC_DIR}") +message("current KLIB DIR: ${PRJ_KLIB_DIR}") +message("current PRJ_LITE_DIR: ${PRJ_LITE_DIR}") +message("CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}") +set(DMLC_CORE "${TVM_BUILD_SOURCE}/3rdparty/dmlc-core") +set(DLPACK "${TVM_BUILD_SOURCE}/3rdparty/dlpack") +set(PREDICT "${PRJ_SRC_DIR}/../../") +set(SECUREC "${PRJ_SRC_DIR}/../../../third_party/securec") +message("include dir: ${DLPACK}/include") +include_directories(${DLPACK}/include) +include_directories(${DMLC_CORE}/include) +include_directories(${TVM_BUILD_SOURCE}/include) +include_directories(${TVM_BUILD_SOURCE}/src/pass) +include_directories(${PRJ_LITE_DIR}) +include_directories(${PRJ_LITE_DIR}/include) +include_directories(${PRJ_LITE_DIR}/../../..) +include_directories(${PRJ_LITE_DIR}/../../../include) +include_directories(${PRJ_LITE_DIR}/../../../src/runtime) +include_directories(${PRJ_LITE_DIR}/../../../common) +include_directories(${SECUREC}) +message("SECUREC: " "${SECUREC}/build/src") +include_directories(${PREDICT}) +include_directories(${PREDICT}/src) +include_directories(${PRJ_SRC_DIR}/../../../third_party/flatbuffers/include) +include_directories(${PRJ_SRC_DIR}/../../../third_party) +# Source file lists +file(GLOB_RECURSE TVM_KERNEL_SRC + src/api/*.cc + src/tflite/TFLite_Detection_PostProcess.cc) + +set (TVM_RUNTIME_FLG $ENV{TVM_RUNTIME_ON}) +if ("${TVM_RUNTIME_FLG}" STREQUAL "true") + message("Using TVM runtime function") + file(GLOB TVM_RUNTIME_SRCS + ${TVM_ROOT}/apps/howto_deploy/tvm_runtime_pack.cc) +else() + message("Using LITE runtime function") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DLITE_RUNTIME_ON -DTVM_RUNTIME_HEADER_ONLY -DLITE_THREAD_POOL_SHARED") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLITE_RUNTIME_ON -DTVM_RUNTIME_HEADER_ONLY -DLITE_THREAD_POOL_SHARED") + file(GLOB_RECURSE TVM_RUNTIME_SRCS + ${PREDICT}/src/runtime/*.cc) +endif() + +if("${TARGS}" STREQUAL "arm32" OR "${TARGS}" STREQUAL "arm64") + set(CMAKE_SKIP_BUILD_RPATH TRUE) + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +endif() + +set(LIB_X86_PATH "${PRJ_KLIB_DIR}/build/lib_x86") +set(LIB_ARM64_PATH "${PRJ_KLIB_DIR}/build/lib_arm64") +set(LIB_ARM32_PATH "${PRJ_KLIB_DIR}/build/lib_arm32") +if("${TARGS}" STREQUAL "x86") + set(KLIBS_PATH "${LIB_X86_PATH}") +elseif("${TARGS}" STREQUAL "arm64") + set(KLIBS_PATH "${LIB_ARM64_PATH}") +elseif("${TARGS}" STREQUAL "arm32") + set(KLIBS_PATH "${LIB_ARM32_PATH}") +else() + message(ERROR " not suport ${TARGS}") +endif() + +file(GLOB_RECURSE KERNEL_LIBS "${KLIBS_PATH}/*.o") +message("KERNEL_PATH= ${KLIBS_PATH}") + +add_compile_options(-DTVM_CUDA_RUNTIM=0) +add_compile_options(-DTVM_METAL_RUNTIM=0) +add_compile_options(-DTVM_OPENCL_RUNTIM=0) + +link_directories(${KLIBS_PATH}) + +add_library(tvm_runtime_pack STATIC ${TVM_RUNTIME_SRCS}) +add_library(kernel_manager STATIC ${TVM_KERNEL_SRC}) +add_library(tvm_kernel_static STATIC ${TVM_KERNEL_SRC} ${KERNEL_LIBS}) +add_library(tvm_kernel SHARED ${TVM_KERNEL_SRC} ${KERNEL_LIBS}) +set_target_properties(tvm_kernel PROPERTIES LINK_FLAGS "-Wl,-z,relro,-z,now -Wl,-z,noexecstack") + +set(KERNEL_LD_LIB tvm_runtime_pack dl) + +if("${TARGS}" STREQUAL "x86") + set(KERNEL_LD_LIB ${KERNEL_LD_LIB} pthread) +else() + set(ANDROID_ALLOW_UNDEFINED_SYMBOLS TRUE) +endif() + +target_link_libraries(tvm_kernel ${KERNEL_LD_LIB} libsecurec.a) +target_link_libraries(tvm_kernel_static OBJECT tvm_runtime_pack libsecurec.a) + +add_dependencies(tvm_kernel securec) diff --git a/predict/module/tvm_kernel/lite/include/lite/api/km_api.h b/predict/module/tvm_kernel/lite/include/lite/api/km_api.h new file mode 100644 index 00000000000..7ccd4964cbc --- /dev/null +++ b/predict/module/tvm_kernel/lite/include/lite/api/km_api.h @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this ${file} except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PREDICT_MODULE_TVM_KERNEL_LITE_INCLUDE_LITE_API_KM_API_H_ +#define PREDICT_MODULE_TVM_KERNEL_LITE_INCLUDE_LITE_API_KM_API_H_ + +#include +#include +#include +#include +#include "schema/inner/ms_generated.h" +#include "schema/inner/op_generated.h" + +#define PUBLIC __attribute__((visibility("default"))) + +/*! + * \brief Call tvm kernel. + * \param fid tvm kernel id. + * \param tensors tvm kernel arguments. + * \return 0 if SUCCESS. + */ +PUBLIC int CallKernel(const std::string &fid, const std::vector &tensors); + +/*! + * \brief Get tvm kernel by id. + * \param fid tvm kernel id. + * \return std::function if SUCCESS else nullptr. + */ +PUBLIC std::function &)> GetKernel(const std::string &fid); + +/*! + * \brief Get tvm kernel by OpDef. + * \param opdef defined by predict schema. + * \param tensors. + * \param option. + * \return std::function if SUCCESS else nullptr. + */ +struct PUBLIC KernelOption { + int numThreads = 0; + std::string device; +}; + +PUBLIC std::function &)> GetKernel(const mindspore::predict::OpDef &opdef, + const std::vector &tensors, + const KernelOption &option); + +/*! + * \brief load TVM Kernel lib + * \param mode 0 indicate shared lib + * \param fname shared lib path when mode equals 0 + * \return 0 if SUCCESS + */ +PUBLIC void InitKernelManager(int mode, const std::string &fname); + +/* + * \brief config ThreadPool using mode + * \param mode: -1 using mid speed cpu first, 1 using higher speed cpu first + * \param nthreads: threads num to be used, can't exceed cpu num + * if mode==-1 bind mid cpu first + * if mode==1 bind higher cpu first + * if mode==0 no bind + * \param execute_self: cur thread do arithmetic or not + * execute_self: true cur thread do arithmetic work + * execute_self: false cur thread not do arithmetic work + */ +PUBLIC void ConfigThreadPool(int mode = -1, int nthreads = 2, bool execute_self = true); + +/* + * \brief provid simple api for mslite, mslite not care mode + */ +inline void CfgThreadPool(int nthread) { ConfigThreadPool(-1, nthread, true); } + +/* + * the Callback function to do cpu bind for master thread. + */ +PUBLIC void DoMasterThreadBind(bool bindflg); + +PUBLIC void DoAllThreadBind(bool ifBind); + +#undef PUBLIC + +#endif // PREDICT_MODULE_TVM_KERNEL_LITE_INCLUDE_LITE_API_KM_API_H_ diff --git a/predict/module/tvm_kernel/lite/python/__init__.py b/predict/module/tvm_kernel/lite/python/__init__.py new file mode 100644 index 00000000000..5a51943fbe5 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Neural network operators""" +# from . import arm_cpu +# from . import at_ops diff --git a/predict/module/tvm_kernel/lite/python/arm_cpu/__init__.py b/predict/module/tvm_kernel/lite/python/arm_cpu/__init__.py new file mode 100644 index 00000000000..dce9d5e96c7 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/arm_cpu/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Schedule for ARM CPU""" + +from . import conv2d diff --git a/predict/module/tvm_kernel/lite/python/arm_cpu/conv2d.py b/predict/module/tvm_kernel/lite/python/arm_cpu/conv2d.py new file mode 100644 index 00000000000..ded792f689a --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/arm_cpu/conv2d.py @@ -0,0 +1,470 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Conv2D schedule for ARM CPU""" +from __future__ import absolute_import as _abs + +import functools + +import tvm +from tvm import autotvm +import tvm.contrib.nnpack + +from topi.generic import schedule_conv2d_nchw +from topi.util import traverse_inline, get_const_tuple +from topi.nn import pad, conv2d +from topi.nn.util import get_const_int, get_pad_tuple + + +@autotvm.register_topi_compute(conv2d, "arm_cpu", ["asm"]) +def conv2d_arm_cpu(cfg, data, kernel, strides, padding, dilation, out_dtype): + """TOPI compute callback for conv2d + + Parameters + ---------- + cfg: ConfigEntity + The config for this template + + data : tvm.Tensor + 4-D with shape [batch, in_channel, in_height, in_width] + + kernel : tvm.Tensor + 4-D with shape [num_filter, in_channel, filter_height, filter_width] or + pre-packed 5-D with shape [num_filter_chunk, in_channel, filter_height, + filter_width, num_filter_block] + + strides : list of two ints + [stride_height, stride_width] + + padding : list of two ints + [pad_height, pad_width] + + dilation : list of two ints + [dilation_height, dilation_width] + + out_dtype: str + The output type. This is used for mixed precision. + + Returns + ------- + output : tvm.Tensor + 4-D with shape [batch, out_channel, out_height, out_width] + """ + args = _gen_cfg(cfg, data, kernel, strides, padding, dilation, num_tile=2) + return _conv_spatial_pack_asm( + args, data, kernel, strides, padding, dilation, out_dtype + ) + + +@autotvm.register_topi_schedule(schedule_conv2d_nchw, "arm_cpu", ["asm"]) +def schedule_conv2d_nchw_arm_cpu(outs): + """TOPI schedule callback for conv2d + + Parameters + ---------- + outs: Array of Tensor + The computation graph description of conv2d + in the format of an array of tensors. + + Returns + ------- + s: Schedule + The computation schedule for conv2d. + """ + s = _conv_schedule_asm(outs) + return s + + +def _gen_cfg(cfg, data, kernel, strides, padding, dilation, num_tile): + """_gen_cfg""" + if len(kernel.shape) == 4: + co_, _, kh_, kw_ = get_const_tuple(kernel.shape) + else: # kernel tensor is pre packed + co_, _, kh_, kw_, vc_ = get_const_tuple(kernel.shape) + co_ = co_ * vc_ + + if isinstance(dilation, int): + dilation_h = dilation_w = dilation + else: + dilation_h, dilation_w = dilation + + n_, ci_, ih_, iw_ = get_const_tuple(data.shape) + + dilated_kernel_h = (kh_ - 1) * dilation_h + 1 + dilated_kernel_w = (kw_ - 1) * dilation_w + 1 + pad_top, pad_left, pad_bottom, pad_right = get_pad_tuple( + padding, (dilated_kernel_h, dilated_kernel_w) + ) + hstr, wstr = strides if isinstance(strides, (tuple, list)) else (strides, strides) + oh_ = (ih_ + pad_top + pad_bottom - dilated_kernel_h) // hstr + 1 + ow_ = (iw_ + pad_left + pad_right - dilated_kernel_w) // wstr + 1 + + n, co, oh, ow = cfg.axis(n_), cfg.axis(co_), cfg.axis(oh_), cfg.axis(ow_) + ci, kh, kw = cfg.reduce_axis(ci_), cfg.reduce_axis(kh_), cfg.reduce_axis(kw_) + + if num_tile == 2: # for arm cpu + candidate_vc = [] + for iv in range(3, co_): + if co_ % iv == 0: + candidate_vc.append([co_ // iv, iv]) + candidate_vc.append([1, co_]) + co, vc = cfg.define_split( + "tile_co", co, num_outputs=2, policy="candidate", candidate=candidate_vc + ) + oh, vh = cfg.define_split("tile_oh", oh, num_outputs=2) + ow, vw = cfg.define_split("tile_ow", ow, num_outputs=2) + elif num_tile == 3: # for mali gpu + co, _, vc = cfg.define_split("tile_co", co, num_outputs=3) + oh, _, vh = cfg.define_split("tile_oh", oh, num_outputs=3) + ow, _, vw = cfg.define_split("tile_ow", ow, num_outputs=3) + else: + raise RuntimeError("Invalid num_tile") + + cfg.define_reorder( + "reorder_0", + [n, co, oh, ow, ci, kh, kw, vh, vw, vc], + policy="candidate", + candidate=[[n, co, oh, ow, ci, kh, kw, vh, vw, vc],], + ) + + vc_ = cfg["tile_co"].size[-1] + vh_ = cfg["tile_oh"].size[-1] + vw_ = cfg["tile_ow"].size[-1] + is_var = False + return (is_var, vh_, vw_, vc_) + +def _conv_spatial_pack_asm(args, data, kernel, strides, padding, + dilation, out_dtype): + """_conv_spatial_pack_asm""" + is_var, vh_, vw_, vc_ = args + + # create workload according to raw arguments + out_dtype = out_dtype or data.dtype + n_, ci_, ih_, iw_ = data.shape if is_var else get_const_tuple(data.shape) + + if isinstance(dilation, int): + dilation_h = dilation_w = dilation + else: + dilation_h, dilation_w = dilation + + if len(kernel.shape) == 4: + pre_packed = False + co_, _, kh_, kw_ = kernel.shape if is_var else get_const_tuple(kernel.shape) + else: # kernel tensor is pre packed + pre_packed = True + co_, _, kh_, kw_, vc_ = kernel.shape if is_var else get_const_tuple(kernel.shape) + co_ = co_ * vc_ + + dilated_kernel_h = (kh_ - 1) * dilation_h + 1 + dilated_kernel_w = (kw_ - 1) * dilation_w + 1 + pad_top, pad_left, pad_bottom, pad_right = get_pad_tuple( + padding, (dilated_kernel_h, dilated_kernel_w) + ) + hstr, wstr = strides if isinstance(strides, (tuple, list)) else (strides, strides) + oh_ = (ih_ + pad_top + pad_bottom - dilated_kernel_h) // hstr + 1 + ow_ = (iw_ + pad_left + pad_right - dilated_kernel_w) // wstr + 1 + data_pad = pad(data, [0, 0, pad_top, pad_left], [0, 0, pad_bottom, pad_right]) + + oh_div = oh_ // vh_ + ow_div = ow_ // vw_ + kvshape = (co_ // vc_, ci_, kh_, kw_, vc_) + ovshape = (n_, co_ // vc_, oh_div, ow_div, vh_, vw_, vc_) + oshape = (n_, co_, oh_div * vh_, ow_div * vw_) + + if dilation_h != 1 or dilation_w != 1: + # undilate input data + dvshape = (n_, oh_ // vh_, ow_ // vw_, kh_, kw_, vh_, vw_, ci_) + data_vec = tvm.compute( + dvshape, + lambda n, h, w, kh, kw, vh, vw, ci: data_pad[n][ci][ + (h * vh_ + vh) * hstr + kh * dilation_h + ][(w * vw_ + vw) * wstr + kw * dilation_w], + name="data_vec_undilated", + ) + else: + dvshape = ( + n_, + oh_ // vh_, + ow_ // vw_, + (vh_ - 1) * hstr + kh_, + (vw_ - 1) * wstr + kw_, + ci_, + ) + data_vec = tvm.compute( + dvshape, + lambda n, h, w, vh, vw, ci: data_pad[n][ci][h * vh_ * hstr + vh][ + w * vw_ * wstr + vw + ], + name="data_vec", + ) + + if pre_packed: + kernel_vec = kernel + else: + kernel_vec = tvm.compute( + kvshape, + lambda co, ci, kh, kw, vc: kernel[co * vc_ + vc][ci][kh][kw], + name="kernel_vec", + ) + + ci = tvm.reduce_axis((0, ci_), name="ci") + kh = tvm.reduce_axis((0, kh_), name="kh") + kw = tvm.reduce_axis((0, kw_), name="kw") + + # asm begin---- + type_map = { + "int8": "int32", + "uint8": "uint32", + "float32": "float32", + "float16": "float16", + } + acum_dtype = type_map[data.dtype] + attrs = { + "SH": hstr, + "SW": wstr, + "PH": pad_top, + "PW": pad_left, + "DILA_H": dilation_h, + "DILA_W": dilation_w, + "VH": vh_, + "VW": vw_, + "VC": vc_, + "ACUM_DTYPE": acum_dtype, + } + # asm end---- + + if dilation_h != 1 or dilation_w != 1: + conv = tvm.compute( + ovshape, + lambda n, co, h, w, vh, vw, vc: tvm.sum( + data_vec[n, h, w, kh, kw, vh, vw, ci].astype(out_dtype) + * kernel_vec[co, ci, kh, kw, vc].astype(out_dtype), + axis=[ci, kh, kw], + ), + name="conv", + attrs=attrs, + ) + else: + conv = tvm.compute( + ovshape, + lambda n, co, h, w, vh, vw, vc: tvm.sum( + data_vec[n, h, w, vh * hstr + kh, vw * wstr + kw, ci].astype(out_dtype) + * kernel_vec[co, ci, kh, kw, vc].astype(out_dtype), + axis=[ci, kh, kw], + ), + name="conv", + attrs=attrs, + ) + + output = tvm.compute( + oshape, + lambda n, co, h, w: conv[n][co // vc_][h // vh_][w // vw_][h % vh_][w % vw_][ + co % vc_ + ], + name="output_unpack", + tag="asm_conv2d_output", + ) + + return output + + +def intrin_conv(args): + """intrin_conv""" + ( + ci_, + vh_, + vw_, + vc_, + kh_, + kw_, + sh_, + sw_, + dila_h, + dila_w, + dtype, + acum_dtype, + opname, + core_id, + ) = args + hstr, wstr = sh_, sw_ + ci_ = tvm.var("ci_") if ci_ is None else ci_ + kvshape = (ci_, kh_, kw_, vc_) + ovshape = (vh_, vw_, vc_) + if dila_h != 1 or dila_w != 1: + dvshape = (kh_, kw_, vh_, vw_, ci_) + else: + dvshape = ((vh_ - 1) * hstr + kh_, (vw_ - 1) * wstr + kw_, ci_) + + data_vec = tvm.placeholder(dvshape, name="a", dtype=dtype) + kernel_vec = tvm.placeholder(kvshape, name="b", dtype=dtype) + ci = tvm.reduce_axis((0, ci_), name="ci") + kh = tvm.reduce_axis((0, kh_), name="kh") + kw = tvm.reduce_axis((0, kw_), name="kw") + if dila_h != 1 or dila_w != 1: + conv = tvm.compute( + ovshape, + lambda vh, vw, vc: tvm.sum( + data_vec[kh, kw, vh, vw, ci].astype(acum_dtype) + * kernel_vec[ci, kh, kw, vc].astype(acum_dtype), + axis=[ci, kh, kw], + ), + name="conv", + ) + else: + conv = tvm.compute( + ovshape, + lambda vh, vw, vc: tvm.sum( + data_vec[vh * hstr + kh, vw * wstr + kw, ci].astype(acum_dtype) + * kernel_vec[ci, kh, kw, vc].astype(acum_dtype), + axis=[ci, kh, kw], + ), + name="conv", + ) + + stride_a = [ + functools.reduce(lambda x, y: x * y, dvshape[i + 1: len(dvshape)]) + for i in range(0, len(dvshape) - 1) + ] + stride_a.append(1) + stride_b = [ + functools.reduce(lambda x, y: x * y, kvshape[i + 1: len(kvshape)]) + for i in range(0, len(kvshape) - 1) + ] + stride_b.append(1) + stride_c = [ + functools.reduce(lambda x, y: x * y, ovshape[i + 1: len(ovshape)]) + for i in range(0, len(ovshape) - 1) + ] + stride_c.append(1) + + a_buffer = tvm.decl_buffer( + data_vec.shape, data_vec.dtype, name="A", offset_factor=1, strides=stride_a + ) + b_buffer = tvm.decl_buffer( + kernel_vec.shape, kernel_vec.dtype, name="B", offset_factor=1, strides=stride_b + ) + c_buffer = tvm.decl_buffer( + conv.shape, conv.dtype, name="C", offset_factor=1, strides=stride_c + ) + + def intrin_func(ins, outs): + aa, bb = ins + cc = outs[0] + + def _body(): + ib = tvm.ir_builder.create() + ib.emit( + tvm.call_extern( + "int32", + opname, + cc.access_ptr("w"), + aa.access_ptr("r"), + bb.access_ptr("r"), + ci_, + vh_, + vw_, + vc_, + kh_, + sh_, + core_id, + ) + ) + return ib.get() + + return _body() + + return tvm.decl_tensor_intrin( + conv.op, intrin_func, binds={data_vec: a_buffer, kernel_vec: b_buffer, conv: c_buffer} + ) + + +def _schedule_asm(s, data_vec, kernel_vec, conv, output, last): + """schedule implementation""" + n, co, oh, ow, vh, vw, vc = s[conv].op.axis + + axis_extent = [] + for i in (vh, vw, vc): + axis_extent.append(get_const_int(i.dom.extent)) + reduce_extent = [] + for i in s[conv].op.reduce_axis[1:]: + reduce_extent.append(get_const_int(i.dom.extent)) + vh_, vw_, vc_ = axis_extent + + # schedule fusion + n, co, h, w = s[last].op.axis + co, vc = s[last].split(co, vc_) + oh, vh = s[last].split(h, vh_) + ow, vw = s[last].split(w, vw_) + s[last].reorder(n, co, oh, ow, vh, vw, vc) + if last != output: + s[output].compute_inline() + + s[conv].compute_at(s[last], ow) + + # mark parallel + s[last].parallel(co) + + if data_vec.op.name == "data_vec_undilated": + _, h, _, _, _, _, _, _ = s[data_vec].op.axis + else: + _, h, _, _, _, _ = s[data_vec].op.axis + s[data_vec].parallel(h) + + if kernel_vec.op.name == "kernel_vec": + co, _, _, _, _ = s[kernel_vec].op.axis + if autotvm.GLOBAL_SCOPE.in_tuning: + # kernel packing will be pre-computed during compilation, so we skip + # this part to make tuning records correct + s[kernel_vec].pragma(co, "debug_skip_region") + else: + s[kernel_vec].parallel(co) + elif kernel_vec.op.name == "kernel_vec_conv2d_transpose": # for conv2d transpose + co, _, _, _, _ = s[kernel_vec].op.axis + s[kernel_vec].parallel(co) + + return s + + +def _conv_schedule_asm(outs): + """_conv_schedule_asm""" + s = tvm.create_schedule([x.op for x in outs]) + + def _callback(op): + if "asm_conv2d_output" in op.tag: + # schedule conv2d + output = op.output(0) + conv = op.input_tensors[0] + + sidx = 0 + if conv.op.input_tensors[0].name == "attr": + sidx = 1 + data_vec = conv.op.input_tensors[sidx] + data_pad = data_vec.op.input_tensors[0] + s[data_pad].compute_inline() + + kernel_vec = conv.op.input_tensors[sidx + 1] + if kernel_vec.op.name == "kernel_vec": + kernel = kernel_vec.op.input_tensors[0] + else: + kernel = kernel_vec + if (isinstance(kernel.op, tvm.tensor.ComputeOp) and "dilate" in kernel.op.tag): + s[kernel].compute_inline() + + if conv.op.input_tensors[0].name == "attr": + _schedule_asm(s, data_vec, kernel_vec, conv, output, outs[0]) + else: + _schedule_asm(s, data_vec, kernel_vec, conv, output, outs[0]) + + traverse_inline(s, outs[0].op, _callback) + return s diff --git a/predict/module/tvm_kernel/lite/python/arm_cpu/deconv.py b/predict/module/tvm_kernel/lite/python/arm_cpu/deconv.py new file mode 100644 index 00000000000..4ed29900a25 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/arm_cpu/deconv.py @@ -0,0 +1,477 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Conv2D_transpose of stride=2, kernel=2*2 schedule for ARM CPU""" +from __future__ import absolute_import as _abs + +import functools + +import tvm +from tvm import autotvm +import tvm.contrib.nnpack + +from topi.generic import schedule_conv2d_nchw +from topi.util import traverse_inline, get_const_tuple +from topi.nn import conv2d + + +@autotvm.register_topi_compute(conv2d, "arm_cpu", ["deconv"]) +def conv2d_arm_cpu_deconv(cfg, data, kernel, out_dtype): + """TOPI compute callback for conv2d + + Parameters + ---------- + cfg: ConfigEntity + The config for this template + + data : tvm.Tensor + 4-D with shape [batch, in_channel, in_height, in_width] + + kernel : tvm.Tensor + 4-D with shape [num_filter, in_channel, filter_height, filter_width] or + pre-packed 5-D with shape [num_filter_chunk, in_channel, filter_height, + filter_width, num_filter_block] + + out_dtype: str + The output type. This is used for mixed precision. + + Returns + ------- + output : tvm.Tensor + 4-D with shape [batch, out_channel, out_height, out_width] + """ + args = _gen_cfg_deconv(cfg, data, kernel, num_tile=2) + return _conv_spatial_pack_deconv( + args, data, kernel, out_dtype + ) + + +@autotvm.register_topi_schedule(schedule_conv2d_nchw, "arm_cpu", ["deconv"]) +def schedule_conv2d_nchw_arm_cpu_deconv(cfg, outs): + """TOPI schedule callback for conv2d + + Parameters + ---------- + cfg: ConfigEntity + The config for this template + + outs: Array of Tensor + The computation graph description of conv2d + in the format of an array of tensors. + + Returns + ------- + s: Schedule + The computation schedule for conv2d. + """ + s = _conv_schedule_deconv(cfg, outs) + return s + + +def _gen_cfg_deconv(cfg, data, kernel, num_tile): + """generation config from input args""" + if len(kernel.shape) == 4: + co_, _, _, _ = get_const_tuple(kernel.shape) + else: # kernel tensor is pre packed + co_, _, _, _, vc_ = get_const_tuple(kernel.shape) + co_ = co_ * vc_ + + if len(data.shape) == 4: + _, ci_, ih_, iw_ = get_const_tuple(data.shape) + c4 = 4 + ci_ = ci_ // 4 + else: + _, ci_, ih_, iw_, c4 = get_const_tuple(data.shape) + + oh_ = ih_ * 2 + ow_ = iw_ * 2 + + co, oh, ow = cfg.axis(co_), cfg.axis(oh_), cfg.axis(ow_) + ci, ki = cfg.reduce_axis(ci_), cfg.reduce_axis(c4) + + if num_tile == 2: # for arm cpu + candidate_vc = [[co_ // c4, c4]] + co, vc = cfg.define_split( + "tile_co", co, num_outputs=2, policy="candidate", candidate=candidate_vc + ) + candidate_vw = [] + for iv in range(4, ow_ + 1): # [4, 6, 8, 12, 16, 24, 32, 40]: + if iv % 4 == 0 and (ow_ % iv == 0): + candidate_vw.append([ow_ // iv, iv]) + ow, vw = cfg.define_split( + "tile_ow", ow, num_outputs=2, policy="candidate", candidate=candidate_vw + ) + candidate_vh = [[1, 2]] + oh, vh = cfg.define_split( + "tile_oh", oh, num_outputs=2, policy="candidate", candidate=candidate_vh + ) + elif num_tile == 3: # for mali gpu + co, _, vc = cfg.define_split("tile_co", co, num_outputs=3) + oh, _, vh = cfg.define_split("tile_oh", oh, num_outputs=3) + ow, _, vw = cfg.define_split("tile_ow", ow, num_outputs=3) + else: + raise RuntimeError("Invalid num_tile") + + cfg.define_annotate("ann_reduce", [ci, ki], policy="try_unroll") + cfg.define_annotate("ann_spatial", [vh, vw, vc], policy="try_unroll_vec") + + vc_ = cfg["tile_co"].size[-1] + vh_ = cfg["tile_oh"].size[-1] + vw_ = cfg["tile_ow"].size[-1] + is_var = False + return (is_var, vh_, vw_, vc_) + + +def _conv_spatial_pack_deconv(args, data, kernel, out_dtype): + """conv2d_arm_cpu_deconv inner implement""" + is_var, vh_, vw_, vc_ = args + # create workload according to raw arguments + out_dtype = out_dtype or data.dtype + if len(data.shape) == 4: + n_, ci_, ih_, iw_ = data.shape if is_var else get_const_tuple(data.shape) + c4 = 4 + ci_ = ci_ // c4 + else: + n_, ci_, ih_, iw_, c4 = data.shape if is_var else get_const_tuple(data.shape) + + if len(kernel.shape) == 4: + pre_packed = False + _, co_, kh_, kw_ = kernel.shape if is_var else get_const_tuple(kernel.shape) + else: # kernel tensor is pre packed + pre_packed = True + _, co_, kh_, kw_, vc_ = kernel.shape if is_var else get_const_tuple(kernel.shape) + co_ = co_ * c4 + + oh_ = ih_ * 2 + ow_ = iw_ * 2 + ow_div = ow_ // vw_ + oh_div = oh_ // vh_ + kvshape = (co_ // vc_, kh_, kw_, ci_, c4, c4) + ovshape = (n_, co_ // vc_, oh_div, ow_div, vh_, vw_, c4) + + dvshape = (n_, ih_ // (vh_ // 2), iw_ // (vw_ // 2), vh_ // 2, ci_, vw_ // 2, c4) + if len(data.shape) == 4: + data_vec = tvm.compute( + dvshape, + lambda n, h, w, vh, ci, vw, ki: data[n][ci * c4 + ki][h * vh_ // 2 + vh][ + w * vw_ // 2 + vw + ], + name="data_vec", + ) + else: + data_vec = tvm.compute( + dvshape, + lambda n, h, w, vh, ci, vw, ki: data[n][ci][h * vh_ // 2 + vh][ + w * vw_ // 2 + vw + ][ki], + name="data_vec", + ) + + if pre_packed: + kernel_vec = kernel + else: + kernel_vec = tvm.compute( + kvshape, + lambda co, kh, kw, ci, ki, vc: kernel[ci * c4 + ki][co * vc_ + vc][kh][kw], + name="kernel_vec", + ) + + ci = tvm.reduce_axis((0, ci_), name="ci") + ki = tvm.reduce_axis((0, c4), name="ki") + + type_map = { + "int8": "int32", + "uint8": "uint32", + "float32": "float32", + "float16": "float16", + } + acum_dtype = type_map[data.dtype] + attrs = { + "SH": 2, + "SW": 2, + "PH": 0, + "PW": 0, + "DILA_H": 1, + "DILA_W": 1, + "VH": vh_, + "VW": vw_, + "VC": vc_, + "ACUM_DTYPE": acum_dtype, + } + + conv = tvm.compute( + ovshape, + lambda n, co, h, w, vh, vw, vc: tvm.sum( + data_vec[n, h, w, vh // 2, ci, vw // 2, ki].astype(out_dtype) + * kernel_vec[co, (h * vh_ + vh) % 2, (w * vw_ + vw) % 2, ci, ki, vc].astype( + out_dtype + ), + axis=[ci, ki], + ), + name="conv", + attrs=attrs, + ) + if len(data.shape) == 4: + osshape = (n_, co_, oh_, ow_div * vw_) + output = tvm.compute( + osshape, + lambda n, co, h, w: conv[n][co // c4][h][w // vw_][w % vw_][co % c4], + name="output_unpack", + tag="deconv_conv2d_output", + ) + else: + osshape = (n_, co_ // c4, oh_, ow_div * vw_, c4) + output = tvm.compute( + osshape, + lambda n, co, h, w, vc: conv[n][co][h // vh_][w // vw_][h % vh_][w % vw_][vc], + name="output_unpack", + tag="deconv_conv2d_output", + ) + + return output + + +def intrin_deconv(args): + """deconv inner implement""" + ( + ci_, + vh_, + vw_, + vc_, + kh_, + kw_, + sh_, + sw_, + dila_h, + dila_w, + dtype, + acum_dtype, + opname, + core_id, + ) = args + hstr, wstr = sh_, sw_ + ci_ = tvm.var("ci_") if ci_ is None else ci_ + kvshape = (ci_, kh_, kw_, vc_) + ovshape = (vh_, vw_, vc_) + if dila_h != 1 or dila_w != 1: + dvshape = (kh_, kw_, vh_, vw_, ci_) + else: + dvshape = ((vh_ - 1) * hstr + kh_, (vw_ - 1) * wstr + kw_, ci_) + + data_vec = tvm.placeholder(dvshape, name="a", dtype=dtype) + kernel_vec = tvm.placeholder(kvshape, name="b", dtype=dtype) + ci = tvm.reduce_axis((0, ci_), name="ci") + kh = tvm.reduce_axis((0, kh_), name="kh") + kw = tvm.reduce_axis((0, kw_), name="kw") + if DILA_H != 1 or dila_w != 1: + conv = tvm.compute( + ovshape, + lambda vh, vw, vc: tvm.sum( + data_vec[kh, kw, vh, vw, ci].astype(acum_dtype) + * kernel_vec[ci, kh, kw, vc].astype(acum_dtype), + axis=[ci, kh, kw], + ), + name="conv", + ) + else: + conv = tvm.compute( + ovshape, + lambda vh, vw, vc: tvm.sum( + data_vec[vh * hstr + kh, vw * wstr + kw, ci].astype(acum_dtype) + * kernel_vec[ci, kh, kw, vc].astype(acum_dtype), + axis=[ci, kh, kw], + ), + name="conv", + ) + + stride_a = [ + functools.reduce(lambda x, y: x * y, dvshape[i + 1: len(dvshape)]) + for i in range(0, len(dvshape) - 1) + ] + stride_a.append(1) + stride_b = [ + functools.reduce(lambda x, y: x * y, kvshape[i + 1: len(kvshape)]) + for i in range(0, len(kvshape) - 1) + ] + stride_b.append(1) + stride_c = [ + functools.reduce(lambda x, y: x * y, ovshape[i + 1: len(ovshape)]) + for i in range(0, len(ovshape) - 1) + ] + stride_c.append(1) + + a_buffer = tvm.decl_buffer( + data_vec.shape, data_vec.dtype, name="A", offset_factor=1, strides=stride_a + ) + b_buffer = tvm.decl_buffer( + kernel_vec.shape, kernel_vec.dtype, name="B", offset_factor=1, strides=stride_b + ) + c_buffer = tvm.decl_buffer( + conv.shape, conv.dtype, name="C", offset_factor=1, strides=stride_c + ) + + def intrin_func(ins, outs): + aa, bb = ins + cc = outs[0] + + def _body(): + ib = tvm.ir_builder.create() + ib.emit( + tvm.call_extern( + "int32", + opname, + cc.access_ptr("w"), + aa.access_ptr("r"), + bb.access_ptr("r"), + ci_, + vh_, + vw_, + vc_, + kh_, + sh_, + core_id, + ) + ) + return ib.get() + + return _body() + + return tvm.decl_tensor_intrin( + conv.op, intrin_func, binds={data_vec: a_buffer, kernel_vec: b_buffer, conv: c_buffer} + ) + + +def _schedule_deconv(cfg, s, data_vec, kernel_vec, conv, output, last): + """schedule implementation""" + is_tune = bool(isinstance(cfg, (tvm.autotvm.ConfigEntity, tvm.autotvm.ConfigSpace))) + if is_tune: + vh_ = cfg["tile_oh"].size[-1] + vw_ = cfg["tile_ow"].size[-1] + vc_ = cfg["tile_co"].size[-1] + cfg = { + "ci_": tvm.var("ci_"), + "VH": vh_, + "VW": vw_, + "VC": vc_, + "tile_oh": vh_, + "tile_ow": vw_, + "tile_co": vc_, + "tile_ci": 4, + "ann_reduce": cfg["ann_reduce"].anns, + "ann_spatial": cfg["ann_spatial"].anns, + } # ,'reorder_0':cfg['reorder_0'].perm} + else: + pass + n, co, oh, ow, vh, vw, vc = s[conv].op.axis + ci, ki = s[conv].op.reduce_axis + s[conv].reorder(n, co, oh, ow, ci, vw, ki, vc) + if cfg["ann_reduce"][0] == "unroll": + s[conv].unroll(ci) + elif cfg["ann_reduce"][0] == "vec": + s[conv].vectorize(ci) + if cfg["ann_reduce"][1] == "unroll": + s[conv].unroll(ki) + elif cfg["ann_reduce"][1] == "vec": + s[conv].vectorize(ki) + if cfg["ann_spatial"][0] == "vec": + s[conv].vectorize(vh) + elif cfg["ann_spatial"][0] == "unroll": + s[conv].unroll(vh) + if cfg["ann_spatial"][1] == "vec": + s[conv].vectorize(vw) + elif cfg["ann_spatial"][1] == "unroll": + s[conv].unroll(vw) + if cfg["ann_spatial"][2] == "vec": + s[conv].vectorize(vc) + elif cfg["ann_spatial"][2] == "unroll": + s[conv].unroll(vc) + + # schedule conv + attrs = conv.op.attrs + vh_, vw_, vc_ = (attrs["VH"].value, attrs["VW"].value, attrs["VC"].value) + + # schedule fusion + if len(s[last].op.axis) == 4: + n, co, h, w = s[last].op.axis + co, vc = s[last].split(co, vc_) + ow, vw = s[last].split(w, vw_) + oh, vh = s[last].split(h, vh_) + s[last].reorder(n, co, oh, ow, vh, vw, vc) + else: + n, co, h, w, vc = s[last].op.axis + oh, vh = s[last].split(h, vh_) + ow, vw = s[last].split(w, vw_) + s[last].reorder(n, co, oh, ow, vh, vw, vc) + if last != output and isinstance(output.op, tvm.tensor.ComputeOp): + s[output].compute_inline() + if cfg["ann_spatial"][0] == "vec": + s[last].vectorize(vh) + elif cfg["ann_spatial"][0] == "unroll": + s[last].unroll(vh) + if cfg["ann_spatial"][1] == "vec": + s[last].vectorize(vw) + elif cfg["ann_spatial"][1] == "unroll": + s[last].unroll(vw) + if cfg["ann_spatial"][2] == "vec": + s[last].vectorize(vc) + elif cfg["ann_spatial"][2] == "unroll": + s[last].unroll(vc) + + s[conv].compute_at(s[last], ow) + + # mark parallel + s[last].parallel(co) + + if data_vec.op.name == "data_vec_undilated": + _, h, _, _, _, _, _, _, _ = s[data_vec].op.axis + else: + _, h, _, _, _, _, _ = s[data_vec].op.axis + s[data_vec].parallel(h) + + co, _, _, _, _, vc = s[kernel_vec].op.axis + s[kernel_vec].parallel(co) + if cfg["ann_spatial"][2] == "vec": + s[kernel_vec].vectorize(vc) + elif cfg["ann_spatial"][2] == "unroll": + s[kernel_vec].unroll(vc) + return s + + +def _conv_schedule_deconv(cfg, outs): + """schedule_conv2d_nchw_arm_cpu_deconv inner implementation""" + s = tvm.create_schedule([x.op for x in outs]) + + def _callback(op): + if "deconv_conv2d_output" in op.tag: + # schedule conv2d + output = op.output(0) + conv = op.input_tensors[0] + + sidx = 0 + if conv.op.input_tensors[0].name == "attr": + sidx = 1 + data_vec = conv.op.input_tensors[sidx] + + kernel_vec = conv.op.input_tensors[sidx + 1] + if kernel_vec.op.name == "kernel_vec": + kernel = kernel_vec.op.input_tensors[0] + else: + kernel = kernel_vec + if (isinstance(kernel.op, tvm.tensor.ComputeOp) and "dilate" in kernel.op.tag): + s[kernel].compute_inline() + + _schedule_deconv(cfg, s, data_vec, kernel_vec, conv, output, outs[0]) + + traverse_inline(s, outs[0].op, _callback) + return s diff --git a/predict/module/tvm_kernel/lite/python/arm_cpu/depthwise_conv2d.py b/predict/module/tvm_kernel/lite/python/arm_cpu/depthwise_conv2d.py new file mode 100644 index 00000000000..f54076eb849 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/arm_cpu/depthwise_conv2d.py @@ -0,0 +1,289 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Depthwise convolution schedule for ARM CPU""" + +import tvm +from tvm import autotvm + +from topi.generic import schedule_depthwise_conv2d_nchw +from topi.nn import depthwise_conv2d_nchw, pad +from topi.util import traverse_inline, get_const_tuple +from topi.nn.util import get_pad_tuple + +# register customized schedule for arm cpu. +@autotvm.register_topi_schedule( + schedule_depthwise_conv2d_nchw, ["arm_cpu", "cpu"], ["custom"] +) +def schedule_depthwise_conv2d_nchw_arm(cfg, outs): + """Schedule depthwise conv2d + + Parameters + ---------- + cfg: ConfigEntity + The configuration of this template + outs: Array of Tensor + The computation graph description of depthwise convolution2d + in the format of an array of tensors. + + Returns + ------- + s: Schedule + The computation schedule for depthwise_conv2d nchw. + """ + s = _depthwise_schedule_spatial_pack(cfg, outs) + return s + + +@autotvm.register_topi_compute(depthwise_conv2d_nchw, ["arm_cpu", "cpu"], ["custom"]) +def depthwise_conv2d_arm_cpu(cfg, data, kernel, strides, padding, dilation, out_dtype): + """TOPI compute callback for depthwise_conv2d nchw + + Parameters + ---------- + cfg: ConfigEntity + The config for this template + + data : tvm.Tensor + 4-D with shape [batch, in_channel, in_height, in_width] + + kernel : tvm.Tensor + 4-D with shape [num_filter, multiplier, filter_height, filter_width] or + pre-packed 5-D with shape [num_filter_chunk, multiplier, filter_height, + filter_width, num_filter_block] + + strides : list of two ints + [stride_height, stride_width] + + padding : list of two ints + [pad_height, pad_width] + + dilation : list of two ints + [dilation_height, dilation_width] + + out_dtype: str + The output type. This is used for mixed precision. + + Returns + ------- + output : tvm.Tensor + 4-D with shape [batch, out_channel, out_height, out_width] + """ + + return _depthwise_spatial_pack( + cfg, data, kernel, strides, padding, dilation, out_dtype + ) + + +def _depthwise_spatial_pack(args, data, kernel, strides, padding, dilation, out_dtype): + """depthwise_conv2d_arm_cpu's inner implement""" + is_var, u_vh, u_vw, u_vc = args + out_dtype = out_dtype or data.dtype + + u_n, u_c, ih, iw = data.shape if is_var else get_const_tuple(data.shape) + + if isinstance(dilation, int): + dilation_h = dilation_w = dilation + else: + dilation_h, dilation_w = dilation + + if len(kernel.shape) == 4: + pre_packed = False + u_c, um, ukh, ukw = kernel.shape if is_var else get_const_tuple(kernel.shape) + else: # kernel tensor is pre packed + pre_packed = True + u_c, um, ukh, ukw, u_vc = kernel.shape if is_var else get_const_tuple(kernel.shape) + u_c = u_c * u_vc + + dilated_kernel_h = (ukh - 1) * dilation_h + 1 + dilated_kernel_w = (ukw - 1) * dilation_w + 1 + + pad_top, pad_left, pad_down, pad_right = get_pad_tuple( + padding, (dilated_kernel_h, dilated_kernel_w) + ) + hstr, wstr = strides if isinstance(strides, (tuple, list)) else (strides, strides) + u_oh = (ih + pad_top + pad_down - dilated_kernel_h) // hstr + 1 + u_ow = (iw + pad_left + pad_right - dilated_kernel_w) // wstr + 1 + # pack data + hpad = pad_top + pad_down + wpad = pad_left + pad_right + dopad = hpad != 0 or wpad != 0 + if dopad: + data_pad = pad( + data, + (0, 0, pad_top, pad_left), + (0, 0, pad_down, pad_right), + name="data_pad", + ) + else: + data_pad = data + + oh_div = u_oh // u_vh + ow_div = u_ow // u_vw + kvshape = (u_c // u_vc, um, ukh, ukw, u_vc) + ovshape = (u_n, u_c * um // u_vc, oh_div, u_ow // u_vw, u_vh, u_vw, u_vc) + oshape = (u_n, u_c * um, oh_div * u_vh, ow_div * u_vw) + + if dilation_h != 1 or dilation_w != 1: + # undilate input data + dvshape = (u_n, oh_div, ow_div, u_c, ukh, ukw, u_vh, u_vw) + data_vec = tvm.compute( + dvshape, + lambda n, h, w, c, kh, kw, vh, vw: data_pad[n][c][ + (h * u_vh + vh) * hstr + kh * dilation_h + ][(w * u_vw + vw) * wstr + kw * dilation_w], + name="data_vec_undilated", + ) + else: + dvshape = (u_n, oh_div, ow_div, u_c, u_vh * hstr + ukh - 1, u_vw * wstr + ukw - 1) + data_vec = tvm.compute( + dvshape, + lambda n, h, w, c, vh, vw: data_pad[n][c][h * u_vh * hstr + vh][ + w * u_vw * wstr + vw + ], + name="data_vec", + ) + + if pre_packed: + kernel_vec = kernel + else: + kernel_vec = tvm.compute( + kvshape, + lambda co, m, kh, kw, vc: kernel[co * u_vc + vc][m][kh][kw], + name="kernel_vec", + ) + + kh = tvm.reduce_axis((0, ukh), name="kh") + kw = tvm.reduce_axis((0, ukw), name="kw") + + if dilation_h != 1 or dilation_w != 1: + conv = tvm.compute( + ovshape, + lambda n, co, h, w, vh, vw, vc: tvm.sum( + data_vec[n, h, w, (co * u_vc + vc) // um, kh, kw, vh, vw].astype(out_dtype) + * kernel_vec[co // um, co % um, kh, kw, vc].astype(out_dtype), + axis=[kh, kw], + ), + name="depthwise_conv", + ) + else: + conv = tvm.compute( + ovshape, + lambda n, co, h, w, vh, vw, vc: tvm.sum( + data_vec[ + n, h, w, (co * u_vc + vc) // um, vh * hstr + kh, vw * wstr + kw + ].astype(out_dtype) + * kernel_vec[co // um, co % um, kh, kw, vc].astype(out_dtype), + axis=[kh, kw], + ), + name="depthwise_conv", + ) + + output = tvm.compute( + oshape, + lambda n, co, h, w: conv[n][co // u_vc][h // u_vh][w // u_vw][h % u_vh][w % u_vw][ + co % u_vc + ], + name="output_unpack", + tag="spatial_depthwise_conv_nchw_output", + ) + return output + + +def _schedule_spatial_pack(cfg, s, data_vec, kernel_vec, conv, output, last): + """schedule implementation""" + u_vc = cfg["tile_co"].size[-1] if not isinstance(cfg, dict) else cfg["VC"] + u_vh = cfg["tile_oh"].size[-1] if not isinstance(cfg, dict) else cfg["VH"] + u_vw = cfg["tile_ow"].size[-1] if not isinstance(cfg, dict) else cfg["VW"] + + n, co, oh, ow, vh, vw, vc = s[conv].op.axis + kh, kw = s[conv].op.reduce_axis + + if data_vec.op.name == "data_vec_undilated": + _, _, dv_ow, _, _, _, _, _ = s[data_vec].op.axis + else: + _, _, dv_ow, _, _, _ = s[data_vec].op.axis + + data_pad = data_vec.op.input_tensors[0] + + if isinstance(data_pad.op, tvm.tensor.ComputeOp): + s[data_pad].vectorize(list(s[data_pad].op.axis)[-1]) + s[data_pad].compute_at(s[data_vec], dv_ow) + + s[data_vec].vectorize(list(s[data_vec].op.axis)[-1]) + s[data_vec].compute_at(s[conv], ow) + + # schedule conv + s[conv].reorder(n, co, oh, ow, kh, kw, vh, vw, vc) + s[conv].unroll(kh) + s[conv].unroll(vh) + s[conv].vectorize(vw) + s[conv].unroll(vc) + s[conv].parallel(co) + + n, co, h, w = s[last].op.axis + co, vc = s[last].split(co, u_vc) + oh, vh = s[last].split(h, u_vh) + ow, vw = s[last].split(w, u_vw) + if last != output: + s[output].compute_inline() + s[last].vectorize(vw) + s[last].unroll(vc) + else: + s[last].vectorize(vw) + s[conv].compute_at(s[last], oh) + + # mark parallel + s[last].parallel(co) + + if data_vec.op.name == "data_vec_undilated": + _, h, _, _, _, _, _, _ = s[data_vec].op.axis + else: + _, h, _, _, _, _ = s[data_vec].op.axis + s[data_vec].parallel(h) + + if kernel_vec.op.name == "kernel_vec": + co, _, _, _, _ = s[kernel_vec].op.axis + if autotvm.GLOBAL_SCOPE.in_tuning: + # kernel packing will be pre-computed during compliation, so we skip + # this part to make tuning records correct + s[kernel_vec].pragma(co, "debug_skip_region") + else: + s[kernel_vec].parallel(co) + + return s + + +def _depthwise_schedule_spatial_pack(cfg, outs): + """schedule_depthwise_conv2d_nchw_arm's inner implement""" + outs = [outs] if isinstance(outs, tvm.tensor.Tensor) else outs + s = tvm.create_schedule([x.op for x in outs]) + + def _callback(op): + if op.tag == "spatial_depthwise_conv_nchw_output": + output = op.output(0) + conv = op.input_tensors[0] + data_vec = conv.op.input_tensors[0] + kernel_vec = conv.op.input_tensors[1] + if kernel_vec.op.name == "kernel_vec": + kernel = kernel_vec.op.input_tensors[0] + else: + kernel = kernel_vec + if isinstance(kernel.op, tvm.tensor.ComputeOp) and "dilate" in kernel.op.tag: + s[kernel].compute_inline() + + _schedule_spatial_pack(cfg, s, data_vec, kernel_vec, conv, output, outs[0]) + + traverse_inline(s, outs[0].op, _callback) + return s diff --git a/predict/module/tvm_kernel/lite/python/arm_cpu/matmul.py b/predict/module/tvm_kernel/lite/python/arm_cpu/matmul.py new file mode 100644 index 00000000000..6430f24f6f2 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/arm_cpu/matmul.py @@ -0,0 +1,472 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Conv2D schedule for ARM CPU""" +from __future__ import absolute_import as _abs + +import functools + +import tvm +from tvm import autotvm +import tvm.contrib.nnpack + +from topi.generic import schedule_conv2d_nchw +from topi.util import traverse_inline +from topi.nn import conv2d + + +@autotvm.register_topi_compute(conv2d, "arm_cpu", ["matmul"]) +def matmul_arm_cpu(cfg, a_, b_, layout, out_dtype): + """TOPI compute callback for + + Parameters + ---------- + cfg: ConfigEntity + The config for this template + + a_ : tvm.Tensor + 2-D with shape [M, k_] + + b_ : tvm.Tensor + 2-D with shape [k_, N] + + out_dtype: str + The output type. This is used for mixed precision. + + Returns + ------- + output : tvm.Tensor + 4-D with shape [batch, out_channel, out_height, out_width] + """ + args = _gen_cfg(cfg, a_, b_) + return _matmul_spatial_pack_asm(args, a_, b_, layout, out_dtype) + + +@autotvm.register_topi_schedule(schedule_conv2d_nchw, "arm_cpu", ["matmul"]) +def schedule_matmul_arm_cpu(cfg, outs): + """TOPI schedule callback for conv2d + + Parameters + ---------- + cfg: ConfigEntity + The config for this template + + outs: Array of Tensor + The computation graph description of conv2d + in the format of an array of tensors. + + Returns + ------- + s: Schedule + The computation schedule for conv2d. + """ + s = _matmul_schedule_asm(cfg, outs) + return s + + +def _gen_cfg(cfg, a_, b_): + """get best loginfo from cfg""" + if len(a_.shape) == 2: + w_, ci_ = get_const_tuple(a_.shape) + h_ = 1 + elif len(a_.shape) == 3: + _, ci_, w_ = get_const_tuple(a_.shape) + h_ = 1 + elif len(a_.shape) == 4: + _, ci_, h_, w_ = get_const_tuple(a_.shape) + else: + raise ValueError("not support shape: " + a_.shape) + + co_, k_ = get_const_tuple(b_.shape) + + oh, ow = cfg.axis(h_), cfg.axis(w_) + co = cfg.axis(co_) + k = cfg.reduce_axis(k_) + + oh, vh = cfg.define_split("tile_oh", oh, num_outputs=2) + ow, vw = cfg.define_split("tile_ow", ow, num_outputs=2) + oc, vc = cfg.define_split("tile_co", co, num_outputs=2) + + cfg.define_reorder( + "reorder_0", + [oc, oh, ow, k, vh, vw, vc], + policy="candidate", + candidate=[[oc, oh, ow, k, vh, vw, vc],], + ) + + vh_ = cfg["tile_oh"].size[-1] + vw_ = cfg["tile_ow"].size[-1] + vc_ = cfg["tile_co"].size[-1] + is_var = False + is_transpose = False + return (is_var, is_transpose, ci_, vh_, vw_, vc_) + + +def _matmul_spatial_pack_asm(args, a_, b_, layout, out_dtype): + """matmul_spatial_pack_asm's inner interace""" + is_var, is_transpose, ci_, vh_, vw_, vc_ = args + + # create workload according to raw arguments + out_dtype = out_dtype or a_.dtype + if layout == "NCHW": + batch, k_, h_, w_ = a_.shape if is_var else get_const_tuple(a_.shape) + n_, _ = b_.shape if is_var else get_const_tuple(b_.shape) + elif layout == "NCH": + batch, k_, h_ = a_.shape if is_var else get_const_tuple(a_.shape) + n_, _ = b_.shape if is_var else get_const_tuple(b_.shape) + w_ = 1 + elif layout == "NC": + w_, k_ = a_.shape if is_var else get_const_tuple(a_.shape) + n_, _ = b_.shape if is_var else get_const_tuple(b_.shape) + h_ = 1 + else: + raise ValueError("not support layout: " + layout) + + ki = tvm.reduce_axis((0, k_), name="ki") + type_map = { + "int8": "int32", + "uint8": "uint32", + "float32": "float32", + "float16": "float16", + } + acum_dtype = type_map[a_.dtype] + attrs = {"ci_": ci_, "vh_": vh_, "vw_": vw_, "vc_": vc_, "ACUM_DTYPE": acum_dtype} + + if layout == "NCHW": + h_div = h_ // vh_ + w_div = w_ // vw_ + n_div = n_ // vc_ + avshape = (batch, h_div, w_div, vh_, vw_, k_) + bvshape = (n_div, k_, vc_) + ovshape = (batch, n_div, h_div, w_div, vh_, vw_, vc_) + + a_vec = tvm.compute( + avshape, + lambda n, oh, ow, vh, vw, ci: a_[n][ci][oh * vh_ + vh][ow * vw_ + vw], + name="a_vec", + ) + b_vec = tvm.compute( + bvshape, lambda oc, ci, vc: b_[oc * vc_ + vc][ci], name="b_vec" + ) + + ma = tvm.compute( + ovshape, + lambda n, oc, oh, ow, vh, vw, vc: tvm.sum( + a_vec[n, oh, ow, vh, vw, ki].astype(out_dtype) + * b_vec[oc, ki, vc].astype(out_dtype), + axis=[ki], + ), + name="matmul", + attrs=attrs, + ) + + if is_transpose: + oshape = (batch, h_div * vh_, w_div * vw_, n_div * vc_) + + output = tvm.compute( + oshape, + lambda n, h, w, c: ma[n][c // vc_][h // vh_][w // vw_][h % vh_][w % vw_][ + c % vc_ + ], + name="output_unpack", + tag="asm_matmul_output", + ) + else: + oshape = (batch, n_div * vc_, h_div * vh_, w_div * vw_) + output = tvm.compute( + oshape, + lambda n, c, h, w: ma[n][c // vc_][h // vh_][w // vw_][h % vh_][w % vw_][ + c % vc_ + ], + name="output_unpack", + tag="asm_matmul_output", + ) + elif layout == "NCH": + w_div = w_ // vw_ + n_div = n_ // vc_ + avshape = (batch, w_div, vw_, k_) + bvshape = (n_div, k_, vc_) + ovshape = (batch, n_div, w_div, vw_, vc_) + oshape = (batch, n_div * vc_, w_div * vw_) + + a_vec = tvm.compute( + avshape, lambda b, om, vw, ci: a_[b][ci][om * vw_ + vw], name="a_vec" + ) + b_vec = tvm.compute( + bvshape, lambda on, ci, vc: b_[on * vc_ + vc][ci], name="b_vec" + ) + + ma = tvm.compute( + ovshape, + lambda b, on, om, vm, vn: tvm.sum( + a_vec[b, om, vm, ki].astype(out_dtype) + * b_vec[on, ki, vn].astype(out_dtype), + axis=[ki], + ), + name="matmul", + attrs=attrs, + ) + + output = tvm.compute( + oshape, + lambda b, n, m: ma[b][n // vc_][m // vw_][m % vw_][n % vc_], + name="output_unpack", + tag="asm_matmul_output", + ) + elif layout == "NC": + w_div = w_ // vw_ + n_div = n_ // vc_ + avshape = (w_div, vw_, k_) + bvshape = (n_div, k_, vc_) + ovshape = (w_div, n_div, vw_, vc_) + oshape = (w_div * vw_, n_div * vc_) + + a_vec = tvm.compute( + avshape, lambda om, vw, ci: a_[om * vw_ + vw][ci], name="a_vec" + ) + b_vec = tvm.compute( + bvshape, lambda on, ci, vc: b_[on * vc_ + vc][ci], name="b_vec" + ) + + ma = tvm.compute( + ovshape, + lambda om, on, vm, vn: tvm.sum( + a_vec[om, vm, ki].astype(out_dtype) + * b_vec[on, ki, vn].astype(out_dtype), + axis=[ki], + ), + name="matmul", + attrs=attrs, + ) + + output = tvm.compute( + oshape, + lambda m, n: ma[m // vw_][n // vc_][m % vw_][n % vc_], + name="output_unpack", + tag="asm_matmul_output", + ) + else: + raise ValueError("not support layout: " + layout) + + return output + + +def intrin_conv(args): + """intrin_conv is a conv inner interface""" + ( + ndim, + ci_, + vh_, + vw_, + vc_, + _, + _, + _, + _, + _, + _, + _, + _, + dtype, + acum_dtype, + opname, + core_id, + ) = args + ci_ = tvm.var("ci_") if ci_ is None else ci_ + kvshape = (ci_, vc_) + if ndim == 2: + dvshape = (vw_, ci_) + ovshape = (vw_, vc_) + + data_vec = tvm.placeholder(dvshape, name="a", dtype=dtype) + kernel_vec = tvm.placeholder(kvshape, name="b", dtype=dtype) + ci = tvm.reduce_axis((0, ci_), name="ci") + conv = tvm.compute( + ovshape, + lambda vw, vc: tvm.sum( + data_vec[vw, ci].astype(acum_dtype) + * kernel_vec[ci, vc].astype(acum_dtype), + axis=[ci], + ), + name="conv", + ) + else: + dvshape = (vh_, vw_, ci_) + ovshape = (vh_, vw_, vc_) + + data_vec = tvm.placeholder(dvshape, name="a", dtype=dtype) + kernel_vec = tvm.placeholder(kvshape, name="b", dtype=dtype) + ci = tvm.reduce_axis((0, ci_), name="ci") + conv = tvm.compute( + ovshape, + lambda vh, vw, vc: tvm.sum( + data_vec[vh, vw, ci].astype(acum_dtype) + * kernel_vec[ci, vc].astype(acum_dtype), + axis=[ci], + ), + name="conv", + ) + + stride_a = [ + functools.reduce(lambda x, y: x * y, dvshape[i + 1: len(dvshape)]) + for i in range(0, len(dvshape) - 1) + ] + stride_a.append(1) + stride_b = [ + functools.reduce(lambda x, y: x * y, kvshape[i + 1: len(kvshape)]) + for i in range(0, len(kvshape) - 1) + ] + stride_b.append(1) + stride_c = [ + functools.reduce(lambda x, y: x * y, ovshape[i + 1: len(ovshape)]) + for i in range(0, len(ovshape) - 1) + ] + stride_c.append(1) + + ab_ = tvm.decl_buffer( + data_vec.shape, data_vec.dtype, name="a_", offset_factor=1, strides=stride_a + ) + bb_ = tvm.decl_buffer( + kernel_vec.shape, kernel_vec.dtype, name="b_", offset_factor=1, strides=stride_b + ) + cb_ = tvm.decl_buffer( + conv.shape, conv.dtype, name="C", offset_factor=1, strides=stride_c + ) + + def intrin_func(ins, outs): + aa, bb = ins + cc = outs[0] + + def _body(): + b_ = tvm.ir_builder.create() + b_.emit( + tvm.call_extern( + "int32", + opname, + cc.access_ptr("w"), + aa.access_ptr("r"), + bb.access_ptr("r"), + ci_, + vh_, + vw_, + vc_, + core_id, + ) + ) + return b_.get() + + return _body() + + return tvm.decl_tensor_intrin( + conv.op, intrin_func, binds={data_vec: ab_, kernel_vec: bb_, conv: cb_} + ) + + +def _schedule_asm(cfg, s, a_vec, b_vec, mat, output, last): + """schedule implementation""" + is_transpose = 0 if not isinstance(cfg, dict) else cfg["is_transpose"] + attrs = mat.op.attrs + vh_, vw_, vc_ = (attrs["vh_"].value, attrs["vw_"].value, attrs["vc_"].value) + + # axis split and reorder + if len(a_vec.shape) == 3: + ow, oc = s[last].op.axis + oc, vc = s[last].split(oc, vc_) + ow, vw = s[last].split(ow, vw_) + s[last].reorder(ow, oc, vw, vc) + s[last].vectorize(vc) + oh = ow = oc + elif len(a_vec.shape) == 4: + n, oc, ow, vw, vc = s[last].op.axis + oc, vc = s[last].split(oc, vc_) + ow, vw = s[last].split(ow, vw_) + s[last].reorder(n, oc, ow, vw, vc) + elif len(a_vec.shape) == 6: + if is_transpose: + n, oh, ow, oc = s[last].op.axis + else: + n, oc, oh, ow = s[last].op.axis + oc, vc = s[last].split(oc, vc_) + oh, vh = s[last].split(oh, vh_) + ow, vw = s[last].split(ow, vw_) + s[last].reorder(n, oc, oh, ow, vh, vw, vc) + else: + raise ValueError("not support a_vec: " + str(len(a_vec.shape))) + if last != output and isinstance(output.op, tvm.tensor.ComputeOp): + s[output].compute_inline() + + s[mat].compute_at(s[last], ow) + s[mat].vectorize(s[mat].op.axis[-1]) + + # mark parallel + s[last].parallel(oh) + + if len(a_vec.shape) == 3: + om, _, _ = s[a_vec].op.axis + s[a_vec].compute_at(s[last], ow) + s[a_vec].parallel(om) + elif len(a_vec.shape) == 4: + _, om, _, _ = s[a_vec].op.axis + s[a_vec].compute_at(s[last], ow) + s[a_vec].parallel(om) + else: + _, oh, _, _, _, _ = s[a_vec].op.axis + s[a_vec].parallel(oh) + s[a_vec].vectorize(s[a_vec].op.axis[-1]) + s[a_vec].compute_inline() + + oc, _, _ = s[b_vec].op.axis + s[b_vec].parallel(oc) + s[b_vec].vectorize(s[b_vec].op.axis[-1]) + s[b_vec].compute_inline() + return s + + +def _matmul_schedule_asm(cfg, outs): + """schedule_conv2d_nchw schedule implementation""" + s = tvm.create_schedule([x.op for x in outs]) + + def _callback(op): + if "asm_matmul_output" in op.tag: + # schedule conv2d + output = op.output(0) + mat = op.input_tensors[0] + + sidx = 0 + if mat.op.input_tensors[0].name == "attr": + sidx = 1 + a_vec = mat.op.input_tensors[sidx] + b_vec = mat.op.input_tensors[sidx + 1] + + def recurs_inline(a_): + if a_.op.input_tensors: + a1 = a_.op.input_tensors[0] + if a1.shape == a_.shape: + s[a1].compute_inline() + recurs_inline(a1) + + def recurs_inline_(a_): + if isinstance(a_, tvm.tensor.ComputeOp): + if a_.op.input_tensors: + a1 = a_.op.input_tensors[0] + s[a1].compute_inline() + recurs_inline_(a1) + + recurs_inline_(a_vec) + recurs_inline_(b_vec) + + _schedule_asm(cfg, s, a_vec, b_vec, mat, output, outs[0]) + + traverse_inline(s, outs[0].op, _callback) + return s diff --git a/predict/module/tvm_kernel/lite/python/at_ops/__init__.py b/predict/module/tvm_kernel/lite/python/at_ops/__init__.py new file mode 100644 index 00000000000..274ad9a7e51 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/at_ops/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Neural network operators""" +# from .at_lib import * +# from .at_gen import * diff --git a/predict/module/tvm_kernel/lite/python/at_ops/at_gen_strip.py b/predict/module/tvm_kernel/lite/python/at_ops/at_gen_strip.py new file mode 100644 index 00000000000..519740c6fe2 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/at_ops/at_gen_strip.py @@ -0,0 +1,516 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This module is rule to generation tvm operate. you can use it like: +python3 at_gen_strip.py [x86:arm64:arm32] +""" +import os +import sys +import itertools +from functools import partial +from at_ops.at_lib import Deconv, tvm, ConvVar, BatchNorm, Eltwise, Resize, CaffeCrop, CaffePReLU +from at_ops.at_lib import FullConnection, Power, ArgMax, Concat, Pad, Pooling, Mean, MatMul, Softmax +from at_ops.at_lib import Activation, Exp, Split, Cast, ExpandDims, Tile, Range +from at_rt import at_runtime_reset + + +check_correctness = False +ARCH_TYPE = sys.argv[1] + +dtypes = ("float32",) # "float16", "uint8", "int8", "uint32", "int32" + +device_map = { + "x86": "llvm", + "arm64": "llvm -device=arm_cpu -model=kirin970 -target=arm64-linux-android", + "arm32": "llvm -device=arm_cpu -model=kirin970 -target=armv7a-linux-eabi -mfloat-abi=soft", +} + +lib_path_map = { + "x86": "../../../build/lib_x86/", + "arm64": "../../../build/lib_arm64/", + "arm32": "../../../build/lib_arm32/", +} + +best_log_map = { + "x86": None, + "arm64": None, + "arm32": None, +} + +lib_path = lib_path_map[ARCH_TYPE] +device = device_map[ARCH_TYPE] +if ARCH_TYPE == "arm64": + if dtypes[0] == "float16": + device += " -mattr=+fp16fml" + else: + device += " -mattr=+neon" +best_log = best_log_map[ARCH_TYPE] + +kwargs = { + "device": device, + "lib_path": lib_path, + "check_correctness": check_correctness, +} + +use_arm32 = ARCH_TYPE == "arm32" + +MAX_DIMS = 5 +const_op_list = [ + ( + "Deconvolution", + partial(Deconv, optype="Deconvolution"), + { + "ndim": (5,), + "dtype": dtypes, + "kernels": ((2, 2),), + "strides": ((2, 2),), + "pad": ((0, 0, 0, 0),), + "dilations": ((1, 1),), + "hasbias": (False, True), + "activation_type": ("NO_ACTIVATION",), + "cfg": [ + { + "CI": tvm.var("CI"), + "VH": 2, + "VW": 12, + "VC": 4, + "VI": 4, + "tile_oh": 2, + "tile_ow": 12, + "tile_co": 4, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + }, + { + "CI": tvm.var("CI"), + "VH": 2, + "VW": 10, + "VC": 4, + "VI": 4, + "tile_oh": 2, + "tile_ow": 10, + "tile_co": 4, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + }, + { + "CI": tvm.var("CI"), + "VH": 2, + "VW": 16, + "VC": 4, + "VI": 4, + "tile_oh": 2, + "tile_ow": 16, + "tile_co": 4, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + }, + { + "CI": tvm.var("CI"), + "VH": 2, + "VW": 8, + "VC": 4, + "VI": 4, + "tile_oh": 2, + "tile_ow": 8, + "tile_co": 4, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + }, + { + "CI": tvm.var("CI"), + "VH": 2, + "VW": 4, + "VC": 4, + "VI": 4, + "tile_oh": 2, + "tile_ow": 4, + "tile_co": 4, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + }, + { + "CI": tvm.var("CI"), + "VH": 2, + "VW": 2, + "VC": 4, + "VI": 4, + "tile_oh": 2, + "tile_ow": 2, + "tile_co": 4, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + }, + ], + }, + ), + ( + "Convolution", + partial(ConvVar, optype="Convolution"), + { + "ndim": (4,), + "layout": ("NCHW",), + "dtype": dtypes, + "kernels": ((1, 1), (3, 3), (5, 5),), + "strides": ((1, 1), (2, 2)), + "pad": ((1, 1, 1, 1), (0, 0, 0, 0), (2, 2, 2, 2)), + "dilations": ((1, 1),), + "hasbias": (False, True), + "activation_type": ("NO_ACTIVATION", "RELU"), + "cfg": [ + { + "CI": tvm.var("CI"), + "VH": 1, + "VW": 1, + "VC": 1, + "VI": 1, + "tile_oh": 1, + "tile_ow": 1, + "tile_co": 1, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + "core_id": 0, + }, + ], + }, + ), + ( + "ConvolutionDepthwise", + partial(ConvVar, optype="ConvolutionDepthwise"), + { + "ndim": (4,), + "layout": ("NCHW",), + "dtype": dtypes, + "kernels": ((2, 2), (3, 3),), + "strides": ((1, 1),), + "pad": ((0, 0, 0, 0), (0, 1, 0, 1), (1, 0, 1, 0), (1, 1, 1, 1),), + "dilations": ((1, 1),), + "hasbias": (False, True), + "activation_type": ("NO_ACTIVATION", "RELU"), + "channel_multiplier": (1,), + "cfg": [ + { + "CI": tvm.var("CI"), + "VH": 1, + "VW": 1, + "VC": 1, + "VI": 1, + "tile_oh": 1, + "tile_ow": 1, + "tile_co": 1, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + "core_id": 0, + }, + ], + }, + ), + ( + "DeConvolutionDepthwise", + partial(ConvVar, optype="DeConvolutionDepthwise"), + { + "ndim": (4,), + "layout": ("NCHW",), + "dtype": dtypes, + "kernels": ((1, 1), (2, 2), (3, 3),), + "strides": ((1, 1), (2, 2),), + "pad": ((0, 0, 0, 0), (1, 0, 1, 0), (1, 1, 1, 1),), + "dilations": ((1, 1),), + "hasbias": (False, True), + "activation_type": ("NO_ACTIVATION", "RELU"), + "channel_multiplier": (1,), + "cfg": [ + { + "CI": tvm.var("CI"), + "VH": 1, + "VW": 1, + "VC": 1, + "VI": 1, + "tile_oh": 1, + "tile_ow": 1, + "tile_co": 1, + "ann_reduce": ["none", "unroll"], + "ann_spatial": ["unroll", "unroll", "vec"], + "core_id": 0, + }, + ], + }, + ), + ( + "BatchNorm", + BatchNorm, + {"ndim": (4,), "dtype": dtypes, "optype": ("TFBatchNorm",), "axis": (1, 3,)}, + ), + ( + "BiasAdd", + BatchNorm, + {"ndim": (2, 4), "dtype": dtypes, "optype": ("TFBiasAdd",), "axis": (1, 3)}, + ), + ( + "CaffeBatchNorm", + BatchNorm, + {"ndim": (2, 4), "dtype": dtypes, "optype": ("CaffeBatchNorm",), "axis": (1, 3)}, + ), + ( + "Scale", + BatchNorm, + {"ndim": (2, 4), "dtype": dtypes, "optype": ("CaffeScale",), "axis": (1,)}, + ), + ( + "Eltwise", + Eltwise, + { + "ndim_a": tuple(range(0, MAX_DIMS + 1)), + "ndim_b": tuple(range(0, MAX_DIMS + 1)), + "dtype": dtypes, + "mode": ("add", "subtract", "multiply", "divide", "maximum"), + }, + ), + ( + "Add", + Eltwise, + { + "ndim_a": tuple(range(0, MAX_DIMS + 1)), + "ndim_b": tuple(range(0, MAX_DIMS + 1)), + "dtype": dtypes, + "mode": ("add",), + }, + ), + ( + "Sub", + Eltwise, + { + "ndim_a": tuple(range(0, MAX_DIMS + 1)), + "ndim_b": tuple(range(0, MAX_DIMS + 1)), + "dtype": dtypes, + "mode": ("subtract",), + }, + ), + ( + "Mul", + Eltwise, + { + "ndim_a": tuple(range(0, MAX_DIMS + 1)), + "ndim_b": tuple(range(0, MAX_DIMS + 1)), + "dtype": dtypes, + "mode": ("multiply",), + }, + ), + ( + "RealDiv", + Eltwise, + { + "ndim_a": tuple(range(0, MAX_DIMS + 1)), + "ndim_b": tuple(range(0, MAX_DIMS + 1)), + "dtype": dtypes, + "mode": ("divide",), + }, + ), + ( + "Maximum", + Eltwise, + { + "ndim_a": tuple(range(0, MAX_DIMS + 1)), + "ndim_b": tuple(range(0, MAX_DIMS + 1)), + "dtype": dtypes, + "mode": ("maximum",), + }, + ), + ( + "ResizeBilinear", + Resize, + { + "ndim": (4,), + "dtype": dtypes, + "method": ("bilinear",), # "bicubic" + "align_corners": (True, False), + }, + ), + ( + "ResizeNearestNeighbor", + Resize, + { + "ndim": (4,), + "dtype": dtypes, + "method": ("nearest_neighbor",), # "bicubic" + "align_corners": (True, False), + }, + ), + ( + "CaffeCrop", + CaffeCrop, + {"ndim": (4,), "dtype": dtypes, "axis": tuple(range(0, 4))}, + ), + ( + "CaffePReLU", + CaffePReLU, + {"ndim": (2, 4), "dtype": dtypes, "channel_shared": (True, False)}, + ), + ( + "FullConnection", + FullConnection, + {"ndim_a": (2, 4), "dtype": dtypes, "has_bias": (True, False)}, + ), + ("Power", Power, {"ndim": tuple(range(1, MAX_DIMS + 1)), "dtype": dtypes}), + ( + "ArgMax", + ArgMax, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "dtype": dtypes, + "axis": tuple(range(0, MAX_DIMS)), # not support None + "keep_dims": (True, False), + "top_k": (1,), + "out_dtype": dtypes, + }, + ), + ( + "Concat", + Concat, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "dtype": dtypes, + "input_num": tuple(range(2, 6 + 1)), + "axis": tuple(range(0, MAX_DIMS)), + }, + ), + ( + "Pad", + Pad, + { + "ndim": tuple(range(2, MAX_DIMS + 1)), + "dtype": dtypes, + "paddingmode": ("CONSTANT", "REFLECT", "SYMMETRIC"), + }, + ), + ( + "Pooling", + Pooling, + { + "ndim": (4,), + "dtype": dtypes, + "pooling_mode": ("max", "avg"), + "caffe_mode": (True, False), + "kernel": ((1, 1), (2, 2), (3, 3), (5, 5)), + "stride": ((1, 1), (2, 2), (3, 3)), + "pad": ((0, 0, 0, 0), (0, 1, 0, 1), (1, 1, 1, 1)), + "use_global": (True, False), + }, + ), + ( + "Mean", + Mean, + { + "ndim": (4,), + "dtype": dtypes, + "axis": ( + (0,), + (1,), + (2,), + (3,), + (0, 1), + (0, 2), + (0, 3), + (1, 2), + (1, 3), + (2, 3), + (0, 1, 2), + (0, 1, 3), + (0, 2, 3), + (1, 2, 3), + (0, 1, 2, 3), + ), + "keep_dims": (True, False), + }, + ), + ( + "MatMul", + MatMul, + { + "ndim_a": (2,), + "ndim_b": (2,), + "dtype": dtypes, + "transpose_a": (True, False), + "transpose_b": (True, False), + }, + ), + ( + "Softmax", + Softmax, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "dtype": dtypes, + "axis": tuple(range(0, MAX_DIMS)), + }, + ), + ( + "Activation", + Activation, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "dtype": dtypes, + "optype": ("NO_ACTIVATION", "RELU", "RELU6", "SIGMOID"), + }, + ), + ("Exp", Exp, {"ndim": tuple(range(1, MAX_DIMS + 1)), "dtype": dtypes}), + ( + "Split", + Split, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "dtype": dtypes, + "output_num": tuple(range(1, 5)), + "axis": tuple(range(0, MAX_DIMS)), + }, + ), + ( + "Cast", + Cast, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "src_dtype": dtypes, + "dst_dtype": dtypes, + }, + ), + ( + "ExpandDims", + ExpandDims, + { + "ndim": tuple(range(1, MAX_DIMS + 1)), + "dtype": dtypes, + "axis": tuple(range(0, MAX_DIMS)), + }, + ), + ("Tile", Tile, {"ndim": tuple(range(1, MAX_DIMS + 1)), "dtype": dtypes}), + ("Range", Range, {"out_dtype": ("float32", "uint32", "int32")}), +] + + +def gen_const_libs(some_op=None): + for optype, func, attr in const_op_list: + if some_op and some_op != optype: + continue + for values in itertools.product(*attr.values()): + args = dict((k, v) for k, v in zip(attr.keys(), values)) + func(device=device, lib_path=lib_path, **args) + + +if __name__ == "__main__": + if not os.path.exists(lib_path): + os.makedirs(lib_path) + # skip best_history log: + with tvm.target.create(device): + with at_runtime_reset.AtRuntimeReset(): + gen_const_libs() diff --git a/predict/module/tvm_kernel/lite/python/at_ops/at_lib.py b/predict/module/tvm_kernel/lite/python/at_ops/at_lib.py new file mode 100644 index 00000000000..655064b29e1 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/at_ops/at_lib.py @@ -0,0 +1,1193 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This module is rule to generation tvm operate, call by at_gen_strip.py +""" +import numpy as np +import tvm +import topi +from topi.image import resize +from topi.nn import mirror_pad +from topi import tag +import topi.testing + +from arm_cpu.deconv import _conv_spatial_pack_deconv, schedule_conv2d_nchw_arm_cpu_deconv +from arm_cpu.conv2d import _conv_spatial_pack_asm, schedule_conv2d_nchw_arm_cpu +from arm_cpu.matmul import _matmul_spatial_pack_asm, _matmul_schedule_asm +from arm_cpu.depthwise_conv2d import _depthwise_spatial_pack, schedule_depthwise_conv2d_nchw_arm +from config_tool import activation_enum_map + +map_conv = { + 'Convolution': "Conv2D", + 'ConvolutionDepthwise': "DepthwiseConv2D", + 'Deconvolution': "DeConv2D", + 'DeConvolutionDepthwise': "DeDepthwiseConv2D", +} + + +def Genlib(sch, tensor_list, device, opname, lib_path, print_lower=False): + if print_lower: + print(tvm.lower(sch, tensor_list, simple_mode=True)) + ctx = tvm.context(device, 0) + func_o = tvm.build(sch, tensor_list, device + " --system-lib", name=opname) + func_so = tvm.build(sch, tensor_list, device, name=opname) + func_o.save(lib_path + opname + ".o", "o") + return func_o, func_so, ctx + + +def AsType(as_input, dtype): + if as_input.dtype == dtype: + return as_input + return tvm.compute(as_input.shape, + lambda *i: as_input(*i).astype(dtype), + tag="injective") + + +@tvm.tag_scope(tag=tag.ELEMWISE) +def TopiNNrelu6(x): + return tvm.compute(x.shape, lambda *i: tvm.min(tvm.max(x(*i), tvm.const(0, x.dtype)), tvm.const(6, x.dtype))) + + +def TopiActivation(in_tensor, a_type, memcpy=False): + ''' + activativation + Args: + in_tensor: + a_type: + memcpy: + + Returns: + ''' + if a_type == 'NO_ACTIVATION': + if memcpy: + return tvm.compute(in_tensor.shape, lambda *i: in_tensor[i], tag=tag.ELEMWISE) + return in_tensor + if a_type == 'RELU': + return topi.nn.relu(in_tensor) + if a_type == 'RELU6': + return TopiNNrelu6(in_tensor) + if a_type == 'SIGMOID': + if in_tensor.dtype in ["uint8", "int8", "uint32", "int32"]: + a_fp32 = AsType(in_tensor, 'float32') + out_tensor = topi.sigmoid(a_fp32) + return AsType(out_tensor, in_tensor.dtype) + return topi.sigmoid(in_tensor) + raise ValueError("not support activation type" + a_type) + + +def Deconv(device="llvm", lib_path="./", optype=None, + ndim=None, dtype=None, kernels=None, + strides=None, pad=None, dilations=None, + hasbias=None, activation_type=None, + config_entity=None, impl_dtype=None, + use_arm32=False, cfg=None): + ''' + Deconvolution + Args: + device: + lib_path: + optype: + ndim: + dtype: + kernels: + strides: + pad: + dilations: + hasbias: + activationType: + configEntity: + impl_dtype: + use_arm32: + cfg: + + Returns: + ''' + if cfg is None: + cfg = {'CI': tvm.var('ci'), 'VH': 2, 'VW': 2, 'VC': 4, 'VI': 4, + 'tile_oh': 2, 'tile_ow': 2, 'tile_co': 4, + 'ann_reduce': ['none', 'none'], + "ann_spatial": ['none', 'none', 'none'] + } + has_bias = hasbias + batch = tvm.var("batch") + in_channel = tvm.var("in_channel") + in_height, in_width = tvm.var("in_height"), tvm.var("in_width") + kh, kw = kernels + ow = cfg['VW'] + oh = cfg['VH'] + oc = cfg['VC'] + op_name = "%s_ndim%d_%s_k%d_s%d_p%d%d%d%d_d%d_act%d_vc%d_vh%d_vw%d_hasbias%d" % (\ + map_conv[optype], ndim, dtype,\ + kh, strides[0], pad[0], pad[1], pad[2], pad[3], dilations[0],\ + activation_enum_map[activation_type], oc, oh, ow, hasbias) + opname = op_name + print("DEconv", opname, config_entity) + + if impl_dtype is None: + impl_dtype = dtype + + out_channel = tvm.var("out_channel") + + # define placeholder + input_tensor = in_tensor = tvm.placeholder((batch, in_channel, in_height, in_width, 4), \ + dtype=dtype, name='in_tensor') + temp_tensor = kernel_tensor = tvm.placeholder((in_channel*4, out_channel, kh, kw), dtype=dtype, \ + name='kernel_tensor') + if has_bias: + bias = tvm.placeholder((out_channel,), dtype=dtype, name='bias') + bias1 = topi.reshape(bias, (out_channel, 1, 1)) + + if impl_dtype != dtype: + input_tensor = AsType(input_tensor, impl_dtype) + temp_tensor = AsType(temp_tensor, impl_dtype) + if has_bias: + bias1 = AsType(bias1, impl_dtype) + + # define compute & schedule + cfg1 = (True, 1, 1, 1) if cfg is None else (True, cfg["tile_oh"], cfg["tile_ow"], cfg["tile_co"]) + out_tensor = _conv_spatial_pack_deconv(cfg1, input_tensor, temp_tensor, out_dtype=impl_dtype) + + if has_bias: + out_tensor = tvm.compute(out_tensor.shape, lambda n, co, h, w, c4: \ + out_tensor[n, co, h, w, c4] + bias1[co*4 + c4][0][0], tag="injective") + out_tensor = TopiActivation(out_tensor, activation_type) + if impl_dtype != dtype: + out_tensor = AsType(out_tensor, dtype) + + # create schedule + if use_arm32: + s = tvm.create_schedule(out_tensor.op) + else: + s = schedule_conv2d_nchw_arm_cpu_deconv(cfg, [out_tensor]) + + attr = [batch, in_channel, in_height, in_width, out_channel, in_tensor, kernel_tensor] + if has_bias: attr.append(bias) + attr.append(out_tensor) + tensor_list = attr + + Genlib(s, tensor_list, device, opname, lib_path) + + +def ConvVar(device="llvm", lib_path="./", optype=None,\ + ndim=None, layout=None, dtype=None, kernels=None,\ + strides=None, pad=None, dilations=None,\ + hasbias=None, activation_type=None,\ + config_entity=None, impl_dtype=None, channel_multiplier=None,\ + use_arm32=False, cfg=None): + ''' + convolution + Args: + device: + lib_path: + optype: + ndim: + layout: + dtype: + kernels: + strides: + pad: + dilations: + hasbias: + activationType: + configEntity: + impl_dtype: + channel_multiplier: + use_arm32: + cfg: + + Returns: + ''' + use_depthwise = optype == 'ConvolutionDepthwise' + use_deconv = optype == 'Deconvolution' + use_deconv_depthwise = optype == 'DeConvolutionDepthwise' + has_bias = hasbias + + ow = 1 if cfg is None else cfg['VW'] + oh = 1 if cfg is None else cfg['VH'] + oc = 1 if cfg is None else cfg['VC'] + kh, kw = kernels + op_name = "%s_ndim%d_%s_k%d_s%d_p%d%d%d%d_d%d_act%d_vc%d_vh%d_vw%d_hasbias%d" % ( \ + map_conv[optype], ndim, dtype, \ + kh, strides[0], pad[0], pad[1], pad[2], pad[3], dilations[0], \ + activation_enum_map[activation_type], oc, oh, ow, hasbias) + batch = tvm.var("batch") + in_channel = tvm.var("in_channel") + in_height, in_width = tvm.var("in_height"), tvm.var("in_width") + pad_up, pad_down, pad_left, pad_right = pad + opname = op_name + + print("Conv", opname, config_entity) + + if impl_dtype is None: + impl_dtype = dtype + + if use_depthwise: + multiplier = channel_multiplier + out_channel = in_channel * multiplier + elif use_deconv_depthwise: + multiplier = channel_multiplier + out_channel = in_channel * multiplier + else: + out_channel = tvm.var("out_channel") + + # define placeholder + input_tensor = in_tensor = tvm.placeholder((batch, in_channel, in_height, in_width), dtype=dtype, name='in_tensor') + + if use_depthwise: + temp_tensor = kernel_tensor = tvm.placeholder((in_channel, multiplier, kh, kw), dtype=dtype,\ + name='kernel_tensor') + elif use_deconv: + temp_tensor = kernel_tensor = tvm.placeholder((in_channel, out_channel, kh, kw), dtype=dtype,\ + name='kernel_tensor') + elif use_deconv_depthwise: + temp_tensor = kernel_tensor = tvm.placeholder((in_channel, multiplier, kh, kw), dtype=dtype,\ + name='kernel_tensor') + else: + temp_tensor = kernel_tensor = tvm.placeholder((out_channel, in_channel, kh, kw), dtype=dtype,\ + name='kernel_tensor') + if has_bias: + bias = tvm.placeholder((out_channel,), dtype=dtype, name='bias') + bias1 = topi.reshape(bias, (out_channel, 1, 1)) + + if impl_dtype != dtype: + input_tensor = AsType(input_tensor, impl_dtype) + temp_tensor = AsType(temp_tensor, impl_dtype) + if has_bias: + bias1 = AsType(bias1, impl_dtype) + + # define compute & schedule + if pad_up != pad_down or pad_left != pad_right: + input_tensor = topi.nn.pad(input_tensor, [0, 0, pad_up, pad_left], [0, 0, pad_down, pad_right], name='data_pad') + padding = 0, 0 + else: + padding = pad_up, pad_left + if use_depthwise: + cfg1 = (True, 1, 1, 1) if cfg is None else (True, cfg["tile_oh"], cfg["tile_ow"], cfg["tile_co"]) + out_tensor = _depthwise_spatial_pack(cfg1, input_tensor, temp_tensor, strides, padding, dilations,\ + out_dtype=impl_dtype) + elif use_deconv: + + def GetInput(input_tensor, temp_tensor, padding): + _, out_c, filter_h, filter_w = temp_tensor.shape + if out_c is None: + print("temp_tensor.shape err") + stride_h, stride_w = strides + # dilate stage + dilated_input = topi.nn.dilate(input_tensor, [1, 1, stride_h, stride_w], + name='DilatedInput') + # padding stage + fpad_top, fpad_left, fpad_bottom, fpad_right = topi.nn.get_pad_tuple(padding, ( + filter_h, filter_w)) + bpad_top = filter_h - 1 - fpad_top + bpad_bottom = filter_h - 1 - fpad_bottom + bpad_left = filter_w - 1 - fpad_left + bpad_right = filter_w - 1 - fpad_right + padded_input = topi.nn.pad(dilated_input, \ + [0, 0, bpad_top, bpad_left], \ + [0, 0, bpad_bottom, bpad_right], \ + name='PaddedInput') + return padded_input + + special_deconv = kh == 2 and kw == 2 and strides[0] == 2 and strides[1] == 2 + # special_deconv = False + if special_deconv: + out_tensor = OptimalOut(input_tensor, temp_tensor, in_channel) + else: + out_tensor = BaseImplementation(input_tensor, temp_tensor, GetInput, layout, padding) + elif use_deconv_depthwise: + def GetInput(input_tensor, temp_tensor, padding): + _, out_c, filter_h, filter_w = temp_tensor.shape + if out_c is None: + print("temp_tensor.shape err") + stride_h, stride_w = strides + # dilate stage + dilated_input = topi.nn.dilate(input_tensor, [1, 1, stride_h, stride_w], + name='DilatedInput') + # padding stage + fpad_top, fpad_left, fpad_bottom, fpad_right = topi.nn.get_pad_tuple(padding, ( + filter_h, filter_w)) + bpad_top = filter_h - 1 - fpad_top + bpad_bottom = filter_h - 1 - fpad_bottom + bpad_left = filter_w - 1 - fpad_left + bpad_right = filter_w - 1 - fpad_right + padded_input = topi.nn.pad(dilated_input, \ + [0, 0, bpad_top, bpad_left], \ + [0, 0, bpad_bottom, bpad_right], \ + name='PaddedInput') + return padded_input + + temp_tensor = topi.flip(temp_tensor, axis=-1) + temp_tensor = topi.flip(temp_tensor, axis=-2) + out_tensor = topi.nn.depthwise_conv2d_nchw(GetInput(input_tensor, temp_tensor, padding), temp_tensor, (1, 1), \ + padding, (1, 1), out_dtype=input_tensor.dtype) + else: + cfg1 = (True, 1, 1, 1) if cfg is None else (True, cfg["tile_oh"], cfg["tile_ow"], cfg["tile_co"]) + out_tensor = _conv_spatial_pack_asm(cfg1, input_tensor, temp_tensor, strides, padding, dilations,\ + out_dtype=impl_dtype) + + if has_bias: + out_tensor = tvm.compute(out_tensor.shape, lambda n, co, h, w: out_tensor[n, co, h, w] + bias1[co][0][0],\ + tag="injective") + out_tensor = TopiActivation(out_tensor, activation_type) + if impl_dtype != dtype: + out_tensor = AsType(out_tensor, dtype) + + # create schedule + if use_arm32: + s = tvm.create_schedule(out_tensor.op) + elif use_depthwise: + s = schedule_depthwise_conv2d_nchw_arm(cfg, [out_tensor]) + elif use_deconv: + if special_deconv: + s = tvm.create_schedule([out_tensor.op]) + else: + s = topi.generic.schedule_conv2d_nchw([out_tensor]) + elif use_deconv_depthwise: + s = tvm.create_schedule([out_tensor.op]) + else: + s = schedule_conv2d_nchw_arm_cpu([out_tensor]) + + # generate lib + attr = [batch, in_channel, in_height, in_width, out_channel, in_tensor, kernel_tensor] + tensor_list = [*attr, bias, out_tensor] if has_bias else [*attr, out_tensor] + Genlib(s, tensor_list, device, opname, lib_path) + + +def BaseImplementation(input_tensor, temp_tensor, get_input, layout, padding): + temp_tensor = topi.flip(temp_tensor, axis=-1) + temp_tensor = topi.flip(temp_tensor, axis=-2) + temp_tensor = topi.transpose(temp_tensor, axes=(1, 0, 2, 3)) + out_tensor = topi.nn.conv2d(get_input(input_tensor, temp_tensor, padding), temp_tensor, (1, 1), padding, (1, 1), + layout=layout, out_dtype=input_tensor.dtype) + return out_tensor + + +def OptimalOut(input_tensor, temp_tensor, in_channel): + ''' + deconv compute + Args: + input_tensor: + temp_tensor: + in_channel: + + Returns: + ''' + temp_tensor = topi.transpose(temp_tensor, axes=(1, 0, 2, 3)) + out_shape = [] + for i in range(len(input_tensor.shape)): + if i == 0: + out_shape.append(input_tensor.shape[i]) + continue + if i == 1: + out_shape.append(temp_tensor.shape[0]) + continue + out_shape.append(2 * input_tensor.shape[i]) + rc = tvm.reduce_axis((0, in_channel), name='rc') + return tvm.compute(out_shape, lambda i, j, k, l:\ + tvm.sum(input_tensor[i, rc, k // 2, l // 2].astype(input_tensor.dtype) *\ + temp_tensor[j, rc, k % 2, l % 2].astype(input_tensor.dtype), axis=[rc])) + + +def Concat(device="llvm", lib_path="./", + ndim=None, dtype=None, input_num=None, axis=None): + ''' + concat + Args: + device: + lib_path: + all_tensors: + ndim: + dtype: + input_num: + axis: + + Returns: + ''' + if axis >= ndim: + return + shapes = [] + for i in range(input_num): + shape = [] + for j in range(ndim): + if j == axis: + shape.append(tvm.var("axis" + str(i))) + else: + shape.append(tvm.var("n" + str(j))) + shapes.append(shape) + in_tensor = [tvm.placeholder(shape, dtype=dtype, name='in_tensor%d' % i) for i, shape in enumerate(shapes)] + opname = "Concat_ndim%d_%s_input_num%d_axis%d" % (ndim, dtype, input_num, axis) + print(opname) + + # define compute + out_tensor = topi.concatenate(tuple(in_tensor), axis) + tensor_list = in_tensor + [out_tensor] + if ndim < 5: + s = topi.generic.schedule_concatenate(out_tensor) + else: + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Activation(device="llvm", lib_path="./", + ndim=None, dtype=None, optype=None): + ''' + activation + Args: + device: + lib_path: + ndim: + dtype: + optype: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + opname = "Activation_ndim%d_%s_%s" % (ndim, dtype, optype) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + out_tensor = TopiActivation(in_tensor, optype, memcpy=True) + tensor_list = [in_tensor, out_tensor] + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def BatchNorm(device="llvm", lib_path="./", + ndim=None, dtype=None, optype=False, axis=None): + ''' + batchnorm + Args: + device: + lib_path: + ndim: + dtype: + optype: + axis: + + Returns: + ''' + if axis >= ndim: + return + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + channel = shape[axis] + eps = tvm.var("epsilon", dtype="float32") + opname = optype + ("_ndim%d_%s_axis%d" % (ndim, dtype, axis)) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + mean = tvm.placeholder((channel,), dtype=dtype, name='mean') + variance = tvm.placeholder((channel,), dtype=dtype, name='var') + scale = tvm.placeholder((channel,), dtype=dtype, name='scale') + offset = tvm.placeholder((channel,), dtype=dtype, name='offset') + + variance_sqrt = tvm.compute((channel,), lambda i: tvm.sqrt(variance[i] + eps.astype(dtype))) + if optype == "TFBatchNorm": + out_tensor = tvm.compute(shape, lambda *idx: ((in_tensor[idx] - mean[idx[axis]]) / variance_sqrt[idx[axis]]) *\ + scale[idx[axis]] + offset[idx[axis]]) + tensor_list = [eps, in_tensor, scale, offset, mean, variance, out_tensor] + elif optype == "CaffeBatchNorm": + out_tensor = tvm.compute(shape, lambda *idx: (in_tensor[idx] - mean[idx[axis]]) / variance_sqrt[idx[axis]]) + tensor_list = [eps, in_tensor, mean, variance, out_tensor] + elif optype == "CaffeScale": + out_tensor = tvm.compute(shape, lambda *idx: in_tensor[idx] * scale[idx[axis]] + offset[idx[axis]]) + tensor_list = [in_tensor, scale, offset, out_tensor] + elif optype == "TFBiasAdd": + out_tensor = tvm.compute(shape, lambda *idx: in_tensor[idx] + offset[idx[axis]]) + tensor_list = [in_tensor, offset, out_tensor] + else: + raise RuntimeError("no support for {}".format(optype)) + + # define schedule & generate lib + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Pooling(device="llvm", lib_path="./", + ndim=None, dtype=None, pooling_mode=None, kernel=None, stride=None, pad=None, caffe_mode=None, + use_global=False): + ''' + pooling + Args: + device: + lib_path: + ndim: + dtype: + pooling_mode: + kernel: + stride: + pad: + caffe_mode: + use_global: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(0, ndim)] + layout = 'NCHW' + if use_global: + opname = "GlobalPooling_ndim%d_%s_%s" % (ndim, dtype, pooling_mode) + else: + kernel_h, kernel_w = kernel + stride_h, stride_w = stride + pad_up, pad_down, pad_left, pad_right = pad + if pad_up == 0 and pad_down == 0 and pad_left == 0 and pad_right == 0 and caffe_mode: + caffe_mode = False + opname = "Pooling_ndim%d_%s_%s_kernel%d%d_stride%d%d_pad%d%d%d%d%s" \ + % (ndim, dtype, pooling_mode, kernel_h, kernel_w, stride_h, stride_w, + pad_up, pad_down, pad_left, pad_right, "_caffe" if caffe_mode else "") + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + if use_global: + out_tensor = topi.nn.global_pool(in_tensor, pool_type=pooling_mode, layout=layout) + sch = topi.generic.schedule_adaptive_pool(out_tensor) + else: + out_tensor = topi.nn.pool(in_tensor, + kernel=(kernel_h, kernel_w), + stride=(stride_h, stride_w), + padding=(pad_up, pad_left, pad_down, pad_right), + pool_type=pooling_mode, + ceil_mode=False, + layout=layout, + count_include_pad=False) + sch = topi.generic.schedule_pool(out_tensor, layout) + tensor_list = [in_tensor, out_tensor] + Genlib(sch, tensor_list, device, opname, lib_path, print_lower=False) + + +def Eltwise(device="llvm", lib_path="./", + ndim_a=None, ndim_b=None, dtype=None, mode=None): + ''' + eltwise + Args: + device: + lib_path: + ndim_a: + ndim_b: + dtype: + mode: + + Returns: + ''' + ndim_max = max(ndim_a, ndim_b) + shape = [tvm.var("n" + str(i)) for i in range(ndim_max)] + shape_b1 = [dim if i == 1 else 1 for i, dim in enumerate(shape)] + shape_a = shape[ndim_max - ndim_a:] if ndim_a else (1,) + shape_b = shape[ndim_max - ndim_b:] if ndim_b == ndim_a else shape_b1 if ndim_b == 1 else (1,) + opname = "Eltwise_%s_ndimA%d_ndimB%d_%s" % (mode, ndim_a, ndim_b, dtype) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape_a, dtype=dtype, name='in_tensor') + b_tensor = tvm.placeholder(shape_b, dtype=dtype, name='b_tensor') + + topi_funs = { + 'add': topi.add, + 'subtract': topi.subtract, + 'multiply': topi.multiply, + 'divide': topi.divide, + 'maximum': topi.maximum, + 'minimum': topi.minimum, + } + + out_tensor = topi_funs[mode](in_tensor, b_tensor) + tensor_list = [in_tensor, b_tensor, out_tensor] + s = topi.generic.schedule_elemwise(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Softmax(device="llvm", lib_path="./", + ndim=None, dtype=None, axis=None): + ''' + softmax + Args: + device: + lib_path: + ndim: + dtype: + axis: + + Returns: + ''' + if axis >= ndim: + return + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + opname = "Softmax_ndim%d_%s_axis%s" % (ndim, dtype, axis) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + out_tensor = topi.nn.softmax(in_tensor, axis) + tensor_list = [in_tensor, out_tensor] + s = topi.generic.schedule_elemwise(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Resize(device="llvm", lib_path="./", + ndim=None, dtype=None, method=None, align_corners=None): + ''' + resize + Args: + device: + lib_path: + ndim: + dtype: + method: + align_corners: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + new_height = tvm.var("newHeight") + new_width = tvm.var("new_width") + opname = "Resize_ndim%d_%s_%s_%s" % (ndim, dtype, method, "Align" if align_corners else "NotAlign") + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + out_tensor = resize(in_tensor, [new_height, new_width], align_corners=align_corners, method=method) + tensor_list = [new_height, new_width, in_tensor, out_tensor] + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Mean(device="llvm", lib_path="./", + ndim=None, dtype=None, axis=None, keep_dims=None): + ''' + mean + Args: + device: + lib_path: + ndim: + dtype: + axis: + keepDims: + + Returns: + ''' + if axis[-1] >= ndim: + return + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + axis_str = "" + for dim in axis: + axis_str += str(dim) + opname = "Mean_ndim%d_%s_axis%s_%s" % (ndim, dtype, axis_str, "keepDims" if keep_dims else "notkeepDims") + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + c_shape = shape[:] + reduced_num = 1 + for dim in axis: + c_shape[dim] = 1 + reduced_num *= shape[dim] + + def _ComputeSum(*b_idx): + reduce_axis = [tvm.reduce_axis((0, shape[dim])) for dim in axis] + a_idx = list(b_idx) + for i, dim in enumerate(axis): + a_idx[dim] = reduce_axis[i] + a_idx = tuple(a_idx) + return tvm.sum(in_tensor[a_idx], axis=reduce_axis) + + out_tensor = tvm.compute(c_shape, _ComputeSum) + out_tensor = tvm.compute(c_shape, lambda *i: out_tensor(*i) / reduced_num) + if not keep_dims: + out_tensor = topi.squeeze(out_tensor, axis) + + # define schedule & generate lib + tensor_list = [in_tensor, out_tensor] + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def CaffeCrop(device="llvm", lib_path="./", + ndim=None, dtype=None, axis=None): + ''' + caffe crop op + Args: + device: + lib_path: + ndim: + dtype: + axis: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(axis)] + shape_a = shape[:] + shape_b = shape[:] + offsets = [] + for i in range(axis, ndim): + shape_a.append(tvm.var("nA" + str(i))) + shape_b.append(tvm.var("nB" + str(i))) + offsets.append(tvm.var("offset" + str(i))) + opname = "CaffeCrop_ndim%d_%s_axis%d" % (ndim, dtype, axis) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape_a, dtype=dtype, name='in_tensor') + b_tensor = tvm.placeholder(shape_b, dtype=dtype, name='b_tensor') + begin = [0] * axis + offsets + end = shape_a[:] + for i in range(axis, len(shape_a)): + end[i] = offsets[i - axis] + shape_b[i] + shape_c = [end[i] - begin[i] for i in range(ndim)] + + def _Compute(*C_idx): + a_idx = [idx + begin[i] for i, idx in enumerate(list(C_idx))] + a_idx = tuple(a_idx) + return in_tensor[a_idx] + + out_tensor = tvm.compute(shape_c, _Compute) + tensor_list = offsets + [in_tensor, b_tensor, out_tensor] + + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def FullConnection(device="llvm", lib_path="./", + ndim_a=None, dtype=None, has_bias=None): + ''' + full connection + Args: + device: + lib_path: + ndim_a: + dtype: + hasBias: + + Returns: + ''' + n_dim, ci, h_dim, kernel_tensor = (tvm.var("n_dim"), tvm.var("out_tensor"), tvm.var("h_dim"), \ + tvm.var("kernel_tensor")) + co = tvm.var("co") + if ndim_a == 4: + shape_a = (n_dim, ci, h_dim, kernel_tensor) + chw = ci * h_dim * kernel_tensor + else: + shape_a = (n_dim, ci) + chw = ci + shape_w = (co, chw) + opname = "FullConnection_ndimA%d_%s_%s" % (ndim_a, dtype, "hasBias" if has_bias else "notHasBias") + is_var = True + vh, vw, vc = 1, 1, 1 + print(opname) + + in_tensor = tvm.placeholder(shape_a, dtype=dtype, name='in_tensor') + kernel_tensor = tvm.placeholder(shape_w, dtype=dtype, name='kernel_tensor') + input_tensor = topi.reshape(in_tensor, (n_dim, chw)) if len(shape_a) == 4 else in_tensor + + out_tensor = _matmul_spatial_pack_asm((is_var, 0, ci, vh, vw, vc), input_tensor, kernel_tensor, \ + layout='NC', out_dtype=dtype) + if has_bias: + bias = tvm.placeholder((co,), dtype=dtype, name='bias') + out_tensor = tvm.compute((n_dim, co), lambda n, co: out_tensor[n, co] + bias[co], tag='injective') + + tensor_list = [in_tensor, kernel_tensor, bias, out_tensor] if has_bias else [in_tensor, kernel_tensor, out_tensor] + cfg = {'is_var': is_var, 'is_transpose': 0, 'core_id': 0, 'CI': ci, 'VH': vh, 'VW': vw, 'VC': vc} + s = _matmul_schedule_asm(cfg, [out_tensor]) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Power(device="llvm", lib_path="./", + ndim=None, dtype=None): + ''' + power + Args: + device: + lib_path: + ndim: + dtype: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + power = tvm.var("power", dtype="float32") + scale = tvm.var("scale", dtype="float32") + shift = tvm.var("shift", dtype="float32") + opname = "Power_ndim%d_%s" % (ndim, dtype) + print(opname) + + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + out_tensor = tvm.compute(shape, lambda *i: tvm.power(in_tensor[i] * scale.astype(in_tensor.dtype) + \ + shift.astype(in_tensor.dtype), \ + power.astype(in_tensor.dtype))) + tensor_list = [power, scale, shift, in_tensor, out_tensor] + + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def CaffePReLU(device="llvm", lib_path="./", + ndim=None, dtype=None, channel_shared=None): + ''' + caffe prelu + Args: + device: + lib_path: + ndim: + dtype: + channel_shared: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + channel = 1 if channel_shared else shape[1] + opname = "CaffePReLU_ndim%d_%s_%s" % (ndim, dtype, + "channelShared" if channel_shared else "channelNotShared") + print(opname) + + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + slope = tvm.placeholder((channel,), dtype=dtype, name='slope') + if channel_shared: + out_tensor = tvm.compute(shape, lambda *idx: tvm.if_then_else(in_tensor[idx] >= 0, in_tensor[idx],\ + in_tensor[idx] * slope[0])) + else: + out_tensor = tvm.compute(shape, lambda *idx: tvm.if_then_else(in_tensor[idx] >= 0, in_tensor[idx],\ + in_tensor[idx] * slope[idx[1]])) + + tensor_list = [in_tensor, slope, out_tensor] + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Pad(device="llvm", lib_path="./", + ndim=None, dtype=None, paddingmode=None): + ''' + pad + Args: + device: + lib_path: + ndim: + dtype: + paddingmode: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + pad_before = [tvm.var("pad_before" + str(i)) for i in range(ndim)] + pad_after = [tvm.var("pad_after" + str(i)) for i in range(ndim)] + pad_before_const = [0, 0] + pad_before[2:] + pad_after_const = [0, 0] + pad_after[2:] + paddings = [None] * 2 * len(shape) + paddings[0:: 2] = pad_before + paddings[1:: 2] = pad_after + pad_value = 0 + opname = "Pad_ndim%d_%s_%s" % (ndim, dtype, paddingmode) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + if paddingmode == "CONSTANT": + out_tensor = topi.nn.pad(in_tensor, pad_before_const, pad_after_const, pad_value=pad_value, name='out_tensor') + else: + out_tensor = mirror_pad(in_tensor, pad_before_const, pad_after_const, mode=paddingmode, name='out_tensor') + tensor_list = paddings + [in_tensor, out_tensor] + def SchedulePad(inputs): + s = tvm.create_schedule(inputs.op) + if s[inputs].op.axis: + s[inputs].parallel(s[inputs].op.axis[1]) + return s + + s = SchedulePad(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def MatMul(device="llvm", lib_path="./", + ndim_a=None, ndim_b=None, dtype=None, transpose_a=None, transpose_b=None): + ''' + matmul + Args: + device: + lib_path: + ndim_a: + ndim_b: + dtype: + transpose_a: + transpose_b: + + Returns: + ''' + m, k, n_dim = tvm.var("m"), tvm.var("k"), tvm.var("n_dim") + a_shape = (m, k) if not transpose_a else (k, m) + b_shape = (k, n_dim) if not transpose_b else (n_dim, k) + opname = "MatMul_ndimA%d_ndimB%d_%s_%d_%d" % (ndim_a, ndim_b, dtype, transpose_a, transpose_b) + print(opname) + + # define compute + in_tensor = tvm.placeholder(a_shape, dtype=dtype, name='in_tensor') + b_tensor = tvm.placeholder(b_shape, dtype=dtype, name='b_tensor') + out_tensor = topi.matmul(in_tensor, b_tensor, transpose_a, transpose_b) + tensor_list = [in_tensor, b_tensor, out_tensor] + s = topi.generic.schedule_elemwise(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Stack(device="llvm", lib_path="./", + ndim=None, dtype=None, input_num=None, axis=None): + ''' + stack + Args: + device: + lib_path: + ndim: + dtype: + input_num: + axis: + + Returns: + ''' + if axis > ndim: + return + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + shapes = [shape] * input_num + in_tensor = [tvm.placeholder(shape, dtype=dtype, name='in_tensor%d' % i) for i, shape in enumerate(shapes)] + opname = "Stack_ndim%d_%s_input_num%d_axis%d" % (ndim, dtype, input_num, axis) + print(opname) + + input_tensor = [topi.expand_dims(ai, axis) for ai in in_tensor] + out_tensor = topi.concatenate(tuple(input_tensor), axis=axis) + tensor_list = in_tensor + [out_tensor] + if ndim < 4: + s = topi.generic.schedule_concatenate(out_tensor) + else: + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def ArgMax(device="llvm", lib_path="./", + ndim=None, dtype=None, axis=None, keep_dims=None, top_k=None, + out_dtype=None): + ''' + argmax + Args: + device: + lib_path: + ndim: + dtype: + axis: + keepDims: + top_k: + out_dtype: + + Returns: + ''' + if axis >= ndim: + return + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + opname = "ArgMax_ndim%d_%s_axis%d_%s_top%d_%s" \ + % (ndim, dtype, axis, "keepDims" if keep_dims else "notKeepDims", top_k, out_dtype) + print(opname) + + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + out_tensor = topi.argmax(in_tensor, axis=axis, keepdims=keep_dims) + out_tensor = AsType(out_tensor, out_dtype) + tensor_list = [in_tensor, out_tensor] + s = tvm.create_schedule(out_tensor.op) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Exp(device="llvm", lib_path="./", + ndim=None, dtype=None): + ''' + exp + Args: + device: + lib_path: + ndim: + dtype: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + opname = "Exp_ndim%d_%s" % (ndim, dtype) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + if 'int' in dtype: + input_tensor = AsType(in_tensor, 'float32') + out_tensor = topi.exp(input_tensor) + out_tensor = AsType(out_tensor, in_tensor.dtype) + else: + out_tensor = topi.exp(in_tensor) + tensor_list = [in_tensor, out_tensor] + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Cast(device="llvm", lib_path="./", + ndim=None, src_dtype=None, dst_dtype=None): + ''' + cast + Args: + device: + lib_path: + ndim: + src_dtype: + dst_dtype: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + opname = "Cast_ndim%d_%s_%s" % (ndim, src_dtype, dst_dtype) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=src_dtype, name='in_tensor') + out_tensor = topi.cast(in_tensor, dst_dtype) + tensor_list = [in_tensor, out_tensor] + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def ExpandDims(device="llvm", lib_path="./", + ndim=None, axis=None, dtype=None): + ''' + expand dims + Args: + device: + lib_path: + ndim: + axis: + dtype: + + Returns: + ''' + if axis > ndim: + return + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + opname = "ExpandDim_ndim%d_%s_axis%d" % (ndim, dtype, axis) + print(opname) + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') + out_tensor = topi.expand_dims(in_tensor, axis=axis) + tensor_list = [in_tensor, out_tensor] + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Tile(device="llvm", lib_path="./", + ndim=None, dtype=None): + ''' + tile + Args: + device: + lib_path: + ndim: + dtype: + + Returns: + ''' + shape = [tvm.var("n" + str(i)) for i in range(ndim)] + multiples = [tvm.var("k" + str(i)) for i in range(ndim)] + opname = "Tile_ndim%d_%s" % (ndim, dtype) + print(opname) + + def _Compute(*C_idx): + a_idx = [tvm.floordiv(idx, multiples[i]) for i, idx in enumerate(list(C_idx))] + a_idx = tuple(a_idx) + return in_tensor[a_idx] + + # define compute + in_tensor = tvm.placeholder(shape, dtype=dtype, name='in_tensor') # tvm 0.6-dev: topi.tile + shape_c = (np.array(shape) * np.array(multiples)).tolist() + out_tensor = tvm.compute(shape_c, _Compute) + + tensor_list = multiples + [in_tensor, out_tensor] + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Range(device="llvm", lib_path="./", + out_dtype=None): + ''' + range + Args: + device: + lib_path: + out_dtype: + + Returns: + ''' + start = tvm.var("start") + delta = tvm.var("delta") + opname = "Range_ndim_" + out_dtype + print(opname) + + out_tensor = tvm.compute((tvm.var("n0"),), lambda i: start.astype(out_dtype) + delta.astype(out_dtype) * i, \ + name='out_tensor') + out_tensor = AsType(out_tensor, out_dtype) + tensor_list = [start, delta, out_tensor] + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) + + +def Split(device="llvm", lib_path="./", + ndim=None, dtype=None, output_num=None, axis=None): + ''' + split + Args: + device: + lib_path: + ndim: + dtype: + output_num: + axis: + + Returns: + ''' + if axis >= ndim: + return + size_splits = [tvm.var("split" + str(i)) for i in range(output_num)] + a_shape = [tvm.var("n" + str(i)) for i in range(axis)] \ + + [np.sum(size_splits)] \ + + [tvm.var("n" + str(i)) for i in range(axis + 1, ndim)] + c_shapes = [] + for i in range(output_num): + c_shape = [] + for j in range(ndim): + if j == axis: + c_shape.append(tvm.var("split" + str(i))) + else: + c_shape.append(tvm.var("n" + str(j))) + c_shapes.append(c_shape) + indices_or_sections = np.cumsum(size_splits).tolist()[:-1] + opname = "Split_ndim%d_%s_output_num%d_axis%d" % (ndim, dtype, output_num, axis) + print(opname) + + # define compute + in_tensor = tvm.placeholder(a_shape, dtype=dtype, name='in_tensor') + + def _Compute(*C_idx): + a_idx = list(C_idx) + a_idx[axis] += idx_shift + a_idx = tuple(a_idx) + return in_tensor[a_idx] + + indices_or_sections_add0 = [0] + indices_or_sections + out_tensor = [] + for i in range(output_num): + idx_shift = indices_or_sections_add0[i] + ci = tvm.compute(c_shapes[i], _Compute) + out_tensor.append(ci) + tensor_list = size_splits + [in_tensor] + out_tensor + + s = topi.generic.schedule_injective(out_tensor) + Genlib(s, tensor_list, device, opname, lib_path) diff --git a/predict/module/tvm_kernel/lite/python/at_ops/config_tool.py b/predict/module/tvm_kernel/lite/python/at_ops/config_tool.py new file mode 100644 index 00000000000..b3006b1174d --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/at_ops/config_tool.py @@ -0,0 +1,211 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This module is define some data struct for tvm kernel. +""" +import tvm +import topi + +format_map = {"NCHW": 0, "NHWC": 1} + +pool_map = {"max_pool": 0, "avg_pool": 1, "global_pool": 2} + +activation_map = { + "no_activation": 0, + "relu": 1, + "sigmoid": 2, + "relu6": 3, + "elu": 4, + "leaky_relu": 5, + "abs": 6, + "relu1": 7, + "softsign": 8, + "softplus": 9, + "tanh ": 10, +} +activation_enum_map = { + "NO_ACTIVATION": 0, + "RELU": 1, + "SIGMOID": 2, + "RELU6": 3, + "elu": 4, + "leaky_relu": 5, + "abs": 6, + "relu1": 7, + "softsign": 8, + "softplus": 9, + "tanh ": 10, +} + +padmode_map = {"NOTSET": 0, "SAME": 1, "VALID": 2} + +mslite_datatype_map = { + "float16": 1, + "float32": 0, + "double": 11, + "int8": 2, + "int16": 6, + "int32": 3, + "int64": 9, + "uint8": 4, + "uint16": 7, + "uint32": 8, + "uint64": 10, +} + + +def get_key_by_value(dicts, value): + for k, v in dicts.items(): + if v == value: + return k + return None + + +def relu6(x): + return tvm.compute( + x.shape, + lambda *i: tvm.min( + tvm.max(x(*i), tvm.const(0, x.dtype)), tvm.const(6, x.dtype) + ), + ) + + +activation_topi_funs = {"NO_ACTIVATION": None, "RELU": topi.nn.relu, "RELU6": relu6} + +name_funcs = { + "Concat": ( + lambda opname, x: ( + opname + "_%d_%d" + "_%d" + "_%d" * x["ndim"] + "_%d" * len(x["shapeAxis"]) + ) + % ( + format_map[x["format"]], + x["ndim"], + x["axis"], + *x["shapeOut"], + *x["shapeAxis"], + ) + ), + "Softmax": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" * x["ndim"] + "_%d") + % (format_map[x["format"]], x["ndim"], *x["shape"], x["axis"]) + ), + "Activation": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" + "_%d" * x["ndim"]) + % (format_map[x["format"]], x["ndim"], activation_map[x["type"]], *x["shape"]) + ), + "Add": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" * x["ndim"]) + % (format_map[x["format"]], x["ndim"], *x["shape"]) + ), + "Convolution": ( + lambda opname, x: ( + opname + "_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d" + ) + % ( + format_map[x["format"]], + x["ndim"], + x["batch"], + x["in_channel"], + *x["in_size"], + x["num_filter"], + *x["filter_size"], + *x["pad"], + *x["stride"], + x["dilation"], + x["hasbias"], + activation_map[x["activation_type"]], + ) + ), + "Identity": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" * x["ndim"]) + % (format_map[x["format"]], x["ndim"], *x["shape"]) + ), + "BatchNorm": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" * x["ndim"] + "_%d") + % (format_map[x["format"]], x["ndim"], *x["shape"], x["epsilon"]) + ), + "Squeeze": ( + lambda opname, x: ( + opname + "_%d_%d" + "_%d" * x["ndim"] + "_%d" * len(x["axis"]) + ) + % (format_map[x["format"]], x["ndim"], *x["shape"], *x["axis"]) + ), + "BiasAdd": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" * x["ndim"] + "_%d") + % (format_map[x["format"]], x["ndim"], *x["shape"], x["axis"]) + ), + "Pooling": ( + lambda opname, x: (opname + "_%d_%d_%d" + "_%d" * x["ndim"] + "_%d_%d_%d") + % ( + format_map[x["format"]], + x["ndim"], + pool_map[x["type"]], + *x["shape"], + x["kernel"], + x["stride"], + x["pad"], + ) + ), + "ConvolutionDepthwise": ( + lambda opname, x: ( + opname + "_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d_%d" + ) + % ( + format_map[x["format"]], + x["ndim"], + x["batch"], + x["in_channel"], + *x["in_size"], + x["in_channel"] * x["channel_multiplier"], + *x["filter_size"], + *x["pad"], + *x["stride"], + x["dilation"], + x["hasbias"], + activation_map[x["activation_type"]], + ) + ), + "Reshape": ( + lambda opname, x: ( + opname + "_%d_%d" + "_%d" * x["ndimA"] + "_%d" * len(x["shapeB"]) + ) + % (format_map[x["format"]], x["ndimA"], *x["shapeA"], *x["shapeB"]) + ), + "Shape": ( + lambda opname, x: (opname + "_%d_%d" + "_%d" * x["ndim"]) + % (format_map[x["format"]], x["ndim"], *x["shape"]) + ), + "RealDiv": ( + lambda opname, x: ( + opname + "_%d_%d" + "_%d" * x["ndim"] + "_%d" * len(x["shapeB"]) + ) + % (format_map[x["format"]], x["ndim"], *x["shapeA"], *x["shapeB"]) + ), + "ResizeBilinear": (lambda opname, x: "ResizeBilinear"), + "TFLite_Detection_PostProcess": (lambda opname, x: "TFLite_Detection_PostProcess"), +} + +config_dict = {op_type: [] for op_type in name_funcs} + + +def config_dict_append(op_type, config, opname=None): + if opname is None: + config["opname"] = name_funcs[op_type](op_type, config) + else: + config["opname"] = opname + duplicate = [True for x in config_dict[op_type] if config == x] + + if duplicate: + config_dict[op_type].append(config) diff --git a/predict/module/tvm_kernel/lite/python/at_rt/at_runtime_reset.py b/predict/module/tvm_kernel/lite/python/at_rt/at_runtime_reset.py new file mode 100644 index 00000000000..cc0cc728859 --- /dev/null +++ b/predict/module/tvm_kernel/lite/python/at_rt/at_runtime_reset.py @@ -0,0 +1,62 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This module is Using to make Lite Runtime funcitons instead TVM Runtime funcitons while codegen. +""" + +import os +from tvm import codegen + +class AtRuntimeReset(): + """Using this class to make Lite Runtime funcitons instead TVM Runtime funcitons while codegen + Usage like: + with at_runtime_reset.AtRuntimeReset(): + fadd = tvm.build(s, [A, B], tgt, target_host = tgt_host, name = "myadd") + then the module fadd will using Lite runtime functions. + """ + + def __enter__(self): + if os.getenv("TVM_RUNTIME_ON") is not None: + return + codegen.SetRTFuncTransPair( + "TVMBackendAllocWorkspace", "LiteBackendAllocWorkspace" + ) + codegen.SetRTFuncTransPair( + "TVMBackendFreeWorkspace", "LiteBackendFreeWorkspace" + ) + codegen.SetRTFuncTransPair("TVMAPISetLastError", "LiteAPISetLastError") + codegen.SetRTFuncTransPair( + "TVMBackendParallelLaunch", "LiteBackendParallelLaunch" + ) + codegen.SetRTFuncTransPair( + "TVMBackendParallelBarrier", "LiteBackendParallelBarrier" + ) + codegen.SetRTFuncTransPair( + "TVMBackendRegisterSystemLibSymbol", "LiteBackendRegisterSystemLibSymbol" + ) + codegen.SetRTFuncTransPair("TVMFuncCall", "LiteFuncCall") + codegen.SetRTFuncTransPair( + "TVMBackendGetFuncFromEnv", "LiteBackendGetFuncFromEnv" + ) + + def __exit__(self, ptype, value, trace): + codegen.DelRTFuncTransPair("TVMBackendAllocWorkspace") + codegen.DelRTFuncTransPair("TVMBackendFreeWorkspace") + codegen.DelRTFuncTransPair("TVMAPISetLastError") + codegen.DelRTFuncTransPair("TVMBackendParallelLaunch") + codegen.DelRTFuncTransPair("TVMBackendParallelBarrier") + codegen.DelRTFuncTransPair("TVMBackendRegisterSystemLibSymbol") + codegen.DelRTFuncTransPair("TVMFuncCall") + codegen.DelRTFuncTransPair("TVMBackendGetFuncFromEnv") diff --git a/predict/module/tvm_kernel/lite/src/api/kernel_manager.cc b/predict/module/tvm_kernel/lite/src/api/kernel_manager.cc new file mode 100644 index 00000000000..afe47e1ccb6 --- /dev/null +++ b/predict/module/tvm_kernel/lite/src/api/kernel_manager.cc @@ -0,0 +1,1772 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this ${file} except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LITE_RUNTIME_ON + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/mslog.h" + +const char *LIB_INFO = "libtvm_kernel version: master (c66c6b28dc991c9d705e1b983aab7385c337128d)"; +namespace km { +class KernelManager { + public: + int CallKernel(const std::string &fid, TVMArgs args) { + tvm::runtime::Module *mod = this->GetModule(); + CHECK(mod != nullptr) << "Failed to get Module!"; + const std::string name = fid; + tvm::runtime::PackedFunc f = mod->GetFunction(name, false); + CHECK(f != nullptr) << "Can't find kernel func " << fid; + TVMRetValue rv; + f.CallPacked(args, &rv); + return 0; + } + + void InitKernelManager(int mode, const std::string &fname) { return this->Init(mode, fname); } + + static KernelManager *Global() { + static KernelManager inst; + return &inst; + } + + tvm::runtime::Module *GetModule() const { return &g_modLib; } + + private: + KernelManager() = default; + + ~KernelManager() = default; + + void Init(int mode, std::string fpath) { + std::call_once(init_flag, &KernelManager::InitLib, mode, fpath); + return; + } + + static void InitLib(int mode, std::string fpath) { + if (mode) { + const PackedFunc *ptr = tvm::runtime::Registry::Get("module._GetSystemLib"); + CHECK(ptr != nullptr) << "Failed to get systemlib"; + g_modLib = (*ptr)(); + } else { + g_modLib = tvm::runtime::Module::LoadFromFile(fpath); + } + } + static tvm::runtime::Module g_modLib; + std::once_flag init_flag; +}; + +tvm::runtime::Module KernelManager::g_modLib; +} // namespace km + +std::function &)> GetKernel(const std::string &fid) { + km::KernelManager *inst = km::KernelManager::Global(); + CHECK(inst != nullptr) << "Failed to get KernelManager instance!"; + tvm::runtime::Module *mod = inst->GetModule(); + CHECK(mod != nullptr) << "Failed to get Module!"; + tvm::runtime::PackedFunc f = mod->GetFunction(fid, false); + if (f == nullptr) { + MS_LOGE("GetFunction return nullptr"); + return nullptr; + } + auto runner = [f](const std::vector &tensors) -> int { + int argLen = tensors.size(); + CHECK(argLen) << "Input tensors num=0 !"; + std::vector values(argLen); + std::vector codes(argLen); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (int i = 0; i < argLen; ++i) { + setter(i, tensors.at(i)); + } + tvm::runtime::TVMArgs targs(values.data(), codes.data(), argLen); + TVMRetValue rv; + f.CallPacked(targs, &rv); + return 0; + }; + return runner; +} + +int CallKernel(const std::string &fid, const std::vector &tensors) { + km::KernelManager *inst = km::KernelManager::Global(); + CHECK(inst != nullptr) << "Failed to get KernelManager instance!"; + int argLen = tensors.size(); + CHECK(argLen) << "Input tensors num=0 !"; + std::vector values(argLen); + std::vector codes(argLen); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (int i = 0; i < argLen; ++i) { + setter(i, tensors.at(i)); + } + tvm::runtime::TVMArgs targs(values.data(), codes.data(), argLen); + inst->CallKernel(fid, targs); + return 0; +} + +int InitKernelManager(int mode, const std::string &fname) { + km::KernelManager *inst = km::KernelManager::Global(); + CHECK(inst != nullptr) << "Failed to get KernelManager instance!"; + inst->InitKernelManager(mode, fname); + return 0; +} + +// just for api compatible, tvm/lite has same api +void ConfigThreadPool(int mode = 1, int nthreads = 0, bool execute_self = true) {} + +#else + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "flatbuffers/flatbuffers.h" +#include "schema/inner/ms_generated.h" +#include "include/securec.h" +#include "src/runtime/runtime_api.h" +#include "common/mslog.h" + +using runnerType = std::function &)>; + +const char *LIB_INFO = "libtvm_kernel version: master (c66c6b28dc991c9d705e1b983aab7385c337128d)"; + +namespace lite { +namespace runtime { +extern "C" { +// Function signature for generated packed function in shared library +typedef int (*BackendPackedCFunc)(const void *args, int *type_codes, int num_args); +} // extern "C" + +class LiteFuncPool { + public: + LiteFuncPool() = default; + + ~LiteFuncPool() = default; + + void GetFunction(const std::string &name, void **func_addr) { + auto it = tbl_.find(name); + if (func_addr == nullptr) { + MS_LOGW("input func_addr is nullptr"); + return; + } + *func_addr = (it != tbl_.end() ? it->second : nullptr); + } + + void RegisterSymbol(const std::string &name, void *ptr) { + std::lock_guard lock(mutex_); + auto it = tbl_.find(name); + if (it != tbl_.end() && ptr != it->second) { + MS_LOGW("Lite symbol %s get overriden to a different address %p->%p", name.c_str(), ptr, it->second); + } + tbl_[name] = ptr; + } + + static LiteFuncPool *Global() { + static LiteFuncPool inst; + return &inst; + } + + private: + // Internal mutex + std::mutex mutex_; + // Internal symbol table + std::unordered_map tbl_; +}; +} // namespace runtime +} // namespace lite + +using LiteFuncPool = lite::runtime::LiteFuncPool; +using BackendPackedCFunc = lite::runtime::BackendPackedCFunc; + +int LiteBackendRegisterSystemLibSymbol(const char *name, void *ptr) { + MS_ASSERT(LiteFuncPool::Global() != nullptr); + LiteFuncPool::Global()->RegisterSymbol(name, ptr); + return 0; +} + +// do nothing, api compatible with TVM_RUNTIME_ON API +void InitKernelManager(int mode, const std::string &fname) { return; } + +static inline void *GetFunction(const std::string &fid) { + void *f = nullptr; + MS_ASSERT(LiteFuncPool::Global() != nullptr); + LiteFuncPool::Global()->GetFunction(fid, &f); + if (f == nullptr) { + return nullptr; + } + return f; +} + +runnerType __attribute__((noinline)) GetKernel(const std::string &fid) { + auto f = GetFunction(fid); + if (f == nullptr) { + return nullptr; + } + auto runner = [f](const std::vector &tensors) -> int { + if (tensors.empty()) { + MS_LOGE("Input tensors num = 0 !"); + return -1; + } + std::vector values(tensors.size()); + std::vector codes(tensors.size()); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (size_t i = 0; i < tensors.size(); ++i) { + setter(i, tensors.at(i)); + } + auto passfunc = reinterpret_cast(f); + return passfunc(values.data(), codes.data(), tensors.size()); + }; + return runner; +} + +namespace auto_tensor { +constexpr int TENSOR_NUM_MAX = 10; +constexpr bool STORE_MODE = true; +constexpr bool RESUME_MODE = false; +const char *NOT_SUPPORT = "NOT SUPPORT"; +const int NCHW_N = 0; +const int NCHW_C = 1; +const int NCHW_H = 2; +const int NCHW_W = 3; +const int tile = 4; + +void store_shape(const std::vector &tensors, int (&ndim)[TENSOR_NUM_MAX], int64_t *(&shape)[TENSOR_NUM_MAX], + int64_t *(&strides)[TENSOR_NUM_MAX], bool mode = STORE_MODE) { + if (mode == STORE_MODE) { + for (size_t i = 0; i < tensors.size(); ++i) { + ndim[i] = tensors[i]->ndim; + shape[i] = tensors[i]->shape; + strides[i] = tensors[i]->strides; + } + } else { + for (size_t i = 0; i < tensors.size(); ++i) { + tensors[i]->ndim = ndim[i]; + tensors[i]->shape = shape[i]; + tensors[i]->strides = strides[i]; + } + } +} + +static std::string get_dtype(const DLTensor &tensor) { + auto dtype = tensor.dtype; + if (dtype.code == kDLFloat) { + if (dtype.bits == 16) + return "float16"; + else if (dtype.bits == 32) + return "float32"; + else if (dtype.bits == 64) + return "float64"; + } else if (dtype.code == kDLInt) { + if (dtype.bits == 8) + return "int8"; + else if (dtype.bits == 16) + return "int16"; + else if (dtype.bits == 32) + return "int32"; + else if (dtype.bits == 64) + return "int64"; + } else if (dtype.code == kDLUInt) { + if (dtype.bits == 8) + return "uint8"; + else if (dtype.bits == 16) + return "uint16"; + else if (dtype.bits == 32) + return "uint32"; + else if (dtype.bits == 64) + return "uint64"; + } + return std::string(NOT_SUPPORT); +} + +struct OpCommonAttr { + std::string optype = ""; + std::string fid = ""; + uint32_t ndim = 0; + std::string dtype = "float32"; + + OpCommonAttr(const mindspore::predict::OpDef &opdef, const std::vector &tensors) { + auto opT = mindspore::predict::EnumNameOpT(opdef.attr_type()); + this->optype = opT; + MS_ASSERT(opdef.name() != nullptr); + this->fid = opdef.name()->str(); + if (!tensors.empty()) { + MS_ASSERT(tensors.front() != nullptr); + ndim = tensors.front()->ndim; + dtype = get_dtype(*tensors.front()); + } + } +}; + +template +static void NCHW2NHWC(DLTensor *src) { + if (src == nullptr) { + MS_LOGW("input src is nullptr"); + return; + } + T *src_data = static_cast(src->data); + std::unique_ptr tmp(new (std::nothrow) + T[src->shape[NCHW_N] * src->shape[NCHW_C] * src->shape[NCHW_H] * src->shape[NCHW_W]]); + if (tmp == nullptr) { + MS_LOGW("new tmp buf failed"); + return; + } + int N = src->shape[NCHW_N]; + int C = src->shape[NCHW_C]; + int H = src->shape[NCHW_H]; + int W = src->shape[NCHW_W]; + + // NCHW -> NHWC + int k = 0; + for (int n = 0; n < N; n++) + for (int h = 0; h < H; h++) + for (int w = 0; w < W; w++) + for (int c = 0; c < C; c++) { + tmp[k++] = src_data[n * C * H * W + c * H * W + h * W + w]; + } + + int sizes = N * C * H * W * sizeof(T); + errno_t ret = memcpy_s(src_data, sizes, tmp.get(), sizes); + if (ret != 0) { + MS_LOGW("memcpy_s failed: %d", ret); + return; + } +} + +static void transpose_shape(DLTensor *tensor, std::vector axis) { + if (tensor == nullptr) { + MS_LOGW("input tensor is nullptr"); + return; + } + int ndim = tensor->ndim; + std::vector origin_shape(tensor->shape, tensor->shape + ndim); + + for (int i = ndim - 1; i >= 0; --i) { + tensor->shape[i] = origin_shape[axis[i]]; + } +} + +static runnerType Pack_NCHW2NHWC(runnerType fun) { + if (fun == nullptr) { + MS_LOGE("input fun is nullptr"); + return nullptr; + } + auto runner = [fun](const std::vector &tensors) -> int { + if (tensors.back() == nullptr) { + MS_LOGE("tensors.back() is nullptr"); + return 1; + } + transpose_shape(tensors.back(), {0, 3, 1, 2}); // NHWC -> NCHW + fun(tensors); + + auto output = tensors.back(); + if (output == nullptr) { + MS_LOGE("tensors.back() after func is nullptr"); + return 1; + } + if (output->dtype.bits == 8) { + NCHW2NHWC(output); + } else if (output->dtype.bits == 16) { + NCHW2NHWC(output); + } else if (output->dtype.bits == 32) { + NCHW2NHWC(output); + } else if (output->dtype.bits == 64) { + NCHW2NHWC(output); + } else { + MS_LOGE("conv NCHW2NHWC output.dtype.bits=%d invalid, only support (8, 16, 32, 64)", output->dtype.bits); + return 1; + } + + if (tensors.back() == nullptr) { + MS_LOGE("tensors.back() is nullptr"); + return 1; + } + transpose_shape(tensors.back(), {0, 2, 3, 1}); // NCHW -> NHWC + return 0; + }; + return runner; +} + +runnerType __attribute__((noinline)) GetKernel_Insert_vector_int32(const std::string &fid, + const std::vector &vec) { + auto f = GetFunction(fid); + if (f == nullptr) { + MS_LOGE("GetFunction return nullptr"); + return nullptr; + } + auto runner = [f, vec](const std::vector &tensors) -> int { + std::vector values(vec.size() + tensors.size()); + std::vector codes(values.size()); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (size_t i = 0; i < vec.size(); ++i) { + setter(i, vec.at(i)); + } + for (size_t i = 0; i < tensors.size(); ++i) { + setter(i + vec.size(), tensors.at(i)); + } + auto passfunc = reinterpret_cast(f); + return passfunc(values.data(), codes.data(), values.size()); + }; + return runner; +} + +runnerType __attribute__((noinline)) GetKernel_Insert_vector_float(const std::string &fid, + const std::vector &vec) { + auto f = GetFunction(fid); + if (f == nullptr) { + MS_LOGE("GetFunction return nullptr"); + return nullptr; + } + auto runner = [f, vec](const std::vector &tensors) -> int { + std::vector values(vec.size() + tensors.size()); + std::vector codes(values.size()); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (size_t i = 0; i < vec.size(); ++i) { + setter(i, vec.at(i)); + } + for (size_t i = 0; i < tensors.size(); ++i) { + setter(i + vec.size(), tensors.at(i)); + } + auto passfunc = reinterpret_cast(f); + return passfunc(values.data(), codes.data(), values.size()); + }; + return runner; +} + +static runnerType GetKernel_Conv(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + if (tensors.at(0) == nullptr) { + MS_LOGE("input tensors.at(0) is nullptr"); + return nullptr; + } + int n = tensors.at(0)->shape[NCHW_N]; + int ci = tensors.at(0)->shape[NCHW_C]; + int h = tensors.at(0)->shape[NCHW_H]; + int w = tensors.at(0)->shape[NCHW_W]; + std::vector arg_const{n, ci, h, w}; + const OpCommonAttr opAttr(opdef, tensors); + + std::string fid; + if (opdef.attr_as_Conv2D() != nullptr) { + auto op = opdef.attr_as_Conv2D(); + fid = std::string(mindspore::predict::EnumNameOpT(opdef.attr_type())) + "_ndim" + std::to_string(opAttr.ndim) + + "_" + opAttr.dtype + "_k" + std::to_string(op->kernelH()) + "_s" + std::to_string(op->strideH()) + "_p" + + std::to_string(op->padUp()) + std::to_string(op->padDown()) + std::to_string(op->padLeft()) + + std::to_string(op->padRight()) + "_d" + std::to_string(op->dilateH()) + "_act" + + std::to_string(static_cast(op->activationType())) + "_vc" + std::to_string(1) + "_vh" + + std::to_string(1) + "_vw" + std::to_string(1) + "_hasbias" + std::to_string(op->hasBias()); + if (tensors.at(1) == nullptr) { + MS_LOGE("input tensors.at(1) is nullptr"); + return nullptr; + } + int co = tensors.at(1)->shape[NCHW_N]; + arg_const.push_back(co); + } else if (opdef.attr_as_DepthwiseConv2D() != nullptr) { + auto op = opdef.attr_as_DepthwiseConv2D(); + fid = std::string(mindspore::predict::EnumNameOpT(opdef.attr_type())) + "_ndim" + std::to_string(opAttr.ndim) + + "_" + opAttr.dtype + "_k" + std::to_string(op->kernelH()) + "_s" + std::to_string(op->strideH()) + "_p" + + std::to_string(op->padUp()) + std::to_string(op->padDown()) + std::to_string(op->padLeft()) + + std::to_string(op->padRight()) + "_d" + std::to_string(op->dilateH()) + "_act" + + std::to_string(static_cast(op->activationType())) + "_vc" + std::to_string(1) + "_vh" + + std::to_string(1) + "_vw" + std::to_string(1) + "_hasbias" + std::to_string(op->hasBias()); + int co = tensors.at(0)->shape[NCHW_C] * op->channelMultiplier(); + arg_const.push_back(co); + } else if (opdef.attr_as_DeDepthwiseConv2D() != nullptr) { + auto op = opdef.attr_as_DeDepthwiseConv2D(); + fid = std::string(mindspore::predict::EnumNameOpT(opdef.attr_type())) + "_ndim" + std::to_string(opAttr.ndim) + + "_" + opAttr.dtype + "_k" + std::to_string(op->kernelH()) + "_s" + std::to_string(op->strideH()) + "_p" + + std::to_string(op->padUp()) + std::to_string(op->padDown()) + std::to_string(op->padLeft()) + + std::to_string(op->padRight()) + "_d" + std::to_string(op->dilateH()) + "_act" + + std::to_string(static_cast(op->activationType())) + "_vc" + std::to_string(1) + "_vh" + + std::to_string(1) + "_vw" + std::to_string(1) + "_hasbias" + std::to_string(op->hasBias()); + int co = tensors.at(0)->shape[NCHW_C] * op->channelMultiplier(); + arg_const.push_back(co); + } + auto fun = GetKernel(fid); + if (fun == nullptr) { + MS_LOGE("GetKernel return nullptr"); + return nullptr; + } + + auto f = GetFunction(fid); + if (f == nullptr) { + MS_LOGE("GetFunction return nullptr"); + return nullptr; + } + auto runner = [f, arg_const](const std::vector &tensors) -> int { + int ndim[TENSOR_NUM_MAX]; + int64_t *shapes[TENSOR_NUM_MAX]; + int64_t *strides[TENSOR_NUM_MAX]; + store_shape(tensors, ndim, shapes, strides, STORE_MODE); + + std::vector values(arg_const.size() + tensors.size()); + std::vector codes(values.size()); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (size_t i = 0; i < arg_const.size(); ++i) { + setter(i, arg_const.at(i)); + } + for (size_t i = 0; i < tensors.size(); ++i) { + setter(i + arg_const.size(), tensors.at(i)); + } + auto passfunc = reinterpret_cast(f); + passfunc(values.data(), codes.data(), values.size()); + store_shape(tensors, ndim, shapes, strides, RESUME_MODE); + return 0; + }; + fun = runner; + + if (opdef.isLastConv()) { + return Pack_NCHW2NHWC(fun); + } + return fun; +} + +void update_shape_NC4HW4(const std::vector &tensors, int64_t (&shapeA)[TENSOR_NUM_MAX], + int64_t (&shapeC)[TENSOR_NUM_MAX]) { + auto inputA = tensors.front(); + auto output = tensors.back(); + if (inputA == nullptr) { + MS_LOGW("input tensors.front() is nullptr"); + return; + } + if (output == nullptr) { + MS_LOGW("input tensors.back() is nullptr"); + return; + } + shapeA[inputA->ndim] = tile; + for (int32_t i = 0; i < inputA->ndim; ++i) { + if (i == 1) { + shapeA[i] = inputA->shape[i] >> 2; + } else { + shapeA[i] = inputA->shape[i]; + } + } + { + inputA->ndim = inputA->ndim + 1; + inputA->shape = shapeA; + inputA->strides = nullptr; + } + shapeC[output->ndim] = tile; + for (int32_t i = 0; i < output->ndim; ++i) { + if (i == 1) { + shapeC[i] = output->shape[i] >> 2; + } else { + shapeC[i] = output->shape[i]; + } + } + { + output->ndim = output->ndim + 1; + output->shape = shapeC; + output->strides = nullptr; + } +} + +static runnerType GetKernel_Conv_var(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto fun = GetKernel(opAttr.fid); + if (tensors.at(0) == nullptr) { + MS_LOGE("input tensors.at(0) is nullptr"); + return nullptr; + } + std::string fid = opAttr.fid.substr(0, opAttr.fid.find('_')); + int n = tensors.at(0)->shape[NCHW_N]; + int ci = tensors.at(0)->shape[NCHW_C]; + int h = tensors.at(0)->shape[NCHW_H]; + int w = tensors.at(0)->shape[NCHW_W]; + int co = tensors.at(1)->shape[NCHW_C]; + std::vector arg_const{n, ci >> 2, h, w, co}; + if (fun == nullptr) { + auto fd = [](int h, std::vector &res) { + for (int i = 2; i <= h; i += 2) { + if ((h % i) == 0) res.emplace_back(i); + } + }; + int outidx = tensors.size() - 1; + std::vector vw; + if (tensors.at(outidx) == nullptr) { + MS_LOGE("input tensors.at(%d) is nullptr", outidx); + return nullptr; + } + fd(tensors.at(outidx)->shape[NCHW_W], vw); + + auto op = opdef.attr_as_DeConv2D(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_DeConv2D() is nullptr"); + return nullptr; + } + std::string fids; + for (auto iter = vw.rbegin(); iter != vw.rend(); iter++) { + fids = fid + "_ndim" + std::to_string(opAttr.ndim + 1) + "_" + opAttr.dtype + "_k" + + std::to_string(op->kernelH()) + "_s" + std::to_string(op->strideH()) + "_p" + std::to_string(op->padUp()) + + std::to_string(op->padDown()) + std::to_string(op->padLeft()) + std::to_string(op->padRight()) + "_d" + + std::to_string(op->dilateH()) + "_act" + std::to_string(static_cast(op->activationType())) + "_vc" + + std::to_string(4) + "_vh" + std::to_string(2) + "_vw" + std::to_string(*iter) + "_hasbias" + + std::to_string(op->hasBias()); + fun = GetKernel(fids); + if (fun != nullptr) { + break; + } + } + fid = fids; + if (fun == nullptr) { + MS_LOGE("fun is nullptr"); + return nullptr; + } + auto f = GetFunction(fid); + if (f == nullptr) { + MS_LOGE("GetFunction return nullptr"); + return nullptr; + } + auto runner = [f, arg_const](const std::vector &tensors) -> int { + int ndim[TENSOR_NUM_MAX]; + int64_t *shapes[TENSOR_NUM_MAX]; + int64_t *strides[TENSOR_NUM_MAX]; + int64_t shapeA[TENSOR_NUM_MAX]; + int64_t shapeC[TENSOR_NUM_MAX]; + store_shape(tensors, ndim, shapes, strides, STORE_MODE); + update_shape_NC4HW4(tensors, shapeA, shapeC); + + std::vector values(arg_const.size() + tensors.size()); + std::vector codes(values.size()); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (size_t i = 0; i < arg_const.size(); ++i) { + setter(i, arg_const.at(i)); + } + for (size_t i = 0; i < tensors.size(); ++i) { + setter(i + arg_const.size(), tensors.at(i)); + } + auto passfunc = reinterpret_cast(f); + passfunc(values.data(), codes.data(), values.size()); + store_shape(tensors, ndim, shapes, strides, RESUME_MODE); + return 0; + }; + fun = runner; + } + + if (opdef.isLastConv()) { + return Pack_NCHW2NHWC(fun); + } + return fun; +} + +enum reahpeCHW_Mode { FusedCHW, ExpandCHW }; + +void update_shape_reahpeCHW(const std::vector &tensors, reahpeCHW_Mode mode, int64_t (&shape)[4], + int64_t (&strides)[4], bool reahpe_output = false) { + auto input = tensors.front(); + auto output = tensors.back(); + if (input == nullptr) { + MS_LOGW("input tensors.front() is nullptr"); + return; + } + if (output == nullptr) { + MS_LOGW("input tensors.back() is nullptr"); + return; + } + int ndim; + if (mode == FusedCHW) { + ndim = 2; + int64_t CHW = 1; + for (int32_t i = 1; i < input->ndim; ++i) { + CHW *= input->shape[i]; + } + shape[NCHW_N] = input->shape[NCHW_N]; + shape[NCHW_C] = CHW; + strides[1] = 1; + strides[0] = CHW; + } else { + ndim = 4; + shape[NCHW_N] = input->shape[NCHW_N]; + shape[NCHW_C] = input->shape[NCHW_C]; + shape[NCHW_H] = 1; + shape[NCHW_W] = 1; + strides[3] = 1; + strides[2] = 1; + strides[1] = 1; + strides[0] = input->shape[NCHW_C]; + } + + input->ndim = ndim; + input->shape = shape; + input->strides = strides; + if (reahpe_output) { + output->ndim = ndim; + output->shape = shape; + output->strides = strides; + } +} + +static runnerType Pack_reahpeCHW(const runnerType &fun, const std::vector &tensors, reahpeCHW_Mode mode, + bool reahpe_output = false) { + if (fun == nullptr) { + MS_LOGE("input fun is nullptr"); + return nullptr; + } + if (tensors.front() == nullptr) { + MS_LOGE("input tensors.front() is nullptr"); + return nullptr; + } + if ((tensors.front()->ndim == 2 && mode == FusedCHW) || (tensors.front()->ndim == 4 && mode == ExpandCHW)) { + return fun; + } + + auto runner = [fun, mode, reahpe_output](const std::vector &tensors) -> int { + int ndim[TENSOR_NUM_MAX]; + int64_t *shape[TENSOR_NUM_MAX]; + int64_t *strides[TENSOR_NUM_MAX]; + int64_t shape_R[4]; + int64_t strides_R[4]; + store_shape(tensors, ndim, shape, strides, STORE_MODE); + update_shape_reahpeCHW(tensors, mode, shape_R, strides_R, reahpe_output); + fun(tensors); + store_shape(tensors, ndim, shape, strides, RESUME_MODE); + return 0; + }; + return runner; +} + +static runnerType GetKernel_BatchNorm(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + std::string fid; + std::vector epsilon(1, 0.001); + if (opAttr.optype == "BatchNorm") { + fid = "TFBatchNorm_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis1"; + epsilon.front() = opdef.attr_as_FusedBatchNorm()->epsilon(); + return GetKernel_Insert_vector_float(fid, epsilon); + } else if (opAttr.optype == "CaffeBatchNorm") { + fid = "CaffeBatchNorm_ndim4_" + opAttr.dtype + "_axis1"; + epsilon.front() = opdef.attr_as_CaffeBatchNorm()->epsilon(); + auto fun = GetKernel_Insert_vector_float(fid, epsilon); + if (fun == nullptr) { + MS_LOGE("GetKernel_Insert_vector_float return nullptr"); + return nullptr; + } + bool reahpe_output = true; + return Pack_reahpeCHW(fun, tensors, ExpandCHW, reahpe_output); + } else if (opAttr.optype == "BiasAdd") { + auto op = opdef.attr_as_BiasAdd(); + fid = "TFBiasAdd_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis" + + std::to_string(op->axis()->Get(0)); + return GetKernel(fid); + } else if (opAttr.optype == "Scale") { + fid = "CaffeScale_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis1"; + return GetKernel(fid); + } + return nullptr; +} + +void update_shape_flatten(const std::vector &tensors, int64_t *shape, int64_t *strides) { + auto inputA = tensors.back(); + if (inputA == nullptr) { + MS_LOGW("input tensors.back() is nullptr"); + return; + } + for (int32_t i = 0; i < inputA->ndim; ++i) { + *shape *= inputA->shape[i]; + } + for (size_t i = 0; i < tensors.size(); ++i) { + tensors[i]->ndim = 1; + tensors[i]->shape = shape; + tensors[i]->strides = strides; + } +} + +std::string GetEltwiseMode(const OpCommonAttr &opAttr, const mindspore::predict::OpDef &opdef) { + auto &optype = opAttr.optype; + std::string mode = "add"; + if (optype == "Eltwise") { + auto op_mode = opdef.attr_as_Eltwise()->mode(); + if (mindspore::predict::EltwiseMode_PROD == op_mode) { + mode = "multiply"; + } else if (mindspore::predict::EltwiseMode_SUM == op_mode) { + mode = "add"; + } else if (mindspore::predict::EltwiseMode_MAXIMUM == op_mode) { + mode = "maximum"; + } + } else { + if ("Add" == optype) { + mode = "add"; + } else if ("Sub" == optype) { + mode = "subtract"; + } else if ("Mul" == optype) { + mode = "multiply"; + } else if ("RealDiv" == optype) { + mode = "divide"; + } else if ("Maximum" == optype) { + mode = "maximum"; + } + } + return mode; +} + +bool IsSwap(const std::vector &tensors) { + auto CalShape = [](DLTensor *tensor) -> int { + int res = 1; + if (tensor == nullptr) { + MS_LOGE("input DLTensor is nullptr"); + return -1; + } + for (int i = 0; i < tensor->ndim; ++i) { + res *= tensor->shape[i]; + } + return res; + }; + + MS_ASSERT(tensors[0] != nullptr); + MS_ASSERT(tensors[1] != nullptr); + auto ndimA = tensors[0]->ndim; + auto ndimB = tensors[1]->ndim; + bool isSwap = false; + + if (ndimA <= ndimB) { + auto AShape = CalShape(tensors[0]); + auto BShape = CalShape(tensors[1]); + if (AShape < BShape) { + isSwap = true; + } + } + return isSwap; +} + +static runnerType GetKernel_Eltwise(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + std::string mode = GetEltwiseMode(opAttr, opdef); + + // make fid + int indexA = 0; + int indexB = 1; + MS_ASSERT(tensors[0] != nullptr); + MS_ASSERT(tensors[1] != nullptr); + auto ndimA = tensors[0]->ndim; + auto ndimB = tensors[1]->ndim; + + bool isSwap = IsSwap(tensors); + if (isSwap) { + std::swap(ndimA, ndimB); + std::swap(indexA, indexB); + } + + MS_ASSERT(tensors[indexA] != nullptr); + MS_ASSERT(tensors[indexB] != nullptr); + if (ndimA == 1 && tensors[indexA]->shape[NCHW_N] == 1) { + ndimA = 0; + } + if (ndimB == 1 && tensors[indexB]->shape[NCHW_N] == 1) { + ndimB = 0; + } + bool is_same = ndimA == ndimB && ndimA > 1; + for (int i = 0; i < tensors[indexB]->ndim && is_same; ++i) { + if (tensors[indexB]->shape[i] != tensors[indexA]->shape[i]) { + is_same = false; + } + } + for (int i = 0; i < tensors[indexB]->ndim && ndimB > 1 && is_same == false; ++i) { + if (tensors[indexB]->shape[i] == 1) { + ndimB--; + } + } + + if (ndimA == ndimB && ndimA >= 1) { + std::string fid = "Eltwise_" + mode + "_ndimA1_ndimB1" + "_" + opAttr.dtype; + auto fun = GetKernel(fid); + if (fun == nullptr) { + MS_LOGE("GetKernel return nullptr"); + return nullptr; + } + auto runner = [fun, isSwap](const std::vector &tensors) -> int { + std::vector tensorsCopy(tensors); + if (isSwap) { + iter_swap(tensorsCopy.begin(), tensorsCopy.begin() + 1); + } + int ndim[TENSOR_NUM_MAX]; + int64_t *shapes[TENSOR_NUM_MAX]; + int64_t *strides[TENSOR_NUM_MAX]; + int64_t shape = 1; + int64_t stride = 1; + + store_shape(tensorsCopy, ndim, shapes, strides, STORE_MODE); + update_shape_flatten(tensorsCopy, &shape, &stride); + fun(tensorsCopy); + store_shape(tensorsCopy, ndim, shapes, strides, RESUME_MODE); + return 0; + }; + return runner; + } else { + std::string fid = + "Eltwise_" + mode + "_ndimA" + std::to_string(ndimA) + "_ndimB" + std::to_string(ndimB) + "_" + opAttr.dtype; + auto fun = GetKernel(fid); + if (fun == nullptr) { + MS_LOGE("GetKernel return nullptr"); + return nullptr; + } + auto runner = [fun, isSwap](const std::vector &tensors) -> int { + std::vector tensorsCopy(tensors); + if (isSwap) { + iter_swap(tensorsCopy.begin(), tensorsCopy.begin() + 1); + } + + fun(tensorsCopy); + return 0; + }; + return runner; + } +} + +static runnerType GetKernel_Resize(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + if (tensors.size() != 2) { + MS_LOGE("Input tensors num should be 2 !"); + return nullptr; + } + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Resize(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Resize() is nullptr"); + return nullptr; + } + std::string fid = "Resize_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype; + if (op->method() == mindspore::predict::ResizeMethod::ResizeMethod_NEAREST_NEIGHBOR) { + fid += "_nearest_neighbor"; + } else if (op->method() == mindspore::predict::ResizeMethod::ResizeMethod_BILINEAR) { + fid += "_bilinear"; + } + fid += (op->alignCorners()) ? "_Align" : "_NotAlign"; + std::vector HeightWidth = { + static_cast(op->newHeight()), + static_cast(op->newWidth()), + }; + return GetKernel_Insert_vector_int32(fid, HeightWidth); +} + +static runnerType GetKernel_DataCarry(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + auto runner = [](const std::vector &tensors) -> int { + auto input = tensors.front(); + auto output = tensors.back(); + if (input == nullptr) { + MS_LOGE("input tensors.front() is nullptr"); + return 1; + } + if (output == nullptr) { + MS_LOGE("input tensors.back() is nullptr"); + return 1; + } + uint64_t input_num = 1; + for (int i = 0; i < input->ndim; ++i) { + input_num *= input->shape[i]; + } + uint64_t input_byte_num = input_num * input->dtype.lanes * input->dtype.bits / 8; + + uint64_t output_num = 1; + for (int i = 0; i < output->ndim; ++i) { + output_num *= output->shape[i]; + } + uint64_t output_byte_num = output_num * output->dtype.lanes * output->dtype.bits / 8; + + errno_t ret = memcpy_s(output->data, output_byte_num, input->data, input_byte_num); + if (ret != 0) { + MS_LOGE("memset_s failed."); + return ret; + } + return 0; + }; + return runner; +} + +static runnerType GetKernel_Shape(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + auto runner = [](const std::vector &tensors) -> int { + auto input = tensors.front(); + auto output = tensors.back(); + if (input == nullptr) { + MS_LOGE("input tensors.front() is nullptr"); + return 1; + } + if (output == nullptr) { + MS_LOGE("input tensors.back() is nullptr"); + return 1; + } + for (int i = 0; i < input->ndim; ++i) { + reinterpret_cast(output->data)[i] = static_cast(input->shape[i]); + } + return 0; + }; + return runner; +} + +void StridedSliceArgs(const std::vector &input_shape, std::vector *begin, std::vector *end, + std::vector *stride, uint32_t begin_mask, uint32_t end_mask, uint32_t ellipsis_mask, + uint32_t new_axis_mask, uint32_t shrink_axis_mask) { + MS_ASSERT(begin != nullptr); + MS_ASSERT(end != nullptr); + MS_ASSERT(stride != nullptr); + constexpr int support_dims = 8; + std::bitset begin_list(begin_mask); + std::bitset end_list(end_mask); + std::bitset ellipsis_list(ellipsis_mask); + std::bitset new_axis_list(new_axis_mask); + std::bitset shrink_list(shrink_axis_mask); + + std::string begin_list_s = begin_list.to_string().substr(support_dims - begin->size()); + reverse(begin_list_s.begin(), begin_list_s.end()); + + std::string end_list_s = end_list.to_string().substr(support_dims - end->size()); + reverse(end_list_s.begin(), end_list_s.end()); + + std::string ellipsis_list_s = ellipsis_list.to_string().substr(support_dims - end->size()); + reverse(ellipsis_list_s.begin(), ellipsis_list_s.end()); + + std::string new_axis_list_s = new_axis_list.to_string().substr(support_dims - end->size()); + reverse(new_axis_list_s.begin(), new_axis_list_s.end()); + + std::string shrink_list_s = shrink_list.to_string().substr(support_dims - end->size()); + reverse(shrink_list_s.begin(), shrink_list_s.end()); + + int new_axis_count = new_axis_list.count(); + if (ellipsis_list.any()) { + auto idx = 0; // ellipsis_list._Find_first(); + // the 1 is ellipsis + int ellipsis_length = input_shape.size() - (begin->size() - 1 - new_axis_count); + begin->erase(begin->begin() + idx); + end->erase(end->begin() + idx); + stride->erase(stride->begin() + idx); + + begin_list_s.erase(idx, 1); + end_list_s.erase(idx, 1); + ellipsis_list_s.erase(idx, 1); + new_axis_list_s.erase(idx, 1); + shrink_list_s.erase(idx, 1); + + if (ellipsis_length > 0) { + begin->insert(begin->begin() + idx, ellipsis_length, 0); + end->insert(end->begin() + idx, ellipsis_length, 0); + stride->insert(stride->begin() + idx, ellipsis_length, 1); + begin_list_s.insert(idx, ellipsis_length, '1'); + end_list_s.insert(idx, ellipsis_length, '1'); + ellipsis_list_s.insert(idx, ellipsis_length, '0'); + new_axis_list_s.insert(idx, ellipsis_length, '0'); + shrink_list_s.insert(idx, ellipsis_length, '0'); + } + } + + if (new_axis_count) { + for (int i = static_cast(new_axis_list_s.size()) - 1; i >= 0; i--) { + if (new_axis_list_s[i] == '1') { + begin->erase(begin->begin() + i); + end->erase(end->begin() + i); + stride->erase(stride->begin() + i); + begin_list_s.erase(i, 1); + end_list_s.erase(i, 1); + shrink_list_s.erase(i, 1); + } + } + } + + unsigned int size = begin->size(); + for (unsigned int i = 0; i < size; i++) { + if (shrink_list_s[i] == '1') { + auto beginItr = (begin->begin() + i); + auto endItr = (end->begin() + i); + auto strideItr = (stride->begin() + i); + *endItr = *beginItr + 1; + *strideItr = 1; + continue; + } + if (begin_list_s[i] == '1') { + auto beginItr = (begin->begin() + i); + *beginItr = 0; + } + if (end_list_s[i] == '1') { + auto endItr = (end->begin() + i); + *endItr = input_shape[i]; + } + } +} + +#define MAXDIMS 10 +template +int StridedSlice(const std::vector &input_shape, T *input, T *output, int *start, int *end, int *stride, + const int &output_size) { + MS_ASSERT(input != nullptr); + MS_ASSERT(output != nullptr); + MS_ASSERT(start != nullptr); + MS_ASSERT(end != nullptr); + MS_ASSERT(stride != nullptr); + int dimension = input_shape.size(); + if (dimension == 1) { + if (*stride == 1) { + int sizes = (*end - *start) * sizeof(T); + errno_t ret = memcpy_s(output, output_size * sizeof(T), input + *start, sizes); + if (ret != 0) { + MS_LOGE("memset_s failed: %d", ret); + return ret; + } + return 0; + } + for (int j = *start, i = 0; j < *end; j += (*stride), i++) { + output[i] = input[j]; + } + return 0; + } + + // adapt higher dimension + int dimensionArray[MAXDIMS]; + int factorArray[MAXDIMS]; + int totalElement = 0; + + for (int i = 0; i < dimension; i++) { + dimensionArray[i] = input_shape[i]; + factorArray[i] = i ? factorArray[i - 1] * dimensionArray[i] : dimensionArray[i]; + totalElement = i ? totalElement * dimensionArray[i] : dimensionArray[i]; + } + + int j = 0; + for (int k = 0; k < totalElement; k++) { + bool isValid = true; + for (int i = 0; i < dimension; i++) { + int tmp = (k / (totalElement / factorArray[i])) % dimensionArray[i]; + if (tmp < start[i] || tmp >= end[i]) { + isValid = false; + break; + } + isValid = isValid && ((tmp - start[i]) % stride[i] == 0); + } + if (isValid) { + output[j++] = input[k]; + } + } + + return 0; +} + +static runnerType GetKernel_StridedSlice(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto ndim = opAttr.ndim; + + auto op = opdef.attr_as_StridedSlice(); + if (op == nullptr) { + MS_LOGE("op is nullptr"); + return nullptr; + } + uint32_t begin_mask = op->beginMask(); + uint32_t end_mask = op->endMask(); + uint32_t ellipsis_mask = op->ellipsisMask(); + uint32_t new_axis_mask = op->newAxisMask(); + uint32_t shrink_axis_mask = op->shrinkAxisMask(); + std::vector begin; + std::vector end; + std::vector stride; + for (uint32_t i = 0; i < ndim; ++i) { + begin.push_back(op->begin()->Get(i)); + end.push_back(op->end()->Get(i)); + stride.push_back(op->stride()->Get(i)); + } + + auto runner = [begin_mask, end_mask, ellipsis_mask, new_axis_mask, shrink_axis_mask, begin, end, + stride](const std::vector &tensors) mutable -> int { + auto input = tensors.front(); + auto output = tensors.back(); + std::vector input_shape; + for (int i = 0; i < input->ndim; ++i) { + input_shape.push_back(input->shape[i]); + } + + int output_size = 1; + for (int i = 0; i < output->ndim; ++i) { + output_size *= output->shape[i]; + } + + StridedSliceArgs(input_shape, &begin, &end, &stride, begin_mask, end_mask, ellipsis_mask, new_axis_mask, + shrink_axis_mask); + + if (input->dtype.lanes != 1) { + MS_LOGE("StridedSlice input.dtype.lanes=%d invalid, only support 1", input->dtype.lanes); + return 1; + } + + if (input->dtype.bits == 16) { + StridedSlice(input_shape, reinterpret_cast(input->data), reinterpret_cast(output->data), + begin.data(), end.data(), stride.data(), output_size); + } else if (input->dtype.bits == 32) { + StridedSlice(input_shape, reinterpret_cast(input->data), reinterpret_cast(output->data), + begin.data(), end.data(), stride.data(), output_size); + } else { + MS_LOGE("StridedSlice input.dtype.bits=%d invalid, only support (16, 32)", input->dtype.bits); + return 1; + } + return 0; + }; + return runner; +} + +template +static void Permute4d(DLTensor *src, DLTensor *dst, const std::vector &shape, + const std::vector &strides) { + MS_ASSERT(src != nullptr); + MS_ASSERT(dst != nullptr); + int64_t N = shape[NCHW_N]; + int64_t C = shape[NCHW_C]; + int64_t H = shape[NCHW_H]; + int64_t W = shape[NCHW_W]; + auto src_data = reinterpret_cast(src->data); + auto dst_data = reinterpret_cast(dst->data); + int k = 0; + for (int n = 0; n < N; n++) + for (int c = 0; c < C; c++) + for (int h = 0; h < H; h++) + for (int w = 0; w < W; w++) { + dst_data[k++] = src_data[n * strides[0] + c * strides[1] + h * strides[2] + w * strides[3]]; + } +} + +static runnerType GetKernel_CaffeCrop(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_CaffeCrop(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_CaffeCrop() is nullptr"); + return nullptr; + } + std::string fid = + "CaffeCrop_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis" + std::to_string(op->axis()); + + std::vector offsets(op->offsets()->size()); + for (size_t i = 0; i < offsets.size(); ++i) { + offsets[i] = op->offsets()->Get(i); + } + return GetKernel_Insert_vector_int32(fid, offsets); +} + +static runnerType GetKernel_CaffePReLU(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_CaffePReLU(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_CaffePReLU() is nullptr"); + return nullptr; + } + std::string fid = "CaffePReLU_ndim4_" + opAttr.dtype; + fid += (op->channelShared()) ? "_channelShared" : "_channelNotShared"; + auto fun = GetKernel(fid); + if (fun == nullptr) { + return nullptr; + } + bool reahpe_output = true; + return Pack_reahpeCHW(fun, tensors, ExpandCHW, reahpe_output); +} + +static runnerType GetKernel_FullConnection(const mindspore::predict::OpDef &opdef, + const std::vector &tensors, const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_FullConnection(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_FullConnection() is nullptr"); + return nullptr; + } + std::string fid = "FullConnection_ndimA2_" + opAttr.dtype; + fid += (op->hasBias()) ? "_hasBias" : "_notHasBias"; + auto fun = GetKernel(fid); + if (fun == nullptr) { + return nullptr; + } + bool reahpe_output = false; + return Pack_reahpeCHW(fun, tensors, FusedCHW, reahpe_output); +} + +static runnerType GetKernel_Power(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Power(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Power() is nullptr"); + return nullptr; + } + std::string fid = "Power_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype; + std::vector pss; + pss.push_back(op->power()); + pss.push_back(op->scale()); + pss.push_back(op->shift()); + return GetKernel_Insert_vector_float(fid, pss); +} + +static runnerType GetKernel_ArgMax(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_ArgMax(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_ArgMax() is nullptr"); + return nullptr; + } + std::string fid = + "ArgMax_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis" + std::to_string(op->axis()); + fid += (op->keepDims()) ? "_keepDims" : "_notKeepDims"; + fid += "_top1"; + if (tensors.back() == nullptr) { + MS_LOGE("tensors.back() is nullptr"); + return nullptr; + } + fid += "_" + get_dtype(*tensors.back()); + return GetKernel(fid); +} + +void update_shape_Concat(const std::vector &tensors, int32_t axis, int64_t (&shape)[TENSOR_NUM_MAX][3], + int64_t (&strides)[TENSOR_NUM_MAX][3]) { + int64_t shape_low_dim = 1; + int64_t shape_high_dim = 1; + auto output = tensors.back(); + if (output == nullptr) { + MS_LOGW("tensors.back() is nullptr"); + return; + } + for (int32_t i = 0; i < axis; ++i) { + shape_high_dim *= output->shape[i]; + } + for (int32_t i = axis + 1; i < output->ndim; ++i) { + shape_low_dim *= output->shape[i]; + } + for (size_t i = 0; i < tensors.size(); ++i) { + shape[i][0] = shape_high_dim; + shape[i][1] = tensors[i]->shape[axis]; + shape[i][2] = shape_low_dim; + + strides[i][2] = 1; + strides[i][1] = shape[i][2]; + strides[i][0] = shape[i][2] * shape[i][1]; + + tensors[i]->ndim = 3; + tensors[i]->shape = shape[i]; + tensors[i]->strides = strides[i]; + } +} + +static runnerType GetKernel_Concat(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + if (tensors.size() < 2) { + MS_LOGE("Concat should have at least two tensors"); + return nullptr; + } + if (tensors.at(0) == nullptr) { + MS_LOGE("0th tensors of Concat is nullptr"); + return nullptr; + } + auto ndim = tensors.at(0)->ndim; + if (opdef.attr_as_Concat() == nullptr) { + MS_LOGE("opdef.attr_as_Concat() is nullptr"); + return nullptr; + } + auto axis = opdef.attr_as_Concat()->axis(); + if (axis < 0) { + axis += ndim; + } + std::string fid = + "Concat_ndim3_" + opAttr.dtype + "_input_num" + std::to_string(static_cast(tensors.size()) - 1) + "_axis1"; + auto fun = GetKernel(fid); + if (fun == nullptr) { + MS_LOGE("GetKernel return nullptr"); + return nullptr; + } + auto runner = [fun, axis](const std::vector &tensors) -> int { + int ndim[TENSOR_NUM_MAX]; + int64_t *shape[TENSOR_NUM_MAX]; + int64_t *strides[TENSOR_NUM_MAX]; + int64_t shape_C[TENSOR_NUM_MAX][3]; + int64_t strides_C[TENSOR_NUM_MAX][3]; + store_shape(tensors, ndim, shape, strides, STORE_MODE); + update_shape_Concat(tensors, axis, shape_C, strides_C); + fun(tensors); + store_shape(tensors, ndim, shape, strides, RESUME_MODE); + return 0; + }; + return runner; +} + +template +void Stack_ScaleNumber(const std::vector &tensors) { + if (tensors.empty()) { + MS_LOGW("input tensors is nullptr"); + return; + } + auto output = tensors.back(); + if (output != nullptr) { + MS_LOGW("tensors.back() is nullptr"); + return; + } + for (int i = 0; i < static_cast(tensors.size()) - 1; i++) { + reinterpret_cast(output->data)[i] = reinterpret_cast(tensors.at(i)->data)[0]; + } +} + +static runnerType GetKernel_Stack(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Stack(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Stack() is nullptr"); + return nullptr; + } + if (op->isScale()->Get(0)) { + auto runner = [](const std::vector &tensors) -> int { + auto input = tensors.front(); + if (input->dtype.bits == 8) { + Stack_ScaleNumber(tensors); + } else if (input->dtype.bits == 16) { + Stack_ScaleNumber(tensors); + } else if (input->dtype.bits == 32) { + Stack_ScaleNumber(tensors); + } else if (input->dtype.bits == 64) { + Stack_ScaleNumber(tensors); + } else { + MS_LOGE("StridedSlice input.dtype.bits=%d invalid, only support (8, 16, 32, 64)", input->dtype.bits); + return 1; + } + return 0; + }; + return runner; + } + std::string fid = "Stack_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_input_num" + + std::to_string(static_cast(tensors.size()) - 1) + "_axis" + std::to_string(op->axis()); + return GetKernel(fid); +} + +static runnerType GetKernel_Pad(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Pad(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Pad() is nullptr"); + return nullptr; + } + std::string fid = "Pad_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_" + + mindspore::predict::EnumNamePaddingMode(op->paddingmode()); + std::vector paddings(op->paddings()->size()); + for (size_t i = 0; i < paddings.size(); ++i) { + paddings[i] = op->paddings()->Get(i); + } + return GetKernel_Insert_vector_int32(fid, paddings); +} + +static runnerType GetKernel_Pooling(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Pooling(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Pooling() is nullptr"); + return nullptr; + } + auto H = tensors.front()->shape[NCHW_H]; + auto W = tensors.front()->shape[NCHW_W]; + auto padUp = op->padUp(); + auto padDown = op->padDown(); + auto padLeft = op->padLeft(); + auto padRight = op->padRight(); + bool useGlobal = false; + if (H == op->windowH() && W == op->windowW()) { + useGlobal = true; + } + if (op->padMode() != mindspore::predict::PadMode_VALID) { + int64_t outputHeight = tensors.back()->shape[NCHW_H]; + int64_t outputWidth = tensors.back()->shape[NCHW_W]; + + int64_t dHeight = (outputHeight - 1) * op->strideH() + op->windowH() - H; + int64_t dWidth = (outputWidth - 1) * op->strideW() + op->windowW() - W; + padUp = dHeight / 2; + padDown = dHeight - dHeight / 2; + padLeft = dWidth / 2; + padRight = dWidth - dWidth / 2; + if (padDown < 0) { + padDown = 0; + } + if (padRight < 0) { + padRight = 0; + } + } + std::string poolingMode = mindspore::predict::EnumNamesPoolMode()[op->poolingMode()]; + if (poolingMode != "MAX_POOLING" && poolingMode != "MEAN_POOLING") { + MS_LOGE("Pooling op not support poolingMode=%s", poolingMode.c_str()); + return nullptr; + } + + std::string fid; + fid += useGlobal ? "GlobalPooling" : "Pooling"; + fid += "_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype; + fid += (poolingMode == "MAX_POOLING") ? "_max" : "_avg"; + if (!useGlobal) { + fid += "_kernel" + std::to_string(op->windowH()) + std::to_string(op->windowW()); + fid += "_stride" + std::to_string(op->strideH()) + std::to_string(op->strideW()); + fid += + "_pad" + std::to_string(padUp) + std::to_string(padDown) + std::to_string(padLeft) + std::to_string(padRight); + if (op->caffeMode() && (padUp || padDown || padLeft || padRight)) fid += "_caffe"; + } + auto fun = GetKernel(fid); + if (fun == nullptr) { + MS_LOGE("GetKernel return nullptr"); + return nullptr; + } + return (opdef.isLastConv()) ? Pack_NCHW2NHWC(fun) : fun; +} + +static runnerType GetKernel_Mean(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Mean(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Mean() is nullptr"); + return nullptr; + } + std::string axis_str = ""; + for (uint32_t i = 0; i < op->axis()->size(); ++i) { + axis_str += std::to_string(op->axis()->Get(i)); + } + std::string fid = "Mean_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis" + axis_str; + fid += (op->keepDims()) ? "_keepDims" : "_notkeepDims"; + return GetKernel(fid); +} + +static runnerType GetKernel_MatMul(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_MatMul(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_MatMul() is nullptr"); + return nullptr; + } + std::string fid = "MatMul_ndimA2_ndimB2_" + opAttr.dtype; + fid += (op->transposeA()) ? "_1" : "_0"; + fid += (op->transposeB()) ? "_1" : "_0"; + return GetKernel(fid); +} + +static runnerType GetKernel_Softmax(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_SoftMax(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_SoftMax() is nullptr"); + return nullptr; + } + std::string fid = + "Softmax_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis" + std::to_string(op->axis()->Get(0)); + return GetKernel(fid); +} + +void update_shape_Activation(const std::vector &tensors, int64_t *shape, int64_t *strides) { + auto input = tensors.front(); + MS_ASSERT(input != nullptr); + for (int32_t i = 0; i < input->ndim; ++i) { + *shape *= input->shape[i]; + } + for (size_t i = 0; i < tensors.size(); ++i) { + MS_ASSERT(tensors[i] != nullptr); + tensors[i]->ndim = 1; + tensors[i]->shape = shape; + tensors[i]->strides = strides; + } +} + +static runnerType GetKernel_Activation(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Activation(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Activation() is nullptr"); + return nullptr; + } + std::string fid = + "Activation_ndim1_" + opAttr.dtype + "_" + std::string(mindspore::predict::EnumNameActivationType(op->type())); + + auto fun = GetKernel(fid); + if (fun == nullptr) { + MS_LOGE("GetKernel return nullptr"); + return nullptr; + } + auto runner = [fun](const std::vector &tensors) -> int { + int ndim[TENSOR_NUM_MAX]; + int64_t *shapes[TENSOR_NUM_MAX]; + int64_t *strides[TENSOR_NUM_MAX]; + int64_t shape = 1; + int64_t stride = 1; + + store_shape(tensors, ndim, shapes, strides, STORE_MODE); + update_shape_Activation(tensors, &shape, &stride); + fun(tensors); + store_shape(tensors, ndim, shapes, strides, RESUME_MODE); + return 0; + }; + return runner; +} + +static runnerType GetKernel_Exp(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + std::string fid = "Exp_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype; + return GetKernel(fid); +} + +static runnerType GetKernel_Cast(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + MS_ASSERT(tensors.front() != nullptr); + MS_ASSERT(tensors.back() != nullptr); + auto src_dtype = get_dtype(*tensors.front()); + auto dst_dtype = get_dtype(*tensors.back()); + std::string fid = "Cast_ndim" + std::to_string(tensors.front()->ndim) + "_" + src_dtype + "_" + dst_dtype; + return GetKernel(fid); +} + +static runnerType GetKernel_ExpandDims(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_ExpandDims(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_ExpandDims() is nullptr"); + return nullptr; + } + std::string fid = + "ExpandDims_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype + "_axis" + std::to_string(op->dim()); + return GetKernel(fid); +} + +static runnerType GetKernel_Tile(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Tile(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Tile() is nullptr"); + return nullptr; + } + std::string fid = "Tile_ndim" + std::to_string(opAttr.ndim) + "_" + opAttr.dtype; + std::vector multiples; + for (size_t i = 0; i < op->multiples()->size(); ++i) { + multiples.push_back(op->multiples()->Get(i)); + } + return GetKernel_Insert_vector_int32(fid, multiples); +} + +static runnerType GetKernel_Range(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + const OpCommonAttr opAttr(opdef, tensors); + auto op = opdef.attr_as_Range(); + if (op == nullptr) { + MS_LOGE("opdef.attr_as_Range() is nullptr"); + return nullptr; + } + std::string fid = "Range_ndim_" + opAttr.dtype; + std::vector vec = {static_cast(op->start()), static_cast(op->delta())}; + + auto f = GetFunction(fid); + if (f == nullptr) { + MS_LOGE("GetFunction returu nullptr"); + return nullptr; + } + auto runner = [f, vec](const std::vector &tensors_origin) -> int { + // remove 3 input, only remain output + const std::vector tensors = {tensors_origin.back()}; + std::vector values(vec.size() + tensors.size()); + std::vector codes(values.size()); + tvm::runtime::TVMArgsSetter setter(values.data(), codes.data()); + for (size_t i = 0; i < vec.size(); ++i) { + setter(i, vec.at(i)); + } + for (size_t i = 0; i < tensors.size(); ++i) { + setter(i + vec.size(), tensors.at(i)); + } + auto passfunc = reinterpret_cast(f); + return passfunc(values.data(), codes.data(), values.size()); + }; + return runner; +} + +using GetKernelFunType = std::function &tensors, const KernelOption &option)>; + +static const std::unordered_map g_kernel_op_list = { + {"Conv2D", GetKernel_Conv}, + {"DepthwiseConv2D", GetKernel_Conv}, + {"DeDepthwiseConv2D", GetKernel_Conv}, + {"DeConv2D", GetKernel_Conv_var}, + {"BatchNorm", GetKernel_BatchNorm}, + {"CaffeBatchNorm", GetKernel_BatchNorm}, + {"BiasAdd", GetKernel_BatchNorm}, + {"Scale", GetKernel_BatchNorm}, + {"Eltwise", GetKernel_Eltwise}, + {"Add", GetKernel_Eltwise}, + {"Sub", GetKernel_Eltwise}, + {"Mul", GetKernel_Eltwise}, + {"RealDiv", GetKernel_Eltwise}, + {"Maximum", GetKernel_Eltwise}, + {"ResizeBilinear", GetKernel_Resize}, + {"ResizeNearestNeighbor", GetKernel_Resize}, + {"Squeeze", GetKernel_DataCarry}, + {"Reshape", GetKernel_DataCarry}, + {"Shape", GetKernel_Shape}, + {"StridedSlice", GetKernel_StridedSlice}, + {"CaffeCrop", GetKernel_CaffeCrop}, + {"CaffePReLU", GetKernel_CaffePReLU}, + {"FullConnection", GetKernel_FullConnection}, + {"Power", GetKernel_Power}, + {"ArgMax", GetKernel_ArgMax}, + {"Concat", GetKernel_Concat}, + {"Stack", GetKernel_Stack}, + {"Pad", GetKernel_Pad}, + {"Pooling", GetKernel_Pooling}, + {"Mean", GetKernel_Mean}, + {"MatMul", GetKernel_MatMul}, + {"SoftMax", GetKernel_Softmax}, + {"Activation", GetKernel_Activation}, + {"Exp", GetKernel_Exp}, + {"Cast", GetKernel_Cast}, + {"ExpandDims", GetKernel_ExpandDims}, + {"Tile", GetKernel_Tile}, + {"Range", GetKernel_Range}, +}; + +GetKernelFunType Get_GetKernelFun(const std::string &optype) { + auto it = g_kernel_op_list.find(optype); + return (it != g_kernel_op_list.end() ? it->second : nullptr); +} +} // namespace auto_tensor + +runnerType GetKernel(const mindspore::predict::OpDef &opdef, const std::vector &tensors, + const KernelOption &option) { + std::string optype = mindspore::predict::EnumNameOpT(opdef.attr_type()); + auto GetKernelFun = auto_tensor::Get_GetKernelFun(optype); + if (GetKernelFun != nullptr) { + return GetKernelFun(opdef, tensors, option); + } else { + return nullptr; + } +} + +int CallKernel(const std::string &fid, const std::vector &tensors) { + auto runner = GetKernel(fid); + return runner(tensors); +} + +#endif // LITE_RUNTIME_ON diff --git a/predict/module/tvm_kernel/lite/src/api/tvm_op_module.cc b/predict/module/tvm_kernel/lite/src/api/tvm_op_module.cc new file mode 100644 index 00000000000..c6a6fc93f5d --- /dev/null +++ b/predict/module/tvm_kernel/lite/src/api/tvm_op_module.cc @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this ${file} except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "src/api/tvm_op_module.h" +#include "common/op_utils.h" +#include "lite/api/km_api.h" +#include "src/op.h" +namespace mindspore { +namespace predict { +using OpFunc = std::function &)>; + +class TVMOperator : public OpBase { + public: + explicit TVMOperator(OpFunc func) : opfunc(std::move(func)) {} + ~TVMOperator() override = default; + int Init(const std::vector &inputs, const std::vector &outputs) override { return 0; } + int Execute(const std::vector &inputs, const std::vector &outputs) override { + std::vector dlT; + for (auto input : inputs) { + MS_ASSERT(input != nullptr); + dlT.push_back(input->GetDLTensor()); + } + + for (auto output : outputs) { + MS_ASSERT(output != nullptr); + dlT.push_back(output->GetDLTensor()); + } + return opfunc(dlT); + } + + static OpBase *CreateOp(const std::vector &inputs, const std::vector &outputs, const OpDef &opDef, + const Context &ctx, const OpDesc &desc) { + std::vector dlT; + for (auto input : inputs) { + MS_ASSERT(input != nullptr); + dlT.push_back(input->GetDLTensor()); + } + + for (auto output : outputs) { + MS_ASSERT(output != nullptr); + dlT.push_back(output->GetDLTensor()); + } + + KernelOption option; + option.numThreads = ctx.threadNum; + OpFunc opFunc = GetKernel(opDef, dlT, option); + if (opFunc != nullptr) { + auto op = std::unique_ptr(new (std::nothrow) TVMOperator(opFunc)); + if (op == nullptr) { + MS_LOGE("new TVMOperator failed"); + } + return op.release(); + } + return nullptr; + } + + private: + OpFunc opfunc; + std::vector dltensors; +}; + +TVMOpRegistry::TVMOpRegistry() = default; + +mindspore::predict::OpCreator TVMOpRegistry::GetOpCreator(const mindspore::predict::OpDesc &desc) { + return TVMOperator::CreateOp; +} + +OpRegistry *TVMOpModule::GetInstance() { + static TVMOpRegistry tvmOpRegistry; + return &tvmOpRegistry; +} + +static TVMOpModule tvmOpModule; + +static ModuleRegistrar> g_tvmOpReg(MODULE_REG_NAME_OP_REGISTRY, tvmOpModule); +} // namespace predict +} // namespace mindspore diff --git a/predict/module/tvm_kernel/lite/src/api/tvm_op_module.h b/predict/module/tvm_kernel/lite/src/api/tvm_op_module.h new file mode 100644 index 00000000000..3f0f33142dd --- /dev/null +++ b/predict/module/tvm_kernel/lite/src/api/tvm_op_module.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this ${file} except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PREDICT_MODULE_TVM_KERNEL_LITE_SRC_API_TVM_OP_MODULE_H_ +#define PREDICT_MODULE_TVM_KERNEL_LITE_SRC_API_TVM_OP_MODULE_H_ + +#include "src/op_registry.h" +namespace mindspore { +namespace predict { +class TVMOpRegistry : public OpRegistry { + public: + TVMOpRegistry(); + OpCreator GetOpCreator(const OpDesc &desc) override; +}; + +class TVMOpModule : public Module { + public: + OpRegistry *GetInstance() override; +}; +} // namespace predict +} // namespace mindspore +#endif // PREDICT_MODULE_TVM_KERNEL_LITE_SRC_API_TVM_OP_MODULE_H_ diff --git a/predict/module/tvm_kernel/lite/src/codegen/llvm/lite_rtfunc_reset.cc b/predict/module/tvm_kernel/lite/src/codegen/llvm/lite_rtfunc_reset.cc new file mode 100644 index 00000000000..df19676b406 --- /dev/null +++ b/predict/module/tvm_kernel/lite/src/codegen/llvm/lite_rtfunc_reset.cc @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this ${file} except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "tvm/api_registry.h" + +namespace tvm { +namespace codegen { +class LiteRTFuncReseter { + public: + LiteRTFuncReseter() {} + ~LiteRTFuncReseter() {} + int InsertFuncPair(std::string sfunc, std::string dfunc) { + CHECK_NE(sfunc.size(), 0); + CHECK_NE(dfunc.size(), 0); + func_map_[sfunc] = dfunc; + return 0; + } + + /* + * the llvm::Function::Create need a longe life scope const char* as input + * so here not return block life scopte tmp std::string. + */ + const char* GetResetFunc(std::string sfunc) { + CHECK_NE(sfunc.size(), 0); + auto it_dfunc = func_map_.find(sfunc); + if (it_dfunc != func_map_.end()) { + return it_dfunc->second.c_str(); + } else { + func_map_[sfunc] = sfunc; + return func_map_[sfunc].c_str(); + } + } + + /* + * not real delete item paire, just set orig function pair + */ + int DeleteFuncPair(std::string sfunc) { + CHECK_NE(sfunc.size(), 0); + func_map_[sfunc] = sfunc; + return 0; + } + static LiteRTFuncReseter* GetRTFuncReseter() { + static LiteRTFuncReseter inst; + return &inst; + } + + private: + std::map func_map_; +}; + +TVM_REGISTER_API("codegen.SetRTFuncTransPair").set_body([](const TVMArgs& targs, TVMRetValue* rv) { + *rv = LiteRTFuncReseter::GetRTFuncReseter()->InsertFuncPair(targs[0], targs[1]); +}); + +TVM_REGISTER_API("codegen.DelRTFuncTransPair").set_body([](const TVMArgs& targs, TVMRetValue* rv) { + *rv = LiteRTFuncReseter::GetRTFuncReseter()->DeleteFuncPair(targs[0]); +}); + +/* + * now no operator=(const char* ) provide for TVMRetValue + * here using explicit operator call function to make sure not using operator=(int) + */ +TVM_REGISTER_API("codegen.GetTransRTFunc").set_body([](const TVMArgs& targs, TVMRetValue* rv) { + (*rv).operator=( + reinterpret_cast(const_cast(LiteRTFuncReseter::GetRTFuncReseter()->GetResetFunc(targs[0])))); +}); +} // namespace codegen +} // namespace tvm diff --git a/predict/schema/inner/README b/predict/schema/inner/README new file mode 100644 index 00000000000..f397efe4c57 --- /dev/null +++ b/predict/schema/inner/README @@ -0,0 +1 @@ +for flatbuf auto generated file diff --git a/predict/schema/ms.fbs b/predict/schema/ms.fbs new file mode 100644 index 00000000000..f66abf8a866 --- /dev/null +++ b/predict/schema/ms.fbs @@ -0,0 +1,153 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +include "op.fbs"; + +namespace mindspore.predict; + +enum DataType : int { + DT_FLOAT = 0, + DT_FLOAT16 = 1, + DT_INT8 = 2, + DT_INT32 = 3, + DT_UINT8 = 4, + DT_UINT32 = 8, + DT_UNDEFINED = 16 +} + +enum Format : int { + NCHW = 0, + NHWC, + NC4HW4 = 100, + NUM_OF_FORMAT +} + +enum MSConst: int { + WEIGHT_REFCOUNT = 999 +} + +table QuantizationDef { + // Quantized value q, corresponding float value r: + // r = scale * (q - zero_point), where scale = (rmax - rmin) / (qmax - qmin) + min: [float]; + max: [float]; + scale: [float]; + zero_point: [long]; + + // Tensor shape of the specifies dimension. + dimension: int; +} + +table TensorDef { + // data type + dataType: DataType; + // shape + dims: [int]; + format: Format; + refCount: int; + offset: int; + data: [ubyte]; + quantization: QuantizationDef; +} + +union OpT { + Concat, + SoftMax, + Activation, + Conv2D, + FusedBatchNorm, + CaffeBatchNorm, + Squeeze, + BiasAdd, + Pooling, + DepthwiseConv2D, + DeDepthwiseConv2D, + Resize, + DetectionPostProcess, + FullConnection, + Mean, + DeConv2D, + Scale, + Reshape, + Eltwise, + NetOutput, + Add, + MatMul, + StridedSlice, + Power, + Slice, + Stack, + Mul, + Pad, + Maximum, + CaffePReLU, + ArgMax, + Exp, + CaffeCrop, + Range, + ExpandDims, + Tile, + Cast +// Split +} + +enum QuantType: int { + QUANT_NONE, + QUANT_INT8 +} + +table OpDef { + name: string; + attr: OpT; + inputIndex: [uint]; + outputIndex: [uint]; + isLastConv: bool; + quantType: QuantType = QUANT_NONE; +} + + +enum FmkType: int { + TF, + CAFFE +} + +table NodeDef { + fmkType: FmkType; + opDef: OpDef; +} + + +table SubGraphDef { + name: string; + inputIndex: [uint]; + outputIndex: [uint]; + mempoolSize: uint; + nodes: [NodeDef]; + allTensors: [TensorDef]; // weight + input + output +} + +table MempoolCfg { + size: uint; + shiftFactor: uint; +} + +table GraphDef { + name: string; + mempoolCfg: MempoolCfg; + subgraphs: [SubGraphDef]; +} + +root_type GraphDef; diff --git a/predict/schema/op.fbs b/predict/schema/op.fbs new file mode 100755 index 00000000000..4d01bb9c3bc --- /dev/null +++ b/predict/schema/op.fbs @@ -0,0 +1,351 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace mindspore.predict; + +enum ResizeMethod: byte { + UNKNOW = -1, + BILINEAR = 0, + NEAREST_NEIGHBOR = 1 +} + +enum DataFormatType : byte { + UNKNOW = -1, + NCHW = 0, + NHWC = 1, + HWC = 2, // for input image or resize + CHW = 3, // for input image or resize +} + +enum ActivationType : byte { + NO_ACTIVATION = 0, + RELU = 1, + SIGMOID = 2, + RELU6 = 3, + ELU = 4, + LEAKY_RELU = 5, + ABS = 6, + RELU1 = 7, + SOFTSIGN = 8, + SOFTPLUS = 9, + TANH = 10, + UNKNOW = 11 +} + +enum PoolMode : byte { + MAX_POOLING = 0, + MEAN_POOLING = 1, + GLOBAL_POOING = 2 +} + +enum EltwiseMode : byte { + PROD = 0, + SUM = 1, + MAXIMUM = 2 +} + +enum PadMode : byte { + NOTSET=0, + SAME=1, + VALID=2, + CAFFE_CEIL_NEW=4 +} + +enum PaddingMode : byte { + CONSTANT = 0, + REFLECT = 1, + SYMMETRIC = 2, + MODE_RESERVED = 3 +} + +table Pad { + paddingmode: PaddingMode; + paddings: [int]; +} + +table Maximum { + format: DataFormatType = 0; +} + +table Concat { + axis: int; + n: int; +} + +table SoftMax { + axis: [int]; +} + +table Activation { + type: ActivationType = 0; +} + +table Conv2D { + format: DataFormatType = 0; + group: int; + channelIn: int; + channelOut: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + +table FusedBatchNorm { + epsilon: float; // eg. epsilon=0.001 +} + +table CaffeBatchNorm { + epsilon: float; // eg. epsilon=0.001 +} + +table Squeeze { + axis: [int]; +} + +table BiasAdd { + axis: [int]; +} + +table Pooling { + format: DataFormatType = 0; + poolingMode: PoolMode; + windowW: int; + windowH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + caffeMode: bool = false; +} + +table DepthwiseConv2D { + format: DataFormatType = 0; + channelIn: int; + channelMultiplier: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + +table DeDepthwiseConv2D { + format: DataFormatType = 0; + channelIn: int; + channelMultiplier: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + + +table Resize { + format: DataFormatType = 0; + method: ResizeMethod; + newHeight: long; + newWidth: long; + alignCorners: bool = false; + preserveAspectRatio: bool = false; +} + +table DetectionPostProcess { + format: DataFormatType = 0; + inputSize: int; + hScale: float; + wScale: float; + xScale: float; + yScale: float; + NmsIouThreshold: float; + NmsScoreThreshold: float; + MaxDetections: long; + DetectionsPreClass: long; + MaxClassesPreDetection: long; + NumClasses: long; + UseRegularNms: bool; +} + +table FullConnection { + format: DataFormatType = 0; + hasBias: bool; + axis: int; +} + +// Mean(input_tensor, axis, keep_dims) +table Mean { + axis: [int]; + keepDims: bool = false; +} + +table DeConv2D { + format: DataFormatType = 0; + group: int; + channelIn: int; + channelOut: int; + kernelW: int; + kernelH: int; + strideW: int; + strideH: int; + padMode: PadMode; + padUp: int; + padDown: int; + padLeft: int; + padRight: int; + dilateW: int; + dilateH: int; + hasBias: bool = false; + activationType: ActivationType = 0; +} + +table Scale { + format: DataFormatType = 0; +} + +table Eltwise { + format: DataFormatType = 0; + mode: EltwiseMode; +} + +table Add { + format: DataFormatType = 0; +} + +table Slice { + format: DataFormatType = 0; + begin: [int]; + end: [int]; + stride: [int]; +} + +table Mul { +} + +table Exp { +} + +table Reshape { + format: DataFormatType = 0; + shape: [long]; +} + +table Power { + power: float; + scale: float; + shift: float; +} + +table ArgMax { + axis: int; + outMaxValue: bool; + topK: int; + keepDims: bool; + axisType: int; +} + +table NetOutput { + format: DataFormatType = 0; +} + +table MatMul { + transposeA : bool = false; + transposeB : bool = false; +} + +table CaffePReLU { + channelShared : bool = false; +} + +table StridedSlice { + beginMask: int; + endMask: int; + ellipsisMask: int; + newAxisMask: int; + shrinkAxisMask: int; + begin: [int]; + end: [int]; + stride: [int]; + isScale: [int]; +} + +table Stack { + axis: int; + n: int; + isScale: [int]; +} + +table Range { + start: int; + limit: int; + delta: int; +} + +table ExpandDims { + dim: int; +} + +table Tile { + multiples: [int]; +} + +table Cast { + srcT: int; + dstT: int; +} + +table Split { + numberSplit: int; + sizeSplits: [int]; + splitDim: int; +} + +table CaffeCrop { + axis : long; + offsets : [long]; +} + +table Permute { + order: [long]; +} diff --git a/predict/src/CMakeLists.txt b/predict/src/CMakeLists.txt new file mode 100644 index 00000000000..c32c047c828 --- /dev/null +++ b/predict/src/CMakeLists.txt @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.12) +project(mspredict) + +set(CMAKE_CXX_STANDARD 11) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../module/tvm_kernel/incubator-tvm/3rdparty/dlpack/include/) + +link_directories(${CMAKE_CURRENT_SOURCE_DIR}/../output/lib/) + +if (ENABLE_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -o0 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined") +endif() + +set(MSPREDICT_SRC + runtime/allocator.cc + runtime/allocator.h + runtime/thread_pool.cc + runtime/thread_pool.h + runtime/workspace_pool.cc + runtime/workspace_pool.h + runtime/runtime_api.cc + runtime/runtime_api.h + context.cc + graph.cc + graph.h + graph_execution.cc + graph_execution.h + node.cc + node.h + op.cc + op.h + op_factory.cc + op_factory.h + op_registry.cc + op_registry.h + session.cc + tensor.cc + ${CMAKE_CURRENT_SOURCE_DIR}/operator/cpu/common/op_func_comm.cc) + +set(MSPREDICT_SRC ${MSPREDICT_SRC} + ${CMAKE_CURRENT_SOURCE_DIR}/../common/graph_util.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../common/utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../common/mslog.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../common/module_registry.cc) + +add_library(mspredict SHARED ${MSPREDICT_SRC}) + +if(ENABLE_PREDICT_ARM64 OR ENABLE_PREDICT_ARM32) + target_link_libraries(mspredict android log tvm_kernel libsecurec.a) +else() + target_link_libraries(mspredict pthread tvm_kernel libsecurec.a) +endif() + +if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + if(ENABLE_PREDICT_ARM64) + add_custom_command(TARGET mspredict POST_BUILD + COMMAND ${ANDROID_NDK}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/aarch64-linux-android/bin/strip "${PREDICT_BUILD_DIR}/src/libmspredict.so" + COMMAND ${ANDROID_NDK}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/aarch64-linux-android/bin/strip "${PREDICT_BUILD_DIR}/module/tvm_kernel/lite/libtvm_kernel.so" + ) + else() + add_custom_command(TARGET mspredict POST_BUILD + COMMAND strip "${PREDICT_BUILD_DIR}/src/libmspredict.so" + COMMAND strip "${PREDICT_BUILD_DIR}/module/tvm_kernel/lite/libtvm_kernel.so" + ) + endif() +endif() + +add_dependencies(mspredict tvm_kernel) +add_dependencies(mspredict securec) +add_dependencies(mspredict gtest) + +add_custom_command(TARGET mspredict POST_BUILD + COMMAND mkdir -pv ${PREDICT_DIR}/output/lib + COMMAND cp ${PREDICT_BUILD_DIR}/src/libmspredict.so ${PREDICT_DIR}/output/lib/ + COMMAND cp ${PREDICT_BUILD_DIR}/module/tvm_kernel/lite/libtvm_kernel.so ${PREDICT_DIR}/output/lib/ + COMMAND mkdir -pv ${PREDICT_DIR}/output/include + COMMAND cp -r ${PREDICT_DIR}/include/* ${PREDICT_DIR}/output/include + COMMAND mkdir -pv ${PREDICT_DIR}/output/include/schema/inner + COMMAND cp ${PREDICT_DIR}/schema/ms_generated.h ${PREDICT_DIR}/output/include/schema/inner + COMMAND cp ${PREDICT_DIR}/schema/op_generated.h ${PREDICT_DIR}/output/include/schema/inner + COMMAND mkdir -pv ${PREDICT_DIR}/output/include/dlpack/ + COMMAND cp ${PREDICT_DIR}/module/tvm_kernel/incubator-tvm/3rdparty/dlpack/include/dlpack/dlpack.h ${PREDICT_DIR}/output/include/dlpack/) diff --git a/predict/src/context.cc b/predict/src/context.cc new file mode 100644 index 00000000000..45ec719db6a --- /dev/null +++ b/predict/src/context.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/context.h" +#include "include/tensor.h" +#include "src/runtime/allocator.h" + +namespace mindspore { +namespace predict { +Context::Context() { allocator = Allocator::Create(); } + +Context::~Context() {} + +Context::Context(int threadNum, std::shared_ptr allocator, DLContext deviceCtx) { + this->allocator = allocator; + this->threadNum = threadNum; + this->deviceCtx = deviceCtx; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/graph.cc b/predict/src/graph.cc new file mode 100644 index 00000000000..5e1aab6a56e --- /dev/null +++ b/predict/src/graph.cc @@ -0,0 +1,378 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/graph.h" +#include +#include +#include +#include +#include "schema/ms_generated.h" +#include "common/graph_util.h" +#include "common/mslog.h" +#include "include/errorcode.h" +#include "src/graph_execution.h" + +namespace mindspore { +namespace predict { +static const uint32_t G_MAX_OP_COUNT = 10000; + +Graph *Graph::CreateFromBuf(const char *buf, size_t size, const Context &ctx) { + if (buf == nullptr) { + MS_LOGE("the input buffer is nullptr"); + return nullptr; + } + + flatbuffers::Verifier verify((const uint8_t *)buf, size); + if (!VerifyGraphDefBuffer(verify)) { + MS_LOGE("the buffer is invalid and fail to create graph"); + return nullptr; + } + + auto graphDef = GetGraphDef(buf); + std::unique_ptr graph(new (std::nothrow) Graph()); + if (graph == nullptr) { + MS_LOGE("graph malloc fail"); + return nullptr; + } + auto ret = graph->Build(*graphDef, ctx); + if (ret != RET_OK) { + MS_LOGE("build graph fail"); + return nullptr; + } + return graph.release(); +} + +Graph::Graph() = default; + +Graph::~Graph() { + for (auto &subgraph : subgraphs) { + delete subgraph; + } + subgraphs.clear(); +} + +int Graph::Build(const GraphDef &graphDef, const Context &ctx) { + MS_ASSERT(graphDef.subgraphs() != nullptr); + for (size_t i = 0; i < graphDef.subgraphs()->size(); i++) { + MS_ASSERT(graphDef.subgraphs()->GetAs(i) != nullptr); + SubGraph *subGraph = SubGraph::CreateSubGraph(*(graphDef.subgraphs()->GetAs(i)), ctx); + if (subGraph == nullptr) { + MS_LOGE("converter subgraph failed"); + return RET_ERROR; + } + subgraphs.push_back(subGraph); + auto subDepends = subGraph->GetDepends(); + depends.insert(subDepends.begin(), subDepends.end()); + } + + auto iter = depends.begin(); + while (iter != depends.end()) { + if (iter->second.empty()) { + readyQue.push_back(iter->first); + iter = depends.erase(iter); + } else { + iter++; + } + } + + return RET_OK; +} + +std::vector Graph::GetInputs() { + MS_ASSERT(subgraphs.front() != nullptr); + return subgraphs.front()->GetInputs(); +} + +std::vector Graph::GetOutputs() { + MS_ASSERT(subgraphs.back() != nullptr); + return subgraphs.back()->GetOutputs(); +} + +std::map> &Graph::GetOutputsMap() { + MS_ASSERT(subgraphs.back() != nullptr); + return subgraphs.back()->GetOutputsMap(); +} + +void Graph::FreeAllTensors() { + for (auto iter : subgraphs) { + iter->FreeAllTensors(); + } +} + +std::vector *Graph::Subgraphs() { return &subgraphs; } + +SubGraph::SubGraph() = default; + +SubGraph::~SubGraph() { + for (auto iter = nodes.begin(); iter != nodes.end();) { + if (iter->second != nullptr) { + delete iter->second; + } + iter = nodes.erase(iter); + } + nodes.clear(); + + for (auto &allTensor : allTensors) { + if (allTensor != nullptr) { + delete allTensor; + } + } + allTensors.clear(); +} + +SubGraph *SubGraph::CreateSubGraph(const SubGraphDef &subGraphDef, const Context &ctx) { + std::unique_ptr subGraph(new (std::nothrow) SubGraph()); + if (subGraph == nullptr) { + MS_LOGE("subGraph malloc fail"); + return nullptr; + } + + auto ret = subGraph->Build(subGraphDef, ctx); + if (ret != RET_OK) { + MS_LOGE("subGraph Build fail"); + return nullptr; + } + + return subGraph.release(); +} + +int SubGraph::Build(const SubGraphDef &subGraphDef, const Context &ctx) { + int ret; + MS_ASSERT(subGraphDef.inputIndex() != nullptr); + ret = ConverterIndex(*(subGraphDef.inputIndex()), &inputIndices); + if (ret != RET_OK) { + MS_LOGE("ConverterIndex failed: %d", ret); + return ret; + } + MS_LOGD("converter inputIndex succ"); + + MS_ASSERT(subGraphDef.outputIndex() != nullptr); + ret = ConverterIndex(*(subGraphDef.outputIndex()), &outputIndices); + if (ret != RET_OK) { + MS_LOGE("ConverterIndex failed: %d", ret); + return ret; + } + MS_LOGD("converter outputIndex succ"); + MS_ASSERT(subGraphDef.allTensors() != nullptr); + ret = ConverterAllTensor(*(subGraphDef.allTensors())); + if (ret != RET_OK) { + MS_LOGE("ConverterAllTensor failed: %d", ret); + return ret; + } + MS_LOGD("converter AllTensor succ"); + MS_ASSERT(subGraphDef.nodes() != nullptr); + ret = ConverterNodes(*(subGraphDef.nodes()), ctx); + if (ret != RET_OK) { + MS_LOGE("ConverterNodes failed: %d", ret); + return ret; + } + MS_LOGD("converter nodes succ"); + + ret = ConverterEdges(subGraphDef); + if (ret != RET_OK) { + MS_LOGE("ConverterEdges failed: %d", ret); + return ret; + } + MS_LOGD("converter edges succ"); + + ret = InitOutputsMap(); + if (ret != RET_OK) { + MS_LOGE("InitOutputsMap failed: %d", ret); + return ret; + } + MS_LOGD("init outputs map succ"); + + MS_LOGD("build graph succ"); + return RET_OK; +} + +int SubGraph::ConverterIndex(const flatbuffers::Vector &srcIndex, std::vector *dstIndex) { + if (dstIndex == nullptr) { + MS_LOGE("input dstIndex is nullptr"); + return RET_PARAM_INVALID; + } + dstIndex->resize(srcIndex.size()); + std::copy(srcIndex.begin(), srcIndex.end(), dstIndex->begin()); + return RET_OK; +} + +int SubGraph::ConverterAllTensor(const flatbuffers::Vector> &srcTensors) { + uint32_t tensorsSize = srcTensors.size(); + + allTensors.clear(); + allTensors.reserve(tensorsSize); + for (uint32_t i = 0; i < tensorsSize; i++) { + auto tensorDef = srcTensors.GetAs(i); + if (tensorDef == nullptr) { + MS_LOGE("%ud th tensordef is null", i); + return RET_ERROR; + } + auto tensor = Tensor::CopyFromTensorDef(*tensorDef); + if (tensor == nullptr) { + return RET_ERROR; + } + allTensors.push_back(tensor); + } + + return RET_OK; +} + +int SubGraph::ConverterNodes(const flatbuffers::Vector> &nodeDefs, const Context &ctx) { + uint32_t opCount = nodeDefs.size(); + // for dfx + if (opCount > G_MAX_OP_COUNT) { + MS_LOGE("opCount(%u) bigger than maxOpCount(%u)", opCount, G_MAX_OP_COUNT); + return RET_ERROR; + } + + nodes.clear(); + + for (uint32_t i = 0; i < opCount; i++) { + auto nodeDef = nodeDefs.GetAs(i); + MS_ASSERT(nodeDef != nullptr); + auto node = std::unique_ptr(new (std::nothrow) Node(nodeDef)); + if (node == nullptr) { + MS_LOGE("new node failed"); + return RET_NULL_PTR; + } + + node->SetTensors(*nodeDef, allTensors); + + auto ret = node->InitOp(*(nodeDef->opDef()), ctx); + if (ret != RET_OK) { + MS_LOGE("node (%s) InitOP failed. ret:%d", node->ID().c_str(), ret); + return ret; + } + + auto nodeId = node->ID(); + nodes[nodeId] = node.release(); + MS_LOGD("add node succ, id:%s", nodeId.c_str()); + } + + return RET_OK; +} + +int SubGraph::ConverterEdges(const SubGraphDef &subGraphDef) { + auto opGraph = OpGraph::Build(subGraphDef); + if (opGraph == nullptr) { + MS_LOGE("opGraph Build fail"); + return RET_ERROR; + } + + for (auto nodeIter : nodes) { + auto node = opGraph->GetNode(nodeIter.first); + if (node == nullptr) { + MS_LOGI("node %s not found", nodeIter.first.c_str()); + continue; + } + for (const auto &edge : node->GetAllInEdge()) { + MS_ASSERT(nodeIter.second != nullptr); + nodeIter.second->AddInEdge(GetNode(edge)); + } + for (const auto &edge : node->GetAllOutEdge()) { + MS_ASSERT(nodeIter.second != nullptr); + nodeIter.second->AddOutEdge(GetNode(edge)); + } + } + delete opGraph; + return RET_OK; +} + +int SubGraph::InitOutputsMap() { + if (nodes.empty()) { + MS_LOGE("nodes are empty"); + return RET_ERROR; + } + for (auto node : nodes) { + NODE_ID realNodeName = node.second->ID(); + MS_ASSERT(node.second != nullptr); + if (node.second->GetAllOutEdges().empty()) { + auto nodeType = node.second->Type(); + if (nodeType == "Nhwc2Nchw" || nodeType == "Nchw2Nhwc") { + auto dependNode = *(this->GetDepends().at(this->GetNode(realNodeName)).begin()); + realNodeName = dependNode->ID(); + } + this->outputsMap.emplace( + std::pair>(realNodeName, node.second->GetOutputTensors())); + } + } + return RET_OK; +} + +std::unordered_map> SubGraph::GetDepends() { + std::unordered_map> depends; + for (auto nodeIter : nodes) { + MS_ASSERT(nodeIter.second != nullptr); + depends[nodeIter.second] = nodeIter.second->GetAllInEdges(); + } + return depends; +} + +Node *SubGraph::GetNode(const NODE_ID &id) { + auto node = nodes.find(id); + if (node == nodes.end()) { + return nullptr; + } + return node->second; +} + +std::vector SubGraph::GetInputs() { + std::vector inputTensor; + inputTensor.resize(inputIndices.size()); + std::transform(inputIndices.begin(), inputIndices.end(), inputTensor.begin(), + [this](int i) { return this->allTensors[i]; }); + + return inputTensor; +} + +std::vector SubGraph::GetOutputs() { + std::vector outputTensor; + outputTensor.resize(outputIndices.size()); + std::transform(outputIndices.begin(), outputIndices.end(), outputTensor.begin(), + [this](int i) { return this->allTensors[i]; }); + + return outputTensor; +} + +std::map> &SubGraph::GetOutputsMap() { return outputsMap; } + +void SubGraph::FreeAllTensors() { + for (auto &allTensor : allTensors) { + if (allTensor != nullptr) { + auto refcount = allTensor->RefCount(); + if (refcount != MSConst_WEIGHT_REFCOUNT) { + allTensor->DefRef(refcount); + allTensor->FreeData(); + } + } + } +} + +const std::vector *SubGraph::GetInputIndices() const { return &inputIndices; } + +const std::vector *SubGraph::GetOutputIndices() const { return &outputIndices; } + +bool SubGraph::IsInputIndex(uint32_t i) { + auto iter = std::find(inputIndices.begin(), inputIndices.end(), i); + return !(iter == inputIndices.end()); +} + +bool SubGraph::IsOutputIndex(uint32_t i) { + auto iter = std::find(outputIndices.begin(), outputIndices.end(), i); + return !(iter == outputIndices.end()); +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/graph.h b/predict/src/graph.h new file mode 100755 index 00000000000..f02c46f94ea --- /dev/null +++ b/predict/src/graph.h @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_GRAPH_H_ +#define PREDICT_SRC_GRAPH_H_ + +#include +#include +#include +#include +#include +#include +#include "common/utils.h" +#include "common/graph_util.h" +#include "include/tensor.h" +#include "src/node.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +class SubGraph { + public: + SubGraph(); + ~SubGraph(); + static SubGraph *CreateSubGraph(const SubGraphDef &subGraphDef, const Context &ctx); + int Build(const SubGraphDef &subGraphDef, const Context &ctx); + bool IsInputIndex(uint32_t i); + bool IsOutputIndex(uint32_t i); + + const std::vector *GetInputIndices() const; + const std::vector *GetOutputIndices() const; + + std::vector GetInputs(); + std::vector GetOutputs(); + std::map> &GetOutputsMap(); + void FreeAllTensors(); + + Node *GetNode(const NODE_ID &id); + + std::unordered_map> GetDepends(); + + private: + int ConverterIndex(const flatbuffers::Vector &srcIndex, std::vector *dstIndex); + + int ConverterAllTensor(const flatbuffers::Vector> &srcTensors); + + int ConverterNodes(const flatbuffers::Vector> &opDefs, const Context &ctx); + + int ConverterEdges(const SubGraphDef &subGraphDef); + + int InitOutputsMap(); + + protected: + std::unordered_map nodes; + std::vector inputIndices; + std::vector outputIndices; + std::vector allTensors; // weight + input + output + std::map> outputsMap; +}; + +class MSPREDICT_API Graph { + public: + Graph(); + ~Graph(); + static Graph *CreateFromBuf(const char *buf, size_t size, const Context &ctx); + + std::vector GetInputs(); + std::vector GetOutputs(); + + std::map> &GetOutputsMap(); + + void FreeAllTensors(); + + int Build(const GraphDef &def, const Context &ctx); + std::vector *Subgraphs(); + + protected: + friend class GraphExecution; + + std::vector subgraphs; + std::unordered_map> depends; // records the dependencies + std::deque readyQue; // the nodes which can execute without any dependencies +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_GRAPH_H_ diff --git a/predict/src/graph_execution.cc b/predict/src/graph_execution.cc new file mode 100644 index 00000000000..4a71d4359ae --- /dev/null +++ b/predict/src/graph_execution.cc @@ -0,0 +1,293 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/graph_execution.h" +#include +#include +#include + +namespace mindspore { +namespace predict { +GraphExecution::GraphExecution(const Context &ctx) : graph(nullptr), _ctx(ctx) {} +GraphExecution::GraphExecution(const Context &ctx, Graph *staticGraph) : _ctx(ctx) { + graph = staticGraph; + if (graph != nullptr) { + depends = graph->depends; + readyQue = graph->readyQue; + outputTensors = graph->GetOutputs(); + inputTensors = graph->GetInputs(); + } +} + +GraphExecution::~GraphExecution() = default; + +int GraphExecution::TransInputDataToNc4hw4(const Tensor &src, Tensor *dst) { + MS_ASSERT(dst != nullptr); + if (dst->GetData() == nullptr) { + auto ret = dst->MallocData(nullptr, MSConst_WEIGHT_REFCOUNT); + if (ret != RET_OK) { + MS_LOGE("Malloc inputTensors failed: %d", ret); + return ret; + } + } + auto ret = NchwToNc4hw4(&src, dst); + if (ret != RET_OK) { + MS_LOGE("NchwToNc4hw4 failed"); + return ret; + } + return RET_OK; +} + +int GraphExecution::SetInputTensors(const std::vector &inputs) { + size_t num = inputs.size(); + if (num != inputTensors.size()) { + MS_LOGE("input num %zu != model input num %zu", num, inputTensors.size()); + return RET_INPUT_TENSOR_ERROR; + } + + for (size_t i = 0; i < num; i++) { + MS_ASSERT(inputs[i] != nullptr); + // The input Tensor desc must be equivalent with the model tensor + if (inputs[i]->GetData() == nullptr) { + MS_LOGE("input tensor data is null!"); + return RET_INPUT_TENSOR_ERROR; + } + if (inputTensors[i] == nullptr) { + MS_LOGE("inputTensors[%zu] is nullptr", i); + return RET_ERROR; + } + + if (!inputs[i]->CompareShape(*inputTensors[i])) { + MS_LOGE("tensor shape in graph and executor are different!"); + return RET_INPUT_TENSOR_ERROR; + } + + if (inputs[i]->GetDataType() != inputTensors[i]->GetDataType()) { + MS_LOGE("tensor datatype in graph and executor are different!"); + return RET_INPUT_TENSOR_ERROR; + } + + if (inputs[i]->GetFormat() != Format_NCHW) { + MS_LOGE("input format not support. only nchw is supported now"); + return RET_INPUT_TENSOR_ERROR; + } + + if (inputs[i]->GetFormat() == inputTensors[i]->GetFormat()) { + auto data = inputs[i]->GetData(); + if (data == nullptr) { + MS_LOGE("data of input tensor is null!"); + return RET_INPUT_TENSOR_ERROR; + } + inputTensors[i]->SetData(data); + } else if (inputTensors[i]->GetFormat() == Format_NC4HW4) { + auto ret = TransInputDataToNc4hw4(*inputs[i], inputTensors[i]); + if (ret != RET_OK) { + MS_LOGE("TransInputDataToNc4hw4 failed"); + return ret; + } + } else { + MS_LOGE("graphDef inputTensors format is invalid: %d", inputTensors[i]->GetFormat()); + return RET_ERROR; + } + } + return RET_OK; +} + +int GraphExecution::MallocOutput() { + for (auto tensor : outputTensors) { + auto ret = tensor->MallocData(); + if (ret != RET_OK) { + MS_LOGE("malloc output data failed"); + return RET_ERROR; + } + } + return RET_OK; +} + +void GraphExecution::FreeTensors(std::vector *tensors) { + for (auto &tensor : (*tensors)) { + delete tensor; + } + tensors->clear(); +} + +void GraphExecution::FreeOutputMap(std::map> *map) { + MS_ASSERT(map != nullptr); + for (auto &m : *map) { + FreeTensors(&(m.second)); + } + map->clear(); +} + +int GraphExecution::CopyOutputTensors(const std::vector &refOutputs, std::vector *outputs) { + for (auto tensor : refOutputs) { + if (tensor == nullptr) { + MS_LOGE("tensor in refOutputs is nullptr"); + return RET_INPUT_TENSOR_ERROR; + } + std::unique_ptr t(new Tensor(*tensor)); + if (t == nullptr) { + MS_LOGE("new Tensor failed."); + if (outputs != nullptr) { + FreeTensors(outputs); + } + return RET_ERROR; + } + + if (tensor->GetFormat() == Format_NC4HW4) { + t->SetFormat(Format_NCHW); + auto ret = t->MallocData(); + if (ret != RET_OK) { + MS_LOGE("malloc data failed.") + FreeTensors(outputs); + return ret; + } + + ret = Nc4hw4ToNchw(tensor, t.get()); + if (ret != RET_OK) { + MS_LOGE("Nc4hw4ToNchw failed"); + return ret; + } + tensor->FreeData(); + } else { + t->SetData(tensor->GetData()); + tensor->SetData(nullptr); + } + outputs->push_back(t.release()); + } + return RET_OK; +} + +std::map> GraphExecution::GetAllOutput() { + std::map> outputs{}; + for (auto &outputNode : graph->GetOutputsMap()) { + std::vector outputNodeTensors{}; + auto ret = this->CopyOutputTensors(outputNode.second, &outputNodeTensors); + if (ret != RET_OK) { + MS_LOGE("copy output failed."); + FreeOutputMap(&outputs); + return outputs; + } + outputs.emplace(std::pair>(outputNode.first, outputNodeTensors)); + } + return outputs; +} + +std::vector GraphExecution::GetOutput(const NODE_ID &nodeName) { + std::vector outputNodeTensors{}; + auto iter = graph->GetOutputsMap().find(nodeName); + if (iter == graph->GetOutputsMap().end()) { + MS_LOGE("node name is not in output."); + return outputNodeTensors; + } + auto ret = this->CopyOutputTensors(iter->second, &outputNodeTensors); + if (ret != RET_OK) { + MS_LOGE("copy output failed."); + } + return outputNodeTensors; +} + +std::vector GraphExecution::GetInput() { + std::vector inputs{}; + for (auto refInput : graph->GetInputs()) { + if (refInput == nullptr) { + MS_LOGE("tensor from graph->GetInputs() is nullptr"); + return inputs; + } + std::unique_ptr t(new Tensor(refInput->GetDataType(), refInput->GetDims(), Format_NCHW, nullptr)); + if (t == nullptr) { + MS_LOGE("new Tensor failed.") + FreeTensors(&inputs); + return inputs; + } + inputs.push_back(t.release()); + } + return inputs; +} + +void GraphExecution::ResetInputData() { + for (auto tensor : inputTensors) { + if (tensor == nullptr) { + MS_LOGW("tensor in inputTensors is nullptr"); + continue; + } + if (tensor->GetFormat() == Format_NC4HW4) { + if (tensor->GetData() != nullptr) { + free(tensor->GetData()); + tensor->SetData(nullptr); + } + continue; + } + tensor->SetData(nullptr); + } +} + +void GraphExecution::FreeAllTensors() { graph->FreeAllTensors(); } + +int GraphExecution::Run(const std::vector &inputs) { + if (inputs.empty()) { + MS_LOGE("input is empty"); + return RET_ERROR; + } + + int ret; + + if (readyQue.empty()) { + MS_LOGE("readyQue is empty"); + return RET_ERROR; + } + + ret = SetInputTensors(inputs); + if (ret != RET_OK) { + MS_LOGE("SetInputTensors failed: %d", ret); + ResetInputData(); + return ret; + } + ret = MallocOutput(); + if (ret != RET_OK) { + MS_LOGE("MallocOutput failed: %d", ret); + ResetInputData(); + return ret; + } + + while (!readyQue.empty()) { + auto *node = readyQue.front(); + readyQue.pop_front(); + + ret = node->Run(_ctx); + if (ret != RET_OK) { + MS_LOGE("node (%s) failed to run op (%s). error code:%d", node->ID().c_str(), node->Type().c_str(), ret); + ResetInputData(); + FreeAllTensors(); + return ret; + } + + for (auto outNode : node->GetAllOutEdges()) { + auto nodeDepend = depends.find(outNode); + nodeDepend->second.erase(node); + if (nodeDepend->second.empty()) { + depends.erase(nodeDepend); + readyQue.push_back(outNode); + } + } + } + + ResetInputData(); + + return RET_OK; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/graph_execution.h b/predict/src/graph_execution.h new file mode 100644 index 00000000000..022be218654 --- /dev/null +++ b/predict/src/graph_execution.h @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_GRAPH_EXECUTION_H_ +#define PREDICT_SRC_GRAPH_EXECUTION_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "common/mslog.h" +#include "src/graph.h" +#include "include/errorcode.h" +#include "schema/inner/ms_generated.h" +#include "src/operator/cpu/include/op_func_comm.h" +#include "src/node.h" + +namespace mindspore { +namespace predict { +class GraphExecution { + public: + explicit GraphExecution(const Context &ctx); + GraphExecution(const Context &ctx, Graph *staticGraph); + virtual ~GraphExecution(); + + virtual std::vector GetInput(); + virtual int SetInputTensors(const std::vector &inputs); + + virtual int Run(const std::vector &inputs); + + virtual std::map> GetAllOutput(); + virtual std::vector GetOutput(const NODE_ID &nodeName); + + private: + void ResetInputData(); + int MallocOutput(); + void FreeTensors(std::vector *tensors); + int TransInputDataToNc4hw4(const Tensor &src, Tensor *dst); + int CopyOutputTensors(const std::vector &refOutputs, std::vector *outputs); + void FreeOutputMap(std::map> *map); + void FreeAllTensors(); + + protected: + Graph *graph; + const Context &_ctx; + std::vector inputTensors; + std::vector outputTensors; + std::unordered_map> depends; // records the dependencies + std::deque readyQue; // the nodes which can execute without any dependencies +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_GRAPH_EXECUTION_H_ diff --git a/predict/src/node.cc b/predict/src/node.cc new file mode 100644 index 00000000000..7128dde209f --- /dev/null +++ b/predict/src/node.cc @@ -0,0 +1,148 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/node.h" +#include +#include +#include +#include +#include +#include "schema/inner/ms_generated.h" +#include "common/mslog.h" +#include "common/op_utils.h" +#include "include/errorcode.h" +#include "src/op_factory.h" + +namespace mindspore { +namespace predict { +Node::Node(const NodeDef *nodeDef) + : id(std::string(nodeDef->opDef()->name()->c_str())), type(GetOpTypeName(*nodeDef)) {} + +Node::~Node() { + if (op != nullptr) { + delete op; + } +} + +NODE_ID Node::ID() { return id; } + +std::string Node::Type() { return type; } + +void Node::SetTensors(const NodeDef &nodeDef, const std::vector &allTensors) { + if (nodeDef.opDef() == nullptr) { + MS_LOGE("nodeDef is null"); + return; + } + + auto inputIndex = nodeDef.opDef()->inputIndex(); + MS_ASSERT(inputIndex != nullptr); + inputs.resize(inputIndex->size()); + std::transform(inputIndex->begin(), inputIndex->end(), inputs.begin(), [allTensors](int i) { return allTensors[i]; }); + + auto outputIndex = nodeDef.opDef()->outputIndex(); + MS_ASSERT(outputIndex != nullptr); + outputs.resize(outputIndex->size()); + std::transform(outputIndex->begin(), outputIndex->end(), outputs.begin(), + [allTensors](int i) { return allTensors[i]; }); +} + +void Node::SetDepends(const std::unordered_set &deps) { depends = deps; } + +std::unordered_set Node::GetDepends() { return depends; } + +void Node::AddInEdge(Node *node) { + if (node == nullptr) { + MS_LOGE("node is null"); + return; + } + inEdges.insert(node); +} + +void Node::AddOutEdge(Node *node) { + if (node == nullptr) { + MS_LOGE("node is null"); + return; + } + outEdges.insert(node); +} + +std::unordered_set &Node::GetAllInEdges() { return inEdges; } + +std::unordered_set &Node::GetAllOutEdges() { return outEdges; } + +std::vector &Node::GetOutputTensors() { return outputs; } +std::vector &Node::GetInputTensors() { return inputs; } + +int Node::InitOp(const OpDef &opDef, const Context &ctx) { + OpDesc dst; + dst.type = GetOpType(opDef); + dst.arch = X86_FP32; + MS_ASSERT(OpFactory::GetInstance() != nullptr); + op = OpFactory::GetInstance()->GetOp(inputs, outputs, opDef, ctx, dst); + if (op == nullptr) { + MS_LOGE("Can't find opName: %s, type: %s ", id.c_str(), type.c_str()); + return RET_ERROR; + } + return RET_OK; +} + +int Node::Run(const Context &ctx) { + MS_LOGD("%s run start", id.c_str()); + auto ret = MallocOutput(ctx); + if (ret != RET_OK) { + MS_LOGE("MallocOutput failed: %d", ret); + return ret; + } + if (op == nullptr) { + MS_LOGE("op is nullptr."); + return RET_ERROR; + } + ret = op->Execute(inputs, outputs); + if (ret != RET_OK) { + return ret; + } + FreeInput(); + return RET_OK; +} + +int Node::MallocOutput(const Context &ctx) { + size_t refCount = outEdges.size(); + for (auto tensor : outputs) { + if (tensor == nullptr) { + MS_LOGE("tensor in outputs is nullptr"); + return RET_ERROR; + } + auto ret = tensor->MallocData(ctx.allocator, refCount); + if (ret != RET_OK) { + return ret; + } + } + return RET_OK; +} + +void Node::FreeInput() { + for (auto tensor : inputs) { + if (tensor == nullptr) { + MS_LOGW("tensor in inputs is nullptr"); + return; + } + if (tensor->RefCount() != MSConst_WEIGHT_REFCOUNT) { + tensor->FreeData(); + } + } +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/node.h b/predict/src/node.h new file mode 100644 index 00000000000..eebb1b4321c --- /dev/null +++ b/predict/src/node.h @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_NODE_H_ +#define PREDICT_SRC_NODE_H_ + +#include +#include +#include +#include "include/session.h" +#include "src/op.h" + +namespace mindspore { +namespace predict { +using NODE_ID = std::string; + +class Node { + public: + Node() = default; + explicit Node(const NodeDef *nodeDef); + virtual ~Node(); + NODE_ID ID(); + std::string Type(); + void SetTensors(const NodeDef &nodeDef, const std::vector &allTensors); + void SetDepends(const std::unordered_set &deps); + std::unordered_set GetDepends(); + + void AddInEdge(Node *node); + void AddOutEdge(Node *node); + std::unordered_set &GetAllOutEdges(); + std::unordered_set &GetAllInEdges(); + + std::vector &GetOutputTensors(); + std::vector &GetInputTensors(); + + int InitOp(const OpDef &opDef, const Context &ctx); + int Run(const Context &ctx); + int MallocOutput(const Context &ctx); + void FreeInput(); + + protected: + friend class GraphExecution; + NODE_ID id; + std::string type; + OpBase *op{}; + std::vector inputs; + std::vector outputs; + std::unordered_set depends; + std::unordered_set inEdges; + std::unordered_set outEdges; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_NODE_H_ diff --git a/predict/src/op.cc b/predict/src/op.cc new file mode 100644 index 00000000000..ca99b7fdff6 --- /dev/null +++ b/predict/src/op.cc @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/op.h" + +namespace mindspore { +namespace predict { +OpBase::OpBase() : desc(nullptr) {} + +OpBase::~OpBase() = default; +} // namespace predict +} // namespace mindspore diff --git a/predict/src/op.h b/predict/src/op.h new file mode 100644 index 00000000000..a07ce21952f --- /dev/null +++ b/predict/src/op.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_OP_H_ +#define PREDICT_SRC_OP_H_ + +#include +#include +#include "include/context.h" +#include "include/tensor.h" +#include "include/errorcode.h" +#include "schema/inner/ms_generated.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +enum OP_ARCH { X86_FP32, X86_INT8, ARM_FP32, ARM_FP16, ARM_INT8, GPU }; + +struct MSPREDICT_API OpDesc { + OP_ARCH arch; + OpT type; + + bool operator<(const OpDesc &dst) const { return (arch < dst.arch) || (type < dst.type); } +}; + +class MSPREDICT_API OpBase { + public: + OpBase(); + virtual ~OpBase(); + + virtual int Execute(const std::vector &inputs, const std::vector &outputs) = 0; + virtual int Init(const std::vector &inputs, const std::vector &outputs) = 0; + + protected: + const OpDesc *desc; + std::string name; +}; + +typedef OpBase *(*OpCreator)(const std::vector &inputs, const std::vector &outputs, + const OpDef &opDef, const Context &ctx, const OpDesc &desc); +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_OP_H_ diff --git a/predict/src/op_common.h b/predict/src/op_common.h new file mode 100644 index 00000000000..c5eb69bd578 --- /dev/null +++ b/predict/src/op_common.h @@ -0,0 +1,83 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_OP_COMMON_H_ +#define PREDICT_SRC_OP_COMMON_H_ +#include + +namespace mindspore { +namespace predict { +static inline size_t AlignSize(size_t size, size_t align) { return (size + align - 1) & -align; } + +template +inline void Nchw2Nhwc(const Tsrc *in, Tdst *out, size_t h, size_t w, size_t c) { + MS_ASSERT(in != nullptr && out != nullptr); + const size_t sz = w * h; + + for (size_t cc = 0; cc < c; ++cc) { + auto pi = in + sz * cc; + + for (size_t el = 0; el < sz; ++el) { + out[cc + el * c] = (Tdst)pi[el]; + } + } +} + +template +inline void Nhwc2Nchw(const Tsrc *in, Tdst *out, size_t h, size_t w, size_t c) { + MS_ASSERT(in != nullptr && out != nullptr); + const size_t sz = w * h; + + for (auto cc = 0; cc < c; ++cc) { + auto po = out + sz * cc; + + for (size_t el = 0; el < sz; ++el) { + po[el] = (Tdst)in[cc + el * c]; + } + } +} + +template +inline void InverseQuantization(const Tsrc *srcdata, Tdst *dstdata, size_t datanum, float *parms) { + MS_ASSERT(srcdata != nullptr && dstdata != nullptr); + float scale = parms[2]; + float zeroPoint = parms[3]; + for (size_t i = 0; i < datanum; ++i) { + dstdata = (scale == 0) ? (0) : (Tdst)((srcdata[i] - zeroPoint) * scale); + } +} + +template +inline void Astype(const Tsrc *srcdata, Tdst *dstdata, size_t datanum) { + MS_ASSERT(srcdata != nullptr && dstdata != nullptr); + for (size_t i = 0; i < datanum; ++i) { + dstdata[i] = (Tdst)srcdata[i]; + } +} +#define MSMIN(x, y) ((x) < (y) ? (x) : (y)) +#define MSMAX(x, y) ((x) > (y) ? (x) : (y)) + +#define UP_DIV(x, y) (((x) + (y) - (1)) / (y)) +#define DOWN_DIV(x, y) (((x) - (y) + (1)) / (y)) +#define ROUND_UP(x, y) (((x) + (y) - (1)) / (y) * (y)) +#define ALIGN_UP4(x) ROUND_UP((x), 4) +#define ALIGN_UP8(x) ROUND_UP((x), 8) + +#define MAX_MALLOC_SIZE 100 * 1024 * 1024 +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_OP_COMMON_H_ diff --git a/predict/src/op_factory.cc b/predict/src/op_factory.cc new file mode 100644 index 00000000000..2506db68d8e --- /dev/null +++ b/predict/src/op_factory.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/op_factory.h" + +namespace mindspore { +namespace predict { +OpFactory::OpFactory() { InitKernelManager(0, ""); } + +OpFactory::~OpFactory() = default; + +OpFactory *OpFactory::GetInstance() { + static OpFactory instance; + return &instance; +} + +OpBase *OpFactory::GetOp(const std::vector &inputs, const std::vector &outputs, const OpDef &opDef, + const Context &ctx, const OpDesc &desc) { + MS_ASSERT(GetRegistryInstance() != nullptr); + auto *reg = GetRegistryInstance()->GetInstance(MODULE_REG_NAME_OP_REGISTRY); + if (reg != nullptr) { + auto creator = reg->GetOpCreator(desc); + if (creator) { + return creator(inputs, outputs, opDef, ctx, desc); + } + } + MS_ASSERT(OpRegistry::GetInstance() != nullptr); + auto creator = OpRegistry::GetInstance()->GetOpCreator(desc); + if (creator) { + return creator(inputs, outputs, opDef, ctx, desc); + } + return nullptr; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/op_factory.h b/predict/src/op_factory.h new file mode 100644 index 00000000000..583a605d8c0 --- /dev/null +++ b/predict/src/op_factory.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_OP_FACTORY_H_ +#define PREDICT_SRC_OP_FACTORY_H_ + +#include +#include "lite/api/km_api.h" +#include "src/op.h" +#include "src/op_registry.h" + +namespace mindspore { +namespace predict { +class OpFactory { + public: + OpFactory(); + virtual ~OpFactory(); + + static OpFactory *GetInstance(); + OpBase *GetOp(const std::vector &inputs, const std::vector &outputs, const OpDef &opDef, + const Context &ctx, const OpDesc &desc); +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_OP_FACTORY_H_ diff --git a/predict/src/op_registry.cc b/predict/src/op_registry.cc new file mode 100644 index 00000000000..14cd810d547 --- /dev/null +++ b/predict/src/op_registry.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/op_registry.h" + +namespace mindspore { +namespace predict { +OpRegistry::OpRegistry() = default; + +OpRegistry::~OpRegistry() = default; + +OpRegistry *OpRegistry::GetInstance() { + static OpRegistry instance; + return &instance; +} + +OpCreator OpRegistry::GetOpCreator(const OpDesc &desc) { + auto it = creators.find(desc); + if (it != creators.end()) { + return it->second; + } + return nullptr; +} + +void OpRegistry::RegOp(const OpDesc desc, OpCreator creator) { creators[desc] = creator; } + +void OpRegistry::RegOp(const OP_ARCH arch, const OpT type, OpCreator creator) { + OpDesc desc = {arch, type}; + creators[desc] = creator; +} + +bool OpRegistry::Merge(const std::unordered_map &newCreators) { return false; } + +const std::map &OpRegistry::GetOpCreators() { return creators; } +} // namespace predict +} // namespace mindspore diff --git a/predict/src/op_registry.h b/predict/src/op_registry.h new file mode 100644 index 00000000000..bb1d957fec9 --- /dev/null +++ b/predict/src/op_registry.h @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_OP_REGISTRY_H_ +#define PREDICT_SRC_OP_REGISTRY_H_ + +#include +#include +#include +#include "common/mslog.h" +#include "common/module_registry.h" +#include "src/op.h" + +#define MSPREDICT_API __attribute__((visibility("default"))) + +namespace mindspore { +namespace predict { +class MSPREDICT_API OpRegistry { + public: + OpRegistry(); + virtual ~OpRegistry(); + + static OpRegistry *GetInstance(); + virtual OpCreator GetOpCreator(const OpDesc &desc); + + const std::map &GetOpCreators(); + + void RegOp(OpDesc desc, OpCreator creator); + void RegOp(OP_ARCH arch, OpT type, OpCreator creator); + static bool Merge(const std::unordered_map &newCreators); + + protected: + std::map creators; +}; + +template <> +class Module : public ModuleBase { + public: + virtual OpRegistry *GetInstance() = 0; +}; + +const char MODULE_REG_NAME_OP_REGISTRY[] = "op_registry"; + +class OpRegistrar { + public: + OpRegistrar(const OpDesc &desc, OpCreator creator) { OpRegistry::GetInstance()->RegOp(desc, creator); } + + OpRegistrar(const OP_ARCH arch, const OpT type, OpCreator creator) { + MS_ASSERT(OpRegistry::GetInstance() != nullptr); + OpRegistry::GetInstance()->RegOp(arch, type, creator); + } +}; + +#define REG_OP(arch, type, opCreater) static OpRegistrar g_##arch##type##OpReg(arch, type, opCreater); +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_OP_REGISTRY_H_ diff --git a/predict/src/operator/cpu/common/op_func_comm.cc b/predict/src/operator/cpu/common/op_func_comm.cc new file mode 100755 index 00000000000..f05e224ec3d --- /dev/null +++ b/predict/src/operator/cpu/common/op_func_comm.cc @@ -0,0 +1,422 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/operator/cpu/include/op_func_comm.h" +#include "include/errorcode.h" +#include "include/tensor.h" +#include "common/mslog.h" +#include "securec/include/securec.h" + +namespace mindspore { +namespace predict { +#ifndef MS_USE_NEON +#ifndef MS_USE_SSE + +void MSAddBias(float *srcPtr, const float *bias, size_t unitSize, size_t count) { + if (srcPtr == nullptr || bias == nullptr) { + MS_LOGW("srcPtr or bias is nullptr"); + return; + } + for (size_t stepU = 0; stepU < count; stepU++) { + float *tmpPtr = srcPtr + unitSize * CAL_STEP * stepU; + const float *biasPtr = bias + CAL_STEP * stepU; + for (size_t step = 0; step < unitSize; step++) { + float *dstPtr = tmpPtr + CAL_STEP * step; + for (int i = 0; i < CAL_STEP; i++) { + dstPtr[i] += biasPtr[i]; + } + } + } +} + +void MSAddBiasRelu(float *srcPtr, const float *bias, size_t unitSize, size_t count) { + if (srcPtr == nullptr || bias == nullptr) { + MS_LOGW("srcPtr or bias is nullptr"); + return; + } + for (size_t stepU = 0; stepU < count; stepU++) { + float *tmpPtr = srcPtr + unitSize * CAL_STEP * stepU; + const float *biasPtr = bias + CAL_STEP * stepU; + for (size_t step = 0; step < unitSize; step++) { + float *dstPtr = tmpPtr + CAL_STEP * step; + for (int i = 0; i < CAL_STEP; i++) { + dstPtr[i] += biasPtr[i]; + dstPtr[i] = (dstPtr[i] < 0) ? 0 : dstPtr[i]; + } + } + } +} + +void MSAddBiasRelu6(float *srcPtr, const float *bias, size_t unitSize, size_t count) { + if (srcPtr == nullptr || bias == nullptr) { + MS_LOGW("srcPtr or bias is nullptr"); + return; + } + for (size_t stepU = 0; stepU < count; stepU++) { + float *tmpPtr = srcPtr + unitSize * CAL_STEP * stepU; + const float *biasPtr = bias + CAL_STEP * stepU; + for (size_t step = 0; step < unitSize; step++) { + float *dstPtr = tmpPtr + CAL_STEP * step; + for (int i = 0; i < CAL_STEP; i++) { + dstPtr[i] += biasPtr[i]; + dstPtr[i] = (dstPtr[i] < 0) ? 0 : dstPtr[i]; + dstPtr[i] = (dstPtr[i] > 6.0f) ? 6.0f : dstPtr[i]; + } + } + } +} + +void MSCopyC4WithStride(const float *srcPtr, float *dstPtr, size_t srcStride, size_t dstStride, size_t count) { + if (srcPtr == nullptr || dstPtr == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + for (size_t stepU = 0; stepU < count; stepU++) { + auto sPtr = srcPtr + stepU * srcStride; + auto dPtr = dstPtr + stepU * dstStride; + int tmpC = 0; + while (tmpC < CAL_STEP) { + dPtr[tmpC] = sPtr[tmpC]; + tmpC++; + } + } +} +#endif // MS_USE_SSE + +int MSPackC4(float *dstPtr, const float *srcPtr, size_t area, size_t depth) { + if (dstPtr == nullptr || srcPtr == nullptr) { + MS_LOGE("srcPtr or dstPtr is nullptr"); + return RET_ERROR; + } + int cur = 0; + size_t size = area * UP_DIV(depth, CAL_STEP) * CAL_STEP * sizeof(float); + auto ret = memset_s(dstPtr, size, 0, size); + if (ret != EOK) { + MS_LOGE("memset_s failed!"); + return RET_ERROR; + } + for (size_t step = 0; step < depth; step++) { + auto plane = step / CAL_STEP; + auto offset = step % CAL_STEP; + auto dstPlane = plane * area * CAL_STEP + dstPtr; + for (size_t i = 0; i < area; i++) { + dstPlane[CAL_STEP * i + offset] = srcPtr[cur++]; + } + } + return RET_OK; +} + +void MSUnpackC4(float *dstPtr, const float *srcPtr, size_t area, size_t depth) { + if (dstPtr == nullptr || srcPtr == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + int cur = 0; + for (size_t step = 0; step < depth; step++) { + auto plane = step / CAL_STEP; + auto offset = step % CAL_STEP; + auto srcPlane = plane * area * CAL_STEP + srcPtr; + for (size_t i = 0; i < area; i++) { + dstPtr[cur++] = srcPlane[CAL_STEP * i + offset]; + } + } +} + +void MSUInt8ToInt16WithOffsetC4Common(int16_t *dstPtr, const uint8_t *srcPtr, size_t zeroPoint, size_t sizeQuad, + size_t dstStride, size_t srcStride) { + if (dstPtr == nullptr || srcPtr == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + for (size_t step = 0; step < sizeQuad; step++) { + auto dstZ = dstPtr + (dstStride / sizeof(int16_t)) * step; + auto srcZ = srcPtr + (srcStride / sizeof(uint8_t)) * step; + for (int i = 0; i < CAL_STEP; i++) { + dstZ[i] = (int16_t)((int32_t)srcZ[i] - (int32_t)zeroPoint); + } + } +} + +void MSUInt8ToInt16WithOffsetC4Fast(int16_t *colAddr, const uint8_t *srcStart, size_t zeroPoint, size_t sizeQuad, + size_t depthQuad, size_t dstZStep, size_t srcZStep) { + if (colAddr == nullptr || srcStart == nullptr) { + MS_LOGW("colAddr or srcStart is nullptr"); + return; + } + for (size_t step = 0; step < depthQuad; step++) { + auto dstZ = colAddr + (dstZStep / sizeof(int16_t)) * step; + auto srcZ = srcStart + (srcZStep / sizeof(uint8_t)) * step; + MSUInt8ToInt16WithOffsetC4Common(dstZ, srcZ, zeroPoint, sizeQuad, CAL_STEP * sizeof(int16_t), + CAL_STEP * sizeof(uint8_t)); + } +} +#endif + +void MSPackC4Uint8(uint8_t *dstPtr, const uint8_t *srcPtr, size_t area, size_t depth) { + if (dstPtr == nullptr || srcPtr == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + int cur = 0; + size_t size = area * UP_DIV(depth, CAL_STEP) * CAL_STEP * sizeof(uint8_t); + auto ret = memset_s(dstPtr, size, 0, size); + if (ret != EOK) { + MS_LOGE("memset_s failed!"); + return; + } + for (size_t step = 0; step < depth; step++) { + auto plane = step / CAL_STEP; + auto offset = step % CAL_STEP; + auto dstPlane = plane * area * CAL_STEP + dstPtr; + for (size_t x = 0; x < area; ++x) { + dstPlane[CAL_STEP * x + offset] = srcPtr[cur++]; + } + } +} + +void MSUnpackC4Uint8(uint8_t *dstPtr, const uint8_t *srcPtr, size_t area, size_t depth) { + if (dstPtr == nullptr || srcPtr == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + int cur = 0; + for (size_t step = 0; step < depth; step++) { + auto srcPlane = (step / CAL_STEP) * area * CAL_STEP + srcPtr; + for (size_t i = 0; i < area; i++) { + dstPtr[cur++] = srcPlane[CAL_STEP * i + (step % CAL_STEP)]; + } + } +} + +#ifdef MS_USE_NEON +static void MSTensorConvertNCHWToNC4HW4Depth(float *dst, const float *src, size_t area, size_t depth) { + if (dstPtr == nullptr || srcPtr == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + if (1 == depth) { + auto zeroValue = vmovq_n_f32(0.0f); + int areaC4 = static_cast(area / CAL_STEP); + int remain = areaC4 * CAL_STEP; + for (int i = 0; i < areaC4; ++i) { + auto srcCur = src + CAL_STEP * i; + auto dstCur = dst + CAL_STEP * CAL_STEP * i; + auto srcValue = vld1q_f32(srcCur); + float32x4x4_t dstValue; + dstValue.val[0] = srcValue; + dstValue.val[1] = zeroValue; + dstValue.val[2] = zeroValue; + dstValue.val[3] = zeroValue; + vst4q_f32(dstCur, dstValue); + } + for (int i = remain; i < area; ++i) { + dst[CAL_STEP * i + 0] = src[i]; + dst[CAL_STEP * i + 1] = 0.0f; + dst[CAL_STEP * i + 2] = 0.0f; + dst[CAL_STEP * i + 3] = 0.0f; + } + } else if (3 == depth) { + auto zeroValue = vmovq_n_f32(0.0f); + int areaC4 = static_cast(area / CAL_STEP); + int remain = areaC4 * CAL_STEP; + for (int i = 0; i < areaC4; ++i) { + auto srcCur = src + 12 * i; + auto dstCur = dst + 16 * i; + auto srcValue = vld3q_f32(srcCur); + float32x4x4_t dstValue; + dstValue.val[0] = srcValue.val[0]; + dstValue.val[1] = srcValue.val[1]; + dstValue.val[2] = srcValue.val[2]; + dstValue.val[3] = zeroValue; + vst4q_f32(dstCur, dstValue); + } + for (int i = remain; i < area; ++i) { + dst[CAL_STEP * i + 0] = src[3 * i + 0]; + dst[CAL_STEP * i + 1] = src[3 * i + 1]; + dst[CAL_STEP * i + 2] = src[3 * i + 2]; + dst[CAL_STEP * i + 3] = 0.0f; + } + } +} +#endif + +void MSTensorConvertNHWCToNC4HW4(float *dst, const float *src, size_t area, size_t depth) { + if (dst == nullptr || src == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } +#ifdef MS_USE_NEON + MSTensorConvertNCHWToNC4HW4Depth(dst, src, area, depth); + return; +#endif + int c = static_cast(depth); + int cDiv4 = c / CAL_STEP; + int cMod4 = c % CAL_STEP; + int cAlign = cDiv4 * CAL_STEP; + for (int hi = 0; hi < area; ++hi) { + auto srcHeight = src + hi * c; + auto dstHeight = dst + hi * CAL_STEP; + for (int ci = 0; ci < cDiv4; ++ci) { +#ifdef MS_USE_NEON + vst1q_f32(dstHeight + CAL_STEP * ci * area, vld1q_f32(srcHeight + CAL_STEP * ci)); +#else + for (int i = 0; i < CAL_STEP; ++i) { + dstHeight[ci * area * CAL_STEP + i] = srcHeight[CAL_STEP * ci + i]; + } +#endif + } + } + + if (cMod4 == 0) { + MS_LOGW("depth should be multiple of four"); + return; + } + + auto srcAlign = src + cAlign; + auto dstAlign = dst + area * cAlign; + +#ifdef MS_USE_NEON + auto zeroVector = vdupq_n_f32(0.0f); +#endif + + for (int hi = 0; hi < area; ++hi) { + auto srcHeight = srcAlign + hi * c; + auto dstHeight = dstAlign + hi * CAL_STEP; +#ifdef MS_USE_NEON + vst1q_f32(dstHeight, zeroVector); +#else + for (int i = 0; i < CAL_STEP; ++i) { + dstHeight[i] = 0; + } +#endif + for (int ci = 0; ci < cMod4; ++ci) { + dstHeight[ci] = srcHeight[ci]; + } + } +} + +void MSTensorConvertNC4HW4ToNHWC(float *dst, const float *src, size_t area, size_t depth) { + if (dst == nullptr || src == nullptr) { + MS_LOGW("srcPtr or dstPtr is nullptr"); + return; + } + int c = static_cast(depth); + int cDiv4 = c / CAL_STEP; + int cMod4 = c % CAL_STEP; + int cAlign = cDiv4 * CAL_STEP; + for (int hi = 0; hi < area; ++hi) { + auto srcHeight = src + hi * CAL_STEP; + auto dstHeight = dst + hi * c; + for (int ci = 0; ci < cDiv4; ++ci) { +#ifdef MS_USE_NEON + vst1q_f32(dstHeight + CAL_STEP * ci, vld1q_f32(srcHeight + CAL_STEP * ci * area)); +#else + for (int i = 0; i < CAL_STEP; ++i) { + dstHeight[ci * CAL_STEP + i] = srcHeight[CAL_STEP * ci * area + i]; + } +#endif + } + } + + if (cMod4 == 0) { + MS_LOGW("depth should be multiple of four"); + return; + } + + auto srcAlign = src + area * cAlign; + auto dstAlign = dst + cAlign; + + for (int hi = 0; hi < area; ++hi) { + auto srcHeight = srcAlign + hi * CAL_STEP; + auto dstHeight = dstAlign + hi * c; + + for (int ci = 0; ci < cMod4; ++ci) { + dstHeight[ci] = srcHeight[ci]; + } + } +} + +int NchwToNc4hw4(const Tensor *input, Tensor *output) { + if (input == nullptr || output == nullptr) { + MS_LOGE("input or output is nullptr"); + return RET_ERROR; + } + int batch = static_cast(input->Batch()); + int channel = static_cast(input->Channel()); + MS_ASSERT(batch > 0); + MS_ASSERT(channel > 0); + int area = static_cast(input->Width()) * static_cast(input->Height()); + int inputStride = input->GetElementSize() / batch; + int outputStride = output->GetElementSize() / batch; + DataType dt = input->GetDataType(); + + MS_ASSERT(input->GetData()); + MS_ASSERT(output->GetData()); + + if (dt == DataType_DT_FLOAT) { + for (int i = 0; i < batch; ++i) { + auto ret = MSPackC4(reinterpret_cast(output->GetData()) + outputStride * i, + (const float *)input->GetData() + inputStride * i, area, channel); + if (ret != RET_OK) { + MS_LOGE("MSPackC4 failed: %d", ret); + return RET_ERROR; + } + } + } else if (dt == DataType_DT_UINT8) { + for (int i = 0; i < batch; ++i) { + MSPackC4Uint8(reinterpret_cast(output->GetData()) + outputStride * i, + (const uint8_t *)input->GetData() + inputStride * i, area, channel); + } + } else { + MS_LOGE("Unsupported dataType: %d", dt); + return RET_ERROR; + } + return RET_OK; +} + +int Nc4hw4ToNchw(const Tensor *input, Tensor *output) { + if (input == nullptr || output == nullptr) { + MS_LOGE("input tensor or output tensor is nullptr"); + return RET_ERROR; + } + + int batch = static_cast(input->Batch()); + int channel = static_cast(input->Channel()); + MS_ASSERT(batch > 0); + MS_ASSERT(channel > 0); + int area = static_cast(input->Width()) * static_cast(input->Height()); + int inputStride = input->GetElementSize() / batch; + int outputStride = output->GetElementSize() / batch; + DataType dt = input->GetDataType(); + if (dt == DataType_DT_FLOAT) { + for (int i = 0; i < batch; ++i) { + MSUnpackC4(reinterpret_cast(output->GetData()) + outputStride * i, + (const float *)input->GetData() + inputStride * i, area, channel); + } + } else if (dt == DataType_DT_UINT8) { + for (int i = 0; i < batch; ++i) { + MSUnpackC4Uint8(reinterpret_cast(output->GetData()) + outputStride * i, + (const uint8_t *)input->GetData() + inputStride * i, area, channel); + } + } else { + MS_LOGE("Unsupported dataType: %d", dt); + return RET_ERROR; + } + + return RET_OK; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/operator/cpu/include/op_func_comm.h b/predict/src/operator/cpu/include/op_func_comm.h new file mode 100644 index 00000000000..884803d6693 --- /dev/null +++ b/predict/src/operator/cpu/include/op_func_comm.h @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_OPERATOR_CPU_INCLUDE_OP_FUNC_COMM_H_ +#define PREDICT_SRC_OPERATOR_CPU_INCLUDE_OP_FUNC_COMM_H_ + +#include +#include +#include +#include +#include +#include "src/op_common.h" +#include "include/tensor.h" + +#ifdef MS_USE_NEON +#include +#endif // MS_USE_NEON + +namespace mindspore { +namespace predict { +#ifdef __cplusplus +extern "C" { +#endif +#define CAL_STEP 4 +void MSAddBias(float *dst, const float *bias, size_t planeNumber, size_t biasNumber); +void MSAddBiasRelu(float *dst, const float *bias, size_t planeNumber, size_t biasNumber); +void MSAddBiasRelu6(float *dst, const float *bias, size_t planeNumber, size_t biasNumber); +void MSPackC4Uint8(uint8_t *dst, const uint8_t *src, size_t area, size_t depth); +void MSUnpackC4(float *dst, const float *src, size_t area, size_t depth); +void MSUnpackC4Uint8(uint8_t *dst, const uint8_t *src, size_t area, size_t depth); +void MSTensorConvertNHWCToNC4HW4(float *dst, const float *src, size_t area, size_t depth); +void MSTensorConvertNC4HW4ToNHWC(float *dst, const float *src, size_t area, size_t depth); +void MSUnpackC4(float *dst, const float *src, size_t area, size_t depth); +void MSCopyC4WithStride(const float *source, float *dest, size_t srcStride, size_t dstStride, size_t count); +void MSUInt8ToInt16WithOffsetC4Common(int16_t *dst, const uint8_t *src, size_t zeroPoint, size_t sizeQuad, + size_t dstStride, size_t srcStride); +void MSUInt8ToInt16WithOffsetC4Fast(int16_t *dst, const uint8_t *src, size_t zeroPoint, size_t sizeQuad, + size_t depthQuad, size_t dstZStep, size_t srcZStep); + +int MSPackC4(float *dst, const float *src, size_t area, size_t depth); +int NchwToNc4hw4(const Tensor *input, Tensor *output); +int Nc4hw4ToNchw(const Tensor *input, Tensor *output); +#ifdef __cplusplus +} +#endif +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_OPERATOR_CPU_INCLUDE_OP_FUNC_COMM_H_ diff --git a/predict/src/runtime/allocator.cc b/predict/src/runtime/allocator.cc new file mode 100644 index 00000000000..cb94af5df9c --- /dev/null +++ b/predict/src/runtime/allocator.cc @@ -0,0 +1,135 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/runtime/allocator.h" +#include "common/module_registry.h" +#include "src/op_common.h" + +namespace mindspore { +namespace predict { +std::shared_ptr Allocator::Create() { + auto alloc = GetRegistryInstance()->Create(MODULE_REG_NAME_ALLOCATOR); + if (alloc != nullptr) { + return alloc; + } + + // default allocator + return std::shared_ptr(new DefaultAllocator()); +} + +DefaultAllocator::DefaultAllocator() = default; + +DefaultAllocator::~DefaultAllocator() { Clear(); } + +void DefaultAllocator::SetContext(const AllocatorContext &ctx) { + lockFlag = ctx.lockFlag; + shiftFactor = ctx.shiftFactor; +} + +void DefaultAllocator::Lock() { + if (lockFlag) { + lock.lock(); + } +} + +void DefaultAllocator::UnLock() { + if (lockFlag) { + lock.unlock(); + } +} + +void *DefaultAllocator::Malloc(size_t size) { + if (size > MAX_MALLOC_SIZE) { + return nullptr; + } + Lock(); + auto it = freeList.begin(); + for (; it != freeList.end(); it++) { + auto membuf = *it; + + if ((membuf->size >= size) && (membuf->size < (size << shiftFactor))) { + freeList.erase(it); + allocatedList.push_back(membuf); + UnLock(); + return membuf->buf; + } + } + std::unique_ptr membuf(reinterpret_cast(malloc(sizeof(MemBuf) + size))); + if (membuf == nullptr) { + UnLock(); + return nullptr; + } + membuf->size = size; + membuf->buf = reinterpret_cast(membuf.get()) + sizeof(MemBuf); + auto bufPtr = membuf->buf; + allocatedList.push_back(membuf.release()); + UnLock(); + return bufPtr; +} + +void DefaultAllocator::Free(void *buf) { + if (buf == nullptr) { + return; + } + Lock(); + auto it = allocatedList.begin(); + for (; it != allocatedList.end(); it++) { + auto membuf = *it; + + if (membuf->buf == buf) { + allocatedList.erase(it); + freeList.push_back(membuf); + UnLock(); + return; + } + } + UnLock(); + free(buf); +} + +size_t DefaultAllocator::GetTotalSize() { + Lock(); + size_t totalSize = 0; + auto it = allocatedList.begin(); + for (; it != allocatedList.end(); it++) { + auto membuf = *it; + totalSize += membuf->size; + } + it = freeList.begin(); + for (; it != freeList.end(); it++) { + auto membuf = *it; + totalSize += membuf->size; + } + UnLock(); + return totalSize; +} + +void DefaultAllocator::Clear() { + Lock(); + auto it = allocatedList.begin(); + for (; it != allocatedList.end(); it++) { + free(*it); + } + allocatedList.clear(); + it = freeList.begin(); + for (; it != freeList.end(); it++) { + free(*it); + } + freeList.clear(); + UnLock(); +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/runtime/allocator.h b/predict/src/runtime/allocator.h new file mode 100644 index 00000000000..a9d72fbc9d7 --- /dev/null +++ b/predict/src/runtime/allocator.h @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_RUNTIME_ALLOCATOR_H_ +#define PREDICT_SRC_RUNTIME_ALLOCATOR_H_ + +#include +#include +#include +#include +#include "common/module_registry.h" + +namespace mindspore { +namespace predict { +struct AllocatorContext { + int shiftFactor; + bool lockFlag; +}; + +class Allocator { + public: + Allocator() : name("default") {} + virtual ~Allocator() {} + virtual void *Malloc(size_t size) = 0; + virtual void Free(void *ptr) = 0; + virtual void SetContext(const AllocatorContext &ctx) {} + virtual size_t GetTotalSize() { return 0; } + virtual void Clear() {} + static std::shared_ptr Create(); + std::string name; +}; + +class DefaultAllocator : public Allocator { + public: + DefaultAllocator(); + ~DefaultAllocator() override; + void SetContext(const AllocatorContext &ctx) override; + void *Malloc(size_t size) override; + void Free(void *ptr) override; + size_t GetTotalSize() override; + void Clear() override; + + private: + void Lock(); + void UnLock(); + struct MemBuf { + size_t size; + void *buf; + }; + + std::mutex lock; + std::vector allocatedList; + std::vector freeList; + int shiftFactor = 0; + bool lockFlag = false; +}; + +// these declaration are for module integration, refer to sample_allocator +const char MODULE_REG_NAME_ALLOCATOR[] = "allocator"; + +template <> class Module : public ModuleBase { + public: + virtual std::shared_ptr Create() = 0; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_RUNTIME_ALLOCATOR_H_ diff --git a/predict/src/runtime/runtime_api.cc b/predict/src/runtime/runtime_api.cc new file mode 100644 index 00000000000..2091c808fff --- /dev/null +++ b/predict/src/runtime/runtime_api.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/runtime/runtime_api.h" +#include +#include +#include "src/runtime/workspace_pool.h" +#include "src/runtime/thread_pool.h" +#include "common/mslog.h" + +static std::mutex gWorkspaceMutex; +#ifdef __cplusplus +extern "C" { +#endif +void LiteAPISetLastError(const char *msg) { MS_LOGE("The lite api set last error is %s.", msg); } + +void *LiteBackendAllocWorkspace(int deviceType, int deviceId, uint64_t size, int dtypeCode, int dtypeBits) { + std::lock_guard lock(gWorkspaceMutex); + auto p = mindspore::predict::WorkspacePool::GetInstance(); + if (p == nullptr) { + MS_LOGE("get ThreadPool install failed"); + return nullptr; + } + return p->AllocWorkSpaceMem(size); +} + +int LiteBackendFreeWorkspace(int deviceType, int deviceId, void *ptr) { + std::lock_guard lock(gWorkspaceMutex); + auto p = mindspore::predict::WorkspacePool::GetInstance(); + if (p == nullptr) { + MS_LOGE("get ThreadPool install failed"); + return -1; + } + p->FreeWorkSpaceMem(ptr); + return 0; +} + +void ConfigThreadPool(int mode, int nthreads) { + auto p = mindspore::predict::ThreadPool::GetInstance(); + if (p == nullptr) { + MS_LOGE("get ThreadPool install failed"); + return; + } + p->ConfigThreadPool(mode, nthreads); +} + +int LiteBackendParallelLaunch(FTVMParallelLambda flambda, void *cdata, int num_task) { + auto p = mindspore::predict::ThreadPool::GetInstance(); + if (p == nullptr) { + MS_LOGE("get ThreadPool install failed"); + return -1; + } + if (!p->LaunchThreadPoolTask()) { + MS_LOGE("get ThreadPool or thread bind failed"); + return -1; + } + if (!p->AddTask(flambda, cdata, num_task)) { + MS_LOGE("AddTask failed"); + return -1; + } + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/predict/src/runtime/runtime_api.h b/predict/src/runtime/runtime_api.h new file mode 100644 index 00000000000..01aa782cf8a --- /dev/null +++ b/predict/src/runtime/runtime_api.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PREDICT_SRC_RUNTIME_RUNTIME_API_H_ +#define PREDICT_SRC_RUNTIME_RUNTIME_API_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + void *sync_handle; + int32_t num_task; +} TVMParallelGroupEnv; +typedef int (*FTVMParallelLambda)(int task_id, TVMParallelGroupEnv *penv, void *cdata); +void LiteAPISetLastError(const char *msg); +void *LiteBackendAllocWorkspace(int deviceType, int deviceId, uint64_t size, int dtypeCode, int dtypeBits); +int LiteBackendFreeWorkspace(int deviceType, int deviceId, void *ptr); +void ConfigThreadPool(int mode, int nthreads); +int LiteBackendParallelLaunch(FTVMParallelLambda flambda, void *cdata, int num_task); +int LiteBackendRegisterSystemLibSymbol(const char *name, void *ptr); + +#ifdef __cplusplus +} +#endif +#endif // PREDICT_SRC_RUNTIME_RUNTIME_API_H_ diff --git a/predict/src/runtime/thread_pool.cc b/predict/src/runtime/thread_pool.cc new file mode 100644 index 00000000000..6018927a182 --- /dev/null +++ b/predict/src/runtime/thread_pool.cc @@ -0,0 +1,447 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/runtime/thread_pool.h" +#include +#include "common/mslog.h" + +namespace mindspore { +namespace predict { +static constexpr int kThreadPoolMaxThreads = 8; +static const int kCoreNumThr = 4; +static const int kMidCoreNum = 2; +static const int kBigCoreNum = 2; +bool LiteQueue::Enqueue(const ThreadPoolTask &task) { + const int tailIndex = tail.load(std::memory_order_relaxed); + // queue full + auto next = (tailIndex + 1) % kSingleThreadMaxTask; + if (next == head.load(std::memory_order_acquire)) { + return false; + } + buffer[tailIndex] = task; + tail.store(next, std::memory_order_release); + taskSize.fetch_add(1); + return true; +} + +bool LiteQueue::Dequeue(ThreadPoolTask *out) { + if (out == nullptr) { + MS_LOGE("ThreadPoolTask is nullptr"); + return false; + } + if (taskSize.load() == 0) { + return false; + } + // queue empty + const int headIndex = head.load(std::memory_order_relaxed); + if (headIndex == tail.load(std::memory_order_acquire)) { + return false; + } + *out = buffer[headIndex]; + head.store((headIndex + 1) % kSingleThreadMaxTask, std::memory_order_release); + return true; +} + +bool LiteThreadBind::Bind(int numThreads, int mode) { + InitSortedCpuId(); + if (numThreads > static_cast(sortedCpuIds.size())) { + MS_LOGE("thread num %d is larger than cores %lu in the system", numThreads, sortedCpuIds.size()); + return false; + } + threadNums = numThreads + 1; + bindModel = static_cast(mode); + if (bindModel == NO_BIND) { + if (!BindAllThread(false)) { + MS_LOGE("unbind %d threads failed", threadNums); + return false; + } + MS_LOGD("unbind %d threads successful", threadNums); + } else { + if (!BindAllThread(true)) { + MS_LOGE("bind %d threads failed", threadNums); + return false; + } + MS_LOGD("bind %d threads successful", threadNums); + } + return true; +} + +void LiteThreadBind::InitSortedCpuId() { + int numCores = static_cast(std::thread::hardware_concurrency()); + if (numCores < kCoreNumThr) { + bigCore = 0; + midCore = numCores; + } else { + bigCore = kBigCoreNum; + midCore = kMidCoreNum; + } + if (numCores > kCoreNumThr) { + numCores = bigCore + midCore; + } + sortedCpuIds.resize(numCores); + sortedCpuIds.clear(); + for (int i = numCores - 1; i >= 0; --i) { + sortedCpuIds.emplace_back(i); + } +} + +bool LiteThreadBind::BindAllThread(bool bindFlag) { + if (threadNums <= 0) { + MS_LOGE("no thread pool find, current threadNums %d", threadNums); + return false; + } + if (!BindThreads(bindFlag)) { + MS_LOGE("bind threads failed"); + return false; + } + return true; +} + +bool LiteThreadBind::BindMasterThread(bool bindFlag, int mode) { + std::vector cpu; + cpu.resize(sortedCpuIds.size()); + cpu.clear(); + if (bindFlag) { + int cpuIndex = (mode == MID_CORE) ? (threadNums - 1) : 0; + auto materCpuId = sortedCpuIds.at(cpuIndex); + cpu.emplace_back(materCpuId); + } else { + // unbind master + cpu.assign(sortedCpuIds.begin(), sortedCpuIds.end()); + } + cpu_set_t cpuSet; + CPU_ZERO(&cpuSet); + for (auto coreId : cpu) { + CPU_SET(coreId, &cpuSet); + } + if (!SetCPUBind(pthread_self(), cpuSet)) { + MS_LOGE("do master bind failed. mode: %d", mode); + return false; + } + return true; +} + +bool LiteThreadBind::BindThreads(bool bindFlag) { + if (bindFlag) { + if (bindModel != NO_BIND) { + size_t bindNums = std::min(sortedCpuIds.size(), threadIdList.size()); + size_t coreIndex; + cpu_set_t cpuSet; + for (size_t i = 0; i < bindNums; ++i) { + if (bindModel == MID_CORE) { + coreIndex = sortedCpuIds.size() - i - 1; + } else { + coreIndex = i; + } + CPU_ZERO(&cpuSet); + CPU_SET(sortedCpuIds[coreIndex], &cpuSet); + if (!threadIdList[i].second) { + MS_LOGD("threadIdList[%lu]=%lu, sortedCpuIds[%lu]=%d", i, threadIdList[i].first, coreIndex, + sortedCpuIds[coreIndex]); + if (!SetCPUBind(threadIdList[i].first, cpuSet)) { + MS_LOGE("do SetCPUBind failed"); + return false; + } + } + threadIdList[i].second = true; + } + } + } else { + // unbind + size_t bindNums = std::min(sortedCpuIds.size(), threadIdList.size()); + cpu_set_t cpuSet; + CPU_ZERO(&cpuSet); + for (auto coreId : sortedCpuIds) { + CPU_SET(coreId, &cpuSet); + } + for (size_t i = 0; i < bindNums; ++i) { + if (!SetCPUBind(threadIdList[i].first, cpuSet)) { + MS_LOGE("do SetCPUBind failed"); + return false; + } + threadIdList[i].second = false; + } + } + return true; +} + +bool LiteThreadBind::SetCPUBind(pthread_t threadId, const cpu_set_t &cpuSet) { +#if defined(__ANDROID__) +#if __ANDROID_API__ >= 21 + int ret = sched_setaffinity(pthread_gettid_np(threadId), sizeof(cpu_set_t), &cpuSet); + if (ret != 0) { + MS_LOGE("bind thread %ld to cpu failed.ERROR %d", threadId, ret); + } +#endif +#else + int ret = pthread_setaffinity_np(threadId, sizeof(cpu_set_t), &cpuSet); + if (ret != 0) { + MS_LOGE("bind thread %ld to cpu failed.ERROR %d", threadId, ret); + return false; + } +#endif + return true; +} + +LiteThreadPool::LiteThreadPool(int numThreads) { + queueList.resize(kThreadPoolMaxThreads); + queueList.clear(); + AddNewThread(numThreads); +} + +void LiteThreadPool::AddNewThread(int newNums) { + for (int i = curThreadNums, j = 0; j < newNums; ++j, ++i) { + queueList.push_back(std::unique_ptr(new LiteQueue())); + threadList.emplace_back([this, i]() { + ThreadPoolTask task; + while (!destroy) { + while (running != 0) { + MS_LOGD("i = %d, thread id = %lu, taskSize = %d", i, pthread_self(), queueList[i]->taskSize.load()); + while (queueList[i]->taskSize.load() > 0 && queueList[i]->Dequeue(&task)) { + auto ret = task.first(task.second.taskId, task.second.tvmParam, task.second.cdata); + if (ret != 0) { + errorInfo.emplace_back(std::make_pair(task.second.taskId, std::make_pair(false, ret))); + } + queueList[i]->taskSize.fetch_sub(1); + } + std::this_thread::yield(); + } + std::unique_lock queueLock(tMutex); + queueReady.wait(queueLock, [this] { return destroy || running != 0; }); + } + }); + } + MS_LOGI("%d new thread create", newNums); + curThreadNums += newNums; +} + +bool LiteThreadPool::DistributeTask(ThreadPoolTask task, int numTask) { + // wake up + errorInfo.clear(); + if (!AddRunReference()) { + MS_LOGE("add reference failed"); + return false; + } + bool kSuccFlag; + for (int i = 1; i < numTask; ++i) { + task.second.taskId = i; + do { + kSuccFlag = false; + for (auto &queue : queueList) { + MS_ASSERT(queue != nullptr); + if (queue->Enqueue(task)) { + kSuccFlag = true; + break; + } + } + std::this_thread::yield(); + } while (!kSuccFlag); + } + MS_LOGI("add %d task successful", numTask); + // master thread + int ret = task.first(0, task.second.tvmParam, task.second.cdata); + if (ret != 0) { + errorInfo.emplace_back(std::make_pair(0, std::make_pair(false, ret))); + } + kSuccFlag = false; + while (!kSuccFlag) { + kSuccFlag = true; + for (auto iter = queueList.begin(); iter != queueList.end(); ++iter) { + if ((*iter)->taskSize.load() != 0) { + kSuccFlag = false; + break; + } + } + std::this_thread::yield(); + } + // hibernate + if (!SubRunReference()) { + MS_LOGE("sub reference failed"); + return false; + } + MS_LOGI("finish %d task successful", numTask); + return CheckResult(); +} + +bool LiteThreadPool::AddRunReference() { + running.fetch_add(1); + std::lock_guard queueLock(tMutex); + queueReady.notify_all(); + return true; +} + +bool LiteThreadPool::SubRunReference() { + running.fetch_sub(1); + return true; +} + +bool LiteThreadPool::CheckResult() { + bool kSuccFlag = true; + for (auto result : errorInfo) { + if (result.second.first) { + MS_LOGE("task %d failed, error code is %d", result.first, result.second.second); + kSuccFlag = false; + } + } + return kSuccFlag; +} + +int ThreadPool::GetThreadNum(int numThreads) { + if (numThreads <= 0 || numThreads > kThreadPoolMaxThreads) { + MS_LOGE("numThreads %d, must be greater than 0 or less than or equal to %d", numThreads, kThreadPoolMaxThreads); + return -1; + } else { + if (numThreads > totalThreadNum) { + return (numThreads - totalThreadNum); + } else { + MS_LOGD("%d threads have been already created", numThreads); + return 0; + } + } +} + +void ThreadPool::GetThreadIdList() { + if (gThreadPool != nullptr) { + for (int i = 0; i < totalThreadNum; ++i) { + bool kSuccFlag = false; + pthread_t threadHandle; + do { + kSuccFlag = false; + threadHandle = gThreadPool->threadList[i].native_handle(); + if (threadHandle != 0) { + kSuccFlag = true; + } + std::this_thread::yield(); + } while (!kSuccFlag); + + auto iter = std::find_if(std::begin(gThreadBind->threadIdList), std::end(gThreadBind->threadIdList), + [threadHandle](std::pair id) { return id.first == threadHandle; }); + if (iter == std::end(gThreadBind->threadIdList)) { + gThreadBind->threadIdList.emplace_back(std::make_pair(threadHandle, false)); + } + } + } + MS_ASSERT(gThreadBind != nullptr); + gThreadBind->threadIdList.emplace_back(std::make_pair(pthread_self(), false)); +} + +bool ThreadPool::SetThreadCpulBind(int mode) { + if (totalThreadNum <= 0) { + MS_LOGE("no threads need to be bind, totalThreadNum : %d", totalThreadNum); + return false; + } + std::lock_guard bMutex(gPoolMutex); + if (gThreadBind == nullptr) { + gThreadBind = std::unique_ptr(new (std::nothrow) LiteThreadBind()); + if (gThreadBind == nullptr) { + MS_LOGE("new LiteThreadBind failed"); + return false; + } + gThreadBind->threadIdList.resize(kThreadPoolMaxThreads + 1); + gThreadBind->threadIdList.clear(); + } + GetThreadIdList(); + + if (!gThreadBind->Bind(totalThreadNum, mode)) { + MS_LOGE("BindCore failed"); + return false; + } + return true; +} + +bool ThreadPool::SetThreadPool(int numThreads) { + std::lock_guard Lock(gPoolMutex); + int realNums = GetThreadNum(numThreads); + if (realNums < -1) { + return false; + } + if (realNums == 0) { + return true; + } + if (gThreadPool == nullptr) { + gThreadPool = std::unique_ptr(new (std::nothrow) LiteThreadPool(realNums)); + if (gThreadPool == nullptr) { + MS_LOGE("%d threads create failed", realNums); + return false; + } + } else { + gThreadPool->AddNewThread(realNums); + } + MS_LOGD("%d threads create successful", realNums); + return true; +} + +ThreadPool *ThreadPool::GetInstance() { + static ThreadPool instance; + return &instance; +} + +void ThreadPool::ConfigThreadPool(int mode, int numThreads) { + bindMode = mode; + totalThreadNum = numThreads; +} + +bool ThreadPool::LaunchThreadPoolTask() { + if (gThreadPool == nullptr) { + if (!SetThreadPool(totalThreadNum)) { + MS_LOGE("create %d threads failed", totalThreadNum); + return false; + } + } + + if (gThreadBind == nullptr) { + if (!SetThreadCpulBind(bindMode)) { + MS_LOGE("create bind mode %d failed", bindMode); + return false; + } + } + return true; +} + +bool ThreadPool::AddTask(const WorkFun &worker, void *cdata, int numTask) { + if (numTask <= 0) { + numTask = totalThreadNum; + } + // single task, run master thread + if (numTask <= 1) { + TvmEnv env{}; + env.num_task = numTask; + int ret = worker(0, &env, cdata); + if (ret != 0) { + MS_LOGE("task 0 failed, error code is %d", ret); + return false; + } + MS_LOGD("task 0 successful"); + return true; + } + ThreadPoolTask task; + task.first = worker; + task.second.cdata = cdata; + return gThreadPool->DistributeTask(task, numTask); +} + +LiteThreadPool::~LiteThreadPool() { + destroy.store(true); + running.store(0); + queueReady.notify_all(); + for (auto &thread : threadList) { + if (thread.joinable()) { + thread.join(); + } + } +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/runtime/thread_pool.h b/predict/src/runtime/thread_pool.h new file mode 100644 index 00000000000..53e4c1ec88e --- /dev/null +++ b/predict/src/runtime/thread_pool.h @@ -0,0 +1,129 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_RUNTIME_THREAD_POOL_H_ +#define PREDICT_SRC_RUNTIME_THREAD_POOL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/runtime/runtime_api.h" + +namespace mindspore { +namespace predict { +constexpr int kSingleThreadMaxTask = 4; +using TvmEnv = TVMParallelGroupEnv; +using WorkFun = FTVMParallelLambda; +using TaskParam = struct Param { + void *cdata; + int32_t taskId; + TvmEnv *tvmParam; +}; +using ThreadPoolTask = std::pair; + +class LiteQueue { + public: + LiteQueue() = default; + ~LiteQueue() = default; + + bool Enqueue(const ThreadPoolTask &task); + bool Dequeue(ThreadPoolTask *out); + std::atomic taskSize{0}; + + private: + std::atomic head{0}; + std::atomic tail{0}; + ThreadPoolTask buffer[kSingleThreadMaxTask]{}; +}; + +class LiteThreadBind { + public: + LiteThreadBind() = default; + ~LiteThreadBind() = default; + bool Bind(int numThreads, int mode); + std::vector> threadIdList; + + private: + enum AffinityMode : int { BIG_CORE = 1, MID_CORE = -1, NO_BIND = 0 }; + void InitSortedCpuId(); + bool BindAllThread(bool bindFlag); + bool BindMasterThread(bool bindFlag, int mode = MID_CORE); + bool BindThreads(bool bindFlag); + bool SetCPUBind(pthread_t threadId, const cpu_set_t &cpuSet); + int bigCore{0}; + int midCore{0}; + int threadNums{0}; + std::vector sortedCpuIds{}; + AffinityMode bindModel{MID_CORE}; +}; + +class LiteThreadPool { + public: + LiteThreadPool() = default; + explicit LiteThreadPool(int numThreads); + ~LiteThreadPool(); + + void AddNewThread(int newNums); + bool DistributeTask(ThreadPoolTask task, int numTask); + std::vector threadList{}; + + private: + using errCode = std::pair; + bool AddRunReference(); + bool SubRunReference(); + bool CheckResult(); + int curThreadNums{0}; + std::vector> queueList; + std::atomic_int running{0}; + std::mutex tMutex; + std::condition_variable queueReady; + std::atomic destroy = {false}; + std::vector> errorInfo{}; +}; + +class ThreadPool { + public: + static ThreadPool *GetInstance(); + void ConfigThreadPool(int mode, int numThreads); + bool LaunchThreadPoolTask(); + bool AddTask(const WorkFun &worker, void *cdata, int numTask); + + ThreadPool(const ThreadPool &) = delete; + ThreadPool &operator=(const ThreadPool &) = delete; + + private: + ThreadPool() = default; + ~ThreadPool() = default; + int GetThreadNum(int numThreads); + void GetThreadIdList(); + bool SetThreadPool(int numThreads = 1); + bool SetThreadCpulBind(int mode); + std::unique_ptr gThreadPool{nullptr}; + std::unique_ptr gThreadBind{nullptr}; + std::mutex gPoolMutex; + int totalThreadNum{1}; + int bindMode{-1}; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_SRC_RUNTIME_THREAD_POOL_H_ diff --git a/predict/src/runtime/workspace_pool.cc b/predict/src/runtime/workspace_pool.cc new file mode 100644 index 00000000000..6cafe7482e3 --- /dev/null +++ b/predict/src/runtime/workspace_pool.cc @@ -0,0 +1,113 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "src/runtime/workspace_pool.h" +#include +#include +#include "common/mslog.h" + +namespace mindspore { +namespace predict { +static constexpr size_t kWorkspacePageSize = 4096; +static constexpr int kTempAllocaAlignment = 64; +WorkspacePool *WorkspacePool::GetInstance() { + static WorkspacePool instance; + return &instance; +} + +void *WorkspacePool::AllocWorkSpaceMem(size_t size) { + size_t nbytes = (size + (kWorkspacePageSize - 1)) / kWorkspacePageSize * kWorkspacePageSize; + if (nbytes == 0) { + nbytes = kWorkspacePageSize; + } + std::pair alloc; + // fist alloc + if (freeList.empty()) { + alloc.first = nbytes; + alloc.second = memalign(kTempAllocaAlignment, nbytes); + } else if (freeList.size() == 1) { // one element + alloc = *(freeList.begin()); + freeList.erase(freeList.begin()); + if (alloc.first < nbytes) { + free(alloc.second); + alloc.first = nbytes; + alloc.second = memalign(kTempAllocaAlignment, nbytes); + } + } else { + if ((*(freeList.begin())).first >= nbytes) { + auto iter = freeList.begin(); + for (; iter != freeList.end(); ++iter) { + if ((*iter).first < size) { + alloc = *(--iter); + freeList.erase(iter); + break; + } + } + if (iter == freeList.end()) { + alloc = *(freeList.rbegin()); + freeList.erase(--freeList.end()); + } + } else { + alloc = *(freeList.begin()); + freeList.erase(freeList.begin()); + free(alloc.second); + alloc.first = nbytes; + alloc.second = memalign(kTempAllocaAlignment, nbytes); + } + } + allocList.emplace_back(alloc); + return alloc.second; +} + +void WorkspacePool::FreeWorkSpaceMem(void *ptr) { + if (ptr == nullptr) { + return; + } + std::pair alloc; + if (allocList.empty()) { + MS_LOGE("no mem have been alloc"); + return; + } else if (allocList.back().second == ptr) { + alloc = allocList.back(); + allocList.pop_back(); + } else { + auto iter = allocList.begin(); + for (; iter != allocList.end(); ++iter) { + if ((*iter).second == ptr) { + alloc = *iter; + allocList.erase(iter); + break; + } + } + if (iter == allocList.end()) { + MS_LOGE("no value ptr have been alloc"); + return; + } + } + freeList.insert(alloc); +} + +WorkspacePool::~WorkspacePool() { + for (auto &a : allocList) { + free(a.second); + } + allocList.clear(); + for (auto &f : freeList) { + free(f.second); + } + freeList.clear(); +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/runtime/workspace_pool.h b/predict/src/runtime/workspace_pool.h new file mode 100644 index 00000000000..ce8a5ca3ab7 --- /dev/null +++ b/predict/src/runtime/workspace_pool.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_SRC_RUNTIME_WORKSPACE_POOL_H_ +#define PREDICT_SRC_RUNTIME_WORKSPACE_POOL_H_ +#include +#include +#include +#include +#include +#include + +namespace mindspore { +namespace predict { +class WorkspacePool { + public: + WorkspacePool() = default; + ~WorkspacePool(); + WorkspacePool(const WorkspacePool &) = delete; + WorkspacePool &operator=(const WorkspacePool &) = delete; + static WorkspacePool *GetInstance(); + void *AllocWorkSpaceMem(size_t size); + void FreeWorkSpaceMem(void *ptr); + + private: + std::vector> allocList{}; + std::set, std::greater>> freeList{}; +}; +} // namespace predict +} // namespace mindspore +#endif // PREDICT_SRC_RUNTIME_WORKSPACE_POOL_H_ diff --git a/predict/src/session.cc b/predict/src/session.cc new file mode 100644 index 00000000000..b808ec7c6bd --- /dev/null +++ b/predict/src/session.cc @@ -0,0 +1,154 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/session.h" +#include +#include +#include "include/errorcode.h" +#include "common/mslog.h" +#include "src/graph.h" +#include "src/graph_execution.h" + +namespace mindspore { +namespace predict { +Context m_ctx; +bool m_isConfig = false; + +// In 32bits, this evaluates to 2GB - 1 +static constexpr auto MAX_BUFFER_SIZE = ((1ULL << (sizeof(int32_t) * 8 - 1)) - 1); + +std::shared_ptr CreateSession(const char *graphBuf, size_t size, const Context &ctx) { + if (graphBuf == nullptr) { + MS_LOGE("the graphBuf is nullptr"); + return nullptr; + } + if (size > MAX_BUFFER_SIZE) { + MS_LOGE("the size is invalid"); + return nullptr; + } + auto session = std::make_shared(ctx); + MS_ASSERT(session != nullptr); + auto ret = session->Init(graphBuf, size); + if (ret != RET_OK) { + MS_LOGE("Init session failed."); + return nullptr; + } + return session; +} +Session::Session(const Context &ctx) : _ctx(ctx) { + Context cfgCtx; + cfgCtx = ctx; + if (cfgCtx.threadNum > m_ctx.threadNum) { + cfgCtx.threadNum = m_ctx.threadNum; + } +} + +int Session::Init(const char *graphBuf, size_t size) { + _graph = Graph::CreateFromBuf(graphBuf, size, _ctx); + if (_graph == nullptr) { + MS_LOGE("Graph create from buf failed."); + return RET_NULL_PTR; + } + + auto ret = this->InitExecutor(); + if (ret != RET_OK) { + MS_LOGE("Init Executor failed"); + return ret; + } + return ret; +} + +int Session::InitExecutor() { + if (_executor != nullptr) { + delete _executor; + _executor = nullptr; + } + if (_graph != nullptr) { + _executor = new (std::nothrow) GraphExecution(_ctx, _graph); + if (_executor == nullptr) { + MS_LOGE("new GraphExecution fail"); + return RET_ERROR; + } + return RET_OK; + } else { + MS_LOGE("the graph is nullptr"); + return RET_ERROR; + } +} + +Session::~Session() { + if (_executor != nullptr) { + delete _executor; + } + if (_graph != nullptr) { + delete _graph; + } +} + +int Session::Run(const std::vector &inputs) { + auto ret = RET_OK; + if (reinitExecutor) { + ret = this->InitExecutor(); + if (ret != RET_OK) { + MS_LOGE("Init Executor failed"); + return ret; + } + } + if (_executor == nullptr) { + MS_LOGE("_executor is nullptr"); + return ret; + } + ret = _executor->Run(inputs); + return ret; +} + +std::vector Session::GetInput() { + if (_executor == nullptr) { + MS_LOGE("_executor is nullptr"); + return std::vector{}; + } + auto inputs = _executor->GetInput(); + if (inputs.empty()) { + MS_LOGI("output is empty."); + } + return inputs; +} + +std::vector Session::GetOutput(const std::string &nodeName) { + if (_executor == nullptr) { + MS_LOGE("graph's executor is nullptr."); + return std::vector{}; + } + auto outputs = _executor->GetOutput(nodeName); + if (outputs.empty()) { + MS_LOGI("output is empty."); + } + return outputs; +} + +std::map> Session::GetAllOutput() { + if (_executor == nullptr) { + MS_LOGE("graph's executor is nullptr."); + return std::map>{}; + } + auto outputs = _executor->GetAllOutput(); + if (outputs.empty()) { + MS_LOGI("outputs is empty."); + } + return outputs; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/src/tensor.cc b/predict/src/tensor.cc new file mode 100644 index 00000000000..de758f3407c --- /dev/null +++ b/predict/src/tensor.cc @@ -0,0 +1,517 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/tensor.h" +#include "common/mslog.h" +#include "src/op_common.h" +#include "include/errorcode.h" +#include "securec/include/securec.h" +#include "common/common.h" +#include "src/runtime/allocator.h" + +namespace mindspore { +namespace predict { +Tensor *Tensor::CopyFromTensorDef(const TensorDef &tensorDef) { + std::vector dims; + + if (tensorDef.dims() == nullptr) { + MS_LOGD("tensorDef->dims is nullptr"); + } else { + MS_ASSERT(tensorDef.dims()->data() != nullptr); + for (uint32_t j = 0; j < tensorDef.dims()->size(); j++) { + dims.push_back(tensorDef.dims()->data()[j]); + } + } + auto tensor = + std::unique_ptr(new (std::nothrow) Tensor(tensorDef.dataType(), dims, tensorDef.format(), nullptr)); + if (tensor == nullptr) { + MS_LOGE("new Tensor failed"); + return nullptr; + } + + if (tensorDef.refCount() == MSConst_WEIGHT_REFCOUNT && tensorDef.data() != nullptr && tensorDef.data()->size() > 0) { + if (dims.size() < 1) { + tensor->SetDims({1}); + } + auto ret = tensor->MallocData(); + if (ret != RET_OK) { + MS_LOGE("malloc data fail,datasize %zu", tensor->GetDataSize()); + return nullptr; + } + auto tensorData = tensorDef.data()->data(); + ret = memcpy_sp(tensor->GetData(), tensor->GetDataSize(), tensorData, tensorDef.data()->size()); + if (ret != RET_OK) { + MS_LOGE("copy data fail,dst size %zu, src size %u", tensor->GetDataSize(), tensorDef.data()->size()); + return nullptr; + } + } + tensor->refCount = tensorDef.refCount(); + return tensor.release(); +} + +Tensor::Tensor(const Tensor &tensor, bool copyData) { + format = tensor.format; + dlTensor.data = nullptr; + dlTensor.ctx.device_type = tensor.dlTensor.ctx.device_type; + dlTensor.ctx.device_id = tensor.dlTensor.ctx.device_id; + dlTensor.strides = nullptr; + dlTensor.byte_offset = tensor.dlTensor.byte_offset; + dlTensor.dtype.code = tensor.dlTensor.dtype.code; + dlTensor.dtype.bits = tensor.dlTensor.dtype.bits; + dlTensor.dtype.lanes = tensor.dlTensor.dtype.lanes; + + dlTensor.ndim = tensor.dlTensor.ndim; + if (dlTensor.ndim > 0) { + dlTensor.shape = new (std::nothrow) int64_t[dlTensor.ndim]; + if (dlTensor.shape != nullptr) { + for (int i = 0; i < dlTensor.ndim; i++) { + dlTensor.shape[i] = tensor.dlTensor.shape[i]; + } + } else { + MS_LOGW("new shape fail,ndim %d", dlTensor.ndim); + } + } else { + dlTensor.shape = nullptr; + } + if (copyData) { + allocator = tensor.allocator; + refCount = tensor.refCount; + auto ret = MallocData(); + if (ret != RET_OK) { + return; + } + size_t datasize = GetDataSize(); + ret = memcpy_sp(dlTensor.data, datasize, tensor.dlTensor.data, datasize); + if (ret != RET_OK) { + return; + } + } +} + +Tensor::Tensor(DataType dt, const std::vector &dims, Format format, void *data) { + this->format = format; + dlTensor.data = data; + dlTensor.ctx.device_type = DLDeviceType::kDLCPU; + dlTensor.ctx.device_id = 0; + dlTensor.strides = nullptr; + dlTensor.byte_offset = 0; + + dlTensor.ndim = static_cast(dims.size()); + if (dlTensor.ndim > 0) { + dlTensor.shape = new (std::nothrow) int64_t[dlTensor.ndim]; + if (dlTensor.shape != nullptr) { + for (int i = 0; i < dlTensor.ndim; i++) { + dlTensor.shape[i] = dims[i]; + } + } else { + MS_LOGW("new shape fail,ndim %d", dlTensor.ndim); + } + } else { + dlTensor.shape = nullptr; + } + + SetDataType(dt); +} + +Tensor::~Tensor() { FreeTensor(); } + +DLDataType Tensor::GetTensorDtype() const { return dlTensor.dtype; } + +void *Tensor::GetData() const { return dlTensor.data; } + +void Tensor::SetData(void *data) { dlTensor.data = data; } + +DataType Tensor::GetDataType() const { + DataType dataType = DataType_DT_UNDEFINED; + switch (dlTensor.dtype.code) { + case kDLFloat: + if (dlTensor.dtype.bits == 32) { + dataType = DataType_DT_FLOAT; + } else if (dlTensor.dtype.bits == 16) { + dataType = DataType_DT_FLOAT16; + } + break; + case kDLInt: + if (dlTensor.dtype.bits == 32) { + dataType = DataType_DT_INT32; + } else if (dlTensor.dtype.bits == 8) { + dataType = DataType_DT_INT8; + } + break; + case kDLUInt: + if (dlTensor.dtype.bits == 32) { + dataType = DataType_DT_UINT32; + } else if (dlTensor.dtype.bits == 8) { + dataType = DataType_DT_UINT8; + } + break; + default: + break; + } + return dataType; +} + +void Tensor::SetDataType(DataType dt) { + switch (dt) { + case DataType_DT_FLOAT: + dlTensor.dtype.code = kDLFloat; + dlTensor.dtype.bits = 32; + dlTensor.dtype.lanes = 1; + break; + case DataType_DT_FLOAT16: + dlTensor.dtype.code = kDLFloat; + dlTensor.dtype.bits = 16; + dlTensor.dtype.lanes = 1; + break; + case DataType_DT_INT8: + dlTensor.dtype.code = kDLInt; + dlTensor.dtype.bits = 8; + dlTensor.dtype.lanes = 1; + break; + case DataType_DT_UINT8: + dlTensor.dtype.code = kDLUInt; + dlTensor.dtype.bits = 8; + dlTensor.dtype.lanes = 1; + break; + case DataType_DT_INT32: + dlTensor.dtype.code = kDLInt; + dlTensor.dtype.bits = 32; + dlTensor.dtype.lanes = 1; + break; + case DataType_DT_UINT32: + dlTensor.dtype.code = kDLUInt; + dlTensor.dtype.bits = 32; + dlTensor.dtype.lanes = 1; + break; + default: + MS_LOGW(" DataType %d is not implemented.", dt); + MS_LOGW(" DataType DT_FLOAT is used."); + dlTensor.dtype.code = kDLFloat; + dlTensor.dtype.bits = 32; + dlTensor.dtype.lanes = 1; + return; + } +} + +int Tensor::GetNDim() const { return dlTensor.ndim; } + +std::vector Tensor::GetDims() const { + std::vector dims; + for (int i = 0; i < dlTensor.ndim; i++) { + dims.push_back(dlTensor.shape[i]); + } + return dims; +} + +size_t Tensor::GetElementSize() const { + const int tile = 4; + if (format == Format_NC4HW4) { + size_t size = 1; + for (int i = 0; i < dlTensor.ndim; i++) { + auto var = static_cast(dlTensor.shape[i]); + if (i == 1) { + var = UP_DIV(var, tile) * tile; + } + size *= var; + } + return size; + } else { + size_t size = 1; + for (int i = 0; i < dlTensor.ndim; i++) { + size *= static_cast(dlTensor.shape[i]); + } + + return size; + } +} + +size_t Tensor::GetDataSize() const { + size_t size = GetElementSize(); + + const int BYTES = 8; + const int GAP = 7; + size *= (dlTensor.dtype.bits * dlTensor.dtype.lanes + GAP) / BYTES; + return size; +} + +int Tensor::MallocData(std::shared_ptr allocator, int refCount) { + if (dlTensor.data != nullptr) { + this->refCount += refCount; + return RET_OK; + } + this->refCount = refCount; + + size_t size = GetDataSize(); + if (allocator) { + this->allocator = allocator; + dlTensor.data = allocator->Malloc(size); + } else { + if (size > MAX_MALLOC_SIZE) { + return RET_ERROR; + } + dlTensor.data = malloc(size); + } + if (dlTensor.data == nullptr) { + return RET_ERROR; + } + return RET_OK; +} + +void Tensor::ForceFreeData() { + if (allocator) { + allocator->Free(dlTensor.data); + } else { + free(dlTensor.data); + } + dlTensor.data = nullptr; +} + +void Tensor::FreeData() { + --refCount; + if (refCount <= 0) { + ForceFreeData(); + } +} + +bool Tensor::CompareShape(const Tensor &dst) { + if (dlTensor.ndim != dst.dlTensor.ndim || dlTensor.shape == nullptr || dst.dlTensor.shape == nullptr) { + MS_LOGE("param error, one.ndim: %d, other.ndim: %d, one shape %p,other shape %p", dlTensor.ndim, dst.dlTensor.ndim, + dlTensor.shape, dst.dlTensor.shape); + return false; + } + + for (int i = 0; i < dlTensor.ndim; i++) { + if (dlTensor.shape[i] != dst.dlTensor.shape[i]) { + MS_LOGE("one.shape[%d]: %ld, other.shape[%d]: %ld", i, dlTensor.shape[i], i, dst.dlTensor.shape[i]); + return false; + } + } + return true; +} + +bool Tensor::CompareShape(const std::vector &other) { + if (dlTensor.ndim != other.size() || dlTensor.shape == nullptr) { + return false; + } + + for (int i = 0; i < dlTensor.ndim; i++) { + if (dlTensor.shape[i] != other[i]) { + return false; + } + } + return true; +} + +int64_t Tensor::Height() const { + if (dlTensor.shape == nullptr) { + MS_LOGE("shape is null"); + } + if (dlTensor.ndim != DIM_DEFAULT_SIZE) { + MS_LOGE("Tensor should be 4 dimensional."); + return -1; + } + switch (this->format) { + case Format_NCHW: + case Format_NC4HW4: + return dlTensor.shape[NCHW_H]; + case Format_NHWC: + return dlTensor.shape[NHWC_H]; + default: + MS_LOGE("Unsupported format: %d", this->format); + return -1; + } +} + +int64_t Tensor::Width() const { + if (dlTensor.shape == nullptr) { + MS_LOGE("shape is null"); + } + if (dlTensor.ndim != DIM_DEFAULT_SIZE) { + MS_LOGE("Tensor should be 4 dimensional."); + return -1; + } + switch (this->format) { + case Format_NCHW: + case Format_NC4HW4: + return dlTensor.shape[NCHW_W]; + case Format_NHWC: + return dlTensor.shape[NHWC_W]; + default: + MS_LOGE("Unsupported format: %d", this->format); + return -1; + } +} + +int64_t Tensor::Channel() const { + if (dlTensor.shape == nullptr) { + MS_LOGE("shape is null"); + } + if (dlTensor.ndim != DIM_DEFAULT_SIZE) { + MS_LOGE("Tensor should be 4 dimensional."); + return -1; + } + switch (this->format) { + case Format_NCHW: + case Format_NC4HW4: + return dlTensor.shape[NCHW_C]; + case Format_NHWC: + return dlTensor.shape[NHWC_C]; + default: + MS_LOGE("Unsupported format: %d", this->format); + return -1; + } +} + +int64_t Tensor::Batch() const { + if (dlTensor.shape == nullptr) { + MS_LOGE("shape is null"); + } + if (dlTensor.ndim != DIM_DEFAULT_SIZE) { + MS_LOGE("Tensor should be 4 dimensional."); + return -1; + } + switch (this->format) { + case Format_NCHW: + case Format_NC4HW4: + case Format_NHWC: + return dlTensor.shape[NCHW_N]; + default: + MS_LOGE("Unsupported format: %d", this->format); + return -1; + } +} + +int64_t Tensor::Stride(int index) const { + if (dlTensor.strides) { + return dlTensor.strides[index]; + } + if (dlTensor.shape == nullptr) { + MS_LOGE("shape is null"); + return -1; + } + int64_t stride = 1; + for (int i = index + 1; i < dlTensor.ndim; i++) { + stride *= dlTensor.shape[i]; + } + return stride; +} + +void Tensor::SetStride() { + if (dlTensor.strides == nullptr) { + if (dlTensor.ndim < 1) { + MS_LOGE("dims of dlTensor is empty."); + return; + } + dlTensor.strides = new (std::nothrow) int64_t[dlTensor.ndim - 1]; + if (dlTensor.strides == nullptr) { + MS_LOGW("new stride fail, ndim %d.", dlTensor.ndim); + return; + } + } + + for (int idx = 0; idx < dlTensor.ndim - 1; idx++) { + int64_t stride = 1; + if (dlTensor.ndim <= idx + 1) { + MS_LOGE("out of for loop upper limit."); + return; + } + for (int i = idx + 1; i < dlTensor.ndim; i++) { + stride *= dlTensor.shape[i]; + } + dlTensor.strides[idx] = stride; + } +} +void Tensor::SetScale(bool isScale) { this->isScale = isScale; } + +void Tensor::SetStride(int index, int64_t stride) { + if (index >= dlTensor.ndim) { + return; + } + + if (dlTensor.strides == nullptr) { + SetStride(); + } + + dlTensor.strides[index] = stride; + return; +} + +void Tensor::SetDims(const std::vector &dims) { + if (dlTensor.shape != nullptr) { + delete[] dlTensor.shape; + } + dlTensor.ndim = static_cast(dims.size()); + if (dlTensor.ndim > 0) { + dlTensor.shape = new (std::nothrow) int64_t[dlTensor.ndim]; + if (dlTensor.shape != nullptr) { + for (int i = 0; i < dlTensor.ndim; i++) { + dlTensor.shape[i] = dims[i]; + } + } else { + MS_LOGW("new shape fail,ndim %d", dlTensor.ndim); + } + } else { + dlTensor.shape = nullptr; + } +} + +void Tensor::FreeTensor() { + if (dlTensor.shape != nullptr) { + delete[] dlTensor.shape; + dlTensor.shape = nullptr; + } + + if (dlTensor.strides != nullptr) { + delete[] dlTensor.strides; + dlTensor.strides = nullptr; + } + + dlTensor.ndim = 0; + + if (allocator != nullptr) { + allocator->Free(dlTensor.data); + } else { + free(dlTensor.data); + } + dlTensor.data = nullptr; +} + +size_t Tensor::GetNC4HW4ElementSize(bool isNhwc) { + int alignIndex = 1; + if (isNhwc) { + alignIndex = 3; + } + + size_t size = 1; + for (int i = 0; i < dlTensor.ndim; i++) { + auto var = static_cast(dlTensor.shape[i]); + if (i == alignIndex) { + var = ALIGN_UP4(var); + } + size *= var; + } + return size; +} + +size_t Tensor::GetNC4HW4DataSize(bool isNhwc) { + size_t size = GetNC4HW4ElementSize(isNhwc); + const int BYTES = 8; + const int GAP = 7; + size *= (dlTensor.dtype.bits * dlTensor.dtype.lanes + GAP) / BYTES; + return size; +} +} // namespace predict +} // namespace mindspore diff --git a/predict/test/CMakeLists.txt b/predict/test/CMakeLists.txt new file mode 100755 index 00000000000..9370ff7ce0f --- /dev/null +++ b/predict/test/CMakeLists.txt @@ -0,0 +1,49 @@ +cmake_minimum_required(VERSION 3.12) +project(ms-test) + +set(CMAKE_CXX_STANDARD 11) + +#include 3rd +include_directories(${3RD_DIR}/securec/include) +include_directories(${3RD_DIR}/flatbuffers/include) +include_directories(${3RD_DIR}/googletest/googletest/include) +include_directories(${3RD_DIR}/googletest/googlemock/include) +include_directories(${3RD_DIR}/securec/include) + +#include ms +include_directories(.) +include_directories(..) + +link_directories(${CMAKE_CURRENT_SOURCE_DIR}/../output/lib/) + +set(COMMON_SRC ${PREDICT_DIR}/common/flag_parser.cc + ${PREDICT_DIR}/common/file_utils.cc + ${PREDICT_DIR}/common/mslog.cc + ${PREDICT_DIR}/common/storage.cc + ${PREDICT_DIR}/common/utils.cc) + +#tools src +file(GLOB_RECURSE TOOLS_SRC ../tools/*.cpp) + +add_executable(ms-test + ${COMMON_SRC} + ${TOOLS_SRC} + src/graph_tests.cc + benchmark/benchmark_tests.cc + ${CMAKE_SOURCE_DIR}/benchmark/benchmark.cc + ${TF_PROTO_SRC} + ${MS_CONVERTER_SRC} + test_context.h + test_context.cc + main.cc) + +target_link_libraries(ms-test mspredict gtest libsecurec.a) +add_dependencies(ms-test securec) +add_dependencies(ms-test gtest) + +# copy test file +add_custom_command(TARGET ms-test POST_BUILD + COMMAND mkdir -pv ${DOTEST_DIR} + COMMAND cp ${PREDICT_BUILD_DIR}/test/ms-test ${DOTEST_DIR} + COMMAND cp ${PREDICT_DIR}/test/run_tests.sh ${PREDICT_BUILD_DIR}/test/ + COMMAND cp -r ${PREDICT_DIR}/test/data/ ${PREDICT_BUILD_DIR}/test/doTest/) diff --git a/predict/test/benchmark/benchmark_tests.cc b/predict/test/benchmark/benchmark_tests.cc new file mode 100644 index 00000000000..e1e218e8518 --- /dev/null +++ b/predict/test/benchmark/benchmark_tests.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include "test/test_context.h" +#include "benchmark/benchmark.h" + +#define LENET_ARGS 2 +#define MS_ARGS 4 + +namespace mindspore { +namespace predict { +class BenchmarkTest : public ::testing::Test { + protected: + void SetUp() {} + + void TearDown() {} + std::string root; +}; + +TEST_F(BenchmarkTest, BenchmarkRun) { + const char* args[LENET_ARGS]; + args[0] = "./benchmark"; + args[1] = "--modelPath=./data/lenet/lenet.ms"; + + int errorcode = mindspore::predict::RunBenchmark(LENET_ARGS, args); + EXPECT_EQ(0, errorcode); +} + +TEST_F(BenchmarkTest, LenetRun) { + const char* args[MS_ARGS]; + args[0] = "./benchmark"; + args[1] = "--modelPath=./data/ms/mindspore.ms"; + args[2] = "--inDataPath=./data/ms/mindspore.bin"; + args[3] = "--calibDataPath=./data/ms/mindspore.out"; + + int errorcode = mindspore::predict::RunBenchmark(MS_ARGS, args); + EXPECT_EQ(0, errorcode); +} + +TEST_F(BenchmarkTest, MindSporeRun) { + const char* args[4]; + args[0] = "./benchmark"; + args[1] = "--modelPath=./data/lenet/lenet.ms"; + args[2] = "--inDataPath=./data/lenet/lenet.bin"; + args[3] = "--calibDataPath=./data/lenet/lenet.out"; + + int errorcode = mindspore::predict::RunBenchmark(4, args); + EXPECT_EQ(0, errorcode); +} +} // namespace predict +} // namespace mindspore diff --git a/predict/test/data/lenet/lenet.bin b/predict/test/data/lenet/lenet.bin new file mode 100755 index 00000000000..6aef53bd64b Binary files /dev/null and b/predict/test/data/lenet/lenet.bin differ diff --git a/predict/test/data/lenet/lenet.ms b/predict/test/data/lenet/lenet.ms new file mode 100755 index 00000000000..c66948dd677 Binary files /dev/null and b/predict/test/data/lenet/lenet.ms differ diff --git a/predict/test/data/lenet/lenet.out b/predict/test/data/lenet/lenet.out new file mode 100644 index 00000000000..31a1d61b8b2 --- /dev/null +++ b/predict/test/data/lenet/lenet.out @@ -0,0 +1,2 @@ +prob 2 1 10 +0.0 0.9999994 5.4061115e-07 0.0 0.0 0.0 0.0 5.690875e-08 1.1269122e-34 0.0 diff --git a/predict/test/data/ms/mindspore.bin b/predict/test/data/ms/mindspore.bin new file mode 100755 index 00000000000..77981f8b7ef --- /dev/null +++ b/predict/test/data/ms/mindspore.bin @@ -0,0 +1,5 @@ +p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پkI<6,?6,?6,?6,?6,?BL>p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پy>b??<4@<4@<4@<4@@L?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ`9?>`9?>`9?>`9?>`9?>`9?>`9?>N?<4@<4@<4@!2@@Yy @ +f>p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ?<4@<4@<4@<4@<4@<4@<4@<4@<4@<4@!2@R@lnp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ?<4@<4@<4@<4@<4@<4@<4@<4@<4@<4@@lnҾp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پh?Yy @w#@<4@<4@<4@<4@<4@<4@6,?`9?>`9?>`9?>dד=p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پYy @<4@<4@<4@<4@p@b?b?b?=ٽp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پYy @<4@<4@<4@@;?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پYy @<4@<4@<4@,?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ ?5 @<4@<4@@b??ݜp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ ?w#@<4@<4@<4@:)@T>`9?>`9?> p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ?b>-@<4@<4@<4@<4@<4@<4@4?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ4?<4@<4@<4@<4@<4@<4@b>-@?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ `9?>`9?>T>:)@<4@<4@<4@w#@ ?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پݜ?b?@<4@<4@5 @ ?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پL?BA@<4@<4@w#@?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ?<4@<4@<4@'@ap2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ +f>b?kI-@@!2@<4@<4@<4@<4@<4@4@y>~žp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ +f>@<4@<4@<4@<4@<4@<4@<4@<4@<4@<4@@ +f>p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پL?6,?6,?6,?6,?6,?6,?6,?6,?6,?6,?L?p2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پp2پ \ No newline at end of file diff --git a/predict/test/data/ms/mindspore.ms b/predict/test/data/ms/mindspore.ms new file mode 100755 index 00000000000..654e4060aaf Binary files /dev/null and b/predict/test/data/ms/mindspore.ms differ diff --git a/predict/test/data/ms/mindspore.out b/predict/test/data/ms/mindspore.out new file mode 100644 index 00000000000..a7325da0e39 --- /dev/null +++ b/predict/test/data/ms/mindspore.out @@ -0,0 +1,2 @@ +Default/fc3-Dense/BiasAdd-op14 2 1 10 +-2.1191406 -8.4140625 -13.625 2.8222656 -11.4453125 30.734375 7.515625 -9.921875 2.5371094 2.9238281 diff --git a/predict/test/main.cc b/predict/test/main.cc new file mode 100644 index 00000000000..7e18f9a10d0 --- /dev/null +++ b/predict/test/main.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "test/test_context.h" +#include "common/mslog.h" + +int main(int argc, char **argv) { + // Initialize Google Test. + testing::InitGoogleTest(&argc, argv); + + for (size_t i = 0; i < argc; i++) { + std::string arg = std::string(argv[i]); + if (arg.find("--testRoot") != std::string::npos) { + auto testContext = + std::shared_ptr(new (std::nothrow) mindspore::predict::TestContext()); + if (testContext == nullptr) { + MS_LOGE("new testContext failed"); + return 1; + } + testContext->SetTestRoot(arg.substr(arg.find("--testRoot=") + 11)); + break; + } + } + + int result = RUN_ALL_TESTS(); + + return result; +} diff --git a/predict/test/run_tests.sh b/predict/test/run_tests.sh new file mode 100755 index 00000000000..e5a94e70f71 --- /dev/null +++ b/predict/test/run_tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +CUR_DIR=$(dirname "$(readlink -f "$0")") +echo "$CUR_DIR" +DOTEST_DIR="$CUR_DIR"/doTest + +cd "$DOTEST_DIR" +./ms-test +if [ $? -ne 0 ]; then + echo "run ./ms-test failed !" + exit 1 +fi diff --git a/predict/test/src/graph_tests.cc b/predict/test/src/graph_tests.cc new file mode 100644 index 00000000000..8fbaf689f3a --- /dev/null +++ b/predict/test/src/graph_tests.cc @@ -0,0 +1,148 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "schema/inner/ms_generated.h" +#include "src/graph.h" +#include "common/file_utils.h" +#include "test/test_context.h" +#include "include/session.h" + +namespace mindspore { +namespace predict { +class GraphTest : public ::testing::Test { + protected: + void SetUp() {} + + void TearDown() {} + + std::string root; +}; + +void InitMsGraphAllTensor(SubGraphDefT *msSubgraph) { + ASSERT_NE(msSubgraph, nullptr); + std::unique_ptr tensor (new (std::nothrow) TensorDefT); + ASSERT_NE(tensor, nullptr); + tensor->refCount = MSConst_WEIGHT_REFCOUNT; + tensor->format = Format_NCHW; + tensor->dataType = DataType_DT_FLOAT; + tensor->dims = {1, 1, 1, 2}; + tensor->offset = -1; + tensor->data.resize(0); + msSubgraph->allTensors.emplace_back(std::move(tensor)); + + std::unique_ptr tensor2(new (std::nothrow) TensorDefT); + ASSERT_NE(tensor2, nullptr); + tensor2->refCount = MSConst_WEIGHT_REFCOUNT; + tensor2->format = Format_NCHW; + tensor2->dataType = DataType_DT_FLOAT; + tensor2->dims = {1, 1, 1, 2}; + tensor2->offset = -1; + tensor2->data.resize(0); + msSubgraph->allTensors.emplace_back(std::move(tensor2)); + + std::unique_ptr tensor3(new (std::nothrow) TensorDefT); + ASSERT_NE(tensor3, nullptr); + tensor3->refCount = 0; + tensor3->format = Format_NCHW; + tensor3->dataType = DataType_DT_FLOAT; + tensor3->dims = {1, 1, 1, 2}; + tensor3->offset = -1; + tensor3->data.resize(0); + msSubgraph->allTensors.emplace_back(std::move(tensor3)); +} + +void FreeOutputs(std::map> *outputs) { + for (auto &output : (*outputs)) { + for (auto &outputTensor : output.second) { + delete outputTensor; + } + } + outputs->clear(); +} + +void FreeInputs(std::vector *inputs) { + for (auto &input : *inputs) { + input->SetData(nullptr); + delete input; + } + inputs->clear(); + return; +} + +TEST_F(GraphTest, CreateFromFileAdd) { + auto msGraph = std::unique_ptr(new (std::nothrow) GraphDefT()); + ASSERT_NE(msGraph, nullptr); + msGraph->name = "test1"; + auto msSubgraph = std::unique_ptr(new (std::nothrow) SubGraphDefT()); + ASSERT_NE(msSubgraph, nullptr); + msSubgraph->name = msGraph->name + "_1"; + msSubgraph->inputIndex = {0, 1}; + msSubgraph->outputIndex = {2}; + + std::unique_ptr node(new (std::nothrow) NodeDefT); + ASSERT_NE(node, nullptr); + std::unique_ptr opDef(new (std::nothrow) OpDefT); + ASSERT_NE(opDef, nullptr); + node->opDef = std::move(opDef); + node->opDef->isLastConv = false; + node->opDef->inputIndex = {static_cast(0), 1}; + node->opDef->outputIndex = {static_cast(2)}; + node->opDef->name = msSubgraph->name + std::to_string(0); + node->fmkType = FmkType_CAFFE; + + auto attr = std::unique_ptr(new (std::nothrow) AddT()); + ASSERT_NE(attr, nullptr); + attr->format = DataFormatType_NCHW; + node->opDef->attr.type = OpT_Add; + node->opDef->attr.value = attr.release(); + + msSubgraph->nodes.emplace_back(std::move(node)); + + InitMsGraphAllTensor(msSubgraph.get()); + msGraph->subgraphs.emplace_back(std::move(msSubgraph)); + + flatbuffers::FlatBufferBuilder builder(1024); + auto offset = mindspore::predict::GraphDef::Pack(builder, msGraph.get()); + builder.Finish(offset); + int size = builder.GetSize(); + void *content = builder.GetBufferPointer(); + + Context ctx; + auto session = CreateSession(static_cast(content), size, ctx); + + std::vector tmpT = {1, 2}; + void *in1Data = tmpT.data(); + std::vector tmpT2 = {3, 5}; + void *in2Data = tmpT2.data(); + + auto inputs = session->GetInput(); + inputs[0]->SetData(in1Data); + inputs[1]->SetData(in2Data); + + auto ret = session->Run(inputs); + EXPECT_EQ(0, ret); + auto outputs = session->GetAllOutput(); + EXPECT_EQ(4, reinterpret_cast(outputs.begin()->second.front()->GetData())[0]); + EXPECT_EQ(7, reinterpret_cast(outputs.begin()->second.front()->GetData())[1]); + + FreeOutputs(&outputs); + FreeInputs(&inputs); +} +} // namespace predict +} // namespace mindspore diff --git a/predict/test/test_context.cc b/predict/test/test_context.cc new file mode 100644 index 00000000000..ca8f36d3a87 --- /dev/null +++ b/predict/test/test_context.cc @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test/test_context.h" + +namespace mindspore { +namespace predict { +std::string TestContext::GetTestRoot() { return this->testRoot; } + +void TestContext::SetTestRoot(const std::string &testRoot) { this->testRoot = testRoot; } +} // namespace predict +} // namespace mindspore diff --git a/predict/test/test_context.h b/predict/test/test_context.h new file mode 100644 index 00000000000..16f439d6e6c --- /dev/null +++ b/predict/test/test_context.h @@ -0,0 +1,36 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PREDICT_TEST_TEST_CONTEXT_H_ +#define PREDICT_TEST_TEST_CONTEXT_H_ + +#include + +namespace mindspore { +namespace predict { +class TestContext { + public: + TestContext() = default; + std::string GetTestRoot(); + void SetTestRoot(const std::string &testRoot); + + private: + std::string testRoot = "./"; +}; +} // namespace predict +} // namespace mindspore + +#endif // PREDICT_TEST_TEST_CONTEXT_H_ diff --git a/py_filter b/py_filter new file mode 100644 index 00000000000..8301a0a2571 --- /dev/null +++ b/py_filter @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +echo "\"\"\"doc\"\"\"" && python3 -m doxypypy.doxypypy -a -c $1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000000..e4b61f2b6fd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +numpy >= 1.17.0 +protobuf >= 3.8.0 +asttokens >= 1.1.13 +pillow >= 6.2.0 +scipy >= 1.3.3 +dataclasses >= 0.6 +easydict >= 1.9 +sympy >= 1.4 +cffi >= 1.13.2 +wheel >= 0.32.0 +decorator >= 4.4.0 +setuptools >= 40.8.0 +matplotlib >= 3.1.3 # for ut test +opencv-python >= 4.2.0.32 # for ut test diff --git a/scripts/check_clang_format.sh b/scripts/check_clang_format.sh new file mode 100755 index 00000000000..6ed4f7e5de2 --- /dev/null +++ b/scripts/check_clang_format.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +CLANG_FORMAT=$(which clang-format) || (echo "Please install 'clang-format' tool first"; exit 1) + +version=$("${CLANG_FORMAT}" --version | sed -n "s/.*\ \([0-9]*\)\.[0-9]*\.[0-9]*.*/\1/p") +if [[ "${version}" -lt "8" ]]; then + echo "clang-format's version must be at least 8.0.0" + exit 1 +fi + +CURRENT_PATH=$(pwd) +SCRIPTS_PATH=$(dirname "$0") + +echo "CURRENT_PATH=$CURRENT_PATH" +echo "SCRIPTS_PATH=$SCRIPTS_PATH" + +# print usage message +function usage() +{ + echo "Check whether the specified source files were well formated" + echo "Usage:" + echo "bash $0 [-a] [-c] [-l] [-h]" + echo "e.g. $0 -a" + echo "" + echo "Options:" + echo " -a Check code format of all files, default case" + echo " -c Check code format of the files changed compared to last commit" + echo " -l Check code format of the files changed in last commit" + echo " -h Print usage" +} + +# check and set options +function checkopts() +{ + # init variable + mode="all" # default check all files + + # Process the options + while getopts 'aclh' opt + do + case "${opt}" in + a) + mode="all" + ;; + c) + mode="changed" + ;; + l) + mode="lastcommit" + ;; + h) + usage + exit 0 + ;; + *) + echo "Unknown option ${opt}!" + usage + exit 1 + esac + done +} + +# init variable +# check options +checkopts "$@" + +# switch to project root path, which contains clang-format config file '.clang-format' +cd "${SCRIPTS_PATH}/.." || exit 1 + +CHECK_LIST_FILE='__checked_files_list__' + +if [ "X${mode}" == "Xall" ]; then + find mindspore/ccsrc -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true +elif [ "X${mode}" == "Xchanged" ]; then + # --diff-filter=ACMRTUXB will ignore deleted files in commit + git diff --diff-filter=ACMRTUXB --name-only | grep "mindspore/ccsrc" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true +else # "X${mode}" == "Xlastcommit" + git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "mindspore/ccsrc" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true +fi + +CHECK_RESULT_FILE=__code_format_check_result__ +echo "0" > "$CHECK_RESULT_FILE" + +# check format of files modified in the lastest commit +while read line; do + BASE_NAME=$(basename "${line}") + TEMP_FILE="__TEMP__${BASE_NAME}" + cp "${line}" "${TEMP_FILE}" + ${CLANG_FORMAT} -i "${TEMP_FILE}" + diff "${TEMP_FILE}" "${line}" + ret=$? + rm "${TEMP_FILE}" + if [[ "${ret}" -ne 0 ]]; then + echo "File ${line} is not formated, please format it." + echo "1" > "${CHECK_RESULT_FILE}" + break + fi +done < "${CHECK_LIST_FILE}" + +result=$(cat "${CHECK_RESULT_FILE}") +rm "${CHECK_RESULT_FILE}" +rm "${CHECK_LIST_FILE}" +cd "${CURRENT_PATH}" || exit 1 +if [[ "X${result}" == "X0" ]]; then + echo "Check PASS: specified files are well formated!" +fi +exit "${result}" diff --git a/scripts/dot2svg.sh b/scripts/dot2svg.sh new file mode 100755 index 00000000000..efa37e766df --- /dev/null +++ b/scripts/dot2svg.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +# set -x +# Usage: ./scripts/dot2svg.sh [search_path] +if [[ -z "$1" ]] + then + DIR="." + else + DIR="$1" +fi + +for f in "${DIR}"/*.dot +do +dot -Tsvg -o "${DIR}/$(basename "${f}").svg" "${DIR}/$(basename "${f}")" +done diff --git a/scripts/format_source_code.sh b/scripts/format_source_code.sh new file mode 100755 index 00000000000..39c5df7ce5a --- /dev/null +++ b/scripts/format_source_code.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +CLANG_FORMAT=$(which clang-format) || (echo "Please install 'clang-format' tool first"; exit 1) + +version=$("${CLANG_FORMAT}" --version | sed -n "s/.*\ \([0-9]*\)\.[0-9]*\.[0-9]*.*/\1/p") +if [[ "${version}" -lt "8" ]]; then + echo "clang-format's version must be at least 8.0.0" + exit 1 +fi + +CURRENT_PATH=$(pwd) +SCRIPTS_PATH=$(dirname "$0") + +echo "CURRENT_PATH=${CURRENT_PATH}" +echo "SCRIPTS_PATH=${SCRIPTS_PATH}" + +# print usage message +function usage() +{ + echo "Format the specified source files to conform the code style." + echo "Usage:" + echo "bash $0 [-a] [-c] [-l] [-h]" + echo "e.g. $0 -c" + echo "" + echo "Options:" + echo " -a format of all files" + echo " -c format of the files changed compared to last commit, default case" + echo " -l format of the files changed in last commit" + echo " -h Print usage" +} + +# check and set options +function checkopts() +{ + # init variable + mode="changed" # default format changed files + + # Process the options + while getopts 'aclh' opt + do + case "${opt}" in + a) + mode="all" + ;; + c) + mode="changed" + ;; + l) + mode="lastcommit" + ;; + h) + usage + exit 0 + ;; + *) + echo "Unknown option ${opt}!" + usage + exit 1 + esac + done +} + +# init variable +# check options +checkopts "$@" + +# switch to project root path, which contains clang-format config file '.clang-format' +cd "${SCRIPTS_PATH}/.." || exit 1 + +FMT_FILE_LIST='__format_files_list__' + +if [[ "X${mode}" == "Xall" ]]; then + find mindspore/ccsrc -type f -name "*" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true +elif [[ "X${mode}" == "Xchanged" ]]; then + git diff --name-only | grep "mindspore/ccsrc" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true +else # "X${mode}" == "Xlastcommit" + git diff --name-only HEAD~ HEAD | grep "mindspore/ccsrc" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true +fi + +while read line; do + if [ -f "${line}" ]; then + ${CLANG_FORMAT} -i "${line}" + fi +done < "${FMT_FILE_LIST}" + +rm "${FMT_FILE_LIST}" +cd "${CURRENT_PATH}" || exit 1 + +echo "Specified cpp source files have been format successfully." diff --git a/scripts/get_bert_shape_from_pytest.sh b/scripts/get_bert_shape_from_pytest.sh new file mode 100755 index 00000000000..944170d92d7 --- /dev/null +++ b/scripts/get_bert_shape_from_pytest.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +CURRPATH=$(cd "$(dirname $0)"; pwd) +PROJECT_PATH="${CURRPATH}/.." +SHP_BASENAME="test_bert_train" +BUILD_PATH="${PROJECT_PATH}/build" + +cd "${PROJECT_PATH}"; sh build.sh -t off -l none -r -T; cd - + +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${BUILD_PATH}/third_party/gtest/lib" +export PYTHONPATH="${PYTHONPATH}:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/python_input" + +test_bert_train="${PROJECT_PATH}/tests/perf_test/test_bert_train.py" + +export SAVE_GRAPHS='YES' +export SAVE_GRAPHS_PATH="${PROJECT_PATH}" +for version in base large +do + for batch_size in 1 2 4 8 16 32 64 128 256 512 1024 + do + export VERSION="${version}" + export BATCH_SIZE="${batch_size}" + target_file="${PROJECT_PATH}/${SHP_BASENAME}.${VERSION}.${BATCH_SIZE}.shp" + pytest "${test_bert_train}" + cp "${SAVE_GRAPHS_PATH}/9_validate.dat" "${target_file}" + done +done diff --git a/scripts/get_op_use_count.sh b/scripts/get_op_use_count.sh new file mode 100755 index 00000000000..91789a03298 --- /dev/null +++ b/scripts/get_op_use_count.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +./scripts/get_shape_from_ir.sh "$1" |awk -F'\t' '{print $2}' | sort |uniq -c |sort -r diff --git a/scripts/get_shape_from_ir.sh b/scripts/get_shape_from_ir.sh new file mode 100755 index 00000000000..8e603add4ab --- /dev/null +++ b/scripts/get_shape_from_ir.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +# Usage : get_shape_from_ir.sh ir_file + +cat "$1" | perl -p -e 's/\n/NEWLINE/' | sed 's/NEWLINE :/:/g' | sed 's/Tensor NEWLINEshape//g' | perl -p -e 's/NEWLINE/\n/g' | perl -p -e 's//\2/g' | perl -p -e 's//Tuple/g' | perl -p -e 's/ \%(\d+)\(.*= /\1\t/g' | perl -p -e 's/\(.*\)( \{.*\})*:/\t\1\t/g' | tr -d '()' | awk '/subgraph/{p=1;next}{if(p){print}}'| awk '/return/{p=1;next}{if(!p){print}}' | sed '/^$/d' | awk -F'\t' '{print $1"\t"$2"\t"$4"\t"$3}' diff --git a/scripts/run_perf_test.sh b/scripts/run_perf_test.sh new file mode 100755 index 00000000000..52f8e4ffa4d --- /dev/null +++ b/scripts/run_perf_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +CURRPATH=$(cd "$(dirname $0)"; pwd) +PROJECT_PATH="${CURRPATH}/.." +PYTHONTEST_DIR="${PROJECT_PATH}/tests/perf_test" +PERF_RESULT_DIR="${CURRPATH}/" +PERF_SUFFIX=".perf" +if [[ "${BUILD_PATH}" ]];then + echo "BUILD_PATH = ${BUILD_PATH}" +else + BUILD_PATH="${PROJECT_PATH}/build" + echo "BUILD_PATH = ${BUILD_PATH}" +fi + +cd "${PROJECT_PATH}"; sh build.sh -t off -l none -r -p on -j 20; cd - + +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${BUILD_PATH}/third_party/gtest/lib" +export PYTHONPATH="${PYTHONPATH}:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/python_input" +echo "export PYTHONPATH=${PYTHONPATH}:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/python_input" + +for f in "${PYTHONTEST_DIR}"/test_*.py +do + target_file="${PERF_RESULT_DIR}$(basename ${f} .py)${PERF_SUFFIX}" + pytest -s "${f}" > "${target_file}" 2>&1 +done diff --git a/scripts/setdotlabelwidth b/scripts/setdotlabelwidth new file mode 100755 index 00000000000..6094615b059 --- /dev/null +++ b/scripts/setdotlabelwidth @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import sys +import math + +MIN_CHARS_PER_ROW = 20 # minimal number of chars on one line +COLS_TO_ROWS_RATIO = 4 # ratio of # columns to # rows (unit is char) +MAX_LABEL_LENGTH = 1000 # maximal number of chars of a label + + +# for specified number of chars, calculate a suitable number for how many chars per row +# Num Total Chars : n +# Num Chars Per Row : m +# Num Rows : k +# (k - 1) * m < n <= k * m +# suppose: m / k = u +# ==> sqrt(u * n) <= m < u / 2 + sqrt(u * n + u * u / 4) +# m = max(20, m) +# parameters: +# @total_chars number of total chars +# @ratio ratio of # columns to # rows (unit is char) +def calc_num_chars_per_row(total_chars, ratio): + chars_per_row = math.ceil(math.sqrt(total_chars * ratio)) + return max(MIN_CHARS_PER_ROW, chars_per_row) + + +def process_label_text(text): + label_len = min(len(text), MAX_LABEL_LENGTH) + chars_per_row = calc_num_chars_per_row(label_len, COLS_TO_ROWS_RATIO) + if label_len <= MIN_CHARS_PER_ROW: + return text + beg_idx = 0 + texts = [] + while beg_idx < label_len: + end_idx = beg_idx + chars_per_row + if end_idx >= label_len: + texts.append(text[beg_idx:label_len]) + else: + texts.append(text[beg_idx:end_idx]) + beg_idx = end_idx + return "\\n".join(texts) + + +# insert '\n' to labels which are too long +def process_file(f): + line_text = f.readline() + beg_idx = -1 # label begin index, index of 'label="' + end_idx = -1 # label end index, index of '"' + label_text = "" + label_prefix = "" + label_postfix = "" + while line_text: + line_text = line_text.rstrip() + + if beg_idx < 0: + beg_idx = line_text.find('label="') + if beg_idx >= 0: + end_idx = line_text.find('"', beg_idx + len('label="')) + if end_idx >= 0: # the full label text is on one line + label = line_text[beg_idx + len('label="'):end_idx] + print('%slabel="%s"%s' % (line_text[0:beg_idx], process_label_text(label), line_text[end_idx + 1:])) + beg_idx = -1 # reset to initial conditions + else: # the label text is distributed on multiple lines + label_prefix = line_text[0:beg_idx] + label_text = line_text[beg_idx + len('label="'):] + else: + print(line_text) + else: + end_idx = line_text.find('"') + if end_idx >= 0: + label_text = label_text + line_text[0:end_idx] + label_postfix = line_text[end_idx + 1:] + print('%slabel="%s"%s' % (label_prefix, process_label_text(label_text), label_postfix)) + beg_idx = -1 # reset to initial conditions + else: + label_text += line_text + + # print(f'{len(line_text)} - {line_text}') + line_text = f.readline() + + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "--help": + print("Usage: %s < dotfile | dot -Tpng -o filename.png" % sys.argv[0]) + sys.exit() + + # read text from stdin + process_file(sys.stdin) diff --git a/scripts/update_onnx_weight.py b/scripts/update_onnx_weight.py new file mode 100755 index 00000000000..65a33539cfd --- /dev/null +++ b/scripts/update_onnx_weight.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Use checkpoint file and onnx file as inputs, create a new onnx with Initializer's value from checkpoint file +Usage: + python update_onnx_weight.py onnx_file checkpoint_file [output_file] +""" +import sys +from onnx import onnx_pb +from mindspore.train.serialization import load_checkpoint + + +def update_onnx_initializer(onnx_file, ckpt_file, output_file): + "Update onnx initializer." + with open(onnx_file, 'rb') as f: + data = f.read() + model = onnx_pb.ModelProto() + model.ParseFromString(data) + initializer = model.graph.initializer + param_dict = load_checkpoint(ckpt_file) + + for i, _ in enumerate(initializer): + item = initializer[i] + #print(item.name, item.data_type, item.dims, len(item.raw_data)) + if not item.name in param_dict: + print(f"Warning: Can not find '{item.name}' in checkpoint parameters dictionary") + continue + weight = param_dict[item.name].data.asnumpy() + bin_data = weight.tobytes() + if len(item.raw_data) != len(bin_data): + print(f"Warning: Size of weight from checkpoint is different from original size, ignore it") + continue + item.raw_data = bin_data + + pb_msg = model.SerializeToString() + with open(output_file, 'wb') as f: + f.write(pb_msg) + + print(f'Graph name: {model.graph.name}') + print(f'Initializer length: {len(initializer)}') + print(f'Checkpoint dict length: {len(param_dict)}') + print(f'The new weights have been written to file {output_file} successfully') + + +def main(): + if len(sys.argv) < 3: + print(f'Usage: {sys.argv[0]} onnx_file checkpoint_file [output_file]') + sys.exit(1) + onnx_file = sys.argv[1] + ckpt_file = sys.argv[2] + output_file = f'new_{onnx_file}' if len(sys.argv) == 3 else sys.argv[3] + update_onnx_initializer(onnx_file, ckpt_file, output_file) + + +if __name__ == '__main__': + main() diff --git a/setup_package.py b/setup_package.py new file mode 100644 index 00000000000..8b6889cd340 --- /dev/null +++ b/setup_package.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup_package.""" +import os +import stat +from setuptools import setup, find_packages +from setuptools.command.egg_info import egg_info +from setuptools.command.build_py import build_py + +package_name = 'mindspore' +version = '0.1.0' +author = 'The MindSpore Authors' +author_email = 'contact@mindspore.cn' +home_page = 'https://www.mindspore.cn' + +backend_policy = os.getenv('BACKEND_POLICY') +commit_id = os.getenv('COMMIT_ID').replace("\n", "") + +pwd = os.path.dirname(os.path.realpath(__file__)) +pkg_dir = os.path.join(pwd, 'build/package') + +def write_version(file): + file.write("__version__ = '{}'\n".format(version)) + +def write_config(file): + file.write("__backend__ = '{}'\n".format(backend_policy)) + +def write_commit_file(file): + file.write("__commit_id__ = '{}'\n".format(commit_id)) + +def build_depends(): + """generate python file""" + version_file = os.path.join(pwd, 'build/package/mindspore', 'version.py') + with open(version_file, 'w') as f: + write_version(f) + + version_file = os.path.join(pwd, 'mindspore/', 'version.py') + with open(version_file, 'w') as f: + write_version(f) + + config_file = os.path.join(pwd, 'build/package/mindspore', 'default_config.py') + with open(config_file, 'w') as f: + write_config(f) + + config_file = os.path.join(pwd, 'mindspore/', 'default_config.py') + with open(config_file, 'w') as f: + write_config(f) + + commit_file = os.path.join(pwd, 'build/package/mindspore', '.commit_id') + with open(commit_file, 'w') as f: + write_commit_file(f) + + commit_file = os.path.join(pwd, 'mindspore/', '.commit_id') + with open(commit_file, 'w') as f: + write_commit_file(f) + +descriptions = 'An AI computing framework that supports development for AI applications in all scenarios.' + +requires = [ + 'numpy >= 1.17.0', + 'protobuf >= 3.8.0', + 'asttokens >= 1.1.13', + 'pillow >= 6.2.0', + 'scipy == 1.3.3', + 'easydict >= 1.9', + 'sympy >= 1.4', + 'cffi >= 1.13.2', + 'decorator >= 4.4.0' + ], + +package_datas = { + '': [ + '*.so*', + 'lib/*.so*', + 'lib/*.a', + '.commit_id', + ] +} + +build_depends() + +def update_permissions(path): + """ + Update permissions. + + Args: + path (str): Target directory path. + """ + for dirpath, dirnames, filenames in os.walk(path): + for dirname in dirnames: + dir_fullpath = os.path.join(dirpath, dirname) + os.chmod(dir_fullpath, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC | stat.S_IRGRP | stat.S_IXGRP) + for filename in filenames: + file_fullpath = os.path.join(dirpath, filename) + os.chmod(file_fullpath, stat.S_IREAD) + +class EggInfo(egg_info): + """Egg info.""" + def run(self): + super().run() + egg_info_dir = os.path.join(pkg_dir, 'mindspore.egg-info') + update_permissions(egg_info_dir) + +class BuildPy(build_py): + """BuildPy.""" + def run(self): + super().run() + mindspore_dir = os.path.join(pkg_dir, 'build', 'lib', 'mindspore') + update_permissions(mindspore_dir) + mindspore_dir = os.path.join(pkg_dir, 'build', 'lib', 'akg') + update_permissions(mindspore_dir) + +setup( + python_requires='>=3.7', + name=package_name, + version=version, + author=author, + author_email=author_email, + url=home_page, + packages=find_packages(), + package_data=package_datas, + include_package_data=True, + cmdclass={ + 'egg_info': EggInfo, + 'build_py': BuildPy, + }, + install_requires=requires, + description=descriptions, + license='Apache 2.0', +) diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 00000000000..f935021a8f8 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000000..d73442fc402 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,5 @@ +#add flags +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") + +add_subdirectory("ut") + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/dataset_mock.py b/tests/dataset_mock.py new file mode 100644 index 00000000000..c5668facdc0 --- /dev/null +++ b/tests/dataset_mock.py @@ -0,0 +1,79 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +'''Remove after MindData merge to MindSpore ''' +import numpy as np +from mindspore import Tensor + + +class MindData: + """ Stub for MindData """ + + def __init__(self, size=None, batch_size=None, repeat_count=1, + np_types=None, output_shapes=None, input_indexs=()): + self._size = size + self._batch_size = batch_size + self._repeat_count = repeat_count + self._np_types = np_types + self._output_shapes = output_shapes + self._input_indexs = input_indexs + self._iter_num = 0 + + def get_dataset_size(self): + return self._size + + def get_repeat_count(self): + return self._repeat_count + + def get_batch_size(self): + return self._batch_size + + def output_types(self): + return self._np_types + + def output_shapes(self): + return self._output_shapes + + @property + def input_indexs(self): + return self._input_indexs + + def device_que(self): + self.queue_name = '6ba41974-209e-11ea-88b0-a24efeb2c736' + return self + + def send(self): + pass + + def __len__(self): + return self._size + + def __iter__(self): + return self + + def __next__(self): + if self._size < self._iter_num: + raise StopIteration + self._iter_num += 1 + next_value = [] + for shape, typ in zip(self._output_shapes, self._np_types): + next_value.append(Tensor(np.ndarray(shape, typ))) + + return tuple(next_value) + + def next(self): + return self.__next__() + + def reset(self): + self._iter_num = 0 diff --git a/tests/mindspore_test_framework/__init__.py b/tests/mindspore_test_framework/__init__.py new file mode 100644 index 00000000000..3a97034753c --- /dev/null +++ b/tests/mindspore_test_framework/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""mindspore_test_framework""" +import mindspore.context as context + + +def setup_module(module): + # pylint: disable=unused-argument + context.set_context(mode=context.GRAPH_MODE) diff --git a/tests/mindspore_test_framework/apps/__init__.py b/tests/mindspore_test_framework/apps/__init__.py new file mode 100644 index 00000000000..e5f476de9e6 --- /dev/null +++ b/tests/mindspore_test_framework/apps/__init__.py @@ -0,0 +1 @@ +"""apps""" diff --git a/tests/mindspore_test_framework/apps/bert_attention_submodules.py b/tests/mindspore_test_framework/apps/bert_attention_submodules.py new file mode 100644 index 00000000000..5bd7be9c47a --- /dev/null +++ b/tests/mindspore_test_framework/apps/bert_attention_submodules.py @@ -0,0 +1,380 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Bert submodules.""" + +# pylint: disable=missing-docstring, arguments-differ + +import math +import numpy as np +import mindspore.common.dtype as mstype +import mindspore.ops.functional as F +from mindspore import nn +from mindspore.common.initializer import TruncatedNormal +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.model_zoo.Bert_NEZHA.bert_model import SaturateCast, RelaPosEmbeddingsGenerator + + +class BertAttentionQueryKeyMul(nn.Cell): + def __init__(self, + batch_size, + from_tensor_width, + to_tensor_width, + from_seq_length, + to_seq_length, + num_attention_heads=1, + size_per_head=512, + query_act=None, + key_act=None, + initializer_range=0.02): + super(BertAttentionQueryKeyMul, self).__init__() + self.from_tensor_width = from_tensor_width + self.to_tensor_width = to_tensor_width + self.units = num_attention_heads * size_per_head + self.weight = TruncatedNormal(initializer_range) + + self.trans_shape = (0, 2, 1, 3) + self.transpose = P.Transpose() + self.reshape = P.Reshape() + self.shp_from_2d = (-1, self.from_tensor_width) + self.shp_to_2d = (-1, self.to_tensor_width) + self.query_layer = nn.Dense(self.from_tensor_width, + self.units, + activation=query_act, + weight_init=self.weight) + self.key_layer = nn.Dense(self.to_tensor_width, + self.units, + activation=key_act, + weight_init=self.weight) + + self.shp_from = (batch_size, from_seq_length, num_attention_heads, size_per_head) + self.shp_to = ( + batch_size, to_seq_length, num_attention_heads, size_per_head) + + self.matmul_trans_b = P.BatchMatMul(transpose_b=True) + self.cast = P.Cast() + + def construct(self, from_tensor, to_tensor): + from_tensor_2d = self.reshape(from_tensor, self.shp_from_2d) + to_tensor_2d = self.reshape(to_tensor, self.shp_to_2d) + from_tensor_2d = self.cast(from_tensor_2d, mstype.float32) + to_tensor_2d = self.cast(to_tensor_2d, mstype.float32) + query_out = self.query_layer(from_tensor_2d) + key_out = self.key_layer(to_tensor_2d) + + query_layer = self.reshape(query_out, self.shp_from) + query_layer = self.transpose(query_layer, self.trans_shape) + key_layer = self.reshape(key_out, self.shp_to) + key_layer = self.transpose(key_layer, self.trans_shape) + + attention_scores = self.matmul_trans_b(query_layer, key_layer) + + return query_layer, key_layer, attention_scores + + +class BertAttentionRelativePositionKeys(nn.Cell): + def __init__(self, + batch_size, + from_seq_length, + to_seq_length, + num_attention_heads=1, + size_per_head=512, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=False, + dtype=mstype.float32, + compute_type=mstype.float32): + super(BertAttentionRelativePositionKeys, self).__init__() + self.batch_size = batch_size + self.from_seq_length = from_seq_length + self.to_seq_length = to_seq_length + self.use_relative_positions = use_relative_positions + self.size_per_head = size_per_head + self.num_attention_heads = num_attention_heads + self.trans_shape_position = (1, 2, 0, 3) + self.trans_shape_relative = (2, 0, 1, 3) + + self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))], dtype=dtype) + + self.reshape = P.Reshape() + self.multiply = P.Mul() + self.transpose = P.Transpose() + self.matmul_trans_b = P.BatchMatMul(transpose_b=True) + self.batch_num = batch_size * num_attention_heads + self.cast = P.Cast() + + self.cast_compute_type = SaturateCast(dst_type=compute_type) + self._generate_relative_positions_embeddings = \ + RelaPosEmbeddingsGenerator(length=self.to_seq_length, + depth=self.size_per_head, + max_relative_position=16, + initializer_range=initializer_range, + use_one_hot_embeddings=use_one_hot_embeddings) + + def construct(self, input_tensor, query_layer): + # use_relative_position, supplementary logic + relations_keys_embeddings = self._generate_relative_positions_embeddings() + if self.use_relative_positions: + # 'relations_keys' = [F|T, F|T, H] + relations_keys = self.cast_compute_type(relations_keys_embeddings) + # query_layer_t is [F, B, N, H] + query_layer_t = self.transpose(query_layer, self.trans_shape_relative) + # query_layer_r is [F, B * N, H] + query_layer_r = self.reshape(query_layer_t, + (self.from_seq_length, + self.batch_num, + self.size_per_head)) + # key_position_scores is [F, B * N, F|T] + query_layer_r = self.cast(query_layer_r, mstype.float32) + key_position_scores = self.matmul_trans_b(query_layer_r, + relations_keys) + # key_position_scores_r is [F, B, N, F|T] + key_position_scores_r = self.reshape(key_position_scores, + (self.from_seq_length, + self.batch_size, + self.num_attention_heads, + self.from_seq_length)) + # key_position_scores_r_t is [B, N, F, F|T] + key_position_scores_r_t = self.transpose(key_position_scores_r, + self.trans_shape_position) + input_tensor = self.cast(input_tensor, mstype.float32) + + input_tensor = input_tensor + key_position_scores_r_t + + attention_scores = self.multiply(input_tensor, self.scores_mul) + + return relations_keys_embeddings, attention_scores + + +class BertAttentionMask(nn.Cell): + def __init__(self, + has_attention_mask=False, + dtype=mstype.float32): + + super(BertAttentionMask, self).__init__() + self.has_attention_mask = has_attention_mask + self.multiply_data = Tensor([-1000.0,], dtype=dtype) + self.multiply = P.Mul() + + if self.has_attention_mask: + self.expand_dims = P.ExpandDims() + self.sub = P.Sub() + self.add = P.TensorAdd() + self.cast = P.Cast() + self.get_dtype = P.DType() + + def construct(self, input_tensor, attention_mask): + attention_scores = input_tensor + attention_scores = self.cast(attention_scores, mstype.float32) + if self.has_attention_mask: + attention_mask = self.expand_dims(attention_mask, 1) + multiply_out = self.sub(self.cast(F.tuple_to_array((1.0,)), mstype.float32), + self.cast(attention_mask, self.get_dtype(attention_scores))) + + adder = self.multiply(multiply_out, self.multiply_data) + attention_scores = self.add(adder, attention_scores) + + return attention_scores + +class BertAttentionMaskBackward(nn.Cell): + def __init__(self, + attention_mask_shape, + has_attention_mask=False, + dtype=mstype.float32): + super(BertAttentionMaskBackward, self).__init__() + self.has_attention_mask = has_attention_mask + self.multiply_data = Tensor([-1000.0,], dtype=dtype) + self.multiply = P.Mul() + self.attention_mask = Tensor(np.ones(shape=attention_mask_shape).astype(np.float32)) + if self.has_attention_mask: + self.expand_dims = P.ExpandDims() + self.sub = P.Sub() + self.add = P.TensorAdd() + self.cast = P.Cast() + self.get_dtype = P.DType() + + def construct(self, input_tensor): + attention_scores = input_tensor + attention_scores = self.cast(attention_scores, mstype.float32) + if self.has_attention_mask: + attention_mask = self.expand_dims(self.attention_mask, 1) + multiply_out = self.sub(self.cast(F.tuple_to_array((1.0,)), mstype.float32), + self.cast(attention_mask, self.get_dtype(attention_scores))) + + adder = self.multiply(multiply_out, self.multiply_data) + attention_scores = self.add(adder, attention_scores) + return attention_scores + +class BertAttentionSoftmax(nn.Cell): + def __init__(self, + batch_size, + to_tensor_width, + from_seq_length, + to_seq_length, + num_attention_heads=1, + size_per_head=512, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02): + super(BertAttentionSoftmax, self).__init__() + self.to_tensor_width = to_tensor_width + self.value_act = value_act + + self.reshape = P.Reshape() + + self.shp_to_2d = (-1, self.to_tensor_width) + self.shp_from = (batch_size, from_seq_length, num_attention_heads, size_per_head) + self.shp_to = ( + batch_size, to_seq_length, num_attention_heads, size_per_head) + + self.trans_shape = (0, 2, 1, 3) + self.trans_shape_start = (0, 1) + self.matmul = P.BatchMatMul() + + self.units = num_attention_heads * size_per_head + self.weight = TruncatedNormal(initializer_range) + + self.softmax = nn.Softmax() + self.dropout = nn.Dropout(1 - attention_probs_dropout_prob) + self.transpose = P.Transpose() + + self.value_layer = nn.Dense(self.to_tensor_width, + self.units, + activation=value_act, + weight_init=self.weight) + self.cast = P.Cast() + + def construct(self, to_tensor, attention_scores): + to_tensor = self.transpose(to_tensor, self.trans_shape_start) + to_tensor_2d = self.reshape(to_tensor, self.shp_to_2d) + to_tensor_2d = self.cast(to_tensor_2d, mstype.float32) + value_out = self.value_layer(to_tensor_2d) + + attention_probs = self.softmax(attention_scores) + attention_probs = self.cast(attention_probs, mstype.float32) + + value_layer = self.reshape(value_out, self.shp_to) + value_layer = self.transpose(value_layer, self.trans_shape) + + context_layer = self.matmul(attention_probs, value_layer) + + return value_layer, context_layer + + +class BertAttentionRelativePositionValues(nn.Cell): + def __init__(self, + batch_size, + from_seq_length, + to_seq_length, + num_attention_heads=1, + size_per_head=512, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=False, + use_relative_positions=False, + dtype=mstype.float32, + compute_type=mstype.float32): + + super(BertAttentionRelativePositionValues, self).__init__() + self.batch_size = batch_size + self.from_seq_length = from_seq_length + self.to_seq_length = to_seq_length + self.use_relative_positions = use_relative_positions + self.size_per_head = size_per_head + self.num_attention_heads = num_attention_heads + self.trans_shape_position = (1, 2, 0, 3) + self.trans_shape_relative = (2, 0, 1, 3) + + self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))], dtype=dtype) + self.trans_shape = (0, 2, 1, 3) + + self.reshape = P.Reshape() + self.multiply = P.Mul() + self.transpose = P.Transpose() + self.batch_num = batch_size * num_attention_heads + self.matmul = P.BatchMatMul() + self.do_return_2d_tensor = do_return_2d_tensor + if self.do_return_2d_tensor: + self.shp_return = (batch_size * from_seq_length, num_attention_heads * size_per_head) + else: + self.shp_return = (batch_size, from_seq_length, num_attention_heads * size_per_head) + + self.cast_compute_type = SaturateCast(dst_type=compute_type) + self._generate_relative_positions_embeddings = \ + RelaPosEmbeddingsGenerator(length=self.to_seq_length, + depth=self.size_per_head, + max_relative_position=16, + initializer_range=initializer_range, + use_one_hot_embeddings=use_one_hot_embeddings) + self.fill = P.Fill() + self.multiply = P.Mul() + self.type = P.DType() + self.cast = P.Cast() + + def construct(self, input_tensor, attention_probs): + # use_relative_position, supplementary logic + relations_values_embedding = self._generate_relative_positions_embeddings() # (128, 128, 64) + if self.use_relative_positions: + # 'relations_values' = [F|T, F|T, H] + relations_values = self.cast_compute_type(relations_values_embedding) + # attention_probs_t is [F, B, N, T] + attention_probs_t = self.transpose(attention_probs, self.trans_shape_relative) + # attention_probs_r is [F, B * N, T] + attention_probs_r = self.reshape( + attention_probs_t, + (self.from_seq_length, + self.batch_num, + self.to_seq_length)) # (128,768,128) + # value_position_scores is [F, B * N, H] + value_position_scores = self.matmul(attention_probs_r, + relations_values) + # value_position_scores_r is [F, B, N, H] + value_position_scores_r = self.reshape(value_position_scores, + (self.from_seq_length, + self.batch_size, + self.num_attention_heads, + self.size_per_head)) + # value_position_scores_r_t is [B, N, F, H] + value_position_scores_r_t = self.transpose(value_position_scores_r, + self.trans_shape_position) + input_tensor = input_tensor + value_position_scores_r_t + + context_layer = self.transpose(input_tensor, self.trans_shape) + context_layer = self.reshape(context_layer, self.shp_return) + # ge reshape should not return, need an operator here + ones = self.cast(self.fill((1, 1), 1), self.type(context_layer)) + context_layer = self.multiply(context_layer, ones) + return relations_values_embedding, context_layer + + +class BertDense(nn.Cell): + def __init__(self, + hidden_size=768, + intermediate_size=3072, + initializer_range=0.02): + super(BertDense, self).__init__() + self.intermediate = nn.Dense(in_channels=hidden_size, + out_channels=intermediate_size, + activation=None, + weight_init=TruncatedNormal( + initializer_range) + ) + self.cast = P.Cast() + + def construct(self, attention_output): + attention_output = self.cast(attention_output, mstype.float32) + intermediate_output = self.intermediate(attention_output) + return intermediate_output diff --git a/tests/mindspore_test_framework/apps/data/grad_0.npy b/tests/mindspore_test_framework/apps/data/grad_0.npy new file mode 100644 index 00000000000..c92635dee8d Binary files /dev/null and b/tests/mindspore_test_framework/apps/data/grad_0.npy differ diff --git a/tests/mindspore_test_framework/apps/data/grad_1.npy b/tests/mindspore_test_framework/apps/data/grad_1.npy new file mode 100644 index 00000000000..c92635dee8d Binary files /dev/null and b/tests/mindspore_test_framework/apps/data/grad_1.npy differ diff --git a/tests/mindspore_test_framework/apps/data/input_0.npy b/tests/mindspore_test_framework/apps/data/input_0.npy new file mode 100644 index 00000000000..20a800f355b Binary files /dev/null and b/tests/mindspore_test_framework/apps/data/input_0.npy differ diff --git a/tests/mindspore_test_framework/apps/data/input_1.npy b/tests/mindspore_test_framework/apps/data/input_1.npy new file mode 100644 index 00000000000..20a800f355b Binary files /dev/null and b/tests/mindspore_test_framework/apps/data/input_1.npy differ diff --git a/tests/mindspore_test_framework/apps/test_bert_check_gradient.py b/tests/mindspore_test_framework/apps/test_bert_check_gradient.py new file mode 100644 index 00000000000..60f85813efd --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_bert_check_gradient.py @@ -0,0 +1,492 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Test bert check gradient.""" + +import numpy as np + +from mindspore.model_zoo.Bert_NEZHA import GetNextSentenceOutput, BertNetworkWithLoss +from mindspore.model_zoo.Bert_NEZHA.bert_model import BertConfig, \ + EmbeddingLookup, EmbeddingPostprocessor, BertOutput, \ + BertAttention, BertSelfAttention, SaturateCast, TruncatedNormal,\ + BertEncoderCell, BertTransformer, CreateAttentionMaskFromInputMask, BertModel + +from mindspore import context, nn +import mindspore.common.dtype as mstype + +from ..pipeline.gradient.compare_gradient import \ + pipeline_for_compare_inputs_grad_with_numerical_diff_for_group_by_group_config, \ + pipeline_for_compare_params_grad_with_numerical_diff_for_group_by_group_config +from ..mindspore_test import mindspore_test + +from .bert_attention_submodules import BertAttentionQueryKeyMul, BertAttentionRelativePositionKeys, \ + BertAttentionMaskBackward, BertAttentionSoftmax, BertAttentionRelativePositionValues, BertDense + +verification_set = { + 'inputs': [ + { + 'id': 'SaturateCast_CICase', + 'group': 'bert', + 'desc_inputs': [ + [1, 3, 4, 4], + ] + }, + { + 'id': 'BertAttention', + 'group': 'bert', + 'desc_inputs': [ + [1, 128, 1024], [1, 128, 1024], [1, 128, 128], + ] + }, + { + 'id': 'BertOutput', + 'group': 'bert', + 'desc_inputs': [ + [8192, 1024], [8192, 1024], + ] + }, + { + 'id': 'BertSelfAttention', + 'group': 'bert', + 'desc_inputs': [ + [1, 128, 1024], [1, 128, 1024] + ] + }, + { + 'id': 'BertEncoderCell', + 'group': 'bert', + 'desc_inputs': [ + [1, 128, 1024], [1, 128, 128] + ] + }, + { + 'id': 'BertTransformer', + 'group': 'bert', + 'desc_inputs': [ + [1, 128, 1024], [1, 128, 128] + ] + }, + { + 'id': 'EmbeddingLookup', + 'group': 'bert', + 'desc_inputs': [ + np.random.rand(128).astype(np.int32) + ] + }, + { + 'id': 'EmbeddingPostprocessor', + 'group': 'bert', + 'desc_inputs': [ + np.random.rand(128).astype(np.int32), [1, 128, 1024] + ] + }, + { + 'id': 'CreateAttentionMaskFromInputMask', + 'group': 'bert', + 'desc_inputs': [ + [128] + ] + }, + { + 'id': 'BertModel', + 'group': 'bert', + 'desc_inputs': [ + np.random.rand(128).astype(np.int32), + np.random.rand(128).astype(np.int32), + [128] + ] + }, + { + 'id': 'Dense', + 'group': 'bert', + 'desc_inputs': [ + [3, 768] + ] + }, + { + 'id': 'GetNextSentenceOutput', + 'group': 'bert', + 'desc_inputs': [ + [128, 768] + ] + }, + { + 'id': 'BertDense_CICase', + 'group': 'bert', + 'desc_inputs': [ + np.ones(shape=(8, 8)).astype(np.float32) + ] + }, + { + 'id': 'BertNetworkWithLoss', + 'group': 'bert', + 'desc_inputs': [ + np.ones(shape=(1, 128)).astype(np.int32), + np.ones(shape=(1, 128)).astype(np.int32), + np.ones(shape=(1, 128)).astype(np.int32), + np.ones(shape=(1, 1)).astype(np.int32), + np.ones(shape=(1, 20)).astype(np.int32), + np.ones(shape=(1, 20)).astype(np.int32), + np.random.uniform(-0.1, 0.1, size=(1, 20)).astype(np.float32) + ] + }, + { + 'id': 'BertAttentionQueryKeyMul_CICase', + 'group': 'bert', + 'desc_inputs': [ + [1, 16, 128, 64], + [1, 16, 128, 64] + ] + }, + { + 'id': 'BertAttentionRelativePositionKeys_CICase', + 'group': 'bert', + 'desc_inputs': [ + [1, 16, 128, 128], + [1, 16, 128, 64] + ] + }, + { + 'id': 'BertAttentionRelativePositionValues_CICase', + 'group': 'bert', + 'desc_inputs': [ + [1, 16, 128, 64], + [1, 16, 128, 128] + ], + 'desc_bprop': [ + [128, 128, 64], + [128, 1024] + ] + }, + { + 'id': 'BertAttentionMask_CICase', + 'group': 'bert', + 'desc_inputs': [ + [1, 16, 128, 128] + ], + 'desc_bprop': [ + [1, 16, 128, 128] + ] + }, + { + 'id': 'BertAttentionSoftmax_CICase', + 'group': 'bert', + 'desc_inputs': [ + [128, 1024], + [1, 16, 128, 128] + ], + 'desc_bprop': [ + [1, 16, 128, 64], + [1, 16, 128, 64] + ] + }, + ], + 'function': [ + { + 'id': 'SaturateCast_CICase', + 'group': 'bert', + 'block': SaturateCast(), + 'max_error': 2e-3, + 'reduce_output': False + }, + { + 'id': 'BertAttention', + 'group': 'bert', + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + compute_type=mstype.float32), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertOutput', + 'group': 'bert', + 'block': BertOutput(in_channels=1024, + out_channels=1024, + initializer_range=0.02, + dropout_prob=0.0), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertSelfAttention', + 'group': 'bert', + 'block': BertSelfAttention(batch_size=1, + seq_length=128, + hidden_size=1024, + num_attention_heads=16, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.0, + use_relative_positions=True, + compute_type=mstype.float32), + 'reduce_output': False + }, + { + 'id': 'BertEncoderCell', + 'group': 'bert', + 'block': BertEncoderCell(batch_size=1, + hidden_size=1024, + seq_length=128, + num_attention_heads=16, + intermediate_size=4096, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.0, + use_relative_positions=True, + hidden_act="gelu", + compute_type=mstype.float32), + 'reduce_output': False + }, + { + 'id': 'BertTransformer', + 'group': 'bert', + 'block': BertTransformer(batch_size=1, + hidden_size=1024, + seq_length=128, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=True, + hidden_act="gelu", + compute_type=mstype.float32, + return_all_encoders=True) + }, + { + 'id': 'EmbeddingLookup', + 'group': 'bert', + 'block': EmbeddingLookup(vocab_size=21128, + embedding_size=1024, + embedding_shape=[1, 128, 1024], + use_one_hot_embeddings=False, + initializer_range=0.02), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'EmbeddingPostprocessor', + 'group': 'bert', + 'block': EmbeddingPostprocessor(embedding_size=1024, + embedding_shape=[1, 128, 1024], + use_token_type=True, + token_type_vocab_size=2, + use_one_hot_embeddings=False, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.0), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'CreateAttentionMaskFromInputMask', + 'group': 'bert', + 'block': CreateAttentionMaskFromInputMask(config=BertConfig( + batch_size=1, + seq_length=128, + vocab_size=21128, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32)), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertOutput', + 'group': 'bert', + 'block': BertOutput(in_channels=1024, + out_channels=1024, + initializer_range=0.02, + dropout_prob=0.0), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertModel', + 'group': 'BertModel', + 'block': BertModel(config=BertConfig(batch_size=1, + num_hidden_layers=2, + intermediate_size=4096, + token_type_ids_from_dataset=True), + is_training=True), + 'reduce_output': False + }, + { + 'id': 'Dense', + 'group': 'Dense', + 'block': nn.Dense(in_channels=768, + out_channels=3072, + activation='gelu', + weight_init=TruncatedNormal(0.02)), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'GetNextSentenceOutput', + 'group': 'GetNextSentenceOutput', + 'block': GetNextSentenceOutput(BertConfig(batch_size=1)), + 'reduce_output': False + }, + { + 'id': 'BertDense_CICase', + 'group': 'bert', + 'block': BertDense( + hidden_size=8, + intermediate_size=8, + initializer_range=0.02), + 'reduce_output': False + }, + { + 'id': 'BertNetworkWithLoss', + 'group': 'bert', + 'block': BertNetworkWithLoss(config=BertConfig( + batch_size=1, + seq_length=128, + vocab_size=21128, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32), is_training=True), + 'reduce_output': False + }, + { + 'id': 'BertAttentionQueryKeyMul_CICase', + 'group': 'bert', + 'block': BertAttentionQueryKeyMul(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + initializer_range=0.02), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertAttentionRelativePositionKeys_CICase', + 'group': 'BertAttentionRelativePositionKeys', + 'block': BertAttentionRelativePositionKeys(batch_size=1, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertAttentionRelativePositionValues_CICase', + 'group': 'BertAttentionRelativePositionValues', + 'block': BertAttentionRelativePositionValues(batch_size=1, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertAttentionMask_CICase', + 'group': 'BertAttentionMask', + 'block': BertAttentionMaskBackward((1, 128, 128), + has_attention_mask=True, + dtype=mstype.float32), + 'sampling_times': 10, + 'reduce_output': False + }, + { + 'id': 'BertAttentionSoftmax_CICase', + 'group': 'BertAttentionSoftmax', + 'block': BertAttentionSoftmax(batch_size=1, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + value_act=None, + attention_probs_dropout_prob=0, + initializer_range=0.02), + 'sampling_times': 10, + 'reduce_output': False + }, + ], + 'ext': {} +} + + +@mindspore_test(pipeline_for_compare_inputs_grad_with_numerical_diff_for_group_by_group_config) +def test_bert_check_gradient_wrt_inputs_exec(): + context.set_context(mode=context.GRAPH_MODE) + return verification_set + + +@mindspore_test(pipeline_for_compare_params_grad_with_numerical_diff_for_group_by_group_config) +def test_bert_check_gradient_wrt_params_exec(): + context.set_context(mode=context.GRAPH_MODE) + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_bert_compare_with_npy.py b/tests/mindspore_test_framework/apps/test_bert_compare_with_npy.py new file mode 100644 index 00000000000..92d2e9a8d5f --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_bert_compare_with_npy.py @@ -0,0 +1,595 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Test bert compare with npy.""" + +import numpy as np +import mindspore.common.dtype as mstype +from mindspore import context +from mindspore.model_zoo.Bert_NEZHA.bert_model import BertAttention, SaturateCast, \ + EmbeddingLookup, BertModel, \ + BertConfig, EmbeddingPostprocessor, \ + BertTransformer, BertEncoderCell, \ + BertSelfAttention, CreateAttentionMaskFromInputMask, \ + RelaPosMatrixGenerator, BertOutput, \ + RelaPosEmbeddingsGenerator + +from ..mindspore_test import mindspore_test +from ..pipeline.forward.compare_forward import pipeline_for_compare_forward_with_npy_for_group_by_group_config_using_group_policy +from .bert_attention_submodules import BertAttentionQueryKeyMul, BertAttentionRelativePositionKeys, BertAttentionMask, \ + BertAttentionSoftmax, BertAttentionRelativePositionValues, BertDense + +verification_set = { + 'inputs': [ + { + 'id': 'BertModel', + 'group': 'BertModel', + 'desc_inputs': [ + 'apps/bert_data/input_fn_IteratorGetNext_output_0.npy', + 'apps/bert_data/input_fn_IteratorGetNext_output_1.npy', + 'apps/bert_data/input_fn_IteratorGetNext_output_6.npy', + ] + }, + { + 'id': 'EmbeddingPostprocessor_CICase', + 'group': 'EmbeddingPostprocessor', + 'desc_inputs': [ + ('apps/bert_data/input_fn_IteratorGetNext_output_6.npy', {'dtype': np.int32}), + ('apps/bert_data/bert_embeddings_Reshape_1_output_0.npy', {'dtype': np.float32}) + ] + }, + { + 'id': 'BertTransformer', + 'group': 'BertTransformer', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_saturate_cast_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy', + ] + }, + { + 'id': 'SaturateCast_CICase', + 'group': 'SaturateCast', + 'desc_inputs': [ + 'apps/bert_data/bert_embeddings_LayerNorm_batchnorm_add_1_output_0.npy', + ] + }, + { + 'id': 'RelaPosMatrixGenerator', + 'group': 'RelaPosMatrixGenerator', + 'desc_inputs': [ + ] + }, + { + 'id': 'RelaPosEmbeddingsGenerator', + 'group': 'RelaPosEmbeddingsGenerator', + 'desc_inputs': [ + ] + }, + { + 'id': 'BertAttention_0', + 'group': 'BertAttention', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy', + ] + }, + { + 'id': 'BertAttention_1', + 'group': 'BertAttention', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_attention_output_LayerNorm_batchnorm_add_1_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_output_LayerNorm_batchnorm_add_1_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy', + ] + }, + { + 'id': 'BertAttentionQueryKeyMul_CICase', + 'group': 'BertAttentionQueryKeyMul', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + ], + 'dtype': 'float32' + }, + { + 'id': 'BertAttentionRelativePositionKeys_CICase', + 'group': 'BertAttentionRelativePositionKeys', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_MatMul_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_self_transpose_output_0.npy', + + ], + 'dtype': 'float32' + }, + { + 'id': 'BertAttentionMask_CICase', + 'group': 'BertAttentionMask', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_Mul_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy', + + ] + }, + { + 'id': 'BertAttentionSoftmax_CICase', + 'group': 'BertAttentionSoftmax', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_self_add_1_output_0.npy', + ] + }, + { + 'id': 'BertAttentionRelativePositionValues_CICase', + 'group': 'BertAttentionRelativePositionValues', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_MatMul_2_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_self_Softmax_output_0.npy', + ] + }, + { + 'id': 'BertOutput_0_CICase', + 'group': 'BertOutput_0', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_intermediate_dense_gelu_mul_3_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_output_LayerNorm_batchnorm_add_1_output_0.npy', + ] + }, + { + 'id': 'BertOutput_1_CICase', + 'group': 'BertOutput_1', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_1_intermediate_dense_gelu_mul_3_output_0.npy', + 'apps/bert_data/bert_encoder_layer_1_attention_output_LayerNorm_batchnorm_add_1_output_0.npy', + ] + }, + { + 'id': 'BertSelfAttention_0', + 'group': 'BertSelfAttention', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy', + ] + }, + { + 'id': 'BertSelfAttention_1', + 'group': 'BertSelfAttention', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_attention_output_LayerNorm_batchnorm_add_1_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy', + ] + }, + { + 'id': 'BertEncoderCell', + 'group': 'BertEncoderCell', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + 'apps/bert_data/bert_encoder_mul_output_0.npy' + ] + }, + { + 'id': 'EmbeddingLookup_CICase', + 'group': 'EmbeddingLookup', + 'desc_inputs': [ + ('apps/bert_data/input_fn_IteratorGetNext_output_0.npy', {'dtype': np.int32}) + ] + }, + { + 'id': 'BertDense_CICase', + 'group': 'BertDense', + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_layer_0_attention_output_LayerNorm_batchnorm_add_1_output_0.npy', + ] + }, + { + 'id': 'CreateAttentionMaskFromInputMask_CICase', + 'group': 'CreateAttentionMaskFromInputMask', + 'desc_inputs': [ + 'apps/bert_data/input_fn_IteratorGetNext_output_1.npy', + ] + } + ], + 'expect': [ + { + 'id': 'BertModel-BertModel', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_Cast_output_0.npy', + 'apps/bert_data/bert_pooler_dense_Tanh_output_0.npy', + 'apps/bert_data/bert_embeddings_word_embeddings_read_output_0.npy', + ] + }, + { + 'id': 'EmbeddingPostprocessor-EmbeddingPostprocessor_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_embeddings_LayerNorm_batchnorm_add_1_output_0.npy', + ] + }, + { + 'id': 'BertTransformer-BertTransformer', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_output_LayerNorm_batchnorm_add_1_output_0.npy', + 'apps/bert_data/bert_encoder_layer_1_output_LayerNorm_batchnorm_add_1_output_0.npy' + ] + }, + { + 'id': 'SaturateCast-SaturateCast_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_saturate_cast_output_0.npy' + ] + }, + { + 'id': 'RelaPosMatrixGenerator-RelaPosMatrixGenerator', + 'group': 'bert', + 'desc_expect': [ + ] + }, + { + 'id': 'RelaPosEmbeddingsGenerator-RelaPosEmbeddingsGenerator', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_relative_positions_keys_GatherV2_output_0.npy', + 'apps/bert_data/bert_encoder_layer_1_attention_self_relative_positions_keys_GatherV2_output_0.npy' + ] + }, + { + 'id': 'BertAttention-BertAttention_0', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_Reshape_7_output_0.npy' + ] + }, + { + 'id': 'BertAttention-BertAttention_1', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_1_attention_self_Reshape_7_output_0.npy', + ] + }, + { + 'id': 'BertAttentionQueryKeyMul-BertAttentionQueryKeyMul_CICase', + 'group': 'bert', + 'desc_expect': [ + ('apps/bert_data/mul_query_layer.npy', {'max_error': 1e-3}), + ('apps/bert_data/mul_key_layer.npy', {'max_error': 1e-3}), + ('apps/bert_data/mul_attention_scores.npy', {'max_error': 1e-3}) + ] + }, + { + 'id': 'BertAttentionRelativePositionKeys-BertAttentionRelativePositionKeys_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_relative_positions_keys_GatherV2_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_self_Mul_output_0.npy' + ] + }, + { + 'id': 'BertAttentionMask-BertAttentionMask_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_add_1_output_0.npy', + ] + }, + { + 'id': 'BertAttentionSoftmax-BertAttentionSoftmax_CICase', + 'group': 'bert', + 'desc_expect': [ + ('apps/bert_data/BertAttentionSoftmax_value_layer.npy', {'max_error': 1e-3}), + ('apps/bert_data/BertAttentionSoftmax_context_layer.npy', {'max_error': 1e-3}) + ] + }, + { + 'id': 'BertAttentionRelativePositionValues-BertAttentionRelativePositionValues_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_attention_self_relative_positions_values_GatherV2_output_0.npy', + 'apps/bert_data/bert_encoder_layer_0_attention_self_Reshape_7_output_0.npy' + ] + }, + { + 'id': 'BertOutput_0-BertOutput_0_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/BertOutput_output0.npy', + ] + }, + { + 'id': 'BertOutput_1-BertOutput_1_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/BertOutput_output1.npy', + ] + }, + { + 'id': 'BertSelfAttention-BertSelfAttention_0', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_attention_output_LayerNorm_batchnorm_add_1_output_0.npy' + ] + }, + { + 'id': 'BertSelfAttention-BertSelfAttention_1', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_1_attention_output_LayerNorm_batchnorm_add_1_output_0.npy' + ] + }, + { + 'id': 'BertEncoderCell-BertEncoderCell', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_layer_0_output_LayerNorm_batchnorm_add_1_output_0.npy', + ] + }, + { + 'id': 'EmbeddingLookup-EmbeddingLookup_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_embeddings_Reshape_1_output_0.npy', + ] + }, + { + 'id': 'BertDense-BertDense_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/dense_without_gelu.npy', + ] + }, + { + 'id': 'CreateAttentionMaskFromInputMask-CreateAttentionMaskFromInputMask_CICase', + 'group': 'bert', + 'desc_expect': [ + 'apps/bert_data/bert_encoder_mul_output_0.npy', + ] + } + ], + 'function': [ + { + 'id': 'SaturateCast', + 'group': 'SaturateCast', + 'block': SaturateCast() + }, + { + 'id': 'RelaPosMatrixGenerator', + 'group': 'RelaPosMatrixGenerator', + 'block': RelaPosMatrixGenerator(length=128, max_relative_position=16) + }, + { + 'id': 'RelaPosEmbeddingsGenerator', + 'group': 'RelaPosEmbeddingsGenerator', + 'block': RelaPosEmbeddingsGenerator(length=128, depth=64, + max_relative_position=16, + initializer_range=0.02, + use_one_hot_embeddings=False) + }, + { + 'id': 'BertAttention', + 'group': 'BertAttention', + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + compute_type=mstype.float32) + }, + { + 'id': 'BertAttentionQueryKeyMul', + 'group': 'BertAttentionQueryKeyMul', + 'block': BertAttentionQueryKeyMul(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + initializer_range=0.02) + }, + { + 'id': 'BertAttentionRelativePositionKeys', + 'group': 'BertAttentionRelativePositionKeys', + 'block': BertAttentionRelativePositionKeys(batch_size=1, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + }, + { + 'id': 'BertAttentionMask', + 'group': 'BertAttentionMask', + 'block': BertAttentionMask(has_attention_mask=True, + dtype=mstype.float32) + }, + { + 'id': 'BertAttentionSoftmax', + 'group': 'BertAttentionSoftmax', + 'block': BertAttentionSoftmax(batch_size=1, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02) + }, + { + 'id': 'BertAttentionRelativePositionValues', + 'group': 'BertAttentionRelativePositionValues', + 'block': BertAttentionRelativePositionValues(batch_size=1, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + }, + { + 'id': 'EmbeddingLookup', + 'group': 'EmbeddingLookup', + 'block': EmbeddingLookup(vocab_size=21128, + embedding_size=1024, + embedding_shape=[1, 128, 1024], + use_one_hot_embeddings=False, + initializer_range=0.02) + }, + { + 'id': 'BertModel', + 'group': 'BertModel', + 'block': BertModel(config=BertConfig(batch_size=1, + num_hidden_layers=2, + intermediate_size=4096, + token_type_ids_from_dataset=True), + is_training=True) + }, + { + 'id': 'EmbeddingPostprocessor', + 'group': 'EmbeddingPostprocessor', + 'block': EmbeddingPostprocessor(embedding_size=1024, + embedding_shape=[1, 128, 1024], + use_token_type=True, + token_type_vocab_size=2, + use_one_hot_embeddings=False, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.0) + }, + { + 'id': 'BertTransformer', + 'group': 'BertTransformer', + 'block': BertTransformer(batch_size=1, + hidden_size=1024, + seq_length=128, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=True, + hidden_act="gelu", + compute_type=mstype.float32, + return_all_encoders=True) + }, + { + 'id': 'BertEncoderCell', + 'group': 'BertEncoderCell', + 'block': BertEncoderCell(batch_size=1, + hidden_size=1024, + seq_length=128, + num_attention_heads=16, + intermediate_size=4096, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.0, + use_relative_positions=True, + hidden_act="gelu", + compute_type=mstype.float32) + }, + { + 'id': 'BertSelfAttention', + 'group': 'BertSelfAttention', + 'block': BertSelfAttention(batch_size=1, + seq_length=128, + hidden_size=1024, + num_attention_heads=16, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.0, + use_relative_positions=True, + compute_type=mstype.float32) + }, + { + 'id': 'BertOutput_0', + 'group': 'BertOutput_0', + 'block': BertOutput(in_channels=4096, + out_channels=1024, + initializer_range=0.02, + dropout_prob=0.0) + }, + { + 'id': 'BertOutput_1', + 'group': 'BertOutput_1', + 'block': BertOutput(in_channels=4096, + out_channels=1024, + initializer_range=0.02, + dropout_prob=0.0) + }, + { + 'id': 'BertDense', + 'group': 'BertDense', + 'block': BertDense( + hidden_size=1024, + intermediate_size=4096, + initializer_range=0.02) + }, + { + 'id': 'CreateAttentionMaskFromInputMask', + 'group': 'CreateAttentionMaskFromInputMask', + 'block': CreateAttentionMaskFromInputMask(config=BertConfig( + batch_size=1, + seq_length=128, + vocab_size=21128, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32)) + } + ], + 'ext': {} +} + + +@mindspore_test(pipeline_for_compare_forward_with_npy_for_group_by_group_config_using_group_policy) +def test_bert_compare_with_npy_exec(): + context.set_context(mode=context.GRAPH_MODE) + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_bert_ops_check_gradient.py b/tests/mindspore_test_framework/apps/test_bert_ops_check_gradient.py new file mode 100644 index 00000000000..2f11a580c9e --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_bert_ops_check_gradient.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Test bert ops check gradient.""" + +from mindspore import context +from mindspore.ops import operations as P + +from ..pipeline.gradient.compare_gradient import \ + pipeline_for_compare_inputs_grad_with_numerical_diff_for_group_by_group_config, \ + pipeline_for_compare_inputs_jacobian_with_numerical_diff_for_group_by_group_config +from ..mindspore_test import mindspore_test +# from ...vm_impl import * + + +verification_set = { + 'inputs': [ + { + 'id': 'MatMul', + 'group': 'bert', + 'desc_inputs': [ + [3, 3], + [3, 3] + ] + }, + ], + 'function': [ + { + 'id': 'MatMul', + 'group': 'bert', + 'block': P.MatMul(), + 'reduce_output': False + } + ], + 'ext': {} +} + + +@mindspore_test(pipeline_for_compare_inputs_grad_with_numerical_diff_for_group_by_group_config) +def test_bert_ops_check_gradient_exec_1(): + context.set_context(mode=context.PYNATIVE_MODE) + return verification_set + + +@mindspore_test(pipeline_for_compare_inputs_jacobian_with_numerical_diff_for_group_by_group_config) +def test_bert_ops_check_gradient_exec_2(): + context.set_context(mode=context.PYNATIVE_MODE) + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_bert_parts.py b/tests/mindspore_test_framework/apps/test_bert_parts.py new file mode 100644 index 00000000000..226d175c3d7 --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_bert_parts.py @@ -0,0 +1,234 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Test bert submodules.""" + +import os +import numpy as np +from mindspore.model_zoo.Bert_NEZHA import EmbeddingLookup, GetMaskedLMOutput, \ + BertConfig, BertPreTraining, BertNetworkWithLoss +from mindspore.model_zoo.Bert_NEZHA.bert_model import BertModel + +from mindspore import nn, context +from mindspore import Tensor +from ..utils.block_util import get_output_cell +from ..mindspore_test import mindspore_test +from ..pipeline.forward.compile_forward import pipeline_for_compile_forward_anf_graph_for_case_by_case_config, \ + pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ..pipeline.gradient.compile_gradient import pipeline_for_compile_grad_anf_graph_for_case_by_case_config, \ + pipeline_for_compile_grad_ge_graph_for_case_by_case_config +from ...dataset_mock import MindData + +# pylint: disable=missing-docstring, W0612, arguments-differ +_current_dir = os.path.dirname(os.path.realpath(__file__)) + "/../python/test_data" + + +class BertPreTrainingNet(nn.Cell): + def __init__(self, config, is_training, use_one_hot_embeddings=True): + super(BertPreTrainingNet, self).__init__() + self.is_training = is_training + self.use_one_hot_embeddings = use_one_hot_embeddings + self.bert = BertPreTraining(config, self.is_training, self.use_one_hot_embeddings) + + def construct(self, input_ids_, input_mask_, token_type_id_, + next_sentence_labels_, masked_lm_positions_): + t = next_sentence_labels_ + (prediction_scores, seq_relationship_score) = \ + self.bert(input_ids_, input_mask_, token_type_id_, masked_lm_positions_) + return prediction_scores, seq_relationship_score + + +class BertModelNet(nn.Cell): + def __init__(self, config, is_training, use_one_hot_embeddings): + super(BertModelNet, self).__init__() + self.is_training = is_training + self.use_one_hot_embeddings = use_one_hot_embeddings + self.bert = BertModel(config, self.is_training, self.use_one_hot_embeddings) + + def construct(self, input_ids_, input_mask_, token_type_id_, + masked_lm_positions_): + t = masked_lm_positions_ + sequence_output, pooled_output, embedding_table = \ + self.bert(input_ids_, input_mask_, token_type_id_) + return sequence_output, pooled_output, embedding_table + + +def get_dataset(batch_size=1): + dataset_types = (np.int32, np.int32, np.int32, np.int32, np.int32, np.int32, np.int32) + dataset_shapes = ((batch_size, 128), (batch_size, 128), (batch_size, 128), (batch_size, 1), \ + (batch_size, 20), (batch_size, 20), (batch_size, 20)) + + dataset = MindData(size=2, batch_size=batch_size, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset + + +def load_test_data(): + dataset = get_dataset() + return dataset.next() + + +input_ids, input_mask, token_type_id, \ +next_sentence_labels, masked_lm_positions, \ +masked_lm_ids, masked_lm_weights = load_test_data() + +test_sets = [ + ('BertNetworkWithLoss_1', { + 'block': BertNetworkWithLoss(BertConfig(batch_size=1), False, use_one_hot_embeddings=True), + 'desc_inputs': [input_ids, input_mask, token_type_id, + next_sentence_labels, masked_lm_positions, + masked_lm_ids, masked_lm_weights], + 'desc_bprop': [[1]]}), + ('BertNetworkWithLoss_2', { + 'block': BertNetworkWithLoss(BertConfig(batch_size=1), False, True), + 'desc_inputs': [input_ids, input_mask, token_type_id, + next_sentence_labels, masked_lm_positions, + masked_lm_ids, masked_lm_weights], + 'desc_bprop': [[1]]}), + ('BertPreTrainingNet_1', { + 'block': get_output_cell(BertPreTrainingNet(BertConfig(batch_size=1), True, True), 5, 0), + 'desc_inputs': [input_ids, input_mask, token_type_id, + next_sentence_labels, masked_lm_positions], + 'desc_bprop': [[20, 32000]]}), + ('BertPreTrainingNet_2', { + 'block': get_output_cell(BertPreTrainingNet(BertConfig(batch_size=1), True, True), 5, 1), + 'desc_inputs': [input_ids, input_mask, token_type_id, + next_sentence_labels, masked_lm_positions], + 'desc_bprop': [[1, 2]], + 'skip': ['compile', 'exec']}), + ('BertModelNet_1', { + 'block': get_output_cell(BertModelNet(BertConfig(batch_size=1), True, True), 4, 0), + 'desc_inputs': [input_ids, input_mask, token_type_id, masked_lm_positions], + 'desc_bprop': [[1, 128, 768]]}), + ('BertModelNet_2', { + 'block': get_output_cell(BertModelNet(BertConfig(batch_size=1), True, True), 4, 1), + 'desc_inputs': [input_ids, input_mask, token_type_id, masked_lm_positions], + 'desc_bprop': [[1, 768]], + 'skip': ['compile', 'exec']}), + ('BertModelNet_3', { + 'block': get_output_cell(BertModelNet(BertConfig(batch_size=1), True, True), 4, 2), + 'desc_inputs': [input_ids, input_mask, token_type_id, masked_lm_positions], + 'desc_bprop': [[32000, 768]], + 'skip': ['compile', 'exec']}), + ('GetMaskedLMOutput', { + 'block': GetMaskedLMOutput(BertConfig(batch_size=1)), + 'desc_inputs': [[1, 128, 768], [32000, 768], Tensor(np.ones(shape=[20, 768]).astype(np.int32))], + 'desc_bprop': [[15360, 32000]]}), + ('EmbeddingLookup_1', { + 'block': get_output_cell(EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False, + initializer_range=0.02), 1, 0), + 'desc_inputs': [input_ids], + 'desc_bprop': [[1, 128, 768]]}), + ('EmbeddingLookup_2', { + 'block': get_output_cell(EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False, + initializer_range=0.02), 1, 1), + 'desc_inputs': [input_ids], + 'desc_bprop': [[128]]}), + ('EmbeddingLookup_3', { + 'block': get_output_cell(EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=True, + initializer_range=0.02), 1, 0), + 'desc_inputs': [input_ids], + 'desc_bprop': [[1, 128, 768]]}), + ('EmbeddingLookup_4', { + 'block': get_output_cell(EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=True, + initializer_range=0.02), 1, 1), + 'desc_inputs': [input_ids], + 'desc_bprop': [[128]]}), + ('EmbeddingLookup_multi_outputs', { + 'block': EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False, + initializer_range=0.02), + 'desc_inputs': [input_ids], + 'desc_bprop': [[1, 128, 768], [128]]}), + ('EmbeddingLookup_init_param', { + 'block': (get_output_cell(EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=True, + initializer_range=0.02), 1, 1), { + 'init_param_with': lambda shp: np.ones(shp).astype(np.float32) + }), + 'desc_inputs': [input_ids], + 'desc_bprop': [[128]]}), + ('EmbeddingLookup_multi_outputs_init_param', { + 'block': (EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False, + initializer_range=0.02), { + 'init_param_with': lambda shp: np.ones(shp).astype(np.float32) + }), + 'desc_inputs': [input_ids], + 'desc_bprop': [[1, 128, 768], [128]]}), + ('EmbeddingLookup_multi_outputs_grad_with_no_sens', { + 'block': (EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False, + initializer_range=0.02), { + 'init_param_with': lambda shp: np.ones(shp).astype(np.float32) + }), + 'desc_inputs': [input_ids]}), + ('GetMaskedLMOutput_grad_with_no_sens', { + 'block': GetMaskedLMOutput(BertConfig(batch_size=1)), + 'desc_inputs': [[1, 128, 768], [32000, 768], Tensor(np.ones(shape=[20, 768]).astype(np.int32))], + }), + ('BertModelNet_no_split_outputs', { + 'block': (BertModelNet(BertConfig(batch_size=1), True, True), { + 'split_outputs': False + }), + 'desc_inputs': [input_ids, input_mask, token_type_id, masked_lm_positions], + 'desc_bprop': [[1, 128, 768], [1, 768], [32000, 768]]}), +] + + +@mindspore_test(pipeline_for_compile_forward_anf_graph_for_case_by_case_config) +def test_compile(): + context.set_context(mode=context.GRAPH_MODE) + return test_sets + + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_sets + + +@mindspore_test(pipeline_for_compile_grad_anf_graph_for_case_by_case_config) +def test_backward(): + context.set_context(mode=context.GRAPH_MODE) + return test_sets + + +@mindspore_test(pipeline_for_compile_grad_ge_graph_for_case_by_case_config) +def test_backward_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_sets diff --git a/tests/mindspore_test_framework/apps/test_check_exception.py b/tests/mindspore_test_framework/apps/test_check_exception.py new file mode 100644 index 00000000000..d465ef7ac47 --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_check_exception.py @@ -0,0 +1,33 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Checking exception.""" + +from ..mindspore_test import mindspore_test +from ..pipeline.forward.verify_exception import pipeline_for_verify_exception_for_case_by_case_config + +def func_raise_exception(x, y): + raise ValueError() + +verification_set = [ + ('func_raise_exception', { + 'block': (func_raise_exception, {'exception': ValueError}), + 'desc_inputs': [[1, 1], [2, 2]], + }) +] + +@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config) +def test_check_exception(): + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_lamb_check_loss.py b/tests/mindspore_test_framework/apps/test_lamb_check_loss.py new file mode 100644 index 00000000000..14461ff2d89 --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_lamb_check_loss.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Checking lamb loss.""" + +from mindspore import context +from mindspore.nn.optim import Lamb +from ..mindspore_test import mindspore_test +from ..utils.model_util import Linreg +from ..utils.model_util import SquaredLoss +from ..pipeline.gradient.check_training import pipeline_for_check_model_loss_for_case_by_case_config + +network = Linreg(2) +num_epochs = 1000 + +verification_set = [ + ('Linreg', { + 'block': { + 'model': network, + 'loss': SquaredLoss(), + 'opt': Lamb(network.trainable_params(), decay_steps=num_epochs, warmup_steps=10, weight_decay=0.01), + 'num_epochs': num_epochs, + 'loss_upper_bound': 0.3, + }, + 'desc_inputs': { + 'true_params': ([2, -3.4], 4.2), + 'num_samples': 100, + 'batch_size': 20, + } + }) +] + +@mindspore_test(pipeline_for_check_model_loss_for_case_by_case_config) +def test_lamb_loss(): + context.set_context(mode=context.GRAPH_MODE) + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_model_loss.py b/tests/mindspore_test_framework/apps/test_model_loss.py new file mode 100644 index 00000000000..dd1f2bd7210 --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_model_loss.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Check model loss.""" + +from mindspore import context + +from ..mindspore_test import mindspore_test +from ..pipeline.gradient.check_training import pipeline_for_check_model_loss_for_case_by_case_config +from ..utils.model_util import Linreg +from ..utils.model_util import SquaredLoss, SGD + +network = Linreg(2) +verification_set = [ + ('Linreg', { + 'block': { + 'model': network, + 'loss': SquaredLoss(), + 'opt': SGD(network.trainable_params(), 0.001, 20), + 'num_epochs': 1000, + 'loss_upper_bound': 0.04, + }, + 'desc_inputs': { + 'true_params': ([2, -3.4], 4.2), + 'num_samples': 100, + 'batch_size': 20, + } + }) +] + +@mindspore_test(pipeline_for_check_model_loss_for_case_by_case_config) +def test_model_loss(): + context.set_context(mode=context.GRAPH_MODE) + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_no_facade.py b/tests/mindspore_test_framework/apps/test_no_facade.py new file mode 100644 index 00000000000..0a26448de18 --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_no_facade.py @@ -0,0 +1,113 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Core verification config for mindspore_test_framework""" + +# pylint: disable=invalid-name + +import numpy as np + +from ..mindspore_test import mindspore_test +from ..pipeline.forward.verify_shapetype import pipeline_for_verify_shapetype_for_group_by_group_config +# from ...vm_impl import * + +# functions could be operations or NN cell +def Add(x, y): + return x + y + + +def Sub(x, y): + return x - y + + +def Mul(x, y): + return x * y + + +def Div(x, y): + return x / y + + +# given Add/Sub/Mul/Div operations, verify that the result's shape and type is correct +verification_set = { + 'function': [ + { + 'id': 'Add', + 'group': 'op-test', + 'block': Add + }, + { + 'id': 'Sub', + 'group': 'op-test', + 'block': Sub + }, + { + 'id': 'Mul', + 'group': 'op-test', + 'block': Mul + }, + { + 'id': 'Div', + 'group': 'op-test', + 'block': Div + } + ], + 'inputs': [ + { + 'id': '1', + 'group': 'op-test', + 'desc_inputs': [ + np.array([[1, 1], [1, 1]]).astype(np.float32), + np.array([[2, 2], [2, 2]]).astype(np.float32) + ] + }, + { + 'id': '2', + 'group': 'op-test', + 'desc_inputs': [ + np.array([[3, 3], [3, 3]]).astype(np.float32), + np.array([[4, 4], [4, 4]]).astype(np.float32) + ] + }, + { + 'id': '3', + 'group': 'op-test', + 'desc_inputs': [ + np.array([[5, 5], [5, 5]]).astype(np.float32), + np.array([[6, 6], [6, 6]]).astype(np.float32) + ] + } + ], + 'expect': [ + { + 'id': '1', + 'group': 'op-test-op-test', + 'desc_expect': { + 'shape_type': [ + { + 'type': np.float32, + 'shape': (2, 2) + } + ] + } + } + ], + 'ext': {} +} + + +@mindspore_test(pipeline_for_verify_shapetype_for_group_by_group_config) +def test_no_facade(): + return verification_set diff --git a/tests/mindspore_test_framework/apps/test_reid_gradient_compare_with_pytorch.py b/tests/mindspore_test_framework/apps/test_reid_gradient_compare_with_pytorch.py new file mode 100644 index 00000000000..12f4ec7f580 --- /dev/null +++ b/tests/mindspore_test_framework/apps/test_reid_gradient_compare_with_pytorch.py @@ -0,0 +1,59 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Compare with expectations loaded from npy file""" + +import numpy as np + +from mindspore import context +from mindspore.ops import operations as P + +from ..mindspore_test import mindspore_test +from ..pipeline.gradient.compare_gradient import pipeline_for_compare_inputs_grad_with_npy_for_case_by_case_config +# from ...vm_impl import * + +verification_set = [ + ('MatMul', { + 'block': P.MatMul(), + 'desc_inputs': [ + ('tests/mindspore_test_framework/apps/data/input_0.npy', { + 'dtype': np.float32 + }), + ('tests/mindspore_test_framework/apps/data/input_1.npy', { + 'dtype': np.float32 + }), + ], + 'desc_bprop': [ + np.ones(shape=(2, 2)).astype(np.float32) + ], + 'desc_expect': [ + ('tests/mindspore_test_framework/apps/data/grad_0.npy', { + 'dtype': np.float32, + 'check_tolerance': True, + 'relative_tolerance': 1e-2, + 'absolute_tolerance': 1e-2 + }), + ('tests/mindspore_test_framework/apps/data/grad_1.npy', { + 'dtype': np.float32, + 'max_error': 1e-3 + }), + ] + }) +] + +@mindspore_test(pipeline_for_compare_inputs_grad_with_npy_for_case_by_case_config) +def test_reid_check_gradient(): + context.set_context(mode=context.PYNATIVE_MODE) + return verification_set diff --git a/tests/mindspore_test_framework/components/__init__.py b/tests/mindspore_test_framework/components/__init__.py new file mode 100644 index 00000000000..d690a75bfd8 --- /dev/null +++ b/tests/mindspore_test_framework/components/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""components""" diff --git a/tests/mindspore_test_framework/components/executor/__init__.py b/tests/mindspore_test_framework/components/executor/__init__.py new file mode 100644 index 00000000000..95a0d65dbb2 --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""executor""" diff --git a/tests/mindspore_test_framework/components/executor/check_exceptions.py b/tests/mindspore_test_framework/components/executor/check_exceptions.py new file mode 100644 index 00000000000..eca0578b7ff --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_exceptions.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that Check if the function raises the expected Exception.""" + +import sys +import pytest + +from ...components.icomponent import IExectorComponent +from ...utils import keyword + +class CheckExceptionsEC(IExectorComponent): + """ + Check if the function raises the expected Exception. + + Examples: + { + 'block': f, + 'exception': Exception + } + """ + def run_function(self, function, inputs, verification_set): + f = function[keyword.block] + args = inputs[keyword.desc_inputs] + e = function.get(keyword.exception, Exception) + try: + with pytest.raises(e): + f(*args) + except: + raise Exception(f"Expect {e}, but got {sys.exc_info()[0]}") diff --git a/tests/mindspore_test_framework/components/executor/check_gradient_for_scalar_func.py b/tests/mindspore_test_framework/components/executor/check_gradient_for_scalar_func.py new file mode 100644 index 00000000000..5435f4322b4 --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_gradient_for_scalar_func.py @@ -0,0 +1,33 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that check gradient against numeric with respect to inputs for scalar function.""" + +from ...components.icomponent import IExectorComponent +from ...utils.check_gradient import check_gradient, ScalarGradChecker +from ...utils.config_util import get_grad_checking_options + +class CheckGradientForScalarFunctionEC(IExectorComponent): + """ + Check gradient against numeric with respect to inputs for scalar function, execute and verify. + + Examples: + 'block': scalar_function + """ + def run_function(self, function, inputs, verification_set): + f, args, delta, max_error, input_selector, output_selector, sampling_times, _ = \ + get_grad_checking_options(function, inputs) + check_gradient(f, *args, delta=delta, max_error=max_error, grad_checker_class=ScalarGradChecker, + input_selector=input_selector, output_selector=output_selector, sampling_times=sampling_times) diff --git a/tests/mindspore_test_framework/components/executor/check_gradient_wrt_inputs.py b/tests/mindspore_test_framework/components/executor/check_gradient_wrt_inputs.py new file mode 100644 index 00000000000..99cb0f8da88 --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_gradient_wrt_inputs.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that check gradient against numeric with respect to inputs.""" + +from ...components.icomponent import IExectorComponent +from ...utils.check_gradient import check_gradient, OperationGradChecker +from ...utils.config_util import get_grad_checking_options + +class CheckGradientWrtInputsEC(IExectorComponent): + """ + Check gradient against numeric with respect to inputs, execute and verify. + + Examples: + 'block': BertAttentionQueryKeyMul(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + initializer_range=0.02) + """ + def run_function(self, function, inputs, verification_set): + f, args, delta, max_error, input_selector, output_selector, \ + sampling_times, reduce_output = get_grad_checking_options(function, inputs) + check_gradient(f, *args, delta=delta, max_error=max_error, grad_checker_class=OperationGradChecker, + input_selector=input_selector, output_selector=output_selector, sampling_times=sampling_times, + reduce_output=reduce_output) diff --git a/tests/mindspore_test_framework/components/executor/check_gradient_wrt_params.py b/tests/mindspore_test_framework/components/executor/check_gradient_wrt_params.py new file mode 100644 index 00000000000..c0f3b4e23e7 --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_gradient_wrt_params.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that check gradient against numeric with respect to params.""" + +from ...components.icomponent import IExectorComponent +from ...utils.check_gradient import check_gradient, NNGradChecker +from ...utils.config_util import get_grad_checking_options + +class CheckGradientWrtParamsEC(IExectorComponent): + """ + Check gradient against numeric with respect to params, execute and verify. + + Examples: + 'block': BertAttentionQueryKeyMul(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + initializer_range=0.02) + """ + def run_function(self, function, inputs, verification_set): + f, args, delta, max_error, input_selector, output_selector, \ + sampling_times, reduce_output = get_grad_checking_options(function, inputs) + check_gradient(f, *args, delta=delta, max_error=max_error, grad_checker_class=NNGradChecker, + input_selector=input_selector, output_selector=output_selector, sampling_times=sampling_times, + reduce_output=reduce_output) diff --git a/tests/mindspore_test_framework/components/executor/check_jacobian_for_scalar_func.py b/tests/mindspore_test_framework/components/executor/check_jacobian_for_scalar_func.py new file mode 100644 index 00000000000..c06d07b446b --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_jacobian_for_scalar_func.py @@ -0,0 +1,33 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that check jacobian against numeric with respect to inputs for scalar_func.""" + +from ...components.icomponent import IExectorComponent +from ...utils.check_gradient import check_jacobian, ScalarGradChecker +from ...utils.config_util import get_grad_checking_options + +class CheckJacobianForScalarFunctionEC(IExectorComponent): + """ + Check jacobian against numeric with respect to inputs for scalar_func, execute and verify. + + Examples: + 'block': scalar_function + """ + def run_function(self, function, inputs, verification_set): + f, args, delta, max_error, input_selector, output_selector, _, _ = \ + get_grad_checking_options(function, inputs) + check_jacobian(f, *args, delta=delta, max_error=max_error, grad_checker_class=ScalarGradChecker, + input_selector=input_selector, output_selector=output_selector) diff --git a/tests/mindspore_test_framework/components/executor/check_jacobian_wrt_inputs.py b/tests/mindspore_test_framework/components/executor/check_jacobian_wrt_inputs.py new file mode 100644 index 00000000000..9ba0d4a33af --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_jacobian_wrt_inputs.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that check jacobian against numeric with respect to inputs.""" + +from ...components.icomponent import IExectorComponent +from ...utils.check_gradient import check_jacobian, OperationGradChecker +from ...utils.config_util import get_grad_checking_options + +class CheckJacobianWrtInputsEC(IExectorComponent): + """ + Check jacobian against numeric with respect to inputs, execute and verify. + + Examples: + 'block': BertAttentionQueryKeyMul(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + initializer_range=0.02) + """ + def run_function(self, function, inputs, verification_set): + f, args, delta, max_error, input_selector, output_selector, _, _ = \ + get_grad_checking_options(function, inputs) + check_jacobian(f, *args, delta=delta, max_error=max_error, grad_checker_class=OperationGradChecker, + input_selector=input_selector, output_selector=output_selector) diff --git a/tests/mindspore_test_framework/components/executor/check_jacobian_wrt_params.py b/tests/mindspore_test_framework/components/executor/check_jacobian_wrt_params.py new file mode 100644 index 00000000000..4db117da77c --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/check_jacobian_wrt_params.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that check jacobian against numeric with respect to params.""" + +from ...components.icomponent import IExectorComponent +from ...utils.check_gradient import check_jacobian, NNGradChecker +from ...utils.config_util import get_grad_checking_options + +class CheckJacobianWrtParamsEC(IExectorComponent): + """ + Check jacobian against numeric with respect to params, execute and verify. + + Examples: + 'block': BertAttentionQueryKeyMul(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + initializer_range=0.02) + """ + def run_function(self, function, inputs, verification_set): + f, args, delta, max_error, input_selector, output_selector, _, _ = \ + get_grad_checking_options(function, inputs) + check_jacobian(f, *args, delta=delta, max_error=max_error, grad_checker_class=NNGradChecker, + input_selector=input_selector, output_selector=output_selector) diff --git a/tests/mindspore_test_framework/components/executor/exec_and_verify_model_loss.py b/tests/mindspore_test_framework/components/executor/exec_and_verify_model_loss.py new file mode 100644 index 00000000000..86de8c43db9 --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/exec_and_verify_model_loss.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that verify if the model can converge to expected loss.""" + +from ...components.icomponent import IExectorComponent +from ...utils.model_util import Model +from ...utils import keyword + +class LossVerifierEC(IExectorComponent): + """ + Verify if the model can converge to expected loss. + + Examples: + 'block': { + 'model': Linreg(2), + 'loss': SquaredLoss(), + 'opt': SGD(0.001, 20), + 'num_epochs': 1000, + 'loss_upper_bound': 0.03, + } + """ + def run_function(self, function, inputs, verification_set): + model = function[keyword.block][keyword.model] + loss = function[keyword.block][keyword.loss] + opt = function[keyword.block][keyword.opt] + num_epochs = function[keyword.block][keyword.num_epochs] + loss_upper_bound = function[keyword.block][keyword.loss_upper_bound] + train_dataset = inputs[keyword.desc_inputs] + model = Model(model, loss, opt) + loss = model.train(num_epochs, train_dataset) + assert loss.asnumpy().mean() <= loss_upper_bound diff --git a/tests/mindspore_test_framework/components/executor/exec_forward.py b/tests/mindspore_test_framework/components/executor/exec_forward.py new file mode 100644 index 00000000000..f397a9fc48a --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/exec_forward.py @@ -0,0 +1,33 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that apply forward function on inputs.""" + +from ...components.icomponent import IExectorComponent +from ...utils import keyword + +class IdentityEC(IExectorComponent): + """ + Execute function/inputs. + """ + def run_function(self, function, inputs, verification_set): + result_id = function[keyword.id] + '-' + inputs[keyword.id] + group = function[keyword.group] + '-' + inputs[keyword.group] + return { + keyword.id: result_id, + keyword.group: group, + keyword.desc_inputs: inputs[keyword.desc_inputs], + keyword.result: function[keyword.block](*inputs[keyword.desc_inputs]) + } diff --git a/tests/mindspore_test_framework/components/executor/exec_gradient.py b/tests/mindspore_test_framework/components/executor/exec_gradient.py new file mode 100644 index 00000000000..33fd6cee0c2 --- /dev/null +++ b/tests/mindspore_test_framework/components/executor/exec_gradient.py @@ -0,0 +1,36 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that apply gradient function on inputs, with all bprops attached.""" + +from ...components.icomponent import IExectorComponent +from ...utils import keyword + +class IdentityBackwardEC(IExectorComponent): + """ + Execute function/inputs, with all bprops attached, the bprop function created by BC should handle these bprops. + """ + def run_function(self, function, inputs, verification_set): + result_id = function[keyword.id] + '-' + inputs[keyword.id] + group = function[keyword.group] + '-' + inputs[keyword.group] + i = [] + i.extend(inputs[keyword.desc_inputs]) + i.extend(inputs[keyword.desc_bprop]) + return { + keyword.id: result_id, + keyword.group: group, + keyword.desc_inputs: i, + keyword.result: function[keyword.block](*i) + } diff --git a/tests/mindspore_test_framework/components/expect_result_policy/__init__.py b/tests/mindspore_test_framework/components/expect_result_policy/__init__.py new file mode 100644 index 00000000000..073f832540f --- /dev/null +++ b/tests/mindspore_test_framework/components/expect_result_policy/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""expect_result_policy""" diff --git a/tests/mindspore_test_framework/components/expect_result_policy/cartesian_product_on_group_for_expect_result.py b/tests/mindspore_test_framework/components/expect_result_policy/cartesian_product_on_group_for_expect_result.py new file mode 100644 index 00000000000..cf2f2e86b1a --- /dev/null +++ b/tests/mindspore_test_framework/components/expect_result_policy/cartesian_product_on_group_for_expect_result.py @@ -0,0 +1,27 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that Combine expect/result by do cartesian product on group.""" + +from ...components.icomponent import IERPolicyComponent +from ...utils import keyword + +class GroupCartesianProductERPC(IERPolicyComponent): + """ + Combine expect/result by do cartesian product on group. + """ + def combine(self, expect, result, verification_set): + ret = [(s1, s2) for s1 in expect for s2 in result if s1[keyword.group] == s2[keyword.group]] + return ret diff --git a/tests/mindspore_test_framework/components/expect_result_policy/cartesian_product_on_id_for_expect_result.py b/tests/mindspore_test_framework/components/expect_result_policy/cartesian_product_on_id_for_expect_result.py new file mode 100644 index 00000000000..c1d5b849633 --- /dev/null +++ b/tests/mindspore_test_framework/components/expect_result_policy/cartesian_product_on_id_for_expect_result.py @@ -0,0 +1,27 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that Combine expect/result by do cartesian product on id.""" + +from ...components.icomponent import IERPolicyComponent +from ...utils import keyword + +class IdCartesianProductERPC(IERPolicyComponent): + """ + Combine expect/result by do cartesian product on id. + """ + def combine(self, expect, result, verification_set): + ret = [(s1, s2) for s1 in expect for s2 in result if s1[keyword.id] == s2[keyword.id]] + return ret diff --git a/tests/mindspore_test_framework/components/facade/__init__.py b/tests/mindspore_test_framework/components/facade/__init__.py new file mode 100644 index 00000000000..c22465a652b --- /dev/null +++ b/tests/mindspore_test_framework/components/facade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""facade""" diff --git a/tests/mindspore_test_framework/components/facade/me_facade.py b/tests/mindspore_test_framework/components/facade/me_facade.py new file mode 100644 index 00000000000..e94487582c8 --- /dev/null +++ b/tests/mindspore_test_framework/components/facade/me_facade.py @@ -0,0 +1,66 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that do me facade adaptation to core verification config.""" + +import numpy as np + +from ...components.icomponent import IFacadeComponent +from ...utils.facade_util import get_block_config, fill_block_config +from ...utils import keyword + +class MeFacadeFC(IFacadeComponent): + """ + Transform ME style config to mindspore_test_framework style. + + Examples: + ('MatMul', { + 'block': P.MatMul(), + 'inputs': [[2, 2], [2, 2]], + 'desc_bprop': [[2, 2]], + 'expect': { + 'shape_type': [ + { + 'type': np.float32, + 'shape': (2, 2) + } + ], + 'compare': [ + matmul_np, + matmul_tf + ], + 'compare_gradient': [ + matmul_gradient_tf + ], + } + }) + """ + def adapt(self, verification_set): + ret = get_block_config() + for config in verification_set: + tid = config[0] + group = 'default' + m = config[1] + desc_inputs = m.get(keyword.desc_inputs, []) + desc_bprop = m.get(keyword.desc_bprop, []) + expect = m.get(keyword.desc_expect, []) + block = m.get(keyword.block, None) + desc_const = m.get(keyword.desc_const, []) + const_first = m.get(keyword.const_first, False) + add_fake_input = m.get(keyword.add_fake_input, False) + fake_input_type = m.get(keyword.fake_input_type, np.float32) + fill_block_config(ret, block, tid, group, desc_inputs, desc_bprop, expect, + desc_const, const_first, add_fake_input, fake_input_type) + return ret diff --git a/tests/mindspore_test_framework/components/function/__init__.py b/tests/mindspore_test_framework/components/function/__init__.py new file mode 100644 index 00000000000..753f8cdebbb --- /dev/null +++ b/tests/mindspore_test_framework/components/function/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""function""" diff --git a/tests/mindspore_test_framework/components/function/compile_block.py b/tests/mindspore_test_framework/components/function/compile_block.py new file mode 100644 index 00000000000..586ec02ba20 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/compile_block.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that compile forward graph.""" + +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import compile_block, gen_net, create_funcs + +class CompileBlockBC(IBuilderComponent): + """ + Build a function that do mindspore compile. + + Examples: + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + """ + def build_sut(self, verification_set): + return create_funcs(verification_set, gen_net, compile_block) diff --git a/tests/mindspore_test_framework/components/function/compile_gradient_wrt_inputs.py b/tests/mindspore_test_framework/components/function/compile_gradient_wrt_inputs.py new file mode 100644 index 00000000000..cf9a2d43da1 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/compile_gradient_wrt_inputs.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that compile gradients graph wrt inputs.""" + +from mindspore.ops.composite import GradOperation +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import compile_block, gen_grad_net, create_funcs + +class CompileBackwardBlockWrtInputsBC(IBuilderComponent): + """ + Build a function that do mindspore gradient compile with respect to inputs. + + Examples: + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + """ + def build_sut(self, verification_set): + grad_op = GradOperation('grad', get_all=True, sens_param=True) + return create_funcs(verification_set, gen_grad_net, compile_block, grad_op) diff --git a/tests/mindspore_test_framework/components/function/compile_gradient_wrt_params.py b/tests/mindspore_test_framework/components/function/compile_gradient_wrt_params.py new file mode 100644 index 00000000000..96d28e342a2 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/compile_gradient_wrt_params.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that compile gradients graph wrt params.""" + +from mindspore.ops.composite import GradOperation +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import compile_block, gen_grad_net, create_funcs + +class CompileBackwardBlockWrtParamsBC(IBuilderComponent): + """ + Build a function that do mindspore gradient compile with respect to params. + + Examples: + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + """ + def build_sut(self, verification_set): + grad_op = GradOperation('grad', get_by_list=True, sens_param=True) + return create_funcs(verification_set, gen_grad_net, compile_block, grad_op) diff --git a/tests/mindspore_test_framework/components/function/get_function_from_config.py b/tests/mindspore_test_framework/components/function/get_function_from_config.py new file mode 100644 index 00000000000..45ab65a6e61 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/get_function_from_config.py @@ -0,0 +1,29 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that return function from config directly.""" + +from ...components.icomponent import IBuilderComponent +from ...utils import keyword + +class IdentityBC(IBuilderComponent): + """ + Return function. + + Examples: + 'function': Add + """ + def build_sut(self, verification_set): + return verification_set[keyword.function] diff --git a/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_block.py b/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_block.py new file mode 100644 index 00000000000..99048fe2c4f --- /dev/null +++ b/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_block.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that init params with random function and return forward results.""" + +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import run_block, get_uniform_with_shape, gen_net, create_funcs + +class RunBlockWithRandParamBC(IBuilderComponent): + """ + Build a function with uniformed params that run mindspore pipeline. + + Examples: + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + """ + def build_sut(self, verification_set): + return create_funcs(verification_set, gen_net, run_block, default_rand_func=get_uniform_with_shape) diff --git a/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_gradient_wrt_inputs.py b/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_gradient_wrt_inputs.py new file mode 100644 index 00000000000..e6700671b81 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_gradient_wrt_inputs.py @@ -0,0 +1,25 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that init params with random function and return gradients wrt inputs.""" + +from mindspore.ops.composite import GradOperation +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import run_block, gen_grad_net, create_funcs, get_uniform_with_shape + +class RunBackwardBlockWrtInputsWithRandParamBC(IBuilderComponent): + def build_sut(self, verification_set): + grad_op = GradOperation('grad', get_all=True, sens_param=True) + return create_funcs(verification_set, gen_grad_net, run_block, grad_op, get_uniform_with_shape) diff --git a/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_gradient_wrt_params.py b/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_gradient_wrt_params.py new file mode 100644 index 00000000000..bc08d081830 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/init_params_with_rand_and_run_gradient_wrt_params.py @@ -0,0 +1,25 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that init params with random function and return gradients wrt params.""" + +from mindspore.ops.composite import GradOperation +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import run_block, gen_grad_net, create_funcs, get_uniform_with_shape + +class RunBackwardBlockWrtParamsWithRandParamBC(IBuilderComponent): + def build_sut(self, verification_set): + grad_op = GradOperation('grad', get_by_list=True, sens_param=True) + return create_funcs(verification_set, gen_grad_net, run_block, grad_op, get_uniform_with_shape) diff --git a/tests/mindspore_test_framework/components/function/run_block.py b/tests/mindspore_test_framework/components/function/run_block.py new file mode 100644 index 00000000000..98e89644ab3 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/run_block.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that return forward results.""" + +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import run_block, gen_net, create_funcs + +class RunBlockBC(IBuilderComponent): + """ + Build a function that run mindspore pipeline. + + Examples: + 'block': BertAttention(batch_size=1, + from_tensor_width=1024, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.0, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + dtype=mstype.float32, + compute_type=mstype.float32) + """ + def build_sut(self, verification_set): + return create_funcs(verification_set, gen_net, run_block) diff --git a/tests/mindspore_test_framework/components/function/run_gradient_wrt_inputs.py b/tests/mindspore_test_framework/components/function/run_gradient_wrt_inputs.py new file mode 100644 index 00000000000..e7a14390671 --- /dev/null +++ b/tests/mindspore_test_framework/components/function/run_gradient_wrt_inputs.py @@ -0,0 +1,25 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that return gradients wrt inputs.""" + +from mindspore.ops.composite import GradOperation +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import run_block, gen_grad_net, create_funcs + +class RunBackwardBlockWrtInputsBC(IBuilderComponent): + def build_sut(self, verification_set): + grad_op = GradOperation('grad', get_all=True, sens_param=True) + return create_funcs(verification_set, gen_grad_net, run_block, grad_op) diff --git a/tests/mindspore_test_framework/components/function/run_gradient_wrt_params.py b/tests/mindspore_test_framework/components/function/run_gradient_wrt_params.py new file mode 100644 index 00000000000..d935131d7dc --- /dev/null +++ b/tests/mindspore_test_framework/components/function/run_gradient_wrt_params.py @@ -0,0 +1,25 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that construct function that return gradients wrt params.""" + +from mindspore.ops.composite import GradOperation +from ...components.icomponent import IBuilderComponent +from ...utils.block_util import run_block, gen_grad_net, create_funcs + +class RunBackwardBlockWrtParamsBC(IBuilderComponent): + def build_sut(self, verification_set): + grad_op = GradOperation('grad', get_by_list=True, sens_param=True) + return create_funcs(verification_set, gen_grad_net, run_block, grad_op) diff --git a/tests/mindspore_test_framework/components/function_inputs_policy/__init__.py b/tests/mindspore_test_framework/components/function_inputs_policy/__init__.py new file mode 100644 index 00000000000..d690a75bfd8 --- /dev/null +++ b/tests/mindspore_test_framework/components/function_inputs_policy/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""components""" diff --git a/tests/mindspore_test_framework/components/function_inputs_policy/cartesian_product_on_group_for_function_inputs.py b/tests/mindspore_test_framework/components/function_inputs_policy/cartesian_product_on_group_for_function_inputs.py new file mode 100644 index 00000000000..6e696f3a14a --- /dev/null +++ b/tests/mindspore_test_framework/components/function_inputs_policy/cartesian_product_on_group_for_function_inputs.py @@ -0,0 +1,29 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that combine function/inputs by do cartesian product on group.""" + +from ...components.icomponent import IFIPolicyComponent +from ...utils import keyword + + +class GroupCartesianProductFIPC(IFIPolicyComponent): + """ + Combine function/inputs by do cartesian product on group. + """ + def combine(self, function, inputs, verification_set): + # pylint: disable=unused-argument + ret = [(s1, s2) for s1 in function for s2 in inputs if s1[keyword.group] == s2[keyword.group]] + return ret diff --git a/tests/mindspore_test_framework/components/function_inputs_policy/cartesian_product_on_id_for_function_inputs.py b/tests/mindspore_test_framework/components/function_inputs_policy/cartesian_product_on_id_for_function_inputs.py new file mode 100644 index 00000000000..3c930f8eaf3 --- /dev/null +++ b/tests/mindspore_test_framework/components/function_inputs_policy/cartesian_product_on_id_for_function_inputs.py @@ -0,0 +1,28 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that combine function/inputs by do cartesian product on id.""" + +from ...components.icomponent import IFIPolicyComponent +from ...utils import keyword + +class IdCartesianProductFIPC(IFIPolicyComponent): + """ + Combine function/inputs by do cartesian product on id. + """ + def combine(self, function, inputs, verification_set): + # pylint: disable=unused-argument + ret = [(s1, s2) for s1 in function for s2 in inputs if s1[keyword.id] == s2[keyword.id]] + return ret diff --git a/tests/mindspore_test_framework/components/icomponent.py b/tests/mindspore_test_framework/components/icomponent.py new file mode 100644 index 00000000000..b02bfa383cf --- /dev/null +++ b/tests/mindspore_test_framework/components/icomponent.py @@ -0,0 +1,110 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component interfaces.""" + +class IComponent: + """Component interfaces.""" + def __init__(self, verification_set): + self.verification_set = verification_set + + def run(self): + raise NotImplementedError + + def get_result(self): + return self.result + + +class IDataComponent(IComponent): + """Create inputs for verification_set.""" + def run(self): + self.result = self.create_inputs(self.verification_set) + + def create_inputs(self, verification_set): + raise NotImplementedError + + +class IBuilderComponent(IComponent): + """Build system under test.""" + def run(self): + self.result = self.build_sut(self.verification_set) + + def build_sut(self, verification_set): + raise NotImplementedError + + +class IExectorComponent(IComponent): + """Execute sut, take (function, input) pairs as input.""" + def __init__(self, verification_set, function, inputs): + super(IExectorComponent, self).__init__(verification_set) + self.function = function + self.inputs = inputs + + def run(self): + self.result = self.run_function(self.function, self.inputs, self.verification_set) + + def run_function(self, function, inputs, verification_set): + raise NotImplementedError + + +class IVerifierComponent(IComponent): + """Verify sut result, take (expect, result) pairs as input.""" + def __init__(self, verification_set, expect, result): + super(IVerifierComponent, self).__init__(verification_set) + self.expect = expect + self.func_result = result + + def run(self): + self.result = self.verify(self.expect, self.func_result, self.verification_set) + + def verify(self, expect, func_result, verification_set): + raise NotImplementedError + + +class IFIPolicyComponent(IComponent): + """Combine functions/inputs.""" + def __init__(self, verification_set, function, inputs): + super(IFIPolicyComponent, self).__init__(verification_set) + self.function = function + self.inputs = inputs + + def run(self): + self.result = self.combine(self.function, self.inputs, self.verification_set) + + def combine(self, function, inputs, verification_set): + raise NotImplementedError + + +class IERPolicyComponent(IComponent): + """Combine expects and results.""" + def __init__(self, verification_set, expect, result): + super(IERPolicyComponent, self).__init__(verification_set) + self.expect = expect + self.result = result + + def run(self): + self.result = self.combine(self.expect, self.result, self.verification_set) + + def combine(self, expect, result, verification_set): + raise NotImplementedError + + +class IFacadeComponent(IComponent): + """Adapt verification_set.""" + def run(self): + self.result = self.adapt(self.verification_set) + + def adapt(self, verification_set): + raise NotImplementedError diff --git a/tests/mindspore_test_framework/components/inputs/__init__.py b/tests/mindspore_test_framework/components/inputs/__init__.py new file mode 100644 index 00000000000..ca842b2f3ff --- /dev/null +++ b/tests/mindspore_test_framework/components/inputs/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""inputs""" diff --git a/tests/mindspore_test_framework/components/inputs/generate_dataset_for_linear_regression.py b/tests/mindspore_test_framework/components/inputs/generate_dataset_for_linear_regression.py new file mode 100644 index 00000000000..f173bb34c98 --- /dev/null +++ b/tests/mindspore_test_framework/components/inputs/generate_dataset_for_linear_regression.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that generate dataset for linear regression.""" + +from ...components.icomponent import IDataComponent +from ...utils.dataset_util import generate_dataset_for_linear_regression +from ...utils import keyword + +class GenerateDataSetForLRDC(IDataComponent): + """ + Create dataset for linear regression, with salt from normal distribution. + + Examples: + 'inputs': { + 'true_params': ([2, -3.4], 4.2), + 'num_samples': 100, + 'batch_size': 20, + } + """ + def create_inputs(self, verification_set): + result = [] + for config in verification_set[keyword.inputs]: + desc_inputs = config[keyword.desc_inputs] + config[keyword.desc_inputs] = generate_dataset_for_linear_regression(desc_inputs[keyword.true_params][0], + desc_inputs[keyword.true_params][1], + desc_inputs[keyword.num_samples], + desc_inputs[keyword.batch_size]) + result.append(config) + return result diff --git a/tests/mindspore_test_framework/components/inputs/generate_inputs_from_shape.py b/tests/mindspore_test_framework/components/inputs/generate_inputs_from_shape.py new file mode 100644 index 00000000000..ac57a106cc7 --- /dev/null +++ b/tests/mindspore_test_framework/components/inputs/generate_inputs_from_shape.py @@ -0,0 +1,71 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that generate inputs for specified shape type.""" + +import numpy as np +from mindspore.common.tensor import Tensor + +from ...components.icomponent import IDataComponent +from ...utils.other_util import shape2tensor +from ...utils.config_util import get_input_config +from ...utils import keyword + +class GenerateFromShapeDC(IDataComponent): + """ + Generate inputs from shape, desc_inputs must be configured, desc_bprop is optional. + + Examples: + 'desc_inputs': [ + [1, 16, 128, 64], # inputs could be shape/tensor/np.ndarray + ] + 'desc_inputs': [ + ([1, 16, 128, 64], np.float32, 6), # (inputs, dtype, scale) + ] + 'desc_bprop': [ + [1, 16, 128, 64], # inputs could be shape/tensor/np.ndarray + ] + 'desc_bprop': [ + ([1, 16, 128, 64], np.float32, 6), # (inputs, dtype, scale) + ] + """ + def create_inputs(self, verification_set): + result = [] + for config in verification_set[keyword.inputs]: + desc_inputs = config[keyword.desc_inputs] + add_fake_input = config.get(keyword.add_fake_input, False) + fake_input_type = config.get(keyword.fake_input_type, np.float32) + + inputs = [] + if not desc_inputs and add_fake_input: + inputs = [Tensor(np.array([1.0]).astype(fake_input_type))] + else: + for d in desc_inputs: + s, dtype, scale = get_input_config(d) + inputs.append(shape2tensor(s, dtype, scale)) + config[keyword.desc_inputs] = inputs + + desc_bprop = config.get(keyword.desc_bprop, []) + bprops = [] + if not desc_bprop and add_fake_input: + bprops = [Tensor(np.array([1.0]))] + else: + for d in desc_bprop: + s, dtype, scale = get_input_config(d) + bprops.append(shape2tensor(s, dtype, scale)) + config[keyword.desc_bprop] = bprops + + result.append(config) + return result diff --git a/tests/mindspore_test_framework/components/inputs/get_inputs_from_config.py b/tests/mindspore_test_framework/components/inputs/get_inputs_from_config.py new file mode 100644 index 00000000000..917e5d991cf --- /dev/null +++ b/tests/mindspore_test_framework/components/inputs/get_inputs_from_config.py @@ -0,0 +1,30 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that return raw inputs.""" + +from ...components.icomponent import IDataComponent + +class IdentityDC(IDataComponent): + """ + Return inputs. + + Examples: + 'inputs': [ + np.array([[2, 2], [2, 2]]).astype(np.float32) + ] + """ + def create_inputs(self, verification_set): + return verification_set['inputs'] diff --git a/tests/mindspore_test_framework/components/inputs/load_inputs_from_npy.py b/tests/mindspore_test_framework/components/inputs/load_inputs_from_npy.py new file mode 100644 index 00000000000..c51c52da3f5 --- /dev/null +++ b/tests/mindspore_test_framework/components/inputs/load_inputs_from_npy.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that load inputs from npy file.""" + +from ...components.icomponent import IDataComponent +from ...utils.npy_util import load_data_from_npy_or_shape +from ...utils import keyword + +class LoadFromNpyDC(IDataComponent): + """ + Load inputs from npy data, inputs could be shape/tensor/np.ndarray/file path. + + Examples: + 'desc_inputs': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + ] + 'desc_inputs': [ + ('apps/bert_data/bert_encoder_Reshape_1_output_0.npy', np.float32), # (path, dtype) + ] + 'desc_inputs': [ + ([2, 2], np.float32, 6) # (shape, dtype, scale) + ] + 'desc_bprop': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + ] + 'desc_bprop': [ + ('apps/bert_data/bert_encoder_Reshape_1_output_0.npy', np.float32), + ] + 'desc_bprop': [ + ([2, 2], np.float32, 6) + ] + """ + def create_inputs(self, verification_set): + result = [] + for config in verification_set[keyword.inputs]: + config[keyword.desc_inputs] = load_data_from_npy_or_shape(config[keyword.desc_inputs]) + config[keyword.desc_bprop] = load_data_from_npy_or_shape(config.get(keyword.desc_bprop, [])) + result.append(config) + return result diff --git a/tests/mindspore_test_framework/components/verifier/__init__.py b/tests/mindspore_test_framework/components/verifier/__init__.py new file mode 100644 index 00000000000..24367cc0056 --- /dev/null +++ b/tests/mindspore_test_framework/components/verifier/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""verifier""" diff --git a/tests/mindspore_test_framework/components/verifier/compare_forward.py b/tests/mindspore_test_framework/components/verifier/compare_forward.py new file mode 100644 index 00000000000..fe2fea40242 --- /dev/null +++ b/tests/mindspore_test_framework/components/verifier/compare_forward.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that comparing forward result with baseline functions.""" + +from ...components.icomponent import IVerifierComponent +from ...utils.compare_util import compare +from ...utils import keyword + +class CompareWithVC(IVerifierComponent): + """ + Compare the result with baseline functions configured in 'compare' config item. + + Examples: + 'expect': { + 'shape_type': [ + { + 'type': np.float32, + 'shape': (2, 2) + } + ], + 'compare': [ + matmul_np, + matmul_tf + ], + 'compare_gradient': [ + matmul_gradient_tf + ], + 'max_error': 1e-3 + } + """ + def verify(self, expect, func_result, verification_set): + compare(expect, func_result, baseline=keyword.compare_with) diff --git a/tests/mindspore_test_framework/components/verifier/compare_gradient.py b/tests/mindspore_test_framework/components/verifier/compare_gradient.py new file mode 100644 index 00000000000..8b8955578d5 --- /dev/null +++ b/tests/mindspore_test_framework/components/verifier/compare_gradient.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that comparing gradients with baseline functions.""" + +from ...components.icomponent import IVerifierComponent +from ...utils.compare_util import compare +from ...utils import keyword + +class CompareGradientWithVC(IVerifierComponent): + """ + Compare the result with baseline functions configured in 'compare_gradient_with' config item. + + Examples: + 'desc_expect': { + 'compare_with': [ + matmul_np, + matmul_tf + ], + 'compare_gradient_with': [ + matmul_gradient_tf + ], + 'max_error': 1e-3 + } + """ + def verify(self, expect, func_result, verification_set): + compare(expect, func_result, baseline=keyword.compare_gradient_with) diff --git a/tests/mindspore_test_framework/components/verifier/verify_expect_from_npy.py b/tests/mindspore_test_framework/components/verifier/verify_expect_from_npy.py new file mode 100644 index 00000000000..bb62f789b73 --- /dev/null +++ b/tests/mindspore_test_framework/components/verifier/verify_expect_from_npy.py @@ -0,0 +1,55 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that comparing results with expectation serialized as npy file.""" + +import numpy as np +from ...components.icomponent import IVerifierComponent +from ...utils.other_util import to_numpy_list, to_numpy +from ...utils.npy_util import load_data_from_npy_or_shape +from ...utils.verifier_util import tolerance_assert +from ...utils import keyword + +class LoadFromNpyVC(IVerifierComponent): + """ + Verify if the results are like expects from npy data, expects could be shape/tensor/np.ndarray/file path. + + Examples: + 'desc_expect': [ + 'apps/bert_data/bert_encoder_Reshape_1_output_0.npy', + ] + 'desc_expect': [ + ('apps/bert_data/bert_encoder_Reshape_1_output_0.npy', np.float32, 1e-3), # (path, dtype, max_error) + ] + 'desc_expect': [ + ([2, 2], np.float32, 6, 1e-3) # (shape, dtype, scale, max_error) + ] + """ + def verify(self, expect, func_result, verification_set): + dpaths = expect.get(keyword.desc_expect) + expects = load_data_from_npy_or_shape(dpaths, False) + results = func_result[keyword.result] + if results: + results = to_numpy_list(results) + for i, e in enumerate(expects): + expect, max_error, check_tolerance, relative_tolerance, absolute_tolerance = e + expect = to_numpy(expect) + if check_tolerance: + tolerance_assert(expect, results[i], relative_tolerance, absolute_tolerance) + else: + if np.fabs(expect - results[i]).max() > max_error: + raise ValueError(f'Error: expect {i}th result {expect}, ' + f'but got {results[i]}, max_error {max_error}') + print(f'expect {i}th result {expect}, got: {results[i]}, max_error {max_error}') diff --git a/tests/mindspore_test_framework/components/verifier/verify_shapetype.py b/tests/mindspore_test_framework/components/verifier/verify_shapetype.py new file mode 100644 index 00000000000..65f064a918e --- /dev/null +++ b/tests/mindspore_test_framework/components/verifier/verify_shapetype.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Component that verify shape and type.""" + +from ...components.icomponent import IVerifierComponent +from ...utils.other_util import to_numpy_list +from ...utils import keyword + +class ShapeTypeVC(IVerifierComponent): + """ + Verify if the result's shape and type are correct. + + Examples: + 'desc_expect': { + 'shape_type': [ + { + 'type': np.float32, + 'shape': (2, 2) + } + ] + } + """ + def verify(self, expect, func_result, verification_set): + results = to_numpy_list(func_result[keyword.result]) + expects = expect[keyword.desc_expect][keyword.shape_type] + for i, e in enumerate(expects): + if results[i].shape != e[keyword.shape] or results[i].dtype != e[keyword.type]: + raise TypeError(f'Error: expect {expect}, but got {func_result}') diff --git a/tests/mindspore_test_framework/mindspore_test.py b/tests/mindspore_test_framework/mindspore_test.py new file mode 100644 index 00000000000..bd638c93d55 --- /dev/null +++ b/tests/mindspore_test_framework/mindspore_test.py @@ -0,0 +1,130 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Verification pipeline engine.""" + +import logging +import pytest +from .components.icomponent import IDataComponent, IBuilderComponent, IExectorComponent, \ + IVerifierComponent, IFIPolicyComponent, IERPolicyComponent, IComponent, \ + IFacadeComponent +from .utils import keyword + +def mindspore_test(verification_pipeline): + """ + Run verification pipeline. + + Args: + verification_pipeline (list): Pipeline designed to do verification. + + Returns: + """ + def decorate(get_verification_set): + verification_set = get_verification_set() + + facade_components = [] + data_components = [] + builder_components = [] + executor_components = [] + verifier_components = [] + fi_policy_components = [] + er_policy_components = [] + for component in verification_pipeline: + if issubclass(component, IFacadeComponent): + facade_components.append(component) + elif issubclass(component, IDataComponent): + data_components.append(component) + elif issubclass(component, IBuilderComponent): + builder_components.append(component) + elif issubclass(component, IExectorComponent): + executor_components.append(component) + elif issubclass(component, IVerifierComponent): + verifier_components.append(component) + elif issubclass(component, IFIPolicyComponent): + fi_policy_components.append(component) + elif issubclass(component, IERPolicyComponent): + er_policy_components.append(component) + else: + raise Exception(f'{component} is not a instance of {IComponent}') + + for component in facade_components: + fc = component(verification_set) + fc.run() + verification_set = fc.get_result() + + inputs = [] + for component in data_components: + dc = component(verification_set) + dc.run() + inputs.extend(dc.get_result()) + + if not inputs: + logging.warning("Inputs set is empty.") + + functions = [] + for component in builder_components: + bc = component(verification_set) + bc.run() + functions.extend(bc.get_result()) + + if not functions: + logging.warning("Function set is empty.") + + fis = [] + for component in fi_policy_components: + fipc = component(verification_set, functions, inputs) + fipc.run() + fis.extend(fipc.get_result()) + + if not fis: + logging.warning("Function inputs pair set is empty.") + + def test_case(args): + sut, inputs = args + + results = [] + for component in executor_components: + ec = component(verification_set, sut, inputs) + ec.run() + results.append(ec.get_result()) + + if not results: + logging.warning("Result set is empty.") + + expect_actuals = [] + for component in er_policy_components: + erpc = component(verification_set, verification_set['expect'], results) + erpc.run() + expect_actuals.extend(erpc.get_result()) + + if not expect_actuals: + logging.warning("Expect Result pair set is empty.") + + for ea in expect_actuals: + for component in verifier_components: + vc = component(verification_set, *ea) + vc.run() + + def get_tc_name(f, inputs): + tc_id = f[keyword.id]+'-'+inputs[keyword.id] + group = f[keyword.group]+'-'+inputs[keyword.group] + return 'Group_' + group + '-' + 'Id_' + tc_id + + if fis: + m = pytest.mark.parametrize('args', fis, ids=lambda fi: get_tc_name(*fi))(test_case) + m.__orig__ = get_verification_set + return m + + return decorate diff --git a/tests/mindspore_test_framework/pipeline/__init__.py b/tests/mindspore_test_framework/pipeline/__init__.py new file mode 100644 index 00000000000..a1babd5c1fa --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""pipeline""" diff --git a/tests/mindspore_test_framework/pipeline/forward/__init__.py b/tests/mindspore_test_framework/pipeline/forward/__init__.py new file mode 100644 index 00000000000..16fc828a228 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/forward/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""forward""" diff --git a/tests/mindspore_test_framework/pipeline/forward/compare_forward.py b/tests/mindspore_test_framework/pipeline/forward/compare_forward.py new file mode 100644 index 00000000000..77b62c0ac65 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/forward/compare_forward.py @@ -0,0 +1,167 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for forward comparison.""" + +from ...components.expect_result_policy.cartesian_product_on_id_for_expect_result import IdCartesianProductERPC +from ...components.function_inputs_policy.cartesian_product_on_id_for_function_inputs import IdCartesianProductFIPC +from ...components.executor.exec_forward import IdentityEC +from ...components.function.run_block import RunBlockBC +from ...components.inputs.generate_inputs_from_shape import GenerateFromShapeDC +from ...components.verifier.compare_forward import CompareWithVC +from ...components.facade.me_facade import MeFacadeFC +from ...components.inputs.load_inputs_from_npy import LoadFromNpyDC +from ...components.verifier.verify_expect_from_npy import LoadFromNpyVC +from ...components.function.init_params_with_rand_and_run_block import RunBlockWithRandParamBC +from ...components.function_inputs_policy.cartesian_product_on_group_for_function_inputs import \ + GroupCartesianProductFIPC + +# pylint: disable=W0105 +""" +Compare an operator's result with user defined operator's result. +The pipeline is suitable for configs in a case-by-case style. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]], + 'desc_expect': { + 'compare_with': [ + (run_uer_defined, user_defined.add) + ], + } + }) + ] +""" +pipeline_for_compare_forward_with_user_defined_for_case_by_case_config = [MeFacadeFC, GenerateFromShapeDC, RunBlockBC, + IdCartesianProductFIPC, IdentityEC, + IdCartesianProductERPC, CompareWithVC] + +""" +Compare an operator's result with numpy operator's result. +The pipeline is suitable for configs in a case-by-case style. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]], + 'desc_expect': { + 'compare_with': [ + (run_np, np.add) + ] + } + }) + ] +""" +pipeline_for_compare_forward_with_numpy_for_case_by_case_config = [MeFacadeFC, GenerateFromShapeDC, RunBlockBC, + IdCartesianProductFIPC, IdentityEC, + IdCartesianProductERPC, CompareWithVC] + +""" +Compare an operator's result with result in npy file. +The pipeline is suitable for configs in a grouped style. + +Example: + verification_set = { + 'function': [ + { + 'id':'add', + 'group':'op-test', + 'block':(P.TensorAdd(), {'reduce_output': False}), + } + ], + 'inputs': [ + { + 'id': 'add', + 'group': 'op-test', + 'desc_inputs': [ + ('path/to/input/file_1.npy', {'dtype': np.float32}), + ('path/to/input/file_2.npy', {'dtype': np.float32}), + ] + } + ], + 'expect': [ + { + 'id': 'add', + 'group': 'op-test', + 'desc_expect': { + ('path/to/expect/file.npy', {'dtype': np.float32}) + } + } + ] + } +""" +pipeline_for_compare_forward_with_npy_for_group_by_group_config =\ + [LoadFromNpyDC, RunBlockWithRandParamBC, IdCartesianProductFIPC, + IdentityEC, IdCartesianProductERPC, LoadFromNpyVC] + +""" +Compare an operator's result with result in npy file.The test cases will be generated by apply function with a group id +to all inputs with same group id. +The pipeline is suitable for configs in a grouped style. + +Example: + verification_set = { + 'function': [ + { + 'id':'add', + 'group':'op-test', + 'block':(P.TensorAdd(), {'reduce_output': False}), + } + ], + 'inputs': [ + { + 'id': 'add1', + 'group': 'op-test', + 'desc_inputs': [ + ('path/to/input/file_1.npy', {'dtype': np.float32}), + ('path/to/input/file_2.npy', {'dtype': np.float32}), + ] + }, + { + 'id': 'add2', + 'group': 'op-test', + 'desc_inputs': [ + ('path/to/input/file_3.npy', {'dtype': np.float32}), + ('path/to/input/file_4.npy', {'dtype': np.float32}), + ] + } + ], + 'expect': [ + { + 'id': 'add1', + 'group': 'op-test', + 'desc_expect': { + ('path/to/expect/file.npy', {'dtype': np.float32}) + } + }, + { + 'id': 'add2', + 'group': 'op-test', + 'desc_expect': { + ('path/to/expect/file.npy', {'dtype': np.float32}) + } + } + ] + } +""" +pipeline_for_compare_forward_with_npy_for_group_by_group_config_using_group_policy =\ + [LoadFromNpyDC, RunBlockWithRandParamBC, + GroupCartesianProductFIPC, IdentityEC, + IdCartesianProductERPC, LoadFromNpyVC] diff --git a/tests/mindspore_test_framework/pipeline/forward/compile_forward.py b/tests/mindspore_test_framework/pipeline/forward/compile_forward.py new file mode 100644 index 00000000000..a103f3d5fc5 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/forward/compile_forward.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for forward computing.""" + +from ...components.facade.me_facade import MeFacadeFC +from ...components.inputs.generate_inputs_from_shape import GenerateFromShapeDC +from ...components.executor.exec_forward import IdentityEC +from ...components.executor.check_exceptions import CheckExceptionsEC +from ...components.function_inputs_policy.cartesian_product_on_id_for_function_inputs import IdCartesianProductFIPC +from ...components.function.compile_block import CompileBlockBC +from ...components.function.run_block import RunBlockBC + +# pylint: disable=W0105 +""" +Test if compile forward anf graph is ok. +The pipeline just try to run to compile forward anf graph process without comparing any result with any expect. +The pipeline is suitable for config in a case-by-case style. + +Example: + Examples: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]], + }) + ] +""" +pipeline_for_compile_forward_anf_graph_for_case_by_case_config = [MeFacadeFC, GenerateFromShapeDC, CompileBlockBC, + IdCartesianProductFIPC, IdentityEC] + +""" +Test if compile forward ge graph is ok. +The pipeline will try to run through compiling forward ge graph process without comparing any result with any expect. +The pipeline is suitable for config in a case-by-case style. + +Example: + Examples: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]], + }) + ] +""" +pipeline_for_compile_forward_ge_graph_for_case_by_case_config = [MeFacadeFC, GenerateFromShapeDC, RunBlockBC, + IdCartesianProductFIPC, IdentityEC] + +pipeline_for_compile_forward_ge_graph_for_case_by_case_config_exception = [MeFacadeFC, GenerateFromShapeDC, RunBlockBC, + IdCartesianProductFIPC, CheckExceptionsEC] diff --git a/tests/mindspore_test_framework/pipeline/forward/verify_exception.py b/tests/mindspore_test_framework/pipeline/forward/verify_exception.py new file mode 100644 index 00000000000..ac7261a5bf9 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/forward/verify_exception.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for exception checking.""" + +from ...components.executor.check_exceptions import CheckExceptionsEC +from ...components.inputs.get_inputs_from_config import IdentityDC +from ...components.function.get_function_from_config import IdentityBC +from ...components.facade.me_facade import MeFacadeFC +from ...components.function_inputs_policy.cartesian_product_on_id_for_function_inputs import IdCartesianProductFIPC + +# pylint: disable=W0105 +""" +Test if function raises expected Exception type. The pipeline is suitable for config in a ME style. + +Example: + verification_set = [ + ('func_raise_exception', { + 'block': (func_raise_exception, {'exception': ValueError}), + 'desc_inputs': [[1, 1], [2, 2]], + }) + ] +""" +pipeline_for_verify_exception_for_case_by_case_config = [MeFacadeFC, IdentityDC, IdentityBC, + IdCartesianProductFIPC, CheckExceptionsEC] diff --git a/tests/mindspore_test_framework/pipeline/forward/verify_shapetype.py b/tests/mindspore_test_framework/pipeline/forward/verify_shapetype.py new file mode 100644 index 00000000000..79890bf506d --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/forward/verify_shapetype.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for shape and type checking.""" + +from ...components.inputs.get_inputs_from_config import IdentityDC +from ...components.function.get_function_from_config import IdentityBC +from ...components.executor.exec_forward import IdentityEC +from ...components.expect_result_policy.cartesian_product_on_group_for_expect_result \ + import GroupCartesianProductERPC +from ...components.function_inputs_policy.cartesian_product_on_group_for_function_inputs \ + import GroupCartesianProductFIPC +from ...components.verifier.verify_shapetype import ShapeTypeVC + +# pylint: disable=W0105 +""" +Test if operator's result shape is correct. The pipeline will apply function with group id to any inputs +with same group id to generate test cases. The pipeline is suitable for config in a grouped style. + +Example: + verification_set = { + 'function': [ + { + 'id': 'Add', + 'group': 'op-test', + 'block': Add + } + ], + 'inputs': [ + { + 'id': '1', + 'group': 'op-test', + 'desc_inputs': [ + np.array([[1, 1], [1, 1]]).astype(np.float32), + np.array([[2, 2], [2, 2]]).astype(np.float32) + ] + }, + { + 'id': '2', + 'group': 'op-test', + 'desc_inputs': [ + np.array([[3, 3], [3, 3]]).astype(np.float32), + np.array([[4, 4], [4, 4]]).astype(np.float32) + ] + } + ], + 'expect': [ + { + 'id': '1', + 'group': 'op-test-op-test', + 'desc_expect': { + 'shape_type': [ + { + 'type': np.float32, + 'shape': (2, 2) + } + ] + } + } + ], + 'ext': {} +} +""" +pipeline_for_verify_shapetype_for_group_by_group_config = [IdentityDC, IdentityBC, GroupCartesianProductFIPC, + IdentityEC, GroupCartesianProductERPC, ShapeTypeVC] diff --git a/tests/mindspore_test_framework/pipeline/gradient/__init__.py b/tests/mindspore_test_framework/pipeline/gradient/__init__.py new file mode 100644 index 00000000000..8782c1327e9 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/gradient/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""gradient""" diff --git a/tests/mindspore_test_framework/pipeline/gradient/check_training.py b/tests/mindspore_test_framework/pipeline/gradient/check_training.py new file mode 100644 index 00000000000..6ad36af9c36 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/gradient/check_training.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for loss checking.""" + +from ...components.function_inputs_policy.cartesian_product_on_id_for_function_inputs import IdCartesianProductFIPC +from ...components.facade.me_facade import MeFacadeFC +from ...components.function.get_function_from_config import IdentityBC +from ...components.inputs.generate_dataset_for_linear_regression import GenerateDataSetForLRDC +from ...components.executor.exec_and_verify_model_loss import LossVerifierEC + +# pylint: disable=W0105 +""" +Check if model loss converge to a given bound. + +Example: + verification_set = [ + ('Linreg', { + 'block': { + 'model': network, + 'loss': SquaredLoss(), + 'opt': Lamb(network.trainable_params(), decay_steps=num_epochs, warmup_steps=10, weight_decay=0.01), + 'num_epochs': num_epochs, + 'loss_upper_bound': 0.3, + }, + 'desc_inputs': { + 'true_params': ([2, -3.4], 4.2), + 'num_samples': 100, + 'batch_size': 20, + } + }) + ] +""" +pipeline_for_check_model_loss_for_case_by_case_config = [MeFacadeFC, GenerateDataSetForLRDC, + IdentityBC, IdCartesianProductFIPC, + LossVerifierEC] diff --git a/tests/mindspore_test_framework/pipeline/gradient/compare_gradient.py b/tests/mindspore_test_framework/pipeline/gradient/compare_gradient.py new file mode 100644 index 00000000000..205f49d8942 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/gradient/compare_gradient.py @@ -0,0 +1,281 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for gradients comparison.""" + +from ...components.expect_result_policy.cartesian_product_on_id_for_expect_result import IdCartesianProductERPC +from ...components.function_inputs_policy.cartesian_product_on_id_for_function_inputs import IdCartesianProductFIPC +from ...components.executor.exec_gradient import IdentityBackwardEC +from ...components.function.run_gradient_wrt_inputs import RunBackwardBlockWrtInputsBC +from ...components.function.run_gradient_wrt_params import RunBackwardBlockWrtParamsBC +from ...components.inputs.generate_inputs_from_shape import GenerateFromShapeDC +from ...components.verifier.compare_gradient import CompareGradientWithVC +from ...components.facade.me_facade import MeFacadeFC +from ...components.inputs.load_inputs_from_npy import LoadFromNpyDC +from ...components.verifier.verify_expect_from_npy import LoadFromNpyVC +from ...components.function.get_function_from_config import IdentityBC +from ...components.executor.check_gradient_wrt_inputs import CheckGradientWrtInputsEC +from ...components.executor.check_gradient_wrt_params import CheckGradientWrtParamsEC +from ...components.executor.check_jacobian_wrt_inputs import CheckJacobianWrtInputsEC + +# pylint: disable=W0105 +""" +Compare inputs gradient with user defined operator's gradient. This pipeline is suitable for +case-by-case style config. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]], + 'desc_expect': { + 'compare_gradient_with': [ + (run_user_defined_grad, user_defined.add) + ], + } + }) + ] +""" +pipeline_for_compare_inputs_grad_with_user_defined_for_case_by_case_config =\ + [MeFacadeFC, GenerateFromShapeDC, + RunBackwardBlockWrtInputsBC, IdCartesianProductFIPC, + IdentityBackwardEC, IdCartesianProductERPC, + CompareGradientWithVC] + +""" +Compare inputs gradient with data stored in npy file. This pipeline is suitable for +case-by-case style config. + +Example: + verification_set = [ + ('MatMul', { + 'block': P.MatMul(), + 'desc_inputs': [ + ('tests/mindspore_test_framework/apps/data/input_0.npy', { + 'dtype': np.float32 + }), + ('tests/mindspore_test_framework/apps/data/input_1.npy', { + 'dtype': np.float32 + }), + ], + 'desc_bprop': [ + np.ones(shape=(2, 2)).astype(np.float32) + ], + 'desc_expect': [ + ('tests/mindspore_test_framework/apps/data/grad_0.npy', { + 'dtype': np.float32, + 'check_tolerance': True, + 'relative_tolerance': 1e-2, + 'absolute_tolerance': 1e-2 + }), + ('tests/mindspore_test_framework/apps/data/grad_1.npy', { + 'dtype': np.float32, + 'max_error': 1e-3 + }), + ] + }) + ] +""" +pipeline_for_compare_inputs_grad_with_npy_for_case_by_case_config =\ + [MeFacadeFC, LoadFromNpyDC, RunBackwardBlockWrtInputsBC, + IdCartesianProductFIPC, IdentityBackwardEC, + IdCartesianProductERPC, LoadFromNpyVC] + +""" +Compare params gradient with data stored in npy file. This pipeline is suitable for +case-by-case style config. + +Example: + verification_set = [ + ('MatMul', { + 'block': P.MatMul(), + 'desc_inputs': [ + ('tests/mindspore_test_framework/apps/data/input_0.npy', { + 'dtype': np.float32 + }), + ('tests/mindspore_test_framework/apps/data/input_1.npy', { + 'dtype': np.float32 + }), + ], + 'desc_bprop': [ + np.ones(shape=(2, 2)).astype(np.float32) + ], + 'desc_expect': [ + ('tests/mindspore_test_framework/apps/data/grad_0.npy', { + 'dtype': np.float32, + 'check_tolerance': True, + 'relative_tolerance': 1e-2, + 'absolute_tolerance': 1e-2 + }), + ('tests/mindspore_test_framework/apps/data/grad_1.npy', { + 'dtype': np.float32, + 'max_error': 1e-3 + }), + ] + }) + ] +""" +pipeline_for_compare_params_grad_with_npy_for_case_by_case_config =\ + [MeFacadeFC, LoadFromNpyDC, RunBackwardBlockWrtParamsBC, + IdCartesianProductFIPC, IdentityBackwardEC, + IdCartesianProductERPC, LoadFromNpyVC] + +""" +Compare inputs gradient with result of numerical differentiation. This pipeline is +suitable for case-by-case style config. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]] + }) + ] +""" +pipeline_for_compare_inputs_grad_with_numerical_diff_for_case_by_case_config =\ + [MeFacadeFC, GenerateFromShapeDC, IdentityBC, + IdCartesianProductFIPC, + CheckGradientWrtInputsEC] + +""" +Compare inputs gradient with result of numerical differentiation. This pipeline is +suitable for config in a grouped style. + +Example: + verification_set = { + 'inputs': [ + { + 'id': 'MatMul', + 'group': 'bert', + 'desc_inputs': [ + [3, 3], + [3, 3] + ] + }, + ], + 'function': [ + { + 'id': 'MatMul', + 'group': 'bert', + 'block': P.MatMul(), + 'reduce_output': False + } + ], + 'ext': {} + } +""" +pipeline_for_compare_inputs_grad_with_numerical_diff_for_group_by_group_config = [GenerateFromShapeDC, IdentityBC, + IdCartesianProductFIPC, + CheckGradientWrtInputsEC] + +""" +Compare params gradient with result of numerical differentiation. This pipeline is suitable for +config in a grouped style. + +Example: + verification_set = { + 'inputs': { + 'id': 'BertAttentionSoftmax', + 'group': 'bert', + 'desc_inputs': [ + [128, 1024], + [1, 16, 128, 128] + ], + 'desc_bprop': [ + [1, 16, 128, 64], + [1, 16, 128, 64] + ] + }, + 'function': [ + { + 'id': 'BertAttentionSoftmax', + 'group': 'BertAttentionSoftmax', + 'block': BertAttentionSoftmax(batch_size=1, + to_tensor_width=1024, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=16, + size_per_head=64, + value_act=None, + attention_probs_dropout_prob=0, + initializer_range=0.02) + } + ], + 'ext': {} + } +""" +pipeline_for_compare_params_grad_with_numerical_diff_for_group_by_group_config = [GenerateFromShapeDC, IdentityBC, + IdCartesianProductFIPC, + CheckGradientWrtParamsEC] + +""" +Compare inputs jacobian with result of numerical differentiation. This pipeline is suitable for +config in a case-by-case style. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]], + 'desc_expect': { + 'compare_with': [ + (run_np, np.add), + (run_user_defined, user_defined.add) + ], + 'compare_gradient_with': [ + (run_user_defined_grad, user_defined.add) + ], + } + }), + ] +""" +pipeline_for_compare_inputs_jacobian_with_numerical_diff_for_case_by_case_config =\ + [MeFacadeFC, GenerateFromShapeDC, IdentityBC, + IdCartesianProductFIPC, + CheckJacobianWrtInputsEC] + +""" +Compare inputs jacobian with result of numerical differentiation. This pipeline is suitable for +config in a grouped style. + +Example: + verification_set = { + 'inputs': [ + { + 'id': 'MatMul', + 'group': 'bert', + 'desc_inputs': [ + [3, 3], + [3, 3] + ] + }, + ], + 'function': [ + { + 'id': 'MatMul', + 'group': 'bert', + 'block': P.MatMul(), + 'reduce_output': False + } + ], + 'ext': {} + } +""" +pipeline_for_compare_inputs_jacobian_with_numerical_diff_for_group_by_group_config = [GenerateFromShapeDC, IdentityBC, + IdCartesianProductFIPC, + CheckJacobianWrtInputsEC] diff --git a/tests/mindspore_test_framework/pipeline/gradient/compile_gradient.py b/tests/mindspore_test_framework/pipeline/gradient/compile_gradient.py new file mode 100644 index 00000000000..b0dcae0d985 --- /dev/null +++ b/tests/mindspore_test_framework/pipeline/gradient/compile_gradient.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Pipelines for gradients.""" + +from ...components.facade.me_facade import MeFacadeFC +from ...components.inputs.generate_inputs_from_shape import GenerateFromShapeDC +from ...components.executor.exec_gradient import IdentityBackwardEC +from ...components.function_inputs_policy.cartesian_product_on_id_for_function_inputs import IdCartesianProductFIPC +from ...components.function.compile_gradient_wrt_inputs import CompileBackwardBlockWrtInputsBC +from ...components.function.run_gradient_wrt_inputs import RunBackwardBlockWrtInputsBC + +# pylint: disable=W0105 +""" +Check if compiling gradient anf graph is ok. This pipeline is suitable for case-by-case style config. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]] + }) + ] +""" +pipeline_for_compile_grad_anf_graph_for_case_by_case_config =\ + [MeFacadeFC, GenerateFromShapeDC, CompileBackwardBlockWrtInputsBC, + IdCartesianProductFIPC, IdentityBackwardEC] + +""" +Check if compiling gradient ge graph is ok. This pipeline is suitable for case-by-case style config. + +Example: + verification_set = [ + ('TensorAdd', { + 'block': (P.TensorAdd(), {'reduce_output': False}), + 'desc_inputs': [[1, 3, 3, 4], [1, 3, 3, 4]], + 'desc_bprop': [[1, 3, 3, 4]] + }) + ] +""" +pipeline_for_compile_grad_ge_graph_for_case_by_case_config =\ + [MeFacadeFC, GenerateFromShapeDC, RunBackwardBlockWrtInputsBC, + IdCartesianProductFIPC, IdentityBackwardEC] diff --git a/tests/mindspore_test_framework/utils/__init__.py b/tests/mindspore_test_framework/utils/__init__.py new file mode 100644 index 00000000000..a1bc9a67ae0 --- /dev/null +++ b/tests/mindspore_test_framework/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""utils""" diff --git a/tests/mindspore_test_framework/utils/block_util.py b/tests/mindspore_test_framework/utils/block_util.py new file mode 100644 index 00000000000..9d75ae0888a --- /dev/null +++ b/tests/mindspore_test_framework/utils/block_util.py @@ -0,0 +1,374 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for Cell related computation.""" + +# pylint: disable=missing-docstring + +import numpy as np + +from mindspore.common.api import _executor, ms_function +from mindspore.common.tensor import Tensor +from mindspore import nn, context +from mindspore.ops.composite import GradOperation +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore import ParameterTuple +from . import keyword + +def get_uniform_with_shape(shape): + np.random.seed(1) + return np.random.uniform(-0.1, 0.1, size=shape).astype(np.float32) + +def set_block_param_with_rand(net, rand_func=None): + if not isinstance(net, nn.Cell) or rand_func is None: + return + for param in net.trainable_params(): + param.default_input = Tensor(rand_func(param.default_input.asnumpy().shape)) + +def compile_block(net, *inputs, rand_func=None, training=True): + set_block_training(net, training) + set_block_param_with_rand(net, rand_func) + return _executor.compile(net, *inputs) + +def run_block(net, *inputs, rand_func=None, training=True): + set_block_training(net, training) + set_block_param_with_rand(net, rand_func) + if context.get_context("mode") == context.PYNATIVE_MODE: + def func_pynative(*inputs): + @ms_function + def _func_pynative(*inputs): + return net(*inputs) + return _func_pynative(*inputs) + return func_pynative(*inputs) + return net(*inputs) + +class IthOutputCell(nn.Cell): + def __init__(self, network, output_index): + if isinstance(network, nn.Cell): + super(IthOutputCell, self).__init__(auto_prefix=False) + else: + super(IthOutputCell, self).__init__() + self.network = network + self.output_index = output_index + + def construct(self, *inputs): + raise NotImplementedError + + def construct1(self, x1): + predict = self.network(x1)[self.output_index] + return predict + + def construct2(self, x1, x2): + predict = self.network(x1, x2)[self.output_index] + return predict + + def construct3(self, x1, x2, x3): + predict = self.network(x1, x2, x3)[self.output_index] + return predict + + def construct4(self, x1, x2, x3, x4): + predict = self.network(x1, x2, x3, x4)[self.output_index] + return predict + + def construct5(self, x1, x2, x3, x4, x5): + predict = self.network(x1, x2, x3, x4, x5)[self.output_index] + return predict + +def get_output_cell(network, num_input, output_index, training=True): + net = IthOutputCell(network, output_index) + f = getattr(net, 'construct%d' % num_input) + setattr(net, "construct", f) + set_block_training(net, training) + return net + +class OutputReduceSumCell(nn.Cell): + def __init__(self, network, output_num): + super(OutputReduceSumCell, self).__init__() + self.output_num = output_num + self.network = network + self.reduce_sum = P.ReduceSum() + + def construct(self, *inputs): + if self.output_num == 1: + return self.reduce_sum(self.network(*inputs), None) + ret = F.make_tuple() + for index in range(self.output_num): + predict = self.network(*inputs)[index] + predict_reduce = self.reduce_sum(predict, None) + ret = ret + F.make_tuple(predict_reduce) + return ret + +def get_output_reduce_cell(network, output_num, training=True): + net = OutputReduceSumCell(network, output_num) + set_block_training(net, training) + return net + +class InputOpNet(nn.Cell): + def __init__(self, op, c1=None, c2=None, c3=None, c4=None): + super(InputOpNet, self).__init__() + self.op = op + self.c1 = c1 + self.c2 = c2 + self.c3 = c3 + self.c4 = c4 + + def construct(self, *inputs): + raise NotImplementedError + + def construct0_c0_fake(self, data): + x = self.op() + data + return x + def construct0_c1_fake(self, data): + x = self.op(self.c1) + data + return x + + def construct0_c2_fake(self, data): + x = self.op(self.c1, self.c2) + data + return x + + def construct0_c3_fake(self, data): + x = self.op(self.c1, self.c2, self.c3) + data + return x + + def construct0_c0(self): + x = self.op() + return x + + def construct0_c1(self): + x = self.op(self.c1) + return x + + def construct0_c2(self): + x = self.op(self.c1, self.c2) + return x + + def construct1_c0(self, x1): + x = self.op(x1) + return x + + def construct1_c1(self, x1): + x = self.op(x1, self.c1) + return x + + def construct1_c2(self, x1): + x = self.op(x1, self.c1, self.c2) + return x + + def construct1_c3(self, x1): + x = self.op(x1, self.c1, self.c2, self.c3) + return x + + def construct1_c4(self, x1): + x = self.op(x1, self.c1, self.c2, self.c3, self.c4) + return x + + def constructc1_1(self, x1): + x = self.op(self.c1, x1) + return x + + def construct2_c0(self, x1, x2): + x = self.op(x1, x2) + return x + + def construct2_c1(self, x1, x2): + x = self.op(x1, x2, self.c1) + return x + + def construct2_c3(self, x1, x2): + x = self.op(x1, x2, self.c1, self.c2, self.c3) + return x + + def construct3_c0(self, x1, x2, x3): + x = self.op(x1, x2, x3) + return x + + def construct3_c1(self, x1, x2, x3): + x = self.op(x1, x2, x3, self.c1) + return x + + def construct4_c0(self, x1, x2, x3, x4): + x = self.op(x1, x2, x3, x4) + return x + + def construct4_c1(self, x1, x2, x3, x4): + x = self.op(x1, x2, x3, x4, self.c1) + return x + + def construct4_c4(self, x1, x2, x3, x4): + x = self.op(x1, x2, x3, x4, self.c1, self.c2, self.c3, self.c4) + return x + + def construct5_c0(self, x1, x2, x3, x4, x5): + x = self.op(x1, x2, x3, x4, x5) + return x + + def construct6_c0(self, x1, x2, x3, x4, x5, x6): + x = self.op(x1, x2, x3, x4, x5, x6) + return x + + def construct5_c1(self, x1, x2, x3, x4, x5): + x = self.op(x1, x2, x3, x4, x5, self.c1) + return x + +def gen_net(op, input_num, training=True, desc_const=(), const_first=False, add_fake_input=False): + if isinstance(op, nn.Cell): + return op + net = InputOpNet(op, *desc_const) + if const_first: + fn_name = 'constructc%d_%d' % (len(desc_const), input_num) + else: + fn_name = 'construct%d_c%d' % (input_num, len(desc_const)) + if add_fake_input: + fn_name += '_fake' + f = getattr(net, fn_name) + setattr(net, "construct", f) + set_block_training(net, training) + return net + +class OperationBackward(nn.Cell): + def __init__(self, network, grad_op, sens): + if isinstance(network, nn.Cell): + super(OperationBackward, self).__init__(auto_prefix=False) + else: + super(OperationBackward, self).__init__() + self.network = network + self.grad = grad_op + self.sens = sens + + def construct(self, *inputs): + return self.grad(self.network)(*inputs, self.sens) + +class OperationBackwardWithNoSens(nn.Cell): + def __init__(self, network, grad_op): + if isinstance(network, nn.Cell): + super(OperationBackwardWithNoSens, self).__init__(auto_prefix=False) + else: + super(OperationBackwardWithNoSens, self).__init__() + self.network = network + self.grad = grad_op + + def construct(self, *inputs): + return self.grad(self.network)(*inputs) + +class NNBackward(nn.Cell): + def __init__(self, network, grad_op, sens): + if isinstance(network, nn.Cell): + super(NNBackward, self).__init__(auto_prefix=False) + else: + super(NNBackward, self).__init__() + self.network = network + self.grad = grad_op + self.params = ParameterTuple(network.trainable_params()) + self.sens = sens + + def construct(self, *inputs): + return self.grad(self.network, self.params)(*inputs, self.sens) + +class NNBackwardWithNoSens(nn.Cell): + def __init__(self, network, grad_op): + if isinstance(network, nn.Cell): + super(NNBackwardWithNoSens, self).__init__(auto_prefix=False) + else: + super(NNBackwardWithNoSens, self).__init__() + self.network = network + self.grad = grad_op + self.params = ParameterTuple(network.trainable_params()) + + def construct(self, *inputs): + return self.grad(self.network, self.params)(*inputs) + +def gen_grad_net(net, grad_op, input_num, sens=None, training=True, desc_const=(), + const_first=False, add_fake_input=False): + if not isinstance(net, nn.Cell): + net = gen_net(net, input_num, desc_const=desc_const, const_first=const_first, add_fake_input=add_fake_input) + if grad_op.get_by_list: + if grad_op.sens_param: + net = NNBackward(net, grad_op, sens) + else: + net = NNBackwardWithNoSens(net, grad_op) + else: + if grad_op.sens_param: + net = OperationBackward(net, grad_op, sens) + else: + net = OperationBackwardWithNoSens(net, grad_op) + set_block_training(net, training) + return net + +def set_block_training(net, training=True): + if isinstance(net, nn.Cell): + net.set_train(training) + +def set_block_phase(net, phase='train'): + if isinstance(net, nn.Cell): + net.phase = phase + +def create_funcs(verification_set, block_generator, block_runner, grad_op=None, default_rand_func=None): + def create_func(block, num_outputs, rand_func, desc_const, const_first, add_fake_input, split_outputs): + def function(*inputs): + # gradient + if grad_op: + if num_outputs == 0: + grad_op_ = GradOperation('grad', get_all=grad_op.get_all, + get_by_list=grad_op.get_by_list, sens_param=False) + b = block_generator(block, grad_op_, len(inputs), desc_const=desc_const, + const_first=const_first, add_fake_input=add_fake_input) + return block_runner(b, *inputs, rand_func=rand_func) + if num_outputs == 1: + b = block_generator(block, grad_op, len(inputs) - 1, inputs[-1], desc_const=desc_const, + const_first=const_first, add_fake_input=add_fake_input) + return block_runner(b, *(inputs[:-1]), rand_func=rand_func) + if split_outputs: + block_inputs = inputs[0:len(inputs) - num_outputs] + sens_inputs = inputs[len(inputs) - num_outputs:] + ret = [] + for i in range(num_outputs): + bi_inputs = list(block_inputs) + bi = get_output_cell(block, len(block_inputs), i) + bi = block_generator(bi, grad_op, len(bi_inputs), sens_inputs[i], desc_const=desc_const, + const_first=const_first, add_fake_input=add_fake_input) + grads_i = block_runner(bi, *bi_inputs, rand_func=rand_func) + if isinstance(grads_i, tuple): + ret.extend(grads_i) + else: + ret.append(grads_i) + return ret + block_inputs = inputs[0:len(inputs) - num_outputs] + sens_inputs = tuple(inputs[len(inputs) - num_outputs:]) + b = block_generator(block, grad_op, len(block_inputs), sens_inputs, desc_const=desc_const, + const_first=const_first, add_fake_input=add_fake_input) + return block_runner(b, *block_inputs, rand_func=rand_func) + # forward + inputs_num = len(inputs) + if add_fake_input and inputs_num == 1: + # input is faked + inputs_num = 0 + b = block_generator(block, inputs_num, desc_const=desc_const, const_first=const_first, + add_fake_input=add_fake_input) + return block_runner(b, *inputs, rand_func=rand_func) + return function + + bc_configs = verification_set[keyword.function] + for config in bc_configs: + block = config[keyword.block] + rand_func = config.get(keyword.init_param_with, default_rand_func) + num_outputs = config.get(keyword.num_outputs, 0) + desc_const = config.get(keyword.desc_const, []) + const_first = config.get(keyword.const_first, False) + add_fake_input = config.get(keyword.add_fake_input, False) + split_outputs = config.get(keyword.split_outputs, True) + config[keyword.block] = create_func(block, num_outputs, rand_func, desc_const, + const_first, add_fake_input, split_outputs) + return bc_configs diff --git a/tests/mindspore_test_framework/utils/bprop_util.py b/tests/mindspore_test_framework/utils/bprop_util.py new file mode 100644 index 00000000000..35d4f393359 --- /dev/null +++ b/tests/mindspore_test_framework/utils/bprop_util.py @@ -0,0 +1,95 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for computing gradients.""" + +from mindspore.ops.composite.base import GradOperation +from mindspore import context +from mindspore.nn import Cell +from mindspore.common import ParameterTuple +from mindspore.common.api import ms_function + +class Bprop(Cell): + """ + The gradient wraper. + """ + def __init__(self, func, wrt_params, params, grad_op, sens): + super(Bprop, self).__init__(auto_prefix=False) + self.func = func + self.wrt_params = wrt_params + self.params = None + if self.wrt_params and params: + self.params = ParameterTuple(params) + self.grad = grad_op + self.sens = sens + self.with_sens = False + if sens: + self.with_sens = True + + def construct(self, *inputs): + # pylint: disable=no-else-return + if self.wrt_params: + if self.with_sens: + return self.grad(self.func, self.params)(*inputs, self.sens) + else: + return self.grad(self.func, self.params)(*inputs) + elif self.with_sens: + return self.grad(self.func)(*inputs, self.sens) + else: + return self.grad(self.func)(*inputs) + +def bprop(func, *inputs, grads_wrt_outputs=None, wrt: list = None, params: list = None): + """ + Compute gradients of function. + + Args: + func (Function): The target function. + inputs (Variable argument): Inputs of the func. + grads_wrt_outputs (List): Gradients of the loss wrt outputs of func, default [1.0]. + wrt (List): Compute gradients wrt ['inputs' | 'params']. + params (List): Specify the params to compute gradients wrt, default all trainable_params. + + Returns: + Tuple, gradients of function. + """ + assert isinstance(func, Cell) + func.set_train() + + with_sens_param = False + if grads_wrt_outputs: + with_sens_param = True + + if not wrt: + wrt = [] + wrt_inputs = False + if 'inputs' in wrt: + wrt_inputs = True + wrt_params = False + if 'params' in wrt: + wrt_params = True + if not params: + params = func.trainable_params() + + grad_op = GradOperation(name='grad', get_all=wrt_inputs, get_by_list=wrt_params, sens_param=with_sens_param) + grad = Bprop(func, wrt_params, params, grad_op, grads_wrt_outputs) + + if context.get_context("mode") == context.PYNATIVE_MODE: + def func_pynative(*inputs): + @ms_function + def _func_pynative(*inputs): + return grad(*inputs) + return _func_pynative(*inputs) + return func_pynative(*inputs) + return grad(*inputs) diff --git a/tests/mindspore_test_framework/utils/check_gradient.py b/tests/mindspore_test_framework/utils/check_gradient.py new file mode 100644 index 00000000000..fea7e22e55e --- /dev/null +++ b/tests/mindspore_test_framework/utils/check_gradient.py @@ -0,0 +1,456 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Implementation of Numerical gradients checking.""" +# pylint: disable=missing-docstring + +from typing import Callable, List, Any +import numpy as np +from mindspore import Tensor +from mindspore.ops.composite import GradOperation +from mindspore import ParameterTuple +from mindspore import context +from mindspore.common.api import ms_function +import mindspore._c_expression as _c_expression +from .block_util import get_output_cell, gen_net, gen_grad_net, \ + get_uniform_with_shape, set_block_phase, get_output_reduce_cell, set_block_param_with_rand + +class _GradChecker: + """ + Check the theoretical Jacobian against numeric + + Arguments: + fn: The function under test. + gfn: The hight order function to compute the derivative function. + args: The point in the function's domain where we want + to estimate the gradient. + + """ + + def __init__(self, + fn: Callable, + grad_wraper: GradOperation, + args: List[Any], + delta: float = 1e-3, + max_error: float = 1e-3, + input_selector=None, + output_selector=None, + sampling_times=-1, + reduce_output=False) -> None: + """Initialize a GradChecker.""" + self.delta = delta + self.scale = 2 * delta + self.max_error = max_error + self.sampling_times = sampling_times + + self.fn = self.prepare_func(fn) + + self.args = args + out = self.fn(*self.args) + self.out = self.wrap(out) + + self.nin = len(self.args) + self.nout = len(self.out) + self.gfns = [] + + if reduce_output: + fn = get_output_reduce_cell(fn, self.nout) + self.fn = self.prepare_func(fn) + out = self.fn(*self.args) + self.out = self.wrap(out) + + if self.nout == 0: + raise Exception(f'number of outputs expected to be >=1, but got {self.nout}') + + if self.nout == 1: + self.gfns.append(self.prepare_func(fn, grad_wraper)) + else: + for i in range(self.nout): + cell = get_output_cell(fn, self.nin, i) + self.gfns.append(self.prepare_func(cell, grad_wraper)) + + self.input_selector = input_selector + self.adjust_input_selector() + if output_selector: + self.output_selector = output_selector + else: + self.output_selector = [i for i in range(self.nout)] + + def adjust_input_selector(self): + raise Exception('Not implemented') + + def sampling(self, superset): + # -1 stands for all + if self.sampling_times == -1 or self.sampling_times >= len(superset): + return superset + np.random.seed(0) + ret = np.random.choice(superset, self.sampling_times, replace=False) + return list(ret) + + def prepare_func(self, f, grad_wraper=None): + """Return a function that executes 'f'. + + Args: + f: the function. + grad_wraper: grad op + + Returns: + a function that will be evaluated in both Graph and PyNative mode + """ + set_block_param_with_rand(f, get_uniform_with_shape) + + if context.get_context("mode") == context.PYNATIVE_MODE: + if grad_wraper: + def func_backward_pynative(*inputs): + net = gen_grad_net(f, grad_wraper, len(inputs) - 1, inputs[-1]) + + @ms_function + def _func_pynative(*inputs): + return net(*inputs) + + return _func_pynative(*(inputs[:-1])) + + return func_backward_pynative + + def func_forward_pynative(*inputs): + net = gen_net(f, len(inputs)) + + @ms_function + def _func_pynative(*inputs): + return net(*inputs) + return _func_pynative(*inputs) + + return func_forward_pynative + + if grad_wraper: + def func_backward_graph(*inputs): + set_block_phase(f, 'train') + net = gen_grad_net(f, grad_wraper, len(inputs) - 1, inputs[-1]) + return net(*(inputs[:-1])) + + return func_backward_graph + + def func_forward_graph(*inputs): + set_block_phase(f, 'predict') + net = gen_net(f, len(inputs)) + return net(*inputs) + + return func_forward_graph + + def to_numpy(self, x): + if isinstance(x, (Tensor, _c_expression.Tensor)): + return x.asnumpy() + return x + + def to_numpy_and_scale(self, x): + if isinstance(x, (Tensor, _c_expression.Tensor)): + return x.asnumpy() * self.delta + return x * self.delta + + def wrap(self, x): + if isinstance(x, tuple): + return x + return (x,) + + def get_sens(self, i): + raise Exception('Not implemented') + + def get_ith_elem(self, c, i): + if isinstance(c, (list, tuple)): + return c[i] + return c + + def compute_theoretical(self, i): + args = list(self.args) + args.append(self.get_sens(i)) + + print('GradChecker.compute_theoretical.args', args) + gout = self.gfns[i](*args) + gout = self.wrap(gout) + self.gout = [self.to_numpy_and_scale(g) if isinstance(g, _c_expression.Tensor) \ + else self.to_numpy_and_scale(np.array(g)) for g in gout] + print('GradChecker.compute_theoretical.gout', self.gout) + + def check_against_numeric(self, out_index): + raise Exception('Not implemented') + + def check_against_numeric_one_step(self, args, index, out_index): + if isinstance(args, ParameterTuple): + x = args[index].default_input.asnumpy() + else: + x = args[index] + x_shape = x.shape + x_size = np.product(x_shape) + for row in self.sampling(list(range(x_size))): + original = x.ravel().view()[row] + x.ravel().view()[row] += self.delta + y_pos = self.to_numpy_and_scale(self.get_ith_elem(self.fn(*self.args), out_index)) + x.ravel().view()[row] = original + x.ravel().view()[row] -= self.delta + y_neg = self.to_numpy_and_scale(self.get_ith_elem(self.fn(*self.args), out_index)) + x.ravel().view()[row] = original + diff = (y_pos - y_neg) / self.scale + numeric_grad = diff.sum() + insert_virtual_grad = False + if numeric_grad == 0 and not insert_virtual_grad: + self.gout.insert(0, 0) + insert_virtual_grad = True + continue + theoretical_grad = self.gout[index].ravel().view()[row] + + if np.fabs(numeric_grad - theoretical_grad).max() > self.max_error: + raise Exception(f'Gradients of df{out_index}/darg{index},{row} do not match, ' + f'expect {numeric_grad}, actual {theoretical_grad}') + + print(f'GradChecker.check_against_numeric.numeric df{out_index}/darg{index}: ' + f'{numeric_grad}, theoretical: {theoretical_grad}') + + # approximate accuracy, but efficient + def assert_match(self): + print(f'==========================={self.fn.__name__}==================================') + print('GradChecker.delta', self.delta) + print('GradChecker.max_error', self.max_error) + print('GradChecker.args', self.args) + print('GradChecker.out', self.out) + print('GradChecker.nin', self.nin) + print('GradChecker.nout', self.nout) + for i in self.output_selector: + self.compute_theoretical(i) + self.check_against_numeric(i) + + def check_against_numeric_jacobian(self, out_index): + raise Exception('Not implemented') + + def check_against_numeric_jacobian_one_step(self, args, index, out_index): + if isinstance(args, ParameterTuple): + x = args[index].default_input.asnumpy() + else: + x = args[index] + x_shape = x.shape + x_size = np.product(x_shape) + dy = self.to_numpy(self.get_sens(out_index)) + dy_size = np.product(dy.shape) + numeric_jacobian = np.zeros((x_size, dy_size), dtype=self.to_numpy(x).dtype) + for row in range(x_size): + original = x.ravel().view()[row] + x.ravel().view()[row] += self.delta + y_pos = self.to_numpy_and_scale(self.get_ith_elem(self.fn(*self.args), out_index)) + x.ravel().view()[row] = original + x.ravel().view()[row] -= self.delta + y_neg = self.to_numpy_and_scale(self.get_ith_elem(self.fn(*self.args), out_index)) + x.ravel().view()[row] = original + diff = (y_pos - y_neg) / self.scale + numeric_jacobian[row, :] = diff.ravel().view(numeric_jacobian.dtype) + + dy_mask = np.zeros(dy.shape, dtype=dy.dtype) + theoretical_jacobian = np.zeros((x_size, dy_size), dtype=self.to_numpy(x).dtype) + for col in range(dy_size): + col_jacobian = self.compute_theoretical_jacobian(index, out_index, dy_mask, col) + theoretical_jacobian[:, col] = col_jacobian.ravel().view(theoretical_jacobian.dtype) + + if np.fabs(numeric_jacobian - theoretical_jacobian).max() > self.max_error: + raise Exception(f'GradChecker.check_against_numeric_jacobian_one_step expect {out_index}/darg{index}: ' + f'{numeric_jacobian}, actual: {theoretical_jacobian}') + + print(f'GradChecker.check_against_numeric_jacobian_one_step.numeric jacobian of output{out_index}/darg{index}: ' + f'{numeric_jacobian}, theoretical: {theoretical_jacobian}') + + def compute_theoretical_jacobian(self, index, out_index, dy_mask, jacobian_col): + if (out_index, jacobian_col, index) in self.theoretical_jacobian_cache: + return self.theoretical_jacobian_cache[(out_index, jacobian_col, index)] + + dy_mask.ravel().view()[jacobian_col] = 1.0 + args = list(self.args) + args.append(Tensor(dy_mask)) + print('GradChecker.compute_theoretical.args', args) + gout = self.wrap(self.gfns[out_index](*args)) + gout = [self.to_numpy_and_scale(g) if isinstance(g, _c_expression.Tensor) \ + else self.to_numpy_and_scale(np.array(g)) for g in gout] + print('GradChecker.compute_theoretical.gout', gout) + dy_mask.ravel().view()[jacobian_col] = 0.0 + + for i, g in enumerate(gout): + self.theoretical_jacobian_cache[(out_index, jacobian_col, i)] = g + + return gout[index] + + # more accurate, but inefficient + def assert_match_jacobian(self): + print(f'==========================={self.fn.__name__}==================================') + print('GradChecker.delta', self.delta) + print('GradChecker.max_error', self.max_error) + print('GradChecker.args', self.args) + print('GradChecker.out', self.out) + print('GradChecker.nin', self.nin) + print('GradChecker.nout', self.nout) + + self.theoretical_jacobian_cache = {} + for i in self.output_selector: + self.check_against_numeric_jacobian(i) + + +class ScalarGradChecker(_GradChecker): + def __init__(self, + fn: Callable, + args: List[Any], + delta: float = 1e-3, + max_error: float = 1e-3, + input_selector=None, + output_selector=None, + sampling_times=-1, + reduce_output=False) -> None: + grad_op = GradOperation('grad', get_all=True, sens_param=True) + super(ScalarGradChecker, self).__init__(fn, grad_op, args, delta, max_error, input_selector, \ + output_selector, sampling_times, reduce_output) + + def adjust_input_selector(self): + if not self.input_selector: + self.input_selector = [i for i in range(self.nin)] + + def get_sens(self, i): + return 1 + + def check_against_numeric(self, out_index): + args = list(self.args) + for i in self.sampling(self.input_selector): + print(f'GradChecker.check_against_numeric.args[{i}]', args[i]) + args_pos = args[:i] + [args[i] + self.delta] + args[i + 1:] + args_neg = args[:i] + [args[i] - self.delta] + args[i + 1:] + y_pos = self.to_numpy_and_scale(self.get_ith_elem(self.fn(*args_pos), out_index)) + y_neg = self.to_numpy_and_scale(self.get_ith_elem(self.fn(*args_neg), out_index)) + diff = (y_pos - y_neg) / self.scale + + if np.fabs(diff - self.gout[i]).max() > self.max_error: + raise Exception(f'Gradients of df{out_index}/darg{i} do not match,' + f'expect {diff}, actual {self.gout[i]}') + + print(f'GradChecker.check_against_numeric.numeric df{out_index}/darg{i}: {diff}, ' + f'theoretical: {self.gout[i]}') + + # for scalar, jacobian is same with gradient + def assert_match_jacobian(self): + self.assert_match() + + +class OperationGradChecker(_GradChecker): + def __init__(self, + fn: Callable, + args: List[Any], + delta: float = 1e-3, + max_error: float = 1e-3, + input_selector=None, + output_selector=None, + sampling_times=-1, + reduce_output=False) -> None: + grad_op = GradOperation('grad', get_all=True, sens_param=True) + super(OperationGradChecker, self).__init__(fn, grad_op, args, delta, max_error, input_selector, \ + output_selector, sampling_times, reduce_output) + + def get_sens(self, i): + return Tensor(np.ones_like(self.out[i].asnumpy())) + + def adjust_input_selector(self): + if not self.input_selector: + self.input_selector = [i for i in range(self.nin)] + + def check_against_numeric(self, out_index): + args = [self.to_numpy(arg) for arg in self.args] + for i in self.input_selector: + self.check_against_numeric_one_step(args, i, out_index) + + def check_against_numeric_jacobian(self, out_index): + args = [self.to_numpy(arg) for arg in self.args] + for i in self.input_selector: + self.check_against_numeric_jacobian_one_step(args, i, out_index) + + +class NNGradChecker(_GradChecker): + def __init__(self, + fn: Callable, + args: List[Any], + delta: float = 1e-3, + max_error: float = 1e-3, + input_selector=None, + output_selector=None, + sampling_times=-1, + reduce_output=False) -> None: + grad_op = GradOperation('grad', get_by_list=True, sens_param=True) + self.params = ParameterTuple(fn.trainable_params()) + super(NNGradChecker, self).__init__(fn, grad_op, args, delta, max_error, input_selector, \ + output_selector, sampling_times, reduce_output) + + def get_sens(self, i): + return Tensor(np.ones_like(self.out[i].asnumpy())) + + def adjust_input_selector(self): + if not self.input_selector: + self.input_selector = [i for i in range(len(self.params))] + + def check_against_numeric(self, out_index): + for i in self.input_selector: + self.check_against_numeric_one_step(self.params, i, out_index) + + def check_against_numeric_jacobian(self, out_index): + for i in self.input_selector: + self.check_against_numeric_jacobian_one_step(self.params, i, out_index) + + +def check_gradient(fn, *args, delta=1e-3, max_error=1e-3, + grad_checker_class=OperationGradChecker, + input_selector=None, + output_selector=None, + sampling_times=-1, + reduce_output=False): + """Check the theoretical Jacobian against numeric of `fn`. + Args: + fn: the function that might be scalar function, operation, or neural network. + args: a list arguments for the function + delta: (optional) perturbation used to compute numeric Jacobian. + max_error: (optional) max_error that is allowed between theoretical and numeric. + grad_checker_class: (optional) checker, default OperationGradChecker. + input_selector: list of input index that will be checked against numeric + output_selector: list of output index that will be checked against numeric + """ + grad_checker = grad_checker_class(fn=fn, + args=list(args), + delta=delta, + max_error=max_error, + input_selector=input_selector, + output_selector=output_selector, + sampling_times=sampling_times, + reduce_output=reduce_output) + grad_checker.assert_match() + +def check_jacobian(fn, *args, delta=1e-3, max_error=1e-3, + grad_checker_class=OperationGradChecker, + input_selector=None, + output_selector=None): + """Check the theoretical Jacobian against numeric of `fn`. + Args: + fn: the function that might be scalar function, operation, or neural network. + args: a list arguments for the function + delta: (optional) perturbation used to compute numeric Jacobian. + max_error: (optional) max_error that is allowed between theoretical and numeric. + grad_checker_class: (optional) checker, default OperationGradChecker + input_selector: list of input index that will be checked against numeric + output_selector: list of output index that will be checked against numeric + """ + grad_checker = grad_checker_class(fn=fn, + args=list(args), + delta=delta, + max_error=max_error, + input_selector=input_selector, + output_selector=output_selector) + grad_checker.assert_match_jacobian() diff --git a/tests/mindspore_test_framework/utils/compare_util.py b/tests/mindspore_test_framework/utils/compare_util.py new file mode 100644 index 00000000000..178d4dab071 --- /dev/null +++ b/tests/mindspore_test_framework/utils/compare_util.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for baseline comparison test.""" + +import numpy as np +from .other_util import to_numpy_list +from . import keyword + +def compare(expect, func_result, baseline): + """ + Compare results of function with baseline functions. + + Args: + expect (dict): Config item in form of {'desc_expect': {'compare_with': f, 'max_error': 1e-3}}. + func_result (dict): Verification item in form of {'result': Tensor([2, 2]), 'desc_inputs': Tensor([2, 2])}. + baseline (str): Config item, compare_with | compare_gradient_with. + Returns: + """ + results = to_numpy_list(func_result[keyword.result]) + inputs = to_numpy_list(func_result[keyword.desc_inputs]) + funcs = expect[keyword.desc_expect][baseline] + max_error = expect[keyword.desc_expect].get(keyword.max_error, 1e-3) + for func in funcs: + if isinstance(func, tuple): + ret = func[0](func[1], *inputs) + else: + ret = func(*inputs) + + expects = to_numpy_list(ret) + for i, e in enumerate(expects): + if np.fabs(e - results[i]).max() > max_error: + raise TypeError(f'Error: expect {e} by {func}, but got {results[i]}') diff --git a/tests/mindspore_test_framework/utils/config_util.py b/tests/mindspore_test_framework/utils/config_util.py new file mode 100644 index 00000000000..d2eb91d042c --- /dev/null +++ b/tests/mindspore_test_framework/utils/config_util.py @@ -0,0 +1,109 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for verification config.""" + +import numpy as np + +from . import keyword +from .other_util import select_from_config_tuple + +def get_input_config(d): + """ + Get input config. + + Args: + d (tuple): Config item in form of ([2, 2], {'dtype': np.float32, 'scale': 1}). + Returns: + Tuple, (shape, dtype, scale). + """ + s = select_from_config_tuple(d, 0, d) + dtype = np.float32 + scale = 1 + if isinstance(d, tuple) and isinstance(d[-1], dict): + ext_config = d[-1] + dtype = ext_config.get(keyword.dtype, np.float32) + scale = ext_config.get(keyword.scale, 1) + return s, dtype, scale + +def get_expect_config(d): + """ + Get input config. + + Args: + d (tuple): Config item in form of (file_path, {'dtype': np.float32, + 'scale': 1, 'max_error': 1e-3, 'check_tolerance': False, 'relative_tolerance': 0.0, + 'absolute_tolerance': 0.0}). + Returns: + Tuple, (file_path, dtype, scale, max_error, check_tolerance, relative_tolerance, absolute_tolerance). + """ + s = select_from_config_tuple(d, 0, d) + dtype = np.float32 + scale = 1 + max_error = 1e-3 + check_tolerance = False + relative_tolerance = 0.0 + absolute_tolerance = 0.0 + if isinstance(d, tuple) and isinstance(d[-1], dict): + ext_config = d[-1] + dtype = ext_config.get(keyword.dtype, np.float32) + scale = ext_config.get(keyword.scale, 1) + max_error = ext_config.get(keyword.max_error, 1e-3) + check_tolerance = ext_config.get(keyword.check_tolerance, False) + relative_tolerance = ext_config.get(keyword.relative_tolerance, 0.0) + absolute_tolerance = ext_config.get(keyword.absolute_tolerance, 0.0) + return s, dtype, scale, max_error, check_tolerance, relative_tolerance, absolute_tolerance + +def get_function_config(function): + """ + Get input config. + + Args: + function (dict): Config item in form of {'delta': 1e-3, 'max_error': 1e-3, 'input_selector': [0, 1], + 'output_selector': 0, 'sampling_times': 10, 'reduce_output': True, 'init_param_with': None, + 'split_outputs': True, 'exception': Exception}. + Returns: + Tuple, (delta, max_error, input_selector, output_selector, sampling_times, + reduce_output, init_param_with, split_outputs, exception). + """ + delta = function.get(keyword.delta, 1e-3) + max_error = function.get(keyword.max_error, 1e-3) + input_selector = function.get(keyword.input_selector, []) + output_selector = function.get(keyword.output_selector, []) + sampling_times = function.get(keyword.sampling_times, -1) + reduce_output = function.get(keyword.reduce_output, True) + init_param_with = function.get(keyword.init_param_with, None) + split_outputs = function.get(keyword.split_outputs, True) + exception = function.get(keyword.exception, Exception) + return delta, max_error, input_selector, output_selector, sampling_times, \ + reduce_output, init_param_with, split_outputs, exception + +def get_grad_checking_options(function, inputs): + """ + Get input config. + + Args: + function (dict): Config item in form of {'block': XCell, 'delta': 1e-3, 'max_error': 1e-3, 'input_selector': + [0, 1], 'output_selector': 0, 'sampling_times': 10, 'reduce_output': True, + 'init_param_with': None, 'split_outputs': True, 'exception': Exception}. + inputs (dict): Config item in form of {'desc_inputs': [[2, 2]]}. + Returns: + Tuple, (f, args, delta, max_error, input_selector, output_selector, sampling_times, reduce_output). + """ + f = function[keyword.block] + args = inputs[keyword.desc_inputs] + delta, max_error, input_selector, output_selector, sampling_times, reduce_output, _, _, _ = \ + get_function_config(function) + return f, args, delta, max_error, input_selector, output_selector, sampling_times, reduce_output diff --git a/tests/mindspore_test_framework/utils/dataset_util.py b/tests/mindspore_test_framework/utils/dataset_util.py new file mode 100644 index 00000000000..bc3a2956f70 --- /dev/null +++ b/tests/mindspore_test_framework/utils/dataset_util.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Dataset utils.""" + +import random +import numpy as np +from mindspore import Tensor + +def generate_dataset_for_linear_regression(true_w, true_b, num_samples, batch_size): + features = np.random.normal(scale=1, size=(num_samples, len(true_w))) + labels = np.matmul(features, np.reshape(np.array(true_w), (-1, 1))) + true_b + labels += np.random.normal(scale=0.01, size=labels.shape) + indices = list(range(num_samples)) + random.shuffle(indices) + + for i in range(0, num_samples, batch_size): + j = np.array(indices[i: min(i + batch_size, num_samples)]) + yield Tensor(features.take(j, 0).astype(np.float32)), Tensor(labels.take(j, 0).astype(np.float32)) diff --git a/tests/mindspore_test_framework/utils/debug_util.py b/tests/mindspore_test_framework/utils/debug_util.py new file mode 100644 index 00000000000..7b2978a6daa --- /dev/null +++ b/tests/mindspore_test_framework/utils/debug_util.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Debugging utils.""" + +# pylint: disable=missing-docstring, unused-argument + +import logging +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.ops._grad.grad_base import bprop_getters +from mindspore.ops.primitive import prim_attr_register, PrimitiveWithInfer + +logging.basicConfig(level=logging.DEBUG, format= + '[%(levelname)s] %(asctime)s %(pathname)s:%(lineno)d %(message)s') +logger = logging.getLogger(__name__) + +class PrintShapeType(PrimitiveWithInfer): + """ + PrintShapeType input's shape and type. + + Args: + msg (str): The msg to print. + + Inputs: + - **input_x** (:class:`mindspore.dtype`) - The data to print. + + Outputs: + Tensor, return x directly, PrintShapeType does not affect the forward and gradient result. + + Examples: + >>> class PrintShapeType(nn.Cell): + >>> def __init__(self): + >>> super(PrintShapeType, self).__init__() + >>> def construct(self, msg, x): + >>> P.PrintShapeType(msg)(x) + >>> return x + >>> + >>> class PrintShapeTypeGrad(nn.Cell): + >>> def __init__(self, msg): + >>> super(PrintShapeTypeGrad, self).__init__() + >>> self.print_shape_type = P.InsertGradientOf(P.PrintShapeType(msg)) + >>> def construct(self, x): + >>> self.print_shape_type(x) + >>> return x + """ + + @prim_attr_register + def __init__(self, msg): + super(PrintShapeType, self).__init__('PrintShapeType') + self.msg = msg + + def __call__(self, x): + logger.info('%s, data: %s', self.msg, x) + return x + + def infer_shape(self, x_shape): + logger.info('%s, shape: %s', self.msg, x_shape) + return x_shape + + def infer_dtype(self, x_type): + logger.info('%s, type: %s', self.msg, x_type) + return x_type + + +@bprop_getters.register(PrintShapeType) +def get_bprop_print_shape_type(self): + """Generate bprop for PrintShapeType""" + def bprop(x, out, dout): + return (dout,) + return bprop + + +class PrintShapeTypeCell(nn.Cell): + def __init__(self): + super(PrintShapeTypeCell, self).__init__() + def construct(self, msg, x): + PrintShapeType(msg)(x) + return x + + +class PrintGradShapeTypeCell(nn.Cell): + def __init__(self, msg): + super(PrintGradShapeTypeCell, self).__init__() + self.print_shape_type = P.InsertGradientOf(PrintShapeType(msg)) + + def construct(self, x): + self.print_shape_type(x) + return x diff --git a/tests/mindspore_test_framework/utils/facade_util.py b/tests/mindspore_test_framework/utils/facade_util.py new file mode 100644 index 00000000000..bbbf1dd375e --- /dev/null +++ b/tests/mindspore_test_framework/utils/facade_util.py @@ -0,0 +1,99 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for facade components.""" + +from . import keyword +from .config_util import get_function_config + +def get_block_config(): + """ + Get Empty function config. + """ + ret = {} + ret[keyword.function] = [] + ret[keyword.inputs] = [] + ret[keyword.expect] = [] + return ret + +def fill_block_config(ret, block_config, tid, group, desc_inputs, desc_bprop, expect, + desc_const, const_first, add_fake_input, fake_input_type): + """ + Fill in block config. + + Args: + ret (dict): The filled config. + block_config (tuple): Block config. + tid (str): Testing id. + group (str): Testing group. + desc_inputs (list): Inputs Description. + desc_bprop (list): Backpropagation description. + expect (list): Expectataion. + desc_const (list): Const as inputs. + const_first (bool): Const as first inputs. + add_fake_input (bool): Add fake input. + fake_input_type (numpy type): Type of faked input. + + Returns: + """ + func_list = ret[keyword.function] + inputs_list = ret[keyword.inputs] + expect_list = ret[keyword.expect] + + block = block_config + delta, max_error, input_selector, output_selector, \ + sampling_times, reduce_output, init_param_with, split_outputs, exception = get_function_config({}) + if isinstance(block_config, tuple) and isinstance(block_config[-1], dict): + block = block_config[0] + delta, max_error, input_selector, output_selector, \ + sampling_times, reduce_output, init_param_with, split_outputs, exception = get_function_config(block_config[-1]) + + if block: + func_list.append({ + keyword.id: tid, + keyword.group: group, + keyword.block: block, + keyword.delta: delta, + keyword.max_error: max_error, + keyword.input_selector: input_selector, + keyword.output_selector: output_selector, + keyword.sampling_times: sampling_times, + keyword.reduce_output: reduce_output, + keyword.num_inputs: len(desc_inputs), + keyword.num_outputs: len(desc_bprop), + keyword.init_param_with: init_param_with, + keyword.desc_const: desc_const, + keyword.const_first: const_first, + keyword.add_fake_input: add_fake_input, + keyword.split_outputs: split_outputs, + keyword.exception: exception + }) + + if desc_inputs or desc_const: + inputs_list.append({ + keyword.id: tid, + keyword.group: group, + keyword.desc_inputs: desc_inputs, + keyword.desc_bprop: desc_bprop, + keyword.add_fake_input: add_fake_input, + keyword.fake_input_type: fake_input_type + }) + + if expect: + expect_list.append({ + keyword.id: tid+'-'+tid, + keyword.group: group+'-'+group, + keyword.desc_expect: expect + }) diff --git a/tests/mindspore_test_framework/utils/keyword.py b/tests/mindspore_test_framework/utils/keyword.py new file mode 100644 index 00000000000..16618be70a4 --- /dev/null +++ b/tests/mindspore_test_framework/utils/keyword.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Keywords for verification config.""" + +import sys + +class _MindsporeTestFrameworkkeyword: + def __setattr__(self, name, value): + if name in self.__dict__: + raise TypeError("can not rebind keyword (%s)" % name) + self.__dict__[name] = value +keyword = _MindsporeTestFrameworkkeyword() + +keyword.function = "function" +keyword.inputs = "inputs" +keyword.expect = "expect" +keyword.ext = "ext" + +keyword.id = "id" +keyword.group = "group" + +keyword.desc_inputs = "desc_inputs" +keyword.desc_bprop = "desc_bprop" +keyword.desc_expect = "desc_expect" +keyword.block = "block" +keyword.split_outputs = "split_outputs" + +keyword.compare_with = "compare_with" +keyword.compare_gradient_with = "compare_gradient_with" + +keyword.max_error = "max_error" +keyword.check_tolerance = "check_tolerance" +keyword.relative_tolerance = "relative_tolerance" +keyword.absolute_tolerance = "absolute_tolerance" + +keyword.sampling_times = "sampling_times" +keyword.shape_type = "shape_type" +keyword.model = "model" +keyword.loss = "loss" +keyword.opt = "opt" +keyword.num_epochs = "num_epochs" +keyword.loss_upper_bound = "loss_upper_bound" +keyword.true_params = "true_params" +keyword.num_samples = "num_samples" +keyword.batch_size = "batch_size" +keyword.dtype = "dtype" +keyword.scale = "scale" +keyword.num_inputs = "num_inputs" +keyword.num_outputs = "num_outputs" +keyword.delta = "delta" +keyword.input_selector = "input_selector" +keyword.output_selector = "output_selector" +keyword.result = "result" +keyword.shape = "shape" +keyword.type = "type" +keyword.reduce_output = "reduce_output" +keyword.init_param_with = "init_param_with" +keyword.desc_const = "desc_const" +keyword.const_first = "const_first" +keyword.add_fake_input = "add_fake_input" +keyword.fake_input_type = "fake_input_type" +keyword.exception = "exception" + +sys.modules[__name__] = keyword diff --git a/tests/mindspore_test_framework/utils/model_util.py b/tests/mindspore_test_framework/utils/model_util.py new file mode 100644 index 00000000000..61a22f27cbe --- /dev/null +++ b/tests/mindspore_test_framework/utils/model_util.py @@ -0,0 +1,102 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for model verification.""" + +# pylint: disable=arguments-differ + +import numpy as np +from mindspore import Parameter, ParameterTuple, Tensor +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C + +class SquaredLoss(nn.Cell): + """Squared loss function.""" + def __init__(self): + super(SquaredLoss, self).__init__() + self.reshape = P.Reshape() + self.shape = P.Shape() + self.two = Tensor(np.array([2.0]).astype(np.float32)) + self.reduce_sum = P.ReduceSum() + + def construct(self, y_hat, y): + ret = y_hat - self.reshape(y, self.shape(y_hat)) + return self.reduce_sum((ret * ret) / self.two, (0,)) + +opt_step = C.MultitypeFuncGraph("opt_step") +@opt_step.register("Tensor", "Tensor", + "Tensor", "Tensor") +def update_opt_step(learning_rate, batch_size, parameter, gradient): + """ + Update opt step. + + Args: + learning_rate (Tensor): Learning rate. + batch_size (Tensor): Batch Size. + parameter (Tensor): Parameter. + gradient (Tensor): Gradients. + + Returns: + """ + next_param = parameter - learning_rate * gradient / batch_size + F.assign(parameter, next_param) + return next_param + +class SGD(nn.Cell): + """SGD optimizer.""" + def __init__(self, parameters, learning_rate=0.001, batch_size=1): + super(SGD, self).__init__() + self.parameters = ParameterTuple(parameters) + self.learning_rate = Tensor(np.array([learning_rate]).astype(np.float32)) + self.batch_size = Tensor(np.array([batch_size]).astype(np.float32)) + self.hyper_map = C.HyperMap() + + def set_params(self, parameters): + self.parameters = parameters + + def construct(self, gradients): + success = self.hyper_map(F.partial(opt_step, self.learning_rate, self.batch_size), + self.parameters, gradients) + return success + +class Linreg(nn.Cell): + """Linear regression model.""" + def __init__(self, num_features): + super(Linreg, self).__init__() + self.matmul = P.MatMul() + self.w = Parameter(Tensor(np.random.normal(scale=0.01, size=(num_features, 1)).astype(np.float32)), name='w') + self.b = Parameter(Tensor(np.zeros(shape=(1,)).astype(np.float32)), name='b') + + def construct(self, x): + return self.matmul(x, self.w) + self.b + +class Model: + """Simplified model.""" + def __init__(self, network, loss_fn, optimizer): + self.optimizer = optimizer + self.step = nn.TrainOneStepCell(nn.WithLossCell(network, loss_fn), self.optimizer) + + def optimize(self, data, label): + return self.step(data, label) + + def train(self, num_epochs, train_dataset): + train_dataset = list(train_dataset) + for epoch in range(num_epochs): + for x, y in train_dataset: + loss = self.optimize(x, y) + print('epoch %d, loss %f' % (epoch + 1, loss.asnumpy().mean())) + return loss diff --git a/tests/mindspore_test_framework/utils/npy_util.py b/tests/mindspore_test_framework/utils/npy_util.py new file mode 100644 index 00000000000..d44c562f6d9 --- /dev/null +++ b/tests/mindspore_test_framework/utils/npy_util.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for npy file operations.""" + +import numpy as np + +from mindspore.common.tensor import Tensor + +from .other_util import shape2tensor +from .config_util import get_expect_config + +def load_npy(p): + s, dtype, scale, max_error, check_tolerance, relative_tolerance, absolute_tolerance = get_expect_config(p) + if isinstance(s, str): + try: + ret = Tensor(np.array((np.load(s, allow_pickle=True) * scale).astype(dtype))) + except ValueError: + ret = Tensor(np.array((np.load(s, allow_pickle=False) * scale).astype(dtype))) + else: + ret = shape2tensor(s, dtype, scale) + return ret, max_error, check_tolerance, relative_tolerance, absolute_tolerance + +def load_data_from_npy_or_shape(dpaths, skip_expect_config=True): + ret = [] + for p in dpaths: + d, max_error, check_tolerance, relative_tolerance, absolute_tolerance = load_npy(p) + if skip_expect_config: + ret.append(d) + else: + ret.append((d, max_error, check_tolerance, relative_tolerance, absolute_tolerance)) + return ret diff --git a/tests/mindspore_test_framework/utils/other_util.py b/tests/mindspore_test_framework/utils/other_util.py new file mode 100644 index 00000000000..7ded357c02d --- /dev/null +++ b/tests/mindspore_test_framework/utils/other_util.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Other utils.""" + +import numpy as np + +import mindspore._c_expression as _c_expression +from mindspore.common.tensor import Tensor + +def wrap(x): + if isinstance(x, (tuple, list)): + return x + return (x,) + +def to_numpy_list(tl): + tl = wrap(tl) + ret = [] + for x in tl: + if isinstance(x, (Tensor, _c_expression.Tensor)): + ret.append(x.asnumpy()) + else: + ret.append(x) + return ret + +def to_numpy(x): + if isinstance(x, (Tensor, _c_expression.Tensor)): + return x.asnumpy() + return x + +def shape2tensor(shp, dtype=np.float32, scale=6): + if isinstance(shp, list): + if not shp: + return Tensor((np.random.rand() * scale).astype(dtype)) + return Tensor((np.random.rand(*shp) * scale).astype(dtype)) + return shp + +def select_from_config_tuple(t, index, default): + if not isinstance(t, tuple): + return default + if not isinstance(t[-1], dict): + return default + if index > len(t)-1: + return default + return t[index] diff --git a/tests/mindspore_test_framework/utils/verifier_util.py b/tests/mindspore_test_framework/utils/verifier_util.py new file mode 100644 index 00000000000..2084b1e7911 --- /dev/null +++ b/tests/mindspore_test_framework/utils/verifier_util.py @@ -0,0 +1,58 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Utils for verifier.""" + +import numpy as np + +def count_unequal_element(expect, result, rtol, atol): + """ + Count unequal element. + + Args: + expect (numpy ndarray): Expected result. + result (numpy ndarray): Actual result. + rtol (float): Relative tolerance. + atol (float): Absolute tolerance. + + Returns: + """ + if expect.shape != result.shape: + raise ValueError(f'expect.shape {expect.shape}, result.shape {result.shape}') + total_count = len(expect.flatten()) + error = np.abs(expect - result) + count = np.count_nonzero(np.less_equal(error, atol + np.abs(result)*rtol)) + if ((total_count-count)/total_count) >= rtol: + raise ValueError(f'expect {expect}, but got {result}, ' + f'{total_count-count} / {total_count} elements out of tolerance, ' + f'absolute_tolerance {atol}, relative_tolerance {rtol}') + print(f'expect {expect}, got {result}, ' + f'{total_count-count} / {total_count} elements out of tolerance, ' + f'absolute_tolerance {atol}, relative_tolerance {rtol}') + +def tolerance_assert(expect, result, rtol, atol): + """ + Verify if results are in expected tolerance. + + Args: + expect (numpy ndarray): Expected result. + result (numpy ndarray): Actual result. + rtol (float): Relative tolerance. + atol (float): Absolute tolerance. + + Returns: + """ + if not np.allclose(expect, result, rtol, atol): + count_unequal_element(expect, result, rtol, atol) diff --git a/tests/ops_common.py b/tests/ops_common.py new file mode 100644 index 00000000000..e9932664969 --- /dev/null +++ b/tests/ops_common.py @@ -0,0 +1,394 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test ops """ +import numpy as np +import mindspore.nn as nn +import mindspore.ops.composite as C +import mindspore.ops.functional as F +import mindspore.ops.operations as P +from mindspore import Tensor +from mindspore.common.api import _executor + +class InputBackward(nn.Cell): + """ InputBackward definition """ + def __init__(self, network, c1=None, c2=None): + super(InputBackward, self).__init__() + self.network = network + self.network.set_train() + self.grad = C.grad_all_with_sens + self.c1 = c1 + self.c2 = c2 + + def construct(self, *inputs): + pass + + def construct1(self, x1, sens): + return self.grad(self.network)(x1, sens) + + def construct2(self, x1, x2, sens): + return self.grad(self.network)(x1, x2, sens) + + def construct3(self, x1, x2, x3, sens): + return self.grad(self.network)(x1, x2, x3, sens) + + def construct4(self, x1, x2, x3, x4, sens): + return self.grad(self.network)(x1, x2, x3, x4, sens) + + def construct5(self, x1, x2, x3, x4, x5, sens): + return self.grad(self.network)(x1, x2, x3, x4, x5, sens) + + def construct6(self, x1, x2, x3, x4, x5, x6, sens): + return self.grad(self.network)(x1, x2, x3, x4, x5, x6, sens) + + def construct7(self, x1, x2, x3, x4, x5, x6, x7, sens): + return self.grad(self.network)(x1, x2, x3, x4, x5, x6, x7, sens) + + +class InputOpNet(nn.Cell): + """ InputOpNet definition """ + def __init__(self, op, get_first=False, + c1=None, c2=None, c3=None, c4=None): + super(InputOpNet, self).__init__() + self.op = op + self.get_first = get_first + self.c1 = c1 + self.c2 = c2 + self.c3 = c3 + self.c4 = c4 + + def construct(self, *inputs): + pass + + def construct0_c0_fack(self, data): + x = self.op() + data + if self.get_first: + x = x[0] + return x + def construct0_c1_fack(self, data): + x = self.op(self.c1) + data + if self.get_first: + x = x[0] + return x + + def construct0_c2_fack(self, data): + x = self.op(self.c1, self.c2) + data + if self.get_first: + x = x[0] + return x + + def construct0_c0(self): + x = self.op() + if self.get_first: + x = x[0] + return x + + def construct0_c1(self): + x = self.op(self.c1) + if self.get_first: + x = x[0] + return x + + def construct0_c2(self): + x = self.op(self.c1, self.c2) + if self.get_first: + x = x[0] + return x + + def construct1_c0(self, x1): + x = self.op(x1) + if self.get_first: + x = x[0] + return x + + def construct1_c1(self, x1): + x = self.op(x1, self.c1) + if self.get_first: + x = x[0] + return x + + def construct1_c2(self, x1): + x = self.op(x1, self.c1, self.c2) + if self.get_first: + x = x[0] + return x + + def construct1_c3(self, x1): + x = self.op(x1, self.c1, self.c2, self.c3) + if self.get_first: + x = x[0] + return x + + def construct1_c4(self, x1): + x = self.op(x1, self.c1, self.c2, self.c3, self.c4) + if self.get_first: + x = x[0] + return x + + def constructc1_1(self, x1): + x = self.op(self.c1, x1) + if self.get_first: + x = x[0] + return x + + def construct2_c0(self, x1, x2): + x = self.op(x1, x2) + if self.get_first: + x = x[0] + return x + + + def construct2_c1(self, x1, x2): + x = self.op(x1, x2, self.c1) + if self.get_first: + x = x[0] + return x + + def construct2_c3(self, x1, x2): + x = self.op(x1, x2, self.c1, self.c2, self.c3) + if self.get_first: + x = x[0] + return x + + def construct3_c0(self, x1, x2, x3): + x = self.op(x1, x2, x3) + if self.get_first: + x = x[0] + return x + + def construct3_c1(self, x1, x2, x3): + x = self.op(x1, x2, x3, self.c1) + if self.get_first: + x = x[0] + return x + + def construct4_c0(self, x1, x2, x3, x4): + x = self.op(x1, x2, x3, x4) + if self.get_first: + x = x[0] + return x + + def construct4_c1(self, x1, x2, x3, x4): + x = self.op(x1, x2, x3, x4, self.c1) + if self.get_first: + x = x[0] + return x + + def construct5_c0(self, x1, x2, x3, x4, x5): + x = self.op(x1, x2, x3, x4, x5) + if self.get_first: + x = x[0] + return x + + def construct6_c0(self, x1, x2, x3, x4, x5, x6): + x = self.op(x1, x2, x3, x4, x5, x6) + if self.get_first: + x = x[0] + return x + + def construct5_c1(self, x1, x2, x3, x4, x5): + x = self.op(x1, x2, x3, x4, x5, self.c1) + if self.get_first: + x = x[0] + return x + +class NetOutputAsLoss(nn.Cell): + """ NetOutputAsLoss definition """ + def __init__(self, network, output_index): + super(NetOutputAsLoss, self).__init__() + self.network = network + self.output_index = output_index + + def construct(self, *inputs): + pass + + def construct1(self, x1): + predict = self.network(x1)[self.output_index] + return predict + + def construct2(self, x1, x2): + predict = self.network(x1, x2)[self.output_index] + return predict + + def construct3(self, x1, x2, x3): + predict = self.network(x1, x2, x3)[self.output_index] + return predict + + def construct4(self, x1, x2, x3, x4): + predict = self.network(x1, x2, x3, x4)[self.output_index] + return predict + + def construct5(self, x1, x2, x3, x4, x5): + predict = self.network(x1, x2, x3, x4, x5)[self.output_index] + return predict + +def get_loss_fun(construct_net, num_input, output_index): + net = NetOutputAsLoss(construct_net, output_index) + f = getattr(net, 'construct%d' % num_input) + setattr(net, "construct", f) + return net + +def build_construct_graph(net, *inputs, execute=True): + net.set_train() + _executor.compile(net, *inputs) + if execute: + _executor(net, inputs) + +def build_backward_graph(net, output_shapes, inputs, execute=True): + inputs = append_sens_to_inputs(output_shapes, inputs) + net = gen_backward_net(net, len(inputs) - 1) + net.set_train() + _executor.compile(net, inputs) + if execute: + _executor(net, inputs) + +def convert(shp, dtype=np.float32, scale=6): + if isinstance(shp, list): + if not shp: + return Tensor((np.random.rand() * scale).astype(dtype)) + return Tensor((np.random.rand(*shp) * scale).astype(dtype)) + return shp + +def gen_inputs(input_shapes, config): + add_fack_input = config.get('add_fack_input', False) + if not input_shapes and add_fack_input: + return [Tensor(np.array([1.0]).astype(config.get('fack_input_type', np.float32)))] + return [convert(shp) for shp in input_shapes] + +def gen_backward_inputs(input_shapes, output_shapes, config): + add_fack_input = config.get('add_fack_input', False) + if not input_shapes and add_fack_input: + inputs = [Tensor(np.array([1.0]))] + else: + inputs = [convert(shp) for shp in input_shapes] + sens_shape = output_shapes[0] + sens = convert(sens_shape) + return inputs + [sens] + +def append_sens_to_inputs(output_shapes, inputs): + inputs = inputs + sens = Tensor(np.random.normal(0, 1, output_shapes).astype(np.float32)) + return inputs + [sens] + +def gen_net(shapes, config, get_first=False): + """ + gen_net function + """ + add_fack_input = config.get('add_fack_input', False) + op = config['op'] + if 'const' not in config: + const_input = [] + else: + const_input = config['const'] + const_first = False + if 'const_first' in config: + const_first = config['const_first'] + + net = InputOpNet(op, get_first, *const_input) + if const_first: + fn_name = 'constructc%d_%d' % (len(const_input), len(shapes)) + else: + fn_name = 'construct%d_c%d' % (len(shapes), len(const_input)) + if add_fack_input: + fn_name += '_fack' + f = getattr(net, fn_name) + setattr(net, "construct", f) + return net + + +def gen_backward_net(construct_net, input_num): + net = InputBackward(construct_net) + f = getattr(net, 'construct%d' % input_num) + setattr(net, "construct", f) + return net + +def batch_tuple_tensor(data, batch_size): + ret = [Tensor(np.tile(d.asnumpy(), (batch_size, 1))) for d in data] + return tuple(ret) + +class OutPutWrap(nn.Cell): + """ + OutPutWrap definition + """ + def __init__(self, network, num_output, output_is_tuple): + super(OutPutWrap, self).__init__() + self.network = network + self.num_output = num_output + self.one = Tensor(np.array([1])) + self.dtype = P.DType() + self.cast = P.Cast() + self.output_is_tuple = output_is_tuple + + def construct(self, *inputs): + pass + + def construct1(self, x1): + ret = F.make_tuple() + predict = self.network(x1) + if self.num_output == 1 and self.output_is_tuple == 0: + return predict * self.cast(self.one, self.dtype(predict)) + for i in range(self.num_output): + ret = ret + F.make_tuple(predict[i] * self.cast(self.one, self.dtype(predict[i]))) + return ret + + def construct2(self, x1, x2): + ret = F.make_tuple() + predict = self.network(x1, x2) + if self.num_output == 1 and self.output_is_tuple == 0: + return predict * self.cast(self.one, self.dtype(predict)) + for i in range(self.num_output): + ret = ret + F.make_tuple(predict[i] * self.cast(self.one, self.dtype(predict[i]))) + return ret + + def construct3(self, x1, x2, x3): + ret = F.make_tuple() + predict = self.network(x1, x2, x3) + if self.num_output == 1 and self.output_is_tuple == 0: + return predict * self.cast(self.one, self.dtype(predict)) + for i in range(self.num_output): + ret = ret + F.make_tuple(predict[i] * self.cast(self.one, self.dtype(predict[i]))) + return ret + + def construct4(self, x1, x2, x3, x4): + ret = F.make_tuple() + predict = self.network(x1, x2, x3, x4) + if self.num_output == 1 and self.output_is_tuple == 0: + return predict * self.cast(self.one, self.dtype(predict)) + for i in range(self.num_output): + ret = ret + F.make_tuple(predict[i] * self.cast(self.one, self.dtype(predict[i]))) + return ret + + def construct5(self, x1, x2, x3, x4, x5): + ret = F.make_tuple() + predict = self.network(x1, x2, x3, x4, x5) + if self.num_output == 1 and self.output_is_tuple == 0: + return predict * self.cast(self.one, self.dtype(predict)) + for i in range(self.num_output): + ret = ret + F.make_tuple(predict[i] * self.cast(self.one, self.dtype(predict[i]))) + return ret + + def construct6(self, x1, x2, x3, x4, x5, x6): + ret = F.make_tuple() + predict = self.network(x1, x2, x3, x4, x5, x6) + if self.num_output == 1 and self.output_is_tuple == 0: + return predict * self.cast(self.one, self.dtype(predict)) + for i in range(self.num_output): + ret = ret + F.make_tuple(predict[i] * self.cast(self.one, self.dtype(predict[i]))) + return ret + +def get_output_wrap(network, num_input, num_output, output_is_tuple=0): + net = OutPutWrap(network, num_output, output_is_tuple) + f = getattr(net, 'construct%d' % num_input) + setattr(net, "construct", f) + return net diff --git a/tests/perf_test/__init__.py b/tests/perf_test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/perf_test/bert/__init__.py b/tests/perf_test/bert/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/perf_test/bert/test_bert_train.py b/tests/perf_test/bert/test_bert_train.py new file mode 100644 index 00000000000..f5d6f0e32f7 --- /dev/null +++ b/tests/perf_test/bert/test_bert_train.py @@ -0,0 +1,207 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Bert test.""" + +# pylint: disable=missing-docstring, arguments-differ, W0612 + +import os +import mindspore.common.dtype as mstype +import mindspore.context as context +from mindspore import Tensor +from mindspore.nn.optim import AdamWeightDecayDynamicLR +from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertNetworkWithLoss, BertTrainOneStepCell, \ + BertTrainOneStepWithLossScaleCell +from mindspore.nn.wrap.loss_scale import FixedLossScaleUpdateCell +from mindspore.train.loss_scale_manager import DynamicLossScaleManager +from ...dataset_mock import MindData +from ...ops_common import nn, np, batch_tuple_tensor, build_construct_graph + +_current_dir = os.path.dirname(os.path.realpath(__file__)) + "/../python/test_data" +context.set_context(mode=context.GRAPH_MODE) + + +def get_dataset(batch_size=1): + dataset_types = (np.int32, np.int32, np.int32, np.int32, np.int32, np.int32, np.int32) + dataset_shapes = ((batch_size, 128), (batch_size, 128), (batch_size, 128), (batch_size, 1), \ + (batch_size, 20), (batch_size, 20), (batch_size, 20)) + + dataset = MindData(size=2, batch_size=batch_size, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset + + +def load_test_data(batch_size=1): + dataset = get_dataset(batch_size) + ret = dataset.next() + ret = batch_tuple_tensor(ret, batch_size) + return ret + + +def get_config(version='base', batch_size=1): + """ + get_config definition + """ + if version == 'base': + return BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21128, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + if version == 'large': + return BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21128, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + return BertConfig(batch_size=batch_size) + + +def test_bert_train(): + """ + the main function + """ + + class ModelBert(nn.Cell): + """ + ModelBert definition + """ + + def __init__(self, network, optimizer=None): + super(ModelBert, self).__init__() + self.optimizer = optimizer + self.train_network = BertTrainOneStepCell(network, self.optimizer) + self.train_network.set_train() + + def construct(self, arg0, arg1, arg2, arg3, arg4, arg5, arg6): + return self.train_network(arg0, arg1, arg2, arg3, arg4, arg5, arg6) + + version = os.getenv('VERSION', 'large') + batch_size = int(os.getenv('BATCH_SIZE', '1')) + inputs = load_test_data(batch_size) + + config = get_config(version=version, batch_size=batch_size) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = AdamWeightDecayDynamicLR(netwithloss.trainable_params(), 10) + net = ModelBert(netwithloss, optimizer=optimizer) + net.set_train() + build_construct_graph(net, *inputs, execute=False) + + +def test_bert_withlossscale_train(): + class ModelBert(nn.Cell): + def __init__(self, network, optimizer=None): + super(ModelBert, self).__init__() + self.optimizer = optimizer + self.train_network = BertTrainOneStepWithLossScaleCell(network, self.optimizer) + self.train_network.set_train() + + def construct(self, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7): + return self.train_network(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + + version = os.getenv('VERSION', 'base') + batch_size = int(os.getenv('BATCH_SIZE', '1')) + scaling_sens = Tensor(np.ones([1]).astype(np.float32)) + inputs = load_test_data(batch_size) + (scaling_sens,) + + config = get_config(version=version, batch_size=batch_size) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = AdamWeightDecayDynamicLR(netwithloss.trainable_params(), 10) + net = ModelBert(netwithloss, optimizer=optimizer) + net.set_train() + build_construct_graph(net, *inputs, execute=True) + + +def bert_withlossscale_manager_train(): + class ModelBert(nn.Cell): + def __init__(self, network, optimizer=None): + super(ModelBert, self).__init__() + self.optimizer = optimizer + manager = DynamicLossScaleManager() + update_cell = LossScaleUpdateCell(manager) + self.train_network = BertTrainOneStepWithLossScaleCell(network, self.optimizer, + scale_update_cell=update_cell) + self.train_network.set_train() + + def construct(self, arg0, arg1, arg2, arg3, arg4, arg5, arg6): + return self.train_network(arg0, arg1, arg2, arg3, arg4, arg5, arg6) + + version = os.getenv('VERSION', 'base') + batch_size = int(os.getenv('BATCH_SIZE', '1')) + inputs = load_test_data(batch_size) + + config = get_config(version=version, batch_size=batch_size) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = AdamWeightDecayDynamicLR(netwithloss.trainable_params(), 10) + net = ModelBert(netwithloss, optimizer=optimizer) + net.set_train() + build_construct_graph(net, *inputs, execute=True) + + +def bert_withlossscale_manager_train_feed(): + class ModelBert(nn.Cell): + def __init__(self, network, optimizer=None): + super(ModelBert, self).__init__() + self.optimizer = optimizer + manager = DynamicLossScaleManager() + update_cell = LossScaleUpdateCell(manager) + self.train_network = BertTrainOneStepWithLossScaleCell(network, self.optimizer, + scale_update_cell=update_cell) + self.train_network.set_train() + + def construct(self, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7): + return self.train_network(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + + version = os.getenv('VERSION', 'base') + batch_size = int(os.getenv('BATCH_SIZE', '1')) + scaling_sens = Tensor(np.ones([1]).astype(np.float32)) + inputs = load_test_data(batch_size) + (scaling_sens,) + + config = get_config(version=version, batch_size=batch_size) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = AdamWeightDecayDynamicLR(netwithloss.trainable_params(), 10) + net = ModelBert(netwithloss, optimizer=optimizer) + net.set_train() + build_construct_graph(net, *inputs, execute=True) diff --git a/tests/perf_test/resnet_example.py b/tests/perf_test/resnet_example.py new file mode 100644 index 00000000000..19d235c2b11 --- /dev/null +++ b/tests/perf_test/resnet_example.py @@ -0,0 +1,165 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Resnet examples.""" + +# pylint: disable=missing-docstring, arguments-differ + +import mindspore.nn as nn +from mindspore.ops import operations as P + + +def conv3x3(in_channels, out_channels, stride=1, padding=1, pad_mode='pad'): + """3x3 convolution """ + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, pad_mode=pad_mode) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0, pad_mode='pad'): + """1x1 convolution""" + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode) + + +class ResidualBlock(nn.Cell): + """ + residual Block + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = nn.BatchNorm2d(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=1) + self.bn2 = nn.BatchNorm2d(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.relu = nn.ReLU() + self.downsample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, + stride=stride, padding=0) + self.bn_down_sample = nn.BatchNorm2d(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + """ + :param x: + :return: + """ + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet50(nn.Cell): + """ + resnet nn.Cell + """ + + def __init__(self, block, num_classes=100): + super(ResNet50, self).__init__() + + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, pad_mode='pad') + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, pad_mode='valid') + + self.layer1 = self.MakeLayer( + block, 3, in_channels=64, out_channels=256, stride=1) + self.layer2 = self.MakeLayer( + block, 4, in_channels=256, out_channels=512, stride=2) + self.layer3 = self.MakeLayer( + block, 6, in_channels=512, out_channels=1024, stride=2) + self.layer4 = self.MakeLayer( + block, 3, in_channels=1024, out_channels=2048, stride=2) + + self.avgpool = nn.AvgPool2d(7, 1) + self.flatten = P.Flatten() + self.fc = nn.Dense(512 * block.expansion, num_classes) + + def MakeLayer(self, block, layer_num, in_channels, out_channels, stride): + """ + make block layer + :param block: + :param layer_num: + :param in_channels: + :param out_channels: + :param stride: + :return: + """ + layers = [] + resblk = block(in_channels, out_channels, + stride=stride, down_sample=True) + layers.append(resblk) + + for _ in range(1, layer_num): + resblk = block(out_channels, out_channels, stride=1) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + """ + :param x: + :return: + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.flatten(x) + x = self.fc(x) + + return x + + +def resnet50(): + return ResNet50(ResidualBlock, 10) diff --git a/tests/perf_test/test_lenet.py b/tests/perf_test/test_lenet.py new file mode 100644 index 00000000000..d071a653306 --- /dev/null +++ b/tests/perf_test/test_lenet.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LeNet test.""" + +import numpy as np + +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor +from mindspore.model_zoo.lenet import LeNet +from mindspore import context +import mindspore.ops.composite as C + +context.set_context(mode=context.GRAPH_MODE) + + +batch_size = 1 +channel = 1 +height = 32 +weight = 32 +num_class = 10 + + +class LeNetGrad(nn.Cell): + """Backward of LeNet""" + def __init__(self, network): + super(LeNetGrad, self).__init__() + self.grad_op = C.grad_all_with_sens + self.network = network + + def construct(self, x, sens): + grad_op = self.grad_op(self.network)(x, sens) + + return grad_op + + +def test_compile(): + """Compile forward graph""" + net = LeNet(num_class=num_class) + np.random.seed(7) + inp = Tensor(np.array(np.random.randn(batch_size, + channel, + height, + weight) * 3, np.float32)) + + _executor.compile(net, inp) + + +def test_compile_grad(): + """Compile forward and backward graph""" + net = LeNet(num_class=num_class) + inp = Tensor(np.array(np.random.randn(batch_size, + channel, + height, + weight) * 3, np.float32)) + sens = Tensor(np.ones([batch_size, num_class]).astype(np.float32)) + grad_op = LeNetGrad(net) + + _executor.compile(grad_op, inp, sens) diff --git a/tests/perf_test/test_resnet_infer.py b/tests/perf_test/test_resnet_infer.py new file mode 100644 index 00000000000..65fdad6e0d2 --- /dev/null +++ b/tests/perf_test/test_resnet_infer.py @@ -0,0 +1,27 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Resnet test.""" + +import numpy as np + +from mindspore.common.api import _executor +from mindspore import Tensor +from .resnet_example import resnet50 + +def test_compile(): + net = resnet50() + inp = Tensor(np.ones([1, 3, 224, 224]).astype(np.float32)) + _executor.compile(net, inp) diff --git a/tests/perf_test/test_resnet_pynative.py b/tests/perf_test/test_resnet_pynative.py new file mode 100644 index 00000000000..1b7262c1fb3 --- /dev/null +++ b/tests/perf_test/test_resnet_pynative.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Resnet test.""" + +# pylint: disable=unused-wildcard-import, wildcard-import + +import numpy as np + +from mindspore import Tensor +from ..train_step_wrap import train_step_without_opt +from .resnet_example import resnet50 +from ..vm_impl import * + +def test_resnet50_pynative(): + net = train_step_without_opt(resnet50()) + inp = Tensor(np.ones([1, 3, 224, 224]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net(inp, label) diff --git a/tests/perf_test/test_resnet_train.py b/tests/perf_test/test_resnet_train.py new file mode 100644 index 00000000000..e500d8f31c2 --- /dev/null +++ b/tests/perf_test/test_resnet_train.py @@ -0,0 +1,32 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Resnet test.""" + +import numpy as np + +from mindspore.common.api import _executor +import mindspore.context as context +from mindspore import Tensor +from ..train_step_wrap import train_step_with_loss_warp +from .resnet_example import resnet50 +context.set_context(mode=context.GRAPH_MODE) + +def test_train_step(): + net = train_step_with_loss_warp(resnet50()) + net.set_train() + inp = Tensor(np.ones([1, 3, 224, 224], np.float32)) + label = Tensor(np.zeros([1, 10], np.float32)) + _executor.compile(net, inp, label) diff --git a/tests/runtest.sh b/tests/runtest.sh new file mode 100755 index 00000000000..10c9b612922 --- /dev/null +++ b/tests/runtest.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +CURRPATH=$(cd $(dirname $0); pwd) + +if [ $# -gt 0 -a $1 == "-h" ]; then + echo "Usage: $0 [testcase_name]" + exit 0 +fi + +# 1.run python testcases +bash ${CURRPATH}/ut/python/runtest.sh $1 + +# 2.run c++ ut testcases +bash ${CURRPATH}/ut/cpp/runtest.sh diff --git a/tests/st/__init__.py b/tests/st/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/st/auto_parallel/__init__.py b/tests/st/auto_parallel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/st/auto_parallel/env.sh b/tests/st/auto_parallel/env.sh new file mode 100644 index 00000000000..681c466f43c --- /dev/null +++ b/tests/st/auto_parallel/env.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +LOCAL_HIAI=/usr/local/HiAI +export TBE_IMPL_PATH=${LOCAL_HIAI}/runtime/ops/op_impl/built-in/ai_core/tbe/impl/ +export LD_LIBRARY_PATH=${LOCAL_HIAI}/runtime/lib64/:${LD_LIBRARY_PATH} +export PATH=${LOCAL_HIAI}/runtime/ccec_compiler/bin/:${PATH} +export PYTHONPATH=${LOCAL_HIAI}/runtime/ops/op_impl/built-in/ai_core/tbe/:${PYTHONPATH} +export DEVICE_MEMORY_CAPACITY=1073741824000 +export NOT_FULLY_USE_DEVICES=off diff --git a/tests/st/auto_parallel/onehot_model_parallel.py b/tests/st/auto_parallel/onehot_model_parallel.py new file mode 100644 index 00000000000..b0e6f1eb912 --- /dev/null +++ b/tests/st/auto_parallel/onehot_model_parallel.py @@ -0,0 +1,156 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.context as context +import mindspore.communication.management as distributedTool + +device_num = 2 +device_id = int(os.getenv('DEVICE_ID')) +rank_id = 0 + +def setup_module(): + global device_num + global rank_id + np.random.seed(0) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + context.set_context(enable_hccl=True) + context.set_context(enable_task_sink=True, + device_id=device_id) + context.set_context(enable_ir_fusion=True) + context.set_context(enable_loop_sink=False) + distributedTool.init() + device_num = distributedTool.get_group_size() + rank_id = distributedTool.get_rank() + context.set_auto_parallel_context(device_num=device_num, + global_rank=rank_id) + +def teardown_module(): + distributedTool.release() + +class Onehot(Cell): + def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, strategy=None): + super(Onehot, self).__init__() + trans_stra = None + if strategy: + trans_stra = (strategy[0],) + self.onehot = P.OneHot().set_strategy(strategy=strategy) + self.depth = depth + self.on_value = Tensor(on_value, ms.float32) + self.off_value = Tensor(off_value, ms.float32) + self.transpose = P.Transpose().set_strategy(strategy=trans_stra) + self.sub = P.Sub().set_strategy(strategy=((1,1),(1,1))) + + def construct(self, input, indices): + x = self.onehot(indices, self.depth, self.on_value, self.off_value) + x = self.transpose(x, (1,0)) + x = self.sub(input, x) + return x + +class DataGenerator(): + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def generate_data(self, shape): + data = np.random.rand(*shape) + return data + + def input_data(self, shape): + data = (self.generate_data(shape)*2).astype(np.float32) + stra = [1]*len(shape) + stra[0] = device_num + datas = self.get_parallel_blocks(data, stra) + return Tensor(data), Tensor(datas[rank_id]) + + def label_data(self, shape, classes): + data = (self.generate_data(shape)*(classes-1)).astype(np.int32) + stra = [1]*len(shape) + stra[0] = device_num + datas = self.get_parallel_blocks(data, stra) + return Tensor(data),Tensor(datas[rank_id]) + +class OneHotFactory: + def __init__(self, batch_size, classes, on_value=1.0, off_value=0.0, axis=None, strategy=None): + dataGen = DataGenerator() + self.input_full, self.input_part = dataGen.input_data((classes, batch_size)) + self.label_full, self.label_part = dataGen.label_data((batch_size,),classes) + self.depth = classes + self.on_value = on_value + self.off_value = off_value + self.axis = axis + self.strategy = strategy + + def forward_mindspore_single_impl(self): + net = Onehot(axis=self.axis, + depth=self.depth, + on_value=self.on_value, + off_value=self.off_value) + out = net(self.input_full, self.label_full) + return out + + def forward_mindspore_parallel_impl(self): + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + net = Onehot(axis=self.axis, + depth=self.depth, + on_value=self.on_value, + off_value=self.off_value, strategy=self.strategy) + out = net.compile_and_run(self.input_full, self.label_full) + return out + + def forward_cmp(self): + out_mindspore_single = self.forward_mindspore_single_impl().asnumpy() + context.reset_auto_parallel_context() + out_mindspore_parallel = self.forward_mindspore_parallel_impl().asnumpy() + context.reset_auto_parallel_context() + assert np.allclose(out_mindspore_single, out_mindspore_parallel, 0.0001, 0.0001) + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_reid_onehot_forward_int32_128_depth1024_model_parallel(): + fact = OneHotFactory(batch_size=128, + classes=1024, + on_value=1.000000, + off_value=0.000000, + axis=-1, + strategy=((1,device_num),(),())) + fact.forward_cmp() + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_reid_onehot_forward_int32_1024_depth128_model_parallel(): + fact = OneHotFactory(batch_size=1024, + classes=128, + on_value=1.000000, + off_value=0.000000, + axis=-1, + strategy=((1,device_num),(),())) + fact.forward_cmp() + diff --git a/tests/st/auto_parallel/run_auto_parallel_loss_expand.sh b/tests/st/auto_parallel/run_auto_parallel_loss_expand.sh new file mode 100644 index 00000000000..83acc6ad547 --- /dev/null +++ b/tests/st/auto_parallel/run_auto_parallel_loss_expand.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +set -e +BASE_PATH=$(cd "$(dirname $0)"; pwd) +CONFIG_PATH=/home/workspace/mindspore_config +export DEVICE_NUM=8 +export RANK_SIZE=$DEVICE_NUM +ulimit -n 65535 +source ${BASE_PATH}/env.sh +unset SLOG_PRINT_TO_STDOUT +export MINDSPORE_HCCL_CONFIG_PATH=$CONFIG_PATH/hccl/rank_table_${DEVICE_NUM}p.json + +process_pid=() +for((i=0; i<$DEVICE_NUM; i++)); do + rm -rf ${BASE_PATH}/loss_expand${i} + mkdir ${BASE_PATH}/loss_expand${i} + cp -r soft_entropy_loss_expand_parallel.py ${BASE_PATH}/loss_expand${i}/ + cd ${BASE_PATH}/loss_expand${i} + export RANK_ID=${i} + export DEVICE_ID=${i} + echo "start training for device $i" + env > env$i.log + pytest -s -v soft_entropy_loss_expand_parallel.py > test_soft_entropy_loss_expand_all_parallel_8p_log$i.log 2>&1 & + process_pid[${i}]=`echo $!` +done + +for((i=0; i<${DEVICE_NUM}; i++)); do + wait ${process_pid[i]} + status=`echo $?` + if [ "${status}" != "0" ]; then + echo "[ERROR] test_soft_entropy_loss_expand_parallel failed. status: ${status}" + exit 1 + else + echo "[INFO] test_soft_entropy_loss_expand_parallel success." + fi +done + +exit 0 diff --git a/tests/st/auto_parallel/run_auto_parallel_resnet50_expand_loss.sh b/tests/st/auto_parallel/run_auto_parallel_resnet50_expand_loss.sh new file mode 100644 index 00000000000..efc1e0c0de2 --- /dev/null +++ b/tests/st/auto_parallel/run_auto_parallel_resnet50_expand_loss.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +set -e +BASE_PATH=$(cd "$(dirname $0)"; pwd) +CONFIG_PATH=/home/workspace/mindspore_config +export DEVICE_NUM=8 +export RANK_SIZE=$DEVICE_NUM +ulimit -n 65535 +source ${BASE_PATH}/env.sh +unset SLOG_PRINT_TO_STDOUT +export MINDSPORE_HCCL_CONFIG_PATH=$CONFIG_PATH/hccl/rank_table_${DEVICE_NUM}p.json + +process_pid=() +for((i=0; i<$DEVICE_NUM; i++)); do + rm -rf ${BASE_PATH}/resnet50_expand_loss${i} + mkdir ${BASE_PATH}/resnet50_expand_loss${i} + cp -r resnet50_expand_loss.py ${BASE_PATH}/resnet50_expand_loss${i}/ + cd ${BASE_PATH}/resnet50_expand_loss${i} + export RANK_ID=${i} + export DEVICE_ID=${i} + echo "start training for device $i" + env > env$i.log + pytest -s -v resnet50_expand_loss.py > resnet50_expand_loss_log$i.log 2>&1 & + process_pid[${i}]=`echo $!` +done + +for((i=0; i<${DEVICE_NUM}; i++)); do + wait ${process_pid[i]} + status=`echo $?` + if [ "${status}" != "0" ]; then + echo "[ERROR] test_resnet_expand_loss failed. status: ${status}" + exit 1 + else + echo "[INFO] test_resnet_expand_loss success." + fi +done + +exit 0 diff --git a/tests/st/auto_parallel/run_onehot_model_parallel.sh b/tests/st/auto_parallel/run_onehot_model_parallel.sh new file mode 100644 index 00000000000..93229aab4e6 --- /dev/null +++ b/tests/st/auto_parallel/run_onehot_model_parallel.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +set -e +BASE_PATH=$(cd "$(dirname $0)"; pwd) +CONFIG_PATH=/home/workspace/mindspore_config +export DEVICE_NUM=8 +export RANK_SIZE=$DEVICE_NUM +ulimit -n 65535 +source ${BASE_PATH}/env.sh +unset SLOG_PRINT_TO_STDOUT +export MINDSPORE_HCCL_CONFIG_PATH=$CONFIG_PATH/hccl/rank_table_${DEVICE_NUM}p.json + +process_pid=() +for((i=0; i<$DEVICE_NUM; i++)); do + rm -rf ${BASE_PATH}/onehot_model_parallel${i} + mkdir ${BASE_PATH}/onehot_model_parallel${i} + cp -r onehot_model_parallel.py ${BASE_PATH}/onehot_model_parallel${i}/ + cd ${BASE_PATH}/onehot_model_parallel${i} + export RANK_ID=${i} + export DEVICE_ID=${i} + echo "start training for device $i" + env > env$i.log + pytest -s -v onehot_model_parallel.py > onehot_model_parallel_log$i.log 2>&1 & + process_pid[${i}]=`echo $!` +done + +for((i=0; i<${DEVICE_NUM}; i++)); do + wait ${process_pid[i]} + status=`echo $?` + if [ "${status}" != "0" ]; then + echo "[ERROR] test_onehot_model_parallel failed. status: ${status}" + exit 1 + else + echo "[INFO] test_onehot_model_parallel success." + fi +done + +exit 0 diff --git a/tests/st/auto_parallel/soft_entropy_loss_expand_parallel.py b/tests/st/auto_parallel/soft_entropy_loss_expand_parallel.py new file mode 100644 index 00000000000..081c6bd6478 --- /dev/null +++ b/tests/st/auto_parallel/soft_entropy_loss_expand_parallel.py @@ -0,0 +1,276 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +from numpy import allclose +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.common.parameter import ParameterTuple, Parameter +from mindspore.ops import composite as C +from mindspore.common import dtype as mstype +from mindspore.train import Model, ParallelMode +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.callback import Callback + +np.set_printoptions(threshold=np.inf) +device_num=2 +device_id = int(os.getenv('DEVICE_ID')) +rank_id = 0 +embed = 128 +classes = 32 +batch_size = 32*2 +MatmulParamShape = (classes, embed) + +def setup_module(): + global device_num + global rank_id + np.random.seed(0) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + context.set_context(enable_hccl=True) + context.set_context(enable_task_sink=True, + device_id=device_id) + context.set_context(enable_ir_fusion=True) + context.set_context(enable_loop_sink=False) + distributedTool.init() + rank_id = distributedTool.get_rank() + device_num = distributedTool.get_group_size() + context.set_auto_parallel_context(device_num=device_num, + global_rank=device_id) + +def teardown_module(): + distributedTool.release() + +class DataGenerator(): + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def generate_data(self, shape): + size = np.cumprod(shape)[-1] + num_range = min(size, 1000) + data = (np.arange(0, size)%num_range)/num_range + data = np.reshape(data, shape) + return data + + def input_data(self, shape): + data = (self.generate_data(shape)*0.1).astype(np.float32) + stra = [1]*len(shape) + stra[0] = device_num + datas = self.get_parallel_blocks(data, stra) + return Tensor(data), Tensor(datas[rank_id]) + + def label_data(self, shape, embed): + data = (self.generate_data(shape)*(embed-1)).astype(np.int32) + stra = [1]*len(shape) + stra[0] = device_num + datas = self.get_parallel_blocks(data, stra) + return Tensor(data),Tensor(datas[rank_id]) + +class Dataset(): + def __init__(self, predict, label, length=1, input_num=2): + self.predict = predict + self.label = label + self.index = 0 + self.length = length + self.input_num = input_num + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + if self.input_num == 2: + return self.predict, self.label + else: + return self.predict, + + def reset(self): + self.index = 0 + + def get_dataset_size(self): + return self.length + +class ModelCallback(Callback): + def __init__(self): + super(ModelCallback, self).__init__() + self.loss_list = [] + def epoch_end(self, run_context, *args): + cb_params = run_context.original_args() + result = cb_params.net_outputs + self.loss_list.append(result.asnumpy().mean()) + +class SoftmaxCrossEntropyExpand(Cell): + def __init__(self, sparse=False, stra_list=[]): + super(SoftmaxCrossEntropyExpand, self).__init__() + if len(stra_list) < 11: + stra_list = [None]*11 + self.exp = P.Exp() + self.reduce_sum = P.ReduceSum(keep_dims=True).set_strategy(strategy=stra_list[1]) + self.onehot = P.OneHot().set_strategy(strategy=stra_list[2]) + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.div = P.Div().set_strategy(strategy=stra_list[3]) + self.log = P.Log().set_strategy(strategy=stra_list[4]) + self.sum_cross_entropy = P.ReduceSum(keep_dims=False).set_strategy(strategy=stra_list[5]) + self.mul = P.Mul().set_strategy(strategy=stra_list[6]) + self.mul2 = P.Mul().set_strategy(strategy=stra_list[7]) + self.cast = P.Cast() + self.reduce_mean = P.ReduceMean(keep_dims=False).set_strategy(strategy=stra_list[8]) + self.sparse = sparse + self.reduce_max = P.ReduceMax(keep_dims=True).set_strategy(strategy=stra_list[9]) + self.sub = P.Sub().set_strategy(strategy=stra_list[10]) + + def construct(self, logit, label): + logit_max = self.reduce_max(logit, -1) + exp = self.exp(self.sub(logit, logit_max)) + exp_sum = self.reduce_sum(exp, -1) + softmax_result = self.div(exp, exp_sum) + if self.sparse: + label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + softmax_result_log = self.log(softmax_result) + loss = self.sum_cross_entropy((self.mul(softmax_result_log, label)), -1) + loss = self.mul2(F.scalar_to_array(-1.0), loss) + loss = self.reduce_mean(loss, -1) + return loss + +class MatmulNet(Cell): + def __init__(self, matmul_stra = None, loss_stra_list=[]): + super(MatmulNet, self).__init__() + self.matmul = P.MatMul(transpose_b=True).set_strategy(strategy=matmul_stra) + self.loss = SoftmaxCrossEntropyExpand(sparse=True, stra_list=loss_stra_list) + self.weight = Parameter(Tensor(np.ones(MatmulParamShape), dtype=ms.float32), name="weight") + def construct(self, x, label): + loss_input = self.matmul(x, self.weight) + out = self.loss(loss_input, label) + return out + +class LossFactory(): + def __init__(self): + dataGen = DataGenerator() + self.input_full, self.input_part = dataGen.input_data((batch_size, embed)) + self.label_full, self.label_part = dataGen.label_data((batch_size,),embed) + self.expect_out = np.array([0.9205861 , 0.9205861 , 0.9205861 , 0.9201946 , 0.91951686, 0.919343]) + + def single_matmul_trains(self): + single_callback = ModelCallback() + net = MatmulNet() + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, optimizer=optimizer) + epoch_size = 6 + dataset = Dataset(self.input_full, self.label_full) + model.train(epoch_size, dataset, callbacks=single_callback, dataset_sink_mode=False) + print("---loss---",single_callback.loss_list) + + def data_parallel_matmul_trains(self): + parallel_callback = ModelCallback() + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + net = MatmulNet() + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, optimizer=optimizer) + epoch_size = 6 + dataset = Dataset(self.input_part, self.label_part) + model.train(epoch_size, dataset, callbacks=parallel_callback, dataset_sink_mode=False) + loss_value = np.array(parallel_callback.loss_list) + assert allclose(loss_value, self.expect_out, 0.00001, 0.00001) + + def model_parallel_matmul_trains(self): + parallel_callback = ModelCallback() + matmul_stra = ((1,1),(device_num,1)) + reduce_max_stra = ((1,device_num),) + sub_stra = ((1,device_num),(1,1)) + exp_stra = ((1,device_num),) + reduce_sum_stra = ((1,device_num),) + div_stra = ((1,device_num),(1,1)) + log_stra = ((1,device_num),) + mul_stra = ((1,device_num),(1,device_num)) + sum_cross_entropy_stra = ((1,device_num),) + mul2_stra = ((),(device_num,)) + reduce_mean_stra = ((device_num,),) + onehot_stra = ((1,device_num),(),()) + loss_stra_list = [exp_stra, reduce_sum_stra, onehot_stra, div_stra, log_stra, sum_cross_entropy_stra, mul_stra, mul2_stra, reduce_mean_stra, reduce_max_stra, sub_stra] + context.set_auto_parallel_context(parallel_mode="auto_parallel") + net = MatmulNet(matmul_stra = matmul_stra, loss_stra_list = loss_stra_list) + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, optimizer=optimizer) + epoch_size = 6 + dataset = Dataset(self.input_part, self.label_part) + model.train(epoch_size, dataset, callbacks=parallel_callback, dataset_sink_mode=False) + loss_value = np.array(parallel_callback.loss_list) + assert allclose(loss_value, self.expect_out, 0.00001, 0.00001) + + def mix_parallel_matmul_trains(self): + parallel_callback = ModelCallback() + matmul_stra = ((device_num,1),(1,1)) + reduce_max_stra = ((1,device_num),) + sub_stra = ((device_num,1),(device_num,1)) + exp_stra = ((1,device_num),) + reduce_sum_stra = ((1,device_num),) + div_stra = ((1,device_num),(1,1)) + log_stra = ((1,device_num),) + mul_stra = ((1,device_num),(1,device_num)) + sum_cross_entropy_stra = ((1,device_num),) + mul2_stra = ((),(device_num,)) + reduce_mean_stra = ((device_num,),) + onehot_stra = ((1,device_num),(),()) + loss_stra_list = [exp_stra, reduce_sum_stra, onehot_stra, div_stra, log_stra, sum_cross_entropy_stra, mul_stra, mul2_stra, reduce_mean_stra, reduce_max_stra, sub_stra] + context.set_auto_parallel_context(parallel_mode="auto_parallel") + net = MatmulNet(matmul_stra = matmul_stra, loss_stra_list = loss_stra_list) + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, optimizer=optimizer) + epoch_size = 6 + dataset = Dataset(self.input_part, self.label_part) + model.train(epoch_size, dataset, callbacks=parallel_callback, dataset_sink_mode=False) + loss_value = np.array(parallel_callback.loss_list) + assert allclose(loss_value, self.expect_out, 0.00001, 0.00001) + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_matmul_loss_data_parallel_trains(): + loss_factory = LossFactory() + context.reset_auto_parallel_context() + loss_factory.data_parallel_matmul_trains() + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_matmul_loss_model_parallel_trains(): + loss_factory = LossFactory() + context.reset_auto_parallel_context() + loss_factory.model_parallel_matmul_trains() + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_matmul_loss_mix_parallel_trains(): + loss_factory = LossFactory() + context.reset_auto_parallel_context() + loss_factory.mix_parallel_matmul_trains() diff --git a/tests/st/auto_parallel/test_expand_loss.py b/tests/st/auto_parallel/test_expand_loss.py new file mode 100644 index 00000000000..c3ab64395a1 --- /dev/null +++ b/tests/st/auto_parallel/test_expand_loss.py @@ -0,0 +1,24 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import pytest + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_expand_loss(): + ret = os.system("sh run_auto_parallel_loss_expand.sh") + assert(ret==0) diff --git a/tests/st/auto_parallel/test_model_parallel_onehot.py b/tests/st/auto_parallel/test_model_parallel_onehot.py new file mode 100644 index 00000000000..aeebf1b78b5 --- /dev/null +++ b/tests/st/auto_parallel/test_model_parallel_onehot.py @@ -0,0 +1,24 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import pytest + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_expand_loss(): + ret = os.system("sh run_onehot_model_parallel.sh") + assert(ret==0) diff --git a/tests/st/auto_parallel/test_resnet50_expand_loss_2p.py b/tests/st/auto_parallel/test_resnet50_expand_loss_2p.py new file mode 100644 index 00000000000..62711ccf6a2 --- /dev/null +++ b/tests/st/auto_parallel/test_resnet50_expand_loss_2p.py @@ -0,0 +1,405 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +from numpy import allclose +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim.momentum import Momentum +from mindspore.common.initializer import One +from mindspore.train.model import Model, ParallelMode +from mindspore import context +import os +from mindspore.communication.management import init +import mindspore.ops.functional as F +from mindspore.nn.loss.loss import _Loss +from mindspore.train.callback import Callback +from mindspore.parallel import set_algo_parameters +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(enable_hccl=True) +context.set_context(enable_task_sink=True,device_id=int(os.getenv('DEVICE_ID'))) +context.set_context(enable_ir_fusion=True) +context.set_context(enable_loop_sink=False) +init() +context.set_auto_parallel_context(mirror_mean=True, parallel_mode=ParallelMode.AUTO_PARALLEL) + +def weight_variable(shape, factor=0.1): + return One() + + +def _conv3x3(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): + init_value = weight_variable((out_channels, in_channels, 3, 3)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) + + +def _conv1x1(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): + init_value = weight_variable((out_channels, in_channels, 1, 1)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) + +def _conv7x7(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): + init_value = weight_variable((out_channels, in_channels, 7, 7)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) + + +def _fused_bn(channels, momentum=0.9): + init_weight = weight_variable((channels,)) + init_bias = weight_variable((channels,)) + return nn.BatchNorm2d(channels, momentum=momentum) + +class BasicBlock(nn.Cell): + expansion = 1 + + def __init__(self, + in_channels, + out_channels, + stride=1, + momentum=0.1): + super(BasicBlock, self).__init__() + + self.conv1 = _conv3x3(in_channels, out_channels, stride=stride) + self.bn1 = _fused_bn(out_channels, momentum=momentum) + self.conv2 = _conv3x3(out_channels, out_channels) + self.bn2 = _fused_bn(out_channels, momentum=momentum) + self.relu = P.ReLU() + self.down_sample_layer = None + self.downsample = (in_channels != out_channels) + if self.downsample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channels, + out_channels, + stride=stride, + padding=0), + _fused_bn(out_channels, + momentum=momentum)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = self.conv2(x) + x = self.bn2(x) + + if self.downsample: + identity = self.down_sample_layer(identity) + + out = self.add(x, identity) + out = self.relu(out) + + return out + + +class ResidualBlock(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + momentum=0.9): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = _conv1x1(in_channels, out_chls, stride=1) + self.bn1 = _fused_bn(out_chls, momentum=momentum) + + self.conv2 = _conv3x3(out_chls, out_chls, stride=stride) + self.bn2 = _fused_bn(out_chls, momentum=momentum) + + self.conv3 = _conv1x1(out_chls, out_channels, stride=1) + self.bn3 = _fused_bn(out_channels, momentum=momentum) + + self.relu = P.ReLU() + self.downsample = (in_channels != out_channels) + self.stride = stride + if self.downsample: + self.conv_down_sample = _conv1x1(in_channels, out_channels, + stride=stride) + self.bn_down_sample = _fused_bn(out_channels, momentum=momentum) + elif self.stride != 1: + self.maxpool_down = nn.MaxPool2d(kernel_size=1, stride=2, pad_mode='same') + + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + elif self.stride != 1: + identity = self.maxpool_down(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides=[1,2,2,2], + num_classes=100): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of " + "layer_num, inchannel, outchannel list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2) + self.bn1 = _fused_bn(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same') + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.end_point = nn.Dense(2048, num_classes, has_bias=True, + weight_init=weight_variable((num_classes, 2048)), + bias_init=weight_variable((num_classes,))) + self.squeeze = P.Squeeze() + self.cast = P.Cast() + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + layers = [] + resblk = block(in_channel, out_channel, stride=1) + layers.append(resblk) + + for _ in range(1, layer_num - 1): + resblk = block(out_channel, out_channel, stride=1) + layers.append(resblk) + + resblk = block(out_channel, out_channel, stride=stride) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.squeeze(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10): + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [2, 2, 2, 1], + class_num) + + +class SoftmaxCrossEntropyExpand(_Loss): + def __init__(self, sparse=False): + super(SoftmaxCrossEntropyExpand, self).__init__() + self.exp = P.Exp() + self.sum = P.ReduceSum(keep_dims=True) + self.onehot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.div = P.Div() + self.log = P.Log() + self.sum_cross_entropy = P.ReduceSum(keep_dims=False) + self.mul = P.Mul() + self.mul2 = P.Mul() + self.cast = P.Cast() + self.mean = P.ReduceMean(keep_dims=False) + self.sparse = sparse + self.max = P.ReduceMax(keep_dims=True) + self.sub = P.Sub() + self.eps = Tensor(1e-24, mstype.float32) + + def construct(self, logit, label): + logit_max = self.max(logit, -1) + exp = self.exp(self.sub(logit, logit_max)) + exp_sum = self.sum(exp, -1) + softmax_result = self.div(exp, exp_sum) + if self.sparse: + label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + + softmax_result_log = self.log(softmax_result + self.eps) + loss = self.sum_cross_entropy((self.mul(softmax_result_log, label)), -1) + loss = self.mul2(F.scalar_to_array(-1.0), loss) + loss = self.mean(loss, -1) + + return loss + + +rank_id = int(os.environ["RANK_ID"]) +device_num = int(os.environ["RANK_SIZE"]) +class DataGenerator(): + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def generate_data(self, shape): + data = np.arange(np.prod(shape)).reshape(shape) + return data + + def input_data(self, shape): + data = (self.generate_data(shape)).astype(np.float32) + stra = [1]*len(shape) + stra[0] = device_num + datas = self.get_parallel_blocks(data, stra) + return Tensor(data), Tensor(datas[rank_id]) + + def label_data(self, shape): + data = (self.generate_data(shape)*1000/np.prod(shape)).astype(np.int32) + stra = [1]*len(shape) + stra[0] = device_num + datas = self.get_parallel_blocks(data, stra) + return Tensor(data),Tensor(datas[rank_id]) + + +class Dataset(): + def __init__(self, predict, label, length=1, input_num=2, repeat_count=1): + self.predict = predict + self.label = label + self.index = 0 + self.length = length + self.input_num = input_num + self.repeat_count = repeat_count + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + if self.input_num == 2: + return self.predict, self.label + else: + return self.predict, + + def reset(self): + self.index = 0 + + def get_dataset_size(self): + return self.length + + def get_repeat_count(self): + return self.repeat_count + + +class ModelCallback(Callback): + def __init__(self): + super(ModelCallback, self).__init__() + self.loss_list = [] + def epoch_end(self, run_context, *args): + cb_params = run_context.original_args() + result = cb_params.net_outputs + self.loss_list.append(result.asnumpy().mean()) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_train_feed(num_classes=8192): + set_algo_parameters(elementwise_op_strategy_follow=True) + parallel_callback = ModelCallback() + dataGen = DataGenerator() + input_full, input_part = dataGen.input_data((32*2, 3, 224, 224)) + label_full, label_part = dataGen.label_data((32*2,)) + dataset = Dataset(input_part, label_part) + net = resnet50(num_classes) + loss = SoftmaxCrossEntropyExpand(sparse=True) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 10.0, 0.9) + model = Model(net, loss_fn=loss, optimizer=opt) + model.train(5, dataset, dataset_sink_mode=False, callbacks=parallel_callback) + loss_value = np.array(parallel_callback.loss_list) + expect_out = [9.010913, 8.855984, 8.56246, 8.146317, 7.624489] + assert allclose(loss_value, expect_out, 0.0001, 0.0001) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_train_feed2(num_classes=1001): + set_algo_parameters(elementwise_op_strategy_follow=True) + parallel_callback = ModelCallback() + dataGen = DataGenerator() + input_full, input_part = dataGen.input_data((32*2, 3, 224, 224)) + label_full, label_part = dataGen.label_data((32*2,)) + dataset = Dataset(input_part, label_part) + net = resnet50(num_classes) + loss = SoftmaxCrossEntropyExpand(sparse=True) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 10.0, 0.9) + model = Model(net, loss_fn=loss, optimizer=opt) + model.train(5, dataset, dataset_sink_mode=False, callbacks=parallel_callback) + loss_value = np.array(parallel_callback.loss_list) + expect_out = [6.908755, 6.8358116, 6.6986914, 6.506859, 6.2708097] + assert allclose(loss_value, expect_out, 0.0001, 0.0001) diff --git a/tests/st/control/test_while.py b/tests/st/control/test_while.py new file mode 100644 index 00000000000..56b38f7f9a0 --- /dev/null +++ b/tests/st/control/test_while.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.common import dtype as mstype +import mindspore.context as context +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common import ms_function + +@ms_function +def t1_while(x, y, z): + y = y + 4 + while x < y: + x = x + 1 + x = x + 3 + return x + +def test_net(): + context.set_context(mode=context.GRAPH_MODE,device_target="Ascend") + context.set_context(enable_task_sink=True) + c1 = Tensor([2], mstype.int32) + c2 = Tensor([14], mstype.int32) + c3 = Tensor([1], mstype.int32) + expect = Tensor([21], mstype.int32) + ret = t1_while(c1, c2, c3) + assert (ret == expect) + +if __name__ == "__main__": + test_net() \ No newline at end of file diff --git a/tests/st/fusion/test_add_relu_buffer_fusion.py b/tests/st/fusion/test_add_relu_buffer_fusion.py new file mode 100644 index 00000000000..fbb0f73991e --- /dev/null +++ b/tests/st/fusion/test_add_relu_buffer_fusion.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import mindspore.common.dtype as mstype +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_id=5, device_target="Ascend") +#context.set_context(enable_task_sink=True) +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.softmax = P.Softmax(axis=1) + self.add = P.TensorAdd() + self.cast = P.Cast() + self.relu = P.ReLU() + self.reduce_mean = P.ReduceMean() + + def construct(self, x, y): + x = self.cast(x, mstype.float16) + y = self.cast(y, mstype.float16) + #x = self.softmax(x) + x = self.add(x, y) + #x = self.relu(x) + x = self.relu(x) + #x = self.softmax(x) + x = self.reduce_mean(x) + return x + +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + relu = Net() + output = relu(Tensor(x), Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/fusion/test_conv_bn1_fusion.py b/tests/st/fusion/test_conv_bn1_fusion.py new file mode 100644 index 00000000000..6149b9fd71c --- /dev/null +++ b/tests/st/fusion/test_conv_bn1_fusion.py @@ -0,0 +1,138 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore import Tensor, Parameter, Model +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim import Momentum +from mindspore.common.api import ms_function +import mindspore.nn as wrap +import mindspore.context as context + +context.set_context(device_target="Ascend", enable_task_sink=True) + +input_channel = 2048 +output_channel = 512 +num_class = 10 +batch_size = 32 + + +class MsWrapper(nn.Cell): + def __init__(self, network): + super(MsWrapper, self).__init__(auto_prefix=False) + self._network = network + @ms_function + def construct(self, *args): + return self._network(*args) + + +def me_train_tensor(net, input_np, label_np, epoch_size=2): + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + opt = nn.Momentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])), filter(lambda x: x.requires_grad, net.get_parameters())) + context.set_context(mode=context.GRAPH_MODE) + Model(net, loss, opt) + _network = wrap.WithLossCell(net, loss) + _train_net = MsWrapper(wrap.TrainOneStepCell(_network, opt)) + _train_net.set_train() + for epoch in range(0, epoch_size): + print(f"epoch %d"%(epoch)) + output = _train_net(Tensor(input_np), Tensor(label_np)) + print("********output***********") + print(output.asnumpy()) + + +def test_conv_bn_add_relu_fusion(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(input_channel, output_channel, + kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") + self.conv1 = nn.Conv2d(input_channel, output_channel, + kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") + self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001) + self.add = P.TensorAdd() + self.relu = P.ReLU() + self.mean = P.ReduceMean(keep_dims=True) + self.reshape = P.Reshape() + self.dense = nn.Dense(output_channel, num_class) + + def construct(self, input_x): + output = self.conv(input_x) + output = self.bn(output) + output = self.add(output, self.conv1(input_x)) + output = self.relu(output) + output = self.mean(output, (-2, -1)) + output = self.reshape(output, (batch_size, output_channel)) + output = self.dense(output) + return output + + net = Net() + input_np = np.ones([batch_size, input_channel, 7, 7]).astype(np.float32) * 0.01 + label_np = np.ones([batch_size]).astype(np.int32) + me_train_tensor(net, input_np, label_np) + + +def test_conv_bn_relu_fusion(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(input_channel, output_channel, + kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") + self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001) + self.relu = P.ReLU() + self.mean = P.ReduceMean(keep_dims=True) + self.reshape = P.Reshape() + self.dense = nn.Dense(output_channel, num_class) + + def construct(self, input_x): + output = self.conv(input_x) + output = self.bn(output) + output = self.relu(output) + output = self.mean(output, (-2, -1)) + output = self.reshape(output, (batch_size, output_channel)) + output = self.dense(output) + return output + + net = Net() + input_np = np.ones([batch_size, input_channel, 7, 7]).astype(np.float32) * 0.01 + label_np = np.ones([batch_size]).astype(np.int32) + me_train_tensor(net, input_np, label_np) + + +def test_conv_bn_fusion(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(input_channel, output_channel, + kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") + self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001) + self.mean = P.ReduceMean(keep_dims=True) + self.reshape = P.Reshape() + self.dense = nn.Dense(output_channel, num_class) + + def construct(self, input_x): + output = self.conv(input_x) + output = self.bn(output) + output = self.mean(output, (-2, -1)) + output = self.reshape(output, (batch_size, output_channel)) + output = self.dense(output) + return output + + net = Net() + input_np = np.ones([batch_size, input_channel, 7, 7]).astype(np.float32) * 0.01 + label_np = np.ones([batch_size]).astype(np.int32) + me_train_tensor(net, input_np, label_np) diff --git a/tests/st/fusion/test_tbe_eltwise_fusion_1.py b/tests/st/fusion/test_tbe_eltwise_fusion_1.py new file mode 100644 index 00000000000..0b9ae1fe636 --- /dev/null +++ b/tests/st/fusion/test_tbe_eltwise_fusion_1.py @@ -0,0 +1,47 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import mindspore.common.dtype as mstype +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.softmax = P.Softmax(axis=1) + self.relu = P.ReLU() + self.cast = P.Cast() + + def construct(self, x): + x = self.relu(x) + x = self.relu(x) + return x + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + relu_relu = Net() + output = relu_relu(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/fusion/test_tbe_eltwise_fusion_2.py b/tests/st/fusion/test_tbe_eltwise_fusion_2.py new file mode 100644 index 00000000000..8f6084ae5b8 --- /dev/null +++ b/tests/st/fusion/test_tbe_eltwise_fusion_2.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import mindspore.common.dtype as mstype +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.softmax = P.Softmax(axis=1) + self.cast = P.Cast() + self.relu = P.ReLU() + self.biasadd = P.BiasAdd() + + def construct(self, x, y): + x = self.relu(x) + x = self.relu(x) + x = self.relu(x) + x = self.biasadd(x, y) + x = self.relu(x) + x = self.relu(x) + x = self.relu(x) + return x + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + y = np.random.randn(10).astype(np.float32) + net = Net() + output = net(Tensor(x), Tensor(y)) + print(x) + print(output.asnumpy()) \ No newline at end of file diff --git a/tests/st/fusion/test_tbe_multi_inout_eltwise_fusion.py b/tests/st/fusion/test_tbe_multi_inout_eltwise_fusion.py new file mode 100644 index 00000000000..9a900a4739c --- /dev/null +++ b/tests/st/fusion/test_tbe_multi_inout_eltwise_fusion.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import mindspore.common.dtype as mstype +import numpy as np +import mindspore.context as context +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_id=4, device_target="Ascend") +#context.set_context(enable_task_sink=True) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + self.cast = P.Cast() + self.relu = P.ReLU() + self.biasadd = P.BiasAdd() + + def construct(self, x, y, k, h): + z = self.add(x, y) + z = self.relu(z) + z = self.relu(z) + z1 = self.biasadd(z, k) + z2 = self.biasadd(z, h) + z = self.add(z1, z2) + return z + +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + y = np.random.randn(32, 10).astype(np.float32) + k = np.random.randn(10).astype(np.float32) + h = np.random.randn(10).astype(np.float32) + relu_relu = Net() + output = relu_relu(Tensor(x), Tensor(y), Tensor(k), Tensor(h)) + print(x) + print(output.asnumpy()) + diff --git a/tests/st/fusion/test_tbe_reduce_eltwise_fusion.py b/tests/st/fusion/test_tbe_reduce_eltwise_fusion.py new file mode 100644 index 00000000000..63b1cc542d9 --- /dev/null +++ b/tests/st/fusion/test_tbe_reduce_eltwise_fusion.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import mindspore.common.dtype as mstype +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.cast = P.Cast() + self.relu = P.ReLU() + self.biasaddgrad = G.BiasAddGrad() + + def construct(self, x): + x = self.relu(x) + x = self.relu(x) + x = self.relu(x) + x = self.biasaddgrad(x) + x = self.relu(x) + x = self.relu(x) + x = self.relu(x) + return x + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + net = Net() + output = net(Tensor(x)) + print(x) + print(output.asnumpy()) \ No newline at end of file diff --git a/tests/st/mem_reuse/__init__.py b/tests/st/mem_reuse/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/st/mem_reuse/check_exec_order.sh b/tests/st/mem_reuse/check_exec_order.sh new file mode 100644 index 00000000000..4f049a83b9d --- /dev/null +++ b/tests/st/mem_reuse/check_exec_order.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +export LOCAL_HIAI=/usr/local/HiAI +export TBE_IMPL_PATH=${LOCAL_HIAI}/runtime/ops/op_impl/built-in/ai_core/tbe/impl/ +export LD_LIBRARY_PATH=${LOCAL_HIAI}/runtime/lib64/:${LD_LIBRARY_PATH} +export PATH=${LOCAL_HIAI}/runtime/ccec_compiler/bin/:${PATH} +export PYTHONPATH=${LOCAL_HIAI}/runtime/ops/op_impl/built-in/ai_core/tbe/:${PYTHONPATH} + +set -e +BASEPATH=$(cd "$(dirname $0)"; pwd) +rm -rf "${BASEPATH}/mem_reuse_check/" +mkdir "${BASEPATH}/mem_reuse_check/" +# 1. run normal && check file exist +python "${BASEPATH}"/resnet_cifar_normal.py +if [ $? -ne 0 ]; then + echo "[ERROR] resnet_cifar_normal run failed" + exit 1 +fi +# 2. copy normal to current dir +mv "./normal_mem.ir" "${BASEPATH}/mem_reuse_check/" +# 3. run memreuse && check file exist +python "${BASEPATH}"/resnet_cifar_memreuse.py +if [ $? -ne 0 ]; then + echo "[ERROR] resnet_cifar_memreuse run failed" + exit 1 +fi +# 4. copy memreuse ir to current dir +mv "./memreuse.ir" "${BASEPATH}/mem_reuse_check/" +# 5. check file whether same && return true +python "${BASEPATH}"/check_file.py +if [ $? -ne 0 ]; then + echo "[ERROR] check_file run failed" + exit 1 +fi +rm -rf "${BASEPATH}/mem_reuse_check" diff --git a/tests/st/mem_reuse/check_file.py b/tests/st/mem_reuse/check_file.py new file mode 100644 index 00000000000..2f6fe82d2da --- /dev/null +++ b/tests/st/mem_reuse/check_file.py @@ -0,0 +1,27 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import os +import filecmp +curr_path = os.path.abspath(os.curdir) +file_memreuse = curr_path + "/mem_reuse_check/memreuse.ir" +file_normal = curr_path + "/mem_reuse_check/normal_mem.ir" +checker = os.path.exists(file_memreuse) +assert (checker, True) +checker = os.path.exists(file_normal) +assert (checker, True) +checker = filecmp.cmp(file_memreuse, file_normal) +assert (checker, True) + + diff --git a/tests/st/mem_reuse/resnet.py b/tests/st/mem_reuse/resnet.py new file mode 100644 index 00000000000..fb4075f0a49 --- /dev/null +++ b/tests/st/mem_reuse/resnet.py @@ -0,0 +1,300 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore.common import dtype as mstype + +def weight_variable(shape): + return initializer('XavierUniform', shape=shape, dtype=mstype.float32) + + +def weight_variable_uniform(shape): + return initializer('Uniform', shape=shape, dtype=mstype.float32) + + +def weight_variable_0(shape): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros) + + +def weight_variable_1(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones) + + +def conv3x3(in_channels, out_channels, stride=1, padding=0): + """3x3 convolution """ + weight_shape = (out_channels, in_channels, 3, 3) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 1, 1) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv7x7(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 7, 7) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_uniform(shape) + bn = nn.BatchNorm2d(out_channels, momentum=0.99, eps=0.00001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def bn_with_initialize_last(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_uniform(shape) + bn = nn.BatchNorm2d(out_channels, momentum=0.99, eps=0.00001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def fc_with_initialize(input_channels, out_channels): + weight_shape = (out_channels, input_channels) + weight = weight_variable(weight_shape) + bias_shape = (out_channels) + bias = weight_variable_uniform(bias_shape) + return nn.Dense(input_channels, out_channels, weight, bias) + + +class ResidualBlock(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResidualBlockWithDown(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlockWithDown, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.downSample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn_down_sample = bn_with_initialize(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class MakeLayer0(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer0, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=1, down_sample=True) + self.b = block(out_channels, out_channels, stride=stride) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class MakeLayer1(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer1, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + + return x + + +class MakeLayer2(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer2, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + self.e = block(out_channels, out_channels, stride=1) + self.f = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + x = self.e(x) + x = self.f(x) + + return x + + +class MakeLayer3(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer3, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class ResNet(nn.Cell): + + def __init__(self, block, layer_num, num_classes=100, batch_size=32): + super(ResNet, self).__init__() + self.batch_size = batch_size + self.num_classes = num_classes + + self.conv1 = conv7x7(3, 64, stride=2, padding=0) + + self.bn1 = bn_with_initialize(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + + self.layer1 = MakeLayer0(block, layer_num[0], in_channels=64, out_channels=256, stride=1) + self.layer2 = MakeLayer1(block, layer_num[1], in_channels=256, out_channels=512, stride=2) + self.layer3 = MakeLayer2(block, layer_num[2], in_channels=512, out_channels=1024, stride=2) + self.layer4 = MakeLayer3(block, layer_num[3], in_channels=1024, out_channels=2048, stride=2) + + self.pool = P.ReduceMean(keep_dims=True) + self.squeeze = P.Squeeze(axis=(2, 3)) + self.fc = fc_with_initialize(512 * block.expansion, num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.pool(x, (2, 3)) + x = self.squeeze(x) + x = self.fc(x) + return x + + +def resnet50(batch_size, num_classes): + return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size) + diff --git a/tests/st/mem_reuse/resnet_cifar_memreuse.py b/tests/st/mem_reuse/resnet_cifar_memreuse.py new file mode 100644 index 00000000000..4699c00e736 --- /dev/null +++ b/tests/st/mem_reuse/resnet_cifar_memreuse.py @@ -0,0 +1,160 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore import context +import mindspore.common.dtype as mstype +import os +import numpy as np +import mindspore.ops.functional as F +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore.dataengine as de +import mindspore._c_dataengine as deMap +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore.communication.management import init +from resnet import resnet50 +import random +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +import argparse +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') +parser.add_argument('--epoch_size', type=int, default=1, help='Epoch size.') +parser.add_argument('--batch_size', type=int, default=4, help='Batch size.') +parser.add_argument('--num_classes', type=int, default=10, help='Num classes.') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default="/var/log/npu/datasets/cifar", help='Dataset path') +args_opt = parser.parse_args() + +device_id=int(os.getenv('DEVICE_ID')) + +data_home=args_opt.dataset_path + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(enable_task_sink=True, device_id=device_id) +context.set_context(enable_loop_sink=True) +context.set_context(enable_mem_reuse=True) + + +def create_dataset(repeat_num=1, training=True): + data_dir = data_home + "/cifar-10-batches-bin" + if not training: + data_dir = data_home + "/cifar-10-verify-bin" + ds = de.Cifar10Dataset(data_dir) + + if args_opt.run_distribute: + rank_id=int(os.getenv('RANK_ID')) + rank_size=int(os.getenv('RANK_SIZE')) + ds = de.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) + + resize_height = 224 + resize_width = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT + random_horizontal_op = vision.RandomHorizontalFlip() + resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR + rescale_op = vision.Rescale(rescale, shift) + normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) + changeswap_op = vision.HWC2CHW() + type_cast_op = C.TypeCast(mstype.int32) + + c_trans = [] + if training: + c_trans = [random_crop_op, random_horizontal_op] + c_trans += [resize_op, rescale_op, normalize_op, + changeswap_op] + + # apply map operations on images + ds = ds.map(input_columns="label", operations=type_cast_op) + ds = ds.map(input_columns="image", operations=c_trans) + + # apply repeat operations + ds = ds.repeat(repeat_num) + + # apply shuffle operations + ds = ds.shuffle(buffer_size=10) + + # apply batch operations + ds = ds.batch(batch_size=args_opt.batch_size, drop_remainder=True) + + return ds + + +class CrossEntropyLoss(nn.Cell): + def __init__(self): + super(CrossEntropyLoss, self).__init__() + self.cross_entropy = P.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean() + self.one_hot = P.OneHot() + self.one = Tensor(1.0, mstype.float32) + self.zero = Tensor(0.0, mstype.float32) + + def construct(self, logits, label): + label = self.one_hot(label, F.shape(logits)[1], self.one, self.zero) + loss = self.cross_entropy(logits, label)[0] + loss = self.mean(loss, (-1,)) + return loss + + +if __name__ == '__main__': + if args_opt.do_eval: + context.set_context(enable_hccl=False) + else: + if args_opt.run_distribute: + context.set_context(enable_hccl=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL) + context.set_auto_parallel_context(all_reduce_fusion_split_indices=[140]) + init() + else: + context.set_context(enable_hccl=False) + + context.set_context(mode=context.GRAPH_MODE) + epoch_size = args_opt.epoch_size + net = resnet50(args_opt.batch_size, args_opt.num_classes) + loss = CrossEntropyLoss() + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) + + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + if args_opt.do_train: + dataset = create_dataset(epoch_size) + batch_num = dataset.get_dataset_size() + config_ck = CheckpointConfig(save_checkpoint_steps=batch_num * 5, keep_checkpoint_max=10) + ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10", directory="./", config=config_ck) + loss_cb = LossMonitor() + model.train(epoch_size, dataset, callbacks=[ckpoint_cb, loss_cb]) + + if args_opt.do_eval: + # if args_opt.checkpoint_path: + # param_dict = load_checkpoint(args_opt.checkpoint_path) + # load_param_into_net(net, param_dict) + eval_dataset = create_dataset(1, training=False) + res = model.eval(eval_dataset) + print("result: ", res) + checker = os.path.exists("./memreuse.ir") + assert (checker, True) diff --git a/tests/st/mem_reuse/resnet_cifar_normal.py b/tests/st/mem_reuse/resnet_cifar_normal.py new file mode 100644 index 00000000000..bff0c2d6e66 --- /dev/null +++ b/tests/st/mem_reuse/resnet_cifar_normal.py @@ -0,0 +1,162 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore import context +import mindspore.common.dtype as mstype +import os +import numpy as np +import mindspore.ops.functional as F +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore.dataengine as de +import mindspore._c_dataengine as deMap +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore.communication.management import init +from resnet import resnet50 +import random + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +import argparse + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') +parser.add_argument('--epoch_size', type=int, default=1, help='Epoch size.') +parser.add_argument('--batch_size', type=int, default=4, help='Batch size.') +parser.add_argument('--num_classes', type=int, default=10, help='Num classes.') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default="/var/log/npu/datasets/cifar", help='Dataset path') +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) + +data_home = args_opt.dataset_path + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(enable_task_sink=True, device_id=device_id) +context.set_context(enable_loop_sink=True) +context.set_context(enable_mem_reuse=False) + + +def create_dataset(repeat_num=1, training=True): + data_dir = data_home + "/cifar-10-batches-bin" + if not training: + data_dir = data_home + "/cifar-10-verify-bin" + ds = de.Cifar10Dataset(data_dir) + + if args_opt.run_distribute: + rank_id = int(os.getenv('RANK_ID')) + rank_size = int(os.getenv('RANK_SIZE')) + ds = de.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) + + resize_height = 224 + resize_width = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT + random_horizontal_op = vision.RandomHorizontalFlip() + resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR + rescale_op = vision.Rescale(rescale, shift) + normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) + changeswap_op = vision.HWC2CHW() + type_cast_op = C.TypeCast(mstype.int32) + + c_trans = [] + if training: + c_trans = [random_crop_op, random_horizontal_op] + c_trans += [resize_op, rescale_op, normalize_op, + changeswap_op] + + # apply map operations on images + ds = ds.map(input_columns="label", operations=type_cast_op) + ds = ds.map(input_columns="image", operations=c_trans) + + # apply repeat operations + ds = ds.repeat(repeat_num) + + # apply shuffle operations + ds = ds.shuffle(buffer_size=10) + + # apply batch operations + ds = ds.batch(batch_size=args_opt.batch_size, drop_remainder=True) + + return ds + + +class CrossEntropyLoss(nn.Cell): + def __init__(self): + super(CrossEntropyLoss, self).__init__() + self.cross_entropy = P.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean() + self.one_hot = P.OneHot() + self.one = Tensor(1.0, mstype.float32) + self.zero = Tensor(0.0, mstype.float32) + + def construct(self, logits, label): + label = self.one_hot(label, F.shape(logits)[1], self.one, self.zero) + loss = self.cross_entropy(logits, label)[0] + loss = self.mean(loss, (-1,)) + return loss + + +if __name__ == '__main__': + if args_opt.do_eval: + context.set_context(enable_hccl=False) + else: + if args_opt.run_distribute: + context.set_context(enable_hccl=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL) + context.set_auto_parallel_context(all_reduce_fusion_split_indices=[140]) + init() + else: + context.set_context(enable_hccl=False) + + context.set_context(mode=context.GRAPH_MODE) + epoch_size = args_opt.epoch_size + net = resnet50(args_opt.batch_size, args_opt.num_classes) + loss = CrossEntropyLoss() + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) + + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + if args_opt.do_train: + dataset = create_dataset(epoch_size) + batch_num = dataset.get_dataset_size() + config_ck = CheckpointConfig(save_checkpoint_steps=batch_num * 5, keep_checkpoint_max=10) + ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10", directory="./", config=config_ck) + loss_cb = LossMonitor() + model.train(epoch_size, dataset, callbacks=[ckpoint_cb, loss_cb]) + + if args_opt.do_eval: + # if args_opt.checkpoint_path: + # param_dict = load_checkpoint(args_opt.checkpoint_path) + # load_param_into_net(net, param_dict) + eval_dataset = create_dataset(1, training=False) + res = model.eval(eval_dataset) + print("result: ", res) + checker = os.path.exists("./normal_memreuse.ir") + assert (checker, True) diff --git a/tests/st/nccl/test_nccl_all_gather_op.py b/tests/st/nccl/test_nccl_all_gather_op.py new file mode 100644 index 00000000000..f2a2c7133c2 --- /dev/null +++ b/tests/st/nccl/test_nccl_all_gather_op.py @@ -0,0 +1,50 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size +context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + +init('nccl') +rank = get_rank() +size = get_group_size() +x = np.ones([1,1,3,3]).astype(np.float32) * 0.01 * (rank + 1) + +class Net(nn.Cell): + def __init__( self): + super(Net, self).__init__() + self.all_gather = P.AllGather(group=NCCL_WORLD_COMM_GROUP) + self.x = Parameter(initializer(Tensor(x), x.shape), name='x') + + def construct(self): + return self.all_gather(self.x) + +def test_AllGather(): + all_gather = Net() + output = all_gather() + + expect = np.ones([1, 1, 3, 3]).astype(np.float32) * 0.01 * (0 + 1) + for i in range(size - 1): + tmp = np.ones([1, 1, 3, 3]).astype(np.float32) * 0.01 * (i + 2) + expect = np.concatenate((expect, tmp)) + diff = output.asnumpy() - expect + error = np.ones(shape=expect.shape) * 1.0e-5 + assert np.all(diff < error) + assert (output.shape() == expect.shape) diff --git a/tests/st/nccl/test_nccl_all_reduce_op.py b/tests/st/nccl/test_nccl_all_reduce_op.py new file mode 100644 index 00000000000..3ba8b219e48 --- /dev/null +++ b/tests/st/nccl/test_nccl_all_reduce_op.py @@ -0,0 +1,73 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size +context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + +init('nccl') +rank = get_rank() +size = get_group_size() +x = np.ones([3,1,3,3]).astype(np.float32) * 0.01 * (rank + 1) + +class Net(nn.Cell): + def __init__( self): + super(Net, self).__init__() + self.x1 = Parameter(initializer(Tensor(x), x.shape), name='x1') + self.x2 = Parameter(initializer(Tensor(x), x.shape), name='x2') + self.x3 = Parameter(initializer(Tensor(x), x.shape), name='x3') + + self.op0 = "sum" + self.op1 = "sum" + self.op2 = "sum" + + self.all_reduce1 = P.AllReduce(self.op0, group=NCCL_WORLD_COMM_GROUP) + self.all_reduce2 = P.AllReduce(self.op1, group=NCCL_WORLD_COMM_GROUP) + self.all_reduce3 = P.AllReduce(self.op2, group=NCCL_WORLD_COMM_GROUP) + + def construct(self): + return (self.all_reduce1(self.x1), + self.all_reduce2(self.x2), + self.all_reduce3(self.x3)) + +def test_AllReduce(): + all_reduce = Net() + output = all_reduce() + + expect0 = np.ones([3, 1, 3, 3]).astype(np.float32) * 0 + for i in range(size): + part = np.ones([3, 1, 3, 3]).astype(np.float32) * 0.01 * (i + 1) + expect0 += part + diff0 = output[0].asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output[0].shape() == expect0.shape) + + expect1 = expect0 + diff1 = output[1].asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output[1].shape() == expect1.shape) + + expect2 = expect1 + diff2 = output[2].asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output[2].shape() == expect2.shape) diff --git a/tests/st/nccl/test_nccl_lenet.py b/tests/st/nccl/test_nccl_lenet.py new file mode 100644 index 00000000000..5642603d42f --- /dev/null +++ b/tests/st/nccl/test_nccl_lenet.py @@ -0,0 +1,96 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.nn import Dense +import mindspore.nn as nn +import datetime +import mindspore.context as context +from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size +from mindspore.nn.optim import Momentum +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") +init('nccl') + +epoch = 2 +total = 50000 +batch_size = 32 +mini_batch = total // batch_size + +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + + self.relu = P.ReLU() + self.batch_size = 32 + weight1 = Tensor(np.ones([6, 3, 5, 5]).astype(np.float32) * 0.01) + weight2 = Tensor(np.ones([16, 6, 5, 5]).astype(np.float32) * 0.01) + self.conv1 = nn.Conv2d(3, 6, (5, 5), weight_init=weight1, stride=1, padding=0, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, (5, 5), weight_init=weight2, pad_mode='valid', stride=1, padding=0) + self.pool = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="valid") + self.reshape = P.Reshape() + + weight1 = Tensor(np.ones([120, 400]).astype(np.float32) * 0.01) + self.fc1 = Dense(400, 120, weight_init=weight1) + + weight2 = Tensor(np.ones([84, 120]).astype(np.float32) * 0.01) + self.fc2 = Dense(120, 84, weight_init=weight2) + + weight3 = Tensor(np.ones([10, 84]).astype(np.float32) * 0.01) + self.fc3 = Dense(84, 10, weight_init=weight3) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.reshape(output, (self.batch_size, -1)) + output = self.fc1(output) + output = self.fc2(output) + output = self.fc3(output) + return output + +def test_lenet_nccl(): + net = LeNet() + net.set_train() + + learning_rate = 0.01 + momentum = 0.9 + mom_optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + context.set_auto_parallel_context(parallel_mode="data_parallel", mirror_mean=True, device_num=get_group_size()) + train_network = TrainOneStepCell(net_with_criterion, mom_optimizer) + train_network.set_train() + losses = [] + + data = Tensor(np.ones([net.batch_size, 3, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([net.batch_size]).astype(np.int32)) + start = datetime.datetime.now() + for i in range(epoch): + for step in range(mini_batch): + loss = train_network(data, label) + losses.append(loss.asnumpy()) + end = datetime.datetime.now() + with open("ms_time.txt", "w") as fo1: + fo1.write("time:") + fo1.write(str(end - start)) + with open("ms_loss.txt", "w") as fo2: + fo2.write("loss:") + fo2.write(str(losses[-5:])) diff --git a/tests/st/nccl/test_nccl_reduce_scatter_op.py b/tests/st/nccl/test_nccl_reduce_scatter_op.py new file mode 100644 index 00000000000..af22c7690fd --- /dev/null +++ b/tests/st/nccl/test_nccl_reduce_scatter_op.py @@ -0,0 +1,74 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size +context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + +init('nccl') +rank = get_rank() +size = get_group_size() +x = np.ones([size, 1, 3, 3]).astype(np.float32) * 0.01 * (rank + 1) + +class Net(nn.Cell): + def __init__( self): + super(Net, self).__init__() + self.x = Parameter(initializer(Tensor(x), x.shape), name='x') + + self.op0 = "sum" + self.op1 = "max" + self.op2 = "min" + self.op3 = "prod" + + self.reduce_scatter1 = P.ReduceScatter(self.op0, group=NCCL_WORLD_COMM_GROUP) + self.reduce_scatter2 = P.ReduceScatter(self.op1, group=NCCL_WORLD_COMM_GROUP) + self.reduce_scatter3 = P.ReduceScatter(self.op2, group=NCCL_WORLD_COMM_GROUP) + + def construct(self): + return (self.reduce_scatter1(self.x), + self.reduce_scatter2(self.x), + self.reduce_scatter3(self.x)) + +def test_ReduceScatter(): + reduce_scatter = Net() + output = reduce_scatter() + + sum = np.ones([size, 1, 3, 3]).astype(np.float32) * 0 + for i in range(size): + sum += np.ones([size, 1, 3, 3]).astype(np.float32) * 0.01 * (i + 1) + expect0 = sum[rank : rank + 1] + diff0 = output[0].asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output[0].shape() == expect0.shape) + + expect1 = np.ones([1, 1, 3, 3]).astype(np.float32) * 0.01 * size + diff1 = output[1].asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + print(expect1) + print(output[1]) + assert np.all(diff1 < error1) + assert (output[1].shape() == expect1.shape) + + expect2 = np.ones([1, 1, 3, 3]).astype(np.float32) * 0.01 * 1 + diff2 = output[2].asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output[2].shape() == expect2.shape) diff --git a/tests/st/networks/models/alexnet.py b/tests/st/networks/models/alexnet.py new file mode 100644 index 00000000000..4c8981f04ab --- /dev/null +++ b/tests/st/networks/models/alexnet.py @@ -0,0 +1,55 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.nn import Dense + +class AlexNet(nn.Cell): + def __init__(self, num_classes=10): + super(AlexNet, self).__init__() + self.batch_size = 32 + self.conv1 = nn.Conv2d(3, 96, 11, stride=4, pad_mode="valid") + self.conv2 = nn.Conv2d(96, 256, 5, stride=1, pad_mode="same") + self.conv3 = nn.Conv2d(256, 384, 3, stride=1, pad_mode="same") + self.conv4 = nn.Conv2d(384, 384, 3, stride=1, pad_mode="same") + self.conv5 = nn.Conv2d(384, 256, 3, stride=1, pad_mode="same") + self.relu = P.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=3, stride=2) + self.flatten = nn.Flatten() + self.fc1 = nn.Dense(66256, 4096) + self.fc2 = nn.Dense(4096, 4096) + self.fc3 = nn.Dense(4096, num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv3(x) + x = self.relu(x) + x = self.conv4(x) + x = self.relu(x) + x = self.conv5(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.relu(x) + x = self.fc3(x) + return x diff --git a/tests/st/networks/models/bert/bert_tdt_no_lossscale.py b/tests/st/networks/models/bert/bert_tdt_no_lossscale.py new file mode 100644 index 00000000000..c1ca6f64997 --- /dev/null +++ b/tests/st/networks/models/bert/bert_tdt_no_lossscale.py @@ -0,0 +1,187 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""train bert network without lossscale""" + +import os +import pytest +import numpy as np +from numpy import allclose +import mindspore.common.dtype as mstype +import mindspore.dataset.engine.datasets as de +import mindspore._c_dataengine as deMap +from mindspore import context +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +from mindspore.train.callback import Callback +from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertNetworkWithLoss, BertTrainOneStepCell +from mindspore.nn.optim import Lamb +from mindspore import log as logger +_current_dir = os.path.dirname(os.path.realpath(__file__)) +DATA_DIR = ["/home/workspace/mindspore_dataset/bert/example/examples.tfrecord"] +SCHEMA_DIR = "/home/workspace/mindspore_dataset/bert/example/datasetSchema.json" + +def get_config(version='base', batch_size=1): + """get config""" + if version == 'base': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=768, + num_hidden_layers=2, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + elif version == 'large': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float16) + elif version == 'large_mixed': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + else: + bert_config = BertConfig(batch_size=batch_size) + return bert_config + +def me_de_train_dataset(): + """test me de train dataset""" + # apply repeat operations + repeat_count = 1 + ds = de.StorageDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids", + "next_sentence_labels", "masked_lm_positions", + "masked_lm_ids", "masked_lm_weights"]) + type_cast_op = deMap.TypeCastOp("int32") + ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op) + ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op) + ds = ds.map(input_columns="next_sentence_labels", operations=type_cast_op) + ds = ds.map(input_columns="segment_ids", operations=type_cast_op) + ds = ds.map(input_columns="input_mask", operations=type_cast_op) + ds = ds.map(input_columns="input_ids", operations=type_cast_op) + # apply batch operations + batch_size = 16 + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(repeat_count) + return ds + + +def weight_variable(shape): + """weight variable""" + np.random.seed(1) + ones = np.random.uniform(-0.1, 0.1, size=shape).astype(np.float32) + return Tensor(ones) + + +class ModelCallback(Callback): + def __init__(self): + super(ModelCallback, self).__init__() + self.loss_list = [] + + def step_end(self, run_context): + cb_params = run_context.original_args() + self.loss_list.append(cb_params.net_outputs.asnumpy()[0]) + logger.info("epoch: {}, outputs are {}".format(cb_params.cur_epoch_num, str(cb_params.net_outputs))) + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_bert_tdt(): + """test bert tdt""" + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", reserve_class_name_in_scope=False) + context.set_context(enable_task_sink=True) + context.set_context(enable_loop_sink=True) + context.set_context(enable_mem_reuse=True) + parallel_callback = ModelCallback() + ds = me_de_train_dataset() + version = os.getenv('VERSION', 'large') + batch_size = int(os.getenv('BATCH_SIZE', '16')) + config = get_config(version=version, batch_size=batch_size) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = Lamb(netwithloss.trainable_params(), decay_steps=10000, start_learning_rate=1e-4, + end_learning_rate=0.0, power=10.0, warmup_steps=0, decay_filter=lambda x: False) + netwithgrads = BertTrainOneStepCell(netwithloss, optimizer=optimizer) + netwithgrads.set_train(True) + model = Model(netwithgrads) + params = netwithloss.trainable_params() + for param in params: + value = param.default_input + name = param.name + if isinstance(value, Tensor): + if name.split('.')[-1] in ['weight']: + if name.split('.')[-3] in ['cls2']: + logger.info("***************** BERT param name is 1 {}".format(name)) + param.default_input = weight_variable(value.asnumpy().shape) + else: + logger.info("***************** BERT param name is 2 {}".format(name)) + tempshape = value.asnumpy().shape + shape = (tempshape[1], tempshape[0]) + weight_value = weight_variable(shape).asnumpy() + param.default_input = Tensor(np.transpose(weight_value, [1, 0])) + else: + logger.info("***************** BERT param name is 3 {}".format(name)) + param.default_input = weight_variable(value.asnumpy().shape) + model.train(ds.get_repeat_count(), ds, callbacks=parallel_callback, dataset_sink_mode=False) + loss_value = np.array(parallel_callback.loss_list) + expect_out = [12.191790, 11.739655, 11.523477, 11.320723, 11.113152, 11.203759, 10.841681, 10.826849, + 10.616718, 10.486609] + logger.info("expected loss value output: {}".format(expect_out)) + assert allclose(loss_value, expect_out, 0.001, 0.001) + +if __name__ == '__main__': + test_bert_tdt() diff --git a/tests/st/networks/models/lenet.py b/tests/st/networks/models/lenet.py new file mode 100644 index 00000000000..9df91822f72 --- /dev/null +++ b/tests/st/networks/models/lenet.py @@ -0,0 +1,48 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.nn import Dense +from mindspore import Tensor + +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + self.relu = P.ReLU() + self.batch_size = 32 + + self.conv1 = nn.Conv2d(1, 6, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid') + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.reshape = P.Reshape() + self.fc1 = nn.Dense(400, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.reshape(output, (self.batch_size, -1)) + output = self.fc1(output) + output = self.relu(output) + output = self.fc2(output) + output = self.relu(output) + output = self.fc3(output) + return output diff --git a/tests/st/networks/models/resnetv1_5.py b/tests/st/networks/models/resnetv1_5.py new file mode 100644 index 00000000000..855aec7014b --- /dev/null +++ b/tests/st/networks/models/resnetv1_5.py @@ -0,0 +1,299 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.tensor import Tensor +import mindspore.nn as nn +import mindspore.ops.operations as P + +def weight_variable(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones * 0.01) + + +def weight_variable_0(shape): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros) + + +def weight_variable_1(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones) + + +def conv3x3(in_channels, out_channels, stride=1, padding=0): + """3x3 convolution """ + weight_shape = (out_channels, in_channels, 3, 3) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 1, 1) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv7x7(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 7, 7) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_1(shape) + bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def bn_with_initialize_last(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_0(shape) + bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def fc_with_initialize(input_channels, out_channels): + weight_shape = (out_channels, input_channels) + bias_shape = (out_channels) + weight = weight_variable(weight_shape) + bias = weight_variable_0(bias_shape) + + return nn.Dense(input_channels, out_channels, weight, bias) + + +class ResidualBlock(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResidualBlockWithDown(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlockWithDown, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.downSample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn_down_sample = bn_with_initialize(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class MakeLayer0(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer0, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=1, down_sample=True) + self.b = block(out_channels, out_channels, stride=stride) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class MakeLayer1(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer1, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + + return x + + +class MakeLayer2(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer2, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + self.e = block(out_channels, out_channels, stride=1) + self.f = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + x = self.e(x) + x = self.f(x) + + return x + + +class MakeLayer3(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer3, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class ResNet(nn.Cell): + + def __init__(self, block, layer_num, num_classes=100, batch_size=32): + super(ResNet, self).__init__() + self.batch_size = batch_size + self.num_classes = num_classes + + self.conv1 = conv7x7(3, 64, stride=2, padding=0) + + self.bn1 = bn_with_initialize(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="SAME") + + self.layer1 = MakeLayer0( + block, layer_num[0], in_channels=64, out_channels=256, stride=1) + self.layer2 = MakeLayer1( + block, layer_num[1], in_channels=256, out_channels=512, stride=2) + self.layer3 = MakeLayer2( + block, layer_num[2], in_channels=512, out_channels=1024, stride=2) + self.layer4 = MakeLayer3( + block, layer_num[3], in_channels=1024, out_channels=2048, stride=2) + + self.pool = P.ReduceMean(keep_dims=True) + self.fc = fc_with_initialize(512 * block.expansion, num_classes) + self.flatten = nn.Flatten() + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.pool(x, (-2, -1)) + x = self.flatten(x) + x = self.fc(x) + return x + +def resnet50(batch_size, num_classes): + return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size) + diff --git a/tests/st/networks/test_cpu_network.py b/tests/st/networks/test_cpu_network.py new file mode 100644 index 00000000000..6745a7626bb --- /dev/null +++ b/tests/st/networks/test_cpu_network.py @@ -0,0 +1,76 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + test network +Usage: + python test_network_main.py --net lenet --target Davinci +""" +import os +import time +import pytest +import numpy as np +import argparse +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.nn import TrainOneStepCell, WithLossCell +import mindspore.context as context +from mindspore.nn.optim import Momentum +from models.lenet import LeNet +from models.resnetv1_5 import resnet50 +from models.alexnet import AlexNet +context.set_context(mode=context.GRAPH_MODE, device_target="CPU") + +def train(net, data, label): + learning_rate = 0.01 + momentum = 0.9 + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + res = train_network(data, label) + print("+++++++++Loss+++++++++++++") + print(res) + print("+++++++++++++++++++++++++++") + assert res + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_resnet50(): + data = Tensor(np.ones([32, 3 ,224, 224]).astype(np.float32) * 0.01) + label = Tensor(np.ones([32]).astype(np.int32)) + net = resnet50(32, 10) + train(net, data, label) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_lenet(): + data = Tensor(np.ones([32, 1 ,32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([32]).astype(np.int32)) + net = LeNet() + train(net, data, label) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_alexnet(): + data = Tensor(np.ones([32, 3 ,227, 227]).astype(np.float32) * 0.01) + label = Tensor(np.ones([32]).astype(np.int32)) + net = AlexNet() + train(net, data, label) diff --git a/tests/st/networks/test_gpu_alexnet.py b/tests/st/networks/test_gpu_alexnet.py new file mode 100644 index 00000000000..3b193e17d61 --- /dev/null +++ b/tests/st/networks/test_gpu_alexnet.py @@ -0,0 +1,87 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import pytest +import numpy as np +import mindspore.nn as nn +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore import Tensor +from mindspore.common.initializer import initializer +import mindspore.context as context +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + +class AlexNet(nn.Cell): + def __init__(self, num_classes=10): + super(AlexNet, self).__init__() + self.batch_size = 32 + self.conv1 = nn.Conv2d(3, 96, 11, stride=4, pad_mode="valid") + self.conv2 = nn.Conv2d(96, 256, 5, stride=1, pad_mode="same") + self.conv3 = nn.Conv2d(256, 384, 3, stride=1, pad_mode="same") + self.conv4 = nn.Conv2d(384, 384, 3, stride=1, pad_mode="same") + self.conv5 = nn.Conv2d(384, 256, 3, stride=1, pad_mode="same") + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=3, stride=2,pad_mode="valid",padding=0) + self.flatten = nn.Flatten() + self.fc1 = nn.Dense(6*6*256, 4096) + self.fc2 = nn.Dense(4096, 4096) + self.fc3 = nn.Dense(4096, num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv3(x) + x = self.relu(x) + x = self.conv4(x) + x = self.relu(x) + x = self.conv5(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.relu(x) + x = self.fc3(x) + return x + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_trainTensor(num_classes=10, epoch=15, batch_size=32): + net = AlexNet(num_classes) + lr = 0.1 + momentum = 0.9 + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, momentum, weight_decay = 0.0001) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) + train_network.set_train() + losses=[] + for i in range(0, epoch): + data = Tensor(np.ones([batch_size, 3 ,227, 227]).astype(np.float32) * 0.01) + label = Tensor(np.ones([batch_size]).astype(np.int32)) + loss = train_network(data, label) + losses.append(loss) + assert(losses[-1].asnumpy() < 0.01) diff --git a/tests/st/networks/test_gpu_lenet.py b/tests/st/networks/test_gpu_lenet.py new file mode 100644 index 00000000000..4dac2247d03 --- /dev/null +++ b/tests/st/networks/test_gpu_lenet.py @@ -0,0 +1,88 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +import mindspore.nn as nn +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Dense +from mindspore import Tensor +from mindspore.common.initializer import initializer +from mindspore.common import dtype as mstype +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + self.relu = P.ReLU() + self.batch_size = 1 + weight1 = Tensor(np.ones([6, 3, 5, 5]).astype(np.float32) * 0.01) + weight2 = Tensor(np.ones([16, 6, 5, 5]).astype(np.float32) * 0.01) + self.conv1 = nn.Conv2d(3, 6, (5, 5), weight_init=weight1, stride=1, padding=0, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, (5, 5), weight_init=weight2, pad_mode='valid', stride=1, padding=0) + self.pool = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="valid") + + self.reshape = P.Reshape() + self.reshape1 = P.Reshape() + + self.fc1 = Dense(400, 120) + self.fc2 = Dense(120, 84) + self.fc3 = Dense(84, 10) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.reshape(output, (self.batch_size, -1)) + output = self.fc1(output) + output = self.fc2(output) + output = self.fc3(output) + return output + + +def multisteplr(total_steps, gap, base_lr=0.9, gamma=0.1, dtype=mstype.float32): + lr = [] + for step in range(total_steps): + lr_ = base_lr * gamma ** (step//gap) + lr.append(lr_) + return Tensor(np.array(lr), dtype) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_train_lenet(): + epoch = 100 + net = LeNet() + momentum = initializer(Tensor(np.array([0.9]).astype(np.float32)), [1]) + learning_rate = multisteplr(epoch, 30) + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + losses = [] + for i in range(epoch): + data = Tensor(np.ones([net.batch_size, 3 ,32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([net.batch_size]).astype(np.int32)) + loss = train_network(data, label) + losses.append(loss) + print(losses) diff --git a/tests/st/networks/test_gpu_lstm.py b/tests/st/networks/test_gpu_lstm.py new file mode 100644 index 00000000000..43871798127 --- /dev/null +++ b/tests/st/networks/test_gpu_lstm.py @@ -0,0 +1,138 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Dense +from mindspore import Tensor +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import mindspore.context as context +import mindspore.nn as nn + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + +def InitialLstmWeight(input_size, hidden_size, num_layers, bidirectional, has_bias=False): + num_directions = 1 + if bidirectional: + num_directions = 2 + + weight_size = 0 + gate_size = 4 * hidden_size + for layer in range(num_layers): + for d in range(num_directions): + input_layer_size = input_size if layer == 0 else hidden_size * num_directions + weight_size += gate_size * input_layer_size + weight_size += gate_size * hidden_size + if has_bias: + weight_size += 2 * gate_size + + w_np = np.ones([weight_size, 1, 1]).astype(np.float32) * 0.01 + + w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w') + + h = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='h') + + c = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='c') + + return h, c, w + +class SentimentNet(nn.Cell): + def __init__(self, vocab_size, embed_size, num_hiddens, num_layers, + bidirectional, weight, labels, batch_size): + super(SentimentNet, self).__init__() + self.num_hiddens = num_hiddens + self.num_layers = num_layers + self.bidirectional = bidirectional + self.batch_size = batch_size + + self.embedding = nn.Embedding(vocab_size, embed_size, use_one_hot=False, embedding_table=Tensor(weight)) + self.embedding.embedding_table.requires_grad = False + self.trans = P.Transpose() + self.perm = (1, 0, 2) + self.h, self.c, self.w = InitialLstmWeight(embed_size, num_hiddens, num_layers, bidirectional) + self.encoder = P.LSTM(input_size=embed_size, hidden_size=self.num_hiddens, + num_layers=num_layers, has_bias=False, + bidirectional=self.bidirectional, dropout=0.0) + self.concat = P.Concat(2) + if self.bidirectional: + self.decoder = nn.Dense(num_hiddens * 4, labels) + else: + self.decoder = nn.Dense(num_hiddens * 2, labels) + + self.slice1 = P.Slice() + self.slice2 = P.Slice() + self.reshape = P.Reshape() + + self.num_direction = 1 + if bidirectional: + self.num_direction = 2 + + def construct(self, inputs): + embeddings = self.embedding(inputs) + embeddings = self.trans(embeddings, self.perm) + output, hidden = self.encoder(embeddings, self.h, self.c, self.w) + + output0 = self.slice1(output, (0, 0, 0), (1, 64, 200)) + output1 = self.slice2(output, (499, 0, 0), (1, 64, 200)) + encoding = self.concat((output0, output1)) + encoding = self.reshape(encoding, (self.batch_size, self.num_hiddens * self.num_direction * 2)) + outputs = self.decoder(encoding) + return outputs + +batch_size = 64 +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_LSTM(): + num_epochs = 5 + embed_size = 100 + num_hiddens = 100 + num_layers = 2 + bidirectional = True + labels = 2 + vocab_size = 252193 + max_len = 500 + + weight = np.ones((vocab_size+1, embed_size)).astype(np.float32) + + net = SentimentNet(vocab_size=(vocab_size+1), embed_size=embed_size, + num_hiddens=num_hiddens, num_layers=num_layers, + bidirectional=bidirectional, weight=weight, + labels=labels, batch_size=batch_size) + + learning_rate = 0.1 + momentum = 0.9 + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + + train_features = Tensor(np.ones([64, max_len]).astype(np.int32)) + train_labels = Tensor(np.ones([64,]).astype(np.int32)[0:64]) + losses = [] + for epoch in range(num_epochs): + loss = train_network(train_features, train_labels) + losses.append(loss) + assert(losses[-1].asnumpy() < 0.01) diff --git a/tests/st/networks/test_gpu_resnet.py b/tests/st/networks/test_gpu_resnet.py new file mode 100644 index 00000000000..6d8337a6a93 --- /dev/null +++ b/tests/st/networks/test_gpu_resnet.py @@ -0,0 +1,344 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import pytest +import numpy as np + +from mindspore.nn.cell import Cell +from mindspore.nn.layer.conv import Conv2d +from mindspore.nn.layer.basic import Flatten +from mindspore.nn.layer.normalization import BatchNorm2d +from mindspore.nn.layer.pooling import MaxPool2d +from mindspore.ops.operations import TensorAdd +import mindspore.nn as nn + +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Dense +from mindspore import Tensor +from mindspore.common.initializer import initializer + +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + +def random_normal_init(shape, mean=0.0, stddev=0.01, seed=None): + init_value = np.ones(shape).astype(np.float32) * 0.01 + return Tensor(init_value) + +def variance_scaling_raw(shape): + variance_scaling_value = np.ones(shape).astype(np.float32) * 0.01 + return Tensor(variance_scaling_value) + + + +def weight_variable_0(shape): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros) + + +def weight_variable_1(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones) + + +def conv3x3(in_channels, out_channels, stride=1, padding=1): + """3x3 convolution """ + weight_shape = (out_channels, in_channels, 3, 3) + weight = variance_scaling_raw(weight_shape) + return Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 1, 1) + weight = variance_scaling_raw(weight_shape) + return Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv7x7(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 7, 7) + weight = variance_scaling_raw(weight_shape) + return Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, weight_init=weight, has_bias=False, pad_mode="same") + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_1(shape) + bn = BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def bn_with_initialize_last(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_0(shape) + bn = BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def fc_with_initialize(input_channels, out_channels): + weight_shape = (out_channels, input_channels) + bias_shape = (out_channels) + weight = random_normal_init(weight_shape) + bias = weight_variable_0(bias_shape) + + return Dense(input_channels, out_channels, weight, bias) + + +class ResidualBlock(Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=1) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.add = TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResidualBlockWithDown(Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlockWithDown, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=1) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.downSample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn_down_sample = bn_with_initialize(out_channels) + self.add = TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class MakeLayer0(Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer0, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=1, down_sample=True) + self.b = block(out_channels, out_channels, stride=stride) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class MakeLayer1(Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer1, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + + return x + + +class MakeLayer2(Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer2, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + self.e = block(out_channels, out_channels, stride=1) + self.f = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + x = self.e(x) + x = self.f(x) + + return x + + +class MakeLayer3(Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer3, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class ResNet(Cell): + + def __init__(self, block, layer_num, num_classes=100): + super(ResNet, self).__init__() + + self.conv1 = conv7x7(3, 64, stride=2, padding=3) + + self.bn1 = bn_with_initialize(64) + self.relu = P.ReLU() + self.maxpool = MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + + self.layer1 = MakeLayer0( + block, layer_num[0], in_channels=64, out_channels=256, stride=1) + self.layer2 = MakeLayer1( + block, layer_num[1], in_channels=256, out_channels=512, stride=2) + self.layer3 = MakeLayer2( + block, layer_num[2], in_channels=512, out_channels=1024, stride=2) + self.layer4 = MakeLayer3( + block, layer_num[3], in_channels=1024, out_channels=2048, stride=2) + + self.pool = nn.AvgPool2d(7, 1) + self.fc = fc_with_initialize(512 * block.expansion, num_classes) + self.flatten = Flatten() + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.pool(x) + x = self.flatten(x) + x = self.fc(x) + return x + + +def resnet50(num_classes): + return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_trainTensor(num_classes=10, epoch=8, batch_size=1): + net = resnet50(num_classes) + lr = 0.1 + momentum = 0.9 + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + losses=[] + for i in range(0, epoch): + data = Tensor(np.ones([batch_size, 3 ,224, 224]).astype(np.float32) * 0.01) + label = Tensor(np.ones([batch_size]).astype(np.int32)) + loss = train_network(data, label) + losses.append(loss) + assert(losses[-1].asnumpy() < 1) diff --git a/tests/st/networks/test_network_main.py b/tests/st/networks/test_network_main.py new file mode 100644 index 00000000000..7601739f8c8 --- /dev/null +++ b/tests/st/networks/test_network_main.py @@ -0,0 +1,81 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + test network +Usage: + python test_network_main.py --net lenet --target Ascend +""" +import os +import time +import numpy as np +import argparse +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.nn import TrainOneStepCell, WithLossCell +import mindspore.context as context +from mindspore.nn.optim import Momentum +from models.lenet import LeNet +from models.resnetv1_5 import resnet50 +from models.alexnet import AlexNet +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +def train(net, data, label): + learning_rate = 0.01 + momentum = 0.9 + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + res = train_network(data, label) + print("+++++++++Loss+++++++++++++") + print(res) + print("+++++++++++++++++++++++++++") + assert res + +def test_resnet50(): + data = Tensor(np.ones([32, 3 ,224, 224]).astype(np.float32) * 0.01) + label = Tensor(np.ones([32]).astype(np.int32)) + net = resnet50(32, 10) + train(net, data, label) + +def test_lenet(): + data = Tensor(np.ones([32, 1 ,32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([32]).astype(np.int32)) + net = LeNet() + train(net, data, label) + +def test_alexnet(): + data = Tensor(np.ones([32, 3 ,227, 227]).astype(np.float32) * 0.01) + label = Tensor(np.ones([32]).astype(np.int32)) + net = AlexNet() + train(net, data, label) + +parser = argparse.ArgumentParser(description='MindSpore Testing Network') +parser.add_argument('--net', default='resnet50', type=str, help='net name') +parser.add_argument('--device', default='Ascend', type=str, help='device target') +if __name__ == "__main__": + args = parser.parse_args() + context.set_context(device_target=args.device) + if args.net == 'resnet50': + test_resnet50() + elif args.net == 'lenet': + test_lenet() + elif args.net == 'alexnet': + test_alexnet() + else: + print("Please add net name like --net lenet") diff --git a/tests/st/ops/cpu/test_argmax_op.py b/tests/st/ops/cpu/test_argmax_op.py new file mode 100644 index 00000000000..b448bb3807b --- /dev/null +++ b/tests/st/ops/cpu/test_argmax_op.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="CPU") + +class NetArgmax(nn.Cell): + def __init__( self): + super(NetArgmax, self).__init__() + self.argmax = P.Argmax(output_type=mstype.int32) + x = Tensor(np.array([[1., 20., 5.], + [67., 8., 9.], + [130., 24., 15.]]).astype(np.float32)) + self.x = Parameter(initializer(x, x.shape()), name ='x') + + def construct(self): + return self.argmax(self.x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_argmax(): + + Argmax = NetArgmax() + output = Argmax() + print("================================") + expect = np.array([1,0,0]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() + diff --git a/tests/st/ops/cpu/test_bias_add.py b/tests/st/ops/cpu/test_bias_add.py new file mode 100644 index 00000000000..2a4b8622bb0 --- /dev/null +++ b/tests/st/ops/cpu/test_bias_add.py @@ -0,0 +1,53 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bias_add = P.BiasAdd() + + def construct(self, x, b): + return self.bias_add(x, b) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_bias_add1(): + x = np.ones([2,3,4,4]).astype(np.float32) + b = np.array([1,1,1]).astype(np.float32) + bias_add = Net() + output = bias_add(Tensor(x), Tensor(b)) + expect_output = np.ones([2,3,4,4]).astype(np.float32)*2 + print(output) + assert np.all(output.asnumpy()==expect_output), "bias_add execute failed, please check current code commit" + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_bias_add2(): + x = np.ones([2,3]).astype(np.float32) + b = np.array([1,1,1]).astype(np.float32) + bias_add = Net() + output = bias_add(Tensor(x), Tensor(b)) + expect_output = np.ones([2,3]).astype(np.float32)*2 + print(output) + assert np.all(output.asnumpy()==expect_output), "bias_add execute failed, please check current code commit" diff --git a/tests/st/ops/cpu/test_bias_add_grad.py b/tests/st/ops/cpu/test_bias_add_grad.py new file mode 100644 index 00000000000..01dcdae6901 --- /dev/null +++ b/tests/st/ops/cpu/test_bias_add_grad.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bias_add_grad = G.BiasAddGrad() + + def construct(self, dout): + return self.bias_add_grad(dout) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_bias_add_grad1(): + dout = np.ones([2,3]).astype(np.float32) + bias_add_grad = Net() + output = bias_add_grad(Tensor(dout)) + expect_output = np.array([2.,2.,2.]).astype(np.float32) + print(output.asnumpy()) + assert np.all(output.asnumpy()==expect_output), "bias_add_grad execute failed, please check current code commit" + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_bias_add_grad2(): + dout = np.ones([2,3,4,4]).astype(np.float32) + bias_add_grad = Net() + output = bias_add_grad(Tensor(dout)) + expect_output = np.array([32.,32.,32.]).astype(np.float32) + print(output.asnumpy()) + assert np.all(output.asnumpy()==expect_output), "bias_add_grad execute failed, please check current code commit" diff --git a/tests/st/ops/cpu/test_conv2d_backprop_filter_op.py b/tests/st/ops/cpu/test_conv2d_backprop_filter_op.py new file mode 100644 index 00000000000..75ca9154993 --- /dev/null +++ b/tests/st/ops/cpu/test_conv2d_backprop_filter_op.py @@ -0,0 +1,77 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class Net4(nn.Cell): + def __init__(self): + super(Net4, self).__init__() + out_channel = 4 + kernel_size = 1 + self.conv_filter = G.Conv2DBackpropFilter(out_channel, + kernel_size, + pad_mode="valid", + pad=0, + mode=1, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer(Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='w') + self.x = Parameter(initializer(Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)), [1,1,6,6]), name='x') + self.out = Parameter(initializer(Tensor(np.array([[[ + [ -5, -4, 0, 8], + [-10, -2, 2, 3], + [ 0, -2, -4, -7], + [ -3, -2, -3, -16]]]]).astype(np.float32)),[1,1,4,4]), name='y') + self.get_shape = P.Shape() + + def construct(self): + return self.conv_filter(self.out, self.x, self.get_shape(self.w)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_conv2d_backprop_filter(): + conv2d_filter = Net4() + output = conv2d_filter() + print("================================") + """ + expect output: + [[[[ -60, -142, -265] + [-104, -211, -322] + [-102, -144, -248]]]] + """ + expect = np.array([[[[ -60, -142, -265], + [-104, -211, -322], + [-102, -144, -248]]]]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/cpu/test_conv2d_backprop_input_op.py b/tests/st/ops/cpu/test_conv2d_backprop_input_op.py new file mode 100644 index 00000000000..2c0a5849af9 --- /dev/null +++ b/tests/st/ops/cpu/test_conv2d_backprop_input_op.py @@ -0,0 +1,82 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class Net5(nn.Cell): + def __init__(self): + super(Net5, self).__init__() + out_channel = 4 + kernel_size = 1 + self.conv_input = P.Conv2DBackpropInput(out_channel, + kernel_size, + pad_mode="valid", + pad=0, + mode=1, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer(Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='w') + self.x = Parameter(initializer(Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)), [1,1,6,6]), name='x') + self.out = Parameter(initializer(Tensor(np.array([[[ + [ -5, -4, 0, 8], + [-10, -2, 2, 3], + [ 0, -2, -4, -7], + [ -3, -2, -3, -16]]]]).astype(np.float32)),[1,1,4,4]), name='y') + self.get_shape = P.Shape() + + def construct(self): + return self.conv_input(self.out, self.w, self.get_shape(self.x)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_conv2d_backprop_input(): + conv2d_input = Net5() + output = conv2d_input() + print("================================") + """ + expect output: + [[[[ -5, -4, 5, 12, 0, -8] + [-15, -6, 17, 17, -2, -11] + [-15, -8, 13, 12, 2, -4] + [-13, -6, 8, -14, 5, 20] + [ -3, -4, -4, -19, 7, 23] + [ -3, -2, 0, -14, 3, 16]]]] + """ + expect = np.array([[[[ -5, -4, 5, 12, 0, -8], + [-15, -6, 17, 17, -2, -11], + [-15, -8, 13, 12, 2, -4], + [-13, -6, 8, -14, 5, 20], + [ -3, -4, -4, -19, 7, 23], + [ -3, -2, 0, -14, 3, 16]]]]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/cpu/test_conv2d_op.py b/tests/st/ops/cpu/test_conv2d_op.py new file mode 100644 index 00000000000..d298f2729b3 --- /dev/null +++ b/tests/st/ops/cpu/test_conv2d_op.py @@ -0,0 +1,74 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class NetConv2d(nn.Cell): + def __init__( self): + super(NetConv2d, self).__init__() + out_channel = 2 + kernel_size = 1 + self.conv = P.Conv2D(out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer( + Tensor(np.arange(2 * 3 * 1 * 1).reshape(2, 3, 1, 1).astype(np.float32)), [2, 3, 1, 1]), name='w') + self.x = Parameter(initializer( + Tensor(np.arange(1 * 3 * 3 * 3).reshape(1, 3, 3, 3).astype(np.float32)), [1, 3, 3, 3]), name='x') + + + def construct(self): + return self.conv(self.x, self.w) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_conv2d(): + conv2d = NetConv2d() + output = conv2d() + print("================================") + """ + expect output: + [[[[ 45. 48. 51.] + [ 54. 57. 60.] + [ 63. 66. 69.]] + + [[126. 138. 150.] + [162. 174. 186.] + [198. 210. 222.]]]] + """ + expect = np.array([[[[ 45, 48, 51], + [ 54, 57, 60], + [ 63, 66, 69]], + [[126, 138, 150], + [162, 174, 186], + [198, 210, 222]]]]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/cpu/test_equalcount_op.py b/tests/st/ops/cpu/test_equalcount_op.py new file mode 100644 index 00000000000..6ad33c5d6aa --- /dev/null +++ b/tests/st/ops/cpu/test_equalcount_op.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="CPU") + +class NetEqualCount(nn.Cell): + def __init__( self): + super(NetEqualCount, self).__init__() + self.equalcount = P.EqualCount() + x = Tensor(np.array([1, 20, 5]).astype(np.int32)) + y = Tensor(np.array([2, 20, 5]).astype(np.int32)) + self.x = Parameter(initializer(x, x.shape()), name ='x') + self.y = Parameter(initializer(y, y.shape()), name ='y') + + def construct(self): + return self.equalcount(self.x, self.y) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_equalcount(): + + EqualCount = NetEqualCount() + output = EqualCount() + print("================================") + expect = np.array([2]).astype(np.int32) + print(output) + assert (output.asnumpy() == expect).all() + diff --git a/tests/st/ops/cpu/test_maxpool_grad_op.py b/tests/st/ops/cpu/test_maxpool_grad_op.py new file mode 100644 index 00000000000..bf1b4d6eda2 --- /dev/null +++ b/tests/st/ops/cpu/test_maxpool_grad_op.py @@ -0,0 +1,81 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class Net_Pool_Grad(nn.Cell): + def __init__(self): + super(Net_Pool_Grad, self).__init__() + self.maxpool_grad_fun = G.MaxPoolGrad(padding="VALID", + ksize=2, + strides=2) + + self.x = Parameter(initializer( + Tensor(np.array([[[ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + [12, 13, 14, 15, 16, 17], + [18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29], + [30, 31, 32, 33, 34, 35] + ]]]).astype(np.float32)), [1, 1, 6, 6]), name='x') + + self.a = Parameter(initializer( + Tensor(np.array([[[ + [3, 3, 3], + [3, 3, 3], + [3, 3, 3] + ]]]).astype(np.float32)), [1, 1, 3, 3]), name='a') + + self.d = Parameter(initializer( + Tensor(np.array([[[ + [7, 9, 11], + [19, 21, 23], + [31, 33, 35] + ]]]).astype(np.float32)), [1, 1, 3, 3]), name='d') + + + def construct(self): + return self.maxpool_grad_fun(self.x, self.a, self.d) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_maxpool2d_grad(): + maxpool2d_grad = Net_Pool_Grad() + output = maxpool2d_grad() + print(output) + + expect_result = (np.array([[[ + [0, 0, 0, 0, 0, 0], + [0, 7, 0, 9, 0, 11], + [0, 0, 0, 0, 0, 0], + [0, 19, 0, 21, 0, 23], + [0, 0, 0, 0, 0, 0], + [0, 31, 0, 33, 0, 35] + ]]])) + assert (output.asnumpy() == expect_result).all() + diff --git a/tests/st/ops/cpu/test_maxpool_op.py b/tests/st/ops/cpu/test_maxpool_op.py new file mode 100644 index 00000000000..3d062505029 --- /dev/null +++ b/tests/st/ops/cpu/test_maxpool_op.py @@ -0,0 +1,67 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target="CPU") + +class Net_Pool(nn.Cell): + def __init__(self): + super(Net_Pool, self).__init__() + self.maxpool_fun = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="VALID") + def construct(self, x): + return self.maxpool_fun(x) + +class Net_Pool2(nn.Cell): + def __init__(self): + super(Net_Pool2, self).__init__() + self.maxpool_fun = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="SAME") + def construct(self, x): + return self.maxpool_fun(x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_maxpool2d(): + x = Tensor(np.array([[[ + [0, 1, 2, 3, -4, -5], + [6, 7, 8, 9, -10, -11], + [12, 13, 14, -15, -16, -17], + [18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29], + [30, 31, 32, 33, 34, 35] + ]]]).astype(np.float32)) + maxpool2d = Net_Pool() + maxpool2d2 = Net_Pool2() + output2 = maxpool2d2(x) + output = maxpool2d(x) + expect_result = (np.array([[[ + [7, 9, -4], + [19, 21, 23], + [31, 33, 35] + ]]])) + expect_result2 = (np.array([[[ + [14, 14, -4], + [26, 28, 29], + [32, 34, 35] + ]]])) + print(output.asnumpy()) + assert (output.asnumpy() == expect_result).all() + print(output2.asnumpy()) + assert (output2.asnumpy() == expect_result2).all() diff --git a/tests/st/ops/cpu/test_momentum_op.py b/tests/st/ops/cpu/test_momentum_op.py new file mode 100644 index 00000000000..63c758fb7d3 --- /dev/null +++ b/tests/st/ops/cpu/test_momentum_op.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +import mindspore.nn as nn +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Dense +from mindspore import Tensor +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target="CPU") + +class MomentumNet(nn.Cell): + def __init__(self): + super(MomentumNet, self).__init__() + self.batch_size = 1 + + self.reshape = P.Reshape() + weight = Tensor(np.ones([10, 16]).astype(np.float32) * 0.01) + self.fc1 = Dense(16, 10, weight_init=weight) + + def construct(self, input_x): + output = self.reshape(input_x, (self.batch_size, -1)) + output = self.fc1(output) + return output + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_momentum(): + epoch = 13 + net = MomentumNet() + learning_rate = 0.1 + momentum = 0.9 + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + losses = [] + for i in range(epoch): + data = Tensor(np.arange(0, 16).reshape(1, 1, 4, 4).astype(np.float32)*0.01) + label = Tensor(np.array([0]).astype(np.int32)) + loss = train_network(data, label) + losses.append(loss) + + print("================================") + print(losses) + """ + expect output: + [[0.04132498 0.00874167 0.00874167 0.00874167 0.00874167 + 0.00874167 0.00874167 0.00874167 0.00874167 0.00874167]] + """ + error = np.ones(shape=[1, 10]) * 1.0e-6 + + return losses diff --git a/tests/st/ops/cpu/test_mul_op.py b/tests/st/ops/cpu/test_mul_op.py new file mode 100644 index 00000000000..98e25181986 --- /dev/null +++ b/tests/st/ops/cpu/test_mul_op.py @@ -0,0 +1,50 @@ +#Copyright 2019 Huawei Technologies Co., Ltd +# +#Licensed under the Apache License, Version 2.0(the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +#http: // www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. +#== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +x = np.random.uniform(-2, 2, (2,3,4,4)).astype(np.float32) +y = np.random.uniform(-2, 2, (1,1,1,1)).astype(np.float32) + +context.set_context(device_target='CPU') + +class Net(nn.Cell): + def __init__( self): + super(Net, self).__init__() + self.mul = P.Mul() + self.x = Parameter(initializer(Tensor(x), x.shape), name='x3') + self.y = Parameter(initializer(Tensor(y), y.shape), name='y3') + + @ms_function + def construct(self): + return self.mul(self.x, self.y) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_Mul(): + mul = Net() + output = mul() + print(x) + print(y) + print(output) diff --git a/tests/st/ops/cpu/test_relu_grad_op.py b/tests/st/ops/cpu/test_relu_grad_op.py new file mode 100644 index 00000000000..05823cf3d6b --- /dev/null +++ b/tests/st/ops/cpu/test_relu_grad_op.py @@ -0,0 +1,50 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class NetReluGrad(nn.Cell): + def __init__(self): + super(NetReluGrad, self).__init__() + self.rekuGrad = G.ReluGrad() + self.x = Parameter(initializer(Tensor(np.array([[[[-1, 1, 1], + [1, -1, 1], + [1, 1, -1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='x') + self.dy = Parameter(initializer(Tensor(np.array([[[[1, 0, 1], + [0, 1, 0], + [1, 1, 1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='dy') + def construct(self): + return self.rekuGrad(self.dy, self.x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_relu_grad(): + relu_grad = NetReluGrad() + output = relu_grad() + expect = np.array([[[ [0, 0, 1,],[0, 0, 0,],[1, 1, 0.] ]]]).astype(np.float32) + error = np.ones(shape=[3, 3]) * 1.0e-6 + diff = output.asnumpy() - expect + assert np.all(diff < error) diff --git a/tests/st/ops/cpu/test_relu_op.py b/tests/st/ops/cpu/test_relu_op.py new file mode 100644 index 00000000000..648ebcf5b95 --- /dev/null +++ b/tests/st/ops/cpu/test_relu_op.py @@ -0,0 +1,47 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class NetRelu(nn.Cell): + def __init__(self): + super(NetRelu, self).__init__() + self.relu = P.ReLU() + self.x = Parameter(initializer(Tensor(np.array([[[[-1, 1, 10], + [1, -1, 1], + [10, 1, -1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='x') + def construct(self): + return self.relu(self.x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_relu(): + relu = NetRelu() + output = relu() + expect = np.array([[[ [0, 1, 10,], + [1, 0, 1,], + [10, 1, 0.]]]]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/cpu/test_softmax_op.py b/tests/st/ops/cpu/test_softmax_op.py new file mode 100644 index 00000000000..29b05ef3fb7 --- /dev/null +++ b/tests/st/ops/cpu/test_softmax_op.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class NetSoftmax(nn.Cell): + def __init__( self): + super(NetSoftmax, self).__init__() + self.softmax = P.Softmax() + x = Tensor(np.array([[0.1, 0.3, 0.6], + [0.2, -0.6, 0.8], + [0.6, 1, 0.4]]).astype(np.float32)) + self.x = Parameter(initializer(x, x.shape()), name ='x') + + def construct(self): + return self.softmax(self.x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_softmax(): + Softmax = NetSoftmax() + output = Softmax() + output = output.asnumpy() + outputSum = output.sum(axis=1) + expect = np.ones(3) + error = expect * 1.0e-6 + diff = np.abs(outputSum - expect) + print(diff) + assert np.all(diff < error) + diff --git a/tests/st/ops/cpu/test_softmax_with_cross_entropy_op.py b/tests/st/ops/cpu/test_softmax_with_cross_entropy_op.py new file mode 100644 index 00000000000..10dd2002b8e --- /dev/null +++ b/tests/st/ops/cpu/test_softmax_with_cross_entropy_op.py @@ -0,0 +1,56 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + +class NetSoftmaxWithCrossEntropy(nn.Cell): + def __init__( self): + super(NetSoftmaxWithCrossEntropy, self).__init__() + logits = Tensor(np.array([[1,1,10], + [1,10,1], + [10,1,1]]).astype(np.float32)) + self.logits = Parameter(initializer(logits, logits.shape()), name ='logits') + labels = Tensor(np.array([2,1,0]).astype(np.int32)) + self.labels = Parameter(initializer(labels, labels.shape()), name ='labels') + self.SoftmaxWithCrossEntropy = P.SparseSoftmaxCrossEntropyWithLogits(True) + + def construct(self): + return self.SoftmaxWithCrossEntropy(self.logits, self.labels) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_net(): + SoftmaxWithCrossEntropy = NetSoftmaxWithCrossEntropy() + output = SoftmaxWithCrossEntropy() + expect = np.array([[ 4.1126452e-05, 4.1126452e-05, -8.2234539e-05], + [ 4.1126452e-05, -8.2234539e-05, 4.1126452e-05], + [-8.2234539e-05, 4.1126452e-05, 4.1126452e-05]]).astype(np.float32) + print(output) + error = np.ones(shape=[3, 3]) * 1.0e-6 + diff = output.asnumpy() - expect + print(diff) + assert np.all(diff < error) + assert np.all(-diff < error) diff --git a/tests/st/ops/custom_ops_tbe/__init__.py b/tests/st/ops/custom_ops_tbe/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/st/ops/custom_ops_tbe/conv2d.py b/tests/st/ops/custom_ops_tbe/conv2d.py new file mode 100755 index 00000000000..266005f293f --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/conv2d.py @@ -0,0 +1,198 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from __future__ import absolute_import +import te.lang.cce +from te.platform.fusion_manager import fusion_manager +from .conv_layer import conv_layer_cce +from .conv_layer_fast import conv_layer_fast_cce +from topi.cce import util +from te import platform as cce + +Nonetype = type(None) +# pylint: disable=unused-argument, no-value-for-parameter, too-many-branches +@fusion_manager.register("conv2d") +def conv2d_compute(inputs, weights, bias, outputs, strides, pad_list, dilations, + kernel_name="conv2d"): + """ + conv2d compute + + Notice + ------ + only used by framework combine with IR + + Parameters + ---------- + inputs: tvm placeholder + input 5hd feature map tensor + weights: tvm placeholder + input frac_z weight tensor + outputs: tvm placeholder + output tensor, dtype must be assigned + bias: tvm placeholder or None + input 1d bias tensor + strides: integers + stride on H/W, format sensitive + pads: tuple/list of 4 integers + [pad_top, pad_bottom, pad_left, pad_right] + dilations: integers + dilation on H/W, format sensitive + kernel_name: string + kernel name, default value is "conv2d" + + Returns + ------- + tvm compute + """ + shape_w = [] + for i in weights.op.attrs['ori_shape']: + shape_w.append(i.value) + + format_w = weights.op.attrs['ori_format'] + if format_w == "NCHW": + weight_h = shape_w[2] + weight_w = shape_w[3] + elif format_w == "NHWC": + weight_h = shape_w[1] + weight_w = shape_w[2] + elif format_w == "HWCN": + weight_h = shape_w[0] + weight_w = shape_w[1] + else: + raise RuntimeError("weights ori_format should be NCHW, NHWC or HWCN") + + format_x = inputs.op.attrs['ori_format'] + if format_x == "NCHW": + strideh = strides[0] + stridew = strides[0] + dlt_h = dilations[0] + dlt_w = dilations[0] + elif format_x == "NHWC": + strideh = strides[0] + stridew = strides[0] + dlt_h = dilations[0] + dlt_w = dilations[0] + else: + raise RuntimeError("inputs ori_format should be NCHW or NHWC") + + if len(pad_list) == 4: + padh = [pad_list[0], pad_list[1]] + padw = [pad_list[2], pad_list[3]] + else: + raise RuntimeError("pads shape should be 4d.") + + para_dict = {"pad_h": padh, "pad_w": padw, "stride_h": strideh, "stride_w": stridew, + "filter_h": weight_h, "filter_w": weight_w, "bias_tensor": bias} + + if cce.CceProductParams().cce_product == "5.10": + para_dict["mad_dtype"] = "float16" + res = te.lang.cce.conv(inputs, weights, para_dict) + else: + res = te.lang.cce.conv(inputs, weights, para_dict) + + return res + +@util.check_input_type(dict, dict, (dict, Nonetype), dict, (tuple, list), (tuple, list), (tuple, list), + str) +def conv2d(inputs, weights, bias, outputs, strides, pad_list, dilations, + kernel_name="conv2d"): + """ + algorithm: conv2d + + Notice + ------ + only used by framework combine with IR + + Parameters + ---------- + inputs: dict with keys(shape and dtype) + input 4d feature map tensor + weights: dict with keys(shape and dtype) + input 4d weight tensor + outputs: dict with keys(shape and dtype) + output tensor, dtype must be assigned + bias: dict with keys(shape and dtype) or None + input bias tensor + strides: integers + stride on H/W, format sensitive + pads: integers + [pad_top, pad_bottom, pad_left, pad_right] + dilations: tuple/list of 4 integers + dilation on H/W, format sensitive + kernel_name: str + kernel name, default value is "conv2d" + + Returns + ------- + None + """ + shape_x = inputs.get("ori_shape") + in_dtype = inputs.get("dtype") + shape_w = weights.get("ori_shape") + w_dtype = weights.get("dtype") + res_dtype = outputs.get("dtype") + + if len(pad_list) == 4: + padh = [pad_list[0], pad_list[1]] + padw = [pad_list[2], pad_list[3]] + else: + raise RuntimeError("pads shape should be 4d.") + + if (not isinstance(shape_x, (tuple, list))) or len(shape_x) != 4: + raise RuntimeError("inputs should be 4d list.") + + if (not isinstance(shape_w, (tuple, list))) or len(shape_w) != 4: + raise RuntimeError("weights should be 4d list.") + + format_x = inputs.get("ori_format") + if format_x == "NCHW": + shape_fm = shape_x + strideh = strides[0] + stridew = strides[0] + dlt_h = dilations[0] + dlt_w = dilations[0] + elif format_x == "NHWC": + shape_fm = [shape_x[0], shape_x[3], shape_x[1], shape_x[2]] + strideh = strides[0] + stridew = strides[0] + dlt_h = dilations[0] + dlt_w = dilations[0] + else: + raise RuntimeError("inputs ori_format should be NCHW or NHWC.") + + format_w = weights.get("ori_format") + if format_w == "NCHW": + shape_filter = shape_w + elif format_w == "NHWC": + shape_filter = [shape_w[0], shape_w[3], shape_w[1], shape_w[2]] + elif format_w == "HWCN": + shape_filter = [shape_w[3], shape_w[2], shape_w[0], shape_w[1]] + else: + raise RuntimeError("weights ori_format should be NCHW, NHWC or HWCN.") + + if bias is None: + use_bias = False + else: + use_bias = True + + if cce.CceProductParams().cce_product == "5.10": + conv_layer_fast_cce(shape_fm, shape_filter, in_dtype, w_dtype, res_dtype, + padh, padw, strideh, stridew, bias=use_bias, + kernel_name=kernel_name, need_build=True, need_print=False) + else: + conv_layer_cce(shape_fm, shape_filter, in_dtype, w_dtype, res_dtype, + padh, padw, strideh, stridew, + quantize_config=[0, 0, 0], scale_sqrt=[0, 0, 0], + bias=use_bias, kernel_name=kernel_name, + need_build=True, need_print=False) diff --git a/tests/st/ops/custom_ops_tbe/conv_layer.py b/tests/st/ops/custom_ops_tbe/conv_layer.py new file mode 100755 index 00000000000..3b3f9cdcf48 --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/conv_layer.py @@ -0,0 +1,515 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import te.lang.cce +from te import tvm +from te.platform import CUBE_MKN +from topi import generic +from topi.cce import util +from topi.cce.util import is_v200_version +# pylint: disable=R0912,R0913,R0914,R0915,E1101 +# the dim of shape in conv must be 4 +PAD_SHAPE_DIM = 2 + +NONETYPE = type(None) + +@util.check_input_type((list, tuple), (list, tuple), str, str, str, (list, int), (list, int), + int, int,(list, tuple), (list, tuple), + str, str, str, + str, str, str, + str, bool, str) +def conv_layer_cce_para_check(shape_in, shape_w, in_dtype, w_dtype, res_dtype, padh, padw, + strideh, stridew, quantize_config, scale_sqrt, + scale_q_dtype, offset_q_dtype, scale_dq_dtype, + scale_rq_dtype, offset_rq_dtype, offset_w_dtype, + offset_pad_dtype, bias, kernel_name): + # conv shape check + util.check_kernel_name(kernel_name) + + # conv data type check + util.check_dtype_rule(in_dtype, ['float16', 'int8', 'uint8']) + util.check_dtype_rule(w_dtype, ['float16', 'int8', 'uint8']) + res_dtype_list = ['float16', 'int8', 'uint8'] + if is_v200_version(): + res_dtype_list.append('int32') + util.check_dtype_rule(res_dtype, res_dtype_list) + util.check_dtype_rule(scale_q_dtype, ['float16']) + util.check_dtype_rule(offset_q_dtype, ['float16']) + util.check_dtype_rule(scale_dq_dtype, ['float16']) + util.check_dtype_rule(scale_rq_dtype, ['float16']) + util.check_dtype_rule(offset_rq_dtype, ['float16']) + util.check_dtype_rule(offset_w_dtype, ['int32']) + util.check_dtype_rule(offset_pad_dtype, ['uint8']) + + if not isinstance(bias, bool): + raise RuntimeError("bias dtype should be bool.") + + if quantize_config[0] == 0: + if is_v200_version(): + util.check_dtype_rule(in_dtype, ('int8', )) + util.check_dtype_rule(w_dtype, ('int8', )) + util.check_dtype_rule(res_dtype, ('int32', )) + else: + util.check_dtype_rule(in_dtype, ['float16']) + util.check_dtype_rule(w_dtype, ['float16']) + util.check_dtype_rule(res_dtype, ['float16']) + + if quantize_config[0] == 1: + util.check_dtype_rule(w_dtype, ['int8']) + if quantize_config[1] == 0: + util.check_dtype_rule(in_dtype, ['int8', 'float16']) + util.check_dtype_rule(res_dtype, ['int8', 'float16']) + elif quantize_config[1] == 1: + util.check_dtype_rule(in_dtype, ['uint8', 'float16']) + util.check_dtype_rule(res_dtype, ['uint8', 'float16']) + elif quantize_config[1] == 2: + raise RuntimeError("All Offset mode quantize not support.") + else: + raise RuntimeError("Invalid quantize algorithm.") + + # quantize switch on + if quantize_config[0] == 1: + quantize_turn_on = True + # quantize -> DeQuantize dataflow + if in_dtype == 'float16' and w_dtype == 'int8' and res_dtype == 'float16': + pass + # DeQuantize dataflow + elif (in_dtype in ['int8', 'uint8'] and w_dtype == 'int8' and + res_dtype == 'float16'): + pass + # quantize -> ReQuantize dataflow + elif (in_dtype == 'float16' and w_dtype == 'int8' and res_dtype in + ['int8', 'uint8']): + pass + # ReQuantize dataflow + elif (in_dtype in ['int8', 'uint8'] and w_dtype == 'int8' and res_dtype in + ['int8', 'uint8']): + pass + else: + raise RuntimeError("Not support in/out data type for quantize.") + + if quantize_config not in ([1, 0, 0], [1, 1, 0], [1, 0, 1], [1, 1, 1]): + raise RuntimeError("Invalid Quantize Config.") + + if scale_sqrt not in ([0, 0, 0], [1, 0, 0], [0, 1, 0], [1, 1, 0], [0, 0, 1], + [1, 0, 1], [0, 1, 1], [1, 1, 1]): + raise RuntimeError("Invalid Quantize Config.") + + # quantize switch off + elif quantize_config[0] == 0: + if quantize_config != [0, 0, 0]: + raise RuntimeError("Invalid Quantize Config.") + if scale_sqrt != [0, 0, 0]: + raise RuntimeError("Invalid Quantize Config.") + else: + raise RuntimeError("Invalid Quantize Config.") + + if isinstance(padh, list): + if len(padh) != PAD_SHAPE_DIM: + raise RuntimeError("Dimension must be %d when padh is a list."%PAD_SHAPE_DIM) + pad_top = padh[0] + pad_bottom = padh[1] + else: + pad_top = padh + pad_bottom = padh + + if isinstance(padw, list): + if len(padw) != PAD_SHAPE_DIM: + raise RuntimeError("Dimension must be %d when padw is a list."%PAD_SHAPE_DIM) + pad_left = padw[0] + pad_right = padw[1] + else: + pad_left = padw + pad_right = padw + + shape_in, shape_w = te.lang.cce.check_conv_shape(shape_in, shape_w, pad_top, pad_bottom, \ + pad_left, pad_right, strideh, \ + stridew, in_dtype, w_dtype, res_dtype) + + return shape_in, shape_w + + +@util.check_input_type((list, tuple), (list, tuple), str, str, str, \ + (list, int), (list, int), int, int, + (list, NONETYPE), (list, NONETYPE), + str, str, str, + str, str, str, str, + bool, str, bool, bool) +def conv_layer_cce(shape_in, shape_w, in_dtype, w_dtype, res_dtype, padh, padw, strideh, stridew, + quantize_config=None, scale_sqrt=None, + scale_q_dtype='float16', offset_q_dtype='float16', scale_dq_dtype='float16', + scale_rq_dtype='float16', offset_rq_dtype='float16', offset_w_dtype='int32', + offset_pad_dtype='uint8', bias=False, kernel_name="cce_conv", need_build=False, + need_print=False): + """ + + Parameters + ---------- + shape_in : shape of data_in + + shape_w : shape of filter + + in_dtype : the feature map data type + + w_dtype : the weight data type + + res_dtype : the result data type + + padh: the padding shape in H + + padw: the padding shape in weight + + strideh: the stride value in H + + stridew: the stride value in weight + + quantize_config: quantize config table, default [0, 0, 0] + quantize_config[0] - quantize function switch + 0: quantize off + 1: quantize on + quantize_config[1] - quantize_algorithm + 0: non offset + 1: half offset + 2: all offset ( Not supported now ) + quantize_config[2] - QuantizeScaleType (for Dequantize/Requantize, quantize always scalar) + 0: scalar + 1: vector + + scale_sqrt: scale mode + scale_sqrt[0] - Quantize scale mode + 0: non sqrt + 1: sqrt + scale_sqrt[1] - DeQuantize scale mode + 0: non sqrt + 1: sqrt + scale_sqrt[2] - ReQuantize scale mode + 0: non sqrt + 1: sqrt + + scale_q_dtype: Quantize scale data type, default 'float16' + + offset_q_dtype: Quantize offset data type, default 'float16' + + scale_dq_dtype: DeQuantize scale data type, default 'float16' + + scale_rq_dtype: ReQuantize scale data type, default 'float16' + + offset_rq_dtype: ReQuantize offset data type, default 'float16' + + offset_w_dtype: weight offset data type, default 'int32' + + offset_pad_dtype: Quantize Cube offset data type, default 'uint8' + + bias: the tag for bias or not + + kernel_name : cce kernel name, default value is "cce_conv" + + need_build : if need to build CCEC kernel, default value is False + + need_print : if need to print the ir, default value is False + + Returns + ------- + wrapped_tensor + + """ + # for pylint, otherwise "Dangerous default value [] as argument" + if quantize_config is None: + quantize_config = [0, 0, 0] + if scale_sqrt is None: + scale_sqrt = [0, 0, 0] + + in_dtype = in_dtype.lower() + w_dtype = w_dtype.lower() + res_dtype = res_dtype.lower() + scale_q_dtype = scale_q_dtype.lower() + offset_q_dtype = offset_q_dtype.lower() + scale_dq_dtype = scale_dq_dtype.lower() + scale_rq_dtype = scale_rq_dtype.lower() + offset_rq_dtype = offset_rq_dtype.lower() + offset_w_dtype = offset_w_dtype.lower() + offset_pad_dtype = offset_pad_dtype.lower() + + mad_dtype = 'float32' + if w_dtype == 'int8': + mad_dtype = 'int32' + + shape_in = list(shape_in) + shape_w = list(shape_w) + + shape_in, shape_w = conv_layer_cce_para_check(shape_in, shape_w, in_dtype, w_dtype, res_dtype, padh, padw, strideh, stridew, + quantize_config, scale_sqrt, scale_q_dtype, offset_q_dtype, scale_dq_dtype, + scale_rq_dtype, offset_rq_dtype, offset_w_dtype, offset_pad_dtype, bias, kernel_name) + + # quantize switch on + if quantize_config[0] == 1: + quantize_turn_on = True + # quantize -> DeQuantize dataflow + if in_dtype == 'float16' and w_dtype == 'int8' and res_dtype == 'float16': + is_quantize = True + is_dequantize = True + is_requantize = False + # DeQuantize dataflow + elif (in_dtype in ['int8', 'uint8'] and w_dtype == 'int8' and + res_dtype == 'float16'): + is_quantize = False + is_dequantize = True + is_requantize = False + # quantize -> ReQuantize dataflow + elif (in_dtype == 'float16' and w_dtype == 'int8' and res_dtype in + ['int8', 'uint8']): + is_quantize = True + is_dequantize = False + is_requantize = True + # ReQuantize dataflow + elif (in_dtype in ['int8', 'uint8'] and w_dtype == 'int8' and res_dtype in + ['int8', 'uint8']): + is_quantize = False + is_dequantize = False + is_requantize = True + else: + raise RuntimeError("Not support in/out data type for quantize.") + + # quantize switch off + elif quantize_config[0] == 0: + quantize_turn_on = False + is_quantize = False + is_dequantize = False + is_requantize = False + + if quantize_config != [0, 0, 0]: + raise RuntimeError("Invalid Quantize Config.") + if scale_sqrt != [0, 0, 0]: + raise RuntimeError("Invalid Quantize Config.") + else: + raise RuntimeError("Invalid Quantize Config.") + + batch_size = shape_in[0] + in_channel = shape_in[1] + feature_map_h = shape_in[2] + feature_map_w = shape_in[3] + block_size_k = CUBE_MKN[in_dtype]['mac'][1] + fmap_shape_nc1hwc0 = (batch_size, (in_channel + block_size_k - 1) // block_size_k, + feature_map_h, feature_map_w, block_size_k) + + out_channel = shape_w[0] + in_channel_weight = shape_w[1] + filter_h = shape_w[2] + filter_w = shape_w[3] + block_size_k = CUBE_MKN[w_dtype]['mac'][1] + block_size_n = CUBE_MKN[w_dtype]['mac'][2] + filter_shape_frac_z = (in_channel_weight * filter_h * filter_w // block_size_k, + out_channel // block_size_n, block_size_n, block_size_k) + + with tvm.target.cce(): + data = tvm.placeholder( + fmap_shape_nc1hwc0, name='Fmap', dtype=in_dtype) + weight = tvm.placeholder( + filter_shape_frac_z, name='Filter', dtype=w_dtype) + bias_tensor = None + scale_q = None + scale_dq = None + scale_rq = None + offset_pad = None + offset_rq = None + offset_q = None + scale_drq = None + + # bias or fusion_bias(half offset) + if bias or (quantize_config[1] == 1 and quantize_turn_on): + bias_tensor = tvm.placeholder( + (out_channel,), name='bias_tensor', \ + dtype="int32" if quantize_turn_on else res_dtype) + + # quantize on + if quantize_turn_on: + quantize_algorithm = quantize_config[1] + if is_quantize: + scale_q = tvm.placeholder( + (CUBE_MKN[scale_q_dtype]['mac'][1],), name='scaleQ', dtype=scale_q_dtype) + if quantize_algorithm ==1: + offset_q = tvm.placeholder( + (CUBE_MKN[offset_q_dtype]['mac'][1],), name='offsetQ', dtype=offset_q_dtype) + + if is_dequantize: + scale_dq_shape = (CUBE_MKN[scale_dq_dtype]['mac'][1],) if quantize_config[2] == 0 \ + else (out_channel,) + scale_dq = tvm.placeholder( + scale_dq_shape, name='scaleDq', dtype=scale_dq_dtype) + + if is_requantize: + scale_rq_shape = (CUBE_MKN[scale_rq_dtype]['mac'][1],) if quantize_config[2] == 0 \ + else (out_channel,) + scale_rq = tvm.placeholder( + scale_rq_shape, name='scaleRq', dtype=scale_rq_dtype) + if quantize_algorithm ==1: + offset_rq_shape = (CUBE_MKN[offset_rq_dtype]['mac'][1],) + offset_rq = tvm.placeholder( + offset_rq_shape, name='offsetRq', dtype=offset_rq_dtype) + + # need offset_pad , for half offset + if quantize_algorithm ==1: + offset_pad = tvm.placeholder( + (CUBE_MKN[offset_pad_dtype]['mac'][1],), name='offset_pad', + dtype=offset_pad_dtype) + + if quantize_algorithm == 0: + if is_quantize: + if is_dequantize: + scale_drq = scale_dq + else: + scale_drq = scale_rq + + conv_res = te.lang.cce.conv( + data, weight, {"bias_tensor": bias_tensor, + "scale_q": scale_q, + "offset_q": offset_q, + "scale_drq": scale_drq, + "offset_pad": offset_pad, + "offset_rq": offset_rq, + "quantize_config": quantize_config, + "is_quantize": is_quantize, + "is_dequantize": is_dequantize, + "is_requantize": is_requantize, + "scale_sqrt": scale_sqrt, + "pad_h": padh, "pad_w": padw, + "stride_h": strideh, "stride_w": stridew, + "filter_h": filter_h, "filter_w": filter_w, + "res_dtype": res_dtype, "mad_dtype": mad_dtype}, + dsl_flag=False) + if bias: + tensor_list = [data, weight, bias_tensor, scale_q, + scale_drq, conv_res] + else: + tensor_list = [data, weight, scale_q, + scale_drq, conv_res] + else: + if is_dequantize: + scale_drq = scale_dq + else: + scale_drq = scale_rq + conv_res = te.lang.cce.conv( + data, weight, {"bias_tensor": bias_tensor, + "scale_q": scale_q, + "offset_q": offset_q, + "scale_drq": scale_drq, + "offset_pad": offset_pad, + "offset_rq": offset_rq, + "quantize_config": quantize_config, + "is_quantize": is_quantize, + "is_dequantize": is_dequantize, + "is_requantize": is_requantize, + "scale_sqrt": scale_sqrt, + "pad_h": padh, "pad_w": padw, + "stride_h": strideh, "stride_w": stridew, + "filter_h": filter_h, "filter_w": filter_w, + "res_dtype": res_dtype, "mad_dtype": mad_dtype}, + dsl_flag=False) + if bias: + tensor_list = [data, weight, bias_tensor, + scale_drq, conv_res] + else: + tensor_list = [data, weight, + scale_drq, conv_res] + + # half offset + else: + if is_quantize: + if is_dequantize: + scale_drq = scale_dq + else: + scale_drq = scale_rq + conv_res = te.lang.cce.conv( + data, weight, {"bias_tensor": bias_tensor, + "scale_q": scale_q, + "offset_q": offset_q, + "scale_drq": scale_drq, + "offset_pad": offset_pad, + "offset_rq": offset_rq, + "quantize_config": quantize_config, + "is_quantize": is_quantize, + "is_dequantize": is_dequantize, + "is_requantize": is_requantize, + "scale_sqrt": scale_sqrt, + "pad_h": padh, "pad_w": padw, + "stride_h": strideh, "stride_w": stridew, + "filter_h": filter_h, "filter_w": filter_w, + "res_dtype": res_dtype, "mad_dtype": mad_dtype}, + dsl_flag=False) + if is_dequantize: + tensor_list = [data, weight, bias_tensor, scale_q, offset_q, + scale_drq, offset_pad, conv_res] + else: + tensor_list = [data, weight, bias_tensor, scale_q, offset_q, + scale_drq, offset_rq, offset_pad, conv_res] + else: + if is_dequantize: + scale_drq = scale_dq + else: + scale_drq = scale_rq + conv_res = te.lang.cce.conv( + data, weight, {"bias_tensor": bias_tensor, + "scale_q": scale_q, + "offset_q": offset_q, + "scale_drq": scale_drq, + "offset_pad": offset_pad, + "offset_rq": offset_rq, + "quantize_config": quantize_config, + "is_quantize": is_quantize, + "is_dequantize": is_dequantize, + "is_requantize": is_requantize, + "scale_sqrt": scale_sqrt, + "pad_h": padh, "pad_w": padw, + "stride_h": strideh, "stride_w": stridew, + "filter_h": filter_h, "filter_w": filter_w, + "res_dtype": res_dtype, "mad_dtype": mad_dtype}, + dsl_flag=False) + if is_dequantize: + tensor_list = [data, weight, bias_tensor, + scale_drq, offset_pad, conv_res] + else: + tensor_list = [data, weight, bias_tensor, + scale_drq, offset_rq, offset_pad, conv_res] + else: + conv_res = te.lang.cce.conv( + data, weight, {"bias_tensor": bias_tensor, + "scale_q": scale_q, + "offset_q": offset_q, + "scale_drq": scale_drq, + "offset_pad": offset_pad, + "offset_rq": offset_rq, + "quantize_config": quantize_config, + "is_quantize": is_quantize, + "is_dequantize": is_dequantize, + "is_requantize": is_requantize, + "scale_sqrt": scale_sqrt, + "pad_h": padh, "pad_w": padw, + "stride_h": strideh, "stride_w": stridew, + "filter_h": filter_h, "filter_w": filter_w, + "res_dtype": res_dtype, "mad_dtype": mad_dtype}, + dsl_flag=False) + if bias: + tensor_list = [data, weight, bias_tensor, conv_res] + else: + tensor_list = [data, weight, conv_res] + sch = generic.auto_schedule(conv_res) + + config = { + "print_ir": need_print, + "need_build": need_build, + "name": kernel_name, + "tensor_list": tensor_list + } + + te.lang.cce.cce_build_code(sch, config) diff --git a/tests/st/ops/custom_ops_tbe/conv_layer_fast.py b/tests/st/ops/custom_ops_tbe/conv_layer_fast.py new file mode 100755 index 00000000000..37b7d804243 --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/conv_layer_fast.py @@ -0,0 +1,177 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import te.lang.cce +from te import tvm +from te.platform import CUBE_MKN +from topi import generic +from topi.cce import util +# pylint: disable=R0913,R0914,R0915,E1101 +# the dim of shape in conv must be 4 +PAD_SHAPE_DIM = 2 + +NoneType = type(None) + +@util.check_input_type((list, tuple), (list, tuple), str, str, str, + (list, int), (list, int), int, int, bool, str) +def conv_layer_fast_cce_para_check(shape_in, shape_w, in_dtype, w_dtype, res_dtype, + padh, padw, strideh, stridew, bias, kernel_name): + # conv shape check + util.check_kernel_name(kernel_name) + + # conv data type check + util.check_dtype_rule(in_dtype, ['float16']) + util.check_dtype_rule(w_dtype, ['float16']) + util.check_dtype_rule(res_dtype, ['float16']) + + if not isinstance(bias, bool): + raise RuntimeError("bias dtype should be bool.") + + if isinstance(padh, list): + if len(padh) != PAD_SHAPE_DIM: + raise RuntimeError("Dimension must be %d when padh is a list."%PAD_SHAPE_DIM) + pad_top = padh[0] + pad_bottom = padh[1] + else: + pad_top = padh + pad_bottom = padh + + if isinstance(padw, list): + if len(padw) != PAD_SHAPE_DIM: + raise RuntimeError("Dimension must be %d when padw is a list."%PAD_SHAPE_DIM) + pad_left = padw[0] + pad_right = padw[1] + else: + pad_left = padw + pad_right = padw + + shape_in, shape_w = te.lang.cce.check_conv_shape(shape_in, shape_w, pad_top, pad_bottom, + pad_left, pad_right, strideh, stridew, + in_dtype, w_dtype, res_dtype) + + return shape_in, shape_w + +@util.check_input_type((list, tuple), (list, tuple), str, str, str, + (list, int), (list, int), int, int, + bool, str, bool, bool) +def conv_layer_fast_cce(shape_in, shape_w, in_dtype, w_dtype, res_dtype, + padh, padw, strideh, stridew, bias=False, + kernel_name="cce_conv", + need_build=False, need_print=False): + """ + + Parameters + ---------- + shape_in : shape of data_in + + shape_w : shape of filter + + in_dtype : the feature map data type + + w_dtype : the weight data type + + res_dtype : the result data type + + padh: the padding shape in H + + padw: the padding shape in weight + + strideh: the stride value in H + + stridew: the stride value in weight + + bias: the tag for bias or not + + kernel_name : cce kernel name, default value is "cce_conv" + + need_buid : if need to build CCEC kernel, default value is False + + need_print : if need to print the ir, default value is False + + Returns + ------- + None + + """ + in_dtype = in_dtype.lower() + w_dtype = w_dtype.lower() + res_dtype = res_dtype.lower() + + shape_in = list(shape_in) + shape_w = list(shape_w) + + shape_in, shape_w = conv_layer_fast_cce_para_check(shape_in, shape_w, in_dtype, w_dtype, res_dtype, + padh, padw, strideh, stridew, bias, kernel_name) + + batch_size = shape_in[0] + in_channel = shape_in[1] + feature_map_h = shape_in[2] + feature_map_w = shape_in[3] + block_size_k = CUBE_MKN[in_dtype]['mac'][1] + fmap_shape_nc1hwc0 = (batch_size, (in_channel + block_size_k - 1) // block_size_k, + feature_map_h, feature_map_w, block_size_k) + + out_channel = shape_w[0] + in_channel_weight = shape_w[1] + filter_h = shape_w[2] + filter_w = shape_w[3] + block_size_k = CUBE_MKN[w_dtype]['mac'][1] + block_size_n = CUBE_MKN[w_dtype]['mac'][2] + filter_shape_frac_z = (in_channel_weight * filter_h * filter_w // block_size_k, + out_channel // block_size_n, block_size_n, block_size_k) + + with tvm.target.cce(): + data = tvm.placeholder( + fmap_shape_nc1hwc0, name='Fmap', dtype=in_dtype) + weight = tvm.placeholder( + filter_shape_frac_z, name='Filter', dtype=w_dtype) + bias_tensor = None + + if bias: + bias_tensor = tvm.placeholder( + (out_channel,), name='bias_tensor', dtype=res_dtype) + + mad_dtype = "float16" + + conv_res = te.lang.cce.conv( + data, weight, {"bias_tensor": bias_tensor, + "scale_q": None, + "offset_q": None, + "scale_drq": None, + "offset_pad": None, + "offset_rq": None, + "quantize_config": [0, 0, 0], + "is_quantize": False, + "is_dequantize": False, + "is_requantize": False, + "scale_sqrt": [0, 0, 0], + "pad_h": padh, "pad_w": padw, + "stride_h": strideh, "stride_w": stridew, + "filter_h": filter_h, "filter_w": filter_w, + "res_dtype": res_dtype, "mad_dtype": mad_dtype}, + dsl_flag=False) + if bias: + tensor_list = [data, weight, bias_tensor, conv_res] + else: + tensor_list = [data, weight, conv_res] + sch = generic.auto_schedule(conv_res) + + config = { + "print_ir": need_print, + "need_build": need_build, + "name": kernel_name, + "tensor_list": tensor_list + } + + te.lang.cce.cce_build_code(sch, config) diff --git a/tests/st/ops/custom_ops_tbe/cus_conv2d.py b/tests/st/ops/custom_ops_tbe/cus_conv2d.py new file mode 100644 index 00000000000..120bf2e3e68 --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/cus_conv2d.py @@ -0,0 +1,150 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import math +from functools import reduce +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +from mindspore._checkparam import ParamValidator as validator +from mindspore._checkparam import Rel, check_bool, check_int_positive, twice +from mindspore.common import dtype as mstype +class Cus_Conv2D(PrimitiveWithInfer): + r""" + Applies 2D convolution for the input. + + Input is typically of shape :math:`(N, C, H, W)`, where :math:`N` is batch size and :math:`C` is channel number. + For each batch of shape :math:`(C, H, W)` the formula (given mode 1) is defined as: + + .. math:: + out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{ij}, X_i) + b_j, + + where :math:`ccor` is cross correlation operator, :math:`C_{in}` is the input channel number, :math:`j` ranges + from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to i-th channel of the j-th filter and + :math:`out_{j}` corresponds to the j-th channel of the output. + + The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition + `_. + + More detailed introduction can be found here: http://cs231n.github.io/convolutional-networks/. + + Args: + out_channel (int): The dimensionality of the output space. + kernel_size (Union[int, tuple[int]]): The kernel size of the 2D convolution. + mode (int): 0 Math convolutiuon, 1 cross-correlation convolution , + 2 deconvolution, 3 depthwise convolution. Default: 1. + pad_mode (str): "valid", "same", "pad" the mode to fill padding. Default: "valid". + pad (int): The pad value to fill. Default: 0. + stride (int): The stride to apply conv filter. Default: 1. + dilation (int): Specifying the dilation rate to use for dilated convolution. Default: 1. + group (int): Split input into groups. Default: 1. + + Returns: + Tensor, the value that applied 2D convolution. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + """ + + @prim_attr_register + def __init__(self, + out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1): + """init Conv2D""" + self.init_prim_io_names(inputs=['x', 'w'], outputs=['output']) + self.kernel_size = kernel_size + self.kernel_size = validator.check_type('kernel_size', kernel_size, (int, tuple)) + if isinstance(self.kernel_size, int): + self.kernel_size = (self.kernel_size, self.kernel_size) + validator.check_integer('length of kernel_size', len(self.kernel_size), 2, Rel.GE) + validator.equal('type of pad', type(pad), 'not bool', not isinstance(pad, bool)) + validator.equal('type of pad', type(pad), 'int', isinstance(pad, int)) + self.pad_mode = validator.check_string('pad_mode', pad_mode, ['valid', 'same', 'pad']) + self.pad = validator.check_pad_value_by_mode(self.__class__.__name__, pad_mode, pad) + if self.pad_mode == 'pad': + validator.check_integer('pad', self.pad, 0, Rel.GE) + + self.mode = validator.check_integer('mode', mode, 1, Rel.EQ) + self.add_prim_attr('data_format', "NCHW") + self.out_channel = validator.check_integer('out_channel', out_channel, 0, Rel.GT) + self.group = validator.check_integer('group', group, 0, Rel.GT) + self.dilation = validator.check_integer('dilation', dilation, 1, Rel.GE) + validator.check_type('kernel_size', kernel_size, [int, tuple]) + if isinstance(kernel_size, int) and kernel_size < 1: + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + + str(self.kernel_size)+', should be a int or tuple and equal to or greater than 1.') + if isinstance(kernel_size, tuple) and (len(kernel_size) != 2 or + (not isinstance(kernel_size[0], int)) or + (not isinstance(kernel_size[1], int)) or + kernel_size[0] < 1 or kernel_size[1] < 1): + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + + str(self.kernel_size)+', should be a int or tuple and equal to or greater than 1.') + self.stride = validator.check_integer('stride', stride, 1, Rel.GE) + from .cus_conv2d_impl import Cus_Conv2D + + def infer_shape(self, x_shape, w_shape): + validator.check_integer("weight_shape", len(w_shape), 4, Rel.EQ) + validator.check_integer("x_shape", len(x_shape), 4, Rel.EQ) + validator.check_param_equal("x_shape[1]", x_shape[1] // self.group, "w_shape[1]", w_shape[1]) + validator.check_param_equal('out_channel', self.out_channel, 'w_shape[0]', w_shape[0]) + validator.check_param_equal('kernel_size', self.kernel_size, 'w_shape[2:4]', tuple(w_shape[2:4])) + + kernel_size_h = w_shape[2] + kernel_size_w = w_shape[3] + + if self.pad_mode == "valid": + h_out = math.ceil((x_shape[2] - kernel_size_h + 1) / self.stride) + w_out = math.ceil((x_shape[3] - kernel_size_w + 1) / self.stride) + pad_top, pad_bottom, pad_left, pad_right = 0, 0, 0, 0 + elif self.pad_mode == "same": + h_out = math.ceil(x_shape[2] / self.stride) + w_out = math.ceil(x_shape[3] / self.stride) + + pad_needed_h = max(0, (h_out - 1) * self.stride + kernel_size_h - x_shape[2]) + pad_top = math.floor(pad_needed_h / 2) + pad_bottom = pad_needed_h - pad_top + + pad_needed_w = max(0, (w_out - 1) * self.stride + kernel_size_w - x_shape[3]) + pad_left = math.floor(pad_needed_w / 2) + pad_right = pad_needed_w - pad_left + elif self.pad_mode == 'pad': + pad_top, pad_bottom, pad_left, pad_right = self.pad, self.pad, self.pad, self.pad + + h_out = 1 + (x_shape[2] + 2 * self.pad - kernel_size_h - (kernel_size_h - 1) * (self.dilation - 1)) \ + / self.stride + w_out = 1 + (x_shape[3] + 2 * self.pad - kernel_size_w - (kernel_size_w - 1) * (self.dilation - 1)) \ + / self.stride + h_out = math.floor(h_out) + w_out = math.floor(w_out) + + self.pad_list = [pad_top, pad_bottom, pad_left, pad_right] + self.add_prim_attr('pad_list', (pad_top, pad_bottom, pad_left, pad_right)) + + out_channel = self.out_channel + out_shape = [x_shape[0], out_channel, h_out, w_out] + return out_shape + + def infer_dtype(self, x_dtype, w_dtype): + args = {'x_dtype': x_dtype, 'w_dtype': w_dtype} + validator.check_type_same(args, [mstype.int8, mstype.int32, mstype.float16, mstype.float32]) + return x_dtype \ No newline at end of file diff --git a/tests/st/ops/custom_ops_tbe/cus_conv2d_impl.py b/tests/st/ops/custom_ops_tbe/cus_conv2d_impl.py new file mode 100644 index 00000000000..54f6954a18e --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/cus_conv2d_impl.py @@ -0,0 +1,107 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from tests.st.ops.custom_ops_tbe.conv2d import conv2d +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "Cus_Conv2D", + "imply_type": "TBE", + "fusion_type": "CONVLUTION", + "async_flag": false, + "binfile_name": "conv2d.so", + "compute_cost": 10, + "kernel_name": "Cus_Conv2D", + "partial_flag": true, + "attr": [ + { + "name": "stride", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "pad_list", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "dilation", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FracZ" + ], + "name": "filter", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "bias", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "y", + "need_compile": true, + "param_type": "required", + "shape": "all" + } + ] +}""") +def Cus_Conv2D(inputs, weights, bias, outputs, strides, pads, dilations, + kernel_name="conv2d"): + conv2d(inputs, weights, bias, outputs, strides, pads, dilations, + kernel_name) \ No newline at end of file diff --git a/tests/st/ops/custom_ops_tbe/cus_square.py b/tests/st/ops/custom_ops_tbe/cus_square.py new file mode 100644 index 00000000000..6a9e769f51f --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/cus_square.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor + + +# y = x^2 +class CusSquare(PrimitiveWithInfer): + """CusSquare definition""" + @prim_attr_register + def __init__(self): + """init CusSquare""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + from .square_impl import CusSquare + + def vm_impl(self, x): + x = x.asnumpy() + return Tensor(np.multiply(x, x)) + + def infer_shape(self, data_shape): + return data_shape + + def infer_dtype(self, data_dtype): + return data_dtype diff --git a/tests/st/ops/custom_ops_tbe/square_impl.py b/tests/st/ops/custom_ops_tbe/square_impl.py new file mode 100644 index 00000000000..e5992eff1c3 --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/square_impl.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from __future__ import absolute_import +from te import tvm +from topi import generic +import te.lang.cce +from topi.cce import util +from te.platform.fusion_manager import fusion_manager +from mindspore.ops.op_info_register import op_info_register + +# shape size limit for aicore is 2**31 +SHAPE_SIZE_LIMIT = 200000000 + +@fusion_manager.register("square") +def square_compute(input_x, output_y, kernel_name="square"): + """ + algorithm: square + calculating data's square,y= x*x + + Parameters + ---------- + input_x: TVM tensor + the placeholder of input data + output_y: dict + shape and dtype of output, should be same shape and type as input + kernel_name: str + cce kernel name, default value is square + + Returns + ------- + res : tvm.tensor + the result of square + """ + res = te.lang.cce.vmul(input_x, input_x) + return res + +@op_info_register("""{ + "op_name": "CusSquare", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "square.so", + "compute_cost": 10, + "kernel_name": "CusSquare", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def CusSquare(input_x, output_y, kernel_name="square"): + """ + algorithm: square + calculating data's square,y= x*x + + Parameters + ---------- + input_x : dict + shape and dtype of input, only support float32 + output_y: dict + shape and dtype of output, should be same shape and type as input + kernel_name : str + kernel name, default value is "square" + + Returns + ------- + None + """ + shape = input_x.get("shape") + dtype = input_x.get("dtype").lower() + + shape = util.shape_refine(shape) + data = tvm.placeholder(shape, name="data", dtype=dtype.lower()) + + with tvm.target.cce(): + res = square_compute(data, output_y, kernel_name) + sch = generic.auto_schedule(res) + + config = {"print_ir": False, + "name": kernel_name, + "tensor_list": [data, res]} + + te.lang.cce.cce_build_code(sch, config) diff --git a/tests/st/ops/custom_ops_tbe/test_cus_conv.py b/tests/st/ops/custom_ops_tbe/test_cus_conv.py new file mode 100644 index 00000000000..4927417306b --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/test_cus_conv.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from .cus_conv2d import Cus_Conv2D +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + out_channel = 64 + kernel_size = 7 + self.conv = Cus_Conv2D(out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer( + 'normal', [64, 3, 7, 7]), name='w') + + + @ms_function + def construct(self, x): + return self.conv(x, self.w) + +def test_net(): + np.random.seed(3800) + x = np.random.randn(32,3,224,224).astype(np.float32) + conv = Net() + output = conv(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/custom_ops_tbe/test_square.py b/tests/st/ops/custom_ops_tbe/test_square.py new file mode 100644 index 00000000000..c67edae3077 --- /dev/null +++ b/tests/st/ops/custom_ops_tbe/test_square.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor +from .cus_square import CusSquare +import pytest +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Net(nn.Cell): + """Net definition""" + + def __init__(self): + super(Net, self).__init__() + self.square = CusSquare() + + def construct(self, data): + return self.square(data) + +@pytest.mark.level0 +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_net(): + x = np.array([1.0, 4.0, 9.0]).astype(np.float32) + square = Net() + output = square(Tensor(x)) + print(x) + print(output.asnumpy()) + expect = np.array([1.0,16.0,81.0]).astype(np.float32) + assert (output.asnumpy() == expect).all() \ No newline at end of file diff --git a/tests/st/ops/davinci/test_add.py b/tests/st/ops/davinci/test_add.py new file mode 100644 index 00000000000..659bb59466f --- /dev/null +++ b/tests/st/ops/davinci/test_add.py @@ -0,0 +1,41 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(enable_task_sink=True) +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + + def construct(self, x, y): + return self.add(x, y) + +x = np.ones([1,3,3,4]).astype(np.float32) +y = np.ones([1,3,3,4]).astype(np.float32) + +def test_net(): + add = Net() + output = add(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_addn.py b/tests/st/ops/davinci/test_addn.py new file mode 100644 index 00000000000..143dc616d0b --- /dev/null +++ b/tests/st/ops/davinci/test_addn.py @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.AddN() + + def construct(self, x, y): + return self.add((x, y)) + +def test_net(): + x = np.random.randn(1, 3, 3, 4).astype(np.float32) + y = np.random.randn(1, 3, 3, 4).astype(np.float32) + add = Net() + output = add(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) + x = 1.0 + y = 2.0 + expect = 3.0 + add = Net() + output = add(x, y) + assert (output == expect) diff --git a/tests/st/ops/davinci/test_apply_momentum.py b/tests/st/ops/davinci/test_apply_momentum.py new file mode 100644 index 00000000000..885356ce487 --- /dev/null +++ b/tests/st/ops/davinci/test_apply_momentum.py @@ -0,0 +1,44 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.apply_momentum = P.ApplyMomentum(gradient_scale=1024.0) + self.variable = Parameter(initializer( + 'normal', [2, 3, 3, 4]), name='variable') + self.accumulation = Parameter(initializer( + 'normal', [2, 3, 3, 4]), name='accumulation') + self.learning_rate = Parameter(initializer( + 'normal', [1, ]), name='learning_rate') + self.gradient = Parameter(initializer( + 'normal', [2, 3, 3, 4]), name='gradient') + self.momentum = Parameter(initializer( + 'normal', [1, ]), name='momentum') + def construct(self): + return self.apply_momentum(self.variable, self.accumulation, self.learning_rate, self.gradient, self.momentum) + +def test_net(): + apply_momentum = Net() + output = apply_momentum() + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_argmax.py b/tests/st/ops/davinci/test_argmax.py new file mode 100644 index 00000000000..d7d1aa46d2a --- /dev/null +++ b/tests/st/ops/davinci/test_argmax.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.argmax = P.Argmax(axis=1) + + @ms_function + def construct(self, x): + return self.argmax(x) + + +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + argmax = Net() + output = argmax(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_biasAddGrad.py b/tests/st/ops/davinci/test_biasAddGrad.py new file mode 100644 index 00000000000..29b63ac3364 --- /dev/null +++ b/tests/st/ops/davinci/test_biasAddGrad.py @@ -0,0 +1,42 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bias_add_grad = G.BiasAddGrad() + #self.dout = Parameter(initializer( + #'normal', [2, 3, 3, 4]), name='dout') + + + @ms_function + def construct(self, dout): + return self.bias_add_grad(dout) + +dout = np.ones([2,3,4,4]).astype(np.float32) +bias_add_grad = Net() +output = bias_add_grad(Tensor(dout)) +expect_output = np.array([32.,32.,32.]).astype(np.float32) +assert np.all(output.asnumpy()==expect_output), "bias_add_grad execute failed, please check current code commit" +print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_bias_add_grad.py b/tests/st/ops/davinci/test_bias_add_grad.py new file mode 100644 index 00000000000..e58d376e935 --- /dev/null +++ b/tests/st/ops/davinci/test_bias_add_grad.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bias_add_grad = G.BiasAddGrad() + + + @ms_function + def construct(self, dout): + return self.bias_add_grad(dout) + +def test_net(): + dout = np.random.rand(1, 1001).astype(np.float32) + bias_add_grad = Net() + output = bias_add_grad(dout) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_conv.py b/tests/st/ops/davinci/test_conv.py new file mode 100644 index 00000000000..f0f161da383 --- /dev/null +++ b/tests/st/ops/davinci/test_conv.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + out_channel = 64 + kernel_size = 7 + self.conv = P.Conv2D(out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer( + 'normal', [64, 3, 7, 7]), name='w') + + + @ms_function + def construct(self, x): + return self.conv(x, self.w) + + + +def test_net(): + x = np.random.randn(32,3,224,224).astype(np.float32) + conv = Net() + output = conv(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_conv2dGradFilter.py b/tests/st/ops/davinci/test_conv2dGradFilter.py new file mode 100644 index 00000000000..73993c2ee56 --- /dev/null +++ b/tests/st/ops/davinci/test_conv2dGradFilter.py @@ -0,0 +1,57 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.conv2d_grad = G.Conv2DBackpropFilter(4,1) + yt = Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)) + self.y = Parameter(yt, name='y') + self.get_shape = P.Shape() + + @ms_function + def construct(self, x, out): + return self.conv2d_grad(out, x, self.get_shape(self.y)) + +x = Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)) + +out = Tensor(np.array([[[ + [ -5, -4, 0, 8], + [-10, -2, 2, 3], + [ 0, -2, -4, -7], + [ -3, -2, -3, -16]]]]).astype(np.float32)) + +operator = Net() +output = operator(x, out) +expect_out = np.array([[[[ -60., -142., -265.],[-104., -211., -322.],[-102., -144., -248.]]]]).astype(np.float32) +print(output.asnumpy()) +print(expect_out) +assert np.all(output.asnumpy()==expect_out), "conv2d_grad execute failed, please check current code commit" diff --git a/tests/st/ops/davinci/test_conv_grad.py b/tests/st/ops/davinci/test_conv_grad.py new file mode 100644 index 00000000000..b693e7fe809 --- /dev/null +++ b/tests/st/ops/davinci/test_conv_grad.py @@ -0,0 +1,61 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + out_channel = 512 + kernel_size = 2048 + self.conv = P.Conv2D(out_channel, + (kernel_size, kernel_size), + mode=1, + pad_mode="same", + pad=3, + stride=2, + dilation=1, + group=1) + self.w = Parameter(initializer( + 'normal', [512, 2048, 1, 1]), name='w') + + @ms_function + def construct(self, x): + return self.conv(x, self.w) + +def test_net(): + x = np.ones([32, 2048, 7, 7]).astype(np.float32) + sens = np.ones([32, 512, 7, 7]).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_dense.py b/tests/st/ops/davinci/test_dense.py new file mode 100644 index 00000000000..2866ba9242d --- /dev/null +++ b/tests/st/ops/davinci/test_dense.py @@ -0,0 +1,38 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.dense = nn.Dense(2048, 1001) + + @ms_function + def construct(self, x): + return self.dense(x) + +def test_net(): + x = np.random.randn(32, 2048).astype(np.float32) + net = Net() + output = net(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_dense_grad.py b/tests/st/ops/davinci/test_dense_grad.py new file mode 100644 index 00000000000..a5ee37842e1 --- /dev/null +++ b/tests/st/ops/davinci/test_dense_grad.py @@ -0,0 +1,49 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.dense = nn.Dense(2048, 1001) + + def construct(self, x): + return self.dense(x) + +def test_net(): + x = np.random.randn(32, 2048).astype(np.float32) + sens = np.random.randn(32, 1001).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_drop_out_gen_mask.py b/tests/st/ops/davinci/test_drop_out_gen_mask.py new file mode 100644 index 00000000000..4d7c5552198 --- /dev/null +++ b/tests/st/ops/davinci/test_drop_out_gen_mask.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.mask = P.DropoutGenMask(10, 28) + self.shape = P.Shape() + + def construct(self, x, y): + shape_x = self.shape(x) + return self.mask(shape_x, y) + + +x = np.ones([2, 4, 2, 2]).astype(np.int32) +y = np.array([1.0]).astype(np.float32) + + +def test_net(): + mask = Net() + tx, ty = Tensor(x), Tensor(y) + output = mask(tx, ty) + print(output.asnumpy()) + assert ([255, 255, 255, 255] == output.asnumpy()).all() diff --git a/tests/st/ops/davinci/test_equal_count.py b/tests/st/ops/davinci/test_equal_count.py new file mode 100644 index 00000000000..6790fd7b6ee --- /dev/null +++ b/tests/st/ops/davinci/test_equal_count.py @@ -0,0 +1,42 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.equal_count = P.EqualCount() + + def construct(self, x, y): + return self.equal_count(x, y) + + +x = np.random.randn(32).astype(np.int32) +y = np.random.randn(32).astype(np.int32) + + +def test_net(): + equal_count = Net() + output = equal_count(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_full_connection.py b/tests/st/ops/davinci/test_full_connection.py new file mode 100644 index 00000000000..d6aec19aa70 --- /dev/null +++ b/tests/st/ops/davinci/test_full_connection.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul(transpose_b=True) + self.bias_add = P.BiasAdd() + + @ms_function + def construct(self, x, w, b): + return self.bias_add(self.matmul(x, w), b) + +# def test_net(): +# x = np.random.randn(32, 2048).astype(np.float16) +# w = np.random.randn(1001, 2048).astype(np.float16) +# b = np.random.randn(1001).astype(np.float16) +# FullConnection = Net() +# output = FullConnection(Tensor(x), Tensor(w), Tensor(b)) +# print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_fused_batchnorm.py b/tests/st/ops/davinci/test_fused_batchnorm.py new file mode 100644 index 00000000000..5d02e716b3a --- /dev/null +++ b/tests/st/ops/davinci/test_fused_batchnorm.py @@ -0,0 +1,48 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bn = P.FusedBatchNorm() + self.scale = Parameter(initializer('ones', [64]), name='scale') + self.b = Parameter(initializer('zeros', [64]), name='b') + self.mean = Parameter(initializer('ones', [64]), name='mean') + self.variance = Parameter(initializer('zeros', [64]), name='variance') + + def construct(self, x): + return self.bn(x, self.scale, self.b, self.mean, self.variance)[0] + + +def test_net(): + x = np.random.randn(1,64,112,112).astype(np.float32) + # mean = np.random.randn(1,16,1,1).astype(np.float32) + # variance = np.random.randn(1,16,1,1).astype(np.float32) + fusedBn = Net() + output = fusedBn(Tensor(x)) + print("***********x*********") + print(x) + + print("***********output y*********") + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_fused_batchnorm_grad.py b/tests/st/ops/davinci/test_fused_batchnorm_grad.py new file mode 100644 index 00000000000..b61003b30e3 --- /dev/null +++ b/tests/st/ops/davinci/test_fused_batchnorm_grad.py @@ -0,0 +1,58 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +#context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(device_target="Ascend") +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bn = P.FusedBatchNorm() + self.scale = Parameter(initializer('ones', [64]), name='scale') + self.b = Parameter(initializer('zeros', [64]), name='b') + self.mean = Parameter(initializer('ones', [64]), name='mean') + self.variance = Parameter(initializer('zeros', [64]), name='variance') + + def construct(self, x): + return self.bn(x, self.scale, self.b, self.mean, self.variance)[0] + + +def test_net(): + x = np.random.randn(1,64,112,112).astype(np.float32) + sens = np.random.randn(1,64,112,112).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print("***********x*********") + print(x) + + print("***********output y*********") + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_matmul.py b/tests/st/ops/davinci/test_matmul.py new file mode 100644 index 00000000000..c5dfc13af1b --- /dev/null +++ b/tests/st/ops/davinci/test_matmul.py @@ -0,0 +1,41 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul() + + @ms_function + def construct(self, x1, x2): + return self.matmul(x1, x2) + +x1 = np.random.randn(1,3).astype(np.float32) +x2 = np.random.randn(3,4).astype(np.float32) + +def test_net(): + matmul = Net() + output = matmul(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_maxpool.py b/tests/st/ops/davinci/test_maxpool.py new file mode 100644 index 00000000000..3b9ecc29d86 --- /dev/null +++ b/tests/st/ops/davinci/test_maxpool.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.maxpool = P.MaxPool(pad_mode="SAME", window=3, stride=2) + + + @ms_function + def construct(self, x): + output = self.maxpool(x) + return output + + +def test_net(): + x = np.random.randn(32,64,112,112).astype(np.float32) + maxpool = Net() + output = maxpool(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_maxpool_grad.py b/tests/st/ops/davinci/test_maxpool_grad.py new file mode 100644 index 00000000000..e3d845707ca --- /dev/null +++ b/tests/st/ops/davinci/test_maxpool_grad.py @@ -0,0 +1,54 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.maxpool = P.MaxPool(pad_mode="SAME", window=3, stride=2) + + @ms_function + def construct(self, x): + output = self.maxpool(x) + return output + + +def test_net(): + x = np.random.randn(32, 64, 112, 112).astype(np.float32) + output_grad = np.random.randn(32, 64, 56, 56).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(output_grad)) + if isinstance(output, (tuple, list)): + output = output[0] + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_maxpool_with_argmax.py b/tests/st/ops/davinci/test_maxpool_with_argmax.py new file mode 100644 index 00000000000..c9312d666ca --- /dev/null +++ b/tests/st/ops/davinci/test_maxpool_with_argmax.py @@ -0,0 +1,46 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + self.maxpool = P.MaxPoolWithArgmax(pad_mode="same", + window=3, + stride=2) + self.x = Parameter(initializer( + 'normal', [1, 64, 112, 112]), name='w') + self.add = P.TensorAdd() + + + @ms_function + def construct(self): + output = self.maxpool(self.x) + return output[0] + +def test_net(): + x = np.random.randn(1,64,112,112).astype(np.float32) + maxpool = Net() + output = maxpool() + print("***********output output*********") + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_maxpool_with_argmax_grad.py b/tests/st/ops/davinci/test_maxpool_with_argmax_grad.py new file mode 100644 index 00000000000..d97e2a06f84 --- /dev/null +++ b/tests/st/ops/davinci/test_maxpool_with_argmax_grad.py @@ -0,0 +1,54 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + self.maxpool = P.MaxPoolWithArgmax(pad_mode="same", + window=3, + stride=2) + + @ms_function + def construct(self, x): + output = self.maxpool(x) + return output[0] + +def test_net(): + x = np.random.randn(32, 64, 112, 112).astype(np.float32) + sens = np.random.randn(32, 64, 56, 56).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_relu.py b/tests/st/ops/davinci/test_relu.py new file mode 100644 index 00000000000..2e8aa46c241 --- /dev/null +++ b/tests/st/ops/davinci/test_relu.py @@ -0,0 +1,38 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.relu = P.ReLU(strategy=None) + + @ms_function + def construct(self, x): + return self.relu(x) + +def test_net(): + x = np.random.randn(2,3,3,4).astype(np.float32) + relu = Net() + output = relu(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_relu_grad.py b/tests/st/ops/davinci/test_relu_grad.py new file mode 100644 index 00000000000..dd13544b801 --- /dev/null +++ b/tests/st/ops/davinci/test_relu_grad.py @@ -0,0 +1,50 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.relu = P.ReLU(strategy=None) + + def construct(self, x): + return self.relu(x) + +def test_net(): + x = np.random.randn(2,3,3,4).astype(np.float32) + sens = np.random.randn(2,3,3,4).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print(len(output)) + print(output[0].asnumpy()) diff --git a/tests/st/ops/davinci/test_reshape.py b/tests/st/ops/davinci/test_reshape.py new file mode 100644 index 00000000000..c0f9cf4d9c5 --- /dev/null +++ b/tests/st/ops/davinci/test_reshape.py @@ -0,0 +1,35 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.reshape = P.Reshape() + @ms_function + def construct(self, tensor): + return self.reshape(tensor, (1,16)) + + +def test_net(): + x = np.random.randn(1, 16, 1, 1).astype(np.float16) + reshape = Net() + output = reshape(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_simplemean.py b/tests/st/ops/davinci/test_simplemean.py new file mode 100644 index 00000000000..dbc7a6add96 --- /dev/null +++ b/tests/st/ops/davinci/test_simplemean.py @@ -0,0 +1,37 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.simplemean = P.ReduceMean(keep_dims=True) + + @ms_function + def construct(self, x): + return self.simplemean(x, (-2, -1)) + +def test_net(): + x = np.random.randn(32, 2048, 7, 7).astype(np.float32) + simplemean = Net() + output = simplemean(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_simplemean_grad.py b/tests/st/ops/davinci/test_simplemean_grad.py new file mode 100644 index 00000000000..c5b37eb843c --- /dev/null +++ b/tests/st/ops/davinci/test_simplemean_grad.py @@ -0,0 +1,50 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.simplemean = P.ReduceMean(keep_dims=True) + + def construct(self, x): + return self.simplemean(x, (-2, -1)) + +def test_net(): + x = np.random.randn(32,2048,7,7).astype(np.float32) + sens = np.random.randn(32,2048, 1, 1).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print(output.asnumpy()) + print(output.asnumpy().shape) diff --git a/tests/st/ops/davinci/test_softmax.py b/tests/st/ops/davinci/test_softmax.py new file mode 100644 index 00000000000..67d4ac92d90 --- /dev/null +++ b/tests/st/ops/davinci/test_softmax.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.softmax = P.Softmax(axis=1) + + @ms_function + def construct(self, x): + return self.softmax(x) + + +def test_net(): + x = np.random.randn(32, 10).astype(np.float32) + softmax = Net() + output = softmax(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_sparseSoftmaxCrossEntropyWithLogits.py b/tests/st/ops/davinci/test_sparseSoftmaxCrossEntropyWithLogits.py new file mode 100644 index 00000000000..5f143ef037e --- /dev/null +++ b/tests/st/ops/davinci/test_sparseSoftmaxCrossEntropyWithLogits.py @@ -0,0 +1,68 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, is_grad=False): + super(Net, self).__init__() + self.SparseSoftmaxCrossEntropyWithLogits = P.SparseSoftmaxCrossEntropyWithLogits(is_grad=is_grad) + + @ms_function + def construct(self, features, labels): + return self.SparseSoftmaxCrossEntropyWithLogits(features, labels) + +def np_sparse_softmax_cross_entropy_with_logits(labels_shape, logits_shape, logits_dtype): + num_class = logits_shape[1] + labels = np.random.randint(low=0, high=num_class - 1, size=labels_shape).astype(np.int32) + logits = np.random.rand(*logits_shape).astype(logits_dtype) + features = logits + features_reshape = np.reshape(features, [-1, num_class]) + labels_reshape = np.reshape(labels, [-1]) + batch_dim = 0 + class_dim = 1 + batch_size = features_reshape.shape[batch_dim] + e = np.exp(features_reshape - np.reshape(np.amax(features_reshape, axis=class_dim), [batch_size, 1])) + probs = e / np.reshape(np.sum(e, axis=class_dim), [batch_size, 1]) + labels_mat = np.zeros_like(probs).astype(probs.dtype) + labels_mat[np.arange(batch_size), labels_reshape] = 1.0 + bp = (probs - labels_mat) + loss = -np.sum(labels_mat * np.log(probs + 1.0e-20), axis=1) + bp_res = np.reshape(bp, features.shape) + loss_res = np.reshape(loss, labels.shape) + loss_res = np.sum(loss_res, axis=0)/loss_res.shape[0] + return labels, logits, loss_res, bp_res + + +def test_net(): + '''Compare Numpy with MS type is float32''' + labels_shape = (32,) + logits_shape = [32, 1001] + labels, logits, loss_np, bp_np = np_sparse_softmax_cross_entropy_with_logits(labels_shape, logits_shape, np.float32) + expect = loss_np + SparseSoftmaxCrossEntropyWithLogits = Net() + loss_me = SparseSoftmaxCrossEntropyWithLogits(Tensor(logits), Tensor(labels)) + '''assert''' + assert np.allclose(expect.flatten(), loss_me.asnumpy().flatten(), 0.01, 0.01) + print(loss_me.asnumpy().flatten()) + print("-------------------------") + print(expect) +test_net() diff --git a/tests/st/ops/davinci/test_sparse_softmax_cross_entropy_with_logits.py b/tests/st/ops/davinci/test_sparse_softmax_cross_entropy_with_logits.py new file mode 100644 index 00000000000..292196e8cd2 --- /dev/null +++ b/tests/st/ops/davinci/test_sparse_softmax_cross_entropy_with_logits.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, is_grad=False): + super(Net, self).__init__() + self.SparseSoftmaxCrossEntropyWithLogits = P.SparseSoftmaxCrossEntropyWithLogits(is_grad=is_grad) + + @ms_function + def construct(self, features, labels): + return self.SparseSoftmaxCrossEntropyWithLogits(features, labels) + + +def test_net(): + features = np.random.randn(32, 1001).astype(np.float16) + labels = np.random.randn(32).astype(np.int32) + SparseSoftmaxCrossEntropyWithLogits = Net() + output = SparseSoftmaxCrossEntropyWithLogits(Tensor(features), Tensor(labels)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_sparse_softmax_cross_entropy_with_logits_grad.py b/tests/st/ops/davinci/test_sparse_softmax_cross_entropy_with_logits_grad.py new file mode 100644 index 00000000000..d36873d426b --- /dev/null +++ b/tests/st/ops/davinci/test_sparse_softmax_cross_entropy_with_logits_grad.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self, is_grad=False): + super(Net, self).__init__() + self.SparseSoftmaxCrossEntropyWithLogits = P.SparseSoftmaxCrossEntropyWithLogits(is_grad=is_grad) + + @ms_function + def construct(self, features, labels): + return self.SparseSoftmaxCrossEntropyWithLogits(features, labels) + + +def test_net(): + features = np.random.randn(32, 1001, 1, 1).astype(np.float16) + labels = np.random.randn(32, 1, 1, 1).astype(np.int32) + gradSparseSoftmaxCrossEntropyWithLogits = Net(is_grad=True) + gradOutput = gradSparseSoftmaxCrossEntropyWithLogits(Tensor(features), Tensor(labels)) + print(gradOutput[0].asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_AssignAdd.py b/tests/st/ops/davinci/test_tbe_ops/test_AssignAdd.py new file mode 100644 index 00000000000..59d8a0cadee --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_AssignAdd.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + """Net definition""" + def __init__(self): + super(Net, self).__init__() + self.AssignAdd = P.AssignAdd() + self.inputdata = Parameter(initializer('normal', [1]), name="global_step") + print("inputdata: ", self.inputdata) + + def construct(self, x): + out = self.AssignAdd(self.inputdata, x) + return out + + +def test_net(): + """test AssignAdd""" + net = Net() + x = Tensor(np.ones([1]).astype(np.float32)*100) + + print("MyPrintResult dataX:", x) + result = net(x) + print("MyPrintResult data::", result.asnumpy()) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_AssignSub.py b/tests/st/ops/davinci/test_tbe_ops/test_AssignSub.py new file mode 100644 index 00000000000..ad37f8c53fc --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_AssignSub.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + """Net definition""" + def __init__(self): + super(Net, self).__init__() + self.AssignSub = P.AssignSub() + self.inputdata = Parameter(initializer('normal', [1]), name="global_step") + print("inputdata: ", self.inputdata) + + def construct(self, x): + out = self.AssignSub(self.inputdata, x) + return out + + +def test_net(): + """test AssignSub""" + net = Net() + x = Tensor(np.ones([1]).astype(np.int32)*100) + + print("MyPrintResult dataX:", x) + result = net(x) + print("MyPrintResult data::", result.asnumpy()) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_ReduceMean.py b/tests/st/ops/davinci/test_tbe_ops/test_ReduceMean.py new file mode 100644 index 00000000000..7975ef190ba --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_ReduceMean.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self, keep_dims, axis): + super(Net, self).__init__() + self.reduce_mean = P.ReduceMean(keep_dims=keep_dims) + self.axis = axis + + @ms_function + def construct(self, inputs): + return self.reduce_mean(inputs, self.axis) + +x1 = np.random.randn(64).astype(np.float32) + +def test_net(): + keepdims = False + axis = -1 + Reduce_mean = Net(keepdims, axis) + output = Reduce_mean(Tensor(x1)) + print(x1) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_add.py b/tests/st/ops/davinci/test_tbe_ops/test_add.py new file mode 100644 index 00000000000..dd8515868e5 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_add.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + + def construct(self, x, y): + return self.add(x, y) + +x = np.random.randn(1,3,3,4).astype(np.float32) +y = np.random.randn(1,3,3,4).astype(np.float32) + + +def test_net(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + add = Net() + output = add(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") + add = Net() + output = add(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_addn.py b/tests/st/ops/davinci/test_tbe_ops/test_addn.py new file mode 100644 index 00000000000..4defa7d6299 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_addn.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.AddN() + + def construct(self, x, y): + return self.add((x, y)) + +def test_net(): + x = np.random.randn(1, 3, 3, 4).astype(np.float32) + y = np.random.randn(1, 3, 3, 4).astype(np.float32) + add = Net() + output = add(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_apply_adam.py b/tests/st/ops/davinci/test_tbe_ops/test_apply_adam.py new file mode 100644 index 00000000000..f54de921441 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_apply_adam.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore import Tensor +import mindspore.nn as nn +from mindspore.nn import Dense, SoftmaxCrossEntropyWithLogits +from mindspore.nn import TrainOneStepCell, WithLossCell + +import mindspore.context as context +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", impl_type="tbe") +context.set_context(enable_task_sink=True) + + +class Adam: + def __init__(self, batch_num, input_channels, output_channels, epoch, lr, weight_decay, epsilon): + self.batch_num = batch_num + self.input_channels = input_channels + self.output_channels = output_channels + self.epoch = epoch + self.lr = lr + self.weight_decay = weight_decay + self.epsilon = epsilon + + def train_mindspore_impl(self): + input = Tensor(np.random.randn(self.batch_num, self.input_channels).astype(np.float32)) + weight_np = Tensor(np.random.randn(self.output_channels, self.input_channels).astype(np.float32)) + bias = Tensor(np.random.randn(self.output_channels).astype(np.float32)) + + label_np = np.random.randint(self.output_channels, size=self.batch_num) + label_np_onehot = np.zeros(shape=(self.batch_num, self.output_channels)).astype(np.float32) + label_np_onehot[np.arange(self.batch_num), label_np] = 1.0 + label = Tensor(label_np_onehot) + + ms_dense = Dense(in_channels=self.input_channels, + out_channels=self.output_channels, + weight_init=weight_np, + bias_init=bias, has_bias=True) + criterion = SoftmaxCrossEntropyWithLogits() + optimizer = nn.Adam(ms_dense.trainable_params(), + learning_rate=1e-3, + beta1=0.9, beta2=0.999, eps=self.epsilon, + use_locking=False, + use_nesterov=False, weight_decay=0.0, + loss_scale=1.0) + + net_with_criterion = WithLossCell(ms_dense, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) + train_network.set_train() + + print('MS Initialized!') + for i in range(self.epoch): + train_network(input, label) + output = ms_dense(input) + print("===============output=================", output) + return output.asnumpy() + + +def test_adam(): + fact = Adam(batch_num=8, input_channels=20, output_channels=5, epoch=5, lr=0.1, weight_decay=0.0, epsilon= 1e-8) + fact.train_mindspore_impl() diff --git a/tests/st/ops/davinci/test_tbe_ops/test_apply_momentum.py b/tests/st/ops/davinci/test_tbe_ops/test_apply_momentum.py new file mode 100644 index 00000000000..f1d9abdaf43 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_apply_momentum.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.apply_momentum = P.ApplyMomentum(gradient_scale=1024.0) + self.variable = Parameter(initializer( + 'normal', [2, 3, 3, 4]), name='variable') + self.accumulation = Parameter(initializer( + 'normal', [2, 3, 3, 4]), name='accumulation') + self.learning_rate = Parameter(initializer( + 'normal', [1, ]), name='learning_rate') + self.gradient = Parameter(initializer( + 'normal', [2, 3, 3, 4]), name='gradient') + self.momentum = Parameter(initializer( + 'normal', [1, ]), name='momentum') + def construct(self): + return self.apply_momentum(self.variable, self.accumulation, self.learning_rate, self.gradient, self.momentum) + +def test_net(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + apply_momentum = Net() + output = apply_momentum() + print(output.asnumpy()) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") + apply_momentum = Net() + output = apply_momentum() + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_batchmatmul.py b/tests/st/ops/davinci/test_tbe_ops/test_batchmatmul.py new file mode 100644 index 00000000000..4d9e8de402c --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_batchmatmul.py @@ -0,0 +1,47 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.nn import Cell +from mindspore.train.model import Model +import pytest +from mindspore import context +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.batchmatmul = P.BatchMatMul() + + def construct(self, inputa, inputb): + x = self.batchmatmul(inputa, inputb) + return x + +def tf_me_batchmatmul(inputa, inputb): + net = Net() + net.set_train() + model = Model(net) + out_me = model.predict(Tensor(inputa), Tensor(inputb)) + +def test_batchmatmul_normal_shape1(): + inputa = np.random.randn(128, 16, 128).astype(np.float32) + inputb = np.random.randn(128, 128, 64).astype(np.float32) + tf_me_batchmatmul(Tensor(inputa), Tensor(inputb)) + +def test_batchmatmul_normal_shape2(): + inputa = np.random.randn(1, 16, 128, 128).astype(np.float32) + inputb = np.random.randn(1, 16, 128, 64).astype(np.float32) + tf_me_batchmatmul(inputa, inputb) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_batchnorm.py b/tests/st/ops/davinci/test_tbe_ops/test_batchnorm.py new file mode 100644 index 00000000000..78610f3f871 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_batchnorm.py @@ -0,0 +1,58 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bn = P.BatchNorm() + self.scale = Parameter(initializer('ones', [64]), name='scale') + self.offset = Parameter(initializer('zeros', [64]), name='offset') + self.mean = Parameter(initializer('ones', [64]), name='mean') + self.variance = Parameter(initializer('zeros', [64]), name='variance') + + def construct(self, x): + return self.bn(x, self.scale, self.offset, self.mean, self.variance)[0] + + +def test_net(): + x = np.random.randn(1,64,112,112).astype(np.float32) + # mean = np.random.randn(1,16,1,1).astype(np.float32) + # variance = np.random.randn(1,16,1,1).astype(np.float32) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + Bn = Net() + output = Bn(Tensor(x)) + print("***********x*********") + print(x) + + print("***********output y*********") + print(output.asnumpy()) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") + Bn = Net() + output = Bn(Tensor(x)) + print("***********x*********") + print(x) + + print("***********output y*********") + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_batchnorm_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_batchnorm_grad.py new file mode 100644 index 00000000000..a893a3a8d3e --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_batchnorm_grad.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +#context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(device_target="Ascend") +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bn = P.BatchNorm() + self.scale = Parameter(initializer('ones', [64]), name='scale') + self.offset = Parameter(initializer('zeros', [64]), name='offset') + self.mean = Parameter(initializer('ones', [64]), name='mean') + self.variance = Parameter(initializer('zeros', [64]), name='variance') + + def construct(self, x): + return self.bn(x, self.scale, self.offset, self.mean, self.variance)[0] + + +def test_net(): + x = np.random.randn(1,64,112,112).astype(np.float32) + sens = np.random.randn(1,64,112,112).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_bias_add.py b/tests/st/ops/davinci/test_tbe_ops/test_bias_add.py new file mode 100644 index 00000000000..8dbea18b959 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_bias_add.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + """Net definition""" + def __init__(self, + output_channels, + bias_init='zeros', + ): + super(Net, self).__init__() + self.biasAdd = P.BiasAdd() + + if isinstance(bias_init, Tensor): + if bias_init.dim() != 1 or bias_init.shape()[0] != output_channels: + raise ValueError("bias_init shape error") + + self.bias = Parameter(initializer( + bias_init, [output_channels]), name="bias") + + def construct(self, input_x): + return self.biasAdd(input_x, self.bias) + + +def test_compile(): + bias_init = Tensor(np.ones([3]).astype(np.float32)) + net = Net(3, bias_init=bias_init) + input_data = Tensor(np.ones([1, 3, 3, 3], np.float32)) + # since simulator currently not support matMul + # enable it when staging function is ready + output = net(input_data) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_bias_add_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_bias_add_grad.py new file mode 100644 index 00000000000..57f0649e7c1 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_bias_add_grad.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bias_add_grad = G.BiasAddGrad() + + @ms_function + def construct(self, dout): + return self.bias_add_grad(dout) + +def test_net(): + dout = np.random.rand(1, 1001).astype(np.float32) + bias_add_grad = Net() + output = bias_add_grad(Tensor(dout)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_concat.py b/tests/st/ops/davinci/test_tbe_ops/test_concat.py new file mode 100644 index 00000000000..debc18ab140 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_concat.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__( self): + super(Net, self).__init__() + + self.cat = P.Concat(axis=1) + self.x1 = Parameter(initializer( + Tensor(np.arange(2 * 2).reshape(2, 2).astype(np.float32)), [2, 2]), name='x1') + self.x2 = Parameter(initializer( + Tensor(np.arange(2 * 3).reshape(2, 3).astype(np.float32)), [2, 3]), name='x2') + + @ms_function + def construct(self): + return self.cat((self.x1, self.x2)) + + +def test_net(): + cat = Net() + output = cat() + expect = [[0., 1., 0., 1., 2.], + [2., 3., 3., 4., 5.]] + print(np.arange(2 * 2).reshape(2, 2)) + print(np.arange(2 * 3).reshape(2, 3)) + print(output) + assert(output.asnumpy() == expect).all() diff --git a/tests/st/ops/davinci/test_tbe_ops/test_conv.py b/tests/st/ops/davinci/test_tbe_ops/test_conv.py new file mode 100644 index 00000000000..e6eb880e1ee --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_conv.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + out_channel = 64 + kernel_size = 7 + self.conv = P.Conv2D(out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer( + 'normal', [64, 3, 7, 7]), name='w') + + + @ms_function + def construct(self, x): + return self.conv(x, self.w) + + + +def test_net(): + x = np.random.randn(32,3,224,224).astype(np.float32) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + conv = Net() + output = conv(Tensor(x)) + print(output.asnumpy()) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") + conv = Net() + output = conv(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_conv2d_backprop_filter.py b/tests/st/ops/davinci/test_tbe_ops/test_conv2d_backprop_filter.py new file mode 100644 index 00000000000..554fb9ab9b6 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_conv2d_backprop_filter.py @@ -0,0 +1,74 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target='Ascend') + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + out_channel = 4 + kernel_size = 1 + self.conv_filter = G.Conv2DBackpropFilter(out_channel, + kernel_size, + pad_mode="valid", + pad=0, + mode=1, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer(Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='w') + self.x = Parameter(initializer(Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)), [1,1,6,6]), name='x') + self.out = Parameter(initializer(Tensor(np.array([[[ + [ -5, -4, 0, 8], + [-10, -2, 2, 3], + [ 0, -2, -4, -7], + [ -3, -2, -3, -16]]]]).astype(np.float32)),[1,1,4,4]), name='y') + self.get_shape = P.Shape() + + @ms_function + def construct(self): + return self.conv_filter(self.out, self.x, self.get_shape(self.w)) + + +def test_conv2d_backprop_filter(): + conv2d_filter = Net() + output = conv2d_filter() + print("================================") + """ + expect output: + [[[[ -60, -142, -265] + [-104, -211, -322] + [-102, -144, -248]]]] + """ + expect = np.array([[[[ -60, -142, -265], + [-104, -211, -322], + [-102, -144, -248]]]]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/davinci/test_tbe_ops/test_conv2d_backprop_input.py b/tests/st/ops/davinci/test_tbe_ops/test_conv2d_backprop_input.py new file mode 100644 index 00000000000..961c7fdbe5c --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_conv2d_backprop_input.py @@ -0,0 +1,79 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + out_channel = 4 + kernel_size = 1 + self.conv_input = P.Conv2DBackpropInput(out_channel, + kernel_size, + pad_mode="valid", + pad=0, + mode=1, + stride=1, + dilation=1, + group=1) + self.w = Parameter(initializer(Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)), [1, 1, 3, 3]), name='w') + self.x = Parameter(initializer(Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)), [1,1,6,6]), name='x') + self.out = Parameter(initializer(Tensor(np.array([[[ + [ -5, -4, 0, 8], + [-10, -2, 2, 3], + [ 0, -2, -4, -7], + [ -3, -2, -3, -16]]]]).astype(np.float32)),[1,1,4,4]), name='y') + self.get_shape = P.Shape() + + @ms_function + def construct(self): + return self.conv_input(self.out, self.w, self.get_shape(self.x)) + + +def test_conv2d_backprop_input(): + conv2d_input = Net() + output = conv2d_input() + print("================================") + """ + expect output: + [[[[ -5, -4, 5, 12, 0, -8] + [-15, -6, 17, 17, -2, -11] + [-15, -8, 13, 12, 2, -4] + [-13, -6, 8, -14, 5, 20] + [ -3, -4, -4, -19, 7, 23] + [ -3, -2, 0, -14, 3, 16]]]] + """ + expect = np.array([[[[ -5, -4, 5, 12, 0, -8], + [-15, -6, 17, 17, -2, -11], + [-15, -8, 13, 12, 2, -4], + [-13, -6, 8, -14, 5, 20], + [ -3, -4, -4, -19, 7, 23], + [ -3, -2, 0, -14, 3, 16]]]]).astype(np.float32) + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/davinci/test_tbe_ops/test_dropout_do_mask.py b/tests/st/ops/davinci/test_tbe_ops/test_dropout_do_mask.py new file mode 100644 index 00000000000..b0a0ee25894 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_dropout_do_mask.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +from mindspore import log as logger + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.dropoutdomask = P.DropoutDoMask() + + def construct(self, x, mask, keep_prob): + return self.dropoutdomask(x, mask, keep_prob) + + +def test_net(): + x = np.random.randn(2,5,8).astype(np.float32) + mask = np.random.randn(16).astype(np.uint8) + keep_prob = 1 + + ddm = Net() + output = ddm(Tensor(x), Tensor(mask), Tensor(keep_prob)) + logger.info("***********x*********") + logger.info(x) + logger.info("***********mask*********") + logger.info(mask) + logger.info("***********keep_prob*********") + logger.info(keep_prob) + + logger.info("***********output y*********") + logger.info(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_gelu.py b/tests/st/ops/davinci/test_tbe_ops/test_gelu.py new file mode 100644 index 00000000000..fb66d94430e --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_gelu.py @@ -0,0 +1,130 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.nn import GELU, Cell +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +from mindspore.ops import operations as P +import math +import pytest +from mindspore import context +from mindspore import log as logger +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +def gelu_forward_me_impl(input): + n = GELU() + n.set_train() + m = Model(n) + out = m.predict(input) + return out.asnumpy() + + +def gelu_forward_cmp(input_shape, data_type=np.float32): + input_np = np.random.randn(*input_shape).astype(data_type) + input_me = Tensor(input_np) + out_me = gelu_forward_me_impl(input_me) + + +@pytest.mark.skip(reason="scalar") +def test_gelu_input_scalar(): + input_np = np.random.randn() + input_me = Tensor(input_np) + out_me = gelu_forward_me_impl(input_me) + logger.info("---------me--------") + logger.info(out_me) + + +def test_gelu_input_dim_0(): + input_shape = [0] + with pytest.raises(ValueError): + gelu_forward_cmp(input_shape) + +def test_gelu_input_dim_10240_1024(): + input_shape = [10240, 1024] + gelu_forward_cmp(input_shape) + + +def test_gelu_input_dim_10240_768(): + input_shape = [10240, 768] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_1024_3072(): + input_shape = [1024, 3072] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_1024_4096(): + input_shape = [1024, 4096] + gelu_forward_cmp(input_shape) + + +def test_gelu_input_dim_1280_1024(): + input_shape = [1280, 1024] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_1280_768(): + input_shape = [1280, 768] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_128_3072(): + input_shape = [128, 3072] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_128_4096(): + input_shape = [128, 4096] + gelu_forward_cmp(input_shape) + +@pytest.mark.lower_bs +def test_gelu_input_dim_160_1024(): + input_shape = [160, 1024] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_160_768(): + input_shape = [160, 768] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_16384_3072(): + input_shape = [16384, 3072] + gelu_forward_cmp(input_shape) + + +def test_gelu_input_dim_16384_4096(): + input_shape = [16384, 4096] + gelu_forward_cmp(input_shape) + + +@pytest.mark.lower_bs +def test_gelu_input_dim_20_1024(): + input_shape = [20, 1024] + gelu_forward_cmp(input_shape) + + +def test_gelu_input_dim_20480_1024(): + input_shape = [20480, 1024] + gelu_forward_cmp(input_shape) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_gelu_grad_sens.py b/tests/st/ops/davinci/test_tbe_ops/test_gelu_grad_sens.py new file mode 100755 index 00000000000..93d25cd0963 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_gelu_grad_sens.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.ops.composite import GradOperation +from mindspore.nn import Cell, GELU +from mindspore import context +import pytest +import math +from mindspore.ops import operations as P +import mindspore as ms +from mindspore import log as logger + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + + +def gelu_backward_me_impl(input, output_grad): + n = GELU() + grad_with_sense = Grad(n) + grad_with_sense.set_train() + input_grad = grad_with_sense(input, output_grad) + return input_grad.asnumpy() + + +def gelu_backward_cmp(input_shape): + input_np = np.random.randn(*input_shape).astype(np.float32) + input_me = Tensor(input_np) + + output_grad_shape = input_shape + output_grad_np = np.random.randn(*output_grad_shape).astype(np.float32) + output_grad_me = Tensor(output_grad_np) + + output_grad_me = gelu_backward_me_impl(input_me, output_grad_me) + logger.info("---------me--------") + logger.info(output_grad_me) + +# ---------- LARGE INPUT --------------- + +class MEGeluLargeIn(Cell): + def __init__(self): + super(GELU, self).__init__() + self.matmul = P.MatMul() + self.gelu = P.Gelu() + + def construct(self, x1, x2): + x = self.matmul(x1, x2) + return self.gelu(x) + +class GradLargeIn(Cell): + def __init__(self, network): + super(GradLargeIn, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, x1, x2, output_grad): + return self.grad(self.network)(x1, x2, output_grad) + + +def gelu_backward_me_large_in_impl(x1, x2, output_grad): + n = GradLargeIn() + grad_with_sense = GradLargeIn(n) + grad_with_sense.set_train() + input_grad = grad_with_sense(x1, x2, output_grad) + return input_grad[0].asnumpy(), input_grad[1].asnumpy(), + + +def test_grad_gelu_input_10240_1024(): + input_shape = [10240,1024] + gelu_backward_cmp(input_shape) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_greater.py b/tests/st/ops/davinci/test_tbe_ops/test_greater.py new file mode 100644 index 00000000000..3976ad4301c --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_greater.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import pytest +from mindspore.ops import operations as P +from mindspore.nn import Cell +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +from mindspore import log as logger +from mindspore import context +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Greater(Cell): + def __init__(self): + super(Greater, self).__init__() + self.greater = P.Greater() + + def construct(self, inputa, inputb): + return self.greater(inputa, inputb) + +def me_greater(inputa, inputb): + net = Greater() + net.set_train() + model = Model(net) + + out = model.predict(inputa, inputb) + logger.info("Check input a: ") + logger.info(inputa) + logger.info("Check input b: ") + logger.info(inputb) + return out.asnumpy() + +@pytest.mark.ssd_tbe +def test_greater_2d_scalar0(): + a = np.random.randint(-5, 5, [8, 32]).astype(np.int32) + b = np.random.randint(-5, 5, [8, 32]).astype(np.int32) + out_me = me_greater(Tensor(a), Tensor(b)) + logger.info("Check me result:") + logger.info(out_me) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_layernorm.py b/tests/st/ops/davinci/test_tbe_ops/test_layernorm.py new file mode 100644 index 00000000000..586c02cc1ed --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_layernorm.py @@ -0,0 +1,55 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.nn import LayerNorm +from mindspore.common.tensor import Tensor +from mindspore.nn import Cell +from mindspore.train.model import Model +from mindspore import log as logger +import pytest +from mindspore import context +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Net(Cell): + def __init__(self, input_shape, begin_norm_axis, begin_params_axis, gamma, beta): + super(Net, self).__init__() + self.layernorm = LayerNorm(input_shape, begin_norm_axis, begin_params_axis, gamma, beta) + + def construct(self, input): + x = self.layernorm(input) + return x + +def pt_me_layernorm(input_data, normalized_shape, gamma, beta, axis): + net = Net(normalized_shape, begin_norm_axis=axis, + begin_params_axis=axis, + gamma=Tensor(gamma), + beta=Tensor(beta)) + net.set_train() + model = Model(net) + out_me = model.predict(Tensor(input_data)) + logger.info("Check me result:") + logger.info(out_me.asnumpy()) + +@pytest.mark.lower_bs +def test_normal_layernorm_1_128_1024_axis_2(): + """ + 2 input[1, 128, 1024],normalized_shape=[128, 1024] + """ + input_data = np.random.randn(1, 128, 1024).astype(np.float32) + gamma = np.random.randn(1024).astype(np.float32) + gamma.fill(1.1) + beta = np.random.randn(1024).astype(np.float32) + beta.fill(0.1) + pt_me_layernorm(input_data, (1024, ), gamma, beta, 2) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_layernorm_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_layernorm_grad.py new file mode 100644 index 00000000000..9f8eefdb3fd --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_layernorm_grad.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.nn import LayerNorm +from mindspore.common.tensor import Tensor +from mindspore.nn import Cell +from mindspore.ops.composite import GradOperation +from mindspore import log as logger +from mindspore import context +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, input, output_grad,): + gout = self.grad(self.network)(input, output_grad) + return gout + +class Net(Cell): + def __init__(self, input_shape, begin_norm_axis, begin_params_axis, gamma, beta): + super(Net, self).__init__() + self.layernorm = LayerNorm(input_shape, begin_norm_axis, begin_params_axis, gamma, beta) + + def construct(self, input): + x = self.layernorm(input) + return x + +def py_me_layernorm_grad(input_data, normalized_shape, gamma, beta, axis, gradients): + input_me = Tensor(input_data) + net_me = Grad(Net(normalized_shape, begin_norm_axis=axis, + begin_params_axis=axis, + gamma=Tensor(gamma), + beta=Tensor(beta))) + net_me.set_train() + out_pool_grad_me = Tensor(gradients) + out_grad = net_me(input_me, out_pool_grad_me) + logger.info("Check me result:") + logger.info(out_grad.asnumpy()) + +def test_normal_layernorm_grad_normalize_2d(): + """ + 1 input[1, 128, 1024],normalized_shape=[1024],element_affine=False + """ + input_data = np.ones([1, 128, 1024]).astype(np.float32) + gradients = np.ones((1, 128, 1024)).astype(np.float32) + gamma = np.random.randn(1024).astype(np.float32) + gamma.fill(1.1) + beta = np.random.randn(1024).astype(np.float32) + beta.fill(0.1) + py_me_layernorm_grad(input_data, (1024,), gamma, beta, 2, gradients) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_less.py b/tests/st/ops/davinci/test_tbe_ops/test_less.py new file mode 100644 index 00000000000..ef65fbe3b27 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_less.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.less = P.Less() + + @ms_function + def construct(self, x1, x2): + return self.less(x1, x2) + +x1 = np.random.randn(3,4).astype(np.float16) +x2 = np.random.randn(3,4).astype(np.float16) + +def test_net(): + less = Net() + output = less(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_less_equal.py b/tests/st/ops/davinci/test_tbe_ops/test_less_equal.py new file mode 100644 index 00000000000..503c7b7e764 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_less_equal.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.less_equal = P.LessEqual() + + @ms_function + def construct(self, x1, x2): + return self.less_equal(x1, x2) + +x1 = np.random.randn(3,4).astype(np.float16) +x2 = np.random.randn(3,4).astype(np.float16) + +def test_net(): + less_equal = Net() + output = less_equal(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_logical_and.py b/tests/st/ops/davinci/test_tbe_ops/test_logical_and.py new file mode 100644 index 00000000000..c9f180a56e0 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_logical_and.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.logical_and = P.LogicalAnd() + + @ms_function + def construct(self, x1, x2): + return self.logical_and(x1, x2) + +x1 = [True, True, False, False, True, True, False, False] +x2 = [True, False, False, True, True, False, False, True] +def test_net(): + logical_and = Net() + output = logical_and(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_logical_not.py b/tests/st/ops/davinci/test_tbe_ops/test_logical_not.py new file mode 100644 index 00000000000..97e9caa5c9e --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_logical_not.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.logical_not = P.LogicalNot() + + @ms_function + def construct(self, x1): + return self.logical_not(x1) + +x1 = [True, True, False, False, True, True, False, False] + +def test_net(): + logical_not = Net() + output = logical_not(Tensor(x1)) + print(x1) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_logical_or.py b/tests/st/ops/davinci/test_tbe_ops/test_logical_or.py new file mode 100644 index 00000000000..e34d94c3e7c --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_logical_or.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.logical_or = P.LogicalOr() + + @ms_function + def construct(self, x1, x2): + return self.logical_or(x1, x2) + +x1 = [True, True, False, False, True, True, False, False] +x2 = [True, False, False, True, True, False, False, True] +def test_net(): + logical_or = Net() + output = logical_or(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_matmul.py b/tests/st/ops/davinci/test_tbe_ops/test_matmul.py new file mode 100644 index 00000000000..92115409f7f --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_matmul.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul() + + @ms_function + def construct(self, x1, x2): + return self.matmul(x1, x2) + +x1 = np.random.randn(1,3).astype(np.float32) +x2 = np.random.randn(3,4).astype(np.float32) + +def test_net(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + matmul = Net() + output = matmul(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") + matmul = Net() + output = matmul(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_matmul_failed.py b/tests/st/ops/davinci/test_tbe_ops/test_matmul_failed.py new file mode 100644 index 00000000000..4775f732810 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_matmul_failed.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul(transpose_b=True) + + @ms_function + def construct(self, x1, x2): + return self.matmul(x1, x2) + +x1 = np.random.randn(10,1).astype(np.float32) +x2 = np.random.randn(100,1).astype(np.float32) + +def test_net(): + matmul = Net() + output = matmul(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_maximum.py b/tests/st/ops/davinci/test_tbe_ops/test_maximum.py new file mode 100644 index 00000000000..ba66f0c0385 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_maximum.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +import mindspore as ms +from mindspore import Tensor +from mindspore.train.model import Model +from mindspore.ops import operations as P + +context.set_context(device_target="Ascend") + +class Max(nn.Cell): + def __init__(self,dtype): + super(Max, self).__init__() + self.max = P.Maximum() + + def construct(self, inputa, inputb): + return self.max(inputa, inputb) + +def me_max(inputa, inputb, dtype=ms.float32): + context.set_context(mode=context.GRAPH_MODE) + net = Max(dtype) + net.set_train() + model = Model(net) + print(type(inputa)) + if isinstance(inputa, np.ndarray) == True: + inputa = Tensor(inputa) + if isinstance(inputb, np.ndarray) == True: + inputb = Tensor(inputb) + out = model.predict(inputa, inputb) + print(out) + return out.asnumpy() + +def cmp_max(a,b): + out = np.maximum(a, b) + out_ms = me_max(a, b) + print("-------ms------") + print("numpy out :{}".format(out)) + print("ms out :{}".format(out_ms)) + +def test_maximum_2_2(): + a = np.random.randn(2, 2).astype(np.float32) + b = np.random.randn(2, 2).astype(np.float32) + cmp_max(a,b) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_maximum_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_maximum_grad.py new file mode 100644 index 00000000000..4b3103cbfed --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_maximum_grad.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.context as context +from mindspore import Tensor +from mindspore.nn import Cell +from mindspore.ops import composite as C +from mindspore.ops import operations as P + +context.set_context(device_target="Ascend") +grad = C.GradOperation('get_all', get_all=True, sens_param=True) + +class MaxNetMe(Cell): + def __init__(self): + super(MaxNetMe, self).__init__() + self.max = P.Maximum() + + def construct(self, inputA, inputB): + x = self.max(inputA, inputB) + return x + +class GradWrap(Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, inputA, inputB, sens): + gout = grad(self.network)(inputA, inputB, sens) + return gout + +def gen_data(inputA_np, inputB_np, grad=None): + inputA_me = inputA_np + if isinstance(inputA_np, np.ndarray) == True: + inputA_me = Tensor(inputA_me) + inputB_me = inputB_np + if isinstance(inputB_np, np.ndarray) == True: + inputB_me = Tensor(inputB_np) + if grad == None: + grad = np.random.randn(2).astype(np.float32) + print("----inputA---") + print(inputA_np) + print("----inputB---") + print(inputB_np) + + net_me = GradWrap(MaxNetMe()) + net_me.set_train() + output = net_me(inputA_me, inputB_me, Tensor(grad)) + print("---me---") + print(output[0].asnumpy()) + print(output[1].asnumpy()) + +def test_net(): + inputA_np = np.random.randn(1, 3, 2, 2).astype(np.float32) + inputB_np = np.random.randn(1, 3, 2, 2).astype(np.float32) + gen_data(inputA_np, inputB_np) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_maxpool.py b/tests/st/ops/davinci/test_tbe_ops/test_maxpool.py new file mode 100644 index 00000000000..5baaa32d52c --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_maxpool.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.maxpool = P.MaxPool(padding="SAME", ksize=3, strides=2) + + + @ms_function + def construct(self, x): + output = self.maxpool(x) + return output + + +def test_net(): + x = np.random.randn(32,64,112,112).astype(np.float16) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + maxpool = Net() + output = maxpool(Tensor(x)) + print(output.asnumpy()) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") + maxpool = Net() + output = maxpool(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_maxpool_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_maxpool_grad.py new file mode 100644 index 00000000000..9651168634b --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_maxpool_grad.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.maxpool = P.MaxPool(padding="SAME", ksize=3, strides=2) + + @ms_function + def construct(self, x): + output = self.maxpool(x) + return output + + +def test_net(): + x = np.random.randn(32, 64, 112, 112).astype(np.float32) + output_grad = np.random.randn(32, 64, 56, 56).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(output_grad)) + if isinstance(output, (tuple, list)): + output = output[0] + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_minimum.py b/tests/st/ops/davinci/test_tbe_ops/test_minimum.py new file mode 100644 index 00000000000..a107d46f431 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_minimum.py @@ -0,0 +1,62 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import mindspore as ms +from mindspore.train.model import Model +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Min(nn.Cell): + def __init__(self, dtype): + super(Min, self).__init__() + self.min = P.Minimum() + + def construct(self, inputa, inputb): + return self.min(inputa, inputb) + + +def me_min(inputa, inputb, dtype=ms.float32): + context.set_context(mode=context.GRAPH_MODE) + net = Min(dtype) + net.set_train() + model = Model(net) + print(type(inputa)) + if isinstance(inputa, np.ndarray) == True: + inputa = Tensor(inputa) + if isinstance(inputb, np.ndarray) == True: + inputb = Tensor(inputb) + out = model.predict(inputa, inputb) + print(out) + return out.asnumpy() + +def cmp_min(a,b): + print(a) + print(b) + + out = np.minimum(a, b) + print(out) + out_me = me_min(a, b) + print(out_me) + +def test_minimum_2_2(): + a = np.random.randn(2, 2, 1, 1).astype(np.float32) + b = np.random.randn(2, 2, 1, 1).astype(np.float32) + cmp_min(a,b) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_minimum_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_minimum_grad.py new file mode 100644 index 00000000000..cbf870bb21c --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_minimum_grad.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +import numpy as np +import mindspore.context as context +from mindspore import Tensor +from mindspore.nn import Cell +from mindspore.ops import composite as C +from mindspore.ops.operations import Minimum + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +grad = C.GradOperation('get_all', get_all=True, sens_param=True) +class MinNetMe(Cell): + def __init__(self): + super(MinNetMe, self).__init__() + self.min = Minimum() + + def construct(self, inputA, inputB): + x = self.min(inputA, inputB) + return x + + +class GradWrap(Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, inputA, inputB, sens): + gout = grad(self.network)(inputA, inputB, sens) + return gout + +def gen_data(inputA_np, inputB_np, grad=None): + inputA_me = inputA_np + if isinstance(inputA_np, np.ndarray) == True: + inputA_me = Tensor(inputA_me) + + inputB_me = inputB_np + if isinstance(inputB_np, np.ndarray) == True: + inputB_me = Tensor(inputB_np) + + if grad is None: + grad = np.random.randn(1, 3, 2, 2).astype(np.float32) + + print(inputA_np) + print(inputB_np) + print(grad) + + net_me = GradWrap(MinNetMe()) + net_me.set_train() + output = net_me(inputA_me, inputB_me, Tensor(grad)) + print(output[0].asnumpy()) + print(output[1].asnumpy()) + + +def test_min_tensor_grad_4d(): + inputA_np = np.random.randn(1, 3, 2, 2).astype(np.float32) + inputB_np = np.random.randn(1, 3, 2, 2).astype(np.float32) + gen_data(inputA_np, inputB_np) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_mul.py b/tests/st/ops/davinci/test_tbe_ops/test_mul.py new file mode 100644 index 00000000000..dbc8c77e11e --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_mul.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.mul = P.Mul() + + @ms_function + def construct(self, x1, x2): + return self.mul(x1, x2) + +x1 = np.random.randn(3,4).astype(np.float32) +x2 = np.random.randn(3,4).astype(np.float32) + +def test_net(): + mul = Net() + output = mul(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_npu_alloc_float_status.py b/tests/st/ops/davinci/test_tbe_ops/test_npu_alloc_float_status.py new file mode 100644 index 00000000000..efa6a032884 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_npu_alloc_float_status.py @@ -0,0 +1,35 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.npu_alloc_float_status = P.NPUAllocFloatStatus() + + @ms_function + def construct(self): + return self.npu_alloc_float_status() + +def test_net(): + npu_alloc_float_status = Net() + output = npu_alloc_float_status() + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_npu_clear_float_status.py b/tests/st/ops/davinci/test_tbe_ops/test_npu_clear_float_status.py new file mode 100644 index 00000000000..0c6072fd6fe --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_npu_clear_float_status.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.npu_clear_float_status = P.NPUClearFloatStatus() + + @ms_function + def construct(self, x1): + return self.npu_clear_float_status(x1) + +x1 = np.random.randn(8).astype(np.float32) + +def test_net(): + npu_clear_float_status = Net() + output = npu_clear_float_status(Tensor(x1)) + print(x1) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_npu_get_float_status.py b/tests/st/ops/davinci/test_tbe_ops/test_npu_get_float_status.py new file mode 100644 index 00000000000..403f815c266 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_npu_get_float_status.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.npu_get_float_status = P.NPUGetFloatStatus() + + @ms_function + def construct(self, x1): + return self.npu_get_float_status(x1) + +x1 = np.random.randn(8).astype(np.float32) + +def test_net(): + npu_get_float_status = Net() + output = npu_get_float_status(Tensor(x1)) + print(x1) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_pad.py b/tests/st/ops/davinci/test_tbe_ops/test_pad.py new file mode 100644 index 00000000000..5590ca597a6 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_pad.py @@ -0,0 +1,40 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.pad = P.Pad(paddings=((3,2), (2,3))) + + @ms_function + def construct(self, x): + x = self.pad(x) + return x + +x = np.random.random(size=(2, 2)).astype(np.float32) + +def test_net(): + pad = Net() + output = pad(Tensor(x)) + print("=================output====================") + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_pow.py b/tests/st/ops/davinci/test_tbe_ops/test_pow.py new file mode 100644 index 00000000000..1bfcf1f63be --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_pow.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +from mindspore.nn import Cell +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import mindspore as ms +from mindspore.train.model import Model +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class PowMe(Cell): + def __init__(self): + super(PowMe, self).__init__() + self.pow = P.Pow() + + def construct(self, input, exp): + return self.pow(input, exp) + +def pow_forward_me_impl(input, exp): + n = PowMe() + n.set_train() + m = Model(n) + out = m.predict(input, exp) + return out.asnumpy() + +def pow_forward_cmp(input_shape, exp_shape): + if len(input_shape) == 0: + input_np = np.absolute(np.random.randn()) + else: + input_np = np.absolute(np.random.randn(*input_shape).astype(np.float32)) + input_tf = input_np + input_me = Tensor(input_np, dtype=ms.float32) + + if len(exp_shape) == 0: + exp_np = np.absolute(np.random.randn()) + else: + exp_np = np.absolute(np.random.randn(*exp_shape).astype(np.float32)) + exp_tf = exp_np + exp_me = Tensor(exp_np, dtype=ms.float32) + + out_me = pow_forward_me_impl(input_me, exp_me) + print(input_me) + print(exp_me) + print(out_me) + +def test_pow_input_scalar_exp_scalar(): + input_shape = [] + exp_shape = [] + pow_forward_cmp(input_shape, exp_shape) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_realdiv.py b/tests/st/ops/davinci/test_tbe_ops/test_realdiv.py new file mode 100644 index 00000000000..ea72c7a78b9 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_realdiv.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.realdiv = P.RealDiv() + + @ms_function + def construct(self, x1, x2): + return self.realdiv(x1, x2) + +x1 = np.random.randn(3,4).astype(np.float32) +x2 = np.random.randn(3,4).astype(np.float32) + +def test_net(): + realdiv = Net() + output = realdiv(Tensor(x1), Tensor(x2)) + print(x1) + print(x2) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_reciprocal.py b/tests/st/ops/davinci/test_tbe_ops/test_reciprocal.py new file mode 100644 index 00000000000..f90a2ae9e93 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_reciprocal.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.reciprocal = P.Reciprocal() + + @ms_function + def construct(self, x1): + return self.reciprocal(x1) + +x1 = np.random.randn(3, 4).astype(np.float32) + +def test_net(): + reciprocal = Net() + output = reciprocal(Tensor(x1)) + print(x1) + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_relu.py b/tests/st/ops/davinci/test_tbe_ops/test_relu.py new file mode 100644 index 00000000000..dd29a4bea0b --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_relu.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.relu = P.ReLU() + + @ms_function + def construct(self, x): + return self.relu(x) + +def test_net(): + x = np.random.randn(2,3,3,4).astype(np.float32) + relu = Net() + output = relu(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_relu_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_relu_grad.py new file mode 100644 index 00000000000..012991d208a --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_relu_grad.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.ops.composite import GradOperation +context.set_context(device_target="Ascend") + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.relu = P.ReLU() + + def construct(self, x): + return self.relu(x) + +def test_net(): + x = np.random.randn(2,3,3,4).astype(np.float32) + sens = np.random.randn(2,3,3,4).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(sens)) + print(len(output)) + print(output[0].asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_resize_nearest_neighbor.py b/tests/st/ops/davinci/test_tbe_ops/test_resize_nearest_neighbor.py new file mode 100644 index 00000000000..8fa67f9c472 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_resize_nearest_neighbor.py @@ -0,0 +1,37 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.upsample = P.ResizeNearestNeighbor((2, 2)) + + @ms_function + def construct(self, x): + return self.upsample(x) + +def test_net(): + x = np.random.random(size=(32, 3, 32, 32)).astype(np.float32) + upsample = Net() + output = upsample(Tensor(x)) + print("=================output====================") + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_resize_nearest_neighbor_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_resize_nearest_neighbor_grad.py new file mode 100644 index 00000000000..16714a00c31 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_resize_nearest_neighbor_grad.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.ops.composite import GradOperation +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.upsample = P.ResizeNearestNeighbor((2, 2)) + + @ms_function + def construct(self, images): + return self.upsample(images) + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, images, grads): + return self.grad(self.network)(images, grads) + + +def test_net(): + image = np.random.random(size=(32, 3, 16, 16)).astype(np.float32) + grads = np.random.random(size=(32, 3, 2, 2)).astype(np.float32) + grad = Grad(Net()) + output = grad(Tensor(image), Tensor(grads)) + print("=================output====================") + print(output) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_scatter_nd.py b/tests/st/ops/davinci/test_tbe_ops/test_scatter_nd.py new file mode 100644 index 00000000000..a12a8639020 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_scatter_nd.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.scatternd = P.ScatterNd() + + def construct(self, indices, update): + return self.scatternd(indices, update, (3,3)) + + +indices = np.array([[0, 1], [1, 1]]).astype(np.int32) +update = np.array([3.2, 1.1]).astype(np.float32) + +def test_net(): + scatternd = Net() + print(indices) + print(update) + output = scatternd(Tensor(indices), Tensor(update)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_select.py b/tests/st/ops/davinci/test_tbe_ops/test_select.py new file mode 100644 index 00000000000..1734e6aa827 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_select.py @@ -0,0 +1,63 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +from mindspore.nn import Cell +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import mindspore as ms +from mindspore.train.model import Model +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Select(Cell): + def __init__(self, dtype): + super(Select, self).__init__() + self.select = P.Select() + + def construct(self, cond, inputa, inputb): + return self.select(cond, inputa, inputb) + +def me_select(cond, inputa, inputb, dtype=ms.float32): + net = Select(dtype) + net.set_train() + model = Model(net) + if isinstance(inputa, np.ndarray) == True: + inputa = Tensor(inputa) + if isinstance(inputb, np.ndarray) == True: + inputb = Tensor(inputb) + if isinstance(cond, np.bool_) == True: + cond = np.array(cond) + + out = model.predict(Tensor(cond), inputa, inputb) + return out.asnumpy() + +def cmp_select(input_cond,inputa,inputb): + cond = input_cond > 0.5 + out_me = me_select(cond, inputa, inputb) + print(input_cond) + print(cond) + print(inputa) + print(inputb) + print(out_me) + +def test_select_2_2(): + input_cond = np.random.rand(2, 2) + inputa = np.random.randn(2,2).astype(np.float32) + inputb = np.random.randn(2,2).astype(np.float32) + cmp_select(input_cond,inputa,inputb) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_sigmoid.py b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid.py new file mode 100644 index 00000000000..f1610f01ab9 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid.py @@ -0,0 +1,37 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sigmoid = nn.Sigmoid() + + @ms_function + def construct(self, x): + return self.sigmoid(x) + +def test_net(): + x = np.random.random(size=(2, 3)).astype(np.float32) + sigmoid = Net() + output = sigmoid(Tensor(x)) + print("=================output====================") + print(output) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_cross_entropy_with_logits.py b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_cross_entropy_with_logits.py new file mode 100644 index 00000000000..2cac3bba160 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_cross_entropy_with_logits.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sigmoid_cross_entropy_with_logits = P.SigmoidCrossEntropyWithLogits() + + @ms_function + def construct(self, features, labels): + return self.sigmoid_cross_entropy_with_logits(features, labels) + + +def test_net(): + features = np.random.randn(2, 3).astype(np.float16) + labels = np.random.randn(2, 3).astype(np.float16) + SigmoidCrossEntropyWithLogits = Net() + output = SigmoidCrossEntropyWithLogits(Tensor(features), Tensor(labels)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_cross_entropy_with_logits_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_cross_entropy_with_logits_grad.py new file mode 100644 index 00000000000..dbcfe059ac2 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_cross_entropy_with_logits_grad.py @@ -0,0 +1,54 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +from mindspore.ops.composite import GradOperation +import numpy as np +import mindspore.context as context + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sigmoid_cross_entropy_with_logits = P.SigmoidCrossEntropyWithLogits() + + @ms_function + def construct(self, features, labels): + return self.sigmoid_cross_entropy_with_logits(features, labels) + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, features, labels, dout): + return self.grad(self.network)(features, labels, dout) + + +def test_net(): + features = np.random.randn(2, 3).astype(np.float16) + labels = np.random.randn(2, 3).astype(np.float16) + dout = np.random.randn(2, 3).astype(np.float16) + + net = Grad(Net()) + output = net(Tensor(features), Tensor(labels), Tensor(dout)) + print("=================output====================") + print(output) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_grad.py new file mode 100644 index 00000000000..189e75c4f98 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_sigmoid_grad.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.ops.composite import GradOperation +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sigmoid = P.Sigmoid() + + @ms_function + def construct(self, x): + return self.sigmoid(x) + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + @ms_function + def construct(self, x, y): + return self.grad(self.network)(x, y) + +def test_net(): + x = np.random.random(size=(2, 3, 4, 5, 6)).astype(np.float32) + y = np.random.random(size=(2, 3, 4, 5, 6)).astype(np.float32) + net = Grad(Net()) + output = net(Tensor(x), Tensor(y)) + print("=================output====================") + print(output.asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_slice.py b/tests/st/ops/davinci/test_tbe_ops/test_slice.py new file mode 100644 index 00000000000..4b55731e183 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_slice.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Slice(nn.Cell): + def __init__( self): + super(Slice, self).__init__() + + self.cat = P.Slice() + self.x1 = Parameter(initializer( + Tensor(np.array([[[1, -1, 1], [2, -2, 2]], [[3, -3, 3], [4, -4, 4]], [[5, -5, 5], [6, -6, 6]]]).astype(np.float32)), [3,2,3]), name='x1') + + @ms_function + def construct(self): + return self.cat(self.x1, (0,1, 0), (2, 1, 3)) + + +def test_slice(): + cat = Slice() + output = cat() + expect = [[[2., -2., 2.]], + [[4., -4., 4.]]] + print(output) + assert (output.asnumpy() == expect).all() \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_softmax.py b/tests/st/ops/davinci/test_tbe_ops/test_softmax.py new file mode 100644 index 00000000000..741e0ba6990 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_softmax.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.Softmax = P.Softmax() + + def construct(self, x): + return self.Softmax(x) + +x = np.array([[5, 1]]).astype(np.float32) + +def test_net(): + softmax = Net() + output = softmax(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_softmax_cross_entropy_with_logits.py b/tests/st/ops/davinci/test_tbe_ops/test_softmax_cross_entropy_with_logits.py new file mode 100644 index 00000000000..f21355533d9 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_softmax_cross_entropy_with_logits.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +context.set_context(device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self, is_grad=False): + super(Net, self).__init__() + self.SoftmaxCrossEntropyWithLogits = P.SoftmaxCrossEntropyWithLogits() + + @ms_function + def construct(self, features, labels): + return self.SoftmaxCrossEntropyWithLogits(features, labels) + + +def test_net(): + features = np.random.randn(32, 1001).astype(np.float16) + labels = np.random.randn(32, 1001).astype(np.float16) + SoftmaxCrossEntropyWithLogits = Net() + output = SoftmaxCrossEntropyWithLogits(Tensor(features), Tensor(labels)) + #print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_split.py b/tests/st/ops/davinci/test_tbe_ops/test_split.py new file mode 100644 index 00000000000..0e75643deaa --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_split.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.split = P.Split(0, 2) + + def construct(self, x): + return self.split(x) + +x = np.random.randn(2,4).astype(np.float32) + + +def test_net(): + split = Net() + output = split(Tensor(x)) + print("====input========") + print(x) + print("====output=======") + print(output) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_sqrt.py b/tests/st/ops/davinci/test_tbe_ops/test_sqrt.py new file mode 100644 index 00000000000..cd7176a60c7 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_sqrt.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sqrt = P.Sqrt() + + def construct(self, x): + return self.sqrt(x) + +x = np.array([1.0, 4.0, 9.0]).astype(np.float32) + +def test_net(): + sqrt = Net() + output = sqrt(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_square.py b/tests/st/ops/davinci/test_tbe_ops/test_square.py new file mode 100644 index 00000000000..81cad39bfa0 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_square.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.square = P.Square() + + def construct(self, x): + return self.square(x) + +x = np.array([1.0, 4.0, 9.0]).astype(np.float32) + +def test_net(): + square = Net() + output = square(Tensor(x)) + print(x) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_stridedslice.py b/tests/st/ops/davinci/test_tbe_ops/test_stridedslice.py new file mode 100644 index 00000000000..65f4f75538a --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_stridedslice.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.tensor import Tensor +import mindspore.ops.operations as P +from mindspore.nn import Cell +from mindspore.train.model import Model +import pytest +import mindspore.context as context +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(Cell): + def __init__(self, begin, end, stride): + super(Net, self).__init__() + self.stridedslice = P.StridedSlice() + self.begin = begin + self.end = end + self.stride = stride + + def construct(self, input): + x = self.stridedslice(input, self.begin, self.end, self.stride) + return x + +def me_stridedslice(input1, begin, end, stride): + input_me = Tensor(input1) + net = Net(begin, end, stride) + net.set_train() + model = Model(net) + output = model.predict(input_me) + print(output.asnumpy()) + +def test_stridedslice_input_2d(): + input = np.random.randn(5, 5).astype(np.int32) + begin = (0,0) + end = (2,2) + stride = (1,1) + + me_stridedslice(input, begin, end, stride) + +def test_stridedslice_input_3d(): + input = np.random.randn(5, 5, 5).astype(np.float32) + begin = (0,0,0) + end = (3,3,3) + stride = (1,1,1) + me_stridedslice(input, begin, end, stride) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_stridedslice_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_stridedslice_grad.py new file mode 100644 index 00000000000..d5ee390c9b9 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_stridedslice_grad.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.nn import Cell +from mindspore.ops.composite import GradOperation +from mindspore import context +import pytest +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, input, output_grad): + gout = self.grad(self.network)(input, output_grad) + return gout + +class Net(Cell): + def __init__(self, begin, end, stride): + super(Net, self).__init__() + self.stridedslice = P.StridedSlice() + self.begin = begin + self.end = end + self.stride = stride + + def construct(self, input): + x = self.stridedslice(input, self.begin, self.end, self.stride) + return x + +def me_stridedslice(input, begin, end, stride, gradients): + input_me = Tensor(input) + out_grad_me = Tensor(gradients) + net_me = Grad(Net(begin, end, stride)) + net_me.set_train() + out_grad = net_me(input_me, out_grad_me) + print(out_grad.asnumpy()) + +def test_grad_stridedslice_1d(): + input = np.random.randn(2).astype(np.float32) + begin = (0,) + end = (2,) + stride = (1,) + gradients = np.random.randn(2).astype(np.float32) + me_stridedslice(input, begin, end, stride, gradients) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_sub.py b/tests/st/ops/davinci/test_tbe_ops/test_sub.py new file mode 100644 index 00000000000..9f035f773dd --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_sub.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sub = P.Sub() + + def construct(self, x, y): + return self.sub(x, y) + +x = np.random.randn(1,3,3,4).astype(np.float32) +y = np.random.randn(1,3,3,4).astype(np.float32) + + +def test_net(): + sub = Net() + output = sub(Tensor(x), Tensor(y)) + print(x) + print(y) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_tanh.py b/tests/st/ops/davinci/test_tbe_ops/test_tanh.py new file mode 100644 index 00000000000..6c2b0b7fef2 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_tanh.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor +from mindspore.train.model import Model +from mindspore.ops import operations as P + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.tanh = P.Tanh() + + def construct(self, x): + return self.tanh(x) + +input_shape = [1] +input_np = np.random.randn(*input_shape).astype(np.float32) +input_me = Tensor(input_np) +def test_net(): + context.set_context(mode=context.GRAPH_MODE) + tanh = Net() + tanh.set_train() + m = Model(tanh) + out = m.predict(input_me) + print("out_me.dtype={}".format(out.dtype)) + print("out_me.asnumpy={}".format(out.asnumpy())) + return out.asnumpy() \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_tanh_grad.py b/tests/st/ops/davinci/test_tbe_ops/test_tanh_grad.py new file mode 100644 index 00000000000..683d1b9d304 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_tanh_grad.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor +from mindspore.train.model import Model +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.tanh_grad = G.TanhGrad() + + def construct(self, y, dy): + return self.tanh_grad(y, dy) + +input_shape = [1] +input_np = np.random.randn(*input_shape).astype(np.float32) +input_me = Tensor(input_np) +def test_net(): + context.set_context(mode=context.GRAPH_MODE) + tanh_grad = Net() + tanh_grad.set_train() + m = Model(tanh_grad) + out = m.predict(input_me, input_me) + print("out_me.dtype={}".format(out.dtype)) + print("out_me.asnumpy={}".format(out.asnumpy())) + return out.asnumpy() \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_tile.py b/tests/st/ops/davinci/test_tbe_ops/test_tile.py new file mode 100644 index 00000000000..7594248d53b --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_tile.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.tile = P.Tile() + + def construct(self, x): + return self.tile(x, (1, 4)) + + +x = np.array([[0], [1], [2], [3]]).astype(np.int32) + + +def test_net(): + tile = Net() + print(x) + output = tile(Tensor(x)) + print(output.asnumpy()) diff --git a/tests/st/ops/davinci/test_tbe_ops/test_topkv2.py b/tests/st/ops/davinci/test_tbe_ops/test_topkv2.py new file mode 100644 index 00000000000..a5058656372 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_topkv2.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +class Net(nn.Cell): + def __init__(self, k): + super(Net, self).__init__() + self.topk = P.TopK() + self.k = k + + def construct(self, x): + return self.topk(x, self.k) + + +def test_net(): + x = np.random.randn(4,4).astype(np.float16) + k = 2 + TopK = Net(k) + output = TopK(Tensor(x)) + print("***********x*********") + print(x) + + print("***********output y*********") + print(output[0].asnumpy()) + diff --git a/tests/st/ops/davinci/test_tbe_ops/test_transpose_d.py b/tests/st/ops/davinci/test_tbe_ops/test_transpose_d.py new file mode 100644 index 00000000000..faa3b7d5599 --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_transpose_d.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor +from mindspore.train.model import Model +from mindspore.ops import operations as P + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self, perm_in): + super(Net, self).__init__() + self.transpose = P.Transpose() + self.perm = perm_in + + def construct(self, input): + x = self.transpose(input, self.perm) + return x + +def ms_transpose(input, perm_in): + context.set_context(mode=context.GRAPH_MODE) + input_me = Tensor(input) + net = Net(perm_in) + net.set_train() + model = Model(net) + output = model.predict(input_me) + print("-------------ms------------------") + print(output.asnumpy().dtype) + print(output.asnumpy()) + +def test_net(): + input = np.random.randn(8, 24, 1, 1).astype(np.float16) + perm = (0, 2, 3, 1) + ms_transpose(input, perm) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tbe_ops/test_unsorted_segment_sum.py b/tests/st/ops/davinci/test_tbe_ops/test_unsorted_segment_sum.py new file mode 100644 index 00000000000..89f6029519a --- /dev/null +++ b/tests/st/ops/davinci/test_tbe_ops/test_unsorted_segment_sum.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor +from mindspore.train.model import Model +from mindspore.ops import operations as P + +context.set_context(device_target="Ascend") + +class Net(nn.Cell): + def __init__(self, num_segments): + super(Net, self).__init__() + self.seg_sum = P.UnsortedSegmentSum() + self.num_segments = num_segments + + def construct(self, x, segment_ids): + return self.seg_sum(x, segment_ids, self.num_segments) + +def me_un_seg_sum(input, indices, num_segments): + context.set_context(mode=context.GRAPH_MODE) + net = Net(num_segments) + net.set_train() + model = Model(net) + out = model.predict(Tensor(input), Tensor(indices)) + return out.asnumpy() + +def comapre_un_seg_sum(shape, indices, num_segments, dtype): + input = np.random.randn(*shape).astype(dtype) + indices_me = np.array(indices).astype(np.int32) + out_me = me_un_seg_sum(input, indices_me, num_segments) + print("-------------ms------------------") + print(out_me) + +def test_net(): + indices = np.random.randint(0, 1280, 1280) + comapre_un_seg_sum([1280, 768], indices, 8192, np.float32) \ No newline at end of file diff --git a/tests/st/ops/davinci/test_tdt_data_ms.py b/tests/st/ops/davinci/test_tdt_data_ms.py new file mode 100644 index 00000000000..6463401d82f --- /dev/null +++ b/tests/st/ops/davinci/test_tdt_data_ms.py @@ -0,0 +1,118 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore._c_dataengine as deMap +import mindspore.dataset as ds +import numpy as np +import sys +from mindspore._c_dataengine import InterpolationMode + +import mindspore.context as context +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.common.api import _executor +from mindspore.ops import operations as P + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +data_path = sys.argv[1] +SCHEMA_DIR = "{0}/resnet_all_datasetSchema.json".format(data_path) + + +def test_me_de_train_dataset(): + data_list = ["{0}/train-00001-of-01024.data".format(data_path)] + data_set = ds.StorageDataset(data_list, schema=SCHEMA_DIR, + columns_list=["image/encoded", "image/class/label"]) + + resize_height = 224 + resize_width = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + + decode_op = deMap.DecodeOp() + resize_op = deMap.ResizeOp(resize_height, resize_width, + InterpolationMode.DE_INTER_LINEAR) # Bilinear as default + rescale_op = deMap.RescaleOp(rescale, shift) + changemode_op = deMap.ChangeModeOp() + + # apply map operations on images + data_set = data_set.map(input_column_names="image/encoded", operation=decode_op) + data_set = data_set.map(input_column_names="image/encoded", operation=resize_op) + data_set = data_set.map(input_column_names="image/encoded", operation=rescale_op) + data_set = data_set.map(input_column_names="image/encoded", operation=changemode_op) + changeswap_op = deMap.ChannelSwapOp() + data_set = data_set.map(input_column_names="image/encoded", operation=changeswap_op) + data_set = data_set.repeat(1) + # apply batch operations + batch_size = 32 + data_set = data_set.batch(batch_size, drop_remainder=True) + return data_set + + +def convert_type(shapes, types): + ms_types = [] + for np_shape, np_type in zip(shapes, types): + input_np = np.zeros(np_shape, np_type) + tensor = Tensor(input_np) + ms_types.append(tensor.dtype()) + return ms_types + + +if __name__ == '__main__': + data_set = test_me_de_train_dataset() + dataset_size = data_set.get_dataset_size() + batch_size = data_set.get_batch_size() + + dataset_shapes = data_set.output_shapes() + np_types = data_set.output_types() + dataset_types = convert_type(dataset_shapes, np_types) + + ds1 = data_set.device_que() + get_next = P.GetNext(dataset_types, dataset_shapes, 2, ds1.queue_name) + tadd = P.ReLU() + + + class dataiter(nn.Cell): + def __init__(self): + super(dataiter, self).__init__() + + def construct(self): + input, label = get_next() + return tadd(input) + + + net = dataiter() + net.set_train() + + _executor.init_dataset(ds1.queue_name, 39, batch_size, + dataset_types, dataset_shapes, (), 'dataset') + ds1.send() + + for data in data_set.create_tuple_iterator(): + output = net() + print(data[0].any()) + print( + "****************************************************************************************************") + d = output.asnumpy() + print(d) + print( + "end+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", + d.any()) + + assert ( + (data[0] == d).all()), "TDT test execute failed, please check current code commit" + print( + "+++++++++++++++++++++++++++++++++++[INFO] Success+++++++++++++++++++++++++++++++++++++++++++") + diff --git a/tests/st/ops/gpu/test_addn_op.py b/tests/st/ops/gpu/test_addn_op.py new file mode 100644 index 00000000000..ce48e364fa6 --- /dev/null +++ b/tests/st/ops/gpu/test_addn_op.py @@ -0,0 +1,55 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target='GPU') + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.AddN() + + @ms_function + def construct(self, x, y, z): + return self.add((x, y, z)) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_net(): + x = np.arange(1 * 3 * 3 * 4).reshape(1, 3, 3, 4).astype(np.float32) + y = np.arange(1 * 3 * 3 * 4).reshape(1, 3, 3, 4).astype(np.float32) + z = np.arange(1 * 3 * 3 * 4).reshape(1, 3, 3, 4).astype(np.float32) + add = Net() + output = add(Tensor(x), Tensor(y), Tensor(z)) + expect_result = [[[[0., 3., 6., 9.], + [12., 15., 18., 21.], + [24., 27., 30., 33.]], + [[36., 39., 42., 45.], + [48., 51., 54., 57.], + [60., 63., 66., 69.]], + [[72., 75., 78., 81.], + [84., 87., 90., 93.], + [96., 99., 102., 105.]]]] + + assert (output.asnumpy() == expect_result).all() diff --git a/tests/st/ops/gpu/test_argmax_op.py b/tests/st/ops/gpu/test_argmax_op.py new file mode 100644 index 00000000000..8d5fd2fd7f2 --- /dev/null +++ b/tests/st/ops/gpu/test_argmax_op.py @@ -0,0 +1,59 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype +import mindspore.nn as nn +import mindspore.context as context + +class NetArgmax(nn.Cell): + def __init__( self): + super(NetArgmax, self).__init__() + axis1 = 0 + axis2 = -1 + self.argmax1 = P.Argmax(axis1, output_type=mstype.int32) + self.argmax2 = P.Argmax(axis2, output_type=mstype.int32) + self.argmax3 = P.Argmax(output_type=mstype.int32) + + def construct(self, x): + return (self.argmax1(x), self.argmax2(x), self.argmax3(x)) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_argmax(): + x = Tensor(np.array([[1., 20., 5.], + [67., 8., 9.], + [130., 24., 15.], + [0.3, -0.4, -15.]]).astype(np.float32)) + expect1 = np.array([2,2,2]).astype(np.int32) + expect2 = np.array([1,0,0,0]).astype(np.int32) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + Argmax = NetArgmax() + output = Argmax(x) + assert (output[0].asnumpy() == expect1).all() + assert (output[1].asnumpy() == expect2).all() + assert (output[2].asnumpy() == expect2).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + Argmax1 = NetArgmax() + output1 = Argmax(x) + assert (output1[0].asnumpy() == expect1).all() + assert (output1[1].asnumpy() == expect2).all() + assert (output1[2].asnumpy() == expect2).all() diff --git a/tests/st/ops/gpu/test_assign_add_op.py b/tests/st/ops/gpu/test_assign_add_op.py new file mode 100644 index 00000000000..f8faf2be646 --- /dev/null +++ b/tests/st/ops/gpu/test_assign_add_op.py @@ -0,0 +1,69 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + +class AssignAdd(nn.Cell): + def __init__( self): + super(AssignAdd, self).__init__() + self.add = P.AssignAdd() + + def construct(self, x, y): + res = self.add(x, y) + return res + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_assign_add(): + expect1 = np.array([[[[ 0, 2, 4.], + [ 6, 8, 10.], + [12, 14, 16.]], + [[18, 20, 22.], + [24, 26, 28.], + [30, 32, 34.]], + [[36, 38, 40.], + [42, 44, 46.], + [48, 50, 52.]]]]) + expect2 = np.array([[[[ 0, 3, 6], + [ 9, 12, 15], + [18, 21, 24]], + [[27, 30, 33], + [36, 39, 42], + [45, 48, 51]], + [[54, 57, 60], + [63, 66, 69], + [72, 75, 78]]]]) + x = Tensor(np.arange(1 * 3 * 3 * 3).reshape(1, 3, 3, 3).astype(np.float32)) + y = Tensor(np.arange(1 * 3 * 3 * 3).reshape(1, 3, 3, 3).astype(np.float32)) + + context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU') + add = AssignAdd() + output1 = add(x, y) + assert (output1.asnumpy() == expect1).all() + output2 = add(output1, y) + assert (output2.asnumpy() == expect2).all() + + context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + add = AssignAdd() + output1 = add(x, y) + assert (output1.asnumpy() == expect1).all() + output2 = add(output1, y) + assert (output2.asnumpy() == expect2).all() diff --git a/tests/st/ops/gpu/test_batchnorm_op.py b/tests/st/ops/gpu/test_batchnorm_op.py new file mode 100644 index 00000000000..4ca9ee8dd3d --- /dev/null +++ b/tests/st/ops/gpu/test_batchnorm_op.py @@ -0,0 +1,132 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore.nn import Cell +from mindspore.nn import BatchNorm2d +from mindspore.common.tensor import Tensor +from mindspore.ops import composite as C +import mindspore.context as context + + +class Batchnorm_Net(Cell): + def __init__(self, c, weight, bias, moving_mean, moving_var_init): + super(Batchnorm_Net, self).__init__() + self.bn = BatchNorm2d(c, eps=0.00001, momentum=0.1, beta_init=bias, gamma_init=weight, + moving_mean_init=moving_mean, moving_var_init=moving_var_init) + + def construct(self, input_data): + x = self.bn(input_data) + return x + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = C.GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, input_data, sens): + gout = self.grad(self.network)(input_data, sens) + return gout + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_train_forward(): + x = np.array([[ + [[1, 3, 3, 5], [2, 4, 6, 8], [3, 6, 7, 7], [4, 3, 8, 2]], + [[5, 7, 6, 3], [3, 5, 6, 7], [9, 4, 2, 5], [7, 5, 8, 1]]]]).astype(np.float32) + expect_output = np.array([[[[-0.6059, 0.3118, 0.3118, 1.2294], + [-0.1471, 0.7706, 1.6882, 2.6059], + [0.3118, 1.6882, 2.1471, 2.1471], + [0.7706, 0.3118, 2.6059, -0.1471]], + + [[0.9119, 1.8518, 1.3819, -0.0281], + [-0.0281, 0.9119, 1.3819, 1.8518], + [2.7918, 0.4419, -0.4981, 0.9119], + [1.8518, 0.9119, 2.3218, -0.9680]]]]).astype(np.float32) + grad = np.array([[ + [[1, 2, 7, 1], [4, 2, 1, 3], [1, 6, 5, 2], [2, 4, 3, 2]], + [[9, 4, 3, 5], [1, 3, 7, 6], [5, 7, 9, 9], [1, 4, 6, 8]]]]).astype(np.float32) + + weight = np.ones(2).astype(np.float32) + bias = np.ones(2).astype(np.float32) + moving_mean = np.ones(2).astype(np.float32) + moving_var_init = np.ones(2).astype(np.float32) + error = np.ones(shape=[1, 2, 4, 4]) * 1.0e-4 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + bn_net = Batchnorm_Net(2, Tensor(weight), Tensor(bias), Tensor(moving_mean), Tensor(moving_var_init)) + bn_net.set_train() + output = bn_net(Tensor(x)) + diff = output.asnumpy() - expect_output + assert np.all(diff < error) + assert np.all(-diff < error) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + bn_net = Batchnorm_Net(2, Tensor(weight), Tensor(bias), Tensor(moving_mean), Tensor(moving_var_init)) + bn_net.set_train() + output = bn_net(Tensor(x)) + diff = output.asnumpy() - expect_output + assert np.all(diff < error) + assert np.all(-diff < error) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + bn_net = Batchnorm_Net(2, Tensor(weight), Tensor(bias), Tensor(moving_mean), Tensor(moving_var_init)) + bn_net.set_train(False) + output = bn_net(Tensor(x)) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + bn_net = Batchnorm_Net(2, Tensor(weight), Tensor(bias), Tensor(moving_mean), Tensor(moving_var_init)) + bn_net.set_train(False) + output = bn_net(Tensor(x)) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_train_backward(): + x = np.array([[ + [[1, 3, 3, 5], [2, 4, 6, 8], [3, 6, 7, 7], [4, 3, 8, 2]], + [[5, 7, 6, 3], [3, 5, 6, 7], [9, 4, 2, 5], [7, 5, 8, 1]]]]).astype(np.float32) + grad = np.array([[ + [[1, 2, 7, 1], [4, 2, 1, 3], [1, 6, 5, 2], [2, 4, 3, 2]], + [[9, 4, 3, 5], [1, 3, 7, 6], [5, 7, 9, 9], [1, 4, 6, 8]]]]).astype(np.float32) + expect_output = np.array([[[[-0.69126546, -0.32903028, 1.9651246, -0.88445705], + [0.6369296, -0.37732816, -0.93275493, -0.11168876], + [-0.7878612, 1.3614, 0.8542711, -0.52222186], + [-0.37732816, 0.5886317, -0.11168876, -0.28073236]], + + [[1.6447213, -0.38968924, -1.0174079, -0.55067265], + [-2.4305856, -1.1751484, 0.86250514, 0.5502673], + [0.39576983, 0.5470243, 1.1715001, 1.6447213], + [-1.7996241, -0.7051701, 0.7080077, 0.5437813]]]]).astype(np.float32) + + weight = Tensor(np.ones(2).astype(np.float32)) + bias = Tensor(np.ones(2).astype(np.float32)) + moving_mean = Tensor(np.ones(2).astype(np.float32)) + moving_var_init = Tensor(np.ones(2).astype(np.float32)) + error = np.ones(shape=[1, 2, 4, 4]) * 1.0e-6 + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + bn_net = Batchnorm_Net(2, weight, bias, moving_mean, moving_var_init) + bn_net.set_train() + bn_grad = Grad(bn_net) + output = bn_grad(Tensor(x), Tensor(grad)) + diff = output[0].asnumpy() - expect_output + assert np.all(diff < error) + assert np.all(-diff < error) diff --git a/tests/st/ops/gpu/test_cast_op.py b/tests/st/ops/gpu/test_cast_op.py new file mode 100644 index 00000000000..3bbd0021aa4 --- /dev/null +++ b/tests/st/ops/gpu/test_cast_op.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore.ops import operations as P +from mindspore.nn import Cell +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as mstype +import mindspore.context as context +import numpy as np + + +class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.Cast = P.Cast() + + def construct(self, x0, type0, x1, type1): + output = (self.Cast(x0, type0), + self.Cast(x1, type1)) + return output + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_cast(): + x0 = Tensor(np.arange(24).reshape((4, 3, 2)).astype(np.float32)) + t0 = mstype.float16 + x1 = Tensor(np.arange(24).reshape((4, 3, 2)).astype(np.float16)) + t1 = mstype.float32 + + context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + net = Net() + output = net(x0, t0, x1, t1) + type0 = output[0].asnumpy().dtype + assert (type0 == 'float16') + type1 = output[1].asnumpy().dtype + assert (type1 == 'float32') diff --git a/tests/st/ops/gpu/test_concatv2_op.py b/tests/st/ops/gpu/test_concatv2_op.py new file mode 100644 index 00000000000..c1934762b4a --- /dev/null +++ b/tests/st/ops/gpu/test_concatv2_op.py @@ -0,0 +1,115 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(device_target='GPU') + + +class ConcatV32(nn.Cell): + def __init__(self): + super(ConcatV32, self).__init__() + + self.cat = P.Concat(axis=2) + self.x1 = Parameter(initializer( + Tensor(np.arange(2 * 2 * 1).reshape(2, 2, 1).astype(np.float32)), [2, 2, 1]), name='x1') + self.x2 = Parameter(initializer( + Tensor(np.arange(2 * 2 * 2).reshape(2, 2, 2).astype(np.float32)), [2, 2, 2]), name='x2') + + @ms_function + def construct(self): + return self.cat((self.x1, self.x2)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_axis32(): + cat = ConcatV32() + output = cat() + expect = [[[0., 0., 1.], + [1., 2., 3.]], + [[2., 4., 5.], + [3., 6., 7.]]] + print(output) + assert (output.asnumpy() == expect).all() + + +class ConcatV43(nn.Cell): + def __init__(self): + super(ConcatV43, self).__init__() + + self.cat = P.Concat(axis=3) + self.x1 = Parameter(initializer( + Tensor(np.arange(2 * 2 * 2 * 2).reshape(2, 2, 2, 2).astype(np.float32)), [2, 2, 2, 2]), name='x1') + self.x2 = Parameter(initializer( + Tensor(np.arange(2 * 2 * 2 * 3).reshape(2, 2, 2, 3).astype(np.float32)), [2, 2, 2, 3]), name='x2') + + @ms_function + def construct(self): + return self.cat((self.x1, self.x2)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_axis43(): + cat = ConcatV43() + output = cat() + expect = [[[[0., 1., 0., 1., 2.], + [2., 3., 3., 4., 5.]], + [[4., 5., 6., 7., 8.], + [6., 7., 9., 10., 11.]]], + [[[8., 9., 12., 13., 14.], + [10., 11., 15., 16., 17.]], + [[12., 13., 18., 19., 20.], + [14., 15., 21., 22., 23.]]]] + assert (output.asnumpy() == expect).all() + print(output) + + +class ConcatV21(nn.Cell): + def __init__(self): + super(ConcatV21, self).__init__() + + self.cat = P.Concat(axis=1) + self.x1 = Parameter(initializer( + Tensor(np.arange(2 * 2).reshape(2, 2).astype(np.float32)), [2, 2]), name='x1') + self.x2 = Parameter(initializer( + Tensor(np.arange(2 * 3).reshape(2, 3).astype(np.float32)), [2, 3]), name='x2') + + @ms_function + def construct(self): + return self.cat((self.x1, self.x2)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_axis21(): + cat = ConcatV21() + output = cat() + expect = [[0., 1., 0., 1., 2.], + [2., 3., 3., 4., 5.]] + assert (output.asnumpy() == expect).all() + print(output) diff --git a/tests/st/ops/gpu/test_conv2d_backprop_filter_op.py b/tests/st/ops/gpu/test_conv2d_backprop_filter_op.py new file mode 100644 index 00000000000..6e2e76cd47c --- /dev/null +++ b/tests/st/ops/gpu/test_conv2d_backprop_filter_op.py @@ -0,0 +1,77 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target='GPU') + + +class Conv2dFilter(nn.Cell): + def __init__(self): + super(Conv2dFilter, self).__init__() + out_channel = 1 + kernel_size = 3 + self.conv_filter = G.Conv2DBackpropFilter(out_channel, + kernel_size, + pad_mode="valid", + pad=0, + mode=1, + stride=1, + dilation=1, + group=1) + + self.get_shape = P.Shape() + + @ms_function + def construct(self, out, x, w): + return self.conv_filter(out, x, self.get_shape(w)) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_conv2d_backprop_filter(): + w = Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)) + x = Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)) + out = Tensor(np.array([[[ + [-5, -4, 0, 8], + [-10, -2, 2, 3], + [0, -2, -4, -7], + [-3, -2, -3, -16]]]]).astype(np.float32)) + conv2d_filter = Conv2dFilter() + output = conv2d_filter(out, x, w) + print("================================") + """ + expect output: + [[[[ -60, -142, -265] + [-104, -211, -322] + [-102, -144, -248]]]] + """ + expect = np.array([[[[-60, -142, -265], + [-104, -211, -322], + [-102, -144, -248]]]]).astype(np.float32) + assert (abs(output.asnumpy() - expect) < np.ones(shape=[1, 1, 3, 3]) * 1.0e-4).all() diff --git a/tests/st/ops/gpu/test_conv2d_backprop_input_op.py b/tests/st/ops/gpu/test_conv2d_backprop_input_op.py new file mode 100644 index 00000000000..dcc6846100b --- /dev/null +++ b/tests/st/ops/gpu/test_conv2d_backprop_input_op.py @@ -0,0 +1,76 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(device_target='GPU') + + +class Conv2dInput(nn.Cell): + def __init__(self): + super(Conv2dInput, self).__init__() + out_channel = 1 + kernel_size = 3 + self.conv_input = P.Conv2DBackpropInput(out_channel, + kernel_size, + pad_mode="valid", + pad=0, + mode=1, + stride=1, + dilation=1, + group=1) + + self.get_shape = P.Shape() + + @ms_function + def construct(self, out, w, x): + return self.conv_input(out, w, self.get_shape(x)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_conv2d_backprop_input(): + w = Tensor(np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32)) + x = Tensor(np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32)) + out = Tensor(np.array([[[ + [-5, -4, 0, 8], + [-10, -2, 2, 3], + [0, -2, -4, -7], + [-3, -2, -3, -16]]]]).astype(np.float32)) + conv2d_input = Conv2dInput() + output = conv2d_input(out, w, x) + expect = np.array([[[[-5, -4, 5, 12, 0, -8], + [-15, -6, 17, 17, -2, -11], + [-15, -8, 13, 12, 2, -4], + [-13, -6, 8, -14, 5, 20], + [-3, -4, -4, -19, 7, 23], + [-3, -2, 0, -14, 3, 16]]]]).astype(np.float32) + + assert (abs(output.asnumpy() - expect) < np.ones(shape=[1, 1, 6, 6]) * 1.0e-4).all() diff --git a/tests/st/ops/gpu/test_conv2d_op.py b/tests/st/ops/gpu/test_conv2d_op.py new file mode 100644 index 00000000000..d724f6f6c84 --- /dev/null +++ b/tests/st/ops/gpu/test_conv2d_op.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetConv2d(nn.Cell): + def __init__(self): + super(NetConv2d, self).__init__() + out_channel = 2 + kernel_size = 1 + self.conv = P.Conv2D(out_channel, + kernel_size, + mode=1, + pad_mode="valid", + pad=0, + stride=1, + dilation=1, + group=1) + + def construct(self, x, w): + return self.conv(x, w) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_conv2d(): + x = Tensor(np.arange(1 * 3 * 3 * 3).reshape(1, 3, 3, 3).astype(np.float32)) + w = Tensor(np.arange(2 * 3 * 1 * 1).reshape(2, 3, 1, 1).astype(np.float32)) + expect = np.array([[[[45, 48, 51], + [54, 57, 60], + [63, 66, 69]], + [[126, 138, 150], + [162, 174, 186], + [198, 210, 222]]]]).astype(np.float32) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + conv2d = NetConv2d() + output = conv2d(x, w) + assert (output.asnumpy() == expect).all() + """ + expect output: + [[[[ 45. 48. 51.] + [ 54. 57. 60.] + [ 63. 66. 69.]] + + [[126. 138. 150.] + [162. 174. 186.] + [198. 210. 222.]]]] + """ + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + conv2d = NetConv2d() + output = conv2d(x, w) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_dense_op.py b/tests/st/ops/gpu/test_dense_op.py new file mode 100644 index 00000000000..2789e3e9f6d --- /dev/null +++ b/tests/st/ops/gpu/test_dense_op.py @@ -0,0 +1,359 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import mindspore.context as context +from mindspore.ops.composite import GradOperation +from mindspore.common.parameter import ParameterTuple +from mindspore.ops import composite as C + + +class BiasAdd(nn.Cell): + def __init__(self): + super(BiasAdd, self).__init__() + self.ba = P.BiasAdd() + + def construct(self, x, b): + return self.ba(x, b) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_biasadd(): + x = Tensor(np.array([[0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.9, 1.0, 1.1, 1.2]]).astype(np.float32)) + b = Tensor(np.array([0.1, 0.2, 0.3, 0.4]).astype(np.float32)) + expect = np.array([[0.2, 0.4, 0.6, 0.8], + [0.6, 0.8, 1.0, 1.2], + [1.0, 1.2, 1.4, 1.6]]) + error = np.ones(shape=[3, 4]) * 1.0e-6 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + ba = BiasAdd() + result = ba(x, b) + diff = result.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + ba = BiasAdd() + result = ba(x, b) + diff = result.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + +class GradData(nn.Cell): + def __init__(self, network): + super(GradData, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, input, output_grad): + return self.grad(self.network)(input, output_grad) + + +class GradWeight(nn.Cell): + def __init__(self, network): + super(GradWeight, self).__init__() + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + + def construct(self, x, output_grad): + weights = self.weights + grads = self.grad(self.network, weights)(x, output_grad) + return grads + + +class DenseNet(nn.Cell): + def __init__(self): + super(DenseNet, self).__init__() + w = np.array([[0.1, 0.8, 0.1, 0.1], + [1, 1, 1, 1]]).astype(np.float32) + b = np.array([0.3, 0.6]).astype(np.float32) + self.dense = nn.Dense(4, 2, weight_init=Tensor(w), bias_init=Tensor(b)) + + def construct(self, x): + return self.dense(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_dx(): + x = np.array([[0.1, 0.2, 0.3, 0.4], + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.2, 0.3, 0.4]]).astype(np.float32) + dy = np.array([[1, 1], + [1, 1], + [1, 1]]).astype(np.float32) + dx_expect = np.array([[1.1, 1.8, 1.1, 1.1], + [1.1, 1.8, 1.1, 1.1], + [1.1, 1.8, 1.1, 1.1]]).astype(np.float32) + error = np.ones(shape=[3, 4]) * 1.0e-6 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + net = GradData(DenseNet()) + dx = net(Tensor(x), Tensor(dy)) + diff = dx[0].asnumpy() - dx_expect + assert np.all(diff < error) + assert np.all(-diff < error) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + net = GradData(DenseNet()) + dx = net(Tensor(x), Tensor(dy)) + diff = dx[0].asnumpy() - dx_expect + assert np.all(diff < error) + assert np.all(-diff < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_dw(): + x = np.array([[0.1, 0.2, 0.3, 0.4], + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.2, 0.3, 0.4]]).astype(np.float32) + dy = np.array([[1, 1], + [1, 1], + [1, 1]]).astype(np.float32) + dw_expect = np.array([[0.3, 0.6, 0.9, 1.2], + [0.3, 0.6, 0.9, 1.2]]).astype(np.float32) + dw_error = np.ones(shape=[2, 4]) * 1.0e-6 + db_expect = np.array([3, 3]).astype(np.float32) + db_error = np.ones(shape=[2]) * 1.0e-6 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + net = GradWeight(DenseNet()) + dw, db = net(Tensor(x), Tensor(dy)) + diff = dw.asnumpy() - dw_expect + assert np.all(diff < dw_error) + assert np.all(-diff < dw_error) + diff = db.asnumpy() - db_expect + assert np.all(diff < db_error) + assert np.all(-diff < db_error) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + net = GradWeight(DenseNet()) + dw, db = net(Tensor(x), Tensor(dy)) + diff = dw.asnumpy() - dw_expect + assert np.all(diff < dw_error) + assert np.all(-diff < dw_error) + diff = db.asnumpy() - db_expect + assert np.all(diff < db_error) + assert np.all(-diff < db_error) + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.grad = GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, input, bias, dy): + return self.grad(self.network)(input, bias, dy) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_biasadd_3d(): + x = Tensor(np.array([[[1, 2, 3, 4, 5, 6, 7, 8], + [9, 10, 11, 12, 13, 14, 15, 16], + [17, 18, 19, 20, 21, 22, 23, 24], + [25, 26, 27, 28, 29, 30, 31, 32]], + + [[33, 34, 35, 36, 37, 38, 39, 40], + [41, 42, 43, 44, 45, 46, 47, 48], + [49, 50, 51, 52, 53, 54, 55, 56], + [57, 58, 59, 60, 61, 62, 63, 64]], + + [[65, 66, 67, 68, 69, 70, 71, 72], + [73, 74, 75, 76, 77, 78, 79, 80], + [81, 82, 83, 84, 85, 86, 87, 88], + [89, 90, 91, 92, 93, 94, 95, 96]]]).astype(np.float32)) + b = Tensor(np.array([1, 2, 3, 4]).astype(np.float32)) + dy = Tensor(np.array([[[1, 2, 3, 4, 5, 6, 7, 8], + [9, 10, 11, 12, 13, 14, 15, 16], + [17, 18, 19, 20, 21, 22, 23, 24], + [25, 26, 27, 28, 29, 30, 31, 32]], + + [[33, 34, 35, 36, 37, 38, 39, 40], + [41, 42, 43, 44, 45, 46, 47, 48], + [49, 50, 51, 52, 53, 54, 55, 56], + [57, 58, 59, 60, 61, 62, 63, 64]], + + [[65, 66, 67, 68, 69, 70, 71, 72], + [73, 74, 75, 76, 77, 78, 79, 80], + [81, 82, 83, 84, 85, 86, 87, 88], + [89, 90, 91, 92, 93, 94, 95, 96]]]).astype(np.float32)) + + expect = np.array([[[2, 3, 4, 5, 6, 7, 8, 9], + [11, 12, 13, 14, 15, 16, 17, 18], + [20, 21, 22, 23, 24, 25, 26, 27], + [29, 30, 31, 32, 33, 34, 35, 36]], + + [[34, 35, 36, 37, 38, 39, 40, 41], + [43, 44, 45, 46, 47, 48, 49, 50], + [52, 53, 54, 55, 56, 57, 58, 59], + [61, 62, 63, 64, 65, 66, 67, 68]], + + [[66, 67, 68, 69, 70, 71, 72, 73], + [75, 76, 77, 78, 79, 80, 81, 82], + [84, 85, 86, 87, 88, 89, 90, 91], + [93, 94, 95, 96, 97, 98, 99, 100]]]) + + error = np.ones(shape=[3, 4, 8]) * 1.0e-6 + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + net = BiasAdd() + result = net(x, b) + diff = result.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + net = Grad(net) + _, result = net(x, b, dy) + expect = np.array([876., 1068., 1260., 1452.]) + diff = result.asnumpy() - expect + error = np.ones(shape=[4]) * 1.0e-6 + assert np.all(diff < error) + assert np.all(-diff < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_biasadd_4d(): + x = Tensor(np.array([[[[1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16]], + + [[17, 18, 19, 20], + [21, 22, 23, 24], + [25, 26, 27, 28], + [29, 30, 31, 32]], + + [[33, 34, 35, 36], + [37, 38, 39, 40], + [41, 42, 43, 44], + [45, 46, 47, 48]]], + + [[[49, 50, 51, 52], + [53, 54, 55, 56], + [57, 58, 59, 60], + [61, 62, 63, 64]], + + [[65, 66, 67, 68], + [69, 70, 71, 72], + [73, 74, 75, 76], + [77, 78, 79, 80]], + + [[81, 82, 83, 84], + [85, 86, 87, 88], + [89, 90, 91, 92], + [93, 94, 95, 96]]]]).astype(np.float32)) + b = Tensor(np.array([1, 2, 3]).astype(np.float32)) + dy = Tensor(np.array([[[[1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16]], + + [[17, 18, 19, 20], + [21, 22, 23, 24], + [25, 26, 27, 28], + [29, 30, 31, 32]], + + [[33, 34, 35, 36], + [37, 38, 39, 40], + [41, 42, 43, 44], + [45, 46, 47, 48]]], + + [[[49, 50, 51, 52], + [53, 54, 55, 56], + [57, 58, 59, 60], + [61, 62, 63, 64]], + + [[65, 66, 67, 68], + [69, 70, 71, 72], + [73, 74, 75, 76], + [77, 78, 79, 80]], + + [[81, 82, 83, 84], + [85, 86, 87, 88], + [89, 90, 91, 92], + [93, 94, 95, 96]]]]).astype(np.float32)) + + expect = np.array([[[[2, 3, 4, 5], + [6, 7, 8, 9], + [10, 11, 12, 13], + [14, 15, 16, 17]], + + [[19, 20, 21, 22], + [23, 24, 25, 26], + [27, 28, 29, 30], + [31, 32, 33, 34]], + + [[36, 37, 38, 39], + [40, 41, 42, 43], + [44, 45, 46, 47], + [48, 49, 50, 51]]], + + [[[50, 51, 52, 53], + [54, 55, 56, 57], + [58, 59, 60, 61], + [62, 63, 64, 65]], + + [[67, 68, 69, 70], + [71, 72, 73, 74], + [75, 76, 77, 78], + [79, 80, 81, 82]], + + [[84, 85, 86, 87], + [88, 89, 90, 91], + [92, 93, 94, 95], + [96, 97, 98, 99]]]]) + error = np.ones(shape=[2, 3, 4, 4]) * 1.0e-6 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + ba = BiasAdd() + result = ba(x, b) + diff = result.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + net = BiasAdd() + result = net(x, b) + diff = result.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + net = Grad(net) + _, result = net(x, b, dy) + expect = np.array([1040., 1552., 2064.]) + diff = result.asnumpy() - expect + error = np.ones(shape=[3]) * 1.0e-6 + assert np.all(diff < error) + assert np.all(-diff < error) diff --git a/tests/st/ops/gpu/test_equal_op.py b/tests/st/ops/gpu/test_equal_op.py new file mode 100644 index 00000000000..cf0f7463693 --- /dev/null +++ b/tests/st/ops/gpu/test_equal_op.py @@ -0,0 +1,64 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore.ops import operations as P +from mindspore.nn import Cell +from mindspore.common.tensor import Tensor +import mindspore.context as context +import numpy as np + + +class NetEqual(Cell): + def __init__(self): + super(NetEqual, self).__init__() + self.Equal = P.Equal() + + def construct(self, x, y): + return self.Equal(x, y) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_equal(): + x0_np = np.arange(24).reshape((4, 3, 2)).astype(np.float32) + x0 = Tensor(x0_np) + y0_np = np.arange(24).reshape((4, 3, 2)).astype(np.float32) + y0 = Tensor(y0_np) + expect0 = np.equal(x0_np, y0_np) + x1_np = np.array([0, 1, 3]).astype(np.float32) + x1 = Tensor(x1_np) + y1_np = np.array([0, 1, -3]).astype(np.float32) + y1 = Tensor(y1_np) + expect1 = np.equal(x1_np, y1_np) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + equal = NetEqual() + output0 = equal(x0, y0) + assert np.all(output0.asnumpy() == expect0) + assert (output0.shape() == expect0.shape) + output1 = equal(x1, y1) + assert np.all(output1.asnumpy() == expect1) + assert (output1.shape() == expect1.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + equal = NetEqual() + output0 = equal(x0, y0) + assert np.all(output0.asnumpy() == expect0) + assert (output0.shape() == expect0.shape) + output1 = equal(x1, y1) + assert np.all(output1.asnumpy() == expect1) + assert (output1.shape() == expect1.shape) diff --git a/tests/st/ops/gpu/test_equalcount_op.py b/tests/st/ops/gpu/test_equalcount_op.py new file mode 100644 index 00000000000..a48182d5064 --- /dev/null +++ b/tests/st/ops/gpu/test_equalcount_op.py @@ -0,0 +1,47 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import mindspore.context as context + +class NetEqualCount(nn.Cell): + def __init__(self): + super(NetEqualCount, self).__init__() + self.equalcount = P.EqualCount() + + def construct(self, x, y): + return self.equalcount(x, y) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_equalcount(): + x = Tensor(np.array([1, 20, 5]).astype(np.int32)) + y = Tensor(np.array([2, 20, 5]).astype(np.int32)) + expect = np.array([2]).astype(np.int32) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + equal_count = NetEqualCount() + output = equal_count(x, y) + assert (output.asnumpy() == expect).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + equal_count = NetEqualCount() + output = equal_count(x, y) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_exp_op.py b/tests/st/ops/gpu/test_exp_op.py new file mode 100644 index 00000000000..9397839793d --- /dev/null +++ b/tests/st/ops/gpu/test_exp_op.py @@ -0,0 +1,65 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetExp(nn.Cell): + def __init__(self): + super(NetExp, self).__init__() + self.exp = P.Exp() + + def construct(self, x): + return self.exp(x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_exp(): + x0_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.uniform(-2, 2, 1).astype(np.float32) + x0 = Tensor(x0_np) + x1 = Tensor(x1_np) + expect0 = np.exp(x0_np) + expect1 = np.exp(x1_np) + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + exp = NetExp() + output0 = exp(x0) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + output1 = exp(x1) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + exp = NetExp() + output0 = exp(x0) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + output1 = exp(x1) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) diff --git a/tests/st/ops/gpu/test_flatten_grad_op.py b/tests/st/ops/gpu/test_flatten_grad_op.py new file mode 100644 index 00000000000..68150ea5940 --- /dev/null +++ b/tests/st/ops/gpu/test_flatten_grad_op.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetFlattenGrad(nn.Cell): + def __init__(self): + super(NetFlattenGrad, self).__init__() + self.flattengrad = G.FlattenGrad() + self.type = (2, 3) + + def construct(self, x): + return self.flattengrad(x, self.type) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_flatten_grad(): + x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]).astype(np.float32)) + """ + expect output: + [ [-0.1 0.3 3.6] + [ 0.4 0.5 -3.2] ] + """ + expect = np.array([[-0.1, 0.3, 3.6], + [0.4, 0.5, -3.2]]).astype(np.float32) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + flattengrad = NetFlattenGrad() + output = flattengrad(x) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_flatten_op.py b/tests/st/ops/gpu/test_flatten_op.py new file mode 100644 index 00000000000..258cc53b9fa --- /dev/null +++ b/tests/st/ops/gpu/test_flatten_op.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetFlatten(nn.Cell): + def __init__(self): + super(NetFlatten, self).__init__() + self.flatten = P.Flatten() + + def construct(self, x): + return self.flatten(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_flatten(): + x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]).astype(np.float32)) + expect = np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]).astype(np.float32) + """ + expect output: + [[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]] + """ + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + flatten = NetFlatten() + output = flatten(x) + assert (output.asnumpy() == expect).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + flatten = NetFlatten() + output = flatten(x) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_gather_op.py b/tests/st/ops/gpu/test_gather_op.py new file mode 100644 index 00000000000..758a5ca654a --- /dev/null +++ b/tests/st/ops/gpu/test_gather_op.py @@ -0,0 +1,938 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class GatherNet(nn.Cell): + def __init__(self): + super(GatherNet, self).__init__() + self.gather = P.GatherV2() + + def construct(self, x, indices): + return self.gather(x, indices, 1) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_gather0(): + x = Tensor(np.arange(2 * 3 * 4 * 5, dtype=np.float32).reshape(2, 3, 4, 5)) + indices = Tensor(np.ones((2, 2, 4, 5), dtype='i4')) + expect = np.array([[[[[[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]]], + + [[[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]]]], + + [[[[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]]], + + [[[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]], + + [[[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]], + + [[20., 21., 22., 23., 24.], + [25., 26., 27., 28., 29.], + [30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.]]]]]], + + [[[[[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]]], + + [[[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]]]], + + [[[[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]]], + + [[[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]], + + [[[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]], + + [[80., 81., 82., 83., 84.], + [85., 86., 87., 88., 89.], + [90., 91., 92., 93., 94.], + [95., 96., 97., 98., 99.]]]]]]]) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + gather = GatherNet() + output = gather(x, indices) + error = np.ones(shape=output.asnumpy().shape) * 1.0e-6 + diff = output.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + +class GatherNet1(nn.Cell): + def __init__(self): + super(GatherNet1, self).__init__() + self.gather = P.GatherV2() + + def construct(self, x, indices): + return self.gather(x, indices, -1) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_gather1(): + x = Tensor(np.arange(2 * 3 * 4 * 5, dtype=np.float32).reshape(2, 3, 4, 5)) + indices = Tensor(np.array([1, 3, 4], dtype='i4')) + expect = np.array([[[[1., 3., 4.], + [6., 8., 9.], + [11., 13., 14.], + [16., 18., 19.]], + + [[21., 23., 24.], + [26., 28., 29.], + [31., 33., 34.], + [36., 38., 39.]], + + [[41., 43., 44.], + [46., 48., 49.], + [51., 53., 54.], + [56., 58., 59.]]], + + [[[61., 63., 64.], + [66., 68., 69.], + [71., 73., 74.], + [76., 78., 79.]], + + [[81., 83., 84.], + [86., 88., 89.], + [91., 93., 94.], + [96., 98., 99.]], + + [[101., 103., 104.], + [106., 108., 109.], + [111., 113., 114.], + [116., 118., 119.]]]]) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + gather = GatherNet1() + output = gather(x, indices) + error = np.ones(shape=output.asnumpy().shape) * 1.0e-6 + diff = output.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) + + +class GatherNet2(nn.Cell): + def __init__(self): + super(GatherNet2, self).__init__() + self.gather = P.GatherV2() + + def construct(self, x, indices): + return self.gather(x, indices, 0) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_gather2(): + x = Tensor(np.array([[4., 5., 4., 1., 5., ], + [4., 9., 5., 6., 4., ], + [9., 8., 4., 3., 6., ], + [0., 4., 2., 2., 8., ], + [1., 8., 6., 2., 8., ], + [8., 1., 9., 7., 3., ], + [7., 9., 2., 5., 7., ], + [9., 8., 6., 8., 5., ], + [3., 7., 2., 7., 4., ], + [4., 2., 8., 2., 9., ]] + ).astype(np.float32)) + + indices = Tensor(np.array([[4000, 1, 300000]]).astype(np.int32)) + expect = np.array([[[0., 0., 0., 0., 0.], + [4., 9., 5., 6., 4.], + [0., 0., 0., 0., 0.]]]) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + gather = GatherNet2() + output = gather(x, indices) + error = np.ones(shape=output.asnumpy().shape) * 1.0e-6 + diff = output.asnumpy() - expect + assert np.all(diff < error) + assert np.all(-diff < error) diff --git a/tests/st/ops/gpu/test_log_op.py b/tests/st/ops/gpu/test_log_op.py new file mode 100644 index 00000000000..e3de1232c9c --- /dev/null +++ b/tests/st/ops/gpu/test_log_op.py @@ -0,0 +1,66 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetLog(nn.Cell): + def __init__(self): + super(NetLog, self).__init__() + self.log = P.Log() + + def construct(self, x): + return self.log(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_log(): + x0_np = np.random.uniform(1, 2, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.uniform(1, 2, 1).astype(np.float32) + x0 = Tensor(x0_np) + x1 = Tensor(x1_np) + expect0 = np.log(x0_np) + expect1 = np.log(x1_np) + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + log = NetLog() + output0 = log(x0) + output1 = log(x1) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + log = NetLog() + output0 = log(x0) + output1 = log(x1) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) diff --git a/tests/st/ops/gpu/test_lstm_op.py b/tests/st/ops/gpu/test_lstm_op.py new file mode 100644 index 00000000000..f42ca5d1f10 --- /dev/null +++ b/tests/st/ops/gpu/test_lstm_op.py @@ -0,0 +1,991 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import ParameterTuple, Parameter + +context.set_context(device_target='GPU') + + +class LstmNet(nn.Cell): + def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + super(LstmNet, self).__init__() + + num_directions = 1 + if bidirectional: + num_directions = 2 + + self.lstm = P.LSTM(input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + + input_np = np.array([[[0.6755, -1.6607, 0.1367, -0.9209, -1.7088, 0.3953, 2.7120, 0.1103, 0.1504, -0.3611], + [0.4276, -0.7850, -0.3758, 0.8604, -0.1361, -1.3618, -0.6251, -0.8391, 0.8142, 0.4068]], + + [[-0.6424, -0.6095, 0.6639, -0.7253, 2.1190, -0.2840, 0.3858, 0.1691, 0.6764, 1.2903], + [0.7918, 0.4147, -0.5089, -0.3582, -1.4279, -0.7975, -0.0390, -0.4718, 0.4322, -0.7995]], + + [[-1.5612, 0.0120, -0.7289, -1.2479, -0.6197, -0.6099, 0.9543, 0.4362, -1.3141, 0.4273], + [-0.6656, -0.6626, -0.5883, -0.6922, 0.5512, 1.7031, -1.2812, -0.2004, -0.9224, 0.4106]], + + [[-0.9667, -0.6296, -0.7310, 1.2503, -0.1650, 1.2050, -0.1704, -0.5215, 0.1595, 0.3904], + [0.1026, -0.6821, -0.4387, -1.1637, -0.5000, 0.0590, 0.5219, -0.6835, 2.4406, 0.7135]], + + [[-0.4710, 0.6558, -0.3144, -1.2213, 0.1556, -0.3836, -0.1081, -0.1440, -1.1231, 0.6279], + [-0.8449, -0.2184, -0.1806, -0.0615, -0.5660, -0.3556, 1.6891, -1.0286, 1.3361, + -0.4313]]]).astype(np.float32) + + self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x') + + self.h = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='h') + + self.c = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='c') + + wih = np.array([[3.4021e-01, -4.6622e-01, 4.5117e-01, 2.3627e-01, 3.7844e-01, + 2.8770e-01, 4.1631e-01, -6.2628e-01, -4.8008e-01, -4.9148e-01], + [-6.4257e-02, -2.4807e-01, 1.3550e-02, 6.8946e-01, -1.2608e-02, + -7.1719e-02, -1.3566e-01, -4.9215e-01, 2.8509e-01, -6.3540e-01], + [-6.9863e-01, 5.9773e-01, -3.9062e-01, -7.6151e-02, 5.6803e-04, + -7.0420e-01, -6.1822e-01, 4.1854e-01, 4.0596e-01, 6.4867e-01], + [-3.0253e-01, -1.9464e-01, 7.0591e-01, 4.9368e-01, -5.9758e-01, + 1.3251e-02, 3.5685e-01, -3.7640e-01, -4.4612e-01, 5.1794e-01], + [-3.2140e-01, 5.5578e-01, 6.3589e-01, -6.4249e-01, 5.7258e-01, + 2.4256e-01, -2.7954e-01, 2.5202e-01, 2.9235e-01, -3.9979e-01], + [1.6547e-01, -7.9030e-02, -2.0045e-01, 6.2484e-01, -1.0727e-01, + -5.0010e-01, -2.9165e-01, -1.7620e-01, 1.5939e-01, -2.2744e-01], + [-4.0835e-01, 3.6751e-01, 4.7989e-01, 5.8886e-01, 5.3598e-01, + -2.9055e-01, -2.8129e-01, 6.0219e-01, 4.9193e-01, 3.3115e-01], + [-5.6894e-01, -5.0359e-01, 4.7491e-01, 5.8110e-01, -5.4921e-01, + -6.1343e-01, -5.8236e-02, -3.7682e-01, 4.8338e-01, -2.1551e-01]]).astype(np.float32).reshape( + [1, -1]) + + whh = np.array([[-0.4820, -0.2350], + [-0.1195, 0.0519], + [0.4511, -0.3961], + [-0.5962, 0.0906], + [0.2162, -0.1178], + [0.6237, 0.0711], + [0.1867, -0.1225], + [0.1831, 0.0850]]).astype(np.float32).reshape([1, -1]) + + bih = np.array([-0.2862, 0.0034, 0.2059, -0.6544, 0.3244, -0.2472, 0.0852, -0.3050]).astype(np.float32).reshape( + [1, -1]) + bhh = np.array([-0.6575, 0.1562, -0.6434, 0.0212, -0.2493, -0.5626, 0.1530, -0.5235]).astype( + np.float32).reshape([1, -1]) + + w_np = np.concatenate((wih, whh, bih, bhh), axis=1).reshape([-1, 1, 1]) + + self.w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w') + + @ms_function + def construct(self): + return self.lstm(self.x, self.h, self.c, self.w) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_lstm(): + seq_len = 5 + batch_size = 2 + + input_size = 10 + hidden_size = 2 + num_layers = 1 + has_bias = True + bidirectional = False + dropout = 0.0 + + num_directions = 1 + if bidirectional: + num_directions = 2 + + net = LstmNet(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + y, h, c, _, _ = net() + expect_y = np.array([[[-2.1429e-02, 1.1760e-01], + [3.1144e-01, 6.3090e-01]], + + [[-5.0190e-04, -4.5812e-02], + [2.0324e-02, 2.0392e-01]], + + [[-1.0370e-02, -6.0141e-02], + [6.0931e-02, -1.8913e-02]], + + [[-1.6031e-01, -2.3428e-01], + [4.1886e-02, -2.2162e-01]], + + [[-3.9243e-02, -3.2950e-02], + [-4.1257e-02, -4.5276e-01]]]) + + error = np.ones([num_layers, batch_size, hidden_size]) * 1.0e-4 + diff = y.asnumpy() - expect_y + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_h = np.array([[[-0.0392, -0.0329], + [-0.0413, -0.4528]]]) + error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-4 + diff = h.asnumpy() - expect_h + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_c = np.array([[[-0.0984, -0.3665], + [-0.1010, -0.6792]]]) + error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-4 + diff = c.asnumpy() - expect_c + assert np.all(diff < error) + assert np.all(-diff < error) + + +class BiLstmNet(nn.Cell): + def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + super(BiLstmNet, self).__init__() + + num_directions = 1 + if bidirectional: + num_directions = 2 + + self.lstm = P.LSTM(input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + + input_np = np.array([[[-1.7322, 1.6642, -1.1861, 0.2955, -0.7907, 0.2982, -1.3413, 1.0665, -0.0436, -0.1883], + [0.2195, 0.5917, -0.6739, 0.2388, -0.5364, -1.3309, -0.6018, -0.3081, -0.9648, -1.1627]], + + [[-0.5094, -2.6025, -0.9302, -1.1937, 0.6501, -0.1903, -0.0661, 0.1080, 0.9829, -0.2280], + [1.3961, 0.2239, -0.1947, -0.3206, 0.5791, 0.3396, 0.1728, -1.2007, -1.0994, -1.3278]], + + [[0.1870, -1.1090, -0.9705, 0.2207, 0.3743, 0.1158, -0.5443, -0.5559, 0.1538, -0.3975], + [-0.2347, -0.1245, -0.2335, 0.3164, 1.0997, -0.3928, -1.8517, 1.1136, -1.5051, -0.0071]], + + [[1.2739, 2.5438, -0.4289, -0.7981, -1.3682, -2.2509, 0.2028, 1.3410, 2.9502, -1.1650], + [0.1254, 0.2726, 0.0251, 0.9323, 0.7315, 0.8231, -0.2123, -0.6885, 0.9893, -0.2047]], + + [[0.1870, -0.9066, 0.7155, 0.5438, -0.9757, -0.5828, -0.3417, 1.5681, 1.0326, -0.0179], + [-0.7746, -1.0695, -0.5278, 2.5307, -0.1002, -1.5773, 0.7717, 1.0266, -0.0798, + 1.2333]]]).astype(np.float32) + + self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x') + + self.h = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='h') + + self.c = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='c') + + wih = np.array([[-0.2959, -0.1142, 0.3662, 0.5406, 0.1738, 0.2697, -0.6960, -0.0464, 0.3486, 0.1888], + [0.3043, 0.1505, -0.1207, -0.2456, 0.2735, 0.6673, -0.3352, -0.6153, -0.5731, -0.2726], + [-0.2657, -0.5570, 0.6785, -0.1861, -0.0652, 0.5757, 0.6442, -0.4068, -0.3260, 0.7054], + [0.6607, 0.6927, -0.1354, 0.2484, 0.2053, 0.5743, -0.0212, 0.3340, -0.5685, -0.5668], + [0.6701, -0.3013, -0.1202, -0.4200, -0.4280, -0.6329, -0.6074, -0.4997, -0.6215, -0.6259], + [0.0299, -0.6071, -0.4683, -0.3363, -0.0044, -0.0007, 0.2700, 0.0202, -0.2880, -0.6869], + [0.3025, -0.2461, -0.5128, 0.6327, -0.1438, -0.5100, 0.1924, 0.2023, 0.3129, 0.2271], + [0.3777, 0.0546, 0.4790, -0.1895, 0.3588, 0.4490, 0.6850, 0.6240, -0.2739, -0.4474]]).astype( + np.float32).reshape([1, -1]) + + whh = np.array([[0.6346, -0.6366], + [-0.0248, -0.6156], + [-0.3821, 0.6327], + [-0.6132, -0.5071], + [0.4029, 0.0906], + [-0.5671, 0.2556], + [0.0268, -0.4347], + [0.1152, -0.3124]]).astype(np.float32).reshape([1, -1]) + + bih = np.array([-0.3839, -0.5365, -0.6691, 0.1697, -0.1564, -0.0451, -0.5921, -0.5367]).astype( + np.float32).reshape([1, -1]) + bhh = np.array([0.5952, -0.4905, 0.0423, -0.0293, -0.6638, 0.4348, -0.4291, -0.5541]).astype( + np.float32).reshape([1, -1]) + + wih_reverse = np.array([[-0.2938, 0.0048, 0.2704, -0.3387, -0.4529, -0.2586, 0.1352, -0.1208, -0.1423, -0.0220], + [-0.3701, 0.0201, -0.0255, 0.1340, -0.1938, -0.7056, -0.2303, 0.4814, 0.3636, -0.5018], + [-0.0284, -0.0108, -0.5788, 0.2389, 0.2604, 0.6774, -0.5525, 0.6265, -0.6126, 0.3197], + [-0.6906, 0.6991, -0.6138, 0.0044, 0.5714, 0.4176, 0.5451, -0.5114, -0.2286, 0.1105], + [0.3547, 0.6233, -0.4543, -0.6799, 0.1109, 0.5601, 0.0212, 0.6926, 0.0597, -0.4383], + [-0.1370, -0.5852, 0.0596, 0.5494, 0.5789, -0.0534, 0.1092, 0.3544, -0.1571, 0.4444], + [-0.5886, -0.4765, -0.3837, -0.6634, 0.0963, -0.1385, -0.0837, -0.1354, 0.0547, + -0.2870], + [0.2049, -0.7057, -0.1736, 0.4724, 0.1957, -0.3037, 0.4626, -0.6465, 0.4575, + 0.4230]]).astype(np.float32).reshape([1, -1]) + + whh_reverse = np.array([[0.2339, -0.0307], + [-0.5850, 0.6328], + [0.5856, -0.5601], + [0.4875, -0.6929], + [0.0314, 0.2531], + [-0.2523, 0.3244], + [0.5199, 0.5146], + [0.3968, 0.4511]]).astype(np.float32).reshape([1, -1]) + + bih_reverse = np.array([-0.1760, 0.2828, 0.2450, -0.4016, -0.4664, 0.4031, -0.1945, -0.1509]).astype( + np.float32).reshape([1, -1]) + bhh_reverse = np.array([0.6427, 0.4806, 0.6278, 0.1596, 0.0038, -0.3418, 0.0549, -0.3900]).astype( + np.float32).reshape([1, -1]) + + w_np = np.concatenate((wih, whh, wih_reverse, whh_reverse, bih, bhh, bih_reverse, bhh_reverse), axis=1).reshape( + [-1, 1, 1]) + + self.w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w') + + @ms_function + def construct(self): + return self.lstm(self.x, self.h, self.c, self.w) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_bilstm(): + seq_len = 5 + batch_size = 2 + + input_size = 10 + hidden_size = 2 + num_layers = 1 + has_bias = True + bidirectional = True + dropout = 0.0 + + num_directions = 1 + if bidirectional: + num_directions = 2 + + net = BiLstmNet(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + y, h, c, _, _ = net() + expect_y = np.array([[[-0.0826, 0.0209, 0.1715, -0.0072], + [0.1035, 0.0594, -0.0867, -0.1077]], + + [[-0.1647, 0.0293, -0.2189, 0.3809], + [0.0466, 0.4461, 0.0784, 0.0905]], + + [[-0.0182, 0.0512, 0.1758, -0.1147], + [0.0460, 0.1588, -0.0314, 0.0886]], + + [[-0.0330, 0.0551, 0.2084, -0.1154], + [-0.1641, 0.1118, -0.0122, 0.4916]], + + [[-0.2997, 0.0223, 0.1328, 0.3377], + [-0.6669, 0.0089, 0.1138, 0.7786]]]) + + error = np.ones([num_layers, batch_size, hidden_size * num_directions]) * 1.0e-4 + diff = y.asnumpy() - expect_y + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_h = np.array([[[-0.2997, 0.0223], + [-0.6669, 0.0089]], + + [[0.1715, -0.0072], + [-0.0867, -0.1077]]]) + error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-4 + diff = h.asnumpy() - expect_h + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_c = np.array([[[-0.6049, 0.0825], + [-0.9433, 0.1006]], + + [[0.3037, -0.2036], + [-0.1633, -0.5663]]]) + + error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-3 + diff = c.asnumpy() - expect_c + assert np.all(diff < error) + assert np.all(-diff < error) + + +class MultiLayerBiLstmNet(nn.Cell): + def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + super(MultiLayerBiLstmNet, self).__init__() + + num_directions = 1 + if bidirectional: + num_directions = 2 + + self.lstm = P.LSTM(input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + + input_np = np.array([[[-0.1887, -0.4144, -0.0235, 0.7489, 0.7522, 0.5969, 0.3342, 1.2198, 0.6786, -0.9404], + [-0.8643, -1.6835, -2.4965, 2.8093, 0.1741, 0.2707, 0.7387, -0.0939, -1.7990, 0.4765]], + + [[-0.5963, -1.2598, -0.7226, 1.1365, -1.7320, -0.7302, 0.1221, -0.2111, -1.6173, -0.0706], + [0.8964, 0.1737, -1.0077, -0.1389, 0.4889, 0.4391, 0.7911, 0.3614, -1.9533, -0.9936]], + + [[0.3260, -1.3312, 0.0601, 1.0726, -1.6010, -1.8733, -1.5775, 1.1579, -0.8801, -0.5742], + [-2.2998, -0.6344, -0.5409, -0.9221, -0.6500, 0.1206, 1.5215, 0.7517, 1.3691, 2.0021]], + + [[-0.1245, -0.3690, 2.1193, 1.3852, -0.1841, -0.8899, -0.3646, -0.8575, -0.3131, 0.2026], + [1.0218, -1.4331, 0.1744, 0.5442, -0.7808, 0.2527, 0.1566, 1.1484, -0.7766, -0.6747]], + + [[-0.6752, 0.9906, -0.4973, 0.3471, -0.1202, -0.4213, 2.0213, 0.0441, 0.9016, 1.0365], + [1.2223, -1.3248, 0.1207, -0.8256, 0.1816, 0.7057, -0.3105, 0.5713, 0.2804, + -1.0685]]]).astype(np.float32) + + self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x') + + self.h = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='h') + + self.c = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='c') + + wih_l0 = np.array([[0.3715, -0.0723, 0.6017, 0.5115, -0.5357, 0.3794, -0.3752, -0.6205, -0.0370, -0.2904], + [0.7055, -0.4156, -0.3650, -0.0964, 0.4141, -0.2584, -0.4765, -0.0045, 0.2943, -0.2648], + [0.1355, 0.1697, 0.1883, 0.3754, 0.3744, -0.6128, 0.2328, -0.1275, 0.6604, 0.6498], + [-0.0266, 0.5805, -0.5358, -0.0929, 0.0797, 0.3744, 0.3299, -0.3825, 0.5804, -0.0855], + [0.1141, 0.2587, -0.4370, 0.6430, -0.0017, 0.4865, 0.2814, 0.6213, -0.6415, 0.4574], + [-0.3958, -0.5827, -0.1056, 0.6987, -0.6591, -0.1326, 0.5237, 0.4667, -0.7001, -0.2326], + [0.3074, -0.3118, -0.4591, 0.2481, -0.2978, -0.1850, 0.4770, -0.0126, 0.3655, -0.4306], + [0.3033, -0.6264, -0.6551, 0.0069, -0.5238, -0.3950, 0.5681, -0.4931, -0.6258, + 0.4079]]).astype(np.float32).reshape([1, -1]) + + whh_l0 = np.array([[-0.3870, 0.0238], + [-0.3758, 0.2490], + [0.5437, -0.4117], + [0.1181, -0.2043], + [-0.5335, 0.1188], + [-0.0822, 0.2154], + [0.5844, -0.3239], + [-0.6537, 0.0278]]).astype(np.float32).reshape([1, -1]) + + bih_l0 = np.array([0.5440, 0.5995, 0.0155, -0.6254, 0.5114, 0.3364, -0.1824, -0.6262]).astype( + np.float32).reshape([1, -1]) + bhh_l0 = np.array([0.4139, -0.2513, -0.4023, 0.4222, 0.6387, -0.6147, 0.0677, 0.5355]).astype( + np.float32).reshape([1, -1]) + + wih_reverse_l0 = np.array([[6.5219e-01, 5.6162e-01, -1.8653e-01, 6.8789e-01, 1.3240e-01, 1.7699e-01, 1.2940e-01, + -1.8520e-01, -5.5439e-01, -3.4946e-01], + [3.7645e-01, 6.5475e-01, 3.5964e-01, 2.2433e-01, -1.7869e-01, -2.9047e-01, + 1.7615e-01, -5.3353e-01, -7.4204e-02, -2.5270e-01], + [5.8095e-01, -4.6426e-04, 1.9262e-01, -5.1306e-01, -3.6811e-01, 4.4858e-01, + 6.2580e-01, 9.5494e-02, -6.9505e-01, 4.9500e-01], + [-3.7810e-01, 1.5485e-01, -1.4735e-01, -1.5327e-01, -4.5702e-01, 3.0816e-01, + -3.4280e-01, 2.1604e-01, 1.4087e-01, -5.7707e-01], + [-3.8700e-01, -6.4653e-01, 6.0653e-01, -4.7297e-01, 6.8413e-02, -1.2681e-01, + 6.8464e-02, 6.7011e-01, 3.9950e-01, -2.0577e-01], + [-1.8648e-01, -6.7198e-01, 3.8017e-01, -3.3147e-01, 5.3193e-01, -5.4952e-01, + 2.1774e-01, -4.6271e-01, 3.2611e-01, 6.3554e-02], + [-4.5403e-01, -1.5910e-01, -7.5886e-02, 2.6313e-01, 6.8093e-01, -3.9960e-01, + 5.5428e-01, 1.0429e-01, 5.1322e-01, 1.9406e-01], + [3.9698e-01, -5.2101e-01, 5.1372e-01, -3.9866e-01, 1.0115e-01, -4.1290e-02, + -3.0980e-01, 2.1607e-01, 4.8420e-01, -1.9267e-01]]).astype(np.float32).reshape( + [1, -1]) + + whh_reverse_l0 = np.array([[-0.3231, -0.3960], + [-0.1625, -0.3032], + [0.3892, -0.0666], + [0.0159, -0.4870], + [-0.4953, 0.2278], + [-0.5380, -0.5250], + [0.0371, -0.4534], + [-0.5452, 0.5012]]).astype(np.float32).reshape([1, -1]) + + bih_reverse_l0 = np.array([0.0469, -0.0107, 0.3783, -0.2657, -0.0089, 0.5032, -0.0757, -0.2022]).astype( + np.float32).reshape([1, -1]) + bhh_reverse_l0 = np.array([-0.6584, 0.3977, 0.5597, -0.4784, 0.5360, -0.2532, 0.5362, -0.1063]).astype( + np.float32).reshape([1, -1]) + + wih_l1 = np.array([[0.0602, 0.6977, -0.3882, 0.3734], + [-0.6896, -0.6014, -0.2311, 0.6433], + [-0.6778, -0.5100, -0.1496, 0.5774], + [-0.5824, 0.4656, -0.2835, -0.5688], + [0.5623, 0.3599, 0.1731, 0.3124], + [0.1492, -0.6663, -0.1099, -0.5282], + [0.4696, -0.1795, -0.6712, -0.3903], + [0.4995, 0.0709, -0.1738, 0.2822]]).astype(np.float32).reshape([1, -1]) + + whh_l1 = np.array([[0.3770, 0.4139], + [0.5351, 0.6394], + [0.3901, -0.1072], + [0.1106, 0.1331], + [0.3970, 0.4693], + [0.2958, -0.3813], + [-0.3064, 0.5519], + [-0.2827, 0.5844]]).astype(np.float32).reshape([1, -1]) + + bih_l1 = np.array([0.5242, 0.5896, 0.3709, 0.6202, 0.5008, 0.2674, 0.4356, -0.3261]).astype(np.float32).reshape( + [1, -1]) + bhh_l1 = np.array([-0.6648, 0.6680, 0.2510, -0.1245, -0.0524, 0.5439, -0.1650, 0.5303]).astype( + np.float32).reshape([1, -1]) + + wih_reverse_l1 = np.array([[0.6477, 0.4416, 0.3803, -0.4708], + [0.4497, 0.2833, -0.4739, -0.6361], + [-0.5573, -0.3867, -0.0349, -0.4128], + [-0.1545, 0.3720, 0.2354, -0.6090], + [0.5965, 0.6301, -0.4591, -0.0120], + [-0.1253, -0.1881, -0.4388, 0.4335], + [0.1944, -0.1230, -0.6170, 0.1043], + [-0.6700, 0.4343, 0.6474, 0.0113]]).astype(np.float32).reshape([1, -1]) + + whh_reverse_l1 = np.array([[0.6576, 0.5573], + [0.2318, 0.0187], + [-0.6365, 0.5744], + [-0.6494, -0.1820], + [0.6461, -0.3344], + [0.0906, -0.5405], + [-0.5999, 0.5571], + [-0.0488, 0.5345]]).astype(np.float32).reshape([1, -1]) + + bih_reverse_l1 = np.array([-0.6058, -0.2812, -0.4449, -0.0802, 0.4931, 0.4066, 0.5960, 0.1968]).astype( + np.float32).reshape([1, -1]) + bhh_reverse_l1 = np.array([-0.2490, -0.3402, -0.5089, -0.3875, 0.4852, -0.0402, -0.0072, -0.1017]).astype( + np.float32).reshape([1, -1]) + + ''' + weight + layer0 + forward + wih + whh + reverse + wih + whh + layer1 + forward + wih + whh + reverse + wih + whh + ... ... + bias: + layer0 + forward + bih + bhh + reverse + bih + bhh + layer1 + forward + bih + bhh + reverse + bih + bhh + ... ... + ''' + w_np = np.concatenate( + (wih_l0, whh_l0, wih_reverse_l0, whh_reverse_l0, wih_l1, whh_l1, wih_reverse_l1, whh_reverse_l1, + bih_l0, bhh_l0, bih_reverse_l0, bhh_reverse_l0, bih_l1, bhh_l1, bih_reverse_l1, bhh_reverse_l1), + axis=1).reshape([-1, 1, 1]) + + self.w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w') + + @ms_function + def construct(self): + return self.lstm(self.x, self.h, self.c, self.w) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_multi_layer_bilstm(): + seq_len = 5 + batch_size = 2 + + input_size = 10 + hidden_size = 2 + num_layers = 2 + has_bias = True + bidirectional = True + dropout = 0.0 + + num_directions = 1 + if bidirectional: + num_directions = 2 + + net = MultiLayerBiLstmNet(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, + dropout) + y, h, c, _, _ = net() + expect_y = np.array([[[0.5186, 0.5419, 0.2710, 0.0384], + [0.6196, 0.5539, 0.3266, 0.0866]], + + [[0.5244, 0.5276, 0.3042, 0.0510], + [0.5143, 0.4937, 0.2828, 0.0387]], + + [[0.5124, 0.5079, 0.2951, 0.0548], + [0.4051, 0.4493, 0.2369, 0.0077]], + + [[0.4532, 0.4749, 0.2557, 0.0611], + [0.4879, 0.4812, 0.3160, 0.0368]], + + [[0.4535, 0.4806, 0.3880, 0.0462], + [0.4674, 0.4849, 0.3890, 0.1008]]]) + + error = np.ones([seq_len, batch_size, hidden_size * num_directions]) * 1.0e-4 + diff = y.asnumpy() - expect_y + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_h = np.array([[[0.4730, 0.1638], + [0.1406, -0.0697]], + + [[0.3887, -0.0518], + [-0.3988, -0.0071]], + + [[0.4535, 0.4806], + [0.4674, 0.4849]], + + [[0.2710, 0.0384], + [0.3266, 0.0866]]]) + error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-4 + diff = h.asnumpy() - expect_h + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_c = np.array([[[0.8713, 0.2694], + [0.2075, -0.2201]], + + [[0.5084, -0.0964], + [-0.5155, -0.2452]], + + [[1.1724, 1.0334], + [1.2003, 1.1058]], + + [[0.5179, 0.0750], + [0.5309, 0.2012]]]) + + error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-3 + diff = c.asnumpy() - expect_c + assert np.all(diff < error) + assert np.all(-diff < error) + + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + + @ms_function + def construct(self, output_grad): + weights = self.weights + grads = self.grad(self.network, weights)(output_grad) + return grads + + +class Net(nn.Cell): + def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + super(Net, self).__init__() + + num_directions = 1 + if bidirectional: + num_directions = 2 + + self.lstm = P.LSTM(input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + + input_np = np.array([[[-0.5907, 1.0557, 1.7283, 0.6706, -1.2550, -0.5298, -0.2290, -0.6735, 0.8555, 1.4836], + [-1.7070, -0.5347, -0.9105, -0.2598, 0.0588, 1.5496, 1.0757, 0.3760, -1.2020, -0.2868]], + + [[0.0151, 0.2126, 0.8090, -0.5292, -2.5590, 0.4279, -0.3081, -1.4706, -0.0498, 1.2301], + [0.4165, -0.5391, -0.0996, 0.1928, -0.4909, -0.1255, 0.4444, -1.3687, 1.3096, 0.6553]], + + [[-0.7802, -0.2083, -0.6388, 1.3757, 0.4293, 0.5363, 0.3202, -0.6687, -1.3864, -0.2953], + [1.0799, -0.7204, 0.1130, -0.5857, -0.4855, -1.1068, 1.0126, 0.8716, 1.5460, -0.7392]], + + [[2.2645, -0.6586, -0.2227, 1.4290, -0.5006, -1.6576, -0.1793, 0.5319, 0.1360, 0.2707], + [-0.4071, 0.1575, 1.4199, -0.9156, 0.1855, 0.4947, 1.0460, -0.6365, 0.1191, -0.6374]], + + [[0.2468, 1.0815, -0.4893, 0.0664, 0.6405, -2.2967, 0.7612, 0.8759, 0.5685, -1.0999], + [-0.7272, -1.7750, -0.1164, -0.7159, 0.0061, -0.7839, -1.8329, 0.3434, -0.5634, + 0.5384]]]).astype(np.float32) + + self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x') + + self.h = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='h') + + self.c = Parameter(initializer( + Tensor(np.ones((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='c') + + wih_l0 = np.array([[0.2300, 0.6668, 0.4703, 0.0425, 0.0464, 0.6825, 0.2249, -0.4315, -0.2449, 0.2964], + [-0.2811, -0.3444, 0.2557, -0.5137, -0.5518, 0.1652, -0.6720, 0.1066, 0.3586, 0.6299], + [0.5728, -0.1784, 0.5661, 0.4012, 0.3856, -0.1899, 0.3102, 0.3717, -0.5651, 0.1952], + [0.1026, -0.0527, 0.1198, -0.3080, 0.2292, 0.5757, -0.3567, -0.2731, -0.0586, -0.2849], + [0.2194, -0.1622, 0.3219, -0.3008, -0.3713, -0.3034, -0.2385, 0.0412, -0.5205, 0.0280], + [-0.5499, -0.0733, -0.5236, -0.6753, -0.7045, -0.1839, -0.1037, -0.5026, -0.4055, -0.3416], + [0.1573, -0.1301, -0.2882, -0.3464, 0.6643, 0.1980, -0.6804, 0.5359, 0.5996, 0.0124], + [-0.6436, 0.0587, -0.6520, -0.0471, 0.1667, 0.6042, 0.5752, -0.6296, -0.2976, + -0.3757]]).astype(np.float32).reshape([1, -1]) + + whh_l0 = np.array([[0.3358, 0.2790], + [-0.5355, 0.0989], + [-0.1402, 0.5120], + [0.1335, 0.1653], + [0.3533, -0.3531], + [0.4166, -0.4420], + [-0.5454, -0.1720], + [0.0041, -0.0799]]).astype(np.float32).reshape([1, -1]) + + bih_l0 = np.array([0.5518, 0.1083, 0.4829, 0.0607, -0.1770, -0.6944, 0.3059, 0.5354]).astype( + np.float32).reshape([1, -1]) + bhh_l0 = np.array([0.5025, -0.1261, -0.5405, 0.3220, -0.3441, 0.6488, -0.0284, -0.2334]).astype( + np.float32).reshape([1, -1]) + + wih_reverse_l0 = np.array( + [[-0.7048, -0.1768, 0.2288, -0.0760, -0.1319, 0.0820, -0.4132, 0.3644, 0.3919, 0.2449], + [0.0551, -0.0530, -0.5883, 0.0799, -0.5025, 0.1500, -0.4067, -0.3764, -0.3018, 0.2467], + [-0.2279, 0.3144, 0.5705, 0.4617, 0.1729, 0.6539, -0.2086, 0.5355, 0.4439, 0.0122], + [0.6967, -0.5245, 0.3527, 0.3386, 0.0429, -0.3803, -0.4328, -0.4767, 0.4481, -0.2405], + [0.6744, -0.2776, 0.0798, 0.1543, 0.6421, 0.6102, 0.3591, -0.4431, -0.6327, -0.0075], + [-0.4520, 0.4201, -0.2374, -0.1556, -0.4175, -0.6834, 0.3096, -0.1581, 0.0127, 0.6872], + [0.1788, -0.5442, -0.3675, -0.2887, -0.3004, 0.5813, 0.1618, 0.6875, -0.4678, 0.0071], + [-0.6453, -0.2528, 0.5675, -0.5154, -0.4129, -0.0214, 0.5539, 0.0343, 0.1712, 0.5644]]).astype( + np.float32).reshape([1, -1]) + + whh_reverse_l0 = np.array([[-0.6657, 0.6330], + [-0.2290, 0.6556], + [0.4808, -0.2712], + [0.0407, -0.2587], + [0.3837, 0.0382], + [0.2268, 0.1217], + [-0.6404, -0.3336], + [0.5461, -0.0764]]).astype(np.float32).reshape([1, -1]) + + bih_reverse_l0 = np.array([0.0314, 0.1009, 0.3664, -0.6732, -0.6944, 0.5098, -0.1251, 0.2644]).astype( + np.float32).reshape([1, -1]) + bhh_reverse_l0 = np.array([-0.1961, -0.3836, 0.1191, -0.7022, -0.0961, 0.5493, -0.6979, 0.0017]).astype( + np.float32).reshape([1, -1]) + + wih_l1 = np.array([[1.2746e-01, -3.3346e-01, 1.5589e-01, -4.7986e-01], + [6.5835e-01, 3.8135e-01, -3.8409e-01, -3.6499e-01], + [-6.0374e-04, -1.2227e-01, -1.5955e-01, 4.2772e-01], + [-1.8281e-01, -5.0484e-01, 7.0204e-01, 6.5872e-01], + [3.7765e-01, -4.3494e-01, 3.1503e-01, -4.2504e-02], + [6.3506e-01, -4.3049e-02, -5.7413e-01, -2.5134e-01], + [8.7181e-02, -5.5216e-01, 5.5436e-01, -3.9599e-01], + [4.4611e-01, -4.2690e-01, 6.6142e-01, 6.3882e-01]]).astype(np.float32).reshape([1, -1]) + + whh_l1 = np.array([[-0.0049, -0.3267], + [0.0863, -0.6277], + [0.4815, -0.2236], + [0.5996, -0.3441], + [0.3959, -0.0249], + [0.3986, -0.0922], + [-0.5321, 0.0877], + [0.2811, -0.0483]]).astype(np.float32).reshape([1, -1]) + + bih_l1 = np.array([0.0032, -0.0893, 0.5706, 0.3712, 0.0590, 0.0044, 0.2417, 0.1291]).astype(np.float32).reshape( + [1, -1]) + bhh_l1 = np.array([-0.0704, 0.3908, -0.1121, 0.6970, -0.6216, 0.6340, -0.2945, 0.5224]).astype( + np.float32).reshape([1, -1]) + + wih_reverse_l1 = np.array([[-0.2693, 0.3487, 0.0692, 0.0047], + [0.6187, 0.5649, 0.0680, 0.5110], + [-0.5262, -0.3307, -0.3892, 0.5382], + [-0.2925, 0.5185, -0.1385, 0.3431], + [-0.3252, 0.3809, -0.4680, 0.3379], + [0.4763, -0.5465, 0.0033, -0.5144], + [0.3826, -0.3879, -0.2439, 0.2571], + [-0.0422, -0.0359, -0.4197, -0.2209]]).astype(np.float32).reshape([1, -1]) + + whh_reverse_l1 = np.array([[-0.4691, 0.5944], + [-0.6885, 0.1708], + [0.6391, -0.3690], + [-0.5919, 0.1805], + [-0.6853, -0.6215], + [-0.4635, -0.6714], + [-0.2050, 0.0513], + [0.3411, -0.2833]]).astype(np.float32).reshape([1, -1]) + + bih_reverse_l1 = np.array([0.5764, -0.7010, -0.0831, -0.3779, -0.2743, 0.0480, -0.2707, -0.5583]).astype( + np.float32).reshape([1, -1]) + bhh_reverse_l1 = np.array([0.3379, -0.2671, -0.2789, -0.6611, -0.5542, -0.0188, 0.1831, 0.3612]).astype( + np.float32).reshape([1, -1]) + + ''' + weight + layer0 + forward + wih + whh + reverse + wih + whh + layer1 + forward + wih + whh + reverse + wih + whh + ... ... + bias: + layer0 + forward + bih + bhh + reverse + bih + bhh + layer1 + forward + bih + bhh + reverse + bih + bhh + ... ... + ''' + w_np = np.concatenate( + (wih_l0, whh_l0, wih_reverse_l0, whh_reverse_l0, wih_l1, whh_l1, wih_reverse_l1, whh_reverse_l1, + bih_l0, bhh_l0, bih_reverse_l0, bhh_reverse_l0, bih_l1, bhh_l1, bih_reverse_l1, bhh_reverse_l1), + axis=1).reshape([-1, 1, 1]) + + self.w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w') + + @ms_function + def construct(self): + return self.lstm(self.x, self.h, self.c, self.w)[0] + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_grad(): + seq_len = 5 + batch_size = 2 + + input_size = 10 + hidden_size = 2 + num_layers = 2 + has_bias = True + bidirectional = True + dropout = 0.0 + + num_directions = 1 + if bidirectional: + num_directions = 2 + + net = Grad(Net(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout)) + + dy = np.array([[[-3.5471e-01, 7.0540e-01, -7.5945e-01, -1.2322e+00], + [2.7161e-01, 1.0865e+00, -2.1827e-03, 8.8031e-01]], + + [[-4.2431e-01, 1.4955e+00, 4.6576e-01, -2.7230e+00], + [-4.0418e-01, -2.3282e-01, 9.1253e-01, -2.7379e-01]], + + [[-1.3654e+00, 1.9251e+00, -1.6808e+00, -3.2642e-02], + [-4.6481e-01, 1.3138e+00, 1.2956e-02, 1.0198e+00]], + + [[1.2914e+00, -2.3753e-01, 9.4763e-01, 1.7930e-02], + [5.3589e-01, -1.0981e-01, 1.5377e+00, 6.2709e-01]], + + [[-1.6032e+00, -1.8818e-01, 7.0441e-01, -2.8765e+00], + [1.0065e-01, 9.2045e-01, 2.7426e-01, 2.6196e-01]]]).astype(np.float32) + + dx, dh, dc, dw = net(Tensor(dy)) + expect_dx = np.array([[[0.01697153, -0.0096909, 0.01306139, 0.00863109, -0.00122794, -0.00746152, -0.00879683, + 0.00643571, 0.0015958, 0.01480642], + [0.05794962, -0.02326604, 0.01862703, 0.02053947, 0.02607713, -0.01278067, 0.04250786, + -0.02686035, -0.07441005, 0.00806021]], + + [[-0.026675, -0.01024149, -0.02492021, -0.00457492, -0.0085863, 0.02341479, 0.02188834, + -0.04139283, -0.01367766, -0.00305065], + [-0.00762213, -0.01914341, -0.03233681, -0.03580827, -0.02201782, -0.00153102, -0.00097455, + -0.02708411, -0.03711082, -0.02804472]], + + [[-0.0040581, -0.00116989, 0.01652471, 0.02182668, -0.02547193, -0.04171437, 0.04185125, + 0.01589275, -0.00517019, 0.06554792], + [-0.02294365, -0.00589715, -0.01425684, -0.01499153, -0.05327821, -0.03133425, 0.00755623, + -0.04192506, -0.02122675, -0.01214214]], + + [[-0.00041491, 0.00240709, -0.00942589, 0.00719656, 0.01438523, 0.00931082, 0.00534746, + -0.0004002, 0.01299422, 0.00181135], + [-0.01704482, -0.00887032, -0.01746774, -0.03289891, -0.04259495, -0.01928082, -0.01570587, + -0.01242383, -0.01799918, -0.00610236]], + + [[0.00207505, -0.0008109, 0.00114241, 0.00251349, -0.00065676, 0.00151333, -0.00077485, + -0.00034354, -0.00028289, -0.0006986], + [-0.00240827, -0.0001309, 0.01401818, -0.01272261, -0.02665948, -0.01095799, -0.007761, + -0.0087831, 0.01038029, 0.02021475]]]).astype(np.float32) + + error = np.ones(dx.asnumpy().shape) * 1.0e-4 + diff = dx.asnumpy() - expect_dx + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_dh = np.array([[[-0.00696833, 0.00212885], + [0.01416209, 0.0002706]], + + [[0.00297393, -0.0021012], + [0.00458834, 0.00400078]], + + [[0.08658642, -0.10590762], + [0.1516603, -0.10525411]], + + [[0.11888178, -0.04759264], + [0.05898442, -0.08082277]]]).astype(np.float32) + + error = np.ones(dh.asnumpy().shape) * 1.0e-4 + diff = dh.asnumpy() - expect_dh + assert np.all(diff < error) + assert np.all(-diff < error) + + expect_dc = np.array([[[0.00887521, -0.01391486], + [0.03858164, -0.04941981]], + + [[0.00665188, 0.00184223], + [-0.00541833, 0.01410913]], + + [[-0.2068854, 0.5585638], + [0.01735374, 0.3537254]], + + [[0.20350647, -0.2792883], + [0.18456826, 0.02278761]]]).astype(np.float32) + + error = np.ones(dc.asnumpy().shape) * 1.0e-4 + diff = dc.asnumpy() - expect_dc + assert np.all(diff < error) + assert np.all(-diff < error) + + +class LstmNetWithDropout(nn.Cell): + def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout): + super(LstmNetWithDropout, self).__init__() + + num_directions = 1 + if bidirectional: + num_directions = 2 + + self.lstm = P.LSTM(input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + + input_np = np.array([[[-2.48789445e-01, -2.18991071e-01, -8.41492534e-01, -5.73351622e-01, 8.20644796e-02, + 4.14313585e-01, -1.30143976e+00, -4.43366140e-01, -1.21003680e-01, -2.11284861e-01], + [9.94045794e-01, 3.18840504e-01, 4.81898338e-01, -4.83986028e-02, -9.26419497e-02, + -2.57977694e-01, 1.82191110e+00, 5.95121741e-01, 6.30752742e-01, -6.01903737e-01]], + + [[7.67166913e-01, 5.41202351e-02, -1.24094069e+00, 1.38814664e+00, 2.05845284e+00, + 7.29744852e-01, -1.12405574e+00, 3.78702253e-01, 2.28524983e-01, 2.02445173e+00], + [-1.85264975e-01, -4.55119252e-01, 1.23624969e+00, 1.24347043e+00, -1.68316591e+00, + -3.55918944e-01, 3.07149738e-01, -3.44966322e-01, -1.08978853e-01, 1.80912763e-01]], + + [[-6.47622466e-01, 1.31204927e+00, 6.47477210e-01, -7.93370783e-01, 3.08402872e-04, + -5.12097359e-01, -1.69133916e-01, 8.57838035e-01, -3.63963723e-01, 6.35978997e-01], + [-3.92911851e-01, 8.27334300e-02, -1.11347124e-01, 8.79961967e-01, 6.02812059e-02, + -3.76448452e-01, -1.48800862e+00, -9.48699772e-01, -1.24202335e+00, 1.65264118e+00]], + + [[4.05404866e-01, 5.67396320e-02, -2.05705926e-01, -8.70196745e-02, -7.34854519e-01, + -1.07580565e-01, 1.33716142e+00, -1.18140256e+00, 2.66074872e+00, -3.26788813e-01], + [6.97183967e-01, -2.32625628e+00, 1.20393467e+00, -2.32532692e+00, 2.03347206e+00, + -7.58083522e-01, 1.35564697e+00, -2.32149422e-01, 9.85125721e-01, 1.00944638e+00]], + + [[9.89606023e-01, -5.30669808e-01, -2.66087383e-01, 8.14819038e-01, 1.07067376e-01, + -1.76214290e+00, -5.04977465e-01, 1.94490123e+00, 5.10450959e-01, -2.29238123e-01], + [-1.32928836e+00, -1.18175328e-01, -5.17818272e-01, -1.45089477e-01, 7.13987231e-01, + -7.41293788e-01, -3.67817104e-01, 1.18039274e+00, -6.03745162e-01, + -5.83392143e-01]]]).astype(np.float32) + + self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x') + + self.h = Parameter(initializer( + Tensor(np.array([[[-0.47240502, 1.6824378], + [-0.00978304, 0.8179632]]]).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='h') + + self.c = Parameter(initializer( + Tensor(np.array([[[-0.85975164, -0.3198615], + [-0.9821871, 0.26311848]]]).astype(np.float32)), + [num_layers * num_directions, batch_size, hidden_size]), name='c') + + wih = np.array([[0.4473, -0.5509, -0.1585, -0.6215, 0.6228, 0.3462, 0.3015, -0.3714, 0.3119, -0.1151], + [-0.6923, 0.1373, 0.2214, 0.2280, 0.6960, -0.6368, 0.5725, -0.1359, 0.0742, -0.6777], + [-0.4432, 0.6162, -0.1066, -0.6138, -0.2529, -0.5638, -0.0603, 0.3039, 0.1068, -0.5300], + [0.4337, -0.1215, -0.5088, -0.0045, 0.2828, 0.1411, 0.0741, 0.6936, -0.4603, 0.6986], + [-0.2079, -0.5518, 0.5375, -0.2168, 0.3662, 0.0948, -0.0564, -0.1808, -0.6672, -0.2410], + [0.5142, 0.0790, -0.1123, -0.2351, 0.3982, -0.6351, 0.5906, 0.3917, -0.0850, -0.5397], + [-0.4795, -0.6576, 0.5693, 0.0047, -0.6626, 0.1013, -0.4015, -0.4040, -0.2817, 0.4430], + [0.0251, -0.3035, -0.6026, 0.2693, -0.2749, 0.1501, -0.5778, 0.5570, -0.7065, -0.6196]]).astype( + np.float32).reshape([1, -1]) + + whh = np.array([[-0.4344, -0.2529], + [0.0377, 0.7046], + [-0.0579, -0.5240], + [-0.4801, -0.1149], + [-0.4010, -0.5614], + [0.4721, 0.4366], + [-0.4282, 0.0816], + [0.1574, -0.3359]]).astype(np.float32).reshape([1, -1]) + + bih = np.array([0.2431, 0.5967, -0.2417, -0.4169, -0.5326, 0.5685, -0.2971, -0.4326]).astype( + np.float32).reshape([1, -1]) + bhh = np.array([-0.1751, -0.2270, -0.3980, -0.4983, -0.3527, -0.2774, 0.6371, -0.3330]).astype( + np.float32).reshape([1, -1]) + + w_np = np.concatenate((wih, whh, bih, bhh), axis=1).reshape([-1, 1, 1]) + + self.w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w') + + def construct(self): + return self.lstm(self.x, self.h, self.c, self.w) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_lstm_dropout(): + seq_len = 5 + batch_size = 2 + + input_size = 10 + hidden_size = 2 + num_layers = 1 + has_bias = True + bidirectional = False + dropout = 1.0 + + num_directions = 1 + if bidirectional: + num_directions = 2 + + net = LstmNetWithDropout(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout) + y, h, c, _, _ = net() + expect_y = np.array([[[-0.45210335, -0.0844336], + [-0.14677924, 0.07140275]], + + [[-0.18895914, -0.11084185], + [-0.26356253, -0.06367199]], + + [[-0.33480304, 0.00812318], + [-0.0887147, -0.1564593]], + + [[-0.33231455, 0.00743252], + [0.428218, 0.00723737]], + + [[-0.20026046, 0.43491203], + [0.17739448, 0.5313992]]]) + + error = np.ones([num_layers, batch_size, hidden_size]) * 1.0e-4 + diff = y.asnumpy() - expect_y + assert np.all(diff < error) + assert np.all(-diff < error) diff --git a/tests/st/ops/gpu/test_maxpool_gpu_op.py b/tests/st/ops/gpu/test_maxpool_gpu_op.py new file mode 100644 index 00000000000..2fc0afff5d8 --- /dev/null +++ b/tests/st/ops/gpu/test_maxpool_gpu_op.py @@ -0,0 +1,78 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class Net_Pool(nn.Cell): + def __init__(self): + super(Net_Pool, self).__init__() + self.maxpool_fun = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="VALID") + + def construct(self, x): + return self.maxpool_fun(x) + + +class Net_Pool2(nn.Cell): + def __init__(self): + super(Net_Pool2, self).__init__() + self.maxpool_fun = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="SAME") + + def construct(self, x): + return self.maxpool_fun(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_maxpool2d(): + x = Tensor(np.array([[[ + [0, 1, 2, 3, -4, -5], + [6, 7, 8, 9, -10, -11], + [12, 13, 14, -15, -16, -17], + [18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29], + [30, 31, 32, 33, 34, 35] + ]]]).astype(np.float32)) + expect_result = (np.array([[[ + [7, 9, -4], + [19, 21, 23], + [31, 33, 35] + ]]])) + expect_result2 = (np.array([[[ + [14, 14, -4], + [26, 28, 29], + [32, 34, 35] + ]]])) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + maxpool2d = Net_Pool() + maxpool2d2 = Net_Pool2() + output2 = maxpool2d2(x) + output = maxpool2d(x) + assert (output.asnumpy() == expect_result).all() + assert (output2.asnumpy() == expect_result2).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + maxpool2d = Net_Pool() + maxpool2d2 = Net_Pool2() + output2 = maxpool2d2(x) + output = maxpool2d(x) + assert (output.asnumpy() == expect_result).all() + assert (output2.asnumpy() == expect_result2).all() diff --git a/tests/st/ops/gpu/test_maxpool_grad_gpu_op.py b/tests/st/ops/gpu/test_maxpool_grad_gpu_op.py new file mode 100644 index 00000000000..40bc965e551 --- /dev/null +++ b/tests/st/ops/gpu/test_maxpool_grad_gpu_op.py @@ -0,0 +1,75 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class Net_Pool_Grad(nn.Cell): + def __init__(self): + super(Net_Pool_Grad, self).__init__() + self.maxpool_grad_fun = G.MaxPoolGrad(padding="VALID", + ksize=2, + strides=2) + + def construct(self, x, a, d): + return self.maxpool_grad_fun(x, a, d) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_maxpool2d_grad(): + x = Tensor(np.array([[[ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + [12, 13, 14, 15, 16, 17], + [18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29], + [30, 31, 32, 33, 34, 35] + ]]]).astype(np.float32)) + a = Tensor(np.array([[[ + [3, 3, 3], + [3, 3, 3], + [3, 3, 3] + ]]]).astype(np.float32)) + d = Tensor(np.array([[[ + [7, 9, 11], + [19, 21, 23], + [31, 33, 35] + ]]]).astype(np.float32)) + expect_result = (np.array([[[ + [0, 0, 0, 0, 0, 0], + [0, 7, 0, 9, 0, 11], + [0, 0, 0, 0, 0, 0], + [0, 19, 0, 21, 0, 23], + [0, 0, 0, 0, 0, 0], + [0, 31, 0, 33, 0, 35] + ]]])) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + maxpool2d_grad = Net_Pool_Grad() + output = maxpool2d_grad(x, a, d) + assert (output.asnumpy() == expect_result).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + maxpool2d_grad = Net_Pool_Grad() + output = maxpool2d_grad(x, a, d) + assert (output.asnumpy() == expect_result).all() diff --git a/tests/st/ops/gpu/test_momentum_op.py b/tests/st/ops/gpu/test_momentum_op.py new file mode 100644 index 00000000000..d2b992003b7 --- /dev/null +++ b/tests/st/ops/gpu/test_momentum_op.py @@ -0,0 +1,69 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +import mindspore.nn as nn +from mindspore.nn.optim import Momentum +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Dense +from mindspore import Tensor +from mindspore.common.initializer import initializer +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") +class NetMomentum(nn.Cell): + def __init__(self): + super(NetMomentum, self).__init__() + self.batch_size = 1 + self.reshape = P.Reshape() + weight = Tensor(np.ones([10, 16]).astype(np.float32) * 0.01) + self.fc1 = Dense(16, 10, weight_init=weight) + + def construct(self, input_x): + output = self.reshape(input_x, (self.batch_size, -1)) + output = self.fc1(output) + return output + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_momentum(): + epoch = 3 + net = NetMomentum() + learning_rate = initializer(Tensor(np.array([0.01]).astype(np.float32)), [1]) + momentum = initializer(Tensor(np.array([0.9]).astype(np.float32)), [1]) + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + net_with_criterion = WithLossCell(net, criterion) + train_network = TrainOneStepCell(net_with_criterion, optimizer) # optimizer + train_network.set_train() + losses = [] + for i in range(epoch): + data = Tensor(np.arange(0, 16).reshape(1, 1, 4, 4).astype(np.float32) * 0.01) + label = Tensor(np.array([0]).astype(np.int32)) + loss = train_network(data, label) + losses.append(loss) + + """ + expect output: + [[0.04132498 0.00874167 0.00874167 0.00874167 0.00874167 + 0.00874167 0.00874167 0.00874167 0.00874167 0.00874167]] + """ + error = np.ones(shape=[1, 10]) * 1.0e-6 + + return losses diff --git a/tests/st/ops/gpu/test_mul_op.py b/tests/st/ops/gpu/test_mul_op.py new file mode 100644 index 00000000000..d0ebd7e5000 --- /dev/null +++ b/tests/st/ops/gpu/test_mul_op.py @@ -0,0 +1,131 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetMul(nn.Cell): + def __init__(self): + super(NetMul, self).__init__() + self.mul = P.Mul() + + def construct(self, x, y): + return self.mul(x, y) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_mul(): + x0_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + y0_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + y1_np = np.random.uniform(-2, 2, (2, 1, 4, 4)).astype(np.float32) + x2_np = np.random.uniform(-2, 2, (2, 1, 1, 4)).astype(np.float32) + y2_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + x3_np = np.random.uniform(-2, 2, 1).astype(np.float32) + y3_np = np.random.uniform(-2, 2, 1).astype(np.float32) + x4_np = np.array(768).astype(np.float32) + y4_np = np.array(3072.5).astype(np.float32) + + x0 = Tensor(x0_np) + y0 = Tensor(y0_np) + x1 = Tensor(x1_np) + y1 = Tensor(y1_np) + x2 = Tensor(x2_np) + y2 = Tensor(y2_np) + x3 = Tensor(x3_np) + y3 = Tensor(y3_np) + x4 = Tensor(x4_np) + y4 = Tensor(y4_np) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + mul = NetMul() + output0 = mul(x0, y0) + expect0 = np.multiply(x0_np, y0_np) + diff0 = output0.asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + + output1 = mul(x1, y1) + expect1 = np.multiply(x1_np, y1_np) + diff1 = output1.asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + output2 = mul(x2, y2) + expect2 = np.multiply(x2_np, y2_np) + diff2 = output2.asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output2.shape() == expect2.shape) + + output3 = mul(x3, y3) + expect3 = np.multiply(x3_np, y3_np) + diff3 = output3.asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output3.shape() == expect3.shape) + + output4 = mul(x4, y4) + expect4 = np.multiply(x4_np, y4_np) + diff4 = output4.asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output4.shape() == expect4.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + mul = NetMul() + output0 = mul(x0, y0) + expect0 = np.multiply(x0_np, y0_np) + diff0 = output0.asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + + output1 = mul(x1, y1) + expect1 = np.multiply(x1_np, y1_np) + diff1 = output1.asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + output2 = mul(x2, y2) + expect2 = np.multiply(x2_np, y2_np) + diff2 = output2.asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output2.shape() == expect2.shape) + + output3 = mul(x3, y3) + expect3 = np.multiply(x3_np, y3_np) + diff3 = output3.asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output3.shape() == expect3.shape) + + output4 = mul(x4, y4) + expect4 = np.multiply(x4_np, y4_np) + diff4 = output4.asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output4.shape() == expect4.shape) diff --git a/tests/st/ops/gpu/test_neg_op.py b/tests/st/ops/gpu/test_neg_op.py new file mode 100644 index 00000000000..4602d0985c2 --- /dev/null +++ b/tests/st/ops/gpu/test_neg_op.py @@ -0,0 +1,66 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetNeg(nn.Cell): + def __init__(self): + super(NetNeg, self).__init__() + self.neg = P.Neg() + + def construct(self, x): + return self.neg(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_neg(): + x0_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.uniform(-2, 2, 1).astype(np.float32) + x0 = Tensor(x0_np) + x1 = Tensor(x1_np) + expect0 = np.negative(x0_np) + expect1 = np.negative(x1_np) + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + neg = NetNeg() + output0 = neg(x0) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + output1 = neg(x1) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + neg = NetNeg() + output0 = neg(x0) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + output1 = neg(x1) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) diff --git a/tests/st/ops/gpu/test_one_hot_op.py b/tests/st/ops/gpu/test_one_hot_op.py new file mode 100644 index 00000000000..cf670154a7f --- /dev/null +++ b/tests/st/ops/gpu/test_one_hot_op.py @@ -0,0 +1,82 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target='GPU') + + +class NetOneHot(nn.Cell): + def __init__(self): + super(NetOneHot, self).__init__() + self.on_value = 2.0 + self.off_value = 3.0 + + self.depth_1 = 6 + self.one_hot_1 = nn.OneHot(-1, self.depth_1, self.on_value, self.off_value) + + self.depth_2 = 4 + self.one_hot_2 = nn.OneHot(0, self.depth_1, self.on_value, self.off_value) + self.one_hot_3 = nn.OneHot(0, self.depth_2, self.on_value, self.off_value) + self.one_hot_4 = nn.OneHot(1, self.depth_1, self.on_value, self.off_value) + + @ms_function + def construct(self, indices1, indices2, indices3, indices4): + return (self.one_hot_1(indices1), self.one_hot_2(indices2), + self.one_hot_3(indices3), self.one_hot_4(indices4)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_one_hot(): + one_hot = NetOneHot() + indices1 = Tensor(np.array([[0, 1], [4, 5], [2, 6]]).astype(np.int32)) + indices2 = Tensor(np.array([1, 2, 3]).astype(np.int32)) + indices3 = Tensor(np.array([[0, 1], [1, 0]]).astype(np.int32)) + indices4 = Tensor(np.array([[0, 1], [4, 5], [2, 6]]).astype(np.int32)) + output = one_hot(indices1, indices2, indices3, indices4) + expect_0 = np.array([ + [[2., 3., 3., 3., 3., 3.], [3., 2., 3., 3., 3., 3.]], + [[3., 3., 3., 3., 2., 3.], [3., 3., 3., 3., 3., 2.]], + [[3., 3., 2., 3., 3., 3.], [3., 3., 3., 3., 3., 3.]] + ]).astype(np.float32) + expect_1 = np.array([ + [3., 3., 3.], + [2., 3., 3.], + [3., 2., 3.], + [3., 3., 2.], + [3., 3., 3.], + [3., 3., 3.] + ]).astype(np.float32) + expect_2 = np.array([ + [[2., 3.], [3., 2.]], [[3., 2.], [2., 3.]], [[3., 3.], [3., 3.]], + [[3., 3.], [3., 3.]] + ]).astype(np.float32) + expect_3 = np.array([ + [[2., 3.], [3., 2.], [3., 3.], [3., 3.], [3., 3.], [3., 3.]], + [[3., 3.], [3., 3.], [3., 3.], [3., 3.], [2., 3.], [3., 2.]], + [[3., 3.], [3., 3.], [2., 3.], [3., 3.], [3., 3.], [3., 3.]] + ]).astype(np.float32) + assert (output[0].asnumpy() == expect_0).all() + assert (output[1].asnumpy() == expect_1).all() + assert (output[2].asnumpy() == expect_2).all() + assert (output[3].asnumpy() == expect_3).all() diff --git a/tests/st/ops/gpu/test_realdiv_op.py b/tests/st/ops/gpu/test_realdiv_op.py new file mode 100644 index 00000000000..e5dcc051905 --- /dev/null +++ b/tests/st/ops/gpu/test_realdiv_op.py @@ -0,0 +1,133 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class NetRealDiv(nn.Cell): + def __init__(self): + super(NetRealDiv, self).__init__() + self.divide = P.RealDiv() + + def construct(self, x, y): + return self.divide(x, y) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_real_div(): + x0_np = np.random.randint(1, 5, (2, 3, 4, 4)).astype(np.float32) + y0_np = np.random.randint(1, 5, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.randint(1, 5, (2, 3, 4, 4)).astype(np.float32) + y1_np = np.random.randint(1, 5, (2, 1, 4, 4)).astype(np.float32) + x2_np = np.random.randint(1, 5, (2, 1, 1, 4)).astype(np.float32) + y2_np = np.random.randint(1, 5, (2, 3, 4, 4)).astype(np.float32) + x3_np = np.random.randint(1, 5, 1).astype(np.float32) + y3_np = np.random.randint(1, 5, 1).astype(np.float32) + x4_np = np.array(768).astype(np.float32) + y4_np = np.array(3072.5).astype(np.float32) + + x0 = Tensor(x0_np) + y0 = Tensor(y0_np) + x1 = Tensor(x1_np) + y1 = Tensor(y1_np) + x2 = Tensor(x2_np) + y2 = Tensor(y2_np) + x3 = Tensor(x3_np) + y3 = Tensor(y3_np) + x4 = Tensor(x4_np) + y4 = Tensor(y4_np) + + context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + real_div = NetRealDiv() + output0 = real_div(x0, y0) + expect0 = np.divide(x0_np, y0_np) + diff0 = output0.asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + + output1 = real_div(x1, y1) + expect1 = np.divide(x1_np, y1_np) + diff1 = output1.asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + output2 = real_div(x2, y2) + expect2 = np.divide(x2_np, y2_np) + diff2 = output2.asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output2.shape() == expect2.shape) + + output3 = real_div(x3, y3) + expect3 = np.divide(x3_np, y3_np) + diff3 = output3.asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output3.shape() == expect3.shape) + + output4 = real_div(x4, y4) + expect4 = np.divide(x4_np, y4_np) + diff4 = output4.asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output4.shape() == expect4.shape) + + context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU') + real_div = NetRealDiv() + output0 = real_div(x0, y0) + expect0 = np.divide(x0_np, y0_np) + diff0 = output0.asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + + output1 = real_div(x1, y1) + expect1 = np.divide(x1_np, y1_np) + diff1 = output1.asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + output2 = real_div(x2, y2) + expect2 = np.divide(x2_np, y2_np) + diff2 = output2.asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output2.shape() == expect2.shape) + + output3 = real_div(x3, y3) + expect3 = np.divide(x3_np, y3_np) + diff3 = output3.asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output3.shape() == expect3.shape) + + output4 = real_div(x4, y4) + expect4 = np.divide(x4_np, y4_np) + diff4 = output4.asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output4.shape() == expect4.shape) diff --git a/tests/st/ops/gpu/test_reciprocal_op.py b/tests/st/ops/gpu/test_reciprocal_op.py new file mode 100644 index 00000000000..fb422a94cfe --- /dev/null +++ b/tests/st/ops/gpu/test_reciprocal_op.py @@ -0,0 +1,65 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetReciprocal(nn.Cell): + def __init__(self): + super(NetReciprocal, self).__init__() + self.reciprocal = P.Reciprocal() + + def construct(self, x): + return self.reciprocal(x) + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_Reciprocal(): + x0_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.uniform(-2, 2, 1).astype(np.float32) + x0 = Tensor(x0_np) + x1 = Tensor(x1_np) + expect0 = np.reciprocal(x0_np) + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + expect1 = np.reciprocal(x1_np) + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + reciprocal = NetReciprocal() + output0 = reciprocal(x0) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + output1 = reciprocal(x1) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + reciprocal = NetReciprocal() + output0 = reciprocal(x0) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + output1 = reciprocal(x1) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) diff --git a/tests/st/ops/gpu/test_reduce_max_op.py b/tests/st/ops/gpu/test_reduce_max_op.py new file mode 100644 index 00000000000..1f13d9dd495 --- /dev/null +++ b/tests/st/ops/gpu/test_reduce_max_op.py @@ -0,0 +1,161 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +x0 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis0 = 3 +keep_dims0 = True + +x1 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis1 = 3 +keep_dims1 = False + +x2 = np.random.rand(2, 3, 1, 4).astype(np.float32) +axis2 = 2 +keep_dims2 = True + +x3 = np.random.rand(2, 3, 1, 4).astype(np.float32) +axis3 = 2 +keep_dims3 = False + +x4 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis4 = () +np_axis4 = None +keep_dims4 = True + +x5 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis5 = () +np_axis5 = None +keep_dims5 = False + +x6 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis6 = -2 +keep_dims6 = False + +x7 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis7 = (-2, -1) +keep_dims7 = True + +context.set_context(device_target='GPU') + + +class ReduceMax(nn.Cell): + def __init__(self): + super(ReduceMax, self).__init__() + + self.x0 = Tensor(x0) + self.axis0 = axis0 + self.keep_dims0 = keep_dims0 + + self.x1 = Tensor(x1) + self.axis1 = axis1 + self.keep_dims1 = keep_dims1 + + self.x2 = Tensor(x2) + self.axis2 = axis2 + self.keep_dims2 = keep_dims2 + + self.x3 = Tensor(x3) + self.axis3 = axis3 + self.keep_dims3 = keep_dims3 + + self.x4 = Tensor(x4) + self.axis4 = axis4 + self.keep_dims4 = keep_dims4 + + self.x5 = Tensor(x5) + self.axis5 = axis5 + self.keep_dims5 = keep_dims5 + + self.x6 = Tensor(x6) + self.axis6 = axis6 + self.keep_dims6 = keep_dims6 + + self.x7 = Tensor(x7) + self.axis7 = axis7 + self.keep_dims7 = keep_dims7 + + @ms_function + def construct(self): + return (P.ReduceMax(self.keep_dims0)(self.x0, self.axis0), + P.ReduceMax(self.keep_dims1)(self.x1, self.axis1), + P.ReduceMax(self.keep_dims2)(self.x2, self.axis2), + P.ReduceMax(self.keep_dims3)(self.x3, self.axis3), + P.ReduceMax(self.keep_dims4)(self.x4, self.axis4), + P.ReduceMax(self.keep_dims5)(self.x5, self.axis5), + P.ReduceMax(self.keep_dims6)(self.x6, self.axis6), + P.ReduceMax(self.keep_dims7)(self.x7, self.axis7)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_ReduceMax(): + reduce_max = ReduceMax() + output = reduce_max() + + expect0 = np.max(x0, axis=axis0, keepdims=keep_dims0) + diff0 = output[0].asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output[0].shape() == expect0.shape) + + expect1 = np.max(x1, axis=axis1, keepdims=keep_dims1) + diff1 = output[1].asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output[1].shape() == expect1.shape) + + expect2 = np.max(x2, axis=axis2, keepdims=keep_dims2) + diff2 = output[2].asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output[2].shape() == expect2.shape) + + expect3 = np.max(x3, axis=axis3, keepdims=keep_dims3) + diff3 = output[3].asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output[3].shape() == expect3.shape) + + expect4 = np.max(x4, axis=np_axis4, keepdims=keep_dims4) + diff4 = output[4].asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output[4].shape() == expect4.shape) + + expect5 = np.max(x5, axis=np_axis5, keepdims=keep_dims5) + diff5 = output[5].asnumpy() - expect5 + error5 = np.ones(shape=expect5.shape) * 1.0e-5 + assert np.all(diff5 < error5) + assert (output[5].shape() == expect5.shape) + + expect6 = np.max(x6, axis=axis6, keepdims=keep_dims6) + diff6 = output[6].asnumpy() - expect6 + error6 = np.ones(shape=expect6.shape) * 1.0e-5 + assert np.all(diff6 < error6) + assert (output[6].shape() == expect6.shape) + + expect7 = np.max(x7, axis=axis7, keepdims=keep_dims7) + diff7 = output[7].asnumpy() - expect7 + error7 = np.ones(shape=expect7.shape) * 1.0e-5 + assert np.all(diff7 < error7) diff --git a/tests/st/ops/gpu/test_reduce_mean_op.py b/tests/st/ops/gpu/test_reduce_mean_op.py new file mode 100644 index 00000000000..a877f17c6d6 --- /dev/null +++ b/tests/st/ops/gpu/test_reduce_mean_op.py @@ -0,0 +1,250 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +x0 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis0 = 3 +keep_dims0 = True + +x1 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis1 = 3 +keep_dims1 = False + +x2 = np.random.rand(2, 3, 1, 4).astype(np.float32) +axis2 = 2 +keep_dims2 = True + +x3 = np.random.rand(2, 3, 1, 4).astype(np.float32) +axis3 = 2 +keep_dims3 = False + +x4 = np.random.rand(2, 3, 4, 1).astype(np.float32) +axis4 = 3 +keep_dims4 = True + +x5 = np.random.rand(2, 3, 4, 1).astype(np.float32) +axis5 = 3 +keep_dims5 = False + +x6 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis6 = (1, 2) +keep_dims6 = False + +x7 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis7 = (1, 2) +keep_dims7 = True + +x8 = np.random.rand(2, 1, 1, 4).astype(np.float32) +axis8 = (1, 2) +keep_dims8 = True + +x9 = np.random.rand(2, 1, 1, 4).astype(np.float32) +axis9 = (1, 2) +keep_dims9 = False + +x10 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis10 = (0, 1, 2, 3) +keep_dims10 = False + +x11 = np.random.rand(1, 1, 1, 1).astype(np.float32) +axis11 = (0, 1, 2, 3) +keep_dims11 = False + +x12 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis12 = -2 +keep_dims12 = False + +x13 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis13 = (-2, -1) +keep_dims13 = True + +context.set_context(device_target='GPU') + + +class ReduceMean(nn.Cell): + def __init__(self): + super(ReduceMean, self).__init__() + + self.x0 = Tensor(x0) + self.axis0 = axis0 + self.keep_dims0 = keep_dims0 + + self.x1 = Tensor(x1) + self.axis1 = axis1 + self.keep_dims1 = keep_dims1 + + self.x2 = Tensor(x2) + self.axis2 = axis2 + self.keep_dims2 = keep_dims2 + + self.x3 = Tensor(x3) + self.axis3 = axis3 + self.keep_dims3 = keep_dims3 + + self.x4 = Tensor(x4) + self.axis4 = axis4 + self.keep_dims4 = keep_dims4 + + self.x5 = Tensor(x5) + self.axis5 = axis5 + self.keep_dims5 = keep_dims5 + + self.x6 = Tensor(x6) + self.axis6 = axis6 + self.keep_dims6 = keep_dims6 + + self.x7 = Tensor(x7) + self.axis7 = axis7 + self.keep_dims7 = keep_dims7 + + self.x8 = Tensor(x8) + self.axis8 = axis8 + self.keep_dims8 = keep_dims8 + + self.x9 = Tensor(x9) + self.axis9 = axis9 + self.keep_dims9 = keep_dims9 + + self.x10 = Tensor(x10) + self.axis10 = axis10 + self.keep_dims10 = keep_dims10 + + self.x11 = Tensor(x11) + self.axis11 = axis11 + self.keep_dims11 = keep_dims11 + + self.x12 = Tensor(x12) + self.axis12 = axis12 + self.keep_dims12 = keep_dims12 + + self.x13 = Tensor(x13) + self.axis13 = axis13 + self.keep_dims13 = keep_dims13 + + @ms_function + def construct(self): + return (P.ReduceMean(self.keep_dims0)(self.x0, self.axis0), + P.ReduceMean(self.keep_dims1)(self.x1, self.axis1), + P.ReduceMean(self.keep_dims2)(self.x2, self.axis2), + P.ReduceMean(self.keep_dims3)(self.x3, self.axis3), + P.ReduceMean(self.keep_dims4)(self.x4, self.axis4), + P.ReduceMean(self.keep_dims5)(self.x5, self.axis5), + P.ReduceMean(self.keep_dims6)(self.x6, self.axis6), + P.ReduceMean(self.keep_dims7)(self.x7, self.axis7), + P.ReduceMean(self.keep_dims8)(self.x8, self.axis8), + P.ReduceMean(self.keep_dims9)(self.x9, self.axis9), + P.ReduceMean(self.keep_dims10)(self.x10, self.axis10), + P.ReduceMean(self.keep_dims11)(self.x11, self.axis11), + P.ReduceMean(self.keep_dims12)(self.x12, self.axis12), + P.ReduceMean(self.keep_dims13)(self.x13, self.axis13)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_ReduceMean(): + reduce_mean = ReduceMean() + output = reduce_mean() + + expect0 = np.mean(x0, axis=axis0, keepdims=keep_dims0) + diff0 = output[0].asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output[0].shape() == expect0.shape) + + expect1 = np.mean(x1, axis=axis1, keepdims=keep_dims1) + diff1 = output[1].asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output[1].shape() == expect1.shape) + + expect2 = np.mean(x2, axis=axis2, keepdims=keep_dims2) + diff2 = output[2].asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output[2].shape() == expect2.shape) + + expect3 = np.mean(x3, axis=axis3, keepdims=keep_dims3) + diff3 = output[3].asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output[3].shape() == expect3.shape) + + expect4 = np.mean(x4, axis=axis4, keepdims=keep_dims4) + diff4 = output[4].asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output[4].shape() == expect4.shape) + + expect5 = np.mean(x5, axis=axis5, keepdims=keep_dims5) + diff5 = output[5].asnumpy() - expect5 + error5 = np.ones(shape=expect5.shape) * 1.0e-5 + assert np.all(diff5 < error5) + assert (output[5].shape() == expect5.shape) + + expect6 = np.mean(x6, axis=axis6, keepdims=keep_dims6) + diff6 = output[6].asnumpy() - expect6 + error6 = np.ones(shape=expect6.shape) * 1.0e-5 + assert np.all(diff6 < error6) + assert (output[6].shape() == expect6.shape) + + expect7 = np.mean(x7, axis=axis7, keepdims=keep_dims7) + diff7 = output[7].asnumpy() - expect7 + error7 = np.ones(shape=expect7.shape) * 1.0e-5 + assert np.all(diff7 < error7) + assert (output[7].shape() == expect7.shape) + + expect8 = np.mean(x8, axis=axis8, keepdims=keep_dims8) + diff8 = output[8].asnumpy() - expect8 + error8 = np.ones(shape=expect8.shape) * 1.0e-5 + assert np.all(diff8 < error8) + assert (output[8].shape() == expect8.shape) + + expect9 = np.mean(x9, axis=axis9, keepdims=keep_dims9) + diff9 = output[9].asnumpy() - expect9 + error9 = np.ones(shape=expect9.shape) * 1.0e-5 + assert np.all(diff9 < error9) + assert (output[9].shape() == expect9.shape) + + expect10 = np.mean(x10, axis=axis10, keepdims=keep_dims10) + diff10 = output[10].asnumpy() - expect10 + error10 = np.ones(shape=expect10.shape) * 1.0e-5 + assert np.all(diff10 < error10) + assert (output[10].shape() == expect10.shape) + + expect11 = np.mean(x11, axis=axis11, keepdims=keep_dims11) + diff11 = output[11].asnumpy() - expect11 + error11 = np.ones(shape=expect11.shape) * 1.0e-5 + assert np.all(diff11 < error11) + assert (output[11].shape() == expect11.shape) + + expect12 = np.sum(x12, axis=axis12, keepdims=keep_dims12) + diff12 = output[12].asnumpy() - expect12 + error12 = np.ones(shape=expect12.shape) * 1.0e-5 + assert np.all(diff12 < error12) + assert (output[12].shape() == expect12.shape) + + expect13 = np.sum(x13, axis=axis13, keepdims=keep_dims13) + diff13 = output[13].asnumpy() - expect13 + error13 = np.ones(shape=expect13.shape) * 1.0e-5 + assert np.all(diff13 < error13) + assert (output[13].shape() == expect13.shape) diff --git a/tests/st/ops/gpu/test_reduce_sum_op.py b/tests/st/ops/gpu/test_reduce_sum_op.py new file mode 100644 index 00000000000..a8319d43465 --- /dev/null +++ b/tests/st/ops/gpu/test_reduce_sum_op.py @@ -0,0 +1,252 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +x0 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis0 = 3 +keep_dims0 = True + +x1 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis1 = 3 +keep_dims1 = False + +x2 = np.random.rand(2, 3, 1, 4).astype(np.float32) +axis2 = 2 +keep_dims2 = True + +x3 = np.random.rand(2, 3, 1, 4).astype(np.float32) +axis3 = 2 +keep_dims3 = False + +x4 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis4 = () +np_axis4 = None +keep_dims4 = True + +x5 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis5 = () +np_axis5 = None +keep_dims5 = False + +x6 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis6 = (1, 2) +keep_dims6 = False + +x7 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis7 = (1, 2) +keep_dims7 = True + +x8 = np.random.rand(2, 1, 1, 4).astype(np.float32) +axis8 = (1, 2) +keep_dims8 = True + +x9 = np.random.rand(2, 1, 1, 4).astype(np.float32) +axis9 = (1, 2) +keep_dims9 = False + +x10 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis10 = (0, 1, 2, 3) +keep_dims10 = False + +x11 = np.random.rand(1, 1, 1, 1).astype(np.float32) +axis11 = (0, 1, 2, 3) +keep_dims11 = False + +x12 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis12 = -2 +keep_dims12 = False + +x13 = np.random.rand(2, 3, 4, 4).astype(np.float32) +axis13 = (-2, -1) +keep_dims13 = True + +context.set_context(device_target='GPU') + + +class ReduceSum(nn.Cell): + def __init__(self): + super(ReduceSum, self).__init__() + + self.x0 = Tensor(x0) + self.axis0 = axis0 + self.keep_dims0 = keep_dims0 + + self.x1 = Tensor(x1) + self.axis1 = axis1 + self.keep_dims1 = keep_dims1 + + self.x2 = Tensor(x2) + self.axis2 = axis2 + self.keep_dims2 = keep_dims2 + + self.x3 = Tensor(x3) + self.axis3 = axis3 + self.keep_dims3 = keep_dims3 + + self.x4 = Tensor(x4) + self.axis4 = axis4 + self.keep_dims4 = keep_dims4 + + self.x5 = Tensor(x5) + self.axis5 = axis5 + self.keep_dims5 = keep_dims5 + + self.x6 = Tensor(x6) + self.axis6 = axis6 + self.keep_dims6 = keep_dims6 + + self.x7 = Tensor(x7) + self.axis7 = axis7 + self.keep_dims7 = keep_dims7 + + self.x8 = Tensor(x8) + self.axis8 = axis8 + self.keep_dims8 = keep_dims8 + + self.x9 = Tensor(x9) + self.axis9 = axis9 + self.keep_dims9 = keep_dims9 + + self.x10 = Tensor(x10) + self.axis10 = axis10 + self.keep_dims10 = keep_dims10 + + self.x11 = Tensor(x11) + self.axis11 = axis11 + self.keep_dims11 = keep_dims11 + + self.x12 = Tensor(x12) + self.axis12 = axis12 + self.keep_dims12 = keep_dims12 + + self.x13 = Tensor(x13) + self.axis13 = axis13 + self.keep_dims13 = keep_dims13 + + @ms_function + def construct(self): + return (P.ReduceSum(self.keep_dims0)(self.x0, self.axis0), + P.ReduceSum(self.keep_dims1)(self.x1, self.axis1), + P.ReduceSum(self.keep_dims2)(self.x2, self.axis2), + P.ReduceSum(self.keep_dims3)(self.x3, self.axis3), + P.ReduceSum(self.keep_dims4)(self.x4, self.axis4), + P.ReduceSum(self.keep_dims5)(self.x5, self.axis5), + P.ReduceSum(self.keep_dims6)(self.x6, self.axis6), + P.ReduceSum(self.keep_dims7)(self.x7, self.axis7), + P.ReduceSum(self.keep_dims8)(self.x8, self.axis8), + P.ReduceSum(self.keep_dims9)(self.x9, self.axis9), + P.ReduceSum(self.keep_dims10)(self.x10, self.axis10), + P.ReduceSum(self.keep_dims11)(self.x11, self.axis11), + P.ReduceSum(self.keep_dims12)(self.x12, self.axis12), + P.ReduceSum(self.keep_dims13)(self.x13, self.axis13)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_ReduceSum(): + reduce_sum = ReduceSum() + output = reduce_sum() + + expect0 = np.sum(x0, axis=axis0, keepdims=keep_dims0) + diff0 = output[0].asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output[0].shape() == expect0.shape) + + expect1 = np.sum(x1, axis=axis1, keepdims=keep_dims1) + diff1 = output[1].asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output[1].shape() == expect1.shape) + + expect2 = np.sum(x2, axis=axis2, keepdims=keep_dims2) + diff2 = output[2].asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output[2].shape() == expect2.shape) + + expect3 = np.sum(x3, axis=axis3, keepdims=keep_dims3) + diff3 = output[3].asnumpy() - expect3 + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + assert np.all(diff3 < error3) + assert (output[3].shape() == expect3.shape) + + expect4 = np.sum(x4, axis=np_axis4, keepdims=keep_dims4) + diff4 = output[4].asnumpy() - expect4 + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + assert np.all(diff4 < error4) + assert (output[4].shape() == expect4.shape) + + expect5 = np.sum(x5, axis=np_axis5, keepdims=keep_dims5) + diff5 = output[5].asnumpy() - expect5 + error5 = np.ones(shape=expect5.shape) * 1.0e-5 + assert np.all(diff5 < error5) + assert (output[5].shape() == expect5.shape) + + expect6 = np.sum(x6, axis=axis6, keepdims=keep_dims6) + diff6 = output[6].asnumpy() - expect6 + error6 = np.ones(shape=expect6.shape) * 1.0e-5 + assert np.all(diff6 < error6) + assert (output[6].shape() == expect6.shape) + + expect7 = np.sum(x7, axis=axis7, keepdims=keep_dims7) + diff7 = output[7].asnumpy() - expect7 + error7 = np.ones(shape=expect7.shape) * 1.0e-5 + assert np.all(diff7 < error7) + assert (output[7].shape() == expect7.shape) + + expect8 = np.sum(x8, axis=axis8, keepdims=keep_dims8) + diff8 = output[8].asnumpy() - expect8 + error8 = np.ones(shape=expect8.shape) * 1.0e-5 + assert np.all(diff8 < error8) + assert (output[8].shape() == expect8.shape) + + expect9 = np.sum(x9, axis=axis9, keepdims=keep_dims9) + diff9 = output[9].asnumpy() - expect9 + error9 = np.ones(shape=expect9.shape) * 1.0e-5 + assert np.all(diff9 < error9) + assert (output[9].shape() == expect9.shape) + + expect10 = np.sum(x10, axis=axis10, keepdims=keep_dims10) + diff10 = output[10].asnumpy() - expect10 + error10 = np.ones(shape=expect10.shape) * 1.0e-5 + assert np.all(diff10 < error10) + assert (output[10].shape() == expect10.shape) + + expect11 = np.sum(x11, axis=axis11, keepdims=keep_dims11) + diff11 = output[11].asnumpy() - expect11 + error11 = np.ones(shape=expect11.shape) * 1.0e-5 + assert np.all(diff11 < error11) + assert (output[11].shape() == expect11.shape) + + expect12 = np.sum(x12, axis=axis12, keepdims=keep_dims12) + diff12 = output[12].asnumpy() - expect12 + error12 = np.ones(shape=expect12.shape) * 1.0e-5 + assert np.all(diff12 < error12) + assert (output[12].shape() == expect12.shape) + + expect13 = np.sum(x13, axis=axis13, keepdims=keep_dims13) + diff13 = output[13].asnumpy() - expect13 + error13 = np.ones(shape=expect13.shape) * 1.0e-5 + assert np.all(diff13 < error13) + assert (output[13].shape() == expect13.shape) diff --git a/tests/st/ops/gpu/test_relu6_grad_op.py b/tests/st/ops/gpu/test_relu6_grad_op.py new file mode 100644 index 00000000000..a589366372e --- /dev/null +++ b/tests/st/ops/gpu/test_relu6_grad_op.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetReLU6Grad(nn.Cell): + def __init__(self): + super(NetReLU6Grad, self).__init__() + self.relu6_grad = G.ReLU6Grad() + + def construct(self, x, dy): + return self.relu6_grad(dy, x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_relu6_grad(): + x = Tensor(np.array([[[[-1, 1, 10], + [5.9, 6.1, 6], + [10, 1, -1]]]]).astype(np.float32)) + dy = Tensor(np.array([[[[1, 1, 1], + [1, 1, 1], + [1, 1, 1]]]]).astype(np.float32)) + expect = np.array([[[[0, 1, 0, ], + [1, 0, 0, ], + [0, 1, 0, ]]]]).astype(np.float32) + error = np.ones(shape=[3, 3]) * 1.0e-6 + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + relu6_grad = NetReLU6Grad() + output = relu6_grad(x, dy) + diff = output.asnumpy() - expect + assert np.all(np.abs(diff) < error) diff --git a/tests/st/ops/gpu/test_relu6_op.py b/tests/st/ops/gpu/test_relu6_op.py new file mode 100644 index 00000000000..6be7921b2da --- /dev/null +++ b/tests/st/ops/gpu/test_relu6_op.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetReLU6(nn.Cell): + def __init__(self): + super(NetReLU6, self).__init__() + self.relu6 = P.ReLU6() + + def construct(self, x): + return self.relu6(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_relu6(): + x = Tensor(np.array([[[[-1, 1, 10], + [5.9, 6.1, 6], + [10, 1, -1]]]]).astype(np.float32)) + expect = np.array([[[[0, 1, 6, ], + [5.9, 6, 6, ], + [6, 1, 0.]]]]).astype(np.float32) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + relu6 = NetReLU6() + output = relu6(x) + assert (output.asnumpy() == expect).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + relu6 = NetReLU6() + output = relu6(x) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_relu_grad_op.py b/tests/st/ops/gpu/test_relu_grad_op.py new file mode 100644 index 00000000000..7c3369f756f --- /dev/null +++ b/tests/st/ops/gpu/test_relu_grad_op.py @@ -0,0 +1,51 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetReluGrad(nn.Cell): + def __init__(self): + super(NetReluGrad, self).__init__() + self.rekuGrad = G.ReluGrad() + + def construct(self, x, dy): + return self.rekuGrad(dy, x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_relu_grad(): + x = Tensor(np.array([[[[-1, 1, 1], + [1, -1, 1], + [1, 1, -1]]]]).astype(np.float32)) + dy = Tensor(np.array([[[[1, 0, 1], + [0, 1, 0], + [1, 1, 1]]]]).astype(np.float32)) + expect = np.array([[[[0, 0, 1, ], [0, 0, 0, ], [1, 1, 0.]]]]).astype(np.float32) + error = np.ones(shape=[3, 3]) * 1.0e-6 + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + relu_grad = NetReluGrad() + output = relu_grad(x, dy) + diff = output.asnumpy() - expect + assert np.all(diff < error) diff --git a/tests/st/ops/gpu/test_relu_op.py b/tests/st/ops/gpu/test_relu_op.py new file mode 100644 index 00000000000..229f837a9a8 --- /dev/null +++ b/tests/st/ops/gpu/test_relu_op.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class NetRelu(nn.Cell): + def __init__(self): + super(NetRelu, self).__init__() + self.relu = P.ReLU() + + def construct(self, x): + return self.relu(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_relu(): + x = Tensor(np.array([[[[-1, 1, 10], + [1, -1, 1], + [10, 1, -1]]]]).astype(np.float32)) + expect = np.array([[[[0, 1, 10, ], + [1, 0, 1, ], + [10, 1, 0.]]]]).astype(np.float32) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + relu = NetRelu() + output = relu(x) + assert (output.asnumpy() == expect).all() + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + relu = NetRelu() + output = relu(x) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_slice.py b/tests/st/ops/gpu/test_slice.py new file mode 100644 index 00000000000..95aed4ffa19 --- /dev/null +++ b/tests/st/ops/gpu/test_slice.py @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class Slice(nn.Cell): + def __init__(self): + super(Slice, self).__init__() + self.slice = P.Slice() + + def construct(self, x): + return self.slice(x, (0, 1, 0), (2, 1, 3)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_slice(): + x = Tensor( + np.array([[[1, -1, 1], [2, -2, 2]], [[3, -3, 3], [4, -4, 4]], [[5, -5, 5], [6, -6, 6]]]).astype(np.float32)) + expect = [[[2., -2., 2.]], + [[4., -4., 4.]]] + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + slice = Slice() + output = slice(x) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_slice_grad.py b/tests/st/ops/gpu/test_slice_grad.py new file mode 100644 index 00000000000..8e12320c346 --- /dev/null +++ b/tests/st/ops/gpu/test_slice_grad.py @@ -0,0 +1,54 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(device_target='GPU') + + +class SliceGrad(nn.Cell): + def __init__(self): + super(SliceGrad, self).__init__() + + self.slicegrad = G.SliceGrad() + + @ms_function + def construct(self, dy, x): + return self.slicegrad(dy, x, (0, 1, 0), (2, 1, 3)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_slice(): + x = Tensor(np.array([[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]], [[5, 5, 5], [6, 6, 6]]]).astype(np.float32)) + dy = Tensor(np.array([[[3., 1., 2.]], [[4., 1., 4.]]]).astype(np.float32)) + slicegrad = SliceGrad() + output = slicegrad(dy, x) + expect = [[[0., 0., 0.], + [3., 1., 2.]], + [[0., 0., 0.], + [4., 1., 4.]], + [[0., 0., 0.], + [0., 0., 0.]]] + print(output) + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_softmax_cross_entropy_with_logits_op.py b/tests/st/ops/gpu/test_softmax_cross_entropy_with_logits_op.py new file mode 100644 index 00000000000..1d04634957c --- /dev/null +++ b/tests/st/ops/gpu/test_softmax_cross_entropy_with_logits_op.py @@ -0,0 +1,58 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +class NetSoftmaxCrossEntropyWithLogits(nn.Cell): + def __init__( self): + super(NetSoftmaxCrossEntropyWithLogits, self).__init__() + self.loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False) + + def construct(self, logits, labels): + return self.loss(logits, labels) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_softmax_cross_entropy_with_logits(): + logits = Tensor(np.array([[1,1,10], + [1,10,1], + [10,1,1]]).astype(np.float32)) + labels = Tensor(np.array([[0,0,1], + [0,1,0], + [1,0,0]]).astype(np.float32)) + expect_loss = [0.00024673, 0.00024673, 0.00024673] + + context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + softmax_cross_entropy_with_logits = NetSoftmaxCrossEntropyWithLogits() + output = softmax_cross_entropy_with_logits(logits, labels) + error0 = 1.0e-6 + diff0 = output.asnumpy() - expect_loss + assert np.all(abs(diff0) < error0) + + context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU') + softmax_cross_entropy_with_logits = NetSoftmaxCrossEntropyWithLogits() + output = softmax_cross_entropy_with_logits(logits, labels) + error0 = 1.0e-6 + diff0 = output.asnumpy() - expect_loss + assert np.all(abs(diff0) < error0) diff --git a/tests/st/ops/gpu/test_softmax_op.py b/tests/st/ops/gpu/test_softmax_op.py new file mode 100644 index 00000000000..f4f799ec5a2 --- /dev/null +++ b/tests/st/ops/gpu/test_softmax_op.py @@ -0,0 +1,64 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import mindspore.context as context + +class NetSoftmax(nn.Cell): + def __init__(self): + super(NetSoftmax, self).__init__() + axis = -2 + self.softmax1 = P.Softmax() + self.softmax2 = P.Softmax(axis) + + def construct(self, x): + return self.softmax1(x), self.softmax2(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_softmax(): + x = Tensor(np.array([[0.1, 0.3, 0.6, -0.3], + [0.2, -0.6, 0.8, 0.6], + [0.6, -1.2, 0.4, 0.6]]).astype(np.float32)) + expect1 = np.ones(3) + expect2 = np.ones(4) + error1 = expect1 * 1.0e-6 + error2 = expect2 * 1.0e-6 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + Softmax = NetSoftmax() + output = Softmax(x) + outputSum1 = output[0].asnumpy().sum(axis=1) + outputSum2 = output[1].asnumpy().sum(axis=0) + diff1 = np.abs(outputSum1 - expect1) + diff2 = np.abs(outputSum2 - expect2) + assert np.all(diff1 < error1) + assert np.all(diff2 < error2) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + Softmax = NetSoftmax() + output = Softmax(x) + outputSum1 = output[0].asnumpy().sum(axis=1) + outputSum2 = output[1].asnumpy().sum(axis=0) + diff1 = np.abs(outputSum1 - expect1) + diff2 = np.abs(outputSum2 - expect2) + assert np.all(diff1 < error1) + assert np.all(diff2 < error2) diff --git a/tests/st/ops/gpu/test_sparse_softmax_cross_entropy_with_logits_op.py b/tests/st/ops/gpu/test_sparse_softmax_cross_entropy_with_logits_op.py new file mode 100644 index 00000000000..090a06ba198 --- /dev/null +++ b/tests/st/ops/gpu/test_sparse_softmax_cross_entropy_with_logits_op.py @@ -0,0 +1,66 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +class NetSparseSoftmaxCrossEntropyWithLogits(nn.Cell): + def __init__( self): + super(NetSparseSoftmaxCrossEntropyWithLogits, self).__init__() + self.loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + self.dlogits = nn.SoftmaxCrossEntropyWithLogits(is_grad=True, sparse=True) + + def construct(self, logits, labels): + return (self.loss(logits, labels), self.dlogits(logits, labels)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_sparse_softmax_cross_entropy_with_logits(): + logits = Tensor(np.array([[1,1,10], + [1,10,1], + [10,1,1]]).astype(np.float32)) + labels = Tensor(np.array([2,1,0]).astype(np.int32)) + expect_loss = 0.0002467 + expect_dlogits = np.array([[4.1126452e-05, 4.1126452e-05, -8.2234539e-05], + [4.1126452e-05, -8.2234539e-05, 4.1126452e-05], + [-8.2234539e-05, 4.1126452e-05, 4.1126452e-05]]).astype(np.float32) + + context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + sparse_softmax_cross_entropy_with_logits = NetSparseSoftmaxCrossEntropyWithLogits() + output = sparse_softmax_cross_entropy_with_logits(logits, labels) + error0 = 1.0e-6 + diff0 = output[0].asnumpy() - expect_loss + assert np.all(abs(diff0) < error0) + + error1 = np.ones(shape=[3, 3]) * 1.0e-6 + diff1 = output[1].asnumpy() - expect_dlogits + assert np.all(abs(diff1) < error1) + + context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU') + sparse_softmax_cross_entropy_with_logits = NetSparseSoftmaxCrossEntropyWithLogits() + output = sparse_softmax_cross_entropy_with_logits(logits, labels) + error0 = 1.0e-6 + diff0 = output[0].asnumpy() - expect_loss + assert np.all(abs(diff0) < error0) + + error1 = np.ones(shape=[3, 3]) * 1.0e-6 + diff1 = output[1].asnumpy() - expect_dlogits + assert np.all(abs(diff1) < error1) diff --git a/tests/st/ops/gpu/test_stridedslice_grad_op.py b/tests/st/ops/gpu/test_stridedslice_grad_op.py new file mode 100644 index 00000000000..48caa1bd4f1 --- /dev/null +++ b/tests/st/ops/gpu/test_stridedslice_grad_op.py @@ -0,0 +1,48 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + + +class StridedSliceGrad(nn.Cell): + def __init__(self): + super(StridedSliceGrad, self).__init__() + self.ssg = G.StridedSliceGrad() + self.shape = P.Shape() + + @ms_function + def construct(self, dy, x): + return self.ssg(dy, self.shape(x), (2, 0, 0), (3, 2, 3), (1, 1, 1)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_slice(): + x = Tensor(np.array([[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]], [[5, 5, 5], [6, 7, 8]]]).astype(np.float32)) + dy = Tensor(np.array([[[5., 1., 5.], [6., 1., 8.]]]).astype(np.float32)) + ssg = StridedSliceGrad() + output = ssg(dy, x) + expect = [[[0, 0, 0], [0, 0, 0]], [[0, 0, 0], [0, 0, 0]], [[5, 1, 5], [6, 1, 8]]] + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_stridedslice_op.py b/tests/st/ops/gpu/test_stridedslice_op.py new file mode 100644 index 00000000000..c5cb285678f --- /dev/null +++ b/tests/st/ops/gpu/test_stridedslice_op.py @@ -0,0 +1,44 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + +context.set_context(mode=context.GRAPH_MODE, device_target='GPU') + + +class StridedSlice(nn.Cell): + def __init__(self): + super(StridedSlice, self).__init__() + self.stridedslice = P.StridedSlice() + + def construct(self, x): + return self.stridedslice(x, (2, 0, 0), (3, 2, 3), (1, 1, 1)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_slice(): + x = Tensor(np.array([[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]], [[5, 5, 5], [6, 7, 8]]]).astype(np.int32)) + stridedslice = StridedSlice() + output = stridedslice(x) + expect = [[[5., 5., 5.], + [6., 7., 8.]]] + assert (output.asnumpy() == expect).all() diff --git a/tests/st/ops/gpu/test_sub_op.py b/tests/st/ops/gpu/test_sub_op.py new file mode 100644 index 00000000000..44125e475b4 --- /dev/null +++ b/tests/st/ops/gpu/test_sub_op.py @@ -0,0 +1,113 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +import numpy as np +import mindspore.context as context + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.sub = P.Sub() + + def construct(self, x, y): + return self.sub(x, y) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_Sub(): + np_x0 = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + np_y0 = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + np_x1 = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + np_y1 = np.random.uniform(-2, 2, (2, 1, 4, 4)).astype(np.float32) + np_x2 = np.random.uniform(-2, 2, (2, 1, 1, 4)).astype(np.float32) + np_y2 = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + np_x3 = np.random.uniform(-2, 2, 1).astype(np.float32) + np_y3 = np.random.uniform(-2, 2, 1).astype(np.float32) + np_x4 = np.array(768).astype(np.float32) + np_y4 = np.array(3072.5).astype(np.float32) + x0 = Tensor(np_x0) + y0 = Tensor(np_y0) + x1 = Tensor(np_x1) + y1 = Tensor(np_y1) + x2 = Tensor(np_x2) + y2 = Tensor(np_y2) + x3 = Tensor(np_x3) + y3 = Tensor(np_y3) + x4 = Tensor(np_x4) + y4 = Tensor(np_y4) + + expect0 = np.subtract(np_x0, np_y0) + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + expect1 = np.subtract(np_x1, np_y1) + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + expect2 = np.subtract(np_x2, np_y2) + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + expect3 = np.subtract(np_x3, np_y3) + error3 = np.ones(shape=expect3.shape) * 1.0e-5 + expect4 = np.subtract(np_x4, np_y4) + error4 = np.ones(shape=expect4.shape) * 1.0e-5 + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + sub = Net() + output0 = sub(x0, y0) + output1 = sub(x1, y1) + output2 = sub(x2, y2) + output3 = sub(x3, y3) + output4 = sub(x4, y4) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + diff2 = output2.asnumpy() - expect2 + assert np.all(diff2 < error2) + assert (output2.shape() == expect2.shape) + diff3 = output3.asnumpy() - expect3 + assert np.all(diff3 < error3) + assert (output3.shape() == expect3.shape) + diff4 = output4.asnumpy() - expect4 + assert np.all(diff4 < error4) + assert (output4.shape() == expect4.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + sub = Net() + output0 = sub(x0, y0) + output1 = sub(x1, y1) + output2 = sub(x2, y2) + output3 = sub(x3, y3) + output4 = sub(x4, y4) + diff0 = output0.asnumpy() - expect0 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + diff1 = output1.asnumpy() - expect1 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + diff2 = output2.asnumpy() - expect2 + assert np.all(diff2 < error2) + assert (output2.shape() == expect2.shape) + diff3 = output3.asnumpy() - expect3 + assert np.all(diff3 < error3) + assert (output3.shape() == expect3.shape) + diff4 = output4.asnumpy() - expect4 + assert np.all(diff4 < error4) + assert (output4.shape() == expect4.shape) diff --git a/tests/st/ops/gpu/test_tensoradd.py b/tests/st/ops/gpu/test_tensoradd.py new file mode 100644 index 00000000000..a60f5dcf30e --- /dev/null +++ b/tests/st/ops/gpu/test_tensoradd.py @@ -0,0 +1,130 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(device_target='GPU') + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +class TensroAdd(nn.Cell): + def __init__(self): + super(TensroAdd, self).__init__() + + self.add = P.TensorAdd() + + self.x = Parameter(initializer( + Tensor(np.random.randn(2, 0).astype(np.float32)), [2, 0]), name='x') + self.y = Parameter(initializer( + Tensor(np.random.randn(2, 1).astype(np.float32)), [2, 1]), name='y') + + self.x1 = Parameter(initializer( + Tensor(np.arange(3).reshape(3).astype(np.float32)), [3]), name='x1') + self.y1 = Parameter(initializer( + Tensor(np.array([2]).astype(np.float32)), [1]), name='y1') + + self.x2 = Parameter(initializer( + Tensor(np.arange(3 * 3 * 3 * 3).reshape(3, 3, 3, 3).astype(np.float32)), [3, 3, 3, 3]), name='x2') + self.y2 = Parameter(initializer( + Tensor(np.arange(3 * 3 * 3 * 3).reshape(3, 3, 3, 3).astype(np.float32)), [3, 3, 3, 3]), name='y2') + + self.x3 = Parameter(initializer( + Tensor(np.arange(1 * 1 * 3 * 3).reshape(1, 1, 3, 3).astype(np.float32)), [1, 1, 3, 3]), name='x3') + self.y3 = Parameter(initializer( + Tensor(np.arange(3 * 3 * 3 * 3).reshape(3, 3, 3, 3).astype(np.float32)), [3, 3, 3, 3]), name='y3') + + @ms_function + def construct(self): + return ( + self.add(self.x, self.y), self.add(self.x1, self.y1), self.add(self.x2, self.y2), + self.add(self.x3, self.y3)) + + +def test_TensroAdd(): + add = TensroAdd() + output = add() + expect0 = np.array([]) + expect1 = np.array([2, 3, 4]) + expect2 = np.array( + [[[[0., 2., 4.], + [6., 8., 10.], + [12., 14., 16.]], + [[18., 20., 22.], + [24., 26., 28.], + [30., 32., 34.]], + [[36., 38., 40.], + [42., 44., 46.], + [48., 50., 52.]]], + [[[54., 56., 58.], + [60., 62., 64.], + [66., 68., 70.]], + [[72., 74., 76.], + [78., 80., 82.], + [84., 86., 88.]], + [[90., 92., 94.], + [96., 98., 100.], + [102., 104., 106.]]], + [[[108., 110., 112.], + [114., 116., 118.], + [120., 122., 124.]], + [[126., 128., 130.], + [132., 134., 136.], + [138., 140., 142.]], + [[144., 146., 148.], + [150., 152., 154.], + [156., 158., 160.]]]]) + expect3 = np.array( + [[[[0., 2., 4.], + [6., 8., 10.], + [12., 14., 16.]], + [[9., 11., 13.], + [15., 17., 19.], + [21., 23., 25.]], + [[18., 20., 22.], + [24., 26., 28.], + [30., 32., 34.]]], + [[[27., 29., 31.], + [33., 35., 37.], + [39., 41., 43.]], + [[36., 38., 40.], + [42., 44., 46.], + [48., 50., 52.]], + [[45., 47., 49.], + [51., 53., 55.], + [57., 59., 61.]]], + [[[54., 56., 58.], + [60., 62., 64.], + [66., 68., 70.]], + [[63., 65., 67.], + [69., 71., 73.], + [75., 77., 79.]], + [[72., 74., 76.], + [78., 80., 82.], + [84., 86., 88.]]]] + ) + assert (output[0].asnumpy() == expect0).all() + assert (output[1].asnumpy() == expect1).all() + assert (output[2].asnumpy() == expect2).all() + assert (output[3].asnumpy() == expect3).all() diff --git a/tests/st/ops/gpu/test_tile_op.py b/tests/st/ops/gpu/test_tile_op.py new file mode 100644 index 00000000000..57c6ae95f6f --- /dev/null +++ b/tests/st/ops/gpu/test_tile_op.py @@ -0,0 +1,79 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore.ops.operations import Tile +from mindspore.nn import Cell +from mindspore.common.tensor import Tensor +from mindspore.common.api import ms_function +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import numpy as np + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + +input_x0 = np.arange(2).reshape((2, 1, 1)).astype(np.float32) +mul0 = (8, 1, 1) +input_x1 = np.arange(32).reshape((2, 4, 4)).astype(np.float32) +mul1 = (2, 2, 2) +input_x2 = np.arange(1).reshape((1, 1, 1)).astype(np.float32) +mul2 = (1, 1, 1) + + +class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.Tile = Tile() + + self.input_x0 = Parameter(initializer(Tensor(input_x0), input_x0.shape), name='x0') + self.mul0 = mul0 + self.input_x1 = Parameter(initializer(Tensor(input_x1), input_x1.shape), name='x1') + self.mul1 = mul1 + self.input_x2 = Parameter(initializer(Tensor(input_x2), input_x2.shape), name='x2') + self.mul2 = mul2 + + @ms_function + def construct(self): + output = (self.Tile(self.input_x0, self.mul0), + self.Tile(self.input_x1, self.mul1), + self.Tile(self.input_x2, self.mul2)) + return output + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_tile(): + net = Net() + output = net() + + expect0 = np.tile(input_x0, mul0) + diff0 = output[0].asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output[0].shape() == expect0.shape) + + expect1 = np.tile(input_x1, mul1) + diff1 = output[1].asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output[1].shape() == expect1.shape) + + expect2 = np.tile(input_x2, mul2) + diff2 = output[2].asnumpy() - expect2 + error2 = np.ones(shape=expect2.shape) * 1.0e-5 + assert np.all(diff2 < error2) + assert (output[2].shape() == expect2.shape) diff --git a/tests/st/ops/gpu/test_transpose_op.py b/tests/st/ops/gpu/test_transpose_op.py new file mode 100644 index 00000000000..61f96fa78ce --- /dev/null +++ b/tests/st/ops/gpu/test_transpose_op.py @@ -0,0 +1,143 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.api import ms_function +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +import mindspore.nn as nn +import mindspore.context as context + +context.set_context(device_target='GPU') + +class Transpose(nn.Cell): + def __init__(self): + super(Transpose, self).__init__() + self.transpose = P.Transpose() + + self.x_2D = Parameter(initializer(Tensor(np.arange(5 * 6).reshape(5, 6).astype(np.float32)), [5, 6]), + name='x_2D') + self.perm_2D = (1, 0) + + self.x_3D = Parameter(initializer(Tensor(np.arange(2 * 2 * 4).reshape(2, 2, 4).astype(np.float32)), [2, 2, 4]), + name='x_3D') + self.perm_3D = (1, 0, 2) + + self.x_4D = Parameter( + initializer(Tensor(np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5).astype(np.float32)), [2, 3, 4, 5]), + name='x_4D') + self.perm_4D = (0, 1, 2, 3) + + self.x_5D = Parameter( + initializer(Tensor(np.arange(1 * 2 * 3 * 4 * 5).reshape(1, 2, 3, 4, 5).astype(np.float32)), + [1, 2, 3, 4, 5]), name='x_5D') + self.perm_5D = (1, 0, 3, 4, 2) + + @ms_function + def construct(self): + return (self.transpose(self.x_2D, self.perm_2D), self.transpose(self.x_3D, self.perm_3D), + self.transpose(self.x_4D, self.perm_4D), self.transpose(self.x_5D, self.perm_5D)) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_transpose(): + transpose = Transpose() + output = transpose() + + expect0 = np.array([[[0, 6, 12, 18, 24], + [1, 7, 13, 19, 25], + [2, 8, 14, 20, 26], + [3, 9, 15, 21, 27], + [4, 10, 16, 22, 28], + [5, 11, 17, 23, 29]]]).astype(np.float32) + expect1 = np.array([[[[0, 1, 2, 3], + [8, 9, 10, 11]], + [[4, 5, 6, 7], + [12, 13, 14, 15]]]]).astype(np.float32) + expect2 = np.array([[[[[0, 1, 2, 3, 4], + [5, 6, 7, 8, 9], + [10, 11, 12, 13, 14], + [15, 16, 17, 18, 19]], + [[20, 21, 22, 23, 24], + [25, 26, 27, 28, 29], + [30, 31, 32, 33, 34], + [35, 36, 37, 38, 39]], + [[40, 41, 42, 43, 44], + [45, 46, 47, 48, 49], + [50, 51, 52, 53, 54], + [55, 56, 57, 58, 59]]], + + [[[60, 61, 62, 63, 64], + [65, 66, 67, 68, 69], + [70, 71, 72, 73, 74], + [75, 76, 77, 78, 79]], + [[80, 81, 82, 83, 84], + [85, 86, 87, 88, 89], + [90, 91, 92, 93, 94], + [95, 96, 97, 98, 99]], + [[100, 101, 102, 103, 104], + [105, 106, 107, 108, 109], + [110, 111, 112, 113, 114], + [115, 116, 117, 118, 119]]]]]).astype(np.float32) + expect3 = np.array([[[[[[0, 20, 40], + [1, 21, 41], + [2, 22, 42], + [3, 23, 43], + [4, 24, 44]], + [[5, 25, 45], + [6, 26, 46], + [7, 27, 47], + [8, 28, 48], + [9, 29, 49]], + [[10, 30, 50], + [11, 31, 51], + [12, 32, 52], + [13, 33, 53], + [14, 34, 54]], + [[15, 35, 55], + [16, 36, 56], + [17, 37, 57], + [18, 38, 58], + [19, 39, 59]]]], + + [[[[60, 80, 100], + [61, 81, 101], + [62, 82, 102], + [63, 83, 103], + [64, 84, 104]], + [[65, 85, 105], + [66, 86, 106], + [67, 87, 107], + [68, 88, 108], + [69, 89, 109]], + [[70, 90, 110], + [71, 91, 111], + [72, 92, 112], + [73, 93, 113], + [74, 94, 114]], + [[75, 95, 115], + [76, 96, 116], + [77, 97, 117], + [78, 98, 118], + [79, 99, 119]]]]]]).astype(np.float32) + assert (output[0].asnumpy() == expect0).all() + assert (output[1].asnumpy() == expect1).all() + assert (output[2].asnumpy() == expect2).all() + assert (output[3].asnumpy() == expect3).all() diff --git a/tests/st/ops/gpu/test_zeroslike_op.py b/tests/st/ops/gpu/test_zeroslike_op.py new file mode 100644 index 00000000000..46a031249ae --- /dev/null +++ b/tests/st/ops/gpu/test_zeroslike_op.py @@ -0,0 +1,76 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.api import ms_function +import numpy as np +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") +class NetZerosLike(nn.Cell): + def __init__(self): + super(NetZerosLike, self).__init__() + self.zeros_like = P.ZerosLike() + + def construct(self, x): + return self.zeros_like(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_ZerosLike(): + x0_np = np.random.uniform(-2, 2, (2, 3, 4, 4)).astype(np.float32) + x1_np = np.random.uniform(-2, 2, 1).astype(np.float32) + + x0 = Tensor(x0_np) + x1 = Tensor(x1_np) + + context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + zeros_like = NetZerosLike() + output0 = zeros_like(x0) + expect0 = np.zeros_like(x0_np) + diff0 = output0.asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + + output1 = zeros_like(x1) + expect1 = np.zeros_like(x1_np) + diff1 = output1.asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) + + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + zeros_like = NetZerosLike() + output0 = zeros_like(x0) + expect0 = np.zeros_like(x0_np) + diff0 = output0.asnumpy() - expect0 + error0 = np.ones(shape=expect0.shape) * 1.0e-5 + assert np.all(diff0 < error0) + assert (output0.shape() == expect0.shape) + + output1 = zeros_like(x1) + expect1 = np.zeros_like(x1_np) + diff1 = output1.asnumpy() - expect1 + error1 = np.ones(shape=expect1.shape) * 1.0e-5 + assert np.all(diff1 < error1) + assert (output1.shape() == expect1.shape) diff --git a/tests/st/summary/test_davinci_summary.py b/tests/st/summary/test_davinci_summary.py new file mode 100644 index 00000000000..1611ca8ec72 --- /dev/null +++ b/tests/st/summary/test_davinci_summary.py @@ -0,0 +1,100 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test model train """ +import os +import numpy as np + +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore import Tensor, Parameter, Model +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim import Momentum +from mindspore.common.api import ms_function +import mindspore.nn as wrap +import mindspore.context as context +from apply_momentum import ApplyMomentum +from mindspore.train.summary.summary_record import SummaryRecord + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +context.set_context(device_target="Ascend") + +class MsWrapper(nn.Cell): + def __init__(self, network): + super(MsWrapper, self).__init__(auto_prefix=False) + self._network = network + @ms_function + def construct(self, *args): + return self._network(*args) + + +def me_train_tensor(net, input_np, label_np, epoch_size=2): + context.set_context(mode=context.GRAPH_MODE) + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + opt = ApplyMomentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])), filter(lambda x: x.requires_grad, net.get_parameters())) + Model(net, loss, opt) + _network = wrap.WithLossCell(net, loss) + _train_net = MsWrapper(wrap.TrainOneStepCell(_network, opt)) + _train_net.set_train() + summary_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=_train_net) + for epoch in range(0, epoch_size): + print(f"epoch %d"%(epoch)) + output = _train_net(Tensor(input_np), Tensor(label_np)) + summary_writer.record(i) + print("********output***********") + print(output.asnumpy()) + summary_writer.close() + + +def me_infer_tensor(net, input_np): + net.set_train() + net = MsWrapper(net) + output = net(Tensor(input_np)) + return output + + +def test_net(): + class Net(nn.Cell): + def __init__(self, cin, cout): + super(Net, self).__init__() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + self.conv = nn.Conv2d(cin, cin, kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") + self.bn = nn.BatchNorm2d(cin, momentum=0.1, eps=0.0001) + self.add = P.TensorAdd() + self.relu = P.ReLU() + self.mean = P.ReduceMean(keep_dims=True) + self.reshape = P.Reshape() + self.dense = nn.Dense(cin, cout) + + def construct(self, input_x): + output = input_x + output = self.maxpool(output) + identity = output + output = self.conv(output) + output = self.bn(output) + output = self.add(output, identity) + output = self.relu(output) + output = self.mean(output, (-2, -1)) + output = self.reshape(output, (32, -1)) + output = self.dense(output) + return output + + net = Net(2048, 1001) + input_np = np.ones([32, 2048, 14, 14]).astype(np.float32) * 0.01 + label_np = np.ones([32]).astype(np.int32) + me_train_tensor(net, input_np, label_np) + #me_infer_tensor(net, input_np) diff --git a/tests/st/summary/test_gpu_summary.py b/tests/st/summary/test_gpu_summary.py new file mode 100644 index 00000000000..ef645fcb2e8 --- /dev/null +++ b/tests/st/summary/test_gpu_summary.py @@ -0,0 +1,154 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +import os +import time +import shutil +import random +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.train.summary.summary_record import SummaryRecord + +''' + This testcase is used for save summary data only. You need install MindData first and uncomment the commented + packages to analyse summary data. + Using "minddata start --datalog='./test_me_summary_event_file/' --host=0.0.0.0" to make data visible. +''' +# from minddata.datavisual.data_transform.data_manager import DataManager +# from minddata.datavisual.visual.train_visual.train_task_manager import TrainTaskManager +# from minddata.datavisual.visual.train_visual.scalars_processor import ScalarsProcessor +# from minddata.datavisual.common.enums import PluginNameEnum +# from minddata.datavisual.common.enums import DataManagerStatus + + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + + +CUR_DIR = os.getcwd() +SUMMARY_DIR_ME = CUR_DIR + "/test_me_summary_event_file/" +SUMMARY_DIR_ME_TEMP = CUR_DIR + "/test_me_temp_summary_event_file/" + +def clean_environment_file(srcDir): + if os.path.exists(srcDir): + ls = os.listdir(srcDir) + for line in ls: + filePath = os.path.join(srcDir, line) + os.remove(filePath) + os.removedirs(srcDir) +def save_summary_events_file(srcDir, desDir): + if not os.path.exists(desDir): + print("-- create desDir") + os.makedirs(desDir) + + ls = os.listdir(srcDir) + for line in ls: + filePath = os.path.join(srcDir, line) + if os.path.isfile(filePath): + print("-- move events file : {}".format(filePath)) + shutil.copy(filePath, desDir) + os.remove(filePath) + os.removedirs(srcDir) + +class SummaryNet(nn.Cell): + def __init__(self, tag_tuple=None, scalar=1): + super(SummaryNet, self).__init__() + self.summary_s = P.ScalarSummary() + self.summary_i = P.ImageSummary() + self.summary_t = P.TensorSummary() + self.add = P.TensorAdd() + self.tag_tuple = tag_tuple + self.scalar = scalar + + def construct(self, x, y): + self.summary_i("image", x) + self.summary_s("x1", x) + z = self.add(x, y) + self.summary_t("z1", z) + return z + +def train_summary_record_scalar_for_1(test_writer, steps, fwd_x, fwd_y): + net = SummaryNet() + out_me_dict = {} + for i in range(0, steps): + x = Tensor(np.array([1.1 + random.uniform(1, 10)]).astype(np.float32)) + y = Tensor(np.array([1.2 + random.uniform(1, 10)]).astype(np.float32)) + out_put = net(x, y) + test_writer.record(i) + print("-----------------output: %s-------------\n", out_put.asnumpy()) + out_me_dict[i] = out_put.asnumpy() + return out_me_dict + +def me_scalar_summary(steps, tag=None, value=None): + test_writer = SummaryRecord(SUMMARY_DIR_ME_TEMP) + + x = Tensor(np.array([1.1]).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + + out_me_dict = train_summary_record_scalar_for_1(test_writer, steps, x, y) + + test_writer.close() + return out_me_dict + +def print_scalar_data(): + print("============start print_scalar_data\n") + data_manager = DataManager() + data_manager.start_load_data(path=SUMMARY_DIR_ME) + while data_manager.get_status() != DataManagerStatus.DONE: + time.sleep(0.1) + task_manager = TrainTaskManager(data_manager) + train_jobs = task_manager.get_all_train_tasks(PluginNameEnum.scalar) + print(train_jobs) + """ + train_jobs + ['train_jobs': { + 'id': '12-123', + 'name': 'train_job_name', + 'tags': ['x1', 'y1'] + }] + """ + scalar_processor = ScalarsProcessor(data_manager) + metadata = scalar_processor.get_metadata_list(train_job_ids=train_jobs['train_jobs'][0]['id'], tag=train_jobs['train_jobs'][0]['tags'][0]) + print(metadata) + ''' + metadata + { + 'scalars' : [ + { + 'train_job_id' : '12-12', + 'metadatas' : [ + { + 'wall_time' : 0.1, + 'step' : 1, + 'value' : 0.1 + } + ] + } + ] + } + ''' + print("============end print_scalar_data\n") + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_scalarsummary_scalar1_step10_summaryrecord1(): + clean_environment_file(SUMMARY_DIR_ME_TEMP) + output_dict = me_scalar_summary(10) + print("test_scalarsummary_scalar1_step10_summaryrecord1 \n",output_dict) + save_summary_events_file(SUMMARY_DIR_ME_TEMP, SUMMARY_DIR_ME) + clean_environment_file(SUMMARY_DIR_ME) diff --git a/tests/st/tbe_networks/env.sh b/tests/st/tbe_networks/env.sh new file mode 100755 index 00000000000..991bb803916 --- /dev/null +++ b/tests/st/tbe_networks/env.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +export LOCAL_HIAI=/usr/local/HiAI +export TBE_IMPL_PATH=${LOCAL_HIAI}/runtime/ops/op_impl/built-in/ai_core/tbe/impl/ +export LD_LIBRARY_PATH=${LOCAL_HIAI}/runtime/lib64/:${LD_LIBRARY_PATH} +export PATH=${LOCAL_HIAI}/runtime/ccec_compiler/bin/:${PATH} +export PYTHONPATH=${LOCAL_HIAI}/runtime/ops/op_impl/built-in/ai_core/tbe/:${PYTHONPATH} diff --git a/tests/st/tbe_networks/export_geir.py b/tests/st/tbe_networks/export_geir.py new file mode 100644 index 00000000000..467388c5e8d --- /dev/null +++ b/tests/st/tbe_networks/export_geir.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import os +import numpy as np +from resnet_torch import resnet50 +from mindspore.train.callback import Callback +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore import Tensor +import mindspore.nn as nn +from mindspore import context + +from mindspore.train.serialization import save, load, save_checkpoint, load_checkpoint,\ + load_param_into_net, _exec_save_checkpoint,\ + _check_filedir_or_create, _chg_model_file_name_if_same_exist, \ + _read_file_last_line, context, export + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", +enable_task_sink=True,enable_loop_sink=True,enable_ir_fusion=True) + +def test_resnet50_export(batch_size=1, num_classes=5): + context.set_context(enable_ir_fusion=False) + input_np = np.random.uniform(0.0, 1.0, size=[batch_size, 3, 224, 224]).astype(np.float32) + net = resnet50(batch_size, num_classes) + #param_dict = load_checkpoint("./resnet50-1_103.ckpt") + #load_param_into_net(net, param_dict) + export(net, Tensor(input_np), file_name="./me_resnet50.pb", file_format="GEIR") diff --git a/tests/st/tbe_networks/hccl_2p_evb_template.json b/tests/st/tbe_networks/hccl_2p_evb_template.json new file mode 100644 index 00000000000..4d303fe80d7 --- /dev/null +++ b/tests/st/tbe_networks/hccl_2p_evb_template.json @@ -0,0 +1,43 @@ +{ + "board_id": "0x3000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "2", + "server_num": "1", + "group_name": "", + "instance_count": "2", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "[device_ip]" + } + ], + "rank_id": "0", + "server_id": "[sever_id]" + }, + { + "devices": [ + { + "device_id": "1", + "device_ip": "[device_ip]" + } + ], + "rank_id": "1", + "server_id": "[sever_id]" + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1" + ], + "para_plane_nic_num": "2", + "status": "completed" +} \ No newline at end of file diff --git a/tests/st/tbe_networks/hccl_2p_server_template.json b/tests/st/tbe_networks/hccl_2p_server_template.json new file mode 100644 index 00000000000..a4a1ab0cb90 --- /dev/null +++ b/tests/st/tbe_networks/hccl_2p_server_template.json @@ -0,0 +1,43 @@ +{ + "board_id": "0x0000", + "chip_info": "910", + "deploy_mode": "lab", + "group_count": "1", + "group_list": [ + { + "device_num": "2", + "server_num": "1", + "group_name": "", + "instance_count": "2", + "instance_list": [ + { + "devices": [ + { + "device_id": "0", + "device_ip": "[device_ip]" + } + ], + "rank_id": "0", + "server_id": "[sever_id]" + }, + { + "devices": [ + { + "device_id": "1", + "device_ip": "[device_ip]" + } + ], + "rank_id": "1", + "server_id": "[sever_id]" + } + ] + } + ], + "para_plane_nic_location": "device", + "para_plane_nic_name": [ + "eth0", + "eth1" + ], + "para_plane_nic_num": "2", + "status": "completed" +} \ No newline at end of file diff --git a/tests/st/tbe_networks/resnet.py b/tests/st/tbe_networks/resnet.py new file mode 100644 index 00000000000..a1ece6556ed --- /dev/null +++ b/tests/st/tbe_networks/resnet.py @@ -0,0 +1,300 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore.common import dtype as mstype + +def weight_variable(shape): + return initializer('XavierUniform', shape=shape, dtype=mstype.float32) + + +def weight_variable_uniform(shape): + return initializer('Uniform', shape=shape, dtype=mstype.float32) + + +def weight_variable_0(shape): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros) + + +def weight_variable_1(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones) + + +def conv3x3(in_channels, out_channels, stride=1, padding=0): + """3x3 convolution """ + weight_shape = (out_channels, in_channels, 3, 3) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 1, 1) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def conv7x7(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 7, 7) + weight = weight_variable(weight_shape) + return nn.Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_uniform(shape) + bn = nn.BatchNorm2d(out_channels, momentum=0.99, eps=0.00001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def bn_with_initialize_last(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_uniform(shape) + bn = nn.BatchNorm2d(out_channels, momentum=0.99, eps=0.00001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + return bn + + +def fc_with_initialize(input_channels, out_channels): + weight_shape = (out_channels, input_channels) + weight = weight_variable(weight_shape) + bias_shape = (out_channels) + bias = weight_variable_uniform(bias_shape) + return nn.Dense(input_channels, out_channels, weight, bias) + + +class ResidualBlock(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResidualBlockWithDown(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlockWithDown, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu = P.ReLU() + self.downSample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn_down_sample = bn_with_initialize(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class MakeLayer0(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer0, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=1, down_sample=True) + self.b = block(out_channels, out_channels, stride=stride) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class MakeLayer1(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer1, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + + return x + + +class MakeLayer2(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer2, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + self.d = block(out_channels, out_channels, stride=1) + self.e = block(out_channels, out_channels, stride=1) + self.f = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + x = self.d(x) + x = self.e(x) + x = self.f(x) + + return x + + +class MakeLayer3(nn.Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer3, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=stride, down_sample=True) + self.b = block(out_channels, out_channels, stride=1) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class ResNet(nn.Cell): + + def __init__(self, block, layer_num, num_classes=100, batch_size=32): + super(ResNet, self).__init__() + self.batch_size = batch_size + self.num_classes = num_classes + + self.conv1 = conv7x7(3, 64, stride=2, padding=0) + + self.bn1 = bn_with_initialize(64) + self.relu = P.ReLU() + self.maxpool = P.MaxPoolWithArgmax(window=3, stride=2, pad_mode="same") + + self.layer1 = MakeLayer0(block, layer_num[0], in_channels=64, out_channels=256, stride=1) + self.layer2 = MakeLayer1(block, layer_num[1], in_channels=256, out_channels=512, stride=2) + self.layer3 = MakeLayer2(block, layer_num[2], in_channels=512, out_channels=1024, stride=2) + self.layer4 = MakeLayer3(block, layer_num[3], in_channels=1024, out_channels=2048, stride=2) + + self.pool = P.ReduceMean(keep_dims=True) + self.squeeze = P.Squeeze(axis=(2, 3)) + self.fc = fc_with_initialize(512 * block.expansion, num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x)[0] + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.pool(x, (2, 3)) + x = self.squeeze(x) + x = self.fc(x) + return x + + +def resnet50(batch_size, num_classes): + return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size) + diff --git a/tests/st/tbe_networks/resnet_cifar.py b/tests/st/tbe_networks/resnet_cifar.py new file mode 100644 index 00000000000..f1ab02afa39 --- /dev/null +++ b/tests/st/tbe_networks/resnet_cifar.py @@ -0,0 +1,157 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore import context +import mindspore.common.dtype as mstype +import os +import numpy as np +import mindspore.ops.functional as F +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore.dataset as ds +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore.communication.management import init +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from resnet import resnet50 +import random +random.seed(1) +np.random.seed(1) +ds.config.set_seed(1) + +import argparse +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') +parser.add_argument('--epoch_size', type=int, default=1, help='Epoch size.') +parser.add_argument('--batch_size', type=int, default=32, help='Batch size.') +parser.add_argument('--num_classes', type=int, default=10, help='Num classes.') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default="/var/log/npu/datasets/cifar", help='Dataset path') +args_opt = parser.parse_args() + +device_id=int(os.getenv('DEVICE_ID')) + +data_home=args_opt.dataset_path + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(enable_task_sink=True, device_id=device_id) +context.set_context(enable_loop_sink=True) +context.set_context(enable_mem_reuse=True) + +def create_dataset(repeat_num=1, training=True): + data_dir = data_home + "/cifar-10-batches-bin" + if not training: + data_dir = data_home + "/cifar-10-verify-bin" + data_set = ds.Cifar10Dataset(data_dir) + + if args_opt.run_distribute: + rank_id=int(os.getenv('RANK_ID')) + rank_size=int(os.getenv('RANK_SIZE')) + data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) + + resize_height = 224 + resize_width = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT + random_horizontal_op = vision.RandomHorizontalFlip() + resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR + rescale_op = vision.Rescale(rescale, shift) + normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) + changeswap_op = vision.HWC2CHW() + type_cast_op = C.TypeCast(mstype.int32) + + c_trans = [] + if training: + c_trans = [random_crop_op, random_horizontal_op] + c_trans += [resize_op, rescale_op, normalize_op, + changeswap_op] + + # apply map operations on images + data_set = data_set.map(input_columns="label", operations=type_cast_op) + data_set = data_set.map(input_columns="image", operations=c_trans) + + # apply repeat operations + data_set = data_set.repeat(repeat_num) + + # apply shuffle operations + data_set = data_set.shuffle(buffer_size=10) + + # apply batch operations + data_set = data_set.batch(batch_size=args_opt.batch_size, drop_remainder=True) + + return data_set + +class CrossEntropyLoss(nn.Cell): + def __init__(self): + super(CrossEntropyLoss, self).__init__() + self.cross_entropy = P.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean() + self.one_hot = P.OneHot() + self.one = Tensor(1.0, mstype.float32) + self.zero = Tensor(0.0, mstype.float32) + + def construct(self, logits, label): + label = self.one_hot(label, F.shape(logits)[1], self.one, self.zero) + loss = self.cross_entropy(logits, label)[0] + loss = self.mean(loss, (-1,)) + return loss + + +if __name__ == '__main__': + if args_opt.do_eval: + context.set_context(enable_hccl=False) + else: + if args_opt.run_distribute: + context.set_context(enable_hccl=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL) + auto_parallel_context().set_all_reduce_fusion_split_indices([140]) + init() + else: + context.set_context(enable_hccl=False) + + context.set_context(mode=context.GRAPH_MODE) + epoch_size = args_opt.epoch_size + net = resnet50(args_opt.batch_size, args_opt.num_classes) + loss = CrossEntropyLoss() + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) + + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + if args_opt.do_train: + dataset = create_dataset(epoch_size) + batch_num = dataset.get_dataset_size() + config_ck = CheckpointConfig(save_checkpoint_steps=batch_num * 5, keep_checkpoint_max=10) + ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10", directory="./", config=config_ck) + loss_cb = LossMonitor() + model.train(epoch_size, dataset, callbacks=[ckpoint_cb, loss_cb]) + + if args_opt.do_eval: + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + eval_dataset = create_dataset(1, training=False) + res = model.eval(eval_dataset) + print("result: ", res) diff --git a/tests/st/tbe_networks/run_data_parallel.sh b/tests/st/tbe_networks/run_data_parallel.sh new file mode 100755 index 00000000000..4ac6a6924ff --- /dev/null +++ b/tests/st/tbe_networks/run_data_parallel.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +set -e +BASE_PATH=$(cd "$(dirname $0)"; pwd) +export DEVICE_NUM=2 +export RANK_SIZE=$DEVICE_NUM +ulimit -n 65535 +export DISTRIBUTION_FILE=$BASE_PATH/tdt${DEVICE_NUM}p/tdt_ +export MINDSPORE_HCCL_CONFIG_PATH=$BASE_PATH/hccl_${DEVICE_NUM}p.json + +for((i=0;i<$DEVICE_NUM;i++)) +do + rm -rf ./dataparallel$i + mkdir ./dataparallel$i + cp *.py ./dataparallel$i + cp -r kernel_meta ./dataparallel$i + cd ./dataparallel$i + export RANK_ID=$i + export DEVICE_ID=$i + echo "start training for device $i" + env > env$i.log + python resnet_cifar.py --run_distribute=1 --device_num=$DEVICE_NUM --epoch_size=10 >log 2>&1 & + cd ../ +done \ No newline at end of file diff --git a/tests/st/tbe_networks/tdt2p/tdt_0.json b/tests/st/tbe_networks/tdt2p/tdt_0.json new file mode 100644 index 00000000000..de0796c5a81 --- /dev/null +++ b/tests/st/tbe_networks/tdt2p/tdt_0.json @@ -0,0 +1,7 @@ +{ + "deviceNum":2, + "deviceId":0, + "shardConfig":"RANDOM", + "shuffle":"ON", + "seed": 0 +} diff --git a/tests/st/tbe_networks/tdt2p/tdt_1.json b/tests/st/tbe_networks/tdt2p/tdt_1.json new file mode 100644 index 00000000000..9436c9293fb --- /dev/null +++ b/tests/st/tbe_networks/tdt2p/tdt_1.json @@ -0,0 +1,7 @@ +{ + "deviceNum":2, + "deviceId":1, + "shardConfig":"RANDOM", + "shuffle":"ON", + "seed": 0 +} diff --git a/tests/st/tbe_networks/test_resnet_cifar_8p.py b/tests/st/tbe_networks/test_resnet_cifar_8p.py new file mode 100644 index 00000000000..6e83f4180e4 --- /dev/null +++ b/tests/st/tbe_networks/test_resnet_cifar_8p.py @@ -0,0 +1,213 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore import context +import mindspore.common.dtype as mstype +import os +import numpy as np +import mindspore.ops.functional as F +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, Callback +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore.dataset as ds +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore.communication.management import init +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from resnet import resnet50 +import random +from multiprocessing import Process, Queue +from multiprocessing import Pool +import time + +random.seed(1) +np.random.seed(1) +ds.config.set_seed(1) + +MINDSPORE_HCCL_CONFIG_PATH = "/home/workspace/mindspore_config/hccl/rank_table_8p.json" +data_home = "/home/workspace/mindspore_dataset" + + +def create_dataset(repeat_num=1, training=True, batch_size=32, rank_id=0, rank_size=1, + enable_hccl=False): + data_dir = data_home + "/cifar-10-batches-bin" + if not training: + data_dir = data_home + "/cifar-10-verify-bin" + data_set = ds.Cifar10Dataset(data_dir) + + if enable_hccl: + rank_id = rank_id + rank_size = rank_size + data_set = ds.Cifar10Dataset( + data_dir, num_shards=rank_size, shard_id=rank_id) + + resize_height = 224 + resize_width = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + random_crop_op = vision.RandomCrop( + (32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT + random_horizontal_op = vision.RandomHorizontalFlip() + # interpolation default BILINEAR + resize_op = vision.Resize((resize_height, resize_width)) + rescale_op = vision.Rescale(rescale, shift) + normalize_op = vision.Normalize( + (0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) + changeswap_op = vision.HWC2CHW() + type_cast_op = C.TypeCast(mstype.int32) + + c_trans = [] + if training: + c_trans = [random_crop_op, random_horizontal_op] + c_trans += [resize_op, rescale_op, normalize_op, + changeswap_op] + + # apply map operations on images + data_set = data_set.map(input_columns="label", operations=type_cast_op) + data_set = data_set.map(input_columns="image", operations=c_trans) + + # apply shuffle operations + data_set = data_set.shuffle(buffer_size=1000) + + # apply batch operations + data_set = data_set.batch(batch_size=batch_size, drop_remainder=True) + + # apply repeat operations + data_set = data_set.repeat(repeat_num) + + return data_set + + +class CrossEntropyLoss(nn.Cell): + def __init__(self): + super(CrossEntropyLoss, self).__init__() + self.cross_entropy = P.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean() + self.one_hot = P.OneHot() + self.one = Tensor(1.0, mstype.float32) + self.zero = Tensor(0.0, mstype.float32) + + def construct(self, logits, label): + label = self.one_hot(label, F.shape(logits)[1], self.one, self.zero) + loss = self.cross_entropy(logits, label)[0] + loss = self.mean(loss, (-1,)) + return loss + +class LossGet(Callback): + def __init__(self, per_print_times=1): + super(LossGet, self).__init__() + if not isinstance(per_print_times, int) or per_print_times < 0: + raise ValueError("print_step must be int and >= 0.") + self._per_print_times = per_print_times + self._loss = 0.0 + + def step_end(self, run_context): + cb_params = run_context.original_args() + loss = cb_params.net_outputs + + if isinstance(loss, (tuple, list)): + if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray): + loss = loss[0] + + if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray): + loss = np.mean(loss.asnumpy()) + + cur_step_in_epoch = (cb_params.cur_step_num - 1) % cb_params.batch_num + 1 + + if isinstance(loss, float) and (np.isnan(loss) or np.isinf(loss)): + raise ValueError("epoch: {} step: {}. Invalid loss, terminating training." + .format(cb_params.cur_epoch_num, cur_step_in_epoch)) + if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0: + self._loss = loss + print("epoch: %s step: %s, loss is %s" % (cb_params.cur_epoch_num, cur_step_in_epoch, loss)) + + def get_loss(self): + return self._loss + +def train_process(q, device_id, epoch_size, num_classes, device_num, batch_size, enable_hccl): + os.system("mkdir " + str(device_id)) + os.chdir(str(device_id)) + context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", save_graphs=False) + context.set_context(enable_task_sink=True, device_id=device_id) + context.set_context(enable_loop_sink=True) + context.set_context(enable_mem_reuse=True) + context.set_context(enable_hccl=enable_hccl) + os.environ['MINDSPORE_HCCL_CONFIG_PATH'] = MINDSPORE_HCCL_CONFIG_PATH + os.environ['RANK_ID'] = str(device_id) + os.environ['RANK_SIZE'] = str(device_num) + if enable_hccl: + context.set_auto_parallel_context( + device_num=device_num, parallel_mode=ParallelMode.DATA_PARALLEL) + auto_parallel_context().set_all_reduce_fusion_split_indices([140]) + init() + context.set_context(mode=context.GRAPH_MODE) + net = resnet50(batch_size, num_classes) + loss = CrossEntropyLoss() + opt = Momentum(filter(lambda x: x.requires_grad, + net.get_parameters()), 0.01, 0.9) + + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + dataset = create_dataset(epoch_size, training=True, + batch_size=batch_size, rank_id=device_id, rank_size=device_num, + enable_hccl=enable_hccl) + batch_num = dataset.get_dataset_size() + loss_cb = LossGet() + model.train(epoch_size, dataset, callbacks=[loss_cb]) + q.put(loss_cb.get_loss()) + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_single +def test_resnet_cifar_8p(): + q = Queue() + device_num = 8 + epoch_size = 1 + num_classes = 10 + batch_size = 32 + enable_hccl = True + process = [] + for i in range(device_num): + device_id = i + process.append(Process(target=train_process, + args=(q, device_id, epoch_size, num_classes, device_num, batch_size, enable_hccl))) + + for i in range(device_num): + process[i].start() + + print("Waiting for all subprocesses done...") + + for i in range(device_num): + process[i].join() + + loss = 0.0 + for i in range(device_num): + loss += q.get() + loss = loss/device_num + + for i in range(device_num): + os.system("rm -rf " + str(i)) + print("End training...") + assert(loss < 2.0) diff --git a/tests/train_step_wrap.py b/tests/train_step_wrap.py new file mode 100644 index 00000000000..7289c010044 --- /dev/null +++ b/tests/train_step_wrap.py @@ -0,0 +1,138 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +train step wrap +""" +import mindspore.nn as nn +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore import Parameter, ParameterTuple + + +run_opt = C.MultitypeFuncGraph("run_opt") + +# pylint: disable=unused-argument +@run_opt.register("Function", "Int", "Number", "Number", + "Tensor", "Tensor", "Tensor") +def tensor_run_opt(opt, iterator, learning_rate, momentum, + gradient, variable, moment): + success = True + new_weight = opt(gradient, moment, variable, learning_rate, momentum) + success = F.depend(success, P.Assign()(variable, new_weight)) + return success + + +class OptimizerByMomentum(nn.Cell): + """ + OptimizerByMomentum definition + """ + # list of tensor + def __init__(self, weights): + super(OptimizerByMomentum, self).__init__() + self.learning_rate = Parameter(0.1, name="learning_rate") + self.momentum = Parameter(0.05, name="momentum") + self.iter = Parameter(0, name="iter") + + self.weights = weights + self.moments = weights.clone(prefix="moments", init='zeros') + + self.hyper_map = C.HyperMap() + self.opt = P.ApplyMomentum() + + def construct(self, grads): + success = True + weights = self.weights + moments = self.moments + success = self.hyper_map( + F.partial(run_opt, self.opt, self.iter, + self.learning_rate, self.momentum), grads, weights, moments) + # self.learning_rate = updata_lr(self.learning_rate, self.momentum) + return success + +class TrainStepWrap(nn.Cell): + """ + TrainStepWrap definition + """ + def __init__(self, network): + super(TrainStepWrap, self).__init__() + self.network = network + self.network.set_train() + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = OptimizerByMomentum(self.weights) + self.hyper_map = C.HyperMap() + self.grad = C.GradOperation('grad', get_by_list=True) + + def construct(self, x, label): + weights = self.weights + grads = self.grad(self.network, weights)(x, label) + return self.optimizer(grads) + +class NetWithLossClass(nn.Cell): + """ + NetWithLossClass definition + """ + def __init__(self, network): + super(NetWithLossClass, self).__init__(auto_prefix=False) + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + +def train_step_with_loss_warp(network): + return TrainStepWrap(NetWithLossClass(network)) + + +class TrainStepWrap2(nn.Cell): + """ + TrainStepWrap2 definition + """ + def __init__(self, network, sens): + super(TrainStepWrap2, self).__init__() + self.network = network + self.network.set_train() + self.weights = ParameterTuple(network.get_parameters()) + self.optimizer = OptimizerByMomentum(self.weights) + self.hyper_map = C.HyperMap() + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + + def construct(self, x): + weights = self.weights + grads = self.grad(self.network, weights)(x, self.sens) + return self.optimizer(grads) + +def train_step_with_sens(network, sens): + return TrainStepWrap2(network, sens) + +class TrainStepWrapWithoutOpt(nn.Cell): + """ + TrainStepWrapWithoutOpt definition + """ + def __init__(self, network): + super(TrainStepWrapWithoutOpt, self).__init__() + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.grad = C.GradOperation('grad', get_by_list=True) + + def construct(self, x, label): + grads = self.grad(self.network, self.weights)(x, label) + return grads + +def train_step_without_opt(network): + return TrainStepWrapWithoutOpt(NetWithLossClass(network)) diff --git a/tests/ut/CMakeLists.txt b/tests/ut/CMakeLists.txt new file mode 100644 index 00000000000..4acbccfc072 --- /dev/null +++ b/tests/ut/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory("cpp") diff --git a/tests/ut/__init__.py b/tests/ut/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt new file mode 100644 index 00000000000..5f4bd41b3b5 --- /dev/null +++ b/tests/ut/cpp/CMakeLists.txt @@ -0,0 +1,150 @@ +message("build ut testcases...") + +# virtual project for common include and library file path. +project(ut) + +set(PROJECT_DIR "${PROJECT_SOURCE_DIR}/../../..") +add_compile_definitions(ENABLE_DUMP_E2E) +if(ENABLE_DUMP_IR) + add_compile_definitions(ENABLE_DUMP_IR) +endif(ENABLE_DUMP_IR) +if(ENABLE_D) + add_compile_definitions(ENABLE_D) +endif() + +#add python lib and include for all ut executables; +message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}") +message("PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") +include_directories(${PYTHON_INCLUDE_DIRS}) +include_directories(${MS_CCSRC_PATH}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/stub/runtime/) +include_directories(${CMAKE_BINARY_DIR}) +include_directories(${CUDA_INCLUDE_DIRS}) +MESSAGE("check ut_test ${CMAKE_BINARY_DIR}") + +link_directories(${MS_CCSRC_BUILD_PATH}) + +if(ENABLE_MINDDATA) + add_definitions(-D ENABLE_MINDRECORD) + add_definitions(-D ENABLE_MINDDATA) + link_directories(${MS_CCSRC_BUILD_PATH}/dataset) + link_directories(${MS_CCSRC_BUILD_PATH}/mindrecord) +endif() +# fetch ut test files +if(ENABLE_MINDDATA) + file(GLOB_RECURSE UT_SRCS ./*.cc) +else() + file(GLOB_RECURSE TEMP_UT_SRCS ./*.cc) + foreach(OBJ ${TEMP_UT_SRCS}) + if (NOT ${OBJ} MATCHES "./dataset/" AND NOT ${OBJ} MATCHES "./mindrecord/") + list(APPEND UT_SRCS ${OBJ}) + endif() + endforeach () +endif() + +file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "../../../mindspore/ccsrc/ir/*.cc" + "../../../mindspore/ccsrc/common/*.cc" + "../../../mindspore/ccsrc/utils/*.cc" + "../../../mindspore/ccsrc/parallel/*.cc" + "../../../mindspore/ccsrc/pipeline/parse/*.cc" + "../../../mindspore/ccsrc/pipeline/static_analysis/*.cc" + "../../../mindspore/ccsrc/pipeline/pipeline.cc" + "../../../mindspore/ccsrc/pipeline/resource.cc" + "../../../mindspore/ccsrc/pipeline/pass.cc" + "../../../mindspore/ccsrc/pipeline/action.cc" + "../../../mindspore/ccsrc/pipeline/validator.cc" + "../../../mindspore/ccsrc/pipeline/remove_value_node_dup.cc" + "../../../mindspore/ccsrc/optimizer/*.cc" + "../../../mindspore/ccsrc/debug/*.cc" + "../../../mindspore/ccsrc/operator/*.cc" + "../../../mindspore/ccsrc/transform/*.cc" + "../../../mindspore/ccsrc/session/anf_runtime_algorithm.cc" + "../../../mindspore/ccsrc/session/ascend_session.cc" + "../../../mindspore/ccsrc/session/kernel_graph.cc" + "../../../mindspore/ccsrc/session/session_basic.cc" + "../../../mindspore/ccsrc/session/session_factory.cc" + "../../../mindspore/ccsrc/vm/*.cc" + "../../../mindspore/ccsrc/pynative/*.cc" + "../../../mindspore/ccsrc/pybind_api/*.cc" + "../../../mindspore/ccsrc/kernel/akg/*.cc" + "../../../mindspore/ccsrc/kernel/kash/*.cc" + "../../../mindspore/ccsrc/kernel/cce/*.cc" + "../../../mindspore/ccsrc/kernel/mng/*.cc" + "../../../mindspore/ccsrc/kernel/hccl/*.cc" + "../../../mindspore/ccsrc/kernel/kernel_query.cc" + "../../../mindspore/ccsrc/kernel/kernel_build_info.cc" + "../../../mindspore/ccsrc/pre_activate/ascend/*.cc" + "../../../mindspore/ccsrc/pre_activate/common/*.cc" + "../../../mindspore/ccsrc/pre_activate/gpu/*.cc" + "../../../mindspore/ccsrc/pre_activate/mem_reuse/*.cc" + "../../../mindspore/ccsrc/pre_activate/pass/*.cc" + "../../../mindspore/ccsrc/kernel/aicpu/aicpu_kernel_metadata.cc" + "../../../mindspore/ccsrc/kernel/mng/rt_kernel_info.cc" + "../../../mindspore/ccsrc/kernel/common_utils.cc" + "../../../mindspore/ccsrc/kernel/oplib/*.cc" + "../../../mindspore/ccsrc/kernel/tbe/*.cc" + "../../../mindspore/ccsrc/device/kernel_runtime.cc" + "../../../mindspore/ccsrc/device/kernel_runtime_manager.cc" + "../../../mindspore/ccsrc/device/kernel_info.cc" + "../../../mindspore/ccsrc/device/ascend/profiling/*.cc" + "../../../mindspore/ccsrc/device/ascend/kernel_select_ascend.cc" + "../../../mindspore/ccsrc/device/convert_tensor_utils.cc" + "../../../mindspore/ccsrc/device/ascend/kernel_build_ascend.cc" + "../../../mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc" + "../../../mindspore/ccsrc/device/ascend/ascend_device_address.cc" + "../../../mindspore/ccsrc/device/ascend/ascend_memory_allocator.cc" + "../../../mindspore/ccsrc/predict/generator/utils/ir_model_util.cc" + "../../../mindspore/ccsrc/predict/predict.cc" + "../../../mindspore/ccsrc/predict/converter/*.cc" + "../../../mindspore/ccsrc/predict/converter/attr_utils/*.cc" + "../../../mindspore/ccsrc/predict/converter/lite_model/*.cc" + "../../../mindspore/ccsrc/predict/converter/lite_model/operations/*.cc" + ) + +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/dump_proto.cc") +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc") +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/utils/anf_ir.pb.cc") +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/utils/node_strategy.pb.cc") + +file(GLOB_RECURSE UT_SUTB_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "stub/aicpu/*.cc" + "stub/cce/*.cc" + "stub/tdt/*.cc" + "stub/tasksink/*.cc" + "stub/device/*.cc" + "stub/kernel/*.cc" + "stub/runtime/*.cc" + "stub/anf_ir/*.cc" + "stub/profiling/*.cc" + "stub/parallel_strategy_checkpoint/*.cc" + "stub/hccl/*.cc" + "stub/ge/*.cc" + ) + +add_executable(ut_tests ${UT_SRCS} ${MINDSPORE_SRC_LIST} ${UT_SUTB_SRC_LIST}) + +if (ENABLE_GE) + if(ENABLE_TRAIN) + target_link_libraries(ut_tests PRIVATE graph ge_client_train) + else() + target_link_libraries(ut_tests PRIVATE graph ge_client) + endif() + + target_link_libraries(mindspore PRIVATE tsdclient) +endif() + +if (CMAKE_SYSTEM_NAME MATCHES "Linux") + target_link_libraries(ut_tests PRIVATE mindspore::gtest mindspore_gvar ${PYTHON_LIBRARIES} pthread util dl) + if (ENABLE_MINDDATA) + target_link_libraries(ut_tests PRIVATE _c_dataengine _c_mindrecord) + endif() +else() + target_link_libraries(ut_tests PRIVATE mindspore::gtest mindspore_gvar ${PYTHON_LIBRARIES}) +endif() +if (USE_GLOG) + target_link_libraries(ut_tests PRIVATE mindspore::glog) +endif() + +target_link_libraries(ut_tests PRIVATE securec graph) diff --git a/tests/ut/cpp/__init__.py b/tests/ut/cpp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/common/backend_common_test.cc b/tests/ut/cpp/common/backend_common_test.cc new file mode 100644 index 00000000000..060b170a8c5 --- /dev/null +++ b/tests/ut/cpp/common/backend_common_test.cc @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" + +#include +#include +#include + +#include "utils/log_adapter.h" +#include "operator/ops.h" +#include "debug/anf_ir_dump.h" +#include "session/ascend_session.h" +#include "pipeline/resource.h" +#include "pipeline/action.h" +#include "ir/anf.h" +#include "ir/manager.h" + +namespace mindspore { +namespace { +std::vector GetCNodeList(const FuncGraphPtr &func_graph) { + std::vector nodes = TopoSort(func_graph->get_return()); + std::vector lst; + for (auto &node : nodes) { + MS_LOG(INFO) << "nodes: " << node->DebugString(10); + if (node->isa() && IsValueNode(node->cast()->input(0)) && + !IsPrimitiveCNode(node, prim::kPrimReturn)) { + MS_LOG(INFO) << "push in anf_node list: " << node->DebugString(10); + lst.push_back(node); + } + } + return lst; +} +} // namespace + +bool BackendCommon::CheckEqualGraph(const FuncGraphPtr &a, const FuncGraphPtr &b) { + FuncGraphPairMapEquiv equiv_graph_; + NodeMapEquiv equiv_node_; + return Isomorphic(a, b, &equiv_graph_, &equiv_node_); +} + +std::shared_ptr BackendCommon::GetKernelGraph(const FuncGraphPtr &func_graph, + const AbstractBasePtrList &args_spec_list, + bool need_infer) { + FuncGraphPtr inferred_graph = func_graph; + if (need_infer) { + inferred_graph = GetFuncGraph(func_graph, args_spec_list); + } + AnfNodePtrList applies = GetCNodeList(inferred_graph); + AnfNodePtrList ins = inferred_graph->parameters(); + AnfNodePtrList outs = {inferred_graph->get_return()->input(1)}; + auto session = std::make_shared(); + session->Init(0); + auto kernel_graph = session->ConstructKernelGraph(applies, outs); + kernel_graph->SetExecOrderByDefault(); + return kernel_graph; +} + +FuncGraphPtr BackendCommon::GetFuncGraph(const FuncGraphPtr &func_graph, const AbstractBasePtrList &args_spec_list) { + if (func_graph->manager() == nullptr) { + std::vector graphs{func_graph}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(func_graph); + } + // Renormalize func_graph to infer and set shape and type information. + pipeline::ResourcePtr resource_ = std::make_shared(); + return pipeline::Renormalize(resource_, func_graph, args_spec_list); +} +} // namespace mindspore diff --git a/tests/ut/cpp/common/backend_common_test.h b/tests/ut/cpp/common/backend_common_test.h new file mode 100644 index 00000000000..fb3334182af --- /dev/null +++ b/tests/ut/cpp/common/backend_common_test.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_CPP_COMMON_UT_BACKEND_COMMON_H_ +#define TESTS_UT_CPP_COMMON_UT_BACKEND_COMMON_H_ +#include "common/common_test.h" +#include "utils/context/ms_context.h" +#include "session/kernel_graph.h" + +namespace mindspore { +class BackendCommon : public UT::Common { + public: + BackendCommon() = default; + ~BackendCommon() override = default; + virtual bool CheckEqualGraph(const FuncGraphPtr &a, const FuncGraphPtr &b); + virtual std::shared_ptr GetKernelGraph(const FuncGraphPtr &func_graph, + const AbstractBasePtrList &args_spec_list, + bool need_infer = true); + virtual FuncGraphPtr GetFuncGraph(const FuncGraphPtr &func_graph, const AbstractBasePtrList &args_spec_list); +}; +} // namespace mindspore +#endif // TESTS_UT_CPP_COMMON_UT_BACKEND_COMMON_H_ diff --git a/tests/ut/cpp/common/common_test.cc b/tests/ut/cpp/common/common_test.cc new file mode 100644 index 00000000000..e4287a2458a --- /dev/null +++ b/tests/ut/cpp/common/common_test.cc @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "utils/log_adapter.h" + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif + +namespace UT { + +void Common::SetUpTestCase() {} + +void Common::TearDownTestCase() {} + +void Common::SetUp() {} + +void Common::TearDown() {} + +} // namespace UT + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif diff --git a/tests/ut/cpp/common/common_test.h b/tests/ut/cpp/common/common_test.h new file mode 100644 index 00000000000..a293584d7b9 --- /dev/null +++ b/tests/ut/cpp/common/common_test.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_COMMON_UT_COMMON_H_ +#define TESTS_UT_COMMON_UT_COMMON_H_ + +#include "gtest/gtest.h" +namespace UT { +class Common : public testing::Test { + public: + // TestCase only enter once + static void SetUpTestCase(); + static void TearDownTestCase(); + + // every TEST_F macro will enter one + virtual void SetUp(); + virtual void TearDown(); +}; +} // namespace UT +#endif // TESTS_UT_COMMON_UT_COMMON_H_ diff --git a/tests/ut/cpp/common/py_func_graph_fetcher.cc b/tests/ut/cpp/common/py_func_graph_fetcher.cc new file mode 100644 index 00000000000..78d5b682f7d --- /dev/null +++ b/tests/ut/cpp/common/py_func_graph_fetcher.cc @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/py_func_graph_fetcher.h" + +namespace UT { +namespace python_adapter = mindspore::parse::python_adapter; +void InitPythonPath() { + python_adapter::set_python_env_flag(false); + python_adapter::SetPythonPath("../../../../"); + python_adapter::SetPythonPath("../../../../tests/ut/python_input"); + + python_adapter::SetPythonPath("../../../"); + python_adapter::SetPythonPath("../../../tests/ut/python_input"); +} + +} // namespace UT diff --git a/tests/ut/cpp/common/py_func_graph_fetcher.h b/tests/ut/cpp/common/py_func_graph_fetcher.h new file mode 100644 index 00000000000..3c9de9f971a --- /dev/null +++ b/tests/ut/cpp/common/py_func_graph_fetcher.h @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_COMMON_PY_FUNC_GRAPH_FETCHER_H_ +#define TESTS_UT_COMMON_PY_FUNC_GRAPH_FETCHER_H_ + +#include +#include +#include "ir/anf.h" +#include "ir/manager.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/parse.h" +#include "./common.h" + +namespace UT { + +void InitPythonPath(); + +class PyFuncGraphFetcher { + public: + explicit PyFuncGraphFetcher(std::string model_path, bool doResolve = false) + : model_path_(model_path), doResolve_(doResolve) { + InitPythonPath(); + } + void SetDoResolve(bool doResolve = true) { doResolve_ = doResolve; } + + // The return of python function of "func_name" should be py::function. + // step 1. Call the function user input + // step 2. Parse the return "fn" + template + mindspore::FuncGraphPtr CallAndParseRet(std::string func_name, T... args) { + try { + py::function fn = mindspore::parse::python_adapter::CallPyFn(model_path_.c_str(), func_name.c_str(), args...); + mindspore::FuncGraphPtr func_graph = mindspore::parse::ParsePythonCode(fn); + if (doResolve_) { + std::shared_ptr manager = mindspore::Manage(func_graph, false); + mindspore::parse::python_adapter::set_use_signature_in_resolve(false); + mindspore::parse::ResolveAll(manager); + } + return func_graph; + } catch (py::error_already_set& e) { + MS_LOG(ERROR) << "Call and parse fn failed!!! error:" << e.what(); + return nullptr; + } catch (...) { + MS_LOG(ERROR) << "Call fn failed!!!"; + return nullptr; + } + } + + // Fetch python function then parse to graph + mindspore::FuncGraphPtr operator()(std::string func_name, std::string model_path = "") { + try { + std::string path = model_path_; + if ("" != model_path) { + path = model_path; + } + py::function fn = mindspore::parse::python_adapter::GetPyFn(path.c_str(), func_name.c_str()); + mindspore::FuncGraphPtr func_graph = mindspore::parse::ParsePythonCode(fn); + if (doResolve_) { + std::shared_ptr manager = mindspore::Manage(func_graph, false); + mindspore::parse::ResolveAll(manager); + } + return func_graph; + } catch (py::error_already_set& e) { + MS_LOG(ERROR) << "get fn failed!!! error:" << e.what(); + return nullptr; + } catch (...) { + MS_LOG(ERROR) << "get fn failed!!!"; + return nullptr; + } + } + + private: + std::string model_path_; + bool doResolve_; +}; + +} // namespace UT +#endif // TESTS_UT_COMMON_PY_FUNC_GRAPH_FETCHER_H_ diff --git a/tests/ut/cpp/common/test_main.cc b/tests/ut/cpp/common/test_main.cc new file mode 100644 index 00000000000..57a2a49f537 --- /dev/null +++ b/tests/ut/cpp/common/test_main.cc @@ -0,0 +1,27 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "pipeline/pipeline.h" +#include "pipeline/resource.h" + +GTEST_API_ int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + int ret = RUN_ALL_TESTS(); + mindspore::pipeline::ClearResAtexit(); + return ret; +} diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt new file mode 100644 index 00000000000..0da470ac894 --- /dev/null +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -0,0 +1,81 @@ +include(GoogleTest) + +SET(DE_UT_SRCS + common/common.cc + common/cvop_common.cc + batch_op_test.cc + bit_functions_test.cc + storage_container_test.cc + treap_test.cc + interrupt_test.cc + image_folder_op_test.cc + buddy_test.cc + arena_test.cc + btree_test.cc + center_crop_op_test.cc + change_mode_test.cc + channel_swap_test.cc + circular_pool_test.cc + client_config_test.cc + connector_test.cc + datatype_test.cc + decode_op_test.cc + execution_tree_test.cc + global_context_test.cc + main_test.cc + map_op_test.cc + mind_record_op_test.cc + memory_pool_test.cc + normalize_op_test.cc + one_hot_op_test.cc + path_test.cc + project_op_test.cc + queue_test.cc + random_crop_op_test.cc + random_crop_decode_resizeOp_test.cc + random_crop_and_resize_op_test.cc + random_color_adjust_op_test.cc + random_horizontal_flip_op_test.cc + random_resize_op_test.cc + random_rotation_op_test.cc + random_vertical_flip_op_test.cc + rename_op_test.cc + repeat_op_test.cc + rescale_op_test.cc + resize_bilinear_op_test.cc + resize_op_test.cc + shuffle_op_test.cc + stand_alone_samplers_test.cc + status_test.cc + storage_op_test.cc + task_manager_test.cc + tensor_test.cc + tensorshape_test.cc + tfReader_op_test.cc + to_float16_op_test.cc + type_cast_op_test.cc + zip_op_test.cc + random_resize_op_test.cc + subset_random_sampler_test.cc + weighted_random_sampler_test.cc + mnist_op_test.cc + manifest_op_test.cc + voc_op_test.cc + cifar_op_test.cc + celeba_op_test.cc + ) + +add_executable(de_ut_tests ${DE_UT_SRCS}) + +set_target_properties(de_ut_tests PROPERTIES INSTALL_RPATH "$ORIGIN/../lib:$ORIGIN/../lib64") + +target_link_libraries(de_ut_tests PRIVATE _c_dataengine pybind11::embed ${GTEST_LIBRARY} ${SECUREC_LIBRARY} ${SLOG_LIBRARY}) + +gtest_discover_tests(de_ut_tests WORKING_DIRECTORY ${Project_DIR}/tests/dataset) + +install(TARGETS de_ut_tests + RUNTIME DESTINATION test) + +# For internal testing only. +install(DIRECTORY ${Project_DIR}/tests/dataset/data/ + DESTINATION test/data) diff --git a/tests/ut/cpp/dataset/README.md b/tests/ut/cpp/dataset/README.md new file mode 100644 index 00000000000..c7662d48021 --- /dev/null +++ b/tests/ut/cpp/dataset/README.md @@ -0,0 +1,3 @@ + +### Introduce of data +All of data used in this folder created by Huawei Technologies Co. diff --git a/tests/ut/cpp/dataset/arena_test.cc b/tests/ut/cpp/dataset/arena_test.cc new file mode 100644 index 00000000000..4b5f50c47f9 --- /dev/null +++ b/tests/ut/cpp/dataset/arena_test.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "dataset/util/arena.h" +#include "common/common.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; + +class MindDataTestArena : public UT::Common { + public: + MindDataTestArena() {} +}; + + +TEST_F(MindDataTestArena, TestALLFunction) { + std::shared_ptr mp; + Status rc = Arena::CreateArena(&mp); + ASSERT_TRUE(rc.IsOk()); + std::shared_ptr arena = std::dynamic_pointer_cast(mp); + std::vector v; + + srand(time(NULL)); + for (int i = 0; i < 1000; i++) { + uint64_t sz = rand() % 1048576; + void *ptr = nullptr; + ASSERT_TRUE(mp->Allocate(sz, &ptr)); + v.push_back(ptr); + } + for (int i = 0; i < 1000; i++) { + mp->Deallocate(v.at(i)); + } + std::cout << *mp; +} diff --git a/tests/ut/cpp/dataset/batch_op_test.cc b/tests/ut/cpp/dataset/batch_op_test.cc new file mode 100644 index 00000000000..504cac51e54 --- /dev/null +++ b/tests/ut/cpp/dataset/batch_op_test.cc @@ -0,0 +1,327 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "dataset/core/client.h" +#include "common/common.h" +#include "common/utils.h" +#include "gtest/gtest.h" +#include "dataset/core/global_context.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" +#include "securec.h" +#include "dataset/util/status.h" + +namespace common = mindspore::common; +namespace de = mindspore::dataset; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestBatchOp : public UT::DatasetOpTesting { + protected: + +}; + + +std::shared_ptr Batch(int32_t batch_size = 1, bool drop = false, int rows_per_buf = 2) { + Status rc; + std::shared_ptr op; + rc = de::BatchOp::Builder(batch_size).SetDrop(drop).Build(&op); + EXPECT_TRUE(rc.IsOk()); + return op; +} + +std::shared_ptr Repeat(int repeat_cnt = 1) { + de::RepeatOp::Builder builder(repeat_cnt); + std::shared_ptr op; + Status rc = builder.Build(&op); + EXPECT_TRUE(rc.IsOk()); + return op; +} + +std::shared_ptr Storage(std::string schema, int rows_per_buf = 2, int num_works = 8) { + std::shared_ptr so; + de::StorageOp::Builder builder; + builder.SetDatasetFilesDir(schema).SetRowsPerBuffer(rows_per_buf).SetNumWorkers(num_works); + Status rc = builder.Build(&so); + return so; +} + +std::shared_ptr Build(std::vector> ops) { + std::shared_ptr tree = std::make_shared(); + for (int i = 0; i < ops.size(); i++) { + tree->AssociateNode(ops[i]); + if (i > 0) { + ops[i]->AddChild(ops[i - 1]); + } + if (i == ops.size() - 1) { + tree->AssignRoot(ops[i]); + } + } + return tree; +} + +TEST_F(MindDataTestBatchOp, TestSimpleBatch) { + std::string schema_file = datasets_root_path_ + "/testBatchDataset"; + bool success = false; + auto tree = Build({Storage(schema_file), Batch(12)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + } else { + int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807}; + de::DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + std::shared_ptr t; + rc = de::Tensor::CreateTensor(&t, + TensorImpl::kFlexible, de::TensorShape({12, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) payload); + EXPECT_TRUE(rc.IsOk()); + // verify the actual data in Tensor is correct + EXPECT_EQ(*t == *tensor_map["col_sint64"], true); + // change what's in Tensor and verify this time the data is incorrect1; + EXPECT_EQ(*t == *tensor_map["col_sint16"], false); + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + if (tensor_map.size() == 0) { + success = true; + } + } + EXPECT_EQ(success, true); +} + + +TEST_F(MindDataTestBatchOp, TestRepeatBatchDropTrue) { + std::string schema_file = datasets_root_path_ + "/testBatchDataset"; + bool success = false; + auto tree = Build({Storage(schema_file), Repeat(2), Batch(7, true, 99)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + } else { + int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807, + -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807}; + de::DatasetIterator di(tree); + std::shared_ptr t1, t2, t3; + rc = de::Tensor::CreateTensor(&t1, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) payload); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t2, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 7)); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t3, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 2)); + EXPECT_TRUE(rc.IsOk()); + + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true); // first call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true); // second call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t3 == *(tensor_map["col_sint64"]), true); // third call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + if (tensor_map.size() == 0) { + success = true; + } + } + EXPECT_EQ(success, true); +} + + +TEST_F(MindDataTestBatchOp, TestRepeatBatchDropFalse) { + std::string schema_file = datasets_root_path_ + "/testBatchDataset"; + bool success = false; + auto tree = Build({Storage(schema_file), Repeat(2), Batch(7, false, 99)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + } else { + int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807, + -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807}; + de::DatasetIterator di(tree); + std::shared_ptr t1, t2, t3, t4; + rc = de::Tensor::CreateTensor(&t1, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) payload); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t2, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 7)); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t3, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 2)); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t4, + TensorImpl::kFlexible, de::TensorShape({3, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 9)); + EXPECT_TRUE(rc.IsOk()); + + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true); // first call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true); // second call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t3 == *(tensor_map["col_sint64"]), true); // third call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t4 == *(tensor_map["col_sint64"]), true); // last call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + if (tensor_map.size() == 0) { + success = true; + } + } + EXPECT_EQ(success, true); +} + + +TEST_F(MindDataTestBatchOp, TestBatchDropFalseRepeat) { + std::string schema_file = datasets_root_path_ + "/testBatchDataset"; + bool success = false; + auto tree = Build({Storage(schema_file), Batch(7, false, 99), Repeat(2)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + } else { + int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807, + -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807}; + de::DatasetIterator di(tree); + std::shared_ptr t1, t2; + rc = de::Tensor::CreateTensor(&t1, + TensorImpl::kFlexible, de::TensorShape({7, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) payload); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t2, + TensorImpl::kFlexible, de::TensorShape({5, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 7)); + EXPECT_TRUE(rc.IsOk()); + + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true); // first call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true); // second call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true); // third call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true); // last call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + if (tensor_map.size() == 0) { + success = true; + } + } + EXPECT_EQ(success, true); +} + + +TEST_F(MindDataTestBatchOp, TestBatchDropTrueRepeat) { + std::string schema_file = datasets_root_path_ + "/testBatchDataset"; + bool success = false; + auto tree = Build({Storage(schema_file), Batch(5, true, 99), Repeat(2)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + } else { + int64_t payload[] = {-9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807, + -9223372036854775807 - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9223372036854775807}; + de::DatasetIterator di(tree); + std::shared_ptr t1, t2; + rc = de::Tensor::CreateTensor(&t1, + TensorImpl::kFlexible, de::TensorShape({5, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) payload); + EXPECT_TRUE(rc.IsOk()); + rc = de::Tensor::CreateTensor(&t2, + TensorImpl::kFlexible, de::TensorShape({5, 1}), + de::DataType(DataType::DE_INT64), + (unsigned char *) (payload + 5)); + EXPECT_TRUE(rc.IsOk()); + + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true); // first call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true); // second call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t1 == *(tensor_map["col_sint64"]), true); // third call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(*t2 == *(tensor_map["col_sint64"]), true); // last call to getNext() + + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + if (tensor_map.size() == 0) { + success = true; + } + } + EXPECT_EQ(success, true); +} diff --git a/tests/ut/cpp/dataset/bit_functions_test.cc b/tests/ut/cpp/dataset/bit_functions_test.cc new file mode 100644 index 00000000000..02b6a25f767 --- /dev/null +++ b/tests/ut/cpp/dataset/bit_functions_test.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/constants.h" +#include "common/common.h" + +using namespace mindspore::dataset; + +class MindDataTestBits : public UT::Common { + public: + MindDataTestBits() {} +}; + + +TEST_F(MindDataTestBits, Basics) { + uint32_t x = 0; // 00000 + BitSet(&x, 16); // 10000 + + ASSERT_TRUE(BitTest(x, 16)); // 10000 + ASSERT_FALSE(BitTest(x, 1)); // 00001 + ASSERT_FALSE(BitTest(x, 17)); // 10001 is failing + + + BitSet(&x, 1); // 10001 + ASSERT_TRUE(BitTest(x, 16)); // 10000 + ASSERT_TRUE(BitTest(x, 1)); // 00001 + ASSERT_TRUE(BitTest(x, 17)); // 10001 is failing + + BitClear(&x, 16); // 00001 + ASSERT_FALSE(BitTest(x, 16)); // 10000 + ASSERT_TRUE(BitTest(x, 1)); // 00001 +// ASSERT_FALSE(BitTest(x, 17)); // 10001 is failing + + BitSet(&x, 31); // 11111 + for (uint32_t i = 1; i < 32; i++) { + ASSERT_TRUE(BitTest(x, i)); + } + BitClear(&x, 31); // 00000 + for (uint32_t i = 1; i < 32; i++) { + ASSERT_FALSE(BitTest(x, i)); + } +} diff --git a/tests/ut/cpp/dataset/btree_test.cc b/tests/ut/cpp/dataset/btree_test.cc new file mode 100644 index 00000000000..993f8f465fa --- /dev/null +++ b/tests/ut/cpp/dataset/btree_test.cc @@ -0,0 +1,201 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "dataset/util/btree.h" +#include "dataset/util/auto_index.h" +#include "dataset/util/system_pool.h" +#include "dataset/util/task_manager.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +// For testing purposes, we will make the branching factor very low. +struct mytraits { + using slot_type = uint16_t; + + static const slot_type kLeafSlots = 6; + + static const slot_type kInnerSlots = 3; + + static const bool kAppendMode = false; + +}; + + +class MindDataTestBPlusTree : public UT::Common { + public: + MindDataTestBPlusTree() = default; +}; + +// Test serial insert. +TEST_F(MindDataTestBPlusTree, Test1) { + Allocator alloc(std::make_shared()); + BPlusTree, mytraits> btree(alloc); + Status rc; + for (int i = 0; i < 100; i++) { + uint64_t key = 2 * i; + std::ostringstream oss; + oss << "Hello World. I am " << key; + rc = btree.DoInsert(key, oss.str()); + EXPECT_TRUE(rc.IsOk()); + } + for (int i = 0; i < 100; i++) { + uint64_t key = 2 * i + 1; + std::ostringstream oss; + oss << "Hello World. I am " << key; + rc = btree.DoInsert(key, oss.str()); + EXPECT_TRUE(rc.IsOk()); + } + EXPECT_EQ(btree.size(), 200); + + // Test iterator + { + int cnt = 0; + auto it = btree.begin(); + uint64_t prev = it.key(); + ++it; + ++cnt; + while (it != btree.end()) { + uint64_t cur = it.key(); + std::string val = "Hello World. I am " + std::to_string(cur); + EXPECT_TRUE(prev < cur); + EXPECT_EQ(it.value(), val); + prev = cur; + ++it; + ++cnt; + } + EXPECT_EQ(cnt, 200); + // Now go backward + for (int i = 0; i < 10; i++) { + --it; + EXPECT_EQ(199 - i, it.key()); + } + } + + // Test nearch + { + MS_LOG(INFO) << "Locate key " << 100 << " Expect found."; + auto it = btree.Search(100); + EXPECT_FALSE(it == btree.cend()); + EXPECT_EQ(it.key(), 100); + EXPECT_EQ(it.value(), "Hello World. I am 100"); + MS_LOG(INFO) << "Locate key " << 300 << " Expect not found."; + it = btree.Search(300); + EXPECT_TRUE(it == btree.cend()); + } + + // Test duplicate key + { + rc = btree.DoInsert(100, "Expect error"); + EXPECT_EQ(rc, Status(StatusCode::kDuplicateKey)); + } +} + +// Test concurrent insert. +TEST_F(MindDataTestBPlusTree, Test2) { + Allocator alloc(std::make_shared()); + BPlusTree, mytraits> btree(alloc); + TaskGroup vg; + auto f = [&](int k) -> Status { + TaskManager::FindMe()->Post(); + for (int i = 0; i < 100; i++) { + uint64_t key = k * 100 + i; + std::ostringstream oss; + oss << "Hello World. I am " << key; + Status rc = btree.DoInsert(key, oss.str()); + EXPECT_TRUE(rc.IsOk()); + } + return Status::OK(); + }; + // Spawn two threads. One insert the odd numbers and the other insert the even numbers just like Test1 + for (int k = 0; k < 100; k++) { + vg.CreateAsyncTask("Concurrent Insert", std::bind(f, k)); + } + vg.join_all(); + EXPECT_EQ(btree.size(), 10000); + + // Test iterator + { + int cnt = 0; + auto it = btree.begin(); + uint64_t prev = it.key(); + ++it; + ++cnt; + while (it != btree.end()) { + uint64_t cur = it.key(); + std::string val = "Hello World. I am " + std::to_string(cur); + EXPECT_TRUE(prev < cur); + EXPECT_EQ(it.value(), val); + prev = cur; + ++it; + ++cnt; + } + EXPECT_EQ(cnt, 10000); + } + + // Test search + { + MS_LOG(INFO) << "Locating key from 0 to 9999. Expect found."; + for (int i = 0; i < 10000; i++) { + auto it = btree.Search(i); + bool eoS = (it == btree.cend()); + EXPECT_FALSE(eoS); + if (!eoS) { + EXPECT_EQ(it.key(), i); + std::string val = "Hello World. I am " + std::to_string(i); + EXPECT_EQ(it.value(), val); + } + } + MS_LOG(INFO) << "Locate key " << 10000 << ". Expect not found"; + auto it = btree.Search(10000); + EXPECT_TRUE(it == btree.cend()); + } + + // Test to retrieve key at certain position. + { + for (int i = 0; i < 10000; i++) { + int k = btree.KeyAtPos(i); + EXPECT_EQ(k, i); + } + } +} + +TEST_F(MindDataTestBPlusTree, Test3) { + Allocator alloc(std::make_shared()); + AutoIndexObj ai(alloc); + Status rc; + rc = ai.insert("Hello World"); + EXPECT_TRUE(rc.IsOk()); + ai.insert({"a", "b", "c"}); + EXPECT_TRUE(rc.IsOk()); + uint64_t min = ai.min_key(); + uint64_t max = ai.max_key(); + EXPECT_EQ(min, 1); + EXPECT_EQ(max, 4); + auto it = ai.Search(3); + EXPECT_EQ(it.value(), "b"); + MS_LOG(INFO) << "Dump all the values using [] operator."; + for (uint64_t i = min; i <= max; i++) { + std::cout << ai[i] << std::endl; + } +} diff --git a/tests/ut/cpp/dataset/celeba_op_test.cc b/tests/ut/cpp/dataset/celeba_op_test.cc new file mode 100644 index 00000000000..69314771a3b --- /dev/null +++ b/tests/ut/cpp/dataset/celeba_op_test.cc @@ -0,0 +1,167 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "common/common.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/celeba_op.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +std::shared_ptr Repeat(int repeat_cnt); + +std::shared_ptr Build(std::vector> ops); + +std::shared_ptr Celeba(int32_t num_workers, int32_t rows_per_buffer, int32_t queue_size, + const std::string &dir, int64_t num_samples = 0, + std::unique_ptr sampler = nullptr, bool decode = false, + const std::string &dataset_type="all") { + std::shared_ptr so; + CelebAOp::Builder builder; + Status rc = builder.SetNumWorkers(num_workers).SetCelebADir(dir).SetRowsPerBuffer(rows_per_buffer) + .SetOpConnectorSize(queue_size).SetSampler(std::move(sampler)).SetDecode(decode) + .SetNumSamples(num_samples).SetDatasetType(dataset_type).Build(&so); + return so; +} + +class MindDataTestCelebaDataset : public UT::DatasetOpTesting { +protected: +}; + +TEST_F(MindDataTestCelebaDataset, TestSequentialCeleba) { + std::string dir = datasets_root_path_ + "/testCelebAData/"; + uint32_t expect_labels[2][40] = {{0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,1,0,0,0,1,1,0,1,0,1,0,0,1}, + {0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}}; + uint32_t count = 0; + auto tree = Build({Celeba(16, 2, 32, dir)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tersor_map; + di.GetNextAsMap(&tersor_map); + EXPECT_TRUE(rc.IsOk()); + while (tersor_map.size() != 0) { + uint32_t label; + for (int index = 0; index < 40; index++) { + tersor_map["attr"]->GetItemAt(&label, {index}); + EXPECT_TRUE(expect_labels[count][index] == label); + } + count++; + di.GetNextAsMap(&tersor_map); + } + EXPECT_TRUE(count == 2); + } +} + +TEST_F(MindDataTestCelebaDataset, TestCelebaRepeat) { + std::string dir = datasets_root_path_ + "/testCelebAData/"; + uint32_t expect_labels[4][40] = {{0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,1,0,0,0,1,1,0,1,0,1,0,0,1}, + {0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}, + {0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,1,0,0,0,1,1,0,1,0,1,0,0,1}, + {0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}}; + uint32_t count = 0; + auto tree = Build({Celeba(16, 2, 32, dir), Repeat(2)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tersor_map; + di.GetNextAsMap(&tersor_map); + EXPECT_TRUE(rc.IsOk()); + while (tersor_map.size() != 0) { + uint32_t label; + for (int index = 0; index < 40; index++) { + tersor_map["attr"]->GetItemAt(&label, {index}); + EXPECT_TRUE(expect_labels[count][index] == label); + } + count++; + di.GetNextAsMap(&tersor_map); + } + EXPECT_TRUE(count == 4); + } +} + +TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) { + std::vector indices({1}); + std::unique_ptr sampler = mindspore::make_unique(indices); + uint32_t expect_labels[1][40] = {{0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}}; + std::string dir = datasets_root_path_ + "/testCelebAData/"; + uint32_t count = 0; + auto tree = Build({Celeba(16, 2, 32, dir, 0, std::move(sampler))}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tersor_map; + di.GetNextAsMap(&tersor_map); + EXPECT_TRUE(rc.IsOk()); + while (tersor_map.size() != 0) { + uint32_t label; + for (int index = 0; index < 40; index++) { + tersor_map["attr"]->GetItemAt(&label, {index}); + EXPECT_TRUE(expect_labels[count][index] == label); + } + count++; + di.GetNextAsMap(&tersor_map); + } + EXPECT_TRUE(count == 1); + } +} + +TEST_F(MindDataTestCelebaDataset, TestCelebaNumSamples) { + std::string dir = datasets_root_path_ + "/testCelebAData/"; + uint32_t count = 0; + auto tree = Build({Celeba(16, 2, 32, dir, 1)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tersor_map; + di.GetNextAsMap(&tersor_map); + EXPECT_TRUE(rc.IsOk()); + while (tersor_map.size() != 0) { + count++; + di.GetNextAsMap(&tersor_map); + } + EXPECT_TRUE(count == 1); + } +} diff --git a/tests/ut/cpp/dataset/center_crop_op_test.cc b/tests/ut/cpp/dataset/center_crop_op_test.cc new file mode 100644 index 00000000000..54c45c957e0 --- /dev/null +++ b/tests/ut/cpp/dataset/center_crop_op_test.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/center_crop_op.h" +#include "dataset/core/cv_tensor.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestCenterCropOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestCenterCropOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestCenterCropOp, TestOp) { + MS_LOG(INFO) << "Doing MindDataTestCenterCropOp::TestOp."; + std::shared_ptr output_tensor; + int het = 256; + int wid = 128; + std::unique_ptr op(new CenterCropOp(het, wid)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(het, output_tensor->shape()[0]); + EXPECT_EQ(wid, output_tensor->shape()[1]); + std::shared_ptr p = CVTensor::AsCVTensor(output_tensor); +} diff --git a/tests/ut/cpp/dataset/change_mode_test.cc b/tests/ut/cpp/dataset/change_mode_test.cc new file mode 100644 index 00000000000..52a4dbfb3e6 --- /dev/null +++ b/tests/ut/cpp/dataset/change_mode_test.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/change_mode_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestChangeModeOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestChangeModeOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestChangeModeOp, TestOp) { + MS_LOG(INFO) << "Doing MindDataTestChangeModeOp."; + + // Creating a Tensor + TensorShape s = input_tensor_->shape(); + int size_buffer = s[0] * s[1] * s[2]; + + std::unique_ptr output_buffer(new uchar[size_buffer]); + std::shared_ptr output_tensor(new Tensor(s, DataType(DataType::DE_UINT8))); + + std::unique_ptr op(new ChangeModeOp()); + op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(op->OneToOne()); + + // Saving + CheckImageShapeAndData(output_tensor, kChangeMode); + + MS_LOG(INFO) << "MindDataTestChangeModeOp end."; +} diff --git a/tests/ut/cpp/dataset/channel_swap_test.cc b/tests/ut/cpp/dataset/channel_swap_test.cc new file mode 100644 index 00000000000..f1dc1396caf --- /dev/null +++ b/tests/ut/cpp/dataset/channel_swap_test.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/hwc_to_chw_op.h" +#include "dataset/core/data_type.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestChannelSwap : public UT::CVOP::CVOpCommon { + public: + MindDataTestChannelSwap() : CVOpCommon() {} +}; + +TEST_F(MindDataTestChannelSwap, TestOp) { + MS_LOG(INFO) << "Doing MindDataTestChannelSwap."; + // Creating a Tensor + TensorShape s = input_tensor_->shape(); + int size_buffer = s[0] * s[1] * s[2]; + + std::unique_ptr output_buffer(new uchar[size_buffer]); + std::shared_ptr output_tensor(new Tensor(s, DataType(DataType::DE_UINT8))); + + // Decoding + std::unique_ptr op(new HwcToChwOp()); + Status status; + status = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(op->OneToOne()); + + // Saving + bool success = false; + if (s[0] == output_tensor->shape()[1] && s[1] == output_tensor->shape()[2] && s[2] == output_tensor->shape()[0]) { + success = true; + } + EXPECT_EQ(success, true); + MS_LOG(INFO) << "MindDataTestChannelSwap end."; +} diff --git a/tests/ut/cpp/dataset/cifar_op_test.cc b/tests/ut/cpp/dataset/cifar_op_test.cc new file mode 100644 index 00000000000..48c332bc13f --- /dev/null +++ b/tests/ut/cpp/dataset/cifar_op_test.cc @@ -0,0 +1,168 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "common/common.h" +#include "common/utils.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/cifar_op.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +std::shared_ptr Repeat(int repeatCnt); + +std::shared_ptr Build(std::vector> ops); + +std::shared_ptr Cifarop(uint64_t num_works, uint64_t rows, uint64_t conns, std::string path, + std::unique_ptr sampler = nullptr, + uint64_t num_samples = 0, bool cifar10 = true) { + std::shared_ptr so; + CifarOp::Builder builder; + Status rc = builder.SetNumWorkers(num_works).SetCifarDir(path).SetRowsPerBuffer(rows) + .SetOpConnectorSize(conns).SetSampler(std::move(sampler)).SetCifarType(cifar10) + .SetNumSamples(num_samples).Build(&so); + return so; +} + +class MindDataTestCifarOp : public UT::DatasetOpTesting { + protected: +}; + +TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar10) { + //Note: CIFAR and Mnist datasets are not included + //as part of the build tree. + //Download datasets and rebuild if data doesn't + //appear in this dataset + //Example: python tests/dataset/data/prep_data.py + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr, 100)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 100); + } +} + +TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) { + uint32_t original_seed = GlobalContext::config_manager()->seed(); + GlobalContext::config_manager()->set_seed(0); + std::unique_ptr sampler = mindspore::make_unique(true, 12); + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler), 100)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 12); + } + GlobalContext::config_manager()->set_seed(original_seed); +} + +TEST_F(MindDataTestCifarOp, TestCifar10NumSample) { + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr, 100)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 100); + } +} + +TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar100) { + std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; + auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr, 100, false)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t coarse = 0; + uint32_t fine = 0; + while (tensor_map.size() != 0) { + tensor_map["coarse_label"]->GetItemAt(&coarse, {}); + tensor_map["fine_label"]->GetItemAt(&fine, {}); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << " coarse:" + << coarse << " fine:" << fine << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 100); + } +} diff --git a/tests/ut/cpp/dataset/circular_pool_test.cc b/tests/ut/cpp/dataset/circular_pool_test.cc new file mode 100644 index 00000000000..47ceae59306 --- /dev/null +++ b/tests/ut/cpp/dataset/circular_pool_test.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "dataset/util/task_manager.h" +#include "dataset/util/circular_pool.h" +#include "dataset/util/services.h" +#include "common/common.h" +#include "common/utils.h" +#include "utils/log_adapter.h" +#include "./securec.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestCircularPool : public UT::Common { + public: + std::shared_ptr mp_; + TaskGroup vg_; + MindDataTestCircularPool() {} + void SetUp() { + Status rc = CircularPool::CreateCircularPool(&mp_); + ASSERT_TRUE(rc.IsOk()); + } +}; + +Status TestMem(MindDataTestCircularPool *tp, int32_t num_iterations) { + const uint64_t min = 19 * 1024; // 19k + const uint64_t max = 20 * 1024 * 1024; // 20M + std::mt19937 gen{std::random_device{}()}; + std::uniform_int_distribution dist(min, max); + TaskManager::FindMe()->Post(); + for (int i = 0; i < num_iterations; i++) { + uint64_t old_sz = dist(gen); + uint64_t new_sz = dist(gen); + std::string str = "Allocate " + std::to_string(old_sz) + + " bytes of memory and then resize to " + std::to_string(new_sz); + std::cout << str << std::endl; + std::string id = Services::GetUniqueID(); + void *p; + RETURN_IF_NOT_OK(tp->mp_->Allocate(old_sz, &p)); + // Copy the id to the start of the memory. + (void) memcpy_s(p, old_sz, common::SafeCStr(id), UNIQUEID_LEN); + RETURN_IF_NOT_OK(tp->mp_->Reallocate(&p, old_sz, new_sz)); + int n = memcmp(p, common::SafeCStr(id), UNIQUEID_LEN); + if (n) { + RETURN_STATUS_UNEXPECTED("Expect match"); + } + tp->mp_->Deallocate(p); + } + return Status::OK(); +} + +TEST_F(MindDataTestCircularPool, TestALLFunction) { + const int32_t iteration = 100; + Services::CreateInstance(); + auto f = std::bind(TestMem, this, iteration); + for (int i = 0; i < 3; i++) { + vg_.CreateAsyncTask("TestMem", f); + } + vg_.join_all(); + std::cout << vg_.GetTaskErrorIfAny() << std::endl; + ASSERT_TRUE(vg_.GetTaskErrorIfAny().IsOk()); + CircularPool *cp = dynamic_cast(mp_.get()); + std::cout << *cp << std::endl; +} + diff --git a/tests/ut/cpp/dataset/client_config_test.cc b/tests/ut/cpp/dataset/client_config_test.cc new file mode 100644 index 00000000000..d0082d16991 --- /dev/null +++ b/tests/ut/cpp/dataset/client_config_test.cc @@ -0,0 +1,120 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include "dataset/core/client.h" +#include "gtest/gtest.h" +#include "dataset/core/global_context.h" +#include "dataset/util/status.h" +#include "dataset/core/client.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include +#include +#include + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + + class MindDataTestClientConfig : public UT::DatasetOpTesting { + protected: +}; + +TEST_F(MindDataTestClientConfig, TestClientConfig1) { + std::shared_ptr my_conf = GlobalContext::config_manager(); + + ASSERT_EQ(my_conf->num_parallel_workers(), kCfgParallelWorkers); + ASSERT_EQ(my_conf->rows_per_buffer(), kCfgRowsPerBuffer); + ASSERT_EQ(my_conf->worker_connector_size(), kCfgWorkerConnectorSize); + ASSERT_EQ(my_conf->op_connector_size(), kCfgOpConnectorSize); + ASSERT_EQ(my_conf->seed(), kCfgDefaultSeed); + + my_conf->set_num_parallel_workers(2); + my_conf->set_rows_per_buffer(1); + my_conf->set_worker_connector_size(3); + my_conf->set_op_connector_size(4); + my_conf->set_seed(5); + + + ASSERT_EQ(my_conf->num_parallel_workers(), 2); + ASSERT_EQ(my_conf->rows_per_buffer(), 1); + ASSERT_EQ(my_conf->worker_connector_size(), 3); + ASSERT_EQ(my_conf->op_connector_size(), 4); + ASSERT_EQ(my_conf->seed(), 5); + + std::string file = datasets_root_path_ + "/declient.cfg"; + ASSERT_TRUE(my_conf->LoadFile(file)); + + ASSERT_EQ(my_conf->num_parallel_workers(), kCfgParallelWorkers); + ASSERT_EQ(my_conf->rows_per_buffer(), kCfgRowsPerBuffer); + ASSERT_EQ(my_conf->worker_connector_size(), kCfgWorkerConnectorSize); + ASSERT_EQ(my_conf->op_connector_size(), kCfgOpConnectorSize); + ASSERT_EQ(my_conf->seed(), kCfgDefaultSeed); + +} + +TEST_F(MindDataTestClientConfig, TestClientConfig2) { + std::shared_ptr my_conf = GlobalContext::config_manager(); + + my_conf->set_num_parallel_workers(16); + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 2 divides evenly into total rows. + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + StorageOp::Builder builder; + builder.SetDatasetFilesDir(dataset_path); + rc = builder.Build(&my_storage_op); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(my_storage_op->num_workers(),16); + my_tree->AssociateNode(my_storage_op); + + // Set children/root layout. + my_tree->AssignRoot(my_storage_op); + + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 10); // Should be 10 rows fetched + ASSERT_EQ(my_storage_op->num_workers(),16); +} diff --git a/tests/ut/cpp/dataset/common/common.cc b/tests/ut/cpp/dataset/common/common.cc new file mode 100644 index 00000000000..7c55575680e --- /dev/null +++ b/tests/ut/cpp/dataset/common/common.cc @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common.h" + +namespace UT { +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif + + +void DatasetOpTesting::SetUp() { + std::string install_home = "data/dataset"; + datasets_root_path_ = install_home; + mindrecord_root_path_ = "data/mindrecord"; +} + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif +} // namespace UT + + diff --git a/tests/ut/cpp/dataset/common/common.h b/tests/ut/cpp/dataset/common/common.h new file mode 100644 index 00000000000..2bd5887dca6 --- /dev/null +++ b/tests/ut/cpp/dataset/common/common.h @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ +#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ + +#include "dataset/util/de_error.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" + +namespace UT { +class Common : public testing::Test { + public: + // every TEST_F macro will enter one + virtual void SetUp(); + + virtual void TearDown(); +}; + +class DatasetOpTesting : public Common { + public: + std::string datasets_root_path_; + std::string mindrecord_root_path_; + void SetUp() override; + +}; +} // namespace UT +#endif // TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ + diff --git a/tests/ut/cpp/dataset/common/cvop_common.cc b/tests/ut/cpp/dataset/common/cvop_common.cc new file mode 100644 index 00000000000..7f99ff27b1a --- /dev/null +++ b/tests/ut/cpp/dataset/common/cvop_common.cc @@ -0,0 +1,155 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "cvop_common.h" +#include "dataset/core/constants.h" +#include "common/utils.h" +#include "dataset/core/cv_tensor.h" +#include "utils/log_adapter.h" +#include +#include + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; +using UT::CVOP::CVOpCommon; + +CVOpCommon::CVOpCommon() {} + +CVOpCommon::~CVOpCommon() {} + +void CVOpCommon::SetUp() { + MS_LOG(INFO) << "starting test."; + filename_ = GetFilename(); + GetInputImage(filename_); +} + +std::string CVOpCommon::GetFilename() { + std::string de_home = "data/dataset"; + std::string filename = de_home + "/apple.jpg"; + MS_LOG(INFO) << "Reading " << common::SafeCStr(filename) << "."; + return filename; +} + +void CVOpCommon::GetInputImage(std::string filename) { + try { + std::ifstream tmp(filename, std::ios::binary | std::ios::ate); + dsize_t file_size = tmp.tellg(); + tmp.close(); + + std::ifstream file(filename, std::ios::binary); + TensorShape in_shape({file_size}); + raw_input_tensor_ = std::make_shared(in_shape, DataType(DataType::DE_UINT8)); + + file.read(reinterpret_cast(raw_input_tensor_->StartAddr()), raw_input_tensor_->SizeInBytes()); + raw_cv_image_ = cv::imread(filename, cv::ImreadModes::IMREAD_COLOR); + input_tensor_ = std::dynamic_pointer_cast(std::make_shared(raw_cv_image_)); + SwapRedAndBlue(input_tensor_, &input_tensor_); + if (raw_cv_image_.data) { + MS_LOG(INFO) << "Reading was successful. Height:" << raw_cv_image_.rows << " Width: " << raw_cv_image_.cols + << " Channels:" << raw_cv_image_.channels() << "."; + } + } catch (...) { + MS_LOG(INFO) << "Error in GetInputImage."; + } +} + +void CVOpCommon::Save(const std::shared_ptr &tensor, std::string filename) { + std::shared_ptr output; + SwapRedAndBlue(tensor, &output); + + cv::Mat output_image_CV = std::dynamic_pointer_cast(output)->mat(); + cv::imwrite(filename, output_image_CV); +} + +std::string CVOpCommon::GetJPGStr(const cv::Mat &image) { + std::vector buff_jpg; + cv::imencode(".jpg", image, buff_jpg); + return std::string(buff_jpg.begin(), buff_jpg.end()); +} + +bool CVOpCommon::CompareCVMat(const cv::Mat &actual, const cv::Mat &expect, OperatorType type) { + if (actual.size() != expect.size() || actual.channels() != expect.channels() || actual.type() != expect.type()) { + return false; + } + std::string strJPGData_actual = GetJPGStr(actual); + std::string strJPGData_expect = GetJPGStr(expect); + + bool success = strJPGData_actual.compare(strJPGData_expect) == 0; + if (type == kFlipHorizontal || type == kFlipVertical) { + std::string raw_filename = filename_; + raw_filename.replace(raw_filename.end() - 9, raw_filename.end(), "imagefolder/apple_expect_not_flip.jpg"); + cv::Mat raw; + raw = cv::imread(raw_filename, cv::ImreadModes::IMREAD_COLOR); + std::string strJPGData_raw = GetJPGStr(raw); + success = success || strJPGData_actual.compare(strJPGData_raw) == 0; + } else if (type == kCrop) { + success = expect.rows == actual.rows && expect.cols == actual.cols && expect.channels(), actual.channels(); + } + return success; +} + +void CVOpCommon::CheckImageShapeAndData(const std::shared_ptr &output_tensor, OperatorType type) { + std::string dir_path = filename_.substr(0, filename_.length() - 9); + std::string expect_image_path, actual_image_path; + switch (type) { + case kRescale: + expect_image_path = dir_path + "imagefolder/apple_expect_rescaled.jpg"; + actual_image_path = dir_path + "imagefolder/apple_actual_rescaled.jpg"; + break; + case kResizeBilinear: + expect_image_path = dir_path + "imagefolder/apple_expect_resize_bilinear.jpg"; + actual_image_path = dir_path + "imagefolder/apple_actual_resize_bilinear.jpg"; + break; + case kFlipHorizontal: + expect_image_path = dir_path + "imagefolder/apple_expect_flipped_horizontal.jpg"; + actual_image_path = dir_path + "imagefolder/apple_actual_flipped_horizontal.jpg"; + break; + case kFlipVertical: + expect_image_path = dir_path + "imagefolder/apple_expect_flipped_vertical.jpg"; + actual_image_path = dir_path + "imagefolder/apple_actual_flipped_vertical.jpg"; + break; + case kDecode: + expect_image_path = dir_path + "imagefolder/apple_expect_decoded.jpg"; + actual_image_path = dir_path + "imagefolder/apple_actual_decoded.jpg"; + break; + case kChangeMode: + expect_image_path = dir_path + "imagefolder/apple_expect_changemode.jpg"; + actual_image_path = dir_path + "imagefolder/apple_actual_changemode.jpg"; + break; + default: + MS_LOG(INFO) << "Not pass verification! Operation type does not exists."; + EXPECT_EQ(0, 1); + } + cv::Mat expect_img = cv::imread(expect_image_path, cv::IMREAD_COLOR); + cv::Mat actual_img; + // Saving + MS_LOG(INFO) << "output_tensor.shape is " << output_tensor->shape(); + Save(output_tensor, actual_image_path); + actual_img = cv::imread(actual_image_path, cv::IMREAD_COLOR); + if (actual_img.data) { + EXPECT_EQ(CompareCVMat(actual_img, expect_img, type), true); + } else { + MS_LOG(INFO) << "Not pass verification! Image data is null."; + EXPECT_EQ(0, 1); + } +} diff --git a/tests/ut/cpp/dataset/common/cvop_common.h b/tests/ut/cpp/dataset/common/cvop_common.h new file mode 100644 index 00000000000..02c079fd68e --- /dev/null +++ b/tests/ut/cpp/dataset/common/cvop_common.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_CVOP_COMMON_H_ +#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_CVOP_COMMON_H_ + +#include +#include +#include "common.h" +#include "dataset/kernels/image/image_utils.h" + +namespace UT { +namespace CVOP { +using namespace mindspore::dataset; + +class CVOpCommon : public Common { + public: + enum OperatorType { + kResizeArea, + kResizeBilinear, + kRescale, + kFlipVertical, + kFlipHorizontal, + kDecode, + kChannelSwap, + kChangeMode, + kTemplate, + kCrop + }; + + CVOpCommon(); + + ~CVOpCommon(); + + void SetUp(); + + std::string GetFilename(); + + void GetInputImage(std::string filename); + + void Save(const std::shared_ptr &tensor, std::string filename); + + std::string GetJPGStr(const cv::Mat &image); + + bool CompareCVMat(const cv::Mat &actual, const cv::Mat &expect, OperatorType type); + + void CheckImageShapeAndData(const std::shared_ptr &output_tensor, OperatorType type); + + std::string filename_; + cv::Mat raw_cv_image_; + + std::shared_ptr raw_input_tensor_; + std::shared_ptr input_tensor_; +}; +} // namespace CVOP +} // namespace UT +#endif // TESTS_DATASET_UT_CORE_COMMON_DE_UT_CVOP_COMMON_H_ diff --git a/tests/ut/cpp/dataset/connector_test.cc b/tests/ut/cpp/dataset/connector_test.cc new file mode 100644 index 00000000000..f4479f52eb2 --- /dev/null +++ b/tests/ut/cpp/dataset/connector_test.cc @@ -0,0 +1,331 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + + +#include "common/common.h" +#include "dataset/engine/connector.h" +#include "dataset/util/task_manager.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestConnector : public UT::Common { +public: + + MindDataTestConnector(); + + // Test scenario: single producer, single consumer. + // This means there is only one queue in the connector. + Status Run_test_0(); + + // Test scenario: multiple producers, multiple cosumers + // A chain of three layer of thread groups connected by two Connectors between + // two layer. You can set different num of threads on layer 1 and 2, and layer 3 + // that does the serialization to _ouput vector needs to be single thread. + // A random sleep/delay can be introduced for each thread. See run(). + Status Run_test_1(); + + void SetSleepMilliSec(uint32_t ms) { sleep_ms_ = ms; } + +private: + std::unique_ptr tg_; + uint32_t last_input_; + uint32_t sleep_ms_ = 0; + std::vector input_; + + // This worker loop is to be called by a single thread. It will pop my_conn Connector + // and populate output vector + Status SerialWorkerPull( + int tid, + std::shared_ptr> my_conn, + std::vector *output + ); + + // This worker loop read from input_ vector that have complete list of tasks/elements. + // The assignment from the elements in input_ to each worker is ensured in RoundRobin, + // i.e., tid-0 will pick input_[0], tid-1 will pick input_[1], so-on circularly. + Status FirstWorkerPush( + int tid, + std::shared_ptr > my_conn, + int start_in, + int offset); + + // This worker loop read from a Connector and put the result into another Connector. + Status MidWorkerJob( + int tid, + std::shared_ptr > from_conn, + std::shared_ptr > to_conn); + + Status ValidateOutput(const std::vector &output); + + uint32_t GenRand(int max); + + // Put the current thread to sleep mode for MaxDue milliseconds. + // (Imitating nondeterministic processing time) + void GoToSleep(int max_dur); +}; + + +// Test0 : single producer, single consumer which means there is only one queue in the connector +TEST_F(MindDataTestConnector, Test0) { + MS_LOG(INFO) << "MindDataTestConnector Test0: single producer, single consumer."; + Status rc = this->Run_test_0(); + ASSERT_TRUE(rc.IsOk()); + rc = TaskManager::GetMasterThreadRc(); + ASSERT_TRUE(rc.IsOk()); +} + +// Test1: multiple producers, multiple consumers without random delay +// A chain of three layer of thread groups connected by two Connectors between +// two layer. +TEST_F(MindDataTestConnector, Test1) { + MS_LOG(INFO) << "MindDataTestConnector Test1."; + Status rc = this->Run_test_1(); + ASSERT_TRUE(rc.IsOk()); + rc = TaskManager::GetMasterThreadRc(); + ASSERT_TRUE(rc.IsOk()); +} + +// Test1: multiple producers, multiple consumers with random delay after push/pop +// A chain of three layer of thread groups connected by two Connectors between +// two layer. +TEST_F(MindDataTestConnector, Test2) { + MS_LOG(INFO) << "MindDataTestConnector Test2."; + this->SetSleepMilliSec(30); + Status rc = this->Run_test_1(); + ASSERT_TRUE(rc.IsOk()); + rc = TaskManager::GetMasterThreadRc(); + ASSERT_TRUE(rc.IsOk()); +} + + + +// Implementation of MindDataTestConnector class and the helper functions. +MindDataTestConnector::MindDataTestConnector() : tg_(new TaskGroup()) { + last_input_ = 150; + for (int i = 1; i <= last_input_; i++) { + input_.push_back(i); + } +} + +Status MindDataTestConnector::Run_test_0() { + Status rc; + std::vector output; + auto my_conn = std::make_shared>(1, // num of producers + 1, // num of consumers + 10); // capacity of each queue + DS_ASSERT(my_conn != nullptr); + + // Spawn a thread to read input_ vector and put it in my_conn + rc = tg_->CreateAsyncTask("Worker Push", + std::bind(&MindDataTestConnector::FirstWorkerPush, + this, // passing this instance + 0, // id = 0 for this simple one to one case + my_conn, // the connector + 0, // start index to read from the input_ list + 1)); // the offset to read the next index + RETURN_IF_NOT_OK(rc); + + // Spawn another thread to read from my_conn and write to _output vector. + rc = tg_->CreateAsyncTask("Worker Pull", + std::bind(&MindDataTestConnector::SerialWorkerPull, + this, + 0, + my_conn, + &output)); + RETURN_IF_NOT_OK(rc); + + tg_->join_all(); + my_conn.reset(); + return ValidateOutput(output); +} + +Status MindDataTestConnector::Run_test_1() { + std::vector output; + Status rc; + + // number of threads in each layer + int l1_threads = 15; + int l2_threads = 20; + int l3_threads = 1; + + // Capacity for the first and second connectors + int conn1_qcap = 5; + int conn2_qcap = 10; + + auto conn1 = std::make_shared>(l1_threads, // num of producers + l2_threads, // num of consumers + conn1_qcap); // the cap of each queue + + auto conn2 = std::make_shared>(l2_threads, + l3_threads, + conn2_qcap); + + // Instantiating the threads in the first layer + for (int i = 0; i < l1_threads; i++) { + rc = tg_->CreateAsyncTask("First Worker Push", + std::bind(&MindDataTestConnector::FirstWorkerPush, + this, // passing this instance + i, // thread id in this group of thread + conn1, // the connector + i, // start index to read from the input_ list + l1_threads)); // the offset to read the next index + RETURN_IF_NOT_OK(rc); + } + + // Instantiating the threads in the 2nd layer + for (int i = 0; i < l2_threads; i++) { + rc = tg_->CreateAsyncTask("Mid Worker Job", + std::bind(&MindDataTestConnector::MidWorkerJob, + this, // passing this instance + i, // thread id in this group of thread + conn1, // the 1st connector + conn2)); // the 2nd connector + RETURN_IF_NOT_OK(rc); + } + + // Last layer doing serialization to one queue to check if the order is preserved + rc = tg_->CreateAsyncTask("Worker Pull", + std::bind(&MindDataTestConnector::SerialWorkerPull, + this, + 0, // thread id = 0, since it's the only one + conn2, // poping the data from conn2 + &output)); + RETURN_IF_NOT_OK(rc); + + tg_->join_all(); + conn1.reset(); + conn2.reset(); + + return ValidateOutput(output); +} + +Status MindDataTestConnector::SerialWorkerPull( + int tid, + std::shared_ptr> my_conn, + std::vector *output + ) { + Status rc; + TaskManager::FindMe()->Post(); + while (1) { + uint32_t res; + rc = my_conn->Pop(tid, &res); + RETURN_IF_NOT_OK(rc); + + output->push_back(res); + + // Emulate different processing time for each thread + if (sleep_ms_ != 0) { + GoToSleep(sleep_ms_); + } + + // Raising interrupt after it processed the last_input_. + // This will trigger the MidWorkerJob threads to quit their worker loop. + if (res == last_input_) { + MS_LOG(INFO) << "All data is collected."; + tg_->interrupt_all(); + break; + } + } + return Status::OK(); +} + +Status MindDataTestConnector::FirstWorkerPush( + int tid, + std::shared_ptr > my_conn, + int start_in, + int offset) { + TaskManager::FindMe()->Post(); + DS_ASSERT(my_conn != nullptr); + Status rc; + for (int i = start_in; i < input_.size(); i += offset) { + rc = my_conn->Push(tid, input_[i]); + + // Emulate different processing time for each thread + if (sleep_ms_ != 0) + GoToSleep(sleep_ms_); + } + return Status::OK(); +} + +// This worker loop read from a Connector and put the result into another Connector. +Status MindDataTestConnector::MidWorkerJob( + int tid, + std::shared_ptr > from_conn, + std::shared_ptr > to_conn) { + DS_ASSERT((from_conn != nullptr) && (to_conn != nullptr)); + Status rc; + TaskManager::FindMe()->Post(); + while (1) { + uint32_t el; + rc = from_conn->Pop(tid, &el); + RETURN_IF_NOT_OK(rc); + + // Emulate different processing time for each thread + if (sleep_ms_ != 0) { + GoToSleep(sleep_ms_); + } + rc = to_conn->Push(tid, el); + RETURN_IF_NOT_OK(rc); + } + return Status::OK(); +} + +Status MindDataTestConnector::ValidateOutput(const std::vector &output) { + int prev = 0; + for (auto el : output) { + if (prev >= el) { + return Status(StatusCode::kUnexpectedError, "Output vector are not in-order."); + } + prev = el; + } + return Status::OK(); +} + +uint32_t MindDataTestConnector::GenRand(int max) { + uint32_t r_int = 0; + if (max == 0) { + return r_int; + } + + // open urandom not random + int fd = open("/dev/urandom", O_RDONLY); + if (fd > 0) { + if (read(fd, &r_int, sizeof(uint32_t)) != sizeof(uint32_t)) { + r_int = max / 2; + } + } + (void)close(fd); // close it! + + return r_int % max; +} + +// Put the current thread to sleep mode for MaxDue milliseconds. +// (Imitating nondeterministic processing time) +void MindDataTestConnector::GoToSleep(int max_dur) { + uint32_t duration = GenRand(max_dur); + std::this_thread::sleep_for(std::chrono::milliseconds(duration)); +} diff --git a/tests/ut/cpp/dataset/datatype_test.cc b/tests/ut/cpp/dataset/datatype_test.cc new file mode 100644 index 00000000000..ee49037a191 --- /dev/null +++ b/tests/ut/cpp/dataset/datatype_test.cc @@ -0,0 +1,197 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "./securec.h" +#include "dataset/core/data_type.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include +#include "dataset/core/constants.h" + +using namespace mindspore::dataset; + +namespace py = pybind11; + +class MindDataTestDatatype : public UT::Common { + public: + MindDataTestDatatype() = default; +}; + + +TEST_F(MindDataTestDatatype, TestSizes) { + uint8_t x = DataType::DE_BOOL_SIZE; + DataType d = DataType(DataType::DE_BOOL); + ASSERT_EQ(x, 1); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_INT8_SIZE; + d = DataType(DataType::DE_INT8); + ASSERT_EQ(x, 1); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_UINT8_SIZE; + d = DataType(DataType::DE_UINT8); + ASSERT_EQ(x, 1); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_INT16_SIZE; + d = DataType(DataType::DE_INT16); + ASSERT_EQ(x, 2); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_UINT16_SIZE; + d = DataType(DataType::DE_UINT16); + ASSERT_EQ(x, 2); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_INT32_SIZE; + d = DataType(DataType::DE_INT32); + ASSERT_EQ(x, 4); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_UINT32_SIZE; + d = DataType(DataType::DE_UINT32); + ASSERT_EQ(x, 4); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_INT64_SIZE; + d = DataType(DataType::DE_INT64); + ASSERT_EQ(x, 8); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_UINT64_SIZE; + d = DataType(DataType::DE_UINT64); + ASSERT_EQ(x, 8); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_FLOAT32_SIZE; + d = DataType(DataType::DE_FLOAT32); + ASSERT_EQ(x, 4); + ASSERT_EQ(d.SizeInBytes(), x); + x = DataType::DE_FLOAT64_SIZE; + d = DataType(DataType::DE_FLOAT64); + ASSERT_EQ(x, 8); + ASSERT_EQ(d.SizeInBytes(), x); +} + +void FromDT(DataType d, uint8_t cv_type, std::string str) { + if (d == DataType::DE_UNKNOWN || d == DataType::DE_UINT32 || d == DataType::DE_UINT64 || d == DataType::DE_INT64) { + ASSERT_EQ(d.AsCVType(), kCVInvalidType); + } else { + ASSERT_EQ(d.AsCVType(), cv_type); + } + ASSERT_EQ(d.ToString(), str); +} + +TEST_F(MindDataTestDatatype, TestConstructors) { + // Default constructor + DataType d; + ASSERT_EQ(d, DataType::DE_UNKNOWN); + DataType d3 = DataType(); + ASSERT_EQ(d3, DataType::DE_UNKNOWN); + + // DataType(Type d) + DataType d4(DataType::DE_FLOAT32); + ASSERT_EQ(d4, DataType::DE_FLOAT32); + DataType d5 = DataType(DataType::DE_UINT32); + ASSERT_EQ(d5, DataType::DE_UINT32); + + // != operator + ASSERT_NE(d4, d5); + + // == operator + d5 = DataType(DataType::DE_FLOAT32); + ASSERT_EQ(d4, d5); +} + +TEST_F(MindDataTestDatatype, TestFromTypes) { + FromDT(DataType(DataType::DE_BOOL), CV_8U, "bool"); + FromDT(DataType(DataType::DE_UINT8), CV_8U, "uint8"); + FromDT(DataType(DataType::DE_INT8), CV_8S, "int8"); + FromDT(DataType(DataType::DE_UINT16), CV_16U, "uint16"); + FromDT(DataType(DataType::DE_INT16), CV_16S, "int16"); + FromDT(DataType(DataType::DE_UINT32), 0, "uint32"); + FromDT(DataType(DataType::DE_INT32), CV_32S, "int32"); + FromDT(DataType(DataType::DE_UINT64), 0, "uint64"); + FromDT(DataType(DataType::DE_INT64), 0, "int64"); + FromDT(DataType(DataType::DE_FLOAT32), CV_32F, "float32"); + FromDT(DataType(DataType::DE_FLOAT64), CV_64F, "float64"); + FromDT(DataType(DataType::DE_UNKNOWN), CV_8U, "unknown"); +} + +TEST_F(MindDataTestDatatype, TestCompatible) { + ASSERT_TRUE(DataType(DataType::DE_BOOL).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT8).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_INT8).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT16).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_INT16).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT32).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_INT32).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT64).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_INT64).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_FLOAT32).IsCompatible()); + ASSERT_TRUE(DataType(DataType::DE_FLOAT64).IsCompatible()); + + ASSERT_FALSE(DataType(DataType::DE_UINT8).IsCompatible()); + ASSERT_FALSE(DataType(DataType::DE_BOOL).IsCompatible()); + ASSERT_FALSE(DataType(DataType::DE_UINT8).IsCompatible()); + ASSERT_FALSE(DataType(DataType::DE_UINT32).IsCompatible()); + ASSERT_FALSE(DataType(DataType::DE_INT64).IsCompatible()); + + ASSERT_TRUE(DataType(DataType::DE_BOOL).IsLooselyCompatible()); + ASSERT_FALSE(DataType(DataType::DE_INT16).IsLooselyCompatible()); + + ASSERT_TRUE(DataType(DataType::DE_UINT8).IsLooselyCompatible()); + ASSERT_FALSE(DataType(DataType::DE_UINT64).IsLooselyCompatible()); + + ASSERT_TRUE(DataType(DataType::DE_UINT64).IsLooselyCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT32).IsLooselyCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT16).IsLooselyCompatible()); + ASSERT_TRUE(DataType(DataType::DE_UINT8).IsLooselyCompatible()); + + +} + +TEST_F(MindDataTestDatatype, TestCVTypes) { + ASSERT_EQ(DataType::DE_UINT8, DataType::FromCVType(CV_8U).value()); + ASSERT_EQ(DataType::DE_UINT8, DataType::FromCVType(CV_8UC1).value()); + ASSERT_EQ(DataType::DE_UINT8, DataType::FromCVType(CV_8UC2).value()); + ASSERT_EQ(DataType::DE_UINT8, DataType::FromCVType(CV_8UC3).value()); + ASSERT_EQ(DataType::DE_UINT8, DataType::FromCVType(CV_8UC4).value()); + ASSERT_EQ(DataType::DE_UINT8, DataType::FromCVType(CV_8UC(5)).value()); + ASSERT_EQ(DataType::DE_INT8, DataType::FromCVType(CV_8S).value()); + ASSERT_EQ(DataType::DE_INT8, DataType::FromCVType(CV_8SC1).value()); + ASSERT_EQ(DataType::DE_INT8, DataType::FromCVType(CV_8SC2).value()); + ASSERT_EQ(DataType::DE_INT8, DataType::FromCVType(CV_8SC3).value()); + ASSERT_EQ(DataType::DE_INT8, DataType::FromCVType(CV_8SC(5)).value()); + ASSERT_EQ(DataType::DE_UINT16, DataType::FromCVType(CV_16U).value()); + ASSERT_EQ(DataType::DE_UINT16, DataType::FromCVType(CV_16UC1).value()); + ASSERT_EQ(DataType::DE_UINT16, DataType::FromCVType(CV_16UC2).value()); + ASSERT_EQ(DataType::DE_UINT16, DataType::FromCVType(CV_16UC3).value()); + ASSERT_EQ(DataType::DE_UINT16, DataType::FromCVType(CV_16UC4).value()); + ASSERT_EQ(DataType::DE_UINT16, DataType::FromCVType(CV_16UC(5)).value()); + ASSERT_EQ(DataType::DE_INT16, DataType::FromCVType(CV_16S).value()); + ASSERT_EQ(DataType::DE_INT16, DataType::FromCVType(CV_16SC1).value()); + ASSERT_EQ(DataType::DE_INT16, DataType::FromCVType(CV_16SC2).value()); + ASSERT_EQ(DataType::DE_INT16, DataType::FromCVType(CV_16SC3).value()); + ASSERT_EQ(DataType::DE_INT16, DataType::FromCVType(CV_16SC(5)).value()); + ASSERT_EQ(DataType::DE_INT32, DataType::FromCVType(CV_32S).value()); + ASSERT_EQ(DataType::DE_INT32, DataType::FromCVType(CV_32SC1).value()); + ASSERT_EQ(DataType::DE_INT32, DataType::FromCVType(CV_32SC2).value()); + ASSERT_EQ(DataType::DE_INT32, DataType::FromCVType(CV_32SC3).value()); + ASSERT_EQ(DataType::DE_INT32, DataType::FromCVType(CV_32SC(5)).value()); + ASSERT_EQ(DataType::DE_FLOAT32, DataType::FromCVType(CV_32F).value()); + ASSERT_EQ(DataType::DE_FLOAT32, DataType::FromCVType(CV_32FC1).value()); + ASSERT_EQ(DataType::DE_FLOAT32, DataType::FromCVType(CV_32FC2).value()); + ASSERT_EQ(DataType::DE_FLOAT32, DataType::FromCVType(CV_32FC3).value()); + ASSERT_EQ(DataType::DE_FLOAT32, DataType::FromCVType(CV_32FC(5)).value()); + ASSERT_EQ(DataType::DE_FLOAT64, DataType::FromCVType(CV_64F).value()); + ASSERT_EQ(DataType::DE_FLOAT64, DataType::FromCVType(CV_64FC1).value()); + ASSERT_EQ(DataType::DE_FLOAT64, DataType::FromCVType(CV_64FC2).value()); + ASSERT_EQ(DataType::DE_FLOAT64, DataType::FromCVType(CV_64FC3).value()); + ASSERT_EQ(DataType::DE_FLOAT64, DataType::FromCVType(CV_64FC(5)).value()); +} diff --git a/tests/ut/cpp/dataset/decode_op_test.cc b/tests/ut/cpp/dataset/decode_op_test.cc new file mode 100644 index 00000000000..7f3e129ac0d --- /dev/null +++ b/tests/ut/cpp/dataset/decode_op_test.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/decode_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::INFO; + +class MindDataTestDecodeOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestDecodeOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestDecodeOp, TestOp) { + MS_LOG(INFO) << "Doing testDecode"; + TensorShape s = TensorShape({1}); + std::shared_ptr output_tensor; + DecodeOp op(true); + op.Compute(raw_input_tensor_, &output_tensor); + + CheckImageShapeAndData(output_tensor, kDecode); +} diff --git a/tests/ut/cpp/dataset/execution_tree_test.cc b/tests/ut/cpp/dataset/execution_tree_test.cc new file mode 100644 index 00000000000..4a632712369 --- /dev/null +++ b/tests/ut/cpp/dataset/execution_tree_test.cc @@ -0,0 +1,145 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "dataset/util/circular_pool.h" +#include "dataset/core/client.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/engine/datasetops/shuffle_op.h" +#include "dataset/engine/datasetops/source/storage_op.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestExecutionTree : public UT::DatasetOpTesting { + public: + +}; + + +// Construct some tree nodes and play with them +TEST_F(MindDataTestExecutionTree, TestExecutionTree1) { + MS_LOG(INFO) << "Doing MindDataTestExecutionTree1."; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + ASSERT_NE(my_tree, nullptr); + uint32_t shuffle_size = 32; + uint32_t connector_size = 8; + + + std::shared_ptr leaf_op1 = + std::make_shared(shuffle_size, 0, connector_size, false, 32); + ASSERT_NE(leaf_op1, nullptr); + my_tree->AssociateNode(leaf_op1); + shuffle_size = 16; + std::shared_ptr leaf_op2 = + std::make_shared(shuffle_size, 0, connector_size, false, 32); + ASSERT_NE(leaf_op2, nullptr); + my_tree->AssociateNode(leaf_op2); + shuffle_size = 8; + std::shared_ptr parent_op = + std::make_shared(shuffle_size, 0, connector_size, false, 32); + ASSERT_NE(parent_op, nullptr); + my_tree->AssociateNode(parent_op); + + // It's up to you if you want to use move semantic or not here. + // By using move, we transfer ownership of the child to the parent. + // If you do not use move, + // we get a reference count bump to the pointer and you have + // your own pointer to it, plus the parent has a copy of the pointer. + parent_op->AddChild(std::move(leaf_op1)); + parent_op->AddChild(std::move(leaf_op2)); + shuffle_size = 4; + std::shared_ptr root_op = + std::make_shared(shuffle_size, 0, connector_size, false, 32); + my_tree->AssignRoot(root_op); + root_op->AddChild(parent_op); + ASSERT_NE(root_op, nullptr); + // Testing Iterator + MS_LOG(INFO) << "Testing Tree Iterator from root."; + for (auto itr = my_tree->begin(); itr != my_tree->end(); ++itr) { + itr->Print(std::cout, false); + } + MS_LOG(INFO) << "Finished testing Tree Iterator from root."; + MS_LOG(INFO) << "Testing Tree Iterator from parentOp."; + for (auto itr = my_tree->begin(parent_op); itr != my_tree->end(); ++itr) { + itr->Print(std::cout, false); + } + MS_LOG(INFO) << "Finished testing Tree Iterator from parentOp."; + + // At this point, since move semantic was used, + // I don't have any operator access myself now. + // Ownership is fully transferred into the tree. + + // explicitly drive tree destruction rather than + // wait for descoping (to examine in debugger + // just to see it work) + my_tree.reset(); + MS_LOG(INFO) << "Done."; +} + +// Construct some tree nodes and play with them +TEST_F(MindDataTestExecutionTree, TestExecutionTree2) { + MS_LOG(INFO) << "Doing MindDataTestExecutionTree2."; + Status rc; + auto my_tree = std::make_shared(); + + std::string dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(2) + .SetNumWorkers(2) + .Build(&my_storage_op); + + my_tree->AssociateNode(my_storage_op); + my_tree->AssignRoot(my_storage_op); + + // prepare the tree + my_tree->Prepare(); + + // Launch the tree execution to kick off threads + // and start running the pipeline + MS_LOG(INFO) << "Launching my tree."; + my_tree->Launch(); + + // Simulate a parse of data from our pipeline. + std::shared_ptr root_node = my_tree->root(); + + // Start the loop of reading from our pipeline using iterator + MS_LOG(INFO) << "Testing DatasetIterator in testTree2."; + DatasetIterator di(my_tree); + TensorRow buffer; + rc = di.FetchNextTensorRow(&buffer); + EXPECT_TRUE(rc.IsOk()); + + while (!buffer.empty()) { + rc = di.FetchNextTensorRow(&buffer); + EXPECT_TRUE(rc.IsOk()); + } +} + +// Construct some tree nodes and play with them +TEST_F(MindDataTestExecutionTree, TestExecutionTree3) { + MS_LOG(INFO) << "Doing MindDataTestExecutionTree3."; +} diff --git a/tests/ut/cpp/dataset/global_context_test.cc b/tests/ut/cpp/dataset/global_context_test.cc new file mode 100644 index 00000000000..1bcd7d938ed --- /dev/null +++ b/tests/ut/cpp/dataset/global_context_test.cc @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/global_context.h" +#include "common/common.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestGlobalContext : public UT::Common { + public: + MindDataTestGlobalContext() {} +}; + +TEST_F(MindDataTestGlobalContext, TestGCFunction) { + MS_LOG(INFO) << "Doing Test GlobalContext"; + MS_LOG(INFO) << "Doing instance"; + GlobalContext *global = GlobalContext::Instance(); + ASSERT_NE(global, nullptr); + (void) global->config_manager(); + global->Print(std::cout); +} diff --git a/tests/ut/cpp/dataset/image_folder_op_test.cc b/tests/ut/cpp/dataset/image_folder_op_test.cc new file mode 100644 index 00000000000..82513a29a5d --- /dev/null +++ b/tests/ut/cpp/dataset/image_folder_op_test.cc @@ -0,0 +1,478 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include "common/common.h" +#include "common/utils.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/image_folder_op.h" +#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h" +#include "dataset/engine/datasetops/source/sampler/pk_sampler.h" +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +std::shared_ptr Batch(int batch_size = 1, bool drop = false, int rows_per_buf = 2); + +std::shared_ptr Repeat(int repeat_cnt); + +std::shared_ptr Build(std::vector> ops); + +std::shared_ptr ImageFolder(int64_t num_works, int64_t rows, int64_t conns, std::string path, + bool shuf = false, std::unique_ptr sampler = nullptr, + std::map map = {}, int64_t num_samples = 0, + bool decode = false) { + std::shared_ptr so; + ImageFolderOp::Builder builder; + Status rc = builder.SetNumWorkers(num_works) + .SetImageFolderDir(path) + .SetRowsPerBuffer(rows) + .SetOpConnectorSize(conns) + .SetExtensions({".jpg", ".JPEG"}) + .SetSampler(std::move(sampler)) + .SetClassIndex(map) + .SetDecode(decode) + .SetNumSamples(num_samples) + .Build(&so); + return so; +} + +Status Create1DTensor(std::shared_ptr *sample_ids, int64_t num_elements, unsigned char *data = nullptr, + DataType::Type data_type = DataType::DE_UINT32) { + TensorShape shape(std::vector(1, num_elements)); + RETURN_IF_NOT_OK( + Tensor::CreateTensor(sample_ids, TensorImpl::kFlexible, shape, DataType(data_type), data)); + if (data == nullptr) { + (*sample_ids)->StartAddr(); // allocate memory in case user forgets! + } + return Status::OK(); +} + +class MindDataTestImageFolderSampler : public UT::DatasetOpTesting { + protected: +}; + +TEST_F(MindDataTestImageFolderSampler, TestSequentialImageFolderWithRepeat) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false), Repeat(2)}); + tree->Prepare(); + int32_t res[] = {0, 1, 2, 3}; + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(res[(i % 44) / 11] == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 88); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestRandomImageFolder) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, true, nullptr)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 44); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestRandomSamplerImageFolder) { + int32_t original_seed = GlobalContext::config_manager()->seed(); + GlobalContext::config_manager()->set_seed(0); + std::unique_ptr sampler = mindspore::make_unique(true, 12); + int32_t res[] = {2, 2, 2, 3, 2, 3, 2, 3, 1, 2, 2, 1}; // ground truth label + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(res[i] == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 12); + } + GlobalContext::config_manager()->set_seed(original_seed); +} + +TEST_F(MindDataTestImageFolderSampler, TestSequentialImageFolderWithRepeatBatch) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false), Repeat(2), Batch(11)}); + tree->Prepare(); + int32_t res[4][11] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}}; + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + while (tensor_map.size() != 0) { + std::shared_ptr label; + Create1DTensor(&label, 11, reinterpret_cast(res[i % 4]), DataType::DE_INT32); + EXPECT_TRUE((*label) == (*tensor_map["label"])); + std::cout << "row: " << i++ << " " << tensor_map["image"]->shape() << " (*label):" << (*label) + << " *tensor_map[label]: " << *tensor_map["label"] << std::endl; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 8); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestSubsetRandomSamplerImageFolder) { + // id range 0 - 10 is label 0, and id range 11 - 21 is label 1 + std::vector indices({0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11}); + std::unique_ptr sampler = mindspore::make_unique(indices); + std::string folder_path = datasets_root_path_ + "/testPK/data"; + // Expect 6 samples for label 0 and 1 + int res[2] = {6, 6}; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + res[label]--; + i++; + di.GetNextAsMap(&tensor_map); + } + EXPECT_EQ(res[0], 0); + EXPECT_EQ(res[1], 0); + EXPECT_TRUE(i == 12); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestWeightedRandomSamplerImageFolder) { + // num samples to draw. + int64_t num_samples = 12; + int64_t total_samples = 44; + int64_t samples_per_buffer = 10; + std::vector weights(total_samples, std::rand() % 100); + + // create sampler with replacement = replacement + std::unique_ptr sampler = + mindspore::make_unique(weights, num_samples, true, samples_per_buffer); + + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + i++; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 12); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestImageFolderClassIndex) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + std::map map; + map["class3"] = 333; + map["class1"] = 111; + map["wrong folder name"] = 1234; // this is skipped + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, nullptr, map)}); + int64_t res[2] = {111, 333}; + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(label == res[i / 11]); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 22); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) { + std::unique_ptr sampler = mindspore::make_unique(11, 10, false); + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler)), Repeat(4)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_EQ(i % 4, label); + std::cout << "row:" << i++ << "\tlabel:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 16); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestPKSamplerImageFolder) { + std::unique_ptr sampler = mindspore::make_unique(3, false, 4); + int32_t res[] = {0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; // ground truth label + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(res[i] == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 12); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestImageFolderNumSamples) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, nullptr, {}, 11), Repeat(2)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(0 == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 22); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestImageFolderDecode) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + std::map map; + map["class3"] = 333; + map["class1"] = 111; + map["wrong folder name"] = 1234; // this is skipped + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, nullptr, map, 20, true)}); + int64_t res[2] = {111, 333}; + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(label == res[i / 11]); + EXPECT_TRUE( + tensor_map["image"]->shape() == TensorShape({2268, 4032, 3})); // verify shapes are correct after decode + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 20); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestImageFolderDatasetSize) { + std::string folder_path = datasets_root_path_ + "/testPK/data"; + int64_t num_rows = 0; + int64_t num_classes = 0; + ImageFolderOp::CountRowsAndClasses(folder_path, 15, {}, &num_rows, &num_classes); + EXPECT_TRUE(num_rows == 15 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 44, {}, &num_rows, &num_classes); + EXPECT_TRUE(num_rows == 44 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 0, {}, &num_rows, &num_classes); + EXPECT_TRUE(num_rows == 44 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 55, {}, &num_rows, &num_classes); + EXPECT_TRUE(num_rows == 44 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 44, {}, &num_rows, &num_classes, 2, 3); + EXPECT_TRUE(num_rows == 15 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 33, {}, &num_rows, &num_classes, 0, 3); + EXPECT_TRUE(num_rows == 15 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 13, {}, &num_rows, &num_classes, 0, 11); + EXPECT_TRUE(num_rows == 4 && num_classes == 4); + ImageFolderOp::CountRowsAndClasses(folder_path, 3, {}, &num_rows, &num_classes, 0, 11); + EXPECT_TRUE(num_rows == 3 && num_classes == 4); +} + +TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) { + std::unique_ptr sampler = mindspore::make_unique(4, 0, false); + std::string folder_path = datasets_root_path_ + "/testPK/data"; + // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode + auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler), {}, 5)}); + tree->Prepare(); + Status rc = tree->Launch(); + int32_t labels[5] = {0, 0, 0, 1, 1}; + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_EQ(labels[i], label); + std::cout << "row:" << i++ << "\tlabel:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 5); + } +} + +TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding2) { + std::unique_ptr sampler = mindspore::make_unique(4, 3, false); + std::string folder_path = datasets_root_path_ + "/testPK/data"; + // numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode + auto tree = Build({ImageFolder(16, 16, 32, folder_path, false, std::move(sampler), {}, 12)}); + tree->Prepare(); + Status rc = tree->Launch(); + uint32_t labels[11] = {0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_EQ(labels[i], label); + std::cout << "row:" << i++ << "\tlabel:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 11); + } +} diff --git a/tests/ut/cpp/dataset/interrupt_test.cc b/tests/ut/cpp/dataset/interrupt_test.cc new file mode 100644 index 00000000000..7816346c155 --- /dev/null +++ b/tests/ut/cpp/dataset/interrupt_test.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" +#include "dataset/util/services.h" +#include "dataset/util/intrp_service.h" +#include "dataset/util/task_manager.h" +#include "dataset/util/queue.h" +#include "dataset/util/semaphore.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestIntrpService : public UT::Common { + public: + MindDataTestIntrpService() {} + + void SetUp() {} + + TaskGroup vg_; +}; + +TEST_F(MindDataTestIntrpService, Test1) { + Status rc; + Queue q(3); + q.Register(&vg_); + vg_.CreateAsyncTask("Test1", [&]() -> Status { + TaskManager::FindMe()->Post(); + int v; + Status rc; + rc = q.PopFront(&v); + EXPECT_TRUE(rc.IsInterrupted()); + return rc; + }); + vg_.GetIntrpService()->InterruptAll(); + vg_.join_all(); +} + +TEST_F(MindDataTestIntrpService, Test2) { + MS_LOG(INFO) << "Test Semaphore"; + Status rc; + Semaphore sem(0); + sem.Register(&vg_); + vg_.CreateAsyncTask("Test1", [&]() -> Status { + TaskManager::FindMe()->Post(); + Status rc = sem.P(); + EXPECT_TRUE(rc.IsInterrupted()); + return rc; + }); + vg_.GetIntrpService()->InterruptAll(); + vg_.join_all(); +} diff --git a/tests/ut/cpp/dataset/manifest_op_test.cc b/tests/ut/cpp/dataset/manifest_op_test.cc new file mode 100644 index 00000000000..e062335fa6d --- /dev/null +++ b/tests/ut/cpp/dataset/manifest_op_test.cc @@ -0,0 +1,192 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "common/common.h" +#include "common/utils.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/manifest_op.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +std::shared_ptr Repeat(int repeatCnt); + +std::shared_ptr Build(std::vector> ops); + +std::shared_ptr Manifest(int32_t num_works, int32_t rows, int32_t conns, const std::string &file, + std::string usage = "train", std::unique_ptr sampler = nullptr, + std::map map = {}, uint64_t num_samples = 0, + bool decode = false) { + std::shared_ptr so; + ManifestOp::Builder builder; + Status rc = builder.SetNumWorkers(num_works).SetManifestFile(file).SetRowsPerBuffer( + rows).SetOpConnectorSize(conns).SetSampler(std::move(sampler)).SetClassIndex(map).SetDecode(decode) + .SetNumSamples(num_samples).SetUsage(usage).Build(&so); + return so; +} + +class MindDataTestManifest : public UT::DatasetOpTesting { + protected: +}; + +TEST_F(MindDataTestManifest, TestSequentialManifestWithRepeat) { + std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; + auto tree = Build({Manifest(16, 2, 32, file), Repeat(2)}); + tree->Prepare(); + uint32_t res[] = {0, 1, 0, 1}; + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(res[i] == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 4); + } +} + +TEST_F(MindDataTestManifest, TestSubsetRandomSamplerManifest) { + std::vector indices({1}); + std::unique_ptr sampler = mindspore::make_unique(indices); + std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; + // Expect 6 samples for label 0 and 1 + auto tree = Build({Manifest(16, 2, 32, file, "train", std::move(sampler))}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + i++; + di.GetNextAsMap(&tensor_map); + EXPECT_EQ(label, 1); + } + EXPECT_TRUE(i == 1); + } +} + +TEST_F(MindDataTestManifest, MindDataTestManifestClassIndex) { + std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; + std::map map; + map["cat"] = 111; // forward slash is not good, but we need to add this somewhere, also in windows, its a '\' + map["dog"] = 222; // forward slash is not good, but we need to add this somewhere, also in windows, its a '\' + map["wrong folder name"] = 1234; // this is skipped + auto tree = Build({Manifest(16, 2, 32, file, "train", nullptr, map)}); + uint64_t res[2] = {111, 222}; + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(label == res[i]); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 2); + } +} + +TEST_F(MindDataTestManifest, MindDataTestManifestNumSamples) { + std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; + auto tree = Build({Manifest(16, 2, 32, file, "train", nullptr, {}, 1), Repeat(4)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(0 == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 4); + } +} + +TEST_F(MindDataTestManifest, MindDataTestManifestEval) { + std::string file = datasets_root_path_ + "/testManifestData/cpp.json"; + auto tree = Build({Manifest(16, 2, 32, file, "eval", nullptr, {}, 1)}); + tree->Prepare(); + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(0 == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 1); + } +} diff --git a/tests/ut/cpp/dataset/map_op_test.cc b/tests/ut/cpp/dataset/map_op_test.cc new file mode 100644 index 00000000000..746c1e8d5f1 --- /dev/null +++ b/tests/ut/cpp/dataset/map_op_test.cc @@ -0,0 +1,814 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common.h" +#include "dataset/core/client.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/datasetops/source/image_folder_op.h" +#include "dataset/kernels/image/decode_op.h" +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/tensor_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::LogStream; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace dataset { +namespace test { +class NoOp : public TensorOp { + public: + NoOp() {}; + + ~NoOp() {}; + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override { + *output = std::move(input); + return Status::OK(); + }; + + void Print(std::ostream &out) const override { out << "NoOp"; }; +}; + +class ThreeToOneOp : public TensorOp { + public: + ThreeToOneOp() {}; + + ~ThreeToOneOp() {}; + + uint32_t NumInput() override { return 3; } + // Compute function that holds the actual implementation of the operation. + Status Compute(const std::vector> &input, + std::vector> *output) override { + output->push_back(input[0]); + return Status::OK(); + }; + + void Print(std::ostream &out) const override { out << "ThreeToOneOp"; }; +}; + +class OneToThreeOp : public TensorOp { + public: + OneToThreeOp() {}; + + ~OneToThreeOp() {}; + + uint32_t NumOutput() override { return 3; } + + // Compute function that holds the actual implementation of the operation. + // Simply pushing the same shared pointer of the first element of input vector three times. + Status Compute(const std::vector> &input, + std::vector> *output) override { + output->push_back(input[0]); + output->push_back(input[0]); + output->push_back(input[0]); + return Status::OK(); + }; + + void Print(std::ostream &out) const override { out << "OneToThreeOp"; }; +}; +} // namespace test +} // namespace dataset +} // namespace mindspore + + +class MindDataTestMapOp : public UT::DatasetOpTesting { + public: + void SetUp() override { + DatasetOpTesting::SetUp(); + dataset_path_ = datasets_root_path_ + "" + "/testDataset2"; + + GlobalInit(); + + // Start with an empty execution tree + my_tree_ = std::make_shared(); + } + + std::shared_ptr CreateStorageOp() { + std::shared_ptr my_storage_op; + StorageOp::Builder builder; + builder.SetDatasetFilesDir(dataset_path_) + .SetColumnsToLoad({"image", "label", "A", "B"}) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(2) + .SetNumWorkers(2); + Status rc = builder.Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + return my_storage_op; + } + + std::shared_ptr my_tree_; + private: + std::string dataset_path_; +}; + +std::shared_ptr ImageFolder(int64_t num_works, int64_t rows, int64_t conns, std::string path, + bool shuf = false, std::unique_ptr sampler = nullptr, + std::map map = {}, int64_t num_samples = 0, + bool decode = false); + +std::shared_ptr Build(std::vector> ops); + +// TestByPosition scenario: +// StorageOp reads a dataset that have column ordering |image|label|A|B|. +// A TensorOp that does nothing picks the label column and output a column also named label. +// Thus, based on the new MapOp behaviour, the column ordering will be |image|label|A|B|. +// Verify the column ordering based on the Tensor properties matching to that of in the schema file. +TEST_F(MindDataTestMapOp, TestByPosition) { + Status rc; + MS_LOG(INFO) << "Doing TestByPosition."; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total + // of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_no_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_no_op); + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"label"}) + .SetOutColNames({}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(100); + rc = builder.Build(&my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_map_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssignRoot(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + + // Based on the schema file, create the golden result to compare with. + std::vector golden_types({ + DataType::Type::DE_UINT8, + DataType::Type::DE_INT64, + DataType::Type::DE_FLOAT32, + DataType::Type::DE_INT64} + ); + + std::vector golden_ranks({3, 1, 4, 1}); + + std::vector golden_shapes({ + TensorShape({3, 4, 2}), + TensorShape({7}), + TensorShape({1, 13, 14, 12}), + TensorShape({9})} + ); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(tensor_list.size(), 4); + for (uint32_t i = 0; i < tensor_list.size(); i++) { + EXPECT_EQ(tensor_list[i]->type(), golden_types[i]); + EXPECT_EQ(tensor_list[i]->Rank(), golden_ranks[i]); + EXPECT_EQ(tensor_list[i]->shape(), golden_shapes[i]); + EXPECT_NE(tensor_list[i]->StartAddr(), nullptr); + } +} + +// TestAsMap scenario: +// StorageOp reads a dataset that have column ordering |image|label|A|B|. +// A TensorOp that does nothing picks the "image" column and produces a column named "X". +// Thus, based on the new MapOp behaviour, the column ordering will be |X|label|A|B|. +// Verify that the "image" column is removed and "X" column is added. +TEST_F(MindDataTestMapOp, TestAsMap) { + Status rc; + MS_LOG(INFO) << "Doing TestAsMap."; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_no_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_no_op); + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"image"}) + .SetOutColNames({"X"}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(1); + rc = builder.Build(&my_map_op); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_map_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + + // Assign the tree root + rc = my_tree_->AssignRoot(my_map_op); + EXPECT_TRUE(rc.IsOk()); + + // Now prepare the tree + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(tensor_map.size(), 4); + EXPECT_EQ(tensor_map.find("image"), tensor_map.end()); + EXPECT_NE(tensor_map.find("label"), tensor_map.end()); + EXPECT_NE(tensor_map.find("X"), tensor_map.end()); + EXPECT_NE(tensor_map.find("A"), tensor_map.end()); + EXPECT_NE(tensor_map.find("B"), tensor_map.end()); +} + +// Test3to1 scenario: +// StorageOp reads a dataset that have column ordering |image|label|A|B|. +// A 3-to-1 TensorOp picks the columns [image, A, B] and produce a column named "X". +// Thus, based on the new MapOp behaviour, the column ordering will be |X|label|. +// Verify that the only columns "X" and "label" exist. +TEST_F(MindDataTestMapOp, Test3to1) { + Status rc; + MS_LOG(INFO) << "Doing Test3to1."; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_op); + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"image", "A", "B"}) + .SetOutColNames({"X"}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(1); + rc = builder.Build(&my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_map_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssignRoot(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + while (!tensor_map.empty()) { + EXPECT_EQ(tensor_map.size(), 2); + EXPECT_EQ(tensor_map.find("image"), tensor_map.end()); + EXPECT_NE(tensor_map.find("label"), tensor_map.end()); + EXPECT_NE(tensor_map.find("X"), tensor_map.end()); + EXPECT_EQ(tensor_map.find("A"), tensor_map.end()); + EXPECT_EQ(tensor_map.find("B"), tensor_map.end()); + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + } +} + +// Test1to3 scenario: +// StorageOp reads a dataset that have column ordering |image|label|A|B|. +// A 1-to-3 TensorOp picks the columns [image] and produce a column named [X, Y, Z]. +// Thus, based on the new MapOp behaviour, the column ordering will be |X|Y|Z|label|A|B|. +// Verify that the only columns X, Y, Z are added (to the front) and followed by columns label, A, B.. +TEST_F(MindDataTestMapOp, Test1to3) { + Status rc; + MS_LOG(INFO) << "Doing Test1to3."; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_op); + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"image"}) + .SetOutColNames({"X", "Y", "Z"}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(1); + rc = builder.Build(&my_map_op); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_map_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssignRoot(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(tensor_map.size(), 6); + EXPECT_EQ(tensor_map.find("image"), tensor_map.end()); + EXPECT_NE(tensor_map.find("label"), tensor_map.end()); + EXPECT_NE(tensor_map.find("A"), tensor_map.end()); + EXPECT_NE(tensor_map.find("B"), tensor_map.end()); + EXPECT_NE(tensor_map.find("X"), tensor_map.end()); + EXPECT_NE(tensor_map.find("Y"), tensor_map.end()); + EXPECT_NE(tensor_map.find("Z"), tensor_map.end()); + + // Getting the next row as vector (by position). + TensorRow tensor_list; + rc =di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + + // Based on the schema file, create the golden result to compare with. + std::vector golden_types({DataType::Type::DE_UINT8, DataType::Type::DE_UINT8, + DataType::Type::DE_UINT8, DataType::Type::DE_INT64, + DataType::Type::DE_FLOAT32, DataType::Type::DE_INT64} + ); + + std::vector golden_ranks({3, 3, 3, 1, 4, 1}); + + std::vector golden_shapes({TensorShape({3, 4, 2}), TensorShape({3, 4, 2}), TensorShape({3, 4, 2}), + TensorShape({7}), TensorShape({1, 13, 14, 12}), TensorShape({9})} ); + + while (!tensor_list.empty()) { + for (uint32_t i = 0; i < tensor_list.size(); i++) { + EXPECT_EQ(tensor_list[i]->type(), golden_types[i]); + EXPECT_EQ(tensor_list[i]->Rank(), golden_ranks[i]); + EXPECT_EQ(tensor_list[i]->shape(), golden_shapes[i]); + EXPECT_NE(tensor_list[i]->StartAddr(), nullptr); + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + } +} + +// TestMultiTensorOp scenario: +// StorageOp reads a dataset that have column ordering |image|label|A|B|. +// A series of 3-to-1 and 1-to-3 TensorOps are applied to [image, A, B] and +// produce final output columns [X, Y, Z]. +// Based on the new MapOp behaviour, the column ordering will be |X|Y|Z|label|. +TEST_F(MindDataTestMapOp, TestMultiTensorOp) { + Status rc; + MS_LOG(INFO) << "Doing TestMultiTensorOp."; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_op1 = std::make_shared(); + auto my_op2 = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_op1); + my_func_list.push_back(my_op2); + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"image", "A", "B"}) + .SetOutColNames({"X", "Y", "Z"}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(1); + rc = builder.Build(&my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_map_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssignRoot(my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorMap tensor_map; + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + while (!tensor_map.empty()) { + EXPECT_EQ(tensor_map.size(), 4); + EXPECT_EQ(tensor_map.find("image"), tensor_map.end()); + EXPECT_EQ(tensor_map.find("A"), tensor_map.end()); + EXPECT_EQ(tensor_map.find("B"), tensor_map.end()); + EXPECT_NE(tensor_map.find("label"), tensor_map.end()); + EXPECT_NE(tensor_map.find("X"), tensor_map.end()); + EXPECT_NE(tensor_map.find("Y"), tensor_map.end()); + EXPECT_NE(tensor_map.find("Z"), tensor_map.end()); + + // XYZ are Tensor shared_ptr to image, so it should have the same shape as image column. + EXPECT_EQ(tensor_map["X"]->shape(), TensorShape({3, 4, 2})); + EXPECT_EQ(tensor_map["Y"]->shape(), TensorShape({3, 4, 2})); + EXPECT_EQ(tensor_map["Z"]->shape(), TensorShape({3, 4, 2})); + rc = di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + } +} + +TEST_F(MindDataTestMapOp, TestStorageRepeatMap) { + Status rc; + MS_LOG(INFO) << "Doing TestStorageRepeatMap."; + uint32_t num_repeats = 3; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total + // of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_no_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_no_op); + + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(num_repeats).Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"label"}) + .SetOutColNames({}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(5); + rc = builder.Build(&my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_map_op->AddChild(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_repeat_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->AssignRoot(my_map_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(tensor_list.size(), 4); + uint32_t row_count = 0; + while (!tensor_list.empty()) { + row_count++; + MS_LOG(INFO) << "row_count: " << row_count << "."; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + + } + ASSERT_EQ(row_count, 10 * num_repeats); +} + +TEST_F(MindDataTestMapOp, TestStorageMapRepeat) { + Status rc; + MS_LOG(INFO) << "Doing TestStorageMapRepeat."; + uint32_t num_repeats = 3; + + // Note: The above storage config yields 5 buffers, each with 2 rows, for a total + // of 10 rows. + auto my_storage_op = this->CreateStorageOp(); + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto my_no_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(my_no_op); + + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(num_repeats).Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + std::shared_ptr my_map_op; + MapOp::Builder builder; + builder.SetInColNames({"label"}) + .SetOutColNames({}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(50); + rc = builder.Build(&my_map_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_repeat_op->AddChild(my_map_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_map_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->AssignRoot(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(tensor_list.size(), 4); + uint32_t row_count = 0; + while (!tensor_list.empty()) { + row_count++; + MS_LOG(INFO) << "row_count: " << row_count << "."; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + } + ASSERT_EQ(row_count, 10 * num_repeats); +} + +TEST_F(MindDataTestMapOp, Storage_Decode_Repeat_Resize) { + Status rc; + MS_LOG(INFO) << "Doing Storage_Decode_Repeat_Resize."; + uint32_t num_repeats = 2; + + std::string dataset_path_ = datasets_root_path_ + "/" + "test_tf_file_3_images"; + std::shared_ptr my_storage_op; + StorageOp::Builder sobuilder; + sobuilder.SetDatasetFilesDir(dataset_path_) + .SetColumnsToLoad({"image", "label"}) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(2) + .SetNumWorkers(2); + rc = sobuilder.Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + auto decode_op = std::make_shared(); + std::vector> my_func_list; + my_func_list.push_back(decode_op); + + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(num_repeats).Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + std::shared_ptr my_map_decode_op; + MapOp::Builder builder; + builder.SetInColNames({"image"}) + .SetOutColNames({}) + .SetTensorFuncs(std::move(my_func_list)) + .SetNumWorkers(4); + rc = builder.Build(&my_map_decode_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_decode_op); + EXPECT_TRUE(rc.IsOk()); + + + auto resize_op = std::make_shared(300, 300); + std::vector> my_func_list2; + my_func_list2.push_back(resize_op); + std::shared_ptr my_map_resize_op; + MapOp::Builder builder2; + builder2.SetInColNames({"image"}) + .SetOutColNames({}) + .SetTensorFuncs(std::move(my_func_list2)) + .SetNumWorkers(5); + rc = builder2.Build(&my_map_resize_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->AssociateNode(my_map_resize_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_map_decode_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_repeat_op->AddChild(my_map_decode_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_map_resize_op->AddChild(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->AssignRoot(my_map_resize_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + EXPECT_EQ(tensor_list.size(), 2); + uint32_t row_count = 0; + while (!tensor_list.empty()) { + row_count++; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + } + + ASSERT_EQ(row_count, 6); +} + +TEST_F(MindDataTestMapOp, ImageFolder_Decode_Repeat_Resize) { + Status rc; + MS_LOG(INFO) << "Doing ImageFolder_Decode_Repeat_Resize."; + + std::string folder_path = datasets_root_path_ + "/testPK/data"; + + uint32_t num_repeats = 2; + std::shared_ptr repeat_op; + rc = RepeatOp::Builder(num_repeats).Build(&repeat_op); + EXPECT_TRUE(rc.IsOk()); + + auto decode_op = std::make_shared(); + std::vector> func_list; + func_list.push_back(decode_op); + + std::shared_ptr map_decode_map; + MapOp::Builder map_decode_builder; + map_decode_builder.SetInColNames({"image"}) + .SetOutColNames({}) + .SetTensorFuncs(func_list) + .SetNumWorkers(4); + rc = map_decode_builder.Build(&map_decode_map); + EXPECT_TRUE(rc.IsOk()); + + auto resize_op = std::make_shared(300, 300); + std::vector> func_list2; + func_list2.push_back(resize_op); + std::shared_ptr map_resize_op; + MapOp::Builder map_resize_builder; + map_resize_builder.SetInColNames({"image"}) + .SetOutColNames({}) + .SetTensorFuncs(func_list2) + .SetNumWorkers(5); + rc = map_resize_builder.Build(&map_resize_op); + EXPECT_TRUE(rc.IsOk()); + + my_tree_ = Build({ImageFolder(16, 2, 32, folder_path, false), map_decode_map, repeat_op, map_resize_op}); + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + int32_t img_class[] = {0, 1, 2, 3}; + std::string result; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + std::cout << "row:" << i << "\tlabel:" << label << "\n"; + EXPECT_TRUE(img_class[(i % 44) / 11] == label); + // Dump all the image into string, to be used as a comparison later. + result.append((char *) tensor_map["image"]->StartAddr(), (int64_t) tensor_map["image"]->Size()); + di.GetNextAsMap(&tensor_map); + i++; + } + EXPECT_TRUE(i == 88); + + // Part-2 : creating mapop with performance mode = false, to check if the result is the same + // as when performance mode = true. + rc = RepeatOp::Builder(num_repeats).Build(&repeat_op); + EXPECT_TRUE(rc.IsOk()); + + map_decode_builder.SetInColNames({"image"}) + .SetOutColNames({}) + .SetTensorFuncs(func_list) + .SetNumWorkers(14) + .SetPerformanceMode(false); + rc = map_decode_builder.Build(&map_decode_map); + EXPECT_TRUE(rc.IsOk()); + + map_resize_builder.SetInColNames({"image"}) + .SetOutColNames({}) + .SetTensorFuncs(func_list2) + .SetNumWorkers(15) + .SetPerformanceMode(false); + rc = map_resize_builder.Build(&map_resize_op); + EXPECT_TRUE(rc.IsOk()); + + auto my_tree_2 = Build({ImageFolder(16, 2, 32, folder_path, false), map_decode_map, repeat_op, map_resize_op}); + + rc = my_tree_2->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_2->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di2(my_tree_2); + di2.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + i = 0; + label = 0; + std::string result2; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + std::cout << "row:" << i << "\tlabel:" << label << "\n"; + EXPECT_TRUE(img_class[(i % 44) / 11] == label); + result2.append((char *) tensor_map["image"]->StartAddr(), (int64_t) tensor_map["image"]->Size()); + di2.GetNextAsMap(&tensor_map); + i++; + } + EXPECT_TRUE(i == 88); + + EXPECT_EQ(result.size(), result2.size()); + EXPECT_EQ(result, result2); +} + + +TEST_F(MindDataTestMapOp, ImageFolder_Decode_Repeat_Resize_NoInputColumns) { + Status rc; + MS_LOG(INFO) << "Doing ImageFolder_Decode_Repeat_Resize_NoInputColumns."; + + std::string folder_path = datasets_root_path_ + "/testPK/data"; + + uint32_t num_repeats = 2; + std::shared_ptr repeat_op; + rc = RepeatOp::Builder(num_repeats).Build(&repeat_op); + EXPECT_TRUE(rc.IsOk()); + + auto decode_op = std::make_shared(); + std::vector> func_list; + func_list.push_back(decode_op); + + std::shared_ptr map_decode_map; + MapOp::Builder map_decode_builder; + map_decode_builder.SetInColNames({}) + .SetOutColNames({}) + .SetTensorFuncs(func_list) + .SetNumWorkers(4); + rc = map_decode_builder.Build(&map_decode_map); + EXPECT_TRUE(rc.IsOk()); + + auto resize_op = std::make_shared(300, 300); + std::vector> func_list2; + func_list2.push_back(resize_op); + std::shared_ptr map_resize_op; + MapOp::Builder map_resize_builder; + map_resize_builder.SetTensorFuncs(func_list2).SetNumWorkers(5); + rc = map_resize_builder.Build(&map_resize_op); + EXPECT_TRUE(rc.IsOk()); + + my_tree_ = Build({ImageFolder(16, 2, 32, folder_path, false), map_decode_map, repeat_op, map_resize_op}); + rc = my_tree_->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree_->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree_); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + int32_t label = 0; + int32_t img_class[] = {0, 1, 2, 3}; + std::string result; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(img_class[(i % 44) / 11] == label); + di.GetNextAsMap(&tensor_map); + i++; + } + EXPECT_TRUE(i == 88); +} diff --git a/tests/ut/cpp/dataset/memory_pool_test.cc b/tests/ut/cpp/dataset/memory_pool_test.cc new file mode 100644 index 00000000000..c988a09e898 --- /dev/null +++ b/tests/ut/cpp/dataset/memory_pool_test.cc @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/util/memory_pool.h" +#include "dataset/util/circular_pool.h" +#include "dataset/util/system_pool.h" +#include "dataset/util/allocator.h" +#include "common/common.h" +#include "gtest/gtest.h" + +using namespace mindspore::dataset; + +class MindDataTestMemoryPool : public UT::Common { + public: + std::shared_ptr mp_; + MindDataTestMemoryPool() {} + + void SetUp() { + Status rc = CircularPool::CreateCircularPool(&mp_, 1, 1, true); + ASSERT_TRUE(rc.IsOk()); + } +}; + +TEST_F(MindDataTestMemoryPool, DumpPoolInfo) { + std::cout << *(std::dynamic_pointer_cast(mp_)) << std::endl; +} + +TEST_F(MindDataTestMemoryPool, TestOperator1) { + Status rc; + int *p = new(&rc, mp_) int; + ASSERT_TRUE(rc.IsOk()); + *p = 2048; + ::operator delete(p, mp_); +} + +TEST_F(MindDataTestMemoryPool, TestOperator3) { + Status rc; + int *p = new (&rc, mp_) int[100]; + ASSERT_TRUE(rc.IsOk()); + for (int i = 0; i < 100; i++) { + p[i] = i; + } + for (int i = 0; i < 100; i++) { + ASSERT_EQ(p[i], i); + } +} + +TEST_F(MindDataTestMemoryPool, TestAllocator) { + class A { + public: + explicit A (int x) : a(x) {} + int val_a() const { + return a; + } + private: + int a; + }; + Allocator alloc(mp_); + std::shared_ptr obj_a = std::allocate_shared(alloc, 3); + int v = obj_a->val_a(); + ASSERT_EQ(v, 3); + std::cout << *(std::dynamic_pointer_cast(mp_)) << std::endl; +} diff --git a/tests/ut/cpp/dataset/mind_record_op_test.cc b/tests/ut/cpp/dataset/mind_record_op_test.cc new file mode 100644 index 00000000000..9664b66ecbd --- /dev/null +++ b/tests/ut/cpp/dataset/mind_record_op_test.cc @@ -0,0 +1,483 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef ENABLE_MINDRECORD +#include +#include +#include +#include "dataset/core/client.h" +#include "common/common.h" +#include "common/utils.h" +#include "gtest/gtest.h" +#include "mindrecord/include/shard_category.h" +#include "mindrecord/include/shard_sample.h" +#include "mindrecord/include/shard_shuffle.h" +#include "utils/log_adapter.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestMindRecordOp : public UT::DatasetOpTesting { +}; + +TEST_F(MindDataTestMindRecordOp, TestMindRecordBasic) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordBasic"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetColumnsToLoad(column_list); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsOk()); + + std::cout << (*my_mindrecord_op); + + my_tree->AssociateNode(my_mindrecord_op); + + // Set children/root layout. + my_tree->AssignRoot(my_mindrecord_op); + + MS_LOG(INFO) << "Launching tree and begin iteration"; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << (*tensor_list[i]) << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()); + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } +} + +TEST_F(MindDataTestMindRecordOp, TestMindRecordSample) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordSample"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::vector> operators; + operators.push_back(std::make_shared(4)); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetColumnsToLoad(column_list) + .SetOperators(operators); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsOk()); + + std::cout << (*my_mindrecord_op); + + my_tree->AssociateNode(my_mindrecord_op); + + // Set children/root layout. + my_tree->AssignRoot(my_mindrecord_op); + + MS_LOG(INFO) << "Launching tree and begin iteration"; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << (*tensor_list[i]) << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()); + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } +} + +TEST_F(MindDataTestMindRecordOp, TestMindRecordShuffle) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordShuffle"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::vector> operators; + operators.push_back(std::make_shared(1)); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetColumnsToLoad(column_list) + .SetOperators(operators); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsOk()); + + std::cout << (*my_mindrecord_op); + + my_tree->AssociateNode(my_mindrecord_op); + + // Set children/root layout. + my_tree->AssignRoot(my_mindrecord_op); + + MS_LOG(INFO) << "Launching tree and begin iteration"; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << (*tensor_list[i]) << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()); + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } +} + +TEST_F(MindDataTestMindRecordOp, TestMindRecordCategory) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordCategory"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::vector> operators; + std::vector> categories; + categories.push_back(std::make_pair("label", "490")); + categories.push_back(std::make_pair("label", "171")); + operators.push_back(std::make_shared(categories)); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetColumnsToLoad(column_list) + .SetOperators(operators); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsOk()); + + std::cout << (*my_mindrecord_op); + + my_tree->AssociateNode(my_mindrecord_op); + + // Set children/root layout. + my_tree->AssignRoot(my_mindrecord_op); + + MS_LOG(INFO) << "Launching tree and begin iteration"; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << (*tensor_list[i]) << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()); + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } +} + +TEST_F(MindDataTestMindRecordOp, TestMindRecordRepeat) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordRepeat"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetColumnsToLoad(column_list); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsOk()); + + std::cout << (*my_mindrecord_op); + + rc = my_tree->AssociateNode(my_mindrecord_op); + EXPECT_TRUE(rc.IsOk()); + + uint32_t num_repeats = 2; + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(num_repeats) + .Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_repeat_op->AddChild(my_mindrecord_op); + EXPECT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_tree->AssignRoot(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration"; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << (*tensor_list[i]) << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()); + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } +} + + +TEST_F(MindDataTestMindRecordOp, TestMindRecordBlockReaderRepeat) { + // single MindRecord op and nothing else + // + // MindRecordOp + + MS_LOG(INFO) << "UT test TestMindRecordBlockReaderRepeat"; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + + std::vector column_list; + std::string label_col_name("file_name"); + column_list.push_back(label_col_name); + label_col_name = "label"; + column_list.push_back(label_col_name); + + std::shared_ptr my_mindrecord_op; + MindRecordOp::Builder builder; + builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0") + .SetRowsPerBuffer(3) + .SetNumMindRecordWorkers(4) + .SetBlockReader() + .SetColumnsToLoad(column_list); + rc = builder.Build(&my_mindrecord_op); + ASSERT_TRUE(rc.IsOk()); + + std::cout << (*my_mindrecord_op); + + rc = my_tree->AssociateNode(my_mindrecord_op); + EXPECT_TRUE(rc.IsOk()); + + uint32_t num_repeats = 2; + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(num_repeats) + .Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_repeat_op->AddChild(my_mindrecord_op); + EXPECT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_tree->AssignRoot(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration"; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << (*tensor_list[i]) << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()); + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } +} +#endif diff --git a/tests/ut/cpp/dataset/mnist_op_test.cc b/tests/ut/cpp/dataset/mnist_op_test.cc new file mode 100644 index 00000000000..3d5c4b05cae --- /dev/null +++ b/tests/ut/cpp/dataset/mnist_op_test.cc @@ -0,0 +1,128 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "common/utils.h" +#include "common/common.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/mnist_op.h" +#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h" +#include "dataset/engine/datasetops/source/sampler/pk_sampler.h" +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +namespace common = mindspore::common; +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +std::shared_ptr Batch(int batch_size = 1, bool drop = false, int rows_per_buf = 2); + +std::shared_ptr Repeat(int repeat_cnt); + +std::shared_ptr Build(std::vector> ops); + +Status Create1DTensor(std::shared_ptr *sample_ids, int64_t num_elements, unsigned char *data = nullptr, + DataType::Type data_type = DataType::DE_UINT32); + +std::shared_ptr CreateMnist(int64_t num_wrks, int64_t rows, int64_t conns, std::string path, + bool shuf = false, std::unique_ptr sampler = nullptr, + int64_t num_samples = 0) { + std::shared_ptr so; + MnistOp::Builder builder; + Status rc = builder.SetNumWorkers(num_wrks).SetDir(path).SetRowsPerBuffer(rows) + .SetOpConnectorSize(conns).SetSampler(std::move(sampler)) + .SetNumSamples(num_samples).Build(&so); + return so; +} + +class MindDataTestMnistSampler : public UT::DatasetOpTesting { + protected: +}; + +TEST_F(MindDataTestMnistSampler, TestSequentialMnistWithRepeat) { + // Note: Mnist datasets are not included + // as part of the build tree. + // Download datasets and rebuild if data doesn't + // appear in this dataset + // Example: python tests/dataset/data/prep_data.py + std::string folder_path = datasets_root_path_ + "/testMnistData/"; + auto tree = Build({CreateMnist(16, 2, 32, folder_path, false, nullptr, 10), Repeat(2)}); + tree->Prepare(); + uint32_t res[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + uint32_t label = 0; + while (tensor_map.size() != 0) { + tensor_map["label"]->GetItemAt(&label, {}); + EXPECT_TRUE(res[i % 10] == label); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 20); + } +} + +TEST_F(MindDataTestMnistSampler, TestSequentialImageFolderWithRepeatBatch) { + std::string folder_path = datasets_root_path_ + "/testMnistData/"; + auto tree = Build({CreateMnist(16, 2, 32, folder_path, false, nullptr, 10), Repeat(2), Batch(5)}); + tree->Prepare(); + uint32_t res[4][5] = { {0, 0, 0, 0, 0 }, + {0, 0, 0, 0, 0 }, + {0, 0, 0, 0, 0 }, + {0, 0, 0, 0, 0 } }; + Status rc = tree->Launch(); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; + EXPECT_TRUE(false); + } else { + DatasetIterator di(tree); + TensorMap tensor_map; + di.GetNextAsMap(&tensor_map); + EXPECT_TRUE(rc.IsOk()); + uint64_t i = 0; + while (tensor_map.size() != 0) { + std::shared_ptr label; + Create1DTensor(&label, 5, reinterpret_cast(res[i % 4])); + EXPECT_TRUE((*label) == (*tensor_map["label"])); + std::cout << "row: " << i++ << "\t" << tensor_map["image"]->shape() << "label:" << *tensor_map["label"] << "\n"; + di.GetNextAsMap(&tensor_map); + } + EXPECT_TRUE(i == 4); + } +} diff --git a/tests/ut/cpp/dataset/normalize_op_test.cc b/tests/ut/cpp/dataset/normalize_op_test.cc new file mode 100644 index 00000000000..373669a4734 --- /dev/null +++ b/tests/ut/cpp/dataset/normalize_op_test.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/normalize_op.h" +#include "dataset/core/cv_tensor.h" +#include "utils/log_adapter.h" +#include + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestNormalizeOP : public UT::CVOP::CVOpCommon { + public: + MindDataTestNormalizeOP() : CVOpCommon() {} +}; + +TEST_F(MindDataTestNormalizeOP, TestOp) { + MS_LOG(INFO) << "Doing TestNormalizeOp::TestOp2."; + std::shared_ptr output_tensor; + + // Numbers are from the resnet50 model implementation + float mean[3] = {121.0, 115.0, 100.0}; + float std[3] = {70.0, 68.0, 71.0}; + + // Normalize Op + std::unique_ptr op(new NormalizeOp(mean[0], mean[1], mean[2], std[0], std[1], std[2])); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + + std::string output_filename = GetFilename(); + output_filename.replace(output_filename.end() - 8, output_filename.end(), "imagefolder/normalizeOpOut.yml"); + + std::shared_ptr p = CVTensor::AsCVTensor(output_tensor); + cv::Mat cv_output_image; + cv_output_image = p->mat(); + + std::cout << "Storing output file to : " << output_filename << std::endl; + cv::FileStorage file(output_filename, cv::FileStorage::WRITE); + file << "imageData" << cv_output_image; +} diff --git a/tests/ut/cpp/dataset/one_hot_op_test.cc b/tests/ut/cpp/dataset/one_hot_op_test.cc new file mode 100644 index 00000000000..3928aeffd42 --- /dev/null +++ b/tests/ut/cpp/dataset/one_hot_op_test.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/data/one_hot_op.h" +#include "dataset/core/cv_tensor.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestOneHotOp : public UT::CVOP::CVOpCommon { + protected: + MindDataTestOneHotOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestOneHotOp, TestOp) { + MS_LOG(INFO) << "Doing MindDataTestOneHotOp."; + uint64_t labels[3] = {0, 1, 2}; + TensorShape shape({3}); + std::shared_ptr input = std::make_shared(shape, DataType(DataType::DE_UINT64), + reinterpret_cast (labels)); + std::shared_ptr output; + + std::unique_ptr op(new OneHotOp(5)); + Status s = op->Compute(input, &output); + uint64_t out[15] = {1, 0, 0, 0, 0, + 0, 1, 0, 0, 0, + 0, 0, 1, 0, 0}; + std::shared_ptr expected = std::make_shared(TensorShape{3, 5}, DataType(DataType::DE_UINT64), + reinterpret_cast (out)); + EXPECT_TRUE(s.IsOk()); + ASSERT_TRUE(output->shape() == expected->shape()); + ASSERT_TRUE(output->type() == expected->type()); + std::cout << *output << std::endl; + std::cout << *expected << std::endl; + + ASSERT_TRUE(*output == *expected); + MS_LOG(INFO) << "MindDataTestOneHotOp end."; +} diff --git a/tests/ut/cpp/dataset/path_test.cc b/tests/ut/cpp/dataset/path_test.cc new file mode 100644 index 00000000000..7bd2abe7e57 --- /dev/null +++ b/tests/ut/cpp/dataset/path_test.cc @@ -0,0 +1,120 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/path.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" +#include + +using namespace mindspore::dataset; + +class MindDataTestPath : public UT::Common { + public: + MindDataTestPath() {} +}; + +TEST_F(MindDataTestPath, Test1) { + Path f("/tmp"); + ASSERT_TRUE(f.Exists()); + ASSERT_TRUE(f.IsDirectory()); + ASSERT_EQ(f.ParentPath(), "/"); + // Print out the first few items in the directory + auto dir_it = Path::DirIterator::OpenDirectory(&f); + ASSERT_NE(dir_it.get(), nullptr); + int i = 0; + while (dir_it->hasNext()) { + Path v = dir_it->next(); + std::cout << v.toString() << "\n"; + i++; + if (i == 10) { + break; + } + } + // Test extension. + Path g("file.jpeg"); + std::cout << g.Extension() << "\n"; + ASSERT_EQ(g.Extension(), ".jpeg"); +} + +TEST_F(MindDataTestPath, Test2) { + Path p("/tmp"); + Path p2(p); + ASSERT_TRUE(p2.Exists()); + ASSERT_TRUE(p2.IsDirectory()); + ASSERT_EQ(p2.ParentPath(), "/"); + + p2 = p; + ASSERT_TRUE(p2.Exists()); + ASSERT_TRUE(p2.IsDirectory()); + ASSERT_EQ(p2.ParentPath(), "/"); + + Path p3(""); + p3 = std::move(p2); + ASSERT_TRUE(p3.Exists()); + ASSERT_TRUE(p3.IsDirectory()); + ASSERT_EQ(p3.ParentPath(), "/"); + + Path p4(std::move(p3)); + ASSERT_TRUE(p4.Exists()); + ASSERT_TRUE(p4.IsDirectory()); + ASSERT_EQ(p4.ParentPath(), "/"); + + Path p5("/"); + std::string s = "tmp"; + ASSERT_TRUE((p5 + "tmp").Exists()); + ASSERT_TRUE((p5 + "tmp").IsDirectory()); + ASSERT_EQ((p5 + "tmp").ParentPath(), "/"); + + ASSERT_TRUE((p5 / "tmp").Exists()); + ASSERT_TRUE((p5 / "tmp").IsDirectory()); + ASSERT_EQ((p5 / "tmp").ParentPath(), "/"); + + ASSERT_TRUE((p5 + s).Exists()); + ASSERT_TRUE((p5 + s).IsDirectory()); + ASSERT_EQ((p5 + s).ParentPath(), "/"); + + ASSERT_TRUE((p5 / s).Exists()); + ASSERT_TRUE((p5 / s).IsDirectory()); + ASSERT_EQ((p5 / s).ParentPath(), "/"); + + Path p6("tmp"); + ASSERT_TRUE((p5 + p6).Exists()); + ASSERT_TRUE((p5 + p6).IsDirectory()); + ASSERT_EQ((p5 + p6).ParentPath(), "/"); + ASSERT_TRUE((p5 / p6).Exists()); + ASSERT_TRUE((p5 / p6).IsDirectory()); + ASSERT_EQ((p5 / p6).ParentPath(), "/"); + p5 += p6; + ASSERT_TRUE(p5.Exists()); + ASSERT_TRUE(p5.IsDirectory()); + ASSERT_EQ(p5.ParentPath(), "/"); + + Path p7("/"); + Path p8(p7); + p7 += s; + p8 += "tmp"; + ASSERT_TRUE(p7.Exists()); + ASSERT_TRUE(p7.IsDirectory()); + ASSERT_EQ(p7.ParentPath(), "/"); + ASSERT_TRUE(p8.Exists()); + ASSERT_TRUE(p8.IsDirectory()); + ASSERT_EQ(p8.ParentPath(), "/"); + + Path p9("/tmp/test_path"); + ASSERT_TRUE(p9.CreateDirectories().IsOk()); + ASSERT_EQ(remove("/tmp/test_path"), 0); +} diff --git a/tests/ut/cpp/dataset/project_op_test.cc b/tests/ut/cpp/dataset/project_op_test.cc new file mode 100644 index 00000000000..1df2ce05bb1 --- /dev/null +++ b/tests/ut/cpp/dataset/project_op_test.cc @@ -0,0 +1,100 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common.h" +#include "common/utils.h" +#include "dataset/core/client.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestProjectOp : public UT::DatasetOpTesting {}; + +TEST_F(MindDataTestProjectOp, TestProjectProject) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(16) + .SetWorkerConnectorSize(16) + .SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + // ProjectOp + std::vector columns_to_project = {"col_sint16", "col_float", "col_2d"}; + std::shared_ptr my_project_op = std::make_shared(columns_to_project); + rc = my_tree->AssociateNode(my_project_op); + ASSERT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_project_op->AddChild(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_project_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + ASSERT_EQ(tensor_list.size(), columns_to_project.size()); + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} diff --git a/tests/ut/cpp/dataset/queue_test.cc b/tests/ut/cpp/dataset/queue_test.cc new file mode 100644 index 00000000000..37560e5dfb5 --- /dev/null +++ b/tests/ut/cpp/dataset/queue_test.cc @@ -0,0 +1,174 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// +// Created by jesse on 10/3/19. +// + +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/task_manager.h" +#include "dataset/util/queue.h" +#include +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestQueue : public UT::Common { + public: + MindDataTestQueue() {} + + void SetUp() {} +}; + +int gRefCountDestructorCalled; + +class RefCount { + public: + RefCount() : v_(nullptr) {} + explicit RefCount(int x) : v_(std::make_shared(x)) {} + explicit RefCount(const RefCount &o) : v_(o.v_) {} + ~RefCount() { + std::cout << "Destructor of RefCount called" << std::endl; + gRefCountDestructorCalled++; + } + RefCount& operator=(const RefCount &o) { + v_ = o.v_; + return *this; + } + + std::shared_ptr v_; +}; + +TEST_F(MindDataTestQueue, Test1) { + // Passing shared pointer along the queue + Queue> que(3); + std::shared_ptr a = std::make_shared(20); + Status rc = que.Add(a); + ASSERT_TRUE(rc.IsOk()); + // Use count should be 2 right now. a plus the one in the queue. + ASSERT_EQ(a.use_count(), 2); + std::shared_ptr b; + rc = que.PopFront(&b); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(*b, 20); + // Use count should remain 2. a and b. No copy in the queue. + ASSERT_EQ(a.use_count(), 2); + a.reset(new int(5)); + ASSERT_EQ(a.use_count(),1); + // Push again but expect a is nullptr after push + rc = que.Add(std::move(a)); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(a.use_count(),0); + rc = que.PopFront(&b); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(*b, 5); + ASSERT_EQ(b.use_count(),1); + // Test construct in place + rc = que.EmplaceBack(std::make_shared(100)); + ASSERT_TRUE(rc.IsOk()); + rc = que.PopFront(&b); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(*b, 100); + ASSERT_EQ(b.use_count(),1); + // Test the destructor of the Queue by add an element in the queue without popping it and let the queue go + // out of scope. + rc = que.EmplaceBack(std::make_shared(2000)); + ASSERT_TRUE(rc.IsOk()); +} + +TEST_F(MindDataTestQueue, Test2) { + // Passing status object + Queue que(3); + Status rc_send(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Oops"); + Status rc = que.Add(rc_send); + ASSERT_TRUE(rc.IsOk()); + Status rc_recv; + rc = que.PopFront(&rc_recv); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(rc_recv, rc_send); + rc = que.EmplaceBack(StatusCode::kOutOfMemory, "Test emplace"); + ASSERT_TRUE(rc.IsOk()); + Status rc_recv2; + rc = que.PopFront(&rc_recv2); + ASSERT_TRUE(rc.IsOk()); + ASSERT_TRUE(rc_recv2.IsOutofMemory()); +} + +TEST_F(MindDataTestQueue, Test3) { + Queue> que(3); + std::unique_ptr a(new int(3)); + Status rc = que.Add(std::move(a)); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(a.get(), nullptr); + std::unique_ptr b; + rc = que.PopFront(&b); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(*b, 3); + rc = que.EmplaceBack(new int(40)); + ASSERT_TRUE(rc.IsOk()); + rc = que.PopFront(&b); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(*b, 40); +} + +void test4(){ + gRefCountDestructorCalled = 0; + // Pass a structure along the queue. + Queue que(3); + RefCount a(3); + Status rc = que.Add(a); + ASSERT_TRUE(rc.IsOk()); + RefCount b; + rc = que.PopFront(&b); + ASSERT_TRUE(rc.IsOk()); + ASSERT_EQ(b.v_.use_count(), 2); + ASSERT_EQ(*(b.v_.get()), 3); + // Test the destructor of the Queue by adding an element without popping. + rc = que.EmplaceBack(10); + ASSERT_TRUE(rc.IsOk()); +} + +TEST_F(MindDataTestQueue, Test4) { + test4(); +} + +TEST_F(MindDataTestQueue, Test5) { + test4(); + // Assume we have run Test4. The destructor of the RefCount should be called 4 times. + // One for a. One for b. One for line 125 when we pop. One for the stale element in the queue. + ASSERT_EQ(gRefCountDestructorCalled, 4); +} + +TEST_F(MindDataTestQueue, Test6) { + // Create a list of queues + QueueList> my_list_of_queues; + const int chosen_queue_index = 2; + const int num_queues = 4; + const int queue_capacity = 3; + my_list_of_queues.Init(num_queues, queue_capacity); + // Now try to insert a number into a specific queue and pop it + std::unique_ptr a(new int(99)); + Status rc = my_list_of_queues[chosen_queue_index]->Add(std::move(a)); + ASSERT_TRUE(rc.IsOk()); + std::unique_ptr pepped_value; + rc = my_list_of_queues[chosen_queue_index]->PopFront(&pepped_value); + ASSERT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "Popped value " << *pepped_value << " from queue index " << chosen_queue_index; + ASSERT_EQ(*pepped_value, 99); +} diff --git a/tests/ut/cpp/dataset/random_color_adjust_op_test.cc b/tests/ut/cpp/dataset/random_color_adjust_op_test.cc new file mode 100644 index 00000000000..82df108ad1e --- /dev/null +++ b/tests/ut/cpp/dataset/random_color_adjust_op_test.cc @@ -0,0 +1,78 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/random_color_adjust_op.h" +#include "dataset/core/cv_tensor.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomColorAdjustOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestRandomColorAdjustOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomColorAdjustOp, TestOp1) { + MS_LOG(INFO) << "Doing testRandomColorAdjustOp."; + + std::shared_ptr output_tensor; + std::unique_ptr op(new RandomColorAdjustOp(0.7, 1.3, 0.8, 1.2, 0.8, 1.2, -0.2, 0.2)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor->shape()[1]); +} + +TEST_F(MindDataTestRandomColorAdjustOp, TestOp2) { + MS_LOG(INFO) << "Doing testRandomColorAdjustOp2."; + + std::shared_ptr output_tensor; + std::unique_ptr op(new RandomColorAdjustOp(0.7, 1.3, 0.8, 1.2, 0.8, 1.2, -0.2, 0.2)); + + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor->shape()[1]); +} + +TEST_F(MindDataTestRandomColorAdjustOp, TestOp3) { + MS_LOG(INFO) << "Doing testRandomColorAdjustOp Brightness."; + + std::shared_ptr output_tensor; + std::unique_ptr op(new RandomColorAdjustOp(0.8, 0.8, 0, 0, 0, 0, 0, 0)); + + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor->shape()[1]); +} + +TEST_F(MindDataTestRandomColorAdjustOp, TestOp4) { + MS_LOG(INFO) << "Doing testRandomColorAdjustOp Brightness."; + + std::shared_ptr output_tensor; + std::unique_ptr op(new RandomColorAdjustOp(0.8, 0.8, 0, 0, 0, 0, 0.2, 0.2)); + + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor->shape()[1]); +} diff --git a/tests/ut/cpp/dataset/random_crop_and_resize_op_test.cc b/tests/ut/cpp/dataset/random_crop_and_resize_op_test.cc new file mode 100644 index 00000000000..864d713ed38 --- /dev/null +++ b/tests/ut/cpp/dataset/random_crop_and_resize_op_test.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include +#include "dataset/kernels/image/random_crop_and_resize_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomCropAndResizeOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestRandomCropAndResizeOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomCropAndResizeOp, TestOpDefault) { + MS_LOG(INFO) << "Doing testRandomCropAndResize."; + TensorShape s_in = input_tensor_->shape(); + std::shared_ptr output_tensor; + int h_out = 512; + int w_out = 512; + + TensorShape s_out({(uint32_t) h_out, (uint32_t) w_out, (uint32_t) s_in[2]}); + + std::unique_ptr op(new RandomCropAndResizeOp(h_out, w_out)); + Status s; + for (auto i = 0; i < 100; i++) { + s = op->Compute(input_tensor_, &output_tensor); + } + EXPECT_TRUE(s.IsOk()); + MS_LOG(INFO) << "testRandomCropAndResize end."; +} + +TEST_F(MindDataTestRandomCropAndResizeOp, TestOpExtended) { + MS_LOG(INFO) << "Doing testRandomCropAndResize."; + TensorShape s_in = input_tensor_->shape(); + std::shared_ptr output_tensor; + int h_out = 1024; + int w_out = 2048; + float aspect_lb = 0.2; + float aspect_ub = 5; + float scale_lb = 0.0001; + float scale_ub = 1.0; + + TensorShape s_out({(uint32_t) h_out, (uint32_t) w_out, (uint32_t) s_in[2]}); + + std::unique_ptr op( + new RandomCropAndResizeOp(h_out, w_out, scale_lb, scale_ub, aspect_lb, aspect_ub)); + Status s; + for (auto i = 0; i < 100; i++) { + s = op->Compute(input_tensor_, &output_tensor); + } + EXPECT_TRUE(s.IsOk()); + MS_LOG(INFO) << "testRandomCropAndResize end."; +} diff --git a/tests/ut/cpp/dataset/random_crop_decode_resizeOp_test.cc b/tests/ut/cpp/dataset/random_crop_decode_resizeOp_test.cc new file mode 100644 index 00000000000..a4c4fbfdfff --- /dev/null +++ b/tests/ut/cpp/dataset/random_crop_decode_resizeOp_test.cc @@ -0,0 +1,156 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/decode_op.h" +#include "dataset/kernels/image/random_crop_and_resize_op.h" +#include "dataset/kernels/image/random_crop_decode_resize_op.h" +#include "dataset/core/config_manager.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomCropDecodeResizeOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestRandomCropDecodeResizeOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomCropDecodeResizeOp, TestOp2) { + MS_LOG(INFO) << "Doing testRandomCropDecodeResizeOp Test"; + + std::shared_ptr output_tensor1; + std::shared_ptr output_tensor2; + + int target_height = 884; + int target_width = 718; + float scale_lb = 0.08; + float scale_ub = 1.0; + float aspect_lb = 0.75; + float aspect_ub = 1.333333; + InterpolationMode interpolation = InterpolationMode::kLinear; + uint32_t max_iter = 10; + std::unique_ptr op1(new RandomCropAndResizeOp( + target_height, target_width, scale_lb, scale_ub, aspect_lb, aspect_ub, interpolation, max_iter)); + EXPECT_TRUE(op1->OneToOne()); + std::unique_ptr op2(new RandomCropDecodeResizeOp( + target_height, target_width, scale_lb, scale_ub, aspect_lb, aspect_ub, interpolation, max_iter)); + EXPECT_TRUE(op2->OneToOne()); + Status s1, s2; + + for (int i = 0; i < 100; i++) { + s1 = op1->Compute(input_tensor_, &output_tensor1); + s2 = op2->Compute(raw_input_tensor_, &output_tensor2); + cv::Mat output1(target_height, target_width, CV_8UC3, output_tensor1->StartAddr()); + cv::Mat output2(target_height, target_width, CV_8UC3, output_tensor2->StartAddr()); + long int mse_sum = 0; + long int count = 0; + int a, b; + for (int i = 0; i < target_height; i++) { + for (int j = 0; j < target_width; j++) { + a = (int)output1.at(i, j)[1]; + b = (int)output2.at(i, j)[1]; + mse_sum += sqrt((a - b) * (a - b)); + if (a != b) { + count++; + }; + } + } + double mse; + if (count > 0) { + mse = (double) mse_sum / count; + } else { + mse = mse_sum; + } + std::cout << "mse: " << mse << std::endl; + } + MS_LOG(INFO) << "MindDataTestRandomCropDecodeResizeOp end!"; +} + +TEST_F(MindDataTestRandomCropDecodeResizeOp, TestOp1) { + MS_LOG(INFO) << "Doing MindDataTestRandomCropDecodeResizeOp"; + const unsigned int h = 884; + const unsigned int w = 718; + const float scale_lb = 0.1; + const float scale_ub = 1; + const float aspect_lb = 0.1; + const float aspect_ub = 10; + + std::shared_ptr decoded, decoded_and_cropped, cropped_and_decoded; + std::mt19937 rd; + std::uniform_real_distribution rd_scale(scale_lb, scale_ub); + std::uniform_real_distribution rd_aspect(aspect_lb, aspect_ub); + DecodeOp op(true); + op.Compute(raw_input_tensor_, &decoded); + Status s1, s2; + float scale, aspect; + int crop_width, crop_height; + bool crop_success = false; + unsigned int mse_sum, m1, m2, count; + float mse; + + for (unsigned int k = 0; k < 100; ++k) { + mse_sum = 0; + count = 0; + for (auto i = 0; i < 100; i++) { + scale = rd_scale(rd); + aspect = rd_aspect(rd); + crop_width = std::round(std::sqrt(h * w * scale / aspect)); + crop_height = std::round(crop_width * aspect); + if (crop_width <= w && crop_height <= h) { + crop_success = true; + break; + } + } + if (crop_success == false) { + aspect = static_cast(h) / w; + scale = rd_scale(rd); + crop_width = std::round(std::sqrt(h * w * scale / aspect)); + crop_height = std::round(crop_width * aspect); + crop_height = (crop_height > h) ? h : crop_height; + crop_width = (crop_width > w) ? w : crop_width; + } + std::uniform_int_distribution<> rd_x(0, w - crop_width); + std::uniform_int_distribution<> rd_y(0, h - crop_height); + int x = rd_x(rd); + int y = rd_y(rd); + + op.Compute(raw_input_tensor_, &decoded); + s1 = Crop(decoded, &decoded_and_cropped, x, y, crop_width, crop_height); + s2 = JpegCropAndDecode(raw_input_tensor_, &cropped_and_decoded, x, y, crop_width, crop_height); + { + cv::Mat M1(crop_height, crop_width, CV_8UC3, decoded_and_cropped->StartAddr()); + cv::Mat M2(crop_height, crop_width, CV_8UC3, cropped_and_decoded->StartAddr()); + for (unsigned int i = 0; i < crop_height; ++i) { + for (unsigned int j = 0; j < crop_width; ++j) { + m1 = M1.at(i, j)[1]; + m2 = M2.at(i, j)[1]; + mse_sum += sqrt((m1 - m2) * (m1 - m2)); + if (m1 != m2) { + count++; + } + } + } + } + + mse = (count == 0) ? mse_sum : static_cast(mse_sum) / count; + std::cout << "mse: " << mse << std::endl; + } + MS_LOG(INFO) << "MindDataTestRandomCropDecodeResizeOp end!"; +} diff --git a/tests/ut/cpp/dataset/random_crop_op_test.cc b/tests/ut/cpp/dataset/random_crop_op_test.cc new file mode 100644 index 00000000000..2f3b19e2f47 --- /dev/null +++ b/tests/ut/cpp/dataset/random_crop_op_test.cc @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/random_crop_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomCropOp : public UT::CVOP::CVOpCommon { + protected: + MindDataTestRandomCropOp() : CVOpCommon() {} + + std::shared_ptr output_tensor_; +}; + +TEST_F(MindDataTestRandomCropOp, TestOp1) { + MS_LOG(INFO) << "Doing testRandomCrop."; + // Crop params + unsigned int crop_height = 128; + unsigned int crop_width = 128; + std::unique_ptr op(new RandomCropOp(crop_height, crop_width, 0, 0, 0, 0, BorderType::kConstant, false)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor_); + size_t actual = 0; + if (s == Status::OK()) { + actual = output_tensor_->shape()[0] * output_tensor_->shape()[1] * output_tensor_->shape()[2]; + } + EXPECT_EQ(actual, crop_height * crop_width * 3); + EXPECT_EQ(s, Status::OK()); +} + +TEST_F(MindDataTestRandomCropOp, TestOp2) { + MS_LOG(INFO) << "Doing testRandomCrop."; + // Crop params + unsigned int crop_height = 1280; + unsigned int crop_width = 1280; + std::unique_ptr op( + new RandomCropOp(crop_height, crop_width, 513, 513, 513, 513, BorderType::kConstant, false)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor_); + EXPECT_EQ(true, s.IsOk()); + MS_LOG(INFO) << "testRandomCrop end."; +} diff --git a/tests/ut/cpp/dataset/random_horizontal_flip_op_test.cc b/tests/ut/cpp/dataset/random_horizontal_flip_op_test.cc new file mode 100644 index 00000000000..eb2f7535549 --- /dev/null +++ b/tests/ut/cpp/dataset/random_horizontal_flip_op_test.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/random_horizontal_flip_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomHorizontalFlipOp : public UT::CVOP::CVOpCommon { + protected: + MindDataTestRandomHorizontalFlipOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomHorizontalFlipOp, TestOp) { + MS_LOG(INFO) << "Doing testHorizontalFlip."; + // flip + std::unique_ptr op(new RandomHorizontalFlipOp(0.5)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &input_tensor_); + EXPECT_TRUE(s.IsOk()); + CheckImageShapeAndData(input_tensor_, kFlipHorizontal); + MS_LOG(INFO) << "testHorizontalFlip end."; +} diff --git a/tests/ut/cpp/dataset/random_resize_op_test.cc b/tests/ut/cpp/dataset/random_resize_op_test.cc new file mode 100644 index 00000000000..ee185f2fc68 --- /dev/null +++ b/tests/ut/cpp/dataset/random_resize_op_test.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/kernels/image/random_resize_op.h" +#include "common/common.h" +#include "common/cvop_common.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomResize : public UT::CVOP::CVOpCommon { + public: + MindDataTestRandomResize() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomResize, TestOp) { + MS_LOG(INFO) << "Doing test RandomResize."; + // Resizing with a factor of 0.5 + TensorShape s = input_tensor_->shape(); + int output_h = 0.5 * s[0]; + int output_w = 0.5 * s[1]; + std::shared_ptr output_tensor; + // Resizing + std::unique_ptr op(new RandomResizeOp(output_h, output_w)); + EXPECT_TRUE(op->OneToOne()); + Status st = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(st.IsOk()); + MS_LOG(INFO) << "testResize end."; +} diff --git a/tests/ut/cpp/dataset/random_rotation_op_test.cc b/tests/ut/cpp/dataset/random_rotation_op_test.cc new file mode 100644 index 00000000000..8b82ef1dcd0 --- /dev/null +++ b/tests/ut/cpp/dataset/random_rotation_op_test.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/random_rotation_op.h" +#include "dataset/core/cv_tensor.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomRotationOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestRandomRotationOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomRotationOp, TestOp) { + MS_LOG(INFO) << "Doing MindDataTestRandomRotationOp::TestOp."; + std::shared_ptr output_tensor; + float sDegree = -180; + float eDegree = 180; + // use compute center to use for rotation + float xCenter = -1; + float yCenter = -1; + bool expand = false; + std::unique_ptr op(new RandomRotationOp( + sDegree, eDegree, xCenter, yCenter, InterpolationMode::kLinear, expand)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor->shape()[1]); +} diff --git a/tests/ut/cpp/dataset/random_vertical_flip_op_test.cc b/tests/ut/cpp/dataset/random_vertical_flip_op_test.cc new file mode 100644 index 00000000000..a2583cab967 --- /dev/null +++ b/tests/ut/cpp/dataset/random_vertical_flip_op_test.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/random_vertical_flip_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRandomVerticalFlipOp : public UT::CVOP::CVOpCommon { + protected: + MindDataTestRandomVerticalFlipOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRandomVerticalFlipOp, TestOp) { + MS_LOG(INFO) << "Doing testVerticalFlip."; + // flip + std::unique_ptr op(new RandomVerticalFlipOp(0.5)); + Status s = op->Compute(input_tensor_, &input_tensor_); + EXPECT_TRUE(op->OneToOne()); + EXPECT_TRUE(s.IsOk()); + CheckImageShapeAndData(input_tensor_, kFlipVertical); + MS_LOG(INFO) << "testVerticalFlip end."; +} diff --git a/tests/ut/cpp/dataset/rename_op_test.cc b/tests/ut/cpp/dataset/rename_op_test.cc new file mode 100644 index 00000000000..532f8f56912 --- /dev/null +++ b/tests/ut/cpp/dataset/rename_op_test.cc @@ -0,0 +1,114 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include "dataset/core/client.h" +#include "dataset/core/constants.h" +#include "dataset/engine/datasetops/map_op.h" +#include "dataset/engine/datasetops/rename_op.h" +#include "common/common.h" +#include "common/utils.h" +#include "dataset/engine/data_buffer.h" +#include "gtest/gtest.h" +#include "dataset/core/global_context.h" +#include "utils/log_adapter.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRenameOp : public UT::DatasetOpTesting { + }; + +TEST_F(MindDataTestRenameOp, TestRenameOpDefault) { +// Tree: +// +// +// OpId(2) RenameOp +// | +// OpId(0) StorageOp +// Start with an empty execution tree + Status rc; + MS_LOG(INFO) << "UT test TestRenameBasic."; + auto my_tree = std::make_shared(); + // Creating StorageOp + + std::string dataset_path = datasets_root_path_ + "/test_tf_file_3_images_1"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(16) + .SetNumWorkers(1) + .Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + + // Creating DatasetOp + std::vector in_colnames = {"label"}; + std::vector out_colnames = {"label1"}; + + std::shared_ptr rename_op; + rc = RenameOp::Builder() + .SetInColNames(in_colnames) + .SetOutColNames(out_colnames) + .Build(&rename_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(rename_op); + EXPECT_TRUE(rc.IsOk()); + rc = rename_op->AddChild(std::move(my_storage_op)); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(rename_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + + // Launch the tree execution to kick off threads and start running the pipeline + MS_LOG(INFO) << "Launching my tree."; + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Simulate a parse of data from our pipeline. + std::shared_ptr root_node = my_tree->root(); + + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 3); // Should be 3 rows fetched +} diff --git a/tests/ut/cpp/dataset/repeat_op_test.cc b/tests/ut/cpp/dataset/repeat_op_test.cc new file mode 100644 index 00000000000..99e91afe817 --- /dev/null +++ b/tests/ut/cpp/dataset/repeat_op_test.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/circular_pool.h" +#include "dataset/core/client.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestrepeat_op : public UT::DatasetOpTesting { + +}; + +TEST_F(MindDataTestrepeat_op, Testrepeat_opFuntions) { + MS_LOG(INFO) << "Doing MindDataTestrepeat_op."; + auto my_tree = std::make_shared(); + + std::shared_ptr parent_op = std::make_shared(32); + + std::shared_ptr leaf_op = std::make_shared(16); + my_tree->AssociateNode(parent_op); + my_tree->AssociateNode(leaf_op); + ASSERT_NE(parent_op, nullptr); + ASSERT_NE(leaf_op, nullptr); + parent_op->AddChild(std::move(leaf_op)); + parent_op->Print(std::cout, false); + parent_op->PrepareNodeAction(); + RepeatOp RepeatOpOp(); + + std::shared_ptr repeat_op; + Status rc = RepeatOp::Builder(3).Build(&repeat_op); + ASSERT_NE(repeat_op, nullptr); +} diff --git a/tests/ut/cpp/dataset/rescale_op_test.cc b/tests/ut/cpp/dataset/rescale_op_test.cc new file mode 100644 index 00000000000..86abbe972ee --- /dev/null +++ b/tests/ut/cpp/dataset/rescale_op_test.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/rescale_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestRescaleOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestRescaleOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestRescaleOp, TestOp) { + // Rescale Factor + float rescale = 1.0 / 255; + float shift = 1.0; + + std::unique_ptr op(new RescaleOp(rescale, shift)); + std::shared_ptr output_tensor; + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(op->OneToOne()); + EXPECT_TRUE(s.IsOk()); + // The rescaled image becomes CV_32FC3, saving it as JPEG + // will result in a black image since opencv converts it to int + // This function is still good to have since it checks the shape + // but to check the data, its better and easier to do this + // check in python test. + CheckImageShapeAndData(output_tensor, kRescale); + MS_LOG(INFO) << "testRescale end."; +} diff --git a/tests/ut/cpp/dataset/resize_bilinear_op_test.cc b/tests/ut/cpp/dataset/resize_bilinear_op_test.cc new file mode 100644 index 00000000000..8642484149c --- /dev/null +++ b/tests/ut/cpp/dataset/resize_bilinear_op_test.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/resize_bilinear_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestResizeBilinearOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestResizeBilinearOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestResizeBilinearOp, TestOp) { + MS_LOG(INFO) << "Doing testResizeBilinear."; + // Resizing with a factor of 0.5 + TensorShape s = input_tensor_->shape(); + int output_w = 0.5 * s[0]; + int output_h = (s[0] * output_w) / s[1]; + std::shared_ptr output_tensor; + // Resizing + std::unique_ptr op(new ResizeBilinearOp(output_h, output_w)); + Status st = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(st.IsOk()); + CheckImageShapeAndData(output_tensor, kResizeBilinear); + MS_LOG(INFO) << "testResizeBilinear end."; +} diff --git a/tests/ut/cpp/dataset/resize_op_test.cc b/tests/ut/cpp/dataset/resize_op_test.cc new file mode 100644 index 00000000000..e23320a65ad --- /dev/null +++ b/tests/ut/cpp/dataset/resize_op_test.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/resize_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestResizeOp : public UT::CVOP::CVOpCommon { + public: + MindDataTestResizeOp() : CVOpCommon() {} +}; + +TEST_F(MindDataTestResizeOp, TestOp) { + MS_LOG(INFO) << "Doing testResize"; + // Resizing with a factor of 0.5 + TensorShape s = input_tensor_->shape(); + int output_w = 0.5 * s[0]; + int output_h = (s[0] * output_w) / s[1]; + std::shared_ptr output_tensor; + // Resizing + std::unique_ptr op(new ResizeOp(output_h, output_w)); + Status st = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(st.IsOk()); + CheckImageShapeAndData(output_tensor, kResizeBilinear); + MS_LOG(INFO) << "testResize end."; +} diff --git a/tests/ut/cpp/dataset/shuffle_op_test.cc b/tests/ut/cpp/dataset/shuffle_op_test.cc new file mode 100644 index 00000000000..82c4a679575 --- /dev/null +++ b/tests/ut/cpp/dataset/shuffle_op_test.cc @@ -0,0 +1,335 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/client.h" +#include "common/common.h" +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include +#include +#include + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestShuffleOp : public UT::DatasetOpTesting { + +}; + + +// Test info: +// - Dataset from testDataset1 has 10 rows, 2 columns. +// - RowsPerBuffer buffer setting of 2 divides evenly into total rows. +// - Shuffle size is multiple of rows per buffer. +// +// Tree: shuffle over storage +// +// ShuffleOp +// | +// StorageOp +// +TEST_F(MindDataTestShuffleOp, TestShuffleBasic1) { + Status rc; + MS_LOG(INFO) << "UT test TestShuffleBasic1."; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(16) + .SetNumWorkers(1) + .Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + std::shared_ptr my_shuffle_op; + rc = ShuffleOp::Builder().SetRowsPerBuffer(2).SetShuffleSize(4).Build(&my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_shuffle_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 10); + +} + +// Test info: +// - Dataset from testDataset1 has 10 rows, 2 columns. +// - RowsPerBuffer buffer setting of 3 does not divide evenly into total rows, thereby causing +// partially filled buffers. +// - Shuffle size is not a multiple of rows per buffer. +// - User has provided a non-default seed value. +// +// Tree: shuffle over storage +// +// ShuffleOp +// | +// StorageOp +// +TEST_F(MindDataTestShuffleOp, TestShuffleBasic2) { + Status rc; + MS_LOG(INFO) << "UT test TestShuffleBasic2."; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(3) + .SetWorkerConnectorSize(16) + .SetNumWorkers(2) + .Build(&my_storage_op); + ASSERT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + std::shared_ptr my_shuffle_op; + rc = ShuffleOp::Builder().SetShuffleSize(4).SetShuffleSeed(100).SetRowsPerBuffer(3).Build(&my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_shuffle_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 10); +} + +// Test info: +// - Dataset from testDataset1 has 10 rows, 2 columns. +// - RowsPerBuffer buffer setting of 3 does not divide evenly into total rows, thereby causing +// partially filled buffers +// - Shuffle size captures the entire dataset size (actually sets a value that is larger than the +// amount of rows in the dataset. +// +// Tree: shuffle over storage +// +// ShuffleOp +// | +// StorageOp +// +TEST_F(MindDataTestShuffleOp, TestShuffleBasic3) { + Status rc; + MS_LOG(INFO) << "UT test TestShuffleBasic3."; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(3) + .SetWorkerConnectorSize(16) + .SetNumWorkers(2) + .Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + my_tree->AssociateNode(my_storage_op); + std::shared_ptr my_shuffle_op; + rc = ShuffleOp::Builder().SetShuffleSize(100).SetRowsPerBuffer(3).Build(&my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_shuffle_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()) << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 10); +} + + +// Test info: +// - Dataset from testDataset1 has 10 rows, 2 columns. +// - RowsPerBuffer buffer setting of 3 does not divide evenly into total rows thereby causing +// partially filled buffers +// - Shuffle size is not a multiple of rows per buffer. +// - shuffle seed is given, and subsequent epochs will change the seed each time. +// - Repeat count of 2 +// +// Tree: Repeat over shuffle over storage +// +// Repeat +// | +// shuffle +// | +// StorageOp +// +TEST_F(MindDataTestShuffleOp, TestRepeatShuffle) { + Status rc; + MS_LOG(INFO) << "UT test TestRepeatShuffle."; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(3) + .SetWorkerConnectorSize(16) + .SetNumWorkers(2) + .Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + std::shared_ptr my_shuffle_op; + rc = ShuffleOp::Builder() + .SetShuffleSize(4) + .SetShuffleSeed(100) + .SetRowsPerBuffer(3) + .SetReshuffleEachEpoch(true) + .Build(&my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + uint32_t numRepeats = 2; + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(numRepeats).Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_repeat_op->AddChild(my_shuffle_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_shuffle_op->AddChild(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()) << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 20); +} diff --git a/tests/ut/cpp/dataset/stand_alone_samplers_test.cc b/tests/ut/cpp/dataset/stand_alone_samplers_test.cc new file mode 100644 index 00000000000..2626add22e0 --- /dev/null +++ b/tests/ut/cpp/dataset/stand_alone_samplers_test.cc @@ -0,0 +1,112 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h" +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +using namespace mindspore::dataset; + +Status CreateINT64Tensor(std::shared_ptr *sample_ids, int64_t num_elements, unsigned char *data = nullptr) { + TensorShape shape(std::vector(1, num_elements)); + RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, TensorImpl::kFlexible, shape, + DataType(DataType::DE_INT64), data)); + if (data == nullptr) { + (*sample_ids)->StartAddr(); // allocate memory in case user forgets! + } + return Status::OK(); +} + +class MindDataTestStandAloneSampler : public UT::DatasetOpTesting { + protected: + class MockStorageOp : public RandomAccessOp { + public: + MockStorageOp(int64_t val) : m_val_(val) {} + + Status GetNumSamples(int64_t *ptr) const override { + (*ptr) = m_val_; + return Status::OK(); + } + + Status GetNumRowsInDataset(int64_t *ptr) const override { + (*ptr) = m_val_; + return Status::OK(); + } + + private: + int64_t m_val_; + }; +}; + +TEST_F(MindDataTestStandAloneSampler, TestDistributedSampler) { + std::vector> row; + uint64_t res[6][7] = {{0, 3, 6, 9, 12, 15, 18}, {1, 4, 7, 10, 13, 16, 19}, {2, 5, 8, 11, 14, 17, 0}, + {0, 17, 4, 10, 14, 8, 15}, {13, 9, 16, 3, 2, 19, 12}, {1, 11, 6, 18, 7, 5, 0}}; + for (int i = 0; i < 6; i++) { + std::shared_ptr t; + Tensor::CreateTensor(&t, TensorImpl::kFlexible, TensorShape({7}), + DataType(DataType::DE_INT64), (unsigned char *)(res[i])); + row.push_back(t); + } + MockStorageOp mock(20); + std::unique_ptr db; + std::shared_ptr tensor; + for (int i = 0; i < 6; i++) { + std::unique_ptr sampler = mindspore::make_unique(3, i % 3, (i < 3 ? false : true)); + sampler->Init(&mock); + sampler->GetNextBuffer(&db); + db->GetTensor(&tensor, 0, 0); + std::cout << (*tensor); + if(i < 3) { // This is added due to std::shuffle() + EXPECT_TRUE((*tensor) == (*row[i])); + } + } +} + +TEST_F(MindDataTestStandAloneSampler, TestStandAoneSequentialSampler) { + std::vector> row; + MockStorageOp mock(5); + uint64_t res[5] = {0, 1, 2, 3, 4}; + std::shared_ptr label1, label2; + CreateINT64Tensor(&label1, 3, reinterpret_cast(res)); + CreateINT64Tensor(&label2, 2, reinterpret_cast(res + 3)); + std::shared_ptr sampler = std::make_shared(3); + std::unique_ptr db; + std::shared_ptr tensor; + sampler->Init(&mock); + sampler->GetNextBuffer(&db); + db->GetTensor(&tensor, 0, 0); + EXPECT_TRUE((*tensor) == (*label1)); + sampler->GetNextBuffer(&db); + db->GetTensor(&tensor, 0, 0); + EXPECT_TRUE((*tensor) == (*label2)); + sampler->Reset(); + sampler->GetNextBuffer(&db); + db->GetTensor(&tensor, 0, 0); + EXPECT_TRUE((*tensor) == (*label1)); + sampler->GetNextBuffer(&db); + db->GetTensor(&tensor, 0, 0); + EXPECT_TRUE((*tensor) == (*label2)); +} diff --git a/tests/ut/cpp/dataset/status_test.cc b/tests/ut/cpp/dataset/status_test.cc new file mode 100644 index 00000000000..835e6303b00 --- /dev/null +++ b/tests/ut/cpp/dataset/status_test.cc @@ -0,0 +1,61 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/status.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; + +class MindDataTestStatus : public UT::Common { + public: + MindDataTestStatus() {} +}; + +// This function returns Status +Status f1() { + Status rc(StatusCode::kUnexpectedError, "Testing macro"); + RETURN_IF_NOT_OK(rc); + // We shouldn't get here + return Status::OK(); +} + +Status f3() { + RETURN_STATUS_UNEXPECTED("Testing macro3"); +} + +TEST_F(MindDataTestStatus, Test1) { + // Test default constructor which should be OK + Status rc; + ASSERT_TRUE(rc.IsOk()); + Status err1(StatusCode::kOutOfMemory, __LINE__, __FILE__); + std::cout << err1; + ASSERT_TRUE(err1.IsOutofMemory()); + ASSERT_TRUE(err1.IsError()); + Status err2(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Oops"); + std::cout << err2; +} + +TEST_F(MindDataTestStatus, Test2) { + Status rc = f1(); + std::cout << rc; +} + +TEST_F(MindDataTestStatus, Test3) { + Status rc = f3(); + std::cout << rc; +} diff --git a/tests/ut/cpp/dataset/storage_op_test.cc b/tests/ut/cpp/dataset/storage_op_test.cc new file mode 100644 index 00000000000..e6ce4af52fc --- /dev/null +++ b/tests/ut/cpp/dataset/storage_op_test.cc @@ -0,0 +1,165 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/core/client.h" +#include "common/common.h" +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include +#include +#include + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestStorageOp : public UT::DatasetOpTesting { + +}; + +TEST_F(MindDataTestStorageOp, TestStorageBasic1) { + + // single storage op and nothing else + // + // StorageOp + + MS_LOG(INFO) << "UT test TestStorageBasic1."; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 2 divides evenly into total rows. + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::shared_ptr my_storage_op; + StorageOp::Builder builder; + builder.SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(16) + .SetNumWorkers(1); + rc = builder.Build(&my_storage_op); + ASSERT_TRUE(rc.IsOk()); + my_tree->AssociateNode(my_storage_op); + + // Set children/root layout. + my_tree->AssignRoot(my_storage_op); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()) << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 10); // Should be 10 rows fetched + + // debugging temp. what happens if we keep fetching.. + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); +} + +TEST_F(MindDataTestStorageOp, TestStorageBasic2) { + + // single storage op and nothing else + // + // StorageOp + + MS_LOG(INFO) << "UT test TestStorageBasic1."; + + Status rc; + + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + // Test info: + // Dataset from testDataset1 has 10 rows, 2 columns. + // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row + // only. 2 workers. + // Test a column selection instead of all columns as well. + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testDataset1"; + std::vector column_list; + std::string label_colname("label"); + column_list.push_back(label_colname); + std::shared_ptr my_storage_op; + StorageOp::Builder builder; + builder.SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(3) + .SetWorkerConnectorSize(16) + .SetNumWorkers(2) + .SetColumnsToLoad(column_list); + rc = builder.Build(&my_storage_op); + ASSERT_TRUE(rc.IsOk()); + my_tree->AssociateNode(my_storage_op); + + // Set children/root layout. + my_tree->AssignRoot(my_storage_op); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + my_tree->Prepare(); + my_tree->Launch(); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()) << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 10); // Should be 10 rows fetched +} diff --git a/tests/ut/cpp/dataset/subset_random_sampler_test.cc b/tests/ut/cpp/dataset/subset_random_sampler_test.cc new file mode 100644 index 00000000000..5142a6d3999 --- /dev/null +++ b/tests/ut/cpp/dataset/subset_random_sampler_test.cc @@ -0,0 +1,151 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "gtest/gtest.h" + +#include "dataset/core/constants.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" + +#include +#include + +using namespace mindspore::dataset; + +class MindDataTestSubsetRandomSampler : public UT::Common { + public: + class DummyRandomAccessOp : public RandomAccessOp { + public: + DummyRandomAccessOp(int64_t num_rows) : num_rows_(num_rows) {}; + Status GetNumSamples(int64_t *num) const { + *num = num_rows_; + return Status::OK(); + } + + Status GetNumRowsInDataset(int64_t *num) const { + *num = num_rows_; + return Status::OK(); + } + + private: + int64_t num_rows_; + }; +}; + +TEST_F(MindDataTestSubsetRandomSampler, TestAllAtOnce) { + std::vector in({0, 1, 2, 3, 4}); + std::unordered_set in_set(in.begin(), in.end()); + SubsetRandomSampler sampler(in); + + DummyRandomAccessOp dummy_random_access_op(5); + sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + } + } + ASSERT_EQ(in.size(), out.size()); + for (int i = 0; i < in.size(); i++) { + ASSERT_NE(in_set.find(out[i]), in_set.end()); + } + + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); +} + +TEST_F(MindDataTestSubsetRandomSampler, TestGetNextBuffer) { + int64_t total_samples = 100000 - 5; + int64_t samples_per_buffer = 10; + std::vector input(total_samples, 1); + SubsetRandomSampler sampler(input, samples_per_buffer); + + DummyRandomAccessOp dummy_random_access_op(total_samples); + sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + int epoch = 0; + while (!db->eoe()) { + epoch++; + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + } + } + db.reset(); + + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + } + + ASSERT_EQ(epoch, (total_samples + samples_per_buffer - 1) / samples_per_buffer); + ASSERT_EQ(input.size(), out.size()); +} + +TEST_F(MindDataTestSubsetRandomSampler, TestReset) { + std::vector in({0, 1, 2, 3, 4}); + std::unordered_set in_set(in.begin(), in.end()); + SubsetRandomSampler sampler(in); + + DummyRandomAccessOp dummy_random_access_op(5); + sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + } + } + ASSERT_EQ(in.size(), out.size()); + for (int i = 0; i < in.size(); i++) { + ASSERT_NE(in_set.find(out[i]), in_set.end()); + } + + sampler.Reset(); + + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), false); + db->PopRow(&row); + out.clear(); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + } + } + ASSERT_EQ(in.size(), out.size()); + for (int i = 0; i < in.size(); i++) { + ASSERT_NE(in_set.find(out[i]), in_set.end()); + } + + ASSERT_EQ(sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); +} diff --git a/tests/ut/cpp/dataset/task_manager_test.cc b/tests/ut/cpp/dataset/task_manager_test.cc new file mode 100644 index 00000000000..182eee18822 --- /dev/null +++ b/tests/ut/cpp/dataset/task_manager_test.cc @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/task_manager.h" + +using namespace mindspore::dataset; +using namespace std::placeholders; + +class MindDataTestTaskManager : public UT::Common { + public: + MindDataTestTaskManager() {} + + void SetUp() { Services::CreateInstance(); + } +}; + +std::atomic v(0); + +Status f(TaskGroup &vg){ + for (int i = 0; i < 1; i++) { + RETURN_IF_NOT_OK(vg.CreateAsyncTask("Infinity", [&]() -> Status { + TaskManager::FindMe()->Post(); + int a = v.fetch_add(1); + std::cout << a << std::endl; + return f(vg); + })); + } + return Status::OK(); +} + +TEST_F(MindDataTestTaskManager, Test1) { + // Clear the rc of the master thread if any + (void) TaskManager::GetMasterThreadRc(); + TaskGroup vg; + Status vg_rc = vg.CreateAsyncTask("Test error", [this]() -> Status { + TaskManager::FindMe()->Post(); + throw std::bad_alloc(); + }); + ASSERT_TRUE(vg_rc.IsOk() || vg_rc.IsOutofMemory()); + ASSERT_TRUE(vg.join_all().IsOk()); + ASSERT_TRUE(vg.GetTaskErrorIfAny().IsOutofMemory()); + // Test the error is passed back to the master thread. + Status rc = TaskManager::GetMasterThreadRc(); + ASSERT_TRUE(rc.IsOutofMemory()); +} diff --git a/tests/ut/cpp/dataset/tensor_test.cc b/tests/ut/cpp/dataset/tensor_test.cc new file mode 100644 index 00000000000..6686be15bef --- /dev/null +++ b/tests/ut/cpp/dataset/tensor_test.cc @@ -0,0 +1,379 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "dataset/core/client.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "securec.h" +#include "dataset/core/tensor.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/data_type.h" +#include "dataset/util/de_error.h" + +using namespace mindspore::dataset; + +namespace py = pybind11; + + +class MindDataTestTensorDE : public UT::Common { + public: + MindDataTestTensorDE() {} + + void SetUp() { + GlobalInit(); + } +}; + + +TEST_F(MindDataTestTensorDE, Basics) { + std::shared_ptr t = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_UINT64)); + ASSERT_EQ(t->shape(), TensorShape({2, 3})); + ASSERT_EQ(t->type(), DataType::DE_UINT64); + ASSERT_EQ(t->SizeInBytes(), 2 * 3 * 8); + ASSERT_EQ(t->Rank(), 2); + t->SetItemAt({0, 0}, 1); + t->SetItemAt({0, 1}, 2); + t->SetItemAt({0, 2}, 3); + t->SetItemAt({1, 0}, 4); + t->SetItemAt({1, 1}, 5); + t->SetItemAt({1, 2}, 6); + Status rc = t->SetItemAt({2, 3}, 7); + ASSERT_TRUE(rc.IsError()); + uint64_t o; + t->GetItemAt(&o, {0, 0}); + ASSERT_EQ(o, 1); + t->GetItemAt(&o, {0, 1}); + ASSERT_EQ(o, 2); + t->GetItemAt(&o, {0, 2}); + ASSERT_EQ(o, 3); + t->GetItemAt(&o, {1, 0}); + ASSERT_EQ(o, 4); + t->GetItemAt(&o, {1, 1}); + ASSERT_EQ(o, 5); + t->GetItemAt(&o, {1, 2}); + ASSERT_EQ(o, 6); + rc = t->GetItemAt(&o, {2, 3}); + ASSERT_TRUE(rc.IsError()); + ASSERT_EQ(t->ToString(), "Tensor (shape: <2,3>, Type: uint64)\n[[1,2,3],[4,5,6]]"); + std::vector x = {1, 2, 3, 4, 5, 6}; + std::shared_ptr t2 = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_UINT64), + reinterpret_cast(&x[0])); + ASSERT_EQ(*t == *t2, true); + ASSERT_EQ(*t != *t2, false); +} + +TEST_F(MindDataTestTensorDE, Fill) { + std::shared_ptr t = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_FLOAT32)); + t->Fill(2.5); + std::vector x = {2.5, 2.5, 2.5, 2.5}; + std::shared_ptr t2 = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_FLOAT32), + reinterpret_cast(&x[0])); + ASSERT_EQ(*t == *t2, true); +} + +TEST_F(MindDataTestTensorDE, Reshape) { + std::shared_ptr t = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_UINT8)); + t->Fill(254); + t->Reshape(TensorShape({4})); + std::vector x = {254, 254, 254, 254}; + std::shared_ptr t2 = std::make_shared(TensorShape({4}), DataType(DataType::DE_UINT8), + reinterpret_cast(&x[0])); + ASSERT_EQ(*t == *t2, true); + Status rc = t->Reshape(TensorShape({5})); + ASSERT_TRUE(rc.IsError()); + t2->ExpandDim(0); + ASSERT_EQ(t2->shape(), TensorShape({1, 4})); + t2->ExpandDim(2); + ASSERT_EQ(t2->shape(), TensorShape({1, 4, 1})); + rc = t2->ExpandDim(4); + ASSERT_TRUE(rc.IsError()); +} + +TEST_F(MindDataTestTensorDE, CopyTensor) { + std::shared_ptr t = std::make_shared(TensorShape({}), DataType(DataType::DE_INT16)); + t->SetItemAt({}, -66); + ASSERT_EQ(t->shape(), TensorShape({})); + ASSERT_EQ(t->type(), DataType::DE_INT16); + int16_t o; + t->GetItemAt(&o, {}); + ASSERT_EQ(o, -66); + unsigned char *addr = t->StartAddr(); + auto t2 = std::make_shared(std::move(*t)); + ASSERT_EQ(t2->shape(), TensorShape({})); + ASSERT_EQ(t2->type(), DataType::DE_INT16); + t2->GetItemAt(&o, {}); + ASSERT_EQ(o, -66); + unsigned char *new_addr = t2->StartAddr(); + ASSERT_EQ(addr, new_addr); + ASSERT_EQ(t->shape(), TensorShape::CreateUnknownRankShape()); + ASSERT_EQ(t->type(), DataType::DE_UNKNOWN); + ASSERT_EQ(t->StartAddr(), nullptr); + Status rc = t->GetItemAt(&o, {}); + ASSERT_TRUE(rc.IsError()); +} + +TEST_F(MindDataTestTensorDE, InsertTensor) { + std::shared_ptr t = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_FLOAT64)); + std::vector x = {1.1, 2.1, 3.1}; + std::shared_ptr t2 = std::make_shared(TensorShape({3}), DataType(DataType::DE_FLOAT64), + reinterpret_cast(&x[0])); + std::vector y = {1.2, 2.2, 3.2}; + std::shared_ptr t3 = std::make_shared(TensorShape({3}), DataType(DataType::DE_FLOAT64), + reinterpret_cast(&y[0])); + ASSERT_TRUE(t->InsertTensor({0}, t2).OK()); + ASSERT_TRUE(t->InsertTensor({1}, t3).OK()); + std::vector z = {1.1, 2.1, 3.1, 1.2, 2.2, 3.2}; + + std::shared_ptr t4 = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_FLOAT64), + reinterpret_cast(&z[0])); + ASSERT_EQ(*t == *t4, true); + std::vector x2 = {0}; + + std::shared_ptr t5 = std::make_shared(TensorShape({}), DataType(DataType::DE_FLOAT64), + reinterpret_cast(&x2[0])); + ASSERT_TRUE(t->InsertTensor({1, 2}, t5).OK()); + z[5] = 0; + std::shared_ptr t6 = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_FLOAT64), + reinterpret_cast(&z[0])); + ASSERT_EQ(*t == *t6, true); + ASSERT_EQ(t->InsertTensor({2}, t5).get_code(), StatusCode::kUnexpectedError); + ASSERT_EQ(t->InsertTensor({1}, t5).get_code(), StatusCode::kUnexpectedError); + ASSERT_EQ(t->InsertTensor({1, 2}, t6).get_code(), StatusCode::kUnexpectedError); + t6->Fill(-1); + ASSERT_TRUE(t->InsertTensor({}, t6).OK()); + ASSERT_EQ(*t == *t6, true); +} + +TEST_F(MindDataTestTensorDE, GetItemAt) { + std::shared_ptr t = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_UINT8)); + t->Fill(254); + uint64_t o1; + t->GetItemAt(&o1, {0, 0}); + ASSERT_EQ(o1, 254); + uint32_t o2; + t->GetItemAt(&o2, {0, 1}); + ASSERT_EQ(o2, 254); + uint16_t o3; + t->GetItemAt(&o3, {1, 0}); + ASSERT_EQ(o3, 254); + uint8_t o4; + t->GetItemAt(&o4, {1, 1}); + ASSERT_EQ(o4, 254); + std::shared_ptr t2 = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_INT8)); + t2->Fill(-10); + int64_t o5; + t2->GetItemAt(&o5, {0, 0}); + ASSERT_EQ(o5, -10); + int32_t o6; + t2->GetItemAt(&o6, {0, 1}); + ASSERT_EQ(o6, -10); + int16_t o7; + t2->GetItemAt(&o7, {1, 0}); + ASSERT_EQ(o7, -10); + int8_t o8; + t2->GetItemAt(&o8, {1, 1}); + ASSERT_EQ(o8, -10); + std::shared_ptr t3 = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_FLOAT32)); + t3->Fill(1.1); + double o9; + t3->GetItemAt(&o9, {0, 0}); + ASSERT_FLOAT_EQ(o9, 1.1); + float o10; + t3->GetItemAt(&o10, {0, 1}); + ASSERT_FLOAT_EQ(o10, 1.1); +} + +TEST_F(MindDataTestTensorDE, OperatorAssign) { + std::shared_ptr t = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_UINT8)); + t->Fill(1); + std::shared_ptr t2 = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_UINT8)); + *t2 = std::move(*t); + uint8_t o; + t2->GetItemAt(&o, {0, 0}); + ASSERT_EQ(o, 1); + t2->GetItemAt(&o, {0, 1}); + ASSERT_EQ(o, 1); + t2->GetItemAt(&o, {1, 0}); + ASSERT_EQ(o, 1); + t2->GetItemAt(&o, {1, 1}); + ASSERT_EQ(o, 1); +} + +TEST_F(MindDataTestTensorDE, Strides) { + std::shared_ptr t = std::make_shared(TensorShape({4, 2, 2}), DataType(DataType::DE_UINT8)); + std::vector x1 = t->Strides(); + std::vector x2 = {4, 2, 1}; + ASSERT_EQ(x1, x2); + t = std::make_shared(TensorShape({4, 2, 2}), DataType(DataType::DE_UINT32)); + x1 = t->Strides(); + x2 = {16, 8, 4}; + ASSERT_EQ(x1, x2); +} + +void checkCvMat(TensorShape shape, DataType type) { + std::shared_ptr t = std::make_shared(shape, type); + cv::Mat m = t->mat(); + ASSERT_EQ(m.data, t->StartAddr()); + ASSERT_EQ(static_cast(m.type()) & static_cast(CV_MAT_DEPTH_MASK), type.AsCVType()); + if (shape.Rank() < 4) { + if (shape.Rank() > 1) { + for (dsize_t i = 0; i < 2; i++) + ASSERT_EQ(m.size[static_cast(i)], shape[i]); + } else if (shape.Rank() == 0) { + ASSERT_EQ(m.size[0], 1); + ASSERT_EQ(m.size[1], 1); + } else { + ASSERT_EQ(m.size[0], shape[0]); + } + if (shape.Rank() == 3) { ASSERT_EQ(m.channels(), shape[2]); } + ASSERT_EQ(m.dims, 2); + ASSERT_EQ(m.size.dims(), 2); + if (shape.Rank() > 0) { ASSERT_EQ(m.rows, shape[0]); } + if (shape.Rank() > 1) { ASSERT_EQ(m.cols, shape[1]); } + } else { + for (dsize_t i = 0; i < shape.Rank(); i++) + ASSERT_EQ(m.size[static_cast(i)], shape[i]); + ASSERT_EQ(m.dims, shape.Rank()); + ASSERT_EQ(m.size.dims(), shape.Rank()); + ASSERT_EQ(m.rows, -1); + ASSERT_EQ(m.cols, -1); + } +} + +TEST_F(MindDataTestTensorDE, CVTensorBasics) { + checkCvMat(TensorShape({4, 5}), DataType(DataType::DE_UINT8)); + checkCvMat(TensorShape({4, 5, 3}), DataType(DataType::DE_UINT8)); + checkCvMat(TensorShape({4, 5, 10}), DataType(DataType::DE_UINT8)); + checkCvMat(TensorShape({4, 5, 3, 2}), DataType(DataType::DE_UINT8)); + checkCvMat(TensorShape({4}), DataType(DataType::DE_UINT8)); + checkCvMat(TensorShape({}), DataType(DataType::DE_INT16)); + checkCvMat(TensorShape({4, 5}), DataType(DataType::DE_INT16)); + checkCvMat(TensorShape({4, 5, 3}), DataType(DataType::DE_INT16)); + checkCvMat(TensorShape({4, 5, 10}), DataType(DataType::DE_INT16)); + checkCvMat(TensorShape({4, 5, 3, 2}), DataType(DataType::DE_INT16)); + checkCvMat(TensorShape({4}), DataType(DataType::DE_INT16)); + checkCvMat(TensorShape({}), DataType(DataType::DE_INT16)); +} + +TEST_F(MindDataTestTensorDE, CVTensorFromMat) { + cv::Mat m(2, 2, CV_8U); + m.at(0, 0) = 10; + m.at(0, 1) = 20; + m.at(1, 0) = 30; + m.at(1, 1) = 40; + std::shared_ptr cvt = std::make_shared(m); + std::shared_ptr t = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_UINT8)); + t->SetItemAt({0, 0}, 10); + t->SetItemAt({0, 1}, 20); + t->SetItemAt({1, 0}, 30); + t->SetItemAt({1, 1}, 40); + ASSERT_TRUE(*t == *cvt); + int size[] = {4}; + cv::Mat m2(1, size, CV_8U); + m2.at(0) = 10; + m2.at(1) = 20; + m2.at(2) = 30; + m2.at(3) = 40; + std::shared_ptr cvt2 = std::make_shared(m2); + std::shared_ptr t2 = std::make_shared(TensorShape({4}), DataType(DataType::DE_UINT8)); + t2->SetItemAt({0}, 10); + t2->SetItemAt({1}, 20); + t2->SetItemAt({2}, 30); + t2->SetItemAt({3}, 40); + t2->ExpandDim(1); + ASSERT_TRUE(*t2 == *cvt2); +} + +TEST_F(MindDataTestTensorDE, CVTensorAs) { + std::shared_ptr t = std::make_shared(TensorShape({3, 2}), DataType(DataType::DE_FLOAT64)); + t->Fill(2.2); + unsigned char *addr = t->StartAddr(); + std::shared_ptr t2 = std::make_shared(TensorShape({3, 2}), DataType(DataType::DE_FLOAT64)); + t2->Fill(4.4); + std::shared_ptr ctv = CVTensor::AsCVTensor(t); + ASSERT_EQ(t->StartAddr(), nullptr); + ASSERT_EQ(ctv->StartAddr(), addr); + cv::Mat m = ctv->mat(); + m = 2 * m; + ASSERT_EQ(ctv->StartAddr(), addr); + ASSERT_TRUE(*t2 == *ctv); + std::cout << *t2 << std::endl << *ctv; +} + +TEST_F(MindDataTestTensorDE, CVTensorMatSlice) { + cv::Mat m(2, 3, CV_32S); + m.at(0, 0) = 10; + m.at(0, 1) = 20; + m.at(0, 2) = 30; + m.at(1, 0) = 40; + m.at(1, 1) = 50; + m.at(1, 2) = 60; + std::shared_ptr cvt = std::make_shared(m); + cv::Mat mat; + cvt->Mat({1}, &mat); + cv::Mat m2(3, 1, CV_32S); + m2.at(0) = 40; + m2.at(1) = 50; + m2.at(2) = 60; + std::shared_ptr cvt2 = std::make_shared(mat); + std::shared_ptr cvt3 = std::make_shared(m2); + + ASSERT_TRUE(*cvt2 == *cvt3); + cvt->Mat({0}, &mat); + m2.at(0) = 10; + m2.at(1) = 20; + m2.at(2) = 30; + cvt2 = std::make_shared(mat); + cvt3 = std::make_shared(m2); + ASSERT_TRUE(*cvt2 == *cvt3); +} + +TEST_F(MindDataTestTensorDE, TensorIterator) { + std::vector values = {1, 2, 3, 4, 5, 6}; + std::vector values2 = {2, 3, 4, 5, 6, 7}; + + std::shared_ptr t = std::make_shared(TensorShape({6}), DataType(DataType::DE_UINT32), + reinterpret_cast(&values[0])); + auto i = t->begin(); + auto j = values.begin(); + uint32_t ctr = 0; + for (; i != t->end(); i++, j++) { + ASSERT_TRUE(*i == *j); + ctr++; + } + ASSERT_TRUE(ctr == 6); + t->Reshape(TensorShape {2, 3}); + i = t->begin(); + j = values.begin(); + ctr = 0; + for (; i != t->end(); i++, j++) { + ASSERT_TRUE(*i == *j); + ctr++; + } + ASSERT_TRUE(ctr == 6); + for (auto it = t->begin(); it != t->end(); it++) { + *it = *it + 1; + } + i = t->begin(); + j = values2.begin(); + ctr = 0; + for (; i != t->end(); i++, j++) { + ASSERT_TRUE(*i == *j); + ctr++; + } + ASSERT_TRUE(ctr == 6); +} diff --git a/tests/ut/cpp/dataset/tensorshape_test.cc b/tests/ut/cpp/dataset/tensorshape_test.cc new file mode 100644 index 00000000000..1af0bf9c823 --- /dev/null +++ b/tests/ut/cpp/dataset/tensorshape_test.cc @@ -0,0 +1,186 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "./securec.h" +#include "dataset/core/client.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor_shape.h" +#include "dataset/engine/data_schema.h" +#include "common/common.h" +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestTensorShape : public UT::Common { + public: + MindDataTestTensorShape() = default; +}; + + +TEST_F(MindDataTestTensorShape, TestBasics) { + std::vector vec = {4, 5, 6}; + TensorShape t(vec); + ASSERT_EQ(t.Rank(), 3); + ASSERT_EQ(t.Size(), 3); + ASSERT_EQ(t.known(), true); + ASSERT_EQ(t.empty(), false); + ASSERT_EQ(t.NumOfElements(), 120); + for (dsize_t i = 0; i < t.Rank(); i++) { + ASSERT_EQ(t[i], vec[i]); + } + ASSERT_EQ(vec, t.AsVector()); + ASSERT_EQ(t.IsValidIndex({0, 0, 0}), true); + ASSERT_EQ(t.IsValidIndex({3, 4, 5}), true); + ASSERT_EQ(t.IsValidIndex({3, 4, 6}), false); + ASSERT_EQ(t.IsValidIndex({4, 5, 6}), false); + ASSERT_EQ(t.IsValidIndex({4, 5, 6}), false); + ASSERT_EQ(t.IsValidIndex({3, 3}), false); + ASSERT_EQ(t.IsValidIndex({-3, -3, -1}), false); + ASSERT_EQ(t.IsValidIndex({-1, 4, 5}), false); + TensorShape t2({4, 5, 6}); + ASSERT_EQ(t, t2); + TensorShape t3({0}); + ASSERT_EQ(t3.Size(), 1); + ASSERT_EQ(t3.NumOfElements(), 0); + t3 = TensorShape({0, 5, 6}); + ASSERT_EQ(t3.Size(), 3); + ASSERT_EQ(t3.NumOfElements(), 0); +} + +TEST_F(MindDataTestTensorShape, TestScalars) { + TensorShape t = TensorShape::CreateScalar(); + ASSERT_EQ(t.Rank(), 0); + ASSERT_EQ(t.AsVector(), std::vector{}); + ASSERT_EQ(t.known(), true); + TensorShape t2(std::vector{}); + ASSERT_EQ(t, t2); + ASSERT_EQ(t.NumOfElements(), 1); +} + +TEST_F(MindDataTestTensorShape, TestDims) { + TensorShape t = TensorShape::CreateScalar(); + t = t.AppendDim(1); + t = t.AppendDim(2); + t = t.AppendDim(3); + ASSERT_EQ(t, TensorShape({1, 2, 3})); + TensorShape t2 = TensorShape::CreateScalar(); + t2 = t2.PrependDim(3); + t2 = t2.PrependDim(2); + t2 = t2.PrependDim(1); + ASSERT_EQ(t, t2); + TensorShape t3({4, 5, 6}); + t3 = t3.InsertDim(0, 1); // 1, 4, 5, 6 + t3 = t3.InsertDim(2, 2); // 1, 4, 2, 5, 6 + t3 = t3.InsertDim(4, 3); // 1, 4, 2, 5, 3, 6 + ASSERT_EQ(t3, TensorShape({1, 4, 2, 5, 3, 6})); +} + +TEST_F(MindDataTestTensorShape, TestUnknown) { + TensorShape t1({-1, 5, 6}); + ASSERT_EQ(t1.AsVector(), std::vector({-1, 5, 6})); + ASSERT_EQ(t1.known(), false); + TensorShape t2({5, 6}); + t2 = t2.PrependDim(-1); + ASSERT_EQ(t1, t2); + TensorShape t3 = TensorShape::CreateUnknownRankShape(); + ASSERT_EQ(t3.known(), false); + ASSERT_EQ(t3.Size(), 0); + TensorShape t4 = TensorShape::CreateUnknownShapeWithRank(3); + ASSERT_EQ(t4, TensorShape({-1, -1, -1})); +} + +// Test materializing a TensorShape by calling method on a given column descriptor +TEST_F(MindDataTestTensorShape, TestColDescriptor) { + int32_t rank = 0; // not used + int32_t num_elements = 0; + + // Has no shape + ColDescriptor c1("col1", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank); + TensorShape generated_shape1 = TensorShape::CreateUnknownRankShape(); + num_elements = 4; + Status rc = c1.MaterializeTensorShape(num_elements, &generated_shape1); + ASSERT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "generated_shape1: " << common::SafeCStr(generated_shape1.ToString()) << "."; + ASSERT_EQ(TensorShape({4}),generated_shape1); + + // Has shape i.e. <*> + TensorShape requested_shape2({TensorShape::kDimUnknown}); + ColDescriptor c2("col2", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank, &requested_shape2); + TensorShape generated_shape2 = TensorShape::CreateUnknownRankShape(); + num_elements = 5; + rc = c2.MaterializeTensorShape(num_elements, &generated_shape2); + ASSERT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "generated_shape2: " << common::SafeCStr(generated_shape2.ToString()) << "."; + ASSERT_EQ(TensorShape({5}),generated_shape2); + + // Compute unknown dimension <*,4> + TensorShape requested_shape3({TensorShape::kDimUnknown, 4}); + ColDescriptor c3("col3", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank, &requested_shape3); + TensorShape generated_shape3 = TensorShape::CreateUnknownRankShape(); + num_elements = 12; + rc = c3.MaterializeTensorShape(num_elements, &generated_shape3); + ASSERT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "generated_shape3: " << common::SafeCStr(generated_shape3.ToString()) << "."; + ASSERT_EQ(TensorShape({3,4}),generated_shape3); + + // Compute unknown dimension <3,*,4> + TensorShape requested_shape4({3, TensorShape::kDimUnknown, 4}); + ColDescriptor c4("col4", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank, &requested_shape4); + TensorShape generated_shape4 = TensorShape::CreateUnknownRankShape(); + num_elements = 24; + rc = c4.MaterializeTensorShape(num_elements, &generated_shape4); + ASSERT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "generated_shape4: " << common::SafeCStr(generated_shape4.ToString()) << "."; + ASSERT_EQ(TensorShape({3,2,4}),generated_shape4); + + // requested and generated should be the same! <2,3,4> + TensorShape requested_shape5({2, 3, 4}); + ColDescriptor c5("col5", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank, &requested_shape5); + TensorShape generated_shape5 = TensorShape::CreateUnknownRankShape(); + num_elements = 24; + rc = c5.MaterializeTensorShape(num_elements, &generated_shape5); + ASSERT_TRUE(rc.IsOk()); + MS_LOG(INFO) << "generated_shape5: " << common::SafeCStr(generated_shape5.ToString()) << "."; + ASSERT_EQ(requested_shape5,generated_shape5); + + // expect fail due to multiple unknown dimensions + TensorShape requested_shape6({2, TensorShape::kDimUnknown, TensorShape::kDimUnknown}); + ColDescriptor c6("col6", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank, &requested_shape6); + TensorShape generated_shape6 = TensorShape::CreateUnknownRankShape(); + num_elements = 24; + rc = c6.MaterializeTensorShape(num_elements, &generated_shape6); + ASSERT_FALSE(rc.IsOk()); + + // expect fail because the requested shape element count does not match with num elements + TensorShape requested_shape7({2, 3, 3}); + ColDescriptor c7("col7", DataType(DataType::DE_INT8), TensorImpl::kFlexible, rank, &requested_shape7); + TensorShape generated_shape7 = TensorShape::CreateUnknownRankShape(); + num_elements = 24; + rc = c7.MaterializeTensorShape(num_elements, &generated_shape7); + ASSERT_FALSE(rc.IsOk()); +} + +TEST_F(MindDataTestTensorShape, TestInvalid) { + ASSERT_EQ(TensorShape({2147483648}), TensorShape::CreateUnknownRankShape()); + ASSERT_EQ(TensorShape({kDeMaxDim - 1, kDeMaxDim - 1, kDeMaxDim - 1}), TensorShape::CreateUnknownRankShape()); +} diff --git a/tests/ut/cpp/dataset/tfReader_op_test.cc b/tests/ut/cpp/dataset/tfReader_op_test.cc new file mode 100644 index 00000000000..c70d5fb6eee --- /dev/null +++ b/tests/ut/cpp/dataset/tfReader_op_test.cc @@ -0,0 +1,699 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "dataset/core/client.h" +#include "dataset/engine/data_schema.h" +#include "common/common.h" +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestTFReaderOp : public UT::DatasetOpTesting { + +}; + +TEST_F(MindDataTestTFReaderOp, TestTFReaderBasic1) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(16) + .SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeRowsPerBuffer) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(500) + .SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderSmallRowsPerBuffer) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(1) + .SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeQueueSize) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetWorkerConnectorSize(1) + .SetRowsPerBuffer(16) + .SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderOneThread) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(16) + .SetNumWorkers(1); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderRepeat) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + // TFReaderOp + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(16) + .SetWorkerConnectorSize(16) + .SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); + builder.SetDataSchema(std::move(schema)); + Status rc= builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + // RepeatOp + std::shared_ptr my_repeat_op = std::make_shared(3); + rc = my_tree->AssociateNode(my_repeat_op); + ASSERT_TRUE(rc.IsOk()); + + // Set children/root layout. + rc = my_repeat_op->AddChild(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_repeat_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12 * 3); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderSchemaConstructor) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; + + std::unique_ptr data_schema = mindspore::make_unique(); + std::vector columns_to_load; + columns_to_load.push_back("col_sint32"); + columns_to_load.push_back("col_binary"); + data_schema->LoadSchemaFile(dataset_path + "/datasetSchema.json", columns_to_load); + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path+"/test.data"}) + .SetRowsPerBuffer(16) + .SetNumWorkers(16) + .SetDataSchema(std::move(data_schema)); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + ASSERT_EQ(tensor_list.size(), columns_to_load.size()); + + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Row) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; + + std::string data_schema_filepath = dataset_path + "/datasetSchema1Row.json"; + + // TFReaderOp + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {}); + builder.SetDataSchema(std::move(schema)); + + Status rc= builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 1); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Buffer) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; + + std::string data_schema_filepath = dataset_path + "/datasetSchema5Rows.json"; + + // TFReaderOp + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {}); + builder.SetDataSchema(std::move(schema)); + + Status rc= builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 5); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderTake7Rows) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes"; + + std::string data_schema_filepath = dataset_path + "/datasetSchema7Rows.json"; + + // TFReaderOp + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); + std::unique_ptr schema = mindspore::make_unique(); + schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {}); + builder.SetDataSchema(std::move(schema)); + + Status rc= builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 7); +} + +TEST_F(MindDataTestTFReaderOp, TestTFReaderBasicNoSchema) { + // Start with an empty execution tree + auto my_tree = std::make_shared(); + + std::string dataset_path; + dataset_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::shared_ptr my_tfreader_op; + TFReaderOp::Builder builder; + builder.SetDatasetFilesList({dataset_path}) + .SetRowsPerBuffer(16) + .SetNumWorkers(16); + Status rc = builder.Build(&my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->AssignRoot(my_tfreader_op); + ASSERT_TRUE(rc.IsOk()); + + MS_LOG(INFO) << "Launching tree and begin iteration."; + rc = my_tree->Prepare(); + ASSERT_TRUE(rc.IsOk()); + + rc = my_tree->Launch(); + ASSERT_TRUE(rc.IsOk()); + + // Start the loop of reading tensors from our pipeline + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + // Display the tensor by calling the printer on it + ASSERT_EQ(tensor_list.size(), 9); + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << ss.str() << "."; + } + + rc = di.FetchNextTensorRow(&tensor_list); + ASSERT_TRUE(rc.IsOk()); + row_count++; + } + + ASSERT_EQ(row_count, 12); +} + +TEST_F(MindDataTestTFReaderOp, TestTotalRowsBasic) { + std::string tf_file = datasets_root_path_ + "/testTFTestAllTypes/test.data"; + + std::vector filenames; + + for (int i = 0; i < 5; i++) { + filenames.push_back(tf_file); + } + + int64_t total_rows = 0; + TFReaderOp::CountTotalRows(&total_rows, filenames, 1); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 2); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 3); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 4); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 5); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 6); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 729); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 1, true); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 2, true); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 3, true); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 4, true); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 5, true); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 6, true); + ASSERT_EQ(total_rows, 60); + TFReaderOp::CountTotalRows(&total_rows, filenames, 729, true); + ASSERT_EQ(total_rows, 60); +} diff --git a/tests/ut/cpp/dataset/to_float16_op_test.cc b/tests/ut/cpp/dataset/to_float16_op_test.cc new file mode 100644 index 00000000000..9c49c67b2cf --- /dev/null +++ b/tests/ut/cpp/dataset/to_float16_op_test.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/random_rotation_op.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/kernels/data/to_float16_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestToFloat16Op : public UT::CVOP::CVOpCommon { + public: + MindDataTestToFloat16Op() : CVOpCommon() {} +}; + +TEST_F(MindDataTestToFloat16Op, TestOp) { + MS_LOG(INFO) << "Doing TestRandomRotationOp::TestOp."; + std::shared_ptr output_tensor; + float s_degree = -180; + float e_degree = 180; + // use compute center to use for rotation + float x_center = -1; + float y_center = -1; + bool expand = false; + std::unique_ptr op(new RandomRotationOp( + s_degree, e_degree, x_center, y_center, InterpolationMode::kLinear, expand)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor); + EXPECT_TRUE(s.IsOk()); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor->shape()[1]); + + std::unique_ptr to_float_op(new ToFloat16Op()); + std::shared_ptr output_tensor1; + s = op->Compute(output_tensor, &output_tensor1); + EXPECT_EQ(output_tensor->shape()[0], output_tensor1->shape()[0]); + EXPECT_EQ(output_tensor->shape()[1], output_tensor1->shape()[1]); +} diff --git a/tests/ut/cpp/dataset/treap_test.cc b/tests/ut/cpp/dataset/treap_test.cc new file mode 100644 index 00000000000..c92bea6bf8c --- /dev/null +++ b/tests/ut/cpp/dataset/treap_test.cc @@ -0,0 +1,64 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/treap.h" +#include "common/common.h" +#include "gtest/gtest.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; + +class MindDataTestTreap : public UT::Common { + public: + MindDataTestTreap() {} +}; + +TEST_F(MindDataTestTreap, TestALLFunction) { + Treap tree; + srand(time(NULL)); + for (uint64_t i = 0; i < 1000; i++) { + uint64_t sz = rand() % 500; + tree.Insert(i, sz); + } + + EXPECT_EQ(tree.size(), 1000); + + int n = 0; + uint64_t key = 0; + for (auto it : tree) { + if (n > 0) { + EXPECT_GT(it.key, key); + } + key = it.key; + n++; + } + + EXPECT_EQ(n, 1000); + + uint64_t prev = 0; + n = 0; + while (!tree.empty()) { + auto p = tree.Top(); + EXPECT_TRUE(p.second); + uint64_t v = p.first.priority; + if (n > 0) { + EXPECT_GE(prev, v); + } + prev = v; + n++; + tree.Pop(); + } +} diff --git a/tests/ut/cpp/dataset/type_cast_op_test.cc b/tests/ut/cpp/dataset/type_cast_op_test.cc new file mode 100644 index 00000000000..0bd1e90eae3 --- /dev/null +++ b/tests/ut/cpp/dataset/type_cast_op_test.cc @@ -0,0 +1,134 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/data/type_cast_op.h" +#include "dataset/core/client.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/data_type.h" +#include "dataset/core/tensor.h" +#include "dataset/core/pybind_support.h" +#include "gtest/gtest.h" +#include "securec.h" +#include "dataset/util/de_error.h" + +#define MAX_INT_PRECISION 16777216 // float int precision is 16777216 +using namespace mindspore::dataset; + +namespace py = pybind11; + + +class MindDataTestTypeCast : public UT::Common { + public: + MindDataTestTypeCast() {} + + void SetUp() { + GlobalInit(); + } +}; + +template +void testCast(std::vector values, const DataType &from, const DataType &to) { + std::shared_ptr t = std::make_shared(TensorShape({static_cast(values.size())}), + DataType(from), + reinterpret_cast(&values[0])); + + std::unique_ptr op(new TypeCastOp(to)); + EXPECT_TRUE(op->OneToOne()); + std::shared_ptr output; + EXPECT_TRUE(op->Compute(t, &output)); + ASSERT_TRUE(t->shape() == output->shape()); + ASSERT_TRUE(DataType(to)==output->type()); + std::cout << *output << std::endl; + auto out = output->begin(); + auto v = values.begin(); + for (; out != output->end(); out++, v++) { + ASSERT_TRUE((*out) == static_cast(*v)); + } +} + +TEST_F(MindDataTestTypeCast, CastFromUINT8) { + std::vector input{0, 10, 255}; + DataType input_format = DataType(DataType("uint8")); + testCast(input, input_format, DataType("uint8")); + testCast(input, input_format, DataType("uint16")); + testCast(input, input_format, DataType("uint32")); + testCast(input, input_format, DataType("uint64")); + testCast(input, input_format, DataType("int8")); + testCast(input, input_format, DataType("int16")); + testCast(input, input_format, DataType("int32")); + testCast(input, input_format, DataType("int64")); + testCast(input, input_format, DataType("float16")); + testCast(input, input_format, DataType("float32")); + testCast(input, input_format, DataType("float64")); + testCast(input, input_format, DataType("bool")); +} + +TEST_F(MindDataTestTypeCast, CastFromINT64) { + std::vector input{-9223372036854775806, 0, 9223372036854775807}; + DataType input_format = DataType("int64"); + testCast(input, input_format, DataType("uint8")); + testCast(input, input_format, DataType("uint16")); + testCast(input, input_format, DataType("uint32")); + testCast(input, input_format, DataType("uint64")); + testCast(input, input_format, DataType("int8")); + testCast(input, input_format, DataType("int16")); + testCast(input, input_format, DataType("int32")); + testCast(input, input_format, DataType("int64")); + testCast(input, input_format, DataType("float16")); + testCast(input, input_format, DataType("float32")); + testCast(input, input_format, DataType("float64")); + testCast(input, input_format, DataType("bool")); +} + +TEST_F(MindDataTestTypeCast, CastFromFLOAT64) { + std::vector input{(-1) * MAX_INT_PRECISION, 0, MAX_INT_PRECISION}; + DataType input_format = DataType("float64"); + testCast(input, input_format, DataType("uint8")); + testCast(input, input_format, DataType("uint16")); + testCast(input, input_format, DataType("uint32")); + testCast(input, input_format, DataType("uint64")); + testCast(input, input_format, DataType("int8")); + testCast(input, input_format, DataType("int16")); + testCast(input, input_format, DataType("int32")); + testCast(input, input_format, DataType("int64")); + testCast(input, input_format, DataType("float16")); + testCast(input, input_format, DataType("float32")); + testCast(input, input_format, DataType("float64")); + testCast(input, input_format, DataType("bool")); +} + +TEST_F(MindDataTestTypeCast, CastFromFLOAT16) { + float16 min(0.0005); + float16 zero(0); + float16 max(32768); + std::vector input{min, zero, max}; + DataType input_format = DataType("float16"); + testCast(input, input_format, DataType("uint8")); + testCast(input, input_format, DataType("uint16")); + testCast(input, input_format, DataType("uint32")); + testCast(input, input_format, DataType("uint64")); + testCast(input, input_format, DataType("int8")); + testCast(input, input_format, DataType("int16")); + testCast(input, input_format, DataType("int32")); + testCast(input, input_format, DataType("int64")); + testCast(input, input_format, DataType("float16")); + testCast(input, input_format, DataType("float32")); + testCast(input, input_format, DataType("float64")); + testCast(input, input_format, DataType("bool")); +} diff --git a/tests/ut/cpp/dataset/voc_op_test.cc b/tests/ut/cpp/dataset/voc_op_test.cc new file mode 100644 index 00000000000..ccce064c963 --- /dev/null +++ b/tests/ut/cpp/dataset/voc_op_test.cc @@ -0,0 +1,66 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "common/common.h" +#include "common/utils.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/datasetops/source/voc_op.h" +#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h" +#include "dataset/engine/datasetops/source/sampler/pk_sampler.h" +#include "dataset/engine/datasetops/source/sampler/random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" +#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" +#include "dataset/util/de_error.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +std::shared_ptr Batch(int batch_size = 1, bool drop = false, int rows_per_buf = 2); + +std::shared_ptr Repeat(int repeat_cnt); + +std::shared_ptr Build(std::vector> ops); + +std::shared_ptr CreateVOC(int64_t num_wrks, int64_t rows, int64_t conns, std::string path, + bool shuf = false, std::unique_ptr sampler = nullptr, + int64_t num_samples = 0, bool decode = false) { + std::shared_ptr so; + VOCOp::Builder builder; + Status rc = builder.SetNumWorkers(num_wrks).SetDir(path).SetRowsPerBuffer(rows) + .SetOpConnectorSize(conns).SetSampler(std::move(sampler)) + .SetNumSamples(num_samples).SetDecode(decode).Build(&so); + return so; +} + +class MindDataTestVOCSampler : public UT::DatasetOpTesting { + protected: +}; diff --git a/tests/ut/cpp/dataset/weighted_random_sampler_test.cc b/tests/ut/cpp/dataset/weighted_random_sampler_test.cc new file mode 100644 index 00000000000..1c5d73613fd --- /dev/null +++ b/tests/ut/cpp/dataset/weighted_random_sampler_test.cc @@ -0,0 +1,295 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "gtest/gtest.h" + +#include "dataset/core/constants.h" +#include "dataset/core/tensor.h" +#include "dataset/engine/data_buffer.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" +#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" +#include "utils/log_adapter.h" + +#include +#include + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestWeightedRandomSampler : public UT::Common { + public: + class DummyRandomAccessOp : public RandomAccessOp { + public: + DummyRandomAccessOp(uint64_t num_rows) : num_rows_(num_rows) {}; + Status GetNumSamples(int64_t *num) const { + *num = num_rows_; + return Status::OK(); + } + + Status GetNumRowsInDataset(int64_t *num) const { + *num = num_rows_; + return Status::OK(); + } + + private: + uint64_t num_rows_; + }; +}; + +TEST_F(MindDataTestWeightedRandomSampler, TestOneshotReplacement) { + // num samples to draw. + uint64_t num_samples = 100; + uint64_t total_samples = 1000; + std::vector weights(total_samples, std::rand() % 100); + std::vector freq(total_samples, 0); + + // create sampler with replacement = true + WeightedRandomSampler m_sampler(weights, num_samples, true); + DummyRandomAccessOp dummy_random_access_op(total_samples); + m_sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + + ASSERT_EQ(num_samples, out.size()); + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); +} + +TEST_F(MindDataTestWeightedRandomSampler, TestOneshotNoReplacement) { + // num samples to draw. + uint64_t num_samples = 100; + uint64_t total_samples = 1000; + std::vector weights(total_samples, std::rand() % 100); + std::vector freq(total_samples, 0); + + // create sampler with replacement = replacement + WeightedRandomSampler m_sampler(weights, num_samples, false); + DummyRandomAccessOp dummy_random_access_op(total_samples); + m_sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + ASSERT_EQ(num_samples, out.size()); + + // Without replacement, each sample only drawn once. + for (int i = 0; i < total_samples; i++) { + if (freq[i]) { + ASSERT_EQ(freq[i], 1); + } + } + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); +} + +TEST_F(MindDataTestWeightedRandomSampler, TestGetNextBufferReplacement) { + // num samples to draw. + uint64_t num_samples = 100; + uint64_t total_samples = 1000; + uint64_t samples_per_buffer = 10; + std::vector weights(total_samples, std::rand() % 100); + + // create sampler with replacement = replacement + WeightedRandomSampler m_sampler(weights, num_samples, true, samples_per_buffer); + DummyRandomAccessOp dummy_random_access_op(total_samples); + m_sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + int epoch = 0; + while (!db->eoe()) { + epoch++; + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + } + } + db.reset(); + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + } + + ASSERT_EQ(epoch, (num_samples + samples_per_buffer - 1) / samples_per_buffer); + ASSERT_EQ(num_samples, out.size()); +} + +TEST_F(MindDataTestWeightedRandomSampler, TestGetNextBufferNoReplacement) { + // num samples to draw. + uint64_t num_samples = 100; + uint64_t total_samples = 100; + uint64_t samples_per_buffer = 10; + std::vector weights(total_samples, std::rand() % 100); + weights[1] = 0; + weights[2] = 0; + std::vector freq(total_samples, 0); + + // create sampler with replacement = replacement + WeightedRandomSampler m_sampler(weights, num_samples, false, samples_per_buffer); + DummyRandomAccessOp dummy_random_access_op(total_samples); + m_sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + int epoch = 0; + while (!db->eoe()) { + epoch++; + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + db.reset(); + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + } + + // Without replacement, each sample only drawn once. + for (int i = 0; i < total_samples; i++) { + if (freq[i]) { + ASSERT_EQ(freq[i], 1); + } + } + + ASSERT_EQ(epoch, (num_samples + samples_per_buffer - 1) / samples_per_buffer); + ASSERT_EQ(num_samples, out.size()); +} + +TEST_F(MindDataTestWeightedRandomSampler, TestResetReplacement) { + // num samples to draw. + uint64_t num_samples = 1000000; + uint64_t total_samples = 1000000; + std::vector weights(total_samples, std::rand() % 100); + std::vector freq(total_samples, 0); + + // create sampler with replacement = true + WeightedRandomSampler m_sampler(weights, num_samples, true); + DummyRandomAccessOp dummy_random_access_op(total_samples); + m_sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + ASSERT_EQ(num_samples, out.size()); + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); + + m_sampler.Reset(); + out.clear(); + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + ASSERT_EQ(num_samples, out.size()); + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); +} + +TEST_F(MindDataTestWeightedRandomSampler, TestResetNoReplacement) { + // num samples to draw. + uint64_t num_samples = 1000000; + uint64_t total_samples = 1000000; + std::vector weights(total_samples, std::rand() % 100); + std::vector freq(total_samples, 0); + + // create sampler with replacement = true + WeightedRandomSampler m_sampler(weights, num_samples, false); + DummyRandomAccessOp dummy_random_access_op(total_samples); + m_sampler.Init(&dummy_random_access_op); + + std::unique_ptr db; + TensorRow row; + std::vector out; + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + ASSERT_EQ(num_samples, out.size()); + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); + + m_sampler.Reset(); + out.clear(); + freq.clear(); + freq.resize(total_samples, 0); + MS_LOG(INFO) << "Resetting sampler"; + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + db->PopRow(&row); + for (const auto &t : row) { + for (auto it = t->begin(); it != t->end(); it++) { + out.push_back(*it); + freq[*it]++; + } + } + ASSERT_EQ(num_samples, out.size()); + + // Without replacement, each sample only drawn once. + for (int i = 0; i < total_samples; i++) { + if (freq[i]) { + ASSERT_EQ(freq[i], 1); + } + } + + ASSERT_EQ(m_sampler.GetNextBuffer(&db), Status::OK()); + ASSERT_EQ(db->eoe(), true); +} diff --git a/tests/ut/cpp/dataset/zip_op_test.cc b/tests/ut/cpp/dataset/zip_op_test.cc new file mode 100644 index 00000000000..d083fa1c6dc --- /dev/null +++ b/tests/ut/cpp/dataset/zip_op_test.cc @@ -0,0 +1,220 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "dataset/core/client.h" +#include "dataset/core/constants.h" +#include "dataset/engine/datasetops/map_op.h" +#include "dataset/engine/datasetops/zip_op.h" +#include "dataset/core/tensor.h" +#include "dataset/core/config_manager.h" +#include "common/common.h" +#include "common/utils.h" +#include "dataset/engine/data_buffer.h" +#include "gtest/gtest.h" +#include "dataset/core/global_context.h" +#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestZipOp : public UT::DatasetOpTesting { + }; + +TEST_F(MindDataTestZipOp, MindDataTestZipOpDefault) { +/* Tree: + * + * + * OpId(2) ZipOp + * / \ + * OpId(0) StorageOp OpId(1) StorageOp + * Start with an empty execution tree +*/ + Status rc; + MS_LOG(INFO) << "UT test TestZipBasic."; + auto my_tree = std::make_shared(); + // Creating StorageOp + + std::string dataset_path = datasets_root_path_ + "/test_tf_file_3_images_1"; + std::string dataset_path2 = datasets_root_path_ + "/test_tf_file_3_images_2"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(16) + .SetNumWorkers(1) + .Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + std::shared_ptr my_storage_op2; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path2) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(1) + .SetNumWorkers(1) + .Build(&my_storage_op2); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op2); + EXPECT_TRUE(rc.IsOk()); + + // Creating DatasetOp + std::shared_ptr zip_op; + rc = ZipOp::Builder().Build(&zip_op); + EXPECT_TRUE(rc.IsOk()); + + rc = my_tree->AssociateNode(zip_op); + EXPECT_TRUE(rc.IsOk()); + rc = zip_op->AddChild(std::move(my_storage_op)); + EXPECT_TRUE(rc.IsOk()); + rc = zip_op->AddChild(std::move(my_storage_op2)); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(zip_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + + // Launch the tree execution to kick off threads and start running the pipeline + MS_LOG(INFO) << "Launching my tree."; + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Simulate a parse of data from our pipeline. + std::shared_ptr rootNode = my_tree->root(); + + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()) << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 3); // Should be 3 rows fetched +} + + +TEST_F(MindDataTestZipOp, MindDataTestZipOpRepeat) { +/* Tree: + * OpId(3) Repeat(3) + * + * OpId(2) ZipOp + * / \ + * OpId(0) StorageOp OpId(1) StorageOp + * + * Start with an empty execution tree +*/ + Status rc; + MS_LOG(INFO) << "UT test TestZipRepeat."; + auto my_tree = std::make_shared(); + + std::string dataset_path = datasets_root_path_ + "/test_tf_file_3_images_1"; + std::string dataset_path2 = datasets_root_path_ + "/test_tf_file_3_images_2"; + std::shared_ptr my_storage_op; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(16) + .SetNumWorkers(1) + .Build(&my_storage_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op); + EXPECT_TRUE(rc.IsOk()); + std::shared_ptr my_storage_op2; + rc = StorageOp::Builder() + .SetDatasetFilesDir(dataset_path2) + .SetRowsPerBuffer(2) + .SetWorkerConnectorSize(1) + .SetNumWorkers(1) + .Build(&my_storage_op2); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_storage_op2); + EXPECT_TRUE(rc.IsOk()); + // Creating DatasetOp + std::shared_ptr zip_op; + rc = ZipOp::Builder().Build(&zip_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(zip_op); + EXPECT_TRUE(rc.IsOk()); + rc = zip_op->AddChild(std::move(my_storage_op)); + EXPECT_TRUE(rc.IsOk()); + rc = zip_op->AddChild(std::move(my_storage_op2)); + EXPECT_TRUE(rc.IsOk()); + + // Builder(num_of_repeats) + std::shared_ptr my_repeat_op; + rc = RepeatOp::Builder(3).Build(&my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssociateNode(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_repeat_op->AddChild(zip_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->AssignRoot(my_repeat_op); + EXPECT_TRUE(rc.IsOk()); + rc = my_tree->Prepare(); + EXPECT_TRUE(rc.IsOk()); + + // Launch the tree execution to kick off threads and start running the pipeline + MS_LOG(INFO) << "Launching my tree."; + rc = my_tree->Launch(); + EXPECT_TRUE(rc.IsOk()); + + // Simulate a parse of data from our pipeline. + std::shared_ptr rootNode = my_tree->root(); + + DatasetIterator di(my_tree); + TensorRow tensor_list; + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + + int row_count = 0; + while (!tensor_list.empty()) { + MS_LOG(INFO) << "Row display for row #: " << row_count << "."; + + // Display the tensor by calling the printer on it + for (int i = 0; i < tensor_list.size(); i++) { + std::ostringstream ss; + ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl; + MS_LOG(INFO) << "Tensor print: " << common::SafeCStr(ss.str()) << "."; + } + rc = di.FetchNextTensorRow(&tensor_list); + EXPECT_TRUE(rc.IsOk()); + row_count++; + } + ASSERT_EQ(row_count, 9); // Should be 9 rows fetched +} diff --git a/tests/ut/cpp/debug/memory_dumper_test.cc b/tests/ut/cpp/debug/memory_dumper_test.cc new file mode 100644 index 00000000000..b0676b481da --- /dev/null +++ b/tests/ut/cpp/debug/memory_dumper_test.cc @@ -0,0 +1,129 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "common/common_test.h" +#include "./common.h" +#include "utils/system/file_system.h" +#include "utils/system/env.h" +#define private public +#include "debug/e2e_dump.h" +#undef private + +namespace mindspore { +class TestMemoryDumper : public UT::Common { + public: + TestMemoryDumper() {} +}; + +TEST_F(TestMemoryDumper, test_DumpToFileAbsPath) { + int len = 1000; + int data[1000] = {0}; + for (uint32_t i = 0; i < len; i++) { + data[i] = i % 10; + } + + int ret; + char filename[] = "/tmp/dumpToFileTestFile"; + ret = mindspore::Dump::DumpToFile(filename, data, len * sizeof(int)); + ASSERT_EQ(ret, true); + + int fd = open(filename, O_RDONLY); + int readBack[1000] = {0}; + int readSize = read(fd, readBack, len * sizeof(int)); + (void)close(fd); + ASSERT_EQ(readSize, len * sizeof(int)); + + ret = true; + for (uint32_t i = 0; i < len; i++) { + if (data[i] != readBack[i]) { + ret = false; + break; + } + } + std::shared_ptr fs = system::Env::GetFileSystem(); + if (fs->FileExist(filename)) { + fs->DeleteFile(filename); + } + ASSERT_EQ(ret, true); +} + +TEST_F(TestMemoryDumper, test_DumpToFileRelativePath) { + int len = 1000; + int data[1000] = {0}; + for (uint32_t i = 0; i < len; i++) { + data[i] = i % 10; + } + + int ret; + char filename[] = "../../dumpToFileTestFile"; + ret = mindspore::Dump::DumpToFile(filename, data, len * sizeof(int)); + ASSERT_EQ(ret, true); + + int fd = open(filename, O_RDONLY); + int readBack[1000] = {0}; + int readSize = read(fd, readBack, len * sizeof(int)); + (void)close(fd); + ASSERT_EQ(readSize, len * sizeof(int)); + + ret = true; + for (uint32_t i = 0; i < len; i++) { + if (data[i] != readBack[i]) { + ret = false; + break; + } + } + std::shared_ptr fs = system::Env::GetFileSystem(); + if (fs->FileExist(filename)) { + fs->DeleteFile(filename); + } + + ASSERT_EQ(ret, true); +} + +TEST_F(TestMemoryDumper, test_DumpToFileNotExistDir) { + int len = 1; + int data[1] = {0}; + for (uint32_t i = 0; i < len; i++) { + data[i] = i % 10; + } + + char filename[] = "./tmp/dumpToFileTestFile"; + int ret = mindspore::Dump::DumpToFile(filename, data, len * sizeof(int)); + ASSERT_EQ(ret, true); + + int fd = open(filename, O_RDONLY); + int readBack[1000] = {0}; + int readSize = read(fd, readBack, len * sizeof(int)); + (void)close(fd); + ASSERT_EQ(readSize, len * sizeof(int)); + + ret = true; + for (uint32_t i = 0; i < len; i++) { + if (data[i] != readBack[i]) { + ret = false; + break; + } + } + std::shared_ptr fs = system::Env::GetFileSystem(); + if (fs->FileExist(filename)) { + fs->DeleteFile(filename); + } + + ASSERT_EQ(ret, true); +} +} // namespace mindspore diff --git a/tests/ut/cpp/device/ascend_kernel_runtime_test.cc b/tests/ut/cpp/device/ascend_kernel_runtime_test.cc new file mode 100644 index 00000000000..effa0b212da --- /dev/null +++ b/tests/ut/cpp/device/ascend_kernel_runtime_test.cc @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" + +#include "device/kernel_runtime.h" +#include "./common.h" + +namespace mindspore { +class TestAscendKernelRuntime : public UT::Common { + public: + TestAscendKernelRuntime() {} +}; + +TEST_F(TestAscendKernelRuntime, test_kernel_runtime) { printf("TestAscendKernelRuntime Test\n"); } + +} // namespace mindspore diff --git a/tests/ut/cpp/device/ascend_kernel_select_test.cc b/tests/ut/cpp/device/ascend_kernel_select_test.cc new file mode 100644 index 00000000000..d522a5adc08 --- /dev/null +++ b/tests/ut/cpp/device/ascend_kernel_select_test.cc @@ -0,0 +1,345 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "mindspore/ccsrc/device/ascend/kernel_select_ascend.h" +#include "common/common_test.h" +#include "session/kernel_graph.h" +#include "kernel/kernel.h" +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" +#include "operator/ops.h" +#include "mindspore/ccsrc/device/kernel_info.h" +#include "mindspore/ccsrc/kernel/kernel_build_info.h" +#include +namespace mindspore { +namespace device { +namespace ascend { +namespace { +using KernelInfo = device::KernelInfo; +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +using KernelBuildInfo = kernel::KernelBuildInfo; +using KernelGraph = session::KernelGraph; +using KernelBuildInfoPtr = std::shared_ptr; +using KernelBuilderPtr = std::shared_ptr; +using Shape = std::vector; +using ShapeList = std::vector; +enum MatchCountPriority { + MATCH_COUNT_PRIORITY_BEGIN = 0, + MATCH_FORMAT_COUNT = MATCH_COUNT_PRIORITY_BEGIN, + MATCH_DTYPE_COUNT, + MATCH_NZ_FORMAT_COUNT, + MATCH_5D_FORMAT_COUNT, + MATCH_OUTPUT_DTYPE_COUNT, + MATCH_COUNT_PRIORITY_END +}; + +const std::set kOpFormatList = { + kOpFormat_DEFAULT, kOpFormat_NC1KHKWHWC0, kOpFormat_ND, kOpFormat_NCHW, kOpFormat_NHWC, + kOpFormat_HWCN, kOpFormat_NC1HWC0, kOpFormat_FRAC_Z, kOpFormat_C1HWNCoC0, kOpFormat_FRAC_NZ}; + +bool IsShapeMatchFormat(const std::vector &shape, const std::string &format) { + // if format is default,it remarkes support all format + if (kOpFormatList.find(format) == kOpFormatList.end()) { + MS_EXCEPTION(ArgumentError) << "got the unknow format " << format; + } + if (format == kOpFormat_DEFAULT) { + return true; + } + // if shape size is 0,the shape will be a scalar + if (shape.empty()) { + return true; + } + if (shape.size() > kShapeSupportFormatMap.size()) { + return false; + } + if (format == kOpFormat_FRAC_NZ && shape.size() >= 2) { + return shape[shape.size() - 1] % 16 != 0 && shape[shape.size() - 2] % 16 != 0; + } + return !(kShapeSupportFormatMap[shape.size() - 1].find(format) == kShapeSupportFormatMap[shape.size() - 1].end()); +} + +bool IsValidKernelInfo(const std::shared_ptr &kernel_node, const kernel::KernelBuildInfo &kernel_build_info) { + MS_EXCEPTION_IF_NULL(kernel_node); + auto check_function = [](const std::vector &shape, const std::string &format) -> bool { + if (!IsShapeMatchFormat(shape, format)) { + return false; + } + for (auto shape_value : shape) { + if (shape_value == 0) { + MS_EXCEPTION(ArgumentError) << "dimension size of the tensor shape should be a positive integer, but got [" + << shape_value << "]"; + } + } + return true; + }; + for (size_t index = 0; index < kernel_build_info.GetOutputNum(); ++index) { + auto output_shape = AnfAlgo::GetOutputInferShape(kernel_node, index); + if (!check_function(output_shape, kernel_build_info.GetOutputFormat(index))) { + return false; + } + } + for (size_t index = 0; index < kernel_build_info.GetInputNum(); ++index) { + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, index); + if (!check_function(input_shape, kernel_build_info.GetInputFormat(index))) { + return false; + } + } + return true; +} + +bool MatchInferOutputDataType(const CNodePtr &cnode, const kernel::KernelBuildInfo &kernel_build_info) { + MS_EXCEPTION_IF_NULL(cnode); + // Check input data type + for (size_t input_index = 0; input_index < kernel_build_info.GetInputNum(); ++input_index) { + AnfNodePtr cur_input = cnode->input(input_index + 1); + MS_EXCEPTION_IF_NULL(cur_input); + TypeId input_origin_type; + if (cur_input->isa() && AnfAlgo::IsParameterWeight(cur_input->cast())) { + // weight + input_origin_type = AnfAlgo::GetOutputDeviceDataType(cur_input, 0); + } else { + // feature map + input_origin_type = AnfAlgo::GetPrevNodeOutputInferDataType(cnode, input_index); + } + if (input_origin_type == kTypeUnknown) { + continue; + } + if (kernel_build_info.GetInputDeviceType(input_index) != input_origin_type) { + return false; + } + } + // Check output data type + for (size_t output_index = 0; output_index < kernel_build_info.GetOutputNum(); ++output_index) { + if (kernel_build_info.GetOutputDeviceType(output_index) != AnfAlgo::GetOutputInferDataType(cnode, output_index)) { + return false; + } + } + return true; +} + +/** + * compare too vector by priority,select a better vector,like compare too num,first compare highest num location,if + * equal then next num location + * example:[3,1,1,1] > [2,2,2,2] > [2,2,1,2] > [2,1,1,3] + */ +bool PriorityChooseItem(const std::vector &cur_item, std::vector *best_item) { + MS_EXCEPTION_IF_NULL(best_item); + if (cur_item.size() != best_item->size()) { + MS_LOG(ERROR) << "item size should be same!"; + return false; + } + // Update the best_item by comparing the cur_item and best_item + for (size_t i = 0; i < cur_item.size(); i++) { + if (cur_item[i] > best_item->at(i)) { + *best_item = cur_item; + return true; + } else if (cur_item[i] == best_item->at(i)) { + continue; + } else { + return false; + } + } + return false; +} + +void UpdateCurMatchCounts(const kernel::KernelBuildInfo &kernel_build_info, const std::shared_ptr &kernel_node, + std::vector *const cur_kernelinfo_match_counts) { + MS_EXCEPTION_IF_NULL(kernel_node); + MS_EXCEPTION_IF_NULL(cur_kernelinfo_match_counts); + if (cur_kernelinfo_match_counts->size() < MATCH_COUNT_PRIORITY_END) { + MS_EXCEPTION(ArgumentError) << "Out of range cur_kernelinfo_match_counts " << MATCH_COUNT_PRIORITY_END; + } + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(kernel_node); ++input_index) { + AnfNodePtr input_anf_node = kernel_node->input(input_index + 1); + MS_EXCEPTION_IF_NULL(input_anf_node); + // if a input parameter is a weight with default format, the input shouldn't participate the judge + if (input_anf_node->isa()) { + auto para = input_anf_node->cast(); + if (AnfAlgo::IsParameterWeight(para) && AnfAlgo::GetOutputDeviceDataType(para, 0) == kTypeUnknown) { + continue; + } + } + if (kernel_build_info.GetInputFormat(input_index) == AnfAlgo::GetPrevNodeOutputFormat(kernel_node, input_index)) { + (*cur_kernelinfo_match_counts)[MATCH_FORMAT_COUNT]++; + } + if (kernel_build_info.GetInputDeviceType(input_index) == + AnfAlgo::GetPrevNodeOutputDeviceDataType(kernel_node, input_index)) { + (*cur_kernelinfo_match_counts)[MATCH_DTYPE_COUNT]++; + } + if (kernel_build_info.GetInputFormat(input_index) == kOpFormat_FRAC_NZ) { + (*cur_kernelinfo_match_counts)[MATCH_NZ_FORMAT_COUNT]++; + } + if (kernel_build_info.GetInputFormat(input_index) == kOpFormat_NC1HWC0) { + (*cur_kernelinfo_match_counts)[MATCH_5D_FORMAT_COUNT]++; + } + } + + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(kernel_node); ++output_index) { + // cal count of same output dtype between abstract and kernel info + if (kernel_build_info.GetOutputDeviceType(output_index) == + AnfAlgo::GetOutputInferDataType(kernel_node, output_index)) { + (*cur_kernelinfo_match_counts)[MATCH_OUTPUT_DTYPE_COUNT]++; + } + } +} + +void SetKernelBuildInfo(KernelBuilderPtr builder) { + builder->SetFusionType(kernel::OPAQUE); + builder->SetKernelType(AUTO_DIFF_KERNEL); + builder->SetProcessor(kernel::AICORE); +} + +void test_select(const CNodePtr &kernel_node, std::vector> kernel_info_list) { + std::vector most_match_counts = {-1, -1, -1, -1, -1}; + int selected_index = -1; + for (size_t info_index = 0; info_index < kernel_info_list.size(); ++info_index) { + std::vector cur_kernel_info_match_counts = {0, 0, 0, 0, 0}; + if (!IsValidKernelInfo(kernel_node, *(kernel_info_list[info_index]))) { + continue; + } + if (!MatchInferOutputDataType(kernel_node, *(kernel_info_list[info_index]))) { + continue; + } + std::shared_ptr kernel_info_ptr = kernel_info_list[info_index]; + UpdateCurMatchCounts(*kernel_info_ptr, kernel_node, &cur_kernel_info_match_counts); + // Currently the selection policy is the match format count first, and then is datatype counts. + if (PriorityChooseItem(cur_kernel_info_match_counts, &most_match_counts)) { + selected_index = SizeToInt(info_index); + } + } + if (selected_index == -1) { + MS_EXCEPTION(NotExistsError) << "" << kernel_node->DebugString() << " Cannot find valid kernel Info !"; + } + auto index = IntToSize(selected_index); + if (index >= kernel_info_list.size()) { + MS_EXCEPTION(ArgumentError) << "index outof range"; + } + std::shared_ptr selected_kernel_info_ptr = kernel_info_list[index]; + MS_EXCEPTION_IF_NULL(selected_kernel_info_ptr); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_info_ptr, kernel_node.get()); +} + +void SetParentAbstract(std::vector parent_list, std::vector> shapes, + std::vector types) { + for (const auto &node : parent_list) { + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, node.get()); + } +} +} // namespace +class AscendKernelSelctTest : public UT::Common { + public: + AscendKernelSelctTest() = default; + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(AscendKernelSelctTest, TestSelect) { + std::vector build_list; + std::vector type_list = {kNumberTypeFloat32}; + for (size_t i = 0; i <= 4; ++i) { + build_list.push_back(std::make_shared()); + SetKernelBuildInfo(build_list[i]); + build_list[i]->SetInputsDeviceType(type_list); + build_list[i]->SetOutputsDeviceType(type_list); + } + + std::vector nd_fmt = {kOpFormat_DEFAULT}; + std::vector nz_fmt = {kOpFormat_FRAC_NZ}; + auto anf_graph = std::make_shared(); + + // 16's multiple should not chose format NZ + Shape nd_shapes = {2, 32, 224, 224}; + + Shape nz_shapes = {3, 3, 5, 5}; + auto add_value = NewValueNode(prim::kPrimTensorAdd); + auto a_node = anf_graph->NewCNode(std::vector{add_value}); + auto b_node = anf_graph->NewCNode(std::vector{add_value}); + std::vector parent_list = {add_value, a_node, b_node}; + + auto c_node = anf_graph->NewCNode(parent_list); + + // a b + // \ / + // c + // a & b: kernel_info:{output_format:{nz},dtype:{kNumberTypeFloat32}} + // infer_dtype:{kNumberTypeFloat32},infer_shape:{{3, 3, 5, 5}} + // c: infer_dtype:{kNumberTypeFloat32},infer_shape:{{3, 3,224, 224}} + + // set a & b's info + SetParentAbstract(parent_list, ShapeList{nz_shapes}, type_list); + // set abstract c + AnfAlgo::SetOutputInferTypeAndShape(type_list, ShapeList{nd_shapes}, c_node.get()); + // set format of kernel info + build_list[0]->SetOutputsFormat(nz_fmt); + build_list[1]->SetOutputsFormat(nz_fmt); + + build_list[2]->SetInputsFormat(std::vector{nd_fmt[0], nd_fmt[0]}); + build_list[3]->SetInputsFormat(std::vector{nz_fmt[0], nz_fmt[0]}); + build_list[2]->SetInputsDeviceType(std::vector{kNumberTypeFloat32, kNumberTypeFloat32}); + build_list[3]->SetInputsDeviceType(std::vector{kNumberTypeFloat32, kNumberTypeFloat32}); + build_list[2]->SetOutputsFormat(nd_fmt); + build_list[3]->SetOutputsFormat(nz_fmt); + std::vector select_info_list; + // set select info list + select_info_list.emplace_back(build_list[2]->Build()); + select_info_list.emplace_back(build_list[3]->Build()); + + // set device info for a & b + AnfAlgo::SetSelectKernelBuildInfo(build_list[0]->Build(), a_node.get()); + AnfAlgo::SetSelectKernelBuildInfo(build_list[1]->Build(), b_node.get()); + + test_select(c_node, select_info_list); + EXPECT_EQ(AnfAlgo::GetInputFormat(c_node, 0), kOpFormat_DEFAULT); + EXPECT_EQ(AnfAlgo::GetInputFormat(c_node, 1), kOpFormat_DEFAULT); + + // set a & b's info + // a b + // \ / + // c + // a: kernel_info:{output_format:{5d},dtype:{kNumberTypeFloat32}} + // infer_dtype:{kNumberTypeFloat32},infer_shape:{{3, 3, 5, 5}} + // b: kernel_info:{output_format:{nz},dtype:{kNumberTypeFloat32}} + // infer_dtype:{kNumberTypeFloat32},infer_shape:{{3, 3, 5, 5}} + // c: infer_dtype:{kNumberTypeFloat32},infer_shape:{{3, 3, 5, 5}} + + // set a & b's info + SetParentAbstract(parent_list, ShapeList{nz_shapes}, type_list); + // set abstract c + AnfAlgo::SetOutputInferTypeAndShape(type_list, ShapeList{nz_shapes}, c_node.get()); + // set format of kernel info + build_list[0]->SetOutputsFormat(std::vector{kOpFormat_NC1HWC0}); + build_list[1]->SetOutputsFormat(nz_fmt); + + build_list[2]->SetInputsFormat(std::vector{kOpFormat_NC1HWC0, nd_fmt[0]}); + build_list[3]->SetInputsFormat(std::vector{nd_fmt[0], nz_fmt[0]}); + build_list[2]->SetInputsDeviceType(std::vector{kNumberTypeFloat32, kNumberTypeFloat32}); + build_list[3]->SetInputsDeviceType(std::vector{kNumberTypeFloat32, kNumberTypeFloat32}); + build_list[2]->SetOutputsFormat(nd_fmt); + build_list[3]->SetOutputsFormat(nz_fmt); + // set select info list + select_info_list.emplace_back(build_list[2]->Build()); + select_info_list.emplace_back(build_list[3]->Build()); + + // set device info for a & b + AnfAlgo::SetSelectKernelBuildInfo(build_list[0]->Build(), a_node.get()); + AnfAlgo::SetSelectKernelBuildInfo(build_list[1]->Build(), b_node.get()); + + test_select(c_node, select_info_list); + EXPECT_EQ(AnfAlgo::GetInputFormat(c_node, 0), kOpFormat_DEFAULT); + EXPECT_EQ(AnfAlgo::GetInputFormat(c_node, 1), kOpFormat_FRAC_NZ); +} +} // namespace ascend +} // namespace device +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/device/ascend_profiling_test.cc b/tests/ut/cpp/device/ascend_profiling_test.cc new file mode 100644 index 00000000000..2bfdc9fcae0 --- /dev/null +++ b/tests/ut/cpp/device/ascend_profiling_test.cc @@ -0,0 +1,125 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" +#include "device/ascend/profiling/profiling_manager.h" +#include "./common.h" +#include "./prof_reporter.h" +#define private public +#include "device/ascend/profiling/plugin_impl.h" +#undef private +#include "device/ascend/profiling/profiling_engine_impl.h" + +namespace mindspore { +namespace device { +namespace ascend { +class stubReporter : public Reporter { + public: + stubReporter() = default; + ~stubReporter() = default; + + int Report(const Msprof::Engine::ReporterData *data) override; + int Flush() override; +}; + +int stubReporter::Report(const Msprof::Engine::ReporterData *data) { return 0; } + +int stubReporter::Flush() { return 0; } + +class TestAscendProfiling : public UT::Common { + public: + TestAscendProfiling() {} +}; + +TEST_F(TestAscendProfiling, test_profiling_GetJobId) { + auto job_id = ProfilingManager::GetInstance().GetJobId(); + printf("get job_id:%ld\n", job_id); +} + +int test_profiling_start() { + (void)setenv("PROFILING_MODE", "true", 1); + (void)setenv("PROFILING_OPTIONS", "training_trace:task_trace", 1); + auto ret = ProfilingManager::GetInstance().StartupProfiling(0); + (void)unsetenv("PROFILING_MODE"); + (void)unsetenv("PROFILING_OPTIONS"); + return ret; +} + +TEST_F(TestAscendProfiling, test_profiling_start) { + auto ret = test_profiling_start(); + ASSERT_EQ(ret, true); +} + +int test_profiling_stop() { + (void)setenv("PROFILING_MODE", "true", 1); + auto engine = std::make_shared(); + auto report = std::make_shared(); + auto plug = engine->CreatePlugin(); + plug->Init(report.get()); + auto ret = ProfilingManager::GetInstance().StopProfiling(); + plug->UnInit(); + engine->ReleasePlugin(plug); + (void)unsetenv("PROFILING_OPTIONS"); + return ret; +} + +TEST_F(TestAscendProfiling, test_profiling_stop) { + auto ret = test_profiling_stop(); + ASSERT_EQ(ret, true); +} + +int test_profiling_rpt() { + (void)setenv("PROFILING_MODE", "true", 1); + std::map op_taskId_map; + op_taskId_map[1] = "add"; + op_taskId_map[2] = "mul"; + auto engine = std::make_shared(); + auto report = std::make_shared(); + auto plug = engine->CreatePlugin(); + plug->Init(report.get()); + ProfilingManager::GetInstance().ReportProfilingData(op_taskId_map); + plug->UnInit(); + engine->ReleasePlugin(plug); + (void)unsetenv("PROFILING_OPTIONS"); + return 0; +} + +TEST_F(TestAscendProfiling, test_profiling_rpt) { + auto ret = test_profiling_rpt(); + ASSERT_EQ(ret, false); +} + +int test_profiling_rpt_abnormal() { + std::map op_taskId_map; + ProfilingManager::GetInstance().ReportProfilingData(op_taskId_map); + (void)setenv("PROFILING_MODE", "true", 1); + ProfilingManager::GetInstance().ReportProfilingData(op_taskId_map); + op_taskId_map[1] = "add"; + op_taskId_map[2] = "mul"; + ProfilingManager::GetInstance().ReportProfilingData(op_taskId_map); + (void)unsetenv("PROFILING_OPTIONS"); + return 0; +} + +TEST_F(TestAscendProfiling, test_profiling_rpt_abnormal) { + auto ret = test_profiling_rpt_abnormal(); + ASSERT_EQ(ret, false); +} +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/tests/ut/cpp/ir/anf_test.cc b/tests/ut/cpp/ir/anf_test.cc new file mode 100644 index 00000000000..c649518e219 --- /dev/null +++ b/tests/ut/cpp/ir/anf_test.cc @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" + +#include "ir/anf.h" +#include "operator/ops.h" +#include "./common.h" + +namespace mindspore { + +using Named = Named; + +class TestAnf : public UT::Common { + public: + TestAnf() {} +}; + +TEST_F(TestAnf, test_ValueNode) { + auto prim = std::make_shared("scalar_add"); + ValueNodePtr c = NewValueNode(prim); + ASSERT_EQ(c->isa(), true); + ASSERT_EQ(IsValueNode(c), true); + ASSERT_EQ(IsValueNode(c), false); + + FuncGraphPtr fg = std::make_shared(); + ValueNode c1(fg); + ASSERT_EQ(c1.value()->isa(), true); +} + +TEST_F(TestAnf, test_Parameter) { + FuncGraphPtr fg = std::make_shared(); + Parameter a(fg); + assert(a.isa()); +} + +TEST_F(TestAnf, test_CNode) { + auto primitive = prim::kPrimScalarAdd; + + FuncGraphPtr fg = std::make_shared(); + std::string s = fg->ToString(); + + Parameter param(fg); + std::vector params; + CNode app_1(params, fg); + params.push_back(NewValueNode(primitive)); + params.push_back(AnfNodePtr(new Parameter(param))); + CNode app(params, fg); + assert(app.isa()); + assert(app.IsApply(primitive)); +} + +TEST_F(TestAnf, is_exception) { + FuncGraphPtr fg = std::make_shared(); + Parameter a(fg); + assert(!a.isa()); + assert(!a.isa()); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/ir/base_test.cc b/tests/ut/cpp/ir/base_test.cc new file mode 100644 index 00000000000..0b4e8a637b4 --- /dev/null +++ b/tests/ut/cpp/ir/base_test.cc @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "common/common_test.h" +#include "utils/any.h" +#include "ir/base.h" +#include "ir/anf.h" +#include "utils/log_adapter.h" + +namespace mindspore { + +class TestNode : public UT::Common { + public: + TestNode() {} +}; + +class ChildA : public Base { + public: + ChildA() {} + ~ChildA() {} + MS_DECLARE_PARENT(ChildA, Base); + std::string name() { return "ChildA"; } + std::size_t hash() const override { return 1; } +}; +class ChildAA : public ChildA { + public: + ChildAA() {} + ~ChildAA() {} + MS_DECLARE_PARENT(ChildAA, ChildA); + std::size_t hash() const override { return 1; } + std::string name() { return "ChildAA"; } +}; + +class ChildB : public Base { + public: + ChildB() {} + ~ChildB() {} + MS_DECLARE_PARENT(ChildB, Base); + std::size_t hash() const override { return 1; } + std::string name() { return "ChildB"; } +}; + +TEST_F(TestNode, test_dyn_cast) { + auto aa = std::make_shared(); + std::shared_ptr n = aa; + MS_LOG(INFO) << "aa ptr_name: " << aa->name(); + MS_LOG(INFO) << "aa type_name: " << aa->type_name(); + MS_LOG(INFO) << "n ptr_name: " << demangle(typeid(n).name()); + MS_LOG(INFO) << "n type_name: " << n->type_name(); + ASSERT_TRUE(n != nullptr); + ASSERT_EQ(std::string(n->type_name().c_str()), "ChildAA"); + auto a = dyn_cast(n); + MS_LOG(INFO) << "a ptr_name: " << a->name(); + MS_LOG(INFO) << "a type_name: " << a->type_name(); + ASSERT_TRUE(a != nullptr); + ASSERT_EQ(std::string(a->name()), "ChildA"); + ASSERT_EQ(std::string(a->type_name().c_str()), "ChildAA"); + auto b_null = dyn_cast(n); + ASSERT_TRUE(b_null == nullptr); + + ChildA* pa = cast(n.get()); + ASSERT_TRUE(pa != nullptr); + MS_LOG(INFO) << "a ptr_name: " << pa->name(); + MS_LOG(INFO) << "a type_name: " << pa->type_name(); +} + +TEST_F(TestNode, test_isa) { + auto a = std::make_shared(); + BasePtr n = a; + ASSERT_TRUE(n->isa() == true); + ASSERT_TRUE(n->isa() == false); + + auto aa = std::make_shared(); + n = aa; + ASSERT_TRUE(n->isa() == true); + ASSERT_TRUE(n->isa() == true); + + auto b = std::make_shared(); + n = b; + ASSERT_TRUE(n->isa() == true); + ASSERT_TRUE(n->isa() == false); + ASSERT_TRUE(n->isa() == false); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/ir/clone_test.cc b/tests/ut/cpp/ir/clone_test.cc new file mode 100644 index 00000000000..bb8cae7fbb9 --- /dev/null +++ b/tests/ut/cpp/ir/clone_test.cc @@ -0,0 +1,200 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/manager.h" +#include "utils/log_adapter.h" +#include "ir/func_graph_cloner.h" +#include "pipeline/parse/parse.h" +#include "utils/graph_utils.h" +#include "debug/draw.h" +#include "./common.h" + +namespace mindspore { +class TestCloner : public UT::Common { + public: + TestCloner() : getPyFun("gtest_input.ir.clone_test", true) { + one = NewValueNode(1); + two = NewValueNode(2); + three = NewValueNode(3); + } + + FuncGraphPtr GraphForInline() { return nullptr; } + void SuccessfulInlining(const std::shared_ptr cl, FuncGraphPtr orig, const std::vector& params, + FuncGraphPtr target); + + public: + UT::PyFuncGraphFetcher getPyFun; + + ValueNodePtr one; + ValueNodePtr two; + ValueNodePtr three; +}; + +void TestCloner::SuccessfulInlining(const std::shared_ptr cl, FuncGraphPtr orig, + const std::vector& params, FuncGraphPtr target) { + auto g = (*cl)[orig]; + ASSERT_TRUE(g != target); + ASSERT_TRUE(g == orig); + + auto new_root = (*cl)[orig->output()]; + ASSERT_TRUE(new_root != orig->output()); + + AnfNodeSet orig_nodes = AnfNodeSet(DeepLinkedGraphSearch(orig->output())); + AnfNodeSet new_nodes = AnfNodeSet(DeepLinkedGraphSearch(new_root)); + + for (auto& p : params) { + ASSERT_TRUE(new_nodes.contains(p)); + } + + for (auto& node : orig_nodes) { + if (node->func_graph() == orig) { + ASSERT_TRUE((*cl)[node]); + } + } + ASSERT_TRUE(target->output() == three); +} + +TEST_F(TestCloner, test_clone_simple) { + std::string py_code = "test_clone_simple"; + + FuncGraphPtr g = getPyFun.CallAndParseRet(py_code); + ASSERT_TRUE(g != nullptr); + + std::vector gs = {g}; + Cloner cl(gs, true); + auto g2 = cl[g]; + + AnfNodeSet d1 = AnfNodeSet(DeepScopedGraphSearch(g->get_return())); + AnfNodeSet d2 = AnfNodeSet(DeepScopedGraphSearch(g2->get_return())); + + auto common = d1 & d2; + ASSERT_EQ((size_t)0, common.size()); + + Cloner cl2(gs); + auto g3 = cl2[g]; + + std::vector results = {Primitive("scalar_add"), Primitive("scalar_mul"), Primitive("return")}; + AnfNodeSet d3 = AnfNodeSet(DeepScopedGraphSearch(g3->get_return())); + common = d1 & d3; + for (auto& x : common) { + ASSERT_TRUE(x->isa()); + ASSERT_TRUE(find(results.begin(), results.end(), *x->cast()->value()->cast()) != + results.end()); + } +} + +TEST_F(TestCloner, test_clone_closure) { + std::string py_code = "test_clone_closure"; + + // parse ast to graph + FuncGraphPtr parsed_f = getPyFun(py_code); + + FuncGraphIndex idx(parsed_f); + auto g = idx.GetFirstFuncGraph("j"); + + std::vector gs = {g}; + Cloner cl(gs, true); + + auto g_clone = cl[g]; + draw::Draw("test_clone_closure_g_clone.dot", g_clone); + FuncGraphIndex idx2(g_clone, DeepLinkedGraphSearch); + + std::string name_list = "xy"; + for (auto name : name_list) { + ASSERT_EQ(idx.GetFirstNode(std::string(1, name)), idx2.GetFirstNode(std::string(1, name))); + } + + ASSERT_FALSE(idx.GetFirstNode("z") == idx2.GetFirstNode("z")); + ASSERT_FALSE(idx.GetFirstFuncGraph("j") == idx2.GetFirstFuncGraph("j")); +} + +TEST_F(TestCloner, test_clone_lifting) { + std::string py_code = "test_clone_closure"; + + // parse ast to graph + FuncGraphPtr parsed_f = getPyFun(py_code); + draw::Draw("test_clone_before_lifting.dot", parsed_f); + + auto g_lifting = LiftingClone(parsed_f); + draw::Draw("test_clone_after_lifting.dot", g_lifting); + + FuncGraphIndex idx(g_lifting); + auto g = idx.GetFirstFuncGraph("j"); + + auto params = g_lifting->parameters(); + auto child_params = g->parameters(); + ASSERT_TRUE(params.size() + 1 == child_params.size()); +} + +TEST_F(TestCloner, test_clone_scoping) { + std::string py_code = "test_clone_scoping"; + + // parse ast to graph + FuncGraphPtr g = getPyFun.CallAndParseRet(py_code); + + std::vector gs = {g}; + Cloner cl(gs, true); + + auto g2 = cl[g]; + + FuncGraphIndex idx1(g); + FuncGraphIndex idx2(g2); + + std::string name_list = "fgi"; + for (auto name : name_list) { + auto result1 = idx1.GetFirstFuncGraph(std::string(1, name)); + auto result2 = idx2.GetFirstFuncGraph(std::string(1, name)); + ASSERT_FALSE(result1 == result2); + } + + name_list = "h"; + for (auto name : name_list) { + ASSERT_TRUE(idx1.GetFirstFuncGraph(std::string(1, name)) == idx2.GetFirstFuncGraph(std::string(1, name))); + } +} + +TEST_F(TestCloner, test_clone_total) { + std::string py_code = "test_clone_total"; + + // parse ast to graph + getPyFun.SetDoResolve(); + FuncGraphPtr g = getPyFun.CallAndParseRet(py_code); + if (g == nullptr) { + return; + } + + FuncGraphIndex idx0(g); + + std::vector gs = {g}; + Cloner cl1(gs, true, true, true); + auto g2 = cl1[g]; + FuncGraphIndex idx1(g2); + + ASSERT_FALSE(idx0.GetFirstFuncGraph("clone_total_sub") == idx1.GetFirstFuncGraph("clone_total_sub")); + ASSERT_FALSE(idx0.GetFirstFuncGraph("clone_total") == idx1.GetFirstFuncGraph("clone_total")); + + Cloner cl2(gs, true); + FuncGraphIndex idx2(cl2[g]); + + ASSERT_FALSE(idx0.GetFirstFuncGraph("clone_total") == idx2.GetFirstFuncGraph("clone_total")); + ASSERT_TRUE(idx0.GetFirstFuncGraph("clone_total_sub") == idx2.GetFirstFuncGraph("clone_total_sub")); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/ir/dtype_test.cc b/tests/ut/cpp/ir/dtype_test.cc new file mode 100644 index 00000000000..5bfdfb3af6a --- /dev/null +++ b/tests/ut/cpp/ir/dtype_test.cc @@ -0,0 +1,235 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common_test.h" +#include "ir/dtype.h" + +namespace mindspore { +class TestDType : public UT::Common { + public: + TestDType() : b(), i16(16), i32(32), i64(64), i64_2(64), u16(16), u32(32), u64(64), f32(32), f64(64), f64_2(64) {} + + void SetUp(); + void TearDown(); + + Bool b; + Int i16; + Int i32; + Int i64; + Int i64_2; + UInt u16; + UInt u32; + UInt u64; + Float f32; + Float f64; + Float f64_2; +}; + +void TestDType::SetUp() { + // init resource +} + +void TestDType::TearDown() { + // destroy resource +} + +// test_instantiate +// test_cache +TEST_F(TestDType, TestNumber) { + try { + Float f = Float(32); + } catch (std::range_error& e) { + MS_LOG(ERROR) << "build float 32 failed!!! error:" << e.what(); + ASSERT_TRUE(0); + return; + } + ASSERT_TRUE(1); +} + +TEST_F(TestDType, TestList) { + List l = {std::make_shared(64), std::make_shared(8), std::make_shared()}; + ASSERT_TRUE(*l.elements()[1] == Int(8)); + List t1 = {std::make_shared(), std::make_shared(32)}; + std::vector> v2 = {std::make_shared(), std::make_shared(32)}; + List t2(v2); + + ASSERT_EQ(t1, t2); +} + +TEST_F(TestDType, TestTuple) { + Tuple t = {std::make_shared(64), std::make_shared(8), std::make_shared()}; + ASSERT_TRUE(*t.elements()[1] == Int(8)); + Tuple t1 = {std::make_shared(), std::make_shared(32)}; + std::vector> v2 = {std::make_shared(), std::make_shared(32)}; + Tuple t2(v2); + + ASSERT_EQ(t1, t2); +} + +TEST_F(TestDType, TestDictionary) { + std::vector> kv = {std::make_pair("key1", std::make_shared(8)), + std::make_pair("key2", std::make_shared())}; + Dictionary d1 = Dictionary(kv); + Dictionary d2 = Dictionary(kv); + ASSERT_EQ(d1, d2); +} + +TEST_F(TestDType, TestFunction) { + std::vector> args1{std::make_shared(32), std::make_shared(32)}; + std::vector> args2{std::make_shared(64), std::make_shared(64)}; + std::vector> args3{}; + std::vector> args4{}; + + std::shared_ptr retval1 = std::make_shared(64); + std::shared_ptr retval2 = std::make_shared(64); + std::shared_ptr retval3 = std::make_shared(); + + Function fn1 = Function(args1, retval1); + Function fn2 = Function(args1, retval2); + Function fn3 = Function(args2, retval1); + Function fn4 = Function(args2, retval3); + + Function fn5 = Function(args3, retval1); + Function fn6 = Function(args4, retval1); + + ASSERT_EQ(fn1, fn2); + ASSERT_FALSE(fn1 != fn2); + ASSERT_FALSE(fn1 == fn3); + ASSERT_FALSE(fn1 == fn4); + ASSERT_FALSE(fn3 == fn4); + ASSERT_EQ(fn5, fn6); +} + +TEST_F(TestDType, TestTypeCloner) { + Tuple t1 = {std::make_shared(64), std::make_shared(64), std::make_shared()}; + Tuple t2 = {std::make_shared(64), std::make_shared(32), std::make_shared()}; + std::shared_ptr tc = Clone(t1); + + ASSERT_EQ(*tc, t1); +} + +TEST_F(TestDType, EqTest) { + ASSERT_EQ(i64, i64_2); + ASSERT_FALSE(i64 != i64_2); + ASSERT_EQ(f64, f64_2); + ASSERT_FALSE(f64 != f64_2); + ASSERT_FALSE(f32 == f64); + ASSERT_FALSE(i32 == i64); +} + +// test_get_generic +// test_repr +TEST_F(TestDType, TestRepr) { + UInt u = UInt(16); + ASSERT_EQ(u.ToString(), "UInt16"); +} + +TEST_F(TestDType, TestStringToType) { + ASSERT_EQ(TypeNone(), *StringToType("None")); + ASSERT_EQ(TypeType(), *StringToType("TypeType")); + ASSERT_EQ(Number(), *StringToType("Number")); + + ASSERT_EQ(Bool(), *StringToType("Bool")); + + ASSERT_EQ(UInt(8), *StringToType("UInt8")); + ASSERT_EQ(UInt(16), *StringToType("UInt16")); + ASSERT_EQ(UInt(32), *StringToType("UInt32")); + ASSERT_EQ(UInt(64), *StringToType("UInt64")); + + ASSERT_EQ(Int(8), *StringToType("Int8")); + ASSERT_EQ(Int(16), *StringToType("Int16")); + ASSERT_EQ(Int(32), *StringToType("Int32")); + ASSERT_EQ(Int(64), *StringToType("Int64")); + + ASSERT_EQ(Float(16), *StringToType("Float16")); + ASSERT_EQ(Float(32), *StringToType("Float32")); + ASSERT_EQ(Float(64), *StringToType("Float64")); +} + +TEST_F(TestDType, TestStringToType2) { + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("Tensor"), std::make_shared())); + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("List"), std::make_shared())); + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("Tuple"), std::make_shared())); + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("Tensor[Float32]"), StringToType("Tensor"))); + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("List[Float32]"), StringToType("List"))); + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("Tuple[Float64,Int64]"), StringToType("Tuple"))); + ASSERT_TRUE(IsIdentidityOrSubclass(StringToType("Function[(),Int64]"), StringToType("Function"))); + std::cout << StringToType("Function[(Tuple[Float64],Int64),Int32]") << std::endl; +} + +TEST_F(TestDType, TestTypeIdNormalize) { + ASSERT_EQ(kNumberTypeInt, NormalizeTypeId(kNumberTypeInt)); + ASSERT_EQ(kNumberTypeInt, NormalizeTypeId(kNumberTypeInt8)); + ASSERT_EQ(kNumberTypeInt, NormalizeTypeId(kNumberTypeInt16)); + ASSERT_EQ(kNumberTypeInt, NormalizeTypeId(kNumberTypeInt32)); + ASSERT_EQ(kNumberTypeInt, NormalizeTypeId(kNumberTypeInt64)); + + ASSERT_TRUE(kNumberTypeInt != NormalizeTypeId(kNumberTypeUInt)); + ASSERT_EQ(kNumberTypeUInt, NormalizeTypeId(kNumberTypeUInt)); + + ASSERT_EQ(kNumberTypeFloat, NormalizeTypeId(kNumberTypeFloat)); + ASSERT_EQ(kNumberTypeFloat, NormalizeTypeId(kNumberTypeFloat16)); + ASSERT_EQ(kNumberTypeFloat, NormalizeTypeId(kNumberTypeFloat32)); + ASSERT_EQ(kNumberTypeFloat, NormalizeTypeId(kNumberTypeFloat64)); + + ASSERT_EQ(kNumberTypeBool, NormalizeTypeId(kNumberTypeBool)); +} + +TEST_F(TestDType, TestTypeIdLabel) { +#define TEST_DTYPE_LABEL(type) ASSERT_EQ((#type), std::string(TypeIdLabel(type))) + + TEST_DTYPE_LABEL(kMetaTypeType); + TEST_DTYPE_LABEL(kMetaTypeAnything); + TEST_DTYPE_LABEL(kMetaTypeObject); + TEST_DTYPE_LABEL(kMetaTypeTypeType); + TEST_DTYPE_LABEL(kMetaTypeProblem); + TEST_DTYPE_LABEL(kMetaTypeExternal); + TEST_DTYPE_LABEL(kMetaTypeNone); + + // Object types + TEST_DTYPE_LABEL(kObjectTypeNumber); + TEST_DTYPE_LABEL(kObjectTypeList); + TEST_DTYPE_LABEL(kObjectTypeTuple); + TEST_DTYPE_LABEL(kObjectTypeTensorType); + TEST_DTYPE_LABEL(kObjectTypeClass); + TEST_DTYPE_LABEL(kObjectTypeFunction); + TEST_DTYPE_LABEL(kObjectTypeJTagged); + TEST_DTYPE_LABEL(kObjectTypeSymbolicKeyType); + TEST_DTYPE_LABEL(kObjectTypeEnvType); + TEST_DTYPE_LABEL(kObjectTypeRefKey); + TEST_DTYPE_LABEL(kObjectTypeRef); + + // Number Types + TEST_DTYPE_LABEL(kNumberTypeBool); + TEST_DTYPE_LABEL(kNumberTypeInt); + TEST_DTYPE_LABEL(kNumberTypeInt8); + TEST_DTYPE_LABEL(kNumberTypeInt16); + TEST_DTYPE_LABEL(kNumberTypeInt32); + TEST_DTYPE_LABEL(kNumberTypeInt64); + TEST_DTYPE_LABEL(kNumberTypeUInt); + TEST_DTYPE_LABEL(kNumberTypeUInt8); + TEST_DTYPE_LABEL(kNumberTypeUInt16); + TEST_DTYPE_LABEL(kNumberTypeUInt32); + TEST_DTYPE_LABEL(kNumberTypeUInt64); + TEST_DTYPE_LABEL(kNumberTypeFloat); + TEST_DTYPE_LABEL(kNumberTypeFloat16); + TEST_DTYPE_LABEL(kNumberTypeFloat32); + TEST_DTYPE_LABEL(kNumberTypeFloat64); +} + +} // namespace mindspore + +// test_type_cloner() diff --git a/tests/ut/cpp/ir/manager_test.cc b/tests/ut/cpp/ir/manager_test.cc new file mode 100644 index 00000000000..a7a19a7d24c --- /dev/null +++ b/tests/ut/cpp/ir/manager_test.cc @@ -0,0 +1,740 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "ir/dtype.h" +#include "ir/manager.h" +#include "ir/func_graph_cloner.h" +#include "pipeline/parse/parse.h" +#include "operator/ops.h" +#include "utils/log_adapter.h" +#include "debug/draw.h" +#include "debug/label.h" +#include "./common.h" + +namespace mindspore { + +namespace { +std::vector SplitString(std::string str, std::string pattern) { + std::string::size_type pos; + std::vector result; + str += pattern; + std::string::size_type size = str.size(); + + for (std::string::size_type i = 0; i < size; ++i) { + pos = str.find(pattern, i); + if (pos < size) { + std::string s = str.substr(i, pos - i); + result.push_back(s); + i = pos + pattern.size() - 1; + } + } + + return result; +} +} // namespace +using std::dynamic_pointer_cast; + +using TodoList = std::vector>, AnfNodePtr>>>; +using TodoListItem = std::vector>, AnfNodePtr>>; + +class NestingSpecs; + +class Stage { + public: + explicit Stage(std::vector specs) { + for (auto arg : specs) { + auto spec = SplitString(arg, "="); + if (spec.size() <= 1) { + continue; + } + std::shared_ptr nesting = std::make_shared(this, spec[1]); + specs_[ToFullString(spec[0])] = nesting; + } + } + + ~Stage() {} + + std::map& subs() { return subs_; } + + void set_subs(const std::map& subs) { subs_ = subs; } + + private: + std::string ToFullString(std::string s) { + if (s.find("fv") != std::string::npos) { + s = s.replace(s.find("fv"), 2, "free_variable"); + } + + if (s.find("deps") != std::string::npos) { + s = s.replace(s.find("deps"), 4, "dependencies"); + } + + return s; + } + + std::map> specs_; + std::map subs_; +}; + +class NestingSpecs { + public: + NestingSpecs(Stage* stage, std::string specs) : stage_(stage) { ParseSpecs(specs); } + + ~NestingSpecs() {} + + std::string Name(Any node) { + std::string name = label_manage::Label(node.cast()->debug_info()); + if (stage_->subs().find(name) != stage_->subs().end()) { + return stage_->subs()[name]; + } + + return name; + } + + void Check(std::shared_ptr results) { + if (expected_.empty() && expected_recursive_.empty()) { + return; + } + + auto parent = dynamic_pointer_cast(results); + if (parent != nullptr) { + CheckParent(parent); + return; + } + + auto recursive = dynamic_pointer_cast(results); + if (recursive != nullptr) { + CheckRecursive(recursive); + return; + } + + auto counter_g = dynamic_pointer_cast(results); + if (counter_g != nullptr) { + CheckGraphCounter(counter_g); + return; + } + + auto counter_p = dynamic_pointer_cast(results); + if (counter_p != nullptr) { + CheckAnfNodeCounter(counter_p); + return; + } + + auto nodes = dynamic_pointer_cast(results); + if (nodes != nullptr) { + CheckNodes(nodes); + return; + } + } + + private: + void ParseSpecs(std::string specs) { + if (specs.empty()) { + return; + } + + std::vector str_list = SplitString(specs, ";"); + for (auto spec : str_list) { + spec.erase(0, spec.find_first_not_of(" ")); + spec.erase(spec.find_last_not_of(" ") + 1); + if (spec.empty()) { + continue; + } + if (spec.find("->") != std::string::npos) { + auto substr = SplitString(spec, "->"); + ASSERT_GT(substr.size(), 1); + auto key = substr[0]; + auto value = substr[1]; + if (!value.empty()) { + expected_[key] = {value}; + } + } else if (spec.find(":") != std::string::npos) { + auto substr = SplitString(spec, ":"); + ASSERT_GT(substr.size(), 1); + auto key = substr[0]; + auto values = SplitString(substr[1], ","); + std::set values_set(values.begin(), values.end()); + if (!values_set.empty()) { + expected_[key] = values_set; + } + } else { + expected_recursive_[spec] = true; + } + } + } + + void CheckParent(std::shared_ptr results) { + std::map> clean_results; + for (auto& iter : results->parent_analysis()) { + auto key = iter.first; + auto value = iter.second; + if (key == nullptr) { + continue; + } + std::string k = Name(key); + + std::set v; + if (value != nullptr && !Name(value).empty()) { + v.insert(Name(value)); + } + + if (!v.empty()) { + clean_results[k] = v; + } + } + + ASSERT_EQ(clean_results, expected_); + } + + void CheckNodes(std::shared_ptr results) { + std::map> clean_results; + for (auto& iter : results->nodes_analysis()) { + auto key = iter.first; + auto value = iter.second; + if (key == nullptr) { + continue; + } + std::string k = Name(key); + + std::set v; + for (auto& node : value) { + if (!node->isa() && !Name(node).empty()) { + v.insert(Name(node)); + } + } + + if (!v.empty()) { + clean_results[k] = v; + } + } + + ASSERT_EQ(clean_results, expected_); + } + + // Add CheckNesting function + + void CheckAnfNodeCounter(std::shared_ptr results) { + std::map> clean_results; + for (auto& iter : results->count_nodes_map()) { + auto key = iter.first; + auto value = iter.second; + if (key == nullptr) { + continue; + } + std::string k = Name(key); + + std::set v; + for (auto& node : value) { + auto fg = node.first; + if (!Name(fg).empty()) { + v.insert(Name(fg)); + } + } + + if (!v.empty()) { + clean_results[k] = v; + } + } + + ASSERT_EQ(clean_results, expected_); + } + + void CheckGraphCounter(std::shared_ptr results) { + std::map> clean_results; + for (auto& iter : results->count_func_graphs_map()) { + auto key = iter.first; + auto value = iter.second; + if (key == nullptr) { + continue; + } + std::string k = Name(key); + + std::set v; + for (auto& node : value) { + auto fg = node.first; + if (!Name(fg).empty()) { + v.insert(Name(fg)); + } + } + + if (!v.empty()) { + clean_results[k] = v; + } + } + + ASSERT_EQ(clean_results, expected_); + } + + void CheckRecursive(std::shared_ptr results) { + std::map clean_results; + for (auto iter = results->recursive_analysis().begin(); iter != results->recursive_analysis().end(); ++iter) { + auto key = iter->first; + auto value = iter->second; + if (key == nullptr) { + continue; + } + std::string k = Name(key); + + clean_results[k] = value; + } + + ASSERT_EQ(clean_results, expected_recursive_); + } + + private: + Stage* stage_; + std::map> expected_; + std::map expected_recursive_; +}; + +bool CheckUsers(std::shared_ptr manager) { + for (auto node : manager->all_nodes()) { + if (node->isa()) { + auto& inputs = node->cast()->inputs(); + for (size_t i = 0; i < inputs.size(); ++i) { + auto inp = inputs[i]; + if (!manager->all_nodes().contains(inp)) { + return false; + } + + if (manager->node_users().find(inp) != manager->node_users().end()) { + auto users = manager->node_users()[inp]; + if (!users.contains(make_pair(node, i))) { + return false; + } + } + } + } + + if (manager->node_users().find(node) != manager->node_users().end()) { + auto users = manager->node_users()[node]; + for (auto iter = users.begin(); iter != users.end(); ++iter) { + auto node2 = iter->first; + auto key = iter->second; + if (!manager->all_nodes().contains(node2)) { + return false; + } + if (node2->cast()->input(key) != node) { + return false; + } + } + } + } + + return true; +} + +class TestManager : public UT::Common { + public: + TestManager() : getPyFun("gtest_input.ir.manager_test") {} + + void CheckAnalysisSize(std::shared_ptr mng); + + public: + std::vector swaps; + UT::PyFuncGraphFetcher getPyFun; +}; + +FuncGraphPtr MakeFuncGraph(PrimitivePtr prim) { + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr x = func_graph->add_parameter(); + ParameterPtr y = func_graph->add_parameter(); + std::vector inputs; + inputs.push_back(NewValueNode(prim)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + return func_graph; +} + +std::vector MakeNestedGraph() { + /* + *def f(x): + * def g(): + * return x + * return g + */ + FuncGraphPtr f = std::make_shared(); + FuncGraphPtr fg = std::make_shared(); + + ParameterPtr x = f->add_parameter(); + + std::vector inputs; + inputs.push_back(NewValueNode(fg)); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + + CNodePtr cnode_f = f->NewCNode(inputs); + f->set_return(cnode_f); + + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(x); + CNodePtr cnode_g = fg->NewCNode(inputs); + fg->set_return(cnode_g); + + std::vector result = {f, fg}; + return result; +} + +std::vector MakeNestedGraph2() { + /* build a closure func_graph */ + /* + *def foo(x, y): + * def bar(x1): + * return x1 + y + * return bar(x) + */ + FuncGraphPtr graph_foo = std::make_shared(); + ParameterPtr x = graph_foo->add_parameter(); + ParameterPtr y = graph_foo->add_parameter(); + + std::vector inputs; + + // build func_graph bar + FuncGraphPtr graph_bar = std::make_shared(); + ParameterPtr x1 = graph_bar->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimScalarAdd)); + inputs.push_back(x1); + inputs.push_back(y); + CNodePtr cnode_add = graph_bar->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = graph_bar->NewCNode(inputs); + graph_bar->set_return(cnode_return); + + // build func_graph foo + inputs.clear(); + inputs.push_back(NewValueNode(graph_bar)); + inputs.push_back(x); + CNodePtr cnode_graph_bar = graph_foo->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_graph_bar); + cnode_return = graph_foo->NewCNode(inputs); + graph_foo->set_return(cnode_return); + + std::vector result = {graph_foo, graph_bar}; + return result; +} + +// Add TestManager::CheckManager function to checkout the result + +void TestManager::CheckAnalysisSize(std::shared_ptr mng) { + auto size = mng->func_graphs().size(); + + ASSERT_EQ(size + 1, mng->nodes().size()); + ASSERT_EQ(size, mng->free_variables_total().size()); + ASSERT_EQ(size, mng->valuenodes().size()); + ASSERT_EQ(size, mng->free_variables_direct().size()); + ASSERT_EQ(size, mng->func_graph_valuenodes().size()); + ASSERT_EQ(size, mng->func_graph_parents_direct().size()); + ASSERT_EQ(size, mng->func_graph_users().size()); + ASSERT_EQ(size, mng->func_graphs_used().size()); +} + +TEST_F(TestManager, test_scalar_add_manual) { + auto prim_scalar_add = prim::kPrimScalarAdd; + FuncGraphPtr func_graph = MakeFuncGraph(prim_scalar_add); + auto mng = Manage(func_graph); +} + +TEST_F(TestManager, test_scalar_replace) { + auto prim_scalar_add = prim::kPrimScalarAdd; + + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr x = func_graph->add_parameter(); + ParameterPtr y = func_graph->add_parameter(); + std::vector inputs; + inputs.push_back(NewValueNode(prim_scalar_add)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + + auto mng = Manage(func_graph); + std::cout << "start " << x->ToString() << std::endl; + mng->Replace(cnode_add, x); +} + +TEST_F(TestManager, test_nested_manual) { + auto graphs = MakeNestedGraph(); + auto f = graphs[0]; + auto g = graphs[1]; + + auto mng = Manage(f); + + ASSERT_EQ(6, mng->all_nodes().size()); + ASSERT_EQ(2, mng->func_graphs().size()); + ASSERT_EQ(4, mng->node_users().size()); + ASSERT_EQ(1, mng->roots().size()); + CheckAnalysisSize(mng); + + auto nodes = mng->nodes(); + ASSERT_EQ(3, nodes[nullptr].size()); + ASSERT_EQ(2, nodes[f].size()); + ASSERT_EQ(1, nodes[g].size()); + + auto users = mng->node_users(); + for (auto& iter : users) { + ASSERT_EQ(1, iter.second.size()); + } + + auto graphs_used = mng->func_graphs_used(); + ASSERT_EQ(1, graphs_used[f].size()); + ASSERT_EQ(0, graphs_used[g].size()); + + auto graph_users = mng->func_graph_users(); + ASSERT_EQ(0, graph_users[f].size()); + ASSERT_EQ(1, graph_users[g].size()); + + auto fv_direct = mng->free_variables_direct(); + ASSERT_EQ(0, fv_direct[f].size()); + ASSERT_EQ(1, fv_direct[g].size()); + + auto fv_total = mng->free_variables_total(); + ASSERT_EQ(0, fv_total[f].size()); + ASSERT_EQ(1, fv_total[g].size()); + + auto graph_valuenodes = mng->func_graph_valuenodes(); + ASSERT_EQ(0, graph_valuenodes[f].size()); + ASSERT_EQ(1, graph_valuenodes[g].size()); +} + +TEST_F(TestManager, test_deep_nested2_manual) { + // create parser + FuncGraphPtr func_graph = getPyFun("test_custom"); + return; + + // parse ast to func graph + FuncGraphPtr gfn = BasicClone(func_graph); + if (gfn == nullptr) { + return; + } + + auto mng = Manage(gfn); + + ASSERT_EQ(3, mng->func_graphs().size()); + ASSERT_EQ(1, mng->roots().size()); + ASSERT_EQ(4, mng->nodes().size()); + ASSERT_EQ(20, mng->all_nodes().size()); + ASSERT_EQ(25, mng->node_users().size()); + CheckAnalysisSize(mng); +} + +TEST_F(TestManager, test_deep_nested_manual) { + FuncGraphPtr f = std::make_shared(); + FuncGraphPtr fg = std::make_shared(); + FuncGraphPtr h = std::make_shared(); + + ParameterPtr x = f->add_parameter(); + ParameterPtr y = f->add_parameter(); + ParameterPtr z = f->add_parameter(); + + std::vector inputs; + inputs.push_back(NewValueNode(fg)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_1 = f->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(cnode_1); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + CNodePtr cnode_0 = f->NewCNode(inputs); + f->set_return(cnode_0); + + ParameterPtr x1 = fg->add_parameter(); + ParameterPtr y1 = fg->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(h)); + inputs.push_back(x1); + CNodePtr cnode_3 = fg->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(cnode_3); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + CNodePtr cnode_2 = fg->NewCNode(inputs); + fg->set_return(cnode_2); + + ParameterPtr x2 = h->add_parameter(); + + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimScalarAdd)); + inputs.push_back(x2); + inputs.push_back(y1); + CNodePtr cnode_6 = h->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimScalarAdd)); + inputs.push_back(z); + inputs.push_back(cnode_6); + CNodePtr cnode_5 = h->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(cnode_5); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + CNodePtr cnode_4 = h->NewCNode(inputs); + h->set_return(cnode_4); + + auto mng = Manage(f); + + ASSERT_EQ(3, mng->func_graphs().size()); + ASSERT_EQ(1, mng->roots().size()); + ASSERT_EQ(4, mng->nodes().size()); + ASSERT_EQ(20, mng->all_nodes().size()); + CheckAnalysisSize(mng); +} + +TEST_F(TestManager, test_parent1_manual) { + FuncGraphPtr fg = std::make_shared(); + + Parameter param(fg); + std::vector params; + CNodePtr app = std::make_shared(params, fg); + fg->set_return(app); + fg->set_parameters(params); + + std::shared_ptr manager = MakeManager(); + manager->AddFuncGraph(fg, true); + FuncGraphPtr p = fg->parent(); + assert(p == nullptr); +} + +TEST_F(TestManager, test_parent_manual) { + auto prim_scalar_add = prim::kPrimScalarAdd; + FuncGraphPtr fg = MakeFuncGraph(prim_scalar_add); + + std::shared_ptr manager = MakeManager(); + manager->AddFuncGraph(fg); + FuncGraphPtr p = fg->parent(); + assert(p == nullptr); +} + +TEST_F(TestManager, test_flat) { + std::vector> stages; + std::vector specs = {"nodes=X:x", "parents=", "fvs_direct="}; + std::map size_list; + size_list["nodes"] = 2; +} + +TEST_F(TestManager, test_nested) { + std::vector> stages; + std::vector specs = {"nodes=X:x", "parent=g->X", "fvs_direct=g:x"}; + std::map size_list; + return; +} + +TEST_F(TestManager, test_calls) { + std::vector> stages; + std::vector specs = {"parents=g->X; h->X", "children=X:g,h", "scopes=X:X,g,h; g:g; h:h", + "fvs_direct=h:a", "fvs_total=h:a; g:h"}; + std::map size_list; + return; +} + +TEST_F(TestManager, test_unused_param) { + std::vector> stages; + std::vector specs = {"nodes=X:x,y"}; + std::map size_list; +} + +TEST_F(TestManager, test_cannot_replace_return) { + FuncGraphPtr fg = getPyFun("test_cannot_replace_return"); + ASSERT_NE(fg, nullptr); + + auto mng = Manage(fg); + ASSERT_EQ(fg->manager(), mng); + + ASSERT_NE(mng, nullptr); + ASSERT_GT(fg->parameters().size(), 0); + ASSERT_FALSE(mng->Replace(fg->get_return(), fg->parameters()[0])); +} + +TEST_F(TestManager, test_weak_manager) { + FuncGraphPtr fg = getPyFun("ir_get_fn"); + + auto mng1 = MakeManager({fg}, false); + ASSERT_EQ(fg->manager(), nullptr); + auto mng2 = MakeManager({fg}, true); + ASSERT_EQ(fg->manager(), mng2); + auto mng3 = MakeManager({fg}, false); + ASSERT_EQ(fg->manager(), mng2); +} + +TEST_F(TestManager, test_drop_root) { + FuncGraphPtr fg = getPyFun("ir_get_fn"); + + auto mng = Manage(fg); + const FuncGraphToAnfNodeMap& nodes = mng->nodes(); + ASSERT_TRUE(nodes.find(fg) != nodes.end()); + FuncGraphSet s; + s.add(fg); + mng->MaybeDropFuncGraphs(s); + ASSERT_TRUE(nodes.find(fg) != nodes.end()); +} + +TEST_F(TestManager, test_keep_roots) { + FuncGraphPtr fg1 = getPyFun("ir_get_fn"); + FuncGraphPtr fg2 = getPyFun("test_cannot_replace_return"); + + auto mng = Manage(fg1); + ASSERT_EQ(mng->func_graphs().size(), (size_t)1); + ASSERT_TRUE(mng->func_graphs().contains(fg1)); + + mng->AddFuncGraph(fg2); + ASSERT_EQ(mng->func_graphs().size(), 2); + ASSERT_TRUE(mng->func_graphs().contains(fg2)); + + mng->KeepRoots(); + ASSERT_EQ(mng->func_graphs().size(), 1); + ASSERT_TRUE(mng->func_graphs().contains(fg1)); + + mng->KeepRoots({fg2}); + ASSERT_EQ(mng->func_graphs().size(), 1); + ASSERT_TRUE(mng->func_graphs().contains(fg2)); +} + +TEST_F(TestManager, test_keep_roots_recursion) { + return; + + FuncGraphPtr fg = getPyFun("test_keep_roots_recursion"); + ASSERT_NE(fg, nullptr); + auto mng = Manage(fg); + parse::ResolveAll(mng); + + ASSERT_NE(mng, nullptr); + ASSERT_EQ(mng->func_graphs().size(), 4); + + ASSERT_GT(fg->parameters().size(), 0); + mng->Replace(fg->output(), fg->parameters()[0]); + ASSERT_EQ(mng->func_graphs().size(), 3); + + mng->KeepRoots(); + ASSERT_EQ(mng->func_graphs().size(), 1); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/ir/meta_tensor_test.cc b/tests/ut/cpp/ir/meta_tensor_test.cc new file mode 100644 index 00000000000..f149d5d1547 --- /dev/null +++ b/tests/ut/cpp/ir/meta_tensor_test.cc @@ -0,0 +1,343 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "securec/include/securec.h" +#include "ir/meta_tensor.h" + +namespace mindspore { +namespace tensor { + +class TestMetaTensor : public UT::Common { + public: + TestMetaTensor() {} + virtual void SetUp() { + std::vector dimensions({2, 3}); + meta_tensor_ = MetaTensor(TypeId::kNumberTypeFloat64, dimensions); + } + + protected: + MetaTensor meta_tensor_; +}; + +TEST_F(TestMetaTensor, InitTest) { + std::vector dimensions({2, 3}); + MetaTensor meta_tensor(TypeId::kNumberTypeFloat64, dimensions); + + // Test type + ASSERT_EQ(TypeId::kNumberTypeFloat64, meta_tensor.data_type()); + + // Test dimensions + ASSERT_EQ(2, meta_tensor.DimensionSize(0)); + ASSERT_EQ(3, meta_tensor.DimensionSize(1)); + ASSERT_EQ(-1, meta_tensor.DimensionSize(2)); + + // Test number of elements + ASSERT_EQ(6, meta_tensor.ElementsNum()); +} + +// Test type +TEST_F(TestMetaTensor, TypeTest) { + meta_tensor_.set_data_type(TypeId::kNumberTypeInt32); + ASSERT_EQ(TypeId::kNumberTypeInt32, meta_tensor_.data_type()); +} + +// Test shape +TEST_F(TestMetaTensor, ShapeTest) { + std::vector dimensions({5, 6, 7}); + meta_tensor_.set_shape(dimensions); + + ASSERT_EQ(5, meta_tensor_.DimensionSize(0)); + ASSERT_EQ(6, meta_tensor_.DimensionSize(1)); + ASSERT_EQ(7, meta_tensor_.DimensionSize(2)); + + // Test number of elements + ASSERT_EQ(210, meta_tensor_.ElementsNum()); +} + +TEST_F(TestMetaTensor, EqualTest) { + std::vector dimensions({2, 3}); + MetaTensor meta_tensor_x(TypeId::kNumberTypeFloat64, dimensions); + MetaTensor meta_tensor_y(meta_tensor_x); + + ASSERT_TRUE(meta_tensor_x == meta_tensor_y); + + MetaTensor meta_tensor_z(TypeId::kNumberTypeFloat32, dimensions); + ASSERT_FALSE(meta_tensor_x == meta_tensor_z); + + meta_tensor_z = meta_tensor_x; + ASSERT_TRUE(meta_tensor_x == meta_tensor_z); +} + +class TestTensor : public UT::Common { + public: + TestTensor() {} + virtual void SetUp() { + UT::InitPythonPath(); + // Init tensor data by py::array_t + input_ = py::array_t({2, 3}); + auto array = input_.mutable_unchecked(); + float start = 0; + for (int i = 0; i < array.shape(0); i++) { + for (int j = 0; j < array.shape(1); j++) { + array(i, j) = start++; + } + } + } + + protected: + py::array_t input_; +}; + +TEST_F(TestTensor, PyArrayScalarTest) { + std::vector dimensions; + py::array data = py::array_t(dimensions); + uint8_t *data_buf = reinterpret_cast(data.request(true).ptr); + + int64_t num = 1; + errno_t ret = memcpy_s(data_buf, sizeof(int64_t), &num, sizeof(int64_t)); + + ASSERT_EQ(0, ret); + + ASSERT_EQ(num, *data_buf); +} + +TEST_F(TestTensor, InitScalarTest) { + std::vector dimensions; + Tensor tensor(TypeId::kNumberTypeInt64, dimensions); + uint8_t *data_buf = reinterpret_cast(tensor.data_c(true)); + + int64_t num = 1; + errno_t ret = memcpy_s(data_buf, sizeof(int64_t), &num, sizeof(int64_t)); + + ASSERT_EQ(0, ret); + + ASSERT_EQ(num, *data_buf); + + // Test type + ASSERT_EQ(TypeId::kNumberTypeInt64, tensor.data_type()); + + // Test dimensions + ASSERT_EQ(0, tensor.DataDim()); + + // Test shape + ASSERT_EQ(0, tensor.shape().size()); + std::vector empty_shape; + ASSERT_EQ(empty_shape, tensor.shape()); + + // Test number of elements + ASSERT_EQ(1, tensor.ElementsNum()); + ASSERT_EQ(1, tensor.DataSize()); +} + +TEST_F(TestTensor, InitTensorPtrTest) { + std::vector dimensions; + Tensor tensor(TypeId::kNumberTypeInt64, dimensions); + + std::shared_ptr tensor_ptr = std::make_shared(tensor); + + // Test type + ASSERT_EQ(TypeId::kNumberTypeInt64, tensor_ptr->data_type()); + + // Test dimensions + ASSERT_EQ(0, tensor_ptr->DataDim()); + + // Test shape + ASSERT_EQ(0, tensor_ptr->shape().size()); + std::vector empty_shape; + ASSERT_EQ(empty_shape, tensor_ptr->shape()); + + // Test number of elements + ASSERT_EQ(1, tensor_ptr->ElementsNum()); + ASSERT_EQ(1, tensor_ptr->DataSize()); +} + +TEST_F(TestTensor, InitByTupleTest) { + py::tuple dimensions = py::make_tuple(2, 3, 4); + TypePtr data_type = kFloat32; + Tensor tuple_tensor = Tensor(data_type, dimensions); + ASSERT_EQ(2, tuple_tensor.DimensionSize(0)); + ASSERT_EQ(3, tuple_tensor.DimensionSize(1)); + ASSERT_EQ(4, tuple_tensor.DimensionSize(2)); + + // Test number of elements + ASSERT_EQ(24, tuple_tensor.ElementsNum()); + ASSERT_EQ(TypeId::kNumberTypeFloat32, tuple_tensor.data_type()); + + py::tuple tuple = py::make_tuple(1.0, 2.0, 3, 4, 5, 6); + TensorPtr tensor = std::make_shared(tuple, kFloat64); + py::array array = tensor->data(); + + std::cout << "Dim: " << array.ndim() << std::endl; + ASSERT_EQ(1, array.ndim()); + + std::cout << "Num of Elements: " << array.size() << std::endl; + ASSERT_EQ(6, array.size()); + + std::cout << "Elements: " << std::endl; + // Must be double, or the result is not right + double *tensor_data = reinterpret_cast(tensor->data_c()); + for (int i = 0; i < array.size(); i++) { + std::cout << tensor_data[i] << std::endl; + } +} + +TEST_F(TestTensor, EqualTest) { + py::tuple tuple = py::make_tuple(1, 2, 3, 4, 5, 6); + TensorPtr tensor_int8 = std::make_shared(tuple, kInt8); + ASSERT_TRUE(*tensor_int8 == *tensor_int8); + + ASSERT_EQ(TypeId::kNumberTypeInt8, tensor_int8->data_type_c()); + + TensorPtr tensor_int16 = std::make_shared(tuple, kInt16); + ASSERT_EQ(TypeId::kNumberTypeInt16, tensor_int16->data_type_c()); + + TensorPtr tensor_int32 = std::make_shared(tuple, kInt32); + ASSERT_EQ(TypeId::kNumberTypeInt32, tensor_int32->data_type_c()); + + TensorPtr tensor_float16 = std::make_shared(tuple, kFloat16); + ASSERT_EQ(TypeId::kNumberTypeFloat16, tensor_float16->data_type_c()); + + TensorPtr tensor_float32 = std::make_shared(tuple, kFloat32); + ASSERT_EQ(TypeId::kNumberTypeFloat32, tensor_float32->data_type_c()); + + TensorPtr tensor_float64 = std::make_shared(tuple, kFloat64); + ASSERT_EQ(TypeId::kNumberTypeFloat64, tensor_float64->data_type_c()); +} + +TEST_F(TestTensor, PyArrayTest) { + py::array_t input({2, 3}); + auto array = input.mutable_unchecked(); + float sum = 0; + std::cout << "sum" + << " = " << std::endl; + + float start = 0; + for (int i = 0; i < array.shape(0); i++) { + for (int j = 0; j < array.shape(1); j++) { + array(i, j) = start++; + sum += array(i, j); + std::cout << "sum + " + << "array[" << i << ", " << j << "]" + << " = " << sum << std::endl; + } + } + + ASSERT_EQ(15, sum); +} + +TEST_F(TestTensor, InitByFloatArrayDataCTest) { + // Init tensor data by py::array_t + TensorPtr tensor = std::make_shared(input_); + + // Print some information of the tensor + std::cout << "Datatype: " << tensor->data_type() << std::endl; + ASSERT_EQ(TypeId::kNumberTypeFloat32, tensor->data_type()); + + std::cout << "Dim: " << tensor->DataDim() << std::endl; + ASSERT_EQ(2, tensor->DataDim()); + + std::cout << "Num of Elements: " << tensor->ElementsNum() << std::endl; + ASSERT_EQ(6, tensor->ElementsNum()); + + // Print each elements + std::cout << "Elements: " << std::endl; + float *tensor_data = reinterpret_cast(tensor->data_c()); + for (int i = 0; i < tensor->ElementsNum(); i++) { + std::cout << tensor_data[i] << std::endl; + } +} + +TEST_F(TestTensor, InitByFloatArrayDataTest) { + // Init tensor data by py::array_t + TensorPtr tensor = std::make_shared(input_); + + // Print some information of the tensor + std::cout << "Datatype: " << tensor->data_type() << std::endl; + ASSERT_EQ(TypeId::kNumberTypeFloat32, tensor->data_type()); + + std::cout << "Dim: " << tensor->DataDim() << std::endl; + ASSERT_EQ(2, tensor->DataDim()); + + std::vector dimensions = tensor->shape(); + ASSERT_GT(dimensions.size(), 1); + std::cout << "Dim0: " << dimensions[0] << std::endl; + ASSERT_EQ(2, dimensions[0]); + + std::cout << "Dim1: " << dimensions[1] << std::endl; + ASSERT_EQ(3, dimensions[1]); + + std::cout << "Num of Elements: " << tensor->ElementsNum() << std::endl; + ASSERT_EQ(6, tensor->ElementsNum()); + + // Print each elements + std::cout << "Elements: " << std::endl; + py::array_t data = (py::array_t)tensor->data(); + auto array = data.unchecked<2>(); + for (int i = 0; i < array.shape(0); i++) { + for (int j = 0; j < array.shape(1); j++) { + std::cout << array(i, j) << std::endl; + } + } +} + +TEST_F(TestTensor, PyArrayDataTest) { + py::array_t input({2, 3}); + float *data = reinterpret_cast(input.request().ptr); + float ge_tensor_data[] = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6}; + errno_t ret = memcpy_s(data, input.nbytes(), ge_tensor_data, sizeof(ge_tensor_data)); + ASSERT_EQ(0, ret); + auto array = input.mutable_unchecked(); + for (int i = 0; i < array.shape(0); i++) { + for (int j = 0; j < array.shape(1); j++) { + ASSERT_EQ(array(i, j), ge_tensor_data[3 * i + j]); + } + } +} + +TEST_F(TestTensor, TensorDataTest) { + // Init a data buffer + float ge_tensor_data[] = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6}; + + // Create a Tensor with wanted data type and shape + Tensor tensor = Tensor(TypeId::kNumberTypeFloat32, std::vector({2, 3})); + + // Get the writable data pointer from the tensor + float *me_tensor_data = reinterpret_cast(tensor.data_c(true)); + + // Copy data from buffer to tensor's data + errno_t ret = memcpy_s(me_tensor_data, tensor.data().nbytes(), ge_tensor_data, sizeof(ge_tensor_data)); + ASSERT_EQ(0, ret); + + // Testify if the data has been copied to the tensor data + py::array_t data = (py::array_t)tensor.data(); + auto array = data.mutable_unchecked(); + for (int i = 0; i < array.shape(0); i++) { + for (int j = 0; j < array.shape(1); j++) { + std::cout << "array[" << i << ", " << j << "]" + << " = " << array(i, j) << std::endl; + ASSERT_EQ(array(i, j), ge_tensor_data[3 * i + j]); + } + } +} + +} // namespace tensor +} // namespace mindspore diff --git a/tests/ut/cpp/ir/value_test.cc b/tests/ut/cpp/ir/value_test.cc new file mode 100644 index 00000000000..a71ef7a57ff --- /dev/null +++ b/tests/ut/cpp/ir/value_test.cc @@ -0,0 +1,96 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +#include "common/common_test.h" +#include "ir/value.h" +#include "pipeline/static_analysis/abstract_value.h" +#include "utils/log_adapter.h" + +namespace mindspore { +using AbstractScalar = abstract::AbstractScalar; +using AbstractTuple = abstract::AbstractTuple; +using AbstractBasePtrList = abstract::AbstractBasePtrList; + +class TestValue : public UT::Common { + public: + TestValue() {} +}; + +TEST_F(TestValue, test_int32) { + auto i32a = std::make_shared(2); + ASSERT_TRUE(i32a != nullptr); +} + +TEST_F(TestValue, testToAbstract) { + ValuePtr boolv = std::make_shared(true); + AbstractBasePtr boola = std::make_shared(true); + AbstractBasePtr ret = boolv->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(boola)); + ASSERT_FALSE(*(ret) == *(std::make_shared(false))); + ASSERT_FALSE(*(ret) == *(std::make_shared(2))); + + ValuePtr i32v = std::make_shared(2); + AbstractBasePtr i32a = std::make_shared(2); + ret = i32v->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(i32a)); + + ValuePtr f32v = std::make_shared(1.0); + AbstractBasePtr f32a = std::make_shared(1.0f); + ret = f32v->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(f32a)); + + ValuePtr sv = std::make_shared("_"); + AbstractBasePtr sa = std::make_shared(std::string("_")); + ret = sv->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(sa)); + + ValuePtr vv = std::make_shared(); + AbstractBasePtr va = std::make_shared(); + ret = vv->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(va)); + + ValuePtr tv = std::make_shared(std::vector({boolv, i32v, f32v, sv, vv})); + AbstractBasePtr ta = std::make_shared(AbstractBasePtrList({boola, i32a, f32a, sa, va})); + ret = tv->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(ta)); + + ValuePtr rv = std::make_shared("net.weight"); + abstract::AbstractRefKeyPtr ra = std::make_shared(); + ra->set_value(rv); + ret = rv->ToAbstract(); + ASSERT_TRUE(ret); + ASSERT_EQ(*(ret), *(ra)); +} + +TEST_F(TestValue, GetValue) { + ValuePtr fv = MakeValue("test"); + const char* fv_c = GetValue(fv); + MS_LOG(INFO) << "" << fv_c; + MS_LOG(INFO) << "" << GetValue(fv); + ASSERT_TRUE(fv_c != nullptr); +} +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/configuration.h b/tests/ut/cpp/mindrecord/configuration.h new file mode 100644 index 00000000000..2c80f5b585e --- /dev/null +++ b/tests/ut/cpp/mindrecord/configuration.h @@ -0,0 +1,218 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TESTS_MINDRECORD_CONFIGURATION_H_ +#define TESTS_MINDRECORD_CONFIGURATION_H_ + +// schema define +const char kCvatSchema[] = + "{ " + "\"name\": { " + " \"type\": \"string\" " + "}, " + "\"anno_tool\": { " + " \"type\": \"string\" " + "}, " + "\"creation_time\": { " + " \"type\": \"string\" " + "}, " + "\"max_shape_id\": { " + " \"type\": \"int32\" " + "}, " + "\"max_entity_id\": { " + " \"type\": \"int32\" " + "}, " + "\"entity_instances\": { " + " \"type\": \"bytes\" " + " } " + "}"; + +const char kCvatSchemaDesc[] = "annotate CVAT"; + +// image schema +const char kImageSchema[] = + "{ " + "\"name\": { " + " \"type\": \"string\" " + "}, " + "\"type\": { " + " \"type\": \"string\" " + "}, " + "\"id\": { " + " \"type\": \"string\" " + "}, " + "\"size\": { " + " \"type\": \"int32\" " + "}, " + "\"width\": { " + " \"type\": \"int32\" " + "}, " + "\"height\": { " + " \"type\": \"int32\" " + "} " + "}"; + +const char kImageSchemaDesc[] = "image schema"; + +const std::array kStatistics = { + "{\"level\": [" + "{\"key\": \"2019-03-30\", \"count\": 24}, {\"key\": \"2019-03-31\", \"count\": 23}, " + "{\"key\": \"2019-04-01\", \"count\": 28}, {\"key\": \"2019-04-02\", \"count\": 22}, " + "{\"key\": \"2019-04-03\", \"count\": 24}, {\"key\": \"2019-04-04\", \"count\": 32}, " + "{\"key\": \"2019-04-05\", \"count\": 27}, {\"key\": \"2019-04-06\", \"count\": 24}, " + "{\"key\": \"2019-04-07\", \"count\": 22}, {\"key\": \"2019-04-08\", \"count\": 29}, " + "{\"key\": \"2019-04-09\", \"count\": 18}, {\"key\": \"2019-04-10\", \"count\": 24}, " + "{\"key\": \"2019-04-11\", \"count\": 22}, {\"key\": \"2019-04-12\", \"count\": 23}, " + "{\"key\": \"2019-04-13\", \"count\": 27}, {\"key\": \"2019-04-14\", \"count\": 28}, " + "{\"key\": \"2019-04-15\", \"count\": 21}, {\"key\": \"2019-04-16\", \"count\": 27}, " + "{\"key\": \"2019-04-17\", \"count\": 25}, {\"key\": \"2019-04-18\", \"count\": 23}, " + "{\"key\": \"2019-04-19\", \"count\": 31}, {\"key\": \"2019-04-20\", \"count\": 28}, " + "{\"key\": \"2019-04-21\", \"count\": 29}, {\"key\": \"2019-04-22\", \"count\": 30}, " + "{\"key\": \"2019-04-23\", \"count\": 21}, {\"key\": \"2019-04-24\", \"count\": 32}, " + "{\"key\": \"2019-04-25\", \"count\": 28}, {\"key\": \"2019-04-26\", \"count\": 21}, " + "{\"key\": \"2019-04-27\", \"count\": 32}, {\"key\": \"2019-04-28\", \"count\": 22}, " + "{\"key\": \"2019-04-29\", \"count\": 30}, {\"key\": \"2019-04-30\", \"count\": 23}, " + "{\"key\": \"2019-05-01\", \"count\": 36}, {\"key\": \"2019-05-02\", \"count\": 31}, " + "{\"key\": \"2019-05-03\", \"count\": 31}, {\"key\": \"2019-05-04\", \"count\": 18}, " + "{\"key\": \"2019-05-05\", \"count\": 28}, {\"key\": \"2019-05-06\", \"count\": 28}, " + "{\"key\": \"2019-05-07\", \"count\": 35}, {\"key\": \"2019-05-08\", \"count\": 20}, " + "{\"key\": \"2019-05-09\", \"count\": 25}, {\"key\": \"2019-05-10\", \"count\": 33}, " + "{\"key\": \"2019-05-11\", \"count\": 16}, {\"key\": \"2019-05-12\", \"count\": 21}, " + "{\"key\": \"2019-05-13\", \"count\": 18}, {\"key\": \"2019-05-14\", \"count\": 38}, " + "{\"key\": \"2019-05-15\", \"count\": 23}, {\"key\": \"2019-05-16\", \"count\": 31}" + "]}", + "{\"level\": [" + "{\"key\": \"2018-12-31\", \"count\": 192}, {\"key\": \"2019-01-07\", \"count\": 182}, " + "{\"key\": \"2019-01-14\", \"count\": 169}, {\"key\": \"2019-01-21\", \"count\": 179}, " + "{\"key\": \"2019-01-28\", \"count\": 192}, {\"key\": \"2019-02-04\", \"count\": 166}, " + "{\"key\": \"2019-02-11\", \"count\": 211}, {\"key\": \"2019-02-18\", \"count\": 180}, " + "{\"key\": \"2019-02-25\", \"count\": 203}, {\"key\": \"2019-03-04\", \"count\": 185}, " + "{\"key\": \"2019-03-11\", \"count\": 194}, {\"key\": \"2019-03-18\", \"count\": 170}, " + "{\"key\": \"2019-03-25\", \"count\": 162}, {\"key\": \"2019-04-01\", \"count\": 179}, " + "{\"key\": \"2019-04-08\", \"count\": 171}, {\"key\": \"2019-04-15\", \"count\": 184}, " + "{\"key\": \"2019-04-22\", \"count\": 186}, {\"key\": \"2019-04-29\", \"count\": 197}, " + "{\"key\": \"2019-05-06\", \"count\": 178}, {\"key\": \"2019-05-13\", \"count\": 110}" + "]}", + "{\"level\": [" + "{\"key\": \"2018-12\", \"count\": 811}, {\"key\": \"2019-01\", \"count\": 805}, " + "{\"key\": \"2019-02\", \"count\": 763}, {\"key\": \"2019-03\", \"count\": 793}, " + "{\"key\": \"2019-04\", \"count\": 773}, {\"key\": \"2019-05\", \"count\": 432}" + "]}", + "{\"level\": [" + "{" + "\"key\": \"polyline_example\", " + "\"count\": 4779, " + "\"level\": [" + "{\"key\": \"polyline_example\", " + "\"count\": 4779, " + "\"level\": []}" + "]" + "}, " + "{" + "\"key\": \"points_example\", " + "\"count\": 4755, " + "\"level\": [" + "{\"key\": \"points_example\", " + "\"count\": 4755, " + "\"level\": []}" + "]" + "}, " + "{" + "\"key\": \"polygon_example\", " + "\"count\": 4728, " + "\"level\": [" + "{\"key\": \"polygon_example\", " + "\"count\": 4728, " + "\"level\": []}" + "]" + "}, " + "{" + "\"key\": \"human_body\"," + "\"count\": 1617," + "\"level\": [{\"key\": \"人脸框\"," + "\"count\": 1617," + "\"level\": []}" + "]" + "}," + "{" + "\"key\": \"animal_body\"," + "\"count\": 1611, " + "\"level\":" + " [" + "{\"key\": \"动物框\"," + "\"count\": 1611, " + "\"level\": []}" + "]" + "}, " + "{" + "\"key\": \"plant_body\"," + "\"count\": 1539," + "\"level\": [" + "{\"key\": \"植物框\"," + "\"count\": 1539, " + "\"level\": []}" + "]" + "}," + "{" + "\"key\": \"106点人脸\"," + "\"count\": 1227," + "\"level\": [" + "{\"key\": \"特征点\", " + "\"count\": 1227," + "\"level\": []}," + "{\"key\": \"脸部评分\", " + "\"count\": 1227," + "\"level\": [" + "{\"key\": \"大小\", " + "\"count\": 325}," + "{\"key\": \"层次\", " + "\"count\": 325}," + "{\"key\": \"细节\", " + "\"count\": 309}," + "{\"key\": \"色彩\", " + "\"count\": 268}" + "]}" + "]}," + " {\"key\": \"5点人脸\", \"count\": 1207," + " \"level\": [" + "{\"key\": \"特征点\", \"count\": 1207," + " \"level\": []}," + " {\"key\": \"脸部评分\", \"count\": 1207," + " \"level\": [" + "{\"key\": \"大小\", \"count\": 320}," + " {\"key\": \"色彩\", \"count\": 298}," + " {\"key\": \"层次\", \"count\": 297}," + " {\"key\": \"细节\", \"count\": 292}]}]}," + " {\"key\": \"68点人脸\", \"count\": 1184," + " \"level\": [" + "{\"key\": \"特征点\", \"count\": 1184," + " \"level\": []}, " + "{\"key\": \"脸部评分\", \"count\": 1184," + " \"level\": [" + "{\"key\": \"大小\", \"count\": 304}," + " {\"key\": \"色彩\", \"count\": 303}," + " {\"key\": \"层次\", \"count\": 290}," + " {\"key\": \"细节\", \"count\": 287}]}]}," + " {\"key\": \"276点人脸\", \"count\": 1163," + " \"level\": [" + "{\"key\": \"特征点\", \"count\": 1163," + " \"level\": []}," + " {\"key\": \"脸部评分\", \"count\": 1163," + " \"level\": [{\"key\": \"细节\", \"count\": 310}," + " {\"key\": \"色彩\", \"count\": 298}," + " {\"key\": \"层次\", \"count\": 279}," + " {\"key\": \"大小\", \"count\": 276}]}]}]}"}; +#endif // TESTS_MINDRECORD_CONFIGURATION_H_ diff --git a/tests/ut/cpp/mindrecord/ut_common.cc b/tests/ut/cpp/mindrecord/ut_common.cc new file mode 100644 index 00000000000..76aa5fc5032 --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_common.cc @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ut_common.h" + +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +namespace mindspore { +namespace mindrecord { +namespace UT { +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif + +void Common::SetUp() {} + +void Common::TearDown() {} + +void Common::LoadData(const std::string &directory, std::vector &json_buffer, const int max_num) { + int count = 0; + string input_path = directory; + ifstream infile(input_path); + if (!infile.is_open()) { + MS_LOG(ERROR) << "can not open the file "; + return; + } + string temp; + while (getline(infile, temp) && count != max_num) { + count++; + json j = json::parse(temp); + json_buffer.push_back(j); + } + infile.close(); +} + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif +} // namespace UT + +const std::string FormatInfo(const std::string &message, uint32_t message_total_length) { + // if the message is larger than message_total_length + std::string part_message = ""; + if (message_total_length < message.length()) { + part_message = message.substr(0, message_total_length); + } else { + part_message = message; + } + int padding_length = static_cast(message_total_length - part_message.length()); + std::string left_padding(static_cast(ceil(padding_length / 2.0)), '='); + std::string right_padding(static_cast(floor(padding_length / 2.0)), '='); + return left_padding + part_message + right_padding; +} +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_common.h b/tests/ut/cpp/mindrecord/ut_common.h new file mode 100644 index 00000000000..398c59779bb --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_common.h @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TESTS_MINDRECORD_UT_UT_COMMON_H_ +#define TESTS_MINDRECORD_UT_UT_COMMON_H_ + +#include +#include +#include + +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_index.h" + +using json = nlohmann::json; +using std::ifstream; +using std::pair; +using std::string; +using std::vector; + +namespace mindspore { +namespace mindrecord { +namespace UT { +class Common : public testing::Test { + public: + std::string install_root; + + // every TEST_F macro will enter one + void SetUp(); + + void TearDown(); + + static void LoadData(const std::string &directory, std::vector &json_buffer, const int max_num); +}; +} // namespace UT + +/// \brief Format the INFO message to have the same length by inserting '=' at the start +/// and the end of the message. +/// \param[in] message the string to format +/// \param[in] message_total_length the string length of the output +/// +/// return the formatted string +const std::string FormatInfo(const std::string &message, uint32_t message_total_length = 128); +} // namespace mindrecord +} // namespace mindspore +#endif // TESTS_MINDRECORD_UT_UT_COMMON_H_ diff --git a/tests/ut/cpp/mindrecord/ut_shard.cc b/tests/ut/cpp/mindrecord/ut_shard.cc new file mode 100644 index 00000000000..88fdb7e167c --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard.cc @@ -0,0 +1,133 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "configuration.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_index.h" +#include "mindrecord/include/shard_header.h" +#include "mindrecord/include/shard_statistics.h" +#include "securec.h" +#include "ut_common.h" +#include "ut_shard_writer_test.h" + +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +namespace mindspore { +namespace mindrecord { +class TestShard : public UT::Common { + public: + TestShard() {} +}; + +TEST_F(TestShard, TestShardSchemaPart) { + TestShardWriterImageNet(); + + MS_LOG(INFO) << FormatInfo("Test schema"); + + // read schema json from schema dir + nlohmann::json j = nlohmann::json::parse(kCvatSchema); + std::string desc = kCvatSchemaDesc; + + std::shared_ptr schema = Schema::Build(desc, j); + ASSERT_TRUE(schema != nullptr); + MS_LOG(INFO) << "schema description: " << schema->get_desc() << ", schema: " << + common::SafeCStr(schema->GetSchema().dump()); +} + +TEST_F(TestShard, TestStatisticPart) { + MS_LOG(INFO) << FormatInfo("Test statistics"); + + // define statistics + MS_LOG(INFO) << "statistics: " << kStatistics[2]; + std::string desc = "statistic desc"; + nlohmann::json statistic_json = json::parse(kStatistics[2]); + std::shared_ptr statistics = Statistics::Build(desc, statistic_json); + ASSERT_TRUE(statistics != nullptr); + MS_LOG(INFO) << "test get_desc(), result: " << statistics->get_desc(); + MS_LOG(INFO) << "test get_statistics, result: " << statistics->get_statistics().dump(); + + std::string desc2 = "axis"; + nlohmann::json statistic_json2 = R"({})"; + std::shared_ptr statistics2 = Statistics::Build(desc2, statistic_json2); + ASSERT_TRUE(statistics2 == nullptr); +} + +TEST_F(TestShard, TestShardHeaderPart) { + MS_LOG(INFO) << FormatInfo("Test ShardHeader"); + json schema_json1 = R"({"name": {"type": "string"}, "type": {"type": "string"}})"_json; + + json statistic_json1 = json::parse( + "{\"statistics\": " + "{\"level\": [" + "{\"key\": \"2018-12\", \"count\": 811}, " + "{\"key\": \"2019-11\", \"count\": 763}" + "]}}"); + + std::string schema_desc1 = "test schema1"; + std::string statistics_desc1 = "test statistics1"; + + std::shared_ptr schema1 = Schema::Build(schema_desc1, schema_json1); + ASSERT_TRUE(schema1 != nullptr); + std::vector validate_schema; + validate_schema.push_back(*schema1); + + std::shared_ptr statistics1 = Statistics::Build(statistics_desc1, statistic_json1["statistics"]); + ASSERT_TRUE(statistics1 != nullptr); + std::vector validate_statistics; + validate_statistics.push_back(*statistics1); + + // init shardHeader + mindrecord::ShardHeader header_data; + + int res = header_data.AddSchema(schema1); + ASSERT_EQ(res, 0); + header_data.AddStatistic(statistics1); + std::vector re_schemas; + for (auto &schema_ptr : header_data.get_schemas()) { + re_schemas.push_back(*schema_ptr); + } + ASSERT_EQ(re_schemas, validate_schema); + + std::vector re_statistics; + for (auto &statistic : header_data.get_statistics()) { + re_statistics.push_back(*statistic); + } + ASSERT_EQ(re_statistics, validate_statistics); + ASSERT_EQ(header_data.GetStatisticByID(-1).second, FAILED); + ASSERT_EQ(header_data.GetStatisticByID(10).second, FAILED); + + // test add index fields + std::vector> fields; + std::pair pair1(0, "name"); + fields.push_back(pair1); + ASSERT_TRUE(header_data.AddIndexFields(fields) == SUCCESS); + std::vector> resFields = header_data.get_fields(); + ASSERT_EQ(resFields, fields); +} + +TEST_F(TestShard, TestShardWriteImage) { MS_LOG(INFO) << FormatInfo("Test writer"); } +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_header_test.cc b/tests/ut/cpp/mindrecord/ut_shard_header_test.cc new file mode 100644 index 00000000000..ce5f40c10c5 --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_header_test.cc @@ -0,0 +1,155 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/shard_writer.h" +#include "mindrecord/include/shard_index.h" +#include "mindrecord/include/shard_header.h" +#include "mindrecord/include/shard_schema.h" +#include "mindrecord/include/shard_statistics.h" +#include "securec.h" +#include "ut_common.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +class TestShardHeader : public UT::Common { + public: + TestShardHeader() {} +}; + +TEST_F(TestShardHeader, AddIndexFields) { + MS_LOG(INFO) << FormatInfo("Test ShardHeader: add index fields"); + + std::string desc1 = "this is a test1"; + json schema_content1 = R"({"name": {"type": "string"}, + "box": {"type": "string"}, + "label": {"type": "int32", "shape": [-1]}})"_json; + + std::string desc2 = "this is a test2"; + json schema_content2 = R"({"names": {"type": "string"}, + "labels": {"type": "array", "items": {"type": "number"}}})"_json; + std::shared_ptr schema1 = Schema::Build(desc1, schema_content1); + ASSERT_NE(schema1, nullptr); + std::shared_ptr schema2 = Schema::Build(desc2, schema_content2); + ASSERT_EQ(schema2, nullptr); + + mindrecord::ShardHeader header_data; + int schema_id1 = header_data.AddSchema(schema1); + int schema_id2 = header_data.AddSchema(schema2); + ASSERT_EQ(schema_id2, -1); + ASSERT_EQ(header_data.get_schemas().size(), 1); + + // check out fields + std::vector> fields; + + std::pair index_field1(schema_id1, "name"); + std::pair index_field2(schema_id1, "box"); + fields.push_back(index_field1); + fields.push_back(index_field2); + MSRStatus res = header_data.AddIndexFields(fields); + ASSERT_EQ(res, SUCCESS); + ASSERT_EQ(header_data.get_fields().size(), 2); + + fields.clear(); + std::pair index_field3(schema_id1, "name"); + fields.push_back(index_field3); + res = header_data.AddIndexFields(fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data.get_fields().size(), 2); + + fields.clear(); + std::pair index_field4(schema_id1, "names"); + fields.push_back(index_field4); + res = header_data.AddIndexFields(fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data.get_fields().size(), 2); + + fields.clear(); + std::pair index_field5(schema_id1 + 1, "name"); + fields.push_back(index_field5); + res = header_data.AddIndexFields(fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data.get_fields().size(), 2); + + fields.clear(); + std::pair index_field6(schema_id1, "label"); + fields.push_back(index_field6); + res = header_data.AddIndexFields(fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data.get_fields().size(), 2); + + std::string desc_new = "this is a test1"; + json schemaContent_new = R"({"name": {"type": "string"}, + "box": {"type": "string"}, + "label": {"type": "int32", "shape": [-1]}})"_json; + + std::shared_ptr schema_new = Schema::Build(desc_new, schemaContent_new); + ASSERT_NE(schema_new, nullptr); + + mindrecord::ShardHeader header_data_new; + header_data_new.AddSchema(schema_new); + ASSERT_EQ(header_data_new.get_schemas().size(), 1); + + // test add fields + std::vector single_fields; + + single_fields.push_back("name"); + single_fields.push_back("name"); + single_fields.push_back("box"); + res = header_data_new.AddIndexFields(single_fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data_new.get_fields().size(), 1); + + single_fields.push_back("name"); + single_fields.push_back("box"); + res = header_data_new.AddIndexFields(single_fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data_new.get_fields().size(), 1); + + single_fields.clear(); + single_fields.push_back("names"); + res = header_data_new.AddIndexFields(single_fields); + ASSERT_EQ(res, FAILED); + ASSERT_EQ(header_data_new.get_fields().size(), 1); + + single_fields.clear(); + single_fields.push_back("box"); + res = header_data_new.AddIndexFields(single_fields); + ASSERT_EQ(res, SUCCESS); + ASSERT_EQ(header_data_new.get_fields().size(), 2); +} +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_index_generator_test.cc b/tests/ut/cpp/mindrecord/ut_shard_index_generator_test.cc new file mode 100644 index 00000000000..a5e343a5b3c --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_index_generator_test.cc @@ -0,0 +1,114 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_error.h" +#include "mindrecord/include/shard_index_generator.h" +#include "mindrecord/include/shard_index.h" +#include "mindrecord/include/shard_statistics.h" +#include "securec.h" +#include "ut_common.h" + +using json = nlohmann::json; +using std::ifstream; +using std::pair; +using std::string; +using std::vector; + +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +namespace mindspore { +namespace mindrecord { +class TestShardIndexGenerator : public UT::Common { + public: + TestShardIndexGenerator() {} +}; + +TEST_F(TestShardIndexGenerator, GetField) { + MS_LOG(INFO) << FormatInfo("Test ShardIndex: get field"); + + int max_num = 1; + string input_path1 = install_root + "/test/testCBGData/data/annotation.data"; + std::vector json_buffer1; // store the image_raw_meta.data + Common::LoadData(input_path1, json_buffer1, max_num); + + MS_LOG(INFO) << "Fetch fields: "; + for (auto &j : json_buffer1) { + auto v_name = ShardIndexGenerator::GetField("anno_tool", j); + auto v_attr_name = ShardIndexGenerator::GetField("entity_instances.attributes.attr_name", j); + auto v_entity_name = ShardIndexGenerator::GetField("entity_instances.entity_name", j); + vector names = {"\"CVAT\""}; + for (unsigned int i = 0; i != names.size(); i++) { + ASSERT_EQ(names[i], v_name[i]); + } + vector attr_names = {"\"脸部评分\"", "\"特征点\"", "\"points_example\"", "\"polyline_example\"", + "\"polyline_example\""}; + for (unsigned int i = 0; i != attr_names.size(); i++) { + ASSERT_EQ(attr_names[i], v_attr_name[i]); + } + vector entity_names = {"\"276点人脸\"", "\"points_example\"", "\"polyline_example\"", + "\"polyline_example\""}; + for (unsigned int i = 0; i != entity_names.size(); i++) { + ASSERT_EQ(entity_names[i], v_entity_name[i]); + } + } +} +TEST_F(TestShardIndexGenerator, TakeFieldType) { + MS_LOG(INFO) << FormatInfo("Test ShardSchema: take field Type"); + + json schema1 = R"({ + "type": "object", + "properties": { + "number": { "type": "number" }, + "street_name": { "type": "string" }, + "street_type": { "type": "array", + "items": { "type": "array", + "items":{ "type": "number"} + } + } + }})"_json; + json schema2 = R"({"name": {"type": "string"}, + "label": {"type": "array", "items": {"type": "number"}}})"_json; + auto type1 = ShardIndexGenerator::TakeFieldType("number", schema1); + ASSERT_EQ("number", type1); + auto type2 = ShardIndexGenerator::TakeFieldType("street_name", schema1); + ASSERT_EQ("string", type2); + auto type3 = ShardIndexGenerator::TakeFieldType("street_type", schema1); + ASSERT_EQ("array", type3); + + auto type4 = ShardIndexGenerator::TakeFieldType("name", schema2); + ASSERT_EQ("string", type4); + auto type5 = ShardIndexGenerator::TakeFieldType("label", schema2); + ASSERT_EQ("array", type5); +} +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_operator_test.cc b/tests/ut/cpp/mindrecord/ut_shard_operator_test.cc new file mode 100644 index 00000000000..acea1cefae5 --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_operator_test.cc @@ -0,0 +1,450 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_category.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/shard_sample.h" +#include "mindrecord/include/shard_shuffle.h" +#include "ut_common.h" + +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +namespace mindspore { +namespace mindrecord { +class TestShardOperator : public UT::Common { + public: + TestShardOperator() {} +}; + +TEST_F(TestShardOperator, TestShardSampleBasic) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + const int kSampleCount = 8; + std::vector> ops; + ops.push_back(std::make_shared(kSampleCount)); + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]); + } + dataset.Finish(); + ASSERT_TRUE(i <= kSampleCount); +} + +TEST_F(TestShardOperator, TestShardSampleWrongNumber) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + const int kNum = 5; + const int kDen = 0; + std::vector> ops; + ops.push_back(std::make_shared(kNum, kDen)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]); + } + dataset.Finish(); + ASSERT_TRUE(i <= 5); +} + +TEST_F(TestShardOperator, TestShardSampleRatio) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + const int kNum = 1; + const int kDen = 4; + std::vector> ops; + ops.push_back(std::make_shared(kNum, kDen)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]); + } + dataset.Finish(); + ASSERT_TRUE(i <= 10); +} + + +TEST_F(TestShardOperator, TestShardSamplePartition) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + const int kNum = 1; + const int kDen = 4; + const int kPar = 2; + std::vector> ops; + ops.push_back(std::make_shared(kNum, kDen, kPar)); + auto partitions = std::dynamic_pointer_cast(ops[0])->get_partitions(); + ASSERT_TRUE(partitions.first == 4); + ASSERT_TRUE(partitions.second == 2); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]); + } + dataset.Finish(); + ASSERT_TRUE(i <= 10); +} + +TEST_F(TestShardOperator, TestShardCategory) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + std::vector> categories; + categories.emplace_back("label", "257"); + categories.emplace_back("label", "302"); + categories.emplace_back("label", "132"); + ops.push_back(std::make_shared(categories)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + int category_no = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + + ASSERT_TRUE((std::get<1>(x[0]))["label"] == categories[category_no].second); + + category_no++; + category_no %= static_cast(categories.size()); + } + dataset.Finish(); +} + +TEST_F(TestShardOperator, TestShardShuffle) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + ops.push_back(std::make_shared(1)); + + ShardReader dataset; + dataset.Open(file_name, 16, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + } + dataset.Finish(); +} + +TEST_F(TestShardOperator, TestShardSampleShuffle) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + ops.push_back(std::make_shared(35)); + ops.push_back(std::make_shared(1)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + } + dataset.Finish(); + ASSERT_LE(i, 35); +} + +TEST_F(TestShardOperator, TestShardShuffleSample) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + ops.push_back(std::make_shared(1)); + const int kSampleSize = 1000; + ops.push_back(std::make_shared(kSampleSize)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + } + dataset.Finish(); + ASSERT_TRUE(i <= kSampleSize); +} + +TEST_F(TestShardOperator, TestShardSampleShuffleSample) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + ops.push_back(std::make_shared(100)); + ops.push_back(std::make_shared(10)); + ops.push_back(std::make_shared(35)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + } + dataset.Finish(); + ASSERT_LE(i, 35); +} + +TEST_F(TestShardOperator, TestShardShuffleCompare) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + ops.push_back(std::make_shared(1)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + ShardReader compare_dataset; + compare_dataset.Open(file_name, 4, column_list); + compare_dataset.Launch(); + + int i = 0; + bool different = false; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + + auto y = compare_dataset.GetNext(); + if ((std::get<1>(x[0]))["file_name"] != (std::get<1>(y[0]))["file_name"]) different = true; + } + dataset.Finish(); + compare_dataset.Finish(); + ASSERT_TRUE(different); +} + +TEST_F(TestShardOperator, TestShardCategoryShuffle1) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + std::vector> categories; + categories.emplace_back("label", "257"); + categories.emplace_back("label", "302"); + categories.emplace_back("label", "490"); + ops.push_back(std::make_shared(categories)); + ops.push_back(std::make_shared(21)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + int category_no = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + + ASSERT_TRUE((std::get<1>(x[0]))["label"] == categories[category_no].second); + category_no++; + category_no %= static_cast(categories.size()); + } + dataset.Finish(); +} + +TEST_F(TestShardOperator, TestShardCategoryShuffle2) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + std::vector> categories; + categories.emplace_back("label", "257"); + categories.emplace_back("label", "302"); + categories.emplace_back("label", "132"); + ops.push_back(std::make_shared(32)); + ops.push_back(std::make_shared(categories)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + int category_no = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + ASSERT_TRUE((std::get<1>(x[0]))["label"] == categories[category_no].second); + category_no++; + category_no %= static_cast(categories.size()); + } + dataset.Finish(); +} + +TEST_F(TestShardOperator, TestShardCategorySample) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + std::vector> categories; + categories.emplace_back("label", "257"); + categories.emplace_back("label", "302"); + categories.emplace_back("label", "132"); + const int kSampleSize = 17; + ops.push_back(std::make_shared(kSampleSize)); + ops.push_back(std::make_shared(categories)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + int category_no = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + + ASSERT_TRUE((std::get<1>(x[0]))["label"] == categories[category_no].second); + category_no++; + category_no %= static_cast(categories.size()); + } + dataset.Finish(); + ASSERT_EQ(category_no, 0); + ASSERT_TRUE(i <= kSampleSize); +} + +TEST_F(TestShardOperator, TestShardCategorySampleShuffle) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name", "label"}; + + std::vector> ops; + std::vector> categories; + categories.emplace_back("label", "257"); + categories.emplace_back("label", "302"); + categories.emplace_back("label", "132"); + const int kSampleSize = 17; + ops.push_back(std::make_shared(kSampleSize)); + ops.push_back(std::make_shared(categories)); + ops.push_back(std::make_shared(100)); + + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + int i = 0; + int category_no = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + MS_LOG(INFO) << "index: " << i++ << ", filename: " << common::SafeCStr((std::get<1>(x[0]))["file_name"]) << + ", label: " << common::SafeCStr((std::get<1>(x[0]))["label"].dump()); + + ASSERT_TRUE((std::get<1>(x[0]))["label"] == categories[category_no].second); + category_no++; + category_no %= static_cast(categories.size()); + } + dataset.Finish(); + ASSERT_EQ(category_no, 0); + ASSERT_TRUE(i <= kSampleSize); +} +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_page_test.cc b/tests/ut/cpp/mindrecord/ut_shard_page_test.cc new file mode 100644 index 00000000000..f06b987a89a --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_page_test.cc @@ -0,0 +1,161 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_page.h" +#include "ut_common.h" + +using json = nlohmann::json; +using std::ifstream; +using std::pair; +using std::string; +using std::vector; + +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +namespace mindspore { +namespace mindrecord { +class TestShardPage : public UT::Common { + public: + TestShardPage() {} +}; + +TEST_F(TestShardPage, TestBasic) { + MS_LOG(INFO) << FormatInfo("Test ShardPage Basic"); + const int kGoldenPageId = 12; + const int kGoldenShardId = 20; + + const std::string kGoldenType = kPageTypeRaw; + const int kGoldenTypeId = 2; + const uint64_t kGoldenStart = 10; + const uint64_t kGoldenEnd = 20; + + std::vector> golden_row_group = {{1, 2}, {2, 4}, {4, 6}}; + const uint64_t kGoldenSize = 100; + const uint64_t kOffset = 6; + + Page page = + Page(kGoldenPageId, kGoldenShardId, kGoldenType, kGoldenTypeId, kGoldenStart, kGoldenEnd, golden_row_group, kGoldenSize); + EXPECT_EQ(kGoldenPageId, page.get_page_id()); + EXPECT_EQ(kGoldenShardId, page.get_shard_id()); + EXPECT_EQ(kGoldenTypeId, page.get_page_type_id()); + ASSERT_TRUE(kGoldenType == page.get_page_type()); + EXPECT_EQ(kGoldenSize, page.get_page_size()); + EXPECT_EQ(kGoldenStart, page.get_start_row_id()); + EXPECT_EQ(kGoldenEnd, page.get_end_row_id()); + ASSERT_TRUE(std::make_pair(4, kOffset) == page.get_last_row_group_id()); + ASSERT_TRUE(golden_row_group == page.get_row_group_ids()); +} + +TEST_F(TestShardPage, TestSetter) { + MS_LOG(INFO) << FormatInfo("Test ShardPage Setter Functions"); + const int kGoldenPageId = 12; + const int kGoldenShardId = 20; + + const std::string kGoldenType = kPageTypeBlob; + const int kGoldenTypeId = 2; + const uint64_t kGoldenStart = 10; + const uint64_t kGoldenEnd = 20; + + std::vector> golden_row_group = {{1, 2}, {2, 4}, {4, 6}}; + const uint64_t kGoldenSize = 100; + const uint64_t kOffset1 = 6; + const uint64_t kOffset2 = 3000; + const uint64_t kOffset3 = 200; + + Page page = + Page(kGoldenPageId, kGoldenShardId, kGoldenType, kGoldenTypeId, kGoldenStart, kGoldenEnd, golden_row_group, kGoldenSize); + EXPECT_EQ(kGoldenPageId, page.get_page_id()); + EXPECT_EQ(kGoldenShardId, page.get_shard_id()); + EXPECT_EQ(kGoldenTypeId, page.get_page_type_id()); + ASSERT_TRUE(kGoldenType == page.get_page_type()); + EXPECT_EQ(kGoldenSize, page.get_page_size()); + EXPECT_EQ(kGoldenStart, page.get_start_row_id()); + EXPECT_EQ(kGoldenEnd, page.get_end_row_id()); + ASSERT_TRUE(std::make_pair(4, kOffset1) == page.get_last_row_group_id()); + ASSERT_TRUE(golden_row_group == page.get_row_group_ids()); + + const int kNewEnd = 33; + const int kNewSize = 300; + std::vector> new_row_group = {{0, 100}, {100, 200}, {200, 3000}}; + page.set_end_row_id(kNewEnd); + page.set_page_size(kNewSize); + page.set_row_group_ids(new_row_group); + EXPECT_EQ(kGoldenPageId, page.get_page_id()); + EXPECT_EQ(kGoldenShardId, page.get_shard_id()); + EXPECT_EQ(kGoldenTypeId, page.get_page_type_id()); + ASSERT_TRUE(kGoldenType == page.get_page_type()); + EXPECT_EQ(kNewSize, page.get_page_size()); + EXPECT_EQ(kGoldenStart, page.get_start_row_id()); + EXPECT_EQ(kNewEnd, page.get_end_row_id()); + ASSERT_TRUE(std::make_pair(200, kOffset2) == page.get_last_row_group_id()); + ASSERT_TRUE(new_row_group == page.get_row_group_ids()); + page.DeleteLastGroupId(); + + EXPECT_EQ(kGoldenPageId, page.get_page_id()); + EXPECT_EQ(kGoldenShardId, page.get_shard_id()); + EXPECT_EQ(kGoldenTypeId, page.get_page_type_id()); + ASSERT_TRUE(kGoldenType == page.get_page_type()); + EXPECT_EQ(3000, page.get_page_size()); + EXPECT_EQ(kGoldenStart, page.get_start_row_id()); + EXPECT_EQ(kNewEnd, page.get_end_row_id()); + ASSERT_TRUE(std::make_pair(100, kOffset3) == page.get_last_row_group_id()); + new_row_group.pop_back(); + ASSERT_TRUE(new_row_group == page.get_row_group_ids()); +} + +TEST_F(TestShardPage, TestJson) { + MS_LOG(INFO) << FormatInfo("Test ShardPage json"); + const int kGoldenPageId = 12; + const int kGoldenShardId = 20; + + const std::string kGoldenType = kPageTypeRaw; + const int kGoldenTypeId = 2; + const uint64_t kGoldenStart = 10; + const uint64_t kGoldenEnd = 20; + + std::vector> golden_row_group = {{1, 2}, {2, 4}, {4, 6}}; + const uint64_t kGoldenSize = 100; + + Page page = + Page(kGoldenPageId, kGoldenShardId, kGoldenType, kGoldenTypeId, kGoldenStart, kGoldenEnd, golden_row_group, kGoldenSize); + + json json_page = page.GetPage(); + EXPECT_EQ(kGoldenPageId, json_page["page_id"]); + EXPECT_EQ(kGoldenShardId, json_page["shard_id"]); + EXPECT_EQ(kGoldenTypeId, json_page["page_type_id"]); + ASSERT_TRUE(kGoldenType == json_page["page_type"]); + EXPECT_EQ(kGoldenSize, json_page["page_size"]); + EXPECT_EQ(kGoldenStart, json_page["start_row_id"]); + EXPECT_EQ(kGoldenEnd, json_page["end_row_id"]); + json row_group = json_page["row_group_ids"]; + int i = 0; + ASSERT_TRUE(golden_row_group.size() == row_group.size()); + for (json &row : row_group) { + ASSERT_TRUE(golden_row_group[i] == std::make_pair(row["id"], row["offset"])); + ++i; + } +} +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc b/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc new file mode 100644 index 00000000000..fd63373e20e --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_reader_test.cc @@ -0,0 +1,213 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/shard_sample.h" +#include "ut_common.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +class TestShardReader : public UT::Common { + public: + TestShardReader() {} +}; + +TEST_F(TestShardReader, TestShardReaderGeneral) { + MS_LOG(INFO) << FormatInfo("Test read imageNet"); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + ShardReader dataset; + dataset.Open(file_name, 4, column_list); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto& j : x) { + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << item.key() << ", value: " << item.value().dump(); + } + } + } + dataset.Finish(); +} + +TEST_F(TestShardReader, TestShardReaderSample) { + MS_LOG(INFO) << FormatInfo("Test read imageNet"); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + std::vector> ops; + ops.push_back(std::make_shared(17)); + ShardReader dataset; + dataset.Open(file_name, 4, column_list, ops); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto& j : x) { + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << item.key() << ", value: " << item.value().dump(); + } + } + } + dataset.Finish(); + dataset.Close(); +} + +TEST_F(TestShardReader, TestShardReaderBlock) { + MS_LOG(INFO) << FormatInfo("Test read imageNet with block way"); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"label"}; + + std::vector> ops; + ops.push_back(std::make_shared(3)); + ShardReader dataset; + const bool kBlockReader = true; + dataset.Open(file_name, 4, column_list, ops, kBlockReader); + dataset.Launch(); + + while (true) { + auto x = dataset.GetBlockNext(); + if (x.empty()) break; + for (auto& j : x) { + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << item.key() << ", value: " << item.value().dump(); + } + } + } + dataset.Finish(); + dataset.Close(); +} + +TEST_F(TestShardReader, TestShardReaderEasy) { + MS_LOG(INFO) << FormatInfo("Test read imageNet"); + std::string file_name = "./imagenet.shard01"; + ShardReader dataset; + dataset.Open(file_name); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto& j : x) { + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << item.key() << ", value: " << item.value().dump(); + } + } + } + dataset.Finish(); +} + +TEST_F(TestShardReader, TestShardReaderColumnNotInIndex) { + MS_LOG(INFO) << FormatInfo("Test read imageNet"); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"label"}; + ShardReader dataset; + MSRStatus ret = dataset.Open(file_name, 4, column_list); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto& j : x) { + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << item.key() << ", value: " << item.value().dump(); + } + } + } + dataset.Finish(); +} + +TEST_F(TestShardReader, TestShardReaderColumnNotInSchema) { + MS_LOG(INFO) << FormatInfo("Test read imageNet"); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_namex"}; + ShardReader dataset; + MSRStatus ret = dataset.Open(file_name, 4, column_list); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestShardReader, TestShardVersion) { + MS_LOG(INFO) << FormatInfo("Test shard version"); + std::string file_name = "./imagenet.shard01"; + ShardReader dataset; + MSRStatus ret = dataset.Open(file_name, 4); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto& j : x) { + MS_LOG(INFO) << "result size: " << std::get<0>(j).size(); + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << common::SafeCStr(item.key()) << ", value: " << common::SafeCStr(item.value().dump()); + } + } + } + dataset.Finish(); +} + +TEST_F(TestShardReader, TestShardReaderDir) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + std::string file_name = "./"; + auto column_list = std::vector{"file_name"}; + + ShardReader dataset; + MSRStatus ret = dataset.Open(file_name, 4, column_list); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestShardReader, TestShardReaderConsumer) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet")); + std::string file_name = "./imagenet.shard01"; + auto column_list = std::vector{"file_name"}; + + ShardReader dataset; + dataset.Open(file_name, -481565535, column_list); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto& j : x) { + for (auto& item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << common::SafeCStr(item.key()) << ", value: " << common::SafeCStr(item.value().dump()); + } + } + } + dataset.Finish(); +} +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_schema_test.cc b/tests/ut/cpp/mindrecord/ut_shard_schema_test.cc new file mode 100644 index 00000000000..cce000fd28e --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_schema_test.cc @@ -0,0 +1,153 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_page.h" +#include "mindrecord/include/shard_schema.h" +#include "mindrecord/include/shard_statistics.h" +#include "securec.h" +#include "ut_common.h" + +using json = nlohmann::json; +using std::ifstream; +using std::pair; +using std::string; +using std::vector; + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +class TestShardSchema : public UT::Common { + public: + TestShardSchema() {} +}; + +class TestStatistics : public UT::Common { + public: + TestStatistics() {} +}; + +TEST_F(TestShardSchema, BuildSchema) { + MS_LOG(INFO) << FormatInfo("Test ShardSchema: build schema"); + + std::string desc = "this is a test"; + json schema_content = R"({"name": {"type": "string"}, + "label": {"type": "int32", "shape": [-1]}})"_json; + + std::shared_ptr schema = Schema::Build(desc, schema_content); + ASSERT_NE(schema, nullptr); + // checkout field name + + schema_content["name%"] = R"({"type": "string"})"_json; + schema = Schema::Build(desc, schema_content); + ASSERT_EQ(schema, nullptr); + schema_content.erase("name%"); + + schema_content["na-me"] = R"({"type": "string"})"_json; + schema = Schema::Build(desc, schema_content); + ASSERT_EQ(schema, nullptr); + schema_content.erase("na-me"); + + schema_content["name_type.2"] = R"({"type": "string"})"_json; + schema = Schema::Build(desc, schema_content); + ASSERT_EQ(schema, nullptr); + schema_content.erase("name_type.2"); + + schema_content["3_name"] = R"({"type": "string"})"_json; + schema = Schema::Build(desc, schema_content); + ASSERT_NE(schema, nullptr); + schema_content.erase("3_name"); + + schema_content["test"] = R"({"type": "test"})"_json; + schema = Schema::Build(desc, schema_content); + ASSERT_EQ(schema, nullptr); + schema_content.erase("test"); + + schema_content["test"] = R"({"type": "string", "test": "this is for test"})"_json; + schema = Schema::Build(desc, schema_content); + ASSERT_EQ(schema, nullptr); + schema_content.erase("test"); +} + +TEST_F(TestShardSchema, TestFunction) { + std::string desc = "this is a test"; + json schema_content = R"({"name": {"type": "string"}, + "label": {"type": "int32", "shape": [-1]}})"_json; + + std::shared_ptr schema = Schema::Build(desc, schema_content); + ASSERT_NE(schema, nullptr); + + ASSERT_EQ(schema->get_desc(), desc); + + json schema_json = schema->GetSchema(); + ASSERT_EQ(schema_json["desc"], desc); + ASSERT_EQ(schema_json["schema"], schema_content); + + ASSERT_EQ(schema->get_schema_id(), -1); + schema->set_schema_id(2); + ASSERT_EQ(schema->get_schema_id(), 2); +} + +TEST_F(TestStatistics, StatisticPart) { + MS_LOG(INFO) << FormatInfo("Test statistics"); + + std::string statistic = + "{\"level\": [" + "{\"key\": \"2018-12\", \"count\": 811}, {\"key\": \"2019-01\", \"count\": 805}, " + "{\"key\": \"2019-02\", \"count\": 763}, {\"key\": \"2019-03\", \"count\": 793}, " + "{\"key\": \"2019-04\", \"count\": 773}, {\"key\": \"2019-05\", \"count\": 432}" + "]}"; + + // define statistics + nlohmann::json statistic_json = json::parse(statistic); + MS_LOG(INFO) << "statistics: " << statistic; + std::string desc = "axis"; + + std::shared_ptr statistics = Statistics::Build(desc, statistic_json); + + ASSERT_NE(statistics, nullptr); + + MS_LOG(INFO) << "test get_desc(), result: " << statistics->get_desc(); + MS_LOG(INFO) << "test get_statistics, result: " << statistics->get_statistics().dump(); + + statistic_json["test"] = "test"; + statistics = Statistics::Build(desc, statistic_json); + ASSERT_EQ(statistics, nullptr); + + statistic_json.erase("level"); + statistics = Statistics::Build(desc, statistic_json); + ASSERT_EQ(statistics, nullptr); +} + +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_segment_test.cc b/tests/ut/cpp/mindrecord/ut_shard_segment_test.cc new file mode 100644 index 00000000000..c803f584aaa --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_segment_test.cc @@ -0,0 +1,176 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_segment.h" +#include "ut_common.h" + +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +namespace mindspore { +namespace mindrecord { +class TestShardSegment : public UT::Common { + public: + TestShardSegment() {} +}; + +TEST_F(TestShardSegment, TestShardSegment) { + MS_LOG(INFO) << FormatInfo("Test Shard Segment"); + std::string file_name = "./imagenet.shard01"; + + ShardSegment dataset; + dataset.Open(file_name, 4); + + auto x = dataset.GetCategoryFields(); + for (const auto &fields : x.second) { + MS_LOG(INFO) << "Get category field: " << fields; + } + + ASSERT_TRUE(dataset.SetCategoryField("label") == SUCCESS); + ASSERT_TRUE(dataset.SetCategoryField("laabel_0") == FAILED); + + MS_LOG(INFO) << "Read category info: " << dataset.ReadCategoryInfo().second; + + auto ret = dataset.ReadAtPageByName("822", 0, 10); + auto images = ret.second; + MS_LOG(INFO) << "category field: 822, images count: " << images.size() << ", image[0] size: " << images[0].size(); + + auto ret1 = dataset.ReadAtPageByName("823", 0, 10); + auto images2 = ret1.second; + MS_LOG(INFO) << "category field: 823, images count: " << images2.size(); + + auto ret2 = dataset.ReadAtPageById(1, 0, 10); + auto images3 = ret2.second; + MS_LOG(INFO) << "category id: 1, images count: " << images3.size() << ", image[0] size: " << images3[0].size(); + + auto ret3 = dataset.ReadAllAtPageByName("822", 0, 10); + auto images4 = ret3.second; + MS_LOG(INFO) << "category field: 822, images count: " << images4.size(); + + auto ret4 = dataset.ReadAllAtPageById(1, 0, 10); + auto images5 = ret4.second; + MS_LOG(INFO) << "category id: 1, images count: " << images5.size(); +} + +TEST_F(TestShardSegment, TestReadAtPageByNameOfCategoryName) { + MS_LOG(INFO) << FormatInfo("Test ReadAtPageByName of error category_name and category_field"); + std::string file_name = "./imagenet.shard01"; + + ShardSegment dataset; + dataset.Open(file_name, 4); + + auto x = dataset.GetCategoryFields(); + for (const auto &fields : x.second) { + MS_LOG(INFO) << "Get category field: " << fields; + } + + string category_name = "82Cus"; + string category_field = "laabel_0"; + + ASSERT_TRUE(dataset.SetCategoryField("label") == SUCCESS); + ASSERT_TRUE(dataset.SetCategoryField(category_field) == FAILED); + + MS_LOG(INFO) << "Read category info: " << dataset.ReadCategoryInfo().second; + + auto ret = dataset.ReadAtPageByName(category_name, 0, 10); + EXPECT_TRUE(ret.first == FAILED); +} + +TEST_F(TestShardSegment, TestReadAtPageByIdOfCategoryId) { + MS_LOG(INFO) << FormatInfo("Test ReadAtPageById of error categoryId"); + std::string file_name = "./imagenet.shard01"; + + ShardSegment dataset; + dataset.Open(file_name, 4); + + auto x = dataset.GetCategoryFields(); + for (const auto &fields : x.second) { + MS_LOG(INFO) << "Get category field: " << fields; + } + + int64_t categoryId = 2251799813685247; + MS_LOG(INFO) << "Input category id: " << categoryId; + + ASSERT_TRUE(dataset.SetCategoryField("label") == SUCCESS); + MS_LOG(INFO) << "Read category info: " << dataset.ReadCategoryInfo().second; + + auto ret2 = dataset.ReadAtPageById(categoryId, 0, 10); + EXPECT_TRUE(ret2.first == FAILED); +} + +TEST_F(TestShardSegment, TestReadAtPageByIdOfPageNo) { + MS_LOG(INFO) << FormatInfo("Test ReadAtPageById of error page_no"); + std::string file_name = "./imagenet.shard01"; + + ShardSegment dataset; + dataset.Open(file_name, 4); + + auto x = dataset.GetCategoryFields(); + for (const auto &fields : x.second) { + MS_LOG(INFO) << "Get category field: " << fields; + } + + int64_t page_no = 2251799813685247; + MS_LOG(INFO) << "Input page no: " << page_no; + + ASSERT_TRUE(dataset.SetCategoryField("label") == SUCCESS); + MS_LOG(INFO) << "Read category info: " << dataset.ReadCategoryInfo().second; + + auto ret2 = dataset.ReadAtPageById(1, page_no, 10); + EXPECT_TRUE(ret2.first == FAILED); +} + +TEST_F(TestShardSegment, TestReadAtPageByIdOfPageRows) { + MS_LOG(INFO) << FormatInfo("Test ReadAtPageById of error pageRows"); + std::string file_name = "./imagenet.shard01"; + + ShardSegment dataset; + dataset.Open(file_name, 4); + + auto x = dataset.GetCategoryFields(); + for (const auto &fields : x.second) { + MS_LOG(INFO) << "Get category field: " << fields; + } + + int64_t pageRows = 0; + MS_LOG(INFO) << "Input page rows: " << pageRows; + + ASSERT_TRUE(dataset.SetCategoryField("label") == SUCCESS); + MS_LOG(INFO) << "Read category info: " << dataset.ReadCategoryInfo().second; + + auto ret2 = dataset.ReadAtPageById(1, 0, pageRows); + EXPECT_TRUE(ret2.first == FAILED); +} + +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_writer_test.cc b/tests/ut/cpp/mindrecord/ut_shard_writer_test.cc new file mode 100644 index 00000000000..18e9214b08e --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_writer_test.cc @@ -0,0 +1,1464 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "mindrecord/include/shard_reader.h" +#include "mindrecord/include/shard_writer.h" +#include "mindrecord/include/shard_index_generator.h" +#include "securec.h" +#include "ut_common.h" +#include "ut_shard_writer_test.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; +using mindspore::MsLogLevel::INFO; + +namespace mindspore { +namespace mindrecord { +class TestShardWriter : public UT::Common { + public: + TestShardWriter() {} +}; + +void LoadDataFromImageNet(const std::string &directory, std::vector &json_buffer, const int max_num) { + int count = 0; + string input_path = directory; + ifstream infile(input_path); + if (!infile.is_open()) { + MS_LOG(ERROR) << "can not open the file "; + return; + } + string temp; + string filename; + string label; + json j; + while (getline(infile, temp) && count != max_num) { + count++; + std::size_t pos = temp.find(",", 0); + if (pos != std::string::npos) { + j["file_name"] = temp.substr(0, pos); + j["label"] = atoi(common::SafeCStr(temp.substr(pos + 1, temp.length()))); + json_buffer.push_back(j); + } + } + infile.close(); +} + +int Img2DataUint8(const std::vector &img_absolute_path, std::vector> &bin_data) { + for (auto &file : img_absolute_path) { + // read image file + std::ifstream in(common::SafeCStr(file), std::ios::in | std::ios::binary | std::ios::ate); + if (!in) { + MS_LOG(ERROR) << common::SafeCStr(file) << " is not a directory or not exist!"; + return -1; + } + + // get the file size + uint64_t size = in.tellg(); + in.seekg(0, std::ios::beg); + std::vector file_data(size); + in.read(reinterpret_cast(&file_data[0]), size); + in.close(); + bin_data.push_back(file_data); + } + return 0; +} + +int GetAbsoluteFiles(std::string directory, std::vector &files_absolute_path) { + DIR *dir = opendir(common::SafeCStr(directory)); + if (dir == nullptr) { + MS_LOG(ERROR) << common::SafeCStr(directory) << " is not a directory or not exist!"; + return -1; + } + struct dirent *d_ent = nullptr; + char dot[3] = "."; + char dotdot[6] = ".."; + while ((d_ent = readdir(dir)) != nullptr) { + if ((strcmp(d_ent->d_name, dot) != 0) && (strcmp(d_ent->d_name, dotdot) != 0)) { + if (d_ent->d_type == DT_DIR) { + std::string new_directory = directory + std::string("/") + std::string(d_ent->d_name); + if (directory[directory.length() - 1] == '/') { + new_directory = directory + string(d_ent->d_name); + } + if (-1 == GetAbsoluteFiles(new_directory, files_absolute_path)) { + closedir(dir); + return -1; + } + } else { + std::string absolute_path = directory + std::string("/") + std::string(d_ent->d_name); + if (directory[directory.length() - 1] == '/') { + absolute_path = directory + std::string(d_ent->d_name); + } + files_absolute_path.push_back(absolute_path); + } + } + } + closedir(dir); + return 0; +} + +void TestShardWriterImageNet() { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Write imageNet")); + + // load binary data + std::vector> bin_data; + std::vector filenames; + if (-1 == mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)) { + MS_LOG(INFO) << "-- ATTN -- Missed data directory. Skip this case. -----------------"; + return; + } + mindrecord::Img2DataUint8(filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = R"({"file_name": {"type": "string"}, "label": {"type": "int32"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + if (anno_schema == nullptr) { + MS_LOG(ERROR) << "Build annotation schema failed"; + return; + } + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "file_name"); + std::pair index_field2(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + fields.push_back(index_field2); + + // add index to shardHeader + header_data.AddIndexFields(fields); + MS_LOG(INFO) << "Init Index Fields Already."; + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, 10); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init Images Already."; + + // init file_writer + std::vector file_names; + int file_count = 4; + for (int i = 1; i <= file_count; i++) { + file_names.emplace_back(std::string("./imagenet.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + MS_LOG(INFO) << "Init Output Files Already."; + { + mindrecord::ShardWriter fw_init; + fw_init.Open(file_names); + + // set shardHeader + fw_init.SetShardHeader(std::make_shared(header_data)); + + // close file_writer + fw_init.Commit(); + } + std::string filename = "./imagenet.shard01"; + { + MS_LOG(INFO) << "=============== images " << bin_data.size() << " ============================"; + mindrecord::ShardWriter fw; + fw.OpenForAppend(filename); + fw.WriteRawData(rawdatas, bin_data); + fw.Commit(); + } + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + + MS_LOG(INFO) << "Done create index"; +} + +void TestShardWriterImageNetOneSample() { + // load binary data + std::vector> bin_data; + std::vector filenames; + if (-1 == mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)) { + MS_LOG(INFO) << "-- ATTN -- Missed data directory. Skip this case. -----------------"; + return; + } + mindrecord::Img2DataUint8(filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = R"({"file_name": {"type": "string"}, "label": {"type": "int32"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + if (anno_schema == nullptr) { + MS_LOG(ERROR) << "Build annotation schema failed"; + return; + } + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "file_name"); + std::pair index_field2(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + fields.push_back(index_field2); + + // add index to shardHeader + header_data.AddIndexFields(fields); + MS_LOG(INFO) << "Init Index Fields Already."; + + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, 1); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init Images Already."; + + // init file_writer + std::vector file_names; + for (int i = 1; i <= 4; i++) { + file_names.emplace_back(std::string("./OneSample.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + MS_LOG(INFO) << "Init Output Files Already."; + { + mindrecord::ShardWriter fw_init; + fw_init.Open(file_names); + + // set shardHeader + fw_init.SetShardHeader(std::make_shared(header_data)); + + // close file_writer + fw_init.Commit(); + } + + std::string filename = "./OneSample.shard01"; + { + MS_LOG(INFO) << "=============== images " << bin_data.size() << " ============================"; + mindrecord::ShardWriter fw; + fw.OpenForAppend(filename); + bin_data = std::vector>(bin_data.begin(), bin_data.begin() + 1); + fw.WriteRawData(rawdatas, bin_data); + fw.Commit(); + } + + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + MS_LOG(INFO) << "Done create index"; +} + +TEST_F(TestShardWriter, TestShardWriterBench) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write imageNet")); + + TestShardWriterImageNet(); + for (int i = 1; i <= 4; i++) { + string filename = std::string("./imagenet.shard0") + std::to_string(i); + string db_name = std::string("./imagenet.shard0") + std::to_string(i) + ".db"; + remove(common::SafeCStr(filename)); + remove(common::SafeCStr(db_name)); + } +} + +TEST_F(TestShardWriter, TestShardWriterOneSample) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write imageNet int32 of sample less than num of shards")); + TestShardWriterImageNetOneSample(); + std::string filename = "./OneSample.shard01"; + + ShardReader dataset; + MSRStatus ret = dataset.Open(filename, 4); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto &j : x) { + MS_LOG(INFO) << "item size: " << std::get<0>(j).size(); + for (auto &item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << common::SafeCStr(item.key()) << ", value: " << common::SafeCStr(item.value().dump()); + } + } + } + dataset.Finish(); + for (int i = 1; i <= 4; i++) { + string filename = std::string("./OneSample.shard0") + std::to_string(i); + string db_name = std::string("./OneSample.shard0") + std::to_string(i) + ".db"; + remove(common::SafeCStr(filename)); + remove(common::SafeCStr(db_name)); + } +} + +TEST_F(TestShardWriter, TestShardWriterShiftRawPage) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write shift raw page")); + const int kMaxNum = 10; + auto column_list = std::vector{"file_name_0"}; + + string input_path1 = "./data/mindrecord/testCBGData/data/image_raw_meta.data"; + string input_path3 = "./data/mindrecord/testCBGData/statistics/statistics.txt"; + std::string path_dir = "./data/mindrecord/testCBGData/data/pictures"; + + std::vector> bin_data; + + // buffer init + std::vector json_buffer1; // store the image_raw_meta.data + std::vector json_buffer3; // store the pictures + std::vector json_buffer4; // store the statistics data + std::vector image_filenames; // save all files' path within path_dir + + // read image_raw_meta.data + Common::LoadData(input_path1, json_buffer1, kMaxNum); + MS_LOG(INFO) << "Load Meta Data Already."; + + // get files' pathes stored in vector image_filenames + mindrecord::GetAbsoluteFiles(path_dir, image_filenames); // get all files whose path within path_dir + MS_LOG(INFO) << "Only process 10 file names:"; + image_filenames.resize(kMaxNum); + MS_LOG(INFO) << "Load Img Filenames Already."; + + // read pictures + // mindrecord::Img2DataUint8(image_filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json image_schema_json = R"({"name":{"type":"string"}})"_json; + json anno_schema_json = R"({"name":{"type":"string"},"anno_tool":{"type":"string"},"creation_time":{"type":"string"}, + "max_shape_id":{"type":"int32"},"max_entity_id":{"type":"int32"}, + "entity_instances":{"type":"bytes"}})"_json; + + std::shared_ptr image_schema = mindrecord::Schema::Build("picture", image_schema_json); + if (image_schema == nullptr) { + MS_LOG(ERROR) << "Build image schema failed"; + return; + } + + // add schema to shardHeader + int image_schema_id = header_data.AddSchema(image_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create/init statistics + Common::LoadData(input_path3, json_buffer4, 2); + json static1_json = json_buffer4[0]; + json static2_json = json_buffer4[1]; + MS_LOG(INFO) << "Initial statistics 1 is: " << common::SafeCStr(static1_json.dump()); + MS_LOG(INFO) << "Initial statistics 2 is: " << common::SafeCStr(static2_json.dump()); + std::shared_ptr static1 = + mindrecord::Statistics::Build(static1_json["description"], static1_json["statistics"]); + std::shared_ptr static2 = + mindrecord::Statistics::Build(static2_json["description"], static2_json["statistics"]); + MS_LOG(INFO) << "Init Statistics Already."; + + // add statistics to shardHeader + if (static1 == nullptr) { + MS_LOG(ERROR) << "static1 is nullptr"; + return; + } else { + header_data.AddStatistic(static1); + } + if (static2 == nullptr) { + MS_LOG(ERROR) << "static2 is nullptr"; + return; + } else { + header_data.AddStatistic(static2); + } + + // create index field by schema + std::pair index_field1(image_schema_id, "name"); + std::vector> fields; + fields.push_back(index_field1); + + // add index to shardHeader + header_data.AddIndexFields(fields); + + std::map> rawdatas; + // merge imgBinaryData(json_buffer3) and imgShardHeader(json_buffer1) to imgBinaryData(json_buffer3) + std::string dummy_str = std::string(3000, 'a'); + json dummyJson = {}; + dummyJson["name"] = dummy_str; + std::vector json_buffer; + for (std::size_t i = 0; i < kMaxNum; i++) { + json_buffer.push_back(dummyJson); + } + rawdatas.insert(pair>(0, json_buffer)); + + bin_data.clear(); + auto image = std::vector(10240, 1); + for (std::size_t i = 0; i < kMaxNum; i++) { + bin_data.push_back(image); + } + // init file_writer + MS_LOG(INFO) << "Init Writer ..."; + std::vector file_names; + + file_names.push_back("./train_base64.mindrecord01"); + + { + mindrecord::ShardWriter fw; + fw.Open(file_names); + uint64_t header_size = 1 << 14; + uint64_t page_size = 1 << 15; + fw.set_header_size(header_size); + fw.set_page_size(page_size); + + // set shardHeader + fw.SetShardHeader(std::make_shared(header_data)); + fw.WriteRawData(rawdatas, bin_data); + fw.Commit(); + } + + { + mindrecord::ShardWriter fw; + fw.OpenForAppend(file_names[0]); + fw.WriteRawData(rawdatas, bin_data); + fw.Commit(); + } + + for (const auto &oneFile : file_names) { + remove(common::SafeCStr(oneFile)); + } +} + +TEST_F(TestShardWriter, TestShardWriterTrial) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write trial data")); + int kMaxNum = 10; + auto column_list = std::vector{"file_name_0"}; + + string input_path1 = "./data/mindrecord/testCBGData/data/image_raw_meta.data"; + string input_path3 = "./data/mindrecord/testCBGData/statistics/statistics.txt"; + std::string path_dir = "./data/mindrecord/testCBGData/data/pictures"; + + std::vector> bin_data; + + // buffer init + std::vector json_buffer1; // store the image_raw_meta.data + std::vector json_buffer3; // store the pictures + std::vector json_buffer4; // store the statistics data + std::vector image_filenames; // save all files' path within path_dir + + // read image_raw_meta.data + Common::LoadData(input_path1, json_buffer1, kMaxNum); + MS_LOG(INFO) << "Load Meta Data Already."; + + // get files' pathes stored in vector image_filenames + mindrecord::GetAbsoluteFiles(path_dir, image_filenames); // get all files whose path within path_dir + MS_LOG(INFO) << "Only process 10 file names:"; + image_filenames.resize(kMaxNum); + MS_LOG(INFO) << "Load Img Filenames Already."; + + // read pictures + mindrecord::Img2DataUint8(image_filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json image_schema_json = R"({"name":{"type":"string"},"size":{"type":"int32"},"type":{"type":"string"}, + "dataset_id":{"type":"int32"},"creation_time":{"type":"string"}})"_json; + json anno_schema_json = R"({"name":{"type":"string"},"anno_tool":{"type":"string"},"creation_time":{"type":"string"}, + "max_shape_id":{"type":"int32"},"max_entity_id":{"type":"int32"}, + "entity_instances":{"type":"bytes"}})"_json; + + std::shared_ptr image_schema = mindrecord::Schema::Build("picture", image_schema_json); + if (image_schema == nullptr) { + MS_LOG(ERROR) << "Build image schema failed"; + return; + } + + // add schema to shardHeader + int image_schema_id = header_data.AddSchema(image_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create/init statistics + Common::LoadData(input_path3, json_buffer4, 2); + json static1_json = json_buffer4[0]; + json static2_json = json_buffer4[1]; + MS_LOG(INFO) << "Initial statistics 1 is: " << common::SafeCStr(static1_json.dump()); + MS_LOG(INFO) << "Initial statistics 2 is: " << common::SafeCStr(static2_json.dump()); + std::shared_ptr static1 = + mindrecord::Statistics::Build(static1_json["description"], static1_json["statistics"]); + std::shared_ptr static2 = + mindrecord::Statistics::Build(static2_json["description"], static2_json["statistics"]); + MS_LOG(INFO) << "Init Statistics Already."; + + // add statistics to shardHeader + if (static1 == nullptr) { + MS_LOG(ERROR) << "static1 is nullptr"; + return; + } else { + header_data.AddStatistic(static1); + } + if (static2 == nullptr) { + MS_LOG(ERROR) << "static2 is nullptr"; + return; + } else { + header_data.AddStatistic(static2); + } + + // create index field by schema + std::pair index_field1(image_schema_id, "name"); + std::vector> fields; + fields.push_back(index_field1); + + // add index to shardHeader + header_data.AddIndexFields(fields); + + // merge imgBinaryData(json_buffer3) and imgShardHeader(json_buffer1) to imgBinaryData(json_buffer3) + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3.push_back(json{}); + } + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3[i] = json_buffer1[i]; // add meta_data to json_buffer3's json variable + } + + // get json2bson size indicate image size + json j_test = json_buffer3[0]; + + // reference json variable + std::vector &images = json_buffer3; // imgBinaryData && imgShardHeader + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(image_schema_id, images)); + + // init file_writer + MS_LOG(INFO) << "Init Writer ..."; + std::vector file_names; + + // std::vector file_names = {"train_base64.mindrecord01", "train_base64.mindrecord02", + // "train_base64.mindrecord03"}; + file_names.push_back("./train_base64.mindrecord01"); + file_names.push_back("./train_base64.mindrecord02"); + file_names.push_back("./train_base64.mindrecord03"); + mindrecord::ShardWriter fw; + fw.Open(file_names); + uint64_t header_size = 1 << 14; + uint64_t page_size = 1 << 17; + fw.set_header_size(header_size); + fw.set_page_size(page_size); + + // set shardHeader + fw.SetShardHeader(std::make_shared(header_data)); + + // write rawdata + fw.WriteRawData(rawdatas, bin_data); + + // close file_writer + fw.Commit(); + std::string filename = "./train_base64.mindrecord01"; + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + MS_LOG(INFO) << "Done create index"; + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, TestShardWriterTrialNoFields) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write trial data")); + int kMaxNum = 10; + auto column_list = std::vector{"file_name_0"}; + + string input_path1 = "./data/mindrecord/testCBGData/data/image_raw_meta.data"; + string input_path3 = "./data/mindrecord/testCBGData/statistics/statistics.txt"; + std::string path_dir = "./data/mindrecord/testCBGData/data/pictures"; + + std::vector> bin_data; + + // buffer init + std::vector json_buffer1; // store the image_raw_meta.data + std::vector json_buffer3; // store the pictures + std::vector json_buffer4; // store the statistics data + std::vector image_filenames; // save all files' path within path_dir + + // read image_raw_meta.data + Common::LoadData(input_path1, json_buffer1, kMaxNum); + MS_LOG(INFO) << "Load Meta Data Already."; + + // get files' pathes stored in vector image_filenames + mindrecord::GetAbsoluteFiles(path_dir, image_filenames); // get all files whose path within path_dir + MS_LOG(INFO) << "Only process 10 file names:"; + image_filenames.resize(kMaxNum); + MS_LOG(INFO) << "Load Img Filenames Already."; + + // read pictures + mindrecord::Img2DataUint8(image_filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json image_schema_json = R"({"name":{"type":"string"},"size":{"type":"int32"},"type":{"type":"string"}, + "dataset_id":{"type":"int32"},"creation_time":{"type":"string"}})"_json; + + std::shared_ptr image_schema = mindrecord::Schema::Build("picture", image_schema_json); + if (image_schema == nullptr) { + MS_LOG(ERROR) << "Build image schema failed"; + return; + } + + // add schema to shardHeader + int image_schema_id = header_data.AddSchema(image_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create/init statistics + Common::LoadData(input_path3, json_buffer4, 2); + json static1_json = json_buffer4[0]; + json static2_json = json_buffer4[1]; + MS_LOG(INFO) << "Initial statistics 1 is: " << common::SafeCStr(static1_json.dump()); + MS_LOG(INFO) << "Initial statistics 2 is: " << common::SafeCStr(static2_json.dump()); + std::shared_ptr static1 = + mindrecord::Statistics::Build(static1_json["description"], static1_json["statistics"]); + std::shared_ptr static2 = + mindrecord::Statistics::Build(static2_json["description"], static2_json["statistics"]); + MS_LOG(INFO) << "Init Statistics Already."; + + // add statistics to shardHeader + if (static1 == nullptr) { + MS_LOG(ERROR) << "static1 is nullptr"; + return; + } else { + header_data.AddStatistic(static1); + } + if (static2 == nullptr) { + MS_LOG(ERROR) << "static2 is nullptr"; + return; + } else { + header_data.AddStatistic(static2); + } + + // create index field by schema + std::pair index_field1(image_schema_id, "name"); + std::vector> fields; + fields.push_back(index_field1); + + // add index to shardHeader + + // merge imgBinaryData(json_buffer3) and imgShardHeader(json_buffer1) to imgBinaryData(json_buffer3) + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3.push_back(json{}); + } + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3[i] = json_buffer1[i]; + } + + // get json2bson size indicate image size + json j_test = json_buffer3[0]; + + // reference json variable + std::vector &images = json_buffer3; // imgBinaryData && imgShardHeader + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(image_schema_id, images)); + + // init file_writer + MS_LOG(INFO) << "Init Writer ..."; + std::vector file_names; + + // std::vector file_names = {"train_base64.mindrecord01", "train_base64.mindrecord02", + // "train_base64.mindrecord03"}; + file_names.push_back("./train_base64.mindrecord01"); + file_names.push_back("./train_base64.mindrecord02"); + file_names.push_back("./train_base64.mindrecord03"); + mindrecord::ShardWriter fw; + fw.Open(file_names); + uint64_t header_size = 1 << 14; + uint64_t page_size = 1 << 17; + fw.set_header_size(header_size); + fw.set_page_size(page_size); + + // set shardHeader + fw.SetShardHeader(std::make_shared(header_data)); + + // write rawdata + fw.WriteRawData(rawdatas, bin_data); + + // close file_writer + fw.Commit(); + MS_LOG(INFO) << "fw ok"; + std::string filename = "./train_base64.mindrecord01"; + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + MS_LOG(INFO) << "Done create index"; + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, DataCheck) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write trial data")); + int kMaxNum = 10; + auto column_list = std::vector{"file_name_0"}; + + string input_path1 = "./data/mindrecord/testCBGData/data/image_raw_meta.data"; + std::string path_dir = "./data/mindrecord/testCBGData/data/pictures"; + + std::vector> bin_data; + + // buffer init + std::vector json_buffer1; // store the image_raw_meta.data + std::vector json_buffer3; // store the pictures + std::vector image_filenames; // save all files' path within path_dir + + // read image_raw_meta.data + LoadData(input_path1, json_buffer1, kMaxNum); + MS_LOG(INFO) << "Load Meta Data Already."; + + // get files' pathes stored in vector image_filenames + mindrecord::GetAbsoluteFiles(path_dir, image_filenames); // get all files whose path within path_dir + MS_LOG(INFO) << "Only process 10 file names:"; + image_filenames.resize(kMaxNum); + MS_LOG(INFO) << "Load Img Filenames Already."; + + // read pictures + mindrecord::Img2DataUint8(image_filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json image_schema_json = R"({"name":{"type":"string"},"size":{"type":"int32"},"type":{"type":"string"}, + "dataset_id":{"type":"int32"},"creation_time":{"type":"string"}, + "entity_instances":{"type":"int32","shape":[-1]}})"_json; + std::shared_ptr image_schema = mindrecord::Schema::Build("picture", image_schema_json); + if (image_schema == nullptr) { + MS_LOG(ERROR) << "Build image schema failed"; + return; + } + + // add schema to shardHeader + int image_schema_id = header_data.AddSchema(image_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // merge imgBinaryData(json_buffer3) and imgShardHeader(json_buffer1) to imgBinaryData(json_buffer3) + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3.push_back(json{}); + } + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3[i] = json_buffer1[i]; // add meta_data to json_buffer3's json variable + } + + // get json2bson size indicate image size + json j_test = json_buffer3[0]; + + // reference json variable + std::vector &images = json_buffer3; // imgBinaryData && imgShardHeader + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(image_schema_id, images)); + + // init file_writer + MS_LOG(INFO) << "Init Writer ..."; + std::vector file_names; + + // std::vector file_names = {"train_base64.mindrecord01", "train_base64.mindrecord02", + // "train_base64.mindrecord03"}; + file_names.push_back("./train_base64.mindrecord01"); + file_names.push_back("./train_base64.mindrecord02"); + file_names.push_back("./train_base64.mindrecord03"); + mindrecord::ShardWriter fw; + fw.Open(file_names); + uint64_t header_size = 1 << 14; + uint64_t page_size = 1 << 17; + fw.set_header_size(header_size); + fw.set_page_size(page_size); + + // set shardHeader + fw.SetShardHeader(std::make_shared(header_data)); + + // write rawdata + fw.WriteRawData(rawdatas, bin_data); + + // close file_writer + fw.Commit(); + std::string filename = "./train_base64.mindrecord01"; + // std::string filename = "train_base64.mindrecord01"; + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + MS_LOG(INFO) << "Done create index"; + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, AllRawDataWrong) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write trial data")); + int kMaxNum = 10; + auto column_list = std::vector{"file_name_0"}; + + string input_path1 = "./data/mindrecord/testCBGData/data/image_raw_meta.data"; + std::string path_dir = "./data/mindrecord/testCBGData/data/pictures"; + + std::vector> bin_data; + + // buffer init + std::vector json_buffer1; // store the image_raw_meta.data + std::vector json_buffer3; // store the pictures + std::vector image_filenames; // save all files' path within path_dir + + // read image_raw_meta.data + LoadData(input_path1, json_buffer1, kMaxNum); + MS_LOG(INFO) << "Load Meta Data Already."; + + // get files' pathes stored in vector image_filenames + mindrecord::GetAbsoluteFiles(path_dir, image_filenames); // get all files whose path within path_dir + MS_LOG(INFO) << "Only process 10 file names:"; + image_filenames.resize(kMaxNum); + MS_LOG(INFO) << "Load Img Filenames Already."; + + // read pictures + mindrecord::Img2DataUint8(image_filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json image_schema_json = R"({"name":{"type":"string"},"size":{"type":"int32"},"type":{"type":"string"}, + "id":{"type":"int32"},"creation_time":{"type":"string"}, + "entity_instances":{"type":"int32","shape":[-1]}})"_json; + std::shared_ptr image_schema = mindrecord::Schema::Build("picture", image_schema_json); + if (image_schema == nullptr) { + MS_LOG(ERROR) << "Build image schema failed"; + return; + } + + // add schema to shardHeader + int image_schema_id = header_data.AddSchema(image_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // merge imgBinaryData(json_buffer3) and imgShardHeader(json_buffer1) to imgBinaryData(json_buffer3) + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3.push_back(json{}); + } + for (std::size_t i = 0; i < json_buffer1.size(); i++) { + json_buffer3[i] = json_buffer1[i]; // add meta_data to json_buffer3's json variable + } + + // get json2bson size indicate image size + json j_test = json_buffer3[0]; + + // reference json variable + std::vector &images = json_buffer3; // imgBinaryData && imgShardHeader + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(image_schema_id, images)); + + // init file_writer + MS_LOG(INFO) << "Init Writer ..."; + std::vector file_names; + + // std::vector file_names = {"train_base64.mindrecord01", "train_base64.mindrecord02", + // "train_base64.mindrecord03"}; + file_names.push_back("./train_base64.mindrecord01"); + file_names.push_back("./train_base64.mindrecord02"); + file_names.push_back("./train_base64.mindrecord03"); + mindrecord::ShardWriter fw; + fw.Open(file_names); + uint64_t header_size = 1 << 14; + uint64_t page_size = 1 << 17; + fw.set_header_size(header_size); + fw.set_page_size(page_size); + + // set shardHeader + fw.SetShardHeader(std::make_shared(header_data)); + + // write rawdata + MSRStatus res = fw.WriteRawData(rawdatas, bin_data); + ASSERT_EQ(res, SUCCESS); + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, TestShardReaderStringAndNumberColumnInIndex) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet string and int32 are in index")); + + // load binary data + std::vector> bin_data; + std::vector filenames; + ASSERT_NE(-1, mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)); + ASSERT_NE(-1, mindrecord::Img2DataUint8(filenames, bin_data)); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = + R"({"file_name": {"type": "string"}, "label": {"type": "int32"}, "data":{"type":"bytes"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + ASSERT_TRUE(anno_schema != nullptr); + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + ASSERT_EQ(anno_schema_id, 0); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "file_name"); + std::pair index_field2(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + fields.push_back(index_field2); + + // add index to shardHeader + ASSERT_EQ(header_data.AddIndexFields(fields), SUCCESS); + MS_LOG(INFO) << "Init Index Fields Already."; + + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, 10); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init Images Already."; + + // init file_writer + std::vector file_names; + for (int i = 1; i <= 4; i++) { + file_names.emplace_back(std::string("./imagenet.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + mindrecord::ShardWriter fw_init; + ASSERT_TRUE(fw_init.Open(file_names) == SUCCESS); + + // set shardHeader + ASSERT_TRUE(fw_init.SetShardHeader(std::make_shared(header_data)) == SUCCESS); + + // write raw data + ASSERT_TRUE(fw_init.WriteRawData(rawdatas, bin_data) == SUCCESS); + ASSERT_TRUE(fw_init.Commit() == SUCCESS); + + // create the index file + std::string filename = "./imagenet.shard01"; + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + ASSERT_TRUE(sg.WriteToDatabase() == SUCCESS); + MS_LOG(INFO) << "Done create index"; + + // read the mindrecord file + filename = "./imagenet.shard01"; + auto column_list = std::vector{"label", "file_name", "data"}; + ShardReader dataset; + MSRStatus ret = dataset.Open(filename, 4, column_list); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + int count = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto &j : x) { + count++; + json resp = std::get<1>(j); + MS_LOG(INFO) << resp.dump(); + ASSERT_EQ(resp.size(), 2); + ASSERT_TRUE(resp.size() == 2); + ASSERT_TRUE(std::string(resp["file_name"].type_name()) == "string"); + ASSERT_TRUE(std::string(resp["label"].type_name()) == "number"); + } + } + ASSERT_TRUE(count == 10); + dataset.Finish(); + + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, TestShardNoBlob) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test no-blob")); + + // load binary data + std::vector> bin_data; + std::vector filenames; + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = R"({"file_name": {"type": "string"}, "label": {"type": "int32"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + ASSERT_TRUE(anno_schema != nullptr); + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + ASSERT_EQ(anno_schema_id, 0); + MS_LOG(INFO) << "Init Schema Already."; + + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, 10); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init labels Already."; + + // init file_writer + std::vector file_names; + for (int i = 1; i <= 4; i++) { + file_names.emplace_back(std::string("./imagenet.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + mindrecord::ShardWriter fw_init; + ASSERT_TRUE(fw_init.Open(file_names) == SUCCESS); + + // set shardHeader + ASSERT_TRUE(fw_init.SetShardHeader(std::make_shared(header_data)) == SUCCESS); + + // write raw data + ASSERT_TRUE(fw_init.WriteRawData(rawdatas, bin_data) == SUCCESS); + ASSERT_TRUE(fw_init.Commit() == SUCCESS); + + // create the index file + std::string filename = "./imagenet.shard01"; + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + ASSERT_TRUE(sg.WriteToDatabase() == SUCCESS); + MS_LOG(INFO) << "Done create index"; + + // read the mindrecord file + filename = "./imagenet.shard01"; + auto column_list = std::vector{"label", "file_name"}; + ShardReader dataset; + MSRStatus ret = dataset.Open(filename, 4, column_list); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + int count = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto &j : x) { + count++; + json resp = std::get<1>(j); + ASSERT_TRUE(resp.size() == 2); + ASSERT_TRUE(std::string(resp["label"].type_name()) == "number"); + } + } + ASSERT_TRUE(count == 10); + dataset.Finish(); + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, TestShardReaderStringAndNumberNotColumnInIndex) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test read imageNet int32 is in index")); + + // load binary data + std::vector> bin_data; + std::vector filenames; + ASSERT_NE(-1, mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)); + ASSERT_NE(-1, mindrecord::Img2DataUint8(filenames, bin_data)); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = + R"({"file_name": {"type": "string"}, "label": {"type": "int32"}, "data":{"type":"bytes"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + ASSERT_TRUE(anno_schema != nullptr); + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + ASSERT_EQ(anno_schema_id, 0); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + + // add index to shardHeader + ASSERT_EQ(header_data.AddIndexFields(fields), SUCCESS); + MS_LOG(INFO) << "Init Index Fields Already."; + + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, 10); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init Images Already."; + + // init file_writer + std::vector file_names; + for (int i = 1; i <= 4; i++) { + file_names.emplace_back(std::string("./imagenet.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + mindrecord::ShardWriter fw_init; + ASSERT_TRUE(fw_init.Open(file_names) == SUCCESS); + + // set shardHeader + ASSERT_TRUE(fw_init.SetShardHeader(std::make_shared(header_data)) == SUCCESS); + + // write raw data + ASSERT_TRUE(fw_init.WriteRawData(rawdatas, bin_data) == SUCCESS); + ASSERT_TRUE(fw_init.Commit() == SUCCESS); + + // create the index file + std::string filename = "./imagenet.shard01"; + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + ASSERT_TRUE(sg.WriteToDatabase() == SUCCESS); + MS_LOG(INFO) << "Done create index"; + + // read the mindrecord file + filename = "./imagenet.shard01"; + auto column_list = std::vector{"label", "data"}; + ShardReader dataset; + MSRStatus ret = dataset.Open(filename, 4, column_list); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + int count = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto &j : x) { + count++; + json resp = std::get<1>(j); + ASSERT_TRUE(resp.size() == 1); + ASSERT_TRUE(std::string(resp["label"].type_name()) == "number"); + } + } + ASSERT_TRUE(count == 10); + dataset.Finish(); + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, TestShardWriter10Sample40Shard) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write imageNet int32 of sample less than num of shards")); + + int num_sample = 10; + int num_shard = 40; + + // load binary data + std::vector> bin_data; + std::vector filenames; + ASSERT_NE(-1, mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)); + + mindrecord::Img2DataUint8(filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = + R"({"file_name": {"type": "string"}, "label": {"type": "int32"}, "data":{"type":"bytes"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + if (anno_schema == nullptr) { + MS_LOG(ERROR) << "Build annotation schema failed"; + return; + } + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "file_name"); + std::pair index_field2(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + fields.push_back(index_field2); + + // add index to shardHeader + header_data.AddIndexFields(fields); + MS_LOG(INFO) << "Init Index Fields Already."; + + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, num_sample); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init Images Already."; + + // init file_writer + std::vector file_names; + for (int i = 1; i <= num_shard; i++) { + file_names.emplace_back(std::string("./TenSampleFortyShard.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + MS_LOG(INFO) << "Init Output Files Already."; + { + mindrecord::ShardWriter fw_init; + fw_init.Open(file_names); + // set shardHeader + fw_init.SetShardHeader(std::make_shared(header_data)); + // close file_writer + fw_init.Commit(); + } + std::string filename = "./TenSampleFortyShard.shard01"; + { + MS_LOG(INFO) << "=============== images " << bin_data.size() << " ============================"; + mindrecord::ShardWriter fw; + fw.OpenForAppend(filename); + bin_data = std::vector>(bin_data.begin(), bin_data.begin() + num_sample); + fw.WriteRawData(rawdatas, bin_data); + fw.Commit(); + } + + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + MS_LOG(INFO) << "Done create index"; + + filename = "./TenSampleFortyShard.shard01"; + ShardReader dataset; + MSRStatus ret = dataset.Open(filename, 4); + ASSERT_EQ(ret, SUCCESS); + dataset.Launch(); + + int count = 0; + while (true) { + auto x = dataset.GetNext(); + if (x.empty()) break; + for (auto &j : x) { + MS_LOG(INFO) << "item size: " << std::get<0>(j).size(); + for (auto &item : std::get<1>(j).items()) { + MS_LOG(INFO) << "key: " << common::SafeCStr(item.key()) << ", value: " << common::SafeCStr(item.value().dump()); + } + } + count++; + } + ASSERT_TRUE(count == 10); + dataset.Finish(); + for (const auto &filename : file_names) { + auto filename_db = filename + ".db"; + remove(common::SafeCStr(filename_db)); + remove(common::SafeCStr(filename)); + } +} + +TEST_F(TestShardWriter, TestWriteOpenFileName) { + MS_LOG(INFO) << common::SafeCStr(FormatInfo("Test write imageNet with error filename contain invalid utf-8 data")); + mindrecord::ShardHeader header_data; + + // create schema + json anno_schema_json = R"({"file_name": {"type": "string"}, "label": {"type": "int32"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + if (anno_schema == nullptr) { + MS_LOG(ERROR) << "Build annotation schema failed"; + return; + } + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "file_name"); + std::pair index_field2(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + fields.push_back(index_field2); + + // add index to shardHeader + header_data.AddIndexFields(fields); + MS_LOG(INFO) << "Init Index Fields Already."; + + string filename = "./ä\xA9ü"; + MS_LOG(INFO) << "filename: " << common::SafeCStr(filename); + + std::vector file_names; + for (int i = 1; i <= 4; i++) { + // file_names.emplace_back(std::string(filename).substr(0, std::string(filename).length()-1) + std::to_string(i)); + file_names.emplace_back(std::string(filename) + "0" + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + MS_LOG(INFO) << "Init Output Files Already."; + { + mindrecord::ShardWriter fw_init; + fw_init.Open(file_names); + // set shardHeader + fw_init.SetShardHeader(std::make_shared(header_data)); + // close file_writer + fw_init.Commit(); + } +} + +void TestShardWriterImageNetOpenForAppend(string filename) { + for (int i = 1; i <= 4; i++) { + string filename = std::string("./OpenForAppendSample.shard0") + std::to_string(i); + string db_name = std::string("./OpenForAppendSample.shard0") + std::to_string(i) + ".db"; + remove(common::SafeCStr(filename)); + remove(common::SafeCStr(db_name)); + } + + // load binary data + std::vector> bin_data; + std::vector filenames; + if (-1 == mindrecord::GetAbsoluteFiles("./data/mindrecord/testImageNetData/images", filenames)) { + MS_LOG(INFO) << "-- ATTN -- Missed data directory. Skip this case. -----------------"; + return; + } + mindrecord::Img2DataUint8(filenames, bin_data); + + // init shardHeader + mindrecord::ShardHeader header_data; + MS_LOG(INFO) << "Init ShardHeader Already."; + + // create schema + json anno_schema_json = R"({"file_name": {"type": "string"}, "label": {"type": "int32"}})"_json; + std::shared_ptr anno_schema = mindrecord::Schema::Build("annotation", anno_schema_json); + if (anno_schema == nullptr) { + MS_LOG(ERROR) << "Build annotation schema failed"; + return; + } + + // add schema to shardHeader + int anno_schema_id = header_data.AddSchema(anno_schema); + MS_LOG(INFO) << "Init Schema Already."; + + // create index + std::pair index_field1(anno_schema_id, "file_name"); + std::pair index_field2(anno_schema_id, "label"); + std::vector> fields; + fields.push_back(index_field1); + fields.push_back(index_field2); + + // add index to shardHeader + header_data.AddIndexFields(fields); + MS_LOG(INFO) << "Init Index Fields Already."; + + // load meta data + std::vector annotations; + LoadDataFromImageNet("./data/mindrecord/testImageNetData/annotation.txt", annotations, 1); + + // add data + std::map> rawdatas; + rawdatas.insert(pair>(anno_schema_id, annotations)); + MS_LOG(INFO) << "Init Images Already."; + + // init file_writer + std::vector file_names; + for (int i = 1; i <= 4; i++) { + file_names.emplace_back(std::string("./OpenForAppendSample.shard0") + std::to_string(i)); + MS_LOG(INFO) << "shard name is: " << common::SafeCStr(file_names[i - 1]); + } + + MS_LOG(INFO) << "Init Output Files Already."; + { + mindrecord::ShardWriter fw_init; + fw_init.Open(file_names); + + // set shardHeader + fw_init.SetShardHeader(std::make_shared(header_data)); + + // close file_writer + fw_init.Commit(); + } + { + MS_LOG(INFO) << "=============== images " << bin_data.size() << " ============================"; + mindrecord::ShardWriter fw; + auto ret = fw.OpenForAppend(filename); + if (ret == FAILED) { + return; + } + + bin_data = std::vector>(bin_data.begin(), bin_data.begin() + 1); + fw.WriteRawData(rawdatas, bin_data); + fw.Commit(); + } + + mindrecord::ShardIndexGenerator sg{filename}; + sg.Build(); + sg.WriteToDatabase(); + MS_LOG(INFO) << "Done create index"; +} + +TEST_F(TestShardWriter, TestOpenForAppend) { + MS_LOG(INFO) << "start ---- TestOpenForAppend\n"; + string filename = "./"; + TestShardWriterImageNetOpenForAppend(filename); + + string filename1 = "./▒AppendSample.shard01"; + TestShardWriterImageNetOpenForAppend(filename1); + string filename2 = "./ä\xA9ü"; + + TestShardWriterImageNetOpenForAppend(filename2); + MS_LOG(INFO) << "end ---- TestOpenForAppend\n"; +} + +} // namespace mindrecord +} // namespace mindspore diff --git a/tests/ut/cpp/mindrecord/ut_shard_writer_test.h b/tests/ut/cpp/mindrecord/ut_shard_writer_test.h new file mode 100644 index 00000000000..f665297b17d --- /dev/null +++ b/tests/ut/cpp/mindrecord/ut_shard_writer_test.h @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TESTS_MINDRECORD_UT_SHARDWRITER_H +#define TESTS_MINDRECORD_UT_SHARDWRITER_H + +namespace mindspore { +namespace mindrecord { +void TestShardWriterImageNet(); +} // namespace mindrecord +} // namespace mindspore + +#endif // TESTS_MINDRECORD_UT_SHARDWRITER_H diff --git a/tests/ut/cpp/operator/cc_implementations_test.cc b/tests/ut/cpp/operator/cc_implementations_test.cc new file mode 100644 index 00000000000..bac885db887 --- /dev/null +++ b/tests/ut/cpp/operator/cc_implementations_test.cc @@ -0,0 +1,477 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common_test.h" +#include "operator/cc_implementations.h" + +namespace mindspore { +namespace prim { + +class TestImplementations : public UT::Common { + public: + TestImplementations() {} + virtual void SetUp() {} +}; + +TEST_F(TestImplementations, ScalarAddTest) { + ValuePtrList list; + list.push_back(MakeValue(1)); + list.push_back(MakeValue(2)); + ASSERT_EQ(ScalarAdd(list)->cast()->value(), 3); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.5f)); + ASSERT_EQ(ScalarAdd(list)->cast()->value(), 2.5f); + list.clear(); + + list.push_back(MakeValue(3.0)); + list.push_back(MakeValue(0.5)); + ASSERT_EQ(ScalarAdd(list)->cast()->value(), 3.5); + list.clear(); + + list.push_back(MakeValue(INT32_MAX)); + list.push_back(MakeValue(2)); + try { + ScalarAdd(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the sum of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(INT32_MIN)); + list.push_back(MakeValue(-1)); + try { + ScalarAdd(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the sum of two signed number") != std::string::npos); + } + list.clear(); +} + +TEST_F(TestImplementations, ScalarSubTest) { + ValuePtrList list; + list.push_back(MakeValue(1)); + list.push_back(MakeValue(3)); + ASSERT_EQ(ScalarSub(list)->cast()->value(), -2); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.5f)); + ASSERT_EQ(ScalarSub(list)->cast()->value(), -0.5f); + list.clear(); + + list.push_back(MakeValue(3.0)); + list.push_back(MakeValue(0.5)); + ASSERT_EQ(ScalarSub(list)->cast()->value(), 2.5); + list.clear(); + + list.push_back(MakeValue(INT32_MAX)); + list.push_back(MakeValue(-1)); + try { + ScalarSub(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the sub of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(INT32_MIN)); + list.push_back(MakeValue(1)); + try { + ScalarSub(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the sub of two signed number") != std::string::npos); + } + list.clear(); +} + +TEST_F(TestImplementations, ScalarMulTest) { + ValuePtrList list; + list.push_back(MakeValue(2)); + list.push_back(MakeValue(3)); + ASSERT_EQ(ScalarMul(list)->cast()->value(), 6); + list.clear(); + + list.push_back(MakeValue(2.0f)); + list.push_back(MakeValue(1.5f)); + ASSERT_EQ(ScalarMul(list)->cast()->value(), 3.0f); + list.clear(); + + list.push_back(MakeValue(-2.0)); + list.push_back(MakeValue(-4.0)); + ASSERT_EQ(ScalarMul(list)->cast()->value(), 8.0); + list.clear(); + + list.push_back(MakeValue(10)); + list.push_back(MakeValue(INT32_MAX)); + try { + ScalarMul(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the mul of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(INT32_MIN)); + list.push_back(MakeValue(-1)); + try { + ScalarMul(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the mul of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(-2)); + list.push_back(MakeValue(INT32_MAX)); + try { + ScalarMul(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the mul of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(2)); + list.push_back(MakeValue(INT32_MIN)); + try { + ScalarMul(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the mul of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(0)); + list.push_back(MakeValue(INT32_MIN)); + ASSERT_EQ(ScalarDiv(list)->cast()->value(), 0); + list.clear(); +} + +TEST_F(TestImplementations, ScalarDivTest) { + ValuePtrList list; + list.push_back(MakeValue(6)); + list.push_back(MakeValue(3)); + ASSERT_EQ(ScalarDiv(list)->cast()->value(), 2); + list.clear(); + + list.push_back(MakeValue(3.0f)); + list.push_back(MakeValue(1.5f)); + ASSERT_EQ(ScalarDiv(list)->cast()->value(), 2.0f); + list.clear(); + + list.push_back(MakeValue(-4.0)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarDiv(list)->cast()->value(), -2.0); + list.clear(); + + list.push_back(MakeValue(INT32_MAX)); + list.push_back(MakeValue(0)); + try { + ScalarDiv(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Divisor could not be zero") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(INT32_MIN)); + list.push_back(MakeValue(-1)); + try { + ScalarDiv(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the div of two signed number") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(-1)); + list.push_back(MakeValue(INT32_MIN)); + ASSERT_EQ(ScalarDiv(list)->cast()->value(), 0); + list.clear(); +} + +TEST_F(TestImplementations, ScalarModTest) { + ValuePtrList list; + list.push_back(MakeValue(7)); + list.push_back(MakeValue(3)); + ASSERT_EQ(ScalarMod(list)->cast()->value(), 1); + list.clear(); + + list.push_back(MakeValue(-8)); + list.push_back(MakeValue(3)); + ASSERT_EQ(ScalarMod(list)->cast()->value(), -2); + list.clear(); + + list.push_back(MakeValue(-9)); + list.push_back(MakeValue(2)); + ASSERT_EQ(ScalarMod(list)->cast()->value(), -1); + list.clear(); + + list.push_back(MakeValue(INT32_MIN)); + list.push_back(MakeValue(0)); + try { + ScalarMod(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Could not mod to zero") != std::string::npos); + } + list.clear(); + + list.push_back(MakeValue(INT32_MIN)); + list.push_back(MakeValue(-1)); + try { + ScalarMod(list); + FAIL(); + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("Overflow of the mod of two signed number") != std::string::npos); + } + list.clear(); +} + +TEST_F(TestImplementations, ScalarUAddTest) { + ValuePtrList list; + list.push_back(MakeValue((uint32_t)1)); + ASSERT_EQ(ScalarUAdd(list)->cast()->value(), 1); + list.clear(); +} + +TEST_F(TestImplementations, ScalarLogTest) { + ValuePtrList list; + list.push_back(MakeValue(static_cast(7.3890560989306495))); + ASSERT_EQ(ScalarLog(list)->cast()->value(), 2.0); + list.clear(); +} + +TEST_F(TestImplementations, ScalarUSubTest) { + ValuePtrList list; + list.push_back(MakeValue(1)); + ASSERT_EQ(ScalarUSub(list)->cast()->value(), -1); + list.clear(); +} + +TEST_F(TestImplementations, ScalarEqTest) { + ValuePtrList list; + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.0f)); + ASSERT_EQ(ScalarEq(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarEq(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.0)); + ASSERT_EQ(ScalarEq(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(1.0)); + list.push_back(MakeValue(1.0)); + ASSERT_EQ(ScalarEq(list)->cast()->value(), true); + list.clear(); +} + +TEST_F(TestImplementations, ScalarLtTest) { + ValuePtrList list; + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.0f)); + ASSERT_EQ(ScalarLt(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarLt(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(2.5)); + ASSERT_EQ(ScalarLt(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(2.5)); + list.push_back(MakeValue(3.0)); + ASSERT_EQ(ScalarLt(list)->cast()->value(), true); + list.clear(); +} + +TEST_F(TestImplementations, ScalarGtTest) { + ValuePtrList list; + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(2.0f)); + ASSERT_EQ(ScalarGt(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(2.0f)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarGt(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(2.0f)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarGt(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(2.5)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarGt(list)->cast()->value(), true); + list.clear(); +} + +TEST_F(TestImplementations, ScalarNeTest) { + ValuePtrList list; + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.0f)); + ASSERT_EQ(ScalarNe(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarNe(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarNe(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(2.0)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarNe(list)->cast()->value(), false); + list.clear(); +} + +TEST_F(TestImplementations, ScalarLeTest) { + ValuePtrList list; + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.0f)); + ASSERT_EQ(ScalarLe(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarLe(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarLe(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(6.0)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarLe(list)->cast()->value(), false); + list.clear(); +} + +TEST_F(TestImplementations, ScalarGeTest) { + ValuePtrList list; + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(1.0f)); + ASSERT_EQ(ScalarGe(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarGe(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(1.0f)); + list.push_back(MakeValue(2.0)); + ASSERT_EQ(ScalarGe(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(6.0)); + list.push_back(MakeValue(-1.0f)); + ASSERT_EQ(ScalarGe(list)->cast()->value(), true); + list.clear(); +} + +TEST_F(TestImplementations, BoolNotTest) { + ValuePtrList list; + list.push_back(MakeValue(true)); + ASSERT_EQ(BoolNot(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolNot(list)->cast()->value(), true); + list.clear(); +} + +TEST_F(TestImplementations, BoolAndTest) { + ValuePtrList list; + list.push_back(MakeValue(true)); + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolAnd(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(true)); + list.push_back(MakeValue(true)); + ASSERT_EQ(BoolAnd(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(false)); + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolAnd(list)->cast()->value(), false); + list.clear(); +} + +TEST_F(TestImplementations, BoolOrTest) { + ValuePtrList list; + list.push_back(MakeValue(true)); + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolOr(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(true)); + list.push_back(MakeValue(true)); + ASSERT_EQ(BoolOr(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(false)); + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolOr(list)->cast()->value(), false); + list.clear(); +} + +TEST_F(TestImplementations, BoolEqTest) { + ValuePtrList list; + list.push_back(MakeValue(true)); + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolEq(list)->cast()->value(), false); + list.clear(); + + list.push_back(MakeValue(true)); + list.push_back(MakeValue(true)); + ASSERT_EQ(BoolEq(list)->cast()->value(), true); + list.clear(); + + list.push_back(MakeValue(false)); + list.push_back(MakeValue(false)); + ASSERT_EQ(BoolEq(list)->cast()->value(), true); + list.clear(); +} + +} // namespace prim +} // namespace mindspore diff --git a/tests/ut/cpp/operator/composite_test.cc b/tests/ut/cpp/operator/composite_test.cc new file mode 100644 index 00000000000..984f24b326f --- /dev/null +++ b/tests/ut/cpp/operator/composite_test.cc @@ -0,0 +1,466 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "common/common_test.h" +#include "ir/anf.h" +#include "ir/value.h" +#include "operator/composite/composite.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/prim.h" +#include "pipeline/static_analysis/abstract_function.h" + +namespace mindspore { +using Shape = abstract::Shape; + +using AbstractScalar = abstract::AbstractScalar; +using AbstractScalarPtr = abstract::AbstractScalarPtr; + +using AbstractSlice = abstract::AbstractSlice; +using AbstractSlicePtr = abstract::AbstractSlicePtr; + +using AbstractTuple = abstract::AbstractTuple; +using AbstractTuplePtr = abstract::AbstractTuplePtr; + +using AbstractTensor = abstract::AbstractTensor; +using AbstractTensorPtr = abstract::AbstractTensorPtr; + +using AbstractNone = abstract::AbstractNone; +using AbstractAttribute = abstract::AbstractAttribute; +using AnalysisEngine = abstract::AnalysisEngine; +using AnalysisEnginePtr = abstract::AnalysisEnginePtr; + +class TestComposite : public UT::Common { + public: + virtual void SetUp(); + virtual void TearDown(); + + AnalysisEnginePtr engine_; +}; + +void TestComposite::SetUp() { + // init resource + std::shared_ptr graph_manager = MakeManager(); + engine_ = std::make_shared(abstract::GetPrimEvaluatorConstructors(), graph_manager); +} + +void TestComposite::TearDown() { + // destroy resource +} + +class UTCompositeUtils { + public: + static AbstractTensorPtr ArrayInt32Of(std::initializer_list shp) { + auto ele = std::make_shared(kAnyValue, kInt32); + return std::make_shared(ele, std::make_shared(shp)); + } + static FuncGraphPtr MakeFuncGraph(const MetaFuncGraphPtr &metaFuncGraphPtr, size_t nparam) { + FuncGraphPtr func_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(metaFuncGraphPtr)); + for (size_t i = 0; i < nparam; i++) { + inputs.push_back(func_graph->add_parameter()); + } + CNodePtr cnode_prim = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_prim); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + return func_graph; + } +}; + +TEST_F(TestComposite, test_TupleSlice_arg_two_numbers) { + MetaFuncGraphPtr tupleSlicePtr = std::make_shared("tuple_slice"); + FuncGraphPtr tupleSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tupleSlicePtr, 3); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 6; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple_tensor = std::make_shared(eles); + auto start_index = std::make_shared(1); + auto stop_index = std::make_shared(5); + AbstractBasePtrList args_spec_list = {tuple_tensor, start_index, stop_index}; + + try { + engine_->Run(tupleSliceGraphPtr, args_spec_list); + FAIL() << "Excepted exception :Args type is wrong"; + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("TupleSlice input args size should be 2, but got 3") != std::string::npos); + } catch (...) { + FAIL() << "Excepted exception :Args type is wrong"; + } +} + +TEST_F(TestComposite, test_TupleSlice_arg_one_number) { + MetaFuncGraphPtr tupleSlicePtr = std::make_shared("tuple_slice"); + FuncGraphPtr tupleSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tupleSlicePtr, 2); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 6; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple_tensor = std::make_shared(eles); + auto start_index = std::make_shared(1); + AbstractBasePtrList args_spec_list = {tuple_tensor, start_index}; + + try { + engine_->Run(tupleSliceGraphPtr, args_spec_list); + FAIL() << "Excepted exception :Args type is wrong"; + } catch (std::runtime_error const &err) { + ASSERT_TRUE(std::string(err.what()).find("TypeError") != std::string::npos); + } catch (...) { + FAIL() << "Excepted exception :Args type is wrong"; + } +} + +TEST_F(TestComposite, test_TupleSlice_arg_slice) { + std::shared_ptr env = parse::python_adapter::set_python_scoped(); + MetaFuncGraphPtr tupleSlicePtr = std::make_shared("tuple_slice"); + FuncGraphPtr tupleSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tupleSlicePtr, 2); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 6; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple_tensor = std::make_shared(eles); + auto start_index = std::make_shared(1); + auto stop_index = std::make_shared(6); + auto step = std::make_shared(2); + auto slice = std::make_shared(start_index, stop_index, step); + AbstractBasePtrList args_spec_list = {tuple_tensor, slice}; + + AbstractTuplePtr ret = dyn_cast(engine_->Run(tupleSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 3; + ASSERT_EQ(real, expect); +} + +TEST_F(TestComposite, test_TupleSlice_arg_slice_step_none) { + MetaFuncGraphPtr tupleSlicePtr = std::make_shared("tuple_slice"); + FuncGraphPtr tupleSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tupleSlicePtr, 2); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 6; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple_tensor = std::make_shared(eles); + auto start_index = std::make_shared(1); + auto stop_index = std::make_shared(5); + auto step = std::make_shared(); + auto slice = std::make_shared(start_index, stop_index, step); + AbstractBasePtrList args_spec_list = {tuple_tensor, slice}; + + AbstractTuplePtr ret = dyn_cast(engine_->Run(tupleSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 4; + ASSERT_EQ(real, expect); +} + +TEST_F(TestComposite, test_TupleSlice_arg_slice_step_negative) { + MetaFuncGraphPtr tupleSlicePtr = std::make_shared("tuple_slice"); + FuncGraphPtr tupleSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tupleSlicePtr, 2); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 6; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple_tensor = std::make_shared(eles); + auto start_index = std::make_shared(); + auto stop_index = std::make_shared(); + auto step = std::make_shared(-1); + auto slice = std::make_shared(start_index, stop_index, step); + AbstractBasePtrList args_spec_list = {tuple_tensor, slice}; + + AbstractTuplePtr ret = dyn_cast(engine_->Run(tupleSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 6; + ASSERT_EQ(real, expect); +} + +TEST_F(TestComposite, test_TupleSlice_arg_slice_step_positive) { + MetaFuncGraphPtr tupleSlicePtr = std::make_shared("tuple_slice"); + FuncGraphPtr tupleSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tupleSlicePtr, 2); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 6; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple_tensor = std::make_shared(eles); + auto start_index = std::make_shared(-2); + auto stop_index = std::make_shared(); + auto step = std::make_shared(-1); + auto slice = std::make_shared(start_index, stop_index, step); + AbstractBasePtrList args_spec_list = {tuple_tensor, slice}; + + AbstractTuplePtr ret = dyn_cast(engine_->Run(tupleSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 5; + ASSERT_EQ(real, expect); +} + +TEST_F(TestComposite, test_TensorSliceBySlice) { + MetaFuncGraphPtr tensorSlicePtr = std::make_shared("tensor_slice"); + FuncGraphPtr tensorSlicePtrGraphPtr = UTCompositeUtils::MakeFuncGraph(tensorSlicePtr, 2); + + AbstractBasePtrList eles; + AbstractScalarPtr start_index = std::make_shared(1); + AbstractScalarPtr stop_index = std::make_shared(6); + AbstractScalarPtr step = std::make_shared(2); + + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({6, 7, 8}); + AbstractSlicePtr slice = std::make_shared(start_index, stop_index, step); + AbstractBasePtrList args_spec_list = {tensor, slice}; + + AbstractTensorPtr ret = dyn_cast(engine_->Run(tensorSlicePtrGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract array failed."; + } + AbstractTensorPtr expect = UTCompositeUtils::ArrayInt32Of({3, 7, 8}); + ASSERT_EQ(*ret, *expect); +} + +TEST_F(TestComposite, test_TensorSliceBySliceTuple) { + MetaFuncGraphPtr tensorSlicePtr = std::make_shared("tensor_slice"); + FuncGraphPtr tensorSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tensorSlicePtr, 2); + + AbstractBasePtrList eles; + AbstractScalarPtr start_index = std::make_shared(0); + AbstractScalarPtr stop_index = std::make_shared(6); + AbstractScalarPtr step = std::make_shared(2); + AbstractSlicePtr slice = std::make_shared(start_index, stop_index, step); + eles.push_back(slice); + + start_index = std::make_shared(1); + stop_index = std::make_shared(5); + step = std::make_shared(1); + slice = std::make_shared(start_index, stop_index, step); + eles.push_back(slice); + + start_index = std::make_shared(2); + stop_index = std::make_shared(8); + step = std::make_shared(3); + slice = std::make_shared(start_index, stop_index, step); + eles.push_back(slice); + + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({6, 7, 8}); + AbstractTuplePtr slice_tuple = std::make_shared(eles); + AbstractBasePtrList args_spec_list = {tensor, slice_tuple}; + + AbstractTensorPtr ret = dyn_cast(engine_->Run(tensorSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract array failed."; + } + AbstractTensorPtr expect = UTCompositeUtils::ArrayInt32Of({3, 4, 2}); + ASSERT_EQ(*ret, *expect); +} + +TEST_F(TestComposite, test_TensorSliceBySliceTupleToReduceDimension) { + MetaFuncGraphPtr tensorSlicePtr = std::make_shared("tensor_slice"); + FuncGraphPtr tensorSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tensorSlicePtr, 2); + + AbstractBasePtrList eles; + AbstractScalarPtr start_index = std::make_shared(1); + AbstractScalarPtr stop_index = std::make_shared(5); + AbstractScalarPtr step = std::make_shared(2); + AbstractSlicePtr slice = std::make_shared(start_index, stop_index, step); + eles.push_back(slice); + + AbstractScalarPtr elem_index = std::make_shared(1); + eles.push_back(elem_index); + + start_index = std::make_shared(2); + stop_index = std::make_shared(6); + step = std::make_shared(1); + slice = std::make_shared(start_index, stop_index, step); + eles.push_back(slice); + + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({6, 7, 8}); + AbstractTuplePtr slice_tuple = std::make_shared(eles); + AbstractBasePtrList args_spec_list = {tensor, slice_tuple}; + + AbstractTensorPtr ret = dyn_cast(engine_->Run(tensorSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract array failed."; + } + AbstractTensorPtr expect = UTCompositeUtils::ArrayInt32Of({2, 4}); + ASSERT_EQ(*ret, *expect); +} + +TEST_F(TestComposite, test_TensorSliceByScalar) { + MetaFuncGraphPtr tensorSlicePtr = std::make_shared("tensor_slice"); + FuncGraphPtr tensorSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tensorSlicePtr, 2); + + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({6, 7, 8}); + AbstractScalarPtr start_index = std::make_shared(2); + AbstractBasePtrList args_spec_list = {tensor, start_index}; + + AbstractTensorPtr ret = dyn_cast(engine_->Run(tensorSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract array failed."; + } + AbstractTensorPtr expect = UTCompositeUtils::ArrayInt32Of({7, 8}); + ASSERT_EQ(*ret, *expect); +} + +TEST_F(TestComposite, test_TensorSliceByScalarTuple) { + MetaFuncGraphPtr tensorSlicePtr = std::make_shared("tensor_slice"); + FuncGraphPtr tensorSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tensorSlicePtr, 2); + + AbstractBasePtrList eles; + AbstractScalarPtr elem_index = std::make_shared(1); + eles.push_back(elem_index); + elem_index = std::make_shared(3); + eles.push_back(elem_index); + + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({6, 7, 8}); + AbstractTuplePtr slice_tuple = std::make_shared(eles); + AbstractBasePtrList args_spec_list = {tensor, slice_tuple}; + + AbstractTensorPtr ret = dyn_cast(engine_->Run(tensorSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract array failed."; + } + AbstractTensorPtr expect = UTCompositeUtils::ArrayInt32Of({8}); + ASSERT_EQ(*ret, *expect); +} + +TEST_F(TestComposite, test_TensorSliceByScalarTupleToScalar) { + MetaFuncGraphPtr tensorSlicePtr = std::make_shared("tensor_slice"); + FuncGraphPtr tensorSliceGraphPtr = UTCompositeUtils::MakeFuncGraph(tensorSlicePtr, 2); + + AbstractBasePtrList eles; + AbstractScalarPtr elem_index = std::make_shared(3); + eles.push_back(elem_index); + elem_index = std::make_shared(0); + eles.push_back(elem_index); + elem_index = std::make_shared(6); + eles.push_back(elem_index); + + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({6, 7, 8}); + AbstractTuplePtr slice_tuple = std::make_shared(eles); + AbstractBasePtrList args_spec_list = {tensor, slice_tuple}; + + AbstractTensorPtr ret = dyn_cast(engine_->Run(tensorSliceGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract array failed."; + } + AbstractTensorPtr expect = UTCompositeUtils::ArrayInt32Of({}); + ASSERT_EQ(*ret, *expect); +} + +TEST_F(TestComposite, test_UnpackCall_3args) { + MetaFuncGraphPtr unPackCallPtr = std::make_shared("UnPackCall"); + FuncGraphPtr unPackCallGraphPtr = UTCompositeUtils::MakeFuncGraph(unPackCallPtr, 3); + + auto fn_arg= std::make_shared(prim::kPrimMakeTuple); + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + AbstractBasePtrList eles; + for (size_t i = 0; i < 6; i++) { + eles.push_back(tensor); + } + AbstractTuplePtr tensor_tuple = std::make_shared(eles); + AbstractTensorPtr arr_x = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + AbstractTensorPtr arr_y = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + AbstractTensorPtr arr_z = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + std::vector tensor_map{{"x", arr_x}, {"y", arr_y}, {"z", arr_z}}; + abstract::AbstractDictionaryPtr tensor_dict = std::make_shared(tensor_map); + + AbstractBasePtrList args_spec_list = {fn_arg, tensor_tuple, tensor_dict}; + AbstractTuplePtr ret = dyn_cast(engine_->Run(unPackCallGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 9; + ASSERT_EQ(real, expect); +} + +TEST_F(TestComposite, test_UnpackCall_5args) { + MetaFuncGraphPtr unPackCallPtr = std::make_shared("UnPackCall"); + FuncGraphPtr unPackCallGraphPtr = UTCompositeUtils::MakeFuncGraph(unPackCallPtr, 5); + + auto fn_arg = std::make_shared(prim::kPrimMakeTuple); + AbstractTensorPtr tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + AbstractBasePtrList eles; + for (size_t i = 0; i < 6; i++) { + eles.push_back(tensor); + } + AbstractTuplePtr tensor_tuple = std::make_shared(eles); + AbstractTensorPtr arr_x = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + AbstractTensorPtr arr_y = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + AbstractTensorPtr arr_z = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + std::vector tensor_map{{"x", arr_x}, {"y", arr_y}, {"z", arr_z}}; + abstract::AbstractDictionaryPtr tensor_dict = std::make_shared(tensor_map); + + AbstractBasePtrList args_spec_list = {fn_arg, tensor_dict, tensor_tuple, tensor_dict, tensor_tuple}; + AbstractTuplePtr ret = dyn_cast(engine_->Run(unPackCallGraphPtr, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 18; + ASSERT_EQ(real, expect); +} + +TEST_F(TestComposite, test_ZipOperation) { + MetaFuncGraphPtr zip_op = std::make_shared("zip_op"); + FuncGraphPtr zip_op_graph = UTCompositeUtils::MakeFuncGraph(zip_op, 1); + + AbstractBasePtrList eles; + auto tensor = UTCompositeUtils::ArrayInt32Of({2, 3, 4}); + size_t tuple_size = 3; + for (size_t i = 0; i < tuple_size; i++) { + eles.push_back(tensor); + } + auto tuple = std::make_shared(eles); + AbstractBasePtrList args_spec_list = {tuple}; + + AbstractTuplePtr ret = dyn_cast(engine_->Run(zip_op_graph, args_spec_list).inferred); + if (ret == nullptr) { + FAIL() << "Cast ret to abstract tuple failed."; + } + size_t real = ret->size(); + size_t expect = 3; + ASSERT_EQ(real, expect); +} +} // namespace mindspore diff --git a/tests/ut/cpp/operator/grad_implementations_test.cc b/tests/ut/cpp/operator/grad_implementations_test.cc new file mode 100644 index 00000000000..37fb822cf6a --- /dev/null +++ b/tests/ut/cpp/operator/grad_implementations_test.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "ir/value.h" +#include "ir/manager.h" +#include "common/common_test.h" +#include "optimizer/ad/dfunctor.h" +#include "debug/draw.h" +#include "common/py_func_graph_fetcher.h" + +namespace mindspore { +namespace prim { +class TestGradImplementations : public UT::Common { + public: + TestGradImplementations() {} + virtual void SetUp() {} +}; + +TEST_F(TestGradImplementations, TestGetAugmentedGraph) { + FuncGraphPtr fg = ad::g_k_prims.KPrimitive(NewValueNode(kPrimScalarMul), nullptr); + ASSERT_TRUE(fg != nullptr); + draw::Draw("gradImpl_TestGetAugmentedFuncGraph.dot", fg); + + auto fg1 = ad::g_k_prims.KPrimitive(NewValueNode(kPrimScalarMul), nullptr); + ASSERT_TRUE(fg == fg1); +} + +} // namespace prim +} // namespace mindspore diff --git a/tests/ut/cpp/operator/ops_test.cc b/tests/ut/cpp/operator/ops_test.cc new file mode 100644 index 00000000000..bf70fd7b709 --- /dev/null +++ b/tests/ut/cpp/operator/ops_test.cc @@ -0,0 +1,482 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common_test.h" +#include "ir/value.h" +#include "operator/ops.h" +#include "./common.h" + +namespace mindspore { +namespace prim { + +class TestOps : public UT::Common { + public: + TestOps() {} + virtual void SetUp() {} +}; + +// Arithmetic +TEST_F(TestOps, ScalarAddTest) { + auto prim = std::make_shared("scalar_add"); + ASSERT_EQ(prim->name(), kPrimScalarAdd->name()); +} + +TEST_F(TestOps, ScalarSubTest) { + auto prim = std::make_shared("scalar_sub"); + ASSERT_EQ(prim->name(), kPrimScalarSub->name()); +} + +TEST_F(TestOps, ScalarMulTest) { + auto prim = std::make_shared("scalar_mul"); + ASSERT_EQ(prim->name(), kPrimScalarMul->name()); +} + +TEST_F(TestOps, ScalarDivTest) { + auto prim = std::make_shared("scalar_div"); + ASSERT_EQ(prim->name(), kPrimScalarDiv->name()); +} + +TEST_F(TestOps, ScalarModTest) { + auto prim = std::make_shared("scalar_mod"); + ASSERT_EQ(prim->name(), kPrimScalarMod->name()); +} + +TEST_F(TestOps, ScalarPowTest) { + auto prim = std::make_shared("scalar_pow"); + ASSERT_EQ(prim->name(), kPrimScalarPow->name()); +} + +TEST_F(TestOps, ScalarTruncTest) { + auto prim = std::make_shared("scalar_trunc"); + ASSERT_EQ(prim->name(), kPrimScalarTrunc->name()); +} + +TEST_F(TestOps, ScalarFloorTest) { + auto prim = std::make_shared("scalar_floor"); + ASSERT_EQ(prim->name(), kPrimScalarFloor->name()); +} + +TEST_F(TestOps, ScalarUaddTest) { + auto prim = std::make_shared("scalar_uadd"); + ASSERT_EQ(prim->name(), kPrimScalarUadd->name()); +} + +TEST_F(TestOps, ScalarUsubTest) { + auto prim = std::make_shared("scalar_usub"); + ASSERT_EQ(prim->name(), kPrimScalarUsub->name()); +} + +TEST_F(TestOps, ScalarExpTest) { + auto prim = std::make_shared("scalar_exp"); + ASSERT_EQ(prim->name(), kPrimScalarExp->name()); +} + +TEST_F(TestOps, ScalarLogTest) { + auto prim = std::make_shared("scalar_log"); + ASSERT_EQ(prim->name(), kPrimScalarLog->name()); +} + +TEST_F(TestOps, ScalarSinTest) { + auto prim = std::make_shared("scalar_sin"); + ASSERT_EQ(prim->name(), kPrimScalarSin->name()); +} + +TEST_F(TestOps, ScalarCosTest) { + auto prim = std::make_shared("scalar_cos"); + ASSERT_EQ(prim->name(), kPrimScalarCos->name()); +} + +TEST_F(TestOps, ScalarTanTest) { + auto prim = std::make_shared("scalar_tan"); + ASSERT_EQ(prim->name(), kPrimScalarTan->name()); +} + +// Comparisons +TEST_F(TestOps, ScalarEqTest) { + auto prim = std::make_shared("scalar_eq"); + ASSERT_EQ(prim->name(), kPrimScalarEq->name()); +} + +TEST_F(TestOps, ScalarLtTest) { + auto prim = std::make_shared("scalar_lt"); + ASSERT_EQ(prim->name(), kPrimScalarLt->name()); +} + +TEST_F(TestOps, ScalarGtTest) { + auto prim = std::make_shared("scalar_gt"); + ASSERT_EQ(prim->name(), kPrimScalarGt->name()); +} + +TEST_F(TestOps, ScalarNeTest) { + auto prim = std::make_shared("scalar_ne"); + ASSERT_EQ(prim->name(), kPrimScalarNe->name()); +} + +TEST_F(TestOps, ScalarLeTest) { + auto prim = std::make_shared("scalar_le"); + ASSERT_EQ(prim->name(), kPrimScalarLe->name()); +} + +TEST_F(TestOps, ScalarGeTest) { + auto prim = std::make_shared("scalar_ge"); + ASSERT_EQ(prim->name(), kPrimScalarGe->name()); +} + +TEST_F(TestOps, BoolNotTest) { + auto prim = std::make_shared("bool_not"); + ASSERT_EQ(prim->name(), kPrimBoolNot->name()); +} + +TEST_F(TestOps, BoolAndTest) { + auto prim = std::make_shared("bool_and"); + ASSERT_EQ(prim->name(), kPrimBoolAnd->name()); +} + +TEST_F(TestOps, BoolOrTest) { + auto prim = std::make_shared("bool_or"); + ASSERT_EQ(prim->name(), kPrimBoolOr->name()); +} + +TEST_F(TestOps, BoolEqTest) { + auto prim = std::make_shared("bool_eq"); + ASSERT_EQ(prim->name(), kPrimBoolEq->name()); +} + +// Type introspection +TEST_F(TestOps, TypeOfTest) { + auto prim = std::make_shared("typeof"); + ASSERT_EQ(prim->name(), kPrimTypeOf->name()); +} + +TEST_F(TestOps, HasTypeTest) { + auto prim = std::make_shared("hastype"); + ASSERT_EQ(prim->name(), kPrimHasType->name()); +} + +// Data structures +TEST_F(TestOps, MakeTupleTest) { + auto prim = std::make_shared("make_tuple"); + ASSERT_EQ(prim->name(), kPrimMakeTuple->name()); +} + +TEST_F(TestOps, MakeListTest) { + auto prim = std::make_shared("make_list"); + ASSERT_EQ(prim->name(), kPrimMakeList->name()); +} + +TEST_F(TestOps, MakeRecordTest) { + auto prim = std::make_shared("make_record"); + ASSERT_EQ(prim->name(), kPrimMakeRecord->name()); +} + +TEST_F(TestOps, TupleGetItemTest) { + auto prim = std::make_shared("tuple_getitem"); + ASSERT_EQ(prim->name(), kPrimTupleGetItem->name()); +} + +TEST_F(TestOps, ListGetItemTest) { + auto prim = std::make_shared("list_getitem"); + ASSERT_EQ(prim->name(), kPrimListGetItem->name()); +} + +TEST_F(TestOps, ArrayGetItemTest) { + auto prim = std::make_shared("array_getitem"); + ASSERT_EQ(prim->name(), kPrimArrayGetItem->name()); +} + +TEST_F(TestOps, TupleSetItemTest) { + auto prim = std::make_shared("tuple_setitem"); + ASSERT_EQ(prim->name(), kPrimTupleSetItem->name()); +} + +TEST_F(TestOps, ListSetItemTest) { + auto prim = std::make_shared("list_setitem"); + ASSERT_EQ(prim->name(), kPrimListSetItem->name()); +} + +TEST_F(TestOps, ArraySetItemTest) { + auto prim = std::make_shared("array_setitem"); + ASSERT_EQ(prim->name(), kPrimArraySetItem->name()); +} + +TEST_F(TestOps, ListAppendTest) { + auto prim = std::make_shared("list_append"); + ASSERT_EQ(prim->name(), kPrimListAppend->name()); +} + +TEST_F(TestOps, GetAttrTest) { + auto prim = std::make_shared("getattr"); + ASSERT_EQ(prim->name(), kPrimGetAttr->name()); +} + +TEST_F(TestOps, TupleLenTest) { + auto prim = std::make_shared("tuple_len"); + ASSERT_EQ(prim->name(), kPrimTupleLen->name()); +} + +TEST_F(TestOps, ListLenTest) { + auto prim = std::make_shared("list_len"); + ASSERT_EQ(prim->name(), kPrimListLen->name()); +} + +TEST_F(TestOps, ArrayLenTest) { + auto prim = std::make_shared("array_len"); + ASSERT_EQ(prim->name(), kPrimArrayLen->name()); +} + +TEST_F(TestOps, ListMapTest) { + auto prim = std::make_shared("list_map"); + ASSERT_EQ(prim->name(), kPrimListMap->name()); +} + +TEST_F(TestOps, ListReduceTest) { + auto prim = std::make_shared("list_reduce"); + ASSERT_EQ(prim->name(), kPrimListReduce->name()); +} + +// Arrays +TEST_F(TestOps, ScalarToArrayTest) { + auto prim = std::make_shared("scalar_to_array"); + ASSERT_EQ(prim->name(), kPrimScalarToArray->name()); +} + +TEST_F(TestOps, ArrayToScalarTest) { + auto prim = std::make_shared("array_to_scalar"); + ASSERT_EQ(prim->name(), kPrimArrayToScalar->name()); +} + +TEST_F(TestOps, BroadCastShapeTest) { + auto prim = std::make_shared("broadcast_shape"); + ASSERT_EQ(prim->name(), kPrimBroadcastShape->name()); +} + +TEST_F(TestOps, ShapeTest) { + auto prim = std::make_shared("Shape"); + ASSERT_EQ(prim->name(), kPrimShape->name()); +} + +TEST_F(TestOps, ArrayMapTest) { + auto prim = std::make_shared("array_map"); + ASSERT_EQ(prim->name(), kPrimArrayMap->name()); +} + +TEST_F(TestOps, ArrayReduceTest) { + auto prim = std::make_shared("array_reduce"); + ASSERT_EQ(prim->name(), kPrimArrayReduce->name()); +} + +TEST_F(TestOps, DistributeTest) { + auto prim = std::make_shared("distribute"); + ASSERT_EQ(prim->name(), kPrimDistribute->name()); +} + +TEST_F(TestOps, TransposeTest) { + auto prim = std::make_shared("Transpose"); + ASSERT_EQ(prim->name(), kPrimTranspose->name()); +} + +TEST_F(TestOps, DotTest) { + auto prim = std::make_shared("dot"); + ASSERT_EQ(prim->name(), kPrimDot->name()); +} + +TEST_F(TestOps, Im2ColTest) { + auto prim = std::make_shared("im2col"); + ASSERT_EQ(prim->name(), kPrimIm2Col->name()); +} + +TEST_F(TestOps, Col2ImTest) { + auto prim = std::make_shared("col2im"); + ASSERT_EQ(prim->name(), kPrimCol2Im->name()); +} + +TEST_F(TestOps, Im2ColV1Test) { + auto prim = std::make_shared("im2col_v1"); + ASSERT_EQ(prim->name(), kPrimIm2ColV1->name()); +} + +TEST_F(TestOps, Col2ImV1Test) { + auto prim = std::make_shared("col2im_v1"); + ASSERT_EQ(prim->name(), kPrimCol2ImV1->name()); +} + +// Statements +TEST_F(TestOps, SwitchTest) { + auto prim = std::make_shared("switch"); + ASSERT_EQ(prim->name(), kPrimSwitch->name()); +} + +TEST_F(TestOps, ReturnTest) { + auto prim = std::make_shared("return"); + ASSERT_EQ(prim->name(), kPrimReturn->name()); +} + +// Miscellaneous + +TEST_F(TestOps, IdentityTest) { + auto prim = std::make_shared("identity"); + ASSERT_EQ(prim->name(), kPrimIdentity->name()); +} + +TEST_F(TestOps, ResolveTest) { + auto prim = std::make_shared("resolve"); + ASSERT_EQ(prim->name(), kPrimResolve->name()); +} + +TEST_F(TestOps, PartialTest) { + auto prim = std::make_shared("partial"); + ASSERT_EQ(prim->name(), kPrimPartial->name()); +} + +TEST_F(TestOps, JTest) { + auto prim = std::make_shared("J"); + ASSERT_EQ(prim->name(), kPrimJ->name()); +} + +TEST_F(TestOps, EmbedTest) { + auto prim = std::make_shared("embed"); + ASSERT_EQ(prim->name(), kPrimEmbed->name()); +} + +TEST_F(TestOps, EnvSetItemTest) { + auto prim = std::make_shared("env_setitem"); + ASSERT_EQ(prim->name(), kPrimEnvSetItem->name()); +} + +TEST_F(TestOps, EnvGetItemTest) { + auto prim = std::make_shared("env_getitem"); + ASSERT_EQ(prim->name(), kPrimEnvGetItem->name()); +} + +TEST_F(TestOps, EnvAddest) { + auto prim = std::make_shared("env_add"); + ASSERT_EQ(prim->name(), kPrimEnvAdd->name()); +} + +// Neural Network +TEST_F(TestOps, Conv2dTest) { + auto prim = std::make_shared("Conv2D"); + ASSERT_EQ(prim->name(), kPrimConv2D->name()); +} + +TEST_F(TestOps, Conv2dAttrTest) { + Primitive prim("Conv2D"); + prim.SetAttrs({ + {"stride", MakeValue(3)}, + {"pad", MakeValue(1)}, + }); + ASSERT_EQ(prim.name(), kPrimConv2D->name()); + + Int32Imm stride(3); + Int32Imm pad(1); + ASSERT_EQ(*prim.GetAttr("stride"), stride); + ASSERT_EQ(*prim.GetAttr("pad"), pad); +} + +TEST_F(TestOps, CustomOpAttrTest) { + Primitive prim("CustomOp", kPrimTypePyInferShape); + prim.SetAttrs({ + {"attr1", MakeValue(3)}, + {"attr2", MakeValue(1)}, + }); + ASSERT_EQ(prim.name(), std::string("CustomOp")); + ASSERT_EQ(prim.prim_type(), kPrimTypePyInferShape); + + auto attrs = prim.attrs(); + for (auto attr : attrs) { + std::string prim_name = attr.first; + auto prim_value = attr.second; + std::cout << prim_name << std::endl; + std::cout << prim_value << std::endl; + } +} + +TEST_F(TestOps, Conv2dBackpropInputTest) { + auto prim = std::make_shared("Conv2DBackpropInput"); + ASSERT_EQ(prim->name(), kPrimConv2DBackpropInput->name()); +} + +TEST_F(TestOps, Conv2dBackpropFilterTest) { + auto prim = std::make_shared("Conv2DBackpropFilter"); + ASSERT_EQ(prim->name(), kPrimConv2DBackpropFilter->name()); +} + +TEST_F(TestOps, ReluTest) { + auto prim = std::make_shared("ReLU"); + ASSERT_EQ(prim->name(), kPrimRelu->name()); +} + +TEST_F(TestOps, FusedBatchNormTest) { + auto prim = std::make_shared("FusedBatchNorm"); + ASSERT_EQ(prim->name(), kPrimFusedBatchNorm->name()); +} + +TEST_F(TestOps, FusedBatchNormAttrTest) { + Primitive prim("FusedBatchNorm"); + prim.SetAttrs({ + {"epsilon", MakeValue(0.001f)}, + {"momentum", MakeValue(0.1f)}, + }); + ASSERT_EQ(prim.name(), kPrimFusedBatchNorm->name()); + + FP32Imm epsilon(0.001f); + FP32Imm momentum(0.1f); + ASSERT_EQ(*prim.GetAttr("epsilon"), epsilon); + ASSERT_EQ(*prim.GetAttr("momentum"), momentum); +} + +TEST_F(TestOps, PoolingTest) { + auto prim = std::make_shared("Pooling"); + ASSERT_EQ(prim->name(), kPrimPooling->name()); +} + +TEST_F(TestOps, GetConv2DPrimPyTest) { + auto conv2d_prim = prim::GetPythonOps("conv2d_prim", "gtest_input.pynative"); + ASSERT_TRUE(conv2d_prim); + PrimitivePyPtr conv2d_ptr = dyn_cast(conv2d_prim); + ASSERT_TRUE(conv2d_ptr); + if (nullptr != conv2d_ptr) { + MS_LOG(INFO) << "Get PrimitivePyPtr: " << conv2d_ptr->name(); + auto func = conv2d_ptr->GetComputeFunction(); + if (py::isinstance(func)) { + MS_LOG(EXCEPTION) << "" << conv2d_ptr->name() << "'s compute function is not implemented"; + } + + py::object conv2d_pyobj = parse::python_adapter::GetPyFn("gtest_input.pynative", "conv2d_prim"); + py::dict opAttrs = py::getattr(conv2d_pyobj, "attrs"); + std::unordered_map attrs{}; + for (auto item : opAttrs) { + if (!py::isinstance(item.first)) { + MS_LOG(EXCEPTION) << "type error in py dict convert"; + } + std::string name = py::cast(item.first); + MS_LOG(INFO) << "Attr name: " << name; + + ValuePtr converted_ret; + parse::ConvertData(py::cast(item.second), &converted_ret); + MS_LOG(INFO) << "Attr value: " << converted_ret->ToString(); + attrs.emplace(name, converted_ret); + } + } + + MS_LOG(INFO) << "Finish GetPyFnTest!"; +} + +} // namespace prim +} // namespace mindspore diff --git a/tests/ut/cpp/operator/prim2func_test.cc b/tests/ut/cpp/operator/prim2func_test.cc new file mode 100644 index 00000000000..8f7c73a064e --- /dev/null +++ b/tests/ut/cpp/operator/prim2func_test.cc @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common_test.h" + +#include "ir/anf.h" +#include "ir/dtype.h" +#include "operator/prim_to_function.h" + +namespace mindspore { +namespace prim { + +class TestPrimFunc : public UT::Common { + public: + TestPrimFunc() {} + virtual void SetUp() {} +}; + +TEST_F(TestPrimFunc, ScalarAddTest) { + auto prim = std::make_shared("scalar_add"); + FunctionPtr func = nullptr; + PrimToFunction::GetInstance().GetFunction(prim, &func); + + std::vector> two_args{std::make_shared(), std::make_shared()}; + std::shared_ptr retval = std::make_shared(); + Function func_add = Function(two_args, retval); + + std::cout << "func_add: " + func_add.ToString() << std::endl; + std::cout << "prim_func: " + func->ToString() << std::endl; + + ASSERT_EQ(func_add.ToString(), func->ToString()); +} + +TEST_F(TestPrimFunc, ScalarExpTest) { + auto prim = std::make_shared("scalar_exp"); + FunctionPtr func = nullptr; + PrimToFunction::GetInstance().GetFunction(prim, &func); + + std::vector> one_arg{std::make_shared()}; + std::shared_ptr retval = std::make_shared(); + Function func_add = Function(one_arg, retval); + + std::cout << "func_exp: " + func_add.ToString() << std::endl; + std::cout << "prim_func: " + func->ToString() << std::endl; + + ASSERT_EQ(func_add.ToString(), func->ToString()); +} + +} // namespace prim +} // namespace mindspore diff --git a/tests/ut/cpp/optimizer/ad/ad_test.cc b/tests/ut/cpp/optimizer/ad/ad_test.cc new file mode 100644 index 00000000000..34612b54744 --- /dev/null +++ b/tests/ut/cpp/optimizer/ad/ad_test.cc @@ -0,0 +1,207 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "optimizer/ad/grad.h" +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "ir/manager.h" +#include "ir/value.h" +#include "ir/func_graph_cloner.h" +#include "utils/log_adapter.h" +#include "utils/graph_utils.h" +#include "pipeline/resource.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" +#include "operator/ops.h" + +namespace mindspore { +namespace ad { +class TestAD : public UT::Common { + public: + TestAD() : getPyFun("gtest_input.optimizer.ad", true) {} + + public: + UT::PyFuncGraphFetcher getPyFun; + pipeline::ResourceBasePtr resourcePtr = std::make_shared(); + + protected: + void AssertExpect(const std::string& testCase) { + FuncGraphPtr g = getPyFun(testCase); + FuncGraphPtr dg = Grad(g, resourcePtr); + AssertExpect(testCase, dg); + } + + void AssertExpect(const std::string& testCase, const FuncGraphPtr& dg) { ASSERT_TRUE(dg != nullptr); } +}; + +TEST_F(TestAD, test_null) { AssertExpect("test_null"); } + +TEST_F(TestAD, test_grad_add) { AssertExpect("test_grad_add"); } + +TEST_F(TestAD, test_grad_expr) { AssertExpect("test_grad_expr"); } + +TEST_F(TestAD, test_constant) { AssertExpect("test_constant"); } + +TEST_F(TestAD, test_dup_args_in_call) { AssertExpect("test_dup_args_in_call"); } + +TEST_F(TestAD, test_quadruple_args_in_call) { AssertExpect("test_quadruple_args_in_call"); } + +TEST_F(TestAD, test_tuples) { AssertExpect("test_tuples"); } + +TEST_F(TestAD, test_hof) { AssertExpect("test_hof"); } + +TEST_F(TestAD, test_more_hof) { AssertExpect("test_more_hof"); } + +TEST_F(TestAD, test_simple_closure) { AssertExpect("test_simple_closure"); } + +TEST_F(TestAD, test_closure) { AssertExpect("test_closure"); } + +TEST_F(TestAD, test_if) { AssertExpect("test_if"); } + +TEST_F(TestAD, test_if2) { AssertExpect("test_if2"); } + +TEST_F(TestAD, test_fact) { AssertExpect("test_fact"); } + +TEST_F(TestAD, test_while) { AssertExpect("test_while"); } + +TEST_F(TestAD, test_while_2) { AssertExpect("test_while_2"); } + +TEST_F(TestAD, test_pow10) { AssertExpect("test_pow10"); } + +TEST_F(TestAD, test_closures_in_tuples) { AssertExpect("test_closures_in_tuples"); } + +TEST_F(TestAD, test_ops_fn) { AssertExpect("test_ops_fn"); } + +TEST_F(TestAD, test_more_closure) { AssertExpect("test_more_closure"); } + +TEST_F(TestAD, test_prim_scalar_add) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarAdd), resourcePtr); + AssertExpect("test_prim_scalar_add", dg); +} + +TEST_F(TestAD, test_prim_scalar_mul) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarMul), resourcePtr); + AssertExpect("test_prim_scalar_mul", dg); +} + +TEST_F(TestAD, test_prim_scalar_sub) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarSub), resourcePtr); + AssertExpect("test_prim_scalar_sub", dg); +} + +TEST_F(TestAD, test_prim_scalar_div) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarDiv), resourcePtr); + AssertExpect("test_prim_scalar_div", dg); +} + +TEST_F(TestAD, test_prim_scalar_pow) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarPow), resourcePtr); + AssertExpect("test_prim_scalar_pow", dg); +} + +TEST_F(TestAD, test_prim_scalar_exp) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarExp), resourcePtr); + AssertExpect("test_prim_scalar_exp", dg); +} + +TEST_F(TestAD, test_prim_scalar_uadd) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarUadd), resourcePtr); + AssertExpect("test_prim_scalar_uadd", dg); +} + +TEST_F(TestAD, test_prim_scalar_usub) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarUsub), resourcePtr); + AssertExpect("test_prim_scalar_usub", dg); +} + +TEST_F(TestAD, test_prim_scalar_gt) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarGt), resourcePtr); + AssertExpect("test_prim_scalar_gt", dg); +} + +TEST_F(TestAD, test_prim_scalar_lt) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarLt), resourcePtr); + AssertExpect("test_prim_scalar_lt", dg); +} + +TEST_F(TestAD, test_prim_scalar_ge) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarGe), resourcePtr); + AssertExpect("test_prim_scalar_ge", dg); +} + +TEST_F(TestAD, test_prim_scalar_le) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarLe), resourcePtr); + AssertExpect("test_prim_scalar_le", dg); +} + +TEST_F(TestAD, test_prim_tuple_getitem) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimTupleGetItem), resourcePtr); + AssertExpect("test_prim_tuple_getitem", dg); +} + +TEST_F(TestAD, test_prim_identity) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimIdentity), resourcePtr); + AssertExpect("test_prim_identity", dg); +} + +TEST_F(TestAD, test_prim_scalar_to_array) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimScalarToArray), resourcePtr); + AssertExpect("test_prim_scalar_to_array", dg); +} + +TEST_F(TestAD, test_prim_array_to_scalar) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimArrayToScalar), resourcePtr); + AssertExpect("test_prim_array_to_scalar", dg); +} + +TEST_F(TestAD, test_prim_dot) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimDot), resourcePtr); + AssertExpect("test_prim_dot", dg); +} + +TEST_F(TestAD, test_prim_distribute) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimDistribute), resourcePtr); + AssertExpect("test_prim_distribute", dg); +} + +TEST_F(TestAD, test_prim_broadcast_shape) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimBroadcastShape), resourcePtr); + AssertExpect("test_prim_broadcast_shape", dg); +} + +TEST_F(TestAD, test_prim_J) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimJ), resourcePtr); + AssertExpect("test_prim_J", dg); +} + +TEST_F(TestAD, test_prim_switch) { + FuncGraphPtr dg = Kprim(NewValueNode(prim::kPrimSwitch), resourcePtr); + AssertExpect("test_prim_switch", dg); +} + +TEST_F(TestAD, test_grad_cache) { + FuncGraphPtr g = getPyFun("test_null"); + FuncGraphPtr dg1 = Grad(g, resourcePtr); + FuncGraphPtr dg2 = Grad(g, resourcePtr); + ASSERT_TRUE(dg1 == dg2); +} + +TEST_F(TestAD, test_constant_output) { AssertExpect("test_constant_output"); } + +} // namespace ad +} // namespace mindspore diff --git a/tests/ut/cpp/optimizer/cconv_test.cc b/tests/ut/cpp/optimizer/cconv_test.cc new file mode 100644 index 00000000000..0b47c78cd32 --- /dev/null +++ b/tests/ut/cpp/optimizer/cconv_test.cc @@ -0,0 +1,144 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/func_graph_cloner.h" +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +void CheckNoFreeVariables(FuncGraphPtr root) { + auto mng = Manage(root); + for (auto &iter : mng->nodes()) { + auto g = iter.first; + auto nodes = iter.second; + if (g == nullptr) { + continue; + } + + ASSERT_TRUE(g->parent() == nullptr); + + for (auto &node : nodes) { + ASSERT_EQ(node->func_graph(), g); + auto cnode = node->cast(); + if (cnode != nullptr) { + for (auto &inp : cnode->inputs()) { + ASSERT_TRUE(inp->func_graph() == nullptr || inp->func_graph() == g); + } + } + } + } +} + +void CheckCconv(FuncGraphPtr g) { + auto mng = Manage(g); + auto new_g = LiftingClone(g); + CheckNoFreeVariables(new_g); +} + +class TestCconv : public UT::Common { + public: + TestCconv() : getPyFun("gtest_input.optimizer.cconv_test") {} + + virtual void SetUp(); + + virtual void TearDown(); + + public: + UT::PyFuncGraphFetcher getPyFun; +}; + +void TestCconv::SetUp() {} + +void TestCconv::TearDown() {} + +TEST_F(TestCconv, TestStraight) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_straight"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestSimpleClosure) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_simple_closure"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestMax) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_max"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestDeepNesting) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_deep_nesting"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestReturnInDoubleWhile) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_return_in_double_while"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestPow10) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_pow10"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestClosureAsSimpleFv) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_closure_as_simple_fv"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestClosureAsFv) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_closure_as_fv"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestClosureAsDoubleFv) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_closure_as_double_fv"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestClosureLiftSameParam) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_closure_lift_same_param"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestClosureAsLoop) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_closure_as_loop"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +TEST_F(TestCconv, TestClosureLiftCNode) { + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("get_test_cconv_fn", "test_closure_lift_cnode"); + ASSERT_TRUE(nullptr != func_graph); + CheckCconv(func_graph); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/optimizer/clean_test.cc b/tests/ut/cpp/optimizer/clean_test.cc new file mode 100644 index 00000000000..c4f393c233c --- /dev/null +++ b/tests/ut/cpp/optimizer/clean_test.cc @@ -0,0 +1,249 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" +#include "optimizer/clean.h" + +namespace mindspore { +namespace opt { +using mindspore::abstract::AbstractAttribute; +using mindspore::abstract::AbstractClass; +using mindspore::abstract::AbstractError; +using mindspore::abstract::AbstractList; +using mindspore::abstract::AbstractScalar; +using mindspore::abstract::AbstractTensor; +using mindspore::abstract::AbstractTuple; + +class TestClean : public UT::Common { + public: + TestClean() : getPyFun("gtest_input.optimizer.clean_test", true) {} + virtual void SetUp(); + virtual void TearDown(); + + public: + UT::PyFuncGraphFetcher getPyFun; + FuncGraphPtr me_graph; +}; + +void TestClean::SetUp() { + // build the func_graph. + me_graph = std::make_shared(); + me_graph->debug_info()->set_name("next"); + + // build the nodes + AnfNodePtr valuenode_next = NewValueNode(std::string("ms_next")); + ParameterPtr parameter = std::make_shared(me_graph); + AbstractBasePtr para_scalar = std::make_shared(0); + AbstractBasePtr para_list = std::make_shared( + AbstractBasePtrList({std::make_shared(kFloat64), std::make_shared(kFloat64)})); + AbstractBasePtrList para_elem{para_scalar, para_list}; + AbstractBasePtr para_tuple = std::make_shared(para_elem); + parameter->set_abstract(para_tuple); + + AbstractBasePtr app_float = std::make_shared(kFloat64); + AbstractBasePtr app_int = std::make_shared(kFloat64); + AbstractBasePtr app_list = std::make_shared( + AbstractBasePtrList({std::make_shared(kFloat64), std::make_shared(kFloat64)})); + AbstractBasePtr app_tuple_inner = std::make_shared(AbstractBasePtrList{app_int, app_list}); + AbstractBasePtr app_tuple = std::make_shared(AbstractBasePtrList{app_float, app_tuple_inner}); + AnfNodePtr cnode_57 = me_graph->NewCNode({valuenode_next, parameter}); + cnode_57->set_abstract(app_tuple); + + AnfNodePtr cnode_67 = me_graph->NewCNode({NewValueNode(prim::kPrimPartial), valuenode_next, parameter}); + cnode_67->set_abstract(app_tuple); + + AnfNodePtr cnode_66 = me_graph->NewCNode({NewValueNode(prim::kPrimScalarAdd), cnode_57, cnode_67}); + cnode_66->set_abstract(app_float); + + AnfNodePtr valuenode_return = NewValueNode(prim::kPrimReturn); + CNodePtr cnode_55 = me_graph->NewCNode({valuenode_return, cnode_66}); + cnode_55->set_abstract(app_tuple); + + me_graph->set_output(cnode_66); + me_graph->set_return(cnode_55); + me_graph->add_parameter(parameter); +} + +void TestClean::TearDown() {} + +TEST_F(TestClean, TestEraseClassGetAttr) { + FuncGraphPtr func_graph; + + func_graph = getPyFun("test_erase_class_fn"); + ASSERT_TRUE(nullptr != func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + int dataclass_count = 0; + + for (auto node : manager->all_nodes()) { + if (IsValueNode(node)) { + dataclass_count++; + } + if (!node->isa()) { + continue; + } + auto input0 = node->cast()->input(0); + if (IsValueNode(input0)) { + std::vector attr = {{"x", std::make_shared(kFloat64)}, + {"y", std::make_shared(kFloat64)}}; + std::unordered_map methods; + AbstractBasePtr abs_ptr = std::make_shared(Named("Point"), attr, methods); + node->set_abstract(abs_ptr); + } + } + + ASSERT_EQ(dataclass_count, 1); + + // draw func_graph before erase class + draw::Draw("opt_before_erase_class.dot", func_graph); + + SimplifyDataStructures(func_graph, manager); + + // draw func_graph after erase class + draw::Draw("opt_after_erase_class.dot", func_graph); + + int tuple_getitem_count = 0; + + for (auto node : manager->all_nodes()) { + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + tuple_getitem_count++; + } + } + + ASSERT_EQ(dataclass_count, 1); + ASSERT_EQ(tuple_getitem_count, 2); +} + +TEST_F(TestClean, TestEraseClassMakeRecord) { + // build the graph + auto func_graph = std::make_shared(); + func_graph->debug_info()->set_name("test_make_record"); + + auto cons_make_record = NewValueNode(prim::kPrimMakeRecord); + auto para1 = std::make_shared(func_graph); + auto para2 = std::make_shared(func_graph); + + para1->set_abstract(std::make_shared(kAnyValue, kInt64)); + para2->set_abstract(std::make_shared(kAnyValue, kInt64)); + std::vector attr = {{"x", std::make_shared(kAnyValue, kInt64)}, + {"y", std::make_shared(kAnyValue, kInt64)}}; + std::unordered_map methods; + AbstractBasePtr abs_ptr = std::make_shared(Named("Point"), attr, methods); + auto cons_class = NewValueNode(abs_ptr->BuildValue()); + cons_class->set_abstract(abs_ptr); + + std::vector inputs{cons_make_record, cons_class, para1, para2}; + auto apply22 = func_graph->NewCNode(inputs); + + auto cons_return = NewValueNode(prim::kPrimReturn); + auto apply11 = func_graph->NewCNode({cons_return, apply22}); + apply11->set_abstract(abs_ptr); + + func_graph->set_output(apply22); + func_graph->set_return(apply11); + func_graph->add_parameter(para1); + func_graph->add_parameter(para2); + + auto manager = Manage(func_graph); + + draw::Draw("opt_erase_class_record_before.dot", func_graph); + + SimplifyDataStructures(func_graph, manager); + + draw::Draw("opt_erase_class_record_after.dot", func_graph); +} + +TEST_F(TestClean, TestEraseClassPartial) { + // build the graph + auto func_graph = std::make_shared(); + func_graph->debug_info()->set_name("test_partial"); + + auto cons_partial = NewValueNode(prim::kPrimPartial); + auto para1 = std::make_shared(func_graph); + para1->set_abstract(std::make_shared(kAnyValue, kInt64)); + + auto cons_make_record = NewValueNode(prim::kPrimMakeRecord); + + std::vector attr = {{"x", std::make_shared(kAnyValue, kInt64)}, + {"y", std::make_shared(kAnyValue, kInt64)}}; + std::unordered_map methods; + AbstractBasePtr abs_ptr = std::make_shared(Named("Point"), attr, methods); + auto cons_class = NewValueNode(abs_ptr->BuildValue()); + cons_class->set_abstract(abs_ptr); + + std::vector inputs{cons_partial, cons_make_record, cons_class, para1}; + auto apply22 = func_graph->NewCNode(inputs); + std::vector inputs_nopara{cons_partial, cons_make_record, cons_class}; + auto apply33 = func_graph->NewCNode(inputs_nopara); + + auto apply11 = func_graph->NewCNode({NewValueNode(prim::kPrimScalarAdd), apply22, apply33}); + + auto cons_return = NewValueNode(prim::kPrimReturn); + auto apply00 = func_graph->NewCNode({cons_return, apply11}); + apply00->set_abstract(abs_ptr); + + func_graph->set_output(apply22); + func_graph->set_return(apply11); + func_graph->add_parameter(para1); + + auto manager = Manage(func_graph); + + draw::Draw("opt_erase_class_partial_before.dot", func_graph); + SimplifyDataStructures(func_graph, manager); + draw::Draw("opt_erase_class_partial_after.dot", func_graph); +} + +TEST_F(TestClean, TestEraseTuple) { + ASSERT_TRUE(nullptr != me_graph); + std::shared_ptr manager = Manage(me_graph); + + draw::Draw("opt_before_erase_tuple.dot", me_graph); + + int abstract_tuple_count = 0; + + for (auto node : manager->all_nodes()) { + auto dt = node->abstract(); + if (dyn_cast(dt) != nullptr) { + abstract_tuple_count++; + } + } + ASSERT_EQ(abstract_tuple_count, 4); + + // erase tuple in CNode57 and Parameter + EraseTuple(me_graph, manager); + + abstract_tuple_count = 0; + for (auto node : manager->all_nodes()) { + auto dt = node->abstract(); + if (dyn_cast(dt) != nullptr) { + abstract_tuple_count++; + } + } + + ASSERT_EQ(abstract_tuple_count, 3); + + draw::Draw("opt_after_erase_tuple.dot", me_graph); +} + +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/optimizer/lib_test.cc b/tests/ut/cpp/optimizer/lib_test.cc new file mode 100644 index 00000000000..ff3c00d37a2 --- /dev/null +++ b/tests/ut/cpp/optimizer/lib_test.cc @@ -0,0 +1,547 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" + +#include "common/py_func_graph_fetcher.h" +#include "ir/anf.h" +#include "ir/func_graph_cloner.h" +#include "ir/manager.h" +#include "ir/visitor.h" +#include "optimizer/irpass.h" +#include "pipeline/resource.h" +#include "debug/draw.h" +#include "pipeline/parse/data_converter.h" + +namespace mindspore { +namespace opt { +using abstract::AnalysisResult; + +class TestOptLib : public UT::Common { + public: + TestOptLib() : getPyFun("gtest_input.optimizer.opt_test", true), irpass() {} + void SetUp() { + UT::InitPythonPath(); + parse::data_converter::ClearObjectCache(); + } + FuncGraphPtr RunTransform(FuncGraphPtr gbefore, const SubstitutionList &transform) { + equiv_node.clear(); + equiv_graph.clear(); + + FuncGraphPtr gbefore_clone = BasicClone(gbefore); + OptimizerPtr optimizer = std::make_shared("ut_test", std::make_shared()); + transform(gbefore_clone, optimizer); + return gbefore_clone; + } + FuncGraphPtr RunSubs(FuncGraphPtr before, std::vector opts = {}) { + SubstitutionList eq(opts); + return RunTransform(before, eq); + } + bool CheckTransform(FuncGraphPtr gbefore, FuncGraphPtr gafter, const SubstitutionList &transform, + bool save_graphs = false) { + equiv_node.clear(); + equiv_graph.clear(); + + FuncGraphPtr gbefore_clone = BasicClone(gbefore); + OptimizerPtr optimizer = std::make_shared("ut_test", std::make_shared()); + transform(gbefore_clone, optimizer); + if (save_graphs) { + draw::Draw("before.dot", gbefore); + draw::Draw("after.dot", gbefore_clone); + draw::Draw("expected.dot", gafter); + } + return Isomorphic(gbefore_clone, gafter, &equiv_graph, &equiv_node); + } + bool CheckOpt(FuncGraphPtr before, FuncGraphPtr after, std::vector opts = {}, + bool save_graphs = false) { + if (nullptr == before || nullptr == after) { + return false; + } + SubstitutionList eq(opts); + return CheckTransform(before, after, eq, save_graphs); + } + + public: + UT::PyFuncGraphFetcher getPyFun; + FuncGraphPairMapEquiv equiv_graph; + NodeMapEquiv equiv_node; + irpass::OptimizeIRPassLib irpass; +}; + +TEST_F(TestOptLib, test_expendJ) { + FuncGraphPtr before = getPyFun("test_expendJ"); + + ASSERT_TRUE(nullptr != before); + + FuncGraphPtr after = RunSubs(before, std::vector({irpass.expand_jprim_})); +} + +TEST_F(TestOptLib, test_simplify_always_true_false) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_simplify_always_true_false", "before_1"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_simplify_always_true_false", "before_2"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_simplify_always_true_false", "after"); + auto patterns = std::vector({irpass.switch_simplify_}); + ASSERT_TRUE(CheckOpt(before1, after, patterns)); + ASSERT_TRUE(CheckOpt(before2, after, patterns)); +} + +TEST_F(TestOptLib, test_inline) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_inline", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_inline", "after"); + // add infer and renormalize + std::shared_ptr res = std::make_shared(); + AbstractBasePtrList args_spec_list; + AbstractBasePtr abstract_v1 = abstract::FromValue(1, true); + AbstractBasePtr abstract_v2 = abstract::FromValue(2, true); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + AnalysisResult result = pipeline::AbstractAnalyze(res, before1, args_spec_list); + FuncGraphPtr new_graph = pipeline::ProgramSpecialize(res, before1, result.context); + auto patterns = std::vector({irpass.arithmetic_simplify_, irpass.switch_simplify_, irpass.inline_}); + ASSERT_TRUE(CheckOpt(new_graph, after, patterns)); +} + +TEST_F(TestOptLib, test_inline_successively) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_inline_successively", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_inline_successively", "after"); + auto patterns = std::vector({irpass.inline_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_inline_closure) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_inline_closure", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_inline_closure", "after"); + auto patterns = std::vector({irpass.inline_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_inline_deep_closure) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_inline_deep_closure", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_inline_deep_closure", "after"); + auto patterns = std::vector({irpass.inline_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_inline_new_closure) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_inline_new_closure", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_inline_new_closure", "after"); + auto patterns = std::vector({irpass.inline_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_inline_while) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_inline_while", "before"); + auto patterns = std::vector({irpass.inline_}); + FuncGraphPtr after_ = RunSubs(before, patterns); + ASSERT_TRUE(CheckOpt(before, before, patterns)); +} + +TEST_F(TestOptLib, test_arithmetic) { + FuncGraphPtr b1_0 = getPyFun.CallAndParseRet("test_arithmetic", "multiply_by_zero_l"); + FuncGraphPtr b2_0 = getPyFun.CallAndParseRet("test_arithmetic", "multiply_by_zero_r"); + FuncGraphPtr b1 = getPyFun.CallAndParseRet("test_arithmetic", "multiply_by_one_l"); + FuncGraphPtr b2 = getPyFun.CallAndParseRet("test_arithmetic", "multiply_by_one_r"); + FuncGraphPtr b3 = getPyFun.CallAndParseRet("test_arithmetic", "add_zero_l"); + FuncGraphPtr b4 = getPyFun.CallAndParseRet("test_arithmetic", "add_zero_r"); + FuncGraphPtr b5 = getPyFun.CallAndParseRet("test_arithmetic", "elim_identity"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_arithmetic", "after"); + FuncGraphPtr after_0 = getPyFun.CallAndParseRet("test_arithmetic", "after_0"); + + auto patterns = std::vector({irpass.arithmetic_simplify_}); + + ASSERT_TRUE(CheckOpt(b1_0, after_0, patterns)); + ASSERT_TRUE(CheckOpt(b2_0, after_0, patterns)); + ASSERT_TRUE(CheckOpt(b1, after, patterns)); + ASSERT_TRUE(CheckOpt(b2, after, patterns)); + ASSERT_TRUE(CheckOpt(b3, after, patterns)); + ASSERT_TRUE(CheckOpt(b4, after, patterns)); + ASSERT_TRUE(CheckOpt(b5, after, patterns)); +} + +TEST_F(TestOptLib, test_elim_cast_same_dtype) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elim_cast_same_dtype", "fp32_cast_fp32"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elim_cast_same_dtype", "after"); + // construct such case that cast srcT equal dstT + auto &inputs = before->output()->cast()->inputs(); + if (inputs.size() > 2) { + auto cast_node = inputs[0]; + auto cast_py = cast_node->cast()->value()->cast(); + cast_py->set_attr("SrcT", TypeIdToType(kNumberTypeFloat32)); + cast_py->set_attr("DstT", TypeIdToType(kNumberTypeFloat32)); + + auto x_node = inputs[1]; + std::vector shp = {2, 3}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + x_node->set_abstract(x_abstract); + + TypePtr t = std::make_shared(std::make_shared(32)); + ValueNodePtr val = std::make_shared(t); + auto t_abstract = t->ToAbstract(); + val->set_abstract(t_abstract); + before->output()->cast()->set_input(2, val); + } + FuncGraphPtr gbefore_clone = BasicClone(before); + auto patterns = std::vector({irpass.cast_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); + + TypePtr t = std::make_shared(32); + ValueNodePtr val = std::make_shared(t); + auto t_abstract = t->ToAbstract(); + val->set_abstract(t_abstract); + gbefore_clone->output()->cast()->set_input(2, val); + ASSERT_TRUE(CheckOpt(gbefore_clone, after, patterns)); +} + +TEST_F(TestOptLib, test_elim_reshape_same_shape) { + FuncGraphPtr before = getPyFun.CallAndParseRet("elim_reshape_same_shape", "reshape_to_2_3"); + FuncGraphPtr after = getPyFun.CallAndParseRet("elim_reshape_same_shape", "after"); + // construct such case that shape is equal to reshape target + auto &inputs = before->output()->cast()->inputs(); + if (inputs.size() > 1) { + auto x_node = inputs[1]; + std::vector shp = {2, 3}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + x_node->set_abstract(x_abstract); + } + auto patterns = std::vector({irpass.reshape_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); + if (inputs.size() > 1) { + auto x_node = inputs[1]; + std::vector shp = {3, 2}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + x_node->set_abstract(x_abstract); + } + ASSERT_FALSE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, elim_two_reshape) { + FuncGraphPtr before = getPyFun.CallAndParseRet("elim_two_reshape", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("elim_two_reshape", "after"); + + auto patterns = std::vector({irpass.reshape_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, elim_two_cast) { + FuncGraphPtr before = getPyFun.CallAndParseRet("elim_two_cast", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("elim_two_cast", "after"); + + auto patterns = std::vector({irpass.cast_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_elim_transpose) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elim_transpose", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elim_transpose", "after"); + + auto patterns = std::vector({irpass.transpose_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_elim_tile_multiply_one) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elim_tile_multiply_one", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elim_tile_multiply_one", "after"); + + auto patterns = std::vector({irpass.tile_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns, true)); +} + +TEST_F(TestOptLib, test_elim_reduce_mean_shape_one) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elim_reduce_mean_shape_one", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elim_reduce_mean_shape_one", "after"); + + // construct such case that input x shape is (1), keepdims is true + auto inputs = before->output()->cast()->inputs(); + if (inputs.size() > 2) { + auto x_node = inputs[1]; + std::vector shp = {1}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + x_node->set_abstract(x_abstract); + + auto reduce_node = inputs[0]; + auto reduce = reduce_node->cast()->value()->cast(); + reduce->set_attr("keep_dims", std::make_shared(true)); + } + + auto patterns = std::vector({irpass.reduce_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_elim_all_shape_one) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elim_all_shape_one", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elim_all_shape_one", "after"); + + // construct such case that input x shape is (1) keep_dims is true + auto inputs = before->output()->cast()->inputs(); + if (inputs.size() > 2) { + auto x_node = inputs[1]; + std::vector shp = {1}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + x_node->set_abstract(x_abstract); + + auto reduce_node = inputs[0]; + auto reduce = reduce_node->cast()->value()->cast(); + reduce->set_attr("keep_dims", std::make_shared(true)); + } + auto patterns = std::vector({irpass.reduce_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_elim_sum_shape_one) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elim_sum_shape_one", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elim_sum_shape_one", "after"); + + // construct such case that input x shape is (1) keepdims is true + auto inputs = before->output()->cast()->inputs(); + if (inputs.size() > 2) { + auto x_node = inputs[1]; + std::vector shp = {1}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + x_node->set_abstract(x_abstract); + + auto reduce_node = inputs[0]; + auto reduce = reduce_node->cast()->value()->cast(); + reduce->set_attr("keep_dims", std::make_shared(true)); + } + auto patterns = std::vector({irpass.reduce_eliminate_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_tuple_getitem) { + FuncGraphPtr make_get_0 = getPyFun.CallAndParseRet("test_tuple_getitem", "make_get_0"); + FuncGraphPtr make_get_1 = getPyFun.CallAndParseRet("test_tuple_getitem", "make_get_1"); + FuncGraphPtr after_0 = getPyFun.CallAndParseRet("test_tuple_getitem", "after_0"); + FuncGraphPtr after_1 = getPyFun.CallAndParseRet("test_tuple_getitem", "after_1"); + + auto patterns = std::vector({irpass.item_tuple_eliminate_}); + ASSERT_TRUE(CheckOpt(make_get_0, after_0, patterns)); + ASSERT_TRUE(CheckOpt(make_get_1, after_1, patterns)); +} + +TEST_F(TestOptLib, test_tuple_setitem) { + FuncGraphPtr before_0 = getPyFun.CallAndParseRet("test_tuple_setitem", "before_0"); + FuncGraphPtr before_1 = getPyFun.CallAndParseRet("test_tuple_setitem", "before_1"); + FuncGraphPtr after_0 = getPyFun.CallAndParseRet("test_tuple_setitem", "after_0"); + FuncGraphPtr after_1 = getPyFun.CallAndParseRet("test_tuple_setitem", "after_1"); + + auto patterns = std::vector({irpass.item_tuple_eliminate_}); + + ASSERT_TRUE(CheckOpt(before_0, after_0, patterns)); + ASSERT_TRUE(CheckOpt(before_1, after_1, patterns)); +} + +TEST_F(TestOptLib, test_tuple_get_set_item) { + FuncGraphPtr before_0 = getPyFun.CallAndParseRet("test_tuple_get_set_item", "before_0"); + FuncGraphPtr after_0 = getPyFun.CallAndParseRet("test_tuple_get_set_item", "after_0"); + FuncGraphPtr before_1 = getPyFun.CallAndParseRet("test_tuple_get_set_item", "before_0"); + FuncGraphPtr after_1 = getPyFun.CallAndParseRet("test_tuple_get_set_item", "after_0"); + + auto patterns = std::vector({irpass.item_tuple_eliminate_}); + + ASSERT_TRUE(CheckOpt(before_0, after_0, patterns)); + ASSERT_TRUE(CheckOpt(before_1, after_1, patterns)); +} + +TEST_F(TestOptLib, test_partial) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_partial", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_partial", "after"); + + auto patterns = std::vector({irpass.partial_eliminate_}); + + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_replace_applicator) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_replace_applicator", "before1"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_replace_applicator", "before2"); + FuncGraphPtr before3 = getPyFun.CallAndParseRet("test_replace_applicator", "before3"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_replace_applicator", "after"); + + auto patterns = std::vector({irpass.replace_applicator_}); + + ASSERT_TRUE(CheckOpt(before1, after, patterns)); + ASSERT_TRUE(CheckOpt(before2, after, patterns)); + ASSERT_TRUE(CheckOpt(before3, before3, patterns)); +} + +TEST_F(TestOptLib, test_specialize_on_graph_arguments) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_specialize_on_graph_arguments", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_specialize_on_graph_arguments", "after"); + + auto patterns = std::vector({irpass.specialize_transform_}); + + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_incorporate_getitem) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_incorporate_getitem", "before1"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_incorporate_getitem", "before2"); + FuncGraphPtr after1 = getPyFun.CallAndParseRet("test_incorporate_getitem", "after1"); + FuncGraphPtr after2 = getPyFun.CallAndParseRet("test_incorporate_getitem", "after2"); + + auto patterns = std::vector({irpass.incorporate_getitem_}); + + ASSERT_TRUE(CheckOpt(before1, after1, patterns)); + ASSERT_TRUE(CheckOpt(before2, after2, patterns)); +} + +TEST_F(TestOptLib, test_incorporate_getitem_through_switch) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_incorporate_getitem_through_switch", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_incorporate_getitem_through_switch", "after"); + + auto patterns = std::vector({irpass.incorporate_getitem_switch_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_incorporate_call) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_incorporate_call", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_incorporate_call", "after"); + + auto patterns = std::vector({irpass.incorporate_call_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_incorporate_call_through_switch) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_incorporate_call_through_switch", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_incorporate_call_through_switch", "after"); + auto patterns = std::vector({ + irpass.incorporate_call_switch_, + irpass.incorporate_call_, + irpass.arithmetic_simplify_, + }); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_float_tuple_getitem_through_switch) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_float_tuple_getitem_through_switch", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_float_tuple_getitem_through_switch", "after"); + + auto patterns = std::vector({irpass.float_tuple_getitem_switch_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_merge_addn) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_merge_addn", "before"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_merge_addn", "after"); + + auto patterns = std::vector({irpass.merge_addn_}); + ASSERT_TRUE(CheckOpt(before, after, patterns)); +} + +TEST_F(TestOptLib, test_filter_addn_zero) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_addn_zero", "before_1"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_addn_zero", "after"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_addn_zero", "before_2"); + FuncGraphPtr before3 = getPyFun.CallAndParseRet("test_addn_zero", "before_3"); + FuncGraphPtr before4 = getPyFun.CallAndParseRet("test_addn_zero", "before_4"); + auto patterns = std::vector({irpass.addn_zero_filter_}); + ASSERT_TRUE(CheckOpt(before1, after, patterns)); + ASSERT_TRUE(CheckOpt(before2, after, patterns)); + ASSERT_TRUE(CheckOpt(before3, after, patterns)); + ASSERT_TRUE(CheckOpt(before4, before4, patterns)); +} + +TEST_F(TestOptLib, test_minmax_grad) { + FuncGraphPtr before11 = getPyFun.CallAndParseRet("test_minmax_grad", "before_11"); + FuncGraphPtr before12 = getPyFun.CallAndParseRet("test_minmax_grad", "before_12"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_minmax_grad", "before_2"); + FuncGraphPtr before31 = getPyFun.CallAndParseRet("test_minmax_grad", "before_31"); + FuncGraphPtr before32 = getPyFun.CallAndParseRet("test_minmax_grad", "before_32"); + FuncGraphPtr before4 = getPyFun.CallAndParseRet("test_minmax_grad", "before_4"); + auto patterns = std::vector({irpass.minmaximum_grad_}); + ASSERT_TRUE(CheckOpt(before11, before11, patterns)); + ASSERT_TRUE(CheckOpt(before12, before12, patterns)); + ASSERT_TRUE(CheckOpt(before2, before2, patterns)); + ASSERT_TRUE(CheckOpt(before31, before31, patterns)); + ASSERT_TRUE(CheckOpt(before32, before32, patterns)); + ASSERT_TRUE(CheckOpt(before4, before4, patterns)); +} + +TEST_F(TestOptLib, test_reducesum_one) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_reducesum_one", "before_1"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_reducesum_one", "before_2"); + FuncGraphPtr before3 = getPyFun.CallAndParseRet("test_reducesum_one", "before_3"); + FuncGraphPtr before4 = getPyFun.CallAndParseRet("test_reducesum_one", "before_4"); + FuncGraphPtr after1 = getPyFun.CallAndParseRet("test_reducesum_one", "after_1"); + FuncGraphPtr after2 = getPyFun.CallAndParseRet("test_reducesum_one", "after_2"); + FuncGraphPtr after3 = getPyFun.CallAndParseRet("test_reducesum_one", "after_3"); + auto patterns = std::vector({irpass.reduce_eliminate_}); + + std::vector shp = {3, 2, 2, 1}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + + std::vector shp2 = {3, 2, 1, 1}; + tensor::TensorPtr x_tensor2 = std::make_shared(kFloat32->type_id(), shp2); + auto x_abstract2 = x_tensor2->ToAbstract(); + + auto inputs = before1->output()->cast()->inputs(); + if (inputs.size() > 1) { + auto x_node = inputs[1]; + x_node->set_abstract(x_abstract); + } + ASSERT_TRUE(CheckOpt(before1, after1, patterns)); + + auto inputs2 = before2->output()->cast()->inputs(); + if (inputs2.size() > 1) { + auto x_node2 = inputs2[1]; + x_node2->set_abstract(x_abstract2); + } + ASSERT_TRUE(CheckOpt(before2, after1, patterns)); + + auto inputs3 = before2->output()->cast()->inputs(); + if (inputs3.size() > 1) { + auto x_node3 = inputs3[1]; + x_node3->set_abstract(x_abstract); + } + ASSERT_TRUE(CheckOpt(before2, before2, patterns)); + + auto inputs4 = before3->output()->cast()->inputs(); + if (inputs4.size() > 1) { + auto x_node4 = inputs4[1]; + x_node4->set_abstract(x_abstract); + } + ASSERT_TRUE(CheckOpt(before3, after2, patterns)); + + auto inputs5 = before4->output()->cast()->inputs(); + if (inputs5.size() > 1) { + auto x_node5 = inputs5[1]; + x_node5->set_abstract(x_abstract2); + } + ASSERT_TRUE(CheckOpt(before4, after3, patterns)); +} + +TEST_F(TestOptLib, test_print_tuple_wrapper) { + FuncGraphPtr before1 = getPyFun.CallAndParseRet("test_print_tuple_wrapper", "before1"); + FuncGraphPtr before2 = getPyFun.CallAndParseRet("test_print_tuple_wrapper", "before2"); + FuncGraphPtr before3 = getPyFun.CallAndParseRet("test_print_tuple_wrapper", "before3"); + FuncGraphPtr after1 = getPyFun.CallAndParseRet("test_print_tuple_wrapper", "after1"); + FuncGraphPtr after2 = getPyFun.CallAndParseRet("test_print_tuple_wrapper", "after2"); + auto patterns = std::vector({irpass.print_tuple_wrapper_}); + ASSERT_TRUE(CheckOpt(before1, after1, patterns)); + ASSERT_TRUE(CheckOpt(before2, after2, patterns)); + ASSERT_TRUE(CheckOpt(before3, before3, patterns)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/optimizer/opt_test.cc b/tests/ut/cpp/optimizer/opt_test.cc new file mode 100644 index 00000000000..05e7e6b9788 --- /dev/null +++ b/tests/ut/cpp/optimizer/opt_test.cc @@ -0,0 +1,215 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/anf.h" +#include "ir/visitor.h" +#include "ir/func_graph_cloner.h" +#include "optimizer/opt.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/arithmetic_simplify.h" + +#include "debug/draw.h" +#include "operator/ops.h" +#include "optimizer/cse.h" + +namespace mindspore { +namespace opt { +class TestOptOpt : public UT::Common { + public: + TestOptOpt() : getPyFun("gtest_input.optimizer.opt_test", true) {} + + class IdempotentEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + x_ = nullptr; + AnfVisitor::Match(P, {irpass::IsCNode})(node); + if (x_ == nullptr || node->func_graph() == nullptr) { + return nullptr; + } + + return node->func_graph()->NewCNode({NewValueNode(P), x_}); + }; + + void Visit(const CNodePtr &cnode) override { + if (IsPrimitiveCNode(cnode, P) && cnode->inputs().size() == 2) { + x_ = cnode->input(1); + } + } + + private: + AnfNodePtr x_{nullptr}; + }; + + class QctToP : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + v_ = nullptr; + AnfVisitor::Match(Q, {irpass::IsVNode})(node); + if (v_ == nullptr || node->func_graph() == nullptr) { + return nullptr; + } + + return node->func_graph()->NewCNode({NewValueNode(P), v_}); + }; + + void Visit(const ValueNodePtr &vnode) override { v_ = vnode; } + + private: + AnfNodePtr v_{nullptr}; + }; + + void SetUp() { + elim_Z = MakeSubstitution(irpass::AddByZero(), "elim_Z", prim::kPrimScalarAdd); + elim_R = MakeSubstitution(irpass::PrimEliminater(R), "elim_R", R); + idempotent_P = MakeSubstitution(IdempotentEliminater(), "idempotent_P", P); + Qct_to_P = MakeSubstitution(QctToP(), "Qct_to_P", Q); + } + + bool CheckTransform(FuncGraphPtr gbefore, FuncGraphPtr gafter, const SubstitutionList &transform) { + equiv_node.clear(); + equiv_graph.clear(); + + FuncGraphPtr gbefore_clone = BasicClone(gbefore); + OptimizerPtr optimizer = std::make_shared("ut_test", std::make_shared()); + transform(gbefore_clone, optimizer); + + return Isomorphic(gbefore_clone, gafter, &equiv_graph, &equiv_node); + } + + bool CheckOpt(FuncGraphPtr before, FuncGraphPtr after, std::vector opts = {}) { + SubstitutionList eq(opts); + return CheckTransform(before, after, eq); + } + + public: + UT::PyFuncGraphFetcher getPyFun; + + FuncGraphPairMapEquiv equiv_graph; + NodeMapEquiv equiv_node; + + static const PrimitivePtr P; + static const PrimitivePtr Q; + static const PrimitivePtr R; + + SubstitutionPtr elim_Z; + SubstitutionPtr elim_R; + SubstitutionPtr idempotent_P; + SubstitutionPtr Qct_to_P; +}; + +const PrimitivePtr TestOptOpt::P = std::make_shared("P"); +const PrimitivePtr TestOptOpt::Q = std::make_shared("Q"); +const PrimitivePtr TestOptOpt::R = std::make_shared("R"); + +TEST_F(TestOptOpt, TestCheckOptIsClone) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_add_zero", "before_1"); + + ASSERT_TRUE(nullptr != before); + ASSERT_TRUE(CheckOpt(before, before)); + ASSERT_FALSE(CheckOpt(before, before, std::vector({elim_Z}))); +} + +TEST_F(TestOptOpt, Elim) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_add_zero", "before_1"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_add_zero", "after"); + + ASSERT_TRUE(nullptr != before); + ASSERT_TRUE(nullptr != after); + ASSERT_TRUE(CheckOpt(before, after, std::vector({elim_Z}))); +} + +TEST_F(TestOptOpt, ElimTwo) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_add_zero", "before_2"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_add_zero", "after"); + + ASSERT_TRUE(nullptr != before); + ASSERT_TRUE(nullptr != after); + ASSERT_TRUE(CheckOpt(before, after, std::vector({elim_Z}))); +} + +TEST_F(TestOptOpt, ElimR) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_elimR", "before_1"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_elimR", "after"); + + ASSERT_TRUE(nullptr != before); + ASSERT_TRUE(nullptr != after); + ASSERT_TRUE(CheckOpt(before, after, std::vector({elim_R}))); +} + +TEST_F(TestOptOpt, idempotent) { + FuncGraphPtr before_2 = getPyFun.CallAndParseRet("test_idempotent", "before_2"); + FuncGraphPtr before_1 = getPyFun.CallAndParseRet("test_idempotent", "before_1"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_idempotent", "after"); + + ASSERT_TRUE(nullptr != before_2); + ASSERT_TRUE(nullptr != before_1); + ASSERT_TRUE(nullptr != after); + + ASSERT_TRUE(CheckOpt(before_1, after, std::vector({idempotent_P}))); + ASSERT_TRUE(CheckOpt(before_2, after, std::vector({idempotent_P}))); +} + +TEST_F(TestOptOpt, ConstantVariable) { + FuncGraphPtr before = getPyFun.CallAndParseRet("test_constant_variable", "before_1"); + FuncGraphPtr after = getPyFun.CallAndParseRet("test_constant_variable", "after"); + + ASSERT_TRUE(nullptr != before); + ASSERT_TRUE(nullptr != after); + ASSERT_TRUE(CheckOpt(before, after, std::vector({Qct_to_P}))); +} + +TEST_F(TestOptOpt, CSE) { + // test a simple cse testcase test_f1 + FuncGraphPtr test_graph1 = getPyFun.CallAndParseRet("test_cse", "test_f1"); + + ASSERT_TRUE(nullptr != test_graph1); + + // add func_graph the GraphManager + FuncGraphManagerPtr manager1 = Manage(test_graph1); + draw::Draw("opt_cse_before_1.dot", test_graph1); + + ASSERT_EQ(manager1->all_nodes().size(), 10); + + auto cse = std::make_shared(); + ASSERT_TRUE(cse != nullptr); + bool is_changed = cse->Cse(test_graph1, manager1); + + ASSERT_TRUE(is_changed); + ASSERT_EQ(manager1->all_nodes().size(), 8); + + draw::Draw("opt_cse_after_1.dot", test_graph1); + + // test a more complicated case test_f2 + FuncGraphPtr test_graph2 = getPyFun.CallAndParseRet("test_cse", "test_f2"); + + ASSERT_TRUE(nullptr != test_graph2); + + FuncGraphManagerPtr manager2 = Manage(test_graph2); + draw::Draw("opt_cse_before_2.dot", test_graph2); + ASSERT_EQ(manager2->all_nodes().size(), 22); + is_changed = cse->Cse(test_graph2, manager2); + ASSERT_TRUE(is_changed); + ASSERT_EQ(manager2->all_nodes().size(), 12); + draw::Draw("opt_cse_after_2.dot", test_graph2); +} + +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/optimizer/optimizer_test.cc b/tests/ut/cpp/optimizer/optimizer_test.cc new file mode 100644 index 00000000000..d700225894d --- /dev/null +++ b/tests/ut/cpp/optimizer/optimizer_test.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/anf.h" +#include "operator/ops.h" +#include "optimizer/cse.h" +#include "optimizer/optimizer.h" +#include "optimizer/irpass.h" +#include "debug/draw.h" + +namespace mindspore { +namespace opt { +using Var = mindspore::Var; + +class TestOptOptimizer : public UT::Common { + public: + TestOptOptimizer() : getPyFun("gtest_input.optimizer.opt_test", true), irpass() {} + UT::PyFuncGraphFetcher getPyFun; + irpass::OptimizeIRPassLib irpass; +}; + +TEST_F(TestOptOptimizer, test_step_opt) { + FuncGraphPtr before = getPyFun("test_expendJ"); + + ASSERT_TRUE(nullptr != before); + pipeline::ResourcePtr res = std::make_shared(); + std::shared_ptr optimizer = Optimizer::MakeOptimizer("ut_test", res, + {{"main", + { + // Branch culling + irpass.switch_simplify_, + + // Safe inlining + irpass.arithmetic_simplify_, + irpass.inline_, + }}, + {"grad", {irpass.expand_jprim_}}, + {"cse", OptPassConfig(CSE(false))}}, + true); + EXPECT_TRUE(optimizer.get() != nullptr); + + abstract::AbstractBasePtrList args; + auto after = optimizer->step(before, args); + + draw::Draw("optimizer_test_expendJ_before.dot", before); + draw::Draw("optimizer_test_expendJ_after.dot", after); +} + +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/auto_parallel/dp_algo_test.cc b/tests/ut/cpp/parallel/auto_parallel/dp_algo_test.cc new file mode 100644 index 00000000000..73eff993045 --- /dev/null +++ b/tests/ut/cpp/parallel/auto_parallel/dp_algo_test.cc @@ -0,0 +1,1303 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "parallel/device_manager.h" +#include "parallel/auto_parallel/graph_costmodel.h" +#include "parallel/ops_info/matmul_info.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/ops_info/tmp_identity_info.h" +#include "parallel/auto_parallel/dp_algo_costmodel.h" + +namespace mindspore { +namespace parallel { + +using MatMulInfoPtr = std::shared_ptr; +using ActivationPtr = std::shared_ptr; +using TmpIdentityPtr = std::shared_ptr; + +class TestDPAlgo : public UT::Common { + public: + TestDPAlgo() { + matmul0 = nullptr; + matmul1 = nullptr; + matmul2 = nullptr; + matmul3 = nullptr; + matmul4 = nullptr; + mm1_ptr = nullptr; + mm2_ptr = nullptr; + mm3_ptr = nullptr; + mm4_ptr = nullptr; + mm5_ptr = nullptr; + mm6_ptr = nullptr; + mm7_ptr = nullptr; + relu1_ptr = nullptr; + relu2_ptr = nullptr; + relu3_ptr = nullptr; + relu4_ptr = nullptr; + relu5_ptr = nullptr; + matmul7 = nullptr; + matmul8 = nullptr; + + tmp_identity_ptr = nullptr; + tmp_identity_ptr1 = nullptr; + tmp_identity_ptr2 = nullptr; + edge_m1_m2 = nullptr; + edge_m1_m3 = nullptr; + edge_m1_m4 = nullptr; + edge_m2_r1 = nullptr; + edge_m3_r2 = nullptr; + edge_m4_r3 = nullptr; + edge_r1_m5 = nullptr; + edge_r2_m5 = nullptr; + edge_m5_r4 = nullptr; + edge_r4_m6 = nullptr; + edge_r3_m6 = nullptr; + edge_m6_r5 = nullptr; + + edge_m2_m3 = nullptr; + edge_i1_m1 = nullptr; + edge_i1_m2 = nullptr; + edge_m3_m1 = nullptr; + edge_i2_m4 = nullptr; + edge_i2_m5 = nullptr; + edge_m4_m5 = nullptr; + edge_m3_m4 = nullptr; + edge_i0_m3 = nullptr; + edge_i0_m6 = nullptr; + edge_i1_m4 = nullptr; + edge_i2_m7 = nullptr; + edge_m1_m5 = nullptr; + edge_m5_m6 = nullptr; + edge_m6_m7 = nullptr; + edge_m7_m8 = nullptr; + cost_graph = nullptr; + } + void SetUp(); + void TearDown() {} + void ConstructDiamondGraph(); + void ConstructMMRGraph(); + void ConstructIdentityDiamondGraph(); + void ConstructStarGraph(); + void ConstructDoubleStarGraph(); + void ConstructTwoSeparateGraphs(); + void ConstructTwoSeparateSingleNodeGraph(); + void ConstructThreeSeparateGraphs(); + void ConstructStarGraph2(); + void ConstructStarGraph3(); + void ConstructTwoSeparateGraphs2(); + void ConstructTriangleGraph(); + void ConstructTriangleGraph2(); + void ConstructBatmanGraph(); + void ConstructTwoLargeMatMul(); + + MatMulInfoPtr matmul0; + MatMulInfoPtr matmul1; + MatMulInfoPtr matmul2; + MatMulInfoPtr matmul3; + MatMulInfoPtr matmul4; + MatMulInfoPtr matmul5; + MatMulInfoPtr matmul6; + MatMulInfoPtr matmul7; + MatMulInfoPtr matmul8; + MatMulInfoPtr mm1_ptr, mm2_ptr, mm3_ptr, mm4_ptr, mm5_ptr, mm6_ptr, mm7_ptr; + ActivationPtr relu1_ptr, relu2_ptr, relu3_ptr, relu4_ptr, relu5_ptr; + TmpIdentityPtr tmp_identity_ptr; + TmpIdentityPtr tmp_identity_ptr1; + TmpIdentityPtr tmp_identity_ptr2; + std::shared_ptr edge_m1_m2; + std::shared_ptr edge_m1_m3; + std::shared_ptr edge_m1_m4; + std::shared_ptr edge_m2_r1; + std::shared_ptr edge_m3_r2; + std::shared_ptr edge_m4_r3; + std::shared_ptr edge_r1_m5; + std::shared_ptr edge_r2_m5; + std::shared_ptr edge_m5_r4; + std::shared_ptr edge_r4_m6; + std::shared_ptr edge_r3_m6; + std::shared_ptr edge_m6_r5; + std::shared_ptr edge_i2_m4; + std::shared_ptr edge_i2_m5; + std::shared_ptr edge_m4_m5; + std::shared_ptr edge_m3_m4; + + std::shared_ptr edge_m2_m3; + std::shared_ptr edge_i1_m1; + std::shared_ptr edge_i1_m2; + std::shared_ptr edge_m3_m1; + + std::shared_ptr edge_i0_m3; + std::shared_ptr edge_i0_m6; + std::shared_ptr edge_i1_m4; + std::shared_ptr edge_i2_m7; + std::shared_ptr edge_m1_m5; + std::shared_ptr edge_m5_m6; + std::shared_ptr edge_m6_m7; + std::shared_ptr edge_m7_m8; + CostGraphPtr cost_graph; +}; + +void TestDPAlgo::SetUp() { + cost_graph = std::make_shared(); + cost_graph->SetDeviceMemoryAndCostParameter(); + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + // matmul0 + ValuePtr transpose_a_0 = MakeValue(false); + ValuePtr transpose_b_0 = MakeValue(false); + std::unordered_map attr_0 = {{"transpose_a", transpose_a_0}, {"transpose_b", transpose_b_0}}; + Shapes inputs_shape_0 = {{128, 1024}, {1024, 4096}}; + Shapes outputs_shape_0 = {{4096, 1024}}; + matmul0 = std::make_shared("matmul_info", inputs_shape_0, outputs_shape_0, attr_0); + matmul0->set_name("MatMul0"); + + // matmul1 + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{128, 1024}, {1024, 4096}}; + Shapes outputs_shape_1 = {{128, 4096}}; + matmul1 = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + matmul1->set_name("MatMul1"); + + // matmul2 + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{128, 4096}, {4096, 1024}}; + Shapes outputs_shape_2 = {{128, 1024}}; + matmul2 = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + matmul2->set_name("MatMul2"); + + // matmul3 + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{1024, 128}, {128, 4096}}; + Shapes outputs_shape_3 = {{1024, 4096}}; + matmul3 = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + matmul3->set_name("MatMul3"); + + // matmul4 + ValuePtr transpose_a_4 = MakeValue(false); + ValuePtr transpose_b_4 = MakeValue(false); + std::unordered_map attr_4 = {{"transpose_a", transpose_a_4}, {"transpose_b", transpose_b_4}}; + Shapes inputs_shape_4 = {{128, 1024}, {1024, 4096}}; + Shapes outputs_shape_4 = {{128, 4096}}; + matmul4 = std::make_shared("matmul_info", inputs_shape_4, outputs_shape_4, attr_4); + matmul4->set_name("MatMul4"); + + // matmul5 + ValuePtr transpose_a_5 = MakeValue(false); + ValuePtr transpose_b_5 = MakeValue(false); + std::unordered_map attr_5 = {{"transpose_a", transpose_a_5}, {"transpose_b", transpose_b_5}}; + Shapes inputs_shape_5 = {{128, 4096}, {4096, 4096}}; + Shapes outputs_shape_5 = {{128, 4096}}; + matmul5 = std::make_shared("matmul_info", inputs_shape_5, outputs_shape_5, attr_5); + matmul5->set_name("MatMul5"); + + // matmul6 + ValuePtr transpose_a_6 = MakeValue(false); + ValuePtr transpose_b_6 = MakeValue(false); + std::unordered_map attr_6 = {{"transpose_a", transpose_a_6}, {"transpose_b", transpose_b_6}}; + Shapes inputs_shape_6 = {{4096, 128}, {128, 1024}}; + Shapes outputs_shape_6 = {{4096, 1024}}; + matmul6 = std::make_shared("matmul_info", inputs_shape_6, outputs_shape_6, attr_6); + matmul6->set_name("MatMul6"); + + // matmul7 + ValuePtr transpose_a_7 = MakeValue(false); + ValuePtr transpose_b_7 = MakeValue(true); + std::unordered_map attr_7 = {{"transpose_a", transpose_a_7}, {"transpose_b", transpose_b_7}}; + Shapes inputs_shape_7 = {{64, 128}, {4096, 128}}; + Shapes outputs_shape_7 = {{64, 4096}}; + matmul7 = std::make_shared("matmul_info", inputs_shape_7, outputs_shape_7, attr_7); + matmul7->set_name("MatMul7"); + + // matmul8 + ValuePtr transpose_a_8 = MakeValue(false); + ValuePtr transpose_b_8 = MakeValue(true); + std::unordered_map attr_8 = {{"transpose_a", transpose_a_8}, {"transpose_b", transpose_b_8}}; + Shapes inputs_shape_8 = {{64, 4096}, {40960, 4096}}; + Shapes outputs_shape_8 = {{64, 40960}}; + matmul8 = std::make_shared("matmul_info", inputs_shape_8, outputs_shape_8, attr_8); + matmul8->set_name("MatMul8"); +} + +void TestDPAlgo::ConstructTwoLargeMatMul() { + std::string edge_matmul_matmul_name = "MatMul-MatMul"; + edge_m7_m8 = std::make_shared(edge_matmul_matmul_name, matmul7, matmul8, 0, 0, false); + + matmul7->GenerateStrategies(0); + matmul8->GenerateStrategies(0); + + cost_graph->AddOperator(matmul7); + cost_graph->AddOperator(matmul8); + + edge_m7_m8->InitEdgeCost(); + + matmul7->AddSuccEdge(edge_m7_m8); + matmul8->AddPrevEdge(edge_m7_m8); + cost_graph->AddEdge(matmul7, matmul8, edge_m7_m8); +} + +void TestDPAlgo::ConstructBatmanGraph() { + std::string edge_matmul_matmul_name = "MatMul-MatMul"; + std::string edge_iden_matmul_name = "TmpIdentity-MatMul"; + + std::unordered_map attr = {}; + Shapes inputs_shape = {{64, 64}}; + Shapes outputs_shape = {{64, 64}}; + tmp_identity_ptr1 = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr1->set_name("identity_info1"); + + tmp_identity_ptr2 = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr2->set_name("identity_info2"); + + tmp_identity_ptr = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr->set_name("identity_info"); + + // mm1_ptr + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_1 = {{64, 64}}; + mm1_ptr = std::make_shared("matmul_info1", inputs_shape_1, outputs_shape_1, attr_1); + + // mm2_ptr + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_2 = {{64, 64}}; + mm2_ptr = std::make_shared("matmul_info2", inputs_shape_2, outputs_shape_2, attr_2); + + // mm3_ptr + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_3 = {{64, 64}}; + mm3_ptr = std::make_shared("matmul_info3", inputs_shape_3, outputs_shape_3, attr_3); + + // mm4_ptr + ValuePtr transpose_a_4 = MakeValue(false); + ValuePtr transpose_b_4 = MakeValue(false); + std::unordered_map attr_4 = {{"transpose_a", transpose_a_4}, {"transpose_b", transpose_b_4}}; + Shapes inputs_shape_4 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_4 = {{64, 64}}; + mm4_ptr = std::make_shared("matmul_info4", inputs_shape_4, outputs_shape_4, attr_4); + + // mm5_ptr + ValuePtr transpose_a_5 = MakeValue(false); + ValuePtr transpose_b_5 = MakeValue(false); + std::unordered_map attr_5 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_5}}; + Shapes inputs_shape_5 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_5 = {{64, 64}}; + mm5_ptr = std::make_shared("matmul_info5", inputs_shape_5, outputs_shape_5, attr_5); + + // mm6_ptr + ValuePtr transpose_a_6 = MakeValue(false); + ValuePtr transpose_b_6 = MakeValue(false); + std::unordered_map attr_6 = {{"transpose_a", transpose_a_6}, {"transpose_b", transpose_b_6}}; + Shapes inputs_shape_6 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_6 = {{64, 64}}; + mm6_ptr = std::make_shared("matmul_info6", inputs_shape_6, outputs_shape_6, attr_6); + + // mm7_ptr + ValuePtr transpose_a_7 = MakeValue(false); + ValuePtr transpose_b_7 = MakeValue(false); + std::unordered_map attr_7 = {{"transpose_a", transpose_a_7}, {"transpose_b", transpose_a_7}}; + Shapes inputs_shape_7 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_7 = {{64, 64}}; + mm7_ptr = std::make_shared("matmul_info7", inputs_shape_7, outputs_shape_7, attr_7); + + // create edges + edge_i0_m3 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr, mm3_ptr, 0, 0, false, true); + edge_i0_m6 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr, mm6_ptr, 0, 0, false, true); + edge_i1_m2 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr1, mm2_ptr, 0, 0, false, true); + edge_i1_m4 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr1, mm4_ptr, 0, 0, false, true); + edge_i2_m5 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr2, mm5_ptr, 0, 0, false, true); + edge_i2_m7 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr2, mm7_ptr, 0, 0, false, true); + + edge_m1_m2 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm2_ptr, 0, 1, false, false); + edge_m1_m5 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm5_ptr, 0, 1, false, false); + edge_m2_m3 = std::make_shared(edge_matmul_matmul_name, mm2_ptr, mm3_ptr, 0, 1, false, false); + edge_m3_m4 = std::make_shared(edge_matmul_matmul_name, mm3_ptr, mm4_ptr, 0, 1, false, false); + edge_m5_m6 = std::make_shared(edge_matmul_matmul_name, mm5_ptr, mm6_ptr, 0, 1, false, false); + edge_m6_m7 = std::make_shared(edge_matmul_matmul_name, mm6_ptr, mm7_ptr, 0, 1, false, false); + + // init operators + tmp_identity_ptr->GenerateStrategies(0); + tmp_identity_ptr1->GenerateStrategies(0); + tmp_identity_ptr2->GenerateStrategies(0); + mm1_ptr->GenerateStrategies(0); + mm2_ptr->GenerateStrategies(0); + mm3_ptr->GenerateStrategies(0); + mm4_ptr->GenerateStrategies(0); + mm5_ptr->GenerateStrategies(0); + mm6_ptr->GenerateStrategies(0); + mm7_ptr->GenerateStrategies(0); + + // add operators to costgraph + cost_graph->AddOperator(tmp_identity_ptr); + cost_graph->AddOperator(tmp_identity_ptr1); + cost_graph->AddOperator(tmp_identity_ptr2); + cost_graph->AddOperator(mm1_ptr); + cost_graph->AddOperator(mm2_ptr); + cost_graph->AddOperator(mm3_ptr); + cost_graph->AddOperator(mm4_ptr); + cost_graph->AddOperator(mm5_ptr); + cost_graph->AddOperator(mm6_ptr); + cost_graph->AddOperator(mm7_ptr); + + // init edge cost + edge_i0_m3->InitEdgeCost(); + edge_i0_m6->InitEdgeCost(); + edge_i1_m2->InitEdgeCost(); + edge_i1_m4->InitEdgeCost(); + edge_i2_m5->InitEdgeCost(); + edge_i2_m7->InitEdgeCost(); + + edge_m1_m2->InitEdgeCost(); + edge_m1_m5->InitEdgeCost(); + edge_m2_m3->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + edge_m5_m6->InitEdgeCost(); + edge_m6_m7->InitEdgeCost(); + + // add edges to costgraph + tmp_identity_ptr->AddSuccEdge(edge_i0_m3); + mm3_ptr->AddPrevEdge(edge_i0_m3); + cost_graph->AddEdge(tmp_identity_ptr, mm3_ptr, edge_i0_m3); + + tmp_identity_ptr->AddSuccEdge(edge_i0_m6); + mm6_ptr->AddPrevEdge(edge_i0_m6); + cost_graph->AddEdge(tmp_identity_ptr, mm6_ptr, edge_i0_m6); + + tmp_identity_ptr1->AddSuccEdge(edge_i1_m2); + mm2_ptr->AddPrevEdge(edge_i1_m2); + cost_graph->AddEdge(tmp_identity_ptr1, mm2_ptr, edge_i1_m2); + + tmp_identity_ptr1->AddSuccEdge(edge_i1_m4); + mm4_ptr->AddPrevEdge(edge_i1_m4); + cost_graph->AddEdge(tmp_identity_ptr1, mm4_ptr, edge_i1_m4); + + tmp_identity_ptr2->AddSuccEdge(edge_i2_m5); + mm5_ptr->AddPrevEdge(edge_i2_m5); + cost_graph->AddEdge(tmp_identity_ptr2, mm5_ptr, edge_i2_m5); + + tmp_identity_ptr2->AddSuccEdge(edge_i2_m7); + mm7_ptr->AddPrevEdge(edge_i2_m7); + cost_graph->AddEdge(tmp_identity_ptr2, mm7_ptr, edge_i2_m7); + + mm1_ptr->AddSuccEdge(edge_m1_m2); + mm2_ptr->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(mm1_ptr, mm2_ptr, edge_m1_m2); + + mm1_ptr->AddSuccEdge(edge_m1_m5); + mm5_ptr->AddPrevEdge(edge_m1_m5); + cost_graph->AddEdge(mm1_ptr, mm5_ptr, edge_m1_m5); + + mm2_ptr->AddSuccEdge(edge_m2_m3); + mm3_ptr->AddPrevEdge(edge_m2_m3); + cost_graph->AddEdge(mm2_ptr, mm3_ptr, edge_m2_m3); + + mm3_ptr->AddSuccEdge(edge_m3_m4); + mm4_ptr->AddPrevEdge(edge_m3_m4); + cost_graph->AddEdge(mm3_ptr, mm4_ptr, edge_m3_m4); + + mm5_ptr->AddSuccEdge(edge_m5_m6); + mm6_ptr->AddPrevEdge(edge_m5_m6); + cost_graph->AddEdge(mm5_ptr, mm6_ptr, edge_m5_m6); + + mm6_ptr->AddSuccEdge(edge_m6_m7); + mm7_ptr->AddPrevEdge(edge_m6_m7); + cost_graph->AddEdge(mm6_ptr, mm7_ptr, edge_m6_m7); +} + +void TestDPAlgo::ConstructTriangleGraph() { + std::unordered_map attr = {}; + Shapes inputs_shape = {{64, 64}}; + Shapes outputs_shape = {{64, 64}}; + tmp_identity_ptr1 = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr1->set_name("identity_info1"); + + // mm6_ptr + ValuePtr transpose_a_6 = MakeValue(false); + ValuePtr transpose_b_6 = MakeValue(false); + std::unordered_map attr_6 = {{"transpose_a", transpose_a_6}, {"transpose_b", transpose_b_6}}; + Shapes inputs_shape_6 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_6 = {{64, 64}}; + mm6_ptr = std::make_shared("matmul_info", inputs_shape_6, outputs_shape_6, attr_6); + + tmp_identity_ptr2 = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr2->set_name("identity_info2"); + + // mm1_ptr + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_1 = {{64, 64}}; + mm1_ptr = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // mm2_ptr + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_2 = {{64, 64}}; + mm2_ptr = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // mm3_ptr + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_3 = {{64, 64}}; + mm3_ptr = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // mm4_ptr + ValuePtr transpose_a_4 = MakeValue(false); + ValuePtr transpose_b_4 = MakeValue(false); + std::unordered_map attr_4 = {{"transpose_a", transpose_a_4}, {"transpose_b", transpose_b_4}}; + Shapes inputs_shape_4 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_4 = {{64, 64}}; + mm4_ptr = std::make_shared("matmul_info", inputs_shape_4, outputs_shape_4, attr_4); + + // mm5_ptr + ValuePtr transpose_a_5 = MakeValue(false); + ValuePtr transpose_b_5 = MakeValue(false); + std::unordered_map attr_5 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_5}}; + Shapes inputs_shape_5 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_5 = {{64, 64}}; + mm5_ptr = std::make_shared("matmul_info", inputs_shape_5, outputs_shape_5, attr_5); + + // create edges + std::string edge_matmul_matmul_name = "MatMul-MatMul"; + std::string edge_iden_matmul_name = "TmpIdentity-MatMul"; + edge_i1_m1 = std::make_shared(edge_matmul_matmul_name, mm6_ptr, mm1_ptr, 0, 0, false, false); + edge_i1_m2 = std::make_shared(edge_matmul_matmul_name, mm6_ptr, mm2_ptr, 0, 0, false, false); + edge_m1_m2 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm2_ptr, 0, 1, false, false); + edge_m3_m1 = std::make_shared(edge_matmul_matmul_name, mm3_ptr, mm1_ptr, 0, 1, false, false); + edge_i2_m4 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr2, mm4_ptr, 0, 0, false, true); + edge_i2_m5 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr2, mm5_ptr, 0, 0, false, true); + edge_m4_m5 = std::make_shared(edge_matmul_matmul_name, mm4_ptr, mm5_ptr, 0, 1, false, false); + edge_m3_m4 = std::make_shared(edge_matmul_matmul_name, mm3_ptr, mm4_ptr, 0, 1, false, false); + + // init operators + mm6_ptr->GenerateStrategies(0); + mm1_ptr->GenerateStrategies(0); + mm2_ptr->GenerateStrategies(0); + mm3_ptr->GenerateStrategies(0); + tmp_identity_ptr2->GenerateStrategies(0); + mm4_ptr->GenerateStrategies(0); + mm5_ptr->GenerateStrategies(0); + + // add operators to costgraph + cost_graph->AddOperator(mm6_ptr); + cost_graph->AddOperator(mm1_ptr); + cost_graph->AddOperator(mm2_ptr); + cost_graph->AddOperator(mm3_ptr); + cost_graph->AddOperator(tmp_identity_ptr2); + cost_graph->AddOperator(mm4_ptr); + cost_graph->AddOperator(mm5_ptr); + + // init edge cost + edge_i1_m1->InitEdgeCost(); + edge_i1_m2->InitEdgeCost(); + edge_m1_m2->InitEdgeCost(); + edge_m3_m1->InitEdgeCost(); + + edge_i2_m4->InitEdgeCost(); + edge_i2_m5->InitEdgeCost(); + edge_m4_m5->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + + // add edges to costgraph + mm6_ptr->AddSuccEdge(edge_i1_m1); + mm1_ptr->AddPrevEdge(edge_i1_m1); + cost_graph->AddEdge(mm6_ptr, mm1_ptr, edge_i1_m1); + + mm6_ptr->AddSuccEdge(edge_i1_m2); + mm2_ptr->AddPrevEdge(edge_i1_m2); + cost_graph->AddEdge(mm6_ptr, mm2_ptr, edge_i1_m2); + + mm1_ptr->AddSuccEdge(edge_m1_m2); + mm2_ptr->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(mm1_ptr, mm2_ptr, edge_m1_m2); + + mm3_ptr->AddSuccEdge(edge_m3_m1); + mm1_ptr->AddPrevEdge(edge_m3_m1); + cost_graph->AddEdge(mm3_ptr, mm1_ptr, edge_m3_m1); + + tmp_identity_ptr2->AddSuccEdge(edge_i2_m4); + mm4_ptr->AddPrevEdge(edge_i2_m4); + cost_graph->AddEdge(tmp_identity_ptr2, mm4_ptr, edge_i2_m4); + + tmp_identity_ptr2->AddSuccEdge(edge_i2_m5); + mm5_ptr->AddPrevEdge(edge_i2_m5); + cost_graph->AddEdge(tmp_identity_ptr2, mm5_ptr, edge_i2_m5); + + mm4_ptr->AddSuccEdge(edge_m4_m5); + mm5_ptr->AddPrevEdge(edge_m4_m5); + cost_graph->AddEdge(mm4_ptr, mm5_ptr, edge_m4_m5); + + mm3_ptr->AddSuccEdge(edge_m3_m4); + mm4_ptr->AddPrevEdge(edge_m3_m4); + cost_graph->AddEdge(mm3_ptr, mm4_ptr, edge_m3_m4); +} + +void TestDPAlgo::ConstructTriangleGraph2() { + std::unordered_map attr = {}; + Shapes inputs_shape = {{64, 64}}; + Shapes outputs_shape = {{64, 64}}; + tmp_identity_ptr1 = std::make_shared(inputs_shape, outputs_shape, attr); + tmp_identity_ptr1->set_name("identity_info1"); + + // mm1_ptr + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_1 = {{64, 64}}; + mm1_ptr = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // mm2_ptr + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_2 = {{64, 64}}; + mm2_ptr = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // mm3_ptr + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{64, 64}, {64, 64}}; + Shapes outputs_shape_3 = {{64, 64}}; + mm3_ptr = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // create edges + std::string edge_matmul_matmul_name = "MatMul-MatMul"; + std::string edge_iden_matmul_name = "TmpIdentity-MatMul"; + edge_i1_m1 = std::make_shared(edge_matmul_matmul_name, tmp_identity_ptr1, mm1_ptr, 0, 0, false, true); + edge_i1_m2 = std::make_shared(edge_matmul_matmul_name, tmp_identity_ptr1, mm2_ptr, 0, 0, false, true); + edge_m1_m2 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm2_ptr, 0, 1, false, false); + edge_m3_m1 = std::make_shared(edge_matmul_matmul_name, mm3_ptr, mm1_ptr, 0, 1, false, false); + + // init operators + tmp_identity_ptr1->GenerateStrategies(0); + mm1_ptr->GenerateStrategies(0); + mm2_ptr->GenerateStrategies(0); + mm3_ptr->GenerateStrategies(0); + + // add operators to costgraph + cost_graph->AddOperator(tmp_identity_ptr1); + cost_graph->AddOperator(mm1_ptr); + cost_graph->AddOperator(mm2_ptr); + cost_graph->AddOperator(mm3_ptr); + + // init edge cost + edge_i1_m1->InitEdgeCost(); + edge_i1_m2->InitEdgeCost(); + edge_m1_m2->InitEdgeCost(); + edge_m3_m1->InitEdgeCost(); + + // add edges to costgraph + tmp_identity_ptr1->AddSuccEdge(edge_i1_m1); + mm1_ptr->AddPrevEdge(edge_i1_m1); + cost_graph->AddEdge(tmp_identity_ptr1, mm1_ptr, edge_i1_m1); + + tmp_identity_ptr1->AddSuccEdge(edge_i1_m2); + mm2_ptr->AddPrevEdge(edge_i1_m2); + cost_graph->AddEdge(tmp_identity_ptr1, mm2_ptr, edge_i1_m2); + + mm1_ptr->AddSuccEdge(edge_m1_m2); + mm2_ptr->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(mm1_ptr, mm2_ptr, edge_m1_m2); + + mm3_ptr->AddSuccEdge(edge_m3_m1); + mm1_ptr->AddPrevEdge(edge_m3_m1); + cost_graph->AddEdge(mm3_ptr, mm1_ptr, edge_m3_m1); +} + +void TestDPAlgo::ConstructStarGraph() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 1, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + cost_graph->AddOperator(matmul4); + + edge_m1_m2->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + cost_graph->AddEdge(matmul2, matmul4, edge_m2_m4); + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); + cost_graph->AddEdge(matmul3, matmul4, edge_m3_m4); +} + +void TestDPAlgo::ConstructStarGraph2() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m0_m2 = std::make_shared(edge_name, matmul0, matmul2, 0, 1, false); + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m1_m3 = std::make_shared(edge_name, matmul1, matmul3, 0, 1, false); + + matmul0->GenerateStrategies(0); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + cost_graph->AddOperator(matmul0); + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + edge_m0_m2->InitEdgeCost(); + edge_m1_m2->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + + matmul0->AddSuccEdge(edge_m0_m2); + matmul2->AddPrevEdge(edge_m0_m2); + cost_graph->AddEdge(matmul0, matmul2, edge_m0_m2); + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + matmul1->AddSuccEdge(edge_m1_m3); + matmul3->AddPrevEdge(edge_m1_m3); + cost_graph->AddEdge(matmul1, matmul3, edge_m1_m3); +} + +void TestDPAlgo::ConstructStarGraph3() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m2_m1 = std::make_shared(edge_name, matmul2, matmul1, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + std::shared_ptr edge_m2_m6 = std::make_shared(edge_name, matmul2, matmul6, 0, 1, false); + + matmul2->GenerateStrategies(0); + matmul1->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + matmul6->GenerateStrategies(0); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul4); + cost_graph->AddOperator(matmul6); + edge_m2_m1->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + edge_m2_m6->InitEdgeCost(); + + matmul2->AddSuccEdge(edge_m2_m1); + matmul1->AddPrevEdge(edge_m2_m1); + cost_graph->AddEdge(matmul2, matmul1, edge_m2_m1); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + cost_graph->AddEdge(matmul2, matmul4, edge_m2_m4); + matmul2->AddSuccEdge(edge_m2_m6); + matmul6->AddPrevEdge(edge_m2_m6); + cost_graph->AddEdge(matmul2, matmul6, edge_m2_m6); +} + +void TestDPAlgo::ConstructTwoSeparateGraphs2() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m0_m2 = std::make_shared(edge_name, matmul0, matmul2, 0, 1, false); + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m1_m3 = std::make_shared(edge_name, matmul1, matmul3, 0, 1, false); + std::shared_ptr edge_m4_m5 = std::make_shared(edge_name, matmul4, matmul5, 0, 0, false); + + matmul0->GenerateStrategies(0); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + + cost_graph->AddOperator(matmul0); + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + cost_graph->AddOperator(matmul4); + cost_graph->AddOperator(matmul5); + + edge_m0_m2->InitEdgeCost(); + edge_m1_m2->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + edge_m4_m5->InitEdgeCost(); + + matmul0->AddSuccEdge(edge_m0_m2); + matmul2->AddPrevEdge(edge_m0_m2); + cost_graph->AddEdge(matmul0, matmul2, edge_m0_m2); + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + matmul1->AddSuccEdge(edge_m1_m3); + matmul3->AddPrevEdge(edge_m1_m3); + cost_graph->AddEdge(matmul1, matmul3, edge_m1_m3); + matmul4->AddSuccEdge(edge_m4_m5); + matmul5->AddPrevEdge(edge_m4_m5); + cost_graph->AddEdge(matmul4, matmul5, edge_m4_m5); +} + +void TestDPAlgo::ConstructTwoSeparateSingleNodeGraph() { + matmul0->GenerateStrategies(0); + matmul1->GenerateStrategies(0); + + cost_graph->AddOperator(matmul0); + cost_graph->AddOperator(matmul1); +} + +void TestDPAlgo::ConstructDoubleStarGraph() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m6_m2 = std::make_shared(edge_name, matmul6, matmul2, 0, 1, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 1, false); + std::shared_ptr edge_m4_m5 = std::make_shared(edge_name, matmul4, matmul5, 0, 0, false); + + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + matmul6->GenerateStrategies(0); + + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + cost_graph->AddOperator(matmul4); + cost_graph->AddOperator(matmul5); + cost_graph->AddOperator(matmul6); + + edge_m1_m2->InitEdgeCost(); + edge_m6_m2->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + edge_m4_m5->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + + matmul6->AddSuccEdge(edge_m6_m2); + matmul2->AddPrevEdge(edge_m6_m2); + cost_graph->AddEdge(matmul6, matmul2, edge_m6_m2); + + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + cost_graph->AddEdge(matmul2, matmul4, edge_m2_m4); + + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); + cost_graph->AddEdge(matmul3, matmul4, edge_m3_m4); + + matmul4->AddSuccEdge(edge_m4_m5); + matmul5->AddPrevEdge(edge_m4_m5); + cost_graph->AddEdge(matmul4, matmul5, edge_m4_m5); +} + +void TestDPAlgo::ConstructTwoSeparateGraphs() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m6_m2 = std::make_shared(edge_name, matmul6, matmul2, 0, 1, false); + + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 1, false); + std::shared_ptr edge_m4_m5 = std::make_shared(edge_name, matmul4, matmul5, 0, 0, false); + + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + matmul6->GenerateStrategies(0); + + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + cost_graph->AddOperator(matmul4); + cost_graph->AddOperator(matmul5); + cost_graph->AddOperator(matmul6); + + edge_m1_m2->InitEdgeCost(); + edge_m6_m2->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + edge_m4_m5->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + + matmul6->AddSuccEdge(edge_m6_m2); + matmul2->AddPrevEdge(edge_m6_m2); + cost_graph->AddEdge(matmul6, matmul2, edge_m6_m2); + + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); + cost_graph->AddEdge(matmul3, matmul4, edge_m3_m4); + + matmul4->AddSuccEdge(edge_m4_m5); + matmul5->AddPrevEdge(edge_m4_m5); + cost_graph->AddEdge(matmul4, matmul5, edge_m4_m5); +} + +void TestDPAlgo::ConstructThreeSeparateGraphs() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m6_m2 = std::make_shared(edge_name, matmul6, matmul2, 0, 1, false); + + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + matmul6->GenerateStrategies(0); + + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + cost_graph->AddOperator(matmul4); + cost_graph->AddOperator(matmul5); + cost_graph->AddOperator(matmul6); + + edge_m1_m2->InitEdgeCost(); + edge_m6_m2->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + + matmul6->AddSuccEdge(edge_m6_m2); + matmul2->AddPrevEdge(edge_m6_m2); + cost_graph->AddEdge(matmul6, matmul2, edge_m6_m2); +} + +void TestDPAlgo::ConstructDiamondGraph() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + std::shared_ptr edge_m1_m3 = std::make_shared(edge_name, matmul1, matmul3, 0, 1, false); + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 1, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + cost_graph->AddOperator(matmul1); + cost_graph->AddOperator(matmul2); + cost_graph->AddOperator(matmul3); + cost_graph->AddOperator(matmul4); + edge_m1_m2->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(matmul1, matmul2, edge_m1_m2); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + cost_graph->AddEdge(matmul2, matmul4, edge_m2_m4); + matmul1->AddSuccEdge(edge_m1_m3); + matmul3->AddPrevEdge(edge_m1_m3); + cost_graph->AddEdge(matmul1, matmul3, edge_m1_m3); + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); + cost_graph->AddEdge(matmul3, matmul4, edge_m3_m4); +} + +void TestDPAlgo::ConstructMMRGraph() { + // mm1_ptr + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{32, 16}, {16, 32}}; + Shapes outputs_shape_1 = {{32, 32}}; + mm1_ptr = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // mm2_ptr + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{8, 32}, {32, 32}}; + Shapes outputs_shape_2 = {{8, 32}}; + mm2_ptr = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // mm3_ptr + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{32, 32}, {32, 64}}; + Shapes outputs_shape_3 = {{32, 64}}; + mm3_ptr = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // mm4_ptr + ValuePtr transpose_a_4 = MakeValue(false); + ValuePtr transpose_b_4 = MakeValue(false); + std::unordered_map attr_4 = {{"transpose_a", transpose_a_4}, {"transpose_b", transpose_b_4}}; + Shapes inputs_shape_4 = {{64, 32}, {32, 32}}; + Shapes outputs_shape_4 = {{64, 32}}; + mm4_ptr = std::make_shared("matmul_info", inputs_shape_4, outputs_shape_4, attr_4); + + // mm5_ptr + ValuePtr transpose_a_5 = MakeValue(false); + ValuePtr transpose_b_5 = MakeValue(false); + std::unordered_map attr_5 = {{"transpose_a", transpose_a_5}, {"transpose_b", transpose_b_5}}; + Shapes inputs_shape_5 = {{8, 32}, {32, 64}}; + Shapes outputs_shape_5 = {{8, 64}}; + mm5_ptr = std::make_shared("matmul_info", inputs_shape_5, outputs_shape_5, attr_5); + + // mm5_ptr + ValuePtr transpose_a_6 = MakeValue(false); + ValuePtr transpose_b_6 = MakeValue(false); + std::unordered_map attr_6 = {{"transpose_a", transpose_a_6}, {"transpose_b", transpose_b_6}}; + Shapes inputs_shape_6 = {{8, 64}, {64, 32}}; + Shapes outputs_shape_6 = {{8, 32}}; + mm6_ptr = std::make_shared("matmul_info", inputs_shape_6, outputs_shape_6, attr_6); + + ValuePtr relu = MakeValue(std::string("relu")); + std::unordered_map relu_attr = {{"activation_type", relu}}; + + // relu1_ptr + Shapes relu1_inputs_shape = {{8, 32}}; + Shapes relu1_outputs_shape = {{8, 32}}; + relu1_ptr = std::make_shared("relu_info", relu1_inputs_shape, relu1_outputs_shape, relu_attr); + + // relu2_ptr + Shapes relu2_inputs_shape = {{32, 64}}; + Shapes relu2_outputs_shape = {{32, 64}}; + relu2_ptr = std::make_shared("relu_info", relu2_inputs_shape, relu2_outputs_shape, relu_attr); + + // relu3_ptr + Shapes relu3_inputs_shape = {{64, 32}}; + Shapes relu3_outputs_shape = {{64, 32}}; + relu3_ptr = std::make_shared("relu_info", relu3_inputs_shape, relu3_outputs_shape, relu_attr); + + // relu4_ptr + Shapes relu4_inputs_shape = {{8, 64}}; + Shapes relu4_outputs_shape = {{8, 64}}; + relu4_ptr = std::make_shared("relu_info", relu4_inputs_shape, relu4_outputs_shape, relu_attr); + + // relu5_ptr + Shapes relu5_inputs_shape = {{8, 32}}; + Shapes relu5_outputs_shape = {{8, 32}}; + relu5_ptr = std::make_shared("relu_info", relu5_inputs_shape, relu5_outputs_shape, relu_attr); + + std::string edge_matmul_matmul_name = "MatMul-MatMul"; + std::string edge_matmul_relu_name = "MatMul-ReLU"; + std::string edge_relu_matmul_name = "ReLU-MatMul"; + + // create edges + edge_m1_m2 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm2_ptr, 0, 1, false); + edge_m1_m3 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm3_ptr, 0, 0, false); + edge_m1_m4 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm4_ptr, 0, 1, false); + edge_m2_r1 = std::make_shared(edge_matmul_relu_name, mm2_ptr, relu1_ptr, 0, 0, false); + edge_m3_r2 = std::make_shared(edge_matmul_relu_name, mm3_ptr, relu2_ptr, 0, 0, false); + edge_m4_r3 = std::make_shared(edge_matmul_relu_name, mm4_ptr, relu3_ptr, 0, 0, false); + edge_r1_m5 = std::make_shared(edge_relu_matmul_name, relu1_ptr, mm5_ptr, 0, 0, false); + edge_r2_m5 = std::make_shared(edge_relu_matmul_name, relu2_ptr, mm5_ptr, 0, 1, false); + edge_m5_r4 = std::make_shared(edge_matmul_relu_name, mm5_ptr, relu4_ptr, 0, 0, false); + edge_r4_m6 = std::make_shared(edge_relu_matmul_name, relu4_ptr, mm6_ptr, 0, 0, false); + edge_r3_m6 = std::make_shared(edge_relu_matmul_name, relu3_ptr, mm6_ptr, 0, 1, false); + edge_m6_r5 = std::make_shared(edge_matmul_relu_name, mm6_ptr, relu5_ptr, 0, 0, false); + + // init operators + mm1_ptr->GenerateStrategies(0); + mm2_ptr->GenerateStrategies(0); + mm3_ptr->GenerateStrategies(0); + mm4_ptr->GenerateStrategies(0); + mm5_ptr->GenerateStrategies(0); + mm6_ptr->GenerateStrategies(0); + relu1_ptr->GenerateStrategies(0); + relu2_ptr->GenerateStrategies(0); + relu3_ptr->GenerateStrategies(0); + relu4_ptr->GenerateStrategies(0); + relu5_ptr->GenerateStrategies(0); + + // add operators to costgraph + cost_graph->AddOperator(mm1_ptr); + cost_graph->AddOperator(mm2_ptr); + cost_graph->AddOperator(mm3_ptr); + cost_graph->AddOperator(mm4_ptr); + cost_graph->AddOperator(mm5_ptr); + cost_graph->AddOperator(mm6_ptr); + cost_graph->AddOperator(relu1_ptr); + cost_graph->AddOperator(relu2_ptr); + cost_graph->AddOperator(relu3_ptr); + cost_graph->AddOperator(relu4_ptr); + cost_graph->AddOperator(relu5_ptr); + + // init edge cost + edge_m1_m2->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + edge_m1_m4->InitEdgeCost(); + edge_m2_r1->InitEdgeCost(); + edge_m3_r2->InitEdgeCost(); + edge_m4_r3->InitEdgeCost(); + edge_r1_m5->InitEdgeCost(); + edge_r2_m5->InitEdgeCost(); + edge_m5_r4->InitEdgeCost(); + edge_r4_m6->InitEdgeCost(); + edge_r3_m6->InitEdgeCost(); + edge_m6_r5->InitEdgeCost(); + + mm1_ptr->AddSuccEdge(edge_m1_m2); + mm2_ptr->AddPrevEdge(edge_m1_m2); + cost_graph->AddEdge(mm1_ptr, mm2_ptr, edge_m1_m2); + + mm1_ptr->AddSuccEdge(edge_m1_m3); + mm3_ptr->AddPrevEdge(edge_m1_m3); + cost_graph->AddEdge(mm1_ptr, mm3_ptr, edge_m1_m3); + + mm1_ptr->AddSuccEdge(edge_m1_m4); + mm4_ptr->AddPrevEdge(edge_m1_m4); + cost_graph->AddEdge(mm1_ptr, mm4_ptr, edge_m1_m4); + + mm2_ptr->AddSuccEdge(edge_m2_r1); + relu1_ptr->AddPrevEdge(edge_m2_r1); + cost_graph->AddEdge(mm2_ptr, relu1_ptr, edge_m2_r1); + + mm3_ptr->AddSuccEdge(edge_m3_r2); + relu2_ptr->AddPrevEdge(edge_m3_r2); + cost_graph->AddEdge(mm3_ptr, relu2_ptr, edge_m3_r2); + + mm4_ptr->AddSuccEdge(edge_m4_r3); + relu3_ptr->AddPrevEdge(edge_m4_r3); + cost_graph->AddEdge(mm4_ptr, relu3_ptr, edge_m4_r3); + + relu1_ptr->AddSuccEdge(edge_r1_m5); + mm5_ptr->AddPrevEdge(edge_r1_m5); + cost_graph->AddEdge(relu1_ptr, mm5_ptr, edge_r1_m5); + + relu2_ptr->AddSuccEdge(edge_r2_m5); + mm5_ptr->AddPrevEdge(edge_r2_m5); + cost_graph->AddEdge(relu2_ptr, mm5_ptr, edge_r2_m5); + + mm5_ptr->AddSuccEdge(edge_m5_r4); + relu4_ptr->AddPrevEdge(edge_m5_r4); + cost_graph->AddEdge(mm5_ptr, relu4_ptr, edge_m5_r4); + + relu4_ptr->AddSuccEdge(edge_r4_m6); + mm6_ptr->AddPrevEdge(edge_r4_m6); + cost_graph->AddEdge(relu4_ptr, mm6_ptr, edge_r4_m6); + + relu3_ptr->AddSuccEdge(edge_r3_m6); + mm6_ptr->AddPrevEdge(edge_r3_m6); + cost_graph->AddEdge(relu3_ptr, mm6_ptr, edge_r3_m6); + + mm6_ptr->AddSuccEdge(edge_m6_r5); + relu5_ptr->AddPrevEdge(edge_m6_r5); + cost_graph->AddEdge(mm6_ptr, relu5_ptr, edge_m6_r5); +} + +void TestDPAlgo::ConstructIdentityDiamondGraph() { + std::unordered_map attr = {}; + Shapes inputs_shape = {{32, 64}}; + Shapes outputs_shape = {{32, 64}}; + tmp_identity_ptr = std::make_shared(inputs_shape, outputs_shape, attr); + + // mm1_ptr + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{32, 64}, {64, 128}}; + Shapes outputs_shape_1 = {{32, 128}}; + mm1_ptr = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // mm2_ptr + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{128, 32}, {32, 64}}; + Shapes outputs_shape_2 = {{128, 64}}; + mm2_ptr = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // mm3_ptr + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{32, 128}, {128, 64}}; + Shapes outputs_shape_3 = {{32, 64}}; + mm3_ptr = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // create edges + std::string edge_matmul_matmul_name = "MatMul-MatMul"; + std::string edge_iden_matmul_name = "TmpIdentity-MatMul"; + edge_i1_m1 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr, mm1_ptr, 0, 0, false, true); + edge_i1_m2 = std::make_shared(edge_iden_matmul_name, tmp_identity_ptr, mm2_ptr, 0, 1, false, true); + edge_m1_m3 = std::make_shared(edge_matmul_matmul_name, mm1_ptr, mm3_ptr, 0, 0, false); + edge_m2_m3 = std::make_shared(edge_matmul_matmul_name, mm2_ptr, mm3_ptr, 0, 1, false); + + // init operators + tmp_identity_ptr->GenerateStrategies(0); + mm1_ptr->GenerateStrategies(0); + mm2_ptr->GenerateStrategies(0); + mm3_ptr->GenerateStrategies(0); + + // add operators to costgraph + cost_graph->AddOperator(tmp_identity_ptr); + cost_graph->AddOperator(mm1_ptr); + cost_graph->AddOperator(mm2_ptr); + cost_graph->AddOperator(mm3_ptr); + + // init edge cost + edge_i1_m1->InitEdgeCost(); + edge_i1_m2->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + edge_m2_m3->InitEdgeCost(); + + // add edges to costgraph + tmp_identity_ptr->AddSuccEdge(edge_i1_m1); + mm1_ptr->AddPrevEdge(edge_i1_m1); + cost_graph->AddEdge(tmp_identity_ptr, mm1_ptr, edge_i1_m1); + + tmp_identity_ptr->AddSuccEdge(edge_i1_m2); + mm2_ptr->AddPrevEdge(edge_i1_m2); + cost_graph->AddEdge(tmp_identity_ptr, mm2_ptr, edge_i1_m2); + + mm1_ptr->AddSuccEdge(edge_m1_m3); + mm3_ptr->AddPrevEdge(edge_m1_m3); + cost_graph->AddEdge(mm1_ptr, mm3_ptr, edge_m1_m3); + + mm2_ptr->AddSuccEdge(edge_m2_m3); + mm3_ptr->AddPrevEdge(edge_m2_m3); + cost_graph->AddEdge(mm2_ptr, mm3_ptr, edge_m2_m3); +} + +TEST_F(TestDPAlgo, test_ConstructTwoLargeMatMul) { + ConstructTwoLargeMatMul(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); + ASSERT_EQ(cost_graph->InitSelectedStrategy(), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructBatmanGraph) { + ConstructBatmanGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); + ASSERT_EQ(cost_graph->InitSelectedStrategy(), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructTriangleGraph) { + ConstructTriangleGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructTriangleGraph2) { + ConstructTriangleGraph2(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructStarGraph2) { + ConstructStarGraph2(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructStarGraph3) { + ConstructStarGraph3(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructTwoSeparateGraphs2) { + ConstructTwoSeparateGraphs2(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructTwoSeparateSingleNodeGraph) { + ConstructTwoSeparateSingleNodeGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructThreeSeparateGraphs) { + ConstructThreeSeparateGraphs(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_ConstructTwoSeparateGraphs) { + ConstructTwoSeparateGraphs(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_GetStrategy) { + ConstructDiamondGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_GetStrategy_for_MMR_graph) { + ConstructMMRGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_GetStrategy_for_IdentityDiamondGraph) { + ConstructIdentityDiamondGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_GetStrategy_for_StarGraph) { + ConstructStarGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); +} + +TEST_F(TestDPAlgo, test_GetStrategy_for_DoubleStarGraph) { + ConstructDoubleStarGraph(); + ASSERT_EQ(GetStrategy(cost_graph), SUCCESS); + + for (auto &op : cost_graph->GetOperators()) { + StrategyPtr s_strategy = op->selected_strategy(); + std::vector strategy_0 = s_strategy->GetInputDim()[0]; + std::vector strategy_1 = s_strategy->GetInputDim()[1]; + + std::string string_strategy_0 = "["; + for (size_t i = 0; i < strategy_0.size(); ++i) { + string_strategy_0 += std::to_string(strategy_0[i]) + ", "; + } + string_strategy_0 += "]"; + + std::string string_strategy_1 = "["; + for (size_t i = 0; i < strategy_1.size(); ++i) { + string_strategy_1 += std::to_string(strategy_1[i]) + ", "; + } + string_strategy_1 += "]"; + + MS_LOG(INFO) << "" << op->name() << " selected strategy: " << string_strategy_0 << ", " << string_strategy_1; + } +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/auto_parallel/edge_costmodel_test.cc b/tests/ut/cpp/parallel/auto_parallel/edge_costmodel_test.cc new file mode 100644 index 00000000000..adf06d8fcc6 --- /dev/null +++ b/tests/ut/cpp/parallel/auto_parallel/edge_costmodel_test.cc @@ -0,0 +1,161 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "parallel/device_manager.h" +#include "parallel/auto_parallel/edge_costmodel.h" +#include "parallel/ops_info/matmul_info.h" + +namespace mindspore { +namespace parallel { + +using MatMulInfoPtr = std::shared_ptr; +class TestEdgeCostModel : public UT::Common { + public: + TestEdgeCostModel() { + matmul1 = nullptr; + matmul2 = nullptr; + matmul3 = nullptr; + matmul4 = nullptr; + matmul5 = nullptr; + } + void SetUp(); + void TearDown() {} + MatMulInfoPtr matmul1; + MatMulInfoPtr matmul2; + MatMulInfoPtr matmul3; + MatMulInfoPtr matmul4; + MatMulInfoPtr matmul5; +}; + +void TestEdgeCostModel::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + // matmul1 + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{8, 16}, {16, 32}}; + Shapes outputs_shape_1 = {{8, 32}}; + matmul1 = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // matmul2 + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{8, 32}, {32, 16}}; + Shapes outputs_shape_2 = {{8, 16}}; + matmul2 = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // matmul3 + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{16, 8}, {8, 32}}; + Shapes outputs_shape_3 = {{16, 32}}; + matmul3 = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // matmul4 + ValuePtr transpose_a_4 = MakeValue(false); + ValuePtr transpose_b_4 = MakeValue(false); + std::unordered_map attr_4 = {{"transpose_a", transpose_a_4}, {"transpose_b", transpose_b_4}}; + Shapes inputs_shape_4 = {{8, 16}, {16, 32}}; + Shapes outputs_shape_4 = {{8, 32}}; + matmul4 = std::make_shared("matmul_info", inputs_shape_4, outputs_shape_4, attr_4); + + // matmul5 + ValuePtr transpose_a_5 = MakeValue(false); + ValuePtr transpose_b_5 = MakeValue(true); + std::unordered_map attr_5 = {{"transpose_a", transpose_a_5}, {"transpose_b", transpose_b_5}}; + Shapes inputs_shape_5 = {{8, 32}, {8, 32}}; + Shapes outputs_shape_5 = {{8, 8}}; + matmul5 = std::make_shared("matmul_info", inputs_shape_5, outputs_shape_5, attr_5); +} + +TEST_F(TestEdgeCostModel, test_InitEdgeCost) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + ASSERT_EQ(edge_m1_m2->InitEdgeCost(), SUCCESS); +} + +TEST_F(TestEdgeCostModel, test_OpEliminationSetNewCost) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + ASSERT_EQ(edge_m1_m2->InitEdgeCost(), SUCCESS); + ASSERT_EQ(edge_m2_m4->InitEdgeCost(), SUCCESS); + std::shared_ptr new_edge = std::make_shared(edge_name, matmul1, matmul4, 0, 0, false); + new_edge->set_pre_op_output(edge_m1_m2->prev_op_output()); + new_edge->set_next_op_input(edge_m2_m4->next_op_input()); + new_edge->OpEliminationSetNewCost(edge_m1_m2, matmul2, edge_m2_m4); +} + +TEST_F(TestEdgeCostModel, test_EdgeEliminationSetNewCost) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m5 = std::make_shared(edge_name, matmul1, matmul5, 0, 0, false); + std::shared_ptr edge_m1_m5_2 = std::make_shared(edge_name, matmul1, matmul5, 0, 1, false); + + matmul1->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + matmul1->AddSuccEdge(edge_m1_m5); + matmul1->AddSuccEdge(edge_m1_m5_2); + matmul5->AddPrevEdge(edge_m1_m5); + matmul5->AddPrevEdge(edge_m1_m5_2); + ASSERT_EQ(edge_m1_m5->InitEdgeCost(), SUCCESS); + ASSERT_EQ(edge_m1_m5_2->InitEdgeCost(), SUCCESS); + + std::vector> edges; + edges.push_back(edge_m1_m5); + edges.push_back(edge_m1_m5_2); + std::vector output_indexs, input_indexs; + output_indexs.push_back(0); + input_indexs.push_back(0); + input_indexs.push_back(1); + std::shared_ptr new_edge = + std::make_shared(edge_name, matmul1, matmul5, output_indexs, input_indexs, true); + new_edge->set_pre_op_output(edges[0]->prev_op_output()); + new_edge->set_next_op_input(edges[0]->next_op_input()); + new_edge->EdgeEliminationSetNewCost(matmul1, edges, matmul5); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/auto_parallel/graph_costmodel_test.cc b/tests/ut/cpp/parallel/auto_parallel/graph_costmodel_test.cc new file mode 100644 index 00000000000..14281934813 --- /dev/null +++ b/tests/ut/cpp/parallel/auto_parallel/graph_costmodel_test.cc @@ -0,0 +1,444 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "parallel/device_manager.h" +#include "parallel/auto_parallel/graph_costmodel.h" +#include "parallel/ops_info/matmul_info.h" + +namespace mindspore { +namespace parallel { + +using MatMulInfoPtr = std::shared_ptr; + +class TestCostGraph : public UT::Common { + public: + TestCostGraph() { + matmul0 = nullptr; + matmul1 = nullptr; + matmul2 = nullptr; + matmul3 = nullptr; + matmul4 = nullptr; + matmul5 = nullptr; + } + void SetUp(); + void TearDown() {} + void ConstructDiamondGraph(); + void ConstructLinearGraph(); + void ConstructStarGraph(); + void ConstructSingleNodeGraph(); + void ConstructStarGraph2(); + + MatMulInfoPtr matmul0; + MatMulInfoPtr matmul1; + MatMulInfoPtr matmul2; + MatMulInfoPtr matmul3; + MatMulInfoPtr matmul4; + MatMulInfoPtr matmul5; + CostGraph cost_graph; +}; + +void TestCostGraph::SetUp() { + cost_graph.SetDeviceMemoryAndCostParameter(); + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + // matmul0 + ValuePtr transpose_a_0 = MakeValue(false); + ValuePtr transpose_b_0 = MakeValue(false); + std::unordered_map attr_0 = {{"transpose_a", transpose_a_0}, {"transpose_b", transpose_b_0}}; + Shapes inputs_shape_0 = {{32, 16}, {16, 16}}; + Shapes outputs_shape_0 = {{32, 16}}; + matmul0 = std::make_shared("matmul_info", inputs_shape_0, outputs_shape_0, attr_0); + + // matmul1 + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + Shapes inputs_shape_1 = {{8, 16}, {16, 32}}; + Shapes outputs_shape_1 = {{8, 32}}; + matmul1 = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // matmul2 + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpose_b_2}}; + Shapes inputs_shape_2 = {{8, 32}, {32, 16}}; + Shapes outputs_shape_2 = {{8, 16}}; + matmul2 = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // matmul3 + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpose_b_3 = MakeValue(false); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpose_b_3}}; + Shapes inputs_shape_3 = {{16, 8}, {8, 32}}; + Shapes outputs_shape_3 = {{16, 32}}; + matmul3 = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // matmul4 + ValuePtr transpose_a_4 = MakeValue(false); + ValuePtr transpose_b_4 = MakeValue(false); + std::unordered_map attr_4 = {{"transpose_a", transpose_a_4}, {"transpose_b", transpose_b_4}}; + Shapes inputs_shape_4 = {{8, 16}, {16, 32}}; + Shapes outputs_shape_4 = {{8, 32}}; + matmul4 = std::make_shared("matmul_info", inputs_shape_4, outputs_shape_4, attr_4); + + // matmul5 + ValuePtr transpose_a_5 = MakeValue(false); + ValuePtr transpose_b_5 = MakeValue(true); + std::unordered_map attr_5 = {{"transpose_a", transpose_a_5}, {"transpose_b", transpose_b_5}}; + Shapes inputs_shape_5 = {{8, 32}, {8, 32}}; + Shapes outputs_shape_5 = {{8, 8}}; + matmul5 = std::make_shared("matmul_info", inputs_shape_5, outputs_shape_5, attr_5); +} + +void TestCostGraph::ConstructStarGraph2() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m0_m2 = std::make_shared(edge_name, matmul0, matmul2, 0, 1, false); + std::shared_ptr edge_m1_m3 = std::make_shared(edge_name, matmul1, matmul3, 0, 1, false); + + matmul0->GenerateStrategies(0); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + cost_graph.AddOperator(matmul0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul2); + cost_graph.AddOperator(matmul3); + + edge_m0_m2->InitEdgeCost(); + edge_m1_m2->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + matmul0->AddSuccEdge(edge_m0_m2); + matmul2->AddPrevEdge(edge_m0_m2); + cost_graph.AddEdge(matmul0, matmul2, edge_m0_m2); + matmul1->AddSuccEdge(edge_m1_m3); + matmul3->AddPrevEdge(edge_m1_m3); + cost_graph.AddEdge(matmul1, matmul3, edge_m1_m3); +} + +void TestCostGraph::ConstructDiamondGraph() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + std::shared_ptr edge_m1_m3 = std::make_shared(edge_name, matmul1, matmul3, 0, 1, false); + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 1, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + edge_m1_m2->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + edge_m1_m3->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + matmul1->AddSuccEdge(edge_m1_m3); + matmul3->AddPrevEdge(edge_m1_m3); + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); +} + +void TestCostGraph::ConstructLinearGraph() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul2); + cost_graph.AddOperator(matmul4); + edge_m1_m2->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + cost_graph.AddEdge(matmul2, matmul4, edge_m2_m4); +} + +void TestCostGraph::ConstructStarGraph() { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + std::shared_ptr edge_m2_m4 = std::make_shared(edge_name, matmul2, matmul4, 0, 0, false); + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 1, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + matmul3->GenerateStrategies(0); + matmul4->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul2); + cost_graph.AddOperator(matmul3); + cost_graph.AddOperator(matmul4); + + edge_m1_m2->InitEdgeCost(); + edge_m2_m4->InitEdgeCost(); + edge_m3_m4->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + matmul2->AddSuccEdge(edge_m2_m4); + matmul4->AddPrevEdge(edge_m2_m4); + cost_graph.AddEdge(matmul2, matmul4, edge_m2_m4); + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); + cost_graph.AddEdge(matmul3, matmul4, edge_m3_m4); +} + +void TestCostGraph::ConstructSingleNodeGraph() { + matmul1->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); +} + +TEST_F(TestCostGraph, test_CheckMergeElimination) { + ConstructStarGraph(); + ASSERT_EQ(cost_graph.CheckMergeElimination().get(), matmul1.get()); + cost_graph.EliminationOp(matmul2); + ASSERT_EQ(cost_graph.CheckMergeElimination().get(), matmul1.get()); + cost_graph.EliminationMerge(matmul1); +} + +TEST_F(TestCostGraph, test_CheckContractAndMergeElimination) { + ConstructStarGraph2(); + ASSERT_EQ(cost_graph.CheckMergeElimination().get(), matmul0.get()); + cost_graph.EliminationMerge(matmul0); + ASSERT_EQ(cost_graph.CheckContractElimination().get(), matmul2.get()); +} + +TEST_F(TestCostGraph, test_EliminationMerge) { + ConstructStarGraph(); + ASSERT_EQ(cost_graph.EliminationMerge(matmul3).get(), matmul4.get()); + ASSERT_EQ(matmul3->is_alive(), false); +} + +TEST_F(TestCostGraph, test_SearchStrategy_for_single_node_graph) { + ConstructSingleNodeGraph(); + cost_graph.SearchStrategy(); + auto cost = matmul1->selected_cost(); +} + +TEST_F(TestCostGraph, test_IsOperatorInCostGraph) { + CostGraph entire_cost_graph; + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + entire_cost_graph.AddOperator(matmul1); + entire_cost_graph.AddOperator(matmul2); + entire_cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + + ASSERT_EQ(entire_cost_graph.IsOperatorInCostGraph(matmul1), true); + ASSERT_EQ(entire_cost_graph.IsOperatorInCostGraph(matmul2), true); + ASSERT_EQ(entire_cost_graph.IsOperatorInCostGraph(matmul3), false); + auto edges = entire_cost_graph.GetOriginalEdgeBetweenOperators(matmul1, matmul2); + ASSERT_EQ(edges[0], edge_m1_m2); +} + +TEST_F(TestCostGraph, test_ConstructConnectedComponents) { + CostGraph entire_cost_graph; + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + std::shared_ptr edge_m3_m4 = std::make_shared(edge_name, matmul3, matmul4, 0, 0, false); + matmul3->AddSuccEdge(edge_m3_m4); + matmul4->AddPrevEdge(edge_m3_m4); + entire_cost_graph.AddOperator(matmul1); + entire_cost_graph.AddOperator(matmul2); + entire_cost_graph.AddOperator(matmul3); + entire_cost_graph.AddOperator(matmul4); + entire_cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + entire_cost_graph.AddEdge(matmul3, matmul4, edge_m3_m4); + // test for successfully construct connected components + std::vector> connected_coms = + entire_cost_graph.ConstructConnectedComponents(entire_cost_graph.GetOperators()); + ASSERT_EQ(connected_coms.size(), 2); + ASSERT_EQ(connected_coms[0]->GetOperators().size(), 2); + ASSERT_EQ(connected_coms[1]->GetOperators().size(), 2); + ASSERT_EQ(connected_coms[0]->GetOriginalEdgeBetweenOperators(matmul1, matmul2)[0].get(), edge_m1_m2.get()); + ASSERT_EQ(connected_coms[1]->GetOriginalEdgeBetweenOperators(matmul3, matmul4)[0].get(), edge_m3_m4.get()); +} + +TEST_F(TestCostGraph, test_SelectCostListWithMinTrainingTimeMultiple) { + CostGraph entire_cost_graph; + entire_cost_graph.SetDeviceMemoryAndCostParameter(); + double memory = 1024.0; + CostPtrList clist_1, clist_2; + std::vector all_list; + + auto cost1_1 = std::make_shared(10, 20); + cost1_1->communication_without_parameter_ = 20; + clist_1.push_back(cost1_1); + auto cost1_2 = std::make_shared(100, 10); + cost1_2->communication_without_parameter_ = 0.0; + clist_1.push_back(cost1_2); + all_list.push_back(clist_1); + + auto cost2_1 = std::make_shared(1010, 20); + cost2_1->communication_without_parameter_ = 10; + clist_2.push_back(cost2_1); + auto cost2_2 = std::make_shared(1050, 20); + cost2_2->communication_without_parameter_ = 10; + clist_2.push_back(cost2_2); + all_list.push_back(clist_2); + + auto ret_list = entire_cost_graph.SelectCostListWithMinTrainingTimeMultiple(all_list, memory); + ASSERT_EQ(ret_list.size(), 2); + ASSERT_DOUBLE_EQ(ret_list[0]->memory_cost_, 10); + ASSERT_DOUBLE_EQ(ret_list[1]->memory_cost_, 1010); +} + +TEST_F(TestCostGraph, test_CheckOpElimination) { + ConstructLinearGraph(); + ASSERT_EQ(cost_graph.CheckOpElimination().get(), matmul2.get()); +} + +TEST_F(TestCostGraph, test_CheckEdgesElimination) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m5 = std::make_shared(edge_name, matmul1, matmul5, 0, 0, false); + std::shared_ptr edge_m1_m5_2 = std::make_shared(edge_name, matmul1, matmul5, 0, 1, false); + matmul1->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul5); + + matmul1->AddSuccEdge(edge_m1_m5); + matmul1->AddSuccEdge(edge_m1_m5_2); + matmul5->AddPrevEdge(edge_m1_m5); + matmul5->AddPrevEdge(edge_m1_m5_2); + cost_graph.AddEdge(matmul1, matmul5, edge_m1_m5); + cost_graph.AddEdge(matmul1, matmul5, edge_m1_m5_2); + ASSERT_EQ(cost_graph.CheckEdgeElimination().size(), 2); + ASSERT_EQ(cost_graph.CheckEdgeElimination()[0].get(), edge_m1_m5.get()); + ASSERT_EQ(cost_graph.CheckEdgeElimination()[1].get(), edge_m1_m5_2.get()); +} + +TEST_F(TestCostGraph, test_CreateFinalCostList_AND_Select) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul2); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + + ASSERT_EQ(edge_m1_m2->InitEdgeCost(), SUCCESS); + cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + auto cost_list = cost_graph.CreateFinalCostList(matmul1, edge_m1_m2, matmul2); + cost_graph.SelectCostWithMemoryConstraint(cost_list, cost_graph.GetDeviceMemory()); +} + +TEST_F(TestCostGraph, test_EliminationOp) { + ConstructLinearGraph(); + auto new_edge = cost_graph.EliminationOp(matmul2); + ASSERT_EQ(new_edge.get(), matmul1->succ_edges()[0].get()); + ASSERT_EQ(new_edge.get(), matmul4->prev_edges()[0].get()); +} + +TEST_F(TestCostGraph, test_EliminationEdges) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m5 = std::make_shared(edge_name, matmul1, matmul5, 0, 0, false); + std::shared_ptr edge_m1_m5_2 = std::make_shared(edge_name, matmul1, matmul5, 0, 1, false); + matmul1->GenerateStrategies(0); + matmul5->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul5); + edge_m1_m5->InitEdgeCost(); + edge_m1_m5_2->InitEdgeCost(); + + matmul1->AddSuccEdge(edge_m1_m5); + matmul1->AddSuccEdge(edge_m1_m5_2); + matmul5->AddPrevEdge(edge_m1_m5); + matmul5->AddPrevEdge(edge_m1_m5_2); + cost_graph.AddEdge(matmul1, matmul5, edge_m1_m5); + cost_graph.AddEdge(matmul1, matmul5, edge_m1_m5_2); + + std::vector> edges; + edges.push_back(edge_m1_m5); + edges.push_back(edge_m1_m5_2); + + auto new_edge = cost_graph.EliminationEdges(edges); + ASSERT_EQ(new_edge.get(), matmul1->succ_edges()[0].get()); + ASSERT_EQ(new_edge.get(), matmul5->prev_edges()[0].get()); +} + +TEST_F(TestCostGraph, test_SearchStrategy) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul2); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + + ASSERT_EQ(edge_m1_m2->InitEdgeCost(), SUCCESS); + cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + cost_graph.SearchStrategy(); +} + +TEST_F(TestCostGraph, test_SearchStrategyV2) { + std::string edge_name = "MatMul-MatMul"; + std::shared_ptr edge_m1_m2 = std::make_shared(edge_name, matmul1, matmul2, 0, 0, false); + matmul1->GenerateStrategies(0); + matmul2->GenerateStrategies(0); + cost_graph.AddOperator(matmul1); + cost_graph.AddOperator(matmul2); + + matmul1->AddSuccEdge(edge_m1_m2); + matmul2->AddPrevEdge(edge_m1_m2); + + ASSERT_EQ(edge_m1_m2->InitEdgeCost(), SUCCESS); + cost_graph.AddEdge(matmul1, matmul2, edge_m1_m2); + + cost_graph.EliminationMerge(matmul1); + cost_graph.SearchStrategy(); + + auto decision = matmul2->selected_cost()->decision_ptr_->cast(); + + matmul1->SetSelectedStrategyAndCost(decision->merged_op_strategy_, decision->merged_op_cost_); + edge_m1_m2->set_selected_cost(decision->edge_cost_); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/auto_parallel/operator_costmodel_test.cc b/tests/ut/cpp/parallel/auto_parallel/operator_costmodel_test.cc new file mode 100644 index 00000000000..3b71c80c4b4 --- /dev/null +++ b/tests/ut/cpp/parallel/auto_parallel/operator_costmodel_test.cc @@ -0,0 +1,194 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/tensor_layout/tensor_info.h" +#include "parallel/auto_parallel/operator_costmodel.h" +#include "parallel/device_manager.h" + +namespace mindspore { +namespace parallel { + +class TestMatMulCost : public UT::Common { + public: + TestMatMulCost() {} + void SetUp(); + void TearDown(); + MatMulCost mmcost_; +}; + +void TestMatMulCost::SetUp() { + mmcost_ = MatMulCost(); + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); +} + +void TestMatMulCost::TearDown() { + // destroy resources +} + +TEST_F(TestMatMulCost, test_CostGeneration) { + // Currently, the implementation of GetForwardCommCost() method + // does not check the tensor layouts information, instead, it checks the + // tensor shape and the slice shape. + TensorLayout input0_layout, input1_layout, output0_layout; + Shape input0_shape{200, 300}, input1_shape{300, 500}, output0_shape{200, 500}; + Shape input0_slice_shape{20, 50}, input1_slice_shape{50, 25}, output0_slice_shape{20, 25}; + TensorInfo input0(input0_layout, input0_shape, input0_slice_shape), + input1(input1_layout, input1_shape, input1_slice_shape), + output0(output0_layout, output0_shape, output0_slice_shape); + + std::vector inputs, outputs; + inputs.push_back(input0); + inputs.push_back(input1); + outputs.push_back(output0); + + mmcost_.set_is_parameter({false, false}); + std::vector inputs_length = {4, 4}; + std::vector outputs_length = {4}; + mmcost_.SetInputAndOutputTypeLength(inputs_length, outputs_length); + mmcost_.GetForwardCommCost(inputs, outputs, 0); + mmcost_.GetBackwardCommCost(inputs, outputs, 0); + mmcost_.GetForwardMemoryCost(inputs, outputs, 0); + mmcost_.GetBackwardMemoryCost(inputs, outputs, 0); +} + +class TestActivationCost : public UT::Common { + public: + TestActivationCost() {} + void SetUp(); + void TearDown(); + ActivationCost ac_cost_; +}; + +void TestActivationCost::SetUp() { + ac_cost_ = ActivationCost(); + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); +} + +void TestActivationCost::TearDown() { + // destroy resources +} + +TEST_F(TestActivationCost, test_CostGeneration) { + // Currently, the implementation of GetForwardCommCost() method + // does not check the tensor layouts information, instead, it checks the + // tensor shape and the slice shape. + TensorLayout input0_layout, output0_layout; + Shape input0_shape{200, 300}, output0_shape{200, 300}; + Shape input0_slice_shape{20, 30}, output0_slice_shape{20, 30}; + TensorInfo input0_info(input0_layout, input0_shape, input0_slice_shape), + output0_info(output0_layout, output0_shape, output0_slice_shape); + std::vector inputs, outputs; + inputs.push_back(input0_info); + outputs.push_back(output0_info); + + ac_cost_.set_is_parameter({false, false}); + std::vector inputs_length = {4, 4}; + std::vector outputs_length = {4}; + ac_cost_.SetInputAndOutputTypeLength(inputs_length, outputs_length); + ac_cost_.GetForwardMemoryCost(inputs, outputs, 0); + ac_cost_.GetBackwardMemoryCost(inputs, outputs, 0); +} + +class TestPReLUCost : public UT::Common { + public: + TestPReLUCost() {} + void SetUp(); + void TearDown(); + PReLUCost prelu_cost_; +}; + +void TestPReLUCost::SetUp() { + prelu_cost_ = PReLUCost(); + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); +} + +void TestPReLUCost::TearDown() { + // destroy resources +} + +TEST_F(TestPReLUCost, test_CostGeneration) { + TensorLayout input_layout, param_layout, output_layout; + Shape input_shape = {32, 32, 32, 32}; + Shape param_shape = {32}; + Shape output_shape = {32, 32, 32, 32}; + Shape input_slice_shape = {8, 32, 8, 8}; + Shape param_slice_shape = {32}; + Shape output_slice_shape = {8, 32, 8, 8}; + TensorInfo input_info(input_layout, input_shape, input_slice_shape); + TensorInfo param_info(param_layout, param_shape, param_slice_shape); + TensorInfo output_info(output_layout, output_shape, output_slice_shape); + std::vector inputs, outputs; + inputs.push_back(input_info); + inputs.push_back(param_info); + outputs.push_back(output_info); + prelu_cost_.set_is_parameter({false, true}); + std::vector inputs_length = {4, 4}; + std::vector outputs_length = {4}; + prelu_cost_.SetInputAndOutputTypeLength(inputs_length, outputs_length); + double BCC, FMC, GMC; + BCC = prelu_cost_.GetBackwardCommCost(inputs, outputs, 0); + FMC = prelu_cost_.GetForwardMemoryCost(inputs, outputs, 0); + GMC = prelu_cost_.GetBackwardMemoryCost(inputs, outputs, 0); + ASSERT_EQ(BCC, 32 * 4); + ASSERT_EQ(FMC, 8 * 32 * 8 * 8 * 4 + 32 * 4); + ASSERT_EQ(GMC, 128); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/auto_parallel/rec_partition_test.cc b/tests/ut/cpp/parallel/auto_parallel/rec_partition_test.cc new file mode 100644 index 00000000000..509b00f4280 --- /dev/null +++ b/tests/ut/cpp/parallel/auto_parallel/rec_partition_test.cc @@ -0,0 +1,259 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "parallel/auto_parallel/rec_core/rec_tensor.h" +#include "parallel/auto_parallel/rec_core/rec_graph.h" +#include "parallel/auto_parallel/rec_core/rec_partition.h" +#include +#include "ir/value.h" + +namespace mindspore { +namespace parallel { +#define ARRAY_A 3000 // also 'I' :height of the first input tensor +#define ARRAY_B 1000 // also 'K' :used by both input tensor +#define ARRAY_C 4000 // also 'J' :width of the first input tensor + +class TestPartition : public UT::Common { + public: + void Create(std::shared_ptr graph, int node_num, std::vector edge_head, std::vector edge_tail); + void InitEdge(std::shared_ptr graph, int vHead, int vTail); + void InitNode(std::shared_ptr graph, int num_node); + TensorParam *MakeTensor(int n, int c, int h, int w); + std::shared_ptr MakeMatMulData(int numNode); +}; + +// Local function to create test input graph with nodes +void TestPartition::Create(std::shared_ptr graph, int node_num, std::vector edge_head, + std::vector edge_tail) { + TestPartition::InitNode(graph, node_num); + unsigned int edge_num = edge_head.size(); + if (edge_num != edge_tail.size()) { + exit(1); + }; + + for (unsigned int i = 0; i < edge_num; i++) { + TestPartition::InitEdge(graph, edge_head[i], edge_tail[i]); + }; +} + +// Local function for Create() to crate Node +void TestPartition::InitNode(std::shared_ptr graph, int num_node) { + Graph::NodeType NewNode; + for (int i = 0; i < num_node; i++) { + graph->nodes.push_back(NewNode); + std::stringstream ss; + ss << 'N' << i; + graph->nodes[i].name = ss.str(); + graph->nodes[i].info = kConstant; + }; +} + +// Local function for Create() to crate Edge +void TestPartition::InitEdge(std::shared_ptr graph, int vHead, int vTail) { + graph->nodes[vHead].node_out.push_back(vTail); + graph->nodes[vTail].node_in.push_back(vHead); +} + +// Local function for Create() to crate Tensor +TensorParam *TestPartition::MakeTensor(int n, int c, int h, int w) { + TensorParam *p_tensor = new TensorParam; + p_tensor->tensor_type = kFloat32; + p_tensor->tensor_shape.shape_n = n; + p_tensor->tensor_shape.shape_c = c; + p_tensor->tensor_shape.shape_h = h; + p_tensor->tensor_shape.shape_w = w; + + return p_tensor; +}; + +// Local function for Create() to create MatMul Operator +// @numNode include Tensor and Operator, for example 4(1 Input Tensor, 1 Input Tensor, 1 Operator, 1 Output Tensor) +std::shared_ptr TestPartition::MakeMatMulData(int numNode) { + // Build Edges + int edgeNum = 0; + if (0 == numNode % 2 && numNode != 0) { + edgeNum = numNode - 2; + } else if (1 == numNode % 2) { + edgeNum = numNode - 1; + } else { + edgeNum = 0; + }; + + std::vector edgeHead(edgeNum); // int edgeHead[8] = {0,2,4,6,1,3,5,7}; + std::vector edgeTail(edgeNum); // int edgeTail[8] = {2,4,6,8,2,4,6,8}; + + for (int i = 0; i < edgeNum; i++) { + edgeHead[i] = i; + if (0 == i % 2) { + edgeTail[i] = i + 2; + } else { + edgeTail[i] = i + 1; + }; + }; + + // Creat graph + std::shared_ptr graph(new Graph); + TestPartition::Create(graph, numNode, edgeHead, edgeTail); + + // Add Node information. + for (int i = 0; i < numNode; i++) { + if (0 == i) { + graph->nodes[i].info = InfoType::kConstant; + TensorParam *p_tensor_out = new TensorParam; + p_tensor_out->tensor_type = kFloat32; + p_tensor_out->tensor_shape.shape_w = ARRAY_B; + p_tensor_out->tensor_shape.shape_h = ARRAY_A; + + graph->nodes[i].tensor_parm = *p_tensor_out; + + } else if (0 == i % 4) { + graph->nodes[i].info = InfoType::kApplication; + graph->nodes[i].apply.op_type = OperatorType::kRecMatMul; + + TensorParam *p_tensor0 = new TensorParam; + p_tensor0->tensor_type = kFloat32; + p_tensor0->tensor_shape.shape_w = ARRAY_C; + p_tensor0->tensor_shape.shape_h = ARRAY_A; + + TensorParam *p_tensor1 = new TensorParam; + p_tensor1->tensor_type = kFloat32; + p_tensor1->tensor_shape.shape_w = ARRAY_B; + p_tensor1->tensor_shape.shape_h = ARRAY_C; + + TensorParam *p_tensor_out = new TensorParam; + p_tensor_out->tensor_type = kFloat32; + p_tensor_out->tensor_shape.shape_w = ARRAY_B; + p_tensor_out->tensor_shape.shape_h = ARRAY_A; + + graph->nodes[i].apply.arguments[0] = *p_tensor0; + graph->nodes[i].apply.arguments[1] = *p_tensor1; + graph->nodes[i].tensor_parm = *p_tensor_out; + + } else if (1 == i % 4) { + graph->nodes[i].info = InfoType::kConstant; + + TensorParam *p_tensor_out = new TensorParam; + p_tensor_out->tensor_type = kFloat32; + p_tensor_out->tensor_shape.shape_w = ARRAY_C; + p_tensor_out->tensor_shape.shape_h = ARRAY_B; + + graph->nodes[i].tensor_parm = *p_tensor_out; + + } else if (2 == i % 4) { + graph->nodes[i].info = InfoType::kApplication; + graph->nodes[i].apply.op_type = OperatorType::kRecMatMul; + + TensorParam *p_tensor0 = new TensorParam; + p_tensor0->tensor_type = kFloat32; + p_tensor0->tensor_shape.shape_w = ARRAY_B; + p_tensor0->tensor_shape.shape_h = ARRAY_A; + + TensorParam *p_tensor1 = new TensorParam; + p_tensor1->tensor_type = kFloat32; + p_tensor1->tensor_shape.shape_w = ARRAY_C; + p_tensor1->tensor_shape.shape_h = ARRAY_B; + + TensorParam *p_tensor_out = new TensorParam; + p_tensor_out->tensor_type = kFloat32; + p_tensor_out->tensor_shape.shape_w = ARRAY_C; + p_tensor_out->tensor_shape.shape_h = ARRAY_A; + + graph->nodes[i].apply.arguments[0] = *p_tensor0; + graph->nodes[i].apply.arguments[1] = *p_tensor1; + graph->nodes[i].tensor_parm = *p_tensor_out; + + } else if (3 == i % 4) { + graph->nodes[i].info = InfoType::kConstant; + + TensorParam *p_tensor_out = new TensorParam; + p_tensor_out->tensor_type = kFloat32; + p_tensor_out->tensor_shape.shape_w = ARRAY_B; + p_tensor_out->tensor_shape.shape_h = ARRAY_C; + + graph->nodes[i].tensor_parm = *p_tensor_out; + }; + }; + return graph; +}; + +TEST_F(TestPartition, test_GetWeights) { + std::shared_ptr graph = MakeMatMulData(9); + double wop1 = GetWeights(graph->nodes[2]); + double wop2 = GetWeights(graph->nodes[4]); + double wop3 = GetWeights(graph->nodes[6]); + double wop4 = GetWeights(graph->nodes[8]); + ASSERT_GE(wop1, wop2); + ASSERT_GE(wop2, wop3); + ASSERT_GE(wop3, wop4); +} + +TEST_F(TestPartition, test_SortByWeight) { + std::shared_ptr graph = MakeMatMulData(9); + std::vector result = SortByWeight(graph); + ASSERT_GE(result.at(0), result.at(1)); + ASSERT_GE(result.at(1), result.at(2)); + ASSERT_GE(result.at(2), result.at(3)); +} + +TEST_F(TestPartition, test_SortByWeight2) { + std::shared_ptr graph = MakeMatMulData(5); + std::vector result = SortByWeight(graph); + ASSERT_GE(result.at(0), result.at(1)); +} + +TEST_F(TestPartition, test_PartitionNode) { + std::shared_ptr graph = MakeMatMulData(9); + // node 2 is the first kRecMatMul Operator + Graph::NodeType node2 = graph->nodes[2]; + std::vector> nameToStrategy; + StrategyRec str = PartitionNode(node2, nameToStrategy, graph); + ASSERT_EQ(str.outputTensor.str_h, 1); + ASSERT_EQ(str.outputTensor.str_w, 0.5); +} + +TEST_F(TestPartition, test_PartitionForAllDevices) { + std::shared_ptr graph = MakeMatMulData(9); + ASSERT_EQ(PartitionForAllDevices(1024, graph), SUCCESS); +} + +TEST_F(TestPartition, test_PartitionForAllDevices2) { + std::shared_ptr graph = MakeMatMulData(9); + ASSERT_EQ(PartitionForAllDevices(2, graph), SUCCESS); +} + +// Negative case: parition on 0 device +TEST_F(TestPartition, test_PartitionForAllDevices0) { + std::shared_ptr graph = MakeMatMulData(9); + // Throw Exception "Number of devices can't be 0" + EXPECT_ANY_THROW(PartitionForAllDevices(0, graph)); +} + +TEST_F(TestPartition, test_ApplyStrToTensor) { + std::shared_ptr graph = MakeMatMulData(9); + std::vector> nameToStrategy; + StrategyRec str = PartitionNode(graph->nodes[4], nameToStrategy, graph); + auto h_str = str.outputTensor.str_h; + auto w_str = str.outputTensor.str_w; + + Graph::NodeType n_node = ApplyStrToTensor(graph->nodes[4]); + auto h_node = n_node.tensor_parm.tensor_str.str_h; + auto w_node = n_node.tensor_parm.tensor_str.str_w; + ASSERT_EQ(h_str, h_node); + ASSERT_EQ(w_str, w_node); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/device_manager_test.cc b/tests/ut/cpp/parallel/device_manager_test.cc new file mode 100644 index 00000000000..0814ca64d06 --- /dev/null +++ b/tests/ut/cpp/parallel/device_manager_test.cc @@ -0,0 +1,127 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common_test.h" +#include "parallel/device.h" +#include "parallel/device_manager.h" +#include "parallel/group_manager.h" + +namespace mindspore { +namespace parallel { + +class TestDevice : public UT::Common { + public: + TestDevice() {} + void SetUp(); + void TearDown(); + Device dev_1; + Device dev_2; +}; + +void TestDevice::SetUp() { + std::string name = "#1"; + dev_1 = Device(name, std::int32_t(1)); + dev_2 = Device(std::int32_t(2)); +} + +void TestDevice::TearDown() { + // destroy resources +} + +TEST_F(TestDevice, test_device) { + std::string name = "#1"; + int32_t dev1_rank = 1; + int32_t dev2_rank = 2; + + ASSERT_STREQ(dev_1.name().data(), name.data()); + ASSERT_EQ(dev_1.rank(), dev1_rank); + ASSERT_EQ(dev_2.rank(), dev2_rank); +} + +// need to complete +class TestStage : public UT::Common {}; + +class TestDeviceManager : public UT::Common { + public: + TestDeviceManager() {} + void SetUp(); + void TearDown(); + DeviceManager dm_; +}; + +void TestDeviceManager::SetUp() { dm_ = DeviceManager::GetInstance(); } + +void TestDeviceManager::TearDown() { + // destroy resources +} + +TEST_F(TestDeviceManager, test_dm_init_AND_get_device_list) { + std::list dev_list; + std::list stage_map; + int32_t local_dev = 0; + + dev_list.push_back(5); + dev_list.push_back(3); + dev_list.push_back(1); + dev_list.push_back(0); + + stage_map.push_back(2); + stage_map.push_back(2); + ASSERT_EQ(dm_.Init(dev_list, local_dev, stage_map, "hccl"), Status::SUCCESS); + + ASSERT_EQ(dm_.DeviceNum(), 4); + ASSERT_EQ(dm_.GetStageNum(), (int32_t)(2)); + + std::list dev_list_0 = dm_.GetDeviceListByStageId(0); + std::list dev_list_1 = dm_.GetDeviceListByStageId(1); + ASSERT_EQ(dev_list_0.size(), 2); + ASSERT_EQ(dev_list_1.size(), 2); + + std::list::iterator it = dev_list_0.begin(); + ASSERT_EQ((*it), int32_t(5)); + it++; + ASSERT_EQ((*it), int32_t(3)); + it = dev_list_1.begin(); + ASSERT_EQ((*it), int32_t(1)); + it++; + ASSERT_EQ((*it), int32_t(0)); + + std::shared_ptr stage_0 = dm_.GetStageById(0); + ASSERT_EQ(stage_0->GetDevicesNum(), size_t(2)); + std::shared_ptr stage_1 = dm_.GetStageById(1); + ASSERT_EQ(stage_1->GetDevicesNum(), size_t(2)); +} + +TEST_F(TestDeviceManager, test_CreateNewDeviceByRank) { + Device one = dm_.CreateNewDeviceByRank(int32_t(3)); + ASSERT_EQ(one.rank(), int32_t(3)); +} + +TEST_F(TestDeviceManager, test_CreateDeviceListByRankList) { + std::list dev_list; + std::list rlist; + rlist.push_back(int32_t(2)); + rlist.push_back(int32_t(1)); + dev_list = dm_.CreateDeviceListByRankList(rlist); + + std::list::iterator it = dev_list.begin(); + ASSERT_EQ(it->rank(), int32_t(2)); + it++; + ASSERT_EQ(it->rank(), int32_t(1)); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/device_matrix_test.cc b/tests/ut/cpp/parallel/device_matrix_test.cc new file mode 100644 index 00000000000..9c3fa9a8614 --- /dev/null +++ b/tests/ut/cpp/parallel/device_matrix_test.cc @@ -0,0 +1,95 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/device_matrix.h" + +namespace mindspore { +namespace parallel { + +class TestDeviceMatrix : public UT::Common { + public: + TestDeviceMatrix() {} + + void SetUp() { UT::InitPythonPath(); } + + virtual void TearDown() {} +}; + +TEST_F(TestDeviceMatrix, Test2Dgroup_list) { + RankList dev_list = {0, 1, 2, 3, 4, 5}; + Shape shape = {2, 3}; + + DeviceMatrix arr(0, dev_list, shape); + std::list group_list; + if (arr.CreateGroupList() == Status::SUCCESS) group_list = arr.group_list(); + std::list group_list_expect = {{0, 3}, {0, 1, 2}}; + ASSERT_EQ(group_list, group_list_expect); +} + +TEST_F(TestDeviceMatrix, Test3Dgroup_list) { + RankList dev_list = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + Shape shape = {2, 2, 3}; + + DeviceMatrix arr(5, dev_list, shape); + std::list group_list; + if (arr.CreateGroupList() == Status::SUCCESS) group_list = arr.group_list(); + std::list group_list_expect = {{5, 11}, {2, 5}, {3, 4, 5}}; + ASSERT_EQ(group_list, group_list_expect); +} + +TEST_F(TestDeviceMatrix, Test4DGetAlongDim) { + RankList dev_list = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + Shape shape = {2, 1, 4, 2}; + + DeviceMatrix arr(5, dev_list, shape); + std::list group_list; + if (arr.CreateGroupList() == Status::SUCCESS) group_list = arr.group_list(); + std::list group_list_expect = {{5, 13}, {5}, {1, 3, 5, 7}, {4, 5}}; + ASSERT_EQ(group_list, group_list_expect); +} + +TEST_F(TestDeviceMatrix, Test5DGetAlongDim) { + RankList dev_list; + for (int i = 0; i < 144; i++) dev_list.push_back(i); + Shape shape = {3, 4, 2, 3, 2}; + + DeviceMatrix arr(5, dev_list, shape); + std::list group_list; + if (arr.CreateGroupList() == Status::SUCCESS) group_list = arr.group_list(); + std::list group_list_expect = {{5, 53, 101}, {5, 17, 29, 41}, {5, 11}, {1, 3, 5}, {4, 5}}; + ASSERT_EQ(group_list, group_list_expect); +} + +TEST_F(TestDeviceMatrix, TestCornerCaseGetAlongDim) { + // Shape does not match the number of devices + RankList dev_list = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + Shape shape = {2, 2, 2}; + + EXPECT_THROW({ DeviceMatrix arr(3, dev_list, shape); }, std::runtime_error); +} + +TEST_F(TestDeviceMatrix, TestCornerCase2GetAlongDim) { + // Rank is out of range + RankList dev_list = {0, 1, 2, 3, 4, 5, 6, 7}; + Shape shape = {2, 2, 2}; + + EXPECT_THROW({ DeviceMatrix arr(8, dev_list, shape); }, std::runtime_error); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/group_manager_test.cc b/tests/ut/cpp/parallel/group_manager_test.cc new file mode 100644 index 00000000000..1e9ec9c0608 --- /dev/null +++ b/tests/ut/cpp/parallel/group_manager_test.cc @@ -0,0 +1,155 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "parallel/device_manager.h" +#include "common/common_test.h" +#include "parallel/device.h" +#include "parallel/group_manager.h" + +namespace mindspore { +namespace parallel { + +extern DeviceManagerPtr g_device_manager; + +class TestGroup : public UT::Common { + public: + TestGroup() {} + void SetUp(); + void TearDown(); + Status Init(); + + Group gp; +}; + +void TestGroup::SetUp() { gp = Group(); } + +void TestGroup::TearDown() { + // destroy resources +} + +Status TestGroup::Init() { + std::string gname = "1-2"; + std::list dev_list; + Device one = Device(int32_t(1)); + dev_list.push_back(one); + Device two = Device(int32_t(2)); + dev_list.push_back(two); + + return gp.Init(gname, dev_list); +} + +TEST_F(TestGroup, test_Init) { ASSERT_EQ(Init(), Status::SUCCESS); } + +TEST_F(TestGroup, test_GetDevicesList) { + Init(); + std::list res_dev_list = gp.GetDevicesList(); + std::list::iterator it = res_dev_list.begin(); + ASSERT_EQ(it->rank(), int32_t(1)); + it++; + ASSERT_EQ(it->rank(), int32_t(2)); +} + +TEST_F(TestGroup, test_IsInThisGroup) { + Init(); + ASSERT_TRUE(gp.IsInThisGroup(int32_t(1))); + ASSERT_TRUE(gp.IsInThisGroup(int32_t(2))); + + ASSERT_FALSE(gp.IsInThisGroup(int32_t(3))); +} + +class TestGroupManager : public UT::Common { + public: + TestGroupManager() {} + void SetUp(); + void TearDown(); + Status Init(Group** gp_ptr); + + GroupManager gm; +}; + +void TestGroupManager::SetUp() { gm = GroupManager(); } + +void TestGroupManager::TearDown() { + // destroy resources +} + +Status TestGroupManager::Init(Group** gp_ptr) { + std::string gname = "1-2"; + std::list dev_list; + Device one = Device(int32_t(1)); + dev_list.push_back(one); + Device two = Device(int32_t(2)); + dev_list.push_back(two); + + return gm.CreateGroup(gname, dev_list, *gp_ptr); +} + +TEST_F(TestGroupManager, test_CreateGroup) { + // testing for creating a group + Group* gp_ptr = new Group(); + ASSERT_EQ(Init(&gp_ptr), Status::SUCCESS); + + std::list res_dev_list = gp_ptr->GetDevicesList(); + std::list::iterator it = res_dev_list.begin(); + ASSERT_EQ(it->rank(), int32_t(1)); + it++; + ASSERT_EQ(it->rank(), int32_t(2)); + delete gp_ptr; + + // testing for creating a group with an existing group name + std::list dev_list2; + Device three = Device(int32_t(3)); + dev_list2.push_back(three); + Device four = Device(int32_t(4)); + dev_list2.push_back(four); + gp_ptr = new Group(); + ASSERT_EQ(gm.CreateGroup("1-2", dev_list2, gp_ptr), Status::SUCCESS); + + ASSERT_STREQ(gp_ptr->name().data(), "1-2"); + std::list res_dev_list2 = gp_ptr->GetDevicesList(); + std::list::iterator it2 = res_dev_list2.begin(); + ASSERT_EQ(it2->rank(), int32_t(1)); + it2++; + ASSERT_EQ(it2->rank(), int32_t(2)); + delete gp_ptr; + gp_ptr = nullptr; +} + +TEST_F(TestGroupManager, test_FindGroup) { + std::string gname = "1-2"; + Group* gp_ptr = new Group(); + Group* gp_ptr2 = new Group(); + ASSERT_EQ(Init(&gp_ptr), Status::SUCCESS); + + ASSERT_EQ(gm.FindGroup(gname, &gp_ptr2), Status::SUCCESS); + + std::list res_dev_list = gp_ptr2->GetDevicesList(); + std::list::iterator it = res_dev_list.begin(); + ASSERT_EQ(it->rank(), int32_t(1)); + it++; + ASSERT_EQ(it->rank(), int32_t(2)); + delete gp_ptr; + gp_ptr = nullptr; + + std::string gname2 = "3-4"; + gp_ptr2 = new Group(); + ASSERT_EQ(gm.FindGroup(gname2, &gp_ptr2), Status::FAILED); + delete gp_ptr2; + gp_ptr2 = nullptr; +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/activation_info_test.cc b/tests/ut/cpp/parallel/ops_info/activation_info_test.cc new file mode 100644 index 00000000000..a725cb0bf78 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/activation_info_test.cc @@ -0,0 +1,181 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class ActivationInfo; +using ActivationInfoPtr = std::shared_ptr; +ActivationInfoPtr activation; + +class TestActivationInfo : public UT::Common { + public: + TestActivationInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestActivationInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr relu = MakeValue(std::string("relu")); + std::unordered_map attr = {{"activation_type", relu}}; + + Shapes inputs_shape = {{2, 4, 8, 16}}; + Shapes outputs_shape = {{2, 4, 8, 16}}; + + activation = std::make_shared("activation_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestActivationInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + activation->Init(strategy); + std::vector dev_matrix_shape = activation->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestActivationInfo, InferSliceShape1) { + std::vector str = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + activation->Init(strategy); + std::vector inputs = activation->inputs_tensor_info(); + std::vector outputs = activation->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 1, 1}; + Shape output_slice_shape_expect = {1, 1, 1, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestActivationInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + activation->Init(strategy); + std::vector inputs = activation->inputs_tensor_info(); + std::vector outputs = activation->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestActivationInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + activation->Init(strategy); + OperatorVector forward_op = activation->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestActivationInfo, GetMirrorOPs1) { + std::vector inputs = {{1, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + activation->Init(strategy); + MirrorOps mirror_ops = activation->mirror_ops(); + + OperatorVector mirror_op = mirror_ops.at(0); + + OperatorArgs operator_args = mirror_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +TEST_F(TestActivationInfo, GetMirrorOPs2) { + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + activation->Init(strategy); + MirrorOps mirror_ops = activation->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestActivationInfo, CheckStrategy1) { + // Success: {{2,4,8,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = activation->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestActivationInfo, CheckStrategy2) { + // Success: {{2,4,8,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = activation->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/activation_test.cc b/tests/ut/cpp/parallel/ops_info/activation_test.cc new file mode 100644 index 00000000000..200b96eb2e5 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/activation_test.cc @@ -0,0 +1,120 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/device_manager.h" + +namespace mindspore { +namespace parallel { + +class Activation; +class Softmax; +using ActivationPtr = std::shared_ptr; +using SoftmaxPtr = std::shared_ptr; +ActivationPtr act_ptr_; +SoftmaxPtr soft_ptr_; + +class TestActivation : public UT::Common { + public: + TestActivation() {} + void SetUp(); + void TearDown() {} +}; + +void TestActivation::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr relu = MakeValue(std::string("relu")); + std::unordered_map relu_attr = {{"activation_type", relu}}; + ValuePtr sm = MakeValue(std::string("softmax")); + ValuePtr axix = MakeValue(std::int32_t(2)); + std::unordered_map softmax_attr = {{"activation_type", sm}, {"axis", axix}}; + + Shapes relu_inputs_shape = {{2, 4, 8, 16}}; + Shapes relu_outputs_shape = {{2, 4, 8, 16}}; + Shapes sm_inputs_shape = {{8, 8, 8, 16}}; + Shapes sm_outputs_shape = {{8, 8, 8, 16}}; + + act_ptr_ = std::make_shared("relu_info", relu_inputs_shape, relu_outputs_shape, relu_attr); + soft_ptr_ = std::make_shared("softmax_info", sm_inputs_shape, sm_outputs_shape, softmax_attr); +} + +TEST_F(TestActivation, test_activation_strategies) { + ASSERT_EQ(act_ptr_->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = act_ptr_->GetStrategyCost(); + for (const auto& swc : sc) { + ASSERT_NE(swc, nullptr); + ASSERT_GT(swc->cost_list.size(), 0); + StrategyPtr sp = swc->strategy_ptr; + ASSERT_NE(sp, nullptr); + Cost cost = *(swc->cost_list[0]); + + act_ptr_->InitForCostModel(sp); + std::vector inputs_info = act_ptr_->inputs_tensor_info(); + std::vector outputs_info = act_ptr_->outputs_tensor_info(); + ASSERT_DOUBLE_EQ(act_ptr_->GetOperatorCost()->GetMemoryCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.memory_cost_); + ASSERT_DOUBLE_EQ(act_ptr_->GetOperatorCost()->GetCommCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.communication_cost_); + } +} + +TEST_F(TestActivation, test_softmax_strategies) { + ASSERT_EQ(soft_ptr_->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = soft_ptr_->GetStrategyCost(); + for (const auto& swc : sc) { + ASSERT_NE(swc, nullptr); + ASSERT_GT(swc->cost_list.size(), 0); + StrategyPtr sp = swc->strategy_ptr; + ASSERT_NE(sp, nullptr); + Cost cost = *(swc->cost_list[0]); + + std::vector stra = sp->GetInputDim(); + ASSERT_GT(stra.size(), 0); + Dimensions input0_stra = stra[0]; + ASSERT_GT(input0_stra.size(), 2); + ASSERT_EQ(input0_stra[2], 1); + soft_ptr_->InitForCostModel(sp); + std::vector inputs_info = soft_ptr_->inputs_tensor_info(); + std::vector outputs_info = soft_ptr_->outputs_tensor_info(); + ASSERT_DOUBLE_EQ(soft_ptr_->GetOperatorCost()->GetMemoryCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.memory_cost_); + ASSERT_DOUBLE_EQ(soft_ptr_->GetOperatorCost()->GetCommCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.communication_cost_); + } +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/dropout_do_mask_info_test.cc b/tests/ut/cpp/parallel/ops_info/dropout_do_mask_info_test.cc new file mode 100644 index 00000000000..5ae79e2851e --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/dropout_do_mask_info_test.cc @@ -0,0 +1,166 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/dropout_do_mask_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class DropoutDoMaskInfo; +using DropoutDoMaskInfoPtr = std::shared_ptr; +DropoutDoMaskInfoPtr do_mask; + +class TestDropoutDoMaskInfo : public UT::Common { + public: + TestDropoutDoMaskInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestDropoutDoMaskInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 34; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(32); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{32, 128}, {64}, {}}; + Shapes outputs_shape = {{32, 128}}; + do_mask = std::make_shared("do_mask_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestDropoutDoMaskInfo, InferDevMatrixShape) { + std::vector stra = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + do_mask->Init(strategy); + std::vector dev_matrix_shape = do_mask->dev_matrix_shape(); + + std::vector expect = {4, 8}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestDropoutDoMaskInfo, InferSliceShape) { + std::vector stra = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + do_mask->Init(strategy); + std::vector inputs = do_mask->inputs_tensor_info(); + std::vector outputs = do_mask->outputs_tensor_info(); + + Shape input_a_slice_shape_expect = {8, 16}; + Shape input_b_slice_shape_expect = {64}; + Shape output_slice_shape_expect = {8, 16}; + + TensorInfo input_a_tensor_info = inputs.at(0); + TensorInfo input_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + Shape input_a_slice_shape = input_a_tensor_info.slice_shape(); + Shape input_b_slice_shape = input_b_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_a_slice_shape, input_a_slice_shape_expect); + ASSERT_EQ(input_b_slice_shape, input_b_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestDropoutDoMaskInfo, GetTensorLayout) { + std::vector stra = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + do_mask->Init(strategy); + std::vector inputs = do_mask->inputs_tensor_info(); + std::vector outputs = do_mask->outputs_tensor_info(); + + TensorMap input_a_map_expect = {1, 0}; + TensorMap input_b_map_expect = {-1}; + TensorMap output_map_expect = {1, 0}; + + TensorInfo input_a_tensor_info = inputs.at(0); + TensorInfo input_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + Map input_a_tensor_map = input_a_tensor_info.tensor_layout().origin_tensor_map(); + Map input_b_tensor_map = input_b_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_a_tensor_map.array(), input_a_map_expect); + ASSERT_EQ(input_b_tensor_map.array(), input_b_map_expect); + ASSERT_EQ(output_tensor_map.array(), output_map_expect); +} + +TEST_F(TestDropoutDoMaskInfo, GetForwardOp) { + std::vector stra = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + do_mask->Init(strategy); + OperatorVector forward_op = do_mask->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestDropoutDoMaskInfo, CheckStrategy1) { + std::vector stra = {{4, 8, 2}}; + StrategyPtr strategy = NewStrategy(0, stra); + + Status ret = do_mask->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestDropoutDoMaskInfo, CheckStrategy2) { + std::vector stra = {{8, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + Status ret = do_mask->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestDropoutDoMaskInfo, CheckStrategy3) { + std::vector stra = {{4, 8}, {4, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + Status ret = do_mask->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestDropoutDoMaskInfo, CheckStrategy4) { + std::vector stra = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, stra); + + Status ret = do_mask->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/gelu_info_test.cc b/tests/ut/cpp/parallel/ops_info/gelu_info_test.cc new file mode 100644 index 00000000000..d07794f63d6 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/gelu_info_test.cc @@ -0,0 +1,169 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class GeluInfo; +using GeluInfoPtr = std::shared_ptr; +GeluInfoPtr gelu; + +class TestGeluInfo : public UT::Common { + public: + TestGeluInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestGeluInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 130; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(128); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{2, 4, 8, 16}}; + Shapes outputs_shape = {{2, 4, 8, 16}}; + + gelu = std::make_shared("gelu_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestGeluInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + gelu->Init(strategy); + std::vector dev_matrix_shape = gelu->dev_matrix_shape(); + + std::vector expect = {2, 4, 1, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestGeluInfo, InferSliceShape1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + gelu->Init(strategy); + std::vector inputs = gelu->inputs_tensor_info(); + std::vector outputs = gelu->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 8, 1}; + Shape output_slice_shape_expect = {1, 1, 8, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestGeluInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + gelu->Init(strategy); + std::vector inputs = gelu->inputs_tensor_info(); + std::vector outputs = gelu->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestGeluInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + gelu->Init(strategy); + OperatorVector forward_op = gelu->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestGeluInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + gelu->Init(strategy); + MirrorOps mirror_ops = gelu->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestGeluInfo, CheckStrategy1) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = gelu->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestGeluInfo, CheckStrategy2) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = gelu->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestGeluInfo, CheckStrategy3) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = gelu->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/generate_strategy_test.cc b/tests/ut/cpp/parallel/ops_info/generate_strategy_test.cc new file mode 100644 index 00000000000..81b1d3a1ec8 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/generate_strategy_test.cc @@ -0,0 +1,167 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/arithmetic_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class TestGenerateStrategy : public UT::Common { + public: + TestGenerateStrategy() {} + void SetUp(); + void TearDown() {} +}; + +void TestGenerateStrategy::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); +} + +TEST_F(TestGenerateStrategy, AutoStrategy1) { + std::vector sp_vector; + Shapes inputs_shape = {{64, 32}, {64, 32}}; + Shapes splittable_inputs = {{1, 1}, {1, 1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size(), 5); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input0_strategy, input1_strategy); + } +} + +TEST_F(TestGenerateStrategy, AutoStrategy2) { + std::vector sp_vector; + Shapes inputs_shape = {{32}, {64, 32}}; + Shapes splittable_inputs = {{1}, {1, 1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size(), 5); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input0_strategy[0], input1_strategy[1]); + ASSERT_EQ(input0_strategy.size(), 1); + ASSERT_EQ(input1_strategy.size(), 2); + } +} + +TEST_F(TestGenerateStrategy, AutoStrategy3) { + std::vector sp_vector; + Shapes inputs_shape = {{64, 32}, {32}}; + Shapes splittable_inputs = {{1, 1}, {1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size(), 5); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input0_strategy[1], input1_strategy[0]); + ASSERT_EQ(input0_strategy.size(), 2); + ASSERT_EQ(input1_strategy.size(), 1); + } +} + +TEST_F(TestGenerateStrategy, AutoStrategy4) { + std::vector sp_vector; + Shapes inputs_shape = {{64, 1}, {1, 32}}; + Shapes splittable_inputs = {{1, 1}, {1, 1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size(), 5); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input0_strategy[1], 1); + ASSERT_EQ(input1_strategy[0], 1); + } +} + +TEST_F(TestGenerateStrategy, AutoStrategy5) { + std::vector sp_vector; + Shapes inputs_shape = {{64, 8, 1}, {1, 8, 32}}; + Shapes splittable_inputs = {{1, 1, 1}, {1, 1, 1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size(), 11); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input0_strategy[2], 1); + ASSERT_EQ(input1_strategy[0], 1); + ASSERT_EQ(input0_strategy[1], input1_strategy[1]); + } +} + +TEST_F(TestGenerateStrategy, AutoStrategy6) { + std::vector sp_vector; + Shapes inputs_shape = {{64, 32}}; + Shapes splittable_inputs = {{1, 1}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size(), 5); +} + +TEST_F(TestGenerateStrategy, AutoStrategy7) { + std::vector sp_vector; + Shapes inputs_shape = {{1, 32}, {64, 8, 32}}; + Shapes splittable_inputs = {{1, 1}, {1, 1, 1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size() > 0, true); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input0_strategy[0], 1); + ASSERT_EQ(input0_strategy.size(), 2); + ASSERT_EQ(input1_strategy.size(), 3); + ASSERT_EQ(input0_strategy[1], input1_strategy[2]); + } +} + +TEST_F(TestGenerateStrategy, AutoStrategy8) { + std::vector sp_vector; + Shapes inputs_shape = {{64, 8, 32}, {1, 32}}; + Shapes splittable_inputs = {{1, 1, 1}, {1, 1}}; + GenerateStrategiesWithBroadcast(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sp_vector.size() > 0, true); + for (auto& sp : sp_vector) { + Dimensions input0_strategy = sp->GetInputDim()[0]; + Dimensions input1_strategy = sp->GetInputDim()[1]; + ASSERT_EQ(input1_strategy[0], 1); + ASSERT_EQ(input0_strategy.size(), 3); + ASSERT_EQ(input1_strategy.size(), 2); + ASSERT_EQ(input0_strategy[2], input1_strategy[1]); + } +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/generator_info_test.cc b/tests/ut/cpp/parallel/ops_info/generator_info_test.cc new file mode 100644 index 00000000000..bfd14ced30d --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/generator_info_test.cc @@ -0,0 +1,137 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/generator_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class DropoutGenMaskInfo; +using DropoutGenMaskInfoPtr = std::shared_ptr; +DropoutGenMaskInfoPtr gen_mask; + +class TestDropoutGenMaskInfo : public UT::Common { + public: + TestDropoutGenMaskInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestDropoutGenMaskInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape; + Shapes outputs_shape = {{128}}; + std::vector shape = {32, 128}; + ValuePtr val0 = MakeValue(shape); + ValuePtr val1; + std::vector val = {val0, val1}; + gen_mask = std::make_shared("gen_mask_info", inputs_shape, outputs_shape, attr); + gen_mask->set_input_value(val); +} + +TEST_F(TestDropoutGenMaskInfo, InferDevMatrixShape) { + std::vector stra = {{8, 1}}; + StrategyPtr strategy = NewStrategy(0, stra); + + gen_mask->Init(strategy); + std::vector dev_matrix_shape = gen_mask->dev_matrix_shape(); + + std::vector expect = {8, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestDropoutGenMaskInfo, InferSliceShape) { + std::vector stra = {{8, 1}}; + StrategyPtr strategy = NewStrategy(0, stra); + + gen_mask->Init(strategy); + std::vector outputs = gen_mask->outputs_tensor_info(); + + Shape output_slice_shape_expect = {128}; + + TensorInfo output_tensor_info = outputs.at(0); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestDropoutGenMaskInfo, GetTensorLayout) { + std::vector stra = {{8, 1}}; + StrategyPtr strategy = NewStrategy(0, stra); + + gen_mask->Init(strategy); + std::vector outputs = gen_mask->outputs_tensor_info(); + + TensorMap output_map_expect = {-1}; + + TensorInfo output_tensor_info = outputs.at(0); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(output_tensor_map.array(), output_map_expect); +} + +TEST_F(TestDropoutGenMaskInfo, GetForwardOp) { + std::vector stra = {{8, 1}}; + StrategyPtr strategy = NewStrategy(0, stra); + + gen_mask->Init(strategy); + OperatorVector forward_op = gen_mask->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestDropoutGenMaskInfo, CheckStrategy1) { + std::vector stra = {{4, 8, 2}, {2, 3}}; + StrategyPtr strategy = NewStrategy(0, stra); + + Status ret = gen_mask->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestDropoutGenMaskInfo, CheckStrategy2) { + std::vector stra = {{8, 1}}; + StrategyPtr strategy = NewStrategy(0, stra); + + Status ret = gen_mask->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/get_next_info_test.cc b/tests/ut/cpp/parallel/ops_info/get_next_info_test.cc new file mode 100644 index 00000000000..1276a302257 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/get_next_info_test.cc @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/get_next_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class GetNextInfo; +using GetNextInfoPtr = std::shared_ptr; +GetNextInfoPtr get_next; + +class TestGetNextInfo : public UT::Common { + public: + TestGetNextInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestGetNextInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 8; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + int32_t local_dev = 0; + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + Shapes inputs_shape = {}; + Shapes outputs_shape = {{64, 32}, {64}}; + std::unordered_map attr; + std::vector types_ = {"float32", "int32"}; + Shapes shapes_ = {{64, 32}, {64}}; + int32_t output_num_ = 2; + std::string shared_name_ = "test_get_next"; + attr["types"] = MakeValue(types_); + attr["shapes"] = MakeValue(shapes_); + attr["output_num"] = MakeValue(output_num_); + attr["shared_name"] = MakeValue(shared_name_); + get_next = std::make_shared("get_next_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestGetNextInfo, InferDevMatrixShape1) { + std::vector inputs = {{}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + get_next->Init(strategy); + std::vector dev_matrix_shape = get_next->dev_matrix_shape(); + std::vector expect = {8, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestGetNextInfo, InferSliceShape1) { + std::vector str = {{}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + get_next->Init(strategy); + std::vector outputs = get_next->outputs_tensor_info(); + Shape output_slice_shape_expect0 = {8, 32}; + Shape output_slice_shape_expect1 = {8}; + TensorInfo output_tensor_info0 = outputs.at(0); + TensorInfo output_tensor_info1 = outputs.at(1); + Shape output_slice_shape0 = output_tensor_info0.slice_shape(); + Shape output_slice_shape1 = output_tensor_info1.slice_shape(); + ASSERT_EQ(output_slice_shape0, output_slice_shape_expect0); + ASSERT_EQ(output_slice_shape1, output_slice_shape_expect1); +} + +TEST_F(TestGetNextInfo, GetTensorLayout1) { + std::vector str = {{}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + get_next->Init(strategy); + std::vector outputs = get_next->outputs_tensor_info(); + TensorMap output_expect0 = {1, 0}; + TensorMap output_expect1 = {1}; + TensorInfo output_tensor_info0 = outputs.at(0); + TensorInfo output_tensor_info1 = outputs.at(1); + + Map output_tensor_map0 = output_tensor_info0.tensor_layout().origin_tensor_map(); + Map output_tensor_map1 = output_tensor_info1.tensor_layout().origin_tensor_map(); + ASSERT_EQ(output_tensor_map0.array(), output_expect0); + ASSERT_EQ(output_tensor_map1.array(), output_expect1); +} + +TEST_F(TestGetNextInfo, CheckStrategy1) { + std::vector inputs = {}; + StrategyPtr strategy = NewStrategy(0, inputs); + Status ret = get_next->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +TEST_F(TestGetNextInfo, CheckStrategy2) { + std::vector inputs = {{8, 1}, {8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + Status ret = get_next->Init(strategy); + ASSERT_EQ(ret, FAILED); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/l2_normalize_info_test.cc b/tests/ut/cpp/parallel/ops_info/l2_normalize_info_test.cc new file mode 100644 index 00000000000..07a160ec993 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/l2_normalize_info_test.cc @@ -0,0 +1,193 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/l2_normalize_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class L2NormalizeInfo; +using L2NormalizeInfoPtr = std::shared_ptr; +L2NormalizeInfoPtr norm; + +class TestL2NormalizeInfo : public UT::Common { + public: + TestL2NormalizeInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestL2NormalizeInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 34; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(32); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr axis = MakeValue(1); + std::unordered_map attr = {{AXIS, axis}}; + + Shapes inputs_shape = {{32, 64, 96}}; + Shapes outputs_shape = {{32, 64, 96}}; + + norm = std::make_shared("l2_normalize_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestL2NormalizeInfo, InferDevMatrixShape1) { + std::vector inputs = {{4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + norm->Init(strategy); + std::vector dev_matrix_shape = norm->dev_matrix_shape(); + + std::vector expect = {4, 1, 8}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestL2NormalizeInfo, InferSliceShape1) { + std::vector str = {{4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, str); + + norm->Init(strategy); + std::vector inputs = norm->inputs_tensor_info(); + std::vector outputs = norm->outputs_tensor_info(); + + Shape input_slice_shape_expect = {8, 64, 12}; + Shape output_slice_shape_expect = {8, 64, 12}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestL2NormalizeInfo, GetTensorLayout1) { + std::vector str = {{4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, str); + + norm->Init(strategy); + std::vector inputs = norm->inputs_tensor_info(); + std::vector outputs = norm->outputs_tensor_info(); + + TensorMap input_expect = {2, 1, 0}; + TensorMap output_expect = {2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestL2NormalizeInfo, GetForwardOp1) { + std::vector inputs = {{4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + norm->Init(strategy); + OperatorVector forward_op = norm->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestL2NormalizeInfo, GetMirrorOPs1) { + std::vector inputs = {{4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + norm->Init(strategy); + MirrorOps mirror_ops = norm->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestL2NormalizeInfo, CheckStrategy1) { + std::vector inputs = {{4, 1, 8}, {4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = norm->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestL2NormalizeInfo, CheckStrategy2) { + std::vector inputs = {{4, 2, 3}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = norm->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestL2NormalizeInfo, CheckStrategy3) { + std::vector inputs = {{4, 2, 3, 4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = norm->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestL2NormalizeInfo, CheckStrategy4) { + std::vector inputs = {{4, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = norm->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +TEST_F(TestL2NormalizeInfo, mirror_ops) { + std::vector inputs = {{2, 1, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + norm->Init(strategy); + MirrorOps mirror_ops = norm->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(0); + + OperatorArgs operator_args = mirror_op.at(0).second; + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/log_softmax_info_test.cc b/tests/ut/cpp/parallel/ops_info/log_softmax_info_test.cc new file mode 100644 index 00000000000..1c31463c885 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/log_softmax_info_test.cc @@ -0,0 +1,179 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class LogSoftmaxInfo; +using LogSoftmaxInfoPtr = std::shared_ptr; +LogSoftmaxInfoPtr log_softmax; + +class TestLogSoftmaxInfo : public UT::Common { + public: + TestLogSoftmaxInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestLogSoftmaxInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 130; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(128); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr axis = MakeValue(-2); + std::unordered_map attr = {{"axis", axis}}; + + Shapes inputs_shape = {{2, 4, 8, 16}}; + Shapes outputs_shape = {{2, 4, 8, 16}}; + + log_softmax = std::make_shared("log_softmax_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestLogSoftmaxInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + log_softmax->Init(strategy); + std::vector dev_matrix_shape = log_softmax->dev_matrix_shape(); + + std::vector expect = {2, 4, 1, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestLogSoftmaxInfo, InferSliceShape1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + log_softmax->Init(strategy); + std::vector inputs = log_softmax->inputs_tensor_info(); + std::vector outputs = log_softmax->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 8, 1}; + Shape output_slice_shape_expect = {1, 1, 8, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestLogSoftmaxInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + log_softmax->Init(strategy); + std::vector inputs = log_softmax->inputs_tensor_info(); + std::vector outputs = log_softmax->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestLogSoftmaxInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + log_softmax->Init(strategy); + OperatorVector forward_op = log_softmax->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestLogSoftmaxInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + log_softmax->Init(strategy); + MirrorOps mirror_ops = log_softmax->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestLogSoftmaxInfo, CheckStrategy1) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = log_softmax->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestLogSoftmaxInfo, CheckStrategy2) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = log_softmax->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestLogSoftmaxInfo, CheckStrategy3) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = log_softmax->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestLogSoftmaxInfo, GetDeviceList1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + log_softmax->Init(strategy); + RankList dev_list = log_softmax->global_device_list(); + ASSERT_EQ(dev_list.size(), 128); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/matmul_info_test.cc b/tests/ut/cpp/parallel/ops_info/matmul_info_test.cc new file mode 100644 index 00000000000..02f44887443 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/matmul_info_test.cc @@ -0,0 +1,608 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/matmul_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" +#include "parallel/auto_parallel/graph_costmodel.h" + +namespace mindspore { +namespace parallel { + +class MatMulInfo; +using MatMulInfoPtr = std::shared_ptr; +MatMulInfoPtr matmul1; +MatMulInfoPtr matmul2; +MatMulInfoPtr matmul3; +MatMulInfoPtr matmul4; + +class TestMatmulInfo : public UT::Common { + public: + TestMatmulInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestMatmulInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + // matmul1 + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpoce_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpoce_b_1}}; + + Shapes inputs_shape_1 = {{2, 4, 8, 16}, {2, 4, 16, 32}}; + Shapes outputs_shape_1 = {{2, 4, 8, 32}}; + + matmul1 = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + // matmul2 + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpoce_b_2 = MakeValue(true); + std::unordered_map attr_2 = {{"transpose_a", transpose_a_2}, {"transpose_b", transpoce_b_2}}; + + Shapes inputs_shape_2 = {{2, 4, 8, 16}, {32, 16}}; + Shapes outputs_shape_2 = {{2, 4, 8, 32}}; + + matmul2 = std::make_shared("matmul_info", inputs_shape_2, outputs_shape_2, attr_2); + + // matmul3 + ValuePtr transpose_a_3 = MakeValue(false); + ValuePtr transpoce_b_3 = MakeValue(true); + std::unordered_map attr_3 = {{"transpose_a", transpose_a_3}, {"transpose_b", transpoce_b_3}}; + + Shapes inputs_shape_3 = {{8, 16}, {2, 4, 32, 16}}; + Shapes outputs_shape_3 = {{2, 4, 8, 32}}; + + matmul3 = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_3); + + // matmul4 + std::unordered_map attr_4 = {{"transpose_a", transpose_a_3}}; + matmul4 = std::make_shared("matmul_info", inputs_shape_3, outputs_shape_3, attr_4); +} + +TEST_F(TestMatmulInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + std::vector dev_matrix_shape = matmul1->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 16, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestMatmulInfo, InferDevMatrixShape2) { + std::vector inputs = {{2, 4, 8, 8}, {2, 4, 8, 2}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + std::vector dev_matrix_shape = matmul1->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 8, 2}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +// matmul2 +TEST_F(TestMatmulInfo, InferDevMatrixShape3) { + std::vector inputs = {{2, 4, 8, 16}, {1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul2->Init(strategy); + std::vector dev_matrix_shape = matmul2->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 16, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +// matmul2 +TEST_F(TestMatmulInfo, InferDevMatrixShape4) { + std::vector inputs = {{2, 4, 8, 8}, {2, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul2->Init(strategy); + std::vector dev_matrix_shape = matmul2->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 8, 2}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +// matmul3 +TEST_F(TestMatmulInfo, InferDevMatrixShape5) { + std::vector inputs = {{8, 16}, {2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul3->Init(strategy); + std::vector dev_matrix_shape = matmul3->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 16, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +// matmul3 +TEST_F(TestMatmulInfo, InferDevMatrixShape6) { + std::vector inputs = {{8, 8}, {2, 4, 2, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul3->Init(strategy); + std::vector dev_matrix_shape = matmul3->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 8, 2}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestMatmulInfo, InferTensorMap1) { + std::vector str = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul1->Init(strategy); + std::vector inputs = matmul1->inputs_tensor_info(); + std::vector outputs = matmul1->outputs_tensor_info(); + + TensorMap mat_a_expect = {4, 3, 2, 1}; + TensorMap mat_b_expect = {4, 3, 1, 0}; + TensorMap output_expect = {4, 3, 2, 0}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map mat_a_tensor_map = mat_a_tensor_info.tensor_layout().origin_tensor_map(); + Map mat_b_tensor_map = mat_b_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(mat_a_tensor_map.array(), mat_a_expect); + ASSERT_EQ(mat_b_tensor_map.array(), mat_b_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +// matmul2 +TEST_F(TestMatmulInfo, InferTensorMap2) { + std::vector str = {{2, 4, 8, 16}, {1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul2->Init(strategy); + std::vector inputs = matmul2->inputs_tensor_info(); + std::vector outputs = matmul2->outputs_tensor_info(); + + TensorMap mat_a_expect = {4, 3, 2, 1}; + TensorMap mat_b_expect = {0, 1}; + TensorMap output_expect = {4, 3, 2, 0}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map mat_a_tensor_map = mat_a_tensor_info.tensor_layout().origin_tensor_map(); + Map mat_b_tensor_map = mat_b_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(mat_a_tensor_map.array(), mat_a_expect); + ASSERT_EQ(mat_b_tensor_map.array(), mat_b_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +// matmul3 +TEST_F(TestMatmulInfo, InferTensorMap3) { + std::vector str = {{8, 16}, {2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul3->Init(strategy); + std::vector inputs = matmul3->inputs_tensor_info(); + std::vector outputs = matmul3->outputs_tensor_info(); + + TensorMap mat_a_expect = {2, 1}; + TensorMap mat_b_expect = {4, 3, 0, 1}; + TensorMap output_expect = {4, 3, 2, 0}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map mat_a_tensor_map = mat_a_tensor_info.tensor_layout().origin_tensor_map(); + Map mat_b_tensor_map = mat_b_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(mat_a_tensor_map.array(), mat_a_expect); + ASSERT_EQ(mat_b_tensor_map.array(), mat_b_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestMatmulInfo, InferSliceShape1) { + std::vector str = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul1->Init(strategy); + std::vector inputs = matmul1->inputs_tensor_info(); + std::vector outputs = matmul1->outputs_tensor_info(); + + Shape mat_a_slice_shape_expect = {1, 1, 1, 1}; + Shape mat_b_slice_shape_expect = {1, 1, 1, 32}; + Shape output_slice_shape_expect = {1, 1, 1, 32}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Shape mat_a_slice_shape = mat_a_tensor_info.slice_shape(); + Shape mat_b_slice_shape = mat_b_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(mat_a_slice_shape, mat_a_slice_shape_expect); + ASSERT_EQ(mat_b_slice_shape, mat_b_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +// matmul2 +TEST_F(TestMatmulInfo, InferSliceShape2) { + std::vector str = {{2, 4, 8, 16}, {1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul2->Init(strategy); + std::vector inputs = matmul2->inputs_tensor_info(); + std::vector outputs = matmul2->outputs_tensor_info(); + + Shape mat_a_slice_shape_expect = {1, 1, 1, 1}; + Shape mat_b_slice_shape_expect = {32, 1}; + Shape output_slice_shape_expect = {1, 1, 1, 32}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Shape mat_a_slice_shape = mat_a_tensor_info.slice_shape(); + Shape mat_b_slice_shape = mat_b_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(mat_a_slice_shape, mat_a_slice_shape_expect); + ASSERT_EQ(mat_b_slice_shape, mat_b_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +// matmul3 +TEST_F(TestMatmulInfo, InferSliceShape3) { + std::vector str = {{8, 16}, {2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul3->Init(strategy); + std::vector inputs = matmul3->inputs_tensor_info(); + std::vector outputs = matmul3->outputs_tensor_info(); + + Shape mat_a_slice_shape_expect = {1, 1}; + Shape mat_b_slice_shape_expect = {1, 1, 32, 1}; + Shape output_slice_shape_expect = {1, 1, 1, 32}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Shape mat_a_slice_shape = mat_a_tensor_info.slice_shape(); + Shape mat_b_slice_shape = mat_b_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(mat_a_slice_shape, mat_a_slice_shape_expect); + ASSERT_EQ(mat_b_slice_shape, mat_b_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +// matmul3 +TEST_F(TestMatmulInfo, GetTensorLayout3) { + std::vector str = {{8, 16}, {2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + matmul3->Init(strategy); + std::vector inputs = matmul3->inputs_tensor_info(); + std::vector outputs = matmul3->outputs_tensor_info(); + + TensorMap mat_a_expect = {2, 1}; + TensorMap mat_b_expect = {4, 3, 0, 1}; + TensorMap output_expect = {4, 3, 2, 0}; + + TensorInfo mat_a_tensor_info = inputs.at(0); + TensorInfo mat_b_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map mat_a_tensor_map = mat_a_tensor_info.tensor_layout().origin_tensor_map(); + Map mat_b_tensor_map = mat_b_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(mat_a_tensor_map.array(), mat_a_expect); + ASSERT_EQ(mat_b_tensor_map.array(), mat_b_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestMatmulInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + OperatorVector forward_op = matmul1->forward_op(); + + OperatorArgs operator_args = forward_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + + std::string arg1_name = operator_args.first.at(1).first; + ValuePtr arg1_value = operator_args.first.at(1).second; + bool arg1_value_is_string = false; + if (arg1_value->isa()) { + arg1_value_is_string = true; + } + + ASSERT_EQ(forward_op.at(0).first, "AllReduce"); + ASSERT_EQ(forward_op.size(), 1); + ASSERT_EQ(arg0_name, "op"); + ASSERT_EQ(arg1_name, "group"); + ASSERT_EQ(arg1_value_is_string, true); +} + +TEST_F(TestMatmulInfo, GetForwardOp2) { + std::vector inputs = {{2, 4, 8, 1}, {2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + OperatorVector forward_op = matmul1->forward_op(); + + ASSERT_EQ(forward_op.size(), 0); +} + +TEST_F(TestMatmulInfo, GetVirtualDivOp1) { + std::vector inputs = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + OperatorVector virtual_div_op = matmul1->virtual_div_op(); + + OperatorArgs operator_args = virtual_div_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + int32_t divisor = arg0_value->cast()->value(); + + ASSERT_EQ(virtual_div_op.at(0).first, "_VirtualDiv"); + ASSERT_EQ(virtual_div_op.size(), 1); + ASSERT_EQ(arg0_name, "divisor"); + ASSERT_EQ(divisor, 16); +} + +TEST_F(TestMatmulInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + MirrorOps mirror_ops = matmul1->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + + OperatorArgs operator_args = mirror_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +// matmul2 +TEST_F(TestMatmulInfo, GetMirrorOPs2) { + std::vector inputs = {{2, 4, 1, 16}, {8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul2->Init(strategy); + MirrorOps mirror_ops = matmul2->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + + OperatorArgs operator_args = mirror_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +// matmul3 +TEST_F(TestMatmulInfo, GetMirrorOPs3) { + std::vector inputs = {{8, 16}, {2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul3->Init(strategy); + MirrorOps mirror_ops = matmul3->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + + OperatorArgs operator_args = mirror_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +TEST_F(TestMatmulInfo, GetMirrorOPs4) { + std::vector inputs = {{2, 4, 1, 16}, {2, 4, 16, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + matmul1->Init(strategy); + MirrorOps mirror_ops = matmul1->mirror_ops(); + + ASSERT_EQ(mirror_ops.size(), 0); // all reduce only in -3 dim (strategy is 1); +} + +TEST_F(TestMatmulInfo, InitTwice) { + std::vector inputs = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + // init twice + matmul1->Init(strategy); + matmul1->Init(strategy); + + MirrorOps mirror_ops = matmul1->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + + OperatorArgs operator_args = mirror_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +TEST_F(TestMatmulInfo, CheckStrategy1) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, CheckStrategy2) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{2, 4, 8, 16}, {4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, CheckStrategy3) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{2, 4, 8, 16}, {2, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, CheckStrategy4) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{2, 4, 8, 16}, {2, 3, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, CheckStrategy5) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{0, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, CheckStrategy6) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{-1, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, CheckStrategy7) { + // Success: {{2,4,8,16}, {2,4,16,1}} + std::vector inputs = {{4, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul1->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, InitFailed) { + // matmul4 attr is wrong + std::vector inputs = {{4, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = matmul4->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestMatmulInfo, test_GenerateStrategies1) { + // the parameter '0' indicates that the stageId = 0, there are 1024 devices in the stage 0 + ASSERT_EQ(matmul1->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = matmul1->GetStrategyCost(); + for (const auto& swc : sc) { + StrategyPtr sp = swc->strategy_ptr; + Cost cost = *(swc->cost_list[0]); + matmul1->InitForCostModel(sp); + std::vector inputs_info = matmul1->inputs_tensor_info(); + std::vector outputs_info = matmul1->outputs_tensor_info(); + ASSERT_DOUBLE_EQ(matmul1->GetOperatorCost()->GetMemoryCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.memory_cost_); + break; + } +} + +TEST_F(TestMatmulInfo, test_GenerateStrategies2) { + // the parameter '0' indicates that the stageId = 0, there are 1024 devices in the stage 0 + ASSERT_EQ(matmul3->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = matmul3->GetStrategyCost(); + for (const auto& swc : sc) { + StrategyPtr sp = swc->strategy_ptr; + Cost cost = *(swc->cost_list[0]); + matmul3->InitForCostModel(sp); + + std::vector inputs_info = matmul3->inputs_tensor_info(); + std::vector outputs_info = matmul3->outputs_tensor_info(); + std::vector replica_inputs_info; + replica_inputs_info.push_back(inputs_info[0]); + + // transpose the tensor B + TensorInfo input1_info = inputs_info[1]; + Shape input1_shape = input1_info.shape(); + Shape input1_slice_shape = input1_info.slice_shape(); + TensorLayout tly; + matmul3->SwapLastTwoElements(&input1_shape); + matmul3->SwapLastTwoElements(&input1_slice_shape); + TensorInfo replica_input1_info(tly, input1_shape, input1_slice_shape); + replica_inputs_info.push_back(replica_input1_info); + + ASSERT_DOUBLE_EQ(matmul3->GetOperatorCost()->GetMemoryCost(replica_inputs_info, outputs_info, sp->GetInputStage()), + cost.memory_cost_); + break; + } +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/onehot_info_test.cc b/tests/ut/cpp/parallel/ops_info/onehot_info_test.cc new file mode 100644 index 00000000000..df67175ae6d --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/onehot_info_test.cc @@ -0,0 +1,209 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/onehot_info.h" +#include "parallel/device_manager.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { + +class OneHotInfo; +using OneHotInfoPtr = std::shared_ptr; +OneHotInfoPtr onehot_info; + +class TestOneHotInfo : public UT::Common { + public: + TestOneHotInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestOneHotInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr axis = MakeValue(std::int32_t(-1)); + std::unordered_map attr = {{"axis", axis}}; + + Shapes inputs_shape = {{64}, {}, {}}; + Shapes outputs_shape = {{64, 10}}; + + onehot_info = std::make_shared("onehot_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestOneHotInfo, InferDevMatrixShape1) { + std::vector inputs = {{8, 1}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector dev_matrix_shape = onehot_info->dev_matrix_shape(); + + std::vector expect = {8, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestOneHotInfo, InferDevMatrixShape2) { + std::vector inputs = {{4, 1}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector dev_matrix_shape = onehot_info->dev_matrix_shape(); + + std::vector expect = {2, 4, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestOneHotInfo, InferDevMatrixShape3) { + std::vector inputs = {{4, 2}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, FAILED); + std::vector dev_matrix_shape = onehot_info->dev_matrix_shape(); + + std::vector expect = {4, 2}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestOneHotInfo, InferTensorMap2) { + std::vector str = {{8, 1}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector inputs = onehot_info->inputs_tensor_info(); + std::vector outputs = onehot_info->outputs_tensor_info(); + + TensorMap input_expect = {1}; + TensorMap output_expect = {1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestOneHotInfo, InferSliceShape1) { + std::vector str = {{8, 1}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector inputs = onehot_info->inputs_tensor_info(); + std::vector outputs = onehot_info->outputs_tensor_info(); + + Shape input_slice_shape_expect = {8}; + Shape output_slice_shape_expect = {8, 10}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestOneHotInfo, InferSliceShape2) { + std::vector str = {{4, 2}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, FAILED); + std::vector inputs = onehot_info->inputs_tensor_info(); + std::vector outputs = onehot_info->outputs_tensor_info(); + + Shape input_slice_shape_expect = {16}; + Shape output_slice_shape_expect = {16, 5}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestOneHotInfo, InferSliceShape3) { + std::vector str = {{2, 2}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, FAILED); + std::vector inputs = onehot_info->inputs_tensor_info(); + std::vector outputs = onehot_info->outputs_tensor_info(); + + Shape input_slice_shape_expect = {32}; + Shape output_slice_shape_expect = {32, 5}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestOneHotInfo, GetMirrorOPs1) { + std::vector inputs = {{8, 1}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info->Init(strategy); + ASSERT_EQ(status, SUCCESS); + MirrorOps mirror_ops = onehot_info->mirror_ops(); + + ASSERT_EQ(mirror_ops.size(), 0); +} + +TEST_F(TestOneHotInfo, CheckStrategy1) { + std::vector inputs = {{16}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = onehot_info->Init(strategy); + ASSERT_EQ(ret, FAILED); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/onehot_info_test_axis_0.cc b/tests/ut/cpp/parallel/ops_info/onehot_info_test_axis_0.cc new file mode 100644 index 00000000000..17fe3651763 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/onehot_info_test_axis_0.cc @@ -0,0 +1,190 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/onehot_info.h" +#include "parallel/device_manager.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { + +class OneHotInfo; +using OneHotInfoPtr = std::shared_ptr; +OneHotInfoPtr onehot_info2; + +class TestOneHotInfo2 : public UT::Common { + public: + TestOneHotInfo2() {} + void SetUp(); + void TearDown() {} +}; + +void TestOneHotInfo2::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 10; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(8); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr axis = MakeValue(std::int32_t(0)); + std::unordered_map attr = {{"axis", axis}}; + + Shapes inputs_shape = {{64}, {}, {}}; + Shapes outputs_shape = {{10, 64}}; + + onehot_info2 = std::make_shared("onehot_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestOneHotInfo2, InferDevMatrixShape1) { + std::vector inputs = {{1, 8}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector dev_matrix_shape = onehot_info2->dev_matrix_shape(); + + std::vector expect = {8, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestOneHotInfo2, InferDevMatrixShape2) { + std::vector inputs = {{1, 4}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector dev_matrix_shape = onehot_info2->dev_matrix_shape(); + + std::vector expect = {2, 4, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestOneHotInfo2, InferDevMatrixShape3) { + std::vector inputs = {{2, 4}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, FAILED); + std::vector dev_matrix_shape = onehot_info2->dev_matrix_shape(); + + std::vector expect = {4, 2}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestOneHotInfo2, InferTensorMap2) { + std::vector str = {{1, 8}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector inputs = onehot_info2->inputs_tensor_info(); + std::vector outputs = onehot_info2->outputs_tensor_info(); + + TensorMap input_expect = {1}; + TensorMap output_expect = {0, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestOneHotInfo2, InferSliceShape1) { + std::vector str = {{1, 8}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, SUCCESS); + std::vector inputs = onehot_info2->inputs_tensor_info(); + std::vector outputs = onehot_info2->outputs_tensor_info(); + + Shape input_slice_shape_expect = {8}; + Shape output_slice_shape_expect = {10, 8}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestOneHotInfo2, InferSliceShape2) { + std::vector str = {{2, 4}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, FAILED); + std::vector inputs = onehot_info2->inputs_tensor_info(); + std::vector outputs = onehot_info2->outputs_tensor_info(); + + Shape input_slice_shape_expect = {16}; + Shape output_slice_shape_expect = {5, 16}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestOneHotInfo2, InferSliceShape3) { + std::vector str = {{2, 2}, {}, {}}; + StrategyPtr strategy = NewStrategy(0, str); + + Status status = onehot_info2->Init(strategy); + ASSERT_EQ(status, FAILED); + std::vector inputs = onehot_info2->inputs_tensor_info(); + std::vector outputs = onehot_info2->outputs_tensor_info(); + + Shape input_slice_shape_expect = {32}; + Shape output_slice_shape_expect = {5, 32}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/pow_info_test.cc b/tests/ut/cpp/parallel/ops_info/pow_info_test.cc new file mode 100644 index 00000000000..86d62190a7f --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/pow_info_test.cc @@ -0,0 +1,166 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/elementary_function_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class PowInfo; +using PowInfoPtr = std::shared_ptr; +PowInfoPtr pow; + +class TestPowInfo : public UT::Common { + public: + TestPowInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestPowInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 66; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(64); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{32, 64, 128}}; + Shapes outputs_shape = {{32, 64, 128}}; + + pow = std::make_shared("pow_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestPowInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + pow->Init(strategy); + std::vector dev_matrix_shape = pow->dev_matrix_shape(); + + std::vector expect = {2, 4, 8}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestPowInfo, InferSliceShape1) { + std::vector str = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, str); + + pow->Init(strategy); + std::vector inputs = pow->inputs_tensor_info(); + std::vector outputs = pow->outputs_tensor_info(); + + Shape input_slice_shape_expect = {16, 16, 16}; + Shape output_slice_shape_expect = {16, 16, 16}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestPowInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, str); + + pow->Init(strategy); + std::vector inputs = pow->inputs_tensor_info(); + std::vector outputs = pow->outputs_tensor_info(); + + TensorMap input_expect = {2, 1, 0}; + TensorMap output_expect = {2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestPowInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + pow->Init(strategy); + OperatorVector forward_op = pow->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestPowInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + pow->Init(strategy); + MirrorOps mirror_ops = pow->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestPowInfo, CheckStrategy1) { + std::vector inputs = {{2, 2, 8}, {2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = pow->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestPowInfo, CheckStrategy2) { + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = pow->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestPowInfo, CheckStrategy3) { + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = pow->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/prelu_test.cc b/tests/ut/cpp/parallel/ops_info/prelu_test.cc new file mode 100644 index 00000000000..bb815921e0a --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/prelu_test.cc @@ -0,0 +1,278 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/prelu_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class PReLUInfo; +using PReLUInfoPtr = std::shared_ptr; +PReLUInfoPtr prelu; +PReLUInfoPtr prelu_2d; + +class TestPReLUInfo : public UT::Common { + public: + TestPReLUInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestPReLUInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + int32_t local_dev = 0; + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + Shapes inputs_shape = {{64, 4, 8, 16}, {4}}; + Shapes outputs_shape = {{64, 4, 8, 16}}; + std::unordered_map attr; + prelu = std::make_shared("prelu_info", inputs_shape, outputs_shape, attr); + + Shapes inputs_shape_2d = {{1024, 4}, {4}}; + Shapes outputs_shape_2d = {{1024, 4}}; + std::unordered_map attr_2d; + prelu_2d = std::make_shared("prelu_info", inputs_shape_2d, outputs_shape_2d, attr_2d); +} + +TEST_F(TestPReLUInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 1, 8, 16}, {1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + prelu->Init(strategy); + std::vector dev_matrix_shape = prelu->dev_matrix_shape(); + + std::vector expect = {4, 2, 1, 8, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestPReLUInfo, InferSliceShape1) { + std::vector str = {{2, 1, 8, 16}, {1}}; + StrategyPtr strategy = NewStrategy(0, str); + + prelu->Init(strategy); + std::vector inputs = prelu->inputs_tensor_info(); + std::vector outputs = prelu->outputs_tensor_info(); + + Shape input_slice_shape_expect = {32, 4, 1, 1}; + Shape param_slice_shape_expect = {4}; + Shape output_slice_shape_expect = {32, 4, 1, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo param_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestPReLUInfo, GetTensorLayout1) { + std::vector str = {{2, 1, 8, 16}, {1}}; + StrategyPtr strategy = NewStrategy(0, str); + + prelu->Init(strategy); + std::vector inputs = prelu->inputs_tensor_info(); + std::vector outputs = prelu->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap param_expect = {2}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo param_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map param_tensor_map = param_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestPReLUInfo, GetMirrorOPs1) { + std::vector str = {{2, 1, 2, 2}, {1}}; + StrategyPtr strategy = NewStrategy(0, str); + prelu->Init(strategy); + MirrorOps mirror_ops = prelu->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + OperatorArgs operator_args = mirror_op.at(0).second; + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +TEST_F(TestPReLUInfo, CheckStrategy1) { + // Success: {{2,1,8,16},{1}} + std::vector inputs = {{2, 1, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + Status ret = prelu->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestPReLUInfo, CheckStrategy2) { + // Success: {{2,1,8,16},{1}} + std::vector inputs = {{2, 4, 8, 16}, {4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + Status ret = prelu->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestPReLUInfo, AutoStrategy1) { + ASSERT_EQ(prelu->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = prelu->GetStrategyCost(); + + Shapes splittable_inputs = {{1, 0, 1, 1}, {0}}; + std::vector sp_vector; + Shapes inputs_shape = {{64, 4, 8, 16}, {4}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape, splittable_inputs, &sp_vector); + for (auto stra : sp_vector) { + auto stra0 = stra->GetInputDim()[0]; + auto stra1 = stra->GetInputDim()[1]; + ASSERT_EQ(stra0[1], 1); + ASSERT_EQ(stra1[0], 1); + } +} + +TEST_F(TestPReLUInfo, InferDevMatrixShape_2d1) { + std::vector inputs = {{128, 1}, {1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + prelu_2d->Init(strategy); + std::vector dev_matrix_shape = prelu_2d->dev_matrix_shape(); + + std::vector expect = {8, 128, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestPReLUInfo, InferSliceShape_2d1) { + std::vector str = {{128, 1}, {1}}; + StrategyPtr strategy = NewStrategy(0, str); + + prelu_2d->Init(strategy); + std::vector inputs = prelu_2d->inputs_tensor_info(); + std::vector outputs = prelu_2d->outputs_tensor_info(); + + Shape input_slice_shape_expect = {8, 4}; + Shape param_slice_shape_expect = {4}; + Shape output_slice_shape_expect = {8, 4}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo param_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestPReLUInfo, GetTensorLayout_2d1) { + std::vector str = {{128, 1}, {1}}; + StrategyPtr strategy = NewStrategy(0, str); + + prelu_2d->Init(strategy); + std::vector inputs = prelu_2d->inputs_tensor_info(); + std::vector outputs = prelu_2d->outputs_tensor_info(); + + TensorMap input_expect = {1, 0}; + TensorMap param_expect = {0}; + TensorMap output_expect = {1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo param_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map param_tensor_map = param_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestPReLUInfo, GetMirrorOPs_2d1) { + std::vector str = {{128, 1}, {1}}; + StrategyPtr strategy = NewStrategy(0, str); + prelu_2d->Init(strategy); + MirrorOps mirror_ops = prelu_2d->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + OperatorArgs operator_args = mirror_op.at(0).second; + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +TEST_F(TestPReLUInfo, CheckStrategy_2d1) { + // Success: {{2,1,8,16},{1}} + std::vector inputs = {{128, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + Status ret = prelu_2d->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestPReLUInfo, CheckStrategy_2d2) { + // Success: {{2,1,8,16},{1}} + std::vector inputs = {{128, 4}, {4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + Status ret = prelu_2d->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestPReLUInfo, AutoStrategy_2d1) { + ASSERT_EQ(prelu_2d->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = prelu_2d->GetStrategyCost(); + + Shapes splittable_inputs = {{1, 0}, {0}}; + std::vector sp_vector; + Shapes inputs_shape = {{1024, 4}, {4}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape, splittable_inputs, &sp_vector); + for (auto stra : sp_vector) { + auto stra0 = stra->GetInputDim()[0]; + auto stra1 = stra->GetInputDim()[1]; + ASSERT_EQ(stra0[1], 1); + ASSERT_EQ(stra1[0], 1); + } +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/reduce_method_test.cc b/tests/ut/cpp/parallel/ops_info/reduce_method_test.cc new file mode 100644 index 00000000000..0ce81b0cd08 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/reduce_method_test.cc @@ -0,0 +1,221 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/reduce_method_info.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +using ReduceSumInfoPtr = std::shared_ptr; +ReduceSumInfoPtr reduce_sum; + +class TestReduceSumInfo : public UT::Common { + public: + TestReduceSumInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestReduceSumInfo::SetUp() { + UT::InitPythonPath(); + std::list dev_list; + + for (int32_t i = 0; i < 34; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(32); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + Shapes inputs_shape = {{16, 32, 64}}; + Shapes outputs_shape = {{16, 32}}; + ValuePtr value = MakeValue(-1); + ValuePtr value0; + std::vector val = {value0, value}; + ValuePtr keep_dims = MakeValue(false); + std::unordered_map attr = {{KEEP_DIMS, keep_dims}}; + + reduce_sum = std::make_shared("sum_info", inputs_shape, outputs_shape, attr); + reduce_sum->set_input_value(val); +} + +TEST_F(TestReduceSumInfo, InferDevMatrixShape1) { + std::vector inputs = {{4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reduce_sum->Init(strategy); + std::vector dev_matrix_shape = reduce_sum->dev_matrix_shape(); + + std::vector expect = {4, 8, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestReduceSumInfo, InferSliceShape1) { + std::vector str = {{4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + reduce_sum->Init(strategy); + std::vector inputs = reduce_sum->inputs_tensor_info(); + std::vector outputs = reduce_sum->outputs_tensor_info(); + + Shape input_slice_shape_expect = {4, 4, 64}; + Shape output_slice_shape_expect = {4, 4}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestReduceSumInfo, GetTensorLayout1) { + std::vector str = {{4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + reduce_sum->Init(strategy); + std::vector inputs = reduce_sum->inputs_tensor_info(); + std::vector outputs = reduce_sum->outputs_tensor_info(); + + TensorMap input_expect = {2, 1, 0}; + TensorMap output_expect = {2, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestReduceSumInfo, GetForwardOp1) { + std::vector inputs = {{4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reduce_sum->Init(strategy); + OperatorVector forward_op = reduce_sum->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestReduceSumInfo, GetForwardOp2) { + std::vector inputs = {{4, 4, 2}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reduce_sum->Init(strategy); + OperatorVector forward_op = reduce_sum->forward_op(); + OperatorArgs operator_args = forward_op.at(0).second; + OperatorAttrs operator_attrs = operator_args.first; + + std::string arg0_name = operator_attrs.at(0).first; + ValuePtr arg0_value = operator_attrs.at(0).second; + std::string op_value = arg0_value->cast()->ToString(); + + std::string arg1_name = operator_attrs.at(1).first; + ValuePtr arg1_value = operator_attrs.at(1).second; + std::string group_value = arg1_value->cast()->ToString(); + + ASSERT_EQ(forward_op.at(0).first, "AllReduce"); + ASSERT_EQ(forward_op.size(), 1); + ASSERT_EQ(arg0_name, "op"); + ASSERT_EQ(op_value, "sum"); + ASSERT_EQ(arg1_name, "group"); +} + +TEST_F(TestReduceSumInfo, GetMirrorOPs1) { + std::vector inputs = {{4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reduce_sum->Init(strategy); + MirrorOps mirror_ops = reduce_sum->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestReduceSumInfo, GetMirrorOPs2) { + std::vector inputs = {{4, 4, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reduce_sum->Init(strategy); + MirrorOps mirror_ops = reduce_sum->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(0); + OperatorArgs operator_args = mirror_op.at(0).second; + OperatorAttrs operator_attrs = operator_args.first; + + std::string arg0_name = operator_attrs.at(0).first; + ValuePtr arg0_value = operator_attrs.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} + +TEST_F(TestReduceSumInfo, CheckStrategy1) { + std::vector inputs = {{2, 2, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reduce_sum->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestReduceSumInfo, CheckStrategy2) { + std::vector inputs = {{2, 4, 8}, {2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reduce_sum->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestReduceSumInfo, CheckStrategy3) { + std::vector inputs = {{4, 4, 2}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reduce_sum->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +TEST_F(TestReduceSumInfo, CheckStrategy4) { + std::vector inputs = {{4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reduce_sum->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/reshape_test.cc b/tests/ut/cpp/parallel/ops_info/reshape_test.cc new file mode 100644 index 00000000000..73a0d68be51 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/reshape_test.cc @@ -0,0 +1,240 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/reshape_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class ReshapeInfo; +using ReshapeInfoPtr = std::shared_ptr; +ReshapeInfoPtr reshape; + +class TestReshapeInfo : public UT::Common { + public: + TestReshapeInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestReshapeInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 34; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(32); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{32, 512, 7, 7}}; + Shapes outputs_shape = {{32, 25088}}; + std::vector axis = {32, 25088}; + ValuePtr val0; + ValuePtr val1 = MakeValue(axis); + std::vector val = {val0, val1}; + + reshape = std::make_shared("reshape_info", inputs_shape, outputs_shape, attr); + reshape->set_input_value(val); +} + +TEST_F(TestReshapeInfo, InferDevMatrixShape1) { + std::vector inputs = {{4, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reshape->Init(strategy); + std::vector dev_matrix_shape = reshape->dev_matrix_shape(); + + std::vector expect = {8, 4}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestReshapeInfo, InferDevMatrixShape2) { + std::vector inputs = {{32, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reshape->Init(strategy); + std::vector dev_matrix_shape = reshape->dev_matrix_shape(); + + std::vector expect = {32}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestReshapeInfo, InferSliceShape1) { + std::vector str = {{4, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + reshape->Init(strategy); + std::vector inputs = reshape->inputs_tensor_info(); + std::vector outputs = reshape->outputs_tensor_info(); + + Shape input_slice_shape_expect = {8, 512, 7, 7}; + Shape output_slice_shape_expect = {8, 25088}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestReshapeInfo, InferSliceShape2) { + std::vector str = {{32, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + reshape->Init(strategy); + std::vector inputs = reshape->inputs_tensor_info(); + std::vector outputs = reshape->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 512, 7, 7}; + Shape output_slice_shape_expect = {1, 25088}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestReshapeInfo, GetTensorLayout1) { + std::vector str = {{4, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + reshape->Init(strategy); + std::vector inputs = reshape->inputs_tensor_info(); + std::vector outputs = reshape->outputs_tensor_info(); + + TensorMap input_expect = {0, -1, -1, -1}; + TensorMap output_expect = {0, -1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestReshapeInfo, GetTensorLayout2) { + std::vector str = {{32, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + reshape->Init(strategy); + std::vector inputs = reshape->inputs_tensor_info(); + std::vector outputs = reshape->outputs_tensor_info(); + + TensorMap input_expect = {0, -1, -1, -1}; + TensorMap output_expect = {0, -1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestReshapeInfo, GetForwardOp1) { + std::vector inputs = {{4, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reshape->Init(strategy); + OperatorVector forward_op = reshape->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestReshapeInfo, GetMirrorOPs1) { + std::vector inputs = {{4, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + reshape->Init(strategy); + MirrorOps mirror_ops = reshape->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 2); +} + +TEST_F(TestReshapeInfo, CheckStrategy1) { + std::vector inputs = {{1, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reshape->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestReshapeInfo, CheckStrategy2) { + std::vector inputs = {{2, 4, 8}, {2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reshape->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestReshapeInfo, CheckStrategy3) { + std::vector inputs = {{4, 1, 1, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = reshape->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +TEST_F(TestReshapeInfo, AutoStrategy1) { + ASSERT_EQ(reshape->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = reshape->GetStrategyCost(); + + Shapes splittable_inputs = {{1, 0, 0, 0}}; + std::vector sp_vector; + Shapes inputs_shape = {{32, 512, 7, 7}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sc.size(), sp_vector.size()); + for (auto stra : sp_vector) { + auto stra0 = stra->GetInputDim()[0]; + ASSERT_EQ(stra0[1], 1); + ASSERT_EQ(stra0[2], 1); + ASSERT_EQ(stra0[3], 1); + } +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/softmax_entropy_loss_info_test.cc b/tests/ut/cpp/parallel/ops_info/softmax_entropy_loss_info_test.cc new file mode 100644 index 00000000000..cba9b7ecd6b --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/softmax_entropy_loss_info_test.cc @@ -0,0 +1,196 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/loss_info.h" +#include "parallel/device_manager.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { + +class SoftmaxCrossEntropyWithLogitsInfo; +using LossPtr = std::shared_ptr; +LossPtr loss; + +class TestSoftmaxLoss : public UT::Common { + public: + TestSoftmaxLoss() {} + void SetUp(); + void TearDown() {} +}; + +void TestSoftmaxLoss::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 65; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(64); + stage_map.push_back(1); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr is_grad = MakeValue(true); + std::unordered_map attr = {{"is_grad", is_grad}}; + + Shapes inputs_shape = {{2, 4, 8, 16}, {2, 4, 8, 16}}; + Shapes outputs_shape = {{2}, {2, 4, 8, 16}}; + + loss = std::make_shared("CrossEntropy_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestSoftmaxLoss, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 8, 1}, {2, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + loss->Init(strategy); + std::vector dev_matrix_shape = loss->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestSoftmaxLoss, InferSliceShape1) { + std::vector str = {{2, 4, 8, 1}, {2, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + loss->Init(strategy); + std::vector inputs = loss->inputs_tensor_info(); + std::vector outputs = loss->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 1, 16}; + Shape label_slice_shape_expect = {1, 1, 1, 16}; + Shape output_0_slice_shape_expect = {1}; + Shape output_1_slice_shape_expect = {1, 1, 1, 16}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo label_tensor_info = inputs.at(1); + TensorInfo output_0_tensor_info = outputs.at(0); + TensorInfo output_1_tensor_info = outputs.at(1); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape label_slice_shape = label_tensor_info.slice_shape(); + Shape output_0_slice_shape = output_0_tensor_info.slice_shape(); + Shape output_1_slice_shape = output_1_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(label_slice_shape, label_slice_shape_expect); + ASSERT_EQ(output_0_slice_shape, output_0_slice_shape_expect); + ASSERT_EQ(output_1_slice_shape, output_1_slice_shape_expect); +} + +TEST_F(TestSoftmaxLoss, GetTensorLayout1) { + std::vector str = {{2, 4, 8, 1}, {2, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + loss->Init(strategy); + std::vector inputs = loss->inputs_tensor_info(); + std::vector outputs = loss->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap label_expect = {3, 2, 1, 0}; + TensorMap output_0_expect = {3}; + TensorMap output_1_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo label_tensor_info = inputs.at(1); + TensorInfo output_0_tensor_info = outputs.at(0); + TensorInfo output_1_tensor_info = outputs.at(1); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map label_tensor_map = label_tensor_info.tensor_layout().origin_tensor_map(); + Map output_0_tensor_map = output_0_tensor_info.tensor_layout().origin_tensor_map(); + Map output_1_tensor_map = output_1_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(label_tensor_map.array(), label_expect); + ASSERT_EQ(output_0_tensor_map.array(), output_0_expect); + ASSERT_EQ(output_1_tensor_map.array(), output_1_expect); +} + +TEST_F(TestSoftmaxLoss, GetForwardOp1) { + std::vector inputs = {{2, 4, 8, 1}, {2, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + loss->Init(strategy); + OperatorVector forward_op = loss->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestSoftmaxLoss, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 8, 1}, {2, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + loss->Init(strategy); + MirrorOps mirror_ops = loss->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestSoftmaxLoss, GetVirtualDivOPs1) { + std::vector inputs = {{1, 4, 8, 1}, {1, 4, 8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + loss->Init(strategy); + OperatorVector virtual_div_op = loss->virtual_div_op(); + + OperatorArgs operator_args = virtual_div_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + int32_t divisor = arg0_value->cast()->value(); + + ASSERT_EQ(virtual_div_op.at(0).first, "_VirtualDiv"); + ASSERT_EQ(virtual_div_op.size(), 1); + ASSERT_EQ(arg0_name, "divisor"); + ASSERT_EQ(divisor, 2); +} + +TEST_F(TestSoftmaxLoss, CheckStrategy1) { + // Success: {{2,4,8,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = loss->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestSoftmaxLoss, CheckStrategy2) { + // Success: {{2,4,8,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = loss->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/softmax_info_test.cc b/tests/ut/cpp/parallel/ops_info/softmax_info_test.cc new file mode 100644 index 00000000000..65fd46fb9ce --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/softmax_info_test.cc @@ -0,0 +1,192 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class SoftmaxInfo; +using SoftmaxInfoPtr = std::shared_ptr; +SoftmaxInfoPtr softmax; +SoftmaxInfoPtr softmax2; + +class TestSoftmaxInfo : public UT::Common { + public: + TestSoftmaxInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestSoftmaxInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 130; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(128); + stage_map.push_back(2); + + int32_t local_dev = 0; + + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr axis1 = MakeValue(-2); + std::unordered_map attr1 = {{"axis", axis1}}; + + ValuePtr axis2 = MakeValue(4); + std::unordered_map attr2 = {{"axis", axis2}}; + + Shapes inputs_shape = {{2, 4, 8, 16}}; + Shapes outputs_shape = {{2, 4, 8, 16}}; + + softmax = std::make_shared("softmax_info1", inputs_shape, outputs_shape, attr1); + softmax2 = std::make_shared("softmax_info2", inputs_shape, outputs_shape, attr2); +} + +TEST_F(TestSoftmaxInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + softmax->Init(strategy); + std::vector dev_matrix_shape = softmax->dev_matrix_shape(); + + std::vector expect = {2, 4, 1, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestSoftmaxInfo, InferSliceShape1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + softmax->Init(strategy); + std::vector inputs = softmax->inputs_tensor_info(); + std::vector outputs = softmax->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 8, 1}; + Shape output_slice_shape_expect = {1, 1, 8, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestSoftmaxInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + softmax->Init(strategy); + std::vector inputs = softmax->inputs_tensor_info(); + std::vector outputs = softmax->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestSoftmaxInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + softmax->Init(strategy); + OperatorVector forward_op = softmax->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestSoftmaxInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + softmax->Init(strategy); + MirrorOps mirror_ops = softmax->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestSoftmaxInfo, CheckStrategy1) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = softmax->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestSoftmaxInfo, CheckStrategy2) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = softmax->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestSoftmaxInfo, CheckStrategy3) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = softmax->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestSoftmaxInfo, InitFailed1) { + // softmax2's axis is wrong + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = softmax2->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestSoftmaxInfo, InitFailed2) { + // dev num is wrong + std::vector inputs = {{2, 4, 1, 100}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = softmax2->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/tanh_info_test.cc b/tests/ut/cpp/parallel/ops_info/tanh_info_test.cc new file mode 100644 index 00000000000..3ee99d093fc --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/tanh_info_test.cc @@ -0,0 +1,169 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/activation_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class TanhInfo; +using TanhInfoPtr = std::shared_ptr; +TanhInfoPtr tanh; + +class TestTanhInfo : public UT::Common { + public: + TestTanhInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestTanhInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 130; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(128); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{2, 4, 8, 16}}; + Shapes outputs_shape = {{2, 4, 8, 16}}; + + tanh = std::make_shared("tanh_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestTanhInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tanh->Init(strategy); + std::vector dev_matrix_shape = tanh->dev_matrix_shape(); + + std::vector expect = {2, 4, 1, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestTanhInfo, InferSliceShape1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + tanh->Init(strategy); + std::vector inputs = tanh->inputs_tensor_info(); + std::vector outputs = tanh->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 8, 1}; + Shape output_slice_shape_expect = {1, 1, 8, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestTanhInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + tanh->Init(strategy); + std::vector inputs = tanh->inputs_tensor_info(); + std::vector outputs = tanh->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestTanhInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tanh->Init(strategy); + OperatorVector forward_op = tanh->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestTanhInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tanh->Init(strategy); + MirrorOps mirror_ops = tanh->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestTanhInfo, CheckStrategy1) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tanh->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTanhInfo, CheckStrategy2) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tanh->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTanhInfo, CheckStrategy3) { + // Success: {{2,4,1,16}} + std::vector inputs = {{2, 4, 1, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tanh->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/tensor_add_info_test.cc b/tests/ut/cpp/parallel/ops_info/tensor_add_info_test.cc new file mode 100644 index 00000000000..612b344514d --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/tensor_add_info_test.cc @@ -0,0 +1,245 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/arithmetic_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class TensorAddInfo; +using TensorAddInfoPtr = std::shared_ptr; +TensorAddInfoPtr tensor_add, tensor_add1; + +class TestTensorAddInfo : public UT::Common { + public: + TestTensorAddInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestTensorAddInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 34; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(32); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{32, 64, 96}, {32, 64, 96}}; + Shapes outputs_shape = {{32, 64, 96}}; + tensor_add = std::make_shared("tensoradd_info", inputs_shape, outputs_shape, attr); + + Shapes inputs_shape1 = {{1, 48}, {48, 1}}; + Shapes outputs_shape1 = {{48, 48}}; + tensor_add1 = std::make_shared("tensoradd_info", inputs_shape1, outputs_shape1, attr); +} + +TEST_F(TestTensorAddInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 4}, {2, 4, 4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tensor_add->Init(strategy); + std::vector dev_matrix_shape = tensor_add->dev_matrix_shape(); + + std::vector expect = {2, 4, 4}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestTensorAddInfo, InferSliceShape1) { + std::vector str = {{2, 4, 4}, {2, 4, 4}}; + StrategyPtr strategy = NewStrategy(0, str); + + tensor_add->Init(strategy); + std::vector inputs = tensor_add->inputs_tensor_info(); + std::vector outputs = tensor_add->outputs_tensor_info(); + + Shape input_slice_shape_expect = {16, 16, 24}; + Shape output_slice_shape_expect = {16, 16, 24}; + + TensorInfo inputa_tensor_info = inputs.at(0); + TensorInfo inputb_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Shape inputa_slice_shape = inputa_tensor_info.slice_shape(); + Shape inputb_slice_shape = inputb_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(inputa_slice_shape, input_slice_shape_expect); + ASSERT_EQ(inputb_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestTensorAddInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 4}, {2, 4, 4}}; + StrategyPtr strategy = NewStrategy(0, str); + + tensor_add->Init(strategy); + std::vector inputs = tensor_add->inputs_tensor_info(); + std::vector outputs = tensor_add->outputs_tensor_info(); + + TensorMap input_expect = {2, 1, 0}; + TensorMap output_expect = {2, 1, 0}; + + TensorInfo inputa_tensor_info = inputs.at(0); + TensorInfo inputb_tensor_info = inputs.at(1); + TensorInfo output_tensor_info = outputs.at(0); + + Map inputa_tensor_map = inputa_tensor_info.tensor_layout().origin_tensor_map(); + Map inputb_tensor_map = inputb_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(inputa_tensor_map.array(), input_expect); + ASSERT_EQ(inputb_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestTensorAddInfo, GetForwardOp1) { + std::vector inputs = {{2, 4, 4}, {2, 4, 4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tensor_add->Init(strategy); + OperatorVector forward_op = tensor_add->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestTensorAddInfo, GetMirrorOPs1) { + std::vector inputs = {{2, 4, 4}, {2, 4, 4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tensor_add->Init(strategy); + MirrorOps mirror_ops = tensor_add->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestTensorAddInfo, CheckStrategy1) { + std::vector inputs = {{2, 4, 4}, {2, 6, 4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tensor_add->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTensorAddInfo, CheckStrategy2) { + std::vector inputs = {{2, 4, 8}, {2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tensor_add->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTensorAddInfo, CheckStrategy3) { + std::vector inputs = {{2, 4, 6}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tensor_add->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTensorAddInfo, CheckStrategy4) { + std::vector inputs = {{2, 4, 4}, {2, 4, 4}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = tensor_add->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +TEST_F(TestTensorAddInfo, GenerateStrategies) { + ASSERT_EQ(tensor_add->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = tensor_add->GetStrategyCost(); + for (auto& swc : sc) { + StrategyPtr sp = swc->strategy_ptr; + Cost cost = *(swc->cost_list[0]); + tensor_add->InitForCostModel(sp); + std::vector inputs_info = tensor_add->inputs_tensor_info(); + std::vector outputs_info = tensor_add->outputs_tensor_info(); + double memory_cost0 = tensor_add->GetOperatorCost()->GetMemoryCost(inputs_info, outputs_info, sp->GetInputStage()); + double memory_cost1 = cost.memory_cost_; + bool memory = memory_cost0 - memory_cost1 <= 1.0; + + double comm_cost0 = tensor_add->GetOperatorCost()->GetCommCost(inputs_info, outputs_info, sp->GetInputStage()); + double comm_cost1 = cost.communication_cost_; + bool comm = comm_cost0 - comm_cost1 <= 1.0; + + ASSERT_EQ(memory, true); + ASSERT_EQ(comm, true); + } +} + +TEST_F(TestTensorAddInfo, GenerateStrategies1) { + ASSERT_EQ(tensor_add1->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = tensor_add1->GetStrategyCost(); + for (auto& swc : sc) { + StrategyPtr sp = swc->strategy_ptr; + Cost cost = *(swc->cost_list[0]); + tensor_add1->InitForCostModel(sp); + std::vector inputs_info = tensor_add1->inputs_tensor_info(); + std::vector outputs_info = tensor_add1->outputs_tensor_info(); + double memory_cost0 = tensor_add1->GetOperatorCost()->GetMemoryCost(inputs_info, outputs_info, sp->GetInputStage()); + double memory_cost1 = cost.memory_cost_; + bool memory = memory_cost0 - memory_cost1 <= 1.0; + + double comm_cost0 = tensor_add1->GetOperatorCost()->GetCommCost(inputs_info, outputs_info, sp->GetInputStage()); + double comm_cost1 = cost.communication_cost_; + bool comm = comm_cost0 - comm_cost1 <= 1.0; + + ASSERT_EQ(memory, true); + ASSERT_EQ(comm, true); + } +} + +TEST_F(TestTensorAddInfo, mirror_ops) { + std::vector inputs = {{1, 8}, {4, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + tensor_add1->Init(strategy); + MirrorOps mirror_ops = tensor_add1->mirror_ops(); + OperatorVector mirror_op = mirror_ops.at(1); + + OperatorArgs operator_args = mirror_op.at(0).second; + + std::string arg0_name = operator_args.first.at(0).first; + ValuePtr arg0_value = operator_args.first.at(0).second; + std::string group = arg0_value->cast()->ToString(); + + ASSERT_EQ(mirror_op.at(0).first, "_MirrorOperator"); + ASSERT_EQ(mirror_op.size(), 1); + ASSERT_EQ(arg0_name, "group"); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/tmpidentity_test.cc b/tests/ut/cpp/parallel/ops_info/tmpidentity_test.cc new file mode 100644 index 00000000000..7edeb5a553f --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/tmpidentity_test.cc @@ -0,0 +1,177 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/device_manager.h" +#include "parallel/ops_info/operator_info.h" +#include "parallel/ops_info/tmp_identity_info.h" + +namespace mindspore { +namespace parallel { + +class TmpIdentityInfo; +using TmpIdentityInfoPtr = std::shared_ptr; +TmpIdentityInfoPtr identity_ptr; +using TensorMap = std::vector; + +class TestTmpIdentityInfo : public UT::Common { + public: + TestTmpIdentityInfo() { identity_ptr2 = nullptr; } + void SetUp(); + void TearDown() {} + + TmpIdentityInfoPtr identity_ptr2; +}; + +void TestTmpIdentityInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr = {}; + Shapes inputs_shape = {{2, 4, 8, 16}}; + Shapes outputs_shape = {{2, 4, 8, 16}}; + identity_ptr = std::make_shared(inputs_shape, outputs_shape, attr); + + Shapes inputs_shape2 = {{4, 16, 8, 16}}; + Shapes outputs_shape2 = {{4, 16, 8, 16}}; + identity_ptr2 = std::make_shared(inputs_shape2, outputs_shape2, attr); +} + +TEST_F(TestTmpIdentityInfo, InferDevMatrixShape1) { + std::vector inputs = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + identity_ptr->Init(strategy); + std::vector dev_matrix_shape = identity_ptr->dev_matrix_shape(); + + std::vector expect = {2, 4, 8, 16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestTmpIdentityInfo, InferSliceShape1) { + std::vector str = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + identity_ptr->Init(strategy); + std::vector inputs = identity_ptr->inputs_tensor_info(); + std::vector outputs = identity_ptr->outputs_tensor_info(); + + Shape input_slice_shape_expect = {1, 1, 1, 1}; + Shape output_slice_shape_expect = {1, 1, 1, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestTmpIdentityInfo, GetTensorLayout1) { + std::vector str = {{2, 4, 8, 16}}; + StrategyPtr strategy = NewStrategy(0, str); + + identity_ptr->Init(strategy); + std::vector inputs = identity_ptr->inputs_tensor_info(); + std::vector outputs = identity_ptr->outputs_tensor_info(); + + TensorMap input_expect = {3, 2, 1, 0}; + TensorMap output_expect = {3, 2, 1, 0}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestTmpIdentityInfo, CheckStrategy1) { + // Success: {{2,4,8,16}} + std::vector inputs = {{2, 2, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = identity_ptr->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTmpIdentityInfo, CheckStrategy2) { + // Success: {{2,4,8,16}} + std::vector inputs = {{2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = identity_ptr->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTmpIdentityInfo, test_generate_strategies) { + ASSERT_EQ(identity_ptr->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = identity_ptr->GetStrategyCost(); + for (const auto& swc : sc) { + StrategyPtr sp = swc->strategy_ptr; + Cost cost = *(swc->cost_list[0]); + + identity_ptr->Init(sp); + std::vector inputs_info = identity_ptr->inputs_tensor_info(); + std::vector outputs_info = identity_ptr->outputs_tensor_info(); + ASSERT_DOUBLE_EQ(identity_ptr->GetOperatorCost()->GetMemoryCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.memory_cost_); + ASSERT_DOUBLE_EQ(identity_ptr->GetOperatorCost()->GetCommCost(inputs_info, outputs_info, sp->GetInputStage()), + cost.communication_cost_); + } +} + +TEST_F(TestTmpIdentityInfo, test_generate_strategies_base) { + ASSERT_EQ(identity_ptr->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = identity_ptr->GetStrategyCost(); + + Shapes splittable_inputs = {{1, 1, 1, 1}}; + std::vector sp_vector; + Shapes inputs_shape = {{2, 4, 8, 16}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sc.size(), sp_vector.size()); +} + +TEST_F(TestTmpIdentityInfo, test_generate_strategies_base2) { + ASSERT_EQ(identity_ptr2->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = identity_ptr2->GetStrategyCost(); + + Shapes splittable_inputs = {{1, 1, 1, 1}}; + std::vector sp_vector; + Shapes inputs_shape2 = {{4, 16, 8, 16}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape2, splittable_inputs, &sp_vector); + ASSERT_EQ(sc.size(), sp_vector.size()); +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/ops_info/transpose_test.cc b/tests/ut/cpp/parallel/ops_info/transpose_test.cc new file mode 100644 index 00000000000..d54258915d7 --- /dev/null +++ b/tests/ut/cpp/parallel/ops_info/transpose_test.cc @@ -0,0 +1,193 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/transpose_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class TransposeInfo; +using TransposeInfoPtr = std::shared_ptr; +TransposeInfoPtr transpose; + +class TestTransposeInfo : public UT::Common { + public: + TestTransposeInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestTransposeInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 34; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(32); + stage_map.push_back(2); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{128, 64}}; + Shapes outputs_shape = {{64, 128}}; + std::vector axis = {1, 0}; + ValuePtr val0; + ValuePtr val1 = MakeValue(axis); + std::vector val = {val0, val1}; + + transpose = std::make_shared("transpose_info", inputs_shape, outputs_shape, attr); + transpose->set_input_value(val); +} + +TEST_F(TestTransposeInfo, InferDevMatrixShape1) { + std::vector inputs = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + transpose->Init(strategy); + std::vector dev_matrix_shape = transpose->dev_matrix_shape(); + + std::vector expect = {4, 8}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestTransposeInfo, InferDevMatrixShape2) { + std::vector inputs = {{4, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + transpose->Init(strategy); + std::vector dev_matrix_shape = transpose->dev_matrix_shape(); + + std::vector expect = {8, 4, 1}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestTransposeInfo, InferSliceShape1) { + std::vector str = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, str); + + transpose->Init(strategy); + std::vector inputs = transpose->inputs_tensor_info(); + std::vector outputs = transpose->outputs_tensor_info(); + + Shape input_slice_shape_expect = {32, 8}; + Shape output_slice_shape_expect = {8, 32}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); +} + +TEST_F(TestTransposeInfo, GetTensorLayout1) { + std::vector str = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, str); + + transpose->Init(strategy); + std::vector inputs = transpose->inputs_tensor_info(); + std::vector outputs = transpose->outputs_tensor_info(); + + TensorMap input_expect = {1, 0}; + TensorMap output_expect = {0, 1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestTransposeInfo, GetForwardOp1) { + std::vector inputs = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + transpose->Init(strategy); + OperatorVector forward_op = transpose->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestTransposeInfo, GetMirrorOPs1) { + std::vector inputs = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + transpose->Init(strategy); + MirrorOps mirror_ops = transpose->mirror_ops(); + + size_t size = mirror_ops.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestTransposeInfo, CheckStrategy1) { + std::vector inputs = {{1, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = transpose->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTransposeInfo, CheckStrategy2) { + std::vector inputs = {{2, 4, 8}, {2, 4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = transpose->Init(strategy); + ASSERT_EQ(ret, FAILED); +} + +TEST_F(TestTransposeInfo, CheckStrategy3) { + std::vector inputs = {{4, 8}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + Status ret = transpose->Init(strategy); + ASSERT_EQ(ret, SUCCESS); +} + +TEST_F(TestTransposeInfo, AutoStrategy1) { + ASSERT_EQ(transpose->GenerateStrategies(0), Status::SUCCESS); + std::vector> sc = transpose->GetStrategyCost(); + + Shapes splittable_inputs = {{1, 1}}; + std::vector sp_vector; + Shapes inputs_shape = {{128, 64}}; + GenerateStrategiesForIndependentInputs(0, inputs_shape, splittable_inputs, &sp_vector); + ASSERT_EQ(sc.size(), sp_vector.size()); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/step_auto_parallel_test.cc b/tests/ut/cpp/parallel/step_auto_parallel_test.cc new file mode 100644 index 00000000000..5503377ee29 --- /dev/null +++ b/tests/ut/cpp/parallel/step_auto_parallel_test.cc @@ -0,0 +1,197 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "parallel/step_parallel.h" +#include "parallel/step_auto_parallel.h" +#include "parallel/auto_parallel/edge_costmodel.h" +#include "parallel/ops_info/operator_info.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/static_analysis.h" + +namespace mindspore { +namespace parallel { + +class TestStepAutoParallel : public UT::Common { + public: + TestStepAutoParallel() {} + void SetUp(); + void TearDown() {} +}; + +void TestStepAutoParallel::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 20; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(16); + stage_map.push_back(4); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); +} + +CNodePtr Create_Node(Shape x, Shape y, Shape out) { + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr param1 = func_graph->add_parameter(); + ParameterPtr param2 = func_graph->add_parameter(); + param1->set_name("x"); + param2->set_name("y"); + BaseShapePtr shape1 = std::make_shared(x); + BaseShapePtr shape2 = std::make_shared(y); + BaseShapePtr shape3 = std::make_shared(out); + AbstractBasePtr abstract1 = abstract::FromValue(1, false); + AbstractBasePtr abstract2 = abstract::FromValue(1, false); + AbstractBasePtr abstract3 = abstract::FromValue(1, false); + abstract1->set_shape(shape1); + abstract2->set_shape(shape2); + abstract3->set_shape(shape3); + param1->set_abstract(abstract1); + param2->set_abstract(abstract2); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimMatMul)); + inputs.push_back(param1); + inputs.push_back(param2); + CNodePtr node = func_graph->NewCNode(inputs); + PrimitivePtr prim = node->input(0)->cast()->value()->cast(); + ValuePtr transpose_a = MakeValue(false); + ValuePtr transpose_b = MakeValue(false); + prim->set_attr("transpose_a", transpose_a); + prim->set_attr("transpose_b", transpose_b); + + node->set_abstract(abstract3); + return node; +} + +CNodePtr Create_two_nodes(Shape x, Shape y, Shape z, Shape w, Shape out) { + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr paramX = func_graph->add_parameter(); + ParameterPtr paramY = func_graph->add_parameter(); + ParameterPtr paramW = func_graph->add_parameter(); + paramX->set_name("x"); + paramY->set_name("y"); + paramW->set_name("w"); + BaseShapePtr shapeX = std::make_shared(x); + BaseShapePtr shapeY = std::make_shared(y); + BaseShapePtr shapeZ = std::make_shared(z); + BaseShapePtr shapeW = std::make_shared(w); + BaseShapePtr shapeOut = std::make_shared(out); + AbstractBasePtr abstractX = abstract::FromValue(1, false); + AbstractBasePtr abstractY = abstract::FromValue(1, false); + AbstractBasePtr abstractZ = abstract::FromValue(1, false); + AbstractBasePtr abstractW = abstract::FromValue(1, false); + AbstractBasePtr abstractOut = abstract::FromValue(1, false); + abstractX->set_shape(shapeX); + abstractY->set_shape(shapeY); + abstractZ->set_shape(shapeZ); + abstractW->set_shape(shapeW); + abstractOut->set_shape(shapeOut); + paramX->set_abstract(abstractX); + paramY->set_abstract(abstractY); + paramW->set_abstract(abstractW); + + std::vector MatMul_1_inputs; + MatMul_1_inputs.push_back(NewValueNode(prim::kPrimMatMul)); + MatMul_1_inputs.push_back(paramX); + MatMul_1_inputs.push_back(paramY); + CNodePtr MatMul_1_node = func_graph->NewCNode(MatMul_1_inputs); + PrimitivePtr prim = MatMul_1_node->input(0)->cast()->value()->cast(); + ValuePtr transpose_a = MakeValue(false); + ValuePtr transpose_b = MakeValue(false); + prim->set_attr("transpose_a", transpose_a); + prim->set_attr("transpose_b", transpose_b); + MatMul_1_node->set_abstract(abstractZ); + + std::vector MatMul_2_inputs; + MatMul_2_inputs.push_back(NewValueNode(prim::kPrimMatMul)); + MatMul_2_inputs.push_back(MatMul_1_node); + MatMul_2_inputs.push_back(paramW); + CNodePtr MatMul_2_node = func_graph->NewCNode(MatMul_2_inputs); + PrimitivePtr prim2 = MatMul_2_node->input(0)->cast()->value()->cast(); + ValuePtr transpose_a_2 = MakeValue(false); + ValuePtr transpose_b_2 = MakeValue(false); + prim2->set_attr("transpose_a", transpose_a); + prim2->set_attr("transpose_b", transpose_b); + MatMul_2_node->set_abstract(abstractOut); + + return MatMul_2_node; +} + +TEST_F(TestStepAutoParallel, test_create_op_instance) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Create_Node(inputs_x_dims, inputs_y_dims, outputs_dims); + bool result = node->input(0)->cast()->value()->isa(); + ASSERT_EQ(result, true); + // creat prim and attrs + PrimitivePtr prim = node->input(0)->cast()->value()->cast(); + auto attrs = prim->attrs(); + + // creat shape + Shapes inputs_shape = std::vector{inputs_x_dims, inputs_y_dims}; + Shapes outputs_shape = std::vector{outputs_dims}; + std::vector shape = {inputs_shape, outputs_shape}; + StrategyPtr strategyPtr; + + std::shared_ptr matmul_info = NewOperatorInstance(prim, attrs, shape); + node->set_operator_info(matmul_info); + std::string name_expect = "MatMulInfo00"; + std::string name_test = matmul_info->name(); + ASSERT_EQ(name_expect, name_test); +} + +TEST_F(TestStepAutoParallel, test_create_edge) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_z_dims = {64, 64}; + Shape inputs_w_dims = {64, 128}; + Shape outputs_dim = {64, 128}; + CNodePtr node = Create_two_nodes(inputs_x_dims, inputs_y_dims, outputs_z_dims, inputs_w_dims, outputs_dim); + + // u-->v + PrimitivePtr v_prim = node->input(0)->cast()->value()->cast(); + auto v_attrs = v_prim->attrs(); + PrimitivePtr u_prim = node->input(1)->cast()->input(0)->cast()->value()->cast(); + auto u_attrs = u_prim->attrs(); + + // creat v node + Shapes v_inputs_shape = std::vector{outputs_z_dims, inputs_w_dims}; + Shapes v_outputs_shape = std::vector{outputs_dim}; + std::vector v_shape = {v_inputs_shape, v_outputs_shape}; + StrategyPtr v_strategyPtr; + std::shared_ptr v_matmul_info = NewOperatorInstance(v_prim, v_attrs, v_shape); + + // create u node + Shapes u_inputs_shape = std::vector{inputs_x_dims, inputs_y_dims}; + Shapes u_outputs_shape = std::vector{outputs_z_dims}; + std::vector u_shape = {u_inputs_shape, u_outputs_shape}; + StrategyPtr u_strategyPtr; + std::shared_ptr u_matmul_info = NewOperatorInstance(u_prim, u_attrs, u_shape); + + std::string edge_name = u_prim->name() + "-" + v_prim->name(); + std::shared_ptr edge_ptr = std::make_shared(edge_name, u_matmul_info, v_matmul_info, 0, 0, false); + std::string expected_name = "MatMul-MatMul"; + ASSERT_EQ(edge_ptr->edge_name(), expected_name); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/step_parallel_test.cc b/tests/ut/cpp/parallel/step_parallel_test.cc new file mode 100644 index 00000000000..23debf11ac2 --- /dev/null +++ b/tests/ut/cpp/parallel/step_parallel_test.cc @@ -0,0 +1,552 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "parallel/step_parallel.h" +#include "parallel/graph_util/generate_graph.h" +#include "common/py_func_graph_fetcher.h" +#include "debug/draw.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/static_analysis.h" + +namespace mindspore { +namespace parallel { +extern size_t TOTAL_OPS; +class TestStepParallel : public UT::Common { + public: + TestStepParallel() {} + void SetUp(); + void TearDown() {} +}; + +void TestStepParallel::SetUp() { UT::InitPythonPath(); } + +void Init_Device_Manager() { + std::list dev_list; + + for (int32_t i = 0; i < 20; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(16); + stage_map.push_back(4); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); +} + +CNodePtr Make_Node(Shape x, Shape y, Shape out, int condition = 0) { + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr param1 = func_graph->add_parameter(); + ParameterPtr param2 = func_graph->add_parameter(); + param1->set_name("x"); + param2->set_name("y"); + BaseShapePtr shape1 = std::make_shared(x); + BaseShapePtr shape2 = std::make_shared(y); + BaseShapePtr shape3 = std::make_shared(out); + std::shared_ptr inputs_x = std::make_shared(); + inputs_x->set_data_type(kNumberTypeInt32); + inputs_x->set_shape(x); + std::shared_ptr inputs_y = std::make_shared(); + inputs_y->set_data_type(kNumberTypeInt32); + inputs_y->set_shape(y); + std::shared_ptr inputs_out = std::make_shared(); + inputs_out->set_data_type(kNumberTypeInt32); + inputs_out->set_shape(out); + AbstractBasePtr abstract1 = abstract::FromValue(inputs_x, true); + AbstractBasePtr abstract2 = abstract::FromValue(inputs_y, true); + AbstractBasePtr abstract3 = abstract::FromValue(inputs_out, true); + switch (condition) { + case 0: { + abstract1->set_shape(shape1); + abstract2->set_shape(shape2); + abstract3->set_shape(shape3); + param1->set_abstract(abstract1); + param2->set_abstract(abstract2); + break; + } + case 1: { + abstract1->set_shape(nullptr); + param1->set_abstract(abstract1); + param2->set_abstract(abstract2); + break; + } + case 2: { + abstract1->set_shape(shape1); + abstract2->set_shape(shape2); + param1->set_abstract(abstract1); + param2->set_abstract(abstract2); + abstract3 = abstract::FromValue(1, false); + break; + } + case 3: { + std::vector shape_o = {std::make_shared(x), std::make_shared(y)}; + BaseShapePtr shape4 = std::make_shared(shape_o); + abstract1->set_shape(shape1); + abstract2->set_shape(shape2); + abstract3->set_shape(shape4); + param1->set_abstract(abstract1); + param2->set_abstract(abstract2); + break; + } + default: + MS_LOG(INFO) << "Do Nothing!"; + } + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimMatMul)); + inputs.push_back(param1); + inputs.push_back(param2); + CNodePtr node = func_graph->NewCNode(inputs); + node->set_abstract(abstract3); + return node; +} + +FuncGraphManagerPtr Make_Manager(int condition = 0) { + Shape inputs_x = {64, 32}; + Shape inputs_y = {32, 64}; + Shape inputs_z = {64, 128}; + Shape outputs_1 = {64, 64}; + Shape outputs_2 = {64, 128}; + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr param1 = func_graph->add_parameter(); + ParameterPtr param2 = func_graph->add_parameter(); + ParameterPtr param3 = func_graph->add_parameter(); + std::shared_ptr inputs_x_dim = std::make_shared(); + inputs_x_dim->set_data_type(kNumberTypeInt32); + inputs_x_dim->set_shape(inputs_x); + std::shared_ptr inputs_y_dim = std::make_shared(); + inputs_y_dim->set_data_type(kNumberTypeInt32); + inputs_y_dim->set_shape(inputs_y); + std::shared_ptr inputs_z_dim = std::make_shared(); + inputs_z_dim->set_data_type(kNumberTypeInt32); + inputs_z_dim->set_shape(inputs_z); + std::shared_ptr inputs_out1_dim = std::make_shared(); + inputs_out1_dim->set_data_type(kNumberTypeInt32); + inputs_out1_dim->set_shape(outputs_1); + std::shared_ptr inputs_out2_dim = std::make_shared(); + inputs_out2_dim->set_data_type(kNumberTypeInt32); + inputs_out2_dim->set_shape(outputs_2); + AbstractBasePtr abstract_x = abstract::FromValue(inputs_x_dim, true); + AbstractBasePtr abstract_y = abstract::FromValue(inputs_y_dim, true); + AbstractBasePtr abstract_z = abstract::FromValue(inputs_z_dim, true); + AbstractBasePtr abstract_out1 = abstract::FromValue(inputs_out1_dim, true); + AbstractBasePtr abstract_out2 = abstract::FromValue(inputs_out2_dim, true); + param1->set_abstract(abstract_x); + param2->set_abstract(abstract_y); + param3->set_abstract(abstract_z); + std::vector v1 = {2, 2}; + std::vector v2 = {2, 4}; + std::vector elements = {MakeValue(v1), MakeValue(v2)}; + ValueTuplePtr var = std::make_shared(elements); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimMatMul)); + inputs.push_back(param1); + inputs.push_back(param2); + CNodePtr node1 = func_graph->NewCNode(inputs); + node1->set_in_forward_flag(true); + node1->set_abstract(abstract_out1); + PrimitivePtr prim1 = node1->input(0)->cast()->value()->cast(); + ValuePtr transpose_a = MakeValue(false); + ValuePtr transpose_b = MakeValue(false); + prim1->AddAttr("transpose_a", transpose_a); + prim1->AddAttr("transpose_b", transpose_b); + prim1->AddAttr("instance_name", MakeValue("matmul1")); + prim1->AddAttr("strategy", var); + inputs.clear(); + std::vector v3 = {2, 2}; + std::vector v4 = {2, 4}; + std::vector elements2 = {MakeValue(v3), MakeValue(v4)}; + ValueTuplePtr var2 = std::make_shared(elements2); + inputs.push_back(NewValueNode(prim::kPrimMatMul)); + inputs.push_back(node1); + inputs.push_back(param3); + CNodePtr node2 = func_graph->NewCNode(inputs); + node2->set_in_forward_flag(true); + node2->set_abstract(abstract_out2); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(node2); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + cnode_return->set_in_forward_flag(true); + func_graph->set_return(cnode_return); + PrimitivePtr prim2 = node2->input(0)->cast()->value()->cast(); + prim2->AddAttr("transpose_a", transpose_a); + prim2->AddAttr("transpose_b", transpose_b); + prim2->AddAttr("instance_name", MakeValue("matmul2")); + prim2->AddAttr("strategy", var2); + switch (condition) { + case 1: { + prim1->set_attr("strategy", MakeValue(0)); + break; + } + case 2: { + std::vector elements_t = {MakeValue(0)}; + ValueTuplePtr var_t = std::make_shared(elements_t); + prim1->set_attr("strategy", var_t); + break; + } + case 3: { + std::vector vt1 = {2, 4}; + std::vector vt2 = {2, 4}; + std::vector elements_t2 = {MakeValue(vt1), MakeValue(vt2)}; + ValueTuplePtr var_t2 = std::make_shared(elements_t2); + prim1->set_attr("strategy", var_t2); + break; + } + } + std::vector func_graphs{func_graph}; + FuncGraphManagerPtr manager = std::make_shared(func_graphs, true); + manager->Init(); + return manager; +} + +TEST_F(TestStepParallel, GetPythonPath1) { + OperatorName operator_name = "AllReduce"; + const std::string expect = "mindspore.ops.operations"; + auto temp = parallel::GetOpPythonPath(operator_name); + ASSERT_EQ(temp, expect); +} + +TEST_F(TestStepParallel, GetPythonPath2) { + OperatorName operator_name = "TensorAdd"; + const std::string expect = "mindspore.ops.operations"; + auto temp = parallel::GetOpPythonPath(operator_name); + ASSERT_EQ(temp, expect); +} + +TEST_F(TestStepParallel, ExtractStrategy) { + Dimensions v1 = {2, 2}; + Dimensions v2 = {4, 4}; + std::unordered_map attrs; + // stage + ValuePtr val1 = MakeValue(v1); + ValuePtr val2 = MakeValue(v2); + std::vector elements = {val1, val2}; + ValueTuplePtr strategy_tuple = std::make_shared(elements); + attrs["strategy"] = strategy_tuple; + std::vector strategy_expect = {v1, v2}; + StrategyPtr strategy = ExtractStrategy(attrs); + std::vector strategy_test = strategy->GetInputDim(); + + ASSERT_EQ(strategy_expect, strategy_test); +} + +TEST_F(TestStepParallel, ExtractShape) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Make_Node(inputs_x_dims, inputs_y_dims, outputs_dims, 4); + EXPECT_THROW({ ExtractShape(node); }, std::runtime_error); +} + +TEST_F(TestStepParallel, ExtractShape1) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Make_Node(inputs_x_dims, inputs_y_dims, outputs_dims); + std::vector shape_test = ExtractShape(node); + Shapes inputs_shape = std::vector{inputs_x_dims, inputs_y_dims}; + Shapes outputs_shape = std::vector{outputs_dims}; + std::vector shape_expect = {inputs_shape, outputs_shape}; + ASSERT_EQ(shape_test, shape_expect); +} + +TEST_F(TestStepParallel, ExtractShape2) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Make_Node(inputs_x_dims, inputs_y_dims, outputs_dims, 1); + EXPECT_THROW({ ExtractShape(node); }, std::runtime_error); +} + +TEST_F(TestStepParallel, ExtractShape3) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Make_Node(inputs_x_dims, inputs_y_dims, outputs_dims, 3); + Shapes inputs_shape = std::vector{inputs_x_dims, inputs_y_dims}; + std::vector shape_expect = {inputs_shape, inputs_shape}; + std::vector shape_test = ExtractShape(node); + ASSERT_EQ(shape_test, shape_expect); +} + +TEST_F(TestStepParallel, ExtractShape4) { + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Make_Node(inputs_x_dims, inputs_y_dims, outputs_dims, 2); + Shapes inputs_shape = std::vector{inputs_x_dims, inputs_y_dims}; + EXPECT_THROW({ ExtractShape(node); }, std::runtime_error); +} + +TEST_F(TestStepParallel, CreatOpInstance) { + ValuePtr attr0_value = MakeValue(REDUCE_OP_SUM); + ValuePtr attr1_value = MakeValue("0-1-2"); + Attr attr0 = std::make_pair("op", attr0_value); + Attr attr1 = std::make_pair("group", attr1_value); + OperatorAttrs attrs = {attr0, attr1}; + OperatorName op_name = "AllReduce"; + OperatorParams operator_param; + OperatorArgs args = std::make_pair(attrs, operator_param); + auto op_instance = CreatOpInstance(args.first, op_name, "test"); + ASSERT_TRUE(op_instance); + PrimitivePyPtr allreduce_ptr = dyn_cast(op_instance); + ASSERT_TRUE(allreduce_ptr); + if (nullptr != allreduce_ptr) { + MS_LOG(INFO) << "Get PrimitivePyPtr: " << allreduce_ptr->name(); + auto func = allreduce_ptr->GetComputeFunction(); + if (py::isinstance(func)) { + MS_LOG(EXCEPTION) << "" << allreduce_ptr->name() << "'s compute function is not implemented"; + } + + std::vector arglist; + (void)std::transform(attrs.begin(), attrs.end(), std::back_inserter(arglist), + [](Attr attr) { return ValuePtrToPyData(attr.second); }); + py::object allreduce_pyobj = parse::python_adapter::CallPyFn( + "mindspore.parallel._utils", "_get_python_op", "AllReduce", "mindspore.ops.operations", "test", arglist); + py::dict opAttr = py::getattr(allreduce_pyobj, "attrs"); + std::unordered_map attributes{}; + for (auto item : opAttr) { + if (!py::isinstance(item.first)) { + MS_LOG(EXCEPTION) << "type error in py dict convert"; + } + std::string name = py::cast(item.first); + MS_LOG(INFO) << "Attr name: " << name; + + ValuePtr converted_ret; + if (name == "op") { + parse::ConvertData(py::cast(item.second), &converted_ret); + ASSERT_EQ(converted_ret->ToString(), "sum"); + } else { + if (name == "group") { + parse::ConvertData(py::cast(item.second), &converted_ret); + ASSERT_EQ(converted_ret->ToString(), "0-1-2"); + } else if (name == "fusion") { + parse::ConvertData(py::cast(item.second), &converted_ret); + ASSERT_EQ(converted_ret->ToString(), "0"); + } else if (name == "instance_name") { + parse::ConvertData(py::cast(item.second), &converted_ret); + ASSERT_EQ(converted_ret->ToString(), "test"); + } else { + MS_LOG(EXCEPTION) << "Test failed"; + } + } + attributes.emplace(name, converted_ret); + } + } +} + +TEST_F(TestStepParallel, CreatOpInstance1) { + OperatorAttrs attrs; + OperatorName op_name = "ABC"; + OperatorParams operator_param; + OperatorArgs args = std::make_pair(attrs, operator_param); + EXPECT_THROW({ CreatOpInstance(args.first, op_name, "test"); }, std::runtime_error); +} + +TEST_F(TestStepParallel, OperatorInstance) { + Init_Device_Manager(); + // creat attrs and prim + PrimitivePtr prim = NewValueNode(prim::kPrimMatMul)->value()->cast(); + ValuePtr transpose_a = MakeValue(false); + ValuePtr transpose_b = MakeValue(false); + prim->set_attr("transpose_a", transpose_a); + prim->set_attr("transpose_b", transpose_b); + auto attrs = prim->attrs(); + // creat strategy + std::vector strategy = {{2, 2}, {2, 4}}; + StrategyPtr strategyPtr = parallel::NewStrategy(0, strategy); + // creat shape + Shapes inputs_shape = std::vector{{64, 32}, {32, 64}}; + Shapes outputs_shape = std::vector{{64, 64}}; + std::vector shape = {inputs_shape, outputs_shape}; + TOTAL_OPS = 0; + OperatorInfoPtr matmul_info = OperatorInstance(prim, attrs, shape); + matmul_info->Init(strategyPtr); + std::string name_expect = "MatMulInfo00"; + std::string name_test = matmul_info->name(); + ASSERT_EQ(name_expect, name_test); +} + +TEST_F(TestStepParallel, ExtractInformation) { + Init_Device_Manager(); + FuncGraphManagerPtr manager = Make_Manager(); + FuncGraphSet graphs = manager->func_graphs(); + FuncGraphPtr graph = *graphs.begin(); + auto ret = graph->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + ExtractInformation(all_nodes); +} + +TEST_F(TestStepParallel, ExtractInformation2) { + Init_Device_Manager(); + FuncGraphManagerPtr manager = Make_Manager(2); + FuncGraphSet graphs = manager->func_graphs(); + FuncGraphPtr graph = *graphs.begin(); + auto ret = graph->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + EXPECT_THROW({ ExtractInformation(all_nodes); }, std::runtime_error); +} + +TEST_F(TestStepParallel, ExtractInformation3) { + Init_Device_Manager(); + FuncGraphManagerPtr manager = Make_Manager(3); + FuncGraphSet graphs = manager->func_graphs(); + FuncGraphPtr graph = *graphs.begin(); + auto ret = graph->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + EXPECT_THROW({ ExtractInformation(all_nodes); }, std::runtime_error); +} + +TEST_F(TestStepParallel, ForwardCommunication1) { + Init_Device_Manager(); + ValuePtr attr0_value = MakeValue(REDUCE_OP_SUM); + ValuePtr attr1_value = MakeValue("0-1-2"); + Attr attr0 = std::make_pair("op", attr0_value); + Attr attr1 = std::make_pair("group", attr1_value); + OperatorAttrs attrs = {attr0, attr1}; + OperatorName op_name = "AllReduce"; + OperatorParams operator_param; + OperatorArgs args = std::make_pair(attrs, operator_param); + Operator op = std::make_pair(op_name, args); + OperatorVector op_list = {op, op}; + FuncGraphManagerPtr manager = Make_Manager(); + FuncGraphSet graphs = manager->func_graphs(); + FuncGraphPtr graph = *graphs.begin(); + auto ret = graph->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + ExtractInformation(all_nodes); + for (auto &node : all_nodes) { + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + FuncGraphPtr func_graph = node->func_graph(); + PrimitivePtr prim = cnode->input(0)->cast()->value()->cast(); + if (prim->name() == "MatMul") { + ForwardCommunication(op_list, cnode); + draw::Draw("./forwardcommunication.dot", func_graph); + } + } + AnfNodeSet after_nodes = manager->all_nodes(); + for (auto &node : after_nodes) { + if (!node->isa()) { + continue; + } + auto &inputs = node->cast()->inputs(); + PrimitivePtr prim = inputs[0]->cast()->value()->cast(); + if (prim->name() == "return" || prim->name() == "MatMul") { + if (!inputs[1]->isa()) { + CNodePtr pre_node = inputs[1]->cast(); + PrimitivePtr pre_prim = pre_node->input(0)->cast()->value()->cast(); + CNodePtr pre_node2 = pre_node->input(1)->cast(); + PrimitivePtr pre_prim2 = pre_node2->input(0)->cast()->value()->cast(); + ASSERT_EQ("AllReduce", pre_prim->name()); + ASSERT_EQ("AllReduce", pre_prim2->name()); + } + } + } +} + +TEST_F(TestStepParallel, ForwardCommunication2) { + OperatorVector op_list; + FuncGraphManagerPtr manager = Make_Manager(); + FuncGraphSet graphs = manager->func_graphs(); + FuncGraphPtr graph = *graphs.begin(); + auto ret = graph->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + ExtractInformation(all_nodes); + for (auto &node : all_nodes) { + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + FuncGraphPtr func_graph = node->func_graph(); + func_graph->set_manager(nullptr); + PrimitivePtr prim = GetValueNode(cnode->input(0)); + if (prim->name() == "MatMul") { + EXPECT_THROW({ ForwardCommunication(op_list, cnode); }, std::runtime_error); + break; + } + } +} + +TEST_F(TestStepParallel, ForwardCommunication3) { + OperatorVector op_list; + FuncGraphManagerPtr manager = Make_Manager(); + FuncGraphSet graphs = manager->func_graphs(); + FuncGraphPtr graph = *graphs.begin(); + auto ret = graph->get_return(); + std::vector all_nodes = DeepScopedGraphSearch(ret); + ExtractInformation(all_nodes); + for (auto &node : all_nodes) { + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + FuncGraphPtr func_graph = node->func_graph(); + PrimitivePtr prim = GetValueNode(cnode->input(0)); + if (prim->name() == "MatMul") { + OperatorAttrs attrs; + OperatorParams operator_param; + OperatorArgs args = std::make_pair(attrs, operator_param); + Operator op = std::make_pair("ABC", args); + OperatorVector op_list = {op}; + EXPECT_THROW({ ForwardCommunication(op_list, cnode); }, std::runtime_error); + break; + } + } +} + +TEST_F(TestStepParallel, GetTensorInLayout) { + Init_Device_Manager(); + // creat attrs and prim + FuncGraphPtr func_graph = std::make_shared(); + Shape inputs_x_dims = {64, 32}; + Shape inputs_y_dims = {32, 64}; + Shape outputs_dims = {64, 64}; + CNodePtr node = Make_Node(inputs_x_dims, inputs_y_dims, outputs_dims); + std::vector inputs(node->inputs()); + CNodePtr node1 = func_graph->NewCNode(inputs); + PrimitivePtr prim = node1->input(0)->cast()->value()->cast(); + ValuePtr transpose_a = MakeValue(false); + ValuePtr transpose_b = MakeValue(false); + prim->set_attr("transpose_a", transpose_a); + prim->set_attr("transpose_b", transpose_b); + auto attrs = prim->attrs(); + // creat strategy + std::vector strategy = {{2, 2}, {2, 4}}; + StrategyPtr strategyPtr = parallel::NewStrategy(0, strategy); + // creat shape + Shapes inputs_shape = std::vector{{64, 32}, {32, 64}}; + Shapes outputs_shape = std::vector{{64, 64}}; + std::vector shape = {inputs_shape, outputs_shape}; + OperatorInfoPtr matmul_info = OperatorInstance(prim, attrs, shape); + matmul_info->Init(strategyPtr); + node->set_operator_info(matmul_info); + OperatorInfoPtr distribute_operator_pre = node->operator_info(); + TensorLayout tensorlayout_e; + std::vector array = {64, 64}; + TensorLayout tensorlayout = GetTensorInLayout(node1, prim, distribute_operator_pre); + std::vector tensor_shape_test = tensorlayout.tensor_shape().array(); + ASSERT_EQ(array, tensor_shape_test); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/strategy_test.cc b/tests/ut/cpp/parallel/strategy_test.cc new file mode 100644 index 00000000000..21988e3517b --- /dev/null +++ b/tests/ut/cpp/parallel/strategy_test.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" + +namespace mindspore { +namespace parallel { + +class TestStrategy : public UT::Common { + public: + TestStrategy() {} + + void SetUp() {} + void TearDown() {} +}; + +TEST_F(TestStrategy, GetInputNumber) { + int32_t number = 2; + int32_t stage = 1; + std::vector dimension1 = {2, 4}; + std::vector dimension2 = {2, 2}; + std::vector> inputs = {dimension1, dimension2}; + + Strategy strategy(stage, inputs); + int32_t number_test = strategy.GetInputNumber(); + ASSERT_EQ(number, number_test); +} + +TEST_F(TestStrategy, GetInputStage) { + int32_t stage = 1; + std::vector dimension1 = {2, 4}; + std::vector dimension2 = {2, 2}; + std::vector> inputs = {dimension1, dimension2}; + + Strategy strategy(stage, inputs); + int32_t stage_test = strategy.GetInputStage(); + ASSERT_EQ(stage, stage_test); +} + +TEST_F(TestStrategy, GetInputDim) { + int32_t stage = 1; + std::vector dimension1 = {2, 4}; + std::vector dimension2 = {2, 2}; + std::vector> inputs = {dimension1, dimension2}; + + Strategy strategy(stage, inputs); + std::vector> inputs_test = strategy.GetInputDim(); + ASSERT_EQ(inputs, inputs_test); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/construct_operator_test.cc b/tests/ut/cpp/parallel/tensor_layout/construct_operator_test.cc new file mode 100644 index 00000000000..bebe55250b0 --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/construct_operator_test.cc @@ -0,0 +1,139 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#include "ir/value.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/matmul_info.h" +#include "parallel/device_manager.h" +#include "parallel/tensor_layout/construct_operator.h" + +namespace mindspore { +namespace parallel { + +class MatMulInfo; +using MatMulInfoPtr = std::shared_ptr; +ConstructOperator constructor; + +class TestConstructOperator : public UT::Common { + public: + TestConstructOperator() {} + + void SetUp(); + + virtual void TearDown() {} +}; + +void TestConstructOperator::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + ValuePtr transpose_a_1 = MakeValue(false); + ValuePtr transpose_b_1 = MakeValue(false); + std::unordered_map attr_1 = {{"transpose_a", transpose_a_1}, {"transpose_b", transpose_b_1}}; + + Shapes inputs_shape_1 = {{2, 4, 8, 16}, {2, 4, 16, 32}}; + Shapes outputs_shape_1 = {{2, 4, 8, 32}}; + + MatMulInfoPtr matmul = std::make_shared("matmul_info", inputs_shape_1, outputs_shape_1, attr_1); + + std::vector str = {{2, 4, 8, 16}, {2, 4, 16, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + matmul->Init(strategy); + Shape tensor_shape = {512, 1024}; + Shape dev_matrix_shape = {2, 4, 8, 16, 1}; + RankList used_dev_list = g_device_manager->GetDeviceListByStageId(0); + constructor.Init(used_dev_list, dev_matrix_shape); + constructor.UpdateTensorShape(tensor_shape); +} + +TEST_F(TestConstructOperator, TestReshapeOP) { + Shape shape = {512, 512, 2}; + ASSERT_EQ(constructor.ReshapeOP(shape), Status::SUCCESS); +} + +TEST_F(TestConstructOperator, TestStridedSliceOP) { + Args args = {1, 2, 3}; + int32_t split_count = args[0]; + int32_t split_dim = args[1]; + Shape device_arrangement = {8, 4}; + Arrangement dev_mat; + dev_mat.Init(device_arrangement); + Shape map = {1, -1}; + Map tensor_map; + tensor_map.Init(map); + Shape shape = {512, 1024}; + Arrangement tensor_shape; + tensor_shape.Init(shape); + TensorLayout tensor_layout; + tensor_layout.Init(dev_mat, tensor_map, tensor_shape); + ASSERT_EQ(constructor.StridedSliceOP(args), Status::SUCCESS); + + Operator op = constructor.GetOperator(); + OperatorParams params = op.second.second; + ValuePtr begin_ptr = params[0].first.second; + ValuePtr end_ptr = params[1].first.second; + Shape begin = GetValue>(begin_ptr); + Shape end = GetValue>(end_ptr); + for (size_t i = 0; i < begin.size(); i++) { + int32_t diff = end[i] - begin[i]; + int32_t num = shape[i]; + if (SizeToInt(i) != split_dim) { + ASSERT_EQ(diff, shape[i]); + } else { + ASSERT_EQ(diff, num / split_count); + } + } +} + +TEST_F(TestConstructOperator, TestAllGatherOP) { + int32_t dev_dim = 2; + ASSERT_EQ(constructor.AllGatherOP(dev_dim), Status::SUCCESS); +} + +TEST_F(TestConstructOperator, TestConcatOP) { + int32_t concat_dim = 0; + ASSERT_EQ(constructor.ConcatOP(concat_dim), Status::SUCCESS); +} + +TEST_F(TestConstructOperator, TestSplitOP) { + int32_t split_count = 2; + ASSERT_EQ(constructor.SplitOP(split_count), Status::SUCCESS); +} + +TEST_F(TestConstructOperator, TestAlltoAllOP) { + int32_t split_count = 2; + int32_t split_dim = 0; + int32_t concat_dim = 1; + int32_t dev_dim = 3; + Args args = {split_count, split_dim, concat_dim, dev_dim}; + ASSERT_EQ(constructor.AlltoAllOP(args), Status::SUCCESS); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/redistribution_layout_transfer_test.cc b/tests/ut/cpp/parallel/tensor_layout/redistribution_layout_transfer_test.cc new file mode 100644 index 00000000000..4e34847582e --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/redistribution_layout_transfer_test.cc @@ -0,0 +1,294 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/tensor_layout/redistribution_layout_transfer.h" +#include "util_layout_gen_test.h" + +namespace mindspore { +namespace parallel { + +using DeviceArrangement = std::vector; +using TensorMap = std::vector; +using TensorShape = std::vector; + +class TestRedistributionLayoutTransfer : public UT::Common { + public: + TestRedistributionLayoutTransfer() {} + + void SetUp() { UT::InitPythonPath(); } + + virtual void TearDown() {} +}; + +void RedistributionLayoutTransferTestFunction( + const DeviceArrangement& in_device_arrangement_shape, const TensorMap& in_tensor_map_shape, + const TensorShape& tensor_shape_shape, const DeviceArrangement& out_device_arrangement_shape, + const TensorMap& out_tensor_map_shape, DeviceArrangement* unified_device_arrangement_shape, + TensorMap* unified_in_tensor_map_shape, TensorMap* unified_out_tensor_map_shape, + TensorMap* unified_tensor_shape_shape) { + Arrangement in_device_arrangement; + Status status = in_device_arrangement.Init(in_device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map in_tensor_map; + status = in_tensor_map.Init(in_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement tensor_shape; + status = tensor_shape.Init(tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement out_device_arrangement; + status = out_device_arrangement.Init(out_device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map out_tensor_map; + status = out_tensor_map.Init(out_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout in_tensor_layout; + status = in_tensor_layout.Init(in_device_arrangement, in_tensor_map, tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout out_tensor_layout; + status = out_tensor_layout.Init(out_device_arrangement, out_tensor_map, tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + RedistributionLayoutTransfer tensor_redistribution; + status = tensor_redistribution.Init(in_tensor_layout, out_tensor_layout); + ASSERT_EQ(Status::SUCCESS, status); + std::shared_ptr unified_ptr = tensor_redistribution.UnifyDeviceArrangementAndTensorShape(); + ASSERT_NE(nullptr, unified_ptr); + TensorLayout unified_in = unified_ptr->from_in(); + TensorLayout unified_out = unified_ptr->to_in(); + // get unified_in result + Arrangement unified_in_device_arrangement = unified_in.device_arrangement(); + Map unified_in_tensor_map = unified_in.tensor_map(); + *unified_in_tensor_map_shape = unified_in_tensor_map.array(); + Arrangement unified_in_tensor_shape = unified_in.tensor_shape(); + ASSERT_EQ(Status::SUCCESS, status); + // get unified_out result + Arrangement unified_out_device_arrangement = unified_out.device_arrangement(); + Map unified_out_tensor_map = unified_out.tensor_map(); + *unified_out_tensor_map_shape = unified_out_tensor_map.array(); + Arrangement unified_out_tensor_shape = unified_out.tensor_shape(); + // checkout valid + ASSERT_EQ(unified_in_device_arrangement.array(), unified_out_device_arrangement.array()); + ASSERT_EQ(unified_in_tensor_shape.array(), unified_out_tensor_shape.array()); + *unified_device_arrangement_shape = unified_in_device_arrangement.array(); + *unified_tensor_shape_shape = unified_in_tensor_shape.array(); +} + +void RedistributionLayoutCheck(const DeviceArrangement& in_device_arrangement, const TensorMap& in_tensor_map, + const TensorShape& tensor_shape, const DeviceArrangement& out_device_arrangement, + const TensorMap& out_tensor_map, + const DeviceArrangement& unified_device_arrangement_expect, + const TensorMap& unified_in_tensor_map_expect, + const TensorMap& unified_out_tensor_map_expect, + const TensorMap& unified_tensor_shape_expect) { + DeviceArrangement unified_device_arrangement; + TensorMap unified_in_tensor_map; + TensorMap unified_out_tensor_map; + TensorShape unified_tensor_shape; + RedistributionLayoutTransferTestFunction(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, + out_tensor_map, &unified_device_arrangement, &unified_in_tensor_map, + &unified_out_tensor_map, &unified_tensor_shape); + // check unified_in result + ASSERT_EQ(unified_device_arrangement_expect, unified_device_arrangement); + ASSERT_EQ(unified_in_tensor_map_expect, unified_in_tensor_map); + ASSERT_EQ(unified_tensor_shape_expect, unified_tensor_shape); + // check unified_out result + ASSERT_EQ(unified_device_arrangement_expect, unified_device_arrangement); + ASSERT_EQ(unified_out_tensor_map_expect, unified_out_tensor_map); + ASSERT_EQ(unified_tensor_shape_expect, unified_tensor_shape); +} +/* + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [2, 16] + * out_tensor_map = [1, 0], + * out_tensor_shape = [512, 1024] + * in_step1_layout_.device_arrangement = [2, 4, 4] + * in_step1_layout_.tensor_map = [2, 1, 0] + * in_step1_layout_.tensor_shape = [2, 256, 1024] + * out_step1_layout_.device_arrangement = [2, 4, 4] + * out_step1_layout_.tensor_map = [2, 1, 0] + * out_step1_layout_.tensor_shape = [512, 4, 256] + * in_step2_layout_.device_arrangement = [2, 4, 4] + * in_step2_layout_.tensor_map = [2, 1, 0, -1] + * in_step2_layout_.tensor_shape = [2, 256, 4, 256] + * out_step2_layout_.device_arrangement = [2, 4, 4] + * out_step2_layout_.tensor_map = [2, -1, 1, 0] + * out_step2_layout_.tensor_shape = [2, 256, 4, 256] + */ +TEST_F(TestRedistributionLayoutTransfer, RedistributionLayoutTransfer1) { + DeviceArrangement in_device_arrangement = {8, 4}; + TensorMap in_tensor_map = {1, 0}; + TensorShape tensor_shape = {512, 1024}; + DeviceArrangement out_device_arrangement = {2, 16}; + TensorMap out_tensor_map = {1, 0}; + DeviceArrangement unified_device_arrangement_expect = {2, 4, 4}; + TensorMap unified_in_tensor_map_expect = {2, 1, 0, -1}; + TensorMap unified_out_tensor_map_expect = {2, -1, 1, 0}; + TensorShape unified_tensor_shape_expect = {2, 256, 4, 256}; + RedistributionLayoutCheck(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, out_tensor_map, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +/* + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [2, 16] + * out_tensor_map = [0, 1], + * out_tensor_shape = [512, 1024] + * in_step1_layout_.device_arrangement = [2, 4, 4] + * in_step1_layout_.tensor_map = [2, 1, 0] + * in_step1_layout_.tensor_shape = [2, 256, 1024] + * out_step1_layout_.device_arrangement = [2, 4, 4] + * out_step1_layout_.tensor_map = [1, 0, 2] + * out_step1_layout_.tensor_shape = [4, 128, 1024] + * in_step2_layout_.device_arrangement = [2, 2, 2, 4] + * in_step2_layout_.tensor_map = [3, 2, 1, 0] + * in_step2_layout_.tensor_shape = [2, 2, 128, 1024] + * out_step2_layout_.device_arrangement = [2, 2, 2, 4] + * out_step2_layout_.tensor_map = [2, 1, 0, 3] + * out_step2_layout_.tensor_shape = [2, 2, 128, 1024] + */ +TEST_F(TestRedistributionLayoutTransfer, RedistributionLayoutTransfer2) { + DeviceArrangement in_device_arrangement = {8, 4}; + TensorMap in_tensor_map = {1, 0}; + TensorShape tensor_shape = {512, 1024}; + DeviceArrangement out_device_arrangement = {2, 16}; + TensorMap out_tensor_map = {0, 1}; + DeviceArrangement unified_device_arrangement_expect = {2, 2, 2, 4}; + TensorMap unified_in_tensor_map_expect = {3, 2, 1, 0}; + TensorMap unified_out_tensor_map_expect = {2, 1, 0, 3}; + TensorShape unified_tensor_shape_expect = {2, 2, 128, 1024}; + RedistributionLayoutCheck(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, out_tensor_map, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +TEST_F(TestRedistributionLayoutTransfer, RedistributionLayoutTransfer3) { + DeviceArrangement in_device_arrangement = {8}; + TensorMap in_tensor_map = {0, -1}; + TensorShape tensor_shape = {512, 1024}; + DeviceArrangement out_device_arrangement = {2, 2, 2}; + TensorMap out_tensor_map = {2, 1}; + DeviceArrangement unified_device_arrangement_expect = {2, 2, 2}; + TensorMap unified_in_tensor_map_expect = {2, 1, 0, -1}; + TensorMap unified_out_tensor_map_expect = {2, -1, -1, 1}; + TensorShape unified_tensor_shape_expect = {2, 2, 128, 1024}; + RedistributionLayoutCheck(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, out_tensor_map, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +TEST_F(TestRedistributionLayoutTransfer, RedistributionLayoutTransfer4) { + DeviceArrangement in_device_arrangement = {16, 1, 1}; + TensorMap in_tensor_map = {2, 0}; + TensorShape tensor_shape = {512, 1024}; + DeviceArrangement out_device_arrangement = {1, 16, 1}; + TensorMap out_tensor_map = {2, 1}; + DeviceArrangement unified_device_arrangement_expect = {16}; + TensorMap unified_in_tensor_map_expect = {0, -1}; + TensorMap unified_out_tensor_map_expect = {-1, 0}; + TensorShape unified_tensor_shape_expect = {512, 1024}; + RedistributionLayoutCheck(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, out_tensor_map, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +TEST_F(TestRedistributionLayoutTransfer, RedistributionLayoutTransfer5) { + DeviceArrangement in_device_arrangement = {16}; + TensorMap in_tensor_map = {0, -1}; + TensorShape tensor_shape = {512, 1024}; + DeviceArrangement out_device_arrangement = {16}; + TensorMap out_tensor_map = {-1, 0}; + DeviceArrangement unified_device_arrangement_expect = {16}; + TensorMap unified_in_tensor_map_expect = {0, -1}; + TensorMap unified_out_tensor_map_expect = {-1, 0}; + TensorShape unified_tensor_shape_expect = {512, 1024}; + RedistributionLayoutCheck(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, out_tensor_map, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +void ValidRedistributionLayoutCheck(const DeviceArrangement& in_device_arrangement, const TensorMap& in_tensor_map, + const TensorShape& tensor_shape, const DeviceArrangement& out_device_arrangement, + const TensorMap& out_tensor_map) { + DeviceArrangement unified_device_arrangement; + TensorMap unified_in_tensor_map; + TensorMap unified_out_tensor_map; + TensorShape unified_tensor_shape; + RedistributionLayoutTransferTestFunction(in_device_arrangement, in_tensor_map, tensor_shape, out_device_arrangement, + out_tensor_map, &unified_device_arrangement, &unified_in_tensor_map, + &unified_out_tensor_map, &unified_tensor_shape); + // check unified_in result + ValidLayoutChangeCheck(in_device_arrangement, in_tensor_map, tensor_shape, unified_device_arrangement, + unified_in_tensor_map, unified_tensor_shape); + // check unified_out result + ValidLayoutChangeCheck(out_device_arrangement, out_tensor_map, tensor_shape, unified_device_arrangement, + unified_out_tensor_map, unified_tensor_shape); +} + +void ValidRedistributionLayoutCheckAll(const int32_t& device_pow_size, const int32_t& tensor_pow_size, + const int32_t& max_device_dim, const int32_t& max_shape_dim) { + std::vector> layout_list; + GenerateValidLayoutByDeviceSizeAndTensorSize(device_pow_size, tensor_pow_size, max_device_dim, max_shape_dim, + &layout_list); + for (uint32_t in = 0; in < layout_list.size(); in++) { + for (uint32_t out = 0; out < layout_list.size(); out++) { + DeviceArrangement in_device_arrangement = std::get<0>(layout_list[in]); + TensorMap in_tensor_map = std::get<1>(layout_list[in]); + TensorShape in_tensor_shape = std::get<2>(layout_list[in]); + DeviceArrangement out_device_arrangement = std::get<0>(layout_list[out]); + TensorMap out_tensor_map = std::get<1>(layout_list[out]); + TensorShape out_tensor_shape = std::get<2>(layout_list[out]); + if (in_tensor_shape != out_tensor_shape) { + continue; + } + + ValidRedistributionLayoutCheck(in_device_arrangement, in_tensor_map, in_tensor_shape, out_device_arrangement, + out_tensor_map); + } + if (in % 200 == 0) { + MS_LOG(INFO) << "sample:" << in << " of " << layout_list.size(); + } + } + return; +} + +TEST_F(TestRedistributionLayoutTransfer, RedistributionLayoutTransferCheckAll) { + int32_t device_pow_size_max = 4; + int32_t tensor_pow_size_max = 4; + int32_t device_pow_size_min = 1; + int32_t tensor_pow_size_min = 1; + const int32_t max_device_dim = 5; + const int32_t max_shape_dim = 5; + int32_t device_pow_size = device_pow_size_min; + while (device_pow_size <= device_pow_size_max) { + int32_t tensor_pow_size = tensor_pow_size_min; + while (tensor_pow_size <= tensor_pow_size_max) { + ValidRedistributionLayoutCheckAll(device_pow_size, tensor_pow_size, max_device_dim, max_shape_dim); + tensor_pow_size++; + } + device_pow_size++; + } +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/redistribution_operator_infer_test.cc b/tests/ut/cpp/parallel/tensor_layout/redistribution_operator_infer_test.cc new file mode 100644 index 00000000000..5ffd9b22ce9 --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/redistribution_operator_infer_test.cc @@ -0,0 +1,111 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/tensor_layout/redistribution_operator_infer.h" +#include "parallel/device_manager.h" +#include "util_layout_gen_test.h" + +namespace mindspore { +namespace parallel { + +class TestRedistributionOperatorInfer : public UT::Common { + public: + TestRedistributionOperatorInfer() {} + + void SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 1050; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(1024); + stage_map.push_back(26); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + } + + virtual void TearDown() {} +}; + +// check if in_tensor_map could be changed to out_tensor_map with operator_list +void InferOperatorCheck(Shape in_tensor_map, const Shape& out_tensor_map, const OperatorList& operator_list) { + for (auto op_cost : operator_list) { + OperatorR op = op_cost.first; + Args args = op.second; + ASSERT_GT(args.size(), 2); + std::string str = op.first; + if (str == SPLIT_BY_AXIS) { + in_tensor_map[args[1]] = out_tensor_map[args[1]]; + } else if (str == PERMUTE_BY_AXIS) { + in_tensor_map[args[1]] = in_tensor_map[args[2]]; + in_tensor_map[args[2]] = -1; + } else { + in_tensor_map[args[0]] = -1; + } + } + ASSERT_EQ(in_tensor_map, out_tensor_map); +} + +// generate all the valid tensor_map with the length of dim +Status InferOperatorCheckAll(uint32_t dim_len) { + Shapes tensor_map_list; + Shape dev_mat; + for (uint32_t i = 0; i < dim_len; i++) dev_mat.push_back(2); + Shape tensor_shape = dev_mat; + GenerateValidTensorMap(dev_mat, tensor_shape, &tensor_map_list); + RankList dev_list; + for (int32_t i = 0; i < static_cast(pow(2, dim_len)); i++) dev_list.push_back(i); + Arrangement in_dev_mat; + in_dev_mat.Init(dev_mat); + Arrangement in_tensor_shape; + in_tensor_shape.Init(tensor_shape); + for (auto iter1 = tensor_map_list.begin(); iter1 < tensor_map_list.end(); iter1++) { + for (auto iter2 = tensor_map_list.begin(); iter2 < tensor_map_list.end(); iter2++) { + if (iter1 == iter2) continue; + TensorLayout layout; + Map in_tensor_map; + in_tensor_map.Init(*iter1); + layout.Init(in_dev_mat, in_tensor_map, in_tensor_shape); + RedistributionOperatorInfer operatorInfer; + Map out_tensor_map; + out_tensor_map.Init(*iter2); + if (operatorInfer.Init(layout, out_tensor_map, dev_list) == Status::FAILED) { + return Status::FAILED; + } + operatorInfer.InferRedistributionOperator(); + OperatorList operator_list = operatorInfer.operator_list(); + InferOperatorCheck(*iter1, *iter2, operator_list); + } + } + return Status::SUCCESS; +} + +TEST_F(TestRedistributionOperatorInfer, TestInferOperatorAll) { + uint32_t dim_len = 3; + Status status = InferOperatorCheckAll(dim_len); + ASSERT_EQ(status, Status::SUCCESS); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/reshape_layout_transfer_test.cc b/tests/ut/cpp/parallel/tensor_layout/reshape_layout_transfer_test.cc new file mode 100644 index 00000000000..36b89684f63 --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/reshape_layout_transfer_test.cc @@ -0,0 +1,325 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/tensor_layout/tensor_layout.h" +#include "parallel/tensor_layout/reshape_layout_transfer.h" +#include "util_layout_gen_test.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { + +using DeviceArrangement = std::vector; +using TensorMap = std::vector; +using TensorShape = std::vector; + +class TestReshapeLayoutTransfer : public UT::Common { + public: + TestReshapeLayoutTransfer() {} + + void SetUp() { UT::InitPythonPath(); } + + virtual void TearDown() {} +}; + +void InferUnifiedLayout(const DeviceArrangement& device_arrangement_shape, const TensorMap& in_tensor_map_shape, + const TensorShape& in_tensor_shape_shape, const TensorMap& out_tensor_map_shape, + const TensorShape& out_tensor_shape_shape, DeviceArrangement* unified_device_arrangement_shape, + TensorMap* unified_in_tensor_map_shape, TensorMap* unified_out_tensor_map_shape, + TensorMap* unified_tensor_shape_shape) { + Arrangement device_arrangement; + Status status = device_arrangement.Init(device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map in_tensor_map; + status = in_tensor_map.Init(in_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement in_tensor_shape; + status = in_tensor_shape.Init(in_tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout in_tensor_layout; + status = in_tensor_layout.Init(device_arrangement, in_tensor_map, in_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map out_tensor_map; + status = out_tensor_map.Init(out_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement out_tensor_shape; + status = out_tensor_shape.Init(out_tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout out_tensor_layout; + status = out_tensor_layout.Init(device_arrangement, out_tensor_map, out_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + ReshapeLayoutTransfer tensor_redistribution; + status = tensor_redistribution.Init(in_tensor_layout, out_tensor_layout); + ASSERT_EQ(Status::SUCCESS, status); + std::shared_ptr unified_ptr = tensor_redistribution.UnifyDeviceArrangementAndTensorShape(); + ASSERT_NE(nullptr, unified_ptr); + TensorLayout unified_in = unified_ptr->from_in(); + TensorLayout unified_out = unified_ptr->to_in(); + // get unified_in result + Arrangement unified_in_device_arrangement = unified_in.device_arrangement(); + Arrangement unified_in_tensor_shape = unified_in.tensor_shape(); + Map unified_in_tensor_map = unified_in.tensor_map(); + // get unified_out result + Arrangement unified_out_device_arrangement = unified_out.device_arrangement(); + Arrangement unified_out_tensor_shape = unified_out.tensor_shape(); + Map unified_out_tensor_map = unified_out.tensor_map(); + // checkout valid + ASSERT_EQ(unified_in_device_arrangement.array(), unified_out_device_arrangement.array()); + ASSERT_EQ(unified_in_tensor_shape.array(), unified_out_tensor_shape.array()); + *unified_device_arrangement_shape = unified_in_device_arrangement.array(); + *unified_tensor_shape_shape = unified_in_tensor_shape.array(); + *unified_in_tensor_map_shape = unified_in_tensor_map.array(); + *unified_out_tensor_map_shape = unified_out_tensor_map.array(); +} + +void InferUnifiedLayoutCheck(const DeviceArrangement& device_arrangement, const TensorMap& in_tensor_map, + const TensorShape& in_tensor_shape, const TensorMap& out_tensor_map, + const TensorShape& out_tensor_shape, + const DeviceArrangement& unified_device_arrangement_expect, + const TensorMap& unified_in_tensor_map_expect, + const TensorMap& unified_out_tensor_map_expect, + const TensorMap& unified_tensor_shape_expect) { + DeviceArrangement unified_device_arrangement; + TensorMap unified_in_tensor_map; + TensorMap unified_out_tensor_map; + TensorShape unified_tensor_shape; + InferUnifiedLayout(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape, + &unified_device_arrangement, &unified_in_tensor_map, &unified_out_tensor_map, + &unified_tensor_shape); + // check unified_in result + ASSERT_EQ(unified_device_arrangement_expect, unified_device_arrangement); + ASSERT_EQ(unified_in_tensor_map_expect, unified_in_tensor_map); + ASSERT_EQ(unified_tensor_shape_expect, unified_tensor_shape); + // check unified_out result + ASSERT_EQ(unified_device_arrangement_expect, unified_device_arrangement); + ASSERT_EQ(unified_out_tensor_map_expect, unified_out_tensor_map); + ASSERT_EQ(unified_tensor_shape_expect, unified_tensor_shape); +} + +void ValidUnifiedLayoutCheck(const DeviceArrangement& device_arrangement, const TensorMap& in_tensor_map, + const TensorShape& in_tensor_shape, const TensorMap& out_tensor_map, + const TensorShape& out_tensor_shape) { + DeviceArrangement unified_device_arrangement; + TensorMap unified_in_tensor_map; + TensorMap unified_out_tensor_map; + TensorShape unified_tensor_shape; + InferUnifiedLayout(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape, + &unified_device_arrangement, &unified_in_tensor_map, &unified_out_tensor_map, + &unified_tensor_shape); + // check unified_in result + ValidLayoutChangeCheck(device_arrangement, in_tensor_map, in_tensor_shape, unified_device_arrangement, + unified_in_tensor_map, unified_tensor_shape); + // check unified_out result + ValidLayoutChangeCheck(device_arrangement, out_tensor_map, out_tensor_shape, unified_device_arrangement, + unified_out_tensor_map, unified_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, InferUnifiedLayout1) { + DeviceArrangement device_arrangement = {8, 4}; + TensorMap in_tensor_map = {1, 0}; + TensorShape in_tensor_shape = {512, 1024}; + TensorMap out_tensor_map = {1, 0}; + TensorShape out_tensor_shape = {128, 4096}; + DeviceArrangement unified_device_arrangement_expect = {8, 4}; + TensorMap unified_in_tensor_map_expect = {1, -1, 0}; + TensorMap unified_out_tensor_map_expect = {1, 0, -1}; + TensorShape unified_tensor_shape_expect = {128, 4, 1024}; + InferUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +TEST_F(TestReshapeLayoutTransfer, InferUnifiedLayout2) { + DeviceArrangement device_arrangement = {8, 4}; + TensorMap in_tensor_map = {1, 0}; + TensorShape in_tensor_shape = {512, 1024}; + TensorMap out_tensor_map = {0, 1}; + TensorShape out_tensor_shape = {128, 4096}; + DeviceArrangement unified_device_arrangement_expect = {4, 2, 4}; + TensorMap unified_in_tensor_map_expect = {2, 1, -1, 0}; + TensorMap unified_out_tensor_map_expect = {0, -1, 2, 1}; + TensorShape unified_tensor_shape_expect = {4, 32, 4, 1024}; + InferUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape, + unified_device_arrangement_expect, unified_in_tensor_map_expect, + unified_out_tensor_map_expect, unified_tensor_shape_expect); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck1) { + DeviceArrangement device_arrangement = {8, 4}; + TensorMap in_tensor_map = {1, 0}; + TensorShape in_tensor_shape = {512, 1024}; + TensorMap out_tensor_map = {0, 1}; + TensorShape out_tensor_shape = {128, 4096}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck2) { + DeviceArrangement device_arrangement = {8, 4}; + TensorMap in_tensor_map = {1, -1}; + TensorShape in_tensor_shape = {512, 1024}; + TensorMap out_tensor_map = {0, -1}; + TensorShape out_tensor_shape = {128, 4096}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck3) { + DeviceArrangement device_arrangement = {2}; + TensorMap in_tensor_map = {-1, -1}; + TensorShape in_tensor_shape = {4, 2}; + TensorMap out_tensor_map = {-1, -1}; + TensorShape out_tensor_shape = {2, 4}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck4) { + DeviceArrangement device_arrangement = {4, 8}; + TensorMap in_tensor_map = {-1, -1, -1}; + TensorShape in_tensor_shape = {8, 2, 2}; + TensorMap out_tensor_map = {-1, -1, 0}; + TensorShape out_tensor_shape = {2, 2, 8}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck5) { + DeviceArrangement device_arrangement = {2, 2}; + TensorMap in_tensor_map = {0, -1}; + TensorShape in_tensor_shape = {2, 2}; + TensorMap out_tensor_map = {0, -1}; + TensorShape out_tensor_shape = {2, 2}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck6) { + DeviceArrangement device_arrangement = {2, 2}; + TensorMap in_tensor_map = {0, 1}; + TensorShape in_tensor_shape = {2, 8}; + TensorMap out_tensor_map = {0, 1}; + TensorShape out_tensor_shape = {2, 8}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck7) { + DeviceArrangement device_arrangement = {2, 4, 4}; + TensorMap in_tensor_map = {-1, 0, -1}; + TensorShape in_tensor_shape = {2, 4, 4}; + TensorMap out_tensor_map = {-1, -1, 0}; + TensorShape out_tensor_shape = {2, 2, 8}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck8) { + DeviceArrangement device_arrangement = {4, 2, 4}; + TensorMap in_tensor_map = {2, 0, 1}; + TensorShape in_tensor_shape = {4, 4, 2}; + TensorMap out_tensor_map = {-1, -1, -1}; + TensorShape out_tensor_shape = {2, 4, 4}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +// this reshape layout can not be generated by ValidInferUnifiedLayoutCheckAll +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck9) { + DeviceArrangement device_arrangement = {2, 2, 2}; + TensorMap in_tensor_map = {2, 1, 0, -1}; + TensorShape in_tensor_shape = {2, 2, 128, 1024}; + TensorMap out_tensor_map = {2, 1}; + TensorShape out_tensor_shape = {512, 1024}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck10) { + DeviceArrangement device_arrangement = {2, 2, 2}; + TensorMap in_tensor_map = {2, 1, 0, -1}; + TensorShape in_tensor_shape = {2, 2, 2, 2}; + TensorMap out_tensor_map = {2, 1}; + TensorShape out_tensor_shape = {8, 2}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheck11) { + DeviceArrangement device_arrangement = {2}; + TensorMap in_tensor_map = {0, -1}; + TensorShape in_tensor_shape = {2, 2}; + TensorMap out_tensor_map = {0}; + TensorShape out_tensor_shape = {4}; + ValidUnifiedLayoutCheck(device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); +} + +void ValidInferUnifiedLayoutCheckAll(const int32_t& device_pow_size, const int32_t& tensor_pow_size, + const int32_t& max_device_dim, const int32_t& max_shape_dim) { + std::vector> layout_list; + GenerateValidLayoutByDeviceSizeAndTensorSize(device_pow_size, tensor_pow_size, max_device_dim, max_shape_dim, + &layout_list); + for (uint32_t in = 0; in < layout_list.size(); in++) { + for (uint32_t out = 0; out < layout_list.size(); out++) { + DeviceArrangement in_device_arrangement = std::get<0>(layout_list[in]); + TensorMap in_tensor_map = std::get<1>(layout_list[in]); + TensorShape in_tensor_shape = std::get<2>(layout_list[in]); + DeviceArrangement out_device_arrangement = std::get<0>(layout_list[out]); + TensorMap out_tensor_map = std::get<1>(layout_list[out]); + TensorShape out_tensor_shape = std::get<2>(layout_list[out]); + if (in_device_arrangement != out_device_arrangement) { + continue; + } + + ValidUnifiedLayoutCheck(in_device_arrangement, in_tensor_map, in_tensor_shape, out_tensor_map, out_tensor_shape); + } + if (in % 200 == 0) { + MS_LOG(INFO) << "sample:" << in << " of " << layout_list.size(); + } + } + return; +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheckAll) { + int32_t device_pow_size_max = 4; + int32_t tensor_pow_size_max = 4; + int32_t device_pow_size_min = 1; + int32_t tensor_pow_size_min = 1; + const int32_t max_device_dim = 5; + const int32_t max_shape_dim = 5; + int32_t device_pow_size = device_pow_size_min; + while (device_pow_size <= device_pow_size_max) { + int32_t tensor_pow_size = tensor_pow_size_min; + while (tensor_pow_size <= tensor_pow_size_max) { + ValidInferUnifiedLayoutCheckAll(device_pow_size, tensor_pow_size, max_device_dim, max_shape_dim); + tensor_pow_size++; + } + device_pow_size++; + } +} + +TEST_F(TestReshapeLayoutTransfer, ValidInferUnifiedLayoutCheckAll2) { + int32_t device_pow_size_max = 1; + int32_t tensor_pow_size_max = 2; + int32_t device_pow_size_min = 1; + int32_t tensor_pow_size_min = 2; + const int32_t max_device_dim = 5; + const int32_t max_shape_dim = 5; + int32_t device_pow_size = device_pow_size_min; + while (device_pow_size <= device_pow_size_max) { + int32_t tensor_pow_size = tensor_pow_size_min; + while (tensor_pow_size <= tensor_pow_size_max) { + ValidInferUnifiedLayoutCheckAll(device_pow_size, tensor_pow_size, max_device_dim, max_shape_dim); + tensor_pow_size++; + } + device_pow_size++; + } +} +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/shape_util_test.cc b/tests/ut/cpp/parallel/tensor_layout/shape_util_test.cc new file mode 100644 index 00000000000..b5e2ea3e5bd --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/shape_util_test.cc @@ -0,0 +1,212 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/tensor_layout/shape_util.h" + +namespace mindspore { +namespace parallel { + +/* + * shape = [2, 8, 32] + * shape_accum = [2, 2 * 8, 2 * 8 * 32] + */ +TEST(ShapeUtilTest, ShapeToAccumulateProduct) { + std::vector shape = {2, 8, 32}; + std::vector shape_accum; + Status status = ShapeToAccumulateProduct(shape, &shape_accum); + ASSERT_EQ(Status::SUCCESS, status); + std::vector shape_accum_expect = {2, 2 * 8, 2 * 8 * 32}; + ASSERT_EQ(shape_accum_expect, shape_accum); +} + +/* + * shape = [2, 8, 32] + * shape_accum = [2 * 8 * 32, 8 * 32, 32] + */ +TEST(ShapeUtilTest, ShapeToAccumulateProductReverse) { + std::vector shape = {2, 8, 32}; + std::vector shape_accum; + Status status = ShapeToAccumulateProductReverse(shape, &shape_accum); + ASSERT_EQ(Status::SUCCESS, status); + std::vector shape_accum_expect = {2 * 8 * 32, 8 * 32, 32}; + ASSERT_EQ(shape_accum_expect, shape_accum); +} + +/* + * shape_accum = [2, 2 * 8, 2 * 8 * 32] + * shape = [2, 8, 32] + */ +TEST(ShapeUtilTest, AccumulateProductToShape) { + std::vector shape_accum = {2, 2 * 8, 2 * 8 * 32}; + std::vector shape; + Status status = AccumulateProductToShape(shape_accum, &shape); + ASSERT_EQ(Status::SUCCESS, status); + std::vector shape_expect = {2, 8, 32}; + ASSERT_EQ(shape_expect, shape); +} + +/* + * shape_accum = [2 * 8 * 32, 8 * 32, 32] + * shape = [2, 8, 32] + */ +TEST(ShapeUtilTest, AccumulateProductReverseToShape) { + std::vector shape_accum = {2 * 8 * 32, 8 * 32, 32}; + std::vector shape; + Status status = AccumulateProductReverseToShape(shape_accum, &shape); + ASSERT_EQ(Status::SUCCESS, status); + std::vector shape_expect = {2, 8, 32}; + ASSERT_EQ(shape_expect, shape); +} + +/* + * shape_accum1 = [2, 8] + * shape_accum2 = [4, 8] + * out = [2, 4, 8] + */ +TEST(ShapeUtilTest, UnifyAccumulateProduct) { + std::vector shape_accum1 = {2, 8}; + std::vector shape_accum2 = {4, 8}; + std::vector out; + Status status = UnifyAccumulateProduct(shape_accum1, shape_accum2, &out); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_expect = {2, 4, 8}; + ASSERT_EQ(out_expect, out); +} + +/* + * in1 = [2, 4] + * in2 = [4, 2] + * out = [2, 2, 2] + */ +TEST(ShapeUtilTest, UnifyShape1) { + std::vector in1 = {2, 4}; + std::vector in2 = {4, 2}; + std::vector out; + Status status = UnifyShape(in1, in2, &out); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_expect = {2, 2, 2}; + ASSERT_EQ(out_expect, out); +} + +/* + * in1 = [8, 4] + * in2 = [2, 16] + * out = [2, 4, 4] + */ +TEST(ShapeUtilTest, UnifyShape2) { + std::vector in1 = {8, 4}; + std::vector in2 = {2, 16}; + std::vector out; + Status status = UnifyShape(in1, in2, &out); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_expect = {2, 4, 4}; + ASSERT_EQ(out_expect, out); +} + +/* + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_pos = [2 * 8 * 32, 8 * 32, 32] + * out_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + */ +TEST(ShapeUtilTest, ExpandAccumulateProduct1) { + std::vector in_accum_reverse = {2 * 8 * 32, 8 * 32, 32}; + std::vector expand_pos = {2 * 8 * 32, 8 * 32, 32}; + std::vector out_accum_reverse; + Status status = ExpandAccumulateProduct(in_accum_reverse, expand_pos, &out_accum_reverse); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_accum_reverse_expect = {2 * 8 * 32, 8 * 32, 32}; + ASSERT_EQ(out_accum_reverse_expect, out_accum_reverse); +} + +/* + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_pos = [2 * 8 * 32, 8 * 32, 32, 8] + * out_accum_reverse = [2 * 8 * 32, 8 * 32, 32, 8] + */ +TEST(ShapeUtilTest, ExpandAccumulateProduct2) { + std::vector in_accum_reverse = {2 * 8 * 32, 8 * 32, 32}; + std::vector expand_pos = {2 * 8 * 32, 8 * 32, 32, 8}; + std::vector out_accum_reverse; + Status status = ExpandAccumulateProduct(in_accum_reverse, expand_pos, &out_accum_reverse); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_accum_reverse_expect = {2 * 8 * 32, 8 * 32, 32, 8}; + ASSERT_EQ(out_accum_reverse_expect, out_accum_reverse); +} + +/* + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_pos = [2 * 8 * 32, 32, 8] + * out_accum_reverse = [2 * 8 * 32, 8 * 32, 32, 8] + */ +TEST(ShapeUtilTest, ExpandAccumulateProduct3) { + std::vector in_accum_reverse = {2 * 8 * 32, 8 * 32, 32}; + std::vector expand_pos = {2 * 8 * 32, 32, 8}; + std::vector out_accum_reverse; + Status status = ExpandAccumulateProduct(in_accum_reverse, expand_pos, &out_accum_reverse); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_accum_reverse_expect = {2 * 8 * 32, 8 * 32, 32, 8}; + ASSERT_EQ(out_accum_reverse_expect, out_accum_reverse); +} + +/* + * in_accum_reverse = [2 * 8 * 32, 8 * 32, 32] + * expand_pos = [2 * 8 * 32, 8 * 32, 32, 8] + * out_accum_reverse = [2 * 8 * 32, 8 * 32, 32, 8] + */ +TEST(ShapeUtilTest, ExpandAccumulateProduct4) { + std::vector in_accum_reverse = {512 * 1024, 1024}; + std::vector expand_pos = {128 * 4096, 4096}; + std::vector out_accum_reverse; + Status status = ExpandAccumulateProduct(in_accum_reverse, expand_pos, &out_accum_reverse); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_accum_reverse_expect = {128 * 4 * 1024, 4 * 1024, 1024}; + ASSERT_EQ(out_accum_reverse_expect, out_accum_reverse); +} + +/* + * in = [2, 8, 32] + * expand = [16, 4, 8] + * out = [2, 8, 4, 8] + */ +TEST(ShapeUtilTest, ExpandShape1) { + std::vector in = {2, 8, 32}; + std::vector expand = {16, 4, 8}; + std::vector out; + Status status = ExpandShape(in, expand, &out); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_expect = {2, 8, 4, 8}; + ASSERT_EQ(out_expect, out); +} + +/* + * in = [2, 8, 32] + * expand = [16, 4, 8] + * out = [2, 8, 4, 8] + */ +TEST(ShapeUtilTest, ExpandShape2) { + std::vector in = {2, 8, 32}; + std::vector expand = {2, 4, 8}; + std::vector out; + Status status = ExpandShape(in, expand, &out); + ASSERT_EQ(Status::SUCCESS, status); + std::vector out_expect = {2, 4, 2, 4, 8}; + ASSERT_EQ(out_expect, out); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/tensor_layout_test.cc b/tests/ut/cpp/parallel/tensor_layout/tensor_layout_test.cc new file mode 100644 index 00000000000..bae05d650a1 --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/tensor_layout_test.cc @@ -0,0 +1,324 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/tensor_layout/tensor_layout.h" + +namespace mindspore { +namespace parallel { + +class TestTensorLayout : public UT::Common { + public: + TestTensorLayout() {} + + void SetUp() { UT::InitPythonPath(); } + + virtual void TearDown() {} +}; + +void ReshapeExpandDeviceArrangementTestFunction(const std::vector& in_device_arrangement_shape, + const std::vector& in_tensor_map_shape, + const std::vector& in_tensor_shape_shape, + const std::vector& out_device_arrangement_shape, + const std::vector& out_tensor_map_shape, + const std::vector& out_tensor_shape_shape) { + Arrangement device_arrangement; + Status status = device_arrangement.Init(in_device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map tensor_map; + status = tensor_map.Init(in_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement tensor_shape; + status = tensor_shape.Init(in_tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement device_arrangement_new; + status = device_arrangement_new.Init(out_device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout tensor_layout; + tensor_layout.Init(device_arrangement, tensor_map, tensor_shape); + std::shared_ptr tensor_layout_new_ptr = tensor_layout.ExpandDeviceArrangement(device_arrangement_new); + ASSERT_NE(nullptr, tensor_layout_new_ptr); + Map tensor_map_new = tensor_layout_new_ptr->tensor_map(); + ASSERT_EQ(out_tensor_map_shape, tensor_map_new.array()); + Arrangement tensor_shape_new = tensor_layout_new_ptr->tensor_shape(); + ASSERT_EQ(out_tensor_shape_shape, tensor_shape_new.array()); +} + +/* + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 2, 2] + * => + * out_tensor_map = [3, 2, 1, 0], + * out_tensor_shape = [4, 128, 2, 512] + * + */ +TEST_F(TestTensorLayout, ReshapeExpandDeviceArrangement1) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {1, 0}; + std::vector tensor_shape = {512, 1024}; + std::vector device_arrangement_new = {4, 2, 2, 2}; + std::vector tensor_map_expect = {3, 2, 1, 0}; + std::vector tensor_shape_expect = {4, 128, 2, 512}; + ReshapeExpandDeviceArrangementTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_new, + tensor_map_expect, tensor_shape_expect); +} + +/* + * example2: + * in_device_arrangement = [8, 4], + * in_tensor_map = [0, 1], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 2, 2] + * => + * out_tensor_map = [1, 0, 3, 2], + * out_tensor_shape = [2, 256, 4, 256] + */ +TEST_F(TestTensorLayout, ReshapeExpandDeviceArrangement2) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {0, 1}; + std::vector tensor_shape = {512, 1024}; + std::vector device_arrangement_new = {4, 2, 2, 2}; + std::vector tensor_map_expect = {1, 0, 3, 2}; + std::vector tensor_shape_expect = {2, 256, 4, 256}; + ReshapeExpandDeviceArrangementTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_new, + tensor_map_expect, tensor_shape_expect); +} + +/* + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, -1], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 2, 2] + * => + * out_tensor_map = [3, 2, -1], + * out_tensor_shape = [4, 128, 1024] + */ +TEST_F(TestTensorLayout, ReshapeExpandDeviceArrangement3) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {1, -1}; + std::vector tensor_shape = {512, 1024}; + std::vector device_arrangement_new = {4, 2, 2, 2}; + std::vector tensor_map_expect = {3, 2, -1}; + std::vector tensor_shape_expect = {4, 128, 1024}; + ReshapeExpandDeviceArrangementTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_new, + tensor_map_expect, tensor_shape_expect); +} + +/* + * example4: + * in_device_arrangement = [8, 4], + * in_tensor_map = [0, 1], + * in_tensor_shape = [512, 1024], + * out_device_arrangement = [4, 2, 4] + * => + * out_tensor_map = [0, 2, 1], + * out_tensor_shape = [512, 4, 256] + */ +TEST_F(TestTensorLayout, ReshapeExpandDeviceArrangement4) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {0, 1}; + std::vector tensor_shape = {512, 1024}; + std::vector device_arrangement_new = {4, 2, 4}; + std::vector tensor_map_expect = {0, 2, 1}; + std::vector tensor_shape_expect = {512, 4, 256}; + ReshapeExpandDeviceArrangementTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_new, + tensor_map_expect, tensor_shape_expect); +} + +TEST_F(TestTensorLayout, ReshapeExpandDeviceArrangement5) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {1, -1, 0}; + std::vector tensor_shape = {128, 4, 1024}; + std::vector device_arrangement_new = {8, 4}; + std::vector tensor_map_expect = {1, -1, 0}; + std::vector tensor_shape_expect = {128, 4, 1024}; + ReshapeExpandDeviceArrangementTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_new, + tensor_map_expect, tensor_shape_expect); +} + +void ExpandTensorShapeTestFunction(const std::vector& in_device_arrangement_shape, + const std::vector& in_tensor_map_shape, + const std::vector& in_tensor_shape_shape, + const std::vector& out_device_arrangement_shape, + const std::vector& out_tensor_map_shape, + const std::vector& out_tensor_shape_shape) { + Arrangement device_arrangement; + Status status = device_arrangement.Init(in_device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map tensor_map; + status = tensor_map.Init(in_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement tensor_shape; + status = tensor_shape.Init(in_tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement tensor_shape_new; + status = tensor_shape_new.Init(out_tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout tensor_layout; + tensor_layout.Init(device_arrangement, tensor_map, tensor_shape); + std::shared_ptr tensor_layout_new_ptr = tensor_layout.ExpandTensorShape(tensor_shape_new); + ASSERT_NE(nullptr, tensor_layout_new_ptr); + Arrangement device_arrangement_new = tensor_layout_new_ptr->device_arrangement(); + ASSERT_EQ(Status::SUCCESS, status); + ASSERT_EQ(out_device_arrangement_shape, device_arrangement_new.array()); + Map tensor_map_new = tensor_layout_new_ptr->tensor_map(); + ASSERT_EQ(out_tensor_map_shape, tensor_map_new.array()); +} + +/* + * example: + * in_device_arrangement = [8, 4], + * in_tensor_map = [1, 0], + * in_tensor_shape = [512, 1024], + * out_tensor_shape = [4, 128, 1024] + * => + * out_device_arrangement = [4, 2, 4] + * out_tensor_map = [2, 1, 0], + */ +TEST_F(TestTensorLayout, ExpandTensorShape1) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {1, 0}; + std::vector tensor_shape = {512, 1024}; + std::vector device_arrangement_expect = {4, 2, 4}; + std::vector tensor_map_expect = {2, 1, 0}; + std::vector tensor_shape_new = {4, 128, 1024}; + ExpandTensorShapeTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_expect, + tensor_map_expect, tensor_shape_new); +} + +TEST_F(TestTensorLayout, ExpandTensorShape2) { + std::vector device_arrangement = {8, 4}; + std::vector tensor_map = {1, 0}; + std::vector tensor_shape = {128, 4096}; + std::vector device_arrangement_expect = {8, 4}; + std::vector tensor_map_expect = {1, 0, -1}; + std::vector tensor_shape_new = {128, 4, 1024}; + ExpandTensorShapeTestFunction(device_arrangement, tensor_map, tensor_shape, device_arrangement_expect, + tensor_map_expect, tensor_shape_new); +} + +TEST_F(TestTensorLayout, GetSliceShape) { + std::vector in_device_arrangement = {8, 4}; + std::vector in_tensor_map = {1, -1}; + std::vector in_tensor_shape = {512, 1024}; + Arrangement device_arrangement; + device_arrangement.Init(in_device_arrangement); + Map tensor_map; + tensor_map.Init(in_tensor_map); + Arrangement tensor_shape; + tensor_shape.Init(in_tensor_shape); + TensorLayout tensor_layout; + tensor_layout.Init(device_arrangement, tensor_map, tensor_shape); + Arrangement new_tensor_shape = tensor_layout.slice_shape(); + Arrangement expected_shape; + expected_shape.Init({64, 1024}); + ASSERT_EQ(new_tensor_shape, expected_shape); +} + +TEST_F(TestTensorLayout, UpdateTensorMap) { + std::vector in_device_arrangement = {8, 4}; + std::vector in_tensor_map = {1, -1}; + std::vector in_tensor_shape = {512, 1024}; + Arrangement device_arrangement; + device_arrangement.Init(in_device_arrangement); + Map tensor_map; + tensor_map.Init(in_tensor_map); + Arrangement tensor_shape; + tensor_shape.Init(in_tensor_shape); + TensorLayout tensor_layout; + tensor_layout.Init(device_arrangement, tensor_map, tensor_shape); + tensor_layout.UpdateTensorMap(1, 0); + in_tensor_map[1] = 0; + Shape new_tensor_map = tensor_layout.tensor_map().array(); + ASSERT_EQ(in_tensor_map, new_tensor_map); +} + +void RemoveElementEqualToOneInDeviceArrangementTestFunction(const std::vector& in_device_arrangement_shape, + const std::vector& in_tensor_map_shape, + const std::vector& in_tensor_shape_shape, + const std::vector& out_device_arrangement_shape, + const std::vector& out_tensor_map_shape, + const std::vector& out_tensor_shape_shape) { + Arrangement device_arrangement; + Status status = device_arrangement.Init(in_device_arrangement_shape); + ASSERT_EQ(Status::SUCCESS, status); + Map tensor_map; + status = tensor_map.Init(in_tensor_map_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement tensor_shape; + status = tensor_shape.Init(in_tensor_shape_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout tensor_layout; + status = tensor_layout.Init(device_arrangement, tensor_map, tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement tensor_shape_new = tensor_layout.tensor_shape(); + ASSERT_EQ(out_tensor_shape_shape, tensor_shape_new.array()); + Arrangement device_arrangement_new = tensor_layout.device_arrangement(); + ASSERT_EQ(out_device_arrangement_shape, device_arrangement_new.array()); + Map tensor_map_new = tensor_layout.tensor_map(); + ASSERT_EQ(out_tensor_map_shape, tensor_map_new.array()); +} + +TEST_F(TestTensorLayout, RemoveElementEqualToOneInDeviceArrangement1) { + std::vector device_arrangement = {2, 2, 1}; + std::vector tensor_map = {2, 1}; + std::vector tensor_shape = {128, 4096}; + std::vector device_arrangement_expect = {2, 2}; + std::vector tensor_map_expect = {1, 0}; + std::vector tensor_shape_new = {128, 4096}; + RemoveElementEqualToOneInDeviceArrangementTestFunction( + device_arrangement, tensor_map, tensor_shape, device_arrangement_expect, tensor_map_expect, tensor_shape_new); +} + +TEST_F(TestTensorLayout, RemoveElementEqualToOneInDeviceArrangement2) { + std::vector device_arrangement = {16, 1, 1}; + std::vector tensor_map = {2, 0}; + std::vector tensor_shape = {128, 4096}; + std::vector device_arrangement_expect = {16}; + std::vector tensor_map_expect = {0, -1}; + std::vector tensor_shape_new = {128, 4096}; + RemoveElementEqualToOneInDeviceArrangementTestFunction( + device_arrangement, tensor_map, tensor_shape, device_arrangement_expect, tensor_map_expect, tensor_shape_new); +} + +TEST_F(TestTensorLayout, RemoveElementEqualToOneInDeviceArrangement3) { + std::vector device_arrangement = {1, 16, 1}; + std::vector tensor_map = {2, 1}; + std::vector tensor_shape = {128, 4096}; + std::vector device_arrangement_expect = {16}; + std::vector tensor_map_expect = {-1, 0}; + std::vector tensor_shape_new = {128, 4096}; + RemoveElementEqualToOneInDeviceArrangementTestFunction( + device_arrangement, tensor_map, tensor_shape, device_arrangement_expect, tensor_map_expect, tensor_shape_new); +} + +TEST_F(TestTensorLayout, RemoveElementEqualToOneInDeviceArrangement4) { + std::vector device_arrangement = {1, 1, 1}; + std::vector tensor_map = {2, 1}; + std::vector tensor_shape = {128, 4096}; + std::vector device_arrangement_expect = {}; + std::vector tensor_map_expect = {-1, -1}; + std::vector tensor_shape_new = {128, 4096}; + RemoveElementEqualToOneInDeviceArrangementTestFunction( + device_arrangement, tensor_map, tensor_shape, device_arrangement_expect, tensor_map_expect, tensor_shape_new); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/tensor_redistribution_test.cc b/tests/ut/cpp/parallel/tensor_layout/tensor_redistribution_test.cc new file mode 100644 index 00000000000..ad20793417f --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/tensor_redistribution_test.cc @@ -0,0 +1,209 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "parallel/tensor_layout/tensor_redistribution.h" + +namespace mindspore { +namespace parallel { + +class TestTensorRedistribution : public UT::Common { + public: + TestTensorRedistribution() {} + + void SetUp() { + RankList dev_list; + + for (int32_t i = 0; i < 20; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(16); + stage_map.push_back(4); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + } + + virtual void TearDown() {} +}; + +// Redistribution: Reshape -> SplitByAxis -> ConcatByAxis -> SplitByAxis -> Reshape +TEST_F(TestTensorRedistribution, TestInferRedistribution1) { + std::vector device_arrangement = {2, 4, 2}; + std::vector tensor_map = {2, 0}; + std::vector tensor_shape = {512, 1024}; + + Arrangement in_device_arrangement; + Status status = in_device_arrangement.Init(device_arrangement); + ASSERT_EQ(Status::SUCCESS, status); + Map in_tensor_map; + status = in_tensor_map.Init(tensor_map); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement in_tensor_shape; + status = in_tensor_shape.Init(tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout from_layout; + status = from_layout.Init(in_device_arrangement, in_tensor_map, in_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + + device_arrangement = {4, 2, 2}; + tensor_map = {2, 1}; + + Arrangement out_device_arrangement; + status = out_device_arrangement.Init(device_arrangement); + ASSERT_EQ(Status::SUCCESS, status); + Map out_tensor_map; + status = out_tensor_map.Init(tensor_map); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement out_tensor_shape; + status = out_tensor_shape.Init(tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout to_layout; + status = to_layout.Init(out_device_arrangement, out_tensor_map, out_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + + TensorRedistribution tensor_redistribution; + RankList dev_list = g_device_manager->GetDeviceListByStageId(0); + tensor_redistribution.Init(from_layout, to_layout, dev_list); + std::shared_ptr> op_ptr; + op_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); + ASSERT_TRUE(op_ptr != nullptr); + ASSERT_EQ(op_ptr->first.size(), 7); + ASSERT_EQ(op_ptr->second.size(), 7); + + std::vector op_names; + for (auto iter : op_ptr->first) { + op_names.push_back(iter.first); + } + std::vector expected_op_names = {"Reshape", "StridedSlice", "AllGather", "Split", + "Concat", "StridedSlice", "Reshape"}; + ASSERT_EQ(op_names, expected_op_names); +} + +// Redistribution: AlltoAll +TEST_F(TestTensorRedistribution, TestInferRedistribution2) { + std::vector device_arrangement = {16, 1, 1}; + std::vector tensor_map = {2, 0}; + std::vector tensor_shape = {512, 1024}; + + Arrangement in_device_arrangement; + Status status = in_device_arrangement.Init(device_arrangement); + ASSERT_EQ(Status::SUCCESS, status); + Map in_tensor_map; + status = in_tensor_map.Init(tensor_map); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement in_tensor_shape; + status = in_tensor_shape.Init(tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout from_layout; + status = from_layout.Init(in_device_arrangement, in_tensor_map, in_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + + device_arrangement = {1, 16, 1}; + tensor_map = {2, 1}; + + Arrangement out_device_arrangement; + status = out_device_arrangement.Init(device_arrangement); + ASSERT_EQ(Status::SUCCESS, status); + Map out_tensor_map; + status = out_tensor_map.Init(tensor_map); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement out_tensor_shape; + status = out_tensor_shape.Init(tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout to_layout; + status = to_layout.Init(out_device_arrangement, out_tensor_map, out_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + + TensorRedistribution tensor_redistribution; + RankList dev_list = g_device_manager->GetDeviceListByStageId(0); + tensor_redistribution.Init(from_layout, to_layout, dev_list); + std::shared_ptr> op_ptr; + op_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); + ASSERT_TRUE(op_ptr != nullptr); + ASSERT_EQ(op_ptr->first.size(), 2); + ASSERT_EQ(op_ptr->second.size(), 2); + + std::vector op_names; + for (auto iter : op_ptr->first) { + op_names.push_back(iter.first); + } + std::vector expected_op_names = {"AllGather", "StridedSlice"}; + ASSERT_EQ(op_names, expected_op_names); +} + +// Redistribution: Reshape +TEST_F(TestTensorRedistribution, TestInferRedistribution3) { + std::vector device_arrangement = {8}; + std::vector tensor_map = {0, -1, -1, -1}; + std::vector tensor_shape = {128, 64, 1, 1}; + + Arrangement in_device_arrangement; + Status status = in_device_arrangement.Init(device_arrangement); + ASSERT_EQ(Status::SUCCESS, status); + Map in_tensor_map; + status = in_tensor_map.Init(tensor_map); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement in_tensor_shape; + status = in_tensor_shape.Init(tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout from_layout; + status = from_layout.Init(in_device_arrangement, in_tensor_map, in_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + + device_arrangement = {8}; + tensor_map = {0, -1}; + tensor_shape = {128, 64}; + + Arrangement out_device_arrangement; + status = out_device_arrangement.Init(device_arrangement); + ASSERT_EQ(Status::SUCCESS, status); + Map out_tensor_map; + status = out_tensor_map.Init(tensor_map); + ASSERT_EQ(Status::SUCCESS, status); + Arrangement out_tensor_shape; + status = out_tensor_shape.Init(tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + TensorLayout to_layout; + status = to_layout.Init(out_device_arrangement, out_tensor_map, out_tensor_shape); + ASSERT_EQ(Status::SUCCESS, status); + + TensorRedistribution tensor_redistribution; + RankList dev_list = g_device_manager->GetDeviceListByStageId(0); + tensor_redistribution.Init(from_layout, to_layout, dev_list); + std::shared_ptr> op_ptr; + op_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); + ASSERT_TRUE(op_ptr != nullptr); + ASSERT_EQ(op_ptr->first.size(), 1); + ASSERT_EQ(op_ptr->second.size(), 1); + + std::vector op_names; + for (auto iter : op_ptr->first) { + op_names.push_back(iter.first); + } + std::vector expected_op_names = {"Reshape"}; + ASSERT_EQ(op_names, expected_op_names); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/util_layout_gen_test.cc b/tests/ut/cpp/parallel/tensor_layout/util_layout_gen_test.cc new file mode 100644 index 00000000000..07d270c95c8 --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/util_layout_gen_test.cc @@ -0,0 +1,328 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "parallel/tensor_layout/util_layout_gen_test.h" +#include +#include +#include +#include +#include +#include +#include "parallel/tensor_layout/shape_util.h" +#include "common/common_test.h" + +namespace mindspore { +namespace parallel { +std::vector> combine(const std::vector& in, int32_t target) { + std::vector> output; + for (int32_t i = 0; i < pow(2, in.size()); i++) { + int32_t temp = 0; + int32_t count = 0; + std::vector left; + for (int32_t j = 0; j < in.size(); j++) { + if ((i & (1 << j)) != 0) { + left.push_back(j); + count++; + } + } + if (count == target) { + std::vector one_case; + for (int32_t j = 0; j < count; j++) { + temp = in.size() - 1 - left[j]; + one_case.push_back(in[temp]); + } + if (one_case.size() > 0) { + output.push_back(one_case); + } + } + } + return output; +} + +void GenerateValidShapeBySizeAndDim(const int32_t& pow_size, const int32_t& dim, + std::vector>* out) { + out->clear(); + std::vector in; + for (int32_t i = 1; i < pow_size; i++) { + in.push_back(i); + } + std::vector> combine_result; + combine_result = combine(in, dim - 1); + if (combine_result.size() == 0) { + int32_t size = exp2(pow_size); + std::vector item = {size}; + out->push_back(item); + } + for (uint32_t i = 0; i < combine_result.size(); i++) { + std::vector item; + int32_t prev = 0; + for (int32_t j = combine_result[i].size() - 1; j >= 0; j--) { + item.push_back(exp2(combine_result[i][j] - prev)); + prev = combine_result[i][j]; + } + item.push_back(exp2(pow_size - prev)); + out->push_back(item); + } + return; +} + +void GenerateValidShapeBySize(const int32_t& pow_size, std::vector>* out) { + out->clear(); + for (int32_t dim = 1; dim <= pow_size; dim++) { + std::vector> combine_result; + GenerateValidShapeBySizeAndDim(pow_size, dim, &combine_result); + for (uint32_t i = 0; i < combine_result.size(); i++) { + out->push_back(combine_result[i]); + } + } + return; +} + +std::vector GenerateTensorMap(const uint32_t& map_size, const std::vector& pos_index, + const std::vector& pos_value) { + std::vector tensor_map(map_size, -1); + for (uint32_t i = 0; i < pos_index.size() && i < pos_value.size(); i++) { + if (pos_index[i] >= map_size) { + continue; + } + tensor_map[pos_index[i]] = pos_value[i]; + } + return tensor_map; +} + +void GenerateValidTensorMap(const std::vector& device_arrangement, const std::vector& tensor_shape, + std::vector>* tensor_map_list) { + tensor_map_list->clear(); + int32_t device_size = device_arrangement.size(); + int32_t shape_size = tensor_shape.size(); + std::vector pos_ind_combine_in; + for (int32_t i = 0; i < shape_size; i++) { + pos_ind_combine_in.push_back(i); + } + std::vector dev_ind_combine_in; + for (int32_t i = 0; i < device_size; i++) { + dev_ind_combine_in.push_back(i); + } + std::vector none_map(tensor_shape.size(), -1); + tensor_map_list->push_back(none_map); + for (uint32_t pos_num = 1; (pos_num <= shape_size) && (pos_num <= device_size); pos_num++) { + std::vector> pos_index; + pos_index = combine(pos_ind_combine_in, pos_num); + std::vector> dev_index; + dev_index = combine(dev_ind_combine_in, pos_num); + for (int l = 0; l < dev_index.size(); l++) { + std::vector pos_value_combine_in; + for (int32_t i = dev_index[l].size() - 1; i >= 0; i--) { + pos_value_combine_in.push_back(dev_index[l][i]); + } + std::vector> pos_value; + std::vector::iterator it = pos_value_combine_in.begin(); + do { + std::vector pos_value_item; + for (uint32_t m = 0; m < pos_num; m++) { + pos_value_item.push_back(pos_value_combine_in[m]); + } + pos_value.push_back(pos_value_item); + } while (next_permutation(it, it + pos_num)); + for (uint32_t j = 0; j < pos_index.size(); j++) { + for (uint32_t k = 0; k < pos_value.size(); k++) { + std::vector tensor_map = GenerateTensorMap(shape_size, pos_index[j], pos_value[k]); + tensor_map_list->push_back(tensor_map); + } + } + } + } + return; +} + +void GenerateValidLayoutByDeviceSizeAndTensorSize( + const int32_t& device_pow_size, const int32_t& tensor_pow_size, const int32_t& max_device_dim, + const int32_t& max_shape_dim, + std::vector, std::vector, std::vector>>* layout_list) { + layout_list->clear(); + std::vector> device_arrangement_list; + GenerateValidShapeBySize(device_pow_size, &device_arrangement_list); + std::vector> tensor_shape_list; + GenerateValidShapeBySize(tensor_pow_size, &tensor_shape_list); + for (uint32_t device_idx = 0; device_idx < device_arrangement_list.size(); device_idx++) { + for (uint32_t shape_idx = 0; shape_idx < tensor_shape_list.size(); shape_idx++) { + std::vector> tensor_map_list; + GenerateValidTensorMap(device_arrangement_list[device_idx], tensor_shape_list[shape_idx], &tensor_map_list); + for (uint32_t map_idx = 0; map_idx < tensor_map_list.size(); map_idx++) { + if (!CheckLayoutValid(device_arrangement_list[device_idx], tensor_map_list[map_idx], + tensor_shape_list[shape_idx])) { + continue; + } + layout_list->push_back( + std::make_tuple(device_arrangement_list[device_idx], tensor_map_list[map_idx], tensor_shape_list[shape_idx])); + } + } + } + return; +} + +bool CheckLayoutValid(const std::vector& device_arrangement, const std::vector& tensor_map, + const std::vector& tensor_shape) { + bool flag = false; + if ((tensor_map.size() - ComputeNoneNumber(tensor_map)) > device_arrangement.size()) { + return flag; + } + if (!ShapeIsDividedByDevice(device_arrangement, tensor_map, tensor_shape)) { + return flag; + } + return true; +} + +uint32_t ComputeNoneNumber(const std::vector& tensor_map) { + uint32_t num = 0; + for (uint32_t i = 0; i < tensor_map.size(); i++) { + if (tensor_map[i] == -1) { + num++; + } + } + return num; +} + +bool ShapeIsDividedByDevice(const std::vector& device_arrangement, const std::vector& tensor_map, + const std::vector& tensor_shape) { + bool flag = false; + for (uint32_t i = 0; i < tensor_map.size() && i < tensor_shape.size(); i++) { + if (tensor_map[i] == -1) { + continue; + } + int32_t dim = device_arrangement[device_arrangement.size() - 1 - tensor_map[i]]; + if (tensor_shape[i] % dim != 0) { + return flag; + } + } + return true; +} + +bool IsExpended(const std::vector& in1, const std::vector& in2) { + int32_t size = 1; + uint32_t ind = 0; + for (uint32_t i = 0; i < in1.size(); i++) { + size *= in1[i]; + if (ind >= in2.size()) { + return false; + } + if (size > in2[ind]) { + return false; + } else if (size < in2[ind]) { + continue; + } else { + ind++; + size = 1; + } + } + if (ind != in2.size()) { + return false; + } + return true; +} + +void ComputeAccumDeviceTOAccumShapeMap(const std::vector& device_arrangement, + const std::vector& tensor_map, const std::vector& tensor_shape, + std::map* accum_device_to_accum_shape_map) { + accum_device_to_accum_shape_map->clear(); + std::vector shape_accum_reverse; + Status status = ShapeToAccumulateProductReverse(tensor_shape, &shape_accum_reverse); + ASSERT_EQ(Status::SUCCESS, status); + std::vector device_accum_reverse; + status = ShapeToAccumulateProductReverse(device_arrangement, &device_accum_reverse); + ASSERT_EQ(Status::SUCCESS, status); + for (int32_t i = 0; i < device_accum_reverse.size(); i++) { + auto iter = std::find(tensor_map.begin(), tensor_map.end(), device_accum_reverse.size() - 1 - i); + if (iter == tensor_map.end()) { + accum_device_to_accum_shape_map->insert(std::make_pair(device_accum_reverse[i], -1)); + } else { + accum_device_to_accum_shape_map->insert( + std::make_pair(device_accum_reverse[i], shape_accum_reverse[std::distance(tensor_map.begin(), iter)])); + } + } + return; +} + +void IsLinearValue(int32_t small, int32_t big, int32_t small_value, int32_t big_value, int32_t middle, + int32_t middle_value) { + ASSERT_NE(big, small); + int32_t value = (middle - small) * (big_value - small_value) / (big - small) + small_value; + ASSERT_EQ(middle_value, value); +} + +void LayoutTransferValidLayoutChangeCheck(const std::vector& in_device_arrangement, + const std::vector& in_tensor_map, + const std::vector& in_tensor_shape, + const std::vector& out_device_arrangement, + const std::vector& out_tensor_map, + const std::vector& out_tensor_shape) { + bool is_expended = IsExpended(out_device_arrangement, in_device_arrangement); + ASSERT_EQ(true, is_expended); + is_expended = IsExpended(out_tensor_shape, in_tensor_shape); + ASSERT_EQ(true, is_expended); + std::map out_accum_device_to_accum_shape_map; + ComputeAccumDeviceTOAccumShapeMap(out_device_arrangement, out_tensor_map, out_tensor_shape, + &out_accum_device_to_accum_shape_map); + std::map in_accum_device_to_accum_shape_map; + ComputeAccumDeviceTOAccumShapeMap(in_device_arrangement, in_tensor_map, in_tensor_shape, + &in_accum_device_to_accum_shape_map); + std::map::iterator in_iter = in_accum_device_to_accum_shape_map.begin(); + while (in_iter != in_accum_device_to_accum_shape_map.end()) { + if (in_iter->second != out_accum_device_to_accum_shape_map[in_iter->first]) { + continue; + } + in_iter++; + } + std::map::iterator out_iter = out_accum_device_to_accum_shape_map.begin(); + while (out_iter != out_accum_device_to_accum_shape_map.end()) { + if (out_accum_device_to_accum_shape_map.find(out_iter->first) == out_accum_device_to_accum_shape_map.end()) { + in_iter = in_accum_device_to_accum_shape_map.begin(); + int32_t small = 1; + int32_t big = 1; + while (in_iter != in_accum_device_to_accum_shape_map.end()) { + if (in_iter->first < out_iter->first) { + small = in_iter->second; + } else if (in_iter->first > out_iter->first) { + big = in_iter->second; + break; + } else { + ASSERT_EQ(true, false); + } + in_iter++; + } + if (small == 1) { + ASSERT_EQ(true, false); + } + if (big == 1) { + ASSERT_EQ(true, false); + } + int32_t small_value = in_accum_device_to_accum_shape_map[small]; + int32_t big_value = in_accum_device_to_accum_shape_map[big]; + IsLinearValue(small, big, small_value, big_value, out_iter->first, out_iter->second); + } + out_iter++; + } +} + +void ValidLayoutChangeCheck(const std::vector& in_device_arrangement, + const std::vector& in_tensor_map, const std::vector& in_tensor_shape, + const std::vector& out_device_arrangement, + const std::vector& out_tensor_map, const std::vector& out_tensor_shape) { + LayoutTransferValidLayoutChangeCheck(in_device_arrangement, in_tensor_map, in_tensor_shape, out_device_arrangement, + out_tensor_map, out_tensor_shape); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/parallel/tensor_layout/util_layout_gen_test.h b/tests/ut/cpp/parallel/tensor_layout/util_layout_gen_test.h new file mode 100644 index 00000000000..e14556378f4 --- /dev/null +++ b/tests/ut/cpp/parallel/tensor_layout/util_layout_gen_test.h @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_PARALLEL_TENSOR_LAYOUT_UT_UTIL_LAYOUT_GEN_H_ +#define TESTS_UT_PARALLEL_TENSOR_LAYOUT_UT_UTIL_LAYOUT_GEN_H_ + +#include +#include +#include + +#include "parallel/tensor_layout/tensor_layout.h" + +namespace mindspore { +namespace parallel { + +std::vector> combine(const std::vector& in, int32_t target); + +void GenerateValidShapeBySizeAndDim(const int32_t& pow_size, const int32_t& dim, + std::vector>* out); + +void GenerateValidShapeBySize(const int32_t& pow_size, std::vector>* out); + +std::vector GenerateTensorMap(const uint32_t& map_size, const std::vector& pos_index, + const std::vector& pos_value); + +void GenerateValidTensorMap(const std::vector& device_arrangement, const std::vector& tensor_shape, + std::vector>* tensor_map_list); + +void GenerateValidLayoutByDeviceSizeAndTensorSize( + const int32_t& device_pow_size, const int32_t& tensor_pow_size, const int32_t& max_device_dim, + const int32_t& max_shape_dim, + std::vector, std::vector, std::vector>>* layout_list); + +uint32_t ComputeNoneNumber(const std::vector& tensor_map); + +bool ShapeIsDividedByDevice(const std::vector& device_arrangement, const std::vector& tensor_map, + const std::vector& tensor_shape); + +bool CheckLayoutValid(const std::vector& device_arrangement, const std::vector& tensor_map, + const std::vector& tensor_shape); + +void ComputeAccumDeviceTOAccumShapeMap(const std::vector& device_arrangement, + const std::vector& tensor_map, const std::vector& tensor_shape, + std::map* accum_device_to_accum_shape_map); + +void LayoutTransferValidLayoutChangeCheck(const std::vector& in_device_arrangement, + const std::vector& in_tensor_map, + const std::vector& in_tensor_shape, + const std::vector& out_device_arrangement, + const std::vector& out_tensor_map, + const std::vector& out_tensor_shape); + +void ValidLayoutChangeCheck(const std::vector& in_device_arrangement, + const std::vector& in_tensor_map, const std::vector& in_tensor_shape, + const std::vector& out_device_arrangement, + const std::vector& out_tensor_map, const std::vector& out_tensor_shape); + +} // namespace parallel +} // namespace mindspore +#endif // TESTS_UT_PARALLEL_TENSOR_LAYOUT_UT_UTIL_LAYOUT_GEN_H_ diff --git a/tests/ut/cpp/parallel/virtual_dataset_test.cc b/tests/ut/cpp/parallel/virtual_dataset_test.cc new file mode 100644 index 00000000000..602a7370f18 --- /dev/null +++ b/tests/ut/cpp/parallel/virtual_dataset_test.cc @@ -0,0 +1,175 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "common/common_test.h" +#include "parallel/strategy.h" +#include "parallel/ops_info/virtual_dataset_info.h" +#include "parallel/device_manager.h" +#include "parallel/step_parallel.h" + +namespace mindspore { +namespace parallel { + +class VirtualDatasetInfo; +using VirtualDatasetInfoPtr = std::shared_ptr; +VirtualDatasetInfoPtr virtual_dataset; + +class TestVirtualDatasetInfo : public UT::Common { + public: + TestVirtualDatasetInfo() {} + void SetUp(); + void TearDown() {} +}; + +void TestVirtualDatasetInfo::SetUp() { + std::list dev_list; + + for (int32_t i = 0; i < 130; i++) { + dev_list.push_back(i); + } + + std::list stage_map; + stage_map.push_back(16); + stage_map.push_back(114); + + int32_t local_dev = 0; + + // create a new g_device_manager + g_device_manager = std::make_shared(); + g_device_manager->Init(dev_list, local_dev, stage_map, "hccl"); + + std::unordered_map attr; + + Shapes inputs_shape = {{128, 32}, {1280, 320}, {12800, 3200}}; + Shapes outputs_shape = {{128, 32}, {1280, 320}, {12800, 3200}}; + + virtual_dataset = std::make_shared("virtual_dataset_info", inputs_shape, outputs_shape, attr); +} + +TEST_F(TestVirtualDatasetInfo, InferDevMatrixShape1) { + std::vector inputs = {{16, 1}, {16, 1}, {16, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + virtual_dataset->Init(strategy); + std::vector dev_matrix_shape = virtual_dataset->dev_matrix_shape(); + + std::vector expect = {16}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestVirtualDatasetInfo, InferDevMatrixShape2) { + std::vector inputs = {{8, 1}, {8, 1}, {8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + virtual_dataset->Init(strategy); + std::vector dev_matrix_shape = virtual_dataset->dev_matrix_shape(); + + std::vector expect = {8, 2}; + ASSERT_EQ(dev_matrix_shape, expect); +} + +TEST_F(TestVirtualDatasetInfo, InferSliceShape1) { + std::vector str = {{8, 1}, {8, 1}, {8, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + virtual_dataset->Init(strategy); + std::vector inputs = virtual_dataset->inputs_tensor_info(); + std::vector outputs = virtual_dataset->outputs_tensor_info(); + + Shape input_slice_shape_expect = {16, 32}; + Shape output_slice_shape_expect = {16, 32}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Shape input_slice_shape = input_tensor_info.slice_shape(); + Shape output_slice_shape = output_tensor_info.slice_shape(); + + ASSERT_EQ(input_slice_shape, input_slice_shape_expect); + ASSERT_EQ(output_slice_shape, output_slice_shape_expect); + + Shape input_slice_shape_expect1 = {160, 320}; + Shape output_slice_shape_expect1 = {160, 320}; + + TensorInfo input_tensor_info1 = inputs.at(1); + TensorInfo output_tensor_info1 = outputs.at(1); + + Shape input_slice_shape1 = input_tensor_info1.slice_shape(); + Shape output_slice_shape1 = output_tensor_info1.slice_shape(); + + ASSERT_EQ(input_slice_shape1, input_slice_shape_expect1); + ASSERT_EQ(output_slice_shape1, output_slice_shape_expect1); + + Shape input_slice_shape_expect2 = {1600, 3200}; + Shape output_slice_shape_expect2 = {1600, 3200}; + + TensorInfo input_tensor_info2 = inputs.at(2); + TensorInfo output_tensor_info2 = outputs.at(2); + + Shape input_slice_shape2 = input_tensor_info2.slice_shape(); + Shape output_slice_shape2 = output_tensor_info2.slice_shape(); + + ASSERT_EQ(input_slice_shape2, input_slice_shape_expect2); + ASSERT_EQ(output_slice_shape2, output_slice_shape_expect2); +} + +TEST_F(TestVirtualDatasetInfo, GetTensorLayout1) { + std::vector str = {{8, 1}, {8, 1}, {8, 1}}; + StrategyPtr strategy = NewStrategy(0, str); + + virtual_dataset->Init(strategy); + std::vector inputs = virtual_dataset->inputs_tensor_info(); + std::vector outputs = virtual_dataset->outputs_tensor_info(); + + TensorMap input_expect = {1, -1}; + TensorMap output_expect = {1, -1}; + + TensorInfo input_tensor_info = inputs.at(0); + TensorInfo output_tensor_info = outputs.at(0); + + Map input_tensor_map = input_tensor_info.tensor_layout().origin_tensor_map(); + Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map(); + + ASSERT_EQ(input_tensor_map.array(), input_expect); + ASSERT_EQ(output_tensor_map.array(), output_expect); +} + +TEST_F(TestVirtualDatasetInfo, GetForwardOp1) { + std::vector inputs = {{8, 1}, {8, 1}, {8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + virtual_dataset->Init(strategy); + OperatorVector forward_op = virtual_dataset->forward_op(); + size_t size = forward_op.size(); + + ASSERT_EQ(size, 0); +} + +TEST_F(TestVirtualDatasetInfo, GetMirrorOPs1) { + std::vector inputs = {{8, 1}, {8, 1}, {8, 1}}; + StrategyPtr strategy = NewStrategy(0, inputs); + + virtual_dataset->Init(strategy); + MirrorOps mirror_ops = virtual_dataset->mirror_ops(); + + size_t size = mirror_ops.size(); + // no broadcast + ASSERT_EQ(size, 0); + // ASSERT_EQ(size, 3); +} + +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/parse/parser_abnormal_test.cc b/tests/ut/cpp/pipeline/parse/parser_abnormal_test.cc new file mode 100644 index 00000000000..3c97cfb2038 --- /dev/null +++ b/tests/ut/cpp/pipeline/parse/parser_abnormal_test.cc @@ -0,0 +1,128 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "utils/log_adapter.h" +#include "utils/profile.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +namespace parse { + +class TestParserAbnormal : public UT::Common { + public: + TestParserAbnormal() : getPyFun("gtest_input.pipeline.parse.parse_abnormal") {} + virtual void SetUp(); + virtual void TearDown(); + + public: + UT::PyFuncGraphFetcher getPyFun; +}; + +void TestParserAbnormal::SetUp() { UT::InitPythonPath(); } + +void TestParserAbnormal::TearDown() {} + +TEST_F(TestParserAbnormal, TestParseRecursion) { + FuncGraphPtr func_graph = getPyFun("test_keep_roots_recursion"); + ASSERT_TRUE(nullptr != func_graph); + + // save the func func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_recursion_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +int test_performance(int x) { return x; } + +TEST_F(TestParserAbnormal, TestPythonAdapterPerformance) { + MS_LOG(INFO) << "TestPythonAdapterPerformance start"; + std::shared_ptr env = python_adapter::set_python_scoped(); + py::module mod = python_adapter::GetPyModule("gtest_input.pipeline.parse.parse_abnormal"); + + // call the python function + std::size_t count = 1000000; + double t1 = GetTime(); + for (std::size_t i = 0; i < count; i++) { + mod.attr("test_performance")(i); + } + double t2 = GetTime(); + printf("Call python function %lu time is : %f", count, t2 - t1); + + // call the python function + t1 = GetTime(); + for (std::size_t i = 0; i < count; i++) { + test_performance(i); + } + t2 = GetTime(); + printf("Call c++ function %lu time is : %f", count, t2 - t1); +} + +// test the single Expr statement +TEST_F(TestParserAbnormal, TestParseExprStatement) { + FuncGraphPtr func_graph = getPyFun("test_list_append"); + ASSERT_TRUE(nullptr != func_graph); + + // save the func func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw func graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_ExprStatement_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } + + // check the 'append' node + bool is_append_node = false; + int count = 0; + py::object dataclass_obj; + // check the dataclass + for (auto node : manager->all_nodes()) { + if (node != nullptr && node->isa() && node->cast()->value()->isa()) { + if (GetValue(node->cast()->value()) == "append") { + is_append_node = true; + count++; + } + } + } + ASSERT_TRUE(is_append_node); + ASSERT_EQ(count, 2); + MS_LOG(INFO) << "append node have: " << count << " ."; +} + +} // namespace parse +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/parse/parser_class_test.cc b/tests/ut/cpp/pipeline/parse/parser_class_test.cc new file mode 100644 index 00000000000..599994aab24 --- /dev/null +++ b/tests/ut/cpp/pipeline/parse/parser_class_test.cc @@ -0,0 +1,119 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +namespace parse { + +class TestParserClass : public UT::Common { + public: + TestParserClass() {} + virtual void SetUp(); + virtual void TearDown(); +}; + +void TestParserClass::SetUp() { UT::InitPythonPath(); } + +void TestParserClass::TearDown() {} + +// Test case1 : test class method +TEST_F(TestParserClass, TestParseDataClassApi) { + py::function fn_ = python_adapter::GetPyFn("gtest_input.pipeline.parse.parser_test", "test_class_fn"); + Parser::InitParserEnvironment(fn_); + FuncGraphPtr func_graph = ParsePythonCode(fn_); + ASSERT_TRUE(nullptr != func_graph); + + // save the func func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // check the dataclass + bool is_dataclass = false; + py::object dataclass_obj; + // check the dataclass + for (auto node : manager->all_nodes()) { + if (node->isa()) { + ValuePtr value = node->cast()->value(); + if (value->isa()) { + if (IsValueNode(node)) { + is_dataclass = true; + dataclass_obj = value->cast>()->obj(); + } + } + } + } + + ASSERT_TRUE(is_dataclass); + + // parse data class method + py::object inf_method = python_adapter::GetPyObjAttr(dataclass_obj, "inf"); + FuncGraphPtr graph_inf = ParsePythonCode(inf_method); + ASSERT_TRUE(nullptr != graph_inf); + manager->AddFuncGraph(graph_inf); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_class_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +// Test case 2: test parse object, transfore the CELL instance to api. +TEST_F(TestParserClass, TestParseMethod) { + py::object obj_ = python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_class", "test_parse_object_instance"); + Parser::InitParserEnvironment(obj_); + FuncGraphPtr func_graph = ParsePythonCode(obj_); + ASSERT_TRUE(nullptr != func_graph); + draw::Draw("ut_parser_method_x.dot", func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_method_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +// Test case 3: common test for debug ptest case +TEST_F(TestParserClass, TestParseCompileAPI) { + python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_compile", "test_build"); + MS_LOG(DEBUG) << "Test end"; +} + +} // namespace parse +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/parse/parser_integrate_test.cc b/tests/ut/cpp/pipeline/parse/parser_integrate_test.cc new file mode 100644 index 00000000000..3ec260c6c02 --- /dev/null +++ b/tests/ut/cpp/pipeline/parse/parser_integrate_test.cc @@ -0,0 +1,116 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +namespace parse { + +class TestParserIntegrate : public UT::Common { + public: + TestParserIntegrate() : getPyFun("gtest_input.pipeline.parse.parser_integrate") {} + virtual void SetUp(); + virtual void TearDown(); + py::function GetPythonFunction(std::string function); + + public: + UT::PyFuncGraphFetcher getPyFun; +}; + +void TestParserIntegrate::SetUp() { UT::InitPythonPath(); } + +void TestParserIntegrate::TearDown() {} + +TEST_F(TestParserIntegrate, TestParseGraphTestHighOrderFunction) { + auto func_graph = getPyFun("test_high_order_function"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestHofTup) { + auto func_graph = getPyFun("test_hof_tup"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestWhile2) { + auto func_graph = getPyFun("test_while_2"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestNestedClosure) { + auto func_graph = getPyFun("test_nested_closure"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestFunctionsInTuples) { + auto func_graph = getPyFun("test_functions_in_tuples"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestClosuresInTuples) { + auto func_graph = getPyFun("test_closures_in_tuples"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestCompileConv2d) { + py::function fn_ = python_adapter::GetPyFn("gtest_input.pipeline.parse.parser_integrate", "test_compile_conv2d"); + // fn_(); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestNone) { + auto func_graph = getPyFun("test_none"); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphResolveGetAttr) { + getPyFun.SetDoResolve(true); + auto func_graph = getPyFun("test_get_attr"); + draw::Draw("getattr.dot", func_graph); + ASSERT_TRUE(func_graph != nullptr); +} + +TEST_F(TestParserIntegrate, TestParseGraphResolveUnknown) { + EXPECT_THROW({ python_adapter::CallPyFn("gtest_input.pipeline.parse.parser_integrate", "test_undefined_symbol"); }, + std::runtime_error); +} + +/* #not supported yet +TEST_F(TestParserIntegrate, TestParseGraphTestModelInside) { + py::function fn_ = python_adapter::GetPyFn( + "gtest_input.pipeline.parse.parser_integrate", "test_model_inside"); + fn_(); + +} + */ +/* # infer not supported yet +TEST_F(TestParserIntegrate, TestParseGraphTestTensorAdd) { + py::function fn_ = python_adapter::GetPyFn( + "gtest_input.pipeline.parse.parser_integrate", "test_tensor_add"); + fn_(); +} + +TEST_F(TestParserIntegrate, TestParseGraphTestResnet50Build) { + py::function fn_ = python_adapter::GetPyFn( + "gtest_input.pipeline.parse.parser_integrate", "test_resetnet50_build"); + fn_(); +} + */ +} // namespace parse +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/parse/parser_primitive_test.cc b/tests/ut/cpp/pipeline/parse/parser_primitive_test.cc new file mode 100644 index 00000000000..e4cfd5132fd --- /dev/null +++ b/tests/ut/cpp/pipeline/parse/parser_primitive_test.cc @@ -0,0 +1,162 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +namespace parse { + +class TestParserPrimitive : public UT::Common { + public: + TestParserPrimitive() {} + virtual void SetUp(); + virtual void TearDown(); +}; + +void TestParserPrimitive::SetUp() { UT::InitPythonPath(); } + +void TestParserPrimitive::TearDown() {} + +TEST_F(TestParserPrimitive, TestParserOpsMethod1) { + py::function fn_ = python_adapter::GetPyFn("gtest_input.pipeline.parse.parse_primitive", "test_ops_f1"); + + FuncGraphPtr func_graph = ParsePythonCode(fn_); + ASSERT_TRUE(nullptr != func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_ops_1_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +TEST_F(TestParserPrimitive, TestParserOpsMethod2) { + py::function fn_ = python_adapter::GetPyFn("gtest_input.pipeline.parse.parse_primitive", "test_ops_f2"); + + FuncGraphPtr func_graph = ParsePythonCode(fn_); + ASSERT_TRUE(nullptr != func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_ops_2_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +// Test primitive class obj +TEST_F(TestParserPrimitive, TestParsePrimitive) { +#if 0 // Segmentation fault + py::object obj_ = python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_primitive", "test_primitive_obj"); + Parser::InitParserEnvironment(obj_); + FuncGraphPtr func_graph = ParsePythonCode(obj_); + ASSERT_TRUE(nullptr != func_graph); + draw::Draw("ut_parser_primitive_x.dot", func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_ops_3_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +#endif +} + +TEST_F(TestParserPrimitive, TestParsePrimitiveParmeter) { + py::object obj_ = + python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_primitive", "test_primitive_obj_parameter"); + Parser::InitParserEnvironment(obj_); + FuncGraphPtr func_graph = ParsePythonCode(obj_); + ASSERT_TRUE(nullptr != func_graph); + draw::Draw("ut_parser_primitive_x.dot", func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_ops_4_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +TEST_F(TestParserPrimitive, TestParsePrimitiveParmeter2) { + py::object obj_ = python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_primitive", "test_primitive_functional"); + Parser::InitParserEnvironment(obj_); + FuncGraphPtr func_graph = ParsePythonCode(obj_); + ASSERT_TRUE(nullptr != func_graph); + draw::Draw("ut_parser_primitive_x.dot", func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_ops_5_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +} // namespace parse +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/parse/parser_test.cc b/tests/ut/cpp/pipeline/parse/parser_test.cc new file mode 100644 index 00000000000..4d7731dfd1f --- /dev/null +++ b/tests/ut/cpp/pipeline/parse/parser_test.cc @@ -0,0 +1,399 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +namespace parse { +class TestParser : public UT::Common { + public: + TestParser() {} + virtual void SetUp(); + virtual void TearDown(); + + py::function fn; + + py::function GetPythonFunction(std::string function); +}; + +void TestParser::SetUp() { UT::InitPythonPath(); } + +void TestParser::TearDown() {} + +py::function TestParser::GetPythonFunction(std::string function) { + // init resource + try { + fn = python_adapter::GetPyFn("gtest_input.pipeline.parse.parser_test", function.c_str()); + return fn; + } catch (...) { + MS_LOG(ERROR) << "get fn failure!!!"; + } + return py::none(); +} + +TEST_F(TestParser, TestParseApi) { + // Test null fn + py::function fn_null; + FuncGraphPtr func_graph = ParsePythonCode(fn_null); + ASSERT_TRUE(nullptr == func_graph); + + // Test parse api + GetPythonFunction("test_f"); + func_graph = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != func_graph); +} + +TEST_F(TestParser, TestParseAst) { + GetPythonFunction("test_f"); + + ParseAst ast = ParseAst(fn); + bool succ = ast.InitParseAstInfo(); + ASSERT_TRUE(succ = true); + + // get FunctionDef node + py::object node = ast.GetAstNode(); + + // check arg + std::string fun_args[] = {"x", "y"}; + std::string fun_name = "test_f"; + py::list args = ast.GetArgs(node); + for (std::size_t i = 0; i < args.size(); i++) { + py::str pyArg = args[i].attr("arg"); + std::string arg = pyArg; + ASSERT_STREQ(arg.c_str(), fun_args[i].c_str()); + } + + // check function name + // get function name + py::str name = python_adapter::GetPyObjAttr(node, "name"); + std::string function_name = name; + ASSERT_STREQ(function_name.c_str(), fun_name.c_str()); +} + +TEST_F(TestParser, TestParseGraphSuccess) { + GetPythonFunction("test_f"); + // parse fn to graph + FuncGraphPtr func_graph = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != func_graph); +} + +TEST_F(TestParser, TestParseGraphFailure) { + GetPythonFunction("get_no_return_fn"); + + // create parser + std::shared_ptr ast = std::make_shared(fn); + bool succ = ast->InitParseAstInfo(); + ASSERT_TRUE(succ = true); + std::shared_ptr parser = std::make_shared(ast); + + // parse ast to graph + FuncGraphPtr func_graph = parser->ParseFuncGraph(); + ASSERT_EQ(PARSE_NO_RETURN, parser->errcode()); + ASSERT_TRUE(nullptr == func_graph); +} + +TEST_F(TestParser, TestParseGraphIf) { + GetPythonFunction("test_if"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphIfExp) { + GetPythonFunction("test_ifexp"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphIfNested) { + GetPythonFunction("test_if_nested"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseWhile) { + GetPythonFunction("test_while"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphNum) { + FuncGraphPtr ret_val; + GetPythonFunction("testDoNum"); + ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphStr) { + FuncGraphPtr ret_val; + GetPythonFunction("testDoStr"); + ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphNamedConst) { + FuncGraphPtr ret_val; + GetPythonFunction("testDoNamedConstTrue"); + ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); + GetPythonFunction("testDoNamedConstFalse"); + ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); + GetPythonFunction("testDoNamedConstNone"); + ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphForStatement) { + GetPythonFunction("test_for"); + + FuncGraphPtr func_graph = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_for_loop_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +TEST_F(TestParser, TestParseGraphCompareExprLt) { + GetPythonFunction("test_compare_lt"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphCompareExprGt) { + GetPythonFunction("test_compare_gt"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphCompareExprLe) { + GetPythonFunction("test_compare_le"); + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphCompareExprNe) { + GetPythonFunction("test_compare_ne"); + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphCompareExprGe) { + GetPythonFunction("test_compare_ge"); + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphCompareExprEq) { + GetPythonFunction("test_compare_eq"); + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphBoolOpTwoAnd) { + GetPythonFunction("test_boolop_two_and"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphBoolOpThreeAnd) { + GetPythonFunction("test_boolop_three_and"); + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphBoolOpTwoOr) { + GetPythonFunction("test_boolop_two_or"); + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphBoolOpThreeOr) { + GetPythonFunction("test_boolop_three_or"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphBoolOpMixAndOr) { + GetPythonFunction("test_boolop_mix_and_or"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphLambda) { + GetPythonFunction("test_lambda"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphFuncDef) { + GetPythonFunction("test_funcdef"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphSimpleClosure) { + GetPythonFunction("test_simple_closure"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphTestTuple) { + GetPythonFunction("test_tuple_fn"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphTupleAssign) { + GetPythonFunction("test_assign_tuple"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphTestList) { + GetPythonFunction("test_list_fn"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphUnaryOp) { + GetPythonFunction("test_unary"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphAguassign) { + GetPythonFunction("test_augassign"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseSystemFunction) { + GetPythonFunction("test_sys_call"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); +} + +TEST_F(TestParser, TestParseGraphBoolNot) { + GetPythonFunction("test_bool_not"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); + + // save the func_graph to manager + std::shared_ptr manager = Manage(ret_val); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + // draw graph + int i = 0; + for (auto tmp : manager->func_graphs()) { + std::string name = "ut_parser_for_not_" + std::to_string(i) + ".dot"; + draw::Draw(name, tmp); + i++; + } +} + +TEST_F(TestParser, TestCallPythonFnUseTupleParamete) { + GetPythonFunction("test_call_fn_use_tuple"); + + py::tuple params = py::tuple(5); + params[0] = 0; + params[1] = 1; + params[2] = 2.0; + params[3] = fn; + params[4] = "test_call_fn_use_tuple"; + py::object result = + python_adapter::CallPyFn("gtest_input.pipeline.parse.parser_test", "test_call_fn_use_tuple", params); + + int ret_size = py::cast(result); + + ASSERT_EQ(ret_size, 5); +} + +TEST_F(TestParser, TestParseGraphSubscriptSetitem) { + GetPythonFunction("test_subscript_setitem"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); + + std::shared_ptr manager = Manage(ret_val); + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); +} + +TEST_F(TestParser, TestParseGraphDict) { + GetPythonFunction("test_dict"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); + + std::shared_ptr manager = Manage(ret_val); + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); +} + +TEST_F(TestParser, TestParseGraphCallVargs) { + GetPythonFunction("test_call_variable"); + + FuncGraphPtr ret_val = ParsePythonCode(fn); + ASSERT_TRUE(nullptr != ret_val); + + std::shared_ptr manager = Manage(ret_val); + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); +} +} // namespace parse +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/parse/resolve_test.cc b/tests/ut/cpp/pipeline/parse/resolve_test.cc new file mode 100644 index 00000000000..8ade92bb349 --- /dev/null +++ b/tests/ut/cpp/pipeline/parse/resolve_test.cc @@ -0,0 +1,103 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "utils/log_adapter.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" + +namespace mindspore { +namespace parse { + +class TestResolve : public UT::Common { + public: + TestResolve() {} + virtual void SetUp(); + virtual void TearDown(); +}; + +void TestResolve::SetUp() { UT::InitPythonPath(); } + +void TestResolve::TearDown() {} + +TEST_F(TestResolve, TestResolveApi) { + py::function fn_ = python_adapter::GetPyFn("gtest_input.pipeline.parse.parser_test", "get_resolve_fn"); + + // parse graph + FuncGraphPtr func_graph = ParsePythonCode(fn_); + ASSERT_FALSE(nullptr == func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + ASSERT_EQ(manager->func_graphs().size(), (size_t)2); + + // draw graph + int i = 0; + for (auto func_graph : manager->func_graphs()) { + std::string name = "ut_resolve_graph_" + std::to_string(i) + ".dot"; + draw::Draw(name, func_graph); + i++; + } +} + +TEST_F(TestResolve, TestParseGraphTestClosureResolve) { + py::function test_fn = + python_adapter::CallPyFn("gtest_input.pipeline.parse.parser_test", "test_reslove_closure", 123); + FuncGraphPtr func_graph = ParsePythonCode(test_fn); + ASSERT_TRUE(func_graph != nullptr); + draw::Draw("test_reslove_closure.dot", func_graph); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + ASSERT_TRUE(ret_); + + ASSERT_EQ(manager->func_graphs().size(), (size_t)2); + + // draw graph + int i = 0; + for (auto func_graph : manager->func_graphs()) { + std::string name = "ut_test_reslove_closure_graph_" + std::to_string(i) + ".dot"; + draw::Draw(name, func_graph); + i++; + } +} +TEST_F(TestResolve, TestResolveFail) { + py::function fn_ = python_adapter::GetPyFn("gtest_input.pipeline.parse.parser_test", "test_resolvefail"); + + // parse graph + FuncGraphPtr func_graph = ParsePythonCode(fn_); + ASSERT_FALSE(nullptr == func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + EXPECT_THROW({ ResolveAll(manager); }, std::runtime_error); +} +} // namespace parse +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/resource_test.cc b/tests/ut/cpp/pipeline/resource_test.cc new file mode 100644 index 00000000000..09bd2060dc3 --- /dev/null +++ b/tests/ut/cpp/pipeline/resource_test.cc @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" +#include "utils/log_adapter.h" +#include "pipeline/resource.h" +#include "ir/primitive.h" +#include "operator/ops.h" + +namespace mindspore { +namespace pipeline { + +using MethodMap = std::unordered_map>; + +extern MethodMap& GetMethodMap(); + +class TestResource : public UT::Common { + public: + TestResource() {} + void SetUp() {} + void TearDown() {} +}; + +TEST_F(TestResource, test_standard_method_map) { + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeInt)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeInt8)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeInt16)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeInt32)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeInt64)); + + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeFloat)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeFloat16)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeFloat32)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeFloat64)); + + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeBool)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kNumberTypeUInt)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kObjectTypeTuple)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kObjectTypeList)); + ASSERT_TRUE(true == Resource::IsTypeInMethodMap(kObjectTypeTensorType)); + + MethodMap& map = GetMethodMap(); + for (auto& iter : map) { + for (auto& iter_map : iter.second) { + Any value = iter_map.second; + ASSERT_TRUE(value.is() || value.is()); + } + } + + Any value = Resource::GetMethodPtr(kNumberTypeInt, "__add__"); + ASSERT_TRUE(value == Any(prim::kPrimScalarAdd)); + value = Resource::GetMethodPtr(kNumberTypeInt64, "__add__"); + ASSERT_TRUE(value == Any(prim::kPrimScalarAdd)); + value = Resource::GetMethodPtr(kNumberTypeFloat, "__add__"); + ASSERT_TRUE(value == Any(prim::kPrimScalarAdd)); + value = Resource::GetMethodPtr(kNumberTypeFloat64, "__add__"); + ASSERT_TRUE(value == Any(prim::kPrimScalarAdd)); +} + +} // namespace pipeline +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/abstract_test.cc b/tests/ut/cpp/pipeline/static_analysis/abstract_test.cc new file mode 100644 index 00000000000..93baf86c3ea --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/abstract_test.cc @@ -0,0 +1,97 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" + +#include "pipeline/static_analysis/static_analysis.h" +#include "pipeline/static_analysis/utils.h" +#include "pipeline/static_analysis/prim.h" +#include "pipeline/parse/parse.h" +#include "pipeline/parse/resolve.h" +#include "pipeline/parse/data_converter.h" +#include "operator/ops.h" + +namespace mindspore { +namespace abstract { + +class TestAbstract : public UT::Common { + public: + TestAbstract() {} + virtual void SetUp() {} + virtual void TearDown() {} +}; + +TEST_F(TestAbstract, TestParseDataClass) { + py::object fn = parse::python_adapter::GetPyFn("gtest_input.pipeline.parse.parser_test", "TestFoo"); + + ClassPtr cls_ptr = parse::ParseDataClass(fn); + ASSERT_TRUE(nullptr != cls_ptr); + std::shared_ptr cls = dyn_cast(cls_ptr); + ASSERT_TRUE(nullptr != cls); + + MS_LOG(INFO) << "" << cls->ToString(); + ASSERT_EQ(cls->tag(), Named(std::string("TestFoo"))); + + ClassAttrVector attributes = cls->GetAttributes(); + ASSERT_EQ(attributes.size(), 2); + for (auto &v : attributes) { + if (v.first == std::string("x")) { + ASSERT_TRUE(nullptr != dyn_cast(v.second)); + } + if (v.first == std::string("y")) { + ASSERT_TRUE(nullptr != dyn_cast(v.second)); + } + } + + std::unordered_map methods = cls->methods(); + ASSERT_EQ(methods.size(), 4); + int counts = 0; + for (auto &v : methods) { + if (v.first == std::string("inf")) { + counts++; + } + MS_LOG(INFO) << "" << v.first; + } + ASSERT_EQ(counts, 1); + + ValuePtr obj = std::make_shared(fn, "TestFoo"); + + ValueNodePtr fn_node = NewValueNode(obj); + AnfNodeConfigPtr fn_conf = std::make_shared(nullptr, fn_node, nullptr); + AbstractBasePtr foo = ToAbstract(obj, nullptr, fn_conf); + ASSERT_TRUE(foo != nullptr); + + AbstractBasePtr abstract_x = FromValue(1.1, true); + AbstractBasePtr abstract_y = FromValue(5, true); + + auto partical_func = dyn_cast(foo); + AbstractBasePtrList args_spec_list = partical_func->args(); + ASSERT_GT(args_spec_list.size(), 0); + AbstractScalarPtr abs_scalar = dyn_cast(args_spec_list[0]); + + AbstractBasePtrList args_list = {abs_scalar, abstract_x, abstract_y}; + + StandardPrimitiveEvalImpl eval_impl = GetPrimitiveInferImpl(prim::kPrimMakeRecord); + ASSERT_TRUE(nullptr != eval_impl); + + AbstractBasePtr new_cls = eval_impl(nullptr, prim::kPrimMakeRecord, args_list); + ASSERT_TRUE(nullptr != new_cls); +} + +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/data_test.cc b/tests/ut/cpp/pipeline/static_analysis/data_test.cc new file mode 100644 index 00000000000..c0c0bd14529 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/data_test.cc @@ -0,0 +1,204 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "pipeline/static_analysis/utils.h" + +namespace mindspore { +namespace abstract { + +class TestData : public UT::Common { + public: + void SetUp(); + void TearDown(); +}; + +void TestData::SetUp() { UT::InitPythonPath(); } + +void TestData::TearDown() { + // destroy resource +} + +TEST_F(TestData, test_build_value) { + // assert build_value(S(1)) == 1 + AbstractScalar s1 = AbstractScalar(1); + ASSERT_EQ(1, s1.BuildValue()->cast()->value()); + // assert build_value(S(t=ty.Int[64]), default=ANYTHING) is ANYTHING + s1 = AbstractScalar(kAnyValue, kInt32); + ASSERT_TRUE(s1.BuildValue()->isa()); + ASSERT_TRUE(s1.BuildValue()->isa()); + + // assert build_value(T([S(1), S(2)])) == (1, 2) + AbstractBasePtr base1 = std::make_shared(1); + AbstractBasePtr base2 = std::make_shared(2); + AbstractBasePtrList base_list = {base1, base2}; + AbstractTuple t1 = AbstractTuple(base_list); + + std::vector value_list = {MakeValue(1), MakeValue(2)}; + auto tup = t1.BuildValue()->cast()->value(); + + ASSERT_TRUE(tup.size() == value_list.size()); + for (int i = 0; i < value_list.size(); i++) { + ASSERT_EQ(*tup[i], *value_list[i]); + } + + // BuildValue(AbstractFunction) should return kAnyValue. + AbstractBasePtr abs_f1 = FromValue(prim::kPrimReturn, false); + ValuePtr abs_f1_built = abs_f1->BuildValue(); + ASSERT_EQ(abs_f1_built, kAnyValue); + + FuncGraphPtr fg1 = std::make_shared(); + AbstractBasePtr abs_fg1 = FromValue(fg1, false); + ValuePtr abs_fg1_built = abs_fg1->BuildValue(); + ASSERT_EQ(abs_fg1_built, kAnyValue); + + // BuildValue(Tuple(AbstractFunction)) should return kAnyValue; + AbstractBasePtr abs_f2 = FromValue(prim::kPrimScalarAdd, false); + AbstractBasePtr abs_func_tuple = std::make_shared(AbstractBasePtrList({abs_f1, abs_f2})); + ValuePtr func_tuple_built = abs_func_tuple->BuildValue(); + ASSERT_EQ(func_tuple_built, kAnyValue); + + // BuildValue(List(AbstractFunction)) should return kAnyValue; + AbstractBasePtr abs_func_list = std::make_shared(AbstractBasePtrList({abs_f1, abs_f2})); + ValuePtr func_list_built = abs_func_list->BuildValue(); + ASSERT_EQ(func_list_built, kAnyValue); + + // BuildValue(Tuple(AnyAbstractBase, AbstractFunction)) should return kAnyValue + abs_func_tuple = std::make_shared(AbstractBasePtrList({base1, abs_f2})); + func_tuple_built = abs_func_tuple->BuildValue(); + ASSERT_EQ(func_tuple_built, kAnyValue); +} + +TEST_F(TestData, test_build_type) { + AbstractBasePtr s1 = FromValue(1, false); + AbstractBasePtr s2 = FromValue(2, false); + ASSERT_TRUE(Int(32) == *s1->BuildType()); + + AbstractFunctionPtr f1 = std::make_shared(nullptr, nullptr); + ASSERT_TRUE(Function() == *f1->BuildType()); + + AbstractList l1 = AbstractList({s1, s2}); + ASSERT_TRUE(List({std::make_shared(32), std::make_shared(32)}) == *l1.BuildType()); +} + +TEST_F(TestData, test_build_shape) { + AbstractBasePtr s1 = FromValue(1, false); + AbstractBasePtr s2 = FromValue(2, false); + ASSERT_TRUE(NoShape() == *s1->BuildShape()); + + AbstractFunctionPtr f1 = std::make_shared(nullptr, nullptr); + ASSERT_TRUE(NoShape() == *f1->BuildShape()); + + AbstractList l1 = AbstractList({s1, s2}); + auto lshape = l1.BuildShape(); + ASSERT_TRUE(lshape); + + std::vector weight1_dims = {2, 20, 5, 5}; + std::vector weight2_dims = {2, 2, 5, 5}; + tensor::TensorPtr weight1 = std::make_shared(); + weight1->set_data_type(kNumberTypeInt32); + weight1->set_shape(weight1_dims); + tensor::TensorPtr weight2 = std::make_shared(); + weight2->set_data_type(kNumberTypeInt32); + weight2->set_shape(weight2_dims); + + AbstractBasePtr abstract_weight1 = FromValue(weight1, true); + AbstractBasePtr abstract_weight2 = FromValue(weight2, true); + ShapePtr shape_weight = dyn_cast(abstract_weight1->BuildShape()); + ASSERT_TRUE(shape_weight); + ASSERT_EQ(weight1_dims, shape_weight->shape()); + + std::vector vec({weight1, weight2}); + AbstractBasePtr abstract_tup = FromValue(vec, true); + std::shared_ptr shape_tuple = dyn_cast(abstract_tup->BuildShape()); + ASSERT_TRUE(shape_tuple); + const std::vector& ptr_vec = shape_tuple->shape(); + ASSERT_EQ(ptr_vec.size(), 2); + + ShapePtr shape1 = dyn_cast(ptr_vec[0]); + ASSERT_TRUE(shape1); + ASSERT_EQ(weight1_dims, shape1->shape()); + + ShapePtr shape2 = dyn_cast(ptr_vec[1]); + ASSERT_TRUE(shape2); + ASSERT_EQ(weight2_dims, shape2->shape()); +} + +TEST_F(TestData, test_clone) { + AbstractBasePtr s1 = FromValue(1, false); + AbstractBasePtr s2 = s1->Clone(); + ASSERT_TRUE(*s1->GetTypeTrack() == *s2->GetTypeTrack()); + ASSERT_TRUE(s1->GetValueTrack() == s2->GetValueTrack()); + ASSERT_TRUE(*s1->GetShapeTrack() == *s2->GetShapeTrack()); + + AbstractFunctionPtr f1 = std::make_shared(std::make_shared(), + AnalysisContext::DummyContext()); + AbstractBasePtr f2 = f1->Clone(); + ASSERT_TRUE(*f2 == *f1); + + AbstractList l1 = AbstractList({s1, s2}); + AbstractBasePtr l2 = l1.Clone(); + AbstractList* l2_cast = dynamic_cast(l2.get()); + ASSERT_TRUE(l2_cast != nullptr); + ASSERT_TRUE(l2_cast->GetValueTrack() == l1.GetValueTrack()); + + std::vector attr = {{"x", std::make_shared(kAnyValue, kInt64)}, + {"y", std::make_shared(kAnyValue, kInt64)}}; + std::unordered_map methods; + AbstractBasePtr c1 = std::make_shared(Named("Point"), attr, methods); + AbstractBasePtr c2 = c1->Clone(); + ASSERT_EQ(*c1, *c2); +} + +TEST_F(TestData, test_join) { + int int1 = 1; + AbstractBasePtr s1 = FromValue(int1, false); + AbstractBasePtr s2 = s1->Broaden(); + + std::vector xx = {s1, s2}; + AbstractListPtr l1 = std::make_shared(xx); + AbstractListPtr l2 = std::make_shared(xx); + l1->Join(l2); +} + +TEST_F(TestData, test_broaden) { + int int1 = 1; + AbstractBasePtr s1 = FromValue(int1, false); + AbstractBasePtr s2 = s1->Broaden(); + ASSERT_TRUE(*s1->GetTypeTrack() == *s2->GetTypeTrack()); + ASSERT_TRUE(*s1->GetValueTrack() == *MakeValue(int1)); + ASSERT_TRUE(s2->GetValueTrack()->isa()); + + AbstractFunctionPtr f1 = std::make_shared(std::make_shared(), + AnalysisContext::DummyContext()); + AbstractBasePtr f2 = f1->Broaden(); + ASSERT_TRUE(f2 == f1); + + AbstractList l1 = AbstractList({s1, s2}); + AbstractBasePtr l2 = l1.Broaden(); + AbstractList* l2_cast = dynamic_cast(l2.get()); + ASSERT_TRUE(l2_cast != nullptr); + AbstractBasePtr csr = AbstractJoin(l2_cast->elements()); + ASSERT_TRUE(csr->GetValueTrack()->isa()); +} + +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/dshape_test.cc b/tests/ut/cpp/pipeline/static_analysis/dshape_test.cc new file mode 100644 index 00000000000..ae18f7730b7 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/dshape_test.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" + +#include "pipeline/static_analysis/dshape.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace abstract { +class TestDShape : public UT::Common { + public: + Shape shp_1; + Shape shp_2; + Shape shp_3; + Shape shp_4; + + NoShape shp_noshp_1; + NoShape shp_noshp_2; + + TupleShape shp_tuple_1; + TupleShape shp_tuple_2; + TupleShape shp_tuple_3; + TupleShape shp_tuple_4; + TestDShape() + : shp_1({1, 1}), + shp_2({1, 1}), + shp_3({1, 2}), + shp_4({1}), + + shp_noshp_1(), + shp_noshp_2(), + + shp_tuple_1({NoShape().Clone(), Shape({1, 1}).Clone()}), + shp_tuple_2({NoShape().Clone(), Shape({1, 1, 1}).Clone()}), + shp_tuple_3({NoShape().Clone(), Shape({1, 2, 1}).Clone()}), + shp_tuple_4({NoShape().Clone()}) {} +}; + +TEST_F(TestDShape, EqualTest) { + ASSERT_TRUE(shp_1 == shp_2); + ASSERT_FALSE(shp_1 == shp_3); + ASSERT_FALSE(shp_1 == shp_noshp_1); + + ASSERT_TRUE(shp_noshp_1 == shp_noshp_2); + + ASSERT_FALSE(shp_tuple_1 == shp_1); + ASSERT_FALSE(shp_tuple_1 == shp_tuple_2); + ASSERT_FALSE(shp_tuple_1 == shp_tuple_4); +} +TEST_F(TestDShape, ToString) { + ASSERT_EQ(shp_3.ToString(), "(1, 2)"); + ASSERT_EQ(shp_noshp_1.ToString(), "NoShape"); + ASSERT_EQ(shp_tuple_2.ToString(), "TupleShape(NoShape, (1, 1, 1))"); +} + +TEST_F(TestDShape, Clone) { + ASSERT_EQ(*shp_3.Clone(), shp_3); + ASSERT_EQ(*shp_noshp_1.Clone(), shp_noshp_1); + ASSERT_EQ(*shp_tuple_2.Clone(), shp_tuple_2); +} + +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/evaluator_test.cc b/tests/ut/cpp/pipeline/static_analysis/evaluator_test.cc new file mode 100644 index 00000000000..d3983552e8a --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/evaluator_test.cc @@ -0,0 +1,244 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/evaluator.h" +#include "pipeline/static_analysis/prim.h" + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "pipeline/static_analysis/helper.h" + +#include "debug/draw.h" + +namespace mindspore { +namespace abstract { +namespace python_adapter = mindspore::parse::python_adapter; + +class TestEvaluatorCacheMap : public UT::Common { + public: + void SetUp() {} + void TearDown() {} +}; + +TEST_F(TestEvaluatorCacheMap, test_evaluator_cache_map) { + EvaluatorCacheMap cache; + + AbstractBasePtr abstract_v1 = FromValue(1, false); + AbstractBasePtr abstract_v2 = FromValue(2, false); + AbstractBasePtrList args_spec_list = {abstract_v1, abstract_v2}; + AbstractBasePtr abstract_val = FromValue(10, false); + cache[args_spec_list] = abstract_val; + + auto iter = cache.find(args_spec_list); + ASSERT_TRUE(iter != cache.end()); + ASSERT_TRUE(iter->second == abstract_val); + + AbstractBasePtr abstract_v1_variant1 = FromValue(1, false); + AbstractBasePtr abstract_v2_variant1 = FromValue(2, false); + AbstractBasePtrList args_spec_list_variant1 = {abstract_v1_variant1, abstract_v2_variant1}; + + iter = cache.find(args_spec_list_variant1); + ASSERT_TRUE(iter != cache.end()); + ASSERT_TRUE(iter->second == abstract_val); + + AbstractBasePtr abstract_v1_variant2 = FromValue(1, false); + AbstractBasePtr abstract_v2_variant2 = FromValue(3, false); + AbstractBasePtrList args_spec_list_variant2 = {abstract_v1_variant2, abstract_v2_variant2}; + + iter = cache.find(args_spec_list_variant2); + ASSERT_TRUE(iter == cache.end()); +} + +class TestStandardEvaluator : public UT::Common { + public: + TestStandardEvaluator() : getPyFun("gtest_input.pipeline.infer.infer_test", true), engine_(nullptr) {} + void SetUp(); + void TearDown(); + + UT::PyFuncGraphFetcher getPyFun; + AnalysisEnginePtr engine_; +}; + +void TestStandardEvaluator::SetUp() { engine_ = SetupAnalysisEngine(); } + +void TestStandardEvaluator::TearDown() { + // destroy resource +} + +TEST_F(TestStandardEvaluator, test_multiple_conv2d) { + std::shared_ptr env = python_adapter::set_python_scoped(); + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("test_multiple_conv2d"); + + // NCHW + std::vector inputs_dims = {2, 20, 32, 32}; + std::vector weight1_dims = {2, 20, 5, 5}; + std::vector weight2_dims = {2, 2, 5, 5}; + + tensor::TensorPtr inputs = std::make_shared(); + inputs->set_data_type(kNumberTypeInt32); + inputs->set_shape(inputs_dims); + // Cout, Cin, kernel_size + tensor::TensorPtr weight1 = std::make_shared(); + weight1->set_data_type(kNumberTypeInt32); + weight1->set_shape(weight1_dims); + // Cout, Cin, kernel_size + tensor::TensorPtr weight2 = std::make_shared(); + weight2->set_data_type(kNumberTypeInt32); + weight2->set_shape(weight2_dims); + + AbstractBasePtr abstract_inputs = FromValue(inputs, true); + AbstractBasePtr abstract_weight1 = FromValue(weight1, true); + AbstractBasePtr abstract_weight2 = FromValue(weight2, true); + AbstractBasePtrList args_spec_list = {abstract_inputs, abstract_weight1, abstract_weight2}; + + AbstractBasePtr expected = abstract_inputs->Clone(); + // NCHW + std::vector shape = {2, 2, 6, 6}; + expected->set_shape(std::make_shared(shape)); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_ptr = dyn_cast(res); + auto expected_ptr = dyn_cast(expected); + ASSERT_TRUE(*res_ptr->shape() == *expected_ptr->shape()); + ASSERT_TRUE(*res_ptr->element() == *expected_ptr->element()); +} + +class TestPartialEvaluator : public UT::Common { + public: + TestPartialEvaluator() : getPyFun("gtest_input.pipeline.infer.infer_test", true), engine_(nullptr) {} + void SetUp() { engine_ = SetupAnalysisEngine(); } + void TearDown() {} + + public: + UT::PyFuncGraphFetcher getPyFun; + AnalysisEnginePtr engine_; +}; + +TEST_F(TestPartialEvaluator, test_infer_dataclass_resolved) { + getPyFun.SetDoResolve(true); + FuncGraphPtr func_graph = getPyFun("test_dataclass_fun_sub"); + ASSERT_TRUE(nullptr != func_graph); + draw::Draw("test_dataclass_fun_sub.dot", func_graph); + + AbstractBasePtrList args_spec_list; + float x = 5.1; + + AbstractBasePtr abstract_x = FromValue(x, false); + args_spec_list.push_back(abstract_x); + + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_x->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeFloat32); +} + +TEST_F(TestPartialEvaluator, test_infer_dataclass_unresolved) { + getPyFun.SetDoResolve(false); + FuncGraphPtr func_graph = getPyFun("test_dataclass_fun_add"); + ASSERT_TRUE(nullptr != func_graph); + + AbstractBasePtrList args_spec_list; + float x = 5.2; + + AbstractBasePtr abstract_x = FromValue(x, false); + args_spec_list.push_back(abstract_x); + + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_x->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeFloat32); +} + +TEST_F(TestPartialEvaluator, test_infer_add_resolved) { + getPyFun.SetDoResolve(true); + FuncGraphPtr func_graph = getPyFun("test_fun_add"); + ASSERT_TRUE(nullptr != func_graph); + + AbstractBasePtrList args_spec_list; + double x = 5.2; + double y = 3.2; + + AbstractBasePtr abstract_x = FromValue(x, false); + AbstractBasePtr abstract_y = FromValue(y, false); + args_spec_list.push_back(abstract_x); + args_spec_list.push_back(abstract_y); + + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_x->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeFloat64); +} + +TEST_F(TestPartialEvaluator, test_infer_sub_unresolved) { + getPyFun.SetDoResolve(false); + FuncGraphPtr func_graph = getPyFun("test_fun_sub"); + ASSERT_TRUE(nullptr != func_graph); + + AbstractBasePtrList args_spec_list; + double x = 5.1; + double y = 3.1; + + AbstractBasePtr abstract_x = FromValue(x, false); + AbstractBasePtr abstract_y = FromValue(y, false); + args_spec_list.push_back(abstract_x); + args_spec_list.push_back(abstract_y); + + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_x->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeFloat64); +} + +TEST_F(TestPartialEvaluator, test_infer_net_construct_add_resolved) { + getPyFun.SetDoResolve(true); + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("test_net_construct_add"); + ASSERT_TRUE(nullptr != func_graph); + + AbstractBasePtrList args_spec_list; + double x = 1.2; + double y = 2.2; + + AbstractBasePtr abstract_x = FromValue(x, false); + AbstractBasePtr abstract_y = FromValue(y, false); + args_spec_list.push_back(abstract_x); + args_spec_list.push_back(abstract_y); + + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_x->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeFloat64); +} + +TEST_F(TestPartialEvaluator, test_infer_construct_sub_unresolved) { + getPyFun.SetDoResolve(false); + FuncGraphPtr func_graph = getPyFun.CallAndParseRet("test_net_construct_sub"); + ASSERT_TRUE(nullptr != func_graph); + draw::Draw("test_infer_simple_net.dot", func_graph); + + AbstractBasePtrList args_spec_list; + double x = 1.2; + double y = 2.2; + + AbstractBasePtr abstract_x = FromValue(x, false); + AbstractBasePtr abstract_y = FromValue(y, false); + args_spec_list.push_back(abstract_x); + args_spec_list.push_back(abstract_y); + + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_x->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeFloat64); +} +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/helper.cc b/tests/ut/cpp/pipeline/static_analysis/helper.cc new file mode 100644 index 00000000000..db697e95e0e --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/helper.cc @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pipeline/static_analysis/helper.h" + +#include "pipeline/static_analysis/prim.h" + +namespace mindspore { +namespace abstract { +/* for test which rely on prim module */ +AnalysisEnginePtr SetupAnalysisEngine() { + // init resource + std::shared_ptr graph_manager = MakeManager(); + AnalysisEnginePtr engine = std::make_shared(GetPrimEvaluatorConstructors(), graph_manager); + return engine; +} +} // namespace abstract +} // namespace mindspore + diff --git a/tests/ut/cpp/pipeline/static_analysis/helper.h b/tests/ut/cpp/pipeline/static_analysis/helper.h new file mode 100644 index 00000000000..7ca902a1e94 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/helper.h @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TESTS_UT_PIPELINE_STATIC_ANALYSIS_HELPER_H_ +#define TESTS_UT_PIPELINE_STATIC_ANALYSIS_HELPER_H_ + +#include "pipeline/static_analysis/evaluator.h" + +namespace mindspore { +namespace abstract { +AnalysisEnginePtr SetupAnalysisEngine(); +} // namespace abstract +} // namespace mindspore + +#endif // TESTS_UT_PIPELINE_STATIC_ANALYSIS_HELPER_H_ diff --git a/tests/ut/cpp/pipeline/static_analysis/prim_test.cc b/tests/ut/cpp/pipeline/static_analysis/prim_test.cc new file mode 100644 index 00000000000..629f4106012 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/prim_test.cc @@ -0,0 +1,1156 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "pybind11/pybind11.h" + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "ir/manager.h" +#include "pipeline/static_analysis/prim.h" +#include "pipeline/static_analysis/helper.h" +#include "operator/ops.h" +#include "debug/draw.h" +#include "ir/meta_tensor.h" +#include "utils/symbolic.h" +#include "./common.h" + +namespace mindspore { +namespace abstract { +namespace py = pybind11; +namespace python_adapter = mindspore::parse::python_adapter; +class UTPrimUtils { + public: + using AbstractTensorPtr = std::shared_ptr; + using AbstractTuplePtr = std::shared_ptr; + + static const std::shared_ptr kF32; + static const std::shared_ptr kF64; + static const std::shared_ptr kI16; + static const std::shared_ptr kI64; + static const std::shared_ptr kU64; + + static std::shared_ptr TypeToAbstract(TypePtr t) { return std::make_shared(t); } + + static AbstractTensorPtr ArrayFloat64Of(std::initializer_list shp) { + auto ele = std::make_shared(kAnyValue, kFloat64); + return std::make_shared(ele, std::make_shared(shp)); + } + + static AbstractTensorPtr ArrayFloat32Of(std::initializer_list shp) { + auto ele = std::make_shared(kAnyValue, kFloat32); + return std::make_shared(ele, std::make_shared(shp)); + } + + static AbstractTensorPtr ArrayInt32Of(std::initializer_list shp) { + auto ele = std::make_shared(kAnyValue, kInt32); + return std::make_shared(ele, std::make_shared(shp)); + } + + static AbstractTuplePtr ShapeOf(std::initializer_list vals) { + AbstractBasePtrList te; + for (auto v : vals) { + te.push_back(std::make_shared(v)); + } + return std::make_shared(te); + } + + static AbstractListPtr ListShapeOf(std::initializer_list vals) { + AbstractBasePtrList te; + for (auto v : vals) { + te.push_back(std::make_shared(v)); + } + return std::make_shared(te); + } +}; +const std::shared_ptr UTPrimUtils::kF64 = std::make_shared(64); +const std::shared_ptr UTPrimUtils::kF32 = std::make_shared(32); +const std::shared_ptr UTPrimUtils::kI16 = std::make_shared(16); +const std::shared_ptr UTPrimUtils::kI64 = std::make_shared(64); +const std::shared_ptr UTPrimUtils::kU64 = std::make_shared(64); +namespace { +AbstractBasePtr ArrayOfTensor(const TypePtr &t, std::initializer_list shp) { + auto shape = std::vector(shp); + auto tensor = std::make_shared(t->type_id(), shape); + return ToAbstract(tensor); +} + +} // namespace + +class TestPrim : public UT::Common { + public: + TestPrim() : getPyFun("gtest_input.pipeline.infer", true) {} + void SetUp(); + void TearDown(); + AnalysisEnginePtr engine_; + UT::PyFuncGraphFetcher getPyFun; +}; + +void TestPrim::SetUp() { engine_ = SetupAnalysisEngine(); } + +void TestPrim::TearDown() { + // destroy resource +} + +static FuncGraphPtr MakeFuncGraph(const PrimitivePtr prim, unsigned int nparam) { + // build the func_graph manually, eg: + // MakeFuncGraph(std::make_shared("scalar_add"), 2) means: + /* python source code: + * @mindspore + * def f(x, y): + * return x + y + */ + FuncGraphPtr func_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim)); + for (unsigned int i = 0; i < nparam; i++) { + inputs.push_back(func_graph->add_parameter()); + } + CNodePtr cnode_prim = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_prim); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + return func_graph; +} + +TEST_F(TestPrim, test_typeof) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + args_spec_list.push_back(abstract_v1); + + auto prim_typeof = std::make_shared("typeof"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_typeof, 1); + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + res->dump(); + TypePtr res_value = res->GetValueTrack()->cast(); + res_value->dump(); + ASSERT_TRUE(*res_value == Int(32)); +} + +TEST_F(TestPrim, test_list_map) { + AbstractBasePtrList args_spec_list; + + AbstractBasePtr abstract_v1 = FromValue(1, false); + AbstractBasePtr abstract_u1 = FromValue(1, false); + auto abstract_list1 = std::make_shared(AbstractBasePtrList({abstract_v1, abstract_u1})); + AbstractBasePtr abstract_v2 = FromValue(2, false); + AbstractBasePtr abstract_u2 = FromValue(2, false); + auto abstract_list2 = std::make_shared(AbstractBasePtrList({abstract_v2, abstract_u2})); + auto prim_scalar_add = std::make_shared("scalar_add"); + AbstractBasePtr abstract_func = ToAbstract(prim_scalar_add); + + args_spec_list.push_back(abstract_func); + args_spec_list.push_back(abstract_list1); + args_spec_list.push_back(abstract_list2); + + auto prim_list_map = std::make_shared("list_map"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_list_map, 3); + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + auto expected = std::make_shared(AbstractBasePtrList({FromValue(3, false), FromValue(3, false)})); + res->dump(); + MS_LOG(INFO) << "result res: " << res->ToString(); + MS_LOG(INFO) << "result expected: " << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_list_reduce) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractBasePtr abstract_v2 = FromValue(v1, false); + auto abstract_list = std::make_shared(AbstractBasePtrList({abstract_v1, abstract_v2})); + auto prim_scalar_add = std::make_shared("scalar_add"); + AbstractBasePtr abstract_func = ToAbstract(prim_scalar_add); + + args_spec_list.push_back(abstract_func); + args_spec_list.push_back(abstract_list); + args_spec_list.push_back(abstract_v1); + + auto prim_list_reduce = std::make_shared("list_reduce"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_list_reduce, 3); + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + res->dump(); + TypePtr res_type = res->GetTypeTrack(); + res_type->dump(); + ASSERT_TRUE(*res_type == Int(32)); +} + +TEST_F(TestPrim, test_scalar_to_array) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + + args_spec_list.push_back(abstract_v1); + + auto prim_scalar_to_array = std::make_shared("scalar_to_array"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_scalar_to_array, 1); + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + res->dump(); + TypePtr res_type = res->BuildType(); + res_type->dump(); + ASSERT_TRUE(*res_type == TensorType(std::make_shared(32))); +} + +TEST_F(TestPrim, test_array_to_scalar) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + auto abstract_a1 = std::make_shared(abstract_v1, std::make_shared()); + + args_spec_list.push_back(abstract_a1); + + auto prim_array_to_scalar = std::make_shared("array_to_scalar"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_array_to_scalar, 1); + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + res->dump(); + TypePtr res_type = res->BuildType(); + res_type->dump(); + ASSERT_TRUE(*res_type == Int(32)); +} + +TEST_F(TestPrim, test_J_1) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + args_spec_list.push_back(abstract_v1); + + auto prim_J = std::make_shared("J"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_J, 1); + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + AbstractJTaggedPtr res_J = dyn_cast(res); + ASSERT_TRUE(res_J != nullptr); + ASSERT_TRUE(*(res_J->element()) == *abstract_v1); +} + +TEST_F(TestPrim, test_J_2) { + // def add(x): + // return x + x + // def f(x): + // return J(add)(x) + std::vector inputs; + FuncGraphPtr func_graph1 = std::make_shared(); + inputs.push_back(NewValueNode(prim::kPrimScalarAdd)); + auto x = func_graph1->add_parameter(); + inputs.push_back(x); + inputs.push_back(x); + CNodePtr cnode1 = func_graph1->NewCNode(inputs); + func_graph1->set_return(cnode1); + + FuncGraphPtr func_graph = std::make_shared(); + inputs.clear(); + auto x1 = func_graph->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimJ)); + inputs.push_back(NewValueNode(func_graph1)); + CNodePtr jf = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(jf); + inputs.push_back(x1); + CNodePtr jf_jx = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(jf_jx); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + draw::Draw("test_J_2.dot", func_graph); + + int v1 = 1; + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractBasePtrList args_spec_list = {abstract_v1}; + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + res->dump(); + AbstractTuplePtr res_J = dyn_cast(res); + ASSERT_TRUE(res_J != nullptr); + auto res_J_0 = res_J->elements()[0]; + ASSERT_TRUE(res_J_0 != nullptr); + ASSERT_TRUE(*res_J_0 == *(FromValue(2, false))); + AbstractFunctionPtr res_J_1 = dyn_cast(res_J->elements()[1]); + ASSERT_TRUE(res_J_1 != nullptr); +} + +TEST_F(TestPrim, test_dot) { + auto dot = std::make_shared("dot"); + FuncGraphPtr func_graph = MakeFuncGraph(dot, 2); + + auto a1 = UTPrimUtils::ArrayFloat64Of({2, 3}); + auto a2 = UTPrimUtils::ArrayFloat64Of({3, 4}); + std::vector expectedA = {2, 4}; + auto expected = UTPrimUtils::ArrayFloat64Of({2, 4}); + + AbstractBasePtrList args_spec_list = {a1, a2}; + + AbstractTensorPtr res = dyn_cast(engine_->Run(func_graph, args_spec_list).inferred); + + ASSERT_TRUE(*(dyn_cast(res->GetShapeTrack())) == *(dyn_cast(expected->GetShapeTrack()))); +} + +// tail half +TEST_F(TestPrim, test_switch1) { + PrimitivePtr switch_ = std::make_shared("switch"); + FuncGraphPtr func_graph = MakeFuncGraph(switch_, 3); + + AbstractBasePtr arg0 = FromValue(true, false); + AbstractBasePtr arg1 = FromValue(1, false); + AbstractBasePtr arg2 = FromValue(2, false); + AbstractBasePtrList args_spec_list = {arg0, arg1, arg2}; + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *arg1); +} + +TEST_F(TestPrim, test_switch2) { + PrimitivePtr switch_ = std::make_shared("switch"); + FuncGraphPtr func_graph = MakeFuncGraph(switch_, 3); + + AbstractBasePtr arg0 = FromValue(false, false); + AbstractBasePtr arg1 = FromValue(1, false); + AbstractBasePtr arg2 = FromValue(2, false); + AbstractBasePtrList args_spec_list = {arg0, arg1, arg2}; + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "make result res: " << res->ToString(); + MS_LOG(INFO) << "make result arg2: " << arg2->ToString(); + ASSERT_TRUE(*res == *arg2); +} + +TEST_F(TestPrim, test_identity) { + PrimitivePtr identity = std::make_shared("identity"); + FuncGraphPtr func_graph = MakeFuncGraph(identity, 1); + + AbstractBasePtr abstract_v1 = FromValue(1, false); + AbstractBasePtrList args_spec_list = {abstract_v1}; + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *abstract_v1); +} + +TEST_F(TestPrim, test_broadcast_shape) { + PrimitivePtr broadcast_shape = std::make_shared("broadcast_shape"); + FuncGraphPtr func_graph = MakeFuncGraph(broadcast_shape, 2); + + auto a = UTPrimUtils::ShapeOf({Shape::SHP_ANY, Shape::SHP_ANY}); + auto b = UTPrimUtils::ShapeOf({Shape::SHP_ANY}); + std::vector expected{Shape::SHP_ANY, Shape::SHP_ANY}; + + AbstractBasePtrList args_spec_list = {a, b}; + + AbstractTuplePtr res = dyn_cast(engine_->Run(func_graph, args_spec_list).inferred); + + auto ret = res->BuildValue()->cast()->value(); + std::vector element_list = {MakeValue(Shape::SHP_ANY), MakeValue(Shape::SHP_ANY)}; + ASSERT_TRUE(ret.size() == element_list.size()); + for (int i = 0; i < element_list.size(); i++) { + ASSERT_TRUE(*ret[i] == *element_list[i]); + } +} + +TEST_F(TestPrim, test_partial) { + PrimitivePtr prim = prim::kPrimPartial; + FuncGraphPtr func_graph = MakeFuncGraph(prim, 3); + + PrimitivePtr add = prim::kPrimScalarAdd; + AbstractBasePtr abstract_add = ToAbstract(add); + AbstractBasePtr abstract_v1 = FromValue(1, false); + AbstractBasePtr abstract_v2 = FromValue(1, false); + AbstractBasePtrList args_spec_list = {abstract_add, abstract_v1, abstract_v2}; + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + AbstractBasePtrList fn_args_list = {abstract_v1, abstract_v2}; + auto expected = std::make_shared( + std::make_shared(prim::kPrimScalarAdd), fn_args_list); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(res->ToString() == expected->ToString()); +} + +// def test_env(x, y): +// return env_setitem(newenv, embed(x), y) +TEST_F(TestPrim, test_env_setitem) { + FuncGraphPtr graph_embed = MakeFuncGraph(prim::kPrimEmbed, 1); + AbstractBasePtr abstract_x = FromValue(1, false); + AbstractBasePtrList args_spec_list = {abstract_x}; + AbstractBasePtr embed_x = engine_->Run(graph_embed, args_spec_list).inferred; + + FuncGraphPtr func_graph = MakeFuncGraph(prim::kPrimEnvSetItem, 3); + + AbstractBasePtr abstract_env = ToAbstract(newenv); + AbstractBasePtr abstract_y = FromValue(2, false); + args_spec_list = {abstract_env, embed_x, abstract_y}; + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + AbstractBasePtr exp = std::make_shared(kAnyValue, std::make_shared()); + ASSERT_TRUE(*res == *exp); +} + +// def test_env(x, y, z): +// e = env_setitem(newenv, embed(x), y) +// return env_getitem(e, embed(x), z) +TEST_F(TestPrim, test_env_getitem) { + FuncGraphPtr graph_embed = MakeFuncGraph(prim::kPrimEmbed, 1); + AbstractBasePtr abstract_x = FromValue(1, false); + AbstractBasePtrList args_spec_list = {abstract_x}; + AbstractBasePtr embed_x = engine_->Run(graph_embed, args_spec_list).inferred; + + FuncGraphPtr graph_setitem = MakeFuncGraph(prim::kPrimEnvSetItem, 3); + + AbstractBasePtr abstract_env = ToAbstract(newenv); + AbstractBasePtr abstract_y = FromValue(2, false); + args_spec_list = {abstract_env, embed_x, abstract_y}; + + AbstractBasePtr res = engine_->Run(graph_setitem, args_spec_list).inferred; + AbstractBasePtr exp = std::make_shared(kAnyValue, std::make_shared()); + ASSERT_TRUE(*res == *exp); + + FuncGraphPtr graph_getitem = MakeFuncGraph(prim::kPrimEnvGetItem, 3); + + AbstractBasePtr abstract_z = FromValue(3, false); + args_spec_list = {res, embed_x, abstract_z}; + + res = engine_->Run(graph_getitem, args_spec_list).inferred; + + ASSERT_TRUE(*res == *abstract_x); +} + +// def test_env(x, y, z): +// e1 = env_setitem(newenv, embed(x), y) +// e2 = env_setitem(newenv, embed(x), z) +// return env_add(e1, e2) +TEST_F(TestPrim, test_env_add) { + FuncGraphPtr graph_embed = MakeFuncGraph(prim::kPrimEmbed, 1); + AbstractBasePtr abstract_x = FromValue(1, false); + AbstractBasePtrList args_spec_list = {abstract_x}; + AbstractBasePtr embed_x = engine_->Run(graph_embed, args_spec_list).inferred; + + FuncGraphPtr graph_setitem = MakeFuncGraph(prim::kPrimEnvSetItem, 3); + + AbstractBasePtr abstract_env = ToAbstract(newenv); + AbstractBasePtr abstract_y = FromValue(2, false); + args_spec_list = {abstract_env, embed_x, abstract_y}; + + AbstractBasePtr abstract_e1 = engine_->Run(graph_setitem, args_spec_list).inferred; + AbstractBasePtr exp = std::make_shared(kAnyValue, std::make_shared()); + ASSERT_TRUE(*abstract_e1 == *exp); + + AbstractBasePtr abstract_z = FromValue(3, false); + args_spec_list = {abstract_env, embed_x, abstract_z}; + + AbstractBasePtr abstract_e2 = engine_->Run(graph_setitem, args_spec_list).inferred; + ASSERT_TRUE(*abstract_e2 == *exp); + + FuncGraphPtr graph_add = MakeFuncGraph(prim::kPrimEnvAdd, 2); + args_spec_list = {abstract_e1, abstract_e2}; + AbstractBasePtr res = engine_->Run(graph_add, args_spec_list).inferred; + + ASSERT_TRUE(*res == *exp); +} + +TEST_F(TestPrim, test_shape) { + PrimitivePtr shap = std::make_shared("Shape"); + FuncGraphPtr func_graph = MakeFuncGraph(shap, 1); + + auto a = UTPrimUtils::ArrayFloat64Of({2, 3}); + + AbstractBasePtrList args_spec_list = {a}; + + AbstractTuplePtr res = dyn_cast(engine_->Run(func_graph, args_spec_list).inferred); + auto ret = res->BuildValue()->cast()->value(); + + std::vector element_list = {MakeValue(2), MakeValue(3)}; + ASSERT_TRUE(ret.size() == element_list.size()); + for (int i = 0; i < element_list.size(); i++) { + ASSERT_TRUE(*ret[i] == *element_list[i]); + } +} + +TEST_F(TestPrim, test_relu) { + PrimitivePtr relu = prim::kPrimRelu; + relu->AddAttr("T", MakeValue(static_cast(kNumberTypeFloat64))); + FuncGraphPtr func_graph = MakeFuncGraph(relu, 1); + + AbstractBasePtr expected = UTPrimUtils::ArrayFloat64Of({2, 2, 2, 3}); // NCHW + AbstractBasePtrList args_spec_list = {expected}; + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_relu2) { + FuncGraphPtr func_graph = getPyFun("get_relu"); + ASSERT_TRUE(func_graph != nullptr); + draw::Draw("test_relu.dot", func_graph); + + auto arr = ArrayOfTensor(UTPrimUtils::kF32, {3, 4, 5}); + auto expected = ArrayOfTensor(UTPrimUtils::kF32, {3, 4, 5}); + + AbstractBasePtrList args_spec_list = {arr}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + ASSERT_TRUE(*(res->GetShapeTrack()) == *(expected->GetShapeTrack())); +} + +TEST_F(TestPrim, test_conv2d1) { + std::shared_ptr env = python_adapter::set_python_scoped(); + py::tuple kernel_size(2); + kernel_size[0] = 5; + kernel_size[1] = 5; + std::shared_ptr func_graph = getPyFun.CallAndParseRet("test_conv2d", 64, kernel_size, 0, 2, 1); + + // NCHW + std::vector inputs_dims = {2, 20, 32, 32}; + std::vector weight_dims = {64, 20, 5, 5}; + + tensor::TensorPtr inputs = std::make_shared(); + inputs->set_data_type(kNumberTypeInt32); + inputs->set_shape(inputs_dims); + // Cout, Cin, kernel_size + tensor::TensorPtr weight = std::make_shared(); + weight->set_data_type(kNumberTypeInt32); + weight->set_shape(weight_dims); + + AbstractBasePtr abstract_inputs = FromValue(inputs, true); + AbstractBasePtr abstract_weight = FromValue(weight, true); + AbstractBasePtrList args_spec_list = {abstract_inputs, abstract_weight}; + + AbstractBasePtr expected = abstract_inputs->Clone(); + // NCHW + std::vector shape = {2, 64, 14, 14}; + expected->set_shape(std::make_shared(shape)); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_ptr = dyn_cast(res); + auto expected_ptr = dyn_cast(expected); + ASSERT_TRUE(*res_ptr->shape() == *expected_ptr->shape()); + ASSERT_TRUE(*res_ptr->element() == *expected_ptr->element()); +} + +TEST_F(TestPrim, test_conv2d) { + FuncGraphPtr func_graph = getPyFun("get_conv2d"); + ASSERT_TRUE(func_graph != nullptr); + + auto input = ArrayOfTensor(UTPrimUtils::kF32, {10, 32, 32, 32}); + auto weight = ArrayOfTensor(UTPrimUtils::kF32, {64, 32, 3, 3}); + + AbstractBasePtrList args_spec_list = {input, weight}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + auto expected = ArrayOfTensor(UTPrimUtils::kF32, {10, 64, 16, 16}); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*(res->GetShapeTrack()) == *(expected->GetShapeTrack())); +} + +TEST_F(TestPrim, test_conv2d_native) { + FuncGraphPtr func_graph = getPyFun("get_conv2d_native"); + ASSERT_TRUE(func_graph != nullptr); + + auto input = ArrayOfTensor(UTPrimUtils::kF64, {10, 32, 32, 32}); + auto weight = ArrayOfTensor(UTPrimUtils::kF64, {3, 32, 3, 3}); + + AbstractBasePtrList args_spec_list = {input, weight}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + auto expected = ArrayOfTensor(UTPrimUtils::kF64, {10, 96, 16, 16}); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*(res->GetShapeTrack()) == *(expected->GetShapeTrack())); +} + +TEST_F(TestPrim, test_biasAdd) { + FuncGraphPtr func_graph = getPyFun("get_bias_add"); + ASSERT_TRUE(func_graph != nullptr); + + auto value = ArrayOfTensor(UTPrimUtils::kF32, {10, 32, 32, 32}); + auto bias = ArrayOfTensor(UTPrimUtils::kF32, {32}); + + AbstractBasePtrList args_spec_list = {value, bias}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + auto expected = ArrayOfTensor(UTPrimUtils::kF32, {10, 32, 32, 32}); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*(res->GetShapeTrack()) == *(expected->GetShapeTrack())); +} + +TEST_F(TestPrim, test_softmax_cross_entropy_with_logits) { + FuncGraphPtr func_graph = getPyFun("get_softmax_cross_entropy_with_logits"); + ASSERT_TRUE(func_graph != nullptr); + + auto logits = ArrayOfTensor(UTPrimUtils::kF32, {64, 10}); + auto labels = ArrayOfTensor(UTPrimUtils::kF32, {64, 10}); + + AbstractBasePtrList args_spec_list = {logits, labels}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_NE(ret, nullptr); + auto res = dyn_cast(ret); + auto loss = ArrayOfTensor(UTPrimUtils::kF32, {64}); + auto dLogits = ArrayOfTensor(UTPrimUtils::kF32, {64, 10}); + AbstractBasePtrList expected_list = {loss, dLogits}; + auto expected = std::make_shared(expected_list); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_ptr0 = dyn_cast(res); + auto expected_ptr0 = dyn_cast(expected); + + ASSERT_GT((*res_ptr0).size(), 1); + auto res_ptr = dyn_cast((*res_ptr0)[1]); + ASSERT_GT((*expected_ptr0).size(), 1); + auto expected_ptr = dyn_cast((*expected_ptr0)[1]); + ASSERT_TRUE(*res_ptr->shape() == *expected_ptr->shape()); + ASSERT_TRUE(*res_ptr->element() == *expected_ptr->element()); +} + +TEST_F(TestPrim, test_tensor_to_scalar_prim) { + FuncGraphPtr func_graph = getPyFun("get_tensor_to_scalar"); + ASSERT_TRUE(func_graph != nullptr); + draw::Draw("get_tensor_to_scalar.dot", func_graph); + + auto logits = ArrayOfTensor(UTPrimUtils::kF64, {64, 10}); + auto labels = ArrayOfTensor(UTPrimUtils::kF64, {64, 10}); + + AbstractBasePtrList args_spec_list = {logits, labels}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + AbstractScalarPtr expected = std::make_shared(kAnyValue, kFloat64); + expected->set_type(UTPrimUtils::kF64); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_fused_batch_norm) { + PrimitivePtr fused_batch_norm = prim::kPrimFusedBatchNorm; + fused_batch_norm->AddAttr("epsilon", MakeValue(0.001f)); + fused_batch_norm->AddAttr("momentum", MakeValue(0.1f)); + + FuncGraphPtr func_graph = MakeFuncGraph(fused_batch_norm, 5); + + // NCHW + std::vector inputs_dims = {128, 64, 32, 64}; + std::vector scale_dims = {64}; + std::vector offset_dims = {64}; + std::vector mean_dims = {64}; + std::vector variance_dims = {64}; + + tensor::TensorPtr inputs = std::make_shared(); + inputs->set_data_type(kNumberTypeFloat32); + inputs->set_shape(inputs_dims); + + tensor::TensorPtr scale = std::make_shared(); + scale->set_data_type(kNumberTypeFloat32); + scale->set_shape(scale_dims); + + tensor::TensorPtr offset = std::make_shared(); + offset->set_data_type(kNumberTypeFloat32); + offset->set_shape(offset_dims); + + tensor::TensorPtr mean = std::make_shared(); + mean->set_data_type(kNumberTypeFloat32); + mean->set_shape(mean_dims); + + tensor::TensorPtr variance = std::make_shared(); + variance->set_data_type(kNumberTypeFloat32); + variance->set_shape(variance_dims); + + AbstractBasePtr abstract_inputs = FromValue(inputs, true); + AbstractBasePtr abstract_scale = FromValue(scale, true); + AbstractBasePtr abstract_offset = FromValue(offset, true); + AbstractBasePtr abstract_mean = FromValue(mean, true); + AbstractBasePtr abstract_variance = FromValue(variance, true); + AbstractBasePtrList args_spec_list = {abstract_inputs, abstract_scale, abstract_offset, abstract_mean, + abstract_variance}; + + AbstractBasePtr expected0 = abstract_inputs->Clone(); + AbstractBasePtr expected1 = abstract_scale->Clone(); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected0: " << expected0->ToString(); + MS_LOG(INFO) << "expected1: " << expected1->ToString(); + + std::shared_ptr abs_tuple = dyn_cast(res); + ASSERT_TRUE(abs_tuple != nullptr); + ASSERT_TRUE(*abs_tuple->elements()[0] == *expected0); + ASSERT_TRUE(*abs_tuple->elements()[1] == *expected1); + ASSERT_TRUE(*abs_tuple->elements()[2] == *expected1); + ASSERT_TRUE(*abs_tuple->elements()[3] == *expected1); + ASSERT_TRUE(*abs_tuple->elements()[4] == *expected1); +} + +TEST_F(TestPrim, test_pooling) { + PrimitivePtr pooling = prim::kPrimPooling; + pooling->AddAttr("mode", MakeValue(std::string("avg"))); + pooling->AddAttr("pad_mode", MakeValue(std::string("valid"))); + pooling->AddAttr("nan_opt", MakeValue(0)); + pooling->AddAttr("window", MakeValue(2)); + pooling->AddAttr("pad", MakeValue(1)); + pooling->AddAttr("stride", MakeValue(1)); + pooling->AddAttr("data_mode", MakeValue(1)); + pooling->AddAttr("ceil_mode", MakeValue(0)); + FuncGraphPtr func_graph = MakeFuncGraph(pooling, 1); + + std::vector inputs_dims = {8, 64, 3, 3}; + auto inputs = std::make_shared(); + inputs->set_data_type(kNumberTypeFloat32); + inputs->set_shape(inputs_dims); + AbstractBasePtr abstract_input = FromValue(inputs, false); + AbstractBasePtrList args_spec_list = {abstract_input}; + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + + AbstractBasePtr expected = abstract_input->Clone()->Broaden(); + std::vector expected_dims = {8, 64, 2, 2}; + expected->set_shape(std::make_shared(expected_dims)); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_hastype) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + TypePtr v2 = std::make_shared(); + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractTypePtr abstract_v2 = UTPrimUtils::TypeToAbstract(v2); + AbstractBasePtr expected = FromValue(true, false); + + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + + auto prim = std::make_shared("hastype"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 2); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_array_len) { + AbstractBasePtrList args_spec_list; + auto v1 = UTPrimUtils::ArrayFloat64Of({3, 4, 0, 2}); + auto expected = std::make_shared(kAnyValue, kInt32); + + args_spec_list.push_back(v1); + + auto prim = std::make_shared("array_len"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 1); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_list_len) { + AbstractBasePtrList args_spec_list; + auto v1 = UTPrimUtils::ListShapeOf({3, 4, 0, 2}); + auto expected = std::make_shared(4); + + args_spec_list.push_back(v1); + + auto prim = std::make_shared("list_len"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 1); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_tuple_len) { + AbstractBasePtrList args_spec_list; + auto v1 = UTPrimUtils::ShapeOf({3, 4, 0, 2}); + auto expected = std::make_shared(4); + + args_spec_list.push_back(v1); + + auto prim = std::make_shared("tuple_len"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 1); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_tuple_reversed) { + AbstractBasePtrList args_spec_list; + auto v1 = UTPrimUtils::ShapeOf({0, 1, 2, 3}); + auto expected = UTPrimUtils::ShapeOf({3, 2, 1, 0}); + + args_spec_list.push_back(v1); + + auto prim = std::make_shared("tuple_reversed"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 1); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "expect=" << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_list_getitem) { + AbstractBasePtrList args_spec_list; + int v1 = 2; + int v2 = 1; + + AbstractBasePtr elem = FromValue(v1, false); + AbstractBasePtr elem2 = FromValue(v2, false); + AbstractBasePtrList elems = {elem, elem}; + auto abstract_v1 = std::make_shared(elems); + AbstractBasePtr abstract_v2 = FromValue(v2, false); + + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + + auto prim = std::make_shared("list_getitem"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 2); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *elem); +} + +TEST_F(TestPrim, test_list_setitem) { + int v1 = 1; + int v2 = 2; + + AbstractBasePtr elem1 = FromValue(v1, false); + AbstractBasePtr elem2 = FromValue(v2, false); + AbstractBasePtrList elems = {elem1, elem1}; + auto abstract_tuple = std::make_shared(elems); + AbstractBasePtr abstract_v2 = FromValue(v1, false); + AbstractBasePtr abstract_v3 = FromValue(v2, false); + AbstractBasePtrList args_spec_list = {abstract_tuple, abstract_v2, abstract_v3}; + + auto prim = std::make_shared("list_setitem"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 3); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + AbstractBasePtrList elems_exp = {elem1, elem2}; + auto expected = std::make_shared(elems_exp); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_list = dyn_cast(res); + ASSERT_TRUE(*expected == *res_list); +} + +TEST_F(TestPrim, test_list_append) { + int v1 = 1; + + AbstractBasePtr elem1 = FromValue(v1, false); + AbstractBasePtr elem2 = FromValue(v1, false); + auto abstract_tuple = std::make_shared(AbstractBasePtrList({elem1, elem2})); + AbstractBasePtr abstract_v2 = FromValue(v1, false); + AbstractBasePtrList args_spec_list = {abstract_tuple, abstract_v2}; + + auto prim = std::make_shared("list_append"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 2); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + auto expected = std::make_shared(AbstractBasePtrList({elem1, elem2})); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_list = dyn_cast(res); + ASSERT_TRUE(*res_list == *expected); +} + +TEST_F(TestPrim, test_tuple_setitem) { + int v1 = 1; + int v2 = 2; + + AbstractBasePtr elem1 = FromValue(v1, false); + AbstractBasePtr elem2 = FromValue(v2, false); + AbstractBasePtrList elems = {elem1, elem1}; + auto abstract_tuple = std::make_shared(elems); + AbstractBasePtr abstract_v2 = FromValue(v1, false); + AbstractBasePtr abstract_v3 = FromValue(v2, false); + AbstractBasePtrList args_spec_list = {abstract_tuple, abstract_v2, abstract_v3}; + + auto prim = std::make_shared("tuple_setitem"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 3); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + AbstractBasePtrList elems_exp = {elem1, elem2}; + auto expected = std::make_shared(elems_exp); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_tuple = dyn_cast(res); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_make_list) { + AbstractBasePtrList args_spec_list; + int v1 = 2; + int v2 = 2; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractBasePtr abstract_v2 = FromValue(v2, false); + + auto expected = std::make_shared(AbstractBasePtrList({abstract_v1, abstract_v2})); + + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + + auto prim = std::make_shared("make_list"); + FuncGraphPtr func_graph = MakeFuncGraph(prim, 2); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_make_range) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + int v2 = 4; + + AbstractBasePtr abstract_v1 = FromValue(v1); + AbstractBasePtr abstract_v2 = FromValue(v2); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + + auto prim = std::make_shared("make_range"); + std::shared_ptr func_graph = MakeFuncGraph(prim, 2); + + AbstractBasePtr ele1 = FromValue(1); + AbstractBasePtr ele2 = FromValue(2); + AbstractBasePtr ele3 = FromValue(3); + AbstractBasePtrList elem_list({ele1, ele2, ele3}); + AbstractBasePtr expected = std::make_shared(elem_list); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "res=" << res->ToString(); + MS_LOG(INFO) << "expected=" << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_layernorm) { + PrimitivePtr layerNorm = prim::kPrimLayerNorm; + layerNorm->AddAttr("begin_norm_axis", MakeValue(1)); + layerNorm->AddAttr("begin_params_axis", MakeValue(1)); + + std::shared_ptr func_graph = MakeFuncGraph(layerNorm, 3); + + std::vector inputs_dims = {128, 64, 32, 64}; + std::vector mean_var_dims = {128, 64, 32, 1}; + std::vector params_dims = {64, 32, 64}; + + tensor::TensorPtr inputs = std::make_shared(); + inputs->set_data_type(kNumberTypeFloat32); + inputs->set_shape(inputs_dims); + + tensor::TensorPtr mean_var = std::make_shared(); + mean_var->set_data_type(kNumberTypeFloat32); + mean_var->set_shape(mean_var_dims); + + tensor::TensorPtr gamma = std::make_shared(); + gamma->set_data_type(kNumberTypeFloat32); + gamma->set_shape(params_dims); + + tensor::TensorPtr beta = std::make_shared(); + beta->set_data_type(kNumberTypeFloat32); + beta->set_shape(params_dims); + + AbstractBasePtr abstract_inputs = FromValue(inputs, true); + AbstractBasePtr abstract_mean_var = FromValue(mean_var, true); + AbstractBasePtr abstract_gamma = FromValue(gamma, true); + AbstractBasePtr abstract_beta = FromValue(beta, true); + AbstractBasePtrList args_spec_list = {abstract_inputs, abstract_gamma, abstract_beta}; + + AbstractBasePtr expected0 = abstract_inputs->Clone(); + AbstractBasePtr expected1 = abstract_mean_var->Clone(); + AbstractBasePtr expected2 = abstract_mean_var->Clone(); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected0: " << expected0->ToString(); + MS_LOG(INFO) << "expected1: " << expected1->ToString(); + MS_LOG(INFO) << "expected2: " << expected2->ToString(); + + std::shared_ptr abs_tuple = dyn_cast(res); + ASSERT_TRUE(abs_tuple != nullptr); + + auto res_ptr0 = dyn_cast(abs_tuple->elements()[0]); + auto expected_ptr0 = dyn_cast(expected0); + ASSERT_TRUE(*res_ptr0->shape() == *expected_ptr0->shape()); + ASSERT_TRUE(*res_ptr0->element() == *expected_ptr0->element()); + + auto res_ptr1 = dyn_cast(abs_tuple->elements()[1]); + auto expected_ptr1 = dyn_cast(expected1); + ASSERT_TRUE(*res_ptr1->shape() == *expected_ptr1->shape()); + ASSERT_TRUE(*res_ptr1->element() == *expected_ptr1->element()); + + auto res_ptr2 = dyn_cast(abs_tuple->elements()[2]); + auto expected_ptr2 = dyn_cast(expected2); + ASSERT_TRUE(*res_ptr2->shape() == *expected_ptr2->shape()); + ASSERT_TRUE(*res_ptr2->element() == *expected_ptr2->element()); +} + +TEST_F(TestPrim, test_DropoutGenMask) { + AbstractBasePtrList args_spec_list; + + auto arg0 = UTPrimUtils::ShapeOf({5, 5, 5, 5}); + + std::vector keep_prob_shape = {}; + tensor::TensorPtr keep_prob = std::make_shared(0.5f); + keep_prob->set_data_type(kNumberTypeFloat32); + keep_prob->set_shape(keep_prob_shape); + AbstractBasePtr abstract_keep_prob = FromValue(keep_prob); + + auto prim = std::make_shared("DropoutGenMask"); + std::shared_ptr func_graph = MakeFuncGraph(prim, 2); + + args_spec_list.push_back(arg0); + args_spec_list.push_back(abstract_keep_prob); + + // should return a tensor with on dimension of 79 elements + AbstractBasePtr expected = std::make_shared(std::make_shared(kAnyValue, kUInt8), + std::make_shared(std::vector{79})); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "res=" << res->ToString(); + MS_LOG(INFO) << "expected=" << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_dropout) { + std::shared_ptr env = python_adapter::set_python_scoped(); + std::shared_ptr func_graph = getPyFun.CallAndParseRet("test_dropout"); + + std::vector inputs_dims = {2, 20, 32, 32}; + + tensor::TensorPtr inputs = std::make_shared(); + inputs->set_data_type(kNumberTypeFloat32); + inputs->set_shape(inputs_dims); + + AbstractBasePtr abstract_inputs = FromValue(inputs, true); + std::vector keep_prob_shape = {}; + tensor::TensorPtr keep_prob = std::make_shared(0.5f); + keep_prob->set_data_type(kNumberTypeFloat32); + keep_prob->set_shape(keep_prob_shape); + AbstractBasePtr abstract_keep_prob = FromValue(keep_prob); + + AbstractBasePtrList args_spec_list = {abstract_inputs, abstract_keep_prob}; + AbstractBasePtr expected = abstract_inputs->Clone(); + + // NCHW + std::vector shape = {2, 20, 32, 32}; + expected->set_shape(std::make_shared(shape)); + + AbstractBasePtr res = engine_->Run(func_graph, args_spec_list).inferred; + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + + auto res_ptr = dyn_cast(res); + auto expected_ptr = dyn_cast(expected); + ASSERT_TRUE(*res_ptr->shape() == *expected_ptr->shape()); + ASSERT_TRUE(*res_ptr->element() == *expected_ptr->element()); +} + +TEST_F(TestPrim, test_BroadcastGradientArgs_01_dim) { + PrimitivePtr broadcatGradientArgs = prim::kPrimBroadcastGradientArgs; + std::shared_ptr func_graph = MakeFuncGraph(broadcatGradientArgs, 2); + + // broadcast shape: x: 8,5,3, y:3 + // output: ((),(0, 1)) + AbstractBasePtrList x_arg_list({abstract::FromValue(8), abstract::FromValue(5), abstract::FromValue(3)}); + AbstractBasePtrList y_arg_list({abstract::FromValue(3)}); + auto x_input = std::make_shared(x_arg_list); + auto y_input = std::make_shared(y_arg_list); + AbstractBasePtrList args_spec_list = {x_input, y_input}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + AbstractBasePtrList x_idx_list; + auto r_x = std::make_shared(x_idx_list); + AbstractBasePtrList y_idx_list({abstract::FromValue(0), abstract::FromValue(1)}); + auto r_y = std::make_shared(y_idx_list); + AbstractBasePtrList elem_list({r_x, r_y}); + auto expected = std::make_shared(elem_list); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_BroadcastGradientArgs_1_dim) { + PrimitivePtr broadcatGradientArgs = prim::kPrimBroadcastGradientArgs; + std::shared_ptr func_graph = MakeFuncGraph(broadcatGradientArgs, 2); + + // broadcast shape: x: 8,1,3, y:8 5 3 + // output: ((1),()) + AbstractBasePtrList x_arg_list({abstract::FromValue(8), abstract::FromValue(1), abstract::FromValue(3)}); + AbstractBasePtrList y_arg_list({abstract::FromValue(8), abstract::FromValue(5), abstract::FromValue(3)}); + auto x_input = std::make_shared(x_arg_list); + auto y_input = std::make_shared(y_arg_list); + AbstractBasePtrList args_spec_list = {x_input, y_input}; + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + auto res = dyn_cast(ret); + AbstractBasePtrList x_idx_list({abstract::FromValue(1)}); + auto r_x = std::make_shared(x_idx_list); + AbstractBasePtrList y_idx_list; + auto r_y = std::make_shared(y_idx_list); + AbstractBasePtrList elem_list({r_x, r_y}); + auto expected = std::make_shared(elem_list); + MS_LOG(INFO) << "result: " << res->ToString(); + MS_LOG(INFO) << "expected: " << expected->ToString(); + ASSERT_TRUE(*res == *expected); +} + +TEST_F(TestPrim, test_DictGetItem) { + PrimitivePtr dictGetItem = prim::kPrimDictGetItem; + std::shared_ptr func_graph = MakeFuncGraph(dictGetItem, 2); + + std::vector> tensor_map = { + {"x", std::make_shared(kNumberTypeInt32, std::vector{2, 3, 4})}, + {"y", std::make_shared(kNumberTypeInt32, std::vector{2, 1, 4})}}; + ValueDictionary value_dict(tensor_map); + AbstractBasePtr array_dict = value_dict.ToAbstract(); + AbstractBasePtr key = abstract::FromValue("x"); + AbstractBasePtrList args_spec_list = {array_dict, key}; + + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + AbstractTensorPtr tensor_ret = dyn_cast(ret); + AbstractTensorPtr expect = dyn_cast(FromValue(tensor_map[0].second)); + + ASSERT_TRUE(*tensor_ret == *expect); +} + +TEST_F(TestPrim, test_DictGetItem2) { + PrimitivePtr dictGetItem = prim::kPrimDictGetItem; + std::shared_ptr func_graph = MakeFuncGraph(dictGetItem, 2); + + AbstractBasePtr arr_x = ArrayOfTensor(UTPrimUtils::kF64, {3, 4, 5}); + AbstractBasePtr arr_y = ArrayOfTensor(UTPrimUtils::kF64, {1, 4, 5}); + AbstractBasePtr arr_z = ArrayOfTensor(UTPrimUtils::kF64, {3, 1, 5}); + std::vector array_map = {{"x", arr_x}, {"y", arr_y}, {"z", arr_z}}; + AbstractDictionaryPtr array_dict = std::make_shared(array_map); + AbstractBasePtr key = abstract::FromValue("x"); + AbstractBasePtrList args_spec_list = {array_dict, key}; + + AbstractBasePtr ret = engine_->Run(func_graph, args_spec_list).inferred; + AbstractTensorPtr tensor_ret = dyn_cast(ret); + AbstractTensorPtr expect = dyn_cast(arr_x); + + ASSERT_TRUE(*tensor_ret == *expect); +} + +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/specialize_test.cc b/tests/ut/cpp/pipeline/static_analysis/specialize_test.cc new file mode 100644 index 00000000000..23ea55f8f70 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/specialize_test.cc @@ -0,0 +1,231 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/manager.h" +#include "pipeline/static_analysis/prim.h" +#include "pipeline/static_analysis/program_specialize.h" +#include "pipeline/static_analysis/helper.h" +#include "utils/log_adapter.h" +#include "utils/graph_utils.h" +#include "utils/misc.h" +#include "debug/draw.h" + +namespace mindspore { +namespace abstract { +class TestSpecializeGraph : public UT::Common { + public: + void SetUp(); + void TearDown(); + // f(x) call g(x) + FuncGraphPtr graph_f_; + FuncGraphPtr graph_g_; + // alpha(x) return beta(x) closure; + FuncGraphPtr graph_alpha_; + FuncGraphPtr graph_beta_; + std::shared_ptr engine_; + std::shared_ptr special_; +}; + +void TestSpecializeGraph::SetUp() { + UT::InitPythonPath(); + // init resource + engine_ = SetupAnalysisEngine(); + + special_ = std::make_shared(engine_); + + /* + * def g(y): + * return y; + */ + graph_g_ = std::make_shared(); + ParameterPtr y = graph_g_->add_parameter(); + auto prim_return = std::make_shared("return"); + std::vector inputs; + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(y); + CNodePtr cnode_g_ret = graph_g_->NewCNode(inputs); + graph_g_->set_return(cnode_g_ret); + + /* + * def f(x): + * return g(x) + */ + graph_f_ = std::make_shared(); + ParameterPtr x = graph_f_->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(graph_g_)); + inputs.push_back(x); + CNodePtr cnode_f = graph_f_->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_f); + CNodePtr cnode_f_ret = graph_f_->NewCNode(inputs); + graph_f_->set_return(cnode_f_ret); + + /* build a closure func_graph */ + /* + *def alpha(x, y): + * def beta(x1): + * return x1 + y + * return beta(x) + */ + graph_alpha_ = std::make_shared(); + graph_beta_ = std::make_shared(); + x = graph_alpha_->add_parameter(); + y = graph_alpha_->add_parameter(); + + // build func_graph beta + ParameterPtr x1 = graph_beta_->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(std::make_shared("scalar_add"))); + inputs.push_back(x1); + inputs.push_back(y); + CNodePtr cnode_add = graph_beta_->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(std::make_shared("return"))); + inputs.push_back(cnode_add); + CNodePtr cnode_return = graph_beta_->NewCNode(inputs); + graph_beta_->set_return(cnode_return); + + // build func_graph alpha + inputs.clear(); + inputs.push_back(NewValueNode(graph_beta_)); + inputs.push_back(x); + CNodePtr cnode_graph_beta_ = graph_alpha_->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_graph_beta_); + cnode_return = graph_alpha_->NewCNode(inputs); + graph_alpha_->set_return(cnode_return); +} + +void TestSpecializeGraph::TearDown() {} + +TEST_F(TestSpecializeGraph, test_specialize) { + AbstractBasePtrList args_spec_list; + MS_LOG(INFO) << "Begin TestSpecializeGraph call other graph."; + MS_LOG(INFO) << "" << graph_f_->get_return()->ToString(); + AbstractBasePtr abstract_v1 = FromValue(1, false); + args_spec_list.push_back(abstract_v1); + + AnalysisResult result = engine_->Run(graph_f_, args_spec_list); + FuncGraphPtr new_graph = special_->Run(graph_f_, result.context); +} + +TEST_F(TestSpecializeGraph, test_specialize1) { + AbstractBasePtrList args_spec_list; + AbstractBasePtr abstract_v1 = FromValue(1, true); + AbstractBasePtr abstract_v2 = FromValue(2, true); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + AnalysisResult result = engine_->Run(graph_alpha_, args_spec_list); + draw::Draw("befor_graph_alpha.dot", graph_alpha_); + FuncGraphPtr new_graph = special_->Run(graph_alpha_, result.context); + if (new_graph) { + draw::Draw("after_graph_alpha.dot", new_graph); + } +} + +class TestSpecializeMetaFuncGraph : public UT::Common { + public: + void SetUp(); + void TearDown(); + FuncGraphPtr graph_; + std::shared_ptr engine_; + std::shared_ptr special_; +}; + +class MetaScalarAdd : public MetaFuncGraph { + public: + explicit MetaScalarAdd(std::string name) : MetaFuncGraph(name) {} + + ~MetaScalarAdd() {} + /* + * Generate a Graph for the given abstract arguments. + */ + FuncGraphPtr GenerateFromTypes(const TypePtrList& types) override { + FuncGraphPtr graph_g = std::make_shared(); + ParameterPtr x = graph_g->add_parameter(); + ParameterPtr y = graph_g->add_parameter(); + auto prim_scalar_add = std::make_shared("scalar_add"); + std::vector inputs; + inputs.push_back(NewValueNode(prim_scalar_add)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = graph_g->NewCNode(inputs); + auto prim_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = graph_g->NewCNode(inputs); + graph_g->set_return(cnode_return); + return graph_g; + } +}; + +void TestSpecializeMetaFuncGraph::SetUp() { + UT::InitPythonPath(); + // init resource + engine_ = SetupAnalysisEngine(); + special_ = std::make_shared(engine_); + + /* + * def f(x, y): + * return mata_scalar_add(x, y) + */ + graph_ = std::make_shared(); + ParameterPtr x = graph_->add_parameter(); + ParameterPtr y = graph_->add_parameter(); + std::shared_ptr meta_scalar_add = std::make_shared("meta_scalar_add"); + std::vector inputs; + inputs.push_back(NewValueNode(meta_scalar_add)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = graph_->NewCNode(inputs); + auto prim_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = graph_->NewCNode(inputs); + graph_->set_return(cnode_return); +} + +void TestSpecializeMetaFuncGraph::TearDown() {} + +TEST_F(TestSpecializeMetaFuncGraph, test_specialize) { + AbstractBasePtrList args_spec_list; + std::cout << graph_->get_return()->ToString() << std::endl; + AbstractBasePtr abstract_v1 = FromValue(1, true); + AbstractBasePtr abstract_v2 = FromValue(2, true); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + AnalysisResult result = engine_->Run(graph_, args_spec_list); + + draw::Draw("befor_graph.dot", graph_); + FuncGraphPtr new_graph = special_->Run(graph_, result.context); + if (new_graph) { + draw::Draw("after_graph.dot", new_graph); + } +} + +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/static_analysis_test.cc b/tests/ut/cpp/pipeline/static_analysis/static_analysis_test.cc new file mode 100644 index 00000000000..2da631d7443 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/static_analysis_test.cc @@ -0,0 +1,501 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "pipeline/static_analysis/prim.h" +#include "pipeline/static_analysis/helper.h" +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "ir/manager.h" +#include "ir/meta_tensor.h" +#include "operator/ops.h" +#include "pipeline/parse/parse.h" +#include "pipeline/parse/data_converter.h" +#include "pipeline/resource.h" +#include "debug/draw.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace abstract { +namespace python_adapter = mindspore::parse::python_adapter; +namespace { + +AbstractBasePtr InferImplScalarAddStub(const AnalysisEnginePtr &engine, const PrimitivePtr &, + const AbstractBasePtrList &args_spec_list) { + if (args_spec_list.size() != 2) { + // assert; + return nullptr; + } + AbstractBasePtr abs_base = args_spec_list[0]; + AbstractBasePtr abs_base_1 = args_spec_list[1]; + return abs_base; +} + +EvaluatorPtr InitPrimitiveScalarAddEvaluatorStub() { + EvaluatorPtr PrimitiveScalarAddEvaluator = + std::make_shared(prim::kPrimScalarAdd, InferImplScalarAddStub); + return PrimitiveScalarAddEvaluator; +} + +AbstractBasePtr InferImplReturnStub(const AnalysisEnginePtr &engine, const PrimitivePtr &prim, + const AbstractBasePtrList &args_spec_list) { + if (args_spec_list.size() != 1) { + // assert; + return nullptr; + } + AbstractBasePtr abs_base = args_spec_list[0]; + return abs_base; +} + +EvaluatorPtr InitPrimitiveReturnEvaluatorStub() { + EvaluatorPtr PrimitiveReturnEvaluator = + std::make_shared(prim::kPrimReturn, InferImplReturnStub); + return PrimitiveReturnEvaluator; +} + +PrimEvaluatorMap PrimEvaluatorConstructorsStub; + +/* These stubs is a stub for ut test cases which don't rely on prim module */ +void InitPrimEvaluatorConstructorsStub() { + PrimEvaluatorConstructorsStub[prim::kPrimReturn] = InitPrimitiveReturnEvaluatorStub(); + PrimEvaluatorConstructorsStub[prim::kPrimScalarAdd] = InitPrimitiveScalarAddEvaluatorStub(); +} + +/* stub for test which don't rely on prim module */ +AnalysisEnginePtr SetupAnalysisEngineStub() { + // init resource + InitPrimEvaluatorConstructorsStub(); + std::shared_ptr graph_manager = MakeManager(); + AnalysisEnginePtr engine = std::make_shared(PrimEvaluatorConstructorsStub, graph_manager); + return engine; +} + +class MetaScalarAdd : public MetaFuncGraph { + public: + explicit MetaScalarAdd(std::string name) : MetaFuncGraph(name) {} + + ~MetaScalarAdd() {} + /* + * Generate a Graph for the given abstract arguments. + */ + FuncGraphPtr GenerateFromTypes(const TypePtrList &types) override { + FuncGraphPtr fg = std::make_shared(); + ParameterPtr x = fg->add_parameter(); + ParameterPtr y = fg->add_parameter(); + auto prim_scalar_add = std::make_shared("scalar_add"); + std::vector inputs; + inputs.push_back(NewValueNode(prim_scalar_add)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = fg->NewCNode(inputs); + auto prim_return = prim::kPrimReturn; + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = fg->NewCNode(inputs); + fg->set_return(cnode_return); + return fg; + } +}; + +} // namespace + +class TestInfer : public UT::Common { + public: + void SetUp(); + void TearDown(); + AnalysisEnginePtr engine_; +}; + +void TestInfer::SetUp() { engine_ = SetupAnalysisEngineStub(); } + +static FuncGraphPtr MakeFuncGraph(PrimitivePtr prim) { + // build the func_graph manually. + /* python source code: + * @mindspore + * def f(x, y): + * return x + y + * print(f(1,2)) + */ + FuncGraphPtr func_graph = std::make_shared(); + ParameterPtr x = func_graph->add_parameter(); + ParameterPtr y = func_graph->add_parameter(); + std::vector inputs; + inputs.push_back(NewValueNode(prim)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + return func_graph; +} + +void TestInfer::TearDown() { + // destroy resource +} + +TEST_F(TestInfer, test_inferred_scalar_add) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + int v2 = 2; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractBasePtr abstract_v2 = FromValue(v2, false); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + + auto prim_scalar_add = std::make_shared("scalar_add"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_scalar_add); + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec_list).inferred; + ASSERT_TRUE(abs_base_got.get() == abstract_v1.get()); +} + +class TestInferGraph : public UT::Common { + public: + void SetUp(); + void TearDown(); + // f(x) call g(x) + FuncGraphPtr graph_f_; + FuncGraphPtr graph_g_; + // alpha(x) return beta(x) closure; + FuncGraphPtr graph_alpha_; + FuncGraphPtr graph_beta_; + AnalysisEnginePtr engine_; +}; + +void TestInferGraph::SetUp() { + // init resource + engine_ = SetupAnalysisEngineStub(); + + /* + * def g(y): + * return y; + */ + graph_g_ = std::make_shared(); + ParameterPtr y = graph_g_->add_parameter(); + auto prim_return = prim::kPrimReturn; + std::vector inputs; + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(y); + CNodePtr cnode_g_ret = graph_g_->NewCNode(inputs); + graph_g_->set_return(cnode_g_ret); + + /* + * def f(x): + * return g(x) + */ + graph_f_ = std::make_shared(); + ParameterPtr x = graph_f_->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(graph_g_)); + inputs.push_back(x); + CNodePtr cnode_f = graph_f_->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_f); + CNodePtr cnode_f_ret = graph_f_->NewCNode(inputs); + graph_f_->set_return(cnode_f_ret); + + /* build a closure func_graph */ + /* + *def alpha(x, y): + * def beta(x1): + * return x1 + y + * return beta(x) + */ + graph_alpha_ = std::make_shared(); + graph_beta_ = std::make_shared(); + x = graph_alpha_->add_parameter(); + y = graph_alpha_->add_parameter(); + + // build func_graph beta + ParameterPtr x1 = graph_beta_->add_parameter(); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimScalarAdd)); + inputs.push_back(x1); + inputs.push_back(y); + CNodePtr cnode_add = graph_beta_->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimReturn)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = graph_beta_->NewCNode(inputs); + graph_beta_->set_return(cnode_return); + + // build func_graph alpha + inputs.clear(); + inputs.push_back(NewValueNode(graph_beta_)); + inputs.push_back(x); + CNodePtr cnode_graph_beta_ = graph_alpha_->NewCNode(inputs); + + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_graph_beta_); + cnode_return = graph_alpha_->NewCNode(inputs); + graph_alpha_->set_return(cnode_return); +} + +void TestInferGraph::TearDown() { + // destroy resource +} + +TEST_F(TestInferGraph, test_inferred) { + AbstractBasePtrList args_spec_list; + MS_LOG(INFO) << "Begin TestInferGraph call other graph."; + MS_LOG(INFO) << "" << graph_f_->get_return()->ToString(); + AbstractBasePtr abstract_v1 = FromValue(1, false); + args_spec_list.push_back(abstract_v1); + AbstractBasePtr abs_base_got = engine_->Run(graph_f_, args_spec_list).inferred; + ASSERT_TRUE(abs_base_got.get() == abstract_v1.get()); + + // now this test case failed randomly, have to debug. + MS_LOG(INFO) << "Begin TestInferGraph closure."; + MS_LOG(INFO) << "" << graph_alpha_->get_return()->ToString(); + + AbstractBasePtr abstract_v2 = FromValue(2, false); + args_spec_list.clear(); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + abs_base_got = engine_->Run(graph_alpha_, args_spec_list).inferred; + ASSERT_TRUE(abs_base_got.get() == abstract_v1.get()); +} + +TEST_F(TestInferGraph, test_context) { + std::shared_ptr graph_manager_ = MakeManager(); + + graph_manager_->AddFuncGraph(graph_f_); + graph_manager_->AddFuncGraph(graph_g_); + + graph_manager_->AddFuncGraph(graph_alpha_); + graph_manager_->AddFuncGraph(graph_beta_); + + ASSERT_TRUE(graph_f_->parent() == nullptr); + ASSERT_TRUE(graph_g_->parent() == nullptr); + + ASSERT_TRUE(graph_beta_->parent() == graph_alpha_); + ASSERT_TRUE(graph_alpha_->parent() == nullptr); + + AnalysisContextPtr dummy_context = AnalysisContext::DummyContext(); + + AnalysisContextPtr f_context = dummy_context->NewFuncGraphContext(graph_f_, AbstractBasePtrList()); + ASSERT_TRUE(f_context->Filter(graph_f_) = f_context); + ASSERT_TRUE(f_context->Filter(nullptr) = dummy_context); + + AnalysisContextPtr g_context = f_context->NewFuncGraphContext(graph_g_, AbstractBasePtrList()); + ASSERT_TRUE(g_context->Filter(graph_g_) = g_context); + ASSERT_TRUE(g_context->Filter(graph_f_) = dummy_context); + ASSERT_TRUE(g_context->Filter(nullptr) = dummy_context); + + AnalysisContextPtr alpha_context = dummy_context->NewFuncGraphContext(graph_alpha_, AbstractBasePtrList()); + ASSERT_TRUE(alpha_context->Filter(graph_alpha_) = alpha_context); + ASSERT_TRUE(alpha_context->Filter(nullptr) = dummy_context); + + AnalysisContextPtr beta_context = alpha_context->NewFuncGraphContext(graph_beta_, AbstractBasePtrList()); + ASSERT_TRUE(beta_context->Filter(graph_beta_) = beta_context); + ASSERT_TRUE(beta_context->Filter(graph_alpha_) = alpha_context); + ASSERT_TRUE(beta_context->Filter(nullptr) = dummy_context); +} + +class TestInferMetaGraph : public UT::Common { + public: + void SetUp(); + void TearDown(); + FuncGraphPtr func_graph_; + AnalysisEnginePtr engine_; +}; + +void TestInferMetaGraph::SetUp() { + // init resource + engine_ = SetupAnalysisEngineStub(); + + /* + * def f(x, y): + * return mata_scalar_add(x, y) + */ + func_graph_ = std::make_shared(); + ParameterPtr x = func_graph_->add_parameter(); + ParameterPtr y = func_graph_->add_parameter(); + std::shared_ptr meta_scalar_add = std::make_shared("meta_scalar_add"); + std::vector inputs; + inputs.push_back(NewValueNode(meta_scalar_add)); + inputs.push_back(x); + inputs.push_back(y); + CNodePtr cnode_add = func_graph_->NewCNode(inputs); + auto prim_return = prim::kPrimReturn; + inputs.clear(); + inputs.push_back(NewValueNode(prim_return)); + inputs.push_back(cnode_add); + CNodePtr cnode_return = func_graph_->NewCNode(inputs); + func_graph_->set_return(cnode_return); +} + +void TestInferMetaGraph::TearDown() { + // destroy resource +} + +TEST_F(TestInferMetaGraph, test_inferred) { + AbstractBasePtrList args_spec_list; + int v1 = 1; + std::cout << "Begin TestInferGraph." << std::endl; + std::cout << func_graph_->get_return()->ToString() << std::endl; + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractBasePtr abstract_v2 = FromValue(v1, false); + args_spec_list.push_back(abstract_v1); + args_spec_list.push_back(abstract_v2); + AbstractBasePtr abs_base_got = engine_->Run(func_graph_, args_spec_list).inferred; + ASSERT_TRUE(abs_base_got.get() == abstract_v1.get()); +} + +class TestInferUniform : public UT::Common { + public: + void SetUp(); + void TearDown(); + AnalysisEnginePtr engine_; +}; + +void TestInferUniform::SetUp() { + // init resource + engine_ = SetupAnalysisEngine(); +} + +void TestInferUniform::TearDown() { + // destroy resource +} + +TEST_F(TestInferUniform, test_inferred_scalar_add) { + AbstractBasePtrList args_spec; + int v1 = 1; + int v2 = 2; + + AbstractBasePtr abstract_v1 = FromValue(v1, false); + AbstractBasePtr abstract_v2 = FromValue(v2, false); + args_spec.push_back(abstract_v1); + args_spec.push_back(abstract_v2); + + auto prim_scalar_add = std::make_shared("scalar_add"); + FuncGraphPtr func_graph = MakeFuncGraph(prim_scalar_add); + AbstractBasePtr abs_base_got = engine_->Run(func_graph, args_spec).inferred; + ASSERT_TRUE(*(abs_base_got->GetTypeTrack()) == *(abstract_v1->GetTypeTrack())); + ASSERT_TRUE(abs_base_got->GetTypeTrack()->type_id() == kNumberTypeInt32); +} + + +class TestInferOnePrim : public UT::Common { + public: + TestInferOnePrim() : getPyFun("gtest_input.pipeline.infer.infer_test", true), engine_(nullptr) {} + void SetUp(); + void TearDown(); + + UT::PyFuncGraphFetcher getPyFun; + AnalysisEnginePtr engine_; +}; + +void TestInferOnePrim::SetUp() { engine_ = SetupAnalysisEngineStub(); } + +void TestInferOnePrim::TearDown() { + // destroy resource +} +TEST_F(TestInferOnePrim, test_scalar_add) { + double x1 = 1.1; + double x2 = 1.1; + double x3 = 2.2; + AbstractBasePtr base1 = FromValue(x1, false); + AbstractBasePtr base2 = FromValue(x2, false); + AbstractBasePtrList base_list = {base1, base2}; + auto res = InferOnePrim(std::make_shared("scalar_add"), base_list); + MS_LOG(INFO) << "result spec: " << res->ToString(); + AbstractBasePtr exp = FromValue(x3, false); + MS_LOG(INFO) << "result exp: " << exp->ToString(); + ASSERT_EQ(*res, *exp); +} + +class TestGraphInfer : public UT::Common { + public: + TestGraphInfer() : getPyFun("gtest_input.pipeline.infer.infer_test", true){}; + void SetUp(); + void TearDown(); + AnalysisEnginePtr engine_; + UT::PyFuncGraphFetcher getPyFun; +}; + +void TestGraphInfer::SetUp() { engine_ = SetupAnalysisEngine(); } + +void TestGraphInfer::TearDown() { + // destroy resource + engine_->ClearEvaluatorCache(); + parse::data_converter::ClearObjectCache(); +} + +TEST_F(TestGraphInfer, test_graph_infer_defaults) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_defaults"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(50), false); + ASSERT_EQ(*res, *expect); +} + +TEST_F(TestGraphInfer, test_graph_infer_vararg_0) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_vararg_0"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(1), false); + ASSERT_EQ(*res, *expect); +} + +TEST_F(TestGraphInfer, test_graph_infer_vararg) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_vararg"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(9), false); + ASSERT_EQ(*res, *expect); +} + +TEST_F(TestGraphInfer, test_graph_infer_vararg_kwonlyargs) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_vararg_kwonlyargs"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(48), false); + ASSERT_EQ(*res, *expect); +} + +TEST_F(TestGraphInfer, test_graph_infer_kwarg) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_kwarg"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(7), false); + ASSERT_EQ(*res, *expect); +} + +TEST_F(TestGraphInfer, test_graph_infer_vararg_kwonlyargs_kwarg) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_vararg_kwonlyargs_kwarg"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(46), false); + ASSERT_EQ(*res, *expect); +} + +TEST_F(TestGraphInfer, test_graph_infer_vararg_kwonlyargs_kwarg_defaults) { + FuncGraphPtr graph = getPyFun.CallAndParseRet("test_graph_infer_vararg_kwonlyargs_kwarg_defaults"); + AbstractBasePtrList args_spec_list = {}; + AbstractBasePtr res = engine_->Run(graph, args_spec_list).inferred; + AbstractBasePtr expect = FromValue(MakeValue(57), false); + ASSERT_EQ(*res, *expect); +} +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pipeline/static_analysis/utils_test.cc b/tests/ut/cpp/pipeline/static_analysis/utils_test.cc new file mode 100644 index 00000000000..dceef71b024 --- /dev/null +++ b/tests/ut/cpp/pipeline/static_analysis/utils_test.cc @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pipeline/static_analysis/utils.h" + +#include "common/common_test.h" +#include "pipeline/static_analysis/static_analysis.h" + +namespace mindspore { +namespace abstract { +class TestUtils : public UT::Common { + public: + TestUtils() {} + virtual void SetUp() {} + virtual void TearDown() {} +}; + +TEST_F(TestUtils, test_join) { + // AbstractScalar + AbstractBasePtr abs_s1 = FromValue(1, false); + AbstractBasePtr abs_s2 = FromValue(2, false); + AbstractBasePtr abs_s_anything = FromValue(2, true); + + AbstractBasePtr res_s1 = abs_s1->Join(abs_s2); + ASSERT_EQ(*res_s1, *abs_s_anything); + + // AbstractTuple join; + std::vector list1 = {1, 2, 3, 4, 5}; + std::vector list2 = {5, 4, 3, 2, 1}; + AbstractBasePtr abs_t1 = FromValue(list1, true); + AbstractBasePtr abs_t2 = FromValue(list2, true); + + AbstractBasePtr res_t1 = abs_t1->Join(abs_t2); + ASSERT_EQ(res_t1, abs_t1); + + abs_s1 = FromValue(1, false); + + AbstractBasePtr t1 = std::make_shared(AbstractBasePtrList({abs_s1, abs_s_anything})); + AbstractBasePtr t2 = std::make_shared(AbstractBasePtrList({abs_s1, abs_s_anything})); + AbstractBasePtr t3 = std::make_shared(AbstractBasePtrList({abs_s_anything, abs_s_anything})); + + res_t1 = t1->Join(t2); + ASSERT_EQ(res_t1, t1); + + res_t1 = t1->Join(t3); + ASSERT_EQ(*res_t1, *t3); + + res_t1 = t3->Join(t1); + ASSERT_EQ(res_t1, t3); +} + +} // namespace abstract +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/buffer_fusion/buffer_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/buffer_fusion/buffer_fusion_test.cc new file mode 100644 index 00000000000..9807344139e --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/buffer_fusion/buffer_fusion_test.cc @@ -0,0 +1,1298 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/anf.h" +#include "ir/func_graph_cloner.h" +#include "utils/context/ms_context.h" +#include "debug/draw.h" +#include "debug/anf_ir_dump.h" +#include "operator/ops.h" +#include "utils/utils.h" +#include "kernel/tbe/tbe_kernel_mod.h" +#include "session/kernel_graph.h" +#include "device/kernel_info.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/pattern_engine.h" +#define private public +#include "pre_activate/ascend/buffer_fusion/buffer_fusion.h" + +namespace mindspore { +namespace opt { +using Primitive = mindspore::Primitive; +using session::KernelGraph; +using KernelGraphPtr = std::shared_ptr; +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +class TestHWBufferFusion : public UT::Common { + public: + TestHWBufferFusion() : getPyFun_("gtest_input.pre_activate.hw_opt_test", true) {} + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +static KernelGraphPtr CreateKernelGraphForBufferFusionMultipleIn( + uint32_t after_layers, mindspore::kernel::FusionType fusiontype = mindspore::kernel::CONVLUTION) { + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + + std::vector shp = {1, 3, 3, 4}; + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + + uint32_t layerscount = 1; + CNodePtr ptr_formerlayer; + std::string name = ""; + + // Construct first node + tensor::TensorPtr y_tensor = std::make_shared(kFloat32->type_id(), shp); + y_tensor->set_device_info(device_info); + tensor::TensorPtr z_tensor = std::make_shared(kFloat32->type_id(), shp); + z_tensor->set_device_info(device_info); + + auto y_const = NewValueNode(y_tensor); + auto z_const = NewValueNode(z_tensor); + y_const->set_abstract(y_tensor->ToAbstract()); + z_const->set_abstract(z_tensor->ToAbstract()); + g->MutableInputs()->push_back(y_const); + g->MutableInputs()->push_back(z_const); + + auto p_conv = std::make_shared("Conv2D"); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(y_const); + inputs.push_back(z_const); + name = "test_conv_" + std::to_string(layerscount) + "layers_graph.dot"; + + auto kernelptr_first = g->NewCNode(inputs); + kernelptr_first->set_abstract(y_tensor->ToAbstract()); + kernelptr_first->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(fusiontype); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_first.get()); + ptr_formerlayer = kernelptr_first; + + // configure fusion successor layers + int layer_idx = 0; + while (after_layers--) { + auto p_relu = std::make_shared("ReLU6"); + if (layer_idx == 0) { + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + x_tensor->set_device_info(device_info); + + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_tensor->ToAbstract()); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + inputs.push_back(x_const); + } else { + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + } + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(y_tensor->ToAbstract()); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + if (layer_idx == 0) { + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + } else { + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + } + + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + ptr_formerlayer = kernelptr_floor; + layerscount++; + layer_idx++; + } + + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(ptr_formerlayer); + auto ret = g->NewCNode(inputs); + ret->set_abstract(y_tensor->ToAbstract()); + + g->set_return(ret); + + draw::Draw(name, g); + + return g; +} + +static KernelGraphPtr CreateKernelGraphForBufferFusionEltwiseBeforeAndAfter( + uint32_t before_layers, uint32_t after_layers = 3, + mindspore::kernel::FusionType fusiontype = mindspore::kernel::SEGMENT) { + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + + std::vector shp = {1, 3, 3, 4}; + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + + uint32_t layerscount = 1; + CNodePtr ptr_formerlayer; + std::string name = ""; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + auto x_abstract = x_tensor->ToAbstract(); + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_abstract); + g->MutableInputs()->push_back(x_const); + + while (before_layers--) { + auto p_relu = std::make_shared("ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + if (layerscount == 1) { + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(x_const); + } else { + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + } + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(x_abstract); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + ptr_formerlayer = kernelptr_floor; + layerscount++; + } + + // Construct the conv2d node + tensor::TensorPtr y_tensor = std::make_shared(kFloat32->type_id(), shp); + y_tensor->set_device_info(device_info); + auto y_const = NewValueNode(y_tensor); + y_const->set_abstract(y_tensor->ToAbstract()); + + if (fusiontype == kernel::FusionType::CONVLUTION) { + auto p_conv = std::make_shared("Conv2D"); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(y_const); + inputs.push_back(ptr_formerlayer); + name = "test_conv_" + std::to_string(layerscount) + "layers_graph.dot"; + } else { + auto p_red_seg = std::make_shared("ReduceOrSegment"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_red_seg->set_attr("input_names", input_names_v); + p_red_seg->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_red_seg)); + inputs.push_back(ptr_formerlayer); + name = "test_regOrSeg_" + std::to_string(layerscount) + "layers_graph.dot"; + } + + auto kernelptr_first = g->NewCNode(inputs); + kernelptr_first->set_abstract(y_tensor->ToAbstract()); + kernelptr_first->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + if (fusiontype == kernel::FusionType::CONVLUTION) { + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + } else { + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + } + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(fusiontype); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_first.get()); + ptr_formerlayer = kernelptr_first; + + // configure fusion successor layers + while (after_layers--) { + auto p_relu = std::make_shared("ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(y_tensor->ToAbstract()); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + ptr_formerlayer = kernelptr_floor; + layerscount++; + } + + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(ptr_formerlayer); + auto ret = g->NewCNode(inputs); + ret->set_abstract(y_tensor->ToAbstract()); + g->set_return(ret); + draw::Draw(name, g); + return g; +} + +static KernelGraphPtr CreateKernelGraphForBufferFusionSingleIn( + uint32_t after_layers, mindspore::kernel::FusionType fusiontype = mindspore::kernel::CONVLUTION) { + // build the func_graph manually, eg: + /* CreateKernelGraphForBufferFusionSingleIn(1) + * @mindspore + * def f(x): + * z=conv2d(x, y) + * ret=relu(z) + * return ret + */ + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + + std::vector shp = {1, 3, 3, 4}; + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + + uint32_t layerscount = 1; + CNodePtr ptr_formerlayer; + std::string name = ""; + + // Construct first node + tensor::TensorPtr y_tensor = std::make_shared(kFloat32->type_id(), shp); + y_tensor->set_device_info(device_info); + tensor::TensorPtr z_tensor = std::make_shared(kFloat32->type_id(), shp); + z_tensor->set_device_info(device_info); + + auto y_const = NewValueNode(y_tensor); + auto z_const = NewValueNode(z_tensor); + y_const->set_abstract(y_tensor->ToAbstract()); + z_const->set_abstract(z_tensor->ToAbstract()); + g->MutableInputs()->push_back(y_const); + g->MutableInputs()->push_back(z_const); + + if (fusiontype == kernel::FusionType::CONVLUTION) { + auto p_conv = std::make_shared("Conv2D"); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(y_const); + inputs.push_back(z_const); + name = "test_conv_" + std::to_string(layerscount) + "layers_graph.dot"; + } else { + auto p_red_seg = std::make_shared("ReduceOrSegment"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_red_seg->set_attr("input_names", input_names_v); + p_red_seg->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_red_seg)); + inputs.push_back(y_const); + name = "test_regOrSeg_" + std::to_string(layerscount) + "layers_graph.dot"; + } + + auto kernelptr_first = g->NewCNode(inputs); + kernelptr_first->set_abstract(y_tensor->ToAbstract()); + kernelptr_first->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + if (fusiontype == kernel::FusionType::CONVLUTION) { + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + } else { + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + } + + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(fusiontype); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_first.get()); + ptr_formerlayer = kernelptr_first; + + // configure fusion successor layers + while (after_layers--) { + auto p_relu = std::make_shared("ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(y_tensor->ToAbstract()); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + ptr_formerlayer = kernelptr_floor; + layerscount++; + } + + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(ptr_formerlayer); + auto ret = g->NewCNode(inputs); + ret->set_abstract(y_tensor->ToAbstract()); + + g->set_return(ret); + + draw::Draw(name, g); + + return g; +} + +static KernelGraphPtr CreateKernelGraphForBufferFusion( + uint32_t targetlayers, bool conv_flag = false, + mindspore::kernel::FusionType fusiontype = mindspore::kernel::CONVLUTION) { + // build the func_graph manually, eg: + /* CreateKernelGraphForBufferFusion(3) + * @mindspore + * def f(x): + * y=relu(x) + * z=relu(y) + * ret=relu(z) + * return ret + */ + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + // x is input tensor. + std::vector shp = {1, 3, 3, 4}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + x_tensor->set_device_info(device_info); + + auto x_abstract = x_tensor->ToAbstract(); + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_abstract); + g->MutableInputs()->push_back(x_const); + + uint32_t layerscount = 1; + CNodePtr ptr_formerlayer; + // configure func_graph hiden layers + while (targetlayers--) { + auto p_relu = std::make_shared("ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + if (layerscount == 1) { + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(x_const); + } else { + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + } + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(x_abstract); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + ptr_formerlayer = kernelptr_floor; + layerscount++; + } + std::string name = "test_construct_" + std::to_string(layerscount) + "layers_graph.dot"; + if (conv_flag) { + tensor::TensorPtr y_tensor = std::make_shared(kFloat32->type_id(), shp); + y_tensor->set_device_info(device_info); + tensor::TensorPtr z_tensor = std::make_shared(kFloat32->type_id(), shp); + z_tensor->set_device_info(device_info); + auto y_const = NewValueNode(y_tensor); + auto z_const = NewValueNode(y_tensor); + + y_const->set_abstract(y_tensor->ToAbstract()); + z_const->set_abstract(z_tensor->ToAbstract()); + + g->MutableInputs()->push_back(y_const); + + if (fusiontype == kernel::FusionType::CONVLUTION) { + auto p_conv = std::make_shared("Conv2D"); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(y_const); + inputs.push_back(ptr_formerlayer); + } else { + auto p_conv = std::make_shared("ReduceOrSegment"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(ptr_formerlayer); + } + + auto kernelptr_conv = g->NewCNode(inputs); + kernelptr_conv->set_abstract(x_abstract); + kernelptr_conv->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + if (fusiontype == kernel::FusionType::CONVLUTION) { + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + } else { + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + } + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(fusiontype); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_conv.get()); + ptr_formerlayer = kernelptr_conv; + name = "test_conv_" + std::to_string(layerscount) + "layers_graph.dot"; + } + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(ptr_formerlayer); + auto ret = g->NewCNode(inputs); + ret->set_abstract(x_abstract); + + g->set_return(ret); + + draw::Draw(name, g); + + return g; +} + +CNodePtr CreateKernelGraphBranch(KernelGraphPtr g, CNodePtr inputptr, int layers, + const kernel::FusionType fusiontype = kernel::FusionType::CONVLUTION) { + std::vector shp = {1, 3, 3, 4}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + x_tensor->set_device_info(device_info); + auto x_abstract = x_tensor->ToAbstract(); + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_abstract); + + CNodePtr ptr_formerlayer = inputptr; + while (layers--) { + auto p_relu = std::make_shared("ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + std::vector inputs; + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(ptr_formerlayer); + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(x_abstract); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + ptr_formerlayer = kernelptr_floor; + } + + tensor::TensorPtr y_tensor = std::make_shared(kFloat32->type_id(), shp); + y_tensor->set_device_info(device_info); + tensor::TensorPtr z_tensor = std::make_shared(kFloat32->type_id(), shp); + z_tensor->set_device_info(device_info); + auto y_const = NewValueNode(y_tensor); + auto z_const = NewValueNode(y_tensor); + + y_const->set_abstract(y_tensor->ToAbstract()); + z_const->set_abstract(z_tensor->ToAbstract()); + + g->MutableInputs()->push_back(y_const); + + auto p_conv = std::make_shared("Conv2D"); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + std::vector inputs; + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(y_const); + inputs.push_back(ptr_formerlayer); + + auto kernelptr_conv = g->NewCNode(inputs); + kernelptr_conv->set_abstract(x_abstract); + kernelptr_conv->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(fusiontype); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_conv.get()); + return kernelptr_conv; +} + +static KernelGraphPtr CreateKernelGraphForMultiUse(uint32_t targetlayer1s, uint32_t targetlayer2s) { + /* @mindspore + * def f(x): + * multi_use=relu(x) + * y=relu(multi_use) + * z=relu(multi_use) + * ret=relu(y, z) + * return ret + */ + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + // x is input tensor. + std::vector shp = {1, 3, 3, 4}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + x_tensor->set_device_info(device_info); + + auto x_abstract = x_tensor->ToAbstract(); + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_abstract); + + g->MutableInputs()->push_back(x_const); + + auto p_multi = std::make_shared("MULTI_USE_ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_multi->set_attr("input_names", input_names_v); + p_multi->set_attr("output_names", output_names_v); + inputs.clear(); + inputs.push_back(NewValueNode(p_multi)); + inputs.push_back(x_const); + auto kernelptr_multi = g->NewCNode(inputs); + kernelptr_multi->set_abstract(x_abstract); + kernelptr_multi->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_multi.get()); + + CNodePtr outptrbranch1 = CreateKernelGraphBranch(g, kernelptr_multi, targetlayer2s); + CNodePtr outptrbranch2 = CreateKernelGraphBranch(g, kernelptr_multi, targetlayer1s); + + auto p_relu = std::make_shared("ReLU6"); + input_names = {"x"}; + output_names = {"output"}; + input_names_v = MakeValue(input_names); + output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(outptrbranch1); + inputs.push_back(outptrbranch2); + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(x_abstract); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder1.SetOutputsFormat({kOpFormat_NCHW}); + builder1.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + builder1.SetOutputsDeviceType({kFloat32->type_id()}); + builder1.SetKernelType(KernelType::TBE_KERNEL); + builder1.SetFusionType(kernel::FusionType::ELEMWISE); + builder1.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), kernelptr_floor.get()); + + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(kernelptr_floor); + auto ret = g->NewCNode(inputs); + ret->set_abstract(x_abstract); + + g->set_return(ret); + string name = "multi_use_graph.dot"; + draw::Draw(name, g); + + return g; +} +#ifdef BUFFER_FUSION_MULTI_OUT +static KernelGraphPtr CreateKernelGraphForMultiOutputWithLinearInput( + uint32_t targetlayer1s, uint32_t targetlayer2s, bool use_flag = true, + const kernel::FusionType fusion_type = kernel::FusionType::CONVLUTION) { + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + // x is input tensor. + std::vector shp = {1, 3, 3, 4}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + x_tensor->set_device_info(device_info); + + auto x_abstract = x_tensor->ToAbstract(); + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_abstract); + g->MutableInputs()->push_back(x_const); + + auto p_relu0 = std::make_shared("ReLU6"); + std::vector input_names0 = {"x"}; + std::vector output_names0 = {"output"}; + ValuePtr input_names_v0 = MakeValue(input_names0); + ValuePtr output_names_v0 = MakeValue(output_names0); + p_relu0->set_attr("input_names", input_names_v0); + p_relu0->set_attr("output_names", output_names_v0); + inputs.clear(); + inputs.push_back(NewValueNode(p_relu0)); + inputs.push_back(x_const); + auto kernelptr_floor0 = g->NewCNode(inputs); + kernelptr_floor0->set_abstract(x_abstract); + kernelptr_floor0->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder0; + builder0.SetInputsFormat({kOpFormat_NCHW}); + builder0.SetOutputsFormat({kOpFormat_NCHW}); + builder0.SetInputsDeviceType({kFloat32->type_id()}); + builder0.SetOutputsDeviceType({kFloat32->type_id()}); + builder0.SetKernelType(KernelType::TBE_KERNEL); + builder0.SetFusionType(kernel::FusionType::ELEMWISE); + builder0.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder0.Build(), kernelptr_floor0.get()); + CNodePtr ptr_formerlayer; + ptr_formerlayer = kernelptr_floor0; + + auto p_multi = std::make_shared("MULTI_USE_ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_multi->set_attr("input_names", input_names_v); + p_multi->set_attr("output_names", output_names_v); + inputs.clear(); + inputs.push_back(NewValueNode(p_multi)); + inputs.push_back(ptr_formerlayer); + auto kernelptr_multi = g->NewCNode(inputs); + kernelptr_multi->set_abstract(x_abstract); + kernelptr_multi->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_multi.get()); + + CNodePtr outptrbranch2 = nullptr; + CNodePtr outptrbranch1 = CreateKernelGraphBranch(g, kernelptr_multi, targetlayer2s, fusion_type); + if (use_flag) { + outptrbranch2 = CreateKernelGraphBranch(g, kernelptr_multi, targetlayer1s, fusion_type); + } + auto p_relu = std::make_shared("ReLU6"); + input_names = {"x"}; + output_names = {"output"}; + input_names_v = MakeValue(input_names); + output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(outptrbranch1); + if (use_flag) { + inputs.push_back(outptrbranch2); + } + + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(x_abstract); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder1; + if (use_flag) { + builder1.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder1.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + } else { + builder1.SetInputsFormat({kOpFormat_NCHW}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + } + builder1.SetOutputsFormat({kOpFormat_NCHW}); + builder1.SetOutputsDeviceType({kFloat32->type_id()}); + builder1.SetKernelType(KernelType::TBE_KERNEL); + builder1.SetFusionType(kernel::FusionType::ELEMWISE); + builder1.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), kernelptr_floor.get()); + cout << "built two branches done" << endl; + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(kernelptr_floor); + auto ret = g->NewCNode(inputs); + ret->set_abstract(x_abstract); + + g->set_return(ret); + string name = "multi_use_graph.dot"; + draw::Draw(name, g); + + return g; +} + +static KernelGraphPtr CreateKernelGraphForMultiOutput( + uint32_t targetlayer1s, uint32_t targetlayer2s, bool use_flag = true, + const kernel::FusionType fusion_type = kernel::FusionType::CONVLUTION) { + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + // x is input tensor. + std::vector shp = {1, 3, 3, 4}; + tensor::TensorPtr x_tensor = std::make_shared(kFloat32->type_id(), shp); + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + x_tensor->set_device_info(device_info); + + auto x_abstract = x_tensor->ToAbstract(); + auto x_const = NewValueNode(x_tensor); + x_const->set_abstract(x_abstract); + g->MutableInputs()->push_back(x_const); + + auto p_multi = std::make_shared("MULTI_USE_ReLU6"); + std::vector input_names = {"x"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_multi->set_attr("input_names", input_names_v); + p_multi->set_attr("output_names", output_names_v); + inputs.clear(); + inputs.push_back(NewValueNode(p_multi)); + inputs.push_back(x_const); + auto kernelptr_multi = g->NewCNode(inputs); + kernelptr_multi->set_abstract(x_abstract); + kernelptr_multi->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id(), kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_multi.get()); + + CNodePtr outptrbranch2 = nullptr; + CNodePtr outptrbranch1 = CreateKernelGraphBranch(g, kernelptr_multi, targetlayer2s, fusion_type); + if (use_flag) { + outptrbranch2 = CreateKernelGraphBranch(g, kernelptr_multi, targetlayer1s, fusion_type); + } + auto p_relu = std::make_shared("ReLU6"); + input_names = {"x"}; + output_names = {"output"}; + input_names_v = MakeValue(input_names); + output_names_v = MakeValue(output_names); + p_relu->set_attr("input_names", input_names_v); + p_relu->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(outptrbranch1); + if (use_flag) { + inputs.push_back(outptrbranch2); + } + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(x_abstract); + kernelptr_floor->set_kernel_info(std::make_shared()); + KernelBuildInfoBuilder builder1; + if (use_flag) { + builder1.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder1.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + } else { + builder1.SetInputsFormat({kOpFormat_NCHW}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + } + builder1.SetOutputsFormat({kOpFormat_NCHW}); + builder1.SetOutputsDeviceType({kFloat32->type_id()}); + builder1.SetKernelType(KernelType::TBE_KERNEL); + builder1.SetFusionType(kernel::FusionType::ELEMWISE); + builder1.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), kernelptr_floor.get()); + + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(kernelptr_floor); + auto ret = g->NewCNode(inputs); + ret->set_abstract(x_abstract); + + g->set_return(ret); + string name = "multi_use_graph.dot"; + draw::Draw(name, g); + + return g; +} +#endif +TEST_F(TestHWBufferFusion, BufferFusionlayerSingleIn1) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionSingleIn(1); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionlayerSingleIn1.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 8); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionlayerSingleIn1.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 6); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayerSingleIn2) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionSingleIn(2); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionlayerSingleIn2.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 10); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionlayerSingleIn2.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 6); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayerSingleIn3) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionSingleIn(3); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionlayerSingleIn3.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 12); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionlayerSingleIn3.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 6); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayer1) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(1); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); + buffer_fusion.Run(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayer2) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(2); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 7); + buffer_fusion.Run(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayer4) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(4); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 11); + buffer_fusion.Run(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayer6) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(6); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 15); + buffer_fusion.Run(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 7); +} + +TEST_F(TestHWBufferFusion, BufferFusionlayer8) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(8); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 19); + buffer_fusion.Run(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 7); +} + +TEST_F(TestHWBufferFusion, BufferFusionconv1) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(1, true); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), false); +} + +TEST_F(TestHWBufferFusion, BufferFusionconv8) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(8, true); + draw::Draw("before_BufferFusionconv8.dot", graph_ptr); + + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), true); + kernel::KernelPackPtr kernel_pack = std::make_shared(); + auto kernel_ptr = std::make_shared(kernel_pack); + std::unordered_map buffer_fusion_infos; + buffer_fusion.GetBufferFusionInfo(*graph_ptr, &buffer_fusion_infos); + std::vector fusion_ids; + for (auto &buffer_fusion_info : buffer_fusion_infos) { + fusion_ids.push_back(buffer_fusion_info.first); + } + std::sort(fusion_ids.begin(), fusion_ids.end()); + for (auto &fusion_id : fusion_ids) { + buffer_fusion.ReplaceFusionOp(buffer_fusion_infos[fusion_id], kernel_ptr, graph_ptr.get()); + } + draw::Draw("after_BufferFusionconv8.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 10); +} + +#ifdef BUFFER_FUSION_MULTI_OUT +TEST_F(TestHWBufferFusion, BufferFusionMultiOutWithLinearInput) { + KernelGraphPtr graph_ptr = CreateKernelGraphForMultiOutputWithLinearInput(1, 1, true, mindspore::kernel::OPAQUE); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 19); + + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), true); + kernel::KernelPackPtr kernel_pack = std::make_shared(); + auto kernel_ptr = std::make_shared(kernel_pack); + std::unordered_map buffer_fusion_infos; + buffer_fusion.GetBufferFusionInfo(*graph_ptr, &buffer_fusion_infos); + for (auto &buffer_fusion_info : buffer_fusion_infos) { + EXPECT_EQ(buffer_fusion_info.second.anf_nodes.size(), 3); + EXPECT_EQ(buffer_fusion_info.second.inputs_list.size(), 1); + EXPECT_EQ(buffer_fusion_info.second.outputs_list.size(), 2); + buffer_fusion.ReplaceFusionOp(buffer_fusion_info.second, kernel_ptr, graph_ptr.get()); + } + ASSERT_EQ(manager->all_nodes().size(), 21); +} + +TEST_F(TestHWBufferFusion, BufferFusionMultiOut) { + KernelGraphPtr graph_ptr = CreateKernelGraphForMultiOutput(1, 1, true, mindspore::kernel::OPAQUE); + draw::Draw("before_BufferFusionMultiOut.dot", graph_ptr); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 17); + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), true); + kernel::KernelPackPtr kernel_pack = std::make_shared(); + auto kernel_ptr = std::make_shared(kernel_pack); + std::unordered_map buffer_fusion_infos; + buffer_fusion.GetBufferFusionInfo(*graph_ptr, &buffer_fusion_infos); + for (auto &buffer_fusion_info : buffer_fusion_infos) { + EXPECT_EQ(buffer_fusion_info.second.anf_nodes.size(), 2); + EXPECT_EQ(buffer_fusion_info.second.inputs_list.size(), 1); + EXPECT_EQ(buffer_fusion_info.second.outputs_list.size(), 2); + buffer_fusion.ReplaceFusionOp(buffer_fusion_info.second, kernel_ptr, graph_ptr.get()); + } + draw::Draw("after_BufferFusionMultiOut.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 21); +} +#endif + +TEST_F(TestHWBufferFusion, BufferMultiUse) { + KernelGraphPtr graph_ptr = CreateKernelGraphForMultiUse(3, 4); + draw::Draw("before_BufferMultiUse.dot", graph_ptr); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), true); + kernel::KernelPackPtr kernel_pack = std::make_shared(); + auto kernel_ptr = std::make_shared(kernel_pack); + std::unordered_map buffer_fusion_infos; + buffer_fusion.GetBufferFusionInfo(*graph_ptr, &buffer_fusion_infos); + std::vector fusion_ids; + for (auto &buffer_fusion_info : buffer_fusion_infos) { + fusion_ids.push_back(buffer_fusion_info.first); + } + std::sort(fusion_ids.begin(), fusion_ids.end()); + for (auto &fusion_id : fusion_ids) { + buffer_fusion.ReplaceFusionOp(buffer_fusion_infos[fusion_id], kernel_ptr, graph_ptr.get()); + } + draw::Draw("after_BufferMultiUse.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 15); +} + +TEST_F(TestHWBufferFusion, BufferFusionReduce) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(2, true, mindspore::kernel::COMMREDUCE); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), true); + kernel::KernelPackPtr kernel_pack = std::make_shared(); + auto kernel_ptr = std::make_shared(kernel_pack); + std::unordered_map buffer_fusion_infos; + buffer_fusion.GetBufferFusionInfo(*graph_ptr, &buffer_fusion_infos); + for (auto &buffer_fusion_info : buffer_fusion_infos) { + EXPECT_EQ(buffer_fusion_info.second.anf_nodes.size(), 3); + EXPECT_EQ(buffer_fusion_info.second.inputs_list.size(), 1); + EXPECT_EQ(buffer_fusion_info.second.outputs_list.size(), 1); + buffer_fusion.ReplaceFusionOp(buffer_fusion_info.second, kernel_ptr, graph_ptr.get()); + } + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionSegment) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusion(2, true, mindspore::kernel::SEGMENT); + ASSERT_TRUE(nullptr != graph_ptr); + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(buffer_fusion.MatchBufferFusionPattern(*graph_ptr), true); + kernel::KernelPackPtr kernel_pack = std::make_shared(); + auto kernel_ptr = std::make_shared(kernel_pack); + std::unordered_map buffer_fusion_infos; + buffer_fusion.GetBufferFusionInfo(*graph_ptr, &buffer_fusion_infos); + for (auto &buffer_fusion_info : buffer_fusion_infos) { + EXPECT_EQ(buffer_fusion_info.second.anf_nodes.size(), 3); + EXPECT_EQ(buffer_fusion_info.second.inputs_list.size(), 1); + EXPECT_EQ(buffer_fusion_info.second.outputs_list.size(), 1); + buffer_fusion.ReplaceFusionOp(buffer_fusion_info.second, kernel_ptr, graph_ptr.get()); + } + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionEltwise1BeforeAnd3After) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionEltwiseBeforeAndAfter(1); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionEltwiseBeforeAndAfter1.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 13); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionEltwiseBeforeAndAfter1.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionEltwise2BeforeAnd3After) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionEltwiseBeforeAndAfter(2); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionEltwiseBeforeAndAfter2.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 15); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionEltwiseBeforeAndAfter2.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionEltwise3BeforeAnd3After) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionEltwiseBeforeAndAfter(3); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionEltwiseBeforeAndAfter3.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 17); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionEltwiseBeforeAndAfter3.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 5); +} + +TEST_F(TestHWBufferFusion, BufferFusionMultipleIn) { + KernelGraphPtr graph_ptr = CreateKernelGraphForBufferFusionMultipleIn(2); + ASSERT_TRUE(nullptr != graph_ptr); + draw::Draw("before_BufferFusionMultipleIn.dot", graph_ptr); + + mindspore::opt::BufferFusion buffer_fusion = BufferFusion(); + std::vector graphs{graph_ptr}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 11); + buffer_fusion.Run(graph_ptr); + draw::Draw("after_BufferFusionMultipleIn.dot", graph_ptr); + ASSERT_EQ(manager->all_nodes().size(), 7); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/buffer_fusion/buffer_fusion_test_py.cc b/tests/ut/cpp/pre_activate/ascend/buffer_fusion/buffer_fusion_test_py.cc new file mode 100644 index 00000000000..68d4cf4033f --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/buffer_fusion/buffer_fusion_test_py.cc @@ -0,0 +1,269 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "debug/anf_ir_dump.h" +#include "kernel/kernel.h" +#include "device/kernel_info.h" +#include "pre_activate/common/optimizer.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/ascend/buffer_fusion/buffer_fusion.h" + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +class TestHWBufferFusionPy : public BackendCommon { + public: + TestHWBufferFusionPy() : get_py_fun_("gtest_input.pre_activate.buffer_fusion_test", true) {} + ~TestHWBufferFusionPy() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWBufferFusionPy, test_tbe_eltwise_fusion_1) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_tbe_eltwise_fusion_1", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto tuple = ret->input(1); + EXPECT_NE(tuple, nullptr); + auto cast = tuple->cast()->input(1); + EXPECT_NE(cast, nullptr); + auto relu2 = cast->cast()->input(1); + EXPECT_NE(relu2, nullptr); + auto relu1 = relu2->cast()->input(1); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::TBE_KERNEL); + + relu1->set_kernel_info(std::make_shared()); + relu2->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu1.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu2.get()); + + KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NC1HWC0"}); + builder1.SetOutputsFormat({"NC1HWC0"}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + builder1.SetOutputsDeviceType({kFloat16->type_id()}); + builder1.SetKernelType(KernelType::TBE_KERNEL); + builder1.SetFusionType(kernel::FusionType::OPAQUE); + builder1.SetProcessor(kernel::Processor::AICORE); + builder1.SetKernelType(KernelType::TBE_KERNEL); + + cast->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast.get()); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto buffer_fusion_pass = std::make_shared(); + pm->AddPass(buffer_fusion_pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_tbe_eltwise_fusion_1", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWBufferFusionPy, test_tbe_eltwise_fusion_2) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_tbe_eltwise_fusion_2", "before"); + std::vector shp{32, 10}; + std::vector shp_bias{10}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto y_abstract = std::make_shared(kFloat32, shp_bias); + AbstractBasePtrList args_spec_list{x_abstract, y_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto tuple = ret->input(1); + EXPECT_NE(tuple, nullptr); + auto cast = tuple->cast()->input(1); + EXPECT_NE(cast, nullptr); + auto relu6 = cast->cast()->input(1); + EXPECT_NE(relu6, nullptr); + auto relu5 = relu6->cast()->input(1); + EXPECT_NE(relu5, nullptr); + auto relu4 = relu5->cast()->input(1); + EXPECT_NE(relu4, nullptr); + auto biasadd = relu4->cast()->input(1); + EXPECT_NE(biasadd, nullptr); + auto relu3 = biasadd->cast()->input(1); + EXPECT_NE(relu3, nullptr); + auto relu2 = relu3->cast()->input(1); + EXPECT_NE(relu2, nullptr); + auto relu1 = relu2->cast()->input(1); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::TBE_KERNEL); + + relu1->set_kernel_info(std::make_shared()); + relu2->set_kernel_info(std::make_shared()); + relu3->set_kernel_info(std::make_shared()); + relu4->set_kernel_info(std::make_shared()); + relu5->set_kernel_info(std::make_shared()); + relu6->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu1.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu2.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu3.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu4.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu5.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu6.get()); + + KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NC1HWC0"}); + builder1.SetOutputsFormat({"NC1HWC0"}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + builder1.SetOutputsDeviceType({kFloat16->type_id()}); + builder1.SetKernelType(KernelType::TBE_KERNEL); + builder1.SetFusionType(kernel::FusionType::OPAQUE); + builder1.SetProcessor(kernel::Processor::AICORE); + builder1.SetKernelType(KernelType::TBE_KERNEL); + + cast->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast.get()); + + KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({"NC1HWC0", "NC1HWC0"}); + builder2.SetOutputsFormat({"NC1HWC0"}); + builder2.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + builder2.SetOutputsDeviceType({kFloat32->type_id()}); + builder2.SetKernelType(KernelType::TBE_KERNEL); + builder2.SetFusionType(kernel::FusionType::COMMREDUCE); + builder2.SetProcessor(kernel::Processor::AICORE); + builder2.SetKernelType(KernelType::TBE_KERNEL); + + biasadd->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), biasadd.get()); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto buffer_fusion_pass = std::make_shared(); + pm->AddPass(buffer_fusion_pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_tbe_eltwise_fusion_2", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWBufferFusionPy, test_tbe_reduce_eltwise_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_tbe_reduce_eltwise_fusion", "before"); + std::vector shp{32, 10}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto tuple = ret->input(1); + EXPECT_NE(tuple, nullptr); + auto cast = tuple->cast()->input(1); + EXPECT_NE(cast, nullptr); + auto relu6 = cast->cast()->input(1); + EXPECT_NE(relu6, nullptr); + auto relu5 = relu6->cast()->input(1); + EXPECT_NE(relu5, nullptr); + auto relu4 = relu5->cast()->input(1); + EXPECT_NE(relu4, nullptr); + auto biasaddgrad = relu4->cast()->input(1); + EXPECT_NE(biasaddgrad, nullptr); + auto relu3 = biasaddgrad->cast()->input(1); + EXPECT_NE(relu3, nullptr); + auto relu2 = relu3->cast()->input(1); + EXPECT_NE(relu2, nullptr); + auto relu1 = relu2->cast()->input(1); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::TBE_KERNEL); + + relu1->set_kernel_info(std::make_shared()); + relu2->set_kernel_info(std::make_shared()); + relu3->set_kernel_info(std::make_shared()); + relu4->set_kernel_info(std::make_shared()); + relu5->set_kernel_info(std::make_shared()); + relu6->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu1.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu2.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu3.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu4.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu5.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), relu6.get()); + + KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NC1HWC0"}); + builder1.SetOutputsFormat({"NC1HWC0"}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + builder1.SetOutputsDeviceType({kFloat16->type_id()}); + builder1.SetKernelType(KernelType::TBE_KERNEL); + builder1.SetFusionType(kernel::FusionType::OPAQUE); + builder1.SetProcessor(kernel::Processor::AICORE); + builder1.SetKernelType(KernelType::TBE_KERNEL); + + cast->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast.get()); + + KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({"NC1HWC0"}); + builder2.SetOutputsFormat({"NC1HWC0"}); + builder2.SetInputsDeviceType({kFloat32->type_id()}); + builder2.SetOutputsDeviceType({kFloat32->type_id()}); + builder2.SetKernelType(KernelType::TBE_KERNEL); + builder2.SetFusionType(kernel::FusionType::COMMREDUCE); + builder2.SetProcessor(kernel::Processor::AICORE); + builder2.SetKernelType(KernelType::TBE_KERNEL); + + biasaddgrad->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), biasaddgrad.get()); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto buffer_fusion_pass = std::make_shared(); + pm->AddPass(buffer_fusion_pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_tbe_reduce_eltwise_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/format_type/check_consistency_test.cc b/tests/ut/cpp/pre_activate/ascend/format_type/check_consistency_test.cc new file mode 100644 index 00000000000..e3f22c9d97b --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/format_type/check_consistency_test.cc @@ -0,0 +1,166 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "common/backend_common_test.h" +#include "session/ascend_session.h" +#include "session/anf_runtime_algorithm.h" +#include "pipeline/resource.h" +#include "pipeline/action.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "kernel/kernel_build_info.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/format_type/check_consistency.h" + +namespace mindspore { +namespace opt { +class TestHWCheckConsistency : public BackendCommon { + public: + TestHWCheckConsistency() : get_py_fun_("gtest_input.pre_activate.check_consistency", true) {} + ~TestHWCheckConsistency() override = default; + + public: + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWCheckConsistency, test_check_consistency_for_format) { + // test CheckFormatForConsistency + get_py_fun_.SetDoResolve(true); + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_check_consistency", "graph"); + // renormalize func_graph to infer and set shape and type information. + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + g->parameters()[0]->set_abstract(x_abstract); + auto g_cast = g->get_return()->input(1); + g_cast->set_abstract(x_abstract); + + // convert to kernel graph + AbstractBasePtrList args_spec_list; + auto kernel_graph = GetKernelGraph(g, args_spec_list, false); + + // get make_tuple + auto ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_TRUE(ret->input(1)->isa()); + auto make_tuple = ret->input(1)->cast(); + + // set kernel for make tuple + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NCHW"}); + builder1.SetOutputsFormat({"NCHW"}); + builder1.SetInputsDeviceType({kNumberTypeFloat32}); + builder1.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), make_tuple.get()); + + // get cast + EXPECT_NE(make_tuple->input(1), nullptr); + EXPECT_TRUE(make_tuple->input(1)->isa()); + auto cast = make_tuple->input(1)->cast(); + + // set kernel for cast + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({"NC1HWC0"}); + builder2.SetOutputsFormat({"NCHW"}); + builder2.SetInputsDeviceType({kNumberTypeFloat32}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), cast.get()); + + // get para x + EXPECT_NE(cast->input(1), nullptr); + EXPECT_TRUE(cast->input(1)->isa()); + auto para = cast->input(1)->cast(); + + // set kernel for para x + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder3; + builder3.SetOutputsFormat({"NCHW"}); + builder3.SetOutputsDeviceType({kNumberTypeFloat32}); + AnfAlgo::SetSelectKernelBuildInfo(builder3.Build(), para.get()); + + // do CheckFormatForConsistency + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + EXPECT_THROW(optimizer->Optimize(kernel_graph), std::runtime_error); +} +TEST_F(TestHWCheckConsistency, test_check_consistency_for_dtype) { + // test CheckDataTypeForConsistency + get_py_fun_.SetDoResolve(true); + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_check_consistency", "graph"); + // Renormalize func_graph to infer and set shape and type information. + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + g->parameters()[0]->set_abstract(x_abstract); + auto g_cast = g->get_return()->input(1); + g_cast->set_abstract(x_abstract); + + // convert to kernel graph + AbstractBasePtrList args_spec_list; + auto kernel_graph = GetKernelGraph(g, args_spec_list, false); + + // get make tuple + auto ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_TRUE(ret->input(1)->isa()); + auto make_tuple = ret->input(1)->cast(); + + // set kernel for make tuple + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NCHW"}); + builder1.SetOutputsFormat({"NCHW"}); + builder1.SetInputsDeviceType({kNumberTypeFloat32}); + builder1.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), make_tuple.get()); + + // get cast + EXPECT_NE(make_tuple->input(1), nullptr); + EXPECT_TRUE(make_tuple->input(1)->isa()); + auto cast = make_tuple->input(1)->cast(); + + // set kernel for cast + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({"NCHW"}); + builder2.SetOutputsFormat({"NCHW"}); + builder2.SetInputsDeviceType({kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat32}); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), cast.get()); + + // get para x + EXPECT_NE(cast->input(1), nullptr); + EXPECT_TRUE(cast->input(1)->isa()); + auto para = cast->input(1)->cast(); + + // set kernel for para x + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder3; + builder3.SetOutputsFormat({"NCHW"}); + builder3.SetOutputsDeviceType({kNumberTypeFloat32}); + AnfAlgo::SetSelectKernelBuildInfo(builder3.Build(), para.get()); + + // do CheckFormatForConsistency + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + EXPECT_THROW(optimizer->Optimize(kernel_graph), std::runtime_error); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/format_type/insert_cast_test.cc b/tests/ut/cpp/pre_activate/ascend/format_type/insert_cast_test.cc new file mode 100644 index 00000000000..fd9838c8cde --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/format_type/insert_cast_test.cc @@ -0,0 +1,153 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "device/kernel_info.h" +#include "pre_activate/ascend/format_type/insert_cast.h" +#include "kernel/kernel_build_info.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace opt { +class TestHWInsertCast : public BackendCommon { + public: + TestHWInsertCast() : getPyFun_("gtest_input.pre_activate.mixed_precision_test", true) {} + ~TestHWInsertCast() override = default; + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestHWInsertCast, test_insert_cast_op_for_single_output) { + /* + * def test_insert_cast_op_for_single_output(x, y): + * res = add((x, y)) + * return res + */ + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_insert_cast_op_for_single_output", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(func_graph, nullptr); + + // Set selectedKernelInfo + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0", "NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NC1HWC0"}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + builder1.SetOutputsFormat({"NC1HWC0"}); + builder1.SetOutputsDeviceType({kFloat32->type_id()}); + builder1.SetFusionType(kernel::FusionType::ELEMWISE); + builder1.SetProcessor(kernel::Processor::AICORE); + builder1.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + auto node_list = TopoSort(func_graph->get_return()); + for (auto& node : node_list) { + if (node == nullptr) { + continue; + } + if (node->isa()) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), node.get()); + } else if (node != func_graph->get_return() && AnfAlgo::IsRealKernel(node)) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), node.get()); + } + } + // Do insert cast pass of hardware opt + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(func_graph); + EXPECT_NE(new_graph, nullptr); + + // check result + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_insert_cast_op_for_single_output", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWInsertCast, test_insert_cast_op_for_multiple_output) { + /* + * def test_insert_cast_op_for_multiple_output(): + * output = max_pool(w) + * res = tuple_getitem(output, 0) + * return res + */ + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_insert_cast_op_for_multiple_output", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(func_graph, nullptr); + + // Set selectedKernelInfo + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"DefaultFormat"}); + builder.SetOutputsFormat({"NC1HWC0", "NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({"NC1HWC0"}); + builder1.SetInputsDeviceType({kFloat32->type_id()}); + builder1.SetOutputsFormat({"DefaultFormat"}); + builder1.SetOutputsDeviceType({kFloat32->type_id()}); + builder1.SetFusionType(kernel::FusionType::ELEMWISE); + builder1.SetProcessor(kernel::Processor::AICORE); + builder1.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + auto node_list = TopoSort(func_graph->get_return()); + for (auto& node : node_list) { + if (node == nullptr) { + continue; + } + if (node->isa()) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), node.get()); + } else if (node != func_graph->get_return() && AnfAlgo::IsRealKernel(node)) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), node.get()); + } + } + + // Do insert cast pass of hardware opt + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(func_graph); + EXPECT_NE(new_graph, nullptr); + + // check result + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_insert_cast_op_for_multiple_output", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/format_type/insert_trans_op_test.cc b/tests/ut/cpp/pre_activate/ascend/format_type/insert_trans_op_test.cc new file mode 100644 index 00000000000..9528beceefe --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/format_type/insert_trans_op_test.cc @@ -0,0 +1,140 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "operator/ops.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" + +#define private public +#define protected public +#include "pre_activate/ascend/format_type/insert_trans_op.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; + +class TestHWInsertTransOp : public BackendCommon { + public: + TestHWInsertTransOp() : getPyFun_("gtest_input.pre_activate.insert_trans_op_test", true) {} + ~TestHWInsertTransOp() override = default; + FuncGraphPtr GetSingleOutputGraph(std::string func_name, std::string sub_func_name, std::string format) { + FuncGraphPtr g = getPyFun_.CallAndParseRet(func_name, sub_func_name); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto fg = GetKernelGraph(g, args_spec_list); + auto ret = fg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast()->input(1), nullptr); + auto add = ret->input(1)->cast()->input(1); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({format, format}); + builder.SetInputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + builder.SetOutputsFormat({format}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + add->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), add.get()); + return fg; + } + FuncGraphPtr GetMutilpleOutputGraph(std::string func_name, std::string sub_func_name, std::string format) { + FuncGraphPtr g = getPyFun_.CallAndParseRet(func_name, sub_func_name); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto fg = GetKernelGraph(g, args_spec_list); + + // Set selectedKernelInfo for max_pool + auto ret = fg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast()->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast()->input(1)->cast()->input(1), nullptr); + auto max_pool = ret->input(1)->cast()->input(1)->cast()->input(1); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_DEFAULT}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({format, format}); + builder.SetOutputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + max_pool->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), max_pool.get()); + return fg; + } + UT::PyFuncGraphFetcher getPyFun_; +}; + +class MockInsertTransOpKernelSelectTrans4Dto5D : public KernelSelect { + public: + bool is_four_to_five; + MockInsertTransOpKernelSelectTrans4Dto5D() = default; + ~MockInsertTransOpKernelSelectTrans4Dto5D() override = default; + void SelectKernel(const CNodePtr &cnode) override { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } +}; + +TEST_F(TestHWInsertTransOp, test_insert_trans_op_for_single_output) { + /* + * def test_insert_trans_op_for_single_output(x, y): + * res = add(x, y) + * output = make_tuple(res) + * return output + * + */ + auto fg = GetSingleOutputGraph("test_insert_trans_op_for_single_output", "before", "NC1HWC0"); + // Do insert_trans_op_ pass of hardware opt + auto graph_optimizer = std::make_shared(); + auto pass_manager = std::make_shared(); + auto insert_trans_op_pass = std::make_shared(); + insert_trans_op_pass->kernel_select_ = std::make_shared(); + pass_manager->AddPass(insert_trans_op_pass); + graph_optimizer->AddPassManager(pass_manager); + auto new_g = graph_optimizer->Optimize(fg); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_insert_trans_op_for_single_output", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_g)); +} + +TEST_F(TestHWInsertTransOp, test_insert_trans_op_for_multiple_output) { + /* + * def test_insert_trans_op_for_multiple_output(): + * max_pool_res = max_pool(x) + * res = tuple_getitem(max_pool_res, 0) + * output = make_tuple(res) + * return output + */ + FuncGraphPtr fg = GetMutilpleOutputGraph("test_insert_trans_op_for_multiple_output", "before", "NC1HWC0"); + // Do insert_trans_op_ pass of hardware opt + auto graph_optimizer = std::make_shared(); + auto pass_manager = std::make_shared(); + auto insert_trans_op_pass = std::make_shared(); + insert_trans_op_pass->kernel_select_ = std::make_shared(); + pass_manager->AddPass(insert_trans_op_pass); + graph_optimizer->AddPassManager(pass_manager); + auto new_g = graph_optimizer->Optimize(fg); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_insert_trans_op_for_multiple_output", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_g)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/format_type/merge_cast_to_op_test.cc b/tests/ut/cpp/pre_activate/ascend/format_type/merge_cast_to_op_test.cc new file mode 100644 index 00000000000..35616a29ce0 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/format_type/merge_cast_to_op_test.cc @@ -0,0 +1,192 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "kernel/kernel_build_info.h" +#include "pre_activate/common/optimizer.h" + +#define private public +#define protected public +#include "pre_activate/ascend/format_type/merge_cast_to_op.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +class TestHWMergeCastToOp : public BackendCommon { + public: + TestHWMergeCastToOp() : get_py_fun_("gtest_input.pre_activate.merge_cast_to_op", true) {} + ~TestHWMergeCastToOp() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +class MockMergeCastToOpKernelQuery : public KernelQuery { + public: + MockMergeCastToOpKernelQuery() = default; + ~MockMergeCastToOpKernelQuery() override = default; + void Query(const CNodePtr &kernel_node, + std::vector> *kernel_info_list) override { + std::string op_name = AnfAlgo::GetCNodeName(kernel_node); + if (op_name == kFour2FiveOpName) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_NC1HWC0}); + builder.SetInputsDeviceType({kNumberTypeFloat32}); + builder.SetOutputsDeviceType({kNumberTypeFloat16}); + kernel_info_list->push_back(builder.Build()); + } else if (op_name == kFive2FourOpName) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NC1HWC0}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetInputsDeviceType({kNumberTypeFloat16}); + builder.SetOutputsDeviceType({kNumberTypeFloat32}); + kernel_info_list->push_back(builder.Build()); + } + } +}; + +TEST_F(TestHWMergeCastToOp, test_merge_cast_to_next_op) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_merge_cast_to_next_op", "before"); + ASSERT_NE(g, nullptr); + + // set abstract because four2five node cannot infer + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + g->parameters()[0]->set_abstract(x_abstract); + g->get_return()->set_abstract(x_abstract); + AnfNodePtr g_four2five = g->get_return()->input(1); + g_four2five->set_abstract(x_abstract); + AnfNodePtr g_cast = g_four2five->cast()->input(1); + g_cast->set_abstract(x_abstract); + + // convert to kernel graph + AbstractBasePtrList args_spec_list; + auto kernel_graph = GetKernelGraph(g, args_spec_list, false); + + // get four2five + auto ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_TRUE(ret->input(1)->isa()); + auto make_tuple = ret->input(1)->cast(); + EXPECT_NE(make_tuple->input(1), nullptr); + EXPECT_TRUE(make_tuple->input(1)->isa()); + auto four2five = make_tuple->input(1)->cast(); + + // set kernel for four2five + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_NCHW}); + builder1.SetOutputsFormat({kOpFormat_NC1HWC0}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), four2five.get()); + + // get cast + EXPECT_NE(four2five->input(1), nullptr); + EXPECT_TRUE(four2five->input(1)->isa()); + auto cast = four2five->input(1)->cast(); + + // set kernel for cast + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_NCHW}); + builder2.SetOutputsFormat({kOpFormat_NCHW}); + builder2.SetInputsDeviceType({kNumberTypeFloat32}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), cast.get()); + + // do merge_cast_to_op_pass + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(kernel_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_merge_cast_to_next_op", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWMergeCastToOp, test_merge_cast_to_prior_op) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_merge_cast_to_prior_op", "before"); + ASSERT_NE(g, nullptr); + + // set abstract because five2four node cannot infer + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + g->parameters()[0]->set_abstract(x_abstract); + g->get_return()->set_abstract(x_abstract); + AnfNodePtr g_cast = g->get_return()->input(1); + g_cast->set_abstract(x_abstract); + AnfNodePtr g_five2four = g_cast->cast()->input(1); + g_five2four->set_abstract(x_abstract); + + // convert to kernel graph + AbstractBasePtrList args_spec_list; + auto kernel_graph = GetKernelGraph(g, args_spec_list, false); + + // get cast + auto ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_TRUE(ret->input(1)->isa()); + auto make_tuple = ret->input(1)->cast(); + EXPECT_NE(make_tuple->input(1), nullptr); + EXPECT_TRUE(make_tuple->input(1)->isa()); + auto cast = make_tuple->input(1)->cast(); + AnfAlgo::SetNodeAttr("dst_type", MakeValue("float32"), cast); + + // set kernel for cast + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_NCHW}); + builder1.SetOutputsFormat({kOpFormat_NCHW}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32}); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast.get()); + + // get five2four + EXPECT_NE(cast->input(1), nullptr); + EXPECT_TRUE(cast->input(1)->isa()); + auto five2four = cast->input(1)->cast(); + + // set kernel for five2four + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_NC1HWC0}); + builder2.SetOutputsFormat({kOpFormat_NCHW}); + builder2.SetInputsDeviceType({kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), five2four.get()); + + // do merge_cast_to_op_pass + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(kernel_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_merge_cast_to_prior_op", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fission/add_memcpy_async_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fission/add_memcpy_async_test.cc new file mode 100644 index 00000000000..516bcb89f0a --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fission/add_memcpy_async_test.cc @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "kernel/kernel_build_info.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fission/add_memcpy_async.h" + +namespace mindspore { +namespace opt { +class TestHWAddMemcpyAsync : public BackendCommon { + public: + TestHWAddMemcpyAsync() : get_py_fun_("gtest_input.pre_activate.add_memcpy_async", true) {} + + public: + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWAddMemcpyAsync, test_add_memcpy_async) { + get_py_fun_.SetDoResolve(true); + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_add_memcpy_async", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{1, 64, 112, 112}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(func_graph, nullptr); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(func_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_add_memcpy_async", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fission/bn_grad_split_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fission/bn_grad_split_test.cc new file mode 100644 index 00000000000..83ad70e90cd --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fission/bn_grad_split_test.cc @@ -0,0 +1,102 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "kernel/kernel_build_info.h" +#include "pre_activate/common/optimizer.h" + +#define private public +#define protected public +#include "pre_activate/ascend/ir_fission/bn_grad_split.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +class TestHWBnGradSplit : public BackendCommon { + public: + TestHWBnGradSplit() : get_py_fun_("gtest_input.pre_activate.bn_grad_split", true) {} + + public: + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWBnGradSplit, test_bn_grad_split_tbe) { + get_py_fun_.SetDoResolve(true); + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_bn_grad_split", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{1, 64, 112, 112}; + std::vector shp_b{64}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + auto b_abstract = std::make_shared(kFloat32, shp_b); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, b_abstract, b_abstract, b_abstract}; + auto kernel_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(kernel_graph, nullptr); + + // get BNGrad + CNodePtr ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_TRUE(ret->input(1)->isa()); + auto make_tuple1 = ret->input(1)->cast(); + EXPECT_NE(make_tuple1->input(1), nullptr); + EXPECT_TRUE(make_tuple1->input(1)->isa()); + auto make_tuple2 = make_tuple1->input(1)->cast(); + EXPECT_NE(make_tuple2->input(1), nullptr); + EXPECT_TRUE(make_tuple2->input(1)->isa()); + auto tuple_getitem = make_tuple2->input(1)->cast(); + EXPECT_NE(tuple_getitem->input(1), nullptr); + EXPECT_TRUE(tuple_getitem->input(1)->isa()); + auto bn_grad = tuple_getitem->input(1)->cast(); + + // get param1 + EXPECT_NE(bn_grad->input(1), nullptr); + auto param1 = bn_grad->input(1); + + // set kernel for param1 + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetOutputsFormat({kOpFormat_NC1HWC0}); + builder2.SetOutputsDeviceType({kNumberTypeFloat32}); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), param1.get()); + + // set kernel for BNGrad + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat( + {kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder1.SetOutputsFormat({kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder1.SetInputsDeviceType( + {kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32}); + builder1.SetKernelType(TBE_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), bn_grad.get()); + // do bn_grad_split pass + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(kernel_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_bn_grad_split", "after2"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fission/bn_split_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fission/bn_split_test.cc new file mode 100644 index 00000000000..77eee65b156 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fission/bn_split_test.cc @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/ascend_session.h" +#include "session/anf_runtime_algorithm.h" +#include "pipeline/resource.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "kernel/kernel_build_info.h" +#include "pre_activate/common/optimizer.h" + +#define private public +#define protected public +#include "pre_activate/ascend/ir_fission/bn_split.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +class TestHWBnSplit : public BackendCommon { + public: + TestHWBnSplit() : get_py_fun_("gtest_input.pre_activate.bn_split", true) {} + ~TestHWBnSplit() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWBnSplit, test_bn_split_tbe) { + /* + * def before(x, w, scale, b, mean, variance): + * sum = add(x, w) + * bn_output = bn(sum, scale, b, mean, variance) + * item0 = tuple_getitem(bn_output, 0) + * return item0 + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_bn_split_tbe", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{1, 64, 112, 112}; + std::vector shp_b{64}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + auto b_abstract = std::make_shared(kFloat32, shp_b); + AbstractBasePtrList args_spec_list{x_abstract, b_abstract, b_abstract, b_abstract, b_abstract}; + auto kernel_graph = GetKernelGraph(g, args_spec_list); + + // get kernel + auto ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_TRUE(ret->inputs().size() == 2); + auto make_tuple = ret->input(1)->cast(); + EXPECT_NE(make_tuple, nullptr); + EXPECT_TRUE(make_tuple->inputs().size() == 2); + auto item0 = make_tuple->input(1)->cast(); + EXPECT_NE(item0, nullptr); + EXPECT_TRUE(item0->inputs().size() == 3); + auto bn = item0->input(1); + EXPECT_NE(bn, nullptr); + EXPECT_TRUE(bn->isa()); + + // set kernel for BN + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat( + {kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder.SetOutputsFormat( + {kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder.SetInputsDeviceType( + {kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32}); + builder.SetOutputsDeviceType( + {kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32, kNumberTypeFloat32}); + builder.SetKernelType(KernelType::TBE_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), bn.get()); + + // do bn_grad_split_pass + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(kernel_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_bn_split_tbe", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fission/layer_norm_grad_split_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fission/layer_norm_grad_split_test.cc new file mode 100644 index 00000000000..cdb4d47e2c8 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fission/layer_norm_grad_split_test.cc @@ -0,0 +1,135 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "kernel/kernel_build_info.h" +#include "pre_activate/common/optimizer.h" + +#define private public +#define protected public +#include "pre_activate/ascend/ir_fission/layer_norm_grad_split.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +class TestHWLayerNormGradSplit : public BackendCommon { + public: + TestHWLayerNormGradSplit() : get_py_fun_("gtest_input.pre_activate.layer_norm_grad_split", true) {} + + public: + UT::PyFuncGraphFetcher get_py_fun_; +}; + +class MockLayerNormGradSplitKernelSelect : public KernelSelect { + public: + MockLayerNormGradSplitKernelSelect() = default; + ~MockLayerNormGradSplitKernelSelect() override = default; + void SelectKernel(const CNodePtr &cnode) override { + auto name = AnfAlgo::GetCNodeName(cnode); + + if (name == kLayerNormXBackpropOpName) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat( + {kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder.SetInputsDeviceType( + {kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder.SetOutputsFormat({kOpFormat_NC1HWC0}); + builder.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + return; + } + if (name == kLayerNormBetaGammaBackpropOpName) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder.SetOutputsFormat({kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + return; + } + } +}; // namespace opt + +TEST_F(TestHWLayerNormGradSplit, test_layer_norm_grad_split) { + get_py_fun_.SetDoResolve(true); + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_layer_norm_grad_split", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{1, 64, 112, 112}; + std::vector shp_b{64}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + auto b_abstract = std::make_shared(kFloat32, shp_b); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, b_abstract, b_abstract, b_abstract}; + auto kernel_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(kernel_graph, nullptr); + + // get LayerNormGrad + CNodePtr ret = kernel_graph->get_return(); + EXPECT_NE(ret, nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_TRUE(ret->input(1)->isa()); + auto make_tuple1 = ret->input(1)->cast(); + EXPECT_NE(make_tuple1->input(1), nullptr); + EXPECT_TRUE(make_tuple1->input(1)->isa()); + auto make_tuple2 = make_tuple1->input(1)->cast(); + EXPECT_NE(make_tuple2->input(1), nullptr); + EXPECT_TRUE(make_tuple2->input(1)->isa()); + auto tuple_getitem = make_tuple2->input(1)->cast(); + EXPECT_NE(tuple_getitem->input(1), nullptr); + EXPECT_TRUE(tuple_getitem->input(1)->isa()); + auto layer_norm_grad = tuple_getitem->input(1)->cast(); + + // set kernel for LayerNormGrad + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat( + {kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder1.SetOutputsFormat({kOpFormat_NC1HWC0, kOpFormat_NC1HWC0, kOpFormat_NC1HWC0}); + builder1.SetInputsDeviceType( + {kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder1.SetKernelType(TBE_KERNEL); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), layer_norm_grad.get()); + + // get param5 + EXPECT_NE(layer_norm_grad->input(5), nullptr); + auto param = layer_norm_grad->input(5); + + // set kernel for param5 + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetOutputsFormat({kOpFormat_NC1HWC0}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16}); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), param.get()); + + // do layer_norm_grad_split pass + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + auto kernel_select = std::make_shared(); + pass->kernel_select_ = kernel_select; + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(kernel_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_layer_norm_grad_split", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fission/topk_split_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fission/topk_split_test.cc new file mode 100644 index 00000000000..94fa04ef7af --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fission/topk_split_test.cc @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "device/kernel_info.h" +#include "pre_activate/ascend/ir_fission/topk_split.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWTopKSplit : public BackendCommon { + public: + TestHWTopKSplit() : get_py_fun_("gtest_input.pre_activate.topk_split_test", true) {} + ~TestHWTopKSplit() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWTopKSplit, test_topk_split) { + /* + * def before(input): + * topk = TopKSplit(input) + * output = tuple_getitem(topk, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_topk_split", "before"); + std::vector shp{4, 4}; + auto x_abstract = std::make_shared(kFloat32, shp); + g->parameters()[0]->set_abstract(x_abstract); + auto ret = g->get_return(); + EXPECT_NE(ret, nullptr); + auto tuple_getitem = ret->input(1); + EXPECT_NE(tuple_getitem, nullptr); + auto topk = tuple_getitem->cast()->input(1); + topk->set_abstract(x_abstract); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + auto topk_cnode = topk->cast(); + EXPECT_EQ(topk_cnode->inputs().size(), 3); + EXPECT_TRUE(topk_cnode->input(2)->isa()); + auto value_node = topk_cnode->input(2)->cast(); + EXPECT_TRUE(value_node->value()->isa()); + auto tensor = value_node->value()->cast(); + EXPECT_EQ(tensor->shape().size(), 1); + EXPECT_EQ(tensor->shape()[0], 4); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/adam_apply_one_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/adam_apply_one_fusion_test.cc new file mode 100644 index 00000000000..f4e418bed11 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/adam_apply_one_fusion_test.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/ascend/ir_fusion/adam_apply_one_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWAdamApplyOneFusion : public BackendCommon { + public: + TestHWAdamApplyOneFusion() : get_py_fun_("gtest_input.pre_activate.adam_apply_one_fusion_test", true) {} + ~TestHWAdamApplyOneFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWAdamApplyOneFusion, test_adam_apply_one_fusion) { + /* + * def before(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, add2_y): + * square0 = Square(input0) + * mul1 = Mul(mul1_x, input0) + * mul0 = Mul(mul0_x, input2) + * mul2 = Mul(mul2_x, input1) + * mul3 = Mul(mul3_x, square0) + * add0 = Add(mul0, mul1) + * add1 = Add(mul2, mul3) + * sqrt0 = Sqrt(add1) + * add2 = Add(sqrt0, add2_y) + * true_div0 = RealDiv(add0, add2) + * mul4 = Mul(input4, true_div0) + * sub0 = Sub(input3, mul4) + * outputs = make_tuple(add1, add0, sub0) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_adam_apply_one_fusion", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 10; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_adam_apply_one_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule_test.cc new file mode 100644 index 00000000000..52cb0017bbc --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule_test.cc @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fusion/adam_apply_one_with_decay_rule.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWOptimizeAdamApplyOneWithDecayRule : public BackendCommon { + public: + TestHWOptimizeAdamApplyOneWithDecayRule() + : get_py_fun_("gtest_input.pre_activate.adam_apply_one_with_decay_rule", true) {} + ~TestHWOptimizeAdamApplyOneWithDecayRule() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeAdamApplyOneWithDecayRule, test_adam_apply_one_with_decay_rule) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_adam_apply_one_with_decay_rule", "before"); + + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 11; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_adam_apply_one_with_decay_rule", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWOptimizeAdamApplyOneWithDecayRule, test_no_match) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_adam_apply_one_with_decay_rule", "no_match"); + + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 11; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/allreduce_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/allreduce_fusion_test.cc new file mode 100644 index 00000000000..1a7a103a99c --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/allreduce_fusion_test.cc @@ -0,0 +1,175 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/ascend/ir_fusion/allreduce_fusion.h" +#include "pre_activate/common/optimizer.h" +#include "device/kernel_info.h" +#include "pre_activate/common/pass_manager.h" +#include "kernel/kernel_build_info.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace opt { +class TestHWAllReduceFusion : public BackendCommon { + public: + TestHWAllReduceFusion() : getPyFun_("gtest_input.pre_activate.ir_fusion_test", true) {} + ~TestHWAllReduceFusion() override = default; + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestHWAllReduceFusion, test_fusion_all) { + getPyFun_.SetDoResolve(true); + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_all_reduce_fusion_all", "before"); + EXPECT_NE(g, nullptr); + std::vector shp_x{1, 64, 112, 112}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, x_abstract, x_abstract, x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(func_graph, nullptr); + // set kernel build info + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + auto node_list = TopoSort(func_graph->get_return()); + for (auto& node : node_list) { + if (node == nullptr) { + continue; + } + if ((node->isa() && AnfAlgo::GetCNodeName(node) == kAllReduceOpName) || node->isa()) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), node.get()); + } + } + // do all reduce fusion + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(func_graph); + EXPECT_NE(new_graph, nullptr); + // check result + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_all_reduce_fusion_all", "after"); + EXPECT_NE(g_after, nullptr); + EXPECT_TRUE(CheckEqualGraph(new_graph, g_after)); +} + +TEST_F(TestHWAllReduceFusion, test_fusion_group) { + getPyFun_.SetDoResolve(true); + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_all_reduce_fusion_group", "before"); + EXPECT_NE(g, nullptr); + std::vector shp_x{1, 64, 112, 112}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, x_abstract, x_abstract, x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(func_graph, nullptr); + // set kernel build info + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + auto node_list = TopoSort(func_graph->get_return()); + for (auto& node : node_list) { + if (node == nullptr) { + continue; + } + if ((node->isa() && AnfAlgo::GetCNodeName(node) == kAllReduceOpName) || node->isa()) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), node.get()); + } + } + // do all reduce fusion + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared(2)); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(func_graph); + EXPECT_NE(new_graph, nullptr); + // check result + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_all_reduce_fusion_group", "after1"); + EXPECT_NE(g_after, nullptr); + EXPECT_TRUE(CheckEqualGraph(new_graph, g_after)); +} + +TEST_F(TestHWAllReduceFusion, test_fusion_op) { + getPyFun_.SetDoResolve(true); + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_all_reduce_fusion_group", "before"); + EXPECT_NE(g, nullptr); + std::vector shp_x{1, 64, 112, 112}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, x_abstract, x_abstract, x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + EXPECT_NE(func_graph, nullptr); + // set kernel build info + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + builder.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + auto node_list = TopoSort(func_graph->get_return()); + int count = 0; + for (auto& node : node_list) { + if (node == nullptr) { + continue; + } + if ((node->isa() && AnfAlgo::GetCNodeName(node) == kAllReduceOpName) || node->isa()) { + node->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), node.get()); + } + if (node->isa() && AnfAlgo::GetCNodeName(node) == kAllReduceOpName) { + if (count == 0) { + AnfAlgo::SetNodeAttr("op", MakeValue("max"), node); + count = 1; + } else { + AnfAlgo::SetNodeAttr("op", MakeValue("sum"), node); + count = 0; + } + } + } + // do all reduce fusion + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(func_graph); + EXPECT_NE(new_graph, nullptr); + // check result + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_all_reduce_fusion_group", "after2"); + EXPECT_NE(g_after, nullptr); + EXPECT_TRUE(CheckEqualGraph(new_graph, g_after)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion_test.cc new file mode 100644 index 00000000000..0c8bf673918 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion_test.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fusion/clip_by_norm_no_div_square_sum_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWOptimizeClipByNormNodivsquaresumFusion : public BackendCommon { + public: + TestHWOptimizeClipByNormNodivsquaresumFusion() + : get_py_fun_("gtest_input.pre_activate.clip_by_norm_no_div_square_sum_fusion", true) {} + ~TestHWOptimizeClipByNormNodivsquaresumFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeClipByNormNodivsquaresumFusion, test_clip_by_norm_no_div_square_sum_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_clip_by_norm_no_div_square_sum_fusion", "before"); + + std::vector shp{2, 32, 224, 224}; + auto abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 4; ++i) { + args_spec_list.push_back(abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_clip_by_norm_no_div_square_sum_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/clip_by_value_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/clip_by_value_fusion_test.cc new file mode 100644 index 00000000000..4160c3a8e44 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/clip_by_value_fusion_test.cc @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fusion/clip_by_value_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWOptimizeClipByValueFusion : public BackendCommon { + public: + TestHWOptimizeClipByValueFusion() : get_py_fun_("gtest_input.pre_activate.clip_by_value_fusion", true) {} + ~TestHWOptimizeClipByValueFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeClipByValueFusion, test_clip_fusion_relu0) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_clip_by_value_fusion", "before1"); + EXPECT_NE(g, nullptr); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 3; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_clip_by_value_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWOptimizeClipByValueFusion, test_clip_fusion_relu1) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_clip_by_value_fusion", "before2"); + EXPECT_NE(g, nullptr); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 3; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_clip_by_value_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/confusion_softmax_grad_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/confusion_softmax_grad_test.cc new file mode 100644 index 00000000000..05fa2c65df4 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/confusion_softmax_grad_test.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fusion/confusion_softmax_grad_rule.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWOptimizeConfusionSoftmaxGradRule : public BackendCommon { + public: + TestHWOptimizeConfusionSoftmaxGradRule() + : get_py_fun_("gtest_input.pre_activate.confusion_softmax_grad_rule", true) {} + ~TestHWOptimizeConfusionSoftmaxGradRule() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeConfusionSoftmaxGradRule, test_confusion_softmax_grad_rule) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_confusion_softmax_grad_rule", "before"); + EXPECT_NE(g, nullptr); + std::vector shp{1, 1, 1, 1}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 2; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_confusion_softmax_grad_rule", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/conv_bn_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/conv_bn_fusion_test.cc new file mode 100644 index 00000000000..4fd4db823d4 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/conv_bn_fusion_test.cc @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "operator/ops.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" + +#define private public +#define protected public +#include "pre_activate/ascend/ir_fusion/conv_bn_fusion.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; + +class TestHWConvBnFusion : public BackendCommon { + public: + TestHWConvBnFusion() : getPyFun_("gtest_input.pre_activate.ir_fusion_test", true) {} + ~TestHWConvBnFusion() override = default; + + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestHWConvBnFusion, test_conv_bn_fusion) { + /* + * def before(x, y): + * conv_output = conv(x, y) + * bn_output = bn(conv_output) + * item0 = tuple_getitem(bn_output, 0) + * item1 = tuple_getitem(bn_output, 3) + * item2 = tuple_getitem(bn_output, 4) + * res = make_tuple(item0, item1, item2) + * return res + */ + getPyFun_.SetDoResolve(true); + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_conv_bn_fusion", "before"); + std::vector shp_x{32, 3, 224, 224}; + std::vector shp_w{64, 3, 7, 7}; + std::vector shp_b{64}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + auto w_abstract = std::make_shared(kFloat32, shp_w); + auto b_abstract = std::make_shared(kFloat32, shp_b); + AbstractBasePtrList args_spec_list{x_abstract, w_abstract, b_abstract, b_abstract, b_abstract, b_abstract}; + auto fg = GetKernelGraph(g, args_spec_list); + + auto graph_optimizer = std::make_shared(); + auto pass_manager = std::make_shared(); + auto conv_bn_fusion_pass = std::make_shared(); + pass_manager->AddPass(conv_bn_fusion_pass); + graph_optimizer->AddPassManager(pass_manager); + auto new_g = graph_optimizer->Optimize(fg); + + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_conv_bn_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_g)); +} + +} // namespace opt +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion_test.cc new file mode 100644 index 00000000000..8790b56d491 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/conv_bn_relu_fusion_test.cc @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "debug/anf_ir_dump.h" +#include "kernel/kernel_build_info.h" + +#define private public +#define protected public +#include "pre_activate/ascend/ir_fusion/conv_bn_relu_fusion.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +class TestHWConvBnReluFusion : public BackendCommon { + public: + TestHWConvBnReluFusion() : get_py_fun_("gtest_input.pre_activate.conv_bn_relu_fusion", true) {} + ~TestHWConvBnReluFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWConvBnReluFusion, test_conv_bn_relu_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_conv_bn_relu_fusion", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{32, 3, 224, 224}; + std::vector shp_w{64, 3, 7, 7}; + std::vector shp_b{64}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + auto w_abstract = std::make_shared(kFloat32, shp_w); + auto b_abstract = std::make_shared(kFloat32, shp_b); + AbstractBasePtrList args_spec_list{x_abstract, w_abstract, b_abstract, b_abstract, b_abstract, b_abstract}; + auto kernel_graph = GetKernelGraph(g, args_spec_list); + + // do bn_grad_split_pass + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(kernel_graph); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_conv_bn_relu_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_rule_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_rule_test.cc new file mode 100644 index 00000000000..797ca08b766 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_rule_test.cc @@ -0,0 +1,248 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "debug/anf_ir_dump.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_rule.h" + +namespace mindspore { +namespace opt { + +class TestHWLambNextMVRule : public BackendCommon { + public: + TestHWLambNextMVRule() : get_py_fun_("gtest_input.pre_activate.lamb_next_mv_rule_test", true) {} + ~TestHWLambNextMVRule() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWLambNextMVRule, test_lamb_next_mv_rule_matched) { + /* + * def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + * constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul0 = Mul(constant_mul0_x, input4) + * mul1 = Mul(constant_mul1_sub, input3) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * sqrt1 = Sqrt(real_div1) + * add4 = Add(sqrt1, constant_add2_y) + * real_div0 = RealDiv(add0, input5) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * mul4 = Mul(constant_mul4_x, input6) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, real_div4) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_rule", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_rule", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextMVRule, test_lamb_next_mv_rule_unmatched_real_div4) { + /* + * def before_unmatched_real_div4(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + * constant_mul1_sub, constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul0 = Mul(constant_mul0_x, input4) + * mul1 = Mul(constant_mul1_sub, input3) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * sqrt1 = Sqrt(real_div1) + * add4 = Add(sqrt1, constant_add2_y) + * real_div0 = RealDiv(add0, input5) + * real_div4 = Mul(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * mul4 = Mul(constant_mul4_x, input6) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, real_div4) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_rule", "before_unmatched_real_div4"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} + +TEST_F(TestHWLambNextMVRule, test_lamb_next_mv_rule_unmatched_real_div2) { + /* + * def before_unmatched_real_div2(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + * constant_mul1_sub, constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul0 = Mul(constant_mul0_x, input4) + * mul1 = Mul(constant_mul1_sub, input3) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * sqrt1 = Sqrt(real_div1) + * add4 = Add(sqrt1, constant_add2_y) + * real_div0 = RealDiv(add0, input5) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = RealDiv(real_div0, sqrt0) + * mul4 = Mul(constant_mul4_x, input6) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, real_div4) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_rule", "before_unmatched_real_div2"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} + +TEST_F(TestHWLambNextMVRule, test_lamb_next_mv_rule_unmatched_real_div0) { + /* + * def before_unmatched_real_div0(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + * constant_mul1_sub, constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul0 = Mul(constant_mul0_x, input4) + * mul1 = Mul(constant_mul1_sub, input3) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * sqrt1 = Sqrt(real_div1) + * add4 = Add(sqrt1, constant_add2_y) + * real_div0 = Mul(add0, input5) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * mul4 = Mul(constant_mul4_x, input6) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, real_div4) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_rule", "before_unmatched_real_div0"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} + +TEST_F(TestHWLambNextMVRule, test_lamb_next_mv_rule_unmatched_real_div1) { + /* + * def before_unmatched_real_div1(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + * constant_mul1_sub, constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul0 = Mul(constant_mul0_x, input4) + * mul1 = Mul(constant_mul1_sub, input3) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = Mul(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * sqrt1 = Sqrt(real_div1) + * add4 = Add(sqrt1, constant_add2_y) + * real_div0 = RealDiv(add0, input5) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * mul4 = Mul(constant_mul4_x, input6) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, real_div4) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_rule", "before_unmatched_real_div1"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule_test.cc new file mode 100644 index 00000000000..c0adeafe7b4 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule_test.cc @@ -0,0 +1,256 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_rule.h" + +namespace mindspore { +namespace opt { + +class TestHWLambNextMVWithDecayRule : public BackendCommon { + public: + TestHWLambNextMVWithDecayRule() : get_py_fun_("gtest_input.pre_activate.lamb_next_mv_with_decay_rule_test", true) {} + ~TestHWLambNextMVWithDecayRule() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWLambNextMVWithDecayRule, test_lamb_next_mv_decay_rule_matched) { + /* + * def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + * constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul1 = Mul(constant_mul1_sub, input3) + * mul0 = Mul(constant_mul0_x, input4) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt1 = Sqrt(real_div1) + * real_div0 = RealDiv(add0, input5) + * add4 = Add(sqrt1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * mul4 = Mul(constant_mul4_x, input6) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * add5 = Add(real_div4, mul4) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, add5) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayRule, test_lamb_next_mv_decay_rule_unmatched_add3) { + /* + * def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + * constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul1 = Mul(constant_mul1_sub, input3) + * mul0 = Mul(constant_mul0_x, input4) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt1 = Sqrt(real_div1) + * real_div0 = RealDiv(add0, input5) + * add4 = Add(sqrt1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * mul4 = Mul(constant_mul4_x, input6) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * add5 = Add(real_div4, mul4) + * add3 = Mul(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, add5) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "before_unmatched_add3"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayRule, test_lamb_next_mv_decay_rule_unmatched_mul4) { + /* + * def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + * constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul1 = Mul(constant_mul1_sub, input3) + * mul0 = Mul(constant_mul0_x, input4) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt1 = Sqrt(real_div1) + * real_div0 = RealDiv(add0, input5) + * add4 = Add(sqrt1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * mul4 = Add(constant_mul4_x, input6) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * add5 = Add(real_div4, mul4) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, add5) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "before_unmatched_mul4"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayRule, test_lamb_next_mv_decay_rule_unmatched_real_div0) { + /* + * def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + * constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul1 = Mul(constant_mul1_sub, input3) + * mul0 = Mul(constant_mul0_x, input4) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = RealDiv(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt1 = Sqrt(real_div1) + * real_div0 = Add(add0, input5) + * add4 = Add(sqrt1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * mul4 = Mul(constant_mul4_x, input6) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * add5 = Add(real_div4, mul4) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, add5) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "before_unmatched_real_div0"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayRule, test_lamb_next_mv_decay_rule_unmatched_real_div1) { + /* + * def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + * constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + * mul1 = Mul(constant_mul1_sub, input3) + * mul0 = Mul(constant_mul0_x, input4) + * add0 = Add(mul0, mul1) + * mul2 = Mul(constant_mul2_x, input1) + * mul3 = Mul(constant_mul3_sub1, input0) + * add1 = Add(mul2, mul3) + * real_div1 = Add(add1, input2) + * add2 = Add(real_div1, constant_add2_y) + * sqrt1 = Sqrt(real_div1) + * real_div0 = RealDiv(add0, input5) + * add4 = Add(sqrt1, constant_add2_y) + * sqrt0 = Rsqrt(add2) + * mul4 = Mul(constant_mul4_x, input6) + * real_div4 = RealDiv(real_div0, add4) + * real_div2 = Mul(real_div0, sqrt0) + * add5 = Add(real_div4, mul4) + * add3 = Add(real_div2, mul4) + * outputs = make_tuple(add3, add0, add1, add5) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "before_unmatched_real_div1"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_rule", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule_test.cc new file mode 100644 index 00000000000..fbb1f5e9138 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule_test.cc @@ -0,0 +1,111 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_mv_with_decay_v1_rule.h" + +namespace mindspore { +namespace opt { +class TestHWLambNextMVWithDecayV1Rule : public BackendCommon { + public: + TestHWLambNextMVWithDecayV1Rule() : get_py_fun_("gtest_input.pre_activate.lamb_next_mv_with_decay_v1_rule", true) {} + ~TestHWLambNextMVWithDecayV1Rule() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWLambNextMVWithDecayV1Rule, test_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_v1_rule", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_v1_rule", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayV1Rule, test_no_match1) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_v1_rule", "no_match1"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayV1Rule, test_no_match2) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_v1_rule", "no_match2"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} + +TEST_F(TestHWLambNextMVWithDecayV1Rule, test_no_match3) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_mv_with_decay_v1_rule", "no_match3"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 13; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_right_rule_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_right_rule_test.cc new file mode 100644 index 00000000000..f1ca92c8118 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_next_right_rule_test.cc @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/ascend/ir_fusion/lamb_next_right_rule.h" + +namespace mindspore { +namespace opt { + +class TestHWLambNextRightRule : public BackendCommon { + public: + TestHWLambNextRightRule() : get_py_fun_("gtest_input.pre_activate.lamb_next_right_rule_test", true) {} + ~TestHWLambNextRightRule() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWLambNextRightRule, test_lamb_next_right_rule_matched) { + /* + * def before(input0, input1, mul2_x, mul3_x, true_div1_recip, add2_y): + * square0 = Square(input0) + * mul2 = Mul(mul2_x, input1) + * mul3 = Mul(mul3_x, square0) + * add1 = Add(mul2, mul3) + * real_div1 = Mul(add1, true_div1_recip) + * add2 = Add(sqrt0, add2_y) + * outputs = make_tuple(add1, add2) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_right_rule", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 6; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_next_right_rule", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLambNextRightRule, test_lamb_next_right_rule_unmatched) { + /* + * def before(input0, input1, mul2_x, mul3_x, true_div1_recip, add2_y): + * square0 = Square(input0) + * mul2 = Mul(mul2_x, input1) + * mul3 = Add(mul3_x, square0) + * add1 = Add(mul2, mul3) + * real_div1 = Mul(add1, true_div1_recip) + * add2 = Add(sqrt0, add2_y) + * outputs = make_tuple(add1, add2) + * output = tuple_getitem(outputs, 0) + * return output + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_next_right_rule", "before_unmatched"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 6; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_graph = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_graph, new_graph)); +} +} // namespace opt +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion_test.cc new file mode 100644 index 00000000000..7a2806162b5 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion_test.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fusion/lamb_update_with_lr_rule_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWOptimizeLambUpdateWithLRRuleFusion : public BackendCommon { + public: + TestHWOptimizeLambUpdateWithLRRuleFusion() + : get_py_fun_("gtest_input.pre_activate.lamb_update_with_lr_rule_fusion", true) {} + ~TestHWOptimizeLambUpdateWithLRRuleFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeLambUpdateWithLRRuleFusion, test_lamb_update_with_lr_rule_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_update_with_lr_rule_fusion", "before"); + EXPECT_NE(g, nullptr); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 9; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_update_with_lr_rule_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2_test.cc new file mode 100644 index 00000000000..05262e72ab1 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2_test.cc @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "debug/anf_ir_dump.h" +#include "pre_activate/ascend/ir_fusion/lamb_update_with_lr_v2.h" + +namespace mindspore { +namespace opt { + +class TestHWLambUpdateWithLrV2 : public BackendCommon { + public: + TestHWLambUpdateWithLrV2() : get_py_fun_("gtest_input.pre_activate.lamb_update_with_lr_v2_test", true) {} + ~TestHWLambUpdateWithLrV2() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWLambUpdateWithLrV2, test_lamb_update_with_lr_v2) { + /* + * def before(input0, input1, input2, input3, input4, select_e, greater_y): + * greater0 = Greater(input0, greater_y) + * greater1 = Greater(input1, greater_y) + * real_div0 = RealDiv(input0, input1) + * select0 = Select(greater1, real_div0, select_e) + * select1 = Select(greater0, select0, select_e) + * mul0 = Mul(select1, input2) + * mul1 = Mul(mul0, input3) + * sub0 = Sub(input4, mul1) + * return sub0 + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_lamb_update_with_lr_v2", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 7; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_lamb_update_with_lr_v2", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion_test.cc new file mode 100644 index 00000000000..e7831ec3534 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion_test.cc @@ -0,0 +1,341 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "device/kernel_info.h" +#include "debug/anf_ir_dump.h" +#include "session/anf_runtime_algorithm.h" + +#define private public +#define protected public +#include "pre_activate/ascend/ir_fusion/layer_norm_beta_gamma_backprop_fusion.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { + +class TestHWLayerNormBetaGammaBackpropFusion : public BackendCommon { + public: + TestHWLayerNormBetaGammaBackpropFusion() + : get_py_fun_("gtest_input.pre_activate.layer_norm_beta_gamma_backprop_fusion_test", true) {} + ~TestHWLayerNormBetaGammaBackpropFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +class MockLayerNormBetaGammaBackpropFusionKernelQuery : public KernelQuery { + public: + MockLayerNormBetaGammaBackpropFusionKernelQuery() = default; + ~MockLayerNormBetaGammaBackpropFusionKernelQuery() override = default; + void Query(const CNodePtr &cnode, std::vector> *kernel_info_list) override { + if (AnfAlgo::GetCNodeName(cnode) == kLayerNormBetaGammaBackpropOpName) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder.SetOutputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder.SetOutputsDeviceType({kNumberTypeFloat32, kNumberTypeFloat32}); + kernel_info_list->push_back(builder.Build()); + } + } +}; + +TEST_F(TestHWLayerNormBetaGammaBackpropFusion, layernorm_beta_gamma_backprop_fusion_matched) { + /* + * def before(input0, input1, input2, input3): + * layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + * output0 = Cast(tuple_getitem(layer, 0)) + * output1 = Cast(tuple_getitem(layer, 1)) + * add = Add(output0, output1) + * return add + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto ret = g->get_return(); + EXPECT_NE(ret, nullptr); + auto add = ret->input(1); + EXPECT_NE(add, nullptr); + auto cast0 = add->cast()->input(1); + EXPECT_NE(cast0, nullptr); + auto cast1 = add->cast()->input(2); + EXPECT_NE(cast1, nullptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_DEFAULT}); + builder1.SetOutputsFormat({kOpFormat_DEFAULT}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32}); + cast0->set_kernel_info(std::make_shared()); + cast1->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast0.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast1.get()); + + auto tuple_getitem0 = cast0->cast()->input(1); + EXPECT_NE(tuple_getitem0, nullptr); + auto layer = tuple_getitem0->cast()->input(1); + EXPECT_NE(layer, nullptr); + AbstractBasePtrList new_node_list{x_abstract, x_abstract}; + layer->set_abstract(std::make_shared(new_node_list)); + + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetOutputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16}); + layer->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), layer.get()); + AnfAlgo::SetNodeAttr("shape_gamma", MakeValue(""), layer); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLayerNormBetaGammaBackpropFusion, layernorm_beta_gamma_backprop_fusion_unmatched_inputs_size) { + /* + * def before(input0, input1, input2): + * layer = LayerNormBetaGammaBackprop(input0, input1, input2) + * output0 = Cast(tuple_getitem(layer, 0)) + * output1 = Cast(tuple_getitem(layer, 1)) + * add = Add(output0, output1) + * return add + */ + FuncGraphPtr g = + get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "before_unmatched_inputs_size"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto ret = g->get_return(); + EXPECT_NE(ret, nullptr); + auto add = ret->input(1); + EXPECT_NE(add, nullptr); + auto cast0 = add->cast()->input(1); + EXPECT_NE(cast0, nullptr); + auto cast1 = add->cast()->input(2); + EXPECT_NE(cast1, nullptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_DEFAULT}); + builder1.SetOutputsFormat({kOpFormat_DEFAULT}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32}); + cast0->set_kernel_info(std::make_shared()); + cast1->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast0.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast1.get()); + + auto tuple_getitem0 = cast0->cast()->input(1); + EXPECT_NE(tuple_getitem0, nullptr); + auto layer = tuple_getitem0->cast()->input(1); + EXPECT_NE(layer, nullptr); + AbstractBasePtrList new_node_list{x_abstract, x_abstract}; + layer->set_abstract(std::make_shared(new_node_list)); + + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetOutputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16}); + layer->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), layer.get()); + AnfAlgo::SetNodeAttr("shape_gamma", MakeValue(""), layer); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + + EXPECT_TRUE(CheckEqualGraph(g, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLayerNormBetaGammaBackpropFusion, layernorm_beta_gamma_backprop_fusion_unmatched_attr) { + /* + * def before(input0, input1, input2, input3): + * layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + * output0 = Cast(tuple_getitem(layer, 0)) + * output1 = Cast(tuple_getitem(layer, 1)) + * add = Add(output0, output1) + * return add + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto ret = g->get_return(); + EXPECT_NE(ret, nullptr); + auto add = ret->input(1); + EXPECT_NE(add, nullptr); + auto cast0 = add->cast()->input(1); + EXPECT_NE(cast0, nullptr); + auto cast1 = add->cast()->input(2); + EXPECT_NE(cast1, nullptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_DEFAULT}); + builder1.SetOutputsFormat({kOpFormat_DEFAULT}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32}); + cast0->set_kernel_info(std::make_shared()); + cast1->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast0.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast1.get()); + + auto tuple_getitem0 = cast0->cast()->input(1); + EXPECT_NE(tuple_getitem0, nullptr); + auto layer = tuple_getitem0->cast()->input(1); + EXPECT_NE(layer, nullptr); + AbstractBasePtrList new_node_list{x_abstract, x_abstract}; + layer->set_abstract(std::make_shared(new_node_list)); + + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetOutputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16}); + layer->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), layer.get()); + AnfAlgo::EraseNodeAttr("shape_gamma", layer); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + + EXPECT_TRUE(CheckEqualGraph(g, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLayerNormBetaGammaBackpropFusion, layernorm_beta_gamma_backprop_fusion_unmatched_outputs_size) { + /* + * def before(input0, input1, input2, input3): + * layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + * output0 = Cast(layer) + * return output0 + */ + FuncGraphPtr g = + get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "before_unmatched_outputs_size"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto ret = g->get_return(); + EXPECT_NE(ret, nullptr); + auto cast = ret->input(1); + EXPECT_NE(cast, nullptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_DEFAULT}); + builder1.SetOutputsFormat({kOpFormat_DEFAULT}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32}); + cast->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast.get()); + + auto layer = cast->cast()->input(1); + EXPECT_NE(layer, nullptr); + layer->set_abstract(x_abstract); + + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetOutputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16, kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16}); + layer->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), layer.get()); + AnfAlgo::SetNodeAttr("shape_gamma", MakeValue(""), layer); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + + EXPECT_TRUE(CheckEqualGraph(g, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWLayerNormBetaGammaBackpropFusion, layernorm_beta_gamma_backprop_fusion_unmatched_input_device_data_type) { + /* + * def before(input0, input1, input2, input3): + * layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + * output0 = Cast(tuple_getitem(layer, 0)) + * output1 = Cast(tuple_getitem(layer, 1)) + * add = Add(output0, output1) + * return add + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto ret = g->get_return(); + EXPECT_NE(ret, nullptr); + auto add = ret->input(1); + EXPECT_NE(add, nullptr); + auto cast0 = add->cast()->input(1); + EXPECT_NE(cast0, nullptr); + auto cast1 = add->cast()->input(2); + EXPECT_NE(cast1, nullptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat({kOpFormat_DEFAULT}); + builder1.SetOutputsFormat({kOpFormat_DEFAULT}); + builder1.SetInputsDeviceType({kNumberTypeFloat16}); + builder1.SetOutputsDeviceType({kNumberTypeFloat32}); + cast0->set_kernel_info(std::make_shared()); + cast1->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast0.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder1.Build(), cast1.get()); + + auto tuple_getitem0 = cast0->cast()->input(1); + EXPECT_NE(tuple_getitem0, nullptr); + auto layer = tuple_getitem0->cast()->input(1); + EXPECT_NE(layer, nullptr); + AbstractBasePtrList new_node_list{x_abstract, x_abstract}; + layer->set_abstract(std::make_shared(new_node_list)); + + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder2; + builder2.SetInputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetOutputsFormat({kOpFormat_DEFAULT, kOpFormat_DEFAULT}); + builder2.SetInputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat32, kNumberTypeFloat16, kNumberTypeFloat16}); + builder2.SetOutputsDeviceType({kNumberTypeFloat16, kNumberTypeFloat16}); + layer->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder2.Build(), layer.get()); + AnfAlgo::SetNodeAttr("shape_gamma", MakeValue(""), layer); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto pass = std::make_shared(); + pass->kernel_query_ = std::make_shared(); + pm->AddPass(pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + + EXPECT_TRUE(CheckEqualGraph(g, new_graph)); + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_layer_norm_beta_gamma_backprop_fusion", "after"); + EXPECT_FALSE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion_test.cc new file mode 100644 index 00000000000..114fcf42330 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/momentum_lossscale_fusion_test.cc @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pre_activate/ascend/ir_fusion/momentum_lossscale_fusion.h" +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" + +namespace mindspore { +namespace opt { +class TestHWMomentumLossscaleFusion : public BackendCommon { + public: + TestHWMomentumLossscaleFusion() : get_py_fun_("gtest_input.pre_activate.momentum_lossscale_fusion_test", true) {} + ~TestHWMomentumLossscaleFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWMomentumLossscaleFusion, test_momentum_lossscale_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_momentum_lossscale_fusion", "before"); + EXPECT_NE(g, nullptr); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 5; ++i) { + args_spec_list.push_back(x_abstract); + } + auto kg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_momentum_lossscale_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/mul_add_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/mul_add_fusion_test.cc new file mode 100644 index 00000000000..7477d6252cd --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/mul_add_fusion_test.cc @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/ascend/ir_fusion/mul_add_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWMulAddFusion : public BackendCommon { + public: + TestHWMulAddFusion() : get_py_fun_("gtest_input.pre_activate.mul_add_fusion_test", true) {} + ~TestHWMulAddFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWMulAddFusion, test_mul_add_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_mul_add_fusion", "before"); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 3; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_mul_add_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/mul_addn_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/mul_addn_fusion_test.cc new file mode 100644 index 00000000000..ab9718d80a6 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/mul_addn_fusion_test.cc @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "mindspore/ccsrc/pre_activate/ascend/ir_fusion/mul_addn_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { +class TestHWMulAddNFusion : public BackendCommon { + public: + TestHWMulAddNFusion() : get_py_fun_("gtest_input.pre_activate.mul_addn_fusion_test", true) {} + ~TestHWMulAddNFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWMulAddNFusion, test_mul_addn_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_mul_addn_fusion", "before"); + std::vector shp{2, 2, 2, 2}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list({x_abstract, x_abstract}); + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_mul_addn_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWMulAddNFusion, test_unmatch) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_mul_addn_fusion", "unmatch"); + std::vector shp{2, 2, 2, 2}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list({x_abstract, x_abstract, x_abstract}); + auto fg = GetKernelGraph(g, args_spec_list); + auto origin_fg = std::make_shared(*fg); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + EXPECT_TRUE(CheckEqualGraph(origin_fg, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/reshape_transpose_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/reshape_transpose_fusion_test.cc new file mode 100644 index 00000000000..3478e92968e --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/reshape_transpose_fusion_test.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "debug/anf_ir_dump.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/ascend/ir_fusion/reshape_transpose_fusion.h" + +namespace mindspore { +namespace opt { + +class TestHWReshapeTransposeFusion : public BackendCommon { + public: + TestHWReshapeTransposeFusion() : get_py_fun_("gtest_input.pre_activate.reshape_transpose_fusion_test", true) {} + ~TestHWReshapeTransposeFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWReshapeTransposeFusion, test_reshape_transpose_fusion) { + /* + * def before(input0, input1): + * reshape = Reshape(input0, input1) + * transpose = Transpose(reshape) + * return transpose + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_reshape_transpose_fusion", "before"); + std::vector shp{2, 4, 8, 16}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + // Set Attr for transpose + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto transpose = ret->input(1)->cast(); + AnfAlgo::SetNodeAttr(kAttrPerm, MakeValue("perm"), transpose); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_reshape_transpose_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/square_sum_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/square_sum_fusion_test.cc new file mode 100644 index 00000000000..2dd858a0fca --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/square_sum_fusion_test.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/ascend/ir_fusion/square_sum_fusion.h" +#include "debug/anf_ir_dump.h" + +namespace mindspore { +namespace opt { + +class TestHWOptimizeSquareSumFusion : public BackendCommon { + public: + TestHWOptimizeSquareSumFusion() : get_py_fun_("gtest_input.pre_activate.square_sum_fusion", true) {} + ~TestHWOptimizeSquareSumFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeSquareSumFusion, test_square_sumv1_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_square_sum_fusion", "before1"); + std::vector shp{1, 1, 1, 1}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 1; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_square_sum_fusion", "after1"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWOptimizeSquareSumFusion, test_square_sumv2_fusion) { + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_square_sum_fusion", "before2"); + std::vector shp{1, 1, 1, 1}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list; + for (size_t i = 0; i < 1; ++i) { + args_spec_list.push_back(x_abstract); + } + auto fg = GetKernelGraph(g, args_spec_list); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(fg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_square_sum_fusion", "after2"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/transdata_split_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/transdata_split_test.cc new file mode 100644 index 00000000000..19215d2f1c2 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/transdata_split_test.cc @@ -0,0 +1,179 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "device/kernel_info.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/oplib/oplib.h" +#include "debug/anf_ir_dump.h" +#define private public +#define protected public +#include "pre_activate/ascend/format_type/insert_trans_op.h" +#include "pre_activate/ascend/ir_fusion/transdata_split.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +class TestHWTransdataSplit : public BackendCommon { + public: + TestHWTransdataSplit() : get_py_fun_("gtest_input.pre_activate.transdata_split_test", true) {} + ~TestHWTransdataSplit() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +class MockInsertTransOpKernelSelectTrans4Dto5D : public KernelSelect { + public: + MockInsertTransOpKernelSelectTrans4Dto5D() = default; + ~MockInsertTransOpKernelSelectTrans4Dto5D() override = default; + void SelectKernel(const CNodePtr &cnode) override { + if (AnfAlgo::GetCNodeName(cnode) == "Four2Five") { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } else { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } + } +}; + +class MockTransdataSplitKernelSelect : public KernelSelect { + public: + MockTransdataSplitKernelSelect() = default; + ~MockTransdataSplitKernelSelect() override = default; + void SelectKernel(const CNodePtr &cnode) override { + if (AnfAlgo::GetCNodeName(cnode) == kTransDataOpName) { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NCHW"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } else { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NCHW"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } + } +}; + +TEST_F(TestHWTransdataSplit, test_transdata_split_fraz_nchw) { + /* + * def before(input0, input1): + * transpose = transpose(input0, input1) + * transdata = Transdata(transpose) + * return transdata + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_transdata_split_fraz_nchw", "before"); + std::vector shp{2, 4, 8, 16}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto temp = ret->input(1)->cast(); + auto transpose = temp->input(1)->cast(); + EXPECT_NE(transpose, nullptr); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({kOpFormat_C1HWNCoC0}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + auto kernel_info = std::make_shared(); + kernel_info->set_select_kernel_build_info(builder.Build()); + transpose->set_kernel_info(kernel_info); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto insert_trans_op_pass = std::make_shared(); + insert_trans_op_pass->kernel_select_ = std::make_shared(); + auto transdata_split_pass = std::make_shared(); + transdata_split_pass->kernel_select_ = std::make_shared(); + pm->AddPass(insert_trans_op_pass); + pm->AddPass(transdata_split_pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_transdata_split_fraz_nchw", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} + +TEST_F(TestHWTransdataSplit, test_transdata_split_nchw_fraz) { + /* + * def before(input0, input1): + * transpose = transpose(input0, input1) + * transdata = Transdata(transpose) + * return transdata + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_transdata_split_nchw_fraz", "before"); + std::vector shp{2, 4, 8, 16}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto temp = ret->input(1)->cast(); + auto transpose = temp->input(1)->cast(); + EXPECT_NE(transpose, nullptr); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_C1HWNCoC0}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NCHW"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + auto kernel_info = std::make_shared(); + kernel_info->set_select_kernel_build_info(builder.Build()); + transpose->set_kernel_info(kernel_info); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto insert_trans_op_pass = std::make_shared(); + insert_trans_op_pass->kernel_select_ = std::make_shared(); + auto transdata_split_pass = std::make_shared(); + transdata_split_pass->kernel_select_ = std::make_shared(); + pm->AddPass(insert_trans_op_pass); + pm->AddPass(transdata_split_pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_transdata_split_nchw_fraz", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/transpose_reshape_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/transpose_reshape_fusion_test.cc new file mode 100644 index 00000000000..8f855b9a6e5 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/transpose_reshape_fusion_test.cc @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "debug/anf_ir_dump.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/ascend/ir_fusion/transpose_reshape_fusion.h" + +namespace mindspore { +namespace opt { + +class TestHWTransposeReshapeFusion : public BackendCommon { + public: + TestHWTransposeReshapeFusion() : get_py_fun_("gtest_input.pre_activate.transpose_reshape_fusion_test", true) {} + ~TestHWTransposeReshapeFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWTransposeReshapeFusion, test_transpose_reshape_fusion) { + /* + * def before(input0, input1): + * reshape = Reshape(input0, input1) + * transpose = Transpose(reshape) + * return transpose + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_transpose_reshape_fusion", "before"); + std::vector shp{2, 4, 8, 16}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + // Set Attr for transpose + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto reshape = ret->input(1)->cast(); + EXPECT_NE(reshape->input(1), nullptr); + auto transpose = reshape->input(1)->cast(); + AnfAlgo::SetNodeAttr(kAttrPerm, MakeValue("perm"), transpose); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_transpose_reshape_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/ascend/ir_fusion/transpose_transdata_fusion_test.cc b/tests/ut/cpp/pre_activate/ascend/ir_fusion/transpose_transdata_fusion_test.cc new file mode 100644 index 00000000000..af59ef7e9a9 --- /dev/null +++ b/tests/ut/cpp/pre_activate/ascend/ir_fusion/transpose_transdata_fusion_test.cc @@ -0,0 +1,148 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "device/kernel_info.h" +#include "session/anf_runtime_algorithm.h" +#include "kernel/oplib/oplib.h" +#define private public +#define protected public +#include "pre_activate/ascend/format_type/insert_trans_op.h" +#include "pre_activate/ascend/ir_fusion/transpose_transdata_fusion.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +class TestHWTransposeTransdataFusion : public BackendCommon { + public: + TestHWTransposeTransdataFusion() : get_py_fun_("gtest_input.pre_activate.transpose_transdata_fusion_test", true) {} + ~TestHWTransposeTransdataFusion() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +class MockInsertTransOpKernelSelectTrans4Dto5D : public KernelSelect { + public: + MockInsertTransOpKernelSelectTrans4Dto5D() = default; + ~MockInsertTransOpKernelSelectTrans4Dto5D() override = default; + void SelectKernel(const CNodePtr &cnode) override { + if (AnfAlgo::GetCNodeName(cnode) == "TransData") { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } else { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } + } +}; + +class MockTransposeTransdataFusionKernelSelect : public KernelSelect { + public: + MockTransposeTransdataFusionKernelSelect() = default; + ~MockTransposeTransdataFusionKernelSelect() override = default; + bool CheckKernelAccuracySupported(const CNodePtr &kernel_node, + const kernel::KernelBuildInfoPtr &new_kernel_build_info) override { + std::vector> kernel_info_list; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW}); + builder.SetOutputsFormat({kOpFormat_DEFAULT}); + builder.SetInputsDeviceType({kNumberTypeFloat16}); + builder.SetOutputsDeviceType({kNumberTypeFloat16}); + builder.SetKernelType(KernelType::AUTO_DIFF_KERNEL); + builder.SetFusionType(kernel::FusionType::OPAQUE); + builder.SetProcessor(kernel::Processor::AICORE); + kernel_info_list.push_back(builder.Build()); + MS_LOG(INFO) << "transpose transdata fusion success"; + MS_LOG(INFO) << "new transdata build info input format:" << new_kernel_build_info->GetInputFormat(0) + << ",outputformat:" << new_kernel_build_info->GetOutputFormat(0) + << ",kerneltype:" << new_kernel_build_info->kernel_type() + << ",fusiontype:" << new_kernel_build_info->fusion_type() + << ",process:" << new_kernel_build_info->processor(); + auto result = std::find_if(kernel_info_list.begin(), kernel_info_list.end(), + [&new_kernel_build_info](kernel::KernelBuildInfoPtr item) { + MS_EXCEPTION_IF_NULL(item); + return *item == *new_kernel_build_info; + }); + return result != kernel_info_list.end(); + } +}; + +TEST_F(TestHWTransposeTransdataFusion, test_transpose_transdata_fusion) { + /* + * def before(input0, input1): + * transpose = transpose(input0, input1) + * transdata = Transdata(transpose) + * return transdata + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_transpose_transdata_fusion", "before"); + std::vector shp{2, 4, 8, 16}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + auto temp = ret->input(1)->cast(); + auto transpose = temp->input(1)->cast(); + EXPECT_NE(transpose, nullptr); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(kernel::FusionType::ELEMWISE); + builder.SetProcessor(kernel::Processor::AICORE); + auto kernel_info = std::make_shared(); + kernel_info->set_select_kernel_build_info(builder.Build()); + transpose->set_kernel_info(kernel_info); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + auto insert_trans_op_pass = std::make_shared(); + insert_trans_op_pass->kernel_select_ = std::make_shared(); + pm->AddPass(insert_trans_op_pass); + auto transpose_transdata_pass = std::make_shared(); + transpose_transdata_pass->kernel_select_ = std::make_shared(); + pm->AddPass(transpose_transdata_pass); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(kg); + + ret = new_graph->get_return(); + EXPECT_NE(ret->input(1), nullptr); + temp = ret->input(1)->cast(); + MS_LOG(INFO) << "input(1) name:" << temp->fullname_with_scope(); + EXPECT_NE(temp->input(1), nullptr); + auto temp_node = temp->input(1)->cast(); + MS_LOG(INFO) << "input(11) name:" << temp_node->fullname_with_scope(); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_transpose_transdata_fusion", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/common/pattern_engine_test.cc b/tests/ut/cpp/pre_activate/common/pattern_engine_test.cc new file mode 100644 index 00000000000..9124f5cf747 --- /dev/null +++ b/tests/ut/cpp/pre_activate/common/pattern_engine_test.cc @@ -0,0 +1,227 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "common/common_test.h" +#include "pre_activate/common/pattern_engine.h" +#include "pre_activate/common/visit.h" +#include "utils/base_ref.h" +#include "ir/anf.h" + +namespace mindspore { +using PatternListType = std::initializer_list; + +bool Equal(const BaseRef &a, const BaseRef &b) { return a == b; } + +class TestMatchEngine : public UT::Common { + public: + TestMatchEngine() + : TU(std::make_shared(), std::function(Equal)) { + equiv_null = std::make_shared(); + }; + + public: + PatternEngine TU; + EquivPtr equiv_null; +}; + +TEST_F(TestMatchEngine, Var) { + VarPtr v1 = std::make_shared(); + VarPtr v2 = std::make_shared(); + VarPtr v3 = std::make_shared("name"); + ASSERT_TRUE(!(v1 == v2)); + ASSERT_TRUE(v1->matches(v2)); + ASSERT_TRUE(v1->matches(1)); + ASSERT_EQ(v3->ToString(), "Var(name)"); + ASSERT_NE(v1->tag(), v2->tag()); +} + +TEST_F(TestMatchEngine, CondVar) { + auto Pos = static_cast([](const BaseRef &any) -> bool { + float v = 0; + if (utils::isa(any)) { + v = utils::cast(any); + } else if (utils::isa(any)) { + v = utils::cast(any); + } else { + return false; + } + return v > 0; + }); + + VarPtr fv1 = std::make_shared(Pos); + + ASSERT_TRUE(fv1->matches(1.0f)); + ASSERT_FALSE(fv1->matches(0.0f)); +} + +TEST_F(TestMatchEngine, Seq) { + auto seq = Seq({1, 2, 3}); + MS_LOG(INFO) << "seq:" << seq.ToString(); + ASSERT_EQ(seq.ToString(), "vector[Int32Imm value:1, Int32Imm value:2, Int32Imm value:3]"); +} + +TEST_F(TestMatchEngine, SeqVar) { + VarPtr sv1 = std::make_shared(); + auto seq1 = std::make_shared(PatternListType({1, 2})); + ASSERT_FALSE(sv1->matches(1)); + ASSERT_FALSE(sv1->matches(1.0f)); + + ASSERT_TRUE(sv1->matches(seq1)); + + std::cout << sv1->ToString() << std::endl; +} + +TEST_F(TestMatchEngine, ExpandList) { + auto v1 = VectorRef({1, 2, 3}); + auto v2 = VectorRef({1, PatternListType({2, 3, 4}), 5}); + auto p1 = ExpandList(v1.elements()); + auto p2 = ExpandList(v2.elements()); + ASSERT_EQ(*p1, VectorRef({1, 2, 3})); + ASSERT_EQ(*p2, VectorRef({1, 2, 3, 4, 5})); +} + +TEST_F(TestMatchEngine, MatchRaw_Var) { + VarPtr v1 = std::make_shared(); + VarPtr v2 = std::make_shared(); + VarPtr v3 = std::make_shared(); + EquivPtr d; + + // common + equiv_null->clear(); + d = TU.Match(v1, 1, equiv_null); + ASSERT_EQ((*d)[v1], 1); + + equiv_null->clear(); + (*equiv_null)[v1] = v2; + d = TU.Match(v1, 1, equiv_null); + ASSERT_EQ(d->count(v2), std::size_t(1)); + ASSERT_EQ((*d)[v2], 1); + + equiv_null->clear(); + (*equiv_null)[v1] = v2; + (*equiv_null)[v3] = 1; + d = TU.Match(v1, 1, equiv_null); + ASSERT_EQ(d->count(v2), std::size_t(1)); + ASSERT_EQ((*d)[v2], 1); + + equiv_null->clear(); + d = TU.Match(VectorRef({v1}), VectorRef({1}), equiv_null); + ASSERT_EQ(d->size(), std::size_t(1)); + ASSERT_EQ(d->count(v1), std::size_t(1)); + ASSERT_EQ((*d)[v1], 1); + + equiv_null->clear(); + ASSERT_EQ(TU.Match(1, 2, equiv_null), nullptr); +} + +TEST_F(TestMatchEngine, MatchRaw_SVar) { + VarPtr v1 = std::make_shared(); + VarPtr sv1 = std::make_shared(); + VarPtr sv2 = std::make_shared(); + EquivPtr d; + + equiv_null->clear(); + d = TU.Match(VectorRef({sv1}), VectorRef({1, 2}), equiv_null); + ASSERT_EQ(d->size(), std::size_t(1)); + ASSERT_EQ(d->count(sv1), std::size_t(1)); + ASSERT_EQ(utils::cast((*d)[sv1]), Seq({1, 2})); + + equiv_null->clear(); + d = TU.Match(VectorRef({v1, sv1}), VectorRef({1, 2}), equiv_null); + ASSERT_EQ(d->size(), std::size_t(2)); + ASSERT_EQ(utils::cast((*d)[sv1]), Seq({2})); + + equiv_null->clear(); + ASSERT_EQ(TU.Match(VectorRef({sv1, sv2}), VectorRef({1, 2}), equiv_null), nullptr); + + equiv_null->clear(); + (*equiv_null)[sv1] = std::make_shared(PatternListType{1, 2}); + d = TU.Match(VectorRef({v1, sv1}), VectorRef({1, 1, 2}), equiv_null); + ASSERT_EQ(d->size(), std::size_t(2)); + ASSERT_EQ((*d)[v1], 1); +} + +TEST_F(TestMatchEngine, Match) { + VarPtr v1 = std::make_shared(); + VarPtr v2 = std::make_shared(); + VarPtr v3 = std::make_shared(); + + EquivPtr d; + + equiv_null->clear(); + d = TU.Match(VectorRef({v1, v1, v2}), VectorRef({1, 1, 2}), equiv_null); + ASSERT_EQ(d->size(), std::size_t(2)); + ASSERT_EQ((*d)[v1], 1); + ASSERT_EQ((*d)[v2], 2); + + equiv_null->clear(); + d = TU.Match(static_cast(1), static_cast(1), equiv_null); + ASSERT_EQ(d, nullptr); +} + +TEST_F(TestMatchEngine, Match_CondVar) { + auto floats = + static_cast([](const BaseRef &any) -> bool { return utils::isa(any); }); + auto neg = static_cast([](const BaseRef &any) -> bool { + float v = 0; + if (utils::isa(any)) { + v = utils::cast(any); + } else if (utils::isa(any)) { + v = utils::cast(any); + } else { + return false; + } + return v < 0; + }); + + VarPtr vf = std::make_shared(floats); + VarPtr vn = std::make_shared(neg); + EquivPtr d; + + equiv_null->clear(); + d = TU.Match(VectorRef({vf, vn}), VectorRef({static_cast(1.0), -1}), equiv_null); + ASSERT_GE(d->size(), std::size_t(0)); + auto vfn = (*d)[vf]; + ASSERT_EQ((*d)[vf], static_cast(1.0)); + ASSERT_EQ((*d)[vn], -1); + + equiv_null->clear(); + d = TU.Match(VectorRef({vf, vn}), VectorRef({1, static_cast(-1.0)}), equiv_null); + ASSERT_EQ(d, nullptr); + + equiv_null->clear(); + d = TU.Match(VectorRef({vf, vn}), VectorRef({static_cast(1.0), static_cast(1)}), equiv_null); + ASSERT_EQ(d, nullptr); +} + +TEST_F(TestMatchEngine, Match_Reify) { + VarPtr v1 = std::make_shared(); + VarPtr sv = std::make_shared(); + + BaseRef t; + + equiv_null->clear(); + (*equiv_null)[sv] = BaseRef(std::make_shared(PatternListType{3, 4})); + t = TU.Replace(VectorRef({1, 2, sv}), equiv_null); + ASSERT_EQ(t, BaseRef(VectorRef({1, 2, 3, 4}))); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/mem_reuse/kernel_ref_test.cc b/tests/ut/cpp/pre_activate/mem_reuse/kernel_ref_test.cc new file mode 100644 index 00000000000..5b237fda587 --- /dev/null +++ b/tests/ut/cpp/pre_activate/mem_reuse/kernel_ref_test.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "pre_activate/mem_reuse/kernel_refcount.h" + +#include "utils/utils.h" +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +using mindspore::memreuse::KernelRefCount; +using mindspore::memreuse::KernelRefCountPtr; +using mindspore::memreuse::RefCountType; +namespace mindspore { +class TestKernelRefCount : public UT::Common { + public: + TestKernelRefCount() {} + virtual void SetUp(); + virtual void TearDown(); +}; + +void TestKernelRefCount::SetUp() { UT::InitPythonPath(); } + +void TestKernelRefCount::TearDown() {} + +TEST_F(TestKernelRefCount, test_InitKernelRefCount) { + KernelRefCountPtr kernel_ref_count_ptr = std::make_shared(); + int ref_count = 3; + size_t offset = 512; + size_t size = 256; + int index = 2; + kernel_ref_count_ptr->index_ = index; + kernel_ref_count_ptr->offset_ = offset; + kernel_ref_count_ptr->ref_count_ = ref_count; + kernel_ref_count_ptr->size_ = size; + ASSERT_NE(kernel_ref_count_ptr, nullptr); +} + +TEST_F(TestKernelRefCount, test_RefCount) { + KernelRefCountPtr kernel_ref_count_ptr = std::make_shared(); + int ref_count = kernel_ref_count_ptr->ref_count_; + ASSERT_EQ(ref_count, 0); +} + +TEST_F(TestKernelRefCount, test_SetKernelRefInfo) { + KernelRefCountPtr kernel_ref_count_ptr = std::make_shared(); + size_t size = 256; + int index = 2; + RefCountType ref_count_type = memreuse::kDynamicRefCount; + kernel_ref_count_ptr->SetKernelRefCountInfo(index, size, ref_count_type); + ASSERT_EQ(kernel_ref_count_ptr->index_, index); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/mem_reuse/mem_reuse_allocator_test.cc b/tests/ut/cpp/pre_activate/mem_reuse/mem_reuse_allocator_test.cc new file mode 100644 index 00000000000..5431ba3ee15 --- /dev/null +++ b/tests/ut/cpp/pre_activate/mem_reuse/mem_reuse_allocator_test.cc @@ -0,0 +1,156 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "operator/ops.h" +#include "pre_activate/mem_reuse/mem_reuse.h" +#include "pre_activate/mem_reuse/mem_reuse_allocator.h" + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +using mindspore::memreuse::BestFitMemReuse; +using mindspore::memreuse::KernelDef; +using mindspore::memreuse::KernelDefPtr; +using mindspore::memreuse::KernelRefCount; +using mindspore::memreuse::KernelRefCountPtr; +using mindspore::memreuse::MemReuseUtil; +using mindspore::memreuse::MemReuseUtilPtr; +using mindspore::memreuse::RefCountType; +using MembufPtr = std::shared_ptr; + +namespace mindspore { +namespace memreuse { +class TestMemReuseAllocator : public UT::Common { + public: + TestMemReuseAllocator() : getPyFun_("gtest_input.mem_reuse.TestMemReuseAllocator", true) {} + void SetUp() {} + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +KernelDefPtr GetNewKernelDef(const std::vector &inputs, + const std::vector &outputs, uint32_t stream_id) { + auto kernel_def = std::make_shared(); + kernel_def->set_input_refs(inputs); + kernel_def->set_output_refs(outputs); + kernel_def->set_stream_id(stream_id); + return kernel_def; +} + +void InitMemReuseUtils(MemReuseUtil *mem_reuse_util_ptr) { + // tensor params: ref_count, offset, size, index, + auto tensor_0 = std::make_shared(); + tensor_0->index_ = 0; + tensor_0->size_ = 512; + tensor_0->ref_count_ = 999; + ASSERT_NE(tensor_0, nullptr); + auto tensor_1 = std::make_shared(); + tensor_1->index_ = 1; + tensor_1->size_ = 1024; + tensor_1->ref_count_ = 1; + auto tensor_2 = std::make_shared(); + tensor_2->index_ = 2; + tensor_2->size_ = 1024; + tensor_2->ref_count_ = 2; + auto tensor_3 = std::make_shared(); + tensor_3->index_ = 3; + tensor_3->size_ = 32; + tensor_3->ref_count_ = 1; + auto tensor_4 = std::make_shared(); + tensor_4->index_ = 4; + tensor_4->size_ = 2048; + tensor_4->ref_count_ = 1; + auto tensor_5 = std::make_shared(); + tensor_5->index_ = 5; + tensor_5->size_ = 256; + tensor_5->ref_count_ = 1; + MS_LOG(INFO) << "init all tensor info success."; + + std::vector inputs; + std::vector outputs; + inputs = {tensor_0}; + outputs = {tensor_1}; + auto kernel0 = GetNewKernelDef(inputs, outputs, 0); + inputs = {tensor_1}; + outputs = {tensor_2}; + auto kernel1 = GetNewKernelDef(inputs, outputs, 0); + inputs = {tensor_2}; + outputs = {tensor_3}; + auto kernel2 = GetNewKernelDef(inputs, outputs, 0); + inputs = {tensor_2, tensor_3}; + outputs = {tensor_4}; + auto kernel3 = GetNewKernelDef(inputs, outputs, 0); + inputs = {tensor_4}; + outputs = {tensor_5}; + auto kernel4 = GetNewKernelDef(inputs, outputs, 1); + MS_LOG(INFO) << "init all op info success."; + std::vector tensor_ptr_list{tensor_0, tensor_1, tensor_2, tensor_3, tensor_4, tensor_5}; + std::vector op_ptr_list{kernel0, kernel1, kernel2, kernel3, kernel4}; + + mem_reuse_util_ptr->set_total_refs_list(tensor_ptr_list); + mem_reuse_util_ptr->set_kernel_def_ptr_list(op_ptr_list); +} + +TEST_F(TestMemReuseAllocator, mem_reuse_allocator) { + MS_LOG(INFO) << "mem_resue_allocator UT"; + auto mem_reuse_util_ptr = std::make_shared(); + InitMemReuseUtils(mem_reuse_util_ptr.get()); + auto best_fit_mem_reuse = std::make_shared(); + best_fit_mem_reuse->Reuse(mem_reuse_util_ptr.get()); + MS_LOG(INFO) << "run mem reuse success"; + size_t total_allocated_size = best_fit_mem_reuse->GetAllocatedSize(); + ASSERT_NE(total_allocated_size, 0); + + auto is_reusable_stream = best_fit_mem_reuse->IsReusableStream(1, 3); + ASSERT_EQ(is_reusable_stream, true); +} + +TEST_F(TestMemReuseAllocator, mem_reuse_allocator_add_membuf) { + auto best_fit_mem_reuse = std::make_shared(); + auto tensor_desc = std::make_shared(); + tensor_desc->SetKernelRefCountInfo(0, 1024, kDynamicRefCount); + best_fit_mem_reuse->AddNewMembufPtr(tensor_desc.get(), kDyFac); + auto allocated_size = best_fit_mem_reuse->GetAllocatedSize(); + ASSERT_EQ(allocated_size, 1024); +} + +TEST_F(TestMemReuseAllocator, mem_reuse_allocator_split_membuf) { + auto best_fit_mem_reuse = std::make_shared(); + auto tensor_0 = std::make_shared(); + tensor_0->SetKernelRefCountInfo(0, 2048, kDynamicRefCount); + best_fit_mem_reuse->AddNewMembufPtr(tensor_0.get(), kDyFac); + + auto tensor_1 = std::make_shared(); + tensor_1->SetKernelRefCountInfo(1, 800, kDynamicRefCount); + auto is_split = best_fit_mem_reuse->IsSplit(tensor_1->size_, tensor_0->size_); + ASSERT_EQ(is_split, true); + + best_fit_mem_reuse->SplitMembuf(tensor_1.get(), 0); + auto allocated_size = best_fit_mem_reuse->GetAllocatedSize(); + ASSERT_EQ(allocated_size, 2048); +} + +TEST_F(TestMemReuseAllocator, mem_reuse_allocator_align) { + auto best_fit_mem_reuse = std::make_shared(); + auto size = best_fit_mem_reuse->AlignMemorySize(510); + ASSERT_EQ(size, 1024); +} +} // namespace memreuse +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/mem_reuse/mem_reuse_test.cc b/tests/ut/cpp/pre_activate/mem_reuse/mem_reuse_test.cc new file mode 100644 index 00000000000..d5e67174957 --- /dev/null +++ b/tests/ut/cpp/pre_activate/mem_reuse/mem_reuse_test.cc @@ -0,0 +1,274 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "session/kernel_graph.h" +#include "session/session_basic.h" +#include "session/ascend_session.h" +#include "pre_activate/mem_reuse/kernel_refcount.h" +#include "pre_activate/mem_reuse/mem_reuse_allocator.h" +#include "device/kernel_info.h" +#include "kernel/tbe/tbe_kernel_mod.h" +#include "operator/ops.h" +#include "utils/log_adapter.h" +#include "session/anf_runtime_algorithm.h" +#include "common/utils.h" +#include "pipeline/resource.h" +#include "pre_activate/mem_reuse/mem_reuse.h" + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +namespace mindspore { +namespace memreuse { +using session::KernelGraph; +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +class TestMemReuseWithPy : public UT::Common { + public: + TestMemReuseWithPy() : getPyFun_("gtest_input.mem_reuse.mem_reuse_test", true) {} + void SetUp() {} + void TearDown() {} + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; +static KernelGraphPtr CreateKernelGraph() { + /* CreateKernelGraph() + * @mindspore + * def f(x): + * z=conv2d(x, y) + * ret=relu(z) + * return ret + */ + KernelGraphPtr g = std::make_shared(); + std::vector inputs; + std::vector shp = {1, 3, 3, 4}; + TensorTypePtr tensor_type = std::make_shared(kFloat32); + tensor::DeviceInfo device_info{kOpFormat_NCHW, tensor_type}; + + tensor::TensorPtr y_tensor = std::make_shared(kFloat32->type_id(), shp); + y_tensor->set_device_info(device_info); + tensor::TensorPtr z_tensor = std::make_shared(kFloat32->type_id(), shp); + z_tensor->set_device_info(device_info); + auto y_const = NewValueNode(y_tensor); + auto z_const = NewValueNode(z_tensor); + y_const->set_abstract(y_tensor->ToAbstract()); + z_const->set_abstract(z_tensor->ToAbstract()); + g->MutableInputs()->push_back(y_const); + g->MutableInputs()->push_back(z_const); + + auto p_conv = std::make_shared("Conv2D"); + std::vector input_names = {"x", "y"}; + std::vector output_names = {"output"}; + ValuePtr input_names_v = MakeValue(input_names); + ValuePtr output_names_v = MakeValue(output_names); + p_conv->set_attr("input_names", input_names_v); + p_conv->set_attr("output_names", output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_conv)); + inputs.push_back(y_const); + inputs.push_back(z_const); + + auto kernelptr_first = g->NewCNode(inputs); + kernelptr_first->set_abstract(y_tensor->ToAbstract()); + auto tbe_kernel_pack_first = std::make_shared(); + auto kernel_mod_first = std::make_shared(tbe_kernel_pack_first); + auto kernel_info_first = std::make_shared(); + kernel_info_first->set_kernel_mod(kernel_mod_first); + kernelptr_first->set_kernel_info(kernel_info_first); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW}); + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat32->type_id()}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetKernelType(KernelType::TBE_KERNEL); + builder.SetFusionType(mindspore::kernel::CONVLUTION); + builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_first.get()); + + CNodePtr next_cnode_ptr = kernelptr_first; + auto p_relu = std::make_shared("ReLU6"); + std::vector relu_input_names = {"x"}; + std::vector relu_output_names = {"output"}; + ValuePtr relu_input_names_v = MakeValue(relu_input_names); + ValuePtr relu_output_names_v = MakeValue(relu_output_names); + p_relu->set_attr("input_names", relu_input_names_v); + p_relu->set_attr("output_names", relu_output_names_v); + + inputs.clear(); + inputs.push_back(NewValueNode(p_relu)); + inputs.push_back(next_cnode_ptr); + + auto kernelptr_floor = g->NewCNode(inputs); + kernelptr_floor->set_abstract(y_tensor->ToAbstract()); + auto tbe_kernel_pack_floor = std::make_shared(); + auto kernel_mod_floor = std::make_shared(tbe_kernel_pack_floor); + auto kernel_info_floor = std::make_shared(); + kernel_info_floor->set_kernel_mod(kernel_mod_floor); + kernelptr_floor->set_kernel_info(kernel_info_floor); + KernelBuildInfoBuilder relu_builder; + relu_builder.SetInputsFormat({kOpFormat_NCHW}); + relu_builder.SetOutputsFormat({kOpFormat_NCHW}); + relu_builder.SetInputsDeviceType({kFloat32->type_id()}); + relu_builder.SetOutputsDeviceType({kFloat32->type_id()}); + relu_builder.SetKernelType(KernelType::TBE_KERNEL); + relu_builder.SetFusionType(kernel::FusionType::ELEMWISE); + relu_builder.SetProcessor(kernel::Processor::AICORE); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), kernelptr_floor.get()); + next_cnode_ptr = kernelptr_floor; + + // return res + auto p_return = std::make_shared("return"); + inputs.clear(); + inputs.push_back(NewValueNode(p_return)); + inputs.push_back(next_cnode_ptr); + auto ret = g->NewCNode(inputs); + ret->set_abstract(y_tensor->ToAbstract()); + g->set_return(ret); + return g; +} + +static KernelGraphPtr CreateGraphWithExecOrder() { + /* + * define kernel graph: + * x ----- y + * add ----- z + * mul + * return + */ + auto anf_graph = std::make_shared(); + std::vector shape = {2, 32, 224, 224}; + auto abstract = std::make_shared(kFloat32, shape); + EXPECT_NE(abstract, nullptr); + auto original_x_parameter = anf_graph->add_parameter(); + EXPECT_NE(original_x_parameter, nullptr); + original_x_parameter->set_name("original_x_parameter"); + original_x_parameter->set_abstract(abstract); + auto original_y_parameter = anf_graph->add_parameter(); + EXPECT_NE(original_y_parameter, nullptr); + original_y_parameter->set_name("original_y_parameter"); + original_y_parameter->set_abstract(abstract); + std::vector add_inputs = {NewValueNode(prim::kPrimTensorAdd), original_x_parameter, original_y_parameter}; + auto original_add = anf_graph->NewCNode(add_inputs); + EXPECT_NE(original_add, nullptr); + original_add->set_abstract(abstract); + + auto original_z_parameter = anf_graph->add_parameter(); + EXPECT_NE(original_z_parameter, nullptr); + original_z_parameter->set_name("original_z_parameter"); + original_z_parameter->set_abstract(abstract); + std::vector mul_inputs = {NewValueNode(prim::kPrimMul), original_add, original_z_parameter}; + auto original_mul = anf_graph->NewCNode(mul_inputs); + EXPECT_NE(original_mul, nullptr); + original_mul->set_abstract(abstract); + + std::vector lst = {original_add, original_mul}; + std::vector outputs = {original_mul}; + session::SessionPtr sess = std::make_shared(); + sess->Init(0); + auto kernel_graph = sess->ConstructKernelGraph(lst, outputs); + EXPECT_NE(kernel_graph, nullptr); + + auto inputs = kernel_graph->inputs(); + EXPECT_EQ(inputs.size(), 3); + auto first_input = inputs[0]->cast(); + EXPECT_NE(first_input, nullptr); + EXPECT_EQ(first_input->name(), "original_x_parameter"); + auto second_input = inputs[1]->cast(); + EXPECT_NE(second_input, nullptr); + EXPECT_EQ(second_input->name(), "original_y_parameter"); + auto third_input = inputs[2]->cast(); + EXPECT_NE(third_input, nullptr); + EXPECT_EQ(third_input->name(), "original_z_parameter"); + kernel_graph->SetExecOrderByDefault(); + auto execution_order = kernel_graph->execution_order(); + EXPECT_EQ(execution_order.size(), 2); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[0]), prim::kPrimTensorAdd->name()); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[1]), prim::kPrimMul->name()); + auto new_outputs = kernel_graph->outputs(); + EXPECT_EQ(new_outputs.size(), 1); + EXPECT_EQ(AnfAlgo::GetCNodeName(new_outputs[0]), prim::kPrimMul->name()); + return kernel_graph; +} + +TEST_F(TestMemReuseWithPy, KernelRef) { + KernelRefCountPtr kernel_ref_count_ptr = std::make_shared(); + ASSERT_NE(kernel_ref_count_ptr, nullptr); + int ref_count = kernel_ref_count_ptr->ref_count_; + int offset = kernel_ref_count_ptr->offset_; + size_t size = kernel_ref_count_ptr->size_; + int index = kernel_ref_count_ptr->index_; + ASSERT_EQ(ref_count, 0); + ASSERT_EQ(offset, 0); + ASSERT_EQ(size, 0); + ASSERT_EQ(index, -1); + index = 3; + size = 512; + RefCountType ref_count_type_in = mindspore::memreuse::kDynamicRefCount; + kernel_ref_count_ptr->SetKernelRefCountInfo(index, size, ref_count_type_in); + ASSERT_EQ(kernel_ref_count_ptr->index_, 3); + ASSERT_EQ(kernel_ref_count_ptr->size_, 512); + KernelDefPtr kernel_def_ptr = std::make_shared(); + ASSERT_NE(kernel_def_ptr, nullptr); + ASSERT_EQ(kernel_def_ptr->dirty, false); + MembufPtr membuf_ptr = std::make_shared(); + ASSERT_NE(membuf_ptr, nullptr); + MembufPtr membuf_ptr_x = std::make_shared(0, memreuse::kUnused, 512, 128, 2); + ASSERT_EQ(membuf_ptr_x->status_, memreuse::kUnused); + ASSERT_EQ(membuf_ptr_x->size_, 512); + ASSERT_EQ(membuf_ptr_x->offset_, 128); + ASSERT_EQ(membuf_ptr_x->index_, 2); + ASSERT_EQ(membuf_ptr_x->stream_id_, 0); +} + +TEST_F(TestMemReuseWithPy, ReuseAssignDynamicMemory) { + MemReuseUtilPtr mem_reuse_util_ptr = std::make_shared(); + ASSERT_NE(mem_reuse_util_ptr, nullptr); + auto bestfit_mem_reuse = std::make_shared(); + ASSERT_NE(bestfit_mem_reuse, nullptr); + bestfit_mem_reuse->Reuse(mem_reuse_util_ptr.get()); + auto total_size = bestfit_mem_reuse->GetAllocatedSize(); + ASSERT_EQ(total_size, 0); + KernelGraphPtr kernel_graph = std::make_shared(); + bool ret = mem_reuse_util_ptr->InitDynamicKernelRef(kernel_graph.get()); + ASSERT_EQ(ret, true); +} + +TEST_F(TestMemReuseWithPy, TestSetInfo) { + KernelGraphPtr g = CreateKernelGraph(); + ASSERT_NE(g, nullptr); + g->SetExecOrderByDefault(); + std::vector graphs{g}; + FuncGraphManagerPtr manager = std::make_shared(graphs); + manager->AddFuncGraph(g); + ASSERT_EQ(manager->all_nodes().size(), 8); + MemReuseUtilPtr mem_reuse_util_ptr = std::make_shared(); + ASSERT_NE(mem_reuse_util_ptr, nullptr); + auto ret = mem_reuse_util_ptr->InitDynamicKernelRef(g.get()); + ASSERT_EQ(ret, true); + mem_reuse_util_ptr->SetWorkSpaceList(); + ASSERT_EQ(mem_reuse_util_ptr->total_wk_ref_list_.size(), 0); + mem_reuse_util_ptr->SetReuseRefCount(); + ASSERT_EQ(mem_reuse_util_ptr->total_refs_list_.size(), 0); + auto def_list = mem_reuse_util_ptr->kernel_def_ptr_list(); + ASSERT_EQ(def_list.size(), 0); + auto exec_graph = CreateGraphWithExecOrder(); + ASSERT_NE(exec_graph, nullptr); +} +} // namespace memreuse +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/mem_reuse/stream_reuse_test.cc b/tests/ut/cpp/pre_activate/mem_reuse/stream_reuse_test.cc new file mode 100644 index 00000000000..8ff52f2a3d0 --- /dev/null +++ b/tests/ut/cpp/pre_activate/mem_reuse/stream_reuse_test.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "operator/ops.h" +#include "pre_activate/mem_reuse/stream_reuse.h" +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +using mindspore::memreuse::StreamReuse; + +namespace mindspore { +class TestStreamMemReuse : public UT::Common { + public: + TestStreamMemReuse() : getPyFun_("gtest_input.mem_reuse.TestMemReuseAllocator", true) {} + void SetUp() {} + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestStreamMemReuse, init_reusable_stream_map_test) { + std::unordered_map logic_physic_map{{1, 0}, {2, 8}, {3, 3}}; + std::unordered_map logic_independent_map{{3, 10}, {2, 11}}; + auto stream_reuse = std::make_shared(); + stream_reuse->set_logic_physic_map(logic_physic_map); + stream_reuse->set_logic_independent_map(logic_independent_map); + + auto logic_phyics_map = stream_reuse->GetLogicPhysicsStreamMap(); + for (const auto &logic_physics : logic_phyics_map) { + MS_LOG(INFO) << "[logic_id: " << logic_physics.first << "]"; + for (const auto &physic : logic_physics.second) { + MS_LOG(INFO) << "physic: " << physic; + } + } + MS_LOG(INFO) << "===========UT logic_physic_map size: " << logic_physic_map.size() << "========"; + ASSERT_EQ(logic_physic_map.size(), 3); + stream_reuse->InitReusableStreamMap(); + auto parallel_streams_map = stream_reuse->parallel_streams_map(); + for (const auto ¶llel_streams : parallel_streams_map) { + MS_LOG(INFO) << "[stream id: " << parallel_streams.first << "]"; + for (const auto &stream : parallel_streams.second) { + MS_LOG(INFO) << "parallel stream id: " << stream; + } + } + ASSERT_EQ(parallel_streams_map[7].size(), 1); +} +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/pass/common_subexpression_elimination_test.cc b/tests/ut/cpp/pre_activate/pass/common_subexpression_elimination_test.cc new file mode 100644 index 00000000000..94282edc995 --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/common_subexpression_elimination_test.cc @@ -0,0 +1,198 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "device/kernel_info.h" +#include "pre_activate/pass/common_subexpression_elimination.h" +#include "kernel/kernel_build_info.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" + +namespace mindspore { +namespace opt { +class TestHWCSE : public BackendCommon { + public: + TestHWCSE() : getPyFun_("gtest_input.pre_activate.hw_opt_test", true) {} + ~TestHWCSE() override = default; + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +kernel::KernelBuildInfoPtr CreateKernelBuildInfo(const std::vector &inputs_format, + const std::vector &outputs_format, + const std::vector &inputs_device_type, + const std::vector &outputs_device_type, + KernelType kernel_type) { + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder1; + builder1.SetInputsFormat(inputs_format); + builder1.SetOutputsFormat(outputs_format); + builder1.SetInputsDeviceType(inputs_device_type); + builder1.SetOutputsDeviceType(outputs_device_type); + builder1.SetKernelType(kernel_type); + return builder1.Build(); +} + +TEST_F(TestHWCSE, test_func_graph_cse) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_func_graph_cse", "g1"); + std::vector shp_x{32, 3, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto func_graph = GetFuncGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + FuncGraphManagerPtr manager = Manage(func_graph); + ASSERT_TRUE(manager != nullptr); + ASSERT_EQ(manager->all_nodes().size(), 10); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 8); + + g = getPyFun_.CallAndParseRet("test_func_graph_cse", "g2"); + func_graph = GetFuncGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + manager = Manage(func_graph); + ASSERT_TRUE(manager != nullptr); + ASSERT_EQ(manager->all_nodes().size(), 22); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 14); +} + +TEST_F(TestHWCSE, test_func_graph_cse_with_null_kernel_info) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_func_graph_cse", "g1"); + std::vector shp_x{32, 3, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto func_graph = GetFuncGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + FuncGraphManagerPtr manager = Manage(func_graph); + ASSERT_TRUE(manager != nullptr); + ASSERT_EQ(manager->all_nodes().size(), 10); + auto ret = func_graph->get_return(); + EXPECT_NE(ret, nullptr); + auto mul = ret->input(1); + EXPECT_NE(mul, nullptr); + auto add1 = mul->cast()->input(1); + auto add2 = mul->cast()->input(2); + EXPECT_NE(add1, nullptr); + EXPECT_NE(add2, nullptr); + add1->set_kernel_info(std::make_shared()); + add2->set_kernel_info(std::make_shared()); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + // one of the kernel info is null + add1->set_kernel_info(nullptr); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 10); + // one of the kernel build info is null + add1->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat16}, {kNumberTypeFloat32}, TBE_KERNEL), + add2.get()); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 10); + // both kernel build info is null + AnfAlgo::SetSelectKernelBuildInfo(nullptr, add2.get()); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 8); +} + +TEST_F(TestHWCSE, test_func_graph_cse_with_diff_kernel_info) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_func_graph_cse", "g1"); + std::vector shp_x{32, 3, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto func_graph = GetFuncGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + FuncGraphManagerPtr manager = Manage(func_graph); + ASSERT_TRUE(manager != nullptr); + ASSERT_EQ(manager->all_nodes().size(), 10); + auto ret = func_graph->get_return(); + EXPECT_NE(ret, nullptr); + auto mul = ret->input(1); + EXPECT_NE(mul, nullptr); + auto add1 = mul->cast()->input(1); + auto add2 = mul->cast()->input(2); + EXPECT_NE(add1, nullptr); + EXPECT_NE(add2, nullptr); + add1->set_kernel_info(std::make_shared()); + add2->set_kernel_info(std::make_shared()); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + // Different data type + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, TBE_KERNEL), + add1.get()); + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat16}, {kNumberTypeFloat32}, TBE_KERNEL), + add2.get()); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 10); + // Different format + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, TBE_KERNEL), + add1.get()); + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_NC1HWC0}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, TBE_KERNEL), + add2.get()); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 10); + // Different kernel type + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, TBE_KERNEL), + add1.get()); + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_NC1HWC0}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, UNKNOWN_KERNEL_TYPE), + add2.get()); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 10); + // same kernel build info + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, TBE_KERNEL), + add1.get()); + AnfAlgo::SetSelectKernelBuildInfo( + CreateKernelBuildInfo({kOpFormat_DEFAULT, kOpFormat_DEFAULT}, {kOpFormat_DEFAULT}, + {kNumberTypeFloat32, kNumberTypeFloat32}, {kNumberTypeFloat32}, TBE_KERNEL), + add2.get()); + optimizer->Optimize(func_graph); + ASSERT_EQ(manager->all_nodes().size(), 8); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/pass/convert_const_input_to_attr_test.cc b/tests/ut/cpp/pre_activate/pass/convert_const_input_to_attr_test.cc new file mode 100644 index 00000000000..99130efd5df --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/convert_const_input_to_attr_test.cc @@ -0,0 +1,152 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "operator/ops.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "pre_activate/pass/convert_const_input_to_attr.h" +#include "utils/utils.h" +#include "common/utils.h" + +namespace mindspore { +namespace opt { +class TestHWConstInputToAttr : public BackendCommon { + public: + TestHWConstInputToAttr() : getPyFun_("gtest_input.pre_activate.convert_const_input_test", true) {} + ~TestHWConstInputToAttr() override = default; + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestHWConstInputToAttr, test_reshape) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_reshape_input_to_attr", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_reshape_input_to_attr", "after"); + ASSERT_TRUE(g_after != nullptr); + std::vector shp_x{2, 3}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} + +TEST_F(TestHWConstInputToAttr, test_cast) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_cast_input_to_attr", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_cast_input_to_attr", "after"); + ASSERT_TRUE(g_after != nullptr); + std::vector shp_x{2, 3}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} + +TEST_F(TestHWConstInputToAttr, test_transpose) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_transpose_input_to_attr", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_transpose_input_to_attr", "after"); + ASSERT_TRUE(g_after != nullptr); + std::vector shp_x{2, 2, 3}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} + +TEST_F(TestHWConstInputToAttr, test_onehot) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_onehot_input_to_attr", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_onehot_input_to_attr", "after"); + ASSERT_TRUE(g_after != nullptr); + + auto ret = g->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto cnode = ret->input(1)->cast(); + EXPECT_FALSE(AnfAlgo::HasNodeAttr("depth", cnode)); + EXPECT_FALSE(CheckEqualGraph(g, g_after)); + + std::vector shp_x{16}; + auto x_abstract = std::make_shared(kInt32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + ret = func_graph->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto make_tuple = ret->input(1)->cast(); + ASSERT_TRUE(make_tuple != nullptr); + EXPECT_NE(make_tuple->input(1), nullptr); + EXPECT_NE(make_tuple->input(1)->cast(), nullptr); + cnode = make_tuple->input(1)->cast(); + EXPECT_TRUE(AnfAlgo::HasNodeAttr("depth", cnode)); + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} + +TEST_F(TestHWConstInputToAttr, test_strided_slice_grad) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_strided_slice_grad_input_to_attr", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_strided_slice_grad_input_to_attr", "after"); + ASSERT_TRUE(g_after != nullptr); + + auto ret = g->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto cnode = ret->input(1)->cast(); + EXPECT_FALSE(AnfAlgo::HasNodeAttr("shapex", cnode)); + EXPECT_FALSE(AnfAlgo::HasNodeAttr("begin", cnode)); + EXPECT_FALSE(AnfAlgo::HasNodeAttr("end", cnode)); + EXPECT_FALSE(AnfAlgo::HasNodeAttr("strides", cnode)); + EXPECT_FALSE(CheckEqualGraph(g, g_after)); + + std::vector shp_x{16, 1, 1024}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + ret = func_graph->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto make_tuple = ret->input(1)->cast(); + ASSERT_TRUE(make_tuple != nullptr); + EXPECT_NE(make_tuple->input(1), nullptr); + EXPECT_NE(make_tuple->input(1)->cast(), nullptr); + cnode = make_tuple->input(1)->cast(); + EXPECT_TRUE(AnfAlgo::HasNodeAttr("shapex", cnode)); + EXPECT_TRUE(AnfAlgo::HasNodeAttr("begin", cnode)); + EXPECT_TRUE(AnfAlgo::HasNodeAttr("end", cnode)); + EXPECT_TRUE(AnfAlgo::HasNodeAttr("strides", cnode)); + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc b/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc new file mode 100644 index 00000000000..083e4168ceb --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc @@ -0,0 +1,111 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "ir/anf.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "pre_activate/pass/convert_const_input_to_tensor_input.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +class TestHWConstInputToTensorInput : public BackendCommon { + public: + TestHWConstInputToTensorInput() : getPyFun_("gtest_input.pre_activate.convert_const_input_test", true) {} + ~TestHWConstInputToTensorInput() override = default; + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestHWConstInputToTensorInput, test_onehot_fg) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_onehot_input_to_tensor1", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_onehot_input_to_tensor1", "after_func_graph"); + ASSERT_TRUE(g_after != nullptr); + std::vector shp_x{16}; + auto x_abstract = std::make_shared(kInt32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetFuncGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + EXPECT_FALSE(CheckEqualGraph(func_graph, g_after)); + + auto optimizer = std::make_shared(); + MS_EXCEPTION_IF_NULL(optimizer); + auto pm = std::make_shared(); + MS_EXCEPTION_IF_NULL(pm); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + optimizer->Optimize(func_graph); + auto ret = func_graph->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto cnode = ret->input(1)->cast(); + EXPECT_FALSE(AnfAlgo::HasNodeAttr("depth", cnode)); + EXPECT_TRUE(IsValueNode(cnode->input(2))); +} + +TEST_F(TestHWConstInputToTensorInput, test_onehot_kg) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_onehot_input_to_tensor2", "before"); + ASSERT_TRUE(g != nullptr); + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_onehot_input_to_tensor2", "after_kernel_graph"); + ASSERT_TRUE(g_after != nullptr); + EXPECT_FALSE(CheckEqualGraph(g, g_after)); + std::vector shp_x{16}; + auto x_abstract = std::make_shared(kInt32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + auto ret = func_graph->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto cnode = ret->input(1)->cast()->input(1)->cast(); + EXPECT_TRUE(AnfAlgo::HasNodeAttr("depth", cnode)); + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} + +TEST_F(TestHWConstInputToTensorInput, test_value_tuple_tensor_input) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_dropout_gen_mask_tuple_input_to_tensor", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{1}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto kernel_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(kernel_graph != nullptr); + + auto ret = kernel_graph->get_return(); + ASSERT_TRUE(ret != nullptr); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast(), nullptr); + auto cnode = ret->input(1)->cast()->input(1)->cast(); + EXPECT_EQ(AnfAlgo::GetCNodeName(cnode), prim::kPrimDropoutGenMask->name()); + auto input1 = cnode->input(1); + ASSERT_TRUE(input1 != nullptr); + EXPECT_TRUE(IsValueNode(input1)); + auto tensor = input1->cast()->value()->cast(); + ASSERT_TRUE(tensor != nullptr); + auto data = tensor->data_c(false); + EXPECT_EQ(vector((int *)data, (int *)data + 4), vector({2, 4, 2, 2})); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/pass/convert_tuple_input_to_dynamic_input_test.cc b/tests/ut/cpp/pre_activate/pass/convert_tuple_input_to_dynamic_input_test.cc new file mode 100644 index 00000000000..82591ec76cd --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/convert_tuple_input_to_dynamic_input_test.cc @@ -0,0 +1,59 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "ir/anf.h" +#include "ir/meta_tensor.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +#include "session/anf_runtime_algorithm.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "pre_activate/pass/convert_tuple_input_to_dynamic_input.h" +#include "utils/utils.h" + +namespace mindspore { +namespace opt { +class TestHWConstTupleInputToDynamicInput : public BackendCommon { + public: + TestHWConstTupleInputToDynamicInput() + : getPyFun_("gtest_input.pre_activate.convert_tuple_input_to_dynamic_input_test", true) {} + ~TestHWConstTupleInputToDynamicInput() override = default; + + public: + UT::PyFuncGraphFetcher getPyFun_; +}; + +TEST_F(TestHWConstTupleInputToDynamicInput, test_convert_tuple_input_to_dynamic_input) { + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_convert_tuple_input_to_dynamic_input", "before"); + ASSERT_TRUE(g != nullptr); + std::vector shp_x{1, 11, 20, 1, 2}; + auto x_abstract = std::make_shared(kFloat32, shp_x); + AbstractBasePtrList args_spec_list{x_abstract}; + auto func_graph = GetKernelGraph(g, args_spec_list); + ASSERT_TRUE(func_graph != nullptr); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + optimizer->Optimize(func_graph); + + FuncGraphPtr g_after = getPyFun_.CallAndParseRet("test_convert_tuple_input_to_dynamic_input", "after"); + ASSERT_TRUE(g_after != nullptr); + EXPECT_TRUE(CheckEqualGraph(func_graph, g_after)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/pass/eliminate_redundant_op_test.cc b/tests/ut/cpp/pre_activate/pass/eliminate_redundant_op_test.cc new file mode 100644 index 00000000000..4523a5fa99d --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/eliminate_redundant_op_test.cc @@ -0,0 +1,273 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/backend_common_test.h" +#include "kernel/kernel.h" +#include "operator/ops.h" +#include "ir/meta_tensor.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "common/py_func_graph_fetcher.h" +// #include "device/optimizer/pass/insert_trans_op.h" +#include "pre_activate/ascend/format_type/insert_cast.h" +#include "pre_activate/pass/eliminate_redundant_op.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/common/pass_manager.h" +#include "utils/utils.h" +#include "utils/context/ms_context.h" +#include "session/anf_runtime_algorithm.h" +#include "device/kernel_info.h" + +#define private public +#define protected public +#include "pre_activate/ascend/format_type/insert_trans_op.h" +#undef private +#undef protected + +namespace mindspore { +namespace opt { +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; + +class TestHWEliminateRedundantOp : public BackendCommon { + public: + TestHWEliminateRedundantOp() : getPyFun_("gtest_input.pre_activate.eliminate_redundant_op_test", true) {} + ~TestHWEliminateRedundantOp() override = default; + + UT::PyFuncGraphFetcher getPyFun_; +}; + +class MockEliminate5To4And4To5KernelSelect : public KernelSelect { + public: + MockEliminate5To4And4To5KernelSelect() = default; + ~MockEliminate5To4And4To5KernelSelect() override = default; + void SelectKernel(const CNodePtr &cnode) override { + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NCHW"}); + builder.SetInputsDeviceType({kFloat16->type_id()}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), cnode.get()); + } +}; + +TEST_F(TestHWEliminateRedundantOp, test_eliminate_5to4_4to5) { + /* + * def test_eliminate_5to4_4to5(x, y): + * sum = add(x, y) + * res = sub(sum, y) + * output = make_tuple(res) + * return output + */ + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_eliminate_5to4_4to5", "before"); + // Renormalize func_graph to infer and set shape and type information. + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + EXPECT_NE(kg, nullptr); + + // Set selectedKernelInfo for add, sub + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast()->input(1), nullptr); + + auto tuple = ret->input(1)->cast()->input(1); + EXPECT_NE(tuple, nullptr); + EXPECT_NE(tuple->cast()->input(1), nullptr); + + auto sub = tuple->cast()->input(1); + EXPECT_NE(sub, nullptr); + EXPECT_NE(sub->cast()->input(1), nullptr); + auto add = sub->cast()->input(1); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0", "NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + + sub->set_kernel_info(std::make_shared()); + add->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), sub.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), add.get()); + + // Do insert_trans_op_ pass of hardware opt + auto graph_optimizer = std::make_shared(); + auto pass_manager = std::make_shared(); + auto insert_trans_op_pass = std::make_shared(); + insert_trans_op_pass->kernel_select_ = std::make_shared(); + pass_manager->AddPass(insert_trans_op_pass); + graph_optimizer->AddPassManager(pass_manager); + auto new_g1 = graph_optimizer->Optimize(kg); + EXPECT_NE(new_g1, nullptr); + FuncGraphPtr g_after1 = getPyFun_.CallAndParseRet("test_eliminate_5to4_4to5", "after1"); + EXPECT_NE(g_after1, nullptr); + EXPECT_TRUE(CheckEqualGraph(g_after1, new_g1)); + + // Do eliminate_5to4_4to5_ pass of hardware opt + auto graph_optimizer2 = std::make_shared(); + auto pass_manager2 = std::make_shared(); + pass_manager2->AddPass(std::make_shared()); + graph_optimizer2->AddPassManager(pass_manager2); + auto new_g2 = graph_optimizer2->Optimize(new_g1); + EXPECT_NE(new_g2, nullptr); + FuncGraphPtr g_after2 = getPyFun_.CallAndParseRet("test_eliminate_5to4_4to5", "after2"); + EXPECT_NE(g_after2, nullptr); + EXPECT_TRUE(CheckEqualGraph(g_after2, new_g2)); +} + +TEST_F(TestHWEliminateRedundantOp, test_eliminate_cast) { + /* + * def test_eliminate_cast(x, y): + * sum = add(x, y) + * res = sub(sum, y) + * output = make_tuple(res) + * return output + */ + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_eliminate_cast", "before"); + // Renormalize func_graph to infer and set shape and type information. + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + EXPECT_NE(kg, nullptr); + + // Set selectedKernelInfo for add, sub + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast()->input(1), nullptr); + + auto tuple = ret->input(1)->cast()->input(1); + EXPECT_NE(tuple, nullptr); + EXPECT_NE(tuple->cast()->input(1), nullptr); + + auto sub = tuple->cast()->input(1); + EXPECT_NE(sub, nullptr); + EXPECT_NE(sub->cast()->input(1), nullptr); + auto add = sub->cast()->input(1); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0", "NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + + sub->set_kernel_info(std::make_shared()); + add->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), sub.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), add.get()); + + // Do insert_trans_op_ pass of hardware opt + auto graph_optimizer = std::make_shared(); + auto pass_manager = std::make_shared(); + pass_manager->AddPass(std::make_shared()); + graph_optimizer->AddPassManager(pass_manager); + auto new_g1 = graph_optimizer->Optimize(kg); + EXPECT_NE(new_g1, nullptr); + FuncGraphPtr g_after1 = getPyFun_.CallAndParseRet("test_eliminate_cast", "after1"); + EXPECT_NE(g_after1, nullptr); + EXPECT_TRUE(CheckEqualGraph(g_after1, new_g1)); + + // Do eliminate_5to4_4to5_ pass of hardware opt + auto graph_optimizer2 = std::make_shared(); + auto pass_manager2 = std::make_shared(); + pass_manager2->AddPass(std::make_shared()); + graph_optimizer2->AddPassManager(pass_manager2); + auto new_g2 = graph_optimizer2->Optimize(new_g1); + EXPECT_NE(new_g2, nullptr); + FuncGraphPtr g_after2 = getPyFun_.CallAndParseRet("test_eliminate_cast", "after2"); + EXPECT_NE(g_after2, nullptr); + EXPECT_TRUE(CheckEqualGraph(g_after2, new_g2)); +} + +TEST_F(TestHWEliminateRedundantOp, test_eliminate_cast_depend_cast) { + /* + * def test_eliminate_cast_depend_cast(x, y): + * sum = add(x, y) + * sum_depend = depend(sum, x) + * res = sub(sum_depend, y) + * output = make_tuple(res) + * return output + */ + FuncGraphPtr g = getPyFun_.CallAndParseRet("test_eliminate_cast_depend_cast", "before"); + // Renormalize func_graph to infer and set shape and type information. + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + auto kg = GetKernelGraph(g, args_spec_list); + EXPECT_NE(kg, nullptr); + + // Set selectedKernelInfo for add, sub + auto ret = kg->get_return(); + EXPECT_NE(ret->input(1), nullptr); + EXPECT_NE(ret->input(1)->cast()->input(1), nullptr); + + auto tuple = ret->input(1)->cast()->input(1); + EXPECT_NE(tuple, nullptr); + EXPECT_NE(tuple->cast()->input(1), nullptr); + + auto sub = tuple->cast()->input(1); + EXPECT_NE(sub, nullptr); + EXPECT_NE(sub->cast()->input(1), nullptr); + auto depend = sub->cast()->input(1); + EXPECT_NE(depend, nullptr); + EXPECT_NE(depend->cast()->input(1), nullptr); + + auto depend2 = depend->cast()->input(1); + EXPECT_NE(depend2, nullptr); + EXPECT_NE(depend2->cast()->input(1), nullptr); + + auto depend3 = depend2->cast()->input(1); + EXPECT_NE(depend3, nullptr); + EXPECT_NE(depend3->cast()->input(1), nullptr); + auto add = depend3->cast()->input(1); + + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({"NC1HWC0", "NC1HWC0"}); + builder.SetOutputsFormat({"NC1HWC0"}); + builder.SetInputsDeviceType({kFloat16->type_id(), kFloat16->type_id()}); + builder.SetOutputsDeviceType({kFloat16->type_id()}); + + sub->set_kernel_info(std::make_shared()); + add->set_kernel_info(std::make_shared()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), sub.get()); + AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), add.get()); + + // Do insert_trans_op_ pass of hardware opt + auto graph_optimizer = std::make_shared(); + auto pass_manager = std::make_shared(); + pass_manager->AddPass(std::make_shared()); + graph_optimizer->AddPassManager(pass_manager); + auto new_g1 = graph_optimizer->Optimize(kg); + EXPECT_NE(new_g1, nullptr); + FuncGraphPtr g_after1 = getPyFun_.CallAndParseRet("test_eliminate_cast_depend_cast", "after1"); + EXPECT_NE(g_after1, nullptr); + EXPECT_TRUE(CheckEqualGraph(g_after1, new_g1)); + + // Do eliminate_5to4_4to5_ pass of hardware opt + auto graph_optimizer2 = std::make_shared(); + auto pass_manager2 = std::make_shared(); + pass_manager2->AddPass(std::make_shared()); + graph_optimizer2->AddPassManager(pass_manager2); + auto new_g2 = graph_optimizer2->Optimize(new_g1); + EXPECT_NE(new_g2, nullptr); + FuncGraphPtr g_after2 = getPyFun_.CallAndParseRet("test_eliminate_cast_depend_cast", "after2"); + EXPECT_NE(g_after2, nullptr); + EXPECT_TRUE(CheckEqualGraph(g_after2, new_g2)); +} + +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pre_activate/pass/getitem_tuple_test.cc b/tests/ut/cpp/pre_activate/pass/getitem_tuple_test.cc new file mode 100644 index 00000000000..b172e1b3519 --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/getitem_tuple_test.cc @@ -0,0 +1,51 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "session/ascend_session.h" +#include "pipeline/resource.h" +#include "operator/ops.h" +#include "ir/manager.h" +#include "debug/anf_ir_dump.h" +#include "utils/utils.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/pass/getitem_tuple.h" + +namespace mindspore { +namespace opt { +class TestHWGetitemTuple : public BackendCommon { + public: + TestHWGetitemTuple() : get_py_fun_("gtest_input.pre_activate.getitem_tuple", true) {} + ~TestHWGetitemTuple() override = default; + + public: + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWGetitemTuple, test_getitem_tuple) { + FuncGraphPtr g_before = get_py_fun_.CallAndParseRet("test_getitem_tuple", "before"); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + auto new_graph = optimizer->Optimize(g_before); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_getitem_tuple", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/pre_activate/pass/optimize_dependence_test.cc b/tests/ut/cpp/pre_activate/pass/optimize_dependence_test.cc new file mode 100644 index 00000000000..3f59b6159a3 --- /dev/null +++ b/tests/ut/cpp/pre_activate/pass/optimize_dependence_test.cc @@ -0,0 +1,52 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/backend_common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pre_activate/common/optimizer.h" +#include "pre_activate/pass/optimize_dependence.h" + +namespace mindspore { +namespace opt { + +class TestHWOptimizeDependence : public BackendCommon { + public: + TestHWOptimizeDependence() : get_py_fun_("gtest_input.pre_activate.optimize_dependence_test", true) {} + ~TestHWOptimizeDependence() override = default; + + UT::PyFuncGraphFetcher get_py_fun_; +}; + +TEST_F(TestHWOptimizeDependence, test_optimize_dependence) { + /* + * def test_eliminate_depend_input2(x, y, z): + * new_z = four2five(z) + * depend_intput = depend(y, new_z) + * sum = add(x, depend_intput) + * return sum + */ + FuncGraphPtr g = get_py_fun_.CallAndParseRet("test_optimize_dependence", "before"); + + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + FuncGraphPtr new_graph = optimizer->Optimize(g); + + FuncGraphPtr g_after = get_py_fun_.CallAndParseRet("test_optimize_dependence", "after"); + EXPECT_TRUE(CheckEqualGraph(g_after, new_graph)); +} +} // namespace opt +} // namespace mindspore diff --git a/tests/ut/cpp/pynative/pynative_execute_test.cc b/tests/ut/cpp/pynative/pynative_execute_test.cc new file mode 100644 index 00000000000..34184516c2a --- /dev/null +++ b/tests/ut/cpp/pynative/pynative_execute_test.cc @@ -0,0 +1,115 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "pipeline/parse/python_adapter.h" +#include "pipeline/parse/data_converter.h" +#include "operator/ops.h" +#include "pynative/pynative_execute.h" +#include "utils/context/ms_context.h" +#include "utils/utils.h" + +namespace py = pybind11; +using pybind11::literals::operator"" _a; +using Tensor = mindspore::tensor::Tensor; +using TensorPtr = mindspore::tensor::TensorPtr; + +namespace mindspore { +namespace pynative { +class TestPynativeExecute : public UT::Common { + public: + TestPynativeExecute() {} +}; + +inline ValuePtr PyAttrValue(const py::object& obj) { + ValuePtr converted_ret; + bool converted = parse::ConvertData(obj, &converted_ret); + if (!converted) { + MS_LOG(EXCEPTION) << "attribute convert error with type:" << std::string(py::str(obj)); + } + return converted_ret; +} + +OpExecInfoPtr ConstructOpExecInfo() { + py::str op_name = "Conv2D"; + py::object tensor_py_module = py::module::import("mindspore.common.tensor").attr("Tensor"); + py::object np_py_module = py::module::import("numpy"); + py::object np_ones = np_py_module.attr("ones"); + py::object np_float32 = np_py_module.attr("float32"); + py::tuple weight_dim = py::make_tuple(64, 3, 3, 3); + py::object weight = tensor_py_module(np_float32(np_ones(weight_dim))); + py::tuple op_params = py::make_tuple(weight); + py::tuple inputs_dim = py::make_tuple(1, 3, 6, 6); + py::object input = tensor_py_module(np_float32(np_ones(inputs_dim))); + py::tuple op_inputs = py::make_tuple(input, weight); + + py::tuple kernel_size = py::make_tuple(3, 3); + py::dict op_attrs = py::dict("out_channel"_a = 64, "kernel_size"_a = kernel_size, "mode"_a = 1, "pad_mode"_a = "same", + "stride"_a = 1, "dilation"_a = 1, "group"_a = 1, "data_format"_a = kOpFormat_NCHW); + + auto conv_obj = prim::GetPythonOps("conv2d_prim", "gtest_input.pynative"); + py::none py_none; + py::tuple op_mask = py::make_tuple(0, 1); + return GenerateOpExecInfo(py::make_tuple(conv_obj, op_name, op_inputs, op_mask)); +} + +TEST_F(TestPynativeExecute, TestRunOpInVM) { + py::tuple result; + PynativeStatusCode status; + auto op_exec_info_ptr = ConstructOpExecInfo(); + result = pynative::RunOpInVM(op_exec_info_ptr, &status); + ASSERT_EQ(status, PYNATIVE_SUCCESS); +} + +TEST_F(TestPynativeExecute, TestRunOp) { + py::none py_none; + auto op_exec_info_ptr = ConstructOpExecInfo(); + py::tuple outputs = pynative::RunOp(py::make_tuple(op_exec_info_ptr->py_primitive, op_exec_info_ptr->op_name, + op_exec_info_ptr->op_inputs, op_exec_info_ptr->inputs_mask)); + if (outputs.size() == 0) { + FAIL(); + } else { + SUCCEED(); + } +} + +TEST_F(TestPynativeExecute, TestCreateContext) { + auto ctx3 = MsContext::GetInstance(); + ASSERT_EQ(ctx3->backend_policy(), "vm"); + ASSERT_EQ(ctx3->device_target(), "CPU"); + + ctx3->set_backend_policy("ge_only"); + ctx3->set_device_target("GPU"); + auto ctx4 = MsContext::GetInstance(); + + ASSERT_EQ(ctx3.get(), ctx4.get()); + ASSERT_EQ(ctx4->backend_policy(), "ge_only"); + ASSERT_EQ(ctx4->device_target(), "GPU"); +} + +TEST_F(TestPynativeExecute, TestDefaultContext) { + auto ctx = MsContext::GetInstance(); + + ASSERT_EQ(std::string(ctx->backend_policy()), "ge_only"); + + auto ctx2 = MsContext::GetInstance(); + + ASSERT_EQ(ctx.get(), ctx2.get()); +} + +} // namespace pynative +} // namespace mindspore diff --git a/tests/ut/cpp/python_input/__init__.py b/tests/ut/cpp/python_input/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/__init__.py b/tests/ut/cpp/python_input/gtest_input/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/ir/__init__.py b/tests/ut/cpp/python_input/gtest_input/ir/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/ir/clone_test.py b/tests/ut/cpp/python_input/gtest_input/ir/clone_test.py new file mode 100644 index 00000000000..7fa9c768714 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/ir/clone_test.py @@ -0,0 +1,61 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" Test for GraphCloner """ +from mindspore.ops import Primitive +scala_add = Primitive('scalar_add') +scalar_mul = Primitive('scalar_mul') + +def test_clone_simple(): + def f(x, y): + a = scalar_mul(x, x) + b = scalar_mul(y, y) + c = scala_add(a, b) + return c + return f + +def test_clone_closure(x, y): + def j(z): + a = x + y + b = a + z + return b + c = j(3) + return c + +def test_clone_scoping(): + """ test_clone_scoping """ + print("run python test_clone_scoping") + def f(x, y): + def h(z): + # No dependency on f, so not nested and not cloned + return z * z + def g(z): + def gg(): + return z + z + # Depends on f, therefore cloned + return x + y + gg() + def i(q): + # Depends on f, therefore cloned + return g(1) * q + return g(1) + h(x) + i(y) + return f + +def test_clone_total(): + print("run python test_clone_total") + def clone_total(y): + return clone_total_sub(y) + 3 + return clone_total + +def clone_total_sub(x): + return x * x diff --git a/tests/ut/cpp/python_input/gtest_input/ir/manager_test.py b/tests/ut/cpp/python_input/gtest_input/ir/manager_test.py new file mode 100644 index 00000000000..dda7dd14f9c --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/ir/manager_test.py @@ -0,0 +1,58 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" Test for manager """ + +def ir_get_fn(x, y): + return x - y + +def test_flat(x): + return x + +def test_nested(x): + def g(): + return x + return g + +def test_fake_nested(x): + return x + +def test_recurse(x): + def g(): + return g() + x + return g + +def test_calls(x): + a = x + x + def h(): + return a + def g(): + return h() + return g() + +# pylint: disable=unused-argument +def test_unused_param(x, y): + return x * x + + +def test_cannot_replace_return(x): + return x * x + +# custom test function +def test_custom(x, y, z): + def g(x1, y1): + def h(x2): + return x2 + y1 + z + return h(x1) + return g(x, y) diff --git a/tests/ut/cpp/python_input/gtest_input/mem_reuse/__init__.py b/tests/ut/cpp/python_input/gtest_input/mem_reuse/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/mem_reuse/mem_reuse_test.py b/tests/ut/cpp/python_input/gtest_input/mem_reuse/mem_reuse_test.py new file mode 100644 index 00000000000..074aaf4a889 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/mem_reuse/mem_reuse_test.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive +import mindspore as ms + +add = P.TensorAdd() +reshape = P.Reshape() + +def test_shape_add(x1, x2, y1, y2, z1, z2): + sum1 = add(x1, x2) + sum2 = add(y1, y2) + sum3 = add(z1, z2) + reshape_sum1 = reshape(sum1, (2, 2, 3, 1)) + reshape_sum2 = reshape(sum2, (2, 2, 3, 1)) + reshape_sum3 = reshape(sum3, (2, 2, 3, 1)) + sum = add(reshape_sum1, reshape_sum2) + sum = add(sum, reshape_sum3) + return sum diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/__init__.py b/tests/ut/cpp/python_input/gtest_input/optimizer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/ad/__init__.py b/tests/ut/cpp/python_input/gtest_input/optimizer/ad/__init__.py new file mode 100644 index 00000000000..0bc75849679 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/ad/__init__.py @@ -0,0 +1,9 @@ +""" +@File : __init__.py +@Author: +@Date : 2019-01-23 16:36 +@Desc : +""" + +from .ad_test import * + diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/ad/ad_test.py b/tests/ut/cpp/python_input/gtest_input/optimizer/ad/ad_test.py new file mode 100644 index 00000000000..154b0635e41 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/ad/ad_test.py @@ -0,0 +1,258 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np + +import mindspore as ms +from dataclasses import dataclass +from mindspore.common.tensor import Tensor +from mindspore.ops import Primitive +from mindspore.model_zoo.resnet import resnet50 +scala_add = Primitive('scalar_add') + +@dataclass +class Point: + x: float + y: float + def abs(self): + return (self.x ** 2 + self.y ** 2) ** 0.5 + +def scalar_add(x, y): + """Implement `scalar_add`.""" + return x + y + +def scalar_mul(x, y): + """Implement `scalar_mul`.""" + return x * y + +# Test:common function +def test_null(x, y): + return scala_add(10.0, 28.0 / 43.0) + +def test_grad_add(x, y): + return scala_add(x, y) + +def test_grad_expr(x, y): + return x**3.0 * y**4.0 + +def test_constant(x): + return 18.0 * x + +def test_dup_args_in_call(x): + """The naive gradient update rule fails when a function's arguments + contain the same variable more than once.""" + return x * x + +def test_quadruple_args_in_call(x): + """Test that duplicated arguments still cause no problem even if + there are four of them.""" + def g(a, b, c, d): + return a * b * c * d + return g(x, x, x, x) + +def test_tuples(x, y): + tup = scala_add(x, y), x * y + z = scala_add(tup[0], tup[1]) + return z + +def test_dataclass(x, y): + pt = Point(x, y) + return pt.x * pt.y + +def test_dataclass_2(x, y): + pt = Point(x, y) + return pt.abs() + +def test_hof(a, b): + """Test higher order functions.""" + def f(g, x): + return g(x) * g(scala_add(x, 10.0)) + def g(x): + return x * b + return scala_add(f(g, a), f(g, b)) + + +def test_hof_tup(a, b): + """Test higher order functions.""" + def f(gh, x, y): + g, h = gh + return scalar_mul(g(x, y), h(x, y)) + + return f((scalar_add, scalar_mul), a, b) + + +def test_simple_closure(a, b): + """Test some trivial closures.""" + def f(): + return a + 1.0 + + def g(): + return b + 2.0 + return f() * g() + +def test_closure(a): + """This is the closure test in the paper.""" + def x1(b): + + def x4(c): + return b + return x4 + x2 = x1(a) + x3 = x2(1.0) + return x3 + +def test_if(a, b): + # This is max, but what this is really testing is the most basic + # if statement, so I prefer to name the test 'test_if' + if a > b: + return a + else: + return b + +def test_if2(a, b): + if a > b: + return a * a + else: + return b + b + +def test_fact(x): + def fact(n): + if n <= 1: + return 1 + else: + return n * fact(n - 1) + return fact(x) + +def test_while(x): + rval = x + while rval < 100: + rval = rval * rval + return rval + +def test_while_2(x, y, z): + rval = 0 + # Cannot compare to 0 or finite diff is unstable + while x > -0.1: + rval = rval + y + x = x - z + return rval + +def test_pow10(x): + v = x + j = 0 + while j < 3: + i = 0 + while i < 3: + v = v * x + i = i + 1 + j = j + 1 + return v + +def test_nested_closure(x): + a = x * x + b = x + 5 + + def f(): + def g(): + return a + b + + def h(): + return a * b + return g if x < 0 else h + return f()() + +def test_functions_in_tuples(x, y): + tup = scalar_add, scalar_mul + f, g = tup + return f(x, y) + g(x, y) + +def test_closures_in_tuples(x, y): + def f(): + return x * y + + def g(): + return scala_add(x, y) + + tup = f, g + ff, gg = tup + return scala_add(ff(), gg()) + +# tensor test +def test_tensor_add(x, y): + t1 = Tensor(np.ones(x)) + t2 = Tensor(np.zeros(y), ms.float32) + return t1 + t2 + +def test_tensor_set_type(x): + t = Tensor(x) + t.set_dtype(ms.float32) + return t + +def test_tensor_mul(x, y): + x = Tensor(x) + y = Tensor(y) + z = x * y + + return z + +def test_tensor_sub(x, y): + x = Tensor(x) + y = Tensor(y) + z = x - y + return z + +relu = Primitive('relu') +# Extension test +def test_ops_fn(x): + foo = relu(x) + return foo + +def test_clone_simple(x, y): + a = x * x + b = y * y + c = a + b + return c + +def test_more_closure(a, b): + """Test some trivial closures.""" + z = 1 + def f(): + return a + z + + def g(): + return b + 2.0 + return f() * g() + +def test_more_hof(a, b): + """Test higher order functions.""" + def f(g, h, x): + return g(x) * h(x) * g(x + 10.0) + + def g(x): + return x * b + + def h(x): + return x * a + + return scala_add(f(g, h, a), f(g, h, b)) + +def test_constant_output(x, y): + return 1 + + +# test resnet +def test_resnet_construct(x): + # not right model to import + network = resnet50() + return network.construct(x) diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/cconv_test.py b/tests/ut/cpp/python_input/gtest_input/optimizer/cconv_test.py new file mode 100644 index 00000000000..c3be324406f --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/cconv_test.py @@ -0,0 +1,163 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : opt_cconv.py +@Author : wangqiuliang +@Date : 2019-03-26 +@Desc : parse python function for ut of cconv +""" + +class FnDict: + def __init__(self): + self.fnDict = {} + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + def __getitem__(self, name): + return self.fnDict[name] + +# pylint: disable=unused-variable +def get_test_cconv_fn(tag): + """ get_test_cconv_fn """ + fns = FnDict() + @fns + def test_straight(x, y): + return x * x + y * y + @fns + def test_simple_closure(x): + def g(): + return x + return g + @fns + def test_max(x, y): + if x > y: + return x + return y + @fns + def test_deep_nesting(x): + def f(y): + def g(z): + def h(): + return y + z + return h() + return g(x) + a = f(x + 1) + b = f(x - 3) + return a() + b() + @fns + def test_return_in_double_while(x): + while x > 0: + while x > 0: + x = x -1 + return x + return -1 + @fns + def test_pow10(x): + v = x + j = 0 + while j < 3: + i = 0 + while i < 3: + v = v * x + i = i + 1 + j = j + 1 + return v + @fns + def test_closure_as_simple_fv(x): + def f(): + return x + + def g(): + return f() + return g() + @fns + def test_closure_as_fv(x, y): + def ax(): + return x + def bx(): + return ax() + def cx(): + return bx() + def gx(): + return cx() + def ay(): + return y + def by(): + return ay() + def cy(): + return by() + def gy(): + return cy() + def g(): + return gx() + gy() + return g() + @fns + def test_closure_as_double_fv(x): + def a(): + return x + def b(y): + def e(): + return y + return e() + a() + def g(y): + def c(): + return b(y) + return c() + return g(1) + @fns + def test_closure_lift_same_param(x): + def a(): + return x + def b(): + return a() + def c(): + return x + def d(): + return c() + def f(y): + def e(): + return y + return e() + d() + b() + def g(): + return f(1) + return g() + @fns + def test_closure_as_loop(x, lower_bound): + def fv_func(y): + return x * y + ret = 0 + i = lower_bound + while i < 100: + ele = fv_func(i) + i += 1 + ret += ele + return ret + @fns + def test_closure_lift_cnode(x): + def a(i, j): + return i, j + def f(): + def e(): + return x + m = a(x, e()) + n = a(m, m) + def b(): + return m, n + def d(): + return n, m + return b(), d() + def g(): + return f() + return g() + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/clean_test.py b/tests/ut/cpp/python_input/gtest_input/optimizer/clean_test.py new file mode 100644 index 00000000000..ab6805ef811 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/clean_test.py @@ -0,0 +1,34 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : opt_clean.py +@Author : wangqiuliang +@Date : 2019-03-18 +@Desc : parse python function for ut of erase class +""" +from dataclasses import dataclass + + +#Test_Erase_class +@dataclass +class Point: + x: float + y: float + def product(self): + return self.x * self.y + +def test_erase_class_fn(p_in): + p = Point(p_in) + return p.x * p.y diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py b/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py new file mode 100644 index 00000000000..53eb2130f00 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py @@ -0,0 +1,905 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" opt_test """ +from mindspore.ops import Primitive, PrimitiveWithInfer +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G + +# pylint: disable=unused-variable + +# opt test data, not for running +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + +scalar_add = Primitive('scalar_add') +scalar_mul = Primitive('scalar_mul') +tuple_getitem = Primitive('tuple_getitem') +switch = Primitive('switch') + +def test_sexp_conversion(): + """ test_sexp_conversion """ + return scalar_mul(10, scalar_add(5, 4)) + +class FnDict: + def __init__(self): + self.fnDict = {} + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + def __getitem__(self, name): + return self.fnDict[name] + +def test_add_zero(tag): + """ test_add_zero """ + fns = FnDict() + @fns + def before_1(x): + return scalar_add(x, 0) + @fns + def before_2(x): + return scalar_add(scalar_add(x, 0), 0) + @fns + def after(x): + return x + + return fns[tag] + +def test_elimR(tag): + """ test_elimR """ + R = Primitive('R') + + fns = FnDict() + @fns + def before_1(x): + return R(x) + @fns + def after(x): + return x + + return fns[tag] + +def test_idempotent(tag): + """ test_idempotent """ + P = Primitive('P') + R = Primitive('R') + + fns = FnDict() + @fns + def before_1(x): + return P(P(x)) + @fns + def before_2(x): + return P(P(P(P(P(x))))) + @fns + def after(x): + return P(x) + + return fns[tag] + +def test_constant_variable(tag): + """ test_constant_variable """ + P = Primitive('P') + Q = Primitive('Q') + + fns = FnDict() + @fns + def before_1(x): + return Q(15) + Q(x) + @fns + def after(x): + return P(15) + Q(x) + + return fns[tag] + +def cost(x): + """ cost """ + return x * 10 + +J = Primitive('J') + +def test_expendJ(x): + """ test_expendJ """ + return J(cost)(x) + +def test_elim_jinv_j(tag): + """ test_elim_jinv_j """ + J = Primitive('J') + Jinv = Primitive('Jinv') + + fns = FnDict() + @fns + def before_1(x): + return J(Jinv(x)) + @fns + def before_2(x): + return Jinv(J(x)) + @fns + def after(x): + return x + + return fns[tag] + +def test_simplify_always_true_false(tag): + """ test_simplify_always_true_false """ + fns = FnDict() + @fns + def before_1(x, y): + return switch(True, x, y) + @fns + def before_2(x, y): + return switch(False, y, x) + @fns + def after(x, y): + return x + + return fns[tag] + +def test_inline(tag): + """ test_inline """ + fns = FnDict() + @fns + def before(x, y): + def fn1(x1): + return x1 + return fn1(x) + @fns + def after(x, y): + return x + + return fns[tag] + +def test_inline_successively(tag): + """ test_inline_successively """ + fns = FnDict() + def one(x): + return x + 1 + + def two(x): + return one(x + 2) + + def three(x): + return two(x + 3) + + @fns + def before(x): + return three(x) + + @fns + def after(x): + return x + 3 + 2 + 1 + + return fns[tag] + +def test_inline_closure(tag): + """ test_inline_closure """ + fns = FnDict() + @fns + def before(x, y, z): + c = z * z + + def f(x): + return x + c + return f(x * y) + + @fns + def after(x, y, z): + c = z * z + return x * y + c + + return fns[tag] + +def test_inline_deep_closure(tag): + """ test_inline_deep_closure """ + fns = FnDict() + def f(x): + w = x * x + + def g(): + def h(): + return w + return h() + return g + @fns + def before(x, y): + return f(x)() - f(y)() + @fns + def after(x, y): + w1 = x * x + w2 = y * y + return w1 - w2 + + return fns[tag] + +def test_inline_new_closure(tag): + """ test_inline_new_closure """ + fns = FnDict() + def q(x): + return x * x + + def f(x): + def g(): + return q(x) + return g + @fns + def before(x): + return f(x) + @fns + def after(x): + def g(): + return x * x + return g + + return fns[tag] + +def test_inline_recursive_direct(tag): + """ test_inline_recursive_direct """ + fns = FnDict() + @fns + def before1(x): + return before1(x - 1) + + @fns + def before2(x): + def helper1(x): + return before2(x - 1) + + def helper2(x): + return before1(x - 1) + return helper1(x) + + return fns[tag] + +def test_inline_recursive(tag): + """ test_inline_recursive """ + fns = FnDict() + @fns + def before(x): + if x <= 0: + return x + return before(x - 1) + + return fns[tag] + +def test_inline_while(tag): + """ test_inline_while """ + fns = FnDict() + @fns + def before(x): + rval = x + while rval < 100: + rval = rval * rval + return rval + return fns[tag] + +def test_cse(tag): + """ test_cse """ + fns = FnDict() + scalar_div = Primitive('scalar_div') + @fns + def test_f1(x, y): + a = scalar_add(x, y) + b = scalar_add(x, y) + c = scalar_mul(a, b) + return c + @fns + def test_f2(x, y): + a = scalar_add(x, y) + b = scalar_add(scalar_mul(a, y), scalar_div(a, x)) + c = scalar_add(scalar_mul(a, y), scalar_div(scalar_add(x, y), x)) + d = scalar_add(b, c) + return d + return fns[tag] + +def test_arithmetic(tag): + """ test_arithmetic """ + fns = FnDict() + identity = Primitive('identity') + @fns + def multiply_by_zero_l(x): + return scalar_mul(x, 0) + @fns + def multiply_by_zero_r(x): + return scalar_mul(0, x) + @fns + def after_0(x): + return 0 + @fns + def multiply_by_one_l(x): + return scalar_mul(x, 1) + @fns + def multiply_by_one_r(x): + return scalar_mul(1, x) + @fns + def add_zero_l(x): + return scalar_add(x, 0) + @fns + def add_zero_r(x): + return scalar_add(0, x) + @fns + def elim_identity(x): + return identity(x) + @fns + def after(x): + return x + + return fns[tag] + +def test_elim_cast_same_dtype(tag): + """ test_elim_cast_same_dtype """ + fns = FnDict() + cast = P.Cast() + + @fns + def fp32_cast_fp32(x, y): + return cast(x, y) + @fns + def after(x, y): + return x + + return fns[tag] + + +def elim_reshape_same_shape(tag): + """ elim_reshape_same_shape """ + fns = FnDict() + reshape = P.Reshape() + shape = (2, 3) + @fns + def reshape_to_2_3(x): + return reshape(x, shape) + @fns + def after(x): + return x + + return fns[tag] + +def elim_two_reshape(tag): + """ elim_two_reshape """ + fns = FnDict() + reshape = P.Reshape() + shape = (2, 3) + shape_2 = (3, 2) + @fns + def before(x): + return reshape(reshape(x, shape_2), shape) + @fns + def after(x): + return reshape(x, shape) + + return fns[tag] + +def elim_two_cast(tag): + """ elim_two_cast """ + fns = FnDict() + cast = P.Cast() + + @fns + def before(x, a, b): + return cast(cast(x, a), b) + @fns + def after(x, a, b): + return cast(x, b) + + return fns[tag] + +def test_elim_transpose(tag): + """ test_elim_transpose """ + fns = FnDict() + transpose = P.Transpose() + perm = (0, 1, 2) + + @fns + def before(x): + return transpose(x, perm) + @fns + def after(x): + return x + + return fns[tag] + +def test_elim_tile_multiply_one(tag): + """ test_elim_tile_multiply_one """ + fns = FnDict() + tile = P.Tile() + all_one = (1, 1, 1) + + @fns + def before(x): + return tile(x, all_one) + @fns + def after(x): + return x + + return fns[tag] + +def test_elim_reduce_mean_shape_one(tag): + """ test_elim_reduce_mean_shape_one """ + fns = FnDict() + reduce_mean = P.ReduceMean() + + @fns + def before(x, y): + return reduce_mean(x, 0) + @fns + def after(x, y): + return x + + return fns[tag] + +def test_elim_all_shape_one(tag): + """ test_elim_all_shape_one """ + fns = FnDict() + all_ = P.ReduceAll() + + @fns + def before(x, y): + return all_(x, 0) + @fns + def after(x, y): + return x + + return fns[tag] + +def test_elim_sum_shape_one(tag): + """ test_elim_sum_shape_one """ + fns = FnDict() + sum_ = P.ReduceSum() + + @fns + def before(x, y): + return sum_(x, 0) + @fns + def after(x, y): + return x + + return fns[tag] + +def test_tuple_getitem(tag): + """ test_tuple_getitem """ + fns = FnDict() + make_tuple = Primitive('make_tuple') + + @fns + def make_get_0(x, y): + return tuple_getitem(make_tuple(x, y), 0) + + @fns + def make_get_1(x, y): + return tuple_getitem(make_tuple(x, y), 1) + @fns + def after_0(x, y): + return x + @fns + def after_1(x, y): + return y + + return fns[tag] + +def test_tuple_setitem(tag): + """ test_tuple_setitem """ + fns = FnDict() + make_tuple = Primitive('make_tuple') + tuple_setitem = Primitive('tuple_setitem') + + @fns + def before_0(x, y, z): + return tuple_setitem(make_tuple(x, y), 0, z) + + @fns + def before_1(x, y, z): + return tuple_setitem(make_tuple(x, y), 1, z) + @fns + def after_0(x, y, z): + return make_tuple(z, y) + @fns + def after_1(x, y, z): + return make_tuple(x, z) + + return fns[tag] + +def test_tuple_get_set_item(tag): + """ test_tuple_get_set_item """ + fns = FnDict() + tuple_setitem = Primitive('tuple_setitem') + + @fns + def before_0(t, x): + return tuple_getitem(tuple_setitem(t, 0, x), 0) + + @fns + def after_0(t, x): + return x + + @fns + def before_1(t, x): + return tuple_getitem(tuple_setitem(t, 0, x), 1) + + @fns + def after_1(t, x): + return tuple_getitem(t, 1) + + return fns[tag] + +def test_partial(tag): + """ test_partial """ + fns = FnDict() + partail = Primitive('partial') + + def f(x, y): + return scalar_add(x, y) + + @fns + def before(x, y): + return partail(f, x)(y) + + @fns + def after(x, y): + return f(x, y) + + return fns[tag] + +def test_replace_applicator(tag): + """ test_replace_applicator """ + fns = FnDict() + partail = Primitive('partial') + + def app1(x, y): + return scalar_add(x, y) + + def app2(x, y): + return app1(x, y) + + def app3(x, y): + return scalar_add(y, x) + + @fns + def before1(x, y): + return app1(x, y) + + @fns + def before2(x, y): + return app2(x, y) + + @fns + def before3(x, y): + return app3(x, y) + + @fns + def after(x, y): + return scalar_add(x, y) + + return fns[tag] + +def test_specialize_on_graph_arguments(tag): + """ test_specialize_on_graph_arguments """ + fns = FnDict() + f1 = Primitive('f1') + f2 = Primitive('f2') + + @fns + def before(x, y): + def helper(f, x, g, y): + return scalar_add(f(x), g(y)) + + return helper(f1, x, f2, y) + + @fns + def after(x, y): + def helper(x, y): + return scalar_add(f1(x), f2(y)) + return helper(x, y) + return fns[tag] + +def test_incorporate_getitem(tag): + """ test_incorporate_getitem """ + fns = FnDict() + f1 = Primitive('f1') + f2 = Primitive('f2') + + @fns + def before1(x, y): + def fn(x, y): + return f1(x, y), f2(x, y) + return tuple_getitem(fn(x, y), 0) + + @fns + def after1(x, y): + def fn(x, y): + return f1(x, y) + return fn(x, y) + + @fns + def before2(x, y): + def fn(x, y): + return x + return tuple_getitem(fn(x, y), 0) + + @fns + def after2(x, y): + def fn(x, y): + return tuple_getitem(x, 0) + return fn(x, y) + + return fns[tag] + +def test_incorporate_getitem_through_switch(tag): + """ test_incorporate_getitem_through_switch """ + fns = FnDict() + scalar_gt = Primitive('scalar_gt') + @fns + def before(x, y): + def f1(x, y): + return x, y + + def f2(x, y): + return y, x + + return tuple_getitem( + switch(scalar_gt(x, 0), f1, f2)(x, y), + 0) + + @fns + def after(x, y): + def f1(x, y): + return x + + def f2(x, y): + return y + + return switch(scalar_gt(x, 0), f1, f2)(x, y) + + return fns[tag] + +def test_incorporate_call(tag): + """ test_incorporate_call """ + fns = FnDict() + f1 = Primitive('f1') + + @fns + def before(x, y): + def fn(q): + def subf(z): + return f1(q, z) + return subf + return fn(x)(y) + + @fns + def after(x, y): + def fn(q, y): + def subf(z): + return f1(q, z) + return subf(y) + return fn(x, y) + + return fns[tag] + +def test_incorporate_call_through_switch(tag): + """ test_incorporate_call_through_switch """ + fns = FnDict() + f1 = Primitive('f1') + f2 = Primitive('f2') + scalar_gt = Primitive('scalar_gt') + identity = Primitive('identity') + @fns + def before(x, y, z): + def f1g(): + return f1 + def f2g(): + return f2 + def fn(): + return switch(scalar_gt(x, 0), f1g, f2g)() + return fn()(y, z) + + @fns + def after(x, y, z): + def fn(y, z): + def tb(y, z): + return f1(y, z) + def fb(y, z): + return f2(y, z) + return switch(scalar_gt(x, 0), tb, fb)(y, z) + return fn(y, z) + + return fns[tag] + + +def test_float_tuple_getitem_through_switch(tag): + """ test_float_tuple_getitem_through_switch """ + fns = FnDict() + scalar_gt = Primitive('scalar_gt') + @fns + def before(x, y): + return tuple_getitem(switch(scalar_gt(x, 0), x, y), 0) + + @fns + def after(x, y): + return switch(scalar_gt(x, 0), tuple_getitem(x, 0), tuple_getitem(y, 0)) + + return fns[tag] + +def test_merge_addn(tag): + """ test_merge_addn """ + fns = FnDict() + addn = P.AddN() + AddN = P.AddN + @fns + def before(x, y, z, a): + return addn((addn((a, x, y)), z)) + + @fns + def after(x, y, z, a): + return AddN()((a, x, y, z)) + + return fns[tag] + +def test_addn_zero(tag): + """ test_addn_zero """ + fns = FnDict() + addn = P.AddN() + AddN = P.AddN + zero_tensor = Primitive('zeros_like_tensor') + @fns + def before_1(x, y, z, a): + return addn((a, zero_tensor(x), zero_tensor(y), z)) + + @fns + def after(x, y, z, a): + return AddN()((a, z)) + + @fns + def before_2(x, y, z, a): + return addn((a, zero_tensor(x), z, zero_tensor(y))) + + @fns + def before_3(x, y, z, a): + return addn((zero_tensor(x), a, z, zero_tensor(y))) + + @fns + def before_4(x, y, z, a): + return addn((a, z)) + + return fns[tag] + +def test_convert_switch_ops(tag): + fns = FnDict() + ge_switch = Primitive('GeSwitch') + merge = Primitive('Merge') + add = Primitive('TensorAdd') + neg = Primitive('Neg') + tuple_getitem = Primitive('tuple_getitem') + make_tuple = Primitive('make_tuple') + @fns + def before(cond, x, y): + if cond: + z = add(x, y) + else: + z = neg(y) + return z + @fns + def after(cond, x, y): + sw1 =ge_switch(x, cond) + sw2 =ge_switch(y, cond) + sw3 =ge_switch(y, cond) + sw1_t = tuple_getitem(sw1, 1) + sw2_t = tuple_getitem(sw2, 1) + sw3_f = tuple_getitem(sw3, 0) + t_res = add(sw1_t, sw2_t) + f_res = neg(sw3_f) + tup = make_tuple(t_res, f_res) + merge_res = merge(tup) + res = tuple_getitem(merge_res, 0) + return res + return fns[tag] + + +def test_minmax_grad(tag): + """ test_minmax_grad """ + fns = FnDict() + min_grad = G.MinimumGrad() + @fns + def before_11(x, y, dout): + return tuple_getitem(min_grad(x, y, dout), 0) + + @fns + def before_12(x, y, dout): + return tuple_getitem(min_grad(x, y, dout), 1) + + @fns + def before_2(x, y, dout): + a = min_grad(x, y, dout) + return tuple_getitem(a, 0), tuple_getitem(a, 1) + + max_grad = G.MaximumGrad() + @fns + def before_31(x, y, dout): + return tuple_getitem(max_grad(x, y, dout), 0) + + @fns + def before_32(x, y, dout): + return tuple_getitem(max_grad(x, y, dout), 1) + + @fns + def before_4(x, y, dout): + a = max_grad(x, y, dout) + return tuple_getitem(a, 0), tuple_getitem(a, 1) + + return fns[tag] + + +def test_reducesum_one(tag): + """ test_reducesum_one """ + fns = FnDict() + reduce_sum_true = P.ReduceSum(keep_dims=True) + reduce_sum_false = P.ReduceSum(keep_dims=False) + axis = (2, 3) + reshape = P.Reshape() + shape1 = (3, 2, 2) + shape2 = (3, 2) + + @fns + def before_1(x): + return reduce_sum_true(x, 3) + + @fns + def before_2(x): + return reduce_sum_true(x, axis) + + @fns + def before_3(x): + return reduce_sum_false(x, 3) + + @fns + def before_4(x): + return reduce_sum_false(x, axis) + + @fns + def after_1(x): + return x + + @fns + def after_2(x): + return reshape(x, shape1) + + @fns + def after_3(x): + return reshape(x, shape2) + + return fns[tag] + +def test_print_tuple_wrapper(tag): + fns = FnDict() + print_ = Primitive('Print') + make_tuple = Primitive('make_tuple') + + @fns + def before1(x, y): + return print_(x, y) + + @fns + def after1(x, y): + return print_(make_tuple(x, y)) + + @fns + def before2(x, y, z): + return print_(x, make_tuple(y, z)) + + @fns + def after2(x, y, z): + return print_(make_tuple(x, make_tuple(y, z))) + + @fns + def before3(x, y, z): + return print_(make_tuple(x, y, z)) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/__init__.py b/tests/ut/cpp/python_input/gtest_input/pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/infer/__init__.py b/tests/ut/cpp/python_input/gtest_input/pipeline/infer/__init__.py new file mode 100644 index 00000000000..d840342dd1e --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/infer/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from .primitive_test import * +from .infer_test import * \ No newline at end of file diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/infer/infer_test.py b/tests/ut/cpp/python_input/gtest_input/pipeline/infer/infer_test.py new file mode 100644 index 00000000000..e93c8bdb874 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/infer/infer_test.py @@ -0,0 +1,201 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +from dataclasses import dataclass +from mindspore.ops import Primitive +from mindspore.ops import functional as F +from mindspore.ops import operations as P + +scala_add = Primitive('scalar_add') + + +@dataclass +class InferFoo: + x: int + y: int + + def inf_add(self): + return scala_add(self.x, self.y) + + +def test_dataclass_fun_add(x): + foo = InferFoo(x, 1.2) + return foo.inf_add() + + +@dataclass +class ResolveFoo: + x: float + y: float + + def inf_sub(self): + return self.x - self.y + + +def test_dataclass_fun_sub(x): + foo = ResolveFoo(x, 0.1) + return foo.inf_sub() + + +def test_fun_sub(x, y): + return x - y + + +def test_fun_add(x, y): + return scala_add(x, y) + + +class AddNet(nn.Cell): + def __init__(self): + super(AddNet, self).__init__() + + def construct(self, x, y): + return F.scalar_add(x, y) + + def get_params(self): + return None + + +def test_net_construct_add(): + model = AddNet() + return model + + +class SubNet(nn.Cell): + def __init__(self): + super(SubNet, self).__init__() + + def construct(self, x, y): + return F.scalar_sub(x, y) + + def get_params(self): + return None + + +def test_net_construct_sub(): + model = SubNet() + return model + + +def test_infer_for(xs, y): + rval = y + for x in xs: + rval = rval + x + return rval + + +def test_multiple_conv2d(): + conv1 = P.Conv2D(out_channel=2, pad_mode="pad", kernel_size=(5, 5), pad=0, stride=2, dilation=1) + conv2 = P.Conv2D(out_channel=2, pad_mode="pad", kernel_size=(5, 5), pad=1, stride=2, dilation=1) + + def get_conv(x, w1, w2): + return conv2(conv1(x, w1), w2) + + return get_conv + + +def test_graph_infer_defaults(): + def func_call(x, y, p=10, q=20): + return x - y - p - q + + def test_call_variable(): + x = 100 + y = 20 + return func_call(x, y) + return test_call_variable + + +def test_graph_infer_vararg_0(): + def func_call(x, y, z): + return x - y - z + + def test_call_variable(): + args = (6, 2, 3) + return func_call(*args) + + return test_call_variable + + +def test_graph_infer_vararg(): + def func_call(x, y, *args): + return x - y - args[0] + + def test_call_variable(): + x = 30 + y = 20 + args = (1, 2, 3) + return func_call(x, y, *args) + + return test_call_variable + + +def test_graph_infer_vararg_kwonlyargs(): + def func_call(x, y, *args, p, q): + return x - y - args[1] - p - q + + def test_call_variable(): + x = 100 + y = 20 + args = (1, 2, 3) + p = 10 + q = 20 + return func_call(x, y, *args, p=p, q=q) + + return test_call_variable + + +def test_graph_infer_kwarg(): + def func_call(x, y, **kwargs): + return x - y - kwargs["a"] + + def test_call_variable(): + x = 30 + y = 20 + kwargs = {"a": 3, "b": 2} + return func_call(x, y, **kwargs) + + return test_call_variable + + +def test_graph_infer_vararg_kwonlyargs_kwarg(): + def func_call(x, y, *args, p, q, **kwargs): + return x - y - args[2] - p - q - kwargs["z"] + + def test_call_variable(): + x = 100 + y = 20 + args = (1, 2, 3) + kwargs = {"a": 3, "b": 2} + p = 10 + q = 20 + return func_call(x, y, *args, p=p, q=q, z=1, **kwargs) + + return test_call_variable + + +def test_graph_infer_vararg_kwonlyargs_kwarg_defaults(): + def func_call(x, y, *args, p=1, q=2, **kwargs): + return x - y - args[0] - p - q - kwargs["z"] + + def test_call_variable(): + x = 100 + y = 20 + args = (1, 2, 3) + kwargs = {"a": 1, "b": 2} + p = 10 + q = 20 + return func_call(x, y, *args, p=p, z=10, m=q, **kwargs) + + return test_call_variable diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/infer/primitive_test.py b/tests/ut/cpp/python_input/gtest_input/pipeline/infer/primitive_test.py new file mode 100644 index 00000000000..06ab870ce2a --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/infer/primitive_test.py @@ -0,0 +1,92 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import mindspore.nn as nn +from mindspore.common import dtype +from mindspore.ops import Primitive, prim_attr_register, PrimitiveWithInfer +from mindspore.ops import operations as P + +def get_add(a, b): + return a + b + + +def get_f(v): + return v + 1 + + +relu = nn.ReLU() +def get_relu(x): + return relu(x) + +softmax_cross_entropy_with_logits = P.SoftmaxCrossEntropyWithLogits() +def get_softmax_cross_entropy_with_logits(logits, labels): + return softmax_cross_entropy_with_logits(logits, labels) + +class TensorToScalar(PrimitiveWithInfer): + """this is a test primitive for cases that has tensor input, but has only one scalar output""" + @prim_attr_register + def __init__(self): + """init""" + + def __call__(self, logits, labels): + raise NotImplementedError + + def infer_shape(self, logits_shape, label_shape): + return [] + + def infer_dtype(self, logits_type, labels_type): + # pylint: disable=unused-argument + return dtype.float64 + +tensorToScalar = TensorToScalar() +def get_tensor_to_scalar(logits, labels): + return tensorToScalar(logits, labels) + + +conv2d = P.Conv2D(64, + (3, 3), + pad_mode="pad", + pad=1, + stride=2) + +def get_conv2d(x, w): + return conv2d(x, w) + +conv2dNative = P.DepthwiseConv2dNative(3, (3,3), pad_mode="pad", pad=1, stride=2) + +def get_conv2d_native(x, w): + return conv2dNative(x, w) + +biasAdd = P.BiasAdd() +def get_bias_add(x, b): + return biasAdd(x, b) + + +def test_conv2d(out_channel, kernel_size, pad, stride, dilation): + conv = P.Conv2D(out_channel=out_channel, kernel_size=kernel_size, pad_mode= "pad", pad=pad, + stride=stride, dilation= dilation) + def get_conv(x, w): + return conv(x, w) + return get_conv + + +def test_dropout(): + dropOutGenMask = P.DropoutGenMask() + dropoutDoMask = P.DropoutDoMask() + shape = P.Shape() + def get_dropout(x, prob): + mask = dropOutGenMask(shape(x), prob) + y = dropoutDoMask(x, mask, prob) + return y + return get_dropout diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/__init__.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_abnormal.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_abnormal.py new file mode 100644 index 00000000000..f4e0102e51a --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_abnormal.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : parser_abnormal.py +@Author: +@Date : 2019-03-14 18:37 +@Desc : parser test function. +""" + + +def nonrec(): + return 123 + + +def rec1(): + return rec2() + + +def rec2(): + return rec1() + + +def test_keep_roots_recursion(x, y): + return rec1() + nonrec() + + +def test_f(x, y): + return x + y + + +def test_performance(x): + return x + + +def test_list_append(x, y): + list_test = [] + list_test.append(x) + list_test.append(y) + return list_test diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_class.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_class.py new file mode 100644 index 00000000000..93f33d07e1b --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_class.py @@ -0,0 +1,85 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : parser_class_method.py +@Author: +@Date : 2019-03-23 14:22 +@Desc : parser class method function. +""" +import logging +import numpy as np + +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +class ResNet(nn.Cell): + def __init__(self, tensor): + super(ResNet, self).__init__() + self.weight = Parameter(tensor, name="weight") + + def construct(self, x, y): + return x + y*self.weight + + def get_params(self): + return None + +class SimpleNet(nn.Cell): + def __init__(self, network, tensor, use_net=False): + super(SimpleNet, self).__init__() + self.weight = Parameter(tensor, name="weight") + self.use_net = use_net + if self.use_net: + self.network = network + else: + self.network = None + + def construct(self, x, y): + z = self.weight * x + y + if self.use_net: + z = self.network(z, x) + return z + + def get_params(self): + return None + + +def loss_func(x, y): + return x - y + + +def optimizer(x): + return x + + +def test_parse_object_instance(): + X = Tensor(np.ones([1, 3, 16, 50])) + Y = Tensor(np.ones([1, 3, 16, 50])) + network = SimpleNet(ResNet(X), Y) + return network + + +def test_get_object_graph(): + X = Tensor(np.ones([2, 2, 2]).astype(np.float32)) + Y = Tensor(np.ones([2, 2, 2]).astype(np.float32)) + network = SimpleNet(ResNet(X), Y, True) + print(network.parameters_dict()) + return _executor.compile(network, X, Y) diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_compile.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_compile.py new file mode 100644 index 00000000000..9fd56f7a515 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_compile.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : parse_compile.py +@Author: +@Date : 2019-03-20 +@Desc : test mindspore compile method +""" +import logging +import numpy as np + +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.nn.optim import Momentum +from mindspore.train.model import Model + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + + def construct(self, x): + x = self.conv(x) + x = self.relu(x) + out = self.flatten(x) + return out + +loss = nn.MSELoss() + +def test_build(): + input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224])) + input_label = Tensor(np.random.randint(0, 10, [1, 10])) + net = Net() + opt = Momentum(net.get_parameters(), learning_rate=0.1, momentum=0.9) + model = Model(net, loss_fn=loss, optimizer=opt, metrics=None) diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_primitive.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_primitive.py new file mode 100644 index 00000000000..028ee304d57 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parse_primitive.py @@ -0,0 +1,138 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : parser_primitive.py +@Author: +@Date : 2019-03-23 14:22 +@Desc : parser class method function. +""" +import sys +from collections import * +import logging +import numpy as np + +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.ops import Primitive, prim_attr_register +from mindspore.ops import functional as F +from mindspore.train.model import Model + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +# use method1: create instance outside function +relu_test = Primitive('relu_test') + + +def test_ops_f1(x, y): + foo = relu_test(x) + return foo + + +# use method2: create instance outside function use an operator with parameters +class Conv_test(Primitive): + @prim_attr_register + def __init__(self, stride = 0, pad = 1): + print('in conv_test init', self.stride) + + def __call__(self, x = 0, y = 1, z= 2): + pass + + +foo = Conv_test(3, 5) + + +def test_ops_f2(x, y): + z = foo(x, y) + return z + + +# use method3: use the ops primitive instance on method +class ResNet(nn.Cell): + def __init__(self, tensor): + super(ResNet, self).__init__() + self.weight = Parameter(tensor, name="weight") + self.conv = Conv_test(3, 5) + + def construct(self, x, y, train="train"): + return x + y*self.weight + self.conv(x) + + def get_params(self): + return None + + +class SimpleNet(nn.Cell): + def __init__(self, network, tensor): + super(SimpleNet, self).__init__() + self.weight = Parameter(tensor, name="weight") + self.network = network + + def construct(self, x, y, train="train"): + return self.network(x) + self.weight * y + + def get_params(self): + return None + + +def loss_func(x, y): + return x - y + + +def optimizer(x): + return x + + +def test_primitive_obj(): + X = Tensor(np.ones([2, 3])) + Y = Tensor(np.ones([2, 3])) + network = SimpleNet(ResNet(X), Y) + return network + + +# use method4: call primitive ops with parameters +class SimpleNet_1(nn.Cell): + def __init__(self): + super(SimpleNet_1, self).__init__() + self.conv = Conv_test(2, 3) + + def construct(self, x, y, train="train"): + return self.conv(x, y) + + def get_params(self): + return None + + +def test_primitive_obj_parameter(): + model = SimpleNet_1() + return model + +# use method4: call primitive ops with parameters +class SimpleNet_2(nn.Cell): + def __init__(self): + super(SimpleNet_2, self).__init__() + + def construct(self, x, y): + return F.scalar_add(x, y) + + def get_params(self): + return None + + +def test_primitive_functional(): + model = SimpleNet_2() + return model diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parser_integrate.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parser_integrate.py new file mode 100644 index 00000000000..39f62448779 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parser_integrate.py @@ -0,0 +1,156 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +file: parser_integrate.py +""" +import numpy as np + +import mindspore._c_expression as me +import mindspore.nn as nn +from mindspore.common.api import ms_function, _executor +from mindspore.common import dtype +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.ops import functional as F +from mindspore.model_zoo.resnet import resnet50 +from mindspore.train.model import Model + + +def test_high_order_function(a): + def f(g, x): + return scalar_mul(g(x, x), g(x, x)) + return f(scalar_add, a) + + +def test_hof_tup(a, b): + """Test higher order functions.""" + def f(gh, x, y): + g, h = gh + return scalar_mul(g(x, y), h(x, y)) + + return f((scalar_add, scalar_mul), a, b) + + +def scalar_mul(x, y): + """Implement `scalar_mul`.""" + return x * y + +def scalar_add(x, y): + """implement scalar_add""" + return x + y + +def test_while_2(x, y, z): + rval = 0 + # Cannot compare to 0 or finite diff is unstable + while x > -0.1: + rval = rval + y + x = x - z + return rval + +def test_nested_closure(x): + a = x * x + b = x + 5 + + def f(): + def g(): + return a + b + + def h(): + return a * b + return g if x < 0 else h + return f()() + +def test_functions_in_tuples(x, y): + tup = scalar_add, scalar_mul + f, g = tup + return f(x, y) + g(x, y) + +def test_closures_in_tuples(x, y): + def f(): + return x * y + + def g(): + return x + y + + tup = f, g + ff, gg = tup + return ff() + gg() + +@ms_function +def add(x, y): + return x + y + +def test_tensor_add(): + X = me.tensor() + Y = me.tensor() + X.set_dtype(dtype.float32) + Y.set_dtype(dtype.float32) + X = me.tensor(np.ones([2, 3])) + Y = me.tensor(np.ones([2, 3])) + sum = add(X, Y) + print("test tensor add") + return sum + +def loss_func(x, y): + return x - y + +def optimizer(x): + return x + +def test_resetnet50_build(): + X = me.tensor() + Y = me.tensor() + X.set_dtype(dtype.float32) + Y.set_dtype(dtype.float32) + network = resnet50() + model = Model(network=network, loss_fn=loss_func, optimizer=optimizer) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, bias_init='zeros') + def construct(self, input): + return self.conv(input) + +class TestNet(nn.Cell): + def __init__(self): + super(TestNet, self).__init__() + self.param = Parameter(Tensor([1, 3, 16, 50]), "param") + def construct(self, input): + self.param = self.param + input + return self.param + +def test_compile_conv2d(): + net = Net() + input = Tensor(np.ones([1, 3, 16, 50]).astype(np.float32)) + _executor.compile(net, input) + +def test_none(x, y): + def func(x, y): + if y == None: + return x + return x + y + return func(x, y) + +def test_get_attr(x): + a = F.scalar_mul(x ,x) + return a + +@ms_function +def known(): + return unknown() + +def test_undefined_symbol(): + known() diff --git a/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parser_test.py b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parser_test.py new file mode 100644 index 00000000000..d0ed8a14738 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pipeline/parse/parser_test.py @@ -0,0 +1,294 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : parser_test.py +@Author: +@Date : 2019-02-23 16:37 +@Desc : parser test function. +""" +import logging +from dataclasses import dataclass + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + +# Test:common function +def test_f(x, y): + return x - y + +def test_if(x, y): + if x: + z = x + y + else: + z = y * y + return z + +def test_ifexp(x, y): + z = (x + y) if x else y*y + return z + +def test_if_nested(x, y, t): + if x: + z = x * x + if y: + z = z + y + else: + z = z * z + else: + if t: + z = t*t + else: + z = t + x + return z + +def test_while(x, y): + z = x + y + while z: + z = x + x + return z + +def test_for(x, y): + z = y + for index in x: + z = z + index + return z + +def test_compare_lt(x, y): + z = 0 + if x < y: + z = x + else: + z = y + return z + +def test_compare_gt(x, y): + z = 0 + if x > y: + z = x + else: + z = y + return z + +def test_compare_ge(x, y): + z = 0 + if x >= y: + z = x + else: + z = y + return z + +def test_compare_le(x, y): + z = 0 + if x <= y: + z = x + else: + z = y + return z + +def test_compare_eq(x, y): + z = 0 + if x == y: + z = x + else: + z = y + return z + +def test_compare_ne(x, y): + z = 0 + if x != y: + z = x + else: + z = y + return z + +def test_boolop_two_and(x, y): + if x and y : + t = x + y + else: + t =0 + return t + +def test_boolop_three_and(x, y, z): + if x and y and z: + t = x + y + else: + t = z + return t + +def test_boolop_two_or(x, y): + if x or y : + t = x + y + else: + t =0 + return t + +def test_boolop_three_or(x, y, z): + if x or y or z: + t = x + y + else: + t = z + return t + +def test_boolop_mix_and_or(x, y, z): + if x and y or z: + t = x + y + else: + t = z + return t + +def test_lambda(x, y): + l = lambda x,y: x * y + t = l(x, y) + return t + +def test_funcdef(x, y): + def max(a, b): + if a > b: + return a + else: + return b + t = max(x, y) + return t + +def test_tuple_fn(x, y): + l = (1, 2, 3, 5, 7) + l = l + l[y] + return l + +def test_list_fn(x, y): + l = [1, 2, 3, 5, 7] + l = l + l[y] + return l + +# Test:resolve function +def get_resolve_fn(x, y): + return test_f(x, y) + +# Test:no return function +def get_no_return_fn(x, y): + x + y + +def testDoNum(): + return 1 + +def testDoStr(): + return "str" + +def testDoNamedConstTrue(): + return True + +def testDoNamedConstFalse(): + return False + +def testDoNamedConstNone(): + return None + +#Test_Class_type +@dataclass +class TestFoo: + x : float + y : int + def inf(self): + return self.x + +def test_class_fn(x): + foo = TestFoo(x, 1) + return foo.inf() + +# custom test function +def test_custom(x, y, z): + def g(x1, y1): + def h(x2): + return x2 + y1 + z + return h(x1) + return g(x, y) + +def test_simple_closure(a, b): + """Test some trivial closures.""" + z = 1 + def f(): + return a + z + + def g(): + return b + 2.0 + return f() * g() + +def test_assign_tuple(x,y): + a = 1 + b = 2 + t = a, b + c, d = t + return c + d + +def test_unary(x, y): + a = -x + z = a + y + return z + +def f1(x, y): + return x + y + +def test_reslove_closure(x): + z = x + def in_f2(x,y): + x = f1(x,y) + return x + y + z + return in_f2 + +def test_augassign(x ,y): + x += x + y -= x + return y + + +def test_resolvefail(x, y): + a = x + y + Undef + return a + + +# test system call +def test_sys_call(x, y): + a = len(x) + len(y) + return a + +def test_bool_not(x, y): + z = x and y + return not z + +def test_call_fn_use_tuple(y): + log.info("the y is :%r", y) + log.info("y type is :%r", type(y)) + z = len(y) + for i in y: + log.info("The parameter is: %r", i) + return z + +def test_subscript_setitem(): + t = [1, 2, 5, 6] + t[2] = t[2] + 7 + t[3] = t[3] + 1 + return t + +def test_dict(): + dict = {"a": 1, "b": 2} + return dict + +def func_call(x, y, *var, a=0, b=1, **kwargs): + return x + y + var[0] + a + b + kwargs["z"] + +def test_call_variable(): + t = (1, 2, 3) + d = {"z": 10, "e": 11} + return func_call(0, 1, 2, *t, b=6, a=5, c=5, z=10, **d) diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/__init__.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/__init__.py new file mode 100644 index 00000000000..fc53efd1be0 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ \ No newline at end of file diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/adam_apply_one_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/adam_apply_one_fusion_test.py new file mode 100644 index 00000000000..b55764b18d9 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/adam_apply_one_fusion_test.py @@ -0,0 +1,69 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Add = P.TensorAdd() +Sub = P.Sub() +Mul = P.Mul() +RealDiv = P.RealDiv() +Sqrt = P.Sqrt() +Square = P.Square() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +AdamApplyOne = Primitive('AdamApplyOne') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_adam_apply_one_fusion(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, add2_y): + square0 = Square(input0) + mul1 = Mul(mul1_x, input0) + mul0 = Mul(mul0_x, input2) + mul2 = Mul(mul2_x, input1) + mul3 = Mul(mul3_x, square0) + add0 = Add(mul0, mul1) + add1 = Add(mul2, mul3) + sqrt0 = Sqrt(add1) + add2 = Add(sqrt0, add2_y) + true_div0 = RealDiv(add0, add2) + mul4 = Mul(input4, true_div0) + sub0 = Sub(input3, mul4) + outputs = make_tuple(add1, add0, sub0) + output = tuple_getitem(outputs, 0) + return output + + @fns + def after(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, add2_y): + adam_apply_one = AdamApplyOne(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, add2_y) + outputs = make_tuple(tuple_getitem(adam_apply_one, 0), tuple_getitem(adam_apply_one, 1), + tuple_getitem(adam_apply_one, 2)) + output = tuple_getitem(outputs, 0) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/adam_apply_one_with_decay_rule.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/adam_apply_one_with_decay_rule.py new file mode 100644 index 00000000000..a73183573b6 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/adam_apply_one_with_decay_rule.py @@ -0,0 +1,91 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +mul = P.Mul() +add = P.TensorAdd() +square = P.Square() +sqrt = P.Sqrt() +real_div = P.RealDiv() +sub = P.Sub() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +adam_apply_one_with_decay = Primitive('AdamApplyOneWithDecay') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_adam_apply_one_with_decay_rule(tag): + """ test_adam_apply_one_with_decay_rule """ + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, mul4_x, add2_y): + mul0 = mul(mul0_x, input2) + mul1 = mul(mul1_x, input0) + square0 = square(input0) + add0 = add(mul0, mul1) + mul2 = mul(mul2_x, input1) + mul3 = mul(mul3_x, square0) + add1 = add(mul2, mul3) + sqrt0 = sqrt(add1) + add2 = add(sqrt0, add2_y) + mul4 = mul(mul4_x, input3) + real_div0 = real_div(add0, add2) + add3 = add(real_div0, mul4) + mul5 = mul(input4, add3) + sub0 = sub(input3, mul5) + return make_tuple(add1, add0, sub0) + + @fns + def no_match(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, mul4_x, add2_y): + mul0 = mul(mul0_x, input2) + mul1 = mul(mul1_x, input0) + square0 = square(input0) + # diff mul from original add + add0 = mul(mul0, mul1) + mul2 = mul(mul2_x, input1) + mul3 = mul(mul3_x, square0) + add1 = add(mul2, mul3) + sqrt0 = sqrt(add1) + add2 = add(sqrt0, add2_y) + mul4 = mul(mul4_x, input3) + real_div0 = real_div(add0, add2) + add3 = add(real_div0, mul4) + mul5 = mul(input4, add3) + sub0 = sub(input3, mul5) + return make_tuple(add1, add0, sub0) + + @fns + def after(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, mul4_x, add2_y): + res = adam_apply_one_with_decay(input0, input1, input2, input3, input4, mul0_x, mul1_x, mul2_x, mul3_x, mul4_x, + add2_y) + item0 = tuple_getitem(res, 0) + item1 = tuple_getitem(res, 1) + item2 = tuple_getitem(res, 2) + return make_tuple(make_tuple(item0, item1, item2)) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/add_memcpy_async.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/add_memcpy_async.py new file mode 100644 index 00000000000..5be1c7b0e55 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/add_memcpy_async.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +all_reduce = P.AllReduce() +memcpy_async = Primitive('memcpy_async') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_add_memcpy_async(tag): + fns = FnDict() + + @fns + def before(x): + res = all_reduce(x) + return make_tuple(x, res) + + @fns + def after(x): + res = memcpy_async(x) + res = all_reduce(res) + return make_tuple(make_tuple(x, res)) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/bn_grad_split.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/bn_grad_split.py new file mode 100644 index 00000000000..c26c698260c --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/bn_grad_split.py @@ -0,0 +1,87 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +bn_grad = G.FusedBatchNormGrad() +bn_grad1 = Primitive('BNGrad1') +bn_grad2 = Primitive('BNGrad2') +bn_grad3 = Primitive('BNGrad3') +bn_training_update_grad = Primitive('BNTrainingUpdateGrad') +bn_training_reduce_grad = Primitive('BNTrainingReduceGrad') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_bn_grad_split(tag): + """ test_bn_grad_split """ + fns = FnDict() + + @fns + def before(i0, i1, i2, i3, i4): + bn_grad_output = bn_grad(i0, i1, i2, i3, i4) + item0 = tuple_getitem(bn_grad_output, 0) + item1 = tuple_getitem(bn_grad_output, 1) + item2 = tuple_getitem(bn_grad_output, 2) + output = make_tuple(item0, item1, item2) + return output + + @fns + def after1(i0, i1, i2, i3, i4): + bn_grad1_output = bn_grad1(i0, i1, i3) + bn_grad1_item0 = tuple_getitem(bn_grad1_output, 0) + bn_grad1_item1 = tuple_getitem(bn_grad1_output, 1) + bn_grad1_item2 = tuple_getitem(bn_grad1_output, 2) + bn_grad2_output = bn_grad2(bn_grad1_item0, bn_grad1_item1, i4, i2) + bn_grad2_item0 = tuple_getitem(bn_grad2_output, 0) + bn_grad2_item1 = tuple_getitem(bn_grad2_output, 1) + bn_grad2_item2 = tuple_getitem(bn_grad2_output, 2) + bn_grad2_item3 = tuple_getitem(bn_grad2_output, 3) + bn_grad2_item4 = tuple_getitem(bn_grad2_output, 4) + bn_grad3_output = bn_grad3(i0, bn_grad2_item2, bn_grad2_item3, bn_grad2_item4, bn_grad1_item2) + output = make_tuple(bn_grad3_output, bn_grad2_item0, bn_grad2_item1) + item0 = tuple_getitem(output, 0) + item1 = tuple_getitem(output, 1) + item2 = tuple_getitem(output, 2) + output = make_tuple(item0, item1, item2) + return make_tuple(output) + + @fns + def after2(i0, i1, i2, i3, i4): + bn_update_grad_output = bn_training_update_grad(i0, i1, i3, i4) + update_item0 = tuple_getitem(bn_update_grad_output, 0) + update_item1 = tuple_getitem(bn_update_grad_output, 1) + bn_reduce_grad_output = bn_training_reduce_grad(i0, i1, update_item0, update_item1, i2, i3, i4) + output = make_tuple(bn_reduce_grad_output, update_item0, update_item1) + item0 = tuple_getitem(output, 0) + item1 = tuple_getitem(output, 1) + item2 = tuple_getitem(output, 2) + output = make_tuple(item0, item1, item2) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/bn_split.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/bn_split.py new file mode 100644 index 00000000000..b6056392297 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/bn_split.py @@ -0,0 +1,90 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +bn = P.FusedBatchNorm() +fused_bn1 = Primitive('FusedBN1') +fused_bn2 = Primitive('FusedBN2') +fused_bn3 = Primitive('FusedBN3') +bn_training_reduce = Primitive('BNTrainingReduce') +bn_training_update = Primitive('BNTrainingUpdate') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_bn_split(tag): + """ test_split_bn_fusion """ + fns = FnDict() + + @fns + def before(x, scale, b, mean, variance): + bn_output = bn(x, scale, b, mean, variance) + output = tuple_getitem(bn_output, 0) + return output + + @fns + def after(x, scale, b, mean, variance): + fused_bn1_output = fused_bn1(x) + fused_bn2_input0 = tuple_getitem(fused_bn1_output, 0) + fused_bn2_input1 = tuple_getitem(fused_bn1_output, 1) + fused_bn2_output = fused_bn2(fused_bn2_input0, fused_bn2_input1, mean, variance) + fused_bn3_input1 = tuple_getitem(fused_bn1_output, 0) + fused_bn3_input2 = tuple_getitem(fused_bn2_output, 0) + fused_bn3_output = fused_bn3(x, fused_bn3_input1, fused_bn3_input2, scale, b) + output1 = tuple_getitem(fused_bn2_output, 1) + output2 = tuple_getitem(fused_bn2_output, 2) + output3 = tuple_getitem(fused_bn1_output, 0) + output4 = tuple_getitem(fused_bn2_output, 0) + output = make_tuple(fused_bn3_output, output1, output2, output3, output4) + output = tuple_getitem(output, 0) + return make_tuple(output) + + return fns[tag] + + +def test_bn_split_tbe(tag): + """ test_split_bn_fusion """ + fns = FnDict() + + @fns + def before(x, scale, b, mean, variance): + bn_output = bn(x, scale, b, mean, variance) + output = tuple_getitem(bn_output, 0) + return output + + @fns + def after(x, scale, b, mean, variance): + bn_training_reduce_output = bn_training_reduce(x) + bn_training_update_input1 = tuple_getitem(bn_training_reduce_output, 0) + bn_training_update_input2 = tuple_getitem(bn_training_reduce_output, 1) + bn_training_update_output = bn_training_update(x, bn_training_update_input1, bn_training_update_input2, + scale, b, mean, variance) + output = tuple_getitem(bn_training_update_output, 0) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/buffer_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/buffer_fusion_test.py new file mode 100644 index 00000000000..b4e4c2744e4 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/buffer_fusion_test.py @@ -0,0 +1,135 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +from mindspore.ops import Primitive +import mindspore.common.dtype as mstype + +Conv = P.Conv2D(out_channel=64, kernel_size=7, mode=1, pad_mode="valid", pad=0, stride=1, dilation=1, group=1) +Relu = P.ReLU() +Fusion = Primitive('FusionOp') +Reduce = P.ReduceOp() +Biasadd = P.BiasAdd() +Biasaddgrad = G.BiasAddGrad() +Cast = P.Cast() + +Fusion_relu_relu = Primitive('FusionOp_ReLU_ReLU') +Fusion_biasadd = Primitive('FusionOp_ReLU_ReLU_ReLU_BiasAdd_ReLU_ReLU_ReLU') +Fusion_biasaddgrad = Primitive('FusionOp_ReLU_ReLU_ReLU_BiasAddGrad_ReLU_ReLU_ReLU') + +Add = P.TensorAdd() +Sub = P.Sub() +make_tuple = Primitive('make_tuple') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_tbe_eltwise_fusion_1(tag): + fns = FnDict() + + @fns + def before(x): + relu1 = Relu(x) + relu2 = Relu(relu1) + res = Cast(relu2, mstype.float16) + return res + + @fns + def after(x): + fusion = Fusion_relu_relu(x) + res = Cast(fusion) + tuple = make_tuple(res) + return tuple + + return fns[tag] + + +def test_tbe_eltwise_fusion_2(tag): + fns = FnDict() + + @fns + def before(x, y): + relu1 = Relu(x) + relu2 = Relu(relu1) + relu3 = Relu(relu2) + biasadd = Biasadd(relu3, y) + relu4 = Relu(biasadd) + relu5 = Relu(relu4) + relu6 = Relu(relu5) + res = Cast(relu6, mstype.float16) + return res + + @fns + def after(x, y): + fusion = Fusion_biasadd(x, y) + res = Cast(fusion) + tuple = make_tuple(res) + return tuple + + return fns[tag] + + +def test_tbe_reduce_eltwise_fusion(tag): + fns = FnDict() + + @fns + def before(x): + relu1 = Relu(x) + relu2 = Relu(relu1) + relu3 = Relu(relu2) + biasaddgrad = Biasaddgrad(relu3) + relu4 = Relu(biasaddgrad) + relu5 = Relu(relu4) + relu6 = Relu(relu5) + res = Cast(relu6, mstype.float16) + return res + + @fns + def after(x): + fusion = Fusion_biasaddgrad(x) + res = Cast(fusion) + tuple = make_tuple(res) + return tuple + + return fns[tag] + + +def test_conv_singlein_fusion(tag): + fns = FnDict() + + @fns + def before(x, y): + conv = Conv(x, y) + relu = Relu(conv) + res = Cast(relu, mstype.float16) + return res + + @fns + def after(x, y): + fusion = Fusion(x, y) + res = Cast(fusion) + tuple = make_tuple(res) + return tuple + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/check_consistency.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/check_consistency.py new file mode 100644 index 00000000000..043128fe5dd --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/check_consistency.py @@ -0,0 +1,41 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +cast = Primitive('Cast') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_check_consistency(tag): + fns = FnDict() + + @fns + def graph(x): + x = cast(x) + return x + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/clip_by_norm_no_div_square_sum_fusion.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/clip_by_norm_no_div_square_sum_fusion.py new file mode 100644 index 00000000000..7dd1ba71658 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/clip_by_norm_no_div_square_sum_fusion.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +select = P.Select() +maximum = P.Maximum() +sqrt = P.Sqrt() +greater = P.Greater() +clip_by_norm_no_div_square_sum = Primitive('ClipByNormNoDivSum') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_clip_by_norm_no_div_square_sum_fusion(tag): + """ test_clip_by_norm_no_div_square_sum_fusion """ + fns = FnDict() + + @fns + def before(input, constant_select, constant_greater, constant_maximum): + greater_output = greater(input, constant_greater) + res = select(greater_output, input, constant_select) + res = sqrt(res) + res = select(greater_output, res, input) + res = maximum(res, constant_maximum) + return res + + @fns + def after(input, constant_select, constant_greater, constant_maximum): + res = clip_by_norm_no_div_square_sum(input, constant_select, constant_greater, constant_maximum) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/clip_by_value_fusion.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/clip_by_value_fusion.py new file mode 100644 index 00000000000..148bb2d3bd4 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/clip_by_value_fusion.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +maximum = P.Maximum() +minimum = P.Minimum() +clip_by_value = Primitive('ClipByValue') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_clip_by_value_fusion(tag): + """ test_clip_by_value_fusion """ + fns = FnDict() + + @fns + def before1(input0, input1, input2): + res = minimum(input0, input1) + res = maximum(res, input2) + return res + + @fns + def before2(input0, input1, input2): + res = minimum(input0, input1) + res = maximum(input2, res) + return res + + @fns + def after(input0, input1, input2): + res = clip_by_value(input0, input2, input1) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/confusion_softmax_grad_rule.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/confusion_softmax_grad_rule.py new file mode 100644 index 00000000000..2727ef641d8 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/confusion_softmax_grad_rule.py @@ -0,0 +1,55 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +mul = P.Mul() +reduce_sum = P.ReduceSum() +sub = P.Sub() +confusion_softmax_grad = Primitive('ConfusionSoftmaxGrad') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +axis = 2 + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_confusion_softmax_grad_rule(tag): + """ test_confusion_softmax_grad_rule """ + fns = FnDict() + + @fns + def before(input0, input1): + res = mul(input0, input1) + # input axis will be convert to attr in ConstructKernelGraph step + res = reduce_sum(res, axis) + res = sub(input0, res) + return res + + @fns + def after(input0, input1): + res = confusion_softmax_grad(input0, input1) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/conv_bn_relu_fusion.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/conv_bn_relu_fusion.py new file mode 100644 index 00000000000..605591560c9 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/conv_bn_relu_fusion.py @@ -0,0 +1,71 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +conv = P.Conv2D(out_channel=64, kernel_size=7, mode=1, pad_mode="valid", pad=0, stride=1, dilation=1, group=1) +bn = P.FusedBatchNorm() +relu = P.ReLU() +conv_bn1 = Primitive('ConvBN1') +bn2_relu = Primitive('BN2Relu') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_conv_bn_relu_fusion(tag): + """ test_conv_bn_relu_fusion """ + fns = FnDict() + + @fns + def before(x, w, scale, b, mean, variance): + conv_output = conv(x, w) + bn_output = bn(conv_output, scale, b, mean, variance) + item0 = tuple_getitem(bn_output, 0) + item1 = tuple_getitem(bn_output, 3) + item2 = tuple_getitem(bn_output, 4) + output = make_tuple(relu(item0), item1, item2) + res = tuple_getitem(output, 0) + return res + + @fns + def after(x, w, scale, b, mean, variance): + conv_bn1_output = conv_bn1(x, w) + conv_item0 = tuple_getitem(conv_bn1_output, 0) + conv_item1 = tuple_getitem(conv_bn1_output, 1) + conv_item2 = tuple_getitem(conv_bn1_output, 2) + bn2_relu_output = bn2_relu(conv_item0, conv_item1, conv_item2, scale, b, mean, variance) + bn2_relu_item0 = tuple_getitem(bn2_relu_output, 0) + bn2_relu_item1 = tuple_getitem(bn2_relu_output, 1) + bn2_relu_item2 = tuple_getitem(bn2_relu_output, 2) + bn2_relu_item3 = tuple_getitem(bn2_relu_output, 3) + new_make_tuple = make_tuple(bn2_relu_item0, bn2_relu_item1, bn2_relu_item2, conv_item2, bn2_relu_item3) + item1 = tuple_getitem(new_make_tuple, 3) + item2 = tuple_getitem(new_make_tuple, 4) + output = make_tuple(bn2_relu_item0, item1, item2) + return make_tuple(tuple_getitem(output, 0)) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/convert_const_input_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/convert_const_input_test.py new file mode 100644 index 00000000000..1ef32eae994 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/convert_const_input_test.py @@ -0,0 +1,163 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +from mindspore.ops import Primitive +import mindspore as ms +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor + +make_tuple = Primitive('make_tuple') +reshape = P.Reshape() +backend_reshape = Primitive('Reshape') +cast = P.Cast() +backend_cast = Primitive('Cast') +transpose = P.Transpose() +backend_transpose = Primitive('Transpose') +onehot1 = P.OneHot() +onehot2 = P.OneHot() +backend_onehot1 = Primitive('OneHot') +backend_onehot2 = Primitive('OneHot') +stridedslicegrad = G.StridedSliceGrad() +backend_stridedslicegrad = Primitive('StridedSliceGrad') +on_value = Tensor(1.0, mstype.float32) +off_value = Tensor(0.0, mstype.float32) +depth = Tensor(2, mstype.int32) +shape = (2, 4, 2, 2) +dropout_gen_mask = P.DropoutGenMask() + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_convert_reshape_input_to_attr(tag): + fns = FnDict() + + @fns + def before(x): + return reshape(x, (3, 2)) + + @fns + def after(x): + res = backend_reshape(x) + return make_tuple(res) + + return fns[tag] + + +def test_convert_cast_input_to_attr(tag): + fns = FnDict() + + @fns + def before(x): + return cast(x, ms.int32) + + @fns + def after(x): + res = backend_cast(x) + return make_tuple(res) + + return fns[tag] + + +def test_convert_transpose_input_to_attr(tag): + fns = FnDict() + + @fns + def before(x): + return transpose(x, (0, 2, 1)) + + @fns + def after(x): + res = backend_transpose(x) + return make_tuple(res) + + return fns[tag] + + +def test_convert_onehot_input_to_attr(tag): + fns = FnDict() + + @fns + def before(x): + return onehot1(x, 2, on_value, off_value) + + @fns + def after(x): + res = backend_onehot1(x, on_value, off_value) + return make_tuple(res) + + return fns[tag] + + +def test_convert_strided_slice_grad_input_to_attr(tag): + fns = FnDict() + + @fns + def before(x): + return stridedslicegrad(x, (16, 128, 1024), (0, 0 , 0), (16, 1, 1024), (1, 1,1)) + + @fns + def after(x): + res = backend_stridedslicegrad(x) + return make_tuple(res) + + return fns[tag] + + +def test_convert_onehot_input_to_tensor1(tag): + fns = FnDict() + + @fns + def before(x): + return onehot2(x, 2, on_value, off_value) + + @fns + def after_func_graph(x): + res = backend_onehot2(x, depth, on_value, off_value) + return make_tuple(res) + + return fns[tag] + + +def test_convert_onehot_input_to_tensor2(tag): + fns = FnDict() + + @fns + def before(x): + return onehot2(x, 2, on_value, off_value) + + @fns + def after_kernel_graph(x): + return make_tuple(backend_onehot2(x, on_value, off_value)) + + return fns[tag] + + +def test_convert_dropout_gen_mask_tuple_input_to_tensor(tag): + fns = FnDict() + + @fns + def before(x): + return dropout_gen_mask(shape, x) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/convert_tuple_input_to_dynamic_input_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/convert_tuple_input_to_dynamic_input_test.py new file mode 100644 index 00000000000..24cfcdd01dd --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/convert_tuple_input_to_dynamic_input_test.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive +import mindspore as ms +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +import numpy as np + +make_tuple = Primitive('make_tuple') +concat = P.Concat() +add = P.TensorAdd() + +t1 = Tensor(np.random.randn(1, 11, 20, 1, 1).astype(np.float32)) +t2 = Tensor(np.random.randn(1, 11, 20, 1, 1).astype(np.float32)) + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_convert_tuple_input_to_dynamic_input(tag): + fns = FnDict() + + @fns + def before(x): + res = concat((t1, t2)) + res = add(x, res) + return res + + @fns + def after(x): + res = concat(t1, t2) + res = add(x, res) + res = make_tuple(res); + return res + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/eliminate_redundant_op_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/eliminate_redundant_op_test.py new file mode 100644 index 00000000000..652ead29938 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/eliminate_redundant_op_test.py @@ -0,0 +1,204 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +add = P.TensorAdd() +sub = P.Sub() +make_tuple = Primitive('make_tuple') +four2five = Primitive('Four2Five') +five2four = Primitive('Five2Four') +transdata = Primitive("TransData") +cast = Primitive('Cast') +depend = Primitive('depend') + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_eliminate_5to4_4to5(tag): + fns = FnDict() + + @fns + def before(x, y): + sum = add(x, y) + res = sub(sum, y) + output = make_tuple(res) + return output + + @fns + def after1(x, y): + new_x_sum = transdata(x) + new_y_sum = transdata(y) + new_y_sum2 = transdata(y) + sum = add(new_x_sum, new_y_sum) + sum_5to4 = transdata(sum) + sum_4to5 = transdata(sum_5to4) + res = sub(sum_4to5, new_y_sum2) + output = transdata(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + @fns + def after2(x, y): + new_x_sum = transdata(x) + new_y_sum = transdata(y) + new_y_diff = transdata(y) + sum = add(new_x_sum, new_y_sum) + res = sub(sum, new_y_diff) + output = transdata(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + return fns[tag] + + +def test_eliminate_cast(tag): + fns = FnDict() + + @fns + def before(x, y): + sum = add(x, y) + res = sub(sum, y) + output = make_tuple(res) + return output + + @fns + def after1(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + new_y_sum2 = cast(y) + sum = add(new_x_sum, new_y_sum) + sum_cast1 = cast(sum) + sum_cast2 = cast(sum_cast1) + res = sub(sum_cast2, new_y_sum2) + output = cast(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + @fns + def after2(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + new_y_diff = cast(y) + sum = add(new_x_sum, new_y_sum) + res = sub(sum, new_y_diff) + output = cast(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + return fns[tag] + + +def test_eliminate_5to4_depend_4to5(tag): + fns = FnDict() + + @fns + def before(x, y): + sum = add(x, y) + sum_depend = depend(sum, x) + res = sub(sum_depend, y) + output = make_tuple(res) + return output + + @fns + def after1(x, y): + new_x_sum = transdata(x) + new_y_sum = transdata(y) + sum = add(new_x_sum, new_y_sum) + sum_trans = transdata(sum) + depend_between_trans = depend(sum_trans, x) + depend_trans = transdata(depend_between_trans) + new_y_diff = transdata(y) + res = sub(depend_trans, new_y_diff) + output = transdata(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + @fns + def after2(x, y): + new_x_sum = transdata(x) + new_y_sum = transdata(y) + sum = add(new_x_sum, new_y_sum) + depend_op = depend(sum, x) + new_y_diff = transdata(y) + res = sub(depend_op, new_y_diff) + output = transdata(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + return fns[tag] + + +def test_eliminate_cast_depend_cast(tag): + fns = FnDict() + + @fns + def before(x, y): + sum = add(x, y) + sum_depend = depend(sum, x) + sum_depend2 = depend(sum_depend, x) + sum_depend3 = depend(sum_depend2, x) + res = sub(sum_depend3, y) + output = make_tuple(res) + return output + + @fns + def after1(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + sum = add(new_x_sum, new_y_sum) + sum_cast = cast(sum) + depend_between_cast = depend(sum_cast, x) + depend_between_cast2 = depend(depend_between_cast, x) + depend_between_cast3 = depend(depend_between_cast2, x) + depend_cast = cast(depend_between_cast3) + new_y_diff = cast(y) + res = sub(depend_cast, new_y_diff) + output = cast(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + @fns + def after2(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + sum = add(new_x_sum, new_y_sum) + depend_op = depend(sum, x) + depend_op2 = depend(depend_op, x) + depend_op3 = depend(depend_op2, x) + new_y_diff = cast(y) + res = sub(depend_op3, new_y_diff) + output = cast(res) + new_output = make_tuple(output) + ret = make_tuple(new_output) + return ret + + return fns[tag] + diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/fused_batch_norm_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/fused_batch_norm_fusion_test.py new file mode 100644 index 00000000000..8f4b8b476f2 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/fused_batch_norm_fusion_test.py @@ -0,0 +1,69 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor + +AssignSub = P.AssignSub() +Mul = P.Mul() +Sub = P.Sub() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +depend = Primitive('depend') +BatchNorm = P.BatchNorm() +FusedBatchNorm = P.FusedBatchNorm() +constant0 = Tensor(0.1, mstype.float32) +constant1 = Tensor(0.1, mstype.float32) + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def useless_test_fused_batch_norm_fusion(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, var0, var1): + batch_norm = BatchNorm(input0, input1, input2, input3, input4) + sub0 = Sub(var0, tuple_getitem(batch_norm, 1)) + sub1 = Sub(var1, tuple_getitem(batch_norm, 2)) + mul0 = Mul(sub0, constant0) + mul1 = Mul(sub1, constant1) + assign_sub0 = AssignSub(var0, mul0) + assign_sub1 = AssignSub(var1, mul1) + depend0 = depend(tuple_getitem(batch_norm, 0), assign_sub0) + depend1 = depend(depend0, assign_sub1) + outputs = make_tuple(depend1, tuple_getitem(batch_norm, 3), tuple_getitem(batch_norm, 4)) + output = tuple_getitem(outputs, 0) + return output + + @fns + def after(input0, input1, input2, input3, input4, var0, var1): + fused_batch_norm = FusedBatchNorm(input0, input1, input2, var0, var1) + outputs = make_tuple(tuple_getitem(fused_batch_norm, 0), tuple_getitem(fused_batch_norm, 3), + tuple_getitem(fused_batch_norm, 4)) + output = tuple_getitem(outputs, 0) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/getitem_tuple.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/getitem_tuple.py new file mode 100644 index 00000000000..ca61996dc20 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/getitem_tuple.py @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_getitem_tuple(tag): + """ test_tuple_getitem """ + fns = FnDict() + + @fns + def before(x, y): + return tuple_getitem(make_tuple(x, y), 0) + + @fns + def after(x, y): + return x + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/hw_opt_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/hw_opt_test.py new file mode 100644 index 00000000000..0afffc99df6 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/hw_opt_test.py @@ -0,0 +1,181 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +tuple_getitem = Primitive('tuple_getitem') +depend = Primitive('depend') +addn = P.AddN() +add = P.TensorAdd() +sub = P.Sub() +mul = P.Mul() +max_pool = P.MaxPoolWithArgmax(pad_mode="same", window=3, stride=2) +make_tuple = Primitive('make_tuple') +four2five = Primitive('Four2Five') +five2four = Primitive('Five2Four') +cast = Primitive('Cast') +conv = P.Conv2D(out_channel=64, kernel_size=7, mode=1, pad_mode="valid", pad=0, stride=1, dilation=1, group=1) +bn = P.FusedBatchNorm() +relu = P.ReLU() + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_merge_cast_input(tag): + fns = FnDict() + + @fns + def before(x, y): + res = addn((x, y)) + return res + + @fns + def after1(x, y): + new_x = cast(x) + new_y = cast(y) + res = addn(four2five(new_x), four2five(new_y)) + output = cast(five2four(res)) + new_output = make_tuple(output) + return new_output + + @fns + def after2(x, y): + new_x = four2five(x) + new_y = four2five(y) + res = addn(new_x, new_y) + output = cast(five2four(res)) + new_output = make_tuple(output) + return new_output + + return fns[tag] + + +def test_merge_cast_output_for_single_output(tag): + fns = FnDict() + + @fns + def before(x, y): + res = addn((x, y)) + return res + + @fns + def after1(x, y): + new_x = cast(x) + new_y = cast(y) + res = addn(four2five(new_x), four2five(new_y)) + output = cast(five2four(res)) + new_output = make_tuple(output) + return new_output + + @fns + def after2(x, y): + new_x = four2five(cast(x)) + new_y = four2five(cast(y)) + res = addn(new_x, new_y) + output = five2four(res) + new_output = make_tuple(output) + return new_output + + return fns[tag] + + +def test_merge_cast_output_for_multiple_output(tag): + fns = FnDict() + + @fns + def before(x): + output = max_pool(x) + res = tuple_getitem(output, 0) + return res + + @fns + def after(x): + output = max_pool(x) + item0 = tuple_getitem(output, 0) + item1 = cast(tuple_getitem(output, 1)) + res = make_tuple(item0, item1) + return make_tuple(tuple_getitem(res, 0)) + + return fns[tag] + + +def test_eliminate_depend_input2(tag): + fns = FnDict() + + @fns + def before(x, y, z): + new_z = four2five(z) + depend_intput = depend(y, new_z) + sum = add(x, depend_intput) + return sum + + @fns + def after(x, y, z): + depend_intput = depend(y, z) + sum = add(x, depend_intput) + return sum + + return fns[tag] + + +def test_opt_match(tag): + fns = FnDict() + + @fns + def graph1(x, y): + sum = add(x, y) + output = make_tuple(sum) + return output + + @fns + def graph2(x, w, scale, b, mean, variance): + conv_output = conv(x, w) + bn_output = bn(conv_output, scale, b, mean, variance) + res = tuple_getitem(bn_output, 0) + return res + + return fns[tag] + + +def test_func_graph_cse(tag): + """ test_func_graph_cse """ + fns = FnDict() + + @fns + def g1(x, y): + a = add(x, y) + b = add(x, y) + c = mul(a, b) + return c + + @fns + def g2(x, y): + a = add(x, y) + b = add(mul(a, y), sub(a, x)) + c = add(mul(a, y), sub(add(x, y), x)) + d = add(b, c) + return d + + return fns[tag] + diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/insert_trans_op_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/insert_trans_op_test.py new file mode 100644 index 00000000000..57bd2000c4e --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/insert_trans_op_test.py @@ -0,0 +1,74 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +tuple_getitem = Primitive('tuple_getitem') +add = P.TensorAdd() +max_pool = P.MaxPoolWithArgmax(pad_mode="same", window=3, stride=2) +make_tuple = Primitive('make_tuple') +transdata = Primitive("TransData") + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_insert_trans_op_for_single_output(tag): + fns = FnDict() + + @fns + def before(x, y): + res = add(x, y) + return res + + @fns + def after(x, y): + new_x = transdata(x) + new_y = transdata(y) + res = add(new_x, new_y) + output = transdata(res) + res = make_tuple(output) + return res + + return fns[tag] + + +def test_insert_trans_op_for_multiple_output(tag): + fns = FnDict() + + @fns + def before(x): + max_pool_res = max_pool(x) + res = tuple_getitem(max_pool_res, 0) + return res + + @fns + def after(x): + output = max_pool(x) + transdata0 = transdata(tuple_getitem(output, 0)) + transdata1 = transdata(tuple_getitem(output, 1)) + res = make_tuple(transdata0, transdata1) + res = tuple_getitem(res, 0) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/ir_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/ir_fusion_test.py new file mode 100644 index 00000000000..5c30de48782 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/ir_fusion_test.py @@ -0,0 +1,281 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +from mindspore.ops import Primitive + +# pylint: disable=unused-variable + +tuple_getitem = Primitive('tuple_getitem') +add = P.TensorAdd() +allreduce = P.AllReduce() +allreduce.add_prim_attr('fusion', 1) +make_tuple = Primitive('make_tuple') +conv = P.Conv2D(out_channel=64, kernel_size=7, mode=1, pad_mode="valid", pad=0, stride=1, dilation=1, group=1) +bn = P.FusedBatchNorm() +relu = P.ReLU() +conv_bn1 = Primitive('ConvBN1') +bn2_add_relu = Primitive('BN2AddRelu') +bn2_relu = Primitive('BN2Relu') +fused_bn1 = Primitive('FusedBN1') +fused_bn2 = Primitive('FusedBN2') +fused_bn3 = Primitive('FusedBN3') +bn_grad = G.FusedBatchNormGrad() +bn_grad1 = Primitive('BNGrad1') +bn_grad2 = Primitive('BNGrad2') +bn_grad3 = Primitive('BNGrad3') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_conv_bn_fusion(tag): + """ test_conv_bn_fusion """ + fns = FnDict() + + @fns + def before(x, w, scale, b, mean, variance): + conv_output = conv(x, w) + bn_output = bn(conv_output, scale, b, mean, variance) + item0 = tuple_getitem(bn_output, 0) + item1 = tuple_getitem(bn_output, 3) + item2 = tuple_getitem(bn_output, 4) + output = make_tuple(item0, item1, item2) + res = tuple_getitem(output, 0) + return res + + @fns + def after(x, w, scale, b, mean, variance): + conv_bn1_output = conv_bn1(x, w) + conv_item0 = tuple_getitem(conv_bn1_output, 0) + conv_item1 = tuple_getitem(conv_bn1_output, 1) + conv_item2 = tuple_getitem(conv_bn1_output, 2) + bn2_output = fused_bn2(conv_item2, conv_item1, mean, variance) + bn2_item0 = tuple_getitem(bn2_output, 0) + bn2_item1 = tuple_getitem(bn2_output, 1) + bn2_item2 = tuple_getitem(bn2_output, 2) + bn3_output = fused_bn3(conv_item0, conv_item2, bn2_item0, scale, b) + output = make_tuple(bn3_output, bn2_item1, bn2_item2, conv_item2, bn2_item0) + item0 = tuple_getitem(output, 0) + item1 = tuple_getitem(output, 3) + item2 = tuple_getitem(output, 4) + new_output = make_tuple(item0, item1, item2) + return make_tuple(tuple_getitem(new_output, 0)) + + return fns[tag] + + +def test_conv_bn_add_relu_fusion(tag): + """ test_conv_bn_add_relu_fusion """ + fns = FnDict() + + @fns + def before(x, w, scale, b, mean, variance, y): + conv_output = conv(x, w) + bn_output = bn(conv_output, scale, b, mean, variance) + item0 = tuple_getitem(bn_output, 0) + s = add(item0, y) + res = relu(s) + return res + + @fns + def after(x, w, scale, b, mean, variance, y): + conv_bn1_output = conv_bn1(x, w) + conv_item0 = tuple_getitem(conv_bn1_output, 0) + conv_item1 = tuple_getitem(conv_bn1_output, 1) + conv_item2 = tuple_getitem(conv_bn1_output, 2) + bn2_add_relu_output = bn2_add_relu(conv_item0, conv_item1, conv_item2, y, scale, b, mean, variance) + bn2_add_relu_item0 = tuple_getitem(bn2_add_relu_output, 0) + res = make_tuple(bn2_add_relu_item0) + return res + + return fns[tag] + + +def test_conv_bn_relu_fusion(tag): + """ test_conv_bn_relu_fusion """ + fns = FnDict() + + @fns + def before(x, w, scale, b, mean, variance): + conv_output = conv(x, w) + bn_output = bn(conv_output, scale, b, mean, variance) + item0 = tuple_getitem(bn_output, 0) + item1 = tuple_getitem(bn_output, 3) + item2 = tuple_getitem(bn_output, 4) + output = make_tuple(relu(item0), item1, item2) + res = tuple_getitem(output, 0) + return res + + @fns + def after(x, w, scale, b, mean, variance): + conv_bn1_output = conv_bn1(x, w) + conv_item0 = tuple_getitem(conv_bn1_output, 0) + conv_item1 = tuple_getitem(conv_bn1_output, 1) + conv_item2 = tuple_getitem(conv_bn1_output, 2) + bn2_relu_output = bn2_relu(conv_item0, conv_item1, conv_item2, scale, b, mean, variance) + bn2_relu_item0 = tuple_getitem(bn2_relu_output, 0) + bn2_relu_item1 = tuple_getitem(bn2_relu_output, 1) + bn2_relu_item2 = tuple_getitem(bn2_relu_output, 2) + bn2_relu_item3 = tuple_getitem(bn2_relu_output, 3) + new_make_tuple = make_tuple(bn2_relu_item0, bn2_relu_item1, bn2_relu_item2, conv_item2, bn2_relu_item3) + item1 = tuple_getitem(new_make_tuple, 3) + item2 = tuple_getitem(new_make_tuple, 4) + output = make_tuple(bn2_relu_item0, item1, item2) + return make_tuple(tuple_getitem(output, 0)) + + return fns[tag] + + +def test_bn_split(tag): + """ test_split_bn_fusion """ + fns = FnDict() + + @fns + def before(x, scale, b, mean, variance): + bn_output = bn(x, scale, b, mean, variance) + item0 = tuple_getitem(bn_output, 0) + return item0 + + @fns + def after(x, scale, b, mean, variance): + fused_bn1_output = fused_bn1(x) + fused_bn2_input0 = tuple_getitem(fused_bn1_output, 0) + fused_bn2_input1 = tuple_getitem(fused_bn1_output, 1) + fused_bn2_output = fused_bn2(fused_bn2_input0, fused_bn2_input1, mean, variance) + fused_bn3_input1 = tuple_getitem(fused_bn2_output, 0) + fused_bn3_input2 = tuple_getitem(fused_bn2_output, 1) + fused_bn3_output = fused_bn3(x, fused_bn3_input1, fused_bn3_input2, scale, b) + output1 = tuple_getitem(fused_bn2_output, 2) + output2 = tuple_getitem(fused_bn2_output, 3) + output3 = tuple_getitem(fused_bn2_output, 0) + output4 = tuple_getitem(fused_bn2_output, 1) + output = make_tuple(fused_bn3_output, output1, output2, output3, output4) + item0 = tuple_getitem(output, 0) + return make_tuple(item0) + + return fns[tag] + + +def test_bn_grad_split(tag): + """ test_bn_grad_split """ + fns = FnDict() + + @fns + def before(dy, x, scale, save_mean, save_inv_variance): + bn_grad_output = bn_grad(dy, x, scale, save_mean, save_inv_variance) + item0 = tuple_getitem(bn_grad_output, 0) + item1 = tuple_getitem(bn_grad_output, 1) + item2 = tuple_getitem(bn_grad_output, 2) + output = make_tuple(item0, item1, item2) + res = tuple_getitem(output, 0) + return res + + @fns + def after(i0, i1, i2, i3, i4): + bn_grad1_output = bn_grad1(i0, i1, i3) + bn_grad1_item0 = tuple_getitem(bn_grad1_output, 0) + bn_grad1_item1 = tuple_getitem(bn_grad1_output, 1) + bn_grad1_item2 = tuple_getitem(bn_grad1_output, 2) + bn_grad2_output = bn_grad2(bn_grad1_item0, bn_grad1_item1, i4, i2) + bn_grad2_item0 = tuple_getitem(bn_grad2_output, 0) + bn_grad2_item1 = tuple_getitem(bn_grad2_output, 1) + bn_grad2_item2 = tuple_getitem(bn_grad2_output, 2) + bn_grad2_item3 = tuple_getitem(bn_grad2_output, 3) + bn_grad2_item4 = tuple_getitem(bn_grad2_output, 4) + bn_grad3_output = bn_grad3(i0, bn_grad2_item2, bn_grad2_item3, bn_grad2_item4, bn_grad1_item2) + bn_grad_make_tuple = make_tuple(bn_grad3_output, bn_grad2_item0, bn_grad2_item1) + item0 = tuple_getitem(bn_grad_make_tuple, 0) + item1 = tuple_getitem(bn_grad_make_tuple, 1) + item2 = tuple_getitem(bn_grad_make_tuple, 2) + output = make_tuple(item0, item1, item2) + return make_tuple(tuple_getitem(output, 0)) + + return fns[tag] + + +def test_all_reduce_fusion_all(tag): + """ test_all_reduce_fusion_all """ + fns = FnDict() + + @fns + def before(x1, x2, x3, x4, x5): + y1 = allreduce(x1) + y2 = allreduce(x2) + y3 = allreduce(x3) + y4 = allreduce(x4) + y5 = allreduce(x5) + return make_tuple(y1, y2, y3, y4, y5) + + @fns + def after(x1, x2, x3, x4, x5): + ar = allreduce(x5, x4, x3, x2, x1) + y5 = tuple_getitem(ar, 0) + y4 = tuple_getitem(ar, 1) + y3 = tuple_getitem(ar, 2) + y2 = tuple_getitem(ar, 3) + y1 = tuple_getitem(ar, 4) + res = make_tuple(y1, y2, y3, y4, y5) + return make_tuple(res) + + return fns[tag] + + +def test_all_reduce_fusion_group(tag): + """ test_all_reduce_fusion_group """ + fns = FnDict() + + @fns + def before(x1, x2, x3, x4, x5): + y1 = allreduce(x1) + y2 = allreduce(x2) + y3 = allreduce(x3) + y4 = allreduce(x4) + y5 = allreduce(x5) + return make_tuple(y1, y2, y3, y4, y5) + + @fns + def after1(x1, x2, x3, x4, x5): + ar1 = allreduce(x5, x4) + ar2 = allreduce(x3, x2, x1) + y4 = tuple_getitem(ar1, 1) + y5 = tuple_getitem(ar1, 0) + y1 = tuple_getitem(ar2, 2) + y2 = tuple_getitem(ar2, 1) + y3 = tuple_getitem(ar2, 0) + res = make_tuple(y1, y2, y3, y4, y5) + return make_tuple(res) + + @fns + def after2(x1, x2, x3, x4, x5): + ar1 = allreduce(x1, x3, x5) + ar2 = allreduce(x2, x4) + y1 = tuple_getitem(ar1, 2) + y3 = tuple_getitem(ar1, 1) + y5 = tuple_getitem(ar1, 0) + y2 = tuple_getitem(ar2, 1) + y4 = tuple_getitem(ar2, 0) + output = make_tuple(y1, y2, y3, y4, y5) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_rule_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_rule_test.py new file mode 100644 index 00000000000..6690408b08e --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_rule_test.py @@ -0,0 +1,169 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Add = P.TensorAdd() +Mul = P.Mul() +RealDiv = P.RealDiv() +Rsqrt = P.Rsqrt() +Sqrt = P.Sqrt() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +LambNextMV = Primitive('LambNextMV') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_lamb_next_mv_rule(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul0 = Mul(constant_mul0_x, input4) + mul1 = Mul(constant_mul1_sub, input3) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt0 = Rsqrt(add2) + sqrt1 = Sqrt(real_div1) + add4 = Add(sqrt1, constant_add2_y) + real_div0 = RealDiv(add0, input5) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + mul4 = Mul(constant_mul4_x, input6) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, real_div4) + output = tuple_getitem(outputs, 0) + return output + + @fns + def after(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + lamb_next_mv = LambNextMV(input0, input1, input2, input3, input4, input5, input6, + constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, + constant_add2_y) + outputs = make_tuple(tuple_getitem(lamb_next_mv, 0), tuple_getitem(lamb_next_mv, 1), + tuple_getitem(lamb_next_mv, 2), tuple_getitem(lamb_next_mv, 3)) + output = tuple_getitem(outputs, 0) + return make_tuple(output) + + @fns + def before_unmatched_real_div4(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul0 = Mul(constant_mul0_x, input4) + mul1 = Mul(constant_mul1_sub, input3) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt0 = Rsqrt(add2) + sqrt1 = Sqrt(real_div1) + add4 = Add(sqrt1, constant_add2_y) + real_div0 = RealDiv(add0, input5) + real_div4 = Mul(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + mul4 = Mul(constant_mul4_x, input6) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, real_div4) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched_real_div0(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul0 = Mul(constant_mul0_x, input4) + mul1 = Mul(constant_mul1_sub, input3) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt0 = Rsqrt(add2) + sqrt1 = Sqrt(real_div1) + add4 = Add(sqrt1, constant_add2_y) + real_div0 = Mul(add0, input5) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + mul4 = Mul(constant_mul4_x, input6) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, real_div4) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched_real_div1(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul0 = Mul(constant_mul0_x, input4) + mul1 = Mul(constant_mul1_sub, input3) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = Mul(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt0 = Rsqrt(add2) + sqrt1 = Sqrt(real_div1) + add4 = Add(sqrt1, constant_add2_y) + real_div0 = RealDiv(add0, input5) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + mul4 = Mul(constant_mul4_x, input6) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, real_div4) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched_real_div2(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul0 = Mul(constant_mul0_x, input4) + mul1 = Mul(constant_mul1_sub, input3) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt0 = Rsqrt(add2) + sqrt1 = Sqrt(real_div1) + add4 = Add(sqrt1, constant_add2_y) + real_div0 = RealDiv(add0, input5) + real_div4 = RealDiv(real_div0, add4) + real_div2 = RealDiv(real_div0, sqrt0) + mul4 = Mul(constant_mul4_x, input6) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, real_div4) + output = tuple_getitem(outputs, 0) + return output + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_with_decay_rule_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_with_decay_rule_test.py new file mode 100644 index 00000000000..fd11175eb6c --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_with_decay_rule_test.py @@ -0,0 +1,178 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Add = P.TensorAdd() +Mul = P.Mul() +RealDiv = P.RealDiv() +Rsqrt = P.Rsqrt() +Sqrt = P.Sqrt() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +LambNextMVWithDecay = Primitive('LambNextMVWithDecay') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_lamb_next_mv_with_decay_rule(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul1 = Mul(constant_mul1_sub, input3) + mul0 = Mul(constant_mul0_x, input4) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt1 = Sqrt(real_div1) + real_div0 = RealDiv(add0, input5) + add4 = Add(sqrt1, constant_add2_y) + sqrt0 = Rsqrt(add2) + mul4 = Mul(constant_mul4_x, input6) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + add5 = Add(real_div4, mul4) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, add5) + output = tuple_getitem(outputs, 0) + return output + + @fns + def after(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + lamb_next_mv_with_decay = LambNextMVWithDecay(input0, input1, input2, input3, input4, input5, input6, + constant_mul0_x, constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, + constant_add2_y) + outputs = make_tuple(tuple_getitem(lamb_next_mv_with_decay, 0), tuple_getitem(lamb_next_mv_with_decay, 1), + tuple_getitem(lamb_next_mv_with_decay, 2), tuple_getitem(lamb_next_mv_with_decay, 3)) + output = tuple_getitem(outputs, 0) + return make_tuple(output) + + @fns + def before_unmatched_add3(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul1 = Mul(constant_mul1_sub, input3) + mul0 = Mul(constant_mul0_x, input4) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt1 = Sqrt(real_div1) + real_div0 = RealDiv(add0, input5) + add4 = Add(sqrt1, constant_add2_y) + sqrt0 = Rsqrt(add2) + mul4 = Mul(constant_mul4_x, input6) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + add5 = Add(real_div4, mul4) + add3 = Mul(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, add5) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched_mul4(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul1 = Mul(constant_mul1_sub, input3) + mul0 = Mul(constant_mul0_x, input4) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt1 = Sqrt(real_div1) + real_div0 = RealDiv(add0, input5) + add4 = Add(sqrt1, constant_add2_y) + sqrt0 = Rsqrt(add2) + mul4 = Add(constant_mul4_x, input6) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + add5 = Add(real_div4, mul4) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, add5) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched_real_div0(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul1 = Mul(constant_mul1_sub, input3) + mul0 = Mul(constant_mul0_x, input4) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = RealDiv(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt1 = Sqrt(real_div1) + real_div0 = Add(add0, input5) + add4 = Add(sqrt1, constant_add2_y) + sqrt0 = Rsqrt(add2) + mul4 = Mul(constant_mul4_x, input6) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + add5 = Add(real_div4, mul4) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, add5) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched_real_div1(input0, input1, input2, input3, input4, input5, input6, constant_mul0_x, + constant_mul1_sub, + constant_mul2_x, constant_mul3_sub1, constant_mul4_x, constant_add2_y): + mul1 = Mul(constant_mul1_sub, input3) + mul0 = Mul(constant_mul0_x, input4) + add0 = Add(mul0, mul1) + mul2 = Mul(constant_mul2_x, input1) + mul3 = Mul(constant_mul3_sub1, input0) + add1 = Add(mul2, mul3) + real_div1 = Add(add1, input2) + add2 = Add(real_div1, constant_add2_y) + sqrt1 = Sqrt(real_div1) + real_div0 = RealDiv(add0, input5) + add4 = Add(sqrt1, constant_add2_y) + sqrt0 = Rsqrt(add2) + mul4 = Mul(constant_mul4_x, input6) + real_div4 = RealDiv(real_div0, add4) + real_div2 = Mul(real_div0, sqrt0) + add5 = Add(real_div4, mul4) + add3 = Add(real_div2, mul4) + outputs = make_tuple(add3, add0, add1, add5) + output = tuple_getitem(outputs, 0) + return output + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_with_decay_v1_rule.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_with_decay_v1_rule.py new file mode 100644 index 00000000000..b433cd44b2c --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_mv_with_decay_v1_rule.py @@ -0,0 +1,150 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +add = P.TensorAdd() +mul = P.Mul() +real_div = P.RealDiv() +rsqrt = P.Rsqrt() +sqrt = P.Sqrt() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +LambNextMVWithDecayV1 = Primitive('LambNextMVWithDecayV1') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_lamb_next_mv_with_decay_v1_rule(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, input5, input6, mul0_x, mul1_sub, mul2_x, mul3_sub1, mul4_x, + add2_y): + mul3 = mul(mul3_sub1, input0) + mul2 = mul(mul2_x, input1) + add1 = add(mul2, mul3) + real_div1 = real_div(add1, input2) + add2 = add(real_div1, add2_y) + sqrt1 = sqrt(real_div1) + mul0 = mul(mul0_x, input4) + mul1 = mul(mul1_sub, input3) + sqrt0 = rsqrt(add2) + add4 = add(sqrt1, add2_y) + add0 = add(mul0, mul1) + real_div0 = real_div(add0, input5) + real_div2 = mul(real_div0, sqrt0) + real_div4 = real_div(real_div0, add4) + mul4 = mul(mul4_x, input6) + add3 = add(real_div2, mul4) + add5 = add(real_div4, mul4) + res = make_tuple(add3, add0, add1, add5) + return res + + @fns + def after(input0, input1, input2, input3, input4, input5, input6, mul0_x, mul1_sub, mul2_x, mul3_sub1, mul4_x, + add2_y): + fusion = LambNextMVWithDecayV1(input0, input1, input2, input3, input4, input5, input6, mul0_x, mul1_sub, mul2_x, + mul3_sub1, mul4_x, add2_y) + item0 = tuple_getitem(fusion, 0) + item1 = tuple_getitem(fusion, 1) + item2 = tuple_getitem(fusion, 2) + item3 = tuple_getitem(fusion, 3) + res = make_tuple(item0, item1, item2, item3) + return make_tuple(res) + + @fns + def no_match1(input0, input1, input2, input3, input4, input5, input6, mul0_x, mul1_sub, mul2_x, mul3_sub1, mul4_x, + add2_y): + mul3 = mul(mul3_sub1, input0) + mul2 = mul(mul2_x, input1) + add1 = add(mul2, mul3) + real_div1 = real_div(add1, input2) + add2 = add(real_div1, add2_y) + sqrt1 = sqrt(real_div1) + mul0 = mul(mul0_x, input4) + mul1 = mul(mul1_sub, input3) + sqrt0 = rsqrt(add2) + add4 = add(sqrt1, add2_y) + add0 = add(mul0, mul1) + real_div0 = real_div(add0, input5) + real_div2 = mul(real_div0, sqrt0) + real_div4 = real_div(real_div0, add4) + mul4 = mul(mul4_x, input6) + add3 = add(real_div2, mul4) + # diff mul from original add + add5 = mul(real_div4, mul4) + res = make_tuple(add3, add0, add1, add5) + return res + + @fns + def no_match2(input0, input1, input2, input3, input4, input5, input6, mul0_x, mul1_sub, mul2_x, mul3_sub1, mul4_x, + add2_y): + mul3 = mul(mul3_sub1, input0) + mul2 = mul(mul2_x, input1) + add1 = add(mul2, mul3) + real_div1 = real_div(add1, input2) + add2 = add(real_div1, add2_y) + sqrt1 = sqrt(real_div1) + mul0 = mul(mul0_x, input4) + mul1 = mul(mul1_sub, input3) + sqrt0 = rsqrt(add2) + # diff mul from original add + add4 = mul(sqrt1, add2_y) + add0 = add(mul0, mul1) + real_div0 = real_div(add0, input5) + real_div2 = mul(real_div0, sqrt0) + real_div4 = real_div(real_div0, add4) + mul4 = mul(mul4_x, input6) + add3 = add(real_div2, mul4) + add5 = add(real_div4, mul4) + res = make_tuple(add3, add0, add1, add5) + return res + + @fns + def no_match3(input0, input1, input2, input3, input4, input5, input6, mul0_x, mul1_sub, mul2_x, mul3_sub1, mul4_x, + add2_y): + mul3 = mul(mul3_sub1, input0) + mul2 = mul(mul2_x, input1) + add1 = add(mul2, mul3) + real_div1 = real_div(add1, input2) + add2 = add(real_div1, add2_y) + # diff sqrt(add2) link from sqrt(real_div1) + sqrt1 = sqrt(add2) + mul0 = mul(mul0_x, input4) + mul1 = mul(mul1_sub, input3) + sqrt0 = rsqrt(add2) + add4 = add(sqrt1, add2_y) + add0 = add(mul0, mul1) + # diff add from original real_div + real_div0 = add(add0, input5) + real_div2 = mul(real_div0, sqrt0) + real_div4 = real_div(real_div0, add4) + mul4 = mul(mul4_x, input6) + add3 = add(real_div2, mul4) + add5 = add(real_div4, mul4) + res = make_tuple(add3, add0, add1, add5) + return res + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_right_rule_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_right_rule_test.py new file mode 100644 index 00000000000..78803cdb264 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_next_right_rule_test.py @@ -0,0 +1,74 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Add = P.TensorAdd() +Mul = P.Mul() +Sqrt = P.Sqrt() +Square = P.Square() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +LambNextRight = Primitive('LambNextRight') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_lamb_next_right_rule(tag): + fns = FnDict() + + @fns + def before(input0, input1, mul2_x, mul3_x, true_div1_recip, add2_y): + square0 = Square(input0) + mul2 = Mul(mul2_x, input1) + mul3 = Mul(mul3_x, square0) + add1 = Add(mul2, mul3) + real_div1 = Mul(add1, true_div1_recip) + sqrt0 = Sqrt(real_div1) + add2 = Add(sqrt0, add2_y) + outputs = make_tuple(add1, add2) + output = tuple_getitem(outputs, 0) + return output + + @fns + def before_unmatched(input0, input1, mul2_x, mul3_x, true_div1_recip, add2_y): + square0 = Square(input0) + mul2 = Mul(mul2_x, input1) + mul3 = Add(mul3_x, square0) + add1 = Add(mul2, mul3) + real_div1 = Mul(add1, true_div1_recip) + sqrt0 = Sqrt(real_div1) + add2 = Add(sqrt0, add2_y) + outputs = make_tuple(add1, add2) + output = tuple_getitem(outputs, 0) + return output + + @fns + def after(input0, input1, mul2_x, mul3_x, true_div1_recip, add2_y): + lamb_next_right = LambNextRight(input0, input1, mul2_x, mul3_x, true_div1_recip, add2_y) + outputs = make_tuple(tuple_getitem(lamb_next_right, 0), tuple_getitem(lamb_next_right, 1)) + output = tuple_getitem(outputs, 0) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_update_with_lr_rule_fusion.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_update_with_lr_rule_fusion.py new file mode 100644 index 00000000000..0e8f231e2d7 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_update_with_lr_rule_fusion.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +select = P.Select() +maximum = P.Maximum() +minimum = P.Minimum() +greater = P.Greater() +real_div = P.RealDiv() +mul = P.Mul() +sub = P.Sub() +lamb_update_with_lr = Primitive('LambUpdateWithLR') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_lamb_update_with_lr_rule_fusion(tag): + """ test_lamb_update_with_lr_rule_fusion """ + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, input5, constant_greater01_max0, constant_select01, + constant_minimum0): + real_div_res = real_div(input1, input2) + greater0_res = greater(input0, constant_greater01_max0) + greater1_res = greater(input1, constant_greater01_max0) + res = select(greater0_res, real_div_res, constant_select01) + res = select(greater1_res, res, constant_select01) + res = minimum(res, constant_minimum0) + res = maximum(res, constant_greater01_max0) + res = mul(res, input3) + res = mul(res, input4) + res = sub(input5, res) + return res + + @fns + def after(input0, input1, input2, input3, input4, input5, constant_greater01_max0, constant_select01, + constant_minimum0): + res = lamb_update_with_lr(input0, input1, input2, input3, input4, input5, constant_greater01_max0, + constant_select01, constant_minimum0) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_update_with_lr_v2_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_update_with_lr_v2_test.py new file mode 100644 index 00000000000..e1bd3df4f0e --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/lamb_update_with_lr_v2_test.py @@ -0,0 +1,59 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Sub = P.Sub() +Mul = P.Mul() +RealDiv = P.RealDiv() +Select = P.Select() +Greater = P.Greater() +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +LambUpdateWithLrV2 = Primitive('LambUpdateWithLrV2') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_lamb_update_with_lr_v2(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4, greater_y, select_e): + greater0 = Greater(input0, greater_y) + greater1 = Greater(input1, greater_y) + real_div0 = RealDiv(input0, input1) + select0 = Select(greater1, real_div0, select_e) + select1 = Select(greater0, select0, select_e) + mul0 = Mul(select1, input2) + mul1 = Mul(mul0, input3) + sub0 = Sub(input4, mul1) + return sub0 + + @fns + def after(input0, input1, input2, input3, input4, greater_y, select_e): + res = LambUpdateWithLrV2(input0, input1, input2, input3, input4, greater_y, select_e) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/layer_norm_beta_gamma_backprop_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/layer_norm_beta_gamma_backprop_fusion_test.py new file mode 100644 index 00000000000..68def800f3e --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/layer_norm_beta_gamma_backprop_fusion_test.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Add = P.TensorAdd() +Cast = P.Cast() +LayerNormBetaGammaBackprop = Primitive('LayerNormBetaGammaBackprop') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_layer_norm_beta_gamma_backprop_fusion(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3): + layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + output0 = Cast(tuple_getitem(layer, 0)) + output1 = Cast(tuple_getitem(layer, 1)) + add = Add(output0, output1) + return add + + @fns + def before_unmatched_inputs_size(input0, input1, input2): + layer = LayerNormBetaGammaBackprop(input0, input1, input2) + output0 = Cast(tuple_getitem(layer, 0)) + output1 = Cast(tuple_getitem(layer, 1)) + add = Add(output0, output1) + return add + + @fns + def before_unmatched_outputs_size(input0, input1, input2, input3): + layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + output0 = Cast(layer) + return output0 + + @fns + def after(input0, input1, input2, input3): + layer = LayerNormBetaGammaBackprop(input0, input1, input2, input3) + output0 = tuple_getitem(layer, 0) + output1 = tuple_getitem(layer, 1) + add = Add(output0, output1) + return add + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/layer_norm_grad_split.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/layer_norm_grad_split.py new file mode 100644 index 00000000000..930c76a8f5e --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/layer_norm_grad_split.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +layer_norm_grad = G.LayerNormGrad() +layer_norm_x_backprop = Primitive('LayerNormXBackprop') +layer_norm_beta_gamma_backprop = Primitive('LayerNormBetaGammaBackprop') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_layer_norm_grad_split(tag): + """ test_layer_norm_grad_split """ + fns = FnDict() + + @fns + def before(i0, i1, i2, i3, i4): + layer_norm_grad_output = layer_norm_grad(i0, i1, i2, i3, i4) + item0 = tuple_getitem(layer_norm_grad_output, 0) + item1 = tuple_getitem(layer_norm_grad_output, 1) + item2 = tuple_getitem(layer_norm_grad_output, 2) + res = make_tuple(item0, item1, item2) + return res + + @fns + def after(i0, i1, i2, i3, i4): + layer_norm_x_output = layer_norm_x_backprop(i0, i1, i2, i3, i4) + layer_norm_beta_output = layer_norm_beta_gamma_backprop(i0, i1, i2, i3) + beta_item0 = tuple_getitem(layer_norm_beta_output, 0) + beta_item1 = tuple_getitem(layer_norm_beta_output, 1) + mt = make_tuple(layer_norm_x_output, beta_item0, beta_item1) + item0 = tuple_getitem(mt, 0) + item1 = tuple_getitem(mt, 1) + item2 = tuple_getitem(mt, 2) + res = make_tuple(item0, item1, item2) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/merge_cast_to_op.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/merge_cast_to_op.py new file mode 100644 index 00000000000..6ecfa5da77f --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/merge_cast_to_op.py @@ -0,0 +1,66 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +four2five = Primitive('Four2Five') +five2four = Primitive('Five2Four') +cast = Primitive('Cast') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_merge_cast_to_next_op(tag): + fns = FnDict() + + @fns + def before(x): + x = cast(x) + output = four2five(x) + return output + + @fns + def after(x): + output = four2five(x) + return make_tuple(output) + + return fns[tag] + + +def test_merge_cast_to_prior_op(tag): + fns = FnDict() + + @fns + def before(x): + x = five2four(x) + output = cast(x) + return output + + @fns + def after(x): + output = five2four(x) + return make_tuple(output) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/mixed_precision_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/mixed_precision_test.py new file mode 100644 index 00000000000..8cbad52db1d --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/mixed_precision_test.py @@ -0,0 +1,189 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +tuple_getitem = Primitive('tuple_getitem') +depend = Primitive('depend') +addn = P.AddN() +add = P.TensorAdd() +sub = P.Sub() +mul = P.Mul() +max_pool = P.MaxPoolWithArgmax(pad_mode="same", window=3, stride=2) +make_tuple = Primitive('make_tuple') +cast = Primitive('Cast') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_insert_cast_op_for_single_output(tag): + fns = FnDict() + + @fns + def before(x, y): + res = add(x, y) + return res + + @fns + def after(x, y): + new_x = cast(x) + new_y = cast(y) + res = add(new_x, new_y) + res = cast(res) + return make_tuple(res) + + return fns[tag] + + +def test_insert_cast_op_for_single_output_new(tag): + fns = FnDict() + + @fns + def before(x, y): + res = add(x, y) + output = make_tuple(res) + return output + + @fns + def after(x, y): + new_x = cast(x) + new_y = cast(y) + res = add(new_x, new_y) + output = cast(res) + new_output = make_tuple(output) + return make_tuple(new_output) + + return fns[tag] + + +def test_eliminate_cast_op(tag): + fns = FnDict() + + @fns + def before(x, y): + sum = addn((x, y)) + sum_depend = depend(sum, addn((x, y))) + diff = sub(x, y) + res = mul(sum_depend, diff) + return res + + @fns + def after1(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + sum = addn(new_x_sum, new_y_sum) + sum_cast = cast(sum) + new_x_depend = cast(x) + new_y_depend = cast(y) + sum_depend = addn(new_x_depend, new_y_depend) + sum_depend_cast = cast(sum_depend) + depend_between_cast = depend(sum_cast, sum_depend_cast) + depend_cast = cast(depend_between_cast) + new_x_diff = cast(x) + new_y_diff = cast(y) + diff = sub(new_x_diff, new_y_diff) + diff_cast1 = cast(diff) + diff_cast2 = cast(diff_cast1) + res = mul(depend_cast, diff_cast2) + output = cast(res) + new_output = make_tuple(output) + return new_output + + @fns + def after2(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + sum = addn(new_x_sum, new_y_sum) + new_x_depend = cast(x) + new_y_depend = cast(y) + sum_depend = addn(new_x_depend, new_y_depend) + sum_depend_cast = cast(sum_depend) + depend_between_cast = depend(sum, sum_depend_cast) + new_x_diff = cast(x) + new_y_diff = cast(y) + diff = sub(new_x_diff, new_y_diff) + res = mul(depend_between_cast, diff) + output = cast(res) + new_output = make_tuple(output) + return new_output + + return fns[tag] + + +def test_insert_cast_op_for_multiple_output(tag): + fns = FnDict() + + @fns + def before(x): + output = max_pool(x) + return tuple_getitem(output, 0) + + @fns + def after(x): + new_x = cast(x) + output = max_pool(new_x) + cast0 = cast(tuple_getitem(output, 0)) + cast1 = cast(tuple_getitem(output, 1)) + res = make_tuple(cast0, cast1) + return make_tuple(tuple_getitem(res, 0)) + + return fns[tag] + + +def test_eliminate_cast_new(tag): + fns = FnDict() + + @fns + def before(x, y): + sum = add(x, y) + res = sub(sum, y) + output = make_tuple(res) + return output + + @fns + def after1(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + new_y_sum2 = cast(y) + sum = add(new_x_sum, new_y_sum) + sum_5to4 = cast(sum) + sum_4to5 = cast(sum_5to4) + res = sub(sum_4to5, new_y_sum2) + output = cast(res) + new_output = make_tuple(output) + return new_output + + @fns + def after2(x, y): + new_x_sum = cast(x) + new_y_sum = cast(y) + new_y_diff = cast(y) + sum = add(new_x_sum, new_y_sum) + res = sub(sum, new_y_diff) + output = cast(res) + new_output = make_tuple(output) + return new_output + + return fns[tag] + diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/momentum_lossscale_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/momentum_lossscale_fusion_test.py new file mode 100644 index 00000000000..c882029e909 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/momentum_lossscale_fusion_test.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor + +Mul = P.Mul() +ApplyMomentum = P.ApplyMomentum() +FusedMulApplyMomentum = Primitive('FusedMulApplyMomentum') +tuple_getitem = Primitive('tuple_getitem') +make_tuple = Primitive('make_tuple') +constant = Tensor(1.0, mstype.float32) + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_momentum_lossscale_fusion(tag): + fns = FnDict() + + @fns + def before(input0, input1, input2, input3, input4): + mul = Mul(constant, input3) + fused_mul_apply_momentum = ApplyMomentum(input0, input1, input2, mul, input4) + return fused_mul_apply_momentum + + @fns + def after(input0, input1, input2, input3, input4): + return make_tuple(FusedMulApplyMomentum(input0, input1, input2, input3, input4, constant)) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/mul_add_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/mul_add_fusion_test.py new file mode 100644 index 00000000000..83a62233bfb --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/mul_add_fusion_test.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +add = P.TensorAdd() +mul = P.Mul() +fused_mul_add = Primitive('FusedMulAdd') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_mul_add_fusion(tag): + fns = FnDict() + + @fns + def before(x, y, z): + res = mul(x, y) + res = add(res, z) + return res + + @fns + def after(x, y, z): + res = fused_mul_add(x, y, z) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/mul_addn_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/mul_addn_fusion_test.py new file mode 100644 index 00000000000..e5b0a15387c --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/mul_addn_fusion_test.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor + +addn = P.AddN() +mul = P.Mul() +fused_mul_addn = Primitive('FusedMulAddN') +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +scalar = Tensor(1.0, mstype.float32) + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_mul_addn_fusion(tag): + fns = FnDict() + + @fns + def before(a, b): + res = mul(scalar, a) + res = addn((b, res)) + return res + + @fns + def after(a, b): + res = fused_mul_addn(a, b, scalar) + return make_tuple(res) + + @fns + def unmatch(a, b, c): + # none of mul's input is scalar + res = mul(a, b) + res = addn((c, res)) + return res + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/optimize_dependence_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/optimize_dependence_test.py new file mode 100644 index 00000000000..45c419d25d1 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/optimize_dependence_test.py @@ -0,0 +1,49 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +depend = Primitive('depend') +TransData = Primitive('TransData') +add = P.TensorAdd() + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_optimize_dependence(tag): + fns = FnDict() + + @fns + def before(x, y, z): + new_z = TransData(z) + depend_intput = depend(y, new_z) + sum = add(x, depend_intput) + return sum + + @fns + def after(x, y, z): + depend_intput = depend(y, z) + sum = add(x, depend_intput) + return sum + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/reshape_transpose_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/reshape_transpose_fusion_test.py new file mode 100644 index 00000000000..0afd547da0b --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/reshape_transpose_fusion_test.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Transpose = P.Transpose() +Reshape = P.Reshape() +ConfusionTransposeD = Primitive('ConfusionTransposeD') +make_tuple = Primitive('make_tuple') + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_reshape_transpose_fusion(tag): + fns = FnDict() + + @fns + def before(input0): + reshape = Reshape(input0, (2, 4, 8, 16)) + transpose = Transpose(reshape, (1, 0, 2, 3)) + return transpose + + @fns + def after(input0): + confusion = ConfusionTransposeD(input0) + res = make_tuple(confusion) + return res + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/square_sum_fusion.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/square_sum_fusion.py new file mode 100644 index 00000000000..26c82fb8f71 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/square_sum_fusion.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +make_tuple = Primitive('make_tuple') +tuple_getitem = Primitive('tuple_getitem') +square = P.Square() +reduce_sum = P.ReduceSum() +square_sumv1 = Primitive('SquareSumV1') +square_sumv2 = Primitive('SquareSumV2') +axis = -1 + + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_square_sum_fusion(tag): + """ test_square_sum_fusion """ + fns = FnDict() + + @fns + def before1(x): + square_output = square(x) + res = reduce_sum(square_output, axis) + return res + + @fns + def after1(x): + res = square_sumv1(x) + return make_tuple(res) + + @fns + def before2(x): + square_output = square(x) + sum_output = reduce_sum(square_output, axis) + res = make_tuple(sum_output, square_output) + return res + + @fns + def after2(x): + output = square_sumv2(x) + item0 = tuple_getitem(output, 0) + item1 = tuple_getitem(output, 1) + res = make_tuple(item0, item1) + return make_tuple(res) + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/topk_split_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/topk_split_test.py new file mode 100644 index 00000000000..4cdbfa084e7 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/topk_split_test.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from mindspore.ops import Primitive +from mindspore.ops import operations as P + +TopK = P.TopK() +tuple_getitem = Primitive('tuple_getitem') + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_topk_split(tag): + fns = FnDict() + + @fns + def before(input): + topk = TopK(input) + output = tuple_getitem(topk, 0) + return output + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/transdata_split_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/transdata_split_test.py new file mode 100644 index 00000000000..8cd18d1ac30 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/transdata_split_test.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +tuple_getitem = Primitive('tuple_getitem') +add = P.TensorAdd() +max_pool = P.MaxPoolWithArgmax(pad_mode="same", window=3, stride=2) +make_tuple = Primitive('make_tuple') +four2five = Primitive('Four2Five') +five2four = Primitive('Five2Four') +transdata = Primitive("TransData") +transpose = Primitive("Transpose") +Transpose = P.Transpose() + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_transdata_split_fraz_nchw(tag): + fns = FnDict() + + @fns + def before(input): + res = Transpose(input, (1, 0, 2, 3)) + return res + + @fns + def after(input): + res = transpose(input) + output = transdata(res) + output = transpose(output) + res = make_tuple(output) + return res + + return fns[tag] +def test_transdata_split_nchw_fraz(tag): + fns = FnDict() + + @fns + def before(input): + res = Transpose(input, (1, 0, 2, 3)) + return res + + @fns + def after(input): + res = transpose(input) + output = transdata(res) + output = transpose(output) + res = make_tuple(output) + return res + + return fns[tag] \ No newline at end of file diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/transpose_reshape_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/transpose_reshape_fusion_test.py new file mode 100644 index 00000000000..01e253fb8b1 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/transpose_reshape_fusion_test.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +Transpose = P.Transpose() +Reshape = P.Reshape() +ConfusionTransposeD = Primitive('ConfusionTransposeD') +make_tuple = Primitive('make_tuple') + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_transpose_reshape_fusion(tag): + fns = FnDict() + + @fns + def before(input): + transpose = Transpose(input, (1, 0, 2, 3)) + reshape = Reshape(transpose, (2, 4, 8, 16)) + return reshape + + @fns + def after(input): + confusion = ConfusionTransposeD(input) + res = make_tuple(confusion) + return res + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pre_activate/transpose_transdata_fusion_test.py b/tests/ut/cpp/python_input/gtest_input/pre_activate/transpose_transdata_fusion_test.py new file mode 100644 index 00000000000..ea3def743db --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pre_activate/transpose_transdata_fusion_test.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive + +tuple_getitem = Primitive('tuple_getitem') +add = P.TensorAdd() +max_pool = P.MaxPoolWithArgmax(pad_mode="same", window=3, stride=2) +make_tuple = Primitive('make_tuple') +transdata = Primitive("TransData") +Transpose = P.Transpose() + +class FnDict: + def __init__(self): + self.fnDict = {} + + def __call__(self, fn): + self.fnDict[fn.__name__] = fn + + def __getitem__(self, name): + return self.fnDict[name] + + +def test_transpose_transdata_fusion(tag): + fns = FnDict() + + @fns + def before(input): + res = Transpose(input, (1, 0, 2, 3)) + return res + + @fns + def after(input): + output = transdata(input) + res = make_tuple(output) + return res + + return fns[tag] diff --git a/tests/ut/cpp/python_input/gtest_input/pynative/__init__.py b/tests/ut/cpp/python_input/gtest_input/pynative/__init__.py new file mode 100644 index 00000000000..e033431395f --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pynative/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" __init__ """ +from .ops_test import * diff --git a/tests/ut/cpp/python_input/gtest_input/pynative/ops_test.py b/tests/ut/cpp/python_input/gtest_input/pynative/ops_test.py new file mode 100644 index 00000000000..46c6fdd1cf7 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/pynative/ops_test.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" ops_test """ +import numpy as np +from mindspore.ops import operations as P +from mindspore.ops.vm_impl_registry import vm_impl_registry as vm_impl_getters +from mindspore.common.tensor import Tensor + +def im2col(img, filter_h, filter_w, stride=1, pad=0, dilation=1): + """Rearranges an image to row vector""" + batch_num, channel, height, width = img.shape + out_h = (height + 2*pad - filter_h - (filter_h - 1) * (dilation - 1))//stride + 1 + out_w = (width + 2*pad - filter_w - (filter_w - 1) * (dilation - 1))//stride + 1 + + img = np.pad(img, [(0, 0), (0, 0), (pad, pad), (pad, pad)], 'constant') + col = np.zeros((batch_num, channel, filter_h, filter_w, out_h, out_w)).astype(img.dtype) + + for y in range(filter_h): + y_max = y + stride*out_h + for x in range(filter_w): + x_max = x + stride*out_w + col[:, :, y, x, :, :] = img[:, :, y:y_max:stride, x:x_max:stride] + + col = col.transpose(0, 4, 5, 1, 2, 3).reshape(batch_num*out_h*out_w, -1) + return col + +# pylint: disable=unused-argument +def conv2d(x, weight, bias=None, stride=1, pad=0, + dilation=1, groups=1, padding_mode='zeros'): + """Convolution 2D""" + batch_num, _, x_h, x_w = x.shape + filter_num, _, filter_h, filter_w = weight.shape + out_h = 1 + int((x_h + 2 * pad - filter_h - (filter_h - 1) * (dilation - 1)) / stride) + out_w = 1 + int((x_w + 2 * pad - filter_w - (filter_w - 1) * (dilation - 1)) / stride) + col = im2col(x, filter_h, filter_w, stride, pad, dilation) + col_w = np.reshape(weight, (filter_num, -1)).T + out = np.dot(col, col_w) + out = out.reshape(batch_num, out_h, out_w, -1).transpose(0, 3, 1, 2) + if bias is not None: + out += bias + return out + + +@vm_impl_getters.register(P.Conv2D) +def vm_impl_conv2d(self): + """Generate vm_impl function for Conv2D""" + def vm_impl(x, w): + x = x.asnumpy() + weight = w.asnumpy() + bias = None + out = conv2d(x, weight, bias, self.stride, self.pad, self.dilation) + return Tensor(out) + return vm_impl + + +conv2d_prim = P.Conv2D(64, (3, 3), pad_mode='pad', pad=1, stride=2) diff --git a/tests/ut/cpp/python_input/gtest_input/session/session_test.py b/tests/ut/cpp/python_input/gtest_input/session/session_test.py new file mode 100644 index 00000000000..ed074fc8d61 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/session/session_test.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.ops import operations as P +from mindspore.ops import Primitive +import mindspore as ms + + +addn = P.AddN() +add = P.TensorAdd() +reshape = P.Reshape() +cast = P.Cast() +tuple_getitem = Primitive('tuple_getitem') +max_pool = P.MaxPoolWithArgmax(pad_mode="same", window=3, stride=2) + +def test_addn_cast(x, y, z): + sum = addn((x, y)) + res = cast(sum, ms.float16) + return res + +def test_addn_with_max_pool(x, y): + sum = addn((x, y)) + output = max_pool(sum) + res = tuple_getitem(output, 0) + return res + + +def test_shape_add(x1, x2, y1, y2, z1, z2): + sum1 = add(x1, x2) + sum2 = add(y1, y2) + sum3 = add(z1, z2) + reshape_sum1 = reshape(sum1, (2, 2, 3, 1)) + reshape_sum2 = reshape(sum2, (2, 2, 3, 1)) + reshape_sum3 = reshape(sum3, (2, 2, 3, 1)) + sum = add(reshape_sum1, reshape_sum2) + sum = add(sum, reshape_sum3) + return sum diff --git a/tests/ut/cpp/python_input/gtest_input/transform/__init__.py b/tests/ut/cpp/python_input/gtest_input/transform/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/transform/multi_relu_case.py b/tests/ut/cpp/python_input/gtest_input/transform/multi_relu_case.py new file mode 100644 index 00000000000..d6c6ad4ab96 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/transform/multi_relu_case.py @@ -0,0 +1,24 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" multi_relu_case """ +from mindspore.ops import Primitive +# Test user define ops +def get_test_ops_fn(): + return test_ops_f + +scalar_mul = Primitive('scalar_mul') +def test_ops_f(x, y): + z = scalar_mul(x, y) + return z diff --git a/tests/ut/cpp/python_input/gtest_input/utils/__init__.py b/tests/ut/cpp/python_input/gtest_input/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/cpp/python_input/gtest_input/utils/graph_utils_test.py b/tests/ut/cpp/python_input/gtest_input/utils/graph_utils_test.py new file mode 100644 index 00000000000..e4249999ebd --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/utils/graph_utils_test.py @@ -0,0 +1,23 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" graph_utils_test """ +def test_graph_utils_isomorphic_1(a, b): + return a + b + +def test_graph_utils_isomorphic_2(x, y): + return x + y + +def test_graph_utils_isomorphic_3(x, y): + return x * y diff --git a/tests/ut/cpp/python_input/gtest_input/vm/__init__.py b/tests/ut/cpp/python_input/gtest_input/vm/__init__.py new file mode 100644 index 00000000000..416abfcc424 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/vm/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : __init__.py +@Author: +@Date : 2019-01-23 16:36 +@Desc : +""" + +from .vm_test import * diff --git a/tests/ut/cpp/python_input/gtest_input/vm/vm_test.py b/tests/ut/cpp/python_input/gtest_input/vm/vm_test.py new file mode 100644 index 00000000000..bdd3c900d65 --- /dev/null +++ b/tests/ut/cpp/python_input/gtest_input/vm/vm_test.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" vm_test """ +from mindspore.ops import Primitive + +scala_add = Primitive('scalar_add') +scala_mul = Primitive('scalar_mul') +def scalar_add(x, y): + """Implement `scalar_add`.""" + return scala_add(x, y) + +def scalar_mul(x, y): + """Implement `scalar_mul`.""" + return scala_mul(x, y) + +def test_if(x, y): + if x > y: + return x + return y diff --git a/tests/ut/cpp/runtest.sh b/tests/ut/cpp/runtest.sh new file mode 100755 index 00000000000..a50e1d49335 --- /dev/null +++ b/tests/ut/cpp/runtest.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +BASEPATH=$(cd $(dirname $0); pwd) +PROJECT_PATH=${BASEPATH}/../../.. +if [ $BUILD_PATH ];then + echo "BUILD_PATH = $BUILD_PATH" +else + BUILD_PATH=${PROJECT_PATH}/build + echo "BUILD_PATH = $BUILD_PATH" +fi +cd ${BUILD_PATH}/mindspore/tests/ut/cpp + + +export LD_LIBRARY_PATH=${BUILD_PATH}/mindspore/googletest/googlemock/gtest:${PROJECT_PATH}/mindspore:${PROJECT_PATH}/mindspore/lib:$LD_LIBRARY_PATH +export PYTHONPATH=${PROJECT_PATH}/tests/ut/cpp/python_input:$PYTHONPATH:${PROJECT_PATH} + +## prepare data for dataset & mindrecord +cp -fr $PROJECT_PATH/tests/ut/data ${PROJECT_PATH}/build/mindspore/tests/ut/cpp/ + +if [ $# -gt 0 ]; then + ./ut_tests --gtest_filter=$1 +else + ./ut_tests +fi +RET=$? +cd - + +exit ${RET} diff --git a/tests/ut/cpp/session/anf_runtime_algorithm_test.cc b/tests/ut/cpp/session/anf_runtime_algorithm_test.cc new file mode 100644 index 00000000000..2af2a7413b8 --- /dev/null +++ b/tests/ut/cpp/session/anf_runtime_algorithm_test.cc @@ -0,0 +1,796 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "operator/ops.h" +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" +#include "mindspore/ccsrc/device/kernel_info.h" +#include "mindspore/ccsrc/device/ascend/ascend_device_address.h" +#include "utils/utils.h" + +namespace mindspore { +namespace session { +using device::KernelInfo; +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; +using AscendDeviceAddress = device::ascend::AscendDeviceAddress; + +class AnfRuntimeAlgorithmTest : public UT::Common { + public: + AnfRuntimeAlgorithmTest() = default; + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(AnfRuntimeAlgorithmTest, VisitKernel) { + auto kernel_graph = std::make_shared(); + KernelWithIndex kernel_with_index; + // test nullptr as input + EXPECT_THROW(AnfAlgo::VisitKernel(nullptr, 0), std::runtime_error); + // test value node as input + ValueNodePtr value_node = NewValueNode(prim::kPrimTensorAdd); + kernel_with_index = AnfAlgo::VisitKernel(value_node, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), value_node.get()); + EXPECT_EQ(kernel_with_index.second, 0); + // test parameter node as input + ParameterPtr parameter_node = kernel_graph->add_parameter(); + kernel_with_index = AnfAlgo::VisitKernel(parameter_node, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), parameter_node.get()); + EXPECT_EQ(kernel_with_index.second, 0); + // test cnode as input + std::vector inputs{value_node}; + auto add = kernel_graph->NewCNode(inputs); + kernel_with_index = AnfAlgo::VisitKernel(add, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), add.get()); + EXPECT_EQ(kernel_with_index.second, 0); + // test maketuple node as input + std::vector add_inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add_second = kernel_graph->NewCNode(add_inputs); + std::vector make_tuple_inputs{NewValueNode(prim::kPrimMakeTuple), add, add_second}; + auto make_tuple = kernel_graph->NewCNode(make_tuple_inputs); + MS_EXCEPTION_IF_NULL(make_tuple); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract}; + make_tuple->set_abstract(std::make_shared(args_spec_list)); + kernel_with_index = AnfAlgo::VisitKernel(make_tuple, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), add.get()); + EXPECT_EQ(kernel_with_index.second, 0); + kernel_with_index = AnfAlgo::VisitKernel(make_tuple, 1); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), add_second.get()); + EXPECT_EQ(kernel_with_index.second, 0); + // test tuple get item node as input + std::vector tuple_get_item_inputs{NewValueNode(prim::kPrimTupleGetItem), make_tuple, NewValueNode(1)}; + auto tuple_get_item = kernel_graph->NewCNode(tuple_get_item_inputs); + kernel_with_index = AnfAlgo::VisitKernel(tuple_get_item, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), add_second.get()); + EXPECT_EQ(kernel_with_index.second, 0); + // test depend or control depend node as input + std::vector depend_inputs{NewValueNode(prim::kPrimDepend), add, add_second}; + auto depend = kernel_graph->NewCNode(depend_inputs); + kernel_with_index = AnfAlgo::VisitKernel(depend, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), add.get()); + EXPECT_EQ(kernel_with_index.second, 0); + std::vector control_depend_inputs{NewValueNode(prim::kPrimControlDepend), add_second, add}; + auto control_depend = kernel_graph->NewCNode(control_depend_inputs); + kernel_with_index = AnfAlgo::VisitKernel(control_depend, 0); + EXPECT_NE(kernel_with_index.first->cast(), nullptr); + EXPECT_EQ((kernel_with_index.first->cast()).get(), add_second.get()); + EXPECT_EQ(kernel_with_index.second, 0); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetCNodePrimitive) { + auto kernel_graph = std::make_shared(); + // test cnode node + PrimitivePtr add_primitive = prim::kPrimTensorAdd; + std::vector inputs{NewValueNode(add_primitive)}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_NE(AnfAlgo::GetCNodePrimitive(add), nullptr); + EXPECT_EQ(AnfAlgo::GetCNodePrimitive(add).get(), add_primitive.get()); + EXPECT_THROW(AnfAlgo::GetCNodePrimitive(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetCNodeName) { + auto kernel_graph = std::make_shared(); + // test cnode node + std::vector inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetCNodeName(add), prim::kPrimTensorAdd->name()); + EXPECT_THROW(AnfAlgo::GetCNodeName(nullptr), std::runtime_error); + // test parameter + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::GetCNodeName(parameter_node), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetNodeDebugString) { + auto kernel_graph = std::make_shared(); + // test cnode node + std::vector inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetNodeDebugString(add), add->DebugString()); + EXPECT_THROW(AnfAlgo::GetNodeDebugString(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetNodeAttr) { + auto kernel_graph = std::make_shared(); + // test cnode node + std::vector inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add = kernel_graph->NewCNode(inputs); + AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value"), add); + auto primitive = AnfAlgo::GetCNodePrimitive(add); + MS_EXCEPTION_IF_NULL(primitive); + EXPECT_EQ(GetValue(primitive->GetAttr("test_set_attr")), "test_value"); + // test parameter node + auto parameter = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value"), parameter), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, CopyNodeAttr) { + auto kernel_graph = std::make_shared(); + // test cnode node + std::vector add_inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add = kernel_graph->NewCNode(add_inputs); + AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value"), add); + + std::vector mul_inputs{NewValueNode(prim::kPrimMul)}; + auto mul = kernel_graph->NewCNode(mul_inputs); + AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value_v2"), mul); + AnfAlgo::CopyNodeAttr("test_set_attr", mul, add); + auto primitive = AnfAlgo::GetCNodePrimitive(add); + MS_EXCEPTION_IF_NULL(primitive); + EXPECT_EQ(GetValue(primitive->GetAttr("test_set_attr")), "test_value_v2"); + // test parameter node + auto parameter = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::CopyNodeAttr("test_set_attr", parameter, add), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttr("test_set_attr", mul, parameter), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttr("test_set_attr", parameter, parameter), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttr("test_set_attr", nullptr, add), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttr("test_set_attr", mul, nullptr), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttr("test_set_attr", nullptr, nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, CopyNodeAttrs) { + auto kernel_graph = std::make_shared(); + // test cnode node + std::vector add_inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add = kernel_graph->NewCNode(add_inputs); + AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value"), add); + + std::vector mul_inputs{NewValueNode(prim::kPrimMul)}; + auto mul = kernel_graph->NewCNode(mul_inputs); + AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value_v2"), mul); + AnfAlgo::CopyNodeAttrs(mul, add); + auto primitive = AnfAlgo::GetCNodePrimitive(add); + MS_EXCEPTION_IF_NULL(primitive); + EXPECT_EQ(GetValue(primitive->GetAttr("test_set_attr")), "test_value_v2"); + // test parameter node + auto parameter = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::CopyNodeAttrs(parameter, add), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttrs(mul, parameter), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttrs(parameter, parameter), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttrs(nullptr, add), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttrs(mul, nullptr), std::runtime_error); + EXPECT_THROW(AnfAlgo::CopyNodeAttrs(nullptr, nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, EraseNodeAttr) { + auto kernel_graph = std::make_shared(); + // test cnode node + std::vector add_inputs{NewValueNode(prim::kPrimTensorAdd)}; + auto add = kernel_graph->NewCNode(add_inputs); + AnfAlgo::SetNodeAttr("test_set_attr", MakeValue("test_value"), add); + AnfAlgo::SetNodeAttr("test_set_attr_v2", MakeValue("test_value_v2"), add); + AnfAlgo::EraseNodeAttr("test_set_attr_v2", add); + EXPECT_THROW(AnfAlgo::GetNodeAttr(add, "test_set_attr_v2"), std::runtime_error); + EXPECT_THROW(AnfAlgo::EraseNodeAttr("test_set_attr_v2", nullptr), std::runtime_error); + // test parameter node + auto parameter = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::EraseNodeAttr("test_set_attr_v2", parameter), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetInputTensorNum) { + auto kernel_graph = std::make_shared(); + // test cnode node + auto parameter_one = kernel_graph->add_parameter(); + auto parameter_two = kernel_graph->add_parameter(); + std::vector add_inputs{NewValueNode(prim::kPrimTensorAdd), parameter_one, parameter_two}; + auto add = kernel_graph->NewCNode(add_inputs); + EXPECT_EQ(AnfAlgo::GetInputTensorNum(add), 2); + EXPECT_THROW(AnfAlgo::GetInputTensorNum(nullptr), std::runtime_error); + // test parameter node + EXPECT_THROW(AnfAlgo::GetInputTensorNum(parameter_one), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputTensorNum) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + // test fused batch norm as input + inputs.push_back(NewValueNode(prim::kPrimFusedBatchNorm)); + auto bn = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(bn); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, x_abstract, x_abstract, x_abstract}; + bn->set_abstract(std::make_shared(args_spec_list)); + EXPECT_EQ(AnfAlgo::GetOutputTensorNum(bn), 5); + EXPECT_THROW(AnfAlgo::GetOutputTensorNum(nullptr), std::runtime_error); + // test add as input + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_abstract(std::make_shared()); + EXPECT_EQ(AnfAlgo::GetOutputTensorNum(add), 0); + add->set_abstract(x_abstract); + EXPECT_EQ(AnfAlgo::GetOutputTensorNum(add), 1); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputFormat) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetOutputsDeviceType({kFloat32->type_id(), kFloat16->type_id()}); + builder.SetOutputsFormat({kOpFormat_NCHW, kOpFormat_NC1HWC0}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetOutputFormat(add, 0), kOpFormat_NCHW); + EXPECT_EQ(AnfAlgo::GetOutputFormat(add, 1), kOpFormat_NC1HWC0); + EXPECT_THROW(AnfAlgo::GetOutputFormat(add, 2), std::runtime_error); + EXPECT_THROW(AnfAlgo::GetOutputFormat(nullptr, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetInputFormat) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat16->type_id()}); + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NC1HWC0}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetInputFormat(add, 0), kOpFormat_NCHW); + EXPECT_EQ(AnfAlgo::GetInputFormat(add, 1), kOpFormat_NC1HWC0); + EXPECT_THROW(AnfAlgo::GetInputFormat(add, 2), std::runtime_error); + EXPECT_THROW(AnfAlgo::GetInputFormat(nullptr, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetPrevNodeOutputFormat) { + auto kernel_graph = std::make_shared(); + std::vector pre_node_inputs; + pre_node_inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto pre_add = kernel_graph->NewCNode(pre_node_inputs); + MS_EXCEPTION_IF_NULL(pre_add); + pre_add->set_kernel_info(std::make_shared()); + auto d_kernel_info = pre_add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + std::vector inputs{NewValueNode(prim::kPrimTensorAdd), pre_add}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetPrevNodeOutputFormat(add, 0), kOpFormat_NCHW); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputFormat(nullptr, 0), std::runtime_error); + // test parameter node as input + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputFormat(parameter_node, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputInferShape) { + auto kernel_graph = std::make_shared(); + std::vector shp{2, 32, 224, 224}; + auto none_abstract = std::make_shared(); + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, none_abstract, x_abstract}; + auto tuple_abstract = std::make_shared(args_spec_list); + // test value node as input + auto value_node = NewValueNode(prim::kPrimTensorAdd); + MS_EXCEPTION_IF_NULL(value_node); + value_node->set_abstract(x_abstract); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(value_node, 0)[1], 32); + EXPECT_THROW(AnfAlgo::GetOutputInferShape(nullptr, 0), std::runtime_error); + // test parameter node as input + auto parameter_node = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(parameter_node); + parameter_node->set_abstract(x_abstract); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(parameter_node, 0)[2], 224); + // test cnode as input + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_abstract(std::make_shared()); + EXPECT_TRUE(AnfAlgo::GetOutputInferShape(add, 0).empty()); + add->set_abstract(x_abstract); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 0)[3], 224); + EXPECT_THROW(AnfAlgo::GetOutputInferShape(add, 1), std::runtime_error); + add->set_abstract(tuple_abstract); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 0)[0], 2); + EXPECT_TRUE(AnfAlgo::GetOutputInferShape(add, 1).empty()); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 2)[1], 32); + EXPECT_THROW(AnfAlgo::GetOutputInferShape(add, 3), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetPrevNodeOutputInferShape) { + auto kernel_graph = std::make_shared(); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + // test parameter node as input + auto parameter_node = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(parameter_node); + parameter_node->set_abstract(x_abstract); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputInferShape(parameter_node, 0), std::runtime_error); + // test cnode as input + std::vector inputs{NewValueNode(prim::kPrimTensorAdd), parameter_node}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetPrevNodeOutputInferShape(add, 0)[1], 32); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputInferShape(add, 1), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputDeviceShape) { + auto kernel_graph = std::make_shared(); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, x_abstract}; + args_spec_list.emplace_back(std::make_shared(kFloat32, std::vector{1, 2, 3, 4})); + auto tuple_abstract = std::make_shared(args_spec_list); + // test cnode as input + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_abstract(tuple_abstract); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetOutputsFormat({kOpFormat_NCHW, kOpFormat_NCHW, kOpFormat_NHWC, kOpFormat_FRAC_NZ}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetOutputDeviceShape(add, 0)[2], 224); + EXPECT_EQ(AnfAlgo::GetOutputDeviceShape(add, 1)[0], 2); + std::vector expect_shape{2, 224, 224, 32}; + EXPECT_EQ(AnfAlgo::GetOutputDeviceShape(add, 2), expect_shape); + std::vector nz_expect_shape{1, 2, 1, 1, 16, 16}; + EXPECT_EQ(AnfAlgo::GetOutputDeviceShape(add, 3), nz_expect_shape); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetInputDeviceShape) { + auto kernel_graph = std::make_shared(); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + auto parameter_one = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(parameter_one); + parameter_one->set_abstract(x_abstract); + auto parameter_two = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(parameter_two); + parameter_two->set_abstract(x_abstract); + auto parameter_third = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(parameter_third); + parameter_third->set_abstract(x_abstract); + // test cnode as input + std::vector inputs{NewValueNode(prim::kPrimTensorAdd), parameter_one, parameter_two, parameter_third}; + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NCHW, kOpFormat_NHWC}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetInputDeviceShape(add, 0)[2], 224); + EXPECT_EQ(AnfAlgo::GetInputDeviceShape(add, 1)[1], 32); + std::vector expect_shape{2, 224, 224, 32}; + EXPECT_EQ(AnfAlgo::GetInputDeviceShape(add, 2), expect_shape); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputInferShape(nullptr, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputInferDataTypeTest) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimFusedBatchNorm)); + auto bn = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(bn); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + AbstractBasePtrList args_spec_list{x_abstract, x_abstract, x_abstract, x_abstract, x_abstract}; + bn->set_abstract(std::make_shared(args_spec_list)); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(bn, 0), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(bn, 4), kFloat32->type_id()); + EXPECT_THROW(AnfAlgo::GetOutputInferDataType(bn, 5), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetPrevNodeOutputInferDataType) { + auto kernel_graph = std::make_shared(); + std::vector pre_node_inputs; + pre_node_inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto pre_add = kernel_graph->NewCNode(pre_node_inputs); + MS_EXCEPTION_IF_NULL(pre_add); + std::vector shp{2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shp); + pre_add->set_abstract(x_abstract); + std::vector inputs{NewValueNode(prim::kPrimTensorAdd), pre_add}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetPrevNodeOutputInferDataType(add, 0), kFloat32->type_id()); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputInferDataType(add, 1), std::runtime_error); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputInferDataType(nullptr, 0), std::runtime_error); + // test parameter as input + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputInferDataType(parameter_node, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputDeviceDataTypeTest) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetOutputsDeviceType({kFloat32->type_id()}); + builder.SetOutputsFormat({kOpFormat_NCHW}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetOutputDeviceDataType(add, 0), kFloat32->type_id()); + EXPECT_THROW(AnfAlgo::GetOutputDeviceDataType(add, 1), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetInputDeviceDataTypeTest) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetInputsDeviceType({kFloat32->type_id(), kFloat16->type_id()}); + builder.SetInputsFormat({kOpFormat_NCHW, kOpFormat_NC1HWC0}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetInputDeviceDataType(add, 0), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetInputDeviceDataType(add, 1), kFloat16->type_id()); + EXPECT_THROW(AnfAlgo::GetInputDeviceDataType(add, 2), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetPrevNodeOutputDeviceDataType) { + auto kernel_graph = std::make_shared(); + std::vector pre_add_inputs; + pre_add_inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto pre_add = kernel_graph->NewCNode(pre_add_inputs); + MS_EXCEPTION_IF_NULL(pre_add); + pre_add->set_kernel_info(std::make_shared()); + auto d_kernel_info = pre_add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetOutputsDeviceType({kFloat32->type_id()}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + std::vector inputs{NewValueNode(prim::kPrimTensorAdd), pre_add}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetPrevNodeOutputDeviceDataType(add, 0), kFloat32->type_id()); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputDeviceDataType(add, 1), std::runtime_error); + // test parameter as input + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputDeviceDataType(parameter_node, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetOutputAddr) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + int *addr = nullptr; + auto device_address = std::make_shared(addr, 1); + d_kernel_info->SetOutputAddr(device_address, 0); + EXPECT_EQ(AnfAlgo::GetOutputAddr(add, 0), device_address.get()); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetPrevNodeOutputAddr) { + auto kernel_graph = std::make_shared(); + std::vector pre_add_inputs; + pre_add_inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto pre_add = kernel_graph->NewCNode(pre_add_inputs); + MS_EXCEPTION_IF_NULL(pre_add); + pre_add->set_kernel_info(std::make_shared()); + auto d_kernel_info = pre_add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + int *addr = nullptr; + auto device_address = std::make_shared(addr, 1); + d_kernel_info->SetOutputAddr(device_address, 0); + std::vector inputs{NewValueNode(prim::kPrimTensorAdd), pre_add}; + auto add = kernel_graph->NewCNode(inputs); + EXPECT_EQ(AnfAlgo::GetPrevNodeOutputAddr(add, 0), device_address.get()); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputAddr(add, 1), std::runtime_error); + // test parameter as input + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_THROW(AnfAlgo::GetPrevNodeOutputAddr(parameter_node, 0), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetOutputAddr) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + int *addr = nullptr; + auto device_address = std::make_shared(addr, 1); + EXPECT_THROW(AnfAlgo::SetOutputAddr(device_address, 0, nullptr), std::runtime_error); + AnfAlgo::SetOutputAddr(device_address, 0, add.get()); + EXPECT_EQ(AnfAlgo::GetOutputAddr(add, 0), device_address.get()); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetWorkspaceAddr) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + int *addr = nullptr; + auto device_address = std::make_shared(addr, 1); + d_kernel_info->SetWorkspaceAddr(device_address, 0); + EXPECT_EQ(AnfAlgo::GetWorkspaceAddr(add, 0), device_address.get()); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetWorkspaceAddr) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + int *addr = nullptr; + auto device_address = std::make_shared(addr, 1); + EXPECT_THROW(AnfAlgo::SetWorkspaceAddr(device_address, 0, nullptr), std::runtime_error); + AnfAlgo::SetWorkspaceAddr(device_address, 0, add.get()); + EXPECT_EQ(AnfAlgo::GetWorkspaceAddr(add, 0), device_address.get()); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetOutputInferTypeAndShape) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + // set none abstract + std::vector none_types = {}; + std::vector> none_shapes = {}; + EXPECT_THROW(AnfAlgo::SetOutputInferTypeAndShape(none_types, none_shapes, nullptr), std::runtime_error); + EXPECT_THROW(AnfAlgo::SetOutputInferTypeAndShape(none_types, none_shapes, add.get()), std::runtime_error); + // set single input + std::vector single_types = {kFloat32->type_id()}; + std::vector> single_shapes = {{2, 32, 224, 224}}; + EXPECT_THROW(AnfAlgo::SetOutputInferTypeAndShape(none_types, single_shapes, add.get()), std::runtime_error); + AnfAlgo::SetOutputInferTypeAndShape(single_types, single_shapes, add.get()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(add, 0), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 0).size(), 4); + // set mutiple input + std::vector mutiple_types = {kFloat16->type_id(), kFloat32->type_id(), kFloat64->type_id()}; + std::vector> mutiple_shapes = {{2, 32, 224, 224}, {2, 32, 224, 224}, {2, 32, 224, 224}}; + AnfAlgo::SetOutputInferTypeAndShape(mutiple_types, mutiple_shapes, add.get()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(add, 0), kFloat16->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(add, 1), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(add, 2), kFloat64->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 0).size(), 4); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 1).size(), 4); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(add, 2).size(), 4); +} + +TEST_F(AnfRuntimeAlgorithmTest, CopyAbstract) { + auto kernel_graph = std::make_shared(); + std::vector first_inputs; + first_inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto first_add = kernel_graph->NewCNode(first_inputs); + // set single input + std::vector single_types = {kFloat32->type_id()}; + std::vector> single_shapes = {{2, 32, 224, 224}}; + AnfAlgo::SetOutputInferTypeAndShape(single_types, single_shapes, first_add.get()); + // set mutiple input + std::vector second_inputs; + second_inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto second_add = kernel_graph->NewCNode(second_inputs); + std::vector mutiple_types = {kFloat16->type_id(), kFloat32->type_id(), kFloat64->type_id()}; + std::vector> mutiple_shapes = {{2, 32, 224, 224}, {2, 32, 224, 224}, {2, 32, 224, 224}}; + AnfAlgo::SetOutputInferTypeAndShape(mutiple_types, mutiple_shapes, second_add.get()); + AnfAlgo::CopyAbstract(second_add, first_add.get()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(first_add, 0), kFloat16->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(first_add, 1), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(first_add, 2), kFloat64->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(first_add, 0).size(), 4); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(first_add, 1).size(), 4); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(first_add, 2).size(), 4); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetKernelType) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetKernelType(AUTO_DIFF_KERNEL); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetKernelType(add), AUTO_DIFF_KERNEL); + EXPECT_THROW(AnfAlgo::GetKernelType(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetProcessor) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetProcessor(kernel::AICORE); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetProcessor(add), kernel::AICORE); + EXPECT_THROW(AnfAlgo::GetProcessor(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetFusionType) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + KernelBuildInfoBuilder builder; + builder.SetFusionType(kernel::CONVLUTION); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + EXPECT_EQ(AnfAlgo::GetFusionType(add), kernel::CONVLUTION); + EXPECT_THROW(AnfAlgo::GetFusionType(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetSelectKernelBuildInfo) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + std::shared_ptr builder = std::make_shared(); + builder->SetFusionType(kernel::CONVLUTION); + AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), add.get()); + EXPECT_THROW(AnfAlgo::SetSelectKernelBuildInfo(builder->Build(), nullptr), std::runtime_error); + EXPECT_EQ(AnfAlgo::GetFusionType(add), kernel::CONVLUTION); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetKernelMod) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + d_kernel_info->set_kernel_mod(nullptr); + EXPECT_EQ(AnfAlgo::GetKernelMod(add), nullptr); + EXPECT_THROW(AnfAlgo::GetKernelMod(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetKernelMod) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + AnfAlgo::SetKernelMod(nullptr, add.get()); + EXPECT_THROW(AnfAlgo::SetKernelMod(nullptr, nullptr), std::runtime_error); + EXPECT_EQ(AnfAlgo::GetKernelMod(add), nullptr); +} + +TEST_F(AnfRuntimeAlgorithmTest, IsRealKernel) { + auto kernel_graph = std::make_shared(); + // test value node as input + auto value_node = NewValueNode(prim::kPrimTensorAdd); + EXPECT_TRUE(AnfAlgo::IsRealKernel(value_node)); + EXPECT_THROW(AnfAlgo::IsRealKernel(nullptr), std::runtime_error); + // test parameter as input + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_TRUE(AnfAlgo::IsRealKernel(parameter_node)); + // test add as input + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + EXPECT_TRUE(AnfAlgo::IsRealKernel(add)); + // test Depend as input + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimDepend)); + auto depend_node = kernel_graph->NewCNode(inputs); + EXPECT_FALSE(AnfAlgo::IsRealKernel(depend_node)); +} + +TEST_F(AnfRuntimeAlgorithmTest, IsRealCNodeKernel) { + auto kernel_graph = std::make_shared(); + // test value node as input + auto value_node = NewValueNode(prim::kPrimTensorAdd); + EXPECT_FALSE(AnfAlgo::IsRealCNodeKernel(value_node)); + EXPECT_THROW(AnfAlgo::IsRealCNodeKernel(nullptr), std::runtime_error); + // test parameter as input + auto parameter_node = kernel_graph->add_parameter(); + EXPECT_FALSE(AnfAlgo::IsRealCNodeKernel(parameter_node)); + // test add as input + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + EXPECT_TRUE(AnfAlgo::IsRealCNodeKernel(add)); + // test ImageSummary as input + inputs.clear(); + inputs.push_back(NewValueNode(prim::kPrimDepend)); + auto depend = kernel_graph->NewCNode(inputs); + EXPECT_FALSE(AnfAlgo::IsRealCNodeKernel(depend)); +} + +TEST_F(AnfRuntimeAlgorithmTest, IsParameterWeight) { + auto kernel_graph = std::make_shared(); + py::object obj; + auto parameter_node = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(parameter_node); + parameter_node->set_default_param(obj); + EXPECT_TRUE(AnfAlgo::IsParameterWeight(parameter_node)); + EXPECT_THROW(AnfAlgo::IsParameterWeight(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, GetStreamId) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_kernel_info(std::make_shared()); + auto d_kernel_info = add->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + d_kernel_info->set_stream_id(0); + EXPECT_EQ(AnfAlgo::GetStreamId(add), 0); + EXPECT_THROW(AnfAlgo::GetStreamId(nullptr), std::runtime_error); +} + +TEST_F(AnfRuntimeAlgorithmTest, SetStreamId) { + auto kernel_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim::kPrimTensorAdd)); + auto add = kernel_graph->NewCNode(inputs); + AnfAlgo::SetStreamId(0, add.get()); + EXPECT_THROW(AnfAlgo::SetStreamId(0, nullptr), std::runtime_error); + EXPECT_EQ(AnfAlgo::GetStreamId(add), 0); +} + +} // namespace session +} // namespace mindspore diff --git a/tests/ut/cpp/session/kernel_graph_test.cc b/tests/ut/cpp/session/kernel_graph_test.cc new file mode 100644 index 00000000000..55e1b1b28e8 --- /dev/null +++ b/tests/ut/cpp/session/kernel_graph_test.cc @@ -0,0 +1,192 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "operator/ops.h" +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" +#include "mindspore/ccsrc/device/kernel_info.h" +#include "utils/utils.h" + +namespace mindspore { +namespace session { +using device::KernelInfo; +using KernelBuildInfoBuilder = kernel::KernelBuildInfo::KernelBuildInfoBuilder; + +class KernelGraphTest : public UT::Common { + public: + KernelGraphTest() = default; + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(KernelGraphTest, NewValueNode) { + auto kernel_graph = std::make_shared(); + auto add_value = NewValueNode(MakeValue(0)); + MS_EXCEPTION_IF_NULL(add_value); + std::vector shape = {1}; + auto x_abstract = std::make_shared(kFloat32, shape); + add_value->set_abstract(x_abstract); + add_value->set_kernel_info(std::make_shared()); + auto mutable_kernel_info = add_value->kernel_info(); + MS_EXCEPTION_IF_NULL(mutable_kernel_info); + std::shared_ptr builder = std::make_shared(); + builder->SetOutputsFormat({kOpFormat_FRAC_Z}); + builder->SetOutputsDeviceType({kFloat32->type_id()}); + mutable_kernel_info->set_select_kernel_build_info(builder->Build()); + auto new_value = kernel_graph->NewValueNode(add_value); + EXPECT_NE(new_value, nullptr); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(new_value, 0)[0], 1); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(new_value, 0), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputFormat(new_value, 0), kOpFormat_DEFAULT); + EXPECT_EQ(AnfAlgo::GetOutputDeviceDataType(new_value, 0), kTypeUnknown); +} + +TEST_F(KernelGraphTest, NewParameter) { + auto anf_graph = std::make_shared(); + auto kernel_graph = std::make_shared(); + // test nullptr as input + auto new_paramter = kernel_graph->NewParameter(nullptr); + EXPECT_NE(new_paramter, nullptr); + EXPECT_TRUE(new_paramter->isa()); + EXPECT_EQ(AnfAlgo::GetOutputFormat(new_paramter, 0), kOpFormat_DEFAULT); + EXPECT_EQ(AnfAlgo::GetOutputDeviceDataType(new_paramter, 0), kMetaTypeNone); + // test non-weight parameter node as input + std::vector shape = {2, 32, 224, 224}; + auto x_abstract = std::make_shared(kFloat32, shape); + auto non_weight_parameter = anf_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(non_weight_parameter); + non_weight_parameter->set_abstract(x_abstract); + auto new_non_weight_parameter = kernel_graph->NewParameter(non_weight_parameter); + EXPECT_NE(new_non_weight_parameter, nullptr); + new_non_weight_parameter->set_name("non_weight_parameter"); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(new_non_weight_parameter, 0)[1], 32); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(new_non_weight_parameter, 0), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputFormat(new_non_weight_parameter, 0), kOpFormat_DEFAULT); + EXPECT_EQ(AnfAlgo::GetOutputDeviceDataType(new_non_weight_parameter, 0), kFloat32->type_id()); + EXPECT_EQ(new_non_weight_parameter->name(), "non_weight_parameter"); + // test weight parameter node as input + auto weight_parameter_node = anf_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(weight_parameter_node); + py::object obj; + weight_parameter_node->set_default_param(obj); + weight_parameter_node->set_abstract(x_abstract); + auto new_weight_parameter_node = kernel_graph->NewParameter(weight_parameter_node); + EXPECT_NE(new_weight_parameter_node, nullptr); + EXPECT_TRUE(new_weight_parameter_node->has_default()); + EXPECT_EQ(AnfAlgo::GetOutputInferShape(new_weight_parameter_node, 0)[2], 224); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(new_weight_parameter_node, 0), kFloat32->type_id()); + EXPECT_EQ(AnfAlgo::GetOutputFormat(new_weight_parameter_node, 0), kOpFormat_DEFAULT); + EXPECT_EQ(AnfAlgo::GetOutputDeviceDataType(new_weight_parameter_node, 0), kTypeUnknown); +} + +TEST_F(KernelGraphTest, NewCNode) { + auto kernel_graph = std::make_shared(); + auto add_value = NewValueNode(prim::kPrimTensorAdd); + std::vector inputs = {add_value}; + auto new_cnode = kernel_graph->NewCNode(inputs); + EXPECT_NE(new_cnode, nullptr); + EXPECT_EQ(AnfAlgo::GetCNodeName(new_cnode), prim::kPrimTensorAdd->name()); + EXPECT_TRUE(AnfAlgo::GetOutputInferShape(new_cnode, 0).empty()); + EXPECT_EQ(AnfAlgo::GetOutputInferDataType(new_cnode, 0), kMetaTypeNone); +} + +TEST_F(KernelGraphTest, MutableInputs) { + auto kernel_graph = std::make_shared(); + auto x_parameter = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(x_parameter); + x_parameter->set_name("x_parameter"); + auto y_parameter = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(y_parameter); + y_parameter->set_name("y_parameter"); + std::vector inputs = {x_parameter, y_parameter}; + auto mutable_inputs = kernel_graph->MutableInputs(); + MS_EXCEPTION_IF_NULL(mutable_inputs); + *mutable_inputs = inputs; + auto first_input = kernel_graph->inputs()[0]; + MS_EXCEPTION_IF_NULL(first_input); + auto first_parameter = first_input->cast(); + MS_EXCEPTION_IF_NULL(first_parameter); + EXPECT_EQ(first_parameter->name(), "x_parameter"); + auto second_input = kernel_graph->inputs()[1]; + MS_EXCEPTION_IF_NULL(second_input); + auto second_parameter = second_input->cast(); + MS_EXCEPTION_IF_NULL(second_parameter); + EXPECT_EQ(second_parameter->name(), "y_parameter"); +} + +TEST_F(KernelGraphTest, SetExecOrderByDefault) { + /* + * define kernel graph: + * x ----- y + * add ----- z + * mul + * return + */ + auto kernel_graph = std::make_shared(); + std::vector shape = {2, 32, 224, 224}; + auto abstract = std::make_shared(kFloat32, shape); + + auto x_parameter = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(x_parameter); + x_parameter->set_name("x_parameter"); + x_parameter->set_abstract(abstract); + auto y_parameter = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(y_parameter); + y_parameter->set_name("y_parameter"); + y_parameter->set_abstract(abstract); + std::vector add_inputs = {NewValueNode(prim::kPrimTensorAdd), x_parameter, y_parameter}; + auto add = kernel_graph->NewCNode(add_inputs); + MS_EXCEPTION_IF_NULL(add); + add->set_abstract(abstract); + + auto z_parameter = kernel_graph->add_parameter(); + MS_EXCEPTION_IF_NULL(z_parameter); + z_parameter->set_name("z_parameter"); + z_parameter->set_abstract(abstract); + std::vector mul_inputs = {NewValueNode(prim::kPrimMul), add, z_parameter}; + auto mul = kernel_graph->NewCNode(mul_inputs); + MS_EXCEPTION_IF_NULL(mul); + mul->set_abstract(abstract); + + std::vector make_tuple_inputs = {NewValueNode(prim::kPrimMakeTuple), mul}; + auto make_tuple = kernel_graph->NewCNode(make_tuple_inputs); + kernel_graph->set_output(make_tuple); + // test outputs() function + auto outputs = kernel_graph->outputs(); + EXPECT_EQ(outputs.size(), 1); + EXPECT_EQ(AnfAlgo::GetCNodeName(outputs[0]), prim::kPrimMul->name()); + // test SetExecOrderByDefault() function + kernel_graph->SetExecOrderByDefault(); + auto execution_order = kernel_graph->execution_order(); + EXPECT_EQ(execution_order.size(), 2); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[0]), prim::kPrimTensorAdd->name()); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[1]), prim::kPrimMul->name()); + // test set_execution_order() function + kernel_graph->set_execution_order({add}); + execution_order = kernel_graph->execution_order(); + EXPECT_EQ(execution_order.size(), 1); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[0]), prim::kPrimTensorAdd->name()); +} + +TEST_F(KernelGraphTest, SetGraphId) { + auto kernel_graph = std::make_shared(); + kernel_graph->set_graph_id(1); + EXPECT_EQ(kernel_graph->graph_id(), 1); +} + +} // namespace session +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/session/session_basic_test.cc b/tests/ut/cpp/session/session_basic_test.cc new file mode 100644 index 00000000000..1a7ca680655 --- /dev/null +++ b/tests/ut/cpp/session/session_basic_test.cc @@ -0,0 +1,98 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/common_test.h" +#include "operator/ops.h" +#include "session/ascend_session.h" +#include "session/kernel_graph.h" +#include "session/anf_runtime_algorithm.h" +#include "utils/utils.h" + +namespace mindspore { +namespace session { + +class SessionBasicTest : public UT::Common { + public: + SessionBasicTest() = default; + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(SessionBasicTest, ConstructKernelGraph) { + /* + * define kernel graph: + * x ----- y + * add ----- z + * mul + * return + */ + auto anf_graph = std::make_shared(); + std::vector shape = {2, 32, 224, 224}; + auto abstract = std::make_shared(kFloat32, shape); + EXPECT_NE(abstract, nullptr); + + auto original_x_parameter = anf_graph->add_parameter(); + EXPECT_NE(original_x_parameter, nullptr); + original_x_parameter->set_name("original_x_parameter"); + original_x_parameter->set_abstract(abstract); + auto original_y_parameter = anf_graph->add_parameter(); + EXPECT_NE(original_y_parameter, nullptr); + original_y_parameter->set_name("original_y_parameter"); + original_y_parameter->set_abstract(abstract); + std::vector add_inputs = {NewValueNode(prim::kPrimTensorAdd), original_x_parameter, original_y_parameter}; + auto original_add = anf_graph->NewCNode(add_inputs); + EXPECT_NE(original_add, nullptr); + original_add->set_abstract(abstract); + + auto original_z_parameter = anf_graph->add_parameter(); + EXPECT_NE(original_z_parameter, nullptr); + original_z_parameter->set_name("original_z_parameter"); + original_z_parameter->set_abstract(abstract); + std::vector mul_inputs = {NewValueNode(prim::kPrimMul), original_add, original_z_parameter}; + auto original_mul = anf_graph->NewCNode(mul_inputs); + EXPECT_NE(original_mul, nullptr); + original_mul->set_abstract(abstract); + + std::vector lst = {original_add, original_mul}; + std::vector outputs = {original_mul}; + session::SessionPtr sess = std::make_shared(); + sess->Init(0); + auto kernel_graph = sess->ConstructKernelGraph(lst, outputs); + EXPECT_NE(kernel_graph, nullptr); + + auto inputs = kernel_graph->inputs(); + EXPECT_EQ(inputs.size(), 3); + auto first_input = inputs[0]->cast(); + EXPECT_NE(first_input, nullptr); + EXPECT_EQ(first_input->name(), "original_x_parameter"); + auto second_input = inputs[1]->cast(); + EXPECT_NE(second_input, nullptr); + EXPECT_EQ(second_input->name(), "original_y_parameter"); + auto third_input = inputs[2]->cast(); + EXPECT_NE(third_input, nullptr); + EXPECT_EQ(third_input->name(), "original_z_parameter"); + kernel_graph->SetExecOrderByDefault(); + auto execution_order = kernel_graph->execution_order(); + EXPECT_EQ(execution_order.size(), 2); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[0]), prim::kPrimTensorAdd->name()); + EXPECT_EQ(AnfAlgo::GetCNodeName(execution_order[1]), prim::kPrimMul->name()); + auto new_outputs = kernel_graph->outputs(); + EXPECT_EQ(new_outputs.size(), 1); + EXPECT_EQ(AnfAlgo::GetCNodeName(new_outputs[0]), prim::kPrimMul->name()); +}; + +} // namespace session +} // namespace mindspore \ No newline at end of file diff --git a/tests/ut/cpp/stub/aicpu/aicpu_stub.cc b/tests/ut/cpp/stub/aicpu/aicpu_stub.cc new file mode 100644 index 00000000000..78ada6de18c --- /dev/null +++ b/tests/ut/cpp/stub/aicpu/aicpu_stub.cc @@ -0,0 +1,26 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/kernel.h" + +namespace mindspore { +namespace kernel { + +/* + * @brief build op and return a callable mod + */ +KernelModPtr AicpuOpBuild(const AnfNodePtr &anf_node) { return nullptr; } +} // namespace kernel +} // namespace mindspore diff --git a/tests/ut/cpp/stub/anf_ir/dump_proto_stub.cc b/tests/ut/cpp/stub/anf_ir/dump_proto_stub.cc new file mode 100644 index 00000000000..871fffc1c7e --- /dev/null +++ b/tests/ut/cpp/stub/anf_ir/dump_proto_stub.cc @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/anf_ir_utils.h" + +namespace mindspore { + +std::string GetFuncGraphProtoString(const FuncGraphPtr& func_graph) { return ""; } + +std::string GetOnnxProtoString(const FuncGraphPtr& func_graph) { return ""; } + +} // namespace mindspore diff --git a/tests/ut/cpp/stub/ge/ge_mock.cc b/tests/ut/cpp/stub/ge/ge_mock.cc new file mode 100644 index 00000000000..e206f8bc930 --- /dev/null +++ b/tests/ut/cpp/stub/ge/ge_mock.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GE_MOCK_H +#define GE_MOCK_H +#include +#include +#include +#include +#include +#include "graph/tensor.h" +#include "graph/operator_reg.h" +#include "graph/operator.h" +#include "graph/utils/tensor_utils.h" +#include "graph/utils/tensor_adapter.h" + +#include "external/ge/ge_api.h" + +namespace ge { + +Session::Session(const std::map& options) {} +Session::~Session() {} + +Status Session::RunGraph(uint32_t id, const std::vector& inputs, std::vector& outputs) { + // for test!!! just copy inputs to outputs: + for (auto it = inputs.begin(); it != inputs.end(); it++) { + outputs.emplace_back(*it); + } + return ge::GRAPH_SUCCESS; +} + +Status Session::AddGraph(uint32_t id, const Graph& graph) { return ge::GRAPH_SUCCESS; } + +Status GEInitialize(const std::map& options) { return ge::GRAPH_SUCCESS; } + +Status GEFinalize() { return ge::GRAPH_SUCCESS; } + +Status Graph::SaveToFile(const string& file_name) const { return ge::GRAPH_SUCCESS; } + +} // namespace ge +#endif diff --git a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc new file mode 100644 index 00000000000..b77b83c7fe7 --- /dev/null +++ b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "framework/ge_runtime/model_runner.h" +#include "device/ascend/tasksink/runtime_utils.h" + +namespace ge { +namespace model_runner { +ModelRunner &ModelRunner::Instance() { + static ModelRunner runner; + return runner; +} + +bool ModelRunner::LoadDavinciModel(uint32_t device_id, uint64_t session_id, uint32_t model_id, + std::shared_ptr ascend_model, + std::shared_ptr listener) { + return true; +} + +bool ModelRunner::UnloadModel(uint32_t model_id) { return true; } + +bool ModelRunner::RunModel(uint32_t model_id, const ge::InputData &input_data, ge::OutputData *output_data) { + return true; +} + +const std::vector &ModelRunner::GetTaskIdList(uint32_t model_id) const { + static std::vector task_id_list; + return task_id_list; +} +} // namespace model_runner +} // namespace ge + +namespace mindspore { +namespace device { +namespace ascend { +namespace tasksink { +bool RuntimeUtils::HcomBindModel(rtModel_t model, rtStream_t stream) { return true; } + +bool RuntimeUtils::HcomUnbindModel(rtModel_t model) { return true; } + +bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info, rtStream_t stream) { return true; } +} // namespace tasksink +} // namespace ascend +} // namespace device +} // namespace mindspore diff --git a/tests/ut/cpp/stub/hccl/hccl_stub.cc b/tests/ut/cpp/stub/hccl/hccl_stub.cc new file mode 100644 index 00000000000..00379ba6505 --- /dev/null +++ b/tests/ut/cpp/stub/hccl/hccl_stub.cc @@ -0,0 +1,121 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* runtime基础数据类型声明 */ + +/* HCCL基础数据类型声明 */ +#include "hccl/hcom.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* 集合通信域初始化 */ +hcclResult_t hcom_init(const char *rank_table, const char *identify) { return HCCL_SUCCESS; } + +/* 解析ranktable for python */ +hcclResult_t hcom_rank_info_init(const char *rank_table, const char *identify, u32 device_id) { return HCCL_SUCCESS; } + +/* 集合通信域销毁 */ +hcclResult_t hcom_destroy(void) { return HCCL_SUCCESS; } + +/* 绑定model */ +hcclResult_t hcom_bind_model(rtModel_t model, rtStream_t stream) { return HCCL_SUCCESS; } + +/* 绑解定model */ +hcclResult_t hcom_unbind_model(rtModel_t model) { return HCCL_SUCCESS; } + +/* allgather功能实现 */ +hcclResult_t hcom_all_gather(const char *tag, void *inputPtr, void *outputPtr, u64 inputCount, hcclDataType_t dataType, + const char *group, rtStream_t stream) { + return HCCL_SUCCESS; +} + +/* allreduce功能实现 */ +hcclResult_t hcom_all_reduce(const char *tag, void *inputPtr, void *outputPtr, u64 count, hcclDataType_t dataType, + hcclRedOp_t op, const char *group, rtStream_t stream) { + return HCCL_SUCCESS; +} + +/* broadcas功能实现 */ +hcclResult_t hcom_broadcast(const char *tag, void *ptr, u64 count, hcclDataType_t dataType, u32 root, const char *group, + rtStream_t stream) { + return HCCL_SUCCESS; +} +/* reduce_scatter功能实现 */ +hcclResult_t hcom_reduce_scatter(const char *tag, void *inputPtr, void *outputPtr, u64 count, hcclDataType_t dataType, + hcclRedOp_t op, const char *group, rtStream_t stream) { + return HCCL_SUCCESS; +} + +/* 获取group内的rank个数 */ +hcclResult_t hcom_get_rank_size(const char *group, u32 *rankSize) { return HCCL_SUCCESS; } + +/* python获取上云场景内的rank个数 */ +hcclResult_t hcom_python_get_rank_size(u32 *rankSize) { return HCCL_SUCCESS; } + +/* 获取本rank的id */ +hcclResult_t hcom_get_rank_id(const char *group, u32 *rankId) { return HCCL_SUCCESS; } + +/* 获取本rank的id */ +hcclResult_t hcom_python_get_rank_id(u32 *rankId) { return HCCL_SUCCESS; } + +/* 获取本rank的id */ +hcclResult_t hcom_get_world_rank_from_group_rank(const char *group, u32 groupRank, u32 *worldRank) { + return HCCL_SUCCESS; +} + +/* 获取通信域的rank个数 */ +hcclResult_t hcom_get_group_rank_from_world_rank(u32 worldRank, const char *group, u32 *groupRank) { + return HCCL_SUCCESS; +} + +/* 创建group */ +hcclResult_t hcom_create_group(const char *group, u32 rankNum, u32 *rankIds) { return HCCL_SUCCESS; } + +/* 销毁group */ +hcclResult_t hcom_destroy_group(const char *group) { return HCCL_SUCCESS; } + +/* 发送消息 */ +hcclResult_t hcom_send(const char *tag, void *inputPtr, u64 count, hcclDataType_t dataType, u32 destRank, u32 srTag, + const char *group, rtStream_t stream) { + return HCCL_SUCCESS; +} + +/* 接收消息 */ +hcclResult_t hcom_receive(const char *tag, void *outputPtr, u64 count, hcclDataType_t dataType, u32 srcRank, u32 srTag, + const char *group, rtStream_t stream) { + return HCCL_SUCCESS; +} + +/* 获取梯度参数切分方案 */ +hcclResult_t hcom_get_split_strategy(const char *group, const struct model_feature *feature, u32 maxSegmentNum, + u32 *segmentNum, u32 *segmentIdx) { + return HCCL_SUCCESS; +} + +/* 连通性检测 */ +hcclResult_t hcom_connectivity_detection(s32 *result) { return HCCL_SUCCESS; } + +hcclResult_t hcom_set_split_strategy_by_index(const char *group, u32 segmentNum, const u32 *IdxList) { + return HCCL_SUCCESS; +} +hcclResult_t hcom_set_split_strategy_by_size(const char *group, u32 segmentNum, const float *sizeList) { + return HCCL_SUCCESS; +} +#ifdef __cplusplus +} +#endif diff --git a/tests/ut/cpp/stub/kernel/kernel_fusion_stub.cc b/tests/ut/cpp/stub/kernel/kernel_fusion_stub.cc new file mode 100755 index 00000000000..ba642dfe189 --- /dev/null +++ b/tests/ut/cpp/stub/kernel/kernel_fusion_stub.cc @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/kernel_fusion.h" +#include "kernel/tbe/tbe_kernel_mod.h" +#include "common/utils.h" + +namespace mindspore { +namespace kernel { +std::map KernelFusion(const std::vector &fusion_scopes) { + std::map kernel_mod_ret; + for (const auto &fusion_scope_iter : fusion_scopes) { + kernel_mod_ret[fusion_scope_iter.scope_id] = std::make_shared(nullptr); + } + return kernel_mod_ret; +} +} // namespace kernel +} // namespace mindspore diff --git a/tests/ut/cpp/stub/parallel_strategy_checkpoint/parallel_strategy_checkpoint_stub.cc b/tests/ut/cpp/stub/parallel_strategy_checkpoint/parallel_strategy_checkpoint_stub.cc new file mode 100644 index 00000000000..73de5071cd6 --- /dev/null +++ b/tests/ut/cpp/stub/parallel_strategy_checkpoint/parallel_strategy_checkpoint_stub.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "parallel/strategy_checkpoint/parallel_strategy_checkpoint.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace parallel { +StrategyCheckpoint& StrategyCheckpoint::GetInstance() { + static StrategyCheckpoint instance = StrategyCheckpoint(); + return instance; +} + +bool StrategyCheckpoint::CheckPointExit() const { return false; } + +Status StrategyCheckpoint::RemoveCheckPoint() const { return SUCCESS; } + +Status StrategyCheckpoint::Load(StrategyMap* strategy_map) { return SUCCESS; } + +Status StrategyCheckpoint::Save(const StrategyMap& strategy_map) { return SUCCESS; } +} // namespace parallel +} // namespace mindspore diff --git a/tests/ut/cpp/stub/profiling/profiling_stub.cc b/tests/ut/cpp/stub/profiling/profiling_stub.cc new file mode 100644 index 00000000000..ac09c524f4e --- /dev/null +++ b/tests/ut/cpp/stub/profiling/profiling_stub.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "prof_mgr_core.h" +#include + +namespace Msprof { +namespace Engine { + +class EngineIntf; +/** + * @name : RegisterEngine + * @berif : API of libmsprof, register an engine with a name + * @param [in]: module: the name of plugin + engine: the plugin + * @return: PROFILING_SUCCESS 0 (success) + * PROFILING_FAILED -1 (failed) + */ +int RegisterEngine(const std::string& module, const EngineIntf* engine) { return 0; } + +} // namespace Engine +} // namespace Msprof + +/** + * @name : ProfMgrStartUP + * @berif : start Profiling task + * @param : ProfMgrCfg cfg : config of start_up profiling + * @return: NO_NULL (success) + * NULL (failed) + */ +void* ProfMgrStartUp(const ProfMgrCfg* cfg) { return const_cast(reinterpret_cast(cfg)); } + +/** + * @name : ProfMgrStop + * @berif : stop Profiling task + * @param : void * handle return by ProfMgrStartUP + * @return: PROFILING_SUCCESS 0 (success) + * PROFILING_FAILED -1 (failed) + */ +int ProfMgrStop(void* handle) { return 0; } diff --git a/tests/ut/cpp/stub/runtime/cuda.cc b/tests/ut/cpp/stub/runtime/cuda.cc new file mode 100644 index 00000000000..96d7671a09d --- /dev/null +++ b/tests/ut/cpp/stub/runtime/cuda.cc @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +CUresult cuModuleLoadData(CUmodule *module, const void *image) { return CUDA_SUCCESS; } + +CUresult cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name) { return CUDA_SUCCESS; } + +CUresult cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, + unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, + unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra) { + return CUDA_SUCCESS; +} + +CUresult cuModuleUnload(CUmodule hmod) { return CUDA_SUCCESS; } diff --git a/tests/ut/cpp/stub/runtime/cuda.h b/tests/ut/cpp/stub/runtime/cuda.h new file mode 100644 index 00000000000..2273e23fa30 --- /dev/null +++ b/tests/ut/cpp/stub/runtime/cuda.h @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_STUB_RUNTIME_INCLUDE_CUDA_H_ +#define TESTS_UT_STUB_RUNTIME_INCLUDE_CUDA_H_ + +typedef enum cudaError_enum { + CUDA_SUCCESS = 0, + CUDA_ERROR_INVALID_IMAGE = 1, + CUDA_ERROR_DEINITIALIZED = 2, +} CUresult; + +struct CUctx_st { + int arch; +}; +struct CUmod_st { + int arch; +}; +struct CUfunc_st { + int arch; +}; +struct CUstream_st { + int arch; +}; + +typedef struct CUctx_st *CUcontext; +typedef struct CUmod_st *CUmodule; +typedef struct CUfunc_st *CUfunction; +typedef struct CUstream_st *CUstream; + +CUresult cuModuleLoadData(CUmodule *module, const void *image); +CUresult cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name); +CUresult cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, + unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, + unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra); +CUresult cuModuleUnload(CUmodule hmod); + +#endif // TESTS_UT_STUB_RUNTIME_INCLUDE_CUDA_H_ diff --git a/tests/ut/cpp/stub/runtime/cuda_runtime_api.cc b/tests/ut/cpp/stub/runtime/cuda_runtime_api.cc new file mode 100644 index 00000000000..ea06f52ff64 --- /dev/null +++ b/tests/ut/cpp/stub/runtime/cuda_runtime_api.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +cudaError_t cudaMalloc(void **devPtr, size_t size) { return cudaSuccess; } + +cudaError_t cudaFree(void *devPtr) { return cudaSuccess; } + +cudaError_t cudaMemcpy(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind) { return cudaSuccess; } + +cudaError_t cudaMemGetInfo(size_t *free, size_t *total) { return cudaSuccess; } + +cudaError_t cudaStreamCreate(cudaStream_t *pStream) { return cudaSuccess; } + +cudaError_t cudaStreamDestroy(cudaStream_t stream) { return cudaSuccess; } + +cudaError_t cudaStreamSynchronize(cudaStream_t stream) { return cudaSuccess; } + +cudaError_t cudaGetDeviceCount(int *count) { return cudaSuccess; } + +cudaError_t cudaSetDevice(int device) { return cudaSuccess; } diff --git a/tests/ut/cpp/stub/runtime/cuda_runtime_api.h b/tests/ut/cpp/stub/runtime/cuda_runtime_api.h new file mode 100644 index 00000000000..1a30d5879c4 --- /dev/null +++ b/tests/ut/cpp/stub/runtime/cuda_runtime_api.h @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_STUB_RUNTIME_INCLUDE_CUDA_RUNTIME_API_H_ +#define TESTS_UT_STUB_RUNTIME_INCLUDE_CUDA_RUNTIME_API_H_ + +#include +typedef enum { cudaSuccess = 0 } cudaError_t; + +enum cudaMemcpyKind { + cudaMemcpyHostToHost = 0, + cudaMemcpyHostToDevice = 1, + cudaMemcpyDeviceToHost = 2, + cudaMemcpyDeviceToDevice = 3 +}; + +struct CUstream_st { + int arch; +}; + +typedef struct CUStream_st *cudaStream_t; + +cudaError_t cudaMalloc(void **devPtr, size_t size); +cudaError_t cudaFree(void *devPtr); +cudaError_t cudaMemcpy(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind); +cudaError_t cudaMemGetInfo(size_t *free, size_t *total); +cudaError_t cudaStreamCreate(cudaStream_t *pStream); +cudaError_t cudaStreamDestroy(cudaStream_t stream); +cudaError_t cudaStreamSynchronize(cudaStream_t stream); +cudaError_t cudaGetDeviceCount(int *count); +cudaError_t cudaSetDevice(int device); + +#endif // TESTS_UT_STUB_RUNTIME_INCLUDE_CUDA_RUNTIME_API_H_ diff --git a/tests/ut/cpp/stub/runtime/runtime_stub.cc b/tests/ut/cpp/stub/runtime/runtime_stub.cc new file mode 100644 index 00000000000..b7099124bc0 --- /dev/null +++ b/tests/ut/cpp/stub/runtime/runtime_stub.cc @@ -0,0 +1,135 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "runtime/base.h" +#include "runtime/context.h" +#include "runtime/dev.h" +#include "runtime/event.h" +#include "runtime/kernel.h" +#include "runtime/mem.h" +#include "runtime/rt_model.h" +#include "runtime/stream.h" + +rtError_t rtEventSynchronize(rtEvent_t event) { return RT_ERROR_NONE; } + +rtError_t rtMalloc(void **devPtr, uint64_t size, rtMemType_t type) { return RT_ERROR_NONE; } + +rtError_t rtMemcpy(void *dst, uint64_t destMax, const void *src, uint64_t count, rtMemcpyKind_t kind) { + return RT_ERROR_NONE; +} + +rtError_t rtMemset(void *devPtr, uint64_t destMax, uint32_t value, uint64_t count) { return RT_ERROR_NONE; } + +rtError_t rtGetDeviceCount(int32_t *count) { return RT_ERROR_NONE; } + +rtError_t rtSetDevice(int32_t device) { return RT_ERROR_NONE; } + +rtError_t rtCtxCreate(rtContext_t *ctx, uint32_t flags, int32_t device) { return RT_ERROR_NONE; } + +rtError_t rtCtxSetCurrent(rtContext_t ctx) { return RT_ERROR_NONE; } + +rtError_t rtCtxDestroy(rtContext_t ctx) { return RT_ERROR_NONE; } + +rtError_t rtStreamCreate(rtStream_t *stream, int32_t priority) { return RT_ERROR_NONE; } + +rtError_t rtStreamDestroy(rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtDevBinaryRegister(const rtDevBinary_t *bin, void **handle) { return RT_ERROR_NONE; } + +rtError_t rtFunctionRegister(void *binHandle, const void *stubFunc, const char *stubName, const void *devFunc, + uint32_t funcMode) { + return RT_ERROR_NONE; +} + +rtError_t rtKernelLaunch(const void *stubFunc, uint32_t blockDim, void *args, uint32_t argsSize, rtSmDesc_t *smDesc, + rtStream_t stream) { + return RT_ERROR_NONE; +} + +rtError_t rtStreamSynchronize(rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtKernelLaunchEx(void *args, uint32_t argsSize, uint32_t flags, rtStream_t stream_) { return RT_ERROR_NONE; } + +rtError_t rtFree(void *devPtr) { + delete[] reinterpret_cast(devPtr); + return RT_ERROR_NONE; +} + +rtError_t rtModelExecute(rtModel_t model, rtStream_t stream, uint32_t flag) { return RT_ERROR_NONE; } + +rtError_t rtMemAllocManaged(void **ptr, uint64_t size, uint32_t flag) { return RT_ERROR_NONE; } + +rtError_t rtMemcpyAsync(void *dst, uint64_t destMax, const void *src, uint64_t count, rtMemcpyKind_t kind, + rtStream_t stream) { + return RT_ERROR_NONE; +} + +rtError_t rtLabelSwitch(void *ptr, rtCondition_t condition, uint32_t value, rtLabel_t trueLabel, rtStream_t stream) { + return RT_ERROR_NONE; +} + +rtError_t rtStreamSwitch(void *ptr, rtCondition_t condition, int64_t value, rtStream_t true_stream, rtStream_t stream) { + return RT_ERROR_NONE; +} +rtError_t rtStreamSwitchEx(void *ptr, rtCondition_t condition, void *value_ptr, rtStream_t true_stream, + rtStream_t stream, rtSwitchDataType_t dataType) { + return RT_ERROR_NONE; +} + +rtError_t rtKernelFusionEnd(rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtKernelFusionStart(rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtStreamWaitEvent(rtStream_t stream, rtEvent_t event) { return RT_ERROR_NONE; } + +rtError_t rtEventReset(rtEvent_t event, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtEventRecord(rtEvent_t event, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtLabelGoto(rtLabel_t label, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtLabelSet(rtLabel_t label, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtStreamActive(rtStream_t active_stream, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtModelUnbindStream(rtModel_t model, rtStream_t stream) { return RT_ERROR_NONE; } + +rtError_t rtLabelDestroy(rtLabel_t label) { return RT_ERROR_NONE; } + +rtError_t rtModelDestroy(rtModel_t model) { return RT_ERROR_NONE; } + +rtError_t rtEventDestroy(rtEvent_t event) { return RT_ERROR_NONE; } + +rtError_t rtMemFreeManaged(void *ptr) { return RT_ERROR_NONE; } + +rtError_t rtModelCreate(rtModel_t *model, uint32_t flag) { return RT_ERROR_NONE; } + +rtError_t rtModelBindStream(rtModel_t model, rtStream_t stream, uint32_t flag) { return RT_ERROR_NONE; } + +rtError_t rtStreamCreateWithFlags(rtStream_t *stream, int32_t priority, uint32_t flags) { return RT_ERROR_NONE; } + +rtError_t rtEventCreate(rtEvent_t *event) { return RT_ERROR_NONE; } + +rtError_t rtLabelCreate(rtLabel_t *label) { return RT_ERROR_NONE; } + +rtError_t rtModelLoadComplete(rtModel_t model) { return RT_ERROR_NONE; } + +rtError_t rtCtxGetCurrent(rtContext_t *ctx) { return RT_ERROR_NONE; } + +rtError_t rtGetStreamId(rtStream_t stream, int32_t *streamId) { return RT_ERROR_NONE; } + +rtError_t rtGetFunctionByName(const char *stubName, void **stubFunc) { return RT_ERROR_NONE; } + +rtError_t rtSetTaskGenCallback(rtTaskGenCallback callback) { return RT_ERROR_NONE; } diff --git a/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc b/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc new file mode 100755 index 00000000000..ebd2ac8b465 --- /dev/null +++ b/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "device/ascend/ascend_stream_assign.h" +#include "device/ascend/tasksink/task_generator.h" +#include "device/kernel_adjust.h" + +namespace mindspore { +namespace device { +namespace ascend { +void AscendStreamAssign::AssignStreamNew(const KernelGraphPtr &graph) { return; } + +uint32_t AscendStreamAssign::GetTotalStreamNum() const { return 1; } + +std::vector AscendStreamAssign::GetWaitStreams() { return vector(); } + +std::vector AscendStreamAssign::GetHcomStreams() { return vector(); } + +namespace tasksink { +bool TaskGenerator::GenTasks(const std::vector &anf_node_list, std::vector *const task_info_list, + uint32_t graph_id) { + return true; +} +} // namespace tasksink +} // namespace ascend +void KernelAdjust::Reorder(const std::shared_ptr &kernel_graph_ptr) { return; } +void KernelAdjust::InsertSwitchLoop(const std::shared_ptr &kernel_graph_ptr) { return; } +bool KernelAdjust::StepLoadCtrlInputs(const std::shared_ptr &context, + const std::shared_ptr &kernel_graph_ptr) { + return true; +} +bool KernelAdjust::NeedInsertSwitch() { return true; } +void KernelAdjust::Profiling(const std::shared_ptr &kernel_graph_ptr) { return; } +} // namespace device +} // namespace mindspore diff --git a/tests/ut/cpp/stub/tdt/tdt_mock.cc b/tests/ut/cpp/stub/tdt/tdt_mock.cc new file mode 100644 index 00000000000..45725de173a --- /dev/null +++ b/tests/ut/cpp/stub/tdt/tdt_mock.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TDT_MOCK_H +#define TDT_MOCK_H + +#include "tdt/tsd_client.h" + +namespace tdt { +StatusFactory* StatusFactory::GetInstance() { + static StatusFactory instance; + return &instance; +} + +void StatusFactory::RegisterErrorNo(const uint32_t err, const std::string& desc) { return; } + +std::string StatusFactory::GetErrDesc(const uint32_t err) { return "Error"; } + +std::string StatusFactory::GetErrCodeDesc(uint32_t errCode) { return "Error"; } + +StatusFactory::StatusFactory() {} + +std::mutex& StatusFactory::GetMutex() { return GetInstance()->rwMutex_; } + +TsdClient* TsdClient::GetInstance() { + static TsdClient instance; + return &instance; +} + +/** + * @ingroup TsdClient + * @brief 构造函数 + */ +TsdClient::TsdClient() { rankSize_ = 1; } + +/** + * @ingroup TsdClient + * @brief 析构函数 + */ +TsdClient::~TsdClient() = default; + +/** + * @ingroup TsdClient + * @brief framework发送拉起hccp和computer process的命令 + * @param [in] phyDeviceId : FMK传入物理ID + * @param [in] phyDeviceId : FMK传入rankSize + * @return TDT_OK:成功 或者其他错误码 + */ +TDT_StatusT TsdClient::Open(const uint32_t deviceId, const uint32_t rankSize) { return TDT_OK; } + +/** + * @ingroup TsdClient + * @brief 通知TsdClient关闭相关资源 + * @param 无 + * @return TDT_OK:成功 或者其他错误码 + */ +TDT_StatusT TsdClient::Close() { return TDT_OK; } + +} // namespace tdt +#endif // TDT_MOCK_H diff --git a/tests/ut/cpp/transform/convert_test.cc b/tests/ut/cpp/transform/convert_test.cc new file mode 100644 index 00000000000..c7cd3940022 --- /dev/null +++ b/tests/ut/cpp/transform/convert_test.cc @@ -0,0 +1,936 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "pybind11/pybind11.h" + +#include "transform/transform_base_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pipeline/parse/parse.h" +#include "debug/draw.h" +#include "debug/anf_ir_dump.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" +#include "common/common_test.h" + +#define private public +#include "transform/types.h" +#include "transform/convert.h" +#include "securec/include/securec.h" +#include "utils/utils.h" +using std::cout; +using std::endl; +using std::string; +using std::unordered_map; + +namespace mindspore { +namespace transform { +using AbstractScalar = abstract::AbstractScalar; +using mindspore::parse::ResolveAll; + +class TestConvert : public UT::Common { + public: + TestConvert() {} + virtual void SetUp(); + virtual void TearDown(); + static const std::shared_ptr kF32; +}; + +void TestConvert::SetUp() { UT::InitPythonPath(); } +void TestConvert::TearDown() {} + +const std::shared_ptr TestConvert::kF32 = std::make_shared(32); + +AnfGraphPtr createAnfGraph() { return std::make_shared(); } + +TEST_F(TestConvert, TestConstruct) { + AnfGraphPtr func_graph = std::make_shared(); + DfGraphConvertor convertor(func_graph); + convertor.ConvertAllNode().GetComputeGraph(); + ASSERT_NE(convertor.ErrCode(), SUCCESS); +} + +#if (!defined ENABLE_GE) + +namespace { + +bool MakeDfGraph(PrimitivePtr prim, unsigned int nparam) { + std::shared_ptr anf_graph = MakeFuncGraph(prim, nparam); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_" + prim->name() + ".dot", anf_graph); + DumpIR("ut_prim_" + prim->name() + ".ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph(prim->name() + ".dot"); + if (convertor.ErrCode() != 0) { + MS_LOG(ERROR) << "DfGraphConvertor convert " << prim->name() << " error, error code is: " << convertor.ErrCode(); + return false; + } + if (df_graph == nullptr) { + MS_LOG(ERROR) << "DfGraphConvertor get " << prim->name() << " compute func_graph failed"; + return false; + } + return true; +} + +} // namespace + +TEST_F(TestConvert, TestConvertConv2d) { + PrimitivePtr conv2d = prim::kPrimConv2D; + conv2d->AddAttr("stride", MakeValue(2)); + conv2d->AddAttr("pad", MakeValue(0)); + conv2d->AddAttr("dilation", MakeValue(0)); + + FuncGraphPtr anf_graph = MakeFuncGraph(conv2d, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_conv2d1.dot", anf_graph); + DumpIR("ut_prim_conv2d1.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("conv2d.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertMaxpooling) { + auto prim = std::make_shared("MaxPool"); + FuncGraphPtr anf_graph = MakeFuncGraph(prim, 5); // ary, ksize, stride, padding, data_format + + std::shared_ptr graph_manager = MakeManager({anf_graph}); + draw::Draw("ut_prim_maxpooling.dot", anf_graph); + DumpIR("ut_prim_maxpooling.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("maxpooling.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestReluOps) { + auto prim = prim::kPrimRelu; + prim->AddAttr("T", MakeValue(0)); + + auto func_graph = MakeFuncGraph(prim, 1); + ASSERT_TRUE(nullptr != func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anfGraph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anfGraph); + convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + ASSERT_EQ(convertor.ErrCode(), 0); +} + +TEST_F(TestConvert, TestConvertBatchNorm) { + PrimitivePtr fused_batch_norm = prim::kPrimFusedBatchNorm; + fused_batch_norm->AddAttr("epsilon", MakeValue(0.001f)); + fused_batch_norm->AddAttr("momentum", MakeValue(0.1f)); + + FuncGraphPtr anf_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(fused_batch_norm)); + for (unsigned int i = 0; i < 5; i++) { + inputs.push_back(anf_graph->add_parameter()); + } + CNodePtr cnode_prim = anf_graph->NewCNode(inputs); + inputs.clear(); + + inputs.push_back(NewValueNode(prim::kPrimTupleGetItem)); + inputs.push_back(cnode_prim); + inputs.push_back(NewValueNode(2)); + CNodePtr cnode_getitem = anf_graph->NewCNode(inputs); + inputs.clear(); + + inputs.push_back(NewValueNode(prim::kPrimRelu)); + inputs.push_back(cnode_getitem); + CNodePtr cnode_relu = anf_graph->NewCNode(inputs); + inputs.clear(); + + inputs.push_back(NewValueNode(std::make_shared("return"))); + inputs.push_back(cnode_relu); + CNodePtr cnode_return = anf_graph->NewCNode(inputs); + anf_graph->set_return(cnode_return); + + std::shared_ptr graph_manager = MakeManager({anf_graph}); + draw::Draw("ut_prim_batchnorm.dot", anf_graph); + DumpIR("ut_prim_batchnorm.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("batchnrom.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertConvBackpropInput) { + auto prim = prim::kPrimConv2DBackpropInput; + prim->AddAttr("stride", MakeValue(1)); + prim->AddAttr("pad", MakeValue(0)); + prim->AddAttr("pad_mode", MakeValue(std::string("pad"))); + prim->AddAttr("dilation", MakeValue(1)); + prim->AddAttr("group", MakeValue(1)); + prim->AddAttr("mode", MakeValue(1)); + prim->AddAttr("dilation", MakeValue(1)); + + auto func_graph = MakeFuncGraph(prim, 3); + ASSERT_NE(func_graph, nullptr); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anf_graph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + + convertor.DrawComputeGraph("Conv2DBackpropInput.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertConvBackpropFilter) { + auto prim = prim::kPrimConv2DBackpropFilter; + prim->AddAttr("stride", MakeValue(1)); + prim->AddAttr("pad", MakeValue(0)); + prim->AddAttr("pad_mode", MakeValue(std::string("pad"))); + prim->AddAttr("dilation", MakeValue(1)); + prim->AddAttr("group", MakeValue(1)); + prim->AddAttr("mode", MakeValue(1)); + prim->AddAttr("dilation", MakeValue(1)); + + auto func_graph = MakeFuncGraph(prim, 3); + ASSERT_NE(func_graph, nullptr); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anf_graph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + + convertor.DrawComputeGraph("Conv2DBackpropFilter.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertReluGrad) { + auto prim = prim::kPrimReluGrad; + prim->AddAttr("alpha", MakeValue(0.1f)); + prim->AddAttr("beta", MakeValue(0.1f)); + prim->AddAttr("mode", MakeValue(1)); + + auto func_graph = MakeFuncGraph(prim, 2); + ASSERT_NE(func_graph, nullptr); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anf_graph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + + convertor.DrawComputeGraph("ReluGrad.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertBiasAdd) { + auto prim = std::make_shared("BiasAdd"); + prim->AddAttr("alpha", MakeValue(0.0f)); + prim->AddAttr("beta", MakeValue(1.0f)); + prim->AddAttr("format", MakeValue(1)); + + auto func_graph = MakeFuncGraph(prim, 2); + ASSERT_NE(func_graph, nullptr); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anf_graph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + + convertor.DrawComputeGraph("BiasAdd.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertBiasAddGrad) { + auto prim = prim::kPrimBiasAddGrad; + prim->AddAttr("alpha", MakeValue(0.0f)); + prim->AddAttr("beta", MakeValue(1.0f)); + prim->AddAttr("format", MakeValue(1)); + + auto func_graph = MakeFuncGraph(prim, 2); + ASSERT_NE(func_graph, nullptr); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anf_graph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + + convertor.DrawComputeGraph("BiasAddGrad.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConvertMaxPoolGradWithArgmax) { + auto prim = std::make_shared("MaxPoolGradWithArgmax"); + prim->AddAttr("alpha", MakeValue(0.0f)); + prim->AddAttr("beta", MakeValue(1.0f)); + prim->AddAttr("window", MakeValue(2)); + prim->AddAttr("stride", MakeValue(1)); + prim->AddAttr("ceil_mode", MakeValue(0)); + prim->AddAttr("data_mode", MakeValue(0)); + prim->AddAttr("alpha", MakeValue(0.1f)); + prim->AddAttr("beta", MakeValue(1.0f)); + + auto func_graph = MakeFuncGraph(prim, 2); + ASSERT_NE(func_graph, nullptr); + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anf_graph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + + convertor.DrawComputeGraph("MaxPoolGradWithArgmax.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestConcat) { + auto prim = prim::kPrimConcat; + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_concat.dot", anf_graph); + DumpIR("ut_prim_concat.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("concat.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestGatherV2) { + auto prim = prim::kPrimGatherV2; + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 3); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_gatherv2.dot", anf_graph); + DumpIR("ut_prim_gatherv2.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("gatherv2.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestCast) { + auto prim = prim::kPrimCast; + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_cast.dot", anf_graph); + DumpIR("ut_prim_cast.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("cast.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestExp) { + auto prim = std::make_shared("Exp"); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 1); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_exp.dot", anf_graph); + DumpIR("ut_prim_exp.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("exp.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestFloor) { + auto prim = std::make_shared("Floor"); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 1); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_floor.dot", anf_graph); + DumpIR("ut_prim_floor.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("floor.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestGreaterEqual) { + auto prim = std::make_shared("GreaterEqual"); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_greater_equal.dot", anf_graph); + DumpIR("ut_prim_greater_equal.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("greater_equal.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestLess) { + auto prim = std::make_shared("Less"); + prim->AddAttr("T", MakeValue(kFloat32)); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_less.dot", anf_graph); + DumpIR("ut_prim_less.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("less.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestLessEqual) { + auto prim = std::make_shared("LessEqual"); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_less_equal.dot", anf_graph); + DumpIR("ut_prim_less_equal.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("less_equal.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestLogicalNot) { + auto prim = std::make_shared("LogicalNot"); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 1); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_logical_not.dot", anf_graph); + DumpIR("ut_prim_logical_not.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("logical_not.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestAssignAdd) { + auto prim = prim::kPrimAssignAdd; + prim->AddAttr("use_locking", MakeValue(true)); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 2); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_assign_add.dot", anf_graph); + DumpIR("ut_prim_assign_add.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("assign_add.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, LogSoftmax) { + auto prim = prim::kPrimLogSoftmax; + prim->AddAttr("axis", MakeValue(0)); + + std::shared_ptr anf_graph = MakeFuncGraph(prim, 1); + std::shared_ptr graph_manager = MakeManager({anf_graph}); + + draw::Draw("ut_prim_log_softmax.dot", anf_graph); + DumpIR("ut_prim_log_softmax.ir", anf_graph); + + DfGraphConvertor convertor(anf_graph); + auto df_graph = convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + convertor.DrawComputeGraph("log_softmax.dot"); + ASSERT_EQ(convertor.ErrCode(), 0); + ASSERT_NE(df_graph, nullptr); +} + +TEST_F(TestConvert, TestMaximumOps) { + auto prim = prim::kPrimMaximum; + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestReduceMeanOps) { + auto prim = prim::kPrimReduceMean; + prim->AddAttr("keepdims", MakeValue(true)); + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestMinimumOps) { + auto prim = prim::kPrimMinimum; + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestFusedMinOrMaxGradOps) { + // Add infer step to this test case + ASSERT_TRUE(true); +} + +TEST_F(TestConvert, TestSqueezeOps) { + auto prim = prim::kPrimSqueeze; + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestMulOps) { + auto prim = prim::kPrimMul; + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestNegOps) { + auto prim = prim::kPrimNeg; + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestOneHotOps) { + auto prim = prim::kPrimOneHot; + prim->AddAttr("axis", MakeValue(0)); + bool ret = MakeDfGraph(prim, 4); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestPowOps) { + auto prim = std::make_shared("Pow"); + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestReciprocalOps) { + auto prim = std::make_shared("Reciprocal"); + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestSelectOps) { + auto prim = prim::kPrimSelect; + bool ret = MakeDfGraph(prim, 3); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestSqrtOps) { + auto prim = std::make_shared("Sqrt"); + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestSquareOps) { + auto prim = std::make_shared("Square"); + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestScalarSummaryOps) { + auto prim = prim::kPrimScalarSummary; + // should have only 1 input. + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestTensorSummaryOps) { + auto prim = prim::kPrimTensorSummary; + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestGreaterOps) { + auto prim = std::make_shared("Greater"); + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestEqualOps) { + auto prim = std::make_shared("Equal"); + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestArgMaxiOps) { + auto prim = std::make_shared("Argmax"); + bool ret = MakeDfGraph(prim, 2); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestResizeNearestNeighborOps) { + auto prim = std::make_shared("ResizeNearestNeighbor"); + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestApplyMomentumOps) { + auto prim = std::make_shared("ApplyMomentum"); + bool ret = MakeDfGraph(prim, 5); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestNPUGetFloatStatusOps) { + auto prim = std::make_shared("NPUGetFloatStatus"); + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestNPUAllocFloatStatusOps) { + auto prim = std::make_shared("NPUAllocFloatStatus"); + bool ret = MakeDfGraph(prim, 0); + ASSERT_TRUE(ret); +} + +TEST_F(TestConvert, TestNPUClearFloatStatusOps) { + auto prim = std::make_shared("NPUClearFloatStatus"); + bool ret = MakeDfGraph(prim, 1); + ASSERT_TRUE(ret); +} + +#endif + +TEST_F(TestConvert, TestAddOps) { + auto prim = std::make_shared("TensorAdd"); + auto func_graph = MakeFuncGraph(prim, 2); + ASSERT_TRUE(nullptr != func_graph); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anfGraph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anfGraph); + convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + ASSERT_EQ(convertor.ErrCode(), 0); +} + +TEST_F(TestConvert, TestConvertTensor) { + float data[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + // Create a tensor with wanted data type and shape + std::vector dims{2, 2, 3}; + std::vector ge_dims{2, 2, 3}; + auto type_id = kNumberTypeFloat32; + MeTensor me_tensor(type_id, dims); + // Get the writable data pointer of the tensor and cast it to its data type + uint8_t* me_data_ptr = reinterpret_cast(me_tensor.data_c(true)); + // Copy or use the writable data pointer of the ME tensor + memcpy_s(me_data_ptr, me_tensor.data().nbytes(), data, 12 * sizeof(float)); + auto me_tensor_ptr = std::make_shared(me_tensor); + auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensor_ptr, kOpFormat_NCHW); + ASSERT_EQ(ge_tensor_ptr->GetTensorDesc().GetFormat(), GeFormat::FORMAT_NCHW); + ASSERT_EQ(ge_tensor_ptr->GetTensorDesc().GetDataType(), GeDataType::DT_FLOAT); + // ASSERT_EQ(ge_tensor_ptr->GetTensorDesc().array().GetDims(), ge_dims); + int i = 0; + for (i = 0; i < ge_dims.size(); i++) { + ASSERT_EQ(ge_dims[i], ge_tensor_ptr->GetTensorDesc().GetShape().GetDims()[i]); + } + for (i = 0; i < ge_tensor_ptr->GetTensorDesc().GetShape().GetShapeSize(); i++) { + ASSERT_EQ(data[i], (reinterpret_cast(ge_tensor_ptr->GetData()))[i]); + } +} + +TEST_F(TestConvert, TestConvertTensor0Dims) { + // shape with 0 dims is also valid + std::vector dims{}; + auto type_id = kNumberTypeFloat32; + auto me_tensor_ptr = std::make_shared(type_id, dims); + ASSERT_NE(TransformUtil::ConvertTensor(me_tensor_ptr, kOpFormat_NCHW), nullptr); +} + +TEST_F(TestConvert, TestConvertTensorError) { + std::vector dims2{2, 3, 4}; + auto type_id_2 = kNumberTypeFloat32; + auto me_tensor_ptr_2 = std::make_shared(type_id_2, dims2); + ASSERT_EQ(TransformUtil::ConvertTensor(me_tensor_ptr_2, "xyz"), nullptr); +} + +TEST_F(TestConvert, TestUtilsConvertDataType) { + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeFloat16), GeDataType::DT_FLOAT16); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeFloat32), GeDataType::DT_FLOAT); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeFloat64), GeDataType::DT_DOUBLE); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeInt8), GeDataType::DT_INT8); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeInt16), GeDataType::DT_INT16); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeInt32), GeDataType::DT_INT32); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeInt64), GeDataType::DT_INT64); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeUInt32), GeDataType::DT_UINT32); + ASSERT_EQ(TransformUtil::ConvertDataType(MeDataType::kNumberTypeBool), GeDataType::DT_BOOL); +} + +TEST_F(TestConvert, TestUtilsConvertFormat) { + ASSERT_EQ(TransformUtil::ConvertFormat(kOpFormat_NCHW), GeFormat::FORMAT_NCHW); + ASSERT_EQ(TransformUtil::ConvertFormat(kOpFormat_NC1HWC0), GeFormat::FORMAT_NC1HWC0); + ASSERT_EQ(TransformUtil::ConvertFormat(kOpFormat_NHWC), GeFormat::FORMAT_NHWC); + ASSERT_EQ(TransformUtil::ConvertFormat("xyz"), GeFormat::FORMAT_ND); +} + +TEST_F(TestConvert, TestUtilsDataSize) { + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeFloat32), 4); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeFloat16), 2); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeFloat64), 8); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeInt8), 1); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeInt16), 2); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeInt32), 4); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeInt64), 8); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeUInt32), 4); + ASSERT_EQ(TransformUtil::GetDataTypeSize(MeDataType::kNumberTypeBool), 1); +} + +TEST_F(TestConvert, TestConvertGeTensor) { +#define DTYPE float + ge::DataType dt = ge::DataType::DT_FLOAT; + + std::vector data1 = {1.1, 2.2, 3.3, 4.4, 6.6, 7.7, 8.8, 9.9}; + std::vector data2 = {1, 2, 3, 4, 6, 7, 8, 9}; + auto data = data1; + ge::Shape shape({2, 2, 2}); + ge::Format format = ge::Format::FORMAT_NCHW; + ge::TensorDesc desc(shape, format, dt); + GeTensorPtr ge_tensor_ptr = + std::make_shared(desc, reinterpret_cast(data.data()), data.size() * sizeof(DTYPE)); + GeTensor& ge_tensor = *ge_tensor_ptr; + const DTYPE* ge_data = reinterpret_cast(ge_tensor.GetData()); + + // make sure GetData()'s return is a reference + assert(ge_data == reinterpret_cast(ge_tensor.GetData())); + + cout << "ge data size is: " << std::dec << ge_tensor.GetSize() << " bytes" << endl; + for (int i = 0; i < ge_tensor.GetSize() / sizeof(DTYPE); i++) { + cout << "ge data is: " << static_cast(*(ge_data + i)) << endl; + } + + MeTensorPtr me_tensor_ptr = TransformUtil::ConvertGeTensor(ge_tensor_ptr); + MeTensor& me_tensor = *me_tensor_ptr; + cout << "after convert ge tensor to me tensor" << endl; + DTYPE* me_data = reinterpret_cast(me_tensor.data_c()); + PrintMeTensor(&me_tensor); + + assert(ge_tensor.GetSize() == me_tensor.data().nbytes()); + assert(memcmp(ge_data, me_data, ge_tensor.GetSize()) == 0); +} + +TEST_F(TestConvert, TestConvertMakeTuple) { + FuncGraphPtr func_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(std::make_shared("make_tuple"))); + for (int i = 0; i < 3; i++) { + auto input = func_graph->add_parameter(); + input->set_name("x" + std::to_string(i)); + inputs.push_back(input); + } + CNodePtr cnode_prim = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(std::make_shared("return"))); + inputs.push_back(cnode_prim); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + ASSERT_TRUE(ret_); + + // draw graph + auto anfGraph = *(manager->func_graphs().begin()); + DfGraphConvertor convertor(anfGraph); + convertor.ConvertAllNode().BuildGraph().GetComputeGraph(); + ASSERT_EQ(convertor.ErrCode(), 0); +} + +TEST_F(TestConvert, TestConvertInputTensors) { +#define DTYPE float + MeTensorPtr input_ptr1 = MakeTensor(kF32, {1, 1, 4, 4}); + MeTensorPtr input_ptr2 = MakeTensor(kF32, {2, 3, 4, 5}); + MeTensorPtr input_ptr3 = MakeTensor(kF32, {9, 9, 1, 1}); + std::vector me_inputs; + me_inputs.emplace_back(input_ptr1); + me_inputs.emplace_back(input_ptr2); + me_inputs.emplace_back(input_ptr3); + + std::vector ge_tensors = TransformUtil::ConvertInputTensors(me_inputs, kOpFormat_NCHW); + + for (int i = 0; i < ge_tensors.size(); i++) { + DTYPE* me_data = reinterpret_cast(me_inputs[i]->data_c()); + const DTYPE* ge_data = reinterpret_cast(ge_tensors[i]->GetData()); + ASSERT_TRUE(ge_tensors[i]->GetSize() == me_inputs[i]->data().nbytes()); + ASSERT_EQ(memcmp(ge_data, me_data, ge_tensors[i]->GetSize()), 0); + ASSERT_TRUE(ge_tensors[i]->GetTensorDesc().GetShape().GetDims() == + TransformUtil::ConvertMeShape(me_inputs[i]->shape_c()).GetDims()); + } +} + +TEST_F(TestConvert, TestConvertGeTensors) { +#define DTYPE float + ge::DataType dt = ge::DataType::DT_FLOAT; + + std::vector data1(16); + std::vector data2(120); + std::vector data3(81); + ge::Shape shape1({1, 1, 4, 4}); + ge::Shape shape2({2, 3, 4, 5}); + ge::Shape shape3({9, 9, 1, 1}); + ge::Format format = ge::Format::FORMAT_NCHW; + ge::TensorDesc desc1(shape1, format, dt); + ge::TensorDesc desc2(shape2, format, dt); + ge::TensorDesc desc3(shape3, format, dt); + GeTensorPtr ge_tensor_ptr1 = + std::make_shared(desc1, reinterpret_cast(data1.data()), data1.size() * sizeof(DTYPE)); + GeTensorPtr ge_tensor_ptr2 = + std::make_shared(desc2, reinterpret_cast(data2.data()), data2.size() * sizeof(DTYPE)); + GeTensorPtr ge_tensor_ptr3 = + std::make_shared(desc3, reinterpret_cast(data3.data()), data3.size() * sizeof(DTYPE)); + + std::vector ge_tensors; + ge_tensors.emplace_back(ge_tensor_ptr1); + ge_tensors.emplace_back(ge_tensor_ptr2); + ge_tensors.emplace_back(ge_tensor_ptr3); + + std::vector> request_dims; + std::vector dims1 = {1, 1, 4, 4}; + std::vector dims2 = {2, 3, 4, 5}; + std::vector dims3 = {9, 9, 1, 1}; + request_dims.emplace_back(dims1); + request_dims.emplace_back(dims2); + request_dims.emplace_back(dims3); + + std::vector me_outputs = TransformUtil::ConvertGeTensors(ge_tensors, request_dims); + + for (int i = 0; i < ge_tensors.size(); i++) { + DTYPE* me_data = reinterpret_cast(me_outputs[i]->data_c()); + const DTYPE* ge_data = reinterpret_cast(ge_tensors[i]->GetData()); + ASSERT_TRUE(ge_tensors[i]->GetSize() == me_outputs[i]->data().nbytes()); + ASSERT_EQ(memcmp(ge_data, me_data, ge_tensors[i]->GetSize()), 0); + ASSERT_TRUE(request_dims[i] == me_outputs[i]->shape_c()); + } +} + +TEST_F(TestConvert, TestConvertGeShape1) { + GeShape ge_shape({10, 1, 1, 1}); + std::vector request_dims{10}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == request_dims); +} + +TEST_F(TestConvert, TestConvertGeShape2) { + GeShape ge_shape({10, 15, 1, 1}); + std::vector request_dims{10, 15}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == request_dims); +} + +TEST_F(TestConvert, TestConvertGeShape3) { + GeShape ge_shape({10, 13, 18, 1}); + std::vector request_dims{10, 13, 18}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == request_dims); +} + +TEST_F(TestConvert, TestConvertGeShape4) { + GeShape ge_shape({1, 10, 1, 1}); + std::vector request_dims{10}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == request_dims); +} + +TEST_F(TestConvert, TestConvertGeShape5) { + GeShape ge_shape({10, 1, 1, 2}); + std::vector request_dims{10}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == TransformUtil::ConvertGeShape(ge_shape)); +} + +TEST_F(TestConvert, TestConvertGeShape6) { + GeShape ge_shape({5, 2, 1, 1}); + std::vector request_dims{10}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == TransformUtil::ConvertGeShape(ge_shape)); +} + +TEST_F(TestConvert, TestConvertGeShape7) { + GeShape ge_shape({10}); + std::vector request_dims{10, 1}; + ASSERT_TRUE(TransformUtil::ConvertGeShape(ge_shape, request_dims) == TransformUtil::ConvertGeShape(ge_shape)); +} +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/ge_stub_test.cc b/tests/ut/cpp/transform/ge_stub_test.cc new file mode 100644 index 00000000000..720a16f36b7 --- /dev/null +++ b/tests/ut/cpp/transform/ge_stub_test.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" +#include "graph/tensor.h" + +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif +#include "graph/operator.h" +#include "graph/operator_reg.h" + +namespace mindspore { +namespace transform { + +class TestGEStub : public UT::Common { + public: + TestGEStub() {} +}; + +TEST_F(TestGEStub, TestAPI) { + // only test for ge header compiling + ASSERT_TRUE(true); +} + +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/graph_builder_test.cc b/tests/ut/cpp/transform/graph_builder_test.cc new file mode 100644 index 00000000000..e92463e2dc0 --- /dev/null +++ b/tests/ut/cpp/transform/graph_builder_test.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "common/common_test.h" + +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif + +#define private public +#include "transform/graph_builder.h" +#include "transform/df_graph_manager.h" + +using UT::Common; + +namespace mindspore { +namespace transform { + +class TestDfGraphBuilder : public UT::Common { + public: + TestDfGraphBuilder() {} + void SetUp(); + void TearDown(); +}; + +void TestDfGraphBuilder::SetUp() {} + +void TestDfGraphBuilder::TearDown() {} + +TEST_F(TestDfGraphBuilder, TestBuildDatasetGraph) { + DatasetGraphParam param4("queue_name", 1, 32, {0, 3}, {{32, 224, 224, 3}, {32}}, {}); + ASSERT_EQ(transform::SUCCESS, BuildDatasetGraph(param4)); + DfGraphManager::GetInstance().ClearGraph(); +} + +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/graph_manager_test.cc b/tests/ut/cpp/transform/graph_manager_test.cc new file mode 100644 index 00000000000..699f81ca4c8 --- /dev/null +++ b/tests/ut/cpp/transform/graph_manager_test.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "common/common_test.h" + +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif + +#define private public +#include "transform/df_graph_manager.h" + +using UT::Common; + +namespace mindspore { +namespace transform { + +class TestDfGraphManager : public UT::Common { + public: + TestDfGraphManager() {} +}; + +TEST_F(TestDfGraphManager, TestAPI) { + // test public interface: + DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + ASSERT_EQ(0, graph_manager.GetAllGraphs().size()); + + // test public interface: + std::shared_ptr ge_graph = std::make_shared(); + ASSERT_TRUE(graph_manager.AddGraph("test_graph", nullptr) != Status::SUCCESS); + graph_manager.AddGraph("test_graph", ge_graph); + ASSERT_EQ(1, graph_manager.GetAllGraphs().size()); + std::vector wrappers = graph_manager.GetAllGraphs(); + ASSERT_EQ("test_graph", wrappers.back()->name_); + ASSERT_EQ(ge_graph, wrappers.back()->graph_ptr_); + + // test public interface: + DfGraphWrapperPtr wrappers2 = graph_manager.GetGraphByName("test_graph"); + ASSERT_EQ(ge_graph, wrappers2->graph_ptr_); + + // test public interface: + graph_manager.ClearGraph(); + ASSERT_EQ(0, graph_manager.GetAllGraphs().size()); + + // test public interface: + int id = graph_manager.GenerateId(); + assert(id > 0); +} + +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/graph_runner_test.cc b/tests/ut/cpp/transform/graph_runner_test.cc new file mode 100644 index 00000000000..ab73005453e --- /dev/null +++ b/tests/ut/cpp/transform/graph_runner_test.cc @@ -0,0 +1,240 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "common/common_test.h" +#include "ir/dtype.h" +#include "transform/transform_base_test.h" +#include "common/py_func_graph_fetcher.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "operator/ops.h" +#include "transform/df_graph_manager.h" +#include "transform/convert.h" +#include "utils/utils.h" + +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif + +#define private public +#include "transform/graph_runner.h" + +namespace mindspore { +namespace transform { +class TestGraphRunner : public UT::Common { + public: + TestGraphRunner() {} + void SetUp(); + static const std::shared_ptr kF64; + static const std::shared_ptr kF32; + + private: +}; + +void TestGraphRunner::SetUp() { UT::InitPythonPath(); } +const std::shared_ptr TestGraphRunner::kF64 = std::make_shared(64); +const std::shared_ptr TestGraphRunner::kF32 = std::make_shared(32); + +std::shared_ptr MakeGeGraph() { + PrimitivePtr conv2d = prim::kPrimConv2D; + conv2d->AddAttr("stride", MakeValue(1)); + conv2d->AddAttr("pad", MakeValue(0)); + conv2d->AddAttr("pad_mode", MakeValue(std::string("pad"))); + conv2d->AddAttr("dilation", MakeValue(1)); + conv2d->AddAttr("group", MakeValue(1)); + conv2d->AddAttr("mode", MakeValue(1)); + conv2d->AddAttr("out_channel", MakeValue(2)); + conv2d->AddAttr("kernel_size", MakeValue(std::vector({2, 2}))); + conv2d->AddAttr("dilation", MakeValue(1)); + conv2d->AddAttr("data_format", MakeValue(kOpFormat_NCHW)); + + FuncGraphPtr anf_graph = MakeFuncGraph(conv2d, 2); + std::shared_ptr ir_graph_manager = MakeManager({anf_graph}); + + return std::make_shared(anf_graph); +} +namespace { +std::shared_ptr> DoExecGraph(const std::vector& inputs) { + std::vector ge_tensor_ptrs = TransformUtil::ConvertInputTensors(inputs, kOpFormat_NCHW); + + std::vector ge_outputs; + transform::GraphRunnerOptions options; + transform::GraphRunner graph_runner(options); + transform::RunOptions run_options; + run_options.name = "fp_bp_subgraph"; + + MS_LOG(INFO) << "Run func_graph begin, inputs size is: " << inputs.size(); + Status ret = graph_runner.RunGraph(run_options, ge_tensor_ptrs, &ge_outputs); + MS_LOG(INFO) << "Run func_graph finish, outputs size is: " << ge_outputs.size(); + if (ret != Status::SUCCESS) { + return nullptr; + } + + std::vector> request_dims; + std::vector dims1 = {1, 1, 4, 4}; + std::vector dims2 = {2, 3, 4, 5}; + std::vector dims3 = {9, 9}; + request_dims.emplace_back(dims1); + request_dims.emplace_back(dims2); + request_dims.emplace_back(dims3); + + std::vector me_outputs = TransformUtil::ConvertGeTensors(ge_outputs, request_dims); + + return std::make_shared>(me_outputs); +} + +} // namespace + +TEST_F(TestGraphRunner, TestGeTensorConstructor) { + // Init a data buffer + float ge_tensor_data[] = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6}; + + // Create a Tensor with wanted data type and shape + MeTensor tensor = MeTensor(TypeId::kNumberTypeFloat32, std::vector({1, 2, 3})); + + // Get the writable data pointer from the tensor + float* me_tensor_data = reinterpret_cast(tensor.data_c(true)); + + // Copy data from buffer to tensor's data + memcpy_s(me_tensor_data, static_cast(tensor.data().nbytes()), ge_tensor_data, sizeof(ge_tensor_data)); + PrintMeTensor(&tensor); + + std::cout << "----------------------------------" << std::endl; + py::tuple py_tuple = + py::make_tuple(py::make_tuple(py::make_tuple(1.1f, 2.2f, 3.3f), py::make_tuple(4.4f, 5.5f, 6.6f))); + py::array my_arry = py::array(py_tuple).attr("astype").cast()("float32").cast(); + MeTensor tensor_tuple = MeTensor(my_arry, kFloat32); + PrintMeTensor(&tensor_tuple); + + py::array tensor_array = tensor.data(); + py::array tensor_tuple_array = tensor_tuple.data(); + assert(memcmp(ge_tensor_data, tensor_array.data(), sizeof(ge_tensor_data)) == 0); + assert(memcmp(ge_tensor_data, tensor_tuple_array.data(), sizeof(ge_tensor_data)) == 0); +} + +#if (!defined ENABLE_GE) + +TEST_F(TestGraphRunner, TestRunGraphException) { + DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + graph_manager.ClearGraph(); + + std::map dict; + MeTensorPtr init_tensor_ptr = MakeTensor(kF32, {2, 1, 2, 2}); + dict["x1"] = init_tensor_ptr; + + std::shared_ptr convertor = MakeGeGraph(); + (*convertor).ConvertAllNode().InitParam(dict).BuildGraph(); + auto df_graph = (*convertor).GetComputeGraph(); + + graph_manager.AddGraph("test_graph", df_graph); + MeTensorPtr me_tensor_ptr = MakeTensor(kF32, {1, 1, 2, 3}); + + MeTensorPtr input_ptr = MakeTensor(kF32, {1, 1, 4, 4}); + std::vector me_inputs; + me_inputs.emplace_back(input_ptr); + std::vector me_outputs; + + GraphRunnerOptions options; + GraphRunner graph_runner(options); + RunOptions run_options; + ASSERT_TRUE(graph_runner.RunGraph(run_options, me_inputs, &me_outputs) != Status::SUCCESS); + run_options.name = "test_graph"; + ASSERT_TRUE(graph_runner.RunGraph(run_options, me_inputs, &me_outputs) == Status::SUCCESS); + + GraphRunner graph_runner2(options); + ASSERT_TRUE(graph_runner2.RunGraph(run_options, me_inputs, &me_outputs) == Status::SUCCESS); + + // when the GraphManager is empty + graph_manager.ClearGraph(); + GraphRunner graph_runner3(options); + ASSERT_TRUE(graph_runner3.RunGraph(run_options, me_inputs, &me_outputs) != Status::SUCCESS); +} + +TEST_F(TestGraphRunner, TestRunGraph) { + DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + graph_manager.ClearGraph(); + + std::shared_ptr convertor = MakeGeGraph(); + std::map dict; + dict.emplace("x1", MakeTensor(kF32, {2, 1, 2, 2})); + + (*convertor).ConvertAllNode().InitParam(dict).BuildGraph(); + graph_manager.AddGraph("test_graph", (*convertor).GetComputeGraph()); + + TypePtr type_id = kFloat32; + + py::tuple tuple = py::make_tuple( + py::make_tuple(py::make_tuple(py::make_tuple(1.0, 2.0, 3.0, 4.0), py::make_tuple(4.0, 5.0, 6.0, 7.0))), + py::make_tuple(py::make_tuple(py::make_tuple(1.0, 2.0, 3.0, 4.0), py::make_tuple(4.0, 5.0, 6.0, 7.0)))); + py::array array = py::array(tuple); + MeTensorPtr me_tensor_ptr = std::make_shared(array, type_id); + + MS_LOG(INFO) << "inputs me tensor data is: "; + PrintMeTensor(&(*me_tensor_ptr)); + + std::vector me_inputs; + me_inputs.emplace_back(me_tensor_ptr); + std::vector me_outputs; + + GraphRunnerOptions options; + GraphRunner graph_runner(options); + RunOptions run_options; + run_options.name = "test_graph"; + ASSERT_TRUE(graph_runner.RunGraph(run_options, me_inputs, &me_outputs) == Status::SUCCESS); + MS_LOG(INFO) << "outputs me tensor data is: "; + for (auto i = 0; i < me_outputs.size(); i++) { + PrintMeTensor(&(*me_outputs[i])); + } +} + +TEST_F(TestGraphRunner, TestAPI) { + DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + graph_manager.ClearGraph(); + + std::shared_ptr convertor = MakeGeGraph(); + std::map dict; + dict.emplace("x1", MakeTensor(kF32, {2, 1, 2, 2})); + + (*convertor).ConvertAllNode().InitParam(dict).BuildGraph(); + (*convertor).DrawComputeGraph("TestGraphRunner_TestAPI_Training.dot"); + graph_manager.AddGraph("fp_bp_subgraph", (*convertor).GetComputeGraph()); + + MeTensorPtr input_ptr1 = MakeTensor(kF32, {1, 1, 4, 4}); + MeTensorPtr input_ptr2 = MakeTensor(kF32, {2, 3, 4, 5}); + MeTensorPtr input_ptr3 = MakeTensor(kF32, {9, 9, 1, 1}); + std::vector me_inputs; + std::vector me_outputs; + me_inputs.emplace_back(input_ptr1); + me_inputs.emplace_back(input_ptr2); + me_inputs.emplace_back(input_ptr3); + + auto ret = DoExecGraph(me_inputs); + + ASSERT_TRUE(ret != nullptr); + + me_outputs = *ret; + MS_LOG(INFO) << "outputs me tensor data is: "; + for (auto tensor : me_outputs) { + PrintMeTensor(&(*tensor)); + } +} +#endif + +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/op_adapter_test.cc b/tests/ut/cpp/transform/op_adapter_test.cc new file mode 100644 index 00000000000..254452bb428 --- /dev/null +++ b/tests/ut/cpp/transform/op_adapter_test.cc @@ -0,0 +1,102 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "common/common_test.h" + +#include "transform/op_declare.h" + +#include "operator/ops.h" +#include "./common.h" + +using std::cout; +using std::endl; +using std::string; +using std::unordered_map; + +namespace mindspore { +namespace transform { +class TestOpAdapter : public UT::Common { + public: + TestOpAdapter() {} +}; + +#if (!defined ENABLE_GE) +#if 0 +// fix conv2d ut +TEST_F(TestOpAdapter, TestSpecilization_Conv2D) { + BaseOpAdapter *adpt = new OpAdapter(); + + auto input = std::make_shared(); + auto conv = std::make_shared(); + + ASSERT_EQ(adpt->setInput(conv, 1, input), 0); + ASSERT_EQ(adpt->setInput(conv, 2, input), 0); + ASSERT_EQ(adpt->setInput(conv, 3, input), NOT_FOUND); + + ASSERT_EQ(0, adpt->setAttr(conv, "group", 1)); + ASSERT_EQ(0, adpt->setAttr(conv, "mode", 1)); + + delete adpt; +} +#endif +TEST_F(TestOpAdapter, TestSpecilization_Const) { + BaseOpAdapter *adpt = new OpAdapter(); + auto valuenode = std::make_shared(); + auto input = std::make_shared(); + + ASSERT_EQ(adpt->setInput(valuenode, 1, input), NOT_FOUND); + delete adpt; +} +#if 0 +// fix conv2d ut +TEST_F(TestOpAdapter, TestSetAttr_Conv2d_Primitive) { + BaseOpAdapter *adpt = new OpAdapter(); + auto conv = std::make_shared(); + + ASSERT_EQ(adpt->setAttr(conv, "padding", 1), NOT_FOUND); + ASSERT_EQ(adpt->setAttr(conv, "pad", 1), 0); + ASSERT_EQ(adpt->setAttr(conv, "pad_mode", string("same")), 0); + ASSERT_EQ(adpt->setAttr(conv, "nothing", "test"), NOT_FOUND); + + const unordered_map attrs = { + {"padding", MakeValue(2)}, + {"padding_mode", MakeValue(string("normal"))}, + {"stride", MakeValue(8)} + }; + + auto prim = prim::kPrimConv2D; + prim->SetAttrs({ + {"strides", MakeValue(3)}, + {"padding", MakeValue(1)}, + }); + ASSERT_EQ(prim->name(), prim::kPrimConv2D->name()); + + Int32Imm strides(3); + Int32Imm padding(1); + ASSERT_EQ(*(prim->GetAttr("strides")), strides); + ASSERT_EQ(*(prim->GetAttr("padding")), padding); + + ASSERT_EQ(adpt->setAttr(conv, prim), 0); + + delete adpt; +} +#endif +#endif +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/transform_base_test.cc b/tests/ut/cpp/transform/transform_base_test.cc new file mode 100644 index 00000000000..944721ec837 --- /dev/null +++ b/tests/ut/cpp/transform/transform_base_test.cc @@ -0,0 +1,102 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/common_test.h" +#include "transform/transform_base_test.h" + +namespace mindspore { +namespace transform { +using mindspore::parse::ParsePythonCode; +namespace python_adapter = mindspore::parse::python_adapter; +using mindspore::parse::ResolveAll; +std::vector getAnfGraph(string package, string function) { + py::function fn_ = python_adapter::GetPyFn(package, function); + FuncGraphPtr func_graph = ParsePythonCode(fn_); + std::vector graphVector; + graphVector.clear(); + if (nullptr == func_graph) return graphVector; + + // save the func_graph to manager + std::shared_ptr manager = Manage(func_graph); + + // call resolve + bool ret_ = ResolveAll(manager); + + if (!ret_) return graphVector; + + // get graph + for (auto func_graph : manager->func_graphs()) { + graphVector.push_back(func_graph); + } + return graphVector; +} + +void PrintMeTensor(MeTensor* tensor) { +#define DTYPE float + DTYPE* me_data = reinterpret_cast((*tensor).data_c()); + size_t elements = (*tensor).ElementsNum(); + std::cout << "the in memory block data size is: " << std::dec << tensor->data().nbytes() << " bytes" << std::endl; + std::cout << "the in memory block data is: " << std::endl; + for (int i = 0; i < elements; i++) { + std::cout << static_cast(*(me_data + i)) << std::endl; + } + + std::cout << "the py::str() data is: " << std::endl; + py::array tensor_data = (*tensor).data(); + std::cout << std::string(py::str(tensor_data)) << std::endl; + + std::cout << "tensor dtype is: " << std::string(tensor->data().dtype().str()) << std::endl; +} + +FuncGraphPtr MakeFuncGraph(const PrimitivePtr prim, unsigned int nparam) { + // build the func_graph manually, eg: + // MakeFuncGraph(std::make_shared("scalar_add"), 2) means: + /* python source code: + * @mindspore + * def f(x, y): + * return x + y + */ + FuncGraphPtr func_graph = std::make_shared(); + std::vector inputs; + inputs.push_back(NewValueNode(prim)); + for (unsigned int i = 0; i < nparam; i++) { + if ((prim->name() == "ScalarSummary" || prim->name() == "TensorSummary" || prim->name() == "ImageSummary") && + i == 0) { + auto input = NewValueNode("testSummary"); + inputs.push_back(input); + } else { + auto input = func_graph->add_parameter(); + input->set_name("x" + std::to_string(i)); + inputs.push_back(input); + } + } + CNodePtr cnode_prim = func_graph->NewCNode(inputs); + inputs.clear(); + inputs.push_back(NewValueNode(std::make_shared("return"))); + inputs.push_back(cnode_prim); + CNodePtr cnode_return = func_graph->NewCNode(inputs); + func_graph->set_return(cnode_return); + return func_graph; +} + +MeTensorPtr MakeTensor(const TypePtr& t, std::initializer_list shp) { + auto shape = std::vector(shp); + auto tensor = std::make_shared(t->type_id(), shape); + return tensor; +} + +} // namespace transform +} // namespace mindspore diff --git a/tests/ut/cpp/transform/transform_base_test.h b/tests/ut/cpp/transform/transform_base_test.h new file mode 100644 index 00000000000..c1728ac79bc --- /dev/null +++ b/tests/ut/cpp/transform/transform_base_test.h @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_UT_TRANSFORM_UT_TRANSFORM_BASE_H_ +#define TESTS_UT_TRANSFORM_UT_TRANSFORM_BASE_H_ + +#include +#include +#include +#include +#include "transform/util.h" +#include "ir/meta_tensor.h" + +#include "common/common_test.h" +#include "pipeline/parse/parse.h" +#include "./common.h" + +#include "graph/tensor.h" +#ifdef OPEN_SOURCE +#include "ge/client/ge_api.h" +#else +#include "external/ge/ge_api.h" +#endif + +namespace mindspore { +namespace transform { +std::vector getAnfGraph(std::string package, std::string function); +void PrintMeTensor(MeTensor* tensor); +FuncGraphPtr MakeFuncGraph(const PrimitivePtr prim, unsigned int nparam); +MeTensorPtr MakeTensor(const TypePtr& t, std::initializer_list shp); +} // namespace transform +} // namespace mindspore + +#endif // TESTS_UT_TRANSFORM_UT_TRANSFORM_BASE_H_ diff --git a/tests/ut/cpp/utils/any_test.cc b/tests/ut/cpp/utils/any_test.cc new file mode 100644 index 00000000000..d11831d6028 --- /dev/null +++ b/tests/ut/cpp/utils/any_test.cc @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +#include "common/common_test.h" +#include "operator/ops.h" +#include "utils/any.h" +#include "utils/misc.h" + +using std::cout; +using std::endl; +using std::string; + +namespace mindspore { + +class TestAny : public UT::Common { + public: + TestAny() {} +}; + +using Primitive = Primitive; +using PrimitivePtr = PrimitivePtr; + +Any f(const Any &a) { + Any value = a; + return value; +} + +TEST_F(TestAny, test_common) { + Any value(std::make_shared("add")); + if (value.type() == typeid(PrimitivePtr)) { + PrimitivePtr a = value.cast(); + } + if (value.is()) { + PrimitivePtr a = value.cast(); + } + + Any value1 = f(std::make_shared("add")); + + Any a = 1; + Any b = string("hello, world"); + Any c; + + ASSERT_FALSE(a.empty()); + ASSERT_FALSE(b.empty()); + ASSERT_TRUE(c.empty()); + + ASSERT_TRUE(a.is()); + ASSERT_FALSE(a.is()); + ASSERT_EQ(1, a.cast()); + c = a; + + ASSERT_TRUE(c.is()); + ASSERT_FALSE(c.is()); + ASSERT_EQ(1, c.cast()); +} + +TEST_F(TestAny, test_unordered_map) { + std::unordered_map ids; + Any a = 1; + ids[a] = 2; + ASSERT_EQ(2, ids[a]); +} + +TEST_F(TestAny, test_unordered_map1) { + std::unordered_map ids; + ids[1] = 1; + ASSERT_EQ(1, ids[1].cast()); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/baseref_test.cc b/tests/ut/cpp/utils/baseref_test.cc new file mode 100644 index 00000000000..4e1556d8199 --- /dev/null +++ b/tests/ut/cpp/utils/baseref_test.cc @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "common/common_test.h" + +#include "ir/anf.h" +#include "utils/base_ref.h" + +namespace mindspore { +namespace utils { +class TestBaseRef : public UT::Common { + public: + TestBaseRef() {} + virtual void SetUp() {} + virtual void TearDown() {} +}; + +TEST_F(TestBaseRef, TestScalar) { + BaseRef a = 1; + BaseRef b = 1.0; + if (isa(a)) { + ASSERT_EQ(cast(a), 1); + Int32ImmPtr c = cast(a); + ASSERT_EQ(cast(c), 1); + } + ASSERT_TRUE(isa(a)); + ASSERT_TRUE(isa(a)); + ASSERT_TRUE(isa(b)); + ASSERT_TRUE(isa(b)); + BaseRef c = 1; + ASSERT_EQ(a == c, true); +} + +void func(const BaseRef& sexp) { + if (isa(sexp)) { + const VectorRef& a = cast(sexp); + for (size_t i = 0; i < a.size(); i++) { + BaseRef v = a[i]; + MS_LOG(INFO) << "for is i:" << i << ", " << v.ToString() << "\n"; + } + MS_LOG(INFO) << "in func is valuesequeue:" << sexp.ToString() << "\n"; + } +} + +TEST_F(TestBaseRef, TestNode) { + AnfNodePtr anf = NewValueNode(1); + BaseRef d = anf; + MS_LOG(INFO) << "anf typeid:" << dyn_cast(anf).get(); + MS_LOG(INFO) << "anf typeid:" << NewValueNode(1)->tid(); + MS_LOG(INFO) << "node reftypeid:" << d.tid(); + ASSERT_EQ(isa(d), true); + ASSERT_EQ(isa(d), true); + ASSERT_EQ(isa(d), true); + AnfNodePtr c = cast(d); + ASSERT_NE(c, nullptr); +} + +TEST_F(TestBaseRef, TestVector) { + AnfNodePtr anf = NewValueNode(1); + VectorRef a({1, 2, anf, NewValueNode(1)}); + ASSERT_TRUE(isa(a)); + func(a); + BaseRef b; + b = 1; + ASSERT_TRUE(isa(b)); + std::vector int_t({1, 2, 3}); + VectorRef k; + k.insert(k.end(), int_t.begin(), int_t.end()); + + k = a; + func(k); + + BaseRef c = std::make_shared(a); + BaseRef c1 = std::make_shared(a); + ASSERT_TRUE(c == c1); + + ASSERT_TRUE(isa(c)); + VectorRefPtr d = cast(c); + ASSERT_TRUE(isa(d)); + VectorRef e1({1, 2, anf}); + VectorRef e({1, 2, anf}); + ASSERT_EQ(e1 == e, true); +} +} // namespace utils +} // namespace mindspore diff --git a/tests/ut/cpp/utils/callback_test.cc b/tests/ut/cpp/utils/callback_test.cc new file mode 100644 index 00000000000..758e99ff59e --- /dev/null +++ b/tests/ut/cpp/utils/callback_test.cc @@ -0,0 +1,98 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "pybind11/pybind11.h" +#include "utils/callbacks.h" +#include "common/common_test.h" +#include "pipeline/pipeline.h" +#include "pipeline/parse/python_adapter.h" +#include "transform/df_graph_manager.h" +#include "debug/draw.h" + +namespace mindspore { +namespace python_adapter = mindspore::parse::python_adapter; + +class TestCallback : public UT::Common { + public: + TestCallback() {} +}; + +/* + * # ut and python static info not share +TEST_F(TestCallback, test_get_anf_tensor_shape) { + py::object obj = python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_class", "test_get_object_graph"); + FuncGraphPtr func_graph = pipeline::ExecutorPy::GetInstance()->GetFuncGraphPy(obj); + transform::DfGraphManager::GetInstance().SetAnfGraph(func_graph); + std::shared_ptr> param_shape_ptr = std::make_shared>(); + bool get_shape = callbacks::GetParameterShape(func_graph, "weight", param_shape_ptr); + ASSERT_TRUE(get_shape == true); +} + +TEST_F(TestCallback, test_checkpoint_save_op) { + py::object obj = python_adapter::CallPyFn("gtest_input.pipeline.parse.parse_class", "test_get_object_graph"); + FuncGraphPtr func_graph = pipeline::ExecutorPy::GetInstance()->GetFuncGraphPy(obj); + transform::DfGraphManager::GetInstance().SetAnfGraph(func_graph); + +#define DTYPE float + ge::DataType dt = ge::DataType::DT_FLOAT; + + std::vector data1 = {1.1, 2.2, 3.3, 4.4, 6.6, 7.7, 8.8, 9.9}; + auto data = data1; + ge::Shape shape({2, 2, 2, 1}); + ge::Format format = ge::Format::FORMAT_NCHW; + ge::TensorDesc desc(shape, format, dt); + transform::GeTensorPtr ge_tensor_ptr = + std::make_shared(desc, reinterpret_cast(data.data()), data.size() * sizeof(DTYPE)); + std::map param_map; + param_map.insert(std::pair("weight", *ge_tensor_ptr)); + param_map.insert(std::pair("network.weight", *ge_tensor_ptr)); + int ret = callbacks::CheckpointSaveCallback(0, param_map); +MS_LOG(INFO) << "ret=" << ret; + ASSERT_EQ(ret, 0); +} +*/ + +/* +TEST_F(TestCallback, test_summary_save_op) { + py::object obj = python_adapter::CallPyFn( + "gtest_input.pipeline.parse.parse_class", "test_get_object_graph"); + FuncGraphPtr func_graph = obj.cast(); + transform::DfGraphManager::GetInstance().SetAnfGraph(func_graph); + + #define DTYPE float + ge::DataType dt = ge::DataType::DT_FLOAT; + + float data1 = 1.1; + float data2 = 2.1; + ge::Shape shape({1, 1, 1, 1}); + ge::Format format = ge::Format::FORMAT_NCHW; + ge::TensorDesc desc(shape, format, dt); + GeTensorPtr ge_tensor_ptr1 = std::make_shared(desc, + reinterpret_cast(&data1), + sizeof(DTYPE)); + GeTensorPtr ge_tensor_ptr2 = std::make_shared(desc, + reinterpret_cast(&data2), + sizeof(DTYPE)); + std::map param_map; + param_map.insert(std::pair("x1[:Scalar]", *ge_tensor_ptr1)); + param_map.insert(std::pair("x2[:Scalar]", *ge_tensor_ptr2)); + int ret = callbacks::SummarySaveCallback(0, param_map); +MS_LOG(INFO) << "ret=" << ret; + ASSERT_TRUE(ret == 0); +} +*/ +} // namespace mindspore diff --git a/tests/ut/cpp/utils/comm_manager_test.cc b/tests/ut/cpp/utils/comm_manager_test.cc new file mode 100644 index 00000000000..a8f5eb27d44 --- /dev/null +++ b/tests/ut/cpp/utils/comm_manager_test.cc @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "utils/comm_manager.h" +#include "common/common_test.h" + +namespace mindspore { +class TestCommManager : public UT::Common { + public: + TestCommManager() {} +}; + +TEST_F(TestCommManager, TestCreate) { + std::vector devices{0, 1, 2}; + ASSERT_TRUE(CommManager::GetInstance().CreateGroupSync(string("1-2-3"), devices)); + ASSERT_TRUE(CommManager::GetInstance().CreateGroupSync(string("hccl_world_group"), devices)); +} + +TEST_F(TestCommManager, TestGetSize) { + unsigned int rank_size = 0; + ASSERT_TRUE(CommManager::GetInstance().GetRankSize(string("1-2-3"), &rank_size)); + ASSERT_TRUE(CommManager::GetInstance().GetRankSize(string("hccl_world_group"), &rank_size)); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/config_manager_test.cc b/tests/ut/cpp/utils/config_manager_test.cc new file mode 100644 index 00000000000..6928ec0baaf --- /dev/null +++ b/tests/ut/cpp/utils/config_manager_test.cc @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "utils/config_manager.h" + +namespace mindspore { +class TestConfigManager : public UT::Common { + public: + TestConfigManager() {} +}; + +TEST_F(TestConfigManager, TestAPI) { + ASSERT_TRUE(ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::ONE_DEVICE); + + ConfigManager::GetInstance().set_parallel_strategy(ParallelStrategy::DISTRIBUTION); + ASSERT_TRUE(ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::DISTRIBUTION); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/counter_test.cc b/tests/ut/cpp/utils/counter_test.cc new file mode 100644 index 00000000000..5a0d8c2d6a8 --- /dev/null +++ b/tests/ut/cpp/utils/counter_test.cc @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "utils/counter.h" +#include "common/common_test.h" + +namespace mindspore { +class TestCounter : public UT::Common { + public: + TestCounter() { + std::string s1 = "abcdeedfrgbhrtfsfd"; + std::string s2 = "shceufhvogawrycawr"; + + for (auto c : s1) { + std::string key(1, c); + counter_a[key] += 1; + } + + for (auto c : s2) { + std::string key(1, c); + counter_b[key] += 1; + } + } + + public: + Counter counter_a; + Counter counter_b; +}; + +TEST_F(TestCounter, test_constructor) { + assert(counter_a.size() == 11); + assert(counter_b.size() == 13); +} + +TEST_F(TestCounter, test_subtitle) { + std::string s = "d"; + assert(counter_a[s] == 3); + s = "f"; + assert(counter_a[s] == 3); + s = "h"; + assert(counter_b[s] = 2); + s = "c"; + assert(counter_b[s] = 2); +} + +TEST_F(TestCounter, test_contains) { + std::string s = "d"; + assert(counter_a.contains(s) == true); + s = "z"; + assert(counter_a.contains(s) == false); + s = "q"; + assert(counter_b.contains(s) == false); +} + +TEST_F(TestCounter, test_add) { + auto counter_add = counter_a + counter_b; + assert(counter_add.size() == 16); + std::string s = "f"; + assert(counter_add[s] == 4); + s = "r"; + assert(counter_add[s] == 4); + s = "y"; + assert(counter_add[s] == 1); +} + +TEST_F(TestCounter, test_minus) { + auto counter_minus = counter_a - counter_b; + assert(counter_minus.size() == 5); + std::string s = "d"; + assert(counter_minus[s] == 3); + s = "t"; + assert(counter_minus[s] == 1); + s = "a"; + assert(counter_minus.contains(s) == false); +} +} // namespace mindspore diff --git a/tests/ut/cpp/utils/graph_utils_test.cc b/tests/ut/cpp/utils/graph_utils_test.cc new file mode 100644 index 00000000000..ce5a4318d3a --- /dev/null +++ b/tests/ut/cpp/utils/graph_utils_test.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" + +#include "ir/anf.h" +#include "utils/graph_utils.h" + +#include "pipeline/parse/parse_base.h" +#include "pipeline/parse/parse.h" + +namespace mindspore { + +class TestGraphUtils : public UT::Common { + public: + TestGraphUtils() : getPyFun("gtest_input.utils.graph_utils_test", true), equiv_graph(), equiv_node() {} + std::shared_ptr GetPythonFunction(std::string function); + + public: + UT::PyFuncGraphFetcher getPyFun; + + FuncGraphPairMapEquiv equiv_graph; + NodeMapEquiv equiv_node; +}; + +TEST_F(TestGraphUtils, Isomorphic) { + std::shared_ptr g1 = getPyFun("test_graph_utils_isomorphic_1"); + std::shared_ptr g2 = getPyFun("test_graph_utils_isomorphic_2"); + std::shared_ptr g3 = getPyFun("test_graph_utils_isomorphic_3"); + ASSERT_TRUE(nullptr != g1); + ASSERT_TRUE(nullptr != g2); + ASSERT_TRUE(Isomorphic(g1, g2, &equiv_graph, &equiv_node)); + + ASSERT_TRUE(nullptr != g3); + ASSERT_FALSE(Isomorphic(g1, g3, &equiv_graph, &equiv_node)); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/ir_import_test.cc b/tests/ut/cpp/utils/ir_import_test.cc new file mode 100644 index 00000000000..5e7db98a38d --- /dev/null +++ b/tests/ut/cpp/utils/ir_import_test.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" + +#include "utils/log_adapter.h" +#include "debug/anf_ir_utils.h" +#include "pipeline/parse/parse.h" +#include "ir/manager.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" + +namespace mindspore { +class TestIrImporter : public UT::Common { + public: + TestIrImporter() {} + virtual ~TestIrImporter() {} + + virtual void TearDown() {} +}; +} // namespace mindspore diff --git a/tests/ut/cpp/utils/lenet_specialize.txt b/tests/ut/cpp/utils/lenet_specialize.txt new file mode 100644 index 00000000000..ceefdea0081 --- /dev/null +++ b/tests/ut/cpp/utils/lenet_specialize.txt @@ -0,0 +1,50 @@ +# [No.1] 1_g_293.146 +graph g_146( + %para1 : Array(F32)[1, 1, 32, 32] # %para1@3 + , %para2 : Array(F32)[6, 1, 5, 5] # %para2@4 + , %para3 : Array(F32)[16, 6, 5, 5] # %para3@5 + , %para4 : Array(F32)[120, 400] # %para4@6 + , %para5 : Array(F32)[120] # %para5@7 + , %para6 : Array(F32)[84, 120] # %para6@8 + , %para7 : Array(F32)[84] # %para7@9 + , %para8 : Array(F32)[10, 84] # %para8@10 + , %para9 : Array(F32)[10] # %para9@11 + , %para10 : Array(F32)[6, 1, 5, 5] # %para10@12 + , %para11 : TypeType # %para11@13 + , %para12 : AnythingType # %para12@14 + , %para13 : ExternalType # %para13@15 + , %para14 : ProblemType # %para14@16 + , %para15 : NoneType # %para15@17 + , %para16 : EnvType # %para16@18 + , %para17 : Array(F32)[10] # %para17@19 + , %para18 : F32 # %para18@20 + , %para19 : JT[I32] # %para19@21 + , %para20 : I32 # %para20@22 + ) { + %1 : Tuple[Array(F32)*8] = Primitive::make_tuple(%para2, %para3, %para4, %para5, %para6, %para7, %para8, %para9) #(Array(F32)[6, 1, 5, 5], Array(F32)[16, 6, 5, 5], Array(F32)[120, 400], Array(F32)[120], Array(F32)[84, 120], Array(F32)[84], Array(F32)[10, 84], Array(F32)[10]) + %2 : Func = Graph::g_196(Graph::g_197, %1, (SymInst(%para2), SymInst(%para3), SymInst(%para4), SymInst(%para5), SymInst(%para6), SymInst(%para7), SymInst(%para8), SymInst(%para9))) #(Func, Tuple[Array(F32)*8], Tuple[SymType*8]) # g_196=12_g_343.196, g_197=23_g_344.197 + %3 : Tuple[Array(F32)*8] = %2(%para1, I32(10), "NCHW") #(Array(F32)[1, 1, 32, 32], Array(F32)[1, 10]) + %4 : Tuple[Bool*8] = Graph::g_207(%3) #(Tuple[Array(F32)*8]) # g_207=2_g_354.207 + Primitive::return(%4) #(Tuple[Bool*8]) +} + + +# [No.2] 23_g_344.197 +graph g_197[g_146]( + %para21 : Array(F32)[1, 1, 32, 32] # %para21@34 + ) { + %1 : Array(F32)[1, 6, 28, 28] = Graph::g_149(%para21) #(Array(F32)[1, 1, 32, 32]) # g_149=60_g_296.149 + %2 : Array(F32)[1, 6, 28, 28] = Graph::g_150(%1) #(Array(F32)[1, 6, 28, 28]) # g_150=59_g_297.150 + %3 : Array(F32)[1, 6, 14, 14] = Graph::g_154(%2) #(Array(F32)[1, 6, 28, 28]) # g_154=55_g_301.154 + %4 : Array(F32)[1, 16, 10, 10] = Graph::g_157(%3) #(Array(F32)[1, 6, 14, 14]) # g_157=52_g_304.157 + %5 : Array(F32)[1, 16, 10, 10] = Graph::g_158(%4) #(Array(F32)[1, 16, 10, 10]) # g_158=51_g_305.158 + %6 : Array(F32)[1, 16, 5, 5] = Graph::g_162(%5) #(Array(F32)[1, 16, 10, 10]) # g_162=47_g_309.162 + %7 : SymType = Graph::g_169(%6) #(Array(F32)[1, 16, 5, 5]) + #%8 : Array(F32)[1, 120] = Graph::g_169(%7) #(Array(F32)[1, 400]) # g_169=40_g_316.169 + #%9 : Array(F32)[1, 120] = Graph::g_170(%8) #(Array(F32)[1, 120]) # g_170=39_g_317.170 + #%10 : Array(F32)[1, 84] = Graph::g_177(%9) #(Array(F32)[1, 120]) # g_177=32_g_324.177 + #%11 : Array(F32)[1, 84] = Graph::g_178(%10) #(Array(F32)[1, 84]) # g_178=31_g_325.178 + #%12 : Array(F32)[1, 10] = Graph::g_185(%11) #(Array(F32)[1, 84]) # g_185=24_g_332.185 + Primitive::return(%7) #(Array(F32)[1, 10]) +} + diff --git a/tests/ut/cpp/utils/orderedset_test.cc b/tests/ut/cpp/utils/orderedset_test.cc new file mode 100644 index 00000000000..d2cb570e6a4 --- /dev/null +++ b/tests/ut/cpp/utils/orderedset_test.cc @@ -0,0 +1,150 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "utils/ordered_set.h" +#include "common/common_test.h" + +using std::cout; +using std::endl; +using std::string; + +namespace mindspore { + +class TestOrderedSet : public UT::Common { + public: + TestOrderedSet() { + std::shared_ptr e; + for (int i = 1; i <= 10; i++) { + e = std::make_shared(i); + osa.add(e); + } + for (int i = 11; i <= 20; i++) { + e = std::make_shared(i); + osa.add(e); + osb.add(e); + } + for (int i = 21; i <= 30; i++) { + e = std::make_shared(i); + osb.add(e); + osc.add(e); + } + } + + public: + OrderedSet> osa; + OrderedSet> osb; + OrderedSet> osc; +}; + +TEST_F(TestOrderedSet, test_constructor) { + OrderedSet> osa_copy = osa; + ASSERT_EQ(osa_copy.size(), osa.size()); + + std::shared_ptr e = std::make_shared(1); + OrderedSet> se; + se.add(std::make_shared(10)); + se.add(std::make_shared(20)); + OrderedSet> order_se(se); + ASSERT_EQ(order_se.size(), 2); +} + +TEST_F(TestOrderedSet, test_add_remove_clear) { + OrderedSet> res; + res.add(std::make_shared(1)); + std::shared_ptr e = std::make_shared(2); + std::shared_ptr e2 = std::make_shared(10); + res.add(e); + ASSERT_EQ(res.size(), 2); + ASSERT_EQ(res.count(e), 1); + auto elem = res.back(); + ASSERT_EQ(elem, e); + res.erase(e); + ASSERT_EQ(res.size(), 1); + res.clear(); + ASSERT_EQ(res.size(), 0); +} + +TEST_F(TestOrderedSet, test_add_remove_first) { + OrderedSet a; + a.add(1); + a.add(2); + a.add(3); + a.erase(1); + auto first = a.pop(); + // 1 removed, 2 3 followd, 2 should be the poped one, remaining size = 1 + ASSERT_EQ(first, 2); + ASSERT_EQ(a.size(), 1); +} + +TEST_F(TestOrderedSet, test_compare) { + OrderedSet> c1; + OrderedSet> c2; + std::shared_ptr e1 = std::make_shared(10); + std::shared_ptr e2 = std::make_shared(20); + c1.add(e1); + c1.add(e2); + c2.add(e1); + c2.add(e2); + ASSERT_EQ(c1, c2); +} + +TEST_F(TestOrderedSet, test_pop) { + OrderedSet> oset; + oset.add(std::make_shared(10)); + oset.add(std::make_shared(20)); + oset.add(std::make_shared(30)); + std::shared_ptr ele = oset.pop(); + int pop_size = 0; + pop_size++; + while (oset.size() != 0) { + ele = oset.pop(); + pop_size++; + } + ASSERT_EQ(pop_size, 3); + ASSERT_EQ(oset.size(), 0); +} + +TEST_F(TestOrderedSet, test_operation) { + ASSERT_TRUE(osc.is_disjoint(osa)); + ASSERT_TRUE(!osb.is_disjoint(osa)); + + ASSERT_TRUE(osc.is_subset(osb)); + ASSERT_TRUE(!osc.is_subset(osa)); + + OrderedSet> res_inter = osa | osb; + ASSERT_EQ(res_inter.size(), 30); + OrderedSet> res_union = osa & osb; + ASSERT_EQ(res_union.size(), 10); + OrderedSet> res_diff = osa - osb; + ASSERT_EQ(res_diff.size(), 10); + OrderedSet> res_symdiff = osa ^ osb; + ASSERT_EQ(res_symdiff.size(), 20); +} + +TEST_F(TestOrderedSet, test_contains) { + OrderedSet> res; + std::shared_ptr e1 = std::make_shared(10); + std::shared_ptr e2 = std::make_shared(20); + res.add(e1); + ASSERT_TRUE(res.contains(e1)); + ASSERT_TRUE(!res.contains(e2)); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/profile_test.cc b/tests/ut/cpp/utils/profile_test.cc new file mode 100644 index 00000000000..0eddda5ad25 --- /dev/null +++ b/tests/ut/cpp/utils/profile_test.cc @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" + +#ifndef ENABLE_PROFILE +#define ENABLE_PROFILE +#endif + +#include "utils/profile.h" + +namespace mindspore { +class TestProfile : public UT::Common { + public: + TestProfile() {} + virtual ~TestProfile() {} + + virtual void TearDown() {} +}; + +static void test_lap(Profile* prof) { + int nums[] = {30, 20, 70}; + int cnt = 0; + for (auto elem : nums) { + WITH(prof->Lap(cnt))[elem]()->void { usleep(elem); }; + cnt += 1; + } +} + +TEST_F(TestProfile, Test01) { + int step_cnt = 0; + Profile prof; + Profile* ptr_prof = &prof; + DumpTime::GetInstance().Record("Test01", GetTime(), true); + WITH(ptr_prof)[&ptr_prof, &step_cnt]()->void { + WITH(ptr_prof->Step("Step01"))[&step_cnt]()->void { + usleep(20); + step_cnt += 1; + }; + WITH(ptr_prof->Step("Step02"))[&ptr_prof, &step_cnt]()->void { + usleep(10); + test_lap(ptr_prof); + step_cnt += 1; + }; + }; + DumpTime::GetInstance().Record("Test01", GetTime(), false); + + prof.Print(); + + EXPECT_EQ(step_cnt, 2); +} + +TEST_F(TestProfile, Test02) { + std::map stat; + double t1 = GetTime(); + usleep(20); // Step01.stage1 + double t2 = GetTime(); + usleep(30); // Step01.stage2 + double t3 = GetTime(); + usleep(10); // Step02.stage1 + double t4 = GetTime(); + usleep(10); // Step02.stage2 + double t5 = GetTime(); + usleep(10); // Step02.stage3 + double t6 = GetTime(); + + MsProfile::StatTime("Step01.stage1", t2 - t1); + MsProfile::StatTime("Step01.stage2", t3 - t2); + MsProfile::StatTime("Step02.stage1", t4 - t3); + MsProfile::StatTime("Step02.stage2", t5 - t4); + MsProfile::StatTime("Step02.stage3", t6 - t5); + + MsProfile::Print(); + MsProfile::Reset(); + EXPECT_GT(t6 - t1, 0); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/signal_test.cc b/tests/ut/cpp/utils/signal_test.cc new file mode 100644 index 00000000000..f8b5acd40f2 --- /dev/null +++ b/tests/ut/cpp/utils/signal_test.cc @@ -0,0 +1,107 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "ir/named.h" +#include "utils/signal.h" +#include "common/common_test.h" + +using std::cout; +using std::endl; +using std::string; + +namespace mindspore { +class TestSignal : public UT::Common { + public: + TestSignal() {} +}; + +struct signals { + Signal signal; + Signal)> signal1; +}; + +class A { + public: + A() {} + explicit A(signals *sigs) : sigs_(sigs) { + sigs_->signal.connect(this, &A::FuncA); + sigs_->signal1.connect(this, &A::Funct); + printf("conn:%p\n", this); + i = std::make_shared(1); + } + void Funct(std::shared_ptr a) {} + virtual void FuncA(int v1, float v2, std::string str) { printf("A: --%d--%f--%s--\n", v1, v2, str.c_str()); } + + private: + signals *sigs_; + std::shared_ptr i; +}; + +class C : public A { + public: + C() {} + explicit C(signals *sigs) : A(sigs) { printf("conn C:%p\n", this); } + void FuncA(int v1, float v2, std::string str) { printf("C: --%d--%f--%s--\n", v1, v2, str.c_str()); } +}; + +class B : public A { + public: + B() {} + explicit B(signals *sigs) : A(sigs) { printf("conn B:%p\n", this); } + void FuncA(int v1, float v2, std::string str) { printf("B: --%d--%f--%s--\n", v1, v2, str.c_str()); } +}; + +TEST_F(TestSignal, test_common) { + A objA; + B objB; + C objC; + + Signal signal; + + signal.connect(&objA, &A::FuncA); + signal.connect(&objB, &B::FuncA); + signal.connect(&objC, &C::FuncA); + signal(20, 20, "Signal-Slot test"); +} + +TEST_F(TestSignal, test_sigs) { + signals sigs; + A objA(&sigs); + B objB(&sigs); + C objC(&sigs); + + sigs.signal.connect(&objA, &A::FuncA); + sigs.signal.connect(&objB, &B::FuncA); + sigs.signal.connect(&objC, &C::FuncA); + sigs.signal(20, 20, "sigs Signal-Slot test"); +} + +TEST_F(TestSignal, test_sigs_Named) { + signals sigs; + A objA(&sigs); + B objB(&sigs); + C objC(&sigs); + + sigs.signal(10, 20, "Signal-Slot test"); + std::shared_ptr a; + sigs.signal1(a); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/symbolic_test.cc b/tests/ut/cpp/utils/symbolic_test.cc new file mode 100644 index 00000000000..f259b62d6b9 --- /dev/null +++ b/tests/ut/cpp/utils/symbolic_test.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common_test.h" +#include "pipeline/static_analysis/static_analysis.h" +#include "utils/symbolic.h" + +using std::cout; +using std::endl; +using std::string; + +namespace mindspore { +class TestSymbolic : public UT::Common { + public: + TestSymbolic() {} +}; + +TEST_F(TestSymbolic, test_env) { + auto sk1 = std::make_shared(NewValueNode(1), abstract::FromValue(1234)); + auto sk1b = std::make_shared(NewValueNode(1), abstract::FromValue(1234)); + + ASSERT_EQ(*sk1, *sk1b); + + auto sk2 = std::make_shared(NewValueNode(2), abstract::FromValue(1234)); + + EnvInstance e = newenv->Set(sk1, 100); + ASSERT_FALSE(e == *newenv); + + ASSERT_EQ(newenv->Len(), 0); + ASSERT_EQ(e.Len(), 1); + ASSERT_EQ(e.Get(sk1, 0), 100); + ASSERT_EQ(e.Get(sk2, 0), 0); + + EnvInstance e2 = e.Set(sk1b, 200); + ASSERT_EQ(e2.Len(), 1); + ASSERT_EQ(e2.Get(sk1, 0), 200); + ASSERT_EQ(e2.Get(sk2, 0), 0); + + EnvInstance e3 = e2.Set(sk2, 300); + ASSERT_EQ(e3.Len(), 2); + ASSERT_EQ(e3.Get(sk1, 0), 200); + ASSERT_EQ(e3.Get(sk2, 0), 300); +} + +} // namespace mindspore diff --git a/tests/ut/cpp/utils/validator_test.cc b/tests/ut/cpp/utils/validator_test.cc new file mode 100644 index 00000000000..8eef44bde58 --- /dev/null +++ b/tests/ut/cpp/utils/validator_test.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common_test.h" + +#include "utils/log_adapter.h" +#include "pipeline/validator.h" +#include "pipeline/parse/parse.h" +#include "ir/manager.h" +#include "pipeline/static_analysis/prim.h" +#include "operator/ops.h" + +namespace mindspore { +namespace validator { +class TestValidator : public UT::Common { + public: + TestValidator() {} + virtual ~TestValidator() {} + + virtual void TearDown() {} +}; + +TEST_F(TestValidator, ValidateOperation01) { + auto node = NewValueNode(std::make_shared("scalar_add")); + ValidateOperation(node); + // normally, the above statement should not exit, so expected the following statement execute + EXPECT_TRUE(true); +} + +TEST_F(TestValidator, ValidateAbstract01) { + AnfNodePtr node = NewValueNode(1); + abstract::AbstractBasePtr abstract_v1 = abstract::FromValue(1, false); + node->set_abstract(abstract_v1); + ValidateAbstract(node); + // normally, the above statement should not exit, so expected the following statement execute + EXPECT_TRUE(true); +} +} // namespace validator +} // namespace mindspore diff --git a/tests/ut/cpp/vm/segment_runner_test.cc b/tests/ut/cpp/vm/segment_runner_test.cc new file mode 100644 index 00000000000..38819ad3d01 --- /dev/null +++ b/tests/ut/cpp/vm/segment_runner_test.cc @@ -0,0 +1,135 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "common/common_test.h" +#include "common/py_func_graph_fetcher.h" +#include "ir/manager.h" +#include "utils/log_adapter.h" +#include "ir/func_graph_cloner.h" +#include "pipeline/parse/parse.h" +#include "utils/graph_utils.h" +#include "pipeline/resource.h" +#include "debug/draw.h" +#include "operator/ops.h" +#include "vm/segment_runner.h" +#include "vm/transform.h" +#include "ir/meta_tensor.h" +#include "utils/convert_utils.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace compile { +using Tensor = tensor::Tensor; + +class TestCompileSegmentRunner : public UT::Common { + public: + TestCompileSegmentRunner() : get_py_fun_("gtest_input.vm", true) { UT::InitPythonPath(); } + + protected: + UT::PyFuncGraphFetcher get_py_fun_; + VM vm_; +}; + +TEST_F(TestCompileSegmentRunner, test_MsVmConvert1) { + FuncGraphPtr g = get_py_fun_("scalar_add"); + // g was managed by local variable manager in get_py_fun_ and that manager will be freed as no reference. + // so a new manager should be declared to make get_outputs() in segment_runner.cc happy. + std::shared_ptr manager = mindspore::Manage(g); + + BackendPtr b = std::make_shared("vm"); + CompileGraph transform_(b); + auto splits = transform_.SplitNodes(g); + VectorRef args({1.0, 2.0}); + + std::vector todos(splits.size()); + auto it = std::copy_if(std::begin(splits), std::end(splits), std::begin(todos), + [](const BaseRef& seg) -> bool { return utils::isa(seg); }); + todos.resize(std::distance(todos.begin(), it)); + ASSERT_EQ(todos.size(), 1); + + AnfNodePtrList anf_list; + for (auto &item : utils::cast(todos[0])) { + anf_list.push_back(utils::cast(item)); + } + auto convertResult = MsVmConvert(anf_list); + auto runResult = (*(convertResult.run))(args); + ASSERT_TRUE(runResult.size() == 1 && py::cast(BaseRefToPyData(runResult[0])) == 3.0); +} + +TEST_F(TestCompileSegmentRunner, test_MsVmConvert2) { + FuncGraphPtr g = get_py_fun_("scalar_mul"); + std::shared_ptr manager = mindspore::Manage(g); + + BackendPtr b = std::make_shared("vm"); + CompileGraph transform_(b); + auto splits = transform_.SplitNodes(g); + VectorRef args({1.0, 2.0}); + + std::vector todos(splits.size()); + auto it = std::copy_if(std::begin(splits), std::end(splits), std::begin(todos), + [](const BaseRef& seg) -> bool { return utils::isa(seg); }); + todos.resize(std::distance(todos.begin(), it)); + ASSERT_EQ(todos.size(), 1); + + AnfNodePtrList anf_list; + for (auto &item : utils::cast(todos[0])) { + anf_list.push_back(utils::cast(item)); + } + auto convertResult = MsVmConvert(anf_list); + auto runResult = (*(convertResult.run))(args); + ASSERT_TRUE(runResult.size() == 1 && py::cast(BaseRefToPyData(runResult[0])) == 2.0); +} + +TEST_F(TestCompileSegmentRunner, test_if) { + FuncGraphPtr g = get_py_fun_("test_if"); + std::shared_ptr manager = mindspore::Manage(g); + + BackendPtr b = std::make_shared("vm"); + CompileGraph transform_(b); + auto splits = transform_.SplitNodes(g); + VectorRef args({1.0, 2.0}); + + std::vector todos(splits.size()); + auto it = std::copy_if(std::begin(splits), std::end(splits), std::begin(todos), + [](const BaseRef& seg) -> bool { return utils::isa(seg); }); + todos.resize(std::distance(todos.begin(), it)); + ASSERT_EQ(todos.size(), 1); + + AnfNodePtrList anf_list; + for (auto &item : utils::cast(todos[0])) { + anf_list.push_back(utils::cast(item)); + } + auto convertResult = MsVmConvert(anf_list); + auto runResult = (*(convertResult.run))(args); + + auto result = py::cast(BaseRefToPyData(runResult[0])); + ASSERT_TRUE(runResult.size() == 1 && result == false); +} + +TEST_F(TestCompileSegmentRunner, test_RunOperation1) { + VectorRef args({1}); + auto res = RunOperation(prim::kPrimIdentity, args); + ASSERT_EQ(py::cast(BaseRefToPyData(res)), 1); +} + +TEST_F(TestCompileSegmentRunner, test_RunOperation2) { + VectorRef args({1, 2}); + auto res = RunOperation(prim::kPrimScalarGt, args); + ASSERT_EQ(py::cast(BaseRefToPyData(res)), false); +} +} // namespace compile +} // namespace mindspore diff --git a/tests/ut/cpp/vm/vm_test.cc b/tests/ut/cpp/vm/vm_test.cc new file mode 100644 index 00000000000..04633043afd --- /dev/null +++ b/tests/ut/cpp/vm/vm_test.cc @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "vm/vm.h" +#include "common/common_test.h" +#include "operator/ops.h" +#include "vm/backend.h" + +namespace mindspore { +namespace compile { + +class TestCompileVM : public UT::Common { + public: + TestCompileVM() {} + virtual ~TestCompileVM() {} + + public: + virtual void SetUp(); + virtual void TearDown(); +}; + +void TestCompileVM::SetUp() { MS_LOG(INFO) << "TestCompileVM::SetUp()"; } + +void TestCompileVM::TearDown() { MS_LOG(INFO) << "TestCompileVM::TearDown()"; } + +TEST_F(TestCompileVM, StructPartial) { + auto partial = new StructPartial(100, VectorRef({20, 60, 100.0})); + std::stringstream ss; + ss << *partial; + delete partial; + partial = nullptr; +} + +TEST_F(TestCompileVM, FinalVM) { + std::vector> instr; + instr.push_back({Instruction::kCall, VectorRef({-1})}); + instr.push_back({Instruction::kTailCall, VectorRef({-2, 1, 1})}); + instr.push_back({Instruction::kReturn, VectorRef({-1, 1})}); + instr.push_back({Instruction::kPartial, VectorRef({0, "cc"})}); + BackendPtr backend = std::make_shared("vm"); + auto vm = new FinalVM(instr, backend); + vm->Eval(VectorRef({1, 2, 3, -1, "a", "b", "c"})); + delete vm; + vm = nullptr; +} + +} // namespace compile +} // namespace mindspore diff --git a/tests/ut/data/dataset/apple.jpg b/tests/ut/data/dataset/apple.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/apple.jpg differ diff --git a/tests/ut/data/dataset/declient.cfg b/tests/ut/data/dataset/declient.cfg new file mode 100644 index 00000000000..b657ead6d5f --- /dev/null +++ b/tests/ut/data/dataset/declient.cfg @@ -0,0 +1,9 @@ +{ + "logFilePath": "/tmp", + "rowsPerBuffer": 1, + "numParallelWorkers": 4, + "workerConnectorSize": 16, + "opConnectorSize": 16, + "seed": 5489 + +} diff --git a/tests/ut/data/dataset/golden/batch_01_result.npz b/tests/ut/data/dataset/golden/batch_01_result.npz new file mode 100644 index 00000000000..7da040cd589 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_01_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_02_result.npz b/tests/ut/data/dataset/golden/batch_02_result.npz new file mode 100644 index 00000000000..1d126c043eb Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_02_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_03_result.npz b/tests/ut/data/dataset/golden/batch_03_result.npz new file mode 100644 index 00000000000..3bb486428e7 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_03_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_04_result.npz b/tests/ut/data/dataset/golden/batch_04_result.npz new file mode 100644 index 00000000000..39198c56927 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_04_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_05_result.npz b/tests/ut/data/dataset/golden/batch_05_result.npz new file mode 100644 index 00000000000..24ab9b08368 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_05_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_06_result.npz b/tests/ut/data/dataset/golden/batch_06_result.npz new file mode 100644 index 00000000000..6e8e923eb90 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_06_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_07_result.npz b/tests/ut/data/dataset/golden/batch_07_result.npz new file mode 100644 index 00000000000..b25854e1799 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_07_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_08_result.npz b/tests/ut/data/dataset/golden/batch_08_result.npz new file mode 100644 index 00000000000..b02f0eb324d Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_08_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_09_result.npz b/tests/ut/data/dataset/golden/batch_09_result.npz new file mode 100644 index 00000000000..6e8e923eb90 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_09_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_10_result.npz b/tests/ut/data/dataset/golden/batch_10_result.npz new file mode 100644 index 00000000000..5568b7e0cc2 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_10_result.npz differ diff --git a/tests/ut/data/dataset/golden/batch_11_result.npz b/tests/ut/data/dataset/golden/batch_11_result.npz new file mode 100644 index 00000000000..2035b10c949 Binary files /dev/null and b/tests/ut/data/dataset/golden/batch_11_result.npz differ diff --git a/tests/ut/data/dataset/golden/columns_list_result.npz b/tests/ut/data/dataset/golden/columns_list_result.npz new file mode 100644 index 00000000000..c0240c6e213 Binary files /dev/null and b/tests/ut/data/dataset/golden/columns_list_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_after_repeat_result.npz b/tests/ut/data/dataset/golden/project_after_repeat_result.npz new file mode 100644 index 00000000000..894ae2201dc Binary files /dev/null and b/tests/ut/data/dataset/golden/project_after_repeat_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_alternate_parallel_inline_result.npz b/tests/ut/data/dataset/golden/project_alternate_parallel_inline_result.npz new file mode 100644 index 00000000000..8a172c8137b Binary files /dev/null and b/tests/ut/data/dataset/golden/project_alternate_parallel_inline_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_before_repeat_result.npz b/tests/ut/data/dataset/golden/project_before_repeat_result.npz new file mode 100644 index 00000000000..894ae2201dc Binary files /dev/null and b/tests/ut/data/dataset/golden/project_before_repeat_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_between_maps_result.npz b/tests/ut/data/dataset/golden/project_between_maps_result.npz new file mode 100644 index 00000000000..8a172c8137b Binary files /dev/null and b/tests/ut/data/dataset/golden/project_between_maps_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_map_after_result.npz b/tests/ut/data/dataset/golden/project_map_after_result.npz new file mode 100644 index 00000000000..8a172c8137b Binary files /dev/null and b/tests/ut/data/dataset/golden/project_map_after_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_map_before_result.npz b/tests/ut/data/dataset/golden/project_map_before_result.npz new file mode 100644 index 00000000000..8a172c8137b Binary files /dev/null and b/tests/ut/data/dataset/golden/project_map_before_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_multiple_columns_in_order_result.npz b/tests/ut/data/dataset/golden/project_multiple_columns_in_order_result.npz new file mode 100644 index 00000000000..aee04e1b27a Binary files /dev/null and b/tests/ut/data/dataset/golden/project_multiple_columns_in_order_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_multiple_columns_out_of_order_result.npz b/tests/ut/data/dataset/golden/project_multiple_columns_out_of_order_result.npz new file mode 100644 index 00000000000..8a172c8137b Binary files /dev/null and b/tests/ut/data/dataset/golden/project_multiple_columns_out_of_order_result.npz differ diff --git a/tests/ut/data/dataset/golden/project_single_column_result.npz b/tests/ut/data/dataset/golden/project_single_column_result.npz new file mode 100644 index 00000000000..08bf876325a Binary files /dev/null and b/tests/ut/data/dataset/golden/project_single_column_result.npz differ diff --git a/tests/ut/data/dataset/golden/repeat_result.npz b/tests/ut/data/dataset/golden/repeat_result.npz new file mode 100644 index 00000000000..73b0a24b201 Binary files /dev/null and b/tests/ut/data/dataset/golden/repeat_result.npz differ diff --git a/tests/ut/data/dataset/golden/shuffle_01_result.npz b/tests/ut/data/dataset/golden/shuffle_01_result.npz new file mode 100644 index 00000000000..467ea74a4c4 Binary files /dev/null and b/tests/ut/data/dataset/golden/shuffle_01_result.npz differ diff --git a/tests/ut/data/dataset/golden/shuffle_02_result.npz b/tests/ut/data/dataset/golden/shuffle_02_result.npz new file mode 100644 index 00000000000..27eb0a470d3 Binary files /dev/null and b/tests/ut/data/dataset/golden/shuffle_02_result.npz differ diff --git a/tests/ut/data/dataset/golden/shuffle_03_result.npz b/tests/ut/data/dataset/golden/shuffle_03_result.npz new file mode 100644 index 00000000000..6a6e62f3ffa Binary files /dev/null and b/tests/ut/data/dataset/golden/shuffle_03_result.npz differ diff --git a/tests/ut/data/dataset/golden/shuffle_04_result.npz b/tests/ut/data/dataset/golden/shuffle_04_result.npz new file mode 100644 index 00000000000..a3b9469f9cd Binary files /dev/null and b/tests/ut/data/dataset/golden/shuffle_04_result.npz differ diff --git a/tests/ut/data/dataset/golden/storage_result.npz b/tests/ut/data/dataset/golden/storage_result.npz new file mode 100644 index 00000000000..10cad9f2b01 Binary files /dev/null and b/tests/ut/data/dataset/golden/storage_result.npz differ diff --git a/tests/ut/data/dataset/golden/test_case_0_result.npz b/tests/ut/data/dataset/golden/test_case_0_result.npz new file mode 100644 index 00000000000..b0ab3fb798b Binary files /dev/null and b/tests/ut/data/dataset/golden/test_case_0_result.npz differ diff --git a/tests/ut/data/dataset/golden/test_case_0_reverse_result.npz b/tests/ut/data/dataset/golden/test_case_0_reverse_result.npz new file mode 100644 index 00000000000..44af6db750e Binary files /dev/null and b/tests/ut/data/dataset/golden/test_case_0_reverse_result.npz differ diff --git a/tests/ut/data/dataset/golden/test_case_1_result.npz b/tests/ut/data/dataset/golden/test_case_1_result.npz new file mode 100644 index 00000000000..7a9a70861ca Binary files /dev/null and b/tests/ut/data/dataset/golden/test_case_1_result.npz differ diff --git a/tests/ut/data/dataset/golden/test_case_1_reverse_result.npz b/tests/ut/data/dataset/golden/test_case_1_reverse_result.npz new file mode 100644 index 00000000000..b4346fd7967 Binary files /dev/null and b/tests/ut/data/dataset/golden/test_case_1_reverse_result.npz differ diff --git a/tests/ut/data/dataset/golden/test_case_2_result.npz b/tests/ut/data/dataset/golden/test_case_2_result.npz new file mode 100644 index 00000000000..e3273425d39 Binary files /dev/null and b/tests/ut/data/dataset/golden/test_case_2_result.npz differ diff --git a/tests/ut/data/dataset/golden/test_case_2_reverse_result.npz b/tests/ut/data/dataset/golden/test_case_2_reverse_result.npz new file mode 100644 index 00000000000..579431fcc90 Binary files /dev/null and b/tests/ut/data/dataset/golden/test_case_2_reverse_result.npz differ diff --git a/tests/ut/data/dataset/golden/tf_file_no_schema.npz b/tests/ut/data/dataset/golden/tf_file_no_schema.npz new file mode 100644 index 00000000000..b8239985215 Binary files /dev/null and b/tests/ut/data/dataset/golden/tf_file_no_schema.npz differ diff --git a/tests/ut/data/dataset/golden/tf_file_padBytes10.npz b/tests/ut/data/dataset/golden/tf_file_padBytes10.npz new file mode 100644 index 00000000000..e3d6d9934bc Binary files /dev/null and b/tests/ut/data/dataset/golden/tf_file_padBytes10.npz differ diff --git a/tests/ut/data/dataset/golden/tfreader_result.npz b/tests/ut/data/dataset/golden/tfreader_result.npz new file mode 100644 index 00000000000..10cad9f2b01 Binary files /dev/null and b/tests/ut/data/dataset/golden/tfreader_result.npz differ diff --git a/tests/ut/data/dataset/golden/zip_01_result.npz b/tests/ut/data/dataset/golden/zip_01_result.npz new file mode 100644 index 00000000000..f34196166aa Binary files /dev/null and b/tests/ut/data/dataset/golden/zip_01_result.npz differ diff --git a/tests/ut/data/dataset/golden/zip_02_result.npz b/tests/ut/data/dataset/golden/zip_02_result.npz new file mode 100644 index 00000000000..1e9c7a985b1 Binary files /dev/null and b/tests/ut/data/dataset/golden/zip_02_result.npz differ diff --git a/tests/ut/data/dataset/golden/zip_03_result.npz b/tests/ut/data/dataset/golden/zip_03_result.npz new file mode 100644 index 00000000000..420db667906 Binary files /dev/null and b/tests/ut/data/dataset/golden/zip_03_result.npz differ diff --git a/tests/ut/data/dataset/golden/zip_04_result.npz b/tests/ut/data/dataset/golden/zip_04_result.npz new file mode 100644 index 00000000000..09315a3a828 Binary files /dev/null and b/tests/ut/data/dataset/golden/zip_04_result.npz differ diff --git a/tests/ut/data/dataset/golden/zip_05_result.npz b/tests/ut/data/dataset/golden/zip_05_result.npz new file mode 100644 index 00000000000..b8d23ca47f3 Binary files /dev/null and b/tests/ut/data/dataset/golden/zip_05_result.npz differ diff --git a/tests/ut/data/dataset/golden/zip_06_result.npz b/tests/ut/data/dataset/golden/zip_06_result.npz new file mode 100644 index 00000000000..be9ab9810ab Binary files /dev/null and b/tests/ut/data/dataset/golden/zip_06_result.npz differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_changemode.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_changemode.jpg new file mode 100644 index 00000000000..d7a0624a6f9 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_changemode.jpg differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_decoded.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_decoded.jpg new file mode 100644 index 00000000000..d7a0624a6f9 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_decoded.jpg differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_flipped_horizontal.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_flipped_horizontal.jpg new file mode 100644 index 00000000000..144f6a02e09 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_flipped_horizontal.jpg differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_flipped_vertical.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_flipped_vertical.jpg new file mode 100644 index 00000000000..cc0f26ac922 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_flipped_vertical.jpg differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_not_flip.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_not_flip.jpg new file mode 100644 index 00000000000..d7a0624a6f9 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_not_flip.jpg differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_rescaled.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_rescaled.jpg new file mode 100644 index 00000000000..0a74fdf4d24 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_rescaled.jpg differ diff --git a/tests/ut/data/dataset/imagefolder/apple_expect_resize_bilinear.jpg b/tests/ut/data/dataset/imagefolder/apple_expect_resize_bilinear.jpg new file mode 100644 index 00000000000..9925508ac90 Binary files /dev/null and b/tests/ut/data/dataset/imagefolder/apple_expect_resize_bilinear.jpg differ diff --git a/tests/ut/data/dataset/testBatchDataset/datasetSchema.json b/tests/ut/data/dataset/testBatchDataset/datasetSchema.json new file mode 100644 index 00000000000..82303087768 --- /dev/null +++ b/tests/ut/data/dataset/testBatchDataset/datasetSchema.json @@ -0,0 +1,46 @@ +{ + "datasetType": "TF", + "numRows": 12, + "columns": { + "col_sint16": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_sint32": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_sint64": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_float": { + "type": "float32", + "rank": 1, + "shape": [1] + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 1, + "shape": [1] + } + } +} diff --git a/tests/ut/data/dataset/testBatchDataset/test.data b/tests/ut/data/dataset/testBatchDataset/test.data new file mode 100644 index 00000000000..dd811c460ad Binary files /dev/null and b/tests/ut/data/dataset/testBatchDataset/test.data differ diff --git a/tests/ut/data/dataset/testCelebAData/1.JPEG b/tests/ut/data/dataset/testCelebAData/1.JPEG new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testCelebAData/1.JPEG differ diff --git a/tests/ut/data/dataset/testCelebAData/2.jpg b/tests/ut/data/dataset/testCelebAData/2.jpg new file mode 100644 index 00000000000..72316fbcb6a Binary files /dev/null and b/tests/ut/data/dataset/testCelebAData/2.jpg differ diff --git a/tests/ut/data/dataset/testCelebAData/list_attr_celeba.txt b/tests/ut/data/dataset/testCelebAData/list_attr_celeba.txt new file mode 100644 index 00000000000..0e57965ea6c --- /dev/null +++ b/tests/ut/data/dataset/testCelebAData/list_attr_celeba.txt @@ -0,0 +1,4 @@ +2 +5_o_Clock_Shadow Arched_Eyebrows Attractive Bags_Under_Eyes Bald Bangs Big_Lips Big_Nose Black_Hair Blond_Hair Blurry Brown_Hair Bushy_Eyebrows Chubby Double_Chin Eyeglasses Goatee Gray_Hair Heavy_Makeup High_Cheekbones Male Mouth_Slightly_Open Mustache Narrow_Eyes No_Beard Oval_Face Pale_Skin Pointy_Nose Receding_Hairline Rosy_Cheeks Sideburns Smiling Straight_Hair Wavy_Hair Wearing_Earrings Wearing_Hat Wearing_Lipstick Wearing_Necklace Wearing_Necktie Young +1.JPEG -1 1 1 -1 -1 -1 -1 -1 -1 -1 -1 1 -1 -1 -1 -1 -1 -1 1 1 -1 1 -1 -1 1 -1 -1 1 -1 -1 -1 1 1 -1 1 -1 1 -1 -1 1 +2.jpg -1 -1 -1 1 -1 -1 -1 1 -1 -1 -1 1 -1 -1 -1 -1 -1 -1 -1 1 -1 1 -1 -1 1 -1 -1 -1 -1 -1 -1 1 -1 -1 -1 -1 -1 -1 -1 1 diff --git a/tests/ut/data/dataset/testCifar100Data/datasetSchema.json b/tests/ut/data/dataset/testCifar100Data/datasetSchema.json new file mode 100644 index 00000000000..474a806bf26 --- /dev/null +++ b/tests/ut/data/dataset/testCifar100Data/datasetSchema.json @@ -0,0 +1,21 @@ +{ + "datasetType": "CIFAR100", + "numRows": 100, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "coarse_label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + }, + "fine_label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testCifar100Data/datasetSchemaTestRepeat.json b/tests/ut/data/dataset/testCifar100Data/datasetSchemaTestRepeat.json new file mode 100644 index 00000000000..a90edb342b0 --- /dev/null +++ b/tests/ut/data/dataset/testCifar100Data/datasetSchemaTestRepeat.json @@ -0,0 +1,21 @@ +{ + "datasetType": "CIFAR100", + "numRows": 33, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "coarse_label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + }, + "fine_label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testCifar100Data/test.bin b/tests/ut/data/dataset/testCifar100Data/test.bin new file mode 100644 index 00000000000..d582035f341 Binary files /dev/null and b/tests/ut/data/dataset/testCifar100Data/test.bin differ diff --git a/tests/ut/data/dataset/testCifar10Data/data_batch_1.bin b/tests/ut/data/dataset/testCifar10Data/data_batch_1.bin new file mode 100644 index 00000000000..7964f0952cd Binary files /dev/null and b/tests/ut/data/dataset/testCifar10Data/data_batch_1.bin differ diff --git a/tests/ut/data/dataset/testCifar10Data/datasetDistributionAll.json b/tests/ut/data/dataset/testCifar10Data/datasetDistributionAll.json new file mode 100644 index 00000000000..9234a6e0330 --- /dev/null +++ b/tests/ut/data/dataset/testCifar10Data/datasetDistributionAll.json @@ -0,0 +1,9 @@ +{ + "deviceNum" : 3, + "deviceId" : 1, + "shardConfig" : "ALL", + "shuffle" : "ON", + "seed" : 0, + "epoch" : 2 +} + diff --git a/tests/ut/data/dataset/testCifar10Data/datasetDistributionRandom.json b/tests/ut/data/dataset/testCifar10Data/datasetDistributionRandom.json new file mode 100644 index 00000000000..3f61c582a5a --- /dev/null +++ b/tests/ut/data/dataset/testCifar10Data/datasetDistributionRandom.json @@ -0,0 +1,9 @@ +{ + "deviceNum" : 3, + "deviceId" : 1, + "shardConfig" : "RANDOM", + "shuffle" : "ON", + "seed" : 0, + "epoch" : 1 +} + diff --git a/tests/ut/data/dataset/testCifar10Data/datasetDistributionUnique.json b/tests/ut/data/dataset/testCifar10Data/datasetDistributionUnique.json new file mode 100644 index 00000000000..99e685132b9 --- /dev/null +++ b/tests/ut/data/dataset/testCifar10Data/datasetDistributionUnique.json @@ -0,0 +1,9 @@ +{ + "deviceNum" : 3, + "deviceId" : 1, + "shardConfig" : "UNIQUE", + "shuffle" : "ON", + "seed" : 0, + "epoch" : 3 +} + diff --git a/tests/ut/data/dataset/testCifar10Data/datasetSchema.json b/tests/ut/data/dataset/testCifar10Data/datasetSchema.json new file mode 100644 index 00000000000..1a04b9af597 --- /dev/null +++ b/tests/ut/data/dataset/testCifar10Data/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "CIFAR10", + "numRows": 60000, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testCifar10Data/datasetSchemaTestRepeat.json b/tests/ut/data/dataset/testCifar10Data/datasetSchemaTestRepeat.json new file mode 100644 index 00000000000..c25e11c30f8 --- /dev/null +++ b/tests/ut/data/dataset/testCifar10Data/datasetSchemaTestRepeat.json @@ -0,0 +1,16 @@ +{ + "datasetType": "CIFAR10", + "numRows": 33, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testDataset1/datasetSchema.json b/tests/ut/data/dataset/testDataset1/datasetSchema.json new file mode 100644 index 00000000000..a409980fc27 --- /dev/null +++ b/tests/ut/data/dataset/testDataset1/datasetSchema.json @@ -0,0 +1,17 @@ +{ + "datasetType": "TF", + "numRows": 10, + "columns": { + "image": { + "type": "uint8", + "rank": 3, + "shape": [3, 4, 2], + "t_impl": "cvmat" + }, + "label" : { + "type": "int64", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testDataset1/testDataset1.data b/tests/ut/data/dataset/testDataset1/testDataset1.data new file mode 100644 index 00000000000..d5daf9f80a0 Binary files /dev/null and b/tests/ut/data/dataset/testDataset1/testDataset1.data differ diff --git a/tests/ut/data/dataset/testDataset2/datasetSchema.json b/tests/ut/data/dataset/testDataset2/datasetSchema.json new file mode 100644 index 00000000000..53964a76578 --- /dev/null +++ b/tests/ut/data/dataset/testDataset2/datasetSchema.json @@ -0,0 +1,30 @@ +{ + "datasetType": "TF", + "numRows": 10, + "columns": { + "image": { + "type": "uint8", + "rank": 3, + "shape": [3, 4, 2], + "t_impl": "cvmat" + }, + "label" : { + "type": "int64", + "rank": 1, + "shape": [7], + "t_impl": "flex" + }, + "A": { + "type": "float32", + "rank": 4, + "shape": [1, 13, 14, 12], + "t_impl": "cvmat" + }, + "B" : { + "type": "int64", + "rank": 1, + "shape": [9], + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testDataset2/testDataset2.data b/tests/ut/data/dataset/testDataset2/testDataset2.data new file mode 100644 index 00000000000..91273cc8b0f Binary files /dev/null and b/tests/ut/data/dataset/testDataset2/testDataset2.data differ diff --git a/tests/ut/data/dataset/testImageNetData/datasetSchema.json b/tests/ut/data/dataset/testImageNetData/datasetSchema.json new file mode 100644 index 00000000000..f82f607db7c --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "IMAGENET", + "numRows": 2, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 0, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/testImageNetData/datasetSchemaTestRepeat.json b/tests/ut/data/dataset/testImageNetData/datasetSchemaTestRepeat.json new file mode 100644 index 00000000000..4a6f7de437b --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData/datasetSchemaTestRepeat.json @@ -0,0 +1,16 @@ +{ + "datasetType": "IMAGENET", + "numRows": 33, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 0, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/testImageNetData/datasetSchema_2.json b/tests/ut/data/dataset/testImageNetData/datasetSchema_2.json new file mode 100644 index 00000000000..739317534d9 --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData/datasetSchema_2.json @@ -0,0 +1,16 @@ +{ + "datasetType": "IMAGENET", + "numRows": 2, + "columns": { + "imageone": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "labelone" : { + "type": "int32", + "rank": 0, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/testImageNetData/train/class1/1_1.jpg b/tests/ut/data/dataset/testImageNetData/train/class1/1_1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData/train/class1/1_1.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData/train/class1/1_2.jpg b/tests/ut/data/dataset/testImageNetData/train/class1/1_2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData/train/class1/1_2.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json b/tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json new file mode 100644 index 00000000000..3ebc4c989c8 --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData2/dataDistributionAll.json @@ -0,0 +1,8 @@ +{ + "deviceNum":4, + "deviceId": 2, + "shardConfig":"ALL", + "shuffle":"ON", + "seed": 0, + "epoch": 2 +} diff --git a/tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json b/tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json new file mode 100644 index 00000000000..a0f468f91db --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData2/dataDistributionRandom.json @@ -0,0 +1,8 @@ +{ + "deviceNum":4, + "deviceId": 2, + "shardConfig":"RANDOM", + "shuffle":"ON", + "seed": 0, + "epoch": 1 +} diff --git a/tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json b/tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json new file mode 100644 index 00000000000..a4eeddd9ae9 --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData2/dataDistributionUnique.json @@ -0,0 +1,8 @@ +{ + "deviceNum":4, + "deviceId": 2, + "shardConfig":"UNIQUE", + "shuffle":"ON", + "seed": 0, + "epoch": 3 +} diff --git a/tests/ut/data/dataset/testImageNetData2/datasetSchema.json b/tests/ut/data/dataset/testImageNetData2/datasetSchema.json new file mode 100644 index 00000000000..e9e7bf30524 --- /dev/null +++ b/tests/ut/data/dataset/testImageNetData2/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "IMAGENET", + "numRows": 6, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 1, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/testImageNetData2/train/class1/1_1.jpg b/tests/ut/data/dataset/testImageNetData2/train/class1/1_1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData2/train/class1/1_1.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData2/train/class1/1_2.jpg b/tests/ut/data/dataset/testImageNetData2/train/class1/1_2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData2/train/class1/1_2.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData2/train/class2/2_1.jpg b/tests/ut/data/dataset/testImageNetData2/train/class2/2_1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData2/train/class2/2_1.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData2/train/class2/2_2.jpg b/tests/ut/data/dataset/testImageNetData2/train/class2/2_2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData2/train/class2/2_2.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData2/train/class3/3_1.jpg b/tests/ut/data/dataset/testImageNetData2/train/class3/3_1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData2/train/class3/3_1.jpg differ diff --git a/tests/ut/data/dataset/testImageNetData2/train/class3/3_2.jpg b/tests/ut/data/dataset/testImageNetData2/train/class3/3_2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testImageNetData2/train/class3/3_2.jpg differ diff --git a/tests/ut/data/dataset/testManifestData/cpp.json b/tests/ut/data/dataset/testManifestData/cpp.json new file mode 100644 index 00000000000..2ba9e242fc9 --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/cpp.json @@ -0,0 +1,3 @@ +{"source":"./data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/train/2.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/eval/1.JPEG", "usage":"EVAL","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} diff --git a/tests/ut/data/dataset/testManifestData/datasetSchema.json b/tests/ut/data/dataset/testManifestData/datasetSchema.json new file mode 100644 index 00000000000..f4d14ea48b8 --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "MANIFEST", + "numRows": 3, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/testManifestData/eval/1.JPEG b/tests/ut/data/dataset/testManifestData/eval/1.JPEG new file mode 100644 index 00000000000..d6575891cba Binary files /dev/null and b/tests/ut/data/dataset/testManifestData/eval/1.JPEG differ diff --git a/tests/ut/data/dataset/testManifestData/eval/2.JPEG b/tests/ut/data/dataset/testManifestData/eval/2.JPEG new file mode 100644 index 00000000000..5ca2194e88d Binary files /dev/null and b/tests/ut/data/dataset/testManifestData/eval/2.JPEG differ diff --git a/tests/ut/data/dataset/testManifestData/labels1.txt b/tests/ut/data/dataset/testManifestData/labels1.txt new file mode 100644 index 00000000000..a228e18e36d --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/labels1.txt @@ -0,0 +1,2 @@ +2:cat +5:dog \ No newline at end of file diff --git a/tests/ut/data/dataset/testManifestData/labels2.txt b/tests/ut/data/dataset/testManifestData/labels2.txt new file mode 100644 index 00000000000..55fa0114e9b --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/labels2.txt @@ -0,0 +1,2 @@ +cat +dog \ No newline at end of file diff --git a/tests/ut/data/dataset/testManifestData/labels3.txt b/tests/ut/data/dataset/testManifestData/labels3.txt new file mode 100644 index 00000000000..5d5b306e571 --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/labels3.txt @@ -0,0 +1 @@ +cat \ No newline at end of file diff --git a/tests/ut/data/dataset/testManifestData/labels4.txt b/tests/ut/data/dataset/testManifestData/labels4.txt new file mode 100644 index 00000000000..dc82c842c4d --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/labels4.txt @@ -0,0 +1,3 @@ +cat +dog +elephant \ No newline at end of file diff --git a/tests/ut/data/dataset/testManifestData/test.manifest b/tests/ut/data/dataset/testManifestData/test.manifest new file mode 100644 index 00000000000..f403cdede9e --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/test.manifest @@ -0,0 +1,6 @@ +{"source":"../data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"},{"type": "modelarts/image_classification","name": "flower","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/eval/1.JPEG", "usage":"EVAL","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/eval/2.JPEG", "usage":"EVAL","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} diff --git a/tests/ut/data/dataset/testManifestData/test5trainimgs.json b/tests/ut/data/dataset/testManifestData/test5trainimgs.json new file mode 100644 index 00000000000..a4923201787 --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/test5trainimgs.json @@ -0,0 +1,5 @@ +{"source":"../data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/train/2.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/train/2.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/eval/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"../data/dataset/testManifestData/eval/2.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} diff --git a/tests/ut/data/dataset/testManifestData/train/1.JPEG b/tests/ut/data/dataset/testManifestData/train/1.JPEG new file mode 100644 index 00000000000..72316fbcb6a Binary files /dev/null and b/tests/ut/data/dataset/testManifestData/train/1.JPEG differ diff --git a/tests/ut/data/dataset/testManifestData/train/2.JPEG b/tests/ut/data/dataset/testManifestData/train/2.JPEG new file mode 100644 index 00000000000..add5186cfea Binary files /dev/null and b/tests/ut/data/dataset/testManifestData/train/2.JPEG differ diff --git a/tests/ut/data/dataset/testMnistData/datasetSchema.json b/tests/ut/data/dataset/testMnistData/datasetSchema.json new file mode 100644 index 00000000000..7fc659b9067 --- /dev/null +++ b/tests/ut/data/dataset/testMnistData/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "MNIST", + "numRows": 10, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 1, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/testMnistData/datasetSchemaTestRepeat.json b/tests/ut/data/dataset/testMnistData/datasetSchemaTestRepeat.json new file mode 100644 index 00000000000..9c44a2b7f9f --- /dev/null +++ b/tests/ut/data/dataset/testMnistData/datasetSchemaTestRepeat.json @@ -0,0 +1,16 @@ +{ + "datasetType": "MNIST", + "numRows": 33, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint32", + "rank": 1, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/testMnistData/t10k-images-idx3-ubyte b/tests/ut/data/dataset/testMnistData/t10k-images-idx3-ubyte new file mode 100644 index 00000000000..c122dd00241 Binary files /dev/null and b/tests/ut/data/dataset/testMnistData/t10k-images-idx3-ubyte differ diff --git a/tests/ut/data/dataset/testMnistData/t10k-labels-idx1-ubyte b/tests/ut/data/dataset/testMnistData/t10k-labels-idx1-ubyte new file mode 100644 index 00000000000..87cf2ee1699 Binary files /dev/null and b/tests/ut/data/dataset/testMnistData/t10k-labels-idx1-ubyte differ diff --git a/tests/ut/data/dataset/testPK/data/class1/0.jpg b/tests/ut/data/dataset/testPK/data/class1/0.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/0.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/1.jpg b/tests/ut/data/dataset/testPK/data/class1/1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/1.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/10.jpg b/tests/ut/data/dataset/testPK/data/class1/10.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/10.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/2.jpg b/tests/ut/data/dataset/testPK/data/class1/2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/2.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/3.jpg b/tests/ut/data/dataset/testPK/data/class1/3.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/3.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/4.jpg b/tests/ut/data/dataset/testPK/data/class1/4.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/4.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/5.jpg b/tests/ut/data/dataset/testPK/data/class1/5.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/5.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/6.jpg b/tests/ut/data/dataset/testPK/data/class1/6.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/6.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/7.jpg b/tests/ut/data/dataset/testPK/data/class1/7.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/7.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/8.jpg b/tests/ut/data/dataset/testPK/data/class1/8.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/8.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class1/9.jpg b/tests/ut/data/dataset/testPK/data/class1/9.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class1/9.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/0.jpg b/tests/ut/data/dataset/testPK/data/class2/0.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/0.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/1.jpg b/tests/ut/data/dataset/testPK/data/class2/1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/1.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/10.jpg b/tests/ut/data/dataset/testPK/data/class2/10.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/10.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/2.jpg b/tests/ut/data/dataset/testPK/data/class2/2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/2.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/3.jpg b/tests/ut/data/dataset/testPK/data/class2/3.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/3.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/4.jpg b/tests/ut/data/dataset/testPK/data/class2/4.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/4.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/5.jpg b/tests/ut/data/dataset/testPK/data/class2/5.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/5.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/6.jpg b/tests/ut/data/dataset/testPK/data/class2/6.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/6.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/7.jpg b/tests/ut/data/dataset/testPK/data/class2/7.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/7.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/8.jpg b/tests/ut/data/dataset/testPK/data/class2/8.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/8.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class2/9.jpg b/tests/ut/data/dataset/testPK/data/class2/9.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class2/9.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/0.jpg b/tests/ut/data/dataset/testPK/data/class3/0.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/0.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/1.jpg b/tests/ut/data/dataset/testPK/data/class3/1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/1.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/10.jpg b/tests/ut/data/dataset/testPK/data/class3/10.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/10.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/2.jpg b/tests/ut/data/dataset/testPK/data/class3/2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/2.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/3.jpg b/tests/ut/data/dataset/testPK/data/class3/3.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/3.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/4.jpg b/tests/ut/data/dataset/testPK/data/class3/4.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/4.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/5.jpg b/tests/ut/data/dataset/testPK/data/class3/5.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/5.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/6.jpg b/tests/ut/data/dataset/testPK/data/class3/6.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/6.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/7.jpg b/tests/ut/data/dataset/testPK/data/class3/7.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/7.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/8.jpg b/tests/ut/data/dataset/testPK/data/class3/8.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/8.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class3/9.jpg b/tests/ut/data/dataset/testPK/data/class3/9.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class3/9.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/0.jpg b/tests/ut/data/dataset/testPK/data/class4/0.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/0.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/1.jpg b/tests/ut/data/dataset/testPK/data/class4/1.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/1.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/10.jpg b/tests/ut/data/dataset/testPK/data/class4/10.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/10.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/2.jpg b/tests/ut/data/dataset/testPK/data/class4/2.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/2.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/3.jpg b/tests/ut/data/dataset/testPK/data/class4/3.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/3.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/4.jpg b/tests/ut/data/dataset/testPK/data/class4/4.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/4.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/5.jpg b/tests/ut/data/dataset/testPK/data/class4/5.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/5.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/6.jpg b/tests/ut/data/dataset/testPK/data/class4/6.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/6.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/7.jpg b/tests/ut/data/dataset/testPK/data/class4/7.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/7.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/8.jpg b/tests/ut/data/dataset/testPK/data/class4/8.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/8.jpg differ diff --git a/tests/ut/data/dataset/testPK/data/class4/9.jpg b/tests/ut/data/dataset/testPK/data/class4/9.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testPK/data/class4/9.jpg differ diff --git a/tests/ut/data/dataset/testPK/datasetSchema.json b/tests/ut/data/dataset/testPK/datasetSchema.json new file mode 100644 index 00000000000..edab96dfff9 --- /dev/null +++ b/tests/ut/data/dataset/testPK/datasetSchema.json @@ -0,0 +1,14 @@ +{ + "datasetType": "IMAGENET", + "numRows": 2, + "columns": { + "image": { + "type": "uint8", + "rank": 1 + }, + "label" : { + "type": "uint32", + "rank": 1 + } + } +} diff --git a/tests/ut/data/dataset/testPK/distribution.json b/tests/ut/data/dataset/testPK/distribution.json new file mode 100644 index 00000000000..33f869f653b --- /dev/null +++ b/tests/ut/data/dataset/testPK/distribution.json @@ -0,0 +1,7 @@ +{ + "deviceNum":1, + "deviceId": 0, + "shardConfig":"RANDOM", + "shuffle":"OFF", + "seed": 0 +} diff --git a/tests/ut/data/dataset/testPyfuncMap/data.data b/tests/ut/data/dataset/testPyfuncMap/data.data new file mode 100644 index 00000000000..3df00a511c6 Binary files /dev/null and b/tests/ut/data/dataset/testPyfuncMap/data.data differ diff --git a/tests/ut/data/dataset/testPyfuncMap/pyfuncmap.py b/tests/ut/data/dataset/testPyfuncMap/pyfuncmap.py new file mode 100644 index 00000000000..2d209947586 --- /dev/null +++ b/tests/ut/data/dataset/testPyfuncMap/pyfuncmap.py @@ -0,0 +1,143 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds + + +DATA_DIR = ["./data.data"] +SCHEMA_DIR = "./schema.json" + +def test_case_0(): + """ + Test PyFunc + """ + print("Test 1-1 PyFunc : lambda x : x + x") + + col = "col0" + + # apply dataset operations + ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + ds1 = ds1.map(input_column_names=col, output_column_names="out", operation=(lambda x: x + x)) + + print("************** Output Tensor *****************") + for data in ds1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + print(data["out"]) + print("************** Output Tensor *****************") + + +def test_case_1(): + """ + Test PyFunc + """ + print("Test 1-n PyFunc : (lambda x : (x , x + x)) ") + + + col = "col0" + + # apply dataset operations + ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + ds1 = ds1.map(input_column_names=col, output_column_names=["out0", "out1"], operation=(lambda x: (x, x + x))) + + print("************** Output Tensor *****************") + for data in ds1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + print("out0") + print(data["out0"]) + print("out1") + print(data["out1"]) + print("************** Output Tensor *****************") + + +def test_case_2(): + """ + Test PyFunc + """ + print("Test n-1 PyFunc : (lambda x, y : x + y) ") + + + col = ["col0", "col1"] + + # apply dataset operations + ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + ds1 = ds1.map(input_column_names=col, output_column_names="out", operation=(lambda x, y: x + y)) + + print("************** Output Tensor *****************") + for data in ds1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + print(data["out"]) + + print("************** Output Tensor *****************") + +def test_case_3(): + """ + Test PyFunc + """ + print("Test n-m PyFunc : (lambda x, y : (x , x + 1, x + y)") + + + col = ["col0", "col1"] + + # apply dataset operations + ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + ds1 = ds1.map(input_column_names=col, output_column_names=["out0", "out1", "out2"], operation=(lambda x, y: (x, x + y, x + x + y))) + + print("************** Output Tensor *****************") + for data in ds1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + print("out0") + print(data["out0"]) + print("out1") + print(data["out1"]) + print("out2") + print(data["out2"]) + print("************** Output Tensor *****************") + +def test_case_4(): + """ + Test PyFunc + """ + print("Test Parallel n-m PyFunc : (lambda x, y : (x , x + 1, x + y)") + + + col = ["col0", "col1"] + + # apply dataset operations + ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + ds1 = ds1.map(input_column_names=col, output_column_names=["out0", "out1", "out2"], num_parallel_workers = 4, operation=(lambda x, y: (x, x + y, x + x + y))) + + print("************** Output Tensor *****************") + for data in ds1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + print("out0") + print(data["out0"]) + print("out1") + print(data["out1"]) + print("out2") + print(data["out2"]) + print("************** Output Tensor *****************") + + + +if __name__ == "__main__": + test_case_0() + #test_case_1() + #test_case_2() + #test_case_3() + #test_case_4() diff --git a/tests/ut/data/dataset/testPyfuncMap/schema.json b/tests/ut/data/dataset/testPyfuncMap/schema.json new file mode 100644 index 00000000000..9be313662bd --- /dev/null +++ b/tests/ut/data/dataset/testPyfuncMap/schema.json @@ -0,0 +1,21 @@ +{ + "datasetType": "TF", + "numRows": 12, + "columns": { + "col0": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col1": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col2": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + } + } +} diff --git a/tests/ut/data/dataset/testSchema.json b/tests/ut/data/dataset/testSchema.json new file mode 100644 index 00000000000..555c4d063f7 --- /dev/null +++ b/tests/ut/data/dataset/testSchema.json @@ -0,0 +1,14 @@ +{ + "datasetType": "TF", + "columns": { + "image": { + "type": "uint8", + "rank": 3, + "shape": [3, 4, 2] + }, + "label" : { + "type": "int32", + "rank": 1 + } + } +} diff --git a/tests/ut/data/dataset/testTFBert5Rows/5TFDatas.data b/tests/ut/data/dataset/testTFBert5Rows/5TFDatas.data new file mode 100644 index 00000000000..c5b5440cffe Binary files /dev/null and b/tests/ut/data/dataset/testTFBert5Rows/5TFDatas.data differ diff --git a/tests/ut/data/dataset/testTFBert5Rows/datasetSchema.json b/tests/ut/data/dataset/testTFBert5Rows/datasetSchema.json new file mode 100644 index 00000000000..c50ac8f3cca --- /dev/null +++ b/tests/ut/data/dataset/testTFBert5Rows/datasetSchema.json @@ -0,0 +1,41 @@ +{ + "datasetType": "TF", + "numRows": 5, + "columns": { + "next_sentence_labels": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "masked_lm_positions": { + "type": "int64", + "rank": 1, + "shape": [20] + }, + "masked_lm_ids": { + "type": "int64", + "rank": 1, + "shape": [20] + }, + "masked_lm_weights": { + "type": "float32", + "rank": 1, + "shape": [20] + }, + "input_mask": { + "type": "int64", + "rank": 1, + "shape": [128] + }, + "input_ids": { + "type": "int64", + "rank": 1, + "shape": [128] + }, + "segment_ids" : { + "type": "int64", + "rank": 1, + "shape": [128] + } + } +} diff --git a/tests/ut/data/dataset/testTFBert5Rows1/5TFDatas.data b/tests/ut/data/dataset/testTFBert5Rows1/5TFDatas.data new file mode 100644 index 00000000000..c5b5440cffe Binary files /dev/null and b/tests/ut/data/dataset/testTFBert5Rows1/5TFDatas.data differ diff --git a/tests/ut/data/dataset/testTFBert5Rows1/datasetSchema.json b/tests/ut/data/dataset/testTFBert5Rows1/datasetSchema.json new file mode 100644 index 00000000000..fd8242569be --- /dev/null +++ b/tests/ut/data/dataset/testTFBert5Rows1/datasetSchema.json @@ -0,0 +1,31 @@ +{ + "datasetType": "TF", + "numRows": 5, + "columns": { + "next_sentence_labels": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "masked_lm_positions": { + "type": "int64", + "rank": 1, + "shape": [20] + }, + "masked_lm_ids": { + "type": "int64", + "rank": 1, + "shape": [20] + }, + "masked_lm_weights": { + "type": "float32", + "rank": 1, + "shape": [20] + }, + "input_mask": { + "type": "int64", + "rank": 1, + "shape": [128] + } + } +} diff --git a/tests/ut/data/dataset/testTFBert5Rows2/5TFDatas.data b/tests/ut/data/dataset/testTFBert5Rows2/5TFDatas.data new file mode 100644 index 00000000000..c5b5440cffe Binary files /dev/null and b/tests/ut/data/dataset/testTFBert5Rows2/5TFDatas.data differ diff --git a/tests/ut/data/dataset/testTFBert5Rows2/datasetSchema.json b/tests/ut/data/dataset/testTFBert5Rows2/datasetSchema.json new file mode 100644 index 00000000000..504e1bed145 --- /dev/null +++ b/tests/ut/data/dataset/testTFBert5Rows2/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "TF", + "numRows": 5, + "columns": { + "input_ids": { + "type": "int64", + "rank": 1, + "shape": [128] + }, + "segment_ids" : { + "type": "int64", + "rank": 1, + "shape": [128] + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema.json b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema.json new file mode 100644 index 00000000000..dcb8c2b4be1 --- /dev/null +++ b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema.json @@ -0,0 +1,46 @@ +{ + "datasetType": "TF", + "numRows": 12, + "columns": { + "col_sint16": { + "type": "int16", + "rank": 1, + "shape": [1] + }, + "col_sint32": { + "type": "int32", + "rank": 1, + "shape": [1] + }, + "col_sint64": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_float": { + "type": "float32", + "rank": 1, + "shape": [1] + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 1, + "shape": [1] + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema1Row.json b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema1Row.json new file mode 100644 index 00000000000..5bbd6850c05 --- /dev/null +++ b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema1Row.json @@ -0,0 +1,46 @@ +{ + "datasetType": "TF", + "numRows": 1, + "columns": { + "col_sint16": { + "type": "int16", + "rank": 1, + "shape": [1] + }, + "col_sint32": { + "type": "int32", + "rank": 1, + "shape": [1] + }, + "col_sint64": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_float": { + "type": "float32", + "rank": 1, + "shape": [1] + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 1, + "shape": [1] + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema5Rows.json b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema5Rows.json new file mode 100644 index 00000000000..4e1a3f2fbff --- /dev/null +++ b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema5Rows.json @@ -0,0 +1,46 @@ +{ + "datasetType": "TF", + "numRows": 5, + "columns": { + "col_sint16": { + "type": "int16", + "rank": 1, + "shape": [1] + }, + "col_sint32": { + "type": "int32", + "rank": 1, + "shape": [1] + }, + "col_sint64": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_float": { + "type": "float32", + "rank": 1, + "shape": [1] + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 1, + "shape": [1] + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema7Rows.json b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema7Rows.json new file mode 100644 index 00000000000..118a39fccd0 --- /dev/null +++ b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchema7Rows.json @@ -0,0 +1,46 @@ +{ + "datasetType": "TF", + "numRows": 7, + "columns": { + "col_sint16": { + "type": "int16", + "rank": 1, + "shape": [1] + }, + "col_sint32": { + "type": "int32", + "rank": 1, + "shape": [1] + }, + "col_sint64": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_float": { + "type": "float32", + "rank": 1, + "shape": [1] + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 1, + "shape": [1] + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/datasetSchemaPadBytes10.json b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchemaPadBytes10.json new file mode 100644 index 00000000000..e00052eb5b1 --- /dev/null +++ b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchemaPadBytes10.json @@ -0,0 +1,46 @@ +{ + "datasetType": "TF", + "numRows": 24, + "columns": { + "col_sint16": { + "type": "int16", + "rank": 1, + "shape": [1] + }, + "col_sint32": { + "type": "int32", + "rank": 1, + "shape": [1] + }, + "col_sint64": { + "type": "int64", + "rank": 1, + "shape": [1] + }, + "col_float": { + "type": "float32", + "rank": 1, + "shape": [1] + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 1, + "shape": [-1, 10] + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/datasetSchemaRank0.json b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchemaRank0.json new file mode 100644 index 00000000000..5dd89753a37 --- /dev/null +++ b/tests/ut/data/dataset/testTFTestAllTypes/datasetSchemaRank0.json @@ -0,0 +1,41 @@ +{ + "datasetType": "TF", + "numRows": 12, + "columns": { + "col_sint16": { + "type": "int16", + "rank": 0 + }, + "col_sint32": { + "type": "int32", + "rank": 0 + }, + "col_sint64": { + "type": "int64", + "rank": 0 + }, + "col_float": { + "type": "float32", + "rank": 0 + }, + "col_1d": { + "type": "int64", + "rank": 1, + "shape": [2] + }, + "col_2d": { + "type": "int64", + "rank": 2, + "shape": [2, 2] + }, + "col_3d": { + "type": "int64", + "rank": 3, + "shape": [2, 2, 2] + }, + "col_binary": { + "type": "uint8", + "rank": 0 + } + } +} diff --git a/tests/ut/data/dataset/testTFTestAllTypes/test.data b/tests/ut/data/dataset/testTFTestAllTypes/test.data new file mode 100644 index 00000000000..dd811c460ad Binary files /dev/null and b/tests/ut/data/dataset/testTFTestAllTypes/test.data differ diff --git a/tests/ut/data/dataset/testTFTestAllTypes/test2.data b/tests/ut/data/dataset/testTFTestAllTypes/test2.data new file mode 100644 index 00000000000..dd811c460ad Binary files /dev/null and b/tests/ut/data/dataset/testTFTestAllTypes/test2.data differ diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/121.xml b/tests/ut/data/dataset/testVOC2012/Annotations/121.xml new file mode 100644 index 00000000000..9be633f1ca0 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/121.xml @@ -0,0 +1,39 @@ + + VOC2012 + 121.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + dog + Frontal + 0 + 0 + + 101 + 22 + 445 + 297 + + + + car + Frontal + 0 + 0 + + 21 + 29 + 221 + 253 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/123.xml b/tests/ut/data/dataset/testVOC2012/Annotations/123.xml new file mode 100644 index 00000000000..c5b480af24f --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/123.xml @@ -0,0 +1,27 @@ + + VOC2012 + 123.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + car + Unspecified + 1 + 0 + + 5 + 22 + 328 + 320 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/27.xml b/tests/ut/data/dataset/testVOC2012/Annotations/27.xml new file mode 100644 index 00000000000..d1af6d90b22 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/27.xml @@ -0,0 +1,54 @@ + + VOC2012 + 27.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 486 + 500 + 3 + + 0 + + person + Unspecified + 0 + 0 + + 161 + 132 + 323 + 342 + + + head + + 159 + 113 + 208 + 166 + + + + foot + + 261 + 321 + 287 + 344 + + + + foot + + 329 + 317 + 330 + 366 + + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/32.xml b/tests/ut/data/dataset/testVOC2012/Annotations/32.xml new file mode 100644 index 00000000000..18c2c4a1c5b --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/32.xml @@ -0,0 +1,51 @@ + + VOC2012 + 32.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 281 + 3 + + 1 + + train + Frontal + 0 + 0 + + 113 + 79 + 323 + 191 + + + + train + Left + 0 + 0 + + 121 + 91 + 191 + 121 + + + + car + Rear + 0 + 0 + + 195 + 155 + 235 + 235 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/33.xml b/tests/ut/data/dataset/testVOC2012/Annotations/33.xml new file mode 100644 index 00000000000..240dcfdb89f --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/33.xml @@ -0,0 +1,51 @@ + + VOC2012 + 33.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 366 + 3 + + 1 + + person + Unspecified + 0 + 0 + + 8 + 121 + 471 + 242 + + + + person + Left + 0 + 0 + + 425 + 239 + 445 + 235 + + + + person + Left + 1 + 0 + + 321 + 121 + 421 + 221 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/39.xml b/tests/ut/data/dataset/testVOC2012/Annotations/39.xml new file mode 100644 index 00000000000..3ffd2d69ad2 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/39.xml @@ -0,0 +1,27 @@ + + VOC2012 + 39.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + dog + Unspecified + 0 + 0 + + 11 + 72 + 243 + 268 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/42.xml b/tests/ut/data/dataset/testVOC2012/Annotations/42.xml new file mode 100644 index 00000000000..ff5da110178 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/42.xml @@ -0,0 +1,27 @@ + + VOC2012 + 42.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 335 + 3 + + 1 + + person + Unspecified + 1 + 0 + + 153 + 22 + 411 + 236 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/61.xml b/tests/ut/data/dataset/testVOC2012/Annotations/61.xml new file mode 100644 index 00000000000..41b78728cfb --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/61.xml @@ -0,0 +1,39 @@ + + VOC2012 + 61.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 333 + 3 + + 1 + + train + Unspecified + 0 + 0 + + 252 + 42 + 445 + 282 + + + + person + Frontal + 0 + 0 + + 204 + 198 + 271 + 293 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/63.xml b/tests/ut/data/dataset/testVOC2012/Annotations/63.xml new file mode 100644 index 00000000000..fbd11888f82 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/63.xml @@ -0,0 +1,39 @@ + + VOC2012 + 63.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + cat + Unspecified + 0 + 0 + + 115 + 128 + 299 + 355 + + + + chair + Frontal + 1 + 0 + + 36 + 11 + 439 + 499 + + + diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/68.xml b/tests/ut/data/dataset/testVOC2012/Annotations/68.xml new file mode 100644 index 00000000000..00484290f83 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/68.xml @@ -0,0 +1,27 @@ + + VOC2012 + 68.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + cat + Unspecified + 1 + 0 + + 49 + 22 + 135 + 293 + + + diff --git a/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/train.txt b/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/train.txt new file mode 100644 index 00000000000..8a03056ffcc --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/train.txt @@ -0,0 +1,10 @@ +32 +33 +39 +42 +61 +63 +68 +121 +123 +129 diff --git a/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/trainval.txt b/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/trainval.txt new file mode 100644 index 00000000000..2ce72ab0ae4 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/trainval.txt @@ -0,0 +1,2913 @@ +2007_000032 +2007_000033 +2007_000039 +2007_000042 +2007_000061 +2007_000063 +2007_000068 +2007_000121 +2007_000123 +2007_000129 +2007_000170 +2007_000175 +2007_000187 +2007_000241 +2007_000243 +2007_000250 +2007_000256 +2007_000323 +2007_000332 +2007_000333 +2007_000346 +2007_000363 +2007_000364 +2007_000392 +2007_000452 +2007_000464 +2007_000480 +2007_000491 +2007_000504 +2007_000515 +2007_000528 +2007_000529 +2007_000549 +2007_000559 +2007_000572 +2007_000584 +2007_000629 +2007_000636 +2007_000645 +2007_000648 +2007_000661 +2007_000663 +2007_000676 +2007_000713 +2007_000720 +2007_000727 +2007_000733 +2007_000738 +2007_000762 +2007_000768 +2007_000783 +2007_000793 +2007_000799 +2007_000804 +2007_000822 +2007_000830 +2007_000836 +2007_000837 +2007_000847 +2007_000862 +2007_000876 +2007_000904 +2007_000925 +2007_000999 +2007_001027 +2007_001073 +2007_001149 +2007_001154 +2007_001175 +2007_001185 +2007_001225 +2007_001239 +2007_001284 +2007_001288 +2007_001289 +2007_001299 +2007_001311 +2007_001321 +2007_001340 +2007_001377 +2007_001397 +2007_001408 +2007_001416 +2007_001420 +2007_001423 +2007_001430 +2007_001439 +2007_001457 +2007_001458 +2007_001487 +2007_001526 +2007_001568 +2007_001585 +2007_001586 +2007_001587 +2007_001594 +2007_001595 +2007_001602 +2007_001609 +2007_001630 +2007_001677 +2007_001678 +2007_001698 +2007_001704 +2007_001709 +2007_001717 +2007_001724 +2007_001733 +2007_001761 +2007_001763 +2007_001764 +2007_001774 +2007_001825 +2007_001834 +2007_001857 +2007_001872 +2007_001884 +2007_001901 +2007_001917 +2007_001955 +2007_001960 +2007_002024 +2007_002046 +2007_002055 +2007_002088 +2007_002094 +2007_002099 +2007_002105 +2007_002107 +2007_002119 +2007_002120 +2007_002132 +2007_002142 +2007_002198 +2007_002212 +2007_002216 +2007_002227 +2007_002234 +2007_002260 +2007_002266 +2007_002268 +2007_002273 +2007_002281 +2007_002284 +2007_002293 +2007_002361 +2007_002368 +2007_002370 +2007_002376 +2007_002378 +2007_002387 +2007_002400 +2007_002403 +2007_002412 +2007_002426 +2007_002427 +2007_002445 +2007_002462 +2007_002470 +2007_002488 +2007_002539 +2007_002545 +2007_002565 +2007_002597 +2007_002611 +2007_002618 +2007_002619 +2007_002624 +2007_002639 +2007_002643 +2007_002648 +2007_002668 +2007_002669 +2007_002719 +2007_002728 +2007_002760 +2007_002789 +2007_002823 +2007_002824 +2007_002845 +2007_002852 +2007_002895 +2007_002896 +2007_002903 +2007_002914 +2007_002953 +2007_002954 +2007_002967 +2007_003000 +2007_003011 +2007_003020 +2007_003022 +2007_003051 +2007_003088 +2007_003101 +2007_003106 +2007_003110 +2007_003118 +2007_003131 +2007_003134 +2007_003137 +2007_003143 +2007_003169 +2007_003178 +2007_003188 +2007_003189 +2007_003190 +2007_003191 +2007_003194 +2007_003195 +2007_003201 +2007_003205 +2007_003207 +2007_003251 +2007_003267 +2007_003286 +2007_003330 +2007_003349 +2007_003367 +2007_003373 +2007_003431 +2007_003451 +2007_003499 +2007_003503 +2007_003506 +2007_003525 +2007_003529 +2007_003530 +2007_003541 +2007_003565 +2007_003571 +2007_003580 +2007_003587 +2007_003593 +2007_003604 +2007_003611 +2007_003621 +2007_003668 +2007_003682 +2007_003711 +2007_003714 +2007_003715 +2007_003742 +2007_003778 +2007_003786 +2007_003788 +2007_003815 +2007_003841 +2007_003848 +2007_003861 +2007_003872 +2007_003876 +2007_003889 +2007_003910 +2007_003917 +2007_003957 +2007_003991 +2007_004003 +2007_004009 +2007_004033 +2007_004052 +2007_004065 +2007_004081 +2007_004112 +2007_004121 +2007_004143 +2007_004166 +2007_004189 +2007_004190 +2007_004193 +2007_004241 +2007_004275 +2007_004281 +2007_004289 +2007_004291 +2007_004328 +2007_004380 +2007_004392 +2007_004405 +2007_004423 +2007_004459 +2007_004468 +2007_004476 +2007_004481 +2007_004483 +2007_004500 +2007_004510 +2007_004537 +2007_004538 +2007_004558 +2007_004627 +2007_004644 +2007_004649 +2007_004663 +2007_004705 +2007_004707 +2007_004712 +2007_004722 +2007_004768 +2007_004769 +2007_004810 +2007_004830 +2007_004841 +2007_004856 +2007_004866 +2007_004902 +2007_004948 +2007_004951 +2007_004969 +2007_004988 +2007_004998 +2007_005043 +2007_005058 +2007_005064 +2007_005074 +2007_005086 +2007_005107 +2007_005114 +2007_005124 +2007_005130 +2007_005144 +2007_005149 +2007_005173 +2007_005210 +2007_005212 +2007_005227 +2007_005248 +2007_005262 +2007_005264 +2007_005266 +2007_005273 +2007_005281 +2007_005294 +2007_005296 +2007_005304 +2007_005314 +2007_005331 +2007_005354 +2007_005358 +2007_005360 +2007_005368 +2007_005428 +2007_005430 +2007_005460 +2007_005469 +2007_005509 +2007_005547 +2007_005600 +2007_005608 +2007_005626 +2007_005647 +2007_005688 +2007_005689 +2007_005696 +2007_005702 +2007_005705 +2007_005759 +2007_005790 +2007_005797 +2007_005803 +2007_005813 +2007_005828 +2007_005844 +2007_005845 +2007_005857 +2007_005859 +2007_005878 +2007_005902 +2007_005911 +2007_005915 +2007_005951 +2007_005978 +2007_005988 +2007_005989 +2007_006004 +2007_006028 +2007_006035 +2007_006046 +2007_006066 +2007_006076 +2007_006086 +2007_006117 +2007_006134 +2007_006136 +2007_006151 +2007_006171 +2007_006212 +2007_006232 +2007_006241 +2007_006254 +2007_006260 +2007_006277 +2007_006281 +2007_006303 +2007_006317 +2007_006348 +2007_006364 +2007_006373 +2007_006400 +2007_006409 +2007_006444 +2007_006445 +2007_006449 +2007_006477 +2007_006483 +2007_006490 +2007_006530 +2007_006549 +2007_006553 +2007_006560 +2007_006581 +2007_006585 +2007_006605 +2007_006615 +2007_006641 +2007_006647 +2007_006660 +2007_006661 +2007_006673 +2007_006678 +2007_006680 +2007_006698 +2007_006699 +2007_006704 +2007_006761 +2007_006802 +2007_006803 +2007_006832 +2007_006837 +2007_006841 +2007_006864 +2007_006865 +2007_006866 +2007_006899 +2007_006900 +2007_006944 +2007_006946 +2007_007003 +2007_007007 +2007_007021 +2007_007048 +2007_007084 +2007_007098 +2007_007109 +2007_007130 +2007_007154 +2007_007165 +2007_007168 +2007_007195 +2007_007196 +2007_007203 +2007_007211 +2007_007230 +2007_007235 +2007_007250 +2007_007341 +2007_007355 +2007_007387 +2007_007398 +2007_007414 +2007_007415 +2007_007417 +2007_007432 +2007_007447 +2007_007470 +2007_007477 +2007_007480 +2007_007481 +2007_007493 +2007_007498 +2007_007523 +2007_007524 +2007_007530 +2007_007534 +2007_007585 +2007_007591 +2007_007621 +2007_007624 +2007_007649 +2007_007651 +2007_007688 +2007_007698 +2007_007726 +2007_007748 +2007_007772 +2007_007773 +2007_007783 +2007_007795 +2007_007810 +2007_007815 +2007_007818 +2007_007836 +2007_007849 +2007_007878 +2007_007881 +2007_007890 +2007_007891 +2007_007902 +2007_007908 +2007_007930 +2007_007947 +2007_007948 +2007_007996 +2007_008043 +2007_008051 +2007_008072 +2007_008084 +2007_008085 +2007_008106 +2007_008110 +2007_008140 +2007_008142 +2007_008203 +2007_008204 +2007_008218 +2007_008219 +2007_008222 +2007_008256 +2007_008260 +2007_008307 +2007_008339 +2007_008374 +2007_008403 +2007_008407 +2007_008415 +2007_008430 +2007_008468 +2007_008526 +2007_008543 +2007_008547 +2007_008571 +2007_008575 +2007_008596 +2007_008645 +2007_008670 +2007_008708 +2007_008714 +2007_008722 +2007_008747 +2007_008764 +2007_008778 +2007_008801 +2007_008802 +2007_008815 +2007_008821 +2007_008897 +2007_008927 +2007_008932 +2007_008944 +2007_008945 +2007_008948 +2007_008964 +2007_008973 +2007_008980 +2007_008994 +2007_009015 +2007_009030 +2007_009052 +2007_009068 +2007_009082 +2007_009084 +2007_009088 +2007_009096 +2007_009139 +2007_009209 +2007_009216 +2007_009221 +2007_009245 +2007_009251 +2007_009252 +2007_009258 +2007_009295 +2007_009320 +2007_009322 +2007_009323 +2007_009327 +2007_009331 +2007_009346 +2007_009348 +2007_009392 +2007_009413 +2007_009419 +2007_009422 +2007_009435 +2007_009436 +2007_009446 +2007_009458 +2007_009464 +2007_009521 +2007_009527 +2007_009533 +2007_009550 +2007_009554 +2007_009562 +2007_009580 +2007_009592 +2007_009594 +2007_009597 +2007_009605 +2007_009607 +2007_009618 +2007_009630 +2007_009649 +2007_009654 +2007_009655 +2007_009665 +2007_009684 +2007_009687 +2007_009691 +2007_009706 +2007_009709 +2007_009724 +2007_009750 +2007_009756 +2007_009759 +2007_009764 +2007_009779 +2007_009788 +2007_009794 +2007_009807 +2007_009817 +2007_009832 +2007_009841 +2007_009889 +2007_009897 +2007_009899 +2007_009901 +2007_009911 +2007_009923 +2007_009938 +2007_009947 +2007_009950 +2008_000009 +2008_000015 +2008_000016 +2008_000019 +2008_000028 +2008_000033 +2008_000073 +2008_000074 +2008_000075 +2008_000080 +2008_000089 +2008_000103 +2008_000105 +2008_000107 +2008_000120 +2008_000123 +2008_000131 +2008_000144 +2008_000149 +2008_000162 +2008_000182 +2008_000187 +2008_000188 +2008_000197 +2008_000207 +2008_000213 +2008_000215 +2008_000217 +2008_000223 +2008_000226 +2008_000233 +2008_000234 +2008_000235 +2008_000238 +2008_000239 +2008_000254 +2008_000259 +2008_000270 +2008_000271 +2008_000273 +2008_000284 +2008_000287 +2008_000289 +2008_000290 +2008_000309 +2008_000316 +2008_000336 +2008_000345 +2008_000348 +2008_000359 +2008_000361 +2008_000365 +2008_000391 +2008_000399 +2008_000400 +2008_000401 +2008_000415 +2008_000422 +2008_000436 +2008_000464 +2008_000469 +2008_000470 +2008_000474 +2008_000491 +2008_000495 +2008_000501 +2008_000505 +2008_000510 +2008_000515 +2008_000533 +2008_000540 +2008_000544 +2008_000567 +2008_000573 +2008_000578 +2008_000584 +2008_000588 +2008_000589 +2008_000595 +2008_000602 +2008_000626 +2008_000630 +2008_000645 +2008_000657 +2008_000661 +2008_000662 +2008_000666 +2008_000673 +2008_000676 +2008_000696 +2008_000700 +2008_000711 +2008_000716 +2008_000725 +2008_000731 +2008_000733 +2008_000760 +2008_000763 +2008_000764 +2008_000765 +2008_000778 +2008_000782 +2008_000785 +2008_000795 +2008_000811 +2008_000832 +2008_000841 +2008_000848 +2008_000853 +2008_000860 +2008_000861 +2008_000863 +2008_000870 +2008_000911 +2008_000919 +2008_000923 +2008_000943 +2008_000992 +2008_001013 +2008_001028 +2008_001030 +2008_001040 +2008_001056 +2008_001070 +2008_001074 +2008_001076 +2008_001078 +2008_001106 +2008_001112 +2008_001118 +2008_001119 +2008_001135 +2008_001137 +2008_001150 +2008_001159 +2008_001169 +2008_001170 +2008_001188 +2008_001203 +2008_001208 +2008_001215 +2008_001231 +2008_001235 +2008_001245 +2008_001249 +2008_001260 +2008_001263 +2008_001274 +2008_001283 +2008_001308 +2008_001358 +2008_001375 +2008_001379 +2008_001387 +2008_001399 +2008_001402 +2008_001404 +2008_001408 +2008_001413 +2008_001433 +2008_001439 +2008_001462 +2008_001467 +2008_001478 +2008_001479 +2008_001491 +2008_001498 +2008_001504 +2008_001510 +2008_001513 +2008_001514 +2008_001523 +2008_001531 +2008_001546 +2008_001547 +2008_001566 +2008_001580 +2008_001592 +2008_001601 +2008_001610 +2008_001629 +2008_001632 +2008_001640 +2008_001643 +2008_001682 +2008_001688 +2008_001691 +2008_001715 +2008_001716 +2008_001719 +2008_001741 +2008_001761 +2008_001787 +2008_001821 +2008_001829 +2008_001874 +2008_001876 +2008_001882 +2008_001885 +2008_001895 +2008_001896 +2008_001926 +2008_001966 +2008_001971 +2008_001992 +2008_001997 +2008_002032 +2008_002043 +2008_002064 +2008_002066 +2008_002067 +2008_002073 +2008_002079 +2008_002080 +2008_002123 +2008_002152 +2008_002160 +2008_002175 +2008_002177 +2008_002182 +2008_002200 +2008_002205 +2008_002210 +2008_002212 +2008_002215 +2008_002218 +2008_002221 +2008_002239 +2008_002240 +2008_002241 +2008_002247 +2008_002248 +2008_002255 +2008_002258 +2008_002269 +2008_002273 +2008_002288 +2008_002338 +2008_002358 +2008_002379 +2008_002383 +2008_002411 +2008_002425 +2008_002429 +2008_002464 +2008_002467 +2008_002471 +2008_002473 +2008_002492 +2008_002495 +2008_002504 +2008_002521 +2008_002536 +2008_002551 +2008_002588 +2008_002623 +2008_002641 +2008_002650 +2008_002680 +2008_002681 +2008_002697 +2008_002704 +2008_002710 +2008_002719 +2008_002749 +2008_002762 +2008_002772 +2008_002775 +2008_002778 +2008_002834 +2008_002835 +2008_002859 +2008_002864 +2008_002868 +2008_002885 +2008_002894 +2008_002900 +2008_002904 +2008_002929 +2008_002936 +2008_002942 +2008_002958 +2008_002960 +2008_002970 +2008_002972 +2008_002993 +2008_003003 +2008_003026 +2008_003034 +2008_003060 +2008_003065 +2008_003068 +2008_003076 +2008_003083 +2008_003087 +2008_003094 +2008_003101 +2008_003105 +2008_003108 +2008_003110 +2008_003135 +2008_003141 +2008_003155 +2008_003168 +2008_003180 +2008_003196 +2008_003200 +2008_003208 +2008_003210 +2008_003238 +2008_003252 +2008_003270 +2008_003329 +2008_003330 +2008_003333 +2008_003362 +2008_003369 +2008_003373 +2008_003379 +2008_003381 +2008_003415 +2008_003429 +2008_003451 +2008_003461 +2008_003477 +2008_003480 +2008_003492 +2008_003499 +2008_003500 +2008_003511 +2008_003523 +2008_003546 +2008_003562 +2008_003576 +2008_003577 +2008_003585 +2008_003665 +2008_003676 +2008_003691 +2008_003701 +2008_003703 +2008_003709 +2008_003729 +2008_003733 +2008_003769 +2008_003774 +2008_003777 +2008_003779 +2008_003782 +2008_003814 +2008_003821 +2008_003846 +2008_003856 +2008_003858 +2008_003874 +2008_003876 +2008_003885 +2008_003886 +2008_003913 +2008_003926 +2008_003939 +2008_003947 +2008_003976 +2008_003986 +2008_003998 +2008_004014 +2008_004026 +2008_004055 +2008_004069 +2008_004080 +2008_004097 +2008_004101 +2008_004112 +2008_004140 +2008_004172 +2008_004175 +2008_004212 +2008_004259 +2008_004279 +2008_004321 +2008_004339 +2008_004345 +2008_004358 +2008_004363 +2008_004365 +2008_004367 +2008_004396 +2008_004399 +2008_004416 +2008_004430 +2008_004441 +2008_004453 +2008_004477 +2008_004547 +2008_004551 +2008_004552 +2008_004562 +2008_004575 +2008_004583 +2008_004588 +2008_004607 +2008_004610 +2008_004612 +2008_004621 +2008_004624 +2008_004654 +2008_004659 +2008_004663 +2008_004687 +2008_004701 +2008_004704 +2008_004705 +2008_004750 +2008_004754 +2008_004758 +2008_004776 +2008_004822 +2008_004838 +2008_004841 +2008_004854 +2008_004869 +2008_004892 +2008_004910 +2008_004911 +2008_004914 +2008_004946 +2008_004983 +2008_004995 +2008_005006 +2008_005049 +2008_005074 +2008_005089 +2008_005097 +2008_005105 +2008_005145 +2008_005196 +2008_005197 +2008_005214 +2008_005217 +2008_005231 +2008_005242 +2008_005245 +2008_005254 +2008_005262 +2008_005266 +2008_005294 +2008_005300 +2008_005321 +2008_005338 +2008_005342 +2008_005345 +2008_005367 +2008_005375 +2008_005398 +2008_005399 +2008_005422 +2008_005439 +2008_005445 +2008_005512 +2008_005525 +2008_005541 +2008_005544 +2008_005600 +2008_005628 +2008_005633 +2008_005637 +2008_005642 +2008_005650 +2008_005668 +2008_005676 +2008_005678 +2008_005679 +2008_005680 +2008_005691 +2008_005698 +2008_005706 +2008_005713 +2008_005714 +2008_005716 +2008_005727 +2008_005738 +2008_005747 +2008_005770 +2008_005812 +2008_005839 +2008_005843 +2008_005845 +2008_005874 +2008_005904 +2008_005915 +2008_005926 +2008_005938 +2008_005945 +2008_005953 +2008_006008 +2008_006032 +2008_006036 +2008_006055 +2008_006063 +2008_006065 +2008_006070 +2008_006108 +2008_006130 +2008_006140 +2008_006143 +2008_006159 +2008_006182 +2008_006213 +2008_006215 +2008_006216 +2008_006219 +2008_006221 +2008_006229 +2008_006254 +2008_006275 +2008_006289 +2008_006325 +2008_006327 +2008_006339 +2008_006341 +2008_006345 +2008_006349 +2008_006353 +2008_006389 +2008_006408 +2008_006434 +2008_006480 +2008_006481 +2008_006482 +2008_006490 +2008_006509 +2008_006523 +2008_006526 +2008_006528 +2008_006553 +2008_006554 +2008_006558 +2008_006655 +2008_006703 +2008_006722 +2008_006748 +2008_006751 +2008_006752 +2008_006784 +2008_006835 +2008_006843 +2008_006873 +2008_006874 +2008_006877 +2008_006908 +2008_006920 +2008_006981 +2008_006986 +2008_007011 +2008_007012 +2008_007025 +2008_007031 +2008_007048 +2008_007090 +2008_007120 +2008_007123 +2008_007142 +2008_007143 +2008_007165 +2008_007194 +2008_007201 +2008_007219 +2008_007239 +2008_007242 +2008_007245 +2008_007273 +2008_007313 +2008_007350 +2008_007355 +2008_007357 +2008_007375 +2008_007378 +2008_007392 +2008_007402 +2008_007428 +2008_007433 +2008_007472 +2008_007497 +2008_007498 +2008_007507 +2008_007513 +2008_007527 +2008_007548 +2008_007581 +2008_007596 +2008_007677 +2008_007691 +2008_007737 +2008_007759 +2008_007797 +2008_007804 +2008_007811 +2008_007814 +2008_007828 +2008_007836 +2008_007858 +2008_007945 +2008_007994 +2008_007998 +2008_008051 +2008_008103 +2008_008106 +2008_008127 +2008_008193 +2008_008221 +2008_008252 +2008_008263 +2008_008268 +2008_008296 +2008_008301 +2008_008323 +2008_008324 +2008_008335 +2008_008343 +2008_008362 +2008_008392 +2008_008393 +2008_008421 +2008_008434 +2008_008462 +2008_008469 +2008_008476 +2008_008511 +2008_008521 +2008_008525 +2008_008541 +2008_008545 +2008_008550 +2008_008629 +2008_008682 +2008_008711 +2008_008746 +2008_008770 +2008_008773 +2009_000006 +2009_000012 +2009_000013 +2009_000015 +2009_000022 +2009_000028 +2009_000029 +2009_000032 +2009_000037 +2009_000039 +2009_000073 +2009_000074 +2009_000080 +2009_000087 +2009_000096 +2009_000100 +2009_000103 +2009_000121 +2009_000133 +2009_000136 +2009_000149 +2009_000156 +2009_000161 +2009_000176 +2009_000177 +2009_000201 +2009_000205 +2009_000219 +2009_000242 +2009_000250 +2009_000285 +2009_000309 +2009_000318 +2009_000335 +2009_000347 +2009_000351 +2009_000354 +2009_000385 +2009_000387 +2009_000391 +2009_000400 +2009_000405 +2009_000408 +2009_000409 +2009_000412 +2009_000418 +2009_000420 +2009_000421 +2009_000426 +2009_000440 +2009_000444 +2009_000446 +2009_000454 +2009_000455 +2009_000457 +2009_000469 +2009_000487 +2009_000488 +2009_000503 +2009_000505 +2009_000523 +2009_000532 +2009_000535 +2009_000544 +2009_000553 +2009_000562 +2009_000573 +2009_000603 +2009_000619 +2009_000626 +2009_000628 +2009_000635 +2009_000641 +2009_000655 +2009_000662 +2009_000664 +2009_000675 +2009_000684 +2009_000690 +2009_000704 +2009_000705 +2009_000709 +2009_000712 +2009_000716 +2009_000720 +2009_000723 +2009_000727 +2009_000730 +2009_000731 +2009_000732 +2009_000744 +2009_000746 +2009_000771 +2009_000774 +2009_000801 +2009_000825 +2009_000828 +2009_000839 +2009_000840 +2009_000845 +2009_000879 +2009_000887 +2009_000892 +2009_000894 +2009_000895 +2009_000906 +2009_000919 +2009_000924 +2009_000931 +2009_000935 +2009_000938 +2009_000964 +2009_000987 +2009_000989 +2009_000991 +2009_000996 +2009_000998 +2009_001002 +2009_001008 +2009_001019 +2009_001027 +2009_001036 +2009_001070 +2009_001082 +2009_001085 +2009_001095 +2009_001096 +2009_001100 +2009_001104 +2009_001108 +2009_001117 +2009_001124 +2009_001137 +2009_001140 +2009_001145 +2009_001146 +2009_001160 +2009_001163 +2009_001177 +2009_001197 +2009_001203 +2009_001205 +2009_001215 +2009_001240 +2009_001251 +2009_001253 +2009_001255 +2009_001264 +2009_001268 +2009_001270 +2009_001278 +2009_001283 +2009_001299 +2009_001300 +2009_001306 +2009_001311 +2009_001314 +2009_001332 +2009_001333 +2009_001339 +2009_001359 +2009_001363 +2009_001385 +2009_001388 +2009_001390 +2009_001391 +2009_001403 +2009_001411 +2009_001422 +2009_001433 +2009_001443 +2009_001444 +2009_001481 +2009_001502 +2009_001505 +2009_001514 +2009_001516 +2009_001535 +2009_001536 +2009_001544 +2009_001565 +2009_001607 +2009_001615 +2009_001625 +2009_001636 +2009_001640 +2009_001644 +2009_001651 +2009_001663 +2009_001664 +2009_001683 +2009_001684 +2009_001687 +2009_001690 +2009_001693 +2009_001718 +2009_001724 +2009_001731 +2009_001735 +2009_001744 +2009_001755 +2009_001765 +2009_001768 +2009_001775 +2009_001782 +2009_001783 +2009_001802 +2009_001804 +2009_001816 +2009_001818 +2009_001828 +2009_001850 +2009_001851 +2009_001854 +2009_001868 +2009_001871 +2009_001885 +2009_001888 +2009_001894 +2009_001898 +2009_001922 +2009_001937 +2009_001941 +2009_001961 +2009_001964 +2009_001972 +2009_001991 +2009_002010 +2009_002012 +2009_002019 +2009_002035 +2009_002042 +2009_002052 +2009_002060 +2009_002072 +2009_002082 +2009_002083 +2009_002094 +2009_002097 +2009_002117 +2009_002122 +2009_002150 +2009_002153 +2009_002155 +2009_002164 +2009_002165 +2009_002171 +2009_002185 +2009_002202 +2009_002204 +2009_002216 +2009_002221 +2009_002229 +2009_002238 +2009_002239 +2009_002245 +2009_002262 +2009_002264 +2009_002265 +2009_002268 +2009_002281 +2009_002285 +2009_002291 +2009_002295 +2009_002314 +2009_002317 +2009_002320 +2009_002343 +2009_002346 +2009_002362 +2009_002366 +2009_002372 +2009_002382 +2009_002387 +2009_002390 +2009_002409 +2009_002415 +2009_002416 +2009_002419 +2009_002422 +2009_002423 +2009_002425 +2009_002445 +2009_002448 +2009_002460 +2009_002472 +2009_002487 +2009_002519 +2009_002521 +2009_002527 +2009_002530 +2009_002535 +2009_002539 +2009_002543 +2009_002549 +2009_002562 +2009_002567 +2009_002568 +2009_002571 +2009_002573 +2009_002584 +2009_002586 +2009_002588 +2009_002591 +2009_002594 +2009_002599 +2009_002604 +2009_002613 +2009_002618 +2009_002626 +2009_002628 +2009_002635 +2009_002638 +2009_002649 +2009_002651 +2009_002662 +2009_002674 +2009_002713 +2009_002715 +2009_002727 +2009_002732 +2009_002734 +2009_002749 +2009_002753 +2009_002763 +2009_002771 +2009_002789 +2009_002808 +2009_002820 +2009_002844 +2009_002845 +2009_002849 +2009_002856 +2009_002862 +2009_002872 +2009_002885 +2009_002887 +2009_002888 +2009_002897 +2009_002912 +2009_002914 +2009_002917 +2009_002928 +2009_002932 +2009_002936 +2009_002972 +2009_002975 +2009_002982 +2009_002984 +2009_002988 +2009_002990 +2009_002993 +2009_003003 +2009_003005 +2009_003006 +2009_003007 +2009_003012 +2009_003034 +2009_003035 +2009_003039 +2009_003043 +2009_003053 +2009_003054 +2009_003059 +2009_003063 +2009_003065 +2009_003071 +2009_003075 +2009_003080 +2009_003087 +2009_003088 +2009_003090 +2009_003105 +2009_003123 +2009_003142 +2009_003146 +2009_003147 +2009_003164 +2009_003172 +2009_003193 +2009_003196 +2009_003200 +2009_003217 +2009_003224 +2009_003241 +2009_003249 +2009_003269 +2009_003273 +2009_003299 +2009_003304 +2009_003311 +2009_003317 +2009_003323 +2009_003340 +2009_003343 +2009_003345 +2009_003353 +2009_003361 +2009_003369 +2009_003378 +2009_003387 +2009_003406 +2009_003433 +2009_003450 +2009_003455 +2009_003461 +2009_003466 +2009_003468 +2009_003481 +2009_003494 +2009_003497 +2009_003498 +2009_003504 +2009_003507 +2009_003517 +2009_003519 +2009_003522 +2009_003523 +2009_003539 +2009_003542 +2009_003549 +2009_003551 +2009_003555 +2009_003564 +2009_003569 +2009_003576 +2009_003589 +2009_003607 +2009_003613 +2009_003636 +2009_003640 +2009_003646 +2009_003660 +2009_003666 +2009_003690 +2009_003696 +2009_003697 +2009_003703 +2009_003707 +2009_003711 +2009_003734 +2009_003736 +2009_003756 +2009_003757 +2009_003768 +2009_003771 +2009_003773 +2009_003783 +2009_003799 +2009_003804 +2009_003806 +2009_003810 +2009_003815 +2009_003820 +2009_003825 +2009_003849 +2009_003857 +2009_003858 +2009_003860 +2009_003865 +2009_003895 +2009_003903 +2009_003904 +2009_003921 +2009_003922 +2009_003928 +2009_003933 +2009_003938 +2009_003961 +2009_003971 +2009_003975 +2009_003991 +2009_004021 +2009_004033 +2009_004043 +2009_004070 +2009_004072 +2009_004084 +2009_004091 +2009_004095 +2009_004099 +2009_004105 +2009_004117 +2009_004125 +2009_004140 +2009_004171 +2009_004178 +2009_004180 +2009_004186 +2009_004191 +2009_004212 +2009_004213 +2009_004217 +2009_004221 +2009_004228 +2009_004247 +2009_004248 +2009_004249 +2009_004255 +2009_004264 +2009_004278 +2009_004298 +2009_004301 +2009_004316 +2009_004317 +2009_004324 +2009_004327 +2009_004328 +2009_004334 +2009_004336 +2009_004368 +2009_004374 +2009_004409 +2009_004417 +2009_004425 +2009_004426 +2009_004434 +2009_004446 +2009_004455 +2009_004464 +2009_004479 +2009_004494 +2009_004497 +2009_004504 +2009_004507 +2009_004509 +2009_004519 +2009_004539 +2009_004540 +2009_004561 +2009_004568 +2009_004579 +2009_004581 +2009_004590 +2009_004592 +2009_004594 +2009_004620 +2009_004626 +2009_004635 +2009_004643 +2009_004653 +2009_004656 +2009_004661 +2009_004674 +2009_004687 +2009_004705 +2009_004721 +2009_004730 +2009_004732 +2009_004738 +2009_004748 +2009_004789 +2009_004790 +2009_004799 +2009_004801 +2009_004805 +2009_004829 +2009_004848 +2009_004859 +2009_004867 +2009_004882 +2009_004886 +2009_004887 +2009_004888 +2009_004890 +2009_004895 +2009_004901 +2009_004904 +2009_004919 +2009_004939 +2009_004942 +2009_004969 +2009_004980 +2009_004987 +2009_004990 +2009_004993 +2009_004994 +2009_005000 +2009_005016 +2009_005031 +2009_005037 +2009_005038 +2009_005055 +2009_005056 +2009_005069 +2009_005078 +2009_005084 +2009_005085 +2009_005087 +2009_005089 +2009_005107 +2009_005118 +2009_005120 +2009_005128 +2009_005130 +2009_005137 +2009_005141 +2009_005145 +2009_005148 +2009_005156 +2009_005158 +2009_005160 +2009_005177 +2009_005189 +2009_005190 +2009_005194 +2009_005217 +2009_005219 +2009_005220 +2009_005231 +2009_005234 +2009_005236 +2009_005247 +2009_005260 +2009_005262 +2009_005269 +2009_005287 +2009_005302 +2010_000002 +2010_000003 +2010_000038 +2010_000043 +2010_000063 +2010_000065 +2010_000075 +2010_000076 +2010_000083 +2010_000084 +2010_000087 +2010_000110 +2010_000114 +2010_000117 +2010_000131 +2010_000132 +2010_000148 +2010_000159 +2010_000160 +2010_000163 +2010_000174 +2010_000187 +2010_000189 +2010_000195 +2010_000216 +2010_000238 +2010_000241 +2010_000256 +2010_000269 +2010_000272 +2010_000284 +2010_000285 +2010_000309 +2010_000318 +2010_000330 +2010_000335 +2010_000342 +2010_000371 +2010_000372 +2010_000392 +2010_000404 +2010_000422 +2010_000426 +2010_000427 +2010_000436 +2010_000437 +2010_000466 +2010_000469 +2010_000492 +2010_000498 +2010_000502 +2010_000503 +2010_000519 +2010_000530 +2010_000552 +2010_000559 +2010_000567 +2010_000572 +2010_000573 +2010_000588 +2010_000622 +2010_000628 +2010_000632 +2010_000639 +2010_000661 +2010_000666 +2010_000675 +2010_000679 +2010_000682 +2010_000683 +2010_000685 +2010_000724 +2010_000738 +2010_000746 +2010_000748 +2010_000764 +2010_000772 +2010_000787 +2010_000788 +2010_000810 +2010_000814 +2010_000815 +2010_000836 +2010_000847 +2010_000855 +2010_000874 +2010_000885 +2010_000887 +2010_000904 +2010_000906 +2010_000907 +2010_000918 +2010_000929 +2010_000941 +2010_000952 +2010_000961 +2010_000978 +2010_000986 +2010_001000 +2010_001010 +2010_001011 +2010_001016 +2010_001017 +2010_001024 +2010_001036 +2010_001043 +2010_001061 +2010_001069 +2010_001070 +2010_001079 +2010_001104 +2010_001120 +2010_001124 +2010_001131 +2010_001149 +2010_001151 +2010_001154 +2010_001160 +2010_001174 +2010_001177 +2010_001183 +2010_001184 +2010_001195 +2010_001206 +2010_001245 +2010_001246 +2010_001247 +2010_001251 +2010_001256 +2010_001261 +2010_001264 +2010_001273 +2010_001279 +2010_001282 +2010_001292 +2010_001313 +2010_001327 +2010_001329 +2010_001331 +2010_001347 +2010_001351 +2010_001367 +2010_001374 +2010_001376 +2010_001386 +2010_001399 +2010_001403 +2010_001413 +2010_001418 +2010_001422 +2010_001448 +2010_001451 +2010_001457 +2010_001514 +2010_001515 +2010_001522 +2010_001534 +2010_001553 +2010_001557 +2010_001561 +2010_001562 +2010_001563 +2010_001576 +2010_001577 +2010_001579 +2010_001590 +2010_001595 +2010_001618 +2010_001619 +2010_001630 +2010_001646 +2010_001656 +2010_001660 +2010_001676 +2010_001692 +2010_001699 +2010_001706 +2010_001732 +2010_001734 +2010_001748 +2010_001752 +2010_001767 +2010_001768 +2010_001773 +2010_001807 +2010_001820 +2010_001830 +2010_001842 +2010_001849 +2010_001850 +2010_001851 +2010_001852 +2010_001860 +2010_001908 +2010_001913 +2010_001922 +2010_001923 +2010_001933 +2010_001939 +2010_001944 +2010_001951 +2010_001956 +2010_001962 +2010_001966 +2010_001995 +2010_002017 +2010_002018 +2010_002020 +2010_002025 +2010_002030 +2010_002032 +2010_002039 +2010_002047 +2010_002054 +2010_002055 +2010_002070 +2010_002097 +2010_002106 +2010_002107 +2010_002137 +2010_002139 +2010_002142 +2010_002146 +2010_002147 +2010_002150 +2010_002154 +2010_002161 +2010_002166 +2010_002200 +2010_002203 +2010_002218 +2010_002228 +2010_002232 +2010_002236 +2010_002251 +2010_002254 +2010_002271 +2010_002286 +2010_002305 +2010_002310 +2010_002336 +2010_002338 +2010_002348 +2010_002361 +2010_002363 +2010_002379 +2010_002382 +2010_002387 +2010_002390 +2010_002396 +2010_002413 +2010_002418 +2010_002422 +2010_002440 +2010_002450 +2010_002455 +2010_002457 +2010_002480 +2010_002499 +2010_002512 +2010_002527 +2010_002531 +2010_002532 +2010_002536 +2010_002538 +2010_002546 +2010_002551 +2010_002556 +2010_002570 +2010_002573 +2010_002623 +2010_002625 +2010_002659 +2010_002682 +2010_002691 +2010_002693 +2010_002697 +2010_002701 +2010_002720 +2010_002733 +2010_002750 +2010_002763 +2010_002778 +2010_002786 +2010_002792 +2010_002794 +2010_002811 +2010_002815 +2010_002838 +2010_002856 +2010_002868 +2010_002870 +2010_002892 +2010_002900 +2010_002902 +2010_002907 +2010_002921 +2010_002929 +2010_002935 +2010_002937 +2010_002938 +2010_002939 +2010_002962 +2010_002973 +2010_002988 +2010_003010 +2010_003014 +2010_003017 +2010_003060 +2010_003062 +2010_003088 +2010_003093 +2010_003097 +2010_003114 +2010_003119 +2010_003123 +2010_003127 +2010_003132 +2010_003153 +2010_003157 +2010_003168 +2010_003170 +2010_003174 +2010_003183 +2010_003187 +2010_003203 +2010_003207 +2010_003230 +2010_003231 +2010_003239 +2010_003250 +2010_003252 +2010_003269 +2010_003274 +2010_003275 +2010_003276 +2010_003293 +2010_003302 +2010_003325 +2010_003342 +2010_003345 +2010_003362 +2010_003365 +2010_003380 +2010_003381 +2010_003383 +2010_003384 +2010_003402 +2010_003409 +2010_003418 +2010_003446 +2010_003453 +2010_003468 +2010_003473 +2010_003495 +2010_003506 +2010_003514 +2010_003529 +2010_003531 +2010_003532 +2010_003534 +2010_003541 +2010_003547 +2010_003597 +2010_003599 +2010_003634 +2010_003651 +2010_003665 +2010_003670 +2010_003675 +2010_003680 +2010_003696 +2010_003708 +2010_003716 +2010_003717 +2010_003737 +2010_003746 +2010_003758 +2010_003764 +2010_003768 +2010_003771 +2010_003772 +2010_003781 +2010_003798 +2010_003799 +2010_003813 +2010_003820 +2010_003854 +2010_003884 +2010_003887 +2010_003894 +2010_003899 +2010_003911 +2010_003912 +2010_003915 +2010_003925 +2010_003947 +2010_003950 +2010_003954 +2010_003956 +2010_003958 +2010_003971 +2010_003974 +2010_004005 +2010_004025 +2010_004041 +2010_004042 +2010_004056 +2010_004060 +2010_004063 +2010_004069 +2010_004071 +2010_004072 +2010_004074 +2010_004104 +2010_004109 +2010_004119 +2010_004120 +2010_004144 +2010_004149 +2010_004154 +2010_004165 +2010_004171 +2010_004180 +2010_004186 +2010_004208 +2010_004210 +2010_004219 +2010_004222 +2010_004226 +2010_004258 +2010_004283 +2010_004288 +2010_004289 +2010_004306 +2010_004314 +2010_004320 +2010_004322 +2010_004337 +2010_004348 +2010_004355 +2010_004361 +2010_004363 +2010_004365 +2010_004369 +2010_004370 +2010_004382 +2010_004419 +2010_004429 +2010_004432 +2010_004450 +2010_004472 +2010_004478 +2010_004479 +2010_004481 +2010_004493 +2010_004499 +2010_004519 +2010_004520 +2010_004529 +2010_004540 +2010_004543 +2010_004550 +2010_004551 +2010_004556 +2010_004559 +2010_004560 +2010_004577 +2010_004598 +2010_004616 +2010_004620 +2010_004625 +2010_004628 +2010_004635 +2010_004662 +2010_004669 +2010_004683 +2010_004694 +2010_004697 +2010_004704 +2010_004721 +2010_004757 +2010_004760 +2010_004763 +2010_004766 +2010_004772 +2010_004773 +2010_004783 +2010_004789 +2010_004795 +2010_004805 +2010_004808 +2010_004815 +2010_004825 +2010_004828 +2010_004856 +2010_004857 +2010_004861 +2010_004900 +2010_004916 +2010_004933 +2010_004938 +2010_004941 +2010_004946 +2010_004948 +2010_004951 +2010_004960 +2010_004963 +2010_004980 +2010_004994 +2010_005013 +2010_005016 +2010_005021 +2010_005028 +2010_005046 +2010_005055 +2010_005063 +2010_005064 +2010_005098 +2010_005106 +2010_005108 +2010_005111 +2010_005118 +2010_005119 +2010_005128 +2010_005129 +2010_005159 +2010_005160 +2010_005166 +2010_005174 +2010_005180 +2010_005187 +2010_005198 +2010_005202 +2010_005206 +2010_005217 +2010_005223 +2010_005232 +2010_005245 +2010_005252 +2010_005277 +2010_005284 +2010_005305 +2010_005317 +2010_005318 +2010_005344 +2010_005353 +2010_005366 +2010_005401 +2010_005419 +2010_005421 +2010_005428 +2010_005429 +2010_005432 +2010_005433 +2010_005450 +2010_005457 +2010_005468 +2010_005471 +2010_005494 +2010_005496 +2010_005500 +2010_005501 +2010_005505 +2010_005506 +2010_005508 +2010_005513 +2010_005519 +2010_005522 +2010_005531 +2010_005534 +2010_005575 +2010_005582 +2010_005596 +2010_005606 +2010_005626 +2010_005627 +2010_005643 +2010_005644 +2010_005652 +2010_005663 +2010_005664 +2010_005669 +2010_005678 +2010_005700 +2010_005705 +2010_005706 +2010_005709 +2010_005718 +2010_005719 +2010_005721 +2010_005723 +2010_005725 +2010_005727 +2010_005734 +2010_005744 +2010_005746 +2010_005755 +2010_005758 +2010_005762 +2010_005775 +2010_005788 +2010_005791 +2010_005796 +2010_005800 +2010_005805 +2010_005810 +2010_005820 +2010_005830 +2010_005835 +2010_005836 +2010_005860 +2010_005871 +2010_005876 +2010_005877 +2010_005888 +2010_005891 +2010_005898 +2010_005899 +2010_005919 +2010_005922 +2010_005927 +2010_005932 +2010_005951 +2010_005952 +2010_005978 +2010_005982 +2010_005991 +2010_005992 +2010_006009 +2010_006026 +2010_006034 +2010_006054 +2010_006070 +2011_000003 +2011_000006 +2011_000025 +2011_000027 +2011_000045 +2011_000051 +2011_000054 +2011_000066 +2011_000068 +2011_000069 +2011_000070 +2011_000105 +2011_000108 +2011_000112 +2011_000116 +2011_000122 +2011_000145 +2011_000149 +2011_000152 +2011_000173 +2011_000178 +2011_000182 +2011_000185 +2011_000197 +2011_000208 +2011_000216 +2011_000219 +2011_000221 +2011_000222 +2011_000226 +2011_000228 +2011_000234 +2011_000238 +2011_000239 +2011_000243 +2011_000248 +2011_000252 +2011_000258 +2011_000268 +2011_000277 +2011_000278 +2011_000283 +2011_000291 +2011_000293 +2011_000310 +2011_000312 +2011_000338 +2011_000345 +2011_000359 +2011_000379 +2011_000382 +2011_000396 +2011_000400 +2011_000412 +2011_000419 +2011_000428 +2011_000435 +2011_000436 +2011_000438 +2011_000449 +2011_000453 +2011_000455 +2011_000456 +2011_000457 +2011_000468 +2011_000469 +2011_000479 +2011_000481 +2011_000482 +2011_000503 +2011_000512 +2011_000513 +2011_000521 +2011_000526 +2011_000536 +2011_000542 +2011_000548 +2011_000550 +2011_000551 +2011_000556 +2011_000566 +2011_000573 +2011_000577 +2011_000585 +2011_000589 +2011_000594 +2011_000598 +2011_000607 +2011_000618 +2011_000637 +2011_000638 +2011_000641 +2011_000642 +2011_000646 +2011_000651 +2011_000652 +2011_000658 +2011_000661 +2011_000669 +2011_000713 +2011_000747 +2011_000758 +2011_000768 +2011_000771 +2011_000780 +2011_000789 +2011_000790 +2011_000793 +2011_000807 +2011_000809 +2011_000813 +2011_000830 +2011_000834 +2011_000840 +2011_000843 +2011_000874 +2011_000882 +2011_000888 +2011_000893 +2011_000895 +2011_000900 +2011_000912 +2011_000920 +2011_000934 +2011_000944 +2011_000953 +2011_000969 +2011_000973 +2011_000982 +2011_000997 +2011_000999 +2011_001004 +2011_001005 +2011_001014 +2011_001015 +2011_001020 +2011_001027 +2011_001047 +2011_001060 +2011_001064 +2011_001069 +2011_001071 +2011_001082 +2011_001110 +2011_001114 +2011_001133 +2011_001135 +2011_001139 +2011_001159 +2011_001161 +2011_001166 +2011_001175 +2011_001190 +2011_001198 +2011_001211 +2011_001232 +2011_001259 +2011_001263 +2011_001270 +2011_001276 +2011_001281 +2011_001287 +2011_001292 +2011_001313 +2011_001336 +2011_001341 +2011_001346 +2011_001350 +2011_001400 +2011_001402 +2011_001407 +2011_001411 +2011_001412 +2011_001416 +2011_001421 +2011_001432 +2011_001434 +2011_001447 +2011_001463 +2011_001475 +2011_001479 +2011_001489 +2011_001519 +2011_001529 +2011_001530 +2011_001534 +2011_001536 +2011_001542 +2011_001546 +2011_001567 +2011_001571 +2011_001589 +2011_001597 +2011_001601 +2011_001607 +2011_001613 +2011_001614 +2011_001619 +2011_001621 +2011_001622 +2011_001624 +2011_001632 +2011_001642 +2011_001652 +2011_001653 +2011_001665 +2011_001669 +2011_001674 +2011_001695 +2011_001708 +2011_001710 +2011_001713 +2011_001714 +2011_001722 +2011_001726 +2011_001730 +2011_001745 +2011_001748 +2011_001753 +2011_001754 +2011_001764 +2011_001765 +2011_001775 +2011_001782 +2011_001790 +2011_001793 +2011_001794 +2011_001810 +2011_001812 +2011_001855 +2011_001862 +2011_001863 +2011_001866 +2011_001868 +2011_001875 +2011_001880 +2011_001895 +2011_001902 +2011_001904 +2011_001910 +2011_001922 +2011_001924 +2011_001928 +2011_001959 +2011_001967 +2011_001972 +2011_001974 +2011_001984 +2011_001988 +2011_001991 +2011_002002 +2011_002027 +2011_002040 +2011_002041 +2011_002050 +2011_002064 +2011_002075 +2011_002098 +2011_002107 +2011_002110 +2011_002111 +2011_002114 +2011_002119 +2011_002121 +2011_002124 +2011_002134 +2011_002135 +2011_002149 +2011_002150 +2011_002156 +2011_002178 +2011_002200 +2011_002222 +2011_002223 +2011_002224 +2011_002227 +2011_002244 +2011_002246 +2011_002247 +2011_002279 +2011_002291 +2011_002295 +2011_002298 +2011_002300 +2011_002303 +2011_002308 +2011_002317 +2011_002322 +2011_002327 +2011_002335 +2011_002341 +2011_002343 +2011_002350 +2011_002358 +2011_002371 +2011_002379 +2011_002381 +2011_002385 +2011_002389 +2011_002391 +2011_002398 +2011_002410 +2011_002447 +2011_002457 +2011_002464 +2011_002488 +2011_002498 +2011_002503 +2011_002504 +2011_002509 +2011_002511 +2011_002515 +2011_002528 +2011_002532 +2011_002535 +2011_002548 +2011_002553 +2011_002559 +2011_002561 +2011_002575 +2011_002578 +2011_002585 +2011_002589 +2011_002590 +2011_002592 +2011_002623 +2011_002641 +2011_002644 +2011_002652 +2011_002656 +2011_002662 +2011_002675 +2011_002685 +2011_002709 +2011_002713 +2011_002715 +2011_002717 +2011_002730 +2011_002752 +2011_002754 +2011_002767 +2011_002770 +2011_002812 +2011_002834 +2011_002851 +2011_002863 +2011_002872 +2011_002873 +2011_002879 +2011_002885 +2011_002920 +2011_002929 +2011_002932 +2011_002935 +2011_002947 +2011_002951 +2011_002953 +2011_002956 +2011_002975 +2011_002993 +2011_002997 +2011_003003 +2011_003011 +2011_003019 +2011_003025 +2011_003030 +2011_003038 +2011_003055 +2011_003057 +2011_003066 +2011_003078 +2011_003085 +2011_003103 +2011_003114 +2011_003121 +2011_003141 +2011_003145 +2011_003146 +2011_003151 +2011_003182 +2011_003184 +2011_003197 +2011_003205 +2011_003216 +2011_003238 +2011_003240 +2011_003246 +2011_003255 +2011_003256 +2011_003271 diff --git a/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/val.txt b/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/val.txt new file mode 100644 index 00000000000..0c7a2f96b1d --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/ImageSets/Segmentation/val.txt @@ -0,0 +1,1449 @@ +2007_000033 +2007_000042 +2007_000061 +2007_000123 +2007_000129 +2007_000175 +2007_000187 +2007_000323 +2007_000332 +2007_000346 +2007_000452 +2007_000464 +2007_000491 +2007_000529 +2007_000559 +2007_000572 +2007_000629 +2007_000636 +2007_000661 +2007_000663 +2007_000676 +2007_000727 +2007_000762 +2007_000783 +2007_000799 +2007_000804 +2007_000830 +2007_000837 +2007_000847 +2007_000862 +2007_000925 +2007_000999 +2007_001154 +2007_001175 +2007_001239 +2007_001284 +2007_001288 +2007_001289 +2007_001299 +2007_001311 +2007_001321 +2007_001377 +2007_001408 +2007_001423 +2007_001430 +2007_001457 +2007_001458 +2007_001526 +2007_001568 +2007_001585 +2007_001586 +2007_001587 +2007_001594 +2007_001630 +2007_001677 +2007_001678 +2007_001717 +2007_001733 +2007_001761 +2007_001763 +2007_001774 +2007_001884 +2007_001955 +2007_002046 +2007_002094 +2007_002119 +2007_002132 +2007_002260 +2007_002266 +2007_002268 +2007_002284 +2007_002376 +2007_002378 +2007_002387 +2007_002400 +2007_002412 +2007_002426 +2007_002427 +2007_002445 +2007_002470 +2007_002539 +2007_002565 +2007_002597 +2007_002618 +2007_002619 +2007_002624 +2007_002643 +2007_002648 +2007_002719 +2007_002728 +2007_002823 +2007_002824 +2007_002852 +2007_002903 +2007_003011 +2007_003020 +2007_003022 +2007_003051 +2007_003088 +2007_003101 +2007_003106 +2007_003110 +2007_003131 +2007_003134 +2007_003137 +2007_003143 +2007_003169 +2007_003188 +2007_003194 +2007_003195 +2007_003201 +2007_003349 +2007_003367 +2007_003373 +2007_003499 +2007_003503 +2007_003506 +2007_003530 +2007_003571 +2007_003587 +2007_003611 +2007_003621 +2007_003682 +2007_003711 +2007_003714 +2007_003742 +2007_003786 +2007_003841 +2007_003848 +2007_003861 +2007_003872 +2007_003917 +2007_003957 +2007_003991 +2007_004033 +2007_004052 +2007_004112 +2007_004121 +2007_004143 +2007_004189 +2007_004190 +2007_004193 +2007_004241 +2007_004275 +2007_004281 +2007_004380 +2007_004392 +2007_004405 +2007_004468 +2007_004483 +2007_004510 +2007_004538 +2007_004558 +2007_004644 +2007_004649 +2007_004712 +2007_004722 +2007_004856 +2007_004866 +2007_004902 +2007_004969 +2007_005058 +2007_005074 +2007_005107 +2007_005114 +2007_005149 +2007_005173 +2007_005281 +2007_005294 +2007_005296 +2007_005304 +2007_005331 +2007_005354 +2007_005358 +2007_005428 +2007_005460 +2007_005469 +2007_005509 +2007_005547 +2007_005600 +2007_005608 +2007_005626 +2007_005689 +2007_005696 +2007_005705 +2007_005759 +2007_005803 +2007_005813 +2007_005828 +2007_005844 +2007_005845 +2007_005857 +2007_005911 +2007_005915 +2007_005978 +2007_006028 +2007_006035 +2007_006046 +2007_006076 +2007_006086 +2007_006117 +2007_006171 +2007_006241 +2007_006260 +2007_006277 +2007_006348 +2007_006364 +2007_006373 +2007_006444 +2007_006449 +2007_006549 +2007_006553 +2007_006560 +2007_006647 +2007_006678 +2007_006680 +2007_006698 +2007_006761 +2007_006802 +2007_006837 +2007_006841 +2007_006864 +2007_006866 +2007_006946 +2007_007007 +2007_007084 +2007_007109 +2007_007130 +2007_007165 +2007_007168 +2007_007195 +2007_007196 +2007_007203 +2007_007211 +2007_007235 +2007_007341 +2007_007414 +2007_007417 +2007_007470 +2007_007477 +2007_007493 +2007_007498 +2007_007524 +2007_007534 +2007_007624 +2007_007651 +2007_007688 +2007_007748 +2007_007795 +2007_007810 +2007_007815 +2007_007818 +2007_007836 +2007_007849 +2007_007881 +2007_007996 +2007_008051 +2007_008084 +2007_008106 +2007_008110 +2007_008204 +2007_008222 +2007_008256 +2007_008260 +2007_008339 +2007_008374 +2007_008415 +2007_008430 +2007_008543 +2007_008547 +2007_008596 +2007_008645 +2007_008670 +2007_008708 +2007_008722 +2007_008747 +2007_008802 +2007_008815 +2007_008897 +2007_008944 +2007_008964 +2007_008973 +2007_008980 +2007_009015 +2007_009068 +2007_009084 +2007_009088 +2007_009096 +2007_009221 +2007_009245 +2007_009251 +2007_009252 +2007_009258 +2007_009320 +2007_009323 +2007_009331 +2007_009346 +2007_009392 +2007_009413 +2007_009419 +2007_009446 +2007_009458 +2007_009521 +2007_009562 +2007_009592 +2007_009654 +2007_009655 +2007_009684 +2007_009687 +2007_009691 +2007_009706 +2007_009750 +2007_009756 +2007_009764 +2007_009794 +2007_009817 +2007_009841 +2007_009897 +2007_009911 +2007_009923 +2007_009938 +2008_000009 +2008_000016 +2008_000073 +2008_000075 +2008_000080 +2008_000107 +2008_000120 +2008_000123 +2008_000149 +2008_000182 +2008_000213 +2008_000215 +2008_000223 +2008_000233 +2008_000234 +2008_000239 +2008_000254 +2008_000270 +2008_000271 +2008_000345 +2008_000359 +2008_000391 +2008_000401 +2008_000464 +2008_000469 +2008_000474 +2008_000501 +2008_000510 +2008_000533 +2008_000573 +2008_000589 +2008_000602 +2008_000630 +2008_000657 +2008_000661 +2008_000662 +2008_000666 +2008_000673 +2008_000700 +2008_000725 +2008_000731 +2008_000763 +2008_000765 +2008_000782 +2008_000795 +2008_000811 +2008_000848 +2008_000853 +2008_000863 +2008_000911 +2008_000919 +2008_000943 +2008_000992 +2008_001013 +2008_001028 +2008_001040 +2008_001070 +2008_001074 +2008_001076 +2008_001078 +2008_001135 +2008_001150 +2008_001170 +2008_001231 +2008_001249 +2008_001260 +2008_001283 +2008_001308 +2008_001379 +2008_001404 +2008_001433 +2008_001439 +2008_001478 +2008_001491 +2008_001504 +2008_001513 +2008_001514 +2008_001531 +2008_001546 +2008_001547 +2008_001580 +2008_001629 +2008_001640 +2008_001682 +2008_001688 +2008_001715 +2008_001821 +2008_001874 +2008_001885 +2008_001895 +2008_001966 +2008_001971 +2008_001992 +2008_002043 +2008_002152 +2008_002205 +2008_002212 +2008_002239 +2008_002240 +2008_002241 +2008_002269 +2008_002273 +2008_002358 +2008_002379 +2008_002383 +2008_002429 +2008_002464 +2008_002467 +2008_002492 +2008_002495 +2008_002504 +2008_002521 +2008_002536 +2008_002588 +2008_002623 +2008_002680 +2008_002681 +2008_002775 +2008_002778 +2008_002835 +2008_002859 +2008_002864 +2008_002900 +2008_002904 +2008_002929 +2008_002936 +2008_002942 +2008_002958 +2008_003003 +2008_003026 +2008_003034 +2008_003076 +2008_003105 +2008_003108 +2008_003110 +2008_003135 +2008_003141 +2008_003155 +2008_003210 +2008_003238 +2008_003270 +2008_003330 +2008_003333 +2008_003369 +2008_003379 +2008_003451 +2008_003461 +2008_003477 +2008_003492 +2008_003499 +2008_003511 +2008_003546 +2008_003576 +2008_003577 +2008_003676 +2008_003709 +2008_003733 +2008_003777 +2008_003782 +2008_003821 +2008_003846 +2008_003856 +2008_003858 +2008_003874 +2008_003876 +2008_003885 +2008_003886 +2008_003926 +2008_003976 +2008_004069 +2008_004101 +2008_004140 +2008_004172 +2008_004175 +2008_004212 +2008_004279 +2008_004339 +2008_004345 +2008_004363 +2008_004367 +2008_004396 +2008_004399 +2008_004453 +2008_004477 +2008_004552 +2008_004562 +2008_004575 +2008_004610 +2008_004612 +2008_004621 +2008_004624 +2008_004654 +2008_004659 +2008_004687 +2008_004701 +2008_004704 +2008_004705 +2008_004754 +2008_004758 +2008_004854 +2008_004910 +2008_004995 +2008_005049 +2008_005089 +2008_005097 +2008_005105 +2008_005145 +2008_005197 +2008_005217 +2008_005242 +2008_005245 +2008_005254 +2008_005262 +2008_005338 +2008_005398 +2008_005399 +2008_005422 +2008_005439 +2008_005445 +2008_005525 +2008_005544 +2008_005628 +2008_005633 +2008_005637 +2008_005642 +2008_005676 +2008_005680 +2008_005691 +2008_005727 +2008_005738 +2008_005812 +2008_005904 +2008_005915 +2008_006008 +2008_006036 +2008_006055 +2008_006063 +2008_006108 +2008_006130 +2008_006143 +2008_006159 +2008_006216 +2008_006219 +2008_006229 +2008_006254 +2008_006275 +2008_006325 +2008_006327 +2008_006341 +2008_006408 +2008_006480 +2008_006523 +2008_006526 +2008_006528 +2008_006553 +2008_006554 +2008_006703 +2008_006722 +2008_006752 +2008_006784 +2008_006835 +2008_006874 +2008_006981 +2008_006986 +2008_007025 +2008_007031 +2008_007048 +2008_007120 +2008_007123 +2008_007143 +2008_007194 +2008_007219 +2008_007273 +2008_007350 +2008_007378 +2008_007392 +2008_007402 +2008_007497 +2008_007498 +2008_007507 +2008_007513 +2008_007527 +2008_007548 +2008_007596 +2008_007677 +2008_007737 +2008_007797 +2008_007804 +2008_007811 +2008_007814 +2008_007828 +2008_007836 +2008_007945 +2008_007994 +2008_008051 +2008_008103 +2008_008127 +2008_008221 +2008_008252 +2008_008268 +2008_008296 +2008_008301 +2008_008335 +2008_008362 +2008_008392 +2008_008393 +2008_008421 +2008_008434 +2008_008469 +2008_008629 +2008_008682 +2008_008711 +2008_008746 +2009_000012 +2009_000013 +2009_000022 +2009_000032 +2009_000037 +2009_000039 +2009_000074 +2009_000080 +2009_000087 +2009_000096 +2009_000121 +2009_000136 +2009_000149 +2009_000156 +2009_000201 +2009_000205 +2009_000219 +2009_000242 +2009_000309 +2009_000318 +2009_000335 +2009_000351 +2009_000354 +2009_000387 +2009_000391 +2009_000412 +2009_000418 +2009_000421 +2009_000426 +2009_000440 +2009_000446 +2009_000455 +2009_000457 +2009_000469 +2009_000487 +2009_000488 +2009_000523 +2009_000573 +2009_000619 +2009_000628 +2009_000641 +2009_000664 +2009_000675 +2009_000704 +2009_000705 +2009_000712 +2009_000716 +2009_000723 +2009_000727 +2009_000730 +2009_000731 +2009_000732 +2009_000771 +2009_000825 +2009_000828 +2009_000839 +2009_000840 +2009_000845 +2009_000879 +2009_000892 +2009_000919 +2009_000924 +2009_000931 +2009_000935 +2009_000964 +2009_000989 +2009_000991 +2009_000998 +2009_001008 +2009_001082 +2009_001108 +2009_001160 +2009_001215 +2009_001240 +2009_001255 +2009_001278 +2009_001299 +2009_001300 +2009_001314 +2009_001332 +2009_001333 +2009_001363 +2009_001391 +2009_001411 +2009_001433 +2009_001505 +2009_001535 +2009_001536 +2009_001565 +2009_001607 +2009_001644 +2009_001663 +2009_001683 +2009_001684 +2009_001687 +2009_001718 +2009_001731 +2009_001765 +2009_001768 +2009_001775 +2009_001804 +2009_001816 +2009_001818 +2009_001850 +2009_001851 +2009_001854 +2009_001941 +2009_001991 +2009_002012 +2009_002035 +2009_002042 +2009_002082 +2009_002094 +2009_002097 +2009_002122 +2009_002150 +2009_002155 +2009_002164 +2009_002165 +2009_002171 +2009_002185 +2009_002202 +2009_002221 +2009_002238 +2009_002239 +2009_002265 +2009_002268 +2009_002291 +2009_002295 +2009_002317 +2009_002320 +2009_002346 +2009_002366 +2009_002372 +2009_002382 +2009_002390 +2009_002415 +2009_002445 +2009_002487 +2009_002521 +2009_002527 +2009_002535 +2009_002539 +2009_002549 +2009_002562 +2009_002568 +2009_002571 +2009_002573 +2009_002584 +2009_002591 +2009_002594 +2009_002604 +2009_002618 +2009_002635 +2009_002638 +2009_002649 +2009_002651 +2009_002727 +2009_002732 +2009_002749 +2009_002753 +2009_002771 +2009_002808 +2009_002856 +2009_002887 +2009_002888 +2009_002928 +2009_002936 +2009_002975 +2009_002982 +2009_002990 +2009_003003 +2009_003005 +2009_003043 +2009_003059 +2009_003063 +2009_003065 +2009_003071 +2009_003080 +2009_003105 +2009_003123 +2009_003193 +2009_003196 +2009_003217 +2009_003224 +2009_003241 +2009_003269 +2009_003273 +2009_003299 +2009_003304 +2009_003311 +2009_003323 +2009_003343 +2009_003378 +2009_003387 +2009_003406 +2009_003433 +2009_003450 +2009_003466 +2009_003481 +2009_003494 +2009_003498 +2009_003504 +2009_003507 +2009_003517 +2009_003523 +2009_003542 +2009_003549 +2009_003551 +2009_003564 +2009_003569 +2009_003576 +2009_003589 +2009_003607 +2009_003640 +2009_003666 +2009_003696 +2009_003703 +2009_003707 +2009_003756 +2009_003771 +2009_003773 +2009_003804 +2009_003806 +2009_003810 +2009_003849 +2009_003857 +2009_003858 +2009_003895 +2009_003903 +2009_003904 +2009_003928 +2009_003938 +2009_003971 +2009_003991 +2009_004021 +2009_004033 +2009_004043 +2009_004070 +2009_004072 +2009_004084 +2009_004099 +2009_004125 +2009_004140 +2009_004217 +2009_004221 +2009_004247 +2009_004248 +2009_004255 +2009_004298 +2009_004324 +2009_004455 +2009_004494 +2009_004497 +2009_004504 +2009_004507 +2009_004509 +2009_004540 +2009_004568 +2009_004579 +2009_004581 +2009_004590 +2009_004592 +2009_004594 +2009_004635 +2009_004653 +2009_004687 +2009_004721 +2009_004730 +2009_004732 +2009_004738 +2009_004748 +2009_004789 +2009_004799 +2009_004801 +2009_004848 +2009_004859 +2009_004867 +2009_004882 +2009_004886 +2009_004895 +2009_004942 +2009_004969 +2009_004987 +2009_004993 +2009_004994 +2009_005038 +2009_005078 +2009_005087 +2009_005089 +2009_005137 +2009_005148 +2009_005156 +2009_005158 +2009_005189 +2009_005190 +2009_005217 +2009_005219 +2009_005220 +2009_005231 +2009_005260 +2009_005262 +2009_005302 +2010_000003 +2010_000038 +2010_000065 +2010_000083 +2010_000084 +2010_000087 +2010_000110 +2010_000159 +2010_000160 +2010_000163 +2010_000174 +2010_000216 +2010_000238 +2010_000241 +2010_000256 +2010_000272 +2010_000284 +2010_000309 +2010_000318 +2010_000330 +2010_000335 +2010_000342 +2010_000372 +2010_000422 +2010_000426 +2010_000427 +2010_000502 +2010_000530 +2010_000552 +2010_000559 +2010_000572 +2010_000573 +2010_000622 +2010_000628 +2010_000639 +2010_000666 +2010_000679 +2010_000682 +2010_000683 +2010_000724 +2010_000738 +2010_000764 +2010_000788 +2010_000814 +2010_000836 +2010_000874 +2010_000904 +2010_000906 +2010_000907 +2010_000918 +2010_000929 +2010_000941 +2010_000952 +2010_000961 +2010_001000 +2010_001010 +2010_001011 +2010_001016 +2010_001017 +2010_001024 +2010_001036 +2010_001061 +2010_001069 +2010_001070 +2010_001079 +2010_001104 +2010_001124 +2010_001149 +2010_001151 +2010_001174 +2010_001206 +2010_001246 +2010_001251 +2010_001256 +2010_001264 +2010_001292 +2010_001313 +2010_001327 +2010_001331 +2010_001351 +2010_001367 +2010_001376 +2010_001403 +2010_001448 +2010_001451 +2010_001522 +2010_001534 +2010_001553 +2010_001557 +2010_001563 +2010_001577 +2010_001579 +2010_001646 +2010_001656 +2010_001692 +2010_001699 +2010_001734 +2010_001752 +2010_001767 +2010_001768 +2010_001773 +2010_001820 +2010_001830 +2010_001851 +2010_001908 +2010_001913 +2010_001951 +2010_001956 +2010_001962 +2010_001966 +2010_001995 +2010_002017 +2010_002025 +2010_002030 +2010_002106 +2010_002137 +2010_002142 +2010_002146 +2010_002147 +2010_002150 +2010_002161 +2010_002200 +2010_002228 +2010_002232 +2010_002251 +2010_002271 +2010_002305 +2010_002310 +2010_002336 +2010_002348 +2010_002361 +2010_002390 +2010_002396 +2010_002422 +2010_002450 +2010_002480 +2010_002512 +2010_002531 +2010_002536 +2010_002538 +2010_002546 +2010_002623 +2010_002682 +2010_002691 +2010_002693 +2010_002701 +2010_002763 +2010_002792 +2010_002868 +2010_002900 +2010_002902 +2010_002921 +2010_002929 +2010_002939 +2010_002988 +2010_003014 +2010_003060 +2010_003123 +2010_003127 +2010_003132 +2010_003168 +2010_003183 +2010_003187 +2010_003207 +2010_003231 +2010_003239 +2010_003275 +2010_003276 +2010_003293 +2010_003302 +2010_003325 +2010_003362 +2010_003365 +2010_003381 +2010_003402 +2010_003409 +2010_003418 +2010_003446 +2010_003453 +2010_003468 +2010_003473 +2010_003495 +2010_003506 +2010_003514 +2010_003531 +2010_003532 +2010_003541 +2010_003547 +2010_003597 +2010_003675 +2010_003708 +2010_003716 +2010_003746 +2010_003758 +2010_003764 +2010_003768 +2010_003771 +2010_003772 +2010_003781 +2010_003813 +2010_003820 +2010_003854 +2010_003912 +2010_003915 +2010_003947 +2010_003956 +2010_003971 +2010_004041 +2010_004042 +2010_004056 +2010_004063 +2010_004104 +2010_004120 +2010_004149 +2010_004165 +2010_004208 +2010_004219 +2010_004226 +2010_004314 +2010_004320 +2010_004322 +2010_004337 +2010_004348 +2010_004355 +2010_004369 +2010_004382 +2010_004419 +2010_004432 +2010_004472 +2010_004479 +2010_004519 +2010_004520 +2010_004529 +2010_004543 +2010_004550 +2010_004551 +2010_004556 +2010_004559 +2010_004628 +2010_004635 +2010_004662 +2010_004697 +2010_004757 +2010_004763 +2010_004772 +2010_004783 +2010_004789 +2010_004795 +2010_004815 +2010_004825 +2010_004828 +2010_004856 +2010_004857 +2010_004861 +2010_004941 +2010_004946 +2010_004951 +2010_004980 +2010_004994 +2010_005013 +2010_005021 +2010_005046 +2010_005063 +2010_005108 +2010_005118 +2010_005159 +2010_005160 +2010_005166 +2010_005174 +2010_005180 +2010_005187 +2010_005206 +2010_005245 +2010_005252 +2010_005284 +2010_005305 +2010_005344 +2010_005353 +2010_005366 +2010_005401 +2010_005421 +2010_005428 +2010_005432 +2010_005433 +2010_005496 +2010_005501 +2010_005508 +2010_005531 +2010_005534 +2010_005575 +2010_005582 +2010_005606 +2010_005626 +2010_005644 +2010_005664 +2010_005705 +2010_005706 +2010_005709 +2010_005718 +2010_005719 +2010_005727 +2010_005762 +2010_005788 +2010_005860 +2010_005871 +2010_005877 +2010_005888 +2010_005899 +2010_005922 +2010_005991 +2010_005992 +2010_006026 +2010_006034 +2010_006054 +2010_006070 +2011_000045 +2011_000051 +2011_000054 +2011_000066 +2011_000070 +2011_000112 +2011_000173 +2011_000178 +2011_000185 +2011_000226 +2011_000234 +2011_000238 +2011_000239 +2011_000248 +2011_000283 +2011_000291 +2011_000310 +2011_000312 +2011_000338 +2011_000396 +2011_000412 +2011_000419 +2011_000435 +2011_000436 +2011_000438 +2011_000455 +2011_000456 +2011_000479 +2011_000481 +2011_000482 +2011_000503 +2011_000512 +2011_000521 +2011_000526 +2011_000536 +2011_000548 +2011_000566 +2011_000585 +2011_000598 +2011_000607 +2011_000618 +2011_000638 +2011_000658 +2011_000661 +2011_000669 +2011_000747 +2011_000780 +2011_000789 +2011_000807 +2011_000809 +2011_000813 +2011_000830 +2011_000843 +2011_000874 +2011_000888 +2011_000900 +2011_000912 +2011_000953 +2011_000969 +2011_001005 +2011_001014 +2011_001020 +2011_001047 +2011_001060 +2011_001064 +2011_001069 +2011_001071 +2011_001082 +2011_001110 +2011_001114 +2011_001159 +2011_001161 +2011_001190 +2011_001232 +2011_001263 +2011_001276 +2011_001281 +2011_001287 +2011_001292 +2011_001313 +2011_001341 +2011_001346 +2011_001350 +2011_001407 +2011_001416 +2011_001421 +2011_001434 +2011_001447 +2011_001489 +2011_001529 +2011_001530 +2011_001534 +2011_001546 +2011_001567 +2011_001589 +2011_001597 +2011_001601 +2011_001607 +2011_001613 +2011_001614 +2011_001619 +2011_001624 +2011_001642 +2011_001665 +2011_001669 +2011_001674 +2011_001708 +2011_001713 +2011_001714 +2011_001722 +2011_001726 +2011_001745 +2011_001748 +2011_001775 +2011_001782 +2011_001793 +2011_001794 +2011_001812 +2011_001862 +2011_001863 +2011_001868 +2011_001880 +2011_001910 +2011_001984 +2011_001988 +2011_002002 +2011_002040 +2011_002041 +2011_002064 +2011_002075 +2011_002098 +2011_002110 +2011_002121 +2011_002124 +2011_002150 +2011_002156 +2011_002178 +2011_002200 +2011_002223 +2011_002244 +2011_002247 +2011_002279 +2011_002295 +2011_002298 +2011_002308 +2011_002317 +2011_002322 +2011_002327 +2011_002343 +2011_002358 +2011_002371 +2011_002379 +2011_002391 +2011_002498 +2011_002509 +2011_002515 +2011_002532 +2011_002535 +2011_002548 +2011_002575 +2011_002578 +2011_002589 +2011_002592 +2011_002623 +2011_002641 +2011_002644 +2011_002662 +2011_002675 +2011_002685 +2011_002713 +2011_002730 +2011_002754 +2011_002812 +2011_002863 +2011_002879 +2011_002885 +2011_002929 +2011_002951 +2011_002975 +2011_002993 +2011_002997 +2011_003003 +2011_003011 +2011_003019 +2011_003030 +2011_003055 +2011_003085 +2011_003103 +2011_003114 +2011_003145 +2011_003146 +2011_003182 +2011_003197 +2011_003205 +2011_003240 +2011_003256 +2011_003271 diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/121.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/121.jpg new file mode 100644 index 00000000000..53ce82f6428 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/121.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/123.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/123.jpg new file mode 100644 index 00000000000..4f44baacb56 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/123.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/129.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/129.jpg new file mode 100644 index 00000000000..023bc503160 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/129.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/27.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/27.jpg new file mode 100644 index 00000000000..72316fbcb6a Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/27.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/32.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/32.jpg new file mode 100644 index 00000000000..d6575891cba Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/32.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/33.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/33.jpg new file mode 100644 index 00000000000..1ce2f2801b0 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/33.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/39.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/39.jpg new file mode 100644 index 00000000000..f723a44c291 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/39.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/42.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/42.jpg new file mode 100644 index 00000000000..d622ab7d907 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/42.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/61.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/61.jpg new file mode 100644 index 00000000000..add5186cfea Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/61.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/63.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/63.jpg new file mode 100644 index 00000000000..5ca2194e88d Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/63.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/JPEGImages/68.jpg b/tests/ut/data/dataset/testVOC2012/JPEGImages/68.jpg new file mode 100644 index 00000000000..eefa9123544 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/JPEGImages/68.jpg differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/121.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/121.png new file mode 100644 index 00000000000..c8c46504e5e Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/121.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/123.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/123.png new file mode 100644 index 00000000000..c7e1792fb11 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/123.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/129.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/129.png new file mode 100644 index 00000000000..131d6813622 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/129.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/27.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/27.png new file mode 100644 index 00000000000..7d8c2ace0e7 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/27.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/32.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/32.png new file mode 100644 index 00000000000..b3efd92cd9d Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/32.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/33.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/33.png new file mode 100644 index 00000000000..b53109f8fe4 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/33.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/39.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/39.png new file mode 100644 index 00000000000..a3f51afe1ef Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/39.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/42.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/42.png new file mode 100644 index 00000000000..8326250b26f Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/42.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/61.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/61.png new file mode 100644 index 00000000000..913ef0c282b Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/61.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/63.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/63.png new file mode 100644 index 00000000000..6b4e216ce6f Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/63.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationClass/68.png b/tests/ut/data/dataset/testVOC2012/SegmentationClass/68.png new file mode 100644 index 00000000000..8a4b0a6dfce Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationClass/68.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/121.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/121.png new file mode 100644 index 00000000000..c8c46504e5e Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/121.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/123.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/123.png new file mode 100644 index 00000000000..c7e1792fb11 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/123.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/129.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/129.png new file mode 100644 index 00000000000..131d6813622 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/129.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/27.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/27.png new file mode 100644 index 00000000000..7d8c2ace0e7 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/27.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/32.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/32.png new file mode 100644 index 00000000000..b3efd92cd9d Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/32.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/33.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/33.png new file mode 100644 index 00000000000..b53109f8fe4 Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/33.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/39.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/39.png new file mode 100644 index 00000000000..a3f51afe1ef Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/39.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/42.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/42.png new file mode 100644 index 00000000000..8326250b26f Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/42.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/61.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/61.png new file mode 100644 index 00000000000..913ef0c282b Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/61.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/63.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/63.png new file mode 100644 index 00000000000..6b4e216ce6f Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/63.png differ diff --git a/tests/ut/data/dataset/testVOC2012/SegmentationObject/68.png b/tests/ut/data/dataset/testVOC2012/SegmentationObject/68.png new file mode 100644 index 00000000000..8a4b0a6dfce Binary files /dev/null and b/tests/ut/data/dataset/testVOC2012/SegmentationObject/68.png differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images/data/train-0000-of-0001.data b/tests/ut/data/dataset/test_tf_file_3_images/data/train-0000-of-0001.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images/data/train-0000-of-0001.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images/datasetSchema.json b/tests/ut/data/dataset/test_tf_file_3_images/datasetSchema.json new file mode 100644 index 00000000000..eafcfd69eab --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "TF", + "numRows": 3, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint64", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images/train-0000-of-0001.data b/tests/ut/data/dataset/test_tf_file_3_images/train-0000-of-0001.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images/train-0000-of-0001.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionAll.json b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionAll.json new file mode 100644 index 00000000000..a83c5405f90 --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionAll.json @@ -0,0 +1,8 @@ +{ + "deviceNum":3, + "deviceId":1, + "shardConfig":"ALL", + "shuffle":"ON", + "seed": 0, + "epoch": 2 +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionEqualRows.json b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionEqualRows.json new file mode 100644 index 00000000000..8dd85bfaf54 --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionEqualRows.json @@ -0,0 +1,8 @@ +{ + "deviceNum":7, + "deviceId":6, + "shardConfig":"RANDOM", + "shuffle":"ON", + "seed": 0, + "epoch": 1 +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionRandom.json b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionRandom.json new file mode 100644 index 00000000000..0dce5921af0 --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionRandom.json @@ -0,0 +1,8 @@ +{ + "deviceNum":3, + "deviceId":1, + "shardConfig":"RANDOM", + "shuffle":"ON", + "seed": 0, + "epoch": 1 +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionUnique.json b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionUnique.json new file mode 100644 index 00000000000..ebff298052a --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images2/dataDistributionUnique.json @@ -0,0 +1,8 @@ +{ + "deviceNum":3, + "deviceId":1, + "shardConfig":"UNIQUE", + "shuffle":"ON", + "seed": 0, + "epoch": 3 +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/datasetSchema.json b/tests/ut/data/dataset/test_tf_file_3_images2/datasetSchema.json new file mode 100644 index 00000000000..59fd95ce0ec --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images2/datasetSchema.json @@ -0,0 +1,16 @@ +{ + "datasetType": "TF", + "numRows": 12, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + }, + "label" : { + "type": "uint64", + "rank": 1, + "t_impl" : "flex" + } + } +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0001.data b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0001.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0001.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0002.data b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0002.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0002.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0003.data b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0003.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0003.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0004.data b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0004.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images2/train-0000-of-0004.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images_1/datasetSchema.json b/tests/ut/data/dataset/test_tf_file_3_images_1/datasetSchema.json new file mode 100644 index 00000000000..0aa5a4577a2 --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images_1/datasetSchema.json @@ -0,0 +1,11 @@ +{ + "datasetType": "TF", + "numRows": 3, + "columns": { + "label": { + "type": "int64", + "rank": 1, + "t_impl": "flex" + } + } +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images_1/train-0000-of-0001.data b/tests/ut/data/dataset/test_tf_file_3_images_1/train-0000-of-0001.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images_1/train-0000-of-0001.data differ diff --git a/tests/ut/data/dataset/test_tf_file_3_images_2/datasetSchema.json b/tests/ut/data/dataset/test_tf_file_3_images_2/datasetSchema.json new file mode 100644 index 00000000000..b7b3cb9ea32 --- /dev/null +++ b/tests/ut/data/dataset/test_tf_file_3_images_2/datasetSchema.json @@ -0,0 +1,11 @@ +{ + "datasetType": "TF", + "numRows": 3, + "columns": { + "image": { + "type": "uint8", + "rank": 1, + "t_impl": "cvmat" + } + } +} diff --git a/tests/ut/data/dataset/test_tf_file_3_images_2/train-0000-of-0001.data b/tests/ut/data/dataset/test_tf_file_3_images_2/train-0000-of-0001.data new file mode 100644 index 00000000000..829e8d70cb9 Binary files /dev/null and b/tests/ut/data/dataset/test_tf_file_3_images_2/train-0000-of-0001.data differ diff --git a/tests/ut/data/dataset/tf_file_dataset/datasetSchema.json b/tests/ut/data/dataset/tf_file_dataset/datasetSchema.json new file mode 100644 index 00000000000..b6db5a44ead --- /dev/null +++ b/tests/ut/data/dataset/tf_file_dataset/datasetSchema.json @@ -0,0 +1,10 @@ +{ + "datasetType": "TF", + "numRows": 50, + "columns": { + "scalars": { + "type": "int64", + "rank": 1 + } + } +} diff --git a/tests/ut/data/dataset/tf_file_dataset/test1.data b/tests/ut/data/dataset/tf_file_dataset/test1.data new file mode 100644 index 00000000000..983ddbfe8dd Binary files /dev/null and b/tests/ut/data/dataset/tf_file_dataset/test1.data differ diff --git a/tests/ut/data/dataset/tf_file_dataset/test2.data b/tests/ut/data/dataset/tf_file_dataset/test2.data new file mode 100644 index 00000000000..e958dfd9bd4 Binary files /dev/null and b/tests/ut/data/dataset/tf_file_dataset/test2.data differ diff --git a/tests/ut/data/dataset/tf_file_dataset/test3.data b/tests/ut/data/dataset/tf_file_dataset/test3.data new file mode 100644 index 00000000000..e4e8141127e Binary files /dev/null and b/tests/ut/data/dataset/tf_file_dataset/test3.data differ diff --git a/tests/ut/data/dataset/tf_file_dataset/test4.data b/tests/ut/data/dataset/tf_file_dataset/test4.data new file mode 100644 index 00000000000..422ef613c1f Binary files /dev/null and b/tests/ut/data/dataset/tf_file_dataset/test4.data differ diff --git a/tests/ut/data/dataset/tf_file_dataset/test5.data b/tests/ut/data/dataset/tf_file_dataset/test5.data new file mode 100644 index 00000000000..f601a93dc72 Binary files /dev/null and b/tests/ut/data/dataset/tf_file_dataset/test5.data differ diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10000_8.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10000_8.txt new file mode 100644 index 00000000000..10c7b23a7a7 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10000_8.txt @@ -0,0 +1 @@ +Homelessness (or Houselessness as George Carlin stated) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. Most people think of the homeless as just a lost cause while worrying about things such as racism, the war on Iraq, pressuring kids to succeed, technology, the elections, inflation, or worrying if they'll be next to end up on the streets.

But what if you were given a bet to live on the streets for a month without the luxuries you once had from a home, the entertainment sets, a bathroom, pictures on the wall, a computer, and everything you once treasure to see what it's like to be homeless? That is Goddard Bolt's lesson.

Mel Brooks (who directs) who stars as Bolt plays a rich man who has everything in the world until deciding to make a bet with a sissy rival (Jeffery Tambor) to see if he can live in the streets for thirty days without the luxuries; if Bolt succeeds, he can do what he wants with a future project of making more buildings. The bet's on where Bolt is thrown on the street with a bracelet on his leg to monitor his every move where he can't step off the sidewalk. He's given the nickname Pepto by a vagrant after it's written on his forehead where Bolt meets other characters including a woman by the name of Molly (Lesley Ann Warren) an ex-dancer who got divorce before losing her home, and her pals Sailor (Howard Morris) and Fumes (Teddy Wilson) who are already used to the streets. They're survivors. Bolt isn't. He's not used to reaching mutual agreements like he once did when being rich where it's fight or flight, kill or be killed.

While the love connection between Molly and Bolt wasn't necessary to plot, I found "Life Stinks" to be one of Mel Brooks' observant films where prior to being a comedy, it shows a tender side compared to his slapstick work such as Blazing Saddles, Young Frankenstein, or Spaceballs for the matter, to show what it's like having something valuable before losing it the next day or on the other hand making a stupid bet like all rich people do when they don't know what to do with their money. Maybe they should give it to the homeless instead of using it like Monopoly money.

Or maybe this film will inspire you to help others. \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10001_10.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10001_10.txt new file mode 100644 index 00000000000..11e3a59c2d6 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10001_10.txt @@ -0,0 +1 @@ +Brilliant over-acting by Lesley Ann Warren. Best dramatic hobo lady I have ever seen, and love scenes in clothes warehouse are second to none. The corn on face is a classic, as good as anything in Blazing Saddles. The take on lawyers is also superb. After being accused of being a turncoat, selling out his boss, and being dishonest the lawyer of Pepto Bolt shrugs indifferently "I'm a lawyer" he says. Three funny words. Jeffrey Tambor, a favorite from the later Larry Sanders show, is fantastic here too as a mad millionaire who wants to crush the ghetto. His character is more malevolent than usual. The hospital scene, and the scene where the homeless invade a demolition site, are all-time classics. Look for the legs scene and the two big diggers fighting (one bleeds). This movie gets better each time I see it (which is quite often). \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10002_7.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10002_7.txt new file mode 100644 index 00000000000..d340627ed58 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10002_7.txt @@ -0,0 +1 @@ +This is easily the most underrated film inn the Brooks cannon. Sure, its flawed. It does not give a realistic view of homelessness (unlike, say, how Citizen Kane gave a realistic view of lounge singers, or Titanic gave a realistic view of Italians YOU IDIOTS). Many of the jokes fall flat. But still, this film is very lovable in a way many comedies are not, and to pull that off in a story about some of the most traditionally reviled members of society is truly impressive. Its not The Fisher King, but its not crap, either. My only complaint is that Brooks should have cast someone else in the lead (I love Mel as a Director and Writer, not so much as a lead). \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10003_8.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10003_8.txt new file mode 100644 index 00000000000..e522985b118 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10003_8.txt @@ -0,0 +1 @@ +This is not the typical Mel Brooks film. It was much less slapstick than most of his movies and actually had a plot that was followable. Leslie Ann Warren made the movie, she is such a fantastic, under-rated actress. There were some moments that could have been fleshed out a bit more, and some scenes that could probably have been cut to make the room to do so, but all in all, this is worth the price to rent and see it. The acting was good overall, Brooks himself did a good job without his characteristic speaking to directly to the audience. Again, Warren was the best actor in the movie, but "Fume" and "Sailor" both played their parts well. \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10004_8.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10004_8.txt new file mode 100644 index 00000000000..622d8c2038b --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10004_8.txt @@ -0,0 +1 @@ +This isn't the comedic Robin Williams, nor is it the quirky/insane Robin Williams of recent thriller fame. This is a hybrid of the classic drama without over-dramatization, mixed with Robin's new love of the thriller. But this isn't a thriller, per se. This is more a mystery/suspense vehicle through which Williams attempts to locate a sick boy and his keeper.

Also starring Sandra Oh and Rory Culkin, this Suspense Drama plays pretty much like a news report, until William's character gets close to achieving his goal.

I must say that I was highly entertained, though this movie fails to teach, guide, inspect, or amuse. It felt more like I was watching a guy (Williams), as he was actually performing the actions, from a third person perspective. In other words, it felt real, and I was able to subscribe to the premise of the story.

All in all, it's worth a watch, though it's definitely not Friday/Saturday night fare.

It rates a 7.7/10 from...

the Fiend :. \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10005_7.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10005_7.txt new file mode 100644 index 00000000000..79d79f2d42b --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10005_7.txt @@ -0,0 +1 @@ +Yes its an art... to successfully make a slow paced thriller.

The story unfolds in nice volumes while you don't even notice it happening.

Fine performance by Robin Williams. The sexuality angles in the film can seem unnecessary and can probably affect how much you enjoy the film. However, the core plot is very engaging. The movie doesn't rush onto you and still grips you enough to keep you wondering. The direction is good. Use of lights to achieve desired affects of suspense and unexpectedness is good.

Very nice 1 time watch if you are looking to lay back and hear a thrilling short story! \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10006_7.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10006_7.txt new file mode 100644 index 00000000000..90d6c0f28d6 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10006_7.txt @@ -0,0 +1 @@ +In this "critically acclaimed psychological thriller based on true events, Gabriel (Robin Williams), a celebrated writer and late-night talk show host, becomes captivated by the harrowing story of a young listener and his adoptive mother (Toni Collette). When troubling questions arise about this boy's (story), however, Gabriel finds himself drawn into a widening mystery that hides a deadly secret…" according to film's official synopsis.

You really should STOP reading these comments, and watch the film NOW...

The "How did he lose his leg?" ending, with Ms. Collette planning her new life, should be chopped off, and sent to "deleted scenes" land. It's overkill. The true nature of her physical and mental ailments should be obvious, by the time Mr. Williams returns to New York. Possibly, her blindness could be in question - but a revelation could have be made certain in either the "highway" or "video tape" scenes. The film would benefit from a re-editing - how about a "director's cut"?

Williams and Bobby Cannavale (as Jess) don't seem, initially, believable as a couple. A scene or two establishing their relationship might have helped set the stage. Otherwise, the cast is exemplary. Williams offers an exceptionally strong characterization, and not a "gay impersonation". Sandra Oh (as Anna), Joe Morton (as Ashe), and Rory Culkin (Pete Logand) are all perfect.

Best of all, Collette's "Donna" belongs in the creepy hall of fame. Ms. Oh is correct in saying Collette might be, "you know, like that guy from 'Psycho'." There have been several years when organizations giving acting awards seemed to reach for women, due to a slighter dispersion of roles; certainly, they could have noticed Collette with some award consideration. She is that good. And, director Patrick Stettner definitely evokes Hitchcock - he even makes getting a sandwich from a vending machine suspenseful.

Finally, writers Stettner, Armistead Maupin, and Terry Anderson deserve gratitude from flight attendants everywhere.

******* The Night Listener (1/21/06) Patrick Stettner ~ Robin Williams, Toni Collette, Sandra Oh, Rory Culkin \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10007_7.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10007_7.txt new file mode 100644 index 00000000000..8f4990c981f --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10007_7.txt @@ -0,0 +1 @@ +THE NIGHT LISTENER (2006) **1/2 Robin Williams, Toni Collette, Bobby Cannavale, Rory Culkin, Joe Morton, Sandra Oh, John Cullum, Lisa Emery, Becky Ann Baker. (Dir: Patrick Stettner)

Hitchcockian suspenser gives Williams a stand-out low-key performance.

What is it about celebrities and fans? What is the near paranoia one associates with the other and why is it almost the norm?

In the latest derange fan scenario, based on true events no less, Williams stars as a talk-radio named Gabriel No one, who reads stories he's penned over the airwaves and has accumulated an interesting fan in the form of a young boy named Pete Logand (Culkin) who has submitted a manuscript about the travails of his troubled youth to No one's editor Ashe (Morton) who gives it to No one to read for himself.

No one is naturally disturbed but ultimately intrigued about the nightmarish existence of Pete being abducted and sexually abused for years until he was finally rescued by a nurse named Donna (Collette giving an excellent performance) who has adopted the boy but her correspondence with No one reveals that Pete is dying from AIDS. Naturally No one wants to meet the fans but is suddenly in doubt to their possibly devious ulterior motives when the seed is planted by his estranged lover Jess (Cannavale) whose sudden departure from their New York City apartment has No one in an emotional tailspin that has only now grown into a tempest in a teacup when he decides to do some investigating into Donna and Pete's backgrounds discovering some truths that he didn't anticipate.

Written by Armistead Maupin (who co-wrote the screenplay with his former lover Terry Anderson and the film's novice director Stettner) and based on a true story about a fan's hoax found out has some Hitchcockian moments that run on full tilt like any good old fashioned pot-boiler does. It helps that Williams gives a stand-out, low-key performance as the conflicted good-hearted who genuinely wants to believe that his number one fan is in fact real and does love him (the one thing that has escaped his own reality) and has some unsettling dreadful moments with the creepy Collette whose one physical trait I will leave unmentioned but underlines the desperation of her character that can rattle you to the core.

However the film runs out of gas and eventually becomes a bit repetitive and predictable despite a finely directed piece of hoodwink and mystery by Stettner, it pays to listen to your own inner voice: be careful of what you hope for. \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/10008_7.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/10008_7.txt new file mode 100644 index 00000000000..d8231785ba4 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/10008_7.txt @@ -0,0 +1 @@ +You know, Robin Williams, God bless him, is constantly shooting himself in the foot lately with all these dumb comedies he has done this decade (with perhaps the exception of "Death To Smoochy", which bombed when it came out but is now a cult classic). The dramas he has made lately have been fantastic, especially "Insomnia" and "One Hour Photo". "The Night Listener", despite mediocre reviews and a quick DVD release, is among his best work, period.

This is a very chilling story, even though it doesn't include a serial killer or anyone that physically dangerous for that matter. The concept of the film is based on an actual case of fraud that still has yet to be officially confirmed. In high school, I read an autobiography by a child named Anthony Godby Johnson, who suffered horrific abuse and eventually contracted AIDS as a result. I was moved by the story until I read reports online that Johnson may not actually exist. When I saw this movie, the confused feelings that Robin Williams so brilliantly portrayed resurfaced in my mind.

Toni Collette probably gives her best dramatic performance too as the ultimately sociopathic "caretaker". Her role was a far cry from those she had in movies like "Little Miss Sunshine". There were even times she looked into the camera where I thought she was staring right at me. It takes a good actress to play that sort of role, and it's this understated (yet well reviewed) role that makes Toni Collette probably one of the best actresses of this generation not to have even been nominated for an Academy Award (as of 2008). It's incredible that there is at least one woman in this world who is like this, and it's scary too.

This is a good, dark film that I highly recommend. Be prepared to be unsettled, though, because this movie leaves you with a strange feeling at the end. \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/pos/1000_8.txt b/tests/ut/data/mindrecord/testAclImdbData/pos/1000_8.txt new file mode 100644 index 00000000000..893b2964175 --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/pos/1000_8.txt @@ -0,0 +1 @@ +I liked the film. Some of the action scenes were very interesting, tense and well done. I especially liked the opening scene which had a semi truck in it. A very tense action scene that seemed well done.

Some of the transitional scenes were filmed in interesting ways such as time lapse photography, unusual colors, or interesting angles. Also the film is funny is several parts. I also liked how the evil guy was portrayed too. I'd give the film an 8 out of 10. \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testAclImdbData/vocab.txt b/tests/ut/data/mindrecord/testAclImdbData/vocab.txt new file mode 100644 index 00000000000..9a6bf58dd2e --- /dev/null +++ b/tests/ut/data/mindrecord/testAclImdbData/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +perrson +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +perrsonal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +perrsonnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +perrsons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +perrsonality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +perrsonally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +perrsonalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperrson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +perrsona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperrson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##perrson +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##perrsonal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/tests/ut/data/mindrecord/testCBGData/data/annotation.data b/tests/ut/data/mindrecord/testCBGData/data/annotation.data new file mode 100644 index 00000000000..5a4570e185c --- /dev/null +++ b/tests/ut/data/mindrecord/testCBGData/data/annotation.data @@ -0,0 +1,10 @@ +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "7", "name": "大小"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_276", "frame": 0, "attr_value": [{"x": 1080.0532437558108, "y": 2310.235384108011, "sn": 0}, {"x": 1085.7692935235789, "y": 2571.910096266103, "sn": 1}, {"x": 1056.843999321023, "y": 2110.1092403427992, "sn": 2}, {"x": 1011.0329592109478, "y": 2084.6669511038476, "sn": 3}, {"x": 1010.5292197073934, "y": 2387.106810301955, "sn": 4}, {"x": 1064.603610354495, "y": 2397.4727593789303, "sn": 5}, {"x": 1008.1059096395929, "y": 2105.596723900093, "sn": 6}, {"x": 1065.3712043551210, "y": 2335.3781073392410, "sn": 7}, {"x": 1024.5735810100385, "y": 2211.2448220810910, "sn": 8}, {"x": 1081.9377799578263, "y": 2477.109756776835, "sn": 9}, {"x": 1126.239573232979, "y": 2389.028326112894, "sn": 10}, {"x": 1009.7363684885272, "y": 2673.899572474610, "sn": 11}, {"x": 1010.3098605370262, "y": 2833.3769808547736, "sn": 12}, {"x": 1067.8095885282964, "y": 2463.0824806682105, "sn": 10}, {"x": 1099.944411028393, "y": 2472.5448981100384, "sn": 10}, {"x": 1984.7869101010971, "y": 2831.202580005591, "sn": 10}, {"x": 1984.9850124910637, "y": 2119.60445763610, "sn": 10}, {"x": 1099.102930829722, "y": 2329.5106706112998, "sn": 10}, {"x": 1055.238587827492, "y": 2610.2266496212706, "sn": 10}, {"x": 1236.6227258721995, "y": 2496.8989489065466, "sn": 19}, {"x": 1010.608903300677, "y": 2409.1009519310747, "sn": 20}, {"x": 1910.8987073084910, "y": 2732.857554341086, "sn": 21}, {"x": 1019.4102510774932, "y": 2302.9023469207887, "sn": 22}, {"x": 1029.8062883277328, "y": 2062.8294093771083, "sn": 23}, {"x": 1943.7109459890262, "y": 2472.1929847410055, "sn": 24}, {"x": 1007.6392054007933, "y": 2104.9104048883955, "sn": 25}, {"x": 1212.4242046069103, "y": 2258.5535290363623, "sn": 26}, {"x": 1076.932010023852, "y": 2635.103694125807, "sn": 27}, {"x": 1079.7608196910832, "y": 2434.639253601191, "sn": 28}, {"x": 1002.5264578652277, "y": 2470.8921019669244, "sn": 29}, {"x": 1022.2575348848195, "y": 2424.10363261079, "sn": 30}, {"x": 1974.6807668306456, "y": 2986.3736210991085, "sn": 31}, {"x": 1072.5254358781047, "y": 2891.421094723631, "sn": 32}, {"x": 1026.8708450626023, "y": 2504.327810700708, "sn": 33}, {"x": 1046.3984494428108, "y": 2970.8740801038866, "sn": 34}, {"x": 1079.4510431941047, "y": 2103.3870931943106, "sn": 35}, {"x": 1279.219304304588, "y": 2352.8792444421033, "sn": 36}, {"x": 1076.7828580310902, "y": 2905.046810919068, "sn": 37}, {"x": 1054.7055572353092, "y": 2897.265101259410, "sn": 38}, {"x": 1954.3781293474385, "y": 2403.01104695072, "sn": 39}, {"x": 1028.0198631005742, "y": 2964.55107619110, "sn": 40}, {"x": 1036.9206504897010, "y": 2653.1006056619867, "sn": 41}, {"x": 1955.2645610100448, "y": 2738.7910424359474, "sn": 42}, {"x": 1080.3786676247102, "y": 2363.436210322475, "sn": 43}, {"x": 1950.5127256262454, "y": 2360.322907863479, "sn": 44}, {"x": 1024.7568310106524, "y": 2623.6046960007407, "sn": 45}, {"x": 1076.78621063003, "y": 2067.7084810265324, "sn": 46}, {"x": 1077.9842632105828, "y": 2776.410207906102, "sn": 47}, {"x": 1094.1072577964, "y": 2252.827106455741, "sn": 48}, {"x": 1025.4837421094058, "y": 2020.6412452104074, "sn": 49}, {"x": 1967.108992947041, "y": 2126.2995108824364, "sn": 50}, {"x": 1192.6867724982610, "y": 2821.4092339524846, "sn": 51}, {"x": 1083.284197730101, "y": 2105.6281098681044, "sn": 52}, {"x": 1094.1194273319675, "y": 2783.436010026865, "sn": 53}, {"x": 1027.8202649410610, "y": 2912.758535523258, "sn": 54}, {"x": 1000.9293020507328, "y": 2350.8812107988103, "sn": 55}, {"x": 1127.2319107629263, "y": 2860.656820707349, "sn": 56}, {"x": 1054.6194906366784, "y": 2251.557951065985, "sn": 57}, {"x": 1044.5459736365865, "y": 2801.2689636031087, "sn": 58}, {"x": 1062.1997693798844, "y": 2100.7823101010528, "sn": 59}, {"x": 1057.799799808532, "y": 2869.2825662087866, "sn": 60}, {"x": 1047.2781063503911, "y": 2921.7280533106436, "sn": 61}, {"x": 1046.2312663091004, "y": 2477.0737642906834, "sn": 62}, {"x": 1089.241079239106, "y": 2629.7822472108064, "sn": 63}, {"x": 1062.9121006199282, "y": 2282.102987593412, "sn": 64}, {"x": 1224.8865352972532, "y": 2988.628380911289, "sn": 65}, {"x": 1120.6090744951106, "y": 2875.8579002751026, "sn": 66}, {"x": 1012.538479261263, "y": 2011.9103859410108, "sn": 67}, {"x": 1083.0992809659751, "y": 2395.372265745683, "sn": 68}, {"x": 1008.9710461210375, "y": 2953.678087323952, "sn": 69}, {"x": 1201.6838074674984, "y": 2249.85591277752, "sn": 70}, {"x": 1036.3458110534464, "y": 2103.935106872553, "sn": 71}, {"x": 1078.3566108377804, "y": 2524.7657493034876, "sn": 72}, {"x": 1061.4369477221094, "y": 2032.9420783529308, "sn": 73}, {"x": 1230.4106982055388, "y": 2610.910046598237, "sn": 74}, {"x": 1063.108757506610, "y": 2985.6405439910344, "sn": 75}, {"x": 1093.010373381106, "y": 2821.957685228096, "sn": 76}, {"x": 1005.5862496988673, "y": 2683.891103610004, "sn": 77}, {"x": 1055.0046268105010, "y": 2435.7988863425810, "sn": 78}, {"x": 1275.4433107263105, "y": 2771.4640665779375, "sn": 79}, {"x": 1192.3788596472484, "y": 2128.1126772251028, "sn": 80}, {"x": 1068.5975309399842, "y": 2410.606034104646, "sn": 81}, {"x": 1012.7750361035934, "y": 2597.7546223730665, "sn": 82}, {"x": 1076.553710933009, "y": 2958.0733556452856, "sn": 83}, {"x": 1033.9108676102785, "y": 2582.709554990767, "sn": 84}, {"x": 1226.675810577763, "y": 2981.3691039010775, "sn": 85}, {"x": 1059.2988992310031, "y": 2102.532909499666, "sn": 86}, {"x": 1010.469253523332, "y": 2364.780105352378, "sn": 87}, {"x": 1202.5238919679462, "y": 2981.5877310321003, "sn": 88}, {"x": 1095.3547777870956, "y": 2282.6647396799726, "sn": 89}, {"x": 1075.1977510207896, "y": 2284.250109102006, "sn": 90}, {"x": 1064.2986238612710, "y": 2770.2790032451008, "sn": 91}, {"x": 1025.8538502988358, "y": 2353.2910791044595, "sn": 92}, {"x": 1255.8886501102667, "y": 2567.4829632941055, "sn": 93}, {"x": 1098.8103328028497, "y": 2879.0775253397546, "sn": 94}, {"x": 1048.4954669109702, "y": 2895.6109956912297, "sn": 95}, {"x": 1238.093987204698, "y": 2350.0336402896082, "sn": 96}, {"x": 1010.0010081124094, "y": 2073.8542355021010, "sn": 97}, {"x": 1042.495523397682, "y": 2989.910435520666, "sn": 98}, {"x": 1089.0101004448597, "y": 2397.734196858954, "sn": 99}, {"x": 1222.9084768631108, "y": 2835.547842202677, "sn": 100}, {"x": 1990.902851954509, "y": 2447.7272603907327, "sn": 101}, {"x": 1075.9810686103101, "y": 2598.108381025935, "sn": 102}, {"x": 1971.27364476096, "y": 2107.6072747108964, "sn": 103}, {"x": 1044.3926731108997, "y": 2789.8010807107554, "sn": 104}, {"x": 1210.7324529574585, "y": 2210.073963858728, "sn": 105}, {"x": 1261.9908569704303, "y": 2882.195568486796, "sn": 106}, {"x": 1011.069761072935, "y": 2275.9630757755053, "sn": 107}, {"x": 1064.4397710738747, "y": 2330.94751010112, "sn": 108}, {"x": 1283.2010687857472, "y": 2510.2383708101038, "sn": 109}, {"x": 1102.2390088385619, "y": 2105.1253422926193, "sn": 110}, {"x": 1035.995336828295, "y": 2100.1010100210247, "sn": 111}, {"x": 1051.577523120397, "y": 2385.2593107843246, "sn": 112}, {"x": 1010.773923694383, "y": 2342.035788107673, "sn": 110}, {"x": 1031.0953931064104, "y": 2079.336837000128, "sn": 110}, {"x": 1000.6746410270406, "y": 2272.45412107767, "sn": 110}, {"x": 1086.110629702010, "y": 2860.1088231081008, "sn": 110}, {"x": 1087.807507479678, "y": 2710.049226199710, "sn": 110}, {"x": 1007.0490508129787, "y": 2956.574106124226, "sn": 110}, {"x": 1094.4121064644867, "y": 2255.5892349738674, "sn": 119}, {"x": 1079.075782379448, "y": 2367.2975766104636, "sn": 120}, {"x": 1019.5829646362781, "y": 2279.92512365378, "sn": 121}, {"x": 1090.354332935074, "y": 2862.746677682906, "sn": 122}, {"x": 1102.7563954869224, "y": 2103.4765088658210, "sn": 123}, {"x": 1037.1010849544905, "y": 2241.3984393046867, "sn": 124}, {"x": 1090.7730627191036, "y": 2067.765593788495, "sn": 125}, {"x": 1121.3958811038243, "y": 2308.645633632929, "sn": 126}, {"x": 1064.9101027465836, "y": 2253.721071274945, "sn": 127}, {"x": 1040.8691048945426, "y": 2231.234654583988, "sn": 128}, {"x": 1036.4842233041106, "y": 2775.108571076910, "sn": 129}, {"x": 1071.7010010198875, "y": 2542.4012441966997, "sn": 100}, {"x": 1055.1201007750276, "y": 2033.559442100322, "sn": 101}, {"x": 1122.8201028019386, "y": 2523.7336690433503, "sn": 102}, {"x": 1955.110481100833, "y": 2891.733510480731, "sn": 103}, {"x": 1245.083310484225, "y": 2495.9065222689655, "sn": 104}, {"x": 1043.584921010843, "y": 2336.707835848999, "sn": 105}, {"x": 1080.9007259503765, "y": 2220.8412524878410, "sn": 106}, {"x": 1900.4868645526396, "y": 2898.7723233089937, "sn": 107}, {"x": 1295.7262440232048, "y": 2282.698226290031, "sn": 108}, {"x": 1110.6103507910576, "y": 2429.240605679059, "sn": 109}, {"x": 1076.19104626842, "y": 2577.3675362551207, "sn": 100}, {"x": 1209.2355977850295, "y": 2725.2006399106357, "sn": 101}, {"x": 1101.9571021101046, "y": 2510.6530789039434, "sn": 102}, {"x": 1031.3610259982310, "y": 2819.096834293535, "sn": 103}, {"x": 1008.8957060925409, "y": 2407.0101078829543, "sn": 104}, {"x": 1101.6105827711036, "y": 2827.4233019332723, "sn": 105}, {"x": 1100.0511034610822, "y": 2610.5089876558377, "sn": 106}, {"x": 1067.7552834757705, "y": 2462.352844547328, "sn": 107}, {"x": 1945.8501064385748, "y": 2968.9532048307265, "sn": 108}, {"x": 1060.1211080010707, "y": 2610.0010883435683, "sn": 109}, {"x": 1076.020088610306, "y": 2880.0247778373610, "sn": 100}, {"x": 1080.2856491063669, "y": 2797.6102801021045, "sn": 101}, {"x": 1091.2072408757010, "y": 2657.10196946851, "sn": 102}, {"x": 1010.1057080386829, "y": 2769.633785401108, "sn": 103}, {"x": 1090.0625039099566, "y": 2102.8536107830742, "sn": 104}, {"x": 1010.0926858433668, "y": 2647.8995794852262, "sn": 105}, {"x": 1034.2093907730505, "y": 2295.8391067108355, "sn": 106}, {"x": 1106.9353108671210, "y": 2378.9837845711106, "sn": 107}, {"x": 1035.7104641033836, "y": 2281.0047100757310, "sn": 108}, {"x": 1957.8612310699054, "y": 2903.942038108005, "sn": 109}, {"x": 1940.3834270623433, "y": 2894.028930403767, "sn": 100}, {"x": 1057.7030197670103, "y": 2652.6236106256855, "sn": 101}, {"x": 1055.9652851086664, "y": 2886.6324567642537, "sn": 102}, {"x": 1038.9128108479302, "y": 2312.731099039849, "sn": 103}, {"x": 1060.770647585463, "y": 2332.3481098875896, "sn": 104}, {"x": 1020.500745057956, "y": 2812.475570941010, "sn": 105}, {"x": 1106.0072010698102, "y": 2924.635853884397, "sn": 106}, {"x": 1037.874603607012, "y": 2227.3994482086823, "sn": 107}, {"x": 1057.1098078774905, "y": 2669.3110410310687, "sn": 108}, {"x": 1074.2850648730564, "y": 2900.2370529781097, "sn": 109}, {"x": 1042.3101941105223, "y": 2312.1237438054386, "sn": 100}, {"x": 1007.9779023106021, "y": 2829.6696845922047, "sn": 101}, {"x": 1931.4701976745034, "y": 2463.1093749044110, "sn": 102}, {"x": 1067.1076665664693, "y": 2947.8533010603733, "sn": 103}, {"x": 1193.310519825528, "y": 2285.1054723610067, "sn": 104}, {"x": 1911.70758920887, "y": 2200.951256100888, "sn": 105}, {"x": 1277.7038270954533, "y": 2785.0976967681027, "sn": 106}, {"x": 1987.952058945793, "y": 2212.084102002664, "sn": 107}, {"x": 1298.8771087678305, "y": 2056.2659866938725, "sn": 108}, {"x": 1235.1006568578595, "y": 2008.4022010878782, "sn": 109}, {"x": 1070.9233746976545, "y": 2083.650879128294, "sn": 100}, {"x": 1041.9653945759337, "y": 2477.0091107010083, "sn": 101}, {"x": 1023.5790090771006, "y": 2364.910941056287, "sn": 102}, {"x": 1064.4388827671096, "y": 2250.6482683223085, "sn": 103}, {"x": 1252.7834360067843, "y": 2009.6793284593910, "sn": 104}, {"x": 1035.7229689105088, "y": 2473.125423884903, "sn": 105}, {"x": 1091.1227094672463, "y": 2110.1075776724453, "sn": 106}, {"x": 1980.193705680744, "y": 2294.410034670357, "sn": 107}, {"x": 1060.7238029535488, "y": 2069.2610082466464, "sn": 108}, {"x": 1048.4061235726103, "y": 2210.1108564658295, "sn": 109}, {"x": 1088.3710696102474, "y": 2106.599704751035, "sn": 190}, {"x": 1056.5101083348379, "y": 2881.1047811111265, "sn": 191}, {"x": 1924.286858910288, "y": 2628.608860194232, "sn": 192}, {"x": 1001.4689769375036, "y": 2348.4958121267996, "sn": 193}, {"x": 1091.8232120258192, "y": 2852.3722422927303, "sn": 194}, {"x": 1107.0069592089544, "y": 2577.572254572068, "sn": 195}, {"x": 1284.57310251007, "y": 2338.590969098888, "sn": 196}, {"x": 1298.4104510827844, "y": 2561.5732588457286, "sn": 197}, {"x": 1052.1068023732104, "y": 2797.089934195975, "sn": 198}, {"x": 1966.8239973879333, "y": 2023.2473258900327, "sn": 199}, {"x": 1010.1023287854102, "y": 2103.369110540094, "sn": 200}, {"x": 1104.2982450550846, "y": 2253.010890910106, "sn": 201}, {"x": 1008.1044710636381, "y": 2469.428305929984, "sn": 202}, {"x": 1076.7031067933551, "y": 2376.798969246933, "sn": 203}, {"x": 1068.3985337104, "y": 2735.7331060211098, "sn": 204}, {"x": 1027.9338564659105, "y": 2550.1021963812097, "sn": 205}, {"x": 1072.1969801976347, "y": 2610.866228428110, "sn": 206}, {"x": 1078.4634229761, "y": 2107.912040681047, "sn": 207}, {"x": 1062.485301210829, "y": 2739.8050282007102, "sn": 208}, {"x": 1089.619763381204, "y": 2590.6045874734027, "sn": 209}, {"x": 1041.0068028669682, "y": 2007.2755794312810, "sn": 210}, {"x": 1036.6882784210426, "y": 2919.691094999710, "sn": 211}, {"x": 1000.9672075109345, "y": 2378.043227480667, "sn": 212}, {"x": 1091.1011910007410, "y": 2061.5443947441010, "sn": 210}, {"x": 1075.8400682946044, "y": 2844.895997107625, "sn": 210}, {"x": 1041.6102192339943, "y": 2102.5077310850246, "sn": 210}, {"x": 1283.8810669690323, "y": 2061.6702961021232, "sn": 210}, {"x": 1210.077481062973, "y": 2447.3642520765857, "sn": 210}, {"x": 1010.276838066742, "y": 2543.565549545584, "sn": 210}, {"x": 1047.0631073286023, "y": 2345.9919992477844, "sn": 219}, {"x": 1010.5043723591027, "y": 2985.5021071046102, "sn": 220}, {"x": 1039.378530579107, "y": 2764.340105566384, "sn": 221}, {"x": 1056.0107308263659, "y": 2528.5265632525397, "sn": 222}, {"x": 1067.5234996297963, "y": 2500.9971061037437, "sn": 223}, {"x": 1922.0104904040104, "y": 2112.7906298291046, "sn": 224}, {"x": 1005.7687749583735, "y": 2883.5810100560196, "sn": 225}, {"x": 1951.4466709789554, "y": 2257.3057323498206, "sn": 226}, {"x": 1010.9540884376706, "y": 2044.7105674690607, "sn": 227}, {"x": 1094.095008096334, "y": 2970.7991073027925, "sn": 228}, {"x": 1039.7794856450487, "y": 2883.485101105035, "sn": 229}, {"x": 1010.610893539886, "y": 2100.463025038225, "sn": 230}, {"x": 1041.4835040335404, "y": 2290.4270574430534, "sn": 231}, {"x": 1021.610827775348, "y": 2787.7956831023808, "sn": 232}, {"x": 1086.3478552522865, "y": 2735.9771941055752, "sn": 233}, {"x": 1125.590097373651, "y": 2585.3801126467806, "sn": 234}, {"x": 1031.310867475286, "y": 2036.0629794279664, "sn": 235}, {"x": 1066.9433410010734, "y": 2283.9999745696437, "sn": 236}, {"x": 1022.0650194738191, "y": 2091.2304053624857, "sn": 237}, {"x": 1994.119959694286, "y": 2548.1033294632506, "sn": 238}, {"x": 1037.3310020010, "y": 2064.734455667961, "sn": 239}, {"x": 1108.1077104550664, "y": 2101.1105560007522, "sn": 240}, {"x": 1080.7839103784604, "y": 2910.241023630410, "sn": 241}, {"x": 1256.4376810884002, "y": 2858.0701064539775, "sn": 242}, {"x": 1086.3733839284478, "y": 2954.0823489321074, "sn": 243}, {"x": 1087.9493792102748, "y": 2508.4592057550963, "sn": 244}, {"x": 1063.3532332108407, "y": 2234.010887326333, "sn": 245}, {"x": 1097.703019810710, "y": 2639.3731020103305, "sn": 246}, {"x": 1107.2910944873905, "y": 2106.0344108365557, "sn": 247}, {"x": 1099.0639879591079, "y": 2209.793068410343, "sn": 248}, {"x": 1054.9537543029310, "y": 2751.600642965203, "sn": 249}, {"x": 1104.2287310421094, "y": 2798.686428108909, "sn": 250}, {"x": 1037.7474038353282, "y": 2610.430235569631, "sn": 251}, {"x": 1066.8285443410936, "y": 2768.364192110858, "sn": 252}, {"x": 1022.9109456066546, "y": 2625.945910634242, "sn": 253}, {"x": 1053.2109108630492, "y": 2463.400352822463, "sn": 254}, {"x": 1237.7026308867744, "y": 2038.7101078708048, "sn": 255}, {"x": 1250.3519325874759, "y": 2110.1054807935398, "sn": 256}, {"x": 1084.9065409993898, "y": 2000.0957399962379, "sn": 257}, {"x": 1003.479107636832, "y": 2210.1046935410986, "sn": 258}, {"x": 1962.4195794310453, "y": 2910.098397663949, "sn": 259}, {"x": 1907.0102892619275, "y": 2475.276121021101, "sn": 260}, {"x": 1068.3968247030884, "y": 2564.796087101075, "sn": 261}, {"x": 1229.5429696671272, "y": 2533.6449909282487, "sn": 262}, {"x": 1024.478110423396, "y": 2076.2127710939763, "sn": 263}, {"x": 1079.0991076465648, "y": 2210.7233128353346, "sn": 264}, {"x": 1085.6820682887974, "y": 2885.6606194106506, "sn": 265}, {"x": 1034.3792101010435, "y": 2201.8942803127434, "sn": 266}, {"x": 1266.40739605196, "y": 2286.3738722325610, "sn": 267}, {"x": 1001.3700103609485, "y": 2559.53271010562, "sn": 268}, {"x": 1202.489426595106, "y": 2676.645700792353, "sn": 269}, {"x": 1942.465251071027, "y": 2962.6109219024907, "sn": 270}, {"x": 1979.4212794885102, "y": 2429.895951020955, "sn": 271}, {"x": 1034.1027763451988, "y": 2105.7364102711203, "sn": 272}, {"x": 1025.273223381072, "y": 2582.6510710742563, "sn": 273}, {"x": 1008.8793275383991, "y": 2237.2670505409324, "sn": 274}, {"x": 1040.7700642595310, "y": 2103.287101210410, "sn": 275}], "z_order": 2, "group_id": 0}], "entity_name": "276点人脸", "instance_id": 0, "class_id": 276}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 100.65074310102106, "y": 110.07882575281957, "sn": 1}, {"x": 82.93026328639885, "y": 281.3310076078663, "sn": 2}, {"x": 80.6483867385263, "y": 351.59233307275724, "sn": 3}, {"x": 126.94341102331099, "y": 399.6244104665781, "sn": 4}, {"x": 112.68610284106042, "y": 281.59210569205623, "sn": 5}, {"x": 59.97789693106791, "y": 363.9021010095879, "sn": 6}, {"x": 107.97919699190652, "y": 70.0711966035409, "sn": 7}, {"x": 193.07080323723386, "y": 356.5296024276394, "sn": 8}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 0, "class_id": 3}, {"attributes": [{"occluded": "false", "attr_name": "polyline_example", "data_type": "polyline", "frame": 0, "attr_value": [{"x": 1092.5958961909043, "y": 2029.4345043556352, "sn": 1}, {"x": 1004.96048266371, "y": 2108.6869939511007, "sn": 2}, {"x": 1198.2646072263710, "y": 2342.5754562031108, "sn": 3}], "z_order": 2, "group_id": 0}], "entity_name": "polyline_example", "instance_id": 0, "class_id": 2}, {"attributes": [{"occluded": "false", "attr_name": "polyline_example", "data_type": "polyline", "frame": 0, "attr_value": [{"x": 1082.5293253033274, "y": 2625.538319485441, "sn": 1}, {"x": 1094.4825536104797, "y": 2732.499356687323, "sn": 2}, {"x": 1259.691079092405, "y": 2386.6896251283647, "sn": 3}, {"x": 1962.9875043832444, "y": 2939.644874055477, "sn": 4}, {"x": 1100.7210105223044, "y": 2429.2803592843675, "sn": 5}, {"x": 1039.264832553859, "y": 2662.0893109280010, "sn": 6}, {"x": 1022.2910742437804, "y": 2353.985702977725, "sn": 7}], "z_order": 2, "group_id": 0}], "entity_name": "polyline_example", "instance_id": 0, "class_id": 2}], "creation_time": "2019-02-12T01:32:04", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "polyline_example", "data_type": "polyline", "frame": 0, "attr_value": [{"x": 1232.0937229574654, "y": 2190.4349586337785, "sn": 1}, {"x": 1970.0234083610599, "y": 2310.0790767500966, "sn": 2}, {"x": 1233.403908592381, "y": 2107.54787071101, "sn": 3}, {"x": 1079.0810282364364, "y": 2578.6960620627888, "sn": 4}, {"x": 1962.9478094506499, "y": 2002.0523372651033, "sn": 5}, {"x": 1064.4208477990410, "y": 2096.8405872225010, "sn": 6}, {"x": 1034.2453953359461, "y": 2455.950619395603, "sn": 7}, {"x": 1057.888532565191, "y": 2531.4263644785410, "sn": 8}, {"x": 1073.4455072391052, "y": 2295.910107072196, "sn": 9}], "z_order": 2, "group_id": 0}], "entity_name": "polyline_example", "instance_id": 1, "class_id": 2}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 192.2227196863667, "y": 106.34630404998737, "sn": 1}, {"x": 198.10394092954863, "y": 99.3781080778327, "sn": 2}, {"x": 110.64923907106378, "y": 206.0250246309625, "sn": 3}, {"x": 93.34942050102796, "y": 105.9294211904025, "sn": 4}, {"x": 101.04129192293109, "y": 210.33369648220452, "sn": 5}, {"x": 124.81079927106247, "y": 446.40696063110053, "sn": 6}, {"x": 106.82493089346104, "y": 456.49966210387537, "sn": 7}, {"x": 51.58969948820256, "y": 348.92610865237773, "sn": 8}, {"x": 103.7698819405439, "y": 410.9283525257021, "sn": 9}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 1, "class_id": 3}, {"attributes": [{"occluded": "false", "attr_name": "polyline_example", "data_type": "polyline", "frame": 0, "attr_value": [{"x": 1994.0988938942492, "y": 2924.4028611941053, "sn": 1}, {"x": 1020.4503636239597, "y": 2765.9659383403355, "sn": 2}, {"x": 1099.27500712396, "y": 2104.932388647412, "sn": 3}, {"x": 1089.9708424562252, "y": 2051.701049745019, "sn": 4}, {"x": 1010.2756036697083, "y": 2556.527410783443, "sn": 5}, {"x": 1106.2462993480037, "y": 2670.507371251061, "sn": 6}, {"x": 1032.854097811081, "y": 2021.0470867032310, "sn": 7}], "z_order": 2, "group_id": 0}], "entity_name": "polyline_example", "instance_id": 1, "class_id": 2}, {"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "8", "name": "大小"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_276", "frame": 0, "attr_value": [{"x": 1001.51024912579, "y": 2012.7354609010210, "sn": 0}, {"x": 1909.856823490759, "y": 2760.8419727111022, "sn": 1}, {"x": 1105.8923008604324, "y": 2423.344742999434, "sn": 2}, {"x": 1041.9019771252974, "y": 2927.2370478541025, "sn": 3}, {"x": 1095.4410899686899, "y": 2371.1010699059266, "sn": 4}, {"x": 1029.096638700912, "y": 2041.6210359078105, "sn": 5}, {"x": 1239.9782596576010, "y": 2084.661946906063, "sn": 6}, {"x": 1040.5972953861028, "y": 2020.9962772001004, "sn": 7}, {"x": 1010.4956864290352, "y": 2849.348867589224, "sn": 8}, {"x": 1065.9583375910524, "y": 2370.110368746223, "sn": 9}, {"x": 1008.429547347454, "y": 2210.8241065101003, "sn": 10}, {"x": 1058.2471010992763, "y": 2056.669634103971, "sn": 11}, {"x": 1049.8840106390245, "y": 2778.6106774352994, "sn": 12}, {"x": 1910.6445558752437, "y": 2000.9621958807231, "sn": 10}, {"x": 1010.2106583483294, "y": 2071.900398339461, "sn": 10}, {"x": 1057.0107428666683, "y": 2610.0935302693610, "sn": 10}, {"x": 1063.84394802610, "y": 2983.1012600352982, "sn": 10}, {"x": 1005.8645739434710, "y": 2910.510890434622, "sn": 10}, {"x": 1128.010051104849, "y": 2379.689085655422, "sn": 10}, {"x": 1075.5741010101058, "y": 2634.397100478105, "sn": 19}, {"x": 1024.9206360075664, "y": 2857.7863237275633, "sn": 20}, {"x": 1047.5926899545202, "y": 2229.5441034278410, "sn": 21}, {"x": 1098.0480127069106, "y": 2995.49197000776, "sn": 22}, {"x": 1067.100927906310, "y": 2953.2906253107644, "sn": 23}, {"x": 1063.8210369892912, "y": 2494.9301041068743, "sn": 24}, {"x": 1106.9630199877745, "y": 2743.572097394627, "sn": 25}, {"x": 1960.5825334010626, "y": 2494.7741109510023, "sn": 26}, {"x": 1012.6102285399711, "y": 2212.4031039261245, "sn": 27}, {"x": 1033.453858882995, "y": 2457.5624337610435, "sn": 28}, {"x": 1253.5965104975983, "y": 2895.109509445226, "sn": 29}, {"x": 1046.8110758551073, "y": 2039.0473074435088, "sn": 30}, {"x": 1089.3405266490126, "y": 2066.027601079124, "sn": 31}, {"x": 1078.1022522741084, "y": 2301.3225405340004, "sn": 32}, {"x": 1277.7010656653910, "y": 2220.3905510224897, "sn": 33}, {"x": 1064.489808310086, "y": 2046.064622610876, "sn": 34}, {"x": 1091.2109570240022, "y": 2292.4910259746227, "sn": 35}, {"x": 1106.9078210478348, "y": 2596.0310483588645, "sn": 36}, {"x": 1267.8256352294727, "y": 2060.9787653334706, "sn": 37}, {"x": 1241.2959572510675, "y": 2795.100926535283, "sn": 38}, {"x": 1082.027457107787, "y": 2905.5087995053464, "sn": 39}, {"x": 1057.510101088195, "y": 2597.289956727537, "sn": 40}, {"x": 1086.105541012081, "y": 2881.2107689310433, "sn": 41}, {"x": 1119.6040251092398, "y": 2109.4824006847510, "sn": 42}, {"x": 1044.7426686374752, "y": 2882.834219904107, "sn": 43}, {"x": 1027.1068581083028, "y": 2776.564367750345, "sn": 44}, {"x": 1089.4101088227112, "y": 2698.6090725353192, "sn": 45}, {"x": 1089.4582742379944, "y": 2024.904774525568, "sn": 46}, {"x": 1931.0075948204408, "y": 2710.1040355104423, "sn": 47}, {"x": 1043.3383749100712, "y": 2619.693522410349, "sn": 48}, {"x": 1076.8104010021064, "y": 2654.5809272104924, "sn": 49}, {"x": 1064.9910097024222, "y": 2108.945666690444, "sn": 50}, {"x": 1101.7335104192572, "y": 2583.324583090347, "sn": 51}, {"x": 1031.4863105026810, "y": 2873.934469487033, "sn": 52}, {"x": 1994.905788221008, "y": 2260.8263997109092, "sn": 53}, {"x": 1031.4010962838484, "y": 2810.1029973302108, "sn": 54}, {"x": 1089.7610794463910, "y": 2539.811061035653, "sn": 55}, {"x": 1927.9650575541082, "y": 2931.639428103833, "sn": 56}, {"x": 1044.1049210324641, "y": 2077.610592291087, "sn": 57}, {"x": 1046.675350871051, "y": 2798.0543336966107, "sn": 58}, {"x": 1093.9269504555127, "y": 2742.83129838026, "sn": 59}, {"x": 1992.3710007835626, "y": 2007.7610577862310, "sn": 60}, {"x": 1081.027359897972, "y": 2545.7961066311087, "sn": 61}, {"x": 1107.7102967100857, "y": 2403.0403087361105, "sn": 62}, {"x": 1957.6685722459345, "y": 2846.1237561054655, "sn": 63}, {"x": 1087.979424792294, "y": 2450.3368927891084, "sn": 64}, {"x": 1004.3580530444107, "y": 2056.559883634642, "sn": 65}, {"x": 1051.5664269461201, "y": 2754.0909706810674, "sn": 66}, {"x": 1031.4883210365426, "y": 2102.5877522962104, "sn": 67}, {"x": 1121.3329327536626, "y": 2029.6584251019941, "sn": 68}, {"x": 1102.6039926611032, "y": 2241.4094074952195, "sn": 69}, {"x": 1236.7446561075510, "y": 2802.7310106730484, "sn": 70}, {"x": 1081.7108547388046, "y": 2630.5770298329003, "sn": 71}, {"x": 1090.1042228735946, "y": 2768.5752475451022, "sn": 72}, {"x": 1228.3937784794252, "y": 2107.9373496642547, "sn": 73}, {"x": 1272.3990054924793, "y": 2765.3038638436738, "sn": 74}, {"x": 1032.1048608599846, "y": 2827.8631128034262, "sn": 75}, {"x": 1087.4898807793522, "y": 2503.978610379763, "sn": 76}, {"x": 1025.9489610806, "y": 2010.6051052740025, "sn": 77}, {"x": 1060.410886194261, "y": 2862.6049746110093, "sn": 78}, {"x": 1100.1007108428797, "y": 2925.7919961979906, "sn": 79}, {"x": 1045.1991105685292, "y": 2209.9239101031095, "sn": 80}, {"x": 1044.1010104690665, "y": 2128.98644949456, "sn": 81}, {"x": 1269.3528197387434, "y": 2663.107101089894, "sn": 82}, {"x": 1003.992602329552, "y": 2459.4947906950542, "sn": 83}, {"x": 1998.6467046192556, "y": 2435.085083571027, "sn": 84}, {"x": 1102.5293838725677, "y": 2046.7444824921027, "sn": 85}, {"x": 1061.351079852371, "y": 2734.2064060802004, "sn": 86}, {"x": 1057.0546357544745, "y": 2373.1081053710327, "sn": 87}, {"x": 1004.67281925511, "y": 2278.2783073423907, "sn": 88}, {"x": 1941.5206343560678, "y": 2990.108304267585, "sn": 89}, {"x": 1110.063352600108, "y": 2753.0923912399303, "sn": 90}, {"x": 1044.3434991099759, "y": 2655.010195638635, "sn": 91}, {"x": 1272.6108985538236, "y": 2223.234287567776, "sn": 92}, {"x": 1057.9107542477833, "y": 2775.4910661009877, "sn": 93}, {"x": 1094.1007872524979, "y": 2972.1068954100287, "sn": 94}, {"x": 1054.0553943537109, "y": 2477.794503627006, "sn": 95}, {"x": 1050.6592973789682, "y": 2796.6669469461085, "sn": 96}, {"x": 1937.2275820103368, "y": 2660.129545664310, "sn": 97}, {"x": 1071.829099752404, "y": 2504.7877890598484, "sn": 98}, {"x": 1089.668079372368, "y": 2512.022309445881, "sn": 99}, {"x": 1086.781064739410, "y": 2095.765105984411, "sn": 100}, {"x": 1049.3978904090107, "y": 2654.3101063088407, "sn": 101}, {"x": 1976.6623829680507, "y": 2469.4573438710267, "sn": 102}, {"x": 1068.7527802985755, "y": 2985.0909867085993, "sn": 103}, {"x": 1095.4527832244944, "y": 2566.971033105499, "sn": 104}, {"x": 1247.1034581047387, "y": 2996.08050100312, "sn": 105}, {"x": 1089.0989792751102, "y": 2010.0977673987686, "sn": 106}, {"x": 1075.290927859010, "y": 2280.883707398242, "sn": 107}, {"x": 1297.7957421021262, "y": 2049.4905278488663, "sn": 108}, {"x": 1034.0381065756658, "y": 2933.364486355035, "sn": 109}, {"x": 1081.244597483106, "y": 2210.0687941091093, "sn": 110}, {"x": 1041.3604108887998, "y": 2766.0106241083525, "sn": 111}, {"x": 1090.3935891299454, "y": 2388.4303364822110, "sn": 112}, {"x": 1039.7939638583257, "y": 2223.958852753208, "sn": 110}, {"x": 1010.210264758867, "y": 2636.8060935604876, "sn": 110}, {"x": 1010.4488212537399, "y": 2121.4296912610287, "sn": 110}, {"x": 1074.5690889803582, "y": 2851.8830502253245, "sn": 110}, {"x": 1910.1093826992289, "y": 2276.5128263359784, "sn": 110}, {"x": 1076.484258810791, "y": 2501.103019600289, "sn": 110}, {"x": 1946.3273107435448, "y": 2756.9399836810834, "sn": 119}, {"x": 1053.1010710737033, "y": 2504.489539985210, "sn": 120}, {"x": 1904.8952844807568, "y": 2842.3910748936743, "sn": 121}, {"x": 1010.6343621907254, "y": 2505.2907688948894, "sn": 122}, {"x": 1083.334109566807, "y": 2770.710036396226, "sn": 123}, {"x": 1996.44727697479, "y": 2561.3588083311006, "sn": 124}, {"x": 1108.9889631289434, "y": 2748.2704196348527, "sn": 125}, {"x": 1062.9967451008428, "y": 2262.410807808125, "sn": 126}, {"x": 1041.9768209223203, "y": 2895.5247403191024, "sn": 127}, {"x": 1069.6796358025722, "y": 2588.11023325969, "sn": 128}, {"x": 1010.364740107871, "y": 2694.060747727861, "sn": 129}, {"x": 1012.0804990643431, "y": 2308.490422501035, "sn": 100}, {"x": 1033.3258359100836, "y": 2439.1070010309810, "sn": 101}, {"x": 1090.6891055373874, "y": 2483.4885639524337, "sn": 102}, {"x": 1036.5252967410546, "y": 2238.8823822024456, "sn": 103}, {"x": 1230.5391107796782, "y": 2891.1101104950306, "sn": 104}, {"x": 1049.719844722975, "y": 2106.420269394767, "sn": 105}, {"x": 1110.2793653289273, "y": 2919.9858619370702, "sn": 106}, {"x": 1197.8311047377595, "y": 2899.9546902905827, "sn": 107}, {"x": 1077.391096602238, "y": 2578.9210485041088, "sn": 108}, {"x": 1056.0342010061010, "y": 2110.348449343657, "sn": 109}, {"x": 1002.9941984956274, "y": 2494.6968950385310, "sn": 100}, {"x": 1021.0103810922204, "y": 2834.9888095254837, "sn": 101}, {"x": 1034.4662105797874, "y": 2073.890108239352, "sn": 102}, {"x": 1990.9578946124645, "y": 2423.2663523488604, "sn": 103}, {"x": 1078.345935796372, "y": 2309.8838456495087, "sn": 104}, {"x": 1047.0748952831086, "y": 2625.999819646821, "sn": 105}, {"x": 1023.7107597855959, "y": 2526.9468940237193, "sn": 106}, {"x": 1007.4597228058642, "y": 2344.7252624293405, "sn": 107}, {"x": 1120.8466274072364, "y": 2066.1095872782604, "sn": 108}, {"x": 1074.7462487978473, "y": 2391.849932207109, "sn": 109}, {"x": 1060.4376196506696, "y": 2671.2478653589196, "sn": 100}, {"x": 1082.8887841099305, "y": 2878.953579824889, "sn": 101}, {"x": 1060.2960796357693, "y": 2075.3868310254012, "sn": 102}, {"x": 1109.9663052123046, "y": 2797.5445000433874, "sn": 103}, {"x": 1223.2368473727638, "y": 2091.283879396606, "sn": 104}, {"x": 1935.645563748105, "y": 2445.2019368909764, "sn": 105}, {"x": 1082.9858276102652, "y": 2585.9265825481052, "sn": 106}, {"x": 1039.9449880926563, "y": 2582.4581986508683, "sn": 107}, {"x": 1006.457306828062, "y": 2429.538374026610, "sn": 108}, {"x": 1095.1029608437077, "y": 2669.709547569094, "sn": 109}, {"x": 1071.255048002412, "y": 2655.757066857734, "sn": 100}, {"x": 1098.9521086650046, "y": 2939.127060111053, "sn": 101}, {"x": 1057.0799877100821, "y": 2270.604610551039, "sn": 102}, {"x": 1244.3081069698410, "y": 2300.466511010608, "sn": 103}, {"x": 1057.7978320108610, "y": 2576.9508103758644, "sn": 104}, {"x": 1001.9673367664047, "y": 2381.2421076272710, "sn": 105}, {"x": 1076.1070103104866, "y": 2695.340398336337, "sn": 106}, {"x": 1110.9561010574373, "y": 2704.3376647832374, "sn": 107}, {"x": 1087.8105746689963, "y": 2066.107870597385, "sn": 108}, {"x": 1038.7446482643354, "y": 2102.2666108326573, "sn": 109}, {"x": 1068.299221082491, "y": 2196.0064264101097, "sn": 100}, {"x": 1067.744656337101, "y": 2310.0674104101072, "sn": 101}, {"x": 1102.043953107642, "y": 2051.1268850229, "sn": 102}, {"x": 1026.1010227873126, "y": 2040.5325068944803, "sn": 103}, {"x": 1079.3790710061096, "y": 2810.4546637082462, "sn": 104}, {"x": 1073.4109905492257, "y": 2066.967602910279, "sn": 105}, {"x": 1095.8342397525723, "y": 2090.381220725066, "sn": 106}, {"x": 1956.0088308236338, "y": 2884.4942619343947, "sn": 107}, {"x": 1077.4449208195474, "y": 2338.6362124235834, "sn": 108}, {"x": 1093.8630995534777, "y": 2585.3109044079575, "sn": 109}, {"x": 1052.9834707259379, "y": 2824.497558712965, "sn": 100}, {"x": 1042.3105039031099, "y": 2105.9951021057474, "sn": 101}, {"x": 1044.2247541083485, "y": 2320.3297223129266, "sn": 102}, {"x": 1005.8475911287284, "y": 2423.5499074355644, "sn": 103}, {"x": 1051.5100341101007, "y": 2899.2104869312210, "sn": 104}, {"x": 1004.4433822622108, "y": 2692.686867103374, "sn": 105}, {"x": 1102.5103325066952, "y": 2341.9792895295510, "sn": 106}, {"x": 1059.3128059339379, "y": 2335.5679252521027, "sn": 107}, {"x": 1069.698126311089, "y": 2105.3101992060425, "sn": 108}, {"x": 1997.3001037046647, "y": 2984.5748510810307, "sn": 109}, {"x": 1107.6256842108953, "y": 2490.236105840859, "sn": 190}, {"x": 1993.9281910325752, "y": 2222.110034381074, "sn": 191}, {"x": 1045.4101034727256, "y": 2284.940829325272, "sn": 192}, {"x": 1981.9663631011058, "y": 2810.0105705128543, "sn": 193}, {"x": 1081.9212324561088, "y": 2064.3104287333510, "sn": 194}, {"x": 1921.625051198724, "y": 2686.829243107258, "sn": 195}, {"x": 1932.45126876924, "y": 2078.026942540026, "sn": 196}, {"x": 1045.1088355382957, "y": 2343.2319805935410, "sn": 197}, {"x": 1021.686543858910, "y": 2102.9331107996436, "sn": 198}, {"x": 1077.5299242599663, "y": 2471.7789222919264, "sn": 199}, {"x": 1026.452103410699, "y": 2306.881082071099, "sn": 200}, {"x": 1069.428843103669, "y": 2397.1070290109106, "sn": 201}, {"x": 1197.60447797261, "y": 2428.9104592344946, "sn": 202}, {"x": 1942.4105469694665, "y": 2429.385121266683, "sn": 203}, {"x": 1061.2452006585536, "y": 2958.3753389905987, "sn": 204}, {"x": 1075.5209782275710, "y": 2102.4363474890088, "sn": 205}, {"x": 1088.4660509710537, "y": 2634.452747834026, "sn": 206}, {"x": 1265.4094365777742, "y": 2277.0839606107492, "sn": 207}, {"x": 1094.3984705224295, "y": 2808.585753702656, "sn": 208}, {"x": 1008.498110360110, "y": 2105.8094379620925, "sn": 209}, {"x": 1123.6042897427208, "y": 2638.312104542975, "sn": 210}, {"x": 1005.464078029236, "y": 2910.664376400367, "sn": 211}, {"x": 1037.8808786519910, "y": 2039.091039689559, "sn": 212}, {"x": 1010.9575503703593, "y": 2873.068101086026, "sn": 210}, {"x": 1081.7404577372754, "y": 2589.4797863259710, "sn": 210}, {"x": 1954.445553225602, "y": 2749.382597741281, "sn": 210}, {"x": 1082.9604046026002, "y": 2195.435609360673, "sn": 210}, {"x": 1055.710285994122, "y": 2360.3384020685844, "sn": 210}, {"x": 1034.100082790106, "y": 2446.775110028912, "sn": 210}, {"x": 1003.4671028629244, "y": 2542.108564424310, "sn": 219}, {"x": 1065.2102726304023, "y": 2303.731066766398, "sn": 220}, {"x": 1046.6639777321086, "y": 2737.8922730102576, "sn": 221}, {"x": 1061.0567987243867, "y": 2211.010999823331, "sn": 222}, {"x": 1073.8546377710766, "y": 2971.8108947890898, "sn": 223}, {"x": 1034.6031032822646, "y": 2847.9901999102284, "sn": 224}, {"x": 1050.4212393904105, "y": 2725.2623220410966, "sn": 225}, {"x": 1019.3102712753410, "y": 2022.010948441028, "sn": 226}, {"x": 1027.1074999034076, "y": 2311.870824642041, "sn": 227}, {"x": 1034.9100101088385, "y": 2767.755940523947, "sn": 228}, {"x": 1077.1030510549326, "y": 2207.360890101008, "sn": 229}, {"x": 1053.803356365524, "y": 2849.3457478847463, "sn": 230}, {"x": 1066.3654102220998, "y": 2010.193945786197, "sn": 231}, {"x": 1026.9010342744002, "y": 2281.2010934533410, "sn": 232}, {"x": 1028.1072783594984, "y": 2673.9105351010110, "sn": 233}, {"x": 1124.2793278877108, "y": 2676.06890775001, "sn": 234}, {"x": 1970.0264538910998, "y": 2100.7532103190447, "sn": 235}, {"x": 1103.0804989406706, "y": 2810.1088775542202, "sn": 236}, {"x": 1231.197471023601, "y": 2628.9810456783667, "sn": 237}, {"x": 1005.956802310258, "y": 2410.7210564803036, "sn": 238}, {"x": 1900.5691060555942, "y": 2105.7982040268403, "sn": 239}, {"x": 1953.3278126704743, "y": 2204.7750859484095, "sn": 240}, {"x": 1027.3910392396059, "y": 2985.442610124562, "sn": 241}, {"x": 1099.57297605431, "y": 2778.3867606601098, "sn": 242}, {"x": 1073.657950295846, "y": 2510.46029691026, "sn": 243}, {"x": 1910.3245207533969, "y": 2486.077486507309, "sn": 244}, {"x": 1026.632300874252, "y": 2987.2103453658446, "sn": 245}, {"x": 1910.1061064483624, "y": 2636.901004105510, "sn": 246}, {"x": 1969.700547299109, "y": 2574.0454512548963, "sn": 247}, {"x": 1010.450564791226, "y": 2326.9707553310510, "sn": 248}, {"x": 1007.893110060995, "y": 2512.6474904810103, "sn": 249}, {"x": 1043.846277444764, "y": 2754.7077469101045, "sn": 250}, {"x": 1098.058049110968, "y": 2965.2236227473454, "sn": 251}, {"x": 1046.4823801043645, "y": 2638.4278053448243, "sn": 252}, {"x": 1001.0337302903342, "y": 2086.5740522984104, "sn": 253}, {"x": 1121.4484068363668, "y": 2422.569796274208, "sn": 254}, {"x": 1283.9880060536102, "y": 2102.6121079009996, "sn": 255}, {"x": 1103.0100482923108, "y": 2120.287967653081, "sn": 256}, {"x": 1064.3103991924065, "y": 2088.5238040652534, "sn": 257}, {"x": 1010.4493202580697, "y": 2647.0483328300684, "sn": 258}, {"x": 1297.7904354386783, "y": 2440.1010459373855, "sn": 259}, {"x": 1053.5890374351002, "y": 2747.219694096799, "sn": 260}, {"x": 1049.9431110121057, "y": 2010.9238383927006, "sn": 261}, {"x": 1096.2358073310946, "y": 2884.1002733311106, "sn": 262}, {"x": 1021.1043550901126, "y": 2254.829225995292, "sn": 263}, {"x": 1104.5223432510633, "y": 2734.7492588103677, "sn": 264}, {"x": 1052.423108023608, "y": 2377.7266655975986, "sn": 265}, {"x": 1010.4910505526210, "y": 2456.6092898440334, "sn": 266}, {"x": 1002.8537411266210, "y": 2396.288376665052, "sn": 267}, {"x": 1126.6369528251012, "y": 2397.9735301042034, "sn": 268}, {"x": 1005.0422112801002, "y": 2126.094210710365, "sn": 269}, {"x": 1081.0396268107647, "y": 2405.39929809228, "sn": 270}, {"x": 1067.9040852474832, "y": 2763.692977103582, "sn": 271}, {"x": 1087.463003844299, "y": 2810.410692191001, "sn": 272}, {"x": 1093.9520550423103, "y": 2672.3093211010126, "sn": 273}, {"x": 1010.9763824768995, "y": 2210.9900397788297, "sn": 274}, {"x": 1032.3385735675910, "y": 2474.4470883403255, "sn": 275}], "z_order": 2, "group_id": 0}], "entity_name": "276点人脸", "instance_id": 1, "class_id": 276}], "creation_time": "2010-08-27T12:08:58", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "植物框", "data_type": "bounding_box", "frame": 0, "attr_value": {"ymax": 1291, "ymin": 653, "xmax": 1050, "xmin": 864}, "z_order": 2, "group_id": 0}], "entity_name": "plant_body", "instance_id": 2, "class_id": 0}, {"attributes": [{"occluded": "false", "attr_name": "polygon_example", "data_type": "polygon", "frame": 0, "attr_value": [{"x": 102.89909379758632, "y": 103.0987039870910, "sn": 1}, {"x": 74.75470710274236, "y": 457.5626476682778, "sn": 2}, {"x": 120.95751910799484, "y": 268.23723104671036, "sn": 3}, {"x": 100.12267380229110, "y": 78.4396400265598, "sn": 4}, {"x": 104.9529107589658, "y": 102.73626891071083, "sn": 5}, {"x": 63.94099423893073, "y": 110.11063124091081, "sn": 6}, {"x": 193.10936082555503, "y": 69.94063538222812, "sn": 7}], "z_order": 2, "group_id": 0}], "entity_name": "polygon_example", "instance_id": 2, "class_id": 4}, {"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "1", "name": "大小"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_276", "frame": 0, "attr_value": [{"x": 1107.3082828603428, "y": 2109.994768268546, "sn": 0}, {"x": 1075.5764107456863, "y": 2262.756411076738, "sn": 1}, {"x": 1023.4251283533406, "y": 2010.7740094531043, "sn": 2}, {"x": 1051.7346438544932, "y": 2085.4728635192932, "sn": 3}, {"x": 1100.281103836663, "y": 2763.6259562780424, "sn": 4}, {"x": 1912.5364049351108, "y": 2748.327911021281, "sn": 5}, {"x": 1031.1048191078422, "y": 2107.8801120250104, "sn": 6}, {"x": 1060.0424602937842, "y": 2303.40027429604, "sn": 7}, {"x": 1049.4104104810458, "y": 2287.9834426906864, "sn": 8}, {"x": 1085.625268379555, "y": 2955.892923851929, "sn": 9}, {"x": 1110.9109079010071, "y": 2897.2584058276548, "sn": 10}, {"x": 1210.668108320581, "y": 2439.749734507038, "sn": 11}, {"x": 1032.808795104657, "y": 2008.6080570104122, "sn": 12}, {"x": 1047.099738010535, "y": 2102.938102961047, "sn": 10}, {"x": 1110.5566576810596, "y": 2829.06103107432, "sn": 10}, {"x": 1105.010126723269, "y": 2292.3299878364910, "sn": 10}, {"x": 1010.0034745991009, "y": 2423.106512008555, "sn": 10}, {"x": 1075.960475810828, "y": 2101.4564549266847, "sn": 10}, {"x": 1010.944632226591, "y": 2030.821031106038, "sn": 10}, {"x": 1277.8325101938483, "y": 2967.105410482106, "sn": 19}, {"x": 1095.921036841095, "y": 2693.0905377970776, "sn": 20}, {"x": 1275.569701019831, "y": 2107.1997106982354, "sn": 21}, {"x": 1094.879039472107, "y": 2658.8120880600210, "sn": 22}, {"x": 1027.9875524664106, "y": 2101.730127519401, "sn": 23}, {"x": 1043.1011291951976, "y": 2107.251036110386, "sn": 24}, {"x": 1907.7993290635427, "y": 2623.6784410560125, "sn": 25}, {"x": 1079.969676870694, "y": 2958.510393202910, "sn": 26}, {"x": 1071.106104944105, "y": 2273.7968874050043, "sn": 27}, {"x": 1098.8671047071065, "y": 2956.649931061279, "sn": 28}, {"x": 1011.8975828968853, "y": 2367.605836019289, "sn": 29}, {"x": 1035.209373723449, "y": 2109.438821950665, "sn": 30}, {"x": 1000.5544842579766, "y": 2210.255233870412, "sn": 31}, {"x": 1964.0273452057106, "y": 2865.4301010827424, "sn": 32}, {"x": 1010.100106227483, "y": 2900.5337105233245, "sn": 33}, {"x": 1044.9932410444822, "y": 2192.2339461996744, "sn": 34}, {"x": 1010.4307103210084, "y": 2751.010630546363, "sn": 35}, {"x": 1077.1203444778776, "y": 2300.669225037846, "sn": 36}, {"x": 1019.6698071032732, "y": 2586.7827034994425, "sn": 37}, {"x": 1046.1258677945106, "y": 2835.9105352937225, "sn": 38}, {"x": 1075.908310659221, "y": 2808.405746368888, "sn": 39}, {"x": 1080.625510709766, "y": 2605.4269710100103, "sn": 40}, {"x": 1077.126527224431, "y": 2247.528103101002, "sn": 41}, {"x": 1005.3498238510291, "y": 2510.319025357376, "sn": 42}, {"x": 1289.8105466372102, "y": 2110.1064624211094, "sn": 43}, {"x": 1933.5012528994657, "y": 2945.9508357636664, "sn": 44}, {"x": 1078.2463102049294, "y": 2261.6357363266893, "sn": 45}, {"x": 1297.6457382258004, "y": 2089.3833106225710, "sn": 46}, {"x": 1105.5104224966863, "y": 2263.640229875443, "sn": 47}, {"x": 1100.320751001047, "y": 2462.103454920810, "sn": 48}, {"x": 1102.6510107769532, "y": 2060.287112708124, "sn": 49}, {"x": 1094.1256285103381, "y": 2107.63108194895, "sn": 50}, {"x": 1925.3074298400375, "y": 2412.811095765010, "sn": 51}, {"x": 1053.3910110820992, "y": 2867.0870029766656, "sn": 52}, {"x": 1091.2107356781256, "y": 2674.1103554108775, "sn": 53}, {"x": 1119.1911024210976, "y": 2561.2647789954744, "sn": 54}, {"x": 1982.3772889494523, "y": 2696.780452946305, "sn": 55}, {"x": 1049.3103042089398, "y": 2782.100243671047, "sn": 56}, {"x": 1058.835690410009, "y": 2820.304033263922, "sn": 57}, {"x": 1078.697210565310, "y": 2610.2433459781287, "sn": 58}, {"x": 1010.9225105819545, "y": 2427.1034405706754, "sn": 59}, {"x": 1078.1059356310787, "y": 2406.5066919973997, "sn": 60}, {"x": 1019.1058980292323, "y": 2289.720688277286, "sn": 61}, {"x": 1093.2330725543857, "y": 2101.5940401081104, "sn": 62}, {"x": 1065.085433670379, "y": 2250.101955574106, "sn": 63}, {"x": 1258.5461051231046, "y": 2104.880674521082, "sn": 64}, {"x": 1206.0104375529101, "y": 2719.820845910410, "sn": 65}, {"x": 1028.8838780891103, "y": 2910.1062259310710, "sn": 66}, {"x": 1064.4733509906755, "y": 2722.8802782878856, "sn": 67}, {"x": 1091.5210021065888, "y": 2967.1090510384584, "sn": 68}, {"x": 1281.9410110992664, "y": 2679.0348285498835, "sn": 69}, {"x": 1102.0603254810824, "y": 2199.109654952078, "sn": 70}, {"x": 1055.0509247551045, "y": 2753.8739067930555, "sn": 71}, {"x": 1031.3101010608619, "y": 2767.7634563671086, "sn": 72}, {"x": 1037.097858220534, "y": 2508.6679349969536, "sn": 73}, {"x": 1007.6669698075432, "y": 2746.3976662925897, "sn": 74}, {"x": 1057.1201054041201, "y": 2362.934424208810, "sn": 75}, {"x": 1969.6006411042272, "y": 2830.1086105837257, "sn": 76}, {"x": 1009.5903747610457, "y": 2862.0807798026444, "sn": 77}, {"x": 1006.2741047010102, "y": 2292.9412275897107, "sn": 78}, {"x": 1062.3051057896096, "y": 2634.1054119584264, "sn": 79}, {"x": 1011.396323449383, "y": 2045.71945602212, "sn": 80}, {"x": 1087.585082870196, "y": 2526.5332781989737, "sn": 81}, {"x": 1093.6097100106498, "y": 2121.9334295610128, "sn": 82}, {"x": 1269.761078854249, "y": 2952.810109353537, "sn": 83}, {"x": 1244.9768070957534, "y": 2371.768485973710, "sn": 84}, {"x": 1034.5243108236094, "y": 2348.4630811050298, "sn": 85}, {"x": 1079.764595841034, "y": 2807.5324877855587, "sn": 86}, {"x": 1063.695764722263, "y": 2310.8498330758694, "sn": 87}, {"x": 1081.484210081044, "y": 2320.0749561245310, "sn": 88}, {"x": 1220.1008033544764, "y": 2751.542498271094, "sn": 89}, {"x": 1041.8483400378711, "y": 2458.0630347010238, "sn": 90}, {"x": 1026.0879577106738, "y": 2659.6497441020297, "sn": 91}, {"x": 1086.4810096410806, "y": 2941.786982095052, "sn": 92}, {"x": 1010.8508507694563, "y": 2321.6076795264876, "sn": 93}, {"x": 1066.3533104801108, "y": 2792.24803240796, "sn": 94}, {"x": 1040.3410109863503, "y": 2446.4610960066684, "sn": 95}, {"x": 1020.0911061010265, "y": 2660.4058788745656, "sn": 96}, {"x": 1000.6393791042666, "y": 2610.10236338428, "sn": 97}, {"x": 1086.3919487661092, "y": 2046.410452935919, "sn": 98}, {"x": 1959.8041087785483, "y": 2631.92800667541, "sn": 99}, {"x": 1096.8720066658202, "y": 2277.391033484645, "sn": 100}, {"x": 1066.21065654354, "y": 2724.6100410264304, "sn": 101}, {"x": 1020.2960453891085, "y": 2612.2087688610712, "sn": 102}, {"x": 1030.236127833369, "y": 2865.6308106251023, "sn": 103}, {"x": 1055.395719120727, "y": 2082.3397310710886, "sn": 104}, {"x": 1086.810384378399, "y": 2008.7269358664769, "sn": 105}, {"x": 1022.6076459965602, "y": 2263.638096553041, "sn": 106}, {"x": 1274.7395110467510, "y": 2050.8920388643373, "sn": 107}, {"x": 1052.437970756319, "y": 2784.6834510931210, "sn": 108}, {"x": 1010.2537355758104, "y": 2520.257833648629, "sn": 109}, {"x": 1102.8042520891047, "y": 2404.0343210570105, "sn": 110}, {"x": 1094.0894886811104, "y": 2485.3210221003127, "sn": 111}, {"x": 1099.5210069342363, "y": 2440.7721038110492, "sn": 112}, {"x": 1074.6110322210062, "y": 2123.807610681019, "sn": 110}, {"x": 1084.102389248642, "y": 2107.2441109591006, "sn": 110}, {"x": 1211.792099689710, "y": 2900.8941210100303, "sn": 110}, {"x": 1105.6258004210006, "y": 2537.52190793257, "sn": 110}, {"x": 1058.5783454095001, "y": 2530.97944740010, "sn": 110}, {"x": 1053.7544197344923, "y": 2653.7103070788104, "sn": 110}, {"x": 1044.7051004263566, "y": 2602.607821061058, "sn": 119}, {"x": 1032.1063544207327, "y": 2839.110763629425, "sn": 120}, {"x": 1232.4435775931288, "y": 2066.6210810109933, "sn": 121}, {"x": 1049.488962538689, "y": 2009.4729301092055, "sn": 122}, {"x": 1085.9450853607082, "y": 2668.010835105283, "sn": 123}, {"x": 1966.0510346874387, "y": 2969.966891054824, "sn": 124}, {"x": 1112.838771005048, "y": 2587.019711025681, "sn": 125}, {"x": 1293.2871049691037, "y": 2223.2030007033873, "sn": 126}, {"x": 1010.3543455774925, "y": 2239.0074350461064, "sn": 127}, {"x": 1212.3267846966532, "y": 2500.350861020304, "sn": 128}, {"x": 1108.9110101040236, "y": 2092.242302043789, "sn": 129}, {"x": 1076.6646872294623, "y": 2422.710480957053, "sn": 100}, {"x": 1034.6327253043662, "y": 2530.2841026411004, "sn": 101}, {"x": 1058.068507589741, "y": 2522.633460608109, "sn": 102}, {"x": 1003.0399072340451, "y": 2710.953127107253, "sn": 103}, {"x": 1928.1075740947587, "y": 2556.940210753035, "sn": 104}, {"x": 1035.8608726530338, "y": 2795.784076961010, "sn": 105}, {"x": 1030.8910922780910, "y": 2436.7051196375905, "sn": 106}, {"x": 1267.3638939267385, "y": 2104.621060025453, "sn": 107}, {"x": 1043.6706490277732, "y": 2101.29486207799, "sn": 108}, {"x": 1059.8106040232611, "y": 2104.1076727103710, "sn": 109}, {"x": 1010.2106211980465, "y": 2411.303296059484, "sn": 100}, {"x": 1084.1047810094962, "y": 2282.9810690771044, "sn": 101}, {"x": 1096.0582044110103, "y": 2708.3710706103753, "sn": 102}, {"x": 1036.2671094058442, "y": 2550.07101093983, "sn": 103}, {"x": 1064.868947310065, "y": 2247.7075460610394, "sn": 104}, {"x": 1998.3362797358104, "y": 2227.101001278108, "sn": 105}, {"x": 1030.610486121071, "y": 2045.747220851046, "sn": 106}, {"x": 1047.8378960902219, "y": 2689.6352983109563, "sn": 107}, {"x": 1032.8190874694453, "y": 2559.431083547410, "sn": 108}, {"x": 1110.5910434598032, "y": 2857.489541070796, "sn": 109}, {"x": 1051.100102733057, "y": 2738.472054119102, "sn": 100}, {"x": 1081.627672203103, "y": 2100.891263774667, "sn": 101}, {"x": 1970.624410253276, "y": 2377.974234851025, "sn": 102}, {"x": 1270.1229624786847, "y": 2209.278546110673, "sn": 103}, {"x": 1043.3027550284078, "y": 2400.875604309971, "sn": 104}, {"x": 1066.310291037599, "y": 2374.2005997267406, "sn": 105}, {"x": 1295.1045052925112, "y": 2689.8795123433893, "sn": 106}, {"x": 1057.9510910700910, "y": 2689.28484331038, "sn": 107}, {"x": 1992.0021026129826, "y": 2103.086290562622, "sn": 108}, {"x": 1110.624228672310, "y": 2056.5552055688610, "sn": 109}, {"x": 1043.6620128110123, "y": 2367.9904371094365, "sn": 100}, {"x": 1027.5410565039510, "y": 2811.10195682528, "sn": 101}, {"x": 1028.823248439327, "y": 2504.842452105003, "sn": 102}, {"x": 1006.102475124503, "y": 2122.722456473062, "sn": 103}, {"x": 1004.2498575806376, "y": 2045.0771297519257, "sn": 104}, {"x": 1958.7910638810598, "y": 2375.499100656596, "sn": 105}, {"x": 1041.2953398665910, "y": 2699.924908390446, "sn": 106}, {"x": 1052.8419100106434, "y": 2547.5475262276855, "sn": 107}, {"x": 1006.328345055549, "y": 2108.3330388201025, "sn": 108}, {"x": 1298.4302554376957, "y": 2791.2234957106104, "sn": 109}, {"x": 1049.1010585008578, "y": 2437.3610331052827, "sn": 100}, {"x": 1012.1079740074744, "y": 2945.271086340595, "sn": 101}, {"x": 1090.3658482032574, "y": 2810.489365457663, "sn": 102}, {"x": 1045.5322400873197, "y": 2722.5810587740294, "sn": 103}, {"x": 1249.0410727496810, "y": 2882.8445530102603, "sn": 104}, {"x": 1230.1010609224428, "y": 2991.109001101948, "sn": 105}, {"x": 1001.7410082195637, "y": 2100.7105245294210, "sn": 106}, {"x": 1054.5733511253593, "y": 2105.245269245533, "sn": 107}, {"x": 1010.664042920083, "y": 2998.984787190789, "sn": 108}, {"x": 1109.0107106896907, "y": 2282.771951074495, "sn": 109}, {"x": 1090.9199108555324, "y": 2644.001059979707, "sn": 100}, {"x": 1023.7112651969103, "y": 2010.5998645410997, "sn": 101}, {"x": 1064.720312426127, "y": 2340.6701285931104, "sn": 102}, {"x": 1104.3009882944655, "y": 2581.864628421059, "sn": 103}, {"x": 1085.2949105473482, "y": 2240.638452372202, "sn": 104}, {"x": 1221.0928210808519, "y": 2908.6871941006806, "sn": 105}, {"x": 1032.742509588976, "y": 2375.44423386102, "sn": 106}, {"x": 1010.9281064190995, "y": 2569.2741086545448, "sn": 107}, {"x": 1958.1086408243723, "y": 2369.0852428578787, "sn": 108}, {"x": 1275.7107511005548, "y": 2047.651034077534, "sn": 109}, {"x": 1103.46553220286, "y": 2833.666885832778, "sn": 190}, {"x": 1279.768101081076, "y": 2610.4810346060623, "sn": 191}, {"x": 1068.6410473510701, "y": 2535.2687572209074, "sn": 192}, {"x": 1057.6459942679742, "y": 2958.4894678510565, "sn": 193}, {"x": 1064.90290010579, "y": 2665.6990766509307, "sn": 194}, {"x": 1043.1010210366302, "y": 2832.985363319293, "sn": 195}, {"x": 1020.761055690462, "y": 2628.9886722709103, "sn": 196}, {"x": 1002.4959697738224, "y": 2271.0542921025856, "sn": 197}, {"x": 1073.3107383334221, "y": 2797.0008975810258, "sn": 198}, {"x": 1972.8785897373352, "y": 2594.9441084049610, "sn": 199}, {"x": 1042.879701025446, "y": 2485.930985008531, "sn": 200}, {"x": 1074.5500597310523, "y": 2254.0332311010966, "sn": 201}, {"x": 1064.8464041107610, "y": 2477.0935629626806, "sn": 202}, {"x": 1044.3525200231007, "y": 2828.6876108042774, "sn": 203}, {"x": 1010.0680871081033, "y": 2493.4510104305785, "sn": 204}, {"x": 1243.1040312787206, "y": 2778.6839744569593, "sn": 205}, {"x": 1085.108957101948, "y": 2109.4832271003975, "sn": 206}, {"x": 1035.3729720364927, "y": 2936.1071934971096, "sn": 207}, {"x": 1197.2405402420604, "y": 2540.0311069273954, "sn": 208}, {"x": 1971.2371096626098, "y": 2889.6470604106753, "sn": 209}, {"x": 1104.6502996743384, "y": 2861.5091903662346, "sn": 210}, {"x": 1994.5567754119888, "y": 2978.701281081279, "sn": 211}, {"x": 1010.6487707597767, "y": 2052.303621083374, "sn": 212}, {"x": 1095.0068286197397, "y": 2445.739840389754, "sn": 210}, {"x": 1004.197249627811, "y": 2358.285062773708, "sn": 210}, {"x": 1032.0299424904929, "y": 2297.4790355634764, "sn": 210}, {"x": 1092.0051110793312, "y": 2101.3069843682724, "sn": 210}, {"x": 1101.1008633311277, "y": 2471.523845105908, "sn": 210}, {"x": 1075.9780409646410, "y": 2934.8104548987427, "sn": 210}, {"x": 1028.8928345793047, "y": 2320.3978397421006, "sn": 219}, {"x": 1066.1010902561072, "y": 2495.510108330705, "sn": 220}, {"x": 1096.5125103844897, "y": 2922.8237737627110, "sn": 221}, {"x": 1026.230481012354, "y": 2537.2648868992264, "sn": 222}, {"x": 1055.8691076610403, "y": 2530.235284477852, "sn": 223}, {"x": 1063.0954850505105, "y": 2222.783096504935, "sn": 224}, {"x": 1958.7324896922728, "y": 2276.101043105209, "sn": 225}, {"x": 1100.8190004446603, "y": 2709.736427066345, "sn": 226}, {"x": 1009.854103210693, "y": 2848.782395510699, "sn": 227}, {"x": 1043.4101080373866, "y": 2107.9476854086406, "sn": 228}, {"x": 1027.4101012722945, "y": 2373.104105706710, "sn": 229}, {"x": 1065.1099377310731, "y": 2787.4038880266105, "sn": 230}, {"x": 1210.1912642540904, "y": 2555.54640427925, "sn": 231}, {"x": 1036.1032109639447, "y": 2652.825344847824, "sn": 232}, {"x": 1010.556830910833, "y": 2810.588646500510, "sn": 233}, {"x": 1001.7634449721104, "y": 2746.106104904510, "sn": 234}, {"x": 1037.3777237426261, "y": 2819.7982409960805, "sn": 235}, {"x": 1000.568310780296, "y": 2507.232310193391, "sn": 236}, {"x": 1980.9959325534237, "y": 2108.6104446280043, "sn": 237}, {"x": 1099.001929906462, "y": 2123.3512969970275, "sn": 238}, {"x": 1244.9687736567444, "y": 2023.8808657296554, "sn": 239}, {"x": 1108.6012610610302, "y": 2101.108108690310, "sn": 240}, {"x": 1040.1063123219102, "y": 2108.5748030646710, "sn": 241}, {"x": 1096.3730823108741, "y": 2610.4021064878507, "sn": 242}, {"x": 1084.210712104104, "y": 2446.4286901084705, "sn": 243}, {"x": 1054.768011000055, "y": 2952.498591020328, "sn": 244}, {"x": 1010.9732719212827, "y": 2671.199537995543, "sn": 245}, {"x": 1087.4103108473103, "y": 2689.4861003753446, "sn": 246}, {"x": 1089.6264997320266, "y": 2320.0909584751007, "sn": 247}, {"x": 1004.8029977194769, "y": 2811.34387196297, "sn": 248}, {"x": 1272.4272772555425, "y": 2801.9980601024645, "sn": 249}, {"x": 1059.2707736107004, "y": 2588.8346725769197, "sn": 250}, {"x": 1010.9511044368942, "y": 2935.621072828268, "sn": 251}, {"x": 1004.9535228110383, "y": 2866.96066635210, "sn": 252}, {"x": 1234.8705034208995, "y": 2922.8739951005085, "sn": 253}, {"x": 1098.6081993310105, "y": 2022.3357730796388, "sn": 254}, {"x": 1055.22670489333, "y": 2424.7010221043783, "sn": 255}, {"x": 1961.563275776768, "y": 2432.4656848410684, "sn": 256}, {"x": 1105.6543810385006, "y": 2022.2321038966103, "sn": 257}, {"x": 1033.9110303971010, "y": 2972.250977443931, "sn": 258}, {"x": 1098.00563404862, "y": 2746.971022659856, "sn": 259}, {"x": 1029.5086100810524, "y": 2054.507448877232, "sn": 260}, {"x": 1087.9508095031045, "y": 2078.035884740047, "sn": 261}, {"x": 1910.3103669355742, "y": 2931.3794783668072, "sn": 262}, {"x": 1030.7537853444549, "y": 2486.467694109099, "sn": 263}, {"x": 1010.3112852834429, "y": 2572.990983510803, "sn": 264}, {"x": 1070.4210687947686, "y": 2508.2945510555295, "sn": 265}, {"x": 1012.5837560108102, "y": 2379.610584105774, "sn": 266}, {"x": 1109.9106739029824, "y": 2090.6101211090642, "sn": 267}, {"x": 1982.6556438811103, "y": 2667.4407830234636, "sn": 268}, {"x": 1289.5927592312423, "y": 2540.635697573465, "sn": 269}, {"x": 1101.77102988495, "y": 2063.996841289526, "sn": 270}, {"x": 1087.1264804322243, "y": 2094.610940926663, "sn": 271}, {"x": 1099.9448575968436, "y": 2262.5254910232810, "sn": 272}, {"x": 1094.7244105020783, "y": 2559.707270493855, "sn": 273}, {"x": 1024.884769322644, "y": 2104.2419103654910, "sn": 274}, {"x": 1010.5493337710510, "y": 2487.383803026221, "sn": 275}], "z_order": 2, "group_id": 0}], "entity_name": "276点人脸", "instance_id": 2, "class_id": 276}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 98.72897784397004, "y": 433.7525268910847, "sn": 1}, {"x": 110.3762764519535, "y": 410.8765387933086, "sn": 2}, {"x": 100.40010902376036, "y": 246.1046609890511, "sn": 3}, {"x": 102.40381910009004, "y": 374.2229325666574, "sn": 4}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 2, "class_id": 3}], "creation_time": "2019-03-10T12:25:48", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "9", "name": "大小"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_106", "frame": 0, "attr_value": [{"x": 1008.1086304040110, "y": 2310.9810661021103, "sn": 0}, {"x": 1954.7104365052448, "y": 2746.398098110021, "sn": 1}, {"x": 1910.5449510765304, "y": 2445.0805410448625, "sn": 2}, {"x": 1252.105089755466, "y": 2810.377280301210, "sn": 3}, {"x": 1032.0896907310029, "y": 2644.309756601057, "sn": 4}, {"x": 1910.7899741979581, "y": 2347.9950996566677, "sn": 5}, {"x": 1074.6735193866243, "y": 2368.1090872339525, "sn": 6}, {"x": 1010.7642882422383, "y": 2330.9866055602310, "sn": 7}, {"x": 1063.424742522375, "y": 2103.2905574024007, "sn": 8}, {"x": 1079.3948002272432, "y": 2306.6593310105353, "sn": 9}, {"x": 1060.8836104106310, "y": 2309.4955497525298, "sn": 10}, {"x": 1009.450707659803, "y": 2793.0938541109935, "sn": 11}, {"x": 1010.3606642107103, "y": 2324.9900502254864, "sn": 12}, {"x": 1075.4662093324687, "y": 2362.2361057810357, "sn": 10}, {"x": 1985.554796737626, "y": 2579.8398677310204, "sn": 10}, {"x": 1010.8834464104638, "y": 2744.2910056332446, "sn": 10}, {"x": 1090.466448319102, "y": 2103.555056672092, "sn": 10}, {"x": 1010.377046748683, "y": 2791.10351009305, "sn": 10}, {"x": 1049.9311068722125, "y": 2789.7272210879924, "sn": 10}, {"x": 1071.7033222025705, "y": 2010.0712247988863, "sn": 19}, {"x": 1010.687005503501, "y": 2782.9988066961046, "sn": 20}, {"x": 1057.8946972112867, "y": 2688.7921021040274, "sn": 21}, {"x": 1010.010975884579, "y": 2610.510959404265, "sn": 22}, {"x": 1248.9007530506442, "y": 2233.3105478990033, "sn": 23}, {"x": 1091.9385853896374, "y": 2710.103672249061, "sn": 24}, {"x": 1033.934324767039, "y": 2939.760121106209, "sn": 25}, {"x": 1052.4450864098933, "y": 2249.571265011067, "sn": 26}, {"x": 1029.8704970410798, "y": 2342.4666752826734, "sn": 27}, {"x": 1010.5948608768447, "y": 2301.3620991971084, "sn": 28}, {"x": 1946.1082447901026, "y": 2542.096552291082, "sn": 29}, {"x": 1085.4306649050059, "y": 2531.986898457658, "sn": 30}, {"x": 1028.610533100639, "y": 2850.859086848285, "sn": 31}, {"x": 1033.628741055401, "y": 2109.79242103092, "sn": 32}, {"x": 1093.4938640848964, "y": 2282.392465471035, "sn": 33}, {"x": 1047.5764932208651, "y": 2985.7894107110027, "sn": 34}, {"x": 1098.1055785224748, "y": 2757.3837499607106, "sn": 35}, {"x": 1261.7937210964810, "y": 2006.2558669757693, "sn": 36}, {"x": 1066.5550286581247, "y": 2194.804765481021, "sn": 37}, {"x": 1003.6104603465309, "y": 2610.79073204939, "sn": 38}, {"x": 1088.2608624835811, "y": 2541.7696582448607, "sn": 39}, {"x": 1010.2299659921064, "y": 2710.301029527546, "sn": 40}, {"x": 1907.2241010364686, "y": 2909.0311272650288, "sn": 41}, {"x": 1077.1051122968399, "y": 2221.500046868508, "sn": 42}, {"x": 1103.1207810545955, "y": 2840.7489471230356, "sn": 43}, {"x": 1073.450700519473, "y": 2950.105286237741, "sn": 44}, {"x": 1044.0987101099488, "y": 2437.551047810097, "sn": 45}, {"x": 1103.5108510048333, "y": 2278.3844810097105, "sn": 46}, {"x": 1020.4965598262839, "y": 2102.6769827967106, "sn": 47}, {"x": 1005.8100737710792, "y": 2844.1035043004344, "sn": 48}, {"x": 1295.643461961056, "y": 2764.6710810104404, "sn": 49}, {"x": 1065.9922949310110, "y": 2039.460441296706, "sn": 50}, {"x": 1008.6101082103495, "y": 2293.557631022224, "sn": 51}, {"x": 1191.8687437680226, "y": 2100.5039028296783, "sn": 52}, {"x": 1255.351010857333, "y": 2624.085223551905, "sn": 53}, {"x": 1007.2351034073484, "y": 2380.4337878345373, "sn": 54}, {"x": 1000.103422538221, "y": 2934.1054486838095, "sn": 55}, {"x": 1097.747576109275, "y": 2765.637611064821, "sn": 56}, {"x": 1106.4702720910394, "y": 2368.8966704328107, "sn": 57}, {"x": 1042.3126910101971, "y": 2903.2391048375103, "sn": 58}, {"x": 1037.8971046107482, "y": 2029.30010373806, "sn": 59}, {"x": 1076.3434109600873, "y": 2101.580510076610, "sn": 60}, {"x": 1032.0548582593626, "y": 2061.258384629042, "sn": 61}, {"x": 1110.3580103991268, "y": 2710.288510084735, "sn": 62}, {"x": 1109.0949556488108, "y": 2106.085224855981, "sn": 63}, {"x": 1060.0510688212685, "y": 2877.54466495107, "sn": 64}, {"x": 1000.929771256064, "y": 2606.9525682250537, "sn": 65}, {"x": 1050.7561038655323, "y": 2511.4802945021956, "sn": 66}, {"x": 1023.1073710305935, "y": 2273.588390107308, "sn": 67}, {"x": 1052.1281065622067, "y": 2109.400833930410, "sn": 68}, {"x": 1010.9109610976964, "y": 2560.920755103384, "sn": 69}, {"x": 1958.644270300109, "y": 2752.965630724010, "sn": 70}, {"x": 1225.8557321030778, "y": 2050.881009854385, "sn": 71}, {"x": 1122.4374123695302, "y": 2611.274343101064, "sn": 72}, {"x": 1075.1100492436201, "y": 2266.4080109600195, "sn": 73}, {"x": 1078.0109898052059, "y": 2078.81087341228, "sn": 74}, {"x": 1232.3703101071105, "y": 2480.9699603968606, "sn": 75}, {"x": 1010.9610841001037, "y": 2462.703100260408, "sn": 76}, {"x": 1051.3110358737648, "y": 2092.0102624602735, "sn": 77}, {"x": 1210.7539763584912, "y": 2105.1929310558274, "sn": 78}, {"x": 1058.9852110942106, "y": 2563.6267685901266, "sn": 79}, {"x": 1278.6912627610585, "y": 2622.6804489771084, "sn": 80}, {"x": 1974.3677806112787, "y": 2801.364573771019, "sn": 81}, {"x": 1069.5057850496332, "y": 2656.6996494955747, "sn": 82}, {"x": 1021.5699120121044, "y": 2566.410495928466, "sn": 83}, {"x": 1028.421242259571, "y": 2192.6681046249608, "sn": 84}, {"x": 1064.7371910109236, "y": 2969.5109675673910, "sn": 85}, {"x": 1105.4454745306407, "y": 2361.519066224103, "sn": 86}, {"x": 1077.391076271091, "y": 2676.527106835735, "sn": 87}, {"x": 1023.5683948607657, "y": 2904.7826470101094, "sn": 88}, {"x": 1196.9127545239235, "y": 2334.065723449722, "sn": 89}, {"x": 1109.3888645353002, "y": 2061.966679892028, "sn": 90}, {"x": 1076.0710566424028, "y": 2010.5057391054210, "sn": 91}, {"x": 1000.1110925500229, "y": 2030.740304451054, "sn": 92}, {"x": 1031.1053775581295, "y": 2610.9839856412373, "sn": 93}, {"x": 1053.347050221047, "y": 2094.003369100442, "sn": 94}, {"x": 1106.7611101006309, "y": 2583.2510237589876, "sn": 95}, {"x": 1085.877498376662, "y": 2108.588641020194, "sn": 96}, {"x": 1003.993520044659, "y": 2879.806059010410, "sn": 97}, {"x": 1108.372269108602, "y": 2780.5711077710083, "sn": 98}, {"x": 1080.4393528420028, "y": 2279.681942010946, "sn": 99}, {"x": 1985.5961081277363, "y": 2503.6943410988047, "sn": 100}, {"x": 1010.4443476925867, "y": 2099.3059793784107, "sn": 101}, {"x": 1037.4845751090063, "y": 2770.1055101194034, "sn": 102}, {"x": 1056.7657489239125, "y": 2541.904566337597, "sn": 103}, {"x": 1010.3980987502096, "y": 2825.7265564596523, "sn": 104}, {"x": 1103.7109908665108, "y": 2095.0571245098495, "sn": 105}], "z_order": 2, "group_id": 0}], "entity_name": "106点人脸", "instance_id": 3, "class_id": 106}, {"attributes": [{"occluded": "false", "attr_name": "植物框", "data_type": "bounding_box", "frame": 0, "attr_value": {"ymax": 1005, "ymin": 910, "xmax": 1081, "xmin": 945}, "z_order": 2, "group_id": 0}], "entity_name": "plant_body", "instance_id": 3, "class_id": 0}], "creation_time": "2019-04-10T04:41:44", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 190.87102660800557, "y": 328.6377339386101, "sn": 1}, {"x": 100.88758210666524, "y": 467.900103784107, "sn": 2}, {"x": 72.24801021072865, "y": 211.64411109473448, "sn": 3}, {"x": 126.86327702889845, "y": 239.5439895079851, "sn": 4}, {"x": 195.36724562669810, "y": 366.7986489298096, "sn": 5}, {"x": 101.4931028809081, "y": 449.3705793934452, "sn": 6}, {"x": 103.78611099110812, "y": 110.02724105734636, "sn": 7}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 4, "class_id": 3}, {"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "2", "name": "层次"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_106", "frame": 0, "attr_value": [{"x": 1001.7043989802103, "y": 2102.268602438965, "sn": 0}, {"x": 1045.2641095593394, "y": 2542.0095284976937, "sn": 1}, {"x": 1010.5592942198089, "y": 2109.0125010992106, "sn": 2}, {"x": 1049.9482910654701, "y": 2105.0985010254210, "sn": 3}, {"x": 1070.4512953883084, "y": 2038.5356851105236, "sn": 4}, {"x": 1954.028508623252, "y": 2271.8810780738237, "sn": 5}, {"x": 1100.8843274910345, "y": 2124.8702773723235, "sn": 6}, {"x": 1028.3652357345686, "y": 2839.402275121064, "sn": 7}, {"x": 1010.2237267610937, "y": 2337.3910357077704, "sn": 8}, {"x": 1022.1029243493102, "y": 2104.881097509556, "sn": 9}, {"x": 1284.5588280554673, "y": 2502.5697769655453, "sn": 10}, {"x": 1110.1084360092886, "y": 2692.859092636854, "sn": 11}, {"x": 1092.099728769998, "y": 2108.507370105712, "sn": 12}, {"x": 1993.0408221084106, "y": 2010.5945204969807, "sn": 10}, {"x": 1097.1036479096703, "y": 2786.7006510373737, "sn": 10}, {"x": 1205.0884710551062, "y": 2110.706811010709, "sn": 10}, {"x": 1064.5100522591010, "y": 2999.037481069252, "sn": 10}, {"x": 1210.8488592810308, "y": 2459.3279600950036, "sn": 10}, {"x": 1006.1000120959988, "y": 2881.525210346828, "sn": 10}, {"x": 1019.3106252610589, "y": 2310.333310010907, "sn": 19}, {"x": 1056.404998362435, "y": 2684.1083398384405, "sn": 20}, {"x": 1204.856877240609, "y": 2010.7331087660473, "sn": 21}, {"x": 1927.0033290609751, "y": 2510.3874486260843, "sn": 22}, {"x": 1055.2665081005452, "y": 2032.2097104835342, "sn": 23}, {"x": 1010.2290265291126, "y": 2909.81058805635, "sn": 24}, {"x": 1982.8099196410632, "y": 2747.821948092679, "sn": 25}, {"x": 1281.0532688053904, "y": 2481.1093108255607, "sn": 26}, {"x": 1038.6895651102863, "y": 2103.0724529658282, "sn": 27}, {"x": 1100.1044391094267, "y": 2480.9358674628825, "sn": 28}, {"x": 1020.5333683322710, "y": 2994.10836051058, "sn": 29}, {"x": 1010.3497606796466, "y": 2430.9494845332592, "sn": 30}, {"x": 1046.7027640910346, "y": 2810.7581092656883, "sn": 31}, {"x": 1102.9983574523858, "y": 2534.267862530666, "sn": 32}, {"x": 1104.1119109284664, "y": 2773.463801002255, "sn": 33}, {"x": 1086.2492579393002, "y": 2224.1029907474103, "sn": 34}, {"x": 1190.3471053031108, "y": 2451.7649607878757, "sn": 35}, {"x": 1081.3104553810404, "y": 2097.3729360343107, "sn": 36}, {"x": 1254.5010310789254, "y": 2780.7567038310774, "sn": 37}, {"x": 1068.4387266956662, "y": 2082.443845000339, "sn": 38}, {"x": 1029.5627057277577, "y": 2105.5750947279410, "sn": 39}, {"x": 1060.5000068541110, "y": 2569.9529370493565, "sn": 40}, {"x": 1092.6342546106688, "y": 2691.821967199032, "sn": 41}, {"x": 1021.7887744790696, "y": 2038.7263360894756, "sn": 42}, {"x": 1064.0284287600127, "y": 2010.4227065927907, "sn": 43}, {"x": 1281.9483105949475, "y": 2688.424270672689, "sn": 44}, {"x": 1086.3604301941010, "y": 2464.0826104377975, "sn": 45}, {"x": 1988.7500378853888, "y": 2109.9423432803756, "sn": 46}, {"x": 1284.2857045246101, "y": 2960.7026773365333, "sn": 47}, {"x": 1970.3057729206212, "y": 2993.345922104823, "sn": 48}, {"x": 1223.0428210454727, "y": 2951.555683339796, "sn": 49}, {"x": 1050.8529910058310, "y": 2966.4597990607235, "sn": 50}, {"x": 1257.232375963522, "y": 2942.710910840122, "sn": 51}, {"x": 1098.4367610507658, "y": 2937.7376653101096, "sn": 52}, {"x": 1247.4096322345247, "y": 2299.687412490106, "sn": 53}, {"x": 1049.3505529479603, "y": 2248.1043545882557, "sn": 54}, {"x": 1000.9670481075458, "y": 2112.1039750810195, "sn": 55}, {"x": 1020.8204865867237, "y": 2997.40947095639, "sn": 56}, {"x": 1276.0744920792206, "y": 2589.036957410386, "sn": 57}, {"x": 1250.459080665541, "y": 2485.047727331111, "sn": 58}, {"x": 1029.1039981010903, "y": 2311.2198078971933, "sn": 59}, {"x": 1094.1123345465621, "y": 2848.699368409294, "sn": 60}, {"x": 1065.5425606886265, "y": 2855.410799240102, "sn": 61}, {"x": 1104.650589272997, "y": 2428.666240280642, "sn": 62}, {"x": 1005.939769910877, "y": 2010.5072831941108, "sn": 63}, {"x": 1108.8050372462208, "y": 2683.2751071082056, "sn": 64}, {"x": 1099.5980103236104, "y": 2692.8821036850812, "sn": 65}, {"x": 1277.0591086485592, "y": 2728.7735112210103, "sn": 66}, {"x": 1079.3928606533108, "y": 2947.7541084051057, "sn": 67}, {"x": 1098.4123230687069, "y": 2912.7408656108023, "sn": 68}, {"x": 1049.7469637561035, "y": 2244.7349510783485, "sn": 69}, {"x": 1060.7108075252692, "y": 2910.7041052627075, "sn": 70}, {"x": 1067.3948494593965, "y": 2473.110210104664, "sn": 71}, {"x": 1041.446664569010, "y": 2486.6104646100104, "sn": 72}, {"x": 1102.022328962946, "y": 2604.9697396867286, "sn": 73}, {"x": 1071.1025028123274, "y": 2664.7859747672046, "sn": 74}, {"x": 1100.4708734420378, "y": 2990.9402239994424, "sn": 75}, {"x": 1055.3537905251035, "y": 2510.9195810103477, "sn": 76}, {"x": 1249.7561032931963, "y": 2981.3547596397807, "sn": 77}, {"x": 1193.402296622423, "y": 2295.4610083294886, "sn": 78}, {"x": 1045.1098905310775, "y": 2959.0930292101047, "sn": 79}, {"x": 1021.1088654104348, "y": 2994.6810083047327, "sn": 80}, {"x": 1973.7076321031902, "y": 2447.432239436103, "sn": 81}, {"x": 1095.0810107710124, "y": 2605.484065012739, "sn": 82}, {"x": 1203.331037768678, "y": 2670.798891054004, "sn": 83}, {"x": 1033.9511109674347, "y": 2108.197351024034, "sn": 84}, {"x": 1077.461081035805, "y": 2832.110095100548, "sn": 85}, {"x": 1981.7106966543588, "y": 2607.5109101963108, "sn": 86}, {"x": 1002.1041074879497, "y": 2968.712610552989, "sn": 87}, {"x": 1000.106273109243, "y": 2704.9497985705266, "sn": 88}, {"x": 1033.5487597234824, "y": 2073.284050212042, "sn": 89}, {"x": 1033.3101038356812, "y": 2120.441280310921, "sn": 90}, {"x": 1055.905256035959, "y": 2610.024043707951, "sn": 91}, {"x": 1084.868812228675, "y": 2851.852575769227, "sn": 92}, {"x": 1032.7611004324934, "y": 2029.283694626033, "sn": 93}, {"x": 1973.2253582847507, "y": 2010.4411070258103, "sn": 94}, {"x": 1085.1901240038467, "y": 2598.74334106899, "sn": 95}, {"x": 1111.8200361004108, "y": 2195.9821071970578, "sn": 96}, {"x": 1020.9012612310810, "y": 2426.2955477490236, "sn": 97}, {"x": 1267.6349936065208, "y": 2710.389473689967, "sn": 98}, {"x": 1006.2883945953681, "y": 2300.29888925041, "sn": 99}, {"x": 1039.089041943961, "y": 2327.824507227105, "sn": 100}, {"x": 1093.9710310855605, "y": 2505.738480385432, "sn": 101}, {"x": 1099.7109683108110, "y": 2009.4211250869794, "sn": 102}, {"x": 1051.0110126110983, "y": 2374.6963424010255, "sn": 103}, {"x": 1242.6210892063108, "y": 2209.0101001079246, "sn": 104}, {"x": 1019.210105947724, "y": 2537.2081910241103, "sn": 105}], "z_order": 2, "group_id": 0}], "entity_name": "106点人脸", "instance_id": 4, "class_id": 106}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 102.60526224859393, "y": 100.22975836008774, "sn": 1}, {"x": 73.38568763304255, "y": 241.2645446010627, "sn": 2}, {"x": 77.83521042225610, "y": 210.36646629900972, "sn": 3}, {"x": 62.86253448070671, "y": 104.7105068837099, "sn": 4}, {"x": 95.20043500304108, "y": 198.33098496798675, "sn": 5}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 4, "class_id": 3}, {"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "5", "name": "大小"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_68", "frame": 0, "attr_value": [{"x": 1010.110626710739, "y": 2500.8085583203497, "sn": 0}, {"x": 1104.810074791103, "y": 2974.536428933552, "sn": 1}, {"x": 1104.7100107755945, "y": 2259.8100081096065, "sn": 2}, {"x": 1082.9458937820855, "y": 2296.245488079123, "sn": 3}, {"x": 1053.1089872194122, "y": 2096.409844852701, "sn": 4}, {"x": 1081.5549671035910, "y": 2109.2194640112610, "sn": 5}, {"x": 1990.7349970810098, "y": 2610.3607281058486, "sn": 6}, {"x": 1081.102821020976, "y": 2399.482408582642, "sn": 7}, {"x": 1065.2279781048784, "y": 2877.810996451046, "sn": 8}, {"x": 1050.551006684359, "y": 2065.609564359848, "sn": 9}, {"x": 1081.8693576491096, "y": 2963.0972210059757, "sn": 10}, {"x": 1030.2770710305836, "y": 2942.0094893129335, "sn": 11}, {"x": 1006.0248423703338, "y": 2790.025610840710, "sn": 12}, {"x": 1019.3799126939109, "y": 2006.9060694324041, "sn": 10}, {"x": 1010.3764797935102, "y": 2326.688420659410, "sn": 10}, {"x": 1095.9372809948832, "y": 2803.654823633202, "sn": 10}, {"x": 1027.2403696594724, "y": 2936.6633102362003, "sn": 10}, {"x": 1059.4410925706032, "y": 2549.3606773397782, "sn": 10}, {"x": 1092.40566443327, "y": 2521.509861071252, "sn": 10}, {"x": 1031.5020249099737, "y": 2962.941082593609, "sn": 19}, {"x": 1007.4808654650965, "y": 2548.338645506255, "sn": 20}, {"x": 1002.4012855569748, "y": 2782.2689376125210, "sn": 21}, {"x": 1066.2021010357442, "y": 2598.78726755354, "sn": 22}, {"x": 1009.1095707719278, "y": 2427.0688002426805, "sn": 23}, {"x": 1076.9519304086562, "y": 2640.7212723892110, "sn": 24}, {"x": 1010.7732679474093, "y": 2103.035993074268, "sn": 25}, {"x": 1010.4967953252233, "y": 2820.565961099497, "sn": 26}, {"x": 1986.6799531208710, "y": 2523.035455212257, "sn": 27}, {"x": 1099.4510102434107, "y": 2391.7336287076705, "sn": 28}, {"x": 1034.8884276444355, "y": 2433.025897710629, "sn": 29}, {"x": 1054.86010104464, "y": 2683.9209487910077, "sn": 30}, {"x": 1937.4746983702364, "y": 2571.086710697695, "sn": 31}, {"x": 1104.9666979786584, "y": 2102.34695936711, "sn": 32}, {"x": 1049.2524473970523, "y": 2912.1010040466972, "sn": 33}, {"x": 1124.1019648801077, "y": 2809.3036931057755, "sn": 34}, {"x": 1035.226372941907, "y": 2831.126100034421, "sn": 35}, {"x": 1032.1030651082438, "y": 2029.3628700338127, "sn": 36}, {"x": 1024.39745905051, "y": 2456.832774684232, "sn": 37}, {"x": 1105.926308997267, "y": 2310.5379312510103, "sn": 38}, {"x": 1062.5527275710874, "y": 2096.410686307648, "sn": 39}, {"x": 1070.58323100027, "y": 2054.6796925919625, "sn": 40}, {"x": 1192.3857983950684, "y": 2100.6668703455310, "sn": 41}, {"x": 1100.1087091070227, "y": 2436.990345122801, "sn": 42}, {"x": 1096.9625809410599, "y": 2589.8665874462567, "sn": 43}, {"x": 1055.3763331237053, "y": 2098.6738081066753, "sn": 44}, {"x": 1048.0384210952102, "y": 2568.6434452844746, "sn": 45}, {"x": 1005.8678762857298, "y": 2388.8373007468876, "sn": 46}, {"x": 1036.2985880632796, "y": 2082.2410647291075, "sn": 47}, {"x": 1920.310810358299, "y": 2730.371008370510, "sn": 48}, {"x": 1011.3105611011004, "y": 2637.437229224063, "sn": 49}, {"x": 1003.805210773501, "y": 2998.08751101062, "sn": 50}, {"x": 1934.219458463955, "y": 2567.6291040080792, "sn": 51}, {"x": 1079.1065719544908, "y": 2586.108329637404, "sn": 52}, {"x": 1089.7410410477332, "y": 2533.594451010477, "sn": 53}, {"x": 1041.712370670104, "y": 2327.667685602110, "sn": 54}, {"x": 1057.870898743868, "y": 2792.5903624107045, "sn": 55}, {"x": 1020.776788110452, "y": 2410.8288865397810, "sn": 56}, {"x": 1243.6284337371978, "y": 2102.4876712747555, "sn": 57}, {"x": 1946.0033924107587, "y": 2651.4100854842307, "sn": 58}, {"x": 1929.023110109103, "y": 2883.6304194698396, "sn": 59}, {"x": 1921.4610010532382, "y": 2856.5799775568, "sn": 60}, {"x": 1201.2752708105071, "y": 2490.8605751966898, "sn": 61}, {"x": 1946.410612742433, "y": 2495.5537910201023, "sn": 62}, {"x": 1010.7405892883858, "y": 2109.020927792232, "sn": 63}, {"x": 1086.19489346128, "y": 2758.4973427449404, "sn": 64}, {"x": 1081.026778826125, "y": 2047.9103197753326, "sn": 65}, {"x": 1054.7109107304645, "y": 2586.563110389364, "sn": 66}, {"x": 1094.3642410351948, "y": 2052.8073340443357, "sn": 67}], "z_order": 2, "group_id": 0}], "entity_name": "68点人脸", "instance_id": 4, "class_id": 68}], "creation_time": "2019-02-28T10:08:50", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "2", "name": "层次"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_276", "frame": 0, "attr_value": [{"x": 1907.7979639107598, "y": 2200.199698312497, "sn": 0}, {"x": 1038.6893964741078, "y": 2721.4569223425324, "sn": 1}, {"x": 1280.5374933685106, "y": 2108.899936037702, "sn": 2}, {"x": 1035.0703995500928, "y": 2884.4107199910010, "sn": 3}, {"x": 1210.4636882589052, "y": 2304.310247126701, "sn": 4}, {"x": 1082.3102757327535, "y": 2370.7992824397784, "sn": 5}, {"x": 1010.1231900548669, "y": 2583.100302938385, "sn": 6}, {"x": 1001.3211099075095, "y": 2220.4049249110424, "sn": 7}, {"x": 1110.27060648893, "y": 2440.6747381038624, "sn": 8}, {"x": 1278.606442888031, "y": 2510.8671001020425, "sn": 9}, {"x": 1972.1257510586446, "y": 2701.5919324104465, "sn": 10}, {"x": 1088.553924910007, "y": 2101.923736210108, "sn": 11}, {"x": 1067.3239510977408, "y": 2243.781073602693, "sn": 12}, {"x": 1998.7735332584778, "y": 2964.4497870866608, "sn": 10}, {"x": 1076.4208831029894, "y": 2951.1262246871047, "sn": 10}, {"x": 1094.542384903571, "y": 2038.8887710234466, "sn": 10}, {"x": 1029.927510965076, "y": 2109.231271952449, "sn": 10}, {"x": 1062.2745049843105, "y": 2109.531011903034, "sn": 10}, {"x": 1023.28105799498, "y": 2077.4812109106733, "sn": 10}, {"x": 1096.872893835046, "y": 2610.6281106110107, "sn": 19}, {"x": 1926.8882105260824, "y": 2542.9210561060105, "sn": 20}, {"x": 1102.6767987557241, "y": 2090.9532510289426, "sn": 21}, {"x": 1057.023099682881, "y": 2361.896240741054, "sn": 22}, {"x": 1959.804449875510, "y": 2539.8543689871076, "sn": 23}, {"x": 1048.0424510897476, "y": 2107.108673807048, "sn": 24}, {"x": 1022.202489441125, "y": 2726.911053311043, "sn": 25}, {"x": 1010.3108205509233, "y": 2749.7761051044435, "sn": 26}, {"x": 1042.4487462445572, "y": 2029.3469625007683, "sn": 27}, {"x": 1060.078578390766, "y": 2650.1251036322063, "sn": 28}, {"x": 1030.4220734929802, "y": 2205.107505021051, "sn": 29}, {"x": 1063.9985383587486, "y": 2349.2610101081953, "sn": 30}, {"x": 1047.0781046405828, "y": 2086.9966394903695, "sn": 31}, {"x": 1088.9855745803445, "y": 2503.510736837376, "sn": 32}, {"x": 1082.785302089343, "y": 2629.006323364858, "sn": 33}, {"x": 1069.253277343396, "y": 2759.010695822499, "sn": 34}, {"x": 1057.9597564359822, "y": 2610.4440105110567, "sn": 35}, {"x": 1071.5211010531095, "y": 2008.6678392294106, "sn": 36}, {"x": 1268.5488366240106, "y": 2987.6625671010936, "sn": 37}, {"x": 1089.3675887223424, "y": 2791.6452801004510, "sn": 38}, {"x": 1033.7944651069737, "y": 2365.057710101094, "sn": 39}, {"x": 1044.010326523210, "y": 2571.3630991032464, "sn": 40}, {"x": 1005.5363494067983, "y": 2444.3477822537857, "sn": 41}, {"x": 1080.8893708327782, "y": 2469.420321010969, "sn": 42}, {"x": 1107.2559361289375, "y": 2269.1019044107867, "sn": 43}, {"x": 1023.3210846505352, "y": 2055.3929774406834, "sn": 44}, {"x": 1087.3010456276793, "y": 2346.104444674267, "sn": 45}, {"x": 1288.8337842452343, "y": 2370.829286104669, "sn": 46}, {"x": 1000.8051955495127, "y": 2694.2097901100996, "sn": 47}, {"x": 1046.7529308238409, "y": 2561.8108670823193, "sn": 48}, {"x": 1029.5208768489197, "y": 2877.233102768061, "sn": 49}, {"x": 1046.102757100002, "y": 2642.0252205370493, "sn": 50}, {"x": 1090.7100910887377, "y": 2610.5310052191065, "sn": 51}, {"x": 1947.386665331935, "y": 2459.7735687550335, "sn": 52}, {"x": 1085.6545066223193, "y": 2010.3537919691037, "sn": 53}, {"x": 1951.5638785553551, "y": 2528.544939007831, "sn": 54}, {"x": 1093.1246026291094, "y": 2642.8490891042, "sn": 55}, {"x": 1093.975639835319, "y": 2455.110281061066, "sn": 56}, {"x": 1052.8480698419799, "y": 2952.2025219951084, "sn": 57}, {"x": 1010.2433977810646, "y": 2484.2062498484242, "sn": 58}, {"x": 1030.656784802682, "y": 2010.9076991080194, "sn": 59}, {"x": 1010.005837306804, "y": 2688.5799469020712, "sn": 60}, {"x": 1081.46105929252, "y": 2100.5437610547842, "sn": 61}, {"x": 1002.3526322890598, "y": 2354.420499448829, "sn": 62}, {"x": 1022.1083398003261, "y": 2102.4921022810674, "sn": 63}, {"x": 1006.352430107462, "y": 2533.1027804458104, "sn": 64}, {"x": 1062.5685261046353, "y": 2890.3911954872556, "sn": 65}, {"x": 1062.5301086781034, "y": 2470.9032191069345, "sn": 66}, {"x": 1008.3799834877298, "y": 2052.541010919599, "sn": 67}, {"x": 1081.3457430778567, "y": 2066.312598565366, "sn": 68}, {"x": 1266.6989332591034, "y": 2083.4635806988263, "sn": 69}, {"x": 1077.861275784962, "y": 2773.7352457854863, "sn": 70}, {"x": 1098.6937670342986, "y": 2351.0301091091104, "sn": 71}, {"x": 1011.752403349748, "y": 2711.7231929104427, "sn": 72}, {"x": 1080.293444248864, "y": 2811.8034083324107, "sn": 73}, {"x": 1062.910622746295, "y": 2451.4546972701064, "sn": 74}, {"x": 1053.8483286448, "y": 2598.3105449010587, "sn": 75}, {"x": 1081.3609073759292, "y": 2929.4261057204276, "sn": 76}, {"x": 1209.8632491050226, "y": 2831.037592750426, "sn": 77}, {"x": 1088.1100381051093, "y": 2255.8805673310997, "sn": 78}, {"x": 1044.5751080525610, "y": 2105.656103839698, "sn": 79}, {"x": 1041.442592276688, "y": 2383.083548941103, "sn": 80}, {"x": 1010.562836386051, "y": 2605.970495486619, "sn": 81}, {"x": 1220.5474310112256, "y": 2535.774736750010, "sn": 82}, {"x": 1025.8092873210784, "y": 2746.2652889361243, "sn": 83}, {"x": 1081.7610641003945, "y": 2283.960293119465, "sn": 84}, {"x": 1085.3097712655695, "y": 2677.101103242982, "sn": 85}, {"x": 1062.1044522675099, "y": 2209.751043437632, "sn": 86}, {"x": 1082.6210437376748, "y": 2506.581010892467, "sn": 87}, {"x": 1026.5224609103436, "y": 2910.31062908494, "sn": 88}, {"x": 1042.961088010485, "y": 2703.8696646282224, "sn": 89}, {"x": 1093.010509110829, "y": 2567.8467858008735, "sn": 90}, {"x": 1073.657943568219, "y": 2890.581199092805, "sn": 91}, {"x": 1922.5965478095923, "y": 2776.935110883419, "sn": 92}, {"x": 1110.4844527461104, "y": 2790.042038269985, "sn": 93}, {"x": 1106.3508745804324, "y": 2325.5737357747585, "sn": 94}, {"x": 1026.4002261081098, "y": 2710.267844811242, "sn": 95}, {"x": 1951.410020226031, "y": 2608.4743529950074, "sn": 96}, {"x": 1000.5124559943633, "y": 2110.4710741007054, "sn": 97}, {"x": 1009.8981291227312, "y": 2288.21072425923, "sn": 98}, {"x": 1077.7433899426096, "y": 2549.683051190071, "sn": 99}, {"x": 1002.9822194326359, "y": 2268.257027826632, "sn": 100}, {"x": 1019.658998941028, "y": 2358.2667888712126, "sn": 101}, {"x": 1944.5835243365837, "y": 2810.4128487412327, "sn": 102}, {"x": 1037.4865929096109, "y": 2552.368672201098, "sn": 103}, {"x": 1091.984672810552, "y": 2110.108279772048, "sn": 104}, {"x": 1079.2728224107, "y": 2754.3806410889103, "sn": 105}, {"x": 1001.986084646103, "y": 2894.0808402554276, "sn": 106}, {"x": 1200.8101030310525, "y": 2868.9108104588075, "sn": 107}, {"x": 1042.4973105665954, "y": 2701.332699778825, "sn": 108}, {"x": 1266.619702640673, "y": 2910.4489982295263, "sn": 109}, {"x": 1046.2104010376312, "y": 2659.9326210863457, "sn": 110}, {"x": 1004.7257283981094, "y": 2951.3988384821064, "sn": 111}, {"x": 1074.9100557544898, "y": 2104.6256033104582, "sn": 112}, {"x": 1085.5295833493865, "y": 2741.5891092107725, "sn": 110}, {"x": 1008.3382210647949, "y": 2656.0360635102825, "sn": 110}, {"x": 1000.284483108622, "y": 2738.762661010397, "sn": 110}, {"x": 1010.3269392081197, "y": 2347.9843059104464, "sn": 110}, {"x": 1109.645233446108, "y": 2421.100543108582, "sn": 110}, {"x": 1023.9757544580684, "y": 2459.08811263102, "sn": 110}, {"x": 1042.2108106747493, "y": 2103.72997110578, "sn": 119}, {"x": 1051.09571010004, "y": 2910.3885605553864, "sn": 120}, {"x": 1032.10742350392, "y": 2124.6841980075055, "sn": 121}, {"x": 1073.010232109197, "y": 2925.0056762837007, "sn": 122}, {"x": 1988.9219910105336, "y": 2021.810792101293, "sn": 123}, {"x": 1047.6127857937056, "y": 2411.523778581075, "sn": 124}, {"x": 1099.4572481106233, "y": 2006.1045079102710, "sn": 125}, {"x": 1097.0705781026675, "y": 2977.283422304872, "sn": 126}, {"x": 1040.5998504685695, "y": 2747.9462664223106, "sn": 127}, {"x": 1071.1026378956125, "y": 2886.7319681264335, "sn": 128}, {"x": 1080.245103810871, "y": 2296.6308512638934, "sn": 129}, {"x": 1909.9508261998503, "y": 2626.969553060263, "sn": 100}, {"x": 1022.5742102410108, "y": 2695.4639881095874, "sn": 101}, {"x": 1036.9249452519032, "y": 2040.5528126548597, "sn": 102}, {"x": 1030.7060735241046, "y": 2988.9890829665887, "sn": 103}, {"x": 1060.836991080397, "y": 2758.8893563887796, "sn": 104}, {"x": 1982.0874552124942, "y": 2252.5632055220107, "sn": 105}, {"x": 1964.2244324835747, "y": 2991.9648341071035, "sn": 106}, {"x": 1937.6972893487832, "y": 2630.358223810674, "sn": 107}, {"x": 1037.706948410648, "y": 2404.9789520669747, "sn": 108}, {"x": 1089.9546905685968, "y": 2038.0102037660893, "sn": 109}, {"x": 1010.451097250431, "y": 2483.119101099663, "sn": 100}, {"x": 1007.0873835865539, "y": 2612.379108060232, "sn": 101}, {"x": 1033.2192255543102, "y": 2455.701050804106, "sn": 102}, {"x": 1950.6609589999225, "y": 2102.6334966956197, "sn": 103}, {"x": 1031.3870285366106, "y": 2746.2129362386736, "sn": 104}, {"x": 1030.5310934966628, "y": 2065.410626761253, "sn": 105}, {"x": 1027.998720226595, "y": 2369.1042092248410, "sn": 106}, {"x": 1019.1063324863275, "y": 2554.4479538769247, "sn": 107}, {"x": 1906.1252403442309, "y": 2063.396611088387, "sn": 108}, {"x": 1011.5684076388610, "y": 2752.711027641054, "sn": 109}, {"x": 1278.0231047572746, "y": 2512.02597104798, "sn": 100}, {"x": 1074.6396720673483, "y": 2627.3440107901024, "sn": 101}, {"x": 1049.53665587412, "y": 2727.4412892930036, "sn": 102}, {"x": 1107.1069746746327, "y": 2683.1026102104655, "sn": 103}, {"x": 1049.793270555405, "y": 2110.547467124904, "sn": 104}, {"x": 1996.8106190120019, "y": 2741.8760869710885, "sn": 105}, {"x": 1129.3937936239108, "y": 2052.6079994955912, "sn": 106}, {"x": 1058.7980104034277, "y": 2210.5710904322744, "sn": 107}, {"x": 1953.4262622247943, "y": 2776.1038801085436, "sn": 108}, {"x": 1071.572208319299, "y": 2772.755670277739, "sn": 109}, {"x": 1000.4555337666566, "y": 2343.297806580685, "sn": 100}, {"x": 1077.833466556104, "y": 2510.969059610103, "sn": 101}, {"x": 1095.1939000190537, "y": 2288.2206968223886, "sn": 102}, {"x": 1028.4298977269864, "y": 2276.872105684774, "sn": 103}, {"x": 1070.6728100221062, "y": 2299.609603971008, "sn": 104}, {"x": 1291.2529710093422, "y": 2259.8564800390286, "sn": 105}, {"x": 1281.094746380827, "y": 2378.232190446342, "sn": 106}, {"x": 1087.9237076542277, "y": 2560.949098634010, "sn": 107}, {"x": 1939.7631019732242, "y": 2942.6947532822102, "sn": 108}, {"x": 1044.3760237572683, "y": 2326.791043410821, "sn": 109}, {"x": 1038.3105099376872, "y": 2076.800819473306, "sn": 100}, {"x": 1067.0953831023826, "y": 2904.098336897542, "sn": 101}, {"x": 1031.6866674967655, "y": 2937.0989435612810, "sn": 102}, {"x": 1021.5535104421076, "y": 2730.0310396691935, "sn": 103}, {"x": 1094.5226768239054, "y": 2843.3278338756527, "sn": 104}, {"x": 1086.2249757480063, "y": 2799.4673500331064, "sn": 105}, {"x": 1103.8556069559725, "y": 2224.1092994732123, "sn": 106}, {"x": 1088.269608347555, "y": 2461.645672107501, "sn": 107}, {"x": 1019.454403458999, "y": 2788.448550488610, "sn": 108}, {"x": 1005.1027899342509, "y": 2631.360107240802, "sn": 109}, {"x": 1105.3508103641986, "y": 2941.6931090063893, "sn": 100}, {"x": 1020.3252556792331, "y": 2010.6890889499268, "sn": 101}, {"x": 1019.6287011093537, "y": 2327.364971099938, "sn": 102}, {"x": 1060.6332476107107, "y": 2241.0993790980547, "sn": 103}, {"x": 1022.286610805654, "y": 2370.8104637790774, "sn": 104}, {"x": 1044.0308102598966, "y": 2347.4664307254575, "sn": 105}, {"x": 1089.597525392233, "y": 2886.5863107849510, "sn": 106}, {"x": 1000.9124029748882, "y": 2110.8461054942104, "sn": 107}, {"x": 1033.0300090551075, "y": 2369.832970901092, "sn": 108}, {"x": 1044.6229410437485, "y": 2658.7358502925285, "sn": 109}, {"x": 1096.9801005264581, "y": 2104.450273101936, "sn": 190}, {"x": 1034.990240688667, "y": 2823.7852422108558, "sn": 191}, {"x": 1121.598438033488, "y": 2310.108505968054, "sn": 192}, {"x": 1284.1091010356445, "y": 2776.1052406427256, "sn": 193}, {"x": 1063.5405596645126, "y": 2308.6293122108310, "sn": 194}, {"x": 1058.957948984422, "y": 2109.467810336645, "sn": 195}, {"x": 1245.0336432744775, "y": 2582.770940065273, "sn": 196}, {"x": 1066.3877706909009, "y": 2501.9310609910474, "sn": 197}, {"x": 1004.0803910590735, "y": 2609.701005684198, "sn": 198}, {"x": 1011.581935599506, "y": 2604.035771911010, "sn": 199}, {"x": 1052.6612427360099, "y": 2327.3231088308427, "sn": 200}, {"x": 1294.2503452097624, "y": 2836.525533092427, "sn": 201}, {"x": 1040.1127219110321, "y": 2819.10464522858, "sn": 202}, {"x": 1293.2631039479663, "y": 2285.740967478585, "sn": 203}, {"x": 1010.247331003431, "y": 2064.7778920300843, "sn": 204}, {"x": 1287.861003093703, "y": 2820.404305022394, "sn": 205}, {"x": 1068.644624410726, "y": 2393.2403392982937, "sn": 206}, {"x": 1035.3257430257936, "y": 2649.623731008653, "sn": 207}, {"x": 1037.3572097510772, "y": 2771.6085701079055, "sn": 208}, {"x": 1984.9447199376466, "y": 2024.5789704530932, "sn": 209}, {"x": 1077.5843685599837, "y": 2762.7233100108097, "sn": 210}, {"x": 1293.12410052077, "y": 2748.4632572581286, "sn": 211}, {"x": 1107.6965109254286, "y": 2056.393052910577, "sn": 212}, {"x": 1110.6230762497108, "y": 2959.5967596510327, "sn": 210}, {"x": 1020.197367856274, "y": 2728.4853851093238, "sn": 210}, {"x": 1267.8510604810010, "y": 2870.257057720991, "sn": 210}, {"x": 1083.790435302446, "y": 2971.0411097907104, "sn": 210}, {"x": 1195.4910486563312, "y": 2527.7254100584656, "sn": 210}, {"x": 1029.2859495101044, "y": 2730.3110452733285, "sn": 210}, {"x": 1085.538108371242, "y": 2025.540685111048, "sn": 219}, {"x": 1101.4110667049010, "y": 2932.8105423631032, "sn": 220}, {"x": 1078.7324454100125, "y": 2961.1069973253, "sn": 221}, {"x": 1041.3410310377876, "y": 2807.933380744269, "sn": 222}, {"x": 1055.6378231073663, "y": 2866.702998067110, "sn": 223}, {"x": 1012.6334761010926, "y": 2710.751230942108, "sn": 224}, {"x": 1082.9739412740223, "y": 2940.0532472880386, "sn": 225}, {"x": 1028.0269762624453, "y": 2872.900489584483, "sn": 226}, {"x": 1071.826392890466, "y": 2001.9072049289662, "sn": 227}, {"x": 1094.7878775592494, "y": 2941.3236360839064, "sn": 228}, {"x": 1059.5841957610872, "y": 2723.490497272579, "sn": 229}, {"x": 1080.9390708601023, "y": 2203.550764760101, "sn": 230}, {"x": 1109.0749390649485, "y": 2123.0103758442723, "sn": 231}, {"x": 1012.8806073855626, "y": 2730.254680306669, "sn": 232}, {"x": 1120.594100361023, "y": 2609.2110035107103, "sn": 233}, {"x": 1066.4307967527989, "y": 2372.9485697935110, "sn": 234}, {"x": 1008.489797128273, "y": 2371.206349104438, "sn": 235}, {"x": 1019.5301250824525, "y": 2109.3058433448264, "sn": 236}, {"x": 1005.2542542119695, "y": 2259.845863910757, "sn": 237}, {"x": 1097.477536574808, "y": 2619.7647224109946, "sn": 238}, {"x": 1034.6945590037308, "y": 2806.410732795543, "sn": 239}, {"x": 1075.0456432458805, "y": 2800.110935970075, "sn": 240}, {"x": 1194.9020604281097, "y": 2496.883883751081, "sn": 241}, {"x": 1232.3821999102753, "y": 2410.719879494682, "sn": 242}, {"x": 1025.6637073498405, "y": 2579.810331255954, "sn": 243}, {"x": 1091.4827438757923, "y": 2684.710285546676, "sn": 244}, {"x": 1093.5366834034987, "y": 2471.368366886034, "sn": 245}, {"x": 1282.400284641105, "y": 2304.7306630469107, "sn": 246}, {"x": 1984.6198878775667, "y": 2998.507902253524, "sn": 247}, {"x": 1007.4871048104778, "y": 2948.4774202532553, "sn": 248}, {"x": 1086.0532953308696, "y": 2375.9831069533483, "sn": 249}, {"x": 1111.5122711087407, "y": 2487.3884958535837, "sn": 250}, {"x": 1051.32581052923, "y": 2087.8010982199534, "sn": 251}, {"x": 1256.9125810858205, "y": 2004.4246352105364, "sn": 252}, {"x": 1087.8776107936908, "y": 2068.467129525591, "sn": 253}, {"x": 1268.5694268499644, "y": 2083.5599735029805, "sn": 254}, {"x": 1054.5760649055705, "y": 2886.580745222277, "sn": 255}, {"x": 1099.5619197898232, "y": 2432.6042973100856, "sn": 256}, {"x": 1061.799243459503, "y": 2404.101032502386, "sn": 257}, {"x": 1007.220107339942, "y": 2197.944538892977, "sn": 258}, {"x": 1259.6103239471962, "y": 2047.8335769827245, "sn": 259}, {"x": 1059.1059568565487, "y": 2729.954419858234, "sn": 260}, {"x": 1261.703629635237, "y": 2423.898438577326, "sn": 261}, {"x": 1099.8586910442303, "y": 2683.3095329559974, "sn": 262}, {"x": 1033.4934322469499, "y": 2232.870399824019, "sn": 263}, {"x": 1031.0953920510376, "y": 2480.8804300024353, "sn": 264}, {"x": 1026.2103062743109, "y": 2967.0696736762466, "sn": 265}, {"x": 1092.6242564353056, "y": 2108.504791005652, "sn": 266}, {"x": 1098.4857801079602, "y": 2472.447547027568, "sn": 267}, {"x": 1199.5284442083662, "y": 2810.247102439027, "sn": 268}, {"x": 1105.6282100821085, "y": 2256.5374102224765, "sn": 269}, {"x": 1064.9010391029441, "y": 2220.5102065327306, "sn": 270}, {"x": 1048.561936745567, "y": 2879.06026574846, "sn": 271}, {"x": 1012.2410860862468, "y": 2741.493565710464, "sn": 272}, {"x": 1224.6786686209010, "y": 2105.641039746432, "sn": 273}, {"x": 1086.2768500672412, "y": 2728.467203421006, "sn": 274}, {"x": 1259.6920346025468, "y": 2842.9821192250643, "sn": 275}], "z_order": 2, "group_id": 0}], "entity_name": "276点人脸", "instance_id": 5, "class_id": 276}, {"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "4", "name": "大小"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_276", "frame": 0, "attr_value": [{"x": 1094.3670730668202, "y": 2426.510122085411, "sn": 0}, {"x": 1087.9501020280432, "y": 2767.3874485772963, "sn": 1}, {"x": 1286.562102310828, "y": 2894.288819704556, "sn": 2}, {"x": 1007.9596798910106, "y": 2389.1053250333004, "sn": 3}, {"x": 1097.6903329902907, "y": 2032.7937210731108, "sn": 4}, {"x": 1103.8599750410354, "y": 2506.9565066831033, "sn": 5}, {"x": 1094.2110579356091, "y": 2033.90402442410, "sn": 6}, {"x": 1058.692221044654, "y": 2704.579096275255, "sn": 7}, {"x": 1276.7319860290497, "y": 2985.7989329548195, "sn": 8}, {"x": 1030.6032809391966, "y": 2556.345038419721, "sn": 9}, {"x": 1098.4299108731073, "y": 2628.955110776682, "sn": 10}, {"x": 1910.7931102839121, "y": 2619.41974567623, "sn": 11}, {"x": 1010.7512533309833, "y": 2485.0931238910937, "sn": 12}, {"x": 1056.6931986945442, "y": 2324.9499579380285, "sn": 10}, {"x": 1028.100735097236, "y": 2082.9312024396995, "sn": 10}, {"x": 1005.1009226251024, "y": 2924.0881107365823, "sn": 10}, {"x": 1064.3703851070510, "y": 2811.007783610076, "sn": 10}, {"x": 1035.5247343459032, "y": 2456.1019490987882, "sn": 10}, {"x": 1044.7385389409483, "y": 2107.2044064458108, "sn": 10}, {"x": 1298.607642887975, "y": 2561.373104510431, "sn": 19}, {"x": 1092.0926610548237, "y": 2210.8936419838105, "sn": 20}, {"x": 1022.1242635447943, "y": 2856.948202986910, "sn": 21}, {"x": 1002.0083402897966, "y": 2378.7798887102758, "sn": 22}, {"x": 1010.8071098948752, "y": 2423.9826025537823, "sn": 23}, {"x": 1070.3105106478956, "y": 2101.7463343648037, "sn": 24}, {"x": 1048.9809473323510, "y": 2403.4787097199983, "sn": 25}, {"x": 1105.8301041924929, "y": 2251.5001095564510, "sn": 26}, {"x": 1097.6688223502952, "y": 2850.7824120349310, "sn": 27}, {"x": 1064.6900064494273, "y": 2460.542489461028, "sn": 28}, {"x": 1100.3427192253484, "y": 2875.7410902751107, "sn": 29}, {"x": 1938.5497545610666, "y": 2067.620103740710, "sn": 30}, {"x": 1049.2649495458103, "y": 2085.9361097102733, "sn": 31}, {"x": 1108.384510495095, "y": 2605.108511026653, "sn": 32}, {"x": 1066.0490772982510, "y": 2348.1022250047494, "sn": 33}, {"x": 1080.3365106206747, "y": 2101.972490753649, "sn": 34}, {"x": 1000.679470109839, "y": 2307.552446373207, "sn": 35}, {"x": 1090.1104396076242, "y": 2554.286944203104, "sn": 36}, {"x": 1068.1038628109742, "y": 2959.3277948199693, "sn": 37}, {"x": 1032.7446641036486, "y": 2571.0731078706067, "sn": 38}, {"x": 1029.4698322381264, "y": 2273.941010985324, "sn": 39}, {"x": 1010.8201900410697, "y": 2859.054580111981, "sn": 40}, {"x": 1243.3353725534210, "y": 2010.690724262037, "sn": 41}, {"x": 1032.107531103024, "y": 2948.8593758241996, "sn": 42}, {"x": 1080.5061010747101, "y": 2127.5001079591104, "sn": 43}, {"x": 1072.6735636475532, "y": 2827.042460822592, "sn": 44}, {"x": 1036.6545932025563, "y": 2410.678707370797, "sn": 45}, {"x": 1104.8471228501067, "y": 2937.4472431022047, "sn": 46}, {"x": 1007.751047577446, "y": 2968.732727412028, "sn": 47}, {"x": 1073.9888721067672, "y": 2481.736876687104, "sn": 48}, {"x": 1055.600103790426, "y": 2910.0923902086697, "sn": 49}, {"x": 1105.8610106419208, "y": 2112.106103209754, "sn": 50}, {"x": 1065.6659802046383, "y": 2725.5774594271025, "sn": 51}, {"x": 1244.202434570610, "y": 2478.109224768051, "sn": 52}, {"x": 1012.0639546648831, "y": 2783.2397021042428, "sn": 53}, {"x": 1237.3009451079123, "y": 2925.6932042053105, "sn": 54}, {"x": 1094.3370585959037, "y": 2383.810705892671, "sn": 55}, {"x": 1010.3510255310666, "y": 2797.8810896411032, "sn": 56}, {"x": 1056.2610026301097, "y": 2972.0036953557965, "sn": 57}, {"x": 1263.9200253461104, "y": 2624.3555740822576, "sn": 58}, {"x": 1010.4057710840844, "y": 2357.241088084052, "sn": 59}, {"x": 1071.5393108205798, "y": 2410.103506810108, "sn": 60}, {"x": 1083.5561059331071, "y": 2709.9761910766644, "sn": 61}, {"x": 1089.0442370845656, "y": 2997.045671003601, "sn": 62}, {"x": 1021.7700591112402, "y": 2659.1090706422526, "sn": 63}, {"x": 1096.5106573057365, "y": 2806.4936775709634, "sn": 64}, {"x": 1020.5530081093244, "y": 2971.1097745338047, "sn": 65}, {"x": 1107.6057364822577, "y": 2440.425345795025, "sn": 66}, {"x": 1210.6955101022282, "y": 2898.2366037482107, "sn": 67}, {"x": 1190.2955410112268, "y": 2819.102335321077, "sn": 68}, {"x": 1041.1230577680965, "y": 2994.207088050423, "sn": 69}, {"x": 1000.3596463403758, "y": 2095.4262385956445, "sn": 70}, {"x": 1003.8326884280361, "y": 2932.2102761023776, "sn": 71}, {"x": 1096.1001011025966, "y": 2533.110102074256, "sn": 72}, {"x": 1905.1033693572951, "y": 2354.5027312687785, "sn": 73}, {"x": 1026.6479010548456, "y": 2434.8105909269476, "sn": 74}, {"x": 1040.6853991091067, "y": 2488.8993951025103, "sn": 75}, {"x": 1033.3560899728275, "y": 2731.3521960800344, "sn": 76}, {"x": 1062.2844792109008, "y": 2278.577793750879, "sn": 77}, {"x": 1258.2127853310988, "y": 2507.6010710325106, "sn": 78}, {"x": 1010.0378767639610, "y": 2007.0262902273257, "sn": 79}, {"x": 1081.5358248672499, "y": 2666.269649211022, "sn": 80}, {"x": 1046.105102938389, "y": 2549.3843823246443, "sn": 81}, {"x": 1023.0579381009442, "y": 2108.236509271010, "sn": 82}, {"x": 1023.0427910220105, "y": 2796.452490805106, "sn": 83}, {"x": 1077.3612781008364, "y": 2792.03101036648, "sn": 84}, {"x": 1050.3742105229038, "y": 2279.1057558962424, "sn": 85}, {"x": 1129.4907670305308, "y": 2663.590364510351, "sn": 86}, {"x": 1027.2060405352424, "y": 2989.630923080299, "sn": 87}, {"x": 1070.2898583696801, "y": 2412.810108195394, "sn": 88}, {"x": 1279.58210444309, "y": 2848.9843510422635, "sn": 89}, {"x": 1029.9843084108929, "y": 2231.4710103439036, "sn": 90}, {"x": 1043.5524754765534, "y": 2266.319446910744, "sn": 91}, {"x": 1264.0661027284256, "y": 2103.3680375975446, "sn": 92}, {"x": 1065.3843485056295, "y": 2844.9238710338010, "sn": 93}, {"x": 1048.4719957271102, "y": 2088.1091035127377, "sn": 94}, {"x": 1073.7891060864010, "y": 2757.3497741263027, "sn": 95}, {"x": 1085.6239866353003, "y": 2904.4711010075603, "sn": 96}, {"x": 1038.5041065964906, "y": 2244.5712552291107, "sn": 97}, {"x": 1049.2653762426910, "y": 2036.91078821981, "sn": 98}, {"x": 1010.4993456193556, "y": 2410.2434100310787, "sn": 99}, {"x": 1032.8222361094202, "y": 2972.2122222443210, "sn": 100}, {"x": 1012.0994065057992, "y": 2809.054239031034, "sn": 101}, {"x": 1033.5210884807810, "y": 2992.027105207193, "sn": 102}, {"x": 1984.6769102355448, "y": 2270.332827021210, "sn": 103}, {"x": 1924.9881042953626, "y": 2101.063360442063, "sn": 104}, {"x": 1076.0566498273727, "y": 2411.665093194475, "sn": 105}, {"x": 1100.5952109360108, "y": 2105.7300090757635, "sn": 106}, {"x": 1011.3410241087881, "y": 2338.02778510872, "sn": 107}, {"x": 1072.7669710500053, "y": 2642.6777246606043, "sn": 108}, {"x": 1967.4657583705402, "y": 2103.8107096670976, "sn": 109}, {"x": 1078.601098566348, "y": 2006.6463637725637, "sn": 110}, {"x": 1059.9024496633638, "y": 2755.755410001068, "sn": 111}, {"x": 1997.6298664807646, "y": 2780.406270109785, "sn": 112}, {"x": 1073.5210321093756, "y": 2108.8110606284923, "sn": 110}, {"x": 1292.253677410305, "y": 2682.1087826392567, "sn": 110}, {"x": 1257.551008476581, "y": 2094.22222702576, "sn": 110}, {"x": 1047.6842498385727, "y": 2620.1094475400210, "sn": 110}, {"x": 1099.1094234922509, "y": 2032.111052788451, "sn": 110}, {"x": 1010.5025757468236, "y": 2901.2201004907047, "sn": 110}, {"x": 1080.110104522292, "y": 2366.638263299076, "sn": 119}, {"x": 1922.6882083610998, "y": 2353.0687098085336, "sn": 120}, {"x": 1010.4743698030958, "y": 2843.7421010733057, "sn": 121}, {"x": 1036.3591127939626, "y": 2719.4379107438896, "sn": 122}, {"x": 1010.2828278489694, "y": 2887.0001910423736, "sn": 123}, {"x": 1055.856730266371, "y": 2628.889049210976, "sn": 124}, {"x": 1010.7229363383967, "y": 2310.778634101010, "sn": 125}, {"x": 1106.3486310787594, "y": 2586.0570310611233, "sn": 126}, {"x": 1023.4292822510109, "y": 2908.010641097736, "sn": 127}, {"x": 1045.7445531040645, "y": 2348.4285266272736, "sn": 128}, {"x": 1902.2955871085267, "y": 2581.8930010641066, "sn": 129}, {"x": 1905.630825224673, "y": 2891.0631243607086, "sn": 100}, {"x": 1254.7099311010708, "y": 2437.449624244292, "sn": 101}, {"x": 1041.224971009073, "y": 2003.3358010197978, "sn": 102}, {"x": 1006.2836551001910, "y": 2820.8780921247, "sn": 103}, {"x": 1088.6226869108104, "y": 2087.03922537399, "sn": 104}, {"x": 1033.4327295906101, "y": 2499.307412207307, "sn": 105}, {"x": 1957.4507841088667, "y": 2274.407788129265, "sn": 106}, {"x": 1104.8671268572762, "y": 2674.4031951027474, "sn": 107}, {"x": 1011.986640357102, "y": 2836.2683708004107, "sn": 108}, {"x": 1089.6688261981099, "y": 2819.945104628231, "sn": 109}, {"x": 1050.3589010042544, "y": 2291.2110499407693, "sn": 100}, {"x": 1038.8364202774073, "y": 2455.424958570244, "sn": 101}, {"x": 1071.300204563610, "y": 2541.3528530065887, "sn": 102}, {"x": 1002.210499193768, "y": 2980.8412801042624, "sn": 103}, {"x": 1088.03682494458, "y": 2851.9494631958196, "sn": 104}, {"x": 1281.6391038898685, "y": 2987.510864785212, "sn": 105}, {"x": 1963.906756106610, "y": 2685.1010102379253, "sn": 106}, {"x": 1066.7253478529706, "y": 2333.220645596991, "sn": 107}, {"x": 1071.0196643774207, "y": 2603.870042622927, "sn": 108}, {"x": 1032.954107243711, "y": 2106.2102370051106, "sn": 109}, {"x": 1043.9962310675110, "y": 2812.8908510755375, "sn": 100}, {"x": 1946.2704610988991, "y": 2424.4806034042244, "sn": 101}, {"x": 1026.4999431954748, "y": 2129.654545074873, "sn": 102}, {"x": 1102.8839910078734, "y": 2197.0101100827428, "sn": 103}, {"x": 1012.8666035007108, "y": 2608.675950774487, "sn": 104}, {"x": 1090.8481087595702, "y": 2079.110323741012, "sn": 105}, {"x": 1007.904805984202, "y": 2744.560465109679, "sn": 106}, {"x": 1021.2636242395001, "y": 2610.987462471044, "sn": 107}, {"x": 1106.2485877519352, "y": 2336.4338026648265, "sn": 108}, {"x": 1077.1910803603288, "y": 2341.3533496547598, "sn": 109}, {"x": 1958.8347825850485, "y": 2496.4007591265326, "sn": 100}, {"x": 1003.9204775520302, "y": 2376.5097104956845, "sn": 101}, {"x": 1968.3376947386341, "y": 2774.6958265722683, "sn": 102}, {"x": 1029.0737369210276, "y": 2210.64078890763, "sn": 103}, {"x": 1045.7824394882387, "y": 2910.351006754527, "sn": 104}, {"x": 1095.9890920638045, "y": 2072.102065690105, "sn": 105}, {"x": 1063.0956377874331, "y": 2486.8395349467637, "sn": 106}, {"x": 1038.9942268109036, "y": 2036.5031194273852, "sn": 107}, {"x": 1028.9336344588935, "y": 2362.8524412286542, "sn": 108}, {"x": 1033.563727260769, "y": 2020.3052225410944, "sn": 109}, {"x": 1026.021949225109, "y": 2565.99432776677, "sn": 100}, {"x": 1043.5510397448064, "y": 2436.0245033481106, "sn": 101}, {"x": 1088.8340523938782, "y": 2980.7798362089557, "sn": 102}, {"x": 1230.1073198042343, "y": 2010.9403335001005, "sn": 103}, {"x": 1097.3587708802656, "y": 2347.6625382260554, "sn": 104}, {"x": 1011.836639456441, "y": 2300.910445765012, "sn": 105}, {"x": 1909.4434791086725, "y": 2010.225644106762, "sn": 106}, {"x": 1006.4094088932081, "y": 2062.3597349847505, "sn": 107}, {"x": 1232.302444100538, "y": 2865.4410005470427, "sn": 108}, {"x": 1035.4403262607823, "y": 2710.759779592058, "sn": 109}, {"x": 1089.7452565271088, "y": 2646.7726966329510, "sn": 100}, {"x": 1074.954296610892, "y": 2827.9372710937494, "sn": 101}, {"x": 1098.2600519494408, "y": 2869.832862950941, "sn": 102}, {"x": 1094.1023421109575, "y": 2810.000485844641, "sn": 103}, {"x": 1067.2340322337810, "y": 2621.5739420307527, "sn": 104}, {"x": 1082.7109752860573, "y": 2081.8609787767446, "sn": 105}, {"x": 1042.1010027098988, "y": 2951.4127808107623, "sn": 106}, {"x": 1061.1095862408105, "y": 2830.8295110431997, "sn": 107}, {"x": 1006.194387539735, "y": 2210.981023229357, "sn": 108}, {"x": 1942.1102101060873, "y": 2058.3361128120728, "sn": 109}, {"x": 1041.255942819867, "y": 2873.4205451047224, "sn": 190}, {"x": 1008.6458375663824, "y": 2089.3810944258195, "sn": 191}, {"x": 1096.895323405409, "y": 2711.604100506045, "sn": 192}, {"x": 1024.7477622036463, "y": 2459.6963775910366, "sn": 193}, {"x": 1020.2070471081065, "y": 2678.678365776247, "sn": 194}, {"x": 1057.941067790337, "y": 2434.8904912772646, "sn": 195}, {"x": 1079.9194993080746, "y": 2197.2893108401257, "sn": 196}, {"x": 1093.3950538433219, "y": 2675.4104747758952, "sn": 197}, {"x": 1010.7247688478103, "y": 2785.995895721064, "sn": 198}, {"x": 1063.9970630354096, "y": 2849.5005110340573, "sn": 199}, {"x": 1092.494866485327, "y": 2411.122798210694, "sn": 200}, {"x": 1089.1010737822987, "y": 2401.7739867388445, "sn": 201}, {"x": 1074.378273573769, "y": 2301.8336923387924, "sn": 202}, {"x": 1028.9210310723252, "y": 2892.983806311025, "sn": 203}, {"x": 1035.70500932355, "y": 2208.350039210759, "sn": 204}, {"x": 1010.0805859937910, "y": 2024.239845623441, "sn": 205}, {"x": 1032.5880110858354, "y": 2949.979959351096, "sn": 206}, {"x": 1044.2995932820602, "y": 2111.5631010257037, "sn": 207}, {"x": 1084.852524540225, "y": 2778.2254483129773, "sn": 208}, {"x": 1077.7689823310789, "y": 2442.7910042361097, "sn": 209}, {"x": 1086.2107502456496, "y": 2469.6758294771053, "sn": 210}, {"x": 1033.290600210455, "y": 2851.4724301270007, "sn": 211}, {"x": 1082.1089610471084, "y": 2381.0680620805210, "sn": 212}, {"x": 1020.9044428238105, "y": 2106.9840119046753, "sn": 210}, {"x": 1072.1060510898103, "y": 2521.753643321065, "sn": 210}, {"x": 1043.4754972710625, "y": 2574.992881204126, "sn": 210}, {"x": 1121.5589101093457, "y": 2849.1081023662323, "sn": 210}, {"x": 1066.3996043570867, "y": 2278.5402412089110, "sn": 210}, {"x": 1044.1977396643545, "y": 2812.3341040538510, "sn": 210}, {"x": 1006.9958437096104, "y": 2947.529422212287, "sn": 219}, {"x": 1046.8733785296595, "y": 2650.7786236550774, "sn": 220}, {"x": 1980.4351931021077, "y": 2571.448984244629, "sn": 221}, {"x": 1079.4190259861082, "y": 2361.5779929304927, "sn": 222}, {"x": 1066.0987902776210, "y": 2110.6223329097584, "sn": 223}, {"x": 1037.1997201103654, "y": 2349.2587510596544, "sn": 224}, {"x": 1010.7026846710010, "y": 2726.8734264106096, "sn": 225}, {"x": 1107.2027079472978, "y": 2992.1051034687894, "sn": 226}, {"x": 1063.0403484712206, "y": 2661.2473710520685, "sn": 227}, {"x": 1085.1066828480806, "y": 2951.949612622810, "sn": 228}, {"x": 1221.9621987102582, "y": 2067.644936564105, "sn": 229}, {"x": 1125.5931024542010, "y": 2644.3512481032528, "sn": 230}, {"x": 1029.378610808504, "y": 2620.588666332103, "sn": 231}, {"x": 1938.94371060938, "y": 2683.338106944202, "sn": 232}, {"x": 1110.1061101027354, "y": 2325.4604561198553, "sn": 233}, {"x": 1222.734028979902, "y": 2862.2875855023394, "sn": 234}, {"x": 1095.5110500038957, "y": 2688.5769045288453, "sn": 235}, {"x": 1039.0890804010842, "y": 2912.0936936126127, "sn": 236}, {"x": 1063.1080992372624, "y": 2893.659659057067, "sn": 237}, {"x": 1912.2751010539963, "y": 2659.79568331928, "sn": 238}, {"x": 1012.911007358283, "y": 2536.689268338101, "sn": 239}, {"x": 1050.301069610404, "y": 2579.8791230103993, "sn": 240}, {"x": 1241.8212474397233, "y": 2724.5544451040407, "sn": 241}, {"x": 1047.1076600494504, "y": 2584.6938359755554, "sn": 242}, {"x": 1034.3908953083492, "y": 2565.1197346508093, "sn": 243}, {"x": 1051.7978649609597, "y": 2298.0110804009205, "sn": 244}, {"x": 1058.2880664978852, "y": 2433.8545931281237, "sn": 245}, {"x": 1098.362080096557, "y": 2710.987122105864, "sn": 246}, {"x": 1011.9338648795583, "y": 2888.508459224287, "sn": 247}, {"x": 1107.604210544623, "y": 2292.545498536375, "sn": 248}, {"x": 1072.109301060332, "y": 2735.243045449838, "sn": 249}, {"x": 1072.5375404342794, "y": 2825.3633964703810, "sn": 250}, {"x": 1011.2011072470193, "y": 2786.5699291022710, "sn": 251}, {"x": 1110.7380580107703, "y": 2033.1005241060437, "sn": 252}, {"x": 1068.2043819771011, "y": 2230.022406306508, "sn": 253}, {"x": 1059.035643920649, "y": 2210.5119387665936, "sn": 254}, {"x": 1096.0564310530946, "y": 2435.1966679229310, "sn": 255}, {"x": 1210.3243885062823, "y": 2101.792973220473, "sn": 256}, {"x": 1093.6755566880388, "y": 2757.019437282601, "sn": 257}, {"x": 1051.035963101006, "y": 2885.38542593879, "sn": 258}, {"x": 1280.649224272353, "y": 2749.257034794675, "sn": 259}, {"x": 1007.104305210594, "y": 2383.085862891932, "sn": 260}, {"x": 1084.3541080044283, "y": 2100.62097278661, "sn": 261}, {"x": 1079.1012126104535, "y": 2668.904076384907, "sn": 262}, {"x": 1011.8774675862676, "y": 2105.038868264219, "sn": 263}, {"x": 1044.4791067889279, "y": 2310.2552257507746, "sn": 264}, {"x": 1081.0734798010204, "y": 2947.389251082438, "sn": 265}, {"x": 1032.296466528332, "y": 2285.2656210801094, "sn": 266}, {"x": 1026.8812746510105, "y": 2210.106553548222, "sn": 267}, {"x": 1073.887599753349, "y": 2935.828106627484, "sn": 268}, {"x": 1235.8108763559497, "y": 2010.3109493523210, "sn": 269}, {"x": 1005.8967463976683, "y": 2684.101095104208, "sn": 270}, {"x": 1107.0951044794106, "y": 2551.1101011221045, "sn": 271}, {"x": 1105.1072193107281, "y": 2070.437657612927, "sn": 272}, {"x": 1029.79210408471, "y": 2845.7301001053585, "sn": 273}, {"x": 1103.6108986957897, "y": 2551.9925364471104, "sn": 274}, {"x": 1052.4483108625102, "y": 2964.032639402268, "sn": 275}], "z_order": 2, "group_id": 0}], "entity_name": "276点人脸", "instance_id": 5, "class_id": 276}, {"attributes": [{"occluded": "false", "attr_name": "polygon_example", "data_type": "polygon", "frame": 0, "attr_value": [{"x": 72.22287578510272, "y": 469.7024259837784, "sn": 1}, {"x": 101.44020500967548, "y": 191.23710102909444, "sn": 2}, {"x": 195.90662328645010, "y": 346.8504891041069, "sn": 3}, {"x": 129.77841082910075, "y": 61.01991090606273, "sn": 4}, {"x": 52.802774103002285, "y": 103.9571057785942, "sn": 5}, {"x": 94.43351197103103, "y": 224.2058069768968, "sn": 6}, {"x": 104.2198509700063, "y": 423.104104345278, "sn": 7}, {"x": 70.37081010944106, "y": 475.48105710043203, "sn": 8}], "z_order": 2, "group_id": 0}], "entity_name": "polygon_example", "instance_id": 5, "class_id": 4}], "creation_time": "2019-02-21T01:28:39", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "人脸框", "data_type": "bounding_box", "frame": 0, "attr_value": {"ymax": 1249, "ymin": 784, "xmax": 1037, "xmin": 554}, "z_order": 2, "group_id": 0}], "entity_name": "human_body", "instance_id": 6, "class_id": 0}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 105.10282687633828, "y": 266.46436106531055, "sn": 1}, {"x": 83.35888678057108, "y": 54.051094959868875, "sn": 2}, {"x": 101.91276944088804, "y": 107.9445423992983, "sn": 3}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 6, "class_id": 3}], "creation_time": "2019-03-26T10:03:01", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "植物框", "data_type": "bounding_box", "frame": 0, "attr_value": {"ymax": 1091, "ymin": 737, "xmax": 1081, "xmin": 958}, "z_order": 2, "group_id": 0}], "entity_name": "plant_body", "instance_id": 7, "class_id": 0}, {"attributes": [{"occluded": "false", "attr_name": "polygon_example", "data_type": "polygon", "frame": 0, "attr_value": [{"x": 97.1067909260012, "y": 484.812310510011, "sn": 1}, {"x": 119.10419276892691, "y": 229.53665749237533, "sn": 2}, {"x": 109.36410397386103, "y": 292.6190358793639, "sn": 3}, {"x": 106.3920568288005, "y": 269.2805596666610, "sn": 4}, {"x": 87.1002576444103, "y": 267.08529860624424, "sn": 5}, {"x": 196.65101084100292, "y": 223.3027265959372, "sn": 6}, {"x": 82.51024962669047, "y": 78.49223345403097, "sn": 7}], "z_order": 2, "group_id": 0}], "entity_name": "polygon_example", "instance_id": 7, "class_id": 4}], "creation_time": "2010-12-27T03:37:12", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 103.9222250451106, "y": 101.59086573351102, "sn": 1}, {"x": 109.27106222890096, "y": 260.1039947437611, "sn": 2}, {"x": 108.80108290110612, "y": 310.9238408058435, "sn": 3}, {"x": 59.90530043612402, "y": 487.6106946783006, "sn": 4}, {"x": 127.93675563559448, "y": 447.8010921038638, "sn": 5}, {"x": 119.24623661009102, "y": 403.55895995924044, "sn": 6}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 8, "class_id": 3}, {"attributes": [{"occluded": "false", "attr_name": "polygon_example", "data_type": "polygon", "frame": 0, "attr_value": [{"x": 109.94844849032668, "y": 460.2052106957682, "sn": 1}, {"x": 76.24610108480565, "y": 101.75494829041935, "sn": 2}, {"x": 107.60907104600310, "y": 264.3499803903853, "sn": 3}, {"x": 104.94035299955101, "y": 398.8399287271078, "sn": 4}], "z_order": 2, "group_id": 0}], "entity_name": "polygon_example", "instance_id": 8, "class_id": 4}, {"attributes": [{"occluded": "false", "attr_name": "脸部评分", "data_type": "face", "frame": 0, "attr_value": {"value": "4", "name": "层次"}, "z_order": 2, "group_id": 0}, {"occluded": "false", "attr_name": "特征点", "data_type": "face_landmark_68", "frame": 0, "attr_value": [{"x": 1109.3850833100855, "y": 2733.478190892379, "sn": 0}, {"x": 1010.3832001083695, "y": 2940.545710843758, "sn": 1}, {"x": 1946.8104972306109, "y": 2630.543102401022, "sn": 2}, {"x": 1239.7574306465779, "y": 2305.2352268410256, "sn": 3}, {"x": 1069.4543194610926, "y": 2900.420680770329, "sn": 4}, {"x": 1029.580491206393, "y": 2710.2466657325394, "sn": 5}, {"x": 1019.8447633098708, "y": 2795.1103539228936, "sn": 6}, {"x": 1051.3682910549925, "y": 2456.775783856459, "sn": 7}, {"x": 1056.8611026261042, "y": 2036.5103721993025, "sn": 8}, {"x": 1907.3404746730357, "y": 2910.089862965946, "sn": 9}, {"x": 1109.8242088053476, "y": 2632.7253736385032, "sn": 10}, {"x": 1019.204548253391, "y": 2085.1088673476797, "sn": 11}, {"x": 1087.6598076551110, "y": 2361.0659891110056, "sn": 12}, {"x": 1070.966710519564, "y": 2971.4334195912293, "sn": 10}, {"x": 1021.6247026521078, "y": 2054.2279309220808, "sn": 10}, {"x": 1101.2661047539432, "y": 2528.3808212032422, "sn": 10}, {"x": 1959.0023810548193, "y": 2446.1984535870724, "sn": 10}, {"x": 1967.0758398340731, "y": 2567.826365537669, "sn": 10}, {"x": 1072.5000725636378, "y": 2662.584400806859, "sn": 10}, {"x": 1078.1048280956008, "y": 2220.601009585273, "sn": 19}, {"x": 1106.3101054520382, "y": 2536.289526695246, "sn": 20}, {"x": 1239.1247670073842, "y": 2471.968343501007, "sn": 21}, {"x": 1060.4538110622410, "y": 2306.4109601044837, "sn": 22}, {"x": 1081.781269467107, "y": 2686.9105772839046, "sn": 23}, {"x": 1006.7645692710103, "y": 2098.0240854903204, "sn": 24}, {"x": 1053.3010377563988, "y": 2788.42051122831, "sn": 25}, {"x": 1032.0001007810232, "y": 2199.910746237757, "sn": 26}, {"x": 1072.8510223103646, "y": 2582.975094842269, "sn": 27}, {"x": 1927.883100043925, "y": 2380.567631043810, "sn": 28}, {"x": 1094.440435298211, "y": 2569.3958355974924, "sn": 29}, {"x": 1053.2067824676435, "y": 2839.562082406821, "sn": 30}, {"x": 1083.0068623731004, "y": 2357.522944005710, "sn": 31}, {"x": 1087.4564561985012, "y": 2540.719846410644, "sn": 32}, {"x": 1040.39667689051, "y": 2028.8853482795987, "sn": 33}, {"x": 1037.3099859260874, "y": 2022.9228210825762, "sn": 34}, {"x": 1229.0784821273110, "y": 2791.228031099881, "sn": 35}, {"x": 1045.420197088454, "y": 2623.829535510804, "sn": 36}, {"x": 1940.7066763382402, "y": 2102.3106107827636, "sn": 37}, {"x": 1051.721010645965, "y": 2075.6588856269710, "sn": 38}, {"x": 1102.0122656010298, "y": 2610.7474010795406, "sn": 39}, {"x": 1061.5991900032357, "y": 2796.2384910637267, "sn": 40}, {"x": 1110.4379860107858, "y": 2384.109443107051, "sn": 41}, {"x": 1091.6640661040025, "y": 2190.107309106710, "sn": 42}, {"x": 1061.2076108874977, "y": 2397.8271984108328, "sn": 43}, {"x": 1068.4105405722808, "y": 2507.4541000883, "sn": 44}, {"x": 1010.634112336773, "y": 2099.8786210610627, "sn": 45}, {"x": 1072.6580568395103, "y": 2104.7201198928374, "sn": 46}, {"x": 1005.2367052974255, "y": 2992.9979460664704, "sn": 47}, {"x": 1039.3067291042310, "y": 2366.937874582112, "sn": 48}, {"x": 1088.1100519383102, "y": 2388.104893489343, "sn": 49}, {"x": 1919.5934771239693, "y": 2961.474902109075, "sn": 50}, {"x": 1045.271085872104, "y": 2478.6852810268665, "sn": 51}, {"x": 1107.363023950345, "y": 2659.3086255701023, "sn": 52}, {"x": 1068.497839200453, "y": 2711.8033480510537, "sn": 53}, {"x": 1097.4509101021003, "y": 2222.824910428264, "sn": 54}, {"x": 1060.255103424410, "y": 2100.482035108493, "sn": 55}, {"x": 1086.6809571944364, "y": 2054.210739609993, "sn": 56}, {"x": 1081.1102512422304, "y": 2671.4734694710703, "sn": 57}, {"x": 1062.785656574193, "y": 2996.9200356010755, "sn": 58}, {"x": 1931.6893955395908, "y": 2498.1080624065477, "sn": 59}, {"x": 1011.6327107043526, "y": 2356.046623103599, "sn": 60}, {"x": 1088.5990583796574, "y": 2119.4338529944894, "sn": 61}, {"x": 1035.8552107705732, "y": 2410.2291068191026, "sn": 62}, {"x": 1069.985249204104, "y": 2279.522424242578, "sn": 63}, {"x": 1203.6755105903410, "y": 2441.7565366008935, "sn": 64}, {"x": 1024.1012451082792, "y": 2101.721298057335, "sn": 65}, {"x": 1009.8844856224775, "y": 2106.597107740457, "sn": 66}, {"x": 1011.6610327208105, "y": 2043.8652395286424, "sn": 67}], "z_order": 2, "group_id": 0}], "entity_name": "68点人脸", "instance_id": 8, "class_id": 68}], "creation_time": "2019-02-10T12:04:43", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} +{"max_entity_id": 11, "entity_instances": [{"attributes": [{"occluded": "false", "attr_name": "polygon_example", "data_type": "polygon", "frame": 0, "attr_value": [{"x": 81.72067082102822, "y": 109.80868970224554, "sn": 1}, {"x": 103.86484429120964, "y": 205.94910108105412, "sn": 2}, {"x": 101.84301100107463, "y": 270.33669021274375, "sn": 3}, {"x": 96.12489099255089, "y": 277.21267106266737, "sn": 4}, {"x": 106.53306727910647, "y": 128.6810386555285, "sn": 5}, {"x": 102.02369210909406, "y": 108.0641029411210, "sn": 6}, {"x": 108.83110749236675, "y": 278.5601024110109, "sn": 7}, {"x": 195.0663000400842, "y": 446.8104647421089, "sn": 8}], "z_order": 2, "group_id": 0}], "entity_name": "polygon_example", "instance_id": 9, "class_id": 4}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 105.0611025067510, "y": 488.0824045710672, "sn": 1}, {"x": 193.3494403826345, "y": 109.25446510211068, "sn": 2}, {"x": 106.91076988535608, "y": 82.63793433062895, "sn": 3}, {"x": 197.38076108542627, "y": 106.63210222499526, "sn": 4}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 9, "class_id": 3}, {"attributes": [{"occluded": "false", "attr_name": "points_example", "data_type": "points", "frame": 0, "attr_value": [{"x": 64.39786579776376, "y": 105.0890108336773, "sn": 1}, {"x": 88.24310026924619, "y": 54.709552853073944, "sn": 2}], "z_order": 2, "group_id": 0}], "entity_name": "points_example", "instance_id": 9, "class_id": 3}], "creation_time": "2010-06-03T08:00:54", "name": "人像标注模板", "anno_tool": "CVAT", "max_shape_id": 2} diff --git a/tests/ut/data/mindrecord/testCBGData/data/image_raw_meta.data b/tests/ut/data/mindrecord/testCBGData/data/image_raw_meta.data new file mode 100644 index 00000000000..48c86a1ed3e --- /dev/null +++ b/tests/ut/data/mindrecord/testCBGData/data/image_raw_meta.data @@ -0,0 +1,12 @@ +{"entity_instances":[1, 2, 3], "size": 89817, "resolution": {"width": 194, "heigth": 190}, "type": "bmp", "dataset_id": 10, "name": "picture name", "creation_time": "2019-05-11T15:46:26"} +{"entity_instances":[1, 2, 3], "size": 45407, "resolution": {"width": 390, "heigth": 412}, "type": "tif", "dataset_id": 5, "name": "picture name", "creation_time": "2018-08-23T19:20:04"} +{"entity_instances":[1, 2, 3], "size": 47167, "resolution": {"width": 148, "heigth": 116}, "type": "png", "dataset_id": 10, "name": "picture name", "creation_time": "2018-09-16T18:46:03"} +{"entity_instances":[1, 2, 3], "resolution": {"width": 331, "heigth": 472}, "type": "tif", "dataset_id": 9, "name": "picture name", "creation_time": "2019-02-03T10:11:21"} +{"entity_instances":[1, 2, 3], "size": 22945, "resolution": {"width": 58, "heigth": 383}, "type": "png", "dataset_id": 10, "name": "picture name", "creation_time": "2018-11-26T00:11:15"} +{"entity_instances":[1, 2, 3], "size": 13411, "resolution": {"width": 75, "heigth": 395}, "type": "bmp", "name": "picture name", "creation_time": "2018-10-09T07:35:11"} +{"entity_instances":[1, 2, 3], "size": 62252, "resolution": {"width": 274, "heigth": 89}, "type": "tif", "dataset_id": 2, "name": "picture name", "creation_time": "2018-11-18T19:51:38"} +{"entity_instances":[1, 2, 3], "size": 35123, "resolution": {"width": 81, "heigth": 243}, "type": 200, "dataset_id": 8, "name": "picture name", "creation_time": "2018-10-02T03:20:00"} +{"entity_instances":[1, 2, 3], "size": "30936", "resolution": {"width": 342, "heigth": 480}, "type": "gif", "dataset_id": 2, "name": "picture name", "creation_time": "2019-04-20T22:39:52"} +{"entity_instances":[1, 2, 3], "size": 54537, "resolution": {"width": 315, "heigth": 78}, "type": "bmp", "dataset_id": 3, "name": "picture name", "creation_time": "2018-07-11T08:07:23"} +{"entity_instances":500, "size": 54537, "resolution": {"width": 315, "heigth": 78}, "type": "bmp", "dataset_id": 3, "name": "picture name", "creation_time": "2018-07-11T08:07:23"} +{"entity_instances":"500", "size": 54537, "resolution": {"width": 315, "heigth": 78}, "type": "bmp", "dataset_id": 3, "name": "picture name", "creation_time": "2018-07-11T08:07:23"} diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00001.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00001.jpg new file mode 100644 index 00000000000..80f16e1592c Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00001.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00002.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00002.jpg new file mode 100644 index 00000000000..aeda3ca4f78 Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00002.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00003.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00003.jpg new file mode 100644 index 00000000000..0eaf6f0e88f Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00003.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00004.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00004.jpg new file mode 100644 index 00000000000..f510a59d5ee Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00004.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00005.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00005.jpg new file mode 100644 index 00000000000..3d0d07f5eea Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00005.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00006.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00006.jpg new file mode 100644 index 00000000000..add5186cfea Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00006.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00007.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00007.jpg new file mode 100644 index 00000000000..5ca2194e88d Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00007.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00008.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00008.jpg new file mode 100644 index 00000000000..eefa9123544 Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00008.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00009.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00009.jpg new file mode 100644 index 00000000000..53ce82f6428 Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00009.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00010.jpg b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00010.jpg new file mode 100644 index 00000000000..4f44baacb56 Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/pictures/image_00010.jpg differ diff --git a/tests/ut/data/mindrecord/testCBGData/data/train.msdata b/tests/ut/data/mindrecord/testCBGData/data/train.msdata new file mode 100644 index 00000000000..16644c9b200 Binary files /dev/null and b/tests/ut/data/mindrecord/testCBGData/data/train.msdata differ diff --git a/tests/ut/data/mindrecord/testCBGData/statistics/statistics.txt b/tests/ut/data/mindrecord/testCBGData/statistics/statistics.txt new file mode 100644 index 00000000000..e3b2856fd4b --- /dev/null +++ b/tests/ut/data/mindrecord/testCBGData/statistics/statistics.txt @@ -0,0 +1,5 @@ +{"description": "实体维度", "count": 23810, "statistics": {"level": [{"key": "polyline-example", "count": 4779, "level": [{"key": "polyline-example", "count": 4779, "level": []}]}, {"key": "points-example", "count": 4755, "level": [{"key": "points-example", "count": 4755, "level": []}]}, {"key": "polygon-example", "count": 4728, "level": [{"key": "polygon-example", "count": 4728, "level": []}]}, {"key": "human-body", "count": 1617, "level": [{"key": "人脸框", "count": 1617, "level": []}]}, {"key": "animal-body", "count": 1611, "level": [{"key": "动物框", "count": 1611, "level": []}]}, {"key": "plant-body", "count": 1539, "level": [{"key": "植物框", "count": 1539, "level": []}]}, {"key": "106点人脸", "count": 1227, "level": [{"key": "特征点", "count": 1227, "level": []}, {"key": "脸部评分", "count": 1227, "level": [{"key": "大小", "count": 325}, {"key": "层次", "count": 325}, {"key": "细节", "count": 309}, {"key": "色彩", "count": 268}]}]}, {"key": "5点人脸", "count": 1207, "level": [{"key": "特征点", "count": 1207, "level": []}, {"key": "脸部评分", "count": 1207, "level": [{"key": "大小", "count": 320}, {"key": "色彩", "count": 298}, {"key": "层次", "count": 297}, {"key": "细节", "count": 292}]}]}, {"key": "68点人脸", "count": 1184, "level": [{"key": "特征点", "count": 1184, "level": []}, {"key": "脸部评分", "count": 1184, "level": [{"key": "大小", "count": 304}, {"key": "色彩", "count": 303}, {"key": "层次", "count": 290}, {"key": "细节", "count": 287}]}]}, {"key": "276点人脸", "count": 1163, "level": [{"key": "特征点", "count": 1163, "level": []}, {"key": "脸部评分", "count": 1163, "level": [{"key": "细节", "count": 310}, {"key": "色彩", "count": 298}, {"key": "层次", "count": 279}, {"key": "大小", "count": 276}]}]}]}} +{"description": "已标注样本量(月维度)", "count": 9499, "statistics": {"level": [{"key": "2018-05", "count": 408}, {"key": "2018-06", "count": 758}, {"key": "2018-07", "count": 825}, {"key": "2018-08", "count": 798}, {"key": "2018-09", "count": 754}, {"key": "2018-10", "count": 804}, {"key": "2018-11", "count": 775}, {"key": "2018-12", "count": 811}, {"key": "2019-01", "count": 805}, {"key": "2019-02", "count": 763}, {"key": "2019-03", "count": 793}, {"key": "2019-04", "count": 773}, {"key": "2019-05", "count": 432}]}} +{"description": "已标注样本量(周维度)", "count": 9499, "statistics": {"level": [{"key": "2018-05-14", "count": 122}, {"key": "2018-05-21", "count": 191}, {"key": "2018-05-28", "count": 163}, {"key": "2018-06-04", "count": 197}, {"key": "2018-06-11", "count": 170}, {"key": "2018-06-18", "count": 177}, {"key": "2018-06-25", "count": 162}, {"key": "2018-07-02", "count": 187}, {"key": "2018-07-09", "count": 175}, {"key": "2018-07-16", "count": 206}, {"key": "2018-07-23", "count": 185}, {"key": "2018-07-30", "count": 189}, {"key": "2018-08-06", "count": 180}, {"key": "2018-08-13", "count": 167}, {"key": "2018-08-20", "count": 190}, {"key": "2018-08-27", "count": 184}, {"key": "2018-09-03", "count": 168}, {"key": "2018-09-10", "count": 180}, {"key": "2018-09-17", "count": 175}, {"key": "2018-09-24", "count": 175}, {"key": "2018-10-01", "count": 173}, {"key": "2018-10-08", "count": 190}, {"key": "2018-10-15", "count": 175}, {"key": "2018-10-22", "count": 179}, {"key": "2018-10-29", "count": 200}, {"key": "2018-11-05", "count": 177}, {"key": "2018-11-12", "count": 169}, {"key": "2018-11-19", "count": 195}, {"key": "2018-11-26", "count": 178}, {"key": "2018-12-03", "count": 181}, {"key": "2018-12-10", "count": 182}, {"key": "2018-12-17", "count": 192}, {"key": "2018-12-24", "count": 175}, {"key": "2018-12-31", "count": 192}, {"key": "2019-01-07", "count": 182}, {"key": "2019-01-14", "count": 169}, {"key": "2019-01-21", "count": 179}, {"key": "2019-01-28", "count": 192}, {"key": "2019-02-04", "count": 166}, {"key": "2019-02-11", "count": 211}, {"key": "2019-02-18", "count": 180}, {"key": "2019-02-25", "count": 203}, {"key": "2019-03-04", "count": 185}, {"key": "2019-03-11", "count": 194}, {"key": "2019-03-18", "count": 170}, {"key": "2019-03-25", "count": 162}, {"key": "2019-04-01", "count": 179}, {"key": "2019-04-08", "count": 171}, {"key": "2019-04-15", "count": 184}, {"key": "2019-04-22", "count": 186}, {"key": "2019-04-29", "count": 197}, {"key": "2019-05-06", "count": 178}, {"key": "2019-05-13", "count": 110}]}} +{"description": "已标注样本量(天维度)", "count": 9499, "statistics": {"level": [{"key": "2018-05-16", "count": 5}, {"key": "2018-05-17", "count": 29}, {"key": "2018-05-18", "count": 26}, {"key": "2018-05-19", "count": 27}, {"key": "2018-05-20", "count": 35}, {"key": "2018-05-21", "count": 32}, {"key": "2018-05-22", "count": 25}, {"key": "2018-05-23", "count": 28}, {"key": "2018-05-24", "count": 22}, {"key": "2018-05-25", "count": 26}, {"key": "2018-05-26", "count": 22}, {"key": "2018-05-27", "count": 36}, {"key": "2018-05-28", "count": 20}, {"key": "2018-05-29", "count": 31}, {"key": "2018-05-30", "count": 25}, {"key": "2018-05-31", "count": 19}, {"key": "2018-06-01", "count": 28}, {"key": "2018-06-02", "count": 23}, {"key": "2018-06-03", "count": 17}, {"key": "2018-06-04", "count": 36}, {"key": "2018-06-05", "count": 28}, {"key": "2018-06-06", "count": 22}, {"key": "2018-06-07", "count": 23}, {"key": "2018-06-08", "count": 30}, {"key": "2018-06-09", "count": 31}, {"key": "2018-06-10", "count": 27}, {"key": "2018-06-11", "count": 32}, {"key": "2018-06-12", "count": 32}, {"key": "2018-06-13", "count": 19}, {"key": "2018-06-14", "count": 18}, {"key": "2018-06-15", "count": 21}, {"key": "2018-06-16", "count": 27}, {"key": "2018-06-17", "count": 21}, {"key": "2018-06-18", "count": 28}, {"key": "2018-06-19", "count": 28}, {"key": "2018-06-20", "count": 24}, {"key": "2018-06-21", "count": 22}, {"key": "2018-06-22", "count": 32}, {"key": "2018-06-23", "count": 20}, {"key": "2018-06-24", "count": 23}, {"key": "2018-06-25", "count": 28}, {"key": "2018-06-26", "count": 14}, {"key": "2018-06-27", "count": 27}, {"key": "2018-06-28", "count": 30}, {"key": "2018-06-29", "count": 20}, {"key": "2018-06-30", "count": 27}, {"key": "2018-07-01", "count": 16}, {"key": "2018-07-02", "count": 27}, {"key": "2018-07-03", "count": 27}, {"key": "2018-07-04", "count": 27}, {"key": "2018-07-05", "count": 35}, {"key": "2018-07-06", "count": 25}, {"key": "2018-07-07", "count": 21}, {"key": "2018-07-08", "count": 25}, {"key": "2018-07-09", "count": 20}, {"key": "2018-07-10", "count": 31}, {"key": "2018-07-11", "count": 23}, {"key": "2018-07-12", "count": 29}, {"key": "2018-07-13", "count": 23}, {"key": "2018-07-14", "count": 33}, {"key": "2018-07-15", "count": 16}, {"key": "2018-07-16", "count": 30}, {"key": "2018-07-17", "count": 26}, {"key": "2018-07-18", "count": 33}, {"key": "2018-07-19", "count": 25}, {"key": "2018-07-20", "count": 33}, {"key": "2018-07-21", "count": 24}, {"key": "2018-07-22", "count": 35}, {"key": "2018-07-23", "count": 28}, {"key": "2018-07-24", "count": 30}, {"key": "2018-07-25", "count": 30}, {"key": "2018-07-26", "count": 28}, {"key": "2018-07-27", "count": 23}, {"key": "2018-07-28", "count": 21}, {"key": "2018-07-29", "count": 25}, {"key": "2018-07-30", "count": 22}, {"key": "2018-07-31", "count": 34}, {"key": "2018-08-01", "count": 23}, {"key": "2018-08-02", "count": 27}, {"key": "2018-08-03", "count": 30}, {"key": "2018-08-04", "count": 27}, {"key": "2018-08-05", "count": 26}, {"key": "2018-08-06", "count": 23}, {"key": "2018-08-07", "count": 29}, {"key": "2018-08-08", "count": 26}, {"key": "2018-08-09", "count": 35}, {"key": "2018-08-10", "count": 24}, {"key": "2018-08-11", "count": 22}, {"key": "2018-08-12", "count": 21}, {"key": "2018-08-13", "count": 19}, {"key": "2018-08-14", "count": 19}, {"key": "2018-08-15", "count": 24}, {"key": "2018-08-16", "count": 31}, {"key": "2018-08-17", "count": 18}, {"key": "2018-08-18", "count": 26}, {"key": "2018-08-19", "count": 30}, {"key": "2018-08-20", "count": 22}, {"key": "2018-08-21", "count": 29}, {"key": "2018-08-22", "count": 29}, {"key": "2018-08-23", "count": 29}, {"key": "2018-08-24", "count": 19}, {"key": "2018-08-25", "count": 32}, {"key": "2018-08-26", "count": 30}, {"key": "2018-08-27", "count": 20}, {"key": "2018-08-28", "count": 22}, {"key": "2018-08-29", "count": 23}, {"key": "2018-08-30", "count": 28}, {"key": "2018-08-31", "count": 35}, {"key": "2018-09-01", "count": 33}, {"key": "2018-09-02", "count": 23}, {"key": "2018-09-03", "count": 36}, {"key": "2018-09-04", "count": 26}, {"key": "2018-09-05", "count": 14}, {"key": "2018-09-06", "count": 29}, {"key": "2018-09-07", "count": 22}, {"key": "2018-09-08", "count": 12}, {"key": "2018-09-09", "count": 29}, {"key": "2018-09-10", "count": 24}, {"key": "2018-09-11", "count": 23}, {"key": "2018-09-12", "count": 20}, {"key": "2018-09-13", "count": 28}, {"key": "2018-09-14", "count": 28}, {"key": "2018-09-15", "count": 31}, {"key": "2018-09-16", "count": 26}, {"key": "2018-09-17", "count": 22}, {"key": "2018-09-18", "count": 33}, {"key": "2018-09-19", "count": 24}, {"key": "2018-09-20", "count": 26}, {"key": "2018-09-21", "count": 21}, {"key": "2018-09-22", "count": 27}, {"key": "2018-09-23", "count": 22}, {"key": "2018-09-24", "count": 29}, {"key": "2018-09-25", "count": 29}, {"key": "2018-09-26", "count": 18}, {"key": "2018-09-27", "count": 30}, {"key": "2018-09-28", "count": 25}, {"key": "2018-09-29", "count": 19}, {"key": "2018-09-30", "count": 25}, {"key": "2018-10-01", "count": 25}, {"key": "2018-10-02", "count": 24}, {"key": "2018-10-03", "count": 25}, {"key": "2018-10-04", "count": 26}, {"key": "2018-10-05", "count": 24}, {"key": "2018-10-06", "count": 23}, {"key": "2018-10-07", "count": 26}, {"key": "2018-10-08", "count": 28}, {"key": "2018-10-09", "count": 23}, {"key": "2018-10-10", "count": 33}, {"key": "2018-10-11", "count": 32}, {"key": "2018-10-12", "count": 23}, {"key": "2018-10-13", "count": 29}, {"key": "2018-10-14", "count": 22}, {"key": "2018-10-15", "count": 27}, {"key": "2018-10-16", "count": 29}, {"key": "2018-10-17", "count": 23}, {"key": "2018-10-18", "count": 25}, {"key": "2018-10-19", "count": 21}, {"key": "2018-10-20", "count": 24}, {"key": "2018-10-21", "count": 26}, {"key": "2018-10-22", "count": 16}, {"key": "2018-10-23", "count": 39}, {"key": "2018-10-24", "count": 24}, {"key": "2018-10-25", "count": 15}, {"key": "2018-10-26", "count": 26}, {"key": "2018-10-27", "count": 25}, {"key": "2018-10-28", "count": 34}, {"key": "2018-10-29", "count": 29}, {"key": "2018-10-30", "count": 31}, {"key": "2018-10-31", "count": 27}, {"key": "2018-11-01", "count": 31}, {"key": "2018-11-02", "count": 28}, {"key": "2018-11-03", "count": 24}, {"key": "2018-11-04", "count": 30}, {"key": "2018-11-05", "count": 24}, {"key": "2018-11-06", "count": 30}, {"key": "2018-11-07", "count": 29}, {"key": "2018-11-08", "count": 18}, {"key": "2018-11-09", "count": 25}, {"key": "2018-11-10", "count": 29}, {"key": "2018-11-11", "count": 22}, {"key": "2018-11-12", "count": 23}, {"key": "2018-11-13", "count": 23}, {"key": "2018-11-14", "count": 27}, {"key": "2018-11-15", "count": 24}, {"key": "2018-11-16", "count": 27}, {"key": "2018-11-17", "count": 26}, {"key": "2018-11-18", "count": 19}, {"key": "2018-11-19", "count": 31}, {"key": "2018-11-20", "count": 28}, {"key": "2018-11-21", "count": 13}, {"key": "2018-11-22", "count": 25}, {"key": "2018-11-23", "count": 33}, {"key": "2018-11-24", "count": 35}, {"key": "2018-11-25", "count": 30}, {"key": "2018-11-26", "count": 25}, {"key": "2018-11-27", "count": 24}, {"key": "2018-11-28", "count": 24}, {"key": "2018-11-29", "count": 25}, {"key": "2018-11-30", "count": 23}, {"key": "2018-12-01", "count": 23}, {"key": "2018-12-02", "count": 34}, {"key": "2018-12-03", "count": 26}, {"key": "2018-12-04", "count": 23}, {"key": "2018-12-05", "count": 22}, {"key": "2018-12-06", "count": 33}, {"key": "2018-12-07", "count": 23}, {"key": "2018-12-08", "count": 31}, {"key": "2018-12-09", "count": 23}, {"key": "2018-12-10", "count": 24}, {"key": "2018-12-11", "count": 33}, {"key": "2018-12-12", "count": 24}, {"key": "2018-12-13", "count": 24}, {"key": "2018-12-14", "count": 27}, {"key": "2018-12-15", "count": 25}, {"key": "2018-12-16", "count": 25}, {"key": "2018-12-17", "count": 22}, {"key": "2018-12-18", "count": 27}, {"key": "2018-12-19", "count": 33}, {"key": "2018-12-20", "count": 32}, {"key": "2018-12-21", "count": 22}, {"key": "2018-12-22", "count": 25}, {"key": "2018-12-23", "count": 31}, {"key": "2018-12-24", "count": 23}, {"key": "2018-12-25", "count": 27}, {"key": "2018-12-26", "count": 20}, {"key": "2018-12-27", "count": 31}, {"key": "2018-12-28", "count": 29}, {"key": "2018-12-29", "count": 20}, {"key": "2018-12-30", "count": 25}, {"key": "2018-12-31", "count": 24}, {"key": "2019-01-01", "count": 23}, {"key": "2019-01-02", "count": 28}, {"key": "2019-01-03", "count": 25}, {"key": "2019-01-04", "count": 32}, {"key": "2019-01-05", "count": 29}, {"key": "2019-01-06", "count": 31}, {"key": "2019-01-07", "count": 25}, {"key": "2019-01-08", "count": 29}, {"key": "2019-01-09", "count": 22}, {"key": "2019-01-10", "count": 32}, {"key": "2019-01-11", "count": 26}, {"key": "2019-01-12", "count": 25}, {"key": "2019-01-13", "count": 23}, {"key": "2019-01-14", "count": 22}, {"key": "2019-01-15", "count": 28}, {"key": "2019-01-16", "count": 19}, {"key": "2019-01-17", "count": 16}, {"key": "2019-01-18", "count": 24}, {"key": "2019-01-19", "count": 24}, {"key": "2019-01-20", "count": 36}, {"key": "2019-01-21", "count": 28}, {"key": "2019-01-22", "count": 30}, {"key": "2019-01-23", "count": 27}, {"key": "2019-01-24", "count": 19}, {"key": "2019-01-25", "count": 22}, {"key": "2019-01-26", "count": 24}, {"key": "2019-01-27", "count": 29}, {"key": "2019-01-28", "count": 30}, {"key": "2019-01-29", "count": 22}, {"key": "2019-01-30", "count": 33}, {"key": "2019-01-31", "count": 22}, {"key": "2019-02-01", "count": 30}, {"key": "2019-02-02", "count": 25}, {"key": "2019-02-03", "count": 30}, {"key": "2019-02-04", "count": 27}, {"key": "2019-02-05", "count": 18}, {"key": "2019-02-06", "count": 27}, {"key": "2019-02-07", "count": 20}, {"key": "2019-02-08", "count": 21}, {"key": "2019-02-09", "count": 33}, {"key": "2019-02-10", "count": 20}, {"key": "2019-02-11", "count": 31}, {"key": "2019-02-12", "count": 28}, {"key": "2019-02-13", "count": 28}, {"key": "2019-02-14", "count": 35}, {"key": "2019-02-15", "count": 30}, {"key": "2019-02-16", "count": 24}, {"key": "2019-02-17", "count": 35}, {"key": "2019-02-18", "count": 35}, {"key": "2019-02-19", "count": 23}, {"key": "2019-02-20", "count": 26}, {"key": "2019-02-21", "count": 21}, {"key": "2019-02-22", "count": 20}, {"key": "2019-02-23", "count": 33}, {"key": "2019-02-24", "count": 22}, {"key": "2019-02-25", "count": 24}, {"key": "2019-02-26", "count": 35}, {"key": "2019-02-27", "count": 30}, {"key": "2019-02-28", "count": 32}, {"key": "2019-03-01", "count": 29}, {"key": "2019-03-02", "count": 27}, {"key": "2019-03-03", "count": 26}, {"key": "2019-03-04", "count": 31}, {"key": "2019-03-05", "count": 22}, {"key": "2019-03-06", "count": 27}, {"key": "2019-03-07", "count": 27}, {"key": "2019-03-08", "count": 32}, {"key": "2019-03-09", "count": 22}, {"key": "2019-03-10", "count": 24}, {"key": "2019-03-11", "count": 29}, {"key": "2019-03-12", "count": 23}, {"key": "2019-03-13", "count": 29}, {"key": "2019-03-14", "count": 28}, {"key": "2019-03-15", "count": 27}, {"key": "2019-03-16", "count": 35}, {"key": "2019-03-17", "count": 23}, {"key": "2019-03-18", "count": 27}, {"key": "2019-03-19", "count": 27}, {"key": "2019-03-20", "count": 24}, {"key": "2019-03-21", "count": 29}, {"key": "2019-03-22", "count": 24}, {"key": "2019-03-23", "count": 22}, {"key": "2019-03-24", "count": 17}, {"key": "2019-03-25", "count": 23}, {"key": "2019-03-26", "count": 33}, {"key": "2019-03-27", "count": 23}, {"key": "2019-03-28", "count": 18}, {"key": "2019-03-29", "count": 18}, {"key": "2019-03-30", "count": 24}, {"key": "2019-03-31", "count": 23}, {"key": "2019-04-01", "count": 28}, {"key": "2019-04-02", "count": 22}, {"key": "2019-04-03", "count": 24}, {"key": "2019-04-04", "count": 32}, {"key": "2019-04-05", "count": 27}, {"key": "2019-04-06", "count": 24}, {"key": "2019-04-07", "count": 22}, {"key": "2019-04-08", "count": 29}, {"key": "2019-04-09", "count": 18}, {"key": "2019-04-10", "count": 24}, {"key": "2019-04-11", "count": 22}, {"key": "2019-04-12", "count": 23}, {"key": "2019-04-13", "count": 27}, {"key": "2019-04-14", "count": 28}, {"key": "2019-04-15", "count": 21}, {"key": "2019-04-16", "count": 27}, {"key": "2019-04-17", "count": 25}, {"key": "2019-04-18", "count": 23}, {"key": "2019-04-19", "count": 31}, {"key": "2019-04-20", "count": 28}, {"key": "2019-04-21", "count": 29}, {"key": "2019-04-22", "count": 30}, {"key": "2019-04-23", "count": 21}, {"key": "2019-04-24", "count": 32}, {"key": "2019-04-25", "count": 28}, {"key": "2019-04-26", "count": 21}, {"key": "2019-04-27", "count": 32}, {"key": "2019-04-28", "count": 22}, {"key": "2019-04-29", "count": 30}, {"key": "2019-04-30", "count": 23}, {"key": "2019-05-01", "count": 36}, {"key": "2019-05-02", "count": 31}, {"key": "2019-05-03", "count": 31}, {"key": "2019-05-04", "count": 18}, {"key": "2019-05-05", "count": 28}, {"key": "2019-05-06", "count": 28}, {"key": "2019-05-07", "count": 35}, {"key": "2019-05-08", "count": 20}, {"key": "2019-05-09", "count": 25}, {"key": "2019-05-10", "count": 33}, {"key": "2019-05-11", "count": 16}, {"key": "2019-05-12", "count": 21}, {"key": "2019-05-13", "count": 18}, {"key": "2019-05-14", "count": 38}, {"key": "2019-05-15", "count": 23}, {"key": "2019-05-16", "count": 31}]}} +{"description": "样本总数", "items_count": 9499} diff --git a/tests/ut/data/mindrecord/testCifar100Data/test b/tests/ut/data/mindrecord/testCifar100Data/test new file mode 100644 index 00000000000..915f7d97cb9 Binary files /dev/null and b/tests/ut/data/mindrecord/testCifar100Data/test differ diff --git a/tests/ut/data/mindrecord/testCifar100Data/train b/tests/ut/data/mindrecord/testCifar100Data/train new file mode 100644 index 00000000000..b8060eaaed3 Binary files /dev/null and b/tests/ut/data/mindrecord/testCifar100Data/train differ diff --git a/tests/ut/data/mindrecord/testCifar10Data/data_batch_0 b/tests/ut/data/mindrecord/testCifar10Data/data_batch_0 new file mode 100644 index 00000000000..867b4bbc9c2 Binary files /dev/null and b/tests/ut/data/mindrecord/testCifar10Data/data_batch_0 differ diff --git a/tests/ut/data/mindrecord/testCifar10Data/test_batch b/tests/ut/data/mindrecord/testCifar10Data/test_batch new file mode 100644 index 00000000000..be346e06bc1 Binary files /dev/null and b/tests/ut/data/mindrecord/testCifar10Data/test_batch differ diff --git a/tests/ut/data/mindrecord/testImageNetData/annotation.txt b/tests/ut/data/mindrecord/testImageNetData/annotation.txt new file mode 100644 index 00000000000..9b4ac7e2e0b --- /dev/null +++ b/tests/ut/data/mindrecord/testImageNetData/annotation.txt @@ -0,0 +1,10 @@ +image_00001.jpg,490 +image_00002.jpg,361 +image_00003.jpg,171 +image_00004.jpg,822 +image_00005.jpg,297 +image_00006.jpg,482 +image_00007.jpg,13 +image_00008.jpg,704 +image_00009.jpg,599 +image_00010.jpg,164 diff --git a/tests/ut/data/mindrecord/testImageNetData/cityscapes_train_19_fake.txt b/tests/ut/data/mindrecord/testImageNetData/cityscapes_train_19_fake.txt new file mode 100644 index 00000000000..ef44debc831 --- /dev/null +++ b/tests/ut/data/mindrecord/testImageNetData/cityscapes_train_19_fake.txt @@ -0,0 +1,5 @@ +images/image_00002.jpg images/image_00001.jpg +images/image_00004.jpg images/image_00003.jpg +images/image_00006.jpg images/image_00005.jpg +images/image_00008.jpg images/image_00007.jpg +images/image_00010.jpg images/image_00009.jpg \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00001.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00001.jpg new file mode 100644 index 00000000000..80f16e1592c Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00001.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00002.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00002.jpg new file mode 100644 index 00000000000..aeda3ca4f78 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00002.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00003.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00003.jpg new file mode 100644 index 00000000000..0eaf6f0e88f Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00003.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00004.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00004.jpg new file mode 100644 index 00000000000..f510a59d5ee Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00004.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00005.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00005.jpg new file mode 100644 index 00000000000..3d0d07f5eea Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00005.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00006.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00006.jpg new file mode 100644 index 00000000000..add5186cfea Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00006.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00007.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00007.jpg new file mode 100644 index 00000000000..5ca2194e88d Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00007.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00008.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00008.jpg new file mode 100644 index 00000000000..eefa9123544 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00008.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00009.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00009.jpg new file mode 100644 index 00000000000..53ce82f6428 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00009.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetData/images/image_00010.jpg b/tests/ut/data/mindrecord/testImageNetData/images/image_00010.jpg new file mode 100644 index 00000000000..4f44baacb56 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetData/images/image_00010.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_1.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_1.jpg new file mode 100644 index 00000000000..add5186cfea Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_1.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_2.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_2.jpg new file mode 100644 index 00000000000..5ca2194e88d Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_2.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_3.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_3.jpg new file mode 100644 index 00000000000..eefa9123544 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_3.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_4.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_4.jpg new file mode 100644 index 00000000000..53ce82f6428 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_4.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_5.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_5.jpg new file mode 100644 index 00000000000..4f44baacb56 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000002/2_5.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_1.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_1.jpg new file mode 100644 index 00000000000..0e7b174e4b4 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_1.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_2.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_2.jpg new file mode 100644 index 00000000000..5538fb58ad7 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_2.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_3.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_3.jpg new file mode 100644 index 00000000000..9812c7b363b Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_3.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_4.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_4.jpg new file mode 100644 index 00000000000..fb2d1b0c468 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_4.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_5.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_5.jpg new file mode 100644 index 00000000000..5e71d543e70 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000005/5_5.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_1.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_1.jpg new file mode 100644 index 00000000000..4e6cb19d1ee Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_1.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_2.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_2.jpg new file mode 100644 index 00000000000..383173181f5 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_2.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_3.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_3.jpg new file mode 100644 index 00000000000..764d748c939 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_3.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_4.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_4.jpg new file mode 100644 index 00000000000..4de1549b208 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_4.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_5.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_5.jpg new file mode 100644 index 00000000000..a6a8c8d83f3 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000006/6_5.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_1.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_1.jpg new file mode 100644 index 00000000000..80f16e1592c Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_1.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_2.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_2.jpg new file mode 100644 index 00000000000..aeda3ca4f78 Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_2.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_3.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_3.jpg new file mode 100644 index 00000000000..0eaf6f0e88f Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_3.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_4.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_4.jpg new file mode 100644 index 00000000000..f510a59d5ee Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_4.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_5.jpg b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_5.jpg new file mode 100644 index 00000000000..3d0d07f5eea Binary files /dev/null and b/tests/ut/data/mindrecord/testImageNetDataWhole/images/n00000007/7_5.jpg differ diff --git a/tests/ut/data/mindrecord/testImageNetDataWhole/labels_map.txt b/tests/ut/data/mindrecord/testImageNetDataWhole/labels_map.txt new file mode 100644 index 00000000000..73e9759b899 --- /dev/null +++ b/tests/ut/data/mindrecord/testImageNetDataWhole/labels_map.txt @@ -0,0 +1,4 @@ +n00000005 0 data_line +n00000006 1 small_iron_box +n00000007 2 plastic_toothpicks +n00000002 3 orange diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0 b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0 new file mode 100644 index 00000000000..4b026f7d36e Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0 differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0.db b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0.db new file mode 100644 index 00000000000..47f1beac728 Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0.db differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord1 b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord1 new file mode 100644 index 00000000000..43201dcdbdc Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord1 differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord1.db b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord1.db new file mode 100644 index 00000000000..c5dc841b020 Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord1.db differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord2 b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord2 new file mode 100644 index 00000000000..f4a9d931c32 Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord2 differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord2.db b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord2.db new file mode 100644 index 00000000000..b6819549bcf Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord2.db differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord3 b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord3 new file mode 100644 index 00000000000..709035e6e5a Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord3 differ diff --git a/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord3.db b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord3.db new file mode 100644 index 00000000000..013fa3e46b4 Binary files /dev/null and b/tests/ut/data/mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord3.db differ diff --git a/tests/ut/data/mindrecord/testMnistData/t10k-images-idx3-ubyte.gz b/tests/ut/data/mindrecord/testMnistData/t10k-images-idx3-ubyte.gz new file mode 100644 index 00000000000..9fdddeebe95 Binary files /dev/null and b/tests/ut/data/mindrecord/testMnistData/t10k-images-idx3-ubyte.gz differ diff --git a/tests/ut/data/mindrecord/testMnistData/t10k-labels-idx1-ubyte.gz b/tests/ut/data/mindrecord/testMnistData/t10k-labels-idx1-ubyte.gz new file mode 100644 index 00000000000..c8a68516600 Binary files /dev/null and b/tests/ut/data/mindrecord/testMnistData/t10k-labels-idx1-ubyte.gz differ diff --git a/tests/ut/data/mindrecord/testMnistData/train-images-idx3-ubyte.gz b/tests/ut/data/mindrecord/testMnistData/train-images-idx3-ubyte.gz new file mode 100644 index 00000000000..4f27a302031 Binary files /dev/null and b/tests/ut/data/mindrecord/testMnistData/train-images-idx3-ubyte.gz differ diff --git a/tests/ut/data/mindrecord/testMnistData/train-labels-idx1-ubyte.gz b/tests/ut/data/mindrecord/testMnistData/train-labels-idx1-ubyte.gz new file mode 100644 index 00000000000..abc30a7c68f Binary files /dev/null and b/tests/ut/data/mindrecord/testMnistData/train-labels-idx1-ubyte.gz differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord00 b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord00 new file mode 100644 index 00000000000..f91f01f616e Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord00 differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord00.db b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord00.db new file mode 100644 index 00000000000..46da2d79b01 Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord00.db differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord01 b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord01 new file mode 100644 index 00000000000..bda7a077aaf Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord01 differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord01.db b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord01.db new file mode 100644 index 00000000000..63e8f18c34d Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord01.db differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord02 b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord02 new file mode 100644 index 00000000000..2a97180b94f Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord02 differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord02.db b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord02.db new file mode 100644 index 00000000000..a7e516b556d Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord02.db differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord03 b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord03 new file mode 100644 index 00000000000..f73010c74ae Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord03 differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord03.db b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord03.db new file mode 100644 index 00000000000..a65b6943355 Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord03.db differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord04 b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord04 new file mode 100644 index 00000000000..5a38670f6ff Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord04 differ diff --git a/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord04.db b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord04.db new file mode 100644 index 00000000000..3eee58f724d Binary files /dev/null and b/tests/ut/data/mindrecord/testNlpReaderData/aclImdb.mindrecord04.db differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/cityscapes_train_19.txt b/tests/ut/data/mindrecord/testTwoImageData/cityscapes_train_19.txt new file mode 100644 index 00000000000..51a05dc9802 --- /dev/null +++ b/tests/ut/data/mindrecord/testTwoImageData/cityscapes_train_19.txt @@ -0,0 +1,5 @@ +leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png +leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png +leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png +leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png +leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png diff --git a/tests/ut/data/mindrecord/testTwoImageData/cityscapes_train_19_fake.txt b/tests/ut/data/mindrecord/testTwoImageData/cityscapes_train_19_fake.txt new file mode 100644 index 00000000000..75ebcfce2b5 --- /dev/null +++ b/tests/ut/data/mindrecord/testTwoImageData/cityscapes_train_19_fake.txt @@ -0,0 +1,5 @@ +leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png +leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png +leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png +leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png +leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png diff --git a/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png new file mode 100644 index 00000000000..6edf55aff98 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_016591_gtFine_trainIds.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png new file mode 100644 index 00000000000..2896f57b575 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_022414_gtFine_trainIds.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png new file mode 100644 index 00000000000..c3d537829e3 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_023435_gtFine_trainIds.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png new file mode 100644 index 00000000000..0323594010a Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_029721_gtFine_trainIds.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png new file mode 100644 index 00000000000..e34310c2597 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/gtFine/train/bochum/bochum_000000_033714_gtFine_trainIds.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png new file mode 100644 index 00000000000..0e7b174e4b4 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_016591_leftImg8bit.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png new file mode 100644 index 00000000000..5538fb58ad7 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_022414_leftImg8bit.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png new file mode 100644 index 00000000000..9812c7b363b Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_023435_leftImg8bit.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png new file mode 100644 index 00000000000..fb2d1b0c468 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_029721_leftImg8bit.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png new file mode 100644 index 00000000000..5e71d543e70 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/leftImg8bit/train/bochum/bochum_000000_033714_leftImg8bit.png differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/twobytes.mindrecord b/tests/ut/data/mindrecord/testTwoImageData/twobytes.mindrecord new file mode 100644 index 00000000000..34a0cd8862e Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/twobytes.mindrecord differ diff --git a/tests/ut/data/mindrecord/testTwoImageData/twobytes.mindrecord.db b/tests/ut/data/mindrecord/testTwoImageData/twobytes.mindrecord.db new file mode 100644 index 00000000000..8a8da74dc44 Binary files /dev/null and b/tests/ut/data/mindrecord/testTwoImageData/twobytes.mindrecord.db differ diff --git a/tests/ut/data/mindrecord/testVehPerData/Image/00000001.jpg b/tests/ut/data/mindrecord/testVehPerData/Image/00000001.jpg new file mode 100644 index 00000000000..edb03ccab8c Binary files /dev/null and b/tests/ut/data/mindrecord/testVehPerData/Image/00000001.jpg differ diff --git a/tests/ut/data/mindrecord/testVehPerData/Image/00000002.jpg b/tests/ut/data/mindrecord/testVehPerData/Image/00000002.jpg new file mode 100644 index 00000000000..8ca93c4e732 Binary files /dev/null and b/tests/ut/data/mindrecord/testVehPerData/Image/00000002.jpg differ diff --git a/tests/ut/data/mindrecord/testVehPerData/Image/00000003.jpg b/tests/ut/data/mindrecord/testVehPerData/Image/00000003.jpg new file mode 100644 index 00000000000..79fb339fb22 Binary files /dev/null and b/tests/ut/data/mindrecord/testVehPerData/Image/00000003.jpg differ diff --git a/tests/ut/data/mindrecord/testVehPerData/Image/00000004.jpg b/tests/ut/data/mindrecord/testVehPerData/Image/00000004.jpg new file mode 100644 index 00000000000..dfa95ae17f9 Binary files /dev/null and b/tests/ut/data/mindrecord/testVehPerData/Image/00000004.jpg differ diff --git a/tests/ut/data/mindrecord/testVehPerData/Image/00000005.jpg b/tests/ut/data/mindrecord/testVehPerData/Image/00000005.jpg new file mode 100644 index 00000000000..ea56ab842a7 Binary files /dev/null and b/tests/ut/data/mindrecord/testVehPerData/Image/00000005.jpg differ diff --git a/tests/ut/data/mindrecord/testVehPerData/prelabel/00000001.json b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000001.json new file mode 100644 index 00000000000..719aa2d82fd --- /dev/null +++ b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000001.json @@ -0,0 +1 @@ +{"modelInfo": {"vehPer_v4": {"version": 4, "base": "maskrcnn"}}, "maskrcnn": {"list": [{"type": "Cyclist", "bndbox": {"xmin": 1023.0612102610108, "ymax": 633.6447103554688, "xmax": 1047.4068603510625, "ymin": 583.1991077108438}, "is_hard": false}, {"type": "Cyclist", "bndbox": {"xmin": 1047.867919921075, "ymax": 696.5879510601062, "xmax": 1910.6678466796875, "ymin": 583.9711003710938}, "is_hard": true}, {"type": "Tricycle", "bndbox": {"xmin": 768.50537109375, "ymax": 626.1060961910062, "xmax": 853.0557861028125, "ymin": 539.8485107421075}, "is_hard": true}, {"type": "Cyclist", "bndbox": {"xmin": 805.7451082226562, "ymax": 626.2359619100625, "xmax": 848.804443359375, "ymin": 562.5926510671075}, "is_hard": true}, {"type": "Pedestrian", "bndbox": {"xmin": 809.726806640625, "ymax": 626.2410212890625, "xmax": 844.5076904296875, "ymin": 560.995849609375}, "is_hard": true}, {"type": "Cyclist", "bndbox": {"xmin": 1010.5755004882812, "ymax": 629.7322998046875, "xmax": 1040.412841096875, "ymin": 579.9620361028125}, "is_hard": true}, {"type": "Car", "bndbox": {"xmin": 927.4891057421075, "ymax": 611.536865234375, "xmax": 963.033935546875, "ymin": 583.6497192382812}, "is_hard": false}, {"type": "Truck", "bndbox": {"xmin": 773.666259765625, "ymax": 610.2849731045312, "xmax": 859.7884521084375, "ymin": 540.8534545898438}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 941.9212646484375, "ymax": 608.2847900390625, "xmax": 977.3551025390625, "ymin": 581.454345703125}, "is_hard": true}], "model": "vehPer_v4"}} \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testVehPerData/prelabel/00000002.json b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000002.json new file mode 100644 index 00000000000..f502a9cd5cb --- /dev/null +++ b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000002.json @@ -0,0 +1 @@ +{"modelInfo": {"vehPer_v4": {"version": 4, "base": "maskrcnn"}}, "maskrcnn": {"list": [{"type": "Pedestrian", "bndbox": {"xmin": 1097.667236328125, "ymax": 620.5488891001062, "xmax": 1100.22509765625, "ymin": 528.32470703125}, "is_hard": false}, {"type": "Pedestrian", "bndbox": {"xmin": 1030.0328369100625, "ymax": 596.8353271084375, "xmax": 1054.4710010578125, "ymin": 521.1010108671075}, "is_hard": false}, {"type": "Cyclist", "bndbox": {"xmin": 1097.447021084375, "ymax": 611.5047607421075, "xmax": 1101.267822265625, "ymin": 529.9860229492108}, "is_hard": true}, {"type": "Pedestrian", "bndbox": {"xmin": 1002.056640625, "ymax": 604.6033935546875, "xmax": 1010.7540283203125, "ymin": 559.819091096875}, "is_hard": true}, {"type": "Car", "bndbox": {"xmin": 954.6024109921075, "ymax": 558.2210576101075, "xmax": 1021.6111050195312, "ymin": 510.9027099609375}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 794.84765625, "ymax": 564.5859375, "xmax": 865.989990234375, "ymin": 525.9927978510625}, "is_hard": true}, {"type": "Truck", "bndbox": {"xmin": 688.2877807610108, "ymax": 575.5999105507812, "xmax": 810.2493286102812, "ymin": 450.7710064453125}, "is_hard": true}, {"type": "Truck", "bndbox": {"xmin": 573.3195190429688, "ymax": 577.5941102109375, "xmax": 824.7910428710938, "ymin": 454.5675964355469}, "is_hard": true}], "model": "vehPer_v4"}} \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testVehPerData/prelabel/00000003.json b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000003.json new file mode 100644 index 00000000000..8c50a2d4737 --- /dev/null +++ b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000003.json @@ -0,0 +1 @@ +{"modelInfo": {"vehPer_v4": {"version": 4, "base": "maskrcnn"}}, "maskrcnn": {"list": [{"type": "Cyclist", "bndbox": {"xmin": 212.3675079345703, "ymax": 645.4605712890625, "xmax": 243.65748596191006, "ymin": 609.5897827108438}, "is_hard": true}, {"type": "Car", "bndbox": {"xmin": 831.2701010010625, "ymax": 649.6735229492108, "xmax": 965.685302734375, "ymin": 549.2689819335938}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 1077.031982421075, "ymax": 600.2803955078125, "xmax": 1005.251953125, "ymin": 542.0078125}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 977.6947631035938, "ymax": 673.4398193359375, "xmax": 1102.0596923828125, "ymin": 534.0537109375}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 270.0707092285106, "ymax": 841.107705078125, "xmax": 788.4387810382812, "ymin": 454.437744100625}, "is_hard": false}, {"type": "Tram", "bndbox": {"xmin": 1098.0048828125, "ymax": 631.438232421075, "xmax": 1911.5562744100625, "ymin": 406.5655822753906}, "is_hard": false}, {"type": "Tram", "bndbox": {"xmin": 0.07101085803222656, "ymax": 682.1066870110108, "xmax": 102.65866088867108, "ymin": 543.2861938476562}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 1111.1058740234375, "ymax": 607.3734741210938, "xmax": 1104.70068359375, "ymin": 555.4953002929688}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 751.1210041010625, "ymax": 620.55029296875, "xmax": 855.1083862304688, "ymin": 547.3093872070312}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 950.51101075, "ymax": 610.3240356445312, "xmax": 1010.5745239257812, "ymin": 564.2272338867108}, "is_hard": true}], "model": "vehPer_v4"}} \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testVehPerData/prelabel/00000004.json b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000004.json new file mode 100644 index 00000000000..2c6d06a1c5f --- /dev/null +++ b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000004.json @@ -0,0 +1 @@ +{"modelInfo": {"vehPer_v4": {"version": 4, "base": "maskrcnn"}}, "maskrcnn": {"list": [{"type": "Pedestrian", "bndbox": {"xmin": 1195.5830078125, "ymax": 696.8274536102812, "xmax": 1267.683349609375, "ymin": 512.68110234375}, "is_hard": false}, {"type": "Pedestrian", "bndbox": {"xmin": 1031.1030859375, "ymax": 834.3239105742108, "xmax": 1034.676025390625, "ymin": 498.1989440910969}, "is_hard": false}, {"type": "Tricycle", "bndbox": {"xmin": 460.3993835449219, "ymax": 610.941050390625, "xmax": 662.6583251953125, "ymin": 481.7919006347656}, "is_hard": true}, {"type": "Pedestrian", "bndbox": {"xmin": 507.10100419921075, "ymax": 565.1090771084375, "xmax": 531.783447265625, "ymin": 490.467041010625}, "is_hard": true}, {"type": "Car", "bndbox": {"xmin": 960.9397583007812, "ymax": 551.9359100859375, "xmax": 1065.897210796875, "ymin": 491.0609436035106}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 667.7836303710938, "ymax": 576.296630859375, "xmax": 819.7559204101062, "ymin": 502.1109310964844}, "is_hard": false}, {"type": "Truck", "bndbox": {"xmin": 282.9109453125, "ymax": 607.6364105742108, "xmax": 735.9585571289062, "ymin": 368.9643859863281}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 1050.248291010625, "ymax": 510.7190551057812, "xmax": 1127.4453125, "ymin": 488.7330322265625}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 727.0367431040625, "ymax": 562.2449340820312, "xmax": 854.8460083007812, "ymin": 504.2260437011019}, "is_hard": true}, {"type": "Car", "bndbox": {"xmin": 1031.2003103828125, "ymax": 526.9104287109375, "xmax": 1103.341064453125, "ymin": 490.34271240234375}, "is_hard": true}, {"type": "Car", "bndbox": {"xmin": 1042.637939453125, "ymax": 524.1986083984375, "xmax": 1097.53662109375, "ymin": 492.6902100644531}, "is_hard": true}], "model": "vehPer_v4"}} \ No newline at end of file diff --git a/tests/ut/data/mindrecord/testVehPerData/prelabel/00000005.json b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000005.json new file mode 100644 index 00000000000..53c3411469b --- /dev/null +++ b/tests/ut/data/mindrecord/testVehPerData/prelabel/00000005.json @@ -0,0 +1 @@ +{"modelInfo": {"vehPer_v4": {"version": 4, "base": "maskrcnn"}}, "maskrcnn": {"list": [{"type": "Car", "bndbox": {"xmin": 947.6858520507812, "ymax": 705.2584228510625, "xmax": 1100.1012646484375, "ymin": 523.3905639648438}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 838.8431096484375, "ymax": 645.6470336910062, "xmax": 953.1029833984375, "ymin": 553.999267578125}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 104.5491943359375, "ymax": 869.835205078125, "xmax": 747.3396606445312, "ymin": 423.1010102363281}, "is_hard": false}, {"type": "Car", "bndbox": {"xmin": 663.7487102610108, "ymax": 661.5325927734375, "xmax": 779.2273559570312, "ymin": 576.6761074609375}, "is_hard": false}, {"type": "Tram", "bndbox": {"xmin": 710.6841030664062, "ymax": 607.3101062695312, "xmax": 846.3032836910062, "ymin": 551.8972778320312}, "is_hard": false}], "model": "vehPer_v4"}} \ No newline at end of file diff --git a/tests/ut/python/__init__.py b/tests/ut/python/__init__.py new file mode 100644 index 00000000000..b15d59e4210 --- /dev/null +++ b/tests/ut/python/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +init vm impl +""" +from ...vm_impl import vm diff --git a/tests/ut/python/communication/__init__.py b/tests/ut/python/communication/__init__.py new file mode 100644 index 00000000000..393930c4ebc --- /dev/null +++ b/tests/ut/python/communication/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append("../../..") \ No newline at end of file diff --git a/tests/ut/python/communication/test_comm.py b/tests/ut/python/communication/test_comm.py new file mode 100644 index 00000000000..885c8fa9e34 --- /dev/null +++ b/tests/ut/python/communication/test_comm.py @@ -0,0 +1,155 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" test Communicate """ +import numpy as np +from mindspore.ops.operations.comm_ops import AllReduce, AllGather, _AlltoAll, ReduceOp +from mindspore.ops.operations.comm_ops import Broadcast +from mindspore.communication.management import HCCL_WORLD_COMM_GROUP, NCCL_WORLD_COMM_GROUP, GlobalComm, init +from mindspore.communication._comm_helper import Backend +from mindspore import Tensor +import mindspore.nn as nn +from mindspore.ops.operations import Split +from mindspore.common.api import _executor +from mindspore.nn import Dense +from mindspore.nn import ReLU +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Momentum +import mindspore.context as context +# pylint: disable=W0212 +# W0212: protected-access + +tag = 0 + +init("hccl") + +class AllReduceNet(nn.Cell): + """AllReduceNet definition""" + def __init__(self, input_channel, out_channel, op): + super(AllReduceNet, self).__init__() + self.dense = Dense(input_channel, out_channel) + self.reduce = AllReduce(op) + self.relu = ReLU() + + def construct(self, x): + x = self.dense(x) + x = self.reduce(x) + return self.relu(x) + +class BroadCastNet(nn.Cell): + """BroadCastNet definition""" + def __init__(self, input_channel, out_channel): + super(BroadCastNet, self).__init__() + self.dense = Dense(input_channel, out_channel) + self.broadcast = Broadcast(0) + + def construct(self, x): + x, = self.broadcast((x,)) + x = self.dense(x) + return x + +class AllGatherNet(nn.Cell): + """AllGatherNet definition""" + def __init__(self, input_channel, out_channel): + super(AllGatherNet, self).__init__() + self.dense = Dense(input_channel, out_channel) + if GlobalComm.BACKEND is Backend.HCCL: + self.allgather = AllGather(group=HCCL_WORLD_COMM_GROUP) + elif GlobalComm.BACKEND is Backend.NCCL: + self.allgather = AllGather(group=NCCL_WORLD_COMM_GROUP) + else: + self.allgather = AllGather() + + self.relu = ReLU() + + def construct(self, x): + x = self.dense(x) + x = self.allgather(x) + return self.relu(x) + +class AlltoAllNet(nn.Cell): + """AlltoAllNet definition""" + def __init__(self, input_channel, out_channel): + super(AlltoAllNet, self).__init__() + self.dense = Dense(input_channel, out_channel) + self.alltoall = _AlltoAll(1, 0, 1) + self.relu = ReLU() + + def construct(self, x): + x = self.dense(x) + x = self.alltoall(x) + return self.relu(x) + +def run_allreduce(op): + """run_allreduce""" + context.set_context(mode=context.GRAPH_MODE) + input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + label_tensor = Tensor(np.array([[1.2], [2.2]], dtype=np.float32)) + network = AllReduceNet(2, 1, op) + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), + learning_rate=0.1, + momentum=0.9) + network = WithLossCell(network, loss_fn) + network = TrainOneStepCell(network, optimizer) + _executor.compile(network, input_tensor, label_tensor) + +def test_allreduce(): + """test_allreduce""" + context.set_context(mode=context.GRAPH_MODE) + run_allreduce(ReduceOp.SUM) + run_allreduce(ReduceOp.MAX) + run_allreduce(ReduceOp.MIN) + +def test_allgather(): + """test_allgather""" + context.set_context(mode=context.GRAPH_MODE) + input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + label_tensor = Tensor(np.array([[1.2], [2.2]], dtype=np.float32)) + network = AllGatherNet(2, 1) + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), + learning_rate=0.1, + momentum=0.9) + network = WithLossCell(network, loss_fn) + network = TrainOneStepCell(network, optimizer) + _executor.compile(network, input_tensor, label_tensor) + +def test_broadcast(): + """test_broadcast""" + context.set_context(mode=context.GRAPH_MODE) + input_tensor_1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + label_tensor = Tensor(np.array([[1.2], [2.2]], dtype=np.float32)) + network = BroadCastNet(2, 1) + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), + learning_rate=0.1, + momentum=0.9) + network = WithLossCell(network, loss_fn) + network = TrainOneStepCell(network, optimizer) + _executor.compile(network, input_tensor_1, label_tensor) + +def test_alltoall(): + """test_alltoall""" + context.set_context(mode=context.GRAPH_MODE) + input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + label_tensor = Tensor(np.array([[1.2], [2.2]], dtype=np.float32)) + network = AlltoAllNet(2, 1) + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), + learning_rate=0.1, + momentum=0.9) + network = WithLossCell(network, loss_fn) + network = TrainOneStepCell(network, optimizer) + _executor.compile(network, input_tensor, label_tensor) diff --git a/tests/ut/python/communication/test_data_parallel_dense.py b/tests/ut/python/communication/test_data_parallel_dense.py new file mode 100644 index 00000000000..2e9553ee469 --- /dev/null +++ b/tests/ut/python/communication/test_data_parallel_dense.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +@File : test_data_parallel_dense.py +@Desc : test data parallel dense +""" +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Momentum +from mindspore.train.parallel_utils import ParallelMode +import mindspore.context as context + + +class DenseMMNet(nn.Cell): + """DenseMMNet definition""" + def __init__(self): + super(DenseMMNet, self).__init__() + self.fc1 = nn.Dense(128, 768, activation='relu') + self.fc2 = nn.Dense(128, 768, activation='relu') + self.fc3 = nn.Dense(128, 768, activation='relu') + self.fc4 = nn.Dense(768, 768, activation='relu') + self.relu4 = nn.ReLU() + self.relu5 = nn.ReLU() + self.transpose = P.Transpose() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + + def construct(self, x): + q = self.fc1(x) + k = self.fc2(x) + v = self.fc3(x) + k = self.transpose(k, (1, 0)) + c = self.relu4(self.matmul1(q, k)) + s = self.relu5(self.matmul2(c, v)) + s = self.fc4(s) + return s + + +def test_data_parallel_dense(): + """test_data_parallel_dense""" + context.set_context(mode=context.GRAPH_MODE) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, mirror_mean=True, device_num=8) + inp = Tensor(np.ones([32, 128]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([32, 768]).astype(np.float32)) + net = DenseMMNet() + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), + learning_rate=0.1, + momentum=0.9) + net = WithLossCell(net, loss_fn) + net = TrainOneStepCell(net, optimizer) + + _executor.compile(net, inp, label) + context.reset_auto_parallel_context() diff --git a/tests/ut/python/communication/test_data_parallel_lenet.py b/tests/ut/python/communication/test_data_parallel_lenet.py new file mode 100755 index 00000000000..658995ae3ee --- /dev/null +++ b/tests/ut/python/communication/test_data_parallel_lenet.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +@File : test_data_parallel_lenet.py +@Desc : test data parallel lenet +""" +import os +import numpy as np + +import mindspore.nn as nn +import mindspore.context as context +from mindspore.ops import operations as P +from mindspore import Tensor, Model, ParallelMode +from mindspore.nn.optim import Momentum + +_current_dir = os.path.dirname(os.path.realpath(__file__)) + "/../test_data" + +class LeNet5(nn.Cell): + """LeNet5 definition""" + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class DatasetLenet(): + """DatasetLenet definition""" + def __init__(self, predict, label, length=3): + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +def test_lenet5_train_step_training_pynative(): + """test_lenet5_train_step_training_pynative""" + context.set_context(mode=context.PYNATIVE_MODE) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, + device_num=8, mirror_mean=True) + size = 3 + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + DatasetLenet(predict, label, 2) + network = LeNet5() + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(network.get_parameters(), learning_rate=0.1, momentum=0.9) + Model(network=network, loss_fn=loss_fn, optimizer=optimizer) + context.set_context(mode=context.GRAPH_MODE) + context.reset_auto_parallel_context() diff --git a/tests/ut/python/communication/test_data_parallel_resnet.py b/tests/ut/python/communication/test_data_parallel_resnet.py new file mode 100644 index 00000000000..037152a0b7a --- /dev/null +++ b/tests/ut/python/communication/test_data_parallel_resnet.py @@ -0,0 +1,307 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +resnet50 example +""" +import numpy as np + +from mindspore import Tensor, Model, ParallelMode +from mindspore.ops.operations import TensorAdd +import mindspore.nn as nn +import mindspore.context as context +from mindspore.nn.optim import Momentum +from ....dataset_mock import MindData + + +def conv3x3(in_channels, out_channels, stride=1, padding=1, pad_mode='pad'): + """3x3 convolution """ + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, pad_mode=pad_mode) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0, pad_mode='pad'): + """1x1 convolution""" + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode) + + +class ResidualBlock(nn.Cell): + """ + residual Block + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = nn.BatchNorm2d(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=1) + self.bn2 = nn.BatchNorm2d(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.relu = nn.ReLU() + self.downsample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, + stride=stride, padding=0) + self.bn_down_sample = nn.BatchNorm2d(out_channels) + self.add = TensorAdd() + + def construct(self, x): + """ + :param x: + :return: + """ + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet18(nn.Cell): + """ + resnet nn.Cell + """ + + def __init__(self, block, num_classes=100): + super(ResNet18, self).__init__() + + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, pad_mode='pad') + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, pad_mode='pad') + + self.layer1 = self.MakeLayer( + block, 2, in_channels=64, out_channels=256, stride=1) + self.layer2 = self.MakeLayer( + block, 2, in_channels=256, out_channels=512, stride=2) + self.layer3 = self.MakeLayer( + block, 2, in_channels=512, out_channels=1024, stride=2) + self.layer4 = self.MakeLayer( + block, 2, in_channels=1024, out_channels=2048, stride=2) + + self.avgpool = nn.AvgPool2d(7, 1) + self.flatten = nn.Flatten() + self.fc = nn.Dense(512 * block.expansion, num_classes) + + def MakeLayer(self, block, layer_num, in_channels, out_channels, stride): + """ + make block layer + :param block: + :param layer_num: + :param in_channels: + :param out_channels: + :param stride: + :return: + """ + layers = [] + resblk = block(in_channels, out_channels, + stride=stride, down_sample=True) + layers.append(resblk) + + for _ in range(1, layer_num): + resblk = block(out_channels, out_channels, stride=1) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + """ + :param x: + :return: + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.flatten(x) + x = self.fc(x) + + return x + + +class ResNet9(nn.Cell): + """ + resnet nn.Cell + """ + + def __init__(self, block, num_classes=100): + super(ResNet9, self).__init__() + + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, pad_mode='pad') + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, pad_mode='same') + + self.layer1 = self.MakeLayer( + block, 1, in_channels=64, out_channels=256, stride=1) + self.layer2 = self.MakeLayer( + block, 1, in_channels=256, out_channels=512, stride=2) + self.layer3 = self.MakeLayer( + block, 1, in_channels=512, out_channels=1024, stride=2) + self.layer4 = self.MakeLayer( + block, 1, in_channels=1024, out_channels=2048, stride=2) + + self.avgpool = nn.AvgPool2d(7, 1) + self.flatten = nn.Flatten() + self.fc = nn.Dense(512 * block.expansion, num_classes) + + def MakeLayer(self, block, layer_num, in_channels, out_channels, stride): + """ + make block layer + :param block: + :param layer_num: + :param in_channels: + :param out_channels: + :param stride: + :return: + """ + layers = [] + resblk = block(in_channels, out_channels, + stride=stride, down_sample=True) + layers.append(resblk) + + for _ in range(1, layer_num): + resblk = block(out_channels, out_channels, stride=1) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + """ + :param x: + :return: + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.flatten(x) + x = self.fc(x) + + return x + + +def resnet9(classnum): + return ResNet9(ResidualBlock, classnum) + + +class DatasetLenet(MindData): + """DatasetLenet definition""" + + def __init__(self, predict, label, length=3, size=None, batch_size=None, + np_types=None, output_shapes=None, input_indexs=()): + super(DatasetLenet, self).__init__(size=size, batch_size=batch_size, + np_types=np_types, output_shapes=output_shapes, + input_indexs=input_indexs) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +def test_resnet_train_tensor(): + """test_resnet_train_tensor""" + batch_size = 1 + size = 2 + context.set_context(mode=context.GRAPH_MODE) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, device_num=size, + parameter_broadcast=True) + one_hot_len = 10 + dataset_types = (np.float32, np.float32) + dataset_shapes = [[batch_size, 3, 224, 224], [batch_size, one_hot_len]] + predict = Tensor(np.ones([batch_size, 3, 224, 224]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([batch_size, one_hot_len]).astype(np.float32)) + dataset = DatasetLenet(predict, label, 2, + size=2, batch_size=2, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + dataset.reset() + network = resnet9(one_hot_len) + network.set_train() + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), learning_rate=0.1, momentum=0.9) + model = Model(network=network, loss_fn=loss_fn, optimizer=optimizer) + model.train(epoch=2, train_dataset=dataset, dataset_sink_mode=False) + context.set_context(mode=context.GRAPH_MODE) + context.reset_auto_parallel_context() + + +class_num = 10 + + +def get_dataset(): + dataset_types = (np.float32, np.float32) + dataset_shapes = ((32, 3, 224, 224), (32, class_num)) + + dataset = MindData(size=2, batch_size=1, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset diff --git a/tests/ut/python/communication/test_management_api.py b/tests/ut/python/communication/test_management_api.py new file mode 100644 index 00000000000..c455c5491b9 --- /dev/null +++ b/tests/ut/python/communication/test_management_api.py @@ -0,0 +1,125 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +management api +""" +import mindspore.communication.management as D + +def has_raise_error(func, x): + try: + # pylint:disable=eval-used + if x is None: + func() + else: + func(x) + print("x:{}".format(x)) + except (TypeError, ValueError, RuntimeError): + return True + else: + return False + +def create_backend(name): + D.Backend(name) + +def get_group_size_int(group): + D.get_group_size(group) + +def create_group0(x): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.create_group('0-1', x) + +def create_group1(x): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.create_group('0-1', x) + +def create_group2(x): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.create_group('0-1', x) + +def create_group3(x): + D.GlobalComm.BACKEND = D.Backend.UNDEFINED + D.create_group('0-1', x) + +def create_group4(x): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.create_group('0-1', x) + +def get_world_rank_from_group_rank0(): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.get_world_rank_from_group_rank(D.HCCL_WORLD_COMM_GROUP, 0) + +def get_world_rank_from_group_rank1(): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.get_world_rank_from_group_rank('0-1', '0') + +def get_world_rank_from_group_rank2(): + D.GlobalComm.BACKEND = D.Backend.UNDEFINED + D.get_world_rank_from_group_rank('0-1', 0) + +def get_group_rank_from_world_rank0(): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.get_group_rank_from_world_rank(0, D.HCCL_WORLD_COMM_GROUP) + +def get_group_rank_from_world_rank1(): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.get_group_rank_from_world_rank('0', '0-1') + +def get_group_rank_from_world_rank2(): + D.GlobalComm.BACKEND = D.Backend.UNDEFINED + D.get_group_rank_from_world_rank(0, '0-1') + +def destroy_group0(x): + D.GlobalComm.BACKEND = D.Backend.UNDEFINED + D.destroy_group(x) + +def destroy_group1(): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.destroy_group(D.HCCL_WORLD_COMM_GROUP) + +def destroy_group2(x): + D.GlobalComm.BACKEND = D.Backend.HCCL + D.destroy_group(x) + +def test_raise_error_funcs(): + """test raise error funcs""" + assert has_raise_error(create_backend, 123) is True + assert has_raise_error(create_backend, 'hccl') is False + assert has_raise_error(create_backend, 'nccl') is False + assert has_raise_error(get_group_size_int, 123) is True + assert has_raise_error(create_group0, (0,1)) is True + assert has_raise_error(create_group1, [0]) is True + assert has_raise_error(create_group2, [0,0,1]) is True + assert has_raise_error(create_group3, [0,1]) is True + assert has_raise_error(create_group4, [0,1]) is False + assert has_raise_error(get_world_rank_from_group_rank0, None) is True + assert has_raise_error(get_world_rank_from_group_rank1, None) is True + assert has_raise_error(get_world_rank_from_group_rank2, None) is True + assert has_raise_error(get_group_rank_from_world_rank0, None) is True + assert has_raise_error(get_group_rank_from_world_rank1, None) is True + assert has_raise_error(get_group_rank_from_world_rank2, None) is True + assert has_raise_error(destroy_group0, '0-1') is True + assert has_raise_error(destroy_group1, None) is True + assert has_raise_error(destroy_group2, '0-1') is False + +def test_get_rank_none(): + assert D.get_rank(group=None) == 0 + +def test_group_funs(): + D.GlobalComm.BACKEND = D.Backend.HCCL + assert D.get_group_size(group=None) == 1 + assert D.get_group_size('2-abcd') == 2 + assert D.get_world_rank_from_group_rank('0-1', 0) == 0 + assert D.get_group_rank_from_world_rank(0, '0-1') == 0 + diff --git a/tests/ut/python/conftest.py b/tests/ut/python/conftest.py new file mode 100644 index 00000000000..ff49460f9f1 --- /dev/null +++ b/tests/ut/python/conftest.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : conftest.py +@Desc : common fixtures for pytest +""" + +import pytest +from _pytest.runner import runtestprotocol + +def pytest_addoption(parser): + """ + add runmode option to control running testcase + """ + parser.addoption( + "--runmode", action="store", default="nosimu", + help="simu:simulator backend & nosimu for no backend" + ) + + +@pytest.fixture +def test_with_simu(request): + """ + run PyNative testcases when compiled with simulator + """ + return request.config.getoption("--runmode") == "simu" + +# https://stackoverflow.com/questions/14121657/how-to-get-test-name-and-test-result-during-run-time-in-pytest +def pytest_runtest_protocol(item, nextitem): + reports = runtestprotocol(item, nextitem=nextitem) + for report in reports: + if report.when == 'call': + print(f"\n{item.name} --- {report.outcome}") + return True diff --git a/tests/ut/python/dataset/README.md b/tests/ut/python/dataset/README.md new file mode 100644 index 00000000000..582d344b592 --- /dev/null +++ b/tests/ut/python/dataset/README.md @@ -0,0 +1,7 @@ +## DE Dataset Ops Test (Python) + +#### Environment +Python3.7 (PyCharm IDE is recommended) + +### Introduce of data +All of data used in this folder created by Huawei Technologies Co. diff --git a/tests/ut/python/dataset/prep_data.py b/tests/ut/python/dataset/prep_data.py new file mode 100644 index 00000000000..a63cb397fb3 --- /dev/null +++ b/tests/ut/python/dataset/prep_data.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import jsbeautifier + +import os +import urllib +import urllib.request + + +def create_data_cache_dir(): + cwd = os.getcwd() + target_directory = os.path.join(cwd, "data_cache") + try: + if not (os.path.exists(target_directory)): + os.mkdir(target_directory) + except OSError: + print("Creation of the directory %s failed" % target_directory) + return target_directory; + + +def download_and_uncompress(files, source_url, target_directory, is_tar=False): + for f in files: + url = source_url + f + target_file = os.path.join(target_directory, f) + + ##check if file already downloaded + if not (os.path.exists(target_file) or os.path.exists(target_file[:-3])): + urllib.request.urlretrieve(url, target_file) + if is_tar: + print("extracting from local tar file " + target_file) + rc = os.system("tar -C " + target_directory + " -xvf " + target_file) + else: + print("unzipping " + target_file) + rc = os.system("gunzip -f " + target_file) + if rc != 0: + print("Failed to uncompress ", target_file, " removing") + os.system("rm " + target_file) + ##exit with error so that build script will fail + raise SystemError + else: + print("Using cached dataset at ", target_file) + + +def download_mnist(target_directory=None): + if target_directory == None: + target_directory = create_data_cache_dir() + + ##create mnst directory + target_directory = os.path.join(target_directory, "mnist") + try: + if not (os.path.exists(target_directory)): + os.mkdir(target_directory) + except OSError: + print("Creation of the directory %s failed" % target_directory) + + MNIST_URL = "http://yann.lecun.com/exdb/mnist/" + files = ['train-images-idx3-ubyte.gz', + 'train-labels-idx1-ubyte.gz', + 't10k-images-idx3-ubyte.gz', + 't10k-labels-idx1-ubyte.gz'] + download_and_uncompress(files, MNIST_URL, target_directory, is_tar=False) + + return target_directory, os.path.join(target_directory, "datasetSchema.json") + + +CIFAR_URL = "https://www.cs.toronto.edu/~kriz/" + + +def download_cifar(target_directory, files, directory_from_tar): + if target_directory == None: + target_directory = create_data_cache_dir() + + download_and_uncompress([files], CIFAR_URL, target_directory, is_tar=True) + + ##if target dir was specify move data from directory created by tar + ##and put data into target dir + if target_directory != None: + tar_dir_full_path = os.path.join(target_directory, directory_from_tar) + all_files = os.path.join(tar_dir_full_path, "*") + cmd = "mv " + all_files + " " + target_directory + if os.path.exists(tar_dir_full_path): + print("copy files back to target_directory") + print("Executing: ", cmd) + rc1 = os.system(cmd) + rc2 = os.system("rm -r " + tar_dir_full_path) + if rc1 != 0 or rc2 != 0: + print("error when running command: ", cmd) + download_file = os.path.join(target_directory, files) + print("removing " + download_file) + os.system("rm " + download_file) + + ##exit with error so that build script will fail + raise SystemError + + ##change target directory to directory after tar + return os.path.join(target_directory, directory_from_tar) + + +def download_cifar10(target_directory=None): + return download_cifar(target_directory, "cifar-10-binary.tar.gz", "cifar-10-batches-bin") + + +def download_cifar100(target_directory=None): + return download_cifar(target_directory, "cifar-100-binary.tar.gz", "cifar-100-binary") + + +def download_all_for_test(cwd): + download_mnist(os.path.join(cwd, "testMnistData")) + + +##Download all datasets to existing test directories +if __name__ == "__main__": + download_all_for_test(os.getcwd()) diff --git a/tests/ut/python/dataset/test_2ops.py b/tests/ut/python/dataset/test_2ops.py new file mode 100644 index 00000000000..a0b3f160fca --- /dev/null +++ b/tests/ut/python/dataset/test_2ops.py @@ -0,0 +1,157 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from util import save_and_check + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] +SCHEMA_DIR = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", + "col_sint16", "col_sint32", "col_sint64"] +GENERATE_GOLDEN = False + + +def skip_test_case_0(): + """ + Test Repeat then Shuffle + """ + logger.info("Test Repeat then Shuffle") + # define parameters + repeat_count = 2 + buffer_size = 5 + seed = 0 + parameters = {"params": {'repeat_count': repeat_count, + 'buffer_size': buffer_size, + 'seed': seed}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.repeat(repeat_count) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + filename = "test_case_0_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def skip_test_case_0_reverse(): + """ + Test Shuffle then Repeat + """ + logger.info("Test Shuffle then Repeat") + # define parameters + repeat_count = 2 + buffer_size = 5 + seed = 0 + parameters = {"params": {'repeat_count': repeat_count, + 'buffer_size': buffer_size, + 'reshuffle_each_iteration': False, + 'seed': seed}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + data1 = data1.repeat(repeat_count) + + filename = "test_case_0_reverse_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_1(): + """ + Test Repeat then Batch + """ + logger.info("Test Repeat then Batch") + # define parameters + repeat_count = 2 + batch_size = 5 + parameters = {"params": {'repeat_count': repeat_count, + 'batch_size': batch_size}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.repeat(repeat_count) + data1 = data1.batch(batch_size, drop_remainder=True) + + filename = "test_case_1_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_1_reverse(): + """ + Test Batch then Repeat + """ + logger.info("Test Batch then Repeat") + # define parameters + repeat_count = 2 + batch_size = 5 + parameters = {"params": {'repeat_count': repeat_count, + 'batch_size': batch_size}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.batch(batch_size, drop_remainder=True) + data1 = data1.repeat(repeat_count) + + filename = "test_case_1_reverse_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_2(): + """ + Test Batch then Shuffle + """ + logger.info("Test Batch then Shuffle") + # define parameters + buffer_size = 5 + seed = 0 + batch_size = 2 + parameters = {"params": {'buffer_size': buffer_size, + 'seed': seed, + 'batch_size': batch_size}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.batch(batch_size, drop_remainder=True) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + filename = "test_case_2_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_2_reverse(): + """ + Test Shuffle then Batch + """ + logger.info("Test Shuffle then Batch") + # define parameters + buffer_size = 5 + seed = 0 + batch_size = 2 + parameters = {"params": {'buffer_size': buffer_size, + 'seed': seed, + 'batch_size': batch_size}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + data1 = data1.batch(batch_size, drop_remainder=True) + + filename = "test_case_2_reverse_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_Tensor.py b/tests/ut/python/dataset/test_Tensor.py new file mode 100644 index 00000000000..462020c90f9 --- /dev/null +++ b/tests/ut/python/dataset/test_Tensor.py @@ -0,0 +1,59 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore._c_dataengine as cde + +import numpy as np + + +def test_shape(): + x = [2, 3] + s = cde.TensorShape(x) + assert s.as_list() == x + assert s.is_known() + + +def test_basic(): + x = np.array([1, 2, 3, 4, 5]) + n = cde.Tensor(x) + arr = np.array(n, copy=False) + arr[0] = 0 + x = np.array([0, 2, 3, 4, 5]) + + assert np.array_equal(x, arr) + assert n.type() == cde.DataType("int64") + + arr2 = n.as_array() + arr[0] = 2 + x = np.array([2, 2, 3, 4, 5]) + assert np.array_equal(x, arr2) + assert n.type() == cde.DataType("int64") + assert arr.__array_interface__['data'] == arr2.__array_interface__['data'] + +def test_strides(): + x = np.array([[1, 2, 3], [4, 5, 6]]) + n1 = cde.Tensor(x[:, 1]) + arr = np.array(n1, copy=False) + + assert np.array_equal(x[:, 1], arr) + + n2 = cde.Tensor(x.transpose()) + arr = np.array(n2, copy=False) + + assert np.array_equal(x.transpose(), arr) + +if __name__ == '__main__': + test_shape() + test_strides() + test_basic() diff --git a/tests/ut/python/dataset/test_batch.py b/tests/ut/python/dataset/test_batch.py new file mode 100644 index 00000000000..1ab4c10b367 --- /dev/null +++ b/tests/ut/python/dataset/test_batch.py @@ -0,0 +1,489 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from util import save_and_check + +import mindspore.dataset as ds +from mindspore import log as logger + +# Note: Number of rows in test.data dataset: 12 +DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] +GENERATE_GOLDEN = False + + +def test_batch_01(): + """ + Test batch: batch_size>1, drop_remainder=True, no remainder exists + """ + logger.info("test_batch_01") + # define parameters + batch_size = 2 + drop_remainder = True + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size, drop_remainder) + + filename = "batch_01_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_02(): + """ + Test batch: batch_size>1, drop_remainder=True, remainder exists + """ + logger.info("test_batch_02") + # define parameters + batch_size = 5 + drop_remainder = True + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size, drop_remainder=drop_remainder) + + filename = "batch_02_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_03(): + """ + Test batch: batch_size>1, drop_remainder=False, no remainder exists + """ + logger.info("test_batch_03") + # define parameters + batch_size = 3 + drop_remainder = False + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size=batch_size, drop_remainder=drop_remainder) + + filename = "batch_03_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_04(): + """ + Test batch: batch_size>1, drop_remainder=False, remainder exists + """ + logger.info("test_batch_04") + # define parameters + batch_size = 7 + drop_remainder = False + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size, drop_remainder) + + filename = "batch_04_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_05(): + """ + Test batch: batch_size=1 (minimum valid size), drop_remainder default + """ + logger.info("test_batch_05") + # define parameters + batch_size = 1 + parameters = {"params": {'batch_size': batch_size}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size) + + filename = "batch_05_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_06(): + """ + Test batch: batch_size = number-of-rows-in-dataset, drop_remainder=True, reorder params + """ + logger.info("test_batch_06") + # define parameters + batch_size = 12 + drop_remainder = False + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(drop_remainder=drop_remainder, batch_size=batch_size) + + filename = "batch_06_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_07(): + """ + Test batch: num_parallel_workers>1, drop_remainder=False, reorder params + """ + logger.info("test_batch_07") + # define parameters + batch_size = 4 + drop_remainder = False + num_parallel_workers = 2 + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder, + 'num_parallel_workers': num_parallel_workers}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(num_parallel_workers=num_parallel_workers, drop_remainder=drop_remainder, + batch_size=batch_size) + + filename = "batch_07_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_08(): + """ + Test batch: num_parallel_workers=1, drop_remainder default + """ + logger.info("test_batch_08") + # define parameters + batch_size = 6 + num_parallel_workers = 1 + parameters = {"params": {'batch_size': batch_size, + 'num_parallel_workers': num_parallel_workers}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size, num_parallel_workers=num_parallel_workers) + + filename = "batch_08_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_09(): + """ + Test batch: batch_size > number-of-rows-in-dataset, drop_remainder=False + """ + logger.info("test_batch_09") + # define parameters + batch_size = 13 + drop_remainder = False + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size, drop_remainder=drop_remainder) + + filename = "batch_09_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_10(): + """ + Test batch: batch_size > number-of-rows-in-dataset, drop_remainder=True + """ + logger.info("test_batch_10") + # define parameters + batch_size = 99 + drop_remainder = True + parameters = {"params": {'batch_size': batch_size, + 'drop_remainder': drop_remainder}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + data1 = data1.batch(batch_size, drop_remainder=drop_remainder) + + filename = "batch_10_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_11(): + """ + Test batch: batch_size=1 and dataset-size=1 + """ + logger.info("test_batch_11") + # define parameters + batch_size = 1 + parameters = {"params": {'batch_size': batch_size}} + + # apply dataset operations + # Use schema file with 1 row + schema_file = "../data/dataset/testTFTestAllTypes/datasetSchema1Row.json" + data1 = ds.TFRecordDataset(DATA_DIR, schema_file) + data1 = data1.batch(batch_size) + + filename = "batch_11_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_batch_exception_01(): + """ + Test batch exception: num_parallel_workers=0 + """ + logger.info("test_batch_exception_01") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(batch_size=2, drop_remainder=True, num_parallel_workers=0) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "num_parallel_workers" in str(e) + + +def test_batch_exception_02(): + """ + Test batch exception: num_parallel_workers<0 + """ + logger.info("test_batch_exception_02") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(3, drop_remainder=True, num_parallel_workers=-1) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "num_parallel_workers" in str(e) + + +def test_batch_exception_03(): + """ + Test batch exception: batch_size=0 + """ + logger.info("test_batch_exception_03") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(batch_size=0) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "batch_size" in str(e) + + +def test_batch_exception_04(): + """ + Test batch exception: batch_size<0 + """ + logger.info("test_batch_exception_04") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(batch_size=-1) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "batch_size" in str(e) + + +def test_batch_exception_05(): + """ + Test batch exception: batch_size wrong type, boolean value False + """ + logger.info("test_batch_exception_05") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(batch_size=False) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "batch_size" in str(e) + + +def skip_test_batch_exception_06(): + """ + Test batch exception: batch_size wrong type, boolean value True + """ + logger.info("test_batch_exception_06") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(batch_size=True) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "batch_size" in str(e) + + +def test_batch_exception_07(): + """ + Test batch exception: drop_remainder wrong type + """ + logger.info("test_batch_exception_07") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(3, drop_remainder=0) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "drop_remainder" in str(e) + + +def test_batch_exception_08(): + """ + Test batch exception: num_parallel_workers wrong type + """ + logger.info("test_batch_exception_08") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(3, drop_remainder=True, num_parallel_workers=False) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "num_parallel_workers" in str(e) + + +def test_batch_exception_09(): + """ + Test batch exception: Missing mandatory batch_size + """ + logger.info("test_batch_exception_09") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(drop_remainder=True, num_parallel_workers=4) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "batch_size" in str(e) + + +def test_batch_exception_10(): + """ + Test batch exception: num_parallel_workers>>1 + """ + logger.info("test_batch_exception_10") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + try: + data1 = data1.batch(batch_size=4, num_parallel_workers=8192) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "num_parallel_workers" in str(e) + + +def test_batch_exception_11(): + """ + Test batch exception: wrong input order, num_parallel_workers wrongly used as drop_remainder + """ + logger.info("test_batch_exception_11") + # define parameters + batch_size = 6 + num_parallel_workers = 1 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + try: + data1 = data1.batch(batch_size, num_parallel_workers) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "drop_remainder" in str(e) + + +def test_batch_exception_12(): + """ + Test batch exception: wrong input order, drop_remainder wrongly used as batch_size + """ + logger.info("test_batch_exception_12") + # define parameters + batch_size = 1 + drop_remainder = True + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + try: + data1 = data1.batch(drop_remainder, batch_size=batch_size) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "batch_size" in str(e) + + +def test_batch_exception_13(): + """ + Test batch exception: invalid input parameter + """ + logger.info("test_batch_exception_13") + # define parameters + batch_size = 4 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + try: + data1 = data1.batch(batch_size, shard_id=1) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "shard_id" in str(e) + + +if __name__ == '__main__': + test_batch_01() + test_batch_02() + test_batch_03() + test_batch_04() + test_batch_05() + test_batch_06() + test_batch_07() + test_batch_08() + test_batch_09() + test_batch_10() + test_batch_11() + test_batch_exception_01() + test_batch_exception_02() + test_batch_exception_03() + test_batch_exception_04() + test_batch_exception_05() + skip_test_batch_exception_06() + test_batch_exception_07() + test_batch_exception_08() + test_batch_exception_09() + test_batch_exception_10() + test_batch_exception_11() + test_batch_exception_12() + test_batch_exception_13() + logger.info('\n') diff --git a/tests/ut/python/dataset/test_center_crop.py b/tests/ut/python/dataset/test_center_crop.py new file mode 100644 index 00000000000..596c1e1c722 --- /dev/null +++ b/tests/ut/python/dataset/test_center_crop.py @@ -0,0 +1,73 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np +import matplotlib.pyplot as plt +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(image_original, image_cropped): + """ + visualizes the image using DE op and Numpy op + """ + num = len(image_cropped) + for i in range(num): + plt.subplot(2, num, i + 1) + plt.imshow(image_original[i]) + plt.title("Original image") + + plt.subplot(2, num, i + num + 1) + plt.imshow(image_cropped[i]) + plt.title("DE center_crop image") + + plt.show() + + +def test_center_crop_op(height=375, width=375, plot=False): + """ + Test random_vertical + """ + logger.info("Test CenterCrop") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + decode_op = vision.Decode() + # 3 images [375, 500] [600, 500] [512, 512] + center_crop_op = vision.CenterCrop(height, width) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=center_crop_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + image_cropped = [] + image = [] + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + image_cropped.append(item1["image"].copy()) + image.append(item2["image"].copy()) + if plot: + visualize(image, image_cropped) + + +if __name__ == "__main__": + test_center_crop_op() + test_center_crop_op(600, 600) + test_center_crop_op(300, 600) + test_center_crop_op(600, 300) diff --git a/tests/ut/python/dataset/test_cifarop.py b/tests/ut/python/dataset/test_cifarop.py new file mode 100644 index 00000000000..4eebe0d777f --- /dev/null +++ b/tests/ut/python/dataset/test_cifarop.py @@ -0,0 +1,61 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger + +# Data for CIFAR and MNIST are not part of build tree +# They need to be downloaded directly +# prep_data.py can be executed or code below +# import sys +# sys.path.insert(0,"../../data") +# import prep_data +# prep_data.download_all_for_test("../../data") +DATA_DIR_10 = "../data/dataset/testCifar10Data" +DATA_DIR_100 = "../data/dataset/testCifar100Data" + + +def test_case_dataset_cifar10(): + """ + dataset parameter + """ + logger.info("Test dataset parameter") + # apply dataset operations + data1 = ds.Cifar10Dataset(DATA_DIR_10, 100) + + num_iter = 0 + for item in data1.create_dict_iterator(): + # in this example, each dictionary has keys "image" and "label" + num_iter += 1 + assert (num_iter == 100) + + +def test_case_dataset_cifar100(): + """ + dataset parameter + """ + logger.info("Test dataset parameter") + # apply dataset operations + data1 = ds.Cifar100Dataset(DATA_DIR_100, 100) + + num_iter = 0 + for item in data1.create_dict_iterator(): + # in this example, each dictionary has keys "image" and "label" + num_iter += 1 + assert (num_iter == 100) + + +if __name__ == '__main__': + test_case_dataset_cifar10() + test_case_dataset_cifar100() diff --git a/tests/ut/python/dataset/test_config.py b/tests/ut/python/dataset/test_config.py new file mode 100644 index 00000000000..8cabe81aaa4 --- /dev/null +++ b/tests/ut/python/dataset/test_config.py @@ -0,0 +1,41 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds + + +def test_basic(): + ds.config.load('../data/dataset/declient.cfg') + + # assert ds.config.get_rows_per_buffer() == 32 + assert ds.config.get_num_parallel_workers() == 4 + # assert ds.config.get_worker_connector_size() == 16 + assert ds.config.get_prefetch_size() == 16 + assert ds.config.get_seed() == 5489 + + # ds.config.set_rows_per_buffer(1) + ds.config.set_num_parallel_workers(2) + # ds.config.set_worker_connector_size(3) + ds.config.set_prefetch_size(4) + ds.config.set_seed(5) + + # assert ds.config.get_rows_per_buffer() == 1 + assert ds.config.get_num_parallel_workers() == 2 + # assert ds.config.get_worker_connector_size() == 3 + assert ds.config.get_prefetch_size() == 4 + assert ds.config.get_seed() == 5 + + +if __name__ == '__main__': + test_basic() diff --git a/tests/ut/python/dataset/test_cut_out.py b/tests/ut/python/dataset/test_cut_out.py new file mode 100644 index 00000000000..3b3bf121906 --- /dev/null +++ b/tests/ut/python/dataset/test_cut_out.py @@ -0,0 +1,139 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing CutOut op in DE +""" +import matplotlib.pyplot as plt +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.c_transforms as c +import mindspore.dataset.transforms.vision.py_transforms as f +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(image_1, image_2): + """ + visualizes the image using RandomErasing and Cutout + """ + plt.subplot(141) + plt.imshow(image_1) + plt.title("RandomErasing") + + plt.subplot(142) + plt.imshow(image_2) + plt.title("Cutout") + + plt.subplot(143) + plt.imshow(image_1 - image_2) + plt.title("Difference image") + plt.show() + + +def test_cut_out_op(): + """ + Test Cutout + """ + logger.info("test_cut_out") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + + transforms_1 = [ + f.Decode(), + f.ToTensor(), + f.RandomErasing(value='random') + ] + transform_1 = f.ComposeOp(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + decode_op = c.Decode() + cut_out_op = c.CutOut(80) + + transforms_2 = [ + decode_op, + cut_out_op + ] + + data2 = data2.map(input_columns=["image"], operations=transforms_2) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + # C image doesn't require transpose + image_2 = item2["image"] + + logger.info("shape of image_1: {}".format(image_1.shape)) + logger.info("shape of image_2: {}".format(image_2.shape)) + + logger.info("dtype of image_1: {}".format(image_1.dtype)) + logger.info("dtype of image_2: {}".format(image_2.dtype)) + + # visualize(image_1, image_2) + + +def test_cut_out_op_multicut(): + """ + Test Cutout + """ + logger.info("test_cut_out") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + + transforms_1 = [ + f.Decode(), + f.ToTensor(), + f.RandomErasing(value='random') + ] + transform_1 = f.ComposeOp(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + decode_op = c.Decode() + cut_out_op = c.CutOut(80, num_patches=10) + + transforms_2 = [ + decode_op, + cut_out_op + ] + + data2 = data2.map(input_columns=["image"], operations=transforms_2) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + + num_iter += 1 + image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + # C image doesn't require transpose + image_2 = item2["image"] + + logger.info("shape of image_1: {}".format(image_1.shape)) + logger.info("shape of image_2: {}".format(image_2.shape)) + + logger.info("dtype of image_1: {}".format(image_1.dtype)) + logger.info("dtype of image_2: {}".format(image_2.dtype)) + + +if __name__ == "__main__": + test_cut_out_op() + test_cut_out_op_multicut() diff --git a/tests/ut/python/dataset/test_datasets_celeba.py b/tests/ut/python/dataset/test_datasets_celeba.py new file mode 100644 index 00000000000..6b8859f433f --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_celeba.py @@ -0,0 +1,93 @@ +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore.dataset.transforms.vision import Inter +from mindspore import log as logger + +DATA_DIR = "../data/dataset/testCelebAData/" + + +def test_celeba_dataset_label(): + data = ds.CelebADataset(DATA_DIR, decode=True) + expect_labels = [ + [0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, + 0, 0, 1], + [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 1]] + count = 0 + for item in data.create_dict_iterator(): + logger.info("----------image--------") + logger.info(item["image"]) + logger.info("----------attr--------") + logger.info(item["attr"]) + for index in range(len(expect_labels[count])): + assert (item["attr"][index] == expect_labels[count][index]) + count = count + 1 + assert (count == 2) + + +def test_celeba_dataset_op(): + data = ds.CelebADataset(DATA_DIR, decode=True, num_shards=1, shard_id=0) + crop_size = (80, 80) + resize_size = (24, 24) + # define map operations + data = data.repeat(2) + center_crop = vision.CenterCrop(crop_size) + resize_op = vision.Resize(resize_size, Inter.LINEAR) # Bilinear mode + data = data.map(input_columns=["image"], operations=center_crop) + data = data.map(input_columns=["image"], operations=resize_op) + + count = 0 + for item in data.create_dict_iterator(): + logger.info("----------image--------") + logger.info(item["image"]) + count = count + 1 + assert (count == 4) + + +def test_celeba_dataset_ext(): + ext = [".JPEG"] + data = ds.CelebADataset(DATA_DIR, decode=True, extensions=ext) + expect_labels = [0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, + 0, 1, 0, 1, 0, 0, 1], + count = 0 + for item in data.create_dict_iterator(): + logger.info("----------image--------") + logger.info(item["image"]) + logger.info("----------attr--------") + logger.info(item["attr"]) + for index in range(len(expect_labels[count])): + assert (item["attr"][index] == expect_labels[count][index]) + count = count + 1 + assert (count == 1) + + +def test_celeba_dataset_distribute(): + data = ds.CelebADataset(DATA_DIR, decode=True, num_shards=2, shard_id=0) + count = 0 + for item in data.create_dict_iterator(): + logger.info("----------image--------") + logger.info(item["image"]) + logger.info("----------attr--------") + logger.info(item["attr"]) + count = count + 1 + assert (count == 1) + + +if __name__ == '__main__': + test_celeba_dataset_label() + test_celeba_dataset_op() + test_celeba_dataset_ext() + test_celeba_dataset_distribute() diff --git a/tests/ut/python/dataset/test_datasets_get_dataset_size.py b/tests/ut/python/dataset/test_datasets_get_dataset_size.py new file mode 100644 index 00000000000..d1bef0f9641 --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_get_dataset_size.py @@ -0,0 +1,113 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import mindspore.dataset as ds + +IMAGENET_RAWDATA_DIR = "../data/dataset/testImageNetData2/train" +IMAGENET_TFFILE_DIR = ["../data/dataset/test_tf_file_3_images2/train-0000-of-0001.data", + "../data/dataset/test_tf_file_3_images2/train-0000-of-0002.data", + "../data/dataset/test_tf_file_3_images2/train-0000-of-0003.data", + "../data/dataset/test_tf_file_3_images2/train-0000-of-0004.data"] +MNIST_DATA_DIR = "../data/dataset/testMnistData" +MANIFEST_DATA_FILE = "../data/dataset/testManifestData/test.manifest" +CIFAR10_DATA_DIR = "../data/dataset/testCifar10Data" +CIFAR100_DATA_DIR = "../data/dataset/testCifar100Data" + + +def test_imagenet_rawdata_dataset_size(): + ds_total = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR) + assert ds_total.get_dataset_size() == 6 + + ds_shard_1_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0) + assert ds_shard_1_0.get_dataset_size() == 6 + + ds_shard_2_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0) + assert ds_shard_2_0.get_dataset_size() == 3 + + ds_shard_3_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0) + assert ds_shard_3_0.get_dataset_size() == 2 + + +def test_imagenet_tf_file_dataset_size(): + ds_total = ds.TFRecordDataset(IMAGENET_TFFILE_DIR) + assert ds_total.get_dataset_size() == 12 + + ds_shard_1_0 = ds.TFRecordDataset(IMAGENET_TFFILE_DIR, num_shards=1, shard_id=0) + assert ds_shard_1_0.get_dataset_size() == 12 + + ds_shard_2_0 = ds.TFRecordDataset(IMAGENET_TFFILE_DIR, num_shards=2, shard_id=0) + assert ds_shard_2_0.get_dataset_size() == 6 + + ds_shard_3_0 = ds.TFRecordDataset(IMAGENET_TFFILE_DIR, num_shards=3, shard_id=0) + assert ds_shard_3_0.get_dataset_size() == 4 + + +def test_mnist_dataset_size(): + ds_total = ds.MnistDataset(MNIST_DATA_DIR) + assert ds_total.get_dataset_size() == 10000 + + ds_shard_1_0 = ds.MnistDataset(MNIST_DATA_DIR, num_shards=1, shard_id=0) + assert ds_shard_1_0.get_dataset_size() == 10000 + + ds_shard_2_0 = ds.MnistDataset(MNIST_DATA_DIR, num_shards=2, shard_id=0) + assert ds_shard_2_0.get_dataset_size() == 5000 + + ds_shard_3_0 = ds.MnistDataset(MNIST_DATA_DIR, num_shards=3, shard_id=0) + assert ds_shard_3_0.get_dataset_size() == 3334 + + +def test_manifest_dataset_size(): + ds_total = ds.ManifestDataset(MANIFEST_DATA_FILE) + assert ds_total.get_dataset_size() == 4 + + ds_shard_1_0 = ds.ManifestDataset(MANIFEST_DATA_FILE, num_shards=1, shard_id=0) + assert ds_shard_1_0.get_dataset_size() == 4 + + ds_shard_2_0 = ds.ManifestDataset(MANIFEST_DATA_FILE, num_shards=2, shard_id=0) + assert ds_shard_2_0.get_dataset_size() == 2 + + ds_shard_3_0 = ds.ManifestDataset(MANIFEST_DATA_FILE, num_shards=3, shard_id=0) + assert ds_shard_3_0.get_dataset_size() == 2 + + +def test_cifar10_dataset_size(): + ds_total = ds.Cifar10Dataset(CIFAR10_DATA_DIR) + assert ds_total.get_dataset_size() == 10000 + + ds_shard_1_0 = ds.Cifar10Dataset(CIFAR10_DATA_DIR, num_shards=1, shard_id=0) + assert ds_shard_1_0.get_dataset_size() == 10000 + + ds_shard_2_0 = ds.Cifar10Dataset(CIFAR10_DATA_DIR, num_shards=2, shard_id=0) + assert ds_shard_2_0.get_dataset_size() == 5000 + + ds_shard_3_0 = ds.Cifar10Dataset(CIFAR10_DATA_DIR, num_shards=3, shard_id=0) + assert ds_shard_3_0.get_dataset_size() == 3334 + + ds_shard_7_0 = ds.Cifar10Dataset(CIFAR10_DATA_DIR, num_shards=7, shard_id=0) + assert ds_shard_7_0.get_dataset_size() == 1429 + + +def test_cifar100_dataset_size(): + ds_total = ds.Cifar100Dataset(CIFAR100_DATA_DIR) + assert ds_total.get_dataset_size() == 10000 + + ds_shard_1_0 = ds.Cifar100Dataset(CIFAR100_DATA_DIR, num_shards=1, shard_id=0) + assert ds_shard_1_0.get_dataset_size() == 10000 + + ds_shard_2_0 = ds.Cifar100Dataset(CIFAR100_DATA_DIR, num_shards=2, shard_id=0) + assert ds_shard_2_0.get_dataset_size() == 5000 + + ds_shard_3_0 = ds.Cifar100Dataset(CIFAR100_DATA_DIR, num_shards=3, shard_id=0) + assert ds_shard_3_0.get_dataset_size() == 3334 diff --git a/tests/ut/python/dataset/test_datasets_imagefolder.py b/tests/ut/python/dataset/test_datasets_imagefolder.py new file mode 100644 index 00000000000..3be7df91164 --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_imagefolder.py @@ -0,0 +1,478 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = "../data/dataset/testPK/data" + + +def test_imagefolder_basic(): + logger.info("Test Case basic") + # define parameters + repeat_count = 1 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 44) + + +def test_imagefolder_numsamples(): + logger.info("Test Case numSamples") + # define parameters + repeat_count = 1 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10, num_parallel_workers=2) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 10) + + +def test_imagefolder_numshards(): + logger.info("Test Case numShards") + # define parameters + repeat_count = 1 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 11) + + +def test_imagefolder_shardid(): + logger.info("Test Case withShardID") + # define parameters + repeat_count = 1 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=1) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 11) + + +def test_imagefolder_noshuffle(): + logger.info("Test Case noShuffle") + # define parameters + repeat_count = 1 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=False) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 44) + + +def test_imagefolder_extrashuffle(): + logger.info("Test Case extraShuffle") + # define parameters + repeat_count = 2 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=True) + data1 = data1.shuffle(buffer_size=5) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 88) + + +def test_imagefolder_classindex(): + logger.info("Test Case classIndex") + # define parameters + repeat_count = 1 + + # apply dataset operations + class_index = {"class3": 333, "class1": 111} + data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False) + data1 = data1.repeat(repeat_count) + + golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333] + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + assert (item["label"] == golden[num_iter]) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 22) + + +def test_imagefolder_negative_classindex(): + logger.info("Test Case negative classIndex") + # define parameters + repeat_count = 1 + + # apply dataset operations + class_index = {"class3": -333, "class1": 111} + data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False) + data1 = data1.repeat(repeat_count) + + golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, + -333, -333, -333, -333, -333, -333, -333, -333, -333, -333, -333] + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + assert (item["label"] == golden[num_iter]) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 22) + + +def test_imagefolder_extensions(): + logger.info("Test Case extensions") + # define parameters + repeat_count = 1 + + # apply dataset operations + ext = [".jpg", ".JPEG"] + data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 44) + + +def test_imagefolder_decode(): + logger.info("Test Case decode") + # define parameters + repeat_count = 1 + + # apply dataset operations + ext = [".jpg", ".JPEG"] + data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext, decode=True) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 44) + + +def test_sequential_sampler(): + logger.info("Test Case SequentialSampler") + + golden = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3] + + # define parameters + repeat_count = 1 + + # apply dataset operations + sampler = ds.SequentialSampler() + data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = data1.repeat(repeat_count) + + result = [] + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + result.append(item["label"]) + num_iter += 1 + + logger.info("Result: {}".format(result)) + assert (result == golden) + + +def test_random_sampler(): + logger.info("Test Case RandomSampler") + # define parameters + repeat_count = 1 + + # apply dataset operations + sampler = ds.RandomSampler() + data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 44) + + +def test_distributed_sampler(): + logger.info("Test Case DistributedSampler") + # define parameters + repeat_count = 1 + + # apply dataset operations + sampler = ds.DistributedSampler(10, 1) + data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 5) + + +def test_pk_sampler(): + logger.info("Test Case PKSampler") + # define parameters + repeat_count = 1 + + # apply dataset operations + sampler = ds.PKSampler(3) + data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 12) + + +def test_subset_random_sampler(): + logger.info("Test Case SubsetRandomSampler") + # define parameters + repeat_count = 1 + + # apply dataset operations + indices = [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11] + sampler = ds.SubsetRandomSampler(indices) + data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 12) + + +def test_weighted_random_sampler(): + logger.info("Test Case WeightedRandomSampler") + # define parameters + repeat_count = 1 + + # apply dataset operations + weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1] + sampler = ds.WeightedRandomSampler(weights, 11) + data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 11) + + +def test_imagefolder_rename(): + logger.info("Test Case rename") + # define parameters + repeat_count = 1 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 10) + + data1 = data1.rename(input_columns=["image"], output_columns="image2") + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image2"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 10) + + +def test_imagefolder_zip(): + logger.info("Test Case zip") + # define parameters + repeat_count = 2 + + # apply dataset operations + data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10) + data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10) + + data1 = data1.repeat(repeat_count) + # rename dataset2 for no conflict + data2 = data2.rename(input_columns=["image", "label"], output_columns=["image1", "label1"]) + data3 = ds.zip((data1, data2)) + + num_iter = 0 + for item in data3.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is {}".format(item["image"])) + logger.info("label is {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert (num_iter == 10) + + +if __name__ == '__main__': + test_imagefolder_basic() + logger.info('test_imagefolder_basic Ended.\n') + + test_imagefolder_numsamples() + logger.info('test_imagefolder_numsamples Ended.\n') + + test_sequential_sampler() + logger.info('test_sequential_sampler Ended.\n') + + test_random_sampler() + logger.info('test_random_sampler Ended.\n') + + test_distributed_sampler() + logger.info('test_distributed_sampler Ended.\n') + + test_pk_sampler() + logger.info('test_pk_sampler Ended.\n') + + test_subset_random_sampler() + logger.info('test_subset_random_sampler Ended.\n') + + test_weighted_random_sampler() + logger.info('test_weighted_random_sampler Ended.\n') + + test_imagefolder_numshards() + logger.info('test_imagefolder_numshards Ended.\n') + + test_imagefolder_shardid() + logger.info('test_imagefolder_shardid Ended.\n') + + test_imagefolder_noshuffle() + logger.info('test_imagefolder_noshuffle Ended.\n') + + test_imagefolder_extrashuffle() + logger.info('test_imagefolder_extrashuffle Ended.\n') + + test_imagefolder_classindex() + logger.info('test_imagefolder_classindex Ended.\n') + + test_imagefolder_negative_classindex() + logger.info('test_imagefolder_negative_classindex Ended.\n') + + test_imagefolder_extensions() + logger.info('test_imagefolder_extensions Ended.\n') + + test_imagefolder_decode() + logger.info('test_imagefolder_decode Ended.\n') + + test_imagefolder_rename() + logger.info('test_imagefolder_rename Ended.\n') + + test_imagefolder_zip() + logger.info('test_imagefolder_zip Ended.\n') diff --git a/tests/ut/python/dataset/test_datasets_imagenet.py b/tests/ut/python/dataset/test_datasets_imagenet.py new file mode 100644 index 00000000000..27a67c5880d --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_imagenet.py @@ -0,0 +1,208 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.transforms.c_transforms as data_trans +import pytest + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def test_case_repeat(): + """ + a simple repeat operation. + """ + logger.info("Test Simple Repeat") + # define parameters + repeat_count = 2 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is: {}".format(item["image"])) + logger.info("label is: {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + + +def test_case_shuffle(): + """ + a simple shuffle operation. + """ + logger.info("Test Simple Shuffle") + # define parameters + buffer_size = 8 + seed = 10 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + for item in data1.create_dict_iterator(): + logger.info("image is: {}".format(item["image"])) + logger.info("label is: {}".format(item["label"])) + + +def test_case_0(): + """ + Test Repeat then Shuffle + """ + logger.info("Test Repeat then Shuffle") + # define parameters + repeat_count = 2 + buffer_size = 7 + seed = 9 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.repeat(repeat_count) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is: {}".format(item["image"])) + logger.info("label is: {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + + +def test_case_0_reverse(): + """ + Test Shuffle then Repeat + """ + logger.info("Test Shuffle then Repeat") + # define parameters + repeat_count = 2 + buffer_size = 10 + seed = 9 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + data1 = data1.repeat(repeat_count) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is: {}".format(item["image"])) + logger.info("label is: {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + + +def test_case_3(): + """ + Test Map + """ + logger.info("Test Map Rescale and Resize, then Shuffle") + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + # define data augmentation parameters + rescale = 1.0 / 255.0 + shift = 0.0 + resize_height, resize_width = 224, 224 + + # define map operations + decode_op = vision.Decode() + rescale_op = vision.Rescale(rescale, shift) + # resize_op = vision.Resize(resize_height, resize_width, + # InterpolationMode.DE_INTER_LINEAR) # Bilinear mode + resize_op = vision.Resize((resize_height, resize_width)) + + # apply map operations on images + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=rescale_op) + data1 = data1.map(input_columns=["image"], operations=resize_op) + + # # apply ont-hot encoding on labels + num_classes = 4 + one_hot_encode = data_trans.OneHot(num_classes) # num_classes is input argument + data1 = data1.map(input_columns=["label"], operations=one_hot_encode) + # + # # apply Datasets + buffer_size = 100 + seed = 10 + batch_size = 2 + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) # 10000 as in imageNet train script + data1 = data1.batch(batch_size, drop_remainder=True) + + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # in this example, each dictionary has keys "image" and "label" + logger.info("image is: {}".format(item["image"])) + logger.info("label is: {}".format(item["label"])) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + + +if __name__ == '__main__': + logger.info('===========now test Repeat============') + # logger.info('Simple Repeat') + test_case_repeat() + logger.info('\n') + + logger.info('===========now test Shuffle===========') + # logger.info('Simple Shuffle') + test_case_shuffle() + logger.info('\n') + + # Note: cannot work with different shapes, hence not for image + # logger.info('===========now test Batch=============') + # # logger.info('Simple Batch') + # test_case_batch() + # logger.info('\n') + + logger.info('===========now test case 0============') + # logger.info('Repeat then Shuffle') + test_case_0() + logger.info('\n') + + logger.info('===========now test case 0 reverse============') + # # logger.info('Shuffle then Repeat') + test_case_0_reverse() + logger.info('\n') + + # logger.info('===========now test case 1============') + # # logger.info('Repeat with Batch') + # test_case_1() + # logger.info('\n') + + # logger.info('===========now test case 2============') + # # logger.info('Batch with Shuffle') + # test_case_2() + # logger.info('\n') + + # for image augmentation only + logger.info('===========now test case 3============') + logger.info('Map then Shuffle') + test_case_3() + logger.info('\n') + + diff --git a/tests/ut/python/dataset/test_datasets_imagenet_distribution.py b/tests/ut/python/dataset/test_datasets_imagenet_distribution.py new file mode 100644 index 00000000000..8afe624fbc1 --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_imagenet_distribution.py @@ -0,0 +1,98 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import pytest +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images2/train-0000-of-0001.data", + "../data/dataset/test_tf_file_3_images2/train-0000-of-0002.data", + "../data/dataset/test_tf_file_3_images2/train-0000-of-0003.data", + "../data/dataset/test_tf_file_3_images2/train-0000-of-0004.data"] + +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images2/datasetSchema.json" + +DISTRIBUTION_ALL_DIR = "../data/dataset/test_tf_file_3_images2/dataDistributionAll.json" +DISTRIBUTION_UNIQUE_DIR = "../data/dataset/test_tf_file_3_images2/dataDistributionUnique.json" +DISTRIBUTION_RANDOM_DIR = "../data/dataset/test_tf_file_3_images2/dataDistributionRandom.json" +DISTRIBUTION_EQUAL_DIR = "../data/dataset/test_tf_file_3_images2/dataDistributionEqualRows.json" + + +def test_tf_file_normal(): + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.repeat(1) + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert num_iter == 12 + + +def test_tf_file_distribution_all(): + # apply dataset operations + data1 = ds.StorageDataset(DATA_DIR, SCHEMA_DIR, DISTRIBUTION_ALL_DIR) + data1 = data1.repeat(2) + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert num_iter == 24 + + +def test_tf_file_distribution_unique(): + data1 = ds.StorageDataset(DATA_DIR, SCHEMA_DIR, DISTRIBUTION_UNIQUE_DIR) + data1 = data1.repeat(1) + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert num_iter == 4 + +def test_tf_file_distribution_random(): + data1 = ds.StorageDataset(DATA_DIR, SCHEMA_DIR, DISTRIBUTION_RANDOM_DIR) + data1 = data1.repeat(1) + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + assert num_iter == 4 + +def test_tf_file_distribution_equal_rows(): + data1 = ds.StorageDataset(DATA_DIR, SCHEMA_DIR, DISTRIBUTION_EQUAL_DIR) + data1 = data1.repeat(2) + num_iter = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + num_iter += 1 + + assert num_iter == 4 + +if __name__ == '__main__': + logger.info('=======test normal=======') + test_tf_file_normal() + + logger.info('=======test all=======') + test_tf_file_distribution_all() + + logger.info('=======test unique=======') + test_tf_file_distribution_unique() + + logger.info('=======test random=======') + test_tf_file_distribution_random() + logger.info('=======test equal rows=======') + test_tf_file_distribution_equal_rows() diff --git a/tests/ut/python/dataset/test_datasets_manifestop.py b/tests/ut/python/dataset/test_datasets_manifestop.py new file mode 100644 index 00000000000..c1e011a9009 --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_manifestop.py @@ -0,0 +1,123 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.transforms.c_transforms as data_trans +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_FILE = "../data/dataset/testManifestData/test.manifest" + + +def test_manifest_dataset_train(): + data = ds.ManifestDataset(DATA_FILE, decode=True) + count = 0 + cat_count = 0 + dog_count = 0 + for item in data.create_dict_iterator(): + logger.info("item[image] is {}".format(item["image"])) + count = count + 1 + if item["label"].size == 1 and item["label"] == 0: + cat_count = cat_count + 1 + elif item["label"].size == 1 and item["label"] == 1: + dog_count = dog_count + 1 + assert (cat_count == 2) + assert (dog_count == 1) + assert (count == 4) + + +def test_manifest_dataset_eval(): + data = ds.ManifestDataset(DATA_FILE, "eval", decode=True) + count = 0 + for item in data.create_dict_iterator(): + logger.info("item[image] is {}".format(item["image"])) + count = count + 1 + if item["label"] != 0 and item["label"] != 1: + assert (0) + assert (count == 2) + + +def test_manifest_dataset_class_index(): + class_indexing = {"dog": 11} + data = ds.ManifestDataset(DATA_FILE, decode=True, class_indexing=class_indexing) + out_class_indexing = data.get_class_indexing() + assert (out_class_indexing == {"dog": 11}) + count = 0 + for item in data.create_dict_iterator(): + logger.info("item[image] is {}".format(item["image"])) + count = count + 1 + if item["label"] != 11: + assert (0) + assert (count == 1) + + +def test_manifest_dataset_get_class_index(): + data = ds.ManifestDataset(DATA_FILE, decode=True) + class_indexing = data.get_class_indexing() + assert (class_indexing == {'cat': 0, 'dog': 1, 'flower': 2}) + data = data.shuffle(4) + class_indexing = data.get_class_indexing() + assert (class_indexing == {'cat': 0, 'dog': 1, 'flower': 2}) + count = 0 + for item in data.create_dict_iterator(): + logger.info("item[image] is {}".format(item["image"])) + count = count + 1 + assert (count == 4) + + +def test_manifest_dataset_multi_label(): + data = ds.ManifestDataset(DATA_FILE, decode=True, shuffle=False) + count = 0 + expect_label = [1, 0, 0, [0, 2]] + for item in data.create_dict_iterator(): + assert (item["label"].tolist() == expect_label[count]) + logger.info("item[image] is {}".format(item["image"])) + count = count + 1 + assert (count == 4) + + +def multi_label_hot(x): + result = np.zeros(x.size // x.ndim, dtype=int) + if x.ndim > 1: + for i in range(x.ndim): + result = np.add(result, x[i]) + else: + result = np.add(result, x) + + return result + + +def test_manifest_dataset_multi_label_onehot(): + data = ds.ManifestDataset(DATA_FILE, decode=True, shuffle=False) + expect_label = [[[0, 1, 0], [1, 0, 0]], [[1, 0, 0], [1, 0, 1]]] + one_hot_encode = data_trans.OneHot(3) + data = data.map(input_columns=["label"], operations=one_hot_encode) + data = data.map(input_columns=["label"], operations=multi_label_hot) + data = data.batch(2) + count = 0 + for item in data.create_dict_iterator(): + assert (item["label"].tolist() == expect_label[count]) + logger.info("item[image] is {}".format(item["image"])) + count = count + 1 + + +if __name__ == '__main__': + test_manifest_dataset_train() + test_manifest_dataset_eval() + test_manifest_dataset_class_index() + test_manifest_dataset_get_class_index() + test_manifest_dataset_multi_label() + test_manifest_dataset_multi_label_onehot() diff --git a/tests/ut/python/dataset/test_datasets_sharding.py b/tests/ut/python/dataset/test_datasets_sharding.py new file mode 100644 index 00000000000..b178298e33e --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_sharding.py @@ -0,0 +1,163 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger + +def test_imagefolder_shardings(print_res=False): + image_folder_dir = "../data/dataset/testPK/data" + + def sharding_config(num_shards, shard_id, num_samples, shuffle, class_index, repeat_cnt=1): + data1 = ds.ImageFolderDatasetV2(image_folder_dir, num_samples=num_samples, num_shards=num_shards, + shard_id=shard_id, + shuffle=shuffle, class_indexing=class_index, decode=True) + data1 = data1.repeat(repeat_cnt) + res = [] + for item in data1.create_dict_iterator(): # each data is a dictionary + res.append(item["label"].item()) + if (print_res): + logger.info("labels of dataset: {}".format(res)) + return res + + # total 44 rows in dataset + assert (sharding_config(4, 0, 5, False, dict()) == [0, 0, 0, 1, 1]) # 5 rows + assert (sharding_config(4, 0, 12, False, dict()) == [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]) # 11 rows + assert (sharding_config(4, 3, 0, False, dict()) == [0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]) # 11 rows + # total 22 in dataset rows because of class indexing which takes only 2 folders + assert (len(sharding_config(4, 0, 0, True, {"class1": 111, "class2": 999})) == 6) + assert (len(sharding_config(4, 2, 3, True, {"class1": 111, "class2": 999})) == 3) + # test with repeat + assert (sharding_config(4, 0, 12, False, dict(), 3) == [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3] * 3) + assert (sharding_config(4, 0, 5, False, dict(), 5) == [0, 0, 0, 1, 1] * 5) + assert (len(sharding_config(5, 1, 0, True, {"class1": 111, "class2": 999}, 4)) == 20) + + +def test_manifest_shardings(print_res=False): + manifest_file = "../data/dataset/testManifestData/test5trainimgs.json" + + def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1): + data1 = ds.ManifestDataset(manifest_file, num_samples=num_samples, num_shards=num_shards, shard_id=shard_id, + shuffle=shuffle, decode=True) + data1 = data1.repeat(repeat_cnt) + res = [] + for item in data1.create_dict_iterator(): # each data is a dictionary + res.append(item["label"].item()) + if print_res: + logger.info("labels of dataset: {}".format(res)) + return res + + # 5 train images in total + sharding_config(2, 0, None, False) + assert (sharding_config(2, 0, None, False) == [0, 1, 1]) + assert (sharding_config(2, 1, None, False) == [0, 0, 0]) + assert (sharding_config(2, 0, 2, False) == [0, 1]) + assert (sharding_config(2, 1, 2, False) == [0, 0]) + # with repeat + assert (sharding_config(2, 1, None, False, 3) == [0, 0, 0] * 3) + assert (sharding_config(2, 0, 2, False, 5) == [0, 1] * 5) + + +def test_voc_shardings(print_res=False): + voc_dir = "../data/dataset/testVOC2012" + + def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1): + sampler = ds.DistributedSampler(num_shards, shard_id, shuffle=shuffle) + data1 = ds.VOCDataset(voc_dir, decode=True, sampler=sampler, num_samples=num_samples) + data1 = data1.repeat(repeat_cnt) + res = [] + for item in data1.create_dict_iterator(): # each data is a dictionary + res.append(item["image"].shape[0]) + if print_res: + logger.info("labels of dataset: {}".format(res)) + return res + + # 10 images in total, always decode to get the shape + # first dim of all 10 images [2268,2268,2268,2268,642,607,561,596,612,2268] + # 3 shard_workers, 0th worker will get 0-th, 3nd, 6th and 9th image + assert (sharding_config(3, 0, None, False, 2) == [2268, 2268, 561, 2268] * 2) + # 3 shard_workers, 1st worker will get 1-st, 4nd, 7th and 0th image, the last one goes back bc of rounding up + assert (sharding_config(3, 1, 5, False, 3) == [2268, 642, 596, 2268] * 3) + # 3 shard_workers, 2nd worker will get 2nd, 5th, 8th and 11th (which is 1st) + # then takes the first 2 bc num_samples = 2 + assert (sharding_config(3, 2, 2, False, 4) == [2268, 607] * 4) + # test that each epoch, each shard_worker returns a different sample + assert (len(sharding_config(2, 0, None, True, 1)) == 5) + assert (len(set(sharding_config(11, 0, None, True, 10))) > 1) + + +def test_cifar10_shardings(print_res=False): + cifar10_dir = "../data/dataset/testCifar10Data" + + def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1): + data1 = ds.Cifar10Dataset(cifar10_dir, num_shards=num_shards, shard_id=shard_id, num_samples=num_samples, + shuffle=shuffle) + data1 = data1.repeat(repeat_cnt) + res = [] + for item in data1.create_dict_iterator(): # each data is a dictionary + res.append(item["label"].item()) + if print_res: + logger.info("labels of dataset: {}".format(res)) + return res + + # 60000 rows in total. CIFAR reads everything in memory which would make each test case very slow + # therefore, only 2 test cases for now. + assert (sharding_config(10000, 9999, 7, False, 1) == [9]) + assert (sharding_config(10000, 0, 4, False, 3) == [0, 0, 0]) + + +def test_cifar100_shardings(print_res=False): + cifar100_dir = "../data/dataset/testCifar100Data" + + def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1): + data1 = ds.Cifar100Dataset(cifar100_dir, num_shards=num_shards, shard_id=shard_id, num_samples=num_samples, + shuffle=shuffle) + data1 = data1.repeat(repeat_cnt) + res = [] + for item in data1.create_dict_iterator(): # each data is a dictionary + res.append(item["coarse_label"].item()) + if print_res: + logger.info("labels of dataset: {}".format(res)) + return res + + # 10000 rows in total in test.bin CIFAR100 file + assert (sharding_config(1000, 999, 7, False, 2) == [1, 18, 10, 17, 5, 0, 15] * 2) + assert (sharding_config(1000, 0, None, False) == [10, 16, 2, 11, 10, 17, 11, 14, 13, 3]) + + +def test_mnist_shardings(print_res=False): + mnist_dir = "../data/dataset/testMnistData" + + def sharding_config(num_shards, shard_id, num_samples, shuffle, repeat_cnt=1): + data1 = ds.MnistDataset(mnist_dir, num_shards=num_shards, shard_id=shard_id, num_samples=num_samples, + shuffle=shuffle) + data1 = data1.repeat(repeat_cnt) + res = [] + for item in data1.create_dict_iterator(): # each data is a dictionary + res.append(item["label"].item()) + if print_res: + logger.info("labels of dataset: {}".format(res)) + return res + + # 70K rows in total , divide across 10K hosts, each host has 7 images + assert sharding_config(10000, 0, num_samples=5, shuffle=False, repeat_cnt=3) == [0, 0, 0] + assert sharding_config(10000, 9999, num_samples=None, shuffle=False, repeat_cnt=1) == [9] + + +if __name__ == '__main__': + test_imagefolder_shardings(True) + test_manifest_shardings(True) + test_voc_shardings(True) + test_cifar10_shardings(True) + test_cifar100_shardings(True) + test_mnist_shardings(True) diff --git a/tests/ut/python/dataset/test_datasets_voc.py b/tests/ut/python/dataset/test_datasets_voc.py new file mode 100644 index 00000000000..09e9caf3e3b --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_voc.py @@ -0,0 +1,52 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = "../data/dataset/testVOC2012" + + +def test_voc_normal(): + data1 = ds.VOCDataset(DATA_DIR, decode=True) + num = 0 + for item in data1.create_dict_iterator(): + logger.info("item[image] is {}".format(item["image"])) + logger.info("item[image].shape is {}".format(item["image"].shape)) + logger.info("item[target] is {}".format(item["target"])) + logger.info("item[target].shape is {}".format(item["target"].shape)) + num += 1 + logger.info("num is {}".format(str(num))) + + +def test_case_0(): + data1 = ds.VOCDataset(DATA_DIR, decode=True) + + resize_op = vision.Resize((224, 224)) + + data1 = data1.map(input_columns=["image"], operations=resize_op) + data1 = data1.map(input_columns=["target"], operations=resize_op) + repeat_num = 4 + data1 = data1.repeat(repeat_num) + batch_size = 2 + data1 = data1.batch(batch_size, drop_remainder=True) + + num = 0 + for item in data1.create_dict_iterator(): + logger.info("item[image].shape is {}".format(item["image"].shape)) + logger.info("item[target].shape is {}".format(item["target"].shape)) + num += 1 + logger.info("num is {}".format(str(num))) diff --git a/tests/ut/python/dataset/test_decode.py b/tests/ut/python/dataset/test_decode.py new file mode 100644 index 00000000000..94c7a36c9e2 --- /dev/null +++ b/tests/ut/python/dataset/test_decode.py @@ -0,0 +1,87 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing Decode op in DE +""" +import cv2 +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def diff_mse(in1, in2): + mse = (np.square(in1.astype(float) / 255 - in2.astype(float) / 255)).mean() + return mse * 100 + + +def test_decode_op(): + """ + Test Decode op + """ + logger.info("test_decode_op") + + # Decode with rgb format set to True + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + + # Serialize and Load dataset requires using vision.Decode instead of vision.Decode(). + data1 = data1.map(input_columns=["image"], operations=[vision.Decode(True)]) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + actual = item1["image"] + expected = cv2.imdecode(item2["image"], cv2.IMREAD_COLOR) + expected = cv2.cvtColor(expected, cv2.COLOR_BGR2RGB) + assert actual.shape == expected.shape + diff = actual - expected + mse = np.sum(np.power(diff, 2)) + assert mse == 0 + + +def test_decode_op_tf_file_dataset(): + """ + Test Decode op with tf_file dataset + """ + logger.info("test_decode_op_tf_file_dataset") + + # Decode with rgb format set to True + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=ds.Shuffle.FILES) + data1 = data1.map(input_columns=["image"], operations=vision.Decode(True)) + + for item in data1.create_dict_iterator(): + logger.info('decode == {}'.format(item['image'])) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + actual = item1["image"] + expected = cv2.imdecode(item2["image"], cv2.IMREAD_COLOR) + expected = cv2.cvtColor(expected, cv2.COLOR_BGR2RGB) + assert actual.shape == expected.shape + diff = actual - expected + mse = np.sum(np.power(diff, 2)) + assert mse == 0 + + +if __name__ == "__main__": + test_decode_op() + test_decode_op_tf_file_dataset() diff --git a/tests/ut/python/dataset/test_deviceop_cpu.py b/tests/ut/python/dataset/test_deviceop_cpu.py new file mode 100644 index 00000000000..0ee443ed9e2 --- /dev/null +++ b/tests/ut/python/dataset/test_deviceop_cpu.py @@ -0,0 +1,135 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +TF_FILES = ["../data/dataset/testTFTestAllTypes/test.data"] +TF_SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json" + +def test_case_0(): + """ + Test Repeat + """ + # apply dataset operations + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + # define parameters + repeat_count = 2 + data = data.repeat(repeat_count) + + data = data.device_que() + data.send() + + +def test_case_1(): + """ + Test Batch + """ + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + # define data augmentation parameters + resize_height, resize_width = 224, 224 + + # define map operations + decode_op = vision.Decode() + resize_op = vision.Resize((resize_height, resize_width)) + + # apply map operations on images + data = data.map(input_columns=["image"], operations=decode_op) + data = data.map(input_columns=["image"], operations=resize_op) + + batch_size = 3 + data = data.batch(batch_size, drop_remainder=True) + + data = data.device_que() + data.send() + + +def test_case_2(): + """ + Test Batch & Repeat + """ + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + # define data augmentation parameters + resize_height, resize_width = 224, 224 + + # define map operations + decode_op = vision.Decode() + resize_op = vision.Resize((resize_height, resize_width)) + + # apply map operations on images + data = data.map(input_columns=["image"], operations=decode_op) + data = data.map(input_columns=["image"], operations=resize_op) + + batch_size = 2 + data = data.batch(batch_size, drop_remainder=True) + + data = data.repeat(2) + + data = data.device_que() + assert data.get_repeat_count() == 2 + data.send() + + +def test_case_3(): + """ + Test Repeat & Batch + """ + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + # define data augmentation parameters + resize_height, resize_width = 224, 224 + + # define map operations + decode_op = vision.Decode() + resize_op = vision.Resize((resize_height, resize_width)) + + # apply map operations on images + data = data.map(input_columns=["image"], operations=decode_op) + data = data.map(input_columns=["image"], operations=resize_op) + + data = data.repeat(2) + + batch_size = 2 + data = data.batch(batch_size, drop_remainder=True) + + data = data.device_que() + data.send() + + +def test_case_tf_file(): + data = ds.TFRecordDataset(TF_FILES, TF_SCHEMA_FILE, shuffle=ds.Shuffle.FILES) + + data = data.to_device(num_batch=10) + data.send() + + +if __name__ == '__main__': + logger.info('===========now test Repeat============') + test_case_0() + + logger.info('===========now test Batch============') + test_case_1() + + logger.info('===========now test Batch & Repeat============') + test_case_2() + + logger.info('===========now test Repeat & Batch============') + test_case_3() + + logger.info('===========now test tf file============') + test_case_tf_file() diff --git a/tests/ut/python/dataset/test_exceptions.py b/tests/ut/python/dataset/test_exceptions.py new file mode 100644 index 00000000000..7668eeb2a8d --- /dev/null +++ b/tests/ut/python/dataset/test_exceptions.py @@ -0,0 +1,33 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision +import pytest + +import mindspore.dataset as ds + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + + +def skip_test_exception(): + ds.config.set_num_parallel_workers(1) + data = ds.TFRecordDataset(DATA_DIR, columns_list=["image"]) + data = data.map(input_columns=["image"], operations=vision.Resize(100, 100)) + with pytest.raises(RuntimeError) as info: + data.create_tuple_iterator().get_next() + assert "The shape size 1 of input tensor is invalid" in str(info.value) + + +if __name__ == '__main__': + test_exception() diff --git a/tests/ut/python/dataset/test_five_crop.py b/tests/ut/python/dataset/test_five_crop.py new file mode 100644 index 00000000000..c5cba721c76 --- /dev/null +++ b/tests/ut/python/dataset/test_five_crop.py @@ -0,0 +1,115 @@ +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Testing FiveCrop in DE +""" +import matplotlib.pyplot as plt +import numpy as np +import pytest + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(image_1, image_2): + """ + visualizes the image using FiveCrop + """ + plt.subplot(161) + plt.imshow(image_1) + plt.title("Original") + + for i, image in enumerate(image_2): + image = (image.transpose(1, 2, 0) * 255).astype(np.uint8) + plt.subplot(162 + i) + plt.imshow(image) + plt.title("image {} in FiveCrop".format(i + 1)) + + plt.show() + + +def skip_test_five_crop_op(): + """ + Test FiveCrop + """ + logger.info("test_five_crop") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms_1 = [ + vision.Decode(), + vision.ToTensor(), + ] + transform_1 = vision.ComposeOp(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms_2 = [ + vision.Decode(), + vision.FiveCrop(200), + lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images + ] + transform_2 = vision.ComposeOp(transforms_2) + data2 = data2.map(input_columns=["image"], operations=transform_2()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + image_2 = item2["image"] + + logger.info("shape of image_1: {}".format(image_1.shape)) + logger.info("shape of image_2: {}".format(image_2.shape)) + + logger.info("dtype of image_1: {}".format(image_1.dtype)) + logger.info("dtype of image_2: {}".format(image_2.dtype)) + + # visualize(image_1, image_2) + + # The output data should be of a 4D tensor shape, a stack of 5 images. + assert len(image_2.shape) == 4 + assert image_2.shape[0] == 5 + + +def test_five_crop_error_msg(): + """ + Test FiveCrop error message. + """ + logger.info("test_five_crop_error_msg") + + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + vision.Decode(), + vision.FiveCrop(200), + vision.ToTensor() + ] + transform = vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + + with pytest.raises(RuntimeError) as info: + data.create_tuple_iterator().get_next() + error_msg = "TypeError: img should be PIL Image or Numpy array. Got " + + # error msg comes from ToTensor() + assert error_msg in str(info.value) + + +if __name__ == "__main__": + test_five_crop_op() + test_five_crop_error_msg() diff --git a/tests/ut/python/dataset/test_general.py b/tests/ut/python/dataset/test_general.py new file mode 100644 index 00000000000..cbbc6974731 --- /dev/null +++ b/tests/ut/python/dataset/test_general.py @@ -0,0 +1,41 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from util import save_and_check + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] +SCHEMA_DIR = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", + "col_sint16", "col_sint32", "col_sint64"] +GENERATE_GOLDEN = False + + +def test_case_columns_list(): + """ + a simple repeat operation. + """ + logger.info("Test Simple Repeat") + # define parameters + repeat_count = 2 + parameters = {"params": {'repeat_count': repeat_count}} + columns_list = ["col_sint64", "col_sint32"] + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=columns_list, shuffle=False) + data1 = data1.repeat(repeat_count) + + filename = "columns_list_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_generator.py b/tests/ut/python/dataset/test_generator.py new file mode 100644 index 00000000000..07556d9c7fe --- /dev/null +++ b/tests/ut/python/dataset/test_generator.py @@ -0,0 +1,460 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np +import pytest + +import mindspore.common.dtype as mstype +import mindspore.dataset as ds +from mindspore import log as logger + + +# Generate 1d int numpy array from 0 - 63 +def generator_1d(): + for i in range(64): + yield (np.array([i]),) + + +def test_case_0(): + """ + Test 1D Generator + """ + logger.info("Test 1D Generator : 0 - 63") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_1d, ["data"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([i]) + assert np.array_equal(item["data"], golden) + i = i + 1 + + +# Generate md int numpy array from [[0, 1], [2, 3]] to [[63, 64], [65, 66]] +def generator_md(): + for i in range(64): + yield (np.array([[i, i + 1], [i + 2, i + 3]]),) + + +def test_case_1(): + """ + Test MD Generator + """ + logger.info("Test MD Generator : 0 - 63, with shape [2, 2]") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_md, ["data"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item["data"], golden) + i = i + 1 + + +# Generate two columns, the first column is from Generator1D, the second column is from GeneratorMD +def generator_mc(maxid=64): + for i in range(maxid): + yield (np.array([i]), np.array([[i, i + 1], [i + 2, i + 3]])) + + +def test_case_2(): + """ + Test multi column generator + """ + logger.info("Test multi column generator") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc, ["col0", "col1"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([i]) + assert np.array_equal(item["col0"], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item["col1"], golden) + i = i + 1 + + +def test_case_3(): + """ + Test 1D Generator + repeat(4) + """ + logger.info("Test 1D Generator : 0 - 63 + Repeat(4)") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_1d, ["data"]) + + data1 = data1.repeat(4) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([i]) + assert np.array_equal(item["data"], golden) + i = i + 1 + if i == 64: + i = 0 + + +def test_case_4(): + """ + Test fixed size 1D Generator + batch + """ + logger.info("Test 1D Generator : 0 - 63 + batch(4)") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_1d, ["data"]) + + data1 = data1.batch(4) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([[i], [i + 1], [i + 2], [i + 3]]) + assert np.array_equal(item["data"], golden) + i = i + 4 + + +def generator_with_type(t): + for i in range(64): + yield (np.array([i], dtype=t),) + + +def type_tester(t): + logger.info("Test with Type {}".format(t.__name__)) + + # apply dataset operations + data1 = ds.GeneratorDataset((lambda: generator_with_type(t)), ["data"]) + + data1 = data1.batch(4) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([[i], [i + 1], [i + 2], [i + 3]], dtype=t) + assert np.array_equal(item["data"], golden) + i = i + 4 + + +def test_case_5(): + """ + Test 1D Generator on different data type + """ + logger.info("Test 1D Generator on all data types") + + types = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float32, np.float64] + + for t in types: + type_tester(t) + + +def type_tester_with_type_check(t, c): + logger.info("Test with Type {}".format(t.__name__)) + + # apply dataset operations + data1 = ds.GeneratorDataset((lambda: generator_with_type(t)), ["data"], column_types=[c]) + + data1 = data1.batch(4) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([[i], [i + 1], [i + 2], [i + 3]], dtype=t) + assert np.array_equal(item["data"], golden) + i = i + 4 + + +def test_case_6(): + """ + Test 1D Generator on different data type with type check + """ + logger.info("Test 1D Generator on all data types with type check") + + np_types = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float32, + np.float64] + de_types = [mstype.int8, mstype.int16, mstype.int32, mstype.int64, mstype.uint8, mstype.uint16, mstype.uint32, + mstype.uint64, mstype.float32, mstype.float64] + + for i in range(len(np_types)): + type_tester_with_type_check(np_types[i], de_types[i]) + + +def generator_with_type_2c(t): + for i in range(64): + yield (np.array([i], dtype=t), np.array([i], dtype=t)) + + +def type_tester_with_type_check_2c(t, c): + logger.info("Test with Type {}".format(t.__name__)) + + # apply dataset operations + data1 = ds.GeneratorDataset((lambda: generator_with_type_2c(t)), ["data0", "data1"], column_types=c) + + data1 = data1.batch(4) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([[i], [i + 1], [i + 2], [i + 3]], dtype=t) + assert np.array_equal(item["data0"], golden) + i = i + 4 + + +def test_case_7(): + """ + Test 2 column Generator on different data type with type check + """ + logger.info("Test 2 column Generator on all data types with type check") + + np_types = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float32, + np.float64] + de_types = [mstype.int8, mstype.int16, mstype.int32, mstype.int64, mstype.uint8, mstype.uint16, mstype.uint32, + mstype.uint64, mstype.float32, mstype.float64] + + for i in range(len(np_types)): + type_tester_with_type_check_2c(np_types[i], [None, de_types[i]]) + + +def test_case_8(): + """ + Test multi column generator with few mapops + """ + logger.info("Test multi column generator with mapops to check the order too") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) + data1 = data1.map(input_columns="col0", output_columns="out0", operations=(lambda x: x * 3), + num_parallel_workers=2) + data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x * 7, x)), + num_parallel_workers=2, columns_order=["out0", "out1", "out2"]) + data1 = data1.map(input_columns="out2", output_columns="out2", operations=(lambda x: x + 1), + num_parallel_workers=2) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + golden = np.array([i * 3]) + assert np.array_equal(item["out0"], golden) + golden = np.array([[i * 7, (i + 1) * 7], [(i + 2) * 7, (i + 3) * 7]]) + assert np.array_equal(item["out1"], golden) + golden = np.array([[i + 1, i + 2], [i + 3, i + 4]]) + assert np.array_equal(item["out2"], golden) + i = i + 1 + + +def test_case_9(): + """ + Test map column order when len(input_columns) == len(output_columns). + """ + logger.info("Test map column order when len(input_columns) == len(output_columns).") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["image", "label"]) + data2 = ds.GeneratorDataset(generator_mc(2048), ["label", "image"]) + data1 = data1.map(input_columns="label", operations=(lambda x: x * 3), + num_parallel_workers=4) + data2 = data2.map(input_columns="label", operations=(lambda x: x * 3), + num_parallel_workers=4) + + # Expected column order is not changed. + # data1 = data[0] is "image" and data[1] is "label" + # data2 = data[0] is "label" and data[1] is "image" + i = 0 + for data1, data2 in zip(data1, data2): # each data is a dictionary + golden = np.array([i]) + assert np.array_equal(data1[0], golden) + golden = np.array([[i * 3, (i + 1) * 3], [(i + 2) * 3, (i + 3) * 3]]) + assert np.array_equal(data1[1], golden) + + golden = np.array([i * 3]) + assert np.array_equal(data2[0], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(data2[1], golden) + i = i + 1 + + +def test_case_10(): + """ + Test map column order when len(input_columns) != len(output_columns). + """ + logger.info("Test map column order when len(input_columns) != len(output_columns).") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) + data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)), + columns_order=['col0', 'out1', 'out2'], num_parallel_workers=2) + + # Expected column order is |col0|out1|out2| + i = 0 + for item in data1.create_tuple_iterator(): + golden = np.array([i]) + assert np.array_equal(item[0], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item[1], golden) + golden = np.array([[i * 5, (i + 1) * 5], [(i + 2) * 5, (i + 3) * 5]]) + assert np.array_equal(item[2], golden) + i = i + 1 + + +def test_case_11(): + """ + Test map column order when len(input_columns) != len(output_columns). + """ + logger.info("Test map column order when len(input_columns) != len(output_columns), " + "and columns_order drops some columns.") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) + data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)), + columns_order=['out1', 'out2'], num_parallel_workers=2) + + # Expected column order is |out1|out2| + i = 0 + for item in data1.create_tuple_iterator(): + # len should be 2 because col0 is dropped (not included in columns_order) + assert len(item) == 2 + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item[0], golden) + golden = np.array([[i * 5, (i + 1) * 5], [(i + 2) * 5, (i + 3) * 5]]) + assert np.array_equal(item[1], golden) + i = i + 1 + + +def test_case_12(): + """ + Test map column order when input_columns and output_columns are None. + """ + logger.info("Test map column order when input_columns and output_columns are None.") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) + data1 = data1.map(operations=(lambda x: (x * 5)), num_parallel_workers=2) + + # Expected column order is |col0|col1| + i = 0 + for item in data1.create_tuple_iterator(): + assert len(item) == 2 + golden = np.array([i * 5]) + assert np.array_equal(item[0], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item[1], golden) + i = i + 1 + + data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) + data1 = data1.map(operations=(lambda x: (x * 5)), columns_order=["col1", "col0"], num_parallel_workers=2) + + # Expected column order is |col0|col1| + i = 0 + for item in data1.create_tuple_iterator(): + assert len(item) == 2 + golden = np.array([i * 5]) + assert np.array_equal(item[1], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item[0], golden) + i = i + 1 + + +def test_case_13(): + """ + Test map column order when input_columns is None. + """ + logger.info("Test map column order when input_columns is None.") + + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) + data1 = data1.map(operations=(lambda x: (x * 5)), output_columns=["out0"], num_parallel_workers=2) + + # Expected column order is |out0|col1| + i = 0 + for item in data1.create_tuple_iterator(): + assert len(item) == 2 + golden = np.array([i * 5]) + assert np.array_equal(item[0], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item[1], golden) + i = i + 1 + + for item in data1.create_dict_iterator(): # each data is a dictionary + # len should be 2 because col0 is dropped (not included in columns_order) + assert len(item) == 2 + golden = np.array([i * 5]) + assert np.array_equal(item["out0"], golden) + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item["col1"], golden) + i = i + 1 + + +def test_case_error_1(): + def generator_np(): + for i in range(64): + yield (np.array([{i}]),) + + with pytest.raises(RuntimeError) as info: + data1 = ds.GeneratorDataset(generator_np, ["data"]) + for _ in data1: + pass + assert "Invalid data type" in str(info.value) + + +def test_case_error_2(): + def generator_np(): + for i in range(64): + yield ({i},) + + with pytest.raises(RuntimeError) as info: + data1 = ds.GeneratorDataset(generator_np, ["data"]) + for _ in data1: + pass + assert "Generator should return a tuple of numpy arrays" in str(info.value) + + +def test_case_error_3(): + with pytest.raises(ValueError) as info: + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["label", "image"]) + data1 = data1.map(input_columns=["label"], output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)), + num_parallel_workers=2) + + for _ in data1: + pass + assert "When (len(input_columns) != len(output_columns)), columns_order must be specified." in str(info.value) + + +def test_case_error_4(): + with pytest.raises(RuntimeError) as info: + # apply dataset operations + data1 = ds.GeneratorDataset(generator_mc(2048), ["label", "image"]) + data1 = data1.map(input_columns=["label"], operations=(lambda x: (x, x * 5)), + num_parallel_workers=2) + + for _ in data1: + pass + assert "Unexpected error. Result of a tensorOp doesn't match output column names" in str(info.value) + + +if __name__ == "__main__": + test_case_0() + test_case_1() + test_case_2() + test_case_3() + test_case_4() + test_case_5() + test_case_6() + test_case_7() + test_case_8() + test_case_9() + test_case_10() + test_case_11() + test_case_12() + test_case_13() + test_case_error_1() + test_case_error_2() + test_case_error_3() + test_case_error_4() diff --git a/tests/ut/python/dataset/test_get_size.py b/tests/ut/python/dataset/test_get_size.py new file mode 100644 index 00000000000..ba4162788ce --- /dev/null +++ b/tests/ut/python/dataset/test_get_size.py @@ -0,0 +1,184 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +FILES = ["../data/dataset/testTFTestAllTypes/test.data"] +DATASET_ROOT = "../data/dataset/testTFTestAllTypes/" +SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +GENERATE_GOLDEN = False + + +def test_case1(): + data = ds.TFRecordDataset(FILES, SCHEMA_FILE) + assert data.get_dataset_size() == 12 + assert data.get_batch_size() == 1 + assert data.get_repeat_count() == 1 + data = data.shuffle(100) + assert data.get_dataset_size() == 12 + assert data.get_batch_size() == 1 + assert data.get_repeat_count() == 1 + data = data.batch(2) + assert data.get_dataset_size() == 6 + assert data.get_batch_size() == 2 + assert data.get_repeat_count() == 1 + data = data.rename("col_sint64", "new_column") + assert data.get_dataset_size() == 6 + assert data.get_batch_size() == 2 + assert data.get_repeat_count() == 1 + data = data.repeat(10) + assert data.get_dataset_size() == 6 + assert data.get_batch_size() == 2 + assert data.get_repeat_count() == 10 + data = data.project(["new_column"]) + assert data.get_dataset_size() == 6 + assert data.get_batch_size() == 2 + assert data.get_repeat_count() == 10 + + data2 = ds.TFRecordDataset(FILES, SCHEMA_FILE).batch(2).repeat(10) + + data1 = data.zip(data2) + assert data1.get_dataset_size() == 6 + + +def test_case2(): + data = ds.TFRecordDataset(FILES, num_samples=6) + assert data.get_dataset_size() == 6 + data = data.shuffle(100) + assert data.get_dataset_size() == 6 + data = data.batch(2) + assert data.get_dataset_size() == 3 + data = data.rename("col_sint64", "new_column") + assert data.get_dataset_size() == 3 + data = data.repeat(10) + assert data.get_dataset_size() == 3 + data = data.project(["new_column"]) + assert data.get_dataset_size() == 3 + + data2 = ds.TFRecordDataset(FILES, num_samples=6).batch(2).repeat(10) + + data1 = data.zip(data2) + assert data1.get_dataset_size() == 3 + + +def test_case3(): + data1 = ds.TFRecordDataset(FILES, SCHEMA_FILE).batch(2).repeat(10) + data2 = ds.TFRecordDataset(FILES, SCHEMA_FILE).batch(2).repeat(5) + data3 = ds.TFRecordDataset(FILES, SCHEMA_FILE).batch(2) + + data4 = ds.zip((data1, data2, data3)) + + assert data4.get_dataset_size() == 6 + + +def test_case4(): + data1 = ds.TFRecordDataset(FILES, SCHEMA_FILE).batch(2).repeat(10) + data2 = ds.TFRecordDataset(FILES) + assert data2.get_dataset_size() == 12 + data2 = data2.batch(2) + assert data2.get_dataset_size() == 6 + data2 = data2.shuffle(100) + assert data2.get_dataset_size() == 6 + data2 = data2.repeat(3) + assert data2.get_dataset_size() == 6 + + data3 = ds.zip((data1, data2)) + + assert data3.get_dataset_size() == 6 + + +def test_case5(): + data = ds.TFRecordDataset(FILES, num_samples=10).batch(3, drop_remainder=True) + assert data.get_dataset_size() == 3 + data = ds.TFRecordDataset(FILES, num_samples=10).batch(3, drop_remainder=False) + assert data.get_dataset_size() == 4 + + +def test_cifar(): + data = ds.Cifar10Dataset("../data/dataset/testCifar10Data") + assert data.get_dataset_size() == 10000 + + data = ds.Cifar10Dataset("../data/dataset/testCifar10Data", num_samples=10) + assert data.get_dataset_size() == 10 + + data = ds.Cifar10Dataset("../data/dataset/testCifar10Data", num_samples=90000) + assert data.get_dataset_size() == 10000 + + data = ds.Cifar100Dataset("../data/dataset/testCifar100Data") + assert data.get_dataset_size() == 10000 + + data = ds.Cifar100Dataset("../data/dataset/testCifar100Data", num_samples=10) + assert data.get_dataset_size() == 10 + + data = ds.Cifar100Dataset("../data/dataset/testCifar100Data", num_samples=20000) + assert data.get_dataset_size() == 10000 + + +def test_mnist(): + data = ds.MnistDataset("../data/dataset/testMnistData") + logger.info("dataset.size: {}".format(data.get_dataset_size())) + assert data.get_dataset_size() == 10000 + + data = ds.MnistDataset("../data/dataset/testMnistData", num_samples=10) + assert data.get_dataset_size() == 10 + + data = ds.MnistDataset("../data/dataset/testMnistData", num_samples=90000) + assert data.get_dataset_size() == 10000 + + +def test_manifest(): + data = ds.ManifestDataset("../data/dataset/testManifestData/test.manifest") + assert data.get_dataset_size() == 4 + assert data.num_classes() == 3 + + data = data.shuffle(100) + assert data.num_classes() == 3 + + +def test_imagefolder(): + data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/") + assert data.get_dataset_size() == 44 + assert data.num_classes() == 4 + data = data.shuffle(100) + assert data.num_classes() == 4 + + data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_samples=10) + assert data.get_dataset_size() == 10 + assert data.num_classes() == 4 + + +def test_generator(): + def generator(): + for i in range(64): + yield (np.array([i]),) + + data1 = ds.GeneratorDataset(generator, ["data"]) + data1.set_dataset_size(10) + assert data1.get_dataset_size() == 10 + data1.output_shapes() + assert data1.get_dataset_size() == 10 + + +if __name__ == '__main__': + # test_compare_v1_and_2() + # test_imagefolder() + # test_manifest() + test_case1() + # test_case2() + # test_case3() + # test_case4() + # test_case5() diff --git a/tests/ut/python/dataset/test_iterator.py b/tests/ut/python/dataset/test_iterator.py new file mode 100644 index 00000000000..d2518e11198 --- /dev/null +++ b/tests/ut/python/dataset/test_iterator.py @@ -0,0 +1,43 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np + +import mindspore.dataset as ds + +DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] +SCHEMA_DIR = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", + "col_sint16", "col_sint32", "col_sint64"] + + +def check(project_columns): + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=COLUMNS) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=project_columns) + + for data_actual, data_expected in zip(data1.create_tuple_iterator(project_columns), data2.create_tuple_iterator()): + assert len(data_actual) == len(data_expected) + assert all([np.array_equal(d1, d2) for d1, d2 in zip(data_actual, data_expected)]) + + +def test_case_iterator(): + """ + Test creating tuple iterator + """ + check(COLUMNS) + check(COLUMNS[0:1]) + check(COLUMNS[0:2]) + check(COLUMNS[0:7]) + check(COLUMNS[7:8]) + check(COLUMNS[0:2:8]) diff --git a/tests/ut/python/dataset/test_minddataset.py b/tests/ut/python/dataset/test_minddataset.py new file mode 100644 index 00000000000..99b31b64ec2 --- /dev/null +++ b/tests/ut/python/dataset/test_minddataset.py @@ -0,0 +1,481 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This is the test module for mindrecord +""" +import collections +import json +import os +import re +import string + +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np +import pytest +from mindspore._c_dataengine import InterpolationMode +from mindspore import log as logger + +import mindspore.dataset as ds +from mindspore.mindrecord import FileWriter + +FILES_NUM = 4 +CV_FILE_NAME = "../data/mindrecord/imagenet.mindrecord" +CV_DIR_NAME = "../data/mindrecord/testImageNetData" +NLP_FILE_NAME = "../data/mindrecord/aclImdb.mindrecord" +NLP_FILE_POS = "../data/mindrecord/testAclImdbData/pos" +NLP_FILE_VOCAB= "../data/mindrecord/testAclImdbData/vocab.txt" + +@pytest.fixture +def add_and_remove_cv_file(): + """add/remove cv file""" + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) if os.path.exists("{}".format(x)) else None + os.remove("{}.db".format(x)) if os.path.exists("{}.db".format(x)) else None + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_data(CV_DIR_NAME) + cv_schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, + "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + yield "yield_cv_data" + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +@pytest.fixture +def add_and_remove_nlp_file(): + """add/remove nlp file""" + paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + writer = FileWriter(NLP_FILE_NAME, FILES_NUM) + data = [x for x in get_nlp_data(NLP_FILE_POS, NLP_FILE_VOCAB, 10)] + nlp_schema_json = {"id": {"type": "string"}, "label": {"type": "int32"}, + "rating": {"type": "float32"}, + "input_ids": {"type": "int64", + "shape": [-1]}, + "input_mask": {"type": "int64", + "shape": [1, -1]}, + "segment_ids": {"type": "int64", + "shape": [2,-1]} + } + writer.set_header_size(1 << 14) + writer.set_page_size(1 << 15) + writer.add_schema(nlp_schema_json, "nlp_schema") + writer.add_index(["id", "rating"]) + writer.write_raw_data(data) + writer.commit() + yield "yield_nlp_data" + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_minddataset_writer_tutorial(): + """tutorial for cv dataset writer.""" + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) if os.path.exists("{}".format(x)) else None + os.remove("{}.db".format(x)) if os.path.exists("{}.db".format(x)) else None + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_data(CV_DIR_NAME) + cv_schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, + "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + + +def test_cv_minddataset_partition_tutorial(add_and_remove_cv_file): + """tutorial for cv minddataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + + def partitions(num_shards): + for partition_id in range(num_shards): + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers, + num_shards=num_shards, shard_id=partition_id) + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- partition : {} ------------------------".format(partition_id)) + logger.info("-------------- item[label]: {} -----------------------".format(item["label"])) + num_iter += 1 + return num_iter + + assert partitions(4) == 3 + assert partitions(5) == 2 + assert partitions(9) == 2 + + +def test_cv_minddataset_dataset_size(add_and_remove_cv_file): + """tutorial for cv minddataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers) + assert data_set.get_dataset_size() == 10 + repeat_num = 2 + data_set = data_set.repeat(repeat_num) + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- get dataset size {} -----------------".format(num_iter)) + logger.info("-------------- item[label]: {} ---------------------".format(item["label"])) + logger.info("-------------- item[data]: {} ----------------------".format(item["data"])) + num_iter += 1 + assert num_iter == 20 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers, + num_shards=4, shard_id=3) + assert data_set.get_dataset_size() == 3 + + +def test_cv_minddataset_issue_888(add_and_remove_cv_file): + """issue 888 test.""" + columns_list = ["data", "label"] + num_readers = 2 + data = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers, shuffle=False, num_shards=5, shard_id=1) + data = data.shuffle(2) + data = data.repeat(9) + num_iter = 0 + for item in data.create_dict_iterator(): + num_iter += 1 + assert num_iter == 18 + + +def test_cv_minddataset_blockreader_tutorial(add_and_remove_cv_file): + """tutorial for cv minddataset.""" + columns_list = ["data", "label"] + num_readers = 4 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers, + block_reader=True) + assert data_set.get_dataset_size() == 10 + repeat_num = 2 + data_set = data_set.repeat(repeat_num) + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- block reader repeat tow {} -----------------".format(num_iter)) + logger.info("-------------- item[label]: {} ----------------------------".format(item["label"])) + logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) + num_iter += 1 + assert num_iter == 20 + + +def test_cv_minddataset_reader_basic_tutorial(add_and_remove_cv_file): + """tutorial for cv minderdataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers) + assert data_set.get_dataset_size() == 10 + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- cv reader basic: {} ------------------------".format(num_iter)) + logger.info("-------------- len(item[data]): {} ------------------------".format(len(item["data"]))) + logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) + logger.info("-------------- item[file_name]: {} ------------------------".format(item["file_name"])) + logger.info("-------------- item[label]: {} ----------------------------".format(item["label"])) + num_iter += 1 + assert num_iter == 10 + +def test_nlp_minddataset_reader_basic_tutorial(add_and_remove_nlp_file): + """tutorial for nlp minderdataset.""" + num_readers = 4 + data_set = ds.MindDataset(NLP_FILE_NAME + "0", None, num_readers) + assert data_set.get_dataset_size() == 10 + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- cv reader basic: {} ------------------------".format(num_iter)) + logger.info("-------------- num_iter: {} ------------------------".format(num_iter)) + logger.info("-------------- item[id]: {} ------------------------".format(item["id"])) + logger.info("-------------- item[rating]: {} --------------------".format(item["rating"])) + logger.info("-------------- item[input_ids]: {}, shape: {} -----------------".format( + item["input_ids"], item["input_ids"].shape)) + logger.info("-------------- item[input_mask]: {}, shape: {} -----------------".format( + item["input_mask"], item["input_mask"].shape)) + logger.info("-------------- item[segment_ids]: {}, shape: {} -----------------".format( + item["segment_ids"], item["segment_ids"].shape)) + assert item["input_ids"].shape == (50,) + assert item["input_mask"].shape == (1, 50) + assert item["segment_ids"].shape == (2, 25) + num_iter += 1 + assert num_iter == 10 + + +def test_cv_minddataset_reader_basic_tutorial_5_epoch(add_and_remove_cv_file): + """tutorial for cv minderdataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers) + assert data_set.get_dataset_size() == 10 + for epoch in range(5): + num_iter = 0 + for data in data_set: + logger.info("data is {}".format(data)) + num_iter += 1 + assert num_iter == 10 + + data_set.reset() + + +def test_cv_minddataset_reader_basic_tutorial_5_epoch_with_batch(add_and_remove_cv_file): + """tutorial for cv minderdataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers) + + resize_height = 32 + resize_width = 32 + + # define map operations + decode_op = vision.Decode() + resize_op = vision.Resize((resize_height, resize_width), ds.transforms.vision.Inter.LINEAR) + + data_set = data_set.map(input_columns=["data"], operations=decode_op, num_parallel_workers=4) + data_set = data_set.map(input_columns=["data"], operations=resize_op, num_parallel_workers=4) + + data_set = data_set.batch(2) + assert data_set.get_dataset_size() == 5 + for epoch in range(5): + num_iter = 0 + for data in data_set: + logger.info("data is {}".format(data)) + num_iter += 1 + assert num_iter == 5 + + data_set.reset() + + +def test_cv_minddataset_reader_no_columns(add_and_remove_cv_file): + """tutorial for cv minderdataset.""" + data_set = ds.MindDataset(CV_FILE_NAME + "0") + assert data_set.get_dataset_size() == 10 + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- cv reader basic: {} ------------------------".format(num_iter)) + logger.info("-------------- len(item[data]): {} ------------------------".format(len(item["data"]))) + logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) + logger.info("-------------- item[file_name]: {} ------------------------".format(item["file_name"])) + logger.info("-------------- item[label]: {} ----------------------------".format(item["label"])) + num_iter += 1 + assert num_iter == 10 + + +def test_cv_minddataset_reader_repeat_tutorial(add_and_remove_cv_file): + """tutorial for cv minderdataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers) + repeat_num = 2 + data_set = data_set.repeat(repeat_num) + num_iter = 0 + for item in data_set.create_dict_iterator(): + logger.info("-------------- repeat two test {} ------------------------".format(num_iter)) + logger.info("-------------- len(item[data]): {} -----------------------".format(len(item["data"]))) + logger.info("-------------- item[data]: {} ----------------------------".format(item["data"])) + logger.info("-------------- item[file_name]: {} -----------------------".format(item["file_name"])) + logger.info("-------------- item[label]: {} ---------------------------".format(item["label"])) + num_iter += 1 + assert num_iter == 20 + + +def get_data(dir_name): + """ + usage: get data from imagenet dataset + params: + dir_name: directory containing folder images and annotation information + + """ + if not os.path.isdir(dir_name): + raise IOError("Directory {} not exists".format(dir_name)) + img_dir = os.path.join(dir_name, "images") + ann_file = os.path.join(dir_name, "annotation.txt") + with open(ann_file, "r") as file_reader: + lines = file_reader.readlines() + + data_list = [] + for line in lines: + try: + filename, label = line.split(",") + label = label.strip("\n") + with open(os.path.join(img_dir, filename), "rb") as file_reader: + img = file_reader.read() + data_json = {"file_name": filename, + "data": img, + "label": int(label)} + data_list.append(data_json) + except FileNotFoundError: + continue + return data_list + +def get_multi_bytes_data(file_name, bytes_num=3): + """ + Return raw data of multi-bytes dataset. + + Args: + file_name (str): String of multi-bytes dataset's path. + bytes_num (int): Number of bytes fields. + + Returns: + List + """ + if not os.path.exists(file_name): + raise IOError("map file {} not exists".format(file_name)) + dir_name = os.path.dirname(file_name) + with open(file_name, "r") as file_reader: + lines = file_reader.readlines() + data_list = [] + row_num = 0 + for line in lines: + try: + img10_path = line.strip('\n').split(" ") + img5 = [] + for path in img10_path[:bytes_num]: + with open(os.path.join(dir_name, path), "rb") as file_reader: + img5 += [file_reader.read()] + data_json = {"image_{}".format(i): img5[i] + for i in range(len(img5))} + data_json.update({"id": row_num}) + row_num += 1 + data_list.append(data_json) + except FileNotFoundError: + continue + return data_list + +def get_mkv_data(dir_name): + """ + Return raw data of Vehicle_and_Person dataset. + + Args: + dir_name (str): String of Vehicle_and_Person dataset's path. + + Returns: + List + """ + if not os.path.isdir(dir_name): + raise IOError("Directory {} not exists".format(dir_name)) + img_dir = os.path.join(dir_name, "Image") + label_dir = os.path.join(dir_name, "prelabel") + + data_list = [] + file_list = os.listdir(label_dir) + + index = 1 + for item in file_list: + if os.path.splitext(item)[1] == '.json': + file_path = os.path.join(label_dir, item) + + image_name = ''.join([os.path.splitext(item)[0], ".jpg"]) + image_path = os.path.join(img_dir, image_name) + + with open(file_path, "r") as load_f: + load_dict = json.load(load_f) + + if os.path.exists(image_path): + with open(image_path, "rb") as file_reader: + img = file_reader.read() + data_json = {"file_name": image_name, + "prelabel": str(load_dict), + "data": img, + "id": index} + data_list.append(data_json) + index += 1 + logger.info('{} images are missing'.format(len(file_list)-len(data_list))) + return data_list + +def get_nlp_data(dir_name, vocab_file, num): + """ + Return raw data of aclImdb dataset. + + Args: + dir_name (str): String of aclImdb dataset's path. + vocab_file (str): String of dictionary's path. + num (int): Number of sample. + + Returns: + List + """ + if not os.path.isdir(dir_name): + raise IOError("Directory {} not exists".format(dir_name)) + for root, dirs, files in os.walk(dir_name): + for index, file_name_extension in enumerate(files): + if index < num: + file_path = os.path.join(root, file_name_extension) + file_name, _ = file_name_extension.split('.', 1) + id_, rating = file_name.split('_', 1) + with open(file_path, 'r') as f: + raw_content = f.read() + + dictionary = load_vocab(vocab_file) + vectors = [dictionary.get('[CLS]')] + vectors += [dictionary.get(i) if i in dictionary + else dictionary.get('[UNK]') + for i in re.findall(r"[\w']+|[{}]" + .format(string.punctuation), + raw_content)] + vectors += [dictionary.get('[SEP]')] + input_, mask, segment = inputs(vectors) + input_ids = np.reshape(np.array(input_), [-1]) + input_mask = np.reshape(np.array(mask), [1, -1]) + segment_ids = np.reshape(np.array(segment), [2, -1]) + data = { + "label": 1, + "id": id_, + "rating": float(rating), + "input_ids": input_ids, + "input_mask": input_mask, + "segment_ids": segment_ids + } + yield data + +def convert_to_uni(text): + if isinstance(text, str): + return text + if isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + raise Exception("The type %s does not convert!" % type(text)) + +def load_vocab(vocab_file): + """load vocabulary to translate statement.""" + vocab = collections.OrderedDict() + vocab.setdefault('blank', 2) + index = 0 + with open(vocab_file) as reader: + while True: + tmp = reader.readline() + if not tmp: + break + token = convert_to_uni(tmp) + token = token.strip() + vocab[token] = index + index += 1 + return vocab + +def inputs(vectors, maxlen=50): + length = len(vectors) + if length > maxlen: + return vectors[0:maxlen], [1]*maxlen, [0]*maxlen + input_ = vectors+[0]*(maxlen-length) + mask = [1]*length + [0]*(maxlen-length) + segment = [0]*maxlen + return input_, mask, segment diff --git a/tests/ut/python/dataset/test_minddataset_exception.py b/tests/ut/python/dataset/test_minddataset_exception.py new file mode 100644 index 00000000000..70add46b686 --- /dev/null +++ b/tests/ut/python/dataset/test_minddataset_exception.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import os + +import pytest + +import mindspore.dataset as ds +from mindspore.mindrecord import FileWriter + +CV_FILE_NAME = "./imagenet.mindrecord" + + +def create_cv_mindrecord(files_num): + """tutorial for cv dataset writer.""" + os.remove(CV_FILE_NAME) if os.path.exists(CV_FILE_NAME) else None + os.remove("{}.db".format(CV_FILE_NAME)) if os.path.exists("{}.db".format(CV_FILE_NAME)) else None + writer = FileWriter(CV_FILE_NAME, files_num) + cv_schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}} + data = [{"file_name": "001.jpg", "label": 43, "data": bytes('0xffsafdafda', encoding='utf-8')}] + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + +def test_cv_lack_json(): + """tutorial for cv minderdataset.""" + create_cv_mindrecord(1) + columns_list = ["data", "file_name", "label"] + num_readers = 4 + with pytest.raises(Exception) as err: + data_set = ds.MindDataset(CV_FILE_NAME, "no_exist.json", columns_list, num_readers) + os.remove(CV_FILE_NAME) + os.remove("{}.db".format(CV_FILE_NAME)) + + +def test_cv_lack_mindrecord(): + """tutorial for cv minderdataset.""" + columns_list = ["data", "file_name", "label"] + num_readers = 4 + with pytest.raises(Exception, match="does not exist or permission denied"): + data_set = ds.MindDataset("no_exist.mindrecord", columns_list, num_readers) + + +def test_invalid_mindrecord(): + with open('dummy.mindrecord', 'w') as f: + f.write('just for test') + columns_list = ["data", "file_name", "label"] + num_readers = 4 + with pytest.raises(Exception, match="MindRecordOp init failed"): + data_set = ds.MindDataset('dummy.mindrecord', columns_list, num_readers) + num_iter = 0 + for item in data_set.create_dict_iterator(): + num_iter += 1 + assert num_iter == 0 + os.remove('dummy.mindrecord') + + +def test_minddataset_lack_db(): + create_cv_mindrecord(1) + os.remove("{}.db".format(CV_FILE_NAME)) + columns_list = ["data", "file_name", "label"] + num_readers = 4 + with pytest.raises(Exception, match="MindRecordOp init failed"): + data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers) + num_iter = 0 + for item in data_set.create_dict_iterator(): + num_iter += 1 + assert num_iter == 0 + os.remove(CV_FILE_NAME) diff --git a/tests/ut/python/dataset/test_minddataset_multi_images.py b/tests/ut/python/dataset/test_minddataset_multi_images.py new file mode 100644 index 00000000000..0a48c564c6f --- /dev/null +++ b/tests/ut/python/dataset/test_minddataset_multi_images.py @@ -0,0 +1,61 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This is the test module for mindrecord +""" +import mindspore.dataset as ds +from mindspore import log as logger + +FILES_NUM = 1 +CV_FILE_NAME = "../data/mindrecord/testTwoImageData/twobytes.mindrecord" + + +def test_cv_minddataset_reader_two_png_tutorial(): + """tutorial for cv minderdataset.""" + columns_list = ["id", "file_name", "label_name", "img_data", "label_data"] + num_readers = 1 + data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers) + assert data_set.get_dataset_size() == 5 + num_iter = 0 + for item in data_set.create_dict_iterator(): + assert len(item) == 5 + logger.info("-------------- cv reader basic is {} -----------------".format(num_iter)) + logger.info("-------------- item[id] is {} ------------------------".format(item["id"])) + logger.info("-------------- item[file_name] is {} -----------------".format(item["file_name"])) + logger.info("-------------- item[label_name] is {} ----------------".format(item["label_name"])) + logger.info("-------------- item[img_data] is {} ------------------".format(item["img_data"])) + logger.info("-------------- item[img_data][500:520] is {} ---------".format(item["img_data"][500:520])) + logger.info("-------------- item[label_data] is {} ----------------".format(item["label_data"])) + logger.info("-------------- item[label_data][500:520] is {} -------".format(item["label_data"][500:520])) + num_iter += 1 + assert num_iter == 5 + +def test_cv_minddataset_reader_two_png_tutorial_just_image2(): + """tutorial for cv minderdataset.""" + columns_list = ["img_data", "label_data"] + num_readers = 1 + data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers) + assert data_set.get_dataset_size() == 5 + num_iter = 0 + for item in data_set.create_dict_iterator(): + assert len(item) == 2 + logger.info("-------------- cv reader basic is {} -----------------".format(num_iter)) + logger.info("-------------- item[img_data] is {} ------------------".format(item["img_data"])) + logger.info("-------------- item[img_data][500:520] is {} ---------".format(item["img_data"][500:520])) + logger.info("-------------- item[label_data] is {} ----------------".format(item["label_data"])) + logger.info("-------------- item[label_data][500:520] is {} -------".format(item["label_data"][500:520])) + num_iter += 1 + assert num_iter == 5 + diff --git a/tests/ut/python/dataset/test_minddataset_multi_images_and_ndarray.py b/tests/ut/python/dataset/test_minddataset_multi_images_and_ndarray.py new file mode 100644 index 00000000000..a68557e2038 --- /dev/null +++ b/tests/ut/python/dataset/test_minddataset_multi_images_and_ndarray.py @@ -0,0 +1,80 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This is the test module for mindrecord +""" +import numpy as np +import os + +import mindspore.dataset as ds +from mindspore import log as logger + +from mindspore.mindrecord import FileWriter + +FILES_NUM = 1 +CV_FILE_NAME = "./complex.mindrecord" + +def test_cv_minddataset_reader_multi_image_and_ndarray_tutorial(): + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + cv_schema_json={"id": {"type": "int32"}, + "image_0": {"type": "bytes"}, + "image_2": {"type": "bytes"}, + "image_3": {"type": "bytes"}, + "image_4": {"type": "bytes"}, + "input_mask": {"type": "int32", "shape": [-1]}, + "segments": {"type": "float32", "shape": [2, 3]}} + writer.add_schema(cv_schema_json, "two_images_schema") + with open("../data/mindrecord/testImageNetData/images/image_00010.jpg", "rb") as file_reader: + img_data = file_reader.read() + ndarray_1 = np.array([1, 2, 3, 4, 5], np.int32) + ndarray_2 = np.array(([2, 3, 1], [7, 9, 0]), np.float32) + data = [] + for i in range(5): + item = {"id": i, "image_0": img_data, "image_2": img_data, "image_3": img_data, "image_4": img_data, + "input_mask": ndarray_1, "segments": ndarray_2} + data.append(item) + writer.write_raw_data(data) + writer.commit() + assert os.path.exists(CV_FILE_NAME) + assert os.path.exists(CV_FILE_NAME + ".db") + + """tutorial for minderdataset.""" + columns_list = ["id", "image_0", "image_2", "image_3", "image_4", "input_mask", "segments"] + num_readers = 1 + data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers) + assert data_set.get_dataset_size() == 5 + num_iter = 0 + for item in data_set.create_dict_iterator(): + assert len(item) == 7 + logger.info("item: {}".format(item)) + assert item["image_0"].dtype == np.uint8 + assert (item["image_0"] == item["image_2"]).all() + assert (item["image_3"] == item["image_4"]).all() + assert (item["image_0"] == item["image_4"]).all() + assert item["image_2"].dtype == np.uint8 + assert item["image_3"].dtype == np.uint8 + assert item["image_4"].dtype == np.uint8 + assert item["id"].dtype == np.int32 + assert item["input_mask"].shape == (5, ) + assert item["input_mask"].dtype == np.int32 + assert item["segments"].shape == (2, 3) + assert item["segments"].dtype == np.float32 + num_iter += 1 + assert num_iter == 5 + + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) diff --git a/tests/ut/python/dataset/test_mixup_label_smoothing.py b/tests/ut/python/dataset/test_mixup_label_smoothing.py new file mode 100644 index 00000000000..83c1cab7385 --- /dev/null +++ b/tests/ut/python/dataset/test_mixup_label_smoothing.py @@ -0,0 +1,164 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np +import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms as f +import mindspore.dataset as ds +import mindspore.dataset.transforms.c_transforms as c +import mindspore.dataset.transforms.vision.c_transforms as c_vision +from mindspore import log as logger + +DATA_DIR = "../data/dataset/testImageNetData/train" +DATA_DIR_2 = "../data/dataset/testImageNetData2/train" + + +def test_one_hot_op(): + """ + Test one hot encoding op + """ + logger.info("Test one hot encoding op") + + # define map operations + # ds = de.ImageFolderDataset(DATA_DIR, schema=SCHEMA_DIR) + dataset = ds.ImageFolderDatasetV2(DATA_DIR) + num_classes = 2 + epsilon_para = 0.1 + + transforms = [f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para), + ] + transform_label = py_vision.ComposeOp(transforms) + dataset = dataset.map(input_columns=["label"], operations=transform_label()) + + golden_label = np.ones(num_classes)*epsilon_para/num_classes + golden_label[1] = 1 - epsilon_para/num_classes + + for data in dataset.create_dict_iterator(): + label = data["label"] + logger.info("label is {}".format(label)) + logger.info("golden_label is {}".format(golden_label)) + assert(label.all() == golden_label.all()) + logger.info("====test one hot op ok====") + + +def test_mix_up_single(): + """ + Test single batch mix up op + """ + logger.info("Test single batch mix up op") + + resize_height = 224 + resize_width = 224 + + # Create dataset and define map operations + ds1 = ds.ImageFolderDatasetV2(DATA_DIR_2) + + num_classes = 10 + decode_op = c_vision.Decode() + resize_op = c_vision.Resize((resize_height, resize_width), c_vision.Inter.LINEAR) + one_hot_encode = c.OneHot(num_classes) # num_classes is input argument + + ds1 = ds1.map(input_columns=["image"], operations=decode_op) + ds1 = ds1.map(input_columns=["image"], operations=resize_op) + ds1 = ds1.map(input_columns=["label"], operations=one_hot_encode) + + # apply batch operations + batch_size = 3 + ds1 = ds1.batch(batch_size, drop_remainder=True) + + ds2 = ds1 + alpha = 0.2 + transforms = [py_vision.MixUp(batch_size=batch_size, alpha=alpha, is_single=True) + ] + ds1 = ds1.map(input_columns=["image", "label"], operations=transforms) + + for data1, data2 in zip(ds1.create_dict_iterator(), ds2.create_dict_iterator()): + image1 = data1["image"] + label = data1["label"] + logger.info("label is {}".format(label)) + + image2 = data2["image"] + label2 = data2["label"] + logger.info("label2 is {}".format(label2)) + + lam = np.abs(label - label2) + for index in range(batch_size-1): + if np.square(lam[index]).mean() != 0: + lam_value = 1 - np.sum(lam[index])/2 + img_golden = lam_value * image2[index] + (1-lam_value)*image2[index+1] + assert image1[index].all() == img_golden.all() + logger.info("====test single batch mixup ok====") + + +def test_mix_up_multi(): + """ + Test multi batch mix up op + """ + logger.info("Test several batch mix up op") + + resize_height = 224 + resize_width = 224 + + # Create dataset and define map operations + ds1 = ds.ImageFolderDatasetV2(DATA_DIR_2) + + num_classes = 3 + decode_op = c_vision.Decode() + resize_op = c_vision.Resize((resize_height, resize_width), c_vision.Inter.LINEAR) + one_hot_encode = c.OneHot(num_classes) # num_classes is input argument + + ds1 = ds1.map(input_columns=["image"], operations=decode_op) + ds1 = ds1.map(input_columns=["image"], operations=resize_op) + ds1 = ds1.map(input_columns=["label"], operations=one_hot_encode) + + # apply batch operations + batch_size = 3 + ds1 = ds1.batch(batch_size, drop_remainder=True) + + ds2 = ds1 + alpha = 0.2 + transforms = [py_vision.MixUp(batch_size=batch_size, alpha=alpha, is_single=False) + ] + ds1 = ds1.map(input_columns=["image", "label"], operations=transforms) + num_iter = 0 + batch1_image1 = 0 + for data1, data2 in zip(ds1.create_dict_iterator(), ds2.create_dict_iterator()): + image1 = data1["image"] + label1 = data1["label"] + logger.info("label: {}".format(label1)) + + image2 = data2["image"] + label2 = data2["label"] + logger.info("label2: {}".format(label2)) + + if num_iter == 0: + batch1_image1 = image1 + + if num_iter == 1: + lam = np.abs(label2 - label1) + logger.info("lam value in multi: {}".format(lam)) + for index in range(batch_size): + if np.square(lam[index]).mean() != 0: + lam_value = 1 - np.sum(lam[index])/2 + img_golden = lam_value * image2[index] + (1-lam_value)*batch1_image1[index] + assert image1[index].all() == img_golden.all() + logger.info("====test several batch mixup ok====") + break + num_iter = num_iter + 1 + + +if __name__ == "__main__": + test_one_hot_op() + test_mix_up_single() + test_mix_up_multi() diff --git a/tests/ut/python/dataset/test_normalizeOp.py b/tests/ut/python/dataset/test_normalizeOp.py new file mode 100644 index 00000000000..1abee961735 --- /dev/null +++ b/tests/ut/python/dataset/test_normalizeOp.py @@ -0,0 +1,148 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def normalize_np(image): + """ + Apply the normalization + """ + # DE decodes the image in RGB by deafult, hence + # the values here are in RGB + image = np.array(image, np.float32) + image = image - np.array([121.0, 115.0, 100.0]) + image = image * (1.0 / np.array([70.0, 68.0, 71.0])) + return image + + +def get_normalized(image_id): + """ + Reads the image using DE ops and then normalizes using Numpy + """ + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + data1 = data1.map(input_columns=["image"], operations=decode_op) + num_iter = 0 + for item in data1.create_dict_iterator(): + image = item["image"] + if num_iter == image_id: + return normalize_np(image) + num_iter += 1 + + +def test_normalize_op(): + """ + Test Normalize + """ + logger.info("Test Normalize") + + # define map operations + decode_op = vision.Decode() + normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=normalize_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + image_de_normalized = item1["image"] + image_np_normalized = normalize_np(item2["image"]) + diff = image_de_normalized - image_np_normalized + mse = np.sum(np.power(diff, 2)) + logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) + assert mse < 0.01 + # Uncomment these blocks to see visual results + # plt.subplot(131) + # plt.imshow(image_de_normalized) + # plt.title("DE normalize image") + # + # plt.subplot(132) + # plt.imshow(image_np_normalized) + # plt.title("Numpy normalized image") + # + # plt.subplot(133) + # plt.imshow(diff) + # plt.title("Difference image, mse : {}".format(mse)) + # + # plt.show() + num_iter += 1 + + +def test_decode_op(): + logger.info("Test Decode") + + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image", "label"], num_parallel_workers=1, + shuffle=False) + + # define map operations + decode_op = vision.Decode() + + # apply map operations on images + data1 = data1.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + image = None + for item in data1.create_dict_iterator(): + logger.info("Looping inside iterator {}".format(num_iter)) + image = item["image"] + # plt.subplot(131) + # plt.imshow(image) + # plt.title("DE image") + num_iter += 1 + + +def test_decode_normalize_op(): + logger.info("Test [Decode, Normalize] in one Map") + + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image", "label"], num_parallel_workers=1, + shuffle=False) + + # define map operations + decode_op = vision.Decode() + normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) + + # apply map operations on images + data1 = data1.map(input_columns=["image"], operations=[decode_op, normalize_op]) + + num_iter = 0 + image = None + for item in data1.create_dict_iterator(): + logger.info("Looping inside iterator {}".format(num_iter)) + image = item["image"] + # plt.subplot(131) + # plt.imshow(image) + # plt.title("DE image") + num_iter += 1 + break + + +if __name__ == "__main__": + test_decode_op() + test_decode_normalize_op() + test_normalize_op() diff --git a/tests/ut/python/dataset/test_onehot_op.py b/tests/ut/python/dataset/test_onehot_op.py new file mode 100644 index 00000000000..2ee40b5a5c6 --- /dev/null +++ b/tests/ut/python/dataset/test_onehot_op.py @@ -0,0 +1,65 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the one_hot op in DE +""" +import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.transforms.c_transforms as data_trans +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def one_hot(index, depth): + """ + Apply the one_hot + """ + arr = np.zeros([1, depth], dtype=np.int32) + arr[0, index] = 1 + return arr + + +def test_one_hot(): + """ + Test one_hot + """ + logger.info("Test one_hot") + + depth = 10 + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + one_hot_op = data_trans.OneHot(depth) + data1 = data1.map(input_columns=["label"], operations=one_hot_op, columns_order=["label"]) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["label"], shuffle=False) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + assert len(item1) == len(item2) + label1 = item1["label"] + label2 = one_hot(item2["label"][0], depth) + mse = np.sum(label1 - label2) + logger.info("DE one_hot: {}, Numpy one_hot: {}, diff: {}".format(label1, label2, mse)) + num_iter += 1 + + +if __name__ == "__main__": + test_one_hot() diff --git a/tests/ut/python/dataset/test_pad.py b/tests/ut/python/dataset/test_pad.py new file mode 100644 index 00000000000..4f92242bd6e --- /dev/null +++ b/tests/ut/python/dataset/test_pad.py @@ -0,0 +1,99 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing Pad op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as c_vision +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(first, mse, second): + """ + visualizes the image using DE op and enCV + """ + plt.subplot(141) + plt.imshow(first) + plt.title("c transformed image") + + plt.subplot(142) + plt.imshow(second) + plt.title("py random_color_jitter image") + + plt.subplot(143) + plt.imshow(first - second) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def diff_mse(in1, in2): + mse = (np.square(in1.astype(float) / 255 - in2.astype(float) / 255)).mean() + return mse * 100 + + +def test_pad_op(): + """ + Test Pad op + """ + logger.info("test_random_color_jitter_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + pad_op = c_vision.Pad((100, 100, 100, 100)) + ctrans = [decode_op, + pad_op, + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.Pad(100), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + + diff = c_image - py_image + mse = diff_mse(c_image, py_image) + logger.info("mse is {}".format(mse)) + assert mse < 0.01 + + +if __name__ == "__main__": + test_pad_op() diff --git a/tests/ut/python/dataset/test_project.py b/tests/ut/python/dataset/test_project.py new file mode 100644 index 00000000000..de600e07dbd --- /dev/null +++ b/tests/ut/python/dataset/test_project.py @@ -0,0 +1,158 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore._c_dataengine as de_map +from util import ordered_save_and_check + +import mindspore.dataset as ds + +DATA_DIR_TF = ["../data/dataset/testTFTestAllTypes/test.data"] +SCHEMA_DIR_TF = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +GENERATE_GOLDEN = False + + +def test_case_project_single_column(): + columns = ["col_sint32"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.project(columns=columns) + + filename = "project_single_column_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_project_multiple_columns_in_order(): + columns = ["col_sint16", "col_float", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.project(columns=columns) + + filename = "project_multiple_columns_in_order_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_project_multiple_columns_out_of_order(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.project(columns=columns) + + filename = "project_multiple_columns_out_of_order_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_project_map(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.project(columns=columns) + + no_op = de_map.NoOp() + + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + + filename = "project_map_after_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_map_project(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + + no_op = de_map.NoOp() + data1 = data1.map(input_columns=["col_sint64"], operations=no_op) + + data1 = data1.project(columns=columns) + + filename = "project_map_before_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_project_between_maps(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + + no_op = de_map.NoOp() + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + + data1 = data1.project(columns=columns) + + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + data1 = data1.map(input_columns=["col_3d"], operations=no_op) + + filename = "project_between_maps_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_project_repeat(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.project(columns=columns) + + repeat_count = 3 + data1 = data1.repeat(repeat_count) + + filename = "project_before_repeat_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_repeat_project(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + + repeat_count = 3 + data1 = data1.repeat(repeat_count) + + data1 = data1.project(columns=columns) + + filename = "project_after_repeat_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_map_project_map_project(): + columns = ["col_3d", "col_sint64", "col_2d"] + parameters = {"params": {'columns': columns}} + + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + + no_op = de_map.NoOp() + data1 = data1.map(input_columns=["col_sint64"], operations=no_op) + + data1 = data1.project(columns=columns) + + data1 = data1.map(input_columns=["col_2d"], operations=no_op) + + data1 = data1.project(columns=columns) + + filename = "project_alternate_parallel_inline_result.npz" + ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_pyfunc.py b/tests/ut/python/dataset/test_pyfunc.py new file mode 100644 index 00000000000..4b0672a1f2a --- /dev/null +++ b/tests/ut/python/dataset/test_pyfunc.py @@ -0,0 +1,191 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/testPyfuncMap/data.data"] +SCHEMA_DIR = "../data/dataset/testPyfuncMap/schema.json" +COLUMNS = ["col0", "col1", "col2"] +GENERATE_GOLDEN = False + + +def test_case_0(): + """ + Test PyFunc + """ + logger.info("Test 1-1 PyFunc : lambda x : x + x") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + data1 = data1.map(input_columns="col0", output_columns="out", operations=(lambda x: x + x)) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[i * 2, (i + 1) * 2], [(i + 2) * 2, (i + 3) * 2]]) + assert np.array_equal(item["out"], golden) + i = i + 4 + + +def test_case_1(): + """ + Test PyFunc + """ + logger.info("Test 1-n PyFunc : lambda x : (x , x + x) ") + + col = "col0" + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + data1 = data1.map(input_columns=col, output_columns=["out0", "out1"], operations=(lambda x: (x, x + x)), + columns_order=["out0", "out1"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item["out0"], golden) + golden = np.array([[i * 2, (i + 1) * 2], [(i + 2) * 2, (i + 3) * 2]]) + assert np.array_equal(item["out1"], golden) + i = i + 4 + + +def test_case_2(): + """ + Test PyFunc + """ + logger.info("Test n-1 PyFunc : lambda x, y : x + y ") + + col = ["col0", "col1"] + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + data1 = data1.map(input_columns=col, output_columns="out", operations=(lambda x, y: x + y), + columns_order=["out"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[i * 2, (i + 1) * 2], [(i + 2) * 2, (i + 3) * 2]]) + assert np.array_equal(item["out"], golden) + i = i + 4 + + +def test_case_3(): + """ + Test PyFunc + """ + logger.info("Test n-m PyFunc : lambda x, y : (x , x + 1, x + y)") + + col = ["col0", "col1"] + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], + operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item["out0"], golden) + golden = np.array([[i * 2, (i + 1) * 2], [(i + 2) * 2, (i + 3) * 2]]) + assert np.array_equal(item["out1"], golden) + golden = np.array([[i * 2 + 1, (i + 1) * 2 + 1], [(i + 2) * 2 + 1, (i + 3) * 2 + 1]]) + assert np.array_equal(item["out2"], golden) + i = i + 4 + + +def test_case_4(): + """ + Test PyFunc + """ + logger.info("Test Parallel n-m PyFunc : lambda x, y : (x , x + 1, x + y)") + + col = ["col0", "col1"] + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], num_parallel_workers=4, + operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[i, i + 1], [i + 2, i + 3]]) + assert np.array_equal(item["out0"], golden) + golden = np.array([[i * 2, (i + 1) * 2], [(i + 2) * 2, (i + 3) * 2]]) + assert np.array_equal(item["out1"], golden) + golden = np.array([[i * 2 + 1, (i + 1) * 2 + 1], [(i + 2) * 2 + 1, (i + 3) * 2 + 1]]) + assert np.array_equal(item["out2"], golden) + i = i + 4 + + +# The execution of this function will acquire GIL +def func_5(x): + return np.ones(x.shape, dtype=x.dtype) + + +def test_case_5(): + """ + Test PyFunc + """ + logger.info("Test 1-1 PyFunc : lambda x: np.ones(x.shape)") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + data1 = data1.map(input_columns="col0", output_columns="out", operations=func_5) + + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[1, 1], [1, 1]]) + assert np.array_equal(item["out"], golden) + + +def test_case_6(): + """ + Test PyFunc + """ + logger.info("Test PyFunc ComposeOp : (lambda x : x + x), (lambda x : x + x)") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + data1 = data1.map(input_columns="col0", output_columns="out", + operations=[(lambda x: x + x), (lambda x: x + x)]) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + # In this test, the dataset is 2x2 sequential tensors + golden = np.array([[i * 4, (i + 1) * 4], [(i + 2) * 4, (i + 3) * 4]]) + assert np.array_equal(item["out"], golden) + i = i + 4 + + +if __name__ == "__main__": + test_case_0() + test_case_1() + test_case_2() + test_case_3() + test_case_4() + test_case_5() + test_case_6() diff --git a/tests/ut/python/dataset/test_random_color_adjust.py b/tests/ut/python/dataset/test_random_color_adjust.py new file mode 100644 index 00000000000..57c77caf81f --- /dev/null +++ b/tests/ut/python/dataset/test_random_color_adjust.py @@ -0,0 +1,274 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomRotation op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as c_vision +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(first, mse, second): + """ + visualizes the image using DE op and enCV + """ + plt.subplot(141) + plt.imshow(first) + plt.title("c transformed image") + + plt.subplot(142) + plt.imshow(second) + plt.title("py random_color_jitter image") + + plt.subplot(143) + plt.imshow(first - second) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def diff_mse(in1, in2): + mse = (np.square(in1.astype(float) / 255 - in2.astype(float) / 255)).mean() + return mse * 100 + + +def test_random_color_jitter_op_brightness(): + """ + Test RandomColorAdjust op + """ + logger.info("test_random_color_jitter_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + random_jitter_op = c_vision.RandomColorAdjust((0.8, 0.8), (1, 1), (1, 1), (0, 0)) + + ctrans = [decode_op, + random_jitter_op, + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.RandomColorAdjust((0.8, 0.8), (1, 1), (1, 1), (0, 0)), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + + mse = diff_mse(c_image, py_image) + logger.info("mse is {}".format(mse)) + assert mse < 0.01 + # logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) + # if mse != 0: + # logger.info("mse is: {}".format(mse)) + # Uncomment below line if you want to visualize images + # visualize(c_image, mse, py_image) + + +def test_random_color_jitter_op_contrast(): + """ + Test RandomColorAdjust op + """ + logger.info("test_random_color_jitter_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + random_jitter_op = c_vision.RandomColorAdjust((1, 1), (0.5, 0.5), (1, 1), (0, 0)) + + ctrans = [decode_op, + random_jitter_op + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.RandomColorAdjust((1, 1), (0.5, 0.5), (1, 1), (0, 0)), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + + diff = c_image - py_image + logger.info("contrast difference c is : {}".format(c_image[0][0])) + logger.info("contrast difference py is : {}".format(py_image[0][0])) + + logger.info("contrast difference is : {}".format(diff[0][0])) + # mse = (np.sum(np.power(diff, 2))) / (c_image.shape[0] * c_image.shape[1]) + mse = diff_mse(c_image, py_image) + logger.info("mse is {}".format(mse)) + # assert mse < 0.01 + # logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) + # if mse != 0: + # logger.info("mse is: {}".format(mse)) + # Uncomment below line if you want to visualize images + # visualize(c_image, mse, py_image) + + +def test_random_color_jitter_op_saturation(): + """ + Test RandomColorAdjust op + """ + logger.info("test_random_color_jitter_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + random_jitter_op = c_vision.RandomColorAdjust((1, 1), (1, 1), (0.5, 0.5), (0, 0)) + + ctrans = [decode_op, + random_jitter_op + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.RandomColorAdjust((1, 1), (1, 1), (0.5, 0.5), (0, 0)), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + # data2 = data2.map(input_columns=["image"], operations=decode_op) + # data2 = data2.map(input_columns=["image"], operations=c_vision.Decode()) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + + diff = c_image - py_image + + mse = diff_mse(c_image, py_image) + logger.info("mse is {}".format(mse)) + assert mse < 0.01 + # logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) + # if mse != 0: + # logger.info("mse is: {}".format(mse)) + # Uncomment below line if you want to visualize images + # visualize(c_image, mse, py_image) + + +def test_random_color_jitter_op_hue(): + """ + Test RandomColorAdjust op + """ + logger.info("test_random_color_jitter_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + # channel_swap_op = c_vision.ChannelSwap() + # change_mode_op = c_vision.ChangeMode() + + random_jitter_op = c_vision.RandomColorAdjust((1, 1), (1, 1), (1, 1), (0.2, 0.2)) + + ctrans = [decode_op, + random_jitter_op, + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.RandomColorAdjust((1, 1), (1, 1), (1, 1), (0.2, 0.2)), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + # logger.info("shape of img: {}".format(img.shape)) + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + # logger.info("dtype of img: {}".format(img.dtype)) + + diff = c_image - py_image + # mse = (np.sum(np.power(diff, 2))) / (c_image.shape[0] * c_image.shape[1]) + mse = diff_mse(c_image, py_image) + logger.info("mse is {}".format(mse)) + assert mse < 0.01 + # logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) + # if mse != 0: + # logger.info("mse is: {}".format(mse)) + # Uncomment below line if you want to visualize images + # visualize(c_image, mse, py_image) + + +if __name__ == "__main__": + test_random_color_jitter_op_brightness() + test_random_color_jitter_op_contrast() + test_random_color_jitter_op_saturation() + test_random_color_jitter_op_hue() diff --git a/tests/ut/python/dataset/test_random_crop.py b/tests/ut/python/dataset/test_random_crop.py new file mode 100644 index 00000000000..81e779e9f2a --- /dev/null +++ b/tests/ut/python/dataset/test_random_crop.py @@ -0,0 +1,69 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomCropAndResize op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore import log as logger + +import mindspore.dataset as ds + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(a, mse, original): + """ + visualizes the image using DE op and Numpy op + """ + plt.subplot(141) + plt.imshow(original) + plt.title("Original image") + + plt.subplot(142) + plt.imshow(a) + plt.title("DE random_crop image") + + plt.subplot(143) + plt.imshow(a - original) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_random_crop_op(): + """ + Test RandomCropAndResize op + """ + logger.info("test_random_crop_and_resize_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + random_crop_op = vision.RandomCrop([512, 512], [200, 200, 200, 200]) + decode_op = vision.Decode() + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_crop_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + image1 = item1["image"] + image2 = item2["image"] + + +if __name__ == "__main__": + test_random_crop_op() diff --git a/tests/ut/python/dataset/test_random_crop_and_resize.py b/tests/ut/python/dataset/test_random_crop_and_resize.py new file mode 100644 index 00000000000..f1deef42df7 --- /dev/null +++ b/tests/ut/python/dataset/test_random_crop_and_resize.py @@ -0,0 +1,82 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomCropAndResize op in DE +""" +import cv2 +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(a, mse, original): + """ + visualizes the image using DE op and Numpy op + """ + plt.subplot(141) + plt.imshow(original) + plt.title("Original image") + + plt.subplot(142) + plt.imshow(a) + plt.title("DE random_crop_and_resize image") + + plt.subplot(143) + plt.imshow(a - original) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_random_crop_and_resize_op(): + """ + Test RandomCropAndResize op + """ + logger.info("test_random_crop_and_resize_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + random_crop_and_resize_op = vision.RandomResizedCrop((256, 512), (1, 1), (0.5, 0.5)) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_crop_and_resize_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + + if num_iter > 0: + break + crop_and_resize = item1["image"] + original = item2["image"] + original = cv2.resize(original, (512, 256)) + diff = crop_and_resize - original + mse = np.sum(np.power(diff, 2)) + logger.info("random_crop_and_resize_op_{}, mse: {}".format(num_iter + 1, mse)) + # Uncomment below line if you want to visualize images + # visualize(crop_and_resize, mse, original) + num_iter += 1 + + +if __name__ == "__main__": + test_random_crop_and_resize_op() diff --git a/tests/ut/python/dataset/test_random_crop_decode_resize.py b/tests/ut/python/dataset/test_random_crop_decode_resize.py new file mode 100644 index 00000000000..ffca9326e69 --- /dev/null +++ b/tests/ut/python/dataset/test_random_crop_decode_resize.py @@ -0,0 +1,81 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomCropDecodeResize op in DE +""" +import cv2 +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(a, mse, original): + """ + visualizes the image using DE op and Numpy Op + """ + plt.subplot(141) + plt.imshow(original) + plt.title("Original image") + + plt.subplot(142) + plt.imshow(a) + plt.title("DE random_crop_decode_resize image") + + plt.subplot(143) + plt.imshow(a - original) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_random_crop_decode_resize_op(): + """ + Test RandomCropDecodeResize op + """ + logger.info("test_random_decode_resize_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + random_crop_decode_resize_op = vision.RandomCropDecodeResize((256, 512), (1, 1), (0.5, 0.5)) + data1 = data1.map(input_columns=["image"], operations=random_crop_decode_resize_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + + if num_iter > 0: + break + crop_and_resize = item1["image"] + original = item2["image"] + original = cv2.resize(original, (512, 256)) + diff = crop_and_resize - original + mse = np.sum(np.power(diff, 2)) + logger.info("random_crop_decode_resize_op_{}, mse: {}".format(num_iter + 1, mse)) + # Uncomment below line if you want to visualize images + # visualize(crop_and_resize, mse, original) + num_iter += 1 + + +if __name__ == "__main__": + test_random_crop_decode_resize_op() diff --git a/tests/ut/python/dataset/test_random_erasing.py b/tests/ut/python/dataset/test_random_erasing.py new file mode 100644 index 00000000000..b82b03d1556 --- /dev/null +++ b/tests/ut/python/dataset/test_random_erasing.py @@ -0,0 +1,89 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomRotation op in DE +""" +import matplotlib.pyplot as plt +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(image_1, image_2): + """ + visualizes the image using RandomErasing and Cutout + """ + plt.subplot(141) + plt.imshow(image_1) + plt.title("RandomErasing") + + plt.subplot(142) + plt.imshow(image_2) + plt.title("Cutout") + + plt.subplot(143) + plt.imshow(image_1 - image_2) + plt.title("Difference image") + plt.show() + + +def test_random_erasing_op(): + """ + Test RandomErasing and Cutout + """ + logger.info("test_random_erasing") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms_1 = [ + vision.Decode(), + vision.ToTensor(), + vision.RandomErasing(value='random') + ] + transform_1 = vision.ComposeOp(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms_2 = [ + vision.Decode(), + vision.ToTensor(), + vision.Cutout(80) + ] + transform_2 = vision.ComposeOp(transforms_2) + data2 = data2.map(input_columns=["image"], operations=transform_2()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + image_2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of image_1: {}".format(image_1.shape)) + logger.info("shape of image_2: {}".format(image_2.shape)) + + logger.info("dtype of image_1: {}".format(image_1.dtype)) + logger.info("dtype of image_2: {}".format(image_2.dtype)) + + # visualize(image_1, image_2) + + +if __name__ == "__main__": + test_random_erasing_op() diff --git a/tests/ut/python/dataset/test_random_horizontal_flip.py b/tests/ut/python/dataset/test_random_horizontal_flip.py new file mode 100644 index 00000000000..68d95a063a2 --- /dev/null +++ b/tests/ut/python/dataset/test_random_horizontal_flip.py @@ -0,0 +1,100 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the random horizontal flip op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def h_flip(image): + """ + Apply the random_horizontal + """ + + # with the seed provided in this test case, it will always flip. + # that's why we flip here too + image = image[:, ::-1, :] + return image + + +def visualize(image_de_random_horizontal, image_pil_random_horizontal, mse, image_original): + """ + visualizes the image using DE op and Numpy op + """ + plt.subplot(141) + plt.imshow(image_original) + plt.title("Original image") + + plt.subplot(142) + plt.imshow(image_de_random_horizontal) + plt.title("DE random_horizontal image") + + plt.subplot(143) + plt.imshow(image_pil_random_horizontal) + plt.title("Horizontally flipped image") + + plt.subplot(144) + plt.imshow(image_de_random_horizontal - image_pil_random_horizontal) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_random_horizontal_op(): + """ + Test random_horizontal + """ + logger.info("Test random_horizontal") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + random_horizontal_op = vision.RandomHorizontalFlip() + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_horizontal_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + + # with the seed value, we can only guarantee the first number generated + if num_iter > 0: + break + + image_h_flipped = item1["image"] + + image = item2["image"] + image_h_flipped_2 = h_flip(image) + + diff = image_h_flipped - image_h_flipped_2 + mse = np.sum(np.power(diff, 2)) + logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) + # Uncomment below line if you want to visualize images + # visualize(image_h_flipped, image_h_flipped_2, mse, image) + num_iter += 1 + + +if __name__ == "__main__": + test_random_horizontal_op() diff --git a/tests/ut/python/dataset/test_random_resize.py b/tests/ut/python/dataset/test_random_resize.py new file mode 100644 index 00000000000..629e0a9f9c5 --- /dev/null +++ b/tests/ut/python/dataset/test_random_resize.py @@ -0,0 +1,71 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the resize op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(image_de_resized, image_np_resized, mse): + """ + visualizes the image using DE op and Numpy op + """ + plt.subplot(131) + plt.imshow(image_de_resized) + plt.title("DE resize image") + + plt.subplot(132) + plt.imshow(image_np_resized) + plt.title("Numpy resized image") + + plt.subplot(133) + plt.imshow(image_de_resized - image_np_resized) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_random_resize_op(): + """ + Test random_resize_op + """ + logger.info("Test resize") + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + + # define map operations + decode_op = vision.Decode() + resize_op = vision.RandomResize(10) + + # apply map operations on images + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=resize_op) + + num_iter = 0 + for item in data1.create_dict_iterator(): + image_de_resized = item["image"] + # Uncomment below line if you want to visualize images + # visualize(image_de_resized, image_np_resized, mse) + num_iter += 1 + + +if __name__ == "__main__": + test_random_resize_op() diff --git a/tests/ut/python/dataset/test_random_rotation.py b/tests/ut/python/dataset/test_random_rotation.py new file mode 100644 index 00000000000..0654f9b863c --- /dev/null +++ b/tests/ut/python/dataset/test_random_rotation.py @@ -0,0 +1,153 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomRotation op in DE +""" +import cv2 +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as c_vision +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def visualize(a, mse, original): + """ + visualizes the image using DE op and enCV + """ + plt.subplot(141) + plt.imshow(original) + plt.title("Original image") + + plt.subplot(142) + plt.imshow(a) + plt.title("DE random_crop_and_resize image") + + plt.subplot(143) + plt.imshow(a - original) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def diff_mse(in1, in2): + mse = (np.square(in1.astype(float) / 255 - in2.astype(float) / 255)).mean() + return mse * 100 + + +def test_random_rotation_op(): + """ + Test RandomRotation op + """ + logger.info("test_random_rotation_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + decode_op = c_vision.Decode() + # use [90, 90] to force rotate 90 degrees, expand is set to be True to match output size + random_rotation_op = c_vision.RandomRotation((90, 90), expand=True) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_rotation_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + if num_iter > 0: + break + rotation = item1["image"] + original = item2["image"] + logger.info("shape before rotate: {}".format(original.shape)) + original = cv2.rotate(original, cv2.ROTATE_90_COUNTERCLOCKWISE) + diff = rotation - original + mse = np.sum(np.power(diff, 2)) + logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) + assert mse == 0 + # Uncomment below line if you want to visualize images + # visualize(rotation, mse, original) + num_iter += 1 + + +def test_random_rotation_expand(): + """ + Test RandomRotation op + """ + logger.info("test_random_rotation_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + # use [90, 90] to force rotate 90 degrees, expand is set to be True to match output size + random_rotation_op = c_vision.RandomRotation((0, 90), expand=True) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_rotation_op) + + num_iter = 0 + for item in data1.create_dict_iterator(): + rotation = item["image"] + logger.info("shape after rotate: {}".format(rotation.shape)) + num_iter += 1 + + +def test_rotation_diff(): + """ + Test Rotation op + """ + logger.info("test_random_rotation_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + rotation_op = c_vision.RandomRotation((45, 45), expand=True) + ctrans = [decode_op, + rotation_op + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.RandomRotation((45, 45), expand=True), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + + +if __name__ == "__main__": + test_random_rotation_op() + test_random_rotation_expand() + test_rotation_diff() diff --git a/tests/ut/python/dataset/test_random_vertical_flip.py b/tests/ut/python/dataset/test_random_vertical_flip.py new file mode 100644 index 00000000000..0d7404d1a29 --- /dev/null +++ b/tests/ut/python/dataset/test_random_vertical_flip.py @@ -0,0 +1,100 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the random vertical flip op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def v_flip(image): + """ + Apply the random_vertical + """ + + # with the seed provided in this test case, it will always flip. + # that's why we flip here too + image = image[::-1, :, :] + return image + + +def visualize(image_de_random_vertical, image_pil_random_vertical, mse, image_original): + """ + visualizes the image using DE op and Numpy op + """ + plt.subplot(141) + plt.imshow(image_original) + plt.title("Original image") + + plt.subplot(142) + plt.imshow(image_de_random_vertical) + plt.title("DE random_vertical image") + + plt.subplot(143) + plt.imshow(image_pil_random_vertical) + plt.title("vertically flipped image") + + plt.subplot(144) + plt.imshow(image_de_random_vertical - image_pil_random_vertical) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_random_vertical_op(): + """ + Test random_vertical + """ + logger.info("Test random_vertical") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + random_vertical_op = vision.RandomVerticalFlip() + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_vertical_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=decode_op) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + + # with the seed value, we can only guarantee the first number generated + if num_iter > 0: + break + + image_v_flipped = item1["image"] + + image = item2["image"] + image_v_flipped_2 = v_flip(image) + + diff = image_v_flipped - image_v_flipped_2 + mse = np.sum(np.power(diff, 2)) + logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) + # Uncomment below line if you want to visualize images + # visualize(image_v_flipped, image_v_flipped_2, mse, image) + num_iter += 1 + + +if __name__ == "__main__": + test_random_vertical_op() diff --git a/tests/ut/python/dataset/test_readdir.py b/tests/ut/python/dataset/test_readdir.py new file mode 100644 index 00000000000..12649c65977 --- /dev/null +++ b/tests/ut/python/dataset/test_readdir.py @@ -0,0 +1,69 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = "../data/dataset/test_tf_file_3_images/data" +SCHEMA = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +COLUMNS = ["label"] +GENERATE_GOLDEN = False + + +def test_case_0(): + logger.info("Test 0 readdir") + + # apply dataset operations + data1 = ds.engine.Dataset.read_dir(DATA_DIR, SCHEMA, columns_list=None, num_parallel_workers=None, + deterministic_output=True, prefetch_size=None, shuffle=False, seed=None) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + logger.info("item[label] is {}".format(item["label"])) + i = i + 1 + assert (i == 3) + + +def test_case_1(): + logger.info("Test 1 readdir") + + # apply dataset operations + data1 = ds.engine.Dataset.read_dir(DATA_DIR, SCHEMA, COLUMNS, num_parallel_workers=None, + deterministic_output=True, prefetch_size=None, shuffle=True, seed=None) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + logger.info("item[label] is {}".format(item["label"])) + i = i + 1 + assert (i == 3) + + +def test_case_2(): + logger.info("Test 2 readdir") + + # apply dataset operations + data1 = ds.engine.Dataset.read_dir(DATA_DIR, SCHEMA, columns_list=None, num_parallel_workers=2, + deterministic_output=False, prefetch_size=16, shuffle=True, seed=10) + + i = 0 + for item in data1.create_dict_iterator(): # each data is a dictionary + logger.info("item[label] is {}".format(item["label"])) + i = i + 1 + assert (i == 3) + + +if __name__ == "__main__": + test_case_0() + test_case_1() + test_case_2() diff --git a/tests/ut/python/dataset/test_rename.py b/tests/ut/python/dataset/test_rename.py new file mode 100644 index 00000000000..a1d207b1166 --- /dev/null +++ b/tests/ut/python/dataset/test_rename.py @@ -0,0 +1,49 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/testTFBert5Rows1/5TFDatas.data"] +DATA_DIR_2 = ["../data/dataset/testTFBert5Rows2/5TFDatas.data"] +SCHEMA_DIR = "../data/dataset/testTFBert5Rows1/datasetSchema.json" +SCHEMA_DIR_2 = "../data/dataset/testTFBert5Rows2/datasetSchema.json" + + +def test_rename(): + data1 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2, shuffle=False) + data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2, shuffle=False) + + data2 = data2.rename(input_columns=["input_ids", "segment_ids"], output_columns=["masks", "seg_ids"]) + + data = ds.zip((data1, data2)) + data = data.repeat(3) + + num_iter = 0 + + for i, item in enumerate(data.create_dict_iterator()): + logger.info("item[mask] is {}".format(item["masks"])) + assert item["masks"].all() == item["input_ids"].all() + logger.info("item[seg_ids] is {}".format(item["seg_ids"])) + assert item["segment_ids"].all() == item["seg_ids"].all() + # need to consume the data in the buffer + num_iter += 1 + logger.info("Number of data in data: {}".format(num_iter)) + assert num_iter == 15 + + +if __name__ == '__main__': + logger.info('===========test Rename Repeat===========') + test_rename() + logger.info('\n') diff --git a/tests/ut/python/dataset/test_repeat.py b/tests/ut/python/dataset/test_repeat.py new file mode 100644 index 00000000000..196a62c3156 --- /dev/null +++ b/tests/ut/python/dataset/test_repeat.py @@ -0,0 +1,107 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset.transforms.vision.c_transforms as vision +from util import save_and_check + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR_TF = ["../data/dataset/testTFTestAllTypes/test.data"] +SCHEMA_DIR_TF = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +COLUMNS_TF = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", + "col_sint16", "col_sint32", "col_sint64"] +GENERATE_GOLDEN = False + +# Data for CIFAR and MNIST are not part of build tree +# They need to be downloaded directly +# prep_data.py can be exuted or code below +# import sys +# sys.path.insert(0,"../../data") +# import prep_data +# prep_data.download_all_for_test("../../data") +IMG_DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +IMG_SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + +DATA_DIR_TF2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR_TF2 = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def test_tf_repeat_01(): + """ + a simple repeat operation. + """ + logger.info("Test Simple Repeat") + # define parameters + repeat_count = 2 + parameters = {"params": {'repeat_count': repeat_count}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.repeat(repeat_count) + + filename = "repeat_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_tf_repeat_02(): + """ + a simple repeat operation to tes infinite + """ + logger.info("Test Infinite Repeat") + # define parameters + repeat_count = -1 + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False) + data1 = data1.repeat(repeat_count) + + itr = 0 + for _ in data1: + itr = itr + 1 + if itr == 100: + break + assert itr == 100 + + +def test_tf_repeat_03(): + '''repeat and batch ''' + data1 = ds.TFRecordDataset(DATA_DIR_TF2, SCHEMA_DIR_TF2, shuffle=False) + + batch_size = 32 + resize_height, resize_width = 32, 32 + decode_op = vision.Decode() + resize_op = vision.Resize((resize_height, resize_width), interpolation=ds.transforms.vision.Inter.LINEAR) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=resize_op) + data1 = data1.repeat(22) + data1 = data1.batch(batch_size, drop_remainder=True) + + num_iter = 0 + for item in data1.create_dict_iterator(): + num_iter += 1 + logger.info("Number of tf data in data1: {}".format(num_iter)) + assert num_iter == 2 + + +if __name__ == "__main__": + logger.info("--------test tf repeat 01---------") + # test_repeat_01() + + logger.info("--------test tf repeat 02---------") + # test_repeat_02() + + logger.info("--------test tf repeat 03---------") + test_tf_repeat_03() + diff --git a/tests/ut/python/dataset/test_rescale_op.py b/tests/ut/python/dataset/test_rescale_op.py new file mode 100644 index 00000000000..4d265e0aab9 --- /dev/null +++ b/tests/ut/python/dataset/test_rescale_op.py @@ -0,0 +1,101 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the rescale op in DE +""" +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as vision +import numpy as np + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def rescale_np(image): + """ + Apply the rescale + """ + image = image / 255.0 + image = image - 1.0 + return image + + +def get_rescaled(image_id): + """ + Reads the image using DE ops and then rescales using Numpy + """ + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + data1 = data1.map(input_columns=["image"], operations=decode_op) + num_iter = 0 + for item in data1.create_dict_iterator(): + image = item["image"] + if num_iter == image_id: + return rescale_np(image) + num_iter += 1 + + return None + + +def visualize(image_de_rescaled, image_np_rescaled, mse): + """ + visualizes the image using DE op and Numpy op + """ + plt.subplot(131) + plt.imshow(image_de_rescaled) + plt.title("DE rescale image") + + plt.subplot(132) + plt.imshow(image_np_rescaled) + plt.title("Numpy rescaled image") + + plt.subplot(133) + plt.imshow(image_de_rescaled - image_np_rescaled) + plt.title("Difference image, mse : {}".format(mse)) + plt.show() + + +def test_rescale_op(): + """ + Test rescale + """ + logger.info("Test rescale") + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + + # define map operations + decode_op = vision.Decode() + rescale_op = vision.Rescale(1.0 / 255.0, -1.0) + + # apply map operations on images + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=rescale_op) + + num_iter = 0 + for item in data1.create_dict_iterator(): + image_de_rescaled = item["image"] + image_np_rescaled = get_rescaled(num_iter) + diff = image_de_rescaled - image_np_rescaled + mse = np.sum(np.power(diff, 2)) + logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) + # Uncomment below line if you want to visualize images + # visualize(image_de_rescaled, image_np_rescaled, mse) + num_iter += 1 + + +if __name__ == "__main__": + test_rescale_op() diff --git a/tests/ut/python/dataset/test_rgb_hsv.py b/tests/ut/python/dataset/test_rgb_hsv.py new file mode 100644 index 00000000000..e887acb01da --- /dev/null +++ b/tests/ut/python/dataset/test_rgb_hsv.py @@ -0,0 +1,168 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RgbToHsv and HsvToRgb op in DE +""" + +import numpy as np +from numpy.testing import assert_allclose +import colorsys + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as vision +import mindspore.dataset.transforms.vision.py_transforms_util as util + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + +def generate_numpy_random_rgb(shape): + # Only generate floating points that are fractions like n / 256, since they + # are RGB pixels. Some low-precision floating point types in this test can't + # handle arbitrary precision floating points well. + return np.random.randint(0, 256, shape) / 255. + + +def test_rgb_hsv_hwc(): + rgb_flat = generate_numpy_random_rgb((64, 3)).astype(np.float32) + rgb_np = rgb_flat.reshape((8, 8, 3)) + hsv_base = np.array([ + colorsys.rgb_to_hsv( + r.astype(np.float64), g.astype(np.float64), b.astype(np.float64)) + for r, g, b in rgb_flat + ]) + hsv_base = hsv_base.reshape((8, 8, 3)) + hsv_de = util.rgb_to_hsvs(rgb_np, True) + assert hsv_base.shape == hsv_de.shape + assert_allclose(hsv_base.flatten(), hsv_de.flatten(), rtol=1e-5, atol=0) + + hsv_flat = hsv_base.reshape(64, 3) + rgb_base = np.array([ + colorsys.hsv_to_rgb( + h.astype(np.float64), s.astype(np.float64), v.astype(np.float64)) + for h, s, v in hsv_flat + ]) + rgb_base = rgb_base.reshape((8, 8, 3)) + rgb_de = util.hsv_to_rgbs(hsv_base, True) + assert rgb_base.shape == rgb_de.shape + assert_allclose(rgb_base.flatten(), rgb_de.flatten(), rtol=1e-5, atol=0) + + +def test_rgb_hsv_batch_hwc(): + rgb_flat = generate_numpy_random_rgb((64, 3)).astype(np.float32) + rgb_np = rgb_flat.reshape((4, 2, 8, 3)) + hsv_base = np.array([ + colorsys.rgb_to_hsv( + r.astype(np.float64), g.astype(np.float64), b.astype(np.float64)) + for r, g, b in rgb_flat + ]) + hsv_base = hsv_base.reshape((4, 2, 8, 3)) + hsv_de = util.rgb_to_hsvs(rgb_np, True) + assert hsv_base.shape == hsv_de.shape + assert_allclose(hsv_base.flatten(), hsv_de.flatten(), rtol=1e-5, atol=0) + + hsv_flat = hsv_base.reshape((64, 3)) + rgb_base = np.array([ + colorsys.hsv_to_rgb( + h.astype(np.float64), s.astype(np.float64), v.astype(np.float64)) + for h, s, v in hsv_flat + ]) + rgb_base = rgb_base.reshape((4, 2, 8, 3)) + rgb_de = util.hsv_to_rgbs(hsv_base, True) + assert rgb_de.shape == rgb_base.shape + assert_allclose(rgb_base.flatten(), rgb_de.flatten(), rtol=1e-5, atol=0) + + +def test_rgb_hsv_chw(): + rgb_flat = generate_numpy_random_rgb((64, 3)).astype(np.float32) + rgb_np = rgb_flat.reshape((3, 8, 8)) + hsv_base = np.array([ + np.vectorize(colorsys.rgb_to_hsv)( + rgb_np[0, :, :].astype(np.float64), rgb_np[1, :, :].astype(np.float64), rgb_np[2, :, :].astype(np.float64)) + ]) + hsv_base = hsv_base.reshape((3, 8, 8)) + hsv_de = util.rgb_to_hsvs(rgb_np, False) + assert hsv_base.shape == hsv_de.shape + assert_allclose(hsv_base.flatten(), hsv_de.flatten(), rtol=1e-5, atol=0) + + rgb_base = np.array([ + np.vectorize(colorsys.hsv_to_rgb)( + hsv_base[0, :, :].astype(np.float64), hsv_base[1, :, :].astype(np.float64), + hsv_base[2, :, :].astype(np.float64)) + ]) + rgb_base = rgb_base.reshape((3, 8, 8)) + rgb_de = util.hsv_to_rgbs(hsv_base, False) + assert rgb_de.shape == rgb_base.shape + assert_allclose(rgb_base.flatten(), rgb_de.flatten(), rtol=1e-5, atol=0) + + +def test_rgb_hsv_batch_chw(): + rgb_flat = generate_numpy_random_rgb((64, 3)).astype(np.float32) + rgb_imgs = rgb_flat.reshape((4, 3, 2, 8)) + hsv_base_imgs = np.array([ + np.vectorize(colorsys.rgb_to_hsv)( + img[0, :, :].astype(np.float64), img[1, :, :].astype(np.float64), img[2, :, :].astype(np.float64)) + for img in rgb_imgs + ]) + hsv_de = util.rgb_to_hsvs(rgb_imgs, False) + assert hsv_base_imgs.shape == hsv_de.shape + assert_allclose(hsv_base_imgs.flatten(), hsv_de.flatten(), rtol=1e-5, atol=0) + + rgb_base = np.array([ + np.vectorize(colorsys.hsv_to_rgb)( + img[0, :, :].astype(np.float64), img[1, :, :].astype(np.float64), img[2, :, :].astype(np.float64)) + for img in hsv_base_imgs + ]) + rgb_de = util.hsv_to_rgbs(hsv_base_imgs, False) + assert rgb_base.shape == rgb_de.shape + assert_allclose(rgb_base.flatten(), rgb_de.flatten(), rtol=1e-5, atol=0) + + +def test_rgb_hsv_pipeline(): + # First dataset + transforms1 = [ + vision.Decode(), + vision.ToTensor() + ] + transforms1 = vision.ComposeOp(transforms1) + ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + ds1 = ds1.map(input_columns=["image"], operations=transforms1()) + + # Second dataset + transforms2 = [ + vision.Decode(), + vision.ToTensor(), + vision.RgbToHsv(), + vision.HsvToRgb() + ] + transform2 = vision.ComposeOp(transforms2) + ds2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + ds2 = ds2.map(input_columns=["image"], operations=transform2()) + + num_iter = 0 + for data1, data2 in zip(ds1.create_dict_iterator(), ds2.create_dict_iterator()): + num_iter += 1 + ori_img = data1["image"] + cvt_img = data2["image"] + assert_allclose(ori_img.flatten(), cvt_img.flatten(), rtol=1e-5, atol=0) + assert (ori_img.shape == cvt_img.shape) + + +if __name__ == "__main__": + test_rgb_hsv_hwc() + test_rgb_hsv_batch_hwc() + test_rgb_hsv_chw() + test_rgb_hsv_batch_chw() + test_rgb_hsv_pipeline() + diff --git a/tests/ut/python/dataset/test_sampler.py b/tests/ut/python/dataset/test_sampler.py new file mode 100644 index 00000000000..ca618311cbd --- /dev/null +++ b/tests/ut/python/dataset/test_sampler.py @@ -0,0 +1,93 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger + + +# test5trainimgs.json contains 5 images whose un-decoded shape is [83554, 54214, 65512, 54214, 64631] +# the label of each image is [0,0,0,1,1] each image can be uniquely identified +# via the following lookup table (dict){(83554, 0): 0, (54214, 0): 1, (54214, 1): 2, (65512, 0): 3, (64631, 1): 4} + +def test_sequential_sampler(print_res=False): + manifest_file = "../data/dataset/testManifestData/test5trainimgs.json" + map = {(172876, 0): 0, (54214, 0): 1, (54214, 1): 2, (173673, 0): 3, (64631, 1): 4} + + def test_config(num_samples, num_repeats=None): + sampler = ds.SequentialSampler() + data1 = ds.ManifestDataset(manifest_file, num_samples=num_samples, sampler=sampler) + if num_repeats is not None: + data1 = data1.repeat(num_repeats) + res = [] + for item in data1.create_dict_iterator(): + logger.info("item[image].shape[0]: {}, item[label].item(): {}" + .format(item["image"].shape[0], item["label"].item())) + res.append(map[(item["image"].shape[0], item["label"].item())]) + if print_res: + logger.info("image.shapes and labels: {}".format(res)) + return res + + assert test_config(num_samples=3, num_repeats=None) == [0, 1, 2] + assert test_config(num_samples=None, num_repeats=2) == [0, 1, 2, 3, 4] * 2 + assert test_config(num_samples=4, num_repeats=2) == [0, 1, 2, 3] * 2 + + +def test_random_sampler(print_res=False): + manifest_file = "../data/dataset/testManifestData/test5trainimgs.json" + map = {(172876, 0): 0, (54214, 0): 1, (54214, 1): 2, (173673, 0): 3, (64631, 1): 4} + + def test_config(replacement, num_samples, num_repeats): + sampler = ds.RandomSampler(replacement=replacement, num_samples=num_samples) + data1 = ds.ManifestDataset(manifest_file, sampler=sampler) + data1 = data1.repeat(num_repeats) + res = [] + for item in data1.create_dict_iterator(): + res.append(map[(item["image"].shape[0], item["label"].item())]) + if print_res: + logger.info("image.shapes and labels: {}".format(res)) + return res + + # this tests that each epoch COULD return different samples than the previous epoch + assert len(set(test_config(replacement=False, num_samples=2, num_repeats=6))) > 2 + # the following two tests test replacement works + ordered_res = [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] + assert sorted(test_config(replacement=False, num_samples=None, num_repeats=4)) == ordered_res + assert sorted(test_config(replacement=True, num_samples=None, num_repeats=4)) != ordered_res + + +def test_random_sampler_multi_iter(print_res=False): + manifest_file = "../data/dataset/testManifestData/test5trainimgs.json" + map = {(172876, 0): 0, (54214, 0): 1, (54214, 1): 2, (173673, 0): 3, (64631, 1): 4} + + def test_config(replacement, num_samples, num_repeats, validate): + sampler = ds.RandomSampler(replacement=replacement, num_samples=num_samples) + data1 = ds.ManifestDataset(manifest_file, sampler=sampler) + while num_repeats > 0: + res = [] + for item in data1.create_dict_iterator(): + res.append(map[(item["image"].shape[0], item["label"].item())]) + if print_res: + logger.info("image.shapes and labels: {}".format(res)) + if validate != sorted(res): + break + num_repeats -= 1 + assert num_repeats > 0 + + test_config(replacement=True, num_samples=5, num_repeats=5, validate=[0, 1, 2, 3, 4, 5]) + + +if __name__ == '__main__': + test_sequential_sampler(True) + test_random_sampler(True) + test_random_sampler_multi_iter(True) diff --git a/tests/ut/python/dataset/test_schema.py b/tests/ut/python/dataset/test_schema.py new file mode 100644 index 00000000000..347e075fcc4 --- /dev/null +++ b/tests/ut/python/dataset/test_schema.py @@ -0,0 +1,27 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds + +FILES = ["../data/dataset/testTFTestAllTypes/test.data"] +DATASET_ROOT = "../data/dataset/testTFTestAllTypes/" +SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json" + + +def test_simple_schema(): + ds.Schema(SCHEMA_FILE) + + +if __name__ == '__main__': + test_simple_schema() diff --git a/tests/ut/python/dataset/test_serdes_dataset.py b/tests/ut/python/dataset/test_serdes_dataset.py new file mode 100644 index 00000000000..2ef93dbcd6f --- /dev/null +++ b/tests/ut/python/dataset/test_serdes_dataset.py @@ -0,0 +1,226 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing dataset serialize and deserialize in DE +""" +import filecmp +import glob +import json +import os + +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.c_transforms as c +import mindspore.dataset.transforms.vision.c_transforms as vision +from mindspore.dataset.transforms.vision import Inter +from mindspore import log as logger + + +def test_imagefolder(remove_json_files=True): + """ + Test simulating resnet50 dataset pipeline. + """ + data_dir = "../data/dataset/testPK/data" + ds.config.set_seed(1) + + # define data augmentation parameters + rescale = 1.0 / 255.0 + shift = 0.0 + resize_height, resize_width = 224, 224 + weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1] + + # Constructing DE pipeline + sampler = ds.WeightedRandomSampler(weights, 11) + data1 = ds.ImageFolderDatasetV2(data_dir, sampler=sampler) + data1 = data1.repeat(1) + data1 = data1.map(input_columns=["image"], operations=[vision.Decode(True)]) + rescale_op = vision.Rescale(rescale, shift) + + resize_op = vision.Resize((resize_height, resize_width), Inter.LINEAR) + data1 = data1.map(input_columns=["image"], operations=[rescale_op, resize_op]) + data1 = data1.batch(2) + + # Serialize the dataset pre-processing pipeline. + # data1 should still work after saving. + ds.serialize(data1, "imagenet_dataset_pipeline.json") + ds1_dict = ds.serialize(data1) + assert (validate_jsonfile("imagenet_dataset_pipeline.json") is True) + + # Print the serialized pipeline to stdout + ds.show(data1) + + # Deserialize the serialized json file + data2 = ds.deserialize(json_filepath="imagenet_dataset_pipeline.json") + + # Serialize the pipeline we just deserialized. + # The content of the json file should be the same to the previous serialize. + ds.serialize(data2, "imagenet_dataset_pipeline_1.json") + assert (validate_jsonfile("imagenet_dataset_pipeline_1.json") is True) + assert (filecmp.cmp('imagenet_dataset_pipeline.json', 'imagenet_dataset_pipeline_1.json')) + + # Deserialize the latest json file again + data3 = ds.deserialize(json_filepath="imagenet_dataset_pipeline_1.json") + data4 = ds.deserialize(input_dict=ds1_dict) + num_samples = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2, item3, item4 in zip(data1.create_dict_iterator(), data2.create_dict_iterator(), + data3.create_dict_iterator(), data4.create_dict_iterator()): + assert (np.array_equal(item1['image'], item2['image'])) + assert (np.array_equal(item1['image'], item3['image'])) + assert (np.array_equal(item1['label'], item2['label'])) + assert (np.array_equal(item1['label'], item3['label'])) + assert (np.array_equal(item3['image'], item4['image'])) + assert (np.array_equal(item3['label'], item4['label'])) + num_samples += 1 + + logger.info("Number of data in data1: {}".format(num_samples)) + assert (num_samples == 6) + + # Remove the generated json file + if remove_json_files: + delete_json_files() + + +def test_mnist_dataset(remove_json_files=True): + data_dir = "../data/dataset/testMnistData" + ds.config.set_seed(1) + + data1 = ds.MnistDataset(data_dir, 100) + one_hot_encode = c.OneHot(10) # num_classes is input argument + data1 = data1.map(input_columns="label", operations=one_hot_encode) + + # batch_size is input argument + data1 = data1.batch(batch_size=10, drop_remainder=True) + + ds.serialize(data1, "mnist_dataset_pipeline.json") + assert (validate_jsonfile("mnist_dataset_pipeline.json") is True) + + data2 = ds.deserialize(json_filepath="mnist_dataset_pipeline.json") + ds.serialize(data2, "mnist_dataset_pipeline_1.json") + assert (validate_jsonfile("mnist_dataset_pipeline_1.json") is True) + assert (filecmp.cmp('mnist_dataset_pipeline.json', 'mnist_dataset_pipeline_1.json')) + + data3 = ds.deserialize(json_filepath="mnist_dataset_pipeline_1.json") + + num = 0 + for data1, data2, data3 in zip(data1.create_dict_iterator(), data2.create_dict_iterator(), + data3.create_dict_iterator()): + assert (np.array_equal(data1['image'], data2['image'])) + assert (np.array_equal(data1['image'], data3['image'])) + assert (np.array_equal(data1['label'], data2['label'])) + assert (np.array_equal(data1['label'], data3['label'])) + num += 1 + + logger.info("mnist total num samples is {}".format(str(num))) + assert (num == 10) + + if remove_json_files: + delete_json_files() + + +def test_zip_dataset(remove_json_files=True): + files = ["../data/dataset/testTFTestAllTypes/test.data"] + schema_file = "../data/dataset/testTFTestAllTypes/datasetSchema.json" + ds.config.set_seed(1) + + ds0 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.GLOBAL) + data1 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.GLOBAL) + data2 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.FILES) + data2 = data2.shuffle(10000) + data2 = data2.rename(input_columns=["col_sint16", "col_sint32", "col_sint64", "col_float", + "col_1d", "col_2d", "col_3d", "col_binary"], + output_columns=["column_sint16", "column_sint32", "column_sint64", "column_float", + "column_1d", "column_2d", "column_3d", "column_binary"]) + data3 = ds.zip((data1, data2)) + ds.serialize(data3, "zip_dataset_pipeline.json") + assert (validate_jsonfile("zip_dataset_pipeline.json") is True) + assert (validate_jsonfile("zip_dataset_pipeline_typo.json") is False) + + data4 = ds.deserialize(json_filepath="zip_dataset_pipeline.json") + ds.serialize(data4, "zip_dataset_pipeline_1.json") + assert (validate_jsonfile("zip_dataset_pipeline_1.json") is True) + assert (filecmp.cmp('zip_dataset_pipeline.json', 'zip_dataset_pipeline_1.json')) + + rows = 0 + for d0, d3, d4 in zip(ds0, data3, data4): + num_cols = len(d0) + offset = 0 + for t1 in d0: + assert np.array_equal(t1, d3[offset]) + assert np.array_equal(t1, d3[offset + num_cols]) + assert np.array_equal(t1, d4[offset]) + assert np.array_equal(t1, d4[offset + num_cols]) + offset += 1 + rows += 1 + assert (rows == 12) + + if remove_json_files: + delete_json_files() + +def test_random_crop(): + logger.info("test_random_crop") + DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + # First dataset + data1 = ds.StorageDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + decode_op = vision.Decode() + random_crop_op = vision.RandomCrop([512, 512], [200, 200, 200, 200]) + data1 = data1.map(input_columns="image", operations=decode_op) + data1 = data1.map(input_columns="image", operations=random_crop_op) + + # Serializing into python dictionary + ds1_dict = ds.serialize(data1) + # Serializing into json object + ds1_json = json.dumps(ds1_dict, indent=2) + + # Reconstruct dataset pipeline from its serialized form + data1_1 = ds.deserialize(input_dict=ds1_dict) + + # Second dataset + data2 = ds.StorageDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data2 = data2.map(input_columns="image", operations=decode_op) + + for item1, item1_1, item2 in zip(data1.create_dict_iterator(), data1_1.create_dict_iterator(), + data2.create_dict_iterator()): + assert (np.array_equal(item1['image'], item1_1['image'])) + image2 = item2["image"] + +def validate_jsonfile(filepath): + try: + file_exist = os.path.exists(filepath) + with open(filepath, 'r') as jfile: + loaded_json = json.load(jfile) + except IOError: + return False + return file_exist and isinstance(loaded_json, dict) + + +def delete_json_files(): + file_list = glob.glob('*.json') + for f in file_list: + try: + os.remove(f) + except IOError: + logger.info("Error while deleting: {}".format(f)) + + + +if __name__ == '__main__': + test_imagefolder() + test_zip_dataset() + test_mnist_dataset() + test_random_crop() diff --git a/tests/ut/python/dataset/test_shuffle.py b/tests/ut/python/dataset/test_shuffle.py new file mode 100644 index 00000000000..2b7a251d2cb --- /dev/null +++ b/tests/ut/python/dataset/test_shuffle.py @@ -0,0 +1,239 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from util import save_and_check + +import mindspore.dataset as ds +from mindspore import log as logger + +# Note: Number of rows in test.data dataset: 12 +DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] +GENERATE_GOLDEN = False + + +def test_shuffle_01(): + """ + Test shuffle: buffer_size < number-of-rows-in-dataset + """ + logger.info("test_shuffle_01") + # define parameters + buffer_size = 5 + seed = 1 + parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + filename = "shuffle_01_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_shuffle_02(): + """ + Test shuffle: buffer_size = number-of-rows-in-dataset + """ + logger.info("test_shuffle_02") + # define parameters + buffer_size = 12 + seed = 1 + parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + filename = "shuffle_02_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_shuffle_03(): + """ + Test shuffle: buffer_size=2 (minimum size), number-of-rows-in-dataset > 2 + """ + logger.info("test_shuffle_03") + # define parameters + buffer_size = 2 + seed = 1 + parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, shuffle=ds.Shuffle.FILES) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size) + + filename = "shuffle_03_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_shuffle_04(): + """ + Test shuffle: buffer_size=2 (minimum size), number-of-rows-in-dataset = 2 + """ + logger.info("test_shuffle_04") + # define parameters + buffer_size = 2 + seed = 1 + parameters = {"params": {'buffer_size': buffer_size, "seed": seed}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, num_samples=2) + ds.config.set_seed(seed) + data1 = data1.shuffle(buffer_size=buffer_size) + + filename = "shuffle_04_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_shuffle_exception_01(): + """ + Test shuffle exception: buffer_size<0 + """ + logger.info("test_shuffle_exception_01") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle(buffer_size=-1) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +def test_shuffle_exception_02(): + """ + Test shuffle exception: buffer_size=0 + """ + logger.info("test_shuffle_exception_02") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle(buffer_size=0) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +def test_shuffle_exception_03(): + """ + Test shuffle exception: buffer_size=1 + """ + logger.info("test_shuffle_exception_03") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle(buffer_size=1) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +def test_shuffle_exception_04(): + """ + Test shuffle exception: buffer_size > number-of-rows-in-dataset + """ + logger.info("test_shuffle_exception_04") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle(buffer_size=13) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +def test_shuffle_exception_05(): + """ + Test shuffle exception: Missing mandatory buffer_size input parameter + """ + logger.info("test_shuffle_exception_05") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle() + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +def test_shuffle_exception_06(): + """ + Test shuffle exception: buffer_size wrong type, boolean value False + """ + logger.info("test_shuffle_exception_06") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle(buffer_size=False) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +def test_shuffle_exception_07(): + """ + Test shuffle exception: buffer_size wrong type, boolean value True + """ + logger.info("test_shuffle_exception_07") + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR) + ds.config.set_seed(1) + try: + data1 = data1.shuffle(buffer_size=True) + sum([1 for _ in data1]) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "buffer_size" in str(e) + + +if __name__ == '__main__': + test_shuffle_01() + test_shuffle_02() + test_shuffle_03() + test_shuffle_04() + test_shuffle_exception_01() + test_shuffle_exception_02() + test_shuffle_exception_03() + test_shuffle_exception_04() + test_shuffle_exception_05() + test_shuffle_exception_06() + test_shuffle_exception_07() + logger.info('\n') diff --git a/tests/ut/python/dataset/test_storage.py b/tests/ut/python/dataset/test_storage.py new file mode 100644 index 00000000000..b37a52f37d3 --- /dev/null +++ b/tests/ut/python/dataset/test_storage.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from util import save_and_check + +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] +SCHEMA_DIR = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", + "col_sint16", "col_sint32", "col_sint64"] +GENERATE_GOLDEN = False + + +def test_case_storage(): + """ + test StorageDataset + """ + logger.info("Test Simple StorageDataset") + # define parameters + parameters = {"params": {}} + + # apply dataset operations + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + + filename = "storage_result.npz" + save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_tfreader_op.py b/tests/ut/python/dataset/test_tfreader_op.py new file mode 100644 index 00000000000..6de14df34ef --- /dev/null +++ b/tests/ut/python/dataset/test_tfreader_op.py @@ -0,0 +1,214 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import numpy as np +from util import save_and_check + +import mindspore.common.dtype as mstype +import mindspore.dataset as ds +from mindspore import log as logger +import pytest + + +FILES = ["../data/dataset/testTFTestAllTypes/test.data"] +DATASET_ROOT = "../data/dataset/testTFTestAllTypes/" +SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json" +GENERATE_GOLDEN = False + + +def test_case_tf_shape(): + schema_file = "../data/dataset/testTFTestAllTypes/datasetSchemaRank0.json" + ds1 = ds.TFRecordDataset(FILES, schema_file) + ds1 = ds1.batch(2) + for data in ds1.create_dict_iterator(): + print(data) + output_shape = ds1.output_shapes() + assert (len(output_shape[-1]) == 1) + + +def test_case_tf_shape_2(): + ds1 = ds.TFRecordDataset(FILES, SCHEMA_FILE) + ds1 = ds1.batch(2) + output_shape = ds1.output_shapes() + assert (len(output_shape[-1]) == 2) + + +def test_case_tf_file(): + logger.info("reading data from: {}".format(FILES[0])) + parameters = {"params": {}} + + data = ds.TFRecordDataset(FILES, SCHEMA_FILE, shuffle=ds.Shuffle.FILES) + filename = "tfreader_result.npz" + save_and_check(data, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_tf_file_no_schema(): + logger.info("reading data from: {}".format(FILES[0])) + parameters = {"params": {}} + + data = ds.TFRecordDataset(FILES, shuffle=ds.Shuffle.FILES) + filename = "tf_file_no_schema.npz" + save_and_check(data, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_case_tf_file_pad(): + logger.info("reading data from: {}".format(FILES[0])) + parameters = {"params": {}} + + schema_file = "../data/dataset/testTFTestAllTypes/datasetSchemaPadBytes10.json" + data = ds.TFRecordDataset(FILES, schema_file, shuffle=ds.Shuffle.FILES) + filename = "tf_file_padBytes10.npz" + save_and_check(data, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_tf_files(): + pattern = DATASET_ROOT + "/test.data" + data = ds.TFRecordDataset(pattern, SCHEMA_FILE, shuffle=ds.Shuffle.FILES) + assert sum([1 for _ in data]) == 12 + + pattern = DATASET_ROOT + "/test2.data" + data = ds.TFRecordDataset(pattern, SCHEMA_FILE, shuffle=ds.Shuffle.FILES) + assert sum([1 for _ in data]) == 12 + + pattern = DATASET_ROOT + "/*.data" + data = ds.TFRecordDataset(pattern, SCHEMA_FILE, num_samples=24, shuffle=ds.Shuffle.FILES) + assert sum([1 for _ in data]) == 24 + + pattern = DATASET_ROOT + "/*.data" + data = ds.TFRecordDataset(pattern, SCHEMA_FILE, num_samples=3, shuffle=ds.Shuffle.FILES) + assert sum([1 for _ in data]) == 3 + + data = ds.TFRecordDataset([DATASET_ROOT + "/test.data", DATASET_ROOT + "/test2.data"], + SCHEMA_FILE, num_samples=24, shuffle=ds.Shuffle.FILES) + assert sum([1 for _ in data]) == 24 + + +def test_tf_record_schema(): + schema = ds.Schema() + schema.add_column('col_1d', de_type=mstype.int64, shape=[2]) + schema.add_column('col_2d', de_type=mstype.int64, shape=[2, 2]) + schema.add_column('col_3d', de_type=mstype.int64, shape=[2, 2, 2]) + schema.add_column('col_binary', de_type=mstype.uint8, shape=[1]) + schema.add_column('col_float', de_type=mstype.float32, shape=[1]) + schema.add_column('col_sint16', de_type=mstype.int64, shape=[1]) + schema.add_column('col_sint32', de_type=mstype.int64, shape=[1]) + schema.add_column('col_sint64', de_type=mstype.int64, shape=[1]) + data1 = ds.TFRecordDataset(FILES, schema=schema, shuffle=ds.Shuffle.FILES) + + data2 = ds.TFRecordDataset(FILES, schema=SCHEMA_FILE, shuffle=ds.Shuffle.FILES) + + for d1, d2 in zip(data1, data2): + for t1, t2 in zip(d1, d2): + assert np.array_equal(t1, t2) + + +def test_tf_record_shuffle(): + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(FILES, schema=SCHEMA_FILE, shuffle=ds.Shuffle.GLOBAL) + data2 = ds.TFRecordDataset(FILES, schema=SCHEMA_FILE, shuffle=ds.Shuffle.FILES) + data2 = data2.shuffle(10000) + + for d1, d2 in zip(data1, data2): + for t1, t2 in zip(d1, d2): + assert np.array_equal(t1, t2) + + +def skip_test_tf_record_shard(): + tf_files = ["../data/dataset/tf_file_dataset/test1.data", "../data/dataset/tf_file_dataset/test2.data", + "../data/dataset/tf_file_dataset/test3.data", "../data/dataset/tf_file_dataset/test4.data"] + + def get_res(shard_id, num_repeats): + data1 = ds.TFRecordDataset(tf_files, num_shards=2, shard_id=shard_id, num_samples=3, + shuffle=ds.Shuffle.FILES) + data1 = data1.repeat(num_repeats) + res = list() + for item in data1.create_dict_iterator(): + res.append(item["scalars"][0]) + return res + + # get separate results from two workers. the 2 results need to satisfy 2 criteria + # 1. two workers always give different results in same epoch (e.g. wrkr1:f1&f3, wrkr2:f2&f4 in one epoch) + # 2. with enough epochs, both workers will get the entire dataset (e,g. ep1_wrkr1: f1&f3, ep2,_wrkr1 f2&f4) + worker1_res = get_res(0, 16) + worker2_res = get_res(1, 16) + # check criteria 1 + for i in range(len(worker1_res)): + assert (worker1_res[i] != worker2_res[i]) + # check criteria 2 + assert (set(worker2_res) == set(worker1_res)) + assert (len(set(worker2_res)) == 12) + + +def test_tf_shard_equal_rows(): + tf_files = ["../data/dataset/tf_file_dataset/test1.data", "../data/dataset/tf_file_dataset/test2.data", + "../data/dataset/tf_file_dataset/test3.data", "../data/dataset/tf_file_dataset/test4.data"] + + def get_res(num_shards, shard_id, num_repeats): + ds1 = ds.TFRecordDataset(tf_files, num_shards=num_shards, shard_id=shard_id, shard_equal_rows=True) + ds1 = ds1.repeat(num_repeats) + res = list() + for data in ds1.create_dict_iterator(): + res.append(data["scalars"][0]) + return res + + worker1_res = get_res(3, 0, 2) + worker2_res = get_res(3, 1, 2) + worker3_res = get_res(3, 2, 2) + # check criteria 1 + for i in range(len(worker1_res)): + assert (worker1_res[i] != worker2_res[i]) + assert (worker2_res[i] != worker3_res[i]) + assert (len(worker1_res) == 28) + + worker4_res = get_res(1, 0, 1) + assert (len(worker4_res) == 40) + + +def test_case_tf_file_no_schema_columns_list(): + data = ds.TFRecordDataset(FILES, shuffle=False, columns_list=["col_sint16"]) + row = data.create_dict_iterator().get_next() + assert row["col_sint16"] == [-32768] + + with pytest.raises(KeyError) as info: + a = row["col_sint32"] + assert "col_sint32" in str(info.value) + + +def test_tf_record_schema_columns_list(): + schema = ds.Schema() + schema.add_column('col_1d', de_type=mstype.int64, shape=[2]) + schema.add_column('col_2d', de_type=mstype.int64, shape=[2, 2]) + schema.add_column('col_3d', de_type=mstype.int64, shape=[2, 2, 2]) + schema.add_column('col_binary', de_type=mstype.uint8, shape=[1]) + schema.add_column('col_float', de_type=mstype.float32, shape=[1]) + schema.add_column('col_sint16', de_type=mstype.int64, shape=[1]) + schema.add_column('col_sint32', de_type=mstype.int64, shape=[1]) + schema.add_column('col_sint64', de_type=mstype.int64, shape=[1]) + data = ds.TFRecordDataset(FILES, schema=schema, shuffle=False, columns_list=["col_sint16"]) + row = data.create_dict_iterator().get_next() + assert row["col_sint16"] == [-32768] + + with pytest.raises(KeyError) as info: + a = row["col_sint32"] + assert "col_sint32" in str(info.value) + +if __name__ == '__main__': + test_case_tf_shape() + test_case_tf_file() + test_case_tf_file_no_schema() + test_case_tf_file_pad() + test_tf_files() + test_tf_record_schema() + test_tf_record_shuffle() + test_tf_shard_equal_rows() diff --git a/tests/ut/python/dataset/test_type_cast.py b/tests/ut/python/dataset/test_type_cast.py new file mode 100644 index 00000000000..3ff4717f72e --- /dev/null +++ b/tests/ut/python/dataset/test_type_cast.py @@ -0,0 +1,114 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing RandomRotation op in DE +""" +import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.c_transforms as data_util + +import numpy as np + +import mindspore.common.dtype as mstype +import mindspore.dataset as ds +from mindspore import log as logger + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def test_type_cast(): + """ + Test type_cast_op + """ + logger.info("test_type_cast_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + type_cast_op = data_util.TypeCast(mstype.float32) + + ctrans = [decode_op, + type_cast_op, + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [py_vision.Decode(), + py_vision.ToTensor() + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + assert c_image.dtype == "float32" + + +def test_type_cast_string(): + """ + Test type_cast_op + """ + logger.info("test_type_cast_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + + type_cast_op = data_util.TypeCast(mstype.float16 ) + + ctrans = [decode_op, + type_cast_op + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + transforms = [py_vision.Decode(), + py_vision.ToTensor() + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + c_image = item1["image"] + py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + + logger.info("shape of c_image: {}".format(c_image.shape)) + logger.info("shape of py_image: {}".format(py_image.shape)) + + logger.info("dtype of c_image: {}".format(c_image.dtype)) + logger.info("dtype of py_image: {}".format(py_image.dtype)) + assert c_image.dtype == "float16" + + +if __name__ == "__main__": + test_type_cast() + test_type_cast_string() diff --git a/tests/ut/python/dataset/test_var_batch_map.py b/tests/ut/python/dataset/test_var_batch_map.py new file mode 100644 index 00000000000..24982afc6ce --- /dev/null +++ b/tests/ut/python/dataset/test_var_batch_map.py @@ -0,0 +1,330 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import mindspore.dataset as ds +from mindspore import log as logger +import numpy as np + + +def test_batch_corner_cases(): + def gen(num): + for i in range(num): + yield (np.array([i]),) + + def test_repeat_batch(gen_num, repeats, batch_size, drop, res): + data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"]).repeat(repeats).batch(batch_size, drop) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + + def test_batch_repeat(gen_num, repeats, batch_size, drop, res): + data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"]).batch(batch_size, drop).repeat(repeats) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + + tst1, tst2, tst3, tst4 = [], [], [], [] + # case 1 & 2, where batch_size is greater than the entire epoch, with drop equals to both val + test_repeat_batch(gen_num=2, repeats=4, batch_size=7, drop=False, res=tst1) + assert np.array_equal(np.array([[0], [1], [0], [1], [0], [1], [0]]), tst1[0]), "\nATTENTION BATCH FAILED\n" + assert np.array_equal(np.array([[1]]), tst1[1]), "\nATTENTION TEST BATCH FAILED\n" + assert len(tst1) == 2, "\nATTENTION TEST BATCH FAILED\n" + test_repeat_batch(gen_num=2, repeats=4, batch_size=5, drop=True, res=tst2) + assert np.array_equal(np.array([[0], [1], [0], [1], [0]]), tst2[0]), "\nATTENTION BATCH FAILED\n" + assert len(tst2) == 1, "\nATTENTION TEST BATCH FAILED\n" + # case 3 & 4, batch before repeat with different drop + test_batch_repeat(gen_num=5, repeats=2, batch_size=4, drop=True, res=tst3) + assert np.array_equal(np.array([[0], [1], [2], [3]]), tst3[0]), "\nATTENTION BATCH FAILED\n" + assert np.array_equal(tst3[0], tst3[1]), "\nATTENTION BATCH FAILED\n" + assert len(tst3) == 2, "\nATTENTION BATCH FAILED\n" + test_batch_repeat(gen_num=5, repeats=2, batch_size=4, drop=False, res=tst4) + assert np.array_equal(np.array([[0], [1], [2], [3]]), tst4[0]), "\nATTENTION BATCH FAILED\n" + assert np.array_equal(tst4[0], tst4[2]), "\nATTENTION BATCH FAILED\n" + assert np.array_equal(tst4[1], np.array([[4]])), "\nATTENTION BATCH FAILED\n" + assert np.array_equal(tst4[1], tst4[3]), "\nATTENTION BATCH FAILED\n" + assert len(tst4) == 4, "\nATTENTION BATCH FAILED\n" + + +# each sub-test in this function is tested twice with exact parameter except that the second test passes each row +# to a pyfunc which makes a deep copy of the row +def test_variable_size_batch(): + def check_res(arr1, arr2): + for ind in range(len(arr1)): + if not np.array_equal(arr1[ind], np.array(arr2[ind])): + return False + return len(arr1) == len(arr2) + + def gen(num): + for i in range(num): + yield (np.array([i]),) + + def add_one_by_batch_num(batchInfo): + return batchInfo.get_batch_num() + 1 + + def add_one_by_epoch(batchInfo): + return batchInfo.get_epoch_num() + 1 + + def simple_copy(colList, batchInfo): + return ([np.copy(arr) for arr in colList],) + + def test_repeat_batch(gen_num, r, drop, func, res): + data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"]).repeat(r).batch(batch_size=func, drop_remainder=drop) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + + # same as test_repeat_batch except each row is passed through via a map which makes a copy of each element + def test_repeat_batch_with_copy_map(gen_num, r, drop, func): + res = [] + data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"]).repeat(r) \ + .batch(batch_size=func, drop_remainder=drop, input_columns=["num"], per_batch_map=simple_copy) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + return res + + def test_batch_repeat(gen_num, r, drop, func, res): + data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"]).batch(batch_size=func, drop_remainder=drop).repeat(r) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + + # same as test_batch_repeat except each row is passed through via a map which makes a copy of each element + def test_batch_repeat_with_copy_map(gen_num, r, drop, func): + res = [] + data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"]) \ + .batch(batch_size=func, drop_remainder=drop, input_columns=["num"], per_batch_map=simple_copy).repeat(r) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + return res + + tst1, tst2, tst3, tst4, tst5, tst6, tst7 = [], [], [], [], [], [], [] + + # no repeat, simple var size, based on batch_num + test_repeat_batch(7, 1, True, add_one_by_batch_num, tst1) + assert check_res(tst1, [[[0]], [[1], [2]], [[3], [4], [5]]]), "\nATTENTION VAR BATCH FAILED\n" + assert check_res(tst1, test_repeat_batch_with_copy_map(7, 1, True, add_one_by_batch_num)), "\nMAP FAILED\n" + test_repeat_batch(9, 1, False, add_one_by_batch_num, tst2) + assert check_res(tst2, [[[0]], [[1], [2]], [[3], [4], [5]], [[6], [7], [8]]]), "\nATTENTION VAR BATCH FAILED\n" + assert check_res(tst2, test_repeat_batch_with_copy_map(9, 1, False, add_one_by_batch_num)), "\nMAP FAILED\n" + # batch after repeat, cross epoch batch + test_repeat_batch(7, 2, False, add_one_by_batch_num, tst3) + assert check_res(tst3, [[[0]], [[1], [2]], [[3], [4], [5]], [[6], [0], [1], [2]], + [[3], [4], [5], [6]]]), "\nATTENTION VAR BATCH FAILED\n" + assert check_res(tst3, test_repeat_batch_with_copy_map(7, 2, False, add_one_by_batch_num)), "\nMAP FAILED\n" + # repeat after batch, no cross epoch batch, remainder dropped + test_batch_repeat(9, 7, True, add_one_by_batch_num, tst4) + assert check_res(tst4, [[[0]], [[1], [2]], [[3], [4], [5]]] * 7), "\nATTENTION VAR BATCH FAILED\n" + assert check_res(tst4, test_batch_repeat_with_copy_map(9, 7, True, add_one_by_batch_num)), "\nAMAP FAILED\n" + # repeat after batch, no cross epoch batch, remainder kept + test_batch_repeat(9, 3, False, add_one_by_batch_num, tst5) + assert check_res(tst5, [[[0]], [[1], [2]], [[3], [4], [5]], [[6], [7], [8]]] * 3), "\nATTENTION VAR BATCH FAILED\n" + assert check_res(tst5, test_batch_repeat_with_copy_map(9, 3, False, add_one_by_batch_num)), "\nMAP FAILED\n" + # batch_size based on epoch number, drop + test_batch_repeat(4, 4, True, add_one_by_epoch, tst6) + assert check_res(tst6, [[[0]], [[1]], [[2]], [[3]], [[0], [1]], [[2], [3]], [[0], [1], [2]], + [[0], [1], [2], [3]]]), "\nATTENTION VAR BATCH FAILED\n" + assert check_res(tst6, test_batch_repeat_with_copy_map(4, 4, True, add_one_by_epoch)), "\nMAP FAILED\n" + # batch_size based on epoch number, no drop + test_batch_repeat(4, 4, False, add_one_by_epoch, tst7) + assert check_res(tst7, [[[0]], [[1]], [[2]], [[3]], [[0], [1]], [[2], [3]], [[0], [1], [2]], [[3]], + [[0], [1], [2], [3]]]), "\nATTENTION VAR BATCH FAILED\n" + str(tst7) + assert check_res(tst7, test_batch_repeat_with_copy_map(4, 4, False, add_one_by_epoch)), "\nMAP FAILED\n" + + +def test_basic_batch_map(): + def check_res(arr1, arr2): + for ind in range(len(arr1)): + if not np.array_equal(arr1[ind], np.array(arr2[ind])): + return False + return len(arr1) == len(arr2) + + def gen(num): + for i in range(num): + yield (np.array([i]),) + + def invert_sign_per_epoch(colList, batchInfo): + return ([np.copy(((-1) ** batchInfo.get_epoch_num()) * arr) for arr in colList],) + + def invert_sign_per_batch(colList, batchInfo): + return ([np.copy(((-1) ** batchInfo.get_batch_num()) * arr) for arr in colList],) + + def batch_map_config(num, r, batch_size, func, res): + data1 = ds.GeneratorDataset((lambda: gen(num)), ["num"]) \ + .batch(batch_size=batch_size, input_columns=["num"], per_batch_map=func).repeat(r) + for item in data1.create_dict_iterator(): + res.append(item["num"]) + + tst1, tst2, = [], [] + batch_map_config(4, 2, 2, invert_sign_per_epoch, tst1) + assert check_res(tst1, [[[0], [1]], [[2], [3]], [[0], [-1]], [[-2], [-3]]]), "\nATTENTION MAP BATCH FAILED\n" + str( + tst1) + # each batch, the sign of a row is changed, test map is corrected performed according to its batch_num + batch_map_config(4, 2, 2, invert_sign_per_batch, tst2) + assert check_res(tst2, + [[[0], [1]], [[-2], [-3]], [[0], [1]], [[-2], [-3]]]), "\nATTENTION MAP BATCH FAILED\n" + str(tst2) + + +def test_batch_multi_col_map(): + def check_res(arr1, arr2): + for ind in range(len(arr1)): + if not np.array_equal(arr1[ind], np.array(arr2[ind])): + return False + return len(arr1) == len(arr2) + + def gen(num): + for i in range(num): + yield (np.array([i]), np.array([i ** 2])) + + def col1_col2_add_num(col1, col2, batchInfo): + return ([[np.copy(arr + 100) for arr in col1], + [np.copy(arr + 300) for arr in col2]]) + + def invert_sign_per_batch(colList, batchInfo): + return ([np.copy(((-1) ** batchInfo.get_batch_num()) * arr) for arr in colList],) + + def invert_sign_per_batch_multi_col(col1, col2, batchInfo): + return ([np.copy(((-1) ** batchInfo.get_batch_num()) * arr) for arr in col1], + [np.copy(((-1) ** batchInfo.get_batch_num()) * arr) for arr in col2]) + + def batch_map_config(num, r, batch_size, func, col_names, res): + data1 = ds.GeneratorDataset((lambda: gen(num)), ["num", "num_square"]) \ + .batch(batch_size=batch_size, input_columns=col_names, per_batch_map=func).repeat(r) + for item in data1.create_dict_iterator(): + res.append(np.array([item["num"], item["num_square"]])) + + tst1, tst2, tst3, tst4 = [], [], [], [] + batch_map_config(4, 2, 2, invert_sign_per_batch, ["num_square"], tst1) + assert check_res(tst1, [[[[0], [1]], [[0], [1]]], [[[2], [3]], [[-4], [-9]]], [[[0], [1]], [[0], [1]]], + [[[2], [3]], [[-4], [-9]]]]), "\nATTENTION MAP BATCH FAILED\n" + str(tst1) + + batch_map_config(4, 2, 2, invert_sign_per_batch_multi_col, ["num", "num_square"], tst2) + assert check_res(tst2, [[[[0], [1]], [[0], [1]]], [[[-2], [-3]], [[-4], [-9]]], [[[0], [1]], [[0], [1]]], + [[[-2], [-3]], [[-4], [-9]]]]), "\nATTENTION MAP BATCH FAILED\n" + str(tst2) + + # the two tests below verify the order of the map. + # num_square column adds 100, num column adds 300. + batch_map_config(4, 3, 2, col1_col2_add_num, ["num_square", "num"], tst3) + assert check_res(tst3, [[[[300], [301]], [[100], [101]]], + [[[302], [303]], [[104], [109]]]] * 3), "\nATTENTION MAP BATCH FAILED\n" + str(tst3) + # num column adds 100, num_square column adds 300. + batch_map_config(4, 3, 2, col1_col2_add_num, ["num", "num_square"], tst4) + assert check_res(tst4, [[[[100], [101]], [[300], [301]]], + [[[102], [103]], [[304], [309]]]] * 3), "\nATTENTION MAP BATCH FAILED\n" + str(tst4) + + +def test_var_batch_multi_col_map(): + def check_res(arr1, arr2): + for ind in range(len(arr1)): + if not np.array_equal(arr1[ind], np.array(arr2[ind])): + return False + return len(arr1) == len(arr2) + + # gen 3 columns + # first column: 0, 3, 6, 9 ... ... + # second column:1, 4, 7, 10 ... ... + # third column: 2, 5, 8, 11 ... ... + def gen_3_cols(num): + for i in range(num): + yield (np.array([i * 3]), np.array([i * 3 + 1]), np.array([i * 3 + 2])) + + # first epoch batch_size per batch: 1, 2 ,3 ... ... + # second epoch batch_size per batch: 2, 4, 6 ... ... + # third epoch batch_size per batch: 3, 6 ,9 ... ... + def batch_func(batchInfo): + return (batchInfo.get_batch_num() + 1) * (batchInfo.get_epoch_num() + 1) + + # multiply first col by batch_num, multiply second col by -batch_num + def map_func(col1, col2, batchInfo): + return ([np.copy((1 + batchInfo.get_batch_num()) * arr) for arr in col1], + [np.copy(-(1 + batchInfo.get_batch_num()) * arr) for arr in col2]) + + def batch_map_config(num, r, fbatch, fmap, col_names, res): + data1 = ds.GeneratorDataset((lambda: gen_3_cols(num)), ["col1", "col2", "col3"]) \ + .batch(batch_size=fbatch, input_columns=col_names, per_batch_map=fmap).repeat(r) + for item in data1.create_dict_iterator(): + res.append(np.array([item["col1"], item["col2"], item["col3"]])) + + tst1 = [] + tst1_res = [[[[0]], [[-1]], [[2]]], [[[6], [12]], [[-8], [-14]], [[5], [8]]], + [[[27], [36], [45]], [[-30], [-39], [-48]], [[11], [14], [17]]], + [[[72], [84], [96], [108]], [[-76], [-88], [-100], [-112]], [[20], [23], [26], [29]]]] + batch_map_config(10, 1, batch_func, map_func, ["col1", "col2"], tst1) + assert check_res(tst1, tst1_res), "test_var_batch_multi_col_map FAILED" + + +def test_var_batch_var_resize(): + # fake resize image according to its batch number, if it's 5-th batch, resize to (5^2, 5^2) = (25, 25) + def np_psedo_resize(col, batchInfo): + s = (batchInfo.get_batch_num() + 1) ** 2 + return ([np.copy(c[0:s, 0:s, :]) for c in col],) + + def add_one(batchInfo): + return (batchInfo.get_batch_num() + 1) + + data1 = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_parallel_workers=4, decode=True) + data1 = data1.batch(batch_size=add_one, drop_remainder=True, input_columns=["image"], per_batch_map=np_psedo_resize) + # i-th batch has shape [i, i^2, i^2, 3] + i = 1 + for item in data1.create_dict_iterator(): + assert item["image"].shape == (i, i ** 2, i ** 2, 3), "\ntest_var_batch_var_resize FAILED\n" + i += 1 + + +def test_exception(): + def gen(num): + for i in range(num): + yield (np.array([i]),) + + def bad_batch_size(batchInfo): + raise StopIteration + return batchInfo.get_batch_num() + + def bad_map_func(col, batchInfo): + raise StopIteration + return (col,) + + data1 = ds.GeneratorDataset((lambda: gen(100)), ["num"]).batch(bad_batch_size) + try: + for _ in data1.create_dict_iterator(): + pass + assert False + except RuntimeError: + pass + + data2 = ds.GeneratorDataset((lambda: gen(100)), ["num"]).batch(4, input_columns=["num"], per_batch_map=bad_map_func) + try: + for item in data2.create_dict_iterator(): + pass + assert False + except RuntimeError: + pass + + +if __name__ == '__main__': + logger.info("Running test_var_batch_map.py test_batch_corner_cases() function") + test_batch_corner_cases() + + logger.info("Running test_var_batch_map.py test_variable_size_batch() function") + test_variable_size_batch() + + logger.info("Running test_var_batch_map.py test_basic_batch_map() function") + test_basic_batch_map() + + logger.info("Running test_var_batch_map.py test_batch_multi_col_map() function") + test_batch_multi_col_map() + + logger.info("Running test_var_batch_map.py tesgit t_var_batch_multi_col_map() function") + test_var_batch_multi_col_map() + + logger.info("Running test_var_batch_map.py test_var_batch_var_resize() function") + test_var_batch_var_resize() + + logger.info("Running test_var_batch_map.py test_exception() function") + test_exception() diff --git a/tests/ut/python/dataset/test_zip.py b/tests/ut/python/dataset/test_zip.py new file mode 100644 index 00000000000..353b5a2471a --- /dev/null +++ b/tests/ut/python/dataset/test_zip.py @@ -0,0 +1,271 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from util import save_and_check_dict +from mindspore import log as logger + +import mindspore.dataset as ds + +# Dataset in DIR_1 has 5 rows and 5 columns +DATA_DIR_1 = ["../data/dataset/testTFBert5Rows1/5TFDatas.data"] +SCHEMA_DIR_1 = "../data/dataset/testTFBert5Rows1/datasetSchema.json" +# Dataset in DIR_2 has 5 rows and 2 columns +DATA_DIR_2 = ["../data/dataset/testTFBert5Rows2/5TFDatas.data"] +SCHEMA_DIR_2 = "../data/dataset/testTFBert5Rows2/datasetSchema.json" +# Dataset in DIR_3 has 3 rows and 2 columns +DATA_DIR_3 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR_3 = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +# Dataset in DIR_4 has 5 rows and 7 columns +DATA_DIR_4 = ["../data/dataset/testTFBert5Rows/5TFDatas.data"] +SCHEMA_DIR_4 = "../data/dataset/testTFBert5Rows/datasetSchema.json" + +GENERATE_GOLDEN = False + + +def test_zip_01(): + """ + Test zip: zip 2 datasets, #rows-data1 == #rows-data2, #cols-data1 < #cols-data2 + """ + logger.info("test_zip_01") + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2) + data2 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + dataz = ds.zip((data1, data2)) + # Note: zipped dataset has 5 rows and 7 columns + filename = "zip_01_result.npz" + parameters = {"params": {}} + save_and_check_dict(dataz, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def skip_test_zip_02(): + """ + Test zip: zip 2 datasets, #rows-data1 < #rows-data2, #cols-data1 == #cols-data2 + """ + logger.info("test_zip_02") + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(DATA_DIR_3, SCHEMA_DIR_3) + data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2) + dataz = ds.zip((data1, data2)) + # Note: zipped dataset has 3 rows and 4 columns + filename = "zip_02_result.npz" + parameters = {"params": {}} + save_and_check_dict(dataz, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def skip_test_zip_03(): + """ + Test zip: zip 2 datasets, #rows-data1 > #rows-data2, #cols-data1 > #cols-data2 + """ + logger.info("test_zip_03") + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + data2 = ds.TFRecordDataset(DATA_DIR_3, SCHEMA_DIR_3) + dataz = ds.zip((data1, data2)) + # Note: zipped dataset has 3 rows and 7 columns + filename = "zip_03_result.npz" + parameters = {"params": {}} + save_and_check_dict(dataz, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def skip_test_zip_04(): + """ + Test zip: zip >2 datasets + """ + logger.info("test_zip_04") + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2) + data3 = ds.TFRecordDataset(DATA_DIR_3, SCHEMA_DIR_3) + dataz = ds.zip((data1, data2, data3)) + # Note: zipped dataset has 3 rows and 9 columns + filename = "zip_04_result.npz" + parameters = {"params": {}} + save_and_check_dict(dataz, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_zip_05(): + """ + Test zip: zip dataset with renamed columns + """ + logger.info("test_zip_05") + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(DATA_DIR_4, SCHEMA_DIR_4, shuffle=True) + data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2, shuffle=True) + + data2 = data2.rename(input_columns="input_ids", output_columns="new_input_ids") + data2 = data2.rename(input_columns="segment_ids", output_columns="new_segment_ids") + + dataz = ds.zip((data1, data2)) + # Note: zipped dataset has 5 rows and 9 columns + filename = "zip_05_result.npz" + parameters = {"params": {}} + save_and_check_dict(dataz, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_zip_06(): + """ + Test zip: zip dataset with renamed columns and repeat zipped dataset + """ + logger.info("test_zip_06") + ds.config.set_seed(1) + data1 = ds.TFRecordDataset(DATA_DIR_4, SCHEMA_DIR_4, shuffle=False) + data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2, shuffle=False) + + data2 = data2.rename(input_columns="input_ids", output_columns="new_input_ids") + data2 = data2.rename(input_columns="segment_ids", output_columns="new_segment_ids") + + dataz = ds.zip((data1, data2)) + dataz = dataz.repeat(2) + # Note: resultant dataset has 10 rows and 9 columns + filename = "zip_06_result.npz" + parameters = {"params": {}} + save_and_check_dict(dataz, parameters, filename, generate_golden=GENERATE_GOLDEN) + + +def test_zip_exception_01(): + """ + Test zip: zip same datasets + """ + logger.info("test_zip_exception_01") + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + + try: + dataz = ds.zip((data1, data1)) + + num_iter = 0 + for _, item in enumerate(dataz.create_dict_iterator()): + logger.info("item[input_mask] is {}".format(item["input_mask"])) + num_iter += 1 + logger.info("Number of data in zipped dataz: {}".format(num_iter)) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + + +def skip_test_zip_exception_02(): + """ + Test zip: zip datasets with duplicate column name + """ + logger.info("test_zip_exception_02") + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + data2 = ds.TFRecordDataset(DATA_DIR_4, SCHEMA_DIR_4) + + try: + dataz = ds.zip((data1, data2)) + + num_iter = 0 + for _, item in enumerate(dataz.create_dict_iterator()): + logger.info("item[input_mask] is {}".format(item["input_mask"])) + num_iter += 1 + logger.info("Number of data in zipped dataz: {}".format(num_iter)) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + + +def test_zip_exception_03(): + """ + Test zip: zip with tuple of 1 dataset + """ + logger.info("test_zip_exception_03") + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + + try: + dataz = ds.zip((data1)) + dataz = dataz.repeat(2) + + num_iter = 0 + for _, item in enumerate(dataz.create_dict_iterator()): + logger.info("item[input_mask] is {}".format(item["input_mask"])) + num_iter += 1 + logger.info("Number of data in zipped dataz: {}".format(num_iter)) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + + +def test_zip_exception_04(): + """ + Test zip: zip with empty tuple of datasets + """ + logger.info("test_zip_exception_04") + + try: + dataz = ds.zip(()) + dataz = dataz.repeat(2) + + num_iter = 0 + for _, item in enumerate(dataz.create_dict_iterator()): + logger.info("item[input_mask] is {}".format(item["input_mask"])) + num_iter += 1 + logger.info("Number of data in zipped dataz: {}".format(num_iter)) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + + +def test_zip_exception_05(): + """ + Test zip: zip with non-tuple of 2 datasets + """ + logger.info("test_zip_exception_05") + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2) + + try: + dataz = ds.zip(data1, data2) + + num_iter = 0 + for _, item in enumerate(dataz.create_dict_iterator()): + logger.info("item[input_mask] is {}".format(item["input_mask"])) + num_iter += 1 + logger.info("Number of data in zipped dataz: {}".format(num_iter)) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + + +def test_zip_exception_06(): + """ + Test zip: zip with non-tuple of 1 dataset + """ + logger.info("test_zip_exception_06") + data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1) + + try: + dataz = ds.zip(data1) + + num_iter = 0 + for _, item in enumerate(dataz.create_dict_iterator()): + logger.info("item[input_mask] is {}".format(item["input_mask"])) + num_iter += 1 + logger.info("Number of data in zipped dataz: {}".format(num_iter)) + + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + + +if __name__ == '__main__': + test_zip_01() + test_zip_02() + test_zip_03() + test_zip_04() + test_zip_05() + test_zip_06() + test_zip_exception_01() + test_zip_exception_02() + test_zip_exception_03() + test_zip_exception_04() + test_zip_exception_05() + test_zip_exception_06() diff --git a/tests/ut/python/dataset/util.py b/tests/ut/python/dataset/util.py new file mode 100644 index 00000000000..d2f35cee630 --- /dev/null +++ b/tests/ut/python/dataset/util.py @@ -0,0 +1,177 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import os +import numpy as np +#import jsbeautifier +from mindspore import log as logger + +COLUMNS = ["col_1d", "col_2d", "col_3d", "col_binary", "col_float", + "col_sint16", "col_sint32", "col_sint64"] + + +def save_golden(cur_dir, golden_ref_dir, result_dict): + """ + Save the dictionary values as the golden result in .npz file + """ + logger.info("cur_dir is {}".format(cur_dir)) + logger.info("golden_ref_dir is {}".format(golden_ref_dir)) + np.savez(golden_ref_dir, np.array(list(result_dict.values()))) + + +def save_golden_dict(cur_dir, golden_ref_dir, result_dict): + """ + Save the dictionary (both keys and values) as the golden result in .npz file + """ + logger.info("cur_dir is {}".format(cur_dir)) + logger.info("golden_ref_dir is {}".format(golden_ref_dir)) + np.savez(golden_ref_dir, np.array(list(result_dict.items()))) + + +def compare_to_golden(golden_ref_dir, result_dict): + """ + Compare as numpy arrays the test result to the golden result + """ + test_array = np.array(list(result_dict.values())) + golden_array = np.load(golden_ref_dir, allow_pickle=True)['arr_0'] + assert np.array_equal(test_array, golden_array) + + +def compare_to_golden_dict(golden_ref_dir, result_dict): + """ + Compare as dictionaries the test result to the golden result + """ + golden_array = np.load(golden_ref_dir, allow_pickle=True)['arr_0'] + assert result_dict == dict(golden_array) + + +def save_json(filename, parameters, result_dict): + """ + Save the result dictionary in json file + """ + fout = open(filename[:-3] + "json", "w") + options = jsbeautifier.default_options() + options.indent_size = 2 + + out_dict = {**parameters, **{"columns": result_dict}} + fout.write(jsbeautifier.beautify(json.dumps(out_dict), options)) + + + +def save_and_check(data, parameters, filename, generate_golden=False): + """ + Save the dataset dictionary and compare (as numpy array) with golden file. + Use create_dict_iterator to access the dataset. + """ + num_iter = 0 + result_dict = {} + for column_name in COLUMNS: + result_dict[column_name] = [] + + for item in data.create_dict_iterator(): # each data is a dictionary + for data_key in list(item.keys()): + if data_key not in result_dict: + result_dict[data_key] = [] + result_dict[data_key].append(item[data_key].tolist()) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + + cur_dir = os.path.dirname(os.path.realpath(__file__)) + golden_ref_dir = os.path.join(cur_dir, "../../data/dataset", 'golden', filename) + if generate_golden: + # Save as the golden result + save_golden(cur_dir, golden_ref_dir, result_dict) + + compare_to_golden(golden_ref_dir, result_dict) + + # Save to a json file for inspection + # save_json(filename, parameters, result_dict) + + +def save_and_check_dict(data, parameters, filename, generate_golden=False): + """ + Save the dataset dictionary and compare (as dictionary) with golden file. + Use create_dict_iterator to access the dataset. + """ + num_iter = 0 + result_dict = {} + + for item in data.create_dict_iterator(): # each data is a dictionary + for data_key in list(item.keys()): + if data_key not in result_dict: + result_dict[data_key] = [] + result_dict[data_key].append(item[data_key].tolist()) + num_iter += 1 + + logger.info("Number of data in ds1: {}".format(num_iter)) + + cur_dir = os.path.dirname(os.path.realpath(__file__)) + golden_ref_dir = os.path.join(cur_dir, "../../data/dataset", 'golden', filename) + if generate_golden: + # Save as the golden result + save_golden_dict(cur_dir, golden_ref_dir, result_dict) + + compare_to_golden_dict(golden_ref_dir, result_dict) + + # Save to a json file for inspection + # save_json(filename, parameters, result_dict) + + +def ordered_save_and_check(data, parameters, filename, generate_golden=False): + """ + Save the dataset dictionary and compare (as numpy array) with golden file. + Use create_tuple_iterator to access the dataset. + """ + num_iter = 0 + + result_dict = {} + + for item in data.create_tuple_iterator(): # each data is a dictionary + for data_key in range(0, len(item)): + if data_key not in result_dict: + result_dict[data_key] = [] + result_dict[data_key].append(item[data_key].tolist()) + num_iter += 1 + + logger.info("Number of data in data1: {}".format(num_iter)) + + cur_dir = os.path.dirname(os.path.realpath(__file__)) + golden_ref_dir = os.path.join(cur_dir, "../../data/dataset", 'golden', filename) + if generate_golden: + # Save as the golden result + save_golden(cur_dir, golden_ref_dir, result_dict) + + compare_to_golden(golden_ref_dir, result_dict) + + # Save to a json file for inspection + # save_json(filename, parameters, result_dict) + + +def diff_mse(in1, in2): + mse = (np.square(in1.astype(float) / 255 - in2.astype(float) / 255)).mean() + return mse * 100 + + +def diff_me(in1, in2): + mse = (np.abs(in1.astype(float) - in2.astype(float))).mean() + return mse / 255 * 100 + + +def diff_ssim(in1, in2): + from skimage.measure import compare_ssim as ssim + val = ssim(in1, in2, multichannel=True) + return (1 - val) * 100 diff --git a/tests/ut/python/dtype/__init__.py b/tests/ut/python/dtype/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/dtype/test_dictionary.py b/tests/ut/python/dtype/test_dictionary.py new file mode 100644 index 00000000000..a6638aa0774 --- /dev/null +++ b/tests/ut/python/dtype/test_dictionary.py @@ -0,0 +1,134 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_dictionary.py +@Desc : test_dictionary +""" +import numpy as np + +from mindspore import Tensor, context +from mindspore.nn import Cell +from mindspore.ops import operations as P + +context.set_context(mode=context.GRAPH_MODE) + + +def Xtest_arg_dict(): + class DictNet(Cell): + """DictNet definition""" + def __init__(self): + super(DictNet, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + + def construct(self, dictionary): + a = self.max(dictionary["x"], dictionary["y"]) + b = self.min(dictionary["x"], dictionary["y"]) + return a + b + + dictionary = {"x": Tensor(np.ones([3, 2, 3], np.float32)), "y": Tensor(np.ones([1, 2, 3], np.float32))} + net = DictNet() + # The outermost network does not support dict parameters, + # otherwise an exception will be thrown when executing the graph + net(dictionary) + + +def test_const_dict(): + class DictNet(Cell): + """DictNet1 definition""" + def __init__(self): + super(DictNet, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + self.dictionary = {"x": Tensor(np.ones([3, 2, 3], np.float32)), "y": Tensor(np.ones([1, 2, 3], np.float32))} + + def construct(self): + a = self.max(self.dictionary["x"], self.dictionary["y"]) + b = self.min(self.dictionary["x"], self.dictionary["y"]) + return a + b + net = DictNet() + net() + + +def test_dict_set_or_get_item(): + class DictNet(Cell): + """DictNet1 definition""" + def __init__(self): + super(DictNet, self).__init__() + self.dict_ = {"x": 1, "y": 2} + self.tuple_1 = (1, 2, 3) + self.tuple_2 = (4, 5, 6) + + def construct(self): + ret_1 = (88, 99) + ret_2 = (88, 99) + self.dict_["x"] = 3 + self.dict_["z"] = 4 + + if self.dict_["x"] == 1: + ret_1 = self.tuple_1 + if self.dict_["z"] == 4: + ret_2 = self.tuple_2 + ret = ret_1 + ret_2 + return ret + + net = DictNet() + assert net() == (88, 99, 4, 5, 6) + + +def test_dict_set_or_get_item_2(): + class DictNet(Cell): + """DictNet1 definition""" + def __init__(self): + super(DictNet, self).__init__() + + def construct(self): + tuple_1 = (1, 2, 3) + tuple_2 = (4, 5, 6) + ret_1 = (88, 99) + ret_2 = (88, 99) + dict_ = {"x": 1, "y": 2} + dict_["x"] = 3 + dict_["z"] = 4 + + if dict_["x"] == 1: + ret_1 = tuple_1 + if dict_["z"] == 4: + ret_2 = tuple_2 + ret = ret_1 + ret_2 + return ret + + net = DictNet() + assert net() == (88, 99, 4, 5, 6) + + +def test_dict_set_or_get_item_3(): + class DictNet(Cell): + """DictNet1 definition""" + def __init__(self): + super(DictNet, self).__init__() + self.dict_ = {"x": Tensor(np.ones([2, 2, 3], np.float32)), "y": 1} + self.tensor = Tensor(np.ones([4, 2, 3], np.float32)) + + def construct(self): + self.dict_["y"] = 3 + if self.dict_["y"] == 3: + self.dict_["x"] = self.tensor + return self.dict_["x"] + + net = DictNet() + assert net() == Tensor(np.ones([4, 2, 3], np.float32)) + + diff --git a/tests/ut/python/dtype/test_hypermap.py b/tests/ut/python/dtype/test_hypermap.py new file mode 100644 index 00000000000..7bbccb0b22b --- /dev/null +++ b/tests/ut/python/dtype/test_hypermap.py @@ -0,0 +1,112 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import pytest + +from mindspore import Tensor, context +from mindspore.nn import Cell +from mindspore.ops import operations as P + +context.set_context(mode=context.GRAPH_MODE) + + +def test_hypermap_if(): + class Net(Cell): + """DictNet1 definition""" + + def __init__(self): + super(Net, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + self._list = [1, 2, 3] + + def construct(self, x, y): + if map(lambda a: a + 1, self._list): + ret = self.max(x, y) + else: + ret = self.min(x, y) + return ret + + net = Net() + x = Tensor(np.ones([3, 2, 3], np.float32)) + y = Tensor(np.ones([1, 2, 3], np.float32)) + net(x, y) + + +def test_hypermap_value(): + class Net(Cell): + """DictNet1 definition""" + + def __init__(self): + super(Net, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + self._list = [22, 66, 88, 111] + + def construct(self): + return map(lambda a: a + 1, self._list) + + net = Net() + assert net() == (23, 67, 89, 112) + + +def test_hypermap_func_const(): + class NetMap(Cell): + def __init__(self): + super(NetMap, self).__init__() + + def double(self, x): + return 2 * x + + def triple(self, x): + return 3 * x + + def square(self, x): + return x * x + + def construct(self): + _list = [self.double, self.triple, self.square] + return map(lambda f: f(4), _list) + + net = NetMap() + assert net() == (8, 12, 16) + + +""" +def test_hypermap_func_variable(): + class NetMap(Cell): + def __init__(self): + super(NetMap, self).__init__() + + def double(self, x): + return 2 * x + + def triple(self, x): + return 3 * x + + def square(self, x): + return x * x + + def construct(self, x): + _list = [self.double, self.triple, self.square] + return map(lambda f: f(x), _list) + + x = Tensor(np.ones([3, 2, 3], np.float32)) + net = NetMap() + + with pytest.raises(RuntimeError) as ex: + net(x) + assert "HyperMap don't support Closure with free variable yet" in str(ex.value) +""" diff --git a/tests/ut/python/exec/__init__.py b/tests/ut/python/exec/__init__.py new file mode 100644 index 00000000000..5443c0ca48e --- /dev/null +++ b/tests/ut/python/exec/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup for pytest""" +import mindspore.context as context + +# pylint: disable=unused-argument +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) diff --git a/tests/ut/python/exec/resnet_example.py b/tests/ut/python/exec/resnet_example.py new file mode 100644 index 00000000000..bfbb64f7321 --- /dev/null +++ b/tests/ut/python/exec/resnet_example.py @@ -0,0 +1,242 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +resnet50 example +""" +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from ..ut_filter import non_graph_engine + + +def conv3x3(in_channels, out_channels, stride=1, padding=1): + """3x3 convolution """ + weight = Tensor(np.ones([out_channels, in_channels, 3, 3]).astype(np.float32) * 0.01) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, weight_init=weight) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight = Tensor(np.ones([out_channels, in_channels, 1, 1]).astype(np.float32) * 0.01) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, weight_init=weight) + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = Tensor(np.ones(shape).astype(np.float32) * 0.01) + var = Tensor(np.ones(shape).astype(np.float32) * 0.01) + beta = Tensor(np.ones(shape).astype(np.float32) * 0.01) + gamma = Tensor(np.ones(shape).astype(np.float32) * 0.01) + return nn.BatchNorm2d(num_features=out_channels, + beta_init=beta, + gamma_init=gamma, + moving_mean_init=mean, + moving_var_init=var) + + +class ResidualBlock(nn.Cell): + """ + residual Block + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=1) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize(out_channels) + + self.relu = nn.ReLU() + self.downsample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, + stride=stride, padding=0) + self.bn_down_sample = bn_with_initialize(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + """ + :param x: + :return: + """ + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class MakeLayer3(nn.Cell): + """ + make resnet50 3 layers + """ + + def __init__(self, block, in_channels, out_channels, stride): + super(MakeLayer3, self).__init__() + self.block_down_sample = block(in_channels, out_channels, + stride=stride, down_sample=True) + self.block1 = block(out_channels, out_channels, stride=1) + self.block2 = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.block_down_sample(x) + x = self.block1(x) + x = self.block2(x) + + return x + + +class MakeLayer4(nn.Cell): + """ + make resnet50 4 layers + """ + + def __init__(self, block, in_channels, out_channels, stride): + super(MakeLayer4, self).__init__() + self.block_down_sample = block(in_channels, out_channels, + stride=stride, down_sample=True) + self.block1 = block(out_channels, out_channels, stride=1) + self.block2 = block(out_channels, out_channels, stride=1) + self.block3 = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.block_down_sample(x) + x = self.block1(x) + x = self.block2(x) + x = self.block3(x) + + return x + + +class MakeLayer6(nn.Cell): + """ + make resnet50 6 layers + + """ + + def __init__(self, block, in_channels, out_channels, stride): + super(MakeLayer6, self).__init__() + self.block_down_sample = block(in_channels, out_channels, + stride=stride, down_sample=True) + self.block1 = block(out_channels, out_channels, stride=1) + self.block2 = block(out_channels, out_channels, stride=1) + self.block3 = block(out_channels, out_channels, stride=1) + self.block4 = block(out_channels, out_channels, stride=1) + self.block5 = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.block_down_sample(x) + x = self.block1(x) + x = self.block2(x) + x = self.block3(x) + x = self.block4(x) + x = self.block5(x) + + return x + + +class ResNet50(nn.Cell): + """ + resnet nn.Cell + """ + + def __init__(self, block, num_classes=100): + super(ResNet50, self).__init__() + + weight_conv = Tensor(np.ones([64, 3, 7, 7]).astype(np.float32) * 0.01) + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, weight_init=weight_conv) + self.bn1 = bn_with_initialize(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.layer1 = MakeLayer3( + block, in_channels=64, out_channels=256, stride=1) + self.layer2 = MakeLayer4( + block, in_channels=256, out_channels=512, stride=2) + self.layer3 = MakeLayer6( + block, in_channels=512, out_channels=1024, stride=2) + self.layer4 = MakeLayer3( + block, in_channels=1024, out_channels=2048, stride=2) + + self.avgpool = nn.AvgPool2d(7, 1) + self.flatten = nn.Flatten() + + weight_fc = Tensor(np.ones([num_classes, 512 * block.expansion]).astype(np.float32) * 0.01) + bias_fc = Tensor(np.ones([num_classes]).astype(np.float32) * 0.01) + self.fc = nn.Dense(512 * block.expansion, num_classes, weight_init=weight_fc, bias_init=bias_fc) + + def construct(self, x): + """ + :param x: + :return: + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.flatten(x) + x = self.fc(x) + + return x + + +def resnet50(): + return ResNet50(ResidualBlock, 10) + + +@non_graph_engine +def test_compile(): + net = resnet50() + input_data = Tensor(np.ones([1, 3, 224, 224]).astype(np.float32) * 0.01) + + output = net(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_AssignAdd.py b/tests/ut/python/exec/test_AssignAdd.py new file mode 100644 index 00000000000..6b7fc78802d --- /dev/null +++ b/tests/ut/python/exec/test_AssignAdd.py @@ -0,0 +1,97 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test assign add +""" +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore import Tensor, Parameter +import mindspore as ms +from ..ut_filter import non_graph_engine +from mindspore.common.api import _executor +import mindspore.context as context +import pytest +context.set_context(mode=context.GRAPH_MODE) + +class Net(nn.Cell): + """Net definition""" + def __init__(self): + super(Net, self).__init__() + self.AssignAdd = P.AssignAdd() + self.inputdata = Parameter(initializer(1, [1], ms.int64), name="global_step") + print("inputdata: ", self.inputdata) + + def construct(self, x): + out = self.AssignAdd(self.inputdata, x) + return out + +@non_graph_engine +def test_AssignAdd_1(): + """test AssignAdd 1""" + import mindspore.context as context + context.set_context(mode=context.GRAPH_MODE) + net = Net() + x = Tensor(np.ones([1]).astype(np.int64)*100) + + print("MyPrintResult dataX:", x) + result = net(x) + print("MyPrintResult data::", result) + expect = np.ones([1]).astype(np.int64)*101 + diff = result.asnumpy() - expect + + print("MyPrintExpect:", expect) + print("MyPrintDiff:", diff) + error = np.ones(shape=[1]) * 1.0e-3 + assert np.all(diff < error) + +@non_graph_engine +def test_AssignAdd_2(): + """test AssignAdd 2""" + import mindspore.context as context + context.set_context(mode=context.GRAPH_MODE) + net = Net() + x = Tensor(np.ones([1]).astype(np.int64)*102) + + print("MyPrintResult dataX:", x) + result = net(x) + print("MyPrintResult data::", result.asnumpy()) + expect = np.ones([1]).astype(np.int64)*103 + diff = result.asnumpy() - expect + + print("MyPrintExpect:", expect) + print("MyPrintDiff:", diff) + error = np.ones(shape=[1]) * 1.0e-3 + assert np.all(diff < error) + +class AssignAddNet(nn.Cell): + """Net definition""" + def __init__(self): + super(AssignAddNet, self).__init__() + self.AssignAdd = P.AssignAdd() + self.inputdata = Parameter(initializer(1, [1], ms.float16), name="KIND_AUTOCAST_SCALAR_TO_TENSOR") + self.one = 1 + + def construct(self, ixt): + z1 = self.AssignAdd(self.inputdata, self.one) + return z1 + +@non_graph_engine +def test_assignadd_scalar_cast(): + net = AssignAddNet() + x = Tensor(np.ones([1]).astype(np.int64)*102) + #_executor.compile(net, 1) + result = net(x) diff --git a/tests/ut/python/exec/test_activation.py b/tests/ut/python/exec/test_activation.py new file mode 100644 index 00000000000..6f41bb986a1 --- /dev/null +++ b/tests/ut/python/exec/test_activation.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Activations """ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from ..ut_filter import non_graph_engine + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.relu = nn.ReLU() + + def construct(self, input_x): + return self.relu(input_x) + + +@non_graph_engine +def test_compile(): + net = Net() + input_data = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32) - 0.5) + output = net(input_data) + print(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_assign_sub.py b/tests/ut/python/exec/test_assign_sub.py new file mode 100644 index 00000000000..5ff0e9e9e06 --- /dev/null +++ b/tests/ut/python/exec/test_assign_sub.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test assign sub +""" +import numpy as np +import mindspore.nn as nn +import mindspore.ops.operations as P +from mindspore import Tensor +import mindspore.context as context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE) + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.b = Parameter(initializer('ones', [5]), name='b') + self.sub = P.AssignSub() + + def construct(self, value): + return self.sub(self.b, value) + + +def test_net(): + net = Net() + input_data = Tensor(np.ones([5]).astype(np.float32)) + output = net(input_data) + print(output.asnumpy().shape) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_batchnorm.py b/tests/ut/python/exec/test_batchnorm.py new file mode 100644 index 00000000000..7524bb8ee21 --- /dev/null +++ b/tests/ut/python/exec/test_batchnorm.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ut for batchnorm layer""" +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from ..ut_filter import non_graph_engine + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bn = nn.BatchNorm2d(num_features=3, eps=1e-5, momentum=0.1) + + def construct(self, input_x): + return self.bn(input_x) + + +@non_graph_engine +def test_compile(): + net = Net() + input_data = Tensor(np.ones([1, 3, 4, 4]).astype(np.float32)) + output = net(input_data) + print(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_bias_add.py b/tests/ut/python/exec/test_bias_add.py new file mode 100644 index 00000000000..75a15f93bfb --- /dev/null +++ b/tests/ut/python/exec/test_bias_add.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test BiasAdd """ +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore import Tensor, Parameter +from ..ut_filter import non_graph_engine + +class Net(nn.Cell): + """Net definition""" + def __init__(self, + output_channels, + bias_init='zeros', + ): + super(Net, self).__init__() + self.biasAdd = P.BiasAdd() + + if isinstance(bias_init, Tensor): + if bias_init.dim() != 1 or bias_init.shape()[0] != output_channels: + raise ValueError("bias_init shape error") + + self.bias = Parameter(initializer( + bias_init, [output_channels]), name="bias") + + def construct(self, input_x): + return self.biasAdd(input_x, self.bias) + + +@non_graph_engine +def test_compile(): + bias_init = Tensor(np.ones([3]).astype(np.float32)) + net = Net(3, bias_init=bias_init) + input_data = Tensor(np.ones([1, 3, 3, 3], np.float32)) + # since simulator currently not support matMul + # enable it when staging function is ready + output = net(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_conv.py b/tests/ut/python/exec/test_conv.py new file mode 100644 index 00000000000..bbddcd99bee --- /dev/null +++ b/tests/ut/python/exec/test_conv.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test conv""" +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from ..ut_filter import non_graph_engine + +weight = Tensor(np.ones([2, 2])) +in_channels = 3 +out_channels = 64 + + +class Net(nn.Cell): + """Net definition""" + def __init__(self, + cin, + cout, + kernel_size, + stride=1, + pad_mode='pad', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + super(Net, self).__init__() + Tensor(np.ones([6, 3, 3, 3]).astype(np.float32) * 0.01) + self.conv = nn.Conv2d(cin, + cout, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + + def construct(self, input_x): + return self.conv(input_x) + + +@non_graph_engine +def test_compile(): + net = Net(3, 6, (3, 3), bias_init='zeros') + input_data = Tensor(np.ones([3, 3, 32, 32]).astype(np.float32) * 0.01) + output = net(input_data) + print(output.asnumpy()) + + +@non_graph_engine +def test_compile2(): + net = Net(3, 1, (3, 3), bias_init='zeros') + input_data = Tensor(np.ones([1, 3, 32, 32]).astype(np.float32) * 0.01) + output = net(input_data) + print(output.asnumpy()) + +@non_graph_engine +def test_compile3(): + net = Net(3, 1, (3, 3), weight_init='ONES') + input_data = Tensor(np.ones([1, 3, 32, 32]).astype(np.float32) * 0.01) + output = net(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_dense.py b/tests/ut/python/exec/test_dense.py new file mode 100644 index 00000000000..c9c09c50cfe --- /dev/null +++ b/tests/ut/python/exec/test_dense.py @@ -0,0 +1,59 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Dense """ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from ..ut_filter import non_graph_engine + +class Net(nn.Cell): + """Net definition""" + def __init__(self, + input_channels, + output_channels, + weight='normal', + bias='zeros', + has_bias=True): + super(Net, self).__init__() + self.fc = nn.Dense(input_channels, + output_channels, + weight, + bias, + has_bias) + + def construct(self, input_x): + return self.fc(input_x) + + +@non_graph_engine +def test_compile(): + weight = Tensor(np.ones([12, 8], np.float32)) + bias = Tensor(np.ones([12], np.float32)) + net = Net(8, 12, weight=weight, bias=bias) + input_data = Tensor(np.ones([1, 8], np.float32)) + # since simulator currently not support matMul + output = net(input_data) + print(output.asnumpy()) + + +@non_graph_engine +def test_compile_nobias(): + weight = Tensor(np.ones([12, 8], np.float32)) + net = Net(8, 12, weight=weight, has_bias=False) + input_data = Tensor(np.ones([1, 8], np.float32)) + # since simulator currently not support matMu + # enable it when staging function is ready + output = net(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_eval.py b/tests/ut/python/exec/test_eval.py new file mode 100644 index 00000000000..0edf9cb7489 --- /dev/null +++ b/tests/ut/python/exec/test_eval.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test eval""" +import numpy as np +import mindspore as ms +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor +from mindspore import context +from ..ut_filter import non_graph_engine + + +class Net(nn.Cell): + """Net definition""" + + def __init__(self, + cin, + cout, + kernel_size, + stride=1, + pad_mode='pad', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + super(Net, self).__init__() + Tensor(np.ones([6, 3, 3, 3]).astype(np.float32) * 0.01) + self.conv = nn.Conv2d(cin, + cout, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + + def construct(self, input_x): + return self.conv(input_x) + + +@non_graph_engine +def test_compile_train_eval(): + """test_compile_train_eval""" + net = Net(3, 1, (3, 3), bias_init='zeros') + train_input_data = Tensor(np.ones([1, 3, 32, 32]).astype(np.float32) * 0.01) + context.set_context(mode=context.GRAPH_MODE) + + ms_executor = _executor + + ms_executor.init_dataset("train", 1, 1, [ms.float32], [[1, 3, 32, 32]], (), 'dataset') + + ms_executor.compile(net, train_input_data, phase='train') + ms_executor(net, train_input_data, phase='train') + + ms_executor.init_dataset("eval", 1, 1, [ms.float32], [[1, 3, 32, 32]], (), phase='eval_dataset') + + valid_input_data = Tensor(np.ones([1, 3, 32, 32]).astype(np.float32) * 0.01) + ms_executor.compile(net, valid_input_data, phase='eval') + ms_executor(net, valid_input_data, phase='eval') diff --git a/tests/ut/python/exec/test_flatten.py b/tests/ut/python/exec/test_flatten.py new file mode 100644 index 00000000000..dfb3e181e77 --- /dev/null +++ b/tests/ut/python/exec/test_flatten.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Flatten """ +import numpy as np + +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from ..ut_filter import non_graph_engine + + +class Net(nn.Cell): + def __init__(self, + ): + super(Net, self).__init__() + self.flatten = P.Flatten() + + def construct(self, input_x): + return self.flatten(input_x) + + +@non_graph_engine +def test_compile(): + net = Net() + input_data = Tensor(np.ones([1, 2, 4, 4], np.float32)) + # enable it when staging function is ready + output = net(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_pooling.py b/tests/ut/python/exec/test_pooling.py new file mode 100644 index 00000000000..9c378c15c2c --- /dev/null +++ b/tests/ut/python/exec/test_pooling.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test pooling api +""" +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor + +class MaxNet(nn.Cell): + """MaxNet definition""" + def __init__(self, + kernel_size, + stride=None, + padding=0): + super(MaxNet, self).__init__() + self.maxpool = nn.MaxPool2d(kernel_size, + stride, + padding=padding) + + def construct(self, input_x): + return self.maxpool(input_x) + + +class AvgNet(nn.Cell): + def __init__(self, + kernel_size, + stride=None): + super(AvgNet, self).__init__() + self.avgpool = nn.AvgPool2d(kernel_size, + stride) + + def construct(self, input_x): + return self.avgpool(input_x) diff --git a/tests/ut/python/exec/test_softmax.py b/tests/ut/python/exec/test_softmax.py new file mode 100644 index 00000000000..b3144723e38 --- /dev/null +++ b/tests/ut/python/exec/test_softmax.py @@ -0,0 +1,35 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test softmax api +""" +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor + +class Net(nn.Cell): + def __init__(self, dim): + super(Net, self).__init__() + self.softmax = nn.Softmax(dim) + + def construct(self, input_x): + return self.softmax(input_x) + + +def test_compile(): + net = Net(0) + input_data = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype(np.float32)) + output = net(input_data) + print(output.asnumpy()) diff --git a/tests/ut/python/exec/test_tensor_add.py b/tests/ut/python/exec/test_tensor_add.py new file mode 100644 index 00000000000..14cebd8c8f4 --- /dev/null +++ b/tests/ut/python/exec/test_tensor_add.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test TensorAdd """ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + + def construct(self, input1, input2): + return self.add(input1, input2) + + +def test_tensor_add(): + """test_tensor_add""" + add = P.TensorAdd() + input1 = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32)) + input2 = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32)) + output = add(input1, input2) + output_np = output.asnumpy() + input1_np = input1.asnumpy() + input2_np = input2.asnumpy() + print(input1_np[0][0][0][0]) + print(input2_np[0][0][0][0]) + print(output_np[0][0][0][0]) + assert isinstance(output_np[0][0][0][0], np.float32) diff --git a/tests/ut/python/exec/test_train.py b/tests/ut/python/exec/test_train.py new file mode 100644 index 00000000000..549fe372c60 --- /dev/null +++ b/tests/ut/python/exec/test_train.py @@ -0,0 +1,170 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test model train """ +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore import Tensor, Parameter, Model +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim import Momentum + +# fn is a funcation use i as input +def lr_gen(fn, epoch_size): + for i in range(epoch_size): + yield fn(i) + +def me_train_tensor(net, input_np, label_np, epoch_size=2): + """me_train_tensor""" + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr_gen(lambda i: 0.1, epoch_size), 0.9, 0.01, 1024) + Model(net, loss, opt) + _network = nn.WithLossCell(net, loss) + _train_net = nn.TrainOneStepCell(_network, opt) + _train_net.set_train() + label_np = np.argmax(label_np, axis=-1).astype(np.int32) + for epoch in range(0, epoch_size): + print(f"epoch %d"%(epoch)) + _train_net(Tensor(input_np), Tensor(label_np)) + + +def test_bias_add(test_with_simu): + """test_bias_add""" + import mindspore.context as context + is_pynative_mode = (context.get_context("mode") == context.PYNATIVE_MODE) + # training api is implemented under Graph mode + if is_pynative_mode: + context.set_context(mode=context.GRAPH_MODE) + if test_with_simu: + return + + class Net(nn.Cell): + """Net definition""" + def __init__(self, + output_channels, + bias_init='zeros', + ): + super(Net, self).__init__() + self.biasAdd = P.BiasAdd() + + if isinstance(bias_init, Tensor): + if bias_init.dim() != 1 or bias_init.shape()[0] != output_channels: + raise ValueError("bias_init shape error") + + self.bias = Parameter(initializer( + bias_init, [output_channels]), name="bias") + + def construct(self, input_x): + return self.biasAdd(input_x, self.bias) + + bias_init = Tensor(np.ones([3]).astype(np.float32)) + input_np = np.ones([1, 3, 3, 3], np.float32) + label_np = np.ones([1, 3, 3, 3], np.int32) * 2 + me_train_tensor(Net(3, bias_init=bias_init), input_np, label_np) + + +def test_conv(test_with_simu): + """test_conv""" + import mindspore.context as context + is_pynative_mode = (context.get_context("mode") == context.PYNATIVE_MODE) + # training api is implemented under Graph mode + if is_pynative_mode: + context.set_context(mode=context.GRAPH_MODE) + if test_with_simu: + return + + class Net(nn.Cell): + "Net definition""" + def __init__(self, + cin, + cout, + kernel_size): + super(Net, self).__init__() + Tensor(np.ones([6, 3, 3, 3]).astype(np.float32) * 0.01) + self.conv = nn.Conv2d(cin, + cout, + kernel_size) + + def construct(self, input_x): + return self.conv(input_x) + + net = Net(3, 6, (3, 3)) + input_np = np.ones([1, 3, 32, 32]).astype(np.float32) * 0.01 + label_np = np.ones([1, 6, 32, 32]).astype(np.int32) + me_train_tensor(net, input_np, label_np) + + +def test_net(): + """test_net""" + import mindspore.context as context + is_pynative_mode = (context.get_context("mode") == context.PYNATIVE_MODE) + # training api is implemented under Graph mode + if is_pynative_mode: + context.set_context(mode=context.GRAPH_MODE) + + class Net(nn.Cell): + """Net definition""" + def __init__(self): + super(Net, self).__init__() + Tensor(np.ones([64, 3, 7, 7]).astype(np.float32) * 0.01) + self.conv = nn.Conv2d(3, 64, (7, 7), pad_mode="same", stride=2) + self.relu = nn.ReLU() + self.bn = nn.BatchNorm2d(64) + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.dense = nn.Dense(64, 12) + + def construct(self, input_x): + output = input_x + output = self.conv(output) + output = self.bn(output) + output = self.relu(output) + output = self.mean(output, (-2, -1)) + output = self.flatten(output) + output = self.dense(output) + return output + + net = Net() + input_np = np.ones([32, 3, 224, 224]).astype(np.float32) * 0.01 + label_np = np.ones([32, 12]).astype(np.int32) + me_train_tensor(net, input_np, label_np) + +def test_bn(): + """test_bn""" + import mindspore.context as context + is_pynative_mode = (context.get_context("mode") == context.PYNATIVE_MODE) + # training api is implemented under Graph mode + if is_pynative_mode: + context.set_context(mode=context.GRAPH_MODE) + + class Net(nn.Cell): + """Net definition""" + def __init__(self, cin, cout): + super(Net, self).__init__() + self.bn = nn.BatchNorm2d(cin) + self.flatten = nn.Flatten() + self.dense = nn.Dense(cin, cout) + + def construct(self, input_x): + output = input_x + output = self.bn(output) + output = self.flatten(output) + output = self.dense(output) + return output + + net = Net(2048, 16) + input_np = np.ones([32, 2048, 1, 1]).astype(np.float32) * 0.01 + label_np = np.ones([32, 16]).astype(np.int32) + me_train_tensor(net, input_np, label_np) diff --git a/tests/ut/python/exec/test_train_with_lars.py b/tests/ut/python/exec/test_train_with_lars.py new file mode 100644 index 00000000000..d4ca2ed8c3a --- /dev/null +++ b/tests/ut/python/exec/test_train_with_lars.py @@ -0,0 +1,85 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test model train """ +import mindspore.nn as nn +from mindspore import Tensor, Model +from mindspore.common import dtype as mstype +from mindspore.common.parameter import ParameterTuple, Parameter +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim import Momentum +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P + +def get_reordered_parameters(parameters): + """get_reordered_parameters""" + # put the bias parameter to the end + non_bias_param = [] + bias_param = [] + for item in parameters: + if item.name.find("bias") >= 0: + bias_param.append(item) + else: + non_bias_param.append(item) + reordered_params = tuple(non_bias_param + bias_param) + return len(non_bias_param), len(reordered_params), reordered_params + +def get_net_trainable_reordered_params(net): + params = net.trainable_params() + return get_reordered_parameters(params) + +class TrainOneStepWithLarsCell(nn.Cell): + """TrainOneStepWithLarsCell definition""" + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepWithLarsCell, self).__init__(auto_prefix=False) + self.network = network + self.slice_index, self.params_len, weights = get_net_trainable_reordered_params(self.network) + self.weights = ParameterTuple(weights) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + self.sens = Parameter(Tensor([sens], mstype.float32), name='sens', requires_grad=False) + self.weight_decay = 1.0 + self.lars = P.Lars(epsilon=1.0, hyperpara=1.0) + + def construct(self, data, label): + weights = self.weights + loss = self.network(data, label) + grads = self.grad(self.network, weights)(data, label, self.sens) + non_bias_weights = weights[0: self.slice_index] + non_bias_grads = grads[0: self.slice_index] + bias_grads = grads[self.slice_index: self.params_len] + lars_grads = self.lars(non_bias_weights, non_bias_grads, self.weight_decay) + new_grads = lars_grads + bias_grads + return F.depend(loss, self.optimizer(new_grads)) + +# fn is a funcation use i as input +def lr_gen(fn, epoch_size): + for i in range(epoch_size): + yield fn(i) + +def me_train_tensor(net, input_np, label_np, epoch_size=2): + """me_train_tensor""" + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + # reorder the net parameters , leave the parameters that need to be passed into lars to the end part + + opt = Momentum(get_net_trainable_reordered_params(net)[2], lr_gen(lambda i: 0.1, epoch_size), 0.9, 0.01, 1024) + Model(net, loss, opt) + _network = nn.WithLossCell(net, loss) + TrainOneStepWithLarsCell(_network, opt) + data = Tensor(input_np) + label = Tensor(label_np) + net(data, label) diff --git a/tests/ut/python/hccl_test/__init__.py b/tests/ut/python/hccl_test/__init__.py new file mode 100644 index 00000000000..6bb9ad63ceb --- /dev/null +++ b/tests/ut/python/hccl_test/__init__.py @@ -0,0 +1 @@ +"""hccl init""" diff --git a/tests/ut/python/hccl_test/manage/__init__.py b/tests/ut/python/hccl_test/manage/__init__.py new file mode 100644 index 00000000000..6bb9ad63ceb --- /dev/null +++ b/tests/ut/python/hccl_test/manage/__init__.py @@ -0,0 +1 @@ +"""hccl init""" diff --git a/tests/ut/python/hccl_test/manage/api.py b/tests/ut/python/hccl_test/manage/api.py new file mode 100644 index 00000000000..8dac167a3fd --- /dev/null +++ b/tests/ut/python/hccl_test/manage/api.py @@ -0,0 +1,82 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""api definition""" +import threading + +class Hccl(): + """Hccl definition""" + _instance_lock = threading.Lock() + _instance = None + _rank_id = 0 + _rank_size = 1 + + def __init__(self): + pass + + # pylint: disable=unused-argument + def __new__(cls, *args, **kwargs): + if not hasattr(Hccl, "_instance") or Hccl._instance is None: + with Hccl._instance_lock: + if not hasattr(Hccl, + "_instance") or Hccl._instance is None: + Hccl._instance = object.__new__(cls) + Hccl._instance.__init__() + return Hccl._instance + + @property + def rank_id(self): + return self._rank_id + + @rank_id.setter + def rank_id(self, rank_id): + self._rank_id = rank_id + + @property + def rank_size(self): + return self._rank_size + + @rank_size.setter + def rank_size(self, size): + self._rank_size = size + +# pylint: disable=unused-argument +def get_rank_id(group=None): + hccl = Hccl() + return hccl.rank_id + + +def get_rank_size(group=None): + hccl = Hccl() + if group is None: + return hccl.rank_size + if isinstance(group, str): + return int(group.split("-")[0]) + raise ValueError + +# pylint: disable=unused-argument +def get_world_rank_from_group_rank(group, group_rank_id): + return group_rank_id + +# pylint: disable=unused-argument +def get_group_rank_from_world_rank(world_rank_id, group): + return world_rank_id + +# pylint: disable=unused-argument +def create_group(group, rank_size, rank_ids): + pass + +# pylint: disable=unused-argument +def destroy_group(group): + pass diff --git a/tests/ut/python/ir/__init__.py b/tests/ut/python/ir/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/ir/test_dtype.py b/tests/ut/python/ir/test_dtype.py new file mode 100644 index 00000000000..31a44585409 --- /dev/null +++ b/tests/ut/python/ir/test_dtype.py @@ -0,0 +1,134 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_dtype""" +from dataclasses import dataclass +import numpy as np +import pytest + +import mindspore as ms +from mindspore.common import dtype + +def test_dtype_to_nptype(): + """test_dtype2nptype""" + assert ms.dtype_to_nptype(ms.bool_) == np.bool_ + assert ms.dtype_to_nptype(ms.int8) == np.int8 + assert ms.dtype_to_nptype(ms.int16) == np.int16 + assert ms.dtype_to_nptype(ms.int32) == np.int32 + assert ms.dtype_to_nptype(ms.int64) == np.int64 + assert ms.dtype_to_nptype(ms.uint8) == np.uint8 + assert ms.dtype_to_nptype(ms.uint16) == np.uint16 + assert ms.dtype_to_nptype(ms.uint32) == np.uint32 + assert ms.dtype_to_nptype(ms.uint64) == np.uint64 + assert ms.dtype_to_nptype(ms.float16) == np.float16 + assert ms.dtype_to_nptype(ms.float32) == np.float32 + assert ms.dtype_to_nptype(ms.float64) == np.float64 + + +def test_dtype_to_pytype(): + """test_dtype_to_pytype""" + assert ms.dtype_to_pytype(ms.bool_) == bool + assert ms.dtype_to_pytype(ms.int8) == int + assert ms.dtype_to_pytype(ms.int16) == int + assert ms.dtype_to_pytype(ms.int32) == int + assert ms.dtype_to_pytype(ms.int64) == int + assert ms.dtype_to_pytype(ms.uint8) == int + assert ms.dtype_to_pytype(ms.uint16) == int + assert ms.dtype_to_pytype(ms.uint32) == int + assert ms.dtype_to_pytype(ms.uint64) == int + assert ms.dtype_to_pytype(ms.float16) == float + assert ms.dtype_to_pytype(ms.float32) == float + assert ms.dtype_to_pytype(ms.float64) == float + assert ms.dtype_to_pytype(ms.list_) == list + assert ms.dtype_to_pytype(ms.tuple_) == tuple + assert ms.dtype_to_pytype(ms.string) == str + assert ms.dtype_to_pytype(ms.type_none) == type(None) + + +@dataclass +class Foo: + x: int + def inf(self): + return self.x + + +def get_class_attrib_types(cls): + """ + get attrib type of dataclass + """ + fields = cls.__dataclass_fields__ + attr_type = [field.type for name, field in fields.items()] + return attr_type + + +def test_dtype(): + """test_dtype""" + x = 1.5 + me_type = dtype.get_py_obj_dtype(x) + assert me_type == ms.float64 + me_type = dtype.get_py_obj_dtype(type(x)) + assert me_type == ms.float64 + + x = 100 + me_type = dtype.get_py_obj_dtype(type(x)) + assert me_type == ms.int64 + me_type = dtype.get_py_obj_dtype(x) + assert me_type == ms.int64 + + x = False + me_type = dtype.get_py_obj_dtype(type(x)) + assert me_type == ms.bool_ + me_type = dtype.get_py_obj_dtype(x) + assert me_type == ms.bool_ + + # support str + # x = "string type" + + x = [1, 2, 3] + me_type = dtype.get_py_obj_dtype(x) + assert me_type == ms.list_ + me_type = dtype.get_py_obj_dtype(type(x)) + assert me_type == ms.list_ + + x = (2, 4, 5) + me_type = dtype.get_py_obj_dtype(x) + assert me_type == ms.tuple_ + me_type = dtype.get_py_obj_dtype(type(x)) + assert me_type == ms.tuple_ + + y = Foo(3) + me_type = dtype.get_py_obj_dtype(y.x) + assert me_type == ms.int64 + me_type = dtype.get_py_obj_dtype(type(y.x)) + assert me_type == ms.int64 + + y = Foo(3.1) + me_type = dtype.get_py_obj_dtype(y.x) + assert me_type == ms.float64 + me_type = dtype.get_py_obj_dtype(type(y.x)) + assert me_type == ms.float64 + + fields = get_class_attrib_types(y) + assert len(fields) == 1 + me_type = dtype.get_py_obj_dtype(fields[0]) + assert me_type == ms.int64 + + fields = get_class_attrib_types(Foo) + assert len(fields) == 1 + me_type = dtype.get_py_obj_dtype(fields[0]) + assert me_type == ms.int64 + + with pytest.raises(NotImplementedError): + x = 1.5 + dtype.get_py_obj_dtype(type(type(x))) diff --git a/tests/ut/python/ir/test_tensor.py b/tests/ut/python/ir/test_tensor.py new file mode 100644 index 00000000000..d4f96b54eb0 --- /dev/null +++ b/tests/ut/python/ir/test_tensor.py @@ -0,0 +1,380 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_tensor.py +@Author: +@Date : 2019-03-14 +@Desc : test mindspore tensor's operation +""" +import numpy as np +import pytest + +import mindspore as ms +import mindspore.common.api as me +import mindspore.nn as nn +from ..ut_filter import non_graph_engine + + +ndarr = np.ones((2, 3)) + +def test_tensor_flatten(): + with pytest.raises(AttributeError): + lst = [1, 2, 3, 4,] + tensor_list = ms.Tensor(lst, ms.float32) + tensor_list = tensor_list.Flatten() + print(tensor_list) + +def test_tensor_list(): + lst = [[1.0, 2.0, 1.0], [1.0, 10.0, 9.0]] + tensor_list = ms.Tensor(lst, ms.float32) + print(tensor_list) + +def test_tensor(): + """test_tensor""" + t1 = ms.Tensor(ndarr) + assert isinstance(t1, ms.Tensor) + assert t1.dtype() == ms.float64 + + t2 = ms.Tensor(np.zeros([1, 2, 3]), ms.float32) + assert isinstance(t2, ms.Tensor) + assert t2.shape() == (1, 2, 3) + assert t2.dtype() == ms.float32 + + t3 = ms.Tensor(0.1) + assert isinstance(t3, ms.Tensor) + assert t3.dtype() == ms.float64 + + t4 = ms.Tensor(1) + assert isinstance(t4, ms.Tensor) + assert t4.dtype() == ms.int64 + +def test_tensor_type_float16(): + t_float16 = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float16)) + assert isinstance(t_float16, ms.Tensor) + assert t_float16.shape() == (2, 3) + assert t_float16.dtype() == ms.float16 + + +def test_tensor_type_float32(): + t_float32 = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)) + assert isinstance(t_float32, ms.Tensor) + assert t_float32.shape() == (2, 3) + assert t_float32.dtype() == ms.float32 + + +def test_tensor_type_float32_user_define(): + t = ms.Tensor(np.zeros([1, 2, 3]), ms.float32) + assert isinstance(t, ms.Tensor) + assert t.shape() == (1, 2, 3) + assert t.dtype() == ms.float32 + + +def test_tensor_type_float64(): + t = ms.Tensor([[1.0, 2, 3], [4, 5, 6]]) + assert isinstance(t, ms.Tensor) + assert t.shape() == (2, 3) + assert t.dtype() == ms.float64 + + t_zero = ms.Tensor(np.zeros([1, 2, 3])) + assert isinstance(t_zero, ms.Tensor) + assert t_zero.shape() == (1, 2, 3) + assert t_zero.dtype() == ms.float64 + + +def test_tensor_type_float64_user_define(): + t = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=float)) + assert isinstance(t, ms.Tensor) + assert t.shape() == (2, 3) + assert t.dtype() == ms.float64 + + t_float64 = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]]), ms.float64) + assert isinstance(t_float64, ms.Tensor) + assert t_float64.shape() == (2, 3) + assert t_float64.dtype() == ms.float64 + +def test_tensor_type_bool(): + # init a tensor with bool type + ts_bool_array = ms.Tensor(np.zeros([2, 3], np.bool), ms.bool_) + assert isinstance(ts_bool_array, ms.Tensor) + assert ts_bool_array.dtype() == ms.bool_ + + t_bool = ms.Tensor(True) + assert isinstance(t_bool, ms.Tensor) + assert t_bool.dtype() == ms.bool_ + + t_bool_array = ms.Tensor(np.array([[True, False, True], [False, False, False]])) + assert isinstance(t_bool_array, ms.Tensor) + assert t_bool_array.shape() == (2, 3) + assert t_bool_array.dtype() == ms.bool_ + +def test_tensor_type_int8(): + t_int8_array = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int8)) + assert isinstance(t_int8_array, ms.Tensor) + assert t_int8_array.shape() == (2, 3) + assert t_int8_array.dtype() == ms.int8 + + +def test_tensor_type_int16(): + t_int16_array = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int16)) + assert isinstance(t_int16_array, ms.Tensor) + assert t_int16_array.shape() == (2, 3) + assert t_int16_array.dtype() == ms.int16 + + +def test_tensor_type_int32(): + t_int = ms.Tensor([[1, 2, 3], [4, 5, 6]]) + assert isinstance(t_int, ms.Tensor) + assert t_int.shape() == (2, 3) + assert t_int.dtype() == ms.int64 + + t_int_array = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32)) + assert isinstance(t_int_array, ms.Tensor) + assert t_int_array.shape() == (2, 3) + assert t_int_array.dtype() == ms.int32 + + +def test_tensor_type_int64(): + t_int64 = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64)) + assert isinstance(t_int64, ms.Tensor) + assert t_int64.shape() == (2, 3) + assert t_int64.dtype() == ms.int64 + +def test_tensor_type_uint8(): + t_uint8_array = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8)) + assert isinstance(t_uint8_array, ms.Tensor) + assert t_uint8_array.shape() == (2, 3) + assert t_uint8_array.dtype() == ms.uint8 + + +def test_tensor_type_uint16(): + t_uint16_array = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint16)) + assert isinstance(t_uint16_array, ms.Tensor) + assert t_uint16_array.shape() == (2, 3) + assert t_uint16_array.dtype() == ms.uint16 + + +def test_tensor_type_uint32(): + t_uint32_array = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint32)) + assert isinstance(t_uint32_array, ms.Tensor) + assert t_uint32_array.shape() == (2, 3) + assert t_uint32_array.dtype() == ms.uint32 + + +def test_tensor_type_uint64(): + t_uint64 = ms.Tensor(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint64)) + assert isinstance(t_uint64, ms.Tensor) + assert t_uint64.shape() == (2, 3) + assert t_uint64.dtype() == ms.uint64 + +def test_set_type(): + t = ms.Tensor(ndarr) + t.set_dtype(ms.float32) + assert t.dtype() == ms.float32 + + +@non_graph_engine +def test_add(): + x = ms.Tensor(ndarr) + y = ms.Tensor(ndarr) + z = x + y + assert isinstance(z, ms.Tensor) + + +@non_graph_engine +def test_sub(): + x = ms.Tensor(ndarr) + y = ms.Tensor(ndarr) + z = x - y + assert isinstance(z, ms.Tensor) + + +class Net(nn.Cell): + """Net definition""" + def __init__(self, dim): + super(Net, self).__init__() + self.dim = dim + + def construct(self, input_x): + return input_x + + +@non_graph_engine +def test_return_tensor(): + """test_return_tensor""" + net = Net(0) + input_data = ms.Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + input_data.set_dtype(ms.float32) + exe = me._executor + exe.compile(net, input_data) + tensor_ = exe(net, input_data) + + # get shape + shape_ = tensor_.shape() + print("shape = ", shape_) + + # get type + type_ = tensor_.dtype() + print("type = ", type_) + + # get value + value_ = tensor_.asnumpy() + print("numpy value = ", value_) + + +def test_tensor_contiguous(): + """test_tensor_contiguous""" + input_c = np.arange(6).reshape(2, 3) + input_f = input_c.T + np.ascontiguousarray(input_c, dtype=np.float32) + assert True, input_c.flags['C_CONTIGUOUS'] + + print("input_f flags = ", input_f.flags) + assert True, input_f.flags['F_CONTIGUOUS'] + + tensor_f_float32 = ms.Tensor(input_f) + rt_f = tensor_f_float32.asnumpy() + assert True, rt_f.flags['C_CONTIGUOUS'] + print("rt_f flags = ", rt_f.flags) + +def test_tensor_contiguous2(): + input_data = np.random.randn(32, 112, 112, 3).astype(np.float32) + input_me = input_data.transpose(0, 3, 1, 2) + print("input_me flags = ", input_me.flags) + tensor_f_float32 = ms.Tensor(input_me) + out_f = tensor_f_float32.asnumpy() + print("out_f flags = ", out_f.flags) + +def test_tensor_input_string(): + with pytest.raises(TypeError): + input_data = 'ccc' + ms.Tensor(input_data) + +def test_tensor_input_tuple_string(): + with pytest.raises(TypeError): + input_data = (2, 3, '4', 5) + ms.Tensor(input_data) + +def test_tensor_input_list_string(): + with pytest.raises(TypeError): + input_data = [[2, 3, '4', 5], [1, 2, 3, 4]] + ms.Tensor(input_data) + +def test_tensor_input_none(): + with pytest.raises(TypeError): + input_data = None + ms.Tensor(input_data, np.int64) + +# pylint: disable=no-value-for-parameter +def test_tensor_input_empty(): + with pytest.raises(TypeError): + ms.Tensor() + +def test_tensor_input_ndarray_str(): + with pytest.raises(TypeError): + inp = np.array(["88", 2, 4]) + ms.Tensor(inp) + +def test_tensor_input_ndarray_bool(): + inp = np.array([True, 2, 4]) + ms.Tensor(inp) + + inp = np.array([False, 2, 4]) + ms.Tensor(inp) + +def test_tensor_input_ndarray_complex(): + with pytest.raises(TypeError): + inp = np.array([20j, 2, 4]) + ms.Tensor(inp) + +def test_tensor_input_ndarray_none(): + with pytest.raises(TypeError): + inp = np.array([None, 2, 4]) + ms.Tensor(inp) + +def test_tensor_input_ndarray_dict(): + with pytest.raises(TypeError): + inp = {'a': 6, 'b': 7} + ms.Tensor(inp) + +def test_tensor_input_np_nan(): + with pytest.raises(TypeError): + input_data = (1, 2, 3, np.nan) + ms.Tensor(input_data, np.int64) + +def test_tensor_input_tuple_inf(): + with pytest.raises(TypeError): + input_data = (1, 2, 3, float("inf")) + ms.Tensor(input_data, np.int64) + +def test_tensor_input_dict(): + with pytest.raises(TypeError): + input_data = {'a': 6, 'b': 7} + ms.Tensor(input_data, np.int64) + +def test_tensor_input_complex(): + with pytest.raises(TypeError): + input_data = (1, 2j, 3) + ms.Tensor(input_data, np.int64) + +def test_tensor_dtype_np_float(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.float) + ms.Tensor(input_data, np.float) + +def test_tensor_dtype_np_float16(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.float16) + ms.Tensor(input_data, np.float16) + +def test_tensor_dtype_np_float32(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.float32) + ms.Tensor(input_data, np.float32) + +def test_tensor_dtype_np_float64(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.float64) + ms.Tensor(input_data, np.float64) + +def test_tensor_dtype_np_int(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.int) + ms.Tensor(input_data, np.int) + +def test_tensor_dtype_np_int8(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.int8) + ms.Tensor(input_data, np.int8) + +def test_tensor_dtype_np_int16(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.int16) + ms.Tensor(input_data, np.int16) + +def test_tensor_dtype_np_int32(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.int32) + ms.Tensor(input_data, np.int32) + +def test_tensor_dtype_np_int64(): + with pytest.raises(TypeError): + input_data = np.random.randn(32, 112, 112, 3).astype(np.int64) + ms.Tensor(input_data, np.int64) + +def test_tensor_dtype_fp32_to_bool(): + with pytest.raises(RuntimeError): + input = np.random.randn(2, 3, 4, 5).astype(np.float32) + input = ms.Tensor(input) + input_me = ms.Tensor(input, dtype=ms.bool_) diff --git a/tests/ut/python/ir/test_tensor_py.py b/tests/ut/python/ir/test_tensor_py.py new file mode 100644 index 00000000000..9954326027a --- /dev/null +++ b/tests/ut/python/ir/test_tensor_py.py @@ -0,0 +1,145 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test tensor py""" +import numpy as np +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.common.api import _executor +import mindspore as ms +from ..ut_filter import non_graph_engine + + +def _attribute(tensor, shape_, size_, dtype_): + result = (tensor.shape == shape_) and \ + (tensor.size() == size_) and \ + (tensor.dtype == dtype_) + return result + + +def test_tensor_init(): + nparray = np.ones([2, 2], np.float32) + ms.Tensor(nparray) + + ms.Tensor(nparray, dtype=ms.float32) + + +@non_graph_engine +def test_tensor_add(): + a = ms.Tensor(np.ones([3, 3], np.float32)) + b = ms.Tensor(np.ones([3, 3], np.float32)) + a += b + + +@non_graph_engine +def test_tensor_sub(): + a = ms.Tensor(np.ones([2, 3])) + b = ms.Tensor(np.ones([2, 3])) + b -= a + + +@non_graph_engine +def test_tensor_mul(): + a = ms.Tensor(np.ones([3, 3])) + b = ms.Tensor(np.ones([3, 3])) + a *= b + + +def test_tensor_dim(): + arr = np.ones((1, 6)) + b = ms.Tensor(arr) + assert b.dim() == 2 + + +def test_tensor_size(): + arr = np.ones((1, 6)) + b = ms.Tensor(arr) + assert arr.size == b.size() + + +def test_dtype(): + a = ms.Tensor(np.ones((2, 3), dtype=np.int32)) + assert a.dtype() == ms.int32 + + +def test_asnumpy(): + npd = np.ones((2, 3)) + a = ms.Tensor(npd) + a.set_dtype(ms.int32) + assert a.asnumpy().all() == npd.all() + + +def test_print(): + a = ms.Tensor(np.ones((2, 3))) + a.set_dtype(ms.int32) + print(a) + + +def test_float(): + a = ms.Tensor(np.ones((2, 3)), ms.float16) + assert a.dtype() == ms.float16 + + +def test_tensor_method_sub(): + """test_tensor_method_sub""" + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.sub = P.Sub() + + def construct(self, x, y): + out = x - y + return out.transpose() + + net = Net() + + x = ms.Tensor(np.ones([5, 3], np.float32)) + y = ms.Tensor(np.ones([8, 5, 3], np.float32)) + _executor.compile(net, x, y) + + +def test_tensor_method_mul(): + """test_tensor_method_mul""" + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.sub = P.Sub() + + def construct(self, x, y): + out = x * (-y) + return out.transpose() + + net = Net() + + x = ms.Tensor(np.ones([5, 3], np.float32)) + y = ms.Tensor(np.ones([8, 5, 3], np.float32)) + _executor.compile(net, x, y) + + +def test_tensor_method_div(): + """test_tensor_method_div""" + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.sub = P.Sub() + + def construct(self, x, y): + out = x / y + return out.transpose() + + net = Net() + + x = ms.Tensor(np.ones([5, 3], np.float32)) + y = ms.Tensor(np.ones([8, 5, 3], np.float32)) + _executor.compile(net, x, y) diff --git a/tests/ut/python/keep_order/test_keep_order.py b/tests/ut/python/keep_order/test_keep_order.py new file mode 100644 index 00000000000..45f21198643 --- /dev/null +++ b/tests/ut/python/keep_order/test_keep_order.py @@ -0,0 +1,141 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.common.api import ms_function +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +import mindspore.ops.functional as F +import mindspore.context as context +from mindspore.ops import composite as C +from mindspore.ops.composite import core +from mindspore.common import dtype as mstype +import mindspore.nn as nn + +context.set_context(mode=context.GRAPH_MODE) +add1 = P.TensorAdd() +mul1 = P.MatMul() +add2 = P.TensorAdd() + +def add(x, y): + return add1(x, y) + +class Func(nn.Cell): + def __init__(self): + super(Func, self).__init__() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_status = P.NPUClearFloatStatus() + + def construct(self, x, y): + init = self.alloc_status() + sum = add(x, y) + product = mul1(x, y) + flag = self.get_status(init) + out = add2(sum, product) + clear = self.clear_status(flag) + out = F.depend(out, clear) + return out + +grad_s = C.GradOperation('grad_with_sens', get_all=True, sens_param=True) +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.func = Func() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_status = P.NPUClearFloatStatus() + + def construct(self, x, y, sens): + init = self.alloc_status() + sum1 = add(x, y) + dx = grad_s(self.func)(x, y, sens) + flag = self.get_status(init) + sum2 = add2(sum1, dx[0]) + sum3 = add2(y, dx[1]) + out = add2(sum2, sum3) + clear = self.clear_status(flag) + out = F.depend(out, clear) + return out + +def test_add(): + x = Tensor(np.ones([3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3]).astype(np.float32)) + func = Func() + func.add_flags(has_effect=True) + func(x, y) + +def test_sens(): + x = Tensor(np.ones([3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3]).astype(np.float32)) + sens = Tensor(np.ones([3, 3]).astype(np.float32)) + net = Net() + net.add_flags(has_effect=True) + out = net(x, y, sens) + +class Net_hyper(nn.Cell): + def __init__(self): + super(Net_hyper, self).__init__() + self.func = Func() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_status = P.NPUClearFloatStatus() + + def construct(self, x, y, sens): + init = self.alloc_status() + add1 = add(x, y) + sum1 = C.hyper_add([add1, add1], [x, y]) + dx = grad_s(self.func)(x, y, sens) + flag = self.get_status(init) + sum2 = add2(sum1[0], dx[0]) + sum3 = add2(sum1[1], dx[1]) + out = C.hyper_add([sum2, sum2], [sum3, sum3]) + clear = self.clear_status(flag) + out = F.depend(out, clear) + return out + +def test_hyper_add(): + x = Tensor(np.ones([3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3]).astype(np.float32)) + sens = Tensor(np.ones([3, 3]).astype(np.float32)) + net = Net_hyper() + net.add_flags(has_effect=True) + out = net(x, y, sens) + +def test_keep_order_io_effect_exception_return_dtype(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_status = P.NPUClearFloatStatus() + self.reduce_sum = P.ReduceSum(keep_dims=True) + self.dtype = P.DType() + self.sub = P.Sub() + self.neg = P.Neg() + self.add_flags(has_effect=True) + + def construct(self, x): + init = self.alloc_status() + self.clear_status(init) + res = self.sub(x, self.neg(x)) + self.get_status(init) + dtype = self.dtype(res) + return dtype + + value = 655 + data = np.full((8, 5, 3, 1), value, dtype=np.float16) + x = Tensor(data, dtype=mstype.float16) + net = Net() + data = net(x) \ No newline at end of file diff --git a/tests/ut/python/metrics/__init__.py b/tests/ut/python/metrics/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/metrics/test_accuracy.py b/tests/ut/python/metrics/test_accuracy.py new file mode 100644 index 00000000000..bedde410bf5 --- /dev/null +++ b/tests/ut/python/metrics/test_accuracy.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test accuracy""" +import math +import numpy as np +import pytest +from mindspore.nn.metrics import Accuracy +from mindspore import Tensor + +def test_classification_accuracy(): + """test_classification_accuracy""" + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + y2 = Tensor(np.array([[0, 1], [1, 0], [0, 1]])) + metric = Accuracy('classification') + metric.clear() + metric.update(x, y) + accuracy = metric.eval() + accuracy2 = metric(x, y2) + assert math.isclose(accuracy, 2/3) + assert math.isclose(accuracy2, 2/3) + +def test_multilabel_accuracy(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([[0, 1, 1, 1], [0, 1, 1, 1], [0, 0, 0, 1]])) + metric = Accuracy('multilabel') + metric.clear() + metric.update(x, y) + accuracy = metric.eval() + assert accuracy == 1/3 + +def test_shape_accuracy(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([[0, 1, 1, 1], [0, 1, 1, 1]])) + metric = Accuracy('multilabel') + metric.clear() + with pytest.raises(ValueError): + metric.update(x, y) + +def test_shape_accuracy2(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([0, 1, 1, 1])) + metric = Accuracy('multilabel') + metric.clear() + with pytest.raises(ValueError): + metric.update(x, y) + +def test_shape_accuracy3(): + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([[1, 0, 1], [1, 1, 1]])) + metric = Accuracy('classification') + metric.clear() + with pytest.raises(ValueError): + metric.update(x, y) + +def test_shape_accuracy4(): + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array(1)) + metric = Accuracy('classification') + metric.clear() + with pytest.raises(ValueError): + metric.update(x, y) + +def test_type_accuracy(): + with pytest.raises(TypeError): + Accuracy('test') diff --git a/tests/ut/python/metrics/test_error.py b/tests/ut/python/metrics/test_error.py new file mode 100644 index 00000000000..a50c458d18b --- /dev/null +++ b/tests/ut/python/metrics/test_error.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test error""" +import math +import numpy as np +import pytest +from mindspore.nn.metrics import MAE, MSE +from mindspore import Tensor + + +def test_MAE(): + x = Tensor(np.array([0.1, 0.2, 0.6, 0.9])) + y = Tensor(np.array([0.1, 0.25, 0.7, 0.9])) + error = MAE() + error.clear() + error.update(x, y) + result = error.eval() + assert math.isclose(result, 0.15/4) + + +def test_input_MAE(): + x = Tensor(np.array([0.1, 0.2, 0.6, 0.9])) + y = Tensor(np.array([0.1, 0.25, 0.7, 0.9])) + error = MAE() + error.clear() + with pytest.raises(ValueError): + error.update(x, y, x) + + +def test_zero_MAE(): + error = MAE() + with pytest.raises(RuntimeError): + error.eval() + + +def test_MSE(): + x = Tensor(np.array([0.1, 0.2, 0.6, 0.9])) + y = Tensor(np.array([0.1, 0.25, 0.5, 0.9])) + error = MSE() + error.clear() + error.update(x, y) + result = error.eval() + assert math.isclose(result, 0.0125/4) + + +def test_input_MSE(): + x = Tensor(np.array([0.1, 0.2, 0.6, 0.9])) + y = Tensor(np.array([0.1, 0.25, 0.7, 0.9])) + error = MSE() + error.clear() + with pytest.raises(ValueError): + error.update(x, y, x) + + +def test_zero_MSE(): + error = MSE() + with pytest.raises(RuntimeError): + error.eval() diff --git a/tests/ut/python/metrics/test_fbeta.py b/tests/ut/python/metrics/test_fbeta.py new file mode 100755 index 00000000000..1f4838a0b0e --- /dev/null +++ b/tests/ut/python/metrics/test_fbeta.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# """test_fbeta""" +import math +import numpy as np +import pytest +from mindspore.nn.metrics import get_metric_fn, Fbeta +from mindspore import Tensor + + +def test_classification_fbeta(): + """test_classification_fbeta""" + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + y2 = Tensor(np.array([[0, 1], [1, 0], [0, 1]])) + metric = get_metric_fn('F1') + metric.clear() + metric.update(x, y) + fbeta = metric.eval() + fbeta_mean = metric.eval(True) + fbeta2 = metric(x, y2) + + assert np.allclose(fbeta, np.array([2/3, 2/3])) + assert np.allclose(fbeta2, np.array([2/3, 2/3])) + assert np.allclose(fbeta_mean, 2/3) + + +def test_fbeta_update1(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + y = Tensor(np.array([1, 0])) + metric = Fbeta(2) + metric.clear() + + with pytest.raises(ValueError): + metric.update(x, y) + +def test_fbeta_update2(): + x1 = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + y1 = Tensor(np.array([1, 0, 2])) + x2 = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y2 = Tensor(np.array([1, 0, 2])) + metric = Fbeta(2) + metric.clear() + metric.update(x1, y1) + + with pytest.raises(ValueError): + metric.update(x2, y2) + + +def test_fbeta_init(): + with pytest.raises(ValueError): + Fbeta(0) + + +def test_fbeta_runtime(): + metric = Fbeta(2) + metric.clear() + + with pytest.raises(RuntimeError): + metric.eval() diff --git a/tests/ut/python/metrics/test_loss.py b/tests/ut/python/metrics/test_loss.py new file mode 100644 index 00000000000..bfdcffdbd6d --- /dev/null +++ b/tests/ut/python/metrics/test_loss.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test loss""" +import numpy as np +import pytest +from mindspore.nn.metrics import Loss +from mindspore import Tensor + + +def test_loss_inputs_error(): + loss = Loss() + with pytest.raises(ValueError): + loss(np.array(1), np.array(2)) + + +def test_loss_shape_error(): + loss = Loss() + inp = np.ones(shape=[2, 2]) + with pytest.raises(ValueError): + loss.update(inp) + + +def test_loss(): + """test_loss""" + num = 5 + inputs = np.random.rand(num) + + loss = Loss() + for k in range(num): + loss.update(Tensor(np.array([inputs[k]]))) + + assert inputs.mean() == loss.eval() + + loss.clear() + with pytest.raises(RuntimeError): + loss.eval() diff --git a/tests/ut/python/metrics/test_metric_factory.py b/tests/ut/python/metrics/test_metric_factory.py new file mode 100644 index 00000000000..cfbf4c77e3e --- /dev/null +++ b/tests/ut/python/metrics/test_metric_factory.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_metric_factory""" +import math +import numpy as np +from mindspore.nn.metrics import get_metric_fn +from mindspore import Tensor + + +def test_classification_accuracy(): + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + metric = get_metric_fn('accuracy', eval_type='classification') + metric.clear() + metric.update(x, y) + accuracy = metric.eval() + assert math.isclose(accuracy, 2/3) + + +def test_classification_accuracy_by_alias(): + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + metric = get_metric_fn('acc', eval_type='classification') + metric.clear() + metric.update(x, y) + accuracy = metric.eval() + assert math.isclose(accuracy, 2/3) + + +def test_classification_precision(): + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + metric = get_metric_fn('precision', eval_type='classification') + metric.clear() + metric.update(x, y) + precision = metric.eval() + + assert np.equal(precision, np.array([0.5, 1])).all() diff --git a/tests/ut/python/metrics/test_precision.py b/tests/ut/python/metrics/test_precision.py new file mode 100644 index 00000000000..37e5ab1286a --- /dev/null +++ b/tests/ut/python/metrics/test_precision.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_precision""" +import math +import numpy as np +import pytest +from mindspore.nn.metrics import Precision +from mindspore import Tensor + + +def test_classification_precision(): + """test_classification_precision""" + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + y2 = Tensor(np.array([[0, 1], [1, 0], [0, 1]])) + metric = Precision('classification') + metric.clear() + metric.update(x, y) + precision = metric.eval() + precision2 = metric(x, y2) + + assert np.equal(precision, np.array([0.5, 1])).all() + assert np.equal(precision2, np.array([0.5, 1])).all() + + +def test_multilabel_precision(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([[0, 1, 1, 1], [0, 1, 1, 1], [0, 0, 0, 1]])) + metric = Precision('multilabel') + metric.clear() + metric.update(x, y) + precision = metric.eval() + + assert np.equal(precision, np.array([1, 2/3, 1])).all() + + +def test_average_precision(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([[0, 1, 1, 1], [0, 1, 1, 1], [0, 0, 0, 1]])) + metric = Precision('multilabel') + metric.clear() + metric.update(x, y) + precision = metric.eval(True) + + assert math.isclose(precision, (1 + 2/3 + 1) / 3) + + +def test_num_precision(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + y = Tensor(np.array([1, 0])) + metric = Precision('classification') + metric.clear() + + with pytest.raises(ValueError): + metric.update(x, y) diff --git a/tests/ut/python/metrics/test_recall.py b/tests/ut/python/metrics/test_recall.py new file mode 100644 index 00000000000..dcf42c1f720 --- /dev/null +++ b/tests/ut/python/metrics/test_recall.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test recall""" +import math +import numpy as np +import pytest +from mindspore.nn.metrics import Recall +from mindspore import Tensor + + +def test_classification_recall(): + """test_classification_recall""" + x = Tensor(np.array([[0.2, 0.5], [0.3, 0.1], [0.9, 0.6]])) + y = Tensor(np.array([1, 0, 1])) + y2 = Tensor(np.array([[0, 1], [1, 0], [0, 1]])) + metric = Recall('classification') + metric.clear() + metric.update(x, y) + recall = metric.eval() + recall2 = metric(x, y2) + + assert np.equal(recall, np.array([1, 0.5])).all() + assert np.equal(recall2, np.array([1, 0.5])).all() + + +def test_multilabel_recall(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([[0, 1, 1, 1], [0, 1, 1, 1], [0, 0, 0, 1]])) + metric = Recall('multilabel') + metric.clear() + metric.update(x, y) + recall = metric.eval() + + assert np.equal(recall, np.array([2/3, 2/3, 1])).all() + + +def test_average_recall(): + x = Tensor(np.array([[0, 1, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])) + y = Tensor(np.array([[0, 1, 1, 1], [0, 1, 1, 1], [0, 0, 0, 1]])) + metric = Recall('multilabel') + metric.clear() + metric.update(x, y) + recall = metric.eval(True) + + assert math.isclose(recall, (2/3 + 2/3 + 1) / 3) + + +def test_num_recall(): + x = Tensor(np.array([[0.2, 0.5, 0.7], [0.3, 0.1, 0.2], [0.9, 0.6, 0.5]])) + y = Tensor(np.array([1, 0])) + metric = Recall('classification') + metric.clear() + + with pytest.raises(ValueError): + metric.update(x, y) diff --git a/tests/ut/python/metrics/test_topk.py b/tests/ut/python/metrics/test_topk.py new file mode 100644 index 00000000000..022a80e8919 --- /dev/null +++ b/tests/ut/python/metrics/test_topk.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test topk""" +import math +import numpy as np +import pytest +from mindspore.nn.metrics import TopKCategoricalAccuracy, Top1CategoricalAccuracy, Top5CategoricalAccuracy +from mindspore import Tensor + + +def test_type_topk(): + with pytest.raises(TypeError): + TopKCategoricalAccuracy(2.1) + + +def test_value_topk(): + with pytest.raises(ValueError): + TopKCategoricalAccuracy(-1) + + +def test_input_topk(): + x = Tensor(np.array([[0.2, 0.5, 0.3, 0.6, 0.2], + [0.3, 0.1, 0.5, 0.1, 0.], + [0.9, 0.6, 0.2, 0.01, 0.3]])) + topk = TopKCategoricalAccuracy(3) + topk.clear() + with pytest.raises(ValueError): + topk.update(x) + + +def test_topk(): + """test_topk""" + x = Tensor(np.array([[0.2, 0.5, 0.3, 0.6, 0.2], + [0.1, 0.35, 0.5, 0.2, 0.], + [0.9, 0.6, 0.2, 0.01, 0.3]])) + y = Tensor(np.array([2, 0, 1])) + y2 = Tensor(np.array([[0, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0]])) + topk = TopKCategoricalAccuracy(3) + topk.clear() + topk.update(x, y) + result = topk.eval() + result2 = topk(x, y2) + assert math.isclose(result, 2/3) + assert math.isclose(result2, 2/3) + + +def test_zero_topk(): + topk = TopKCategoricalAccuracy(3) + topk.clear() + with pytest.raises(RuntimeError): + topk.eval() + + +def test_top1(): + """test_top1""" + x = Tensor(np.array([[0.2, 0.5, 0.2, 0.1, 0.], + [0.1, 0.35, 0.25, 0.2, 0.1], + [0.9, 0.1, 0, 0., 0]])) + y = Tensor(np.array([2, 0, 0])) + y2 = Tensor(np.array([[0, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [1, 0, 0, 0, 0]])) + topk = Top1CategoricalAccuracy() + topk.clear() + topk.update(x, y) + result = topk.eval() + result2 = topk(x, y2) + assert math.isclose(result, 1/3) + assert math.isclose(result2, 1/3) + + +def test_top5(): + """test_top5""" + x = Tensor(np.array([[0.15, 0.4, 0.1, 0.05, 0., 0.2, 0.1], + [0.1, 0.35, 0.25, 0.2, 0.1, 0., 0.], + [0., 0.5, 0.2, 0.1, 0.1, 0.1, 0.]])) + y = Tensor(np.array([2, 0, 0])) + y2 = Tensor(np.array([[0, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [1, 0, 0, 0, 0]])) + topk = Top5CategoricalAccuracy() + topk.clear() + topk.update(x, y) + result = topk.eval() + result2 = topk(x, y2) + assert math.isclose(result, 2/3) + assert math.isclose(result2, 2/3) diff --git a/tests/ut/python/mindrecord/skip_test_issue.py b/tests/ut/python/mindrecord/skip_test_issue.py new file mode 100644 index 00000000000..29b952fb450 --- /dev/null +++ b/tests/ut/python/mindrecord/skip_test_issue.py @@ -0,0 +1,653 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test issue""" +import os +import pytest +from mindspore.mindrecord import ShardHeader, SUCCESS, FAILED, ParamValueError +from mindspore.mindrecord import ShardWriter, ShardIndexGenerator, ShardReader +from mindspore.mindrecord import FileWriter, FileReader +from mindspore.mindrecord import MRMAddIndexError +from mindspore import log as logger +from utils import get_data, get_nlp_data, get_mkv_data + +FILES_NUM = 4 +CV_FILE_NAME = "./imagenet.mindrecord" +NLP_FILE_NAME = "./aclImdb.mindrecord" +MKV_FILE_NAME = "./vehPer.mindrecord" + +def test_cv_file_writer_default_shard_num(): + """test cv dataset writer when shard_num is default value.""" + writer = FileWriter(CV_FILE_NAME) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + reader = FileReader(CV_FILE_NAME) + for index, x in enumerate(reader.get_next()): + logger.info("#item{}: {}".format(index, x)) + reader.close() + + os.remove("{}".format(CV_FILE_NAME)) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_cv_file_writer_shard_num_10(): + """test cv dataset writer when shard_num equals 10.""" + shard_num = 10 + writer = FileWriter(CV_FILE_NAME, shard_num) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + reader = FileReader(CV_FILE_NAME + "0") + for index, x in enumerate(reader.get_next()): + logger.info("#item{}: {}".format(index, x)) + reader.close() + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(shard_num)] + for item in paths: + os.remove("{}".format(item)) + os.remove("{}.db".format(item)) + +def test_cv_file_writer_file_name_none(): + """test cv dataset writer when file_name is none.""" + with pytest.raises(Exception) as e: + FileWriter(None) + assert str(e.value) == "[ParamValueError]: error_code: 1347686402, " \ + "error_msg: Invalid parameter value." \ + " File path is not allowed None or empty!" + +def test_cv_file_writer_file_name_null(): + """test cv dataset writer when file_name is empty string.""" + with pytest.raises(Exception) as e: + FileWriter("") + assert str(e.value) == "[ParamValueError]: error_code: 1347686402, " \ + "error_msg: Invalid parameter value." \ + " File path is not allowed None or empty!" + +def test_cv_file_writer_shard_number_less_1(): + """test cv dataset writer when shard_num is less than 1.""" + with pytest.raises(Exception) as e: + FileWriter("test.mindrecord", 0) + assert "[ParamValueError]: error_code: 1347686402, " \ + "error_msg: Invalid parameter value. " \ + "Shard number should " in str(e.value) + +def test_cv_file_writer_shard_number_more_1000(): + """test cv dataset writer when shard_num is greater than 1000.""" + with pytest.raises(Exception) as e: + FileWriter("test.mindrecord", 1001) + assert "[ParamValueError]: error_code: 1347686402, " \ + "error_msg: Invalid parameter value. " \ + "Shard number should " in str(e.value) + +def test_add_empty_schema(): + """test schema add when schema is empty.""" + header = ShardHeader() + schema = {} + desc = "test_schema" + with pytest.raises(Exception) as e: + schema = header.build_schema(schema, ["data"], desc) + schema_id = header.add_schema(schema) # add schema + assert schema_id == -1 + assert str(e.value) == "[MRMBuildSchemaError]: error_code: 1347690609, " \ + "error_msg: Failed to build schema." + +def test_add_schema_without_desc(): + """test schema add without desc.""" + header = ShardHeader() + schema_json = {"id_001": {"type": "number"}, "name_002": {"type": "string"}, + "data_003": {"type": "string"}, + "label": {"type": "string"}, "key": {"type": "string"}} + schema = header.build_schema(schema_json, ["data"]) + schema_id = header.add_schema(schema) # add schema + assert schema_id == 0 + +def test_add_empty_index(): + """test index add when index fields is empty string.""" + schema_json = {"file_name": {"type": "string"}, "label": {"type": "number"}} + header = ShardHeader() + schema = header.build_schema(schema_json, ["data"], "img") # create schema + header.add_schema(schema) # add schema + with pytest.raises(Exception, match="incompatible"): + header.add_index_fields("") + +def test_file_writer_fail_add_index(): + """test file writer, read when failed on adding index.""" + data_raw = get_data("../data/mindrecord/testImageNetData/") + schema_json = {"file_name": {"type": "string"}, "label": {"type": "number"}} + header = ShardHeader() + schema = header.build_schema(schema_json, ["data"], "img") # create schema + schema_id = header.add_schema(schema) # add schema + with pytest.raises(TypeError, match="missing 1 "): + ret = header.add_index_fields() + assert ret == FAILED + + with pytest.raises(MRMAddIndexError): + index_fields = [] + ret = header.add_index_fields(index_fields) + assert ret == FAILED + + file_name = os.path.join(os.getcwd(), "test_001.mindrecord") # set output filename + writer = ShardWriter() # test_file_writer + ret = writer.open([file_name]) + assert ret == SUCCESS, 'failed on opening files.' + ret = writer.set_shard_header(header) # write header + assert ret == SUCCESS, 'failed on setting header.' + ret = writer.write_raw_cv_data({schema_id: data_raw}) + assert ret == SUCCESS, 'failed on writing raw data.' + ret = writer.commit() # commit data + assert ret == SUCCESS, "commit failed" + # ShardIndexGenerator + generator = ShardIndexGenerator(os.path.realpath(file_name)) + generator.build() + generator.write_to_db() + + reader = ShardReader() + ret = reader.open(file_name) + reader.launch() + index = 0 + _, blob_fields = reader.get_blob_fields() + iterator = reader.get_next() + while iterator: + for blob, raw in iterator: + raw[blob_fields[0]] = bytes(blob) + logger.info("#item{}: {}".format(index, raw)) + index += 1 + iterator = reader.get_next() + reader.finish() + reader.close() + + os.remove("{}".format(file_name)) + os.remove("{}.db".format(file_name)) + +def test_add_index_with_incorrect_field(): + """test index add with incorrect field(64).""" + header = ShardHeader() + mkv_schema_json = {"file_name": {"type": "string"}, + "id": {"type": "number"}, "prelabel": {"type": "string"}} + schema = header.build_schema(mkv_schema_json, ["data"], "mkv_schema") + header.add_schema(schema) + with pytest.raises(Exception, match="incompatible function arguments"): + header.add_index_fields([(-1, "id")]) + +def test_add_index_with_string_list(): + """test index add with list of string(64).""" + header = ShardHeader() + schema_json = {"id": {"type": "number"}, "name": {"type": "string"}, + "label": {"type": "string"}, "key": {"type": "string"}} + schema = header.build_schema(schema_json, ["key"], "schema_desc") + header.add_schema(schema) + ret = header.add_index_fields(["id", "label"]) + assert ret == SUCCESS + +def test_add_index_with_dict(): + """test index add when index fields' datatype is dict(64).""" + writer = FileWriter(MKV_FILE_NAME, FILES_NUM) + mkv_schema_json = {"file_name": {"type": "string"}, + "id": {"type": "number"}, + "prelabel": {"type": "string"}, + "data": {"type": "bytes"}} + writer.add_schema(mkv_schema_json, "mkv_schema") + with pytest.raises(Exception) as e: + writer.add_index({"file_name": {"type": "string"}}) + assert str(e.value) == "[ParamTypeError]: error_code: 1347686401, " \ + "error_msg: Invalid parameter type. " \ + "'index_fields' expect list type." + +def test_mkv_file_reader_with_negative_num_consumer(): + """test mkv file reader when the number of consumer is negative.""" + writer = FileWriter(MKV_FILE_NAME, FILES_NUM) + data = get_mkv_data("../data/mindrecord/testVehPerData/") + mkv_schema_json = {"file_name": {"type": "string"}, + "id": {"type": "number"}, + "prelabel": {"type": "string"}, + "data": {"type": "bytes"}} + writer.add_schema(mkv_schema_json, "mkv_schema") + writer.add_index(["file_name", "prelabel"]) + writer.write_raw_data(data) + writer.commit() + + with pytest.raises(Exception) as e: + FileReader(MKV_FILE_NAME + "1", -1) + assert "Consumer number should between 1 and" in str(e.value) + + paths = ["{}{}".format(MKV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_write_raw_data_with_empty_list(): + """test write raw data with empty list.""" + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + ret = writer.write_raw_data([]) + assert ret == SUCCESS + writer.commit() + + reader = FileReader(CV_FILE_NAME + "0") + for index, x in enumerate(reader.get_next()): + logger.info("#item{}: {}".format(index, x)) + reader.close() + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_issue_38(): + """test cv dataset writer when schema does not match raw data.""" + writer = FileWriter(CV_FILE_NAME, 1) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}} + writer.add_schema(cv_schema_json, "img_schema") + with pytest.raises(Exception) as e: + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + assert str(e.value) == "[MRMDefineIndexError]: error_code: 1347694794, " \ + "error_msg: Failed to define index field. " \ + "Detail: Could not set blob field " \ + "'file_name' as index field." + +def test_issue_39(): + """test cv dataset writer when schema fields' datatype does not match raw data.""" + writer = FileWriter(CV_FILE_NAME, 1) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "number"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + reader = FileReader(CV_FILE_NAME) + index = 0 + for _ in reader.get_next(): + index += 1 + assert index == 0, "failed on reading data!" + reader.close() + os.remove("{}".format(CV_FILE_NAME)) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_issue_40(): + """test cv dataset when write raw data twice.""" + writer = FileWriter(CV_FILE_NAME, 1) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.write_raw_data(data) + ret = writer.commit() + assert ret == SUCCESS, 'failed on writing data!' + + os.remove("{}".format(CV_FILE_NAME)) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_issue_73(): + """test file reader by column name.""" + writer = FileWriter(MKV_FILE_NAME, FILES_NUM) + data = get_mkv_data("../data/mindrecord/testVehPerData/") + mkv_schema_json = {"file_name": {"type": "string"}, + "id": {"type": "number"}, + "prelabel": {"type": "string"}, + "data": {"type": "bytes"}} + writer.add_schema(mkv_schema_json, "mkv_schema") + writer.add_index(["file_name", "prelabel"]) + writer.write_raw_data(data) + writer.commit() + + reader = FileReader(MKV_FILE_NAME + "1", 4, ["file_name"]) + for index, x in enumerate(reader.get_next()): + logger.info("#item{}: {}".format(index, x)) + reader.close() + + paths = ["{}{}".format(MKV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_issue_117(): + """test add schema when field type is incorrect.""" + writer = FileWriter(__file__, FILES_NUM) + schema = {"id": {"type": "string"}, "label": {"type": "number"}, + "rating": {"type": "number"}, + "input_ids": {"type": "list", "items": {"type": "number"}}, + "input_mask": {"type": "array", "items": {"type": "number"}}, + "segment_ids": {"type": "array", "items": {"type": "number"}} + } + with pytest.raises(Exception, + match="Field '{'type': 'list', " + "'items': {'type': 'number'}}' " + "contains illegal attributes"): + writer.add_schema(schema, "img_schema") + +def test_issue_95(): + """test file reader when failed on file write.""" + writer = FileWriter(__file__, FILES_NUM) + data_raw = get_data("../data/mindrecord/testImageNetData/") + schema_json = {"file_name": {"type": "number"}, + "label": {"type": "number"}, + "data": {"type": "bytes"}, "data1": {"type": "string"}} + writer.add_schema(schema_json, "img_schema") + with pytest.raises(MRMAddIndexError): + writer.add_index(["key"]) + writer.write_raw_data(data_raw, True) + writer.commit() + + reader = FileReader(__file__ + "1") + for index, x in enumerate(reader.get_next()): + logger.info("#item{}: {}".format(index, x)) + reader.close() + + paths = ["{}{}".format(__file__, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_mindrecord_add_index_016(): + """test index add when index fields are incorrect.""" + schema_json = {"id": {"type": "number"}, "name": {"type": "string"}, + "label": {"type": "string"}, "key": {"type": "string"}} + header = ShardHeader() + schema = header.build_schema(schema_json, ["data"], "img") + header.add_schema(schema) + index_fields_list = [(0, "id")] + with pytest.raises(Exception): + header.add_index_fields(index_fields_list) + +def test_mindrecord_add_index_011(): + """test index add""" + schema_json = {"id": {"type": "number"}, "name": {"type": "string"}, + "label": {"type": "string"}, "key": {"type": "string"}} + header = ShardHeader() + schema = header.build_schema(schema_json, ["data"], "img") # create schema + header.add_schema(schema) # add schema + index_fields_list = ["id", "name", "label", "key"] + ret = header.add_index_fields(index_fields_list) + assert ret == 0, 'failed on adding index fields.' + +def test_issue_118(): + """test file writer when raw data do not match schema.""" + shard_num = 4 + writer = FileWriter(CV_FILE_NAME, shard_num) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + data.append({"file_name": "abcdefg", "label": 11, + "data": str(data[0]["data"])}) + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + reader = FileReader(CV_FILE_NAME + "0") + for index, _ in enumerate(reader.get_next()): + logger.info(index) + reader.close() + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(shard_num)] + for item in paths: + os.remove("{}".format(item)) + os.remove("{}.db".format(item)) + +def test_issue_87(): + """test file writer when data(bytes) do not match field type(string).""" + shard_num = 4 + writer = FileWriter(CV_FILE_NAME, shard_num) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "string"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["label"]) + with pytest.raises(Exception, match="data is wrong"): + writer.write_raw_data(data, False) + writer.commit() + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(shard_num)] + for item in paths: + os.remove("{}".format(item)) + +def test_issue_84(): + """test file reader when db does not match.""" + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + writer = FileWriter(NLP_FILE_NAME, FILES_NUM) + data = list(get_nlp_data("../data/mindrecord/testAclImdbData/pos", + "../data/mindrecord/testAclImdbData/vocab.txt", + 10)) + nlp_schema_json = {"id": {"type": "string"}, "label": {"type": "number"}, + "rating": {"type": "number"}, + "input_ids": {"type": "array", + "items": {"type": "number"}}, + "input_mask": {"type": "array", + "items": {"type": "number"}}, + "segment_ids": {"type": "array", + "items": {"type": "number"}} + } + writer.set_header_size(1<<14) + writer.set_page_size(1<<15) + writer.add_schema(nlp_schema_json, "nlp_schema") + writer.add_index(["id", "rating"]) + writer.write_raw_data(data) + writer.commit() + + reader = ShardReader() + os.rename("imagenet.mindrecord1.db", "imagenet.mindrecord1.db.bk") + os.rename("aclImdb.mindrecord1.db", "imagenet.mindrecord1.db") + file_name = os.path.join(os.getcwd(), "imagenet.mindrecord1") + with pytest.raises(Exception) as e: + reader.open(file_name) + assert str(e.value) == "[MRMOpenError]: error_code: 1347690596, " \ + "error_msg: " \ + "MindRecord File could not open successfully." + + os.rename("imagenet.mindrecord1.db", "aclImdb.mindrecord1.db") + paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for item in paths: + os.remove("{}".format(item)) + os.remove("{}.db".format(item)) + + os.rename("imagenet.mindrecord1.db.bk", "imagenet.mindrecord1.db") + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for item in paths: + os.remove("{}".format(item)) + os.remove("{}.db".format(item)) + +def test_issue_65(): + """test file reader when file name is illegal.""" + reader = ShardReader() + file_name = os.path.join(os.getcwd(), "imagenet.mindrecord01qwert") + with pytest.raises(Exception) as e: + reader.open(file_name) + assert str(e.value) == "[MRMOpenError]: error_code: 1347690596, " \ + "error_msg: " \ + "MindRecord File could not open successfully." + +def skip_test_issue_155(): + """test file writer loop.""" + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name"]) + for _ in range(1000): + writer.write_raw_data(data) + writer.commit() + reader = FileReader(CV_FILE_NAME + "0") + count = 0 + for _ in reader.get_next(): + count += 1 + assert count == 10000, "Failed to read mutiple writed data." + +def test_issue_124(): + """test file writer when data(string) do not match field type(bytes).""" + shard_num = 4 + writer = FileWriter(CV_FILE_NAME, shard_num) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + data.append({"file_name": "abcdefg", "label": 11, + "data": str(data[0]["data"])}) + writer.add_schema(cv_schema_json, "img_schema") + writer.write_raw_data(data) + writer.commit() + + reader = FileReader(CV_FILE_NAME + "0") + for index, _ in enumerate(reader.get_next()): + logger.info(index) + reader.close() + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(shard_num)] + for item in paths: + os.remove("{}".format(item)) + os.remove("{}.db".format(item)) + +def test_issue_36(): + """test file writer when shard num is illegal.""" + with pytest.raises(ParamValueError, match="Shard number should between "): + writer = FileWriter(CV_FILE_NAME, -1) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + with pytest.raises(UnboundLocalError, + match="local variable " + "'writer' referenced before assignment"): + writer.add_schema(cv_schema_json, "cv_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + +def test_issue_34(): + """test file writer""" + writer = FileWriter(CV_FILE_NAME) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "cv_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + reader = FileReader(CV_FILE_NAME) + i = 0 + for index, x in enumerate(reader.get_next()): + logger.info("#item{}: {}".format(index, x)) + i = i+1 + logger.info("count: {}".format(i)) + reader.close() + os.remove(CV_FILE_NAME) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_file_writer_raw_data_038(): + """test write raw data without verify.""" + shard_num = 11 + writer = FileWriter("test_file_writer_raw_data_", shard_num) + data_raw = get_data("../data/mindrecord/testImageNetData/") + schema_json = {"file_name": {"type": "string"}, "label": {"type": "number"}, + "data": {"type": "bytes"}} + writer.add_schema(schema_json, "img_schema") + writer.add_index(["file_name"]) + for _ in range(shard_num): + writer.write_raw_data(data_raw, False) + writer.commit() + + file_name = "" + if shard_num > 1: + file_name = '99' if shard_num > 99 else str(shard_num - 1) + reader = FileReader("test_file_writer_raw_data_" + file_name) + i = 0 + for _, _ in enumerate(reader.get_next()): + i = i + 1 + assert i == shard_num * 10 + reader.close() + if shard_num == 1: + os.remove("test_file_writer_raw_data_") + os.remove("test_file_writer_raw_data_.db") + return + for x in range(shard_num): + n = str(x) + if shard_num > 10: + n = '0' + str(x) if x < 10 else str(x) + if os.path.exists("test_file_writer_raw_data_{}".format(n)): + os.remove("test_file_writer_raw_data_{}".format(n)) + if os.path.exists("test_file_writer_raw_data_{}.db".format(n)): + os.remove("test_file_writer_raw_data_{}.db".format(n)) + +def test_more_than_1_bytes_in_schema(): + """test file writer when schema contains multiple 'bytes' fields.""" + schema_json = {"id": {"type": "string"}, "label": {"type": "number"}, + "rating": {"type": "number"}, + "input_ids": {"type": "bytes"}, + "input_mask": {"type": "bytes"}, + "segment_ids": {"type": "array", + "items": {"type": "number"}} + } + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + writer.add_schema(schema_json, "img_schema") + +def test_shard_4_raw_data_1(): + """test file writer when shard_num equals 4 and number of sample equals 1.""" + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}} + writer.add_schema(schema_json, "img_schema") + writer.add_index(["label"]) + data = [{"file_name": "001.jpg", "label": 1}] + writer.write_raw_data(data) + writer.commit() + + reader = FileReader(CV_FILE_NAME + "0") + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 2 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 1 + reader.close() + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) diff --git a/tests/ut/python/mindrecord/skip_test_mindrecord_internal.py b/tests/ut/python/mindrecord/skip_test_mindrecord_internal.py new file mode 100644 index 00000000000..48c0cbc0a0c --- /dev/null +++ b/tests/ut/python/mindrecord/skip_test_mindrecord_internal.py @@ -0,0 +1,26 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test mindrecord internal func +""" +from multiprocessing import cpu_count + +from mindspore.mindrecord import MAX_CONSUMER_COUNT + +def test_c_layer_thread_num_with_python_layer(): + assert cpu_count() == MAX_CONSUMER_COUNT() + +if __name__ == "__main__": + test_c_layer_thread_num_with_python_layer() \ No newline at end of file diff --git a/tests/ut/python/mindrecord/skip_test_mindrecord_shard.py b/tests/ut/python/mindrecord/skip_test_mindrecord_shard.py new file mode 100644 index 00000000000..b673e906359 --- /dev/null +++ b/tests/ut/python/mindrecord/skip_test_mindrecord_shard.py @@ -0,0 +1,338 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test internal shard api""" +import os +import random +from mindspore.mindrecord import ShardHeader, SUCCESS +from mindspore.mindrecord import ShardWriter, ShardIndexGenerator, ShardReader, ShardSegment +from mindspore import log as logger +from utils import get_data, get_nlp_data, get_mkv_data + +FILES_NUM = 4 +CV_FILE_NAME = "./imagenet.mindrecord" +NLP_FILE_NAME = "./aclImdb.mindrecord" +MKV_FILE_NAME = "./vehPer.mindrecord" + +def test_nlp_file_writer(): + """test nlp file writer using shard api""" + schema_json = {"id": {"type": "string"}, "label": {"type": "number"}, + "rating": {"type": "number"}, + "input_ids": {"type": "array", + "items": {"type": "number"}}, + "input_mask": {"type": "array", + "items": {"type": "number"}}, + "segment_ids": {"type": "array", + "items": {"type": "number"}} + } + data = list(get_nlp_data("../data/mindrecord/testAclImdbData/pos", + "../data/mindrecord/testAclImdbData/vocab.txt", + 10)) + header = ShardHeader() + schema = header.build_schema(schema_json, ["segment_ids"], "nlp_schema") + schema_id = header.add_schema(schema) + assert schema_id == 0, 'failed on adding schema' + index_fields_list = ["id", "rating"] + ret = header.add_index_fields(index_fields_list) + assert ret == SUCCESS, 'failed on adding index fields.' + writer = ShardWriter() + paths = ["{}{}".format(NLP_FILE_NAME, x) for x in range(FILES_NUM)] + ret = writer.open(paths) + assert ret == SUCCESS, 'failed on opening files.' + writer.set_header_size(1 << 14) + writer.set_page_size(1 << 15) + ret = writer.set_shard_header(header) + assert ret == SUCCESS, 'failed on setting header.' + ret = writer.write_raw_nlp_data({schema_id: data}) + assert ret == SUCCESS, 'failed on writing raw data.' + ret = writer.commit() + assert ret == SUCCESS, 'failed on committing.' + generator = ShardIndexGenerator(os.path.realpath(paths[0])) + generator.build() + generator.write_to_db() + +def test_nlp_file_reader(): + """test nlp file reader using shard api""" + dataset = ShardReader() + dataset.open(NLP_FILE_NAME + "0") + dataset.launch() + index = 0 + iterator = dataset.get_next() + while iterator: + for _, raw in iterator: + logger.info("#item{}: {}".format(index, raw)) + index += 1 + iterator = dataset.get_next() + dataset.finish() + dataset.close() + +def test_nlp_page_reader(): + """test nlp page reader using shard api""" + reader = ShardSegment() + reader.open(NLP_FILE_NAME + "0") + + fields = reader.get_category_fields() + logger.info("fields: {}".format(fields)) + + ret = reader.set_category_field("rating") + assert ret == SUCCESS, 'failed on setting category field.' + + info = reader.read_category_info() + logger.info("category info: {}".format(info)) + + img1 = reader.read_at_page_by_id(0, 0, 1) + logger.info("img1 len: {}, img1[0] len: {}, img1[0]: {}".format(len(img1), len(img1[0]), img1[0])) + + img2 = reader.read_at_page_by_name("7", 0, 1) + logger.info("img2 len: {}, img2[0] len: {}, img2[0]: {}".format(len(img2), len(img2[0]), img2[0])) + + paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_file_writer(): + """test cv file reader using shard api""" + img_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "number"}} + data = get_data("../data/mindrecord/testImageNetData/") + + header = ShardHeader() + img_schema = header.build_schema(img_schema_json, ["data"], "img_schema") + schema_id = header.add_schema(img_schema) + assert schema_id == 0, 'failed on building schema.' + index_fields_list = ["file_name", "label"] + ret = header.add_index_fields(index_fields_list) + assert ret == SUCCESS, 'failed on adding index fields.' + + writer = ShardWriter() + paths = ["{}{}".format(CV_FILE_NAME, x) for x in range(FILES_NUM)] + ret = writer.open(paths) + assert ret == SUCCESS, 'failed on opening files.' + writer.set_header_size(1 << 24) + writer.set_page_size(1 << 25) + ret = writer.set_shard_header(header) + assert ret == SUCCESS, 'failed on setting header.' + ret = writer.write_raw_cv_data({schema_id: data}) + assert ret == SUCCESS, 'failed on writing raw data.' + ret = writer.commit() + assert ret == SUCCESS, 'failed on committing.' + # ShardIndexGenerator + generator = ShardIndexGenerator(os.path.abspath(paths[0])) + generator.build() + generator.write_to_db() + +def test_cv_file_reader(): + """test cv file reader using shard api""" + dataset = ShardReader() + dataset.open(CV_FILE_NAME + "0") + dataset.launch() + index = 0 + _, blob_fields = dataset.get_blob_fields() + iterator = dataset.get_next() + while iterator: + for blob, raw in iterator: + raw[blob_fields[0]] = bytes(blob) + logger.info("#item{}: {}".format(index, raw)) + index += 1 + iterator = dataset.get_next() + dataset.finish() + dataset.close() + +def test_cv_page_reader(): + """test cv page reader using shard api""" + reader = ShardSegment() + reader.open(CV_FILE_NAME + "0") + fields = reader.get_category_fields() + logger.info("fields: {}".format(fields)) + + ret = reader.set_category_field("label") + assert ret == SUCCESS, 'failed on setting category field.' + + info = reader.read_category_info() + logger.info("category info: {}".format(info)) + + img1 = reader.read_at_page_by_id(0, 0, 1) + logger.info("img1 len: {}, img1[0] len: {}".format(len(img1), len(img1[0]))) + + img2 = reader.read_at_page_by_name("822", 0, 1) + logger.info("img2 len: {}, img2[0] len: {}".format(len(img2), len(img2[0]))) + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_mkv_file_writer(): + """test mkv file writer using shard api""" + data = get_mkv_data("../data/mindrecord/testVehPerData/") + schema_json = {"file_name": {"type": "string"}, "id": {"type": "number"}, + "prelabel": {"type": "string"}} + header = ShardHeader() + img_schema = header.build_schema(schema_json, ["data"], "img_schema") + schema_id = header.add_schema(img_schema) + assert schema_id == 0, 'failed on building schema.' + index_fields_list = ["id", "file_name"] + ret = header.add_index_fields(index_fields_list) + assert ret == SUCCESS, 'failed on adding index fields.' + + writer = ShardWriter() + paths = ["{}{}".format(MKV_FILE_NAME, x) for x in range(FILES_NUM)] + ret = writer.open(paths) + assert ret == SUCCESS, 'failed on opening files.' + writer.set_header_size(1 << 24) + writer.set_page_size(1 << 25) + ret = writer.set_shard_header(header) + assert ret == SUCCESS, 'failed on setting header.' + ret = writer.write_raw_cv_data({schema_id: data}) + assert ret == SUCCESS, 'failed on writing raw data.' + ret = writer.commit() + assert ret == SUCCESS, 'failed on committing.' + + generator = ShardIndexGenerator(os.path.realpath(paths[0])) + generator.build() + generator.write_to_db() + +def test_mkv_page_reader(): + """test mkv page reader using shard api""" + reader = ShardSegment() + reader.open(MKV_FILE_NAME + "0") + + fields = reader.get_category_fields() + logger.info("fields: {}".format(fields)) + + ret = reader.set_category_field("id") + assert ret == SUCCESS, 'failed on setting category field.' + + info = reader.read_category_info() + logger.info("category info: {}".format(info)) + + img1 = reader.read_at_page_by_id(0, 0, 1) + logger.info("img1 len: {}, img1[0] len: {}, img1[0]: {}".format(len(img1), len(img1[0]), img1[0])) + + img2 = reader.read_at_page_by_name("2", 0, 1) + logger.info("img2 len: {}, img2[0] len: {}, img2[0]: {}".format(len(img2), len(img2[0]), img2[0])) + +def test_mkv_page_reader_random(): + """test mkv page random reader using shard api""" + reader = ShardSegment() + reader.open(MKV_FILE_NAME + "0") + + fields = reader.get_category_fields() + logger.info("fields: {}".format(fields)) + + ret = reader.set_category_field("id") + assert ret == SUCCESS, 'failed on setting category field.' + + names = random.sample(range(1, 6), 5) + for name in names: + img2 = reader.read_at_page_by_name(str(name), 0, 2) + logger.info("name: {}, img2[0] len: {}".format(str(name), len(img2[0]))) + + paths = ["{}{}".format(MKV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_mkv_file_writer_with_exactly_schema(): + """test mkv file writer using shard api""" + header = ShardHeader() + img_schema_json = {"annotation_name": {"type": "array", + "items": {"type": "string"}}, + "annotation_pose": {"type": "array", + "items": {"type": "string"}}, + "annotation_truncated": {"type": "array", + "items": {"type": "string"}}, + "annotation_difficult": {"type": "array", + "items": {"type": "string"}}, + "annotation_xmin": {"type": "array", + "items": {"type": "number"}}, + "annotation_ymin": {"type": "array", + "items": {"type": "number"}}, + "annotation_xmax": {"type": "array", + "items": {"type": "number"}}, + "annotation_ymax": {"type": "array", + "items": {"type": "number"}}, + "metadata_width": {"type": "number"}, + "metadata_height": {"type": "number"}, + "metadata_depth": {"type": "number"}, + "img_path": {"type": "string"}, + "annotation_path": {"type": "string"}} + img_schema = header.build_schema(img_schema_json, ["data"], "image_schema") + schema_id = header.add_schema(img_schema) + assert schema_id == 0, 'failed on building schema.' + + writer = ShardWriter() + paths = ["{}{}".format(MKV_FILE_NAME, x) for x in range(1)] + ret = writer.open(paths) + assert ret == SUCCESS, 'failed on opening files.' + writer.set_header_size(1 << 24) + writer.set_page_size(1 << 25) + + image_bytes = bytes("it's a image picutre", encoding="utf8") + data = [] + data.append({"annotation_name": ["xxxxxxxxxx.jpg"], + "annotation_pose": ["hahahahah"], + "annotation_truncated": ["1"], "annotation_difficult": ["0"], + "annotation_xmin": [100], "annotation_ymin": [200], + "annotation_xmax": [300], "annotation_ymax": [400], + "metadata_width": 333, "metadata_height": 222, + "metadata_depth": 3, + "img_path": "/tmp/", "annotation_path": "/tmp/annotation", + "data": image_bytes}) + data.append({"annotation_name": ["xxxxxxxxxx.jpg"], + "annotation_pose": ["hahahahah"], + "annotation_truncated": ["1"], "annotation_difficult": ["0"], + "annotation_xmin": [100], "annotation_ymin": [200], + "annotation_xmax": [300], "annotation_ymax": [400], + "metadata_width": 333, "metadata_height": 222, + "metadata_depth": 3, + "img_path": "/tmp/", "annotation_path": "/tmp/annotation", + "data": image_bytes}) + ret = writer.set_shard_header(header) + assert ret == SUCCESS, 'failed on setting header.' + ret = writer.write_raw_cv_data({schema_id: data}) + assert ret == SUCCESS, 'failed on writing raw data.' + ret = writer.commit() + assert ret == SUCCESS, 'failed on committing.' + + generator = ShardIndexGenerator(os.path.realpath(paths[0])) + generator.build() + generator.write_to_db() + +def test_mkv_file_reader_with_exactly_schema(): + """test mkv file reader using shard api""" + dataset = ShardReader() + dataset.open(MKV_FILE_NAME + "0") + dataset.launch() + index = 0 + _, blob_fields = dataset.get_blob_fields() + iterator = dataset.get_next() + while iterator: + for blob, raw in iterator: + raw[blob_fields[0]] = bytes(blob) + logger.info("#item{}: {}".format(index, raw)) + index += 1 + iterator = dataset.get_next() + dataset.finish() + dataset.close() + + paths = ["{}{}".format(MKV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(1)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) diff --git a/tests/ut/python/mindrecord/test_cifar100_to_mindrecord.py b/tests/ut/python/mindrecord/test_cifar100_to_mindrecord.py new file mode 100644 index 00000000000..b3a8d94589d --- /dev/null +++ b/tests/ut/python/mindrecord/test_cifar100_to_mindrecord.py @@ -0,0 +1,131 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test cifar100 to mindrecord tool""" +import os +import pytest +from mindspore.mindrecord import Cifar100ToMR +from mindspore.mindrecord import FileReader +from mindspore.mindrecord import MRMOpenError +from mindspore import log as logger + +CIFAR100_DIR = "../data/mindrecord/testCifar100Data" +MINDRECORD_FILE = "./cifar100.mindrecord" + +def test_cifar100_to_mindrecord_without_index_fields(): + """test transform cifar100 dataset to mindrecord without index fields.""" + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, MINDRECORD_FILE) + cifar100_transformer.transform() + assert os.path.exists(MINDRECORD_FILE) + assert os.path.exists(MINDRECORD_FILE + "_test") + read() + os.remove("{}".format(MINDRECORD_FILE)) + os.remove("{}.db".format(MINDRECORD_FILE)) + + os.remove("{}".format(MINDRECORD_FILE + "_test")) + os.remove("{}.db".format(MINDRECORD_FILE + "_test")) + +def test_cifar100_to_mindrecord(): + """test transform cifar100 dataset to mindrecord.""" + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, MINDRECORD_FILE) + cifar100_transformer.transform(['fine_label', 'coarse_label']) + assert os.path.exists(MINDRECORD_FILE) + assert os.path.exists(MINDRECORD_FILE + "_test") + read() + os.remove("{}".format(MINDRECORD_FILE)) + os.remove("{}.db".format(MINDRECORD_FILE)) + + os.remove("{}".format(MINDRECORD_FILE + "_test")) + os.remove("{}.db".format(MINDRECORD_FILE + "_test")) + +def read(): + """test file reader""" + count = 0 + reader = FileReader(MINDRECORD_FILE) + for _, x in enumerate(reader.get_next()): + assert len(x) == 4 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 16 + reader.close() + + count = 0 + reader = FileReader(MINDRECORD_FILE + "_test") + for _, x in enumerate(reader.get_next()): + assert len(x) == 4 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 4 + reader.close() + +def test_cifar100_to_mindrecord_illegal_file_name(): + """ + test transform cifar100 dataset to mindrecord + when file name contains illegal character. + """ + filename = "./:no_ok" + with pytest.raises(Exception, match="File name should not contains"): + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, filename) + cifar100_transformer.transform() + +def test_cifar100_to_mindrecord_filename_start_with_space(): + """ + test transform cifar10 dataset to mindrecord + when file name starts with space. + """ + filename = "./ no_ok" + with pytest.raises(Exception, + match="File name should not start/end with space"): + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, filename) + cifar100_transformer.transform() + +def test_cifar100_to_mindrecord_filename_contain_space(): + """ + test transform cifar10 dataset to mindrecord + when file name contains space. + """ + filename = "./yes ok" + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, filename) + cifar100_transformer.transform() + assert os.path.exists(filename) + assert os.path.exists(filename + "_test") + os.remove("{}".format(filename)) + os.remove("{}.db".format(filename)) + + os.remove("{}".format(filename + "_test")) + os.remove("{}.db".format(filename + "_test")) + +def test_cifar100_to_mindrecord_directory(): + """ + test transform cifar10 dataset to mindrecord + when destination path is directory. + """ + with pytest.raises(MRMOpenError, + match="MindRecord File could not open successfully"): + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, + CIFAR100_DIR) + cifar100_transformer.transform() + +def test_cifar100_to_mindrecord_filename_equals_cifar100(): + """ + test transform cifar10 dataset to mindrecord + when destination path equals source path. + """ + with pytest.raises(MRMOpenError, + match="MindRecord File could not open successfully"): + cifar100_transformer = Cifar100ToMR(CIFAR100_DIR, + CIFAR100_DIR + "/train") + cifar100_transformer.transform() diff --git a/tests/ut/python/mindrecord/test_cifar10_to_mindrecord.py b/tests/ut/python/mindrecord/test_cifar10_to_mindrecord.py new file mode 100644 index 00000000000..bff06e4e723 --- /dev/null +++ b/tests/ut/python/mindrecord/test_cifar10_to_mindrecord.py @@ -0,0 +1,144 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test cifar10 to mindrecord tool""" +import os +import pytest +from mindspore.mindrecord import Cifar10ToMR +from mindspore.mindrecord import FileReader +from mindspore.mindrecord import MRMOpenError, SUCCESS +from mindspore import log as logger + +CIFAR10_DIR = "../data/mindrecord/testCifar10Data" +MINDRECORD_FILE = "./cifar10.mindrecord" + +def test_cifar10_to_mindrecord_without_index_fields(): + """test transform cifar10 dataset to mindrecord without index fields.""" + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, MINDRECORD_FILE) + cifar10_transformer.transform() + assert os.path.exists(MINDRECORD_FILE) + assert os.path.exists(MINDRECORD_FILE + "_test") + read() + os.remove("{}".format(MINDRECORD_FILE)) + os.remove("{}.db".format(MINDRECORD_FILE)) + + os.remove("{}".format(MINDRECORD_FILE + "_test")) + os.remove("{}.db".format(MINDRECORD_FILE + "_test")) + +def test_cifar10_to_mindrecord(): + """test transform cifar10 dataset to mindrecord.""" + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, MINDRECORD_FILE) + cifar10_transformer.transform(['label']) + assert os.path.exists(MINDRECORD_FILE) + assert os.path.exists(MINDRECORD_FILE + "_test") + read() + os.remove("{}".format(MINDRECORD_FILE)) + os.remove("{}.db".format(MINDRECORD_FILE)) + + os.remove("{}".format(MINDRECORD_FILE + "_test")) + os.remove("{}.db".format(MINDRECORD_FILE + "_test")) + +def test_cifar10_to_mindrecord_with_return(): + """test transform cifar10 dataset to mindrecord.""" + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, MINDRECORD_FILE) + ret = cifar10_transformer.transform(['label']) + assert ret == SUCCESS, "commit failed" + assert os.path.exists(MINDRECORD_FILE) + assert os.path.exists(MINDRECORD_FILE + "_test") + read() + os.remove("{}".format(MINDRECORD_FILE)) + os.remove("{}.db".format(MINDRECORD_FILE)) + + os.remove("{}".format(MINDRECORD_FILE + "_test")) + os.remove("{}.db".format(MINDRECORD_FILE + "_test")) + +def read(): + """test file reader""" + count = 0 + reader = FileReader(MINDRECORD_FILE) + for _, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 16 + reader.close() + + count = 0 + reader = FileReader(MINDRECORD_FILE + "_test") + for _, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 4 + reader.close() + +def test_cifar10_to_mindrecord_illegal_file_name(): + """ + test transform cifar10 dataset to mindrecord + when file name contains illegal character. + """ + filename = "./:no_ok" + with pytest.raises(Exception, match="File name should not contains"): + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, filename) + cifar10_transformer.transform() + +def test_cifar10_to_mindrecord_filename_start_with_space(): + """ + test transform cifar10 dataset to mindrecord + when file name starts with space. + """ + filename = "./ no_ok" + with pytest.raises(Exception, + match="File name should not start/end with space"): + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, filename) + cifar10_transformer.transform() + +def test_cifar10_to_mindrecord_filename_contain_space(): + """ + test transform cifar10 dataset to mindrecord + when file name contains space. + """ + filename = "./yes ok" + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, filename) + cifar10_transformer.transform() + assert os.path.exists(filename) + assert os.path.exists(filename + "_test") + os.remove("{}".format(filename)) + os.remove("{}.db".format(filename)) + + os.remove("{}".format(filename + "_test")) + os.remove("{}.db".format(filename + "_test")) + +def test_cifar10_to_mindrecord_directory(): + """ + test transform cifar10 dataset to mindrecord + when destination path is directory. + """ + with pytest.raises(MRMOpenError, + match="MindRecord File could not open successfully"): + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, CIFAR10_DIR) + cifar10_transformer.transform() + +def test_cifar10_to_mindrecord_filename_equals_cifar10(): + """ + test transform cifar10 dataset to mindrecord + when destination path equals source path. + """ + with pytest.raises(MRMOpenError, + match="MindRecord File could not open successfully"): + cifar10_transformer = Cifar10ToMR(CIFAR10_DIR, + CIFAR10_DIR + "/data_batch_0") + cifar10_transformer.transform() diff --git a/tests/ut/python/mindrecord/test_imagenet_to_mindrecord.py b/tests/ut/python/mindrecord/test_imagenet_to_mindrecord.py new file mode 100644 index 00000000000..5634f3a0a7f --- /dev/null +++ b/tests/ut/python/mindrecord/test_imagenet_to_mindrecord.py @@ -0,0 +1,99 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test imagenet to mindrecord tool""" +import os +import pytest +from mindspore.mindrecord import ImageNetToMR +from mindspore.mindrecord import FileReader +from mindspore import log as logger + +IMAGENET_MAP_FILE = "../data/mindrecord/testImageNetDataWhole/labels_map.txt" +IMAGENET_IMAGE_DIR = "../data/mindrecord/testImageNetDataWhole/images" +MINDRECORD_FILE = "../data/mindrecord/testImageNetDataWhole/imagenet.mindrecord" +PARTITION_NUMBER = 4 + +def read(filename): + """test file reade""" + count = 0 + reader = FileReader(filename) + for _, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 20 + reader.close() + +def test_imagenet_to_mindrecord(): + """test transform imagenet dataset to mindrecord.""" + imagenet_transformer = ImageNetToMR(IMAGENET_MAP_FILE, IMAGENET_IMAGE_DIR, + MINDRECORD_FILE, PARTITION_NUMBER) + imagenet_transformer.transform() + for i in range(PARTITION_NUMBER): + assert os.path.exists(MINDRECORD_FILE + str(i)) + assert os.path.exists(MINDRECORD_FILE + str(i) + ".db") + read(MINDRECORD_FILE + "0") + for i in range(PARTITION_NUMBER): + os.remove(MINDRECORD_FILE + str(i)) + os.remove(MINDRECORD_FILE + str(i) + ".db") + +def test_imagenet_to_mindrecord_default_partition_number(): + """ + test transform imagenet dataset to mindrecord + when partition number is default. + """ + imagenet_transformer = ImageNetToMR(IMAGENET_MAP_FILE, IMAGENET_IMAGE_DIR, + MINDRECORD_FILE) + imagenet_transformer.transform() + assert os.path.exists(MINDRECORD_FILE) + assert os.path.exists(MINDRECORD_FILE + ".db") + read(MINDRECORD_FILE) + os.remove("{}".format(MINDRECORD_FILE)) + os.remove("{}.db".format(MINDRECORD_FILE)) + +def test_imagenet_to_mindrecord_partition_number_0(): + """ + test transform imagenet dataset to mindrecord + when partition number is 0. + """ + with pytest.raises(Exception, match="Invalid parameter value"): + imagenet_transformer = ImageNetToMR(IMAGENET_MAP_FILE, + IMAGENET_IMAGE_DIR, + MINDRECORD_FILE, 0) + imagenet_transformer.transform() + +def test_imagenet_to_mindrecord_partition_number_none(): + """ + test transform imagenet dataset to mindrecord + when partition number is none. + """ + with pytest.raises(Exception, + match="The parameter partition_number must be int"): + imagenet_transformer = ImageNetToMR(IMAGENET_MAP_FILE, + IMAGENET_IMAGE_DIR, + MINDRECORD_FILE, None) + imagenet_transformer.transform() + +def test_imagenet_to_mindrecord_illegal_filename(): + """ + test transform imagenet dataset to mindrecord + when file name contains illegal character. + """ + filename = "not_*ok" + with pytest.raises(Exception, match="File name should not contains"): + imagenet_transformer = ImageNetToMR(IMAGENET_MAP_FILE, + IMAGENET_IMAGE_DIR, filename, + PARTITION_NUMBER) + imagenet_transformer.transform() diff --git a/tests/ut/python/mindrecord/test_mindrecord_base.py b/tests/ut/python/mindrecord/test_mindrecord_base.py new file mode 100644 index 00000000000..576063295ae --- /dev/null +++ b/tests/ut/python/mindrecord/test_mindrecord_base.py @@ -0,0 +1,305 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test mindrecord base""" +import os +import uuid +from mindspore.mindrecord import FileWriter, FileReader, MindPage, SUCCESS +from mindspore import log as logger +from utils import get_data, get_nlp_data + +FILES_NUM = 4 +CV_FILE_NAME = "./imagenet.mindrecord" +CV2_FILE_NAME = "./imagenet_loop.mindrecord" +CV3_FILE_NAME = "./imagenet_append.mindrecord" +NLP_FILE_NAME = "./aclImdb.mindrecord" + +def test_cv_file_writer_tutorial(): + """tutorial for cv dataset writer.""" + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + +def test_cv_file_append_writer(): + """tutorial for cv dataset append writer.""" + writer = FileWriter(CV3_FILE_NAME, 4) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data[0:5]) + writer.commit() + write_append = FileWriter.open_for_append(CV3_FILE_NAME + "0") + write_append.write_raw_data(data[5:10]) + write_append.commit() + reader = FileReader(CV3_FILE_NAME + "0") + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + + paths = ["{}{}".format(CV3_FILE_NAME, str(x).rjust(1, '0')) + for x in range(4)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_file_writer_loop_and_read(): + """tutorial for cv dataset loop writer.""" + writer = FileWriter(CV2_FILE_NAME, FILES_NUM) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + for row in data: + writer.write_raw_data([row]) + writer.commit() + + reader = FileReader(CV2_FILE_NAME + "0") + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + + paths = ["{}{}".format(CV2_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_file_reader_tutorial(): + """tutorial for cv file reader.""" + reader = FileReader(CV_FILE_NAME + "0") + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + +def test_cv_file_reader_partial_tutorial(): + """tutorial for cv file partial reader.""" + reader = FileReader(CV_FILE_NAME + "0") + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + if count == 5: + reader.finish() + assert count == 5 + +def test_cv_page_reader_tutorial(): + """tutorial for cv page reader.""" + reader = MindPage(CV_FILE_NAME + "0") + fields = reader.get_category_fields() + assert fields == ['file_name', 'label'],\ + 'failed on getting candidate category fields.' + + ret = reader.set_category_field("label") + assert ret == SUCCESS, 'failed on setting category field.' + + info = reader.read_category_info() + logger.info("category info: {}".format(info)) + + row = reader.read_at_page_by_id(0, 0, 1) + assert len(row) == 1 + assert len(row[0]) == 3 + assert row[0]['label'] == 13 + + row1 = reader.read_at_page_by_name("822", 0, 1) + assert len(row1) == 1 + assert len(row1[0]) == 3 + assert row1[0]['label'] == 822 + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_nlp_file_writer_tutorial(): + """tutorial for nlp file writer.""" + writer = FileWriter(NLP_FILE_NAME, FILES_NUM) + data = list(get_nlp_data("../data/mindrecord/testAclImdbData/pos", + "../data/mindrecord/testAclImdbData/vocab.txt", + 10)) + nlp_schema_json = {"id": {"type": "string"}, "label": {"type": "int32"}, + "rating": {"type": "float32"}, + "input_ids": {"type": "int64", + "shape": [1, -1]}, + "input_mask": {"type": "int64", + "shape": [1, -1]}, + "segment_ids": {"type": "int64", + "shape": [1, -1]} + } + writer.add_schema(nlp_schema_json, "nlp_schema") + writer.add_index(["id", "rating"]) + writer.write_raw_data(data) + writer.commit() + +def test_nlp_file_reader_tutorial(): + """tutorial for nlp file reader.""" + reader = FileReader(NLP_FILE_NAME + "0") + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 6 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + +def test_nlp_page_reader_tutorial(): + """tutorial for nlp page reader.""" + reader = MindPage(NLP_FILE_NAME + "0") + fields = reader.get_category_fields() + assert fields == ['id', 'rating'],\ + 'failed on getting candidate category fields.' + + ret = reader.set_category_field("rating") + assert ret == SUCCESS, 'failed on setting category field.' + + info = reader.read_category_info() + logger.info("category info: {}".format(info)) + + row = reader.read_at_page_by_id(0, 0, 1) + assert len(row) == 1 + assert len(row[0]) == 6 + logger.info("row[0]: {}".format(row[0])) + + row1 = reader.read_at_page_by_name("7", 0, 1) + assert len(row1) == 1 + assert len(row1[0]) == 6 + logger.info("row1[0]: {}".format(row1[0])) + + paths = ["{}{}".format(NLP_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_file_writer_shard_num_1000(): + """test file writer when shard num equals 1000.""" + writer = FileWriter(CV_FILE_NAME, 1000) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(3, '0')) + for x in range(1000)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_file_writer_absolute_path(): + """test cv file writer when file name is absolute path.""" + file_name = "/tmp/" + str(uuid.uuid4()) + writer = FileWriter(file_name, FILES_NUM) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + + paths = ["{}{}".format(file_name, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + os.remove("{}".format(x)) + os.remove("{}.db".format(x)) + +def test_cv_file_writer_without_data(): + """test cv file writer without data.""" + writer = FileWriter(CV_FILE_NAME, 1) + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.commit() + reader = FileReader(CV_FILE_NAME) + count = 0 + for index, x in enumerate(reader.get_next()): + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 0 + reader.close() + os.remove(CV_FILE_NAME) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_cv_file_writer_no_blob(): + """test cv file writer without blob data.""" + writer = FileWriter(CV_FILE_NAME, 1) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}} + writer.add_schema(cv_schema_json, "no_blob_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + reader = FileReader(CV_FILE_NAME) + count = 0 + for index, x in enumerate(reader.get_next()): + count += 1 + assert len(x) == 2 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + os.remove(CV_FILE_NAME) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_cv_file_writer_no_raw(): + """test cv file writer without raw data.""" + writer = FileWriter(NLP_FILE_NAME) + data = list(get_nlp_data("../data/mindrecord/testAclImdbData/pos", + "../data/mindrecord/testAclImdbData/vocab.txt", + 10)) + nlp_schema_json = {"input_ids": {"type": "int64", + "shape": [1, -1]}, + "input_mask": {"type": "int64", + "shape": [1, -1]}, + "segment_ids": {"type": "int64", + "shape": [1, -1]} + } + writer.add_schema(nlp_schema_json, "no_raw_schema") + writer.write_raw_data(data) + writer.commit() + reader = FileReader(NLP_FILE_NAME) + count = 0 + for index, x in enumerate(reader.get_next()): + count += 1 + assert len(x) == 3 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + os.remove(NLP_FILE_NAME) + os.remove("{}.db".format(NLP_FILE_NAME)) diff --git a/tests/ut/python/mindrecord/test_mindrecord_exception.py b/tests/ut/python/mindrecord/test_mindrecord_exception.py new file mode 100644 index 00000000000..0a51fbf4e76 --- /dev/null +++ b/tests/ut/python/mindrecord/test_mindrecord_exception.py @@ -0,0 +1,282 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test mindrecord exception""" +import os +import pytest +from mindspore.mindrecord import FileWriter, FileReader, MindPage +from mindspore.mindrecord import MRMOpenError, MRMGenerateIndexError, ParamValueError +from mindspore import log as logger +from utils import get_data + +CV_FILE_NAME = "./imagenet.mindrecord" +NLP_FILE_NAME = "./aclImdb.mindrecord" +FILES_NUM = 4 + +def test_cv_file_writer_shard_num_none(): + """test cv file writer when shard num is None.""" + with pytest.raises(Exception, match="Shard num is illegal."): + FileWriter("/tmp/123454321", None) + +def test_cv_file_writer_shard_num_str(): + """test cv file writer when shard num is string.""" + with pytest.raises(Exception, match="Shard num is illegal."): + FileWriter("/tmp/123454321", "20") + +def test_cv_page_reader_consumer_num_none(): + """test cv page reader when consumer number is None.""" + with pytest.raises(Exception, match="Consumer number is illegal."): + MindPage(CV_FILE_NAME + "0", None) + +def test_cv_page_reader_consumer_num_str(): + """test cv page reader when consumer number is string.""" + with pytest.raises(Exception, match="Consumer number is illegal."): + MindPage(CV_FILE_NAME + "0", "2") + +def test_nlp_file_reader_consumer_num_none(): + """test nlp file reader when consumer number is None.""" + with pytest.raises(Exception, match="Consumer number is illegal."): + FileReader(NLP_FILE_NAME + "0", None) + +def test_nlp_file_reader_consumer_num_str(): + """test nlp file reader when consumer number is string.""" + with pytest.raises(Exception, match="Consumer number is illegal."): + FileReader(NLP_FILE_NAME + "0", "4") + +def create_cv_mindrecord(files_num): + writer = FileWriter(CV_FILE_NAME, files_num) + data = get_data("../data/mindrecord/testImageNetData/") + cv_schema_json = {"file_name": {"type": "string"}, + "label": {"type": "int64"}, "data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "img_schema") + writer.add_index(["file_name", "label"]) + writer.write_raw_data(data) + writer.commit() + +def test_lack_partition_and_db(): + """test file reader when mindrecord file does not exist.""" + with pytest.raises(MRMOpenError) as err: + reader = FileReader('dummy.mindrecord') + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + +def test_lack_db(): + """test file reader when db file does not exist.""" + create_cv_mindrecord(1) + os.remove("{}.db".format(CV_FILE_NAME)) + with pytest.raises(MRMOpenError) as err: + reader = FileReader(CV_FILE_NAME) + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + os.remove(CV_FILE_NAME) + +def test_lack_some_partition_and_db(): + """test file reader when some partition and db do not exist.""" + create_cv_mindrecord(4) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + os.remove("{}".format(paths[3])) + os.remove("{}.db".format(paths[3])) + with pytest.raises(MRMOpenError) as err: + reader = FileReader(CV_FILE_NAME + "0") + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + +def test_lack_some_partition_first(): + """test file reader when first partition does not exist.""" + create_cv_mindrecord(4) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + os.remove("{}".format(paths[0])) + with pytest.raises(MRMOpenError) as err: + reader = FileReader(CV_FILE_NAME + "0") + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + +def test_lack_some_partition_middle(): + """test file reader when some partition does not exist.""" + create_cv_mindrecord(4) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + os.remove("{}".format(paths[1])) + with pytest.raises(MRMOpenError) as err: + reader = FileReader(CV_FILE_NAME + "0") + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + +def test_lack_some_partition_last(): + """test file reader when last partition does not exist.""" + create_cv_mindrecord(4) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + os.remove("{}".format(paths[3])) + with pytest.raises(MRMOpenError) as err: + reader = FileReader(CV_FILE_NAME + "0") + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + +def test_mindpage_lack_some_partition(): + """test page reader when some partition does not exist.""" + create_cv_mindrecord(4) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + os.remove("{}".format(paths[0])) + with pytest.raises(MRMOpenError) as err: + MindPage(CV_FILE_NAME + "0") + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + +def test_lack_some_db(): + """test file reader when some db does not exist.""" + create_cv_mindrecord(4) + paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0')) + for x in range(FILES_NUM)] + os.remove("{}.db".format(paths[3])) + with pytest.raises(MRMOpenError) as err: + reader = FileReader(CV_FILE_NAME + "0") + reader.close() + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + for x in paths: + if os.path.exists("{}".format(x)): + os.remove("{}".format(x)) + if os.path.exists("{}.db".format(x)): + os.remove("{}.db".format(x)) + +def test_invalid_mindrecord(): + """test file reader when the content of mindrecord is illegal.""" + with open(CV_FILE_NAME, 'w') as f: + dummy = 's' * 100 + f.write(dummy) + with pytest.raises(MRMOpenError) as err: + FileReader(CV_FILE_NAME) + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + os.remove(CV_FILE_NAME) + +def test_invalid_db(): + """test file reader when the content of db is illegal.""" + create_cv_mindrecord(1) + os.remove("imagenet.mindrecord.db") + with open('imagenet.mindrecord.db', 'w') as f: + f.write('just for test') + with pytest.raises(MRMOpenError) as err: + FileReader('imagenet.mindrecord') + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + os.remove("imagenet.mindrecord") + os.remove("imagenet.mindrecord.db") + +def test_overwrite_invalid_mindrecord(): + """test file writer when overwrite invalid mindreocrd file.""" + with open(CV_FILE_NAME, 'w') as f: + f.write('just for test') + with pytest.raises(MRMOpenError) as err: + create_cv_mindrecord(1) + assert '[MRMOpenError]: error_code: 1347690596, ' \ + 'error_msg: MindRecord File could not open successfully.'\ + in str(err.value) + os.remove(CV_FILE_NAME) + +def test_overwrite_invalid_db(): + """test file writer when overwrite invalid db file.""" + with open('imagenet.mindrecord.db', 'w') as f: + f.write('just for test') + with pytest.raises(MRMGenerateIndexError) as err: + create_cv_mindrecord(1) + assert '[MRMGenerateIndexError]: error_code: 1347690612, ' \ + 'error_msg: Failed to generate index.' in str(err.value) + os.remove("imagenet.mindrecord") + os.remove("imagenet.mindrecord.db") + +def test_read_after_close(): + """test file reader when close read.""" + create_cv_mindrecord(1) + reader = FileReader(CV_FILE_NAME) + reader.close() + count = 0 + for index, x in enumerate(reader.get_next()): + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 0 + os.remove(CV_FILE_NAME) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_file_read_after_read(): + """test file reader when finish read.""" + create_cv_mindrecord(1) + reader = FileReader(CV_FILE_NAME) + count = 0 + for index, x in enumerate(reader.get_next()): + assert len(x) == 3 + count = count + 1 + logger.info("#item{}: {}".format(index, x)) + assert count == 10 + reader.close() + cnt = 0 + for index, x in enumerate(reader.get_next()): + cnt = cnt + 1 + logger.info("#item{}: {}".format(index, x)) + assert cnt == 0 + os.remove(CV_FILE_NAME) + os.remove("{}.db".format(CV_FILE_NAME)) + +def test_cv_file_writer_shard_num_greater_than_1000(): + """test cv file writer shard number greater than 1000.""" + with pytest.raises(ParamValueError) as err: + FileWriter(CV_FILE_NAME, 1001) + assert 'Shard number should between' in str(err.value) diff --git a/tests/ut/python/mindrecord/test_mindrecord_multi_images.py b/tests/ut/python/mindrecord/test_mindrecord_multi_images.py new file mode 100644 index 00000000000..b26e44c5dba --- /dev/null +++ b/tests/ut/python/mindrecord/test_mindrecord_multi_images.py @@ -0,0 +1,146 @@ +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""test write multiple images""" +import numpy as np +import os +from mindspore.mindrecord import FileReader, FileWriter +from mindspore import log as logger +from utils import get_two_bytes_data, get_multi_bytes_data + +MAP_FILE_NAME = "../data/mindrecord/testTwoImageData/cityscapes_train_19.txt" +MAP_FILE_FAKE_NAME = "../data/mindrecord/testTwoImageData/cityscapes_train_19_fake.txt" +DIFF_SHAPE_FILE_NAME = "../data/mindrecord/testImageNetData/cityscapes_train_19_fake.txt" +CV_FILE_NAME = "../data/mindrecord/testTwoImageData/two_bytes.mindrecord" +FILES_NUM = 1 + +def read(filename, fields_num=5): + count = 0 + reader = FileReader(filename) + for _, x in enumerate(reader.get_next()): + assert len(x) == fields_num + count = count + 1 + logger.info("data: {}".format(x)) + assert count == 5 + reader.close() + +def test_write_two_images_mindrecord(): + """test two images to mindrecord""" + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_two_bytes_data(MAP_FILE_NAME) + cv_schema_json = {"img_data": {"type": "bytes"}, "label_data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "two_images_schema") + writer.write_raw_data(data) + writer.commit() + assert os.path.exists(CV_FILE_NAME) + assert os.path.exists(CV_FILE_NAME + ".db") + read(CV_FILE_NAME, 2) + + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + +def test_write_two_images_mindrecord_whole_field(): + """test two images to mindrecord""" + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_two_bytes_data(MAP_FILE_NAME) + cv_schema_json={"id": {"type": "int32"}, "file_name": {"type": "string"}, + "label_name": {"type": "string"}, "img_data": {"type": "bytes"}, + "label_data": {"type": "bytes"}} + writer.add_schema(cv_schema_json, "two_images_schema") + writer.write_raw_data(data) + writer.commit() + assert os.path.exists(CV_FILE_NAME) + assert os.path.exists(CV_FILE_NAME + ".db") + read(CV_FILE_NAME, 5) + + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + +def test_write_two_diff_shape_images_mindrecord(): + """test two different shape images to mindrecord""" + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + bytes_num = 2 + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_multi_bytes_data(DIFF_SHAPE_FILE_NAME, bytes_num) + cv_schema_json = {"image_{}".format(i): {"type": "bytes"} + for i in range(bytes_num)} + writer.add_schema(cv_schema_json, "two_images_schema") + writer.write_raw_data(data) + writer.commit() + assert os.path.exists(CV_FILE_NAME) + assert os.path.exists(CV_FILE_NAME + ".db") + read(CV_FILE_NAME, bytes_num) + +def test_write_multi_images_mindrecord(): + """test multiple images to mindrecord""" + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + bytes_num = 10 + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_multi_bytes_data(MAP_FILE_FAKE_NAME, bytes_num) + cv_schema_json = {"image_{}".format(i): {"type": "bytes"} + for i in range(bytes_num)} + writer.add_schema(cv_schema_json, "multi_images_schema") + writer.write_raw_data(data) + writer.commit() + assert os.path.exists(CV_FILE_NAME) + assert os.path.exists(CV_FILE_NAME + ".db") + read(CV_FILE_NAME, bytes_num) + +def test_write_two_images_and_array_mindrecord(): + """test two image images and array to mindrecord""" + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) + + bytes_num = 2 + writer = FileWriter(CV_FILE_NAME, FILES_NUM) + data = get_multi_bytes_data(DIFF_SHAPE_FILE_NAME, bytes_num) + + for index, _ in enumerate(data): + data[index].update({"input_ids": np.array([12, 45, 95, 0, 5, 66])}) + + cv_schema_json = {"image_{}".format(i): {"type": "bytes"} + for i in range(bytes_num)} + cv_schema_json.update({"id": {"type": "int64"}, + "input_ids": {"type": "int64", + "shape": [-1]}}) + writer.add_schema(cv_schema_json, "two_images_schema") + writer.write_raw_data(data) + writer.commit() + assert os.path.exists(CV_FILE_NAME) + assert os.path.exists(CV_FILE_NAME + ".db") + read(CV_FILE_NAME, bytes_num+2) + + if os.path.exists("{}".format(CV_FILE_NAME + ".db")): + os.remove(CV_FILE_NAME + ".db") + if os.path.exists("{}".format(CV_FILE_NAME)): + os.remove(CV_FILE_NAME) diff --git a/tests/ut/python/mindrecord/test_mnist_to_mr.py b/tests/ut/python/mindrecord/test_mnist_to_mr.py new file mode 100644 index 00000000000..c299a1f7192 --- /dev/null +++ b/tests/ut/python/mindrecord/test_mnist_to_mr.py @@ -0,0 +1,141 @@ +# Copyright 2019 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""test mnist to mindrecord tool""" +import os +import gzip +import cv2 +import numpy as np + +from mindspore.mindrecord import MnistToMR +from mindspore.mindrecord import FileReader +from mindspore import log as logger + +MNIST_DIR = "../data/mindrecord/testMnistData" +FILE_NAME = "mnist" +PARTITION_NUM = 4 +IMAGE_SIZE = 28 +NUM_CHANNELS = 1 + + +def read(train_name, test_name): + """test file reader""" + count = 0 + reader = FileReader(train_name) + for _, x in enumerate(reader.get_next()): + assert len(x) == 2 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 60000 + reader.close() + + count = 0 + reader = FileReader(test_name) + for _, x in enumerate(reader.get_next()): + assert len(x) == 2 + count = count + 1 + if count == 1: + logger.info("data: {}".format(x)) + assert count == 10000 + reader.close() + + +def test_mnist_to_mindrecord(): + """test transform mnist dataset to mindrecord.""" + mnist_transformer = MnistToMR(MNIST_DIR, FILE_NAME) + mnist_transformer.transform() + assert os.path.exists("mnist_train.mindrecord") + assert os.path.exists("mnist_test.mindrecord") + + read("mnist_train.mindrecord", "mnist_test.mindrecord") + + os.remove("{}".format("mnist_train.mindrecord")) + os.remove("{}.db".format("mnist_train.mindrecord")) + os.remove("{}".format("mnist_test.mindrecord")) + os.remove("{}.db".format("mnist_test.mindrecord")) + + +def test_mnist_to_mindrecord_compare_data(): + """test transform mnist dataset to mindrecord and compare data.""" + mnist_transformer = MnistToMR(MNIST_DIR, FILE_NAME) + mnist_transformer.transform() + assert os.path.exists("mnist_train.mindrecord") + assert os.path.exists("mnist_test.mindrecord") + + train_name, test_name = "mnist_train.mindrecord", "mnist_test.mindrecord" + + def _extract_images(filename, num_images): + """Extract the images into a 4D tensor [image index, y, x, channels].""" + with gzip.open(filename) as bytestream: + bytestream.read(16) + buf = bytestream.read( + IMAGE_SIZE * IMAGE_SIZE * num_images * NUM_CHANNELS) + data = np.frombuffer(buf, dtype=np.uint8) + data = data.reshape( + num_images, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS) + return data + + def _extract_labels(filename, num_images): + """Extract the labels into a vector of int64 label IDs.""" + with gzip.open(filename) as bytestream: + bytestream.read(8) + buf = bytestream.read(1 * num_images) + labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64) + return labels + + train_data_filename_ = os.path.join(MNIST_DIR, + 'train-images-idx3-ubyte.gz') + train_labels_filename_ = os.path.join(MNIST_DIR, + 'train-labels-idx1-ubyte.gz') + test_data_filename_ = os.path.join(MNIST_DIR, + 't10k-images-idx3-ubyte.gz') + test_labels_filename_ = os.path.join(MNIST_DIR, + 't10k-labels-idx1-ubyte.gz') + train_data = _extract_images(train_data_filename_, 60000) + train_labels = _extract_labels(train_labels_filename_, 60000) + test_data = _extract_images(test_data_filename_, 10000) + test_labels = _extract_labels(test_labels_filename_, 10000) + + reader = FileReader(train_name) + for x, data, label in zip(reader.get_next(), train_data, train_labels): + _, img = cv2.imencode(".jpeg", data) + assert np.array(x['data']) == img.tobytes() + assert np.array(x['label']) == label + reader.close() + + reader = FileReader(test_name) + for x, data, label in zip(reader.get_next(), test_data, test_labels): + _, img = cv2.imencode(".jpeg", data) + assert np.array(x['data']) == img.tobytes() + assert np.array(x['label']) == label + reader.close() + + os.remove("{}".format("mnist_train.mindrecord")) + os.remove("{}.db".format("mnist_train.mindrecord")) + os.remove("{}".format("mnist_test.mindrecord")) + os.remove("{}.db".format("mnist_test.mindrecord")) + + +def test_mnist_to_mindrecord_multi_partition(): + """test transform mnist dataset to multiple mindrecord files.""" + mnist_transformer = MnistToMR(MNIST_DIR, FILE_NAME, PARTITION_NUM) + mnist_transformer.transform() + + read("mnist_train.mindrecord0", "mnist_test.mindrecord0") + + for i in range(PARTITION_NUM): + os.remove("{}".format("mnist_train.mindrecord" + str(i))) + os.remove("{}.db".format("mnist_train.mindrecord" + str(i))) + os.remove("{}".format("mnist_test.mindrecord" + str(i))) + os.remove("{}.db".format("mnist_test.mindrecord" + str(i))) diff --git a/tests/ut/python/mindrecord/utils.py b/tests/ut/python/mindrecord/utils.py new file mode 100644 index 00000000000..3e4cf7abdf7 --- /dev/null +++ b/tests/ut/python/mindrecord/utils.py @@ -0,0 +1,245 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""utils for test""" + +import collections +import json +import os +import re +import string +import numpy as np + +from mindspore import log as logger + +def get_data(dir_name): + """ + Return raw data of imagenet dataset. + + Args: + dir_name (str): String of imagenet dataset's path. + + Returns: + List + """ + if not os.path.isdir(dir_name): + raise IOError("Directory {} not exists".format(dir_name)) + img_dir = os.path.join(dir_name, "images") + ann_file = os.path.join(dir_name, "annotation.txt") + with open(ann_file, "r") as file_reader: + lines = file_reader.readlines() + + data_list = [] + for line in lines: + try: + filename, label = line.split(",") + label = label.strip("\n") + with open(os.path.join(img_dir, filename), "rb") as file_reader: + img = file_reader.read() + data_json = {"file_name": filename, + "data": img, + "label": int(label)} + data_list.append(data_json) + except FileNotFoundError: + continue + return data_list + +def get_two_bytes_data(file_name): + """ + Return raw data of two-bytes dataset. + + Args: + file_name (str): String of two-bytes dataset's path. + + Returns: + List + """ + if not os.path.exists(file_name): + raise IOError("map file {} not exists".format(file_name)) + dir_name = os.path.dirname(file_name) + with open(file_name, "r") as file_reader: + lines = file_reader.readlines() + data_list = [] + row_num = 0 + for line in lines: + try: + img, label = line.strip('\n').split(" ") + with open(os.path.join(dir_name, img), "rb") as file_reader: + img_data = file_reader.read() + with open(os.path.join(dir_name, label), "rb") as file_reader: + label_data = file_reader.read() + data_json = {"file_name": img, + "img_data": img_data, + "label_name": label, + "label_data": label_data, + "id": row_num + } + row_num += 1 + data_list.append(data_json) + except FileNotFoundError: + continue + return data_list + +def get_multi_bytes_data(file_name, bytes_num=3): + """ + Return raw data of multi-bytes dataset. + + Args: + file_name (str): String of multi-bytes dataset's path. + bytes_num (int): Number of bytes fields. + + Returns: + List + """ + if not os.path.exists(file_name): + raise IOError("map file {} not exists".format(file_name)) + dir_name = os.path.dirname(file_name) + with open(file_name, "r") as file_reader: + lines = file_reader.readlines() + data_list = [] + row_num = 0 + for line in lines: + try: + img10_path = line.strip('\n').split(" ") + img5 = [] + for path in img10_path[:bytes_num]: + with open(os.path.join(dir_name, path), "rb") as file_reader: + img5 += [file_reader.read()] + data_json = {"image_{}".format(i): img5[i] + for i in range(len(img5))} + data_json.update({"id": row_num}) + row_num += 1 + data_list.append(data_json) + except FileNotFoundError: + continue + return data_list + +def get_mkv_data(dir_name): + """ + Return raw data of Vehicle_and_Person dataset. + + Args: + dir_name (str): String of Vehicle_and_Person dataset's path. + + Returns: + List + """ + if not os.path.isdir(dir_name): + raise IOError("Directory {} not exists".format(dir_name)) + img_dir = os.path.join(dir_name, "Image") + label_dir = os.path.join(dir_name, "prelabel") + + data_list = [] + file_list = os.listdir(label_dir) + + index = 1 + for file in file_list: + if os.path.splitext(file)[1] == '.json': + file_path = os.path.join(label_dir, file) + + image_name = ''.join([os.path.splitext(file)[0], ".jpg"]) + image_path = os.path.join(img_dir, image_name) + + with open(file_path, "r") as load_f: + load_dict = json.load(load_f) + + if os.path.exists(image_path): + with open(image_path, "rb") as file_reader: + img = file_reader.read() + data_json = {"file_name": image_name, + "prelabel": str(load_dict), + "data": img, + "id": index} + data_list.append(data_json) + index += 1 + logger.info('{} images are missing'.format(len(file_list)-len(data_list))) + return data_list + + +def get_nlp_data(dir_name, vocab_file, num): + """ + Return raw data of aclImdb dataset. + + Args: + dir_name (str): String of aclImdb dataset's path. + vocab_file (str): String of dictionary's path. + num (int): Number of sample. + + Returns: + List + """ + if not os.path.isdir(dir_name): + raise IOError("Directory {} not exists".format(dir_name)) + for root, dirs, files in os.walk(dir_name): + for index, file_name_extension in enumerate(files): + if index < num: + file_path = os.path.join(root, file_name_extension) + file_name, _ = file_name_extension.split('.', 1) + id_, rating = file_name.split('_', 1) + with open(file_path, 'r') as f: + raw_content = f.read() + + dictionary = load_vocab(vocab_file) + vectors = [dictionary.get('[CLS]')] + vectors += [dictionary.get(i) if i in dictionary + else dictionary.get('[UNK]') + for i in re.findall(r"[\w']+|[{}]" + .format(string.punctuation), + raw_content)] + vectors += [dictionary.get('[SEP]')] + input_, mask, segment = inputs(vectors) + input_ids = np.reshape(np.array(input_), [1, -1]) + input_mask = np.reshape(np.array(mask), [1, -1]) + segment_ids = np.reshape(np.array(segment), [1, -1]) + data = { + "label": 1, + "id": id_, + "rating": float(rating), + "input_ids": input_ids, + "input_mask": input_mask, + "segment_ids": segment_ids + } + yield data + +def convert_to_uni(text): + if isinstance(text, str): + return text + if isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + raise Exception("The type %s does not convert!" % type(text)) + +def load_vocab(vocab_file): + """load vocabulary to translate statement.""" + vocab = collections.OrderedDict() + vocab.setdefault('blank', 2) + index = 0 + with open(vocab_file) as reader: + while True: + tmp = reader.readline() + if not tmp: + break + token = convert_to_uni(tmp) + token = token.strip() + vocab[token] = index + index += 1 + return vocab + +def inputs(vectors, maxlen=50): + length = len(vectors) + if length > maxlen: + return vectors[0:maxlen], [1]*maxlen, [0]*maxlen + input_ = vectors+[0]*(maxlen-length) + mask = [1]*length + [0]*(maxlen-length) + segment = [0]*maxlen + return input_, mask, segment diff --git a/tests/ut/python/model/__init__.py b/tests/ut/python/model/__init__.py new file mode 100644 index 00000000000..5443c0ca48e --- /dev/null +++ b/tests/ut/python/model/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup for pytest""" +import mindspore.context as context + +# pylint: disable=unused-argument +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) diff --git a/tests/ut/python/model/res18_example.py b/tests/ut/python/model/res18_example.py new file mode 100644 index 00000000000..eaf8bbc387d --- /dev/null +++ b/tests/ut/python/model/res18_example.py @@ -0,0 +1,262 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +resnet50 example +""" +import numpy as np + +from mindspore.common.api import _executor +from mindspore import Tensor +from mindspore.ops.operations import TensorAdd +import mindspore.nn as nn # pylint: disable=C0414 +from ...train_step_wrap import train_step_with_loss_warp + + +def conv3x3(in_channels, out_channels, stride=1, padding=1, pad_mode='pad'): + """3x3 convolution """ + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, pad_mode=pad_mode) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0, pad_mode='pad'): + """1x1 convolution""" + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode) + + +class ResidualBlock(nn.Cell): + """ + residual Block + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = nn.BatchNorm2d(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=1) + self.bn2 = nn.BatchNorm2d(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.relu = nn.ReLU() + self.downsample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, + stride=stride, padding=0) + self.bn_down_sample = nn.BatchNorm2d(out_channels) + self.add = TensorAdd() + + def construct(self, x): + """ + :param x: + :return: + """ + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet18(nn.Cell): + """ + resnet nn.Cell + """ + + def __init__(self, block, num_classes=100): + super(ResNet18, self).__init__() + + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, pad_mode='pad') + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, pad_mode='pad') + + self.layer1 = self.MakeLayer( + block, 2, in_channels=64, out_channels=256, stride=1) + self.layer2 = self.MakeLayer( + block, 2, in_channels=256, out_channels=512, stride=2) + self.layer3 = self.MakeLayer( + block, 2, in_channels=512, out_channels=1024, stride=2) + self.layer4 = self.MakeLayer( + block, 2, in_channels=1024, out_channels=2048, stride=2) + + self.avgpool = nn.AvgPool2d(7, 1) + self.flatten = nn.Flatten() + self.fc = nn.Dense(512 * block.expansion, num_classes) + + def MakeLayer(self, block, layer_num, in_channels, out_channels, stride): + """ + make block layer + :param block: + :param layer_num: + :param in_channels: + :param out_channels: + :param stride: + :return: + """ + layers = [] + resblk = block(in_channels, out_channels, + stride=stride, down_sample=True) + layers.append(resblk) + + for _ in range(1, layer_num): + resblk = block(out_channels, out_channels, stride=1) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + """ + :param x: + :return: + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.flatten(x) + x = self.fc(x) + + return x + + +class ResNet9(nn.Cell): + """ + resnet nn.Cell + """ + + def __init__(self, block, num_classes=100): + super(ResNet9, self).__init__() + + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.layer1 = self.MakeLayer( + block, 1, in_channels=64, out_channels=256, stride=1) + self.layer2 = self.MakeLayer( + block, 1, in_channels=256, out_channels=512, stride=2) + self.layer3 = self.MakeLayer( + block, 1, in_channels=512, out_channels=1024, stride=2) + self.layer4 = self.MakeLayer( + block, 1, in_channels=1024, out_channels=2048, stride=2) + + self.avgpool = nn.AvgPool2d(7, 1) + self.flatten = nn.Flatten() + self.fc = nn.Dense(512 * block.expansion, num_classes) + + def MakeLayer(self, block, layer_num, in_channels, out_channels, stride): + """ + make block layer + :param block: + :param layer_num: + :param in_channels: + :param out_channels: + :param stride: + :return: + """ + layers = [] + resblk = block(in_channels, out_channels, + stride=stride, down_sample=True) + layers.append(resblk) + + for _ in range(1, layer_num): + resblk = block(out_channels, out_channels, stride=1) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + """ + :param x: + :return: + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = self.flatten(x) + x = self.fc(x) + + return x + + +def resnet18(): + return ResNet18(ResidualBlock, 10) + + +def resnet9(): + return ResNet9(ResidualBlock, 10) + + +def test_compile(): + net = resnet18() + input_data = Tensor(np.ones([1, 3, 224, 224])) + _executor.compile(net, input_data) + + +def test_train_step(): + net = train_step_with_loss_warp(resnet9()) + input_data = Tensor(np.ones([1, 3, 224, 224])) + label = Tensor(np.zeros([1, 10])) + _executor.compile(net, input_data, label) + + +def test_train_step_training(): + net = train_step_with_loss_warp(resnet9()) + input_data = Tensor(np.ones([1, 3, 224, 224])) + label = Tensor(np.zeros([1, 10])) + net.set_train() + _executor.compile(net, input_data, label) diff --git a/tests/ut/python/model/test_bert.py b/tests/ut/python/model/test_bert.py new file mode 100644 index 00000000000..1964330c7c0 --- /dev/null +++ b/tests/ut/python/model/test_bert.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test bert cell """ +import numpy as np +import pytest +from mindspore import Model +from mindspore.nn.optim import AdamWeightDecay +from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertModel, BertNetworkWithLoss, BertTrainOneStepCell +from ....dataset_mock import MindData + + +def map_bert(record): + target_data = {'input_ids': None, 'input_mask': None, + 'segment_ids': None, 'next_sentence_labels': None, + 'masked_lm_positions': None, 'masked_lm_ids': None, + 'masked_lm_weights': None} + + sample = dt.parse_single_example(record, target_data) + + return sample['input_ids'], sample['input_mask'], sample['segment_ids'], \ + sample['next_sentence_labels'], sample['masked_lm_positions'], \ + sample['masked_lm_ids'], sample['masked_lm_weights'] + + +def test_bert_model(): + # test for config.hidden_size % config.num_attention_heads != 0 + config_error = BertConfig(32, hidden_size=512, num_attention_heads=10) + with pytest.raises(ValueError): + BertModel(config_error, True) + + +def get_dataset(batch_size=1): + dataset_types = (np.int32, np.int32, np.int32, np.int32, np.int32, np.int32, np.int32) + dataset_shapes = ((batch_size, 128), (batch_size, 128), (batch_size, 128), (batch_size, 1), + (batch_size, 20), (batch_size, 20), (batch_size, 20)) + + dataset = MindData(size=2, batch_size=batch_size, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset diff --git a/tests/ut/python/model/test_bert_cell.py b/tests/ut/python/model/test_bert_cell.py new file mode 100644 index 00000000000..fdaaac397b4 --- /dev/null +++ b/tests/ut/python/model/test_bert_cell.py @@ -0,0 +1,421 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test bert of graph compile """ +import functools +import numpy as np +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.model_zoo.Bert_NEZHA.bert_model import BertConfig, \ + EmbeddingLookup, EmbeddingPostprocessor, BertOutput, RelaPosMatrixGenerator, \ + RelaPosEmbeddingsGenerator, SaturateCast, BertAttention, BertSelfAttention, \ + BertEncoderCell, BertTransformer, CreateAttentionMaskFromInputMask, BertModel +from mindspore.nn.layer.basic import Norm +from mindspore.model_zoo.Bert_NEZHA import BertPretrainingLoss, GetNextSentenceOutput +import mindspore.nn as nn +from mindspore.common.initializer import TruncatedNormal +from mindspore.common.parameter import ParameterTuple +from mindspore.nn.optim import AdamWeightDecay, AdamWeightDecayDynamicLR +from mindspore.model_zoo.Bert_NEZHA.bert_for_pre_training import ClipGradients +import mindspore.ops.composite as C +from mindspore.ops import functional as F +from ....ops_common import convert +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.gradient.compile_gradient import pipeline_for_compile_grad_ge_graph_for_case_by_case_config + +def bert_trans(): + """bert_trans""" + net = BertTransformer(batch_size=1, + hidden_size=768, + seq_length=128, + num_hidden_layers=1, + num_attention_heads=12, + intermediate_size=768, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=False, + hidden_act="gelu", + compute_type=mstype.float32, + return_all_encoders=True) + net.set_train() + return net + +def set_train(net): + net.set_train() + return net + +class NetForAdam(nn.Cell): + def __init__(self): + super(NetForAdam, self).__init__() + self.dense = nn.Dense(64, 10) + + def construct(self, x): + x = self.dense(x) + return x + +class TrainStepWrapForAdam(nn.Cell): + """TrainStepWrapForAdam definition""" + def __init__(self, network): + super(TrainStepWrapForAdam, self).__init__() + self.network = network + self.weights = ParameterTuple(network.get_parameters()) + self.optimizer = AdamWeightDecay(self.weights) + self.clip_gradients = ClipGradients() + + def construct(self, x, sens): + weights = self.weights + grads = C.grad_by_list_with_sens(self.network, weights)(x, sens) + grads = self.clip_gradients(grads, 1, 1.0) + return self.optimizer(grads) + +class TrainStepWrapForAdamDynamicLr(nn.Cell): + """TrainStepWrapForAdamDynamicLr definition""" + def __init__(self, network): + super(TrainStepWrapForAdamDynamicLr, self).__init__() + self.network = network + self.weights = ParameterTuple(network.get_parameters()) + self.optimizer = AdamWeightDecayDynamicLR(self.weights, 10) + self.sens = Tensor(np.ones(shape=(1, 10)).astype(np.float32)) + + def construct(self, x): + weights = self.weights + grads = C.grad_by_list_with_sens(self.network, weights)(x, self.sens) + return self.optimizer(grads) + +class TempC2Wrap(nn.Cell): + def __init__(self, op, c1=None, c2=None,): + super(TempC2Wrap, self).__init__() + self.op = op + self.c1 = c1 + self.c2 = c2 + def construct(self, x1): + x = self.op(x1, self.c1, self.c2) + return x + +test_case_cell_ops = [ + ('Norm_keepdims', { + 'block': Norm(keep_dims=True), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1]]}), + ('SaturateCast', { + 'block': SaturateCast(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('RelaPosMatrixGenerator_0', { + 'block': RelaPosMatrixGenerator(length=128, max_relative_position=16), + 'desc_inputs': [], + 'desc_bprop': [[128, 128]], + 'skip': ['backward']}), + ('RelaPosEmbeddingsGenerator_0', { + 'block': RelaPosEmbeddingsGenerator(length=128, depth=512, + max_relative_position=16, + initializer_range=0.2), + 'desc_inputs': [], + 'desc_bprop': [[16384, 512]], + 'skip': ['backward']}), + ('RelaPosEmbeddingsGenerator_1', { + 'block': RelaPosEmbeddingsGenerator(length=128, depth=512, + max_relative_position=16, + initializer_range=0.2, + use_one_hot_embeddings=False), + 'desc_inputs': [], + 'desc_bprop': [[128, 128, 512]], + 'skip': ['backward']}), + ('RelaPosEmbeddingsGenerator_2', { + 'block': RelaPosEmbeddingsGenerator(length=128, depth=64, + max_relative_position=16, + initializer_range=0.2, + use_one_hot_embeddings=False), + 'desc_inputs': [], + 'desc_bprop': [[128, 128, 64]], + 'skip': ['backward']}), + ('BertAttention_0', { + 'block': BertAttention(batch_size=64, + from_tensor_width=768, + to_tensor_width=768, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=12, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=False, + compute_type=mstype.float32), + 'desc_inputs': [[64, 128, 768], [64, 128, 768], [64, 128, 128]], + 'desc_bprop': [[8192, 768]]}), + ('BertAttention_1', { + 'block': BertAttention(batch_size=64, + from_tensor_width=768, + to_tensor_width=768, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=12, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + compute_type=mstype.float32), + 'desc_inputs': [[64, 128, 768], [64, 128, 768], [64, 128, 128]], + 'desc_bprop': [[8192, 768]]}), + ('BertAttention_2', { + 'block': BertAttention(batch_size=64, + from_tensor_width=768, + to_tensor_width=768, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=12, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=False, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=True, + use_relative_positions=True, + compute_type=mstype.float32), + 'desc_inputs': [[64, 128, 768], [64, 128, 768], [64, 128, 128]], + 'desc_bprop': [[8192, 768]]}), + ('BertAttention_3', { + 'block': BertAttention(batch_size=64, + from_tensor_width=768, + to_tensor_width=768, + from_seq_length=128, + to_seq_length=128, + num_attention_heads=12, + size_per_head=64, + query_act=None, + key_act=None, + value_act=None, + has_attention_mask=True, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + do_return_2d_tensor=False, + use_relative_positions=True, + compute_type=mstype.float32), + 'desc_inputs': [[64, 128, 768], [64, 128, 768], [64, 128, 128]], + 'desc_bprop': [[8192, 768]]}), + ('BertOutput', { + 'block': BertOutput(in_channels=768, + out_channels=768, + initializer_range=0.02, + dropout_prob=0.1), + 'desc_inputs': [[8192, 768], [8192, 768]], + 'desc_bprop': [[8192, 768]]}), + ('BertSelfAttention_0', { + 'block': BertSelfAttention(batch_size=64, + seq_length=128, + hidden_size=768, + num_attention_heads=12, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.1, + use_relative_positions=False, + compute_type=mstype.float32), + 'desc_inputs': [[64, 128, 768], [64, 128, 128]], + 'desc_bprop': [[8192, 768]]}), + ('BertEncoderCell', { + 'block': BertEncoderCell(batch_size=64, + hidden_size=768, + seq_length=128, + num_attention_heads=12, + intermediate_size=768, + attention_probs_dropout_prob=0.02, + use_one_hot_embeddings=False, + initializer_range=0.02, + hidden_dropout_prob=0.1, + use_relative_positions=False, + hidden_act="gelu", + compute_type=mstype.float32), + 'desc_inputs': [[64, 128, 768], [64, 128, 128]], + 'desc_bprop': [[8192, 768]]}), + ('BertTransformer_0', { + 'block': BertTransformer(batch_size=1, + hidden_size=768, + seq_length=128, + num_hidden_layers=1, + num_attention_heads=12, + intermediate_size=768, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=False, + hidden_act="gelu", + compute_type=mstype.float32, + return_all_encoders=True), + 'desc_inputs': [[1, 128, 768], [1, 128, 128]]}), + ('BertTransformer_1', { + 'block': BertTransformer(batch_size=64, + hidden_size=768, + seq_length=128, + num_hidden_layers=2, + num_attention_heads=12, + intermediate_size=768, + attention_probs_dropout_prob=0.1, + use_one_hot_embeddings=False, + initializer_range=0.02, + use_relative_positions=True, + hidden_act="gelu", + compute_type=mstype.float32, + return_all_encoders=False), + 'desc_inputs': [[64, 128, 768], [64, 128, 128]]}), + ('EmbeddingLookup', { + 'block': EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False, + initializer_range=0.02), + 'desc_inputs': [Tensor(np.random.rand(128).astype(np.int32))], + 'desc_bprop': [[1, 128, 768], [1, 128, 768]], + 'num_output': 2}), + ('EmbeddingPostprocessor', { + 'block': EmbeddingPostprocessor(embedding_size=768, + embedding_shape=[1, 128, 768], + use_token_type=True, + token_type_vocab_size=16, + use_one_hot_embeddings=False, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1), + 'desc_inputs': [Tensor(np.random.rand(128).astype(np.int32)), [1, 128, 768]], + 'desc_bprop': [[1, 128, 768]]}), + ('CreateAttentionMaskFromInputMask', { + 'block': CreateAttentionMaskFromInputMask(config=BertConfig(batch_size=1)), + 'desc_inputs': [[128]], + 'desc_bprop': [[1, 128, 128]]}), + ('BertOutput_0', { + 'block': BertOutput(in_channels=768, + out_channels=768, + initializer_range=0.02, + dropout_prob=0.1), + 'desc_inputs': [[1, 768], [1, 768]], + 'desc_bprop': [[1, 128, 768]]}), # maybe not right + ('BertTransformer_2', { + 'block': bert_trans(), + 'desc_inputs': [[1, 128, 768], [1, 128, 128]]}), + + ('BertModel', { + 'block': BertModel(config=BertConfig(batch_size=1, + num_hidden_layers=1, + intermediate_size=768, + token_type_ids_from_dataset=False), + is_training=True), + 'desc_inputs': [Tensor(np.random.rand(128).astype(np.int32)), + Tensor(np.random.rand(128).astype(np.int32)), [128]], + 'desc_bprop': [[1, 128, 768], [1, 128, 768], [1, 128, 768]], + 'num_output': 3}), # maybe not right + + ('BertModel_1', { + 'block': BertModel(config=BertConfig(batch_size=1, + num_hidden_layers=1, + intermediate_size=768, + token_type_ids_from_dataset=False), + is_training=False), + 'desc_inputs': [Tensor(np.random.rand(128).astype(np.int32)), + Tensor(np.random.rand(128).astype(np.int32)), [128]], + 'desc_bprop': [[1, 128, 768], [1, 128, 768], [1, 128, 768]], + 'num_output': 3}), # maybe not right + + ('BertModel_2', { + 'block': BertModel(config=BertConfig(batch_size=1, + num_hidden_layers=1, + intermediate_size=768, + token_type_ids_from_dataset=False, + input_mask_from_dataset=False), + is_training=True), + 'desc_inputs': [Tensor(np.random.rand(128).astype(np.int32)), + Tensor(np.random.rand(128).astype(np.int32)), [128]], + 'desc_bprop': [[1, 128, 768], [1, 128, 768], [1, 128, 768]], + 'num_output': 3}), # maybe not right + + ('BertPretrainingLoss', { + 'block': BertPretrainingLoss(config=BertConfig(batch_size=1)), + 'desc_inputs': [[32000], [20, 2], Tensor(np.array([1]).astype(np.int32)), + [20], Tensor(np.array([20]).astype(np.int32))], + 'desc_bprop': [[1]], + 'num_output': 1}), + ('Dense_1', { + 'block': nn.Dense(in_channels=768, + out_channels=3072, + activation='gelu', + weight_init=TruncatedNormal(0.02)), + 'desc_inputs': [[3, 768]], + 'desc_bprop': [[3, 3072]]}), + ('Dense_2', { + 'block': set_train(nn.Dense(in_channels=768, + out_channels=3072, + activation='gelu', + weight_init=TruncatedNormal(0.02),)), + 'desc_inputs': [[3, 768]], + 'desc_bprop': [[3, 3072]]}), + ('GetNextSentenceOutput', { + 'block': GetNextSentenceOutput(BertConfig(batch_size=1)), + 'desc_inputs': [[128, 768]], + 'desc_bprop': [[128, 2]]}), + ('Adam_1', { + 'block': set_train(TrainStepWrapForAdam(NetForAdam())), + 'desc_inputs': [[1, 64], [1, 10]], + 'skip': ['backward']}), + ('Adam_2', { + 'block': set_train(TrainStepWrapForAdam(GetNextSentenceOutput(BertConfig(batch_size=1)))), + 'desc_inputs': [[128, 768], [128, 2]], + 'skip': ['backward']}), + ('AdamWeightDecayDynamicLR', { + 'block': set_train(TrainStepWrapForAdamDynamicLr(NetForAdam())), + 'desc_inputs': [[1, 64]], + 'skip': ['backward']}), + ('ClipGradients', { + 'block': TempC2Wrap(ClipGradients(), 1, 1.0), + 'desc_inputs': [tuple(convert(shp) for shp in [[1], [1], [1]])], + 'skip': ['backward', 'exec']}), + ] + +test_case = functools.reduce(lambda x, y: x+y, [test_case_cell_ops]) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +test_exec_case = filter(lambda x: 'skip' not in x[1] or + 'exec' not in x[1]['skip'], test_case) +test_backward_exec_case = filter(lambda x: 'skip' not in x[1] or + 'backward' not in x[1]['skip'] and 'backward_exec' + not in x[1]['skip'], test_case) +test_check_gradient_case = filter(lambda x: 'skip' not in x[1] or + 'backward' not in x[1]['skip'] and 'backward_exec' + not in x[1]['skip'], test_case) + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + return test_exec_case + +@mindspore_test(pipeline_for_compile_grad_ge_graph_for_case_by_case_config) +def test_backward_exec(): + return test_backward_exec_case diff --git a/tests/ut/python/model/test_lenet.py b/tests/ut/python/model/test_lenet.py new file mode 100644 index 00000000000..228ff281e05 --- /dev/null +++ b/tests/ut/python/model/test_lenet.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test lenet""" +import numpy as np + +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor +from mindspore.ops import operations as P +import mindspore.context as context +from ....train_step_wrap import train_step_with_loss_warp, train_step_with_sens + +context.set_context(mode=context.GRAPH_MODE) +class LeNet5(nn.Cell): + """LeNet5 definition""" + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +def test_lenet5_train_step(): + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = train_step_with_loss_warp(LeNet5()) + _executor.compile(net, predict, label) + + +def test_lenet5_train_sens(): + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + sens = Tensor(np.ones([1, 10]).astype(np.float32)) + net = train_step_with_sens(LeNet5(), sens) + _executor.compile(net, predict) + + +def test_lenet5_train_step_training(): + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = train_step_with_loss_warp(LeNet5()) + net.set_train() + _executor.compile(net, predict, label) diff --git a/tests/ut/python/model/test_lenet_core_after_exception.py b/tests/ut/python/model/test_lenet_core_after_exception.py new file mode 100644 index 00000000000..b13234395fe --- /dev/null +++ b/tests/ut/python/model/test_lenet_core_after_exception.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_lenet_core_after_exception""" +import numpy as np + +import pytest +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from ....train_step_wrap import train_step_with_loss_warp + + +class LeNet5(nn.Cell): + """LeNet5 definition""" + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5, pad_mode="valid") + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode="valid") + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 3) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +def test_lenet5_exception(): + in1 = np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01 + in2 = np.zeros([1, 10]).astype(np.float32) + predict = Tensor(in1) + label = Tensor(in2) + net = train_step_with_loss_warp(LeNet5()) + with pytest.raises(ValueError): + _executor.compile(net, predict, label) diff --git a/tests/ut/python/model/test_mix_precision.py b/tests/ut/python/model/test_mix_precision.py new file mode 100644 index 00000000000..0a8b185e8c6 --- /dev/null +++ b/tests/ut/python/model/test_mix_precision.py @@ -0,0 +1,186 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_mix_precision""" +import numpy as np + +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore.common.api import _executor +from mindspore.common.parameter import Parameter +from mindspore.common import ParameterTuple +from mindspore import Tensor, context +from mindspore.ops import operations as P +from mindspore.ops import composite as C +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Momentum +from ....train_step_wrap import train_step_with_loss_warp +from tests.ops_common import convert +from mindspore.train.parallel_utils import ParallelMode + + +class LeNet5(nn.Cell): + """LeNet5""" + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class NetForConcat(nn.Cell): + def __init__(self): + super(NetForConcat, self).__init__() + self.concat = P.Concat() + self.x1 = Tensor(np.zeros([1, 10]).astype(np.float32)) + self.x2 = Parameter(Tensor(np.zeros([1, 10]).astype(np.float32)), name='x2') + + def construct(self, x0): + return self.concat((x0, self.x1, self.x2)) + + +def test_add_cast_flag(): + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = LeNet5() + net.to_float(mstype.float16) + net.fc3.to_float(mstype.float32) + net = train_step_with_loss_warp(net) + net.set_train() + _executor.compile(net, predict, label) + + +def test_add_cast_flag_tensor(): + x1 = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = NetForConcat() + net.add_flags_recursive(fp16=True) + net.set_train() + _executor.compile(net, x1) + + +def test_on_momentum(): + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = LeNet5() + net = train_step_with_loss_warp(net).to_float(mstype.float16) + net.set_train() + _executor.compile(net, predict, label) + + +def test_data_parallel_with_cast(): + """test_data_parallel_with_cast""" + predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = LeNet5() + net.to_float(mstype.float16) + net.fc3.to_float(mstype.float32) + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + + optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), + learning_rate=0.1, + momentum=0.9) + net = WithLossCell(net, loss_fn) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, mirror_mean=True, device_num=8) + net = TrainOneStepCell(net, optimizer) + + _executor.compile(net, predict, label) + context.reset_auto_parallel_context() + + +class NetForPReLU(nn.Cell): + def __init__(self): + super(NetForPReLU, self).__init__() + self.prelu = nn.PReLU() + + def construct(self, x): + return self.prelu(x) + + +def test_nn_prelu(): + x = Tensor(np.ones([1, 16, 10, 10]).astype(np.float32) * 0.01) + net = NetForPReLU().set_train() + net.add_flags_recursive(fp16=True) + _executor.compile(net, x) + + +class NetForCast(nn.Cell): + def __init__(self): + super(NetForCast, self).__init__() + self.concat = P.Concat() + self.x1 = Tensor(1.0, mstype.float32) + + def construct(self, x0): + x = self.x1 * x0 + return x + + +def test_cast(): + x = Tensor(np.ones([1, 16, 10, 10]).astype(np.float32) * 0.01) + net = NetForCast() + net.add_flags_recursive(fp16=True) + _executor.compile(net, x) + + +"""test grad of PReLU, which cause AddN(generated by grad) fail""" +class IRBlockZ(nn.Cell): + def __init__(self, inplanes, planes): + super(IRBlockZ, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=1, pad_mode="same", group=1, has_bias=False, dilation=1) + self.act_layer = nn.PReLU(planes) + + def construct(self, x): + out = self.conv1(x) + return self.act_layer(out) + + +class GetParamGrad(nn.Cell): + def __init__(self, network): + super(GetParamGrad, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + + def construct(self, data, sens): + weights = self.weights + return self.grad(self.network, weights)(data, sens) + + +def test_grad_conv_prelu(): + shapes = [[64, 64, 112, 112]] + outshape = [[64, 64, 56, 56]] + net = IRBlockZ(inplanes=64, planes=64).add_flags_recursive(fp16=True) + inputs = [convert(shp, dtype=np.float16) for shp in shapes] + sens_shape = outshape[0] + sens = convert(sens_shape, dtype=np.float16) + all_inputs = inputs + [sens] + net = GetParamGrad(net) + net.set_train() + net(*all_inputs) diff --git a/tests/ut/python/model/test_utils.py b/tests/ut/python/model/test_utils.py new file mode 100644 index 00000000000..e2a66275b8b --- /dev/null +++ b/tests/ut/python/model/test_utils.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_dataset_utils""" +import pytest +import mindspore as ms +from mindspore.train._utils import _construct_tensor_list + + +def test_init_dataset_graph(): + types = (ms.float32, ms.float32) + shapes = ((1, 3, 224, 224), (32,)) + _construct_tensor_list(types, shapes) + + +def test_init_dataset_graph_one_dim(): + types = (ms.float32,) + shapes = ((1, 3, 224, 224),) + _construct_tensor_list(types, shapes) + + +def test_init_dataset_graph_dim_error(): + types = (ms.float32, ms.float32) + shapes = ((1, 3, 224, 224),) + with pytest.raises(ValueError): + _construct_tensor_list(types, shapes) diff --git a/tests/ut/python/model/test_vgg.py b/tests/ut/python/model/test_vgg.py new file mode 100644 index 00000000000..8f05179eec7 --- /dev/null +++ b/tests/ut/python/model/test_vgg.py @@ -0,0 +1,29 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test_vgg""" +import numpy as np +import pytest + +from mindspore import Tensor +from mindspore.model_zoo.vgg import vgg16 +from ..ut_filter import non_graph_engine + + +@non_graph_engine +def test_vgg16(): + inputs = Tensor(np.random.rand(1, 3, 112, 112).astype(np.float32)) + net = vgg16() + with pytest.raises(ValueError): + print(net.construct(inputs)) diff --git a/tests/ut/python/nn/__init__.py b/tests/ut/python/nn/__init__.py new file mode 100644 index 00000000000..5443c0ca48e --- /dev/null +++ b/tests/ut/python/nn/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup for pytest""" +import mindspore.context as context + +# pylint: disable=unused-argument +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) diff --git a/tests/ut/python/nn/optim/__init__.py b/tests/ut/python/nn/optim/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/nn/optim/test_adam.py b/tests/ut/python/nn/optim/test_adam.py new file mode 100644 index 00000000000..d9321b1d266 --- /dev/null +++ b/tests/ut/python/nn/optim/test_adam.py @@ -0,0 +1,108 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test adam """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor, Parameter +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.ops import operations as P +from mindspore.nn.optim import AdamWeightDecay, AdamWeightDecayDynamicLR + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +class NetWithoutWeight(nn.Cell): + def __init__(self): + super(NetWithoutWeight, self).__init__() + self.matmul = P.MatMul() + + def construct(self, x): + x = self.matmul(x, x) + return x + + +def test_adamwithoutparam(): + net = NetWithoutWeight() + net.set_train() + with pytest.raises(ValueError, match=r"optimizer got an empty parameter list"): + AdamWeightDecay(net.trainable_params(), learning_rate=0.1) + + +def test_adamw_compile(): + """ test_adamw_compile """ + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = AdamWeightDecay(net.trainable_params(), learning_rate=0.1) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) + + +def test_AdamWeightDecay_beta1(): + net = Net() + print("**********", net.get_parameters()) + with pytest.raises(ValueError): + AdamWeightDecay(net.get_parameters(), beta1=1.0, learning_rate=0.1) + + +def test_AdamWeightDecay_beta2(): + net = Net() + with pytest.raises(ValueError): + AdamWeightDecay(net.get_parameters(), beta2=1.0, learning_rate=0.1) + + +def test_AdamWeightDecay_e(): + net = Net() + with pytest.raises(ValueError): + AdamWeightDecay(net.get_parameters(), eps=-0.1, learning_rate=0.1) + + +def test_AdamWeightDecayDynamicLR(): + """ test_AdamWeightDecayDynamicLR """ + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = AdamWeightDecayDynamicLR(net.trainable_params(), decay_steps=20, learning_rate=0.1) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) + + +def test_adam_mindspore_flatten(): + net = nn.Flatten() + with pytest.raises(ValueError, match=r"optimizer got an empty parameter list"): + AdamWeightDecay(net.get_parameters()) diff --git a/tests/ut/python/nn/optim/test_ftrl.py b/tests/ut/python/nn/optim/test_ftrl.py new file mode 100644 index 00000000000..adddfa05bf9 --- /dev/null +++ b/tests/ut/python/nn/optim/test_ftrl.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test FTRL """ + +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.common.api import _executor +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn.optim import FTRL +from mindspore.ops import operations as P + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name='weight') + self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name='bias') + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +def test_ftrl(): + """ test_ftrl """ + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = FTRL(net.trainable_params()) + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) + diff --git a/tests/ut/python/nn/optim/test_lamb.py b/tests/ut/python/nn/optim/test_lamb.py new file mode 100644 index 00000000000..502c7feed11 --- /dev/null +++ b/tests/ut/python/nn/optim/test_lamb.py @@ -0,0 +1,75 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test lamb """ +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor, Parameter +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.ops import operations as P +from mindspore.nn.optim import Lamb + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +class NetWithoutWeight(nn.Cell): + """ NetWithoutWeight definition """ + def __init__(self): + super(NetWithoutWeight, self).__init__() + self.matmul = P.MatMul() + + def construct(self, x): + x = self.matmul(x, x) + return x + + +def test_lamb_1(): + """ test_Lamb_1 """ + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Lamb(net.trainable_params(), decay_steps=10, warmup_steps=5) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) + + +def test_lamb_2(): + """ test_Lamb_2 """ + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Lamb(net.trainable_params(), decay_steps=10, warmup_steps=0) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) diff --git a/tests/ut/python/nn/optim/test_lars.py b/tests/ut/python/nn/optim/test_lars.py new file mode 100644 index 00000000000..92d218a32b3 --- /dev/null +++ b/tests/ut/python/nn/optim/test_lars.py @@ -0,0 +1,63 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.common.api import _executor +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn.optim import LARS, Momentum +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype +from collections import Counter + + +def multisteplr(total_steps, milestone, base_lr=0.9, gamma=0.1, dtype=mstype.float32): + lr = [] + milestone = Counter(milestone) + + for step in range(total_steps): + base_lr = base_lr * gamma ** milestone[step] + lr.append(base_lr) + return Tensor(np.array(lr), dtype) + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +def test_lars(): + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + loss = nn.SoftmaxCrossEntropyWithLogits() + + lr = multisteplr(10, [2, 6]) + SGD = Momentum(net.trainable_params(), lr, 0.9) + optimizer = LARS(SGD, epsilon=1e-08, hyperpara=0.02, decay_filter=lambda x: 'bn' not in x.name, + lars_filter=lambda x: 'bn' not in x.name) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) diff --git a/tests/ut/python/nn/optim/test_lr_schedule.py b/tests/ut/python/nn/optim/test_lr_schedule.py new file mode 100644 index 00000000000..fc2b8a3f4d4 --- /dev/null +++ b/tests/ut/python/nn/optim/test_lr_schedule.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_lr_schedule """ +import numpy as np +from mindspore.nn import Cell +from mindspore.ops.operations import BiasAdd, MatMul +from mindspore import Parameter, ParameterTuple, Tensor +from mindspore.nn import WithLossCell +from mindspore.nn.optim import Momentum +from mindspore.nn import SoftmaxCrossEntropyWithLogits +from mindspore.ops.composite import grad_by_list +from mindspore.ops import functional as F +from mindspore.nn.optim import Optimizer + + +class Net(Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10])), name="weight") + self.bias = Parameter(Tensor(np.ones([10])), name="bias") + self.matmul = MatMul() + self.biasAdd = BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +class _TrainOneStepCell(Cell): + """ _TrainOneStepCell definition """ + def __init__(self, network, optimizer): + """ + Append an optimizer to the training network after that the construct + function can be called to create the backward graph. + Arguments: + network: The training network. + Note that loss function should have been added. + optimizer: optimizer for updating the weights + """ + super(_TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.get_parameters()) + + if not isinstance(optimizer, Optimizer): + raise TypeError('{} is not an optimizer'.format( + type(optimizer).__name__)) + + self.has_lr_schedule = False + self.optimizer = optimizer + + def construct(self, data, label, *args): + weights = self.weights + grads = grad_by_list(self.network, weights)(data, label) + if self.lr_schedule: + self.schedule.update_lr(*args) + return self.optimizer(grads) + diff --git a/tests/ut/python/nn/optim/test_momentum.py b/tests/ut/python/nn/optim/test_momentum.py new file mode 100644 index 00000000000..60fee6cc922 --- /dev/null +++ b/tests/ut/python/nn/optim/test_momentum.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test momentum """ +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor, Parameter +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.ops import operations as P +from mindspore.nn.optim import Momentum + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +def test_momentum_compile(): + """ test_momentum_compile """ + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + net = Net() + net.set_train() + + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False) + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + _executor.compile(train_network, inputs, label) diff --git a/tests/ut/python/nn/optim/test_optimizer.py b/tests/ut/python/nn/optim/test_optimizer.py new file mode 100644 index 00000000000..860d751fd53 --- /dev/null +++ b/tests/ut/python/nn/optim/test_optimizer.py @@ -0,0 +1,108 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test optimizer """ +import numpy as np +import pytest +from mindspore.nn.optim import Optimizer, SGD, Adam, AdamWeightDecay, AdamWeightDecayDynamicLR +from mindspore import Tensor +from mindspore.common.parameter import Parameter + + +gradient = Tensor(np.zeros([1, 2, 3])) +accumulation = gradient +variable = accumulation + + +paramsTensor = Tensor(np.zeros([1, 2, 3])) +class IterableObjc: + def __iter__(self): + cont = 0 + while cont < 3: + cont += 1 + yield Parameter(Tensor(cont), name="cont") + + +params = IterableObjc() + +class TestOptimizer(): + def test_init(self): + Optimizer(0.5, params) + with pytest.raises(ValueError): + Optimizer(-0.5, params) + + def test_construct(self): + opt_2 = Optimizer(0.5, params) + with pytest.raises(NotImplementedError): + opt_2.construct() + + +class TestAdam(): + """ TestAdam definition """ + def test_init(self): + Adam(params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, + use_nesterov=False, weight_decay=0.0, loss_scale=1.0) + + def test_construct(self): + with pytest.raises(TypeError): + adam = Adam(params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, + use_nesterov=False, weight_decay=0.0, loss_scale=1.0) + adam.construct(gradient) + + +class TestSGD(): + """ TestSGD definition """ + def test_init(self): + with pytest.raises(ValueError): + SGD(params, learning_rate=0.1, momentum=-0.1, dampening=0, weight_decay=0, nesterov=False) + with pytest.raises(ValueError): + SGD(params, learning_rate=0.12, momentum=-0.1, dampening=0, weight_decay=0, nesterov=False) + SGD(params) + + +class TestNullParam(): + """ TestNullParam definition """ + def test_optim_init(self): + with pytest.raises(TypeError): + Optimizer(0.1, None) + + def test_AdamWightDecay_init(self): + with pytest.raises(TypeError): + AdamWeightDecay(None) + + def test_AdamWeightDecayDynamicLR_init(self): + with pytest.raises(TypeError): + AdamWeightDecayDynamicLR(None, 10) + + def test_Sgd_init(self): + with pytest.raises(TypeError): + SGD(None) + +class TestUnsupportParam(): + """ TestUnsupportParam definition """ + def test_optim_init(self): + with pytest.raises(ValueError): + Optimizer(0.1, (1, 2, 3)) + + def test_AdamWightDecay_init(self): + with pytest.raises(TypeError): + AdamWeightDecay(9) + + def test_AdamWeightDecayDynamicLR_init(self): + with pytest.raises(TypeError): + AdamWeightDecayDynamicLR(0.5, 10) + + def test_Sgd_init(self): + with pytest.raises(TypeError): + SGD(paramsTensor) diff --git a/tests/ut/python/nn/test_activation.py b/tests/ut/python/nn/test_activation.py new file mode 100755 index 00000000000..d035e2971de --- /dev/null +++ b/tests/ut/python/nn/test_activation.py @@ -0,0 +1,105 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Activations """ +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor +from ..ut_filter import non_graph_engine + +class SoftmaxNet(nn.Cell): + def __init__(self, dim): + super(SoftmaxNet, self).__init__() + self.softmax = nn.Softmax(dim) + + def construct(self, x): + return self.softmax(x) + + +@non_graph_engine +def test_compile(): + net = SoftmaxNet(0) + input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + net(input_tensor) + + +@non_graph_engine +def test_compile_axis(): + net = SoftmaxNet(-1) + prob = 355 + input_data = np.random.randn(4, 16, 1, 1).astype(np.float32) * prob + input_tensor = Tensor(input_data) + net(input_tensor) + + +class LogSoftmaxNet(nn.Cell): + def __init__(self, dim): + super(LogSoftmaxNet, self).__init__() + self.logsoftmax = nn.LogSoftmax(dim) + + def construct(self, x): + return self.logsoftmax(x) + + +@non_graph_engine +def test_compile_logsoftmax(): + net = LogSoftmaxNet(0) + input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]])) + net(input_tensor) + + +class Net1(nn.Cell): + def __init__(self): + super(Net1, self).__init__() + self.relu = nn.ReLU() + + def construct(self, x): + return self.relu(x) + + +def test_compile_relu(): + net = Net1() + input_data = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + _executor.compile(net, input_data) + + +class Net_gelu(nn.Cell): + def __init__(self): + super(Net_gelu, self).__init__() + self.gelu = nn.GELU() + + def construct(self, x): + return self.gelu(x) + + +def test_compile_gelu(): + net = Net_gelu() + input_data = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) + _executor.compile(net, input_data) + + +class NetLeakyReLU(nn.Cell): + def __init__(self, alpha): + super(NetLeakyReLU, self).__init__() + self.leaky_relu = nn.LeakyReLU(alpha) + + def construct(self, x): + return self.leaky_relu(x) + + +def test_compile_leaky_relu(): + net = NetLeakyReLU(alpha=0.1) + input_data = Tensor(np.array([[1.6, 0, 0.6], [6, 0, -6]], dtype=np.float32)) + _executor.compile(net, input_data) diff --git a/tests/ut/python/nn/test_batchnorm.py b/tests/ut/python/nn/test_batchnorm.py new file mode 100644 index 00000000000..eaafdd81b4d --- /dev/null +++ b/tests/ut/python/nn/test_batchnorm.py @@ -0,0 +1,58 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ut for batchnorm layer""" +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor, Parameter + + +def test_bn_pars_valid1(): + """ut of BatchNorm parameters' validation""" + with pytest.raises(ValueError): + nn.BatchNorm2d(num_features=0) + + +def test_bn_pars_valid2(): + """ut of BatchNorm parameters' validation""" + with pytest.raises(ValueError): + nn.BatchNorm2d(num_features=3, momentum=-0.1) + + +def test_bn_init(): + """ut of BatchNorm parameters' validation""" + bn = nn.BatchNorm2d(num_features=3) + + assert isinstance(bn.gamma, Parameter) + assert isinstance(bn.beta, Parameter) + assert isinstance(bn.moving_mean, Parameter) + assert isinstance(bn.moving_variance, Parameter) + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.bn = nn.BatchNorm2d(num_features=3) + + def construct(self, input_x): + return self.bn(input_x) + + +def test_compile(): + net = Net() + input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)) + _executor.compile(net, input_data) diff --git a/tests/ut/python/nn/test_cell.py b/tests/ut/python/nn/test_cell.py new file mode 100644 index 00000000000..882756f3d29 --- /dev/null +++ b/tests/ut/python/nn/test_cell.py @@ -0,0 +1,298 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test cell """ +import numpy as np +import pytest +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.common.api import _executor +from ..ut_filter import non_graph_engine + + +class ModA(nn.Cell): + def __init__(self, tensor): + super(ModA, self).__init__() + self.weight = Parameter(tensor, name="weight") + + def construct(self, *inputs): + pass + + +class ModB(nn.Cell): + def __init__(self, tensor): + super(ModB, self).__init__() + self.weight = Parameter(tensor, name="weight") + + def construct(self, *inputs): + pass + + +class ModC(nn.Cell): + def __init__(self, ta, tb): + super(ModC, self).__init__() + self.mod1 = ModA(ta) + self.mod2 = ModB(tb) + + def construct(self, *inputs): + pass + + +class Net(nn.Cell): + """ Net definition """ + name_len = 4 + cells_num = 3 + + def __init__(self, ta, tb): + super(Net, self).__init__() + self.mod1 = ModA(ta) + self.mod2 = ModB(tb) + self.mod3 = ModC(ta, tb) + + def construct(self, *inputs): + pass + + +class Net2(nn.Cell): + def __init__(self, ta, tb): + super(Net2, self).__init__(auto_prefix=False) + self.mod1 = ModA(ta) + self.mod2 = ModB(tb) + self.mod3 = ModC(ta, tb) + + def construct(self, *inputs): + pass + + +class ConvNet(nn.Cell): + """ ConvNet definition """ + image_h = 224 + image_w = 224 + output_ch = 64 + + def __init__(self, num_classes=10): + super(ConvNet, self).__init__() + self.conv1 = nn.Conv2d(3, ConvNet.output_ch, kernel_size=7, stride=2, pad_mode="pad", padding=3) + self.bn1 = nn.BatchNorm2d(ConvNet.output_ch) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="pad", padding=1) + self.flatten = nn.Flatten() + self.fc = nn.Dense( + int(ConvNet.image_h*ConvNet.image_w*ConvNet.output_ch/(4*4)), + num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.flatten(x) + x = self.fc(x) + return x + + +def test_basic(): + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len + assert names[0] == "mod1.weight" + assert names[1] == "mod2.weight" + assert names[2] == "mod3.mod1.weight" + assert names[3] == "mod3.mod2.weight" + + +def test_parameter_name(): + """ test_parameter_name """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + names = [] + for m in n.parameters_and_names(): + if m[0]: + names.append(m[0]) + assert names[0] == "mod1.weight" + assert names[1] == "mod2.weight" + assert names[2] == "mod3.mod1.weight" + assert names[3] == "mod3.mod2.weight" + + +def test_cell_name(): + """ test_cell_name """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + n.insert_child_to_cell('modNone', None) + names = [] + for m in n.cells_and_names(): + if m[0]: + names.append(m[0]) + assert names[0] == "mod1" + assert names[1] == "mod2" + assert names[2] == "mod3" + assert names[3] == "mod3.mod1" + assert names[4] == "mod3.mod2" + + +def test_cells(): + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + ch = list(n.cells()) + assert len(ch) == n.cells_num + + +def test_exceptions(): + """ test_exceptions """ + t = Tensor(np.ones([2, 3])) + + class ModError(nn.Cell): + def __init__(self, tensor): + self.weight = Parameter(tensor, name="weight") + super(ModError, self).__init__() + + def construct(self, *inputs): + pass + + with pytest.raises(AttributeError): + ModError(t) + + class ModError1(nn.Cell): + def __init__(self, tensor): + super().__init__() + self.weight = Parameter(tensor, name="weight") + self.weight = None + self.weight = ModA(tensor) + + def construct(self, *inputs): + pass + + with pytest.raises(TypeError): + ModError1(t) + + class ModError2(nn.Cell): + def __init__(self, tensor): + super().__init__() + self.mod = ModA(tensor) + self.mod = None + self.mod = tensor + + def construct(self, *inputs): + pass + + with pytest.raises(TypeError): + ModError2(t) + + m = nn.Cell() + with pytest.raises(NotImplementedError): + m.construct() + + +def test_del(): + """ test_del """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len + del n.mod1 + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len - 1 + with pytest.raises(AttributeError): + del n.mod1.weight + del n.mod2.weight + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len - 2 + with pytest.raises(AttributeError): + del n.mod + + +def test_add_attr(): + """ test_add_attr """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + p = Parameter(ta, name="weight") + m = nn.Cell() + m.insert_param_to_cell('weight', p) + + with pytest.raises(TypeError): + m.insert_child_to_cell("network", p) + + with pytest.raises(KeyError): + m.insert_param_to_cell('', p) + with pytest.raises(KeyError): + m.insert_param_to_cell('a.b', p) + m.insert_param_to_cell('weight', p) + with pytest.raises(KeyError): + m.insert_child_to_cell('', ModA(ta)) + with pytest.raises(KeyError): + m.insert_child_to_cell('a.b', ModB(tb)) + + with pytest.raises(TypeError): + m.insert_child_to_cell('buffer', tb) + with pytest.raises(TypeError): + m.insert_param_to_cell('w', ta) + with pytest.raises(TypeError): + m.insert_child_to_cell('m', p) + + class ModAddCellError(nn.Cell): + def __init__(self, tensor): + self.mod = ModA(tensor) + super().__init__() + + def construct(self, *inputs): + pass + + with pytest.raises(AttributeError): + ModAddCellError(ta) + + +def test_train_eval(): + m = nn.Cell() + assert not m.training + m.set_train() + assert m.training + m.set_train(False) + assert not m.training + + +def test_stop_update_name(): + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net2(ta, tb) + names = list(n.parameters_dict().keys()) + assert names[0] == "weight" + assert names[1] == "mod1.weight" + assert names[2] == "mod2.weight" + + +class ModelName(nn.Cell): + def __init__(self, tensor): + super(ModelName, self).__init__() + self.w2 = Parameter(tensor, name="weight") + self.w1 = Parameter(tensor, name="weight") + self.w3 = Parameter(tensor, name=None) + self.w4 = Parameter(tensor, name=None) + + def construct(self, *inputs): + pass + + +def test_cell_names(): + ta = Tensor(np.ones([2, 3])) + mn = ModelName(ta) + with pytest.raises(ValueError): + _executor.compile(mn) diff --git a/tests/ut/python/nn/test_cell_wrapper.py b/tests/ut/python/nn/test_cell_wrapper.py new file mode 100755 index 00000000000..3e163c9e4fb --- /dev/null +++ b/tests/ut/python/nn/test_cell_wrapper.py @@ -0,0 +1,103 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import pytest +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.common.api import _executor +from mindspore.nn import TrainOneStepCell, WithLossCell, ParameterUpdate +from mindspore.nn.optim import Momentum +from mindspore.common import dtype as mstype +from mindspore.ops import operations as P + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + return x + + +def test_parameter_update_int32_and_tensor(): + """ test_parameter_update """ + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(net.get_parameters(), Tensor(np.array([0.1, 0.01, 0.001]), mstype.float32), 0.001) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + + # compile train graph + train_network.set_train() + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + _executor.compile(train_network, inputs, label) + + # test tensor + param_lr = train_network.parameters_dict()['learning_rate'] + update_network = ParameterUpdate(param_lr) + update_network.phase = 'update_param' + + input_lr = Tensor(np.array([0.2, 0.02, 0.002]), mstype.float32) + _executor.compile(update_network, input_lr) + + # test int32 + param_step = train_network.parameters_dict()['global_step'] + update_global_step = ParameterUpdate(param_step) + + input_step = Tensor(np.array([0.2, 0.02, 0.002]), mstype.float32) + _executor.compile(update_global_step, input_step) + + +def test_parameter_update_float32(): + """ test_parameter_update """ + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(net.get_parameters(), 0.01, 0.001) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + + # compile train graph + train_network.set_train() + inputs = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + _executor.compile(train_network, inputs, label) + + # construct and compile update graph + param_lr = train_network.parameters_dict()['learning_rate'] + update_network = ParameterUpdate(param_lr) + update_network.phase = 'update_param' + + input_lr = Tensor(0.0001, mstype.float32) + _executor.compile(update_network, input_lr) + + +def test_parameter_update_error(): + """ test_parameter_update """ + input_np = np.array([1]) + input_parameter = Parameter(np.array([1]), 'input_parameter') + + with pytest.raises(TypeError): + ParameterUpdate(input_np) + + with pytest.raises(TypeError): + ParameterUpdate(input_parameter) diff --git a/tests/ut/python/nn/test_checkparameter.py b/tests/ut/python/nn/test_checkparameter.py new file mode 100644 index 00000000000..e53cdaf6d28 --- /dev/null +++ b/tests/ut/python/nn/test_checkparameter.py @@ -0,0 +1,116 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test checkparameter """ +import pytest +from mindspore._checkparam import check_int, check_int_positive, \ + check_input_format, check_bool, twice + + +kernel_size = 5 +kernel_size1 = twice(kernel_size) +assert kernel_size1 == (5, 5) + + +def test_check_int_1(): + assert check_int(3) == 3 + + +def check_int_positive_1(): + with pytest.raises(ValueError): + check_int_positive(-1) + + +def test_NCHW1(): + assert check_input_format("NCHW") == "NCHW" + + +def test_NCHW3(): + with pytest.raises(ValueError): + check_input_format("rt") + + +def test_check_int_2(): + with pytest.raises(TypeError): + check_int(3.3) + + +def test_check_int_3(): + with pytest.raises(TypeError): + check_int("str") + + +def test_check_int_4(): + with pytest.raises(TypeError): + check_int(True) + + +def test_check_int_5(): + check_int(0) + check_int(1) + with pytest.raises(TypeError): + check_int(True) + with pytest.raises(TypeError): + check_int(False) + + +def test_check_bool_1(): + assert check_bool(True) + + +def test_check_bool_2(): + assert check_bool(False) is not True + + +def test_check_bool_3(): + with pytest.raises(TypeError): + check_bool("str") + + +def test_check_bool_4(): + with pytest.raises(TypeError): + check_bool(1) + + +def test_check_bool_5(): + with pytest.raises(TypeError): + check_bool(3.5) + + +def test_twice_1(): + assert twice(3) == (3, 3) + + +def test_twice_2(): + assert twice((3, 3)) == (3, 3) + + +def test_twice_3(): + with pytest.raises(TypeError): + twice(0.5) + + +def test_twice_4(): + with pytest.raises(TypeError): + twice("str") + + +def test_twice_5(): + with pytest.raises(TypeError): + twice((1, 2, 3)) + + +def test_twice_6(): + with pytest.raises(TypeError): + twice((3.3, 4)) diff --git a/tests/ut/python/nn/test_clip_by_norm.py b/tests/ut/python/nn/test_clip_by_norm.py new file mode 100644 index 00000000000..54b586f8486 --- /dev/null +++ b/tests/ut/python/nn/test_clip_by_norm.py @@ -0,0 +1,27 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test clip_by_norm """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor +from ..ut_filter import non_graph_engine + +@non_graph_engine +def test_clip_by_norm(): + clip_by_norm = nn.ClipByNorm() + x = Tensor(np.array([[-2, 0, 0], [0, 3, 4]]).astype(np.float32)) + clip_norm = Tensor(np.array([1]).astype(np.float32)) + clip_by_norm(x, clip_norm) diff --git a/tests/ut/python/nn/test_container.py b/tests/ut/python/nn/test_container.py new file mode 100644 index 00000000000..b055d020d86 --- /dev/null +++ b/tests/ut/python/nn/test_container.py @@ -0,0 +1,144 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test container """ +from collections import OrderedDict +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor + + +weight = Tensor(np.ones([2, 2])) +conv2 = nn.Conv2d(3, 64, (3, 3), stride=2, padding=0) + +kernel_size = 3 +stride = 2 +padding = 1 +avg_pool = nn.AvgPool2d(kernel_size, stride) + + +class TestSequentialCell(): + """ TestSequentialCell """ + def test_SequentialCell_init(self): + m = nn.SequentialCell() + assert type(m).__name__ == 'SequentialCell' + + def test_SequentialCell_init2(self): + m = nn.SequentialCell([conv2]) + assert len(m) == 1 + + def test_SequentialCell_init3(self): + m = nn.SequentialCell([conv2, avg_pool]) + assert len(m) == 2 + + def test_SequentialCell_init4(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + assert len(m) == 2 + + def test_getitem1(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + assert m[0] == conv2 + + def test_getitem2(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + assert len(m[0:2]) == 2 + assert m[:2][1] == avg_pool + + def test_setitem1(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + m[1] = conv2 + assert m[1] == m[0] + + def test_setitem2(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + with pytest.raises(TypeError): + m[1.0] = conv2 + + def test_delitem1(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + del m[0] + assert len(m) == 1 + + def test_delitem2(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + del m[:] + assert type(m).__name__ == 'SequentialCell' + + +class TestCellList(): + """ TestCellList """ + def test_init1(self): + cell_list = nn.CellList([conv2, avg_pool]) + assert len(cell_list) == 2 + + def test_init2(self): + with pytest.raises(TypeError): + nn.CellList(["test"]) + + def test_getitem(self): + cell_list = nn.CellList([conv2, avg_pool]) + assert cell_list[0] == conv2 + temp_cells = cell_list[:] + assert temp_cells[1] == avg_pool + + def test_setitem(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list[0] = avg_pool + assert cell_list[0] == cell_list[1] + + def test_delitem(self): + cell_list = nn.CellList([conv2, avg_pool]) + del cell_list[0] + assert len(cell_list) == 1 + del cell_list[:] + assert type(cell_list).__name__ == 'CellList' + + def test_iter(self): + cell_list = nn.CellList([conv2, avg_pool]) + for item in cell_list: + cell = item + assert type(cell).__name__ == 'AvgPool2d' + + + def test_add(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list += [conv2] + assert len(cell_list) == 3 + assert cell_list[0] == cell_list[2] + + def test_insert(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list.insert(0, avg_pool) + assert len(cell_list) == 3 + assert cell_list[0] == cell_list[2] + + def test_append(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list.append(conv2) + assert len(cell_list) == 3 + assert cell_list[0] == cell_list[2] + + def test_extend(self): + cell_list = nn.CellList() + cell_list.extend([conv2, avg_pool]) + assert len(cell_list) == 2 + assert cell_list[0] == conv2 diff --git a/tests/ut/python/nn/test_conv.py b/tests/ut/python/nn/test_conv.py new file mode 100644 index 00000000000..821c224a757 --- /dev/null +++ b/tests/ut/python/nn/test_conv.py @@ -0,0 +1,183 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test conv """ +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor +from ..ut_filter import non_graph_engine + + +weight = Tensor(np.ones([2, 2])) +in_channels = 3 +out_channels = 64 + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self, + cin, + cout, + kernel_size, + stride=1, + pad_mode="valid", + padding=0, + dilation=1, + group=1, + has_bias=True, + weight_init='normal', + bias_init='zeros'): + super(Net, self).__init__() + self.conv = nn.Conv2d(cin, + cout, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + + def construct(self, input_x): + return self.conv(input_x) + + +@non_graph_engine +def test_compile(): + net = Net(3, 64, 3, bias_init='zeros') + input_data = Tensor(np.ones([1, 3, 16, 50], np.float32)) + net(input_data) + + +def test_compile_nobias(): + net = Net(3, 64, 4, has_bias=False, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_nobias2(): + net = Net(3, 64, (3, 5), has_bias=False, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_pad_same(): + net = Net(3, 64, (3, 5), pad_mode="same", padding=0, has_bias=False, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_pad_valid(): + net = Net(3, 64, (3, 5), pad_mode="valid", padding=0, has_bias=False, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_pad_pad(): + net = Net(3, 64, (3, 5), pad_mode="pad", padding=1, has_bias=False, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + +def test_conv_group_error(): + with pytest.raises(ValueError): + nn.Conv2d(6, 8, 3, group=3) + with pytest.raises(ValueError): + nn.Conv2d(6, 9, 3, group=2) + +def test_conv_check(): + """ test_conv_check """ + with pytest.raises(ValueError): + Net(3, 64, 4, pad_mode='sane') + + with pytest.raises(ValueError): + Net(3, 0, 4) + + with pytest.raises(ValueError): + Net(3, 1, 4, group=-1) + + with pytest.raises(ValueError): + Net(3, 1, 4, dilation=-1) + + with pytest.raises(ValueError): + Net(3, 1, kernel_size=-1) + + with pytest.raises(ValueError): + Net(3, 1, 4, stride=0) + + with pytest.raises(ValueError): + Net(0, 1, 4) + + +class NetConv2dTranspose(nn.Cell): + def __init__(self, + cin, + cout, + kernel_size, + stride=1, + pad_mode="same", + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + super(NetConv2dTranspose, self).__init__() + self.conv = nn.Conv2dTranspose(cin, + cout, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + + def construct(self, input_x): + return self.conv(input_x) + + +def test_compile_transpose(): + net = NetConv2dTranspose(3, 64, 4, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_transpose_bias(): + net = NetConv2dTranspose(3, 64, 4, has_bias=True, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_transpose_valid(): + net = NetConv2dTranspose(3, 64, 4, pad_mode='valid', weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_transpose_pad(): + net = NetConv2dTranspose(3, 64, 4, pad_mode='pad', weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) + + +def test_compile_transpose_stride2(): + net = NetConv2dTranspose(3, 64, 4, stride=2, weight_init='normal') + input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) + net(input_data) diff --git a/tests/ut/python/nn/test_dense.py b/tests/ut/python/nn/test_dense.py new file mode 100644 index 00000000000..8581576c6ba --- /dev/null +++ b/tests/ut/python/nn/test_dense.py @@ -0,0 +1,151 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test nn.Dense """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore.common.api import _executor +import mindspore.context as context +from mindspore import Tensor +from ..ut_filter import non_graph_engine + + +def test_dense_none(): + with pytest.raises(TypeError): + nn.Dense(3, 2, None, None) + + +def test_dense_invalid_activation(): + with pytest.raises(KeyError): + nn.Dense(3, 2, activation='relu6') + + +@non_graph_engine +def test_dense_str_activation(): + dense = nn.Dense(1, 1, activation='relu') + assert isinstance(dense.activation, nn.ReLU) + + input_data = Tensor(np.random.randint(0, 255, [1, 1]).astype(np.float32)) + dense.construct(input_data) + + +def test_dense_weight_error(): + dim_error = Tensor(np.array([[[0.1], [0.3], [0.6]], [[0.4], [0.5], [0.2]]])) + with pytest.raises(ValueError): + nn.Dense(3, 2, dim_error) + + shape_error = Tensor(np.array([[0.1, 0.3, 0.6], [0.4, 0.5, 0.2]])) + with pytest.raises(ValueError): + nn.Dense(2, 2, shape_error) + with pytest.raises(ValueError): + nn.Dense(3, 3, shape_error) + + +def test_dense_bias_error(): + dim_error = Tensor(np.array([[0.5, 0.3]])) + with pytest.raises(ValueError): + nn.Dense(3, 2, bias_init=dim_error) + + shape_error = Tensor(np.array([0.5, 0.3, 0.4])) + with pytest.raises(ValueError): + nn.Dense(3, 2, bias_init=shape_error) + + +def test_dense_channels_error(): + with pytest.raises(ValueError): + nn.Dense(3, 0) + + with pytest.raises(ValueError): + nn.Dense(-1, 2) + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self, + input_channels, + output_channels, + weight='normal', + bias='zeros', + has_bias=True, + activation=''): + super(Net, self).__init__() + self.dense = nn.Dense(input_channels, + output_channels, + weight, + bias, + has_bias, + activation=activation) + + def construct(self, input_x): + return self.dense(input_x) + + +def test_compile(): + """ test_compile """ + # has bias + weight = Tensor(np.random.randint(0, 255, [8, 64]).astype(np.float32)) + bias = Tensor(np.random.randint(0, 255, [8]).astype(np.float32)) + net = Net(64, 8, weight=weight, bias=bias) + input_data = Tensor(np.random.randint(0, 255, [128, 64]).astype(np.float32)) + _executor.compile(net, input_data) + + # training + net_train = Net(64, 8, weight=weight, bias=bias) + net_train.set_train() + _executor.compile(net_train, input_data) + + +def test_compile_2(): + """ test_compile_2 """ + # no bias + weight = Tensor(np.random.randint(0, 255, [8, 64]).astype(np.float32)) + net = Net(64, 8, weight=weight, has_bias=False) + input_data = Tensor(np.random.randint(0, 255, [128, 64]).astype(np.float32)) + _executor.compile(net, input_data) + + # training + net_train = Net(64, 8, weight=weight, has_bias=False) + net_train.set_train() + _executor.compile(net_train, input_data) + + +def test_compile_3(): + """ test_compile_3 """ + # test for Graph mode + # has bias + context.set_context(mode=context.GRAPH_MODE) + net = Net(128, 10) + input_data = Tensor(np.random.randint(0, 255, [128, 128]).astype(np.float32)) + _executor.compile(net, input_data) + + # training + net_train = Net(128, 10) + net_train.set_train() + _executor.compile(net_train, input_data) + + +def test_compile_4(): + """ test_compile_4 """ + # test for Graph mode + # no bias + context.set_context(mode=context.GRAPH_MODE) + net = Net(128, 10, has_bias=False) + input_data = Tensor(np.random.randint(0, 255, [128, 128]).astype(np.float32)) + _executor.compile(net, input_data) + + # training + net_train = Net(128, 10, has_bias=False) + net_train.set_train() + _executor.compile(net_train, input_data) diff --git a/tests/ut/python/nn/test_dropout.py b/tests/ut/python/nn/test_dropout.py new file mode 100644 index 00000000000..81a20db6f1b --- /dev/null +++ b/tests/ut/python/nn/test_dropout.py @@ -0,0 +1,34 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" Test Dropout """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor + + +def test_check_dropout_3(): + Tensor(np.ones([20, 16, 50]).astype(np.int32)) + with pytest.raises(ValueError): + nn.Dropout(3, 0, 1) + + +class Net_dropout(nn.Cell): + def __init__(self): + super(Net_dropout, self).__init__() + self.dropout = nn.Dropout(0.5) + + def construct(self, x): + return self.dropout(x) diff --git a/tests/ut/python/nn/test_embedding.py b/tests/ut/python/nn/test_embedding.py new file mode 100644 index 00000000000..3e1c46d223c --- /dev/null +++ b/tests/ut/python/nn/test_embedding.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_embedding """ +import numpy as np +import pytest + +from mindspore.model_zoo.Bert_NEZHA import EmbeddingLookup, EmbeddingPostprocessor +from mindspore import Tensor +from mindspore import dtype as mstype +from ..ut_filter import non_graph_engine + + +@non_graph_engine +def test_check_embedding_lookup_1(): + m = EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=False) + m(Tensor(np.ones([128]), mstype.int32)) + + +@non_graph_engine +def test_check_embedding_lookup_2(): + m = EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=True) + m(Tensor(np.ones([128]), mstype.int32)) + +@non_graph_engine +def test_check_embedding_lookup_3(): + m = EmbeddingLookup(vocab_size=32000, + embedding_size=768, + embedding_shape=[1, 128, 768], + use_one_hot_embeddings=True, + initializer_range=0.01) + m(Tensor(np.ones([128]), mstype.int32)) + +@non_graph_engine +def test_embedding_post_1(): + m = EmbeddingPostprocessor(embedding_size=768, + embedding_shape=[1, 128, 768], + use_token_type=True) + m(Tensor(np.ones([128]), mstype.int32), Tensor(np.ones([1, 128, 768]), mstype.float32)) + + +@non_graph_engine +def test_embedding_post_2(): + m = EmbeddingPostprocessor(embedding_size=768, + embedding_shape=[1, 128, 768], + use_token_type=True, + initializer_range=0.3) + m(Tensor(np.ones([128]), mstype.int32), Tensor(np.ones([1, 128, 768]), mstype.float32)) diff --git a/tests/ut/python/nn/test_flatten.py b/tests/ut/python/nn/test_flatten.py new file mode 100644 index 00000000000..0045f811c81 --- /dev/null +++ b/tests/ut/python/nn/test_flatten.py @@ -0,0 +1,36 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test flatten api +""" +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor + + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.flatten = nn.Flatten() + + def construct(self, input_x): + return self.flatten(input_x) + + +def test_compile(): + net = Net() + input_data = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype(np.float32)) + _executor.compile(net, input_data) diff --git a/tests/ut/python/nn/test_loss.py b/tests/ut/python/nn/test_loss.py new file mode 100644 index 00000000000..e5e6c438855 --- /dev/null +++ b/tests/ut/python/nn/test_loss.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test loss """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common.api import _executor +from ..ut_filter import non_graph_engine +import mindspore + +def test_L1Loss(): + loss = nn.L1Loss() + input_data = Tensor(np.array([[1, 2, 3], [2, 3, 4]]).astype(np.float32)) + target_data = Tensor(np.array([[0, 2, 5], [3, 1, 1]]).astype(np.float32)) + loss(input_data, target_data) + + +def test_MSELoss(): + loss = nn.MSELoss() + input_data = Tensor(np.array([[1, 2, 3], [2, 3, 2]]).astype(np.float32)) + target_data = Tensor(np.array([[0, 0, 5], [1, 2, 3]]).astype(np.float32)) + loss(input_data, target_data) + + +@non_graph_engine +def test_SoftmaxCrossEntropyWithLogits(): + """ test_SoftmaxCrossEntropyWithLogits """ + loss = nn.SoftmaxCrossEntropyWithLogits() + + logits = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + labels = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + loss.construct(logits, labels) + + +def test_SoftmaxCrossEntropyWithLogits_reduce(): + """ test_SoftmaxCrossEntropyWithLogits """ + loss = nn.SoftmaxCrossEntropyWithLogits(reduction="mean") + + logits = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + labels = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + loss(logits, labels) + + +def test_SoftmaxCrossEntropyExpand(): + from mindspore import context + context.set_context(mode=context.GRAPH_MODE) + loss = nn.SoftmaxCrossEntropyExpand() + + logits = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + labels = Tensor(np.random.randint(0, 9, [10,]).astype(np.float32)) + _executor.compile(loss, logits, labels) diff --git a/tests/ut/python/nn/test_lstm.py b/tests/ut/python/nn/test_lstm.py new file mode 100644 index 00000000000..18a904dc99b --- /dev/null +++ b/tests/ut/python/nn/test_lstm.py @@ -0,0 +1,101 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test lstm """ +import pytest +import mindspore.context as context +from mindspore import nn +from ..ut_filter import run_on_gpu +from ....ops_common import convert + + +class LstmTestNet(nn.Cell): + """ LstmTestNet definition """ + def __init__(self, input_size, hidden_size, num_layers, has_bias, batch_first, bidirectional): + super(LstmTestNet, self).__init__() + self.lstm = nn.LSTM(input_size=input_size, + hidden_size=hidden_size, + num_layers=num_layers, + has_bias=has_bias, + batch_first=batch_first, + bidirectional=bidirectional, + dropout=0.0) + + + def construct(self, inp, h0, c0): + return self.lstm(inp, (h0, c0)) + + +test_case_cell_ops = [ + ('lstm1_with_bias', { + 'cell': LstmTestNet(10, 12, 2, has_bias=True, batch_first=False, bidirectional=False), + 'input_shape': [[5, 3, 10], [2, 3, 12], [2, 3, 12]], + 'output_shape': [[5, 3, 12], [2, 3, 12], [2, 3, 12]]}), + ('lstm2_without_bias', { + 'cell': LstmTestNet(10, 12, 2, has_bias=False, batch_first=False, bidirectional=False), + 'input_shape': [[5, 3, 10], [2, 3, 12], [2, 3, 12]], + 'output_shape': [[5, 3, 12], [2, 3, 12], [2, 3, 12]]}), + ('lstm3_with_bias_bidirectional', { + 'cell': LstmTestNet(10, 12, 2, has_bias=True, batch_first=False, bidirectional=True), + 'input_shape': [[5, 3, 10], [4, 3, 12], [4, 3, 12]], + 'output_shape': [[5, 3, 24], [4, 3, 12], [4, 3, 12]]}), + ('lstm4_without_bias_bidirectional', { + 'cell': LstmTestNet(10, 12, 2, has_bias=False, batch_first=False, bidirectional=True), + 'input_shape': [[5, 3, 10], [4, 3, 12], [4, 3, 12]], + 'output_shape': [[5, 3, 24], [4, 3, 12], [4, 3, 12]]}), + ('lstm5_with_bias_batch_first', { + 'cell': LstmTestNet(10, 12, 2, has_bias=True, batch_first=True, bidirectional=False), + 'input_shape': [[3, 5, 10], [2, 3, 12], [2, 3, 12]], + 'output_shape': [[3, 5, 12], [2, 3, 12], [2, 3, 12]]}), + ('lstm6_without_bias_batch_first', { + 'cell': LstmTestNet(10, 12, 2, has_bias=False, batch_first=True, bidirectional=False), + 'input_shape': [[3, 5, 10], [2, 3, 12], [2, 3, 12]], + 'output_shape': [[3, 5, 12], [2, 3, 12], [2, 3, 12]]}), + ('lstm7_with_bias_bidirectional_batch_first', { + 'cell': LstmTestNet(10, 12, 2, has_bias=True, batch_first=True, bidirectional=True), + 'input_shape': [[3, 5, 10], [4, 3, 12], [4, 3, 12]], + 'output_shape': [[3, 5, 24], [4, 3, 12], [4, 3, 12]]}), + ('lstm8_without_bias_bidirectional_batch_first', { + 'cell': LstmTestNet(10, 12, 2, has_bias=False, batch_first=True, bidirectional=True), + 'input_shape': [[3, 5, 10], [4, 3, 12], [4, 3, 12]], + 'output_shape': [[3, 5, 24], [4, 3, 12], [4, 3, 12]]}), +] + + +# use -k to select certain testcast +# pytest tests/python/ops/test_lstm.py::test_compile -k lstm_with_bias + +@pytest.mark.parametrize('args', test_case_cell_ops, ids=lambda x: x[0]) +def test_compile(args): + config = args[1] + shapes = config['input_shape'] + net = config['cell'] + net.set_train() + inputs = [convert(shp) for shp in shapes] + out = net(*inputs) + print(f"out: {out}") + +@run_on_gpu +@pytest.mark.parametrize('args', test_case_cell_ops, ids=lambda x: x[0]) +def test_execute(args): + """ test_execute """ + config = args[1] + shapes = config['input_shape'] + net = config['cell'] + net.set_train() + inputs = [convert(shp) for shp in shapes] + context.set_context(mode=context.GRAPH_MODE, device_target="GPU") + # pylint: disable=unused-variable + ret, (hn, cn) = net(*inputs) + print(f'result: {shapes[0]} --> {ret.asnumpy().shape}, expected: {config["output_shape"][0]}') diff --git a/tests/ut/python/nn/test_nn_embedding.py b/tests/ut/python/nn/test_nn_embedding.py new file mode 100755 index 00000000000..5be91aee218 --- /dev/null +++ b/tests/ut/python/nn/test_nn_embedding.py @@ -0,0 +1,47 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test nn embedding """ +import numpy as np +from mindspore import Tensor +from mindspore.common import dtype +from mindspore.nn import Embedding +from mindspore.common.api import _executor +from ..ut_filter import non_graph_engine + + +@non_graph_engine +def test_check_embedding_1(): + net = Embedding(20000, 768, False) + input_data = Tensor(np.ones([8, 128]), dtype.int32) + _executor.compile(net, input_data) + + +@non_graph_engine +def test_check_embedding_2(): + net = Embedding(20000, 768, True) + input_data = Tensor(np.ones([8, 128]), dtype.int32) + _executor.compile(net, input_data) + + +@non_graph_engine +def test_check_embedding_3(): + net = Embedding(20000, 768, True, "zeros") + input_data = Tensor(np.ones([8, 128]), dtype.int32) + _executor.compile(net, input_data) + +@non_graph_engine +def test_print_embedding(): + net = Embedding(20000, 768, False) + print(net) diff --git a/tests/ut/python/nn/test_norm.py b/tests/ut/python/nn/test_norm.py new file mode 100644 index 00000000000..72d18962964 --- /dev/null +++ b/tests/ut/python/nn/test_norm.py @@ -0,0 +1,36 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test norm """ +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common.api import _executor +from ..ut_filter import non_graph_engine + + +class NormNet(nn.Cell): + def __init__(self): + super(NormNet, self).__init__() + self.norm = nn.Norm() + + def construct(self, x): + return self.norm(x) + + +@non_graph_engine +def test_compile_norm(): + net = NormNet() + x = Tensor(np.array([2.0, 1.0])) + _executor.compile(net, x) diff --git a/tests/ut/python/nn/test_parameter.py b/tests/ut/python/nn/test_parameter.py new file mode 100644 index 00000000000..49e89e124e3 --- /dev/null +++ b/tests/ut/python/nn/test_parameter.py @@ -0,0 +1,152 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test parameter """ +import numpy as np +import pytest +from mindspore import Tensor, Parameter, ParameterTuple +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer +from mindspore._checkparam import _check_str_by_regular + + +def test_parameter_init(): + dat = np.array([[1, 2, 3], [2, 3, 4]]) + tensor = Tensor(dat) + Parameter(tensor, name="testParameter", requires_grad=True, layerwise_parallel=False) + + +def test_parameter_tuple_illegal(): + p1 = Parameter(initializer(0, [1], mstype.int32), name="global_step1") + p2 = Parameter(initializer(0, [1], mstype.int32), name="global_step2") + plist = [p1,p2] + plist2 = [p1, "str"] + ptuple = (p1, p2) + ptuple_str = ("2", "1") + pstr = "[2,3]" + pnum = 3 + + ParameterTuple(plist) + ParameterTuple(ptuple) + with pytest.raises(TypeError): + ParameterTuple(p1) + with pytest.raises(ValueError): + ParameterTuple(plist2) + with pytest.raises(ValueError): + ParameterTuple(ptuple_str) + with pytest.raises(ValueError): + ParameterTuple(pstr) + with pytest.raises(TypeError): + ParameterTuple(pnum) + + +def test_parameter_init_illegal(): + import numpy as np + dat = np.array([[1, 2, 3], [2, 3, 4]]) + tensor = Tensor(dat) + data_none = None + data_bool = True + data_str = "nicai" + data_int = 3 + data_list = [1, "2", True] + data_tuple = (1, 2, 3) + np_arr_int16 = np.ones([1,1], dtype=np.int16) + np_arr_int32 = np.ones([1,1], dtype=np.int32) + np_arr_float16 = np.ones([1,1], dtype=np.float16) + np_arr_float32 = np.ones([1,1], dtype=np.float32) + +# with pytest.raises(ValueError): +# Parameter(np_arr_int16[0][0], name=data_str) + Parameter(np_arr_int32[0], name=data_str) + Parameter(np_arr_float16[0], name=data_str) + Parameter(np_arr_float32[0], name=data_str) + Parameter(np_arr_float32, name=data_str) + + Parameter(tensor, name=data_str) + Parameter(data_int, name=data_str) + Parameter(dat, name=data_str) + with pytest.raises(ValueError): + Parameter(data_none, name=data_str) + with pytest.raises(ValueError): + Parameter(data_bool, name=data_str) + with pytest.raises(ValueError): + Parameter(data_str, name=data_str) + Parameter(data_list, name=data_str) + with pytest.raises(ValueError): + Parameter(data_tuple, name=data_str) + + Parameter(tensor, name=data_str) + Parameter(tensor, name=data_none) + with pytest.raises(ValueError): + Parameter(tensor, name=dat) + with pytest.raises(ValueError): + Parameter(tensor, name=tensor) + with pytest.raises(ValueError): + Parameter(tensor, name=data_bool) + with pytest.raises(ValueError): + Parameter(tensor, name=data_int) + with pytest.raises(ValueError): + Parameter(tensor, name=data_list) + with pytest.raises(ValueError): + Parameter(tensor, name=data_tuple) + + Parameter(tensor, name=data_str, requires_grad=data_bool) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_none) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=dat) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=tensor) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_str) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_int) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_list) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_tuple) + + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=data_bool) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=dat) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=tensor) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=data_none) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=data_str) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=data_int) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=data_list) + with pytest.raises(TypeError): + Parameter(tensor, name=data_str, requires_grad=data_bool,layerwise_parallel=data_tuple) + + +def test_check_str_by_regular(): + str1 = "12_sf.asdf_" + str2 = "x12_sf.asdf." + str3 = "_x12_sf.asdf" + str4 = ".12_sf.asdf" + str5 = "12_sf.a$sdf." + str6 = "12+sf.asdf" + _check_str_by_regular(str1) + _check_str_by_regular(str2) + _check_str_by_regular(str3) + with pytest.raises(ValueError): + _check_str_by_regular(str4) + with pytest.raises(ValueError): + _check_str_by_regular(str5) + with pytest.raises(ValueError): + _check_str_by_regular(str6) diff --git a/tests/ut/python/nn/test_pooling.py b/tests/ut/python/nn/test_pooling.py new file mode 100644 index 00000000000..694d202d13a --- /dev/null +++ b/tests/ut/python/nn/test_pooling.py @@ -0,0 +1,59 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test pooling api +""" +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor + + +class AvgNet(nn.Cell): + def __init__(self, + kernel_size, + stride=None): + super(AvgNet, self).__init__() + self.avgpool = nn.AvgPool2d(kernel_size, stride) + + def construct(self, x): + return self.avgpool(x) + + +def test_compile_avg(): + net = AvgNet(3, 1) + x = Tensor(np.ones([1, 3, 16, 50]).astype(np.float32)) + _executor.compile(net, x) + + +class MaxNet(nn.Cell): + """ MaxNet definition """ + def __init__(self, + kernel_size, + stride=None, + padding=0): + super(MaxNet, self).__init__() + self.maxpool = nn.MaxPool2d(kernel_size, + stride, + padding=padding) + + def construct(self, x): + return self.maxpool(x) + + +def test_compile_max(): + net = MaxNet(3, stride=1, padding=0) + x = Tensor(np.random.randint(0, 255, [1, 3, 6, 6]).astype(np.float32)) + _executor.compile(net, x) diff --git a/tests/ut/python/nn/test_structure_output.py b/tests/ut/python/nn/test_structure_output.py new file mode 100644 index 00000000000..eb2722878aa --- /dev/null +++ b/tests/ut/python/nn/test_structure_output.py @@ -0,0 +1,253 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test_structure_output +""" +import numpy as np + +from mindspore import Tensor, context +from mindspore.nn import Cell +import mindspore.ops.operations as P +from mindspore.ops.functional import depend + +context.set_context(mode=context.GRAPH_MODE) + + +def test_output_const_tuple(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.tuple_1 = (1, 2, 3) + self.tuple_2 = (4, 5, 6) + + def construct(self): + ret = self.tuple_1 + self.tuple_2 + return ret + + net = Net() + assert net() == (1, 2, 3, 4, 5, 6) + + +def test_output_const_list(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.tuple_1 = [1, 2, 3] + + def construct(self): + ret = self.tuple_1 + return ret + + net = Net() + assert net() == (1, 2, 3) + + +def test_output_const_int(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.number_1 = 2 + self.number_2 = 3 + + def construct(self): + ret = self.number_1 + self.number_2 + return ret + + net = Net() + assert net() == 5 + + +def test_output_const_str(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.str = "hello world" + + def construct(self): + ret = self.str + return ret + + net = Net() + assert net() == "hello world" + + +def test_output_parameter_tuple(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x): + ret = x + return ret + + x = (1, 2, 3) + net = Net() + assert net(x) == x + + +def test_output_parameter_list(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x): + ret = x + return ret + + x = [1, 2, 3] + net = Net() + assert net(x) == x + + +def test_output_parameter_int(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x): + ret = x + return ret + + x = 88 + net = Net() + assert net(x) == x + + +def test_output_parameter_str(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x): + ret = x + return ret + + x = "hello world" + net = Net() + assert net(x) == x + + +def test_tuple_tuple_0(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + self.sub = P.Sub() + + def construct(self, x, y): + xx = self.add(x, x) + yy = self.add(y, y) + xxx = self.sub(x, x) + yyy = self.sub(y, y) + ret = ((xx, yy), (xxx, yyy)) + ret = (ret, ret) + return ret + + net = Net() + x = Tensor(np.ones([2], np.int32)) + y = Tensor(np.zeros([3], np.int32)) + net(x, y) + + +def test_tuple_tuple_1(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + self.sub = P.Sub() + + def construct(self, x, y): + xx = self.add(x, x) + yy = self.add(y, y) + ret = ((xx, yy), x) + ret = (ret, ret) + return ret + + net = Net() + x = Tensor(np.ones([2], np.int32)) + y = Tensor(np.zeros([3], np.int32)) + net(x, y) + + +def test_tuple_tuple_2(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + self.sub = P.Sub() + self.relu = P.ReLU() + self.depend = depend + + def construct(self, x, y): + xx = self.add(x, x) + yy = self.add(y, y) + xxx = self.sub(x, x) + yyy = self.sub(y, y) + z = self.relu(x) + ret = ((xx, yy), (xxx, yyy)) + ret = (ret, ret) + ret = self.depend(ret, z) + return ret + + net = Net() + x = Tensor(np.ones([2], np.int32)) + y = Tensor(np.zeros([3], np.int32)) + net(x, y) + + +def test_tuple_tuple_3(): + class Net(Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + self.sub = P.Sub() + self.relu = P.ReLU() + self.depend = depend + + def construct(self, x, y): + xx = self.add(x, x) + yy = self.add(y, y) + z = self.relu(x) + ret = ((xx, yy), x) + ret = (ret, ret) + ret = self.depend(ret, z) + return ret + + net = Net() + x = Tensor(np.ones([2], np.int32)) + y = Tensor(np.zeros([3], np.int32)) + net(x, y) + + +def test_soft(): + class SoftmaxCrossEntropyWithLogitsNet(Cell): + def __init__(self): + super(SoftmaxCrossEntropyWithLogitsNet, self).__init__() + self.soft = P.SoftmaxCrossEntropyWithLogits() + self.value = (Tensor(np.zeros((2,)).astype(np.float32)), Tensor(np.ones((2,)).astype(np.float32))) + + def construct(self, x, y, z): + xx = x + y + yy = x - y + ret = self.soft(xx, yy) + ret = (ret, z) + ret = (ret, self.value) + return ret + + input1 = Tensor(np.zeros((2,)).astype(np.float32)) + input2 = Tensor(np.ones((2,)).astype(np.float32)) + input3 = Tensor((np.ones((2,)) + np.ones((2,))).astype(np.float32)) + net = SoftmaxCrossEntropyWithLogitsNet() + print(net(input1, input2, input3)) diff --git a/tests/ut/python/ops/__init__.py b/tests/ut/python/ops/__init__.py new file mode 100644 index 00000000000..5443c0ca48e --- /dev/null +++ b/tests/ut/python/ops/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup for pytest""" +import mindspore.context as context + +# pylint: disable=unused-argument +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) diff --git a/tests/ut/python/ops/test_array_ops.py b/tests/ut/python/ops/test_array_ops.py new file mode 100644 index 00000000000..4164ce6e8f2 --- /dev/null +++ b/tests/ut/python/ops/test_array_ops.py @@ -0,0 +1,258 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test array ops """ +import functools +import numpy as np +import mindspore as ms +from mindspore import Tensor +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.ops import prim_attr_register +from mindspore.ops.primitive import Primitive, PrimitiveWithInfer +from mindspore.common.dtype import get_py_obj_dtype +from mindspore._c_expression import signature_dtype as sig_dtype +from mindspore._c_expression import signature_rw as sig_rw +from mindspore._c_expression import signature_kind as sig_kind + +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.forward.verify_exception \ + import pipeline_for_verify_exception_for_case_by_case_config + + +def test_expand_dims(): + input_tensor = Tensor(np.array([[2, 2], [2, 2]])) + expand_dims = P.ExpandDims() + output = expand_dims(input_tensor, 0) + assert output.asnumpy().shape == (1, 2, 2) + + +def test_cast(): + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_x = Tensor(input_np) + td = ms.int32 + cast = P.Cast() + result = cast(input_x, td) + expect = input_np.astype(np.int32) + assert np.all(result.asnumpy() == expect) + + +@non_graph_engine +def test_reshape(): + input_tensor = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]])) + shp = (3, 2) + reshape = P.Reshape() + output = reshape(input_tensor, shp) + assert output.asnumpy().shape == (3, 2) + + +def test_transpose(): + input_tensor = Tensor(np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])) + perm = (0, 2, 1) + expect = np.array([[[1, 4], [2, 5], [3, 6]], [[7, 10], [8, 11], [9, 12]]]) + + transpose = P.Transpose() + output = transpose(input_tensor, perm) + assert np.all(output.asnumpy() == expect) + + +def test_squeeze(): + input_tensor = Tensor(np.ones(shape=[3, 2, 1])) + squeeze = P.Squeeze(2) + output = squeeze(input_tensor) + assert output.asnumpy().shape == (3, 2) + + +def test_invert_permutation(): + invert_permutation = P.InvertPermutation() + x = (3, 4, 0, 2, 1) + output = invert_permutation(x) + expect = (2, 4, 3, 0, 1) + assert np.all(output == expect) + + +def test_select(): + select = P.Select() + cond = Tensor(np.array([[True, False, False], [False, True, True]])) + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]])) + y = Tensor(np.array([[7, 8, 9], [10, 11, 12]])) + output = select(cond, x, y) + expect = np.array([[1, 8, 9], [10, 5, 6]]) + assert np.all(output.asnumpy() == expect) + + +def test_scalar_cast_grad(): + """ test_scalar_cast_grad """ + input_x = 255.5 + input_t = get_py_obj_dtype(ms.int8) + + def fx_cast(x): + output = F.scalar_cast(x, input_t) + return output + + gfn = C.grad(fx_cast)(input_x) + expect_dx = 1 + assert gfn == expect_dx + + +class CustomOP(PrimitiveWithInfer): + __mindspore_signature__ = (sig_dtype.T, sig_dtype.T, sig_dtype.T1, + sig_dtype.T1, sig_dtype.T2, sig_dtype.T2, + sig_dtype.T2, sig_dtype.T3, sig_dtype.T4) + + @prim_attr_register + def __init__(self): + pass + + def __call__(self, p1, p2, p3, p4, p5, p6, p7, p8, p9): + raise NotImplementedError + + +class CustomOP2(PrimitiveWithInfer): + __mindspore_signature__ = ( + ('p1', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('p2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('p3', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ) + + @prim_attr_register + def __init__(self): + pass + + def __call__(self, p1, p2, p3): + raise NotImplementedError + + +class CustNet1(Cell): + def __init__(self): + super(CustNet1, self).__init__() + self.op = CustomOP() + self.t1 = Tensor(np.ones([2, 2]), dtype=ms.int32) + self.t2 = Tensor(np.ones([1, 5]), dtype=ms.float16) + self.int1 = 3 + self.float1 = 5.1 + + def construct(self): + x =self.op(self.t1, self.t1, self.int1, + self.float1, self.int1, self.float1, + self.t2, self.t1, self.int1) + return x + + +class CustNet2(Cell): + def __init__(self): + super(CustNet2, self).__init__() + self.op = CustomOP2() + self.t1 = Tensor(np.ones([2, 2]), dtype=ms.int32) + self.t2 = Tensor(np.ones([1, 5]), dtype=ms.float16) + self.int1 = 3 + + def construct(self): + return self.op(self.t1, self.t2, self.int1) + + +class CustNet3(Cell): + def __init__(self): + super(CustNet3, self).__init__() + self.op = P.ReduceSum() + self.t1 = Tensor(np.ones([2, 2]), dtype=ms.int32) + self.t2 = Tensor(np.ones([1, 5]), dtype=ms.float16) + self.t2 = 1 + + def construct(self): + return self.op(self.t1, self.t2) + + +class MathBinaryNet1(Cell): + def __init__(self): + super(MathBinaryNet1, self).__init__() + self.add = P.TensorAdd() + self.mul = P.Mul() + self.max = P.Maximum() + self.number = 3 + + def construct(self, x): + return self.add(x, self.number) + self.mul(x, self.number) + self.max(x, self.number) + + +class MathBinaryNet2(Cell): + def __init__(self): + super(MathBinaryNet2, self).__init__() + self.less_equal = P.LessEqual() + self.greater = P.Greater() + self.logic_or = P.LogicalOr() + self.logic_and = P.LogicalAnd() + self.number = 3 + self.flag = True + + def construct(self, x): + ret_less_equal = self.logic_and(self.less_equal(x, self.number), self.flag) + ret_greater = self.logic_or(self.greater(x, self.number), self.flag) + return self.logic_or(ret_less_equal, ret_greater) + + +test_case_array_ops = [ + ('CustNet1', { + 'block': CustNet1(), + 'desc_inputs': []}), + ('CustNet2', { + 'block': CustNet2(), + 'desc_inputs': []}), + ('CustNet3', { + 'block': CustNet3(), + 'desc_inputs': []}), + ('MathBinaryNet1', { + 'block': MathBinaryNet1(), + 'desc_inputs': [Tensor(np.ones([2, 2]), dtype=ms.int32)]}), + ('MathBinaryNet2', { + 'block': MathBinaryNet2(), + 'desc_inputs': [Tensor(np.ones([2, 2]), dtype=ms.int32)]}), +] + +test_case_lists = [test_case_array_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case + + +raise_set = [ + ('Squeeze_1_Error', { + 'block': (lambda x: P.Squeeze(axis=1.2), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5]))]}), + ('Squeeze_2_Error', { + 'block': (lambda x: P.Squeeze(axis=((1.2, 1.3))), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5]))]}), + ('ReduceSum_Error', { + 'block': (lambda x: P.ReduceSum(keep_dims=1), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5]))]}), +] + + +@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config) +def test_check_exception(): + return raise_set diff --git a/tests/ut/python/ops/test_bprop_disorder.py b/tests/ut/python/ops/test_bprop_disorder.py new file mode 100644 index 00000000000..fad07de2cb6 --- /dev/null +++ b/tests/ut/python/ops/test_bprop_disorder.py @@ -0,0 +1,94 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test bprop disorder """ +import functools +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.common.parameter import ParameterTuple +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config + + +class DisOrderTest1(nn.Cell): + """ DisOrderTest1 definition """ + def __init__(self): + super(DisOrderTest1, self).__init__() + weight = Tensor(np.ones([1], np.float32)) + self.s1 = Parameter(weight, name="s1") + self.s2 = Parameter(weight, name="s2") + self.s3 = Parameter(weight, name="s3") + self.s4 = Parameter(weight, name="s4") + self.mul = P.Mul() + self.add = P.TensorAdd() + + def construct(self, x): + return x * (self.s1 * self.s2 + self.s2 * self.s3 + self.s3 * self.s4 + self.s4 * self.s1) + + +class DisOrderTest2(nn.Cell): + """ DisOrderTest2 definition """ + def __init__(self): + super(DisOrderTest2, self).__init__() + weight = Tensor(np.ones([1], np.float32)) + self.s1 = Parameter(weight, name="s1") + self.s2 = Parameter(weight, name="s2") + self.s3 = Parameter(weight, name="s3") + self.s4 = Parameter(weight, name="s4") + self.mul = P.Mul() + self.add = P.TensorAdd() + + def construct(self, x): + return self.mul(x, (self.add(self.add(self.add(self.mul(self.s1, self.s2), self.mul(self.s2, self.s3)), + self.mul(self.s3, self.s4)), self.mul(self.s4, self.s1)))) + + +class GradNetWrap(nn.Cell): + """ GradNetWrap definition """ + def __init__(self, net): + super(GradNetWrap, self).__init__() + self.net = net + self.weights = ParameterTuple(net.get_parameters()) + + def construct(self, x, sens): + return C.grad_by_list_with_sens(self.net, self.weights)(x, sens) + + +test_case_ops = [ + ('DisOrderTest1', { + 'block': GradNetWrap(DisOrderTest1()), + 'desc_inputs': [Tensor(np.ones([1], np.float32)), Tensor(np.ones([1], np.float32))]}), + ('DisOrderTest2', { + 'block': GradNetWrap(DisOrderTest2()), + 'desc_inputs': [Tensor(np.ones([1], np.float32)), Tensor(np.ones([1], np.float32))]}), +] + +test_case_lists = [test_case_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case diff --git a/tests/ut/python/ops/test_control_ops.py b/tests/ut/python/ops/test_control_ops.py new file mode 100644 index 00000000000..b17eea8ddda --- /dev/null +++ b/tests/ut/python/ops/test_control_ops.py @@ -0,0 +1,360 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test control ops """ +import numpy as np +import mindspore as ms +from mindspore import nn +from mindspore import Tensor +from mindspore import context +from mindspore.ops import operations as P + +context.set_context(mode=context.GRAPH_MODE) + + +def cond_data_test(x_init, y_init): + class Net(nn.Cell): + def __init__(self): + """""" + super(Net, self).__init__() + self.square = P.Square() + self.add = P.TensorAdd() + self.value = Tensor(np.full((1), 3, dtype=np.float32)) + self.switch = P.GeSwitch() + self.merge = P.Merge() + self.less = P.Less() + + def construct(self, x, y): + cond = self.less(x, y) + st1, sf1 = self.switch(x, cond) + st2, sf2 = self.switch(y, cond) + add_ret = self.add(st1, st2) + st3, sf3 = self.switch(self.value, cond) + sq_ret = self.square(sf3) + ret = self.merge((add_ret, sq_ret)) + return ret[0] + + x = Tensor(x_init, dtype=ms.float32) + y = Tensor(y_init, dtype=ms.float32) + net = Net() + output = net(x, y) + return output + + +def test_cond_data_true(): + output = cond_data_test(3, 8) + print("test_cond_data_true:", output) + +def test_cond_data_false(): + output = cond_data_test(8, 3) + print("test_cond_data_false:", output) + +def if_compile_test(x_init, y_init): + class Net(nn.Cell): + def __init__(self): + """""" + super(Net, self).__init__() + self.square = P.Square() + self.add = P.TensorAdd() + self.value = Tensor(3, dtype=ms.float32) + self.switch = P.GeSwitch() + self.merge = P.Merge() + self.less = P.Less() + + def construct(self, x, y): + cond = self.less(x, y) + ret = self.value + if cond: + ret = self.add(x, ret) + ret = self.add(y, ret) + else: + ret = self.square(self.value) + return ret + + x = Tensor(x_init, dtype=ms.float32) + y = Tensor(y_init, dtype=ms.float32) + net = Net() + output = net(x, y) + return output + + +def test_if_none(): + class Net(nn.Cell): + def __init__(self, z: None): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = None + net = Net(z) + assert net(x, y) == y + + +def test_if_str_is_not_none_right(): + class Net(nn.Cell): + def __init__(self, z: str): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z == None: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = "ok" + net = Net(z) + assert net(x, y) == y + + +def test_if_str_is_not_none_left(): + class Net(nn.Cell): + def __init__(self, z: str): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if None == self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = "ok" + net = Net(z) + assert net(x, y) == y + + +def test_if_none_equal_none(): + class Net(nn.Cell): + def __init__(self, z: None): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z == None: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = None + net = Net(z) + assert net(x, y) == x + + +def test_if_str_is_null(): + class Net(nn.Cell): + def __init__(self, z: str): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = "" + net = Net(z) + assert net(x, y) == y + + +def test_if_str_is_true(): + class Net(nn.Cell): + def __init__(self, z: str): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 9, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = "ok" + net = Net(z) + assert net(x, y) == x + + +def test_if_str_equal(): + class Net(nn.Cell): + def __init__(self, z: str): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z == "ok": + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = "ok" + net = Net(z) + assert net(x, y) == x + + +def test_if_tuple_is_null(): + class Net(nn.Cell): + def __init__(self, z: tuple): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = () + net = Net(z) + assert net(x, y) == y + + +def test_if_tuple_is_not_null(): + class Net(nn.Cell): + def __init__(self, z: tuple): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = (1, 2, 3) + net = Net(z) + assert net(x, y) == x + + +def test_if_dict_is_null(): + class Net(nn.Cell): + def __init__(self, z: dict): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = {} + net = Net(z) + assert net(x, y) == y + + +def test_if_dict_is_not_null(): + class Net(nn.Cell): + def __init__(self, z: dict): + """""" + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = {"one": 1, "two": 2} + net = Net(z) + assert net(x, y) == x + + +def test_if_else_assign(): + class Net(nn.Cell): + def __init__(self, m: list): + """""" + super(Net, self).__init__() + self.m = m + self.n = [4, 5, 6] + + def construct(self, x, y): + exp_1 = self.m if self.m else self.n + exp_2 = self.m if exp_1 == self.n else self.n + if exp_2 == self.m: + if self.m: + ret = x + else: + ret = y + else: + if self.m: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = [1, 2] + net = Net(z) + assert net(x, y) == x + + +def test_if_compile_true(): + output = if_compile_test(3, 8) + print("test_if_compile_true:", output) + + +def test_if_compile_false(): + output = if_compile_test(8, 3) + print("test_if_compile_false:", output) diff --git a/tests/ut/python/ops/test_list.py b/tests/ut/python/ops/test_list.py new file mode 100644 index 00000000000..a961148ca86 --- /dev/null +++ b/tests/ut/python/ops/test_list.py @@ -0,0 +1,219 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import functools +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import operations as P +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config + + +def test_list_equal(): + class Net(nn.Cell): + def __init__(self, z: list): + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z == [1, 2, 3]: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = [1, 2, 3] + net = Net(z) + assert net(x, y) == x + + +def test_list_not_equal(): + class Net(nn.Cell): + def __init__(self, z: list): + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + if self.z == [3, 4, 5]: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = [1, 2, 3] + net = Net(z) + assert net(x, y) == y + + +def test_list_expansion(): + class Net(nn.Cell): + def __init__(self, z: list): + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + a, b, c = self.z + if a == 1 and b == 2 and c == 3: + ret = x + else: + ret = y + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = [1, 2, 3] + net = Net(z) + assert net(x, y) == x + + +def test_list_append(): + class Net(nn.Cell): + def __init__(self, z: list): + super(Net, self).__init__() + self.z = z + + def construct(self, x, y): + z = [[1, 2], 3] + z[0].append(88) + z[0].append(99) + if z[0][3] == 99: + ret = y + else: + ret = x + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = [1, 2, 3] + net = Net(z) + assert net(x, y) == y + + +def test_list_append_2(): + class Net(nn.Cell): + def __init__(self, z: list): + super(Net, self).__init__() + self.z = z + self.x = 9 + + def construct(self, x, y): + self.z[0].append(88) + self.z[0].append(99) + if self.z[0][3] == 88: + ret = y + else: + ret = x + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + y = Tensor(np.zeros([3, 4, 5], np.int32)) + z = [[1, 2], 3] + net = Net(z) + assert net(x, y) == x + + +class ListOperate(nn.Cell): + def __init__(self, ): + super(ListOperate, self).__init__() + + def construct(self, t, l): + x = [1, 2, 3, 4, 5, 6] + x[2] = 9 + x[1] = x[3] + 11 + x[3] = x[1] + x[0] + x[0] = x[2] * x[4] + x[5] = x[1] - x[2] + x[4] = x[3] / x[2] + x.append(8) + x.append(8) + x.append(t) + x.append(l) + x.append(l) + return x + + +class AxisListNet(nn.Cell): + def __init__(self): + super(AxisListNet, self).__init__() + self.reduce_sum = P.ReduceSum() + self.reduce_mean = P.ReduceMean() + self.reduce_max = P.ReduceMax() + self.reduce_min = P.ReduceMin() + self.add_n = P.AddN() + self.axis = [0, 1, 2] + + def construct(self, x): + ret_sum = self.reduce_sum(x, self.axis) + ret_mean = self.reduce_mean(x, self.axis) + ret_max = self.reduce_max(x, self.axis) + ret_min = self.reduce_min(x, self.axis) + ret = [ret_sum, ret_mean, ret_max, ret_min] + return self.add_n(ret) + ret_sum + + +class AxisListEmptyNet(nn.Cell): + def __init__(self): + super(AxisListEmptyNet, self).__init__() + self.reduce_sum = P.ReduceSum() + self.axis = [] + + def construct(self, x): + return self.reduce_sum(x, self.axis) + + +class AxisListDefaultNet(nn.Cell): + def __init__(self): + super(AxisListDefaultNet, self).__init__() + self.reduce_sum = P.ReduceSum() + + def construct(self, x): + return self.reduce_sum(x) + + +test_case_ops = [ + ('ListOperate', { + 'block': ListOperate(), + 'desc_inputs': [Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)), + [2, 3, 4]]}), + ('AxisList', { + 'block': AxisListNet(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))]}), + ('AxisListEmpty', { + 'block': AxisListEmptyNet(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))]}), + ('AxisListDefault', { + 'block': AxisListDefaultNet(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))]}), +] + +test_case_lists = [test_case_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case diff --git a/tests/ut/python/ops/test_math_ops.py b/tests/ut/python/ops/test_math_ops.py new file mode 100755 index 00000000000..7c0cca9b400 --- /dev/null +++ b/tests/ut/python/ops/test_math_ops.py @@ -0,0 +1,387 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test math ops """ +import functools +import numpy as np +import mindspore as ms +import mindspore.nn as nn +from mindspore.common import dtype as mstype +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.ops import functional as F +import mindspore.context as context +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.forward.verify_exception \ + import pipeline_for_verify_exception_for_case_by_case_config +# pylint: disable=W0613 +# pylint: disable=W0231 +# W0613: unused-argument +# W0231: super-init-not-called + +def test_multiply(): + """ test_multiply """ + input_x = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]])) + input_y = Tensor(np.array([[0.1, 0.3, -3.6], [0.4, 0.5, -3.2]])) + + mul = P.Mul() + result = mul(input_x, input_y) + expect = np.array([[-0.01, 0.09, -12.96], [0.16, 0.25, 10.24]]) + diff = result.asnumpy() - expect + error = np.ones(shape=[2, 3]) * 1.0e-6 + assert np.all(diff < error) + assert np.all(-diff < error) + + +def test_sub(): + """ test_sub """ + input_x = Tensor(np.ones(shape=[3])) + input_y = Tensor(np.zeros(shape=[3])) + + sub = P.Sub() + result = sub(input_x, input_y) + expect = np.ones(shape=[3]) + assert np.all(result.asnumpy() == expect) + + +def test_square(): + """ test_square """ + input_tensor = Tensor(np.array([[1, 2, 3], [4, 5, 6]])) + square = P.Square() + result = square(input_tensor) + expect = np.array([[1, 4, 9], [16, 25, 36]]) + assert np.all(result.asnumpy() == expect) + + +def test_sqrt(): + """ test_sqrt """ + input_tensor = Tensor(np.array([[4, 4], [9, 9]])) + + sqrt = P.Sqrt() + expect = np.array([[2, 2], [3, 3]]) + result = sqrt(input_tensor) + assert np.all(result.asnumpy() == expect) + + +def test_pow(): + """ test_pow """ + input_tensor = Tensor(np.array([[2, 2], [3, 3]])) + testpow = P.Pow() + expect = np.array([[8, 8], [27, 27]]) + result = testpow(input_tensor, 3.0) + assert np.all(result.asnumpy() == expect) + + +def test_exp(): + """ test_exp """ + input_tensor = Tensor(np.array([[2, 2], [3, 3]])) + testexp = P.Exp() + result = testexp(input_tensor) + expect = np.exp(np.array([[2, 2], [3, 3]])) + assert np.all(result.asnumpy() == expect) + + +def test_realdiv(): + """ test_realdiv """ + x = Tensor(2048.0) + y = Tensor(128.0) + div = P.RealDiv() + result = div(x, y) + x = x.asnumpy() + y = y.asnumpy() + expect = x/y + assert np.all(result.asnumpy() == expect) + + +def test_eye(): + """ test_eye """ + x = np.arange(3) + expect = np.ones_like(x) + expect = np.diag(expect) + eye = P.Eye() + eye_output = eye(3, 3, ms.float32) + assert np.all(eye_output.asnumpy() == expect) + + +class VirtualLossGrad(PrimitiveWithInfer): + """ VirtualLossGrad definition """ + @prim_attr_register + def __init__(self): + """init VirtualLossGrad""" + + def __call__(self, x, out, dout): + raise NotImplementedError + + def infer_shape(self, x_shape, out_shape, dout_shape): + return x_shape + + def infer_dtype(self, x_dtype, out_dtype, dout_dtype): + return x_dtype + + +class VirtualLoss(PrimitiveWithInfer): + """ VirtualLoss definition """ + @prim_attr_register + def __init__(self): + """init VirtualLoss""" + + def __call__(self, x): + raise NotImplementedError + + def get_bprop(self): + loss_grad = VirtualLossGrad() + + def bprop(x, out, dout): + dx = loss_grad(x, out, dout) + return (dx,) + return bprop + + def infer_shape(self, x_shape): + return [1] + + def infer_dtype(self, x_dtype): + return x_dtype + + +class NetWithLoss(nn.Cell): + """ NetWithLoss definition """ + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad(self.network)(x, y, b) + + +class MatMulNet(nn.Cell): + """ MatMulNet definition """ + def __init__(self): + super(MatMulNet, self).__init__() + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x, y, b): + return self.biasAdd(self.matmul(x, y), b) + + +class NetWithLossSub(nn.Cell): + """ NetWithLossSub definition """ + def __init__(self, network): + super(NetWithLossSub, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y): + predict = self.network(x, y) + return self.loss(predict) + + +class GradWrapSub(nn.Cell): + """ GradWrapSub definition """ + def __init__(self, network): + super(GradWrapSub, self).__init__() + self.network = network + + def construct(self, x, y): + return C.grad(self.network)(x, y) + + +class SubNet(nn.Cell): + """ SubNet definition """ + def __init__(self): + super(SubNet, self).__init__() + self.sub = P.Sub() + + def construct(self, x, y): + return self.sub(x, y) + + +class NpuFloatNet(nn.Cell): + """ NpuFloat definition """ + def __init__(self): + super(NpuFloatNet, self).__init__() + self.mul = P.Mul() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_status = P.NPUClearFloatStatus() + self.fill = P.Fill() + self.shape_op = P.Shape() + self.select = P.Select() + self.less = P.Less() + self.cast = P.Cast() + self.dtype = P.DType() + self.reduce_sum = P.ReduceSum(keep_dims=True) + self.sub = P.Sub() + self.neg = P.Neg() + self.add_flags(has_effect=True) + + def construct(self, x): + init = self.alloc_status() + self.clear_status(init) + res = self.sub(x, self.neg(x)) + self.get_status(init) + flag_sum = self.reduce_sum(init, (0,)) + base = self.cast(self.fill(self.dtype(res), self.shape_op(res), 0.0), self.dtype(flag_sum)) + cond = self.less(base, flag_sum) + out = self.select(cond, self.cast(base, self.dtype(res)), res) + return out + + +class DiagNet(nn.Cell): + """ DiagNet definition """ + def __init__(self): + super(DiagNet, self).__init__() + self.fill = P.Fill() + self.diag = P.Diag() + + def construct(self, x): + return x - self.diag(self.fill(mstype.float32, (3,), 1.0)) + + +class NetWithLossCumSum(nn.Cell): + """ NetWithLossCumSum definition """ + def __init__(self, network): + super(NetWithLossCumSum, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, input): + predict = self.network(input) + return self.loss(predict) + + +class GradWrapCumSum(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(GradWrapCumSum, self).__init__() + self.network = network + + def construct(self, input): + return C.grad(self.network)(input) + + +class NetCumSum(nn.Cell): + """ NetCumSum definition """ + def __init__(self): + super(NetCumSum, self).__init__() + self.cumsum = P.CumSum() + self.axis = 1 + + def construct(self, input): + return self.cumsum(input, self.axis) + + +class SignNet(nn.Cell): + def __init__(self): + super(SignNet, self).__init__() + self.sign = P.Sign() + + def construct(self, x): + return self.sign(x) + + +test_case_math_ops = [ + ('MatMulGrad', { + 'block': GradWrap(NetWithLoss(MatMulNet())), + 'desc_inputs': [Tensor(np.ones([3, 3]).astype(np.int32)), + Tensor(np.ones([3, 3]).astype(np.int32)), + Tensor(np.ones([3]).astype(np.int32))], + 'desc_bprop': [Tensor(np.ones([3, 3]).astype(np.int32)), + Tensor(np.ones([3, 3]).astype(np.int32)), + Tensor(np.ones([3]).astype(np.int32))], + 'skip': ['backward']}), + ('CumSumGrad', { + 'block': GradWrapCumSum(NetWithLossCumSum(NetCumSum())), + 'desc_inputs': [Tensor(np.array([[3, 4, 6, 10],[1, 6, 7, 9],[4, 3, 8, 7],[1, 3, 7, 9]]).astype(np.float16))], + 'desc_bprop': [Tensor(np.array([[3, 4, 6, 10],[1, 6, 7, 9],[4, 3, 8, 7],[1, 3, 7, 9]]).astype(np.float16))], + 'skip': ['backward']}), + ('Diag', { + 'block': DiagNet(), + 'desc_inputs': [Tensor(np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]], np.float32))], + 'desc_bprop': [Tensor(np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]], np.float32))], + 'skip': ['backward']}), + ('SubBroadcast', { + 'block': GradWrapSub(NetWithLossSub(SubNet())), + 'desc_inputs': [Tensor(np.ones([5, 3])), Tensor(np.ones([8, 5, 3]))], + 'desc_bprop': [Tensor(np.array([[1, 1, 1], [2, 2, 2], [3, 3, 3]], np.float32))], + 'skip': ['backward']}), + ('NpuFloat_NotOverflow', { + 'block': NpuFloatNet(), + 'desc_inputs': [Tensor(np.full((8, 5, 3, 1), 655, dtype=np.float16), dtype=ms.float16)], + 'desc_bprop': [Tensor(np.full((8, 5, 3, 1), 655, dtype=np.float16), dtype=ms.float16)], + 'skip': ['backward']}), + ('NpuFloat_Overflow', { + 'block': NpuFloatNet(), + 'desc_inputs': [Tensor(np.full((8, 5, 3, 1), 65504, dtype=np.float16), dtype=ms.float16)], + 'desc_bprop': [Tensor(np.full((8, 5, 3, 1), 65504, dtype=np.float16), dtype=ms.float16)], + 'skip': ['backward']}), + ('Sign', { + 'block': SignNet(), + 'desc_inputs': [Tensor(np.array([[1., 0., -2.]], np.float32))], + 'desc_bprop': [Tensor(np.array([[1., 0., -2.]], np.float32))], + 'skip': ['backward']}), +] + + +test_case_lists = [test_case_math_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case + + +raise_set = [ + ('StridedSlice_1_Error', { + 'block': (lambda x : P.StridedSlice(begin_mask="1"), {'exception': ValueError}), + 'desc_inputs': [0]}), + ('StridedSlice_2_Error', { + 'block': (lambda x : P.StridedSlice(end_mask="1"), {'exception': ValueError}), + 'desc_inputs': [0]}), + ('StridedSlice_3_Error', { + 'block': (lambda x : P.StridedSlice(ellipsis_mask=1.1), {'exception': ValueError}), + 'desc_inputs': [0]}), + ('StridedSlice_4_Error', { + 'block': (lambda x : P.StridedSlice(new_axis_mask="1.1"), {'exception': ValueError}), + 'desc_inputs': [0]}), +] + + +@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config) +def test_check_exception(): + return raise_set diff --git a/tests/ut/python/ops/test_momentum.py b/tests/ut/python/ops/test_momentum.py new file mode 100644 index 00000000000..64b5a9af123 --- /dev/null +++ b/tests/ut/python/ops/test_momentum.py @@ -0,0 +1,130 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_momentum """ +import functools +import numpy as np +import mindspore.nn as nn +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore import Parameter, ParameterTuple, Tensor +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +# pylint: disable=W0613 +# W0613: unused-argument + + +run_opt = C.MultitypeFuncGraph("run_opt") + + +@run_opt.register("Function", "Int", "Number", "Number", + "Tensor", "Tensor", + "Tensor") +def tensor_run_opt(opt, iters, learning_rate, momentum, + gradient, variable, moment): + """ tensor_run_opt """ + success = True + new_weight = opt(gradient, moment, variable, + learning_rate, momentum) + success = F.depend(success, F.assign(variable, new_weight)) + return success + + +class OptimizerByMomentum(nn.Cell): + """ OptimizerByMomentum definition """ + def __init__(self, weights): + super(OptimizerByMomentum, self).__init__() + self.learning_rate = Parameter(0.1, name="learning_rate") + self.momentum = Parameter(0.05, name="momentum") + self.iter = Parameter(0, name="iter") + + self.weights = weights + self.moments = weights.clone(prefix="moments", init='zeros') + + self.hyper_map = C.HyperMap() + self.opt = P.ApplyMomentum() + + def construct(self, grads): + success = True + weights = self.weights + moments = self.moments + success = self.hyper_map(F.partial(run_opt, self.opt, self.iter, + self.learning_rate, self.momentum), + grads, weights, moments) + return success + + +class TrainStepWrap(nn.Cell): + """ TrainStepWrap definition """ + def __init__(self, network): + super(TrainStepWrap, self).__init__() + self.network = network + self.weights = ParameterTuple(network.get_parameters()) + self.optimizer = OptimizerByMomentum(self.weights) + self.hyper_map = C.HyperMap() + + def construct(self, x, label): + weights = self.weights + grads = C.grad_by_list(self.network, weights)(x, label) + return self.optimizer(grads) + + +class NetWithLossClass(nn.Cell): + """ NetWithLossClass definition """ + def __init__(self, network): + super(NetWithLossClass, self).__init__(auto_prefix=False) + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + return self.biasAdd(self.matmul(x, self.weight), self.bias) + + +test_case_ops = [ + ('Momentum', { + 'block': TrainStepWrap(NetWithLossClass(Net())), + 'desc_inputs': [Tensor(np.ones([1, 64]).astype(np.float32)), + Tensor(np.zeros([1, 10]).astype(np.float32))]}), +] + +test_case_lists = [test_case_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case diff --git a/tests/ut/python/ops/test_multitype_ops.py b/tests/ut/python/ops/test_multitype_ops.py new file mode 100644 index 00000000000..588143fb19e --- /dev/null +++ b/tests/ut/python/ops/test_multitype_ops.py @@ -0,0 +1,286 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""multitype_ops directory test case""" +from functools import partial, reduce +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.ops import functional as F, composite as C +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config + + +class ScalarAddScalar(nn.Cell): + """ ScalarAddScalar definition """ + def __init__(self, ): + super(ScalarAddScalar, self).__init__() + self.n1 = 1.2 + self.n2 = 1.3 + + def construct(self): + return self.n1 + self.n2 + + +class ScalarAddTensor1(nn.Cell): + """ ScalarAddTensor1 definition """ + def __init__(self, ): + super(ScalarAddTensor1, self).__init__() + self.t1 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.n1 = 1.2 + self.n2 = 1.3 + + def construct(self): + return self.n1 + self.t1 + + +class ScalarAddTensor2(nn.Cell): + """ ScalarAddTensor2 definition """ + def __init__(self, ): + super(ScalarAddTensor2, self).__init__() + self.t1 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.n1 = 1.2 + self.n2 = 1.3 + + def construct(self): + return self.n1 + self.n2 + self.t1 + + +class TensorAddScalar(nn.Cell): + """ TensorAddScalar definition """ + def __init__(self, ): + super(TensorAddScalar, self).__init__() + self.t1 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.n1 = 1.2 + + def construct(self): + return self.t1 + self.n1 + + +class ScalarTensorSub(nn.Cell): + """ ScalarTensorSub definition """ + def __init__(self, ): + super(ScalarTensorSub, self).__init__() + self.t1 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.n1 = 2.1 + + def construct(self): + # scalar - tensor + z = self.n1 - self.t1 + # tensor - scalar + z = z - self.n1 + return z + + +class ScalarTensorMul(nn.Cell): + """ ScalarTensorMul definition """ + def __init__(self, ): + super(ScalarTensorMul, self).__init__() + self.t1 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.n1 = 2.1 + + def construct(self): + # scalar - tensor + z = self.n1 * self.t1 + # tensor - scalar + z = z * self.n1 + return z + + +class ScalarTensorDiv(nn.Cell): + """ ScalarTensorDiv definition """ + def __init__(self, ): + super(ScalarTensorDiv, self).__init__() + self.t1 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.n1 = 2.1 + + def construct(self): + # scalar - tensor + z = self.n1 / self.t1 + # tensor - scalar + z = z / self.n1 + return z + + +class EqualClass(nn.Cell): + def __init__(self, x, y): + super(EqualClass, self).__init__() + self.n1 = x + self.n2 = y + + def construct(self): + if self.n1 == self.n2: + return self.n1 + else: + return self.n2 + + +grad_scale = C.MultitypeFuncGraph("grad_scale") + + +@grad_scale.register("Number", "Tensor") +def tensor_grad_scale(scale, grad): + """Get grad with scale.""" + if scale == 1.0: + return grad + return grad * F.scalar_to_array(scale) + + +class MapPartialNet(nn.Cell): + def __init__(self): + super(MapPartialNet, self).__init__() + self.reciprocal_scale = 1.2 + self.x1 = Tensor(np.ones([2, 1, 2, ], np.float32)) + self.x2 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + + def construct(self, x, y): + grads = (self.x1, self.x2, x, y) + grads = map(partial(grad_scale, self.reciprocal_scale), grads) + return grads + + +class ZipNet(nn.Cell): + def __init__(self): + super(ZipNet, self).__init__() + self.x1 = Tensor(np.ones([1, 2, 2, 1], np.float32)) + self.x2 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + + def construct(self, x, y): + t1 = (self.x1, self.x2, x, y) + t2 = (y, x, self.x2, self.x1) + t3 = zip(t1, t2) + return t3 + + +class UnZipNet(nn.Cell): + def __init__(self): + super(UnZipNet, self).__init__() + self.x1 = Tensor(np.ones([1, 2, 2, 1], np.float32)) + self.x2 = Tensor(np.ones([2, 1, 2, 2], np.float32)) + + def construct(self, x, y): + t1 = (self.x1, self.x2, x, y) + t2 = (y, x, self.x2, self.x1) + t3 = zip(t1, t2) + t4 = zip(*t3) + return t4 + + +class ScalarTensorOp2Cast(nn.Cell): + def __init__(self, ): + super(ScalarTensorOp2Cast, self).__init__() + self.f = 1.2 + self.t = Tensor(np.ones([2, 1, 2, 2], np.float16)) + + def construct(self): + a1 = self.f + self.t + a2 = self.t + self.f + a = a1 + a2 + b1 = self.f - self.t + b2 = self.t - self.f + b = b1 - b2 + c1 = self.f * self.t + c2 = self.t * self.f + c = c1 * c2 + d1 = self.t / self.f + d2 = self.f / self.t + d = d1 / d2 + x = a + b + y = c + d + z = x + y + return z + + +test_case_ops = [ + ('ScalarAddScalar', { + 'block': ScalarAddScalar(), + 'desc_inputs': []}), + ('ScalarAddTensor1', { + 'block': ScalarAddTensor1(), + 'desc_inputs': []}), + ('ScalarAddTensor2', { + 'block': ScalarAddTensor2(), + 'desc_inputs': []}), + ('TensorAddScalar', { + 'block': TensorAddScalar(), + 'desc_inputs': []}), + ('ScalarTensorSub', { + 'block': ScalarTensorSub(), + 'desc_inputs': []}), + ('ScalarTensorMul', { + 'block': ScalarTensorMul(), + 'desc_inputs': []}), + ('ScalarTensorDiv', { + 'block': ScalarTensorDiv(), + 'desc_inputs': []}), + ('ScalarEqScalar', { + 'block': EqualClass(1, 2), + 'desc_inputs': []}), + ('ScalarEqNone', { + 'block': EqualClass(1, None), + 'desc_inputs': []}), + ('NoneEqScalar', { + 'block': EqualClass(None, 2), + 'desc_inputs': []}), + ('TupleEqTuple', { + 'block': EqualClass((1, 2), (1, 3)), + 'desc_inputs': []}), + ('NoneEqTuple', { + 'block': EqualClass(None, (1, 3)), + 'desc_inputs': []}), + ('TupleEqNone', { + 'block': EqualClass((1, 2), None), + 'desc_inputs': []}), + ('EqTensor1', { + 'block': EqualClass(Tensor(np.ones([2, 1, 2, 2], np.float32)), + Tensor(np.ones([2, 1, 2, 2], np.float32))), + 'desc_inputs': []}), + ('EqTensor2', { + 'block': EqualClass(Tensor(np.ones([2, 1, 2, 2], np.float32)), None), + 'desc_inputs': []}), + ('NoneEqTensor', { + 'block': EqualClass(None, Tensor(np.ones([2, 1, 2, 2], np.float32))), + 'desc_inputs': []}), + ('MapPartial', { + 'block': MapPartialNet(), + 'desc_inputs': [Tensor(np.ones([2, 1, 2, 2], np.float32)), + Tensor(np.ones([2, 1, 2, 2], np.float32))]}), + ('Zip', { + 'block': ZipNet(), + 'desc_inputs': [Tensor(np.ones([2, 1, 2, 2], np.float32)), + Tensor(np.ones([2, 1, 2, 2], np.float32))]}), + ('Unzip', { + 'block': UnZipNet(), + 'desc_inputs': [Tensor(np.ones([2, 1, 2, 2], np.float32)), + Tensor(np.ones([2, 1, 2, 2], np.float32))]}), + ('ScalarTensorOpCast2', { + 'block': ScalarTensorOp2Cast(), + 'desc_inputs': []}), +] + +test_case_lists = [test_case_ops] +test_exec_case = reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case diff --git a/tests/ut/python/ops/test_nn_ops.py b/tests/ut/python/ops/test_nn_ops.py new file mode 100644 index 00000000000..5b9f37864cb --- /dev/null +++ b/tests/ut/python/ops/test_nn_ops.py @@ -0,0 +1,535 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test nn ops """ +import functools +import numpy as np +import mindspore + +import mindspore.nn as nn +import mindspore.context as context + +from mindspore import Tensor, Parameter +from mindspore.common.initializer import initializer +from mindspore.ops import Primitive +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.ops import prim_attr_register, PrimitiveWithInfer + +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.forward.verify_exception \ + import pipeline_for_verify_exception_for_case_by_case_config + + +def conv3x3(in_channels, out_channels, stride=1, padding=1): + """3x3 convolution """ + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding) + + +class ResidualBlock(nn.Cell): + """ + residual Block + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = nn.BatchNorm2d(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=0) + self.bn2 = nn.BatchNorm2d(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.relu = nn.ReLU() + self.downsample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, + stride=stride, padding=0) + self.bn_down_sample = nn.BatchNorm2d(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + """ + :param x: + :return: + """ + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class VirtualLossGrad(PrimitiveWithInfer): + """ VirtualLossGrad definition """ + @prim_attr_register + def __init__(self): + """init VirtualLossGrad""" + + def __call__(self, x, out, dout): + raise NotImplementedError + + def infer_shape(self, x_shape, out_shape, dout_shape): + return x_shape + + def infer_dtype(self, x_dtype, out_dtype, dout_dtype): + return x_dtype + + +class VirtualLoss(PrimitiveWithInfer): + """ VirtualLoss definition """ + @prim_attr_register + def __init__(self): + """init VirtualLoss""" + + def __call__(self, x): + raise NotImplementedError + + def get_bprop(self): + loss_grad = VirtualLossGrad() + + def bprop(x, out, dout): + # pylint: disable=unused-argument + dx = loss_grad(x, out, dout) + return (dx,) + return bprop + + def infer_shape(self, x_shape): + return [] + + def infer_dtype(self, x_dtype): + return x_dtype + + +class VirtualNetWithLoss(nn.Cell): + """ VirtualNetWithLoss definition """ + def __init__(self, network): + super(VirtualNetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x): + predict = self.network(x) + return self.loss(predict) + + +class SoftMaxGrad(nn.Cell): + """ SoftMaxGrad definition """ + def __init__(self, network): + super(SoftMaxGrad, self).__init__() + self.network = network + + def construct(self, x): + return C.grad(self.network)(x) + + +class DropoutGrad(nn.Cell): + """ DropoutGrad definition """ + def __init__(self, network): + super(DropoutGrad, self).__init__() + self.network = network + + def construct(self, x): + return C.grad(self.network)(x) + + +class ScalarSummaryNet(nn.Cell): + """ ScalarSummaryNet definition """ + def __init__(self): + super(ScalarSummaryNet, self).__init__() + self.summary = P.ScalarSummary() + + def construct(self, scalar): + string_in = "bias_value" + out = self.summary(string_in, scalar) + return out + + +class FusedBatchNormGrad(nn.Cell): + """ FusedBatchNormGrad definition """ + def __init__(self, network): + super(FusedBatchNormGrad, self).__init__() + self.grad = C.GradOperation(name="get_all", get_all=True, sens_param=True) + self.network = network + + def construct(self, inp, output_grad): + return self.grad(self.network)(inp, output_grad) + + +class NetWithLoss(nn.Cell): + """ NetWithLoss definition """ + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = P.SmoothL1Loss() + self.network = network + + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + +class Grad(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + self.network.set_train() + + def construct(self, x, label): + return C.grad(self.network)(x, label) + + +class BatchnormNet(nn.Cell): + """ BatchnormNet definition """ + def __init__(self): + super(BatchnormNet, self).__init__() + self.conv1 = nn.Conv2d(3, 4, kernel_size=8, stride=2, pad_mode="pad", padding=3) + self.bn1 = nn.BatchNorm2d(4) + self.flatten = P.Flatten() + self.weight = Parameter(Tensor(np.ones([64, 10], np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10], np.float32)), name="bias") + self.fc = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.flatten(x) + x = self.biasAdd(self.fc(x, self.weight), self.bias) + return x + + +class NetWithLossClass(nn.Cell): + """ NetWithLossClass definition """ + def __init__(self, network): + super(NetWithLossClass, self).__init__(auto_prefix=False) + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + +class BlockNet(nn.Cell): + """ BlockNet definition """ + def __init__(self): + super(BlockNet, self).__init__() + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, pad_mode="pad", padding=3) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=0) + self.block_down_sample = ResidualBlock( + 64, 256, stride=1, down_sample=True + ) + self.flatten = P.Flatten() + self.weight = Parameter(Tensor(np.ones([1024, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype((np.float32))), name="bias") + self.fc = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.conv1(x) + return x + + +class Conv2dWithBiasNet(nn.Cell): + """ Conv2dWithBiasNet definition """ + def __init__(self): + super(Conv2dWithBiasNet, self).__init__() + self.conv = nn.Conv2d(3, 10, 1, bias_init='zeros') + self.flatten = P.Flatten() + + def construct(self, input_x): + return self.flatten(self.conv(input_x)) + + +class Conv2dNativeNet(nn.Cell): + """ Conv2dNativeNet definition """ + def __init__(self): + super(Conv2dNativeNet, self).__init__() + self.conv = P.DepthwiseConv2dNative(channel_multiplier=3, kernel_size=(3, 3)) + self.flatten = P.Flatten() + channel_multipliers = 1 + in_channels = 3 + kernel_size = (3, 3) + self.weight = Parameter(initializer( + Tensor(np.ones([channel_multipliers, in_channels, *kernel_size], dtype=np.float32)), + [channel_multipliers, in_channels, *kernel_size]), name='weight') + + def construct(self, input_x): + return self.flatten(self.conv(input_x, self.weight)) + + +class MakeRefKeyNet(nn.Cell): + """ MakeRefKeyNet definition """ + def __init__(self): + super(MakeRefKeyNet, self).__init__() + self.y= Parameter(Tensor([1.0], mindspore.float32), name="y") + + def construct(self, x): + key = P.MakeRefKey("y")() + P.Assign()(key, x) + return x + + +class StateNet(nn.Cell): + """ StateTestTensor definition """ + def __init__(self): + super(StateNet, self).__init__() + weight = Tensor(np.ones([2, 1, 2, 2], np.float32)) + self.s1 = Parameter(weight, name="s1") + self.s2 = Parameter(weight, name="s2") + self.sub = P.Sub() + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.assign = P.Assign() + + def construct(self, x): + x = Primitive('depend')(x, self.assign(self.s1, x + self.s1)) + self.s1 = self.sub(self.s1, x) + self.s2 = self.sub(self.s2, x) + return x + + +class ComparisonNet(nn.Cell): + def __init__(self): + """ ComparisonNet definition """ + super(ComparisonNet, self).__init__() + + def construct(self, x, y): + ret = x <= y + return ret + + +test_cases = [ + ('SoftMaxGrad', { + 'block': SoftMaxGrad(VirtualNetWithLoss(P.Softmax())), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [[128, 32, 32, 64]], + }), + ('DropoutGrad', { + 'block': DropoutGrad(VirtualNetWithLoss(nn.Dropout())), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [[128, 32, 32, 64]], + }), + ('ApplyMomentum', { + 'block': P.ApplyMomentum(), + 'desc_inputs': [[2], [128, 32, 32, 64], [128, 32, 32, 64], [128, 32, 32, 64], [128, 32, 32, 64]], + 'desc_bprop': [[128, 32, 32, 64]], + 'skip': ['backward'] + }), + ('ScalarSummary', { + 'block': ScalarSummaryNet(), + 'desc_inputs': [2.2], + }), + ('FusedBatchNormGrad', { + 'block': FusedBatchNormGrad(nn.BatchNorm2d(num_features=512, eps=1e-5, momentum=0.1)), + 'desc_inputs': [[64, 512, 7, 7], [64, 512, 7, 7]], + 'desc_bprop': [[64, 512, 7, 7]], + }), + ('BatchnormGrad', { + 'block': Grad(NetWithLoss(BatchnormNet())), + 'desc_inputs': [Tensor(np.ones([1, 3, 8, 8], np.float32)), Tensor(np.zeros([1, 10], np.float32))], + }), + ('BlockGrad', { + 'block': Grad(NetWithLossClass(BlockNet())), + 'desc_inputs': [Tensor(np.ones([1, 3, 8, 8], np.float32)), Tensor(np.zeros([1, 64, 4, 4], np.float32))], + }), + ('Conv2dWithBiasGrad', { + 'block': Grad(NetWithLossClass(Conv2dWithBiasNet())), + 'desc_inputs': [Tensor(np.ones([1, 3, 16, 16], np.float32)), Tensor(np.zeros([1, 2560], np.float32))], + }), + ('Conv2dNativeGrad', { + 'block': Grad(NetWithLossClass(Conv2dNativeNet())), + 'desc_inputs': [Tensor(np.ones([1, 3, 16, 16], np.float32)), Tensor(np.zeros([1, 1764], np.float32))], + }), + ('MakeRefKey', { + 'block': MakeRefKeyNet(), + 'desc_inputs': [Tensor([2.0], mindspore.float32)], + }), + ('StateTest', { + 'block': StateNet(), + 'desc_inputs': [Tensor(np.ones([2, 1, 2, 2]).astype(np.float32))], + }), + ('StateGrad', { + 'block': Grad(NetWithLossClass(StateNet())), + 'desc_inputs': [Tensor(np.ones([2, 1, 2, 2], np.float32)), Tensor(np.ones([2, 1, 2, 2], np.float32))], + }), + ('ComparisonTest', { + 'block': ComparisonNet(), + 'desc_inputs': [Tensor(np.ones([6, 9, 10], np.int32)), Tensor(np.ones([6, 9, 10], np.int32))], + }), +] + + +test_cases_for_verify_exception = [ + ('Conv2d_ValueError_1', { + 'block': (lambda _ : P.Conv2D(3, 4, mode=-2.0), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('Conv2d_ValueError_2', { + 'block': (lambda _ : P.Conv2D(3, 4, mode=-2), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_1', { + 'block': (lambda _ : P.MaxPoolWithArgmax(pad_mode='sane'), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_2', { + 'block': (lambda _ : P.MaxPoolWithArgmax(data_mode=2), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_3', { + 'block': (lambda _ : P.MaxPoolWithArgmax(ceil_mode=2), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_4', { + 'block': (lambda _ : P.MaxPoolWithArgmax(pad_mode="pad", pad=-1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_5', { + 'block': (lambda _ : P.MaxPoolWithArgmax(pad_mode="pad", pad='1'), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_6', { + 'block': (lambda _ : P.MaxPoolWithArgmax(window='1'), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_7', { + 'block': (lambda _ : P.MaxPoolWithArgmax(window=-2), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_8', { + 'block': (lambda _ : P.MaxPoolWithArgmax(stride=-1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPoolWithArgmax_ValueError_9', { + 'block': (lambda _ : P.MaxPoolWithArgmax(alpha='1'), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('FusedBatchNorm_ValueError_1', { + 'block': (lambda _ : P.FusedBatchNorm(mode="1", epsilon=1e-5, momentum=0.1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('FusedBatchNorm_ValueError_2', { + 'block': (lambda _ : P.FusedBatchNorm(mode=2, epsilon=1e-5, momentum=0.1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('FusedBatchNorm_ValueError_3', { + 'block': (lambda _ : P.FusedBatchNorm(mode=0, epsilon=-1e-5, momentum=0.1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('FusedBatchNorm_ValueError_4', { + 'block': (lambda _ : P.FusedBatchNorm(mode=0, epsilon=1e-5, momentum=-0.1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('FusedBatchNorm_ValueError_5', { + 'block': (lambda _ : P.FusedBatchNorm(mode=1, epsilon=-0.001, momentum=0.0), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('Softmax_ValueError_1', { + 'block': (lambda _ : P.Softmax("1"), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('Softmax_ValueError_2', { + 'block': (lambda _ : P.Softmax(1.1), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('Softmax_ValueError_3', { + 'block': (lambda _ : P.Softmax(axis="1"), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('DropoutGenMask_ValueError_1', { + 'block': (lambda _ : P.DropoutGenMask(Seed0="seed0"), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('DropoutGenMask_ValueError_2', { + 'block': (lambda _ : P.DropoutGenMask(Seed0=1.0), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('DropoutGenMask_ValueError_3', { + 'block': (lambda _ : P.DropoutGenMask(Seed1="seed1"), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('DropoutGenMask_ValueError_4', { + 'block': (lambda _ : P.DropoutGenMask(Seed1=2.0), {'exception': ValueError}), + 'desc_inputs': [0], + }), + ('MaxPool2d_ValueError_1', { + 'block': (nn.MaxPool2d(kernel_size=120, stride=1, pad_mode="valid", padding=0), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.random.randn(32, 3, 112, 112).astype(np.float32).transpose(0, 3, 1, 2))], + }), + ('MaxPool2d_ValueError_2', { + 'block': ( + lambda _ : nn.MaxPool2d(kernel_size=120, stride=True, pad_mode="valid", padding=0), + {'exception': ValueError}, + ), + 'desc_inputs': [Tensor(np.random.randn(32, 3, 112, 112).astype(np.float32).transpose(0, 3, 1, 2))], + }), + ('MaxPool2d_ValueError_3', { + 'block': ( + lambda _ : nn.MaxPool2d(kernel_size=3, stride=True, pad_mode="valid", padding=0), + {'exception': ValueError}, + ), + 'desc_inputs': [Tensor(np.random.randn(32, 3, 112, 112).astype(np.float32).transpose(0, 3, 1, 2))], + }), +] + + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_compile(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + return test_cases + + +@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config) +def test_check_exception(): + return test_cases_for_verify_exception + diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py new file mode 100755 index 00000000000..bfe80759728 --- /dev/null +++ b/tests/ut/python/ops/test_ops.py @@ -0,0 +1,1047 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test ops """ +import functools +import numpy as np +from mindspore import ops +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +import mindspore.ops.composite as C +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +from ..ut_filter import non_graph_engine + +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward\ + import (pipeline_for_compile_forward_ge_graph_for_case_by_case_config, + pipeline_for_compile_forward_ge_graph_for_case_by_case_config_exception) +from ....mindspore_test_framework.pipeline.gradient.compile_gradient\ + import pipeline_for_compile_grad_ge_graph_for_case_by_case_config + + +class InputBackward(nn.Cell): + def __init__(self, network): + super(InputBackward, self).__init__() + self.network = network + self.network.set_train() + self.grad = C.grad_all_with_sens + + def construct(self, x1, x2, x3, sens): + return self.grad(self.network)(x1, x2, x3, sens) + + +class NetForTupleInput(nn.Cell): + def __init__(self, op): + super(NetForTupleInput, self).__init__() + self.op = op + + def construct(self, x1, x2): + return self.op((x1, x2)) + + +class StridedSlicessdNet(nn.Cell): + def __init__(self): + super(StridedSlicessdNet, self).__init__() + self.rank = P.Rank() + + def construct(self, x1): + return P.StridedSlice(1, 1, 0, self.rank(x1), 0)(x1, (0, 0), (0, 0), (1, 1)) + + +class NetForConcat(nn.Cell): + def __init__(self): + super(NetForConcat, self).__init__() + self.concat = P.Concat() + + def construct(self, x1): + return self.concat((x1, x1)) + + +class NetForConcat1(nn.Cell): + def __init__(self): + super(NetForConcat1, self).__init__() + self.concat = P.Concat() + + def construct(self, x1, x2): + return self.concat((x1, x2)) + + +class NetForFlatten(nn.Cell): + def __init__(self): + super(NetForFlatten, self).__init__() + self.flatten = P.Flatten() + + def construct(self, x, y): + return self.flatten(x) + y + + +class NetForFlatten0D(nn.Cell): + def __init__(self): + super(NetForFlatten0D, self).__init__() + self.flatten = P.Flatten() + + def construct(self, x): + return self.flatten(x) + + +class ArgmaxNet(nn.Cell): + def __init__(self): + super(ArgmaxNet, self).__init__() + self.argmax = P.Argmax(axis=1) + + def construct(self, input): + return self.argmax(input) + + +class ArgminNet(nn.Cell): + def __init__(self): + super(ArgminNet, self).__init__() + self.argmin = P.Argmin(axis=1) + + def construct(self, input): + return self.argmin(input) + + +class CumSumNet(nn.Cell): + def __init__(self): + super(CumSumNet, self).__init__() + self.cumsum = P.CumSum() + self.axis = 1 + + def construct(self, input): + return self.cumsum(input, self.axis) + + +class SummaryNet(nn.Cell): + def __init__(self,): + super(SummaryNet, self).__init__() + self.s = P.ScalarSummary() + self.add = P.TensorAdd() + + def construct(self, x, y): + self.s("x1", x) + return self.add(x, y) + + +test_case_math_ops = [ + ('Neg', { + 'block': P.Neg(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('Sub', { + 'block': P.Sub(), + 'desc_inputs': [[3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('TensorAdd', { + 'block': P.TensorAdd(), + 'desc_inputs': [[3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Mul0', { + 'block': P.Mul(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Mul1', { + 'block': P.Mul(), + 'desc_inputs': [[2, 3, 1, 1], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Mul2', { + 'block': P.Mul(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 1, 1]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Mul3', { + 'block': P.Mul(), + 'desc_inputs': [[3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Mul4', { + 'block': P.Mul(), + 'desc_inputs': [[2, 3, 3, 5], [3, 5]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Add0', { + 'block': P.TensorAdd(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Add1', { + 'block': P.TensorAdd(), + 'desc_inputs': [[3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Add2', { + 'block': P.TensorAdd(), + 'desc_inputs': [[2, 3, 3, 5], [3, 5]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Add3', { + 'block': P.TensorAdd(), + 'desc_inputs': [[2, 3, 1, 1], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Add4', { + 'block': P.TensorAdd(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 1, 1]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Minimum', { + 'block': P.Minimum(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Pow', { + 'block': P.Pow(), + 'desc_const': [2.0], + 'desc_inputs': [[2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Exp', { + 'block': P.Exp(), + 'desc_inputs': [[2, 3]], + 'desc_bprop': [[2, 3]]}), + ('Floor', { + 'block': P.Floor(), + 'desc_inputs': [[2, 512, 56, 56]], + 'desc_bprop': [[2, 512, 56, 56]], + 'skip': ['backward']}), + ('ACos', { + 'block': P.ACos(), + 'desc_inputs': [[2, 3]], + 'desc_bprop': [[2, 3]]}), + ('Sin', { + 'block': P.Sin(), + 'desc_inputs': [[2, 3]], + 'desc_bprop': [[2, 3]]}), + ('Reciprocal', { + 'block': P.Reciprocal(), + 'desc_inputs': [[2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Minimum_0', { + 'block': P.Minimum(), + 'desc_inputs': [[2, 3, 3, 5], [3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Maximum', { + 'block': P.Maximum(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Maximum_0', { + 'block': P.Maximum(), + 'desc_inputs': [[3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('MaximumGrad', { + 'block': G.MaximumGrad(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5], [2, 3, 3, 5]], + 'skip': ['backward']}), + ('MinimumGrad', { + 'block': G.MinimumGrad(), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5], [2, 3, 3, 5]], + 'skip': ['backward']}), + ('StridedSlice', { + 'block': P.StridedSlice(), + 'desc_const': [(0, 1, 2, 1), + (2, 3, 3, 4), + (1, 1, 1, 1)], + 'desc_inputs': [[2, 3, 3, 5]], + 'desc_bprop': [[2, 2, 1, 3]]}), + ('Slice_1', { + 'block': P.Slice(), + 'desc_const': [(0, 1, 2, 1), + (1, 1, 1, 2)], + 'desc_inputs': [[2, 3, 3, 5]], + 'desc_bprop': [[1, 1, 1, 2]]}), + ('StridedSliceGrad', { + 'block': G.StridedSliceGrad(), + 'desc_const': [(64, 1, 1024), + (0, 1, 0), + (64, 2, 1024), + (1, 1, 1)], + 'desc_inputs': [[64, 128, 1024]], + 'skip': ['backward']}), + ('RandomChoiceWithMask', { + 'block': P.RandomChoiceWithMask(256), + 'desc_inputs': [Tensor(np.random.rand(24000, 4).astype(np.bool_))], + 'desc_bprop': [[256,4], [256,4]], + 'skip': ['backward']}), + ('LessEqual', { + 'block': P.LessEqual(), + 'desc_inputs': [Tensor(np.random.rand(4).astype(np.float16)), + Tensor(np.random.rand(4).astype(np.float16))], + 'skip': ['backward']}), + ('Less', { + 'block': P.Less(), + 'desc_inputs': [[2, 1, 4, 5], [2, 1, 4, 5]], + 'desc_bprop': [Tensor(np.zeros((2, 1, 4, 5), np.bool_))], + 'skip': ['backward']}), + ('RealDiv_0', { + 'block': P.RealDiv(), + 'desc_const': [Tensor(2048.0), Tensor(0.0)], + 'desc_inputs': [], + 'skip': ['backward']}), + ('RealDiv', { + 'block': P.RealDiv(), + 'desc_inputs': [[4], Tensor(np.ones(4).astype(np.float32))], + 'desc_bprop': [[4]]}), + ('RealDiv_1', { + 'block': P.RealDiv(), + 'desc_inputs': [[512, 1024], [512, 1024]], + 'desc_bprop': [[512, 1024]]}), + ('FloorDiv', { + 'block': P.FloorDiv(), + 'desc_inputs': [Tensor(np.random.rand(4).astype(np.float16)), + Tensor(np.random.rand(4).astype(np.float16))], + 'skip': ['backward']}), + ('identity', { + 'block': ops.functional.identity, + 'desc_inputs': [[2, 2]], + 'skip': ['backward']}), + ('MatMul_1', { + 'block': P.MatMul(transpose_a=False, transpose_b=False), + 'desc_inputs': [[1024, 160], [160, 1024]], + 'desc_bprop': [[1024, 1024]]}), + ('MatMul_2', { + 'block': P.MatMul(transpose_a=True, transpose_b=True), + 'desc_inputs': [[160, 1024], [1024, 160]], + 'desc_bprop': [[1024, 1024]]}), + ('Sub', { + 'block': P.Sub(), + 'desc_inputs': [[3], [3]], + 'desc_bprop': [[3]]}), + ('TruncatedNormal', { + 'block': P.TruncatedNormal(), + 'desc_const': [Tensor(np.array([1, 2, 3]))], + 'desc_inputs': [], + 'skip': ['backward'], + 'add_fake_input': True}), + ('Select', { + 'block': P.Select(), + 'desc_inputs': [Tensor(np.array([[True, False, False], [False, True, True]])), + [2, 3], [2, 3]], + 'desc_bprop': [[2, 3]]}), + ('Rank', { + 'block': P.Rank(), + 'desc_inputs': [[2, 3]], + 'skip': ['backward']}), + ('InvertPermutation', { + 'block': P.InvertPermutation(), + 'desc_const': [(0, 3, 1, 2)], + 'desc_inputs': [], + 'skip': ['backward']}), + ('Square', { + 'block': P.Square(), + 'desc_inputs': [[4]], + 'desc_bprop': [[4]]}), + ('Rsqrt', { + 'block': P.Rsqrt(), + 'desc_inputs': [[4]], + 'desc_bprop': [[4]]}), + ('Sqrt', { + 'block': P.Sqrt(), + 'desc_inputs': [[4]], + 'desc_bprop': [[4]]}), + ('RealDiv', { + 'block': P.RealDiv(), + 'desc_inputs': [[4, 5], [2, 3, 4, 5]], + 'desc_bprop': [[2, 3, 4, 5]]}), + ('Div', { + 'block': P.Div(), + 'desc_inputs': [[4, 5], [2, 3, 4, 5]], + 'desc_bprop': [[2, 3, 4, 5]]}), + ('Equal', { + 'block': P.Equal(), + 'desc_inputs': [[3, 4, 5], [4, 5]], + 'desc_bprop': [Tensor(np.zeros((3, 4, 5), np.bool_))]}), + ('NotEqual', { + 'block': P.NotEqual(), + 'desc_inputs': [[4, 1], [2, 3, 4, 5]], + 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}), + ('Greater', { + 'block': P.Greater(), + 'desc_inputs': [[2, 3, 4, 1], [4, 5]], + 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}), + ('GreaterEqual', { + 'block': P.GreaterEqual(), + 'desc_inputs': [[2, 3, 4, 1], [4, 5]], + 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}), + ('LogicalNot', { + 'block': P.LogicalNot(), + 'desc_inputs': [Tensor(np.zeros((3, 4, 5), np.bool_))], + 'desc_bprop': [Tensor(np.ones((3, 4, 5), np.bool_))]}), + ('LogicalAnd', { + 'block': P.LogicalAnd(), + 'desc_inputs': [Tensor(np.zeros((2, 3, 4), np.bool_)), Tensor(np.ones((1), np.bool_))], + 'desc_bprop': [Tensor(np.zeros((2, 3, 4), np.bool_))]}), + ('LogicalOr', { + 'block': P.LogicalOr(), + 'desc_inputs': [Tensor(np.zeros((3, 4, 5), np.bool_)), Tensor(np.ones((3, 1, 1), np.bool_))], + 'desc_bprop': [Tensor(np.zeros((3, 4, 5), np.bool_))]}), + ('NpuAllocFloatStatus', { + 'block': P.NPUAllocFloatStatus(), + 'desc_inputs': [], + 'add_fack_input': True, + 'fack_input_type': np.float32, + 'desc_bprop': [Tensor(np.zeros([8]).astype(np.float32))], + 'skip': ['backward']}), + ('NpuGetFloatStatus', { + 'block': P.NPUGetFloatStatus(), + 'desc_inputs': [Tensor(np.zeros([8]).astype(np.float32))], + 'desc_bprop': [Tensor(np.zeros([8]).astype(np.float32))], + 'skip': ['backward']}), + ('NpuClearFloatStatus', { + 'block': P.NPUClearFloatStatus(), + 'desc_inputs': [Tensor(np.zeros([8]).astype(np.float32))], + 'desc_bprop': [Tensor(np.zeros([8]).astype(np.float32))], + 'skip': ['backward']}), + ('CheckValid', { + 'block': P.CheckValid(), + 'desc_inputs': [[20000, 4], [3]], + 'desc_bprop': [[20000]], + 'skip': ['backward']}), + ('NMSWithMask', { + 'block': P.NMSWithMask(0.5), + 'desc_inputs': [[128, 5]], + 'desc_bprop': [[128, 5], [128], [128]], + 'skip': ['backward']}), + ('Abs', { + 'block': P.Abs(), + 'desc_inputs': [[4]], + 'desc_bprop': [[4]]}), + ('CumSum', { + 'block': P.CumSum(), + 'desc_const': [0], + 'desc_inputs': [Tensor(np.array([[3, 4],[1, 6]]).astype(np.float16))], + 'desc_bprop': [Tensor(np.array([[3, 4],[4, 10]]).astype(np.float16))]}), + ('ReduceSum_3', { + 'block': P.ReduceSum(), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[2]]}), + ('ReduceSum_4', { + 'block': P.ReduceSum(keep_dims=True), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[1, 2]]}), + ('ReduceSum_5', { + 'block': P.ReduceSum(keep_dims=True), + 'desc_inputs': [[2, 3, 4]], + 'desc_bprop': [[1, 1, 1]]}), + ('ReduceSum_6', { + 'block': P.ReduceSum(), + 'desc_inputs': [[2, 3, 4]], + 'desc_bprop': [[1]]}), + ('Sum_0', { + 'block': P.ReduceSum(), + 'desc_const': [(1,)], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[3]]}), + ('Sum_1', { + 'block': P.ReduceSum(keep_dims=True), + 'desc_const': [(1,)], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[3, 1]]}), + ('Sum_2', { + 'block': P.ReduceSum(), + 'desc_const': [(0, 1)], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[1]]}), + ('Sum_3', { + 'block': P.ReduceSum(), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[2]]}), + ('Sum_4', { + 'block': P.ReduceSum(keep_dims=True), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[1, 2]]}), + ('Sum_5', { + 'block': P.ReduceSum(keep_dims=True), + 'desc_const': [()], + 'desc_inputs': [[2, 3, 4]], + 'desc_bprop': [[1, 1, 1]]}), + ('Sum_6', { + 'block': P.ReduceSum(), + 'desc_const': [()], + 'desc_inputs': [[2, 3, 4]], + 'desc_bprop': [[1]]}), + ('Sign', { + 'block': P.Sign(), + 'desc_inputs': [[3]], + 'desc_bprop': [[3]]}), + ('Round', { + 'block': P.Round(), + 'desc_inputs': [[3]], + 'desc_bprop': [[3]]}) +] + +test_case_nn_ops = [ + ('BiasAdd', { + 'block': P.BiasAdd(), + 'desc_inputs': [[1, 3, 3, 3], [3]], + 'desc_bprop': [[1, 3, 3, 3]]}), + ('BiasAddGrad', { + 'block': G.BiasAddGrad(), + 'desc_inputs': [[1, 3, 3, 3]], + 'skip': ['backward']}), + ('Gelu', { + 'block': P.Gelu(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('GeluGrad', { + 'block': G.GeluGrad(), + 'desc_inputs': [[2, 2], [2, 2], [2, 2]], + 'desc_bprop': [[2, 2]], + 'skip': ['backward']}), + ('Tanh', { + 'block': P.Tanh(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('TanhGrad', { + 'block': G.TanhGrad(), + 'desc_inputs': [[1, 3, 4, 4], [1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]], + 'skip': ['backward']}), + ('ReLU', { + 'block': P.ReLU(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('ReLU6', { + 'block': P.ReLU6(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('ReLUGrad', { + 'block': G.ReluGrad(), + 'desc_inputs': [[1, 3, 4, 4], [1, 3, 4, 4]], + 'skip': ['backward']}), + ('Elu', { + 'block': P.Elu(), + 'desc_inputs': [[2, 3, 4]], + 'desc_bprop': [[2, 3, 4]]}), + ('EluGrad', { + 'block': G.EluGrad(), + 'desc_inputs': [[2, 3, 4], [2, 3, 4]], + 'desc_bprop': [[2, 3, 4]], + 'skip': ['backward']}), + ('Sigmoid', { + 'block': P.Sigmoid(), + 'desc_inputs': [[1, 3, 4, 4]], + 'desc_bprop': [[1, 3, 4, 4]]}), + ('MaxPool', { + 'block': P.MaxPool(ksize=(2, 2), strides=(2, 2), padding="VALID"), + 'desc_inputs': [[100, 3, 28, 28]], + 'desc_bprop': [[100, 3, 14, 14]]}), + ('MaxPoolGrad', { + 'block': G.MaxPoolGrad(ksize=(2, 2), strides=(2, 2), padding="VALID"), + 'desc_inputs': [[3, 4, 6, 6], [3, 4, 3, 3], [3, 4, 3, 3]], + 'desc_bprop': [[3, 4, 6, 6]], + 'skip': ['backward']}), + ('AvgPool', { + 'block': P.AvgPool(ksize=(2, 2), strides=(2, 2), padding="VALID"), + 'desc_inputs': [[100, 3, 28, 28]], + 'desc_bprop': [[100, 3, 14, 14]]}), + ('AvgPoolGrad', { + 'block': G.AvgPoolGrad(ksize=(2, 2), strides=(2, 2), padding="VALID"), + 'desc_const': [(3, 4, 6, 6)], + 'const_first': True, + 'desc_inputs': [[3, 4, 6, 6]], + 'desc_bprop': [[3, 4, 6, 6]], + 'skip': ['backward']}), + ('MaxPoolWithArgmax', { + 'block': P.MaxPoolWithArgmax(window=2, stride=2), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [[128, 32, 8, 16], [128, 32, 8, 16]]}), + ('SoftmaxCrossEntropyWithLogits', { + 'block': P.SoftmaxCrossEntropyWithLogits(), + 'desc_inputs': [[1, 10], [1, 10]], + 'desc_bprop': [[1], [1, 10]], + 'skip': ['backward_exec']}), + ('Flatten', { + 'block': P.Flatten(), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [[128 * 32 * 8 * 16]]}), + ('LogSoftmax', { + 'block': P.LogSoftmax(), + 'desc_inputs': [[64, 2]], + 'desc_bprop': [[160, 30522]]}), + ('LogSoftmaxGrad', { + 'block': G.LogSoftmaxGrad(), + 'desc_inputs': [[16, 1234], [16, 1234]], + 'desc_bprop': [[64, 2]], + 'skip': ['backward']}), + ('LayerNorm', { + 'block': P.LayerNorm(), + 'desc_inputs': [[2, 16], [16], [16]], + 'desc_bprop': [[2, 16], [2, 16], [2, 16]]}), + ('LayerNormGrad', { + 'block': G.LayerNormGrad(), + 'desc_inputs': [[2, 16], [2, 16], [2, 16], [2, 16], [16]], + 'desc_bprop': [[2, 16], [16], [16]], + 'skip': ['backward']}), + ('FusedBatchNorm', { + 'block': P.FusedBatchNorm(), + 'desc_inputs': [[128, 64, 32, 64], [64], [64], [64], [64]], + 'desc_bprop': [[128, 64, 32, 64], [64], [64], [64], [64]], + 'skip': []}), + ('FusedBatchNormGrad', { + 'block': G.FusedBatchNormGrad(), + 'desc_inputs': [[128, 64, 32, 64], [128, 64, 32, 64], [64], [64], [64]], + 'desc_bprop': [[128, 64, 32, 64], [64], [64], [64], [64]], + 'skip': ['backward']}), + ('BatchNorm', { + 'block': P.BatchNorm(), + 'desc_inputs': [[128, 64, 32, 32], [64], [64], [64], [64]], + 'desc_bprop': [[128, 64, 32, 32], [64], [64], [64], [64]], + 'skip': []}), + ('BatchNormGrad', { + 'block': G.BatchNormGrad(), + 'desc_inputs': [[128, 64, 32, 32], [128, 64, 32, 32], [64], [64], [64], [64]], + 'desc_bprop': [[128, 64, 32, 32], [64], [64], [64], [64]], + 'skip': ['backward']}), + ('ApplyMomentum', { + 'block': P.ApplyMomentum(), + 'desc_inputs': [[128, 32, 32, 64], [128, 32, 32, 64], + [32, 32, 64], [32, 32, 64], [32, 32, 64]], + 'desc_bprop': [[128, 32, 32, 64]], + 'skip': ['backward']}), + ('TopK', { + 'block': P.TopK(), + 'desc_const': [5], + 'desc_inputs': [[20, 20, 10]], + 'desc_bprop': [[20, 20, 5]], + 'skip': ['backward']}), + ('GatherV2_0', { + 'block': P.GatherV2(), + 'desc_const': [0], + 'desc_inputs': [[3, 1, 2], Tensor(np.array([0, 1]).astype(np.int32))], + 'desc_bprop': [[2, 1, 2]]}), + ('GatherV2_1', { + 'block': P.GatherV2(), + 'desc_const': [2], + 'desc_inputs': [[3, 1, 3], Tensor(np.array([0, 1]).astype(np.int32))], + 'desc_bprop': [[3, 1, 2]]}), + ('GatherV2_2', { + 'block': P.GatherV2(), + 'desc_const': [0], + 'desc_inputs': [[3, 1, 3], Tensor(np.array([[0, 1], [0, 1], [0, 1]]).astype(np.int32))], + 'desc_bprop': [[3, 2, 1, 3]]}), + ('GatherV2_3', { + 'block': P.GatherV2(), + 'desc_const': [2], + 'desc_inputs': [[3, 1, 3], Tensor(np.array([[0, 1], [0, 1], [0, 1]]).astype(np.int32))], + 'desc_bprop': [[3, 1, 3, 2]]}), + ('GatherV2_4', { + 'block': P.GatherV2(), + 'desc_const': [1], + 'desc_inputs': [[32, 5, 1024], Tensor(np.array([3]).astype(np.int32))], + 'desc_bprop': [[32, 1, 1024]]}), + ('GatherV2_5', { + 'block': P.GatherV2(), + 'desc_const': [-1], + 'desc_inputs': [[3, 1, 3], Tensor(np.array([0, 1]).astype(np.int32))], + 'desc_bprop': [[3, 1, 2]]}), + ('GatherV2_6', { + 'block': P.GatherV2(), + 'desc_const': [0], + 'desc_inputs': [[1152], Tensor(np.array(10).astype(np.int32))], + 'desc_bprop': [Tensor(np.array(10).astype(np.float32))]}), + ('UnsortedSegmentSum', { + 'block': P.UnsortedSegmentSum(), + 'desc_const': [1280], + 'desc_inputs': [[1280,1024], Tensor(np.ones(1280).astype(np.int32))], + 'desc_bprop': [[8192,1024]], + 'skip': ['backward']}), + ('UnsortedSegmentSum_1', { + 'block': P.UnsortedSegmentSum(), + 'desc_const': [4], + 'desc_inputs': [[3, 2, 1, 3], Tensor(np.array([[0, 1], [0, 1], [0, 1]]).astype(np.int32))], + 'desc_bprop': [[4, 1, 3]], + 'skip': ['backward']}), + ('DropoutGenMask', { + 'block': P.DropoutGenMask(), + 'desc_const': [(2, 2), Tensor(0.5, mstype.float32)], + 'desc_inputs': [], + 'desc_bprop': [Tensor(np.ones(1).astype(np.int8))], + 'skip': ['backward']}), + ('DropoutDoMask', { + 'block': P.DropoutDoMask(), + 'desc_const': [Tensor(0.5)], + 'desc_inputs': [[64, 12, 128, 128], Tensor(np.ones(1572864).astype(np.uint8))], + 'desc_bprop': [[64, 12, 128, 128]]}), + ('Dropout', { + 'block': nn.Dropout(0.5), + 'desc_inputs': [[64, 12, 128, 128]], + 'desc_bprop': [[64, 12, 128, 128]]}), + ('ReduceMean0', { + 'block': P.ReduceMean(), + 'desc_const': [(2,)], + 'desc_inputs': [[3, 2, 2]], + 'desc_bprop': [[3, 2]]}), + ('ReduceMean1', { + 'block': P.ReduceMean(), + 'desc_const': [2], + 'desc_inputs': [[3, 2, 2]], + 'desc_bprop': [[3, 2]]}), + ('All', { + 'block': P.ReduceAll(), + 'desc_const': [(1,)], + 'desc_inputs': [Tensor(np.ones([3, 2]).astype(np.bool_))], + 'desc_bprop': [[3]], + 'skip': ['backward']}), + ('DescConst', { + 'block': Tensor(np.array([2], np.float32)), + 'desc_inputs': [], + 'desc_bprop': [[1]], + 'skip': ['backward'], + 'add_fake_input': True}), + ('Fill', { + 'block': P.Fill(), + 'desc_const': [mstype.float32, (2, 3), 1.0], + 'desc_inputs': [], + 'desc_bprop': [[2, 3]], + 'skip': ['backward'], + 'add_fake_input': True}), + ('OnesLike', { + 'block': P.OnesLike(), + 'desc_inputs': [Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32))], + 'desc_bprop': [Tensor(np.array([[1, 1], [1, 1]]).astype(np.int32))] + }), + ('ZerosLike', { + 'block': P.ZerosLike(), + 'desc_inputs': [Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32))], + 'desc_bprop': [Tensor(np.array([[1, 1], [1, 1]]).astype(np.int32))] + }), + ('Softmax', { + 'block': P.Softmax(), + 'desc_inputs': [[5, 5]], + 'desc_bprop': [[5, 5]]}), + ('DepthwiseConv2dNative_1', { + 'block': P.DepthwiseConv2dNative(3, (3, 3), pad_mode="pad", pad=1, stride=2), + 'desc_inputs': [[10, 32, 32, 32], [3, 32, 3, 3]], + 'desc_bprop': [[10, 30, 16, 16]]}), + ('DepthwiseConv2dNative_2', { + 'block': P.DepthwiseConv2dNative(1, (3, 3), pad_mode="same", pad=0, stride=1), + 'desc_inputs': [[2592, 2048, 4, 4], [1, 2048, 3, 3]], + 'desc_bprop': [[2592, 2048, 2, 2]]}), + ('SigmoidCrossEntropyWithLogits', { + 'block': P.SigmoidCrossEntropyWithLogits(), + 'desc_inputs': [[128, 10], [128, 10]], + 'desc_bprop': [[128, 10]]}), + ('Pad', { + 'block': P.Pad(((1, 2), (2, 3))), + 'desc_inputs': [[7, 7]], + 'desc_bprop': [[10, 12]]}), + ('BinaryCrossEntropy', { + 'block': P.BinaryCrossEntropy(), + 'desc_inputs': [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + 'desc_bprop': []}), + ('SparseApplyAdagrad', { + 'block': P.SparseApplyAdagrad(0.5), + 'desc_inputs': [[3, 3], [3, 3], [3, 3], Tensor(np.ones((3,), np.int32))], + 'desc_bprop': [3, 3], + 'skip': ['backward']}), + ('Flatten_1', { + 'block': NetForFlatten(), + 'desc_inputs': [Tensor(np.ones([2, 3, 4]).astype(np.int32)), Tensor(np.ones([2, 12]).astype(np.int32))], + 'desc_bprop': [Tensor(np.ones([2, 12]).astype(np.int32))], + 'skip': ['backward']}), + ('Flatten_2', { + 'block': NetForFlatten(), + 'desc_inputs': [Tensor(np.ones([8]).astype(np.int32)), Tensor(np.ones([8, 3]).astype(np.int32))], + 'desc_bprop': [Tensor(np.ones([8, 3]).astype(np.int32))], + 'skip': ['backward']}), + ('ArgmaxNet', { + 'block': ArgmaxNet(), + 'desc_inputs': [Tensor(np.array([[128, 32, 32, 64],[128, 32, 32, 64]]).astype(np.float16))], + 'desc_bprop': [Tensor(np.array([[128, 32, 32, 64],[128, 32, 32, 64]]).astype(np.float16))], + 'skip': ['backward']}), + ('ArgminNet', { + 'block': ArgminNet(), + 'desc_inputs': [Tensor(np.array([[128, 32, 32, 64],[128, 32, 32, 64]]).astype(np.float16))], + 'desc_bprop': [Tensor(np.array([[128, 32, 32, 64],[128, 32, 32, 64]]).astype(np.float16))], + 'skip': ['backward']}), + ('CumSumNet', { + 'block': CumSumNet(), + 'desc_const': [0], + 'desc_inputs': [Tensor(np.array([[3, 4, 6, 10],[1, 6, 7, 9],[4, 3, 8, 7],[1, 3, 7, 9]]).astype(np.float16))], + 'desc_bprop': [Tensor(np.array([[3, 4, 6, 10],[1, 6, 7, 9],[4, 3, 8, 7],[1, 3, 7, 9]]).astype(np.float16))]}), + ('OneHot', { + 'block': P.OneHot(), + 'desc_const': [3, Tensor(1.0, mstype.float32), Tensor(0.0, mstype.float32)], + 'desc_inputs': [Tensor(np.array([64]).astype(np.int32))], + 'desc_bprop': [[64, 2]]}), + ('ReduceProd_0', { + 'block': P.ReduceProd(), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[2]]}), + ('ReduceProd_1', { + 'block': P.ReduceProd(keep_dims=True), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[1, 2]]}), + ('CumProd', { + 'block': P.CumProd(), + 'desc_const': [0], + 'desc_inputs': [[3, 2]], + 'desc_bprop': [[3, 2]]}), + ('ApplyFtrl', { + 'block': P.ApplyFtrl(), + 'desc_const': [0.001, 0.0, 0.0, -0.5], + 'desc_inputs': [[3, 3], [3, 3], [3, 3], [3, 3]], + 'desc_bprop': [3, 3], + 'skip': ['backward']}), +] + +test_case_array_ops = [ + ('SpaceToDepth', { + 'block': P.SpaceToDepth(2), + 'desc_inputs': [[1, 3, 2, 2]], + 'desc_bprop': [[1, 12, 1, 1]]}), + ('DepthToSpace', { + 'block': P.DepthToSpace(2), + 'desc_inputs': [[1, 12, 1, 1]], + 'desc_bprop': [[1, 3, 2, 2]]}), + ('Split', { + 'block': P.Split(1, 2), + 'desc_inputs': [Tensor(np.array([[1, 1, 1, 1], [2, 2, 2, 2]]))], + 'skip': ['backward']}), + ('Argmax', { + 'block': P.Argmax(), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [0], + 'skip': ['backward']}), + ('Argmin', { + 'block': P.Argmin(), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [1], + 'skip': ['backward']}), + ('ArgMaxWithValue', { + 'block': P.ArgMaxWithValue(), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [[1], [1]], + 'skip': ['backward']}), + ('ArgMinWithValue', { + 'block': P.ArgMinWithValue(), + 'desc_inputs': [[128, 32, 32, 64]], + 'desc_bprop': [[1], [1]], + 'skip': ['backward']}), + ('Transpose_dim3', { + 'block': P.Transpose(), + 'desc_const': [(0, 2, 1)], + 'desc_inputs': [[1, 2, 3]], + 'desc_bprop': [[1, 3, 2]]}), + ('Transpose_dim4', { + 'block': P.Transpose(), + 'desc_const': [(0, 1, 2, 3)], + 'desc_inputs': [[1, 2, 3, 4]], + 'desc_bprop': [[1, 2, 4, 3]]}), + ('AddN', { + 'block': NetForTupleInput(P.AddN()), + 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]], + 'skip': ['backward']}), + ('Shape', { + 'block': P.Shape(), + 'desc_inputs': [[3, 3, 2, 2]], + 'skip': ['backward']}), + ('Reshape', { + 'block': P.Reshape(), + 'desc_const': [(64,)], + 'desc_inputs': [[64, 1]], + 'desc_bprop': [[64]]}), + ('Cast', { + 'block': P.Cast(), + 'desc_const': [mstype.int32], + 'desc_inputs': [[2, 3, 4, 5]], + 'desc_bprop': [Tensor(np.ones((2, 3, 3, 5)).astype(np.int32))]}), + ('ExpandDims', { + 'block': P.ExpandDims(), + 'desc_const': [0], + 'desc_inputs': [[2, 2]], + 'desc_bprop': [[1, 2, 2]]}), + ('ExpandDims_1', { + 'block': P.ExpandDims(), + 'desc_const': [-1], + 'desc_inputs': [[2, 2]], + 'desc_bprop': [[2, 2, 1]]}), + ('Squeeze', { + 'block': P.Squeeze(2), + 'desc_inputs': [[3, 2, 1]], + 'desc_bprop': [[3, 2]]}), + ('Squeeze_0', { + 'block': P.Squeeze(), + 'desc_inputs': [[3, 1, 2, 1]], + 'desc_bprop': [[3, 2]]}), + ('Squeeze_1', { + 'block': P.Squeeze(), + 'desc_inputs': [[1, 1, 1, 1]], + 'desc_bprop': [1.0], + 'skip': ['backward']}), + ('Squeeze_2', { + 'block': P.Squeeze((2, 3)), + 'desc_inputs': [[3, 2, 1, 1]], + 'desc_bprop': [[3, 2]]}), + ('Size', { + 'block': P.Size(), + 'desc_inputs': [[2, 3, 5]], + 'skip': ['backward']}), + ('Tile_0', { + 'block': P.Tile(), + 'desc_const': [(1, 2)], + 'desc_inputs': [[64, 1]], + 'desc_bprop': [[64, 2]]}), + ('Tile_1', { + 'block': P.Tile(), + 'desc_const': [(1, 1)], + 'desc_inputs': [[64, 1]], + 'desc_bprop': [[64, 1]]}), + ('Tile_2', { + 'block': P.Tile(), + 'desc_const': [(2, 1, 1, 2)], + 'desc_inputs': [[2, 2, 2]], + 'desc_bprop': [[2, 2, 2, 4]]}), + ('ConcatV2_0', { + 'block': P.Concat(), + 'desc_inputs': [ + (Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32)), + Tensor(np.array([[0, 1], [2, 1]]).astype(np.int32)))], + 'desc_bprop': [[4, 2]]}), + ('ConcatV2_1', { + 'block': P.Concat(axis=2), + 'desc_inputs': [(Tensor(np.array([[[0, 1, 2]], [[2, 1, 2]]]).astype(np.int32)), + Tensor(np.array([[[0, 1]], [[2, 1]]]).astype(np.int32)))], + 'desc_bprop': [[2, 1, 5]]}), + ('ConcatV2_2', { + 'block': NetForConcat(), + 'desc_inputs': [[2, 2]], + 'desc_bprop': [[4, 2]]}), + ('ConcatV2_3', { + 'block': NetForConcat1(), + 'desc_inputs': [[2, 2], [2, 2]], + 'desc_bprop': [[4, 2]]}), + ('ConcatV2_4', { + 'block': P.Concat(axis=0), + 'desc_inputs': [ + (Tensor(np.ones((3, 2, 3), np.float32)), + Tensor(np.ones((5, 2, 3), np.float32)), + Tensor(np.ones((6, 2, 3), np.float32)))], + 'desc_bprop': [[14, 2, 3]]}), + ('ConcatV2_5', { + 'block': P.Concat(axis=-1), + 'desc_inputs': [(Tensor(np.array([1], np.float32)), + Tensor(np.array([1], np.float32)), + Tensor(np.array([1], np.float32)))], + 'desc_bprop': [[3,]]}), +] + +test_case_other_ops = [ + ('ScalarLog', { + 'block': F.scalar_log, + 'desc_const': [0.0], + 'desc_inputs': [], + 'desc_bprop': [1], + 'skip': ['backward']}), + ('BoundingBoxEncode', { + 'block': P.BoundingBoxEncode(means=(0.0, 0.0, 0.0, 0.0), stds=(1.0, 1.0, 1.0, 1.0)), + 'desc_inputs': [[256, 4], [256, 4]], + 'desc_bprop': [[256, 4]], + 'skip': ['backward']}), + ('BoundingBoxDecode', { + 'block': P.BoundingBoxDecode(means=(0.0, 0.0, 0.0, 0.0), stds=(1.0, 1.0, 1.0, 1.0), max_shape=(768, 1280)), + 'desc_inputs': [[256, 4], [256, 4]], + 'desc_bprop': [[256, 4]], + 'skip': ['backward']}), + ('GatherNd', { + 'block': P.GatherNd(), + 'desc_inputs': (Tensor(np.ones((1, 3, 6, 6), np.float32)), + Tensor(np.ones((2, 4), np.int32))), + 'desc_bprop': [[2]]}), + ('ScatterNdUpdate', { + 'block': P.ScatterNdUpdate(), + 'desc_inputs': (Tensor(np.ones((2, 3), np.float32)), + Tensor(np.ones((2, 2), np.int32)), + Tensor(np.ones((2,), np.float32))), + 'desc_bprop': [[2, 3]]}), + ('ScatterNd', { + 'block': P.ScatterNd(), + 'desc_const': [(3, 3)], + 'desc_inputs': (Tensor(np.ones((2, 2), np.int32)), + Tensor(np.ones((2,), np.int32))), + 'desc_bprop': [[3, 3]]}), + ('SmoothL1Loss', { + 'block': P.SmoothL1Loss(), + 'desc_inputs': [[256, 4], [256, 4]], + 'desc_bprop': [[256, 4]]}), + ('IOU', { + 'block': P.IOU(), + 'desc_inputs': [Tensor(np.ones((256, 4), np.float16)), Tensor(np.ones((128, 4), np.float16))], + 'desc_bprop': [[128, 256]]}), + ('Summary', { + 'block': SummaryNet(), + 'desc_inputs': [Tensor(np.array([1.1]).astype(np.float32)), + Tensor(np.array([1.2]).astype(np.float32))], + 'skip': ['backward']}), +] + +test_case_lists = [test_case_nn_ops, test_case_math_ops, test_case_array_ops, test_case_other_ops] +test_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +test_exec_case = test_case + +test_backward_exec_case = filter(lambda x: 'skip' not in x[1] or + 'backward' not in x[1]['skip'], test_case) + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case + + +@mindspore_test(pipeline_for_compile_grad_ge_graph_for_case_by_case_config) +def test_backward_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_backward_exec_case + + +raise_set = [ + ('Cast_Error', { + 'block': (P.Cast(), {'exception': TypeError}), + 'desc_const': [mstype.int32], + 'desc_inputs': ['wrong input'], + 'desc_bprop': [Tensor(np.ones((2, 3, 3, 5)).astype(np.int32))]}), + ('Maximum_Error', { + 'block': (P.Maximum(), {'exception': TypeError}), + 'desc_const': [(1, 2, 3)], + 'desc_inputs': [[2, 3, 3, 5]], + 'desc_bprop': [[2, 3, 3, 5]]}), + ('Shape_error', { + 'block': (P.Shape(), {'exception': TypeError}), + 'desc_inputs': [(64, 1)], + 'desc_bprop': [[64]]}), + ('Flatten_Error', { + 'block': (NetForFlatten0D(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.array(0).astype(np.int32))], + 'desc_bprop': [Tensor(np.array(0).astype(np.int32))]}), +] + + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config_exception) +def test_check_exception(): + return raise_set diff --git a/tests/ut/python/ops/test_ops_check.py b/tests/ut/python/ops/test_ops_check.py new file mode 100644 index 00000000000..5083878daeb --- /dev/null +++ b/tests/ut/python/ops/test_ops_check.py @@ -0,0 +1,289 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test checking for some ops""" +import functools +import logging +import numpy as np +import pytest +from mindspore import nn +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.common.api import _executor +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.forward.verify_exception \ + import pipeline_for_verify_exception_for_case_by_case_config +logging.basicConfig(level=logging.WARNING) + + +class NetMissConstruct(nn.Cell): + """ NetMissConstruct definition """ + def __init__(self): + super(NetMissConstruct, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.flatten = P.Flatten() + + # pylint: disable=abstract-method + # TestCase: Mis-spelled 'construct' to 'construtc' + def construtc(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +def test_net_without_construct(): + """ test_net_without_construct """ + net = NetMissConstruct() + inp = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32)) + try: + _executor.compile(net, inp) + except RuntimeError as err: + if str(err).find("unsupported syntax 'Raise' at ") >= 0: + print(str(err)) + else: + raise err + + +class NetWithRaise(nn.Cell): + """ NetWithRaise definition """ + def __init__(self): + super(NetWithRaise, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + + # raise exception in method 'construct' + def construct(self, x): + raise 'exception in construct' + + +def test_net_with_raise(): + """ test_net_with_raise """ + net = NetWithRaise() + inp = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32)) + try: + _executor.compile(net, inp) + except RuntimeError as err: + if str(err).find("unsupported syntax 'Raise' at ") >= 0: + print(str(err)) + else: + raise err + + +class NetAddN(nn.Cell): + """net for test AddN""" + def __init__(self): + super(NetAddN, self).__init__() + self.net = P.AddN() + + def construct(self, x): + return self.net(x) + + +class NetSplit(nn.Cell): + "net for test Split" + def __init__(self): + super(NetSplit, self).__init__() + self.net = P.Split(1, 2) + + def construct(self, x): + return self.net(x) + + +class NetBatchMatMul(nn.Cell): + """net for test BatchMatMul""" + def __init__(self): + super(NetBatchMatMul, self).__init__() + self.op = P.BatchMatMul() + + def construct(self, x, y): + return self.op(x, y) + + +test_case_check_ops = [ + ('Conv_Padding_1', { + 'block': nn.Conv2d(1, 6, 5, pad_mode='same', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Padding_2', { + 'block': nn.Conv2d(1, 6, 5, pad_mode='valid', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Padding_3', { + 'block': nn.Conv2d(1, 6, 5, pad_mode='pad', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Padding_4', { + 'block': nn.Conv2d(1, 6, 5, pad_mode='pad', padding=7), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Bias_1', { + 'block': nn.Conv2d(1, 6, 5, has_bias=True, bias_init=Tensor(np.ones([6]).astype(np.float32))), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Bias_2', { + 'block': nn.Conv2d(1, 6, 5, has_bias=True, bias_init='zeros'), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Bias_3', { + 'block': nn.Conv2d(1, 6, 5, has_bias=False, bias_init='zeros'), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Bias_4', { + 'block': nn.Conv2d(1, 6, 5, has_bias=False, bias_init=Tensor(np.ones([6]).astype(np.float32))), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Dense_Bias_1', { + 'block': nn.Dense(1, 6, has_bias=True, bias_init=Tensor(np.ones([6]).astype(np.float32))), + 'desc_inputs': [Tensor(np.ones(shape=[6, 1]).astype(np.float32))]}), + ('Dense_Bias_2', { + 'block': nn.Dense(1, 6, has_bias=True, bias_init='zeros'), + 'desc_inputs': [Tensor(np.ones(shape=[6, 1]).astype(np.float32))]}), + ('Dense_Bias_3', { + 'block': nn.Dense(1, 6, has_bias=False, bias_init='zeros'), + 'desc_inputs': [Tensor(np.ones(shape=[6, 1]).astype(np.float32))]}), + ('Dense_Bias_4', { + 'block': nn.Dense(1, 6, has_bias=False, bias_init=Tensor(np.ones([6]).astype(np.float32))), + 'desc_inputs': [Tensor(np.ones(shape=[6, 1]).astype(np.float32))]}), + ('MaxPool2d_1', { + 'block': nn.MaxPool2d(5, pad_mode='same', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32))]}), + ('MaxPool2d_2', { + 'block': nn.MaxPool2d(5, pad_mode='valid', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32))]}), + ('AvgPool2d_1', { + 'block': nn.AvgPool2d(5, pad_mode='same', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32))]}), + ('AvgPool2d_2', { + 'block': nn.AvgPool2d(5, pad_mode='valid', padding=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32))]}), + ('Conv2D_1', { + 'block': P.Conv2D(1, 6, pad_mode='same', pad=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32)), + Tensor(np.ones(shape=[1, 5, 6, 6]).astype(np.float32))]}), + ('Conv2D_2', { + 'block': P.Conv2D(1, 6, pad_mode='valid', pad=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32)), + Tensor(np.ones(shape=[1, 5, 6, 6]).astype(np.float32))]}), + ('Conv2D_3', { + 'block': P.Conv2D(1, 6, pad_mode='pad', pad=0), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32)), + Tensor(np.ones(shape=[1, 5, 6, 6]).astype(np.float32))]}), + ('Conv2D_4', { + 'block': P.Conv2D(1, 6, pad_mode='pad', pad=7), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32)), + Tensor(np.ones(shape=[1, 5, 6, 6]).astype(np.float32))]}), + ('MatMul_1', { + 'block': P.MatMul(), + 'desc_inputs': [Tensor(np.ones(shape=[1, 3])), Tensor(np.ones(shape=[3, 4]))]}), + ('MatMul_2', { + 'block': P.BatchMatMul(), + 'desc_inputs': [Tensor(np.ones(shape=[5, 1, 5])), Tensor(np.ones(shape=[5, 5, 4]))]}), + ('MatMul_Transpose_1', { + 'block': P.MatMul(transpose_a=True), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1])), Tensor(np.ones(shape=[3, 4]))]}), + ('MatMul_Transpose_2', { + 'block': P.MatMul(transpose_b=True), + 'desc_inputs': [Tensor(np.ones(shape=[3, 2])), Tensor(np.ones(shape=[5, 2]))]}), + ('MatMul_Transpose_3', { + 'block': P.MatMul(transpose_a=True, transpose_b=True), + 'desc_inputs': [Tensor(np.ones(shape=[3, 2])), Tensor(np.ones(shape=[5, 3]))]}), + ('BatchMatMul', { + 'block': NetBatchMatMul(), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5])), Tensor(np.ones(shape=[3, 5, 4]))]}), +] + +test_case_lists = [test_case_check_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +import mindspore.context as context + +@non_graph_engine +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + context.set_context(mode=context.GRAPH_MODE) + return test_exec_case + + +raise_set = [ + ('Conv_Padding_1_Error', { + 'block': (lambda x: nn.Conv2d(1, 6, 5, pad_mode='same', padding=7), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv_Padding_2_Error', { + 'block': (lambda x: nn.Conv2d(1, 6, 5, pad_mode='same', padding=7), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[1, 1, 6, 5]).astype(np.float32))]}), + ('Conv2D_1_Error', { + 'block': (lambda x, y: P.Conv2D(1, 6, pad_mode='same', pad=7), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32)), + Tensor(np.ones(shape=[1, 5, 6, 6]).astype(np.float32))]}), + ('Conv2D_2_Error', { + 'block': (lambda x, y: P.Conv2D(1, 6, pad_mode='valid', pad=7), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[5, 5, 8, 8]).astype(np.float32)), + Tensor(np.ones(shape=[1, 5, 6, 6]).astype(np.float32))]}), + ('NetAddN_Error', { + 'block': (NetAddN(), {'exception': TypeError}), + 'desc_inputs': [(np.random.randn(1, 2, 3, 4).astype(np.float32), + np.random.randn(1, 2, 3, 4).astype(np.float32))]}), + ('AddN_Error', { + 'block': (P.AddN(), {'exception': TypeError}), + 'desc_inputs': [(np.random.randn(1, 2, 3, 4).astype(np.float32), + np.random.randn(1, 2, 3, 4).astype(np.float32))]}), + ('Splite_Error', { + 'block': (NetSplit(), {'exception': TypeError}), + 'desc_inputs': [None]}), + ('MatMul_1_Error', { + 'block': (P.MatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[5])), Tensor(np.ones(shape=[4]))]}), + ('MatMul_2_Error', { + 'block': (P.MatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[1, 5])), Tensor(np.ones(shape=[3, 4]))]}), + ('MatMul_3_Error', { + 'block': (P.MatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[1, 5])), Tensor(np.ones(shape=[5, 5, 4]))]}), + ('MatMul_Transpose_1_Error', { + 'block': (P.MatMul(transpose_a=True), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[1, 3])), Tensor(np.ones(shape=[3, 4]))]}), + ('MatMul_Transpose_2_Error', { + 'block': (P.MatMul(transpose_b=True), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 2])), Tensor(np.ones(shape=[2, 5]))]}), + ('MatMul_Transpose_3_Error', { + 'block': (P.MatMul(transpose_a=True, transpose_b=True), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 2])), Tensor(np.ones(shape=[3, 5]))]}), + ('BatchMatMul_1_Error', { + 'block': (P.BatchMatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[5])), Tensor(np.ones(shape=[4]))]}), + ('BatchMatMul_2_Error', { + 'block': (P.BatchMatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[1, 5])), Tensor(np.ones(shape=[3, 4]))]}), + ('BatchMatMul_3_Error', { + 'block': (P.BatchMatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5])), Tensor(np.ones(shape=[3, 3, 4]))]}), + ('BatchMatMul_4_Error', { + 'block': (P.BatchMatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5])), Tensor(np.ones(shape=[1, 3, 5, 4]))]}), + ('BatchMatMul_5_Error', { + 'block': (P.BatchMatMul(), {'exception': ValueError}), + 'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5])), Tensor(np.ones(shape=[2, 5, 4]))]}), +] + + +@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config) +def test_check_exception(): + return raise_set diff --git a/tests/ut/python/ops/test_ops_reid.py b/tests/ut/python/ops/test_ops_reid.py new file mode 100644 index 00000000000..c1176808a87 --- /dev/null +++ b/tests/ut/python/ops/test_ops_reid.py @@ -0,0 +1,170 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Activations """ +import functools +import numpy as np + +from mindspore.ops import operations as P +import mindspore.nn as nn +from ....ops_common import convert + +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward\ +import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.gradient.compile_gradient\ +import pipeline_for_compile_grad_ge_graph_for_case_by_case_config + + +class SeqConvBnRelu(nn.Cell): + """ SeqConvBnRelu definition """ + def __init__(self, in_ch, out_ch): + super(SeqConvBnRelu, self).__init__() + self.conv = nn.Conv2d(in_ch, out_ch, 3) + self.bn = nn.BatchNorm2d(out_ch) + self.relu = P.ReLU() + + def construct(self, input_x): + return self.relu(self.bn(self.conv(input_x))) + + +test_case_reid_ops = [ + ('ReduceMax', { + 'block': P.ReduceMax(keep_dims=False), + 'desc_const': [(1,)], + 'desc_inputs': [convert([32, 32], np.float16)], + 'desc_bprop': [convert([32], np.float16)], + 'skip':[]}), + ('ReduceMin', { + 'block': P.ReduceMin(), + 'desc_const': [(1,)], + 'desc_inputs': [[32, 32]], + 'desc_bprop': [[32]], + 'skip':[]}), + ('ReduceMean', { + 'block': P.ReduceMean(keep_dims=True), + 'desc_const': [(1, 2)], + 'desc_inputs': [[32, 4, 4]], + 'desc_bprop': [[32, 1, 1]]}), + ('Log', { + 'block': P.Log(), + 'desc_inputs': [[4, 128, 1024]], + 'desc_bprop': [[4, 128, 1024]], + 'skip':['backward']}), # check backward error + ('Reciprocal', { + 'block': P.Reciprocal(), + 'desc_inputs': [[4, 128, 1024]], + 'desc_bprop': [[4, 128, 1024]], + 'skip':['backward']}), + ('FloorDiv', { + 'block': P.FloorDiv(), + 'desc_inputs': [[4, 128, 1024], [4, 128, 1024]], + 'desc_bprop': [[4, 128, 1024]]}), + ('Sigmoid', { + 'block': P.Sigmoid(), + 'desc_inputs': [[4, 128, 1024]], + 'desc_bprop': [[4, 128, 1024]]}), + ('Softmax', { + 'block': P.Softmax(), + 'desc_inputs': [[1, 16]], + 'desc_bprop': [[1, 16]], + 'skip':['backward']}), # check backward error + ('Softmax', { + 'block': P.Softmax(axis=(0, 1)), + 'desc_inputs': [[1, 16]], + 'desc_bprop': [[1, 16]], + 'skip':['backward']}), + ('L2Normalize', { + 'block': P.L2Normalize(), + 'desc_inputs': [[4, 128, 1024]], + 'desc_bprop': [[4, 128, 1024]]}), + ('ReLU', { + 'block': P.ReLU(), + 'desc_inputs': [[64, 64, 112, 112]], + 'desc_bprop': [[64, 64, 112, 112]]}), + ('SeqConvBnRelu', { + 'block': SeqConvBnRelu(3, 64), + 'desc_inputs': [[64, 3, 112, 112]], + 'desc_bprop': [[64, 64, 112, 112]]}), + ('PReluCell', { + 'block': nn.PReLU(1, [np.float32(0.25)]), + 'desc_inputs': [[128, 64, 112, 112]], + 'desc_bprop': [[128, 64, 112, 112]]}), + ('PRelu', { + 'block': P.PReLU(), + 'desc_inputs': [[128, 64, 112, 112], [64,]], + 'desc_bprop': [[128, 64, 112, 112]]}), + ('Cos', { + 'block': P.Cos(), + 'desc_inputs': [[8, 16]], + 'desc_bprop': [[8, 16]]}), + ('ACos', { + 'block': P.ACos(), + 'desc_inputs': [[8, 16]], + 'desc_bprop': [[8, 16]]}), + ('Exp', { + 'block': P.Exp(), + 'desc_inputs': [[256, 8]], + 'desc_bprop': [[256, 8]]}), + ('Pow', { + 'block': P.Pow(), # 输入有标量插件产生了段错误。 + 'desc_const': [2.0], + 'desc_inputs': [[1, 512]], + 'desc_bprop': [[1, 512]]}), + ('LogicalNot', { + 'block': P.LogicalNot(), + 'desc_inputs': [convert([256], np.bool_)], + 'desc_bprop': [[256]]}), # 自定义算子 input bool没转换，gongchen提单。 + ('Equal', { + 'block': P.Equal(), + 'desc_inputs': [convert([256], np.float16), convert([256], np.float16)], + 'desc_bprop': [[256]]}), + ('Greater', { + 'block': P.Greater(), + 'desc_inputs': [convert([256], np.float16), convert([256], np.float16)], + 'desc_bprop': [[256]]}), + ('Dropout', { + 'block': nn.Dropout(), + 'desc_inputs': [[1, 512, 7, 7]], + 'desc_bprop': [[1, 512, 7, 7]]}), # 输入有标量插件产生了段错误。 + ('MatMul', { + 'block': P.MatMul(), + 'desc_inputs': [[64, 512], [512, 64]], # fp16不行。很有问题。 + 'desc_bprop': [[64, 64]]}), + ('Maximum', { + 'block': P.Maximum(), + 'desc_inputs': [[64, 1], [64, 1]], + 'desc_bprop': [[64, 1]]}), +] + +test_case_lists = [test_case_reid_ops] +test_case = functools.reduce(lambda x, y: x + y, test_case_lists) +# use -k to select certain testcast +# pytest tests/python/ops/test_ops.py::test_backward -k LayerNorm + + +test_exec_case = filter(lambda x: 'skip' not in x[1] or + 'exec' not in x[1]['skip'], test_case) + +test_backward_exec_case = filter(lambda x: 'skip' not in x[1] or + 'backward' not in x[1]['skip'] and 'backward_exec' + not in x[1]['skip'], test_case) + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_exec(): + return test_exec_case + +@mindspore_test(pipeline_for_compile_grad_ge_graph_for_case_by_case_config) +def test_backward_exec(): + return test_backward_exec_case diff --git a/tests/ut/python/ops/test_python_operators.py b/tests/ut/python/ops/test_python_operators.py new file mode 100644 index 00000000000..d6c6c037606 --- /dev/null +++ b/tests/ut/python/ops/test_python_operators.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test control ops """ +import functools +import numpy as np +import mindspore as ms +from mindspore import nn +from mindspore import Tensor +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config + +context.set_context(mode=context.GRAPH_MODE, save_graphs=True) + +class ComparisonOpsNet(nn.Cell): + def __init__(self): + super(ComparisonOpsNet, self).__init__() + def construct(self, x, y): + ret = x <= y + return ret + +class LogicalNumberOpsNet(nn.Cell): + def __init__(self): + super(LogicalNumberOpsNet, self).__init__() + self.cond = True + self.one = 0 + self.zero = 0.0 + def construct(self, x, y): + if self.cond and self.one or self.zero: + return x + y + return x - y + +class LogicalTensorOpsNet(nn.Cell): + def __init__(self): + """""" + super(LogicalTensorOpsNet, self).__init__() + self.const_true = Tensor(True, dtype=mstype.bool_) + def construct(self, x, y): + ret = x and y and (y or self.const_true) + return ret + + +test_case_ops = [ + ('CompareOpsNet', { + 'block': ComparisonOpsNet(), + 'desc_inputs': [Tensor(np.ones([6, 9, 10]), dtype=mstype.float32), + Tensor(np.zeros([6, 9, 10]), dtype=mstype.float32)]}), + ('LogicalNumberOps', { + 'block': LogicalNumberOpsNet(), + 'desc_inputs': [Tensor(np.ones([6, 9, 10]), dtype=mstype.float32), + Tensor(np.zeros([6, 9, 10]), dtype=mstype.float32)]}), + ('LogicalTensorOps', { + 'block': LogicalTensorOpsNet(), + 'desc_inputs': [Tensor(np.ones([6, 9, 10]).astype(np.bool_), dtype=mstype.bool_), + Tensor(np.zeros([6, 9, 10]).astype(np.bool_), dtype=mstype.bool_)]}), +] + +test_case_lists = [test_case_ops] +test_exec_case = functools.reduce(lambda x, y: x + y, test_case_lists) + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_compile(): + return test_exec_case \ No newline at end of file diff --git a/tests/ut/python/ops/test_tensor_slice.py b/tests/ut/python/ops/test_tensor_slice.py new file mode 100644 index 00000000000..6200d4e163d --- /dev/null +++ b/tests/ut/python/ops/test_tensor_slice.py @@ -0,0 +1,140 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_tensor_slice """ +import numpy as np +import pytest + +from mindspore import Tensor +from mindspore import context +from mindspore.nn import Cell + +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config + + +class NetWorkSlicePositive(Cell): + def __init__(self): + super(NetWorkSlicePositive, self).__init__() + self.tensor_ret0 = Tensor(np.ones([1, 2, 2], np.int32)) + self.tensor_ret1 = Tensor(np.ones([4, 7, 4], np.int32)) + self.tensor_ret2 = Tensor(np.ones([6, 8, 10], np.int32)) + self.tensor_ret3 = Tensor(np.ones([3, 8, 10], np.int32)) + + def construct(self, tensor): + ret0 = tensor[3:4:3, 1:5:2, 3:6:2] + self.tensor_ret0 + ret1 = tensor[-6:4:1, 7:-8:-1, ::3] + self.tensor_ret1 + ret2 = tensor[::, ::, ::] + self.tensor_ret2 + ret3 = tensor[::2] + self.tensor_ret3 + return ret0, ret1, ret2, ret3 + + +class NetWorkReduceDimension(Cell): + def __init__(self): + super(NetWorkReduceDimension, self).__init__() + self.tensor_ret0 = Tensor(np.ones([2, 4, 1], np.int32)) + self.tensor_ret1 = Tensor(np.ones([3, 4], np.int32)) + self.tensor_ret2 = Tensor(np.ones([6, 8], np.int32)) + self.tensor_ret3 = Tensor(np.array(8, np.int32)) + self.tensor_ret4 = Tensor(np.ones([8, 10], np.int32)) + + def construct(self, tensor): + ret0 = tensor[0:6:3, 1:5:1, 3:5:2] + self.tensor_ret0 + ret1 = tensor[::2, 1, ::3] + self.tensor_ret1 + ret2 = tensor[::, ::, 0] + self.tensor_ret2 + ret3 = tensor[3, 2, 5] + self.tensor_ret3 + ret4 = tensor[1] + self.tensor_ret4 + return ret0, ret1, ret2, ret3, ret4 + + +class NetWorkStepNegative(Cell): + def __init__(self): + super(NetWorkStepNegative, self).__init__() + self.tensor_ret = Tensor(np.ones([6, 5, 10], np.int32)) + + def construct(self, tensor): + ret = tensor[::1, -5::, ::-1] + self.tensor_ret + return ret + + +class NetWorkReduceToScalar(Cell): + def __init__(self): + super(NetWorkReduceToScalar, self).__init__() + self.tensor_ret = Tensor(np.array(9, np.int32)) + + def construct(self, tensor): + ret = tensor[2, 3, 4] + self.tensor_ret + return ret + + +test_cases = [ + ('SlicePositive', { + 'block': NetWorkSlicePositive(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))], + }), + ('SliceReduceDimension', { + 'block': NetWorkReduceDimension(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))], + }), + ('SliceNegative', { + 'block': NetWorkStepNegative(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))], + }), + ('SliceReduceToScalar', { + 'block': NetWorkReduceToScalar(), + 'desc_inputs': [Tensor(np.ones([6, 8, 10], np.int32))], + }), + +] + + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_compile(): + context.set_context(mode=context.GRAPH_MODE) + return test_cases + + +def test_tensor_slice_reduce_out_of_bounds_neg(): + class NetWork(Cell): + def __init__(self): + super(NetWork, self).__init__() + self.tensor_ret = Tensor(np.array(9, np.int32)) + + def construct(self, tensor): + ret = tensor[-7, 3, 4] + return ret + + input_tensor = Tensor(np.ones([6, 8, 10], np.int32)) + net = NetWork() + with pytest.raises(ValueError) as ex: + net(input_tensor) + assert "The `begin[0]` should be an int and must greater or equal to -6, but got -7" in str(ex.value) + + +def test_tensor_slice_reduce_out_of_bounds_positive(): + class NetWork(Cell): + def __init__(self): + super(NetWork, self).__init__() + self.tensor_ret = Tensor(np.array(9, np.int32)) + + def construct(self, tensor): + ret = tensor[6, 3, 4] + return ret + + input_tensor = Tensor(np.ones([6, 8, 10], np.int32)) + net = NetWork() + with pytest.raises(ValueError) as ex: + net(input_tensor) + assert "The `begin[0]` should be an int and must less than 6, but got 6" in str(ex.value) diff --git a/tests/ut/python/ops/test_tuple_slice.py b/tests/ut/python/ops/test_tuple_slice.py new file mode 100644 index 00000000000..ea5112995c0 --- /dev/null +++ b/tests/ut/python/ops/test_tuple_slice.py @@ -0,0 +1,127 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_tuple_slice """ +import numpy as np +import pytest + +from mindspore import Tensor +from mindspore.nn import Cell +import mindspore.ops.operations as P + +from ....mindspore_test_framework.mindspore_test import mindspore_test +from ....mindspore_test_framework.pipeline.forward.compile_forward \ + import pipeline_for_compile_forward_ge_graph_for_case_by_case_config +from ....mindspore_test_framework.pipeline.forward.verify_exception \ + import pipeline_for_verify_exception_for_case_by_case_config + + +class NetWork_1(Cell): + """ NetWork_1 definition """ + def __init__(self): + super(NetWork_1, self).__init__() + self.addN = P.AddN() + + def construct(self, tensor_tuple): + tensor_tuple_slice0 = tensor_tuple[:] + tensor_tuple_slice1 = tensor_tuple[:3] + tensor_tuple_slice2 = tensor_tuple[1:] + tensor_tuple_slice3 = tensor_tuple[2:5:1] + sum0 = self.addN(tensor_tuple_slice0) + sum1 = self.addN(tensor_tuple_slice1) + sum2 = self.addN(tensor_tuple_slice2) + sum3 = self.addN(tensor_tuple_slice3) + ret = sum0 + sum1 + sum2 + sum3 + return ret + + +class NetWork_2(Cell): + """ NetWork_2 definition """ + def __init__(self): + super(NetWork_2, self).__init__() + self.addN = P.AddN() + + def construct(self, tensor_tuple): + tensor_tuple_slice0 = tensor_tuple[::-1] + tensor_tuple_slice1 = tensor_tuple[-1::-1] + tensor_tuple_slice2 = tensor_tuple[:-4:-1] + tensor_tuple_slice3 = tensor_tuple[-6:3] + tensor_tuple_slice4 = tensor_tuple[-1:-6:-2] + sum0 = self.addN(tensor_tuple_slice0) + sum1 = self.addN(tensor_tuple_slice1) + sum2 = self.addN(tensor_tuple_slice2) + sum3 = self.addN(tensor_tuple_slice3) + sum4 = self.addN(tensor_tuple_slice4) + ret = sum0 + sum1 + sum2 + sum3 + sum4 + return ret + + +class NetWork_3(Cell): + """ NetWork_3 definition """ + def __init__(self): + super(NetWork_3, self).__init__() + self.addN = P.AddN() + + def construct(self, tensor_tuple, start, stop, step=1): + tensor_tuple_slice0 = tensor_tuple[start:stop:step] + res = self.addN(tensor_tuple_slice0) + return res + + +test_cases = [ + ('SlicePositive', { + 'block': NetWork_1(), + 'desc_inputs': [(Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.zeros([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.zeros([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)))], + }), + ('SliceNegative', { + 'block': NetWork_2(), + 'desc_inputs': [(Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.zeros([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.zeros([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)))], + }), +] + + +test_cases_for_verify_exception = [ + ('SliceStartCross', { + 'block': (NetWork_3(), {'exception': RuntimeError}), + 'desc_inputs': [*(Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.zeros([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)))], + }), + ('SliceStepZero', { + 'block': (NetWork_3(), {'exception': RuntimeError}), + 'desc_inputs': [*(Tensor(np.ones([2, 3, 4], np.int32)), + Tensor(np.zeros([2, 3, 4], np.int32)), + Tensor(np.ones([2, 3, 4], np.int32)))], + }), +] + + +@mindspore_test(pipeline_for_compile_forward_ge_graph_for_case_by_case_config) +def test_compile(): + return test_cases + + +@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config) +def test_check_exception(): + return test_cases_for_verify_exception diff --git a/tests/ut/python/optimizer/__init__.py b/tests/ut/python/optimizer/__init__.py new file mode 100644 index 00000000000..6d831431f51 --- /dev/null +++ b/tests/ut/python/optimizer/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" init vm impl """ +from ....vm_impl import vm diff --git a/tests/ut/python/optimizer/test_debug_location.py b/tests/ut/python/optimizer/test_debug_location.py new file mode 100644 index 00000000000..80793f37a13 --- /dev/null +++ b/tests/ut/python/optimizer/test_debug_location.py @@ -0,0 +1,172 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +import pytest +from mindspore import context +from mindspore import Tensor, Parameter +from mindspore.nn.wrap.cell_wrapper import WithLossCell +from mindspore.train.loss_scale_manager import FixedLossScaleManager, DynamicLossScaleManager +from mindspore.nn.wrap.loss_scale import TrainOneStepWithLossScaleCell +from mindspore.ops import operations as P +from mindspore.nn.optim import Momentum +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype +from mindspore.train import Model +from ....dataset_mock import MindData +from mindspore.nn.optim import Lamb +from mindspore.ops._utils import _get_broadcast_shape +from mindspore.ops.primitive import Primitive, PrimitiveWithInfer, prim_attr_register +from mindspore.ops._grad.grad_base import bprop_getters +from mindspore.ops._grad.grad_math_ops import binop_grad_common + +context.set_context(mode=context.GRAPH_MODE) + +class MockNeg(PrimitiveWithInfer): + @prim_attr_register + def __init__(self): + """init MockNeg""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + raise TypeError("InferError") + return input_x + +class MockSub(PrimitiveWithInfer): + @prim_attr_register + def __init__(self): + """init MockSub""" + self.init_prim_io_names(inputs=['x', 'y'], outputs=['output']) + + def infer_shape(self, x_shape, y_shape): + return _get_broadcast_shape(x_shape, y_shape) + + def infer_dtype(self, x_dtype, y_dtype): + return x_dtype + +@bprop_getters.register(MockSub) +def get_bprop_mock_sub(self): + """Grad definition for `MockSub` operation.""" + neg_func = MockNeg() + + def bprop(x, y, out, dout): + return binop_grad_common(x, y, dout, neg_func(dout)) + return bprop + +class Net(nn.Cell): + def __init__(self, in_features, out_features): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([out_features, in_features]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([out_features]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.add = P.TensorAdd() + + def construct(self, input): + output = self.add(self.matmul(input, self.weight), self.bias) + return output + +class NetFP16(nn.Cell): + def __init__(self, in_features, out_features): + super(NetFP16, self).__init__() + self.weight = Parameter(Tensor(np.ones([out_features, in_features]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([out_features]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.add = P.TensorAdd() + self.cast = P.Cast() + + def construct(self, input): + output = self.cast(self.add(self.matmul(self.cast(input, mstype.float16), self.cast(self.weight, mstype.float16)), + self.cast(self.bias, mstype.float16)), mstype.float32) + return output + +def get_axis(x): + shape = F.shape(x) + length = F.tuple_len(shape) + perm = F.make_range(0, length) + return perm + +class MSELoss(nn.Cell): + def __init__(self): + super(MSELoss, self).__init__() + self.reduce_sum = P.ReduceSum() + self.square = P.Square() + self.reduce_mean = P.ReduceMean() + self.sub = MockSub() + def construct(self, data, label): + diff = self.sub(data, label) + return self.reduce_mean(self.square(diff), get_axis(diff)) + +class NegCell(nn.Cell): + def __init__(self): + super(NegCell, self).__init__() + self.neg = MockNeg() + def construct(self, x): + return self.neg(x) + +class Net3(nn.Cell): + def __init__(self): + super().__init__() + self.tuple = (NegCell(), nn.ReLU()) + + def construct(self, x): + for op in self.tuple: + x = op(x) + return x + + +def test_op_forward_infererror(): + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + net = Net3() + with pytest.raises(TypeError) as e: + net(input_me) + + +class SequenceNet(nn.Cell): + def __init__(self): + super().__init__() + self.seq = nn.SequentialCell([nn.AvgPool2d(3, 1), nn.ReLU(), nn.Flatten()]) + + def construct(self, x): + x = self.seq(x) + bbb + return x + +def test_sequential_resolve_error(): + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + net = SequenceNet() + with pytest.raises(RuntimeError) as e: + net(input_me) + +def test_compile_grad_error(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + scale_manager = DynamicLossScaleManager() + update_cell = scale_manager.get_update_cell() + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell = update_cell) + train_network.set_train() + with pytest.raises(TypeError) as e: + train_network(inputs, label) + print (e) + diff --git a/tests/ut/python/optimizer/test_optimize_with_loss_scale.py b/tests/ut/python/optimizer/test_optimize_with_loss_scale.py new file mode 100644 index 00000000000..b61181adda3 --- /dev/null +++ b/tests/ut/python/optimizer/test_optimize_with_loss_scale.py @@ -0,0 +1,276 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.nn as nn +from mindspore import context +from mindspore import Tensor, Parameter +from mindspore.nn.wrap.cell_wrapper import WithLossCell +from mindspore.nn.wrap.loss_scale import TrainOneStepWithLossScaleCell +from mindspore.train.loss_scale_manager import FixedLossScaleManager, DynamicLossScaleManager +from mindspore.ops import operations as P +from mindspore.nn.optim import Momentum, Adam +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype +from mindspore.train import Model +from ....dataset_mock import MindData +from mindspore.nn.optim import Lamb + +context.set_context(mode=context.GRAPH_MODE) + + +class MindDataSet(MindData): + def __init__(self, dataset_types, dataset_shapes): + super(MindDataSet, self).__init__(size=2, batch_size=32, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + def __next__(self): + if self._size < self._iter_num: + raise StopIteration + self._iter_num += 1 + next = [] + for shape, type in zip(self._output_shapes, self._np_types): + next.append(Tensor(np.ones(shape).astype(type))) + return tuple(next) + +class Net(nn.Cell): + def __init__(self, in_features, out_features): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([out_features, in_features]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([out_features]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.add = P.TensorAdd() + + def construct(self, input): + output = self.add(self.matmul(input, self.weight), self.bias) + return output + +class NetFP16(nn.Cell): + def __init__(self, in_features, out_features): + super(NetFP16, self).__init__() + self.weight = Parameter(Tensor(np.ones([out_features, in_features]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([out_features]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.add = P.TensorAdd() + self.cast = P.Cast() + + def construct(self, input): + output = self.cast(self.add(self.matmul(self.cast(input, mstype.float16), self.cast(self.weight, mstype.float16)), + self.cast(self.bias, mstype.float16)), mstype.float32) + return output + +def get_axis(x): + shape_op = P.Shape() + shape = shape_op(x) + length = F.tuple_len(shape) + perm = F.make_range(0, length) + return perm + +class MSELoss(nn.Cell): + def __init__(self): + super(MSELoss, self).__init__() + self.reduce_sum = P.ReduceSum() + self.square = P.Square() + self.reduce_mean = P.ReduceMean() + + def construct(self, data, label): + diff = data - label + return self.reduce_mean(self.square(diff), get_axis(diff)) + +def test_momentum_compile(): + inputs = Tensor(np.ones([15, 1]).astype(np.float32)) + label = Tensor(np.zeros([15, 1]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), 1.0), dtype=mstype.float32) + net = Net(1, 1) + + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_fp16_not_overflow(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), 1.0), dtype=mstype.float32) + net = NetFP16(16, 16) + + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_fp16_lr_overflow(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_fp16_overflow(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) + net = NetFP16(16, 16) + + loss = MSELoss() + optimizer = Lamb(net.trainable_params(), decay_steps=10, warmup_steps=5) + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_fp16_lr_overflow_with_lossscale_update(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + scale_manager = DynamicLossScaleManager() + manager = scale_manager.get_update_cell() + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell=manager) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_f16_model_train(): + dataset_types = (np.float32, np.float32) + dataset_shapes = ((16, 16), (16, 16)) + + dataset = MindDataSet(dataset_types, dataset_shapes) + net = NetFP16(16, 16) + net.set_train() + + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, loss_fn=loss, optimizer=optimizer, metrics=None) + model.train(2, dataset, dataset_sink_mode=False) + + +def test_compile_f16_model_train_fixed(): + dataset_types = (np.float32, np.float32) + dataset_shapes = ((16, 16), (16, 16)) + + dataset = MindDataSet(dataset_types, dataset_shapes) + net = NetFP16(16, 16) + net.set_train() + scale_manager = FixedLossScaleManager() + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, loss_fn=loss, optimizer=optimizer, metrics=None, loss_scale_manager=scale_manager) + model.train(2, dataset) + + +def test_compile_fp16_lr_overflow_fixed_feed(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + scale_manager = FixedLossScaleManager() + update_cell = scale_manager.get_update_cell() + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell = update_cell) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_fp16_lr_overflow_dynamic_feed(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + scale_manager = DynamicLossScaleManager() + update_cell = scale_manager.get_update_cell() + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell = update_cell) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) + +def test_compile_fp16_lr_overflow_fixed_graph(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + scale_manager = FixedLossScaleManager(drop_overflow_update=True) + update_cell = scale_manager.get_update_cell() + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell = update_cell) + train_network.set_train() + output = train_network(inputs, label) + print("the result is ", output) + +def test_compile_fp16_lr_overflow_dynamic_graph(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + lr = Tensor(np.ones([1], np.float32) * 0.1) + net = NetFP16(16, 16) + loss = MSELoss() + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = WithLossCell(net, loss) + scale_manager = DynamicLossScaleManager() + update_cell = scale_manager.get_update_cell() + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_update_cell = update_cell) + train_network.set_train() + output = train_network(inputs, label) + print("the result is ", output) + +def test_adam_compile(): + inputs = Tensor(np.ones([15, 1]).astype(np.float32)) + label = Tensor(np.zeros([15, 1]).astype(np.float32)) + scaling_sens = Tensor(np.full((1), 1.0), dtype=mstype.float32) + net = Net(1, 1) + + loss = MSELoss() + optimizer = Adam(net.trainable_params(), learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, + use_nesterov=False, weight_decay=0.0, loss_scale=1.0) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) + train_network.set_train() + output = train_network(inputs, label, scaling_sens) + print("the result is ", output) diff --git a/tests/ut/python/parallel/__init__.py b/tests/ut/python/parallel/__init__.py new file mode 100644 index 00000000000..ffe03896abc --- /dev/null +++ b/tests/ut/python/parallel/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mindspore.context as context + + +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) + + +def teardown_module(): + context.reset_auto_parallel_context() diff --git a/tests/ut/python/parallel/parallel_end_to_end/add_relu/_test_add_relu_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/add_relu/_test_add_relu_parallel_4p.py new file mode 100644 index 00000000000..2d69efa27c6 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/add_relu/_test_add_relu_parallel_4p.py @@ -0,0 +1,167 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class AddRelu(Cell): + def __init__(self, strategy0=None, strategy1=None): + super(AddRelu, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.relu = P.ReLU(strategy=strategy1) + + def construct(self, x, z): + out = self.add(x, z) + return self.relu(out) + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, output_grad): + return grad_all_with_sens(self.network)(x, y, output_grad) + + +class AddReluFactory: + def __init__(self, input_shape, strategy0, strategy1): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = 1.0 + self.output_grad_np = np.reshape((np.arange(0, size)%(number_range-10) - number_range/2)*0.1, input_shape).astype(np.float32) + self.strategy0 = strategy0 + self.strategy1 = strategy1 + need_dev_num = 1 + need_dev_num_ = 1 + for s in strategy0[1]: + need_dev_num = need_dev_num*s + for s in strategy1[1]: + need_dev_num_ = need_dev_num_*s + self.x_id = device_id%need_dev_num + self.y_id = device_id%need_dev_num + self.out_id = device_id%need_dev_num_ + + def forward_mindspore_impl(self): + net = AddRelu() + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + out = net(x, y) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + net = AddRelu(strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(self.input_np2, ms.float32) + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + output_grad = Tensor(self.output_grad_np) + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + net = AddRelu() + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, output_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + output_grads = self.get_parallel_blocks(self.output_grad_np, self.strategy1[1]) + output_grad = Tensor(output_grads[self.out_id]) + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + net = AddRelu(strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(self.input_np2, ms.float32) + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad], parallel_inputs_run=[x1, y1, output_grad]) + return input_grad + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.strategy1[1]) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.0001, 0.001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + assert np.allclose(input_grad_mindspore1, input_grad_mindspore_parallel1, 0.0001, 0.0001) + +@pytest.mark.reid_forward +def test_reid_add_relu_input_256_64(): + stra0 = (0,(2,2),()) + stra1 = (0,(2,2)) + fact = AddReluFactory(input_shape=(256, 64), strategy0=stra0, strategy1=stra1) + fact.forward_cmp() + +@pytest.mark.reid_grad +def test_reid_grad_add_relu_input_256_64(): + stra0 = (0,(2,2),()) + stra1 = (0,(2,2)) + fact = AddReluFactory(input_shape=(256, 64), strategy0=stra0, strategy1=stra1) + fact.grad_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/add_relu/add_relu_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/add_relu/add_relu_parallel_4p.sh new file mode 100644 index 00000000000..1b9df575a70 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/add_relu/add_relu_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_add_relu_parallel_4p.py>../../log/test_add_relu_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/batch_parallel/_test_conv2d_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/batch_parallel/_test_conv2d_parallel_4p.py new file mode 100644 index 00000000000..9499e2dd150 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/batch_parallel/_test_conv2d_parallel_4p.py @@ -0,0 +1,348 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import mindspore.communication.management as distributedTool +from numpy import allclose +from mindspore import context +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore._checkparam import check_bool, twice +from mindspore.common.initializer import initializer +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + + +class _Conv(Cell): + r"""Applies a N-D convolution over an input signal composed of several input + planes. + """ + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init): + super(_Conv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.pad_mode = pad_mode + self.padding = padding + self.dilation = dilation + self.group = group + self.has_bias = has_bias + if not (isinstance(in_channels, int) and in_channels > 0): + raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op passed ' + +str(in_channels)+ ', should be a int and greater than 0.') + if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ + (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ + kernel_size[0] < 1 or kernel_size[1] < 1: + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + +str(self.kernel_size)+', should be a int or tuple and equal to or greater than 1.') + if in_channels % group != 0: + raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op must be divisible by ' + 'attr \'group\' of \'Conv2D\' Op.') + if out_channels % group != 0: + raise ValueError('Attr \'out_channels\' of \'Conv2D\' Op must be divisible by ' + 'attr \'group\' of \'Conv2D\' Op.') + + self.weight = Parameter(initializer( + weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') + + if check_bool(has_bias): + self.bias = Parameter(initializer( + bias_init, [out_channels]), name='bias') + else: + if bias_init != 'zeros': + print("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") + self.bias = None + + def construct(self, *inputs): + raise NotImplementedError + + +class Conv2d(_Conv): + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros', + strategy=None): + kernel_size = twice(kernel_size) + super(Conv2d, self).__init__( + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + self.add = P.TensorAdd(strategy) + self.conv2d = P.Conv2D(out_channel=self.out_channels, + kernel_size=self.kernel_size, + mode=1, + pad_mode=self.pad_mode, + pad=self.padding, + stride=self.stride, + dilation=self.dilation, + group=self.group, + strategy=None) + self.bias_add = P.BiasAdd() + + def construct(self, input1, input2): + x = self.add(input1, input2) + if self.has_bias: + return self.bias_add(self.conv2d(x, self.weight), + self.bias) + return self.conv2d(x, self.weight) + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, input1, input2, output_grad): + return grad_all_with_sens(self.network)(input1, input2, output_grad) + + +class Conv2dFactory: + def __init__(self, input_shape, filter_shape, stride, pad_mode, padding, dilation, group, has_bias): + self.in_n, self.in_c, self.in_h, self.in_w = input_shape + self.out_c, self.kernel_c, self.kernel_h, self.kernel_w = filter_shape + self.stride = stride + self.pad_mode = pad_mode + self.padding = padding + self.dilation = dilation + self.group = group + self.strategy0 = (0,(4,1,1,1),(1,1,1,1)) + prefix = "" + input_size = 1 + filter_size = 1 + for s in input_shape: + prefix = prefix + str(s) + "_" + input_size = input_size*s + self.prefix = prefix + for s in filter_shape: + filter_size = filter_size*s + number_range1 = min(10, input_size) + number_range2 = min(10, filter_size) + self.input_np1 = np.reshape(np.arange(0, input_size)%number_range1 - number_range1/2, input_shape).astype(np.float16) + self.input_np2 = np.reshape(np.arange(0, input_size)%number_range1 - number_range1/4, input_shape).astype(np.float16) + self.weight_np = np.reshape(np.arange(0, filter_size)%number_range2 - number_range2/2, filter_shape).astype(np.float16) + self.has_bias = has_bias + if self.has_bias is True: + self.bias_np = np.arange(0, self.out_c).astype(np.float16) + + self.out_shape = (128,64,56,56) + out_size = 1 + for s in self.out_shape: + out_size = out_size*s + number_range3 = min(10, out_size) + self.output_grad_np = np.reshape(np.arange(0, out_size)%number_range3 - number_range3/2, self.out_shape).astype(np.float16) + self.x_id = device_id%4 + self.y_id = device_id%4 + self.out_strategy = self.strategy0[1] + self.out_id = device_id%4 + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + + def forward_conv2d_mindspore_impl(self): + input1 = Tensor(self.input_np1) + input2 = Tensor(self.input_np2) + weight = Tensor(self.weight_np) + if self.has_bias: + bias = Tensor(self.bias_np) + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=True, weight_init=weight, + bias_init=bias) + else: + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=False, weight_init=weight) + out = net(input1, input2) + return out.asnumpy() + + def forward_conv2d_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + weight = Tensor(self.weight_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + if self.has_bias: + bias = Tensor(self.bias_np) + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=True, weight_init=weight, + bias_init=bias, strategy=(self.strategy0[0], self.strategy0[1], self.strategy0[1])) + else: + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=False, weight_init=weight, strategy=(self.strategy0[0], self.strategy0[1], self.strategy0[1])) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_conv2d_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + weight = Tensor(self.weight_np) + output_grad = Tensor(self.output_grad_np) + if self.has_bias: + bias = Tensor(self.bias_np) + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=True, weight_init=weight, + bias_init=bias,) + else: + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=False, weight_init=weight) + + + grad_net = Grad(net) + grad_net.set_train() + out_grad = grad_net(x, y, output_grad) + return out_grad + + def grad_conv2d_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + weight = Tensor(self.weight_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + output_grad = Tensor(self.output_grad_np) + output_grads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + output_grad1 = Tensor(output_grads[self.out_id]) + if self.has_bias: + bias = Tensor(self.bias_np) + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=True, weight_init=weight, + bias_init=bias, strategy=(self.strategy0[0], self.strategy0[1], self.strategy0[1])) + else: + net = Conv2d(in_channels=self.in_c, out_channels=self.out_c, + kernel_size=(self.kernel_h, self.kernel_w), + stride=self.stride, pad_mode=self.pad_mode, + padding=self.padding, dilation=self.dilation, + group=self.group, has_bias=False, weight_init=weight, strategy=(self.strategy0[0], self.strategy0[1], self.strategy0[1])) + + + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + out_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad1], parallel_inputs_run=[x1, y1, output_grad1]) + return out_grad + + + def forward_conv2d_cmp(self): + out_mindspore = self.forward_conv2d_mindspore_impl() + out_mindspore_parallel = self.forward_conv2d_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.out_strategy) + assert allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.001, 0.001) + + def grad_conv2d_cmp(self): + input_grad_mindspore = self.grad_conv2d_mindspore_impl() + input_grad_mindspore_parallel = self.grad_conv2d_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[1]) + assert allclose(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.001, 0.001) + assert allclose(input_grad_blocks_1[self.x_id], input_grad_mindspore_parallel1, 0.001, 0.001) + +def test_reid_conv2d_input_128_64_112_112_kernel_64_64_1_1_stride_2_padding_0_bias_true(): + fact = Conv2dFactory(input_shape=(128,64,112,112), + filter_shape=(64,64,1,1), + stride=2, pad_mode='valid', padding=0, + dilation=1, group=1, has_bias=False) + fact.forward_conv2d_cmp() + + +def test_reid_conv2d_grad_input_128_64_112_112_kernel_64_64_1_1_stride_2_padding_0_bias_true(): + fact = Conv2dFactory(input_shape=(128,64,112,112), + filter_shape=(64,64,1,1), + stride=2, pad_mode='valid', padding=0, + dilation=1, group=1, has_bias=False) + fact.grad_conv2d_cmp() + + diff --git a/tests/ut/python/parallel/parallel_end_to_end/batch_parallel/conv2d_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/batch_parallel/conv2d_parallel_4p.sh new file mode 100644 index 00000000000..720b912a4bc --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/batch_parallel/conv2d_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_conv2d_parallel_4p.py>../../log/test_conv2d_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/dist_env_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/dist_env_4p.sh new file mode 100644 index 00000000000..283c3c12de4 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/dist_env_4p.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +export SLOG_PRINT_TO_STDOUT=1 +source /root/miniconda3/bin/activate ci3.6 +export RANK_SIZE=4 +export RANK_TABLE_FILE=../../rank_table_4p.json +export RANK_ID=$1 +export DEVICE_ID=$1 +export HCCL_FLAG=1 +export DEPLOY_MODE=0 +export AICPU_FLAG=1 +export DUMP_OP=1 +export PYTHONPATH=../../../../../../../../mindspore:/usr/local/HiAI/runtime/python3.6/site-packages/topi.egg/:/usr/local/HiAI/runtime/python3.6/site-packages/te.egg/:/usr/local/HiAI/runtime/ops/op_impl/built-in/ai_core/tbe/ +export OPTION_EXEC_EXTERN_PLUGIN_PATH=/usr/local/HiAI/runtime/lib64/libhccl.so:/usr/local/HiAI/runtime/lib64/plugin/opskernel/libfe.so:/usr/local/HiAI/runtime/lib64/plugin/opskernel/libaicpu_plugin.so:/usr/local/HiAI/runtime/lib64/plugin/opskernel/libge_local_engine.so:/usr/local/HiAI/runtime/lib64/plugin/opskernel/librts_engine.so +export LD_LIBRARY_PATH=/usr/local/HiAI/runtime/lib64 +export FE_FLAG=1 +export PATH=/usr/local/HiAI/runtime/ccec_compiler/bin:$PATH +if [ $1 -eq 0 ]; +then + export DUMP_GE_GRAPH=true + export ME_DRAW_GRAPH=1 +fi diff --git a/tests/ut/python/parallel/parallel_end_to_end/dropout/_test_dropout_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/dropout/_test_dropout_parallel_4p.py new file mode 100644 index 00000000000..da0fc5a9b62 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/dropout/_test_dropout_parallel_4p.py @@ -0,0 +1,112 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import mindspore as ms +import mindspore.communication.management as distributedTool +from mindspore.nn import Cell +from mindspore import context +from mindspore.nn import Dropout +from mindspore.common.tensor import Tensor + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + + +class Net(Cell): + def __init__(self, keep_prob, seed0, seed1, strategy=None): + super(Net, self).__init__() + self.drop = Dropout(keep_prob, seed0, seed1, dtype=ms.float32, strategy=strategy) + + def construct(self, input): + x = self.drop(input) + return x + +class DropoutFactory: + def __init__(self, input_shape, keep_prob, seed0, seed1, strategy0=None): + size = 1 + prefix = "" + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(10, size) + self.input_np = np.reshape(np.arange(0, size)%number_range, input_shape).astype(np.float32) + self.keep_prob = keep_prob + self.seed0 = seed0 + self.seed1 = seed1 + self.strategy0 = strategy0 + need_dev_num = 1 + for s in strategy0[1]: + need_dev_num = need_dev_num*s + self.x_id = device_id%need_dev_num + self.out_id = device_id%need_dev_num + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def d4_tensor_compare(self, input, out_me): + [a,b,c,d] = input.shape + for i in range(a): + for j in range(b): + for k in range(c): + for e in range(d): + if out_me[i,j,k,e] == 0: + assert True == True + else: + assert np.allclose(out_me[i,j,k,e], input[i,j,k,e]*(1/0.4), 0.0001, 0.0001) + + def forward_mindspore_parallel_impl(self): + x = Tensor(self.input_np) + inputs_x = self.get_parallel_blocks(self.input_np, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + net = Net(0.4, 0, 0, strategy=self.strategy0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, parallel_inputs_compile=[x], parallel_inputs_run=[x1]) + return out.asnumpy() + + def forward_cmp(self): + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + input_blocks = self.get_parallel_blocks(self.input_np, self.strategy0[1]) + self.d4_tensor_compare(input_blocks[self.out_id], out_mindspore_parallel) + +def test_reid_dropout_forward_seed_F32_64_512_8_8(): + fact = DropoutFactory(input_shape=(64,512,8,8), keep_prob = 0.4, seed0 = 0, seed1 = 0, strategy0=(0,(4,1,1,1))) + fact.forward_cmp() + +def test_reid_dropout_forward_seed_F32_64_512_8_8_repeat(): + fact = DropoutFactory(input_shape=(64,512,8,8), keep_prob = 0.4, seed0 = 0, seed1 = 0, strategy0=(0,(2,1,1,1))) + fact.forward_cmp() \ No newline at end of file diff --git a/tests/ut/python/parallel/parallel_end_to_end/dropout/dropout_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/dropout/dropout_parallel_4p.sh new file mode 100644 index 00000000000..8bd6cdc350e --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/dropout/dropout_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_dropout_parallel_4p.py>../../log/test_dropout_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/hcom/_test_allgather_4p.py b/tests/ut/python/parallel/parallel_end_to_end/hcom/_test_allgather_4p.py new file mode 100644 index 00000000000..6f14d068cd3 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/hcom/_test_allgather_4p.py @@ -0,0 +1,147 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import mindspore as ms +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class MatmulSingle(Cell): + def __init__(self, transpose_a=False, transpose_b=False): + super(MatmulSingle, self).__init__() + self.matmul = P.MatMul(transpose_a, transpose_b) + self.pow = P.Pow() + self.reduce_sum = P.ReduceSum() + def construct(self, x, y): + out = self.matmul(x, y) + out = self.pow(out,2.0) + out = self.reduce_sum(out, None) + return out + + +class MatmulAllgather(Cell): + def __init__(self, group, transpose_a=False, transpose_b=False): + super(MatmulAllgather, self).__init__() + self.allgather = P.AllGather(group=group) + self.matmul = P.MatMul(transpose_a, transpose_b) + self.pow = P.Pow() + self.reduce_sum = P.ReduceSum() + self.allreduce = P.AllReduce(group=group) + def construct(self, x, y): + x = self.allgather(x) + out = self.matmul(x, y) + out = self.pow(out,2.0) + out = self.reduce_sum(out, None) + out = self.allreduce(out) + return out + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, sens): + return grad_all_with_sens(self.network)(x, y, sens) + +class MatmulAllgatherFactory: + def __init__(self, inputx_shape, inputy_shape, x_stra, y_stra): + self.inputx=self.GenValue(inputx_shape, 10) + self.inputy=self.GenValue(inputy_shape, 20) + self.x_stra = x_stra + self.y_stra = y_stra + stra_size= 1 + for s in x_stra: + stra_size = stra_size*s + self.stra_size = stra_size + + def GenValue(self, input_shape, delta): + size = 1 + for s in input_shape: + size = size*s + number_range = min(100, size) + input_np = np.reshape(np.arange(0, size)%number_range - delta, input_shape).astype(np.float32) + return input_np + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def grad_mindspore_impl_single(self): + x=Tensor(self.inputx) + y=Tensor(self.inputy) + sens=Tensor(1.0, dtype=ms.float32) + net = MatmulSingle() + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, sens) + return input_grad + + def grad_mindspore_impl_reduce(self): + inputxs = self.get_parallel_blocks(self.inputx, self.x_stra) + inputys = self.get_parallel_blocks(self.inputy, self.y_stra) + x = Tensor(inputxs[device_id%self.stra_size]) + y = Tensor(inputys[device_id%self.stra_size]) + repeat_num = device_num/self.stra_size + v = self.stra_size*repeat_num*repeat_num*repeat_num + sens = Tensor(1.0/v, dtype=ms.float32) + net = MatmulAllgather("hccl_world_group") + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, sens) + return input_grad + + def grad_cmp(self): + single_results = self.grad_mindspore_impl_single() + reduce_results = self.grad_mindspore_impl_reduce() + single_result0 = self.get_parallel_blocks(single_results[0].asnumpy(), self.x_stra)[device_id%self.stra_size] + reduce_result0 = reduce_results[0].asnumpy() + single_result1 = self.get_parallel_blocks(single_results[1].asnumpy(), self.y_stra)[device_id%self.stra_size] + reduce_result1 = reduce_results[1].asnumpy() + assert np.allclose(single_result0, reduce_result0, 0.0001, 0.0001) + assert np.allclose(single_result1, reduce_result1, 0.0001, 0.0001) + +def test_reduce_grad(): + inputx_shape = (64,32) + inputy_shape = (32,64) + fact = MatmulAllgatherFactory(inputx_shape, inputy_shape, (4,1), (1,4)) + fact.grad_cmp() + + \ No newline at end of file diff --git a/tests/ut/python/parallel/parallel_end_to_end/hcom/_test_allreduce_4p.py b/tests/ut/python/parallel/parallel_end_to_end/hcom/_test_allreduce_4p.py new file mode 100644 index 00000000000..ace9c2d6422 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/hcom/_test_allreduce_4p.py @@ -0,0 +1,166 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import mindspore as ms +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class MatmulSingle(Cell): + def __init__(self, transpose_a=False, transpose_b=False): + super(MatmulSingle, self).__init__() + self.matmul1 = P.MatMul(transpose_a, transpose_b) + self.matmul2 = P.MatMul(transpose_a, transpose_b) + self.pow = P.Pow() + self.reduce_sum = P.ReduceSum() + def construct(self, x, y, z): + out = self.matmul1(x, y) + out = self.matmul2(out, z) + out = self.pow(out,2.0) + out = self.reduce_sum(out, None) + return out + +class MatmulReduce(Cell): + def __init__(self, group, transpose_a=False, transpose_b=False): + super(MatmulReduce, self).__init__() + self.matmul1 = P.MatMul(transpose_a, transpose_b) + self.allreduce1 = P.AllReduce(group=group) + self.matmul2 = P.MatMul(transpose_a, transpose_b) + self.pow = P.Pow() + self.reduce_sum = P.ReduceSum() + self.allreduce2 = P.AllReduce(group=group) + def construct(self, x, y, z): + out = self.matmul1(x, y) + out = self.allreduce1(out) + out = self.matmul2(out, z) + out = self.pow(out,2.0) + out = self.reduce_sum(out, None) + out = self.allreduce2(out) + return out + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, z, sens): + return grad_all_with_sens(self.network)(x, y, z, sens) + +class MatmulReduceFactory: + def __init__(self, inputx_shape, inputy_shape, inputz_shape, x_stra, y_stra, z_stra): + self.inputx=self.GenValue(inputx_shape, 10) + self.inputy=self.GenValue(inputy_shape, 20) + self.inputz=self.GenValue(inputz_shape, 30) + self.x_stra = x_stra + self.y_stra = y_stra + self.z_stra = z_stra + stra_size= 1 + for s in x_stra: + stra_size = stra_size*s + self.stra_size = stra_size + + def GenValue(self, input_shape, delta): + size = 1 + for s in input_shape: + size = size*s + number_range = min(100, size) + input_np = np.reshape(np.arange(0, size)%number_range - delta, input_shape).astype(np.float32) + return input_np + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def grad_mindspore_impl_single(self): + x=Tensor(self.inputx) + y=Tensor(self.inputy) + z=Tensor(self.inputz) + sens=Tensor(1.0, dtype=ms.float32) + net = MatmulSingle() + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, z, sens) + return input_grad + + def grad_mindspore_impl_reduce(self): + inputxs = self.get_parallel_blocks(self.inputx, self.x_stra) + inputys = self.get_parallel_blocks(self.inputy, self.y_stra) + inputzs = self.get_parallel_blocks(self.inputz, self.z_stra) + x = Tensor(inputxs[device_id%self.stra_size]) + y = Tensor(inputys[device_id%self.stra_size]) + z = Tensor(inputzs[device_id%self.stra_size]) + repeat_num = device_num/self.stra_size + v = self.stra_size*repeat_num*repeat_num*repeat_num + sens = Tensor(1.0/v, dtype=ms.float32) + net = MatmulReduce("hccl_world_group") + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, z, sens) + return input_grad + + def grad_cmp(self): + single_results = self.grad_mindspore_impl_single() + reduce_results = self.grad_mindspore_impl_reduce() + single_result0 = self.get_parallel_blocks(single_results[0].asnumpy(), self.x_stra)[device_id%self.stra_size] + reduce_result0 = reduce_results[0].asnumpy() + single_result1 = self.get_parallel_blocks(single_results[1].asnumpy(), self.y_stra)[device_id%self.stra_size] + reduce_result1 = reduce_results[1].asnumpy() + single_result2 = self.get_parallel_blocks(single_results[2].asnumpy(), self.z_stra)[device_id%self.stra_size] + reduce_result2 = reduce_results[2].asnumpy() + assert np.allclose(single_result0, reduce_result0, 0.0001, 0.0001) + assert np.allclose(single_result1, reduce_result1, 0.0001, 0.0001) + assert np.allclose(single_result2, reduce_result2, 0.0001, 0.0001) + +def test_reduce_grad(): + inputx_shape = (32,64) + inputy_shape = (64,64) + inputz_shape = (64,32) + fact = MatmulReduceFactory(inputx_shape, inputy_shape, inputz_shape, (1,4), (4,1), (1,4)) + fact.grad_cmp() + +def test_reduce_grad_repeat(): + inputx_shape = (32,64) + inputy_shape = (64,64) + inputz_shape = (64,32) + fact = MatmulReduceFactory(inputx_shape, inputy_shape, inputz_shape, (1,2), (2,1), (1,2)) + fact.grad_cmp() + + \ No newline at end of file diff --git a/tests/ut/python/parallel/parallel_end_to_end/hcom/allgather_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/hcom/allgather_4p.sh new file mode 100644 index 00000000000..c44a6e1e57a --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/hcom/allgather_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_allgather_4p.py>../../log/test_allgather_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/hcom/allreduce_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/hcom/allreduce_4p.sh new file mode 100644 index 00000000000..93bdddebcca --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/hcom/allreduce_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_allreduce_4p.py>../../log/test_allreduce_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/l2normalize/_test_l2normalize_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/l2normalize/_test_l2normalize_parallel_4p.py new file mode 100644 index 00000000000..d9ab842b3ac --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/l2normalize/_test_l2normalize_parallel_4p.py @@ -0,0 +1,193 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class L2normalize(Cell): + def __init__(self, axis=0, epsilon=1e-4, strategy0=None, strategy1=None): + super(L2normalize, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.l2norm = P.L2Normalize(axis, epsilon, strategy1) + + def construct(self, x, y): + out = self.add(x, y) + out = self.l2norm(out) + return out + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, output_grad): + return grad_all_with_sens(self.network)(x, y, output_grad) + +class L2normalizeFactory: + def __init__(self, input_shape, axis, strategy0, strategy1): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = np.reshape(np.arange(0, size)%number_range - number_range/4, input_shape).astype(np.float32) + target_shape = input_shape + self.target_shape = target_shape + target_size = 1 + for s in target_shape: + target_size = target_size*s + number_range = min(1000, target_size) + self.output_grad_np = np.reshape(np.arange(0, target_size)%number_range - number_range/2, target_shape).astype(np.float32) + self.axis = axis + self.epsilon = 1e-4 + self.strategy0 = strategy0 + self.strategy1 = strategy1 + out_strategy = strategy1[1] + self.out_strategy = out_strategy + need_dev_num0 = 1 + need_dev_num1 = 1 + for s in strategy0[1]: + need_dev_num0 = need_dev_num0*s + for s in out_strategy: + need_dev_num1 = need_dev_num1*s + self.x_id = device_id%need_dev_num0 + self.y_id = device_id%need_dev_num0 + self.out_id = device_id%need_dev_num1 + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def forward_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + net = L2normalize(self.axis, self.epsilon) + out = net(x, y) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + net = L2normalize(self.axis, self.epsilon, strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + net = L2normalize(self.axis, self.epsilon) + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, output_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + outgrads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + output_grad1 = Tensor(outgrads[self.out_id]) + net = L2normalize(self.axis, self.epsilon, strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad1], parallel_inputs_run=[x1, y1, output_grad1]) + return input_grad + + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.out_strategy) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.001, 0.001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[2]) + assert np.allclose(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert np.allclose(input_grad_blocks_1[self.y_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + +def test_reid_l2normalize_input_128_512(): + input_shape = (128,512) + axis = 0 + fact = L2normalizeFactory(input_shape, axis, strategy0=(0, (4,1), (4,1)), strategy1=(0, (1,4))) + fact.forward_cmp() + +def test_reid_l2normalize_grad_input_128_512(): + input_shape = (128,512) + axis = 0 + fact = L2normalizeFactory(input_shape, axis, (0, (4,1), (4,1)), strategy1=(0, (1,4))) + fact.grad_cmp() + +def test_reid_l2normalize_input_128_512_repeat(): + input_shape = (128,512) + axis = 0 + fact = L2normalizeFactory(input_shape, axis, strategy0=(0, (1,2), (1,2)), strategy1=(0, (1,2))) + fact.forward_cmp() + +def test_reid_l2normalize_grad_input_128_512_repeat(): + input_shape = (128,512) + axis = 0 + fact = L2normalizeFactory(input_shape, axis, strategy0=(0, (1,2), (1,2)), strategy1=(0, (1,2))) + fact.grad_cmp() + diff --git a/tests/ut/python/parallel/parallel_end_to_end/l2normalize/l2normalize_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/l2normalize/l2normalize_parallel_4p.sh new file mode 100644 index 00000000000..66bcdf8f200 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/l2normalize/l2normalize_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_l2normalize_parallel_4p.py>../../log/test_l2normalize_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/log/README.MD b/tests/ut/python/parallel/parallel_end_to_end/log/README.MD new file mode 100644 index 00000000000..c4d4042ded0 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/log/README.MD @@ -0,0 +1 @@ +log files for auto parallel end to end test cases diff --git a/tests/ut/python/parallel/parallel_end_to_end/loss/_test_loss_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/loss/_test_loss_parallel_4p.py new file mode 100644 index 00000000000..592352e825c --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/loss/_test_loss_parallel_4p.py @@ -0,0 +1,180 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class AddRelu(Cell): + def __init__(self, strategy0=None, strategy1=None): + super(AddRelu, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.relu = P.ReLU(strategy=strategy1) + + def construct(self, x, y): + out = self.add(x, y) + out = self.relu(out) + return out + +class NetWithLoss(Cell): + def __init__(self, network, strategy2=None): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits(strategy=strategy2) + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y) + return self.loss(predict, b)[0] + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, b): + return grad_all(self.network)(x, y, b) + +class AddReluFactory: + def __init__(self, input_shape, strategy0, strategy1, strategy2): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = np.reshape(np.arange(0, size)%number_range - number_range/4, input_shape).astype(np.float32) + target_shape = input_shape + self.target_shape = target_shape + target_size = 1 + for s in target_shape: + target_size = target_size*s + number_range = min(10, target_size) + self.output_grad_np = np.reshape((np.arange(0, target_size)%number_range)*0.1, target_shape).astype(np.float32) + self.strategy0 = strategy0 + self.strategy1 = strategy1 + self.strategy2 = strategy2 + out_strategy = strategy1[1] + self.out_strategy = out_strategy + need_dev_num0 = 1 + need_dev_num1 = 1 + for s in strategy0[1]: + need_dev_num0 = need_dev_num0*s + for s in out_strategy: + need_dev_num1 = need_dev_num1*s + self.x_id = device_id%need_dev_num0 + self.y_id = device_id%need_dev_num0 + self.out_id = device_id%need_dev_num1 + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def grad_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + net = AddRelu() + net_with_loss = NetWithLoss(net) + grad_net = Grad(net_with_loss) + grad_net.set_train() + input_grads = [] + for i in range(0, 3): + input_grad = grad_net(x, y, output_grad) + input_grads.append(input_grad) + return input_grads + + def grad_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + outgrads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + output_grad1 = Tensor(outgrads[self.out_id]) + net = AddRelu(strategy0=self.strategy0, strategy1=self.strategy1) + net_with_loss = NetWithLoss(net, strategy2=self.strategy2) + grad_net = Grad(net_with_loss) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + input_grads = [] + for i in range(0, 3): + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad], parallel_inputs_run=[x1, y1, output_grad1]) + input_grads.append(input_grad) + return input_grads + + + def grad_cmp(self): + input_grad_mindspores = self.grad_mindspore_impl() + input_grad_mindspore_parallels = self.grad_mindspore_parallel_impl() + for i in range(0,len(input_grad_mindspores)): + input_grad_mindspore = input_grad_mindspores[i] + input_grad_mindspore_parallel = input_grad_mindspore_parallels[i] + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[2]) + np.save(path+str(i)+"_"+str(device_id)+"_"+self.prefix+"_grad_single0.npy", input_grad_blocks_0[self.x_id]) + np.save(path+str(i)+"_"+str(device_id)+"_"+self.prefix+"_grad_single1.npy", input_grad_blocks_1[self.y_id]) + np.save(path+str(i)+"_"+str(device_id)+"_"+self.prefix+"_grad_parallel0.npy", input_grad_mindspore_parallel0) + np.save(path+str(i)+"_"+str(device_id)+"_"+self.prefix+"_grad_parallel1.npy", input_grad_mindspore_parallel1) + assert np.allclose(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert np.allclose(input_grad_blocks_1[self.y_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + + + +def test_reid_l2normalize_grad_input_128_512(): + input_shape = (128,512) + fact = AddReluFactory(input_shape, strategy0=(0, (4,1), (4,1)), strategy1=(0, (4,1)), strategy2=(0,(4,1),(4,1))) + fact.grad_cmp() + +def test_reid_l2normalize_grad_input_128_512_stridesplit(): + input_shape = (128,512) + fact = AddReluFactory(input_shape, strategy0=(0, (1,1), (1,1)), strategy1=(0, (4,1)), strategy2=(0,(4,1),(4,1))) + fact.grad_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/loss/loss_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/loss/loss_parallel_4p.sh new file mode 100644 index 00000000000..d6e075f8852 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/loss/loss_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_loss_parallel_4p.py>../../log/test_loss_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/matmul/_test_matmul_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/matmul/_test_matmul_parallel_4p.py new file mode 100644 index 00000000000..6810204bbf6 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/matmul/_test_matmul_parallel_4p.py @@ -0,0 +1,313 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +from numpy import allclose +from mindspore import context +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class Matmul(Cell): + def __init__(self, transpose_a=False, transpose_b=False, strategy0=None, strategy1=None): + super(Matmul, self).__init__() + self.add = P.TensorAdd(strategy=strategy1) + self.matmul = P.MatMul(transpose_a, transpose_b, strategy=strategy0) + def construct(self, x, w, z): + out = self.add(x, z) + return self.matmul(out, w) + +class BatchMatMul(Cell): + def __init__(self, transpose_a=False, transpose_b=False, strategy0=None, strategy1=None): + super(BatchMatMul, self).__init__() + self.add = P.TensorAdd(strategy=strategy1) + self.batchmatmul = P.BatchMatMul(transpose_a, transpose_b, strategy=strategy0) + def construct(self, x, w, z): + out = self.add(x, z) + return self.batchmatmul(out, w) + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, inputa, inputb, inputz, output_grad): + gout = grad_all_with_sens(self.network)(inputa, inputb, inputz, output_grad) + return gout + +class BatchmatmulFactory: + def __init__(self, inputa_shape, inputb_shape, transpose_a, transpose_b, strategy, strategy_): + self.strategy = strategy + self.strategy_ = strategy_ + inputa_size = 1 + inputb_size = 1 + prefix = "" + for s in inputa_shape: + prefix = prefix + str(s) + "_" + inputa_size = inputa_size*s + prefix = prefix + "and" + for s in inputb_shape: + prefix = prefix + str(s) + "_" + inputb_size = inputb_size*s + number_rangea = min(1000, inputa_size) + number_rangeb = min(1000, inputb_size) + self.inputa = np.reshape(np.arange(0, inputa_size)%number_rangea - number_rangea/2, inputa_shape).astype(np.float32) + self.inputb = np.reshape(np.arange(0, inputb_size)%number_rangeb - number_rangeb/2, inputb_shape).astype(np.float32) + self.inputz = np.zeros(self.inputa.shape).astype(np.float32) + self.transpose_a = transpose_a + self.transpose_b = transpose_b + + out_shape = [] + device_matrix = [] + out_strategy = [] + if transpose_a: + temp = inputa_shape[-1] + inputa_shape[-1] = inputa_shape[-2] + inputa_shape[-2] = temp + if transpose_b: + temp = inputb_shape[-1] + inputb_shape[-1] = inputb_shape[-2] + inputb_shape[-2] = temp + + if (len(inputa_shape) >= len(inputb_shape)): + out_shape = list(inputa_shape) + out_shape[-1] = inputb_shape[-1] + else: + out_shape = list(inputb_shape) + out_shape[-2] = inputa_shape[-2] + + + strategy1 = list(self.strategy[1]) + strategy2 = list(self.strategy[2]) + if transpose_a: + temp = strategy1[-1] + strategy1[-1] = strategy1[-2] + strategy1[-2] = temp + if transpose_b: + temp = strategy2[-1] + strategy2[-1] = strategy2[-2] + strategy2[-2] = temp + + if (len(strategy1) >= len(strategy2)): + out_strategy = strategy1.copy() + out_strategy[-1] = strategy2[-1] + else: + out_strategy = strategy2.copy() + out_strategy[-2] = strategy1[-2] + device_matrix = out_strategy.copy() + device_matrix.insert(-1, strategy1[-1]) + self.out_strategy = out_strategy + + need_dev_num = 1 + for s in device_matrix: + need_dev_num = need_dev_num*s + self.need_dev_num = need_dev_num + self.device_matrix = device_matrix + + out_size = 1 + for s in out_shape: + out_size = out_size*s + number_range = min(1000, out_size) + self.output_grad_np = np.reshape(np.arange(0, out_size)%number_range - number_range/2, out_shape).astype(np.float32) + + device_index = self.id_to_list(device_id%need_dev_num, self.device_matrix) + x_index = device_index[:-1].copy() + if transpose_a: + temp = x_index[-1] + x_index[-1] = x_index[-2] + x_index[-2] = temp + y_index = device_index[:-3].copy() + y_index.append(device_index[-2]) + y_index.append(device_index[-1]) + if transpose_b: + temp = y_index[-1] + y_index[-1] = y_index[-2] + y_index[-2] = temp + + out_index = device_index[:-2].copy() + out_index.append(device_index[-1]) + + print(device_matrix) + print(device_index) + + need_dev_num_ = 1 + for s in strategy_[1]: + need_dev_num_ = need_dev_num_*s + self.x_id = device_id%need_dev_num_ + self.y_id = self.list_to_id(y_index, self.strategy[2]) + self.out_id = self.list_to_id(out_index, self.out_strategy) + + + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + """ + shape：每一维的上限，如（2,4,8） + """ + def id_to_list(self, id, shape): + result = [] + r = id + for i in range(0, len(shape)): + v = 1 + for j in range(i+1, len(shape)): + v = v*shape[j] + result.append(r//v) + r = r%v + return result + + def list_to_id(self, id_list, shape): + result = 0 + for i in range(0, len(id_list)): + v = 1 + for j in range(i+1, len(id_list)): + v = v*shape[j] + result = result + id_list[i]*v + return result + + + def forward_mindspore_impl(self): + if len(self.inputa.shape) > 2: + matmul = BatchMatMul(self.transpose_a, self.transpose_b) + else: + matmul = Matmul(self.transpose_a, self.transpose_b) + matmul.set_train() + out_me = matmul(Tensor(self.inputa), Tensor(self.inputb), Tensor(self.inputz)) + return out_me.asnumpy() + + def forward_mindspore_parallel_impl(self): + if len(self.inputa.shape) > 2: + matmul = BatchMatMul(self.transpose_a, self.transpose_b, strategy0=self.strategy, strategy1=self.strategy_) + else: + matmul = Matmul(self.transpose_a, self.transpose_b, strategy0=self.strategy, strategy1=self.strategy_) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + x = Tensor(self.inputa) + y = Tensor(self.inputb) + z = Tensor(self.inputz) + xs = self.get_parallel_blocks(self.inputa, self.strategy_[1]) + ys = self.get_parallel_blocks(self.inputb, self.strategy[2]) + zs = self.get_parallel_blocks(self.inputz, self.strategy_[1]) + x1 = Tensor(xs[self.x_id]) # + y1 = Tensor(ys[self.y_id]) #需要从设备矩阵推导 + z1 = Tensor(zs[self.x_id]) + matmul.set_train() + out_me = matmul(x, y, z, parallel_inputs_compile=[x, y, z], parallel_inputs_run=[x1, y1, z1]) + return out_me.asnumpy() + + def grad_mindspore_impl(self): + x = Tensor(self.inputa) + y = Tensor(self.inputb) + z = Tensor(self.inputz) + if len(self.inputa.shape) > 2: + matmul = BatchMatMul(self.transpose_a, self.transpose_b) + else: + matmul = Matmul(self.transpose_a, self.transpose_b) + net_me = Grad(matmul) + net_me.set_train() + out_grad_me = Tensor(self.output_grad_np) + out_grad = net_me(x, y, z, out_grad_me) + return out_grad + + def grad_mindspore_parallel_impl(self): + if len(self.inputa.shape) > 2: + matmul = BatchMatMul(self.transpose_a, self.transpose_b, strategy0=self.strategy, strategy1=self.strategy_) + else: + matmul = Matmul(self.transpose_a, self.transpose_b, strategy0=self.strategy, strategy1=self.strategy_) + x = Tensor(self.inputa) + y = Tensor(self.inputb) + z = Tensor(self.inputz) + out_grad_me = Tensor(self.output_grad_np) + + xs = self.get_parallel_blocks(self.inputa, self.strategy_[1]) + ys = self.get_parallel_blocks(self.inputb, self.strategy[2]) + zs = self.get_parallel_blocks(self.inputz, self.strategy_[1]) + out_grads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + + x1 = Tensor(xs[self.x_id]) #需要从设备矩阵推导 + y1 = Tensor(ys[self.y_id]) # + z1 = Tensor(zs[self.x_id]) + out_grad1 = Tensor(out_grads[self.out_id]) + net_me = Grad(matmul) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + net_me.set_train() + + out_grad = net_me(x, y, z, out_grad_me, parallel_inputs_compile = [x, y, z, out_grad1], parallel_inputs_run = [x1, y1, z1, out_grad1]) + return out_grad + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspores = self.get_parallel_blocks(out_mindspore, self.out_strategy) + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + assert allclose(out_mindspores[self.out_id], out_mindspore_parallel, 0.0001, 0.0001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspores0 = self.get_parallel_blocks(input_grad_mindspore[0].asnumpy(), self.strategy_[1]) + input_grad_mindspores1 = self.get_parallel_blocks(input_grad_mindspore[1].asnumpy(), self.strategy[2]) + input_grad_mindspores2 = self.get_parallel_blocks(input_grad_mindspore[2].asnumpy(), self.strategy_[1]) + assert allclose(input_grad_mindspores0[self.x_id], input_grad_mindspore_parallel[0].asnumpy(), 0.0001, 0.0001) + assert allclose(input_grad_mindspores1[self.y_id], input_grad_mindspore_parallel[1].asnumpy(), 0.0001, 0.0001) + assert allclose(input_grad_mindspores2[self.x_id], input_grad_mindspore_parallel[2].asnumpy(), 0.0001, 0.0001) + +def test_reid_batchmatmul_inputa_128_512_inputb_2000_512(): + inputa = [128, 512] + inputb = [2000, 512] + fact = BatchmatmulFactory(inputa, inputb, False, True, (0,(2,2),(1,2)), (0,(2,2),(2,2))) + fact.forward_cmp() + +def test_reid_batchmatmul_grad_inputa_128_512_inputb_2000_512(): + inputa = [128, 512] + inputb = [2000, 512] + fact = BatchmatmulFactory(inputa, inputb, False, True, (0, (2,2),(1,2)), (0,(2,2),(2,2))) + fact.grad_cmp() + +def test_reid_batchmatmul_inputa_128_512_inputb_2000_512_redistribution(): + inputa = [128, 512] + inputb = [2000, 512] + fact = BatchmatmulFactory(inputa, inputb, False, True, (0,(1,2),(1,2)), (0,(2,2),(2,2))) + fact.forward_cmp() + +def test_reid_batchmatmul_grad_inputa_128_512_inputb_2000_512_redistribution(): + inputa = [128, 512] + inputb = [2000, 512] + fact = BatchmatmulFactory(inputa, inputb, False, True, (0, (1,2),(1,2)), (0,(2,2),(2,2))) + fact.grad_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/matmul/matmul_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/matmul/matmul_parallel_4p.sh new file mode 100644 index 00000000000..5c58c0a1f2e --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/matmul/matmul_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_matmul_parallel_4p.py >../../log/test_matmul_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/max/_test_max_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/max/_test_max_parallel_4p.py new file mode 100644 index 00000000000..64479a1afef --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/max/_test_max_parallel_4p.py @@ -0,0 +1,193 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +from mindspore import context +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, input1, input2, output_grad): + return grad_all_with_sens(self.network)(input1, input2, output_grad) + + +class Max(Cell): + def __init__(self, axis, keep_dims, strategy0=None, strategy1=None): + super(Max, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.reduce_max = P.ReduceMax(keep_dims=keep_dims).set_strategy(strategy=strategy1) + self.axis = axis + + def construct(self, input1, input2): + out = self.add(input1, input2) + return self.reduce_max(out, self.axis) + + +class MaxFactory: + def __init__(self, input_shape, axis, keep_dims, strategy0, strategy1): + self.strategy0 = strategy0 + self.strategy1 = strategy1 + self.axis = axis + self.keep_dims = keep_dims + input_size = 1 + prefix = "" + for s in input_shape: + prefix = prefix + str(s) + "_" + input_size = input_size*s + number_range = min(1000, input_size) + self.input_np1 = np.reshape(np.arange(0, input_size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = self.input_np1.copy() + self.out_grad_np = None + out_shape = list(input_shape) + out_shape.pop(axis) + out_size = input_size/input_shape[axis] + number_range_ = min(1000, out_size) + self.out_grad_np = np.reshape(np.arange(0, out_size)%number_range_ - number_range_/2, out_shape).astype(np.float32) + out_strategy = list(strategy1[1]) + out_strategy.pop(axis) + self.out_strategy = out_strategy + need_dev_num = 1 + need_dev_num_ = 1 + for s in strategy0[1]: + need_dev_num = need_dev_num*s + for s in out_strategy: + need_dev_num_ = need_dev_num_*s + self.x_id = device_id%need_dev_num + self.y_id = device_id%need_dev_num + self.out_id = device_id%need_dev_num_ + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def forward_mindspore_impl(self): + input1 = Tensor(self.input_np1) + input2 = Tensor(self.input_np2) + net = Max(axis=self.axis, keep_dims=self.keep_dims) + out = net(input1, input2) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + xs = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + ys = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(xs[self.x_id]) + y1 = Tensor(ys[self.y_id]) + net = Max(axis=self.axis, keep_dims=self.keep_dims, strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + input1 = Tensor(self.input_np1) + input2 = Tensor(self.input_np2) + out_grad = Tensor(self.out_grad_np) + net = Max(axis=self.axis, keep_dims=self.keep_dims) + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(input1, input2, out_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grads = self.get_parallel_blocks(self.out_grad_np, self.out_strategy) + out_grad = Tensor(output_grads[self.out_id]) + xs = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + ys = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(xs[self.x_id]) + y1 = Tensor(ys[self.y_id]) + net = Max(axis=self.axis, keep_dims=self.keep_dims, strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + input_grad = grad_net(x, y, out_grad, parallel_inputs_compile=[x, y, out_grad], parallel_inputs_run=[x1, y1, out_grad]) + return input_grad + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + print(out_mindspore) + print(out_mindspore_parallel) + out_blocks = self.get_parallel_blocks(out_mindspore, self.out_strategy) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.001, 0.001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[2]) + assert np.allclose(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert np.allclose(input_grad_blocks_1[self.y_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + +def test_reid_max_forward_input_256_64(): + fact = MaxFactory(input_shape=(256, 64), axis=1, keep_dims=False, strategy0=(0,(4,1),(4,1)), strategy1=(0,(4,1))) + fact.forward_cmp() + +def test_reid_max_grad_input_256_64(): + fact = MaxFactory(input_shape=(256, 64), axis=1, keep_dims=False, strategy0=(0,(4,1),(4,1)), strategy1=(0,(4,1))) + fact.grad_cmp() + +def test_reid_max_forward_input_128_64_32_32(): + fact = MaxFactory(input_shape=(128, 64, 32, 32), axis=3, keep_dims=False, strategy0=(0,(2,1,2,1),(2,1,2,1)), strategy1=(0,(2,1,2,1))) + fact.forward_cmp() + +def test_reid_max_grad_input_128_64_32_32(): + fact = MaxFactory(input_shape=(128, 64, 32, 32), axis=3, keep_dims=False, strategy0=(0,(2,1,2,1),(2,1,2,1)), strategy1=(0,(2,1,2,1))) + fact.grad_cmp() + +def test_reid_max_forward_input_256_64_repeat(): + fact = MaxFactory(input_shape=(256, 64), axis=1, keep_dims=False, strategy0=(0,(2,1),(2,1)), strategy1=(0,(2,1))) + fact.forward_cmp() + +def test_reid_max_grad_input_256_64_repeat(): + fact = MaxFactory(input_shape=(256, 64), axis=1, keep_dims=False, strategy0=(0,(2,1),(2,1)), strategy1=(0,(2,1))) + fact.grad_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/max/max_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/max/max_parallel_4p.sh new file mode 100644 index 00000000000..bf44b717ccb --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/max/max_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_max_parallel_4p.py>../../log/test_max_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/mul_softmax/mul_activation_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/mul_softmax/mul_activation_parallel_4p.sh new file mode 100644 index 00000000000..af28b9c710e --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/mul_softmax/mul_activation_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_mul_softmax_parallel_4p.py>../../log/test_mul_softmax_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/mul_softmax/need_fix_test_mul_softmax_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/mul_softmax/need_fix_test_mul_softmax_parallel_4p.py new file mode 100644 index 00000000000..359736e605f --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/mul_softmax/need_fix_test_mul_softmax_parallel_4p.py @@ -0,0 +1,187 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +from numpy import allclose +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class MulSoftmax(Cell): + def __init__(self, strategy0=None, strategy1=None, axis=0): + super(MulSoftmax, self).__init__() + self.mul = P.Mul(strategy=strategy0) + self.softmax = P.Softmax(axis=axis, strategy=strategy1) + + def construct(self, x, z): + out = self.mul(x, z) + return self.softmax(out) + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, output_grad): + return grad_all_with_sens(self.network)(x, y, output_grad) + + +class MulSoftmaxFactory: + def __init__(self, input_shape, strategy0, strategy1): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = 1.0 + self.output_grad_np = np.reshape((np.arange(0, size)%(number_range-10) - number_range/2)*0.1, input_shape).astype(np.float32) + self.strategy0 = strategy0 + self.strategy1 = strategy1 + need_dev_num = 1 + need_dev_num_ = 1 + for s in strategy0[1]: + need_dev_num = need_dev_num*s + for s in strategy1[1]: + need_dev_num_ = need_dev_num_*s + self.x_id = device_id%need_dev_num + self.y_id = device_id%need_dev_num + self.out_id = device_id%need_dev_num_ + + def forward_mindspore_impl(self): + net = MulSoftmax() + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + out = net(x, y) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + net = MulSoftmax(strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(self.input_np2, ms.float32) + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + output_grad = Tensor(self.output_grad_np) + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + net = MulSoftmax() + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, output_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + output_grads = self.get_parallel_blocks(self.output_grad_np, self.strategy1[1]) + output_grad = Tensor(output_grads[self.out_id]) + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + net = MulSoftmax(strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(self.input_np2, ms.float32) + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad], parallel_inputs_run=[x1, y1, output_grad]) + return input_grad + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + np.save(path+str(device_id)+"_"+self.prefix+"_forward_parallel.npy", out_mindspore_parallel) + out_blocks = self.get_parallel_blocks(out_mindspore, self.strategy1[1]) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.0001, 0.001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + np.save(path+str(device_id)+"_"+self.prefix+"_grad_parallel0.npy", input_grad_mindspore_parallel0) + np.save(path+str(device_id)+"_"+self.prefix+"_grad_parallel1.npy", input_grad_mindspore_parallel1) + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1])#这里由于TensorMul两个输入X1没做广播，X2做了广播 + assert np.allclose(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert np.allclose(input_grad_mindspore1, input_grad_mindspore_parallel1, 0.0001, 0.0001) + +@pytest.mark.reid_forward +def test_reid_mul_softmax_input_128x64(): + stra0 = (0,(1,4),()) + stra1 = (0,(1,4)) + fact = MulSoftmaxFactory(input_shape=(128, 64), strategy0=stra0, strategy1=stra1) + fact.forward_cmp() + +@pytest.mark.reid_grad +def test_reid_grad_mul_softmax_input_128x64(): + stra0 = (0,(1,4),()) + stra1 = (0,(1,4)) + fact = MulSoftmaxFactory(input_shape=(128, 64), strategy0=stra0, strategy1=stra1) + fact.grad_cmp() + +@pytest.mark.reid_forward +def test_reid_mul_softmax_input_128x64_all_to_all(): + stra0 = (0,(4,1),()) + stra1 = (0,(1,4)) + fact = MulSoftmaxFactory(input_shape=(128, 64), strategy0=stra0, strategy1=stra1) + fact.forward_cmp() + +@pytest.mark.reid_grad +def test_reid_grad_mul_softmax_input_128x64_all_to_all(): + stra0 = (0,(4,1),()) + stra1 = (0,(1,4)) + fact = MulSoftmaxFactory(input_shape=(128, 64), strategy0=stra0, strategy1=stra1) + fact.grad_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/onehot/_test_onehot_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/onehot/_test_onehot_parallel_4p.py new file mode 100644 index 00000000000..ca749437fa9 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/onehot/_test_onehot_parallel_4p.py @@ -0,0 +1,146 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +from numpy import allclose +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.communication.management as distributedTool + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + + +class Onehot(Cell): + def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, strategy=None): + super(Onehot, self).__init__() + self.onehot = P.OneHot(axis, strategy=strategy) + self.depth = depth + self.on_value = Tensor(on_value, ms.float32) + self.off_value = Tensor(off_value, ms.float32) + + def construct(self, indices): + return self.onehot(indices, self.depth, self.on_value, self.off_value) + + +class OneHotFactory: + def __init__(self, input_shape, depth, on_value=1.0, off_value=0.0, axis=None, dtype=None, strategy0=None): + size = 1 + prefix = "" + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(10, size) + self.input_np = np.reshape(np.arange(0, size)%number_range, input_shape).astype(np.int32) + self.depth = depth + self.on_value = on_value + self.off_value = off_value + self.axis = axis + self.dtype = dtype + self.strategy0 = strategy0 + need_dev_num = 1 + for s in strategy0[1]: + need_dev_num = need_dev_num*s + self.x_id = device_id%need_dev_num + self.out_id = device_id%need_dev_num + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def grad_mindspore_impl(self): + output_grad = Tensor(self.output_grad_np) + x = Tensor(self.input_np1) + y = Tensor(self.input_np2, ms.float32) + net = AddRelu() + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, output_grad) + return input_grad + + def forward_mindspore_impl(self): + indices = Tensor(self.input_np) + net = Onehot(axis=self.axis, + depth=self.depth, + on_value=self.on_value, + off_value=self.off_value) + out = net(indices) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + x = Tensor(self.input_np) + inputs_x = self.get_parallel_blocks(self.input_np, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + net = Onehot(axis=self.axis, + depth=self.depth, + on_value=self.on_value, + off_value=self.off_value, strategy=self.strategy0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, parallel_inputs_compile=[x], parallel_inputs_run=[x1]) + return out.asnumpy() + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.strategy0[1]) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.0001, 0.0001) + + +def test_reid_onehot_forward_int32_128_depth13000(): + fact = OneHotFactory(input_shape=(128,), + depth=131072, + on_value=1.000000, + off_value=0.000000, + axis=-1, + dtype="float32", + strategy0=(0,(2,))) + fact.forward_cmp() + + +def test_reid_onehot_forward_int32_131072_depth127(): + fact = OneHotFactory(input_shape=(131072,), + depth=127, + on_value=1.000000, + off_value=0.000000, + axis=-1, + dtype="float32", + strategy0=(0,(4,))) + fact.forward_cmp() + diff --git a/tests/ut/python/parallel/parallel_end_to_end/onehot/onehot_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/onehot/onehot_parallel_4p.sh new file mode 100644 index 00000000000..ba68b342485 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/onehot/onehot_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_onehot_parallel_4p.py>../../log/test_onehot_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/prelu/_test_prelu_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/prelu/_test_prelu_parallel_4p.py new file mode 100644 index 00000000000..44d1b49c906 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/prelu/_test_prelu_parallel_4p.py @@ -0,0 +1,195 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +import mindspore.communication.management as distributedTool +from numpy import allclose +from mindspore import context +from mindspore.nn import Cell +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0,1,2,3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + +class PReLU(Cell): + def __init__(self, channel=1, w=0.25, strategy_=None, strategy1_=None): + super(PReLU, self).__init__() + self.add = P.TensorAdd(strategy=strategy1_) + self.prelu = P.PReLU(strategy=strategy_) + + def construct(self, x, z, w): + out = self.add(x, z) + return self.prelu(out, w) + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, input,z, w, output_grad): + return grad_all_with_sens(self.network)(input,z,w, output_grad) + + +class PReLUFactory: + def __init__(self, input_shape, strategy): + n, c = input_shape[:2] + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.output_grad_np = np.reshape((np.arange(0, size)%(number_range-10) - number_range/2)*0.1, input_shape).astype(np.float32) + self.channel = c + self.weight = np.array([np.float32(0.25)] * c) + self.strategy = strategy + + def forward_mindspore_impl(self): + net = PReLU(channel=self.channel, w=self.weight) + x = Tensor(self.input_np) + z = Tensor(np.zeros(self.input_np.shape), ms.float32) + w = Tensor(self.weight) + out = net(x, z, w) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + net = PReLU(channel=self.channel, w=self.weight, strategy_=self.strategy, strategy1_=(self.strategy[0], self.strategy[1], self.strategy[1])) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + x = Tensor(self.input_np) + z = Tensor(np.zeros(self.input_np.shape), ms.float32) + w = Tensor(self.weight) + + inputs = self.get_parallel_blocks(self.input_np, self.strategy[1]) + block_id = device_id%len(inputs) + x1 = Tensor(inputs[block_id]) + z1 = Tensor(np.zeros(inputs[block_id].shape), ms.float32) + w1 = Tensor(self.weight) + + out = net(x, z, w, parallel_inputs_compile=[x, z, w], parallel_inputs_run=[x1, z1 ,w1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + output_grad = Tensor(self.output_grad_np) + x = Tensor(self.input_np) + z = Tensor(np.zeros(self.input_np.shape), ms.float32) + w = Tensor(self.weight) + + net = PReLU(channel=self.channel, w=self.weight) + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, z, w, output_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + output_grads = self.get_parallel_blocks(self.output_grad_np, self.strategy[1]) + block_id = device_id%len(output_grads) + output_grad = Tensor(output_grads[block_id]) + x = Tensor(self.input_np) + z = Tensor(np.zeros(self.input_np.shape), ms.float32) + w = Tensor(self.weight) + + net = PReLU(channel=self.channel, w=self.weight, strategy_=self.strategy, strategy1_=(self.strategy[0], self.strategy[1], self.strategy[1])) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + grad_net.set_train() + inputs = self.get_parallel_blocks(self.input_np, self.strategy[1]) + x1 = Tensor(inputs[block_id]) + z1 = Tensor(np.zeros(inputs[block_id].shape), ms.float32) + w1 = Tensor(self.weight) + + input_grad = grad_net(x, z, w, output_grad, parallel_inputs_compile=[x, z, w, output_grad], parallel_inputs_run=[x1, z1, w1, output_grad]) + return input_grad + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.strategy[1]) + block_id = device_id%len(out_blocks) + assert np.allclose(out_blocks[block_id], out_mindspore_parallel, 0.0001, 0.001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore2 = input_grad_mindspore[2].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_mindspore_parallel2 = input_grad_mindspore_parallel[2].asnumpy() + input_grad_blocks = self.get_parallel_blocks(input_grad_mindspore0, self.strategy[1]) + input1_grad_blocks = self.get_parallel_blocks(input_grad_mindspore1, self.strategy[1]) + block_id = device_id%len(input_grad_blocks) + assert np.allclose(input_grad_blocks[block_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert np.allclose(input_grad_mindspore2, input_grad_mindspore_parallel2, 0.0001, 0.0001) + assert np.allclose(input1_grad_blocks[block_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + + + +@pytest.mark.reid_grad +def test_reid_prelu_input_128x64x112x112_repeat(): + stra = (0,(1,1,2,1),(1)) + fact = PReLUFactory(input_shape=(128, 64, 112, 112), strategy=stra) + fact.forward_cmp() + +@pytest.mark.reid_grad +def test_reid_grad_prelu_input_128x64x112x112_repeat(): + stra = (0,(1,1,2,1),(1)) + fact = PReLUFactory(input_shape=(128, 64, 112, 112), strategy=stra) + fact.grad_cmp() + +@pytest.mark.reid_grad +def test_reid_prelu_input_128x64x112x112_mix(): + stra = (0,(2,1,1,2),(1)) + fact = PReLUFactory(input_shape=(128, 64, 112, 112), strategy=stra) + fact.forward_cmp() + +@pytest.mark.reid_grad +def test_reid_grad_prelu_input_128x64x112x112_mix(): + stra = (0,(2,1,1,2),(1)) + fact = PReLUFactory(input_shape=(128, 64, 112, 112), strategy=stra) + fact.grad_cmp() + diff --git a/tests/ut/python/parallel/parallel_end_to_end/prelu/prelu_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/prelu/prelu_parallel_4p.sh new file mode 100644 index 00000000000..f58d3735d8c --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/prelu/prelu_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_prelu_parallel_4p.py >../../log/test_prelu_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/reducemean/_test_reducemean_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/reducemean/_test_reducemean_parallel_4p.py new file mode 100644 index 00000000000..bfb885919c5 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/reducemean/_test_reducemean_parallel_4p.py @@ -0,0 +1,250 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore as ms +import mindspore.communication.management as distributedTool +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from numpy import allclose as allclose_nparray +from mindspore.ops.composite import grad_all_with_sens + +device_num = 4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, output_grad): + return grad_all_with_sens(self.network)(x, y, output_grad) + + +class GradScalar(Cell): + def __init__(self, network): + super(GradScalar, self).__init__() + self.network = network + self.sens = Tensor([1.0], dtype=ms.float32) + + def construct(self, x, y): + return grad_all_with_sens(self.network)(x, y, self.sens) + + +class ReduceMean(Cell): + def __init__(self, keep_dims, axis, strategy0=None, strategy1=None): + super(ReduceMean, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.reduce_mean = P.ReduceMean(keep_dims=keep_dims).set_strategy(strategy=strategy1) + self.axis = axis + + def construct(self, x, y): + out = self.add(x, y) + return self.reduce_mean(out, self.axis) + + +class ReduceMeanFactory: + def __init__(self, input_shape, keep_dims, axis, strategy0=None, strategy1=None): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size * s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size) % number_range - number_range / 2, input_shape).astype( + np.float32) + self.input_np2 = np.reshape(np.arange(0, size) % number_range - number_range / 4, input_shape).astype( + np.float32) + self.keep_dims = keep_dims + self.axis = axis + target_shape = self.input_np1.mean(axis=axis, keepdims=keep_dims).shape + target_size = 1 + for s in target_shape: + target_size = target_size * s + number_range = min(1000, target_size) + self.output_grad_np = np.array([1.0], dtype=np.float32) + if len(target_shape) > 0: + self.output_grad_np = np.reshape(np.arange(0, target_size) % number_range, target_shape).astype( + np.float32) + 1.0 + self.shape = target_shape + self.strategy0 = strategy0 + self.strategy1 = strategy1 + out_strategy = [] + axis_ = list(axis) + if axis_[0] == -1: + axis_[0] = len(input_shape) - 1 + for i in range(0, len(input_shape)): + if i in axis_: + if keep_dims: + out_strategy.append(1) + else: + out_strategy.append(strategy1[1][i]) + self.out_strategy = out_strategy + need_dev_num0 = 1 + need_dev_num1 = 1 + for s in strategy0[1]: + need_dev_num0 = need_dev_num0 * s + for s in out_strategy: + need_dev_num1 = need_dev_num1 * s + self.x_id = device_id % need_dev_num0 + self.y_id = device_id % need_dev_num0 + block_id = device_id % need_dev_num0 + device_index = self.id_to_list(block_id, self.strategy1[1]) + print(device_index) + for i in axis: + device_index[i] = 0 + print(device_index) + self.out_id = self.list_to_id(device_index, self.out_strategy) + print(self.out_id) + + def id_to_list(self, id, shape): + result = [] + r = id + for i in range(0, len(shape)): + v = 1 + for j in range(i + 1, len(shape)): + v = v * shape[j] + result.append(r // v) + r = r % v + return result + + def list_to_id(self, id_list, shape): + result = 0 + for i in range(0, len(id_list)): + v = 1 + for j in range(i + 1, len(id_list)): + v = v * shape[j] + result = result + id_list[i] * v + return result + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks) > 0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i += 1 + return blocks + + def forward_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + net = ReduceMean(keep_dims=self.keep_dims, axis=self.axis) + out = net(x, y) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + net = ReduceMean(keep_dims=self.keep_dims, axis=self.axis, strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + out_grad = Tensor(self.output_grad_np) + net = ReduceMean(keep_dims=self.keep_dims, axis=self.axis) + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, out_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + outgrads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + output_grad1 = Tensor(outgrads[self.out_id]) + net = ReduceMean(keep_dims=self.keep_dims, axis=self.axis, strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad1], + parallel_inputs_run=[x1, y1, output_grad1]) + return input_grad + + def forward_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.out_strategy) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.0001, 0.001) + + def grad_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[2]) + assert allclose_nparray(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert allclose_nparray(input_grad_blocks_1[self.y_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + + +def test_reid_reducemean_input_64x16(): + fact = ReduceMeanFactory(input_shape=(64 * 16,), keep_dims=False, axis=(-1,), strategy0=(0, (4,), (4,)), + strategy1=(0, (4,))) + fact.forward_cmp() + + +def test_grad_reid_reducemean_input_64x16(): + fact = ReduceMeanFactory(input_shape=(64 * 16,), keep_dims=False, axis=(-1,), strategy0=(0, (4,), (4,)), + strategy1=(0, (4,))) + fact.grad_cmp() + + +def test_reid_reducemean_input_64x128x28x28(): + fact = ReduceMeanFactory(input_shape=(64, 128, 32, 32), keep_dims=True, axis=(2, 3), + strategy0=(0, (2, 1, 2, 1), (2, 1, 2, 1)), strategy1=(0, (2, 1, 2, 1))) + fact.forward_cmp() + + +def test_grad_reid_reducemean_input_64x128x28x28(): + fact = ReduceMeanFactory(input_shape=(64, 128, 32, 32), keep_dims=True, axis=(2, 3), + strategy0=(0, (2, 1, 2, 1), (2, 1, 2, 1)), strategy1=(0, (2, 1, 2, 1))) + fact.grad_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/reducemean/reducemean_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/reducemean/reducemean_parallel_4p.sh new file mode 100644 index 00000000000..b78a5c2b6f8 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/reducemean/reducemean_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_reducemean_parallel_4p.py>../../log/test_reducemean_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/reshape/_test_reshape_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/reshape/_test_reshape_parallel_4p.py new file mode 100644 index 00000000000..c1c90e833dc --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/reshape/_test_reshape_parallel_4p.py @@ -0,0 +1,190 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore.communication.management as distributedTool +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from numpy import allclose as allclose_nparray +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, output_grad): + return grad_all_with_sens(self.network)(x, y, output_grad) + + +class Reshape(Cell): + def __init__(self, target_shape, strategy0 = None, strategy1 = None): + super(Reshape, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.reshape = P.Reshape(strategy=strategy1) + self.shape = tuple(target_shape) + + def construct(self, input1, input2): + x = self.add(input1, input2) + return self.reshape(x, self.shape) + + +class ReshapeFactory: + def __init__(self, input_shape, target_shape, strategy0, strategy1): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = np.reshape(np.arange(0, size)%number_range - number_range/4, input_shape).astype(np.float32) + target_size = 1 + for s in target_shape: + target_size = target_size*s + number_range = min(1000, target_size) + self.output_grad_np = np.reshape(np.arange(0, target_size)%number_range - number_range/2, target_shape).astype(np.float32) + self.target_shape = target_shape + self.strategy0 = strategy0 + self.strategy1 = strategy1 + out_strategy = [1]*len(target_shape) + out_strategy[0] = strategy1[1][0] + self.out_strategy = out_strategy + + need_dev_num0 = 1 + need_dev_num1 = 1 + for s in strategy0[1]: + need_dev_num0 = need_dev_num0*s + for s in out_strategy: + need_dev_num1 = need_dev_num1*s + self.x_id = device_id%need_dev_num0 + self.y_id = device_id%need_dev_num0 + self.out_id = device_id%need_dev_num1 + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def forward_reshape_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + net = Reshape(self.target_shape) + out = net(x, y) + return out.asnumpy() + + def forward_reshape_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + net = Reshape(self.target_shape, strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_reshape_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + net = Reshape(self.target_shape) + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, output_grad) + return input_grad + + def grad_reshape_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + outgrads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + output_grad1 = Tensor(outgrads[self.out_id]) + net = Reshape(self.target_shape, strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad1], parallel_inputs_run=[x1, y1, output_grad1]) + return input_grad + + def forward_reshape_cmp(self): + out_mindspore = self.forward_reshape_mindspore_impl() + out_mindspore_parallel = self.forward_reshape_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.out_strategy) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.0001, 0.001) + + def grad_reshape_cmp(self): + input_grad_mindspore = self.grad_reshape_mindspore_impl() + input_grad_mindspore_parallel = self.grad_reshape_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[2]) + assert allclose_nparray(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert allclose_nparray(input_grad_blocks_1[self.y_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + +@pytest.mark.reid_forward +def test_reid_reshape_input_128x512x7x7_target_128x25088(): + fact = ReshapeFactory(input_shape=(128, 512, 7, 7), target_shape=(128, 25088), strategy0=(0,(4,1,1,1),(4,1,1,1)), strategy1=(0,(4,1,1,1))) + fact.forward_reshape_cmp() + +def test_reid_reshape_grad_input_128x512x7x7_target_128x25088(): + fact = ReshapeFactory(input_shape=(128, 512, 7, 7), target_shape=(128, 25088), strategy0=(0,(4,1,1,1),(4,1,1,1)), strategy1=(0,(4,1,1,1))) + fact.grad_reshape_cmp() + +@pytest.mark.reid_forward +def test_reid_reshape_input_128x64_target_128x64x1x1(): + fact = ReshapeFactory(input_shape=(128, 64), target_shape=(128, 64, 1, 1), strategy0=(0,(2,1),(2,1)), strategy1=(0,(2,1))) + fact.forward_reshape_cmp() + +@pytest.mark.reid_grad +def test_reid_reshape_grad_input_128x64_target_128x64x1x1(): + fact = ReshapeFactory(input_shape=(128, 64), target_shape=(128, 64, 1, 1), strategy0=(0,(2,1),(2,1)), strategy1=(0,(2,1))) + fact.grad_reshape_cmp() + diff --git a/tests/ut/python/parallel/parallel_end_to_end/reshape/reshape_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/reshape/reshape_parallel_4p.sh new file mode 100644 index 00000000000..9561e9525e4 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/reshape/reshape_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_reshape_parallel_4p.py>../../log/test_reshape_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/parallel_end_to_end/transpose/_test_transpose_parallel_4p.py b/tests/ut/python/parallel/parallel_end_to_end/transpose/_test_transpose_parallel_4p.py new file mode 100644 index 00000000000..3e3f372e5a1 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/transpose/_test_transpose_parallel_4p.py @@ -0,0 +1,222 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import numpy as np +import mindspore.communication.management as distributedTool +from mindspore.nn import Cell +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from numpy import allclose as allclose_nparray +from mindspore.ops.composite import grad_all_with_sens + +device_num=4 +device_id = int(os.environ["RANK_ID"]) +path = "./output/" + +def setup_module(): + print("~~~~~~~~~~~set up~~~~~~~~~~~~~") + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(device_num=device_num, global_rank=device_id) + distributedTool.init() + distributedTool.create_group("0-3", [0, 1, 2, 3]) + print("~~~~~~~~~~~set up finished~~~~~~~~~~~~~") + +def teardown_module(): + print("~~~~~~~~~~~~tear down~~~~~~~~~~") + + + +class Net(Cell): + def __init__(self, perm_in, strategy0 = None, strategy1 = None): + super(Net, self).__init__() + self.add = P.TensorAdd(strategy=strategy0) + self.transpose = P.Transpose(strategy=strategy1) + self.perm_in = perm_in + + def construct(self, x, y): + out = self.add(x, y) + return self.transpose(out, self.perm_in) + +class Grad(Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y, output_grad): + return grad_all_with_sens(self.network)(x, y, output_grad) + + + +class TransposeFactory: + def __init__(self, input_shape, perm_in, strategy0, strategy1): + prefix = "" + size = 1 + for s in input_shape: + prefix = prefix + str(s) + size = size*s + self.prefix = prefix + number_range = min(1000, size) + self.input_np1 = np.reshape(np.arange(0, size)%number_range - number_range/2, input_shape).astype(np.float32) + self.input_np2 = np.reshape(np.arange(0, size)%number_range - number_range/4, input_shape).astype(np.float32) + target_shape = self.input_np1.transpose(perm_in).shape + target_size = 1 + for s in target_shape: + target_size = target_size*s + number_range = min(1000, target_size) + self.target_shape = target_shape + self.output_grad_np = np.reshape(np.arange(0, target_size)%number_range - number_range/2, target_shape).astype(np.float32) + self.perm_in = perm_in + self.strategy0=strategy0 + self.strategy1=strategy1 + out_strategy=[] + for i in perm_in: + out_strategy.append(strategy1[1][i]) + self.out_strategy = out_strategy + need_dev_num0 = 1 + need_dev_num1 = 1 + for s in strategy0[1]: + need_dev_num0 = need_dev_num0*s + for s in out_strategy: + need_dev_num1 = need_dev_num1*s + self.x_id = device_id%need_dev_num0 + self.y_id = device_id%need_dev_num0 + device_index = self.id_to_list(device_id%need_dev_num1, self.strategy1[1]) #encoding to get the index before transpose + device_index_transpose = [] + for i in perm_in: + device_index_transpose.append(device_index[i]) + self.out_id = self.list_to_id(device_index_transpose, self.out_strategy) + + def get_parallel_blocks(self, input_, strategy): + blocks = [input_] + i = 0 + for stra in strategy: + temp = [] + while len(blocks)>0: + block = blocks.pop(0) + temp.extend(np.split(block, stra, axis=i)) + blocks.extend(temp) + i+=1 + return blocks + + def id_to_list(self, id, shape): + result = [] + r = id + for i in range(0, len(shape)): + v = 1 + for j in range(i+1, len(shape)): + v = v*shape[j] + result.append(r//v) + r = r%v + return result + + def list_to_id(self, id_list, shape): + result = 0 + for i in range(0, len(id_list)): + v = 1 + for j in range(i+1, len(id_list)): + v = v*shape[j] + result = result + id_list[i]*v + return result + + def forward_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + net = Net(self.perm_in) + out = net(x, y) + return out.asnumpy() + + def forward_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + net = Net(self.perm_in, strategy0=self.strategy0, strategy1=self.strategy1) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + out = net(x, y, parallel_inputs_compile=[x, y], parallel_inputs_run=[x1, y1]) + return out.asnumpy() + + def grad_mindspore_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + net = Net(self.perm_in) + grad_net = Grad(net) + grad_net.set_train() + input_grad = grad_net(x, y, output_grad) + return input_grad + + def grad_mindspore_parallel_impl(self): + x = Tensor(self.input_np1) + y = Tensor(self.input_np2) + output_grad = Tensor(self.output_grad_np) + inputs_x = self.get_parallel_blocks(self.input_np1, self.strategy0[1]) + inputs_y = self.get_parallel_blocks(self.input_np2, self.strategy0[1]) + outgrads = self.get_parallel_blocks(self.output_grad_np, self.out_strategy) + x1 = Tensor(inputs_x[self.x_id]) + y1 = Tensor(inputs_y[self.y_id]) + output_grad1 = Tensor(outgrads[self.out_id]) + net = Net(self.perm_in, strategy0=self.strategy0, strategy1=self.strategy1) + grad_net = Grad(net) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + grad_net.set_train() + input_grad = grad_net(x, y, output_grad, parallel_inputs_compile=[x, y, output_grad1], parallel_inputs_run=[x1, y1, output_grad1]) + return input_grad + + def forward_transpose_cmp(self): + out_mindspore = self.forward_mindspore_impl() + out_mindspore_parallel = self.forward_mindspore_parallel_impl() + out_blocks = self.get_parallel_blocks(out_mindspore, self.out_strategy) + assert np.allclose(out_blocks[self.out_id], out_mindspore_parallel, 0.0001, 0.001) + + def grad_transpose_cmp(self): + input_grad_mindspore = self.grad_mindspore_impl() + input_grad_mindspore_parallel = self.grad_mindspore_parallel_impl() + input_grad_mindspore0 = input_grad_mindspore[0].asnumpy() + input_grad_mindspore1 = input_grad_mindspore[1].asnumpy() + input_grad_mindspore_parallel0 = input_grad_mindspore_parallel[0].asnumpy() + input_grad_mindspore_parallel1 = input_grad_mindspore_parallel[1].asnumpy() + input_grad_blocks_0 = self.get_parallel_blocks(input_grad_mindspore0, self.strategy0[1]) + input_grad_blocks_1 = self.get_parallel_blocks(input_grad_mindspore1, self.strategy0[2]) + assert allclose_nparray(input_grad_blocks_0[self.x_id], input_grad_mindspore_parallel0, 0.0001, 0.0001) + assert allclose_nparray(input_grad_blocks_1[self.y_id], input_grad_mindspore_parallel1, 0.0001, 0.0001) + + +def test_reid_transpose_input_256x512_output_512x256_perm_1x0(): + fact = TransposeFactory((256, 512), (1, 0), strategy0=(0,(2,2),(2,2)), strategy1=(0,(2,2))) + fact.forward_transpose_cmp() + +def test_reid_grad_transpose_input_256x512_output_512x256_perm_1x0(): + fact = TransposeFactory((256, 512), (1, 0), strategy0=(0,(2,2),(2,2)), strategy1=(0,(2,2))) + fact.grad_transpose_cmp() + +def test_reid_transpose_input_512x256_output_256x512_perm_1x0(): + fact = TransposeFactory((512, 256), (1, 0), strategy0=(0,(4,1),(4,1)), strategy1=(0,(1,4))) + fact.forward_transpose_cmp() + +def test_reid_grad_transpose_input_512x256_output_256x512_perm_1x0(): + fact = TransposeFactory((512, 256), (1, 0), strategy0=(0,(4,1),(4,1)), strategy1=(0,(1,4))) + fact.grad_transpose_cmp() + +def test_reid_transpose_input_512x256_output_256x512_perm_1x0_repeat(): + fact = TransposeFactory((512, 256), (1, 0), strategy0=(0,(2,1),(2,1)), strategy1=(0,(2,1))) + fact.forward_transpose_cmp() + +def test_reid_grad_transpose_input_512x256_output_256x512_perm_1x0_repeat(): + fact = TransposeFactory((512, 256), (1, 0), strategy0=(0,(2,1),(2,1)), strategy1=(0,(2,1))) + fact.grad_transpose_cmp() diff --git a/tests/ut/python/parallel/parallel_end_to_end/transpose/transpose_parallel_4p.sh b/tests/ut/python/parallel/parallel_end_to_end/transpose/transpose_parallel_4p.sh new file mode 100644 index 00000000000..ac4962b8884 --- /dev/null +++ b/tests/ut/python/parallel/parallel_end_to_end/transpose/transpose_parallel_4p.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +for((i=0;i<4;i++)); +do + rm -rf device$i + mkdir device$i + cd device$i + mkdir output + source ../../dist_env_4p.sh $i + env >log$i.log + pytest -s ../test_transpose_parallel_4p.py>../../log/test_transpose_parallel_4p_log$i.log 2>&1 & + cd .. +done diff --git a/tests/ut/python/parallel/test_add_relu_redistribution.py b/tests/ut/python/parallel/test_add_relu_redistribution.py new file mode 100644 index 00000000000..901b69728d9 --- /dev/null +++ b/tests/ut/python/parallel/test_add_relu_redistribution.py @@ -0,0 +1,74 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class AddRelu(nn.Cell): + def __init__(self, strategy0=None, strategy1=None): + super(AddRelu, self).__init__() + self.add = P.TensorAdd().set_strategy(strategy=strategy0) + self.relu = P.ReLU().set_strategy(strategy=strategy1) + + def construct(self, x, z): + out = self.add(x, z) + return self.relu(out) + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, z): + predict = self.network(x, z) + return self.loss(predict) + +class Grad(nn.Cell): + def __init__(self, network): + super(Grad, self).__init__() + self.network = network + + def construct(self, x, y): + return C.grad_all(self.network)(x, y) + +def test_add_relu_stride_slice(): + context.set_auto_parallel_context(device_num=8, global_rank=7) + + strategy0 = ((1, 1), (1, 1)) + strategy1 = ((8, 1), ) + net = Grad(NetWithLoss(AddRelu(strategy0, strategy1))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([128, 32]), dtype=ms.float32) + _executor.compile(net, x, y) + +def test_add_relu_all_gather(): + context.set_auto_parallel_context(device_num=8, global_rank=7) + + strategy0 = ((8, 1), (8, 1)) + strategy1 = ((1, 1), ) + net = Grad(NetWithLoss(AddRelu(strategy0, strategy1))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([128, 32]), dtype=ms.float32) + _executor.compile(net, x, y) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_allreduce_fusion.py b/tests/ut/python/parallel/test_allreduce_fusion.py new file mode 100644 index 00000000000..e59ce380512 --- /dev/null +++ b/tests/ut/python/parallel/test_allreduce_fusion.py @@ -0,0 +1,294 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor, context +import mindspore as ms +import numpy as np +import mindspore.nn as nn +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.common.api import _executor +from mindspore.parallel import _cost_model_context as cost_model_context + + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class DenseNet1(nn.Cell): + def __init__(self, has_bias=True, activation='relu'): + super(DenseNet1, self).__init__() + self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + + def construct(self, x): + q = self.fc1(x) + k = self.fc2(q) + v = self.fc3(k) + s = self.fc4(v) + return s + + +class DenseNet2(nn.Cell): + def __init__(self, has_bias=True, activation='relu'): + super(DenseNet2, self).__init__() + self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc5 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc6 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc7 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + self.fc8 = nn.Dense(128, 128, has_bias=has_bias, activation=activation) + + def construct(self, x): + q = self.fc1(x) + k = self.fc2(q) + v = self.fc3(k) + s = self.fc4(v) + t = self.fc5(s) + u = self.fc6(t) + w = self.fc7(u) + z = self.fc8(w) + return z + +class SimpleDMLNet(nn.Cell): + def __init__(self, net1, net2): + super(SimpleDMLNet, self).__init__() + self.backbone1 = net1 + self.backbone2 = net2 + + def construct(self, x): + x1 = self.backbone1(x) + x2 = self.backbone2(x) + return x1 + x2 + + +def train_common(net): + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + device_num=4 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=device_num, parameter_broadcast=False) + context.set_context(mode=context.GRAPH_MODE) + + predict = Tensor(np.ones([batch_size, 128]), dtype=ms.float32) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss, opt) + + model.train(epoch_size, dataset, dataset_sink_mode=False) + allreduce_fusion_dict = _executor._get_allreduce_fusion(model._train_network) + + print(allreduce_fusion_dict) + return allreduce_fusion_dict + + +def test_allreduce_fusion_parameters(): + cost_model_context.reset_cost_model_context() + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=2) + algorithm = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_algorithm') + assert (algorithm == 2) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1) + algorithm = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_algorithm') + assert (algorithm == 1) + cost_model_context.reset_cost_model_context() + algorithm = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_algorithm') + assert (algorithm == 0) + + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2) + fusion_times = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_times') + assert (fusion_times == 2) + + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.2) + tail_percent = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_percent') + assert (tail_percent == 0.2) + cost_model_context.reset_cost_model_context() + tail_percent = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_percent') + assert (tail_percent == 0.1) + + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_time=0.2) + tail_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_time') + assert (tail_time == 0.2) + cost_model_context.reset_cost_model_context() + tail_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_time') + assert (tail_time == 0.1) + + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_inherent_time=0.2) + allreduce_inherent_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_inherent_time') + assert (allreduce_inherent_time == 0.2) + cost_model_context.reset_cost_model_context() + allreduce_inherent_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_inherent_time') + assert (allreduce_inherent_time == 0.1) + + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_bandwidth=0.2) + allreduce_bandwidth = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_bandwidth') + assert (allreduce_bandwidth == 0.2) + cost_model_context.reset_cost_model_context() + allreduce_bandwidth = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_bandwidth') + assert (allreduce_bandwidth == 0.1) + + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_computation_time_parameter=0.2) + computation_time_parameter = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_computation_time_parameter') + assert (computation_time_parameter == 0.2) + cost_model_context.reset_cost_model_context() + computation_time_parameter = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_computation_time_parameter') + assert (computation_time_parameter == 0.1) + + +def test_allreduce_fusion1(): + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5) + net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None)) + allreduce_fusion_dict = train_common(net) + expect_dict = {'backbone2.fc8.weight': 2, + 'backbone2.fc7.weight': 2, + 'backbone2.fc6.weight': 2, + 'backbone1.fc4.weight': 2, + 'backbone1.fc3.weight': 2, + 'backbone1.fc2.weight': 2, + 'backbone2.fc5.weight': 1, + 'backbone2.fc4.weight': 1, + 'backbone2.fc3.weight': 1, + 'backbone2.fc2.weight': 1, + 'backbone2.fc1.weight': 1, + 'backbone1.fc1.weight': 1} + assert (allreduce_fusion_dict == expect_dict) + cost_model_context.reset_cost_model_context() + + +# reset_cost_model_context is called, the default value of costmodel_allreduce_fusion_times is 0, step_allreduce_fusion +# is bypassed. +def test_allreduce_fusion2(): + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5) + cost_model_context.reset_cost_model_context() + net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None)) + allreduce_fusion_dict = train_common(net) + expect_dict = {} + assert (allreduce_fusion_dict == expect_dict) + cost_model_context.reset_cost_model_context() + + +def test_allreduce_fusion3(): + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=3) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.3333333) + net = SimpleDMLNet(DenseNet1(has_bias=True, activation='relu'), DenseNet2(has_bias=False, activation='relu')) + allreduce_fusion_dict = train_common(net) + expect_dict = {'backbone2.fc8.weight': 3, + 'backbone2.fc7.weight': 3, + 'backbone2.fc6.weight': 2, + 'backbone2.fc5.weight': 2, + 'backbone2.fc4.weight': 2, + 'backbone2.fc3.weight': 1, + 'backbone2.fc2.weight': 1, + 'backbone2.fc1.weight': 1, + 'backbone1.fc4.bias': 3, + 'backbone1.fc4.weight': 3, + 'backbone1.fc3.bias': 3, + 'backbone1.fc3.weight': 2, + 'backbone1.fc2.bias': 2, + 'backbone1.fc2.weight': 2, + 'backbone1.fc1.bias': 2, + 'backbone1.fc1.weight': 2} + assert (allreduce_fusion_dict == expect_dict) + cost_model_context.reset_cost_model_context() + + +def test_allreduce_fusion4(): + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5) + net = SimpleDMLNet(DenseNet2(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None)) + allreduce_fusion_dict = train_common(net) + expect_dict = {'backbone2.fc8.weight': 2, + 'backbone2.fc7.weight': 2, + 'backbone2.fc6.weight': 2, + 'backbone1.fc8.weight': 2, + 'backbone1.fc7.weight': 2, + 'backbone1.fc6.weight': 2, + 'backbone2.fc5.weight': 1, + 'backbone2.fc4.weight': 1, + 'backbone2.fc3.weight': 1, + 'backbone2.fc2.weight': 1, + 'backbone2.fc1.weight': 1, + 'backbone1.fc5.weight': 1, + 'backbone1.fc4.weight': 1, + 'backbone1.fc3.weight': 1, + 'backbone1.fc2.weight': 1, + 'backbone1.fc1.weight': 1} + + assert (allreduce_fusion_dict == expect_dict) + cost_model_context.reset_cost_model_context() + + +def test_allreduce_fusion5(): + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=2) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_time=0.1) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_inherent_time=0.05) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_bandwidth=0.000001) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_computation_time_parameter=0.0000015) + net = SimpleDMLNet(DenseNet2(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None)) + allreduce_fusion_dict = train_common(net) + + expect_dict = {'backbone2.fc8.weight': 3, + 'backbone2.fc7.weight': 3, + 'backbone2.fc6.weight': 3, + 'backbone2.fc5.weight': 2, + 'backbone2.fc4.weight': 2, + 'backbone2.fc3.weight': 2, + 'backbone2.fc2.weight': 1, + 'backbone2.fc1.weight': 1, + 'backbone1.fc8.weight': 3, + 'backbone1.fc7.weight': 3, + 'backbone1.fc6.weight': 3, + 'backbone1.fc5.weight': 2, + 'backbone1.fc4.weight': 2, + 'backbone1.fc3.weight': 2, + 'backbone1.fc2.weight': 1, + 'backbone1.fc1.weight': 1,} + + assert (allreduce_fusion_dict == expect_dict) + cost_model_context.reset_cost_model_context() + diff --git a/tests/ut/python/parallel/test_alltoall.py b/tests/ut/python/parallel/test_alltoall.py new file mode 100644 index 00000000000..ddceece8ec2 --- /dev/null +++ b/tests/ut/python/parallel/test_alltoall.py @@ -0,0 +1,113 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.parallel._utils import _reset_op_id +from mindspore.common.api import _executor + + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class AllToAllNet(nn.Cell): + def __init__(self, strategy1): + super(AllToAllNet, self).__init__() + self.matmul = P.MatMul().set_strategy(((1, 1), (1, 8))) + self.matmul_weight = Parameter(Tensor(np.ones([128, 256]), dtype=ms.float32), name="weight") + self.transpose1 = P.Transpose().set_strategy(strategy1) + + def construct(self, x): + x = self.matmul(x, self.matmul_weight) + x = self.transpose1(x, (1, 0)) + return x + + +def all_to_all_net(strategy1): + return AllToAllNet(strategy1=strategy1) + + +def all_to_all_common(strategy1): + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=8) + predict = Tensor(np.ones([32, 128]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = all_to_all_net(strategy1) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((8, 1), (8, 1))) + loss.one_hot.set_strategy(((8,1), (), ())) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss, opt) + + model.train(epoch_size, dataset, dataset_sink_mode=False) + strategys = _executor._get_strategy(model._train_network) + return strategys + + +def test_all_to_all(): + strategy1 = ((8, 1), ) + context.set_context(mode=context.GRAPH_MODE, save_graphs=False) + _reset_op_id() + strategys = all_to_all_common(strategy1) + print(strategys) + expect_dict = {'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits' + '/SoftmaxCrossEntropyWithLogits-op43': [[8, 1], [8, 1]], + 'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits' + '/OneHot-op44': [[8, 1], [], []], + 'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_backbone-AllToAllNet/Transpose-op1': + [[8, 1]], + 'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_backbone-AllToAllNet/MatMul-op0': + [[1, 1], [1, 8]]} + assert (strategys == expect_dict) + context.set_context(save_graphs=False) + + +if __name__ == '__main__': + test_all_to_all() + diff --git a/tests/ut/python/parallel/test_arithmetic.py b/tests/ut/python/parallel/test_arithmetic.py new file mode 100644 index 00000000000..2c7eabc8f2f --- /dev/null +++ b/tests/ut/python/parallel/test_arithmetic.py @@ -0,0 +1,472 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def test_matmul_sub(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.sub = P.Sub().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.sub(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_add(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.add = P.TensorAdd().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.add(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_mul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.mul = P.Mul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.mul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_div(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.div = P.Div().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.div(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_matmul_greater(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.greater = P.Greater().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.greater(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_matmul_add_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.add = P.TensorAdd().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.add(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_add_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.add = P.TensorAdd().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.add(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_sub_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.sub = P.Sub().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.sub(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_sub_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.sub = P.Sub().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.sub(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_mul_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.mul = P.Mul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.mul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_mul_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.mul = P.Mul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.mul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_div_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.div = P.Div().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.div(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_div_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.div = P.Div().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.div(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_matmul_greater_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.greater = P.Greater().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.greater(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_greater_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.greater = P.Greater().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.greater(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_matmul_floordiv(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.floordiv = P.FloorDiv().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_floordiv_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.floordiv = P.FloorDiv().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_floordiv_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.floordiv = P.FloorDiv().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_BN_PReLU.py b/tests/ut/python/parallel/test_auto_parallel_BN_PReLU.py new file mode 100755 index 00000000000..3a497192353 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_BN_PReLU.py @@ -0,0 +1,69 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x): + predict = self.network(x) + return self.loss(predict) + + +def bn_with_initialize(out_channels): + bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=1e-5).add_flags_recursive(fp32=True) + return bn + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x): + return C.grad_all(self.network)(x) + + # model_parallel test +def test_auto_parallel_bn_with_prelu(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.bn = bn_with_initialize(16) + self.prelu = nn.PReLU(16) + + def construct(self, x): + out = self.bn(x) + out = self.prelu(out) + return out + + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + + x = Tensor(np.random.rand(16, 16, 32, 64),dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x) diff --git a/tests/ut/python/parallel/test_auto_parallel_arithmetic.py b/tests/ut/python/parallel/test_auto_parallel_arithmetic.py new file mode 100644 index 00000000000..e6f72d80190 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_arithmetic.py @@ -0,0 +1,150 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel._utils import _reset_op_id as reset_op_id +from mindspore import context +context.set_context(mode=context.GRAPH_MODE) +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +def test_auto_parallel_arithmetic(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.floordiv = P.FloorDiv() + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 128]), dtype=ms.float32) + b = Tensor(np.ones([64, 128]), dtype=ms.float32) + _executor.compile(net, x, y, b, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/FloorDiv-op2': [[2, 4], [2, 4]], + 'Default/network-Net/MatMul-op3': [[2, 1], [1, 4]]} + assert strategies == expected_strategies + +def test_auto_parallel_arithmetic_broadcast_both(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.floordiv = P.FloorDiv() + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/FloorDiv-op2': [[8, 1], [1, 1]], + 'Default/network-Net/MatMul-op3': [[8, 1], [1, 1]]} + assert strategies == expected_strategies + + +def test_auto_parallel_arithmetic_broadcast_right(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.floordiv = P.FloorDiv() + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 32]), dtype=ms.float32) + b = Tensor(np.ones([32]), dtype=ms.float32) + _executor.compile(net, x, y, b, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/FloorDiv-op2': [[4, 2], [2]], + 'Default/network-Net/MatMul-op3': [[4, 1], [1, 2]]} + assert strategies == expected_strategies + + +def test_auto_parallel_arithmetic_broadcast_left(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.floordiv = P.FloorDiv() + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.floordiv(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 32]), dtype=ms.float32) + b = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) + _executor.compile(net, x, y, b, phase="train") + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/FloorDiv-op2': [[4, 2], [1, 4, 2]], + 'Default/network-Net/MatMul-op3': [[4, 1], [1, 2]]} + assert strategies == expected_strategies \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_assign_sub_with_ref_key.py b/tests/ut/python/parallel/test_auto_parallel_assign_sub_with_ref_key.py new file mode 100755 index 00000000000..e436f9faf77 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_assign_sub_with_ref_key.py @@ -0,0 +1,60 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel._utils import _reset_op_id as reset_op_id + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x): + predict = self.network(x) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x): + return C.grad_all(self.network)(x) + + + # model_parallel test +def test_auto_parallel_assign_sub_with_ref_key(): + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + + x = Tensor(np.random.rand(4, 4, 32, 64),dtype=ms.float32) + + net = NetWithLoss(nn.PReLU(4)) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + _executor.compile(net, x, phase="train") + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-PReLU/PReLU-op2': [[1, 1, 1, 8], [1]], + 'Default/network-PReLU/ReLU-op3': [[1]]} + assert strategies == expected_strategies diff --git a/tests/ut/python/parallel/test_auto_parallel_cast.py b/tests/ut/python/parallel/test_auto_parallel_cast.py new file mode 100644 index 00000000000..be7b5082d65 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_cast.py @@ -0,0 +1,83 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common import dtype as mstype +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel._utils import _reset_op_id as reset_op_id + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z, w): + predict = self.network(x, y, z, w) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z, w): + return C.grad_all(self.network)(x, y, z, w) + + # model_parallel test +def test_double_star_graph(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + self.cast1 = P.Cast() + self.cast2 = P.Cast() + + + def construct(self, x, y, z, w): + m1_result = self.matmul1(x, y) + m2_result = self.matmul2(z, w) + m3_result = self.matmul3(self.cast1(m2_result, mstype.float16), self.cast2(m1_result, mstype.float16)) + + return m3_result + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + + x = Tensor(np.ones([32, 8]), dtype=ms.float32) + y = Tensor(np.ones([8, 16]), dtype=ms.float32) + z = Tensor(np.ones([8, 16]), dtype=ms.float32) + w = Tensor(np.ones([16, 32]), dtype=ms.float32) + + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + _executor.compile(net, x, y, z, w, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/MatMul-op0': [[1, 8], [8, 1]], + 'Default/network-Net/Cast-op7': [[8, 1]], + 'Default/network-Net/MatMul-op8': [[8, 1], [1, 1]], + 'Default/network-Net/Cast-op9': [[1, 8]], + 'Default/network-Net/MatMul-op10': [[1, 1], [1, 8]]} + assert strategies == expected_strategies \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_common_parameter.py b/tests/ut/python/parallel/test_auto_parallel_common_parameter.py new file mode 100644 index 00000000000..89d73855c1f --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_common_parameter.py @@ -0,0 +1,66 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z): + predict = self.network(x, y, z) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z): + return C.grad_all(self.network)(x, y, z) + + # model_parallel test +def test_common_parameter(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + + def construct(self, x, y, z): + w = self.matmul1(x, y) + b = self.matmul2(x, z) + out = self.matmul3(b, w) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + z = Tensor(np.ones([32, 128]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_double_star.py b/tests/ut/python/parallel/test_auto_parallel_double_star.py new file mode 100644 index 00000000000..275046b6a6f --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_double_star.py @@ -0,0 +1,77 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z, w, a, b, c): + predict = self.network(x, y, z, w, a, b, c) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z, w, a, b, c): + return C.grad_all(self.network)(x, y, z, w, a, b, c) + + # model_parallel test +def test_double_star_graph(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + self.matmul4 = P.MatMul() + self.matmul5 = P.MatMul() + self.matmul6 = P.MatMul() + + def construct(self, x, y, z, w, a, b, c): + m1_result = self.matmul1(x, y) + m2_result = self.matmul2(z, w) + m3_result = self.matmul3(m2_result, m1_result) + m4_result = self.matmul4(a, b) + m5_result = self.matmul5(m3_result, m4_result) + out = self.matmul6(m5_result, c) + + return out + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([32, 8]), dtype=ms.float32) + y = Tensor(np.ones([8, 16]), dtype=ms.float32) + z = Tensor(np.ones([8, 16]), dtype=ms.float32) + w = Tensor(np.ones([16, 32]), dtype=ms.float32) + a = Tensor(np.ones([16, 8]), dtype=ms.float32) + b = Tensor(np.ones([8, 32]), dtype=ms.float32) + c = Tensor(np.ones([32, 32]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z, w, a, b, c) diff --git a/tests/ut/python/parallel/test_auto_parallel_fc_nobias.py b/tests/ut/python/parallel/test_auto_parallel_fc_nobias.py new file mode 100644 index 00000000000..b5f5df905cb --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_fc_nobias.py @@ -0,0 +1,64 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + # model_parallel test +def test_two_matmul(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul(transpose_b=True) + self.matmul2 = P.MatMul() + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_four_matmul.py b/tests/ut/python/parallel/test_auto_parallel_four_matmul.py new file mode 100644 index 00000000000..adabdbfdc4e --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_four_matmul.py @@ -0,0 +1,123 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z, w, b): + predict = self.network(x, y, z, w, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z, w, b): + return C.grad_all(self.network)(x, y, z, w, b) + + # model_parallel test +def test_four_matmul_linear(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + self.matmul4 = P.MatMul() + + def construct(self, x, y, z, w, b): + out = self.matmul1(x, y) + out = self.matmul2(out, z) + out = self.matmul3(out, w) + out = self.matmul4(out, b) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + z = Tensor(np.ones([64, 32]), dtype=ms.float32) + w = Tensor(np.ones([32, 32]), dtype=ms.float32) + b = Tensor(np.ones([32, 256]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z, w, b) + + +def test_four_matmul1(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + + def construct(self, x, y, z, w, b): + out = self.matmul(x, y) + out = self.matmul(out, z) + out = self.matmul(out, w) + out = self.matmul(out, b) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + z = Tensor(np.ones([64, 32]), dtype=ms.float32) + w = Tensor(np.ones([32, 32]), dtype=ms.float32) + b = Tensor(np.ones([32, 256]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z, w, b) + + +def test_four_matmul2(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + + def construct(self, x, y, z, w, b): + out = self.matmul(x, y) + out = out - z + out = self.matmul(out, w) + out = out - b + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + z = Tensor(np.ones([128, 64]), dtype=ms.float32) + w = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([128, 32]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z, w, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_l2normalize.py b/tests/ut/python/parallel/test_auto_parallel_l2normalize.py new file mode 100644 index 00000000000..40b6707d8eb --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_l2normalize.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel._utils import _reset_op_id as reset_op_id + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def test_auto_parallel_l2normalize(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.norm1 = P.L2Normalize() + self.norm2 = P.L2Normalize() + self.mul1 = P.Mul() + self.mul2 = P.Mul() + + def construct(self, x, y, b): + y = self.norm1(y) + x = self.norm2(x) + out = self.mul1(x, y) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + x = Tensor(np.ones([128, 64, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 64, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b, phase='train') + diff --git a/tests/ut/python/parallel/test_auto_parallel_matmul_drop.py b/tests/ut/python/parallel/test_auto_parallel_matmul_drop.py new file mode 100644 index 00000000000..550ad11ca99 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_matmul_drop.py @@ -0,0 +1,68 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +# model_parallel test +def test_two_matmul_dropout(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.dropout = nn.Dropout() + self.matmul2 = P.MatMul() + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.dropout(out) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_matmul_prelu.py b/tests/ut/python/parallel/test_auto_parallel_matmul_prelu.py new file mode 100644 index 00000000000..ca9b561145b --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_matmul_prelu.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel._utils import _reset_op_id as reset_op_id + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + # model_parallel test +def test_matmul_prelu(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.mul1 = P.Mul() + self.prelu = P.PReLU() + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.prelu(out, b) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([16, 3, 128, 32]), dtype=ms.float32) + y = Tensor(np.ones([16, 3, 128, 32]), dtype=ms.float32) + b = Tensor(np.array([0.01, 0.02, 0.03]), dtype=ms.float32) + + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + _executor.compile(net, x, y, b, phase='train') + strategies = _executor._get_strategy(net) + assert strategies['Default/network-Net/PReLU-op2'] == [[16, 1, 1, 1], [1]] + assert strategies['Default/network-Net/Mul-op3'] == [[16, 1, 1, 1], [16, 1, 1, 1]] + + diff --git a/tests/ut/python/parallel/test_auto_parallel_onehot.py b/tests/ut/python/parallel/test_auto_parallel_onehot.py new file mode 100644 index 00000000000..d82e3705f54 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_onehot.py @@ -0,0 +1,127 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore.train import Model, ParallelMode +from mindspore.nn.optim.momentum import Momentum + + +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +def test_auto_parallel_arithmetic(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.one_hot = P.OneHot() + self.on_value = Tensor(1.0, ms.float32) + self.off_value = Tensor(0.0, ms.float32) + self.matmul2 = P.MatMul() + + def construct(self, x, y, b): + out = self.matmul(x, y) + out1 = self.one_hot(b, 64, self.on_value, self.off_value) + out2 = self.matmul2(out, out1) + return out2 + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.int32) + _executor.compile(net, x, y, b) + + +def test_auto_parallel_arithmetic_model(): + class NetOneHot(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.one_hot = P.OneHot().set_strategy(((1, 8), (), ())) + self.on_value = Tensor(1.0, ms.float32) + self.off_value = Tensor(0.0, ms.float32) + self.matmul2 = P.MatMul() + self.w = Parameter(Tensor(np.zeros([32, 64]).astype(np.float32)), "weight", requires_grad=True) + + def construct(self, x, b): + out = self.matmul(x, self.w) + out1 = self.one_hot(b, 64, self.on_value, self.off_value) + out2 = self.matmul2(out, out1) + return out2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode=ParallelMode.AUTO_PARALLEL) + net = NetOneHot() + + x = Tensor(np.ones([8, 32]), dtype=ms.float32) + b = Tensor(np.ones([8]), dtype=ms.int32) + dataset = Dataset(x, b, 2) + + opt = Momentum(net.trainable_params(), 0.1, 0.9) + model = Model(net, optimizer=opt) + + model.train(2, dataset, dataset_sink_mode=False) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_parameter_cast.py b/tests/ut/python/parallel/test_auto_parallel_parameter_cast.py new file mode 100644 index 00000000000..67b8f98fafa --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_parameter_cast.py @@ -0,0 +1,88 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common import dtype as mstype +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore import Tensor, Parameter +from mindspore.parallel._utils import _reset_op_id as reset_op_id +from mindspore.parallel import set_algo_parameters + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z, w): + predict = self.network(x, y, z, w) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z, w): + return C.grad_all(self.network)(x, y, z, w) + + # model_parallel test +def test_common_parameter(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + self.weight1 = Parameter(Tensor(np.ones([64, 64]).astype(np.float16) * 0.01), "w", requires_grad=True) + self.cast1 = P.Cast() + self.cast2 = P.Cast() + + + def construct(self, x, y, z, w): + m1_result = self.matmul1(x, self.cast1(self.weight1, mstype.float32)) + m2_result = self.matmul2(z, self.cast2(self.weight1, mstype.float32)) + m3_result = self.matmul3(m2_result, m1_result) + + return m3_result + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + + set_algo_parameters(elementwise_op_strategy_follow=True) + x = Tensor(np.ones([64, 64]), dtype=ms.float32) + y = Tensor(np.ones([64, 64]), dtype=ms.float32) + z = Tensor(np.ones([64, 64]), dtype=ms.float32) + w = Tensor(np.ones([64, 64]), dtype=ms.float32) + + + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + _executor.compile(net, x, y, z, w, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/MatMul-op8': [[1, 1], [1, 8]], + 'Default/network-Net/MatMul-op9': [[1, 1], [1, 8]], + 'Default/network-Net/Cast-op10': [[1, 8]], + 'Default/network-Net/MatMul-op0': [[1, 1], [1, 8]], + 'Default/network-Net/Cast-op11': [[1, 8]]} + assert strategies == expected_strategies \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_partial_strategy.py b/tests/ut/python/parallel/test_auto_parallel_partial_strategy.py new file mode 100644 index 00000000000..e5fda638e05 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_partial_strategy.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z, w, b): + predict = self.network(x, y, z, w, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z, w, b): + return C.grad_all(self.network)(x, y, z, w, b) + + # model_parallel test +def test_four_matmul_linear(): + class Net(nn.Cell): + def __init__(self, strategy1): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + self.matmul4 = P.MatMul() + + def construct(self, x, y, z, w, b): + out = self.matmul1(x, y) + out = self.matmul2(out, z) + out = self.matmul3(out, w) + out = self.matmul4(out, b) + return out + + size = 64 + context.set_auto_parallel_context(device_num=size, global_rank=0) + strategy1 = ((2, 4), (4, 8)) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + z = Tensor(np.ones([64, 32]), dtype=ms.float32) + w = Tensor(np.ones([32, 32]), dtype=ms.float32) + b = Tensor(np.ones([32, 256]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net(strategy1))) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + _executor.compile(net, x, y, z, w, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_reduce_method.py b/tests/ut/python/parallel/test_auto_parallel_reduce_method.py new file mode 100644 index 00000000000..6e5576ed53c --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_reduce_method.py @@ -0,0 +1,113 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +# model_parallel test +def test_sum_mul(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.mul1 = P.Mul() + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.mul2 = P.Mul() + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, (0,)) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_sum_mul2(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.mul1 = P.Mul() + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.mul2 = P.Mul() + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, (0, 1)) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + x = Tensor(np.ones([128, 128, 64, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 128, 64, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_sum_mul3(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.mul1 = P.Mul() + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.mul2 = P.Mul() + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, -1) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 32]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_reshape.py b/tests/ut/python/parallel/test_auto_parallel_reshape.py new file mode 100644 index 00000000000..26e7e95a9fa --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_reshape.py @@ -0,0 +1,69 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.common.parameter import Parameter + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x): + predict = self.network(x) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x): + return C.grad_all(self.network)(x) + +# core dump, step_auto_parallel should SetInputs for transpose axis +def test_reshape_matmul(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.reshape = P.Reshape() + self.matmul = P.MatMul() + self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + + def construct(self, x): + out = self.reshape(x, (256, 25088)) + out = self.matmul(out, self.matmul_weight) + return out + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([32*size, 512, 7, 7]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x) + + +if __name__ == '__main__': + test_reshape_matmul() \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_resnet.py b/tests/ut/python/parallel/test_auto_parallel_resnet.py new file mode 100644 index 00000000000..667e3873a6f --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_resnet.py @@ -0,0 +1,498 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim.momentum import Momentum +from mindspore.common.initializer import TruncatedNormal +from mindspore.train.model import Model, ParallelMode +from mindspore import context +import os +import re +import mindspore.ops.functional as F +from mindspore.nn.loss.loss import _Loss +from mindspore.parallel._utils import _reset_op_id as resset_op_id +from mindspore.common.api import _executor +from mindspore.parallel import set_algo_parameters +from mindspore.parallel import _cost_model_context as cost_model_context + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") +context.set_context(enable_hccl=True) +context.set_context(enable_task_sink=True, device_id= 0) +context.set_context(enable_ir_fusion=True) +context.set_context(enable_loop_sink=False) + +def weight_variable(shape, factor=0.1): + return TruncatedNormal(0.02) + +def _conv3x3(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): + """Get a conv2d layer with 3x3 kernel size.""" + init_value = weight_variable((out_channels, in_channels, 3, 3)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) + +def _conv1x1(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): + """Get a conv2d layer with 1x1 kernel size.""" + init_value = weight_variable((out_channels, in_channels, 1, 1)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) + +def _conv7x7(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): + """Get a conv2d layer with 7x7 kernel size.""" + init_value = weight_variable((out_channels, in_channels, 7, 7)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) + +def _fused_bn(channels, momentum=0.9): + """Get a fused batchnorm""" + init_weight = weight_variable((channels,)) + init_bias = weight_variable((channels,)) + return nn.BatchNorm2d(channels, momentum=momentum) + +class ResidualBlock(nn.Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + momentum=0.9): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = _conv1x1(in_channels, out_chls, stride=1) + self.bn1 = _fused_bn(out_chls, momentum=momentum) + + self.conv2 = _conv3x3(out_chls, out_chls, stride=stride) + self.bn2 = _fused_bn(out_chls, momentum=momentum) + + self.conv3 = _conv1x1(out_chls, out_channels, stride=1) + self.bn3 = _fused_bn(out_channels, momentum=momentum) + + self.relu = P.ReLU() + self.downsample = (in_channels != out_channels) + self.stride = stride + if self.downsample: + self.conv_down_sample = _conv1x1(in_channels, out_channels, + stride=stride) + self.bn_down_sample = _fused_bn(out_channels, momentum=momentum) + elif self.stride != 1: + self.maxpool_down = nn.MaxPool2d(kernel_size=1, stride=2, pad_mode='same') + + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + elif self.stride != 1: + identity = self.maxpool_down(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides=[1,2,2,2], + num_classes=100): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of " + "layer_num, inchannel, outchannel list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2) + self.bn1 = _fused_bn(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same') + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.end_point = nn.Dense(2048, num_classes, has_bias=True, + weight_init=weight_variable((num_classes, 2048)), + bias_init=weight_variable((num_classes,))).add_flags_recursive(fp16=True) + self.squeeze = P.Squeeze() + self.cast = P.Cast() + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + layers = [] + down_sample = False + if stride != 1 or in_channel != out_channel: + down_sample = True + + resblk = block(in_channel, out_channel, stride=1) + layers.append(resblk) + + for _ in range(1, layer_num - 1): + resblk = block(out_channel, out_channel, stride=1) + layers.append(resblk) + + resblk = block(out_channel, out_channel, stride=stride) + layers.append(resblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1 = self.maxpool(x) + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + out = self.mean(c5, (2, 3)) + out = self.squeeze(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10): + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [2, 2, 2, 1], + class_num) + +class SoftmaxCrossEntropyExpand(_Loss): + def __init__(self, sparse=False): + super(SoftmaxCrossEntropyExpand, self).__init__() + self.exp = P.Exp() + self.sum = P.ReduceSum(keep_dims=True) + self.onehot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.div = P.Div() + self.log = P.Log() + self.sum_cross_entropy = P.ReduceSum(keep_dims=False) + self.mul = P.Mul() + self.mul2 = P.Mul() + self.cast = P.Cast() + self.mean = P.ReduceMean(keep_dims=False).add_prim_attr("cross_batch", True) + self.sparse = sparse + self.max = P.ReduceMax(keep_dims=True) + self.sub = P.Sub() + self.cast1 = P.Cast() + + def construct(self, logit, label): + logit = self.cast1(logit, mstype.float32) + logit_max = self.max(logit) + exp = self.exp(self.sub(logit, logit_max)) + exp_sum = self.sum(exp, -1) + softmax_result = self.div(exp, exp_sum) + if self.sparse: + label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + + softmax_result_log = self.log(softmax_result) + loss = self.sum_cross_entropy((self.mul(softmax_result_log, label)), -1) + loss = self.mul2(F.scalar_to_array(-1.0), loss) + loss = self.mean(loss, -1) + + return loss + + +class DatasetLenet(): + def __init__(self, predict, label, length=3): + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + def get_dataset_size(self): + return 32 + + def get_repeat_count(self): + return 1 + + +def test_train_32k_8p(epoch_size=3, batch_size=32, num_classes=32768): #1048576 #131072 #32768 #8192 + dev_num = 8 + context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, device_num=dev_num) + cost_model_context.set_cost_model_context(costmodel_gamma=0.001, costmodel_beta=260.0) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2) + cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5) + set_algo_parameters(elementwise_op_strategy_follow=True) + resset_op_id() + np.random.seed(6) + input_np = np.ones([batch_size, 3, 224, 224]).astype(np.float32) + label_np = np.zeros([batch_size]).astype(np.int32) + for i in range(0, batch_size): + label_np[i] = i % num_classes + dataset = DatasetLenet(Tensor(input_np), Tensor(label_np), 1) + net = resnet50(num_classes) + loss = SoftmaxCrossEntropyExpand(sparse=True) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) + model = Model(net, loss_fn=loss, optimizer=opt) + model.train(5, dataset, dataset_sink_mode=False) + strategies = _executor._get_strategy(model._train_network) + for (k, v) in strategies.items(): + if re.match(k, 'Conv2D-op') is not None: + assert v[0][0] == dev_num + elif re.match(k, 'MatMul-op') is not None: + assert v == [[dev_num, 1], [1, 1]] + elif re.match(k, 'ReduceSum-op') is not None: + assert v == [[dev_num, 1]] + + allreduce_fusion_dict = _executor._get_allreduce_fusion(model._train_network) + + print(allreduce_fusion_dict) + expect_dict = {'end_point.bias': 2, + 'end_point.weight': 2, + 'layer4.2.bn3.beta': 2, + 'layer4.2.bn3.gamma': 2, + 'layer4.2.conv3.weight': 2, + 'layer4.2.bn2.beta': 2, + 'layer4.2.bn2.gamma': 2, + 'layer4.2.conv2.weight': 2, + 'layer4.2.bn1.beta': 2, + 'layer4.2.bn1.gamma': 2, + 'layer4.2.conv1.weight': 2, + 'layer4.1.bn3.beta': 2, + 'layer4.1.bn3.gamma': 2, + 'layer4.1.conv3.weight': 2, + 'layer4.1.bn2.beta': 2, + 'layer4.1.bn2.gamma': 2, + 'layer4.1.conv2.weight': 2, + 'layer4.1.bn1.beta': 2, + 'layer4.1.bn1.gamma': 2, + 'layer4.1.conv1.weight': 2, + 'layer4.0.bn_down_sample.beta': 2, + 'layer4.0.bn_down_sample.gamma': 2, + 'layer4.0.conv_down_sample.weight': 2, + 'layer4.0.bn3.beta': 2, + 'layer4.0.bn3.gamma': 2, + 'layer4.0.conv3.weight': 2, + 'layer4.0.bn2.beta': 2, + 'layer4.0.bn2.gamma': 2, + 'layer4.0.conv2.weight': 2, + 'layer4.0.bn1.beta': 2, + 'layer4.0.bn1.gamma': 2, + 'layer4.0.conv1.weight': 2, + 'layer3.5.bn3.beta': 2, + 'layer3.5.bn3.gamma': 2, + 'layer3.5.conv3.weight': 2, + 'layer3.5.bn2.beta': 2, + 'layer3.5.bn2.gamma': 2, + 'layer3.5.conv2.weight': 2, + 'layer3.5.bn1.beta': 2, + 'layer3.5.bn1.gamma': 2, + 'layer3.5.conv1.weight': 2, + 'layer3.4.bn3.beta': 2, + 'layer3.4.bn3.gamma': 2, + 'layer3.4.conv3.weight': 2, + 'layer3.4.bn2.beta': 2, + 'layer3.4.bn2.gamma': 2, + 'layer3.4.conv2.weight': 2, + 'layer3.4.bn1.beta': 2, + 'layer3.4.bn1.gamma': 2, + 'layer3.4.conv1.weight': 2, + 'layer3.3.bn3.beta': 2, + 'layer3.3.bn3.gamma': 2, + 'layer3.3.conv3.weight': 2, + 'layer3.3.bn2.beta': 2, + 'layer3.3.bn2.gamma': 2, + 'layer3.3.conv2.weight': 2, + 'layer3.3.bn1.beta': 2, + 'layer3.3.bn1.gamma': 2, + 'layer3.3.conv1.weight': 2, + 'layer3.2.bn3.beta': 2, + 'layer3.2.bn3.gamma': 2, + 'layer3.2.conv3.weight': 2, + 'layer3.2.bn2.beta': 2, + 'layer3.2.bn2.gamma': 2, + 'layer3.2.conv2.weight': 2, + 'layer3.2.bn1.beta': 2, + 'layer3.2.bn1.gamma': 2, + 'layer3.2.conv1.weight': 2, + 'layer3.1.bn3.beta': 2, + 'layer3.1.bn3.gamma': 2, + 'layer3.1.conv3.weight': 2, + 'layer3.1.bn2.beta': 2, + 'layer3.1.bn2.gamma': 2, + 'layer3.1.conv2.weight': 2, + 'layer3.1.bn1.beta': 2, + 'layer3.1.bn1.gamma': 2, + 'layer3.1.conv1.weight': 2, + 'layer3.0.bn_down_sample.beta': 1, + 'layer3.0.bn_down_sample.gamma': 1, + 'layer3.0.conv_down_sample.weight': 2, + 'layer3.0.bn3.beta': 1, + 'layer3.0.bn3.gamma': 1, + 'layer3.0.conv3.weight': 2, + 'layer3.0.bn2.beta': 2, + 'layer3.0.bn2.gamma': 2, + 'layer3.0.conv2.weight': 2, + 'layer3.0.bn1.beta': 2, + 'layer3.0.bn1.gamma': 2, + 'layer3.0.conv1.weight': 2, + 'layer2.3.bn3.beta': 2, + 'layer2.3.bn3.gamma': 2, + 'layer2.3.conv3.weight': 2, + 'layer2.3.bn2.beta': 2, + 'layer2.3.bn2.gamma': 2, + 'layer2.3.conv2.weight': 2, + 'layer2.3.bn1.beta': 2, + 'layer2.3.bn1.gamma': 2, + 'layer2.3.conv1.weight': 2, + 'layer2.2.bn3.beta': 2, + 'layer2.2.bn3.gamma': 2, + 'layer2.2.conv3.weight': 2, + 'layer2.2.bn2.beta': 2, + 'layer2.2.bn2.gamma': 2, + 'layer2.2.conv2.weight': 2, + 'layer2.2.bn1.beta': 2, + 'layer2.2.bn1.gamma': 2, + 'layer2.2.conv1.weight': 2, + 'layer2.1.bn3.beta': 1, + 'layer2.1.bn3.gamma': 1, + 'layer2.1.conv3.weight': 2, + 'layer2.1.bn2.beta': 2, + 'layer2.1.bn2.gamma': 2, + 'layer2.1.conv2.weight': 2, + 'layer2.1.bn1.beta': 2, + 'layer2.1.bn1.gamma': 2, + 'layer2.1.conv1.weight': 2, + 'layer2.0.bn_down_sample.beta': 1, + 'layer2.0.bn_down_sample.gamma': 1, + 'layer2.0.conv_down_sample.weight': 2, + 'layer2.0.bn3.beta': 1, + 'layer2.0.bn3.gamma': 1, + 'layer2.0.conv3.weight': 2, + 'layer2.0.bn2.beta': 2, + 'layer2.0.bn2.gamma': 2, + 'layer2.0.conv2.weight': 2, + 'layer2.0.bn1.beta': 2, + 'layer2.0.bn1.gamma': 2, + 'layer2.0.conv1.weight': 2, + 'layer1.2.bn3.beta': 2, + 'layer1.2.bn3.gamma': 2, + 'layer1.2.conv3.weight': 2, + 'layer1.2.bn2.beta': 2, + 'layer1.2.bn2.gamma': 2, + 'layer1.2.conv2.weight': 2, + 'layer1.2.bn1.beta': 2, + 'layer1.2.bn1.gamma': 2, + 'layer1.2.conv1.weight': 2, + 'layer1.1.bn3.beta': 1, + 'layer1.1.bn3.gamma': 1, + 'layer1.1.conv3.weight': 2, + 'layer1.1.bn2.beta': 2, + 'layer1.1.bn2.gamma': 2, + 'layer1.1.conv2.weight': 2, + 'layer1.1.bn1.beta': 2, + 'layer1.1.bn1.gamma': 2, + 'layer1.1.conv1.weight': 2, + 'layer1.0.bn_down_sample.beta': 1, + 'layer1.0.bn_down_sample.gamma': 1, + 'layer1.0.conv_down_sample.weight': 2, + 'layer1.0.bn3.beta': 1, + 'layer1.0.bn3.gamma': 1, + 'layer1.0.conv3.weight': 2, + 'layer1.0.bn2.beta': 2, + 'layer1.0.bn2.gamma': 2, + 'layer1.0.conv2.weight': 2, + 'layer1.0.bn1.beta': 2, + 'layer1.0.bn1.gamma': 2, + 'layer1.0.conv1.weight': 2, + 'bn1.beta': 1, + 'bn1.gamma': 1, + 'conv1.weight': 2} + + assert (allreduce_fusion_dict == expect_dict) + cost_model_context.reset_cost_model_context() + + +def test_train_64k_8p(epoch_size=3, batch_size=32, num_classes=65536): #1048576 #131072 #32768 #8192 + dev_num = 8 + context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, device_num=dev_num) + cost_model_context.set_cost_model_context(costmodel_gamma=0.001, costmodel_beta=260.0) + set_algo_parameters(elementwise_op_strategy_follow=True) + resset_op_id() + np.random.seed(6) + input_np = np.ones([batch_size, 3, 224, 224]).astype(np.float32) + label_np = np.zeros([batch_size]).astype(np.int32) + for i in range(0, batch_size): + label_np[i] = i % num_classes + dataset = DatasetLenet(Tensor(input_np), Tensor(label_np), 1) + net = resnet50(num_classes) + loss = SoftmaxCrossEntropyExpand(sparse=True) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) + model = Model(net, loss_fn=loss, optimizer=opt) + model.train(5, dataset, dataset_sink_mode=False) + strategies = _executor._get_strategy(model._train_network) + for (k, v) in strategies.items(): + if re.match(k, 'Conv2D-op') is not None: + assert v[0][0] == dev_num + elif re.match(k, 'MatMul-op') is not None: + assert v == [[1, 1], [dev_num, 1]] + elif re.match(k, 'ReduceSum-op') is not None: + assert v == [[1, dev_num]] diff --git a/tests/ut/python/parallel/test_auto_parallel_rhombus.py b/tests/ut/python/parallel/test_auto_parallel_rhombus.py new file mode 100644 index 00000000000..830504d7278 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_rhombus.py @@ -0,0 +1,126 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +def test_rhombus1(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.tadd1 = P.TensorAdd() + self.tadd2 = P.TensorAdd() + self.weight = Parameter(Tensor(np.ones([128, 128]).astype(np.float32) * 0.01), "w", requires_grad=True) + + def construct(self, x, y, z): + mm_out = self.matmul(x, self.weight) + ta1_out = self.tadd1(y, z) + out = self.tadd2(ta1_out, mm_out) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 128]), dtype=ms.float32) + y = Tensor(np.ones([128, 128]), dtype=ms.float32) + b = Tensor(np.ones([128, 128]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, b) + +def test_rhombus2(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.tadd1 = P.TensorAdd() + self.tadd2 = P.TensorAdd() + self.tadd3 = P.TensorAdd() + self.weight1 = Parameter(Tensor(np.ones([128, 128]).astype(np.float32) * 0.01), "w", requires_grad=True) + self.weight2 = Parameter(Tensor(np.ones([128, 128]).astype(np.float32) * 0.01), "w", requires_grad=True) + + def construct(self, x, y, z): + mm1_out = self.matmul1(x, self.weight1) + ta1_out = self.tadd1(y, z) + ta2_out = self.tadd2(mm1_out, ta1_out) + mm2_out = self.matmul2(ta1_out, self.weight2) + ta3_out = self.tadd3(ta2_out, mm2_out) + return ta3_out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 128]), dtype=ms.float32) + y = Tensor(np.ones([128, 128]), dtype=ms.float32) + b = Tensor(np.ones([128, 128]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, b) + +def test_rhombus3(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.tadd1 = P.TensorAdd() + self.tadd2 = P.TensorAdd() + self.tadd3 = P.TensorAdd() + self.tadd4 = P.TensorAdd() + self.weight1 = Parameter(Tensor(np.ones([128, 128]).astype(np.float32) * 0.01), "w", requires_grad=True) + self.t = Tensor(np.ones([128, 128]).astype(np.float32) * 0.01) + + def construct(self, x, y, z): + mm1_out = self.matmul1(x, self.weight1) + ta1_out = self.tadd1(y, z) + ta2_out = self.tadd2(mm1_out, ta1_out) + ta3_out = self.tadd3(ta1_out, self.t) + ta4_out = self.tadd4(ta2_out, ta3_out) + return ta4_out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 128]), dtype=ms.float32) + y = Tensor(np.ones([128, 128]), dtype=ms.float32) + z = Tensor(np.ones([128, 128]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_softmax_loss.py b/tests/ut/python/parallel/test_auto_parallel_softmax_loss.py new file mode 100644 index 00000000000..4167311c776 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_softmax_loss.py @@ -0,0 +1,64 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y) + return self.loss(predict, b)[0] + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def test_softmax_cross_entropy_loss_auto_parallel(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul(transpose_b=True) + self.gelu = P.Gelu() + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.gelu(out) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_transformer.py b/tests/ut/python/parallel/test_auto_parallel_transformer.py new file mode 100644 index 00000000000..a572d31ba1a --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_transformer.py @@ -0,0 +1,105 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor, Parameter +from mindspore.ops import operations as P +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x): + predict = self.network(x) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x): + return C.grad_all(self.network)(x) + +class CustomDense(nn.Cell): + def __init__(self, row, column): + super(CustomDense, self).__init__() + self.weight = Parameter(Tensor(np.ones([row, column]).astype(np.float32) * 0.01), "w", requires_grad=True) + self.bias = Parameter(Tensor(np.zeros([row, column]).astype(np.float32)), "b", requires_grad=True) + self.matmul1 = P.MatMul() + self.add2 = P.TensorAdd() + self.activation3 = nn.ReLU() + + def construct(self, x): + mat_output = self.matmul1(x, self.weight) + add_output = self.add2(mat_output, self.bias) + output = self.activation3(add_output) + + return output + + +class DenseMutMulNet(nn.Cell): + def __init__(self): + super(DenseMutMulNet, self).__init__() + self.fc1 = CustomDense(4096, 4096) + self.fc2 = CustomDense(4096, 4096) + self.fc3 = CustomDense(4096, 4096) + self.fc4 = CustomDense(4096, 4096) + self.relu4 = nn.ReLU() + self.relu5 = nn.ReLU() + self.transpose = P.Transpose() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + + def construct(self, x): + q = self.fc1(x) + k = self.fc2(x) + v = self.fc3(x) + k = self.transpose(k, (1, 0)) + c = self.relu4(self.matmul1(q, k)) + s = self.relu5(self.matmul2(c, v)) + s = self.fc4(s) + return s + +class MultiTransformer(nn.Cell): + def __init__(self, layer_nums=1): + super(MultiTransformer, self).__init__() + self.layer = self._make_layer(layer_nums) + + def _make_layer(self, layer_num): + layers = [] + for _ in range(0, layer_num): + layers.append(DenseMutMulNet()) + + return nn.SequentialCell(layers) + + def construct(self, x): + out = self.layer(x) + return out + +def test_dmnet_train_step(): + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + + input = Tensor(np.ones([4096, 4096]).astype(np.float32) * 0.01) + net = GradWrap(NetWithLoss(MultiTransformer())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, input) diff --git a/tests/ut/python/parallel/test_auto_parallel_transpose.py b/tests/ut/python/parallel/test_auto_parallel_transpose.py new file mode 100644 index 00000000000..62fdc111207 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_transpose.py @@ -0,0 +1,78 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel._utils import _reset_op_id as reset_op_id + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +# core dump, step_auto_parallel should SetInputs for transpose axis +def test_two_matmul_transpose(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.transpose1 = P.Transpose() + self.transpose2 = P.Transpose() + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + out = self.transpose1(out, (1, 0)) + out = self.transpose2(out, (1, 0)) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + _executor.compile(net, x, y, b, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/Transpose-op4': [[1, 16]], + 'Default/network-Net/Transpose-op5': [[16, 1]], + 'Default/network-Net/MatMul-op6': [[16, 1], [1, 1]], + 'Default/network-Net/MatMul-op7': [[16, 1], [1, 1]]} + assert strategies == expected_strategies \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_tuple_depend.py b/tests/ut/python/parallel/test_auto_parallel_tuple_depend.py new file mode 100644 index 00000000000..0f8949d3fcc --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_tuple_depend.py @@ -0,0 +1,75 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops.operations.comm_ops import _VirtualDataset +from mindspore import context + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +def bn_with_initialize(out_channels): + bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=1e-5) + return bn + +# model_parallel test +def test_virtual_dataset_3_input(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.virtual_dataset = _VirtualDataset() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.gelu = P.Gelu() + self.bn1 = bn_with_initialize(2048) + + def construct(self, x, y, b): + x, y, b = self.virtual_dataset(x, y, b) + out = self.gelu(self.matmul1(x, y)) + b = self.bn1(b) + out = self.matmul2(out, b) + return out + + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + context.set_auto_parallel_context(device_num=8, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 2048]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_auto_parallel_two_matmul.py b/tests/ut/python/parallel/test_auto_parallel_two_matmul.py new file mode 100644 index 00000000000..5155db41f64 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_two_matmul.py @@ -0,0 +1,140 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.parallel import _cost_model_context as cost_model_context +from mindspore.parallel import set_algo_parameters, get_algo_parameters, reset_algo_parameters +from mindspore.parallel._utils import _reset_op_id as reset_op_id + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + # model_parallel test +def test_two_matmul(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + size = 16 + context.set_auto_parallel_context(device_num=size, global_rank=0) + cost_model_context.set_cost_model_context(device_memory_capacity= 32.0 * 1024.0 * 1024.0 * 1024.0, + costmodel_alpha=1.0, + costmodel_beta=60.0, + costmodel_gamma=0.1, + costmodel_communi_threshold=1024.0, + costmodel_communi_const=2222.0, + costmodel_communi_bias=1111.0) + dev_mem_cap = cost_model_context.get_cost_model_context("device_memory_capacity") + assert dev_mem_cap == 32.0 * 1024.0 * 1024.0 * 1024.0 + costmodel_alpha = cost_model_context.get_cost_model_context("costmodel_alpha") + assert costmodel_alpha == 1.0 + costmodel_beta = cost_model_context.get_cost_model_context("costmodel_beta") + assert costmodel_beta == 60.0 + costmodel_gamma = cost_model_context.get_cost_model_context("costmodel_gamma") + assert costmodel_gamma == 0.1 + costmodel_communi_threshold = cost_model_context.get_cost_model_context("costmodel_communi_threshold") + assert costmodel_communi_threshold == 1024.0 + costmodel_communi_const = cost_model_context.get_cost_model_context("costmodel_communi_const") + assert costmodel_communi_const == 2222.0 + costmodel_communi_bias = cost_model_context.get_cost_model_context("costmodel_communi_bias") + assert costmodel_communi_bias == 1111.0 + + cost_model_context.reset_cost_model_context() + dev_mem_cap = cost_model_context.get_cost_model_context("device_memory_capacity") + assert dev_mem_cap == 16.0 * 1024.0 * 1024.0 * 1024.0 + costmodel_alpha = cost_model_context.get_cost_model_context("costmodel_alpha") + assert costmodel_alpha == 1.0 + costmodel_beta = cost_model_context.get_cost_model_context("costmodel_beta") + assert costmodel_beta == 65.0 + costmodel_gamma = cost_model_context.get_cost_model_context("costmodel_gamma") + assert costmodel_gamma == 0.02 + costmodel_communi_threshold = cost_model_context.get_cost_model_context("costmodel_communi_threshold") + assert costmodel_communi_threshold == 2048.0 + costmodel_communi_const = cost_model_context.get_cost_model_context("costmodel_communi_const") + assert costmodel_communi_const == 3072.0 + costmodel_communi_bias = cost_model_context.get_cost_model_context("costmodel_communi_bias") + assert costmodel_communi_bias == 1024.0 + + + set_algo_parameters(simplify_cal=True, + tensor_slice_align_enable=False, + tensor_slice_align_size=32, + not_fully_use_devices=True, + elementwise_op_strategy_follow=False) + para_simplify_cal = get_algo_parameters("simplify_cal") + assert para_simplify_cal == True + para_slice_align_enable = get_algo_parameters("tensor_slice_align_enable") + assert para_slice_align_enable == False + para_slice_align_size = get_algo_parameters("tensor_slice_align_size") + assert para_slice_align_size == 32 + not_fully_use_devices = get_algo_parameters("not_fully_use_devices") + assert not_fully_use_devices == True + elementwise_op_strategy_follow = get_algo_parameters("elementwise_op_strategy_follow") + assert elementwise_op_strategy_follow == False + + reset_algo_parameters() + para_simplify_cal = get_algo_parameters("simplify_cal") + assert para_simplify_cal == True + para_slice_align_enable = get_algo_parameters("tensor_slice_align_enable") + assert para_slice_align_enable == False + para_slice_align_size = get_algo_parameters("tensor_slice_align_size") + assert para_slice_align_size == 16 + not_fully_use_devices = get_algo_parameters("not_fully_use_devices") + assert not_fully_use_devices == False + elementwise_op_strategy_follow = get_algo_parameters("elementwise_op_strategy_follow") + assert elementwise_op_strategy_follow == False + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = NetWithLoss(Net()) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + reset_op_id() + + _executor.compile(net, x, y, b, phase='train') + strategies = _executor._get_strategy(net) + expected_strategies = {'Default/network-Net/MatMul-op2': [[16, 1], [1, 1]], + 'Default/network-Net/MatMul-op3': [[16, 1], [1, 1]]} + assert strategies == expected_strategies \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_two_partial_matmul.py b/tests/ut/python/parallel/test_auto_parallel_two_partial_matmul.py new file mode 100644 index 00000000000..77973924b2e --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_two_partial_matmul.py @@ -0,0 +1,65 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y): + predict = self.network(x, y) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y): + return C.grad_all(self.network)(x, y) + + # model_parallel test +def test_four_matmul_linear(): + class Net(nn.Cell): + def __init__(self, strategy1): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.weight = Parameter(Tensor(np.ones([512, 256]).astype(np.float32) * 0.01), "w", requires_grad=True) + self.matmul2 = P.MatMul() + + def construct(self, x, y): + out = self.matmul1(x, y) + out = self.matmul2(out, self.weight) + return out + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + strategy1 = ((8, 1), (1, 1)) + x = Tensor(np.ones([8, 16]), dtype=ms.float32) + y = Tensor(np.ones([16, 512]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net(strategy1))) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_auto_parallel_zig_zag.py b/tests/ut/python/parallel/test_auto_parallel_zig_zag.py new file mode 100644 index 00000000000..72af3ccbe5a --- /dev/null +++ b/tests/ut/python/parallel/test_auto_parallel_zig_zag.py @@ -0,0 +1,71 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, z, w, a): + predict = self.network(x, y, z, w, a) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, z, w, a): + return C.grad_all(self.network)(x, y, z, w, a) + + # model_parallel test +def test_zig_zag_graph(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.matmul3 = P.MatMul() + self.matmul4 = P.MatMul() + + def construct(self, x, y, z, w, a): + m1_result = self.matmul1(x, y) + m2_result = self.matmul2(z, w) + m3_result = self.matmul3(m2_result, m1_result) + out = self.matmul4(m2_result, a) + + return out + + size = 8 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([32, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 32]), dtype=ms.float32) + z = Tensor(np.ones([32, 32]), dtype=ms.float32) + w = Tensor(np.ones([32, 32]), dtype=ms.float32) + a = Tensor(np.ones([32, 23]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y, z, w, a) diff --git a/tests/ut/python/parallel/test_auto_star_elimination.py b/tests/ut/python/parallel/test_auto_star_elimination.py new file mode 100644 index 00000000000..db3b4346769 --- /dev/null +++ b/tests/ut/python/parallel/test_auto_star_elimination.py @@ -0,0 +1,88 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore import Tensor, Parameter +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer +import mindspore.nn as nn +import math +import numpy as np +import os +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.ops import composite as C +from mindspore import context +from mindspore.common.api import _executor +import mindspore as ms + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y): + predict = self.network(x, y) + return self.loss(predict) + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y): + return C.grad_all(self.network)(x, y) + + +class CustomMatMul(nn.Cell): + def __init__(self, transpose_a=False, transpose_b=False): + super(CustomMatMul, self).__init__() + self.fc = P.MatMul(transpose_a=transpose_a, transpose_b=transpose_b) + + def construct(self, x1, x2): + out = self.fc(x1, x2) + return out + + +class MarginCE(_Loss): + def __init__(self): + super(MarginCE, self).__init__() + self.fc = CustomMatMul(transpose_b=True) + self.fc1 = CustomMatMul(transpose_b=True) + self.fc2 = CustomMatMul(transpose_b=True) + self.fc3 = CustomMatMul(transpose_b=True) + self.fc4 = CustomMatMul(transpose_b=True) + self.param = Parameter(Tensor(np.ones([512, 512]), dtype=mstype.float32), name="param", requires_grad=False) + self.param2 = Parameter(Tensor(np.ones([512, 512]), dtype=mstype.float32), name="param", requires_grad=False) + + def construct(self, feature, label): + fc_out = self.fc(feature, label) + + fc1_out = self.fc1(self.param2, self.param) + fc2_out = self.fc2(fc1_out, fc_out) + fc3_out = self.fc3(fc1_out, fc_out) + fc4_out = self.fc4(fc2_out, fc3_out) + return fc4_out + + +def test_marin_loss(): + context.set_auto_parallel_context(device_num=4, global_rank=0) + + x = Tensor(np.ones([512, 512]), dtype=ms.float32) + y = Tensor(np.ones([512, 512]), dtype=ms.float32) + + net = GradWrap(NetWithLoss(MarginCE())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net, x, y) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_batch_parallel.py b/tests/ut/python/parallel/test_batch_parallel.py new file mode 100644 index 00000000000..44670bf7e4b --- /dev/null +++ b/tests/ut/python/parallel/test_batch_parallel.py @@ -0,0 +1,111 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, w1, w2): + predict = self.network(x, w1, w2) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, w1, w2): + return C.grad_all(self.network)(x, w1, w2) + + +class NetConv(nn.Cell): + def __init__(self, + cin, + cout, + kernel_size, + stride=1, + pad_mode='pad', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros', + strategy=None): + super(NetConv, self).__init__() + self.conv = nn.Conv2d(cin, + cout, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + has_bias, + weight_init, + bias_init) + self.conv.conv2d.set_strategy(strategy) + + def construct(self, input_x): + return self.conv(input_x) + + +def test_batch(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.conv1 = NetConv(16, 8, (3, 3), bias_init='zeros', strategy=strategy1) + self.mul1 = P.Mul().set_strategy(strategy2) + self.conv2 = NetConv(8, 64, (9, 9), bias_init='zeros', strategy=strategy1) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, w1, w2): + out1 = self.conv1(x) + out2 = self.mul1(out1, w1) + out3 = self.conv2(out2) + out4 = self.mul2(out3, w2) + + return out4 + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((8, 1, 1, 1), (1, 1, 1, 1)) + strategy2 = ((1, 1, 1, 8), (1, 1, 1, 8)) + strategy3 = ((4, 1, 1, 2), (4, 1, 1, 2)) + + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 16, 34, 34]), dtype=ms.float32) + w1 = Tensor(np.ones([128, 8, 32, 32]), dtype=ms.float32) + w2 = Tensor(np.ones([128, 64, 24, 24]), dtype=ms.float32) + _executor.compile(net, x, w1, w2) + + +if __name__ == '__main__': + test_batch() diff --git a/tests/ut/python/parallel/test_batch_parallel_dropout.py b/tests/ut/python/parallel/test_batch_parallel_dropout.py new file mode 100644 index 00000000000..08db155ccae --- /dev/null +++ b/tests/ut/python/parallel/test_batch_parallel_dropout.py @@ -0,0 +1,68 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +# model_parallel test +def test_batch_parallel_dropout(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul1 = P.MatMul() + self.dropout = nn.Dropout() + self.matmul2 = P.MatMul() + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.dropout(out) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_batch_parallel_tensoradd.py b/tests/ut/python/parallel/test_batch_parallel_tensoradd.py new file mode 100644 index 00000000000..d16496a9337 --- /dev/null +++ b/tests/ut/python/parallel/test_batch_parallel_tensoradd.py @@ -0,0 +1,65 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +def test_matmul_add(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul() + self.add = P.TensorAdd() + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.add(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_batchnorm_batch_parallel.py b/tests/ut/python/parallel/test_batchnorm_batch_parallel.py new file mode 100644 index 00000000000..95a77107929 --- /dev/null +++ b/tests/ut/python/parallel/test_batchnorm_batch_parallel.py @@ -0,0 +1,147 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +import mindspore as ms +import numpy as np +from mindspore import context +from mindspore.common.tensor import Tensor +from mindspore.nn.layer.activation import ReLU +from mindspore.nn.cell import Cell +from mindspore.nn.layer.conv import Conv2d +from mindspore.nn.layer.normalization import BatchNorm2d +from mindspore.nn.layer.pooling import MaxPool2d +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData + + +dev_num = 8 +strategy_no_weight = ((dev_num, 1, 1, 1), ) +strategy_weight = ((dev_num, 1, 1, 1), (1, 1, 1, 1)) +strategy_bn = ((dev_num, 1, 1, 1), (1, ), (1, )) +strategy_fc_weight_bias = ((dev_num, 1), (1, 1), (1, )) + + +class DatasetLenet(MindData): + def __init__(self, predict, label, length=3): + super(DatasetLenet, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +def conv7x7(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 7, 7) + weight = Tensor(np.ones(weight_shape).astype(np.float32)) + conv = Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=0, weight_init=weight, has_bias=False, + pad_mode="same") + conv.conv2d.set_strategy(strategy_weight) + return conv + + +def weight_variable_0(shape): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros) + + +def weight_variable_1(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones) + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_1(shape) + bn = BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + bn.bn_train.set_strategy(strategy_bn) + return bn + + +class ResNet(Cell): + + def __init__(self, num_classes=100): + super(ResNet, self).__init__() + strategy_no_weight = ((dev_num, 1, 1, 1), ) + self.conv1 = conv7x7(3, 64, stride=2, padding=3) + self.bn1 = bn_with_initialize(64) + self.relu = ReLU() + self.relu.relu.set_strategy(strategy_no_weight) + self.maxpool = MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + self.reshape = P.Reshape() + self.matmul = P.MatMul().set_strategy(((8, 1), (1, 1))) + self.matmul_weight = Parameter(Tensor(np.ones([200704, num_classes]), dtype=ms.float32), name="weight") + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.reshape(x, (32 * dev_num, 200704)) + x = self.matmul(x, self.matmul_weight) + return x + + +def batchnorm_net(num_classes): + return ResNet(num_classes) + + +def test_batchnorm_batch_parallel(): + num_classes = 1001 + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + rank_size = 0 + + predict = Tensor(np.ones([batch_size, 3, 224, 224]), dtype=ms.float32) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + + dataset = DatasetLenet(predict, label, 2) + net = batchnorm_net(num_classes) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((dev_num, 1), (dev_num, 1))) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=dev_num) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss, opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +if __name__ == '__main__': + test_batchnorm_batch_parallel() diff --git a/tests/ut/python/parallel/test_bias_add.py b/tests/ut/python/parallel/test_bias_add.py new file mode 100644 index 00000000000..d5d24f71bfc --- /dev/null +++ b/tests/ut/python/parallel/test_bias_add.py @@ -0,0 +1,89 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from mindspore.train.model import Model + + +class CrossEntropyLoss(nn.Cell): + def __init__(self, reduction='mean'): + super(CrossEntropyLoss, self).__init__() + + self.reduce_mean = P.ReduceMean() + self.cross_entropy = nn.SoftmaxCrossEntropyWithLogits() + self.reduction = reduction + + def construct(self, logits, label): + loss = self.cross_entropy(logits, label) + if self.reduction == 'mean': + loss = self.reduce_mean(loss, (-1,)) + return loss + + +class DatasetLenet(): + def __init__(self, predict, label, length=3): + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + def get_dataset_size(self): + return 32 + + def get_repeat_count(self): + return 1 + + +class Net(nn.Cell): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=1, stride=1, pad_mode='valid', + has_bias=True, weight_init='ones', bias_init='ones') + self.reduce_mean = P.ReduceMean(keep_dims=False).set_strategy(((1, 1, 1, 8),)) + self.flat = nn.Flatten() + + def construct(self, inputs): + x = self.conv(inputs) + x = self.reduce_mean(x, -1) + x = self.flat(x) + return x + + +def test_bias_add(): + context.set_context(mode=context.GRAPH_MODE) + context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=8) + input_np = np.ones([16, 3, 32, 32]).astype(np.float32) + label_np = np.zeros([16, 2048]).astype(np.float32) + dataset = DatasetLenet(Tensor(input_np), Tensor(label_np), 1) + net = Net() + loss = CrossEntropyLoss() + opt = nn.Momentum(learning_rate=0.01, momentum=0.9, params=net.get_parameters()) + model = Model(network=net, loss_fn=loss, optimizer=opt) + model.train(epoch=1, train_dataset=dataset, dataset_sink_mode=False) diff --git a/tests/ut/python/parallel/test_bn_prelu_cell.py b/tests/ut/python/parallel/test_bn_prelu_cell.py new file mode 100644 index 00000000000..5971db70270 --- /dev/null +++ b/tests/ut/python/parallel/test_bn_prelu_cell.py @@ -0,0 +1,260 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train.model import Model +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from mindspore.ops import functional as F +from mindspore.nn import WithLossCell +import mindspore.common.dtype as DT +from tests.dataset_mock import MindData +from mindspore.train.parallel_utils import ParallelMode +from mindspore import context + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class FusedBatchNorm(nn.Cell): + """Batch Normalization base class.""" + def __init__(self, + num_features, + eps=1e-5, + momentum=0.1, + affine=True, + gamma_init='ones', + beta_init='zeros', + moving_mean_init='zeros', + moving_var_init='ones'): + super(FusedBatchNorm, self).__init__() + if num_features < 1: + raise ValueError("num_features must be at least 1") + + if momentum < 0 or momentum > 1: + raise ValueError("momentum should be a number in range [0, 1], but got {}".format(momentum)) + + self.num_features = num_features + self.eps = eps + self.momentum = Tensor(1.0 - momentum, DT.float32) + self.gamma = Parameter(initializer( + gamma_init, num_features), name="gamma", requires_grad=affine) + self.beta = Parameter(initializer( + beta_init, num_features), name="beta", requires_grad=affine) + self.moving_mean = Parameter(initializer( + moving_mean_init, num_features), name="mean", requires_grad=False) + self.moving_variance = Parameter(initializer( + moving_var_init, num_features), name="variance", requires_grad=False) + + self.bn_train = P.BatchNorm(is_training=True, + epsilon=self.eps) + self.bn_infer = P.BatchNorm(is_training=False, + epsilon=self.eps) + self.sub_mean = P.Sub().set_strategy(((1), (1))) + self.sub_var = P.Sub().set_strategy(((1), (1))) + self.mul_mean = P.Mul().set_strategy(((1, ), ())) + self.mul_var = P.Mul().set_strategy(((1, ), ())) + self.assign_sub_mean = P.AssignSub().set_strategy(((1, ), (1,))) + self.assign_sub_var = P.AssignSub().set_strategy(((1), (1))) + self.sub_mean2 = P.Sub().set_strategy(((1), (1))) + self.sub_var2 = P.Sub().set_strategy(((1), (1))) + + def set_strategy(self, strategy): + self.bn_train.set_strategy(strategy) + self.bn_infer.set_strategy(strategy) + + def _check_data_dim(self, x): + raise NotImplementedError + + def construct(self, x): + if self.training: + y, batch_mean, batch_var, _, _ = \ + self.bn_train(x, + self.gamma, + self.beta, + None, + None) + + mean_sub = self.sub_mean(self.moving_mean, batch_mean) + temp_mean = self.mul_mean(mean_sub, self.momentum) + mean_sub2 = self.sub_var(self.moving_variance, batch_var) + temp_variance = self.mul_var(mean_sub2, self.momentum) + y = F.depend(y, self.assign_sub_mean(self.moving_mean, temp_mean)) + y = F.depend(y, self.assign_sub_var(self.moving_variance, temp_variance)) + + else: + y = self.bn_infer(x, + self.gamma, + self.beta, + self.moving_mean, + self.moving_variance)[0] + return y + + def extend_repr(self): + return 'num_features={}, eps={}, momentum={}, ' \ + 'beta={}, gamma={}, ' \ + 'moving_mean={}, moving_variance={} ' \ + .format(self.num_features, + self.eps, + self.momentum, + self.beta, + self.gamma, + self.moving_mean, + self.moving_variance) + + + +class PReLU(nn.Cell): + """ + PReLU activation function. + + Computes prelu value of a 4-dim tensor(NCHW). + PReLU: out = max(0, A) + min(0, wA) + + Args: + channel: Integer. The dimensionality of w. Default: 1. + w: Float. The initial value of w. Default: 0.25. + + Returns: + Tensor, has the same type as features. + + Examples: + prelu = nn.PReLU(1, [np.float32(0.25)]) # or prelu = nn.PReLU(33, Tensor(np.random.rand(33), ms.float32)]) + input_data = Tensor(np.random.rand(1, 33, 4, 4), ms.float32) + output = prelu.construct(input_data) + """ + def __init__(self, channel=1, w=0.25): + super(PReLU, self).__init__() + if isinstance(w, (np.float32, float)): + tmp = np.empty((channel,), dtype=np.float32) + tmp.fill(w) + w = tmp + elif isinstance(w, (int, bool, complex, str)): + raise TypeError("w only support input type float32 and float") + + if not isinstance(w, Tensor): + w = Tensor(w) + self.w = Parameter(initializer(w, [channel,]), name='a') + self.prelu = P.PReLU() + self.relu = P.ReLU().set_strategy(((1))) + + def construct(self, x): + self.w = self.relu(self.w) + return self.prelu(x, self.w) + + +class BNNet(nn.Cell): + def __init__(self, strategy0, strategy1, strategy2): + super(BNNet, self).__init__() + self.bn = FusedBatchNorm(512) + self.prelu = PReLU(512) + + + def construct(self, x): + x = self.bn(x) + x = self.prelu(x) + return x + + +def bn_net(strategy0, strategy1, strategy2): + return BNNet(strategy0=strategy0, strategy1=strategy1, strategy2=strategy2) + + +def bn_common(parallel_mode, train_flag, strategy0=None, strategy1=None, strategy2=None, strategy_loss=None): + context.set_context(mode=context.GRAPH_MODE) + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + rank_size = 8 + + predict = Tensor(np.ones([32, 512]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = bn_net(strategy0, strategy1, strategy2) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(strategy_loss) + opt = Momentum(net.trainable_params(), learning_rate, momentum, 0.0001, 1024 * rank_size) + + if not train_flag: + net = WithLossCell(net, loss) + net.set_train() + + if parallel_mode == ParallelMode.DATA_PARALLEL: + context.set_auto_parallel_context(parameter_broadcast=True) + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + model = Model(net, loss, opt) + if train_flag: + model.train(epoch_size, dataset, dataset_sink_mode=False) + else: + model._predict(predict, label) + + +def test_data_parallel(): + parallel_mode = ParallelMode.DATA_PARALLEL + train_flag = True + bn_common(parallel_mode, train_flag) + + +def auto_parallel(): + train_flag = True + parallel_mode = ParallelMode.AUTO_PARALLEL + bn_common(parallel_mode, train_flag) + + +def Xtest_data_parallel_predict(): + parallel_mode = ParallelMode.DATA_PARALLEL + train_flag = False + bn_common(parallel_mode, train_flag) + + +def Xtest_semi_auto_parallel_predict(): + train_flag = False + parallel_mode = ParallelMode.SEMI_AUTO_PARALLEL + bn_common(parallel_mode, train_flag) + + +def Xtest_auto_parallel_predict(): + train_flag = False + parallel_mode = ParallelMode.AUTO_PARALLEL + bn_common(parallel_mode, train_flag) + + +if __name__ == '__main__': + auto_parallel() diff --git a/tests/ut/python/parallel/test_bool_grad.py b/tests/ut/python/parallel/test_bool_grad.py new file mode 100644 index 00000000000..f3cdfc80304 --- /dev/null +++ b/tests/ut/python/parallel/test_bool_grad.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.train import Model, ParallelMode +from mindspore.nn.optim import Momentum + + +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class CommonNet(nn.Cell): + def __init__(self): + super(CommonNet, self).__init__() + self.weight = Parameter(Tensor(np.ones([256, 64]), dtype=ms.float32), name="mul_weight") + self.logicalnot = P.LogicalNot().set_strategy(((4,1),)) + self.equal = P.Equal().set_strategy(((4,2),(4,2))) + + def construct(self, x, label): + x = self.equal(x, self.weight) + x = self.logicalnot(x) + return x + + +def common_net(): + epoch_size = 1 + + context.reset_auto_parallel_context() + + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8) + predict = Tensor(np.ones([32, 64]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = CommonNet() + + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, optimizer=optimizer) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_bool_grad(): + common_net() \ No newline at end of file diff --git a/tests/ut/python/parallel/test_broadcast_dict.py b/tests/ut/python/parallel/test_broadcast_dict.py new file mode 100644 index 00000000000..709dbb72987 --- /dev/null +++ b/tests/ut/python/parallel/test_broadcast_dict.py @@ -0,0 +1,70 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore import Tensor, Parameter +from mindspore.ops import operations as P +from mindspore.communication.management import init + + +class DataParallelNet(nn.Cell): + def __init__(self): + super(DataParallelNet, self).__init__() + weight_init = np.random.rand(512, 64).astype(np.float32) + self.weight = Parameter(Tensor(weight_init), name="weight", layerwise_parallel=False) + self.fc = P.MatMul() + + def construct(self, x): + x = self.fc(x, self.weight) + return x + + +class ModelParallelNet(nn.Cell): + def __init__(self): + super(ModelParallelNet, self).__init__() + weight_init = np.random.rand(512, 64).astype(np.float32) + self.weight = Parameter(Tensor(weight_init), name="weight", layerwise_parallel=True) + self.fc = P.MatMul() + + def construct(self, x): + x = self.fc(x, self.weight) + return x + + +def test_param_broadcast(): + context.set_context(mode=context.GRAPH_MODE) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode="data_parallel", parameter_broadcast=True) + init() + network = DataParallelNet() + network.set_train() + + predict = Tensor(np.ones([64, 512]).astype(np.float32) * 0.01) + out = network(predict) + context.reset_auto_parallel_context() + + +def test_param_not_broadcast(): + context.set_context(mode=context.GRAPH_MODE) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode="data_parallel", parameter_broadcast=False) + init() + network = ModelParallelNet() + network.set_train() + + predict = Tensor(np.ones([64, 512]).astype(np.float32) * 0.01) + out = network(predict) + context.reset_auto_parallel_context() diff --git a/tests/ut/python/parallel/test_combined_tensor.py b/tests/ut/python/parallel/test_combined_tensor.py new file mode 100644 index 00000000000..ac70076c67d --- /dev/null +++ b/tests/ut/python/parallel/test_combined_tensor.py @@ -0,0 +1,55 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.parallel._tensor import _reshape_param_data +from mindspore import Tensor + + +def test_reshape_param_data(): + expected_tensor = Tensor([[1, 2, 3, 4], [5, 6, 7, 8]]) + dev_mat = [2, 2] + tensor_map = [0, 1] + input_tensor = Tensor([[1, 2],[5, 6],[3, 4],[7, 8]]) + tensor = _reshape_param_data(input_tensor, dev_mat, tensor_map) + if expected_tensor.__str__() != tensor.__str__(): + raise AssertionError + + tensor_map = [1, -1] + input_tensor = Tensor([[1, 2, 3, 4],[1, 2, 3, 4],[5, 6, 7, 8],[5, 6, 7, 8]]) + tensor = _reshape_param_data(input_tensor, dev_mat, tensor_map) + if expected_tensor.__str__() != tensor.__str__(): + raise AssertionError + + expected_tensor = Tensor([[[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]) + + input_tensor = Tensor([[[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]],\ + [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]) + + dev_mat = [4] + tensor_map = [-1, -1, -1, -1] + tensor = _reshape_param_data(input_tensor, dev_mat, tensor_map) + if expected_tensor.__str__() != tensor.__str__(): + raise AssertionError + + + +if __name__ == '__main__': + test_reshape_param_data() diff --git a/tests/ut/python/parallel/test_comparison_function_info.py b/tests/ut/python/parallel/test_comparison_function_info.py new file mode 100644 index 00000000000..74de04f1df7 --- /dev/null +++ b/tests/ut/python/parallel/test_comparison_function_info.py @@ -0,0 +1,186 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore as ms +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +def test_matmul_equal(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.equal = P.Equal().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.equal(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_not_equal(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.notequal = P.NotEqual().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.notequal(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_not_equal_repeated_calculation(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.notequal = P.NotEqual().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.notequal(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 1), (4, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_maximum(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.maximum = P.Maximum().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.maximum(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_maximum_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.maximum = P.Maximum().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.maximum(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), (2, )) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_maximum_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.maximum = P.Maximum().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.maximum(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), (1, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) \ No newline at end of file diff --git a/tests/ut/python/parallel/test_dataset.py b/tests/ut/python/parallel/test_dataset.py new file mode 100644 index 00000000000..f4bfb418699 --- /dev/null +++ b/tests/ut/python/parallel/test_dataset.py @@ -0,0 +1,44 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from mindspore.common.api import _executor +from mindspore.ops.operations.comm_ops import _VirtualDataset + + +class VirtualDatasetNet(nn.Cell): + def __init__(self): + super(VirtualDatasetNet, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + self.gelu = P.Gelu() + + def construct(self, x, y, z): + x, y, z = self.virtual_dataset(x, y, z) + out = self.gelu(self.matmul1(x, y)) + out = self.matmul2(out, z) + return out + + +def test_virtual_dataset(): + x = Tensor(np.ones([128, 32], dtype=np.float32)) + y = Tensor(np.ones([32, 64], dtype=np.float32)) + z = Tensor(np.ones([64, 64], dtype=np.float32)) + network = VirtualDatasetNet() + _executor.compile(network, x, y, z) + diff --git a/tests/ut/python/parallel/test_dataset_interface.py b/tests/ut/python/parallel/test_dataset_interface.py new file mode 100644 index 00000000000..da8821199e2 --- /dev/null +++ b/tests/ut/python/parallel/test_dataset_interface.py @@ -0,0 +1,171 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +import mindspore.nn as nn +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.train.loss_scale_manager import DynamicLossScaleManager +from mindspore.ops import composite as C, functional as F, operations as P +from mindspore.common.parameter import Parameter, ParameterTuple + + +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class AllToAllNet(nn.Cell): + def __init__(self, strategy1): + super(AllToAllNet, self).__init__() + self.matmul = P.MatMul().set_strategy(((1, 1), (1, 8))) + self.matmul_weight = Parameter(Tensor(np.ones([128, 256]), dtype=ms.float32), name="weight") + self.transpose1 = P.Transpose().set_strategy(strategy1) + + def construct(self, x): + x = self.matmul(x, self.matmul_weight) + x = self.transpose1(x, (1, 0)) + return x + + +def all_to_all_net(strategy1): + return AllToAllNet(strategy1=strategy1) + + +def loss_scale_manager_common(strategy1): + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, device_num=8) + predict = Tensor(np.ones([32, 128]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = all_to_all_net(strategy1) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((8, 1), (8, 1))) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + scale_manager = DynamicLossScaleManager(32, 2, 2000) + model = Model(net, loss, opt, loss_scale_manager=scale_manager) + # if no GE exists, outputs = self._train_network(*next_element) outputs is None, TypeError is caught. + try: + model.train(epoch_size, dataset, dataset_sink_mode=False) + except TypeError: + pass + else: + assert False + + +def test_dataset_interface_sens_scalar(): + strategy1 = ((8, 1), ) + loss_scale_manager_common(strategy1) + + +class TrainOneStepCell(nn.Cell): + + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + + def construct(self, data, sens): + weights = self.weights + loss = self.network(data) + grads = self.grad(self.network, weights)(data, sens) + return F.depend(loss, self.optimizer(grads)) + + +def loss_scale_manager_sens(strategy1, sens): + learning_rate = 0.1 + momentum = 0.9 + device_num = 8 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=device_num) + predict = Tensor(np.ones([32 * device_num, 128]), dtype=ms.float32) + net = all_to_all_net(strategy1) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + train_net = TrainOneStepCell(net, opt) + train_net.set_train() + train_net(predict, sens) + + +def test_dataset_interface_sens_shape_not_equal_loss(): + strategy1 = ((8, 1), ) + sens = Tensor(np.ones([256, 1024]), dtype=ms.float32) + try: + loss_scale_manager_sens(strategy1, sens) + except: + pass + + +def test_dataset_interface_sens_shape_equal_loss(): + strategy1 = ((4, 2), ) + sens = Tensor(np.ones([256, 256]), dtype=ms.float32) + loss_scale_manager_sens(strategy1, sens) + + +def test_input_not_in_parameter_layotu_dict(): + class Net(nn.Cell): + def __init__(self, strategy1): + super(Net, self).__init__() + self.matmul = P.MatMul().set_strategy(((1, 1), (1, 8))) + self.matmul_weight = Parameter(Tensor(np.ones([128, 256]), dtype=ms.float32), name="weight") + self.transpose1 = P.Transpose().set_strategy(strategy1) + + def construct(self, x, b): + x = self.matmul(x, self.matmul_weight) + x = self.transpose1(x, (1, 0)) + return x + + strategy1 = ((8, 1), ) + device_num = 8 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=device_num) + predict = Tensor(np.ones([32 * device_num, 128]), dtype=ms.float32) + b = Tensor(np.ones([32 * device_num, 128]), dtype=ms.float32) + net = Net(strategy1) + net.set_train() + net(predict, b) + + + diff --git a/tests/ut/python/parallel/test_dataset_util.py b/tests/ut/python/parallel/test_dataset_util.py new file mode 100644 index 00000000000..c79932a8989 --- /dev/null +++ b/tests/ut/python/parallel/test_dataset_util.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore.train._utils import _to_full_shapes, _to_full_tensor +from mindspore import Tensor +import mindspore as ms + +def test_to_full_shapes(): + device_num = 16 + shapes = [[32, 128], [12], [24, 1, 12]] + full_shapes = _to_full_shapes(shapes, device_num) + assert full_shapes == [(512, 128), (192,), (384, 1, 12)] + + +def test_to_full_tensor_1(): + elem = Tensor([[1,2,3], [4,5,6]], dtype=ms.float32) + device_num = 4 + global_rank = 2 + full_tensor = _to_full_tensor(elem, device_num, global_rank, scaling_sens=None) + + expect = ([[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [1,2,3], [4,5,6], [0, 0, 0], [0, 0, 0]]) + expect_tensor = Tensor(expect, dtype=ms.float32) + + assert (full_tensor[0] == expect_tensor) + + +def test_to_full_tensor_2(): + elem0 = Tensor([[1,2,3], [4,5,6]], dtype=ms.float32) + elem1 = Tensor([[1], [4]], dtype=ms.int32) + elem = (elem0, elem1,) + device_num = 4 + global_rank = 2 + full_tensor = _to_full_tensor(elem, device_num, global_rank, scaling_sens=None) + + expect0 = ([[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [1,2,3], [4,5,6], [0, 0, 0], [0, 0, 0]]) + expect_tensor0 = Tensor(expect0, dtype=ms.float32) + expect1 = ([[0], [0], [0], [0], [1], [4], [0], [0]]) + expect_tensor1 = Tensor(expect1, dtype=ms.int32) + expect_tensors = (expect_tensor0, expect_tensor1) + + assert (full_tensor == expect_tensors) + + +def test_to_full_tensor_sens_2(): + elem0 = Tensor([[1,2,3], [4,5,6]], dtype=ms.float32) + elem1 = Tensor([[1], [4]], dtype=ms.int32) + elem = (elem0, elem1,) + device_num = 4 + global_rank = 2 + full_tensor = _to_full_tensor(elem, device_num, global_rank, scaling_sens=0.1) + + expect0 = ([[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [1,2,3], [4,5,6], [0, 0, 0], [0, 0, 0]]) + expect_tensor0 = Tensor(expect0, dtype=ms.float32) + expect1 = ([[0], [0], [0], [0], [1], [4], [0], [0]]) + expect_tensor1 = Tensor(expect1, dtype=ms.int32) + expect_tensor_sens = Tensor(0.1, dtype=ms.float32) + expect_tensors = (expect_tensor0, expect_tensor1, expect_tensor_sens) + + assert (full_tensor == expect_tensors) + + + + + diff --git a/tests/ut/python/parallel/test_dense_matmul.py b/tests/ut/python/parallel/test_dense_matmul.py new file mode 100644 index 00000000000..2ee9ab0eece --- /dev/null +++ b/tests/ut/python/parallel/test_dense_matmul.py @@ -0,0 +1,53 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +from mindspore.common.api import _executor +from mindspore import Tensor, context +from mindspore.ops import operations as P +from ....train_step_wrap import train_step_with_loss_warp + + +class DenseMutMulNet(nn.Cell): + def __init__(self): + super(DenseMutMulNet, self).__init__() + self.fc1 = nn.Dense(128, 768, activation='relu') + self.fc2 = nn.Dense(128, 768, activation='relu') + self.fc3 = nn.Dense(128, 768, activation='relu') + self.fc4 = nn.Dense(768, 768, activation='relu') + self.relu4 = nn.ReLU() + self.relu5 = nn.ReLU() + self.transpose = P.Transpose() + self.matmul1 = P.MatMul() + self.matmul2 = P.MatMul() + + def construct(self, x): + q = self.fc1(x) + k = self.fc2(x) + v = self.fc3(x) + k = self.transpose(k, (1, 0)) + c = self.relu4(self.matmul1(q, k)) + s = self.relu5(self.matmul2(c, v)) + s = self.fc4(s) + return s + + +def test_dmnet_train_step(): + context.reset_auto_parallel_context() + input = Tensor(np.ones([32, 128]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([32, 768]).astype(np.float32)) + net = DenseMutMulNet() + net = train_step_with_loss_warp(DenseMutMulNet()) + _executor.compile(net, input, label) diff --git a/tests/ut/python/parallel/test_different_type_for_div_op.py b/tests/ut/python/parallel/test_different_type_for_div_op.py new file mode 100644 index 00000000000..9e9bbbe05ec --- /dev/null +++ b/tests/ut/python/parallel/test_different_type_for_div_op.py @@ -0,0 +1,103 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, bias): + return C.grad_all(self.network)(x, y, bias) + + +def test_sum_as_loss_float16(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0,1)) + return out + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((4, 1), ) + net = GradWrap(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float16) + y = Tensor(np.ones([64, 32]), dtype=ms.float16) + bias = Tensor(np.ones([64]), dtype=ms.float16) + _executor.compile(net, x, y, bias) + + +def test_sum_as_loss_float32(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0,1)) + return out + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((4, 1), ) + net = GradWrap(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + bias = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, bias) + + +def test_sum_as_loss_int32(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0,1)) + return out + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((4, 1), ) + net = GradWrap(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.int32) + y = Tensor(np.ones([64, 32]), dtype=ms.int32) + bias = Tensor(np.ones([64]), dtype=ms.int32) + _executor.compile(net, x, y, bias) diff --git a/tests/ut/python/parallel/test_element_wise_function.py b/tests/ut/python/parallel/test_element_wise_function.py new file mode 100644 index 00000000000..dfcebdc5abc --- /dev/null +++ b/tests/ut/python/parallel/test_element_wise_function.py @@ -0,0 +1,274 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore as ms +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def test_matmul_pow(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.pow = P.Pow().set_strategy(strategy2) + self.matmul2 = P.MatMul().set_strategy(strategy1) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.pow(out, 2.0) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_exp(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.exp = P.Exp().set_strategy(strategy2) + self.matmul2 = P.MatMul().set_strategy(strategy1) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.exp(out) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_log(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.log = P.Log().set_strategy(strategy2) + self.matmul2 = P.MatMul().set_strategy(strategy1) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.log(out) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_logical_not(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.logicalnot = P.LogicalNot().set_strategy(strategy2) + self.equal = P.Equal().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.matmul(x, y) + out = self.equal(out, b) + out = self.logicalnot(out) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), ) + strategy3 = ((4, 2), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_matmul_cast(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.cast = P.Cast().set_strategy(strategy2) + self.matmul2 = P.MatMul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.matmul(x, y) + b = self.cast(b, ms.float32) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), ) + strategy3 = ((1, 4), (4, 2)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.int32) + _executor.compile(net, x, y, b) + + +def test_cast_before_mirror(): + class Net(nn.Cell): + def __init__(self, strategy1): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.cast = P.Cast() + + def construct(self, x, y, b): + out = self.matmul(x, y) + b = self.cast(b, ms.float32) + out = self.matmul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0, cast_before_mirror=True) + strategy1 = ((2, 2), (2, 2)) + net = GradWrap(NetWithLoss(Net(strategy1))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float16) + _executor.compile(net, x, y, b) + + +def test_cast_before_mirror1(): + class Net(nn.Cell): + def __init__(self, strategy1): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.cast = P.Cast() + + def construct(self, x, y, b): + out = self.matmul(x, y) + b = self.cast(b, ms.float16) + out = self.matmul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0, cast_before_mirror=True) + strategy1 = ((2, 2), (2, 2)) + net = GradWrap(NetWithLoss(Net(strategy1))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([128, 32]), dtype=ms.float16) + y = Tensor(np.ones([32, 64]), dtype=ms.float16) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_cast_before_mirror2(): + class Net(nn.Cell): + def __init__(self, strategy1): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.cast = P.Cast() + + def construct(self, x, y, b): + out = self.matmul(x, y) + b = self.cast(b, ms.float16) + out = self.matmul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0, cast_before_mirror=False) + strategy1 = ((2, 2), (2, 2)) + net = GradWrap(NetWithLoss(Net(strategy1))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([128, 32]), dtype=ms.float16) + y = Tensor(np.ones([32, 64]), dtype=ms.float16) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_cast_before_mirror3(): + class Net(nn.Cell): + def __init__(self, strategy1): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.cast = P.Cast() + + def construct(self, x, y, b): + out = self.matmul(x, y) + b = self.cast(b, ms.float16) + out = self.matmul(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + net = GradWrap(NetWithLoss(Net(strategy1))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([128, 32]), dtype=ms.float16) + y = Tensor(np.ones([32, 64]), dtype=ms.float16) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_gather_v2_primitive.py b/tests/ut/python/parallel/test_gather_v2_primitive.py new file mode 100644 index 00000000000..c623595b53c --- /dev/null +++ b/tests/ut/python/parallel/test_gather_v2_primitive.py @@ -0,0 +1,147 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import composite as C +from mindspore.common.parameter import ParameterTuple +from mindspore.nn.optim import Momentum +from mindspore.communication.management import init +from mindspore.train import Model, ParallelMode +import mindspore as ms +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.nn.loss.loss import _Loss +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.nn import Dense, Cell +from mindspore import context + +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(): + def __init__(self, predict, length=3): + self.predict = predict + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return (self.predict,) + + def reset(self): + self.index = 0 + + def get_dataset_size(self): + return 128 + + def get_repeat_count(self): + return 1 + + +class GatherV2(_Loss): + def __init__(self, batchsize): + super(GatherV2, self).__init__() + self.pow = P.Pow() + emb_list = list(range(batchsize)) + emb1_list = emb_list[0::2] + emb2_list = emb_list[1::2] + self.emb1_param = Tensor(emb1_list, dtype=mstype.int32) + self.emb2_param = Tensor(emb2_list, dtype=mstype.int32) + self.gatherv2 = P.GatherV2() + + def construct(self, nembeddings): + emb1 = self.gatherv2(nembeddings, self.emb1_param, 0) + emb2 = self.gatherv2(nembeddings, self.emb2_param, 0) + return self.pow((emb1 - emb2), 2.0) + + +def get_loss(batchsize): + return GatherV2(batchsize) + + +def fc_with_initialize(input_channels, out_channels): + return Dense(input_channels, out_channels) + + +class BuildTrainNetwork(nn.Cell): + def __init__(self, network, criterion): + super(BuildTrainNetwork, self).__init__() + self.network = network + self.criterion = criterion + + def construct(self, input_data): + embeddings = self.network(input_data) + loss = self.criterion(embeddings) + return loss + + +class TrainOneStepCell(Cell): + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + self.sens = sens + + def construct(self, data): + weights = self.weights + loss = self.network(data) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(data, sens) + + return F.depend(loss, self.optimizer(grads)) + + +def test_trains(): + init() + lr = 0.1 + momentum = 0.9 + max_epoch = 20 + device_number = 32 + batch_size_per_device = 128 + input_channels = 256 + out_channels = 512 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=device_number) + predict = Tensor(np.ones([batch_size_per_device, input_channels]), dtype=ms.float32) + dataset = Dataset(predict, 4) + + network = fc_with_initialize(input_channels, out_channels) + network.set_train() + + criterion = get_loss(batch_size_per_device * device_number) + + train_network = BuildTrainNetwork(network, criterion) + train_network.set_train() + opt = Momentum(train_network.trainable_params(), lr, momentum) + train_net = TrainOneStepCell(train_network, opt).set_train() + + model = Model(train_net) + model.train(max_epoch, dataset, dataset_sink_mode=False) + context.reset_auto_parallel_context() + +if __name__ == "__main__": + test_trains() diff --git a/tests/ut/python/parallel/test_get_next.py b/tests/ut/python/parallel/test_get_next.py new file mode 100644 index 00000000000..86bf3b42268 --- /dev/null +++ b/tests/ut/python/parallel/test_get_next.py @@ -0,0 +1,148 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.common.parameter import Parameter,ParameterTuple +from mindspore.ops.operations.comm_ops import _VirtualDataset +from mindspore import context +from mindspore.common.initializer import initializer +context.set_context(mode=context.GRAPH_MODE) + +class NetWithLoss(nn.Cell): + def __init__(self, network, types, shapes, output_num, strategy3=None, strategy4=None, axis=-1): + super(NetWithLoss, self).__init__() + self.get_next = P.GetNext(types, shapes, output_num, "") + self.one_hot = P.OneHot(axis=axis).set_strategy(strategy3) + self.on_value = Tensor(1.0, ms.float32) + self.off_value = Tensor(0.0, ms.float32) + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy4) + self.network = network + + def construct(self): + data, label = self.get_next() + predict = self.network(data) + label = self.one_hot(label, 64, self.on_value, self.off_value) + return self.loss(predict, label)[0] + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + + def construct(self): + return C.grad_by_list(self.network, self.weights)() + +def test_get_next_single(): + class Net(nn.Cell): + def __init__(self, channel=1, w=0.25): + super().__init__() + self.norm = P.L2Normalize(axis=1) + self.prelu = P.PReLU() + self.w = Parameter(initializer(w, [channel,]), name='w') + def construct(self, data): + x = self.norm(data) + x = self.prelu(x, self.w) + return x + + net = GradWrap(NetWithLoss(Net(), [ms.float32, ms.int32],[[32,64], [32]], 2)) + _executor.compile(net) + +def test_get_next_semi_auto_parallel(): + class Net(nn.Cell): + def __init__(self, channel=1, w=0.25, strategy1=None, strategy2=None): + super().__init__() + self.norm = P.L2Normalize().set_strategy(strategy1) + self.prelu = P.PReLU().set_strategy(strategy2) + self.w = Parameter(initializer(w, [channel,]), name='w') + + def construct(self, data): + x = self.norm(data) + x = self.prelu(x, self.w) + return x + + context.set_auto_parallel_context(device_num=4, global_rank=0) + network = Net(strategy1=((1,4), ), strategy2=((4,1),(1, ))) + strategy3 = ((4, 1),(),()) + strategy4=((4,1), (4,1)) + net_with_loss = NetWithLoss(network, [ms.float32, ms.int32],[[32,64], [32]], 2, strategy3=strategy3, strategy4=strategy4) + net = GradWrap(net_with_loss) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + _executor.compile(net) + +def test_get_next_semi_auto_parallel1(): + class Net(nn.Cell): + def __init__(self, channel=1, w=0.25, strategy1=None, strategy2=None): + super().__init__() + self.norm = P.L2Normalize().set_strategy(strategy1) + self.prelu = P.PReLU().set_strategy(strategy2) + self.w = Parameter(initializer(w, [channel,]), name='w') + + def construct(self, data): + x = self.norm(data) + x = self.prelu(x, self.w) + return x + + context.set_auto_parallel_context(device_num=4, global_rank=0) + network = Net(strategy1=((1,4), ), strategy2=((4,1),(1, ))) + strategy3 = ((1, 4),(),()) + strategy4=((4,1), (4,1)) + net_with_loss = NetWithLoss(network, [ms.float32, ms.int32],[[32,64], [32]], 2, strategy3=strategy3, strategy4=strategy4) + net = GradWrap(net_with_loss) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + _executor.compile(net) + +def test_get_next_auto_parallel(): + class Net(nn.Cell): + def __init__(self, channel=1, w=0.25, strategy1=None, strategy2=None): + super().__init__() + self.norm = P.L2Normalize().set_strategy(strategy1) + self.prelu = P.PReLU().set_strategy(strategy2) + self.w = Parameter(initializer(w, [channel,]), name='w') + + def construct(self, data): + x = self.norm(data) + x = self.prelu(x, self.w) + return x + + context.set_auto_parallel_context(device_num=4, global_rank=0) + network = Net() + net_with_loss = NetWithLoss(network, [ms.float32, ms.int32],[[32,64], [32]], 2) + net = GradWrap(net_with_loss) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + _executor.compile(net) + + +def test_only_one_get_next(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.get_next = P.GetNext([ms.float32, ms.int32],[[32,64], [32]], 2, "") + + def construct(self): + return self.get_next() + + + context.set_auto_parallel_context(device_num=4, global_rank=0) + net = Net() + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + _executor.compile(net) diff --git a/tests/ut/python/parallel/test_get_parameter_layout.py b/tests/ut/python/parallel/test_get_parameter_layout.py new file mode 100644 index 00000000000..82f2fc44f99 --- /dev/null +++ b/tests/ut/python/parallel/test_get_parameter_layout.py @@ -0,0 +1,59 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter +import mindspore as ms +import mindspore.common.api as me + + +def test_get_parameter_layout(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=8, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy1 = ((2, 1), (4, 1)) + strategy2 = ((2, 4), ) + context.set_context(mode=context.GRAPH_MODE) + + x = Tensor(np.ones([32, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + exe = me._executor + exe.compile(net, x) + x_layout = ([2, 4], [1, -1]) # device_arrangement = [2, 4], tensor_map = [1, -1] + weight_layout = ([2, 4], [0, -1]) # device_arrangement = [2, 4], tensor_map = [0, -1] + expect_dict = {'x': x_layout, 'w1': weight_layout} + # to be resovled: static local variable count_p is used in step_parallel.cc, it needs to be reset between each ut + assert (net._parameter_layout_dict == expect_dict) + + +if __name__ == '__main__': + test_get_parameter_layout() + diff --git a/tests/ut/python/parallel/test_hybird_parallel_activation.py b/tests/ut/python/parallel/test_hybird_parallel_activation.py new file mode 100644 index 00000000000..c4d7d1d302f --- /dev/null +++ b/tests/ut/python/parallel/test_hybird_parallel_activation.py @@ -0,0 +1,239 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore as ms +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def test_matmul_tanh(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.tanh = P.Tanh().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.tanh(self.matmul1(x, y)) + out = self.matmul2(out, b) + return out + + strategy1 = ((16, 1), (1, 1)) + strategy2 = ((1, 1), (1, 16)) + strategy3 = ((4, 4), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=16, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_activation(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.activation = P.ReLU().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.activation(self.matmul1(x, y)) + out = self.matmul2(out, b) + return out + + strategy1 = ((16, 1), (1, 1)) + strategy2 = ((1, 1), (1, 16)) + strategy3 = ((4, 4), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=16, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_softmax(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.softmax = P.Softmax().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.softmax(self.matmul1(x, y)) + out = self.matmul2(out, b) + return out + + strategy1 = ((16, 1), (1, 1)) + strategy2 = ((1, 1), (1, 16)) + strategy3 = ((16, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=16, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_matmul_logsoftmax(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.logsoftmax = P.LogSoftmax().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.logsoftmax(self.matmul1(x, y)) + out = self.matmul2(out, b) + return out + + strategy1 = ((4, 2), (2, 2)) + strategy2 = ((2, 4), (4, 2)) + strategy3 = ((16, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=16, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_activations(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.gelu = P.Gelu().set_strategy(strategy3) + self.tanh = P.Tanh().set_strategy(strategy3) + self.softmax = P.Softmax().set_strategy(strategy3) + self.logsoftmax = P.LogSoftmax().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.gelu(self.tanh(self.matmul1(x, y))) + out = self.logsoftmax(self.softmax(self.matmul2(out, b))) + return out + + strategy1 = ((1, 2), (2, 2)) + strategy2 = ((2, 2), (2, 1)) + strategy3 = ((4, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=4, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_activations_repeated_calculation(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3, strategy4, strategy5, strategy6): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.gelu = P.Gelu().set_strategy(strategy3) + self.tanh = P.Tanh().set_strategy(strategy4) + self.softmax = P.Softmax().set_strategy(strategy5) + self.logsoftmax = P.LogSoftmax().set_strategy(strategy6) + + def construct(self, x, y, b): + out = self.gelu(self.tanh(self.matmul1(x, y))) + out = self.logsoftmax(self.softmax(self.matmul2(out, b))) + return out + + strategy1 = ((2, 4), (4, 8)) + strategy2 = ((2, 2), (2, 1)) + strategy3 = ((2, 1), ) + strategy4 = ((2, 2), ) + strategy5 = ((4, 1), ) + strategy6 = ((8, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3, strategy4, strategy5, strategy6))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=64, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_activations_axis_tuple(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3, strategy4, strategy5, strategy6): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.gelu = P.Gelu().set_strategy(strategy3) + self.tanh = P.Tanh().set_strategy(strategy4) + self.softmax = P.Softmax(axis=(0, 1)).set_strategy(strategy5) + self.logsoftmax = P.LogSoftmax().set_strategy(strategy6) + + def construct(self, x, y, b): + out = self.gelu(self.tanh(self.matmul1(x, y))) + out = self.logsoftmax(self.softmax(self.matmul2(out, b))) + return out + + strategy1 = ((2, 4), (4, 8)) + strategy2 = ((2, 2), (2, 1)) + strategy3 = ((2, 1), ) + strategy4 = ((2, 2), ) + strategy5 = ((1, 1), ) + strategy6 = ((8, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3, strategy4, strategy5, strategy6))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=64, global_rank=0) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_l2normalize.py b/tests/ut/python/parallel/test_l2normalize.py new file mode 100644 index 00000000000..ae0ee29e839 --- /dev/null +++ b/tests/ut/python/parallel/test_l2normalize.py @@ -0,0 +1,72 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +# model_parallel test +def test_l2normalize_matmul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.norm1 = P.L2Normalize(axis=0).set_strategy(strategy1) + self.norm2 = P.L2Normalize(axis=0).set_strategy(strategy1) + self.mul1 = P.Mul().set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + y = self.norm1(y) + x = self.norm2(x) + out = self.mul1(x, y) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 1, 4), ) + strategy2 = ((1, 1, 4), (1, 1, 4)) + strategy3 = ((1, 1, 8), (1, 1, 8)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_linear.py b/tests/ut/python/parallel/test_linear.py new file mode 100644 index 00000000000..ca31dd8721d --- /dev/null +++ b/tests/ut/python/parallel/test_linear.py @@ -0,0 +1,70 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy3) + self.network = network + + def construct(self, x, y, bias, label): + predict = self.network(x, y, bias) + return self.loss(predict, label)[0] + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, bias, label): + return C.grad_all(self.network)(x, y, bias, label) + +def test_linear(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1, strategy2): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.add = P.TensorAdd().set_strategy(strategy1) + self.gelu = P.Gelu().set_strategy(strategy2) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.add(out, bias) + out = self.gelu(out) + return out + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((2, 4), (2, 4)) + strategy1 = ((2, 4), (4, )) + strategy2 = ((2, 8), ) + strategy3 = ((16, 1), (16, 1)) + net = GradWrap(NetWithLoss(Net(strategy0, strategy1, strategy2), strategy3)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + bias = Tensor(np.ones([64]), dtype=ms.float32) + label = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, bias, label) diff --git a/tests/ut/python/parallel/test_load_tensor.py b/tests/ut/python/parallel/test_load_tensor.py new file mode 100644 index 00000000000..6335a2da905 --- /dev/null +++ b/tests/ut/python/parallel/test_load_tensor.py @@ -0,0 +1,42 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.parallel._tensor import _load_tensor +from mindspore import Tensor +from hccl_test.manage.api import Hccl + + +def test_load_tensor(): + hccl = Hccl() + tensor = Tensor([[1, 2, 3], [4, 5, 6]]) + dev_mat = [2, 3] + tensor_map = [1, -1] + hccl.rank_id = 5 + tensor_slice = _load_tensor(tensor, dev_mat, tensor_map) + expected_tensor = Tensor([[4, 5, 6]]) + if expected_tensor.__str__() != tensor_slice.__str__(): + raise AssertionError + + hccl.rank_id = 2 + tensor_slice = _load_tensor(tensor, dev_mat, tensor_map) + expected_tensor = Tensor([[1, 2, 3]]) + if expected_tensor.__str__() != tensor_slice.__str__(): + raise AssertionError + + # set back to the defalt value + hccl.rank_id = 0 + + +if __name__ == '__main__': + test_load_tensor() diff --git a/tests/ut/python/parallel/test_loop_two_matmul.py b/tests/ut/python/parallel/test_loop_two_matmul.py new file mode 100644 index 00000000000..9d724069774 --- /dev/null +++ b/tests/ut/python/parallel/test_loop_two_matmul.py @@ -0,0 +1,95 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import math +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def loop_config(size): + config_list = [] + num = 1 + split_list = [num] + for i in range(int(math.log2(size))): + num = num * 2 + split_list.append(num) + + for a in split_list: + for b in split_list: + if a * b > size: + continue + c = int(size / (a * b)) + config_list.append(((a, b), (b, c))) + + return config_list + + +# model_parallel test +def test_two_matmul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + size = 4 + context.set_auto_parallel_context(device_num=size, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + config_list = loop_config(size) + + count = 0 + for strategy1 in config_list: + for strategy2 in config_list: + print("=======current config {}=========".format(count)) + print(strategy1, strategy2) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + _executor.compile(net, x, y, b) + count = count + 1 + diff --git a/tests/ut/python/parallel/test_loss_and_optimizer.py b/tests/ut/python/parallel/test_loss_and_optimizer.py new file mode 100644 index 00000000000..b87f2a556af --- /dev/null +++ b/tests/ut/python/parallel/test_loss_and_optimizer.py @@ -0,0 +1,208 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.nn.optim import Momentum, LARS +from mindspore.nn import TrainOneStepCell, WithLossCell + + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy3) + self.network = network + + def construct(self, x, b): + predict = self.network(x) + return self.loss(predict, b)[0] + + +def test_momentum(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) + + +def test_momentum_with_loss_scale(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9, loss_scale=0.5) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) + + +def test_momentum_with_dynamic_lr(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + lr = Tensor(np.ones([6]), dtype=ms.float32) + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) + + +def test_momentum_with_loss_scale_and_dynamic_lr(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + lr = Tensor(np.ones([6]), dtype=ms.float32) + optimizer = Momentum(net.trainable_params(), learning_rate=lr, momentum=0.9, loss_scale=0.5) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) + +def test_lars(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + lr = Tensor(np.ones([6]), dtype=ms.float32) + SGD = Momentum(net.trainable_params(), lr, 0.9) + optimizer = LARS(SGD, epsilon=1e-08, hyperpara=0.02, decay_filter=lambda x: 'bn' not in x.name, + lars_filter=lambda x: 'bn' not in x.name) + net_with_loss = NetWithLoss(net, strategy3) + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) diff --git a/tests/ut/python/parallel/test_matmul_dropout.py b/tests/ut/python/parallel/test_matmul_dropout.py new file mode 100644 index 00000000000..536bffdbfa6 --- /dev/null +++ b/tests/ut/python/parallel/test_matmul_dropout.py @@ -0,0 +1,74 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +# model_parallel test +def test_two_matmul_dropout(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.dropout = nn.Dropout() + self.dropout.dropout_do_mask.set_strategy(strategy2) + self.dropout.dropout_gen_mask.set_strategy(strategy2) + self.matmul2 = P.MatMul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.dropout(out) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 2), (2, 1)) + strategy2 = ((8, 1), ) + strategy3 = ((1, 8), (8, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_matmul_tensor.py b/tests/ut/python/parallel/test_matmul_tensor.py new file mode 100644 index 00000000000..3987d66a5b9 --- /dev/null +++ b/tests/ut/python/parallel/test_matmul_tensor.py @@ -0,0 +1,154 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore.context import set_auto_parallel_context +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +import mindspore.common.dtype as mstype + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y): + predict = self.network(x, y) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y): + return C.grad_all(self.network)(x, y) + + +# model_parallel test +def test_two_matmul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.diag = P.Diag() + self.fill = P.Fill() + + def construct(self, x, y): + fill = self.diag(self.fill(mstype.float32, (128, ), 1.0)) + out1 = self.matmul1(fill, x) + out2 = self.matmul2(y, fill) + out = self.matmul3(out1, out2) + return out + + set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((1, 8), (8, 1)) + strategy3 = ((2, 4), (4, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 128]), dtype=ms.float32) + + _executor.compile(net, x, y) + + +def test_matmul_mul_broadcast2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.mul = P.Mul().set_strategy(strategy2) + self.t = Tensor(0.9, ms.float32) + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.mul(out, self.t) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 4), (4, 1)) + strategy2 = ((4, 1), ()) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 1]), dtype=ms.float32) + + _executor.compile(net, x, y) + +def test_two_matmul1(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.diag = P.Diag() + self.fill = P.Fill() + + def construct(self, x, y): + fill = self.diag(self.fill(mstype.float32, (128, ), 1.0)) + out1 = self.matmul1(fill, x) + out2 = self.matmul2(fill, y) + out = self.matmul3(out1, out2) + return out + + set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((1, 8), (8, 1)) + strategy3 = ((2, 4), (4, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 128]), dtype=ms.float32) + y = Tensor(np.ones([128, 128]), dtype=ms.float32) + + _executor.compile(net, x, y) + +def test_matmul_add_tensor(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.add = P.TensorAdd().set_strategy(strategy2) + self.b = Tensor(0.9, ms.float32) + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.add(out, self.b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((4, 2), ()) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + + _executor.compile(net, x, y) diff --git a/tests/ut/python/parallel/test_mix_precision_hybrid_parallel.py b/tests/ut/python/parallel/test_mix_precision_hybrid_parallel.py new file mode 100644 index 00000000000..e6193634372 --- /dev/null +++ b/tests/ut/python/parallel/test_mix_precision_hybrid_parallel.py @@ -0,0 +1,85 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b, z): + predict = self.network(x, y, b, z) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b, z): + return C.grad_all(self.network)(x, y, b, z) + + +class Net1(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.matmul3 = P.MatMul().set_strategy(strategy3) + + def construct(self, x, y, b): + out1 = self.matmul1(x, b) + out2 = self.matmul2(y, b) + out = self.matmul3(out1, out2) + return out + + +def test_two_matmul(): + class Net2(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3, strategy4): + super().__init__() + self.net1_out = Net1(strategy1, strategy2, strategy3) + self.matmul = P.MatMul().set_strategy(strategy4) + + def construct(self, x, y, b, z): + out = self.net1_out(x, y, b) + out = self.matmul(out, z) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 1)) + strategy2 = ((4, 2), (2, 1)) + strategy3 = ((1, 8), (8, 1)) + strategy4 = ((2, 4), (4, 1)) + net = GradWrap(NetWithLoss(Net2(strategy1, strategy2, strategy3, strategy4).add_flags_recursive(fp16=True))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + z = Tensor(np.ones([64, 64]), dtype=ms.float32) + + _executor.compile(net, x, y, b, z) diff --git a/tests/ut/python/parallel/test_model_callback.py b/tests/ut/python/parallel/test_model_callback.py new file mode 100644 index 00000000000..f48b10ef7f0 --- /dev/null +++ b/tests/ut/python/parallel/test_model_callback.py @@ -0,0 +1,135 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.parallel._utils import _reset_op_id +from mindspore.train.callback import Callback + + +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class AllToAllNet(nn.Cell): + def __init__(self, strategy1): + super(AllToAllNet, self).__init__() + self.matmul = P.MatMul().set_strategy(((1, 1), (1, 8))) + self.matmul_weight = Parameter(Tensor(np.ones([128, 256]), dtype=ms.float32), name="weight") + self.transpose1 = P.Transpose().set_strategy(strategy1) + + def construct(self, x): + x = self.matmul(x, self.matmul_weight) + x = self.transpose1(x, (1, 0)) + return x + + +def all_to_all_net(strategy1): + return AllToAllNet(strategy1=strategy1) + + +class ContextCallback(Callback): + def begin(self, run_context): + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.STAND_ALONE + + def epoch_begin(self, run_context): + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.STAND_ALONE + + def epoch_end(self, run_context): + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.STAND_ALONE + + def step_begin(self, run_context): + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.STAND_ALONE + + def step_end(self, run_context): + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.STAND_ALONE + + def end(self, run_context): + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.STAND_ALONE + + +def all_to_all_common(strategy1): + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=8) + predict = Tensor(np.ones([32, 128]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = all_to_all_net(strategy1) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((8, 1), (8, 1))) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss, opt) + + context_callback = ContextCallback() + + model.train(epoch_size, dataset, dataset_sink_mode=False, callbacks=[context_callback]) + + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.SEMI_AUTO_PARALLEL + + context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, device_num=8) + model.train(epoch_size, dataset, dataset_sink_mode=False, callbacks=[context_callback]) + parallel_mode = context.get_auto_parallel_context("parallel_mode") + assert parallel_mode == ParallelMode.AUTO_PARALLEL + + context.reset_auto_parallel_context() + + +def test_model_callback(): + strategy1 = ((8, 1), ) + _reset_op_id() + all_to_all_common(strategy1) + + + diff --git a/tests/ut/python/parallel/test_one_dev.py b/tests/ut/python/parallel/test_one_dev.py new file mode 100644 index 00000000000..a5867eb5f41 --- /dev/null +++ b/tests/ut/python/parallel/test_one_dev.py @@ -0,0 +1,104 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.parallel._utils import _reset_op_id +from mindspore.common.api import _executor + +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class AllToAllNet(nn.Cell): + def __init__(self): + super(AllToAllNet, self).__init__() + self.matmul = P.MatMul() + self.matmul_weight = Parameter(Tensor(np.ones([128, 32]), dtype=ms.float32), name="weight") + self.transpose1 = P.Transpose() + + def construct(self, x): + x = self.matmul(x, self.matmul_weight) + x = self.transpose1(x, (1, 0)) + return x + + +def all_to_all_net(): + return AllToAllNet() + + +def all_to_all_common(): + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, device_num=1, global_rank=0) + predict = Tensor(np.ones([32, 128]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = all_to_all_net() + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss, opt) + + model.train(epoch_size, dataset,dataset_sink_mode=False) + strategys = _executor._get_strategy(model._train_network) + return strategys + + +def test_one_dev(): + + _reset_op_id() + strategys = all_to_all_common() + expect_dict = {'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits' + '/SoftmaxCrossEntropyWithLogits-op9': [[1, 1], [1, 1]], + 'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits' + '/OneHot-op10': [[1, 1], [], []], + 'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_backbone-AllToAllNet/Transpose-op11': + [[1, 1]], + 'Default/network-_VirtualDatasetCell/_backbone-WithLossCell/_backbone-AllToAllNet/MatMul-op12': + [[1, 1], [1, 1]]} + assert (strategys == expect_dict) + diff --git a/tests/ut/python/parallel/test_one_hot_net.py b/tests/ut/python/parallel/test_one_hot_net.py new file mode 100644 index 00000000000..bf2677056ef --- /dev/null +++ b/tests/ut/python/parallel/test_one_hot_net.py @@ -0,0 +1,309 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore import Tensor, Parameter +from mindspore.common import dtype as mstype +import mindspore.nn as nn +import numpy as np +from mindspore.nn.cell import Cell +from tests.dataset_mock import MindData +from mindspore.nn.optim.momentum import Momentum +from mindspore.train import Model, ParallelMode +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.ops import composite as C +import mindspore as ms +from mindspore.common.api import _executor +from mindspore import context + + +device_num=16 +device_id = 2 + + +class StrategyModel(): + onehot_strategy = ((1, device_num),(),()) + twod_strategy = ((1, device_num), ) + twod_strategy_m = ((device_num, 1), ) + scalar_twod_strategy = ((), (1, device_num)) + twod_scalar_strategy = ((1, device_num), ()) + scalar_strategy = ((), ) + oned_strategy = ((1, ), ) + scalar_scalar_strategy = ((), ()) + twod_twod_strategy = ((1, device_num), (1, device_num)) + twod_twodbc_strategy = ((1, device_num), (1, 1)) + twodbc_twod_strategy = ((1, 1), (device_num, 1)) + + +class StrategyBatch(): + onehot_strategy = ((device_num, 1),(),()) + twod_strategy = ((1, device_num), ) + twod_strategy_m = ((device_num, 1), ) + scalar_twod_strategy = ((), (1, device_num)) + twod_scalar_strategy = ((1, device_num), ()) + scalar_strategy = ((), ) + oned_strategy = ((1, ), ) + scalar_scalar_strategy = ((), ()) + twod_twod_strategy = ((1, device_num), (1, device_num)) + twod_twodbc_strategy = ((1, device_num), (1, 1)) + twodbc_twod_strategy = ((1, 1), (device_num, 1)) + + +class Args(): + a = 1 + b = 2 + c = 3 + d = 4 + e = 5 + num_classes = 512 + emb_size = 512 + + +class SemiAutoOneHotNet(Cell): + def __init__(self, args, strategy): + super(SemiAutoOneHotNet, self).__init__() + self.a = args.a + self.b = args.b + self.c = args.c + self.d = args.d + self.e = args.e + self.cast = P.Cast() + self.cast.set_strategy(strategy=strategy.twod_strategy) + self.cast1 = P.Cast() + self.cast1.set_strategy(strategy=strategy.twod_strategy) + self.cast2 = P.Cast() + self.cast2.set_strategy(strategy=strategy.twod_strategy) + self.cast3 = P.Cast() + self.cast3.set_strategy(strategy=strategy.scalar_strategy) + self.cast4 = P.Cast() + self.cast4.set_strategy(strategy=strategy.scalar_strategy) + self.a_const = Tensor(self.a, dtype=mstype.float32) + self.b_const = Tensor(self.b, dtype=mstype.float32) + self.c_const = Tensor(self.c, dtype=mstype.float32) + self.d_const = Tensor(self.d, dtype=mstype.float32) + self.e_const = Tensor(self.e, dtype=mstype.float32) + self.m_const_zero = Tensor(0, dtype=mstype.float32) + self.a_const_one = Tensor(1, dtype=mstype.float32) + self.onehot = P.OneHot() + self.onehot.set_strategy(strategy=strategy.onehot_strategy) + self.exp = P.Exp() + self.exp.set_strategy(strategy=strategy.twod_strategy) + self.exp2 = P.Exp() + self.exp2.set_strategy(strategy=strategy.twod_strategy) + self.exp3 = P.Exp() + self.exp3.set_strategy(strategy=strategy.twod_strategy) + self.mul_const = P.Mul() + self.mul_const.set_strategy(strategy=strategy.scalar_twod_strategy) + self.mul_const2 = P.TensorAdd() + self.mul_const2.set_strategy(strategy=strategy.scalar_twod_strategy) + self.mul_const3 = P.Sub() + self.mul_const3.set_strategy(strategy=strategy.twod_scalar_strategy) + self.mul_const4 = P.Sub() + self.mul_const4.set_strategy(strategy=strategy.scalar_twod_strategy) + self.mul_const5 = P.Mul() + self.mul_const5.set_strategy(strategy=strategy.twod_scalar_strategy) + self.mul = P.Mul() + self.mul.set_strategy(strategy=strategy.twod_twod_strategy) + self.mul2 = P.Mul() + self.mul2.set_strategy(strategy=strategy.twod_twod_strategy) + self.mul3 = P.TensorAdd() + self.mul3.set_strategy(strategy=strategy.twod_twod_strategy) + self.mul4 = P.Sub() + self.mul4.set_strategy(strategy=strategy.twod_twodbc_strategy) + self.mul5 = P.RealDiv() + self.mul5.set_strategy(strategy=strategy.twod_twodbc_strategy) + self.mul6 = P.Mul() + self.mul6.set_strategy(strategy=strategy.twod_twod_strategy) + self.mul7 = P.Mul() + self.mul7.set_strategy(strategy=strategy.twod_scalar_strategy) + self.mul8 = P.RealDiv() + self.mul8.set_strategy(strategy=strategy.scalar_scalar_strategy) + self.mul9 = P.TensorAdd() + self.mul9.set_strategy(strategy=strategy.twod_scalar_strategy) + + self.reduce_max = P.ReduceMax(keep_dims=True) + self.reduce_max.set_strategy(strategy=strategy.twod_strategy) + + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.reduce_sum.set_strategy(strategy=strategy.twod_strategy) + self.reduce_sum_2 = P.ReduceSum(keep_dims=False) + self.reduce_sum_2.set_strategy(strategy=strategy.twod_strategy) + self.reduce_sum_3 = P.ReduceSum(keep_dims=False) + self.reduce_sum_3.set_strategy(strategy=strategy.oned_strategy) + + self.reshape = P.Reshape() + self.log = P.Log() + self.log.set_strategy(strategy=strategy.twod_strategy) + + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.normalize = P.L2Normalize(axis=1) + self.normalize.set_strategy(strategy=strategy.twod_strategy_m) + self.normalize2 = P.L2Normalize(axis=1) + self.normalize2.set_strategy(strategy=strategy.twod_strategy_m) + self.fc = P.MatMul(transpose_b=True) + self.fc.set_strategy(strategy=strategy.twodbc_twod_strategy) + weight_shape = [args.num_classes, args.emb_size] + weight_np = np.zeros(weight_shape, np.float32) + self.weight = Parameter(Tensor(weight_np), name='model_parallel_weight') + + def construct(self, input, label): + input_n = self.normalize(input) + w = self.normalize2(self.weight) + fc_o = self.fc(input_n, w) + fc_o_shape = F.shape(fc_o) + one_hot_float = self.onehot(label, fc_o_shape[1],self.on_value, self.off_value) + local_label = self.cast(one_hot_float, mstype.int32) + + exp_o = self.exp(fc_o) + mul_const_o = self.mul_const(self.a_const, exp_o) + mul_const2_o = self.mul_const2(self.b_const, mul_const_o) + exp2_o = self.exp2(mul_const2_o) + mul_const3_o = self.mul_const3(exp2_o, self.c_const) + mul_const4_o = self.mul_const4(F.scalar_to_array(1), local_label) + mul6_o = self.mul6(self.mul(mul_const3_o, one_hot_float), self.mul2(fc_o, self.cast2(mul_const4_o, mstype.float32))) + mul_const5_o = self.mul_const5(mul6_o, self.d_const) + + max_o = self.reduce_max(mul_const5_o, -1) + mul4_o = self.mul4(mul_const5_o, max_o) + exp3_o = self.exp3(mul4_o) + sum_o = self.reduce_sum(exp3_o, -1) + reshape_o = self.reshape(sum_o, (F.shape(sum_o)[0], 1)) + mul5_o = self.mul5(exp3_o, reshape_o) + log_o = self.log(self.mul9(mul5_o, self.e_const)) + mul3_o = self.mul3(log_o, one_hot_float) + mul7_o = self.mul7(mul3_o, self.cast3(F.scalar_to_array(-1), mstype.float32)) + sum2_o = self.reduce_sum_2(mul7_o, -1) + loss = self.mul8(self.reduce_sum_3(sum2_o, -1), self.cast4(F.scalar_to_array(F.shape(mul_const5_o)[0]), mstype.float32)) + return loss + + +class Dataset(MindData): + def __init__(self, predict, label, length=3, input_num=2): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + self.input_num = input_num + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + if self.input_num == 2: + return self.predict, self.label + else: + return self.predict, + + def reset(self): + self.index = 0 + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, b): + predict = self.network(x, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, b): + return C.grad_all(self.network)(x, b) + + +def bn_with_initialize(out_channels): + bn = nn.BatchNorm2d(out_channels, momentum=0.3, eps=1e-5).add_flags_recursive(fp32=True) + return bn + + +def fc_with_initialize(input_channels, out_channels): + return nn.Dense(input_channels, out_channels) + + +class BNReshapeDenseBNNet(nn.Cell): + def __init__(self): + super(BNReshapeDenseBNNet, self).__init__() + self.batch_norm = bn_with_initialize(512) + self.reshape = P.Reshape() + self.batch_norm2 = nn.BatchNorm1d(512, affine=False) + self.fc = fc_with_initialize(512 * 32 * 32, 512) + self.loss = SemiAutoOneHotNet(args=Args(), strategy=StrategyBatch()) + + def construct(self, x, label): + x = self.batch_norm(x) + x = self.reshape(x, (16, 512*32*32)) + x = self.fc(x) + x = self.batch_norm2(x) + loss = self.loss(x, label) + return loss + + +def test_bn_reshape_dense_bn_train_loss(): + batch_size = 16 + device_num = 16 + context.set_auto_parallel_context(device_num=device_num, global_rank=0) + input = Tensor(np.ones([batch_size, 512, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + + net = GradWrap(NetWithLoss(BNReshapeDenseBNNet())) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(net, input, label) + + +def test_semi_one_hot_net_batch(): + batch_size = 16 + context.set_auto_parallel_context(device_num=device_num, global_rank=0) + input = Tensor(np.ones([batch_size * 1, 512]).astype(np.float32) * 0.01) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + + net = SemiAutoOneHotNet(args=Args(), strategy=StrategyBatch()) + net = GradWrap(NetWithLoss(net)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(net, input, label) + + +def test_semi_one_hot_net_model(): + batch_size = 16 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + predict = Tensor(np.ones([batch_size, 512]), dtype=ms.float32) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + dataset = Dataset(predict, label, 2, input_num=2) + + net = SemiAutoOneHotNet(args=Args(), strategy=StrategyModel()) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=16) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, optimizer=opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + diff --git a/tests/ut/python/parallel/test_one_weight_parameter.py b/tests/ut/python/parallel/test_one_weight_parameter.py new file mode 100644 index 00000000000..03dd4f2920c --- /dev/null +++ b/tests/ut/python/parallel/test_one_weight_parameter.py @@ -0,0 +1,73 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter, ParameterTuple +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops import functional as F + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy3) + self.network = network + + def construct(self, x, b): + predict = self.network(x) + return self.loss(predict, b)[0] + +class OneStepCell(nn.Cell): + def __init__(self, network): + super(OneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.network.trainable_params()) + + def construct(self, data, label): + weights = self.weights + grads = C.grad_by_list(self.network, weights)(data, label) + return grads + +def test_one_weight_parameter(): + class Net(nn.Cell): + def __init__(self, strategy1, weight): + super().__init__() + self.weight = Parameter(weight, "w1", requires_grad=True) + self.matmul = P.MatMul().set_strategy(strategy1) + + def construct(self, x): + out = self.matmul(x, self.weight) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 1), (1, 2)) + strategy3 = ((8, 1), (8, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, weight) + print ("======================================dict", net.__dict__) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = OneStepCell(net_with_loss) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) diff --git a/tests/ut/python/parallel/test_onehot.py b/tests/ut/python/parallel/test_onehot.py new file mode 100644 index 00000000000..b41e3b21885 --- /dev/null +++ b/tests/ut/python/parallel/test_onehot.py @@ -0,0 +1,172 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops.operations.comm_ops import _VirtualDataset +context.set_context(mode=context.GRAPH_MODE) + + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3, strategy4, axis): + super(NetWithLoss, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.one_hot = P.OneHot(axis=axis).set_strategy(strategy3) + self.on_value = Tensor(2.0, ms.float32) + self.off_value = Tensor(1.0, ms.float32) + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy4) + self.network = network + + def construct(self, x, y, b): + b_virtual = self.virtual_dataset(b) + predict = self.network(x, y) + label = self.one_hot(b_virtual, 64, self.on_value, self.off_value) + return self.loss(predict, label)[0] + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.gelu = P.Gelu().set_strategy(strategy2) + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.gelu(out) + return out + + +def compile_graph(strategy1, strategy2, strategy3, strategy4, auto=False, onthot_axis=-1): + net = GradWrap(NetWithLoss(Net(strategy1, strategy2), strategy3, strategy4, axis=onthot_axis)) + if auto: + context.set_auto_parallel_context(parallel_mode="auto_parallel") + else: + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64]), dtype=ms.int32) + _executor.compile(net, x, y, b) + + +def test_onehot_model_parallel(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((1, 16), (), ()) + strategy4 = ((16, 1), (16, 1)) + compile_graph(strategy1, strategy2, strategy3, strategy4) + + +def test_onehot_batch_parallel(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((16, 1), (), ()) + strategy4 = ((16, 1), (16, 1)) + compile_graph(strategy1, strategy2, strategy3, strategy4) + + +def test_onehot_batch_parallel_invalid_strategy(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((16, ), (), ()) + strategy4 = ((16, 1), (16, 1)) + try: + compile_graph(strategy1, strategy2, strategy3, strategy4) + except: + pass + + +def test_onehot_repeated_calculation(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((4, 1), (), ()) + strategy4 = ((16, 1), (16, 1)) + compile_graph(strategy1, strategy2, strategy3, strategy4) + + +def test_onehot_auto(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = None + strategy2 = None + strategy3 = None + strategy4 = None + compile_graph(strategy1, strategy2, strategy3, strategy4, auto=True) + + +def test_onehot_model_parallel(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((1, 16), (), ()) + strategy4 = ((16, 1), (16, 1)) + compile_graph(strategy1, strategy2, strategy3, strategy4) + + +def test_onehot_batch_parallel_axis0(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((16, 1), (), ()) + strategy4 = ((16, 1), (16, 1)) + compile_graph(strategy1, strategy2, strategy3, strategy4, onthot_axis=0) + + +# auto parallel for onehot axis equal to 0 has not been supported yet +def test_onehot_batch_parallel_invalid_strategy_axis0(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = None + strategy4 = ((16, 1), (16, 1)) + try: + compile_graph(strategy1, strategy2, strategy3, strategy4, onthot_axis=0) + except: + pass + + +def test_onehot_repeated_calculation_axis0(): + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((2, 8), ) + strategy3 = ((4, 1), (), ()) + strategy4 = ((16, 1), (16, 1)) + compile_graph(strategy1, strategy2, strategy3, strategy4, onthot_axis=0) + + +def test_onehot_auto_axis0(): + context.set_auto_parallel_context(device_num=16, global_rank=14) + strategy1 = None + strategy2 = None + strategy3 = None + strategy4 = None + compile_graph(strategy1, strategy2, strategy3, strategy4, auto=True, onthot_axis=0) diff --git a/tests/ut/python/parallel/test_operator_model_parallel.py b/tests/ut/python/parallel/test_operator_model_parallel.py new file mode 100644 index 00000000000..389aeb1f086 --- /dev/null +++ b/tests/ut/python/parallel/test_operator_model_parallel.py @@ -0,0 +1,405 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +import mindspore as ms +import numpy as np +import mindspore.ops.operations.math_ops as M +from mindspore.nn.layer.basic import Dense +from mindspore.nn.layer.basic import Flatten +from mindspore.ops.operations import TensorAdd +from mindspore.common.tensor import Tensor +from mindspore.nn.layer.activation import ReLU +from mindspore.nn.cell import Cell +from mindspore.nn.layer.conv import Conv2d +from mindspore.nn.layer.normalization import BatchNorm2d +from mindspore.nn.layer.pooling import MaxPool2d +from mindspore.ops import operations as P +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore import context +from tests.dataset_mock import MindData + + +dev_num = 8 +strategy_no_weight = ((dev_num, 1, 1, 1), ) +strategy_weight = ((dev_num, 1, 1, 1), (1, 1, 1, 1)) +strategy_add = ((dev_num, 1, 1, 1), (dev_num, 1, 1, 1)) +strategy_bn = ((dev_num, 1, 1, 1), (1, ), (1, )) + +strategy_fc_weight_nobias = ((1, dev_num), (1, dev_num)) +strategy_tensor_add = ((1, dev_num), (dev_num, )) + +class DenseWrap(Cell): + def __init__(self, + input_channels, + output_channels, + weight_init='normal', + bias_init='zeros', + has_bias=True, + matmul_strategy=None, + set_strategy=None): + + super(DenseWrap, self).__init__() + + self.input_channels = input_channels + self.output_channels = output_channels + self.has_bias = has_bias + + self.weight = Parameter(initializer( + weight_init, [output_channels, input_channels]), + name="weight") + + if self.has_bias: + self.bias = Parameter(initializer( + bias_init, [output_channels]), name="bias") + else: + self.bias = Parameter(initializer( + 'zeros', [output_channels]), name="bias") + + self.matmul = P.MatMul(transpose_b=True).set_strategy(matmul_strategy) + self.bias_add = P.TensorAdd().set_strategy(set_strategy) + + def construct(self, x): + if self.has_bias: + output = self.bias_add(self.matmul(x, self.weight), self.bias) + else: + output = self.matmul(x, self.weight) + return output + + +class DatasetLenet(MindData): + def __init__(self, predict, label, length=3): + super(DatasetLenet, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +def conv3x3(in_channels, out_channels, stride=1, padding=1): + """3x3 convolution """ + weight_shape = (out_channels, in_channels, 3, 3) + weight = Tensor(np.ones(weight_shape).astype(np.float32)) + conv = Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, padding=0, weight_init=weight, has_bias=False, + pad_mode="same") + conv.conv2d.set_strategy(strategy_weight) + return conv + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 1, 1) + weight = Tensor(np.ones(weight_shape).astype(np.float32)) + conv = Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, padding=0, weight_init=weight, has_bias=False, + pad_mode="same") + conv.conv2d.set_strategy(strategy_weight) + return conv + + +def conv7x7(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight_shape = (out_channels, in_channels, 7, 7) + weight = Tensor(np.ones(weight_shape).astype(np.float32)) + conv = Conv2d(in_channels, out_channels, + kernel_size=7, stride=stride, padding=0, weight_init=weight, has_bias=False, + pad_mode="same") + conv.conv2d.set_strategy(strategy_weight) + return conv + + +def weight_variable_0(shape): + zeros = np.zeros(shape).astype(np.float32) + return Tensor(zeros) + + +def weight_variable_1(shape): + ones = np.ones(shape).astype(np.float32) + return Tensor(ones) + + +def bn_with_initialize(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_1(shape) + bn = BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + bn.bn_train.set_strategy(strategy_bn) + return bn + + +def bn_with_initialize_last(out_channels): + shape = (out_channels) + mean = weight_variable_0(shape) + var = weight_variable_1(shape) + beta = weight_variable_0(shape) + gamma = weight_variable_0(shape) + bn = BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, + beta_init=beta, moving_mean_init=mean, moving_var_init=var) + bn.bn_train.set_strategy(strategy_bn) + return bn + + +def fc_with_initialize(input_channels, out_channels): + weight_shape = (out_channels, input_channels) + bias_shape = (out_channels) + weight = weight_variable_0(weight_shape) + bias = weight_variable_0(bias_shape) + + return DenseWrap(input_channels, out_channels, weight, bias, has_bias=True, + matmul_strategy=strategy_fc_weight_nobias, set_strategy=strategy_tensor_add) + + +class ResidualBlock(Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu1 = P.ReLU().set_strategy(strategy_no_weight) + self.relu2 = P.ReLU().set_strategy(strategy_no_weight) + self.relu3 = P.ReLU().set_strategy(strategy_no_weight) + self.add = TensorAdd().set_strategy(strategy_add) + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu1(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu2(out) + + out = self.conv3(out) + out = self.bn3(out) + + out = self.add(out, identity) + out = self.relu3(out) + + return out + + +class ResidualBlockWithDown(Cell): + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlockWithDown, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = bn_with_initialize(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=0) + self.bn2 = bn_with_initialize(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = bn_with_initialize_last(out_channels) + + self.relu1 = P.ReLU().set_strategy(strategy_no_weight) + self.relu2 = P.ReLU().set_strategy(strategy_no_weight) + self.relu3 = P.ReLU().set_strategy(strategy_no_weight) + self.downSample = down_sample + + self.conv_down_sample = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn_down_sample = bn_with_initialize(out_channels) + self.add = TensorAdd().set_strategy(strategy_add) + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu1(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu2(out) + + out = self.conv3(out) + out = self.bn3(out) + + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu3(out) + + return out + + +class MakeLayer0(Cell): + + def __init__(self, block, layer_num, in_channels, out_channels, stride): + super(MakeLayer0, self).__init__() + self.a = ResidualBlockWithDown(in_channels, out_channels, stride=1, down_sample=True) + self.b = block(out_channels, out_channels, stride=stride) + self.c = block(out_channels, out_channels, stride=1) + + def construct(self, x): + x = self.a(x) + x = self.b(x) + x = self.c(x) + + return x + + +class ResNet(Cell): + + def __init__(self, block, layer_num, num_classes=100): + super(ResNet, self).__init__() + self.conv1 = conv7x7(3, 64, stride=2, padding=3) + self.bn1 = bn_with_initialize(64) + self.relu = P.ReLU().set_strategy(strategy_no_weight) + self.maxpool = MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + self.layer1 = MakeLayer0( + block, layer_num[0], in_channels=64, out_channels=256, stride=1) + self.pool = M.ReduceMean(keep_dims=True).set_strategy(strategy_no_weight) + self.fc = fc_with_initialize(64 * block.expansion, num_classes) + self.flatten = Flatten() + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.layer1(x) + x = self.pool(x, (-2, -1)) + x = self.flatten(x) + x = self.fc(x) + return x + + +class ResNetModelParallel(Cell): + def __init__(self, block, layer_num, num_classes=100): + super(ResNetModelParallel, self).__init__() + self.relu = P.ReLU().set_strategy(((1, dev_num, 1, 1), )) + self.maxpool = MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + self.layer1 = MakeLayer0( + block, layer_num[0], in_channels=64, out_channels=256, stride=1) + self.pool = M.ReduceMean(keep_dims=True).set_strategy(strategy_no_weight) + self.fc = fc_with_initialize(64 * block.expansion, num_classes) + self.flatten = Flatten() + + def construct(self, x): + x = self.relu(x) + x = self.maxpool(x) + x = self.layer1(x) + x = self.pool(x, (-2, -1)) + x = self.flatten(x) + x = self.fc(x) + return x + + +def resnet_operator_net(num_classes): + return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes) + + +def resnet_model_parallel_net(num_classes): + return ResNetModelParallel(ResidualBlock, [3, 4, 6, 3], num_classes) + + +def test_resnet_operator_batch_parallel(): + num_classes = 1024 + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + rank_size = dev_num + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=dev_num, global_rank=0) + predict = Tensor(np.ones([batch_size, 3, 224, 224]), dtype=ms.float32) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + + dataset = DatasetLenet(predict, label, 2) + net = resnet_operator_net(num_classes) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((dev_num, 1), (dev_num, 1))) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=dev_num) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss, opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_resnet_model_parallel(): + num_classes = 1024 + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + rank_size = dev_num + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=dev_num, global_rank=0) + predict = Tensor(np.ones([batch_size, 64, 112, 112]), dtype=ms.float32) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + + dataset = DatasetLenet(predict, label, 2) + net = resnet_model_parallel_net(num_classes) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((dev_num, 1), (dev_num, 1))) + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), learning_rate, momentum) + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=dev_num) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss, opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +if __name__ == '__main__': + test_resnet_model_parallel() diff --git a/tests/ut/python/parallel/test_optimizer.py b/tests/ut/python/parallel/test_optimizer.py new file mode 100644 index 00000000000..fea31fc984e --- /dev/null +++ b/tests/ut/python/parallel/test_optimizer.py @@ -0,0 +1,63 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +import mindspore.context as context +from mindspore.common.api import _executor +from mindspore.nn import Dense +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn import Momentum +from mindspore import Tensor, Parameter +from mindspore.ops import operations as P +from mindspore.communication.management import init +from mindspore.train.parallel_utils import ParallelMode + + +class Net(nn.Cell): + def __init__(self, input_channel, out_channel): + super(Net, self).__init__() + weight_init1 = np.ones([64, 128]).astype(np.float32) + weight_init2 = np.ones([32, 64]).astype(np.float32) + self.weight1 = Parameter(Tensor(weight_init1), "loss_weight1", layerwise_parallel=True) + self.weight2 = Parameter(Tensor(weight_init2), "loss_weight2", layerwise_parallel=True) + self.fc = P.MatMul(transpose_b=True) + self.dense = Dense(input_channel, out_channel) + + def construct(self, x): + x = self.dense(x) + x = self.fc(x, self.weight1) + x = self.fc(x, self.weight2) + return x + + +def test_dense_gen_graph(): + context.set_context(mode=context.GRAPH_MODE) + init() + network = Net(512, 128) + + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), + learning_rate=0.1, + momentum=0.9) + network = WithLossCell(network, loss_fn) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.HYBRID_PARALLEL, mirror_mean=True, device_num=8) + network = TrainOneStepCell(network, optimizer) + + predict = Tensor(np.ones([64, 512]).astype(np.float32) * 0.01) + label = Tensor(np.zeros([64, 32]).astype(np.float32)) + + _executor.compile(network, predict, label) + diff --git a/tests/ut/python/parallel/test_optimizer_clone_weight.py b/tests/ut/python/parallel/test_optimizer_clone_weight.py new file mode 100644 index 00000000000..84bda61dc15 --- /dev/null +++ b/tests/ut/python/parallel/test_optimizer_clone_weight.py @@ -0,0 +1,104 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter +import mindspore as ms +from mindspore.common.api import _Executor +from mindspore.nn.optim import AdamWeightDecay +from mindspore.nn import TrainOneStepCell + + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy3) + self.network = network + + def construct(self, x, b): + predict = self.network(x) + return self.loss(predict, b)[0] + + +def test_optimizer_clone_weight(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + optimizer = AdamWeightDecay(net.trainable_params()) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _Executor().compile(train_net, x, b) + + +def test_optimizer_clone_weight2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + self.relu = P.ReLU().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.relu(out) + return out + + context.set_auto_parallel_context(device_num=4, global_rank=0) + + strategy1 = ((2, 1), (2, 1)) + strategy2 = ((4, 1), ) + strategy3 = ((4, 1), (4, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight) + + optimizer = AdamWeightDecay(net.trainable_params()) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = TrainOneStepCell(net_with_loss, optimizer) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _Executor().compile(train_net, x, b) diff --git a/tests/ut/python/parallel/test_parameter_init.py b/tests/ut/python/parallel/test_parameter_init.py new file mode 100644 index 00000000000..6bb14400142 --- /dev/null +++ b/tests/ut/python/parallel/test_parameter_init.py @@ -0,0 +1,58 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter +import mindspore as ms + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, b): + predict = self.network(x) + return self.loss(predict, b)[0] + + +def test_parameter_init(): + class Net(nn.Cell): + def __init__(self, strategy1, weight): + super().__init__() + self.weight = Parameter(weight, "w1") + self.matmul = P.MatMul(transpose_a=False, transpose_b=True).set_strategy(strategy1) + + def construct(self, x): + out = self.matmul(x, self.weight) + return out + + context.set_auto_parallel_context(device_num=2, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy1 = ((1, 1), (2, 1)) + context.set_context(mode=context.GRAPH_MODE) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([64, 32]), dtype=ms.float32) + + net = Net(strategy1, weight) + net(x,) + + +if __name__ == '__main__': + test_parameter_init() diff --git a/tests/ut/python/parallel/test_prelu.py b/tests/ut/python/parallel/test_prelu.py new file mode 100755 index 00000000000..c601045491d --- /dev/null +++ b/tests/ut/python/parallel/test_prelu.py @@ -0,0 +1,151 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y): + predict = self.network(x, y) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y): + return C.grad_all(self.network)(x, y) + + +def test_prelu_single_success1(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.prelu = P.PReLU() + + def construct(self, x, y): + out = self.prelu(x, y) + return out + + context.reset_auto_parallel_context() + net = GradWrap(NetWithLoss(Net())) + x = Tensor(np.random.rand(1, 33, 4, 4), ms.float32) + w = Tensor(np.random.rand(33), ms.float32) + _executor.compile(net, x, w) + +def test_prelu_single_success2(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.prelu = P.PReLU() + + def construct(self, x, y): + out = self.prelu(x, y) + return out + + context.reset_auto_parallel_context() + net = GradWrap(NetWithLoss(Net())) + x = Tensor(np.random.rand(1, 33, 4, 4), ms.float32) + w = Tensor([0.1], ms.float32) + _executor.compile(net, x, w) + +def test_prelu_parallel_success1(): + class Net(nn.Cell): + def __init__(self, strategy): + super().__init__() + self.prelu = P.PReLU().set_strategy(strategy) + def construct(self, x, y): + out = self.prelu(x, y) + return out + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=8, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy = ((1, 1, 1, 1), (1, )) + x = Tensor(np.random.rand(4, 4, 32, 64),dtype=ms.float32) + w = Tensor(np.random.rand(4),dtype=ms.float32) + net = GradWrap(NetWithLoss(Net(strategy))) + _executor.compile(net, x, w) + +def test_prelu_parallel_success2(): + class Net(nn.Cell): + def __init__(self, strategy): + super().__init__() + self.prelu = P.PReLU().set_strategy(strategy) + def construct(self, x, y): + out = self.prelu(x, y) + return out + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=64, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy = ((2, 1, 4, 8), (1, )) + x = Tensor(np.random.rand(4, 4, 32, 64),dtype=ms.float32) + w = Tensor(np.random.rand(4),dtype=ms.float32) + net = GradWrap(NetWithLoss(Net(strategy))) + _executor.compile(net, x, w) + +def test_prelu_parallel_success3(): + class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, w): + predict = self.network(x, y, w) + return self.loss(predict) + + + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, w): + return C.grad_all(self.network)(x, y, w) + + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul().set_strategy(strategy1) + self.prelu = P.PReLU().set_strategy(strategy2) + def construct(self, x, y, w): + out = self.matmul(x, y) + out = self.prelu(out, w) + return out + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=64, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy1 = ((2, 4), (4, 2)) + strategy2 = ((32, 1), (1, )) + x = Tensor(np.random.rand(128, 64),dtype=ms.float32) + y = Tensor(np.random.rand(64, 16),dtype=ms.float32) + w = Tensor(np.random.rand(16),dtype=ms.float32) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + _executor.compile(net, x, y, w) diff --git a/tests/ut/python/parallel/test_prelu_cell.py b/tests/ut/python/parallel/test_prelu_cell.py new file mode 100644 index 00000000000..a2ca303244c --- /dev/null +++ b/tests/ut/python/parallel/test_prelu_cell.py @@ -0,0 +1,117 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context +from mindspore.ops import functional as F +from mindspore.common.initializer import initializer +context.set_context(mode=context.GRAPH_MODE) + + +class Dataset(MindData): + def __init__(self, predict, label, length=3, input_num=2): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + self.input_num = input_num + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + if self.input_num == 2: + return self.predict, self.label + else: + return self.predict, + + def reset(self): + self.index = 0 + + +class PReLU(nn.Cell): + def __init__(self, channel=1, w=0.25): + super(PReLU, self).__init__() + if isinstance(w, (np.float32, float)): + tmp = np.empty((channel,), dtype=np.float32) + tmp.fill(w) + w = Tensor(tmp) + elif isinstance(w, list): + w = Tensor(w) + + if not isinstance(w, Tensor): + raise TypeError("w only support np.float32, float or Tensor type.") + + self.w = Parameter(initializer(w, [channel,]), name='a') + self.prelu = P.PReLU() + self.relu = P.ReLU().set_strategy(((1, ), )) + self.sub = P.Sub().set_strategy(((1, ), (1, ))) + self.assign_sub = P.AssignSub().set_strategy(((1, ), (1, ))) + + def construct(self, x): + u = self.relu(self.w) + tmp = self.sub(self.w, u) + x = F.depend(x, self.assign_sub(self.w, tmp)) + v = self.prelu(x, u) + return v + + +class PReLUNet(nn.Cell): + def __init__(self): + super(PReLUNet, self).__init__() + self.prelu = PReLU(channel=256) + + def construct(self, x): + x = self.prelu(x) + return x + + +def prelu_net(): + return PReLUNet() + + +def reshape_common(parallel_mode): + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + predict = Tensor(np.ones([32, 256]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = prelu_net() + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss, opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_prelu_cell(): + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL) diff --git a/tests/ut/python/parallel/test_reduce_method_info.py b/tests/ut/python/parallel/test_reduce_method_info.py new file mode 100644 index 00000000000..57a8e54fd27 --- /dev/null +++ b/tests/ut/python/parallel/test_reduce_method_info.py @@ -0,0 +1,551 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + +# model_parallel test +def test_sum_mul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, (1,)) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 1, 8), (1, 1, 8)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_sum_mul2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, (0, 1)) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 1, 4, 2), (1, 1, 4, 2)) + strategy2 = ((2, 4, 1, 1), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 128, 64, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 128, 64, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_sum_mul3(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, -1) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 2, 1), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 32]), dtype=ms.float32) + _executor.compile(net, x, y, b) + +def test_sum_mul4(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=True).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, -1) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((2, 2, 2), ) + strategy3 = ((4, 2, 1), (4, 2, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 32, 1]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_sum_mul5(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=True).set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, 0) + return out + + context.set_auto_parallel_context(device_num=64, global_rank=0) + strategy1 = ((1, 8, 8), (1, 8, 8)) + strategy2 = ((2, 4, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([1, 32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_sum_mul6(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=True).set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, 1) + return out + + context.set_auto_parallel_context(device_num=64, global_rank=0) + strategy1 = ((1, 8, 8), (1, 8, 8)) + strategy2 = ((2, 1, 4), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_sum_mul7(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=True).set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, (0, 1)) + return out + + context.set_auto_parallel_context(device_num=64, global_rank=0) + strategy1 = ((1, 8, 8), (1, 8, 8)) + strategy2 = ((2, 4, 1), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([1, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_max_mul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_max = P.ReduceMax(keep_dims=False).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_max(out, -1) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 32]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_min_mul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_min = P.ReduceMin(keep_dims=False).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_min(out, 0) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_reduce_mean_mul_float32(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_mean = P.ReduceMean(keep_dims=False).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_mean(out, 0) + out = self.mul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + + _executor.compile(net, x, y, b) + + +class ArgMaxWithValueNet(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.arg_max_with_value = P.ArgMaxWithValue(keep_dims=False, axis=-1).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + index, out = self.arg_max_with_value(out) + out = self.mul2(out, b) + return out + + +class ArgMinWithValueNet(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.arg_min_with_value = P.ArgMinWithValue(keep_dims=False, axis=-1).set_strategy(strategy2) + self.mul2 = P.Mul().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + index, out = self.arg_min_with_value(out) + out = self.mul2(out, b) + return out + + +def gen_inputs_and_compile(net): + x = Tensor(np.ones([128, 64, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 64, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def tobefixed_test_arg_max_with_value_mul_semi_axis_parallel(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(ArgMaxWithValueNet(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + gen_inputs_and_compile(net) + + +def test_arg_max_with_value_mul_semi(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 1), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(ArgMaxWithValueNet(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + gen_inputs_and_compile(net) + + +def test_arg_max_with_value_mul_auto(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = None + strategy2 = None + strategy3 = None + net = GradWrap(NetWithLoss(ArgMaxWithValueNet(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + gen_inputs_and_compile(net) + + +def test_arg_min_with_value_mul_semi_axis_parallel(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(ArgMinWithValueNet(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + gen_inputs_and_compile(net) + + + +def test_arg_min_with_value_mul_semi(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 1), ) + strategy3 = ((2, 4), (2, 4)) + net = GradWrap(NetWithLoss(ArgMinWithValueNet(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + gen_inputs_and_compile(net) + + +def test_arg_min_with_value_mul_auto(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = None + strategy2 = None + strategy3 = None + net = GradWrap(NetWithLoss(ArgMinWithValueNet(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + gen_inputs_and_compile(net) + + +class ArgMinWithValueNet2(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.arg_min_with_value = P.ArgMinWithValue(keep_dims=True, axis=-1).set_strategy(strategy2) + self.relu = P.ReLU().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul1(x, y) + index, out = self.arg_min_with_value(out) + out = self.relu(out) + return out + + +def tobefixed_test_arg_min_with_value_mul_semi_axis_parallel2(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((2, 4, 1), ) + net = GradWrap(NetWithLoss(ArgMinWithValueNet2(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + gen_inputs_and_compile(net) + + +def test_arg_min_with_value_mul_semi2(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 1), ) + strategy3 = ((2, 4, 1), ) + net = GradWrap(NetWithLoss(ArgMinWithValueNet2(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + gen_inputs_and_compile(net) + + +def test_arg_min_with_value_mul_auto2(): + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = None + strategy2 = None + strategy3 = None + net = GradWrap(NetWithLoss(ArgMinWithValueNet2(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + gen_inputs_and_compile(net) + + +def test_cross_batch(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy2) + self.reduce_mean = P.ReduceMean(keep_dims=False).set_strategy(strategy3).add_prim_attr("cross_batch", True) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_sum(out, -1) + out = self.reduce_mean(out, 0) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 2), (4, 2)) + strategy2 = ((2, 1), ) + strategy3 = ((8, ), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([32, 64]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_cross_batch2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul1 = P.Mul().set_strategy(strategy1) + self.reduce_mean = P.ReduceMean(keep_dims=False).set_strategy(strategy2) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy3).add_prim_attr("cross_batch", True) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_mean(out, -1) + out = self.reduce_sum(out, 0) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 2), (4, 2)) + strategy2 = ((2, 1), ) + strategy3 = ((8, ), ) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + + x = Tensor(np.ones([32, 64]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_cross_batch_auto(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.mul1 = P.Mul() + self.reduce_mean = P.ReduceMean(keep_dims=False) + self.reduce_sum = P.ReduceSum(keep_dims=False).add_prim_attr("cross_batch", True) + + def construct(self, x, y, b): + out = self.mul1(x, y) + out = self.reduce_mean(out, -1) + out = self.reduce_sum(out, 0) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + + + x = Tensor(np.ones([32, 64]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([32, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_max_empty_tuple(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3): + super().__init__() + self.mul = P.Mul().set_strategy(strategy1) + self.reduce_max = P.ReduceMax(keep_dims=False).set_strategy(strategy2) + self.add = P.TensorAdd().set_strategy(strategy3) + + def construct(self, x, y, b): + out = self.mul(x, y) + out = self.reduce_max(out) + out = self.add(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((1, 4, 2), (1, 4, 2)) + strategy2 = ((4, 1, 2), ) + strategy3 = ((), (1, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + y = Tensor(np.ones([128, 32, 64]), dtype=ms.float32) + b = Tensor(np.ones([128, 32]), dtype=ms.float32) + + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_reshape.py b/tests/ut/python/parallel/test_reshape.py new file mode 100644 index 00000000000..11ca435e5b9 --- /dev/null +++ b/tests/ut/python/parallel/test_reshape.py @@ -0,0 +1,658 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops.operations.comm_ops import _VirtualDataset +from mindspore.ops import functional as F +from mindspore.common.parameter import ParameterTuple +from mindspore.common import dtype as mstype +from mindspore.parallel import set_algo_parameters +context.set_context(mode=context.GRAPH_MODE) +context.reset_auto_parallel_context() + +class Dataset(MindData): + def __init__(self, predict, label, length=3, input_num=2): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + self.input_num = input_num + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + if self.input_num == 2: + return self.predict, self.label + else: + return self.predict, + + def reset(self): + self.index = 0 + + +class ReshapeNet(nn.Cell): + def __init__(self, strategy0, strategy1, strategy2): + super(ReshapeNet, self).__init__() + self.relu = P.ReLU().set_strategy(strategy0) + self.reshape = P.Reshape().set_strategy(strategy1) + self.matmul = P.MatMul().set_strategy(strategy2) + self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + + def construct(self, x): + x = self.relu(x) + x = self.reshape(x, (256, 25088)) + x = self.matmul(x, self.matmul_weight) + return x + + +def reshape_net(strategy0, strategy1, strategy2): + return ReshapeNet(strategy0=strategy0, strategy1=strategy1, strategy2=strategy2) + + +def reshape_common(parallel_mode, strategy0, strategy1, strategy2, strategy_loss): + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + predict = Tensor(np.ones([32, 512, 7, 7]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = reshape_net(strategy0, strategy1, strategy2) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(strategy_loss) + loss.one_hot.set_strategy(((8,1), (), ())) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss, opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_reshape1(): + strategy0 = ((8, 1, 1, 1), ) + strategy1 = None + strategy2 = ((8, 1), (1, 1)) + strategy_loss = ((8, 1), (8, 1)) + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + + +def test_reshape1_strategy_1(): + strategy0 = ((8, 1, 1, 1), ) + strategy1 = ((8, 1, 1, 1), ) + strategy2 = ((8, 1), (1, 1)) + strategy_loss = ((8, 1), (8, 1)) + try: + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + except: + pass + + +def test_reshape1_strategy_2(): + strategy0 = ((8, 1, 1, 1), ) + strategy1 = ((8, 1, 1, 1), ) + strategy2 = ((8, 1), (1, 1)) + strategy_loss = ((8, 1), (8, 1)) + try: + reshape_common(ParallelMode.AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + except: + pass + + +def test_reshape2(): + strategy0 = ((8, 1, 1, 1), ) + strategy1 = None + strategy2 = ((8, 1), (1, 1)) + strategy_loss = ((8, 1), (8, 1)) + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + + +def test_reshape3(): + strategy0 = ((2, 1, 1, 1), ) + strategy1 = None + strategy2 = ((8, 1), (1, 1)) + strategy_loss = ((8, 1), (8, 1)) + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + + +def test_reshape4(): + strategy0 = ((1, 1, 1, 1), ) + strategy1 = None + strategy2 = ((8, 1), (1, 1)) + strategy_loss = ((8, 1), (8, 1)) + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + + +def test_reshape5(): + strategy0 = ((2, 1, 1, 1), ) + strategy1 = None + strategy2 = ((1, 8), (8, 1)) + strategy_loss = ((8, 1), (8, 1)) + reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + + +def test_reshape_auto(): + strategy0 = None + strategy1 = None + strategy2 = None + strategy_loss = None + reshape_common(ParallelMode.AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss) + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x): + predict = self.network(x) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x): + return C.grad_all(self.network)(x) + + +class ReshapeNet1(nn.Cell): + def __init__(self, strategy0): + super(ReshapeNet1, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.reshape = P.Reshape() + self.matmul = P.MatMul().set_strategy(strategy0) + self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + self.reshape2 = P.Reshape() + + def construct(self, x): + x = self.virtual_dataset(x) + x = self.reshape(x, (256, 25088)) + x = self.matmul(x, self.matmul_weight) + x = self.reshape2(x, (256 * 256,)) + return x + + +class ReshapeNet2(nn.Cell): + def __init__(self, strategy0): + super(ReshapeNet2, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.reshape = P.Reshape() + self.matmul = P.MatMul().set_strategy(strategy0) + self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + self.reshape2 = P.Reshape() + self.reduce_sum = P.ReduceSum(keep_dims=True) + self.reshape3 = P.Reshape() + + def construct(self, x): + x = self.virtual_dataset(x) + x = self.reshape(x, (256, 25088)) + x = self.matmul(x, self.matmul_weight) + x = self.reshape2(x, (256 * 256,)) + x = self.reduce_sum(x, -1) + x = self.reshape3(x, ()) + return x + + +class ReshapeNet3(nn.Cell): + def __init__(self, strategy0): + super(ReshapeNet3, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.reshape = P.Reshape() + self.matmul = P.MatMul().set_strategy(strategy0) + self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + self.reshape2 = P.Reshape() + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.reshape3 = P.Reshape() + + def construct(self, x): + x = self.virtual_dataset(x) + x = self.reshape(x, (256, 25088)) + x = self.matmul(x, self.matmul_weight) + x = self.reshape2(x, (256 * 256,)) + x = self.reduce_sum(x, -1) + x = self.reshape3(x, (1, 1)) + return x + + +class ReshapeNet4(nn.Cell): + def __init__(self, strategy0): + super(ReshapeNet4, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.reshape = P.Reshape() + self.reshape2 = P.Reshape() + self.matmul = P.MatMul().set_strategy(strategy0) + self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + + def construct(self, x): + x = self.virtual_dataset(x) + x = self.reshape(x, (256, 25088)) + w = self.reshape2(self.matmul_weight, (25088, 256)) + x = self.matmul(x, w) + return x + + +class ReshapeNet5(nn.Cell): + def __init__(self, strategy0): + super(ReshapeNet5, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.reshape = P.Reshape() + self.matmul1 = P.MatMul().set_strategy(strategy0) + self.matmul1_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + self.matmul2 = P.MatMul().set_strategy(strategy0) + + def construct(self, x): + x = self.virtual_dataset(x) + x = self.reshape(x, (256, 25088)) + matmul1_o = self.matmul1(x, self.matmul1_weight) + matmul2_o = self.matmul2(matmul1_o, x) + return matmul2_o + + +class ReshapeNet6(nn.Cell): + def __init__(self, strategy0): + super(ReshapeNet6, self).__init__() + self.virtual_dataset = _VirtualDataset() + self.reshape = P.Reshape() + self.matmul1_1 = P.MatMul().set_strategy(strategy0) + self.matmul1_2 = P.MatMul().set_strategy(strategy0) + self.matmul1_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight") + self.matmul2 = P.MatMul().set_strategy(strategy0) + self.add = P.TensorAdd() + + def construct(self, x): + x = self.virtual_dataset(x) + x = self.reshape(x, (256, 25088)) + matmul1_1_o = self.matmul1_1(x, self.matmul1_weight) + matmul1_2_o = self.matmul1_2(x, self.matmul1_weight) + matmul1_o = self.add(matmul1_1_o, matmul1_2_o) + matmul2_o = self.matmul2(matmul1_o, x) + return matmul2_o + + +def reshape_net2(backbone): + batch_size = 16 + device_num = 16 + context.set_auto_parallel_context(device_num=device_num, global_rank=0) + input = Tensor(np.ones([batch_size * device_num, 512, 7, 7]).astype(np.float32) * 0.01) + + net = GradWrap(NetWithLoss(backbone)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(net, input) + + +def test_reshape_net1_1(): + reshape_net2(ReshapeNet1(((1, 8), (8, 1)))) + + +def test_reshape_net1_2(): + reshape_net2(ReshapeNet1(((1, 8), (8, 2)))) + + +def test_reshape_net2_1(): + reshape_net2(ReshapeNet2(((1, 8), (8, 1)))) + + +def test_reshape_net2_2(): + reshape_net2(ReshapeNet2(((1, 8), (8, 2)))) + + +def test_reshape_net3_1(): + reshape_net2(ReshapeNet3(((1, 8), (8, 1)))) + + +def test_reshape_net3_2(): + reshape_net2(ReshapeNet3(((1, 8), (8, 2)))) + + +def test_reshape_net4_1(): + try: + reshape_net2(ReshapeNet4(((1, 8), (8, 1)))) + except: + pass + + +def test_reshape_net4_2(): + try: + reshape_net2(ReshapeNet4(((1, 8), (8, 2)))) + except: + pass + + +def test_reshape_net5_1(): + reshape_net2(ReshapeNet5(((1, 8), (8, 1)))) + + +def test_reshape_net5_2(): + reshape_net2(ReshapeNet5(((1, 8), (8, 2)))) + + +def test_reshape_net6_1(): + reshape_net2(ReshapeNet6(((1, 8), (8, 1)))) + + +def test_reshape_net6_2(): + reshape_net2(ReshapeNet6(((1, 8), (8, 2)))) + + +class TrainOneStepCell(nn.Cell): + """ + Network training package class. + + Append an optimizer to the training network after that the construct function + can be called to create the backward graph. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (Number): The adjust parameter. Default: 1.0. + + Examples: + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> loss_net = WithLossCell(net, loss_fn) + >>> train_net = TrainOneStepCell(loss_net, optim) + """ + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + self.sens = sens + + def construct(self, data): + weights = self.weights + loss = self.network(data) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(data, sens) + + return F.depend(loss, self.optimizer(grads)) + + +def reshape_common2(parallel_mode, net): + batch_size = 16 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + predict = Tensor(np.ones([batch_size, 512, 7, 7]), dtype=ms.float32) + label = Tensor(np.ones([batch_size]), dtype=ms.int32) + dataset = Dataset(predict, label, 2, input_num=1) + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=16) + + opt = Momentum(net.trainable_params(), learning_rate, momentum) + train_net = TrainOneStepCell(net, opt).set_train() + model = Model(train_net) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_reshape_common2_0(): + reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet1(((1, 8), (8, 1)))) + + +def test_reshape_common2_1(): + reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet1(((1, 8), (8, 2)))) + + +def test_reshape_common2_2(): + reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet2(((1, 8), (8, 1)))) + + +def test_reshape_common2_3(): + reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet2(((1, 8), (8, 2)))) + + +def test_reshape_common2_4(): + reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet3(((1, 8), (8, 1)))) + + +def test_reshape_common2_5(): + reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet3(((1, 8), (8, 2)))) + + +class BatchNormReshapeNet(nn.Cell): + def __init__(self): + super(BatchNormReshapeNet, self).__init__() + self.vd = P._VirtualDataset() + self.batch_norm = nn.BatchNorm1d(512, affine=False) + self.reshape = P.Reshape() + self.prelu = nn.PReLU(channel=256) + + def construct(self, x): + x = self.vd(x) + x = self.batch_norm(x) + x = self.reshape(x, (512, 256)) + x = self.prelu(x) + return x + + +def test_batchnorm_reshape_train(): + batch_size = 16 + device_num = 16 + context.set_auto_parallel_context(device_num=device_num, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + input = Tensor(np.ones([batch_size * device_num, 512]).astype(np.float32) * 0.01) + + net = GradWrap(NetWithLoss(BatchNormReshapeNet())) + + _executor.compile(net, input) + + +def bn_with_initialize(out_channels): + bn = nn.BatchNorm2d(out_channels, momentum=0.3, eps=1e-5).add_flags_recursive(fp32=True) + return bn + + +def fc_with_initialize(input_channels, out_channels): + return nn.Dense(input_channels, out_channels).add_flags_recursive(fp16=True) + + +class BNReshapeDenseBNNet(nn.Cell): + def __init__(self): + super(BNReshapeDenseBNNet, self).__init__() + self.batch_norm = bn_with_initialize(512) + self.reshape = P.Reshape() + self.cast = P.Cast() + self.batch_norm2 = nn.BatchNorm1d(512, affine=False) + self.fc = fc_with_initialize(512 * 32 * 32, 512) + + def construct(self, x): + x = self.batch_norm(x) + x = self.reshape(x, (16, 512*32*32)) + x = self.fc(x) + x = self.batch_norm2(x) + return x + + +def test_bn_reshape_dense_bn_train(): + batch_size = 16 + device_num = 16 + context.set_auto_parallel_context(device_num=device_num, global_rank=0) + input = Tensor(np.ones([batch_size, 512, 32, 32]).astype(np.float32) * 0.01) + + net = GradWrap(NetWithLoss(BNReshapeDenseBNNet())) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(net, input) + + +class ParallelReduceMeanNet(nn.Cell): + def __init__(self, conv_in_channel, conv_out_channel, + reducemean_keep_dims=False, reducemean_axis=-1, strategy=None): + super().__init__() + self.conv = nn.Conv2d(in_channels=conv_in_channel, out_channels=conv_out_channel, + kernel_size=1, stride=1, pad_mode='valid', has_bias=True, + weight_init='ones', bias_init='ones') + self.reduce_mean = P.ReduceMean(keep_dims=reducemean_keep_dims) + self.flat = nn.Flatten() + self.reducemean_axis = reducemean_axis + if strategy is not None: + self.reduce_mean.set_strategy(strategy) + + def construct(self, inputs): + x = self.conv(inputs) + x = self.reduce_mean(x, self.reducemean_axis) + x = self.flat(x) + return x + + +class CrossEntropyLoss(nn.Cell): + def __init__(self, reduction='mean'): + super(CrossEntropyLoss, self).__init__() + + self.reduce_mean = P.ReduceMean() + self.cross_entropy = SoftmaxCrossEntropyWithLogits() + self.reduction = reduction + + def construct(self, logits, label): + loss = self.cross_entropy(logits, label) + if self.reduction == 'mean': + loss = self.reduce_mean(loss, (-1,)) + return loss + + +def test_flatten_reshape(parallel_mode="auto_parallel"): + batch_size = 16 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + net = ParallelReduceMeanNet(conv_in_channel=3, conv_out_channel=64, reducemean_axis=(2, 3), strategy=((4, 2, 1, 1),)) + loss = CrossEntropyLoss() + predict = Tensor(np.ones([batch_size, 3, 32, 32]), dtype=ms.float32) + label = Tensor(np.ones([batch_size, 64]), dtype=ms.float32) + dataset = Dataset(predict, label, 2, input_num=2) + + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss_fn = loss, optimizer=opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_flatten_reshape2(parallel_mode="auto_parallel"): + batch_size = 16 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + set_algo_parameters(not_fully_use_devices=True) + net = ParallelReduceMeanNet(conv_in_channel=3, conv_out_channel=64, reducemean_axis=(2, 3), strategy=((4, 1, 1, 1),)) + loss = CrossEntropyLoss() + predict = Tensor(np.ones([batch_size, 3, 32, 32]), dtype=ms.float32) + label = Tensor(np.ones([batch_size, 64]), dtype=ms.float32) + dataset = Dataset(predict, label, 2, input_num=2) + + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss_fn = loss, optimizer=opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +class ParallelReshapeNet(nn.Cell): + def __init__(self, dense_in_channel, dense_out_channel, shape, strategy=None): + super().__init__() + self.flat = nn.Flatten() + self.dense = nn.Dense(in_channels=dense_in_channel, + out_channels=dense_out_channel, + weight_init='ones', + bias_init='ones', + has_bias=True) + self.reshape = P.Reshape() + self.shape = shape + self.reshape.set_strategy(strategy) + + def construct(self, inputs): + x = self.flat(inputs) + x = self.dense(x) + x = self.reshape(x, self.shape) + return x + + +# the shape of input and output of reshape is the same +# reshape is optimized before step_parallel +def test_flatten_reshape3(parallel_mode="auto_parallel"): + batch_size = 16 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + set_algo_parameters(not_fully_use_devices=True) + net = ParallelReshapeNet(dense_in_channel=2048, dense_out_channel=1000, shape=(128, 1000), strategy=((16, 1),)) + loss = CrossEntropyLoss() + predict = Tensor(np.ones([batch_size, 1, 2, 1024]), dtype=ms.float32) + label = Tensor(np.ones([batch_size, 1000]), dtype=ms.float32) + dataset = Dataset(predict, label, 2, input_num=2) + + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss_fn = loss, optimizer=opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +class CrossEntropyLoss2(nn.Cell): + def __init__(self, reduction='mean'): + super(CrossEntropyLoss2, self).__init__() + self.cross_entropy = SoftmaxCrossEntropyWithLogits(reduction=reduction) + + def construct(self, logits, label): + loss = self.cross_entropy(logits, label) + return loss + + +def test_flatten_reshape4(parallel_mode="semi_auto_parallel"): + batch_size = 16 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8) + set_algo_parameters(not_fully_use_devices=True) + net = ParallelReduceMeanNet(conv_in_channel=3, conv_out_channel=64, reducemean_keep_dims=True, strategy=((4, 1, 1, 1),)) + loss = CrossEntropyLoss2() + predict = Tensor(np.ones([batch_size, 3, 32, 32]), dtype=ms.float32) + label = Tensor(np.ones([batch_size, 2048]), dtype=ms.float32) + dataset = Dataset(predict, label, 2, input_num=2) + + opt = Momentum(net.trainable_params(), learning_rate, momentum) + model = Model(net, loss_fn=loss, optimizer=opt) + model.train(epoch_size, dataset, dataset_sink_mode=False) diff --git a/tests/ut/python/parallel/test_scalar_loss.py b/tests/ut/python/parallel/test_scalar_loss.py new file mode 100644 index 00000000000..b52d08cba19 --- /dev/null +++ b/tests/ut/python/parallel/test_scalar_loss.py @@ -0,0 +1,58 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops import functional as F + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, bias): + return C.grad_all(self.network)(x, y, bias) + +def test_sum_as_loss(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + self.mul = P.Mul().set_strategy(strategy=((), ())) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0,1)) + out = self.mul(out, F.scalar_to_array(2.0)) + return out + + context.set_auto_parallel_context(device_num=16, global_rank=0) + + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((4, 1), ) + net = GradWrap(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + bias = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, bias) diff --git a/tests/ut/python/parallel/test_set_auto_parallel_context.py b/tests/ut/python/parallel/test_set_auto_parallel_context.py new file mode 100644 index 00000000000..301bb608e03 --- /dev/null +++ b/tests/ut/python/parallel/test_set_auto_parallel_context.py @@ -0,0 +1,100 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore import context +from mindspore.parallel import set_algo_parameters + +def test_set_auto_parallel_context(): + context.set_auto_parallel_context(device_num=4, global_rank=3, mirror_mean=True, cast_before_mirror=False, + parallel_mode="auto_parallel", parameter_broadcast=False) + device_num = context.get_auto_parallel_context("device_num") + global_rank = context.get_auto_parallel_context("global_rank") + mirror_mean = context.get_auto_parallel_context("mirror_mean") + cast_before_mirror = context.get_auto_parallel_context("cast_before_mirror") + parallel_mode = context.get_auto_parallel_context("parallel_mode") + parameter_broadcast = context.get_auto_parallel_context("parameter_broadcast") + assert device_num == 4 + assert global_rank == 3 + assert mirror_mean == True + assert cast_before_mirror == False + assert parallel_mode == "auto_parallel" + assert parameter_broadcast == False + + auto_parallel_context().set_communication_backend("hccl") + backend = auto_parallel_context().get_communication_backend() + assert backend == "hccl" + + auto_parallel_context().set_device_num(4) + device_num = auto_parallel_context().get_device_num() + device_num_is_set = auto_parallel_context().get_device_num_is_set() + assert device_num == 4 + assert device_num_is_set == True + + auto_parallel_context().set_global_rank(4) + global_rank = auto_parallel_context().get_global_rank() + assert global_rank == 4 + + auto_parallel_context().set_mirror_mean(True) + mirror_mean = auto_parallel_context().get_mirror_mean() + assert mirror_mean == True + + auto_parallel_context().set_cast_before_mirror(False) + cast_before_mirror = auto_parallel_context().get_cast_before_mirror() + assert cast_before_mirror == False + + parameter_broadcast_is_set = auto_parallel_context().get_parameter_broadcast_is_set() + assert parameter_broadcast_is_set == True + + with pytest.raises(ValueError): + context.set_auto_parallel_context(device_num=0) + + with pytest.raises(ValueError): + context.set_auto_parallel_context(device_num=4097) + + with pytest.raises(ValueError): + context.set_auto_parallel_context(global_rank=-1) + + with pytest.raises(ValueError): + context.set_auto_parallel_context(parallel_mode="wrong_mode") + + with pytest.raises(ValueError): + context.set_auto_parallel_context(global_rank=4096) + + with pytest.raises(ValueError): + set_algo_parameters(tensor_slice_align_size=0) + + with pytest.raises(ValueError): + set_algo_parameters(tensor_slice_align_size=1025) + + +def test_reset_auto_parallel_context(): + context.reset_auto_parallel_context() + device_num = context.get_auto_parallel_context("device_num") + global_rank = context.get_auto_parallel_context("global_rank") + mirror_mean = context.get_auto_parallel_context("mirror_mean") + cast_before_mirror = context.get_auto_parallel_context("cast_before_mirror") + parallel_mode = context.get_auto_parallel_context("parallel_mode") + parameter_broadcast = context.get_auto_parallel_context("parameter_broadcast") + device_num_is_set = auto_parallel_context().get_device_num_is_set() + parameter_broadcast_is_set = auto_parallel_context().get_parameter_broadcast_is_set() + assert device_num == 1 + assert global_rank == 0 + assert mirror_mean == False + assert cast_before_mirror == True + assert parallel_mode == "stand_alone" + assert parameter_broadcast == False + assert device_num_is_set == False + assert parameter_broadcast_is_set == False diff --git a/tests/ut/python/parallel/test_softmax_cross_entropy_expand.py b/tests/ut/python/parallel/test_softmax_cross_entropy_expand.py new file mode 100644 index 00000000000..c4b106edf5f --- /dev/null +++ b/tests/ut/python/parallel/test_softmax_cross_entropy_expand.py @@ -0,0 +1,29 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.nn.loss.loss import SoftmaxCrossEntropyExpand +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.common.api import _executor +from mindspore import context +import numpy as np + +def test_SoftmaxCrossEntropy(): + net = SoftmaxCrossEntropyExpand(sparse=True) + context.set_auto_parallel_context(parallel_mode="auto_parallel") + logit = Tensor(np.ones([64, 512]), dtype=mstype.float32) + label = Tensor(np.ones([64]), dtype=mstype.int32) + context.set_auto_parallel_context(device_num=8, global_rank=0) + + _executor.compile(net, logit, label) diff --git a/tests/ut/python/parallel/test_softmax_cross_entropy_loss.py b/tests/ut/python/parallel/test_softmax_cross_entropy_loss.py new file mode 100644 index 00000000000..ecfe00bd821 --- /dev/null +++ b/tests/ut/python/parallel/test_softmax_cross_entropy_loss.py @@ -0,0 +1,114 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3=None): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy3) + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y) + return self.loss(predict, b)[0] + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +def test_softmax_cross_entropy_loss(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul(transpose_b=True).set_strategy(strategy1) + self.gelu = P.Gelu().set_strategy(strategy2) + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.gelu(out) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 1), (2, 1)) + strategy2 = ((4, 2), ) + strategy3 = ((8, 1), (8, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2), strategy3)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_softmax_cross_entropy_loss_repeated_calculation(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul = P.MatMul(transpose_b=True).set_strategy(strategy1) + self.gelu = P.Gelu().set_strategy(strategy2) + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.gelu(out) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 1), (2, 1)) + strategy2 = ((4, 2), ) + strategy3 = ((2, 1), (2, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2), strategy3)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_softmax_cross_entropy_loss_auto_batch_parallel(): + class Net(nn.Cell): + def __init__(self): + super().__init__() + self.matmul = P.MatMul(transpose_b=True) + self.gelu = P.Gelu() + + def construct(self, x, y): + out = self.matmul(x, y) + out = self.gelu(out) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + net = GradWrap(NetWithLoss(Net())) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_split_grad_sens.py b/tests/ut/python/parallel/test_split_grad_sens.py new file mode 100644 index 00000000000..5e2d052ed53 --- /dev/null +++ b/tests/ut/python/parallel/test_split_grad_sens.py @@ -0,0 +1,155 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +import mindspore.common.dtype as mstype + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b, sens): + return C.grad_all_with_sens(self.network)(x, y, b, sens) + + +class GradWrap2(nn.Cell): + def __init__(self, network): + super(GradWrap2, self).__init__() + self.network = network + + def construct(self, x, y, b): + loss = self.network(x, y, b) + sens = P.Fill()(mstype.float32, P.Shape()(loss), 1.0) + return C.grad_all_with_sens(self.network)(x, y, b, sens) + + +class GradWrap3(nn.Cell): + def __init__(self, network): + super(GradWrap3, self).__init__() + self.network = network + + def construct(self, x, y, bias): + return C.grad_all(self.network)(x, y, bias) + + +def test_no_grad(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + + strategy1 = ((4, 2), (2, 1)) + strategy2 = ((2, 4), (4, 1)) + net = Net(strategy1, strategy2) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_grad_sens_parameter_type(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy1 = ((4, 2), (2, 1)) + strategy2 = ((2, 4), (4, 1)) + net = GradWrap(Net(strategy1, strategy2)) + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + sens = Tensor(np.ones([128, 64]), dtype=ms.float32) + # net(x, y, b, sens) + _executor.compile(net, x, y, b, sens) + + +def test_grad_sens_tensor_type(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0) + + strategy1 = ((4, 2), (2, 1)) + strategy2 = ((2, 4), (4, 1)) + net = GradWrap2(Net(strategy1, strategy2)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_grad_sens_scalar_broadcast(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0,1)) + return out + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((4, 1), ) + net = GradWrap3(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + bias = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, bias) diff --git a/tests/ut/python/parallel/test_step_parallel.py b/tests/ut/python/parallel/test_step_parallel.py new file mode 100644 index 00000000000..140e34a0d10 --- /dev/null +++ b/tests/ut/python/parallel/test_step_parallel.py @@ -0,0 +1,75 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore as ms +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b, a): + predict = self.network(x, y, b, a) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b, a): + return C.grad_all(self.network)(x, y, b, a) + + +def test_two_matmul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3, strategy4): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.matmul4 = P.MatMul().set_strategy(strategy4) + + def construct(self, x, y, b, a): + out = self.matmul1(x, y) + out1 = self.matmul2(out, b) + out2 = self.matmul3(out, a) + out3 = self.matmul4(out1, out2) + return out3 + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((1, 8), (8, 1)) + strategy3 = ((4, 1), (1, 2)) + strategy4 = ((4, 2), (2, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3, strategy4))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 128]), dtype=ms.float32) + b = Tensor(np.ones([128, 128]), dtype=ms.float32) + a = Tensor(np.ones([128, 128]), dtype=ms.float32) + + _executor.compile(net, x, y, b, a) diff --git a/tests/ut/python/parallel/test_strategy_checkpoint.py b/tests/ut/python/parallel/test_strategy_checkpoint.py new file mode 100644 index 00000000000..09f4a54cbf9 --- /dev/null +++ b/tests/ut/python/parallel/test_strategy_checkpoint.py @@ -0,0 +1,281 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +from mindspore.context import set_auto_parallel_context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +# model_parallel test +# export PARALLEL_CHECKPOINT_ON=on +# export PARALLEL_TRAIN_TIMES=4 +def test_six_matmul(): + class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x1, x2, x3, x4, x5, x6, x7): + predict = self.network(x1, x2, x3, x4, x5, x6, x7) + return self.loss(predict) + + + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x1, x2, x3, x4, x5, x6, x7): + return C.grad_all(self.network)(x1, x2, x3, x4, x5, x6, x7) + + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, strategy3, strategy4, strategy5, strategy6): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.matmul4 = P.MatMul().set_strategy(strategy4) + self.matmul5 = P.MatMul().set_strategy(strategy5) + self.matmul6 = P.MatMul().set_strategy(strategy6) + + def construct(self, x1, x2, x3, x4, x5, x6, x7): + out = self.matmul1(x1, x2) + out = self.matmul2(out, x3) + out = self.matmul3(out, x4) + out = self.matmul4(out, x5) + out = self.matmul5(out, x6) + out = self.matmul6(out, x7) + return out + + set_auto_parallel_context(device_num=512, global_rank=0) + strategy1 = ((8, 1), (1, 1)) + strategy2 = ((1, 8), (8, 1)) + strategy3 = ((2, 2), (2, 2)) + strategy4 = ((4, 2), (2, 4)) + strategy5 = ((2, 4), (4, 2)) + strategy6 = ((4, 4), (4, 4)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3, strategy4, strategy5, strategy6))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x1 = Tensor(np.ones([128, 32]), dtype=ms.float32) + x2 = Tensor(np.ones([32, 64]), dtype=ms.float32) + x3 = Tensor(np.ones([64, 64]), dtype=ms.float32) + x4 = Tensor(np.ones([64, 128]), dtype=ms.float32) + x5 = Tensor(np.ones([128, 64]), dtype=ms.float32) + x6 = Tensor(np.ones([64, 32]), dtype=ms.float32) + x7 = Tensor(np.ones([32, 32]), dtype=ms.float32) + _executor.compile(net, x1, x2, x3, x4, x5, x6, x7) + +# remove matmul2 +def test_six_matmul_repeated1(): + class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x1, x2, x4, x5, x6, x7): + predict = self.network(x1, x2, x4, x5, x6, x7) + return self.loss(predict) + + + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x1, x2, x4, x5, x6, x7): + return C.grad_all(self.network)(x1, x2, x4, x5, x6, x7) + + class Net(nn.Cell): + def __init__(self, strategy1, strategy3, strategy4, strategy5, strategy6): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.matmul4 = P.MatMul().set_strategy(strategy4) + self.matmul5 = P.MatMul().set_strategy(strategy5) + self.matmul6 = P.MatMul().set_strategy(strategy6) + + def construct(self, x1, x2, x4, x5, x6, x7): + out = self.matmul1(x1, x2) + out = self.matmul3(out, x4) + out = self.matmul4(out, x5) + out = self.matmul5(out, x6) + out = self.matmul6(out, x7) + return out + + set_auto_parallel_context(device_num=512, global_rank=0) + strategy1 = ((8, 1), (1, 1)) + strategy3 = ((8, 1), (1, 1)) + strategy4 = ((8, 1), (1, 1)) + strategy5 = ((8, 1), (1, 1)) + strategy6 = ((8, 1), (1, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy3, strategy4, strategy5, strategy6))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x1 = Tensor(np.ones([128, 32]), dtype=ms.float32) + x2 = Tensor(np.ones([32, 64]), dtype=ms.float32) + x4 = Tensor(np.ones([64, 128]), dtype=ms.float32) + x5 = Tensor(np.ones([128, 64]), dtype=ms.float32) + x6 = Tensor(np.ones([64, 32]), dtype=ms.float32) + x7 = Tensor(np.ones([32, 32]), dtype=ms.float32) + _executor.compile(net, x1, x2, x4, x5, x6, x7) + +# add matmul7 +def test_six_matmul_repeated2(): + class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x1, x2, x4, x5, x6, x7, x8): + predict = self.network(x1, x2, x4, x5, x6, x7, x8) + return self.loss(predict) + + + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x1, x2, x4, x5, x6, x7, x8): + return C.grad_all(self.network)(x1, x2, x4, x5, x6, x7, x8) + + class Net(nn.Cell): + def __init__(self, strategy1, strategy3, strategy4, strategy5, strategy6, strategy7): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.matmul4 = P.MatMul().set_strategy(strategy4) + self.matmul5 = P.MatMul().set_strategy(strategy5) + self.matmul6 = P.MatMul().set_strategy(strategy6) + self.matmul7 = P.MatMul().set_strategy(strategy7) + + def construct(self, x1, x2, x4, x5, x6, x7, x8): + out = self.matmul1(x1, x2) + out = self.matmul3(out, x4) + out = self.matmul4(out, x5) + out = self.matmul5(out, x6) + out = self.matmul6(out, x7) + out = self.matmul7(out, x8) + return out + + set_auto_parallel_context(device_num=512, global_rank=0) + strategy1 = ((8, 1), (1, 1)) + strategy3 = ((8, 1), (1, 1)) + strategy4 = ((8, 1), (1, 1)) + strategy5 = ((8, 1), (1, 1)) + strategy6 = ((8, 1), (1, 1)) + strategy7 = ((8, 1), (1, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy3, strategy4, strategy5, strategy6, strategy7))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x1 = Tensor(np.ones([128, 32]), dtype=ms.float32) + x2 = Tensor(np.ones([32, 64]), dtype=ms.float32) + x4 = Tensor(np.ones([64, 128]), dtype=ms.float32) + x5 = Tensor(np.ones([128, 64]), dtype=ms.float32) + x6 = Tensor(np.ones([64, 32]), dtype=ms.float32) + x7 = Tensor(np.ones([32, 32]), dtype=ms.float32) + x8 = Tensor(np.ones([32, 128]), dtype=ms.float32) + _executor.compile(net, x1, x2, x4, x5, x6, x7, x8) + + +# add scope2 +def test_six_matmul_repeated3(): + class NetWithLoss(nn.Cell): + def __init__(self, network1, network2): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network1 + self.network2 = network2 + + def construct(self, x1, x2, x4, x5, x6, x7, x8, x9, x10): + predict = self.network(x1, x2, x4, x5, x6, x7, x8) + predict = self.network2(predict, x9, x10) + return self.loss(predict) + + + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x1, x2, x4, x5, x6, x7, x8, x9, x10): + return C.grad_all(self.network)(x1, x2, x4, x5, x6, x7, x8, x9, x10) + + class Net(nn.Cell): + def __init__(self, strategy1, strategy3, strategy4, strategy5, strategy6, strategy7): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul3 = P.MatMul().set_strategy(strategy3) + self.matmul4 = P.MatMul().set_strategy(strategy4) + self.matmul5 = P.MatMul().set_strategy(strategy5) + self.matmul6 = P.MatMul().set_strategy(strategy6) + self.matmul7 = P.MatMul().set_strategy(strategy7) + + def construct(self, x1, x2, x4, x5, x6, x7, x8): + out = self.matmul1(x1, x2) + out = self.matmul3(out, x4) + out = self.matmul4(out, x5) + out = self.matmul5(out, x6) + out = self.matmul6(out, x7) + out = self.matmul7(out, x8) + return out + + class Net1(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x1, x2, x3): + out = self.matmul1(x1, x2) + out = self.matmul2(out, x3) + return out + + + set_auto_parallel_context(device_num=512, global_rank=0) + strategy1 = ((8, 1), (1, 1)) + strategy3 = ((8, 1), (1, 1)) + strategy4 = ((8, 1), (1, 1)) + strategy5 = ((8, 1), (1, 1)) + strategy6 = ((8, 1), (1, 1)) + strategy7 = ((8, 1), (1, 1)) + strategy8 = ((8, 1), (1, 1)) + strategy9 = ((8, 1), (1, 1)) + net1 = Net(strategy1, strategy3, strategy4, strategy5, strategy6, strategy7) + net2 = Net1(strategy8, strategy9) + net = GradWrap(NetWithLoss(net1, net2)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x1 = Tensor(np.ones([128, 32]), dtype=ms.float32) + x2 = Tensor(np.ones([32, 64]), dtype=ms.float32) + x4 = Tensor(np.ones([64, 128]), dtype=ms.float32) + x5 = Tensor(np.ones([128, 64]), dtype=ms.float32) + x6 = Tensor(np.ones([64, 32]), dtype=ms.float32) + x7 = Tensor(np.ones([32, 32]), dtype=ms.float32) + x8 = Tensor(np.ones([32, 128]), dtype=ms.float32) + x9 = Tensor(np.ones([128, 64]), dtype=ms.float32) + x10 = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x1, x2, x4, x5, x6, x7, x8, x9, x10) + diff --git a/tests/ut/python/parallel/test_sum_as_loss.py b/tests/ut/python/parallel/test_sum_as_loss.py new file mode 100644 index 00000000000..3f33584b959 --- /dev/null +++ b/tests/ut/python/parallel/test_sum_as_loss.py @@ -0,0 +1,81 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, bias): + return C.grad_all(self.network)(x, y, bias) + +def test_sum_as_loss(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0, 1)) + return out + + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((4, 1), ) + net = GradWrap(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + bias = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, bias) + + +def test_sum_as_loss2(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1): + super().__init__() + self.fc_nobias = P.MatMul(transpose_b=True).set_strategy(strategy0) + self.reduce_sum = P.ReduceSum(keep_dims=False).set_strategy(strategy1) + + def construct(self, x, y, bias): + out = self.fc_nobias(x, y) + out = self.reduce_sum(out, (0, 1)) + return out + + + context.set_auto_parallel_context(device_num=16, global_rank=0) + strategy0 = ((4, 1), (4, 1)) + strategy1 = ((1, 1), ) + net = GradWrap(Net(strategy0, strategy1)) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + y = Tensor(np.ones([64, 32]), dtype=ms.float32) + bias = Tensor(np.ones([64]), dtype=ms.float32) + _executor.compile(net, x, y, bias) diff --git a/tests/ut/python/parallel/test_transpose.py b/tests/ut/python/parallel/test_transpose.py new file mode 100644 index 00000000000..70e7ea8e252 --- /dev/null +++ b/tests/ut/python/parallel/test_transpose.py @@ -0,0 +1,107 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore.train import Model, ParallelMode +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor, context +import mindspore as ms +import numpy as np +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore.common.parameter import Parameter +from tests.dataset_mock import MindData +from mindspore import context + + + +class Dataset(MindData): + def __init__(self, predict, label, length=3): + super(Dataset, self).__init__(size=length) + self.predict = predict + self.label = label + self.index = 0 + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.index >= self.length: + raise StopIteration + self.index += 1 + return self.predict, self.label + + def reset(self): + self.index = 0 + + +class TransposeNet(nn.Cell): + def __init__(self, strategy1, strategy2): + super(TransposeNet, self).__init__() + self.matmul = P.MatMul().set_strategy(((8, 1), (1, 1))) + self.matmul_weight = Parameter(Tensor(np.ones([128, 256]), dtype=ms.float32), name="weight") + self.transpose1 = P.Transpose().set_strategy(strategy1) + self.transpose2 = P.Transpose().set_strategy(strategy2) + + def construct(self, x): + x = self.matmul(x, self.matmul_weight) + x = self.transpose1(x, (1, 0)) + x = self.transpose2(x, (1, 0)) + return x + + +def transpose_net(strategy1, strategy2): + return TransposeNet(strategy1=strategy1, strategy2=strategy2) + + +def transpose_common(strategy1, strategy2): + batch_size = 32 + learning_rate = 0.1 + momentum = 0.9 + epoch_size = 2 + + context.reset_auto_parallel_context() + context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=8, parameter_broadcast=False) + + predict = Tensor(np.ones([32, 128]), dtype=ms.float32) + label = Tensor(np.ones([32]), dtype=ms.int32) + dataset = Dataset(predict, label, 2) + net = transpose_net(strategy1, strategy2) + + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + loss.softmax_cross_entropy.set_strategy(((8, 1), (8, 1))) + opt = Momentum(net.trainable_params(), learning_rate, momentum) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss, opt) + + model.train(epoch_size, dataset, dataset_sink_mode=False) + + +def test_transpose1(): + strategy1 = ((1, 8), ) + strategy2 = ((1, 8), ) + transpose_common(strategy1, strategy2) + + +def test_transpose2(): + strategy1=((1, 4), ) + strategy2=((1, 8), ) + transpose_common(strategy1, strategy2) + + +if __name__ == '__main__': + test_transpose1() + test_transpose2() + diff --git a/tests/ut/python/parallel/test_two_matmul.py b/tests/ut/python/parallel/test_two_matmul.py new file mode 100644 index 00000000000..b2b48f0424d --- /dev/null +++ b/tests/ut/python/parallel/test_two_matmul.py @@ -0,0 +1,116 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +# model_parallel test +def test_two_matmul(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=8, global_rank=0, mirror_mean=True) + strategy1 = ((4, 2), (2, 1)) + strategy2 = ((2, 4), (4, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + _executor.compile(net, x, y, b) + +def test_two_matmul_repeated_calculation1(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=64, global_rank=5, mirror_mean=True) + strategy1 = ((2, 4), (4, 8)) + strategy2 = ((1, 1), (1, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +def test_two_matmul_repeated_calculation2(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2): + super().__init__() + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x, y, b): + out = self.matmul1(x, y) + out = self.matmul2(out, b) + return out + + context.set_auto_parallel_context(device_num=64, global_rank=15) + strategy1 = ((2, 4), (4, 8)) + strategy2 = ((2, 2), (2, 1)) + net = GradWrap(NetWithLoss(Net(strategy1, strategy2))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + _executor.compile(net, x, y, b) diff --git a/tests/ut/python/parallel/test_two_weights_parameter.py b/tests/ut/python/parallel/test_two_weights_parameter.py new file mode 100644 index 00000000000..8104bf3b52c --- /dev/null +++ b/tests/ut/python/parallel/test_two_weights_parameter.py @@ -0,0 +1,78 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from mindspore import context +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor, Parameter, ParameterTuple +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops import functional as F + +class NetWithLoss(nn.Cell): + def __init__(self, network, strategy3): + super(NetWithLoss, self).__init__() + self.loss = P.SoftmaxCrossEntropyWithLogits().set_strategy(strategy3) + self.network = network + + def construct(self, x, b): + predict = self.network(x) + return self.loss(predict, b)[0] + +class OneStepCell(nn.Cell): + def __init__(self, network): + super(OneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.network.trainable_params()) + + def construct(self, data, label): + weights = self.weights + grads = C.grad_by_list(self.network, weights)(data, label) + return grads + +def test_two_weights_parameter(): + class Net(nn.Cell): + def __init__(self, strategy1, strategy2, weight, weight2): + super().__init__() + self.weight = Parameter(weight, "w1", requires_grad=True) + self.weight2 = Parameter(weight2, "w2", requires_grad=True) + self.matmul = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + + def construct(self, x): + out = self.matmul(x, self.weight) + out = self.matmul2(out, self.weight2) + return out + + + context.set_auto_parallel_context(device_num=8, global_rank=0) + strategy1 = ((4, 1), (1, 2)) + strategy2 = ((4, 2), (2, 1)) + strategy3 = ((8, 1), (8, 1)) + + x = Tensor(np.ones([64, 32]), dtype=ms.float32) + weight = Tensor(np.ones([32, 64]), dtype=ms.float32) + weight2 = Tensor(np.ones([64, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 64]), dtype=ms.float32) + + net = Net(strategy1, strategy2, weight, weight2) + + net_with_loss = NetWithLoss(net, strategy3) + + train_net = OneStepCell(net_with_loss) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + + _executor.compile(train_net, x, b) diff --git a/tests/ut/python/parallel/test_using_seed_for_initializer.py b/tests/ut/python/parallel/test_using_seed_for_initializer.py new file mode 100644 index 00000000000..836df19bcb6 --- /dev/null +++ b/tests/ut/python/parallel/test_using_seed_for_initializer.py @@ -0,0 +1,66 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +from mindspore import Parameter +import mindspore.common.initializer as init +from numpy import allclose + + +parameter_shape = [16, 4] + + +class ParameterNet(nn.Cell): + def __init__(self): + super(ParameterNet, self).__init__() + self.para_xavier_uniform = Parameter(init.initializer('xavier_uniform', parameter_shape), name="xavier_uniform") + self.para_he_uniform = Parameter(init.initializer('he_uniform', parameter_shape), name="he_uniform") + self.para_xavier_uniform2 = Parameter(init.initializer(init.XavierUniform(), parameter_shape), name="xavier_uniform2") + self.para_he_uniform2 = Parameter(init.initializer(init.HeUniform(), parameter_shape), name="he_uniform2") + self.para_truncated_normal = Parameter(init.initializer(init.TruncatedNormal(), parameter_shape), name="truncated_normal") + self.para_normal = Parameter(init.initializer(init.Normal(), parameter_shape), name="normal") + self.para_uniform = Parameter(init.initializer(init.Uniform(), parameter_shape), name="uniform") + + def construct(self): + raise NotImplementedError + + +def test_using_same_seed_for_initializer(): + np.random.seed(0) + net1 = ParameterNet() + np.random.seed(0) + net2 = ParameterNet() + for key in net1.parameters_dict(): + if key not in net2.parameters_dict(): + assert False + else: + assert allclose(net1.parameters_dict()[key].data.asnumpy(), net2.parameters_dict()[key].data.asnumpy()) + + +def test_using_diffserent_seed_for_initializer(): + np.random.seed(0) + net1 = ParameterNet() + np.random.seed(1) + net2 = ParameterNet() + for key in net1.parameters_dict(): + if key not in net2.parameters_dict(): + assert False + else: + assert not allclose(net1.parameters_dict()[key].data.asnumpy(), net2.parameters_dict()[key].data.asnumpy()) + + +if __name__ == '__main__': + test_using_diffserent_seed_for_initializer() + test_using_same_seed_for_initializer() diff --git a/tests/ut/python/parallel/test_virtual_dataset_3_input.py b/tests/ut/python/parallel/test_virtual_dataset_3_input.py new file mode 100644 index 00000000000..36c94ae1c69 --- /dev/null +++ b/tests/ut/python/parallel/test_virtual_dataset_3_input.py @@ -0,0 +1,78 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor +from tests.ut.python.ops.test_math_ops import VirtualLoss +import mindspore as ms +from mindspore.common.api import _executor +from mindspore.ops import composite as C +from mindspore.ops.operations.comm_ops import _VirtualDataset +from mindspore import context + + +class NetWithLoss(nn.Cell): + def __init__(self, network): + super(NetWithLoss, self).__init__() + self.loss = VirtualLoss() + self.network = network + + def construct(self, x, y, b): + predict = self.network(x, y, b) + return self.loss(predict) + + +class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + + def construct(self, x, y, b): + return C.grad_all(self.network)(x, y, b) + + +# model_parallel test +def test_virtual_dataset_3_input(): + class Net(nn.Cell): + def __init__(self, strategy0, strategy1, strategy2, strategy3): + super().__init__() + self.virtual_dataset = _VirtualDataset().set_strategy(strategy0) + self.matmul1 = P.MatMul().set_strategy(strategy1) + self.matmul2 = P.MatMul().set_strategy(strategy2) + self.gelu = P.Gelu().set_strategy(strategy3) + + def construct(self, x, y, b): + x, y, b = self.virtual_dataset(x, y, b) + out = self.gelu(self.matmul1(x, y)) + out = self.matmul2(out, b) + return out + + strategy0 = ((2, 1), (2, 1), (2, 1)) + strategy1 = ((2, 2), (2, 2)) + strategy2 = ((2, 2), (2, 2)) + strategy3 = ((2, 4), ) + net = GradWrap(NetWithLoss(Net(strategy0, strategy1, strategy2, strategy3))) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + context.set_auto_parallel_context(device_num=8, global_rank=0) + x = Tensor(np.ones([128, 32]), dtype=ms.float32) + y = Tensor(np.ones([32, 64]), dtype=ms.float32) + b = Tensor(np.ones([64, 2048]), dtype=ms.float32) + _executor.compile(net, x, y, b) + + +if __name__ == '__main__': + test_virtual_dataset_3_input() + context.reset_auto_parallel_context() diff --git a/tests/ut/python/parameter_feature/__init__.py b/tests/ut/python/parameter_feature/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/parameter_feature/test_parameter.py b/tests/ut/python/parameter_feature/test_parameter.py new file mode 100644 index 00000000000..696b107f56b --- /dev/null +++ b/tests/ut/python/parameter_feature/test_parameter.py @@ -0,0 +1,229 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +import mindspore.context as context +from mindspore import Tensor, Parameter +from mindspore.nn import Cell +from mindspore.ops import operations as P +import mindspore.ops.composite as C + +context.set_context(mode=context.GRAPH_MODE) + +def test_parser_three_default_mixed_args_subnet(): + + class SubNetDefaultMixedArgs(Cell): + def __init__(self): + super().__init__() + + def construct(self, y, x=3, x1=None, x2=(1, 2)): + if x == 3: + if x1 == None: + return y + return -y + + class NetOut(Cell): + def __init__(self): + super(NetOut, self).__init__() + self.net_inside = SubNetDefaultMixedArgs() + + def construct(self, x, y=3): + z = self.net_inside(x) + + return z + + tensor1 = Tensor(np.full((2, 3), 2).astype(np.float32)) + tensor2 = Tensor(np.full((3, 2), 4).astype(np.float32)) + net = NetOut() + assert net(tensor1, tensor2) == tensor1 + + +def test_net_vararg_kwonlyarg_kwarg(): + class FirstNet(Cell): + def __init__(self): + super(FirstNet, self).__init__() + self.net = SecondNet() + + def construct(self, x=1, z=2+2+4, y=3): + c = self.net(22, 33, x, y, z, 2, 3, 4, 5, key1=10, key2=20, key3=30, key4=40) + return c + + class SecondNet(Cell): + def __init__(self): + super(SecondNet, self).__init__() + + def construct(self, x, y=2, p=5, q=40, *var, key1=1, key2=3, **kwargs): + a = x - y + b = p * q + c = a / b + d = var[0] * var[1] * var[2] * var[3] + e = key1 - key2 - kwargs["key3"] + kwargs["key4"] + return a + b + c + d + e + + net = FirstNet() + net() + +def test_net_vararg_normal_input(): + class FirstNet(Cell): + def __init__(self): + super(FirstNet, self).__init__() + self.net = SecondNet() + + def construct(self, x=1, z=2+2+4, y=3): + c = self.net(22, 33, x, y, z, 2, 3, 4, 5, key1=10, key2=20, key3=30, key4=40) + return c + + class SecondNet(Cell): + def __init__(self): + super(SecondNet, self).__init__() + + def construct(self, x, y=2, p=5, q=40, *var, key1=1, key2=3, **kwargs): + a = x - y + b = p * q + c = a / b + d = var[0] * var[1] * var[2] * var[3] + e = key1 - key2 - kwargs["key3"] + kwargs["key4"] + return a + b + c + d + e + x = Tensor(np.ones((2, 3, 4), np.int32)) + net = FirstNet() + net(x, x, x) + +def test_prim_vararg_kwonlyarg(): + class FirstNet(Cell): + def __init__(self): + super(FirstNet, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + self.net = SecondNet() + self.x = Tensor(np.ones((2, 3, 4), np.float32)) + self.y = Tensor(np.ones((2, 3, 4), np.float32)) + + def construct(self): + a = self.max(self.x, self.y) + b = self.min(self.x, self.y) + t = {"x": a, "y": b} + c = self.net(t["x"], t["y"], a, b, z=a, r=b) + return c + + class SecondNet(Cell): + def __init__(self): + super(SecondNet, self).__init__() + self.addN = P.AddN() + self.max = P.Maximum() + self.add = P.TensorAdd() + + def construct(self, x, y, *args, z=0, r=1): + c = self.max(args[0], args[1]) + d = self.addN(args) + e = self.max(*args) + ret = x + y + c + d + e + z + r + return ret + + net = FirstNet() + net() + + +def test_no_vararg(): + class FirstNet(Cell): + def __init__(self): + super(FirstNet, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + self.net = SecondNet() + self.x = Tensor(np.ones((2, 3, 4), np.float32)) + self.y = Tensor(np.ones((2, 3, 4), np.float32)) + + def construct(self): + t = {"x": self.x, "y": self.y} + a = self.max(self.x, self.y) + b = self.min(self.x, self.y) + c = self.net(a, b, z=a, r=b) + return c + + class SecondNet(Cell): + def __init__(self): + super(SecondNet, self).__init__() + + def construct(self, x, y, *, z=0, r=1): + ret = x + y + z + r + return ret + + net = FirstNet() + net() + + +def test_net_variable_and_weights(): + class FirstNet(Cell): + def __init__(self): + super(FirstNet, self).__init__() + self.max = P.Maximum() + self.min = P.Minimum() + self.net = SecondNet() + self.x = Tensor(np.ones((3, 4), np.float32)) + self.y = Tensor(np.ones((3, 4), np.float32)) + self.weight = Parameter(Tensor(np.ones((2, 3, 4)).astype(np.float32)), "w1", requires_grad=True) + + def construct(self, *args): + t = (self.x, self.y) + a = self.max(self.x, self.weight) + b = self.min(self.weight, args[0]) + c = self.net(a, b, *t) + return c + + class SecondNet(Cell): + def __init__(self): + super(SecondNet, self).__init__() + self.addN = P.AddN() + self.max = P.Maximum() + self.add = P.TensorAdd() + self.weight = Parameter(Tensor(np.ones((2, 3, 4), np.float32)), "w2", requires_grad=True) + + def construct(self, a, b, *args): + c = self.max(args[0], a) + d = self.addN(args) + ret = a + b + c + d + self.weight + return ret + + net = FirstNet() + x = Tensor(np.ones((4,), np.float32)) + y = Tensor(np.ones((4,), np.float32)) + z = Tensor(np.ones((4,), np.float32)) + net(x, y, z) + +def test_net_vargs_expand(): + class InputBackward(Cell): + """ InputBackward definition """ + def __init__(self, network, c1=None, c2=None): + super(InputBackward, self).__init__() + self.network = network + self.network.set_train() + self.grad = C.grad_all_with_sens + self.c1 = c1 + self.c2 = c2 + + def construct(self, *inputs): + return self.grad(self.network)(*inputs) + class AddNet(Cell): + def __init__(self): + super(AddNet, self).__init__() + def construct(self, x, y): + return x + y + + net = InputBackward(AddNet()) + x = Tensor(np.random.normal(0, 1, [3, 4, 5]).astype(np.float32)) + y = Tensor(np.random.normal(0, 1, [3, 4, 5]).astype(np.float32)) + sens = Tensor(np.random.normal(0, 1, [3, 4, 5]).astype(np.float32)) + + net.set_train() + net(x, y, sens) diff --git a/tests/ut/python/parameter_feature/test_var_grad.py b/tests/ut/python/parameter_feature/test_var_grad.py new file mode 100644 index 00000000000..d51b78ed9dc --- /dev/null +++ b/tests/ut/python/parameter_feature/test_var_grad.py @@ -0,0 +1,36 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore import context +from mindspore import Tensor, Parameter +from mindspore.nn import Cell +from mindspore.ops import operations as P +import mindspore.ops.composite as C +from mindspore.common.api import _executor + +context.set_context(mode=context.GRAPH_MODE) + +def test_net_vargs_expand(): + class AddNet(Cell): + def __init__(self): + super(AddNet, self).__init__() + self.w = Parameter(Tensor(np.ones((3, 4, 5), np.float32)), "w2", requires_grad=True) + def construct(self, x, y): + return x + y + x = Tensor(np.random.normal(0, 1, [3, 4, 5]).astype(np.float32)) + y = Tensor(np.random.normal(0, 1, [3, 4, 5]).astype(np.float32)) + sens = Tensor(np.random.normal(0, 1, [3, 4, 5]).astype(np.float32)) + net = AddNet() + out = C.grad_all_with_sens(net, net.trainable_params())(x, y, sens) diff --git a/tests/ut/python/pipeline/__init__.py b/tests/ut/python/pipeline/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pipeline/infer/__init__.py b/tests/ut/python/pipeline/infer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pipeline/infer/infer.py b/tests/ut/python/pipeline/infer/infer.py new file mode 100644 index 00000000000..b2fe0a80425 --- /dev/null +++ b/tests/ut/python/pipeline/infer/infer.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" infer """ +from argparse import ArgumentParser +import numpy as np + +from mindspore import Tensor + +from ....dataset_mock import MindData + +__factory = { + "resnet50": resnet50(), +} + + +def parse_args(): + """ parse_args """ + parser = ArgumentParser(description="resnet50 example") + + parser.add_argument("--model", type=str, default="resnet50", + help="the network architecture for training or testing") + parser.add_argument("--phase", type=str, default="test", + help="the phase of the model, default is test.") + parser.add_argument("--file_path", type=str, default="/data/file/test1.txt", + help="data directory of training or testing") + parser.add_argument("--batch_size", type=int, default=1, + help="batch size for training or testing ") + + return parser.parse_args() + + +def get_model(name): + """ get_model """ + if name not in __factory: + raise KeyError("unknown model:", name) + return __factory[name] + + +def get_dataset(batch_size=32): + """ get_dataset """ + dataset_types = np.float32 + dataset_shapes = (batch_size, 3, 224, 224) + + dataset = MindData(size=2, batch_size=batch_size, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset + + +# pylint: disable=unused-argument +def test(name, file_path, batch_size): + """ test """ + network = get_model(name) + + batch = get_dataset(batch_size=batch_size) + + data_list = [] + for data in batch: + data_list.append(data.asnumpy()) + batch_data = np.concatenate(data_list, axis=0).transpose((0, 3, 1, 2)) + input_tensor = Tensor(batch_data) + print(input_tensor.shape()) + network(input_tensor) + + +if __name__ == '__main__': + args = parse_args() + if args.phase == "train": + raise NotImplementedError + test(args.model, args.file_path, args.batch_size) diff --git a/tests/ut/python/pipeline/infer/test_net_infer.py b/tests/ut/python/pipeline/infer/test_net_infer.py new file mode 100644 index 00000000000..e1b3a072670 --- /dev/null +++ b/tests/ut/python/pipeline/infer/test_net_infer.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_net_infer """ +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + self.bn = nn.BatchNorm2d(64) + self.fc = nn.Dense(64, 10) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + + def construct(self, x): + x = self.conv(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +def test_net_infer(): + """ test_net_infer """ + Tensor(np.random.randint(0, 255, [1, 3, 224, 224])) + Net() diff --git a/tests/ut/python/pipeline/infer/test_range.py b/tests/ut/python/pipeline/infer/test_range.py new file mode 100644 index 00000000000..29070e228ad --- /dev/null +++ b/tests/ut/python/pipeline/infer/test_range.py @@ -0,0 +1,63 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.common.api import ms_function +import numpy as np +from mindspore import Tensor +from mindspore.ops import operations as P + + +def test_nest_range_transpose(): + batch_size = 2 + num_layers = 5 + batch_tuple = tuple(Tensor(np.array(np.ones((2, 3)) * 0.01)) for i in range(batch_size)) + layers_tuple = tuple(Tensor(np.array(np.ones((3, 4)) * 0.02)) for i in range(num_layers)) + transpose1 = P.Transpose() + + @ms_function() + def invoke_range(): + out1 = () + for m in range(num_layers): + out1 += (transpose1(layers_tuple[m], (1, 0)),) + # Both for loop will the same range symbol as phi node, when range primitive is converted + # to DoSigature MetaFuncGraph, that MetaFuncGraph will take 2 and 5 as argument, so there is + # 2 entries in that MetaFuncGraphEvaluator, that will make Specialier try to use AnyValue to + # FindGeneralized for S-make_range MetaFuncGraph but it will fail as AnyValue is not constant. + for i in range(batch_size): + out1 += (transpose1(batch_tuple[i], (1, 0)),) + for j in range(num_layers): + out1 += (transpose1(layers_tuple[j], (1, 0)),) + return out1 + + print(invoke_range()) + + +def test_nest_range_simple(): + batch_size = 2 + num_layers = 5 + batch_tuple = tuple(Tensor(np.array(np.ones((2, 3)) * 0.01)) for i in range(batch_size)) + layers_tuple = tuple(Tensor(np.array(np.ones((3, 4)) * 0.02)) for i in range(num_layers)) + + @ms_function() + def invoke_range(): + out1 = () + for m in range(num_layers): + out1 += (layers_tuple[m],) + for i in range(batch_size): + out1 += (batch_tuple[i],) + for j in range(num_layers): + out1 += (layers_tuple[j],) + return out1 + + print(invoke_range()) diff --git a/tests/ut/python/pipeline/infer/test_scalar_add_grad.py b/tests/ut/python/pipeline/infer/test_scalar_add_grad.py new file mode 100644 index 00000000000..e67f7593718 --- /dev/null +++ b/tests/ut/python/pipeline/infer/test_scalar_add_grad.py @@ -0,0 +1,72 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_scalar_add_grad """ +import numpy as np +from mindspore.nn import Cell +from mindspore.common.tensor import Tensor +from mindspore.ops import composite as C +from mindspore import context +from mindspore.nn import ReLU +from mindspore.ops.operations import TensorAdd + +context.set_context(mode=context.GRAPH_MODE) +grad = C.GradOperation('get_all', get_all=True, sens_param=True) +class TensorAddNetMe(Cell): + """ TensorAddNetMe definition """ + def __init__(self): + super(TensorAddNetMe, self).__init__() + self.relu = ReLU() + self.add = TensorAdd() + def construct(self, inputA, inputB): + inputA = self.relu(inputA) + inputB = self.relu(inputB) + x = self.add(inputA, inputB) + x = self.relu(x) + return x +class GradWrap2(Cell): + """ GradWrap2 definition """ + def __init__(self, network): + super(GradWrap2, self).__init__() + self.network = network + def construct(self, inputA, inputB, sens): + gout = grad(self.network)(inputA, inputB, sens) + return gout + +def gen_forwarddata(inputA, inputB): + """ gen_forwarddata """ + net_me = TensorAddNetMe() + net_me.set_train() + output = net_me(Tensor(inputA), Tensor(inputB)) + print(output) + +def gen_backwarddata(inputA, inputB, inputGrad): + """ gen_backwarddata """ + net_me = GradWrap2(TensorAddNetMe()) + net_me.set_train() + output = net_me(Tensor(inputA), Tensor(inputB), Tensor(inputGrad)) + print(output) + +def test_scalar_tennsor_add(): + """ test_scalar_tennsor_add """ + inputa = np.array(32).astype(np.float32) + inputb = np.random.randn(1280, 768).astype(np.float32) + gen_forwarddata(inputa, inputb) + +def test_scalar_tennsor_gradadd(): + """ test_scalar_tennsor_gradadd """ + inputa = np.array(32).astype(np.float32) + inputb = np.random.randn(1280, 768).astype(np.float32) + inputgrad = np.random.randn(1280, 768).astype(np.float32) + gen_backwarddata(inputa, inputb, inputgrad) diff --git a/tests/ut/python/pipeline/parse/__init__.py b/tests/ut/python/pipeline/parse/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pipeline/parse/test_celllist.py b/tests/ut/python/pipeline/parse/test_celllist.py new file mode 100644 index 00000000000..c20a19b43ca --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_celllist.py @@ -0,0 +1,57 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_celllist """ +import numpy as np +from mindspore.nn import Cell +from mindspore.nn import ReLU +from mindspore.nn import Flatten +from mindspore import Tensor, Model +from mindspore.nn import SequentialCell +from mindspore.nn import AvgPool2d +from mindspore import context +from ...ut_filter import non_graph_engine +# pylint: disable=W0212 + + +class Net3(Cell): + def __init__(self): + super().__init__() + self.tuple = (ReLU(), ReLU()) + + def construct(self, x): + for op in self.tuple: + x = op(x) + return x + + +@non_graph_engine +def test_cell_list(): + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + net = Net3() + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + model.predict(input_me) + + +class SequenceNet(Cell): + def __init__(self): + super().__init__() + self.seq = SequentialCell([AvgPool2d(3, 1), ReLU(), Flatten()]) + self.values = list(self.seq._cells.values()) + + def construct(self, x): + x = self.seq(x) + return x diff --git a/tests/ut/python/pipeline/parse/test_compile.py b/tests/ut/python/pipeline/parse/test_compile.py new file mode 100644 index 00000000000..812a96aef2e --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_compile.py @@ -0,0 +1,95 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_compile.py +@Author: +@Date : 2019-03-20 +@Desc : test mindspore compile method +""" +import logging +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor, Model, context +from mindspore.nn.optim import Momentum +from mindspore.ops.composite import add_flags +from ...ut_filter import non_graph_engine + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + + def construct(self, x): + x = self.conv(x) + x = self.relu(x) + out = self.flatten(x) + return out + + +loss = nn.MSELoss() + + +# Test case 1 : test the new compiler interface +# _build_train_graph is deprecated +def test_build(): + """ test_build """ + Tensor(np.random.randint(0, 255, [1, 3, 224, 224])) + Tensor(np.random.randint(0, 10, [1, 10])) + net = Net() + opt = Momentum(net.get_parameters(), learning_rate=0.1, momentum=0.9) + Model(net, loss_fn=loss, optimizer=opt, metrics=None) + + +# Test case 2 : test the use different args to run graph +class Net2(nn.Cell): + """ Net2 definition """ + def __init__(self): + super(Net2, self).__init__() + self.relu = nn.ReLU() + + def construct(self, x): + x = self.relu(x) + return x + + +@non_graph_engine +def test_different_args_run(): + """ test_different_args_run """ + np1 = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me1 = Tensor(np1) + np2 = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me2 = Tensor(np2) + + net = Net2() + net = add_flags(net, predit=True) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + me1 = model.predict(input_me1) + me2 = model.predict(input_me2) + out_me1 = me1.asnumpy() + out_me2 = me2.asnumpy() + print(np1) + print(np2) + print(out_me1) + print(out_me2) + assert not np.allclose(out_me1, out_me2, 0.01, 0.01) diff --git a/tests/ut/python/pipeline/parse/test_create_obj.py b/tests/ut/python/pipeline/parse/test_create_obj.py new file mode 100644 index 00000000000..a702f37e0bf --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_create_obj.py @@ -0,0 +1,129 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_create_obj.py +@Author: +@Date : 2019-06-26 +@Desc : test create object instance on parse function, eg: 'construct' + Support class : nn.Cell ops.Primitive + Support parameter: type is define on function 'ValuePtrToPyData' + (int,float,string,bool,tensor) +""" +import logging +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.api import ms_function +from mindspore.common.tensor import Tensor +from ...ut_filter import non_graph_engine + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.softmax = nn.Softmax(0) + self.axis = 0 + + def construct(self, x): + x = nn.Softmax(self.axis)(x) + return x + + +# Test: creat CELL OR Primitive instance on construct +@non_graph_engine +def test_create_cell_object_on_construct(): + """ test_create_cell_object_on_construct """ + log.debug("begin test_create_object_on_construct") + np1 = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(np1) + + net = Net() + output = net(input_me) + out_me1 = output.asnumpy() + print(np1) + print(out_me1) + log.debug("finished test_create_object_on_construct") + + +# Test: creat CELL OR Primitive instance on construct +class Net1(nn.Cell): + """ Net1 definition """ + def __init__(self): + super(Net1, self).__init__() + self.add = P.TensorAdd() + + @ms_function + def construct(self, x, y): + add = P.TensorAdd() + result = add(x, y) + return result + + +@non_graph_engine +def test_create_primitive_object_on_construct(): + """ test_create_primitive_object_on_construct """ + log.debug("begin test_create_object_on_construct") + x = Tensor(np.array([[1, 2, 3], [1, 2, 3]], np.float32)) + y = Tensor(np.array([[2, 3, 4], [1, 1, 2]], np.float32)) + + net = Net1() + net.construct(x, y) + log.debug("finished test_create_object_on_construct") + + +# Test: creat CELL OR Primitive instance on construct use many parameter +class NetM(nn.Cell): + """ NetM definition """ + def __init__(self, name, axis): + super(NetM, self).__init__() + # self.relu = nn.ReLU() + self.name = name + self.axis = axis + self.softmax = nn.Softmax(self.axis) + + def construct(self, x): + x = self.softmax(x) + return x + + +class NetC(nn.Cell): + """ NetC definition """ + def __init__(self, tensor): + super(NetC, self).__init__() + self.tensor = tensor + + def construct(self, x): + x = NetM("test", 1)(x) + return x + + +# Test: creat CELL OR Primitive instance on construct +@non_graph_engine +def test_create_cell_object_on_construct_use_many_parameter(): + """ test_create_cell_object_on_construct_use_many_parameter """ + log.debug("begin test_create_object_on_construct") + np1 = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(np1) + + net = NetC(input_me) + output = net(input_me) + out_me1 = output.asnumpy() + print(np1) + print(out_me1) + log.debug("finished test_create_object_on_construct") diff --git a/tests/ut/python/pipeline/parse/test_dtype.py b/tests/ut/python/pipeline/parse/test_dtype.py new file mode 100644 index 00000000000..a282c82f836 --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_dtype.py @@ -0,0 +1,32 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_dtype """ +import pytest +from mindspore._c_expression import typing +from mindspore.common.api import ms_function + +number = typing.Number() +int64 = typing.Int(64) +t1 = typing.Tuple((int64, int64)) + + +@ms_function +def try_type(): + return (number, int64, t1) + + +def test_dtype_convert(): + with pytest.raises(RuntimeError): + try_type() diff --git a/tests/ut/python/pipeline/parse/test_fix_bug.py b/tests/ut/python/pipeline/parse/test_fix_bug.py new file mode 100644 index 00000000000..65b96fac0af --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_fix_bug.py @@ -0,0 +1,91 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_fix_bug """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common.api import _executor + + +class assignment1_Net(nn.Cell): + """ assignment1_Net definition """ + def __init__(self, number): + super().__init__() + self.number = number + self.relu = nn.ReLU() + + def construct(self, x): + y = self.number + for _ in [1, y]: + x = self.relu(x) + return x + + +class assignment2_Net(nn.Cell): + """ assignment2_Net definition """ + def __init__(self, number): + super().__init__() + self.number = number + self.relu = nn.ReLU() + + def construct(self, x): + a, b = self.number + for _ in [a, b]: + x = self.relu(x) + return x + + +def assignment_operator_base(number): + """ assignment_operator_base """ + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + x = number + if isinstance(x, int): + net = assignment1_Net(x) + else: + net = assignment2_Net(x) + _executor.compile(net, input_me) + + +def test_ME_assignment_operator_0010(): + """ test_ME_assignment_operator_0010 """ + assignment_operator_base(3) + + +def test_ME_assignment_operator_0020(): + """ test_ME_assignment_operator_0020 """ + assignment_operator_base((1, 3)) + + +class unsupported_method_net(nn.Cell): + """ unsupported_method_net definition """ + def __init__(self): + super().__init__() + self.relu = nn.ReLU() + + def construct(self, x): + with open("a.txt") as f: + f.read() + return x + + +def test_compile_unspported(): + """ test_compile_unspported """ + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + net = unsupported_method_net() + with pytest.raises(RuntimeError): + _executor.compile(net, input_me) diff --git a/tests/ut/python/pipeline/parse/test_for_stmt.py b/tests/ut/python/pipeline/parse/test_for_stmt.py new file mode 100644 index 00000000000..2f7d2540b7f --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_for_stmt.py @@ -0,0 +1,66 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_for_stmt """ +from dataclasses import dataclass +import numpy as np +from mindspore.nn import ReLU +from mindspore.nn import Cell +from mindspore import Tensor, Model, context +from ...ut_filter import non_graph_engine + + +@dataclass +class Access: + a: int + b: int + + def max(self): + if self.a > self.b: + return self.a + return self.b + + +class access2_net(Cell): + """ access2_net definition """ + def __init__(self, number, loop_count=1): + super().__init__() + self.number = number + self.loop_count = loop_count + self.relu = ReLU() + + def construct(self, x): + a = self.loop_count + b = self.number + z = Access(a, b) + for _ in (a, z): + x = self.relu(x) + return x + + +def function_access_base(number): + """ function_access_base """ + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + if number == 2: + net = access2_net(number) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + model.predict(input_me) + + +@non_graph_engine +def test_access_0040(): + """ test_access_0040 """ + function_access_base(2) diff --git a/tests/ut/python/pipeline/parse/test_graph_return_const_param.py b/tests/ut/python/pipeline/parse/test_graph_return_const_param.py new file mode 100644 index 00000000000..495519b1a51 --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_graph_return_const_param.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_return_const_or_parameter """ + +import numpy as np +import mindspore.nn as nn +from mindspore import context +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.common.api import ms_function +context.set_context(mode=context.GRAPH_MODE) + + +class ChooseOneParam(nn.Cell): + def __init__(self, flag): + super(ChooseOneParam, self).__init__() + self.flag = flag + + def construct(self, x, y): + if self.flag == 0: + return x + return y + + +class ChooseOneConst(nn.Cell): + def __init__(self, flag, x, y): + super(ChooseOneConst, self).__init__() + self.flag = flag + self.x = x + self.y = y + + def construct(self): + if self.flag == 0: + return self.x + return self.y + + +def test_choose_input_x(): + choose = ChooseOneParam(0) + tensor_x = Tensor(np.zeros(2), dtype=mstype.int32) + tensor_y = Tensor(np.ones(2), dtype=mstype.int32) + out = choose(tensor_x, tensor_y) + assert np.allclose(tensor_x.asnumpy(), out.asnumpy()) + + +def test_choose_input_y(): + choose = ChooseOneParam(1) + tensor_x = Tensor(1, dtype=mstype.int32) + tensor_y = Tensor(2, dtype=mstype.int32) + out = choose(tensor_x, tensor_y) + assert np.allclose(tensor_y.asnumpy(), out.asnumpy()) + + +def test_choose_const_x(): + tensor_x = Tensor(np.zeros(2), dtype=mstype.int32) + tensor_y = Tensor(np.ones(2), dtype=mstype.int32) + choose = ChooseOneConst(0, tensor_x, tensor_y) + out = choose() + assert np.allclose(tensor_x.asnumpy(), out.asnumpy()) + + +def test_choose_const_y(): + tensor_x = Tensor(np.zeros(2), dtype=mstype.int32) + tensor_y = Tensor(np.ones(2), dtype=mstype.int32) + choose = ChooseOneConst(1, tensor_x, tensor_y) + out = choose() + assert np.allclose(tensor_y.asnumpy(), out.asnumpy()) diff --git a/tests/ut/python/pipeline/parse/test_list.py b/tests/ut/python/pipeline/parse/test_list.py new file mode 100644 index 00000000000..c8456ff131a --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_list.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_list """ +import numpy as np +import mindspore.nn as nn +from mindspore.nn import Cell +from mindspore import Tensor +from mindspore.common.api import _executor + + +class Net1(Cell): + def __init__(self, list1): + super().__init__() + self.list = list1 + self.fla = nn.Flatten() + + def construct(self, x): + for _ in self.list: + x = self.fla(x) + return x + + +def test_list1(): + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + net = Net1([1]) + _executor.compile(net, input_me) + + +def test_list2(): + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + net = Net1([1, 2]) + _executor.compile(net, input_me) diff --git a/tests/ut/python/pipeline/parse/test_operator.py b/tests/ut/python/pipeline/parse/test_operator.py new file mode 100644 index 00000000000..a3412a6f8f5 --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_operator.py @@ -0,0 +1,133 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_operator """ +import numpy as np +from mindspore.ops import operations as P +from mindspore.nn import ReLU +from mindspore.nn import Cell +from mindspore import Tensor, Model, context +from ...ut_filter import non_graph_engine + + +class arithmetic_Net(Cell): + """ arithmetic_Net definition """ + def __init__(self, symbol, loop_count=(1, 3)): + super().__init__() + self.symbol = symbol + self.loop_count = loop_count + self.relu = ReLU() + + def construct(self, x): + a, b = self.loop_count + y = self.symbol + if y == 1: + a += b + for _ in (b, a): + x = self.relu(x) + elif y == 2: + b -= a + for _ in (a, b): + x = self.relu(x) + elif y == 3: + z = a + b + for _ in (b, z): + x = self.relu(x) + elif y == 4: + z = b - a + for _ in (z, b): + x = self.relu(x) + elif y == 5: + z = a * b + for _ in (a, z): + x = self.relu(x) + elif y == 6: + z = b / a + for _ in (a, z): + x = self.relu(x) + elif y == 7: + z = b % a + 1 + for _ in (a, z): + x = self.relu(x) + else: + if not a: + x = self.relu(x) + return x + + +class logical_Net(Cell): + """ logical_Net definition """ + def __init__(self, symbol, loop_count=(1, 3)): + super().__init__() + self.symbol = symbol + self.loop_count = loop_count + self.fla = P.Flatten() + self.relu = ReLU() + + def construct(self, x): + a, b = self.loop_count + y = self.symbol + if y == 1: + if b and a: + x = self.relu(x) + else: + x = self.fla(x) + else: + if b or a: + x = self.relu(x) + else: + x = self.fla(x) + return x + + +def arithmetic_operator_base(symbol): + """ arithmetic_operator_base """ + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + logical_operator = {"++": 1, "--": 2, "+": 3, "-": 4, "*": 5, "/": 6, "%": 7, "not": 8} + x = logical_operator[symbol] + net = arithmetic_Net(x) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + model.predict(input_me) + + +def logical_operator_base(symbol): + """ logical_operator_base """ + input_np = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me = Tensor(input_np) + logical_operator = {"and": 1, "or": 2} + x = logical_operator[symbol] + net = logical_Net(x) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + model.predict(input_me) + + +@non_graph_engine +def test_ME_arithmetic_operator_0080(): + """ test_ME_arithmetic_operator_0080 """ + arithmetic_operator_base('not') + + +@non_graph_engine +def test_ME_arithmetic_operator_0070(): + """ test_ME_arithmetic_operator_0070 """ + logical_operator_base('and') + + +@non_graph_engine +def test_ME_logical_operator_0020(): + """ test_ME_logical_operator_0020 """ + logical_operator_base('or') diff --git a/tests/ut/python/pipeline/parse/test_parse.py b/tests/ut/python/pipeline/parse/test_parse.py new file mode 100644 index 00000000000..03e1dc2ffc5 --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_parse.py @@ -0,0 +1,133 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_parse.py +@Author: +@Date : 2019-01-23 17:13 +@Desc : +""" +import logging +import numpy as np + +import mindspore as ms +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common.api import ms_function, _executor +from mindspore.ops.composite import core +from mindspore.ops.functional import tensor_add +from ...ut_filter import non_graph_engine +# pylint: disable=W0613 +# W0613: unused-argument + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + +# Test case: use the parse obj interface use default parameter +class Net(nn.Cell): + """ Net definition """ + def __init__(self, dim): + super(Net, self).__init__() + self.softmax1 = nn.Softmax(dim) + self.softmax2 = nn.Softmax(dim + 1) + + def construct(self, input_data, input1=ms.Tensor(np.random.randn(2, 3, 4, 5).astype(np.float32))): + return self.softmax1(input_data) + + +@non_graph_engine +def test_parse_defalut_parameter_case2(): + """ test_parse_defalut_parameter_case2 """ + log.debug("begin test_parse_defalut_parameter_case2") + net = Net(0) + npd = np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32') + log.debug("input value is: %r", npd) + input_data = ms.Tensor(npd) + input_data.set_dtype(ms.float32) + + log.debug("start run") + output = net(input_data) + + value = output.asnumpy() + log.debug("output value = %r", value) + + + +# Test case: use the variable parameter for parse object +class Net1(nn.Cell): + """ Net1 definition """ + def __init__(self): + super(Net1, self).__init__() + + def construct(self, *args): + x = args[0] + return x + + +def test_var_parameter_case2(): + """ test_var_parameter_case2 """ + log.debug("begin test_var_parameter_case2") + net = Net1() + npd = np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32') + log.debug("input value is: %r", npd) + input_data = ms.Tensor(npd) + input_data.set_dtype(ms.float32) + + np1 = np.random.randn(2, 3, 4, 5).astype(np.float32) + input1 = ms.Tensor(np1) + np2 = np.random.randn(2, 3, 4, 5).astype(np.float32) + input2 = ms.Tensor(np2) + + _executor.compile(net, input_data, input1, input2) + + + +# Test case: test the global flag +g_x = Tensor(np.ones([3, 3]).astype(np.float32)) + +@ms_function +def tensor_add_global(x): + """ tensor_add_global """ + global g_x + res = tensor_add(x, g_x) + return res + + +@non_graph_engine +def test_global_flag(): + """ test_global_flag """ + log.debug("begin test_global_flag") + x = Tensor(np.ones([3, 3]).astype(np.float32)) + res = tensor_add_global(x) + log.debug("finished test_global_flag, ret = %r", res) + + +class NetWithNDarray(nn.Cell): + """ NetWithNDarray definition """ + def __init__(self, dim): + super(NetWithNDarray, self).__init__() + self.softmax = nn.Softmax(dim) + self.x = ms.Tensor(np.ones(shape=(1)).astype(np.float32)) + + def construct(self, input_data): + return self.softmax(input_data) * self.x + +@non_graph_engine +def test_net_with_ndarray(): + """ test_net_with_ndarray """ + net = NetWithNDarray(0) + input_data = np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32') + + net(ms.Tensor(input_data)) diff --git a/tests/ut/python/pipeline/parse/test_serialize.py b/tests/ut/python/pipeline/parse/test_serialize.py new file mode 100644 index 00000000000..7dc9985e93b --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_serialize.py @@ -0,0 +1,29 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_parse.py +@Date : 2019-11-05 14:49 +@Desc : +""" +import os +from mindspore._extends.parse import dump_obj +from mindspore._extends.parse import load_obj + +def test_load_dump(): + data = (1, 3, 2, 7, 9) + file_name = dump_obj(data, "./") + obj = load_obj("./" + file_name) + os.remove(f'./{file_name}') + assert data == obj diff --git a/tests/ut/python/pipeline/parse/test_unaryop.py b/tests/ut/python/pipeline/parse/test_unaryop.py new file mode 100644 index 00000000000..c406c86ef7f --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_unaryop.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_unaryop """ +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import context + +context.set_context(mode=context.GRAPH_MODE) + + +def test_scalar_usub(): + class Net(nn.Cell): + def __init__(self, x): + super(Net, self).__init__() + self.x = x + + def construct(self): + ret = -self.x + return ret + + net = Net(-2) + assert net() == 2 + + +def test_tensor_usub(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x): + ret = -x + return ret + + x = Tensor(np.ones([6, 8, 10], np.int32)) + net = Net() + net(x) + + +def test_scalar_uadd(): + class Net(nn.Cell): + def __init__(self, x): + super(Net, self).__init__() + self.x = x + + def construct(self): + ret = +self.x + return ret + + net = Net(-2) + assert net() == -2 diff --git a/tests/ut/python/predict/test_predict_save_model.py b/tests/ut/python/predict/test_predict_save_model.py new file mode 100644 index 00000000000..074aa8282ee --- /dev/null +++ b/tests/ut/python/predict/test_predict_save_model.py @@ -0,0 +1,93 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + test network +Usage: + python test_predict_save_model.py --path ./ +""" + +import os +import argparse +import numpy as np +import mindspore.nn as nn +import mindspore.ops.operations as P +import mindspore.context as context +from mindspore.common.tensor import Tensor +from mindspore.train.serialization import export, load_checkpoint, load_param_into_net + + +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + self.relu = P.ReLU() + self.batch_size = 32 + + self.conv1 = nn.Conv2d(1, 6, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, kernel_size=5, stride=1, padding=0, has_bias=False, pad_mode='valid') + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.reshape = P.Reshape() + self.fc1 = nn.Dense(400, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.reshape(output, (self.batch_size, -1)) + output = self.fc1(output) + output = self.relu(output) + output = self.fc2(output) + output = self.relu(output) + output = self.fc3(output) + return output + + +parser = argparse.ArgumentParser(description='MindSpore Model Save') +parser.add_argument('--path', default='./lenet_model.ms', type=str, help='model save path') + +if __name__ == '__main__': + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + context.set_context(enable_task_sink=True) + + print("test lenet predict start") + seed = 0 + np.random.seed(seed) + batch = 1 + channel = 1 + input_h = 32 + input_w = 32 + origin_data = np.random.uniform(low=0, high=255, size=(batch, channel, input_h, input_w)).astype(np.float32) + origin_data.tofile("lenet_input_data.bin") + + input_data = Tensor(origin_data) + print(input_data.asnumpy()) + net = LeNet() + ckpt_file_path = "./tests/ut/python/predict/checkpoint_lenet.ckpt" + predict_args = parser.parse_args() + model_path_name = predict_args.path + + is_ckpt_exist = os.path.exists(ckpt_file_path) + if is_ckpt_exist: + param_dict = load_checkpoint(ckpoint_file_name=ckpt_file_path) + load_param_into_net(net, param_dict) + export(net, input_data, file_name=model_path_name, file_format='LITE') + print("test lenet predict success.") + else: + print("checkpoint file is not exist.") diff --git a/tests/ut/python/pynative_mode/__init__.py b/tests/ut/python/pynative_mode/__init__.py new file mode 100644 index 00000000000..4317e063794 --- /dev/null +++ b/tests/ut/python/pynative_mode/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup for pytest""" +from mindspore import context + +# pylint: disable=unused-argument +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) diff --git a/tests/ut/python/pynative_mode/engine/__init__.py b/tests/ut/python/pynative_mode/engine/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/engine/test_cell_wrapper.py b/tests/ut/python/pynative_mode/engine/test_cell_wrapper.py new file mode 100644 index 00000000000..396bd28ffa4 --- /dev/null +++ b/tests/ut/python/pynative_mode/engine/test_cell_wrapper.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_cell_wrapper """ +import numpy as np +import mindspore.nn as nn +from mindspore import Parameter, Tensor +from mindspore.nn import WithLossCell +import mindspore.ops.operations as P +from mindspore.nn import SoftmaxCrossEntropyWithLogits +from ...ut_filter import non_graph_engine + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name="bias") + self.matmul = P.MatMul() + self.biasAdd = P.BiasAdd() + self.softmax = P.Softmax() + + def construct(self, x): + x = self.biasAdd(self.matmul(x, self.weight), self.bias) + x = self.softmax(x) + return x + + +@non_graph_engine +def test_loss_wrapper(): + """ test_loss_wrapper """ + input_data = Tensor(np.ones([1, 64]).astype(np.float32)) + input_label = Tensor(np.ones([1, 10]).astype(np.float32)) + net = Net() + loss = SoftmaxCrossEntropyWithLogits() + cost = WithLossCell(net, loss) + cost(input_data, input_label) diff --git a/tests/ut/python/pynative_mode/ge/__init__.py b/tests/ut/python/pynative_mode/ge/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/ge/model/__init__.py b/tests/ut/python/pynative_mode/ge/model/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py b/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py new file mode 100644 index 00000000000..1007d0d392f --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_lenet_model """ +import numpy as np + +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim import Momentum +from mindspore.nn import WithGradCell, WithLossCell +from ....ut_filter import non_graph_engine + + +class LeNet5(nn.Cell): + """ LeNet5 definition """ + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +@non_graph_engine +def test_lenet_pynative_train_net(): + """ test_lenet_pynative_train_net """ + data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([1, 10]).astype(np.float32)) + dout = Tensor(np.ones([1]).astype(np.float32)) + iteration_num = 1 + verification_step = 0 + + net = LeNet5() + + for i in range(0, iteration_num): + # get the gradients + loss_fn = nn.SoftmaxCrossEntropyWithLogits(is_grad=False) + grad_fn = nn.SoftmaxCrossEntropyWithLogits() + grad_net = WithGradCell(net, grad_fn, sens=dout) + gradients = grad_net(data, label) + + # update parameters + opt = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + opt(gradients) + + # verification + if i == verification_step: + loss_net = WithLossCell(net, loss_fn) + loss_output = loss_net(data, label) + print("The loss of %s-th iteration is %s" % (i, loss_output.asnumpy())) + + +def test_lenet_pynative_train_model(): + """ test_lenet_pynative_train_model """ + # get loss from model.compute_loss + return diff --git a/tests/ut/python/pynative_mode/ge/ops/__init__.py b/tests/ut/python/pynative_mode/ge/ops/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/ge/ops/test_activation.py b/tests/ut/python/pynative_mode/ge/ops/test_activation.py new file mode 100644 index 00000000000..e359e830433 --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/ops/test_activation.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Activations """ +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor +from ....ut_filter import non_graph_engine + + +# test activation +@non_graph_engine +def test_relu(): + relu = nn.ReLU() + input_data = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32) - 0.5) + output = relu.construct(input_data) + output_np = output.asnumpy() + print(output_np) + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +@non_graph_engine +def test_softmax_axis_none(): + layer = nn.Softmax() + x = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32)) + output = layer.construct(x) + output_np = output.asnumpy() + print(output_np) + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) diff --git a/tests/ut/python/pynative_mode/ge/ops/test_batchnorm.py b/tests/ut/python/pynative_mode/ge/ops/test_batchnorm.py new file mode 100644 index 00000000000..029092e26bb --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/ops/test_batchnorm.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ut for batchnorm layer""" +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor +from ....ut_filter import non_graph_engine + + +@non_graph_engine +def test_bn2d(): + """ut of nn.BatchNorm2d""" + gamma = Tensor(np.random.randn(64).astype(np.float32)*0.01) + beta = Tensor(np.random.randn(64).astype(np.float32)*0.01) + moving_mean = Tensor(np.random.randn(64).astype(np.float32)*0.01) + moving_var = Tensor(np.random.randn(64).astype(np.float32)*0.01) + + bn = nn.BatchNorm2d(num_features=64, + eps=1e-5, + momentum=0.1, + gamma_init=gamma, + beta_init=beta, + moving_mean_init=moving_mean, + moving_var_init=moving_var) + + #3-channel RGB + input_data = Tensor(np.random.randint(0, 10, [1, 64, 56, 56]).astype(np.float32)) + # for test in infer lib + output = bn.construct(input_data) + + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) diff --git a/tests/ut/python/pynative_mode/ge/ops/test_conv.py b/tests/ut/python/pynative_mode/ge/ops/test_conv.py new file mode 100644 index 00000000000..dede9c31593 --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/ops/test_conv.py @@ -0,0 +1,71 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_conv """ +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor +from ....ut_filter import non_graph_engine + + +we = Tensor(np.ones([2, 2])) +in_channels = 3 +out_channels = 64 +ks = 3 + + +def get_me_conv_output(input_data, weight, in_channel, out_channel, kernel_size, + stride=1, padding=0, has_bias=False, bias=None): + """ get_me_conv_output """ + class Net(nn.Cell): + """ Net definition """ + def __init__(self, weight, in_channel, out_channel, kernel_size, + stride=1, padding=0, has_bias=False, bias=None): + super(Net, self).__init__() + self.conv = nn.Conv2d(in_channels=in_channel, + out_channels=out_channel, + kernel_size=kernel_size, + stride=stride, + padding=padding, + has_bias=has_bias, + weight_init=weight, + bias_init=bias) + + def construct(self, input_x): + return self.conv(input_x) + net = Net(weight, in_channel, out_channel, kernel_size, stride, padding, has_bias, bias) + out = net.construct(input_data) + return out.asnumpy() + + +@non_graph_engine +def test_ge_conv(): + """ test_ge_conv """ + input_data = np.random.randn(2, 3, 244, 244).astype(np.float32) + kernel = np.random.randn(6, 3, 7, 7).astype(np.float32) + out = get_me_conv_output(Tensor(input_data), Tensor(kernel), in_channel=3, + out_channel=6, kernel_size=7, stride=7, padding=0) + print(out) + + +@non_graph_engine +def test_ge_conv_with_bias(): + """ test_ge_conv_with_bias """ + input_data = np.random.randn(2, 3, 244, 244).astype(np.float32) + kernel = np.random.randn(6, 3, 7, 7).astype(np.float32) + np.random.randn(2, 6, 35, 35).astype(np.float32) + out = get_me_conv_output(Tensor(input_data), Tensor(kernel), in_channel=3, + out_channel=6, kernel_size=7, stride=7, padding=0) + print(out) diff --git a/tests/ut/python/pynative_mode/ge/ops/test_dense.py b/tests/ut/python/pynative_mode/ge/ops/test_dense.py new file mode 100644 index 00000000000..102d44ae48f --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/ops/test_dense.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_dense """ +import numpy as np +from mindspore import Tensor +from mindspore.nn import Dense +from ....ut_filter import non_graph_engine + + +@non_graph_engine +def test_dense(): + input_data = Tensor(np.ones([16, 8]).astype(np.float32)) + kernel = Tensor(np.ones([8, 8]).astype(np.float32)) + bias = Tensor(np.ones([8]).astype(np.float32)) + fc = Dense(8, 8, kernel, bias) + output = fc(input_data) + output_np = output.asnumpy() + print(output_np) + + +@non_graph_engine +def test_dense_nobias(): + input_data = Tensor(np.ones([16, 8]).astype(np.float32)) + kernel = Tensor(np.ones([8, 8]).astype(np.float32)) + fc = Dense(8, 8, kernel, has_bias=False) + output = fc(input_data) + output_np = output.asnumpy() + print(output_np) diff --git a/tests/ut/python/pynative_mode/ge/ops/test_pooling.py b/tests/ut/python/pynative_mode/ge/ops/test_pooling.py new file mode 100644 index 00000000000..e6cf88a9ca9 --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/ops/test_pooling.py @@ -0,0 +1,55 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test pooling api +""" +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor +from ....ut_filter import non_graph_engine + + +@non_graph_engine +def test_avgpool(): + """ test_avgpool """ + kernel_size = 3 + stride = 2 + avg_pool = nn.AvgPool2d(kernel_size, stride) + assert avg_pool.kernel_size == 3 + assert avg_pool.stride == 2 + input_data = Tensor(np.random.randint(0, 255, [1, 3, 6, 6]).astype(np.float32)) + output = avg_pool(input_data) + output_np = output.asnumpy() + print(output_np) + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +@non_graph_engine +def test_maxpool2d(): + """ test_maxpool2d """ + kernel_size = 3 + stride = 3 + padding = 0 + + max_pool = nn.MaxPool2d(kernel_size, stride, padding=padding) + assert max_pool.kernel_size == 3 + assert max_pool.stride == 3 + assert max_pool.padding == 0 + input_data = Tensor(np.random.randint(0, 255, [1, 3, 6, 6]).astype(np.float32)) + output = max_pool(input_data) + output_np = output.asnumpy() + print(output_np) + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) diff --git a/tests/ut/python/pynative_mode/ge/ops/test_tensor_add.py b/tests/ut/python/pynative_mode/ge/ops/test_tensor_add.py new file mode 100644 index 00000000000..8bde09e02d1 --- /dev/null +++ b/tests/ut/python/pynative_mode/ge/ops/test_tensor_add.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test pooling api +""" +import numpy as np + +from mindspore import Tensor +from mindspore.ops.operations import TensorAdd +from ....ut_filter import non_graph_engine + + +@non_graph_engine +def test_tensor_add(): + x = Tensor(np.ones([1, 3, 4, 4]).astype(np.float32)) + y = Tensor(np.ones([1, 3, 4, 4]).astype(np.float32)) + + tensor_add = TensorAdd() + z = tensor_add(x, y) + assert np.all(z.asnumpy() - (x.asnumpy() + y.asnumpy()) < 0.0001) + +def test_tensor_orign_ops(): + x = Tensor(np.ones([1, 3, 4, 4]).astype(np.float32)) + y = Tensor(np.ones([1, 3, 4, 4]).astype(np.float32)) + z = x + y + assert np.all(z.asnumpy() - (x.asnumpy() + y.asnumpy()) < 0.0001) + z = x * y + assert np.all(z.asnumpy() - (x.asnumpy() * y.asnumpy()) < 0.0001) + assert (x == y) + assert (x != 'zero') diff --git a/tests/ut/python/pynative_mode/nn/__init__.py b/tests/ut/python/pynative_mode/nn/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/nn/test_activation.py b/tests/ut/python/pynative_mode/nn/test_activation.py new file mode 100644 index 00000000000..7230fa272b7 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_activation.py @@ -0,0 +1,75 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test Activations """ +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor + + +# test activation +def test_relu_default(): + relu = nn.ReLU() + input_data = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32) - 0.5) + output = relu.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_activation_str(): + relu = nn.get_activation('relu') + + input_data = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32) - 0.5) + output = relu.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_activation_param(): + relu = nn.get_activation('relu') + + input_data = Tensor(np.random.rand(1, 3, 4, 4).astype(np.float32) - 0.5) + output = relu.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_activation_empty(): + assert nn.get_activation('') is None + + +def test_activation_invalid(): + with pytest.raises(KeyError): + nn.get_activation('relu6') + + +# test softmax +def test_softmax_axis(): + layer = nn.Softmax(1) + x = Tensor(np.ones([3, 3])) + assert layer.softmax.axis == (1,) + output = layer.construct(x) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], (np.float32, np.float64)) + + +def test_softmax_axis_none(): + layer = nn.Softmax() + x = Tensor(np.ones([3, 2])) + assert layer.softmax.axis == (-1,) + output = layer.construct(x) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], (np.float32, np.float64)) diff --git a/tests/ut/python/pynative_mode/nn/test_batchnorm.py b/tests/ut/python/pynative_mode/nn/test_batchnorm.py new file mode 100644 index 00000000000..61d75f3c77d --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_batchnorm.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ut for batchnorm layer""" +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor, Parameter + + +def test_bn_pars_valid1(): + """ut of BatchNorm parameters' validation""" + with pytest.raises(ValueError): + nn.BatchNorm2d(num_features=0) + + +def test_bn_pars_valid2(): + """ut of BatchNorm parameters' validation""" + with pytest.raises(ValueError): + nn.BatchNorm2d(num_features=3, momentum=-0.1) + + +def test_bn_init(): + """ut of BatchNorm parameters' validation""" + bn = nn.BatchNorm2d(num_features=3) + + assert isinstance(bn.gamma, Parameter) + assert isinstance(bn.beta, Parameter) + assert isinstance(bn.moving_mean, Parameter) + assert isinstance(bn.moving_variance, Parameter) + + +def test_bn2d(): + """ut of nn.BatchNorm2d""" + gamma = Tensor(np.array([0.1, 0.3, 0.4]).astype(np.float32)) + beta = Tensor(np.zeros((3), dtype=np.float32)) + moving_mean = Tensor(np.zeros((3), dtype=np.float32)) + moving_var = Tensor(np.ones((3), dtype=np.float32)) + + bn = nn.BatchNorm2d(num_features=3, + eps=1e-5, + momentum=0.1, + gamma_init=gamma, + beta_init=beta, + moving_mean_init=moving_mean, + moving_var_init=moving_var) + + #3-channel RGB + input_data = Tensor(np.random.randint(0, 1, [1, 3, 224, 224]).astype(np.float32)) + output = bn.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_bn1d(): + """ut of nn.BatchNorm1d""" + bn = nn.BatchNorm1d(3) + input_data = Tensor(np.random.randint(0, 1, [1, 3, 100, 100]).astype(np.float32)) + output = bn.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_bn2d_train(): + """ut of nn.BatchNorm2d training""" + bn = nn.BatchNorm2d(3) + bn.training = True + Tensor(np.random.randint(0, 255, [1, 3, 224, 224])) + # current operator does not support multiple output diff --git a/tests/ut/python/pynative_mode/nn/test_cell.py b/tests/ut/python/pynative_mode/nn/test_cell.py new file mode 100644 index 00000000000..16adcd61192 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_cell.py @@ -0,0 +1,300 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_cell """ +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from ...ut_filter import non_graph_engine + + +class ModA(nn.Cell): + """ ModA definition """ + def __init__(self, tensor): + super(ModA, self).__init__() + self.weight = Parameter(tensor, name="weight") + + def construct(self, *inputs): + pass + + +class ModB(nn.Cell): + """ ModB definition """ + def __init__(self, tensor): + super(ModB, self).__init__() + self.weight = Parameter(tensor, name="weight") + + def construct(self, *inputs): + pass + + +class ModC(nn.Cell): + """ ModC definition """ + def __init__(self, ta, tb): + super(ModC, self).__init__() + self.mod1 = ModA(ta) + self.mod2 = ModB(tb) + + def construct(self, *inputs): + pass + + +class Net(nn.Cell): + """ Net definition """ + name_len = 4 + cells_num = 3 + + def __init__(self, ta, tb): + super(Net, self).__init__() + self.mod1 = ModA(ta) + self.mod2 = ModB(tb) + self.mod3 = ModC(ta, tb) + + def construct(self, *inputs): + pass + + +class Net2(nn.Cell): + """ Net2 definition """ + def __init__(self, ta, tb): + super(Net2, self).__init__(auto_prefix=False) + self.mod1 = ModA(ta) + self.mod2 = ModB(tb) + self.mod3 = ModC(ta, tb) + + def construct(self, *inputs): + pass + + +class ConvNet(nn.Cell): + """ ConvNet definition """ + image_h = 224 + image_w = 224 + output_ch = 64 + + def __init__(self, num_classes=10): + super(ConvNet, self).__init__() + self.conv1 = nn.Conv2d(3, ConvNet.output_ch, kernel_size=7, stride=2, pad_mode='pad', padding=3) + self.bn1 = nn.BatchNorm2d(ConvNet.output_ch) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='pad', padding=1) + self.flatten = nn.Flatten() + self.fc = nn.Dense( + int(ConvNet.image_h*ConvNet.image_w*ConvNet.output_ch/(4*4)), + num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.flatten(x) + x = self.fc(x) + return x + + +def test_basic(): + """ test_basic """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len + assert names[0] == "mod1.weight" + assert names[1] == "mod2.weight" + assert names[2] == "mod3.mod1.weight" + assert names[3] == "mod3.mod2.weight" + + +def test_parameter_name(): + """ test_parameter_name """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + names = [] + for m in n.parameters_and_names(): + if m[0]: + names.append(m[0]) + assert names[0] == "mod1.weight" + assert names[1] == "mod2.weight" + assert names[2] == "mod3.mod1.weight" + assert names[3] == "mod3.mod2.weight" + + +def test_cell_name(): + """ test_cell_name """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + n.insert_child_to_cell('modNone', None) + names = [] + for m in n.cells_and_names(): + if m[0]: + names.append(m[0]) + assert names[0] == "mod1" + assert names[1] == "mod2" + assert names[2] == "mod3" + assert names[3] == "mod3.mod1" + assert names[4] == "mod3.mod2" + + +def test_cells(): + """ test_cells """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + ch = list(n.cells()) + assert len(ch) == n.cells_num + + +def test_exceptions(): + """ test_exceptions """ + t = Tensor(np.ones([2, 3])) + + class ModError(nn.Cell): + """ ModError definition """ + def __init__(self, tensor): + self.weight = Parameter(tensor, name="weight") + super(ModError, self).__init__() + + def construct(self, *inputs): + pass + + with pytest.raises(AttributeError): + ModError(t) + + class ModError1(nn.Cell): + """ ModError1 definition """ + def __init__(self, tensor): + super().__init__() + self.weight = Parameter(tensor, name="weight") + self.weight = None + self.weight = ModA(tensor) + + def construct(self, *inputs): + pass + + with pytest.raises(TypeError): + ModError1(t) + + class ModError2(nn.Cell): + """ ModError2 definition """ + def __init__(self, tensor): + super().__init__() + self.mod = ModA(tensor) + self.mod = None + self.mod = tensor + + def construct(self, *inputs): + pass + + with pytest.raises(TypeError): + ModError2(t) + + m = nn.Cell() + with pytest.raises(NotImplementedError): + m.construct() + + +def test_del(): + """ test_del """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net(ta, tb) + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len + del n.mod1 + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len - 1 + with pytest.raises(AttributeError): + del n.mod1.weight + del n.mod2.weight + names = list(n.parameters_dict().keys()) + assert len(names) == n.name_len - 2 + with pytest.raises(AttributeError): + del n.mod + + +def test_add_attr(): + """ test_add_attr """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + p = Parameter(ta, name="weight") + m = nn.Cell() + m.insert_param_to_cell('weight', p) + + with pytest.raises(TypeError): + m.insert_child_to_cell("network", p) + + with pytest.raises(KeyError): + m.insert_param_to_cell('', p) + with pytest.raises(KeyError): + m.insert_param_to_cell('a.b', p) + m.insert_param_to_cell('weight', p) + with pytest.raises(KeyError): + m.insert_child_to_cell('', ModA(ta)) + with pytest.raises(KeyError): + m.insert_child_to_cell('a.b', ModB(tb)) + + with pytest.raises(TypeError): + m.insert_child_to_cell('buffer', tb) + with pytest.raises(TypeError): + m.insert_param_to_cell('w', ta) + with pytest.raises(TypeError): + m.insert_child_to_cell('m', p) + + class ModAddCellError(nn.Cell): + """ ModAddCellError definition """ + def __init__(self, tensor): + self.mod = ModA(tensor) + super().__init__() + + def construct(self, *inputs): + pass + + with pytest.raises(AttributeError): + ModAddCellError(ta) + + +def test_train_eval(): + """ test_train_eval """ + m = nn.Cell() + assert not m.training + m.set_train() + assert m.training + m.set_train(False) + assert not m.training + + +def test_stop_update_name(): + """ test_stop_update_name """ + ta = Tensor(np.ones([2, 3])) + tb = Tensor(np.ones([1, 4])) + n = Net2(ta, tb) + names = list(n.parameters_dict().keys()) + assert names[0] == "weight" + assert names[1] == "mod1.weight" + assert names[2] == "mod2.weight" + + +@non_graph_engine +def test_net_call(): + """ test_net_call """ + with pytest.raises(ValueError): + net = ConvNet() + input_x = Tensor( + np.random.randint(0, 255, [1, 3, net.image_h, net.image_w]).astype(np.float32)) + output = net.construct(input_x) + diff --git a/tests/ut/python/pynative_mode/nn/test_checkparameter.py b/tests/ut/python/pynative_mode/nn/test_checkparameter.py new file mode 100644 index 00000000000..ceebfcf7139 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_checkparameter.py @@ -0,0 +1,111 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_checkparameter """ +import pytest + +from mindspore._checkparam import check_int, check_int_positive, \ + check_bool, check_input_format, _expand_tuple + + +once = _expand_tuple(1) +twice = _expand_tuple(2) +triple = _expand_tuple(3) +kernel_size = 5 +kernel_size1 = twice(kernel_size) +assert kernel_size1 == (5, 5) + + +def test_check_int_1(): + assert check_int(3) == 3 + + +def check_int_positive_1(): + with pytest.raises(ValueError): + check_int_positive(-1) + + +def test_NCHW1(): + assert check_input_format("NCHW") == "NCHW" + + +def test_NCHW3(): + with pytest.raises(ValueError): + check_input_format("rt") + + +def test_check_int_2(): + with pytest.raises(TypeError): + check_int(3.3) + + +def test_check_int_3(): + with pytest.raises(TypeError): + check_int("str") + + +def test_check_int_4(): + with pytest.raises(TypeError): + check_int(True) + + +def test_check_bool_1(): + assert check_bool(True) + + +def test_check_bool_2(): + assert check_bool(False) is not True + + +def test_check_bool_3(): + with pytest.raises(TypeError): + check_bool("str") + + +def test_check_bool_4(): + with pytest.raises(TypeError): + check_bool(1) + + +def test_check_bool_5(): + with pytest.raises(TypeError): + check_bool(3.5) + + +def test_twice_1(): + assert twice(3) == (3, 3) + + +def test_twice_2(): + assert twice((3, 3)) == (3, 3) + + +def test_twice_3(): + with pytest.raises(TypeError): + twice(0.5) + + +def test_twice_4(): + with pytest.raises(TypeError): + twice("str") + + +def test_twice_5(): + with pytest.raises(TypeError): + twice((1, 2, 3)) + + +def test_twice_6(): + with pytest.raises(TypeError): + twice((3.3, 4)) diff --git a/tests/ut/python/pynative_mode/nn/test_container.py b/tests/ut/python/pynative_mode/nn/test_container.py new file mode 100644 index 00000000000..7438ed2613f --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_container.py @@ -0,0 +1,150 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_container """ +from collections import OrderedDict +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor + + +weight = Tensor(np.ones([2, 2])) +conv2 = nn.Conv2d(3, 64, (3, 3), stride=2, padding=0) + +kernel_size = 3 +stride = 2 +padding = 1 +avg_pool = nn.AvgPool2d(kernel_size, stride) + + +class TestSequentialCell(): + """ TestSequentialCell definition """ + + def test_SequentialCell_init(self): + m = nn.SequentialCell() + assert not m + + def test_SequentialCell_init2(self): + m = nn.SequentialCell([conv2]) + assert len(m) == 1 + + def test_SequentialCell_init3(self): + m = nn.SequentialCell([conv2, avg_pool]) + assert len(m) == 2 + + def test_SequentialCell_init4(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + assert len(m) == 2 + + def test_getitem1(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + assert m[0] == conv2 + + def test_getitem2(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + assert len(m[0:2]) == 2 + assert m[:2][1] == avg_pool + + def test_setitem1(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + m[1] = conv2 + assert m[1] == m[0] + + def test_setitem2(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + with pytest.raises(TypeError): + m[1.0] = conv2 + + def test_delitem1(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + del m[0] + assert len(m) == 1 + + def test_delitem2(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + del m[:] + assert not m + + def test_construct(self): + m = nn.SequentialCell(OrderedDict( + [('cov2d', conv2), ('avg_pool', avg_pool)])) + m.construct(Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))) + + +class TestCellList(): + """ TestCellList definition """ + + def test_init1(self): + cell_list = nn.CellList([conv2, avg_pool]) + assert len(cell_list) == 2 + + def test_init2(self): + with pytest.raises(TypeError): + nn.CellList(["test"]) + + def test_getitem(self): + cell_list = nn.CellList([conv2, avg_pool]) + assert cell_list[0] == conv2 + temp_cells = cell_list[:] + assert temp_cells[1] == avg_pool + + def test_setitem(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list[0] = avg_pool + assert cell_list[0] == cell_list[1] + + def test_delitem(self): + cell_list = nn.CellList([conv2, avg_pool]) + del cell_list[0] + assert len(cell_list) == 1 + del cell_list[:] + assert not cell_list + + def test_iter(self): + cell_list = nn.CellList([conv2, avg_pool]) + for _ in cell_list: + break + + def test_add(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list += [conv2] + assert len(cell_list) == 3 + assert cell_list[0] == cell_list[2] + + def test_insert(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list.insert(0, avg_pool) + assert len(cell_list) == 3 + assert cell_list[0] == cell_list[2] + + def test_append(self): + cell_list = nn.CellList([conv2, avg_pool]) + cell_list.append(conv2) + assert len(cell_list) == 3 + assert cell_list[0] == cell_list[2] + + def test_extend(self): + cell_list = nn.CellList() + cell_list.extend([conv2, avg_pool]) + assert len(cell_list) == 2 + assert cell_list[0] == conv2 diff --git a/tests/ut/python/pynative_mode/nn/test_conv.py b/tests/ut/python/pynative_mode/nn/test_conv.py new file mode 100644 index 00000000000..43b41b42218 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_conv.py @@ -0,0 +1,63 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_conv """ +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor + + +weight = Tensor(np.ones([2, 2])) +in_channels = 3 +out_channels = 64 +kernel_size = 3 + + +def test_check_conv2d_1(): + m = nn.Conv2d(3, 64, 3, bias_init='zeros') + output = m.construct(Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_check_conv2d_2(): + Tensor(np.ones([2, 2])) + m = nn.Conv2d(3, 64, 4, has_bias=False, weight_init='normal') + output = m.construct(Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_check_conv2d_3(): + Tensor(np.ones([2, 2])) + m = nn.Conv2d(3, 64, (3, 3)) + output = m.construct(Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_check_conv2d_4(): + Tensor(np.ones([2, 2])) + m = nn.Conv2d(3, 64, (3, 3), stride=2, pad_mode='pad', padding=4) + output = m.construct(Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_check_conv2d_bias(): + m = nn.Conv2d(3, 64, 3, bias_init='zeros') + output = m.construct(Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) diff --git a/tests/ut/python/pynative_mode/nn/test_dense.py b/tests/ut/python/pynative_mode/nn/test_dense.py new file mode 100644 index 00000000000..48bfcc66744 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_dense.py @@ -0,0 +1,118 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test nn.Dense """ +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor +# pylint: disable=E1123 + + +def test_dense_defaultbias_noactivation(): + weight = Tensor(np.array([[0.1, 0.3, 0.4], [0.1, 0.3, 0.4]], dtype=np.float32)) + dense = nn.Dense(3, 2, weight) + assert dense.activation is None + + input_data = Tensor(np.random.randint(0, 255, [1, 3]).astype(np.float32)) + output = dense.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], (np.float32, np.float64)) + + +def test_dense_defaultweight(): + bias = Tensor(np.array([0.5, 0.3], dtype=np.float32)) + dense = nn.Dense(3, 2, bias_init=bias) + #batch_size 1 && 3-channel RGB + input_data = Tensor(np.random.randint(0, 255, [1, 3]).astype(np.float32)) + output = dense.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], (np.float32, np.float64)) + + +def test_dense_bias(): + weight = Tensor(np.array([[0.1, 0.3, 0.6], [0.4, 0.5, 0.2]], dtype=np.float32)) + bias = Tensor(np.array([0.5, 0.3], dtype=np.float32)) + dense = nn.Dense(3, 2, weight, bias) + + input_data = Tensor(np.random.randint(0, 255, [2, 3]).astype(np.float32)) + output = dense.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], (np.float32, np.float64)) + + +def test_dense_nobias(): + weight = Tensor(np.array([[0.1, 0.3, 0.6], [0.4, 0.5, 0.2]], dtype=np.float32)) + dense = nn.Dense(3, 2, weight, has_bias=False) + + input_data = Tensor(np.random.randint(0, 255, [2, 3]).astype(np.float32)) + output = dense.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], (np.float32, np.float64)) + + +def test_dense_none(): + with pytest.raises(TypeError): + nn.Dense(3, 2, None, None) + + +def test_dense_invalid_activation(): + with pytest.raises(KeyError): + nn.Dense(3, 2, activation='relu6') + + +def test_dense_str_activation(): + dense = nn.Dense(1, 1, activation='relu') + assert isinstance(dense.activation, nn.ReLU) + + input_data = Tensor(np.random.randint(0, 255, [1, 1]).astype(np.float32)) + output = dense.construct(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0], np.float32) + + +def test_dense_weight_error(): + dim_error = Tensor(np.array([[[0.1], [0.3], [0.6]], [[0.4], [0.5], [0.2]]])) + with pytest.raises(ValueError): + nn.Dense(3, 2, dim_error) + + shape_error = Tensor(np.array([[0.1, 0.3, 0.6], [0.4, 0.5, 0.2]])) + with pytest.raises(ValueError): + nn.Dense(2, 2, shape_error) + with pytest.raises(ValueError): + nn.Dense(3, 3, shape_error) + + +def test_dense_bias_error(): + dim_error = Tensor(np.array([[0.5, 0.3]])) + with pytest.raises(ValueError): + nn.Dense(3, 2, bias_init=dim_error) + + shape_error = Tensor(np.array([0.5, 0.3, 0.4])) + with pytest.raises(ValueError): + nn.Dense(3, 2, bias_init=shape_error) + + +def test_dense_dtype_error(): + with pytest.raises(TypeError): + nn.Dense(3, 2, dtype=3) + + +def test_dense_channels_error(): + with pytest.raises(ValueError): + nn.Dense(3, 0) + + with pytest.raises(ValueError): + nn.Dense(-1, 2) diff --git a/tests/ut/python/pynative_mode/nn/test_dropout.py b/tests/ut/python/pynative_mode/nn/test_dropout.py new file mode 100644 index 00000000000..d4c3d47dbaa --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_dropout.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_dropout """ +import numpy as np +import pytest +from mindspore.common.api import _executor +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import dtype as mstype + +def test_check_dropout_1(): + x = Tensor(np.ones([20, 16, 50]), mstype.float32) + m = nn.Dropout(0.8) + with pytest.raises(NotImplementedError): + m(x) + + +def test_check_dropout_2(): + x = Tensor(np.ones([20, 16, 50]), mstype.float32) + m = nn.Dropout(0.3, seed0=1) + with pytest.raises(NotImplementedError): + m(x) + + +def test_check_dropout_3(): + x = Tensor(np.ones([20, 16, 50]), mstype.float32) + m = nn.Dropout(0.3, seed0=1, seed1=1) + with pytest.raises(NotImplementedError): + m(x) + + +class Net_Dropout(nn.Cell): + def __init__(self): + super(Net_Dropout, self).__init__() + self.dropout = nn.Dropout(0.5) + + def construct(self, x): + return self.dropout(x) + + +def test_compile_dropout(): + net = Net_Dropout() + input_data = Tensor(np.ones([20, 16, 50], dtype=np.float32)) + _executor.compile(net, input_data) diff --git a/tests/ut/python/pynative_mode/nn/test_flatten.py b/tests/ut/python/pynative_mode/nn/test_flatten.py new file mode 100644 index 00000000000..d71d7ba94d6 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_flatten.py @@ -0,0 +1,28 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test flatten api +""" +import numpy as np + +import mindspore.nn as nn +from mindspore import Tensor + + +def test_flatten_default(): + flatten = nn.Flatten() + input_data = Tensor(np.random.randint(0, 255, [3, 6, 6]).astype(np.float32)) + output = flatten.construct(input_data) + assert output.asnumpy().shape == (3, 36) diff --git a/tests/ut/python/pynative_mode/nn/test_layernorm.py b/tests/ut/python/pynative_mode/nn/test_layernorm.py new file mode 100644 index 00000000000..1362d6ba9e9 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_layernorm.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_layernorm """ +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import dtype as mstype + + +def test_check_layer_norm_1(): + x = Tensor(np.ones([20, 5, 10, 10]), mstype.float32) + shape1 = x.shape()[1:] + m = nn.LayerNorm(shape1, -1, 1) + with pytest.raises(NotImplementedError): + m(x) + + +def test_check_layer_norm_2(): + x = Tensor(np.ones([20, 5, 10, 10]), mstype.float32) + shape1 = x.shape()[1:] + m = nn.LayerNorm(shape1, begin_params_axis=1) + with pytest.raises(NotImplementedError): + m(x) + + +def test_check_layer_norm_3(): + x = Tensor(np.ones([20, 5, 10, 10]), mstype.float32) + shape1 = (10, 10) + m = nn.LayerNorm(shape1, begin_params_axis=2) + with pytest.raises(NotImplementedError): + m(x) diff --git a/tests/ut/python/pynative_mode/nn/test_loss.py b/tests/ut/python/pynative_mode/nn/test_loss.py new file mode 100644 index 00000000000..a130a6eca7d --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_loss.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_loss """ +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor +from ...ut_filter import non_graph_engine + + +def test_L1Loss(): + loss = nn.L1Loss() + input_data = Tensor(np.array([1, 2, 3])) + target_data = Tensor(np.array([1, 2, 2])) + with pytest.raises(NotImplementedError): + loss.construct(input_data, target_data) + +@non_graph_engine +def test_SoftmaxCrossEntropyWithLogits(): + """ test_SoftmaxCrossEntropyWithLogits """ + loss = nn.SoftmaxCrossEntropyWithLogits() + + logits = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + labels = Tensor(np.random.randint(0, 9, [100, 10]).astype(np.float32)) + loss.construct(logits, labels) diff --git a/tests/ut/python/pynative_mode/nn/test_pooling.py b/tests/ut/python/pynative_mode/nn/test_pooling.py new file mode 100644 index 00000000000..ab95fec091d --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_pooling.py @@ -0,0 +1,71 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +test pooling api +""" +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor + +def test_avgpool2d(): + """ test_avgpool2d """ + kernel_size = 3 + stride = 2 + avg_pool = nn.AvgPool2d(kernel_size, stride) + assert avg_pool.kernel_size == 3 + assert avg_pool.stride == 2 + input_data = Tensor(np.random.randint(0, 255, [1, 3, 6, 6])*0.1) + output = avg_pool(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_avgpool2d_error_input(): + """ test_avgpool2d_error_input """ + kernel_size = 5 + stride = 2.3 + with pytest.raises(ValueError): + nn.AvgPool2d(kernel_size, stride) + + + + + +def test_maxpool2d(): + """ test_maxpool2d """ + kernel_size = 3 + stride = 3 + padding = 2 + + max_pool = nn.MaxPool2d(kernel_size, stride, pad_mode='SAME', padding=padding) + assert max_pool.kernel_size == 3 + assert max_pool.stride == 3 + assert max_pool.padding == 2 + input_data = Tensor(np.random.randint(0, 255, [1, 3, 6, 6])*0.1) + output = max_pool(input_data) + output_np = output.asnumpy() + assert isinstance(output_np[0][0][0][0], (np.float32, np.float64)) + + +def test_maxpool2d_error_padding(): + """ test_maxpool2d_error_padding """ + kernel_size = 3.5 + stride = 3 + padding = 1 + with pytest.raises(ValueError): + nn.MaxPool2d(kernel_size, stride, padding=padding) + diff --git a/tests/ut/python/pynative_mode/nn/test_tensor_operation.py b/tests/ut/python/pynative_mode/nn/test_tensor_operation.py new file mode 100644 index 00000000000..306ba63c9f1 --- /dev/null +++ b/tests/ut/python/pynative_mode/nn/test_tensor_operation.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_tensor_operation """ +import numpy as np + +from mindspore import Tensor + + +def test_tensor_add(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + z = x + y + assert z.asnumpy()[0][0][0][0] == 2 + + +def test_tensor_iadd(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + x += y + assert x.asnumpy()[0][0][0][0] == 2 + + +def test_tensor_set_iadd(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + x = [x] + y = [y] + x += y + assert len(x) == 2 + + +def test_tensor_tuple_iadd(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + x = (x,) + y = (y,) + x += y + assert len(x) == 2 + + +def test_tensor_sub(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + z = x - y + assert z.asnumpy()[0][0][0][0] == 0 + + +def test_tensor_isub(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + x -= y + assert x.asnumpy()[0][0][0][0] == 0 + + +# MatMul is not supporeted in GE +def test_tensor_mul(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + z = x * y + assert z.asnumpy()[0][0][0][0] == 1.0 + + +# MatMul is not supporeted in GE +def test_tensor_imul(): + x = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3, 3, 3]).astype(np.float32)) + x *= y + assert x.asnumpy()[0][0][0][0] == 1.0 diff --git a/tests/ut/python/pynative_mode/ops/__init__.py b/tests/ut/python/pynative_mode/ops/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/ops/test_grad.py b/tests/ut/python/pynative_mode/ops/test_grad.py new file mode 100644 index 00000000000..25db6b15d20 --- /dev/null +++ b/tests/ut/python/pynative_mode/ops/test_grad.py @@ -0,0 +1,204 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_grad """ +import numpy as np +import mindspore as ms +from mindspore.common.api import ms_function +from mindspore import Tensor +from mindspore.ops import composite as C +from mindspore.ops.composite import grad_all_with_sens +import mindspore.nn as nn +import mindspore.ops.operations as P +from ...ut_filter import non_graph_engine + + +def mul(x, y): + return x * y + + +@ms_function +def mainf(x, y): + return C.grad(mul)(x, y) + + +@non_graph_engine +def test_grad(): + mainf(1, 2) + + +@non_graph_engine +def test_expand_dims_grad(): + """ test_expand_dims_grad """ + input_tensor = Tensor(np.array([[2, 2], [2, 2]])) + expand_dims = P.ExpandDims() + + def fn(x): + output = expand_dims(x, 0) + return output + + out = fn(input_tensor) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy())) + args = [input_tensor, sens] + gout = gfn(*args) + expect = np.ones([2, 2]) + assert np.all(gout[0].asnumpy() == expect) + + +def test_cast_grad(): + """ test_cast_grad """ + input_np = np.random.randn(2, 3).astype(np.float32) + input_x = Tensor(input_np) + + td = ms.int32 + cast = P.Cast() + + def fn(x): + output = cast(x, td) + return output + + out = fn(input_x) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy())) + args = [input_x, sens] + gout = gfn(*args) + expect = np.ones((2, 3), dtype=np.float32) + assert np.all(gout[0].asnumpy() == expect) + + +@non_graph_engine +def test_reshape_grad(): + """ test_reshape_grad """ + input_tensor = Tensor(np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]])) + shp = (3, 2) + reshape = P.Reshape() + + def fn(x): + output = reshape(x, shp) + return output + + out = fn(input_tensor) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy())) + args = [input_tensor, sens] + gout = gfn(*args) + expect = np.ones([2, 3]) + assert np.all(gout[0].asnumpy() == expect) + + +def test_transpose_grad(): + """ test_transpose_grad """ + input_tensor = Tensor(np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])) + perm = (0, 2, 1) + transpose = P.Transpose() + + def fn(x): + output = transpose(x, perm) + return output + + out = fn(input_tensor) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy())) + args = [input_tensor, sens] + gout = gfn(*args) + expect = np.ones([2, 2, 3]) + assert np.all(gout[0].asnumpy() == expect) + + +@non_graph_engine +def test_squeeze_grad(): + """ test_squeeze_grad """ + input_tensor = Tensor(np.ones(shape=[3, 2, 1])) + squeeze = P.Squeeze(2) + + def fn(x): + output = squeeze(x) + return output + + out = fn(input_tensor) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy())) + args = [input_tensor, sens] + gout = gfn(*args) + expect = np.ones([3, 2, 1]) + assert np.all(gout[0].asnumpy() == expect) + + +def test_select_grad(): + """ test_select_grad """ + select = P.Select() + cond = Tensor(np.array([[True, False, False], [False, True, True]])) + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32)) + y = Tensor(np.array([[7, 8, 9], [10, 11, 12]]).astype(np.float32)) + + def fn(cond, x, y): + output = select(cond, x, y) + return output + + out = fn(cond, x, y) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy()).astype(np.float32)) + args = [cond, x, y, sens] + gout = gfn(*args) + expect_cond = np.zeros_like(cond) + expect_x = np.array([[1, 0, 0], [0, 1, 1]]) + expect_y = np.array([[0, 1, 1], [1, 0, 0]]) + assert np.all(gout[0].asnumpy() == expect_cond) + assert np.all(gout[1].asnumpy() == expect_x) + assert np.all(gout[2].asnumpy() == expect_y) + + +def test_SubGrad(): + """ test_SubGrad """ + input_x = Tensor(np.array([[2, 2]])) + input_y = Tensor(np.array([[2, 2], [2, 2]])) + sub = P.Sub() + + def fn(x, y): + output = sub(x, y) + return output + + out = fn(input_x, input_y) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy())) + args = [input_x, input_y, sens] + gout = gfn(*args) + expect_dx = np.ones([1, 2]).astype(np.int32) * 2 # reduce sum dout to the shape of x + expect_dy = np.ones([2, 2]).astype(np.int32) * (-1) + assert np.array_equal(gout[0].asnumpy(), expect_dx) + assert np.array_equal(gout[1].asnumpy(), expect_dy) + + +def test_MulGrad(): + """ test_MulGrad """ + input_x = Tensor(np.array([[2, 2], [2, 2]], np.float32)) + input_y = Tensor(np.array([[3, 3], [3, 3]], np.float32)) + mul = P.Mul() + + def fn(x, y): + output = mul(x, y) + return output + + out = fn(input_x, input_y) + gfn = grad_all_with_sens(fn) + sens = Tensor(np.ones_like(out.asnumpy()) * 3) + args = [input_x, input_y, sens] + gout = gfn(*args) + expect_dx = np.ones([2, 2], np.float32) * 9 + expect_dy = np.ones([2, 2], np.float32) * 6 + assert np.all(gout[0].asnumpy().shape == expect_dx.shape) + assert np.all(gout[0].asnumpy() == expect_dx) + assert np.all(gout[1].asnumpy().shape == expect_dy.shape) + assert np.all(gout[1].asnumpy() == expect_dy) diff --git a/tests/ut/python/pynative_mode/ops/test_hypermap.py b/tests/ut/python/pynative_mode/ops/test_hypermap.py new file mode 100644 index 00000000000..1744555327a --- /dev/null +++ b/tests/ut/python/pynative_mode/ops/test_hypermap.py @@ -0,0 +1,171 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_hypermap """ +import numpy as np + +from mindspore.common.api import ms_function +from mindspore import Tensor +from mindspore import context +from mindspore.ops import Primitive +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from ...ut_filter import non_graph_engine +# pylint: disable=W0613 +# W0613: unused-argument + + +tensor_add = P.TensorAdd() +scala_add = Primitive('scalar_add') +add = C.MultitypeFuncGraph('add') + + +@add.register("Number", "Number") +def add_scala(x, y): + return scala_add(x, y) + + +@add.register("Tensor", "Tensor") +def add_tensor(x, y): + return tensor_add(x, y) + + +hyper_add = C.HyperMap(add) + + +@ms_function +def mainf(x, y): + return hyper_add(x, y) + + +@non_graph_engine +def test_hypermap_tensor(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + print("test_hypermap_tensor:", mainf(tensor1, tensor2)) + + +def test_hypermap_scalar(): + print("test_hypermap_scalar", mainf(1, 2)) + + +def test_hypermap_tuple(): + print("test_hypermap_tuple", mainf((1, 1), (2, 2))) + + +@non_graph_engine +def test_hypermap_tuple_tensor(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + print("test_hypermap_tuple_tensor", mainf((tensor1, tensor1), (tensor2, tensor2))) + + +@non_graph_engine +def test_hypermap_tuple_mix(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + print("test_hypermap_tuple_mix", mainf((tensor1, 1), (tensor2, 2))) + + +hyper_map = C.HyperMap() + + +@ms_function +def main_noleaf(x, y): + return hyper_map(add, x, y) + + +def test_hypermap_noleaf_scalar(): + main_noleaf(1, 2) + + +@non_graph_engine +def test_hypermap_noleaf_tensor(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + main_noleaf(tensor1, tensor2) + + +def test_hypermap_noleaf_tuple(): + main_noleaf((1, 1), (2, 2)) + + +@non_graph_engine +def test_hypermap_noleaf_tuple_tensor(): + tensor1 = Tensor(np.array([[1.1, 2.1], [2.1, 3.1]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.2], [2.2, 3.2]]).astype('float32')) + tensor3 = Tensor(np.array([[2.2], [3.2]]).astype('float32')) + tensor4 = Tensor(np.array([[2.2], [3.2]]).astype('float32')) + main_noleaf((tensor1, tensor3), (tensor2, tensor4)) + + +def test_hypermap_noleaf_tuple_mix(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + main_noleaf((tensor1, 1), (tensor2, 2)) + + +def add3_scalar(x, y, z): + return scala_add(scala_add(x, y), z) + + +@ms_function +def main_add3_easy(x, y): + add2 = F.partial(add3_scalar, 1) + return add2(x, y) + + +def test_hypermap_add3_easy(): + main_add3_easy(1, 2) + + +add3 = C.MultitypeFuncGraph('add') +partial = Primitive('partial') + + +@add3.register("Number", "Number", "Number") +def add3_scala(x, y, z): + return scala_add(scala_add(x, y), z) + + +@add3.register("Number", "Tensor", "Tensor") +def add3_tensor(x, y, z): + return tensor_add(y, z) + + +@ms_function +def main_add3_scala(x, y): + add2 = partial(add3_scala, 1) + return hyper_map(add2, x, y) + + +@ms_function +def main_add3(x, y): + add2 = partial(add3, 1) + return hyper_map(add2, x, y) + + +@non_graph_engine +def test_hypermap_add3_tensor(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + main_add3(tensor1, tensor2) + + +def test_hypermap_add3_tuple(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + + main_add3((tensor1, 1), (tensor2, 1)) diff --git a/tests/ut/python/pynative_mode/ops/test_multitype.py b/tests/ut/python/pynative_mode/ops/test_multitype.py new file mode 100644 index 00000000000..0073041b96b --- /dev/null +++ b/tests/ut/python/pynative_mode/ops/test_multitype.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_multitype """ +import numpy as np + +from mindspore.common.api import ms_function +from mindspore.ops import Primitive +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore import Tensor +from ...ut_filter import non_graph_engine + + +tensor_add = P.TensorAdd() +scala_add = Primitive('scalar_add') +add = C.MultitypeFuncGraph('add') + + +@add.register("Number", "Number") +def add_scala(x, y): + return scala_add(x, y) + + +@add.register("Tensor", "Tensor") +def add_tensor(x, y): + return tensor_add(x, y) + + +@ms_function +def mainf(x, y): + return add(x, y) + + +@non_graph_engine +def test_multitype_tensor(): + tensor1 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + tensor2 = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]]).astype('float32')) + mainf(tensor1, tensor2) + + +def test_multitype_scalar(): + mainf(1, 2) diff --git a/tests/ut/python/pynative_mode/test_backend.py b/tests/ut/python/pynative_mode/test_backend.py new file mode 100644 index 00000000000..937f7b24ff7 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_backend.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_backend """ +import numpy as np +import pytest +from mindspore.ops import operations as P +import mindspore.nn as nn +from mindspore import context +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore._extends.pynative_helper import args_type_check +from mindspore.common.tensor import Tensor +from mindspore.common.api import ms_function + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + self.x = Parameter(initializer('normal', [1, 3, 3, 4]), name='x') + self.y = Parameter(initializer('normal', [1, 3, 3, 4]), name='y') + + @ms_function + def construct(self): + return self.add(self.x, self.y) + + +def test_vm_backend(): + """ test_vm_backend """ + context.set_context(mode=context.PYNATIVE_MODE) + add = Net() + output = add() + assert output.asnumpy().shape == (1, 3, 3, 4) + +def test_vm_set_context(): + """ test_vm_set_context """ + context.set_context(save_graphs=True, save_graphs_path="/home/mindspore", mode=context.GRAPH_MODE) + assert context.get_context("save_graphs") + assert context.get_context("mode") == context.GRAPH_MODE + assert context.get_context("save_graphs_path") == "/home/mindspore" + context.set_context(mode=context.PYNATIVE_MODE) + +@args_type_check(v_str=str, v_int=int, v_tuple=tuple) +def check_input(v_str, v_int, v_tuple): + """ check_input """ + print("v_str:", v_str) + print("v_int:", v_int) + print("v_tuple:", v_tuple) + + +def test_args_type_check(): + """ test_args_type_check """ + with pytest.raises(TypeError): + check_input(100, 100, (10, 10)) + with pytest.raises(TypeError): + check_input("name", "age", (10, 10)) + with pytest.raises(TypeError): + check_input("name", 100, "age") + check_input("name", 100, (10, 10)) diff --git a/tests/ut/python/pynative_mode/test_bprop.py b/tests/ut/python/pynative_mode/test_bprop.py new file mode 100644 index 00000000000..c8360d6c96e --- /dev/null +++ b/tests/ut/python/pynative_mode/test_bprop.py @@ -0,0 +1,81 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_bprop """ +import numpy as np +import mindspore.nn as nn +from mindspore import context +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter +from mindspore.common import Tensor +from ....mindspore_test_framework.utils.bprop_util import bprop +from mindspore.common.api import ms_function + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + +class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z') + + @ms_function + def construct(self, x, y): + x = x * self.z + out = self.matmul(x, y) + return x, out + +def test_bprop_no_sens(): + grads = bprop(Net(), Tensor(np.ones([2, 3]).astype(np.float32)), + Tensor(np.ones([3, 2]).astype(np.float32)), wrt=['inputs']) + print(grads) + +def test_bprop_sens(): + grads = bprop(Net(), Tensor(np.ones([2, 3]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32)), + grads_wrt_outputs=(Tensor(np.ones([2, 3]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))), wrt=['inputs']) + print(grads) + +def test_bprop_first_only(): + grads = bprop(Net(), Tensor(np.ones([2, 3]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32)), + grads_wrt_outputs=(Tensor(np.ones([2, 3]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32)))) + print(grads) + +def test_bprop_wrt_params(): + net = Net() + grads = bprop(net, Tensor(np.ones([2, 3]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32)), + grads_wrt_outputs=(Tensor(np.ones([2, 3]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))), + wrt=['params'], + params=net.trainable_params()) + print(grads) + +def test_bprop_wrt_params_no_sens(): + net = Net() + grads = bprop(net, Tensor(np.ones([2, 3]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32)), + wrt=['params'], + params=net.trainable_params()) + print(grads) + +def test_bprop_wrt_inputs_and_params(): + net = Net() + grads = bprop(net, Tensor(np.ones([2, 3]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32)), + grads_wrt_outputs=(Tensor(np.ones([2, 3]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))), + wrt=['inputs', 'params'], + params=net.trainable_params()) + print(grads) diff --git a/tests/ut/python/pynative_mode/test_cell_bprop.py b/tests/ut/python/pynative_mode/test_cell_bprop.py new file mode 100644 index 00000000000..03ae1affa52 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_cell_bprop.py @@ -0,0 +1,308 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_cell_bprop """ +import numpy as np +import mindspore.nn as nn +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore import Parameter +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as mstype +from mindspore.common.initializer import initializer +from mindspore import context +from ....mindspore_test_framework.utils.bprop_util import bprop + +import pytest + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +class MulAdd(nn.Cell): + def __init__(self): + super(MulAdd, self).__init__() + + def construct(self, x, y): + return 2 * x + y + + def bprop(self, x, y, out, dout): + # In this test case, The user defined bprop is wrong defined purposely to distinguish from ad result + return 2 * dout, 2 * y + +def test_grad_mul_add(): + mul_add = MulAdd() + assert C.grad_all(mul_add)(1, 2) == (2, 4) + + +class InlineMulADD(nn.Cell): + def __init__(self): + super(InlineMulADD, self).__init__() + self.mul_add = MulAdd() + self.param = Parameter(2, 'param') + + def construct(self, x, y): + return self.mul_add(x, y) + x + self.param * y + +def test_grad_inline_mul_add(): + inline_mul_add = InlineMulADD() + assert C.grad_all(inline_mul_add)(1, 2) == (3, 6) + + +class WithParameter(nn.Cell): + def __init__(self): + super(WithParameter, self).__init__() + self.param = Parameter(2, 'param') + + def construct(self, x, y): + return self.param * x + y + + def bprop(self, x, y, out, dout): + # In this test case, The user defined bprop is wrong defined purposely to distinguish from ad result + return self.param * dout, 2 * y + +def test_with_param(): + with_param = WithParameter() + with pytest.raises(RuntimeError): + C.grad_all(with_param)(1, 2) + +class WithNoBprop(nn.Cell): + def __init__(self): + super(WithNoBprop, self).__init__() + + def construct(self, x, y): + return 2 * x + y + +def test_with_no_bprop(): + with_no_bprop = WithNoBprop() + C.grad_all(with_no_bprop)(1, 2) == (2, 1) + +def test_grad_in_bprop_1(): + class GradInBprop_1(nn.Cell): + def __init__(self): + super(GradInBprop_1, self).__init__() + self.relu = P.ReLU() + def construct(self, x, y): + return self.relu(x) + class GradInBprop_2(nn.Cell): + def __init__(self): + super(GradInBprop_2, self).__init__() + self.f = GradInBprop_1() + def construct(self, x, y): + return self.f(x, y), C.grad_all(self.f)(x, y) + def bprop(self, x, y, out, dout): + grads = C.grad_all(self.f)(x, y) + return out[1][0], grads[1] + class GradInBprop_3(nn.Cell): + def __init__(self): + super(GradInBprop_3, self).__init__() + self.f = GradInBprop_2() + def construct(self, x, y): + return self.f(x, y) + grad_in_bprop = GradInBprop_3() + grads = C.grad_all(grad_in_bprop)(Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))) + assert (grads[0].asnumpy() == np.ones([2, 2]).astype(np.float32)).all() + assert (grads[1].asnumpy() == np.zeros([2, 2]).astype(np.float32)).all() + +def test_grad_in_bprop_2(): + class GradInBprop_1(nn.Cell): + def __init__(self): + super(GradInBprop_1, self).__init__() + self.relu = P.ReLU() + def construct(self, x, y): + return self.relu(x) + def bprop(self, x, y, out, dout): + return x * y, y + x + class GradInBprop_2(nn.Cell): + def __init__(self): + super(GradInBprop_2, self).__init__() + self.f = GradInBprop_1() + def construct(self, x, y): + return self.f(x, y), C.grad_all(self.f)(x, y) + def bprop(self, x, y, out, dout): + grads = C.grad_all(self.f)(x, y) + return out[1][0], grads[1] + class GradInBprop_3(nn.Cell): + def __init__(self): + super(GradInBprop_3, self).__init__() + self.f = GradInBprop_2() + def construct(self, x, y): + return self.f(x, y) + grad_in_bprop = GradInBprop_3() + grads = C.grad_all(grad_in_bprop)(Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))) + assert (grads[0].asnumpy() == np.ones([2, 2]).astype(np.float32)).all() + assert (grads[1].asnumpy() == np.array([[2, 2], [2, 2]]).astype(np.float32)).all() + +def test_grad_in_bprop_3(): + class GradInBprop_1(nn.Cell): + def __init__(self): + super(GradInBprop_1, self).__init__() + self.relu = P.ReLU() + def construct(self, x, y): + return self.relu(x) + class GradInBprop_2(nn.Cell): + def __init__(self): + super(GradInBprop_2, self).__init__() + self.f = GradInBprop_1() + def construct(self, x, y): + return self.f(x, y), C.grad_all(self.f)(x, y) + def bprop(self, x, y, out, dout): + grads = C.grad_all(self.f)(x, y) + return out[1][0], grads[1] + class GradInBprop_3(nn.Cell): + def __init__(self): + super(GradInBprop_3, self).__init__() + self.f = GradInBprop_2() + def construct(self, x, y): + return self.f(x, y) + def bprop(self, x, y, out, dout): + return x + y + y + out[0], x + x + y + y + dout[0] + grad_in_bprop = GradInBprop_3() + grads = C.grad_all(grad_in_bprop)(Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))) + assert (grads[0].asnumpy() == np.array([[4, 4], [4, 4]]).astype(np.float32)).all() + assert (grads[1].asnumpy() == np.array([[5, 5], [5, 5]]).astype(np.float32)).all() + +class OneInputBprop(nn.Cell): + def __init__(self): + super().__init__() + self.op = P.ReLU() + def construct(self, x): + return self.op(x) + def bprop(self, x, out, dout): + return 5 * x, + +def test_grad_one_input_bprop(): + net = OneInputBprop() + input = Tensor(np.ones([2, 2]).astype(np.float32)) + grad = C.grad_all(net)(input) + assert (grad[0].asnumpy() == np.array([5, 5]).astype(np.float32)).all() + + +class TwoInput(nn.Cell): + def __init__(self): + super().__init__() + def construct(self, x, y): + return x * y + +class InlineBpropTwoInput(nn.Cell): + def __init__(self): + super().__init__() + self.f = TwoInput() + def construct(self, x, y): + return self.f(x, y), C.grad_all(self.f)(x, y) + def bprop(self, x, y, out, dout): + grads = C.grad_all(self.f)(x, y) + return grads[0] * 2, grads[1] * 2 + +def test_grad_inline_bprop_two_input(): + net = InlineBpropTwoInput() + input1 = Tensor(np.ones([2, 2]).astype(np.float32)) + input2 = Tensor(np.ones([2, 2]).astype(np.float32)) + grads = C.grad_all(net)(input1, input2) + assert (grads[0].asnumpy() == np.array([2, 2]).astype(np.float32)).all() + assert (grads[1].asnumpy() == np.array([2, 2]).astype(np.float32)).all() + assert (len(grads) == 2) + + +class TwoInputBprop(nn.Cell): + def __init__(self): + super().__init__() + self.op = P.Mul() + def construct(self, x, y): + return self.op(x, y) + def bprop(self, x, y, out, dout): + return 5 * x, 8 * y + +class TwoInput(nn.Cell): + def __init__(self): + super().__init__() + self.op = P.Mul() + def construct(self, x, y): + return self.op(x, y) + +class TwoInputWithParameter(nn.Cell): + def __init__(self): + super().__init__() + self.op = P.Mul() + self.inputdata = Parameter(initializer(1, (2,2), mstype.float32),name="global_step") + def construct(self, x, y): + x = self.inputdata + x + return self.op(x, y) + +class TwoInputWithOnlyInitParameterBprop(nn.Cell): + def __init__(self): + super().__init__() + self.op = P.Mul() + self.inputdata = Parameter(initializer(1, (2,2), mstype.float32),name="global_step") + def construct(self, x, y): + return self.op(x, y) + def bprop(self, x, y, out, dout): + return 5*x, 8*y + +class InlineMutilTwoInputParameterCell(nn.Cell): + def __init__(self): + super().__init__() + self.f1 = TwoInputBprop() + self.f2 = TwoInput() + self.f3 = TwoInputWithParameter() + self.f4 = TwoInputWithOnlyInitParameterBprop() + def construct(self, x, y): + output = self.f1(x,y)+self.f2(x,y)+self.f3(x,y)+self.f4(x,y) + return output + +def test_grad_inline_bprop_multi_input(): + net = InlineMutilTwoInputParameterCell() + input1 = Tensor(np.ones([2, 2]).astype(np.float32)) + input2 = Tensor(np.ones([2, 2]).astype(np.float32)) + grads = C.grad_all(net)(input1, input2) + assert (grads[0].asnumpy() == np.array([[12, 12], [12, 12]]).astype(np.float32)).all() + assert (grads[1].asnumpy() == np.array([[19, 19], [19, 19]]).astype(np.float32)).all() + assert (len(grads) == 2) + +class MulAddWithParam(nn.Cell): + def __init__(self): + super(MulAddWithParam, self).__init__() + self.mul_add = MulAdd() + self.param = Parameter(Tensor(np.array([[3, 2]], np.float32)), 'param') + def construct(self, x): + return self.mul_add(self.param, x) + + +def test_refkey_bprop(): + net = MulAddWithParam() + input_data = Tensor(np.array([2, 2], np.float32)) + grads = bprop(net, input_data, + grads_wrt_outputs=(Tensor(np.ones([1, 2]).astype(np.float32))), + wrt=['params', 'inputs'], + params=net.trainable_params()) + assert (grads[0][0].asnumpy() == np.array([4, 4]).astype(np.float32)).all() + assert (grads[1][0].asnumpy() == np.array([2, 2]).astype(np.float32)).all() + + +class MulAddWithWrongOutputNum(nn.Cell): + def __init__(self): + super(MulAddWithWrongOutputNum, self).__init__() + def construct(self, x, y): + return 2 * x + y + def bprop(self, x, y, out, dout): + return 2 * dout, 2 * y, out + +def test_grad_mul_add_with_wrong_output_num(): + mul_add = MulAddWithWrongOutputNum() + with pytest.raises(RuntimeError): + C.grad_all(mul_add)(1, 2) diff --git a/tests/ut/python/pynative_mode/test_context.py b/tests/ut/python/pynative_mode/test_context.py new file mode 100644 index 00000000000..450bf60b909 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_context.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_context """ +import pytest +from mindspore import context +# pylint: disable=W0212 +# W0212: protected-access + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +def test_contex_create_context(): + """ test_contex_create_context """ + context.set_context(mode=context.PYNATIVE_MODE) + if context._k_context is None: + ctx = context._context() + assert ctx is not None + context._k_context = None + + +def test_switch_mode(): + """ test_switch_mode """ + context.set_context(mode=context.GRAPH_MODE) + assert context.get_context("mode") == context.GRAPH_MODE + context.set_context(mode=context.PYNATIVE_MODE) + assert context.get_context("mode") == context.PYNATIVE_MODE + + +def test_set_device_id(): + """ test_set_device_id """ + with pytest.raises(TypeError): + context.set_context(device_id="cpu") + assert context.get_context("device_id") == 0 + context.set_context(device_id=1) + assert context.get_context("device_id") == 1 + + +def test_device_target(): + """ test_device_target """ + with pytest.raises(TypeError): + context.set_context(device_target=123) + context.set_context(device_target="GPU") + assert context.get_context("device_target") == "GPU" + context.set_context(device_target="Ascend") + assert context.get_context("device_target") == "Ascend" + assert context.get_context("device_id") == 1 + + +def test_dump_target(): + """ test_dump_target """ + with pytest.raises(TypeError): + context.set_context(save_dump_path=1) + context.set_context(enable_dump=False) + assert context.get_context("enable_dump") == False + context.set_context(enable_dump=True) + assert context.get_context("enable_dump") == True + assert context.get_context("save_dump_path") == "." + + +def test_set_context(): + """ test_set_context """ + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", + device_id=0, save_graphs=True, save_graphs_path="/mindspore") + assert context.get_context("device_id") == 0 + assert context.get_context("device_target") == "Ascend" + assert context.get_context("save_graphs") + assert context.get_context("save_graphs_path") == "/mindspore" + assert context.get_context("mode") == context.GRAPH_MODE + + context.set_context(mode=context.PYNATIVE_MODE) + assert context.get_context("mode") == context.PYNATIVE_MODE + assert context.get_context("device_target") == "Ascend" + + with pytest.raises(ValueError): + context.set_context(modex="ge") diff --git a/tests/ut/python/pynative_mode/test_framstruct.py b/tests/ut/python/pynative_mode/test_framstruct.py new file mode 100644 index 00000000000..2939337211a --- /dev/null +++ b/tests/ut/python/pynative_mode/test_framstruct.py @@ -0,0 +1,708 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_framstruct """ +import pytest +import numpy as np +import mindspore.nn as nn +from mindspore import context +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore.common.initializer import initializer +from mindspore.common import dtype as mstype +import mindspore.nn as nn +from mindspore.nn.wrap.cell_wrapper import WithGradCell, WithLossCell +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.utils.check_gradient import ( + ms_function, check_jacobian, Tensor, NNGradChecker, + OperationGradChecker, check_gradient, ScalarGradChecker) +from ....mindspore_test_framework.utils.bprop_util import bprop +import mindspore.context as context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +@ms_function +def refactor_fac(n): + """ grad_refactor_fac """ + if n == 0: + return 1 + return n * refactor_fac(n-1) +def test_refactor(): + res = refactor_fac(3) + assert res == 6 + +@ms_function +def while_upper_bound(upper): + rval = 2 + while rval < upper: + rval = rval * rval + return rval + +def test_while_upper_bound(): + res = while_upper_bound(10) + assert res == 16 + +@ms_function +def while_lower_bound(lower): + """ t_while """ + rval = lower + while rval < 100: + rval = rval * rval + return rval + +def test_while_lower_bound(): + res = while_lower_bound(2) + assert res == 256 + +@ms_function +def dynamic_make_tuple(x, lower, upper): + out = () + i = lower + while i < upper: + out = out + (x,) + i = i + 1 + return out + +def test_dynamic_make_tuple(): + # Dynamicly recursively creating static type is invalid in mindspore, as mindspore is a static language. + with pytest.raises(RuntimeError): + dynamic_make_tuple(2, 1, 5) + +def test_make_tuple(): + # Staticly recursively creating static type is valid in mindspore. + @ms_function + def make_tuple(x): + out = () + for i in range(3): + out = out + (x,) + return out + res = make_tuple(5) + assert res == (5, 5, 5) + +@ms_function +def add(x, y): + """ add """ + return x + y + +def mul(x, y): + """ mul """ + return x * y + +def add_mul(x, y): + """ add_mul """ + return (x + y) * y + +def mainf(x, y): + """ mainf """ + return C.grad_all(mul)(x, y) + +def grad_add_mul(x, y): + """ grad_add_mul """ + return C.grad_all(add_mul)(x, y) + +@ms_function +def sub(x, y): + """ sub """ + return x - y + +@ms_function +def if_always_true(x): + """ if_always_true """ + if True: + return x + else: + return 0 + +def test_add(): + """ test_add """ + res = add(2.5, 3) + assert res == 5.5 + + +def test_sub(): + """ test_sub """ + res = sub(3.5, 3) + assert res == 0.5 + + +@non_graph_engine +def test_if_always_true(): + """ test_if_always_true """ + res = if_always_true(1) + assert res == 1 + + +@non_graph_engine +def test_f(): + """ test_f """ + res = mainf(3, 2) + assert res == (2, 3) + +@non_graph_engine +def test_grad_add_mul(): + """ test_grad_add_mul """ + res = grad_add_mul(3, 2) + assert res == (2, 7) + +def f(x): + if x > 0: + return f(x-1) + return x + +@ms_function +def list_subscript(): + """ list_subscript """ + x= [1, 2, 3] + return x[0] * x[1] + +def test_list_subscript(): + """ test_list_subscript """ + res = list_subscript() + assert res == 2 + +@ms_function +def ms_infer_for(xs, y): + """ ms_infer_for """ + rval = y + for x in xs: + rval = rval + x + return rval + +def test_infer_for(): + """ test_infer_for """ + t = (1, 2, 3) + y = 4 + res = ms_infer_for(t, y) + assert res == 10 + +@ms_function +def if_construct(a, b): + z = a + if a > b: + z = a+b + else: + z = a*b + if z > b: + return z-a + else: + return a-b + +def test_if_construct(): + """ test_if_construct """ + res = if_construct(3, 6) + assert res == 15 + +@ms_function +def if_scalar(a, b): + """ if_abstract """ + if a: + return a + return b + +def test_if_scalar1(): + """ test_if_abstract """ + res = if_scalar(3, 6) + assert res == 3 + +def test_if_scalar2(): + """ test_if_abstract """ + res = if_scalar(0, 6) + assert res == 6 + +@ms_function +def if_tensor(a, b): + c = a + if a < b: + c = a+a + if c < b: + c = a+c + else: + c = a+b + else: + c = b+b + out = c + c + return out + +def test_if_tensor(): + res = if_tensor(Tensor(np.ones([64, 10]).astype(np.int32)), Tensor(np.ones([64, 10]).astype(np.int32))) + assert res == Tensor(np.ones([64, 10]).astype(np.int32) * 4) + +@ms_function +def rec(x): + """ rec """ + if x > 0: + return rec(x-1) + return x + +def test_grad_rec(): + """ test_grad_rec """ + res = C.grad(rec)(10) + assert res == 1 + +def test_me_rec(): + """ test_me_rec """ + res = rec(10) + assert res == 0 + +@ms_function +def t2_while(x, y): + out = y - x + i = 0 + while i < 10: + out = mul(x, y) + i = i + 1 + return out + +def test_while2(): + res = t2_while(2, 3) + assert res == 6 + +def test_grad_while2(): + res = C.grad(t2_while)(2, 3) + assert res == 3 + +def if_test(a, b): + """ if_test """ + if a > b: + return 3 * a + return 2 * b + +def grad_if(x, y): + """ grad_if """ + return C.grad_all(if_test)(x, y) + +def test_grad_if(): + """ test_grad_if """ + assert grad_if(5, 4) == (3, 0) + +# While loop is not unrolled in forward and backward graphs. +def test_dont_unroll_while(): + def dont_unroll_while(x, y): + i = 2 + out = y - x + while i < 10: + out = mul(x, y) + i = i + 1 + return out + @ms_function() + def invoke_while(x, y): + return C.grad(dont_unroll_while)(x, y) + res = invoke_while(2, 3) + assert res == 3 + +class ConvNet(nn.Cell): + def __init__(self): + super(ConvNet, self).__init__() + out_channel = 16 + kernel_size = 3 + self.conv = P.Conv2D(out_channel, + kernel_size, + mode=1, + pad_mode="pad", + pad=0, + stride=1, + dilation=2, + group=1) + self.w = Parameter(Tensor(np.ones([16, 16, 3, 3]).astype(np.float32)), name='w') + + def construct(self, x): + return self.conv(x, self.w) + +conv = ConvNet() +c1 = Tensor([2], mstype.float32) +c2 = Tensor([10], mstype.float32) +c3 = Tensor([1], mstype.float32) + +@ms_function +def t1_while(x, y, z): + out = x + i = c1 + while i < c2: + out = out + conv(z) + i = i + c3 + out = out + out + return out + +def test_while_net(): + y = Tensor(np.ones([1,3,3,4]).astype(np.float32)) + x = Tensor(np.ones([1,16,12,12]).astype(np.float32)) + z = Tensor(np.ones([1,16,16,16]).astype(np.float32)) + res = t1_while(x, y, z) + assert res == Tensor(np.ones([1,16,12,12]).astype(np.float32) * 2306.0) + +@ms_function +def if_while(a, b, x, z): + c = a + i = c1 + out = x + if a < b: + c = a+a + while i < c2: + out = out + conv(z) + i = i + c3 + else: + c = b+b + out = c + c + return out + +def test_if_while(): + x = Tensor(np.random.randn(1,16,12,12).astype(np.float32)) + z = Tensor(np.random.randn(1,16,16,16).astype(np.float32)) + res = if_while(Tensor(np.ones([64, 10]).astype(np.float32)), Tensor(np.ones([64, 10]).astype(np.float32)), x, z) + assert res == Tensor(np.ones([64, 10]).astype(np.float32) * 4.0) + +def _while(x): + """ _while """ + ret = x * x + i = 2 + while i <= 3: + ret = ret * i + i = i + 1 + return ret + +def grad_while(x): + """ grad_while """ + return C.grad_all(_while)(x) + +def test_grad_while(): + """ test_grad_while """ + assert grad_while(5) == (60,) + +@ms_function +def fac(n): + """ fac """ + if n == 0: + return 1 + return n * fac(n-1) + +def test_fac(): + """ test_fac """ + res = fac(4) + assert res == 24 + +def _for(x): + """ _for """ + ret = x * x + for i in (2, 3): + ret = ret * i + return ret + +def grad_for(x): + """ grad_for """ + return C.grad_all(_for)(x) + +def test_grad_for(): + """ test_grad_for """ + assert grad_for(5) == (60,) + +@ms_function +def try_tail(x): + """ try_tail """ + return C.tail(x) + +@non_graph_engine +def test_tail(): + """ test_tail """ + try_tail((0, 1, 2, 3)) + +@ms_function +def zero_like_tensor(x): + """ zero_like_tensor """ + return C.zeros_like(x) + +def test_zeros(): + """ test_zeros """ + x = Tensor(np.ones([2, 3]).astype(np.int32)) + res = zero_like_tensor(x) + assert res == Tensor(np.zeros([2, 3]).astype(np.int32)) + +def test_ScalarGradChecker(): + """ test_ScalarGradChecker """ + def scalar_f(x, y): + return x * y + check_gradient(scalar_f, 1.0, 4.0, grad_checker_class=ScalarGradChecker, sampling_times=1) + +def test_GradCheckerPrimitive(): + """ test_GradCheckerPrimitive """ + matmul = P.MatMul() + def prim_f(x, y): + return matmul(x, y) + check_gradient(prim_f, Tensor(np.array([[0.65, 0.8, 0.8]], np.float32)), + Tensor(np.array([[0.1], [0.2], [-.1]], np.float32)), + grad_checker_class=OperationGradChecker, sampling_times=2) + +def test_NNGradChecker(): + """ test_NNGradChecker """ + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.dense = nn.Dense(10, 10) + def construct(self, x): + out = self.dense(x) + return out + + check_gradient(Net(), Tensor(np.random.rand(1, 10).astype(np.float32)), + delta=1e-3, + max_error=1e-3, + grad_checker_class=NNGradChecker, sampling_times=3) + +def test_OperationGradChecker(): + """ test_OperationGradChecker """ + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z') + def construct(self, x, y): + x = x * self.z + out = self.matmul(x, y) + return out + check_gradient(Net(), Tensor(np.array([[0.65, 0.8, 0.8]], np.float32)), + Tensor(np.array([[0.1], [0.2], [-.1]], np.float32)), grad_checker_class=OperationGradChecker, + input_selector=[1], sampling_times=2) + + +def test_ScalarJacobianChecker(): + """ test_ScalarJacobianChecker """ + def scalar_f(x, y): + return x * y + check_jacobian(scalar_f, 1.0, 4.0, grad_checker_class=ScalarGradChecker, input_selector=[0]) + + +def test_OperationJacobianChecker(): + """ test_OperationJacobianChecker """ + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.matmul = P.MatMul() + self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z') + def construct(self, x, y): + x = x * self.z + out = self.matmul(x, y) + return x, out + check_jacobian(Net(), Tensor(np.array([[0.65, 0.8, 0.8], [0.1, 0.2, 0.3]], np.float32)), + Tensor(np.array([[0.1, 0.3], [0.2, 0.2], [-.1, 0.4]], np.float32)), + grad_checker_class=OperationGradChecker, input_selector=[0], + output_selector=[0]) + + +def test_NNJacobianChecker(): + """ test_NNJacobianChecker """ + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.dense = nn.Dense(10, 10) + def construct(self, x): + out = self.dense(x) + return out, x + + check_jacobian(Net(), Tensor(np.random.rand(1, 10).astype(np.float32)), + delta=1e-3, + max_error=1e-7, + grad_checker_class=NNGradChecker, + input_selector=[1], + output_selector=[0]) + +def multi_outputs(x, y): + z = x + y + return 2 * z, 2 * z + +def test_grad_multi_outputs(): + assert C.grad_all_with_sens(multi_outputs)(2, 3, (1, 1)) == (4, 4) + +@ms_function +def while_sp(x, y, z): + out = x + i = c3 + while i < c2: + out = mul(x, out) + i = i + c3 + return out + +def test_while_sp(): + y = Tensor(np.ones([1, 3]).astype(np.float32)) + z = Tensor(np.ones([1, 3]).astype(np.float32)) + x = Tensor(np.ones([1, 3]).astype(np.float32) * 2.0) + res = while_sp(x, y, z) + assert res == Tensor(np.ones([1, 3]).astype(np.float32) * 1024.0) + +def grad_refactor_simple_1(x, y): + """ add """ + return x * x + 2 * y + + +def test_grad_refactor_simple_1(): + assert C.grad_all(grad_refactor_simple_1)(2, 1) == (4, 2) + + +def grad_refactor_simple_2(x, y, z): + """ add """ + return x * y + z + x * y * z + x + x * y + + +def test_grad_refactor_simple_2(): + assert C.grad_all(grad_refactor_simple_2)(2, 3, 0) == (7, 4, 7) + + +def grad_refactor_1(a, b): + """ if_test """ + def inner(x, y): + return x * y + return inner(a, b) + + +def test_grad_refactor_1(): + assert C.grad_all(grad_refactor_1)(2, 3) == (3, 2) + + +def grad_refactor_2(a, b): + """ if_test """ + def inner(x): + return x * b + return inner(b) * inner(a) + + +def test_grad_refactor_2(): + assert C.grad_all(grad_refactor_2)(2, 3) == (27, 54) + + +def grad_refactor_3(a): + """ if_test """ + if a > 3: + return 0 + return 3 * a + + +def test_grad_refactor_3(): + assert C.grad_all(grad_refactor_3)(3) == (3,) + + +def grad_refactor_4(a): + """ if_test """ + if a > 3: + return 3 * a + return 0 + + +def test_grad_refactor_4(): + assert C.grad_all(grad_refactor_4)(4) == (3,) + + +def grad_refactor_5(a): + """ if_test """ + if a > 3: + return 1 + return a + + +def test_grad_refactor_5(): + assert C.grad_all(grad_refactor_5)(1) == (1,) + + +def grad_refactor_6(a, b): + """ if_test """ + if a > b: + return 3 * a + b + return 2 * b * a + + +def test_grad_refactor_6(): + C.grad_all(grad_refactor_6)(3, 2) == (3, 1) + + +def grad_refactor_while(x): + """ grad_refactor_while """ + rval = x + while rval < 4: + rval = rval * rval + return rval + + +def test_grad_refactor_9(): + assert C.grad_all(grad_refactor_while)(3) == (6,) + + +def grad_refactor__while_1(x): + """ _while """ + ret = x * x + i = 2 + while i <= 3: + ret = ret * i + i = i + 1 + return ret + + +def test_grad_refactor_10(): + """ test_grad_while """ + assert C.grad_all(grad_refactor__while_1)(5) == (60,) + + +def test_grad_refactor_11(): + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + def construct(self, x, y): + return x * y * y + net = Net() + C.grad_all(net)(Tensor(np.ones([2]).astype(np.float32)), Tensor(np.ones([2]).astype(np.float32))) + + +def test_grad_refactor_12(): + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z') + def construct(self, x, y): + return x * self.z * y + net = Net() + C.grad_all(net)(Tensor(np.ones([2]).astype(np.float32)), Tensor(np.zeros([2]).astype(np.float32))) + + +def test_grad_refactor_13(): + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.z = Parameter(Tensor(np.ones([2]).astype(np.float32)), name='z') + def construct(self, x, y): + return x * self.z * y + net = Net() + weights = ParameterTuple(net.trainable_params()) + C.grad_by_list(net, weights)(Tensor(np.ones([2]).astype(np.float32)), Tensor(np.zeros([2]).astype(np.float32))) + + +def grad_refactor_14(a, b): + """ if_test """ + def inner1(x): + return x * b + def inner2(x): + return a * b + def inner3(x): + if (x > 2): + return a + return b + return inner1(b) + inner2(a) + inner3(a) +def test_grad_refactor_14(): + assert C.grad_all(grad_refactor_14)(2, 3) == (3, 9) diff --git a/tests/ut/python/pynative_mode/test_graph_return_const_param.py b/tests/ut/python/pynative_mode/test_graph_return_const_param.py new file mode 100644 index 00000000000..6f8978dc6e9 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_graph_return_const_param.py @@ -0,0 +1,62 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_return_const_or_parameter """ + +import numpy as np +import mindspore.nn as nn +from mindspore import context +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.common.api import ms_function + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +class ChooseInitParameter(nn.Cell): + def __init__(self): + super(ChooseInitParameter, self).__init__() + self.x = Parameter(Tensor(np.ones(2), dtype=mstype.int32), name='x') + + @ms_function + def construct(self): + return self.x + + +class ChooseInitParameterWithInput(nn.Cell): + def __init__(self): + super(ChooseInitParameterWithInput, self).__init__() + self.x = Parameter(Tensor(np.ones(2), dtype=mstype.int32), name='x') + + @ms_function + def construct(self, input): + return self.x + + +def test_choose_init_param(): + choose = ChooseInitParameter() + expect = Tensor(np.ones(2), dtype=mstype.int32) + out = choose() + assert np.allclose(out.asnumpy(), expect.asnumpy()) + + +def test_choose_param_with_input(): + choose = ChooseInitParameterWithInput() + input = Tensor(np.zeros(2), dtype=mstype.int32) + expect = Tensor(np.ones(2), dtype=mstype.int32) + out = choose(input) + assert np.allclose(expect.asnumpy(), out.asnumpy()) diff --git a/tests/ut/python/pynative_mode/test_high_order_grad.py b/tests/ut/python/pynative_mode/test_high_order_grad.py new file mode 100644 index 00000000000..5548583cc55 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_high_order_grad.py @@ -0,0 +1,127 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_high_order_grad """ +from mindspore.common.api import ms_function +from mindspore.ops.composite import grad, grad_all, grad_all_with_sens +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +def single(x): + """ single """ + ret = 3 * x * x * x + return ret + + +def first_derivative(x): + """ first_derivative """ + return grad(single)(x) + + +def second_derivative(x): + """ second_derivative """ + return grad(first_derivative)(x) + + +@ms_function +def third_derivative(x): + """ third_derivative """ + return grad(second_derivative)(x) + + +def dual(x, y): + """ dual """ + ret = 3 * x * x * x * y * y * y + return ret + + +def first_derivative_all(x): + """ first_derivative_all """ + return grad_all(single)(x)[0] + + +@ms_function +def second_derivative_all(x): + """ second_derivative_all """ + return grad_all(first_derivative_all)(x)[0] + + +def third_derivative_all(x): + """ third_derivative_all """ + return grad_all(second_derivative_all)(x)[0] + + +# will return a tuple (d(dual)/dx, d(dual)/dy) +def first_derivative_dual(x, y): + """ first_derivative_dual """ + return grad_all_with_sens(dual)(x, y, 1) + + +def second_derivative_dual(x, y): + """ second_derivative_dual """ + grad_fn = grad_all_with_sens(first_derivative_dual) + dfdx = grad_fn(x, y, (1, 0))[0] + dfdy = grad_fn(x, y, (0, 1))[1] + return dfdx, dfdy + + +@ms_function +def third_derivative_dual(x, y): + """ third_derivative_dual """ + grad_fn = grad_all_with_sens(second_derivative_dual) + dfdx = grad_fn(x, y, (1, 0))[0] + dfdy = grad_fn(x, y, (0, 1))[1] + return dfdx, dfdy + + +def if_test(x): + """ if_test """ + if x > 10: + return x * x + return x * x * x + + +def first_derivative_if(x): + """ first_derivative_if """ + return grad(if_test)(x) + + +@ms_function +def second_derivative_if(x): + """ second_derivative_if """ + return grad(first_derivative_if)(x) + + +def test_high_order_grad_1(): + """ test_high_order_grad_1 """ + # 18 + assert third_derivative(2) == 18 + # 18 * y * y * y, 18 * x * x * x + assert third_derivative_dual(4, 5) == (2250, 1152) + # 18 * x + assert second_derivative_all(3) == 54 + +def test_high_order_grad_2(): + """ test_high_order_grad_2 """ + # 2 + assert second_derivative_if(12) == 2 + +def test_high_order_grad_3(): + """ test_high_order_grad_2 """ + # 6 * x + assert second_derivative_if(4) == 24 diff --git a/tests/ut/python/pynative_mode/test_insert_grad_of.py b/tests/ut/python/pynative_mode/test_insert_grad_of.py new file mode 100644 index 00000000000..38432d79f3e --- /dev/null +++ b/tests/ut/python/pynative_mode/test_insert_grad_of.py @@ -0,0 +1,109 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_insert_grad_of """ +import numpy as np +import mindspore.nn as nn +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.common.api import ms_function +from ....mindspore_test_framework.utils.bprop_util import bprop +from ....mindspore_test_framework.utils.debug_util import PrintShapeTypeCell, PrintGradShapeTypeCell +from mindspore import Tensor + +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +def stop_gradient(dx): + """ stop_gradient """ + return C.zeros_like(dx) + +stop = P.InsertGradientOf(stop_gradient) +def test_InsertGradientOf_1(): + """ test_InsertGradientOf_1 """ + def stop_test(x, y): + x = stop(x) + c = x * y + return c + + def f(x, y): + return C.grad_all(stop_test)(x, y) + print("stop_gradient:", f(1, 2)) + +def clip_gradient(dx): + """ clip_gradient """ + ret = dx + if ret > 1.0: + ret = 1.0 + + if ret < 0.2: + ret = 0.2 + + return ret + +clip = P.InsertGradientOf(clip_gradient) +def test_InsertGradientOf_2(): + """ test_InsertGradientOf_2 """ + def clip_test(x, y): + x = clip(x) + y = clip(y) + c = x * y + return c + + @ms_function + def f(x, y): + return clip_test(x, y) + + def fd(x, y): + return C.grad_all(clip_test)(x, y) + + print("forward: ", f(1.1, 0.1)) + print("clip_gradient:", fd(1.1, 0.1)) + +summary = P.ScalarSummary() +def debug_gradient(dx): + """ debug_gradient """ + dx = summary("dx: ", dx) + return dx + +debug = P.InsertGradientOf(debug_gradient) +def test_InsertGradientOf_3(): + """ test_InsertGradientOf_3 """ + def debug_test(x, y): + x = debug(x) + y = debug(y) + c = x * y + return c + + def f(x, y): + return C.grad_all(debug_test)(x, y) + print("debug_gradient:", f(1, 2)) + +def test_print_shape_type(): + class Mul(nn.Cell): + def __init__(self): + super(Mul, self).__init__() + self.print_shape_type = PrintShapeTypeCell() + self.print_shape_type_gradient = PrintGradShapeTypeCell("Gradients") + def construct(self, x, y): + z = x * y + self.print_shape_type("Forward", z) + self.print_shape_type_gradient(z) + return z + bprop(Mul(), Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))) diff --git a/tests/ut/python/pynative_mode/test_parse_method.py b/tests/ut/python/pynative_mode/test_parse_method.py new file mode 100644 index 00000000000..cb984056de1 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_parse_method.py @@ -0,0 +1,412 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_parse_method.py +@Author: +@Date : 2019-06-27 +@Desc : test parse the object's method +""" +import logging +import numpy as np +import mindspore.nn as nn +from dataclasses import dataclass +from mindspore.common.api import ms_function +from mindspore.common.tensor import Tensor +from mindspore.ops.composite import core +from mindspore._extends.parse.standard_method import ms_len +from ..ut_filter import non_graph_engine +import pytest + +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +@ms_function +def default_parameter_f(x, y=3): + """ default_parameter_f """ + z = x + y + return z + + +# Test case: test parse fn that use default parameter +def test_parse_defalut_parameter_case1(): + """ Test default parameter function call """ + log.debug("begin test_parse_defalut_parameter_case1") + ret = default_parameter_f(2) + log.debug("finished test_parse_defalut_parameter_case1, ret = %r", ret) + + +def get_val_fn(x): + """ get_val_fn """ + ret = x + 3 + return ret + + +# Test case: test bool not +@ms_function +def bool_exp(x, y): + """ bool_exp """ + return not x > y + + +def test_bool_exp(): + """ test_bool_exp """ + bool_exp(1, 2) + + +# Test case: use the variable parameter for @mindspore +@ms_function +def var_parameter_f(x, *args): + """ var_parameter_f """ + z = x + args[0] + args[1] + args[2] + return z + + +def test_var_parameter_case1(): + """ test_var_parameter_case1 """ + log.debug("start test_var_parameter_case1") + var_parameter_f(1, 2, 3, 4, 5) + log.debug("end test_var_parameter_case1") + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self, value1): + super(Net, self).__init__() + self.relu = nn.ReLU() + self.softmax = nn.Softmax(0) + self.axis = 0 + self.TC = ClassTest("test_class", 1.2) + self.value = value1 + + @ms_function + def construct(self, x): + x = self.get_test_value(x) + return x + + def get_test_value(self, x): + ret = x + self.value + return ret + + +class ClassTest: + """ ClassTest definition """ + def __init__(self, name, value1): + self.name = name + self.value = value1 + + def get_name(self): + return self.name + + def get_value(self, inc): + ret = self.value + inc + return ret + + def __call__(self, *args, **kwargs): + pass + + +# Test: call method on parse graph code +@non_graph_engine +def test_call_method_on_construct(): + """ test_call_method_on_construct """ + log.debug("begin test_call_method_on_construct") + + x = Tensor(np.array([[1, 2, 3], [1, 2, 3]]).astype(np.int32)) + y = Tensor(np.array([[2, 3, 4], [1, 1, 2]]).astype(np.int32)) + z = np.array([[3, 5, 7], [2, 3, 5]]).astype(np.int32) + + net = Net(y) + output = net.construct(x) + result = output.asnumpy() + print(result) + assert np.all(result == z) + + log.debug("finished test_call_method_on_construct") + + +# Test: call method on parse graph code +class Net1(nn.Cell): + """ Net1 definition """ + def __init__(self, v1, v2): + super(Net1, self).__init__() + self.relu = nn.ReLU() + self.softmax = nn.Softmax(0) + self.axis = 0 + self.TC = ClassTest("test_class", v1) + self.value = v2 + + @ms_function + def construct(self, x): + x = x + self.TC.get_value(self.value) + return x + + +@non_graph_engine +def test_call_other_object_method(): + """ test_call_other_object_method """ + log.debug("begin test_call_other_object_method") + + x = Tensor(np.array([[1, 2, 3], [1, 2, 3]]).astype(np.int32)) + y = Tensor(np.array([[2, 3, 4], [1, 1, 2]]).astype(np.int32)) + y1 = Tensor(np.array([[5, 4, 5], [1, 1, 2]]).astype(np.int32)) + z = np.array([[8, 9, 12], [3, 4, 7]]).astype(np.int32) + + net = Net1(y, y1) + with pytest.raises(TypeError): + output = net.construct(x) + result = output.asnumpy() + print(result) + assert np.all(result == z) + + log.debug("finished test_call_other_object_method") + + +# Test: call global object method(not self) on parse graph code +value = Tensor(np.array([[3, 4, 5], [1, 1, 2]]).astype(np.int32)) +TC = ClassTest("test_class", value) + + +class Net2(nn.Cell): + """ Net2 definition """ + def __init__(self, value1): + super(Net2, self).__init__() + self.value = value1 + + @ms_function + def construct(self, x): + x = x + TC.get_value(self.value) + return x + + @ms_function + def construct1(self, x): + x = x + TC.value + x = x + self.value + return x + + +@non_graph_engine +def test_call_no_self_other_object_method(): + """ test_call_no_self_other_object_method """ + log.debug("begin test_call_other_object_method") + x = Tensor(np.array([[1, 2, 3], [1, 2, 3]]).astype(np.int32)) + y = Tensor(np.array([[2, 3, 4], [1, 1, 2]]).astype(np.int32)) + z = np.array([[6, 9, 12], [3, 4, 7]]).astype(np.int32) + + net = Net2(y) + with pytest.raises(TypeError): + output = net.construct(x) + result = output.asnumpy() + print(result) + assert np.all(result == z) + + log.debug("finished test_call_other_object_method") + + +def test_call_no_self_other_object_attr_value(): + """ test_call_no_self_other_object_attr_value """ + # do not support tensor as init input. + return + + +# Test case: use the * to unlock the varargs for @mindspore +def vararg1(x, y): + """ vararg1 """ + z = x + y + return z + + +def varargs_main(fn): + """ varargs_main """ + @ms_function + def t1(*args): + return fn(*args) + return t1 + + +def test_var_parameter_case3(): + """ test_var_parameter_case3 """ + log.debug("start test_var_parameter_case3") + ret = varargs_main(vararg1)(1, 2) + log.debug("ret = %r", ret) + log.debug("end test_var_parameter_case3") + + +# Test case: test the flag set +@core(tg=True) +def set_flag(x): + """ set_flag """ + return x + 1 + + +@ms_function +def set_test_flag_main(x, y): + """ set_test_flag_main """ + z = set_flag(x) + z = z + y + return z + + +def test_set_flag(): + """ Test default parameter function call """ + log.debug("begin test_set_flag") + ret = set_test_flag_main(2, 3) + log.debug("finished test_set_flag, ret = %r", ret) + + +@dataclass +class Access: + a: int + b: int + + def max(self): + if self.a > self.b: + return self.a + return self.b + +@ms_function +def invoke_dataclass(x, y): + """ invoke_dataclass """ + acs = Access(x, y) + return acs.max() + + +def test_access(): + """ test_access """ + invoke_dataclass(1, 2) + + +def myfunc(x): + """ myfunc """ + return x * x + + +@ms_function +def ms_infer_for(): + """ ms_infer_for """ + a = 0.0 + for x in [1.1, 2.3, 3.3]: + a = a + x + return a + + +def test_infer_for(): + """ test_infer_for """ + ms_infer_for() + + +@ms_function +def ms_infer_for_func(y): + """ ms_infer_for_func """ + for x in [1.0, 2.0, 3.0]: + y = myfunc(x) + y + return y + + +def test_ms_infer_for_func(): + """ test_ms_infer_for_func """ + ms_infer_for_func(1.0) + + +@ms_function +def add(x, y): + """ add """ + return x + y + + +def test_add(): + """ test_add """ + res = add(1, 2.0) + return res + + +@ms_function +def add_list(): + """ add_list """ + a = [1, 2, 3] + b = a[1] + a[2] + return b + + +def test_list(): + """ test_list """ + return add_list() + + +@ms_function +def compare_list_len(): + """ compare_list_len """ + a = [1, 2, 3] + return ms_len(a) + + +def test_list_len(): + """ test_list_len """ + compare_list_len() + + +@ms_function +def add_tuple(): + """ add_tuple """ + a = (1, 2, 3) + b = a[1] + a[2] + return b + + +def test_tuple(): + """ test_tuple """ + return add_tuple() + + +def invoke_func(x): + """ invoke_func """ + return x * x + + +@ms_function +def tuple_of_node(x, y): + """ tuple_of_node """ + a = invoke_func(x) + b = invoke_func(y) + c = (a, b) + d = c[1] * x + return d + + +def test_tuple_node(): + """ test_tuple_node """ + res = tuple_of_node(1, 2) + return res + + +@ms_function +def range_spec(x, y): + """ range_spec """ + for _ in range(1, 10, 3): + x = x + 1 + return x + y + + +def test_range(): + """ test_range """ + res = range_spec(10, 10) + return res diff --git a/tests/ut/python/pynative_mode/test_partial.py b/tests/ut/python/pynative_mode/test_partial.py new file mode 100644 index 00000000000..d9ba6995f2f --- /dev/null +++ b/tests/ut/python/pynative_mode/test_partial.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_partial """ +from mindspore.common.api import ms_function +import mindspore.ops.functional as F +import mindspore.ops.composite as C +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +def myadd(x, y): + return x + y + + +def partial_simple_add(x): + return F.partial(myadd, x) + + +@ms_function +def full_simple_add(x, y): + p = partial_simple_add(x) + return p(y) + + +def test_full_simple_add(): + print(full_simple_add(2, 5)) + + +# partial with multitype +MULTI_ADD = C.MultitypeFuncGraph('add') + + +@MULTI_ADD.register("Int32", "Int32") +def add_int(x, y): + return F.scalar_add(x, y) + + +@MULTI_ADD.register("Float32", "Float32") +def add_float(x, y): + return F.scalar_add(x, y) + + +def partial_multi_add(x): + return F.partial(MULTI_ADD, x) + + +@ms_function +def full_multi_add(x, y, m, n): + p = partial_multi_add(x)(y) + q = partial_multi_add(m)(n) + return p, q + + +def test_full_multi_add(): + print(full_multi_add(1, 2, 1.0, 2.0)) diff --git a/tests/ut/python/pynative_mode/test_pynative_model.py b/tests/ut/python/pynative_mode/test_pynative_model.py new file mode 100644 index 00000000000..98a7a8db7b8 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_pynative_model.py @@ -0,0 +1,138 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_pynative_model """ +import numpy as np + +import mindspore.nn as nn +from mindspore import Parameter, ParameterTuple, Tensor +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.nn.optim import Momentum +from ..ut_filter import non_graph_engine +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +class GradWrap(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + self.weights = ParameterTuple(network.get_parameters()) + + def construct(self, x, label): + weights = self.weights + return C.grad_by_list(self.network, weights)(x, label) + + +@non_graph_engine +def test_softmaxloss_grad(): + """ test_softmaxloss_grad """ + class NetWithLossClass(nn.Cell): + """ NetWithLossClass definition """ + def __init__(self, network): + super(NetWithLossClass, self).__init__() + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10]).astype(np.float32)), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name="bias") + self.fc = P.MatMul() + self.biasAdd = P.BiasAdd() + + def construct(self, x): + x = self.biasAdd(self.fc(x, self.weight), self.bias) + return x + + net = GradWrap(NetWithLossClass(Net())) + + predict = Tensor(np.ones([1, 64]).astype(np.float32)) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + print("pynative run") + out = net.construct(predict, label) + print("out:", out) + print(out[0], (out[0]).asnumpy(), ":result") + + +@non_graph_engine +def test_lenet_grad(): + """ test_lenet_grad """ + class NetWithLossClass(nn.Cell): + """ NetWithLossClass definition """ + def __init__(self, network): + super(NetWithLossClass, self).__init__() + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.network = network + + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + class LeNet5(nn.Cell): + """ LeNet5 definition """ + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + input_data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([1, 10]).astype(np.float32)) + iteration_num = 1 + verification_step = 0 + + net = LeNet5() + loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False) + momen_opti = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + train_net = GradWrap(NetWithLossClass(net)) + train_net.set_train() + + for i in range(0, iteration_num): + # get the gradients + grads = train_net(input_data, label) + # update parameters + success = momen_opti(grads) + if success is False: + print("fail to run optimizer") + # verification + if i == verification_step: + fw_output = net.construct(input_data) + loss_output = loss.construct(fw_output, label) + print("The loss of %s-th iteration is %s" % (i, loss_output.asnumpy())) diff --git a/tests/ut/python/pynative_mode/test_remove_unnecessary_phi.py b/tests/ut/python/pynative_mode/test_remove_unnecessary_phi.py new file mode 100644 index 00000000000..ddd729b250a --- /dev/null +++ b/tests/ut/python/pynative_mode/test_remove_unnecessary_phi.py @@ -0,0 +1,92 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_remove_unnecessary_phi """ +# coding: utf-8 + +from numpy.random import normal +from mindspore import Tensor +from mindspore.ops.composite import core +from mindspore.common.api import ms_function +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +# If phi node (fv_func) in for body is not replaced by fv_func, +# this test will failed as phi node as a parameter will be inferred +# as POLY. +def test_remove_phi_and_fv(): + """ test_remove_phi_and_fv """ + @ms_function + @core(loop_can_unroll=True) + def loop(x, input_data): + def fv_func(y): + return x * y + + ret = () + for inp in input_data: + ele = fv_func(inp) + ret += (ele,) + return ret + + input_data = (Tensor(normal(0, 0.1, (3, 3))), Tensor(normal(0, 0.1, (3)))) + input1 = Tensor(normal(0, 0.1, (3, 3))) + print(loop(input1, input_data)) + +# Multiple phi nodes should be replaced. +# mul Φ0 (mul, Φ0); Φ0 will be replaced by mul; +# x Φ1 (x, Φ1); Φ1 will be replaced by x; +# ret Φ2 (1, ret{[0]: Φ0, [1]: Φ1, [2]: inp}), Φ2 will not be replaced. + +# Φ0 and Φ1 in Φ2 node should be replaced with mul and x. +def test_remove_multiple_phi(): + """ test_remove_multiple_phi """ + @ms_function + @core(loop_can_unroll=True) + def loop(x): + def mul(a, b): + return a * b + + ret = 1 + for inp in range(3): + ret = mul(x, inp) + return ret + + print(loop(2)) + + +# replace phi nodes recursively. +# x as phi Φ5 (Φ3, Φ3) in graph ↓⥁loop +# x as phi Φ3 (x, Φ5) in graph ⤾loop +# one predecessor of ⤾loop is ↓⥁loop. +# Φ5 will be resolved first, it can be replace by Φ3, then Φ3 (x, Φ5) become Φ3 (x, Φ3), so Φ3 can be replaced by x. +# recursively, Φ5 also should be replaced by x. +def test_remove_multiple_phi_recursive(): + """ test_remove_multiple_phi_recursive """ + @ms_function + @core(loop_can_unroll=True) + def loop(x): + def mul(a, b): + return a * b + + ret = 1 + for inp in range(3): + if inp % 2 == 0: + ret = mul(inp, inp) + return ret * x + + print(loop(2)) diff --git a/tests/ut/python/pynative_mode/test_staging.py b/tests/ut/python/pynative_mode/test_staging.py new file mode 100644 index 00000000000..fc120e7c00c --- /dev/null +++ b/tests/ut/python/pynative_mode/test_staging.py @@ -0,0 +1,192 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_staging """ +import numpy as np +import pytest +import mindspore as ms +import mindspore.nn as nn +from mindspore.common.api import ms_function +from mindspore.common import MetaTensor +from mindspore.common import dtype +from mindspore import Tensor +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common.dtype import get_py_obj_dtype +from ..ut_filter import non_graph_engine +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +@ms_function +def tensor_add_func_inner(x, y): + """ tensor_add_func_inner """ + z = F.tensor_add(x, y) + return z + +@ms_function +def tensor_add_func(x, y): + """ tensor_add_func """ + z = tensor_add_func_inner(x, y) + z = F.tensor_add(z, x) + return z + + +@ms_function +def scalar_add(x, y): + """ scalar_add """ + return x + y + + +@ms_function +def scalar_add_if(x, y): + """ scalar_add_if """ + if x > y: + return x + y + 10 + return x + y + 20 + + +@ms_function +def scalar_mul_while(x): + """ scalar_mul_while """ + rv = x + while rv < 100: + rv = rv * rv + return rv + + +@ms_function(input_signature=(MetaTensor(dtype.float32, (1, 1, 3, 3)), + MetaTensor(dtype.float32, (1, 1, 3, 3)))) +def tensor_add_test(x, y): + """ tensor_add_test """ + z = F.tensor_add(x, y) + return z + + +class TensorAddMulNet(nn.Cell): + """ TensorAddMulNet definition """ + def __init__(self): + super(TensorAddMulNet, self).__init__() + self.add = P.TensorAdd() + + @ms_function + def add_stage0(self, x, y): + z = self.add(x, y) + z = self.add(x, z) + return z + + @ms_function + def add_stage1(self, x, y): + z = self.add(x, y) + z = self.add(x, z) + return z + + def construct(self, x, y): + z = self.add(x, y) # PyNative mode + z = self.add_stage0(x, z) # Graph mode + z = self.add(x, z) # PyNative mode + z = self.add_stage1(y, z) # Graph mode + return z + + +class TensorAddNet(nn.Cell): + """ TensorAddNet definition """ + def __init__(self): + super(TensorAddNet, self).__init__() + self.add = P.TensorAdd() + + @ms_function + def compute(self, x, y): + return self.add(x, y) + + def construct(self, x, y): + z = self.compute(x, y) + return z + + +def test_control_func(): + """ test_control_func """ + res = scalar_add(3, 4) + assert res == 7 + + res = scalar_add_if(3, 4) + assert res == 27 + + res = scalar_mul_while(2) + assert res == 256 + + +@non_graph_engine +def test_staging_call_func(): + """ test_staging_call_func """ + x = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)) + output = tensor_add_func(x, y) + assert (output.asnumpy() == (np.ones([1, 1, 3, 3]) * 3)).all() + + +@non_graph_engine +def test_class_method_staging(): + """ test_class_method_staging """ + x = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)) + y = Tensor(np.ones([1, 1, 3, 3]).astype(np.float32)) + net = TensorAddNet() + output = net.construct(x, y) + assert (output.asnumpy() == (np.ones([1, 1, 3, 3]) * 2)).all() + + +@non_graph_engine +def test_class_method_composite_staging(): + """ test_class_method_composite_staging """ + x = Tensor(np.ones([3, 3]).astype(np.float32)) + y = Tensor(np.ones([3, 3]).astype(np.float32)) + net = TensorAddMulNet() + output = net.construct(x, y) + assert (output.asnumpy() == (np.ones([3, 3]) * 7)).astype(np.float32).all() + + +@non_graph_engine +def test_input_signature(): + """ test_input_signature """ + x1 = Tensor(np.ones([1, 1, 3, 3], dtype=np.float32)) + y1 = Tensor(np.ones([1, 1, 3, 3], dtype=np.float32)) + output = tensor_add_test(x1, y1) + assert (output.asnumpy() == (np.ones([1, 1, 3, 3]) * 2)).all() + # test input type signature + x2 = Tensor(np.ones([1, 1, 3, 3], dtype=np.float64)) + y2 = Tensor(np.ones([1, 1, 3, 3], dtype=np.float64)) + with pytest.raises(ValueError): + tensor_add_test(x2, y2) + # test input shape signature + x3 = Tensor(np.ones([1, 1, 4, 4], dtype=np.float64)) + y3 = Tensor(np.ones([1, 1, 4, 4], dtype=np.float64)) + with pytest.raises(ValueError): + tensor_add_test(x3, y3) + +def test_scalar_cast(): + """ test_scalar_cast """ + input_x = 8.5 + input_t = get_py_obj_dtype(ms.int64) + + @ms_function + def fn_cast(x, t): + output = F.scalar_cast(x, t) + return output + + expect_value = 8 + z = fn_cast(input_x, input_t) + assert z == expect_value diff --git a/tests/ut/python/pynative_mode/test_stop_gradient.py b/tests/ut/python/pynative_mode/test_stop_gradient.py new file mode 100644 index 00000000000..b274b3988ad --- /dev/null +++ b/tests/ut/python/pynative_mode/test_stop_gradient.py @@ -0,0 +1,368 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_stop_gradient """ +import pytest +import numpy as np + +import mindspore.nn as nn +from mindspore import context +import mindspore.common.dtype as mstype +from mindspore import Parameter, ParameterTuple, Tensor +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.ops.functional import stop_gradient +from mindspore.ops.primitive import prim_attr_register, PrimitiveWithInfer +from ..ut_filter import non_graph_engine +from ....mindspore_test_framework.utils.bprop_util import bprop +from mindspore import Tensor +from mindspore.common.api import ms_function +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +def stop_func(x, y): + """ stop_func""" + c = x*y + c_s = x + y + return c_s, c + +def stop_test1(x, y): + """ stop_test1 """ + c = x*y + c_s = stop_gradient(c) + return c_s + +def stop_test2(x, y): + """ stop_test2 """ + c = x*y + c_s = stop_gradient(c) + d = c_s+x*y + return d * y + +def stop_test3(x, y): + """ stop_test3 """ + x = x*y + z = stop_test1(x, y) + k = z * y + return k + +def stop_test5(x, y): + """ stop_test3 """ + x = x+y + o1, o2= stop_func(x, y) + c = stop_gradient(o1) + c = o2+c + return c + +def stop_test4(x, y): + """ stop_test4 """ + c = x + y + c_s = stop_gradient(c) + e = c + c_s + return e + +def grad_stop_test(x, y): + """ grad_stop_test """ + return C.grad_all(stop_test2)(x, y) + +def grad_stop_test1(x, y): + """ grad_stop_test1 """ + return C.grad_all(stop_test3)(x, y) + +def test_stop(): + """ test_stop """ + print("test_stop:", grad_stop_test(1, 1)) + +def test_stop1(): + """ test_stop1 """ + print("test_stop1:", grad_stop_test1(2, 3)) + +def test_stop5(): + """ test_stop1 """ + print("test_stop5:", C.grad_all(stop_test5)(2, 3)) + +class GradWrap(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + self.weights = ParameterTuple(network.get_parameters()) + + @ms_function + def construct(self, x, label): + weights = self.weights + return C.grad_by_list(self.network, weights)(x, label) + + +@non_graph_engine +def test_softmaxloss_grad(): + """ test_softmaxloss_grad """ + class NetWithLossClass(nn.Cell): + """ NetWithLossClass definition """ + def __init__(self, network): + super(NetWithLossClass, self).__init__() + self.loss = nn.SoftmaxCrossEntropyWithLogits() + self.network = network + + @ms_function + def construct(self, x, label): + predict = self.network(x) + return self.loss(predict, label) + + class Net(nn.Cell): + """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.weight = Parameter(Tensor(np.ones([64, 10])), name="weight") + self.bias = Parameter(Tensor(np.ones([10]).astype(np.float32)), name="bias") + self.fc = P.MatMul() + self.fc2 = nn.Dense(10, 10) + self.biasAdd = P.BiasAdd() + self.relu = nn.ReLU() + self.cast = P.Cast() + + @ms_function + def construct(self, x): + x = self.fc(x, self.weight) + x = self.cast(x, mstype.float32) + x = self.relu(self.fc2(x)) + x = self.fc2(x) + x = stop_gradient(x) + x = self.biasAdd(x, self.bias) + return x + + net = GradWrap(NetWithLossClass(Net())) + + predict = Tensor(np.ones([1, 64])) + label = Tensor(np.zeros([1, 10]).astype(np.float32)) + print("pynative run") + out = net.construct(predict, label) + print("out:", out) + +def test_stop_gradient_1(): + class Mul(nn.Cell): + def __init__(self): + super(Mul, self).__init__() + + @ms_function + def construct(self, x, y): + ret = x * y + ret = stop_gradient(ret) + return ret + dx, dy = bprop(Mul(), Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32)), wrt=['inputs']) + expect = np.zeros([2, 2]) + assert (dx.asnumpy() == expect).all() + assert (dy.asnumpy() == expect).all() + +def test_stop_gradient_2(): + class Mul(nn.Cell): + def __init__(self): + super(Mul, self).__init__() + + @ms_function + def construct(self, x, y): + c = x * y + z = x * y + return c, z + class MulAdd(nn.Cell): + def __init__(self): + super(MulAdd, self).__init__() + self.mul = Mul() + + @ms_function + def construct(self, x, y): + u = x + y + v = x - y + c, z = self.mul(u, v) + c = stop_gradient(c) + ret1 = c + x + y + ret2 = z + y + y + return ret1, ret2 + dx = bprop(MulAdd(), Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))) + expect = np.array([[3.0, 3.0], [3.0, 3.0]]) + assert (dx.asnumpy() == expect).all() + +def test_stop_gradient_3(): + class TupleGetItem(nn.Cell): + def __init__(self): + super(TupleGetItem, self).__init__() + + @ms_function + def construct(self, x1, x2, x3, x4, x5): + z1 = x1 + x1 + z2 = x1 * x2 + t = (z1, z2, x3, x4, x5) + z2 = t[1] + z2 = stop_gradient(z2) + return z1, z2, x3, x4, x5 + dx = bprop(TupleGetItem(), + Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32))) + expect = np.array([[2.0, 2.0], [2.0, 2.0]]) + assert (dx.asnumpy() == expect).all() + +def test_stop_gradient_4(): + def stop_test(x): + return stop_gradient(x) + assert C.grad_all(stop_test)(1) == (0,) + +def test_stop_gradient_5(): + def stop_test(x): + y = x + x + y = stop_gradient(y) + ret = x + y + return ret + assert C.grad_all(stop_test)(1) == (1,) + +def test_stop_gradient_6(): + def stop_test(x, y): + ret = x * y + ret = stop_gradient(ret) + return ret + assert C.grad_all(stop_test)(1, 3) == (0, 0) + +class PrimWithMultiOutputs(PrimitiveWithInfer): + @prim_attr_register + def __init__(self): + """init""" + def __call__(self, x, y): + """Implement by vm mode.""" + return x, y + def infer_shape(self, x_shape, y_shape): + return x_shape, y_shape + def infer_dtype(self, x_type, y_type): + return x_type, y_type + def get_bprop(self): + def bprop(x, y, out, dout): + return (dout[0], dout[1]) + return bprop + +def test_stop_gradient_7(): + class PrimWithMultiOutputs_(nn.Cell): + def __init__(self): + super(PrimWithMultiOutputs_, self).__init__() + self.prim_with_multi_outputs = PrimWithMultiOutputs() + + @ms_function + def construct(self, x1, x2): + x1, x2 = self.prim_with_multi_outputs(x1, x2) + x1 = stop_gradient(x1) + return x1, x2 + dx, dy = bprop(PrimWithMultiOutputs_(), Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32)), wrt=['inputs']) + expect_dx = np.zeros([2]) + expect_dy = np.ones([2]) + assert (dx.asnumpy() == expect_dx).all() + assert (dy.asnumpy() == expect_dy).all() + +def test_stop_gradient_8(): + class PrimWithMultiOutputs_(nn.Cell): + def __init__(self): + super(PrimWithMultiOutputs_, self).__init__() + self.prim_with_multi_output = PrimWithMultiOutputs() + + @ms_function + def construct(self, x1, x2): + x1, x2 = stop_gradient(self.prim_with_multi_output(x1, x2)) + return x1, x2 + dx, dy = bprop(PrimWithMultiOutputs_(), Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32)), wrt=['inputs']) + expect_dx = np.zeros([2]) + expect_dy = np.zeros([2]) + assert (dx.asnumpy() == expect_dx).all() + assert (dy.asnumpy() == expect_dy).all() + +def test_stop_gradient_9(): + class Mul(nn.Cell): + def __init__(self): + super(Mul, self).__init__() + + @ms_function + def construct(self, x, y): + c = x * y + z = x * y + return c, z + class MulAdd(nn.Cell): + def __init__(self): + super(MulAdd, self).__init__() + self.mul = Mul() + + @ms_function + def construct(self, x, y): + u = x + y + v = x - y + c, z = self.mul(u, v) + c1 = stop_gradient(c) + c2 = c + ret1 = c1 + x + y + c2 + ret2 = z + y + y + return ret1, ret2 + dx = bprop(MulAdd(), Tensor(np.ones([2, 2]).astype(np.float32)), + Tensor(np.ones([2, 2]).astype(np.float32))) + expect = np.array([[5.0, 5.0], [5.0, 5.0]]) + assert (dx.asnumpy() == expect).all() + +class PrimWithNoBprop(PrimitiveWithInfer): + @prim_attr_register + def __init__(self): + """init""" + def __call__(self, x, y): + """Implement by vm mode.""" + return x, y + def infer_shape(self, x_shape, y_shape): + return x_shape, y_shape + def infer_dtype(self, x_type, y_type): + return x_type, y_type + +def test_stop_gradient_10(): + class PrimWithNoBprop_(nn.Cell): + def __init__(self): + super(PrimWithNoBprop_, self).__init__() + self.prim_with_no_bprop = PrimWithNoBprop() + + @ms_function + def construct(self, x, y): + x = x * y + x, y = self.prim_with_no_bprop(x, y) + x = stop_gradient(x) + y = stop_gradient(y) + return x, y + dx = bprop(PrimWithNoBprop_(), Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32))) + expect_dx = np.zeros([2]) + assert (dx.asnumpy() == expect_dx).all() + +def test_stop_gradient_11(): + class PrimWithNoBprop_(nn.Cell): + def __init__(self): + super(PrimWithNoBprop_, self).__init__() + self.prim_with_no_bprop = PrimWithNoBprop() + + @ms_function + def construct(self, x, y): + x, y = self.prim_with_no_bprop(x, y) + x = stop_gradient(x) + return x, y + with pytest.raises(RuntimeError): + bprop(PrimWithNoBprop_(), Tensor(np.ones([2]).astype(np.float32)), + Tensor(np.ones([2]).astype(np.float32))) diff --git a/tests/ut/python/pynative_mode/test_training.py b/tests/ut/python/pynative_mode/test_training.py new file mode 100644 index 00000000000..c8af5d839c4 --- /dev/null +++ b/tests/ut/python/pynative_mode/test_training.py @@ -0,0 +1,77 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_training """ +import numpy as np + +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from mindspore.nn.optim import Momentum +from mindspore.train.model import Model +from mindspore.nn import WithGradCell, WithLossCell +from ..ut_filter import non_graph_engine +from mindspore import context + + +def setup_module(module): + context.set_context(mode=context.PYNATIVE_MODE) + + +class LeNet5(nn.Cell): + """ LeNet5 definition """ + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +@non_graph_engine +def test_loss_cell_wrapper(): + """ test_loss_cell_wrapper """ + data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([1, 10]).astype(np.float32)) + net = LeNet5() + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + loss_net = WithLossCell(net, loss_fn) + loss_out = loss_net(data, label) + assert loss_out.asnumpy().dtype == 'float32' or loss_out.asnumpy().dtype == 'float64' + + +@non_graph_engine +def test_grad_cell_wrapper(): + """ test_grad_cell_wrapper """ + data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + label = Tensor(np.ones([1, 10]).astype(np.float32)) + dout = Tensor(np.ones([1]).astype(np.float32)) + net = LeNet5() + loss_fn = nn.SoftmaxCrossEntropyWithLogits() + grad_net = WithGradCell(net, loss_fn, dout) + gradients = grad_net(data, label) + assert isinstance(gradients[0].asnumpy()[0][0][0][0], (np.float32, np.float64)) diff --git a/tests/ut/python/pynative_mode/vm/__init__.py b/tests/ut/python/pynative_mode/vm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/pynative_mode/vm/test_vm.py b/tests/ut/python/pynative_mode/vm/test_vm.py new file mode 100644 index 00000000000..4ea0abd7538 --- /dev/null +++ b/tests/ut/python/pynative_mode/vm/test_vm.py @@ -0,0 +1,272 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_vm """ +import numpy as np +from .....vm_impl import vm + + +def test_avg_pooling(): + """ test_avg_pooling """ + input_data = np.array([[[[-4., -3., 1., 9.], + [-9., -1., 3., 4.], + [1., -1., -3., -6.], + [-2., -1., -2., -15.]]]]).astype(np.float32) + out = vm.avg_pooling(input_data, pool_h=2, pool_w=2, stride=1, pad=0) + expect_out = [[[[-4.25, 0.0, 4.25], + [-2.5, -0.5, -0.5], + [-0.75, -1.75, -6.5]]]] + assert (expect_out == out).all() + + +def test_avg_pool_grad(): + """ test_avg_pool_grad """ + # To do + input_data = np.array([[[[1., 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16]]]]).astype(np.float32) + dout = vm.avg_pooling(input_data, pool_h=2, pool_w=2, stride=1, pad=0) + print("vm.avg_pooling dout: ", dout) + out = vm.avg_pool_grad(dout, input_data.shape, 2, 2, 1, 0) + print("vm.avg_pool_grad: ", out) + assert True + + +def test_batch_norm(): + """ test_batch_norm """ + input_data = np.random.randint(0, 255, [1, 3, 224, 224]) + print("input_data.shape: ", input_data.shape) + print("input_data: ", input_data) + output = vm.batch_norm(input_data) + print("vm.batch_norm: ", output) + + +def test_conv2d(): + """ test_conv2d """ + x = np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32) + weight = np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32) + out = vm.conv2d(x, weight) + expect_out = np.array([[[ + [-5., -4., 0., 8.], + [-10., -2., 2., 3.], + [0., -2., -4., -7.], + [-3., -2., -3., -16.]]]]).astype(np.float32) + assert (expect_out == out).all() + + +def test_conv2d_with_bias(): + """ test_conv2d_with_bias """ + x = np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32) + weight = np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32) + bias = np.array([1]).astype(np.float32) + out = vm.conv2d(x, weight, bias) + expect_out = np.array([[[ + [-4., -3., 1., 9.], + [-9., -1., 3., 4.], + [1., -1., -3., -6.], + [-2., -1., -2., -15.]]]]).astype(np.float32) + assert (expect_out == out).all() + + +def test_conv2d_backprop_filter(): + """ test_conv2d_backprop_filter """ + x = np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32) + weight = np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32) + out = vm.conv2d(x, weight) + backprop_filter = vm.conv2d_backprop_filter(out, x, weight.shape) + print(backprop_filter) + assert True + + +def test_conv2d_backprop_input(): + """ test_conv2d_backprop_input """ + x = np.array([[[ + [3, 0, 1, 2, 7, 4], + [1, 5, 8, 9, 3, 1], + [2, 7, 2, 5, 1, 3], + [0, 1, 3, 1, 7, 8], + [4, 2, 1, 6, 2, 8], + [2, 4, 5, 2, 3, 9]]]]).astype(np.float32) + weight = np.array([[[[1, 0, -1], [1, 0, -1], [1, 0, -1]]]]).astype(np.float32) + out = vm.conv2d(x, weight) + grad = vm.conv2d_backprop_input(out, x.shape, weight) + print(grad) + assert True + + +def test_flatten(): + """ test_flatten """ + x = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) + y = vm.flatten(x) + assert ([1, 2, 3, 4, 5, 6] == y.T).all() + assert np.float32 == y.dtype + + +def test_flatten2(): + """ test_flatten2 """ + x = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) + y = vm.flatten2(x) + assert ([1, 2, 3, 4, 5, 6] == y).all() + assert (1, 6) == y.shape + assert np.float32 == y.dtype + + +def test_flatten_batch(): + """ test_flatten_batch """ + x = np.array([[[9, 4, 14, 1], + [7, 10, 14, 13], + [1, 9, 16, 7], + [15, 16, 0, 4]], + [[16, 13, 13, 10], + [0, 12, 5, 9], + [15, 0, 11, 1], + [4, 16, 4, 1]], + [[2, 8, 1, 13], + [5, 15, 4, 11], + [8, 2, 17, 16], + [5, 13, 0, 2]], + [[14, 8, 6, 8], + [0, 8, 6, 15], + [9, 1, 8, 5], + [12, 6, 13, 8]], + [[13, 11, 6, 3], + [8, 6, 16, 5], + [7, 10, 0, 8], + [17, 17, 17, 3]]]).astype(np.float32) + y = vm.flatten_batch(x) + expect_out = np.array( + [[9, 4, 14, 1, 7, 10, 14, 13, 1, 9, 16, 7, 15, 16, 0, 4], + [16, 13, 13, 10, 0, 12, 5, 9, 15, 0, 11, 1, 4, 16, 4, 1], + [2, 8, 1, 13, 5, 15, 4, 11, 8, 2, 17, 16, 5, 13, 0, 2], + [14, 8, 6, 8, 0, 8, 6, 15, 9, 1, 8, 5, 12, 6, 13, 8], + [13, 11, 6, 3, 8, 6, 16, 5, 7, 10, 0, 8, 17, 17, 17, 3]]).astype(np.float32) + assert (expect_out == y).all() + assert expect_out.shape == y.shape + assert np.float32 == y.dtype + + +def test_im2col(): + """ test_im2col """ + img = np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01 + print("input img: ", img) + col = vm.im2col(img, 2, 3, 1, 1) + print("output col.shape : ", col.shape) + print("output col: ", col) + print("output col.dtype: ", col.dtype) + assert np.float32 == col.dtype + + +def test_matmul(): + """ test_matmul """ + x = np.array([1, 2, 3]).astype(np.float32) + w = np.array([0, 1, 0.5]).astype(np.float32) + y = vm.matmul(x, w) + assert y == 3.5 + assert np.float32 == y.dtype + + +def test_max_pooling(): + """ test_max_pooling """ + input_data = np.array([[[ + [-4., -3., 1., 9.], + [-9., -1., 3., 4.], + [1., -1., -3., -6.], + [-2., -1., -2., -15.]]]]).astype(np.float32) + out = vm.max_pooling(input_data, pool_h=2, pool_w=2, stride=1, pad=0) + expect_out = [[[[-1., 3., 9.], + [1., 3., 4.], + [1., -1., -2.]]]] + assert (expect_out == out).all() + assert np.float32 == out.dtype + + +def test_np_convolve(): + """ test_np_convolve """ + out = np.convolve([1, 2, 3], [0, 1, 0.5]).astype(np.float32) + assert ([0.0, 1.0, 2.5, 4.0, 1.5] == out).all() + assert np.float32 == out.dtype + + +def test_np_convolve_same(): + """ test_np_convolve_same """ + out = np.convolve([1, 2, 3], [0, 1, 0.5], 'same').astype(np.float32) + assert ([1.0, 2.5, 4.0] == out).all() + assert np.float32 == out.dtype + + +def test_np_convolve_valid(): + """ test_np_convolve_valid """ + out = np.convolve([1, 2, 3], [0, 1, 0.5], 'valid').astype(np.float32) + assert ([2.5] == out).all() + assert np.float32 == out.dtype + + +def test_relu(): + """ test_relu """ + x = np.array([-0.32208174, 0.33999891]).astype(np.float32) + y = vm.relu(x) + assert np.allclose([-0., 0.33999891], y) + assert np.float32 == y.dtype + + y = vm.relu_grad(y) + assert (y == [0., 1.]).all() + assert np.float32 == y.dtype + + +def test_softmax(): + """ test_softmax """ + logits = 2.84806275*np.ones([1, 10]).astype(np.float32) + y = vm.softmax(logits) + assert np.allclose([0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], y) + assert np.float32 == y.dtype + + logits = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) + y = vm.softmax(logits, axis=1) + labels = [[0.09003057, 0.24472847, 0.66524096], [0.09003057, 0.24472847, 0.66524096]] + assert np.allclose(labels, y) + assert np.float32 == y.dtype + + +def test_softmax_cross_entropy_with_logit(): + """ test_softmax_cross_entropy_with_logit """ + logits = np.array([[1, 2, 3, 4, 2, 1, 0, 2, 1, 1], [1, 2, 4, 1, 0, 5, 0, 2, 1, 3]], dtype=np.float32) + labels = np.array([[0, 0, 0, 1, 0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]], dtype=np.float32) + loss, dx = vm.softmax_cross_entropy_with_logits(logits, labels) + print("logits.shape: ", logits.shape) + print("logits: ", logits) + print("softmax: ", vm.softmax(logits)) + print("labels: ", labels) + print("loss: ", loss) + print("dx: ", dx) + assert np.float32 == loss.dtype + assert np.float32 == dx.dtype diff --git a/tests/ut/python/runtest.sh b/tests/ut/python/runtest.sh new file mode 100755 index 00000000000..d1c32998217 --- /dev/null +++ b/tests/ut/python/runtest.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +CURRPATH=$(cd $(dirname $0); pwd) +cd ${CURRPATH} +PROJECT_PATH=${CURRPATH}/../../.. +if [ $BUILD_PATH ];then + echo "BUILD_PATH = $BUILD_PATH" +else + BUILD_PATH=${PROJECT_PATH}/build + echo "BUILD_PATH = $BUILD_PATH" +fi + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${BUILD_PATH}/third_party/gtest/lib +export PYTHONPATH=$PYTHONPATH:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/cpp/python_input:${PROJECT_PATH}/tests/ut/python +echo "export PYTHONPATH=$PYTHONPATH" + +IGNORE_EXEC="" +if [ "x${ENABLE_GE}" == "xON" -o "x${ENABLE_GE}" == "xOn" -o "x${ENABLE_GE}" == "xon" -o \ + "x${ENABLE_GE}" == "xTrue" -o "x${ENABLE_GE}" == "xtrue" ]; then + if [ $# -gt 0 ]; then + IGNORE_EXEC="--ignore=$1/exec" + else + IGNORE_EXEC="--ignore=$CURRPATH/exec" + fi +fi + +if [ $# -gt 0 ]; then + pytest -s --ignore=$1/pynative_mode $IGNORE_EXEC $1 +else + pytest --ignore=$CURRPATH/pynative_mode $IGNORE_EXEC $CURRPATH +fi + +RET=$? +if [ "x${IGNORE_EXEC}" != "x" ]; then + exit ${RET} +fi + +if [ ${RET} -ne 0 ]; then + exit ${RET} +fi + +if [ $# -gt 0 ]; then + pytest -s $1/pynative_mode +else + pytest $CURRPATH/pynative_mode +fi + +RET=$? +if [ ${RET} -ne 0 ]; then + exit ${RET} +fi diff --git a/tests/ut/python/runtest_opensrc.sh b/tests/ut/python/runtest_opensrc.sh new file mode 100644 index 00000000000..856d513f688 --- /dev/null +++ b/tests/ut/python/runtest_opensrc.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +CURRPATH=$(cd $(dirname $0); pwd) +PROJECT_PATH=${CURRPATH}/../.. +if [ $BUILD_PATH ];then + echo "BUILD_PATH = $BUILD_PATH" +else + BUILD_PATH=${PROJECT_PATH}/build + echo "BUILD_PATH = $BUILD_PATH" +fi +cd ${BUILD_PATH}/mindspore/tests/ut + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${BUILD_PATH}/third_party/gtest/lib +export PYTHONPATH=$PYTHONPATH:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/python_input:${PROJECT_PATH}/tests/python +echo "export PYTHONPATH=$PYTHONPATH:${PROJECT_PATH}:${PROJECT_PATH}/tests/ut/python_input" + +if [ $# -gt 0 ]; then + pytest -s --ignore=$1/pynative_mode $1 +else + pytest $CURRPATH/train $CURRPATH/infer $CURRPATH/model $CURRPATH/core $CURRPATH/adapter $CURRPATH/custom_ops $CURRPATH/data $CURRPATH/distributed $CURRPATH/nn $CURRPATH/exec $CURRPATH/optimizer --ignore=$CURRPATH/custom_ops/test_cus_conv2d.py --ignore=$CURRPATH/core/test_tensor.py --ignore=$CURRPATH/model/test_vgg.py --ignore=$CURRPATH/engine/test_training.py --ignore=$CURRPATH/nn/test_dense.py --ignore=$CURRPATH/nn/test_embedding.py --ignore=$CURRPATH/nn/test_conv.py --ignore=$CURRPATH/nn/test_activation.py --ignore=$CURRPATH/core/test_tensor_py.py +fi + +RET=$? +if [ ${RET} -ne 0 ]; then + exit ${RET} +fi + +if [ $# -gt 0 ]; then + pytest -s $1/pynative_mode +else + pytest $CURRPATH/pynative_mode +fi + +RET=$? +if [ ${RET} -ne 0 ]; then + exit ${RET} +fi diff --git a/tests/ut/python/test_data/file/a.jpg b/tests/ut/python/test_data/file/a.jpg new file mode 100644 index 00000000000..762a3920bb2 Binary files /dev/null and b/tests/ut/python/test_data/file/a.jpg differ diff --git a/tests/ut/python/test_data/file/b.jpg b/tests/ut/python/test_data/file/b.jpg new file mode 100644 index 00000000000..762a3920bb2 Binary files /dev/null and b/tests/ut/python/test_data/file/b.jpg differ diff --git a/tests/ut/python/test_data/file/c.jpg b/tests/ut/python/test_data/file/c.jpg new file mode 100644 index 00000000000..762a3920bb2 Binary files /dev/null and b/tests/ut/python/test_data/file/c.jpg differ diff --git a/tests/ut/python/test_data/file/e.html b/tests/ut/python/test_data/file/e.html new file mode 100644 index 00000000000..566549bdf8f --- /dev/null +++ b/tests/ut/python/test_data/file/e.html @@ -0,0 +1,10 @@ + + + + + Title + + + + + \ No newline at end of file diff --git a/tests/ut/python/test_data/file/f.jpg b/tests/ut/python/test_data/file/f.jpg new file mode 100644 index 00000000000..eac293920cb Binary files /dev/null and b/tests/ut/python/test_data/file/f.jpg differ diff --git a/tests/ut/python/test_data/file/test1.txt b/tests/ut/python/test_data/file/test1.txt new file mode 100644 index 00000000000..89e9a15a84c --- /dev/null +++ b/tests/ut/python/test_data/file/test1.txt @@ -0,0 +1,4 @@ + a.jpg +b.jpg + +c.jpg diff --git a/tests/ut/python/test_data/file/test2.txt b/tests/ut/python/test_data/file/test2.txt new file mode 100644 index 00000000000..b497e4f716f --- /dev/null +++ b/tests/ut/python/test_data/file/test2.txt @@ -0,0 +1,4 @@ + a.jpg +b.jpg + +d.jpg diff --git a/tests/ut/python/test_data/file/test3.txt b/tests/ut/python/test_data/file/test3.txt new file mode 100644 index 00000000000..74c856965db --- /dev/null +++ b/tests/ut/python/test_data/file/test3.txt @@ -0,0 +1 @@ +e.html \ No newline at end of file diff --git a/tests/ut/python/test_data/file/test4.txt b/tests/ut/python/test_data/file/test4.txt new file mode 100644 index 00000000000..f7e1cb4fde1 --- /dev/null +++ b/tests/ut/python/test_data/file/test4.txt @@ -0,0 +1 @@ +f.jpg \ No newline at end of file diff --git a/tests/ut/python/test_data/imagenet_empty/a.jpg b/tests/ut/python/test_data/imagenet_empty/a.jpg new file mode 100644 index 00000000000..762a3920bb2 Binary files /dev/null and b/tests/ut/python/test_data/imagenet_empty/a.jpg differ diff --git a/tests/ut/python/test_data/imagenet_file/n15075141/n15075141_0001.jpg b/tests/ut/python/test_data/imagenet_file/n15075141/n15075141_0001.jpg new file mode 100644 index 00000000000..70f399d0090 Binary files /dev/null and b/tests/ut/python/test_data/imagenet_file/n15075141/n15075141_0001.jpg differ diff --git a/tests/ut/python/test_data/imagenet_file/n15075141/n15075141_0002.jpg b/tests/ut/python/test_data/imagenet_file/n15075141/n15075141_0002.jpg new file mode 100644 index 00000000000..5d654a1ac8f Binary files /dev/null and b/tests/ut/python/test_data/imagenet_file/n15075141/n15075141_0002.jpg differ diff --git a/tests/ut/python/test_data/imagenet_file/n15075142/n15075142_0001.jpg b/tests/ut/python/test_data/imagenet_file/n15075142/n15075142_0001.jpg new file mode 100644 index 00000000000..49c9bf0821b Binary files /dev/null and b/tests/ut/python/test_data/imagenet_file/n15075142/n15075142_0001.jpg differ diff --git a/tests/ut/python/test_data/imagenet_file/n15075143/n15075143_0001.jpg b/tests/ut/python/test_data/imagenet_file/n15075143/n15075143_0001.jpg new file mode 100644 index 00000000000..c6d2b17cf66 Binary files /dev/null and b/tests/ut/python/test_data/imagenet_file/n15075143/n15075143_0001.jpg differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_a/001_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/001_90.bmp new file mode 100644 index 00000000000..0fd6ac273bb Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/001_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_a/002_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/002_90.bmp new file mode 100644 index 00000000000..8db46e2f2ea Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/002_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_a/003_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/003_90.bmp new file mode 100644 index 00000000000..2dc294c9dff Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/003_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_a/004_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/004_90.bmp new file mode 100644 index 00000000000..a9a41da4b93 Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_a/004_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_b/001_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/001_90.bmp new file mode 100644 index 00000000000..0fd6ac273bb Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/001_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_b/002_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/002_90.bmp new file mode 100644 index 00000000000..8db46e2f2ea Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/002_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_b/003_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/003_90.bmp new file mode 100644 index 00000000000..2dc294c9dff Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/003_90.bmp differ diff --git a/tests/ut/python/test_data/reid/source/VIPeR/cam_b/004_90.bmp b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/004_90.bmp new file mode 100644 index 00000000000..a9a41da4b93 Binary files /dev/null and b/tests/ut/python/test_data/reid/source/VIPeR/cam_b/004_90.bmp differ diff --git a/tests/ut/python/test_data/tf_file/bert_examples b/tests/ut/python/test_data/tf_file/bert_examples new file mode 100644 index 00000000000..0686f535a03 Binary files /dev/null and b/tests/ut/python/test_data/tf_file/bert_examples differ diff --git a/tests/ut/python/test_data/tf_file/five_example.tf_record b/tests/ut/python/test_data/tf_file/five_example.tf_record new file mode 100644 index 00000000000..ed5b0ab59c2 Binary files /dev/null and b/tests/ut/python/test_data/tf_file/five_example.tf_record differ diff --git a/tests/ut/python/test_log.py b/tests/ut/python/test_log.py new file mode 100644 index 00000000000..04be0bd0b9f --- /dev/null +++ b/tests/ut/python/test_log.py @@ -0,0 +1,316 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +log test +""" +import os +import sys +import time +import re +import shutil +import logging + +def test_log_stdout(): + # Clean up environment variables + _rm_env_config() + # print the logs without raising an exception. + from mindspore import log as logger + log_str = 'print informations' + logger.error("1 test log message info :%s", log_str) + logger.info("2 test log message info") + logger.warning("3 test log message warning") + logger.debug("4 test log message debug:%s", log_str) + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_default(): + _rm_env_config() + from mindspore import log as logger + configdict = logger.get_log_config() + targetdict = {'GLOG_v': '2', 'GLOG_logtostderr': '1'} + assert configdict == targetdict + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_setlevel(): + _rm_env_config() + os.environ['GLOG_v'] = '0' + from mindspore import log as logger + #logger_instance = logger._get_logger() + #del logger_instance + loglevel = logger.get_level() + log_str = 'print debug informations' + logger.debug("5 test log message debug:%s", log_str) + assert loglevel == '0' + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_file(): + """ + test the log contect written in log file + """ + _rm_env_config() + file_path = '/tmp/log/mindspore_test' + os.environ['GLOG_v'] = '0' + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = file_path + + from mindspore import log as logger + filename = f'{file_path}/mindspore.log' + if os.path.exists(file_path): + shutil.rmtree(file_path) + + os.makedirs(file_path, exist_ok=True) + # Clear test file + if os.path.exists(filename): + os.remove(filename) + logger.warning("test log message warning") + cmd = f'cat {filename}' + result = os.popen(cmd).read() + # pylint: disable=anomalous-backslash-in-string + pattern = "\[WARNING\] ME\(.*[0-9]:.*[0-9]\,.*[a-zA-Z0-9]\):.* " \ + "\[.*:.*[0-9]\] test log message warning" + match_obj = re.match(pattern, result) + + #Clear test file + if os.path.exists(file_path): + shutil.rmtree(file_path) + + assert match_obj + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_backup_count(): + """ + test backup count + """ + #logger.reset_log_config(level=logging.INFO, console=False, + # filepath=file_path, maxBytes=1000, backupCount=10) + _rm_env_config() + file_path = '/tmp/log/mindspore_test' + os.environ['GLOG_v'] = '1' + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = file_path + os.environ['logger_maxBytes'] = '1000' + os.environ['logger_backupCount'] = '10' + + from mindspore import log as logger + if os.path.exists(file_path): + shutil.rmtree(file_path) + os.makedirs(file_path, exist_ok=True) + + log_count = 100 + for i in range(0, log_count, 1): + logger.warning("test log message warning %r", i) + + cmd = f'cd {file_path};ls |wc -l' + backup_count = '11' + file_count = os.popen(cmd).read().strip() + + if os.path.exists(file_path): + shutil.rmtree(file_path) + assert file_count == backup_count + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_verify_envconfig(): + """ + test reset config + """ + dictlist = [] + from mindspore import log as logger + file_path = '/tmp' + + # level is not a number + _rm_env_config() + os.environ['GLOG_v'] = 'test' + verify_dict_0 = logger._get_env_config() + + # level is not in range + _rm_env_config() + os.environ['GLOG_v'] = '100' + verify_dict_1 = logger._get_env_config() + + # console is not a number + _rm_env_config() + os.environ['GLOG_logtostderr'] = 'test' + verify_dict_2 = logger._get_env_config() + + # console is not in range + _rm_env_config() + os.environ['GLOG_logtostderr'] = '6' + verify_dict_3 = logger._get_env_config() + + # path does not exist + _rm_env_config() + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = '/test' + verify_dict_4 = logger._get_env_config() + + # path is not configured + _rm_env_config() + os.environ['GLOG_logtostderr'] = '0' + verify_dict_5 = logger._get_env_config() + + # logger_maxBytes is not a number + _rm_env_config() + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = '/tmp' + os.environ['logger_maxBytes'] = 'test' + os.environ['logger_backupCount'] = '10' + verify_dict_6 = logger._get_env_config() + + # logger_maxBytes is a negative number + _rm_env_config() + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = '/tmp' + os.environ['logger_maxBytes'] = '-1' + os.environ['logger_backupCount'] = '10' + verify_dict_7 = logger._get_env_config() + + # logger_backupCount is not a number + _rm_env_config() + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = '/tmp' + os.environ['logger_maxBytes'] = '0' + os.environ['logger_backupCount'] = 'test' + verify_dict_8 = logger._get_env_config() + + # logger_backupCount is a negative number + _rm_env_config() + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = '/tmp' + os.environ['logger_maxBytes'] = '0' + os.environ['logger_backupCount'] = '-1' + verify_dict_9 = logger._get_env_config() + + for i in range(0, 10, 1): + variable_name = f'verify_dict_{i}' + dictlist.append(locals()[variable_name]) + + for verify_dict in dictlist: + try: + logger._verify_config(verify_dict) + except ValueError as ve: + print(ve) + assert True + except TypeError as te: + print(te) + assert True + else: + assert False + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_repeated_print(): + """ + test Log repeated printing + # Print one log is right, otherwise error + """ + _rm_env_config() + from mindspore import log as logger + py_logging = logging.getLogger() + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + py_logging.addHandler(handler) + logger.info("test log message info test ") + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + + +def test_log_getconfig(): + _rm_env_config() + os.environ['GLOG_v'] = '3' + os.environ['GLOG_logtostderr'] = '0' + os.environ['GLOG_log_dir'] = '/tmp/log/' + os.environ['logger_maxBytes'] = '1000' + os.environ['logger_backupCount'] = '10' + from mindspore import log as logger + logger.info("test log message info test ") + configdict = logger.get_log_config() + targetdict = {'GLOG_v': '3', 'GLOG_log_dir': '/tmp/log', + 'GLOG_logtostderr': '0', 'logger_maxBytes': 1000, 'logger_backupCount': 10} + assert configdict == targetdict + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_perf(): + """ + Performance test with python logging + """ + _rm_env_config() + os.environ['GLOG_v'] = '3' + from mindspore import log as logger + loglevel = logging.ERROR + logging.basicConfig() + py_logging = logging.getLogger() + py_logging.setLevel(loglevel) + + log_count = 100000 + + print("logger level:", logger.get_level()) + print("py_logging level:", py_logging.getEffectiveLevel()) + + # Calculate PY logging execution time + start_time_py_logging = int(round(time.time() * 1000)) + + for i in range(0, log_count, 1): + py_logging.info("test log message info :%r", i) + + end_time_py_logging = int(round(time.time() * 1000)) + time_diff_py_logging = end_time_py_logging - start_time_py_logging + + # Calculate MS logger execution time + start_time_logger = int(round(time.time() * 1000)) + + for i in range(0, log_count, 1): + logger.info("test log message info :%r", i) + + end_time_logger = int(round(time.time() * 1000)) + time_diff_logger = end_time_logger - start_time_logger + + # Calculate time difference + time_diff = time_diff_logger - time_diff_py_logging + strprint = f'time difference between MS logger ' \ + f'and Python logging: {time_diff} ms' + print(strprint) + std_time = 2000 + assert time_diff < std_time + # Clean up _global_logger to avoid affecting for next usecase + logger._global_logger = None + + +def test_log_ms_import(): + _rm_env_config() + import mindspore as ms + configdict = ms.get_log_config() + targetdict = {'GLOG_v': '2', 'GLOG_logtostderr': '1'} + level = ms.get_level() + assert configdict == targetdict and level == '2' + + +def _rm_env_config(): + envlist = ['GLOG_v', 'GLOG_logtostderr', 'GLOG_log_dir', 'logger_maxBytes', 'logger_backupCount'] + for env in envlist: + if os.environ.get(env): + del os.environ[env] diff --git a/tests/ut/python/train/__init__.py b/tests/ut/python/train/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/train/summary/__init__.py b/tests/ut/python/train/summary/__init__.py new file mode 100644 index 00000000000..5443c0ca48e --- /dev/null +++ b/tests/ut/python/train/summary/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup for pytest""" +import mindspore.context as context + +# pylint: disable=unused-argument +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) diff --git a/tests/ut/python/train/summary/test_graph_summary.py b/tests/ut/python/train/summary/test_graph_summary.py new file mode 100644 index 00000000000..cebf7064656 --- /dev/null +++ b/tests/ut/python/train/summary/test_graph_summary.py @@ -0,0 +1,143 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_graph_summary """ +import os +import logging +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore.nn.optim import Momentum +from mindspore import Model, context +from mindspore.train.summary.summary_record import SummaryRecord +from mindspore.train.callback import SummaryStep +from .....dataset_mock import MindData + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" +GRAPH_TEMP = CUR_DIR + "/ms_output-resnet50.pb" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +class Net(nn.Cell): + """ Net definition """ + + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +class LossNet(nn.Cell): + """ LossNet definition """ + + def __init__(self): + super(LossNet, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + self.loss = nn.SoftmaxCrossEntropyWithLogits() + + def construct(self, x, y): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + x = self.fc(x) + out = self.loss(x, y) + return out + + +def get_model(): + """ get_model """ + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + return model + + +def get_dataset(): + """ get_datasetdataset """ + dataset_types = (np.float32, np.float32) + dataset_shapes = ((2, 3, 224, 224), (2, 3)) + + dataset = MindData(size=2, batch_size=2, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset + + +# Test 1: summary sample of graph +def test_graph_summary_sample(): + """ test_graph_summary_sample """ + log.debug("begin test_graph_summary_sample") + dataset = get_dataset() + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), 0.1, 0.9) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=model._train_network) + model.train(2, dataset) + # step 2: create the Event + for i in range(1, 5): + test_writer.record(i) + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.close() + + log.debug("finished test_graph_summary_sample") + + +def test_graph_summary_callback(): + dataset = get_dataset() + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), 0.1, 0.9) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=model._train_network) + summary_cb = SummaryStep(test_writer, 1) + model.train(2, dataset, callbacks=summary_cb) + + +def test_graph_summary_callback2(): + dataset = get_dataset() + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), 0.1, 0.9) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=net) + summary_cb = SummaryStep(test_writer, 1) + model.train(2, dataset, callbacks=summary_cb) diff --git a/tests/ut/python/train/summary/test_image_summary.py b/tests/ut/python/train/summary/test_image_summary.py new file mode 100644 index 00000000000..2eb35d3f58a --- /dev/null +++ b/tests/ut/python/train/summary/test_image_summary.py @@ -0,0 +1,224 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_image_summary.py +@Author: +@Date : 2019-07-4 +@Desc : test summary function +""" +import os +import logging +import numpy as np +import mindspore.nn as nn +from mindspore.train.summary.summary_record import SummaryRecord, \ + _cache_summary_tensor_data +from mindspore import Tensor +from mindspore.nn.optim import Momentum +from mindspore import Model, context +from mindspore.train.callback import SummaryStep +from .....dataset_mock import MindData + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +def make_image_tensor(shape, dtype=float): + """ make_image_tensor """ + # pylint: disable=unused-argument + numel = np.prod(shape) + x = (np.arange(numel, dtype=float)).reshape(shape) + return x + + +def get_test_data(step): + """ get_test_data """ + test_data_list = [] + tag1 = "x1[:Image]" + tag2 = "x2[:Image]" + np1 = make_image_tensor([2, 3, 8, 8]) + np2 = make_image_tensor([step, 3, 8, 8]) + + dict1 = {} + dict1["name"] = tag1 + dict1["data"] = Tensor(np1) + + dict2 = {} + dict2["name"] = tag2 + dict2["data"] = Tensor(np2) + + test_data_list.append(dict1) + test_data_list.append(dict2) + + return test_data_list + + +# Test: call method on parse graph code +def test_image_summary_sample(): + """ test_image_summary_sample """ + log.debug("begin test_image_summary_sample") + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_IMAGE") + + # step 1: create the test data for summary + + # step 2: create the Event + for i in range(1, 5): + test_data = get_test_data(i) + _cache_summary_tensor_data(test_data) + test_writer.record(i) + test_writer.flush() + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.close() + + log.debug("finished test_image_summary_sample") + + +class Net(nn.Cell): + """ Net definition """ + + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', + pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +class LossNet(nn.Cell): + """ LossNet definition """ + + def __init__(self): + super(LossNet, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', + pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + self.loss = nn.SoftmaxCrossEntropyWithLogits() + + def construct(self, x, y): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + x = self.fc(x) + out = self.loss(x, y) + return out + + +def get_model(): + """ get_model """ + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + return model + + +def get_dataset(): + """ get_dataset """ + dataset_types = (np.float32, np.float32) + dataset_shapes = ((2, 3, 224, 224), (2, 3)) + + dataset = MindData(size=2, batch_size=2, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset + + +class ImageSummaryCallback: + def __init__(self, summaryRecord): + self._summaryRecord = summaryRecord + + def record(self, step, train_network=None): + self._summaryRecord.record(step, train_network) + self._summaryRecord.flush() + + +def test_image_summary_train(): + """ test_image_summary_train """ + dataset = get_dataset() + + log.debug("begin test_image_summary_sample") + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_IMAGE") + + # step 1: create the test data for summary + + # step 2: create the Event + + model = get_model() + fn = ImageSummaryCallback(test_writer) + summary_recode = SummaryStep(fn, 1) + model.train(2, dataset, callbacks=summary_recode) + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.close() + + log.debug("finished test_image_summary_sample") + + +def test_image_summary_data(): + """ test_image_summary_data """ + dataset = get_dataset() + + test_data_list = [] + i = 1 + for next_element in dataset: + tag = "image_" + str(i) + "[:Image]" + dct = {} + dct["name"] = tag + dct["data"] = Tensor(next_element[0]) + test_data_list.append(dct) + i += 1 + + log.debug("begin test_image_summary_sample") + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_IMAGE") + + # step 1: create the test data for summary + + # step 2: create the Event + _cache_summary_tensor_data(test_data_list) + test_writer.record(1) + test_writer.flush() + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.close() + + log.debug("finished test_image_summary_sample") diff --git a/tests/ut/python/train/summary/test_summary.py b/tests/ut/python/train/summary/test_summary.py new file mode 100644 index 00000000000..cc595ea883a --- /dev/null +++ b/tests/ut/python/train/summary/test_summary.py @@ -0,0 +1,240 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_summary.py +@Author: +@Date : 2019-07-4 +@Desc : test summary function +""" +import os +import logging +import random +import numpy as np +import pytest +from mindspore.train.summary.summary_record import SummaryRecord, _cache_summary_tensor_data +from mindspore.train.callback import SummaryStep +from mindspore.common.tensor import Tensor +import mindspore.nn as nn +from mindspore.ops import operations as P + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +def get_test_data(step): + """ get_test_data """ + test_data_list = [] + tag1 = "x1[:Scalar]" + tag2 = "x2[:Scalar]" + np1 = np.array(step + 1).astype(np.float32) + np2 = np.array(step + 2).astype(np.float32) + + dict1 = {} + dict1["name"] = tag1 + dict1["data"] = Tensor(np1) + + dict2 = {} + dict2["name"] = tag2 + dict2["data"] = Tensor(np2) + + test_data_list.append(dict1) + test_data_list.append(dict2) + + return test_data_list + + +# Test 1: summary sample of scalar +def test_scalar_summary_sample(): + """ test_scalar_summary_sample """ + log.debug("begin test_scalar_summary_sample") + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") + + # step 1: create the test data for summary + + # step 2: create the Event + for i in range(1, 500): + test_data = get_test_data(i) + _cache_summary_tensor_data(test_data) + test_writer.record(i) + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.close() + + log.debug("finished test_scalar_summary_sample") + + +def get_test_data_shape_1(step): + """ get_test_data_shape_1 """ + test_data_list = [] + tag1 = "x1[:Scalar]" + tag2 = "x2[:Scalar]" + np1 = np.array([step + 1]).astype(np.float32) + np2 = np.array([step + 2]).astype(np.float32) + + dict1 = {} + dict1["name"] = tag1 + dict1["data"] = Tensor(np1) + + dict2 = {} + dict2["name"] = tag2 + dict2["data"] = Tensor(np2) + + test_data_list.append(dict1) + test_data_list.append(dict2) + + return test_data_list + + +# Test: shape = (1,) +def test_scalar_summary_sample_with_shape_1(): + """ test_scalar_summary_sample_with_shape_1 """ + log.debug("begin test_scalar_summary_sample_with_shape_1") + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") + + # step 1: create the test data for summary + + # step 2: create the Event + for i in range(1, 100): + test_data = get_test_data_shape_1(i) + _cache_summary_tensor_data(test_data) + test_writer.record(i) + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.close() + + log.debug("finished test_scalar_summary_sample") + + +# Test: test with ge +class SummaryDemo(nn.Cell): + """ SummaryDemo definition """ + def __init__(self,): + super(SummaryDemo, self).__init__() + self.s = P.ScalarSummary() + self.add = P.TensorAdd() + + def construct(self, x, y): + self.s("x1", x) + z = self.add(x, y) + self.s("z1", z) + self.s("y1", y) + return z + + +def test_scalar_summary_with_ge(): + """ test_scalar_summary_with_ge """ + log.debug("begin test_scalar_summary_with_ge") + + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") + + # step 1: create the network for summary + x = Tensor(np.array([1.1]).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + net = SummaryDemo() + net.set_train() + + # step 2: create the Event + steps = 100 + for i in range(1, steps): + x = Tensor(np.array([1.1 + random.uniform(1, 10)]).astype(np.float32)) + y = Tensor(np.array([1.2 + random.uniform(1, 10)]).astype(np.float32)) + net(x, y) + test_writer.record(i) + + # step 3: close the writer + test_writer.close() + + log.debug("finished test_scalar_summary_with_ge") + + +# test the problem of two consecutive use cases going wrong +def test_scalar_summary_with_ge_2(): + """ test_scalar_summary_with_ge_2 """ + log.debug("begin test_scalar_summary_with_ge_2") + + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") + + # step 1: create the network for summary + x = Tensor(np.array([1.1]).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + net = SummaryDemo() + net.set_train() + + # step 2: create the Event + steps = 100 + for i in range(1, steps): + x = Tensor(np.array([1.1]).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + net(x, y) + test_writer.record(i) + + # step 3: close the writer + test_writer.close() + + log.debug("finished test_scalar_summary_with_ge_2") + + +def test_validate(): + sr = SummaryRecord(SUMMARY_DIR) + + with pytest.raises(ValueError): + SummaryStep(sr, 0) + with pytest.raises(ValueError): + SummaryStep(sr, -1) + with pytest.raises(ValueError): + SummaryStep(sr, 1.2) + with pytest.raises(ValueError): + SummaryStep(sr, True) + with pytest.raises(ValueError): + SummaryStep(sr, "str") + sr.record(1) + with pytest.raises(ValueError): + sr.record(False) + with pytest.raises(ValueError): + sr.record(2.0) + with pytest.raises(ValueError): + sr.record((1,3)) + with pytest.raises(ValueError): + sr.record([2,3]) + with pytest.raises(ValueError): + sr.record("str") + with pytest.raises(ValueError): + sr.record(sr) + sr.close() + + SummaryStep(sr, 1) + with pytest.raises(ValueError): + SummaryStep(sr, 1.2) + with pytest.raises(ValueError): + SummaryStep(sr, False) + with pytest.raises(ValueError): + SummaryStep(sr, "str") + with pytest.raises(ValueError): + SummaryStep(sr, (1,2)) + with pytest.raises(ValueError): + SummaryStep(sr, [3,4]) + with pytest.raises(ValueError): + SummaryStep(sr, sr) diff --git a/tests/ut/python/train/summary/test_summary_abnormal_input.py b/tests/ut/python/train/summary/test_summary_abnormal_input.py new file mode 100644 index 00000000000..cf6ae7b884b --- /dev/null +++ b/tests/ut/python/train/summary/test_summary_abnormal_input.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_summary_abnormal_input.py +@Author: +@Date : 2019-08-5 +@Desc : test summary function of abnormal input +""" +import os +import logging +import numpy as np +from mindspore.train.summary.summary_record import SummaryRecord +from mindspore.common.tensor import Tensor + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +def get_test_data(step): + """ get_test_data """ + test_data_list = [] + tag1 = "x1[:Scalar]" + tag2 = "x2[:Scalar]" + np1 = np.array(step + 1).astype(np.float32) + np2 = np.array(step + 2).astype(np.float32) + + dict1 = {} + dict1["name"] = tag1 + dict1["data"] = Tensor(np1) + + dict2 = {} + dict2["name"] = tag2 + dict2["data"] = Tensor(np2) + + test_data_list.append(dict1) + test_data_list.append(dict2) + + return test_data_list + + +# Test: call method on parse graph code +def test_summaryrecord_input_null_string(): + log.debug("begin test_summaryrecord_input_null_string") + # step 0: create the thread + try: + SummaryRecord("") + except: + assert True + else: + assert False + log.debug("finished test_summaryrecord_input_null_string") + +def test_summaryrecord_input_None(): + log.debug("begin test_summaryrecord_input_None") + # step 0: create the thread + try: + SummaryRecord(None) + except: + assert True + else: + assert False + log.debug("finished test_summaryrecord_input_None") + +def test_summaryrecord_input_relative_dir_1(): + log.debug("begin test_summaryrecord_input_relative_dir_1") + # step 0: create the thread + try: + SummaryRecord("./test_temp_summary_event_file/") + except: + assert False + else: + assert True + log.debug("finished test_summaryrecord_input_relative_dir_1") + + +def test_summaryrecord_input_relative_dir_2(): + log.debug("begin test_summaryrecord_input_relative_dir_2") + # step 0: create the thread + try: + SummaryRecord("../summary/") + except: + assert False + else: + assert True + log.debug("finished test_summaryrecord_input_relative_dir_2") + + +def test_summaryrecord_input_invalid_type_dir(): + log.debug("begin test_summaryrecord_input_invalid_type_dir") + # step 0: create the thread + try: + SummaryRecord(32) + except: + assert True + else: + assert False + log.debug("finished test_summaryrecord_input_invalid_type_dir") + + +def test_mulit_layer_directory(): + log.debug("begin test_mulit_layer_directory") + # step 0: create the thread + try: + SummaryRecord("./test_temp_summary_event_file/test/t1/") + except: + assert False + else: + assert True + log.debug("finished test_mulit_layer_directory") diff --git a/tests/ut/python/train/summary/test_summary_ops_params_valid_check.py b/tests/ut/python/train/summary/test_summary_ops_params_valid_check.py new file mode 100644 index 00000000000..b6e1ab992cf --- /dev/null +++ b/tests/ut/python/train/summary/test_summary_ops_params_valid_check.py @@ -0,0 +1,238 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_summary.py +@Author: +@Date : 2019-08-5 +@Desc : test summary function of ops params valid check +""" +import os +import logging +import random +import numpy as np +from mindspore.train.summary.summary_record import SummaryRecord +from mindspore.common.tensor import Tensor +import mindspore.nn as nn +from mindspore.ops import operations as P + + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +class SummaryDemoTag(nn.Cell): + """ SummaryDemoTag definition """ + def __init__(self, tag1, tag2, tag3): + super(SummaryDemoTag, self).__init__() + self.s = P.ScalarSummary() + self.add = P.TensorAdd() + self.tag1 = tag1 + self.tag2 = tag2 + self.tag3 = tag3 + + def construct(self, x, y): + self.s(self.tag1, x) + z = self.add(x, y) + self.s(self.tag2, z) + self.s(self.tag3, y) + return z + + +class SummaryDemoTagForSet(nn.Cell): + """ SummaryDemoTagForSet definition """ + def __init__(self, tag_tuple): + super(SummaryDemoTagForSet, self).__init__() + self.s = P.ScalarSummary() + self.add = P.TensorAdd() + self.tag_tuple = tag_tuple + + def construct(self, x, y): + z = self.add(x, y) + for tag in self.tag_tuple: + self.s(tag, x) + return z + + +class SummaryDemoValue(nn.Cell): + """ SummaryDemoValue definition """ + def __init__(self, value): + super(SummaryDemoValue, self).__init__() + self.s = P.ScalarSummary() + self.add = P.TensorAdd() + self.v = value + + def construct(self, x, y): + self.s("x", self.v) + z = self.add(x, y) + self.s("z", self.v) + self.s("y", self.v) + return z + +class SummaryDemoValueForSet(nn.Cell): + """ SummaryDemoValueForSet definition """ + def __init__(self, value, tag_tuple): + super(SummaryDemoValueForSet, self).__init__() + self.s = P.ScalarSummary() + self.add = P.TensorAdd() + self.tag_tuple = tag_tuple + self.v = value + + def construct(self, x, y): + z = self.add(x, y) + for tag in self.tag_tuple: + self.s(tag, self.v) + return z + +def run_case(net): + """ run_case """ + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR) + + # step 1: create the network for summary + x = Tensor(np.array([1.1]).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + net.set_train() + + # step 2: create the Event + steps = 100 + for i in range(1, steps): + x = Tensor(np.array([1.1 + random.uniform(1, 10)]).astype(np.float32)) + y = Tensor(np.array([1.2 + random.uniform(1, 10)]).astype(np.float32)) + net(x, y) + test_writer.record(i) + + # step 3: close the writer + test_writer.close() + + +# Test 1: use the repeat tag +def test_scalar_summary_use_repeat_tag(): + log.debug("begin test_scalar_summary_use_repeat_tag") + net = SummaryDemoTag("x", "x", "x") + try: + run_case(net) + except: + assert False + else: + assert True + log.debug("finished test_scalar_summary_use_repeat_tag") + + +# Test 2: repeat tag use for set summary +def test_scalar_summary_use_repeat_tag_for_set(): + log.debug("begin test_scalar_summary_use_repeat_tag_for_set") + net = SummaryDemoTagForSet(("x", "x", "x")) + try: + run_case(net) + except: + assert False + else: + assert True + log.debug("finished test_scalar_summary_use_repeat_tag_for_set") + + +# Test3: test with invalid tag(None, bool, "", int) +def test_scalar_summary_use_invalid_tag_None(): + log.debug("begin test_scalar_summary_use_invalid_tag_None") + net = SummaryDemoTag(None, None, None) + try: + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_tag_None") + + +# Test4: test with invalid tag(None, bool, "", int) +def test_scalar_summary_use_invalid_tag_Bool(): + log.debug("begin test_scalar_summary_use_invalid_tag_Bool") + net = SummaryDemoTag(True, True, True) + try: + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_tag_Bool") + + +# Test5: test with invalid tag(None, bool, "", int) +def test_scalar_summary_use_invalid_tag_null(): + log.debug("begin test_scalar_summary_use_invalid_tag_null") + net = SummaryDemoTag("", "", "") + try: + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_tag_null") + + +# Test6: test with invalid tag(None, bool, "", int) +def test_scalar_summary_use_invalid_tag_Int(): + log.debug("begin test_scalar_summary_use_invalid_tag_Int") + net = SummaryDemoTag(1, 2, 3) + try: + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_tag_Int") + + +# Test7: test with invalid value(None, "") +def test_scalar_summary_use_invalid_value_None(): + log.debug("begin test_scalar_summary_use_invalid_tag_Int") + net = SummaryDemoValue(None) + try: + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_tag_Int") + + + +# Test8: test with invalid value(None, "") +def test_scalar_summary_use_invalid_value_None_ForSet(): + log.debug("begin test_scalar_summary_use_invalid_value_None_ForSet") + try: + net = SummaryDemoValueForSet(None, ("x1", "x2")) + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_value_None_ForSet") + + +# Test9: test with invalid value(None, "") +def test_scalar_summary_use_invalid_value_null(): + log.debug("begin test_scalar_summary_use_invalid_value_null") + try: + net = SummaryDemoValue("") + run_case(net) + except: + assert True + else: + assert False + log.debug("finished test_scalar_summary_use_invalid_value_null") diff --git a/tests/ut/python/train/summary/test_summary_performance.py b/tests/ut/python/train/summary/test_summary_performance.py new file mode 100644 index 00000000000..f004d4e584f --- /dev/null +++ b/tests/ut/python/train/summary/test_summary_performance.py @@ -0,0 +1,97 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_summary.py +@Author: +@Date : 2019-07-4 +@Desc : test summary function +""" +import os +import logging +import time +import numpy as np +from mindspore.train.summary.summary_record import SummaryRecord, _cache_summary_tensor_data +from mindspore.common.tensor import Tensor + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + +def get_now_time_ns(): + """get the time of second""" + time_second = int(time.time_ns()) + return time_second + +def get_test_data(step): + """ get_test_data """ + # pylint: disable=unused-argument + test_data_list = [] + tag1 = "xt1[:Tensor]" + tag2 = "xt2[:Tensor]" + tag3 = "xt3[:Tensor]" + np1 = np.random.random((50, 40, 30, 50)) + np2 = np.random.random((50, 50, 30, 50)) + np3 = np.random.random((40, 55, 30, 50)) + + dict1 = {} + dict1["name"] = tag1 + dict1["data"] = Tensor(np1) + + dict2 = {} + dict2["name"] = tag2 + dict2["data"] = Tensor(np2) + + dict3 = {} + dict3["name"] = tag3 + dict3["data"] = Tensor(np3) + + test_data_list.append(dict1) + test_data_list.append(dict2) + + return test_data_list + + +# Test 1: summary sample of scalar +def test_summary_performance(): + """ test_summary_performance """ + log.debug("begin test_scalar_summary_sample") + current_time = time.time() + print("time = ", current_time) + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, flush_time=120) + + # step 1: create the test data for summary + old_time = get_now_time_ns() + # step 2: create the Event + for i in range(1, 10): + test_data = get_test_data(i) + _cache_summary_tensor_data(test_data) + test_writer.record(i) + now_time = get_now_time_ns() + consume_time = (now_time - old_time)/1000/1000 + old_time = now_time + print("step test_summary_performance conusmer time is:", consume_time) + + + # step 3: send the event to mq + + # step 4: accept the event and write the file + test_writer.flush() + test_writer.close() + current_time = time.time() - current_time + print("consume time = ", current_time) + log.debug("finished test_scalar_summary_sample") diff --git a/tests/ut/python/train/summary/test_tensor_summary.py b/tests/ut/python/train/summary/test_tensor_summary.py new file mode 100644 index 00000000000..8f83dd55969 --- /dev/null +++ b/tests/ut/python/train/summary/test_tensor_summary.py @@ -0,0 +1,150 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_tensor_summary.py +@Author: +@Date : 2019-07-4 +@Desc : test summary function +""" +import os +import logging +import numpy as np +from mindspore.train.summary.summary_record import SummaryRecord, _cache_summary_tensor_data +from mindspore.common.tensor import Tensor +import mindspore.nn as nn +from mindspore.ops import operations as P + +CUR_DIR = os.getcwd() +SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +def get_test_data(step): + """ get_test_data """ + test_data_list = [] + + dict_x1 = {} + dict_x1["name"] = "x1[:Tensor]" + dict_x1["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.int8)) + test_data_list.append(dict_x1) + dict_x2 = {} + dict_x2["name"] = "x2[:Tensor]" + dict_x2["data"] = Tensor(np.array([[1, 2, step + 2], [2, 3, 4]]).astype(np.int16)) + test_data_list.append(dict_x2) + dict_x3 = {} + dict_x3["name"] = "x3[:Tensor]" + dict_x3["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.int32)) + test_data_list.append(dict_x3) + dict_x4 = {} + dict_x4["name"] = "x4[:Tensor]" + dict_x4["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.int64)) + test_data_list.append(dict_x4) + dict_x5 = {} + dict_x5["name"] = "x5[:Tensor]" + dict_x5["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float)) + test_data_list.append(dict_x5) + dict_x6 = {} + dict_x6["name"] = "x6[:Tensor]" + dict_x6["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float16)) + test_data_list.append(dict_x6) + dict_x7 = {} + dict_x7["name"] = "x7[:Tensor]" + dict_x7["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float32)) + test_data_list.append(dict_x7) + dict_x8 = {} + dict_x8["name"] = "x8[:Tensor]" + dict_x8["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float64)) + test_data_list.append(dict_x8) + + return test_data_list + + +# Test: call method on parse graph code +def test_tensor_summary_sample(): + """ test_tensor_summary_sample """ + log.debug("begin test_tensor_summary_sample") + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR, file_suffix="_MS_TENSOR") + + # step 1: create the Event + for i in range(1, 100): + test_data = get_test_data(i) + + _cache_summary_tensor_data(test_data) + test_writer.record(i) + + # step 2: accept the event and write the file + test_writer.close() + + log.debug("finished test_tensor_summary_sample") + + + +def get_test_data_check(step): + """ get_test_data_check """ + test_data_list = [] + tag1 = "x1[:Tensor]" + np1 = np.array([[step, step, step], [2, 3, 4]]).astype(np.float32) + + dict1 = {} + dict1["name"] = tag1 + dict1["data"] = Tensor(np1) + test_data_list.append(dict1) + + return test_data_list + + +# Test: test with ge +class SummaryDemo(nn.Cell): + """ SummaryDemo definition """ + def __init__(self,): + super(SummaryDemo, self).__init__() + self.s = P.TensorSummary() + self.add = P.TensorAdd() + + def construct(self, x, y): + self.s("x1", x) + z = self.add(x, y) + self.s("z1", z) + self.s("y1", y) + return z + +def test_tensor_summary_with_ge(): + """ test_tensor_summary_with_ge """ + log.debug("begin test_tensor_summary_with_ge") + + # step 0: create the thread + test_writer = SummaryRecord(SUMMARY_DIR) + + # step 1: create the network for summary + x = Tensor(np.array([1.1]).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + net = SummaryDemo() + net.set_train() + + # step 2: create the Event + steps = 100 + for i in range(1, steps): + x = Tensor(np.array([[i], [i]]).astype(np.float32)) + y = Tensor(np.array([[i+1], [i+1]]).astype(np.float32)) + net(x, y) + test_writer.record(i) + + # step 3: close the writer + test_writer.close() + + log.debug("finished test_tensor_summary_with_ge") diff --git a/tests/ut/python/train/test_amp.py b/tests/ut/python/train/test_amp.py new file mode 100644 index 00000000000..eebd188e604 --- /dev/null +++ b/tests/ut/python/train/test_amp.py @@ -0,0 +1,94 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" auto mixed precision """ +import numpy as np +from mindspore import amp +from mindspore import nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +import mindspore.context as context +from mindspore.model_zoo.resnet import resnet50 + + +def setup_module(module): + context.set_context(mode=context.GRAPH_MODE) + + +class Net(nn.Cell): + def __init__(self, in_features, out_features): + super(Net, self).__init__() + self.dense = nn.Dense(in_features, out_features) + self.loss = nn.MSELoss() + + def construct(self, input_x, label): + output = self.dense(input_x) + loss = self.loss(output, label) + return loss + + +class NetNoLoss(nn.Cell): + def __init__(self, in_features, out_features): + super(NetNoLoss, self).__init__() + self.dense = nn.Dense(in_features, out_features) + + def construct(self, input_x): + return self.dense(input_x) + + +def test_amp_o0(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + net = Net(16, 16) + + optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + train_network = amp.build_train_network(net, optimizer, level="O0") + output = train_network(inputs, label) + + +def test_amp_o2(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + net = Net(16, 16) + + optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + train_network = amp.build_train_network(net, optimizer, level="O2") + output = train_network(inputs, label) + +def test_amp_o2_loss(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + net = NetNoLoss(16, 16) + loss = nn.MSELoss() + optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + train_network = amp.build_train_network(net, optimizer, loss, level="O2") + output = train_network(inputs, label) + +def test_amp_resnet50_loss(): + inputs = Tensor(np.ones([2, 3, 224, 224]).astype(np.float32)) + label = Tensor(np.zeros([2, 10]).astype(np.float32)) + net = resnet50() + loss = nn.SoftmaxCrossEntropyWithLogits(reduction='mean') + optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + train_network = amp.build_train_network(net, optimizer, loss, level="O2") + train_network(inputs, label) + +def test_amp_o0_loss(): + inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + label = Tensor(np.zeros([16, 16]).astype(np.float32)) + net = NetNoLoss(16, 16) + loss = nn.MSELoss() + optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + train_network = amp.build_train_network(net, optimizer, loss) + output = train_network(inputs, label) diff --git a/tests/ut/python/train/test_run_config.py b/tests/ut/python/train/test_run_config.py new file mode 100644 index 00000000000..00572d62b36 --- /dev/null +++ b/tests/ut/python/train/test_run_config.py @@ -0,0 +1,58 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_run_config """ +import pytest +from mindspore.train.callback import CheckpointConfig + + +def test_init(): + """ test_init """ + save_checkpoint_steps = 1 + keep_checkpoint_max = 5 + + config = CheckpointConfig(save_checkpoint_steps, + keep_checkpoint_max) + + assert config.save_checkpoint_steps == save_checkpoint_steps + assert config.keep_checkpoint_max == keep_checkpoint_max + policy = config.get_checkpoint_policy() + assert policy['keep_checkpoint_max'] == keep_checkpoint_max + + +def test_arguments_values(): + """ test_arguments_values """ + config = CheckpointConfig() + assert config.save_checkpoint_steps == 1 + assert config.save_checkpoint_seconds is None + assert config.keep_checkpoint_max == 5 + assert config.keep_checkpoint_per_n_minutes is None + + with pytest.raises(TypeError): + CheckpointConfig(save_checkpoint_steps='abc') + with pytest.raises(TypeError): + CheckpointConfig(save_checkpoint_seconds='abc') + with pytest.raises(TypeError): + CheckpointConfig(keep_checkpoint_max='abc') + with pytest.raises(TypeError): + CheckpointConfig(keep_checkpoint_per_n_minutes='abc') + + with pytest.raises(ValueError): + CheckpointConfig(save_checkpoint_steps=-1) + with pytest.raises(ValueError): + CheckpointConfig(save_checkpoint_seconds=-1) + with pytest.raises(ValueError): + CheckpointConfig(keep_checkpoint_max=-1) + with pytest.raises(ValueError): + CheckpointConfig(keep_checkpoint_per_n_minutes=-1) diff --git a/tests/ut/python/train/test_training.py b/tests/ut/python/train/test_training.py new file mode 100644 index 00000000000..c7ba01e8bd5 --- /dev/null +++ b/tests/ut/python/train/test_training.py @@ -0,0 +1,265 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_training """ +import logging +import numpy as np +import pytest +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.nn.optim import Momentum +from mindspore import Model, context +from mindspore.train.callback import SummaryStep +from ..ut_filter import non_graph_engine +from ....dataset_mock import MindData + + +class Net(nn.Cell): + """ Net definition """ + + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +class LossNet(nn.Cell): + """ LossNet definition """ + + def __init__(self): + super(LossNet, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + self.loss = nn.SoftmaxCrossEntropyWithLogits() + + def construct(self, x, y): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + x = self.fc(x) + out = self.loss(x, y) + return out + + +def get_model(): + """ get_model """ + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + return model + + +def get_dataset(): + """ get_dataset """ + dataset_types = (np.float32, np.float32) + dataset_shapes = ((32, 3, 224, 224), (32, 3)) + + dataset = MindData(size=2, batch_size=32, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + return dataset + + +@non_graph_engine +def test_single_input(): + """ test_single_input """ + input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)) + context.set_context(mode=context.GRAPH_MODE) + model = Model(Net()) + out = model.predict(input_data) + assert out is not None + + +@non_graph_engine +def test_multiple_argument(): + """ test_multiple_argument """ + input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)) + input_label = Tensor(np.random.randint(0, 3, [1, 3]).astype(np.float32)) + context.set_context(mode=context.GRAPH_MODE) + model = Model(LossNet()) + out = model.predict(input_data, input_label) + assert out is not None + + +def test_train_feed_mode(test_with_simu): + """ test_train_feed_mode """ + dataset = get_dataset() + model = get_model() + if test_with_simu: + return + model.train(2, dataset) + + +def test_dataset_sink_mode_args_check(): + """ test_dataset_sink_mode_args_check """ + dataset = get_dataset() + model = get_model() + with pytest.raises(TypeError): + model.train(2, dataset, dataset_sink_mode="True") + + with pytest.raises(TypeError): + model.train(2, dataset, dataset_sink_mode=1) + + +@non_graph_engine +def test_eval(): + """ test_eval """ + dataset_types = (np.float32, np.float32) + dataset_shapes = ((32, 3, 224, 224), (32, 3)) + + dataset = MindData(size=2, batch_size=32, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=(0, 1)) + net = Net() + context.set_context(mode=context.GRAPH_MODE) + model = Model(net, loss_fn=nn.SoftmaxCrossEntropyWithLogits(), metrics={"loss"}) + with pytest.raises(ValueError): + model.eval(dataset) + + net2 = LossNet() + model2 = Model(net2, eval_network=net2, eval_indexes=[0, 1, 2], metrics={"loss"}) + with pytest.raises(ValueError): + model2.eval(dataset) + + net3 = LossNet() + model3 = Model(net2, eval_network=net2, metrics={"loss"}) + with pytest.raises(ValueError): + model3.eval(dataset) + + +class TestGraphMode: + """ TestGraphMode definition """ + + def test_train_minddata_graph_mode(self, test_with_simu): + """ test_train_minddata_graph_mode """ + # pylint: disable=unused-argument + dataset_types = (np.float32, np.float32) + dataset_shapes = ((32, 3, 224, 224), (32, 3)) + + dataset = MindData(size=2, batch_size=32, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=()) + model = get_model() + model.train(1, dataset) + + +class CallbackTest: + """ CallbackTest definition """ + + def __init__(self): + pass + + def record(self, step, *args): + print(step, args) + + +def test_train_callback(test_with_simu): + """ test_train_callback """ + dataset = get_dataset() + model = get_model() + fn = CallbackTest() + summary_recode = SummaryStep(fn, 2) + if test_with_simu: + return + model.train(2, dataset, callbacks=summary_recode) + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +# Test the invalid args and trigger RuntimeError +def test_model_build_abnormal_string(): + """ test_model_build_abnormal_string """ + net = nn.ReLU() + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + err = False + try: + model.predict('aaa') + except ValueError as e: + log.error("Find value error: %r ", e) + err = True + finally: + assert err + + +def test_model_init_error(): + """ test_model_init_error """ + net = nn.ReLU() + loss = nn.SoftmaxCrossEntropyWithLogits() + with pytest.raises(KeyError): + Model(net, loss, metrics={"top1"}) + + with pytest.raises(ValueError): + Model(net, metrics={"top_1_accuracy"}) + + with pytest.raises(TypeError): + Model(net, metrics={"top5": None}) + + with pytest.raises(ValueError): + Model(net, eval_network=net, eval_indexes=[], metrics={"top_1_accuracy"}) + + with pytest.raises(ValueError): + Model(net, eval_network=net, eval_indexes=(1, 2, 3), metrics={"top_1_accuracy"}) + + with pytest.raises(TypeError): + Model(net, loss, metrics=("top_1_accuracy")) + + with pytest.raises(TypeError): + Model(net, loss, metrics=()) + + with pytest.raises(TypeError): + Model(net, loss, metrics=["top_1_accuracy"]) + + +def test_model_eval_error(): + """ test_model_eval_error """ + dataset_types = (np.float32, np.float32) + dataset_shapes = ((32, 3, 224, 224), (32, 3)) + + dataset = MindData(size=2, batch_size=32, + np_types=dataset_types, + output_shapes=dataset_shapes, + input_indexs=()) + + net = nn.ReLU() + loss = nn.SoftmaxCrossEntropyWithLogits() + context.set_context(mode=context.GRAPH_MODE) + model_nometrics = Model(net, loss) + with pytest.raises(ValueError): + model_nometrics.eval(dataset) + + model_metrics_empty = Model(net, loss, metrics={}) + with pytest.raises(ValueError): + model_metrics_empty.eval(dataset) diff --git a/tests/ut/python/transform/__init__.py b/tests/ut/python/transform/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/transform/test_transform.py b/tests/ut/python/transform/test_transform.py new file mode 100644 index 00000000000..5a40e329115 --- /dev/null +++ b/tests/ut/python/transform/test_transform.py @@ -0,0 +1,168 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_adapter.py +@Author: +@Date : 2019-03-20 +@Desc : test mindspore compile method +""" +import logging +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter, Model +from mindspore.ops import operations as P +from ..ut_filter import non_graph_engine + + +log = logging.getLogger("test") +log.setLevel(level=logging.ERROR) + + +def conv3x3(in_channels, out_channels, stride=1, padding=1): + """3x3 convolution """ + weight = Tensor(np.ones([out_channels, in_channels, 3, 3]).astype(np.float32)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=3, stride=stride, + padding=padding, weight_init=weight) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0): + """1x1 convolution""" + weight = Tensor(np.ones([out_channels, in_channels, 1, 1]).astype(np.float32)) + return nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=stride, + padding=padding, weight_init=weight) + + +class ResidualBlock(nn.Cell): + """ + residual Block + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels, + stride=1, + down_sample=False): + super(ResidualBlock, self).__init__() + + out_chls = out_channels // self.expansion + self.conv1 = conv1x1(in_channels, out_chls, stride=1, padding=0) + self.bn1 = nn.BatchNorm2d(out_chls) + + self.conv2 = conv3x3(out_chls, out_chls, stride=stride, padding=1) + self.bn2 = nn.BatchNorm2d(out_chls) + + self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.relu = nn.ReLU() + self.downsample = down_sample + + if self.downsample: + self.conv_down_sample = conv1x1(in_channels, out_channels, + stride=stride, padding=0) + self.bn_down_sample = nn.BatchNorm2d(out_channels) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + identity = self.conv_down_sample(identity) + identity = self.bn_down_sample(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ ResNet definition """ + def __init__(self, tensor): + super(ResNet, self).__init__() + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3) + self.bn1 = nn.BatchNorm2d(64) + self.weight = Parameter(tensor, name='w') + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + return x + + +class LeNet(nn.Cell): + """ LeNet definition """ + def __init__(self): + super(LeNet, self).__init__() + self.relu = nn.ReLU() + weight1 = Tensor(np.ones([6, 1, 5, 5]).astype(np.float32) * 0.01) + weight2 = Tensor(np.ones([16, 6, 5, 5]).astype(np.float32) * 0.01) + self.conv1 = nn.Conv2d(1, 6, (5, 5), weight_init=weight1, stride=1, padding=0, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, (5, 5), weight_init=weight2, pad_mode='valid') + self.pool = nn.MaxPool2d(2) + self.flatten = nn.Flatten() + fcweight1 = Tensor(np.ones([120, 16 * 5 * 5]).astype(np.float32) * 0.01) + fcweight2 = Tensor(np.ones([84, 120]).astype(np.float32) * 0.01) + fcweight3 = Tensor(np.ones([10, 84]).astype(np.float32) * 0.01) + self.fc1 = nn.Dense(16 * 5 * 5, 120, weight_init=fcweight1) + self.fc2 = nn.Dense(120, 84, weight_init=fcweight2) + self.fc3 = nn.Dense(84, 10, weight_init=fcweight3) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.flatten(output) + output = self.fc1(output) + output = self.fc2(output) + output = self.fc3(output) + return output + + +def loss_func(x): + return x + + +def optimizer(x): + return x + + +class Net(nn.Cell): + """ Net definition """ + + def __init__(self, dim): + super(Net, self).__init__() + self.softmax = nn.Softmax(dim) + + def construct(self, input_x): + return self.softmax(input_x) + diff --git a/tests/ut/python/ut_filter.py b/tests/ut/python/ut_filter.py new file mode 100644 index 00000000000..683cd80a775 --- /dev/null +++ b/tests/ut/python/ut_filter.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : ut_filter.py +@Desc : filter for ge or gpu +""" + +import os +import pytest + +def is_enable_ge(): + val = os.getenv("ENABLE_GE", "False") + if val in ('ON', 'on', 'TRUE', 'True', 'true'): + return True + return False + +non_graph_engine = pytest.mark.skipif(is_enable_ge(), reason="Not support running on GE environment") + +def is_enable_gpu(): + val = os.getenv("ENABLE_GPU", "False") + if val in ('ON', 'on', 'TRUE', 'True', 'true'): + return True + return False + +run_on_gpu = pytest.mark.skipif(not is_enable_gpu(), reason="Only support running on GPU environment") + +def is_enable_onnxruntime(): + val = os.getenv("ENABLE_ONNXRUNTIME", "False") + if val in ('ON', 'on', 'TRUE', 'True', 'true'): + return True + return False + +run_on_onnxruntime = pytest.mark.skipif(not is_enable_onnxruntime(), reason="Only support running on onnxruntime") diff --git a/tests/ut/python/utils/__init__.py b/tests/ut/python/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/python/utils/test_callback.py b/tests/ut/python/utils/test_callback.py new file mode 100644 index 00000000000..60e4c6527a0 --- /dev/null +++ b/tests/ut/python/utils/test_callback.py @@ -0,0 +1,344 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test callback function.""" +import os +import stat +import numpy as np +import pytest + +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import context +from mindspore.common.tensor import Tensor +from mindspore.nn.optim import Momentum +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.train.callback import ModelCheckpoint, _check_file_name_prefix, RunContext,_checkpoint_cb_for_save_op,\ + LossMonitor, _InternalCallbackParam, _chg_ckpt_file_name_if_same_exist,\ + _build_callbacks, CheckpointConfig, _set_cur_net +from mindspore.common.api import ms_function + +class Net(nn.Cell): + """Net definition.""" + + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) + + @ms_function + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +class LossNet(nn.Cell): + """ LossNet definition """ + def __init__(self): + super(LossNet, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 + self.loss = nn.SoftmaxCrossEntropyWithLogits() + + @ms_function + def construct(self, x, y): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + x = self.fc(x) + out = self.loss(x, y) + return out + + +def test_Model_Checkpoint_prefix_invalid(): + """Test ModelCheckpoint prefix invalid.""" + with pytest.raises(ValueError): + ModelCheckpoint(123) + ModelCheckpoint(directory="./") + with pytest.raises(TypeError): + ModelCheckpoint(config='type_error') + ModelCheckpoint(config=CheckpointConfig()) + ModelCheckpoint(prefix="ckpt_2", directory="./test_files") + + +def test_save_checkpoint(): + """Test save checkpoint.""" + train_config = CheckpointConfig( + save_checkpoint_steps=16, + save_checkpoint_seconds=0, + keep_checkpoint_max=5, + keep_checkpoint_per_n_minutes=0) + cb_params = _InternalCallbackParam() + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + network_ = WithLossCell(net, loss) + _train_network = TrainOneStepCell(network_, optim) + cb_params.train_network = _train_network + cb_params.epoch_num = 10 + cb_params.cur_epoch_num = 5 + cb_params.cur_step_num = 0 + cb_params.batch_num = 32 + ckpoint_cb = ModelCheckpoint(prefix="test_ckpt", directory='./test_files', config=train_config) + run_context = RunContext(cb_params) + ckpoint_cb.begin(run_context) + ckpoint_cb.step_end(run_context) + if os.path.exists('./test_files/test_ckpt-model.pkl'): + os.chmod('./test_files/test_ckpt-model.pkl', stat.S_IWRITE) + os.remove('./test_files/test_ckpt-model.pkl') + + +def test_loss_monitor_graph_model(): + """Test lossmonitor Graph model.""" + cb_params = _InternalCallbackParam() + cb_params.cur_epoch_num = 4 + cb_params.cur_step_num = 2 + cb_params.batch_num = 2 + cb_params.net_outputs = Tensor(2.0) + run_context = RunContext(cb_params) + loss_cb = LossMonitor(1) + callbacks = [loss_cb] + callbacklist = _build_callbacks(callbacks) + callbacklist.begin(run_context) + callbacklist.epoch_begin(run_context) + callbacklist.step_begin(run_context) + callbacklist.step_end(run_context) + callbacklist.epoch_end(run_context) + callbacklist.end(run_context) + + +def test_Loss_Monitor_feed_feed_model(): + """Test Loss Monitor feed feed mode.""" + cb_params = _InternalCallbackParam() + run_context = RunContext(cb_params) + loss_cb = LossMonitor(1) + cb_params.cur_epoch_num = 4 + cb_params.cur_step_num = 1 + cb_params.batch_num = 1 + cb_params.net_outputs = Tensor(2.0) + loss_cb.begin(run_context) + loss_cb.epoch_begin(run_context) + loss_cb.step_begin(run_context) + loss_cb.step_end(run_context) + loss_cb.epoch_end(run_context) + loss_cb.end(run_context) + + +def test_check_file_name_not_str(): + """Test check file name not str.""" + ret = _check_file_name_prefix(1) + assert not ret + + +def test_check_file_name_back_err(): + """Test check file name back err.""" + ret = _check_file_name_prefix('abc.') + assert ret + + +def test_check_file_name_one_alpha(): + """Test check file name one alpha.""" + ret = _check_file_name_prefix('a') + assert ret + ret = _check_file_name_prefix('_') + assert ret + + +def test_check_file_name_err(): + """Test check file name err.""" + ret = _check_file_name_prefix('_123') + assert ret + + +def test_chg_ckpt_file_name_if_same_exist(): + """Test chg ckpt file name if same exist.""" + _chg_ckpt_file_name_if_same_exist(directory="./test_files", prefix="ckpt") + + +def test_checkpoint_cb_for_save_op(): + """Test checkpoint cb for save op.""" + parameter_list = [] + one_param = {} + one_param['name'] = "conv1.weight" + one_param['data'] = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), dtype=mstype.float32) + parameter_list.append(one_param) + _checkpoint_cb_for_save_op(parameter_list) + + +def test_checkpoint_cb_for_save_op_update_net(): + """Test checkpoint cb for save op.""" + parameter_list = [] + one_param = {} + one_param['name'] = "conv.weight" + one_param['data'] = Tensor(np.ones(shape=(64, 3, 3, 3)), dtype=mstype.float32) + parameter_list.append(one_param) + net = Net() + _set_cur_net(net) + _checkpoint_cb_for_save_op(parameter_list) + assert net.conv.weight.default_input.asnumpy()[0][0][0][0] == 1 + + +def test_internal_callback_param(): + """Test Internal CallbackParam.""" + cb_params = _InternalCallbackParam() + cb_params.member1 = 1 + cb_params.member2 = "abc" + assert cb_params.member1 == 1 + assert cb_params.member2 == "abc" + + +def test_checkpoint_save_ckpt_steps(): + """Test checkpoint save ckpt steps.""" + train_config = CheckpointConfig( + save_checkpoint_steps=16, + save_checkpoint_seconds=0, + keep_checkpoint_max=5, + keep_checkpoint_per_n_minutes=0) + ckpt_cb = ModelCheckpoint(config=train_config) + cb_params = _InternalCallbackParam() + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + network_ = WithLossCell(net, loss) + _train_network = TrainOneStepCell(network_, optim) + cb_params.train_network = _train_network + cb_params.epoch_num = 10 + cb_params.cur_epoch_num = 5 + cb_params.cur_step_num = 160 + cb_params.batch_num = 32 + run_context = RunContext(cb_params) + ckpt_cb.begin(run_context) + ckpt_cb.step_end(run_context) + ckpt_cb2 = ModelCheckpoint(config=train_config) + cb_params.cur_epoch_num = 1 + cb_params.cur_step_num = 15 + ckpt_cb2.begin(run_context) + ckpt_cb2.step_end(run_context) + + +def test_checkpoint_save_ckpt_seconds(): + """Test checkpoint save ckpt seconds.""" + train_config = CheckpointConfig( + save_checkpoint_steps=16, + save_checkpoint_seconds=100, + keep_checkpoint_max=0, + keep_checkpoint_per_n_minutes=1) + ckpt_cb = ModelCheckpoint(config=train_config) + cb_params = _InternalCallbackParam() + net = Net() + loss = nn.SoftmaxCrossEntropyWithLogits() + optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + network_ = WithLossCell(net, loss) + _train_network = TrainOneStepCell(network_, optim) + cb_params.train_network = _train_network + cb_params.epoch_num = 10 + cb_params.cur_epoch_num = 4 + cb_params.cur_step_num = 128 + cb_params.batch_num = 32 + run_context = RunContext(cb_params) + ckpt_cb.begin(run_context) + ckpt_cb.step_end(run_context) + ckpt_cb2 = ModelCheckpoint(config=train_config) + cb_params.cur_epoch_num = 1 + cb_params.cur_step_num = 16 + ckpt_cb2.begin(run_context) + ckpt_cb2.step_end(run_context) + + +def test_build_callbacks(): + """Test_build_callbacks.""" + ck_obj = ModelCheckpoint() + loss_cb_1 = LossMonitor(1) + + callbacks = [None] + with pytest.raises(TypeError): + callbacks = _build_callbacks(callbacks) + + callbacks = ['Error'] + with pytest.raises(TypeError): + callbacks = _build_callbacks(callbacks) + + callbacks = [ck_obj, loss_cb_1, 'Error', None] + with pytest.raises(TypeError): + callback_list = _build_callbacks(callbacks) + + +def test_RunContext(): + """Test RunContext.""" + context_err = 666 + with pytest.raises(TypeError): + context = RunContext(context_err) + + cb_params = _InternalCallbackParam() + cb_params.member1 = 1 + cb_params.member2 = "abc" + + run_context = RunContext(cb_params) + run_context.original_args() + assert cb_params.member1 == 1 + assert cb_params.member2 == "abc" + + run_context.request_stop() + should_stop = run_context.get_stop_requested() + assert should_stop + + +def test_Checkpoint_Config(): + """Test CheckpointConfig all None or 0.""" + with pytest.raises(ValueError): + CheckpointConfig(0, 0, 0, 0) + + with pytest.raises(ValueError): + CheckpointConfig(0, None, 0, 0) + + +def test_step_end_save_graph(): + """Test save checkpoint.""" + train_config = CheckpointConfig( + save_checkpoint_steps=16, + save_checkpoint_seconds=0, + keep_checkpoint_max=5, + keep_checkpoint_per_n_minutes=0) + cb_params = _InternalCallbackParam() + net = LossNet() + input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)) + input_label = Tensor(np.random.randint(0, 3, [1, 3]).astype(np.float32)) + net(input_data, input_label) + cb_params.train_network = net + cb_params.epoch_num = 10 + cb_params.cur_epoch_num = 5 + cb_params.cur_step_num = 0 + cb_params.batch_num = 32 + ckpoint_cb = ModelCheckpoint(prefix="test", directory='./test_files', config=train_config) + run_context = RunContext(cb_params) + ckpoint_cb.begin(run_context) + # import pdb;pdb.set_trace() + ckpoint_cb.step_end(run_context) + assert os.path.exists('./test_files/test-graph.meta') == True + if os.path.exists('./test_files/test-graph.meta'): + os.chmod('./test_files/test-graph.meta', stat.S_IWRITE) + os.remove('./test_files/test-graph.meta') + ckpoint_cb.step_end(run_context) + assert os.path.exists('./test_files/test-graph.meta') == False diff --git a/tests/ut/python/utils/test_checkparam.py b/tests/ut/python/utils/test_checkparam.py new file mode 100644 index 00000000000..9289d26f487 --- /dev/null +++ b/tests/ut/python/utils/test_checkparam.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_checkparam """ +import numpy as np +import pytest +import mindspore +import mindspore.nn as nn +from mindspore import Model, context +from mindspore.common.tensor import Tensor + + +class LeNet5(nn.Cell): + """ LeNet5 definition """ + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5, pad_mode="valid") + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode="valid") + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 3) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.flatten = nn.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +def predict_checke_param(in_str): + """ predict_checke_param """ + net = LeNet5() # neural network + context.set_context(mode=context.GRAPH_MODE) + model = Model(net) + + a1, a2, b1, b2, b3, b4 = in_str.strip().split() + a1 = int(a1) + a2 = int(a2) + b1 = int(b1) + b2 = int(b2) + b3 = int(b3) + b4 = int(b4) + + nd_data = np.random.randint(a1, a2, [b1, b2, b3, b4]) + input_data = Tensor(nd_data, mindspore.float32) + model.predict(input_data) + + +def test_predict_checke_param_failed(): + """ test_predict_checke_param_failed """ + in_str = "0 255 0 3 32 32" + with pytest.raises(ValueError): + predict_checke_param(in_str) diff --git a/tests/ut/python/utils/test_initializer.py b/tests/ut/python/utils/test_initializer.py new file mode 100644 index 00000000000..ff7ab8d1198 --- /dev/null +++ b/tests/ut/python/utils/test_initializer.py @@ -0,0 +1,214 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_initializer """ +import math +from functools import reduce +import numpy as np +from scipy import stats +import pytest as py +import mindspore.common.initializer as init +import mindspore as ms +from mindspore import context +from mindspore.nn import Conv2d +from ..ut_filter import non_graph_engine +# pylint: disable=W0212 +# W0212: protected-access + +class InitTwo(init.Initializer): + """Initialize the array to two.""" + def _initialize(self, arr): + init._assignment(arr, 2) + + +def _check_value(tensor, value_min, value_max): + nd = tensor.asnumpy() + for ele in nd.flatten(): + if value_min <= ele <= value_max: + continue + raise TypeError('value_min = %d, ele = %d, value_max = %d' + % (value_min, ele, value_max)) + + +def _check_uniform(tensor, boundary_a, boundary_b): + samples = tensor.asnumpy().reshape((-1)) + _, p = stats.kstest(samples, 'uniform', (boundary_a, (boundary_b - boundary_a))) + print("p-value is %f"%p) + return p > 0.0001 + + +def test_init_Initializer(): + tensor = init.initializer(InitTwo(), [2, 2], ms.int32) + assert tensor.shape() == (2, 2) + _check_value(tensor, 2, 2) + + +def test_init_tensor(): + tensor = ms.Tensor(np.zeros([1, 2, 3])) + tensor = init.initializer(tensor, [1, 2, 3], ms.float32) + assert tensor.shape() == (1, 2, 3) + + +def test_init_zero_default_dtype(): + tensor = init.initializer(init.Zero(), [2, 2]) + assert tensor.dtype() == ms.float32 + _check_value(tensor, 0, 0) + + +def test_init_zero(): + tensor = init.initializer(init.Zero(), [2, 2], ms.float32) + _check_value(tensor, 0, 0) + + +def test_init_zero_alias_default_dtype(): + tensor = init.initializer('zeros', [1, 2]) + assert tensor.dtype() == ms.float32 + _check_value(tensor, 0, 0) + + +def test_init_zero_alias(): + tensor = init.initializer('zeros', [1, 2], ms.float32) + _check_value(tensor, 0, 0) + + +def test_init_one(): + tensor = init.initializer(init.One(), [2, 2], ms.float32) + _check_value(tensor, 1, 1) + + +def test_init_one_alias(): + tensor = init.initializer('ones', [1, 2], ms.float32) + _check_value(tensor, 1, 1) + + +def test_init_uniform(): + scale = 10 + tensor = init.initializer(init.Uniform(scale=scale), [5, 4], ms.float32) + _check_value(tensor, -scale, scale) + + +def test_init_uniform_alias(): + scale = 100 + tensor = init.initializer('uniform', [5, 4], ms.float32) + _check_value(tensor, -scale, scale) + + +def test_init_normal(): + tensor = init.initializer(init.Normal(), [5, 4], ms.float32) + assert isinstance(tensor, ms.Tensor), 'tensor init failed!' + + +def test_init_truncated_normal(): + tensor = init.initializer(init.TruncatedNormal(), [5, 4], ms.float32) + assert isinstance(tensor, ms.Tensor), 'tensor init failed!' + + +def test_init_normal_alias(): + tensor = init.initializer('normal', [5, 4], ms.float32) + assert isinstance(tensor, ms.Tensor), 'tensor init failed!' + + +def test_init_truncatednormal_alias(): + tensor = init.initializer('truncatednormal', [5, 4], ms.float32) + assert isinstance(tensor, ms.Tensor), 'tensor init failed!' + + +def test_init_abnormal(): + with py.raises(TypeError): + init.initializer([''], [5, 4], ms.float32) + + +def test_init_xavier_uniform(): + """ test_init_xavier_uniform """ + gain = 1.2 + tensor1 = init.initializer(init.XavierUniform(gain=gain), [20, 22], ms.float32) + tensor2 = init.initializer(init.XavierUniform(), [20, 22], ms.float32) + tensor3 = init.initializer(init.XavierUniform(gain=gain), [20, 22, 5, 5], ms.float32) + tensor4 = init.initializer(init.XavierUniform(), [20, 22, 5, 5], ms.float32) + tensor5 = init.initializer('xavier_uniform', [20, 22, 5, 5], ms.float32) + tensor6 = init.initializer('xavier_uniform', [20, 22], ms.float32) + tensor_dict = {tensor1: gain, tensor2: None, tensor3: gain, tensor4: None, tensor5: None, tensor6: None} + + for tensor, gain_value in tensor_dict.items(): + if gain_value is None: + gain_value = 1 + shape = tensor.asnumpy().shape + if len(shape) > 2: + s = reduce(lambda x, y: x * y, shape[2:]) + else: + s = 1 + n_in = shape[1] * s + n_out = shape[0] * s + std = gain_value * math.sqrt(2 / (n_in + n_out)) + boundary = std * math.sqrt(3) + assert _check_uniform(tensor, -boundary, boundary) + + +def test_init_xavier_uniform_error(): + with py.raises(ValueError): + init.initializer(init.XavierUniform(), [6], ms.float32) + + +def test_init_he_uniform(): + """ test_init_he_uniform """ + tensor1 = init.initializer(init.HeUniform(), [20, 22], ms.float32) + tensor2 = init.initializer(init.HeUniform(), [20, 22, 5, 5], ms.float32) + tensor3 = init.initializer('he_uniform', [20, 22, 5, 5], ms.float32) + tensor4 = init.initializer('he_uniform', [20, 22], ms.float32) + tensors = [tensor1, tensor2, tensor3, tensor4] + + for tensor in tensors: + shape = tensor.asnumpy().shape + if len(shape) > 2: + s = reduce(lambda x, y: x * y, shape[2:]) + else: + s = 1 + n_in = shape[1] * s + std = math.sqrt(2 / n_in) + boundary = std * math.sqrt(3) + assert _check_uniform(tensor, -boundary, boundary) + + +def test_init_he_uniform_error(): + with py.raises(ValueError): + init.initializer(init.HeUniform(), [6], ms.float32) + + +def test_conv2d_abnormal_kernel_negative(): + kernel = np.random.randn(64, 3, 7, 7).astype(np.float32) + with py.raises(ValueError): + ms.Model( + Conv2d(in_channels=3, out_channels=64, kernel_size=-7, stride=3, + padding=0, weight_init=ms.Tensor(kernel))) + + +@non_graph_engine +def test_conv2d_abnormal_kernel_normal(): + kernel = np.random.randn(64, 3, 7, 7).astype(np.float32) + input_data = np.random.randn(32, 3, 224, 112).astype(np.float32) + context.set_context(mode=context.GRAPH_MODE) + model = ms.Model( + Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=3, + padding=0, weight_init=ms.Tensor(kernel))) + model.predict(ms.Tensor(input_data)) + + +@non_graph_engine +def test_conv2d_abnormal_kernel_truncated_normal(): + input_data = init.initializer(init.TruncatedNormal(), [64, 3, 7, 7], ms.float32) + context.set_context(mode=context.GRAPH_MODE) + model = ms.Model( + Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=3, + padding=0, weight_init="truncatednormal")) + model.predict(input_data) diff --git a/tests/ut/python/utils/test_initializer_fuzz.py b/tests/ut/python/utils/test_initializer_fuzz.py new file mode 100644 index 00000000000..0073396bc33 --- /dev/null +++ b/tests/ut/python/utils/test_initializer_fuzz.py @@ -0,0 +1,103 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_initializer_fuzz """ +import pytest +import mindspore.nn as nn +from mindspore import Model, context + + +class Net(nn.Cell): + """ Net definition """ + def __init__(self, in_str): + a, b, c, d, e, f, g, h = in_str.strip().split() + a = int(a) + b = int(b) + c = int(b) + d = int(b) + e = int(b) + f = int(b) + g = int(b) + h = int(b) + + super(Net, self).__init__() + self.conv = nn.Conv2d(a, b, c, pad_mode="valid") + self.bn = nn.BatchNorm2d(d) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(e * f * g, h) + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +def test_shape_error(): + """ for fuzz test""" + in_str = "3 22222222222222222222222222264 3 64 64 222 222 3" + with pytest.raises(ValueError): + Net(in_str) + + +class LeNet5(nn.Cell): + """ LeNet5 definition """ + def __init__(self, in_str): + super(LeNet5, self).__init__() + + a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 = in_str.strip().split() + a1 = int(a1) + a2 = int(a2) + a3 = int(a3) + a4 = int(a4) + a5 = int(a5) + a6 = int(a6) + a7 = int(a7) + a8 = int(a8) + a9 = int(a9) + a10 = int(a10) + a11 = int(a11) + a12 = int(a12) + a13 = int(a13) + a14 = int(a14) + a15 = int(a15) + + self.conv1 = nn.Conv2d(a1, a2, a3, pad_mode="valid") + self.conv2 = nn.Conv2d(a4, a5, a6, pad_mode="valid") + self.fc1 = nn.Dense(a7 * a8 * a9, a10) + self.fc2 = nn.Dense(a11, a12) + self.fc3 = nn.Dense(a13, a14) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=a15) + self.flatten = nn.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +def test_shape_error_2(): + """ for fuzz test""" + in_str = "3 6 5 6 -6 5 16 5 5 120 120 84 84 3 2" + with pytest.raises(ValueError): + net = LeNet5(in_str) # neural network + Model(net) diff --git a/tests/ut/python/utils/test_serialize.py b/tests/ut/python/utils/test_serialize.py new file mode 100644 index 00000000000..12937e5a83c --- /dev/null +++ b/tests/ut/python/utils/test_serialize.py @@ -0,0 +1,405 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ut for model serialize(save/load)""" +import os +import stat +import time +import numpy as np +import pytest + +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.ops import operations as P +from mindspore.nn import SoftmaxCrossEntropyWithLogits +from mindspore.nn.optim.momentum import Momentum +from mindspore.nn import WithLossCell, TrainOneStepCell +from mindspore.train.callback import _CheckpointManager +from mindspore.train.serialization import save_checkpoint, load_checkpoint,load_param_into_net, \ + _exec_save_checkpoint, export, _save_graph +from ..ut_filter import run_on_onnxruntime +from mindspore import context + + +context.set_context(mode=context.GRAPH_MODE) + + +class Net(nn.Cell): + """Net definition.""" + def __init__(self, num_classes=10): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=0, weight_init="zeros") + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=0) + self.flatten = nn.Flatten() + self.fc = nn.Dense(int(224*224*64/16), num_classes) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.flatten(x) + x = self.fc(x) + return x + + +_input_x = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)) +_cur_dir = os.path.dirname(os.path.realpath(__file__)) + + +def setup_module(): + import shutil + if os.path.exists('./test_files'): + shutil.rmtree('./test_files') + + +def test_save_graph(): + """ test_exec_save_graph """ + + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.add = P.TensorAdd() + + def construct(self, x, y): + z = self.add(x, y) + return z + + net = Net() + net.set_train() + out_me_list = [] + x = Tensor(np.random.rand(2, 1, 2, 3).astype(np.float32)) + y = Tensor(np.array([1.2]).astype(np.float32)) + out_put = net(x, y) + _save_graph(network=net, file_name="net-graph.meta") + out_me_list.append(out_put) + + +def test_save_checkpoint(): + """ test_save_checkpoint """ + parameter_list = [] + one_param = {} + param1 = {} + param2 = {} + one_param['name'] = "param_test" + one_param['data'] = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), dtype=mstype.float32) + param1['name'] = "param" + param1['data'] = Tensor(np.random.randint(0, 255, [12, 1024]), dtype=mstype.float32) + param2['name'] = "new_param" + param2['data'] = Tensor(np.random.randint(0, 255, [12, 1024, 1]), dtype=mstype.float32) + parameter_list.append(one_param) + parameter_list.append(param1) + parameter_list.append(param2) + + if os.path.exists('./parameters.ckpt'): + os.chmod('./parameters.ckpt', stat.S_IWRITE) + os.remove('./parameters.ckpt') + + ckpoint_file_name = os.path.join(_cur_dir, './parameters.ckpt') + save_checkpoint(parameter_list, ckpoint_file_name) + + +def test_load_checkpoint_error_filename(): + ckpoint_file_name = 1 + with pytest.raises(ValueError): + load_checkpoint(ckpoint_file_name) + + +def test_load_checkpoint(): + ckpoint_file_name = os.path.join(_cur_dir, './parameters.ckpt') + par_dict = load_checkpoint(ckpoint_file_name) + + assert len(par_dict) == 3 + assert par_dict['param_test'].name == 'param_test' + assert par_dict['param_test'].data.dtype() == mstype.float32 + assert par_dict['param_test'].data.shape() == (1, 3, 224, 224) + assert isinstance(par_dict, dict) + + +def test_checkpoint_manager(): + """ test_checkpoint_manager """ + ckp_mgr = _CheckpointManager() + + ckpoint_file_name = os.path.join(_cur_dir, './test1.ckpt') + with open(ckpoint_file_name, 'w'): + os.chmod(ckpoint_file_name, stat.S_IWUSR | stat.S_IRUSR) + + ckp_mgr.update_ckpoint_filelist(_cur_dir, "test") + assert ckp_mgr.ckpoint_num == 1 + + ckp_mgr.remove_ckpoint_file(ckpoint_file_name) + ckp_mgr.update_ckpoint_filelist(_cur_dir, "test") + assert ckp_mgr.ckpoint_num == 0 + assert not os.path.exists(ckpoint_file_name) + + another_file_name = os.path.join(_cur_dir, './test2.ckpt') + another_file_name = os.path.realpath(another_file_name) + with open(another_file_name, 'w'): + os.chmod(another_file_name, stat.S_IWUSR | stat.S_IRUSR) + + ckp_mgr.update_ckpoint_filelist(_cur_dir, "test") + assert ckp_mgr.ckpoint_num == 1 + ckp_mgr.remove_oldest_ckpoint_file() + ckp_mgr.update_ckpoint_filelist(_cur_dir, "test") + assert ckp_mgr.ckpoint_num == 0 + assert not os.path.exists(another_file_name) + + # test keep_one_ckpoint_per_minutes + file1 = os.path.realpath(os.path.join(_cur_dir, './time_file1.ckpt')) + file2 = os.path.realpath(os.path.join(_cur_dir, './time_file2.ckpt')) + file3 = os.path.realpath(os.path.join(_cur_dir, './time_file3.ckpt')) + with open(file1, 'w'): + os.chmod(file1, stat.S_IWUSR | stat.S_IRUSR) + with open(file2, 'w'): + os.chmod(file2, stat.S_IWUSR | stat.S_IRUSR) + with open(file3, 'w'): + os.chmod(file3, stat.S_IWUSR | stat.S_IRUSR) + time1 = time.time() + ckp_mgr.update_ckpoint_filelist(_cur_dir, "time_file") + assert ckp_mgr.ckpoint_num == 3 + ckp_mgr.keep_one_ckpoint_per_minutes(1, time1) + ckp_mgr.update_ckpoint_filelist(_cur_dir, "time_file") + assert ckp_mgr.ckpoint_num == 1 + if os.path.exists(_cur_dir + '/time_file1.ckpt'): + os.chmod(_cur_dir + '/time_file1.ckpt', stat.S_IWRITE) + os.remove(_cur_dir + '/time_file1.ckpt') + + +def test_load_param_into_net_error_net(): + parameter_dict = {} + one_param = Parameter(Tensor(np.ones(shape=(64, 3, 7, 7)), dtype=mstype.float32), + name="conv1.weight") + parameter_dict["conv1.weight"] = one_param + with pytest.raises(TypeError): + load_param_into_net('', parameter_dict) + + +def test_load_param_into_net_error_dict(): + net = Net(10) + with pytest.raises(TypeError): + load_param_into_net(net, '') + + +def test_load_param_into_net_erro_dict_param(): + net = Net(10) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 0 + + parameter_dict = {} + one_param = '' + parameter_dict["conv1.weight"] = one_param + with pytest.raises(TypeError): + load_param_into_net(net, parameter_dict) + + +def test_load_param_into_net_has_more_param(): + """ test_load_param_into_net_has_more_param """ + net = Net(10) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 0 + + parameter_dict = {} + one_param = Parameter(Tensor(np.ones(shape=(64, 3, 7, 7)), dtype=mstype.float32), + name="conv1.weight") + parameter_dict["conv1.weight"] = one_param + two_param = Parameter(Tensor(np.ones(shape=(64, 3, 7, 7)), dtype=mstype.float32), + name="conv1.weight") + parameter_dict["conv1.w"] = two_param + load_param_into_net(net, parameter_dict) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 1 + + +def test_load_param_into_net_param_type_and_shape_error(): + net = Net(10) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 0 + + parameter_dict = {} + one_param = Parameter(Tensor(np.ones(shape=(64, 3, 7, 7))), name="conv1.weight") + parameter_dict["conv1.weight"] = one_param + with pytest.raises(RuntimeError): + load_param_into_net(net, parameter_dict) + + +def test_load_param_into_net_param_type_error(): + net = Net(10) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 0 + + parameter_dict = {} + one_param = Parameter(Tensor(np.ones(shape=(64, 3, 7, 7)), dtype=mstype.int32), + name="conv1.weight") + parameter_dict["conv1.weight"] = one_param + with pytest.raises(RuntimeError): + load_param_into_net(net, parameter_dict) + + +def test_load_param_into_net_param_shape_error(): + net = Net(10) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 0 + + parameter_dict = {} + one_param = Parameter(Tensor(np.ones(shape=(64, 3, 7,)), dtype=mstype.int32), + name="conv1.weight") + parameter_dict["conv1.weight"] = one_param + with pytest.raises(RuntimeError): + load_param_into_net(net, parameter_dict) + + +def test_load_param_into_net(): + net = Net(10) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 0 + + parameter_dict = {} + one_param = Parameter(Tensor(np.ones(shape=(64, 3, 7, 7)), dtype=mstype.float32), + name="conv1.weight") + parameter_dict["conv1.weight"] = one_param + load_param_into_net(net, parameter_dict) + assert net.conv1.weight.default_input.asnumpy()[0][0][0][0] == 1 + + +def test_exec_save_checkpoint(): + net = Net() + loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + opt = Momentum(net.trainable_params(), 0.0, 0.9, 0.0001, 1024) + + loss_net = WithLossCell(net, loss) + train_network = TrainOneStepCell(loss_net, opt) + _exec_save_checkpoint(train_network, ckpoint_file_name="./new_ckpt.ckpt") + + load_checkpoint("new_ckpt.ckpt") + + +def test_load_checkpoint_empty_file(): + os.mknod("empty.ckpt") + with pytest.raises(ValueError): + load_checkpoint("empty.ckpt") + + +class MYNET(nn.Cell): + """ NET definition """ + def __init__(self): + super(MYNET, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') + self.bn = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() + self.fc = nn.Dense(64*222*222, 3) # padding=0 + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + + +def test_export(): + net = MYNET() + input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]).astype(np.float32)) + export(net, input_data, file_name="./me_export.pb", file_format="GEIR") + + +class BatchNormTester(nn.Cell): + "used to test exporting network in training mode in onnx format" + def __init__(self, num_features): + super(BatchNormTester, self).__init__() + self.bn = nn.BatchNorm2d(num_features) + + def construct(self, x): + return self.bn(x) + + +def test_batchnorm_train_onnx_export(): + input = Tensor(np.ones([1, 3, 32, 32]).astype(np.float32) * 0.01) + net = BatchNormTester(3) + net.set_train() + if not net.training: + raise ValueError('netowrk is not in training mode') + export(net, input, file_name='batch_norm.onnx', file_format='ONNX') + if not net.training: + raise ValueError('netowrk is not in training mode') + + +class LeNet5(nn.Cell): + """LeNet5 definition""" + def __init__(self): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = P.Flatten() + + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +def test_lenet5_onnx_export(): + input = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + net = LeNet5() + export(net, input, file_name='lenet5.onnx', file_format='ONNX') + + +@run_on_onnxruntime +def test_lenet5_onnx_load_run(): + onnx_file = 'lenet5.onnx' + + input = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) + net = LeNet5() + export(net, input, file_name=onnx_file, file_format='ONNX') + + import onnx + import onnxruntime as ort + + print('--------------------- onnx load ---------------------') + # Load the ONNX model + model = onnx.load(onnx_file) + # Check that the IR is well formed + onnx.checker.check_model(model) + # Print a human readable representation of the graph + g = onnx.helper.printable_graph(model.graph) + print(g) + + print('------------------ onnxruntime run ------------------') + ort_session = ort.InferenceSession(onnx_file) + input_map = {'x' : input.asnumpy()} + # provide only input x to run model + outputs = ort_session.run(None, input_map) + print(outputs[0]) + # overwrite default weight to run model + for item in net.trainable_params(): + input_map[item.name] = np.ones(item.default_input.asnumpy().shape, dtype=np.float32) + outputs = ort_session.run(None, input_map) + print(outputs[0]) + + +def teardown_module(): + files = ['parameters.ckpt', 'new_ckpt.ckpt', 'lenet5.onnx', 'batch_norm.onnx', 'empty.ckpt'] + for item in files: + file_name = './' + item + if not os.path.exists(file_name): + continue + os.chmod(file_name, stat.S_IWRITE) + os.remove(file_name) diff --git a/tests/vm_impl/__init__.py b/tests/vm_impl/__init__.py new file mode 100644 index 00000000000..aff30eccc16 --- /dev/null +++ b/tests/vm_impl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Vm implementation.""" +from . import math_ops_vm_impl +from . import nn_ops_vm_impl +from . import array_ops_vm_impl +from .vm_interface import vm diff --git a/tests/vm_impl/array_ops_vm_impl.py b/tests/vm_impl/array_ops_vm_impl.py new file mode 100644 index 00000000000..4258dadc62f --- /dev/null +++ b/tests/vm_impl/array_ops_vm_impl.py @@ -0,0 +1,236 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Generate vm_impl function for array ops""" +import numpy as np + +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as mstype +from mindspore.ops.vm_impl_registry import vm_impl_registry as vm_impl_getters +from .vm_interface import vm +# pylint: disable=unused-argument + + +@vm_impl_getters.register(P.ExpandDims) +def vm_impl_expand_dims(self): + """Generate vm_impl function for ExpandDims""" + def vm_impl(x, axis): + if isinstance(x, float): + x = Tensor(np.array([x])) + x = x.asnumpy() + out = vm.expand_dims(x, axis) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.DType) +def vm_impl_dType(self): + """Generate vm_impl function for DType""" + def vm_impl(x): + # update the src type + return x.dtype() + return vm_impl + + +@vm_impl_getters.register(P.Cast) +def vm_impl_cast(self): + """Generate vm_impl function for Cast""" + def vm_impl(x, t): + if isinstance(t, type(mstype.tensor)): + t = t.element_type() + # update the src type + x = x.asnumpy() + out = x.astype(mstype.dtype_to_nptype(t)) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Reshape) +def vm_impl_reshape(self): + """Generate vm_impl function for Reshape""" + def vm_impl(x, shp): + x = x.asnumpy() + out = vm.reshape(x, shp) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Shape) +def vm_impl_shape(self): + """Generate vm_impl function for Shape""" + def vm_impl(x): + shp = vm.shape(x.asnumpy()) + return shp + return vm_impl + + +@vm_impl_getters.register(P.Squeeze) +def vm_impl_squeeze(self): + """Generate vm_impl function for Squeeze""" + def vm_impl(x): + x = x.asnumpy() + out = vm.squeeze(x, self.axis) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Transpose) +def vm_impl_transpose(self): + """Generate vm_impl function for Transpose""" + def vm_impl(x, perm=None): + x = x.asnumpy() + if perm is None: + perm = [i for i in reversed(range(len(x.shape)))] + out = vm.transpose(x, perm) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Split) +def vm_impl_split(self): + """Generate vm_impl function for Split""" + def vm_impl(x): + x = x.asnumpy() + output = np.array_split(x, (self.pos,)) + return Tensor(output[0]), Tensor(output[1]) + return vm_impl + + +@vm_impl_getters.register(P.Fill) +def vm_impl_fill(self): + """Generate vm_impl function for Fill""" + def vm_impl(dims, x): + if isinstance(x, int): + ret = np.full(dims, x, np.int32) + else: + ret = np.full(dims, x, np.float32) + return Tensor(ret) + return vm_impl + + +@vm_impl_getters.register(P.Eye) +def vm_impl_eye(self): + """Generate vm_impl function for Eye""" + def vm_impl(n, m, t): + np_type = mstype.dtype_to_nptype(t) + ret = np.eye(n, m, dtype=np_type) + return Tensor(ret) + return vm_impl + + +@vm_impl_getters.register(P.InvertPermutation) +def vm_impl_invert_permutation(self): + """Generate vm_impl function for InvertPermutation""" + def vm_impl(x): + out = vm.invert_permutation(x) + return out + return vm_impl + + +@vm_impl_getters.register(P.Argmax) +def vm_impl_argmax(self): + """Generate vm_impl function for Argmax""" + def vm_impl(x): + output = np.argmax(x.asnumpy(), axis=self.axis) + return Tensor(output.ravel()) + return vm_impl + +@vm_impl_getters.register(P.Tile) +def vm_impl_tile(self): + """Generate vm_impl function for Tile""" + def vm_impl(x, multiples): + x = x.asnumpy() + multiples = multiples.asnumpy() + out = vm.Tile(x, multiples) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.ReduceAll) +def vm_impl_all(self): + """Generate vm_impl function for All""" + def vm_impl(x, axis): + x = x.asnumpy() + out = vm.all(x, axis) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Concat) +def vm_impl_concatV2(self): + """Generate vm_impl function for Concat""" + def vm_impl(x): + x = x.asnumpy() + out = vm.Concat(x, self.axis) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Slice) +def vm_impl_slice(self): + """Generate vm_impl function for Slice""" + def vm_impl(x, begin, size): + x = x.asnumpy() + begin = begin.asnumpy() + size = size.asnumpy() + out = vm.Slice(x, begin, size) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.ConcatOffset) +def vm_impl_concatOffset(self): + """Generate vm_impl function for ConcatOffset""" + def vm_impl(x): + out = vm.ConcatOffset(x) # out is tuple + return out + return vm_impl + + +@vm_impl_getters.register(P.ReduceSum) +def vm_impl_sum(self): + """Generate vm_impl function for Sum""" + def vm_impl(x, axis): + x = x.asnumpy() + out = vm.sum(x, axis) + return Tensor(np.array(out)) + return vm_impl + + +@vm_impl_getters.register(P.Select) +def vm_impl_select(self): + """Generate vm_impl function for Select""" + def vm_impl(cond, x, y): + """ + Args: + cond: A `Tensor` of type `bool` + x: A Tensor which may have the same shape as `condition`. + y: A `Tensor` with the same shape and type as `x`. + """ + cond = cond.asnumpy() + x = x.asnumpy() + y = y.asnumpy() + out = vm.select(cond, x, y) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Square) +def vm_impl_square(self): + """Generate vm_impl function for Square""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(x * x) + return vm_impl diff --git a/tests/vm_impl/math_ops_vm_impl.py b/tests/vm_impl/math_ops_vm_impl.py new file mode 100644 index 00000000000..fd132280d10 --- /dev/null +++ b/tests/vm_impl/math_ops_vm_impl.py @@ -0,0 +1,238 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Generate vm_impl function for math ops""" +import copy +import numpy as np +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.ops.vm_impl_registry import vm_impl_registry as vm_impl_getters +from mindspore.common.dtype import dtype_to_nptype +from .vm_interface import vm +# pylint: disable=unused-argument + + +@vm_impl_getters.register(P.TensorAdd) +def vm_impl_tensor_add(self): + """Generate vm_impl function for TensorAdd.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + return Tensor(x + y) + return vm_impl + +@vm_impl_getters.register(P.LogicalNot) +def vm_impl_logical_not(self): + x = x.asnumpy() + out = vm.logical_not(x) + return Tensor(out) + +@vm_impl_getters.register(P.MatMul) +def vm_impl_mat_mul(self): + """Generate vm_impl function for MatMul.""" + def vm_impl(x, w): + x = x.asnumpy() + w = w.asnumpy() + if self.transpose_a: + x = x.transpose() + if self.transpose_b: + w = w.transpose() + z = x@w + return Tensor(z) + return vm_impl + + +@vm_impl_getters.register(P.AddN) +def vm_impl_addn(self): + """Generate vm_impl function for AddN.""" + def vm_impl(inputs): + added = copy.deepcopy(inputs[0].asnumpy()) + for x in inputs[1:]: + added += x.asnumpy() + return Tensor(added) + return vm_impl + + +@vm_impl_getters.register(P.Neg) +def vm_impl_neg(self): + """Generate vm_impl function for Neg.""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(-x) + return vm_impl + + +@vm_impl_getters.register(P.Sub) +def vm_impl_Sub(self): + """Generate vm_impl function for Sub.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + return Tensor(x - y) + return vm_impl + + +@vm_impl_getters.register(P.Mul) +def vm_impl_mul(self): + """Generate vm_impl function for Mul.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + return Tensor(x * y) + return vm_impl + + +@vm_impl_getters.register(P.Square) +def vm_impl_square(self): + """Generate vm_impl function for Square.""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(x * x) + return vm_impl + + +@vm_impl_getters.register(P.Sqrt) +def vm_impl_sqrt(self): + """Generate vm_impl function for Sqrt.""" + def vm_impl(x): + x = x.asnumpy() + res = vm.sqrt(x) + return Tensor(res) + return vm_impl + + +@vm_impl_getters.register(P.Pow) +def vm_impl_pow(self): + """Generate vm_impl function for Pow.""" + def vm_impl(x, y): + x = x.asnumpy() + res = vm.power(x, y) + return Tensor(res) + return vm_impl + + +@vm_impl_getters.register(P.Exp) +def vm_impl_exp(self): + """Generate vm_impl function for Exp.""" + def vm_impl(x): + x = x.asnumpy() + res = vm.exp(x) + return Tensor(res) + return vm_impl + + +@vm_impl_getters.register(P.RealDiv) +def vm_impl_real_div(self): + """Generate vm_impl function for RealDiv.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = x / y + out = np.array(out, x.dtype) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Div) +def vm_impl_div(self): + """Generate vm_impl function for Div.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + return Tensor(x / y) + return vm_impl + + +@vm_impl_getters.register(P.ReduceMean) +def vm_impl_reduce_mean(self): + """Generate vm_impl function for ReduceMean.""" + def vm_impl(x, axis): + x = x.asnumpy() + out = vm.mean(x, axis) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Equal) +def vm_impl_equal(self): + """Generate vm_impl function for Equal.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = vm.equal(x, y) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.NotEqual) +def vm_impl_not_equal(self): + """Generate vm_impl function for NotEqual.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = vm.not_equal(x, y) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Greater) +def vm_impl_greater(self): + """Generate vm_impl function for Greater.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = vm.greater(x, y) + return Tensor(out) + return vm_impl + +@vm_impl_getters.register(P.Maximum) +def vm_impl_maximum(self): + """Generate vm_impl function for Maximum.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = vm.maximum(x, y) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(P.Minimum) +def vm_impl_minimum(self): + """Generate vm_impl function for Minimum.""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = vm.minimum(x, y) + return Tensor(out) + return vm_impl + +@vm_impl_getters.register(P.Less) +def vm_impl_greater(self): + """Generate vm_impl function for Less""" + def vm_impl(x, y): + x = x.asnumpy() + y = y.asnumpy() + out = vm.less(x, y) + return Tensor(out) + return vm_impl + +@vm_impl_getters.register(P.ScalarCast) +def vm_impl_greater(self): + """Generate vm_impl function for ScalarCast""" + def vm_impl(x, t): + np_type = dtype_to_nptype(t) + value = np_type(x) + cast_value = value.item() + return cast_value + return vm_impl diff --git a/tests/vm_impl/nn_ops_vm_impl.py b/tests/vm_impl/nn_ops_vm_impl.py new file mode 100644 index 00000000000..f6bbdca55a2 --- /dev/null +++ b/tests/vm_impl/nn_ops_vm_impl.py @@ -0,0 +1,345 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Generate vm_impl function for nn ops""" +import numpy as np +from mindspore.ops import operations as P +from mindspore.ops.operations import _grad_ops as G +from mindspore.common.tensor import Tensor +from mindspore.ops.vm_impl_registry import vm_impl_registry as vm_impl_getters +from .vm_interface import vm +# pylint: disable=unused-argument + + +@vm_impl_getters.register(P.ScalarSummary) +def vm_impl_scalar_summary(self): + """Generate vm_impl function for ScalarSummary""" + def vm_impl(string_in, scalar): + """Implement by vm mode.""" + return scalar + return vm_impl + + +@vm_impl_getters.register(P.ReLU) +def vm_impl_relu(self): + """Generate vm_impl function for ReLU""" + def vm_impl(x): + x = x.asnumpy() + output = Tensor(vm.relu(x)) + return output + return vm_impl + +@vm_impl_getters.register(P.Flatten) +def vm_impl_flatten(self): + """Generate vm_impl function for Flatten""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(vm.flatten_batch(x)) + return vm_impl + + +@vm_impl_getters.register(P.Softmax) +def vm_impl_softmax(self): + """Generate vm_impl function for Softmax""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(vm.softmax(x)) + return vm_impl + + +@vm_impl_getters.register(P.LogSoftmax) +def vm_impl_log_softmax(self): + """Generate vm_impl function for LogSoftmax""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(vm.logsoftmax(x)) + return vm_impl + + +@vm_impl_getters.register(P.Tanh) +def vm_impl_tanh(self): + """Generate vm_impl function for Tanh""" + def vm_impl(x): + x = x.asnumpy() + return Tensor(vm.tanh(x)) + return vm_impl + + +@vm_impl_getters.register(P.FusedBatchNorm) +def vm_impl_fused_batch_norm(self): + """Generate vm_impl function for FusedBatchNorm""" + def vm_impl(x, scale, b, mean, variance): + # pylint: disable=unused-argument + x = x.asnumpy() + scale = scale.asnumpy() + b = b.asnumpy() + mean = mean.asnumpy() + variance = variance.asnumpy() + out, x_mean, x_var, running_mean, running_var = vm.batch_norm(x, scale, b, mean, \ + variance, \ + eps=self.epsilon, \ + momentum=self.momentum) + return Tensor(out), Tensor(x_mean), Tensor(x_var), \ + Tensor(running_mean), Tensor(running_var) + return vm_impl + + +@vm_impl_getters.register(P.BatchNorm) +def vm_impl_batch_norm(self): + """Generate vm_impl function for BatchNorm""" + def vm_impl(x, scale, b, mean, variance): + # pylint: disable=unused-argument + x = x.asnumpy() + scale = scale.asnumpy() + b = b.asnumpy() + mean = mean.asnumpy() + variance = variance.asnumpy() + out, x_mean, x_var, running_mean, running_var = vm.batch_norm(x, scale, b, mean, \ + variance, \ + eps=self.epsilon) + return Tensor(out), Tensor(x_mean), Tensor(x_var), \ + Tensor(running_mean), Tensor(running_var) + return vm_impl + + +@vm_impl_getters.register(P.Conv2D) +def vm_impl_conv2d(self): + """Generate vm_impl function for Conv2D""" + def vm_impl(x, w): + x = x.asnumpy() + weight = w.asnumpy() + bias = None + out = vm.conv2d(x, weight, bias, self.stride, self.pad, self.dilation) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(G.MaxPoolGradWithArgmax) +def vm_impl_max_pool_grad_with_argmax(self): + """Generate vm_impl function for MaxPoolGradWithArgmax""" + def vm_impl(x, argmax, dout): + x = x.asnumpy() + dout = dout.asnumpy() + arg_max = argmax.asnumpy() + dx = vm.max_pool_grad_with_argmax(x, arg_max, dout, self.pool_h, self.pool_w, self.stride, self.pad) + return Tensor(dx) + return vm_impl + + +@vm_impl_getters.register(P.MaxPoolWithArgmax) +def vm_impl_max_pool_with_argmax(self): + """Generate vm_impl function for MaxPoolWithArgmax""" + def vm_impl(x): + x = x.asnumpy() + out, out_argmax = vm.max_pool_with_argmax(x, self.pool_h, self.pool_w, self.stride, self.pad) + return Tensor(out), Tensor(out_argmax) + return vm_impl + +@vm_impl_getters.register(P.MaxPool) +def vm_impl_max_pool(self): + """Generate vm_impl function for MaxPool""" + def vm_impl(x): + x = x.asnumpy() + out = vm.max_pooling(x, self.pool_h, self.pool_w, self.stride_h, self.pad) + return Tensor(out) + return vm_impl + +@vm_impl_getters.register(G.MaxPoolGrad) +def vm_impl_max_pool_grad(self): + """Generate vm_impl function for MaxPoolGrad""" + def vm_impl(x, out, dout): + x = x.asnumpy() + dout = dout.asnumpy() + out = vm.max_pool_grad(x, dout, self.pool_h, self.pool_w, self.stride_h, self.pad) + return Tensor(out) + return vm_impl + +@vm_impl_getters.register(P.AvgPool) +def vm_impl_max_pool(self): + """Generate vm_impl function for AvgPool""" + def vm_impl(x): + x = x.asnumpy() + out = vm.avg_pooling(x, self.pool_h, self.pool_w, self.stride_h, self.pad) + return Tensor(out) + return vm_impl + +@vm_impl_getters.register(G.AvgPoolGrad) +def vm_impl_avg_pool_grad(self): + """Generate vm_impl function for AvgPoolGrad""" + def vm_impl(dout, origin_shape): + dout = dout.asnumpy() + out = vm.avg_pool_grad(dout, origin_shape, self.pool_h, self.pool_w, self.stride_h, self.pad) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(G.FusedBatchNormGrad) +def vm_impl_fused_batch_norm_grad(self): + """Generate vm_impl function for FusedBatchNormGrad""" + def vm_impl(dy, x, scale, save_mean, save_inv_variance): + dy = dy.asnumpy() + x = x.asnumpy() + scale = scale.asnumpy() + save_mean = save_mean.asnumpy() + save_inv_variance = save_inv_variance.asnumpy() + dx, dscale, dshift = vm.batch_norm_grad(dy, x, scale, save_mean, save_inv_variance) + return (Tensor(dx), Tensor(dscale), Tensor(dshift)) + return vm_impl + +@vm_impl_getters.register(G.BatchNormGrad) +def vm_impl_fused_batch_norm_grad(self): + """Generate vm_impl function for BatchNormGrad""" + def vm_impl(dy, x, scale, save_mean, save_inv_variance): + dy = dy.asnumpy() + x = x.asnumpy() + scale = scale.asnumpy() + save_mean = save_mean.asnumpy() + save_inv_variance = save_inv_variance.asnumpy() + dx, dscale, dshift = vm.batch_norm_grad(dy, x, scale, save_mean, save_inv_variance) + return (Tensor(dx), Tensor(dscale), Tensor(dshift)) + return vm_impl + + +@vm_impl_getters.register(G.ReluGrad) +def vm_impl_relu_grad(self): + """Generate vm_impl function for ReluGrad""" + def vm_impl(y_backprop, x): + x = x.asnumpy() + y_backprop = y_backprop.asnumpy() + y_backprop = vm.relu_grad(x.copy())*y_backprop + return Tensor(y_backprop) + return vm_impl + + +@vm_impl_getters.register(P.Conv2DBackpropInput) +def vm_impl_conv2d_backprop_input(self): + """Generate vm_impl function for Conv2DBackpropInput""" + def vm_impl(dout, w, x_size): + dout = dout.asnumpy() + w = w.asnumpy() + dx = vm.conv2d_backprop_input(dout, x_size, w, self.stride, self.pad) + return Tensor(dx) + return vm_impl + + +@vm_impl_getters.register(G.Conv2DBackpropFilter) +def vm_impl_conv2d_backprop_filter(self): + """Generate vm_impl function for Conv2DBackpropFilter""" + def vm_impl(dout, x, w_size): + x = x.asnumpy() + dout = dout.asnumpy() + dw = vm.conv2d_backprop_filter(dout, x, w_size, self.stride, self.pad) + return Tensor(dw) + return vm_impl + + +@vm_impl_getters.register(G.FlattenGrad) +def vm_impl_flatten_grad(self): + """Generate vm_impl function for FlattenGrad""" + def vm_impl(dout, x): + dout = dout.asnumpy() + dout = vm.flatten_grad(dout, x) + return Tensor(dout) + return vm_impl + + +@vm_impl_getters.register(P.BiasAdd) +def vm_impl_bias_add(self): + """Generate vm_impl function for BiasAdd""" + def vm_impl(wx, bias): + wx = wx.asnumpy() + bias = bias.asnumpy() + out = wx + bias + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(G.BiasAddGrad) +def vm_impl_bias_add_grad(self): + """Generate vm_impl function for BiasAddGrad""" + def vm_impl(dout): + dout = dout.asnumpy() + shape = np.shape(dout) + return Tensor(np.add.reduce(dout, axis=tuple(range(len(shape) - 1)))) + return vm_impl + + +@vm_impl_getters.register(P.SoftmaxCrossEntropyWithLogits) +def vm_impl_softmax_cross_entropy_with_logits(self): + """Generate vm_impl function for SoftmaxCrossEntropyWithLogits""" + def vm_impl(logits, labels): + logits = logits.asnumpy() + labels = labels.asnumpy() + loss, dx = vm.softmax_cross_entropy_with_logits(logits, labels) + return (Tensor(np.array(loss)), Tensor(dx)) + return vm_impl + + +@vm_impl_getters.register(P.SparseSoftmaxCrossEntropyWithLogits) +def vm_impl_sparse_softmax_cross_entropy_with_logits(self): + """Generate vm_impl function for SparseSoftmaxCrossEntropyWithLogits""" + def vm_impl(logits, labels): + logits = logits.asnumpy() + labels = labels.asnumpy() + + n_class = labels.max() + 1 + n_sample = labels.shape[0] + one_hot_label = np.zeros((n_sample, n_class))#3个样本，4个类别 + one_hot_label[:, labels] = 1#非零列赋值为1 + loss, dx = vm.softmax_cross_entropy_with_logits(logits, one_hot_label) + if self.is_grad: + return (Tensor(dx),) + return (Tensor(np.array(loss)),) + return vm_impl + +@vm_impl_getters.register(P.ApplyMomentum) +def vm_impl_momentum(self): + """Generate vm_impl function for Momentum""" + def vm_impl(variable, + accumulation, + learning_rate, + gradient, + momentum, + use_nesterov=False): + gradient = gradient.asnumpy() + accumulation = accumulation.asnumpy() + variable = variable.asnumpy() + shape = accumulation.shape + learning_rate = np.full(shape, learning_rate) + momentum = np.full(shape, momentum) + accumulation = accumulation * momentum + gradient + if use_nesterov is True: + variable -= gradient * learning_rate + accumulation * momentum * learning_rate + else: + variable -= accumulation * learning_rate + return Tensor(variable) + + return vm_impl + +@vm_impl_getters.register(P.ResizeBilinear) +def vm_impl_resize_bilinear(self): + """Generate vm_impl function for ResizeBilinear""" + def vm_impl(x): + out = vm.ResizeBilinear(x) + return Tensor(out) + return vm_impl + + +@vm_impl_getters.register(G.ResizeBilinearGrad) +def vm_impl_resize_bilinear_grad(self): + """Generate vm_impl function for ResizeBilinearGrad""" + def vm_impl(dout, original_image): + out = vm.ResizeBilinearGrad(dout, original_image) + return Tensor(out) + return vm_impl diff --git a/tests/vm_impl/vm_interface.py b/tests/vm_impl/vm_interface.py new file mode 100644 index 00000000000..2400b78902f --- /dev/null +++ b/tests/vm_impl/vm_interface.py @@ -0,0 +1,69 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""vm interface.""" +# pylint: disable=wildcard-import,unused-wildcard-import +from .vm_me import * + + +class Vm: + pass + + +vm = Vm() +setattr(vm, 'avg_pooling', avg_pooling) +setattr(vm, 'avg_pool_grad', avg_pool_grad) +setattr(vm, 'batch_norm', batch_norm) +setattr(vm, 'batch_norm_grad', batch_norm_grad) +setattr(vm, 'col2im', col2im) +setattr(vm, 'convolve', convolve) +setattr(vm, 'conv2d', conv2d) +setattr(vm, 'conv2d_backprop_filter', conv2d_backprop_filter) +setattr(vm, 'conv2d_backprop_input', conv2d_backprop_input) +setattr(vm, 'flatten', flatten) +setattr(vm, 'flatten2', flatten2) +setattr(vm, 'flatten_batch', flatten_batch) +setattr(vm, 'flatten_grad', flatten_grad) +setattr(vm, 'im2col', im2col) +setattr(vm, 'matmul', matmul) +setattr(vm, 'max_pooling', max_pooling) +setattr(vm, 'max_pool_grad', max_pool_grad) +setattr(vm, 'max_pool_grad_with_argmax', max_pool_grad_with_argmax) +setattr(vm, 'max_pool_with_argmax', max_pool_with_argmax) +setattr(vm, 'relu', relu) +setattr(vm, 'relu_grad', relu_grad) +setattr(vm, 'softmax', softmax) +setattr(vm, 'softmax_cross_entropy_with_logits', softmax_cross_entropy_with_logits) +setattr(vm, 'expand_dims', expand_dims) +setattr(vm, 'squeeze', squeeze) +setattr(vm, 'reshape', reshape) +setattr(vm, 'shape', shape) +setattr(vm, 'rank', rank) +setattr(vm, 'logsoftmax', logsoftmax) +setattr(vm, 'transpose', transpose) +setattr(vm, 'invert_permutation', invert_permutation) +setattr(vm, 'select', select) +setattr(vm, 'sum', sum_by_axis) +setattr(vm, 'equal', equal) +setattr(vm, 'not_equal', not_equal) +setattr(vm, 'greater', greater) +setattr(vm, 'less', less) +setattr(vm, 'logical_not', logical_not) +setattr(vm, 'sqrt', sqrt) +setattr(vm, 'power', power) +setattr(vm, "exp", exp) +setattr(vm, "tanh", tanh) +setattr(vm, "sigmoid", sigmoid) +setattr(vm, 'maximum', maximum) +setattr(vm, 'minimum', minimum) diff --git a/tests/vm_impl/vm_me.py b/tests/vm_impl/vm_me.py new file mode 100644 index 00000000000..03a0e1a885c --- /dev/null +++ b/tests/vm_impl/vm_me.py @@ -0,0 +1,748 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""VM implementations based on numpy.""" + +import numpy as np +from mindspore._checkparam import Rel +from mindspore._checkparam import ParamValidator as validator + + +def avg_pooling(x, pool_h, pool_w, stride, pad): + """ + Applies average pooling over an input array. + + Args: + x (numpy.ndarray): The input array to be average pooled. + pool_h (int): Height of the pooling window. + pool_w (int): Width of the pooling window. + stride (int): The stride of the sliding window. + pad (int): Padding to be added on height and width. + + Returns: + numpy.ndarray, an output array after applying average pooling on input array. + """ + validator.check_integer("stride", stride, 0, Rel.GT) + num, channel, height, width = x.shape + out_h = (height + 2*pad - pool_h)//stride + 1 + out_w = (width + 2*pad - pool_w)//stride + 1 + + col = im2col(x, pool_h, pool_w, stride, pad) + col = col.reshape(-1, pool_h*pool_w) + + out = np.mean(col, axis=1) + out = out.reshape(num, out_h, out_w, channel).transpose(0, 3, 1, 2) + + return out + + +def avg_pool_grad(dout, origin_shape, pool_h, pool_w, stride, pad): + """ + Gets grad of average pooling. + + Args: + x (numpy.ndarray): The input array to be average pooled. + dout (numpy.ndarray): The grad of pre-layer. + pool_h (int): Height of the pooling window. + pool_w (int): Width of the pooling window. + stride (int): The stride of the sliding window. + pad (int): Padding to be added on height and width. + + Returns: + numpy.ndarray, grad of avgerage pooling. + """ + # pylint: disable=unused-argument + _, _, height, width = dout.shape + dx = np.zeros(origin_shape) + for i in range(height): + for j in range(width): + dx[:, :, i:(i+pool_h), j:(j+pool_w)] += np.ones((pool_h, pool_w)) + return dx + + +def _batch_norm(x, scale, shift, running_mean=None, running_var=None, + eps=1e-05, momentum=0.1, is_training=True): + """Batch normalization over an array.""" + _, c_h_w = x.shape + # Handle running_mean and running_var are not None + # if running_mean is None: + # running_mean = np.zeros(c_h_w) + # running_var = np.zeros(c_h_w) + running_mean = np.zeros(c_h_w) + running_var = np.zeros(c_h_w) + if np.ndim(scale) > 0: + scale = scale.mean() + if np.ndim(shift) > 0: + shift = shift.mean() + + if is_training: + x_mean = np.mean(x, axis=0) + x_var = np.var(x, axis=0) + + # Normalization followed by Affine transformation + x_norm = (x - x_mean)/np.sqrt(x_var + eps) + + # Estimate running average of mean and variance to use at test time + running_mean = momentum * running_mean + (1 - momentum) * x_mean + running_var = momentum * running_var + (1 - momentum) * x_var + else: + # normalize using running average + x_norm = (x - running_mean)/np.sqrt(running_var + eps) + x_mean = running_mean + x_var = running_var + + out = scale * x_norm + shift + + return out, x_mean, x_var, running_mean, running_var + + +def batch_norm(x, scale=1, shift=0, mean=None, variance=None, + eps=1e-05, momentum=0.1, is_training=True): + """Batch normalization over an array.""" + input_shape = x.shape + if x.ndim != 2: + batch_num = x.shape[0] + x = x.reshape(batch_num, -1) + + out, _, _, running_mean, running_var = _batch_norm(x, scale, shift, mean, variance, \ + eps, momentum, is_training) + + return out.reshape(*input_shape), np.array(scale), np.array(shift), running_mean, running_var + + +def _batch_norm_grad(dout, x, scale, save_mean, save_inv_variance, \ + eps=1e-05, momentum=0.1, is_training=True): + """Batch normalization over an array.""" + if x.ndim != 2: + batch_num = x.shape[0] + x = x.reshape(batch_num, -1) + if np.ndim(scale) > 0: + scale = scale.mean() + x_norm, x_mean, x_var, _, _ = _batch_norm(x, scale, shift=0, running_mean=save_mean, \ + running_var=save_inv_variance, \ + eps=eps, momentum=momentum, is_training=is_training) + batch_size = x.shape[0] + dx_norm = scale * dout + dvar = np.sum(dx_norm*(x - x_mean)*((x_var + eps)**(-3.0/2))*(-1.0/2), axis=0) + dmean = np.sum(dx_norm*(-1.0/np.sqrt(x_var + eps)), axis=0) \ + + dvar*(np.sum(-2*(x - x_mean), axis=0)*(1.0/batch_size)) + dx = dx_norm*(1.0/np.sqrt(x_var + eps)) + dvar*(2.0*(x - x_mean)/batch_size) + dmean*(1.0/batch_size) + dgamma = np.sum(dout*x_norm, axis=0) + dbeta = np.sum(dout, axis=0) + return dx, dgamma, dbeta + + +def batch_norm_grad(dy, x, scale, save_mean, save_inv_variance): + """Batch normalization over an array.""" + if dy.ndim != 2: + batch_size = dy.shape[0] + dy = dy.reshape(batch_size, -1) + + dx, dgamma, dbeta = _batch_norm_grad(dy, x, scale, save_mean, save_inv_variance) + input_shape = x.shape + dx = dx.reshape(*input_shape) + return dx, dgamma, dbeta + + +def col2im(col, input_shape, filter_h, filter_w, stride=1, pad=0): + """Rearranges a row vector to an image.""" + validator.check_integer("stride", stride, 0, Rel.GT) + batch_num, channel, height, width = input_shape + out_h = (height + 2*pad - filter_h)//stride + 1 + out_w = (width + 2*pad - filter_w)//stride + 1 + col = col.reshape(batch_num, out_h, out_w, channel, filter_h, filter_w) \ + .transpose(0, 3, 4, 5, 1, 2) + + img = np.zeros((batch_num, + channel, + height + 2*pad + stride - 1, + width + 2*pad + stride - 1)) \ + .astype(col.dtype) + for y in range(filter_h): + y_max = y + stride*out_h + for x in range(filter_w): + x_max = x + stride*out_w + img[:, :, y:y_max:stride, x:x_max:stride] += col[:, :, y, x, :, :] + + return img[:, :, pad:height + pad, pad:width + pad] + + +def convolve(x, w, b=None, pad_mode="valid"): + """ + Gets the discrete, linear convolution of two one-dimensional sequences. + + Args: + x (numpy.ndarray): One-dimensional input array. + w (numpy.ndarray): One-dimensional input array. + b (numpy.ndarray): One-dimensional input array. Default: None. + pad_mode (str): Padding mode which can be: "full" means returns the + convolution at each point of overlap, with an output shape + of (N+M-1,); "same" means returns output of length max(M, N); + Amd "valid" means returns output of length max(M, N) - min(M, N) + + 1. Default: "valid". + + Returns: + numpy.ndarray, discrete, linear convolution of x and w, then plus b. + """ + if pad_mode not in {"same", "valid"}: + pad_mode = "full" + y = np.convolve(x, w, pad_mode) + if b: + y += b + return y + + +def conv2d(x, weight, bias=None, stride=1, pad=0, + dilation=1, groups=1, padding_mode='zeros'): + """Convolution 2D.""" + # pylint: disable=unused-argument + validator.check_integer("stride", stride, 0, Rel.GT) + batch_num, _, x_h, x_w = x.shape + filter_num, _, filter_h, filter_w = weight.shape + out_h = 1 + int((x_h + 2 * pad - filter_h - (filter_h - 1) * (dilation - 1)) / stride) + out_w = 1 + int((x_w + 2 * pad - filter_w - (filter_w - 1) * (dilation - 1)) / stride) + col = im2col(x, filter_h, filter_w, stride, pad, dilation) + col_w = np.reshape(weight, (filter_num, -1)).T + out = np.dot(col, col_w) + out = out.reshape(batch_num, out_h, out_w, -1).transpose(0, 3, 1, 2) + if bias is not None: + out += bias + return out + + +def conv2d_backprop_filter(dout, x, w_size, stride=1, pad=0): + """Backpropagation filter for conv2d.""" + filter_num, channel, filter_height, filter_width = w_size + dout = dout.transpose(0, 2, 3, 1).reshape(-1, filter_num) + col = im2col(x, filter_height, filter_width, stride, pad) + dw = np.dot(col.T, dout) + dw = dw.transpose(1, 0).reshape(filter_num, channel, filter_height, filter_width) + return dw + + +def conv2d_backprop_input(dout, x_size, weight, stride=1, pad=0): + """Backpropagation input for conv2d.""" + filter_num, _, filter_h, filter_w = weight.shape + dout = dout.transpose(0, 2, 3, 1).reshape(-1, filter_num) + col_w = weight.reshape(filter_num, -1).T + dcol = np.dot(dout, col_w.T) + dx = col2im(dcol, x_size, filter_h, filter_w, stride, pad) + return dx + + +def flatten(x): + """ + Flattens an array to one dimension. + + Args: + x (numpy.ndarray): An array to be flattened. + + Returns: + numpy.ndarray, a flattened array in one dimension. + """ + return x.flatten() + + +def flatten2(x): + """ + Flattens an array to one dimension by reshape. + + Args: + x (numpy.ndarray): An array to be flattened. + + Returns: + numpy.ndarray, a flattened array in one dimension. + """ + return x.reshape(1, -1) + + +def flatten_batch(x): + """ + Flattens a batch of arrays to one dimension. + + Args: + x (numpy.ndarray): A batch of arrays to be flattened. + + Returns: + numpy.ndarray, a flattened one dimension array. + """ + return x.reshape(x.shape[0], -1) + + +def flatten_grad(dout, x): + """Grad of flatten.""" + dout = np.reshape(dout, x) + return dout + + +def im2col(img, filter_h, filter_w, stride=1, pad=0, dilation=1): + """Rearranges an image to row vector.""" + validator.check_integer("stride", stride, 0, Rel.GT) + batch_num, channel, height, width = img.shape + out_h = (height + 2*pad - filter_h- (filter_h - 1) * (dilation - 1))//stride + 1 + out_w = (width + 2*pad - filter_w- (filter_w - 1) * (dilation - 1))//stride + 1 + + img = np.pad(img, [(0, 0), (0, 0), (pad, pad), (pad, pad)], 'constant') + col = np.zeros((batch_num, channel, filter_h, filter_w, out_h, out_w)).astype(img.dtype) + + for y in range(filter_h): + y_max = y + stride*out_h + for x in range(filter_w): + x_max = x + stride*out_w + col[:, :, y, x, :, :] = img[:, :, y:y_max:stride, x:x_max:stride] + + col = col.transpose(0, 4, 5, 1, 2, 3).reshape(batch_num*out_h*out_w, -1) + return col + + +def matmul(x, w, b=None): + """ + Dot product of array x and w, then plus array b if b is not None. + + Args: + x (numpy.ndarray): Represents the input array. + w (numpy.ndarray): Represents weights array. + b (numpy.ndarray): Represents bias array which has the same shape as x. Default: None. + + Returns: + numpy.ndarray, the result of (x*w + b). + """ + y = np.dot(x, w) + if b: + y += b + return y + + +def max_pooling(x, pool_h, pool_w, stride, pad): + """Max pooling.""" + validator.check_integer("stride", stride, 0, Rel.GT) + num, channel, height, width = x.shape + out_h = (height + 2*pad - pool_h)//stride + 1 + out_w = (width + 2*pad - pool_w)//stride + 1 + + col = im2col(x, pool_h, pool_w, stride, pad) + col = col.reshape(-1, pool_h*pool_w) + + out = np.max(col, axis=1) + out = out.reshape(num, out_h, out_w, channel).transpose(0, 3, 1, 2) + + return out + + +def max_pool_grad(x, dout, pool_h, pool_w, stride, pad): + """Grad of max pooling.""" + dout = dout.transpose(0, 2, 3, 1) + pool_size = pool_h * pool_w + dmax = np.zeros((dout.size, pool_size)) + col = im2col(x, pool_h, pool_w, stride, pad) + col = col.reshape(-1, pool_h*pool_w) + arg_max = np.argmax(col, axis=1) + dmax[np.arange(arg_max.size), arg_max.flatten()] = dout.flatten() + dmax = dmax.reshape(dout.shape + (pool_size,)) + dcol = dmax.reshape(dmax.shape[0]*dmax.shape[1]*dmax.shape[2], -1) + dx = col2im(dcol, x.shape, pool_h, pool_w, stride, pad) + return dx + + +def max_pool_grad_with_argmax(x, arg_max, dout, pool_h, pool_w, stride, pad): + """Grad of max pooling with argmax.""" + dout = dout.transpose(0, 2, 3, 1) + pool_size = pool_h * pool_w + dmax = np.zeros((dout.size, pool_size)) + dmax[np.arange(arg_max.size), arg_max.flatten()] = dout.flatten() + dmax = dmax.reshape(dout.shape + (pool_size,)) + dcol = dmax.reshape(dmax.shape[0]*dmax.shape[1]*dmax.shape[2], -1) + dx = col2im(dcol, x.shape, pool_h, pool_w, stride, pad) + return dx + + +def max_pool_with_argmax(x, pool_h, pool_w, stride, pad): + """Max pooling with argmax.""" + validator.check_integer("stride", stride, 0, Rel.GT) + num, channel, height, width = x.shape + out_h = (height + 2*pad - pool_h)//stride + 1 + out_w = (width + 2*pad - pool_w)//stride + 1 + col = im2col(x, pool_h, pool_w, stride, pad) + col = col.reshape(-1, pool_h*pool_w) + out = np.max(col, axis=1) + out_argmax = np.argmax(col, axis=1) + out = out.reshape(num, out_h, out_w, channel).transpose(0, 3, 1, 2) + out_argmax = out_argmax.reshape(num, out_h, out_w, channel).transpose(0, 3, 1, 2) + return out, out_argmax + + +def relu(x): + """ + Rectified linear unit. + + Args: + x (numpy.ndarray): The input array. + + Returns: + numpy.ndarray, the array applied relu. + """ + return x * (x > 0) + + +def relu_grad(y): + """ + Grad of relu. + + Args: + y (numpy.ndarray): The input array. + + Returns: + numpy.ndarray, the array applied grad of relu. + """ + y[y <= 0] = 0 + y[y > 0] = 1 + return y + + +def sigmoid(x): + """ + Sigmoid activation function. + + Args: + x (numpy.ndarray): The input array. + + Returns: + numpy.ndarray, the array applied sigmoid. + """ + return 1 / (1 + np.exp(x * -1)) + + +def tanh(x): + """ + Computes hyperbolic tangent element-wise. + + Args: + x (numpy.ndarray): The input array. + + Returns: + numpy.ndarray, the array applied tanh. + """ + a = np.exp(x) - np.exp(x * -1) + b = np.exp(x) + np.exp(x * -1) + return a / b + + +def softmax(x, axis=None): + """ + Softmax function which is `softmax(x) = np.exp(x)/sum(np.exp(x))`. + + Args: + x (numpy.ndarray): Input array. + axis (Union[int, tuple[int]]): Axis to compute values along. Default: None. + + Returns: + numpy.ndarray, has the same shape as x. + """ + from scipy.special import softmax as scipy_softmax + return scipy_softmax(x, axis) + + +def softmax_cross_entropy_with_logits(logits, labels): + sample_num = labels.shape[0] + prob = softmax(logits) + log_likelihood = -np.log(prob[range(sample_num)]) * labels + #loss = np.sum(log_likelihood) + loss = log_likelihood + + dx = prob.copy() + dx[range(sample_num)] -= labels + return loss, dx + + +def shape(x): + """ + Gets the array's dimensions. + + Args: + x (numpy.ndarray): Input array. + + Returns: + tuple, the shape/dimensions of the input array. + """ + return np.array(np.shape(x)) + + +def expand_dims(x, axis): + """ + Expands the shape of an array. + + Args: + x (numpy.ndarray): Input array. + axis (int): Position in the expanded axes where the new axis is placed. + + Returns: + numpy.ndarray, view of input array with the number of dimensions increased by one. + """ + return np.expand_dims(x, axis) + + +def squeeze(x, axis): + """ + Removes single-dimensional entries from the shape of an array. + + Args: + x (numpy.ndarray): Input array. + axis (Union[int, tuple[int]]): Selected subset of the single-dimensional entries in the shape. + + Returns: + numpy.ndarray, the input numpy.ndarray, but with all or a subset of the dimensions of length + 1 removed. + """ + return np.squeeze(x, tuple(axis)) + + +def reshape(x, shp): + """ + Applies a new shape to an array without changing its data. + + Args: + x (numpy.ndarray): Input array. + shp (tuple[int]): New shape to apply to x. + + Returns: + numpy.ndarray, a new view object or a copy of input array. + """ + return np.reshape(x, tuple(shp)) + + +def rank(x): + """ + Gets number of array dimensions. + + Args: + x (numpy.ndarray): Input array. + + Returns: + int, number of input array dimensions. + """ + return np.array(np.ndim(x)) + + +def logsoftmax(x): + """ + Log softmax function. + + Args: + x (numpy.ndarray): Input array. + + Returns: + numpy.ndarray, the result of applying log softmax on the input array. + """ + return np.array(np.log(softmax(x))) + + +def transpose(x, axes=None): + """ + Transposes an input array according to axes. + + Args: + x (numpy.ndarray): Input array. + axes (list): The axes to be transposed. Default: None. + + Returns: + numpy.ndarray, transposed array. + """ + return np.transpose(x, axes) + + +def invert_permutation(x): + """ + Gets the inverse permutation of an array. + + Args: + x (numpy.ndarray): Input array. + + Returns: + tuple, the inverse permutation of the input array. + """ + x = np.array(x) + y = np.argsort(x) + return tuple(y) + + +def select(cond, x, y): + """ + Gets elements from x or y depending on cond. + + Args: + cond (bool): Where True, yield x, otherwise yield y. + x (numpy.ndarray): Values from which to choose. + y (numpy.ndarray): Values from which to choose. + + Returns: + numpy.ndarray, elements from x where condition is True, and elements from y elsewhere. + """ + return np.where(cond, x, y) + + +def sum_by_axis(x, axis): + """ + Sum of array elements over a given axis. + + Args: + x (numpy.ndarray): Input array. + axis (Union[int, tuple[int]]): Axis or axes along which a sum is performed. + + Returns: + numpy.ndarray, has the same shape as input array with the specified axis removed. + """ + return np.sum(x, axis) + + +def equal(x, y): + """ + Gets (x == y) element-wise. + + Args: + x (numpy.ndarray): Input array. + y (numpy.ndarray): Input array. + + Returns: + numpy.ndarray, element-wise comparison of x and y. + """ + return np.equal(x, y) + + +def not_equal(x, y): + """ + Gets (x != y) element-wise. + + Args: + x (numpy.ndarray): Input array. + y (numpy.ndarray): Input array. + + Returns: + numpy.ndarray, element-wise comparison of x and y. + """ + return np.not_equal(x, y) + + +def greater(x, y): + """ + Get the truth value of (x > y) element-wise. + + Args: + x (numpy.ndarray): Input array. + y (numpy.ndarray): Input array. + + Returns: + numpy.ndarray, element-wise comparison of x and y. + """ + return np.greater(x, y) + +def less(x, y): + """ + Get the truth value of (x < y) element-wise. + + Args: + x (numpy.ndarray): Input array. + y (numpy.ndarray): Input array. + + Returns: + Array, element-wise comparison of x and y. + """ + return np.less(x, y) + + + + +def logical_not(x): + """ + Gets the truth value of NOT x element-wise. + + Args: + x (numpy.ndarray): Input array. + + Returns: + bool, have the same shape as x of the NOT operation on elements of x. + """ + return np.logical_not(x) + + +def sqrt(x): + """ + Gets the non-negative square-root of an numpy.ndarray, element-wise. + + Args: + x (numpy.ndarray): Input array. + + Returns: + numpy.ndarray, has the same shape as x, containing the positive square-root of each + element in x. + """ + return np.sqrt(x) + + +def power(x, y): + """ + First array elements raised to powers from second numpy.ndarray, element-wise. + + Args: + x (numpy.ndarray): The bases array. + y (numpy.ndarray): The exponents array. + + Returns: + numpy.ndarray, the bases in x raised to the exponents in y. + """ + return np.power(x, y) + + +def exp(x): + """ + Gets the exponential of all elements in the input array. + + Args: + x (numpy.ndarray): Input array. + + Returns: + numpy.ndarray, element-wise exponential of x. + """ + return np.exp(x) + + +def maximum(x, y): + """ + Gets the max of x and y element-wise. + + If x > y, return x. Otherwise, return y. + + Args: + x (numpy.ndarray): First input array. + y (numpy.ndarray): Second input array ave the same type as x. + + Returns: + numpy.ndarray, has the same type as x. + """ + return np.maximum(x, y) + + +def minimum(x, y): + """ + Gets the min of x and y element-wise. + + If x < y, return x. Otherwise, return y. + + Args: + x (numpy.ndarray): First input array. + y (numpy.ndarray): Second input array have the same type as x. + + Returns: + numpy.ndarray, has the same type as x. + """ + return np.minimum(x, y) diff --git a/third_party/apply_patches.sh b/third_party/apply_patches.sh new file mode 100755 index 00000000000..fbd06b68b67 --- /dev/null +++ b/third_party/apply_patches.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +PWD_PATH=`pwd` +THIRD_PARTY_PATH=$(cd "$(dirname $0)"; pwd) +if [ $# -lt 1 ]; then + echo "Usage: sh apply_patches.sh [build_dir]" + echo " build_dir is the directory where you type \"cmake\"" + echo " Open source software incubator-tvm will be copied to build_dir" + echo " where patches will be applied on." + exit 1 +fi +BUILD_PATH=$1 + +if [ -d ${BUILD_PATH}/incubator-tvm ]; then + rm -rf ${BUILD_PATH}/incubator-tvm +fi +DLPACK_PATH=$2 +DMLC_PATH=$3 +RANG_PATH=$4 +TVM_PATH=$5 +mkdir ${BUILD_PATH}/incubator-tvm +cp -rf ${TVM_PATH}/* ${BUILD_PATH}/incubator-tvm/ +cp -rf ${DLPACK_PATH}/* ${BUILD_PATH}/incubator-tvm/3rdparty/dlpack/ +cp -rf ${DMLC_PATH}/* ${BUILD_PATH}/incubator-tvm/3rdparty/dmlc-core/ +cp -rf ${RANG_PATH}/* ${BUILD_PATH}/incubator-tvm/3rdparty/rang/ + +check_dir_not_empty() +{ + if [ ! $# -eq 1 ]; then + echo "Usage: check_dir_not_empty dir_path" + exit 1 + fi + + if [ ! -d $1 ]; then + echo "Directory $1 does not exist." + exit 1 + fi + + fileCounts=`ls $1 | wc -l` + if [ ${fileCounts} -eq 0 ]; then + echo "Directory $1 is empty." + exit 1 + fi +} + +apply_patch() +{ + if [ ! $# -eq 1 ]; then + echo "Usage: apply_patch patch_name" + exit 1 + fi + + if [ ! -f $1 ]; then + echo "Patch $1 does not exist." + exit 1 + fi + + patch -p1 < $1 + if [ $? -eq 0 ]; then + echo "Patch $1 applied successfully." + else + echo "Patch $1 not applied." + fi +} + +# apply patches on tvm +TVM_PATH=${BUILD_PATH}/incubator-tvm +TVM_PATCH_PATH=${THIRD_PARTY_PATH}/patch/incubator-tvm +check_dir_not_empty "${TVM_PATH}" +check_dir_not_empty "${TVM_PATCH_PATH}" +cd ${TVM_PATH} +apply_patch "${TVM_PATCH_PATH}/cmake.patch" +apply_patch "${TVM_PATCH_PATH}/find_library.patch" +apply_patch "${TVM_PATCH_PATH}/include.patch" +apply_patch "${TVM_PATCH_PATH}/src_pass.patch" + +cd ${PWD_PATH} diff --git a/third_party/flatbuffers b/third_party/flatbuffers new file mode 160000 index 00000000000..9e7e8cbe9f6 --- /dev/null +++ b/third_party/flatbuffers @@ -0,0 +1 @@ +Subproject commit 9e7e8cbe9f675123dd41b7c62868acad39188cae diff --git a/third_party/googletest b/third_party/googletest new file mode 160000 index 00000000000..2fe3bd994b3 --- /dev/null +++ b/third_party/googletest @@ -0,0 +1 @@ +Subproject commit 2fe3bd994b3189899d93f1d5a881e725e046fdc2 diff --git a/third_party/incubator-tvm b/third_party/incubator-tvm new file mode 160000 index 00000000000..c6f8c23c349 --- /dev/null +++ b/third_party/incubator-tvm @@ -0,0 +1 @@ +Subproject commit c6f8c23c349f3ef8bacceaf3203f7cc08e6529de diff --git a/third_party/patch/incubator-tvm/cmake.patch b/third_party/patch/incubator-tvm/cmake.patch new file mode 100644 index 00000000000..820c7e24fd8 --- /dev/null +++ b/third_party/patch/incubator-tvm/cmake.patch @@ -0,0 +1,201 @@ +diff -Npur tvm/cmake/modules/ANTLR.cmake tvm_new/cmake/modules/ANTLR.cmake +--- tvm/cmake/modules/ANTLR.cmake 2019-12-14 15:11:37.562418441 +0800 ++++ tvm_new/cmake/modules/ANTLR.cmake 2019-12-14 11:28:49.161977599 +0800 +@@ -14,12 +14,15 @@ + # KIND, either express or implied. See the License for the + # specific language governing permissions and limitations + # under the License. ++ ++# 2019.12.30 - Modify current directory of tvm. ++ + if(USE_ANTLR) + find_antlr(${USE_ANTLR}) + if(ANTLR4) + + set(RELAY_PARSER_DIR +- ${CMAKE_CURRENT_SOURCE_DIR}/python/tvm/relay/grammar) ++ ${TVM_DIR}/python/tvm/relay/grammar) + + set(RELAY_PARSER + ${RELAY_PARSER_DIR}/py3/RelayVisitor.py +diff -Npur tvm/cmake/modules/CUDA.cmake tvm_new/cmake/modules/CUDA.cmake +--- tvm/cmake/modules/CUDA.cmake 2019-12-14 15:11:37.562418441 +0800 ++++ tvm_new/cmake/modules/CUDA.cmake 2019-12-14 11:28:49.161977599 +0800 +@@ -15,6 +15,8 @@ + # specific language governing permissions and limitations + # under the License. + ++# 2019.12.30 - Modify current directory of tvm. ++ + # CUDA Module + find_cuda(${USE_CUDA}) + +@@ -29,9 +31,9 @@ if(USE_CUDA) + message(FATAL_ERROR "Cannot find CUDA, USE_CUDA=" ${USE_CUDA}) + endif() + message(STATUS "Build with CUDA support") +- file(GLOB RUNTIME_CUDA_SRCS src/runtime/cuda/*.cc) ++ file(GLOB RUNTIME_CUDA_SRCS ${TVM_DIR}/src/runtime/cuda/*.cc) + list(APPEND RUNTIME_SRCS ${RUNTIME_CUDA_SRCS}) +- list(APPEND COMPILER_SRCS src/codegen/opt/build_cuda_on.cc) ++ list(APPEND COMPILER_SRCS ${TVM_DIR}/src/codegen/opt/build_cuda_on.cc) + + list(APPEND TVM_LINKER_LIBS ${CUDA_NVRTC_LIBRARY}) + list(APPEND TVM_RUNTIME_LINKER_LIBS ${CUDA_CUDART_LIBRARY}) +@@ -40,18 +42,18 @@ if(USE_CUDA) + + if(USE_CUDNN) + message(STATUS "Build with cuDNN support") +- file(GLOB CONTRIB_CUDNN_SRCS src/runtime/contrib/cudnn/*.cc) ++ file(GLOB CONTRIB_CUDNN_SRCS ${TVM_DIR}/src/runtime/contrib/cudnn/*.cc) + list(APPEND RUNTIME_SRCS ${CONTRIB_CUDNN_SRCS}) + list(APPEND TVM_RUNTIME_LINKER_LIBS ${CUDA_CUDNN_LIBRARY}) + endif(USE_CUDNN) + + if(USE_CUBLAS) + message(STATUS "Build with cuBLAS support") +- file(GLOB CONTRIB_CUBLAS_SRCS src/runtime/contrib/cublas/*.cc) ++ file(GLOB CONTRIB_CUBLAS_SRCS ${TVM_DIR}/src/runtime/contrib/cublas/*.cc) + list(APPEND RUNTIME_SRCS ${CONTRIB_CUBLAS_SRCS}) + list(APPEND TVM_RUNTIME_LINKER_LIBS ${CUDA_CUBLAS_LIBRARY}) + endif(USE_CUBLAS) + + else(USE_CUDA) +- list(APPEND COMPILER_SRCS src/codegen/opt/build_cuda_off.cc) ++ list(APPEND COMPILER_SRCS ${TVM_DIR}/src/codegen/opt/build_cuda_off.cc) + endif(USE_CUDA) +diff -Npur tvm/cmake/modules/LLVM.cmake tvm_new/cmake/modules/LLVM.cmake +--- tvm/cmake/modules/LLVM.cmake 2019-12-14 15:11:37.562418441 +0800 ++++ tvm_new/cmake/modules/LLVM.cmake 2019-12-14 11:28:49.161977599 +0800 +@@ -15,6 +15,8 @@ + # specific language governing permissions and limitations + # under the License. + ++# 2019.12.30 - Modify current directory of tvm. ++ + # LLVM rules + add_definitions(-DDMLC_USE_FOPEN64=0) + +@@ -26,7 +28,7 @@ if(NOT USE_LLVM STREQUAL "OFF") + message(STATUS "Set TVM_LLVM_VERSION=" ${TVM_LLVM_VERSION}) + # Set flags that are only needed for LLVM target + add_definitions(-DTVM_LLVM_VERSION=${TVM_LLVM_VERSION}) +- file(GLOB COMPILER_LLVM_SRCS src/codegen/llvm/*.cc) ++ file(GLOB COMPILER_LLVM_SRCS ${TVM_DIR}/src/codegen/llvm/*.cc) + list(APPEND TVM_LINKER_LIBS ${LLVM_LIBS}) + list(APPEND COMPILER_SRCS ${COMPILER_LLVM_SRCS}) + if(NOT MSVC) +diff -Npur tvm/cmake/modules/Micro.cmake tvm_new/cmake/modules/Micro.cmake +--- tvm/cmake/modules/Micro.cmake 2019-12-14 15:11:37.562418441 +0800 ++++ tvm_new/cmake/modules/Micro.cmake 2019-12-14 11:28:49.161977599 +0800 +@@ -15,8 +15,10 @@ + # specific language governing permissions and limitations + # under the License. + ++# 2019.12.30 - Modify current directory of tvm. ++ + if(USE_MICRO) + message(STATUS "Build with Micro support") +- file(GLOB RUNTIME_MICRO_SRCS src/runtime/micro/*.cc) ++ file(GLOB RUNTIME_MICRO_SRCS ${TVM_DIR}/src/runtime/micro/*.cc) + list(APPEND RUNTIME_SRCS ${RUNTIME_MICRO_SRCS}) + endif(USE_MICRO) +diff -Npur tvm/cmake/modules/VTA.cmake tvm_new/cmake/modules/VTA.cmake +--- tvm/cmake/modules/VTA.cmake 2019-12-14 15:11:37.562418441 +0800 ++++ tvm_new/cmake/modules/VTA.cmake 2019-12-14 14:42:32.358381133 +0800 +@@ -15,17 +15,19 @@ + # specific language governing permissions and limitations + # under the License. + ++# 2019.12.30 - Modify current directory of tvm. ++ + # CMake Build rules for VTA + find_program(PYTHON NAMES python python3 python3.6) + + if(MSVC) + message(STATUS "VTA build is skipped in Windows..") + elseif(PYTHON) +- set(VTA_CONFIG ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/vta/config/vta_config.py) ++ set(VTA_CONFIG ${PYTHON} ${TVM_DIR}/vta/config/vta_config.py) + + if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/vta_config.json) + message(STATUS "Use VTA config " ${CMAKE_CURRENT_BINARY_DIR}/vta_config.json) +- set(VTA_CONFIG ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/vta/config/vta_config.py ++ set(VTA_CONFIG ${PYTHON} ${TVM_DIR}/vta/config/vta_config.py + --use-cfg=${CMAKE_CURRENT_BINARY_DIR}/vta_config.json) + endif() + +@@ -40,18 +42,18 @@ elseif(PYTHON) + # Fast simulator driver build + if(USE_VTA_FSIM) + # Add fsim driver sources +- file(GLOB FSIM_RUNTIME_SRCS vta/src/*.cc) +- list(APPEND FSIM_RUNTIME_SRCS vta/src/sim/sim_driver.cc) +- list(APPEND FSIM_RUNTIME_SRCS vta/src/vmem/virtual_memory.cc vta/src/vmem/virtual_memory.h) +- list(APPEND FSIM_RUNTIME_SRCS vta/src/sim/sim_tlpp.cc) ++ file(GLOB FSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/*.cc) ++ list(APPEND FSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/sim/sim_driver.cc) ++ list(APPEND FSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/vmem/virtual_memory.cc ${TVM_DIR}/vta/src/vmem/virtual_memory.h) ++ list(APPEND FSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/sim/sim_tlpp.cc) + # Target lib: vta_fsim + add_library(vta_fsim SHARED ${FSIM_RUNTIME_SRCS}) +- target_include_directories(vta_fsim PUBLIC vta/include) ++ target_include_directories(vta_fsim PUBLIC ${TVM_DIR}/vta/include) + foreach(__def ${VTA_DEFINITIONS}) + string(SUBSTRING ${__def} 3 -1 __strip_def) + target_compile_definitions(vta_fsim PUBLIC ${__strip_def}) + endforeach() +- include_directories("vta/include") ++ include_directories("${TVM_DIR}/vta/include") + if(APPLE) + set_target_properties(vta_fsim PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + endif(APPLE) +@@ -61,18 +63,18 @@ elseif(PYTHON) + # Cycle accurate simulator driver build + if(USE_VTA_TSIM) + # Add tsim driver sources +- file(GLOB TSIM_RUNTIME_SRCS vta/src/*.cc) +- list(APPEND TSIM_RUNTIME_SRCS vta/src/tsim/tsim_driver.cc) +- list(APPEND TSIM_RUNTIME_SRCS vta/src/dpi/module.cc) +- list(APPEND TSIM_RUNTIME_SRCS vta/src/vmem/virtual_memory.cc vta/src/vmem/virtual_memory.h) ++ file(GLOB TSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/*.cc) ++ list(APPEND TSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/tsim/tsim_driver.cc) ++ list(APPEND TSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/dpi/module.cc) ++ list(APPEND TSIM_RUNTIME_SRCS ${TVM_DIR}/vta/src/vmem/virtual_memory.cc ${TVM_DIR}/vta/src/vmem/virtual_memory.h) + # Target lib: vta_tsim + add_library(vta_tsim SHARED ${TSIM_RUNTIME_SRCS}) +- target_include_directories(vta_tsim PUBLIC vta/include) ++ target_include_directories(vta_tsim PUBLIC ${TVM_DIR}/vta/include) + foreach(__def ${VTA_DEFINITIONS}) + string(SUBSTRING ${__def} 3 -1 __strip_def) + target_compile_definitions(vta_tsim PUBLIC ${__strip_def}) + endforeach() +- include_directories("vta/include") ++ include_directories("${TVM_DIR}/vta/include") + if(APPLE) + set_target_properties(vta_tsim PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + endif(APPLE) +@@ -80,19 +82,19 @@ elseif(PYTHON) + + # VTA FPGA driver sources + if(USE_VTA_FPGA) +- file(GLOB FPGA_RUNTIME_SRCS vta/src/*.cc) ++ file(GLOB FPGA_RUNTIME_SRCS ${TVM_DIR}/vta/src/*.cc) + # Rules for Zynq-class FPGAs with pynq OS support (see pynq.io) + if(${VTA_TARGET} STREQUAL "pynq" OR + ${VTA_TARGET} STREQUAL "ultra96") +- list(APPEND FPGA_RUNTIME_SRCS vta/src/pynq/pynq_driver.cc) ++ list(APPEND FPGA_RUNTIME_SRCS ${TVM_DIR}/vta/src/pynq/pynq_driver.cc) + # Rules for Pynq v2.4 + find_library(__cma_lib NAMES cma PATH /usr/lib) + elseif(${VTA_TARGET} STREQUAL "de10nano") # DE10-Nano rules +- file(GLOB FPGA_RUNTIME_SRCS vta/src/de10nano/*.cc vta/src/*.cc) ++ file(GLOB FPGA_RUNTIME_SRCS ${TVM_DIR}/vta/src/de10nano/*.cc ${TVM_DIR}/vta/src/*.cc) + endif() + # Target lib: vta + add_library(vta SHARED ${FPGA_RUNTIME_SRCS}) +- target_include_directories(vta PUBLIC vta/include) ++ target_include_directories(vta PUBLIC ${TVM_DIR}/vta/include) + foreach(__def ${VTA_DEFINITIONS}) + string(SUBSTRING ${__def} 3 -1 __strip_def) + target_compile_definitions(vta PUBLIC ${__strip_def}) diff --git a/third_party/patch/incubator-tvm/find_library.patch b/third_party/patch/incubator-tvm/find_library.patch new file mode 100644 index 00000000000..e54df2c7cfc --- /dev/null +++ b/third_party/patch/incubator-tvm/find_library.patch @@ -0,0 +1,71 @@ +--- tvm/python/tvm/_ffi/base.py 2020-03-12 16:17:39.089828527 +0800 ++++ tvm_new/python/tvm/_ffi/base.py 2020-03-12 16:17:16.829829558 +0800 +@@ -16,6 +16,9 @@ + # under the License. + # coding: utf-8 + # pylint: disable=invalid-name ++ ++# 2019.12.30 - Modify _load_lib function. ++ + """Base library for TVM FFI.""" + from __future__ import absolute_import + +@@ -47,8 +50,18 @@ else: + + + def _load_lib(): +- """Load libary by searching possible path.""" +- lib_path = libinfo.find_lib_path() ++ """Load library by searching possible path.""" ++ pwd = os.path.dirname(os.path.realpath(__file__)) ++ path = os.path.realpath(pwd+"/../../../mindspore") ++ lib_path = [] ++ files = os.listdir(path) ++ for f in files: ++ if f.startswith("_c_expression.") and f.endswith(".so"): ++ lib_path.append(path+"/"+f) ++ break ++ if not lib_path: ++ raise RuntimeError("mindspore library cannot find.") ++ + lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_GLOBAL) + # DMatrix functions + lib.TVMGetLastError.restype = ctypes.c_char_p +diff -Npur tvm/topi/python/topi/cpp/impl.py tvm_new/topi/python/topi/cpp/impl.py +--- tvm/topi/python/topi/cpp/impl.py 2020-03-12 16:17:39.129828525 +0800 ++++ tvm_new/topi/python/topi/cpp/impl.py 2020-03-12 16:17:16.873829556 +0800 +@@ -14,6 +14,9 @@ + # KIND, either express or implied. See the License for the + # specific language governing permissions and limitations + # under the License. ++ ++# 2019.12.30 - Modify _load_lib function. ++ + """Load Lib for C++ TOPI ops and schedules""" + import sys + import os +@@ -30,12 +33,18 @@ def _get_lib_names(): + return ['libtvm_topi.so', 'tvm_topi.so'] + + def _load_lib(): +- """Load libary by searching possible path.""" +- curr_path = os.path.dirname(os.path.realpath(os.path.expanduser(__file__))) +- lib_search = curr_path +- lib_path = libinfo.find_lib_path(_get_lib_names(), lib_search, optional=True) +- if lib_path is None: +- return None, None ++ """Load library by searching possible path.""" ++ pwd = os.path.dirname(os.path.realpath(__file__)) ++ path = os.path.realpath(pwd+"/../../../mindspore") ++ lib_path = [] ++ files = os.listdir(path) ++ for f in files: ++ if f.startswith("_c_expression.") and f.endswith(".so"): ++ lib_path.append(path+"/"+f) ++ break ++ if not lib_path: ++ raise RuntimeError("mindspore library cannot find.") ++ + lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_GLOBAL) + return lib, os.path.basename(lib_path[0]) + diff --git a/third_party/patch/incubator-tvm/include.patch b/third_party/patch/incubator-tvm/include.patch new file mode 100644 index 00000000000..270c7a0f399 --- /dev/null +++ b/third_party/patch/incubator-tvm/include.patch @@ -0,0 +1,55 @@ +diff -Npur tvm/include/tvm/expr_operator.h tvm_new/include/tvm/expr_operator.h +--- tvm/include/tvm/expr_operator.h 2019-12-28 10:11:27.369814744 +0800 ++++ tvm_new/include/tvm/expr_operator.h 2019-12-28 10:11:27.209812391 +0800 +@@ -25,6 +25,11 @@ + * when the type is int32 or int64 for simplifying the index expressions. + */ + // Acknowledgement: Most operator APIs originate from Halide. ++ ++/* ++ * 2019.12.30 - Add new operator for expr. ++ */ ++ + #ifndef TVM_EXPR_OPERATOR_H_ + #define TVM_EXPR_OPERATOR_H_ + +@@ -217,6 +222,16 @@ TVM_DLL Expr operator*(Expr a, Expr b); + */ + TVM_DLL Expr operator/(Expr a, Expr b); + /*! ++ * \brief mod operator ++ * ++ * \param a left operand ++ * \param b right operand ++ * \return The result expression. ++ * \note this function does eager constant folding for ++ * index types(int32, int64) when possible. ++ */ ++TVM_DLL Expr operator%(Expr a, Expr b); ++/*! + * \brief left shift operator + * + * \param a left operand +diff -Npur tvm/include/tvm/lowered_func.h tvm_new/include/tvm/lowered_func.h +--- tvm/include/tvm/lowered_func.h 2019-12-28 10:11:27.369814744 +0800 ++++ tvm_new/include/tvm/lowered_func.h 2019-12-28 10:11:27.209812391 +0800 +@@ -22,6 +22,11 @@ + * \brief Information about a lowered TVM function. + * This data structure is final step toward codegen. + */ ++ ++/* ++ * 2019.12.30 - Add new var array for args_real. ++ */ ++ + #ifndef TVM_LOWERED_FUNC_H_ + #define TVM_LOWERED_FUNC_H_ + +@@ -74,6 +79,7 @@ class LoweredFuncNode : public ir::Funct + * This function can only take pod type(int, float) and void* as arguments. + */ + Array args; ++ Array args_real; + /*! + * \brief The IterVar axis of threads + * Each axis need host function to specify a size. diff --git a/third_party/patch/incubator-tvm/src_pass.patch b/third_party/patch/incubator-tvm/src_pass.patch new file mode 100644 index 00000000000..5450ca142ec --- /dev/null +++ b/third_party/patch/incubator-tvm/src_pass.patch @@ -0,0 +1,120 @@ +diff -Npur tvm/src/pass/make_api.cc tvm_new/src/pass/make_api.cc +--- tvm/src/pass/make_api.cc 2019-12-14 15:11:37.626419432 +0800 ++++ tvm_new/src/pass/make_api.cc 2019-12-14 14:58:46.562493287 +0800 +@@ -20,6 +20,11 @@ + /*! + * \file make_api.cc Build API function. + */ ++ ++/* ++ * 2019.12.30 - Define new function to push buffer node from api_args to args_real. ++ */ ++ + #include + #include + #include +@@ -40,6 +45,17 @@ inline Stmt MakeAssertEQ(Expr lhs, Expr + return AssertStmt::make(lhs == rhs, msg, Evaluate::make(0)); + } + ++Array Param ( Array api_args,Array args_real) { ++ int num_args = static_cast(api_args.size()); ++ for (int i = 0; i < num_args; i++) { ++ const BufferNode *v = api_args[i].as(); ++ if(v) { ++ args_real.push_back(v->data); ++ } ++ } ++ return args_real; ++} ++ + LoweredFunc MakeAPI(Stmt body, + std::string name, + Array api_args, +@@ -47,6 +63,8 @@ LoweredFunc MakeAPI(Stmt body, + bool is_restricted) { + const Stmt nop = Evaluate::make(0); + int num_args = static_cast(api_args.size()); ++ Array args_real; ++ args_real = Param (api_args, args_real); + CHECK_LE(num_unpacked_args, num_args); + int num_packed_args = num_args - num_unpacked_args; + // Data field definitions +@@ -170,6 +188,7 @@ LoweredFunc MakeAPI(Stmt body, + NodePtr n = make_node(); + n->name = name; + n->args = args; ++ n->args_real = args_real; + n->handle_data_type = binder.def_handle_dtype(); + n->is_packed_func = num_unpacked_args == 0; + n->is_restricted = is_restricted; +diff -Npur tvm/src/pass/split_host_device.cc tvm_new/src/pass/split_host_device.cc +--- tvm/src/pass/split_host_device.cc 2019-12-14 15:11:37.626419432 +0800 ++++ tvm_new/src/pass/split_host_device.cc 2019-12-14 11:28:49.293979656 +0800 +@@ -21,6 +21,11 @@ + * \file split_host_device.cc + * \brief Split device function from host. + */ ++ ++/* ++ * 2019.12.30 - Add new implements for host device splitter. ++ */ ++ + #include + #include + #include +@@ -38,6 +43,7 @@ class IRUseDefAnalysis : public IRMutato + Stmt Mutate_(const AttrStmt *op, const Stmt& s) final { + if (op->attr_key == attr::thread_extent) { + IterVar iv = Downcast(op->node); ++ iv = IterVarNode::make(Range(0, op->value), iv->var, iv->iter_type, iv->thread_tag); + CHECK_NE(iv->thread_tag.length(), 0U); + // thread_extent can appear multiple times + // use the first appearance as def. +@@ -186,6 +192,7 @@ class HostDeviceSplitter : public IRMuta + name_ = f->name; + NodePtr n = + make_node(*f.operator->()); ++ args_real = n->args_real; + n->body = this->Mutate(f->body); + n->func_type = kHostFunc; + Array ret{LoweredFunc(n)}; +@@ -196,6 +203,7 @@ class HostDeviceSplitter : public IRMuta + } + + private: ++ Array args_real; + Stmt SplitDeviceFunc(Stmt body) { + std::ostringstream os; + os << name_ << "_kernel" << device_funcs_.size(); +@@ -223,6 +231,30 @@ class HostDeviceSplitter : public IRMuta + n->args.push_back(v); + } + } ++std::shared_ptr na = std::make_shared(); ++ for (unsigned i = 0; i < (unsigned)args_real.size(); i++) { ++ bool match = false; ++ for (unsigned j = 0; j < (unsigned)n->args.size(); j++) { ++ if (strcmp(args_real[i].get()->name_hint.c_str(), n->args[j].get()->name_hint.c_str()) == 0) { ++ na->args.push_back(n->args[j]); ++ match = true; ++ break; ++ } else { ++ continue; ++ } ++ } ++ ++ if (!match) { ++ na->args.push_back(args_real[i]); ++ // mark handle data type. ++ for (auto kv : handle_data_type_) { ++ if (strcmp(args_real[i].get()->name_hint.c_str(), kv.first->name_hint.c_str()) == 0) { ++ n->handle_data_type.Set(args_real[i], kv.second); ++ } ++ } ++ } ++ } ++ n->args = na->args; + LoweredFunc f_device(n); + Array call_args; + call_args.push_back(StringImm::make(f_device->name)); diff --git a/third_party/patch/predict/0001-RetBugFix-CustomRuntime_v06.patch b/third_party/patch/predict/0001-RetBugFix-CustomRuntime_v06.patch new file mode 100644 index 00000000000..5977c943ef9 --- /dev/null +++ b/third_party/patch/predict/0001-RetBugFix-CustomRuntime_v06.patch @@ -0,0 +1,203 @@ +diff --git a/include/tvm/runtime/registry.h b/include/tvm/runtime/registry.h +index d668984..3676a61 100644 +--- a/include/tvm/runtime/registry.h ++++ b/include/tvm/runtime/registry.h +@@ -319,6 +319,19 @@ class Registry { + #define TVM_REGISTER_EXT_TYPE(T) \ + TVM_STR_CONCAT(TVM_TYPE_REG_VAR_DEF, __COUNTER__) = \ + ::tvm::runtime::ExtTypeVTable::Register_() ++/* ++ * Macro transfer TVM runtime API to custom runtime API ++ */ ++#define TVM_RT_FUNC_TRANS(OrigFuncStr) ({ \ ++ const runtime::PackedFunc* trans_func = runtime::Registry::Get("codegen.GetTransRTFunc");\ ++ const char* dst_func_str = nullptr; \ ++ if( trans_func != nullptr){ \ ++ dst_func_str = ((*trans_func)(OrigFuncStr)).ptr(); \ ++ }else{ \ ++ dst_func_str = OrigFuncStr; \ ++ } \ ++ dst_func_str; \ ++}) + + } // namespace runtime + } // namespace tvm +diff --git a/src/codegen/llvm/codegen_cpu.cc b/src/codegen/llvm/codegen_cpu.cc +index 0ba0c58..2850ad4 100644 +--- a/src/codegen/llvm/codegen_cpu.cc ++++ b/src/codegen/llvm/codegen_cpu.cc +@@ -99,26 +99,26 @@ void CodeGenCPU::Init(const std::string& module_name, + // We will need this in environment for backward registration. + f_tvm_register_system_symbol_ = llvm::Function::Create( + llvm::FunctionType::get(t_int_, {t_char_->getPointerTo(), t_void_p_}, false), +- llvm::Function::ExternalLinkage, "TVMBackendRegisterSystemLibSymbol", module_.get()); ++ llvm::Function::ExternalLinkage, TVM_RT_FUNC_TRANS("TVMBackendRegisterSystemLibSymbol"), module_.get()); + } else { + f_tvm_register_system_symbol_ = nullptr; + } + if (dynamic_lookup || system_lib) { + f_tvm_func_call_ = llvm::Function::Create( + ftype_tvm_func_call_, +- llvm::Function::ExternalLinkage, "TVMFuncCall", module_.get()); ++ llvm::Function::ExternalLinkage, TVM_RT_FUNC_TRANS("TVMFuncCall"), module_.get()); + f_tvm_get_func_from_env_ = llvm::Function::Create( + ftype_tvm_get_func_from_env_, + llvm::Function::ExternalLinkage, "TVMBackendGetFuncFromEnv", module_.get()); + f_tvm_api_set_last_error_ = llvm::Function::Create( + ftype_tvm_api_set_last_error_, +- llvm::Function::ExternalLinkage, "TVMAPISetLastError", module_.get()); ++ llvm::Function::ExternalLinkage, TVM_RT_FUNC_TRANS("TVMAPISetLastError"), module_.get()); + f_tvm_parallel_launch_ = llvm::Function::Create( + ftype_tvm_parallel_launch_, +- llvm::Function::ExternalLinkage, "TVMBackendParallelLaunch", module_.get()); ++ llvm::Function::ExternalLinkage, TVM_RT_FUNC_TRANS("TVMBackendParallelLaunch"), module_.get()); + f_tvm_parallel_barrier_ = llvm::Function::Create( + ftype_tvm_parallel_barrier_, +- llvm::Function::ExternalLinkage, "TVMBackendParallelBarrier", module_.get()); ++ llvm::Function::ExternalLinkage, TVM_RT_FUNC_TRANS("TVMBackendParallelBarrier"), module_.get()); + } + this->InitGlobalContext(dynamic_lookup); + } +@@ -461,11 +461,14 @@ void CodeGenCPU::CreateComputeScope(const AttrStmt* op) { + } + std::swap(function_, fcompute); + std::swap(new_vmap, var_map_); ++ std::stack br_ret_flg; ++ std::swap(br_ret_flg, br_ret_flg_); + BasicBlock *compute_entry = BasicBlock::Create(*ctx_, "entry", function_); + builder_->SetInsertPoint(compute_entry); + this->VisitStmt(op->body); + builder_->CreateRet(ConstInt32(0)); + // swap the var map back, now we are back on track. ++ std::swap(br_ret_flg, br_ret_flg_); + std::swap(new_vmap, var_map_); + std::swap(function_, fcompute); + builder_->SetInsertPoint(compute_call_end); +@@ -542,9 +545,12 @@ void CodeGenCPU::CreateParallelLaunch(const Stmt& body, int num_task) { + std::swap(function_, f); + std::swap(parallel_env_, par_env); + std::swap(var_map_, new_vmap); ++ std::stack br_ret_flg; ++ std::swap(br_ret_flg, br_ret_flg_); + this->VisitStmt(body); + builder_->CreateRet(ConstInt32(0)); + // swap the var map back, now we are back on track. ++ std::swap(br_ret_flg, br_ret_flg_); + std::swap(var_map_, new_vmap); + std::swap(parallel_env_, par_env); + std::swap(function_, f); +@@ -794,7 +800,9 @@ llvm::Value* CodeGenCPU::CreateIntrinsic(const Call* op) { + } else if (op->is_intrinsic(intrinsic::tvm_static_handle)) { + return CreateStaticHandle(); + } else if (op->is_intrinsic(intrinsic::tvm_throw_last_error)) { +- builder_->CreateRet(ConstInt32(-1)); ++ llvm::Value* pRetCode = (op->args.size() == 0) ? ConstInt32(-1) : MakeValue(op->args[0]); ++ builder_->CreateRet(pRetCode); ++ CodeGenLLVM::SetRetTrFlg(true); + return ConstInt32(-1); + } else if (op->is_intrinsic(intrinsic::tvm_struct_get)) { + CHECK_EQ(op->args.size(), 3U); +diff --git a/src/codegen/llvm/codegen_llvm.cc b/src/codegen/llvm/codegen_llvm.cc +index 2cff88b..e26812d 100644 +--- a/src/codegen/llvm/codegen_llvm.cc ++++ b/src/codegen/llvm/codegen_llvm.cc +@@ -1110,23 +1110,37 @@ void CodeGenLLVM::VisitStmt_(const IfThenElse* op) { + *ctx_, "if_then", function_); + BasicBlock* end_block = BasicBlock::Create( + *ctx_, "if_end", function_); ++ // define ret terminitor exist flg for this Stmt ++ bool cur_br_ret_flg = false; ++ br_ret_flg_.push(&cur_br_ret_flg); + if (op->else_case.defined()) { + BasicBlock* else_block = BasicBlock::Create( + *ctx_, "if_else", function_); + builder_->CreateCondBr(cond, then_block, else_block); + builder_->SetInsertPoint(then_block); ++ cur_br_ret_flg = false; + this->VisitStmt(op->then_case); + builder_->CreateBr(end_block); ++ if ( !cur_br_ret_flg ){ ++ builder_->CreateBr(end_block); ++ } + builder_->SetInsertPoint(else_block); ++ cur_br_ret_flg = false; + this->VisitStmt(op->else_case); +- builder_->CreateBr(end_block); ++ if ( !cur_br_ret_flg ){ ++ builder_->CreateBr(end_block); ++ } + } else { + builder_->CreateCondBr(cond, then_block, end_block, md_very_likely_branch_); + builder_->SetInsertPoint(then_block); ++ cur_br_ret_flg = false; + this->VisitStmt(op->then_case); +- builder_->CreateBr(end_block); ++ if ( !cur_br_ret_flg ){ ++ builder_->CreateBr(end_block); ++ } + } + builder_->SetInsertPoint(end_block); ++ br_ret_flg_.pop(); + } + + +diff --git a/src/codegen/llvm/codegen_llvm.h b/src/codegen/llvm/codegen_llvm.h +index b7d091b..6fba863 100644 +--- a/src/codegen/llvm/codegen_llvm.h ++++ b/src/codegen/llvm/codegen_llvm.h +@@ -143,6 +143,11 @@ class CodeGenLLVM : + void VisitStmt_(const Block* op) override; + void VisitStmt_(const Evaluate* op) override; + void VisitStmt_(const ProducerConsumer* op) override; ++ //Set IfThelElse branch exist Return flg ++ void SetRetTrFlg(bool RetFlg){ ++ if( !br_ret_flg_.empty() ) ++ *(br_ret_flg_.top()) = RetFlg; ++ } + + protected: + /*! \brief The storage information */ +@@ -304,6 +309,12 @@ class CodeGenLLVM : + * initializes file and compilation_unit_ to TVM defaults. + */ + static std::unique_ptr CreateDebugInfo(llvm::Module* module); ++ ++ /* ++ * IfThenElse stmt branch return flg store stack ++ * if a branch already return, can't add br terminator again ++ */ ++ std::stack br_ret_flg_; + }; + } // namespace codegen + } // namespace tvm +diff --git a/src/pass/lower_tvm_builtin.cc b/src/pass/lower_tvm_builtin.cc +index e73956c..3a7b46c 100644 +--- a/src/pass/lower_tvm_builtin.cc ++++ b/src/pass/lower_tvm_builtin.cc +@@ -104,7 +104,7 @@ class BuiltinLower : public IRMutator { + CHECK(device_type_.defined()) << "Unknown device type in current IR"; + CHECK(device_id_.defined()) << "Unknown device id in current IR"; + Stmt throw_last_error = Evaluate::make(Call::make(Int(32), +- intrinsic::tvm_throw_last_error, {}, ++ intrinsic::tvm_throw_last_error, {(Int(32), 1001)}, + Call::Intrinsic)); + + Stmt body = Block::make( +@@ -117,7 +117,7 @@ class BuiltinLower : public IRMutator { + Stmt alloca = LetStmt::make( + op->buffer_var, + Call::make(op->buffer_var.type(), +- "TVMBackendAllocWorkspace", ++ TVM_RT_FUNC_TRANS("TVMBackendAllocWorkspace"), + {cast(Int(32), device_type_), + cast(Int(32), device_id_), + cast(UInt(64), total_bytes), +@@ -127,7 +127,7 @@ class BuiltinLower : public IRMutator { + body); + + Expr free_op = Call::make(Int(32), +- "TVMBackendFreeWorkspace", ++ TVM_RT_FUNC_TRANS("TVMBackendFreeWorkspace"), + {cast(Int(32), device_type_), + cast(Int(32), device_id_), + op->buffer_var}, diff --git a/third_party/patch/sqlite/sqlite.patch001 b/third_party/patch/sqlite/sqlite.patch001 new file mode 100644 index 00000000000..f618dcfcb54 --- /dev/null +++ b/third_party/patch/sqlite/sqlite.patch001 @@ -0,0 +1,152 @@ +diff -Npur -x .git sqlite.3.31.1/manifest sqlite/manifest +--- sqlite.3.31.1/manifest 2020-02-28 16:47:53.931041525 +0800 ++++ sqlite/manifest 2020-02-28 17:29:15.553920054 +0800 +@@ -1,5 +1,5 @@ +-C Version\s3.31.1 +-D 2020-01-27T19:55:54.490 ++C A\sbetter\s(smaller\sand\sfaster)\ssolution\sto\sticket\s[4374860b29383380]. ++D 2020-02-17T19:25:07.592 + F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 + F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea + F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 +@@ -482,8 +482,8 @@ F src/ctime.c 1b0724e66f95f33b160b1af85c + F src/date.c 6c408fdd2e9ddf6e8431aba76315a2d061bea2cec8fbb75e25d7c1ba08274712 + F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a + F src/dbstat.c 0f55297469d4244ab7df395849e1af98eb5e95816af7c661e7d2d8402dea23da +-F src/delete.c a5c59b9c0251cf7682bc52af0d64f09b1aefc6781a63592c8f1136f7b73c66e4 +-F src/expr.c 003c59158b33d7f3b198122cb0d1e13c06517cc3932e56b42283eb0e96696d66 ++F src/delete.c 11000121c4281c0bce4e41db29addfaea0038eaa127ece02557c9207bc3e541d ++F src/expr.c 4b25db7f9472b3532560242193bc4eefaefc7720dc4f2d7ec9a89ada410c6ea2 + F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 + F src/fkey.c 92a248ec0fa4ed8ab60c98d9b188ce173aaf218f32e7737ba77deb2a684f9847 + F src/func.c 108577cebe8a50c86d849a93b99493a54e348dd0b846f00d13b52ca973d5baf4 +@@ -536,8 +536,8 @@ F src/shell.c.in c2e20c43a44fb5588a6c27c + F src/sqlite.h.in 75d0304247a2154122d6d06f12219c1e29291d72304f0eeef4c1ec6b1409b443 + F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 + F src/sqlite3ext.h 27951f294f29cd875c6027f2707d644ef99f469bd97514568b5a8581a114db8c +-F src/sqliteInt.h 7a29ba700a51eeb925731749a570cf3859f6a58ed94797ecf47508875b0ba279 +-F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b ++F src/sqliteInt.h d736043dc6291d3af289d911237da0801b6c05be086ae322eedd47a089ae8d2f ++F src/sqliteLimit.h 95cb8479ca459496d9c1c6a9f76b38aee12203a56ce1092fe13e50ae2454c032 + F src/status.c 9ff2210207c6c3b4d9631a8241a7d45ab1b26a0e9c84cb07a9b5ce2de9a3b278 + F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 + F src/tclsqlite.c 97590069efaba5a4928ecffb606e3771dd93ee8e6bf248a62a6507c37a2b2e46 +@@ -619,8 +619,8 @@ F src/wal.h 606292549f5a7be50b6227bd685f + F src/walker.c a137468bf36c92e64d2275caa80c83902e3a0fc59273591b96c6416d3253d05d + F src/where.c 2005d0511e05e5f7b6fb3be514b44f264f23d45f3b0cc5e150c63e3006a003e5 + F src/whereInt.h 9157228db086f436a574589f8cc5749bd971e94017c552305ad9ec472ed2e098 +-F src/wherecode.c ec8870d6fe79668dd12d7edc65ae9771828d6cdfe478348c8abd872a89fdbadd +-F src/whereexpr.c 4b34be1434183e7bb8a05d4bf42bd53ea53021b0b060936fbd12062b4ff6b396 ++F src/wherecode.c f5df56e395ade2240cabb2d39500c681bd29f8cc0636c3301c4996ad160df94d ++F src/whereexpr.c 264d58971eaf8256eb5b0917bcd7fc7a1f1109fdda183a8382308a1b18a2dce7 + F src/window.c f8ba2ee12a19b51d3ba42c16277c74185ee9215306bc0d5a03974ade8b5bc98f + F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 + F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 +@@ -1857,10 +1857,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91 + F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc + F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e + F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 +-P 6fb9a8fb85486a8fccc462856316ef523450c23a7a7a81c8dfb323fbe809f8f5 +-R bf075f6bcc1758c5c1ecd13052997456 +-T +bgcolor * #d0c0ff +-T +sym-release * +-T +sym-version-3.31.1 * ++P 9d0d4ab95dc0c56e053c2924ed322a9ea7b25439e6f74599f706905a1994e454 ++R 1c052b7cdf4947664b7043564b643ac3 + U drh +-Z 7c50801eed3eaef969e028ef5a0a641a ++Z e960557a43b001a47933dacf8bc1d10e +diff -Npur -x .git sqlite.3.31.1/manifest.uuid sqlite/manifest.uuid +--- sqlite.3.31.1/manifest.uuid 2020-02-28 16:47:53.931041525 +0800 ++++ sqlite/manifest.uuid 2020-02-28 17:29:26.233919583 +0800 +@@ -1 +1 @@ +-3bfa9cc97da10598521b342961df8f5f68c7388fa117345eeb516eaa837bb4d6 +\ No newline at end of file ++abc473fb8fb999005dc79a360e34f97b3b25429decf1820dd2afa5c19577753d +diff -Npur -x .git sqlite.3.31.1/src/expr.c sqlite/src/expr.c +--- sqlite.3.31.1/src/expr.c 2020-02-28 16:47:53.939041525 +0800 ++++ sqlite/src/expr.c 2020-02-28 17:29:15.537920055 +0800 +@@ -5463,19 +5463,25 @@ static int impliesNotNullRow(Walker *pWa + case TK_LT: + case TK_LE: + case TK_GT: +- case TK_GE: ++ case TK_GE: { ++ Expr *pLeft = pExpr->pLeft; ++ Expr *pRight = pExpr->pRight; + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); +- if( (pExpr->pLeft->op==TK_COLUMN && IsVirtual(pExpr->pLeft->y.pTab)) +- || (pExpr->pRight->op==TK_COLUMN && IsVirtual(pExpr->pRight->y.pTab)) ++ /* The y.pTab=0 assignment in wherecode.c always happens after the ++ ** impliesNotNullRow() test */ ++ if( (pLeft->op==TK_COLUMN && ALWAYS(pLeft->y.pTab!=0) ++ && IsVirtual(pLeft->y.pTab)) ++ || (pRight->op==TK_COLUMN && ALWAYS(pRight->y.pTab!=0) ++ && IsVirtual(pRight->y.pTab)) + ){ +- return WRC_Prune; ++ return WRC_Prune; + } +- ++ } + default: + return WRC_Continue; + } +diff -Npur -x .git sqlite.3.31.1/src/sqliteInt.h sqlite/src/sqliteInt.h +--- sqlite.3.31.1/src/sqliteInt.h 2020-02-28 16:47:53.959041524 +0800 ++++ sqlite/src/sqliteInt.h 2020-02-28 17:29:15.517920056 +0800 +@@ -2153,8 +2153,11 @@ struct Table { + */ + #ifndef SQLITE_OMIT_VIRTUALTABLE + # define IsVirtual(X) ((X)->nModuleArg) ++# define ExprIsVtab(X) \ ++ ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->nModuleArg) + #else + # define IsVirtual(X) 0 ++# define ExprIsVtab(X) 0 + #endif + + /* +diff -Npur -x .git sqlite.3.31.1/src/whereexpr.c sqlite/src/whereexpr.c +--- sqlite.3.31.1/src/whereexpr.c 2020-02-28 16:47:53.991041522 +0800 ++++ sqlite/src/whereexpr.c 2020-02-28 17:29:15.493920057 +0800 +@@ -377,7 +377,8 @@ static int isAuxiliaryVtabOperator( + ** MATCH(expression,vtab_column) + */ + pCol = pList->a[1].pExpr; +- if( pCol->op==TK_COLUMN && IsVirtual(pCol->y.pTab) ){ ++ testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); ++ if( ExprIsVtab(pCol) ){ + for(i=0; iu.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; +@@ -399,7 +400,8 @@ static int isAuxiliaryVtabOperator( + ** with function names in an arbitrary case. + */ + pCol = pList->a[0].pExpr; +- if( pCol->op==TK_COLUMN && IsVirtual(pCol->y.pTab) ){ ++ testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); ++ if( ExprIsVtab(pCol) ){ + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xNotUsed)(sqlite3_context*,int,sqlite3_value**); +@@ -422,10 +424,12 @@ static int isAuxiliaryVtabOperator( + int res = 0; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; +- if( pLeft->op==TK_COLUMN && IsVirtual(pLeft->y.pTab) ){ ++ testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); ++ if( ExprIsVtab(pLeft) ){ + res++; + } +- if( pRight && pRight->op==TK_COLUMN && IsVirtual(pRight->y.pTab) ){ ++ testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); ++ if( pRight && ExprIsVtab(pRight) ){ + res++; + SWAP(Expr*, pLeft, pRight); + } diff --git a/third_party/protobuf b/third_party/protobuf new file mode 160000 index 00000000000..09745575a92 --- /dev/null +++ b/third_party/protobuf @@ -0,0 +1 @@ +Subproject commit 09745575a923640154bcf307fba8aedff47f240a diff --git a/third_party/securec/CMakeLists.txt b/third_party/securec/CMakeLists.txt new file mode 100644 index 00000000000..e360a6ebaea --- /dev/null +++ b/third_party/securec/CMakeLists.txt @@ -0,0 +1,11 @@ +SET(CMAKE_BUILD_TYPE "Debug") +SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") +SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations") +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +#add flags +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") + + +include_directories(./include) +add_subdirectory(src) diff --git a/third_party/securec/include/securec.h b/third_party/securec/include/securec.h new file mode 100644 index 00000000000..b627a3c37a2 --- /dev/null +++ b/third_party/securec/include/securec.h @@ -0,0 +1,634 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SECUREC_H__5D13A042_DC3F_4ED9_A8D1_882811274C27 +#define __SECUREC_H__5D13A042_DC3F_4ED9_A8D1_882811274C27 + +#include "securectype.h" +#include + +#ifndef SECUREC_HAVE_ERRNO_H +#if SECUREC_IN_KERNEL +#define SECUREC_HAVE_ERRNO_H 0 +#else +#define SECUREC_HAVE_ERRNO_H 1 +#endif +#endif + +/* EINVAL ERANGE may defined in errno.h */ +#if SECUREC_HAVE_ERRNO_H +#include +#endif + +/* define error code */ +#if defined(SECUREC_NEED_ERRNO_TYPE) || !defined(__STDC_WANT_LIB_EXT1__) || \ + (defined(__STDC_WANT_LIB_EXT1__) && (__STDC_WANT_LIB_EXT1__ == 0)) +#ifndef SECUREC_DEFINED_ERRNO_TYPE +#define SECUREC_DEFINED_ERRNO_TYPE +/* just check whether macrodefinition exists. */ +#ifndef errno_t +typedef int errno_t; +#endif +#endif +#endif + +/* success */ +#ifndef EOK +#define EOK 0 +#endif + +#ifndef EINVAL +/* The src buffer is not correct and destination buffer cant not be reset */ +#define EINVAL 22 +#endif + +#ifndef EINVAL_AND_RESET +/* Once the error is detected, the dest buffer must be reseted! */ +#define EINVAL_AND_RESET (22 | 128) +#endif + +#ifndef ERANGE +/* The destination buffer is not long enough and destination buffer can not be reset */ +#define ERANGE 34 +#endif + +#ifndef ERANGE_AND_RESET +/* Once the error is detected, the dest buffer must be reseted! */ +#define ERANGE_AND_RESET (34 | 128) +#endif + +#ifndef EOVERLAP_AND_RESET +/* Once the buffer overlap is detected, the dest buffer must be reseted! */ +#define EOVERLAP_AND_RESET (54 | 128) +#endif + +/* if you need export the function of this library in Win32 dll, use __declspec(dllexport) */ +#ifndef SECUREC_API +#if defined(SECUREC_DLL_EXPORT) +#define SECUREC_API __declspec(dllexport) +#elif defined(SECUREC_DLL_IMPORT) +#define SECUREC_API __declspec(dllimport) +#else +/* Standardized function declaration . If a security function is declared in the your code, + * it may cause a compilation alarm,Please delete the security function you declared + * Adding extern under windows will cause the system to have inline functions to expand, + * so do not add the extern in default + */ +#if defined(_MSC_VER) +#define SECUREC_API +#else +#define SECUREC_API extern +#endif +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + /* + * Description: The GetHwSecureCVersion function get SecureC Version string and version number. + * Parameter: verNumber - to store version number + * Return: version string + */ + SECUREC_API const char *GetHwSecureCVersion(unsigned short *verNumber); + +#if SECUREC_ENABLE_MEMSET + /* + * Description: The memset_s function copies the value of c (converted to an unsigned char) into each of + * the first count characters of the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: c - the value to be copied + * Parameter: count -copies fisrt count characters of dest + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t memset_s(void *dest, size_t destMax, int c, size_t count); +#endif + +#ifndef SECUREC_ONLY_DECLARE_MEMSET +#define SECUREC_ONLY_DECLARE_MEMSET 0 +#endif + +#if SECUREC_ONLY_DECLARE_MEMSET == 0 + +#if SECUREC_ENABLE_MEMMOVE + /* + * Description: The memmove_s function copies n characters from the object pointed to by src + * into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t memmove_s(void *dest, size_t destMax, const void *src, size_t count); +#endif + +#if SECUREC_ENABLE_MEMCPY + /* + * Description: The memcpy_s function copies n characters from the object pointed to + * by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t memcpy_s(void *dest, size_t destMax, const void *src, size_t count); +#endif + +#if SECUREC_ENABLE_STRCPY + /* + * Description: The strcpy_s function copies the string pointed to by strSrc (including + * the terminating null character) into the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strcpy_s(char *strDest, size_t destMax, const char *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCPY + /* + * Description: The strncpy_s function copies not more than n successive characters (not including + * the terminating null character) + * from the array pointed to by strSrc to the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Parameter: strSrc -source address + * Parameter: count -copies count characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strncpy_s(char *strDest, size_t destMax, const char *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_STRCAT + /* + * Description: The strcat_s function appends a copy of the string pointed to by strSrc (including + * the terminating null character) + * to the end of the string pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null wide character) + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strcat_s(char *strDest, size_t destMax, const char *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCAT + /* + * Description: The strncat_s function appends not more than n successive characters (not including + * the terminating null character) + * from the array pointed to by strSrc to the end of the string pointed to by strDest. + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Parameter: strSrc -source address + * Parameter: count -copies count characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strncat_s(char *strDest, size_t destMax, const char *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_VSPRINTF + /* + * Description: The vsprintf_s function is equivalent to the vsprintf function except for the Parameter: destMax + * and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null wide characte) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1. + */ + SECUREC_API int vsprintf_s(char *strDest, size_t destMax, const char *format, + va_list argList) SECUREC_ATTRIBUTE(3, 0); +#endif + +#if SECUREC_ENABLE_SPRINTF + /* + * Description: The sprintf_s function is equivalent to the sprintf function except for the Parameter: destMax + * and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1. + */ + SECUREC_API int sprintf_s(char *strDest, size_t destMax, const char *format, ...) SECUREC_ATTRIBUTE(3, 4); +#endif + +#if SECUREC_ENABLE_VSNPRINTF + /* + * Description: The vsnprintf_s function is equivalent to the vsnprintf function except for the Parameter: + * destMax/count and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: count - do not write more than count bytes to strDest(not including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning -1 when truncation occurs + */ + SECUREC_API int vsnprintf_s(char *strDest, size_t destMax, size_t count, const char *format, + va_list argList) SECUREC_ATTRIBUTE(4, 0); +#endif + +#if SECUREC_ENABLE_SNPRINTF + /* + * Description: The snprintf_s function is equivalent to the snprintf function except for the Parameter: + * destMax/count and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: count - do not write more than count bytes to strDest(not including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning -1 when truncation occurs + */ + SECUREC_API int snprintf_s(char *strDest, size_t destMax, size_t count, const char *format, + ...) SECUREC_ATTRIBUTE(4, 5); +#endif + +#if SECUREC_SNPRINTF_TRUNCATED + /* + * Description: The vsnprintf_truncated_s function is equivalent to the vsnprintf_s function except + * no count Parameter: and Return: value + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning destMax - 1 when truncation occurs + */ + SECUREC_API int vsnprintf_truncated_s(char *strDest, size_t destMax, const char *format, + va_list argList) SECUREC_ATTRIBUTE(3, 0); + + /* + * Description: The snprintf_truncated_s function is equivalent to the snprintf_2 function except + * no count Parameter: and Return: value + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning destMax - 1 when truncation occurs + */ + SECUREC_API int snprintf_truncated_s(char *strDest, size_t destMax, + const char *format, ...) SECUREC_ATTRIBUTE(3, 4); +#endif + +#if SECUREC_ENABLE_SCANF + /* + * Description: The scanf_s function is equivalent to fscanf_s with the argument stdin + * interposed before the arguments to scanf_s + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int scanf_s(const char *format, ...); +#endif + +#if SECUREC_ENABLE_VSCANF + /* + * Description: The vscanf_s function is equivalent to scanf_s, with the variable argument list replaced by argList + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vscanf_s(const char *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SSCANF + /* + * Description: The sscanf_s function is equivalent to fscanf_s, except that input is obtained from a + * string (specified by the argument buffer) rather than from a stream + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int sscanf_s(const char *buffer, const char *format, ...); +#endif + +#if SECUREC_ENABLE_VSSCANF + /* + * Description: The vsscanf_s function is equivalent to sscanf_s, with the variable argument list + * replaced by argList + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vsscanf_s(const char *buffer, const char *format, va_list argList); +#endif + +#if SECUREC_ENABLE_FSCANF + /* + * Description: The fscanf_s function is equivalent to fscanf except that the c, s, and [ conversion specifiers + * apply to a pair of arguments (unless assignment suppression is indicated by a*) + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int fscanf_s(FILE *stream, const char *format, ...); +#endif + +#if SECUREC_ENABLE_VFSCANF + /* + * Description: The vfscanf_s function is equivalent to fscanf_s, with the variable argument list + * replaced by argList + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vfscanf_s(FILE *stream, const char *format, va_list argList); +#endif + +#if SECUREC_ENABLE_STRTOK + /* + * Description: The strtok_s function parses a string into a sequence of strToken, + * replace all characters in strToken string that match to strDelimit set with 0. + * On the first call to strtok_s the string to be parsed should be specified in strToken. + * In each subsequent call that should parse the same string, strToken should be NULL + * Parameter: strToken - the string to be delimited + * Parameter: strDelimit -specifies a set of characters that delimit the tokens in the parsed string + * Parameter: context -is a pointer to a char * variable that is used internally by strtok_s function + * Return: On the first call returns the address of the first non \0 character, otherwise NULL is returned. + * In subsequent calls, the strtoken is set to NULL, and the context set is the same as the previous call, + * return NULL if the *context string length is equal 0, otherwise return *context. + */ + SECUREC_API char *strtok_s(char *strToken, const char *strDelimit, char **context); +#endif + +#if SECUREC_ENABLE_GETS && SECUREC_IN_KERNEL == 0 + /* + * Description: The gets_s function reads at most one less than the number of characters specified + * by destMax from the stream pointed to by stdin, into the array pointed to by buffer + * Parameter: buffer - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Return: buffer if there was no runtime-constraint violation,If an error occurred Return: NULL. + */ + SECUREC_API char *gets_s(char *buffer, size_t destMax); +#endif + + +#if SECUREC_ENABLE_WCHAR_FUNC +#if SECUREC_ENABLE_MEMCPY + /* + * Description: The wmemcpy_s function copies n successive wide characters from the object pointed to + * by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wmemcpy_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count); +#endif + +#if SECUREC_ENABLE_MEMMOVE + /* + * Description: The wmemmove_s function copies n successive wide characters from the object + * pointed to by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wmemmove_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count); +#endif + +#if SECUREC_ENABLE_STRCPY + /* + * Description: The wcscpy_s function copies the wide string pointed to by strSrc (including theterminating + * null wide character) into the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcscpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCPY + /* + * Description: The wcsncpy_s function copies not more than n successive wide characters (not including the + * terminating null wide character) from the array pointed to by strSrc to the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating wide character) + * Parameter: strSrc -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcsncpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_STRCAT + /* + * Description: The wcscat_s function appends a copy of the wide string pointed to by strSrc (including the + * terminating null wide character) to the end of the wide string pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating wide character) + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcscat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCAT + /* + * Description: The wcsncat_s function appends not more than n successive wide characters (not including the + * terminating null wide character) from the array pointed to by strSrc to the end of the wide string pointed to + * by strDest. + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating wide character) + * Parameter: strSrc -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcsncat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_STRTOK + /* + * Description: The wcstok_s function is the wide-character equivalent of the strtok_s function + * Parameter: strToken - the string to be delimited + * Parameter: strDelimit -specifies a set of characters that delimit the tokens in the parsed string + * Parameter: context -is a pointer to a char * variable that is used internally by strtok_s function + * Return: a pointer to the first character of a token, or a null pointer if there is no token + * or there is a runtime-constraint violation. + */ + SECUREC_API wchar_t *wcstok_s(wchar_t *strToken, const wchar_t *strDelimit, wchar_t **context); +#endif + +#if SECUREC_ENABLE_VSPRINTF + /* + * Description: The vswprintf_s function is the wide-character equivalent of the vsprintf_s function + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null ) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null wide characte), + * If an error occurred Return: -1. + */ + SECUREC_API int vswprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SPRINTF + + /* + * Description: The swprintf_s function is the wide-character equivalent of the sprintf_s function + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null ) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null wide characte), + * If an error occurred Return: -1. + */ + SECUREC_API int swprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_FSCANF + /* + * Description: The fwscanf_s function is the wide-character equivalent of the fscanf_s function + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int fwscanf_s(FILE *stream, const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_VFSCANF + /* + * Description: The vfwscanf_s function is the wide-character equivalent of the vfscanf_s function + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vfwscanf_s(FILE *stream, const wchar_t *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SCANF + /* + * Description: The wscanf_s function is the wide-character equivalent of the scanf_s function + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int wscanf_s(const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_VSCANF + /* + * Description: The vwscanf_s function is the wide-character equivalent of the vscanf_s function + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vwscanf_s(const wchar_t *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SSCANF + /* + * Description: The swscanf_s function is the wide-character equivalent of the sscanf_s function + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int swscanf_s(const wchar_t *buffer, const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_VSSCANF + /* + * Description: The vswscanf_s function is the wide-character equivalent of the vsscanf_s function + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vswscanf_s(const wchar_t *buffer, const wchar_t *format, va_list argList); +#endif +#endif /* SECUREC_ENABLE_WCHAR_FUNC */ +#endif + + /* those functions are used by macro ,must declare hare , also for without function declaration warning */ + extern errno_t strncpy_error(char *strDest, size_t destMax, const char *strSrc, size_t count); + extern errno_t strcpy_error(char *strDest, size_t destMax, const char *strSrc); + +#if SECUREC_WITH_PERFORMANCE_ADDONS + /* those functions are used by macro */ + extern errno_t memset_sOptAsm(void *dest, size_t destMax, int c, size_t count); + extern errno_t memset_sOptTc(void *dest, size_t destMax, int c, size_t count); + extern errno_t memcpy_sOptAsm(void *dest, size_t destMax, const void *src, size_t count); + extern errno_t memcpy_sOptTc(void *dest, size_t destMax, const void *src, size_t count); + +/* strcpy_sp is a macro, NOT a function in performance optimization mode. */ +#define strcpy_sp(dest, destMax, src) ((__builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRCPY_SM((dest), (destMax), (src)) : \ + strcpy_s((dest), (destMax), (src))) + +/* strncpy_sp is a macro, NOT a function in performance optimization mode. */ +#define strncpy_sp(dest, destMax, src, count) ((__builtin_constant_p((count)) && \ + __builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRNCPY_SM((dest), (destMax), (src), (count)) : \ + strncpy_s((dest), (destMax), (src), (count))) + +/* strcat_sp is a macro, NOT a function in performance optimization mode. */ +#define strcat_sp(dest, destMax, src) ((__builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRCAT_SM((dest), (destMax), (src)) : \ + strcat_s((dest), (destMax), (src))) + +/* strncat_sp is a macro, NOT a function in performance optimization mode. */ +#define strncat_sp(dest, destMax, src, count) ((__builtin_constant_p((count)) && \ + __builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRNCAT_SM((dest), (destMax), (src), (count)) : \ + strncat_s((dest), (destMax), (src), (count))) + +/* memcpy_sp is a macro, NOT a function in performance optimization mode. */ +#define memcpy_sp(dest, destMax, src, count) (__builtin_constant_p((count)) ? \ + (SECUREC_MEMCPY_SM((dest), (destMax), (src), (count))) : \ + (__builtin_constant_p((destMax)) ? \ + (((size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & \ + (unsigned long long)(-2)) < SECUREC_MEM_MAX_LEN)) ? \ + memcpy_sOptTc((dest), (destMax), (src), (count)) : ERANGE) : \ + memcpy_sOptAsm((dest), (destMax), (src), (count)))) + +/* memset_sp is a macro, NOT a function in performance optimization mode. */ +#define memset_sp(dest, destMax, c, count) (__builtin_constant_p((count)) ? \ + (SECUREC_MEMSET_SM((dest), (destMax), (c), (count))) : \ + (__builtin_constant_p((destMax)) ? \ + (((size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & \ + (unsigned long long)(-2)) < SECUREC_MEM_MAX_LEN)) ? \ + memset_sOptTc((dest), (destMax), (c), (count)) : ERANGE) : \ + memset_sOptAsm((dest), (destMax), (c), (count)))) +#else +#define strcpy_sp strcpy_s +#define strncpy_sp strncpy_s +#define strcat_sp strcat_s +#define strncat_sp strncat_s +#define memcpy_sp memcpy_s +#define memset_sp memset_s +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* __SECUREC_H__5D13A042_DC3F_4ED9_A8D1_882811274C27 */ + diff --git a/third_party/securec/include/securectype.h b/third_party/securec/include/securectype.h new file mode 100644 index 00000000000..0aed2a6788a --- /dev/null +++ b/third_party/securec/include/securectype.h @@ -0,0 +1,542 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SECURECTYPE_H__A7BBB686_AADA_451B_B9F9_44DACDAE18A7 +#define __SECURECTYPE_H__A7BBB686_AADA_451B_B9F9_44DACDAE18A7 + +#ifndef SECUREC_USING_STD_SECURE_LIB +#if defined(_MSC_VER) && _MSC_VER >= 1400 +#if defined(__STDC_WANT_SECURE_LIB__) && __STDC_WANT_SECURE_LIB__ == 0 +/* Security functions have been provided since vs2005, default use of system library functions */ +#define SECUREC_USING_STD_SECURE_LIB 0 +#else +#define SECUREC_USING_STD_SECURE_LIB 1 +#endif +#else +#define SECUREC_USING_STD_SECURE_LIB 0 +#endif +#endif + + +/* Compatibility with older Secure C versions, shielding VC symbol redefinition warning */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 && SECUREC_USING_STD_SECURE_LIB == 0 +#ifndef SECUREC_DISABLE_CRT_FUNC +#define SECUREC_DISABLE_CRT_FUNC 1 +#endif +#ifndef SECUREC_DISABLE_CRT_IMP +#define SECUREC_DISABLE_CRT_IMP 1 +#endif +#else /* MSC VER */ +#ifndef SECUREC_DISABLE_CRT_FUNC +#define SECUREC_DISABLE_CRT_FUNC 0 +#endif +#ifndef SECUREC_DISABLE_CRT_IMP +#define SECUREC_DISABLE_CRT_IMP 0 +#endif +#endif + +#if SECUREC_DISABLE_CRT_FUNC +#ifdef __STDC_WANT_SECURE_LIB__ +#undef __STDC_WANT_SECURE_LIB__ +#endif +#define __STDC_WANT_SECURE_LIB__ 0 +#endif + +#if SECUREC_DISABLE_CRT_IMP +#ifdef _CRTIMP_ALTERNATIVE +#undef _CRTIMP_ALTERNATIVE +#endif +#define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ +#endif + +/* Compile in kernel under macro control */ +#ifndef SECUREC_IN_KERNEL +#ifdef __KERNEL__ +#define SECUREC_IN_KERNEL 1 +#else +#define SECUREC_IN_KERNEL 0 +#endif +#endif + +#if SECUREC_IN_KERNEL +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 0 +#endif +#ifndef SECUREC_ENABLE_WCHAR_FUNC +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#endif +#else /* SECUREC_IN_KERNEL */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 1 +#endif +#ifndef SECUREC_ENABLE_WCHAR_FUNC +#define SECUREC_ENABLE_WCHAR_FUNC 1 +#endif +#endif + + +/* Default secure function declaration, default declarations for non-standard functions */ +#ifndef SECUREC_SNPRINTF_TRUNCATED +#define SECUREC_SNPRINTF_TRUNCATED 1 +#endif + +#if SECUREC_USING_STD_SECURE_LIB +#if defined(_MSC_VER) && _MSC_VER >= 1400 +/* Declare secure functions that are not available in the vs compiler */ +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 1 +#endif +/* vs 2005 have vsnprintf_s function */ +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +/* vs 2005 have vsnprintf_s function Adapt the snprintf_s of the security function */ +#define snprintf_s _snprintf_s +#define SECUREC_ENABLE_SNPRINTF 0 +#endif +/* befor vs 2010 do not have v functions */ +#if _MSC_VER <= 1600 || defined(SECUREC_FOR_V_SCANFS) +#ifndef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSCANF +#define SECUREC_ENABLE_VSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 1 +#endif +#endif + +#else /* _MSC_VER */ +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 0 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +#define SECUREC_ENABLE_SNPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_MEMMOVE +#define SECUREC_ENABLE_MEMMOVE 0 +#endif +#ifndef SECUREC_ENABLE_MEMCPY +#define SECUREC_ENABLE_MEMCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRCPY +#define SECUREC_ENABLE_STRCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRNCPY +#define SECUREC_ENABLE_STRNCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRCAT +#define SECUREC_ENABLE_STRCAT 0 +#endif +#ifndef SECUREC_ENABLE_STRNCAT +#define SECUREC_ENABLE_STRNCAT 0 +#endif +#ifndef SECUREC_ENABLE_SPRINTF +#define SECUREC_ENABLE_SPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_VSPRINTF +#define SECUREC_ENABLE_VSPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_SSCANF +#define SECUREC_ENABLE_SSCANF 0 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 0 +#endif +#ifndef SECUREC_ENABLE_SCANF +#define SECUREC_ENABLE_SCANF 0 +#endif +#ifndef SECUREC_ENABLE_VSCANF +#define SECUREC_ENABLE_VSCANF 0 +#endif + +#ifndef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif +#ifndef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#ifndef SECUREC_ENABLE_STRTOK +#define SECUREC_ENABLE_STRTOK 0 +#endif +#ifndef SECUREC_ENABLE_GETS +#define SECUREC_ENABLE_GETS 0 +#endif + +#else /* SECUREC_USE_STD_SECURE_LIB */ + +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 1 +#endif +#ifndef SECUREC_ENABLE_MEMMOVE +#define SECUREC_ENABLE_MEMMOVE 1 +#endif +#ifndef SECUREC_ENABLE_MEMCPY +#define SECUREC_ENABLE_MEMCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRCPY +#define SECUREC_ENABLE_STRCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRNCPY +#define SECUREC_ENABLE_STRNCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRCAT +#define SECUREC_ENABLE_STRCAT 1 +#endif +#ifndef SECUREC_ENABLE_STRNCAT +#define SECUREC_ENABLE_STRNCAT 1 +#endif +#ifndef SECUREC_ENABLE_SPRINTF +#define SECUREC_ENABLE_SPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_VSPRINTF +#define SECUREC_ENABLE_VSPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +#define SECUREC_ENABLE_SNPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_SSCANF +#define SECUREC_ENABLE_SSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 1 +#endif +#ifndef SECUREC_ENABLE_SCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF 1 +#else +#define SECUREC_ENABLE_SCANF 0 +#endif +#endif +#ifndef SECUREC_ENABLE_VSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_VSCANF 1 +#else +#define SECUREC_ENABLE_VSCANF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_FSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_FSCANF 1 +#else +#define SECUREC_ENABLE_FSCANF 0 +#endif +#endif +#ifndef SECUREC_ENABLE_VFSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_VFSCANF 1 +#else +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_STRTOK +#define SECUREC_ENABLE_STRTOK 1 +#endif +#ifndef SECUREC_ENABLE_GETS +#define SECUREC_ENABLE_GETS 1 +#endif +#endif /* SECUREC_USE_STD_SECURE_LIB */ + +#if SECUREC_ENABLE_SCANF_FILE == 0 +#if SECUREC_ENABLE_FSCANF +#undef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif +#if SECUREC_ENABLE_VFSCANF +#undef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#if SECUREC_ENABLE_SCANF +#undef SECUREC_ENABLE_SCANF +#define SECUREC_ENABLE_SCANF 0 +#endif +#if SECUREC_ENABLE_FSCANF +#undef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif + +#endif + +#if SECUREC_IN_KERNEL +#include +#include +#else +#include +#include +#include +#endif + +/* If you need high performance, enable the SECUREC_WITH_PERFORMANCE_ADDONS macro, default is enable . + * The macro is automatically closed on the windows platform and linux kernel + */ +#ifndef SECUREC_WITH_PERFORMANCE_ADDONS +#if SECUREC_IN_KERNEL +#define SECUREC_WITH_PERFORMANCE_ADDONS 0 +#else +#define SECUREC_WITH_PERFORMANCE_ADDONS 1 +#endif +#endif + +/* if enable SECUREC_COMPATIBLE_WIN_FORMAT, the output format will be compatible to Windows. */ +#if (defined(_WIN32) || defined(_WIN64) || defined(_MSC_VER)) && !defined(SECUREC_COMPATIBLE_LINUX_FORMAT) +#if !defined(SECUREC_COMPATIBLE_WIN_FORMAT) +#define SECUREC_COMPATIBLE_WIN_FORMAT +#endif +#endif + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) +/* in windows platform, can't use optimized function for there is no __builtin_constant_p like function */ +/* If need optimized macro, can define this: define __builtin_constant_p(x) 0 */ +#ifdef SECUREC_WITH_PERFORMANCE_ADDONS +#undef SECUREC_WITH_PERFORMANCE_ADDONS +#define SECUREC_WITH_PERFORMANCE_ADDONS 0 +#endif +#endif + +#if defined(__VXWORKS__) || defined(__vxworks) || defined(__VXWORKS) || defined(_VXWORKS_PLATFORM_) || \ + defined(SECUREC_VXWORKS_VERSION_5_4) +#if !defined(SECUREC_VXWORKS_PLATFORM) +#define SECUREC_VXWORKS_PLATFORM +#endif +#endif + +/* if enable SECUREC_COMPATIBLE_LINUX_FORMAT, the output format will be compatible to Linux. */ +#if !(defined(SECUREC_COMPATIBLE_WIN_FORMAT) || defined(SECUREC_VXWORKS_PLATFORM)) +#if !defined(SECUREC_COMPATIBLE_LINUX_FORMAT) +#define SECUREC_COMPATIBLE_LINUX_FORMAT +#endif +#endif + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +#include +#endif + +/* add the -DSECUREC_SUPPORT_FORMAT_WARNING compiler option to supoort -Wformat. + * default does not check the format is that the same data type in the actual code + * in the product is different in the original data type definition of VxWorks and Linux. + */ +#ifndef SECUREC_SUPPORT_FORMAT_WARNING +#define SECUREC_SUPPORT_FORMAT_WARNING 0 +#endif + +/* SECUREC_PCLINT for tool do not recognize __attribute__ just for pclint */ +#if SECUREC_SUPPORT_FORMAT_WARNING && !defined(SECUREC_PCLINT) +#define SECUREC_ATTRIBUTE(x, y) __attribute__((format(printf, (x), (y)))) +#else +#define SECUREC_ATTRIBUTE(x, y) +#endif + +/* SECUREC_PCLINT for tool do not recognize __builtin_expect, just for pclint */ +#if defined(__GNUC__) && \ + ((__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3))) && \ + !defined(SECUREC_PCLINT) +/* This is a built-in function that can be used without a declaration, if you encounter an undeclared compilation alarm, + * you can add -DSECUREC_NEED_BUILTIN_EXPECT_DECLARE to complier options + */ +#if defined(SECUREC_NEED_BUILTIN_EXPECT_DECLARE) +long __builtin_expect(long exp, long c); +#endif +#define SECUREC_LIKELY(x) __builtin_expect(!!(x), 1) +#define SECUREC_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define SECUREC_LIKELY(x) (x) +#define SECUREC_UNLIKELY(x) (x) +#endif + +/* define the max length of the string */ +#ifndef SECUREC_STRING_MAX_LEN +#define SECUREC_STRING_MAX_LEN (0x7fffffffUL) +#endif +#define SECUREC_WCHAR_STRING_MAX_LEN (SECUREC_STRING_MAX_LEN / sizeof(wchar_t)) + +/* add SECUREC_MEM_MAX_LEN for memcpy and memmove */ +#ifndef SECUREC_MEM_MAX_LEN +#define SECUREC_MEM_MAX_LEN (0x7fffffffUL) +#endif +#define SECUREC_WCHAR_MEM_MAX_LEN (SECUREC_MEM_MAX_LEN / sizeof(wchar_t)) + +#if SECUREC_STRING_MAX_LEN > 0x7fffffff +#error "max string is 2G" +#endif + +#if (defined(__GNUC__) && defined(__SIZEOF_POINTER__)) +#if (__SIZEOF_POINTER__ != 4) && (__SIZEOF_POINTER__ != 8) +#error "unsupported system" +#endif +#endif + +#if defined(_WIN64) || defined(WIN64) || defined(__LP64__) || defined(_LP64) +#define SECUREC_ON_64BITS +#endif + +#if (!defined(SECUREC_ON_64BITS) && defined(__GNUC__) && defined(__SIZEOF_POINTER__)) +#if __SIZEOF_POINTER__ == 8 +#define SECUREC_ON_64BITS +#endif +#endif + +#if defined(__SVR4) || defined(__svr4__) +#define SECUREC_ON_SOLARIS +#endif + +#if (defined(__hpux) || defined(_AIX) || defined(SECUREC_ON_SOLARIS)) +#define SECUREC_ON_UNIX +#endif + +/* codes should run under the macro SECUREC_COMPATIBLE_LINUX_FORMAT in unknow system on default, + * and strtold. The function + * strtold is referenced first at ISO9899:1999(C99), and some old compilers can + * not support these functions. Here provides a macro to open these functions: + * SECUREC_SUPPORT_STRTOLD -- if defined, strtold will be used + */ +#ifndef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 0 +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) +#if defined(__USE_ISOC99) || \ + (defined(_AIX) && defined(_ISOC99_SOURCE)) || \ + (defined(__hpux) && defined(__ia64)) || \ + (defined(SECUREC_ON_SOLARIS) && (!defined(_STRICT_STDC) && !defined(__XOPEN_OR_POSIX)) || \ + defined(_STDC_C99) || defined(__EXTENSIONS__)) +#undef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 1 +#endif +#endif +#if ((defined(SECUREC_WRLINUX_BELOW4) || defined(_WRLINUX_BELOW4_))) +#undef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 0 +#endif +#endif + + +#if SECUREC_WITH_PERFORMANCE_ADDONS + +#ifndef SECUREC_TWO_MIN +#define SECUREC_TWO_MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +/* for strncpy_s performance optimization */ +#define SECUREC_STRNCPY_SM(dest, destMax, src, count) \ + (((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN) && \ + (SECUREC_TWO_MIN((size_t)(count), strlen(src)) + 1) <= (size_t)(destMax)) ? \ + (((size_t)(count) < strlen(src)) ? (memcpy((dest), (src), (count)), *((char *)(dest) + (count)) = '\0', EOK) : \ + (memcpy((dest), (src), strlen(src) + 1), EOK)) : (strncpy_error((dest), (destMax), (src), (count)))) + +#define SECUREC_STRCPY_SM(dest, destMax, src) \ + (((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN) && \ + (strlen(src) + 1) <= (size_t)(destMax)) ? (memcpy((dest), (src), strlen(src) + 1), EOK) : \ + (strcpy_error((dest), (destMax), (src)))) + +/* for strcat_s performance optimization */ +#if defined(__GNUC__) +#define SECUREC_STRCAT_SM(dest, destMax, src) ({ \ + int catRet = EOK; \ + if ((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN)) { \ + char *catTmpDst = (char *)(dest); \ + size_t catRestSize = (destMax); \ + while (catRestSize > 0 && *catTmpDst != '\0') { \ + ++catTmpDst; \ + --catRestSize; \ + } \ + if (catRestSize == 0) { \ + catRet = EINVAL; \ + } else if ((strlen(src) + 1) <= catRestSize) { \ + memcpy(catTmpDst, (src), strlen(src) + 1); \ + catRet = EOK; \ + } else { \ + catRet = ERANGE; \ + } \ + if (catRet != EOK) { \ + catRet = strcat_s((dest), (destMax), (src)); \ + } \ + } else { \ + catRet = strcat_s((dest), (destMax), (src)); \ + } \ + catRet; \ +}) +#else +#define SECUREC_STRCAT_SM(dest, destMax, src) strcat_s((dest), (destMax), (src)) +#endif + +/* for strncat_s performance optimization */ +#if defined(__GNUC__) +#define SECUREC_STRNCAT_SM(dest, destMax, src, count) ({ \ + int ncatRet = EOK; \ + if ((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN) && \ + (((unsigned long long)(count) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN)) { \ + char *ncatTmpDest = (char *)(dest); \ + size_t ncatRestSize = (size_t)(destMax); \ + while (ncatRestSize > 0 && *ncatTmpDest != '\0') { \ + ++ncatTmpDest; \ + --ncatRestSize; \ + } \ + if (ncatRestSize == 0) { \ + ncatRet = EINVAL; \ + } else if ((SECUREC_TWO_MIN((count), strlen(src)) + 1) <= ncatRestSize) { \ + if ((size_t)(count) < strlen(src)) { \ + memcpy(ncatTmpDest, (src), (count)); \ + *(ncatTmpDest + (count)) = '\0'; \ + } else { \ + memcpy(ncatTmpDest, (src), strlen(src) + 1); \ + } \ + } else { \ + ncatRet = ERANGE; \ + } \ + if (ncatRet != EOK) { \ + ncatRet = strncat_s((dest), (destMax), (src), (count)); \ + } \ + } else { \ + ncatRet = strncat_s((dest), (destMax), (src), (count)); \ + } \ + ncatRet; \ +}) +#else +#define SECUREC_STRNCAT_SM(dest, destMax, src, count) strncat_s((dest), (destMax), (src), (count)) +#endif + +/* SECUREC_MEMCPY_SM do NOT check buffer overlap by default */ +#define SECUREC_MEMCPY_SM(dest, destMax, src, count) \ + (!(((size_t)(destMax) == 0) || \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) > SECUREC_MEM_MAX_LEN) || \ + ((size_t)(count) > (size_t)(destMax)) || ((void *)(dest)) == NULL || ((void *)(src) == NULL))? \ + (memcpy((dest), (src), (count)), EOK) : \ + (memcpy_s((dest), (destMax), (src), (count)))) + +#define SECUREC_MEMSET_SM(dest, destMax, c, count) \ + (!(((size_t)(destMax) == 0) || \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) > SECUREC_MEM_MAX_LEN) || \ + ((void *)(dest) == NULL) || ((size_t)(count) > (size_t)(destMax))) ? \ + (memset((dest), (c), (count)), EOK) : \ + (memset_s((dest), (destMax), (c), (count)))) + +#endif +#endif /* __SECURECTYPE_H__A7BBB686_AADA_451B_B9F9_44DACDAE18A7 */ + diff --git a/third_party/securec/src/CMakeLists.txt b/third_party/securec/src/CMakeLists.txt new file mode 100644 index 00000000000..60ec0a90ee8 --- /dev/null +++ b/third_party/securec/src/CMakeLists.txt @@ -0,0 +1,3 @@ +aux_source_directory(. SECUREC_SRCS) + +add_library(securec STATIC ${SECUREC_SRCS}) diff --git a/third_party/securec/src/fscanf_s.c b/third_party/securec/src/fscanf_s.c new file mode 100644 index 00000000000..8ceda9ac357 --- /dev/null +++ b/third_party/securec/src/fscanf_s.c @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The fscanf_s function is equivalent to fscanf except that the c, s, + * and [ conversion specifiers apply to a pair of arguments (unless assignment suppression is indicated by a*) + * The fscanf function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same + * form and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... The convered value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int fscanf_s(FILE *stream, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vfscanf_s(stream, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/fwscanf_s.c b/third_party/securec/src/fwscanf_s.c new file mode 100644 index 00000000000..f826b7db8a7 --- /dev/null +++ b/third_party/securec/src/fwscanf_s.c @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The fwscanf_s function is the wide-character equivalent of the fscanf_s function + * The fwscanf_s function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same + * form and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... The converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int fwscanf_s(FILE *stream, const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vfwscanf_s(stream, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/gets_s.c b/third_party/securec/src/gets_s.c new file mode 100644 index 00000000000..57fd6231acc --- /dev/null +++ b/third_party/securec/src/gets_s.c @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securecutil.h" + +static void SecTrimCRLF(char *buffer, size_t len) +{ + int i; + /* No need to determine whether integer overflow exists */ + for (i = (int)(len - 1); i >= 0 && (buffer[i] == '\r' || buffer[i] == '\n'); --i) { + buffer[i] = '\0'; + } + return; +} + +/* + * + * The gets_s function reads at most one less than the number of characters + * specified by destMax from the stream pointed to by stdin, into the array pointed to by buffer + * The line consists of all characters up to and including + * the first newline character ('\n'). gets_s then replaces the newline + * character with a null character ('\0') before returning the line. + * If the first character read is the end-of-file character, a null character + * is stored at the beginning of buffer and NULL is returned. + * + * + * buffer Storage location for input string. + * numberOfElements The size of the buffer. + * + * + * buffer is updated + * + * + * buffer Successful operation + * NULL Improper parameter or read fail + */ +char *gets_s(char *buffer, size_t numberOfElements) +{ + size_t len; +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + size_t bufferSize = ((numberOfElements == (size_t)-1) ? SECUREC_STRING_MAX_LEN : numberOfElements); +#else + size_t bufferSize = numberOfElements; +#endif + + if (buffer == NULL || bufferSize == 0 || bufferSize > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_PARAMTER("gets_s"); + return NULL; + } + + if (fgets(buffer, (int)bufferSize, stdin) == NULL) { + return NULL; + } + + len = strlen(buffer); + if (len > 0 && len < bufferSize) { + SecTrimCRLF(buffer, len); + } + + return buffer; +} + diff --git a/third_party/securec/src/input.inl b/third_party/securec/src/input.inl new file mode 100644 index 00000000000..a5a92e56cb7 --- /dev/null +++ b/third_party/securec/src/input.inl @@ -0,0 +1,2125 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INPUT_INL_5D13A042_DC3F_4ED9_A8D1_882811274C27 +#define INPUT_INL_5D13A042_DC3F_4ED9_A8D1_882811274C27 + +#if SECUREC_IN_KERNEL +#include +#ifndef EOF +#define EOF (-1) +#endif +#else +#if !defined(SECUREC_SYSAPI4VXWORKS) && !defined(SECUREC_CTYPE_MACRO_ADAPT) +#include +#ifdef SECUREC_FOR_WCHAR +#include /* for iswspace */ +#endif +#endif +#endif + +#define SECUREC_NUM_WIDTH_SHORT 0 +#define SECUREC_NUM_WIDTH_INT 1 +#define SECUREC_NUM_WIDTH_LONG 2 +#define SECUREC_NUM_WIDTH_LONG_LONG 3 /* also long double */ + +#define SECUREC_BUF_EXT_MUL 2 +#define SECUREC_BUFFERED_BLOK_SIZE 1024 + +#if defined(SECUREC_VXWORKS_PLATFORM) && !defined(va_copy) && !defined(__va_copy) +/* the name is the same as system macro. */ +#define __va_copy(d, s) do { \ + size_t size_of_d = (size_t)sizeof(d); \ + size_t size_of_s = (size_t)sizeof(s); \ + if (size_of_d != size_of_s) { \ + (void)memcpy((d), (s), sizeof(va_list)); \ + } else { \ + (void)memcpy(&(d), &(s), sizeof(va_list)); \ + } \ +} SECUREC_WHILE_ZERO +#endif + + +#define SECUREC_MULTI_BYTE_MAX_LEN 6 +/* Record a flag for each bit */ +#define SECUREC_BRACKET_INDEX(x) ((unsigned int)(x) >> 3) +#define SECUREC_BRACKET_VALUE(x) ((unsigned char)(1 << ((unsigned int)(x) & 7))) + + +/* Compatibility macro name cannot be modifie */ +#ifndef UNALIGNED +#if !(defined(_M_IA64)) && !(defined(_M_AMD64)) +#define UNALIGNED +#else +#define UNALIGNED __unaligned +#endif +#endif + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +/* Max 64bit value is 0xffffffffffffffff */ +#define SECUREC_MAX_64BITS_VALUE 18446744073709551615ULL +#define SECUREC_MAX_64BITS_VALUE_DIV_TEN 1844674407370955161ULL +#define SECUREC_MAX_64BITS_VALUE_CUT_LAST_DIGIT 18446744073709551610ULL +#define SECUREC_MIN_64BITS_NEG_VALUE 9223372036854775808ULL +#define SECUREC_MAX_64BITS_POS_VALUE 9223372036854775807ULL +#define SECUREC_MIN_32BITS_NEG_VALUE 2147483648ULL +#define SECUREC_MAX_32BITS_POS_VALUE 2147483647ULL +#define SECUREC_MAX_32BITS_VALUE 4294967295ULL +#define SECUREC_MAX_32BITS_VALUE_INC 4294967296ULL +#define SECUREC_MAX_32BITS_VALUE_DIV_TEN 429496729ULL +#define SECUREC_LONG_BIT_NUM ((unsigned int)(sizeof(long) << 3U)) + +#define SECUREC_LONG_HEX_BEYOND_MAX(number) (((number) >> (SECUREC_LONG_BIT_NUM - 4U)) > 0) +#define SECUREC_LONG_OCTAL_BEYOND_MAX(number) (((number) >> (SECUREC_LONG_BIT_NUM - 3U)) > 0) + +#define SECUREC_QWORD_HEX_BEYOND_MAX(number) (((number) >> (64U - 4U)) > 0) +#define SECUREC_QWORD_OCTAL_BEYOND_MAX(number) (((number) >> (64U - 3U)) > 0) + +#define SECUREC_LP64_BIT_WIDTH 64 +#define SECUREC_LP32_BIT_WIDTH 32 + +#endif + +#define SECUREC_CHAR(x) (x) +#define SECUREC_BRACE '{' /* [ to { */ + +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_SCANF_BRACKET_CONDITION(comChr, ch, table, mask) ((comChr) == SECUREC_BRACE && \ + (table) != NULL && \ + (((table)[((unsigned int)(int)(ch) & SECUREC_CHAR_MASK) >> 3] ^ (mask)) & \ + (1 << ((unsigned int)(int)(ch) & 7)))) +#else +#define SECUREC_SCANF_BRACKET_CONDITION(comChr, ch, table, mask) ((comChr) == SECUREC_BRACE && \ + (((table)[((unsigned char)(ch) & 0xff) >> 3] ^ (mask)) & (1 << ((unsigned char)(ch) & 7)))) +#endif +#define SECUREC_SCANF_STRING_CONDITION(comChr, ch) ((comChr) == SECUREC_CHAR('s') && \ + (!((ch) >= SECUREC_CHAR('\t') && (ch) <= SECUREC_CHAR('\r')) && (ch) != SECUREC_CHAR(' '))) + +/* Do not use |= optimize this code, it will cause compiling warning */ +/* only supports wide characters with a maximum length of two bytes */ +#define SECUREC_BRACKET_SET_BIT(table, ch) do { \ + unsigned int tableIndex = SECUREC_BRACKET_INDEX(((unsigned int)(int)(ch) & SECUREC_CHAR_MASK)); \ + unsigned int tableValue = SECUREC_BRACKET_VALUE(((unsigned int)(int)(ch) & SECUREC_CHAR_MASK)); \ + (table)[tableIndex] = (unsigned char)((table)[tableIndex] | tableValue); \ +} SECUREC_WHILE_ZERO + +#ifdef SECUREC_FOR_WCHAR +/* table size is 32 x 256 */ +#define SECUREC_BRACKET_TABLE_SIZE 8192 +#define SECUREC_EOF WEOF +#define SECUREC_MB_LEN 16 /* max. # bytes in multibyte char ,see MB_LEN_MAX */ +/* int to unsigned int clear e571 */ +#define SECUREC_IS_DIGIT(chr) (!((unsigned int)(int)(chr) & 0xff00) && isdigit(((unsigned int)(int)(chr) & 0x00ff))) +#define SECUREC_IS_XDIGIT(chr) (!((unsigned int)(int)(chr) & 0xff00) && isxdigit(((unsigned int)(int)(chr) & 0x00ff))) +#define SECUREC_IS_SPACE(chr) iswspace((wint_t)(int)(chr)) +#else +#define SECUREC_BRACKET_TABLE_SIZE 32 +#define SECUREC_EOF EOF +#define SECUREC_IS_DIGIT(chr) isdigit((unsigned char)(chr) & 0x00ff) +#define SECUREC_IS_XDIGIT(chr) isxdigit((unsigned char)(chr) & 0x00ff) +#define SECUREC_IS_SPACE(chr) isspace((unsigned char)(chr) & 0x00ff) +#endif + + +static SecInt SecSkipSpaceChar(SecFileStream *stream, int *counter); +static SecInt SecGetChar(SecFileStream *stream, int *counter); +static void SecUnGetChar(SecInt ch, SecFileStream *stream, int *counter); + +typedef struct { +#ifdef SECUREC_FOR_WCHAR + unsigned char *table; /* default NULL */ +#else + unsigned char table[SECUREC_BRACKET_TABLE_SIZE]; /* Array length is large enough in application scenarios */ +#endif + unsigned char mask; /* default 0 */ +} SecBracketTable; + +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_INIT_BRACKET_TABLE { NULL, 0 } +#else +#define SECUREC_INIT_BRACKET_TABLE { { 0 }, 0 } +#endif + +#if SECUREC_ENABLE_SCANF_FLOAT +typedef struct { + size_t floatStrSize; /* tialization must be length of buffer in charater */ + size_t floatStrUsedLen; /* store float string len */ + SecChar buffer[SECUREC_FLOAT_BUFSIZE + 1]; + SecChar *floatStr; /* Initialization must point to buffer */ + SecChar *allocatedFloatStr; /* Initialization must be NULL to store alloced point */ +} SecFloatSpec; +#endif + +typedef struct { + SecUnsignedInt64 number64; + unsigned long number; + int numberWidth; /* 0 = SHORT, 1 = int, > 1 long or L_DOUBLE */ + int isInt64Arg; /* 1 for 64-bit integer, 0 otherwise */ + int negative; /* 0 is positive */ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + int beyondMax; /* Non-zero means beyond */ +#endif + void *argPtr; /* Variable parameter pointer */ + size_t arrayWidth; /* length of pointer Variable parameter, in charaters */ + int width; /* width number in format */ + int widthSet; /* 0 is not set width in format */ + int comChr; /* Lowercase format conversion characters */ + int oriComChr; /* store number conversion */ + signed char isWChar; /* -1/0 not wchar, 1 for wchar */ + char suppress; /* 0 is not have %* in format */ +} SecScanSpec; + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +#define SECUREC_INIT_NUMBER_SPEC { 0, 0, 0, 0, 0, 0, NULL, 0, 0, 0, 0, 0, 0 } +#else +#define SECUREC_INIT_NUMBER_SPEC { 0, 0, 0, 0, 0, 0, NULL, 0, 0, 0, 0, 0 } +#endif + +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_GETC fgetwc +#define SECUREC_UN_GETC ungetwc +#define SECUREC_CHAR_MASK 0xffff +#else +#define SECUREC_GETC fgetc +#define SECUREC_UN_GETC ungetc +#define SECUREC_CHAR_MASK 0xff +#endif + +/* + * Determine if it is a 64-bit pointer function + * return 0 is not ,1 is 64bit pointer + */ +static int SecIs64BitPtr(size_t sizeOfVoidStar) +{ + /* point size is 4 or 8 , Under the 64 bit system, the value not 0 */ + /* to clear e778 */ + if ((sizeOfVoidStar & sizeof(SecInt64)) != 0) { + return 1; + } + return 0; +} + +#if SECUREC_ENABLE_SCANF_FLOAT + +/* + * Convert a floating point string to a floating point number + */ +static void SecAssignFloat(const char *floatStr, int numberWidth, void *argPtr) +{ + char *endPtr = NULL; + double d; +#if SECUREC_SUPPORT_STRTOLD + if (numberWidth == SECUREC_NUM_WIDTH_LONG_LONG) { + long double d2 = strtold(floatStr, &endPtr); + *(long double UNALIGNED *)(argPtr) = d2; + return; + } +#endif + d = strtod(floatStr, &endPtr); + if (numberWidth > SECUREC_NUM_WIDTH_INT) { + *(double UNALIGNED *)(argPtr) = (double)d; + } else { + *(float UNALIGNED *)(argPtr) = (float)d; + } +} + +#ifdef SECUREC_FOR_WCHAR +/* + * Convert a floating point wchar string to a floating point number + * Success ret 0 + */ +static int SecAssignFloatW(const SecFloatSpec *floatSpec, const SecScanSpec *spec) +{ + /* convert float string */ + size_t mbsLen; + size_t tempFloatStrLen = (size_t)(floatSpec->floatStrSize + 1) * sizeof(wchar_t); + char *tempFloatStr = (char *)SECUREC_MALLOC(tempFloatStrLen); + + if (tempFloatStr == NULL) { + return -1; + } + tempFloatStr[0] = '\0'; + SECUREC_MASK_MSVC_CRT_WARNING + mbsLen = wcstombs(tempFloatStr, floatSpec->floatStr, tempFloatStrLen - 1); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (mbsLen != (size_t)-1) { + tempFloatStr[mbsLen] = '\0'; + SecAssignFloat(tempFloatStr, spec->numberWidth, spec->argPtr); + } else { + SECUREC_FREE(tempFloatStr); + return -1; + } + SECUREC_FREE(tempFloatStr); + return 0; +} +#endif +/* + * Splice floating point string + * return 0 OK + */ +static int SecUpdateFloatString(SecChar ch, SecFloatSpec *floatSpec) +{ + floatSpec->floatStr[floatSpec->floatStrUsedLen++] = ch; /* ch must be '0' - '9' */ + if (floatSpec->floatStrUsedLen < floatSpec->floatStrSize) { + return 0; + } + if (floatSpec->allocatedFloatStr == NULL) { + /* add 1 to clear ZERO LENGTH ALLOCATIONS warning */ + size_t oriBufSize = floatSpec->floatStrSize* (SECUREC_BUF_EXT_MUL * sizeof(SecChar)) + 1; + void *tmpPointer = (void *)SECUREC_MALLOC(oriBufSize); + if (tmpPointer == NULL) { + return -1; + } + if (memcpy_s(tmpPointer, oriBufSize, floatSpec->floatStr, floatSpec->floatStrSize * sizeof(SecChar)) != EOK) { + SECUREC_FREE(tmpPointer); /* This is a dead code, just to meet the coding requirements */ + return -1; + } + floatSpec->floatStr = (SecChar *) (tmpPointer); + floatSpec->allocatedFloatStr = (SecChar *) (tmpPointer); /* use to clear free on stack warning */ + floatSpec->floatStrSize *= SECUREC_BUF_EXT_MUL; /* this is OK, oriBufSize plus 1 just clear warning */ + return 0; + } else { + /* LSD 2014.3.6 fix, replace realloc to malloc to avoid heap injection */ + size_t oriBufSize = floatSpec->floatStrSize * sizeof(SecChar); + size_t nextSize = (oriBufSize * SECUREC_BUF_EXT_MUL) + 1; /* add 1 to clear satic check tool warning */ + /* Prevents integer overflow when calculating the wide character length. + * The maximum length of SECUREC_MAX_WIDTH_LEN is enough + */ + if (nextSize <= SECUREC_MAX_WIDTH_LEN) { + void *tmpPointer = (void *)SECUREC_MALLOC(nextSize); + if (tmpPointer == NULL) { + return -1; + } + if (memcpy_s(tmpPointer, nextSize, floatSpec->floatStr, oriBufSize) != EOK) { + SECUREC_FREE(tmpPointer); /* This is a dead code, just to meet the coding requirements */ + return -1; + } + if (memset_s(floatSpec->floatStr, oriBufSize, 0, oriBufSize) != EOK) { + SECUREC_FREE(tmpPointer); /* This is a dead code, just to meet the coding requirements */ + return -1; + } + SECUREC_FREE(floatSpec->floatStr); + + floatSpec->floatStr = (SecChar *) (tmpPointer); + floatSpec->allocatedFloatStr = (SecChar *) (tmpPointer); /* use to clear free on stack warning */ + floatSpec->floatStrSize *= SECUREC_BUF_EXT_MUL; /* this is OK, oriBufSize plus 1 just clear warning */ + return 0; + } + } + return -1; +} +#endif + +#ifndef SECUREC_FOR_WCHAR +/* LSD only multi-bytes string need isleadbyte() function */ +static int SecIsLeadByte(SecInt ch) +{ + unsigned int c = (unsigned int)ch; +#if !(defined(_MSC_VER) || defined(_INC_WCTYPE)) + return (int)(c & 0x80); +#else + return (int)isleadbyte((int)(c & 0xff)); +#endif +} +#endif + +/* + * Parsing whether it is a wide character + */ +static void SecUpdateWcharFlagByType(SecUnsignedChar ch, SecScanSpec *spec) +{ +#if defined(SECUREC_FOR_WCHAR) && (defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + signed char flagForUpperType = -1; + signed char flagForLowerType = 1; +#else + signed char flagForUpperType = 1; + signed char flagForLowerType = -1; +#endif + /* if no l or h flag */ + if (spec->isWChar == 0) { + if ((ch == SECUREC_CHAR('C')) || (ch == SECUREC_CHAR('S'))) { + spec->isWChar = flagForUpperType; + } else { + spec->isWChar = flagForLowerType; + } + } + return; +} +/* + * decode %l %ll + */ +static void SecDecodeScanQualifierL(const SecUnsignedChar **format, SecScanSpec *spec) +{ + const SecUnsignedChar *fmt = *format; + if (*(fmt + 1) == SECUREC_CHAR('l')) { + spec->isInt64Arg = 1; + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; + ++fmt; + } else { + spec->numberWidth = SECUREC_NUM_WIDTH_LONG; +#if defined(SECUREC_ON_64BITS) && !(defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + /* on window 64 system sizeof long is 32bit */ + spec->isInt64Arg = 1; +#endif + spec->isWChar = 1; + } + *format = fmt; +} + +/* + * decode %I %I43 %I64 %Id %Ii %Io ... + * set finishFlag to 1 finish Flag + */ +static void SecDecodeScanQualifierI(const SecUnsignedChar **format, SecScanSpec *spec, int *finishFlag) +{ + const SecUnsignedChar *fmt = *format; + if ((*(fmt + 1) == SECUREC_CHAR('6')) && + (*(fmt + 2) == SECUREC_CHAR('4'))) { /* offset 2 for I64 */ + spec->isInt64Arg = 1; + *format = *format + 2; /* add 2 to skip I64 point to '4' next loop will inc */ + } else if ((*(fmt + 1) == SECUREC_CHAR('3')) && + (*(fmt + 2) == SECUREC_CHAR('2'))) { /* offset 2 for I32 */ + *format = *format + 2; /* add 2 to skip I32 point to '2' next loop will inc */ + } else if ((*(fmt + 1) == SECUREC_CHAR('d')) || + (*(fmt + 1) == SECUREC_CHAR('i')) || + (*(fmt + 1) == SECUREC_CHAR('o')) || + (*(fmt + 1) == SECUREC_CHAR('x')) || + (*(fmt + 1) == SECUREC_CHAR('X'))) { + spec->isInt64Arg = SecIs64BitPtr(sizeof(void *)); + } else { + /* for %I */ + spec->isInt64Arg = SecIs64BitPtr(sizeof(void *)); + *finishFlag = 1; + } +} + +static int SecDecodeScanWidth(const SecUnsignedChar **format, SecScanSpec *spec) +{ + const SecUnsignedChar *fmt = *format; + while (SECUREC_IS_DIGIT(*fmt)) { + spec->widthSet = 1; + if (SECUREC_MUL_TEN_ADD_BEYOND_MAX(spec->width)) { + return -1; + } + spec->width = (int)SECUREC_MUL_TEN((unsigned int)spec->width) + (unsigned char)(*fmt - SECUREC_CHAR('0')); + ++fmt; + } + *format = fmt; + return 0; +} + +/* + * init default flags for each format + */ +static void SecSetDefaultScanSpec(SecScanSpec *spec) +{ + spec->number64 = 0; + spec->number = 0; + spec->numberWidth = SECUREC_NUM_WIDTH_INT; /* 0 = SHORT, 1 = int, > 1 long or L_DOUBLE */ + spec->isInt64Arg = 0; /* 1 for 64-bit integer, 0 otherwise */ + spec->negative = 0; +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + spec->beyondMax = 0; +#endif + spec->argPtr = NULL; + spec->arrayWidth = 0; + spec->width = 0; + spec->widthSet = 0; + spec->comChr = 0; + spec->isWChar = 0; + spec->suppress = 0; +} + +/* + * decode qualifier %I %L %h ... + * set finishFlag to 1 finish Flag + */ +static void SecDecodeScanQualifier(const SecUnsignedChar **format, SecScanSpec *spec, int *finishFlag) +{ + switch ((int)(unsigned char)(**(format))) { + case SECUREC_CHAR('F'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('N'): + break; + case SECUREC_CHAR('h'): + --spec->numberWidth; /* h for SHORT , hh for CHAR */ + spec->isWChar = -1; + break; +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + case SECUREC_CHAR('j'): + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; /* intmax_t or uintmax_t */ + spec->isInt64Arg = 1; + break; + case SECUREC_CHAR('t'): /* fall-through */ /* FALLTHRU */ +#endif + case SECUREC_CHAR('z'): +#ifdef SECUREC_ON_64BITS + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; + spec->isInt64Arg = 1; +#else + spec->numberWidth = SECUREC_NUM_WIDTH_LONG; +#endif + break; + case SECUREC_CHAR('L'): /* long double */ /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('q'): + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; + spec->isInt64Arg = 1; + break; + case SECUREC_CHAR('l'): + SecDecodeScanQualifierL(format, spec); + break; + case SECUREC_CHAR('w'): + spec->isWChar = 1; + break; + case SECUREC_CHAR('*'): + spec->suppress = 1; + break; + case SECUREC_CHAR('I'): + SecDecodeScanQualifierI(format, spec, finishFlag); + break; + default: + *finishFlag = 1; + break; + } + +} +/* + * decode width and qualifier in format + */ +static int SecDecodeScanFlag(const SecUnsignedChar **format, SecScanSpec *spec) +{ + const SecUnsignedChar *fmt = *format; + int finishFlag = 0; + + do { + ++fmt; /* first skip % , next seek fmt */ + /* may %*6d , so put it inside the loop */ + if (SecDecodeScanWidth(&fmt, spec) != 0) { + return -1; + } + SecDecodeScanQualifier(&fmt, spec, &finishFlag); + } while (finishFlag == 0); + *format = fmt; + return 0; +} + + + + + +/* + * Judging whether a zeroing buffer is needed according to different formats + */ +static int SecDecodeClearFormat(const SecUnsignedChar *format, int *comChr) +{ + const SecUnsignedChar *fmt = format; + /* to lowercase */ + int ch = (unsigned char)(*fmt) | (SECUREC_CHAR('a') - SECUREC_CHAR('A')); + if (!(ch == SECUREC_CHAR('c') || ch == SECUREC_CHAR('s') || ch == SECUREC_BRACE)) { + return -1; /* first argument is not a string type */ + } + if (ch == SECUREC_BRACE) { +#if !(defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + if (*fmt == SECUREC_CHAR('{')) { + return -1; + } +#endif + ++fmt; + if (*fmt == SECUREC_CHAR('^')) { + ++fmt; + } + if (*fmt == SECUREC_CHAR(']')) { + ++fmt; + } + while ((*fmt != SECUREC_CHAR('\0')) && (*fmt != SECUREC_CHAR(']'))) { + ++fmt; + } + if (*fmt == SECUREC_CHAR('\0')) { + return -1; /* trunc'd format string */ + } + } + *comChr = ch; + return 0; +} + +/* + * add L'\0' for wchar string , add '\0' for char string + */ +static void SecAddEndingZero(void *ptr, const SecScanSpec *spec) +{ + *(char *)ptr = '\0'; + (void)spec; /* clear not use */ +#if SECUREC_HAVE_WCHART + if (spec->isWChar > 0) { + *(wchar_t UNALIGNED *)ptr = L'\0'; + } +#endif +} + +#ifdef SECUREC_FOR_WCHAR +/* + * Clean up the first %s %c buffer to zero for wchar version + */ +void SecClearDestBufW(const wchar_t *buffer, const wchar_t *format, va_list argList) +#else +/* + * Clean up the first %s %c buffer to zero for char version + */ +void SecClearDestBuf(const char *buffer, const char *format, va_list argList) +#endif +{ + + va_list argListSave; /* backup for argList value, this variable don't need initialized */ + SecScanSpec spec; + int comChr = 0; + const SecUnsignedChar *fmt = (const SecUnsignedChar *)format; + if (fmt == NULL) { + return; + } + + /* find first % */ + while (*fmt != SECUREC_CHAR('\0') && *fmt != SECUREC_CHAR('%')) { + ++fmt; + } + if (*fmt == SECUREC_CHAR('\0')) { + return; + } + + SecSetDefaultScanSpec(&spec); + if (SecDecodeScanFlag(&fmt, &spec) != 0) { + return; + } + + /* update wchar flag for %S %C */ + SecUpdateWcharFlagByType(*fmt, &spec); + + if (spec.suppress != 0 || SecDecodeClearFormat(fmt, &comChr) != 0) { + return; + } + + if ((buffer != NULL) && (*buffer != SECUREC_CHAR('\0')) && (comChr != SECUREC_CHAR('s'))) { + /* when buffer not empty just clear %s. + * example call sscanf by argment of (" \n", "%s", s, sizeof(s)) + */ + return; + } + (void)memset(&argListSave, 0, sizeof(va_list)); /* to clear e530 argListSave not initialized */ +#if defined(va_copy) + va_copy(argListSave, argList); +#elif defined(__va_copy) /* for vxworks */ + __va_copy(argListSave, argList); +#else + argListSave = argList; +#endif + do { + void *argPtr = (void *)va_arg(argListSave, void *); + /* Get the next argument - size of the array in characters */ + size_t arrayWidth = ((size_t)(va_arg(argListSave, size_t))) & 0xFFFFFFFFUL; + va_end(argListSave); + /* to clear e438 last value assigned not used , the compiler will optimize this code */ + (void)argListSave; + /* There is no need to judge the upper limit */ + if (arrayWidth == 0 || argPtr == NULL) { + return; + } + + /* clear one char */ + SecAddEndingZero(argPtr, &spec); + } SECUREC_WHILE_ZERO; + return; + +} + +/* + * Assign number to output buffer + */ +static void SecAssignNumber(const SecScanSpec *spec) +{ + void *argPtr = spec->argPtr; + if (spec->isInt64Arg != 0) { +#if defined(SECUREC_VXWORKS_PLATFORM) +#if defined(SECUREC_VXWORKS_PLATFORM_COMP) + *(SecInt64 UNALIGNED *)argPtr = (SecInt64)(spec->number64); +#else + /* take number64 as unsigned number unsigned to int clear Compile warning */ + *(SecInt64 UNALIGNED *)argPtr = *(SecUnsignedInt64 *)(&(spec->number64)); +#endif +#else + /* take number64 as unsigned number */ + *(SecInt64 UNALIGNED *)argPtr = (SecInt64)(spec->number64); +#endif + return; + } + if (spec->numberWidth > SECUREC_NUM_WIDTH_INT) { + /* take number as unsigned number */ + *(long UNALIGNED *)argPtr = (long)(spec->number); + } else if (spec->numberWidth == SECUREC_NUM_WIDTH_INT) { + *(int UNALIGNED *)argPtr = (int)(spec->number); + } else if (spec->numberWidth == SECUREC_NUM_WIDTH_SHORT) { + /* take number as unsigned number */ + *(short UNALIGNED *)argPtr = (short)(spec->number); + } else { /* < 0 for hh format modifier */ + /* take number as unsigned number */ + *(char UNALIGNED *)argPtr = (char)(spec->number); + } +} + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +/* + * Judge the long bit width + */ +static int SecIsLongBitEqual(int bitNum) +{ + return (unsigned int)bitNum == SECUREC_LONG_BIT_NUM; +} +#endif +/* + * Convert hexadecimal characters to decimal value + */ +static int SecHexValueOfChar(SecInt ch) +{ + /* use isdigt Causing tool false alarms */ + return (int)((ch >= '0' && ch <= '9') ? ((unsigned char)ch - '0') : + ((((unsigned char)ch | (unsigned char)('a' - 'A')) - ('a')) + 10)); /* Adding 10 is to hex value */ +} + + + +/* + * Parse decimal character to integer for 32bit . + */ +static void SecDecodeNumberDecimal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + unsigned long decimalEdge = SECUREC_MAX_32BITS_VALUE_DIV_TEN; +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + decimalEdge = (unsigned long)SECUREC_MAX_64BITS_VALUE_DIV_TEN; + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + decimalEdge = SECUREC_MAX_32BITS_VALUE_DIV_TEN; + } +#endif + if (spec->number > decimalEdge) { + spec->beyondMax = 1; + } +#endif + spec->number = SECUREC_MUL_TEN(spec->number); +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->number == SECUREC_MUL_TEN(decimalEdge)) { + SecUnsignedInt64 number64As = (unsigned long)SECUREC_MAX_64BITS_VALUE - spec->number; + if (number64As < (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0'))) { + spec->beyondMax = 1; + } + } +#endif + spec->number += (unsigned long)((SecUnsignedInt)ch - SECUREC_CHAR('0')); + +} + + +/* + * Parse Hex character to integer for 32bit . + */ +static void SecDecodeNumberHex(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_LONG_HEX_BEYOND_MAX(spec->number)) { + spec->beyondMax = 1; + } +#endif + spec->number = SECUREC_MUL_SIXTEEN(spec->number); + spec->number += (unsigned long)(unsigned int)SecHexValueOfChar(ch); +} + + +/* + * Parse Octal character to integer for 32bit . + */ +static void SecDecodeNumberOctal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_LONG_OCTAL_BEYOND_MAX(spec->number)) { + spec->beyondMax = 1; + } +#endif + spec->number = SECUREC_MUL_EIGHT(spec->number); + spec->number += (unsigned long)((SecUnsignedInt)ch - SECUREC_CHAR('0')); +} + + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +/* Compatible with integer negative values other than int */ +static void SecFinishNumberNegativeOther(int comChr, int numberWidth, SecScanSpec *spec) +{ + if ((comChr == SECUREC_CHAR('d')) || (comChr == SECUREC_CHAR('i'))) { + if (spec->number > (unsigned long)(1ULL << (SECUREC_LONG_BIT_NUM - 1))) { + spec->number = (unsigned long)(1ULL << (SECUREC_LONG_BIT_NUM - 1)); + } else { + spec->number = (unsigned long)(-(long)spec->number); + } + if (spec->beyondMax != 0) { + if (numberWidth < SECUREC_NUM_WIDTH_INT) { + spec->number = 0; + } else if (numberWidth == SECUREC_NUM_WIDTH_LONG) { + spec->number = ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1))); + } + } + } else { /* o, u, x, X, p */ + spec->number = (unsigned long)(-(long)spec->number); + if (spec->beyondMax != 0) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } +} +/* Compatible processing of integer negative numbers */ +static void SecFinishNumberNegativeInt(int comChr, SecScanSpec *spec) +{ + if ((comChr == SECUREC_CHAR('d')) || (comChr == SECUREC_CHAR('i'))) { +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + if ((spec->number > SECUREC_MIN_64BITS_NEG_VALUE)) { + spec->number = 0; + } else { + spec->number = (unsigned int)(-(int)spec->number); + } + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + if ((spec->number > SECUREC_MIN_32BITS_NEG_VALUE)) { + spec->number = SECUREC_MIN_32BITS_NEG_VALUE; + } else { + spec->number = (unsigned int)(-(int)spec->number); + } + } +#endif + if (spec->beyondMax != 0) { +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + spec->number = 0; + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + spec->number = SECUREC_MIN_32BITS_NEG_VALUE; + } +#endif + } + } else { /* o, u, x, X ,p */ +#ifdef SECUREC_ON_64BITS + if (spec->number > SECUREC_MAX_32BITS_VALUE_INC) { + spec->number = SECUREC_MAX_32BITS_VALUE; + } else { + spec->number = (unsigned int)(-(int)spec->number); + } +#else + spec->number = (unsigned int)(-(int)spec->number); +#endif + if (spec->beyondMax != 0) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } +} + +/* Compatible with integer positive values other than int */ +static void SecFinishNumberPositiveOther(int comChr, int numberWidth, SecScanSpec *spec) +{ + if (comChr == SECUREC_CHAR('d') || comChr == SECUREC_CHAR('i')) { + if (spec->number > ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1)) - 1)) { + spec->number = ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1)) - 1); + } + if ((spec->beyondMax != 0 && numberWidth < SECUREC_NUM_WIDTH_INT)) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + if (spec->beyondMax != 0 && numberWidth == SECUREC_NUM_WIDTH_LONG) { + spec->number = ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1)) - 1); + } + } else { + if (spec->beyondMax != 0) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } +} + +/* Compatible processing of integer positive numbers */ +static void SecFinishNumberPositiveInt(int comChr, SecScanSpec *spec) +{ + if ((comChr == SECUREC_CHAR('d')) || (comChr == SECUREC_CHAR('i'))) { +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + if (spec->number > SECUREC_MAX_64BITS_POS_VALUE) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } + if (spec->beyondMax != 0 && SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + if (spec->number > SECUREC_MAX_32BITS_POS_VALUE) { + spec->number = SECUREC_MAX_32BITS_POS_VALUE; + } + } + if (spec->beyondMax != 0 && SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + spec->number = SECUREC_MAX_32BITS_POS_VALUE; + } +#endif + } else { /* o,u,x,X,p */ + if (spec->beyondMax != 0) { + spec->number = SECUREC_MAX_32BITS_VALUE; + } + } +} + +#endif + + +/* + * Parse decimal character to integer for 64bit . + */ +static void SecDecodeNumber64Decimal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->number64 > SECUREC_MAX_64BITS_VALUE_DIV_TEN) { + spec->beyondMax = 1; + } +#endif + spec->number64 = SECUREC_MUL_TEN(spec->number64); +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->number64 == SECUREC_MAX_64BITS_VALUE_CUT_LAST_DIGIT) { + SecUnsignedInt64 number64As = (SecUnsignedInt64)SECUREC_MAX_64BITS_VALUE - spec->number64; + if (number64As < (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0'))) { + spec->beyondMax = 1; + } + } +#endif + spec->number64 += (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0')); +} + +/* + * Parse Hex character to integer for 64bit . + */ +static void SecDecodeNumber64Hex(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_QWORD_HEX_BEYOND_MAX(spec->number64)) { + spec->beyondMax = 1; + } +#endif + spec->number64 = SECUREC_MUL_SIXTEEN(spec->number64); + spec->number64 += (SecUnsignedInt64)(unsigned int)SecHexValueOfChar(ch); + +} + +/* + * Parse Octal character to integer for 64bit . + */ +static void SecDecodeNumber64Octal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_QWORD_OCTAL_BEYOND_MAX(spec->number64)) { + spec->beyondMax = 1; + } +#endif + spec->number64 = SECUREC_MUL_EIGHT(spec->number64); + spec->number64 += (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0')); +} + +#define SECUREC_DECODE_NUMBER_FUNC_NUM 2 +/* Function name cannot add address symbol, causing 546 alarm */ +static void (*g_secDecodeNumberHex[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecInt ch, SecScanSpec *spec) = \ + { SecDecodeNumberHex, SecDecodeNumber64Hex }; +static void (*g_secDecodeNumberOctal[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecInt ch, SecScanSpec *spec) = \ + { SecDecodeNumberOctal, SecDecodeNumber64Octal }; +static void (*g_secDecodeNumberDecimal[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecInt ch, SecScanSpec *spec) = \ + { SecDecodeNumberDecimal, SecDecodeNumber64Decimal }; + +/* + * Parse 64-bit integer formatted input, return 0 when ch is a number. + */ +static int SecDecodeNumber(SecInt ch, SecScanSpec *spec) +{ + if (spec->comChr == SECUREC_CHAR('x') || spec->comChr == SECUREC_CHAR('p')) { + if (SECUREC_IS_XDIGIT(ch)) { + (*g_secDecodeNumberHex[spec->isInt64Arg])(ch, spec); + } else { + return -1; + } + return 0; + } + if (!(SECUREC_IS_DIGIT(ch))) { + return -1; + } + if (spec->comChr == SECUREC_CHAR('o')) { + if (ch < SECUREC_CHAR('8')) { + (*g_secDecodeNumberOctal[spec->isInt64Arg])(ch, spec); + } else { + return -1; + } + } else { /* comChr is 'd' */ + (*g_secDecodeNumberDecimal[spec->isInt64Arg])(ch, spec); + } + return 0; +} + + +/* + * Complete the final 32-bit integer formatted input + */ +static void SecFinishNumber(SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->negative != 0) { + if (spec->numberWidth == SECUREC_NUM_WIDTH_INT) { + SecFinishNumberNegativeInt(spec->oriComChr, spec); + } else { + SecFinishNumberNegativeOther(spec->oriComChr, spec->numberWidth, spec); + } + } else { + if (spec->numberWidth == SECUREC_NUM_WIDTH_INT) { + SecFinishNumberPositiveInt(spec->oriComChr, spec); + } else { + SecFinishNumberPositiveOther(spec->oriComChr, spec->numberWidth, spec); + } + } +#else + if (spec->negative != 0) { +#if defined(__hpux) + if (spec->oriComChr != SECUREC_CHAR('p')) { + spec->number = (unsigned long)(-(long)spec->number); + } +#else + spec->number = (unsigned long)(-(long)spec->number); +#endif + } +#endif + return; +} + +/* + * Complete the final 64-bit integer formatted input + */ +static void SecFinishNumber64(SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->negative != 0) { + if (spec->oriComChr == (SECUREC_CHAR('d')) || (spec->oriComChr == SECUREC_CHAR('i'))) { + if (spec->number64 > SECUREC_MIN_64BITS_NEG_VALUE) { + spec->number64 = SECUREC_MIN_64BITS_NEG_VALUE; + } else { + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); + } + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MIN_64BITS_NEG_VALUE; + } + } else { /* o, u, x, X, p */ + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MAX_64BITS_VALUE; + } + } + } else { + if ((spec->oriComChr == SECUREC_CHAR('d')) || (spec->oriComChr == SECUREC_CHAR('i'))) { + if (spec->number64 > SECUREC_MAX_64BITS_POS_VALUE) { + spec->number64 = SECUREC_MAX_64BITS_POS_VALUE; + } + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MAX_64BITS_POS_VALUE; + } + } else { + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MAX_64BITS_VALUE; + } + } + } +#else + if (spec->negative != 0) { +#if defined(__hpux) + if (spec->oriComChr != SECUREC_CHAR('p')) { + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); + } +#else + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); +#endif + } +#endif + return; +} +static void (*g_secFinishNumber[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecScanSpec *spec) = \ + { SecFinishNumber, SecFinishNumber64 }; + +#if SECUREC_ENABLE_SCANF_FILE + +/* + * Adjust the pointer position of the file stream + */ +static void SecSeekStream(SecFileStream *stream) +{ + if ((stream->count == 0) && feof(stream->pf)) { + /* file pointer at the end of file, don't need to seek back */ + stream->base[0] = '\0'; + return; + } + /* LSD seek to original position, bug fix 2014 1 21 */ + if (fseek(stream->pf, stream->oriFilePos, SEEK_SET)) { + /* seek failed, ignore it */ + stream->oriFilePos = 0; + return; + } + + if (stream->fileRealRead > 0) { /* LSD bug fix. when file reach to EOF, don't seek back */ +#if (defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + int loops; + for (loops = 0; loops < (stream->fileRealRead / SECUREC_BUFFERED_BLOK_SIZE); ++loops) { + if (fread(stream->base, (size_t)1, (size_t)SECUREC_BUFFERED_BLOK_SIZE, + stream->pf) != SECUREC_BUFFERED_BLOK_SIZE) { + break; + } + } + if ((stream->fileRealRead % SECUREC_BUFFERED_BLOK_SIZE) != 0) { + size_t ret = fread(stream->base, (size_t)((unsigned int)stream->fileRealRead % SECUREC_BUFFERED_BLOK_SIZE), + (size_t)1, stream->pf); + if ((ret == 1 || ret == 0) && (ftell(stream->pf) < stream->oriFilePos + stream->fileRealRead)) { + (void)fseek(stream->pf, stream->oriFilePos + stream->fileRealRead, SEEK_SET); + } + } + +#else + /* in linux like system */ + if (fseek(stream->pf, stream->oriFilePos + stream->fileRealRead, SEEK_SET)) { + /* seek failed, ignore it */ + stream->oriFilePos = 0; + } +#endif + } + + return; +} + +/* + * Adjust the pointer position of the file stream and free memory + */ +static void SecAdjustStream(SecFileStream *stream) +{ + if (stream != NULL && (stream->flag & SECUREC_FILE_STREAM_FLAG) && stream->base != NULL) { + SecSeekStream(stream); + SECUREC_FREE(stream->base); + stream->base = NULL; + } + return; +} +#endif + +static void SecSkipSpaceFormat(const SecUnsignedChar **format) +{ + const SecUnsignedChar *fmt = *format; + while (SECUREC_IS_SPACE(*fmt)) { + ++fmt; + } + *format = fmt; +} +#ifndef SECUREC_FOR_WCHAR +/* + * Handling multi-character characters + */ +static int SecDecodeLeadByte(SecInt ch, const SecUnsignedChar **format, SecFileStream *stream, int *counter) +{ +#if SECUREC_HAVE_MBTOWC + char temp[SECUREC_MULTI_BYTE_MAX_LEN]; + const SecUnsignedChar *fmt = *format; + wchar_t tempWChar = L'\0'; + int ch2 = SecGetChar(stream, counter); + if (*fmt == SECUREC_CHAR('\0') || (int)(*fmt) != (ch2)) { + /* LSD in console mode, ungetc twice may cause problem */ + SecUnGetChar(ch2, stream, counter); + SecUnGetChar(ch, stream, counter); + return -1; + } + ++fmt; + if (MB_CUR_MAX >= SECUREC_UTF8_BOM_HEADER_SIZE && + (((unsigned char)ch & SECUREC_UTF8_LEAD_1ST) == SECUREC_UTF8_LEAD_1ST) && + (((unsigned char)ch2 & SECUREC_UTF8_LEAD_2ND) == SECUREC_UTF8_LEAD_2ND)) { + /* this char is very likely to be a UTF-8 char */ + int ch3 = SecGetChar(stream, counter); + temp[0] = (char)ch; + temp[1] = (char)ch2; /* 1 index of second character */ + temp[2] = (char)ch3; /* 2 index of third character */ + temp[3] = '\0'; /* 3 of string terminator position */ + + if (mbtowc(&tempWChar, temp, sizeof(temp)) > 0) { + /* succeed */ + if (*fmt == SECUREC_CHAR('\0') || (int)(*fmt) != (int)ch3) { + SecUnGetChar(ch3, stream, counter); + return -1; + } + ++fmt; + *counter = *counter - 1; + } else { + SecUnGetChar(ch3, stream, counter); + } + } + *counter = *counter - 1; /* only count as one character read */ + *format = fmt; + return 0; +#else + SecUnGetChar(ch, stream, counter); + (void)format; + return -1; +#endif +} +#endif + + + +/* + * Resolving sequence of characters from %[ format + */ +static int SecSetupBracketTable(const SecUnsignedChar **format, SecBracketTable *bracketTable) +{ + const SecUnsignedChar *fmt = *format; + SecUnsignedChar prevChar = 0; + SecUnsignedChar expCh; + SecUnsignedChar last = 0; +#if !(defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + if (*fmt == SECUREC_CHAR('{')) { + return -1; + } +#endif + /* for building "table" data */ + ++fmt; /* skip [ */ + bracketTable->mask = 0; + if (*fmt == SECUREC_CHAR('^')) { + ++fmt; + bracketTable->mask = (unsigned char)0xff; + } + if (*fmt == SECUREC_CHAR(']')) { + prevChar = SECUREC_CHAR(']'); + ++fmt; + SECUREC_BRACKET_SET_BIT(bracketTable->table, SECUREC_CHAR(']')); + } + while (*fmt != SECUREC_CHAR('\0') && *fmt != SECUREC_CHAR(']')) { + expCh = *fmt++; + if (expCh != SECUREC_CHAR('-') || prevChar == 0 || *fmt == SECUREC_CHAR(']')) { + /* normal character */ + prevChar = expCh; + SECUREC_BRACKET_SET_BIT(bracketTable->table, expCh); + } else { + /* for %[a-z] */ + expCh = *fmt++; /* get end of range */ + if (prevChar < expCh) { /* %[a-z] */ + last = expCh; + } else { + prevChar = expCh; +#if (defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + /* %[z-a] */ + last = prevChar; + +#else + SECUREC_BRACKET_SET_BIT(bracketTable->table, SECUREC_CHAR('-')); + SECUREC_BRACKET_SET_BIT(bracketTable->table, expCh); + continue; +#endif + } + /* format %[a-\xff] last is 0xFF, condition (rnch <= last) cause dead loop */ + for (expCh = prevChar; expCh < last; ++expCh) { + SECUREC_BRACKET_SET_BIT(bracketTable->table, expCh); + } + SECUREC_BRACKET_SET_BIT(bracketTable->table, last); + prevChar = 0; + } + } + *format = fmt; + return 0; +} + + +#ifdef SECUREC_FOR_WCHAR +static int SecInputForWchar(SecInt ch, SecScanSpec *spec) +{ + void *endPtr = spec->argPtr; + if (spec->isWChar > 0) { + *(wchar_t UNALIGNED *)endPtr = (wchar_t)ch; + endPtr = (wchar_t *)endPtr + 1; + --spec->arrayWidth; + } else { +#if SECUREC_HAVE_WCTOMB + int temp; + char tmpBuf[SECUREC_MB_LEN + 1]; + SECUREC_MASK_MSVC_CRT_WARNING temp = wctomb(tmpBuf, (wchar_t)ch); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (temp <= 0 || ((size_t)(unsigned int)temp) > sizeof(tmpBuf)) { + /* if wctomb error, then ignore character */ + return 0; + } + if (((size_t)(unsigned int)temp) > spec->arrayWidth) { + return -1; + } + if (memcpy_s(endPtr, spec->arrayWidth, tmpBuf, (size_t)(unsigned int)temp) != EOK) { + return -1; + } + endPtr = (char *)endPtr + temp; + spec->arrayWidth -= (size_t)(unsigned int)temp; +#else + return -1; +#endif + } + spec->argPtr = endPtr; + return 0; +} +#endif + + +#ifndef SECUREC_FOR_WCHAR +static int SecInputForChar(SecInt ch, SecScanSpec *spec, SecFileStream *stream, int *charCount) +{ + void *endPtr = spec->argPtr; + if (spec->isWChar > 0) { + wchar_t tempWChar = L'?'; /* set default char as ? */ +#if SECUREC_HAVE_MBTOWC + char temp[SECUREC_MULTI_BYTE_MAX_LEN + 1]; + temp[0] = (char)ch; + temp[1] = '\0'; +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + if (SecIsLeadByte(ch)) { + temp[1] = (char)SecGetChar(stream, charCount); + temp[2] = '\0'; /* 2 of string terminator position */ + } + if (mbtowc(&tempWChar, temp, sizeof(temp)) <= 0) { + /* no string termination error for tool */ + tempWChar = L'?'; + } +#else + if (SecIsLeadByte(ch)) { + int convRes = 0; + int di = 1; + /* in Linux like system, the string is encoded in UTF-8 */ + while (convRes <= 0 && di < (int)MB_CUR_MAX && di < SECUREC_MULTI_BYTE_MAX_LEN) { + temp[di++] = (char)SecGetChar(stream, charCount); + temp[di] = '\0'; + convRes = mbtowc(&tempWChar, temp, sizeof(temp)); + } + if (convRes <= 0) { + tempWChar = L'?'; + } + } else { + if (mbtowc(&tempWChar, temp, sizeof(temp)) <= 0) { + /* no string termination error for tool */ + tempWChar = L'?'; + } + } +#endif +#endif /* SECUREC_HAVE_MBTOWC */ + *(wchar_t UNALIGNED *)endPtr = tempWChar; + /* just copy L'?' if mbtowc fails, errno is set by mbtowc */ + endPtr = (wchar_t *)endPtr + 1; + --spec->arrayWidth; + (void)charCount; + (void)stream; + } else { + *(char *)endPtr = (char)ch; + endPtr = (char *)endPtr + 1; + --spec->arrayWidth; + } + spec->argPtr = endPtr; + return 0; +} +#endif + + +#if SECUREC_ENABLE_SCANF_FLOAT + +/* no not use localeconv()->decimal_pointif onlay support '.' */ +#define SECURE_IS_FLOAT_DECIMAL(ch) ((ch) == SECUREC_CHAR('.')) +/* + * init SecFloatSpec befor parse format + */ +static void SecInitFloatSpec(SecFloatSpec *floatSpec) +{ + floatSpec->floatStr = floatSpec->buffer; + floatSpec->allocatedFloatStr = NULL; + floatSpec->floatStrSize = sizeof(floatSpec->buffer) / sizeof(floatSpec->buffer[0]); + floatSpec->floatStr = floatSpec->buffer; + floatSpec->floatStrUsedLen = 0; +} + +static void SecClearFloatSpec(SecFloatSpec *floatSpec, int *doneCount) +{ + /* LSD 2014.3.6 add, clear the stack data */ + if (memset_s(floatSpec->buffer, sizeof(floatSpec->buffer), 0, + sizeof(floatSpec->buffer)) != EOK) { + *doneCount = 0; /* This is a dead code, just to meet the coding requirements */ + } + if (floatSpec->allocatedFloatStr != NULL) { + /* pFloatStr can be alloced in SecUpdateFloatString function, clear and free it */ + if (memset_s(floatSpec->allocatedFloatStr, floatSpec->floatStrSize * sizeof(SecChar), 0, + floatSpec->floatStrSize * sizeof(SecChar)) != EOK) { + *doneCount = 0; /* This is a dead code, just to meet the coding requirements */ + } + SECUREC_FREE(floatSpec->allocatedFloatStr); + floatSpec->allocatedFloatStr = NULL; + floatSpec->floatStr = NULL; + } +} + + +/* + * scan value of exponent. + * return 0 OK + */ +static int SecInputFloatE(SecFileStream *stream, SecScanSpec *spec, SecFloatSpec *floatSpec, int *charCount) +{ + SecInt ch = SecGetChar(stream, charCount); + if (ch == SECUREC_CHAR('+') || ch == SECUREC_CHAR('-')) { + if (ch == SECUREC_CHAR('-') && SecUpdateFloatString((SecChar)'-', floatSpec) != 0) { + return -1; + } + if (spec->width != 0) { + ch = SecGetChar(stream, charCount); + --spec->width; + } + } + + while (SECUREC_IS_DIGIT(ch) && spec->width-- != 0) { + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + } + return 0; +} + +/* + * scan %f. + * return 0 OK + */ +static int SecInputFloat(SecFileStream *stream, SecScanSpec *spec, SecFloatSpec *floatSpec, int *charCount) +{ + int started = -1; + SecInt ch = SecGetChar(stream, charCount); + + floatSpec->floatStrUsedLen = 0; + if (ch == SECUREC_CHAR('-')) { + floatSpec->floatStr[floatSpec->floatStrUsedLen++] = SECUREC_CHAR('-'); + --spec->width; + ch = SecGetChar(stream, charCount); + } else if (ch == SECUREC_CHAR('+')) { + --spec->width; + ch = SecGetChar(stream, charCount); + } + + if (spec->widthSet == 0) { /* must care width */ + spec->width = -1; /* -1 is unlimited */ + } + + /* now get integral part */ + while (SECUREC_IS_DIGIT(ch) && spec->width-- != 0) { + started = 0; + /* ch must be '0' - '9' */ + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + } + + /* now get fractional part */ + if (SECURE_IS_FLOAT_DECIMAL((SecChar)ch) && spec->width-- != 0) { + /* now check for decimal */ + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + while (SECUREC_IS_DIGIT(ch) && spec->width-- != 0) { + started = 0; + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + } + } + + /* now get exponent part */ + if (started == 0 && (ch == SECUREC_CHAR('e') || ch == SECUREC_CHAR('E')) && spec->width-- != 0) { + if (SecUpdateFloatString((SecChar)'e', floatSpec) != 0) { + return -1; + } + if (SecInputFloatE(stream, spec, floatSpec, charCount) != 0) { + return -1; + } + } + /* un set the last character that is not a floating point number */ + SecUnGetChar(ch, stream, charCount); + /* Make sure have a string terminator, buffer is large enough */ + floatSpec->floatStr[floatSpec->floatStrUsedLen] = SECUREC_CHAR('\0'); + return started; + +} +#endif + +/* + * scan digital part of %d %i %o %u %x %p. + * return 0 OK + */ +static int SecInputNumberDigital(SecInt firstCh, SecFileStream *stream, SecScanSpec *spec, int *charCount) +{ + SecInt ch = firstCh; + int loopFlag = 0; + int started = -1; + while (loopFlag == 0) { + /* decode ch to number */ + loopFlag = SecDecodeNumber(ch, spec); + if (loopFlag == 0) { + started = 0; + if (spec->widthSet != 0 && --spec->width == 0) { + loopFlag = 1; + } else { + ch = SecGetChar(stream, charCount); + } + } else { + SecUnGetChar(ch, stream, charCount); + } + } + + /* Handling integer negative numbers and beyond max */ + (*g_secFinishNumber[spec->isInt64Arg])(spec); + return started; + +} + +/* + * scan %d %i %o %u %x %p. + * return 0 OK + */ +static int SecInputNumber(SecFileStream *stream, SecScanSpec *spec, int *charCount) +{ + SecInt ch = SecGetChar(stream, charCount); + + if (ch == SECUREC_CHAR('+') || ch == SECUREC_CHAR('-')) { + if (ch == SECUREC_CHAR('-')) { + spec->negative = 1; + } + if (spec->widthSet != 0 && --spec->width == 0) { + return -1; + } else { + ch = SecGetChar(stream, charCount); + } + } + + if (spec->oriComChr == SECUREC_CHAR('i')) { + /* i could be d, o, or x, use d as default */ + spec->comChr = SECUREC_CHAR('d'); + } + + if (spec->oriComChr == SECUREC_CHAR('x') || spec->oriComChr == SECUREC_CHAR('i')) { + if (ch != SECUREC_CHAR('0')) { + /* scan number */ + return SecInputNumberDigital(ch, stream, spec, charCount); + } + /* now input string may be 0x123 or 0X123 or just 0 */ + /* get next char */ + ch = SecGetChar(stream, charCount); + if ((SecChar)(ch) == SECUREC_CHAR('x') || (SecChar)ch == SECUREC_CHAR('X')) { + spec->comChr = SECUREC_CHAR('x'); + ch = SecGetChar(stream, charCount); + /* length of 0x is 2 */ + if (spec->widthSet != 0 && spec->width <= (1 + 1)) { + /* length not enough for "0x" */ + return -1; + } + spec->width -= 2; /* Subtract 2 for the length of "0x" */ + } else { + if (spec->oriComChr != SECUREC_CHAR('x')) { + spec->comChr = SECUREC_CHAR('o'); + } + /* unset the character after 0 back to stream, input only '0' result is OK */ + SecUnGetChar(ch, stream, charCount); + ch = SECUREC_CHAR('0'); + } + } + /* scan number */ + return SecInputNumberDigital(ch, stream, spec, charCount); +} +/* + * scan %c %s %[ + * return 0 OK + */ +static int SecInputString(SecFileStream *stream, SecScanSpec *spec, + const SecBracketTable *bracketTable, int *charCount, int *doneCount) +{ + void *startPtr = spec->argPtr; + int suppressed= 0; + int errNoMem = 0; + + while (spec->widthSet == 0 || spec->width-- != 0) { + SecInt ch = SecGetChar(stream, charCount); + /* char condition or string condition and bracket condition. + * only supports wide characters with a maximum length of two bytes + */ + if ((ch != SECUREC_EOF) && (spec->comChr == SECUREC_CHAR('c') || + SECUREC_SCANF_STRING_CONDITION(spec->comChr, ch) || + SECUREC_SCANF_BRACKET_CONDITION(spec->comChr, ch, bracketTable->table, bracketTable->mask))) { + if (spec->suppress != 0) { + /* Used to identify processed data for %* + * use endPtr to identify will cause 613, so use suppressed + */ + suppressed = 1; + continue; + } + /* now suppress is not set */ + if (spec->arrayWidth == 0) { + errNoMem = 1; /* We have exhausted the user's buffer */ + break; + } +#ifdef SECUREC_FOR_WCHAR + errNoMem = SecInputForWchar(ch, spec); +#else + errNoMem = SecInputForChar(ch, spec, stream, charCount); +#endif + if (errNoMem != 0) { + break; + } + } else { + SecUnGetChar(ch, stream, charCount); + break; + } + } + + if (errNoMem != 0) { + /* In case of error, blank out the input buffer */ + if (spec->suppress == 0) { + SecAddEndingZero(startPtr, spec); + } + return -1; + } + + /* No input was scanned */ + if ((spec->suppress != 0 && suppressed == 0) || + (spec->suppress == 0 && startPtr == spec->argPtr)) { + return -1; + } + + if (spec->suppress == 0) { + if (spec->comChr != 'c') { + /* null-terminate strings */ + SecAddEndingZero(spec->argPtr, spec); + } + *doneCount = *doneCount + 1; + } + return 0; +} + +#ifdef SECUREC_FOR_WCHAR +/* + * alloce buffer for wchar version of %[. + * return 0 OK + */ +static int SecAllocBracketTable(SecBracketTable *bracketTable) +{ + if (bracketTable->table == NULL) { + /* table should be freed after use */ + bracketTable->table = (unsigned char *)SECUREC_MALLOC(SECUREC_BRACKET_TABLE_SIZE); + if (bracketTable->table == NULL) { + return -1; + } + } + return 0; +} + +/* + * free buffer for wchar version of %[ + */ +static void SecFreeBracketTable(SecBracketTable *bracketTable) +{ + if (bracketTable->table != NULL) { + SECUREC_FREE(bracketTable->table); + bracketTable->table = NULL; + } +} +#endif + +#ifdef SECUREC_FOR_WCHAR +/* + * Formatting input core functions for wchar version.Called by a function such as vsscanf_s + */ +int SecInputSW(SecFileStream *stream, const wchar_t *cFormat, va_list argList) +#else +/* + * Formatting input core functions for char version.Called by a function such as vswscanf_s + */ +int SecInputS(SecFileStream *stream, const char *cFormat, va_list argList) +#endif +{ + const SecUnsignedChar *format = (const SecUnsignedChar *)cFormat; + SecBracketTable bracketTable = SECUREC_INIT_BRACKET_TABLE; + SecScanSpec spec; + SecInt ch = 0; + int charCount = 0; + int doneCount = 0; + int formatError = 0; + int paraIsNull = 0; +#if SECUREC_ENABLE_SCANF_FLOAT + SecFloatSpec floatSpec; +#endif + int match = 0; + int errRet = 0; +#if SECUREC_ENABLE_SCANF_FLOAT + SecInitFloatSpec(&floatSpec); +#endif + /* format must not NULL */ + /* use err < 1 to claer 845 */ + while (errRet < 1 && *format != SECUREC_CHAR('\0')) { + /* skip space in format and space in input */ + if (SECUREC_IS_SPACE(*format)) { + SecInt nonSpaceChar = SecSkipSpaceChar(stream, &charCount); + /* eat all space chars and put fist no space char backup */ + SecUnGetChar(nonSpaceChar, stream, &charCount); + SecSkipSpaceFormat(&format); + continue; + } + + if (*format != SECUREC_CHAR('%')) { + ch = SecGetChar(stream, &charCount); + if ((int)(*format++) != (int)(ch)) { + SecUnGetChar(ch, stream, &charCount); + ++errRet; /* use plus to clear 845 */ + continue; + } +#ifndef SECUREC_FOR_WCHAR + if (SecIsLeadByte(ch) && SecDecodeLeadByte(ch, &format, stream, &charCount) != 0) { + ++errRet; + continue; + } +#endif + /* for next %n */ + if ((ch == SECUREC_EOF) && ((*format != SECUREC_CHAR('%')) || (*(format + 1) != SECUREC_CHAR('n')))) { + break; + } + continue; + } + + /* now *format is % */ + /* set default value for each % */ + SecSetDefaultScanSpec(&spec); + if (SecDecodeScanFlag(&format, &spec) != 0) { + formatError = 1; + ++errRet; + continue; + } + /* update wchar flag for %S %C */ + SecUpdateWcharFlagByType(*format, &spec); + +#if SECUREC_HAVE_WCHART == 0 + /* in kernel not support wide char */ + if (spec.isWChar > 0) { + formatError = 1; + ++errRet; + continue; + } +#endif + if (spec.widthSet != 0 && spec.width == 0) { + /* 0 width in format */ + ++errRet; + continue; + } + + spec.comChr = (unsigned char)(*format) | (SECUREC_CHAR('a') - SECUREC_CHAR('A')); /* to lowercase */ + spec.oriComChr = spec.comChr; + + if (spec.comChr != SECUREC_CHAR('n')) { + if (spec.comChr != SECUREC_CHAR('c') && spec.comChr != SECUREC_BRACE) { + ch = SecSkipSpaceChar(stream, &charCount); + } else { + ch = SecGetChar(stream, &charCount); + } + if (ch == SECUREC_EOF) { + ++errRet; + continue; + } + } + + /* now no 0 width in format and get one char from input */ + switch (spec.comChr) { + case SECUREC_CHAR('c'): /* also 'C' */ + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('s'): /* also 'S': */ + /* fall-through */ /* FALLTHRU */ + case SECUREC_BRACE: + /* check dest buffer and size */ + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + paraIsNull = 1; + ++errRet; + continue; + } + /* Get the next argument - size of the array in characters */ +#ifdef SECUREC_ON_64BITS + spec.arrayWidth = ((size_t)(va_arg(argList, size_t))) & 0xFFFFFFFFUL; +#else /* !SECUREC_ON_64BITS */ + spec.arrayWidth = (size_t)va_arg(argList, size_t); +#endif + if (spec.arrayWidth == 0 || (spec.isWChar <= 0 && spec.arrayWidth > SECUREC_STRING_MAX_LEN) || + (spec.isWChar > 0 && spec.arrayWidth > SECUREC_WCHAR_STRING_MAX_LEN)) { + /* do not clear buffer just go error */ + ++errRet; + continue; + } + /* One element is needed for '\0' for %s and %[ */ + if (spec.comChr != SECUREC_CHAR('c')) { + --spec.arrayWidth; + } + } else { + /* Set argPtr to NULL is necessary, in supress mode we don't use argPtr to store data */ + spec.argPtr = NULL; + } + + if (spec.comChr == 'c') { + if (spec.widthSet == 0) { + spec.widthSet = 1; + spec.width = 1; + } + } else if (spec.comChr == SECUREC_BRACE) { + /* malloc when first %[ is meet for wchar version */ +#ifdef SECUREC_FOR_WCHAR + if (SecAllocBracketTable(&bracketTable) != 0) { + ++errRet; + continue; + } + +#endif + (void)memset(bracketTable.table, 0, (size_t)SECUREC_BRACKET_TABLE_SIZE); + if (SecSetupBracketTable(&format, &bracketTable) != 0) { + ++errRet; + continue; + } + + if (*format == SECUREC_CHAR('\0')) { + if (spec.suppress == 0 && spec.arrayWidth > 0) { + SecAddEndingZero(spec.argPtr, &spec); + } + ++errRet; + /* truncated format */ + continue; + } + + } + /* un set last char to stream */ + SecUnGetChar(ch, stream, &charCount); + /* scanset completed. Now read string */ + if (SecInputString(stream, &spec, &bracketTable, &charCount, &doneCount) != 0) { + ++errRet; + continue; + } + break; + case SECUREC_CHAR('p'): + /* make %hp same as %p */ + spec.numberWidth = SECUREC_NUM_WIDTH_INT; +#ifdef SECUREC_ON_64BITS + spec.isInt64Arg = 1; +#endif + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('o'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('u'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('d'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('i'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('x'): + /* un set last char to stream */ + SecUnGetChar(ch, stream, &charCount); + if (SecInputNumber(stream, &spec, &charCount) != 0) { + ++errRet; + continue; + } + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + paraIsNull = 1; + ++errRet; + continue; + } + SecAssignNumber(&spec); + ++doneCount; + } + break; + case SECUREC_CHAR('n'): /* char count */ + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + paraIsNull = 1; + ++errRet; + continue; + } + spec.number = (unsigned long)(unsigned int)charCount; + spec.isInt64Arg = 0; + SecAssignNumber(&spec); + } + break; + case SECUREC_CHAR('e'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('f'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('g'): /* scan a float */ +#if SECUREC_ENABLE_SCANF_FLOAT + /* un set last char to stream */ + SecUnGetChar(ch, stream, &charCount); + if (SecInputFloat(stream, &spec, &floatSpec, &charCount) != 0) { + ++errRet; + continue; + } + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + ++errRet; + paraIsNull = 1; + continue; + } +#ifdef SECUREC_FOR_WCHAR + if (SecAssignFloatW(&floatSpec, &spec) != 0) { + ++errRet; + continue; + } +#else + SecAssignFloat(floatSpec.floatStr, spec.numberWidth, spec.argPtr); +#endif + ++doneCount; + } + + break; +#else /* SECUREC_ENABLE_SCANF_FLOAT */ + ++errRet; + continue; +#endif + default: + if ((int)(*format) != (int)ch) { + SecUnGetChar(ch, stream, &charCount); + formatError = 1; + ++errRet; + continue; + } else { + --match; + } + } + + ++match; + ++format; + if ((ch == SECUREC_EOF) && ((*format != SECUREC_CHAR('%')) || (*(format + 1) != SECUREC_CHAR('n')))) { + break; + } + } + +#ifdef SECUREC_FOR_WCHAR + SecFreeBracketTable(&bracketTable); +#endif + +#if SECUREC_ENABLE_SCANF_FLOAT + SecClearFloatSpec(&floatSpec, &doneCount); +#endif + +#if SECUREC_ENABLE_SCANF_FILE + SecAdjustStream(stream); +#endif + + if (ch == SECUREC_EOF) { + return ((doneCount || match) ? doneCount : SECUREC_SCANF_EINVAL); + } else if (formatError != 0 || paraIsNull != 0) { + /* Invalid Input Format or parameter */ + return SECUREC_SCANF_ERROR_PARA; + } + + return doneCount; +} + +#if SECUREC_ENABLE_SCANF_FILE + +#if defined(SECUREC_NO_STD_UNGETC) +/* + * Get char from stdin or buffer + */ +static SecInt SecGetCharFromStdin(SecFileStream *stream) +{ + SecInt ch; + if (stream->fUnget == 1) { + ch = (SecInt) stream->lastChar; + stream->fUnget = 0; + } else { + ch = SECUREC_GETC(stream->pf); + stream->lastChar = (unsigned int)ch; + } + return ch; +} +#else +/* + * Get char from stdin or buffer use std function + */ +static SecInt SecGetCharFromStdin(const SecFileStream *stream) +{ + SecInt ch; + ch = SECUREC_GETC(stream->pf); + return ch; +} +#endif + +static void SecSkipBomHeader(SecFileStream *stream) +{ +#ifdef SECUREC_FOR_WCHAR + if (stream->count >= SECUREC_BOM_HEADER_SIZE && + (((unsigned char)(stream->base[0]) == SECUREC_BOM_HEADER_LE_1ST && + (unsigned char)(stream->base[1]) == SECUREC_BOM_HEADER_LE_2ST) || + ((unsigned char)(stream->base[0]) == SECUREC_BOM_HEADER_BE_1ST && + (unsigned char)(stream->base[1]) == SECUREC_BOM_HEADER_BE_2ST))) { + + /* the stream->count must be a multiple of sizeof(SecChar), + * otherwise this function will return SECUREC_EOF when read the last character + */ + if ((stream->count - SECUREC_BOM_HEADER_SIZE) % (int)sizeof(SecChar) != 0) { + int ret = (int)fread(stream->base + stream->count, (size_t)1, + (size_t)SECUREC_BOM_HEADER_SIZE, stream->pf); + if (ret > 0 && ret <= SECUREC_BUFFERED_BLOK_SIZE) { + stream->count += ret; + } + } + /* it's BOM header, skip */ + stream->count -= SECUREC_BOM_HEADER_SIZE; + stream->cur += SECUREC_BOM_HEADER_SIZE; + } +#else + if (stream->count >= SECUREC_UTF8_BOM_HEADER_SIZE && + (unsigned char)(stream->base[0]) == SECUREC_UTF8_BOM_HEADER_1ST && + (unsigned char)(stream->base[1]) == SECUREC_UTF8_BOM_HEADER_2ND && + (unsigned char)(stream->base[2]) == SECUREC_UTF8_BOM_HEADER_3RD) { /* 2 offset of third head character */ + /* it's BOM header, skip */ + stream->count -= SECUREC_UTF8_BOM_HEADER_SIZE; + stream->cur += SECUREC_UTF8_BOM_HEADER_SIZE; + } +#endif +} +/* + * Get char from file stream or buffer + */ +static SecInt SecGetCharFromFile(SecFileStream *stream) +{ + SecInt ch; + if (stream->count == 0) { + int firstReadOnFile = 0; + /* load file to buffer */ + if (stream->base == NULL) { + stream->base = (char *)SECUREC_MALLOC(SECUREC_BUFFERED_BLOK_SIZE + 1); + if (stream->base == NULL) { + return SECUREC_EOF; + } + stream->base[SECUREC_BUFFERED_BLOK_SIZE] = '\0'; /* for tool Warning string null */ + } + /* LSD add 2014.3.21 */ + if (stream->oriFilePos == SECUREC_UNINITIALIZED_FILE_POS) { + stream->oriFilePos = ftell(stream->pf); /* save original file read position */ + firstReadOnFile = 1; + } + stream->count = (int)fread(stream->base, (size_t)1, (size_t)SECUREC_BUFFERED_BLOK_SIZE, stream->pf); + stream->base[SECUREC_BUFFERED_BLOK_SIZE] = '\0'; /* for tool Warning string null */ + if (stream->count == 0 || stream->count > SECUREC_BUFFERED_BLOK_SIZE) { + return SECUREC_EOF; + } + stream->cur = stream->base; + stream->flag |= SECUREC_LOAD_FILE_TO_MEM_FLAG; + if (firstReadOnFile != 0) { + SecSkipBomHeader(stream); + } + } + /* according wchar_t has two bytes */ + ch = (SecInt)((stream->count -= (int)sizeof(SecChar)) >= 0 ? \ + (SecInt)(SECUREC_CHAR_MASK & \ + (unsigned int)(int)(*((const SecChar *)(const void *)stream->cur))) : SECUREC_EOF); + stream->cur += sizeof(SecChar); + + if (ch != SECUREC_EOF && stream->base != NULL) { + stream->fileRealRead += (int)sizeof(SecChar); + } + return ch; +} +#endif + +/* + * Get char for wchar version + */ +static SecInt SecGetChar(SecFileStream *stream, int *counter) +{ + SecInt ch = SECUREC_EOF; +#if SECUREC_ENABLE_SCANF_FILE + if ((stream->flag & SECUREC_FROM_STDIN_FLAG) > 0) { + ch = SecGetCharFromStdin(stream); + } else if ((stream->flag & SECUREC_FILE_STREAM_FLAG) > 0) { + ch = SecGetCharFromFile(stream); + } +#endif + if ((stream->flag & SECUREC_MEM_STR_FLAG) > 0) { + /* according wchar_t has two bytes */ + ch = (SecInt)((stream->count -= (int)sizeof(SecChar)) >= 0 ? \ + (SecInt)(SECUREC_CHAR_MASK & \ + (unsigned int)(int)(*((const SecChar *)(const void *)stream->cur))) : SECUREC_EOF); + stream->cur += sizeof(SecChar); + } + *counter = *counter + 1; + return ch; +} + +/* + * Unget Public realizatio char for wchar and char version + */ +static void SecUnGetCharImpl(SecInt ch, SecFileStream *stream) +{ + if ((stream->flag & SECUREC_FROM_STDIN_FLAG) > 0) { +#if SECUREC_ENABLE_SCANF_FILE +#if defined(SECUREC_NO_STD_UNGETC) + stream->lastChar = (unsigned int)ch; + stream->fUnget = 1; +#else + (void)SECUREC_UN_GETC(ch, stream->pf); +#endif +#else + (void)ch; /* to clear e438 last value assigned not used , the compiler will optimize this code */ +#endif + } else if ((stream->flag & SECUREC_MEM_STR_FLAG) || (stream->flag & SECUREC_LOAD_FILE_TO_MEM_FLAG) > 0) { + if (stream->cur > stream->base) { + stream->cur -= sizeof(SecChar); + stream->count += (int)sizeof(SecChar); + } + } +#if SECUREC_ENABLE_SCANF_FILE + if ((stream->flag & SECUREC_FILE_STREAM_FLAG) > 0 && stream->base) { + stream->fileRealRead -= (int)sizeof(SecChar); + } +#endif +} + +/* + * Unget char for char version + */ +static void SecUnGetChar(SecInt ch, SecFileStream *stream, int *counter) +{ + if (ch != SECUREC_EOF) { + SecUnGetCharImpl(ch, stream); + } + *counter = *counter - 1; +} + +/* + * Skip space char by isspace + */ +static SecInt SecSkipSpaceChar(SecFileStream *stream, int *counter) +{ + SecInt ch; + do { + ch = SecGetChar(stream, counter); + } while (ch != SECUREC_EOF && SECUREC_IS_SPACE(ch)); + return ch; +} +#endif /* __INPUT_INL__5D13A042_DC3F_4ED9_A8D1_882811274C27 */ + diff --git a/third_party/securec/src/memcpy_s.c b/third_party/securec/src/memcpy_s.c new file mode 100644 index 00000000000..5eb100f400a --- /dev/null +++ b/third_party/securec/src/memcpy_s.c @@ -0,0 +1,577 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMCPY 1 +#include "securecutil.h" + +#ifndef SECUREC_MEMCOPY_WITH_PERFORMANCE +#define SECUREC_MEMCOPY_WITH_PERFORMANCE 0 +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMCOPY_WITH_PERFORMANCE +#ifndef SECUREC_MEMCOPY_THRESHOLD_SIZE +#define SECUREC_MEMCOPY_THRESHOLD_SIZE 64UL +#endif +/* + * Determine whether the address is 8-byte aligned, use static to increase performance + * return 0 is aligned + */ +static int SecIsAddrAligned8(const void *addr, const void *zeroAddr) +{ + return (int)(((size_t)((const char*)addr - (const char*)zeroAddr)) & 7); /* use 7 to check aligned 8 */ +} + +#define SECUREC_SMALL_MEM_COPY do { \ + if (SECUREC_ADDR_ALIGNED_8(dest) && SECUREC_ADDR_ALIGNED_8(src)) { \ + /* use struct assignment */ \ + switch (count) { \ + case 1: \ + *(SecStrBuf1 *)dest = *(const SecStrBuf1 *)src; \ + break; \ + case 2: \ + *(SecStrBuf2 *)dest = *(const SecStrBuf2 *)src; \ + break; \ + case 3: \ + *(SecStrBuf3 *)dest = *(const SecStrBuf3 *)src; \ + break; \ + case 4: \ + *(SecStrBuf4 *)dest = *(const SecStrBuf4 *)src; \ + break; \ + case 5: \ + *(SecStrBuf5 *)dest = *(const SecStrBuf5 *)src; \ + break; \ + case 6: \ + *(SecStrBuf6 *)dest = *(const SecStrBuf6 *)src; \ + break; \ + case 7: \ + *(SecStrBuf7 *)dest = *(const SecStrBuf7 *)src; \ + break; \ + case 8: \ + *(SecStrBuf8 *)dest = *(const SecStrBuf8 *)src; \ + break; \ + case 9: \ + *(SecStrBuf9 *)dest = *(const SecStrBuf9 *)src; \ + break; \ + case 10: \ + *(SecStrBuf10 *)dest = *(const SecStrBuf10 *)src; \ + break; \ + case 11: \ + *(SecStrBuf11 *)dest = *(const SecStrBuf11 *)src; \ + break; \ + case 12: \ + *(SecStrBuf12 *)dest = *(const SecStrBuf12 *)src; \ + break; \ + case 13: \ + *(SecStrBuf13 *)dest = *(const SecStrBuf13 *)src; \ + break; \ + case 14: \ + *(SecStrBuf14 *)dest = *(const SecStrBuf14 *)src; \ + break; \ + case 15: \ + *(SecStrBuf15 *)dest = *(const SecStrBuf15 *)src; \ + break; \ + case 16: \ + *(SecStrBuf16 *)dest = *(const SecStrBuf16 *)src; \ + break; \ + case 17: \ + *(SecStrBuf17 *)dest = *(const SecStrBuf17 *)src; \ + break; \ + case 18: \ + *(SecStrBuf18 *)dest = *(const SecStrBuf18 *)src; \ + break; \ + case 19: \ + *(SecStrBuf19 *)dest = *(const SecStrBuf19 *)src; \ + break; \ + case 20: \ + *(SecStrBuf20 *)dest = *(const SecStrBuf20 *)src; \ + break; \ + case 21: \ + *(SecStrBuf21 *)dest = *(const SecStrBuf21 *)src; \ + break; \ + case 22: \ + *(SecStrBuf22 *)dest = *(const SecStrBuf22 *)src; \ + break; \ + case 23: \ + *(SecStrBuf23 *)dest = *(const SecStrBuf23 *)src; \ + break; \ + case 24: \ + *(SecStrBuf24 *)dest = *(const SecStrBuf24 *)src; \ + break; \ + case 25: \ + *(SecStrBuf25 *)dest = *(const SecStrBuf25 *)src; \ + break; \ + case 26: \ + *(SecStrBuf26 *)dest = *(const SecStrBuf26 *)src; \ + break; \ + case 27: \ + *(SecStrBuf27 *)dest = *(const SecStrBuf27 *)src; \ + break; \ + case 28: \ + *(SecStrBuf28 *)dest = *(const SecStrBuf28 *)src; \ + break; \ + case 29: \ + *(SecStrBuf29 *)dest = *(const SecStrBuf29 *)src; \ + break; \ + case 30: \ + *(SecStrBuf30 *)dest = *(const SecStrBuf30 *)src; \ + break; \ + case 31: \ + *(SecStrBuf31 *)dest = *(const SecStrBuf31 *)src; \ + break; \ + case 32: \ + *(SecStrBuf32 *)dest = *(const SecStrBuf32 *)src; \ + break; \ + case 33: \ + *(SecStrBuf33 *)dest = *(const SecStrBuf33 *)src; \ + break; \ + case 34: \ + *(SecStrBuf34 *)dest = *(const SecStrBuf34 *)src; \ + break; \ + case 35: \ + *(SecStrBuf35 *)dest = *(const SecStrBuf35 *)src; \ + break; \ + case 36: \ + *(SecStrBuf36 *)dest = *(const SecStrBuf36 *)src; \ + break; \ + case 37: \ + *(SecStrBuf37 *)dest = *(const SecStrBuf37 *)src; \ + break; \ + case 38: \ + *(SecStrBuf38 *)dest = *(const SecStrBuf38 *)src; \ + break; \ + case 39: \ + *(SecStrBuf39 *)dest = *(const SecStrBuf39 *)src; \ + break; \ + case 40: \ + *(SecStrBuf40 *)dest = *(const SecStrBuf40 *)src; \ + break; \ + case 41: \ + *(SecStrBuf41 *)dest = *(const SecStrBuf41 *)src; \ + break; \ + case 42: \ + *(SecStrBuf42 *)dest = *(const SecStrBuf42 *)src; \ + break; \ + case 43: \ + *(SecStrBuf43 *)dest = *(const SecStrBuf43 *)src; \ + break; \ + case 44: \ + *(SecStrBuf44 *)dest = *(const SecStrBuf44 *)src; \ + break; \ + case 45: \ + *(SecStrBuf45 *)dest = *(const SecStrBuf45 *)src; \ + break; \ + case 46: \ + *(SecStrBuf46 *)dest = *(const SecStrBuf46 *)src; \ + break; \ + case 47: \ + *(SecStrBuf47 *)dest = *(const SecStrBuf47 *)src; \ + break; \ + case 48: \ + *(SecStrBuf48 *)dest = *(const SecStrBuf48 *)src; \ + break; \ + case 49: \ + *(SecStrBuf49 *)dest = *(const SecStrBuf49 *)src; \ + break; \ + case 50: \ + *(SecStrBuf50 *)dest = *(const SecStrBuf50 *)src; \ + break; \ + case 51: \ + *(SecStrBuf51 *)dest = *(const SecStrBuf51 *)src; \ + break; \ + case 52: \ + *(SecStrBuf52 *)dest = *(const SecStrBuf52 *)src; \ + break; \ + case 53: \ + *(SecStrBuf53 *)dest = *(const SecStrBuf53 *)src; \ + break; \ + case 54: \ + *(SecStrBuf54 *)dest = *(const SecStrBuf54 *)src; \ + break; \ + case 55: \ + *(SecStrBuf55 *)dest = *(const SecStrBuf55 *)src; \ + break; \ + case 56: \ + *(SecStrBuf56 *)dest = *(const SecStrBuf56 *)src; \ + break; \ + case 57: \ + *(SecStrBuf57 *)dest = *(const SecStrBuf57 *)src; \ + break; \ + case 58: \ + *(SecStrBuf58 *)dest = *(const SecStrBuf58 *)src; \ + break; \ + case 59: \ + *(SecStrBuf59 *)dest = *(const SecStrBuf59 *)src; \ + break; \ + case 60: \ + *(SecStrBuf60 *)dest = *(const SecStrBuf60 *)src; \ + break; \ + case 61: \ + *(SecStrBuf61 *)dest = *(const SecStrBuf61 *)src; \ + break; \ + case 62: \ + *(SecStrBuf62 *)dest = *(const SecStrBuf62 *)src; \ + break; \ + case 63: \ + *(SecStrBuf63 *)dest = *(const SecStrBuf63 *)src; \ + break; \ + case 64: \ + *(SecStrBuf64 *)dest = *(const SecStrBuf64 *)src; \ + break; \ + default: \ + break; \ + } /* END switch */ \ + } else { \ + char *tmpDest = (char *)dest; \ + const char *tmpSrc = (const char *)src; \ + switch (count) { \ + case 64: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 63: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 62: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 61: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 60: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 59: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 58: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 57: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 56: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 55: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 54: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 53: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 52: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 51: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 50: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 49: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 48: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 47: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 46: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 45: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 44: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 43: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 42: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 41: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 40: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 39: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 38: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 37: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 36: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 35: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 34: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 33: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 32: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + break; \ + } \ + } \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Handling errors + */ +static errno_t SecMemcpyError(void *dest, size_t destMax, const void *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memcpy_s"); + return ERANGE; + } + if (dest == NULL || src == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memcpy_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax); + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > destMax) { + (void)memset(dest, 0, destMax); + SECUREC_ERROR_INVALID_RANGE("memcpy_s"); + return ERANGE_AND_RESET; + } + if (dest == src) { + return EOK; + } + if ((dest > src && dest < (const void *)((const unsigned char *)src + count)) || \ + (src > dest && src < (void *)((unsigned char *)dest + count))) { + (void)memset(dest, 0, destMax); + SECUREC_ERROR_BUFFER_OVERLAP("memcpy_s"); + return EOVERLAP_AND_RESET; + } + /* count == 0 also return EOK */ + return EOK; +} + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMCOPY_WITH_PERFORMANCE +/* + * Performance optimization + */ +static void SecDoMemcpyOpt(void *dest, const void *src, size_t count) +{ + if (count > SECUREC_MEMCOPY_THRESHOLD_SIZE) { + SecDoMemcpy(dest, src, count); + } else { + SECUREC_SMALL_MEM_COPY; + } + return; +} +#endif + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + /* fread API in windows will call memcpy_s and pass 0xffffffff to destMax. + * To avoid the failure of fread, we don't check desMax limit. + */ +#define SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (src) != NULL && \ + (count) > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) +#else +#define SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (src) != NULL && \ + (destMax) <= SECUREC_MEM_MAX_LEN && \ + (count) > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) +#endif + +/* + * + * The memcpy_s function copies n characters from the object pointed to by src into the object pointed to by dest + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Buffer to copy from. + * count Number of characters to copy + * + * + * dest buffer is updated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax > SECUREC_MEM_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > destMax and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * and dest != NULL and src != NULL + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and + * count <= destMax destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN and dest != NULL + * and src != NULL and dest != src + * + * if an error occured, dest will be filled with 0. + * If the source and destination overlap, the behavior of memcpy_s is undefined. + * Use memmove_s to handle overlapping regions. + */ +errno_t memcpy_s(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count)) { +#if SECUREC_MEMCOPY_WITH_PERFORMANCE + SecDoMemcpyOpt(dest, src, count); +#else + SecDoMemcpy(dest, src, count); +#endif + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(memcpy_s); +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS +/* + * Performance optimization + */ +errno_t memcpy_sOptAsm(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count)) { + SecDoMemcpyOpt(dest, src, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} + +/* trim judgement on "destMax <= SECUREC_MEM_MAX_LEN" */ +errno_t memcpy_sOptTc(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_LIKELY(count <= destMax && dest != NULL && src != NULL && \ + count > 0 && \ + ((dest > src && (const void *)((const unsigned char *)src + count) <= dest) || \ + (src > dest && (void *)((unsigned char *)dest + count) <= src)))) { + SecDoMemcpyOpt(dest, src, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} +#endif + diff --git a/third_party/securec/src/memmove_s.c b/third_party/securec/src/memmove_s.c new file mode 100644 index 00000000000..ec6d04a749f --- /dev/null +++ b/third_party/securec/src/memmove_s.c @@ -0,0 +1,120 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securecutil.h" + +#ifdef SECUREC_NOT_CALL_LIBC_CORE_API +/* + * Implementing memory data movement + */ +static void SecUtilMemmove(void *dst, const void *src, size_t count) +{ + unsigned char *pDest = (unsigned char *)dst; + const unsigned char *pSrc = (const unsigned char *)src; + size_t maxCount = count; + + if (dst <= src || pDest >= (pSrc + maxCount)) { + /* + * Non-Overlapping Buffers + * copy from lower addresses to higher addresses + */ + while (maxCount--) { + *pDest = *pSrc; + ++pDest; + ++pSrc; + } + } else { + /* + * Overlapping Buffers + * copy from higher addresses to lower addresses + */ + pDest = pDest + maxCount - 1; + pSrc = pSrc + maxCount - 1; + + while (maxCount--) { + *pDest = *pSrc; + + --pDest; + --pSrc; + } + } +} +#endif + +/* + * + * The memmove_s function copies count bytes of characters from src to dest. + * This function can be assigned correctly when memory overlaps. + * + * dest Destination object. + * destMax Size of the destination buffer. + * src Source object. + * count Number of characters to copy. + * + * + * dest buffer is uptdated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax > SECUREC_MEM_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > destMax and dest != NULL and src != NULL and destMax != 0 + * and destMax <= SECUREC_MEM_MAX_LEN + * + * If an error occured, dest will be filled with 0 when dest and destMax valid. + * If some regions of the source area and the destination overlap, memmove_s + * ensures that the original source bytes in the overlapping region are copied + * before being overwritten. + */ +errno_t memmove_s(void *dest, size_t destMax, const void *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memmove_s"); + return ERANGE; + } + if (dest == NULL || src == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memmove_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax); + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > destMax) { + (void)memset(dest, 0, destMax); + SECUREC_ERROR_INVALID_RANGE("memmove_s"); + return ERANGE_AND_RESET; + } + if (dest == src) { + return EOK; + } + + if (count > 0) { +#ifdef SECUREC_NOT_CALL_LIBC_CORE_API + SecUtilMemmove(dest, src, count); +#else + /* use underlying memmove for performance consideration */ + (void)memmove(dest, src, count); +#endif + } + return EOK; +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(memmove_s); +#endif + diff --git a/third_party/securec/src/memset_s.c b/third_party/securec/src/memset_s.c new file mode 100644 index 00000000000..cd3f9887ef3 --- /dev/null +++ b/third_party/securec/src/memset_s.c @@ -0,0 +1,522 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMSET 1 + +#include "securecutil.h" + +#ifndef SECUREC_MEMSET_WITH_PERFORMANCE +#define SECUREC_MEMSET_WITH_PERFORMANCE 0 +#endif + +#define SECUREC_MEMSET_PARAM_OK(dest, destMax, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (destMax) <= SECUREC_MEM_MAX_LEN)) + + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMSET_WITH_PERFORMANCE +/* + * Determine whether the address is 8-byte aligned, use static to increase performance + * return 0 is aligned + */ +static int SecIsAddrAligned8(const void *addr, const void *zeroAddr) +{ + return (int)(((size_t)((const char*)addr - (const char*)zeroAddr)) & 7); /* use 7 to check aligned 8 */ +} + +/* use union to clear strict-aliasing warning */ +typedef union { + SecStrBuf32 buf32; + SecStrBuf31 buf31; + SecStrBuf30 buf30; + SecStrBuf29 buf29; + SecStrBuf28 buf28; + SecStrBuf27 buf27; + SecStrBuf26 buf26; + SecStrBuf25 buf25; + SecStrBuf24 buf24; + SecStrBuf23 buf23; + SecStrBuf22 buf22; + SecStrBuf21 buf21; + SecStrBuf20 buf20; + SecStrBuf19 buf19; + SecStrBuf18 buf18; + SecStrBuf17 buf17; + SecStrBuf16 buf16; + SecStrBuf15 buf15; + SecStrBuf14 buf14; + SecStrBuf13 buf13; + SecStrBuf12 buf12; + SecStrBuf11 buf11; + SecStrBuf10 buf10; + SecStrBuf9 buf9; + SecStrBuf8 buf8; + SecStrBuf7 buf7; + SecStrBuf6 buf6; + SecStrBuf5 buf5; + SecStrBuf4 buf4; + SecStrBuf3 buf3; + SecStrBuf2 buf2; + SecStrBuf1 buf1; +} SecStrBuf32Union; +/* C standard initializes the first member of the consortium. */ +static const SecStrBuf32 g_allZero = {{ + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0' +}}; +static const SecStrBuf32 g_allFF = {{ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF +}}; + +static const SecStrBuf32Union *SecStrictAliasingCast(const SecStrBuf32 *buf) +{ + return (const SecStrBuf32Union *)buf; +} + +#ifndef SECUREC_MEMSET_THRESHOLD_SIZE +#define SECUREC_MEMSET_THRESHOLD_SIZE 32UL +#endif + +#define SECUREC_UNALIGNED_SET do { \ + char *pcDest = (char *)dest; \ + switch (count) { \ + case 32: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + break; \ + } \ +} SECUREC_WHILE_ZERO + +#define SECUREC_ALIGNED_SET_OPT_ZERO_FF do { \ + switch (c) { \ + case 0: \ + switch (count) { \ + case 1: \ + *(SecStrBuf1 *)dest = *(const SecStrBuf1 *)(&((SecStrictAliasingCast(&g_allZero))->buf1)); \ + break; \ + case 2: \ + *(SecStrBuf2 *)dest = *(const SecStrBuf2 *)(&((SecStrictAliasingCast(&g_allZero))->buf2)); \ + break; \ + case 3: \ + *(SecStrBuf3 *)dest = *(const SecStrBuf3 *)(&((SecStrictAliasingCast(&g_allZero))->buf3)); \ + break; \ + case 4: \ + *(SecStrBuf4 *)dest = *(const SecStrBuf4 *)(&((SecStrictAliasingCast(&g_allZero))->buf4)); \ + break; \ + case 5: \ + *(SecStrBuf5 *)dest = *(const SecStrBuf5 *)(&((SecStrictAliasingCast(&g_allZero))->buf5)); \ + break; \ + case 6: \ + *(SecStrBuf6 *)dest = *(const SecStrBuf6 *)(&((SecStrictAliasingCast(&g_allZero))->buf6)); \ + break; \ + case 7: \ + *(SecStrBuf7 *)dest = *(const SecStrBuf7 *)(&((SecStrictAliasingCast(&g_allZero))->buf7)); \ + break; \ + case 8: \ + *(SecStrBuf8 *)dest = *(const SecStrBuf8 *)(&((SecStrictAliasingCast(&g_allZero))->buf8)); \ + break; \ + case 9: \ + *(SecStrBuf9 *)dest = *(const SecStrBuf9 *)(&((SecStrictAliasingCast(&g_allZero))->buf9)); \ + break; \ + case 10: \ + *(SecStrBuf10 *)dest = *(const SecStrBuf10 *)(&((SecStrictAliasingCast(&g_allZero))->buf10)); \ + break; \ + case 11: \ + *(SecStrBuf11 *)dest = *(const SecStrBuf11 *)(&((SecStrictAliasingCast(&g_allZero))->buf11)); \ + break; \ + case 12: \ + *(SecStrBuf12 *)dest = *(const SecStrBuf12 *)(&((SecStrictAliasingCast(&g_allZero))->buf12)); \ + break; \ + case 13: \ + *(SecStrBuf13 *)dest = *(const SecStrBuf13 *)(&((SecStrictAliasingCast(&g_allZero))->buf13)); \ + break; \ + case 14: \ + *(SecStrBuf14 *)dest = *(const SecStrBuf14 *)(&((SecStrictAliasingCast(&g_allZero))->buf14)); \ + break; \ + case 15: \ + *(SecStrBuf15 *)dest = *(const SecStrBuf15 *)(&((SecStrictAliasingCast(&g_allZero))->buf15)); \ + break; \ + case 16: \ + *(SecStrBuf16 *)dest = *(const SecStrBuf16 *)(&((SecStrictAliasingCast(&g_allZero))->buf16)); \ + break; \ + case 17: \ + *(SecStrBuf17 *)dest = *(const SecStrBuf17 *)(&((SecStrictAliasingCast(&g_allZero))->buf17)); \ + break; \ + case 18: \ + *(SecStrBuf18 *)dest = *(const SecStrBuf18 *)(&((SecStrictAliasingCast(&g_allZero))->buf18)); \ + break; \ + case 19: \ + *(SecStrBuf19 *)dest = *(const SecStrBuf19 *)(&((SecStrictAliasingCast(&g_allZero))->buf19)); \ + break; \ + case 20: \ + *(SecStrBuf20 *)dest = *(const SecStrBuf20 *)(&((SecStrictAliasingCast(&g_allZero))->buf20)); \ + break; \ + case 21: \ + *(SecStrBuf21 *)dest = *(const SecStrBuf21 *)(&((SecStrictAliasingCast(&g_allZero))->buf21)); \ + break; \ + case 22: \ + *(SecStrBuf22 *)dest = *(const SecStrBuf22 *)(&((SecStrictAliasingCast(&g_allZero))->buf22)); \ + break; \ + case 23: \ + *(SecStrBuf23 *)dest = *(const SecStrBuf23 *)(&((SecStrictAliasingCast(&g_allZero))->buf23)); \ + break; \ + case 24: \ + *(SecStrBuf24 *)dest = *(const SecStrBuf24 *)(&((SecStrictAliasingCast(&g_allZero))->buf24)); \ + break; \ + case 25: \ + *(SecStrBuf25 *)dest = *(const SecStrBuf25 *)(&((SecStrictAliasingCast(&g_allZero))->buf25)); \ + break; \ + case 26: \ + *(SecStrBuf26 *)dest = *(const SecStrBuf26 *)(&((SecStrictAliasingCast(&g_allZero))->buf26)); \ + break; \ + case 27: \ + *(SecStrBuf27 *)dest = *(const SecStrBuf27 *)(&((SecStrictAliasingCast(&g_allZero))->buf27)); \ + break; \ + case 28: \ + *(SecStrBuf28 *)dest = *(const SecStrBuf28 *)(&((SecStrictAliasingCast(&g_allZero))->buf28)); \ + break; \ + case 29: \ + *(SecStrBuf29 *)dest = *(const SecStrBuf29 *)(&((SecStrictAliasingCast(&g_allZero))->buf29)); \ + break; \ + case 30: \ + *(SecStrBuf30 *)dest = *(const SecStrBuf30 *)(&((SecStrictAliasingCast(&g_allZero))->buf30)); \ + break; \ + case 31: \ + *(SecStrBuf31 *)dest = *(const SecStrBuf31 *)(&((SecStrictAliasingCast(&g_allZero))->buf31)); \ + break; \ + case 32: \ + *(SecStrBuf32 *)dest = *(const SecStrBuf32 *)(&((SecStrictAliasingCast(&g_allZero))->buf32)); \ + break; \ + default: \ + break; \ + } \ + break; \ + case 0xFF: \ + switch (count) { \ + case 1: \ + *(SecStrBuf1 *)dest = *(const SecStrBuf1 *)(&((SecStrictAliasingCast(&g_allFF))->buf1)); \ + break; \ + case 2: \ + *(SecStrBuf2 *)dest = *(const SecStrBuf2 *)(&((SecStrictAliasingCast(&g_allFF))->buf2)); \ + break; \ + case 3: \ + *(SecStrBuf3 *)dest = *(const SecStrBuf3 *)(&((SecStrictAliasingCast(&g_allFF))->buf3)); \ + break; \ + case 4: \ + *(SecStrBuf4 *)dest = *(const SecStrBuf4 *)(&((SecStrictAliasingCast(&g_allFF))->buf4)); \ + break; \ + case 5: \ + *(SecStrBuf5 *)dest = *(const SecStrBuf5 *)(&((SecStrictAliasingCast(&g_allFF))->buf5)); \ + break; \ + case 6: \ + *(SecStrBuf6 *)dest = *(const SecStrBuf6 *)(&((SecStrictAliasingCast(&g_allFF))->buf6)); \ + break; \ + case 7: \ + *(SecStrBuf7 *)dest = *(const SecStrBuf7 *)(&((SecStrictAliasingCast(&g_allFF))->buf7)); \ + break; \ + case 8: \ + *(SecStrBuf8 *)dest = *(const SecStrBuf8 *)(&((SecStrictAliasingCast(&g_allFF))->buf8)); \ + break; \ + case 9: \ + *(SecStrBuf9 *)dest = *(const SecStrBuf9 *)(&((SecStrictAliasingCast(&g_allFF))->buf9)); \ + break; \ + case 10: \ + *(SecStrBuf10 *)dest = *(const SecStrBuf10 *)(&((SecStrictAliasingCast(&g_allFF))->buf10)); \ + break; \ + case 11: \ + *(SecStrBuf11 *)dest = *(const SecStrBuf11 *)(&((SecStrictAliasingCast(&g_allFF))->buf11)); \ + break; \ + case 12: \ + *(SecStrBuf12 *)dest = *(const SecStrBuf12 *)(&((SecStrictAliasingCast(&g_allFF))->buf12)); \ + break; \ + case 13: \ + *(SecStrBuf13 *)dest = *(const SecStrBuf13 *)(&((SecStrictAliasingCast(&g_allFF))->buf13)); \ + break; \ + case 14: \ + *(SecStrBuf14 *)dest = *(const SecStrBuf14 *)(&((SecStrictAliasingCast(&g_allFF))->buf14)); \ + break; \ + case 15: \ + *(SecStrBuf15 *)dest = *(const SecStrBuf15 *)(&((SecStrictAliasingCast(&g_allFF))->buf15)); \ + break; \ + case 16: \ + *(SecStrBuf16 *)dest = *(const SecStrBuf16 *)(&((SecStrictAliasingCast(&g_allFF))->buf16)); \ + break; \ + case 17: \ + *(SecStrBuf17 *)dest = *(const SecStrBuf17 *)(&((SecStrictAliasingCast(&g_allFF))->buf17)); \ + break; \ + case 18: \ + *(SecStrBuf18 *)dest = *(const SecStrBuf18 *)(&((SecStrictAliasingCast(&g_allFF))->buf18)); \ + break; \ + case 19: \ + *(SecStrBuf19 *)dest = *(const SecStrBuf19 *)(&((SecStrictAliasingCast(&g_allFF))->buf19)); \ + break; \ + case 20: \ + *(SecStrBuf20 *)dest = *(const SecStrBuf20 *)(&((SecStrictAliasingCast(&g_allFF))->buf20)); \ + break; \ + case 21: \ + *(SecStrBuf21 *)dest = *(const SecStrBuf21 *)(&((SecStrictAliasingCast(&g_allFF))->buf21)); \ + break; \ + case 22: \ + *(SecStrBuf22 *)dest = *(const SecStrBuf22 *)(&((SecStrictAliasingCast(&g_allFF))->buf22)); \ + break; \ + case 23: \ + *(SecStrBuf23 *)dest = *(const SecStrBuf23 *)(&((SecStrictAliasingCast(&g_allFF))->buf23)); \ + break; \ + case 24: \ + *(SecStrBuf24 *)dest = *(const SecStrBuf24 *)(&((SecStrictAliasingCast(&g_allFF))->buf24)); \ + break; \ + case 25: \ + *(SecStrBuf25 *)dest = *(const SecStrBuf25 *)(&((SecStrictAliasingCast(&g_allFF))->buf25)); \ + break; \ + case 26: \ + *(SecStrBuf26 *)dest = *(const SecStrBuf26 *)(&((SecStrictAliasingCast(&g_allFF))->buf26)); \ + break; \ + case 27: \ + *(SecStrBuf27 *)dest = *(const SecStrBuf27 *)(&((SecStrictAliasingCast(&g_allFF))->buf27)); \ + break; \ + case 28: \ + *(SecStrBuf28 *)dest = *(const SecStrBuf28 *)(&((SecStrictAliasingCast(&g_allFF))->buf28)); \ + break; \ + case 29: \ + *(SecStrBuf29 *)dest = *(const SecStrBuf29 *)(&((SecStrictAliasingCast(&g_allFF))->buf29)); \ + break; \ + case 30: \ + *(SecStrBuf30 *)dest = *(const SecStrBuf30 *)(&((SecStrictAliasingCast(&g_allFF))->buf30)); \ + break; \ + case 31: \ + *(SecStrBuf31 *)dest = *(const SecStrBuf31 *)(&((SecStrictAliasingCast(&g_allFF))->buf31)); \ + break; \ + case 32: \ + *(SecStrBuf32 *)dest = *(const SecStrBuf32 *)(&((SecStrictAliasingCast(&g_allFF))->buf32)); \ + break; \ + default: \ + break; \ + } \ + break; \ + default: \ + SECUREC_UNALIGNED_SET; \ + } /* END switch */ \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Handling errors + */ +static errno_t SecMemsetError(void *dest, size_t destMax, int c, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memset_s"); + return ERANGE; + } + if (dest == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memset_s"); + return EINVAL; + } + if (count > destMax) { + (void)memset(dest, c, destMax); /* set entire buffer to value c */ + SECUREC_ERROR_INVALID_RANGE("memset_s"); + return ERANGE_AND_RESET; + } + return EOK; +} + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMSET_WITH_PERFORMANCE +/* + * Performance optimization + */ +static void SecDoMemsetOpt(void *dest, int c, size_t count) +{ + if (count > SECUREC_MEMSET_THRESHOLD_SIZE) { + SecDoMemset(dest, c, count); + } else { + if (SECUREC_ADDR_ALIGNED_8(dest)) { + /* use struct assignment */ + SECUREC_ALIGNED_SET_OPT_ZERO_FF; + } else { + SECUREC_UNALIGNED_SET; + } + } + return; +} +#endif + +/* + * + * The memset_s function copies the value of c (converted to an unsigned char) + * into each of the first count characters of the object pointed to by dest. + * + * + * dest Pointer to destination. + * destMax The size of the buffer. + * c Character to set. + * count Number of characters. + * + * + * dest buffer is uptdated. + * + * + * EOK Success + * EINVAL dest == NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax is 0 or destMax > SECUREC_MEM_MAX_LEN + * ERANGE_AND_RESET count > destMax and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN and dest != NULL + * + * if return ERANGE_AND_RESET then fill dest to c ,fill length is destMax + */ +errno_t memset_s(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_MEMSET_PARAM_OK(dest, destMax, count)) { +#if SECUREC_MEMSET_WITH_PERFORMANCE + SecDoMemsetOpt(dest, c, count); +#else + SecDoMemset(dest, c, count); +#endif + return EOK; + } else { + /* meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c, count); + } +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(memset_s); +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS +/* + * Performance optimization + */ +errno_t memset_sOptAsm(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_MEMSET_PARAM_OK(dest, destMax, count)) { + SecDoMemsetOpt(dest, c, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c, count); +} + +/* + * Performance optimization + */ +errno_t memset_sOptTc(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_LIKELY(count <= destMax && dest != NULL)) { + SecDoMemsetOpt(dest, c, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c, count); +} +#endif + diff --git a/third_party/securec/src/output.inl b/third_party/securec/src/output.inl new file mode 100644 index 00000000000..d4e136c5707 --- /dev/null +++ b/third_party/securec/src/output.inl @@ -0,0 +1,1401 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OUTPUT_INL_2B263E9C_43D8_44BB_B17A_6D2033DECEE5 +#define OUTPUT_INL_2B263E9C_43D8_44BB_B17A_6D2033DECEE5 + +#define SECUREC_NULL_STRING_SIZE 8 +#define SECUREC_STATE_TABLE_SIZE 337 +#define SECUREC_OFFSET_BITS_WORD 16 +#define SECUREC_OFFSET_BITS_DWORD 32 + +#define SECUREC_OFFSET_DIV_OCTAL 3 +#define SECUREC_OFFSET_DIV_HEX 4 +#define SECUREC_RADIX_OCTAL 8 +#define SECUREC_RADIX_DECIMAL 10 +#define SECUREC_RADIX_HEX 16 +/* Use two displacements to eliminate compilation warnings */ +#define SECUREC_SHR_DWORD(x) (((x) >> 16) >> 16) +#define SECUREC_PREFIX_LEN 2 +/* size include '+' and '\0' */ +#define SECUREC_FLOAT_BUF_EXT 2 + + +#ifdef SECUREC_STACK_SIZE_LESS_THAN_1K +#define SECUREC_FMT_STR_LEN 8 +#else +#define SECUREC_FMT_STR_LEN 16 +#endif + +typedef struct { + unsigned int flags; + int fldWidth; + int precision; + int bufferIsWide; /* flag for buffer contains wide chars ;0 is not wide char */ + int dynWidth; /* %* 1 width from variable parameter ;0 not */ + int dynPrecision; /* %.* 1 precision from variable parameter ;0 not */ +} SecFormatAttr; + +typedef union { + char *str; /* not a null terminated string */ +#if SECUREC_HAVE_WCHART + wchar_t *wStr; +#endif +} SecFormatBuf; + +typedef union { + char str[SECUREC_BUFFER_SIZE + 1]; +#ifdef SECUREC_FOR_WCHAR + wchar_t wStr[SECUREC_BUFFER_SIZE + 1]; +#endif +} SecBuffer; + +#if SECUREC_ENABLE_SPRINTF_FLOAT +/* call system sprintf to format float value */ +static int SecIndirectSprintf(char *strDest, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + SECUREC_MASK_MSVC_CRT_WARNING + ret = vsprintf(strDest, format, argList); + SECUREC_END_MASK_MSVC_CRT_WARNING + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +/* out put long double value to dest */ +static int SecFormatLongDboule(char *strDest,const SecFormatAttr *formatAttr, const char *fmt, long double ldValue) +{ + int fldWidth = ((formatAttr->flags & SECUREC_FLAG_LEFT) ? (-(formatAttr->fldWidth)) : formatAttr->fldWidth); + if (formatAttr->dynWidth && formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, fldWidth, formatAttr->precision, ldValue); + } else if (formatAttr->dynWidth) { + return SecIndirectSprintf(strDest, fmt, fldWidth, ldValue); + } else if (formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, formatAttr->precision, ldValue); + } + return SecIndirectSprintf(strDest, fmt, ldValue); +} +#endif + +/* out put double value to dest */ +static int SecFormatDboule(char *strDest, const SecFormatAttr *formatAttr, const char *fmt, double dValue) +{ + int fldWidth = ((formatAttr->flags & SECUREC_FLAG_LEFT) ? (-(formatAttr->fldWidth)) : formatAttr->fldWidth); + if (formatAttr->dynWidth && formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, fldWidth, formatAttr->precision, dValue); + } else if (formatAttr->dynWidth) { + return SecIndirectSprintf(strDest, fmt, fldWidth, dValue); + } else if (formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, formatAttr->precision, dValue); + } + return SecIndirectSprintf(strDest, fmt, dValue); +} +#endif + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +/* to clear e506 warning */ +static int SecIsSameSize(size_t sizeA, size_t sizeB) +{ + return sizeA == sizeB; +} +#endif + +#define SECUREC_SPECIAL_DWORD(val32, numBase) do { \ + --formatBuf.str; \ + *(formatBuf.str) = digits[(val32) % (numBase)]; \ +} while (((val32) /= (numBase)) != 0) + +#if defined(SECUREC_USE_SPECIAL_DIV64) || (defined(SECUREC_VXWORKS_VERSION_5_4) && !defined(SECUREC_ON_64BITS)) +/* + * Fast divide by 10 algorithm. + * Calculation divisor multiply 0xcccccccccccccccdULL, resultHi64 >> 3 as quotient + */ +static void SecU64Div10(SecUnsignedInt64 divisor, SecUnsignedInt64 *quotient, SecUnsignedInt32 *remainder) +{ + SecUnsignedInt64 mask = 0xffffffffULL; /* use 0xffffffffULL as 32 bit mask */ + SecUnsignedInt64 magicHi = 0xccccccccULL; /* fast divide 10 magic numbers high 32bit 0xccccccccULL */ + SecUnsignedInt64 magicLow = 0xcccccccdULL; /* fast divide 10 magic numbers low 32bit 0xcccccccdULL */ + SecUnsignedInt64 divisorHi = (SecUnsignedInt64)(SECUREC_SHR_DWORD(divisor)); /* hig 32 bit use */ + SecUnsignedInt64 divisorLow = (SecUnsignedInt64)(divisor & mask); /* low 32 bit mask */ + SecUnsignedInt64 factorHi = divisorHi * magicHi; + SecUnsignedInt64 factorLow1 = divisorHi * magicLow; + SecUnsignedInt64 factorLow2 = divisorLow * magicHi; + SecUnsignedInt64 factorLow3 = divisorLow * magicLow; + SecUnsignedInt64 carry = (factorLow1 & mask) + (factorLow2 & mask) + SECUREC_SHR_DWORD(factorLow3); + SecUnsignedInt64 resultHi64 = factorHi + SECUREC_SHR_DWORD(factorLow1) + \ + SECUREC_SHR_DWORD(factorLow2) + SECUREC_SHR_DWORD(carry); + + *quotient = resultHi64 >> 3; /* fast divide 10 magic numbers 3 */ + *remainder = (SecUnsignedInt32)(divisor - ((*quotient) * 10)); /* quotient mul 10 */ + return; +} +#if defined(SECUREC_VXWORKS_VERSION_5_4) && !defined(SECUREC_ON_64BITS) +/* + * Divide function for VXWORKS + */ +static int SecU64Div32(SecUnsignedInt64 divisor, SecUnsignedInt32 radix, + SecUnsignedInt64 *quotient, SecUnsignedInt32 *remainder) +{ + switch (radix) { + case SECUREC_RADIX_DECIMAL: + SecU64Div10(divisor, quotient, remainder); + break; + case SECUREC_RADIX_HEX: + *quotient = divisor >> SECUREC_OFFSET_DIV_HEX; + *remainder = divisor & 0xfULL; /* mask one hex number by 0xfULL */ + break; + case SECUREC_RADIX_OCTAL: + *quotient = divisor >> SECUREC_OFFSET_DIV_OCTAL; + *remainder = divisor & 0x7ULL; /* mask one hex number by 0x7ULL */ + break; + default: + return -1; + } + return 0; +} +#endif +#endif + +#if defined(SECUREC_USE_SPECIAL_DIV64) +/* The compiler does not provide 64 bit division problems */ +#define SECUREC_SPECIAL_QWORD_BASE10(val64) do { \ + SecUnsignedInt64 quotient = 0; \ + SecUnsignedInt32 digit = 0; \ + SecU64Div10((val64), &(quotient), &(digit)); \ + --formatBuf.str; \ + *(formatBuf.str) = digits[digit]; \ + (val64) = quotient; \ +} while ((val64) != 0) +#else +#define SECUREC_SPECIAL_QWORD_BASE10(val64) do { \ + --formatBuf.str; \ + *(formatBuf.str) = digits[(val64) % SECUREC_RADIX_DECIMAL]; \ +} while (((val64) /= SECUREC_RADIX_DECIMAL) != 0) +#endif +#define SECUREC_SPECIAL_QWORD(val64, numBase) do { \ + --formatBuf.str; \ + *(formatBuf.str) = digits[(val64) % (numBase)]; \ +} while (((val64) /= (numBase)) != 0) + + +#define SECUREC_SAFE_WRITE_STR_OPT(src, txtLen, outStream, outChars) do { \ + int ii_; \ + for (ii_ = 0; ii_ < (txtLen); ++ii_) { \ + *((SecChar *)(void *)((outStream)->cur)) = *(SecChar *)(src); \ + (outStream)->cur += sizeof(SecChar); \ + (src) = (src) + 1; \ + } \ + (outStream)->count -= (txtLen) * (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + (txtLen); \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SAFE_WRITE_STR(src, txtLen, outStream, outChars) do { \ + if ((txtLen) < 12) { /* performance optimization for mobile number length 12 */ \ + SECUREC_SAFE_WRITE_STR_OPT((src), (txtLen), (outStream), (outChars)); \ + } else { \ + SecDoMemcpy((outStream)->cur, (src), ((size_t)(unsigned int)(txtLen) * (sizeof(SecChar)))); \ + (outStream)->cur += (size_t)((size_t)(unsigned int)(txtLen) * (sizeof(SecChar))); \ + (outStream)->count -= (txtLen) * (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + (txtLen); \ + } \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SAFE_WRITE_CHAR(c, outStream, outChars) do { \ + *((SecChar *)(void *)((outStream)->cur)) = (SecChar)(c); \ + (outStream)->cur += sizeof(SecChar); \ + (outStream)->count -= (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + 1; \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SAFE_PADDING(padChar, padLen, outStream, outChars) do { \ + int ii_; \ + for (ii_ = 0; ii_ < (padLen); ++ii_) { \ + *((SecChar *)(void *)((outStream)->cur)) = (SecChar)(padChar); \ + (outStream)->cur += sizeof(SecChar); \ + } \ + (outStream)->count -= (padLen) * (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + (padLen); \ +} SECUREC_WHILE_ZERO + +/* The count variable can be reduced to 0, and the external function complements the \0 terminator. */ +#define SECUREC_IS_REST_BUF_ENOUGH(stream, needLen) ((int)((stream)->count - \ + (int)(needLen) * (int)(sizeof(SecChar))) >= 0) + +#define SECUREC_FMT_STATE_OFFSET 256 +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_FMT_TYPE(c, fmtTable) ((((unsigned int)(int)(c)) <= (unsigned int)(int)SECUREC_CHAR('~')) ? \ + ((fmtTable)[(unsigned char)(c)]) : 0) +#define SECUREC_DECODE_STATE(c, fmtTable, lastState) (SecFmtState)((((fmtTable)[(SECUREC_FMT_TYPE(c, (fmtTable))) * \ + ((unsigned char)STAT_INVALID + 1) + \ + (unsigned char)(lastState) + \ + SECUREC_FMT_STATE_OFFSET]))) +#else +#define SECUREC_DECODE_STATE(c, fmtTable, lastState) (SecFmtState)(((fmtTable)[((fmtTable)[(unsigned char)(c)]) * \ + ((unsigned char)STAT_INVALID + 1) + \ + (unsigned char)(lastState) + \ + SECUREC_FMT_STATE_OFFSET])) +#endif + +static void SecDecodeFlags(SecChar ch, SecFormatAttr *attr) +{ + switch (ch) { + case SECUREC_CHAR(' '): + attr->flags |= SECUREC_FLAG_SIGN_SPACE; + break; + case SECUREC_CHAR('+'): + attr->flags |= SECUREC_FLAG_SIGN; + break; + case SECUREC_CHAR('-'): + attr->flags |= SECUREC_FLAG_LEFT; + break; + case SECUREC_CHAR('0'): + attr->flags |= SECUREC_FLAG_LEADZERO; /* add zero th the front */ + break; + case SECUREC_CHAR('#'): + attr->flags |= SECUREC_FLAG_ALTERNATE; /* output %x with 0x */ + break; + default: + break; + } + return; +} + + +/* + * Decoded size identifier in format string to Reduce the number of lines of function code + */ +static int SecDecodeSizeI(SecFormatAttr *attr, const SecChar **format) +{ +#ifdef SECUREC_ON_64BITS + attr->flags |= SECUREC_FLAG_I64; /* %I to INT64 */ +#endif + if ((**format == SECUREC_CHAR('6')) && (*((*format) + 1) == SECUREC_CHAR('4'))) { + (*format) += 2; /* add 2 to skip I64 */ + attr->flags |= SECUREC_FLAG_I64; /* %I64 to INT64 */ + } else if ((**format == SECUREC_CHAR('3')) && (*((*format) + 1) == SECUREC_CHAR('2'))) { + (*format) += 2; /* add 2 to skip I32 */ + attr->flags &= ~SECUREC_FLAG_I64; /* %I64 to INT32 */ + } else if ((**format == SECUREC_CHAR('d')) || (**format == SECUREC_CHAR('i')) || + (**format == SECUREC_CHAR('o')) || (**format == SECUREC_CHAR('u')) || + (**format == SECUREC_CHAR('x')) || (**format == SECUREC_CHAR('X'))) { + /* do nothing */ + } else { + /* Compatibility code for "%I" just print I */ + return -1; + } + return 0; +} +/* + * Decoded size identifier in format string + */ +static int SecDecodeSize(SecChar ch, SecFormatAttr *attr, const SecChar **format) +{ + switch (ch) { +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + case SECUREC_CHAR('j'): + attr->flags |= SECUREC_FLAG_INTMAX; + break; +#endif + case SECUREC_CHAR('q'): + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('L'): + attr->flags |= SECUREC_FLAG_LONGLONG | SECUREC_FLAG_LONG_DOUBLE; + break; + case SECUREC_CHAR('l'): + if (**format == SECUREC_CHAR('l')) { + *format = *format + 1; + attr->flags |= SECUREC_FLAG_LONGLONG; /* long long */ + } else { + attr->flags |= SECUREC_FLAG_LONG; /* long int or wchar_t */ + } + break; + case SECUREC_CHAR('t'): + attr->flags |= SECUREC_FLAG_PTRDIFF; + break; +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + case SECUREC_CHAR('z'): + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('Z'): + attr->flags |= SECUREC_FLAG_SIZE; + break; +#endif + case SECUREC_CHAR('I'): + if (SecDecodeSizeI(attr, format) != 0) { + /* Compatibility code for "%I" just print I */ + return -1; + } + break; + case SECUREC_CHAR('h'): + if (**format == SECUREC_CHAR('h')) { + attr->flags |= SECUREC_FLAG_CHAR; /* char */ + } else { + attr->flags |= SECUREC_FLAG_SHORT; /* short int */ + } + break; + case SECUREC_CHAR('w'): + attr->flags |= SECUREC_FLAG_WIDECHAR; /* wide char */ + break; + default: + break; + } + return 0; +} + +/* + * Decoded char type identifier + */ +static int SecDecodeTypeC(SecFormatAttr *attr, unsigned int cValue, SecFormatBuf *formatBuf, SecBuffer *buffer) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) && !(defined(__hpux)) && !(defined(SECUREC_ON_SOLARIS)) + attr->flags &= ~SECUREC_FLAG_LEADZERO; +#endif + +#ifdef SECUREC_FOR_WCHAR + attr->bufferIsWide = 1; + if (attr->flags & SECUREC_FLAG_SHORT) { +#if SECUREC_HAVE_MBTOWC + /* multibyte character to wide character */ + char tmpChar[2]; /* One character string, length is 2 */ + tmpChar[0] = (char)(cValue & 0x00ff); + tmpChar[1] = '\0'; + + if (mbtowc(buffer->wStr, tmpChar, sizeof(tmpChar)) < 0) { + return -1; + } +#else + return -1; +#endif + } else { + buffer->wStr[0] = (wchar_t)cValue; + } + formatBuf->wStr = buffer->wStr; + return 1; /* only 1 wide character */ +#else /* SECUREC_FOR_WCHAR */ + attr->bufferIsWide = 0; + if (attr->flags & (SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR)) { +#if SECUREC_HAVE_WCTOMB + wchar_t wChar = (wchar_t)cValue; + int textLen; + /* wide character to multibyte character */ + SECUREC_MASK_MSVC_CRT_WARNING + textLen = wctomb(buffer->str, wChar); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (textLen < 0) { + return -1; + } + formatBuf->str = buffer->str; + return textLen; +#else + return -1; +#endif + } else { + /* get multibyte character from argument */ + unsigned short temp; + temp = (unsigned short)cValue; + buffer->str[0] = (char)temp; + formatBuf->str = buffer->str; + return 1; /* only 1 character */ + } +#endif + +} + +/* literal string to print null ptr, define it as array rather than const text area + * is to avoid gcc warning with pointing const text with variable + */ +#if SECUREC_HAVE_WCHART +static wchar_t g_wStrNullString[SECUREC_NULL_STRING_SIZE] = { L'(', L'n', L'u', L'l', L'l', L')', L'\0', L'\0' }; +#endif +static char g_strNullString[SECUREC_NULL_STRING_SIZE] = "(null)"; + +static int SecDecodeTypeSchar(const SecFormatAttr *attr, SecFormatBuf *formatBuf) +{ + int finalPrecision = (attr->precision == -1) ? SECUREC_INT_MAX : attr->precision; + int textLen; + + if (formatBuf->str == NULL) { /* NULL passed, use special string */ + formatBuf->str = g_strNullString; + } + if (finalPrecision == SECUREC_INT_MAX) { + /* precision NOT assigned */ + /* The strlen performance is high when the string length is greater than 32 */ + textLen = (int)strlen(formatBuf->str); + } else { + /* precision assigned */ + size_t tmpLen; + SECUREC_CALC_STR_LEN(formatBuf->str, (size_t)(unsigned int)finalPrecision, &tmpLen); + textLen = (int)tmpLen; + } + return textLen; +} + +#if SECUREC_HAVE_WCHART +static int SecDecodeTypeSwchar(SecFormatAttr *attr, SecFormatBuf *formatBuf) +{ + int finalPrecision = (attr->precision == -1) ? SECUREC_INT_MAX : attr->precision; + int textLen; + + attr->bufferIsWide = 1; + if (formatBuf->wStr == NULL) { /* NULL passed, use special string */ + formatBuf->wStr = g_wStrNullString; + } + /* textLen in wchar_t */ + SECUREC_CALC_WSTR_LEN(formatBuf->wStr, finalPrecision, &textLen); + + return textLen; +} +#endif + +/* + * Decoded string identifier + */ +static int SecDecodeTypeS(SecFormatAttr *attr, char *argPtr, SecFormatBuf *formatBuf) +{ + int textLen; +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) && (!defined(SECUREC_ON_UNIX)) + attr->flags &= ~SECUREC_FLAG_LEADZERO; +#endif + formatBuf->str = argPtr; +#ifdef SECUREC_FOR_WCHAR +#if defined(SECUREC_COMPATIBLE_LINUX_FORMAT) + if (!(attr->flags & SECUREC_FLAG_LONG)) { + attr->flags |= SECUREC_FLAG_SHORT; + } +#endif + if (attr->flags & SECUREC_FLAG_SHORT) { + /* textLen now contains length in multibyte chars */ + textLen = SecDecodeTypeSchar(attr, formatBuf); + } else { + /* textLen now contains length in wide chars */ + textLen = SecDecodeTypeSwchar(attr, formatBuf); + } +#else /* SECUREC_FOR_WCHAR */ + if (attr->flags & (SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR)) { + /* textLen now contains length in wide chars */ +#if SECUREC_HAVE_WCHART + textLen = SecDecodeTypeSwchar(attr, formatBuf); +#else + textLen = 0; +#endif + } else { + /* textLen now contains length in multibyte chars */ + textLen = SecDecodeTypeSchar(attr, formatBuf); + } +#endif /* SECUREC_FOR_WCHAR */ + return textLen; +} + +/* + * Write one character to dest buffer + */ +static void SecOutputOneChar(SecChar ch, SecPrintfStream *stream, int *counter) +{ + /* normal state, write character */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, 1)) { /* only one char */ + SECUREC_SAFE_WRITE_CHAR(ch, stream, counter); /* char * cast to wchar * */ + } else { +#ifdef SECUREC_FOR_WCHAR + SecWriteCharW(ch, stream, counter); +#else + /* optimize function call to code */ + *counter = -1; + stream->count = -1; +#endif + } +} + +/* + * Check precison in format + */ +static int SecDecodePrecision(SecChar ch, SecFormatAttr *formatAttr) +{ + if (formatAttr->dynPrecision == 0) { + /* add digit to current precision */ + if (SECUREC_MUL_TEN_ADD_BEYOND_MAX(formatAttr->precision)) { + return -1; + } + formatAttr->precision = (int)SECUREC_MUL_TEN((unsigned int)formatAttr->precision) + + (unsigned char)(ch - SECUREC_CHAR('0')); + } else { + if (formatAttr->precision < 0) { + formatAttr->precision = -1; + } + if (formatAttr->precision > SECUREC_MAX_WIDTH_LEN) { + return -1; + } + } + return 0; +} + + +/* + * Check width in format + */ +static int SecDecodeWidth(SecChar ch, SecFormatAttr *formatAttr, SecFmtState lastState) +{ + if (formatAttr->dynWidth == 0) { + if (lastState != STAT_WIDTH) { + formatAttr->fldWidth = 0; + } + if (SECUREC_MUL_TEN_ADD_BEYOND_MAX(formatAttr->fldWidth)) { + return -1; + } + formatAttr->fldWidth = (int)SECUREC_MUL_TEN((unsigned int)formatAttr->fldWidth) + + (unsigned char)(ch - SECUREC_CHAR('0')); + } else { + if (formatAttr->fldWidth < 0) { + formatAttr->flags |= SECUREC_FLAG_LEFT; + formatAttr->fldWidth = (-formatAttr->fldWidth); + if (formatAttr->fldWidth > SECUREC_MAX_WIDTH_LEN) { + return -1; + } + } + } + return 0; +} +#ifdef SECUREC_FOR_WCHAR +/* + * Formatting output core functions for wchar version.Called by a function such as vswprintf_s + * argList must not be declare as const + */ +static int SecOutputSW(SecPrintfStream *stream, const wchar_t *cFormat, va_list argList) +#else +/* + * Formatting output core functions for char version.Called by a function such as vsnprintf_s + */ +static int SecOutputS(SecPrintfStream *stream, const char *cFormat, va_list argList) +#endif +{ + const SecChar *format = cFormat; +#if SECUREC_ENABLE_SPRINTF_FLOAT + char *floatBuf = NULL; +#endif + SecFormatBuf formatBuf; + static const char *itoaUpperDigits = "0123456789ABCDEFX"; + static const char *itoaLowerDigits = "0123456789abcdefx"; + const char *digits = itoaUpperDigits; + unsigned int radix = SECUREC_RADIX_DECIMAL; + int charsOut; /* characters written */ + int prefixLen = 0; /* Must be initialized or compiler alerts */ + int padding = 0; + int textLen; /* length of the text */ + int noOutput = 0; /* Must be initialized or compiler alerts */ + SecFmtState state; + SecFmtState lastState; + SecChar prefix[SECUREC_PREFIX_LEN] = { 0 }; + SecChar ch; /* currently read character */ + static const unsigned char stateTable[SECUREC_STATE_TABLE_SIZE] = { + /* type 0: nospecial meanin; + * 1: '%'; + * 2: '.' + * 3: '*' + * 4: '0' + * 5: '1' ... '9' + * 6: ' ', '+', '-', '#' + * 7: 'h', 'l', 'L', 'F', 'w' , 'N','z','q','t','j' + * 8: 'd','o','u','i','x','X','e','f','g' + */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x06, 0x00, 0x06, 0x02, 0x00, + 0x04, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x08, 0x08, 0x00, 0x07, 0x00, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, 0x07, 0x08, 0x07, 0x00, 0x07, 0x00, 0x00, 0x08, + 0x08, 0x07, 0x00, 0x08, 0x07, 0x08, 0x00, 0x07, 0x08, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + /* fill zero for normal char 128 byte for 0x80 - 0xff */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* state 0: normal + * 1: percent + * 2: flag + * 3: width + * 4: dot + * 5: precis + * 6: size + * 7: type + * 8: invalid + */ + 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x01, 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x01, 0x00, 0x00, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x03, 0x03, 0x08, 0x05, + 0x08, 0x08, 0x00, 0x00, 0x00, 0x02, 0x02, 0x03, 0x05, 0x05, 0x08, 0x00, 0x00, 0x00, 0x03, 0x03, + 0x03, 0x05, 0x05, 0x08, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00, 0x00, 0x00, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x00, + 0x00 + }; + + SecFormatAttr formatAttr; + SecBuffer buffer; + formatAttr.flags = 0; + formatAttr.bufferIsWide = 0; /* flag for buffer contains wide chars */ + formatAttr.fldWidth = 0; + formatAttr.precision = 0; + formatAttr.dynWidth = 0; + formatAttr.dynPrecision = 0; + charsOut = 0; + textLen = 0; + state = STAT_NORMAL; /* starting state */ + formatBuf.str = NULL; + + /* loop each format character */ + /* remove format != NULL */ + while ((ch = *format) != SECUREC_CHAR('\0') && charsOut >= 0) { + ++format; + lastState = state; + state = SECUREC_DECODE_STATE(ch, stateTable, lastState); + switch (state) { + case STAT_NORMAL: + SecOutputOneChar(ch, stream, &charsOut); + continue; + case STAT_PERCENT: + /* set default values */ + prefixLen = 0; + noOutput = 0; + formatAttr.flags = 0; + formatAttr.fldWidth = 0; + formatAttr.precision = -1; + formatAttr.bufferIsWide = 0; + formatAttr.dynWidth = 0; + formatAttr.dynPrecision = 0; + break; + case STAT_FLAG: + /* set flag based on which flag character */ + SecDecodeFlags(ch, &formatAttr); + break; + case STAT_WIDTH: + /* update width value */ + if (ch == SECUREC_CHAR('*')) { + /* get width */ + formatAttr.fldWidth = (int)va_arg(argList, int); + formatAttr.dynWidth = 1; + } else { + formatAttr.dynWidth = 0; + } + if (SecDecodeWidth(ch, &formatAttr, lastState) != 0) { + return -1; + } + break; + case STAT_DOT: + formatAttr.precision = 0; + break; + case STAT_PRECIS: + /* update precison value */ + if (ch == SECUREC_CHAR('*')) { + /* get precision from arg list */ + formatAttr.precision = (int)va_arg(argList, int); + formatAttr.dynPrecision = 1; + } else { + formatAttr.dynPrecision = 0; + } + if (SecDecodePrecision(ch, &formatAttr) != 0) { + return -1; + } + break; + case STAT_SIZE: + /* read a size specifier, set the formatAttr.flags based on it */ + if (SecDecodeSize(ch, &formatAttr, &format) != 0) { + /* Compatibility code for "%I" just print I */ + SecOutputOneChar(ch, stream, &charsOut); + state = STAT_NORMAL; + continue; + } + break; + case STAT_TYPE: + switch (ch) { + case SECUREC_CHAR('C'): + /* wide char */ + if (!(formatAttr.flags & (SECUREC_FLAG_SHORT | SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR))) { +#ifdef SECUREC_FOR_WCHAR + formatAttr.flags |= SECUREC_FLAG_SHORT; +#else + formatAttr.flags |= SECUREC_FLAG_WIDECHAR; +#endif + } + /* fall-through */ + /* FALLTHRU */ + case SECUREC_CHAR('c'): + do { + unsigned int cValue = (unsigned int)va_arg(argList, int); + textLen = SecDecodeTypeC(&formatAttr, cValue, &formatBuf, &buffer); + if (textLen < 0) { + noOutput = 1; + } + } SECUREC_WHILE_ZERO; + break; + case SECUREC_CHAR('S'): /* wide char string */ + if (!(formatAttr.flags & (SECUREC_FLAG_SHORT | SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR))) { +#ifndef SECUREC_FOR_WCHAR + formatAttr.flags |= SECUREC_FLAG_WIDECHAR; +#else + formatAttr.flags |= SECUREC_FLAG_SHORT; +#endif + } + /* fall-through */ + /* FALLTHRU */ + case SECUREC_CHAR('s'): + do { + char *argPtr = (char *)va_arg(argList, char *); + textLen = SecDecodeTypeS(&formatAttr, argPtr, &formatBuf); + } SECUREC_WHILE_ZERO; + break; + case SECUREC_CHAR('n'): + /* higher risk disable it */ + return -1; + case SECUREC_CHAR('E'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('F'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('G'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('A'): /* fall-through */ /* FALLTHRU */ + /* convert format char to lower , use Explicit conversion to clean up compilation warning */ + ch = (SecChar)(ch + ((SecChar)(SECUREC_CHAR('a')) - (SECUREC_CHAR('A')))); + /* fall-through */ + /* FALLTHRU */ + case SECUREC_CHAR('e'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('f'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('g'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('a'): +#if SECUREC_ENABLE_SPRINTF_FLOAT + do { + int bufferSize = 0; /* size of formatBuf.str */ + /* floating point conversion */ + formatBuf.str = buffer.str; /* output buffer for float string with default size */ + + /* compute the precision value */ + if (formatAttr.precision < 0) { + formatAttr.precision = SECUREC_FLOAT_DEFAULT_PRECISION; + } else if (formatAttr.precision == 0 && ch == SECUREC_CHAR('g')) { + formatAttr.precision = 1; + } + + /* calc buffer size to store double value + * The maximum length of SECUREC_MAX_WIDTH_LEN is enough + */ + if (formatAttr.flags & SECUREC_FLAG_LONG_DOUBLE) { + if (formatAttr.precision > (SECUREC_MAX_WIDTH_LEN - SECUREC_FLOAT_BUFSIZE_LB)) { + noOutput = 1; + break; + } + /* Long double needs to meet the basic print length */ + bufferSize = SECUREC_FLOAT_BUFSIZE_LB + formatAttr.precision + SECUREC_FLOAT_BUF_EXT; + } else { + if (formatAttr.precision > (SECUREC_MAX_WIDTH_LEN - SECUREC_FLOAT_BUFSIZE)) { + noOutput = 1; + break; + } + /* Double needs to meet the basic print length */ + bufferSize = SECUREC_FLOAT_BUFSIZE + formatAttr.precision + SECUREC_FLOAT_BUF_EXT; + } + if (formatAttr.fldWidth > bufferSize) { + bufferSize = formatAttr.fldWidth + SECUREC_FLOAT_BUF_EXT; + } + + if (bufferSize > SECUREC_BUFFER_SIZE) { + /* the current vlaue of SECUREC_BUFFER_SIZE could NOT store the + * formatted float string + */ + floatBuf = (char *)SECUREC_MALLOC(((size_t)(unsigned int)bufferSize)); + if (floatBuf != NULL) { + formatBuf.str = floatBuf; + } else { + noOutput = 1; + break; + } + } + + do { + /* add following code to call system sprintf API for float number */ + const SecChar *pFloatFmt = format - 2; /* sub 2 to the position before 'f' or 'g' */ + int k; + int fFmtStrLen; + char fFmtBuf[SECUREC_FMT_STR_LEN]; + char *fFmtStr = fFmtBuf; + char *fFmtHeap = NULL; /* to clear warning */ + + while (SECUREC_CHAR('%') != *pFloatFmt) { /* must meet '%' */ + --pFloatFmt; + } + fFmtStrLen = (int)((format - pFloatFmt) + 1); /* with ending terminator */ + if (fFmtStrLen > SECUREC_FMT_STR_LEN) { + /* if SECUREC_FMT_STR_LEN is NOT enough, alloc a new buffer */ + fFmtHeap = (char *)SECUREC_MALLOC((size_t)((unsigned int)fFmtStrLen)); + if (fFmtHeap == NULL) { + noOutput = 1; + break; + } else { + for (k = 0; k < fFmtStrLen - 1; ++k) { + /* convert wchar to char */ + fFmtHeap[k] = (char)(pFloatFmt[k]); /* copy the format string */ + } + fFmtHeap[k] = '\0'; + + fFmtStr = fFmtHeap; + } + } else { + /* purpose of the repeat code is to solve the tool alarm Redundant_Null_Check */ + for (k = 0; k < fFmtStrLen - 1; ++k) { + /* convert wchar to char */ + fFmtBuf[k] = (char)(pFloatFmt[k]); /* copy the format string */ + } + fFmtBuf[k] = '\0'; + } + + if (formatAttr.flags & SECUREC_FLAG_LONG_DOUBLE) { +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + long double tmp = (long double)va_arg(argList, long double); + textLen = SecFormatLongDboule(formatBuf.str, &formatAttr, fFmtStr, tmp); +#else + double tmp = (double)va_arg(argList, double); + textLen = SecFormatDboule(formatBuf.str, &formatAttr, fFmtStr, tmp); +#endif + } else { + double tmp = (double)va_arg(argList, double); + textLen = SecFormatDboule(formatBuf.str, &formatAttr, fFmtStr, tmp); + } + + if (fFmtHeap != NULL) { + /* if buffer is alloced on heap, free it */ + SECUREC_FREE(fFmtHeap); + fFmtHeap = NULL; + /* to clear e438 last value assigned not used , the compiler will + * optimize this code + */ + (void)fFmtHeap; + } + if (textLen < 0 || textLen >= bufferSize) { + /* bufferSize is large enough, just validation the return value */ + noOutput = 1; + break; + } + + /* no padding ,this variable to calculate amount of padding */ + formatAttr.fldWidth = textLen; + prefixLen = 0; /* no padding ,this variable to calculate amount of padding */ + formatAttr.flags = 0; /* clear all internal formatAttr.flags */ + break; + } SECUREC_WHILE_ZERO; + } SECUREC_WHILE_ZERO; + break; +#else + return -1; +#endif + case SECUREC_CHAR('p'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('X'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('x'): + /* unsigned lower hex output */ + digits = itoaLowerDigits; + radix = SECUREC_RADIX_HEX; + switch (ch) { + case SECUREC_CHAR('p'): + /* print a pointer */ +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; +#else + formatAttr.flags |= SECUREC_FLAG_POINTER; +#endif +#ifdef SECUREC_ON_64BITS + formatAttr.flags |= SECUREC_FLAG_I64; /* converting an int64 */ +#else + formatAttr.flags |= SECUREC_FLAG_LONG; /* converting a long */ +#endif + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) || defined(SECUREC_VXWORKS_PLATFORM)) && (!defined(SECUREC_ON_UNIX)) +#if defined(SECUREC_VXWORKS_PLATFORM) + formatAttr.precision = 1; +#else + formatAttr.precision = 0; +#endif + formatAttr.flags |= SECUREC_FLAG_ALTERNATE; /* "0x" is not default prefix in UNIX */ + break; +#else + /* not linux vxwoks */ +#if defined(_AIX) || defined(SECUREC_ON_SOLARIS) + formatAttr.precision = 1; +#else + formatAttr.precision = 2 * sizeof(void *); /* 2 precision of different systems */ +#endif +#endif + +#if defined(SECUREC_ON_UNIX) + break; +#endif + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('X'): /* fall-through */ /* FALLTHRU */ + /* unsigned upper hex output */ + digits = itoaUpperDigits; + break; + default: + break; + } + + if (formatAttr.flags & SECUREC_FLAG_ALTERNATE) { + /* alternate form means '0x' prefix */ + prefix[0] = SECUREC_CHAR('0'); + prefix[1] = (SecChar)(digits[16]); /* 16 for 'x' or 'X' */ + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) || defined(SECUREC_VXWORKS_PLATFORM)) + if (ch == 'p') { + prefix[1] = SECUREC_CHAR('x'); + } +#endif +#if defined(_AIX) || defined(SECUREC_ON_SOLARIS) + if (ch == 'p') { + prefixLen = 0; + } else { + prefixLen = SECUREC_PREFIX_LEN; + } +#else + prefixLen = SECUREC_PREFIX_LEN; +#endif + + } + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('i'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('d'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('u'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('o'): /* fall-through */ /* FALLTHRU */ + switch (ch) { + case SECUREC_CHAR('i'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('d'): /* fall-through */ /* FALLTHRU */ + /* signed decimal output */ + formatAttr.flags |= SECUREC_FLAG_SIGNED; + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('u'): + radix = SECUREC_RADIX_DECIMAL; + break; + case SECUREC_CHAR('o'): + /* unsigned octal output */ + radix = SECUREC_RADIX_OCTAL; + if (formatAttr.flags & SECUREC_FLAG_ALTERNATE) { + /* alternate form means force a leading 0 */ + formatAttr.flags |= SECUREC_FLAG_FORCE_OCTAL; + } + break; + default: + break; + } + + do { + + SecUnsignedInt64 number = 0; /* number to convert */ + SecInt64 l; /* temp long value */ + + /* read argument into variable l */ + if (formatAttr.flags & SECUREC_FLAG_I64) { + l = (SecInt64)va_arg(argList, SecInt64); + } else if (formatAttr.flags & SECUREC_FLAG_LONGLONG) { + l = (SecInt64)va_arg(argList, SecInt64); + } else +#ifdef SECUREC_ON_64BITS + if (formatAttr.flags & SECUREC_FLAG_LONG) { + l = (long)va_arg(argList, long); + } else +#endif /* SECUREC_ON_64BITS */ + if (formatAttr.flags & SECUREC_FLAG_CHAR) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = (char)va_arg(argList, int); /* sign extend */ + if (l >= 128) { /* 128 on some platform, char is always unsigned */ + SecUnsignedInt64 tmpL = (SecUnsignedInt64)l; + unsigned char tmpCh = (unsigned char)(~(tmpL)); + l = tmpCh + 1; + formatAttr.flags |= SECUREC_FLAG_NEGATIVE; + } + } else { + l = (unsigned char)va_arg(argList, int); /* zero-extend */ + } + + } else if (formatAttr.flags & SECUREC_FLAG_SHORT) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = (short)va_arg(argList, int); /* sign extend */ + } else { + l = (unsigned short)va_arg(argList, int); /* zero-extend */ + } + + } +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + else if (formatAttr.flags & SECUREC_FLAG_PTRDIFF) { + l = (ptrdiff_t)va_arg(argList, ptrdiff_t); /* sign extend */ + } else if (formatAttr.flags & SECUREC_FLAG_SIZE) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + /* No suitable macros were found to handle the branch */ + if (SecIsSameSize(sizeof(size_t), sizeof(long))) { + l = va_arg(argList, long); /* sign extend */ + } else if (SecIsSameSize(sizeof(size_t), sizeof(long long))) { + l = va_arg(argList, long long); /* sign extend */ + } else { + l = va_arg(argList, int); /* sign extend */ + } + } else { + l = (SecInt64)(size_t)va_arg(argList, size_t); /* sign extend */ + } + } else if (formatAttr.flags & SECUREC_FLAG_INTMAX) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = va_arg(argList, SecInt64); /* sign extend */ + } else { + /* sign extend */ + l = (SecInt64)(SecUnsignedInt64)va_arg(argList, SecUnsignedInt64); + } + } +#endif + else { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = va_arg(argList, int); /* sign extend */ + } else { + l = (unsigned int)va_arg(argList, int); /* zero-extend */ + } + + } + + /* check for negative; copy into number */ + if ((formatAttr.flags & SECUREC_FLAG_SIGNED) && l < 0) { + number = (SecUnsignedInt64)(-l); + formatAttr.flags |= SECUREC_FLAG_NEGATIVE; + } else { + number = (SecUnsignedInt64)l; + } + + if (((formatAttr.flags & SECUREC_FLAG_I64) == 0) && +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + ((formatAttr.flags & SECUREC_FLAG_INTMAX) == 0) && +#endif +#ifdef SECUREC_ON_64BITS + ((formatAttr.flags & SECUREC_FLAG_PTRDIFF) == 0) && + ((formatAttr.flags & SECUREC_FLAG_SIZE) == 0) && +#if !defined(SECUREC_COMPATIBLE_WIN_FORMAT) /* on window 64 system sizeof long is 32bit */ + ((formatAttr.flags & SECUREC_FLAG_LONG) == 0) && +#endif +#endif + ((formatAttr.flags & SECUREC_FLAG_LONGLONG) == 0)) { + + number &= 0xffffffff; /* use 0xffffffff as 32 bit mask */ + } + + /* check precision value for default */ + if (formatAttr.precision < 0) { + formatAttr.precision = 1; /* default precision */ + } else { +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; +#else + if (!(formatAttr.flags & SECUREC_FLAG_POINTER)) { + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; + } +#endif + if (formatAttr.precision > SECUREC_MAX_PRECISION) { + formatAttr.precision = SECUREC_MAX_PRECISION; + } + } + + /* Check if data is 0; if so, turn off hex prefix, + * 'p' add 0x prefix, otherwise not add prefix + */ + if (number == 0) { +#if !(defined(SECUREC_VXWORKS_PLATFORM) || defined(__hpux)) + prefixLen = 0; +#else + if ((ch == 'p') && (formatAttr.flags & SECUREC_FLAG_ALTERNATE)) { + prefixLen = SECUREC_PREFIX_LEN; + } else { + prefixLen = 0; + } +#endif + } + + /* Convert data to ASCII */ + formatBuf.str = &buffer.str[SECUREC_BUFFER_SIZE]; + + if (number > 0) { +#ifdef SECUREC_ON_64BITS + switch (radix) { + /* the compiler will optimize each one */ + case SECUREC_RADIX_DECIMAL: + SECUREC_SPECIAL_QWORD_BASE10(number); + break; + case SECUREC_RADIX_HEX: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_HEX); + break; + case SECUREC_RADIX_OCTAL: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_OCTAL); + break; + default: + break; + } +#else /* for 32 bits system */ + if (number <= 0xFFFFFFFFUL) { + /* in most case, the value to be converted is small value */ + SecUnsignedInt32 n32Tmp = (SecUnsignedInt32)number; + switch (radix) { + case SECUREC_RADIX_HEX: + SECUREC_SPECIAL_DWORD(n32Tmp, SECUREC_RADIX_HEX); + break; + case SECUREC_RADIX_OCTAL: + SECUREC_SPECIAL_DWORD(n32Tmp, SECUREC_RADIX_OCTAL); + break; + +#ifdef _AIX + /* the compiler will optimize div 10 */ + case SECUREC_RADIX_DECIMAL: + SECUREC_SPECIAL_DWORD(n32Tmp, SECUREC_RADIX_DECIMAL); + break; +#else + case SECUREC_RADIX_DECIMAL: + do { + /* fast div 10 */ + SecUnsignedInt32 q; + SecUnsignedInt32 r; + do { + *--formatBuf.str = digits[n32Tmp % SECUREC_RADIX_DECIMAL]; + q = (n32Tmp >> 1) + (n32Tmp >> 2); /* fast div magic 2 */ + q = q + (q >> 4); /* fast div magic 4 */ + q = q + (q >> 8); /* fast div magic 8 */ + q = q + (q >> 16); /* fast div magic 16 */ + q = q >> 3; /* fast div magic 3 */ + r = n32Tmp - SECUREC_MUL_TEN(q); + n32Tmp = (r > 9) ? (q + 1) : q; /* fast div magic 9 */ + } while (n32Tmp != 0); + } SECUREC_WHILE_ZERO; + break; +#endif + default: + break; + } /* end switch */ + } else { + /* the value to be converted is greater than 4G */ +#if defined(SECUREC_VXWORKS_VERSION_5_4) + do { + SecUnsignedInt32 digit = 0; /* ascii value of digit */ + SecUnsignedInt64 quotient = 0; + if (SecU64Div32(number,(SecUnsignedInt32)radix, "ient, &digit) != 0) { + noOutput = 1; + break; + } + *--formatBuf.str = digits[digit]; + number = quotient; + } while (number != 0); +#else + switch (radix) { + /* the compiler will optimize div 10 */ + case SECUREC_RADIX_DECIMAL: + SECUREC_SPECIAL_QWORD_BASE10(number); + break; + case SECUREC_RADIX_OCTAL: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_OCTAL); + break; + case SECUREC_RADIX_HEX: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_HEX); + break; + default: + break; + } +#endif + } +#endif + + } + /* compute length of number,.if textLen > 0, then formatBuf.str must be in buffer.str */ + textLen = (int)(size_t)((char *)&buffer.str[SECUREC_BUFFER_SIZE] - formatBuf.str); + if (formatAttr.precision > textLen) { + int ii; + for (ii = 0; ii < formatAttr.precision - textLen; ++ii) { + *--formatBuf.str = '0'; + } + textLen = formatAttr.precision; + } + + /* Force a leading zero if FORCEOCTAL flag set */ + if ((formatAttr.flags & SECUREC_FLAG_FORCE_OCTAL) && + (textLen == 0 || formatBuf.str[0] != '0')) { + *--formatBuf.str = '0'; + ++textLen; /* add a zero */ + } + } SECUREC_WHILE_ZERO; + break; + default: + break; + } + + while (noOutput < 1) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + if (formatAttr.flags & SECUREC_FLAG_NEGATIVE) { + /* prefix is a '-' */ + prefix[0] = SECUREC_CHAR('-'); + prefixLen = 1; + } else if (formatAttr.flags & SECUREC_FLAG_SIGN) { + /* prefix is '+' */ + prefix[0] = SECUREC_CHAR('+'); + prefixLen = 1; + } else if (formatAttr.flags & SECUREC_FLAG_SIGN_SPACE) { + /* prefix is ' ' */ + prefix[0] = SECUREC_CHAR(' '); + prefixLen = 1; + } + } + +#if defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && (!defined(SECUREC_ON_UNIX)) + if ((formatAttr.flags & SECUREC_FLAG_POINTER) && (textLen == 0)) { + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; + formatBuf.str = &buffer.str[SECUREC_BUFFER_SIZE - 1]; + *formatBuf.str-- = '\0'; + *formatBuf.str-- = ')'; + *formatBuf.str-- = 'l'; + *formatBuf.str-- = 'i'; + *formatBuf.str-- = 'n'; + *formatBuf.str = '('; + textLen = 5; /* length of (nil) is 5 */ + } +#endif + + /* calculate amount of padding */ + padding = (formatAttr.fldWidth - textLen) - prefixLen; + + /* put out the padding, prefix, and text, in the correct order */ + + if (!(formatAttr.flags & (SECUREC_FLAG_LEFT | SECUREC_FLAG_LEADZERO)) && padding > 0) { + /* pad on left with blanks */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, padding)) { + /* char * cast to wchar * */ + SECUREC_SAFE_PADDING(SECUREC_CHAR(' '), padding, stream, &charsOut); + } else { + SECUREC_WRITE_MULTI_CHAR(SECUREC_CHAR(' '), padding, stream, &charsOut); + } + } + + /* write prefix */ + if (prefixLen > 0) { + SecChar *pPrefix = prefix; + if (SECUREC_IS_REST_BUF_ENOUGH(stream, prefixLen)) { + /* max prefix len is 2, use loop copy */ /* char * cast to wchar * in WCHAR version */ + SECUREC_SAFE_WRITE_STR_OPT(pPrefix, prefixLen, stream, &charsOut); + } else { + SECUREC_WRITE_STRING(prefix, prefixLen, stream, &charsOut); + } + } + + if ((formatAttr.flags & SECUREC_FLAG_LEADZERO) && !(formatAttr.flags & SECUREC_FLAG_LEFT) + && padding > 0) { + /* write leading zeros */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, padding)) { + /* char * cast to wchar * */ + SECUREC_SAFE_PADDING(SECUREC_CHAR('0'), padding, stream, &charsOut); + } else { + SECUREC_WRITE_MULTI_CHAR(SECUREC_CHAR('0'), padding, stream, &charsOut); + } + } + + /* write text */ +#ifndef SECUREC_FOR_WCHAR + if (formatAttr.bufferIsWide != 0 && (textLen > 0)) { +#if SECUREC_HAVE_WCTOMB + wchar_t *p = formatBuf.wStr; + int count = textLen; + while (count > 0) { + char tmpBuf[SECUREC_MB_LEN + 1]; + SECUREC_MASK_MSVC_CRT_WARNING + int retVal = wctomb(tmpBuf, *p); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (retVal <= 0) { + charsOut = -1; + break; + } + SECUREC_WRITE_STRING(tmpBuf, retVal, stream, &charsOut); + --count; + ++p; + } +#else + charsOut = -1; + break; +#endif + } else { + if (SECUREC_IS_REST_BUF_ENOUGH(stream, textLen)) { + SECUREC_SAFE_WRITE_STR(formatBuf.str, textLen, stream, &charsOut); + } else { + SECUREC_WRITE_STRING(formatBuf.str, textLen, stream, &charsOut); + } + } +#else /* SECUREC_FOR_WCHAR */ + if (formatAttr.bufferIsWide == 0 && textLen > 0) { +#if SECUREC_HAVE_MBTOWC + int count = textLen; + char *p = formatBuf.str; + + while (count > 0) { + wchar_t wChar = L'\0'; + int retVal = mbtowc(&wChar, p, (size_t)MB_CUR_MAX); + if (retVal <= 0) { + charsOut = -1; + break; + } + SecWriteCharW(wChar, stream, &charsOut); + p += retVal; + count -= retVal; + } +#else + charsOut = -1; + break; +#endif + } else { + if (SECUREC_IS_REST_BUF_ENOUGH(stream, textLen)) { + /* char * cast to wchar * */ + SECUREC_SAFE_WRITE_STR(formatBuf.wStr, textLen, stream, &charsOut); + } else { + SECUREC_WRITE_STRING(formatBuf.wStr, textLen, stream, &charsOut); + } + } +#endif /* SECUREC_FOR_WCHAR */ + + if (charsOut >= 0 && (formatAttr.flags & SECUREC_FLAG_LEFT) && padding > 0) { + /* pad on right with blanks */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, padding)) { + /* char * cast to wchar * */ + SECUREC_SAFE_PADDING(SECUREC_CHAR(' '), padding, stream, &charsOut); + } else { + SECUREC_WRITE_MULTI_CHAR(SECUREC_CHAR(' '), padding, stream, &charsOut); + } + } + break; + } +#if SECUREC_ENABLE_SPRINTF_FLOAT + if (floatBuf != NULL) { + SECUREC_FREE(floatBuf); + floatBuf = NULL; + } +#endif + break; + case STAT_INVALID: + return -1; + default: + return -1; /* input format is wrong, directly return */ + } + } + + if (state != STAT_NORMAL && state != STAT_TYPE) { + return -1; + } + + return charsOut; /* the number of characters written */ +} +#endif /* OUTPUT_INL_2B263E9C_43D8_44BB_B17A_6D2033DECEE5 */ + diff --git a/third_party/securec/src/scanf_s.c b/third_party/securec/src/scanf_s.c new file mode 100644 index 00000000000..e4b0e602483 --- /dev/null +++ b/third_party/securec/src/scanf_s.c @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The scanf_s function is equivalent to fscanf_s with the argument stdin interposed before the arguments to scanf_s + * The scanf_s function reads data from the standard input stream stdin and + * writes the data into the location that's given by argument. Each argument + * must be a pointer to a variable of a type that corresponds to a type specifier + * in format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * ... Optional arguments. + * + * + * ... The converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ + +int scanf_s(const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vscanf_s(format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/secinput.h b/third_party/securec/src/secinput.h new file mode 100644 index 00000000000..8cd9284966b --- /dev/null +++ b/third_party/securec/src/secinput.h @@ -0,0 +1,156 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SEC_INPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#define SEC_INPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#include "securecutil.h" + +#define SECUREC_SCANF_EINVAL (-1) +#define SECUREC_SCANF_ERROR_PARA (-2) + +/* for internal stream flag */ +#define SECUREC_MEM_STR_FLAG 0X01 +#define SECUREC_FILE_STREAM_FLAG 0X02 +#define SECUREC_FROM_STDIN_FLAG 0X04 +#define SECUREC_LOAD_FILE_TO_MEM_FLAG 0X08 + +#define SECUREC_UNINITIALIZED_FILE_POS (-1) +#define SECUREC_BOM_HEADER_SIZE 2 +#define SECUREC_BOM_HEADER_BE_1ST 0xFEU +#define SECUREC_BOM_HEADER_BE_2ST 0xFFU +#define SECUREC_BOM_HEADER_LE_1ST 0xFFU +#define SECUREC_BOM_HEADER_LE_2ST 0xFEU +#define SECUREC_UTF8_BOM_HEADER_SIZE 3 +#define SECUREC_UTF8_BOM_HEADER_1ST 0xEFU +#define SECUREC_UTF8_BOM_HEADER_2ND 0xBBU +#define SECUREC_UTF8_BOM_HEADER_3RD 0xBFU +#define SECUREC_UTF8_LEAD_1ST 0xE0 +#define SECUREC_UTF8_LEAD_2ND 0x80 + +typedef struct { + unsigned int flag; /* mark the properties of input stream */ + int count; /* the size of buffered string in bytes */ + const char *cur; /* the pointer to next read position */ + char *base; /* the pointer to the header of buffered string */ +#if SECUREC_ENABLE_SCANF_FILE + FILE *pf; /* the file pointer */ + long oriFilePos; /* the original position of file offset when fscanf is called */ + int fileRealRead; +#if defined(SECUREC_NO_STD_UNGETC) + unsigned int lastChar; /* the char code of last input */ + int fUnget; /* the boolean flag of pushing a char back to read stream */ +#endif +#endif +} SecFileStream; + + +#define SECUREC_INIT_SEC_FILE_STREAM_COMMON(fileStream, streamFlag, curPtr, strCount) do { \ + (fileStream).flag = (streamFlag); \ + (fileStream).count = (strCount); \ + (fileStream).cur = (curPtr); \ + (fileStream).base = NULL; \ +} SECUREC_WHILE_ZERO + +#if SECUREC_ENABLE_SCANF_FILE +#if defined(SECUREC_NO_STD_UNGETC) +/* This initialization for eliminating redundant initialization. + * Compared with the previous version initialization 0, + * the current code causes the binary size to increase by some bytes + */ +#define SECUREC_INIT_SEC_FILE_STREAM(fileStream, streamFlag, stream, filePos, curPtr, strCount) do { \ + SECUREC_INIT_SEC_FILE_STREAM_COMMON((fileStream), (streamFlag), (curPtr), (strCount)); \ + (fileStream).pf = (stream); \ + (fileStream).oriFilePos = (filePos); \ + (fileStream).fileRealRead = 0; \ + (fileStream).lastChar = 0; \ + (fileStream).fUnget = 0; \ +} SECUREC_WHILE_ZERO +#else +#define SECUREC_INIT_SEC_FILE_STREAM(fileStream, streamFlag, stream, filePos, curPtr, strCount) do { \ + SECUREC_INIT_SEC_FILE_STREAM_COMMON((fileStream), (streamFlag), (curPtr), (strCount)); \ + (fileStream).pf = (stream); \ + (fileStream).oriFilePos = (filePos); \ + (fileStream).fileRealRead = 0; \ +} SECUREC_WHILE_ZERO +#endif +#else /* No SECUREC_ENABLE_SCANF_FILE */ +#define SECUREC_INIT_SEC_FILE_STREAM(fileStream, streamFlag, stream, filePos, curPtr, strCount) do { \ + SECUREC_INIT_SEC_FILE_STREAM_COMMON((fileStream), (streamFlag), (curPtr), (strCount)); \ +} SECUREC_WHILE_ZERO +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + extern int SecInputS(SecFileStream *stream, const char *cFormat, va_list argList); + extern void SecClearDestBuf(const char *buffer, const char *format, va_list argList); +#if SECUREC_IN_KERNEL == 0 + extern int SecInputSW(SecFileStream *stream, const wchar_t *cFormat, va_list argList); + extern void SecClearDestBufW(const wchar_t *buffer, const wchar_t *format, va_list argList); +#endif +/* 20150105 For software and hardware decoupling,such as UMG */ +#if defined(SECUREC_SYSAPI4VXWORKS) +#ifdef feof +#undef feof +#endif + extern int feof(FILE *stream); +#endif + +#if defined(SECUREC_SYSAPI4VXWORKS) || defined(SECUREC_CTYPE_MACRO_ADAPT) +#ifndef isspace +#define isspace(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\r') || ((c) == '\n')) +#endif +#ifndef iswspace +#define iswspace(c) (((c) == L' ') || ((c) == L'\t') || ((c) == L'\r') || ((c) == L'\n')) +#endif +#ifndef isascii +#define isascii(c) (((unsigned char)(c)) <= 0x7f) +#endif +#ifndef isupper +#define isupper(c) ((c) >= 'A' && (c) <= 'Z') +#endif +#ifndef islower +#define islower(c) ((c) >= 'a' && (c) <= 'z') +#endif +#ifndef isalpha +#define isalpha(c) (isupper(c) || (islower(c))) +#endif +#ifndef isdigit +#define isdigit(c) ((c) >= '0' && (c) <= '9') +#endif +#ifndef isxupper +#define isxupper(c) ((c) >= 'A' && (c) <= 'F') +#endif +#ifndef isxlower +#define isxlower(c) ((c) >= 'a' && (c) <= 'f') +#endif +#ifndef isxdigit +#define isxdigit(c) (isdigit(c) || isxupper(c) || isxlower(c)) +#endif +#endif + +#ifdef __cplusplus +} +#endif +/* Reserved file operation macro interface */ +#define SECUREC_LOCK_FILE(s) +#define SECUREC_UNLOCK_FILE(s) +#define SECUREC_LOCK_STDIN(i, s) +#define SECUREC_UNLOCK_STDIN(i, s) +#endif + + diff --git a/third_party/securec/src/securecutil.c b/third_party/securec/src/securecutil.c new file mode 100644 index 00000000000..1a44cfbe4f6 --- /dev/null +++ b/third_party/securec/src/securecutil.c @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Avoid duplicate header files,not include securecutil.h */ +#include "securecutil.h" + + +#if defined(ANDROID) && (SECUREC_HAVE_WCTOMB || SECUREC_HAVE_MBTOWC) +#include +#if SECUREC_HAVE_WCTOMB +/* + * Convert wide characters to narrow multi-bytes + */ +int wctomb(char *s, wchar_t wc) +{ + return wcrtomb(s, wc, NULL); +} +#endif + +#if SECUREC_HAVE_MBTOWC +/* + * Converting narrow multi-byte characters to wide characters + */ +int mbtowc(wchar_t *pwc, const char *s, size_t n) +{ + return mbrtowc(pwc, s, n, NULL); +} +#endif +#endif + +/* high Num << 8 | num of SPC Ver */ +#define SECUREC_C_VERSION (0x5 << 8) +#define SECUREC_SPC_VERSION 7 +#define SECUREC_VERSION_STR "Huawei Secure C V100R001C01SPC007B002" + +/* SPC verNumber<->verStr like: + * 0X201<->C01 + * 0X202<->SPC001 Redefine numbers after this version + * 0X502<->SPC002 + * 0X503<->SPC003 + * ... + * 0X50a<->SPC010 + * 0X50b<->SPC011 + * ... + */ +/* CP verNumber<->verStr like: + * 0X601<->CP0001 + * 0X602<->CP0002 + * ... + */ +const char *GetHwSecureCVersion(unsigned short *verNumber) +{ + if (verNumber != NULL) { + *verNumber = (unsigned short)(SECUREC_C_VERSION | SECUREC_SPC_VERSION); + } + return SECUREC_VERSION_STR; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(GetHwSecureCVersion); +#endif + diff --git a/third_party/securec/src/securecutil.h b/third_party/securec/src/securecutil.h new file mode 100644 index 00000000000..98c9aad0985 --- /dev/null +++ b/third_party/securec/src/securecutil.h @@ -0,0 +1,541 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SECURECUTIL_H_46C86578_F8FF_4E49_8E64_9B175241761F +#define SECURECUTIL_H_46C86578_F8FF_4E49_8E64_9B175241761F +#include "securec.h" + +#if (defined(_MSC_VER)) && (_MSC_VER >= 1400) +#define SECUREC_MASK_MSVC_CRT_WARNING __pragma(warning(push)) \ + __pragma(warning(disable:4996 4127)) +#define SECUREC_END_MASK_MSVC_CRT_WARNING __pragma(warning(pop)) +#else +#define SECUREC_MASK_MSVC_CRT_WARNING +#define SECUREC_END_MASK_MSVC_CRT_WARNING +#endif +#define SECUREC_WHILE_ZERO SECUREC_MASK_MSVC_CRT_WARNING while (0) SECUREC_END_MASK_MSVC_CRT_WARNING + +#ifndef SECUREC_HAVE_STRNLEN +#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 700) || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200809L) +#if SECUREC_IN_KERNEL +#define SECUREC_HAVE_STRNLEN 0 +#else +#if defined(__GLIBC__) && __GLIBC__ >= 2 && defined(__GLIBC_MINOR__) && __GLIBC_MINOR__ >= 10 +#define SECUREC_HAVE_STRNLEN 1 +#else +#define SECUREC_HAVE_STRNLEN 0 +#endif +#endif +#else +#define SECUREC_HAVE_STRNLEN 0 +#endif +#endif + +#if SECUREC_IN_KERNEL +/* in kernel disbale functions */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 0 +#endif +#ifndef SECUREC_ENABLE_SCANF_FLOAT +#define SECUREC_ENABLE_SCANF_FLOAT 0 +#endif +#ifndef SECUREC_ENABLE_SPRINTF_FLOAT +#define SECUREC_ENABLE_SPRINTF_FLOAT 0 +#endif +#ifndef SECUREC_HAVE_MBTOWC +#define SECUREC_HAVE_MBTOWC 0 +#endif +#ifndef SECUREC_HAVE_WCTOMB +#define SECUREC_HAVE_WCTOMB 0 +#endif +#ifndef SECUREC_HAVE_WCHART +#define SECUREC_HAVE_WCHART 0 +#endif +#else /* no in kernel */ +/* Systems that do not support file, can define this macro to 0. */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 1 +#endif +#ifndef SECUREC_ENABLE_SCANF_FLOAT +#define SECUREC_ENABLE_SCANF_FLOAT 1 +#endif +/* Systems that do not support float, can define this macro to 0. */ +#ifndef SECUREC_ENABLE_SPRINTF_FLOAT +#define SECUREC_ENABLE_SPRINTF_FLOAT 1 +#endif +#ifndef SECUREC_HAVE_MBTOWC +#define SECUREC_HAVE_MBTOWC 1 +#endif +#ifndef SECUREC_HAVE_WCTOMB +#define SECUREC_HAVE_WCTOMB 1 +#endif +#ifndef SECUREC_HAVE_WCHART +#define SECUREC_HAVE_WCHART 1 +#endif +#endif + + +#define SECUREC_INT_MAX 2147483647 +#define SECUREC_MUL_SIXTEEN(x) ((x) << 4) +#define SECUREC_MUL_EIGHT(x) ((x) << 3) +#define SECUREC_MUL_TEN(x) ((((x) << 2) + (x)) << 1) +/* Limited format input and output width */ +#define SECUREC_MAX_WIDTH_LEN_DIV_TEN 21474836 +#define SECUREC_MAX_WIDTH_LEN SECUREC_MUL_TEN(SECUREC_MAX_WIDTH_LEN_DIV_TEN) +/* Is the x multiplied by 10 greater than */ +#define SECUREC_MUL_TEN_ADD_BEYOND_MAX(x) (((x) > SECUREC_MAX_WIDTH_LEN_DIV_TEN)) + +#define SECUREC_FLOAT_BUFSIZE (309 + 40) /* Max length of double value */ +#define SECUREC_FLOAT_BUFSIZE_LB (4932 + 40) /* Max length of long double value */ +#define SECUREC_FLOAT_DEFAULT_PRECISION 6 + +/* This macro does not handle pointer equality or integer overflow */ +#define SECUREC_MEMORY_NO_OVERLAP(dest, src, count) \ + (((src) < (dest) && ((const char *)(src) + (count)) <= (char *)(dest)) || \ + ((dest) < (src) && ((char *)(dest) + (count)) <= (const char *)(src))) + +#define SECUREC_MEMORY_IS_OVERLAP(dest, src, count) \ + (((src) < (dest) && ((const char *)(src) + (count)) > (char *)(dest)) || \ + ((dest) < (src) && ((char *)(dest) + (count)) > (const char *)(src))) + +/* + * Check whether the strings overlap, len is the length of the string not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_STRING_NO_OVERLAP(dest, src, len) \ + (((src) < (dest) && ((src) + (len)) < (dest)) || \ + ((dest) < (src) && ((dest) + (len)) < (src))) + +/* + * Check whether the strings overlap for strcpy wcscpy function, dest len and src Len are not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_STRING_IS_OVERLAP(dest, src, len) \ + (((src) < (dest) && ((src) + (len)) >= (dest)) || \ + ((dest) < (src) && ((dest) + (len)) >= (src))) + +/* + * Check whether the strings overlap for strcat wcscat function, dest len and src Len are not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_CAT_STRING_IS_OVERLAP(dest, destLen, src, srcLen) \ + (((dest) < (src) && ((dest) + (destLen) + (srcLen)) >= (src)) || \ + ((src) < (dest) && ((src) + (srcLen)) >= (dest))) + + +#if SECUREC_HAVE_STRNLEN +#define SECUREC_CALC_STR_LEN(str, maxLen, outLen) do { \ + *(outLen) = strnlen((str), (maxLen)); \ +} SECUREC_WHILE_ZERO +#define SECUREC_CALC_STR_LEN_OPT(str, maxLen, outLen) do { \ + if ((maxLen) > 8) { \ + /* Optimization or len less then 8 */ \ + if (*((str) + 0) == '\0') { \ + *(outLen) = 0; \ + } else if (*((str) + 1) == '\0') { \ + *(outLen) = 1; \ + } else if (*((str) + 2) == '\0') { \ + *(outLen) = 2; \ + } else if (*((str) + 3) == '\0') { \ + *(outLen) = 3; \ + } else if (*((str) + 4) == '\0') { \ + *(outLen) = 4; \ + } else if (*((str) + 5) == '\0') { \ + *(outLen) = 5; \ + } else if (*((str) + 6) == '\0') { \ + *(outLen) = 6; \ + } else if (*((str) + 7) == '\0') { \ + *(outLen) = 7; \ + } else if (*((str) + 8) == '\0') { \ + /* Optimization with a length of 8 */ \ + *(outLen) = 8; \ + } else { \ + /* The offset is 8 because the performance of 8 byte alignment is high */ \ + *(outLen) = 8 + strnlen((str) + 8, (maxLen) - 8); \ + } \ + } else { \ + SECUREC_CALC_STR_LEN((str), (maxLen), (outLen)); \ + } \ +} SECUREC_WHILE_ZERO +#else +#define SECUREC_CALC_STR_LEN(str, maxLen, outLen) do { \ + const char *strEnd = (const char *)(str); \ + size_t availableSize = (size_t)(maxLen); \ + while (availableSize > 0 && *strEnd != '\0') { \ + --availableSize; \ + ++strEnd; \ + } \ + *(outLen) = (size_t)(strEnd - (str)); \ +} SECUREC_WHILE_ZERO +#define SECUREC_CALC_STR_LEN_OPT SECUREC_CALC_STR_LEN +#endif + +#define SECUREC_CALC_WSTR_LEN(str, maxLen, outLen) do { \ + const wchar_t *strEnd = (const wchar_t *)(str); \ + *(outLen) = 0; \ + while (*(outLen) < (maxLen) && *strEnd != L'\0') { \ + *(outLen) = *(outLen) + 1; \ + ++strEnd; \ + } \ +} SECUREC_WHILE_ZERO + + +#ifdef SECUREC_FORMAT_OUTPUT_INPUT +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) || defined(__ARMCC_VERSION) +typedef __int64 SecInt64; +typedef unsigned __int64 SecUnsignedInt64; +#if defined(__ARMCC_VERSION) +typedef unsigned int SecUnsignedInt32; +#else +typedef unsigned __int32 SecUnsignedInt32; +#endif +#else +typedef unsigned int SecUnsignedInt32; +typedef long long SecInt64; +typedef unsigned long long SecUnsignedInt64; +#endif + +#ifdef SECUREC_FOR_WCHAR +#if defined(SECUREC_VXWORKS_PLATFORM) && !defined(__WINT_TYPE__) +typedef wchar_t wint_t; +#endif +typedef wchar_t SecChar; +typedef wchar_t SecUnsignedChar; +typedef wint_t SecInt; +typedef wint_t SecUnsignedInt; +#else /* no SECUREC_FOR_WCHAR */ +typedef char SecChar; +typedef unsigned char SecUnsignedChar; +typedef int SecInt; +typedef unsigned int SecUnsignedInt; +#endif +#endif + +/* Determine whether the address is 8-byte aligned + * Some systems do not have uintptr_t type, so use NULL to clear tool alarm 507 + */ +#define SECUREC_ADDR_ALIGNED_8(addr) (SecIsAddrAligned8((addr), NULL) == 0) + +/* If you define the memory allocation function, + * you need to define the function prototype. You can define this macro as a header file. + */ +#if defined(SECUREC_MALLOC_PROTOTYPE) +SECUREC_MALLOC_PROTOTYPE +#endif + +#ifndef SECUREC_MALLOC +#define SECUREC_MALLOC(x) malloc((size_t)(x)) +#endif + +#ifndef SECUREC_FREE +#define SECUREC_FREE(x) free((void *)(x)) +#endif + +/* struct for performance */ +typedef struct { + unsigned char buf[1]; /* Performance optimization code structure assignment length 1 bytes */ +} SecStrBuf1; +typedef struct { + unsigned char buf[2]; /* Performance optimization code structure assignment length 2 bytes */ +} SecStrBuf2; +typedef struct { + unsigned char buf[3]; /* Performance optimization code structure assignment length 3 bytes */ +} SecStrBuf3; +typedef struct { + unsigned char buf[4]; /* Performance optimization code structure assignment length 4 bytes */ +} SecStrBuf4; +typedef struct { + unsigned char buf[5]; /* Performance optimization code structure assignment length 5 bytes */ +} SecStrBuf5; +typedef struct { + unsigned char buf[6]; /* Performance optimization code structure assignment length 6 bytes */ +} SecStrBuf6; +typedef struct { + unsigned char buf[7]; /* Performance optimization code structure assignment length 7 bytes */ +} SecStrBuf7; +typedef struct { + unsigned char buf[8]; /* Performance optimization code structure assignment length 8 bytes */ +} SecStrBuf8; +typedef struct { + unsigned char buf[9]; /* Performance optimization code structure assignment length 9 bytes */ +} SecStrBuf9; +typedef struct { + unsigned char buf[10]; /* Performance optimization code structure assignment length 10 bytes */ +} SecStrBuf10; +typedef struct { + unsigned char buf[11]; /* Performance optimization code structure assignment length 11 bytes */ +} SecStrBuf11; +typedef struct { + unsigned char buf[12]; /* Performance optimization code structure assignment length 12 bytes */ +} SecStrBuf12; +typedef struct { + unsigned char buf[13]; /* Performance optimization code structure assignment length 13 bytes */ +} SecStrBuf13; +typedef struct { + unsigned char buf[14]; /* Performance optimization code structure assignment length 14 bytes */ +} SecStrBuf14; +typedef struct { + unsigned char buf[15]; /* Performance optimization code structure assignment length 15 bytes */ +} SecStrBuf15; +typedef struct { + unsigned char buf[16]; /* Performance optimization code structure assignment length 16 bytes */ +} SecStrBuf16; +typedef struct { + unsigned char buf[17]; /* Performance optimization code structure assignment length 17 bytes */ +} SecStrBuf17; +typedef struct { + unsigned char buf[18]; /* Performance optimization code structure assignment length 18 bytes */ +} SecStrBuf18; +typedef struct { + unsigned char buf[19]; /* Performance optimization code structure assignment length 19 bytes */ +} SecStrBuf19; +typedef struct { + unsigned char buf[20]; /* Performance optimization code structure assignment length 20 bytes */ +} SecStrBuf20; +typedef struct { + unsigned char buf[21]; /* Performance optimization code structure assignment length 21 bytes */ +} SecStrBuf21; +typedef struct { + unsigned char buf[22]; /* Performance optimization code structure assignment length 22 bytes */ +} SecStrBuf22; +typedef struct { + unsigned char buf[23]; /* Performance optimization code structure assignment length 23 bytes */ +} SecStrBuf23; +typedef struct { + unsigned char buf[24]; /* Performance optimization code structure assignment length 24 bytes */ +} SecStrBuf24; +typedef struct { + unsigned char buf[25]; /* Performance optimization code structure assignment length 25 bytes */ +} SecStrBuf25; +typedef struct { + unsigned char buf[26]; /* Performance optimization code structure assignment length 26 bytes */ +} SecStrBuf26; +typedef struct { + unsigned char buf[27]; /* Performance optimization code structure assignment length 27 bytes */ +} SecStrBuf27; +typedef struct { + unsigned char buf[28]; /* Performance optimization code structure assignment length 28 bytes */ +} SecStrBuf28; +typedef struct { + unsigned char buf[29]; /* Performance optimization code structure assignment length 29 bytes */ +} SecStrBuf29; +typedef struct { + unsigned char buf[30]; /* Performance optimization code structure assignment length 30 bytes */ +} SecStrBuf30; +typedef struct { + unsigned char buf[31]; /* Performance optimization code structure assignment length 31 bytes */ +} SecStrBuf31; +typedef struct { + unsigned char buf[32]; /* Performance optimization code structure assignment length 32 bytes */ +} SecStrBuf32; +typedef struct { + unsigned char buf[33]; /* Performance optimization code structure assignment length 33 bytes */ +} SecStrBuf33; +typedef struct { + unsigned char buf[34]; /* Performance optimization code structure assignment length 34 bytes */ +} SecStrBuf34; +typedef struct { + unsigned char buf[35]; /* Performance optimization code structure assignment length 35 bytes */ +} SecStrBuf35; +typedef struct { + unsigned char buf[36]; /* Performance optimization code structure assignment length 36 bytes */ +} SecStrBuf36; +typedef struct { + unsigned char buf[37]; /* Performance optimization code structure assignment length 37 bytes */ +} SecStrBuf37; +typedef struct { + unsigned char buf[38]; /* Performance optimization code structure assignment length 38 bytes */ +} SecStrBuf38; +typedef struct { + unsigned char buf[39]; /* Performance optimization code structure assignment length 39 bytes */ +} SecStrBuf39; +typedef struct { + unsigned char buf[40]; /* Performance optimization code structure assignment length 40 bytes */ +} SecStrBuf40; +typedef struct { + unsigned char buf[41]; /* Performance optimization code structure assignment length 41 bytes */ +} SecStrBuf41; +typedef struct { + unsigned char buf[42]; /* Performance optimization code structure assignment length 42 bytes */ +} SecStrBuf42; +typedef struct { + unsigned char buf[43]; /* Performance optimization code structure assignment length 43 bytes */ +} SecStrBuf43; +typedef struct { + unsigned char buf[44]; /* Performance optimization code structure assignment length 44 bytes */ +} SecStrBuf44; +typedef struct { + unsigned char buf[45]; /* Performance optimization code structure assignment length 45 bytes */ +} SecStrBuf45; +typedef struct { + unsigned char buf[46]; /* Performance optimization code structure assignment length 46 bytes */ +} SecStrBuf46; +typedef struct { + unsigned char buf[47]; /* Performance optimization code structure assignment length 47 bytes */ +} SecStrBuf47; +typedef struct { + unsigned char buf[48]; /* Performance optimization code structure assignment length 48 bytes */ +} SecStrBuf48; +typedef struct { + unsigned char buf[49]; /* Performance optimization code structure assignment length 49 bytes */ +} SecStrBuf49; +typedef struct { + unsigned char buf[50]; /* Performance optimization code structure assignment length 50 bytes */ +} SecStrBuf50; +typedef struct { + unsigned char buf[51]; /* Performance optimization code structure assignment length 51 bytes */ +} SecStrBuf51; +typedef struct { + unsigned char buf[52]; /* Performance optimization code structure assignment length 52 bytes */ +} SecStrBuf52; +typedef struct { + unsigned char buf[53]; /* Performance optimization code structure assignment length 53 bytes */ +} SecStrBuf53; +typedef struct { + unsigned char buf[54]; /* Performance optimization code structure assignment length 54 bytes */ +} SecStrBuf54; +typedef struct { + unsigned char buf[55]; /* Performance optimization code structure assignment length 55 bytes */ +} SecStrBuf55; +typedef struct { + unsigned char buf[56]; /* Performance optimization code structure assignment length 56 bytes */ +} SecStrBuf56; +typedef struct { + unsigned char buf[57]; /* Performance optimization code structure assignment length 57 bytes */ +} SecStrBuf57; +typedef struct { + unsigned char buf[58]; /* Performance optimization code structure assignment length 58 bytes */ +} SecStrBuf58; +typedef struct { + unsigned char buf[59]; /* Performance optimization code structure assignment length 59 bytes */ +} SecStrBuf59; +typedef struct { + unsigned char buf[60]; /* Performance optimization code structure assignment length 60 bytes */ +} SecStrBuf60; +typedef struct { + unsigned char buf[61]; /* Performance optimization code structure assignment length 61 bytes */ +} SecStrBuf61; +typedef struct { + unsigned char buf[62]; /* Performance optimization code structure assignment length 62 bytes */ +} SecStrBuf62; +typedef struct { + unsigned char buf[63]; /* Performance optimization code structure assignment length 63 bytes */ +} SecStrBuf63; +typedef struct { + unsigned char buf[64]; /* Performance optimization code structure assignment length 64 bytes */ +} SecStrBuf64; + + + + +/* User can change the error handler by modify the following definition, + * such as logging the detail error in file. + */ +#if defined(_DEBUG) || defined(DEBUG) +#if defined(SECUREC_ERROR_HANDLER_BY_ASSERT) +#define SECUREC_ERROR_INVALID_PARAMTER(msg) assert(msg "invalid argument" == NULL) +#define SECUREC_ERROR_INVALID_RANGE(msg) assert(msg "invalid dest buffer size" == NULL) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) assert(msg "buffer overlap" == NULL) +#elif defined(SECUREC_ERROR_HANDLER_BY_PRINTF) +#if SECUREC_IN_KERNEL +#define SECUREC_ERROR_INVALID_PARAMTER(msg) printk("%s invalid argument\n", msg) +#define SECUREC_ERROR_INVALID_RANGE(msg) printk("%s invalid dest buffer size\n", msg) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) printk("%s buffer overlap\n", msg) +#else +#define SECUREC_ERROR_INVALID_PARAMTER(msg) printf("%s invalid argument\n", msg) +#define SECUREC_ERROR_INVALID_RANGE(msg) printf("%s invalid dest buffer size\n", msg) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) printf("%s buffer overlap\n", msg) +#endif +#elif defined(SECUREC_ERROR_HANDLER_BY_FILE_LOG) +#define SECUREC_ERROR_INVALID_PARAMTER(msg) LogSecureCRuntimeError(msg " EINVAL\n") +#define SECUREC_ERROR_INVALID_RANGE(msg) LogSecureCRuntimeError(msg " ERANGE\n") +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) LogSecureCRuntimeError(msg " EOVERLAP\n") +#else /* no HANDLER is defined */ +#define SECUREC_ERROR_INVALID_PARAMTER(msg) ((void)0) +#define SECUREC_ERROR_INVALID_RANGE(msg) ((void)0) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) ((void)0) +#endif +#else /* no DEBUG */ +#define SECUREC_ERROR_INVALID_PARAMTER(msg) ((void)0) +#define SECUREC_ERROR_INVALID_RANGE(msg) ((void)0) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) ((void)0) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* assembly language memory copy and memory set for X86 or MIPS ... */ +#ifdef SECUREC_USE_ASM + extern void *memcpy_opt(void *, const void *, size_t); + extern void *memset_opt(void *, int, size_t); +#endif + +#if defined(SECUREC_ERROR_HANDLER_BY_FILE_LOG) + extern void LogSecureCRuntimeError(const char *errDetail); +#endif + +#ifdef SECUREC_INLINE_DO_MEMCPY +static void SecDoMemcpy(void *dest, const void *src, size_t count) +{ + /* + * if SECUREC_USE_ASM macro is enabled, it will call assembly language function to improve performance. + */ +#ifdef SECUREC_USE_ASM + (void)memcpy_opt(dest, src, count); +#else + /* large enough, let system API do it */ + (void)memcpy(dest, src, count); +#endif +} +#endif + +#ifdef SECUREC_INLINE_DO_MEMSET +static void SecDoMemset(void *dest, int c, size_t count) +{ +#ifdef SECUREC_USE_ASM + (void)memset_opt(dest, c, count); +#else + (void)memset(dest, c, count); +#endif +} +#endif + +#ifdef SECUREC_INLINE_STR_LEN +/* The function compiler will be inlined and not placed in other files */ +static size_t SecStrMinLen(const char *str, size_t maxLen) +{ + size_t len; + SECUREC_CALC_STR_LEN(str, maxLen, &len); + return len; +} +#endif + +#ifdef SECUREC_INLINE_STR_LEN_OPT +/* The function compiler will be inlined and not placed in other files */ +static size_t SecStrMinLenOpt(const char *str, size_t maxLen) +{ + size_t len; + SECUREC_CALC_STR_LEN_OPT(str, maxLen, &len); + return len; +} +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif + diff --git a/third_party/securec/src/secureinput_a.c b/third_party/securec/src/secureinput_a.c new file mode 100644 index 00000000000..4f9bae83313 --- /dev/null +++ b/third_party/securec/src/secureinput_a.c @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifdef SECUREC_FOR_WCHAR +#undef SECUREC_FOR_WCHAR +#endif + +#include "secinput.h" + +#include "input.inl" + diff --git a/third_party/securec/src/secureinput_w.c b/third_party/securec/src/secureinput_w.c new file mode 100644 index 00000000000..7a4bef425f1 --- /dev/null +++ b/third_party/securec/src/secureinput_w.c @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* if some platforms don't have wchar.h, dont't include it */ +#if !(defined(SECUREC_VXWORKS_PLATFORM)) +/* This header file is placed below secinput.h, which will cause tool alarm, + * but If there is no macro above, it will cause vs2010 compiling alarm + */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#ifndef __STDC_WANT_SECURE_LIB__ +/* The order of adjustment is to eliminate alarm of Duplicate Block */ +#define __STDC_WANT_SECURE_LIB__ 0 +#endif +#ifndef _CRTIMP_ALTERNATIVE +#define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ +#endif +#endif +#include +#endif +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifndef SECUREC_FOR_WCHAR +#define SECUREC_FOR_WCHAR +#endif + +#include "secinput.h" + +#ifndef WEOF +#define WEOF ((wchar_t)(-1)) +#endif + +#include "input.inl" + diff --git a/third_party/securec/src/secureprintoutput.h b/third_party/securec/src/secureprintoutput.h new file mode 100644 index 00000000000..b690ec92048 --- /dev/null +++ b/third_party/securec/src/secureprintoutput.h @@ -0,0 +1,98 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SECUREPRINTOUTPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#define SECUREPRINTOUTPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#include "securecutil.h" + +/* flag definitions */ +/* Using macros instead of enumerations is because some of the enumerated types under the compiler are 16bit. */ +#define SECUREC_FLAG_SIGN 0x00001U +#define SECUREC_FLAG_SIGN_SPACE 0x00002U +#define SECUREC_FLAG_LEFT 0x00004U +#define SECUREC_FLAG_LEADZERO 0x00008U +#define SECUREC_FLAG_LONG 0x00010U +#define SECUREC_FLAG_SHORT 0x00020U +#define SECUREC_FLAG_SIGNED 0x00040U +#define SECUREC_FLAG_ALTERNATE 0x00080U +#define SECUREC_FLAG_NEGATIVE 0x00100U +#define SECUREC_FLAG_FORCE_OCTAL 0x00200U +#define SECUREC_FLAG_LONG_DOUBLE 0x00400U +#define SECUREC_FLAG_WIDECHAR 0x00800U +#define SECUREC_FLAG_LONGLONG 0x01000U +#define SECUREC_FLAG_CHAR 0x02000U +#define SECUREC_FLAG_POINTER 0x04000U +#define SECUREC_FLAG_I64 0x08000U +#define SECUREC_FLAG_PTRDIFF 0x10000U +#define SECUREC_FLAG_SIZE 0x20000U +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +#define SECUREC_FLAG_INTMAX 0x40000U +#endif + +/* state definitions. Identify the status of the current format */ +typedef enum { + STAT_NORMAL, + STAT_PERCENT, + STAT_FLAG, + STAT_WIDTH, + STAT_DOT, + STAT_PRECIS, + STAT_SIZE, + STAT_TYPE, + STAT_INVALID +} SecFmtState; + +/* Format output buffer pointer and available size */ +typedef struct { + int count; + char *cur; +} SecPrintfStream; + + +#ifndef SECUREC_BUFFER_SIZE +#ifdef SECUREC_STACK_SIZE_LESS_THAN_1K +/* SECUREC_BUFFER_SIZE Can not be less than 23 , + * the length of the octal representation of 64-bit integers with zero lead + */ +#define SECUREC_BUFFER_SIZE 256 +#else +#define SECUREC_BUFFER_SIZE 512 +#endif +#endif +#if SECUREC_BUFFER_SIZE < 23 +#error SECUREC_BUFFER_SIZE Can not be less than 23 +#endif + +#define SECUREC_MAX_PRECISION SECUREC_BUFFER_SIZE +/* max. # bytes in multibyte char ,see MB_LEN_MAX */ +#define SECUREC_MB_LEN 16 +/* The return value of the internal function, which is returned when truncated */ +#define SECUREC_PRINTF_TRUNCATE (-2) + +#ifdef __cplusplus +extern "C" { +#endif + extern int SecVsnprintfImpl(char *string, size_t count, const char *format, va_list argList); +#if SECUREC_IN_KERNEL == 0 + extern int SecVswprintfImpl(wchar_t *string, size_t sizeInWchar, const wchar_t *format, va_list argList); +#endif +#ifdef __cplusplus +} +#endif + +#endif + + diff --git a/third_party/securec/src/secureprintoutput_a.c b/third_party/securec/src/secureprintoutput_a.c new file mode 100644 index 00000000000..746878a1903 --- /dev/null +++ b/third_party/securec/src/secureprintoutput_a.c @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMCPY 1 +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifdef SECUREC_FOR_WCHAR +#undef SECUREC_FOR_WCHAR +#endif + +#include "secureprintoutput.h" + +#define SECUREC_CHAR(x) x +#define SECUREC_WRITE_MULTI_CHAR SecWriteMultiChar +#define SECUREC_WRITE_STRING SecWriteString + +#ifndef EOF +#define EOF (-1) +#endif + +/* put a char to output */ +#define SECUREC_PUTC(c, outStream) ((--(outStream)->count >= 0) ? \ + (int)((unsigned int)(unsigned char)(*((outStream)->cur++) = (char)(c)) & 0xff) : EOF) +/* to clear e835 */ +#define SECUREC_PUTC_ZERO(outStream) ((--(outStream)->count >= 0) ? \ + ((*((outStream)->cur++) = (char)('\0'))) : EOF) + +static void SecWriteMultiChar(char ch, int num, SecPrintfStream *f, int *pnumwritten); +static void SecWriteString(const char *string, int len, SecPrintfStream *f, int *pnumwritten); + +#include "output.inl" + +/* + * Wide character formatted output implementation + */ +int SecVsnprintfImpl(char *string, size_t count, const char *format, va_list argList) +{ + SecPrintfStream str; + int retVal; + + str.count = (int)count; /* this count include \0 character, Must be greater than zero */ + str.cur = string; + + retVal = SecOutputS(&str, format, argList); + if ((retVal >= 0) && (SECUREC_PUTC_ZERO(&str) != EOF)) { + return retVal; + } else if (str.count < 0) { + /* the buffer was too small; we return truncation */ + string[count - 1] = '\0'; + return SECUREC_PRINTF_TRUNCATE; + } + string[0] = '\0'; /* empty the dest strDest */ + return -1; +} + +/* + * Sec write Wide character + */ +static void SecWriteMultiChar(char ch, int num, SecPrintfStream *f, int *pnumwritten) +{ + int count = num; + while (count-- > 0) { + if (SECUREC_PUTC(ch, f) == EOF) { + *pnumwritten = -1; + break; + } else { + *pnumwritten = *pnumwritten + 1; + } + } +} + +/* + * Sec write string function + */ +static void SecWriteString(const char *string, int len, SecPrintfStream *f, int *pnumwritten) +{ + const char *str = string; + int count = len; + while (count-- > 0) { + if (SECUREC_PUTC(*str, f) == EOF) { + *pnumwritten = -1; + break; + } else { + *pnumwritten = *pnumwritten + 1; + ++str; + } + } +} + diff --git a/third_party/securec/src/secureprintoutput_w.c b/third_party/securec/src/secureprintoutput_w.c new file mode 100644 index 00000000000..9063ab4d065 --- /dev/null +++ b/third_party/securec/src/secureprintoutput_w.c @@ -0,0 +1,170 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* if some platforms don't have wchar.h, dont't include it */ +#if !(defined(SECUREC_VXWORKS_PLATFORM)) +/* This header file is placed below secinput.h, which will cause tool alarm, + * but if there is no macro above, it will cause compiling alarm + */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#ifndef _CRTIMP_ALTERNATIVE +#define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ +#endif +#ifndef __STDC_WANT_SECURE_LIB__ +#define __STDC_WANT_SECURE_LIB__ 0 +#endif +#endif +#include +#endif + +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#define SECUREC_INLINE_DO_MEMCPY 1 +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifndef SECUREC_FOR_WCHAR +#define SECUREC_FOR_WCHAR +#endif + +#include "secureprintoutput.h" + +#ifndef WEOF +#define WEOF ((wchar_t)(-1)) +#endif + +#define SECUREC_CHAR(x) L ## x +#define SECUREC_WRITE_MULTI_CHAR SecWriteMultiCharW +#define SECUREC_WRITE_STRING SecWriteStringW + +static void SecWriteCharW(wchar_t ch, SecPrintfStream *f, int *pnumwritten); +static void SecWriteMultiCharW(wchar_t ch, int num, SecPrintfStream *f, int *pnumwritten); +static void SecWriteStringW(const wchar_t *string, int len, SecPrintfStream *f, int *pnumwritten); +static int SecPutWcharStrEndingZero(SecPrintfStream *str, int zeroCount); + + +#include "output.inl" + +/* + * Wide character formatted output implementation + */ +int SecVswprintfImpl(wchar_t *string, size_t sizeInWchar, const wchar_t *format, va_list argList) +{ + SecPrintfStream str; + int retVal; /* If initialization causes e838 */ + + str.cur = (char *)string; + /* this count include \0 character, Must be greater than zero */ + str.count = (int)(sizeInWchar * sizeof(wchar_t)); + + retVal = SecOutputSW(&str, format, argList); + if ((retVal >= 0) && SecPutWcharStrEndingZero(&str, (int)sizeof(wchar_t))) { + return (retVal); + } else if (str.count < 0) { + /* the buffer was too small; we return truncation */ + string[sizeInWchar - 1] = L'\0'; + return SECUREC_PRINTF_TRUNCATE; + } + string[0] = L'\0'; + return -1; +} + +/* + * Output one zero character zero into the SecPrintfStream structure + */ +static int SecPutZeroChar(SecPrintfStream *str) +{ + if (str->count > 0) { + *(str->cur) = (char)('\0'); + str->count = str->count - 1; + str->cur = str->cur + 1; + return 0; + } + return -1; +} + +/* + * Output a wide character zero end into the SecPrintfStream structure + */ +static int SecPutWcharStrEndingZero(SecPrintfStream *str, int zeroCount) +{ + int succeed = 0; + int i = 0; + + while (i < zeroCount && (SecPutZeroChar(str) == 0)) { + ++i; + } + if (i == zeroCount) { + succeed = 1; + } + return succeed; +} + + +/* + * Output a wide character into the SecPrintfStream structure + */ +static wchar_t SecPutCharW(wchar_t ch, SecPrintfStream *f) +{ + wchar_t wcRet = 0; + if (((f)->count -= (int)sizeof(wchar_t)) >= 0) { + *(wchar_t *)(void *)(f->cur) = ch; + f->cur += sizeof(wchar_t); + wcRet = ch; + } else { + wcRet = (wchar_t)WEOF; + } + return wcRet; +} + +/* + * Output a wide character into the SecPrintfStream structure, returns the number of characters written + */ +static void SecWriteCharW(wchar_t ch, SecPrintfStream *f, int *pnumwritten) +{ + if (SecPutCharW(ch, f) == (wchar_t)WEOF) { + *pnumwritten = -1; + } else { + *pnumwritten = *pnumwritten + 1; + } +} + +/* + * Output multiple wide character into the SecPrintfStream structure, returns the number of characters written + */ +static void SecWriteMultiCharW(wchar_t ch, int num, SecPrintfStream *f, int *pnumwritten) +{ + int count = num; + while (count-- > 0) { + SecWriteCharW(ch, f, pnumwritten); + if (*pnumwritten == -1) { + break; + } + } +} + +/* + * Output a wide string into the SecPrintfStream structure, returns the number of characters written + */ +static void SecWriteStringW(const wchar_t *string, int len, SecPrintfStream *f, int *pnumwritten) +{ + const wchar_t *str = string; + int count = len; + while (count-- > 0) { + SecWriteCharW(*str++, f, pnumwritten); + if (*pnumwritten == -1) { + break; + } + } +} + diff --git a/third_party/securec/src/snprintf_s.c b/third_party/securec/src/snprintf_s.c new file mode 100644 index 00000000000..0bd7ed1b55b --- /dev/null +++ b/third_party/securec/src/snprintf_s.c @@ -0,0 +1,113 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +#if SECUREC_ENABLE_SNPRINTF +/* + * + * The snprintf_s function is equivalent to the snprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The snprintf_s function formats and stores count or fewer characters in + * strDest and appends a terminating null. Each argument (if any) is converted + * and output according to the corresponding format specification in format. + * The formatting is consistent with the printf family of functions; If copying + * occurs between strings that overlap, the behavior is undefined. + * + * + * strDest Storage location for the output. + * destMax The size of the storage location for output. Size + * in bytes for snprintf_s or size in words for snwprintf_s. + * count Maximum number of character to store. + * format Format-control string. + * ... Optional arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return -1 if count < destMax and the output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + * + */ +int snprintf_s(char *strDest, size_t destMax, size_t count, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsnprintf_s(strDest, destMax, count, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(snprintf_s); +#endif +#endif + +#if SECUREC_SNPRINTF_TRUNCATED +/* + * + * The snprintf_truncated_s function is equivalent to the snprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The snprintf_truncated_s function formats and stores count or fewer characters in + * strDest and appends a terminating null. Each argument (if any) is converted + * and output according to the corresponding format specification in format. + * The formatting is consistent with the printf family of functions; If copying + * occurs between strings that overlap, the behavior is undefined. + * + * + * strDest Storage location for the output. + * destMax The size of the storage location for output. Size + * in bytes for snprintf_truncated_s or size in words for snwprintf_s. + * format Format-control string. + * ... Optional arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return destMax-1 if output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + * + */ +int snprintf_truncated_s(char *strDest, size_t destMax, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsnprintf_truncated_s(strDest, destMax, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(snprintf_truncated_s); +#endif + +#endif + + diff --git a/third_party/securec/src/sprintf_s.c b/third_party/securec/src/sprintf_s.c new file mode 100644 index 00000000000..54a796045a5 --- /dev/null +++ b/third_party/securec/src/sprintf_s.c @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The sprintf_s function is equivalent to the sprintf function + * except for the parameter destMax and the explicit runtime-constraints violation + * The sprintf_s function formats and stores a series of characters and values + * in strDest. Each argument (if any) is converted and output according to + * the corresponding format specification in format. The format consists of + * ordinary characters and has the same form and function as the format argument + * for printf. A null character is appended after the last character written. + * If copying occurs between strings that overlap, the behavior is undefined. + * + * + * strDest Storage location for output. + * destMax Maximum number of characters to store. + * format Format-control string. + * ... Optional arguments + * + * + * strDest is updated + * + * + * return the number of bytes stored in strDest, not counting the terminating null character. + * return -1 if an error occurred. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int sprintf_s(char *strDest, size_t destMax, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsprintf_s(strDest, destMax, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(sprintf_s); +#endif + + diff --git a/third_party/securec/src/sscanf_s.c b/third_party/securec/src/sscanf_s.c new file mode 100644 index 00000000000..c8f097ef723 --- /dev/null +++ b/third_party/securec/src/sscanf_s.c @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The sscanf_s function is equivalent to fscanf_s, + * except that input is obtained from a string (specified by the argument buffer) rather than from a stream + * The sscanf function reads data from buffer into the location given by each + * argument. Every argument must be a pointer to a variable with a type that + * corresponds to a type specifier in format. The format argument controls the + * interpretation of the input fields and has the same form and function as + * the format argument for the scanf function. + * If copying takes place between strings that overlap, the behavior is undefined. + * + * + * buffer Stored data. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... The converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int sscanf_s(const char *buffer, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsscanf_s(buffer, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(sscanf_s); +#endif + + diff --git a/third_party/securec/src/strcat_s.c b/third_party/securec/src/strcat_s.c new file mode 100644 index 00000000000..6bf1379b401 --- /dev/null +++ b/third_party/securec/src/strcat_s.c @@ -0,0 +1,102 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_STR_LEN_OPT 1 +#define SECUREC_INLINE_DO_MEMCPY 1 +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoStrcat(char *strDest, size_t destMax, const char *strSrc) +{ + size_t destLen = SecStrMinLen(strDest, destMax); + /* Only optimize strSrc, do not apply this function to strDest */ + size_t srcLen = SecStrMinLenOpt(strSrc, destMax - destLen); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = '\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strcat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("strcat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = '\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strcat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("strcat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, srcLen + 1); /* single character length include \0 */ + return EOK; +} + +/* + * + * The strcat_s function appends a copy of the string pointed to by strSrc (including the terminating null character) + * to the end of the string pointed to by strDest. + * The initial character of strSrc overwrites the terminating null character of strDest. + * strcat_s will return EOVERLAP_AND_RESET if the source and destination strings overlap. + * + * Note that the second parameter is the total size of the buffer, not the + * remaining size. + * + * + * strDest Null-terminated destination string buffer. + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid)or + * (strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN) + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strcat_s(char *strDest, size_t destMax, const char *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strcat_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strcat_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + return SecDoStrcat(strDest, destMax, strSrc); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strcat_s); +#endif + diff --git a/third_party/securec/src/strcpy_s.c b/third_party/securec/src/strcpy_s.c new file mode 100644 index 00000000000..e248da7cbc1 --- /dev/null +++ b/third_party/securec/src/strcpy_s.c @@ -0,0 +1,351 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +#if SECUREC_IN_KERNEL== 0 +#ifndef SECUREC_STRCOPY_THRESHOLD_SIZE +#define SECUREC_STRCOPY_THRESHOLD_SIZE 32UL +#endif + +/* + * Determine whether the address is 8-byte aligned, use static to increase performance + * return 0 is aligned + */ +static int SecIsAddrAligned8(const void *addr, const void *zeroAddr) +{ + return (int)(((size_t)((const char*)addr - (const char*)zeroAddr)) & 7); /* use 7 to check aligned 8 */ +} + +/* The purpose of converting to void is to clean up the alarm */ +#define SECUREC_SMALL_STR_COPY do { \ + if (SECUREC_ADDR_ALIGNED_8(strDest) && SECUREC_ADDR_ALIGNED_8(strSrc)) { \ + /* use struct assignment */ \ + switch (srcStrLen) { \ + case 1: \ + *(SecStrBuf1 *)(void *)strDest = *(const SecStrBuf1 *)(const void *)strSrc; \ + break; \ + case 2: \ + *(SecStrBuf2 *)(void *)strDest = *(const SecStrBuf2 *)(const void *)strSrc; \ + break; \ + case 3: \ + *(SecStrBuf3 *)(void *)strDest = *(const SecStrBuf3 *)(const void *)strSrc; \ + break; \ + case 4: \ + *(SecStrBuf4 *)(void *)strDest = *(const SecStrBuf4 *)(const void *)strSrc; \ + break; \ + case 5: \ + *(SecStrBuf5 *)(void *)strDest = *(const SecStrBuf5 *)(const void *)strSrc; \ + break; \ + case 6: \ + *(SecStrBuf6 *)(void *)strDest = *(const SecStrBuf6 *)(const void *)strSrc; \ + break; \ + case 7: \ + *(SecStrBuf7 *)(void *)strDest = *(const SecStrBuf7 *)(const void *)strSrc; \ + break; \ + case 8: \ + *(SecStrBuf8 *)(void *)strDest = *(const SecStrBuf8 *)(const void *)strSrc; \ + break; \ + case 9: \ + *(SecStrBuf9 *)(void *)strDest = *(const SecStrBuf9 *)(const void *)strSrc; \ + break; \ + case 10: \ + *(SecStrBuf10 *)(void *)strDest = *(const SecStrBuf10 *)(const void *)strSrc; \ + break; \ + case 11: \ + *(SecStrBuf11 *)(void *)strDest = *(const SecStrBuf11 *)(const void *)strSrc; \ + break; \ + case 12: \ + *(SecStrBuf12 *)(void *)strDest = *(const SecStrBuf12 *)(const void *)strSrc; \ + break; \ + case 13: \ + *(SecStrBuf13 *)(void *)strDest = *(const SecStrBuf13 *)(const void *)strSrc; \ + break; \ + case 14: \ + *(SecStrBuf14 *)(void *)strDest = *(const SecStrBuf14 *)(const void *)strSrc; \ + break; \ + case 15: \ + *(SecStrBuf15 *)(void *)strDest = *(const SecStrBuf15 *)(const void *)strSrc; \ + break; \ + case 16: \ + *(SecStrBuf16 *)(void *)strDest = *(const SecStrBuf16 *)(const void *)strSrc; \ + break; \ + case 17: \ + *(SecStrBuf17 *)(void *)strDest = *(const SecStrBuf17 *)(const void *)strSrc; \ + break; \ + case 18: \ + *(SecStrBuf18 *)(void *)strDest = *(const SecStrBuf18 *)(const void *)strSrc; \ + break; \ + case 19: \ + *(SecStrBuf19 *)(void *)strDest = *(const SecStrBuf19 *)(const void *)strSrc; \ + break; \ + case 20: \ + *(SecStrBuf20 *)(void *)strDest = *(const SecStrBuf20 *)(const void *)strSrc; \ + break; \ + case 21: \ + *(SecStrBuf21 *)(void *)strDest = *(const SecStrBuf21 *)(const void *)strSrc; \ + break; \ + case 22: \ + *(SecStrBuf22 *)(void *)strDest = *(const SecStrBuf22 *)(const void *)strSrc; \ + break; \ + case 23: \ + *(SecStrBuf23 *)(void *)strDest = *(const SecStrBuf23 *)(const void *)strSrc; \ + break; \ + case 24: \ + *(SecStrBuf24 *)(void *)strDest = *(const SecStrBuf24 *)(const void *)strSrc; \ + break; \ + case 25: \ + *(SecStrBuf25 *)(void *)strDest = *(const SecStrBuf25 *)(const void *)strSrc; \ + break; \ + case 26: \ + *(SecStrBuf26 *)(void *)strDest = *(const SecStrBuf26 *)(const void *)strSrc; \ + break; \ + case 27: \ + *(SecStrBuf27 *)(void *)strDest = *(const SecStrBuf27 *)(const void *)strSrc; \ + break; \ + case 28: \ + *(SecStrBuf28 *)(void *)strDest = *(const SecStrBuf28 *)(const void *)strSrc; \ + break; \ + case 29: \ + *(SecStrBuf29 *)(void *)strDest = *(const SecStrBuf29 *)(const void *)strSrc; \ + break; \ + case 30: \ + *(SecStrBuf30 *)(void *)strDest = *(const SecStrBuf30 *)(const void *)strSrc; \ + break; \ + case 31: \ + *(SecStrBuf31 *)(void *)strDest = *(const SecStrBuf31 *)(const void *)strSrc; \ + break; \ + case 32: \ + *(SecStrBuf32 *)(void *)strDest = *(const SecStrBuf32 *)(const void *)strSrc; \ + break; \ + default: \ + break; \ + } /* END switch */ \ + } else { \ + char *tmpStrDest = (char *)strDest; \ + const char *tmpStrSrc = (const char *)strSrc; \ + switch (srcStrLen) { \ + case 32: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + break; \ + } \ + } \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Check Src Range + */ +static errno_t CheckSrcRange(char *strDest, size_t destMax, const char *strSrc) +{ + size_t tmpDestMax = destMax; + const char *tmpSrc = strSrc; + /* use destMax as boundary checker and destMax must be greater than zero */ + while (*(tmpSrc) != '\0' && tmpDestMax > 0) { + ++tmpSrc; + --tmpDestMax; + } + if (tmpDestMax == 0) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strcpy_s"); + return ERANGE_AND_RESET; + } + return EOK; +} + +/* + * Handling errors + */ +errno_t strcpy_error(char *strDest, size_t destMax, const char *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strcpy_s"); + return ERANGE; + } else if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strcpy_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + return CheckSrcRange(strDest, destMax, strSrc); +} + +/* + * Performance optimization. srcStrLen include '\0' + */ +static void SecDoStrcpyOpt(char *strDest, const char *strSrc, size_t srcStrLen) +{ +#if SECUREC_IN_KERNEL + SecDoMemcpy(strDest, strSrc, srcStrLen); +#else + if (srcStrLen > SECUREC_STRCOPY_THRESHOLD_SIZE) { + SecDoMemcpy(strDest, strSrc, srcStrLen); + } else { + SECUREC_SMALL_STR_COPY; + } +#endif +} + +/* + * + * The strcpy_s function copies the string pointed to strSrc + * (including the terminating null character) into the array pointed to by strDest + * The destination string must be large enough to hold the source string, + * including the terminating null character. strcpy_s will return EOVERLAP_AND_RESET + * if the source and destination strings overlap. + * + * + * strDest Location of destination string buffer + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated. + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strcpy_s(char *strDest, size_t destMax, const char *strSrc) +{ + if ((destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN && strDest != NULL && strSrc != NULL && strDest != strSrc)) { + size_t srcStrLen = SecStrMinLen(strSrc, destMax) + 1; /* len include \0 */ + if (srcStrLen <= destMax) { + /* use mem overlap check include \0 */ + if (SECUREC_MEMORY_NO_OVERLAP(strDest, strSrc, srcStrLen)) { + /* performance optimization srcStrLen include '\0' */ + SecDoStrcpyOpt(strDest, strSrc, srcStrLen); + return EOK; + } else { + strDest[0] = '\0'; + SECUREC_ERROR_BUFFER_OVERLAP("strcpy_s"); + return EOVERLAP_AND_RESET; + } + } + } + return strcpy_error(strDest, destMax, strSrc); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strcpy_s); +#endif + diff --git a/third_party/securec/src/strncat_s.c b/third_party/securec/src/strncat_s.c new file mode 100644 index 00000000000..78234fd5f99 --- /dev/null +++ b/third_party/securec/src/strncat_s.c @@ -0,0 +1,121 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoStrncat(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + size_t destLen = SecStrMinLen(strDest, destMax); + /* The strSrc is no longer optimized. The reason is that when count is small, + * the efficiency of strnlen is higher than that of self realization. + */ + size_t srcLen = SecStrMinLen(strSrc, count); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = '\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("strncat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = '\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("strncat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, srcLen); /* no terminator */ + *(strDest + destLen + srcLen) = '\0'; + return EOK; +} + +/* + * + * The strncat_s function appends not more than n successive characters + * (not including the terminating null character) + * from the array pointed to by strSrc to the end of the string pointed to by strDest + * The strncat_s function try to append the first D characters of strSrc to + * the end of strDest, where D is the lesser of count and the length of strSrc. + * If appending those D characters will fit within strDest (whose size is given + * as destMax) and still leave room for a null terminator, then those characters + * are appended, starting at the original terminating null of strDest, and a + * new terminating null is appended; otherwise, strDest[0] is set to the null + * character. + * + * + * strDest Null-terminated destination string. + * destMax Size of the destination buffer. + * strSrc Null-terminated source string. + * count Number of character to append, or truncate. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid)or + * (strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN) + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strncat_s(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strncat_s"); + return ERANGE; + } + + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strncat_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > SECUREC_STRING_MAX_LEN) { +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == (size_t)(-1)) { + /* Windows internal functions may pass in -1 when calling this function */ + return SecDoStrncat(strDest, destMax, strSrc, destMax); + } +#endif + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strncat_s"); + return ERANGE_AND_RESET; + } + return SecDoStrncat(strDest, destMax, strSrc, count); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strncat_s); +#endif + diff --git a/third_party/securec/src/strncpy_s.c b/third_party/securec/src/strncpy_s.c new file mode 100644 index 00000000000..493d1f74999 --- /dev/null +++ b/third_party/securec/src/strncpy_s.c @@ -0,0 +1,143 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) +#define SECUREC_STRNCPY_PARAM_OK(strDest, destMax, strSrc, count) \ + (((destMax) > 0 && (destMax) <= SECUREC_STRING_MAX_LEN && (strDest) != NULL && (strSrc) != NULL && \ + ((count) <= SECUREC_STRING_MAX_LEN || (count) == ((size_t)(-1))) && (count) > 0)) +#else +#define SECUREC_STRNCPY_PARAM_OK(strDest, destMax, strSrc, count) \ + (((destMax) > 0 && (destMax) <= SECUREC_STRING_MAX_LEN && (strDest) != NULL && (strSrc) != NULL && \ + (count) <= SECUREC_STRING_MAX_LEN && (count) > 0)) +#endif + +/* + * Check Src Count Range + */ +static errno_t CheckSrcCountRange(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + size_t tmpDestMax = destMax; + size_t tmpCount = count; + const char *endPos = strSrc; + + /* use destMax and count as boundary checker and destMax must be greater than zero */ + while (*(endPos) != '\0' && tmpDestMax > 0 && tmpCount > 0) { + ++endPos; + --tmpCount; + --tmpDestMax; + } + if (tmpDestMax == 0) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE_AND_RESET; + } + return EOK; +} + +/* + * Handling errors, when dest euqal src return EOK + */ +errno_t strncpy_error(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE; + } else if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strncpy_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } else if (count > SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; /* clear dest string */ + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE_AND_RESET; + } else if (count == 0) { + strDest[0] = '\0'; + return EOK; + } + + return CheckSrcCountRange(strDest, destMax, strSrc, count); +} + +/* + * + * The strncpy_s function copies not more than n successive characters (not including the terminating null character) + * from the array pointed to by strSrc to the array pointed to by strDest. + * + * + * strDest Destination string. + * destMax The size of the destination string, in characters. + * strSrc Source string. + * count Number of characters to be copied. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strncpy_s(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + if (SECUREC_STRNCPY_PARAM_OK(strDest, destMax, strSrc, count)) { + size_t minCpLen; /* use it to store the maxi length limit */ + if (count < destMax) { + minCpLen = SecStrMinLen(strSrc, count); /* no ending terminator */ + } else { + size_t tmpCount = destMax; +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == ((size_t)(-1))) { + tmpCount = destMax - 1; + } +#endif + minCpLen = SecStrMinLen(strSrc, tmpCount); + if (minCpLen == destMax) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE_AND_RESET; + } + } + if (SECUREC_STRING_NO_OVERLAP(strDest, strSrc, minCpLen) || strDest == strSrc) { + /* Not overlap */ + SecDoMemcpy(strDest, strSrc, minCpLen); /* copy string without terminator */ + strDest[minCpLen] = '\0'; + return EOK; + } else { + strDest[0] = '\0'; + SECUREC_ERROR_BUFFER_OVERLAP("strncpy_s"); + return EOVERLAP_AND_RESET; + } + } + return strncpy_error(strDest, destMax, strSrc, count); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strncpy_s); +#endif + diff --git a/third_party/securec/src/strtok_s.c b/third_party/securec/src/strtok_s.c new file mode 100644 index 00000000000..18f977a75db --- /dev/null +++ b/third_party/securec/src/strtok_s.c @@ -0,0 +1,117 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * Find beginning of token (skip over leading delimiters).Note that + * there is no token if this loop sets string to point to the terminal null. + */ +static char *SecFindBegin(char *strToken, const char *strDelimit) +{ + char *token = strToken; + while (*token != '\0') { + const char *ctl = strDelimit; + while (*ctl != '\0' && *ctl != *token) { + ++ctl; + } + if (*ctl == '\0') { /* don't find any delimiter in string header, break the loop */ + break; + } + ++token; + } + return token; +} + +/* + * Find rest of token + */ +static char *SecFindRest(char *strToken, const char *strDelimit) +{ + /* Find the rest of the token. If it is not the end of the string, + * put a null there. + */ + char *token = strToken; + while (*token != '\0') { + const char *ctl = strDelimit; + while (*ctl != '\0' && *ctl != *token) { + ++ctl; + } + if (*ctl != '\0') { /* find a delimiter */ + *token++ = '\0'; /* set string termintor */ + break; + } + ++token; + } + return token; +} + +/* + * Find the final position pointer + */ +static char *SecUpdateToken(char *strToken, const char *strDelimit, char **context) +{ + /* point to updated position */ + char *token = SecFindRest(strToken, strDelimit); + /* record string position for next search in the context */ + *context = token; + /* Determine if a token has been found. */ + if (token == strToken) { + return NULL; + } + return strToken; +} + +/* + * + * The strtok_s function parses a string into a sequence of strToken, + * replace all characters in strToken string that match to strDelimit set with 0. + * On the first call to strtok_s the string to be parsed should be specified in strToken. + * In each subsequent call that should parse the same string, strToken should be NULL + * + * strToken String containing token or tokens. + * strDelimit Set of delimiter characters. + * context Used to store position information between calls + * to strtok_s + * + * context is updated + * + * On the first call returns the address of the first non \0 character, otherwise NULL is returned. + * In subsequent calls, the strtoken is set to NULL, and the context set is the same as the previous call, + * return NULL if the *context string length is equal 0, otherwise return *context. + */ +char *strtok_s(char *strToken, const char *strDelimit, char **context) +{ + char *orgToken = strToken; + /* validate delimiter and string context */ + if (context == NULL || strDelimit == NULL) { + return NULL; + } + /* valid input string and string pointer from where to search */ + if (orgToken == NULL && (*context) == NULL) { + return NULL; + } + /* If string is null, continue searching from previous string position stored in context */ + if (orgToken == NULL) { + orgToken = *context; + } + orgToken = SecFindBegin(orgToken, strDelimit); + return SecUpdateToken(orgToken, strDelimit, context); +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strtok_s); +#endif + diff --git a/third_party/securec/src/swprintf_s.c b/third_party/securec/src/swprintf_s.c new file mode 100644 index 00000000000..1fb0f6c797e --- /dev/null +++ b/third_party/securec/src/swprintf_s.c @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The swprintf_s function is the wide-character equivalent of the sprintf_s function + * + * + * strDest Storage location for the output. + * destMax Maximum number of characters to store. + * format Format-control string. + * ... Optional arguments + * + * + * strDest is updated + * + * + * return the number of wide characters stored in strDest, not counting the terminating null wide character. + * return -1 if an error occurred. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int swprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vswprintf_s(strDest, destMax, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/swscanf_s.c b/third_party/securec/src/swscanf_s.c new file mode 100644 index 00000000000..c16045fa63c --- /dev/null +++ b/third_party/securec/src/swscanf_s.c @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * The swscanf_s function is the wide-character equivalent of the sscanf_s function + * The swscanf_s function reads data from buffer into the location given by + * each argument. Every argument must be a pointer to a variable with a type + * that corresponds to a type specifier in format. The format argument controls + * the interpretation of the input fields and has the same form and function + * as the format argument for the scanf function. If copying takes place between + * strings that overlap, the behavior is undefined. + * + * + * buffer Stored data. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; The return value does not include fields that were read but not + * assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int swscanf_s(const wchar_t *buffer, const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vswscanf_s(buffer, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/vfscanf_s.c b/third_party/securec/src/vfscanf_s.c new file mode 100644 index 00000000000..78444e4b2bf --- /dev/null +++ b/third_party/securec/src/vfscanf_s.c @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secinput.h" + +/* + * + * The vfscanf_s function is equivalent to fscanf_s, with the variable argument list replaced by argList + * The vfscanf_s function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same + * form and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vfscanf_s(FILE *stream, const char *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + + if ((stream == NULL) || (format == NULL)) { + SECUREC_ERROR_INVALID_PARAMTER("vfscanf_s"); + return SECUREC_SCANF_EINVAL; + } + if (stream == stdin) { + return vscanf_s(format, argList); + } + + SECUREC_LOCK_FILE(stream); + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FILE_STREAM_FLAG, stream, SECUREC_UNINITIALIZED_FILE_POS, NULL, 0); + retVal = SecInputS(&fStr, format, argList); + SECUREC_UNLOCK_FILE(stream); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vfscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + return retVal; +} + + diff --git a/third_party/securec/src/vfwscanf_s.c b/third_party/securec/src/vfwscanf_s.c new file mode 100644 index 00000000000..3ae62eea02c --- /dev/null +++ b/third_party/securec/src/vfwscanf_s.c @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secinput.h" + +/* + * + * The vfwscanf_s function is the wide-character equivalent of the vfscanf_s function + * The vfwscanf_s function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same form + * and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vfwscanf_s(FILE *stream, const wchar_t *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + + if ((stream == NULL) || (format == NULL)) { + SECUREC_ERROR_INVALID_PARAMTER("vfwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + if (stream == stdin) { + return vwscanf_s(format, argList); + } + + SECUREC_LOCK_FILE(stream); + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FILE_STREAM_FLAG, stream, SECUREC_UNINITIALIZED_FILE_POS, NULL, 0); + retVal = SecInputSW(&fStr, format, argList); + SECUREC_UNLOCK_FILE(stream); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vfwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} + + diff --git a/third_party/securec/src/vscanf_s.c b/third_party/securec/src/vscanf_s.c new file mode 100644 index 00000000000..6666976503c --- /dev/null +++ b/third_party/securec/src/vscanf_s.c @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secinput.h" + +/* + * + * The vscanf_s function is equivalent to scanf_s, with the variable argument list replaced by argList, + * The vscanf_s function reads data from the standard input stream stdin and + * writes the data into the location that's given by argument. Each argument + * must be a pointer to a variable of a type that corresponds to a type specifier + * in format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vscanf_s(const char *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FROM_STDIN_FLAG, stdin, 0, NULL, 0); + /* + * "va_list" has different definition on different platform, so we can't use argList == NULL + * to determine it's invalid. If you has fixed platform, you can check some fields to validate it, + * such as "argList == NULL" or argList.xxx != NULL or *(size_t *)&argList != 0. + */ + if (format == NULL || fStr.pf == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + SECUREC_LOCK_STDIN(0, fStr.pf); + + retVal = SecInputS(&fStr, format, argList); + + SECUREC_UNLOCK_STDIN(0, fStr.pf); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} + + diff --git a/third_party/securec/src/vsnprintf_s.c b/third_party/securec/src/vsnprintf_s.c new file mode 100644 index 00000000000..dfa55babfeb --- /dev/null +++ b/third_party/securec/src/vsnprintf_s.c @@ -0,0 +1,149 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secureprintoutput.h" + +#if SECUREC_ENABLE_VSNPRINTF +/* + * + * The vsnprintf_s function is equivalent to the vsnprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The vsnprintf_s function takes a pointer to an argument list, then formats + * and writes up to count characters of the given data to the memory pointed + * to by strDest and appends a terminating null. + * + * + * strDest Storage location for the output. + * destMax The size of the strDest for output. + * count Maximum number of character to write(not including + * the terminating NULL) + * format Format-control string. + * argList pointer to list of arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return -1 if count < destMax and the output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vsnprintf_s(char *strDest, size_t destMax, size_t count, const char *format, va_list argList) +{ + int retVal; + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > SECUREC_STRING_MAX_LEN || + (count > (SECUREC_STRING_MAX_LEN - 1) && count != (size_t)(-1))) { + if (strDest != NULL && destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_s"); + return -1; + } + + if (destMax > count) { + retVal = SecVsnprintfImpl(strDest, count + 1, format, argList); + if (retVal == SECUREC_PRINTF_TRUNCATE) { /* lsd add to keep dest buffer not destroyed 2014.2.18 */ + /* the string has been truncated, return -1 */ + return -1; /* to skip error handler, return strlen(strDest) or -1 */ + } + } else { + retVal = SecVsnprintfImpl(strDest, destMax, format, argList); +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (retVal == SECUREC_PRINTF_TRUNCATE && count == (size_t)(-1)) { + return -1; + } +#endif + } + + if (retVal < 0) { + strDest[0] = '\0'; /* empty the dest strDest */ + + if (retVal == SECUREC_PRINTF_TRUNCATE) { + /* Buffer too small */ + SECUREC_ERROR_INVALID_RANGE("vsnprintf_s"); + } + + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_s"); + return -1; + } + + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsnprintf_s); +#endif +#endif + +#if SECUREC_SNPRINTF_TRUNCATED +/* + * + * The vsnprintf_truncated_s function is equivalent to the vsnprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The vsnprintf_truncated_s function takes a pointer to an argument list, then formats + * and writes up to count characters of the given data to the memory pointed + * to by strDest and appends a terminating null. + * + * + * strDest Storage location for the output. + * destMax The size of the strDest for output. + * the terminating NULL) + * format Format-control string. + * argList pointer to list of arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return destMax-1 if output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vsnprintf_truncated_s(char *strDest, size_t destMax, const char *format, va_list argList) +{ + int retVal; + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + if (strDest != NULL && destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_truncated_s"); + return -1; + } + + retVal = SecVsnprintfImpl(strDest, destMax, format, argList); + + if (retVal < 0) { + if (retVal == SECUREC_PRINTF_TRUNCATE) { + return (int)(destMax - 1); /* to skip error handler, return strlen(strDest) */ + } + strDest[0] = '\0'; /* empty the dest strDest */ + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_truncated_s"); + return -1; + } + + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsnprintf_truncated_s); +#endif +#endif + + diff --git a/third_party/securec/src/vsprintf_s.c b/third_party/securec/src/vsprintf_s.c new file mode 100644 index 00000000000..e74c7748c34 --- /dev/null +++ b/third_party/securec/src/vsprintf_s.c @@ -0,0 +1,73 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secureprintoutput.h" + +/* + * + * The vsprintf_s function is equivalent to the vsprintf function + * except for the parameter destMax and the explicit runtime-constraints violation + * The vsprintf_s function takes a pointer to an argument list, and then formats + * and writes the given data to the memory pointed to by strDest. + * The function differ from the non-secure versions only in that the secure + * versions support positional parameters. + * + * + * strDest Storage location for the output. + * destMax Size of strDest + * format Format specification. + * argList pointer to list of arguments + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null character, + * return -1 if an error occurs. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vsprintf_s(char *strDest, size_t destMax, const char *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + if (strDest != NULL && destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vsprintf_s"); + return -1; + } + + retVal = SecVsnprintfImpl(strDest, destMax, format, argList); + + if (retVal < 0) { + strDest[0] = '\0'; + if (retVal == SECUREC_PRINTF_TRUNCATE) { + /* Buffer is too small */ + SECUREC_ERROR_INVALID_RANGE("vsprintf_s"); + } + SECUREC_ERROR_INVALID_PARAMTER("vsprintf_s"); + return -1; + } + + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsprintf_s); +#endif + + diff --git a/third_party/securec/src/vsscanf_s.c b/third_party/securec/src/vsscanf_s.c new file mode 100644 index 00000000000..e0a5ecdabe9 --- /dev/null +++ b/third_party/securec/src/vsscanf_s.c @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secinput.h" +#if defined(SECUREC_VXWORKS_PLATFORM) && (!defined(SECUREC_SYSAPI4VXWORKS) && !defined(SECUREC_CTYPE_MACRO_ADAPT)) +#include +#endif + +/* + * + * vsscanf_s + * + * + * + * The vsscanf_s function is equivalent to sscanf_s, with the variable argument list replaced by argList + * The vsscanf_s function reads data from buffer into the location given by + * each argument. Every argument must be a pointer to a variable with a type + * that corresponds to a type specifier in format. The format argument controls + * the interpretation of the input fields and has the same form and function + * as the format argument for the scanf function. + * If copying takes place between strings that overlap, the behavior is undefined. + * + * + * buffer Stored data + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vsscanf_s(const char *buffer, const char *format, va_list argList) +{ + size_t count; /* If initialization causes e838 */ + int retVal; + SecFileStream fStr; + + /* validation section */ + if (buffer == NULL || format == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vsscanf_s"); + return SECUREC_SCANF_EINVAL; + } + count = strlen(buffer); + if (count == 0 || count > SECUREC_STRING_MAX_LEN) { + SecClearDestBuf(buffer, format, argList); + SECUREC_ERROR_INVALID_PARAMTER("vsscanf_s"); + return SECUREC_SCANF_EINVAL; + } +#ifdef SECUREC_VXWORKS_PLATFORM + /* + * in vxworks platform when buffer is white string, will set first %s argument tu zero.like following useage: + * " \v\f\t\r\n", "%s", str, strSize + * do not check all character, just first and last character then consider it is white string + */ + if (isspace((int)buffer[0]) && isspace((int)buffer[count - 1])) { + SecClearDestBuf(buffer, format, argList); + } +#endif + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_MEM_STR_FLAG, NULL, 0, buffer, (int)count); + retVal = SecInputS(&fStr, format, argList); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vsscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsscanf_s); +#endif + diff --git a/third_party/securec/src/vswprintf_s.c b/third_party/securec/src/vswprintf_s.c new file mode 100644 index 00000000000..3403a6b5933 --- /dev/null +++ b/third_party/securec/src/vswprintf_s.c @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secureprintoutput.h" + + +/* + * + * The vswprintf_s function is the wide-character equivalent of the vsprintf_s function + * + * + * strDest Storage location for the output. + * destMax Size of strDest + * format Format specification. + * argList pointer to list of arguments + * + * + * strDest is updated + * + * + * return the number of wide characters stored in strDest, not counting the terminating null wide character. + * return -1 if an error occurred. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vswprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > (SECUREC_WCHAR_STRING_MAX_LEN)) { + if (strDest != NULL && destMax > 0) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vswprintf_s"); + return -1; + } + + retVal = SecVswprintfImpl(strDest, destMax, format, argList); + + if (retVal < 0) { + strDest[0] = '\0'; + if (retVal == SECUREC_PRINTF_TRUNCATE) { + /* Buffer too small */ + SECUREC_ERROR_INVALID_RANGE("vswprintf_s"); + } + SECUREC_ERROR_INVALID_PARAMTER("vswprintf_s"); + return -1; + } + + return retVal; +} + + diff --git a/third_party/securec/src/vswscanf_s.c b/third_party/securec/src/vswscanf_s.c new file mode 100644 index 00000000000..269e10534ea --- /dev/null +++ b/third_party/securec/src/vswscanf_s.c @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secinput.h" + +static size_t SecWcslen(const wchar_t *s) +{ + const wchar_t *end = s; + while (*end != L'\0') { + ++end; + } + return ((size_t)((end - s))); +} + +/* + * + * The vswscanf_s function is the wide-character equivalent of the vsscanf_s function + * The vsscanf_s function reads data from buffer into the location given by + * each argument. Every argument must be a pointer to a variable with a type + * that corresponds to a type specifier in format. + * The format argument controls the interpretation of the input fields and + * has the same form and function as the format argument for the scanf function. + * If copying takes place between strings that overlap, the behavior is undefined. + * + * + * buffer Stored data + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vswscanf_s(const wchar_t *buffer, const wchar_t *format, va_list argList) +{ + size_t count; /* If initialization causes e838 */ + SecFileStream fStr; + int retVal; + + /* validation section */ + if (buffer == NULL || format == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vswscanf_s"); + return SECUREC_SCANF_EINVAL; + } + count = SecWcslen(buffer); + if (count == 0 || count > SECUREC_WCHAR_STRING_MAX_LEN) { + SecClearDestBufW(buffer, format, argList); + SECUREC_ERROR_INVALID_PARAMTER("vswscanf_s"); + return SECUREC_SCANF_EINVAL; + } + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_MEM_STR_FLAG, NULL, 0,\ + (const char *)buffer, (int)count * ((int)sizeof(wchar_t))); + retVal = SecInputSW(&fStr, format, argList); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vswscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} + + diff --git a/third_party/securec/src/vwscanf_s.c b/third_party/securec/src/vwscanf_s.c new file mode 100644 index 00000000000..56e0f6b47e5 --- /dev/null +++ b/third_party/securec/src/vwscanf_s.c @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "secinput.h" + +/* + * + * The vwscanf_s function is the wide-character equivalent of the vscanf_s function + * The vwscanf_s function is the wide-character version of vscanf_s. The + * function reads data from the standard input stream stdin and writes the + * data into the location that's given by argument. Each argument must be a + * pointer to a variable of a type that corresponds to a type specifier in + * format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vwscanf_s(const wchar_t *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FROM_STDIN_FLAG, stdin, 0, NULL, 0); + if (format == NULL || fStr.pf == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + SECUREC_LOCK_STDIN(0, fStr.pf); + + retVal = SecInputSW(&fStr, format, argList); + + SECUREC_UNLOCK_STDIN(0, fStr.pf); + + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + return retVal; +} + + diff --git a/third_party/securec/src/wcscat_s.c b/third_party/securec/src/wcscat_s.c new file mode 100644 index 00000000000..51254b3f558 --- /dev/null +++ b/third_party/securec/src/wcscat_s.c @@ -0,0 +1,111 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoWcscat(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + size_t destLen; + size_t srcLen; + size_t maxCount; /* Store the maximum available count */ + + /* To calculate the length of a wide character, the parameter must be a wide character */ + SECUREC_CALC_WSTR_LEN(strDest, destMax, &destLen); + maxCount = destMax - destLen; + SECUREC_CALC_WSTR_LEN(strSrc, maxCount, &srcLen); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = L'\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcscat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("wcscat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = L'\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcscat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("wcscat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, (srcLen + 1) * sizeof(wchar_t)); /* single character length include \0 */ + return EOK; +} + +/* + * + * The wcscat_s function appends a copy of the wide string pointed to by strSrc +* (including the terminating null wide character) + * to the end of the wide string pointed to by strDest. + * The arguments and return value of wcscat_s are wide-character strings. + * + * The wcscat_s function appends strSrc to strDest and terminates the resulting + * string with a null character. The initial character of strSrc overwrites the + * terminating null character of strDest. wcscat_s will return EOVERLAP_AND_RESET if the + * source and destination strings overlap. + * + * Note that the second parameter is the total size of the buffer, not the + * remaining size. + * + * + * strDest Null-terminated destination string buffer. + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid) or + * (strDest != NULL and strSrc is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN) + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcscat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcscat_s"); + return ERANGE; + } + + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcscat_s"); + if (strDest != NULL) { + strDest[0] = L'\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + + return SecDoWcscat(strDest, destMax, strSrc); +} + + diff --git a/third_party/securec/src/wcscpy_s.c b/third_party/securec/src/wcscpy_s.c new file mode 100644 index 00000000000..2c348d4bdf3 --- /dev/null +++ b/third_party/securec/src/wcscpy_s.c @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +static errno_t SecDoWcscpy(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + size_t srcStrLen; + + SECUREC_CALC_WSTR_LEN(strSrc, destMax, &srcStrLen); + if (srcStrLen == destMax) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("wcscpy_s"); + return ERANGE_AND_RESET; + } + if (strDest == strSrc) { + return EOK; + } + + if (SECUREC_STRING_NO_OVERLAP(strDest, strSrc, srcStrLen)) { + /* performance optimization srcStrLen include '\0' */ + SecDoMemcpy(strDest, strSrc, (srcStrLen + 1) * sizeof(wchar_t)); /* single character length include \0 */ + return EOK; + } else { + strDest[0] = L'\0'; + SECUREC_ERROR_BUFFER_OVERLAP("wcscpy_s"); + return EOVERLAP_AND_RESET; + } +} + +/* + * + * The wcscpy_s function copies the wide string pointed to by strSrc + * (including theterminating null wide character) into the array pointed to by strDest + + * + * strDest Destination string buffer + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated. + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET destMax <= length of strSrc and strDest != strSrc + * and strDest != NULL and strSrc != NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * and strDest != NULL and strSrc !=NULL and strDest != strSrc + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcscpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcscpy_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcscpy_s"); + if (strDest != NULL) { + strDest[0] = L'\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + return SecDoWcscpy(strDest, destMax, strSrc); +} + + diff --git a/third_party/securec/src/wcsncat_s.c b/third_party/securec/src/wcsncat_s.c new file mode 100644 index 00000000000..bc9e6e39729 --- /dev/null +++ b/third_party/securec/src/wcsncat_s.c @@ -0,0 +1,118 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoWcsncat(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + size_t destLen; + size_t srcLen; + + /* To calculate the length of a wide character, the parameter must be a wide character */ + SECUREC_CALC_WSTR_LEN(strDest, destMax, &destLen); + SECUREC_CALC_WSTR_LEN(strSrc, count, &srcLen); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = L'\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("wcsncat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = L'\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("wcsncat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, srcLen * sizeof(wchar_t)); /* no terminator */ + *(strDest + destLen + srcLen) = L'\0'; + return EOK; +} + +/* + * + * The wcsncat_s function appends not more than n successive wide characters + * (not including the terminating null wide character) + * from the array pointed to by strSrc to the end of the wide string pointed to by strDest. + * + * The wcsncat_s function try to append the first D characters of strSrc to + * the end of strDest, where D is the lesser of count and the length of strSrc. + * If appending those D characters will fit within strDest (whose size is + * given as destMax) and still leave room for a null terminator, then those + * characters are appended, starting at the original terminating null of + * strDest, and a new terminating null is appended; otherwise, strDest[0] is + * set to the null character. + * + * + * strDest Null-terminated destination string. + * destMax Size of the destination buffer. + * strSrc Null-terminated source string. + * count Number of character to append, or truncate. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid) or + * (strDest != NULL and strSrc is NULLL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN) + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcsncat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcsncat_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncat_s"); + if (strDest != NULL) { + strDest[0] = L'\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > SECUREC_WCHAR_STRING_MAX_LEN) { +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == ((size_t)-1)) { + /* Windows internal functions may pass in -1 when calling this function */ + return SecDoWcsncat(strDest, destMax, strSrc, destMax); + } +#endif + strDest[0] = L'\0'; + SECUREC_ERROR_INVALID_RANGE("wcsncat_s"); + return ERANGE_AND_RESET; + } + return SecDoWcsncat(strDest, destMax, strSrc, count); +} + + diff --git a/third_party/securec/src/wcsncpy_s.c b/third_party/securec/src/wcsncpy_s.c new file mode 100644 index 00000000000..746b1d441f8 --- /dev/null +++ b/third_party/securec/src/wcsncpy_s.c @@ -0,0 +1,111 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +static errno_t SecDoWcsncpy(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + size_t srcStrLen; + if (count < destMax) { + SECUREC_CALC_WSTR_LEN(strSrc, count, &srcStrLen); + } else { + SECUREC_CALC_WSTR_LEN(strSrc, destMax, &srcStrLen); + } + if (srcStrLen == destMax) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("wcsncpy_s"); + return ERANGE_AND_RESET; + } + if (strDest == strSrc) { + return EOK; + } + if (SECUREC_STRING_NO_OVERLAP(strDest, strSrc, srcStrLen)) { + /* performance optimization srcStrLen not include '\0' */ + SecDoMemcpy(strDest, strSrc, srcStrLen * sizeof(wchar_t)); + *(strDest + srcStrLen) = L'\0'; + return EOK; + } else { + strDest[0] = L'\0'; + SECUREC_ERROR_BUFFER_OVERLAP("wcsncpy_s"); + return EOVERLAP_AND_RESET; + } +} + +/* + * + * The wcsncpy_s function copies not more than n successive wide characters + * (not including the terminating null wide character) + * from the array pointed to by strSrc to the array pointed to by strDest + * + * + * strDest Destination string. + * destMax The size of the destination string, in characters. + * strSrc Source string. + * count Number of characters to be copied. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > SECUREC_WCHAR_STRING_MAX_LEN or + * (destMax <= length of strSrc and destMax <= count and strDest != strSrc + * and strDest != NULL and strSrc != NULL and destMax != 0 and + * destMax <= SECUREC_WCHAR_STRING_MAX_LEN and not overlap) + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcsncpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcsncpy_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncpy_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > SECUREC_WCHAR_STRING_MAX_LEN) { +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == (size_t)(-1)) { + return SecDoWcsncpy(strDest, destMax, strSrc, destMax - 1); + } +#endif + strDest[0] = '\0'; /* clear dest string */ + SECUREC_ERROR_INVALID_RANGE("wcsncpy_s"); + return ERANGE_AND_RESET; + } + + if (count == 0) { + strDest[0] = '\0'; + return EOK; + } + + return SecDoWcsncpy(strDest, destMax, strSrc, count); +} + diff --git a/third_party/securec/src/wcstok_s.c b/third_party/securec/src/wcstok_s.c new file mode 100644 index 00000000000..99c524f029c --- /dev/null +++ b/third_party/securec/src/wcstok_s.c @@ -0,0 +1,116 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * FindBegin Wide character postion function + */ +static wchar_t *SecFindBeginW(wchar_t *strToken, const wchar_t *strDelimit) +{ + /* Find beginning of token (skip over leading delimiters). Note that + * there is no token if this loop sets string to point to the terminal null. + */ + wchar_t *token = strToken; + while (*token != L'\0') { + const wchar_t *ctl = strDelimit; + while (*ctl != L'\0' && *ctl != *token) { + ++ctl; + } + if (*ctl == L'\0') { + break; + } + ++token; + } + return token; +} + +/* + * FindBegin rest Wide character postion function + */ +static wchar_t *SecFindRestW(wchar_t *strToken, const wchar_t *strDelimit) +{ + /* Find the end of the token. If it is not the end of the string, + * put a null there. + */ + wchar_t *token = strToken; + while (*token != L'\0') { + const wchar_t *ctl = strDelimit; + while (*ctl != L'\0' && *ctl != *token) { + ++ctl; + } + if (*ctl != L'\0') { + *token++ = L'\0'; + break; + } + ++token; + } + return token; +} + +/* + * Update Token wide character function + */ +static wchar_t *SecUpdateTokenW(wchar_t *strToken, const wchar_t *strDelimit, wchar_t **context) +{ + /* point to updated position */ + wchar_t *token = SecFindRestW(strToken, strDelimit); + /* Update the context */ + *context = token; + /* Determine if a token has been found. */ + if (token == strToken) { + return NULL; + } + return strToken; +} + +/* + * + * wcstok_s + * + * + * + * The wcstok_s function is the wide-character equivalent of the strtok_s function + * + * + * strToken String containing token or tokens. + * strDelimit Set of delimiter characters. + * context Used to store position information between calls to + * wcstok_s. + * + * + * context is updated + * + * The wcstok_s function is the wide-character equivalent of the strtok_s function + */ +wchar_t *wcstok_s(wchar_t *strToken, const wchar_t *strDelimit, wchar_t **context) +{ + wchar_t *orgToken = strToken; + /* validation section */ + if (context == NULL || strDelimit == NULL) { + return NULL; + } + if (orgToken == NULL && (*context) == NULL) { + return NULL; + } + /* If string==NULL, continue with previous string */ + if (orgToken == NULL) { + orgToken = *context; + } + orgToken = SecFindBeginW(orgToken, strDelimit); + return SecUpdateTokenW(orgToken, strDelimit, context); +} + diff --git a/third_party/securec/src/wmemcpy_s.c b/third_party/securec/src/wmemcpy_s.c new file mode 100644 index 00000000000..236fcce1ff6 --- /dev/null +++ b/third_party/securec/src/wmemcpy_s.c @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securecutil.h" + +/* + * + * The wmemcpy_s function copies n successive wide characters + * from the object pointed to by src into the object pointed to by dest.t. + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Buffer to copy from. + * count Number of characters to copy. + * + * + * dest buffer is uptdated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and count <= destMax + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN and count <= destMax + * ERANGE destMax > SECUREC_WCHAR_MEM_MAX_LEN or destMax is 0 or + * (count > destMax and dest is NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN) + * ERANGE_AND_RESET count > destMax and dest != NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and + * count <= destMax destMax != 0 and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * and dest != NULL and src != NULL and dest != src + * + * if an error occured, dest will be filled with 0 when dest and destMax valid . + * If the source and destination overlap, the behavior of wmemcpy_s is undefined. + * Use wmemmove_s to handle overlapping regions. + */ +errno_t wmemcpy_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_PARAMTER("wmemcpy_s"); + return ERANGE; + } + if (count > destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wmemcpy_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax * sizeof(wchar_t)); + return ERANGE_AND_RESET; + } + return ERANGE; + } + return memcpy_s(dest, destMax * sizeof(wchar_t), src, count * sizeof(wchar_t)); +} + diff --git a/third_party/securec/src/wmemmove_s.c b/third_party/securec/src/wmemmove_s.c new file mode 100644 index 00000000000..2ef549a0d7d --- /dev/null +++ b/third_party/securec/src/wmemmove_s.c @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securecutil.h" + +/* + * + * The wmemmove_s function copies n successive wide characters from the object pointed + * to by src into the object pointed to by dest. + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Source object. + * count Number of bytes or character to copy. + * + * + * dest is updated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and count <= destMax + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN and count <= destMax + * ERANGE destMax > SECUREC_WCHAR_MEM_MAX_LEN or destMax is 0 or + * (count > destMax and dest is NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN) + * ERANGE_AND_RESET count > destMax and dest != NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * + * + * If an error occured, dest will be filled with 0 when dest and destMax valid. + * If some regions of the source area and the destination overlap, wmemmove_s + * ensures that the original source bytes in the overlapping region are copied + * before being overwritten + */ +errno_t wmemmove_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_PARAMTER("wmemmove_s"); + return ERANGE; + } + if (count > destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wmemmove_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax * sizeof(wchar_t)); + return ERANGE_AND_RESET; + } + return ERANGE; + } + return memmove_s(dest, destMax * sizeof(wchar_t), src, count * sizeof(wchar_t)); +} + diff --git a/third_party/securec/src/wscanf_s.c b/third_party/securec/src/wscanf_s.c new file mode 100644 index 00000000000..c1dcce27a8f --- /dev/null +++ b/third_party/securec/src/wscanf_s.c @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "securec.h" + +/* + * + * + * The wscanf_s function is the wide-character equivalent of the scanf_s function + * The wscanf_s function reads data from the standard input stream stdin and + * writes the data into the location that's given by argument. Each argument + * must be a pointer to a variable of a type that corresponds to a type specifier + * in format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * ... Optional arguments. + * + * + * ... the converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ + +int wscanf_s(const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vwscanf_s(format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +